2024-09-12T23:10:04.486 INFO:root:teuthology version: 1.2.2.dev2+g3752d38 2024-09-12T23:10:04.508 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-12T23:10:04.614 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-09-11_01:16:05-upgrade:quincy-x-reef-distro-default-smithi/7901063 branch: reef description: upgrade:quincy-x/parallel/{0-random-distro$/{ubuntu_22.04} 0-start 1-tasks mon_election/classic upgrade-sequence workload/{ec-rados-default rados_api rados_loadgenbig rbd_import_export test_rbd_api test_rbd_python}} email: ceph-qa@ceph.com first_in_suite: false job_id: '7901063' kernel: kdb: 1 sha1: distro ktype: distro last_in_suite: false machine_type: smithi meta: - desc: 'run run randomized correctness test for rados operations on an erasure-coded pool ' - desc: 'object class functional tests ' - desc: 'generate read/write load with rados objects ranging from 1MB to 25MB ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'librbd python api tests ' name: teuthology-2024-09-11_01:16:05-upgrade:quincy-x-reef-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 4 size: 10 os_type: ubuntu os_version: '22.04' overrides: admin_socket: branch: reef ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: 7a853b1706d076d58e0da7a5d3b9b9eb36276880 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: extra_system_packages: - python3-pytest flavor: default sha1: 7a853b1706d076d58e0da7a5d3b9b9eb36276880 workunit: branch: reef sha1: 7a853b1706d076d58e0da7a5d3b9b9eb36276880 owner: scheduled_teuthology@teuthology priority: 830 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - mds.a - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mds.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b seed: 4534 sha1: 7a853b1706d076d58e0da7a5d3b9b9eb36276880 sleep_before_teardown: 0 subset: 0/1 suite: upgrade:quincy-x suite_branch: reef suite_path: /home/teuthworker/src/git.ceph.com_ceph_7a853b1706d076d58e0da7a5d3b9b9eb36276880/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 7a853b1706d076d58e0da7a5d3b9b9eb36276880 targets: smithi136.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEBb0j2zpeuUgQqmDXwjj/Cpkv1sk+F75zhCaKOJ/RSelxwDSQj+1kkmCHDmknWQyrHTnSFaS31kO5qCxzvfpqw= smithi181.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOXcWA13SzjC1Vv8gXJqWJgEANUl9b6aQZT8lyjMX633aRe652qzrd+SbGBeBBQVrJy0PT2TUlaie5lCMse7ZRk= tasks: - install: branch: quincy exclude_packages: - ceph-volume - print: '**** done install task...' - print: '**** done start installing quincy cephadm ...' - cephadm: cephadm_branch: quincy cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:quincy - print: '**** done end installing quincy cephadm ...' - print: '**** done start cephadm.shell ceph config set mgr...' - cephadm.shell: mon.a: - ceph config set mgr mgr/cephadm/use_repo_digest true --force - print: '**** done cephadm.shell ceph config set mgr...' - print: '**** done start telemetry quincy...' - workunit: clients: client.0: - test_telemetry_quincy.sh - print: '**** done end telemetry quincy...' - print: '**** done start parallel' - parallel: - workload - upgrade-sequence - print: '**** done end parallel' - print: '**** done start telemetry x...' - workunit: clients: client.0: - test_telemetry_quincy_x.sh - print: '**** done end telemetry x...' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a timestamp: 2024-09-11_01:16:05 tube: smithi upgrade-sequence: sequential: - print: '**** done start upgrade, wait' - cephadm.shell: env: - sha1 mon.a: - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - print: '**** done end upgrade, wait...' user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.3401856 workload: full_sequential: - print: '**** done start ec-rados-default.yaml' - rados: clients: - client.0 ec_pool: true objects: 50 op_weights: append: 100 copy_from: 50 delete: 50 read: 100 rmattr: 25 rollback: 50 setattr: 25 snap_create: 50 snap_remove: 50 write: 0 ops: 4000 write_append_excl: false - print: '**** done end ec-rados-default.yaml' - print: '**** done start rados_api.yaml' - workunit: branch: quincy clients: client.0: - cls - print: '**** done end rados_api.yaml' - print: '**** done start rados_loadgenbig.yaml' - workunit: branch: quincy clients: client.0: - rados/load-gen-big.sh - print: '**** done end rados_loadgenbig.yaml' - print: '**** done start rbd_import_export.yaml' - workunit: branch: quincy clients: client.1: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done end rbd_import_export.yaml' - print: '**** done start test_rbd_api.yaml' - workunit: branch: quincy clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done end test_rbd_api.yaml' - print: '**** done start test_rbd_python.yaml' - workunit: branch: quincy clients: client.0: - rbd/test_librbd_python.sh env: RBD_FEATURES: '61' - print: '**** done end test_rbd_python.yaml' 2024-09-12T23:10:04.615 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_7a853b1706d076d58e0da7a5d3b9b9eb36276880/qa; will attempt to use it 2024-09-12T23:10:04.615 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_7a853b1706d076d58e0da7a5d3b9b9eb36276880/qa/tasks 2024-09-12T23:10:04.615 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-09-12T23:10:04.617 INFO:teuthology.task.internal:Checking packages... 2024-09-12T23:10:04.639 INFO:teuthology.task.internal:Checking packages for os_type 'ubuntu', flavor 'default' and ceph hash '7a853b1706d076d58e0da7a5d3b9b9eb36276880' 2024-09-12T23:10:04.639 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-12T23:10:04.639 INFO:teuthology.packaging:ref: None 2024-09-12T23:10:04.639 INFO:teuthology.packaging:tag: None 2024-09-12T23:10:04.639 INFO:teuthology.packaging:branch: reef 2024-09-12T23:10:04.639 INFO:teuthology.packaging:sha1: 7a853b1706d076d58e0da7a5d3b9b9eb36276880 2024-09-12T23:10:04.639 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&ref=reef 2024-09-12T23:10:04.856 INFO:teuthology.task.internal:Found packages for ceph version 18.2.4-841-g72e3ed98-1jammy 2024-09-12T23:10:04.858 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-09-12T23:10:04.864 INFO:teuthology.task.internal:no buildpackages task found 2024-09-12T23:10:04.864 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-09-12T23:10:04.903 INFO:teuthology.task.internal:Saving configuration 2024-09-12T23:10:04.934 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-09-12T23:10:04.941 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-09-12T23:10:04.970 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi136.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-11_01:16:05-upgrade:quincy-x-reef-distro-default-smithi/7901063', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '22.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-12 23:03:18.748220', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIH2TZmL4d+wIdhPXXb3kzaoWLtRHZI/NIpOdY/HMj5V4'} 2024-09-12T23:10:04.990 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi181.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-11_01:16:05-upgrade:quincy-x-reef-distro-default-smithi/7901063', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '22.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-12 23:03:18.750193', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINp3H5cJIgQPmldG8NdikObQnNpg8Q9bEB4rqr93yiqf'} 2024-09-12T23:10:04.991 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-09-12T23:10:04.997 INFO:teuthology.task.internal:roles: ubuntu@smithi136.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'mds.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-09-12T23:10:04.997 INFO:teuthology.task.internal:roles: ubuntu@smithi181.front.sepia.ceph.com - ['mon.b', 'mds.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-09-12T23:10:04.997 INFO:teuthology.run_tasks:Running task console_log... 2024-09-12T23:10:05.084 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7fe81102b490>, signals=[15]) 2024-09-12T23:10:05.084 INFO:teuthology.run_tasks:Running task internal.connect... 2024-09-12T23:10:05.090 INFO:teuthology.task.internal:Opening connections... 2024-09-12T23:10:05.090 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi136.front.sepia.ceph.com 2024-09-12T23:10:05.091 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi136.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-12T23:10:05.171 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi181.front.sepia.ceph.com 2024-09-12T23:10:05.171 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi181.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-12T23:10:05.251 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-09-12T23:10:05.257 DEBUG:teuthology.orchestra.run.smithi136:> uname -m 2024-09-12T23:10:05.263 INFO:teuthology.orchestra.run.smithi136.stdout:x86_64 2024-09-12T23:10:05.264 DEBUG:teuthology.orchestra.run.smithi136:> cat /etc/os-release 2024-09-12T23:10:05.308 INFO:teuthology.orchestra.run.smithi136.stdout:PRETTY_NAME="Ubuntu 22.04.1 LTS" 2024-09-12T23:10:05.308 INFO:teuthology.orchestra.run.smithi136.stdout:NAME="Ubuntu" 2024-09-12T23:10:05.308 INFO:teuthology.orchestra.run.smithi136.stdout:VERSION_ID="22.04" 2024-09-12T23:10:05.308 INFO:teuthology.orchestra.run.smithi136.stdout:VERSION="22.04.1 LTS (Jammy Jellyfish)" 2024-09-12T23:10:05.309 INFO:teuthology.orchestra.run.smithi136.stdout:VERSION_CODENAME=jammy 2024-09-12T23:10:05.309 INFO:teuthology.orchestra.run.smithi136.stdout:ID=ubuntu 2024-09-12T23:10:05.309 INFO:teuthology.orchestra.run.smithi136.stdout:ID_LIKE=debian 2024-09-12T23:10:05.309 INFO:teuthology.orchestra.run.smithi136.stdout:HOME_URL="https://www.ubuntu.com/" 2024-09-12T23:10:05.309 INFO:teuthology.orchestra.run.smithi136.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2024-09-12T23:10:05.309 INFO:teuthology.orchestra.run.smithi136.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2024-09-12T23:10:05.309 INFO:teuthology.orchestra.run.smithi136.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2024-09-12T23:10:05.309 INFO:teuthology.orchestra.run.smithi136.stdout:UBUNTU_CODENAME=jammy 2024-09-12T23:10:05.313 INFO:teuthology.lock.ops:Updating smithi136.front.sepia.ceph.com on lock server 2024-09-12T23:10:05.336 DEBUG:teuthology.orchestra.run.smithi181:> uname -m 2024-09-12T23:10:05.341 INFO:teuthology.orchestra.run.smithi181.stdout:x86_64 2024-09-12T23:10:05.342 DEBUG:teuthology.orchestra.run.smithi181:> cat /etc/os-release 2024-09-12T23:10:05.388 INFO:teuthology.orchestra.run.smithi181.stdout:PRETTY_NAME="Ubuntu 22.04.1 LTS" 2024-09-12T23:10:05.388 INFO:teuthology.orchestra.run.smithi181.stdout:NAME="Ubuntu" 2024-09-12T23:10:05.388 INFO:teuthology.orchestra.run.smithi181.stdout:VERSION_ID="22.04" 2024-09-12T23:10:05.389 INFO:teuthology.orchestra.run.smithi181.stdout:VERSION="22.04.1 LTS (Jammy Jellyfish)" 2024-09-12T23:10:05.389 INFO:teuthology.orchestra.run.smithi181.stdout:VERSION_CODENAME=jammy 2024-09-12T23:10:05.389 INFO:teuthology.orchestra.run.smithi181.stdout:ID=ubuntu 2024-09-12T23:10:05.389 INFO:teuthology.orchestra.run.smithi181.stdout:ID_LIKE=debian 2024-09-12T23:10:05.389 INFO:teuthology.orchestra.run.smithi181.stdout:HOME_URL="https://www.ubuntu.com/" 2024-09-12T23:10:05.389 INFO:teuthology.orchestra.run.smithi181.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2024-09-12T23:10:05.389 INFO:teuthology.orchestra.run.smithi181.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2024-09-12T23:10:05.389 INFO:teuthology.orchestra.run.smithi181.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2024-09-12T23:10:05.389 INFO:teuthology.orchestra.run.smithi181.stdout:UBUNTU_CODENAME=jammy 2024-09-12T23:10:05.389 INFO:teuthology.lock.ops:Updating smithi181.front.sepia.ceph.com on lock server 2024-09-12T23:10:05.411 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-09-12T23:10:05.421 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-09-12T23:10:05.494 INFO:teuthology.task.internal:Checking for old test directory... 2024-09-12T23:10:05.495 DEBUG:teuthology.orchestra.run.smithi136:> test '!' -e /home/ubuntu/cephtest 2024-09-12T23:10:05.497 DEBUG:teuthology.orchestra.run.smithi181:> test '!' -e /home/ubuntu/cephtest 2024-09-12T23:10:05.502 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-09-12T23:10:05.530 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-09-12T23:10:05.530 DEBUG:teuthology.orchestra.run.smithi136:> test -z $(ls -A /var/lib/ceph) 2024-09-12T23:10:05.542 DEBUG:teuthology.orchestra.run.smithi181:> test -z $(ls -A /var/lib/ceph) 2024-09-12T23:10:05.560 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-09-12T23:10:05.612 INFO:teuthology.run_tasks:Running task kernel... 2024-09-12T23:10:05.626 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-09-12T23:10:05.626 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-09-12T23:10:05.627 DEBUG:teuthology.orchestra.run.smithi136:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-12T23:10:05.627 DEBUG:teuthology.orchestra.run.smithi181:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-12T23:10:05.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-12T23:10:05.632 DEBUG:teuthology.orchestra.run.smithi136:> uname -r 2024-09-12T23:10:05.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-12T23:10:05.633 DEBUG:teuthology.orchestra.run.smithi181:> uname -r 2024-09-12T23:10:05.681 INFO:teuthology.orchestra.run.smithi181.stdout:5.15.0-56-generic 2024-09-12T23:10:05.681 INFO:teuthology.task.kernel:Running kernel on smithi181: 5.15.0-56-generic 2024-09-12T23:10:05.681 DEBUG:teuthology.orchestra.run.smithi181:> sudo apt-get clean 2024-09-12T23:10:05.682 INFO:teuthology.orchestra.run.smithi136.stdout:5.15.0-56-generic 2024-09-12T23:10:05.682 INFO:teuthology.task.kernel:Running kernel on smithi136: 5.15.0-56-generic 2024-09-12T23:10:05.683 DEBUG:teuthology.orchestra.run.smithi136:> sudo apt-get clean 2024-09-12T23:10:05.794 DEBUG:teuthology.orchestra.run.smithi136:> sudo apt-get update 2024-09-12T23:10:05.829 DEBUG:teuthology.orchestra.run.smithi181:> sudo apt-get update 2024-09-12T23:10:06.060 INFO:teuthology.orchestra.run.smithi136.stdout:Hit:1 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-09-12T23:10:06.060 INFO:teuthology.orchestra.run.smithi136.stdout:Get:2 http://security.ubuntu.com/ubuntu jammy-security InRelease [129 kB] 2024-09-12T23:10:06.115 INFO:teuthology.orchestra.run.smithi136.stdout:Get:3 http://archive.ubuntu.com/ubuntu jammy-updates InRelease [128 kB] 2024-09-12T23:10:06.116 INFO:teuthology.orchestra.run.smithi181.stdout:Hit:1 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-09-12T23:10:06.116 INFO:teuthology.orchestra.run.smithi181.stdout:Get:2 http://security.ubuntu.com/ubuntu jammy-security InRelease [129 kB] 2024-09-12T23:10:06.163 INFO:teuthology.orchestra.run.smithi181.stdout:Get:3 http://archive.ubuntu.com/ubuntu jammy-updates InRelease [128 kB] 2024-09-12T23:10:06.447 INFO:teuthology.orchestra.run.smithi136.stdout:Get:4 http://security.ubuntu.com/ubuntu jammy-security/main i386 Packages [533 kB] 2024-09-12T23:10:06.470 INFO:teuthology.orchestra.run.smithi136.stdout:Get:5 http://archive.ubuntu.com/ubuntu jammy-backports InRelease [127 kB] 2024-09-12T23:10:06.520 INFO:teuthology.orchestra.run.smithi181.stdout:Get:4 http://archive.ubuntu.com/ubuntu jammy-backports InRelease [127 kB] 2024-09-12T23:10:06.535 INFO:teuthology.orchestra.run.smithi181.stdout:Get:5 http://security.ubuntu.com/ubuntu jammy-security/main i386 Packages [533 kB] 2024-09-12T23:10:06.570 INFO:teuthology.orchestra.run.smithi136.stdout:Get:6 http://security.ubuntu.com/ubuntu jammy-security/main amd64 Packages [1,773 kB] 2024-09-12T23:10:06.608 INFO:teuthology.orchestra.run.smithi136.stdout:Get:7 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages [2,021 kB] 2024-09-12T23:10:06.627 INFO:teuthology.orchestra.run.smithi136.stdout:Get:8 http://security.ubuntu.com/ubuntu jammy-security/main Translation-en [291 kB] 2024-09-12T23:10:06.649 INFO:teuthology.orchestra.run.smithi136.stdout:Get:9 http://security.ubuntu.com/ubuntu jammy-security/main amd64 c-n-f Metadata [13.3 kB] 2024-09-12T23:10:06.660 INFO:teuthology.orchestra.run.smithi181.stdout:Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/main i386 Packages [696 kB] 2024-09-12T23:10:06.665 INFO:teuthology.orchestra.run.smithi136.stdout:Get:10 http://security.ubuntu.com/ubuntu jammy-security/restricted i386 Packages [37.7 kB] 2024-09-12T23:10:06.683 INFO:teuthology.orchestra.run.smithi136.stdout:Get:11 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 Packages [2,330 kB] 2024-09-12T23:10:06.723 INFO:teuthology.orchestra.run.smithi136.stdout:Get:12 http://security.ubuntu.com/ubuntu jammy-security/restricted Translation-en [401 kB] 2024-09-12T23:10:06.745 INFO:teuthology.orchestra.run.smithi136.stdout:Get:13 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 c-n-f Metadata [584 B] 2024-09-12T23:10:06.762 INFO:teuthology.orchestra.run.smithi136.stdout:Get:14 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 Packages [901 kB] 2024-09-12T23:10:06.789 INFO:teuthology.orchestra.run.smithi136.stdout:Get:15 http://security.ubuntu.com/ubuntu jammy-security/universe i386 Packages [626 kB] 2024-09-12T23:10:06.812 INFO:teuthology.orchestra.run.smithi136.stdout:Get:16 http://security.ubuntu.com/ubuntu jammy-security/universe Translation-en [176 kB] 2024-09-12T23:10:06.826 INFO:teuthology.orchestra.run.smithi181.stdout:Get:7 http://security.ubuntu.com/ubuntu jammy-security/main amd64 Packages [1,773 kB] 2024-09-12T23:10:06.831 INFO:teuthology.orchestra.run.smithi136.stdout:Get:17 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 c-n-f Metadata [19.2 kB] 2024-09-12T23:10:06.849 INFO:teuthology.orchestra.run.smithi136.stdout:Get:18 http://security.ubuntu.com/ubuntu jammy-security/multiverse amd64 Packages [37.2 kB] 2024-09-12T23:10:06.853 INFO:teuthology.orchestra.run.smithi181.stdout:Get:8 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages [2,021 kB] 2024-09-12T23:10:06.866 INFO:teuthology.orchestra.run.smithi136.stdout:Get:19 http://security.ubuntu.com/ubuntu jammy-security/multiverse i386 Packages [1,356 B] 2024-09-12T23:10:06.884 INFO:teuthology.orchestra.run.smithi136.stdout:Get:20 http://security.ubuntu.com/ubuntu jammy-security/multiverse Translation-en [7,588 B] 2024-09-12T23:10:06.926 INFO:teuthology.orchestra.run.smithi136.stdout:Get:21 http://archive.ubuntu.com/ubuntu jammy-updates/main i386 Packages [696 kB] 2024-09-12T23:10:06.961 INFO:teuthology.orchestra.run.smithi136.stdout:Get:22 http://archive.ubuntu.com/ubuntu jammy-updates/main Translation-en [352 kB] 2024-09-12T23:10:06.961 INFO:teuthology.orchestra.run.smithi136.stdout:Get:23 http://security.ubuntu.com/ubuntu jammy-security/multiverse amd64 c-n-f Metadata [228 B] 2024-09-12T23:10:06.976 INFO:teuthology.orchestra.run.smithi136.stdout:Get:24 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 c-n-f Metadata [17.8 kB] 2024-09-12T23:10:06.976 INFO:teuthology.orchestra.run.smithi136.stdout:Get:25 http://archive.ubuntu.com/ubuntu jammy-updates/restricted i386 Packages [39.3 kB] 2024-09-12T23:10:06.978 INFO:teuthology.orchestra.run.smithi136.stdout:Get:26 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 Packages [2,437 kB] 2024-09-12T23:10:06.998 INFO:teuthology.orchestra.run.smithi181.stdout:Get:9 http://security.ubuntu.com/ubuntu jammy-security/main Translation-en [291 kB] 2024-09-12T23:10:07.011 INFO:teuthology.orchestra.run.smithi181.stdout:Get:10 http://security.ubuntu.com/ubuntu jammy-security/main amd64 c-n-f Metadata [13.3 kB] 2024-09-12T23:10:07.012 INFO:teuthology.orchestra.run.smithi181.stdout:Get:11 http://archive.ubuntu.com/ubuntu jammy-updates/main Translation-en [352 kB] 2024-09-12T23:10:07.012 INFO:teuthology.orchestra.run.smithi181.stdout:Get:12 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 Packages [2,330 kB] 2024-09-12T23:10:07.026 INFO:teuthology.orchestra.run.smithi181.stdout:Get:13 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 c-n-f Metadata [17.8 kB] 2024-09-12T23:10:07.027 INFO:teuthology.orchestra.run.smithi181.stdout:Get:14 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 Packages [2,437 kB] 2024-09-12T23:10:07.046 INFO:teuthology.orchestra.run.smithi136.stdout:Get:27 http://archive.ubuntu.com/ubuntu jammy-updates/restricted Translation-en [419 kB] 2024-09-12T23:10:07.063 INFO:teuthology.orchestra.run.smithi136.stdout:Get:28 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 c-n-f Metadata [616 B] 2024-09-12T23:10:07.063 INFO:teuthology.orchestra.run.smithi136.stdout:Get:29 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 Packages [1,124 kB] 2024-09-12T23:10:07.094 INFO:teuthology.orchestra.run.smithi136.stdout:Get:30 http://archive.ubuntu.com/ubuntu jammy-updates/universe i386 Packages [732 kB] 2024-09-12T23:10:07.096 INFO:teuthology.orchestra.run.smithi181.stdout:Get:15 http://archive.ubuntu.com/ubuntu jammy-updates/restricted i386 Packages [39.3 kB] 2024-09-12T23:10:07.096 INFO:teuthology.orchestra.run.smithi181.stdout:Get:16 http://security.ubuntu.com/ubuntu jammy-security/restricted i386 Packages [37.7 kB] 2024-09-12T23:10:07.113 INFO:teuthology.orchestra.run.smithi136.stdout:Get:31 http://archive.ubuntu.com/ubuntu jammy-updates/universe Translation-en [261 kB] 2024-09-12T23:10:07.119 INFO:teuthology.orchestra.run.smithi136.stdout:Get:32 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 c-n-f Metadata [26.2 kB] 2024-09-12T23:10:07.119 INFO:teuthology.orchestra.run.smithi136.stdout:Get:33 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse i386 Packages [4,752 B] 2024-09-12T23:10:07.120 INFO:teuthology.orchestra.run.smithi136.stdout:Get:34 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse amd64 Packages [43.3 kB] 2024-09-12T23:10:07.121 INFO:teuthology.orchestra.run.smithi136.stdout:Get:35 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse Translation-en [10.8 kB] 2024-09-12T23:10:07.169 INFO:teuthology.orchestra.run.smithi181.stdout:Get:17 http://archive.ubuntu.com/ubuntu jammy-updates/restricted Translation-en [419 kB] 2024-09-12T23:10:07.169 INFO:teuthology.orchestra.run.smithi181.stdout:Get:18 http://security.ubuntu.com/ubuntu jammy-security/restricted Translation-en [401 kB] 2024-09-12T23:10:07.172 INFO:teuthology.orchestra.run.smithi181.stdout:Get:19 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 c-n-f Metadata [616 B] 2024-09-12T23:10:07.172 INFO:teuthology.orchestra.run.smithi181.stdout:Get:20 http://archive.ubuntu.com/ubuntu jammy-updates/universe i386 Packages [732 kB] 2024-09-12T23:10:07.173 INFO:teuthology.orchestra.run.smithi136.stdout:Get:36 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse amd64 c-n-f Metadata [444 B] 2024-09-12T23:10:07.173 INFO:teuthology.orchestra.run.smithi136.stdout:Get:37 http://archive.ubuntu.com/ubuntu jammy-backports/main i386 Packages [59.9 kB] 2024-09-12T23:10:07.173 INFO:teuthology.orchestra.run.smithi181.stdout:Get:21 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 c-n-f Metadata [584 B] 2024-09-12T23:10:07.173 INFO:teuthology.orchestra.run.smithi181.stdout:Get:22 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 Packages [901 kB] 2024-09-12T23:10:07.174 INFO:teuthology.orchestra.run.smithi136.stdout:Get:38 http://archive.ubuntu.com/ubuntu jammy-backports/main amd64 Packages [67.8 kB] 2024-09-12T23:10:07.178 INFO:teuthology.orchestra.run.smithi181.stdout:Get:23 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 Packages [1,124 kB] 2024-09-12T23:10:07.181 INFO:teuthology.orchestra.run.smithi181.stdout:Get:24 http://security.ubuntu.com/ubuntu jammy-security/universe i386 Packages [626 kB] 2024-09-12T23:10:07.188 INFO:teuthology.orchestra.run.smithi181.stdout:Get:25 http://archive.ubuntu.com/ubuntu jammy-updates/universe Translation-en [261 kB] 2024-09-12T23:10:07.188 INFO:teuthology.orchestra.run.smithi181.stdout:Get:26 http://security.ubuntu.com/ubuntu jammy-security/universe Translation-en [176 kB] 2024-09-12T23:10:07.190 INFO:teuthology.orchestra.run.smithi181.stdout:Get:27 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 c-n-f Metadata [19.2 kB] 2024-09-12T23:10:07.190 INFO:teuthology.orchestra.run.smithi181.stdout:Get:28 http://security.ubuntu.com/ubuntu jammy-security/multiverse amd64 Packages [37.2 kB] 2024-09-12T23:10:07.190 INFO:teuthology.orchestra.run.smithi181.stdout:Get:29 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 c-n-f Metadata [26.2 kB] 2024-09-12T23:10:07.190 INFO:teuthology.orchestra.run.smithi181.stdout:Get:30 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse amd64 Packages [43.3 kB] 2024-09-12T23:10:07.194 INFO:teuthology.orchestra.run.smithi181.stdout:Get:31 http://security.ubuntu.com/ubuntu jammy-security/multiverse i386 Packages [1,356 B] 2024-09-12T23:10:07.194 INFO:teuthology.orchestra.run.smithi181.stdout:Get:32 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse i386 Packages [4,752 B] 2024-09-12T23:10:07.203 INFO:teuthology.orchestra.run.smithi136.stdout:Get:39 http://archive.ubuntu.com/ubuntu jammy-backports/main Translation-en [11.1 kB] 2024-09-12T23:10:07.204 INFO:teuthology.orchestra.run.smithi136.stdout:Get:40 http://archive.ubuntu.com/ubuntu jammy-backports/main amd64 c-n-f Metadata [388 B] 2024-09-12T23:10:07.204 INFO:teuthology.orchestra.run.smithi136.stdout:Get:41 http://archive.ubuntu.com/ubuntu jammy-backports/universe amd64 Packages [28.8 kB] 2024-09-12T23:10:07.205 INFO:teuthology.orchestra.run.smithi136.stdout:Get:42 http://archive.ubuntu.com/ubuntu jammy-backports/universe i386 Packages [17.3 kB] 2024-09-12T23:10:07.205 INFO:teuthology.orchestra.run.smithi136.stdout:Get:43 http://archive.ubuntu.com/ubuntu jammy-backports/universe Translation-en [16.5 kB] 2024-09-12T23:10:07.254 INFO:teuthology.orchestra.run.smithi136.stdout:Get:44 http://archive.ubuntu.com/ubuntu jammy-backports/universe amd64 c-n-f Metadata [672 B] 2024-09-12T23:10:07.257 INFO:teuthology.orchestra.run.smithi181.stdout:Get:33 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse Translation-en [10.8 kB] 2024-09-12T23:10:07.261 INFO:teuthology.orchestra.run.smithi181.stdout:Get:34 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse amd64 c-n-f Metadata [444 B] 2024-09-12T23:10:07.261 INFO:teuthology.orchestra.run.smithi181.stdout:Get:35 http://archive.ubuntu.com/ubuntu jammy-backports/main i386 Packages [59.9 kB] 2024-09-12T23:10:07.261 INFO:teuthology.orchestra.run.smithi181.stdout:Get:36 http://archive.ubuntu.com/ubuntu jammy-backports/main amd64 Packages [67.8 kB] 2024-09-12T23:10:07.277 INFO:teuthology.orchestra.run.smithi181.stdout:Get:37 http://archive.ubuntu.com/ubuntu jammy-backports/main Translation-en [11.1 kB] 2024-09-12T23:10:07.277 INFO:teuthology.orchestra.run.smithi181.stdout:Get:38 http://archive.ubuntu.com/ubuntu jammy-backports/main amd64 c-n-f Metadata [388 B] 2024-09-12T23:10:07.277 INFO:teuthology.orchestra.run.smithi181.stdout:Get:39 http://archive.ubuntu.com/ubuntu jammy-backports/universe amd64 Packages [28.8 kB] 2024-09-12T23:10:07.278 INFO:teuthology.orchestra.run.smithi181.stdout:Get:40 http://archive.ubuntu.com/ubuntu jammy-backports/universe i386 Packages [17.3 kB] 2024-09-12T23:10:07.278 INFO:teuthology.orchestra.run.smithi181.stdout:Get:41 http://archive.ubuntu.com/ubuntu jammy-backports/universe Translation-en [16.5 kB] 2024-09-12T23:10:07.339 INFO:teuthology.orchestra.run.smithi181.stdout:Get:42 http://archive.ubuntu.com/ubuntu jammy-backports/universe amd64 c-n-f Metadata [672 B] 2024-09-12T23:10:07.377 INFO:teuthology.orchestra.run.smithi181.stdout:Get:43 http://security.ubuntu.com/ubuntu jammy-security/multiverse Translation-en [7,588 B] 2024-09-12T23:10:07.377 INFO:teuthology.orchestra.run.smithi181.stdout:Get:44 http://security.ubuntu.com/ubuntu jammy-security/multiverse amd64 c-n-f Metadata [228 B] 2024-09-12T23:10:13.038 INFO:teuthology.orchestra.run.smithi136.stdout:Fetched 15.9 MB in 2s (6,402 kB/s) 2024-09-12T23:10:13.259 INFO:teuthology.orchestra.run.smithi181.stdout:Fetched 15.9 MB in 3s (5,843 kB/s) 2024-09-12T23:10:14.444 INFO:teuthology.orchestra.run.smithi136.stdout:Reading package lists... 2024-09-12T23:10:14.463 DEBUG:teuthology.orchestra.run.smithi136:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2024-09-12T23:10:14.530 INFO:teuthology.orchestra.run.smithi136.stdout:Reading package lists... 2024-09-12T23:10:14.737 INFO:teuthology.orchestra.run.smithi136.stdout:Building dependency tree... 2024-09-12T23:10:14.737 INFO:teuthology.orchestra.run.smithi136.stdout:Reading state information... 2024-09-12T23:10:14.779 INFO:teuthology.orchestra.run.smithi181.stdout:Reading package lists... 2024-09-12T23:10:14.798 DEBUG:teuthology.orchestra.run.smithi181:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2024-09-12T23:10:14.861 INFO:teuthology.orchestra.run.smithi181.stdout:Reading package lists... 2024-09-12T23:10:14.924 INFO:teuthology.orchestra.run.smithi136.stdout:The following packages were automatically installed and are no longer required: 2024-09-12T23:10:14.924 INFO:teuthology.orchestra.run.smithi136.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-09-12T23:10:14.924 INFO:teuthology.orchestra.run.smithi136.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-09-12T23:10:14.925 INFO:teuthology.orchestra.run.smithi136.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-09-12T23:10:14.925 INFO:teuthology.orchestra.run.smithi136.stdout: python2 python2-dev python2-minimal python2.7 python2.7-dev 2024-09-12T23:10:14.925 INFO:teuthology.orchestra.run.smithi136.stdout: python2.7-minimal 2024-09-12T23:10:14.925 INFO:teuthology.orchestra.run.smithi136.stdout:Use 'sudo apt autoremove' to remove them. 2024-09-12T23:10:14.926 INFO:teuthology.orchestra.run.smithi136.stdout:The following additional packages will be installed: 2024-09-12T23:10:14.926 INFO:teuthology.orchestra.run.smithi136.stdout: linux-generic linux-headers-5.15.0-121 linux-headers-5.15.0-121-generic 2024-09-12T23:10:14.926 INFO:teuthology.orchestra.run.smithi136.stdout: linux-headers-generic linux-image-5.15.0-121-generic 2024-09-12T23:10:14.927 INFO:teuthology.orchestra.run.smithi136.stdout: linux-modules-5.15.0-121-generic linux-modules-extra-5.15.0-121-generic 2024-09-12T23:10:14.928 INFO:teuthology.orchestra.run.smithi136.stdout:Suggested packages: 2024-09-12T23:10:14.928 INFO:teuthology.orchestra.run.smithi136.stdout: fdutils linux-doc | linux-source-5.15.0 linux-tools 2024-09-12T23:10:14.928 INFO:teuthology.orchestra.run.smithi136.stdout:Recommended packages: 2024-09-12T23:10:14.928 INFO:teuthology.orchestra.run.smithi136.stdout: thermald 2024-09-12T23:10:14.979 INFO:teuthology.orchestra.run.smithi136.stdout:The following NEW packages will be installed: 2024-09-12T23:10:14.980 INFO:teuthology.orchestra.run.smithi136.stdout: linux-headers-5.15.0-121 linux-headers-5.15.0-121-generic 2024-09-12T23:10:14.980 INFO:teuthology.orchestra.run.smithi136.stdout: linux-image-5.15.0-121-generic linux-modules-5.15.0-121-generic 2024-09-12T23:10:14.980 INFO:teuthology.orchestra.run.smithi136.stdout: linux-modules-extra-5.15.0-121-generic 2024-09-12T23:10:14.981 INFO:teuthology.orchestra.run.smithi136.stdout:The following packages will be upgraded: 2024-09-12T23:10:14.982 INFO:teuthology.orchestra.run.smithi136.stdout: linux-generic linux-headers-generic linux-image-generic 2024-09-12T23:10:15.033 INFO:teuthology.orchestra.run.smithi136.stdout:3 upgraded, 5 newly installed, 0 to remove and 344 not upgraded. 2024-09-12T23:10:15.033 INFO:teuthology.orchestra.run.smithi136.stdout:Need to get 113 MB of archives. 2024-09-12T23:10:15.033 INFO:teuthology.orchestra.run.smithi136.stdout:After this operation, 584 MB of additional disk space will be used. 2024-09-12T23:10:15.033 INFO:teuthology.orchestra.run.smithi136.stdout:Get:1 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-modules-5.15.0-121-generic amd64 5.15.0-121.131 [22.7 MB] 2024-09-12T23:10:15.049 INFO:teuthology.orchestra.run.smithi181.stdout:Building dependency tree... 2024-09-12T23:10:15.050 INFO:teuthology.orchestra.run.smithi181.stdout:Reading state information... 2024-09-12T23:10:15.220 INFO:teuthology.orchestra.run.smithi181.stdout:The following packages were automatically installed and are no longer required: 2024-09-12T23:10:15.220 INFO:teuthology.orchestra.run.smithi181.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-09-12T23:10:15.221 INFO:teuthology.orchestra.run.smithi181.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-09-12T23:10:15.221 INFO:teuthology.orchestra.run.smithi181.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-09-12T23:10:15.221 INFO:teuthology.orchestra.run.smithi181.stdout: python2 python2-dev python2-minimal python2.7 python2.7-dev 2024-09-12T23:10:15.221 INFO:teuthology.orchestra.run.smithi181.stdout: python2.7-minimal 2024-09-12T23:10:15.221 INFO:teuthology.orchestra.run.smithi181.stdout:Use 'sudo apt autoremove' to remove them. 2024-09-12T23:10:15.222 INFO:teuthology.orchestra.run.smithi181.stdout:The following additional packages will be installed: 2024-09-12T23:10:15.222 INFO:teuthology.orchestra.run.smithi181.stdout: linux-generic linux-headers-5.15.0-121 linux-headers-5.15.0-121-generic 2024-09-12T23:10:15.222 INFO:teuthology.orchestra.run.smithi181.stdout: linux-headers-generic linux-image-5.15.0-121-generic 2024-09-12T23:10:15.223 INFO:teuthology.orchestra.run.smithi181.stdout: linux-modules-5.15.0-121-generic linux-modules-extra-5.15.0-121-generic 2024-09-12T23:10:15.224 INFO:teuthology.orchestra.run.smithi181.stdout:Suggested packages: 2024-09-12T23:10:15.224 INFO:teuthology.orchestra.run.smithi181.stdout: fdutils linux-doc | linux-source-5.15.0 linux-tools 2024-09-12T23:10:15.224 INFO:teuthology.orchestra.run.smithi181.stdout:Recommended packages: 2024-09-12T23:10:15.224 INFO:teuthology.orchestra.run.smithi181.stdout: thermald 2024-09-12T23:10:15.265 INFO:teuthology.orchestra.run.smithi181.stdout:The following NEW packages will be installed: 2024-09-12T23:10:15.265 INFO:teuthology.orchestra.run.smithi181.stdout: linux-headers-5.15.0-121 linux-headers-5.15.0-121-generic 2024-09-12T23:10:15.266 INFO:teuthology.orchestra.run.smithi181.stdout: linux-image-5.15.0-121-generic linux-modules-5.15.0-121-generic 2024-09-12T23:10:15.266 INFO:teuthology.orchestra.run.smithi181.stdout: linux-modules-extra-5.15.0-121-generic 2024-09-12T23:10:15.267 INFO:teuthology.orchestra.run.smithi181.stdout:The following packages will be upgraded: 2024-09-12T23:10:15.268 INFO:teuthology.orchestra.run.smithi181.stdout: linux-generic linux-headers-generic linux-image-generic 2024-09-12T23:10:15.447 INFO:teuthology.orchestra.run.smithi181.stdout:3 upgraded, 5 newly installed, 0 to remove and 344 not upgraded. 2024-09-12T23:10:15.447 INFO:teuthology.orchestra.run.smithi181.stdout:Need to get 113 MB of archives. 2024-09-12T23:10:15.447 INFO:teuthology.orchestra.run.smithi181.stdout:After this operation, 584 MB of additional disk space will be used. 2024-09-12T23:10:15.447 INFO:teuthology.orchestra.run.smithi181.stdout:Get:1 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-modules-5.15.0-121-generic amd64 5.15.0-121.131 [22.7 MB] 2024-09-12T23:10:15.464 INFO:teuthology.orchestra.run.smithi136.stdout:Get:2 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-image-5.15.0-121-generic amd64 5.15.0-121.131 [11.5 MB] 2024-09-12T23:10:15.553 INFO:teuthology.orchestra.run.smithi136.stdout:Get:3 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-modules-extra-5.15.0-121-generic amd64 5.15.0-121.131 [63.9 MB] 2024-09-12T23:10:16.341 INFO:teuthology.orchestra.run.smithi136.stdout:Get:4 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-generic amd64 5.15.0.121.121 [1,694 B] 2024-09-12T23:10:16.341 INFO:teuthology.orchestra.run.smithi136.stdout:Get:5 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-image-generic amd64 5.15.0.121.121 [2,464 B] 2024-09-12T23:10:16.341 INFO:teuthology.orchestra.run.smithi136.stdout:Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-5.15.0-121 all 5.15.0-121.131 [12.3 MB] 2024-09-12T23:10:16.465 INFO:teuthology.orchestra.run.smithi136.stdout:Get:7 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-5.15.0-121-generic amd64 5.15.0-121.131 [2,827 kB] 2024-09-12T23:10:16.485 INFO:teuthology.orchestra.run.smithi136.stdout:Get:8 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-generic amd64 5.15.0.121.121 [2,316 B] 2024-09-12T23:10:16.846 INFO:teuthology.orchestra.run.smithi181.stdout:Get:2 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-image-5.15.0-121-generic amd64 5.15.0-121.131 [11.5 MB] 2024-09-12T23:10:16.885 INFO:teuthology.orchestra.run.smithi136.stdout:Fetched 113 MB in 1s (75.7 MB/s) 2024-09-12T23:10:17.025 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package linux-modules-5.15.0-121-generic. 2024-09-12T23:10:17.195 INFO:teuthology.orchestra.run.smithi181.stdout:Get:3 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-modules-extra-5.15.0-121-generic amd64 5.15.0-121.131 [63.9 MB] 2024-09-12T23:10:19.004 INFO:teuthology.orchestra.run.smithi136.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 129542 files and directories currently installed.) 2024-09-12T23:10:19.008 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../0-linux-modules-5.15.0-121-generic_5.15.0-121.131_amd64.deb ... 2024-09-12T23:10:19.158 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking linux-modules-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-12T23:10:19.455 INFO:teuthology.orchestra.run.smithi181.stdout:Get:4 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-generic amd64 5.15.0.121.121 [1,694 B] 2024-09-12T23:10:19.455 INFO:teuthology.orchestra.run.smithi181.stdout:Get:5 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-image-generic amd64 5.15.0.121.121 [2,464 B] 2024-09-12T23:10:19.456 INFO:teuthology.orchestra.run.smithi181.stdout:Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-5.15.0-121 all 5.15.0-121.131 [12.3 MB] 2024-09-12T23:10:19.801 INFO:teuthology.orchestra.run.smithi181.stdout:Get:7 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-5.15.0-121-generic amd64 5.15.0-121.131 [2,827 kB] 2024-09-12T23:10:19.879 INFO:teuthology.orchestra.run.smithi181.stdout:Get:8 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-generic amd64 5.15.0.121.121 [2,316 B] 2024-09-12T23:10:20.290 INFO:teuthology.orchestra.run.smithi181.stdout:Fetched 113 MB in 5s (24.6 MB/s) 2024-09-12T23:10:20.482 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package linux-modules-5.15.0-121-generic. 2024-09-12T23:10:21.143 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package linux-image-5.15.0-121-generic. 2024-09-12T23:10:21.164 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../1-linux-image-5.15.0-121-generic_5.15.0-121.131_amd64.deb ... 2024-09-12T23:10:21.247 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking linux-image-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-12T23:10:21.705 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package linux-modules-extra-5.15.0-121-generic. 2024-09-12T23:10:21.725 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../2-linux-modules-extra-5.15.0-121-generic_5.15.0-121.131_amd64.deb ... 2024-09-12T23:10:21.763 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking linux-modules-extra-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-12T23:10:22.533 INFO:teuthology.orchestra.run.smithi181.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 129542 files and directories currently installed.) 2024-09-12T23:10:22.537 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../0-linux-modules-5.15.0-121-generic_5.15.0-121.131_amd64.deb ... 2024-09-12T23:10:22.887 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking linux-modules-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-12T23:10:24.997 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package linux-image-5.15.0-121-generic. 2024-09-12T23:10:25.019 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../1-linux-image-5.15.0-121-generic_5.15.0-121.131_amd64.deb ... 2024-09-12T23:10:25.104 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking linux-image-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-12T23:10:25.575 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package linux-modules-extra-5.15.0-121-generic. 2024-09-12T23:10:25.596 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../2-linux-modules-extra-5.15.0-121-generic_5.15.0-121.131_amd64.deb ... 2024-09-12T23:10:25.633 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking linux-modules-extra-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-12T23:10:28.216 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../3-linux-generic_5.15.0.121.121_amd64.deb ... 2024-09-12T23:10:28.344 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking linux-generic (5.15.0.121.121) over (5.15.0.56.54) ... 2024-09-12T23:10:28.659 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../4-linux-image-generic_5.15.0.121.121_amd64.deb ... 2024-09-12T23:10:28.781 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking linux-image-generic (5.15.0.121.121) over (5.15.0.56.54) ... 2024-09-12T23:10:29.025 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package linux-headers-5.15.0-121. 2024-09-12T23:10:29.036 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../5-linux-headers-5.15.0-121_5.15.0-121.131_all.deb ... 2024-09-12T23:10:29.075 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking linux-headers-5.15.0-121 (5.15.0-121.131) ... 2024-09-12T23:10:32.653 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../3-linux-generic_5.15.0.121.121_amd64.deb ... 2024-09-12T23:10:32.786 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking linux-generic (5.15.0.121.121) over (5.15.0.56.54) ... 2024-09-12T23:10:33.127 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../4-linux-image-generic_5.15.0.121.121_amd64.deb ... 2024-09-12T23:10:33.239 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking linux-image-generic (5.15.0.121.121) over (5.15.0.56.54) ... 2024-09-12T23:10:33.559 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package linux-headers-5.15.0-121. 2024-09-12T23:10:33.570 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../5-linux-headers-5.15.0-121_5.15.0-121.131_all.deb ... 2024-09-12T23:10:33.609 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking linux-headers-5.15.0-121 (5.15.0-121.131) ... 2024-09-12T23:10:34.241 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package linux-headers-5.15.0-121-generic. 2024-09-12T23:10:34.272 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../6-linux-headers-5.15.0-121-generic_5.15.0-121.131_amd64.deb ... 2024-09-12T23:10:34.316 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking linux-headers-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-12T23:10:37.173 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../7-linux-headers-generic_5.15.0.121.121_amd64.deb ... 2024-09-12T23:10:37.296 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking linux-headers-generic (5.15.0.121.121) over (5.15.0.56.54) ... 2024-09-12T23:10:37.595 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up linux-headers-5.15.0-121 (5.15.0-121.131) ... 2024-09-12T23:10:37.718 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up linux-headers-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-12T23:10:37.844 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up linux-headers-generic (5.15.0.121.121) ... 2024-09-12T23:10:37.972 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up linux-image-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-12T23:10:39.310 INFO:teuthology.orchestra.run.smithi136.stdout:I: /vmlinuz.old is now a symlink to boot/vmlinuz-5.15.0-56-generic 2024-09-12T23:10:39.310 INFO:teuthology.orchestra.run.smithi136.stdout:I: /initrd.img.old is now a symlink to boot/initrd.img-5.15.0-56-generic 2024-09-12T23:10:39.310 INFO:teuthology.orchestra.run.smithi136.stdout:I: /vmlinuz is now a symlink to boot/vmlinuz-5.15.0-121-generic 2024-09-12T23:10:39.311 INFO:teuthology.orchestra.run.smithi136.stdout:I: /initrd.img is now a symlink to boot/initrd.img-5.15.0-121-generic 2024-09-12T23:10:39.468 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up linux-modules-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-12T23:10:39.637 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package linux-headers-5.15.0-121-generic. 2024-09-12T23:10:39.660 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../6-linux-headers-5.15.0-121-generic_5.15.0-121.131_amd64.deb ... 2024-09-12T23:10:39.703 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking linux-headers-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-12T23:10:40.901 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up linux-modules-extra-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-12T23:10:42.620 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up linux-image-generic (5.15.0.121.121) ... 2024-09-12T23:10:42.746 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up linux-generic (5.15.0.121.121) ... 2024-09-12T23:10:42.874 INFO:teuthology.orchestra.run.smithi136.stdout:Processing triggers for linux-image-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-12T23:10:42.923 INFO:teuthology.orchestra.run.smithi136.stdout:/etc/kernel/postinst.d/initramfs-tools: 2024-09-12T23:10:42.924 INFO:teuthology.orchestra.run.smithi136.stdout:update-initramfs: Generating /boot/initrd.img-5.15.0-121-generic 2024-09-12T23:10:43.308 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../7-linux-headers-generic_5.15.0.121.121_amd64.deb ... 2024-09-12T23:10:43.423 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking linux-headers-generic (5.15.0.121.121) over (5.15.0.56.54) ... 2024-09-12T23:10:43.773 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up linux-headers-5.15.0-121 (5.15.0-121.131) ... 2024-09-12T23:10:43.895 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up linux-headers-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-12T23:10:44.029 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up linux-headers-generic (5.15.0.121.121) ... 2024-09-12T23:10:44.190 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up linux-image-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-12T23:10:45.519 INFO:teuthology.orchestra.run.smithi181.stdout:I: /vmlinuz.old is now a symlink to boot/vmlinuz-5.15.0-56-generic 2024-09-12T23:10:45.519 INFO:teuthology.orchestra.run.smithi181.stdout:I: /initrd.img.old is now a symlink to boot/initrd.img-5.15.0-56-generic 2024-09-12T23:10:45.519 INFO:teuthology.orchestra.run.smithi181.stdout:I: /vmlinuz is now a symlink to boot/vmlinuz-5.15.0-121-generic 2024-09-12T23:10:45.519 INFO:teuthology.orchestra.run.smithi181.stdout:I: /initrd.img is now a symlink to boot/initrd.img-5.15.0-121-generic 2024-09-12T23:10:45.685 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up linux-modules-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-12T23:10:47.136 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up linux-modules-extra-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-12T23:10:48.704 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up linux-image-generic (5.15.0.121.121) ... 2024-09-12T23:10:48.855 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up linux-generic (5.15.0.121.121) ... 2024-09-12T23:10:48.981 INFO:teuthology.orchestra.run.smithi181.stdout:Processing triggers for linux-image-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-12T23:10:49.044 INFO:teuthology.orchestra.run.smithi181.stdout:/etc/kernel/postinst.d/initramfs-tools: 2024-09-12T23:10:49.044 INFO:teuthology.orchestra.run.smithi181.stdout:update-initramfs: Generating /boot/initrd.img-5.15.0-121-generic 2024-09-12T23:10:57.376 INFO:teuthology.orchestra.run.smithi136.stdout:/etc/kernel/postinst.d/zz-update-grub: 2024-09-12T23:10:57.376 INFO:teuthology.orchestra.run.smithi136.stdout:Sourcing file `/etc/default/grub' 2024-09-12T23:10:57.394 INFO:teuthology.orchestra.run.smithi136.stdout:Sourcing file `/etc/default/grub.d/init-select.cfg' 2024-09-12T23:10:57.416 INFO:teuthology.orchestra.run.smithi136.stdout:Generating grub configuration file ... 2024-09-12T23:10:57.676 INFO:teuthology.orchestra.run.smithi136.stdout:Found linux image: /boot/vmlinuz-5.15.0-121-generic 2024-09-12T23:10:57.698 INFO:teuthology.orchestra.run.smithi136.stdout:Found initrd image: /boot/initrd.img-5.15.0-121-generic 2024-09-12T23:10:57.986 INFO:teuthology.orchestra.run.smithi136.stdout:Found linux image: /boot/vmlinuz-5.15.0-56-generic 2024-09-12T23:10:57.991 INFO:teuthology.orchestra.run.smithi136.stdout:Found initrd image: /boot/initrd.img-5.15.0-56-generic 2024-09-12T23:10:58.054 INFO:teuthology.orchestra.run.smithi136.stdout:Found linux image: /boot/vmlinuz-5.4.0-135-generic 2024-09-12T23:10:58.059 INFO:teuthology.orchestra.run.smithi136.stdout:Found initrd image: /boot/initrd.img-5.4.0-135-generic 2024-09-12T23:10:58.213 INFO:teuthology.orchestra.run.smithi136.stdout:Warning: os-prober will not be executed to detect other bootable partitions. 2024-09-12T23:10:58.213 INFO:teuthology.orchestra.run.smithi136.stdout:Systems on them will not be added to the GRUB boot configuration. 2024-09-12T23:10:58.213 INFO:teuthology.orchestra.run.smithi136.stdout:Check GRUB_DISABLE_OS_PROBER documentation entry. 2024-09-12T23:10:58.253 INFO:teuthology.orchestra.run.smithi136.stdout:done 2024-09-12T23:10:58.695 DEBUG:teuthology.orchestra.run.smithi136:> dpkg -s linux-image-generic 2024-09-12T23:10:58.716 INFO:teuthology.orchestra.run.smithi136.stdout:Package: linux-image-generic 2024-09-12T23:10:58.717 INFO:teuthology.orchestra.run.smithi136.stdout:Status: install ok installed 2024-09-12T23:10:58.717 INFO:teuthology.orchestra.run.smithi136.stdout:Priority: optional 2024-09-12T23:10:58.717 INFO:teuthology.orchestra.run.smithi136.stdout:Section: kernel 2024-09-12T23:10:58.717 INFO:teuthology.orchestra.run.smithi136.stdout:Installed-Size: 21 2024-09-12T23:10:58.717 INFO:teuthology.orchestra.run.smithi136.stdout:Maintainer: Ubuntu Kernel Team 2024-09-12T23:10:58.717 INFO:teuthology.orchestra.run.smithi136.stdout:Architecture: amd64 2024-09-12T23:10:58.717 INFO:teuthology.orchestra.run.smithi136.stdout:Source: linux-meta 2024-09-12T23:10:58.717 INFO:teuthology.orchestra.run.smithi136.stdout:Version: 5.15.0.121.121 2024-09-12T23:10:58.717 INFO:teuthology.orchestra.run.smithi136.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.4), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-121), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.4) 2024-09-12T23:10:58.717 INFO:teuthology.orchestra.run.smithi136.stdout:Depends: linux-image-5.15.0-121-generic, linux-modules-extra-5.15.0-121-generic, linux-firmware, intel-microcode, amd64-microcode 2024-09-12T23:10:58.717 INFO:teuthology.orchestra.run.smithi136.stdout:Recommends: thermald 2024-09-12T23:10:58.717 INFO:teuthology.orchestra.run.smithi136.stdout:Description: Generic Linux kernel image 2024-09-12T23:10:58.717 INFO:teuthology.orchestra.run.smithi136.stdout: This package will always depend on the latest generic kernel image 2024-09-12T23:10:58.718 INFO:teuthology.orchestra.run.smithi136.stdout: available. 2024-09-12T23:10:58.718 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.15.0-56-generic Expected: 5.15.0-121-generic 2024-09-12T23:10:58.718 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-12T23:10:58.718 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-12T23:10:58.718 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-09-12T23:10:58.718 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi136.front.sepia.ceph.com, path=None, version=distro) 2024-09-12T23:10:58.718 DEBUG:teuthology.orchestra.run.smithi136:> sudo apt-get clean 2024-09-12T23:10:58.917 DEBUG:teuthology.orchestra.run.smithi136:> sudo apt-get update 2024-09-12T23:10:59.054 INFO:teuthology.orchestra.run.smithi136.stdout:Hit:1 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-09-12T23:10:59.055 INFO:teuthology.orchestra.run.smithi136.stdout:Hit:2 http://security.ubuntu.com/ubuntu jammy-security InRelease 2024-09-12T23:10:59.061 INFO:teuthology.orchestra.run.smithi136.stdout:Hit:3 http://archive.ubuntu.com/ubuntu jammy-updates InRelease 2024-09-12T23:10:59.078 INFO:teuthology.orchestra.run.smithi136.stdout:Hit:4 http://archive.ubuntu.com/ubuntu jammy-backports InRelease 2024-09-12T23:11:00.508 INFO:teuthology.orchestra.run.smithi136.stdout:Reading package lists... 2024-09-12T23:11:00.527 DEBUG:teuthology.orchestra.run.smithi136:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2024-09-12T23:11:00.586 INFO:teuthology.orchestra.run.smithi136.stdout:Reading package lists... 2024-09-12T23:11:00.782 INFO:teuthology.orchestra.run.smithi136.stdout:Building dependency tree... 2024-09-12T23:11:00.783 INFO:teuthology.orchestra.run.smithi136.stdout:Reading state information... 2024-09-12T23:11:00.954 INFO:teuthology.orchestra.run.smithi136.stdout:linux-image-generic is already the newest version (5.15.0.121.121). 2024-09-12T23:11:00.954 INFO:teuthology.orchestra.run.smithi136.stdout:The following packages were automatically installed and are no longer required: 2024-09-12T23:11:00.954 INFO:teuthology.orchestra.run.smithi136.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-09-12T23:11:00.954 INFO:teuthology.orchestra.run.smithi136.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-09-12T23:11:00.954 INFO:teuthology.orchestra.run.smithi136.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-09-12T23:11:00.955 INFO:teuthology.orchestra.run.smithi136.stdout: linux-image-5.4.0-135-generic linux-modules-5.4.0-135-generic python2 2024-09-12T23:11:00.955 INFO:teuthology.orchestra.run.smithi136.stdout: python2-dev python2-minimal python2.7 python2.7-dev python2.7-minimal 2024-09-12T23:11:00.955 INFO:teuthology.orchestra.run.smithi136.stdout:Use 'sudo apt autoremove' to remove them. 2024-09-12T23:11:00.996 INFO:teuthology.orchestra.run.smithi136.stdout:0 upgraded, 0 newly installed, 0 to remove and 344 not upgraded. 2024-09-12T23:11:00.997 DEBUG:teuthology.orchestra.run.smithi136:> dpkg -s linux-image-generic 2024-09-12T23:11:01.051 INFO:teuthology.orchestra.run.smithi136.stdout:Package: linux-image-generic 2024-09-12T23:11:01.051 INFO:teuthology.orchestra.run.smithi136.stdout:Status: install ok installed 2024-09-12T23:11:01.051 INFO:teuthology.orchestra.run.smithi136.stdout:Priority: optional 2024-09-12T23:11:01.051 INFO:teuthology.orchestra.run.smithi136.stdout:Section: kernel 2024-09-12T23:11:01.051 INFO:teuthology.orchestra.run.smithi136.stdout:Installed-Size: 21 2024-09-12T23:11:01.051 INFO:teuthology.orchestra.run.smithi136.stdout:Maintainer: Ubuntu Kernel Team 2024-09-12T23:11:01.051 INFO:teuthology.orchestra.run.smithi136.stdout:Architecture: amd64 2024-09-12T23:11:01.052 INFO:teuthology.orchestra.run.smithi136.stdout:Source: linux-meta 2024-09-12T23:11:01.052 INFO:teuthology.orchestra.run.smithi136.stdout:Version: 5.15.0.121.121 2024-09-12T23:11:01.052 INFO:teuthology.orchestra.run.smithi136.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.4), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-121), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.4) 2024-09-12T23:11:01.052 INFO:teuthology.orchestra.run.smithi136.stdout:Depends: linux-image-5.15.0-121-generic, linux-modules-extra-5.15.0-121-generic, linux-firmware, intel-microcode, amd64-microcode 2024-09-12T23:11:01.052 INFO:teuthology.orchestra.run.smithi136.stdout:Recommends: thermald 2024-09-12T23:11:01.052 INFO:teuthology.orchestra.run.smithi136.stdout:Description: Generic Linux kernel image 2024-09-12T23:11:01.052 INFO:teuthology.orchestra.run.smithi136.stdout: This package will always depend on the latest generic kernel image 2024-09-12T23:11:01.052 INFO:teuthology.orchestra.run.smithi136.stdout: available. 2024-09-12T23:11:01.053 DEBUG:teuthology.orchestra.run.smithi136:> mktemp 2024-09-12T23:11:01.095 INFO:teuthology.orchestra.run.smithi136.stdout:/tmp/tmp.wa9ZhSZTZm 2024-09-12T23:11:01.096 DEBUG:teuthology.orchestra.run.smithi136:> sudo cp /boot/grub/grub.cfg /tmp/tmp.wa9ZhSZTZm 2024-09-12T23:11:01.158 DEBUG:teuthology.orchestra.run.smithi136:> sudo chmod 0666 /tmp/tmp.wa9ZhSZTZm 2024-09-12T23:11:01.283 DEBUG:teuthology.orchestra.remote:smithi136:/tmp/tmp.wa9ZhSZTZm is 10KB 2024-09-12T23:11:01.297 DEBUG:teuthology.orchestra.run.smithi136:> rm -fr /tmp/tmp.wa9ZhSZTZm 2024-09-12T23:11:01.303 DEBUG:teuthology.orchestra.run.smithi136:> sudo rm -f -- /etc/grub.d/01_ceph_kernel 2024-09-12T23:11:01.356 DEBUG:teuthology.orchestra.run.smithi136:> set -ex 2024-09-12T23:11:01.356 DEBUG:teuthology.orchestra.run.smithi136:> sudo dd of=/etc/grub.d/01_ceph_kernel 2024-09-12T23:11:01.356 DEBUG:teuthology.orchestra.run.smithi136:> sudo chmod 755 /etc/grub.d/01_ceph_kernel 2024-09-12T23:11:01.433 INFO:teuthology.task.kernel:Distro Kernel Version: 5.15.0-121-generic 2024-09-12T23:11:01.433 DEBUG:teuthology.orchestra.run.smithi136:> sudo update-grub 2024-09-12T23:11:02.682 INFO:teuthology.orchestra.run.smithi136.stderr:Sourcing file `/etc/default/grub' 2024-09-12T23:11:02.684 INFO:teuthology.orchestra.run.smithi136.stderr:Sourcing file `/etc/default/grub.d/init-select.cfg' 2024-09-12T23:11:02.685 INFO:teuthology.orchestra.run.smithi136.stderr:Generating grub configuration file ... 2024-09-12T23:11:02.858 INFO:teuthology.orchestra.run.smithi136.stderr:Found linux image: /boot/vmlinuz-5.15.0-121-generic 2024-09-12T23:11:02.863 INFO:teuthology.orchestra.run.smithi136.stderr:Found initrd image: /boot/initrd.img-5.15.0-121-generic 2024-09-12T23:11:03.090 INFO:teuthology.orchestra.run.smithi136.stderr:Found linux image: /boot/vmlinuz-5.15.0-56-generic 2024-09-12T23:11:03.096 INFO:teuthology.orchestra.run.smithi136.stderr:Found initrd image: /boot/initrd.img-5.15.0-56-generic 2024-09-12T23:11:03.138 INFO:teuthology.orchestra.run.smithi136.stderr:Found linux image: /boot/vmlinuz-5.4.0-135-generic 2024-09-12T23:11:03.143 INFO:teuthology.orchestra.run.smithi136.stderr:Found initrd image: /boot/initrd.img-5.4.0-135-generic 2024-09-12T23:11:03.302 INFO:teuthology.orchestra.run.smithi136.stderr:Warning: os-prober will not be executed to detect other bootable partitions. 2024-09-12T23:11:03.303 INFO:teuthology.orchestra.run.smithi136.stderr:Systems on them will not be added to the GRUB boot configuration. 2024-09-12T23:11:03.303 INFO:teuthology.orchestra.run.smithi136.stderr:Check GRUB_DISABLE_OS_PROBER documentation entry. 2024-09-12T23:11:03.323 INFO:teuthology.orchestra.run.smithi136.stderr:done 2024-09-12T23:11:03.340 DEBUG:teuthology.orchestra.run.smithi136:> sudo shutdown -r now 2024-09-12T23:11:03.407 INFO:teuthology.orchestra.run.smithi181.stdout:/etc/kernel/postinst.d/zz-update-grub: 2024-09-12T23:11:03.408 INFO:teuthology.orchestra.run.smithi181.stdout:Sourcing file `/etc/default/grub' 2024-09-12T23:11:03.435 INFO:teuthology.orchestra.run.smithi181.stdout:Sourcing file `/etc/default/grub.d/init-select.cfg' 2024-09-12T23:11:03.453 INFO:teuthology.orchestra.run.smithi181.stdout:Generating grub configuration file ... 2024-09-12T23:11:03.742 INFO:teuthology.orchestra.run.smithi181.stdout:Found linux image: /boot/vmlinuz-5.15.0-121-generic 2024-09-12T23:11:03.763 INFO:teuthology.orchestra.run.smithi181.stdout:Found initrd image: /boot/initrd.img-5.15.0-121-generic 2024-09-12T23:11:04.093 INFO:teuthology.orchestra.run.smithi181.stdout:Found linux image: /boot/vmlinuz-5.15.0-56-generic 2024-09-12T23:11:04.098 INFO:teuthology.orchestra.run.smithi181.stdout:Found initrd image: /boot/initrd.img-5.15.0-56-generic 2024-09-12T23:11:04.163 INFO:teuthology.orchestra.run.smithi181.stdout:Found linux image: /boot/vmlinuz-5.4.0-135-generic 2024-09-12T23:11:04.169 INFO:teuthology.orchestra.run.smithi181.stdout:Found initrd image: /boot/initrd.img-5.4.0-135-generic 2024-09-12T23:11:04.332 INFO:teuthology.orchestra.run.smithi181.stdout:Warning: os-prober will not be executed to detect other bootable partitions. 2024-09-12T23:11:04.332 INFO:teuthology.orchestra.run.smithi181.stdout:Systems on them will not be added to the GRUB boot configuration. 2024-09-12T23:11:04.332 INFO:teuthology.orchestra.run.smithi181.stdout:Check GRUB_DISABLE_OS_PROBER documentation entry. 2024-09-12T23:11:04.377 INFO:teuthology.orchestra.run.smithi181.stdout:done 2024-09-12T23:11:04.862 DEBUG:teuthology.orchestra.run.smithi181:> dpkg -s linux-image-generic 2024-09-12T23:11:04.893 INFO:teuthology.orchestra.run.smithi181.stdout:Package: linux-image-generic 2024-09-12T23:11:04.893 INFO:teuthology.orchestra.run.smithi181.stdout:Status: install ok installed 2024-09-12T23:11:04.893 INFO:teuthology.orchestra.run.smithi181.stdout:Priority: optional 2024-09-12T23:11:04.893 INFO:teuthology.orchestra.run.smithi181.stdout:Section: kernel 2024-09-12T23:11:04.893 INFO:teuthology.orchestra.run.smithi181.stdout:Installed-Size: 21 2024-09-12T23:11:04.893 INFO:teuthology.orchestra.run.smithi181.stdout:Maintainer: Ubuntu Kernel Team 2024-09-12T23:11:04.893 INFO:teuthology.orchestra.run.smithi181.stdout:Architecture: amd64 2024-09-12T23:11:04.893 INFO:teuthology.orchestra.run.smithi181.stdout:Source: linux-meta 2024-09-12T23:11:04.893 INFO:teuthology.orchestra.run.smithi181.stdout:Version: 5.15.0.121.121 2024-09-12T23:11:04.893 INFO:teuthology.orchestra.run.smithi181.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.4), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-121), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.4) 2024-09-12T23:11:04.894 INFO:teuthology.orchestra.run.smithi181.stdout:Depends: linux-image-5.15.0-121-generic, linux-modules-extra-5.15.0-121-generic, linux-firmware, intel-microcode, amd64-microcode 2024-09-12T23:11:04.894 INFO:teuthology.orchestra.run.smithi181.stdout:Recommends: thermald 2024-09-12T23:11:04.894 INFO:teuthology.orchestra.run.smithi181.stdout:Description: Generic Linux kernel image 2024-09-12T23:11:04.894 INFO:teuthology.orchestra.run.smithi181.stdout: This package will always depend on the latest generic kernel image 2024-09-12T23:11:04.894 INFO:teuthology.orchestra.run.smithi181.stdout: available. 2024-09-12T23:11:04.894 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.15.0-56-generic Expected: 5.15.0-121-generic 2024-09-12T23:11:04.894 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-12T23:11:04.894 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-12T23:11:04.894 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-09-12T23:11:04.894 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi181.front.sepia.ceph.com, path=None, version=distro) 2024-09-12T23:11:04.895 DEBUG:teuthology.orchestra.run.smithi181:> sudo apt-get clean 2024-09-12T23:11:05.053 DEBUG:teuthology.orchestra.run.smithi181:> sudo apt-get update 2024-09-12T23:11:05.165 INFO:teuthology.orchestra.run.smithi181.stdout:Hit:1 http://security.ubuntu.com/ubuntu jammy-security InRelease 2024-09-12T23:11:05.165 INFO:teuthology.orchestra.run.smithi181.stdout:Hit:2 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-09-12T23:11:05.176 INFO:teuthology.orchestra.run.smithi181.stdout:Hit:3 http://archive.ubuntu.com/ubuntu jammy-updates InRelease 2024-09-12T23:11:05.193 INFO:teuthology.orchestra.run.smithi181.stdout:Hit:4 http://archive.ubuntu.com/ubuntu jammy-backports InRelease 2024-09-12T23:11:06.609 INFO:teuthology.orchestra.run.smithi181.stdout:Reading package lists... 2024-09-12T23:11:06.627 DEBUG:teuthology.orchestra.run.smithi181:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2024-09-12T23:11:06.682 INFO:teuthology.orchestra.run.smithi181.stdout:Reading package lists... 2024-09-12T23:11:06.870 INFO:teuthology.orchestra.run.smithi181.stdout:Building dependency tree... 2024-09-12T23:11:06.870 INFO:teuthology.orchestra.run.smithi181.stdout:Reading state information... 2024-09-12T23:11:07.037 INFO:teuthology.orchestra.run.smithi181.stdout:linux-image-generic is already the newest version (5.15.0.121.121). 2024-09-12T23:11:07.038 INFO:teuthology.orchestra.run.smithi181.stdout:The following packages were automatically installed and are no longer required: 2024-09-12T23:11:07.038 INFO:teuthology.orchestra.run.smithi181.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-09-12T23:11:07.038 INFO:teuthology.orchestra.run.smithi181.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-09-12T23:11:07.038 INFO:teuthology.orchestra.run.smithi181.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-09-12T23:11:07.038 INFO:teuthology.orchestra.run.smithi181.stdout: linux-image-5.4.0-135-generic linux-modules-5.4.0-135-generic python2 2024-09-12T23:11:07.038 INFO:teuthology.orchestra.run.smithi181.stdout: python2-dev python2-minimal python2.7 python2.7-dev python2.7-minimal 2024-09-12T23:11:07.039 INFO:teuthology.orchestra.run.smithi181.stdout:Use 'sudo apt autoremove' to remove them. 2024-09-12T23:11:07.081 INFO:teuthology.orchestra.run.smithi181.stdout:0 upgraded, 0 newly installed, 0 to remove and 344 not upgraded. 2024-09-12T23:11:07.083 DEBUG:teuthology.orchestra.run.smithi181:> dpkg -s linux-image-generic 2024-09-12T23:11:07.139 INFO:teuthology.orchestra.run.smithi181.stdout:Package: linux-image-generic 2024-09-12T23:11:07.139 INFO:teuthology.orchestra.run.smithi181.stdout:Status: install ok installed 2024-09-12T23:11:07.139 INFO:teuthology.orchestra.run.smithi181.stdout:Priority: optional 2024-09-12T23:11:07.139 INFO:teuthology.orchestra.run.smithi181.stdout:Section: kernel 2024-09-12T23:11:07.139 INFO:teuthology.orchestra.run.smithi181.stdout:Installed-Size: 21 2024-09-12T23:11:07.140 INFO:teuthology.orchestra.run.smithi181.stdout:Maintainer: Ubuntu Kernel Team 2024-09-12T23:11:07.140 INFO:teuthology.orchestra.run.smithi181.stdout:Architecture: amd64 2024-09-12T23:11:07.140 INFO:teuthology.orchestra.run.smithi181.stdout:Source: linux-meta 2024-09-12T23:11:07.140 INFO:teuthology.orchestra.run.smithi181.stdout:Version: 5.15.0.121.121 2024-09-12T23:11:07.140 INFO:teuthology.orchestra.run.smithi181.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.4), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-121), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.4) 2024-09-12T23:11:07.140 INFO:teuthology.orchestra.run.smithi181.stdout:Depends: linux-image-5.15.0-121-generic, linux-modules-extra-5.15.0-121-generic, linux-firmware, intel-microcode, amd64-microcode 2024-09-12T23:11:07.140 INFO:teuthology.orchestra.run.smithi181.stdout:Recommends: thermald 2024-09-12T23:11:07.140 INFO:teuthology.orchestra.run.smithi181.stdout:Description: Generic Linux kernel image 2024-09-12T23:11:07.141 INFO:teuthology.orchestra.run.smithi181.stdout: This package will always depend on the latest generic kernel image 2024-09-12T23:11:07.141 INFO:teuthology.orchestra.run.smithi181.stdout: available. 2024-09-12T23:11:07.141 DEBUG:teuthology.orchestra.run.smithi181:> mktemp 2024-09-12T23:11:07.187 INFO:teuthology.orchestra.run.smithi181.stdout:/tmp/tmp.Geh793lVa6 2024-09-12T23:11:07.187 DEBUG:teuthology.orchestra.run.smithi181:> sudo cp /boot/grub/grub.cfg /tmp/tmp.Geh793lVa6 2024-09-12T23:11:07.241 DEBUG:teuthology.orchestra.run.smithi181:> sudo chmod 0666 /tmp/tmp.Geh793lVa6 2024-09-12T23:11:07.370 DEBUG:teuthology.orchestra.remote:smithi181:/tmp/tmp.Geh793lVa6 is 10KB 2024-09-12T23:11:07.382 DEBUG:teuthology.orchestra.run.smithi181:> rm -fr /tmp/tmp.Geh793lVa6 2024-09-12T23:11:07.387 DEBUG:teuthology.orchestra.run.smithi181:> sudo rm -f -- /etc/grub.d/01_ceph_kernel 2024-09-12T23:11:07.438 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2024-09-12T23:11:07.439 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd of=/etc/grub.d/01_ceph_kernel 2024-09-12T23:11:07.439 DEBUG:teuthology.orchestra.run.smithi181:> sudo chmod 755 /etc/grub.d/01_ceph_kernel 2024-09-12T23:11:07.501 INFO:teuthology.task.kernel:Distro Kernel Version: 5.15.0-121-generic 2024-09-12T23:11:07.502 DEBUG:teuthology.orchestra.run.smithi181:> sudo update-grub 2024-09-12T23:11:08.756 INFO:teuthology.orchestra.run.smithi181.stderr:Sourcing file `/etc/default/grub' 2024-09-12T23:11:08.757 INFO:teuthology.orchestra.run.smithi181.stderr:Sourcing file `/etc/default/grub.d/init-select.cfg' 2024-09-12T23:11:08.759 INFO:teuthology.orchestra.run.smithi181.stderr:Generating grub configuration file ... 2024-09-12T23:11:08.998 INFO:teuthology.orchestra.run.smithi181.stderr:Found linux image: /boot/vmlinuz-5.15.0-121-generic 2024-09-12T23:11:09.004 INFO:teuthology.orchestra.run.smithi181.stderr:Found initrd image: /boot/initrd.img-5.15.0-121-generic 2024-09-12T23:11:09.240 INFO:teuthology.orchestra.run.smithi181.stderr:Found linux image: /boot/vmlinuz-5.15.0-56-generic 2024-09-12T23:11:09.246 INFO:teuthology.orchestra.run.smithi181.stderr:Found initrd image: /boot/initrd.img-5.15.0-56-generic 2024-09-12T23:11:09.287 INFO:teuthology.orchestra.run.smithi181.stderr:Found linux image: /boot/vmlinuz-5.4.0-135-generic 2024-09-12T23:11:09.293 INFO:teuthology.orchestra.run.smithi181.stderr:Found initrd image: /boot/initrd.img-5.4.0-135-generic 2024-09-12T23:11:09.430 INFO:teuthology.orchestra.run.smithi181.stderr:Warning: os-prober will not be executed to detect other bootable partitions. 2024-09-12T23:11:09.431 INFO:teuthology.orchestra.run.smithi181.stderr:Systems on them will not be added to the GRUB boot configuration. 2024-09-12T23:11:09.431 INFO:teuthology.orchestra.run.smithi181.stderr:Check GRUB_DISABLE_OS_PROBER documentation entry. 2024-09-12T23:11:09.452 INFO:teuthology.orchestra.run.smithi181.stderr:done 2024-09-12T23:11:09.455 DEBUG:teuthology.orchestra.run.smithi181:> sudo shutdown -r now 2024-09-12T23:11:33.366 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-09-12T23:11:33.367 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi136.front.sepia.ceph.com' 2024-09-12T23:11:33.368 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi136.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-12T23:11:39.461 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-09-12T23:11:39.461 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi181.front.sepia.ceph.com' 2024-09-12T23:11:39.462 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi181.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-12T23:11:51.869 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.136 2024-09-12T23:11:58.013 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.181 2024-09-12T23:12:00.873 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi136.front.sepia.ceph.com' 2024-09-12T23:12:00.875 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi136.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-12T23:12:03.933 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.136 2024-09-12T23:12:07.017 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi181.front.sepia.ceph.com' 2024-09-12T23:12:07.017 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi181.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-12T23:12:10.077 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.181 2024-09-12T23:12:15.937 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi136.front.sepia.ceph.com' 2024-09-12T23:12:15.937 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi136.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-12T23:12:22.083 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi181.front.sepia.ceph.com' 2024-09-12T23:12:22.084 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi181.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-12T23:12:40.509 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.181 2024-09-12T23:12:47.421 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.136 2024-09-12T23:12:55.518 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi181.front.sepia.ceph.com' 2024-09-12T23:12:55.519 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi181.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-12T23:12:56.001 DEBUG:teuthology.orchestra.run.smithi181:> true 2024-09-12T23:12:56.574 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi181.front.sepia.ceph.com' 2024-09-12T23:12:56.574 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.15.0-121-generic"... 2024-09-12T23:12:56.574 DEBUG:teuthology.orchestra.run.smithi181:> uname -r 2024-09-12T23:12:56.622 INFO:teuthology.orchestra.run.smithi181.stdout:5.15.0-121-generic 2024-09-12T23:12:56.622 DEBUG:teuthology.task.kernel:current kernel version is 5.15.0-121-generic vs 5.15.0-121-generic 2024-09-12T23:12:56.623 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-12T23:12:56.623 DEBUG:teuthology.task.kernel:Distro of this test job: ubuntu 2024-09-12T23:12:57.623 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-09-12T23:12:57.624 DEBUG:teuthology.orchestra.run.smithi181:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-12T23:12:57.989 INFO:teuthology.orchestra.run.smithi181.stdout:ttyS1 2024-09-12T23:12:58.012 DEBUG:teuthology.parallel:result is None 2024-09-12T23:13:02.423 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi136.front.sepia.ceph.com' 2024-09-12T23:13:02.423 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi136.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-12T23:13:02.748 DEBUG:teuthology.orchestra.run.smithi136:> true 2024-09-12T23:13:03.235 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi136.front.sepia.ceph.com' 2024-09-12T23:13:03.235 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.15.0-121-generic"... 2024-09-12T23:13:03.235 DEBUG:teuthology.orchestra.run.smithi136:> uname -r 2024-09-12T23:13:03.283 INFO:teuthology.orchestra.run.smithi136.stdout:5.15.0-121-generic 2024-09-12T23:13:03.283 DEBUG:teuthology.task.kernel:current kernel version is 5.15.0-121-generic vs 5.15.0-121-generic 2024-09-12T23:13:03.283 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-12T23:13:03.283 DEBUG:teuthology.task.kernel:Distro of this test job: ubuntu 2024-09-12T23:13:04.284 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-09-12T23:13:04.284 DEBUG:teuthology.orchestra.run.smithi136:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-12T23:13:04.623 INFO:teuthology.orchestra.run.smithi136.stdout:ttyS1 2024-09-12T23:13:04.653 DEBUG:teuthology.parallel:result is None 2024-09-12T23:13:04.654 INFO:teuthology.run_tasks:Running task internal.base... 2024-09-12T23:13:04.900 INFO:teuthology.task.internal:Creating test directory... 2024-09-12T23:13:04.900 DEBUG:teuthology.orchestra.run.smithi136:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-12T23:13:04.903 DEBUG:teuthology.orchestra.run.smithi181:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-12T23:13:04.909 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-09-12T23:13:04.964 INFO:teuthology.run_tasks:Running task internal.archive... 2024-09-12T23:13:05.017 INFO:teuthology.task.internal:Creating archive directory... 2024-09-12T23:13:05.017 DEBUG:teuthology.orchestra.run.smithi136:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-12T23:13:05.020 DEBUG:teuthology.orchestra.run.smithi181:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-12T23:13:05.111 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-09-12T23:13:05.117 INFO:teuthology.task.internal:Enabling coredump saving... 2024-09-12T23:13:05.117 DEBUG:teuthology.orchestra.run.smithi136:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-12T23:13:05.120 DEBUG:teuthology.orchestra.run.smithi181:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-12T23:13:05.138 INFO:teuthology.orchestra.run.smithi136.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-12T23:13:05.143 INFO:teuthology.orchestra.run.smithi181.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-12T23:13:05.145 INFO:teuthology.orchestra.run.smithi136.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-12T23:13:05.151 INFO:teuthology.orchestra.run.smithi181.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-12T23:13:05.153 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-09-12T23:13:05.160 INFO:teuthology.task.internal:Configuring sudo... 2024-09-12T23:13:05.161 DEBUG:teuthology.orchestra.run.smithi136:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-12T23:13:05.195 DEBUG:teuthology.orchestra.run.smithi181:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-12T23:13:05.213 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-09-12T23:13:05.223 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-09-12T23:13:05.224 DEBUG:teuthology.orchestra.run.smithi136:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-12T23:13:05.256 DEBUG:teuthology.orchestra.run.smithi181:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-12T23:13:05.262 DEBUG:teuthology.orchestra.run.smithi136:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-12T23:13:05.307 DEBUG:teuthology.orchestra.run.smithi136:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-12T23:13:05.357 DEBUG:teuthology.orchestra.run.smithi136:> set -ex 2024-09-12T23:13:05.357 DEBUG:teuthology.orchestra.run.smithi136:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-12T23:13:05.423 DEBUG:teuthology.orchestra.run.smithi181:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-12T23:13:05.429 DEBUG:teuthology.orchestra.run.smithi181:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-12T23:13:05.478 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2024-09-12T23:13:05.478 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-12T23:13:05.546 DEBUG:teuthology.orchestra.run.smithi136:> sudo service rsyslog restart 2024-09-12T23:13:05.549 DEBUG:teuthology.orchestra.run.smithi181:> sudo service rsyslog restart 2024-09-12T23:13:05.627 INFO:teuthology.run_tasks:Running task internal.timer... 2024-09-12T23:13:05.634 INFO:teuthology.task.internal:Starting timer... 2024-09-12T23:13:05.634 INFO:teuthology.run_tasks:Running task pcp... 2024-09-12T23:13:05.668 INFO:teuthology.run_tasks:Running task selinux... 2024-09-12T23:13:05.716 DEBUG:teuthology.task.selinux:Excluding smithi136: OS 'ubuntu' does not support SELinux 2024-09-12T23:13:05.717 DEBUG:teuthology.task.selinux:Excluding smithi181: OS 'ubuntu' does not support SELinux 2024-09-12T23:13:05.717 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-09-12T23:13:05.717 DEBUG:teuthology.task.selinux:Existing SELinux modes: {} 2024-09-12T23:13:05.717 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-09-12T23:13:05.717 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-09-12T23:13:05.767 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-09-12T23:13:05.770 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-09-12T23:13:05.885 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-09-12T23:13:05.896 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-09-12T23:13:05.897 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi136.front.sepia.ceph.com,smithi181.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-09-12T23:20:35.347 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi136.front.sepia.ceph.com'), Remote(name='ubuntu@smithi181.front.sepia.ceph.com')] 2024-09-12T23:20:35.349 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi136.front.sepia.ceph.com' 2024-09-12T23:20:35.350 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi136.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-12T23:20:35.431 DEBUG:teuthology.orchestra.run.smithi136:> true 2024-09-12T23:20:35.508 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi136.front.sepia.ceph.com' 2024-09-12T23:20:35.509 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi181.front.sepia.ceph.com' 2024-09-12T23:20:35.509 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi181.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-12T23:20:35.591 DEBUG:teuthology.orchestra.run.smithi181:> true 2024-09-12T23:20:35.672 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi181.front.sepia.ceph.com' 2024-09-12T23:20:35.673 INFO:teuthology.run_tasks:Running task clock... 2024-09-12T23:20:35.691 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-09-12T23:20:35.691 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-12T23:20:35.692 DEBUG:teuthology.orchestra.run.smithi136:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-12T23:20:35.694 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-12T23:20:35.694 DEBUG:teuthology.orchestra.run.smithi181:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-12T23:20:35.722 INFO:teuthology.orchestra.run.smithi136.stdout:12 Sep 23:20:35 ntpd[18489]: ntpd 4.2.8p15@1.3728-o Wed Feb 16 17:13:02 UTC 2022 (1): Starting 2024-09-12T23:20:35.723 INFO:teuthology.orchestra.run.smithi136.stdout:12 Sep 23:20:35 ntpd[18489]: Command line: ntpd -gq 2024-09-12T23:20:35.723 INFO:teuthology.orchestra.run.smithi136.stdout:12 Sep 23:20:35 ntpd[18489]: ---------------------------------------------------- 2024-09-12T23:20:35.723 INFO:teuthology.orchestra.run.smithi136.stdout:12 Sep 23:20:35 ntpd[18489]: ntp-4 is maintained by Network Time Foundation, 2024-09-12T23:20:35.723 INFO:teuthology.orchestra.run.smithi136.stdout:12 Sep 23:20:35 ntpd[18489]: Inc. (NTF), a non-profit 501(c)(3) public-benefit 2024-09-12T23:20:35.723 INFO:teuthology.orchestra.run.smithi136.stdout:12 Sep 23:20:35 ntpd[18489]: corporation. Support and training for ntp-4 are 2024-09-12T23:20:35.723 INFO:teuthology.orchestra.run.smithi136.stdout:12 Sep 23:20:35 ntpd[18489]: available at https://www.nwtime.org/support 2024-09-12T23:20:35.723 INFO:teuthology.orchestra.run.smithi136.stdout:12 Sep 23:20:35 ntpd[18489]: ---------------------------------------------------- 2024-09-12T23:20:35.724 INFO:teuthology.orchestra.run.smithi136.stdout:12 Sep 23:20:35 ntpd[18489]: proto: precision = 0.062 usec (-24) 2024-09-12T23:20:35.724 INFO:teuthology.orchestra.run.smithi136.stdout:12 Sep 23:20:35 ntpd[18489]: basedate set to 2022-02-04 2024-09-12T23:20:35.724 INFO:teuthology.orchestra.run.smithi136.stdout:12 Sep 23:20:35 ntpd[18489]: gps base set to 2022-02-06 (week 2196) 2024-09-12T23:20:35.724 INFO:teuthology.orchestra.run.smithi136.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2024-09-12T23:20:35.725 INFO:teuthology.orchestra.run.smithi136.stdout:12 Sep 23:20:35 ntpd[18489]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2024-09-12T23:20:35.725 INFO:teuthology.orchestra.run.smithi136.stdout:12 Sep 23:20:35 ntpd[18489]: restrict ::: KOD does nothing without LIMITED. 2024-09-12T23:20:35.725 INFO:teuthology.orchestra.run.smithi136.stderr:restrict ::: KOD does nothing without LIMITED. 2024-09-12T23:20:35.726 INFO:teuthology.orchestra.run.smithi136.stdout:12 Sep 23:20:35 ntpd[18489]: Listen and drop on 0 v6wildcard [::]:123 2024-09-12T23:20:35.726 INFO:teuthology.orchestra.run.smithi136.stdout:12 Sep 23:20:35 ntpd[18489]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2024-09-12T23:20:35.726 INFO:teuthology.orchestra.run.smithi136.stdout:12 Sep 23:20:35 ntpd[18489]: Listen normally on 2 lo 127.0.0.1:123 2024-09-12T23:20:35.726 INFO:teuthology.orchestra.run.smithi136.stdout:12 Sep 23:20:35 ntpd[18489]: Listen normally on 3 enp3s0f1 172.21.15.136:123 2024-09-12T23:20:35.726 INFO:teuthology.orchestra.run.smithi136.stdout:12 Sep 23:20:35 ntpd[18489]: Listen normally on 4 lo [::1]:123 2024-09-12T23:20:35.726 INFO:teuthology.orchestra.run.smithi136.stdout:12 Sep 23:20:35 ntpd[18489]: Listen normally on 5 enp3s0f1 [fe80::ec4:7aff:fe88:6e43%5]:123 2024-09-12T23:20:35.726 INFO:teuthology.orchestra.run.smithi136.stdout:12 Sep 23:20:35 ntpd[18489]: Listening on routing socket on fd #22 for interface updates 2024-09-12T23:20:35.739 INFO:teuthology.orchestra.run.smithi181.stdout:12 Sep 23:20:35 ntpd[18487]: ntpd 4.2.8p15@1.3728-o Wed Feb 16 17:13:02 UTC 2022 (1): Starting 2024-09-12T23:20:35.740 INFO:teuthology.orchestra.run.smithi181.stdout:12 Sep 23:20:35 ntpd[18487]: Command line: ntpd -gq 2024-09-12T23:20:35.740 INFO:teuthology.orchestra.run.smithi181.stdout:12 Sep 23:20:35 ntpd[18487]: ---------------------------------------------------- 2024-09-12T23:20:35.740 INFO:teuthology.orchestra.run.smithi181.stdout:12 Sep 23:20:35 ntpd[18487]: ntp-4 is maintained by Network Time Foundation, 2024-09-12T23:20:35.740 INFO:teuthology.orchestra.run.smithi181.stdout:12 Sep 23:20:35 ntpd[18487]: Inc. (NTF), a non-profit 501(c)(3) public-benefit 2024-09-12T23:20:35.740 INFO:teuthology.orchestra.run.smithi181.stdout:12 Sep 23:20:35 ntpd[18487]: corporation. Support and training for ntp-4 are 2024-09-12T23:20:35.740 INFO:teuthology.orchestra.run.smithi181.stdout:12 Sep 23:20:35 ntpd[18487]: available at https://www.nwtime.org/support 2024-09-12T23:20:35.740 INFO:teuthology.orchestra.run.smithi181.stdout:12 Sep 23:20:35 ntpd[18487]: ---------------------------------------------------- 2024-09-12T23:20:35.741 INFO:teuthology.orchestra.run.smithi181.stdout:12 Sep 23:20:35 ntpd[18487]: proto: precision = 0.045 usec (-24) 2024-09-12T23:20:35.741 INFO:teuthology.orchestra.run.smithi181.stdout:12 Sep 23:20:35 ntpd[18487]: basedate set to 2022-02-04 2024-09-12T23:20:35.741 INFO:teuthology.orchestra.run.smithi181.stdout:12 Sep 23:20:35 ntpd[18487]: gps base set to 2022-02-06 (week 2196) 2024-09-12T23:20:35.741 INFO:teuthology.orchestra.run.smithi181.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2024-09-12T23:20:35.742 INFO:teuthology.orchestra.run.smithi181.stdout:12 Sep 23:20:35 ntpd[18487]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2024-09-12T23:20:35.742 INFO:teuthology.orchestra.run.smithi181.stdout:12 Sep 23:20:35 ntpd[18487]: restrict ::: KOD does nothing without LIMITED. 2024-09-12T23:20:35.742 INFO:teuthology.orchestra.run.smithi181.stderr:restrict ::: KOD does nothing without LIMITED. 2024-09-12T23:20:35.743 INFO:teuthology.orchestra.run.smithi181.stdout:12 Sep 23:20:35 ntpd[18487]: Listen and drop on 0 v6wildcard [::]:123 2024-09-12T23:20:35.743 INFO:teuthology.orchestra.run.smithi181.stdout:12 Sep 23:20:35 ntpd[18487]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2024-09-12T23:20:35.743 INFO:teuthology.orchestra.run.smithi181.stdout:12 Sep 23:20:35 ntpd[18487]: Listen normally on 2 lo 127.0.0.1:123 2024-09-12T23:20:35.743 INFO:teuthology.orchestra.run.smithi181.stdout:12 Sep 23:20:35 ntpd[18487]: Listen normally on 3 enp3s0f1 172.21.15.181:123 2024-09-12T23:20:35.743 INFO:teuthology.orchestra.run.smithi181.stdout:12 Sep 23:20:35 ntpd[18487]: Listen normally on 4 lo [::1]:123 2024-09-12T23:20:35.743 INFO:teuthology.orchestra.run.smithi181.stdout:12 Sep 23:20:35 ntpd[18487]: Listen normally on 5 enp3s0f1 [fe80::ec4:7aff:fe88:7b9d%5]:123 2024-09-12T23:20:35.743 INFO:teuthology.orchestra.run.smithi181.stdout:12 Sep 23:20:35 ntpd[18487]: Listening on routing socket on fd #22 for interface updates 2024-09-12T23:20:36.724 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:36 ntpd[18489]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-12T23:20:36.724 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:36 ntpd[18489]: can't open /var/log/ntpstats/rawstats.20240912: Permission denied 2024-09-12T23:20:36.724 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:36 ntpd[18489]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-12T23:20:36.724 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:36 ntpd[18489]: can't open /var/log/ntpstats/rawstats.20240912: Permission denied 2024-09-12T23:20:36.724 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:36 ntpd[18489]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-12T23:20:36.724 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:36 ntpd[18489]: can't open /var/log/ntpstats/rawstats.20240912: Permission denied 2024-09-12T23:20:36.724 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:36 ntpd[18489]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-12T23:20:36.725 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:36 ntpd[18489]: can't open /var/log/ntpstats/peerstats.20240912: Permission denied 2024-09-12T23:20:36.725 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:36 ntpd[18489]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-12T23:20:36.725 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:36 ntpd[18489]: can't open /var/log/ntpstats/rawstats.20240912: Permission denied 2024-09-12T23:20:36.725 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:36 ntpd[18489]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-12T23:20:36.725 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:36 ntpd[18489]: can't open /var/log/ntpstats/peerstats.20240912: Permission denied 2024-09-12T23:20:36.740 INFO:teuthology.orchestra.run.smithi181.stderr:12 Sep 23:20:36 ntpd[18487]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-12T23:20:36.740 INFO:teuthology.orchestra.run.smithi181.stderr:12 Sep 23:20:36 ntpd[18487]: can't open /var/log/ntpstats/rawstats.20240912: Permission denied 2024-09-12T23:20:36.741 INFO:teuthology.orchestra.run.smithi181.stderr:12 Sep 23:20:36 ntpd[18487]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-12T23:20:36.741 INFO:teuthology.orchestra.run.smithi181.stderr:12 Sep 23:20:36 ntpd[18487]: can't open /var/log/ntpstats/rawstats.20240912: Permission denied 2024-09-12T23:20:36.741 INFO:teuthology.orchestra.run.smithi181.stderr:12 Sep 23:20:36 ntpd[18487]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-12T23:20:36.741 INFO:teuthology.orchestra.run.smithi181.stderr:12 Sep 23:20:36 ntpd[18487]: can't open /var/log/ntpstats/rawstats.20240912: Permission denied 2024-09-12T23:20:36.741 INFO:teuthology.orchestra.run.smithi181.stderr:12 Sep 23:20:36 ntpd[18487]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-12T23:20:36.741 INFO:teuthology.orchestra.run.smithi181.stderr:12 Sep 23:20:36 ntpd[18487]: can't open /var/log/ntpstats/peerstats.20240912: Permission denied 2024-09-12T23:20:37.723 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:37 ntpd[18489]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-12T23:20:37.723 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:37 ntpd[18489]: can't open /var/log/ntpstats/rawstats.20240912: Permission denied 2024-09-12T23:20:38.726 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:38 ntpd[18489]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-12T23:20:38.726 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:38 ntpd[18489]: can't open /var/log/ntpstats/rawstats.20240912: Permission denied 2024-09-12T23:20:38.726 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:38 ntpd[18489]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-12T23:20:38.726 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:38 ntpd[18489]: can't open /var/log/ntpstats/rawstats.20240912: Permission denied 2024-09-12T23:20:38.726 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:38 ntpd[18489]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-12T23:20:38.726 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:38 ntpd[18489]: can't open /var/log/ntpstats/rawstats.20240912: Permission denied 2024-09-12T23:20:38.726 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:38 ntpd[18489]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-12T23:20:38.726 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:38 ntpd[18489]: can't open /var/log/ntpstats/peerstats.20240912: Permission denied 2024-09-12T23:20:38.726 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:38 ntpd[18489]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-12T23:20:38.726 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:38 ntpd[18489]: can't open /var/log/ntpstats/rawstats.20240912: Permission denied 2024-09-12T23:20:38.726 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:38 ntpd[18489]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-12T23:20:38.727 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:38 ntpd[18489]: can't open /var/log/ntpstats/peerstats.20240912: Permission denied 2024-09-12T23:20:38.741 INFO:teuthology.orchestra.run.smithi181.stderr:12 Sep 23:20:38 ntpd[18487]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-12T23:20:38.741 INFO:teuthology.orchestra.run.smithi181.stderr:12 Sep 23:20:38 ntpd[18487]: can't open /var/log/ntpstats/rawstats.20240912: Permission denied 2024-09-12T23:20:38.741 INFO:teuthology.orchestra.run.smithi181.stderr:12 Sep 23:20:38 ntpd[18487]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-12T23:20:38.741 INFO:teuthology.orchestra.run.smithi181.stderr:12 Sep 23:20:38 ntpd[18487]: can't open /var/log/ntpstats/rawstats.20240912: Permission denied 2024-09-12T23:20:38.741 INFO:teuthology.orchestra.run.smithi181.stderr:12 Sep 23:20:38 ntpd[18487]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-12T23:20:38.741 INFO:teuthology.orchestra.run.smithi181.stderr:12 Sep 23:20:38 ntpd[18487]: can't open /var/log/ntpstats/rawstats.20240912: Permission denied 2024-09-12T23:20:38.741 INFO:teuthology.orchestra.run.smithi181.stderr:12 Sep 23:20:38 ntpd[18487]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-12T23:20:38.741 INFO:teuthology.orchestra.run.smithi181.stderr:12 Sep 23:20:38 ntpd[18487]: can't open /var/log/ntpstats/peerstats.20240912: Permission denied 2024-09-12T23:20:38.741 INFO:teuthology.orchestra.run.smithi181.stderr:12 Sep 23:20:38 ntpd[18487]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-12T23:20:38.741 INFO:teuthology.orchestra.run.smithi181.stderr:12 Sep 23:20:38 ntpd[18487]: can't open /var/log/ntpstats/rawstats.20240912: Permission denied 2024-09-12T23:20:38.742 INFO:teuthology.orchestra.run.smithi181.stderr:12 Sep 23:20:38 ntpd[18487]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-12T23:20:38.742 INFO:teuthology.orchestra.run.smithi181.stderr:12 Sep 23:20:38 ntpd[18487]: can't open /var/log/ntpstats/peerstats.20240912: Permission denied 2024-09-12T23:20:40.725 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:40 ntpd[18489]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-12T23:20:40.725 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:40 ntpd[18489]: can't open /var/log/ntpstats/rawstats.20240912: Permission denied 2024-09-12T23:20:40.725 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:40 ntpd[18489]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-12T23:20:40.725 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:40 ntpd[18489]: can't open /var/log/ntpstats/rawstats.20240912: Permission denied 2024-09-12T23:20:40.725 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:40 ntpd[18489]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-12T23:20:40.725 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:40 ntpd[18489]: can't open /var/log/ntpstats/rawstats.20240912: Permission denied 2024-09-12T23:20:40.725 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:40 ntpd[18489]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-12T23:20:40.725 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:40 ntpd[18489]: can't open /var/log/ntpstats/rawstats.20240912: Permission denied 2024-09-12T23:20:40.725 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:40 ntpd[18489]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-12T23:20:40.725 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:40 ntpd[18489]: can't open /var/log/ntpstats/peerstats.20240912: Permission denied 2024-09-12T23:20:40.726 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:40 ntpd[18489]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-12T23:20:40.726 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:40 ntpd[18489]: can't open /var/log/ntpstats/rawstats.20240912: Permission denied 2024-09-12T23:20:40.726 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:40 ntpd[18489]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-12T23:20:40.726 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:40 ntpd[18489]: can't open /var/log/ntpstats/peerstats.20240912: Permission denied 2024-09-12T23:20:40.726 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:40 ntpd[18489]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-12T23:20:40.726 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:40 ntpd[18489]: can't open /var/log/ntpstats/rawstats.20240912: Permission denied 2024-09-12T23:20:40.726 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:40 ntpd[18489]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-12T23:20:40.726 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:40 ntpd[18489]: can't open /var/log/ntpstats/peerstats.20240912: Permission denied 2024-09-12T23:20:40.741 INFO:teuthology.orchestra.run.smithi181.stderr:12 Sep 23:20:40 ntpd[18487]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-12T23:20:40.741 INFO:teuthology.orchestra.run.smithi181.stderr:12 Sep 23:20:40 ntpd[18487]: can't open /var/log/ntpstats/rawstats.20240912: Permission denied 2024-09-12T23:20:40.741 INFO:teuthology.orchestra.run.smithi181.stderr:12 Sep 23:20:40 ntpd[18487]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-12T23:20:40.742 INFO:teuthology.orchestra.run.smithi181.stderr:12 Sep 23:20:40 ntpd[18487]: can't open /var/log/ntpstats/rawstats.20240912: Permission denied 2024-09-12T23:20:40.742 INFO:teuthology.orchestra.run.smithi181.stderr:12 Sep 23:20:40 ntpd[18487]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-12T23:20:40.742 INFO:teuthology.orchestra.run.smithi181.stderr:12 Sep 23:20:40 ntpd[18487]: can't open /var/log/ntpstats/rawstats.20240912: Permission denied 2024-09-12T23:20:40.742 INFO:teuthology.orchestra.run.smithi181.stderr:12 Sep 23:20:40 ntpd[18487]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-12T23:20:40.742 INFO:teuthology.orchestra.run.smithi181.stderr:12 Sep 23:20:40 ntpd[18487]: can't open /var/log/ntpstats/rawstats.20240912: Permission denied 2024-09-12T23:20:40.742 INFO:teuthology.orchestra.run.smithi181.stderr:12 Sep 23:20:40 ntpd[18487]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-12T23:20:40.742 INFO:teuthology.orchestra.run.smithi181.stderr:12 Sep 23:20:40 ntpd[18487]: can't open /var/log/ntpstats/peerstats.20240912: Permission denied 2024-09-12T23:20:40.742 INFO:teuthology.orchestra.run.smithi181.stderr:12 Sep 23:20:40 ntpd[18487]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-12T23:20:40.742 INFO:teuthology.orchestra.run.smithi181.stderr:12 Sep 23:20:40 ntpd[18487]: can't open /var/log/ntpstats/rawstats.20240912: Permission denied 2024-09-12T23:20:40.743 INFO:teuthology.orchestra.run.smithi181.stderr:12 Sep 23:20:40 ntpd[18487]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-12T23:20:40.743 INFO:teuthology.orchestra.run.smithi181.stderr:12 Sep 23:20:40 ntpd[18487]: can't open /var/log/ntpstats/peerstats.20240912: Permission denied 2024-09-12T23:20:40.743 INFO:teuthology.orchestra.run.smithi181.stderr:12 Sep 23:20:40 ntpd[18487]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-12T23:20:40.743 INFO:teuthology.orchestra.run.smithi181.stderr:12 Sep 23:20:40 ntpd[18487]: can't open /var/log/ntpstats/rawstats.20240912: Permission denied 2024-09-12T23:20:40.743 INFO:teuthology.orchestra.run.smithi181.stderr:12 Sep 23:20:40 ntpd[18487]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-12T23:20:40.743 INFO:teuthology.orchestra.run.smithi181.stderr:12 Sep 23:20:40 ntpd[18487]: can't open /var/log/ntpstats/peerstats.20240912: Permission denied 2024-09-12T23:20:42.724 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:42 ntpd[18489]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-12T23:20:42.725 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:42 ntpd[18489]: can't open /var/log/ntpstats/rawstats.20240912: Permission denied 2024-09-12T23:20:42.725 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:42 ntpd[18489]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-12T23:20:42.725 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:42 ntpd[18489]: can't open /var/log/ntpstats/rawstats.20240912: Permission denied 2024-09-12T23:20:42.725 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:42 ntpd[18489]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-12T23:20:42.725 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:42 ntpd[18489]: can't open /var/log/ntpstats/rawstats.20240912: Permission denied 2024-09-12T23:20:42.725 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:42 ntpd[18489]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-12T23:20:42.725 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:42 ntpd[18489]: can't open /var/log/ntpstats/rawstats.20240912: Permission denied 2024-09-12T23:20:42.725 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:42 ntpd[18489]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-12T23:20:42.725 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:42 ntpd[18489]: can't open /var/log/ntpstats/peerstats.20240912: Permission denied 2024-09-12T23:20:42.726 INFO:teuthology.orchestra.run.smithi136.stdout:12 Sep 23:20:42 ntpd[18489]: ntpd: time slew -0.000973 s 2024-09-12T23:20:42.726 INFO:teuthology.orchestra.run.smithi136.stdout:ntpd: time slew -0.000973s 2024-09-12T23:20:42.726 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:42 ntpd[18489]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied 2024-09-12T23:20:42.726 INFO:teuthology.orchestra.run.smithi136.stderr:12 Sep 23:20:42 ntpd[18489]: can't open /var/log/ntpstats/loopstats.20240912: Permission denied 2024-09-12T23:20:42.741 INFO:teuthology.orchestra.run.smithi181.stderr:12 Sep 23:20:42 ntpd[18487]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-12T23:20:42.741 INFO:teuthology.orchestra.run.smithi181.stderr:12 Sep 23:20:42 ntpd[18487]: can't open /var/log/ntpstats/rawstats.20240912: Permission denied 2024-09-12T23:20:42.741 INFO:teuthology.orchestra.run.smithi181.stderr:12 Sep 23:20:42 ntpd[18487]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-12T23:20:42.741 INFO:teuthology.orchestra.run.smithi181.stderr:12 Sep 23:20:42 ntpd[18487]: can't open /var/log/ntpstats/rawstats.20240912: Permission denied 2024-09-12T23:20:42.741 INFO:teuthology.orchestra.run.smithi181.stderr:12 Sep 23:20:42 ntpd[18487]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-12T23:20:42.741 INFO:teuthology.orchestra.run.smithi181.stderr:12 Sep 23:20:42 ntpd[18487]: can't open /var/log/ntpstats/rawstats.20240912: Permission denied 2024-09-12T23:20:42.741 INFO:teuthology.orchestra.run.smithi181.stderr:12 Sep 23:20:42 ntpd[18487]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-12T23:20:42.742 INFO:teuthology.orchestra.run.smithi181.stderr:12 Sep 23:20:42 ntpd[18487]: can't open /var/log/ntpstats/rawstats.20240912: Permission denied 2024-09-12T23:20:42.742 INFO:teuthology.orchestra.run.smithi181.stderr:12 Sep 23:20:42 ntpd[18487]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-12T23:20:42.742 INFO:teuthology.orchestra.run.smithi181.stderr:12 Sep 23:20:42 ntpd[18487]: can't open /var/log/ntpstats/peerstats.20240912: Permission denied 2024-09-12T23:20:42.742 INFO:teuthology.orchestra.run.smithi181.stdout:12 Sep 23:20:42 ntpd[18487]: ntpd: time slew -0.001260 s 2024-09-12T23:20:42.742 INFO:teuthology.orchestra.run.smithi181.stdout:ntpd: time slew -0.001260s 2024-09-12T23:20:42.742 INFO:teuthology.orchestra.run.smithi181.stderr:12 Sep 23:20:42 ntpd[18487]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied 2024-09-12T23:20:42.742 INFO:teuthology.orchestra.run.smithi181.stderr:12 Sep 23:20:42 ntpd[18487]: can't open /var/log/ntpstats/loopstats.20240912: Permission denied 2024-09-12T23:20:42.773 INFO:teuthology.orchestra.run.smithi136.stdout: remote refid st t when poll reach delay offset jitter 2024-09-12T23:20:42.773 INFO:teuthology.orchestra.run.smithi136.stdout:============================================================================== 2024-09-12T23:20:42.774 INFO:teuthology.orchestra.run.smithi136.stdout: hv01.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-09-12T23:20:42.774 INFO:teuthology.orchestra.run.smithi136.stdout: hv02.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-09-12T23:20:42.774 INFO:teuthology.orchestra.run.smithi136.stdout: hv03.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-09-12T23:20:42.774 INFO:teuthology.orchestra.run.smithi136.stdout: hv04.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-09-12T23:20:42.797 INFO:teuthology.orchestra.run.smithi181.stdout: remote refid st t when poll reach delay offset jitter 2024-09-12T23:20:42.797 INFO:teuthology.orchestra.run.smithi181.stdout:============================================================================== 2024-09-12T23:20:42.797 INFO:teuthology.orchestra.run.smithi181.stdout: hv01.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-09-12T23:20:42.797 INFO:teuthology.orchestra.run.smithi181.stdout: hv02.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-09-12T23:20:42.797 INFO:teuthology.orchestra.run.smithi181.stdout: hv03.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-09-12T23:20:42.797 INFO:teuthology.orchestra.run.smithi181.stdout: hv04.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-09-12T23:20:42.799 INFO:teuthology.run_tasks:Running task install... 2024-09-12T23:20:42.808 DEBUG:teuthology.task.install:project ceph 2024-09-12T23:20:42.808 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'extra_system_packages': ['python3-pytest'], 'flavor': 'default', 'sha1': '7a853b1706d076d58e0da7a5d3b9b9eb36276880'}} 2024-09-12T23:20:42.808 DEBUG:teuthology.task.install:config {'branch': 'quincy', 'exclude_packages': ['ceph-volume'], 'extra_system_packages': ['python3-pytest'], 'flavor': 'default', 'sha1': '7a853b1706d076d58e0da7a5d3b9b9eb36276880'} 2024-09-12T23:20:42.808 INFO:teuthology.task.install:Using flavor: default 2024-09-12T23:20:42.814 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-09-12T23:20:42.814 INFO:teuthology.task.install:extra packages: [] 2024-09-12T23:20:42.814 DEBUG:teuthology.orchestra.run.smithi136:> sudo apt-key list | grep Ceph 2024-09-12T23:20:42.815 DEBUG:teuthology.orchestra.run.smithi181:> sudo apt-key list | grep Ceph 2024-09-12T23:20:42.869 INFO:teuthology.orchestra.run.smithi136.stderr:Warning: apt-key is deprecated. Manage keyring files in trusted.gpg.d instead (see apt-key(8)). 2024-09-12T23:20:42.894 INFO:teuthology.orchestra.run.smithi136.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2024-09-12T23:20:42.894 INFO:teuthology.orchestra.run.smithi136.stdout:uid [ unknown] Ceph.com (release key) 2024-09-12T23:20:42.895 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse on remote deb x86_64 2024-09-12T23:20:42.896 INFO:teuthology.task.install.deb:Installing system (non-project) packages: python3-pytest on remote deb x86_64 2024-09-12T23:20:42.896 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-12T23:20:42.896 INFO:teuthology.packaging:ref: None 2024-09-12T23:20:42.896 INFO:teuthology.packaging:tag: None 2024-09-12T23:20:42.896 INFO:teuthology.packaging:branch: quincy 2024-09-12T23:20:42.896 INFO:teuthology.packaging:sha1: 7a853b1706d076d58e0da7a5d3b9b9eb36276880 2024-09-12T23:20:42.896 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&ref=quincy 2024-09-12T23:20:42.902 INFO:teuthology.orchestra.run.smithi181.stderr:Warning: apt-key is deprecated. Manage keyring files in trusted.gpg.d instead (see apt-key(8)). 2024-09-12T23:20:42.924 INFO:teuthology.orchestra.run.smithi181.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2024-09-12T23:20:42.924 INFO:teuthology.orchestra.run.smithi181.stdout:uid [ unknown] Ceph.com (release key) 2024-09-12T23:20:42.924 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse on remote deb x86_64 2024-09-12T23:20:42.924 INFO:teuthology.task.install.deb:Installing system (non-project) packages: python3-pytest on remote deb x86_64 2024-09-12T23:20:42.924 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-12T23:20:42.924 INFO:teuthology.packaging:ref: None 2024-09-12T23:20:42.924 INFO:teuthology.packaging:tag: None 2024-09-12T23:20:42.925 INFO:teuthology.packaging:branch: quincy 2024-09-12T23:20:42.925 INFO:teuthology.packaging:sha1: 7a853b1706d076d58e0da7a5d3b9b9eb36276880 2024-09-12T23:20:42.925 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&ref=quincy 2024-09-12T23:20:43.046 INFO:teuthology.task.install.deb:Pulling from https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default/ 2024-09-12T23:20:43.046 INFO:teuthology.task.install.deb:Package version is 17.2.7-1jammy 2024-09-12T23:20:43.068 INFO:teuthology.task.install.deb:Pulling from https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default/ 2024-09-12T23:20:43.069 INFO:teuthology.task.install.deb:Package version is 17.2.7-1jammy 2024-09-12T23:20:43.082 DEBUG:teuthology.orchestra.run.smithi136:> set -ex 2024-09-12T23:20:43.082 DEBUG:teuthology.orchestra.run.smithi136:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2024-09-12T23:20:43.097 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2024-09-12T23:20:43.097 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2024-09-12T23:20:43.098 DEBUG:teuthology.orchestra.run.smithi136:> sudo apt-get update 2024-09-12T23:20:43.114 DEBUG:teuthology.orchestra.run.smithi181:> sudo apt-get update 2024-09-12T23:20:43.238 INFO:teuthology.orchestra.run.smithi136.stdout:Ign:1 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy InRelease 2024-09-12T23:20:43.253 INFO:teuthology.orchestra.run.smithi181.stdout:Ign:1 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy InRelease 2024-09-12T23:20:43.261 INFO:teuthology.orchestra.run.smithi136.stdout:Hit:2 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-09-12T23:20:43.266 INFO:teuthology.orchestra.run.smithi136.stdout:Get:3 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy Release [7,641 B] 2024-09-12T23:20:43.267 INFO:teuthology.orchestra.run.smithi181.stdout:Get:2 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy Release [7,641 B] 2024-09-12T23:20:43.268 INFO:teuthology.orchestra.run.smithi136.stdout:Ign:4 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy Release.gpg 2024-09-12T23:20:43.269 INFO:teuthology.orchestra.run.smithi181.stdout:Ign:3 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy Release.gpg 2024-09-12T23:20:43.270 INFO:teuthology.orchestra.run.smithi136.stdout:Hit:5 http://archive.ubuntu.com/ubuntu jammy-updates InRelease 2024-09-12T23:20:43.273 INFO:teuthology.orchestra.run.smithi136.stdout:Hit:6 http://security.ubuntu.com/ubuntu jammy-security InRelease 2024-09-12T23:20:43.280 INFO:teuthology.orchestra.run.smithi181.stdout:Hit:4 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-09-12T23:20:43.280 INFO:teuthology.orchestra.run.smithi181.stdout:Hit:5 http://security.ubuntu.com/ubuntu jammy-security InRelease 2024-09-12T23:20:43.287 INFO:teuthology.orchestra.run.smithi136.stdout:Get:7 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 Packages [17.3 kB] 2024-09-12T23:20:43.288 INFO:teuthology.orchestra.run.smithi181.stdout:Get:6 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main i386 Packages [3,331 B] 2024-09-12T23:20:43.292 INFO:teuthology.orchestra.run.smithi136.stdout:Hit:8 http://archive.ubuntu.com/ubuntu jammy-backports InRelease 2024-09-12T23:20:43.293 INFO:teuthology.orchestra.run.smithi136.stdout:Get:9 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main i386 Packages [3,331 B] 2024-09-12T23:20:43.295 INFO:teuthology.orchestra.run.smithi181.stdout:Hit:7 http://archive.ubuntu.com/ubuntu jammy-updates InRelease 2024-09-12T23:20:43.296 INFO:teuthology.orchestra.run.smithi181.stdout:Get:8 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 Packages [17.3 kB] 2024-09-12T23:20:43.306 INFO:teuthology.orchestra.run.smithi181.stdout:Hit:9 http://archive.ubuntu.com/ubuntu jammy-backports InRelease 2024-09-12T23:20:43.612 INFO:teuthology.orchestra.run.smithi136.stdout:Fetched 28.3 kB in 0s (83.3 kB/s) 2024-09-12T23:20:43.634 INFO:teuthology.orchestra.run.smithi181.stdout:Fetched 28.3 kB in 0s (82.9 kB/s) 2024-09-12T23:20:44.613 INFO:teuthology.orchestra.run.smithi181.stdout:Reading package lists... 2024-09-12T23:20:44.629 DEBUG:teuthology.orchestra.run.smithi181:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=17.2.7-1jammy cephadm=17.2.7-1jammy ceph-mds=17.2.7-1jammy ceph-mgr=17.2.7-1jammy ceph-common=17.2.7-1jammy ceph-fuse=17.2.7-1jammy ceph-test=17.2.7-1jammy radosgw=17.2.7-1jammy python3-rados=17.2.7-1jammy python3-rgw=17.2.7-1jammy python3-cephfs=17.2.7-1jammy python3-rbd=17.2.7-1jammy libcephfs2=17.2.7-1jammy libcephfs-dev=17.2.7-1jammy librados2=17.2.7-1jammy librbd1=17.2.7-1jammy rbd-fuse=17.2.7-1jammy 2024-09-12T23:20:44.649 INFO:teuthology.orchestra.run.smithi136.stdout:Reading package lists... 2024-09-12T23:20:44.668 DEBUG:teuthology.orchestra.run.smithi136:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=17.2.7-1jammy cephadm=17.2.7-1jammy ceph-mds=17.2.7-1jammy ceph-mgr=17.2.7-1jammy ceph-common=17.2.7-1jammy ceph-fuse=17.2.7-1jammy ceph-test=17.2.7-1jammy radosgw=17.2.7-1jammy python3-rados=17.2.7-1jammy python3-rgw=17.2.7-1jammy python3-cephfs=17.2.7-1jammy python3-rbd=17.2.7-1jammy libcephfs2=17.2.7-1jammy libcephfs-dev=17.2.7-1jammy librados2=17.2.7-1jammy librbd1=17.2.7-1jammy rbd-fuse=17.2.7-1jammy 2024-09-12T23:20:44.684 INFO:teuthology.orchestra.run.smithi181.stdout:Reading package lists... 2024-09-12T23:20:44.733 INFO:teuthology.orchestra.run.smithi136.stdout:Reading package lists... 2024-09-12T23:20:44.872 INFO:teuthology.orchestra.run.smithi181.stdout:Building dependency tree... 2024-09-12T23:20:44.872 INFO:teuthology.orchestra.run.smithi181.stdout:Reading state information... 2024-09-12T23:20:44.928 INFO:teuthology.orchestra.run.smithi136.stdout:Building dependency tree... 2024-09-12T23:20:44.928 INFO:teuthology.orchestra.run.smithi136.stdout:Reading state information... 2024-09-12T23:20:45.056 INFO:teuthology.orchestra.run.smithi181.stdout:The following packages were automatically installed and are no longer required: 2024-09-12T23:20:45.057 INFO:teuthology.orchestra.run.smithi181.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-09-12T23:20:45.057 INFO:teuthology.orchestra.run.smithi181.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-09-12T23:20:45.057 INFO:teuthology.orchestra.run.smithi181.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-09-12T23:20:45.057 INFO:teuthology.orchestra.run.smithi181.stdout: linux-image-5.4.0-135-generic linux-modules-5.4.0-135-generic python2 2024-09-12T23:20:45.057 INFO:teuthology.orchestra.run.smithi181.stdout: python2-dev python2-minimal python2.7 python2.7-dev python2.7-minimal 2024-09-12T23:20:45.057 INFO:teuthology.orchestra.run.smithi181.stdout:Use 'sudo apt autoremove' to remove them. 2024-09-12T23:20:45.058 INFO:teuthology.orchestra.run.smithi181.stdout:The following additional packages will be installed: 2024-09-12T23:20:45.058 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local 2024-09-12T23:20:45.058 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd ceph-volume jq 2024-09-12T23:20:45.058 INFO:teuthology.orchestra.run.smithi181.stdout: libdouble-conversion3 libfmt8 libjq1 liblttng-ust1 liboath0 libonig5 2024-09-12T23:20:45.058 INFO:teuthology.orchestra.run.smithi181.stdout: libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 libradosstriper1 2024-09-12T23:20:45.059 INFO:teuthology.orchestra.run.smithi181.stdout: librdkafka1 librgw2 libsqlite3-mod-ceph libthrift-0.16.0 nvme-cli 2024-09-12T23:20:45.059 INFO:teuthology.orchestra.run.smithi181.stdout: python-babel-localedata python-pastedeploy-tpl python3-babel python3-bcrypt 2024-09-12T23:20:45.059 INFO:teuthology.orchestra.run.smithi181.stdout: python3-cachetools python3-ceph-argparse python3-ceph-common python3-certifi 2024-09-12T23:20:45.059 INFO:teuthology.orchestra.run.smithi181.stdout: python3-cheroot python3-cherrypy3 python3-google-auth python3-idna 2024-09-12T23:20:45.060 INFO:teuthology.orchestra.run.smithi181.stdout: python3-iniconfig python3-jaraco.classes python3-jaraco.collections 2024-09-12T23:20:45.060 INFO:teuthology.orchestra.run.smithi181.stdout: python3-jaraco.functools python3-jaraco.text python3-jinja2 python3-joblib 2024-09-12T23:20:45.060 INFO:teuthology.orchestra.run.smithi181.stdout: python3-kubernetes python3-logutils python3-mako python3-markupsafe 2024-09-12T23:20:45.060 INFO:teuthology.orchestra.run.smithi181.stdout: python3-natsort python3-openssl python3-paste python3-pastedeploy 2024-09-12T23:20:45.060 INFO:teuthology.orchestra.run.smithi181.stdout: python3-pastescript python3-pecan python3-pluggy python3-portend 2024-09-12T23:20:45.060 INFO:teuthology.orchestra.run.smithi181.stdout: python3-prettytable python3-psutil python3-py python3-pyasn1 2024-09-12T23:20:45.060 INFO:teuthology.orchestra.run.smithi181.stdout: python3-pyasn1-modules python3-pygments python3-pyinotify python3-pytest 2024-09-12T23:20:45.060 INFO:teuthology.orchestra.run.smithi181.stdout: python3-repoze.lru python3-requests python3-requests-oauthlib python3-routes 2024-09-12T23:20:45.060 INFO:teuthology.orchestra.run.smithi181.stdout: python3-rsa python3-simplegeneric python3-simplejson python3-singledispatch 2024-09-12T23:20:45.060 INFO:teuthology.orchestra.run.smithi181.stdout: python3-sklearn python3-sklearn-lib python3-tempita python3-tempora 2024-09-12T23:20:45.060 INFO:teuthology.orchestra.run.smithi181.stdout: python3-threadpoolctl python3-toml python3-urllib3 python3-waitress 2024-09-12T23:20:45.060 INFO:teuthology.orchestra.run.smithi181.stdout: python3-wcwidth python3-webob python3-websocket python3-webtest 2024-09-12T23:20:45.060 INFO:teuthology.orchestra.run.smithi181.stdout: python3-werkzeug python3-zc.lockfile qttranslations5-l10n socat xmlstarlet 2024-09-12T23:20:45.061 INFO:teuthology.orchestra.run.smithi181.stdout:Suggested packages: 2024-09-12T23:20:45.061 INFO:teuthology.orchestra.run.smithi181.stdout: python3-influxdb python-jinja2-doc python3-beaker python-mako-doc 2024-09-12T23:20:45.061 INFO:teuthology.orchestra.run.smithi181.stdout: python-natsort-doc python-openssl-doc python3-openssl-dbg httpd-wsgi 2024-09-12T23:20:45.061 INFO:teuthology.orchestra.run.smithi181.stdout: libapache2-mod-python libapache2-mod-scgi libjs-mochikit python-pecan-doc 2024-09-12T23:20:45.061 INFO:teuthology.orchestra.run.smithi181.stdout: python-psutil-doc subversion python-pygments-doc python-pyinotify-doc 2024-09-12T23:20:45.061 INFO:teuthology.orchestra.run.smithi181.stdout: python3-socks python-requests-doc python3-dap python-sklearn-doc ipython3 2024-09-12T23:20:45.061 INFO:teuthology.orchestra.run.smithi181.stdout: python-waitress-doc python-webob-doc python-webtest-doc python-werkzeug-doc 2024-09-12T23:20:45.061 INFO:teuthology.orchestra.run.smithi181.stdout: python3-watchdog 2024-09-12T23:20:45.061 INFO:teuthology.orchestra.run.smithi181.stdout:Recommended packages: 2024-09-12T23:20:45.061 INFO:teuthology.orchestra.run.smithi181.stdout: btrfs-tools 2024-09-12T23:20:45.115 INFO:teuthology.orchestra.run.smithi136.stdout:The following packages were automatically installed and are no longer required: 2024-09-12T23:20:45.115 INFO:teuthology.orchestra.run.smithi136.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-09-12T23:20:45.115 INFO:teuthology.orchestra.run.smithi136.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-09-12T23:20:45.115 INFO:teuthology.orchestra.run.smithi136.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-09-12T23:20:45.116 INFO:teuthology.orchestra.run.smithi136.stdout: linux-image-5.4.0-135-generic linux-modules-5.4.0-135-generic python2 2024-09-12T23:20:45.116 INFO:teuthology.orchestra.run.smithi136.stdout: python2-dev python2-minimal python2.7 python2.7-dev python2.7-minimal 2024-09-12T23:20:45.116 INFO:teuthology.orchestra.run.smithi136.stdout:Use 'sudo apt autoremove' to remove them. 2024-09-12T23:20:45.116 INFO:teuthology.orchestra.run.smithi136.stdout:The following additional packages will be installed: 2024-09-12T23:20:45.116 INFO:teuthology.orchestra.run.smithi136.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local 2024-09-12T23:20:45.117 INFO:teuthology.orchestra.run.smithi136.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd ceph-volume jq 2024-09-12T23:20:45.117 INFO:teuthology.orchestra.run.smithi136.stdout: libdouble-conversion3 libfmt8 libjq1 liblttng-ust1 liboath0 libonig5 2024-09-12T23:20:45.117 INFO:teuthology.orchestra.run.smithi136.stdout: libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 libradosstriper1 2024-09-12T23:20:45.118 INFO:teuthology.orchestra.run.smithi136.stdout: librdkafka1 librgw2 libsqlite3-mod-ceph libthrift-0.16.0 nvme-cli 2024-09-12T23:20:45.118 INFO:teuthology.orchestra.run.smithi136.stdout: python-babel-localedata python-pastedeploy-tpl python3-babel python3-bcrypt 2024-09-12T23:20:45.118 INFO:teuthology.orchestra.run.smithi136.stdout: python3-cachetools python3-ceph-argparse python3-ceph-common python3-certifi 2024-09-12T23:20:45.119 INFO:teuthology.orchestra.run.smithi136.stdout: python3-cheroot python3-cherrypy3 python3-google-auth python3-idna 2024-09-12T23:20:45.119 INFO:teuthology.orchestra.run.smithi136.stdout: python3-iniconfig python3-jaraco.classes python3-jaraco.collections 2024-09-12T23:20:45.119 INFO:teuthology.orchestra.run.smithi136.stdout: python3-jaraco.functools python3-jaraco.text python3-jinja2 python3-joblib 2024-09-12T23:20:45.119 INFO:teuthology.orchestra.run.smithi136.stdout: python3-kubernetes python3-logutils python3-mako python3-markupsafe 2024-09-12T23:20:45.119 INFO:teuthology.orchestra.run.smithi136.stdout: python3-natsort python3-openssl python3-paste python3-pastedeploy 2024-09-12T23:20:45.119 INFO:teuthology.orchestra.run.smithi136.stdout: python3-pastescript python3-pecan python3-pluggy python3-portend 2024-09-12T23:20:45.119 INFO:teuthology.orchestra.run.smithi136.stdout: python3-prettytable python3-psutil python3-py python3-pyasn1 2024-09-12T23:20:45.119 INFO:teuthology.orchestra.run.smithi136.stdout: python3-pyasn1-modules python3-pygments python3-pyinotify python3-pytest 2024-09-12T23:20:45.119 INFO:teuthology.orchestra.run.smithi136.stdout: python3-repoze.lru python3-requests python3-requests-oauthlib python3-routes 2024-09-12T23:20:45.119 INFO:teuthology.orchestra.run.smithi136.stdout: python3-rsa python3-simplegeneric python3-simplejson python3-singledispatch 2024-09-12T23:20:45.119 INFO:teuthology.orchestra.run.smithi136.stdout: python3-sklearn python3-sklearn-lib python3-tempita python3-tempora 2024-09-12T23:20:45.119 INFO:teuthology.orchestra.run.smithi136.stdout: python3-threadpoolctl python3-toml python3-urllib3 python3-waitress 2024-09-12T23:20:45.119 INFO:teuthology.orchestra.run.smithi136.stdout: python3-wcwidth python3-webob python3-websocket python3-webtest 2024-09-12T23:20:45.119 INFO:teuthology.orchestra.run.smithi136.stdout: python3-werkzeug python3-zc.lockfile qttranslations5-l10n socat xmlstarlet 2024-09-12T23:20:45.120 INFO:teuthology.orchestra.run.smithi136.stdout:Suggested packages: 2024-09-12T23:20:45.120 INFO:teuthology.orchestra.run.smithi136.stdout: python3-influxdb python-jinja2-doc python3-beaker python-mako-doc 2024-09-12T23:20:45.120 INFO:teuthology.orchestra.run.smithi136.stdout: python-natsort-doc python-openssl-doc python3-openssl-dbg httpd-wsgi 2024-09-12T23:20:45.120 INFO:teuthology.orchestra.run.smithi136.stdout: libapache2-mod-python libapache2-mod-scgi libjs-mochikit python-pecan-doc 2024-09-12T23:20:45.120 INFO:teuthology.orchestra.run.smithi136.stdout: python-psutil-doc subversion python-pygments-doc python-pyinotify-doc 2024-09-12T23:20:45.120 INFO:teuthology.orchestra.run.smithi136.stdout: python3-socks python-requests-doc python3-dap python-sklearn-doc ipython3 2024-09-12T23:20:45.120 INFO:teuthology.orchestra.run.smithi136.stdout: python-waitress-doc python-webob-doc python-webtest-doc python-werkzeug-doc 2024-09-12T23:20:45.120 INFO:teuthology.orchestra.run.smithi136.stdout: python3-watchdog 2024-09-12T23:20:45.120 INFO:teuthology.orchestra.run.smithi136.stdout:Recommended packages: 2024-09-12T23:20:45.120 INFO:teuthology.orchestra.run.smithi136.stdout: btrfs-tools 2024-09-12T23:20:45.161 INFO:teuthology.orchestra.run.smithi181.stdout:The following NEW packages will be installed: 2024-09-12T23:20:45.161 INFO:teuthology.orchestra.run.smithi181.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2024-09-12T23:20:45.161 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents 2024-09-12T23:20:45.161 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-mgr-modules-core ceph-mon ceph-osd ceph-test ceph-volume cephadm jq 2024-09-12T23:20:45.162 INFO:teuthology.orchestra.run.smithi181.stdout: libcephfs-dev libcephfs2 libdouble-conversion3 libfmt8 libjq1 liblttng-ust1 2024-09-12T23:20:45.162 INFO:teuthology.orchestra.run.smithi181.stdout: liboath0 libonig5 libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 2024-09-12T23:20:45.162 INFO:teuthology.orchestra.run.smithi181.stdout: librados2 libradosstriper1 librbd1 librdkafka1 librgw2 libsqlite3-mod-ceph 2024-09-12T23:20:45.162 INFO:teuthology.orchestra.run.smithi181.stdout: libthrift-0.16.0 nvme-cli python-babel-localedata python-pastedeploy-tpl 2024-09-12T23:20:45.162 INFO:teuthology.orchestra.run.smithi181.stdout: python3-babel python3-bcrypt python3-cachetools python3-ceph-argparse 2024-09-12T23:20:45.162 INFO:teuthology.orchestra.run.smithi181.stdout: python3-ceph-common python3-cephfs python3-certifi python3-cheroot 2024-09-12T23:20:45.162 INFO:teuthology.orchestra.run.smithi181.stdout: python3-cherrypy3 python3-google-auth python3-idna python3-iniconfig 2024-09-12T23:20:45.163 INFO:teuthology.orchestra.run.smithi181.stdout: python3-jaraco.classes python3-jaraco.collections python3-jaraco.functools 2024-09-12T23:20:45.163 INFO:teuthology.orchestra.run.smithi181.stdout: python3-jaraco.text python3-jinja2 python3-joblib python3-kubernetes 2024-09-12T23:20:45.163 INFO:teuthology.orchestra.run.smithi181.stdout: python3-logutils python3-mako python3-markupsafe python3-natsort 2024-09-12T23:20:45.163 INFO:teuthology.orchestra.run.smithi181.stdout: python3-openssl python3-paste python3-pastedeploy python3-pastescript 2024-09-12T23:20:45.163 INFO:teuthology.orchestra.run.smithi181.stdout: python3-pecan python3-pluggy python3-portend python3-prettytable 2024-09-12T23:20:45.163 INFO:teuthology.orchestra.run.smithi181.stdout: python3-psutil python3-py python3-pyasn1 python3-pyasn1-modules 2024-09-12T23:20:45.163 INFO:teuthology.orchestra.run.smithi181.stdout: python3-pygments python3-pyinotify python3-pytest python3-rados python3-rbd 2024-09-12T23:20:45.163 INFO:teuthology.orchestra.run.smithi181.stdout: python3-repoze.lru python3-requests python3-requests-oauthlib python3-rgw 2024-09-12T23:20:45.163 INFO:teuthology.orchestra.run.smithi181.stdout: python3-routes python3-rsa python3-simplegeneric python3-simplejson 2024-09-12T23:20:45.163 INFO:teuthology.orchestra.run.smithi181.stdout: python3-singledispatch python3-sklearn python3-sklearn-lib python3-tempita 2024-09-12T23:20:45.163 INFO:teuthology.orchestra.run.smithi181.stdout: python3-tempora python3-threadpoolctl python3-toml python3-urllib3 2024-09-12T23:20:45.163 INFO:teuthology.orchestra.run.smithi181.stdout: python3-waitress python3-wcwidth python3-webob python3-websocket 2024-09-12T23:20:45.163 INFO:teuthology.orchestra.run.smithi181.stdout: python3-webtest python3-werkzeug python3-zc.lockfile qttranslations5-l10n 2024-09-12T23:20:45.163 INFO:teuthology.orchestra.run.smithi181.stdout: radosgw rbd-fuse socat xmlstarlet 2024-09-12T23:20:45.220 INFO:teuthology.orchestra.run.smithi136.stdout:The following NEW packages will be installed: 2024-09-12T23:20:45.221 INFO:teuthology.orchestra.run.smithi136.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2024-09-12T23:20:45.221 INFO:teuthology.orchestra.run.smithi136.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents 2024-09-12T23:20:45.221 INFO:teuthology.orchestra.run.smithi136.stdout: ceph-mgr-modules-core ceph-mon ceph-osd ceph-test ceph-volume cephadm jq 2024-09-12T23:20:45.222 INFO:teuthology.orchestra.run.smithi136.stdout: libcephfs-dev libcephfs2 libdouble-conversion3 libfmt8 libjq1 liblttng-ust1 2024-09-12T23:20:45.222 INFO:teuthology.orchestra.run.smithi136.stdout: liboath0 libonig5 libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 2024-09-12T23:20:45.222 INFO:teuthology.orchestra.run.smithi136.stdout: librados2 libradosstriper1 librbd1 librdkafka1 librgw2 libsqlite3-mod-ceph 2024-09-12T23:20:45.222 INFO:teuthology.orchestra.run.smithi136.stdout: libthrift-0.16.0 nvme-cli python-babel-localedata python-pastedeploy-tpl 2024-09-12T23:20:45.222 INFO:teuthology.orchestra.run.smithi136.stdout: python3-babel python3-bcrypt python3-cachetools python3-ceph-argparse 2024-09-12T23:20:45.222 INFO:teuthology.orchestra.run.smithi136.stdout: python3-ceph-common python3-cephfs python3-certifi python3-cheroot 2024-09-12T23:20:45.222 INFO:teuthology.orchestra.run.smithi136.stdout: python3-cherrypy3 python3-google-auth python3-idna python3-iniconfig 2024-09-12T23:20:45.222 INFO:teuthology.orchestra.run.smithi136.stdout: python3-jaraco.classes python3-jaraco.collections python3-jaraco.functools 2024-09-12T23:20:45.223 INFO:teuthology.orchestra.run.smithi136.stdout: python3-jaraco.text python3-jinja2 python3-joblib python3-kubernetes 2024-09-12T23:20:45.223 INFO:teuthology.orchestra.run.smithi136.stdout: python3-logutils python3-mako python3-markupsafe python3-natsort 2024-09-12T23:20:45.223 INFO:teuthology.orchestra.run.smithi136.stdout: python3-openssl python3-paste python3-pastedeploy python3-pastescript 2024-09-12T23:20:45.223 INFO:teuthology.orchestra.run.smithi136.stdout: python3-pecan python3-pluggy python3-portend python3-prettytable 2024-09-12T23:20:45.223 INFO:teuthology.orchestra.run.smithi136.stdout: python3-psutil python3-py python3-pyasn1 python3-pyasn1-modules 2024-09-12T23:20:45.223 INFO:teuthology.orchestra.run.smithi136.stdout: python3-pygments python3-pyinotify python3-pytest python3-rados python3-rbd 2024-09-12T23:20:45.223 INFO:teuthology.orchestra.run.smithi136.stdout: python3-repoze.lru python3-requests python3-requests-oauthlib python3-rgw 2024-09-12T23:20:45.223 INFO:teuthology.orchestra.run.smithi136.stdout: python3-routes python3-rsa python3-simplegeneric python3-simplejson 2024-09-12T23:20:45.223 INFO:teuthology.orchestra.run.smithi136.stdout: python3-singledispatch python3-sklearn python3-sklearn-lib python3-tempita 2024-09-12T23:20:45.223 INFO:teuthology.orchestra.run.smithi136.stdout: python3-tempora python3-threadpoolctl python3-toml python3-urllib3 2024-09-12T23:20:45.223 INFO:teuthology.orchestra.run.smithi136.stdout: python3-waitress python3-wcwidth python3-webob python3-websocket 2024-09-12T23:20:45.223 INFO:teuthology.orchestra.run.smithi136.stdout: python3-webtest python3-werkzeug python3-zc.lockfile qttranslations5-l10n 2024-09-12T23:20:45.223 INFO:teuthology.orchestra.run.smithi136.stdout: radosgw rbd-fuse socat xmlstarlet 2024-09-12T23:20:45.224 INFO:teuthology.orchestra.run.smithi181.stdout:0 upgraded, 107 newly installed, 0 to remove and 331 not upgraded. 2024-09-12T23:20:45.224 INFO:teuthology.orchestra.run.smithi181.stdout:Need to get 187 MB of archives. 2024-09-12T23:20:45.224 INFO:teuthology.orchestra.run.smithi181.stdout:After this operation, 815 MB of additional disk space will be used. 2024-09-12T23:20:45.224 INFO:teuthology.orchestra.run.smithi181.stdout:Get:1 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libfmt8 amd64 8.1.1+ds1-2 [60.2 kB] 2024-09-12T23:20:45.225 INFO:teuthology.orchestra.run.smithi181.stdout:Get:2 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 librados2 amd64 17.2.7-1jammy [3,198 kB] 2024-09-12T23:20:45.264 INFO:teuthology.orchestra.run.smithi136.stdout:0 upgraded, 107 newly installed, 0 to remove and 331 not upgraded. 2024-09-12T23:20:45.264 INFO:teuthology.orchestra.run.smithi136.stdout:Need to get 187 MB of archives. 2024-09-12T23:20:45.264 INFO:teuthology.orchestra.run.smithi136.stdout:After this operation, 815 MB of additional disk space will be used. 2024-09-12T23:20:45.264 INFO:teuthology.orchestra.run.smithi136.stdout:Get:1 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 librados2 amd64 17.2.7-1jammy [3,198 kB] 2024-09-12T23:20:45.281 INFO:teuthology.orchestra.run.smithi181.stdout:Get:3 http://archive.ubuntu.com/ubuntu jammy/main amd64 liblttng-ust1 amd64 2.13.1-1ubuntu1 [190 kB] 2024-09-12T23:20:45.282 INFO:teuthology.orchestra.run.smithi136.stdout:Get:2 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libfmt8 amd64 8.1.1+ds1-2 [60.2 kB] 2024-09-12T23:20:45.289 INFO:teuthology.orchestra.run.smithi181.stdout:Get:4 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 librbd1 amd64 17.2.7-1jammy [3,067 kB] 2024-09-12T23:20:45.292 INFO:teuthology.orchestra.run.smithi136.stdout:Get:3 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 librbd1 amd64 17.2.7-1jammy [3,067 kB] 2024-09-12T23:20:45.319 INFO:teuthology.orchestra.run.smithi181.stdout:Get:5 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-wcwidth all 0.2.5+dfsg1-1 [21.9 kB] 2024-09-12T23:20:45.320 INFO:teuthology.orchestra.run.smithi181.stdout:Get:6 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-prettytable all 2.5.0-2 [31.3 kB] 2024-09-12T23:20:45.322 INFO:teuthology.orchestra.run.smithi181.stdout:Get:7 http://archive.ubuntu.com/ubuntu jammy/universe amd64 librdkafka1 amd64 1.8.0-1build1 [633 kB] 2024-09-12T23:20:45.335 INFO:teuthology.orchestra.run.smithi181.stdout:Get:8 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs2 amd64 17.2.7-1jammy [632 kB] 2024-09-12T23:20:45.335 INFO:teuthology.orchestra.run.smithi136.stdout:Get:4 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs2 amd64 17.2.7-1jammy [632 kB] 2024-09-12T23:20:45.341 INFO:teuthology.orchestra.run.smithi136.stdout:Get:5 http://archive.ubuntu.com/ubuntu jammy/main amd64 liblttng-ust1 amd64 2.13.1-1ubuntu1 [190 kB] 2024-09-12T23:20:45.353 INFO:teuthology.orchestra.run.smithi181.stdout:Get:9 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libdouble-conversion3 amd64 3.1.7-4 [39.0 kB] 2024-09-12T23:20:45.355 INFO:teuthology.orchestra.run.smithi181.stdout:Get:10 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libpcre2-16-0 amd64 10.39-3ubuntu0.1 [203 kB] 2024-09-12T23:20:45.360 INFO:teuthology.orchestra.run.smithi181.stdout:Get:11 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5core5a amd64 5.15.3+dfsg-2ubuntu0.2 [2,006 kB] 2024-09-12T23:20:45.364 INFO:teuthology.orchestra.run.smithi136.stdout:Get:6 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-rados amd64 17.2.7-1jammy [344 kB] 2024-09-12T23:20:45.364 INFO:teuthology.orchestra.run.smithi181.stdout:Get:12 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-rados amd64 17.2.7-1jammy [344 kB] 2024-09-12T23:20:45.378 INFO:teuthology.orchestra.run.smithi136.stdout:Get:7 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-wcwidth all 0.2.5+dfsg1-1 [21.9 kB] 2024-09-12T23:20:45.380 INFO:teuthology.orchestra.run.smithi136.stdout:Get:8 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-prettytable all 2.5.0-2 [31.3 kB] 2024-09-12T23:20:45.383 INFO:teuthology.orchestra.run.smithi136.stdout:Get:9 http://archive.ubuntu.com/ubuntu jammy/universe amd64 librdkafka1 amd64 1.8.0-1build1 [633 kB] 2024-09-12T23:20:45.391 INFO:teuthology.orchestra.run.smithi136.stdout:Get:10 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-argparse all 17.2.7-1jammy [23.1 kB] 2024-09-12T23:20:45.392 INFO:teuthology.orchestra.run.smithi181.stdout:Get:13 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-argparse all 17.2.7-1jammy [23.1 kB] 2024-09-12T23:20:45.402 INFO:teuthology.orchestra.run.smithi181.stdout:Get:14 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5dbus5 amd64 5.15.3+dfsg-2ubuntu0.2 [222 kB] 2024-09-12T23:20:45.405 INFO:teuthology.orchestra.run.smithi181.stdout:Get:15 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5network5 amd64 5.15.3+dfsg-2ubuntu0.2 [731 kB] 2024-09-12T23:20:45.412 INFO:teuthology.orchestra.run.smithi136.stdout:Get:11 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-cephfs amd64 17.2.7-1jammy [164 kB] 2024-09-12T23:20:45.413 INFO:teuthology.orchestra.run.smithi181.stdout:Get:16 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-cephfs amd64 17.2.7-1jammy [164 kB] 2024-09-12T23:20:45.414 INFO:teuthology.orchestra.run.smithi136.stdout:Get:12 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libdouble-conversion3 amd64 3.1.7-4 [39.0 kB] 2024-09-12T23:20:45.414 INFO:teuthology.orchestra.run.smithi181.stdout:Get:17 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libthrift-0.16.0 amd64 0.16.0-2 [267 kB] 2024-09-12T23:20:45.416 INFO:teuthology.orchestra.run.smithi136.stdout:Get:13 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libpcre2-16-0 amd64 10.39-3ubuntu0.1 [203 kB] 2024-09-12T23:20:45.417 INFO:teuthology.orchestra.run.smithi181.stdout:Get:18 http://archive.ubuntu.com/ubuntu jammy/main amd64 liboath0 amd64 2.6.7-3build1 [41.1 kB] 2024-09-12T23:20:45.418 INFO:teuthology.orchestra.run.smithi181.stdout:Get:19 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-openssl all 21.0.0-1 [45.2 kB] 2024-09-12T23:20:45.419 INFO:teuthology.orchestra.run.smithi181.stdout:Get:20 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-bcrypt amd64 3.2.0-1build1 [32.7 kB] 2024-09-12T23:20:45.419 INFO:teuthology.orchestra.run.smithi181.stdout:Get:21 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.functools all 3.4.0-2 [9,030 B] 2024-09-12T23:20:45.420 INFO:teuthology.orchestra.run.smithi181.stdout:Get:22 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-cheroot all 8.5.2+ds1-1ubuntu3.1 [71.1 kB] 2024-09-12T23:20:45.421 INFO:teuthology.orchestra.run.smithi181.stdout:Get:23 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.classes all 3.2.1-3 [6,452 B] 2024-09-12T23:20:45.421 INFO:teuthology.orchestra.run.smithi136.stdout:Get:14 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5core5a amd64 5.15.3+dfsg-2ubuntu0.2 [2,006 kB] 2024-09-12T23:20:45.421 INFO:teuthology.orchestra.run.smithi181.stdout:Get:24 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.text all 3.6.0-2 [8,716 B] 2024-09-12T23:20:45.439 INFO:teuthology.orchestra.run.smithi181.stdout:Get:25 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.collections all 3.4.0-2 [11.4 kB] 2024-09-12T23:20:45.439 INFO:teuthology.orchestra.run.smithi181.stdout:Get:26 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempora all 4.1.2-1 [14.8 kB] 2024-09-12T23:20:45.439 INFO:teuthology.orchestra.run.smithi181.stdout:Get:27 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-portend all 3.0.0-1 [7,240 B] 2024-09-12T23:20:45.440 INFO:teuthology.orchestra.run.smithi181.stdout:Get:28 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-zc.lockfile all 2.0-1 [8,980 B] 2024-09-12T23:20:45.440 INFO:teuthology.orchestra.run.smithi181.stdout:Get:29 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cherrypy3 all 18.6.1-4 [208 kB] 2024-09-12T23:20:45.443 INFO:teuthology.orchestra.run.smithi181.stdout:Get:30 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-natsort all 8.0.2-1 [35.3 kB] 2024-09-12T23:20:45.443 INFO:teuthology.orchestra.run.smithi181.stdout:Get:31 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-logutils all 0.3.3-8 [17.6 kB] 2024-09-12T23:20:45.444 INFO:teuthology.orchestra.run.smithi181.stdout:Get:32 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-markupsafe amd64 2.0.1-2build1 [12.7 kB] 2024-09-12T23:20:45.444 INFO:teuthology.orchestra.run.smithi181.stdout:Get:33 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-mako all 1.1.3+ds1-2ubuntu0.1 [60.5 kB] 2024-09-12T23:20:45.450 INFO:teuthology.orchestra.run.smithi181.stdout:Get:34 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-common all 17.2.7-1jammy [47.7 kB] 2024-09-12T23:20:45.450 INFO:teuthology.orchestra.run.smithi136.stdout:Get:15 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-common all 17.2.7-1jammy [47.7 kB] 2024-09-12T23:20:45.455 INFO:teuthology.orchestra.run.smithi181.stdout:Get:35 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplegeneric all 0.8.1-3 [11.3 kB] 2024-09-12T23:20:45.462 INFO:teuthology.orchestra.run.smithi136.stdout:Get:16 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5dbus5 amd64 5.15.3+dfsg-2ubuntu0.2 [222 kB] 2024-09-12T23:20:45.462 INFO:teuthology.orchestra.run.smithi181.stdout:Get:36 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-singledispatch all 3.4.0.3-3 [7,320 B] 2024-09-12T23:20:45.463 INFO:teuthology.orchestra.run.smithi181.stdout:Get:37 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-webob all 1:1.8.6-1.1ubuntu0.1 [86.7 kB] 2024-09-12T23:20:45.463 INFO:teuthology.orchestra.run.smithi181.stdout:Get:38 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-rbd amd64 17.2.7-1jammy [313 kB] 2024-09-12T23:20:45.463 INFO:teuthology.orchestra.run.smithi136.stdout:Get:17 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-rbd amd64 17.2.7-1jammy [313 kB] 2024-09-12T23:20:45.464 INFO:teuthology.orchestra.run.smithi181.stdout:Get:39 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-waitress all 1.4.4-1.1ubuntu1 [47.3 kB] 2024-09-12T23:20:45.465 INFO:teuthology.orchestra.run.smithi181.stdout:Get:40 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempita all 0.5.2-6ubuntu1 [15.1 kB] 2024-09-12T23:20:45.465 INFO:teuthology.orchestra.run.smithi136.stdout:Get:18 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5network5 amd64 5.15.3+dfsg-2ubuntu0.2 [731 kB] 2024-09-12T23:20:45.466 INFO:teuthology.orchestra.run.smithi181.stdout:Get:41 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-paste all 3.5.0+dfsg1-1 [456 kB] 2024-09-12T23:20:45.470 INFO:teuthology.orchestra.run.smithi181.stdout:Get:42 http://archive.ubuntu.com/ubuntu jammy/main amd64 python-pastedeploy-tpl all 2.1.1-1 [4,892 B] 2024-09-12T23:20:45.470 INFO:teuthology.orchestra.run.smithi181.stdout:Get:43 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pastedeploy all 2.1.1-1 [26.6 kB] 2024-09-12T23:20:45.471 INFO:teuthology.orchestra.run.smithi181.stdout:Get:44 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-webtest all 2.0.35-1 [28.5 kB] 2024-09-12T23:20:45.473 INFO:teuthology.orchestra.run.smithi181.stdout:Get:45 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pecan all 1.3.3-4ubuntu2 [87.3 kB] 2024-09-12T23:20:45.474 INFO:teuthology.orchestra.run.smithi136.stdout:Get:19 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libthrift-0.16.0 amd64 0.16.0-2 [267 kB] 2024-09-12T23:20:45.477 INFO:teuthology.orchestra.run.smithi136.stdout:Get:20 http://archive.ubuntu.com/ubuntu jammy/main amd64 liboath0 amd64 2.6.7-3build1 [41.1 kB] 2024-09-12T23:20:45.478 INFO:teuthology.orchestra.run.smithi136.stdout:Get:21 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-openssl all 21.0.0-1 [45.2 kB] 2024-09-12T23:20:45.479 INFO:teuthology.orchestra.run.smithi136.stdout:Get:22 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-bcrypt amd64 3.2.0-1build1 [32.7 kB] 2024-09-12T23:20:45.479 INFO:teuthology.orchestra.run.smithi136.stdout:Get:23 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.functools all 3.4.0-2 [9,030 B] 2024-09-12T23:20:45.480 INFO:teuthology.orchestra.run.smithi136.stdout:Get:24 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-cheroot all 8.5.2+ds1-1ubuntu3.1 [71.1 kB] 2024-09-12T23:20:45.481 INFO:teuthology.orchestra.run.smithi136.stdout:Get:25 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.classes all 3.2.1-3 [6,452 B] 2024-09-12T23:20:45.482 INFO:teuthology.orchestra.run.smithi136.stdout:Get:26 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.text all 3.6.0-2 [8,716 B] 2024-09-12T23:20:45.489 INFO:teuthology.orchestra.run.smithi181.stdout:Get:46 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-certifi all 2020.6.20-1 [150 kB] 2024-09-12T23:20:45.489 INFO:teuthology.orchestra.run.smithi136.stdout:Get:27 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 librgw2 amd64 17.2.7-1jammy [5,086 kB] 2024-09-12T23:20:45.490 INFO:teuthology.orchestra.run.smithi181.stdout:Get:47 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 librgw2 amd64 17.2.7-1jammy [5,086 kB] 2024-09-12T23:20:45.491 INFO:teuthology.orchestra.run.smithi181.stdout:Get:48 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-idna all 3.3-1ubuntu0.1 [52.1 kB] 2024-09-12T23:20:45.492 INFO:teuthology.orchestra.run.smithi181.stdout:Get:49 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-urllib3 all 1.26.5-1~exp1ubuntu0.1 [98.2 kB] 2024-09-12T23:20:45.494 INFO:teuthology.orchestra.run.smithi181.stdout:Get:50 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-requests all 2.25.1+dfsg-2ubuntu0.1 [48.8 kB] 2024-09-12T23:20:45.494 INFO:teuthology.orchestra.run.smithi181.stdout:Get:51 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-werkzeug all 2.0.2+dfsg1-1ubuntu0.22.04.2 [181 kB] 2024-09-12T23:20:45.497 INFO:teuthology.orchestra.run.smithi181.stdout:Get:52 http://archive.ubuntu.com/ubuntu jammy/main amd64 python-babel-localedata all 2.8.0+dfsg.1-7 [4,982 kB] 2024-09-12T23:20:45.498 INFO:teuthology.orchestra.run.smithi136.stdout:Get:28 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.collections all 3.4.0-2 [11.4 kB] 2024-09-12T23:20:45.499 INFO:teuthology.orchestra.run.smithi136.stdout:Get:29 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempora all 4.1.2-1 [14.8 kB] 2024-09-12T23:20:45.499 INFO:teuthology.orchestra.run.smithi136.stdout:Get:30 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-portend all 3.0.0-1 [7,240 B] 2024-09-12T23:20:45.500 INFO:teuthology.orchestra.run.smithi136.stdout:Get:31 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-zc.lockfile all 2.0-1 [8,980 B] 2024-09-12T23:20:45.500 INFO:teuthology.orchestra.run.smithi136.stdout:Get:32 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cherrypy3 all 18.6.1-4 [208 kB] 2024-09-12T23:20:45.503 INFO:teuthology.orchestra.run.smithi136.stdout:Get:33 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-natsort all 8.0.2-1 [35.3 kB] 2024-09-12T23:20:45.504 INFO:teuthology.orchestra.run.smithi136.stdout:Get:34 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-logutils all 0.3.3-8 [17.6 kB] 2024-09-12T23:20:45.504 INFO:teuthology.orchestra.run.smithi136.stdout:Get:35 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-markupsafe amd64 2.0.1-2build1 [12.7 kB] 2024-09-12T23:20:45.504 INFO:teuthology.orchestra.run.smithi136.stdout:Get:36 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-mako all 1.1.3+ds1-2ubuntu0.1 [60.5 kB] 2024-09-12T23:20:45.516 INFO:teuthology.orchestra.run.smithi136.stdout:Get:37 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplegeneric all 0.8.1-3 [11.3 kB] 2024-09-12T23:20:45.522 INFO:teuthology.orchestra.run.smithi136.stdout:Get:38 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-singledispatch all 3.4.0.3-3 [7,320 B] 2024-09-12T23:20:45.523 INFO:teuthology.orchestra.run.smithi136.stdout:Get:39 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-webob all 1:1.8.6-1.1ubuntu0.1 [86.7 kB] 2024-09-12T23:20:45.524 INFO:teuthology.orchestra.run.smithi136.stdout:Get:40 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-waitress all 1.4.4-1.1ubuntu1 [47.3 kB] 2024-09-12T23:20:45.525 INFO:teuthology.orchestra.run.smithi136.stdout:Get:41 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempita all 0.5.2-6ubuntu1 [15.1 kB] 2024-09-12T23:20:45.525 INFO:teuthology.orchestra.run.smithi136.stdout:Get:42 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-paste all 3.5.0+dfsg1-1 [456 kB] 2024-09-12T23:20:45.531 INFO:teuthology.orchestra.run.smithi136.stdout:Get:43 http://archive.ubuntu.com/ubuntu jammy/main amd64 python-pastedeploy-tpl all 2.1.1-1 [4,892 B] 2024-09-12T23:20:45.531 INFO:teuthology.orchestra.run.smithi136.stdout:Get:44 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pastedeploy all 2.1.1-1 [26.6 kB] 2024-09-12T23:20:45.531 INFO:teuthology.orchestra.run.smithi136.stdout:Get:45 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-webtest all 2.0.35-1 [28.5 kB] 2024-09-12T23:20:45.533 INFO:teuthology.orchestra.run.smithi136.stdout:Get:46 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pecan all 1.3.3-4ubuntu2 [87.3 kB] 2024-09-12T23:20:45.549 INFO:teuthology.orchestra.run.smithi181.stdout:Get:53 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-babel all 2.8.0+dfsg.1-7 [85.1 kB] 2024-09-12T23:20:45.549 INFO:teuthology.orchestra.run.smithi136.stdout:Get:47 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-certifi all 2020.6.20-1 [150 kB] 2024-09-12T23:20:45.550 INFO:teuthology.orchestra.run.smithi181.stdout:Get:54 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-jinja2 all 3.0.3-1ubuntu0.2 [108 kB] 2024-09-12T23:20:45.551 INFO:teuthology.orchestra.run.smithi181.stdout:Get:55 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2024-09-12T23:20:45.551 INFO:teuthology.orchestra.run.smithi181.stdout:Get:56 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-routes all 2.5.1-1ubuntu1 [89.0 kB] 2024-09-12T23:20:45.552 INFO:teuthology.orchestra.run.smithi136.stdout:Get:48 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-idna all 3.3-1ubuntu0.1 [52.1 kB] 2024-09-12T23:20:45.552 INFO:teuthology.orchestra.run.smithi181.stdout:Get:57 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn-lib amd64 0.23.2-5ubuntu6 [2,058 kB] 2024-09-12T23:20:45.552 INFO:teuthology.orchestra.run.smithi136.stdout:Get:49 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-urllib3 all 1.26.5-1~exp1ubuntu0.1 [98.2 kB] 2024-09-12T23:20:45.554 INFO:teuthology.orchestra.run.smithi136.stdout:Get:50 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-requests all 2.25.1+dfsg-2ubuntu0.1 [48.8 kB] 2024-09-12T23:20:45.555 INFO:teuthology.orchestra.run.smithi136.stdout:Get:51 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-werkzeug all 2.0.2+dfsg1-1ubuntu0.22.04.2 [181 kB] 2024-09-12T23:20:45.558 INFO:teuthology.orchestra.run.smithi136.stdout:Get:52 http://archive.ubuntu.com/ubuntu jammy/main amd64 python-babel-localedata all 2.8.0+dfsg.1-7 [4,982 kB] 2024-09-12T23:20:45.570 INFO:teuthology.orchestra.run.smithi181.stdout:Get:58 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-joblib all 0.17.0-4ubuntu1 [204 kB] 2024-09-12T23:20:45.571 INFO:teuthology.orchestra.run.smithi181.stdout:Get:59 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-threadpoolctl all 3.1.0-1 [21.3 kB] 2024-09-12T23:20:45.572 INFO:teuthology.orchestra.run.smithi181.stdout:Get:60 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn all 0.23.2-5ubuntu6 [1,829 kB] 2024-09-12T23:20:45.583 INFO:teuthology.orchestra.run.smithi136.stdout:Get:53 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-rgw amd64 17.2.7-1jammy [102 kB] 2024-09-12T23:20:45.585 INFO:teuthology.orchestra.run.smithi181.stdout:Get:61 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-rgw amd64 17.2.7-1jammy [102 kB] 2024-09-12T23:20:45.585 INFO:teuthology.orchestra.run.smithi136.stdout:Get:54 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 libradosstriper1 amd64 17.2.7-1jammy [415 kB] 2024-09-12T23:20:45.586 INFO:teuthology.orchestra.run.smithi181.stdout:Get:62 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 libradosstriper1 amd64 17.2.7-1jammy [415 kB] 2024-09-12T23:20:45.586 INFO:teuthology.orchestra.run.smithi181.stdout:Get:63 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cachetools all 5.0.0-1 [9,722 B] 2024-09-12T23:20:45.611 INFO:teuthology.orchestra.run.smithi136.stdout:Get:55 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-common amd64 17.2.7-1jammy [25.3 MB] 2024-09-12T23:20:45.619 INFO:teuthology.orchestra.run.smithi181.stdout:Get:64 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-common amd64 17.2.7-1jammy [25.3 MB] 2024-09-12T23:20:45.619 INFO:teuthology.orchestra.run.smithi181.stdout:Get:65 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyasn1 all 0.4.8-1 [50.9 kB] 2024-09-12T23:20:45.619 INFO:teuthology.orchestra.run.smithi181.stdout:Get:66 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyasn1-modules all 0.2.1-1 [33.0 kB] 2024-09-12T23:20:45.620 INFO:teuthology.orchestra.run.smithi181.stdout:Get:67 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-rsa all 4.8-1 [28.4 kB] 2024-09-12T23:20:45.620 INFO:teuthology.orchestra.run.smithi181.stdout:Get:68 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-google-auth all 1.5.1-3 [35.7 kB] 2024-09-12T23:20:45.621 INFO:teuthology.orchestra.run.smithi181.stdout:Get:69 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-requests-oauthlib all 1.3.0+ds-0.1 [18.7 kB] 2024-09-12T23:20:45.621 INFO:teuthology.orchestra.run.smithi181.stdout:Get:70 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-websocket all 1.2.3-1 [34.7 kB] 2024-09-12T23:20:45.621 INFO:teuthology.orchestra.run.smithi181.stdout:Get:71 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-kubernetes all 12.0.1-1ubuntu1 [353 kB] 2024-09-12T23:20:45.624 INFO:teuthology.orchestra.run.smithi181.stdout:Get:72 http://archive.ubuntu.com/ubuntu jammy/main amd64 libonig5 amd64 6.9.7.1-2build1 [172 kB] 2024-09-12T23:20:45.634 INFO:teuthology.orchestra.run.smithi136.stdout:Get:56 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-babel all 2.8.0+dfsg.1-7 [85.1 kB] 2024-09-12T23:20:45.635 INFO:teuthology.orchestra.run.smithi136.stdout:Get:57 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-jinja2 all 3.0.3-1ubuntu0.2 [108 kB] 2024-09-12T23:20:45.636 INFO:teuthology.orchestra.run.smithi136.stdout:Get:58 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2024-09-12T23:20:45.636 INFO:teuthology.orchestra.run.smithi136.stdout:Get:59 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-routes all 2.5.1-1ubuntu1 [89.0 kB] 2024-09-12T23:20:45.636 INFO:teuthology.orchestra.run.smithi181.stdout:Get:73 http://archive.ubuntu.com/ubuntu jammy/main amd64 libjq1 amd64 1.6-2.1ubuntu3 [133 kB] 2024-09-12T23:20:45.637 INFO:teuthology.orchestra.run.smithi136.stdout:Get:60 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn-lib amd64 0.23.2-5ubuntu6 [2,058 kB] 2024-09-12T23:20:45.644 INFO:teuthology.orchestra.run.smithi181.stdout:Get:74 http://archive.ubuntu.com/ubuntu jammy/main amd64 jq amd64 1.6-2.1ubuntu3 [52.5 kB] 2024-09-12T23:20:45.644 INFO:teuthology.orchestra.run.smithi181.stdout:Get:75 http://archive.ubuntu.com/ubuntu jammy/main amd64 socat amd64 1.7.4.1-3ubuntu4 [349 kB] 2024-09-12T23:20:45.648 INFO:teuthology.orchestra.run.smithi181.stdout:Get:76 http://archive.ubuntu.com/ubuntu jammy/universe amd64 xmlstarlet amd64 1.6.1-2.1 [265 kB] 2024-09-12T23:20:45.650 INFO:teuthology.orchestra.run.smithi181.stdout:Get:77 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 nvme-cli amd64 1.16-3ubuntu0.3 [474 kB] 2024-09-12T23:20:45.655 INFO:teuthology.orchestra.run.smithi136.stdout:Get:61 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-joblib all 0.17.0-4ubuntu1 [204 kB] 2024-09-12T23:20:45.656 INFO:teuthology.orchestra.run.smithi181.stdout:Get:78 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-iniconfig all 1.1.1-2 [6,024 B] 2024-09-12T23:20:45.656 INFO:teuthology.orchestra.run.smithi181.stdout:Get:79 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pastescript all 2.0.2-4 [54.6 kB] 2024-09-12T23:20:45.656 INFO:teuthology.orchestra.run.smithi181.stdout:Get:80 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pluggy all 0.13.0-7.1 [19.0 kB] 2024-09-12T23:20:45.656 INFO:teuthology.orchestra.run.smithi181.stdout:Get:81 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-psutil amd64 5.9.0-1build1 [158 kB] 2024-09-12T23:20:45.657 INFO:teuthology.orchestra.run.smithi181.stdout:Get:82 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-py all 1.10.0-1 [71.9 kB] 2024-09-12T23:20:45.657 INFO:teuthology.orchestra.run.smithi136.stdout:Get:62 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-threadpoolctl all 3.1.0-1 [21.3 kB] 2024-09-12T23:20:45.657 INFO:teuthology.orchestra.run.smithi136.stdout:Get:63 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn all 0.23.2-5ubuntu6 [1,829 kB] 2024-09-12T23:20:45.671 INFO:teuthology.orchestra.run.smithi181.stdout:Get:83 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pygments all 2.11.2+dfsg-2 [750 kB] 2024-09-12T23:20:45.672 INFO:teuthology.orchestra.run.smithi136.stdout:Get:64 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cachetools all 5.0.0-1 [9,722 B] 2024-09-12T23:20:45.672 INFO:teuthology.orchestra.run.smithi136.stdout:Get:65 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyasn1 all 0.4.8-1 [50.9 kB] 2024-09-12T23:20:45.672 INFO:teuthology.orchestra.run.smithi136.stdout:Get:66 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyasn1-modules all 0.2.1-1 [33.0 kB] 2024-09-12T23:20:45.673 INFO:teuthology.orchestra.run.smithi136.stdout:Get:67 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-rsa all 4.8-1 [28.4 kB] 2024-09-12T23:20:45.673 INFO:teuthology.orchestra.run.smithi136.stdout:Get:68 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-google-auth all 1.5.1-3 [35.7 kB] 2024-09-12T23:20:45.674 INFO:teuthology.orchestra.run.smithi136.stdout:Get:69 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-requests-oauthlib all 1.3.0+ds-0.1 [18.7 kB] 2024-09-12T23:20:45.677 INFO:teuthology.orchestra.run.smithi181.stdout:Get:84 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyinotify all 0.9.6-1.3 [24.8 kB] 2024-09-12T23:20:45.677 INFO:teuthology.orchestra.run.smithi181.stdout:Get:85 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-toml all 0.10.2-1 [16.5 kB] 2024-09-12T23:20:45.677 INFO:teuthology.orchestra.run.smithi181.stdout:Get:86 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pytest all 6.2.5-1ubuntu2 [214 kB] 2024-09-12T23:20:45.688 INFO:teuthology.orchestra.run.smithi181.stdout:Get:87 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplejson amd64 3.17.6-1build1 [54.7 kB] 2024-09-12T23:20:45.689 INFO:teuthology.orchestra.run.smithi181.stdout:Get:88 http://archive.ubuntu.com/ubuntu jammy/universe amd64 qttranslations5-l10n all 5.15.3-1 [1,983 kB] 2024-09-12T23:20:45.689 INFO:teuthology.orchestra.run.smithi136.stdout:Get:70 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-websocket all 1.2.3-1 [34.7 kB] 2024-09-12T23:20:45.689 INFO:teuthology.orchestra.run.smithi136.stdout:Get:71 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-kubernetes all 12.0.1-1ubuntu1 [353 kB] 2024-09-12T23:20:45.691 INFO:teuthology.orchestra.run.smithi136.stdout:Get:72 http://archive.ubuntu.com/ubuntu jammy/main amd64 libonig5 amd64 6.9.7.1-2build1 [172 kB] 2024-09-12T23:20:45.704 INFO:teuthology.orchestra.run.smithi136.stdout:Get:73 http://archive.ubuntu.com/ubuntu jammy/main amd64 libjq1 amd64 1.6-2.1ubuntu3 [133 kB] 2024-09-12T23:20:45.705 INFO:teuthology.orchestra.run.smithi136.stdout:Get:74 http://archive.ubuntu.com/ubuntu jammy/main amd64 jq amd64 1.6-2.1ubuntu3 [52.5 kB] 2024-09-12T23:20:45.706 INFO:teuthology.orchestra.run.smithi136.stdout:Get:75 http://archive.ubuntu.com/ubuntu jammy/main amd64 socat amd64 1.7.4.1-3ubuntu4 [349 kB] 2024-09-12T23:20:45.710 INFO:teuthology.orchestra.run.smithi136.stdout:Get:76 http://archive.ubuntu.com/ubuntu jammy/universe amd64 xmlstarlet amd64 1.6.1-2.1 [265 kB] 2024-09-12T23:20:45.712 INFO:teuthology.orchestra.run.smithi136.stdout:Get:77 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 nvme-cli amd64 1.16-3ubuntu0.3 [474 kB] 2024-09-12T23:20:45.716 INFO:teuthology.orchestra.run.smithi136.stdout:Get:78 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-iniconfig all 1.1.1-2 [6,024 B] 2024-09-12T23:20:45.716 INFO:teuthology.orchestra.run.smithi136.stdout:Get:79 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pastescript all 2.0.2-4 [54.6 kB] 2024-09-12T23:20:45.721 INFO:teuthology.orchestra.run.smithi136.stdout:Get:80 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pluggy all 0.13.0-7.1 [19.0 kB] 2024-09-12T23:20:45.721 INFO:teuthology.orchestra.run.smithi136.stdout:Get:81 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-psutil amd64 5.9.0-1build1 [158 kB] 2024-09-12T23:20:45.724 INFO:teuthology.orchestra.run.smithi136.stdout:Get:82 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-py all 1.10.0-1 [71.9 kB] 2024-09-12T23:20:45.740 INFO:teuthology.orchestra.run.smithi136.stdout:Get:83 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pygments all 2.11.2+dfsg-2 [750 kB] 2024-09-12T23:20:45.748 INFO:teuthology.orchestra.run.smithi136.stdout:Get:84 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyinotify all 0.9.6-1.3 [24.8 kB] 2024-09-12T23:20:45.748 INFO:teuthology.orchestra.run.smithi136.stdout:Get:85 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-toml all 0.10.2-1 [16.5 kB] 2024-09-12T23:20:45.748 INFO:teuthology.orchestra.run.smithi136.stdout:Get:86 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pytest all 6.2.5-1ubuntu2 [214 kB] 2024-09-12T23:20:45.750 INFO:teuthology.orchestra.run.smithi136.stdout:Get:87 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplejson amd64 3.17.6-1build1 [54.7 kB] 2024-09-12T23:20:45.751 INFO:teuthology.orchestra.run.smithi136.stdout:Get:88 http://archive.ubuntu.com/ubuntu jammy/universe amd64 qttranslations5-l10n all 5.15.3-1 [1,983 kB] 2024-09-12T23:20:46.277 INFO:teuthology.orchestra.run.smithi136.stdout:Get:89 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-base amd64 17.2.7-1jammy [6,378 kB] 2024-09-12T23:20:46.280 INFO:teuthology.orchestra.run.smithi181.stdout:Get:89 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-base amd64 17.2.7-1jammy [6,378 kB] 2024-09-12T23:20:46.376 INFO:teuthology.orchestra.run.smithi181.stdout:Get:90 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-modules-core all 17.2.7-1jammy [224 kB] 2024-09-12T23:20:46.376 INFO:teuthology.orchestra.run.smithi136.stdout:Get:90 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-modules-core all 17.2.7-1jammy [224 kB] 2024-09-12T23:20:46.400 INFO:teuthology.orchestra.run.smithi181.stdout:Get:91 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 libsqlite3-mod-ceph amd64 17.2.7-1jammy [115 kB] 2024-09-12T23:20:46.401 INFO:teuthology.orchestra.run.smithi136.stdout:Get:91 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 libsqlite3-mod-ceph amd64 17.2.7-1jammy [115 kB] 2024-09-12T23:20:46.427 INFO:teuthology.orchestra.run.smithi181.stdout:Get:92 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr amd64 17.2.7-1jammy [1,429 kB] 2024-09-12T23:20:46.428 INFO:teuthology.orchestra.run.smithi136.stdout:Get:92 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr amd64 17.2.7-1jammy [1,429 kB] 2024-09-12T23:20:46.477 INFO:teuthology.orchestra.run.smithi181.stdout:Get:93 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mon amd64 17.2.7-1jammy [8,201 kB] 2024-09-12T23:20:46.477 INFO:teuthology.orchestra.run.smithi136.stdout:Get:93 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mon amd64 17.2.7-1jammy [8,201 kB] 2024-09-12T23:20:46.695 INFO:teuthology.orchestra.run.smithi181.stdout:Get:94 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-osd amd64 17.2.7-1jammy [28.4 MB] 2024-09-12T23:20:46.697 INFO:teuthology.orchestra.run.smithi136.stdout:Get:94 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-osd amd64 17.2.7-1jammy [28.4 MB] 2024-09-12T23:20:47.385 INFO:teuthology.orchestra.run.smithi181.stdout:Get:95 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph amd64 17.2.7-1jammy [3,690 B] 2024-09-12T23:20:47.388 INFO:teuthology.orchestra.run.smithi181.stdout:Get:96 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-fuse amd64 17.2.7-1jammy [783 kB] 2024-09-12T23:20:47.390 INFO:teuthology.orchestra.run.smithi136.stdout:Get:95 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph amd64 17.2.7-1jammy [3,690 B] 2024-09-12T23:20:47.393 INFO:teuthology.orchestra.run.smithi136.stdout:Get:96 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-fuse amd64 17.2.7-1jammy [783 kB] 2024-09-12T23:20:47.394 INFO:teuthology.orchestra.run.smithi181.stdout:Get:97 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mds amd64 17.2.7-1jammy [2,120 kB] 2024-09-12T23:20:47.399 INFO:teuthology.orchestra.run.smithi136.stdout:Get:97 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mds amd64 17.2.7-1jammy [2,120 kB] 2024-09-12T23:20:47.410 INFO:teuthology.orchestra.run.smithi181.stdout:Get:98 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 cephadm amd64 17.2.7-1jammy [80.2 kB] 2024-09-12T23:20:47.416 INFO:teuthology.orchestra.run.smithi136.stdout:Get:98 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 cephadm amd64 17.2.7-1jammy [80.2 kB] 2024-09-12T23:20:47.423 INFO:teuthology.orchestra.run.smithi181.stdout:Get:99 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-cephadm all 17.2.7-1jammy [118 kB] 2024-09-12T23:20:47.423 INFO:teuthology.orchestra.run.smithi136.stdout:Get:99 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-cephadm all 17.2.7-1jammy [118 kB] 2024-09-12T23:20:47.451 INFO:teuthology.orchestra.run.smithi181.stdout:Get:100 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-dashboard all 17.2.7-1jammy [1,824 kB] 2024-09-12T23:20:47.451 INFO:teuthology.orchestra.run.smithi136.stdout:Get:100 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-dashboard all 17.2.7-1jammy [1,824 kB] 2024-09-12T23:20:47.488 INFO:teuthology.orchestra.run.smithi181.stdout:Get:101 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-diskprediction-local all 17.2.7-1jammy [8,615 kB] 2024-09-12T23:20:47.488 INFO:teuthology.orchestra.run.smithi136.stdout:Get:101 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-diskprediction-local all 17.2.7-1jammy [8,615 kB] 2024-09-12T23:20:47.648 INFO:teuthology.orchestra.run.smithi181.stdout:Get:102 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-k8sevents all 17.2.7-1jammy [3,814 B] 2024-09-12T23:20:47.659 INFO:teuthology.orchestra.run.smithi181.stdout:Get:103 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-test amd64 17.2.7-1jammy [52.7 MB] 2024-09-12T23:20:47.665 INFO:teuthology.orchestra.run.smithi136.stdout:Get:102 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-k8sevents all 17.2.7-1jammy [3,814 B] 2024-09-12T23:20:47.665 INFO:teuthology.orchestra.run.smithi136.stdout:Get:103 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-test amd64 17.2.7-1jammy [52.7 MB] 2024-09-12T23:20:48.803 INFO:teuthology.orchestra.run.smithi136.stdout:Get:104 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-volume all 17.2.7-1jammy [240 kB] 2024-09-12T23:20:48.806 INFO:teuthology.orchestra.run.smithi136.stdout:Get:105 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs-dev amd64 17.2.7-1jammy [22.4 kB] 2024-09-12T23:20:48.806 INFO:teuthology.orchestra.run.smithi136.stdout:Get:106 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 radosgw amd64 17.2.7-1jammy [15.4 MB] 2024-09-12T23:20:48.830 INFO:teuthology.orchestra.run.smithi181.stdout:Get:104 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-volume all 17.2.7-1jammy [240 kB] 2024-09-12T23:20:48.832 INFO:teuthology.orchestra.run.smithi181.stdout:Get:105 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs-dev amd64 17.2.7-1jammy [22.4 kB] 2024-09-12T23:20:48.833 INFO:teuthology.orchestra.run.smithi181.stdout:Get:106 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 radosgw amd64 17.2.7-1jammy [15.4 MB] 2024-09-12T23:20:49.019 INFO:teuthology.orchestra.run.smithi136.stdout:Get:107 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 rbd-fuse amd64 17.2.7-1jammy [75.8 kB] 2024-09-12T23:20:49.036 INFO:teuthology.orchestra.run.smithi181.stdout:Get:107 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 rbd-fuse amd64 17.2.7-1jammy [75.8 kB] 2024-09-12T23:20:49.558 INFO:teuthology.orchestra.run.smithi181.stdout:Fetched 187 MB in 4s (48.3 MB/s) 2024-09-12T23:20:49.587 INFO:teuthology.orchestra.run.smithi136.stdout:Fetched 187 MB in 4s (49.2 MB/s) 2024-09-12T23:20:49.692 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package libfmt8:amd64. 2024-09-12T23:20:49.702 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package libfmt8:amd64. 2024-09-12T23:20:49.749 INFO:teuthology.orchestra.run.smithi136.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 165990 files and directories currently installed.) 2024-09-12T23:20:49.753 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../000-libfmt8_8.1.1+ds1-2_amd64.deb ... 2024-09-12T23:20:49.804 INFO:teuthology.orchestra.run.smithi181.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 165990 files and directories currently installed.) 2024-09-12T23:20:49.809 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../000-libfmt8_8.1.1+ds1-2_amd64.deb ... 2024-09-12T23:20:49.824 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking libfmt8:amd64 (8.1.1+ds1-2) ... 2024-09-12T23:20:49.884 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking libfmt8:amd64 (8.1.1+ds1-2) ... 2024-09-12T23:20:50.161 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package liblttng-ust1:amd64. 2024-09-12T23:20:50.174 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../001-liblttng-ust1_2.13.1-1ubuntu1_amd64.deb ... 2024-09-12T23:20:50.219 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2024-09-12T23:20:50.237 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package liblttng-ust1:amd64. 2024-09-12T23:20:50.264 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../001-liblttng-ust1_2.13.1-1ubuntu1_amd64.deb ... 2024-09-12T23:20:50.304 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2024-09-12T23:20:50.539 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package librados2. 2024-09-12T23:20:50.565 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../002-librados2_17.2.7-1jammy_amd64.deb ... 2024-09-12T23:20:50.606 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking librados2 (17.2.7-1jammy) ... 2024-09-12T23:20:50.624 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package librados2. 2024-09-12T23:20:50.651 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../002-librados2_17.2.7-1jammy_amd64.deb ... 2024-09-12T23:20:50.690 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking librados2 (17.2.7-1jammy) ... 2024-09-12T23:20:51.042 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package librbd1. 2024-09-12T23:20:51.068 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../003-librbd1_17.2.7-1jammy_amd64.deb ... 2024-09-12T23:20:51.100 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking librbd1 (17.2.7-1jammy) ... 2024-09-12T23:20:51.118 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package librbd1. 2024-09-12T23:20:51.138 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../003-librbd1_17.2.7-1jammy_amd64.deb ... 2024-09-12T23:20:51.177 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking librbd1 (17.2.7-1jammy) ... 2024-09-12T23:20:51.486 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package libcephfs2. 2024-09-12T23:20:51.512 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../004-libcephfs2_17.2.7-1jammy_amd64.deb ... 2024-09-12T23:20:51.554 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package libcephfs2. 2024-09-12T23:20:51.578 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking libcephfs2 (17.2.7-1jammy) ... 2024-09-12T23:20:51.581 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../004-libcephfs2_17.2.7-1jammy_amd64.deb ... 2024-09-12T23:20:51.613 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking libcephfs2 (17.2.7-1jammy) ... 2024-09-12T23:20:51.855 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-rados. 2024-09-12T23:20:51.881 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-rados. 2024-09-12T23:20:51.882 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../005-python3-rados_17.2.7-1jammy_amd64.deb ... 2024-09-12T23:20:51.897 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../005-python3-rados_17.2.7-1jammy_amd64.deb ... 2024-09-12T23:20:51.922 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-rados (17.2.7-1jammy) ... 2024-09-12T23:20:51.940 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-rados (17.2.7-1jammy) ... 2024-09-12T23:20:52.250 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-ceph-argparse. 2024-09-12T23:20:52.260 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-ceph-argparse. 2024-09-12T23:20:52.276 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../006-python3-ceph-argparse_17.2.7-1jammy_all.deb ... 2024-09-12T23:20:52.286 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../006-python3-ceph-argparse_17.2.7-1jammy_all.deb ... 2024-09-12T23:20:52.316 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-ceph-argparse (17.2.7-1jammy) ... 2024-09-12T23:20:52.326 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-ceph-argparse (17.2.7-1jammy) ... 2024-09-12T23:20:52.519 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-cephfs. 2024-09-12T23:20:52.534 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../007-python3-cephfs_17.2.7-1jammy_amd64.deb ... 2024-09-12T23:20:52.569 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-cephfs (17.2.7-1jammy) ... 2024-09-12T23:20:52.579 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-cephfs. 2024-09-12T23:20:52.594 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../007-python3-cephfs_17.2.7-1jammy_amd64.deb ... 2024-09-12T23:20:52.640 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-cephfs (17.2.7-1jammy) ... 2024-09-12T23:20:52.855 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-ceph-common. 2024-09-12T23:20:52.881 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../008-python3-ceph-common_17.2.7-1jammy_all.deb ... 2024-09-12T23:20:52.890 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-ceph-common. 2024-09-12T23:20:52.916 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../008-python3-ceph-common_17.2.7-1jammy_all.deb ... 2024-09-12T23:20:52.921 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-ceph-common (17.2.7-1jammy) ... 2024-09-12T23:20:52.948 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-ceph-common (17.2.7-1jammy) ... 2024-09-12T23:20:53.251 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-wcwidth. 2024-09-12T23:20:53.277 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../009-python3-wcwidth_0.2.5+dfsg1-1_all.deb ... 2024-09-12T23:20:53.293 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-wcwidth. 2024-09-12T23:20:53.317 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-wcwidth (0.2.5+dfsg1-1) ... 2024-09-12T23:20:53.319 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../009-python3-wcwidth_0.2.5+dfsg1-1_all.deb ... 2024-09-12T23:20:53.351 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-wcwidth (0.2.5+dfsg1-1) ... 2024-09-12T23:20:53.612 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-prettytable. 2024-09-12T23:20:53.638 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../010-python3-prettytable_2.5.0-2_all.deb ... 2024-09-12T23:20:53.662 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-prettytable. 2024-09-12T23:20:53.679 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-prettytable (2.5.0-2) ... 2024-09-12T23:20:53.689 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../010-python3-prettytable_2.5.0-2_all.deb ... 2024-09-12T23:20:53.729 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-prettytable (2.5.0-2) ... 2024-09-12T23:20:53.948 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-rbd. 2024-09-12T23:20:53.963 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../011-python3-rbd_17.2.7-1jammy_amd64.deb ... 2024-09-12T23:20:53.990 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-rbd. 2024-09-12T23:20:54.006 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-rbd (17.2.7-1jammy) ... 2024-09-12T23:20:54.016 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../011-python3-rbd_17.2.7-1jammy_amd64.deb ... 2024-09-12T23:20:54.057 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-rbd (17.2.7-1jammy) ... 2024-09-12T23:20:54.351 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package librdkafka1:amd64. 2024-09-12T23:20:54.376 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package librdkafka1:amd64. 2024-09-12T23:20:54.377 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../012-librdkafka1_1.8.0-1build1_amd64.deb ... 2024-09-12T23:20:54.403 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../012-librdkafka1_1.8.0-1build1_amd64.deb ... 2024-09-12T23:20:54.417 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking librdkafka1:amd64 (1.8.0-1build1) ... 2024-09-12T23:20:54.443 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking librdkafka1:amd64 (1.8.0-1build1) ... 2024-09-12T23:20:54.712 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package libdouble-conversion3:amd64. 2024-09-12T23:20:54.737 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package libdouble-conversion3:amd64. 2024-09-12T23:20:54.738 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../013-libdouble-conversion3_3.1.7-4_amd64.deb ... 2024-09-12T23:20:54.764 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../013-libdouble-conversion3_3.1.7-4_amd64.deb ... 2024-09-12T23:20:54.770 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking libdouble-conversion3:amd64 (3.1.7-4) ... 2024-09-12T23:20:54.804 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking libdouble-conversion3:amd64 (3.1.7-4) ... 2024-09-12T23:20:55.149 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package libpcre2-16-0:amd64. 2024-09-12T23:20:55.173 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package libpcre2-16-0:amd64. 2024-09-12T23:20:55.175 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../014-libpcre2-16-0_10.39-3ubuntu0.1_amd64.deb ... 2024-09-12T23:20:55.200 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../014-libpcre2-16-0_10.39-3ubuntu0.1_amd64.deb ... 2024-09-12T23:20:55.215 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2024-09-12T23:20:55.240 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2024-09-12T23:20:55.644 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package libqt5core5a:amd64. 2024-09-12T23:20:55.671 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../015-libqt5core5a_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-09-12T23:20:55.676 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package libqt5core5a:amd64. 2024-09-12T23:20:55.704 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../015-libqt5core5a_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-09-12T23:20:55.719 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-12T23:20:55.760 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-12T23:20:56.089 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package libqt5dbus5:amd64. 2024-09-12T23:20:56.115 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../016-libqt5dbus5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-09-12T23:20:56.155 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package libqt5dbus5:amd64. 2024-09-12T23:20:56.155 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-12T23:20:56.181 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../016-libqt5dbus5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-09-12T23:20:56.221 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-12T23:20:56.509 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package libqt5network5:amd64. 2024-09-12T23:20:56.535 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../017-libqt5network5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-09-12T23:20:56.549 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package libqt5network5:amd64. 2024-09-12T23:20:56.573 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../017-libqt5network5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-09-12T23:20:56.575 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-12T23:20:56.607 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-12T23:20:56.887 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package libthrift-0.16.0:amd64. 2024-09-12T23:20:56.913 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../018-libthrift-0.16.0_0.16.0-2_amd64.deb ... 2024-09-12T23:20:56.944 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package libthrift-0.16.0:amd64. 2024-09-12T23:20:56.945 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking libthrift-0.16.0:amd64 (0.16.0-2) ... 2024-09-12T23:20:56.971 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../018-libthrift-0.16.0_0.16.0-2_amd64.deb ... 2024-09-12T23:20:57.011 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking libthrift-0.16.0:amd64 (0.16.0-2) ... 2024-09-12T23:20:57.240 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package librgw2. 2024-09-12T23:20:57.266 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../019-librgw2_17.2.7-1jammy_amd64.deb ... 2024-09-12T23:20:57.306 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking librgw2 (17.2.7-1jammy) ... 2024-09-12T23:20:57.339 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package librgw2. 2024-09-12T23:20:57.365 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../019-librgw2_17.2.7-1jammy_amd64.deb ... 2024-09-12T23:20:57.405 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking librgw2 (17.2.7-1jammy) ... 2024-09-12T23:20:57.775 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-rgw. 2024-09-12T23:20:57.791 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../020-python3-rgw_17.2.7-1jammy_amd64.deb ... 2024-09-12T23:20:57.834 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-rgw (17.2.7-1jammy) ... 2024-09-12T23:20:57.883 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-rgw. 2024-09-12T23:20:57.909 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../020-python3-rgw_17.2.7-1jammy_amd64.deb ... 2024-09-12T23:20:57.949 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-rgw (17.2.7-1jammy) ... 2024-09-12T23:20:58.170 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package liboath0:amd64. 2024-09-12T23:20:58.196 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../021-liboath0_2.6.7-3build1_amd64.deb ... 2024-09-12T23:20:58.236 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking liboath0:amd64 (2.6.7-3build1) ... 2024-09-12T23:20:58.244 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package liboath0:amd64. 2024-09-12T23:20:58.271 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../021-liboath0_2.6.7-3build1_amd64.deb ... 2024-09-12T23:20:58.310 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking liboath0:amd64 (2.6.7-3build1) ... 2024-09-12T23:20:58.556 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package libradosstriper1. 2024-09-12T23:20:58.583 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../022-libradosstriper1_17.2.7-1jammy_amd64.deb ... 2024-09-12T23:20:58.622 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package libradosstriper1. 2024-09-12T23:20:58.622 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking libradosstriper1 (17.2.7-1jammy) ... 2024-09-12T23:20:58.648 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../022-libradosstriper1_17.2.7-1jammy_amd64.deb ... 2024-09-12T23:20:58.689 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking libradosstriper1 (17.2.7-1jammy) ... 2024-09-12T23:20:59.060 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package ceph-common. 2024-09-12T23:20:59.086 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../023-ceph-common_17.2.7-1jammy_amd64.deb ... 2024-09-12T23:20:59.108 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package ceph-common. 2024-09-12T23:20:59.126 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking ceph-common (17.2.7-1jammy) ... 2024-09-12T23:20:59.134 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../023-ceph-common_17.2.7-1jammy_amd64.deb ... 2024-09-12T23:20:59.174 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking ceph-common (17.2.7-1jammy) ... 2024-09-12T23:21:00.482 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package ceph-base. 2024-09-12T23:21:00.508 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../024-ceph-base_17.2.7-1jammy_amd64.deb ... 2024-09-12T23:21:00.547 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package ceph-base. 2024-09-12T23:21:00.558 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking ceph-base (17.2.7-1jammy) ... 2024-09-12T23:21:00.573 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../024-ceph-base_17.2.7-1jammy_amd64.deb ... 2024-09-12T23:21:00.625 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking ceph-base (17.2.7-1jammy) ... 2024-09-12T23:21:01.169 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-openssl. 2024-09-12T23:21:01.196 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../025-python3-openssl_21.0.0-1_all.deb ... 2024-09-12T23:21:01.237 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-openssl (21.0.0-1) ... 2024-09-12T23:21:01.276 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-openssl. 2024-09-12T23:21:01.302 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../025-python3-openssl_21.0.0-1_all.deb ... 2024-09-12T23:21:01.342 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-openssl (21.0.0-1) ... 2024-09-12T23:21:01.505 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2024-09-12T23:21:01.531 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../026-ceph-mgr-modules-core_17.2.7-1jammy_all.deb ... 2024-09-12T23:21:01.572 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking ceph-mgr-modules-core (17.2.7-1jammy) ... 2024-09-12T23:21:01.596 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2024-09-12T23:21:01.622 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../026-ceph-mgr-modules-core_17.2.7-1jammy_all.deb ... 2024-09-12T23:21:01.670 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking ceph-mgr-modules-core (17.2.7-1jammy) ... 2024-09-12T23:21:01.985 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-bcrypt. 2024-09-12T23:21:02.012 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../027-python3-bcrypt_3.2.0-1build1_amd64.deb ... 2024-09-12T23:21:02.042 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-bcrypt. 2024-09-12T23:21:02.051 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-bcrypt (3.2.0-1build1) ... 2024-09-12T23:21:02.068 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../027-python3-bcrypt_3.2.0-1build1_amd64.deb ... 2024-09-12T23:21:02.108 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-bcrypt (3.2.0-1build1) ... 2024-09-12T23:21:02.354 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-jaraco.functools. 2024-09-12T23:21:02.381 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../028-python3-jaraco.functools_3.4.0-2_all.deb ... 2024-09-12T23:21:02.386 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-jaraco.functools. 2024-09-12T23:21:02.402 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../028-python3-jaraco.functools_3.4.0-2_all.deb ... 2024-09-12T23:21:02.421 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-jaraco.functools (3.4.0-2) ... 2024-09-12T23:21:02.436 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-jaraco.functools (3.4.0-2) ... 2024-09-12T23:21:02.672 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-cheroot. 2024-09-12T23:21:02.699 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../029-python3-cheroot_8.5.2+ds1-1ubuntu3.1_all.deb ... 2024-09-12T23:21:02.715 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-cheroot. 2024-09-12T23:21:02.731 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2024-09-12T23:21:02.742 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../029-python3-cheroot_8.5.2+ds1-1ubuntu3.1_all.deb ... 2024-09-12T23:21:02.782 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2024-09-12T23:21:03.017 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-jaraco.classes. 2024-09-12T23:21:03.043 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../030-python3-jaraco.classes_3.2.1-3_all.deb ... 2024-09-12T23:21:03.075 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-jaraco.classes (3.2.1-3) ... 2024-09-12T23:21:03.202 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-jaraco.classes. 2024-09-12T23:21:03.228 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../030-python3-jaraco.classes_3.2.1-3_all.deb ... 2024-09-12T23:21:03.355 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-jaraco.text. 2024-09-12T23:21:03.381 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../031-python3-jaraco.text_3.6.0-2_all.deb ... 2024-09-12T23:21:03.385 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-jaraco.classes (3.2.1-3) ... 2024-09-12T23:21:03.421 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-jaraco.text (3.6.0-2) ... 2024-09-12T23:21:03.721 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-jaraco.text. 2024-09-12T23:21:03.724 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-jaraco.collections. 2024-09-12T23:21:03.748 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../031-python3-jaraco.text_3.6.0-2_all.deb ... 2024-09-12T23:21:03.751 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../032-python3-jaraco.collections_3.4.0-2_all.deb ... 2024-09-12T23:21:03.788 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-jaraco.text (3.6.0-2) ... 2024-09-12T23:21:03.791 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-jaraco.collections (3.4.0-2) ... 2024-09-12T23:21:04.091 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-jaraco.collections. 2024-09-12T23:21:04.117 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../032-python3-jaraco.collections_3.4.0-2_all.deb ... 2024-09-12T23:21:04.157 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-jaraco.collections (3.4.0-2) ... 2024-09-12T23:21:04.327 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-tempora. 2024-09-12T23:21:04.353 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../033-python3-tempora_4.1.2-1_all.deb ... 2024-09-12T23:21:04.385 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-tempora (4.1.2-1) ... 2024-09-12T23:21:04.427 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-tempora. 2024-09-12T23:21:04.453 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../033-python3-tempora_4.1.2-1_all.deb ... 2024-09-12T23:21:04.485 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-tempora (4.1.2-1) ... 2024-09-12T23:21:04.688 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-portend. 2024-09-12T23:21:04.714 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../034-python3-portend_3.0.0-1_all.deb ... 2024-09-12T23:21:04.755 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-portend (3.0.0-1) ... 2024-09-12T23:21:04.788 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-portend. 2024-09-12T23:21:04.814 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../034-python3-portend_3.0.0-1_all.deb ... 2024-09-12T23:21:04.855 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-portend (3.0.0-1) ... 2024-09-12T23:21:05.057 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-zc.lockfile. 2024-09-12T23:21:05.084 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../035-python3-zc.lockfile_2.0-1_all.deb ... 2024-09-12T23:21:05.124 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-zc.lockfile (2.0-1) ... 2024-09-12T23:21:05.157 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-zc.lockfile. 2024-09-12T23:21:05.172 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../035-python3-zc.lockfile_2.0-1_all.deb ... 2024-09-12T23:21:05.215 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-zc.lockfile (2.0-1) ... 2024-09-12T23:21:05.410 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-cherrypy3. 2024-09-12T23:21:05.436 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../036-python3-cherrypy3_18.6.1-4_all.deb ... 2024-09-12T23:21:05.477 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-cherrypy3 (18.6.1-4) ... 2024-09-12T23:21:05.485 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-cherrypy3. 2024-09-12T23:21:05.500 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../036-python3-cherrypy3_18.6.1-4_all.deb ... 2024-09-12T23:21:05.535 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-cherrypy3 (18.6.1-4) ... 2024-09-12T23:21:05.822 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-natsort. 2024-09-12T23:21:05.848 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../037-python3-natsort_8.0.2-1_all.deb ... 2024-09-12T23:21:05.872 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-natsort. 2024-09-12T23:21:05.888 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-natsort (8.0.2-1) ... 2024-09-12T23:21:05.898 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../037-python3-natsort_8.0.2-1_all.deb ... 2024-09-12T23:21:05.940 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-natsort (8.0.2-1) ... 2024-09-12T23:21:06.200 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-logutils. 2024-09-12T23:21:06.226 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../038-python3-logutils_0.3.3-8_all.deb ... 2024-09-12T23:21:06.250 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-logutils. 2024-09-12T23:21:06.266 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-logutils (0.3.3-8) ... 2024-09-12T23:21:06.276 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../038-python3-logutils_0.3.3-8_all.deb ... 2024-09-12T23:21:06.317 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-logutils (0.3.3-8) ... 2024-09-12T23:21:06.552 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-markupsafe. 2024-09-12T23:21:06.579 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../039-python3-markupsafe_2.0.1-2build1_amd64.deb ... 2024-09-12T23:21:06.610 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-markupsafe (2.0.1-2build1) ... 2024-09-12T23:21:06.661 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-markupsafe. 2024-09-12T23:21:06.687 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../039-python3-markupsafe_2.0.1-2build1_amd64.deb ... 2024-09-12T23:21:06.736 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-markupsafe (2.0.1-2build1) ... 2024-09-12T23:21:06.905 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-mako. 2024-09-12T23:21:06.931 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../040-python3-mako_1.1.3+ds1-2ubuntu0.1_all.deb ... 2024-09-12T23:21:06.972 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-mako (1.1.3+ds1-2ubuntu0.1) ... 2024-09-12T23:21:07.039 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-mako. 2024-09-12T23:21:07.065 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../040-python3-mako_1.1.3+ds1-2ubuntu0.1_all.deb ... 2024-09-12T23:21:07.105 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-mako (1.1.3+ds1-2ubuntu0.1) ... 2024-09-12T23:21:07.291 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-simplegeneric. 2024-09-12T23:21:07.318 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../041-python3-simplegeneric_0.8.1-3_all.deb ... 2024-09-12T23:21:07.358 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-simplegeneric (0.8.1-3) ... 2024-09-12T23:21:07.476 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-simplegeneric. 2024-09-12T23:21:07.502 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../041-python3-simplegeneric_0.8.1-3_all.deb ... 2024-09-12T23:21:07.542 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-simplegeneric (0.8.1-3) ... 2024-09-12T23:21:07.661 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-singledispatch. 2024-09-12T23:21:07.687 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../042-python3-singledispatch_3.4.0.3-3_all.deb ... 2024-09-12T23:21:07.727 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-singledispatch (3.4.0.3-3) ... 2024-09-12T23:21:07.845 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-singledispatch. 2024-09-12T23:21:07.871 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../042-python3-singledispatch_3.4.0.3-3_all.deb ... 2024-09-12T23:21:07.911 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-singledispatch (3.4.0.3-3) ... 2024-09-12T23:21:08.022 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-webob. 2024-09-12T23:21:08.048 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../043-python3-webob_1%3a1.8.6-1.1ubuntu0.1_all.deb ... 2024-09-12T23:21:08.080 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-webob (1:1.8.6-1.1ubuntu0.1) ... 2024-09-12T23:21:08.256 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-webob. 2024-09-12T23:21:08.282 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../043-python3-webob_1%3a1.8.6-1.1ubuntu0.1_all.deb ... 2024-09-12T23:21:08.322 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-webob (1:1.8.6-1.1ubuntu0.1) ... 2024-09-12T23:21:08.391 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-waitress. 2024-09-12T23:21:08.406 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../044-python3-waitress_1.4.4-1.1ubuntu1_all.deb ... 2024-09-12T23:21:08.444 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-waitress (1.4.4-1.1ubuntu1) ... 2024-09-12T23:21:08.727 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-tempita. 2024-09-12T23:21:08.754 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../045-python3-tempita_0.5.2-6ubuntu1_all.deb ... 2024-09-12T23:21:08.794 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-tempita (0.5.2-6ubuntu1) ... 2024-09-12T23:21:08.809 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-waitress. 2024-09-12T23:21:08.836 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../044-python3-waitress_1.4.4-1.1ubuntu1_all.deb ... 2024-09-12T23:21:08.880 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-waitress (1.4.4-1.1ubuntu1) ... 2024-09-12T23:21:09.097 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-paste. 2024-09-12T23:21:09.123 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../046-python3-paste_3.5.0+dfsg1-1_all.deb ... 2024-09-12T23:21:09.163 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-paste (3.5.0+dfsg1-1) ... 2024-09-12T23:21:09.196 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-tempita. 2024-09-12T23:21:09.222 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../045-python3-tempita_0.5.2-6ubuntu1_all.deb ... 2024-09-12T23:21:09.262 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-tempita (0.5.2-6ubuntu1) ... 2024-09-12T23:21:09.565 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-paste. 2024-09-12T23:21:09.591 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../046-python3-paste_3.5.0+dfsg1-1_all.deb ... 2024-09-12T23:21:09.631 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-paste (3.5.0+dfsg1-1) ... 2024-09-12T23:21:09.683 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python-pastedeploy-tpl. 2024-09-12T23:21:09.700 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../047-python-pastedeploy-tpl_2.1.1-1_all.deb ... 2024-09-12T23:21:09.741 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python-pastedeploy-tpl (2.1.1-1) ... 2024-09-12T23:21:09.993 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python-pastedeploy-tpl. 2024-09-12T23:21:10.020 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../047-python-pastedeploy-tpl_2.1.1-1_all.deb ... 2024-09-12T23:21:10.044 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-pastedeploy. 2024-09-12T23:21:10.060 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python-pastedeploy-tpl (2.1.1-1) ... 2024-09-12T23:21:10.071 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../048-python3-pastedeploy_2.1.1-1_all.deb ... 2024-09-12T23:21:10.111 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-pastedeploy (2.1.1-1) ... 2024-09-12T23:21:10.329 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-pastedeploy. 2024-09-12T23:21:10.356 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../048-python3-pastedeploy_2.1.1-1_all.deb ... 2024-09-12T23:21:10.372 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-webtest. 2024-09-12T23:21:10.385 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../049-python3-webtest_2.0.35-1_all.deb ... 2024-09-12T23:21:10.396 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-pastedeploy (2.1.1-1) ... 2024-09-12T23:21:10.413 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-webtest (2.0.35-1) ... 2024-09-12T23:21:10.692 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-pecan. 2024-09-12T23:21:10.699 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-webtest. 2024-09-12T23:21:10.718 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../050-python3-pecan_1.3.3-4ubuntu2_all.deb ... 2024-09-12T23:21:10.725 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../049-python3-webtest_2.0.35-1_all.deb ... 2024-09-12T23:21:10.758 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-pecan (1.3.3-4ubuntu2) ... 2024-09-12T23:21:10.765 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-webtest (2.0.35-1) ... 2024-09-12T23:21:11.077 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-pecan. 2024-09-12T23:21:11.103 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../050-python3-pecan_1.3.3-4ubuntu2_all.deb ... 2024-09-12T23:21:11.129 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-certifi. 2024-09-12T23:21:11.143 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-pecan (1.3.3-4ubuntu2) ... 2024-09-12T23:21:11.156 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../051-python3-certifi_2020.6.20-1_all.deb ... 2024-09-12T23:21:11.204 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-certifi (2020.6.20-1) ... 2024-09-12T23:21:11.498 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-certifi. 2024-09-12T23:21:11.524 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-idna. 2024-09-12T23:21:11.525 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../051-python3-certifi_2020.6.20-1_all.deb ... 2024-09-12T23:21:11.551 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../052-python3-idna_3.3-1ubuntu0.1_all.deb ... 2024-09-12T23:21:11.573 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-certifi (2020.6.20-1) ... 2024-09-12T23:21:11.590 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-idna (3.3-1ubuntu0.1) ... 2024-09-12T23:21:11.851 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-idna. 2024-09-12T23:21:11.876 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-urllib3. 2024-09-12T23:21:11.878 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../052-python3-idna_3.3-1ubuntu0.1_all.deb ... 2024-09-12T23:21:11.904 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../053-python3-urllib3_1.26.5-1~exp1ubuntu0.1_all.deb ... 2024-09-12T23:21:11.909 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-idna (3.3-1ubuntu0.1) ... 2024-09-12T23:21:11.943 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-urllib3 (1.26.5-1~exp1ubuntu0.1) ... 2024-09-12T23:21:12.204 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-urllib3. 2024-09-12T23:21:12.229 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-requests. 2024-09-12T23:21:12.230 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../053-python3-urllib3_1.26.5-1~exp1ubuntu0.1_all.deb ... 2024-09-12T23:21:12.245 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../054-python3-requests_2.25.1+dfsg-2ubuntu0.1_all.deb ... 2024-09-12T23:21:12.270 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-urllib3 (1.26.5-1~exp1ubuntu0.1) ... 2024-09-12T23:21:12.288 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-requests (2.25.1+dfsg-2ubuntu0.1) ... 2024-09-12T23:21:12.532 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-werkzeug. 2024-09-12T23:21:12.548 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../055-python3-werkzeug_2.0.2+dfsg1-1ubuntu0.22.04.2_all.deb ... 2024-09-12T23:21:12.582 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-werkzeug (2.0.2+dfsg1-1ubuntu0.22.04.2) ... 2024-09-12T23:21:12.590 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-requests. 2024-09-12T23:21:12.617 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../054-python3-requests_2.25.1+dfsg-2ubuntu0.1_all.deb ... 2024-09-12T23:21:12.656 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-requests (2.25.1+dfsg-2ubuntu0.1) ... 2024-09-12T23:21:12.877 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2024-09-12T23:21:12.904 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../056-libsqlite3-mod-ceph_17.2.7-1jammy_amd64.deb ... 2024-09-12T23:21:12.943 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking libsqlite3-mod-ceph (17.2.7-1jammy) ... 2024-09-12T23:21:12.976 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-werkzeug. 2024-09-12T23:21:13.003 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../055-python3-werkzeug_2.0.2+dfsg1-1ubuntu0.22.04.2_all.deb ... 2024-09-12T23:21:13.043 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-werkzeug (2.0.2+dfsg1-1ubuntu0.22.04.2) ... 2024-09-12T23:21:13.254 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package ceph-mgr. 2024-09-12T23:21:13.282 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../057-ceph-mgr_17.2.7-1jammy_amd64.deb ... 2024-09-12T23:21:13.330 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking ceph-mgr (17.2.7-1jammy) ... 2024-09-12T23:21:13.380 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2024-09-12T23:21:13.406 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../056-libsqlite3-mod-ceph_17.2.7-1jammy_amd64.deb ... 2024-09-12T23:21:13.446 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking libsqlite3-mod-ceph (17.2.7-1jammy) ... 2024-09-12T23:21:13.733 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package ceph-mon. 2024-09-12T23:21:13.760 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../058-ceph-mon_17.2.7-1jammy_amd64.deb ... 2024-09-12T23:21:13.791 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package ceph-mgr. 2024-09-12T23:21:13.799 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking ceph-mon (17.2.7-1jammy) ... 2024-09-12T23:21:13.818 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../057-ceph-mgr_17.2.7-1jammy_amd64.deb ... 2024-09-12T23:21:13.857 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking ceph-mgr (17.2.7-1jammy) ... 2024-09-12T23:21:14.203 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package ceph-mon. 2024-09-12T23:21:14.229 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../058-ceph-mon_17.2.7-1jammy_amd64.deb ... 2024-09-12T23:21:14.260 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking ceph-mon (17.2.7-1jammy) ... 2024-09-12T23:21:14.669 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package ceph-osd. 2024-09-12T23:21:14.696 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../059-ceph-osd_17.2.7-1jammy_amd64.deb ... 2024-09-12T23:21:14.736 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking ceph-osd (17.2.7-1jammy) ... 2024-09-12T23:21:15.081 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package ceph-osd. 2024-09-12T23:21:15.107 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../059-ceph-osd_17.2.7-1jammy_amd64.deb ... 2024-09-12T23:21:15.147 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking ceph-osd (17.2.7-1jammy) ... 2024-09-12T23:21:16.322 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package ceph. 2024-09-12T23:21:16.338 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../060-ceph_17.2.7-1jammy_amd64.deb ... 2024-09-12T23:21:16.372 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking ceph (17.2.7-1jammy) ... 2024-09-12T23:21:16.658 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package ceph-fuse. 2024-09-12T23:21:16.685 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../061-ceph-fuse_17.2.7-1jammy_amd64.deb ... 2024-09-12T23:21:16.725 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking ceph-fuse (17.2.7-1jammy) ... 2024-09-12T23:21:16.767 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package ceph. 2024-09-12T23:21:16.784 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../060-ceph_17.2.7-1jammy_amd64.deb ... 2024-09-12T23:21:16.825 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking ceph (17.2.7-1jammy) ... 2024-09-12T23:21:17.095 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package ceph-mds. 2024-09-12T23:21:17.095 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package ceph-fuse. 2024-09-12T23:21:17.121 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../062-ceph-mds_17.2.7-1jammy_amd64.deb ... 2024-09-12T23:21:17.121 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../061-ceph-fuse_17.2.7-1jammy_amd64.deb ... 2024-09-12T23:21:17.153 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking ceph-fuse (17.2.7-1jammy) ... 2024-09-12T23:21:17.161 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking ceph-mds (17.2.7-1jammy) ... 2024-09-12T23:21:17.506 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package ceph-mds. 2024-09-12T23:21:17.533 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../062-ceph-mds_17.2.7-1jammy_amd64.deb ... 2024-09-12T23:21:17.539 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package cephadm. 2024-09-12T23:21:17.566 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../063-cephadm_17.2.7-1jammy_amd64.deb ... 2024-09-12T23:21:17.572 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking ceph-mds (17.2.7-1jammy) ... 2024-09-12T23:21:17.606 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking cephadm (17.2.7-1jammy) ... 2024-09-12T23:21:17.833 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python-babel-localedata. 2024-09-12T23:21:17.848 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../064-python-babel-localedata_2.8.0+dfsg.1-7_all.deb ... 2024-09-12T23:21:17.891 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python-babel-localedata (2.8.0+dfsg.1-7) ... 2024-09-12T23:21:17.959 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package cephadm. 2024-09-12T23:21:17.986 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../063-cephadm_17.2.7-1jammy_amd64.deb ... 2024-09-12T23:21:18.026 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking cephadm (17.2.7-1jammy) ... 2024-09-12T23:21:18.262 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python-babel-localedata. 2024-09-12T23:21:18.277 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../064-python-babel-localedata_2.8.0+dfsg.1-7_all.deb ... 2024-09-12T23:21:18.312 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python-babel-localedata (2.8.0+dfsg.1-7) ... 2024-09-12T23:21:19.606 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-babel. 2024-09-12T23:21:19.633 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../065-python3-babel_2.8.0+dfsg.1-7_all.deb ... 2024-09-12T23:21:19.664 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-babel (2.8.0+dfsg.1-7) ... 2024-09-12T23:21:20.208 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-jinja2. 2024-09-12T23:21:20.224 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../066-python3-jinja2_3.0.3-1ubuntu0.2_all.deb ... 2024-09-12T23:21:20.250 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-babel. 2024-09-12T23:21:20.258 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-jinja2 (3.0.3-1ubuntu0.2) ... 2024-09-12T23:21:20.277 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../065-python3-babel_2.8.0+dfsg.1-7_all.deb ... 2024-09-12T23:21:20.317 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-babel (2.8.0+dfsg.1-7) ... 2024-09-12T23:21:20.536 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2024-09-12T23:21:20.563 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../067-ceph-mgr-cephadm_17.2.7-1jammy_all.deb ... 2024-09-12T23:21:20.594 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking ceph-mgr-cephadm (17.2.7-1jammy) ... 2024-09-12T23:21:20.637 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-jinja2. 2024-09-12T23:21:20.663 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../066-python3-jinja2_3.0.3-1ubuntu0.2_all.deb ... 2024-09-12T23:21:20.703 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-jinja2 (3.0.3-1ubuntu0.2) ... 2024-09-12T23:21:20.923 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-repoze.lru. 2024-09-12T23:21:20.950 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../068-python3-repoze.lru_0.7-2_all.deb ... 2024-09-12T23:21:20.990 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2024-09-12T23:21:21.040 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2024-09-12T23:21:21.066 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../067-ceph-mgr-cephadm_17.2.7-1jammy_all.deb ... 2024-09-12T23:21:21.106 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking ceph-mgr-cephadm (17.2.7-1jammy) ... 2024-09-12T23:21:21.292 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-routes. 2024-09-12T23:21:21.319 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../069-python3-routes_2.5.1-1ubuntu1_all.deb ... 2024-09-12T23:21:21.359 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-routes (2.5.1-1ubuntu1) ... 2024-09-12T23:21:21.452 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-repoze.lru. 2024-09-12T23:21:21.478 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../068-python3-repoze.lru_0.7-2_all.deb ... 2024-09-12T23:21:21.518 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2024-09-12T23:21:21.679 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2024-09-12T23:21:21.705 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../070-ceph-mgr-dashboard_17.2.7-1jammy_all.deb ... 2024-09-12T23:21:21.745 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking ceph-mgr-dashboard (17.2.7-1jammy) ... 2024-09-12T23:21:21.796 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-routes. 2024-09-12T23:21:21.823 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../069-python3-routes_2.5.1-1ubuntu1_all.deb ... 2024-09-12T23:21:21.854 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-routes (2.5.1-1ubuntu1) ... 2024-09-12T23:21:22.140 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-sklearn-lib:amd64. 2024-09-12T23:21:22.164 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../071-python3-sklearn-lib_0.23.2-5ubuntu6_amd64.deb ... 2024-09-12T23:21:22.191 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2024-09-12T23:21:22.198 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2024-09-12T23:21:22.218 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../070-ceph-mgr-dashboard_17.2.7-1jammy_all.deb ... 2024-09-12T23:21:22.257 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking ceph-mgr-dashboard (17.2.7-1jammy) ... 2024-09-12T23:21:22.661 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-joblib. 2024-09-12T23:21:22.678 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-sklearn-lib:amd64. 2024-09-12T23:21:22.688 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../072-python3-joblib_0.17.0-4ubuntu1_all.deb ... 2024-09-12T23:21:22.705 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../071-python3-sklearn-lib_0.23.2-5ubuntu6_amd64.deb ... 2024-09-12T23:21:22.727 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-joblib (0.17.0-4ubuntu1) ... 2024-09-12T23:21:22.744 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2024-09-12T23:21:23.072 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-threadpoolctl. 2024-09-12T23:21:23.100 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../073-python3-threadpoolctl_3.1.0-1_all.deb ... 2024-09-12T23:21:23.130 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-threadpoolctl (3.1.0-1) ... 2024-09-12T23:21:23.239 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-joblib. 2024-09-12T23:21:23.266 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../072-python3-joblib_0.17.0-4ubuntu1_all.deb ... 2024-09-12T23:21:23.306 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-joblib (0.17.0-4ubuntu1) ... 2024-09-12T23:21:23.417 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-sklearn. 2024-09-12T23:21:23.444 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../074-python3-sklearn_0.23.2-5ubuntu6_all.deb ... 2024-09-12T23:21:23.483 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-sklearn (0.23.2-5ubuntu6) ... 2024-09-12T23:21:23.669 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-threadpoolctl. 2024-09-12T23:21:23.695 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../073-python3-threadpoolctl_3.1.0-1_all.deb ... 2024-09-12T23:21:23.735 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-threadpoolctl (3.1.0-1) ... 2024-09-12T23:21:24.038 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-sklearn. 2024-09-12T23:21:24.064 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../074-python3-sklearn_0.23.2-5ubuntu6_all.deb ... 2024-09-12T23:21:24.112 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-sklearn (0.23.2-5ubuntu6) ... 2024-09-12T23:21:24.158 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2024-09-12T23:21:24.186 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../075-ceph-mgr-diskprediction-local_17.2.7-1jammy_all.deb ... 2024-09-12T23:21:24.225 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking ceph-mgr-diskprediction-local (17.2.7-1jammy) ... 2024-09-12T23:21:24.888 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2024-09-12T23:21:24.915 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../075-ceph-mgr-diskprediction-local_17.2.7-1jammy_all.deb ... 2024-09-12T23:21:24.954 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking ceph-mgr-diskprediction-local (17.2.7-1jammy) ... 2024-09-12T23:21:25.203 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-cachetools. 2024-09-12T23:21:25.230 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../076-python3-cachetools_5.0.0-1_all.deb ... 2024-09-12T23:21:25.269 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-cachetools (5.0.0-1) ... 2024-09-12T23:21:25.572 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-pyasn1. 2024-09-12T23:21:25.600 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../077-python3-pyasn1_0.4.8-1_all.deb ... 2024-09-12T23:21:25.639 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-pyasn1 (0.4.8-1) ... 2024-09-12T23:21:25.882 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-cachetools. 2024-09-12T23:21:25.909 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../076-python3-cachetools_5.0.0-1_all.deb ... 2024-09-12T23:21:25.949 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-cachetools (5.0.0-1) ... 2024-09-12T23:21:25.959 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-pyasn1-modules. 2024-09-12T23:21:25.986 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../078-python3-pyasn1-modules_0.2.1-1_all.deb ... 2024-09-12T23:21:26.025 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-pyasn1-modules (0.2.1-1) ... 2024-09-12T23:21:26.252 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-pyasn1. 2024-09-12T23:21:26.279 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../077-python3-pyasn1_0.4.8-1_all.deb ... 2024-09-12T23:21:26.318 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-pyasn1 (0.4.8-1) ... 2024-09-12T23:21:26.336 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-rsa. 2024-09-12T23:21:26.363 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../079-python3-rsa_4.8-1_all.deb ... 2024-09-12T23:21:26.403 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-rsa (4.8-1) ... 2024-09-12T23:21:26.647 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-pyasn1-modules. 2024-09-12T23:21:26.674 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../078-python3-pyasn1-modules_0.2.1-1_all.deb ... 2024-09-12T23:21:26.707 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-google-auth. 2024-09-12T23:21:26.713 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-pyasn1-modules (0.2.1-1) ... 2024-09-12T23:21:26.734 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../080-python3-google-auth_1.5.1-3_all.deb ... 2024-09-12T23:21:26.773 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-google-auth (1.5.1-3) ... 2024-09-12T23:21:27.025 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-rsa. 2024-09-12T23:21:27.052 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../079-python3-rsa_4.8-1_all.deb ... 2024-09-12T23:21:27.091 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-rsa (4.8-1) ... 2024-09-12T23:21:27.093 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-requests-oauthlib. 2024-09-12T23:21:27.120 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../081-python3-requests-oauthlib_1.3.0+ds-0.1_all.deb ... 2024-09-12T23:21:27.159 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-requests-oauthlib (1.3.0+ds-0.1) ... 2024-09-12T23:21:27.361 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-google-auth. 2024-09-12T23:21:27.377 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../080-python3-google-auth_1.5.1-3_all.deb ... 2024-09-12T23:21:27.411 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-google-auth (1.5.1-3) ... 2024-09-12T23:21:27.453 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-websocket. 2024-09-12T23:21:27.481 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../082-python3-websocket_1.2.3-1_all.deb ... 2024-09-12T23:21:27.520 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-websocket (1.2.3-1) ... 2024-09-12T23:21:27.706 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-requests-oauthlib. 2024-09-12T23:21:27.733 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../081-python3-requests-oauthlib_1.3.0+ds-0.1_all.deb ... 2024-09-12T23:21:27.764 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-requests-oauthlib (1.3.0+ds-0.1) ... 2024-09-12T23:21:27.908 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-kubernetes. 2024-09-12T23:21:27.936 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../083-python3-kubernetes_12.0.1-1ubuntu1_all.deb ... 2024-09-12T23:21:27.974 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-kubernetes (12.0.1-1ubuntu1) ... 2024-09-12T23:21:28.050 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-websocket. 2024-09-12T23:21:28.077 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../082-python3-websocket_1.2.3-1_all.deb ... 2024-09-12T23:21:28.117 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-websocket (1.2.3-1) ... 2024-09-12T23:21:28.522 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-kubernetes. 2024-09-12T23:21:28.549 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../083-python3-kubernetes_12.0.1-1ubuntu1_all.deb ... 2024-09-12T23:21:28.588 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-kubernetes (12.0.1-1ubuntu1) ... 2024-09-12T23:21:28.647 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2024-09-12T23:21:28.663 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../084-ceph-mgr-k8sevents_17.2.7-1jammy_all.deb ... 2024-09-12T23:21:28.705 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking ceph-mgr-k8sevents (17.2.7-1jammy) ... 2024-09-12T23:21:29.024 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package libonig5:amd64. 2024-09-12T23:21:29.052 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../085-libonig5_6.9.7.1-2build1_amd64.deb ... 2024-09-12T23:21:29.091 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking libonig5:amd64 (6.9.7.1-2build1) ... 2024-09-12T23:21:29.360 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package libjq1:amd64. 2024-09-12T23:21:29.376 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../086-libjq1_1.6-2.1ubuntu3_amd64.deb ... 2024-09-12T23:21:29.410 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking libjq1:amd64 (1.6-2.1ubuntu3) ... 2024-09-12T23:21:29.435 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2024-09-12T23:21:29.462 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../084-ceph-mgr-k8sevents_17.2.7-1jammy_all.deb ... 2024-09-12T23:21:29.502 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking ceph-mgr-k8sevents (17.2.7-1jammy) ... 2024-09-12T23:21:29.671 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package jq. 2024-09-12T23:21:29.688 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../087-jq_1.6-2.1ubuntu3_amd64.deb ... 2024-09-12T23:21:29.729 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking jq (1.6-2.1ubuntu3) ... 2024-09-12T23:21:29.755 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package libonig5:amd64. 2024-09-12T23:21:29.783 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../085-libonig5_6.9.7.1-2build1_amd64.deb ... 2024-09-12T23:21:29.821 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking libonig5:amd64 (6.9.7.1-2build1) ... 2024-09-12T23:21:29.982 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package socat. 2024-09-12T23:21:29.998 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../088-socat_1.7.4.1-3ubuntu4_amd64.deb ... 2024-09-12T23:21:30.040 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking socat (1.7.4.1-3ubuntu4) ... 2024-09-12T23:21:30.158 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package libjq1:amd64. 2024-09-12T23:21:30.185 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../086-libjq1_1.6-2.1ubuntu3_amd64.deb ... 2024-09-12T23:21:30.233 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking libjq1:amd64 (1.6-2.1ubuntu3) ... 2024-09-12T23:21:30.494 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package jq. 2024-09-12T23:21:30.510 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../087-jq_1.6-2.1ubuntu3_amd64.deb ... 2024-09-12T23:21:30.535 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package xmlstarlet. 2024-09-12T23:21:30.552 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking jq (1.6-2.1ubuntu3) ... 2024-09-12T23:21:30.562 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../089-xmlstarlet_1.6.1-2.1_amd64.deb ... 2024-09-12T23:21:30.601 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking xmlstarlet (1.6.1-2.1) ... 2024-09-12T23:21:30.779 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package socat. 2024-09-12T23:21:30.795 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../088-socat_1.7.4.1-3ubuntu4_amd64.deb ... 2024-09-12T23:21:30.829 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking socat (1.7.4.1-3ubuntu4) ... 2024-09-12T23:21:30.939 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package ceph-test. 2024-09-12T23:21:30.955 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../090-ceph-test_17.2.7-1jammy_amd64.deb ... 2024-09-12T23:21:30.989 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking ceph-test (17.2.7-1jammy) ... 2024-09-12T23:21:31.124 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package xmlstarlet. 2024-09-12T23:21:31.152 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../089-xmlstarlet_1.6.1-2.1_amd64.deb ... 2024-09-12T23:21:31.191 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking xmlstarlet (1.6.1-2.1) ... 2024-09-12T23:21:31.554 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package ceph-test. 2024-09-12T23:21:31.581 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../090-ceph-test_17.2.7-1jammy_amd64.deb ... 2024-09-12T23:21:31.612 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking ceph-test (17.2.7-1jammy) ... 2024-09-12T23:21:33.334 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package ceph-volume. 2024-09-12T23:21:33.350 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../091-ceph-volume_17.2.7-1jammy_all.deb ... 2024-09-12T23:21:33.384 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking ceph-volume (17.2.7-1jammy) ... 2024-09-12T23:21:33.714 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package libcephfs-dev. 2024-09-12T23:21:33.742 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../092-libcephfs-dev_17.2.7-1jammy_amd64.deb ... 2024-09-12T23:21:33.780 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking libcephfs-dev (17.2.7-1jammy) ... 2024-09-12T23:21:34.015 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package ceph-volume. 2024-09-12T23:21:34.031 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../091-ceph-volume_17.2.7-1jammy_all.deb ... 2024-09-12T23:21:34.065 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking ceph-volume (17.2.7-1jammy) ... 2024-09-12T23:21:34.159 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package nvme-cli. 2024-09-12T23:21:34.186 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../093-nvme-cli_1.16-3ubuntu0.3_amd64.deb ... 2024-09-12T23:21:34.225 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking nvme-cli (1.16-3ubuntu0.3) ... 2024-09-12T23:21:34.353 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package libcephfs-dev. 2024-09-12T23:21:34.369 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../092-libcephfs-dev_17.2.7-1jammy_amd64.deb ... 2024-09-12T23:21:34.412 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking libcephfs-dev (17.2.7-1jammy) ... 2024-09-12T23:21:34.639 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-iniconfig. 2024-09-12T23:21:34.667 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../094-python3-iniconfig_1.1.1-2_all.deb ... 2024-09-12T23:21:34.705 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-iniconfig (1.1.1-2) ... 2024-09-12T23:21:34.757 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package nvme-cli. 2024-09-12T23:21:34.784 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../093-nvme-cli_1.16-3ubuntu0.3_amd64.deb ... 2024-09-12T23:21:34.815 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking nvme-cli (1.16-3ubuntu0.3) ... 2024-09-12T23:21:35.008 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-pastescript. 2024-09-12T23:21:35.037 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../095-python3-pastescript_2.0.2-4_all.deb ... 2024-09-12T23:21:35.066 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-pastescript (2.0.2-4) ... 2024-09-12T23:21:35.214 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-iniconfig. 2024-09-12T23:21:35.242 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../094-python3-iniconfig_1.1.1-2_all.deb ... 2024-09-12T23:21:35.280 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-iniconfig (1.1.1-2) ... 2024-09-12T23:21:35.345 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-pluggy. 2024-09-12T23:21:35.372 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../096-python3-pluggy_0.13.0-7.1_all.deb ... 2024-09-12T23:21:35.411 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-pluggy (0.13.0-7.1) ... 2024-09-12T23:21:35.608 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-pastescript. 2024-09-12T23:21:35.635 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../095-python3-pastescript_2.0.2-4_all.deb ... 2024-09-12T23:21:35.674 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-pastescript (2.0.2-4) ... 2024-09-12T23:21:35.740 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-psutil. 2024-09-12T23:21:35.767 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../097-python3-psutil_5.9.0-1build1_amd64.deb ... 2024-09-12T23:21:35.806 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-psutil (5.9.0-1build1) ... 2024-09-12T23:21:35.995 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-pluggy. 2024-09-12T23:21:36.022 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../096-python3-pluggy_0.13.0-7.1_all.deb ... 2024-09-12T23:21:36.061 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-pluggy (0.13.0-7.1) ... 2024-09-12T23:21:36.126 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-py. 2024-09-12T23:21:36.153 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../098-python3-py_1.10.0-1_all.deb ... 2024-09-12T23:21:36.192 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-py (1.10.0-1) ... 2024-09-12T23:21:36.339 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-psutil. 2024-09-12T23:21:36.354 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../097-python3-psutil_5.9.0-1build1_amd64.deb ... 2024-09-12T23:21:36.388 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-psutil (5.9.0-1build1) ... 2024-09-12T23:21:36.495 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-pygments. 2024-09-12T23:21:36.523 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../099-python3-pygments_2.11.2+dfsg-2_all.deb ... 2024-09-12T23:21:36.562 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-pygments (2.11.2+dfsg-2) ... 2024-09-12T23:21:36.725 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-py. 2024-09-12T23:21:36.752 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../098-python3-py_1.10.0-1_all.deb ... 2024-09-12T23:21:36.800 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-py (1.10.0-1) ... 2024-09-12T23:21:37.041 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-pyinotify. 2024-09-12T23:21:37.068 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../100-python3-pyinotify_0.9.6-1.3_all.deb ... 2024-09-12T23:21:37.103 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-pygments. 2024-09-12T23:21:37.107 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-pyinotify (0.9.6-1.3) ... 2024-09-12T23:21:37.130 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../099-python3-pygments_2.11.2+dfsg-2_all.deb ... 2024-09-12T23:21:37.169 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-pygments (2.11.2+dfsg-2) ... 2024-09-12T23:21:37.410 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-toml. 2024-09-12T23:21:37.437 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../101-python3-toml_0.10.2-1_all.deb ... 2024-09-12T23:21:37.476 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-toml (0.10.2-1) ... 2024-09-12T23:21:37.548 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-pyinotify. 2024-09-12T23:21:37.564 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../100-python3-pyinotify_0.9.6-1.3_all.deb ... 2024-09-12T23:21:37.598 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-pyinotify (0.9.6-1.3) ... 2024-09-12T23:21:37.754 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-pytest. 2024-09-12T23:21:37.781 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../102-python3-pytest_6.2.5-1ubuntu2_all.deb ... 2024-09-12T23:21:37.820 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-pytest (6.2.5-1ubuntu2) ... 2024-09-12T23:21:37.909 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-toml. 2024-09-12T23:21:37.936 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../101-python3-toml_0.10.2-1_all.deb ... 2024-09-12T23:21:37.975 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-toml (0.10.2-1) ... 2024-09-12T23:21:38.149 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-simplejson. 2024-09-12T23:21:38.177 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../103-python3-simplejson_3.17.6-1build1_amd64.deb ... 2024-09-12T23:21:38.215 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-simplejson (3.17.6-1build1) ... 2024-09-12T23:21:38.311 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-pytest. 2024-09-12T23:21:38.339 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../102-python3-pytest_6.2.5-1ubuntu2_all.deb ... 2024-09-12T23:21:38.378 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-pytest (6.2.5-1ubuntu2) ... 2024-09-12T23:21:38.493 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package qttranslations5-l10n. 2024-09-12T23:21:38.509 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../104-qttranslations5-l10n_5.15.3-1_all.deb ... 2024-09-12T23:21:38.551 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking qttranslations5-l10n (5.15.3-1) ... 2024-09-12T23:21:38.689 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package python3-simplejson. 2024-09-12T23:21:38.717 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../103-python3-simplejson_3.17.6-1build1_amd64.deb ... 2024-09-12T23:21:38.756 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking python3-simplejson (3.17.6-1build1) ... 2024-09-12T23:21:39.000 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package qttranslations5-l10n. 2024-09-12T23:21:39.016 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../104-qttranslations5-l10n_5.15.3-1_all.deb ... 2024-09-12T23:21:39.058 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking qttranslations5-l10n (5.15.3-1) ... 2024-09-12T23:21:39.280 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package radosgw. 2024-09-12T23:21:39.308 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../105-radosgw_17.2.7-1jammy_amd64.deb ... 2024-09-12T23:21:39.355 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking radosgw (17.2.7-1jammy) ... 2024-09-12T23:21:39.854 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package radosgw. 2024-09-12T23:21:39.882 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../105-radosgw_17.2.7-1jammy_amd64.deb ... 2024-09-12T23:21:39.920 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking radosgw (17.2.7-1jammy) ... 2024-09-12T23:21:40.417 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package rbd-fuse. 2024-09-12T23:21:40.445 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../106-rbd-fuse_17.2.7-1jammy_amd64.deb ... 2024-09-12T23:21:40.475 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking rbd-fuse (17.2.7-1jammy) ... 2024-09-12T23:21:40.840 INFO:teuthology.orchestra.run.smithi136.stdout:Selecting previously unselected package rbd-fuse. 2024-09-12T23:21:40.868 INFO:teuthology.orchestra.run.smithi136.stdout:Preparing to unpack .../106-rbd-fuse_17.2.7-1jammy_amd64.deb ... 2024-09-12T23:21:40.906 INFO:teuthology.orchestra.run.smithi136.stdout:Unpacking rbd-fuse (17.2.7-1jammy) ... 2024-09-12T23:21:40.967 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-iniconfig (1.1.1-2) ... 2024-09-12T23:21:41.201 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-iniconfig (1.1.1-2) ... 2024-09-12T23:21:41.229 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up libdouble-conversion3:amd64 (3.1.7-4) ... 2024-09-12T23:21:41.339 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up nvme-cli (1.16-3ubuntu0.3) ... 2024-09-12T23:21:41.452 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up libdouble-conversion3:amd64 (3.1.7-4) ... 2024-09-12T23:21:41.578 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up nvme-cli (1.16-3ubuntu0.3) ... 2024-09-12T23:21:41.579 INFO:teuthology.orchestra.run.smithi181.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmefc-boot-connections.service -> /lib/systemd/system/nvmefc-boot-connections.service. 2024-09-12T23:21:41.842 INFO:teuthology.orchestra.run.smithi136.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmefc-boot-connections.service -> /lib/systemd/system/nvmefc-boot-connections.service. 2024-09-12T23:21:41.873 INFO:teuthology.orchestra.run.smithi181.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmf-autoconnect.service -> /lib/systemd/system/nvmf-autoconnect.service. 2024-09-12T23:21:42.196 INFO:teuthology.orchestra.run.smithi136.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmf-autoconnect.service -> /lib/systemd/system/nvmf-autoconnect.service. 2024-09-12T23:21:42.302 INFO:teuthology.orchestra.run.smithi181.stdout:nvmf-connect.target is a disabled or a static unit, not starting it. 2024-09-12T23:21:42.365 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up cephadm (17.2.7-1jammy) ... 2024-09-12T23:21:42.625 INFO:teuthology.orchestra.run.smithi136.stdout:nvmf-connect.target is a disabled or a static unit, not starting it. 2024-09-12T23:21:42.688 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up cephadm (17.2.7-1jammy) ... 2024-09-12T23:21:42.909 INFO:teuthology.orchestra.run.smithi181.stdout:Adding system user cephadm....done 2024-09-12T23:21:43.046 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-waitress (1.4.4-1.1ubuntu1) ... 2024-09-12T23:21:43.147 INFO:teuthology.orchestra.run.smithi136.stdout:Adding system user cephadm....done 2024-09-12T23:21:43.294 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-waitress (1.4.4-1.1ubuntu1) ... 2024-09-12T23:21:43.325 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-jaraco.classes (3.2.1-3) ... 2024-09-12T23:21:43.593 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-jaraco.functools (3.4.0-2) ... 2024-09-12T23:21:43.595 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-jaraco.classes (3.2.1-3) ... 2024-09-12T23:21:43.844 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-repoze.lru (0.7-2) ... 2024-09-12T23:21:43.846 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-jaraco.functools (3.4.0-2) ... 2024-09-12T23:21:44.097 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-repoze.lru (0.7-2) ... 2024-09-12T23:21:44.111 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up liboath0:amd64 (2.6.7-3build1) ... 2024-09-12T23:21:44.224 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-py (1.10.0-1) ... 2024-09-12T23:21:44.348 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up liboath0:amd64 (2.6.7-3build1) ... 2024-09-12T23:21:44.449 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-py (1.10.0-1) ... 2024-09-12T23:21:44.514 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-joblib (0.17.0-4ubuntu1) ... 2024-09-12T23:21:44.742 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-joblib (0.17.0-4ubuntu1) ... 2024-09-12T23:21:44.874 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-cachetools (5.0.0-1) ... 2024-09-12T23:21:45.125 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-openssl (21.0.0-1) ... 2024-09-12T23:21:45.369 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-cachetools (5.0.0-1) ... 2024-09-12T23:21:45.393 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-bcrypt (3.2.0-1build1) ... 2024-09-12T23:21:45.637 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-openssl (21.0.0-1) ... 2024-09-12T23:21:45.644 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-pyinotify (0.9.6-1.3) ... 2024-09-12T23:21:45.903 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-threadpoolctl (3.1.0-1) ... 2024-09-12T23:21:45.913 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-bcrypt (3.2.0-1build1) ... 2024-09-12T23:21:46.138 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-ceph-argparse (17.2.7-1jammy) ... 2024-09-12T23:21:46.155 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-pyinotify (0.9.6-1.3) ... 2024-09-12T23:21:46.388 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2024-09-12T23:21:46.406 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-threadpoolctl (3.1.0-1) ... 2024-09-12T23:21:46.498 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-markupsafe (2.0.1-2build1) ... 2024-09-12T23:21:46.649 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-ceph-argparse (17.2.7-1jammy) ... 2024-09-12T23:21:46.774 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2024-09-12T23:21:46.908 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2024-09-12T23:21:47.034 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-markupsafe (2.0.1-2build1) ... 2024-09-12T23:21:47.050 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-psutil (5.9.0-1build1) ... 2024-09-12T23:21:47.360 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2024-09-12T23:21:47.410 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python-babel-localedata (2.8.0+dfsg.1-7) ... 2024-09-12T23:21:47.486 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-psutil (5.9.0-1build1) ... 2024-09-12T23:21:47.535 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-natsort (8.0.2-1) ... 2024-09-12T23:21:47.828 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-routes (2.5.1-1ubuntu1) ... 2024-09-12T23:21:47.862 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python-babel-localedata (2.8.0+dfsg.1-7) ... 2024-09-12T23:21:47.988 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-natsort (8.0.2-1) ... 2024-09-12T23:21:48.079 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-simplejson (3.17.6-1build1) ... 2024-09-12T23:21:48.256 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-routes (2.5.1-1ubuntu1) ... 2024-09-12T23:21:48.372 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-pygments (2.11.2+dfsg-2) ... 2024-09-12T23:21:48.515 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-simplejson (3.17.6-1build1) ... 2024-09-12T23:21:48.825 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-pygments (2.11.2+dfsg-2) ... 2024-09-12T23:21:49.024 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-tempita (0.5.2-6ubuntu1) ... 2024-09-12T23:21:49.284 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python-pastedeploy-tpl (2.1.1-1) ... 2024-09-12T23:21:49.409 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up qttranslations5-l10n (5.15.3-1) ... 2024-09-12T23:21:49.518 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-tempita (0.5.2-6ubuntu1) ... 2024-09-12T23:21:49.552 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-wcwidth (0.2.5+dfsg1-1) ... 2024-09-12T23:21:49.778 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python-pastedeploy-tpl (2.1.1-1) ... 2024-09-12T23:21:49.861 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-certifi (2020.6.20-1) ... 2024-09-12T23:21:49.904 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up qttranslations5-l10n (5.15.3-1) ... 2024-09-12T23:21:50.030 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-wcwidth (0.2.5+dfsg1-1) ... 2024-09-12T23:21:50.112 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-paste (3.5.0+dfsg1-1) ... 2024-09-12T23:21:50.322 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-certifi (2020.6.20-1) ... 2024-09-12T23:21:50.455 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2024-09-12T23:21:50.573 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-paste (3.5.0+dfsg1-1) ... 2024-09-12T23:21:50.723 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-werkzeug (2.0.2+dfsg1-1ubuntu0.22.04.2) ... 2024-09-12T23:21:50.941 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2024-09-12T23:21:51.075 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-jaraco.text (3.6.0-2) ... 2024-09-12T23:21:51.259 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-werkzeug (2.0.2+dfsg1-1ubuntu0.22.04.2) ... 2024-09-12T23:21:51.334 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up socat (1.7.4.1-3ubuntu4) ... 2024-09-12T23:21:51.485 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-ceph-common (17.2.7-1jammy) ... 2024-09-12T23:21:51.611 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-jaraco.text (3.6.0-2) ... 2024-09-12T23:21:51.770 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-idna (3.3-1ubuntu0.1) ... 2024-09-12T23:21:51.862 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up socat (1.7.4.1-3ubuntu4) ... 2024-09-12T23:21:51.988 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-ceph-common (17.2.7-1jammy) ... 2024-09-12T23:21:52.063 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-sklearn (0.23.2-5ubuntu6) ... 2024-09-12T23:21:52.272 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-idna (3.3-1ubuntu0.1) ... 2024-09-12T23:21:52.581 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-sklearn (0.23.2-5ubuntu6) ... 2024-09-12T23:21:53.227 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-12T23:21:53.361 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-toml (0.10.2-1) ... 2024-09-12T23:21:53.620 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up librdkafka1:amd64 (1.8.0-1build1) ... 2024-09-12T23:21:53.687 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-12T23:21:53.721 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up xmlstarlet (1.6.1-2.1) ... 2024-09-12T23:21:53.821 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-toml (0.10.2-1) ... 2024-09-12T23:21:53.822 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-urllib3 (1.26.5-1~exp1ubuntu0.1) ... 2024-09-12T23:21:54.072 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up librdkafka1:amd64 (1.8.0-1build1) ... 2024-09-12T23:21:54.090 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-pluggy (0.13.0-7.1) ... 2024-09-12T23:21:54.198 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up xmlstarlet (1.6.1-2.1) ... 2024-09-12T23:21:54.324 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-urllib3 (1.26.5-1~exp1ubuntu0.1) ... 2024-09-12T23:21:54.341 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-zc.lockfile (2.0-1) ... 2024-09-12T23:21:54.592 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-12T23:21:54.608 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-pluggy (0.13.0-7.1) ... 2024-09-12T23:21:54.718 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up libfmt8:amd64 (8.1.1+ds1-2) ... 2024-09-12T23:21:54.844 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-pyasn1 (0.4.8-1) ... 2024-09-12T23:21:54.859 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-zc.lockfile (2.0-1) ... 2024-09-12T23:21:55.102 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-12T23:21:55.137 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-singledispatch (3.4.0.3-3) ... 2024-09-12T23:21:55.228 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up libfmt8:amd64 (8.1.1+ds1-2) ... 2024-09-12T23:21:55.372 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-logutils (0.3.3-8) ... 2024-09-12T23:21:55.387 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-pyasn1 (0.4.8-1) ... 2024-09-12T23:21:55.606 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up ceph-mgr-modules-core (17.2.7-1jammy) ... 2024-09-12T23:21:55.689 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-singledispatch (3.4.0.3-3) ... 2024-09-12T23:21:55.740 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-tempora (4.1.2-1) ... 2024-09-12T23:21:55.932 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-logutils (0.3.3-8) ... 2024-09-12T23:21:56.000 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-simplegeneric (0.8.1-3) ... 2024-09-12T23:21:56.183 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up ceph-mgr-modules-core (17.2.7-1jammy) ... 2024-09-12T23:21:56.276 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-prettytable (2.5.0-2) ... 2024-09-12T23:21:56.317 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-tempora (4.1.2-1) ... 2024-09-12T23:21:56.576 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-simplegeneric (0.8.1-3) ... 2024-09-12T23:21:56.777 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2024-09-12T23:21:56.827 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-prettytable (2.5.0-2) ... 2024-09-12T23:21:56.903 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-websocket (1.2.3-1) ... 2024-09-12T23:21:57.095 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2024-09-12T23:21:57.162 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up libonig5:amd64 (6.9.7.1-2build1) ... 2024-09-12T23:21:57.221 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-websocket (1.2.3-1) ... 2024-09-12T23:21:57.263 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-mako (1.1.3+ds1-2ubuntu0.1) ... 2024-09-12T23:21:57.497 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up libonig5:amd64 (6.9.7.1-2build1) ... 2024-09-12T23:21:57.531 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-webob (1:1.8.6-1.1ubuntu0.1) ... 2024-09-12T23:21:57.623 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-mako (1.1.3+ds1-2ubuntu0.1) ... 2024-09-12T23:21:57.841 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-jaraco.collections (3.4.0-2) ... 2024-09-12T23:21:57.907 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-webob (1:1.8.6-1.1ubuntu0.1) ... 2024-09-12T23:21:58.100 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up librados2 (17.2.7-1jammy) ... 2024-09-12T23:21:58.208 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-jaraco.collections (3.4.0-2) ... 2024-09-12T23:21:58.226 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-babel (2.8.0+dfsg.1-7) ... 2024-09-12T23:21:58.459 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up librados2 (17.2.7-1jammy) ... 2024-09-12T23:21:58.585 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-babel (2.8.0+dfsg.1-7) ... 2024-09-12T23:21:58.619 INFO:teuthology.orchestra.run.smithi181.stdout:update-alternatives: using /usr/bin/pybabel-python3 to provide /usr/bin/pybabel (pybabel) in auto mode 2024-09-12T23:21:58.862 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up libsqlite3-mod-ceph (17.2.7-1jammy) ... 2024-09-12T23:21:58.963 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up libjq1:amd64 (1.6-2.1ubuntu3) ... 2024-09-12T23:21:58.980 INFO:teuthology.orchestra.run.smithi136.stdout:update-alternatives: using /usr/bin/pybabel-python3 to provide /usr/bin/pybabel (pybabel) in auto mode 2024-09-12T23:21:59.064 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-pytest (6.2.5-1ubuntu2) ... 2024-09-12T23:21:59.238 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up libsqlite3-mod-ceph (17.2.7-1jammy) ... 2024-09-12T23:21:59.364 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up libjq1:amd64 (1.6-2.1ubuntu3) ... 2024-09-12T23:21:59.449 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-pastedeploy (2.1.1-1) ... 2024-09-12T23:21:59.523 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-pytest (6.2.5-1ubuntu2) ... 2024-09-12T23:21:59.708 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-portend (3.0.0-1) ... 2024-09-12T23:21:59.916 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-pastedeploy (2.1.1-1) ... 2024-09-12T23:21:59.967 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-12T23:22:00.093 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up libcephfs2 (17.2.7-1jammy) ... 2024-09-12T23:22:00.175 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-portend (3.0.0-1) ... 2024-09-12T23:22:00.219 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-jinja2 (3.0.3-1ubuntu0.2) ... 2024-09-12T23:22:00.443 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-12T23:22:00.529 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up libradosstriper1 (17.2.7-1jammy) ... 2024-09-12T23:22:00.569 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up libcephfs2 (17.2.7-1jammy) ... 2024-09-12T23:22:00.630 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-pyasn1-modules (0.2.1-1) ... 2024-09-12T23:22:00.687 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-jinja2 (3.0.3-1ubuntu0.2) ... 2024-09-12T23:22:00.914 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-requests (2.25.1+dfsg-2ubuntu0.1) ... 2024-09-12T23:22:00.988 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up libradosstriper1 (17.2.7-1jammy) ... 2024-09-12T23:22:01.114 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-pyasn1-modules (0.2.1-1) ... 2024-09-12T23:22:01.182 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up librbd1 (17.2.7-1jammy) ... 2024-09-12T23:22:01.308 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up jq (1.6-2.1ubuntu3) ... 2024-09-12T23:22:01.423 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-requests (2.25.1+dfsg-2ubuntu0.1) ... 2024-09-12T23:22:01.434 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-rsa (4.8-1) ... 2024-09-12T23:22:01.674 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up librbd1 (17.2.7-1jammy) ... 2024-09-12T23:22:01.710 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-webtest (2.0.35-1) ... 2024-09-12T23:22:01.800 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up jq (1.6-2.1ubuntu3) ... 2024-09-12T23:22:01.926 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-rsa (4.8-1) ... 2024-09-12T23:22:01.978 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up ceph-fuse (17.2.7-1jammy) ... 2024-09-12T23:22:02.189 INFO:teuthology.orchestra.run.smithi181.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2024-09-12T23:22:02.189 INFO:teuthology.orchestra.run.smithi181.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2024-09-12T23:22:02.219 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-webtest (2.0.35-1) ... 2024-09-12T23:22:02.486 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up ceph-fuse (17.2.7-1jammy) ... 2024-09-12T23:22:02.663 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up libcephfs-dev (17.2.7-1jammy) ... 2024-09-12T23:22:02.678 INFO:teuthology.orchestra.run.smithi136.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2024-09-12T23:22:02.678 INFO:teuthology.orchestra.run.smithi136.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2024-09-12T23:22:02.764 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-requests-oauthlib (1.3.0+ds-0.1) ... 2024-09-12T23:22:03.049 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-cherrypy3 (18.6.1-4) ... 2024-09-12T23:22:03.104 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up libcephfs-dev (17.2.7-1jammy) ... 2024-09-12T23:22:03.255 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-requests-oauthlib (1.3.0+ds-0.1) ... 2024-09-12T23:22:03.425 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-pastescript (2.0.2-4) ... 2024-09-12T23:22:03.726 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-pecan (1.3.3-4ubuntu2) ... 2024-09-12T23:22:03.862 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-cherrypy3 (18.6.1-4) ... 2024-09-12T23:22:04.147 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up libthrift-0.16.0:amd64 (0.16.0-2) ... 2024-09-12T23:22:04.256 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-rados (17.2.7-1jammy) ... 2024-09-12T23:22:04.284 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-pastescript (2.0.2-4) ... 2024-09-12T23:22:04.383 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up librgw2 (17.2.7-1jammy) ... 2024-09-12T23:22:04.509 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-rbd (17.2.7-1jammy) ... 2024-09-12T23:22:04.577 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-pecan (1.3.3-4ubuntu2) ... 2024-09-12T23:22:04.635 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up rbd-fuse (17.2.7-1jammy) ... 2024-09-12T23:22:04.761 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-google-auth (1.5.1-3) ... 2024-09-12T23:22:05.014 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up libthrift-0.16.0:amd64 (0.16.0-2) ... 2024-09-12T23:22:05.037 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-rgw (17.2.7-1jammy) ... 2024-09-12T23:22:05.123 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-rados (17.2.7-1jammy) ... 2024-09-12T23:22:05.138 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-cephfs (17.2.7-1jammy) ... 2024-09-12T23:22:05.258 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up librgw2 (17.2.7-1jammy) ... 2024-09-12T23:22:05.264 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-kubernetes (12.0.1-1ubuntu1) ... 2024-09-12T23:22:05.401 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-rbd (17.2.7-1jammy) ... 2024-09-12T23:22:05.542 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up rbd-fuse (17.2.7-1jammy) ... 2024-09-12T23:22:05.902 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-google-auth (1.5.1-3) ... 2024-09-12T23:22:06.179 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-rgw (17.2.7-1jammy) ... 2024-09-12T23:22:06.296 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-cephfs (17.2.7-1jammy) ... 2024-09-12T23:22:06.406 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up python3-kubernetes (12.0.1-1ubuntu1) ... 2024-09-12T23:22:06.476 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up ceph-common (17.2.7-1jammy) ... 2024-09-12T23:22:07.011 INFO:teuthology.orchestra.run.smithi181.stdout:Adding group ceph....done 2024-09-12T23:22:07.506 INFO:teuthology.orchestra.run.smithi181.stdout:Adding system user ceph....done 2024-09-12T23:22:07.593 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up ceph-common (17.2.7-1jammy) ... 2024-09-12T23:22:07.656 INFO:teuthology.orchestra.run.smithi181.stdout:Setting system user ceph properties....done 2024-09-12T23:22:07.674 INFO:teuthology.orchestra.run.smithi181.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2024-09-12T23:22:07.808 INFO:teuthology.orchestra.run.smithi181.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /lib/systemd/system/ceph.target. 2024-09-12T23:22:08.144 INFO:teuthology.orchestra.run.smithi136.stdout:Adding group ceph....done 2024-09-12T23:22:08.160 INFO:teuthology.orchestra.run.smithi181.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service -> /lib/systemd/system/rbdmap.service. 2024-09-12T23:22:08.504 INFO:teuthology.orchestra.run.smithi136.stdout:Adding system user ceph....done 2024-09-12T23:22:08.656 INFO:teuthology.orchestra.run.smithi136.stdout:Setting system user ceph properties....done 2024-09-12T23:22:08.669 INFO:teuthology.orchestra.run.smithi136.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2024-09-12T23:22:08.678 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up ceph-test (17.2.7-1jammy) ... 2024-09-12T23:22:08.788 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up radosgw (17.2.7-1jammy) ... 2024-09-12T23:22:08.805 INFO:teuthology.orchestra.run.smithi136.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /lib/systemd/system/ceph.target. 2024-09-12T23:22:09.162 INFO:teuthology.orchestra.run.smithi136.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service -> /lib/systemd/system/rbdmap.service. 2024-09-12T23:22:09.373 INFO:teuthology.orchestra.run.smithi181.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2024-09-12T23:22:09.373 INFO:teuthology.orchestra.run.smithi181.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2024-09-12T23:22:09.652 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up ceph-test (17.2.7-1jammy) ... 2024-09-12T23:22:09.761 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up radosgw (17.2.7-1jammy) ... 2024-09-12T23:22:10.001 INFO:teuthology.orchestra.run.smithi136.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2024-09-12T23:22:10.001 INFO:teuthology.orchestra.run.smithi136.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2024-09-12T23:22:10.016 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up ceph-base (17.2.7-1jammy) ... 2024-09-12T23:22:10.357 INFO:teuthology.orchestra.run.smithi181.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service -> /lib/systemd/system/ceph-crash.service. 2024-09-12T23:22:10.663 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up ceph-base (17.2.7-1jammy) ... 2024-09-12T23:22:10.842 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up ceph-mds (17.2.7-1jammy) ... 2024-09-12T23:22:10.979 INFO:teuthology.orchestra.run.smithi136.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service -> /lib/systemd/system/ceph-crash.service. 2024-09-12T23:22:11.046 INFO:teuthology.orchestra.run.smithi181.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2024-09-12T23:22:11.046 INFO:teuthology.orchestra.run.smithi181.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2024-09-12T23:22:11.423 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up ceph-mds (17.2.7-1jammy) ... 2024-09-12T23:22:11.502 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up ceph-mgr (17.2.7-1jammy) ... 2024-09-12T23:22:11.636 INFO:teuthology.orchestra.run.smithi136.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2024-09-12T23:22:11.637 INFO:teuthology.orchestra.run.smithi136.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2024-09-12T23:22:11.720 INFO:teuthology.orchestra.run.smithi181.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2024-09-12T23:22:11.720 INFO:teuthology.orchestra.run.smithi181.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2024-09-12T23:22:12.092 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up ceph-mgr (17.2.7-1jammy) ... 2024-09-12T23:22:12.178 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up ceph-osd (17.2.7-1jammy) ... 2024-09-12T23:22:12.307 INFO:teuthology.orchestra.run.smithi136.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2024-09-12T23:22:12.307 INFO:teuthology.orchestra.run.smithi136.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2024-09-12T23:22:12.424 INFO:teuthology.orchestra.run.smithi181.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2024-09-12T23:22:12.424 INFO:teuthology.orchestra.run.smithi181.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2024-09-12T23:22:12.809 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up ceph-osd (17.2.7-1jammy) ... 2024-09-12T23:22:12.930 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up ceph-mgr-k8sevents (17.2.7-1jammy) ... 2024-09-12T23:22:13.056 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up ceph-mgr-diskprediction-local (17.2.7-1jammy) ... 2024-09-12T23:22:13.083 INFO:teuthology.orchestra.run.smithi136.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2024-09-12T23:22:13.083 INFO:teuthology.orchestra.run.smithi136.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2024-09-12T23:22:13.215 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up ceph-mon (17.2.7-1jammy) ... 2024-09-12T23:22:13.427 INFO:teuthology.orchestra.run.smithi181.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2024-09-12T23:22:13.427 INFO:teuthology.orchestra.run.smithi181.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2024-09-12T23:22:13.594 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up ceph-mgr-k8sevents (17.2.7-1jammy) ... 2024-09-12T23:22:13.711 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up ceph-mgr-diskprediction-local (17.2.7-1jammy) ... 2024-09-12T23:22:13.862 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up ceph-mon (17.2.7-1jammy) ... 2024-09-12T23:22:13.891 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up ceph-mgr-cephadm (17.2.7-1jammy) ... 2024-09-12T23:22:14.034 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up ceph (17.2.7-1jammy) ... 2024-09-12T23:22:14.075 INFO:teuthology.orchestra.run.smithi136.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2024-09-12T23:22:14.075 INFO:teuthology.orchestra.run.smithi136.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2024-09-12T23:22:14.160 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up ceph-mgr-dashboard (17.2.7-1jammy) ... 2024-09-12T23:22:14.303 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up ceph-volume (17.2.7-1jammy) ... 2024-09-12T23:22:14.405 INFO:teuthology.orchestra.run.smithi181.stdout:Processing triggers for man-db (2.10.2-1) ... 2024-09-12T23:22:14.513 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up ceph-mgr-cephadm (17.2.7-1jammy) ... 2024-09-12T23:22:14.673 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up ceph (17.2.7-1jammy) ... 2024-09-12T23:22:14.799 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up ceph-mgr-dashboard (17.2.7-1jammy) ... 2024-09-12T23:22:14.950 INFO:teuthology.orchestra.run.smithi136.stdout:Setting up ceph-volume (17.2.7-1jammy) ... 2024-09-12T23:22:15.093 INFO:teuthology.orchestra.run.smithi136.stdout:Processing triggers for man-db (2.10.2-1) ... 2024-09-12T23:22:15.497 INFO:teuthology.orchestra.run.smithi181.stdout:Processing triggers for libc-bin (2.35-0ubuntu3.1) ... 2024-09-12T23:22:15.978 INFO:teuthology.orchestra.run.smithi181.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2024-09-12T23:22:15.981 DEBUG:teuthology.orchestra.run.smithi181:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install python3-pytest 2024-09-12T23:22:16.080 INFO:teuthology.orchestra.run.smithi181.stdout:Reading package lists... 2024-09-12T23:22:16.195 INFO:teuthology.orchestra.run.smithi136.stdout:Processing triggers for libc-bin (2.35-0ubuntu3.1) ... 2024-09-12T23:22:16.291 INFO:teuthology.orchestra.run.smithi181.stdout:Building dependency tree... 2024-09-12T23:22:16.292 INFO:teuthology.orchestra.run.smithi181.stdout:Reading state information... 2024-09-12T23:22:16.480 INFO:teuthology.orchestra.run.smithi181.stdout:python3-pytest is already the newest version (6.2.5-1ubuntu2). 2024-09-12T23:22:16.481 INFO:teuthology.orchestra.run.smithi181.stdout:python3-pytest set to manually installed. 2024-09-12T23:22:16.481 INFO:teuthology.orchestra.run.smithi181.stdout:The following packages were automatically installed and are no longer required: 2024-09-12T23:22:16.481 INFO:teuthology.orchestra.run.smithi181.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-09-12T23:22:16.481 INFO:teuthology.orchestra.run.smithi181.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-09-12T23:22:16.481 INFO:teuthology.orchestra.run.smithi181.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-09-12T23:22:16.481 INFO:teuthology.orchestra.run.smithi181.stdout: linux-image-5.4.0-135-generic linux-modules-5.4.0-135-generic python2 2024-09-12T23:22:16.482 INFO:teuthology.orchestra.run.smithi181.stdout: python2-dev python2-minimal python2.7 python2.7-dev python2.7-minimal 2024-09-12T23:22:16.482 INFO:teuthology.orchestra.run.smithi181.stdout:Use 'sudo apt autoremove' to remove them. 2024-09-12T23:22:16.532 INFO:teuthology.orchestra.run.smithi181.stdout:0 upgraded, 0 newly installed, 0 to remove and 331 not upgraded. 2024-09-12T23:22:16.532 INFO:teuthology.orchestra.run.smithi181.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2024-09-12T23:22:16.537 DEBUG:teuthology.parallel:result is None 2024-09-12T23:22:16.646 INFO:teuthology.orchestra.run.smithi136.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2024-09-12T23:22:16.648 DEBUG:teuthology.orchestra.run.smithi136:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install python3-pytest 2024-09-12T23:22:16.704 INFO:teuthology.orchestra.run.smithi136.stdout:Reading package lists... 2024-09-12T23:22:16.910 INFO:teuthology.orchestra.run.smithi136.stdout:Building dependency tree... 2024-09-12T23:22:16.911 INFO:teuthology.orchestra.run.smithi136.stdout:Reading state information... 2024-09-12T23:22:17.089 INFO:teuthology.orchestra.run.smithi136.stdout:python3-pytest is already the newest version (6.2.5-1ubuntu2). 2024-09-12T23:22:17.089 INFO:teuthology.orchestra.run.smithi136.stdout:python3-pytest set to manually installed. 2024-09-12T23:22:17.089 INFO:teuthology.orchestra.run.smithi136.stdout:The following packages were automatically installed and are no longer required: 2024-09-12T23:22:17.089 INFO:teuthology.orchestra.run.smithi136.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-09-12T23:22:17.089 INFO:teuthology.orchestra.run.smithi136.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-09-12T23:22:17.089 INFO:teuthology.orchestra.run.smithi136.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-09-12T23:22:17.090 INFO:teuthology.orchestra.run.smithi136.stdout: linux-image-5.4.0-135-generic linux-modules-5.4.0-135-generic python2 2024-09-12T23:22:17.090 INFO:teuthology.orchestra.run.smithi136.stdout: python2-dev python2-minimal python2.7 python2.7-dev python2.7-minimal 2024-09-12T23:22:17.090 INFO:teuthology.orchestra.run.smithi136.stdout:Use 'sudo apt autoremove' to remove them. 2024-09-12T23:22:17.138 INFO:teuthology.orchestra.run.smithi136.stdout:0 upgraded, 0 newly installed, 0 to remove and 331 not upgraded. 2024-09-12T23:22:17.139 INFO:teuthology.orchestra.run.smithi136.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2024-09-12T23:22:17.142 DEBUG:teuthology.parallel:result is None 2024-09-12T23:22:17.143 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-12T23:22:17.143 INFO:teuthology.packaging:ref: None 2024-09-12T23:22:17.143 INFO:teuthology.packaging:tag: None 2024-09-12T23:22:17.143 INFO:teuthology.packaging:branch: quincy 2024-09-12T23:22:17.143 INFO:teuthology.packaging:sha1: 7a853b1706d076d58e0da7a5d3b9b9eb36276880 2024-09-12T23:22:17.143 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&ref=quincy 2024-09-12T23:22:17.279 DEBUG:teuthology.orchestra.run.smithi136:> dpkg-query -W -f '${Version}' ceph 2024-09-12T23:22:17.297 INFO:teuthology.orchestra.run.smithi136.stdout:17.2.7-1jammy 2024-09-12T23:22:17.297 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-1jammy 2024-09-12T23:22:17.297 INFO:teuthology.task.install:The correct ceph version 17.2.7-1jammy is installed. 2024-09-12T23:22:17.299 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-12T23:22:17.299 INFO:teuthology.packaging:ref: None 2024-09-12T23:22:17.299 INFO:teuthology.packaging:tag: None 2024-09-12T23:22:17.299 INFO:teuthology.packaging:branch: quincy 2024-09-12T23:22:17.299 INFO:teuthology.packaging:sha1: 7a853b1706d076d58e0da7a5d3b9b9eb36276880 2024-09-12T23:22:17.299 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&ref=quincy 2024-09-12T23:22:17.433 DEBUG:teuthology.orchestra.run.smithi181:> dpkg-query -W -f '${Version}' ceph 2024-09-12T23:22:17.452 INFO:teuthology.orchestra.run.smithi181.stdout:17.2.7-1jammy 2024-09-12T23:22:17.452 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-1jammy 2024-09-12T23:22:17.452 INFO:teuthology.task.install:The correct ceph version 17.2.7-1jammy is installed. 2024-09-12T23:22:17.454 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-09-12T23:22:17.455 DEBUG:teuthology.orchestra.run.smithi136:> set -ex 2024-09-12T23:22:17.455 DEBUG:teuthology.orchestra.run.smithi136:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-09-12T23:22:17.468 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2024-09-12T23:22:17.468 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-09-12T23:22:17.508 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-09-12T23:22:17.508 DEBUG:teuthology.orchestra.run.smithi136:> set -ex 2024-09-12T23:22:17.508 DEBUG:teuthology.orchestra.run.smithi136:> sudo dd of=/usr/bin/daemon-helper 2024-09-12T23:22:17.523 DEBUG:teuthology.orchestra.run.smithi136:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-09-12T23:22:17.579 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2024-09-12T23:22:17.579 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd of=/usr/bin/daemon-helper 2024-09-12T23:22:17.591 DEBUG:teuthology.orchestra.run.smithi181:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-09-12T23:22:17.646 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-09-12T23:22:17.646 DEBUG:teuthology.orchestra.run.smithi136:> set -ex 2024-09-12T23:22:17.646 DEBUG:teuthology.orchestra.run.smithi136:> sudo dd of=/usr/bin/adjust-ulimits 2024-09-12T23:22:17.660 DEBUG:teuthology.orchestra.run.smithi136:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-09-12T23:22:17.715 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2024-09-12T23:22:17.715 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd of=/usr/bin/adjust-ulimits 2024-09-12T23:22:17.727 DEBUG:teuthology.orchestra.run.smithi181:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-09-12T23:22:17.783 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-09-12T23:22:17.783 DEBUG:teuthology.orchestra.run.smithi136:> set -ex 2024-09-12T23:22:17.783 DEBUG:teuthology.orchestra.run.smithi136:> sudo dd of=/usr/bin/stdin-killer 2024-09-12T23:22:17.796 DEBUG:teuthology.orchestra.run.smithi136:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-09-12T23:22:17.855 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2024-09-12T23:22:17.856 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd of=/usr/bin/stdin-killer 2024-09-12T23:22:17.868 DEBUG:teuthology.orchestra.run.smithi181:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-09-12T23:22:17.922 INFO:teuthology.run_tasks:Running task print... 2024-09-12T23:22:17.932 INFO:teuthology.task.print:**** done install task... 2024-09-12T23:22:17.932 INFO:teuthology.run_tasks:Running task print... 2024-09-12T23:22:17.974 INFO:teuthology.task.print:**** done start installing quincy cephadm ... 2024-09-12T23:22:17.974 INFO:teuthology.run_tasks:Running task cephadm... 2024-09-12T23:22:18.090 INFO:tasks.cephadm:Config: {'cephadm_branch': 'quincy', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:quincy', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': '7a853b1706d076d58e0da7a5d3b9b9eb36276880'} 2024-09-12T23:22:18.091 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:quincy 2024-09-12T23:22:18.091 INFO:tasks.cephadm:Cluster fsid is da58a47c-715d-11ef-bceb-c7b262605968 2024-09-12T23:22:18.091 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-09-12T23:22:18.091 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.136', 'mon.c': '[v2:172.21.15.136:3301,v1:172.21.15.136:6790]', 'mon.b': '172.21.15.181'} 2024-09-12T23:22:18.091 INFO:tasks.cephadm:First mon is mon.a on smithi136 2024-09-12T23:22:18.092 INFO:tasks.cephadm:First mgr is y 2024-09-12T23:22:18.092 INFO:tasks.cephadm:Normalizing hostnames... 2024-09-12T23:22:18.092 DEBUG:teuthology.orchestra.run.smithi136:> sudo hostname $(hostname -s) 2024-09-12T23:22:18.106 DEBUG:teuthology.orchestra.run.smithi181:> sudo hostname $(hostname -s) 2024-09-12T23:22:18.121 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref quincy)... 2024-09-12T23:22:18.121 DEBUG:teuthology.orchestra.run.smithi136:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-12T23:22:18.383 INFO:teuthology.orchestra.run.smithi136.stdout:-rw-rw-r-- 1 ubuntu ubuntu 378585 Sep 12 23:22 /home/ubuntu/cephtest/cephadm 2024-09-12T23:22:18.384 DEBUG:teuthology.orchestra.run.smithi181:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-12T23:22:18.604 INFO:teuthology.orchestra.run.smithi181.stdout:-rw-rw-r-- 1 ubuntu ubuntu 378585 Sep 12 23:22 /home/ubuntu/cephtest/cephadm 2024-09-12T23:22:18.605 DEBUG:teuthology.orchestra.run.smithi136:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-12T23:22:18.613 DEBUG:teuthology.orchestra.run.smithi181:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-12T23:22:18.633 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:quincy on all hosts... 2024-09-12T23:22:18.633 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-09-12T23:22:18.657 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-09-12T23:22:18.815 INFO:teuthology.orchestra.run.smithi136.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-09-12T23:22:18.830 INFO:teuthology.orchestra.run.smithi181.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-09-12T23:23:14.911 INFO:teuthology.orchestra.run.smithi136.stdout:{ 2024-09-12T23:23:14.912 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph_version": "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)", 2024-09-12T23:23:14.912 INFO:teuthology.orchestra.run.smithi136.stdout: "image_id": "214985049bdbd91e1f28ec4bd8bf7dfb097e67a214bb40938f958f4a20cc6f0d", 2024-09-12T23:23:14.912 INFO:teuthology.orchestra.run.smithi136.stdout: "repo_digests": [ 2024-09-12T23:23:14.912 INFO:teuthology.orchestra.run.smithi136.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:9e84e422c5f3e13e9ae13013f0985fd8f54e85e87f1d2d371f86d595f20a9905" 2024-09-12T23:23:14.912 INFO:teuthology.orchestra.run.smithi136.stdout: ] 2024-09-12T23:23:14.912 INFO:teuthology.orchestra.run.smithi136.stdout:} 2024-09-12T23:23:15.978 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2024-09-12T23:23:15.979 INFO:teuthology.orchestra.run.smithi181.stdout: "ceph_version": "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)", 2024-09-12T23:23:15.979 INFO:teuthology.orchestra.run.smithi181.stdout: "image_id": "214985049bdbd91e1f28ec4bd8bf7dfb097e67a214bb40938f958f4a20cc6f0d", 2024-09-12T23:23:15.979 INFO:teuthology.orchestra.run.smithi181.stdout: "repo_digests": [ 2024-09-12T23:23:15.979 INFO:teuthology.orchestra.run.smithi181.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:9e84e422c5f3e13e9ae13013f0985fd8f54e85e87f1d2d371f86d595f20a9905" 2024-09-12T23:23:15.979 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2024-09-12T23:23:15.979 INFO:teuthology.orchestra.run.smithi181.stdout:} 2024-09-12T23:23:16.017 DEBUG:teuthology.orchestra.run.smithi136:> sudo mkdir -p /etc/ceph 2024-09-12T23:23:16.032 DEBUG:teuthology.orchestra.run.smithi181:> sudo mkdir -p /etc/ceph 2024-09-12T23:23:16.046 DEBUG:teuthology.orchestra.run.smithi136:> sudo chmod 777 /etc/ceph 2024-09-12T23:23:16.086 DEBUG:teuthology.orchestra.run.smithi181:> sudo chmod 777 /etc/ceph 2024-09-12T23:23:16.101 INFO:tasks.cephadm:Writing seed config... 2024-09-12T23:23:16.102 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2024-09-12T23:23:16.102 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2024-09-12T23:23:16.102 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-09-12T23:23:16.103 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-09-12T23:23:16.103 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-09-12T23:23:16.103 INFO:tasks.cephadm: override: [global] mon election default strategy = 1 2024-09-12T23:23:16.103 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-09-12T23:23:16.103 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-09-12T23:23:16.103 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-09-12T23:23:16.103 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-09-12T23:23:16.103 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-09-12T23:23:16.104 DEBUG:teuthology.orchestra.run.smithi136:> set -ex 2024-09-12T23:23:16.104 DEBUG:teuthology.orchestra.run.smithi136:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-09-12T23:23:16.135 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = da58a47c-715d-11ef-bceb-c7b262605968 mon election default strategy = 1 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-09-12T23:23:16.135 DEBUG:teuthology.orchestra.run.smithi136:mon.a> sudo journalctl -f -n 0 -u ceph-da58a47c-715d-11ef-bceb-c7b262605968@mon.a.service 2024-09-12T23:23:16.180 DEBUG:teuthology.orchestra.run.smithi136:mgr.y> sudo journalctl -f -n 0 -u ceph-da58a47c-715d-11ef-bceb-c7b262605968@mgr.y.service 2024-09-12T23:23:16.223 INFO:tasks.cephadm:Bootstrapping... 2024-09-12T23:23:16.224 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy -v bootstrap --fsid da58a47c-715d-11ef-bceb-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.136 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-09-12T23:23:16.437 INFO:teuthology.orchestra.run.smithi136.stdout:-------------------------------------------------------------------------------- 2024-09-12T23:23:16.437 INFO:teuthology.orchestra.run.smithi136.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:quincy', '-v', 'bootstrap', '--fsid', 'da58a47c-715d-11ef-bceb-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.136', '--skip-admin-label'] 2024-09-12T23:23:16.437 INFO:teuthology.orchestra.run.smithi136.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2024-09-12T23:23:16.438 INFO:teuthology.orchestra.run.smithi136.stdout:Verifying podman|docker is present... 2024-09-12T23:23:16.438 INFO:teuthology.orchestra.run.smithi136.stdout:Verifying lvm2 is present... 2024-09-12T23:23:16.438 INFO:teuthology.orchestra.run.smithi136.stdout:Verifying time synchronization is in place... 2024-09-12T23:23:16.443 INFO:teuthology.orchestra.run.smithi136.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-09-12T23:23:16.443 INFO:teuthology.orchestra.run.smithi136.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-09-12T23:23:16.447 INFO:teuthology.orchestra.run.smithi136.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-09-12T23:23:16.447 INFO:teuthology.orchestra.run.smithi136.stdout:systemctl: stdout inactive 2024-09-12T23:23:16.451 INFO:teuthology.orchestra.run.smithi136.stdout:Non-zero exit code 1 from systemctl is-enabled chronyd.service 2024-09-12T23:23:16.451 INFO:teuthology.orchestra.run.smithi136.stdout:systemctl: stderr Failed to get unit file state for chronyd.service: No such file or directory 2024-09-12T23:23:16.456 INFO:teuthology.orchestra.run.smithi136.stdout:Non-zero exit code 3 from systemctl is-active chronyd.service 2024-09-12T23:23:16.456 INFO:teuthology.orchestra.run.smithi136.stdout:systemctl: stdout inactive 2024-09-12T23:23:16.460 INFO:teuthology.orchestra.run.smithi136.stdout:Non-zero exit code 1 from systemctl is-enabled systemd-timesyncd.service 2024-09-12T23:23:16.460 INFO:teuthology.orchestra.run.smithi136.stdout:systemctl: stdout masked 2024-09-12T23:23:16.464 INFO:teuthology.orchestra.run.smithi136.stdout:Non-zero exit code 3 from systemctl is-active systemd-timesyncd.service 2024-09-12T23:23:16.464 INFO:teuthology.orchestra.run.smithi136.stdout:systemctl: stdout inactive 2024-09-12T23:23:16.467 INFO:teuthology.orchestra.run.smithi136.stdout:Non-zero exit code 1 from systemctl is-enabled ntpd.service 2024-09-12T23:23:16.467 INFO:teuthology.orchestra.run.smithi136.stdout:systemctl: stderr Failed to get unit file state for ntpd.service: No such file or directory 2024-09-12T23:23:16.470 INFO:teuthology.orchestra.run.smithi136.stdout:Non-zero exit code 3 from systemctl is-active ntpd.service 2024-09-12T23:23:16.470 INFO:teuthology.orchestra.run.smithi136.stdout:systemctl: stdout inactive 2024-09-12T23:23:16.475 INFO:teuthology.orchestra.run.smithi136.stdout:systemctl: stdout enabled 2024-09-12T23:23:16.478 INFO:teuthology.orchestra.run.smithi136.stdout:systemctl: stdout active 2024-09-12T23:23:16.478 INFO:teuthology.orchestra.run.smithi136.stdout:Unit ntp.service is enabled and running 2024-09-12T23:23:16.478 INFO:teuthology.orchestra.run.smithi136.stdout:Repeating the final host check... 2024-09-12T23:23:16.479 INFO:teuthology.orchestra.run.smithi136.stdout:docker (/usr/bin/docker) is present 2024-09-12T23:23:16.479 INFO:teuthology.orchestra.run.smithi136.stdout:systemctl is present 2024-09-12T23:23:16.479 INFO:teuthology.orchestra.run.smithi136.stdout:lvcreate is present 2024-09-12T23:23:16.482 INFO:teuthology.orchestra.run.smithi136.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-09-12T23:23:16.482 INFO:teuthology.orchestra.run.smithi136.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-09-12T23:23:16.485 INFO:teuthology.orchestra.run.smithi136.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-09-12T23:23:16.485 INFO:teuthology.orchestra.run.smithi136.stdout:systemctl: stdout inactive 2024-09-12T23:23:16.489 INFO:teuthology.orchestra.run.smithi136.stdout:Non-zero exit code 1 from systemctl is-enabled chronyd.service 2024-09-12T23:23:16.489 INFO:teuthology.orchestra.run.smithi136.stdout:systemctl: stderr Failed to get unit file state for chronyd.service: No such file or directory 2024-09-12T23:23:16.492 INFO:teuthology.orchestra.run.smithi136.stdout:Non-zero exit code 3 from systemctl is-active chronyd.service 2024-09-12T23:23:16.492 INFO:teuthology.orchestra.run.smithi136.stdout:systemctl: stdout inactive 2024-09-12T23:23:16.496 INFO:teuthology.orchestra.run.smithi136.stdout:Non-zero exit code 1 from systemctl is-enabled systemd-timesyncd.service 2024-09-12T23:23:16.496 INFO:teuthology.orchestra.run.smithi136.stdout:systemctl: stdout masked 2024-09-12T23:23:16.499 INFO:teuthology.orchestra.run.smithi136.stdout:Non-zero exit code 3 from systemctl is-active systemd-timesyncd.service 2024-09-12T23:23:16.499 INFO:teuthology.orchestra.run.smithi136.stdout:systemctl: stdout inactive 2024-09-12T23:23:16.503 INFO:teuthology.orchestra.run.smithi136.stdout:Non-zero exit code 1 from systemctl is-enabled ntpd.service 2024-09-12T23:23:16.503 INFO:teuthology.orchestra.run.smithi136.stdout:systemctl: stderr Failed to get unit file state for ntpd.service: No such file or directory 2024-09-12T23:23:16.506 INFO:teuthology.orchestra.run.smithi136.stdout:Non-zero exit code 3 from systemctl is-active ntpd.service 2024-09-12T23:23:16.506 INFO:teuthology.orchestra.run.smithi136.stdout:systemctl: stdout inactive 2024-09-12T23:23:16.510 INFO:teuthology.orchestra.run.smithi136.stdout:systemctl: stdout enabled 2024-09-12T23:23:16.514 INFO:teuthology.orchestra.run.smithi136.stdout:systemctl: stdout active 2024-09-12T23:23:16.514 INFO:teuthology.orchestra.run.smithi136.stdout:Unit ntp.service is enabled and running 2024-09-12T23:23:16.514 INFO:teuthology.orchestra.run.smithi136.stdout:Host looks OK 2024-09-12T23:23:16.514 INFO:teuthology.orchestra.run.smithi136.stdout:Cluster fsid: da58a47c-715d-11ef-bceb-c7b262605968 2024-09-12T23:23:16.514 INFO:teuthology.orchestra.run.smithi136.stdout:Acquiring lock 140351158945104 on /run/cephadm/da58a47c-715d-11ef-bceb-c7b262605968.lock 2024-09-12T23:23:16.514 INFO:teuthology.orchestra.run.smithi136.stdout:Lock 140351158945104 acquired on /run/cephadm/da58a47c-715d-11ef-bceb-c7b262605968.lock 2024-09-12T23:23:16.514 INFO:teuthology.orchestra.run.smithi136.stdout:Verifying IP 172.21.15.136 port 3300 ... 2024-09-12T23:23:16.514 INFO:teuthology.orchestra.run.smithi136.stdout:Verifying IP 172.21.15.136 port 6789 ... 2024-09-12T23:23:16.514 INFO:teuthology.orchestra.run.smithi136.stdout:Base mon IP(s) is [172.21.15.136:3300, 172.21.15.136:6789], mon addrv is [v2:172.21.15.136:3300,v1:172.21.15.136:6789] 2024-09-12T23:23:16.517 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/sbin/ip: stdout default via 172.21.15.254 dev enp3s0f1 2024-09-12T23:23:16.517 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/sbin/ip: stdout default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.136 metric 100 2024-09-12T23:23:16.517 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/sbin/ip: stdout 158.69.67.47 via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.136 metric 100 2024-09-12T23:23:16.517 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/sbin/ip: stdout 172.17.0.0/16 dev docker0 proto kernel scope link src 172.17.0.1 linkdown 2024-09-12T23:23:16.517 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/sbin/ip: stdout 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.136 metric 100 2024-09-12T23:23:16.517 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/sbin/ip: stdout 172.21.0.1 dev enp3s0f1 proto dhcp scope link src 172.21.15.136 metric 100 2024-09-12T23:23:16.518 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/sbin/ip: stdout 172.21.0.2 dev enp3s0f1 proto dhcp scope link src 172.21.15.136 metric 100 2024-09-12T23:23:16.518 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/sbin/ip: stdout 172.21.15.254 dev enp3s0f1 proto dhcp scope link src 172.21.15.136 metric 100 2024-09-12T23:23:16.519 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2024-09-12T23:23:16.519 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/sbin/ip: stdout fe80::/64 dev enp3s0f1 proto kernel metric 256 pref medium 2024-09-12T23:23:16.519 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 100 expires 1796sec pref medium 2024-09-12T23:23:16.521 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-09-12T23:23:16.521 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/sbin/ip: stdout inet6 ::1/128 scope host 2024-09-12T23:23:16.521 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-09-12T23:23:16.521 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/sbin/ip: stdout 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-09-12T23:23:16.522 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/sbin/ip: stdout inet6 fe80::ec4:7aff:fe88:6e43/64 scope link 2024-09-12T23:23:16.522 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-09-12T23:23:16.523 INFO:teuthology.orchestra.run.smithi136.stdout:Mon IP `172.21.15.136` is in CIDR network `172.21.0.0/20` 2024-09-12T23:23:16.523 INFO:teuthology.orchestra.run.smithi136.stdout:Mon IP `172.21.15.136` is in CIDR network `172.21.0.0/20` 2024-09-12T23:23:16.523 INFO:teuthology.orchestra.run.smithi136.stdout:Mon IP `172.21.15.136` is in CIDR network `172.21.0.1/32` 2024-09-12T23:23:16.523 INFO:teuthology.orchestra.run.smithi136.stdout:Mon IP `172.21.15.136` is in CIDR network `172.21.0.1/32` 2024-09-12T23:23:16.523 INFO:teuthology.orchestra.run.smithi136.stdout:Mon IP `172.21.15.136` is in CIDR network `172.21.0.2/32` 2024-09-12T23:23:16.523 INFO:teuthology.orchestra.run.smithi136.stdout:Mon IP `172.21.15.136` is in CIDR network `172.21.0.2/32` 2024-09-12T23:23:16.523 INFO:teuthology.orchestra.run.smithi136.stdout:Mon IP `172.21.15.136` is in CIDR network `172.21.15.254/32` 2024-09-12T23:23:16.523 INFO:teuthology.orchestra.run.smithi136.stdout:Mon IP `172.21.15.136` is in CIDR network `172.21.15.254/32` 2024-09-12T23:23:16.523 INFO:teuthology.orchestra.run.smithi136.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20', '172.21.0.1/32', '172.21.0.1/32', '172.21.0.2/32', '172.21.0.2/32', '172.21.15.254/32', '172.21.15.254/32'] 2024-09-12T23:23:16.523 INFO:teuthology.orchestra.run.smithi136.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-09-12T23:23:16.524 INFO:teuthology.orchestra.run.smithi136.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-09-12T23:23:16.817 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/docker: stdout quincy: Pulling from ceph-ci/ceph 2024-09-12T23:23:16.817 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/docker: stdout Digest: sha256:9e84e422c5f3e13e9ae13013f0985fd8f54e85e87f1d2d371f86d595f20a9905 2024-09-12T23:23:16.818 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/docker: stdout Status: Image is up to date for quay.ceph.io/ceph-ci/ceph:quincy 2024-09-12T23:23:16.818 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/docker: stdout quay.ceph.io/ceph-ci/ceph:quincy 2024-09-12T23:23:18.392 INFO:teuthology.orchestra.run.smithi136.stdout:ceph: stdout ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable) 2024-09-12T23:23:18.393 INFO:teuthology.orchestra.run.smithi136.stdout:Ceph version: ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable) 2024-09-12T23:23:18.393 INFO:teuthology.orchestra.run.smithi136.stdout:Extracting ceph user uid/gid from container image... 2024-09-12T23:23:19.704 INFO:teuthology.orchestra.run.smithi136.stdout:stat: stdout 167 167 2024-09-12T23:23:19.704 INFO:teuthology.orchestra.run.smithi136.stdout:Creating initial keys... 2024-09-12T23:23:21.052 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-authtool: stdout AQDod+Nm2dE0HxAAf657NrEiGdZMX1ZYVPy77g== 2024-09-12T23:23:22.499 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-authtool: stdout AQDpd+NmpfudOhAAXLRkDraAe1RhIvkZH1VXMg== 2024-09-12T23:23:23.789 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-authtool: stdout AQDrd+Nm/lpEExAAiA5TWGfNkpStsvoJm2nIGQ== 2024-09-12T23:23:23.790 INFO:teuthology.orchestra.run.smithi136.stdout:Creating initial monmap... 2024-09-12T23:23:25.104 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-12T23:23:25.104 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = octopus 2024-09-12T23:23:25.104 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to da58a47c-715d-11ef-bceb-c7b262605968 2024-09-12T23:23:25.104 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-12T23:23:25.104 INFO:teuthology.orchestra.run.smithi136.stdout:monmaptool for a [v2:172.21.15.136:3300,v1:172.21.15.136:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-12T23:23:25.104 INFO:teuthology.orchestra.run.smithi136.stdout:setting min_mon_release = octopus 2024-09-12T23:23:25.105 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/monmaptool: set fsid to da58a47c-715d-11ef-bceb-c7b262605968 2024-09-12T23:23:25.105 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-12T23:23:25.105 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-12T23:23:25.105 INFO:teuthology.orchestra.run.smithi136.stdout:Creating mon... 2024-09-12T23:23:26.532 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.993+0000 7f80b1461900 0 set uid:gid to 167:167 (ceph:ceph) 2024-09-12T23:23:26.533 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 1 imported monmap: 2024-09-12T23:23:26.533 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr epoch 0 2024-09-12T23:23:26.533 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr fsid da58a47c-715d-11ef-bceb-c7b262605968 2024-09-12T23:23:26.533 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr last_changed 2024-09-12T23:23:24.585690+0000 2024-09-12T23:23:26.533 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr created 2024-09-12T23:23:24.585690+0000 2024-09-12T23:23:26.533 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr min_mon_release 15 (octopus) 2024-09-12T23:23:26.533 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr election_strategy: 1 2024-09-12T23:23:26.533 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr 0: [v2:172.21.15.136:3300/0,v1:172.21.15.136:6789/0] mon.a 2024-09-12T23:23:26.534 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr 2024-09-12T23:23:26.534 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 0 /usr/bin/ceph-mon: set fsid to da58a47c-715d-11ef-bceb-c7b262605968 2024-09-12T23:23:26.534 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: RocksDB version: 6.15.5 2024-09-12T23:23:26.534 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr 2024-09-12T23:23:26.534 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-12T23:23:26.534 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Compile date Sep 12 2024 2024-09-12T23:23:26.534 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: DB SUMMARY 2024-09-12T23:23:26.534 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr 2024-09-12T23:23:26.534 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: DB Session ID: CE9P84ZBAEB1TZ7SOPPH 2024-09-12T23:23:26.534 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr 2024-09-12T23:23:26.534 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 0, files: 2024-09-12T23:23:26.534 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr 2024-09-12T23:23:26.534 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 2024-09-12T23:23:26.535 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr 2024-09-12T23:23:26.535 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.error_if_exists: 0 2024-09-12T23:23:26.535 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.create_if_missing: 1 2024-09-12T23:23:26.535 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.paranoid_checks: 1 2024-09-12T23:23:26.535 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-12T23:23:26.535 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.env: 0x560c03ec9440 2024-09-12T23:23:26.535 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.fs: Posix File System 2024-09-12T23:23:26.535 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.info_log: 0x560c05390660 2024-09-12T23:23:26.535 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.max_file_opening_threads: 16 2024-09-12T23:23:26.535 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.statistics: (nil) 2024-09-12T23:23:26.535 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.use_fsync: 0 2024-09-12T23:23:26.535 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.max_log_file_size: 0 2024-09-12T23:23:26.535 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-12T23:23:26.536 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.log_file_time_to_roll: 0 2024-09-12T23:23:26.536 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.keep_log_file_num: 1000 2024-09-12T23:23:26.536 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.recycle_log_file_num: 0 2024-09-12T23:23:26.536 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.allow_fallocate: 1 2024-09-12T23:23:26.536 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.allow_mmap_reads: 0 2024-09-12T23:23:26.536 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.allow_mmap_writes: 0 2024-09-12T23:23:26.536 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.use_direct_reads: 0 2024-09-12T23:23:26.536 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-12T23:23:26.536 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.create_missing_column_families: 0 2024-09-12T23:23:26.536 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.db_log_dir: 2024-09-12T23:23:26.536 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-09-12T23:23:26.536 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.table_cache_numshardbits: 6 2024-09-12T23:23:26.536 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-09-12T23:23:26.537 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-09-12T23:23:26.537 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-12T23:23:26.537 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-12T23:23:26.537 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-09-12T23:23:26.537 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.advise_random_on_open: 1 2024-09-12T23:23:26.537 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.db_write_buffer_size: 0 2024-09-12T23:23:26.537 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.write_buffer_manager: 0x560c053d9e90 2024-09-12T23:23:26.537 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-12T23:23:26.537 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-12T23:23:26.537 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-12T23:23:26.537 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.use_adaptive_mutex: 0 2024-09-12T23:23:26.537 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.rate_limiter: (nil) 2024-09-12T23:23:26.537 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-12T23:23:26.538 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.wal_recovery_mode: 2 2024-09-12T23:23:26.538 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.enable_thread_tracking: 0 2024-09-12T23:23:26.538 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.enable_pipelined_write: 0 2024-09-12T23:23:26.538 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.unordered_write: 0 2024-09-12T23:23:26.538 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-12T23:23:26.538 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-12T23:23:26.538 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-12T23:23:26.538 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-12T23:23:26.538 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.row_cache: None 2024-09-12T23:23:26.538 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.wal_filter: None 2024-09-12T23:23:26.538 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-12T23:23:26.538 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.allow_ingest_behind: 0 2024-09-12T23:23:26.538 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.preserve_deletes: 0 2024-09-12T23:23:26.539 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.two_write_queues: 0 2024-09-12T23:23:26.539 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.manual_wal_flush: 0 2024-09-12T23:23:26.539 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.atomic_flush: 0 2024-09-12T23:23:26.539 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-12T23:23:26.539 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.persist_stats_to_disk: 0 2024-09-12T23:23:26.539 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-09-12T23:23:26.539 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.log_readahead_size: 0 2024-09-12T23:23:26.539 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-12T23:23:26.539 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.best_efforts_recovery: 0 2024-09-12T23:23:26.540 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-12T23:23:26.540 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-12T23:23:26.540 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.allow_data_in_errors: 0 2024-09-12T23:23:26.540 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.db_host_id: __hostname__ 2024-09-12T23:23:26.540 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.max_background_jobs: 2 2024-09-12T23:23:26.540 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.max_background_compactions: -1 2024-09-12T23:23:26.540 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.max_subcompactions: 1 2024-09-12T23:23:26.540 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-12T23:23:26.540 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-12T23:23:26.540 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.delayed_write_rate : 16777216 2024-09-12T23:23:26.540 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.max_total_wal_size: 0 2024-09-12T23:23:26.540 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-12T23:23:26.541 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.stats_dump_period_sec: 600 2024-09-12T23:23:26.541 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.stats_persist_period_sec: 600 2024-09-12T23:23:26.541 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-12T23:23:26.541 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.max_open_files: -1 2024-09-12T23:23:26.541 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.bytes_per_sync: 0 2024-09-12T23:23:26.541 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-09-12T23:23:26.541 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-09-12T23:23:26.541 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.compaction_readahead_size: 0 2024-09-12T23:23:26.541 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Options.max_background_flushes: -1 2024-09-12T23:23:26.541 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Compression algorithms supported: 2024-09-12T23:23:26.541 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: kZSTD supported: 0 2024-09-12T23:23:26.541 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: kXpressCompression supported: 0 2024-09-12T23:23:26.541 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: kBZip2Compression supported: 0 2024-09-12T23:23:26.541 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-12T23:23:26.542 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: kLZ4Compression supported: 1 2024-09-12T23:23:26.542 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: kZlibCompression supported: 1 2024-09-12T23:23:26.542 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: kLZ4HCCompression supported: 1 2024-09-12T23:23:26.542 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: kSnappyCompression supported: 1 2024-09-12T23:23:26.542 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-09-12T23:23:26.542 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:25.997+0000 7f80b1461900 4 rocksdb: [db/db_impl/db_impl_open.cc:281] Creating manifest 1 2024-09-12T23:23:26.542 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr 2024-09-12T23:23:26.542 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 2024-09-12T23:23:26.542 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr 2024-09-12T23:23:26.542 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-09-12T23:23:26.542 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr 2024-09-12T23:23:26.542 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-12T23:23:26.542 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.merge_operator: 2024-09-12T23:23:26.543 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.compaction_filter: None 2024-09-12T23:23:26.543 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.compaction_filter_factory: None 2024-09-12T23:23:26.543 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.sst_partitioner_factory: None 2024-09-12T23:23:26.543 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-09-12T23:23:26.543 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.table_factory: BlockBasedTable 2024-09-12T23:23:26.543 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x560c05330ee0) 2024-09-12T23:23:26.543 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr cache_index_and_filter_blocks: 1 2024-09-12T23:23:26.543 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr cache_index_and_filter_blocks_with_high_priority: 0 2024-09-12T23:23:26.543 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-12T23:23:26.543 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr pin_top_level_index_and_filter: 1 2024-09-12T23:23:26.543 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr index_type: 0 2024-09-12T23:23:26.543 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr data_block_index_type: 0 2024-09-12T23:23:26.543 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr index_shortening: 1 2024-09-12T23:23:26.544 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr data_block_hash_table_util_ratio: 0.750000 2024-09-12T23:23:26.544 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr hash_index_allow_collision: 1 2024-09-12T23:23:26.544 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr checksum: 1 2024-09-12T23:23:26.544 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr no_block_cache: 0 2024-09-12T23:23:26.545 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr block_cache: 0x560c053691f0 2024-09-12T23:23:26.545 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr block_cache_name: BinnedLRUCache 2024-09-12T23:23:26.545 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr block_cache_options: 2024-09-12T23:23:26.545 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr capacity : 536870912 2024-09-12T23:23:26.545 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr num_shard_bits : 4 2024-09-12T23:23:26.545 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr strict_capacity_limit : 0 2024-09-12T23:23:26.545 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr high_pri_pool_ratio: 0.000 2024-09-12T23:23:26.545 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr block_cache_compressed: (nil) 2024-09-12T23:23:26.545 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr persistent_cache: (nil) 2024-09-12T23:23:26.546 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr block_size: 4096 2024-09-12T23:23:26.546 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr block_size_deviation: 10 2024-09-12T23:23:26.546 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr block_restart_interval: 16 2024-09-12T23:23:26.546 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr index_block_restart_interval: 1 2024-09-12T23:23:26.546 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr metadata_block_size: 4096 2024-09-12T23:23:26.546 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr partition_filters: 0 2024-09-12T23:23:26.546 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr use_delta_encoding: 1 2024-09-12T23:23:26.546 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr filter_policy: rocksdb.BuiltinBloomFilter 2024-09-12T23:23:26.546 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr whole_key_filtering: 1 2024-09-12T23:23:26.546 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr verify_compression: 0 2024-09-12T23:23:26.546 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr read_amp_bytes_per_bit: 0 2024-09-12T23:23:26.546 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr format_version: 4 2024-09-12T23:23:26.546 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr enable_index_compression: 1 2024-09-12T23:23:26.547 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr block_align: 0 2024-09-12T23:23:26.547 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr 2024-09-12T23:23:26.547 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.write_buffer_size: 33554432 2024-09-12T23:23:26.547 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.max_write_buffer_number: 2 2024-09-12T23:23:26.547 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.compression: NoCompression 2024-09-12T23:23:26.547 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.bottommost_compression: Disabled 2024-09-12T23:23:26.547 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.prefix_extractor: nullptr 2024-09-12T23:23:26.547 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-12T23:23:26.547 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.num_levels: 7 2024-09-12T23:23:26.547 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-12T23:23:26.547 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-12T23:23:26.547 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-12T23:23:26.547 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-12T23:23:26.547 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-12T23:23:26.548 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-12T23:23:26.548 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-12T23:23:26.548 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-12T23:23:26.548 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-12T23:23:26.548 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-12T23:23:26.548 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.compression_opts.window_bits: -14 2024-09-12T23:23:26.548 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.compression_opts.level: 32767 2024-09-12T23:23:26.548 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.compression_opts.strategy: 0 2024-09-12T23:23:26.548 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-12T23:23:26.548 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-12T23:23:26.548 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-12T23:23:26.548 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.compression_opts.enabled: false 2024-09-12T23:23:26.549 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-12T23:23:26.549 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-12T23:23:26.549 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-12T23:23:26.549 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.target_file_size_base: 67108864 2024-09-12T23:23:26.549 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.target_file_size_multiplier: 1 2024-09-12T23:23:26.549 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-12T23:23:26.549 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-12T23:23:26.549 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-12T23:23:26.549 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-12T23:23:26.549 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-12T23:23:26.549 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-12T23:23:26.549 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-12T23:23:26.549 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-12T23:23:26.549 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-12T23:23:26.550 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-12T23:23:26.550 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-12T23:23:26.550 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-12T23:23:26.550 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.arena_block_size: 4194304 2024-09-12T23:23:26.550 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-12T23:23:26.550 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-12T23:23:26.550 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-12T23:23:26.550 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.disable_auto_compactions: 0 2024-09-12T23:23:26.550 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-12T23:23:26.550 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-12T23:23:26.550 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-12T23:23:26.550 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-12T23:23:26.550 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-12T23:23:26.551 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-12T23:23:26.551 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-12T23:23:26.551 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-12T23:23:26.551 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-12T23:23:26.551 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-12T23:23:26.551 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-12T23:23:26.551 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.inplace_update_support: 0 2024-09-12T23:23:26.551 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-09-12T23:23:26.551 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-12T23:23:26.551 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-12T23:23:26.551 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.memtable_huge_page_size: 0 2024-09-12T23:23:26.551 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.bloom_locality: 0 2024-09-12T23:23:26.551 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.max_successive_merges: 0 2024-09-12T23:23:26.551 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-09-12T23:23:26.552 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.paranoid_file_checks: 0 2024-09-12T23:23:26.552 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.force_consistency_checks: 1 2024-09-12T23:23:26.552 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.report_bg_io_stats: 0 2024-09-12T23:23:26.552 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.ttl: 2592000 2024-09-12T23:23:26.552 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-09-12T23:23:26.552 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.enable_blob_files: false 2024-09-12T23:23:26.552 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.min_blob_size: 0 2024-09-12T23:23:26.552 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.blob_file_size: 268435456 2024-09-12T23:23:26.552 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.blob_compression_type: NoCompression 2024-09-12T23:23:26.552 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.enable_blob_garbage_collection: false 2024-09-12T23:23:26.552 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-12T23:23:26.552 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-12T23:23:26.552 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr 2024-09-12T23:23:26.553 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2024-09-12T23:23:26.553 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr 2024-09-12T23:23:26.553 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: [db/version_set.cc:4082] Creating manifest 3 2024-09-12T23:23:26.553 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr 2024-09-12T23:23:26.553 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x560c0539a700 2024-09-12T23:23:26.553 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80b1461900 4 rocksdb: DB pointer 0x560c0536c000 2024-09-12T23:23:26.553 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80a8bef640 4 rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-09-12T23:23:26.553 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.001+0000 7f80a8bef640 4 rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-12T23:23:26.553 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr ** DB Stats ** 2024-09-12T23:23:26.553 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr Uptime(secs): 0.0 total, 0.0 interval 2024-09-12T23:23:26.553 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-12T23:23:26.553 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-12T23:23:26.553 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-12T23:23:26.553 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-12T23:23:26.554 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-12T23:23:26.554 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-12T23:23:26.554 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr 2024-09-12T23:23:26.554 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr ** Compaction Stats [default] ** 2024-09-12T23:23:26.554 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-12T23:23:26.554 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-12T23:23:26.554 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-12T23:23:26.554 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-12T23:23:26.554 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr 2024-09-12T23:23:26.554 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr ** Compaction Stats [default] ** 2024-09-12T23:23:26.554 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-12T23:23:26.554 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-12T23:23:26.554 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr Uptime(secs): 0.0 total, 0.0 interval 2024-09-12T23:23:26.554 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr Flush(GB): cumulative 0.000, interval 0.000 2024-09-12T23:23:26.555 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr AddFile(GB): cumulative 0.000, interval 0.000 2024-09-12T23:23:26.555 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr AddFile(Total Files): cumulative 0, interval 0 2024-09-12T23:23:26.555 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr AddFile(L0 Files): cumulative 0, interval 0 2024-09-12T23:23:26.555 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr AddFile(Keys): cumulative 0, interval 0 2024-09-12T23:23:26.555 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-12T23:23:26.555 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-12T23:23:26.555 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-12T23:23:26.555 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr 2024-09-12T23:23:26.555 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr ** File Read Latency Histogram By Level [default] ** 2024-09-12T23:23:26.555 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr 2024-09-12T23:23:26.555 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr ** Compaction Stats [default] ** 2024-09-12T23:23:26.555 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-12T23:23:26.555 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-12T23:23:26.555 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-12T23:23:26.556 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-12T23:23:26.556 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr 2024-09-12T23:23:26.556 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr ** Compaction Stats [default] ** 2024-09-12T23:23:26.556 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-12T23:23:26.556 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-12T23:23:26.556 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr Uptime(secs): 0.0 total, 0.0 interval 2024-09-12T23:23:26.556 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr Flush(GB): cumulative 0.000, interval 0.000 2024-09-12T23:23:26.556 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr AddFile(GB): cumulative 0.000, interval 0.000 2024-09-12T23:23:26.556 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr AddFile(Total Files): cumulative 0, interval 0 2024-09-12T23:23:26.556 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr AddFile(L0 Files): cumulative 0, interval 0 2024-09-12T23:23:26.556 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr AddFile(Keys): cumulative 0, interval 0 2024-09-12T23:23:26.556 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-12T23:23:26.556 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-12T23:23:26.557 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-12T23:23:26.557 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr 2024-09-12T23:23:26.557 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr ** File Read Latency Histogram By Level [default] ** 2024-09-12T23:23:26.557 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr 2024-09-12T23:23:26.557 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.005+0000 7f80b1461900 4 rocksdb: [db/db_impl/db_impl.cc:446] Shutdown: canceling all background work 2024-09-12T23:23:26.557 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.005+0000 7f80b1461900 4 rocksdb: [db/db_impl/db_impl.cc:625] Shutdown complete 2024-09-12T23:23:26.557 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-12T23:23:26.005+0000 7f80b1461900 0 /usr/bin/ceph-mon: created monfs at /var/lib/ceph/mon/ceph-a for mon.a 2024-09-12T23:23:26.557 INFO:teuthology.orchestra.run.smithi136.stdout:create mon.a on 2024-09-12T23:23:26.747 INFO:teuthology.orchestra.run.smithi136.stdout:systemctl: stderr Removed /etc/systemd/system/multi-user.target.wants/ceph.target. 2024-09-12T23:23:26.937 INFO:teuthology.orchestra.run.smithi136.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /etc/systemd/system/ceph.target. 2024-09-12T23:23:27.146 INFO:teuthology.orchestra.run.smithi136.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-da58a47c-715d-11ef-bceb-c7b262605968.target -> /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968.target. 2024-09-12T23:23:27.146 INFO:teuthology.orchestra.run.smithi136.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-da58a47c-715d-11ef-bceb-c7b262605968.target -> /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968.target. 2024-09-12T23:23:27.367 INFO:teuthology.orchestra.run.smithi136.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-da58a47c-715d-11ef-bceb-c7b262605968@mon.a 2024-09-12T23:23:27.368 INFO:teuthology.orchestra.run.smithi136.stdout:systemctl: stderr Failed to reset failed state of unit ceph-da58a47c-715d-11ef-bceb-c7b262605968@mon.a.service: Unit ceph-da58a47c-715d-11ef-bceb-c7b262605968@mon.a.service not loaded. 2024-09-12T23:23:27.573 INFO:teuthology.orchestra.run.smithi136.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968.target.wants/ceph-da58a47c-715d-11ef-bceb-c7b262605968@mon.a.service -> /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service. 2024-09-12T23:23:27.581 INFO:teuthology.orchestra.run.smithi136.stdout:firewalld does not appear to be present 2024-09-12T23:23:27.581 INFO:teuthology.orchestra.run.smithi136.stdout:Not possible to enable service . firewalld.service is not available 2024-09-12T23:23:27.581 INFO:teuthology.orchestra.run.smithi136.stdout:Waiting for mon to start... 2024-09-12T23:23:27.581 INFO:teuthology.orchestra.run.smithi136.stdout:Waiting for mon... 2024-09-12T23:23:27.722 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:27 smithi136 systemd[1]: Started Ceph mon.a for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:23:29.191 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:29 smithi136 bash[22122]: cluster 2024-09-12T23:23:29.001065+0000 mon.a (mon.0) 0 : cluster [INF] mkfs da58a47c-715d-11ef-bceb-c7b262605968 2024-09-12T23:23:29.191 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:29 smithi136 bash[22122]: cluster 2024-09-12T23:23:28.986482+0000 mon.a (mon.0) 1 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2024-09-12T23:23:29.715 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout cluster: 2024-09-12T23:23:29.716 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout id: da58a47c-715d-11ef-bceb-c7b262605968 2024-09-12T23:23:29.716 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2024-09-12T23:23:29.716 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout 2024-09-12T23:23:29.716 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout services: 2024-09-12T23:23:29.716 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.18414s) 2024-09-12T23:23:29.716 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2024-09-12T23:23:29.716 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2024-09-12T23:23:29.716 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout 2024-09-12T23:23:29.716 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout data: 2024-09-12T23:23:29.716 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2024-09-12T23:23:29.716 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2024-09-12T23:23:29.717 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2024-09-12T23:23:29.717 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout pgs: 2024-09-12T23:23:29.717 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout 2024-09-12T23:23:29.717 INFO:teuthology.orchestra.run.smithi136.stdout:mon is available 2024-09-12T23:23:29.717 INFO:teuthology.orchestra.run.smithi136.stdout:Assimilating anything we can from ceph.conf... 2024-09-12T23:23:30.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:30 smithi136 bash[22122]: cluster 2024-09-12T23:23:28.998046+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2024-09-12T23:23:30.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:30 smithi136 bash[22122]: cluster 2024-09-12T23:23:28.998219+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e1: 1 mons at {a=[v2:172.21.15.136:3300/0,v1:172.21.15.136:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-12T23:23:30.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:30 smithi136 bash[22122]: cluster 2024-09-12T23:23:29.002516+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap 2024-09-12T23:23:30.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:30 smithi136 bash[22122]: cluster 2024-09-12T23:23:29.004697+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2024-09-12T23:23:30.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:30 smithi136 bash[22122]: cluster 2024-09-12T23:23:29.005132+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e1: no daemons active 2024-09-12T23:23:30.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:30 smithi136 bash[22122]: audit 2024-09-12T23:23:29.182128+0000 mon.a (mon.0) 7 : audit [DBG] from='client.? 172.21.15.136:0/3628989682' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-12T23:23:31.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:31 smithi136 bash[22122]: audit 2024-09-12T23:23:30.806721+0000 mon.a (mon.0) 8 : audit [INF] from='client.? 172.21.15.136:0/3016992511' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-12T23:23:31.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:31 smithi136 bash[22122]: audit 2024-09-12T23:23:30.812276+0000 mon.a (mon.0) 9 : audit [INF] from='client.? 172.21.15.136:0/3016992511' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-09-12T23:23:31.282 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout 2024-09-12T23:23:31.282 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout [global] 2024-09-12T23:23:31.282 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout fsid = da58a47c-715d-11ef-bceb-c7b262605968 2024-09-12T23:23:31.282 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout mon_cluster_log_file_level = debug 2024-09-12T23:23:31.282 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.136:3300,v1:172.21.15.136:6789] 2024-09-12T23:23:31.282 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-09-12T23:23:31.282 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-09-12T23:23:31.283 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-09-12T23:23:31.283 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-09-12T23:23:31.283 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout 2024-09-12T23:23:31.283 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout [mgr] 2024-09-12T23:23:31.283 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-09-12T23:23:31.283 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout 2024-09-12T23:23:31.283 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout [osd] 2024-09-12T23:23:31.283 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-09-12T23:23:31.283 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-09-12T23:23:31.283 INFO:teuthology.orchestra.run.smithi136.stdout:Generating new minimal ceph.conf... 2024-09-12T23:23:32.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:32 smithi136 bash[22122]: audit 2024-09-12T23:23:32.357578+0000 mon.a (mon.0) 10 : audit [DBG] from='client.? 172.21.15.136:0/1903211511' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:23:32.894 INFO:teuthology.orchestra.run.smithi136.stdout:Restarting the monitor... 2024-09-12T23:23:33.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:32 smithi136 systemd[1]: Stopping Ceph mon.a for da58a47c-715d-11ef-bceb-c7b262605968... 2024-09-12T23:23:33.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:33 smithi136 bash[22122]: debug 2024-09-12T23:23:33.025+0000 7fdd57537640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-09-12T23:23:33.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:33 smithi136 bash[22122]: debug 2024-09-12T23:23:33.025+0000 7fdd57537640 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-09-12T23:23:33.804 INFO:teuthology.orchestra.run.smithi136.stdout:Setting public_network to 172.21.0.0/20,172.21.0.1/32,172.21.15.254/32,172.21.0.2/32 in mon config section 2024-09-12T23:23:33.938 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:33 smithi136 bash[22505]: ceph-da58a47c-715d-11ef-bceb-c7b262605968-mon-a 2024-09-12T23:23:33.938 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:33 smithi136 bash[22565]: Error response from daemon: No such container: ceph-da58a47c-715d-11ef-bceb-c7b262605968-mon-a 2024-09-12T23:23:33.938 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:33 smithi136 systemd[1]: ceph-da58a47c-715d-11ef-bceb-c7b262605968@mon.a.service: Deactivated successfully. 2024-09-12T23:23:33.939 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:33 smithi136 systemd[1]: Stopped Ceph mon.a for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:23:33.939 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:33 smithi136 systemd[1]: Started Ceph mon.a for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:23:35.452 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.233+0000 7ff5e313d900 0 set uid:gid to 167:167 (ceph:ceph) 2024-09-12T23:23:35.452 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.233+0000 7ff5e313d900 0 ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable), process ceph-mon, pid 7 2024-09-12T23:23:35.452 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.233+0000 7ff5e313d900 0 pidfile_write: ignore empty --pid-file 2024-09-12T23:23:35.452 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.233+0000 7ff5e313d900 0 load: jerasure load: lrc 2024-09-12T23:23:35.452 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: RocksDB version: 6.15.5 2024-09-12T23:23:35.453 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-12T23:23:35.453 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Compile date Sep 12 2024 2024-09-12T23:23:35.453 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: DB SUMMARY 2024-09-12T23:23:35.453 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: DB Session ID: H8TK4Z3TRN7Q066YZJGR 2024-09-12T23:23:35.453 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: CURRENT file: CURRENT 2024-09-12T23:23:35.453 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: IDENTITY file: IDENTITY 2024-09-12T23:23:35.453 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-09-12T23:23:35.453 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-09-12T23:23:35.453 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 83189 ; 2024-09-12T23:23:35.453 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.error_if_exists: 0 2024-09-12T23:23:35.453 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.create_if_missing: 0 2024-09-12T23:23:35.453 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.paranoid_checks: 1 2024-09-12T23:23:35.453 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-12T23:23:35.454 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.env: 0x55b4c4621440 2024-09-12T23:23:35.454 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.fs: Posix File System 2024-09-12T23:23:35.454 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.info_log: 0x55b4c54f6ee0 2024-09-12T23:23:35.454 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.max_file_opening_threads: 16 2024-09-12T23:23:35.454 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.statistics: (nil) 2024-09-12T23:23:35.454 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.use_fsync: 0 2024-09-12T23:23:35.454 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.max_log_file_size: 0 2024-09-12T23:23:35.454 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-12T23:23:35.454 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.log_file_time_to_roll: 0 2024-09-12T23:23:35.454 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.keep_log_file_num: 1000 2024-09-12T23:23:35.454 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.recycle_log_file_num: 0 2024-09-12T23:23:35.454 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.allow_fallocate: 1 2024-09-12T23:23:35.454 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.allow_mmap_reads: 0 2024-09-12T23:23:35.455 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.allow_mmap_writes: 0 2024-09-12T23:23:35.455 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.use_direct_reads: 0 2024-09-12T23:23:35.455 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-12T23:23:35.455 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.create_missing_column_families: 0 2024-09-12T23:23:35.455 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.db_log_dir: 2024-09-12T23:23:35.455 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-09-12T23:23:35.455 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.table_cache_numshardbits: 6 2024-09-12T23:23:35.455 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-09-12T23:23:35.455 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-09-12T23:23:35.455 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-12T23:23:35.455 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-12T23:23:35.455 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-09-12T23:23:35.456 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.advise_random_on_open: 1 2024-09-12T23:23:35.456 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.db_write_buffer_size: 0 2024-09-12T23:23:35.456 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.write_buffer_manager: 0x55b4c55b4780 2024-09-12T23:23:35.456 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-12T23:23:35.456 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-12T23:23:35.456 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-12T23:23:35.456 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.use_adaptive_mutex: 0 2024-09-12T23:23:35.456 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.rate_limiter: (nil) 2024-09-12T23:23:35.456 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-12T23:23:35.456 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.wal_recovery_mode: 2 2024-09-12T23:23:35.456 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.enable_thread_tracking: 0 2024-09-12T23:23:35.458 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.enable_pipelined_write: 0 2024-09-12T23:23:35.458 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.unordered_write: 0 2024-09-12T23:23:35.458 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-12T23:23:35.458 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-12T23:23:35.458 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-12T23:23:35.458 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-12T23:23:35.458 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.row_cache: None 2024-09-12T23:23:35.458 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.wal_filter: None 2024-09-12T23:23:35.458 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-12T23:23:35.458 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.allow_ingest_behind: 0 2024-09-12T23:23:35.459 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.preserve_deletes: 0 2024-09-12T23:23:35.459 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.two_write_queues: 0 2024-09-12T23:23:35.459 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.manual_wal_flush: 0 2024-09-12T23:23:35.459 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.atomic_flush: 0 2024-09-12T23:23:35.459 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-12T23:23:35.459 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.persist_stats_to_disk: 0 2024-09-12T23:23:35.459 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-09-12T23:23:35.459 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.log_readahead_size: 0 2024-09-12T23:23:35.459 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-12T23:23:35.459 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.best_efforts_recovery: 0 2024-09-12T23:23:35.459 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-12T23:23:35.459 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-12T23:23:35.459 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.allow_data_in_errors: 0 2024-09-12T23:23:35.459 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.db_host_id: __hostname__ 2024-09-12T23:23:35.460 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.max_background_jobs: 2 2024-09-12T23:23:35.460 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.max_background_compactions: -1 2024-09-12T23:23:35.460 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.max_subcompactions: 1 2024-09-12T23:23:35.460 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-12T23:23:35.460 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-12T23:23:35.460 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.delayed_write_rate : 16777216 2024-09-12T23:23:35.460 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.max_total_wal_size: 0 2024-09-12T23:23:35.460 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-12T23:23:35.460 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.stats_dump_period_sec: 600 2024-09-12T23:23:35.460 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.stats_persist_period_sec: 600 2024-09-12T23:23:35.460 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-12T23:23:35.460 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.max_open_files: -1 2024-09-12T23:23:35.460 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.bytes_per_sync: 0 2024-09-12T23:23:35.461 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-09-12T23:23:35.461 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-09-12T23:23:35.461 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.compaction_readahead_size: 0 2024-09-12T23:23:35.461 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.max_background_flushes: -1 2024-09-12T23:23:35.461 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Compression algorithms supported: 2024-09-12T23:23:35.461 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: kZSTD supported: 0 2024-09-12T23:23:35.461 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: kXpressCompression supported: 0 2024-09-12T23:23:35.461 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: kBZip2Compression supported: 0 2024-09-12T23:23:35.461 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-12T23:23:35.461 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: kLZ4Compression supported: 1 2024-09-12T23:23:35.461 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: kZlibCompression supported: 1 2024-09-12T23:23:35.461 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: kLZ4HCCompression supported: 1 2024-09-12T23:23:35.461 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: kSnappyCompression supported: 1 2024-09-12T23:23:35.462 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-09-12T23:23:35.462 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2024-09-12T23:23:35.462 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-09-12T23:23:35.462 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-12T23:23:35.462 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.merge_operator: 2024-09-12T23:23:35.462 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.compaction_filter: None 2024-09-12T23:23:35.462 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.compaction_filter_factory: None 2024-09-12T23:23:35.462 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.sst_partitioner_factory: None 2024-09-12T23:23:35.462 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-09-12T23:23:35.462 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.table_factory: BlockBasedTable 2024-09-12T23:23:35.462 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55b4c549fa38) 2024-09-12T23:23:35.462 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: cache_index_and_filter_blocks: 1 2024-09-12T23:23:35.462 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-12T23:23:35.462 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-12T23:23:35.463 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: pin_top_level_index_and_filter: 1 2024-09-12T23:23:35.463 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: index_type: 0 2024-09-12T23:23:35.463 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: data_block_index_type: 0 2024-09-12T23:23:35.463 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: index_shortening: 1 2024-09-12T23:23:35.463 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: data_block_hash_table_util_ratio: 0.750000 2024-09-12T23:23:35.463 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: hash_index_allow_collision: 1 2024-09-12T23:23:35.463 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: checksum: 1 2024-09-12T23:23:35.463 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: no_block_cache: 0 2024-09-12T23:23:35.463 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: block_cache: 0x55b4c54d3350 2024-09-12T23:23:35.463 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: block_cache_name: BinnedLRUCache 2024-09-12T23:23:35.463 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: block_cache_options: 2024-09-12T23:23:35.463 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: capacity : 536870912 2024-09-12T23:23:35.463 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: num_shard_bits : 4 2024-09-12T23:23:35.463 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: strict_capacity_limit : 0 2024-09-12T23:23:35.464 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: high_pri_pool_ratio: 0.000 2024-09-12T23:23:35.464 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: block_cache_compressed: (nil) 2024-09-12T23:23:35.464 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: persistent_cache: (nil) 2024-09-12T23:23:35.464 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: block_size: 4096 2024-09-12T23:23:35.464 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: block_size_deviation: 10 2024-09-12T23:23:35.464 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: block_restart_interval: 16 2024-09-12T23:23:35.464 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: index_block_restart_interval: 1 2024-09-12T23:23:35.464 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: metadata_block_size: 4096 2024-09-12T23:23:35.464 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: partition_filters: 0 2024-09-12T23:23:35.464 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: use_delta_encoding: 1 2024-09-12T23:23:35.464 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-12T23:23:35.464 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: whole_key_filtering: 1 2024-09-12T23:23:35.464 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: verify_compression: 0 2024-09-12T23:23:35.465 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: read_amp_bytes_per_bit: 0 2024-09-12T23:23:35.465 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: format_version: 4 2024-09-12T23:23:35.465 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: enable_index_compression: 1 2024-09-12T23:23:35.465 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: block_align: 0 2024-09-12T23:23:35.465 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.write_buffer_size: 33554432 2024-09-12T23:23:35.465 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.max_write_buffer_number: 2 2024-09-12T23:23:35.465 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.compression: NoCompression 2024-09-12T23:23:35.465 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.bottommost_compression: Disabled 2024-09-12T23:23:35.465 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.prefix_extractor: nullptr 2024-09-12T23:23:35.465 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-12T23:23:35.466 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.num_levels: 7 2024-09-12T23:23:35.466 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-12T23:23:35.466 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-12T23:23:35.466 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-12T23:23:35.466 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-12T23:23:35.466 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-12T23:23:35.466 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-12T23:23:35.466 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-12T23:23:35.466 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-12T23:23:35.466 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-12T23:23:35.466 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-12T23:23:35.466 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.compression_opts.window_bits: -14 2024-09-12T23:23:35.466 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.compression_opts.level: 32767 2024-09-12T23:23:35.466 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.compression_opts.strategy: 0 2024-09-12T23:23:35.467 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-12T23:23:35.467 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-12T23:23:35.467 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-12T23:23:35.467 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.compression_opts.enabled: false 2024-09-12T23:23:35.467 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-12T23:23:35.467 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-12T23:23:35.467 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-12T23:23:35.467 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.target_file_size_base: 67108864 2024-09-12T23:23:35.467 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.target_file_size_multiplier: 1 2024-09-12T23:23:35.467 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-12T23:23:35.467 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-12T23:23:35.467 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-12T23:23:35.467 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-12T23:23:35.467 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-12T23:23:35.468 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-12T23:23:35.468 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-12T23:23:35.468 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-12T23:23:35.468 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-12T23:23:35.468 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-12T23:23:35.468 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-12T23:23:35.468 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-12T23:23:35.468 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.arena_block_size: 4194304 2024-09-12T23:23:35.468 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-12T23:23:35.469 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-12T23:23:35.469 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-12T23:23:35.469 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.disable_auto_compactions: 0 2024-09-12T23:23:35.469 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-12T23:23:35.469 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-12T23:23:35.469 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-12T23:23:35.469 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-12T23:23:35.469 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-12T23:23:35.469 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-12T23:23:35.469 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-12T23:23:35.469 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-12T23:23:35.469 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-12T23:23:35.469 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-12T23:23:35.469 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-12T23:23:35.470 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.inplace_update_support: 0 2024-09-12T23:23:35.470 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-09-12T23:23:35.470 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-12T23:23:35.470 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-12T23:23:35.470 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.memtable_huge_page_size: 0 2024-09-12T23:23:35.470 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.bloom_locality: 0 2024-09-12T23:23:35.470 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.max_successive_merges: 0 2024-09-12T23:23:35.470 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-09-12T23:23:35.470 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.paranoid_file_checks: 0 2024-09-12T23:23:35.470 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.force_consistency_checks: 1 2024-09-12T23:23:35.470 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.report_bg_io_stats: 0 2024-09-12T23:23:35.470 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.ttl: 2592000 2024-09-12T23:23:35.470 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-09-12T23:23:35.470 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.enable_blob_files: false 2024-09-12T23:23:35.471 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.min_blob_size: 0 2024-09-12T23:23:35.471 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.blob_file_size: 268435456 2024-09-12T23:23:35.471 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.blob_compression_type: NoCompression 2024-09-12T23:23:35.471 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.enable_blob_garbage_collection: false 2024-09-12T23:23:35.471 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-12T23:23:35.471 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-12T23:23:35.471 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 5 2024-09-12T23:23:35.471 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: [db/version_set.cc:4082] Creating manifest 13 2024-09-12T23:23:35.471 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726183415243183, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-09-12T23:23:35.471 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #10 mode 2 2024-09-12T23:23:35.471 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 3 rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-12T23:23:35.471 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726183415244045, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79780, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 78019, "index_size": 223, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 9999, "raw_average_key_size": 47, "raw_value_size": 72339, "raw_average_value_size": 342, "num_data_blocks": 10, "num_entries": 211, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726183415, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "a6cd1e2e-cca4-4d2f-8e18-1a5c2d73d75a", "db_session_id": "H8TK4Z3TRN7Q066YZJGR"}} 2024-09-12T23:23:35.471 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: [db/version_set.cc:4082] Creating manifest 15 2024-09-12T23:23:35.472 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.237+0000 7ff5e313d900 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726183415244602, "job": 1, "event": "recovery_finished"} 2024-09-12T23:23:35.472 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.241+0000 7ff5e313d900 4 rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-12T23:23:35.472 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.241+0000 7ff5e313d900 4 rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x55b4c5508700 2024-09-12T23:23:35.472 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.241+0000 7ff5e313d900 4 rocksdb: DB pointer 0x55b4c54d6000 2024-09-12T23:23:35.472 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.241+0000 7ff5e313d900 0 starting mon.a rank 0 at public addrs [v2:172.21.15.136:3300/0,v1:172.21.15.136:6789/0] at bind addrs [v2:172.21.15.136:3300/0,v1:172.21.15.136:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid da58a47c-715d-11ef-bceb-c7b262605968 2024-09-12T23:23:35.472 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.241+0000 7ff5e313d900 1 mon.a@-1(???) e1 preinit fsid da58a47c-715d-11ef-bceb-c7b262605968 2024-09-12T23:23:35.472 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.245+0000 7ff5e313d900 0 mon.a@-1(???).mds e1 new map 2024-09-12T23:23:35.472 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.245+0000 7ff5e313d900 0 mon.a@-1(???).mds e1 print_map 2024-09-12T23:23:35.473 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: e1 2024-09-12T23:23:35.473 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: enable_multiple, ever_enabled_multiple: 1,1 2024-09-12T23:23:35.473 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-12T23:23:35.473 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: legacy client fscid: -1 2024-09-12T23:23:35.473 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: 2024-09-12T23:23:35.473 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: No filesystems configured 2024-09-12T23:23:35.473 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.245+0000 7ff5d8f09640 4 rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-09-12T23:23:35.473 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.245+0000 7ff5d8f09640 4 rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-12T23:23:35.474 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: ** DB Stats ** 2024-09-12T23:23:35.474 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: Uptime(secs): 0.0 total, 0.0 interval 2024-09-12T23:23:35.474 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-12T23:23:35.474 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-12T23:23:35.474 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-12T23:23:35.474 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-12T23:23:35.474 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-12T23:23:35.474 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-12T23:23:35.474 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: ** Compaction Stats [default] ** 2024-09-12T23:23:35.474 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-12T23:23:35.474 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-12T23:23:35.474 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: L0 2/0 79.70 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 135.6 0.00 0.00 1 0.001 0 0 2024-09-12T23:23:35.474 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: Sum 2/0 79.70 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 135.6 0.00 0.00 1 0.001 0 0 2024-09-12T23:23:35.475 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 135.6 0.00 0.00 1 0.001 0 0 2024-09-12T23:23:35.475 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: ** Compaction Stats [default] ** 2024-09-12T23:23:35.475 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-12T23:23:35.475 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-12T23:23:35.475 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 135.6 0.00 0.00 1 0.001 0 0 2024-09-12T23:23:35.475 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: Uptime(secs): 0.0 total, 0.0 interval 2024-09-12T23:23:35.475 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: Flush(GB): cumulative 0.000, interval 0.000 2024-09-12T23:23:35.475 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-12T23:23:35.475 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: AddFile(Total Files): cumulative 0, interval 0 2024-09-12T23:23:35.475 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: AddFile(L0 Files): cumulative 0, interval 0 2024-09-12T23:23:35.475 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: AddFile(Keys): cumulative 0, interval 0 2024-09-12T23:23:35.475 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: Cumulative compaction: 0.00 GB write, 10.57 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-12T23:23:35.475 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: Interval compaction: 0.00 GB write, 10.57 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-12T23:23:35.475 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-12T23:23:35.476 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: ** File Read Latency Histogram By Level [default] ** 2024-09-12T23:23:35.476 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: ** Compaction Stats [default] ** 2024-09-12T23:23:35.476 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-12T23:23:35.476 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-12T23:23:35.476 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: L0 2/0 79.70 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 135.6 0.00 0.00 1 0.001 0 0 2024-09-12T23:23:35.476 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: Sum 2/0 79.70 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 135.6 0.00 0.00 1 0.001 0 0 2024-09-12T23:23:35.476 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-12T23:23:35.476 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: ** Compaction Stats [default] ** 2024-09-12T23:23:35.476 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-12T23:23:35.476 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-12T23:23:35.476 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 135.6 0.00 0.00 1 0.001 0 0 2024-09-12T23:23:35.476 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: Uptime(secs): 0.0 total, 0.0 interval 2024-09-12T23:23:35.476 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: Flush(GB): cumulative 0.000, interval 0.000 2024-09-12T23:23:35.477 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-12T23:23:35.477 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: AddFile(Total Files): cumulative 0, interval 0 2024-09-12T23:23:35.477 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: AddFile(L0 Files): cumulative 0, interval 0 2024-09-12T23:23:35.477 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: AddFile(Keys): cumulative 0, interval 0 2024-09-12T23:23:35.477 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: Cumulative compaction: 0.00 GB write, 10.49 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-12T23:23:35.477 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-12T23:23:35.477 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-12T23:23:35.477 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: ** File Read Latency Histogram By Level [default] ** 2024-09-12T23:23:35.477 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.245+0000 7ff5e313d900 0 mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-12T23:23:35.477 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.245+0000 7ff5e313d900 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-12T23:23:35.477 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.245+0000 7ff5e313d900 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-12T23:23:35.477 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.245+0000 7ff5e313d900 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-12T23:23:35.477 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: debug 2024-09-12T23:23:35.245+0000 7ff5e313d900 1 mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-09-12T23:23:35.477 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: cluster 2024-09-12T23:23:35.254114+0000 mon.a (mon.0) 1 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2024-09-12T23:23:35.478 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: cluster 2024-09-12T23:23:35.254203+0000 mon.a (mon.0) 2 : cluster [DBG] monmap e1: 1 mons at {a=[v2:172.21.15.136:3300/0,v1:172.21.15.136:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-12T23:23:35.478 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: cluster 2024-09-12T23:23:35.254782+0000 mon.a (mon.0) 3 : cluster [DBG] fsmap 2024-09-12T23:23:35.478 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: cluster 2024-09-12T23:23:35.254823+0000 mon.a (mon.0) 4 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2024-09-12T23:23:35.478 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:35 smithi136 bash[22601]: cluster 2024-09-12T23:23:35.256101+0000 mon.a (mon.0) 5 : cluster [DBG] mgrmap e1: no daemons active 2024-09-12T23:23:36.016 INFO:teuthology.orchestra.run.smithi136.stdout:Wrote config to /etc/ceph/ceph.conf 2024-09-12T23:23:36.016 INFO:teuthology.orchestra.run.smithi136.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-09-12T23:23:36.016 INFO:teuthology.orchestra.run.smithi136.stdout:Creating mgr... 2024-09-12T23:23:36.017 INFO:teuthology.orchestra.run.smithi136.stdout:Verifying port 0.0.0.0:9283 ... 2024-09-12T23:23:36.017 INFO:teuthology.orchestra.run.smithi136.stdout:Verifying port 0.0.0.0:8765 ... 2024-09-12T23:23:36.233 INFO:teuthology.orchestra.run.smithi136.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-da58a47c-715d-11ef-bceb-c7b262605968@mgr.y 2024-09-12T23:23:36.233 INFO:teuthology.orchestra.run.smithi136.stdout:systemctl: stderr Failed to reset failed state of unit ceph-da58a47c-715d-11ef-bceb-c7b262605968@mgr.y.service: Unit ceph-da58a47c-715d-11ef-bceb-c7b262605968@mgr.y.service not loaded. 2024-09-12T23:23:36.373 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:36 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:23:36.461 INFO:teuthology.orchestra.run.smithi136.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968.target.wants/ceph-da58a47c-715d-11ef-bceb-c7b262605968@mgr.y.service -> /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service. 2024-09-12T23:23:36.468 INFO:teuthology.orchestra.run.smithi136.stdout:firewalld does not appear to be present 2024-09-12T23:23:36.469 INFO:teuthology.orchestra.run.smithi136.stdout:Not possible to enable service . firewalld.service is not available 2024-09-12T23:23:36.469 INFO:teuthology.orchestra.run.smithi136.stdout:firewalld does not appear to be present 2024-09-12T23:23:36.469 INFO:teuthology.orchestra.run.smithi136.stdout:Not possible to open ports <[9283, 8765]>. firewalld.service is not available 2024-09-12T23:23:36.469 INFO:teuthology.orchestra.run.smithi136.stdout:Waiting for mgr to start... 2024-09-12T23:23:36.469 INFO:teuthology.orchestra.run.smithi136.stdout:Waiting for mgr... 2024-09-12T23:23:36.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:36 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:23:36.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:36 smithi136 bash[22601]: audit 2024-09-12T23:23:35.441394+0000 mon.a (mon.0) 6 : audit [INF] from='client.? 172.21.15.136:0/251755390' entity='client.admin' 2024-09-12T23:23:38.111 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:38 smithi136 bash[22852]: debug 2024-09-12T23:23:38.032+0000 7f04d886bec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-12T23:23:38.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:38 smithi136 bash[22601]: audit 2024-09-12T23:23:38.101975+0000 mon.a (mon.0) 7 : audit [DBG] from='client.? 172.21.15.136:0/1319920617' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-12T23:23:38.621 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout 2024-09-12T23:23:38.621 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout { 2024-09-12T23:23:38.621 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "fsid": "da58a47c-715d-11ef-bceb-c7b262605968", 2024-09-12T23:23:38.621 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "health": { 2024-09-12T23:23:38.621 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-09-12T23:23:38.621 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-09-12T23:23:38.621 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-09-12T23:23:38.621 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout }, 2024-09-12T23:23:38.621 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-09-12T23:23:38.621 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-09-12T23:23:38.621 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout 0 2024-09-12T23:23:38.622 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout ], 2024-09-12T23:23:38.622 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-09-12T23:23:38.622 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "a" 2024-09-12T23:23:38.622 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout ], 2024-09-12T23:23:38.622 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "quorum_age": 2, 2024-09-12T23:23:38.622 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "monmap": { 2024-09-12T23:23:38.622 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-12T23:23:38.622 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-09-12T23:23:38.622 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-09-12T23:23:38.622 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout }, 2024-09-12T23:23:38.622 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-09-12T23:23:38.622 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-12T23:23:38.622 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-09-12T23:23:38.622 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-09-12T23:23:38.623 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-09-12T23:23:38.623 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-09-12T23:23:38.623 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-09-12T23:23:38.623 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-09-12T23:23:38.626 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout }, 2024-09-12T23:23:38.626 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-09-12T23:23:38.626 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-09-12T23:23:38.626 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-09-12T23:23:38.626 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-09-12T23:23:38.626 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-09-12T23:23:38.626 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-09-12T23:23:38.626 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-09-12T23:23:38.627 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-09-12T23:23:38.627 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-09-12T23:23:38.627 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout }, 2024-09-12T23:23:38.627 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-09-12T23:23:38.627 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-12T23:23:38.627 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-09-12T23:23:38.627 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-09-12T23:23:38.627 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout }, 2024-09-12T23:23:38.627 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-09-12T23:23:38.627 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "available": false, 2024-09-12T23:23:38.627 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-09-12T23:23:38.627 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "modules": [ 2024-09-12T23:23:38.627 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "iostat", 2024-09-12T23:23:38.627 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "nfs", 2024-09-12T23:23:38.627 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "restful" 2024-09-12T23:23:38.628 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout ], 2024-09-12T23:23:38.628 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-12T23:23:38.628 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout }, 2024-09-12T23:23:38.628 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-09-12T23:23:38.628 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-12T23:23:38.628 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "modified": "2024-09-12T23:23:29.001436+0000", 2024-09-12T23:23:38.628 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-12T23:23:38.628 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout }, 2024-09-12T23:23:38.628 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-09-12T23:23:38.628 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout } 2024-09-12T23:23:38.628 INFO:teuthology.orchestra.run.smithi136.stdout:mgr not available, waiting (1/15)... 2024-09-12T23:23:39.508 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:39 smithi136 bash[22852]: debug 2024-09-12T23:23:39.208+0000 7f04d886bec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-12T23:23:39.508 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:39 smithi136 bash[22852]: debug 2024-09-12T23:23:39.272+0000 7f04d886bec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-12T23:23:39.508 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:39 smithi136 bash[22852]: debug 2024-09-12T23:23:39.420+0000 7f04d886bec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-12T23:23:40.330 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:40 smithi136 bash[22852]: debug 2024-09-12T23:23:40.036+0000 7f04d886bec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-12T23:23:40.330 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:40 smithi136 bash[22852]: debug 2024-09-12T23:23:40.104+0000 7f04d886bec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-12T23:23:40.330 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:40 smithi136 bash[22852]: debug 2024-09-12T23:23:40.172+0000 7f04d886bec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-12T23:23:40.693 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:40 smithi136 bash[22852]: debug 2024-09-12T23:23:40.324+0000 7f04d886bec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-12T23:23:40.694 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:40 smithi136 bash[22852]: debug 2024-09-12T23:23:40.400+0000 7f04d886bec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-12T23:23:40.694 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:40 smithi136 bash[22852]: debug 2024-09-12T23:23:40.464+0000 7f04d886bec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-12T23:23:40.999 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:40 smithi136 bash[22852]: debug 2024-09-12T23:23:40.688+0000 7f04d886bec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-12T23:23:40.999 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:40 smithi136 bash[22852]: debug 2024-09-12T23:23:40.756+0000 7f04d886bec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-12T23:23:40.999 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:40 smithi136 bash[22852]: debug 2024-09-12T23:23:40.920+0000 7f04d886bec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-12T23:23:40.999 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:41 smithi136 bash[22852]: debug 2024-09-12T23:23:40.992+0000 7f04d886bec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-12T23:23:41.258 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:41 smithi136 bash[22852]: debug 2024-09-12T23:23:41.224+0000 7f04d886bec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-12T23:23:41.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:41 smithi136 bash[22852]: debug 2024-09-12T23:23:41.384+0000 7f04d886bec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-12T23:23:41.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:41 smithi136 bash[22852]: debug 2024-09-12T23:23:41.464+0000 7f04d886bec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-12T23:23:41.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:41 smithi136 bash[22852]: debug 2024-09-12T23:23:41.620+0000 7f04d886bec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-12T23:23:42.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:41 smithi136 bash[22601]: audit 2024-09-12T23:23:41.783831+0000 mon.a (mon.0) 8 : audit [DBG] from='client.? 172.21.15.136:0/3051596582' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-12T23:23:42.307 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout 2024-09-12T23:23:42.307 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout { 2024-09-12T23:23:42.307 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "fsid": "da58a47c-715d-11ef-bceb-c7b262605968", 2024-09-12T23:23:42.307 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "health": { 2024-09-12T23:23:42.308 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-09-12T23:23:42.308 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-09-12T23:23:42.308 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-09-12T23:23:42.309 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout }, 2024-09-12T23:23:42.309 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-09-12T23:23:42.309 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-09-12T23:23:42.309 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout 0 2024-09-12T23:23:42.309 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout ], 2024-09-12T23:23:42.309 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-09-12T23:23:42.309 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "a" 2024-09-12T23:23:42.309 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout ], 2024-09-12T23:23:42.309 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "quorum_age": 6, 2024-09-12T23:23:42.309 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "monmap": { 2024-09-12T23:23:42.309 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-12T23:23:42.309 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-09-12T23:23:42.309 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-09-12T23:23:42.310 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout }, 2024-09-12T23:23:42.310 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-09-12T23:23:42.310 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-12T23:23:42.310 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-09-12T23:23:42.310 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-09-12T23:23:42.310 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-09-12T23:23:42.310 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-09-12T23:23:42.310 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-09-12T23:23:42.310 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-09-12T23:23:42.310 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout }, 2024-09-12T23:23:42.310 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-09-12T23:23:42.310 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-09-12T23:23:42.310 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-09-12T23:23:42.311 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-09-12T23:23:42.311 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-09-12T23:23:42.311 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-09-12T23:23:42.311 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-09-12T23:23:42.311 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-09-12T23:23:42.311 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-09-12T23:23:42.311 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout }, 2024-09-12T23:23:42.311 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-09-12T23:23:42.311 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-12T23:23:42.311 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-09-12T23:23:42.311 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-09-12T23:23:42.311 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout }, 2024-09-12T23:23:42.311 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-09-12T23:23:42.311 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "available": false, 2024-09-12T23:23:42.311 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-09-12T23:23:42.312 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "modules": [ 2024-09-12T23:23:42.312 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "iostat", 2024-09-12T23:23:42.312 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "nfs", 2024-09-12T23:23:42.312 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "restful" 2024-09-12T23:23:42.312 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout ], 2024-09-12T23:23:42.312 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-12T23:23:42.312 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout }, 2024-09-12T23:23:42.312 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-09-12T23:23:42.312 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-12T23:23:42.312 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "modified": "2024-09-12T23:23:29.001436+0000", 2024-09-12T23:23:42.312 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-12T23:23:42.312 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout }, 2024-09-12T23:23:42.313 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-09-12T23:23:42.313 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout } 2024-09-12T23:23:42.313 INFO:teuthology.orchestra.run.smithi136.stdout:mgr not available, waiting (2/15)... 2024-09-12T23:23:42.660 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:42 smithi136 bash[22852]: debug 2024-09-12T23:23:42.388+0000 7f04d886bec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-12T23:23:42.660 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:42 smithi136 bash[22852]: debug 2024-09-12T23:23:42.464+0000 7f04d886bec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-12T23:23:43.008 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:42 smithi136 bash[22852]: debug 2024-09-12T23:23:42.656+0000 7f04d886bec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-12T23:23:43.008 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:42 smithi136 bash[22852]: debug 2024-09-12T23:23:42.728+0000 7f04d886bec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-12T23:23:43.189 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:43 smithi136 bash[22852]: debug 2024-09-12T23:23:43.036+0000 7f04d886bec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-12T23:23:43.189 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:43 smithi136 bash[22852]: debug 2024-09-12T23:23:43.112+0000 7f04d886bec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-12T23:23:43.190 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:43 smithi136 bash[22601]: cluster 2024-09-12T23:23:43.139021+0000 mon.a (mon.0) 9 : cluster [INF] Activating manager daemon y 2024-09-12T23:23:43.190 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:43 smithi136 bash[22601]: cluster 2024-09-12T23:23:43.145861+0000 mon.a (mon.0) 10 : cluster [DBG] mgrmap e2: y(active, starting, since 0.00712764s) 2024-09-12T23:23:43.190 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:43 smithi136 bash[22601]: audit 2024-09-12T23:23:43.146996+0000 mon.a (mon.0) 11 : audit [DBG] from='mgr.14100 172.21.15.136:0/1842971196' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-12T23:23:43.190 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:43 smithi136 bash[22601]: audit 2024-09-12T23:23:43.147250+0000 mon.a (mon.0) 12 : audit [DBG] from='mgr.14100 172.21.15.136:0/1842971196' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-12T23:23:43.190 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:43 smithi136 bash[22601]: audit 2024-09-12T23:23:43.147487+0000 mon.a (mon.0) 13 : audit [DBG] from='mgr.14100 172.21.15.136:0/1842971196' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-12T23:23:43.190 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:43 smithi136 bash[22601]: audit 2024-09-12T23:23:43.147708+0000 mon.a (mon.0) 14 : audit [DBG] from='mgr.14100 172.21.15.136:0/1842971196' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-12T23:23:43.190 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:43 smithi136 bash[22601]: audit 2024-09-12T23:23:43.147942+0000 mon.a (mon.0) 15 : audit [DBG] from='mgr.14100 172.21.15.136:0/1842971196' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-12T23:23:43.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:43 smithi136 bash[22601]: audit 2024-09-12T23:23:43.148090+0000 mon.a (mon.0) 16 : audit [DBG] from='mgr.14100 172.21.15.136:0/1842971196' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-12T23:23:43.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:43 smithi136 bash[22601]: audit 2024-09-12T23:23:43.148227+0000 mon.a (mon.0) 17 : audit [DBG] from='mgr.14100 172.21.15.136:0/1842971196' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:23:43.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:43 smithi136 bash[22601]: audit 2024-09-12T23:23:43.148798+0000 mon.a (mon.0) 18 : audit [DBG] from='mgr.14100 172.21.15.136:0/1842971196' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-12T23:23:43.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:43 smithi136 bash[22601]: cluster 2024-09-12T23:23:43.175110+0000 mon.a (mon.0) 19 : cluster [INF] Manager daemon y is now available 2024-09-12T23:23:43.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:43 smithi136 bash[22601]: audit 2024-09-12T23:23:43.187230+0000 mon.a (mon.0) 20 : audit [INF] from='mgr.14100 172.21.15.136:0/1842971196' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-12T23:23:44.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:44 smithi136 bash[22601]: audit 2024-09-12T23:23:43.190492+0000 mon.a (mon.0) 21 : audit [INF] from='mgr.14100 172.21.15.136:0/1842971196' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-12T23:23:44.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:44 smithi136 bash[22601]: audit 2024-09-12T23:23:43.195951+0000 mon.a (mon.0) 22 : audit [INF] from='mgr.14100 172.21.15.136:0/1842971196' entity='mgr.y' 2024-09-12T23:23:44.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:44 smithi136 bash[22601]: audit 2024-09-12T23:23:43.199588+0000 mon.a (mon.0) 23 : audit [INF] from='mgr.14100 172.21.15.136:0/1842971196' entity='mgr.y' 2024-09-12T23:23:44.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:44 smithi136 bash[22601]: audit 2024-09-12T23:23:43.202152+0000 mon.a (mon.0) 24 : audit [INF] from='mgr.14100 172.21.15.136:0/1842971196' entity='mgr.y' 2024-09-12T23:23:44.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:44 smithi136 bash[22601]: cluster 2024-09-12T23:23:44.154037+0000 mon.a (mon.0) 25 : cluster [DBG] mgrmap e3: y(active, since 1.0153s) 2024-09-12T23:23:46.323 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout 2024-09-12T23:23:46.323 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout { 2024-09-12T23:23:46.324 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "fsid": "da58a47c-715d-11ef-bceb-c7b262605968", 2024-09-12T23:23:46.324 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "health": { 2024-09-12T23:23:46.324 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-09-12T23:23:46.324 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-09-12T23:23:46.324 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-09-12T23:23:46.324 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout }, 2024-09-12T23:23:46.324 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-09-12T23:23:46.324 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-09-12T23:23:46.324 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout 0 2024-09-12T23:23:46.324 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout ], 2024-09-12T23:23:46.324 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-09-12T23:23:46.325 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "a" 2024-09-12T23:23:46.325 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout ], 2024-09-12T23:23:46.325 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "quorum_age": 10, 2024-09-12T23:23:46.325 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "monmap": { 2024-09-12T23:23:46.325 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-12T23:23:46.325 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-09-12T23:23:46.325 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-09-12T23:23:46.325 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout }, 2024-09-12T23:23:46.325 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-09-12T23:23:46.325 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-12T23:23:46.325 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-09-12T23:23:46.325 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-09-12T23:23:46.326 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-09-12T23:23:46.326 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-09-12T23:23:46.326 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-09-12T23:23:46.326 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-09-12T23:23:46.326 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout }, 2024-09-12T23:23:46.326 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-09-12T23:23:46.326 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-09-12T23:23:46.326 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-09-12T23:23:46.326 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-09-12T23:23:46.326 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-09-12T23:23:46.326 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-09-12T23:23:46.326 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-09-12T23:23:46.326 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-09-12T23:23:46.326 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-09-12T23:23:46.327 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout }, 2024-09-12T23:23:46.327 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-09-12T23:23:46.327 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-12T23:23:46.327 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-09-12T23:23:46.327 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-09-12T23:23:46.327 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout }, 2024-09-12T23:23:46.327 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-09-12T23:23:46.327 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "available": true, 2024-09-12T23:23:46.327 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-09-12T23:23:46.327 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "modules": [ 2024-09-12T23:23:46.327 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "iostat", 2024-09-12T23:23:46.327 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "nfs", 2024-09-12T23:23:46.328 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "restful" 2024-09-12T23:23:46.328 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout ], 2024-09-12T23:23:46.328 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-12T23:23:46.328 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout }, 2024-09-12T23:23:46.328 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-09-12T23:23:46.328 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-12T23:23:46.328 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "modified": "2024-09-12T23:23:29.001436+0000", 2024-09-12T23:23:46.328 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-12T23:23:46.328 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout }, 2024-09-12T23:23:46.328 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-09-12T23:23:46.328 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout } 2024-09-12T23:23:46.328 INFO:teuthology.orchestra.run.smithi136.stdout:mgr is available 2024-09-12T23:23:46.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:46 smithi136 bash[22601]: cluster 2024-09-12T23:23:45.201676+0000 mon.a (mon.0) 26 : cluster [DBG] mgrmap e4: y(active, since 2s) 2024-09-12T23:23:46.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:46 smithi136 bash[22601]: audit 2024-09-12T23:23:45.824377+0000 mon.a (mon.0) 27 : audit [DBG] from='client.? 172.21.15.136:0/1413255260' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-12T23:23:48.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:47 smithi136 bash[22601]: audit 2024-09-12T23:23:47.653796+0000 mon.a (mon.0) 28 : audit [INF] from='client.? 172.21.15.136:0/3948126828' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-12T23:23:48.144 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout 2024-09-12T23:23:48.144 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout [global] 2024-09-12T23:23:48.144 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout fsid = da58a47c-715d-11ef-bceb-c7b262605968 2024-09-12T23:23:48.144 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout mon_cluster_log_file_level = debug 2024-09-12T23:23:48.144 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-09-12T23:23:48.144 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-09-12T23:23:48.144 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-09-12T23:23:48.144 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-09-12T23:23:48.144 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout 2024-09-12T23:23:48.144 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout [mgr] 2024-09-12T23:23:48.145 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-09-12T23:23:48.145 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout 2024-09-12T23:23:48.145 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout [osd] 2024-09-12T23:23:48.145 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-09-12T23:23:48.145 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-09-12T23:23:48.145 INFO:teuthology.orchestra.run.smithi136.stdout:Enabling cephadm module... 2024-09-12T23:23:50.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:49 smithi136 bash[22601]: audit 2024-09-12T23:23:49.530891+0000 mon.a (mon.0) 29 : audit [INF] from='client.? 172.21.15.136:0/1311120770' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-09-12T23:23:50.008 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:49 smithi136 bash[22852]: ignoring --setuser ceph since I am not root 2024-09-12T23:23:50.008 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:49 smithi136 bash[22852]: ignoring --setgroup ceph since I am not root 2024-09-12T23:23:50.009 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:49 smithi136 bash[22852]: debug 2024-09-12T23:23:49.724+0000 7f2ce17ffec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-12T23:23:50.904 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:50 smithi136 bash[22601]: audit 2024-09-12T23:23:49.587715+0000 mon.a (mon.0) 30 : audit [INF] from='client.? 172.21.15.136:0/1311120770' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-09-12T23:23:50.904 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:50 smithi136 bash[22601]: cluster 2024-09-12T23:23:49.587755+0000 mon.a (mon.0) 31 : cluster [DBG] mgrmap e5: y(active, since 6s) 2024-09-12T23:23:51.258 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:50 smithi136 bash[22852]: debug 2024-09-12T23:23:50.900+0000 7f2ce17ffec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-12T23:23:51.258 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:50 smithi136 bash[22852]: debug 2024-09-12T23:23:50.968+0000 7f2ce17ffec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-12T23:23:51.258 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:51 smithi136 bash[22852]: debug 2024-09-12T23:23:51.120+0000 7f2ce17ffec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-12T23:23:51.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:51 smithi136 bash[22601]: audit 2024-09-12T23:23:51.478959+0000 mon.a (mon.0) 32 : audit [DBG] from='client.? 172.21.15.136:0/1007751284' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-12T23:23:51.973 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout { 2024-09-12T23:23:51.973 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "epoch": 5, 2024-09-12T23:23:51.973 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "available": true, 2024-09-12T23:23:51.973 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-09-12T23:23:51.973 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-09-12T23:23:51.973 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout } 2024-09-12T23:23:51.973 INFO:teuthology.orchestra.run.smithi136.stdout:Waiting for the mgr to restart... 2024-09-12T23:23:51.973 INFO:teuthology.orchestra.run.smithi136.stdout:Waiting for mgr epoch 5... 2024-09-12T23:23:52.008 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:51 smithi136 bash[22852]: debug 2024-09-12T23:23:51.752+0000 7f2ce17ffec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-12T23:23:52.009 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:51 smithi136 bash[22852]: debug 2024-09-12T23:23:51.820+0000 7f2ce17ffec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-12T23:23:52.009 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:51 smithi136 bash[22852]: debug 2024-09-12T23:23:51.888+0000 7f2ce17ffec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-12T23:23:52.418 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:52 smithi136 bash[22852]: debug 2024-09-12T23:23:52.044+0000 7f2ce17ffec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-12T23:23:52.418 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:52 smithi136 bash[22852]: debug 2024-09-12T23:23:52.120+0000 7f2ce17ffec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-12T23:23:52.418 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:52 smithi136 bash[22852]: debug 2024-09-12T23:23:52.188+0000 7f2ce17ffec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-12T23:23:52.724 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:52 smithi136 bash[22852]: debug 2024-09-12T23:23:52.412+0000 7f2ce17ffec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-12T23:23:52.724 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:52 smithi136 bash[22852]: debug 2024-09-12T23:23:52.476+0000 7f2ce17ffec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-12T23:23:52.724 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:52 smithi136 bash[22852]: debug 2024-09-12T23:23:52.644+0000 7f2ce17ffec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-12T23:23:53.008 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:52 smithi136 bash[22852]: debug 2024-09-12T23:23:52.720+0000 7f2ce17ffec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-12T23:23:53.008 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:52 smithi136 bash[22852]: debug 2024-09-12T23:23:52.956+0000 7f2ce17ffec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-12T23:23:53.508 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:53 smithi136 bash[22852]: debug 2024-09-12T23:23:53.116+0000 7f2ce17ffec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-12T23:23:53.508 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:53 smithi136 bash[22852]: debug 2024-09-12T23:23:53.192+0000 7f2ce17ffec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-12T23:23:53.508 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:53 smithi136 bash[22852]: debug 2024-09-12T23:23:53.344+0000 7f2ce17ffec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-12T23:23:54.370 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:54 smithi136 bash[22852]: debug 2024-09-12T23:23:54.100+0000 7f2ce17ffec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-12T23:23:54.370 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:54 smithi136 bash[22852]: debug 2024-09-12T23:23:54.172+0000 7f2ce17ffec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-12T23:23:54.751 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:54 smithi136 bash[22852]: debug 2024-09-12T23:23:54.364+0000 7f2ce17ffec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-12T23:23:54.751 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:54 smithi136 bash[22852]: debug 2024-09-12T23:23:54.436+0000 7f2ce17ffec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-12T23:23:55.008 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:54 smithi136 bash[22852]: debug 2024-09-12T23:23:54.748+0000 7f2ce17ffec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-12T23:23:55.008 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:54 smithi136 bash[22852]: debug 2024-09-12T23:23:54.820+0000 7f2ce17ffec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-12T23:23:55.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:54 smithi136 bash[22601]: cluster 2024-09-12T23:23:54.847018+0000 mon.a (mon.0) 33 : cluster [INF] Active manager daemon y restarted 2024-09-12T23:23:55.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:54 smithi136 bash[22601]: cluster 2024-09-12T23:23:54.847415+0000 mon.a (mon.0) 34 : cluster [INF] Activating manager daemon y 2024-09-12T23:23:55.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:54 smithi136 bash[22601]: cluster 2024-09-12T23:23:54.854652+0000 mon.a (mon.0) 35 : cluster [DBG] osdmap e2: 0 total, 0 up, 0 in 2024-09-12T23:23:55.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:54 smithi136 bash[22601]: cluster 2024-09-12T23:23:54.854861+0000 mon.a (mon.0) 36 : cluster [DBG] mgrmap e6: y(active, starting, since 0.00761276s) 2024-09-12T23:23:55.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:54 smithi136 bash[22601]: audit 2024-09-12T23:23:54.856088+0000 mon.a (mon.0) 37 : audit [DBG] from='mgr.14118 172.21.15.136:0/2117272645' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:23:55.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:54 smithi136 bash[22601]: audit 2024-09-12T23:23:54.857627+0000 mon.a (mon.0) 38 : audit [DBG] from='mgr.14118 172.21.15.136:0/2117272645' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-12T23:23:55.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:54 smithi136 bash[22601]: audit 2024-09-12T23:23:54.858895+0000 mon.a (mon.0) 39 : audit [DBG] from='mgr.14118 172.21.15.136:0/2117272645' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-12T23:23:55.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:54 smithi136 bash[22601]: audit 2024-09-12T23:23:54.859257+0000 mon.a (mon.0) 40 : audit [DBG] from='mgr.14118 172.21.15.136:0/2117272645' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-12T23:23:55.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:54 smithi136 bash[22601]: audit 2024-09-12T23:23:54.859598+0000 mon.a (mon.0) 41 : audit [DBG] from='mgr.14118 172.21.15.136:0/2117272645' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-12T23:23:55.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:54 smithi136 bash[22601]: cluster 2024-09-12T23:23:54.883102+0000 mon.a (mon.0) 42 : cluster [INF] Manager daemon y is now available 2024-09-12T23:23:55.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:54 smithi136 bash[22601]: audit 2024-09-12T23:23:54.900687+0000 mon.a (mon.0) 43 : audit [INF] from='mgr.14118 172.21.15.136:0/2117272645' entity='mgr.y' 2024-09-12T23:23:56.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:55 smithi136 bash[22601]: cephadm 2024-09-12T23:23:54.897022+0000 mgr.y (mgr.14118) 1 : cephadm [INF] Found migration_current of "None". Setting to last migration. 2024-09-12T23:23:56.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:55 smithi136 bash[22601]: audit 2024-09-12T23:23:54.906137+0000 mon.a (mon.0) 44 : audit [INF] from='mgr.14118 172.21.15.136:0/2117272645' entity='mgr.y' 2024-09-12T23:23:56.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:55 smithi136 bash[22601]: audit 2024-09-12T23:23:54.909555+0000 mon.a (mon.0) 45 : audit [DBG] from='mgr.14118 172.21.15.136:0/2117272645' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:23:56.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:55 smithi136 bash[22601]: audit 2024-09-12T23:23:54.912051+0000 mon.a (mon.0) 46 : audit [DBG] from='mgr.14118 172.21.15.136:0/2117272645' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:23:56.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:55 smithi136 bash[22601]: audit 2024-09-12T23:23:54.922171+0000 mon.a (mon.0) 47 : audit [INF] from='mgr.14118 172.21.15.136:0/2117272645' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-12T23:23:56.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:55 smithi136 bash[22601]: audit 2024-09-12T23:23:54.924638+0000 mon.a (mon.0) 48 : audit [INF] from='mgr.14118 172.21.15.136:0/2117272645' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-12T23:23:56.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:55 smithi136 bash[22601]: cluster 2024-09-12T23:23:55.855501+0000 mon.a (mon.0) 49 : cluster [DBG] mgrmap e7: y(active, since 1.00826s) 2024-09-12T23:23:56.430 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout { 2024-09-12T23:23:56.430 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2024-09-12T23:23:56.430 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "initialized": true 2024-09-12T23:23:56.430 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout } 2024-09-12T23:23:56.430 INFO:teuthology.orchestra.run.smithi136.stdout:mgr epoch 5 is available 2024-09-12T23:23:56.430 INFO:teuthology.orchestra.run.smithi136.stdout:Setting orchestrator backend to cephadm... 2024-09-12T23:23:56.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:56 smithi136 bash[22852]: debug 2024-09-12T23:23:56.304+0000 7f2c93bcf640 -1 log_channel(cephadm) log [ERR] : [12/Sep/2024:23:23:56] ENGINE Error in HTTPServer.serve 2024-09-12T23:23:56.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:56 smithi136 bash[22852]: Traceback (most recent call last): 2024-09-12T23:23:56.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:56 smithi136 bash[22852]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-12T23:23:56.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:56 smithi136 bash[22852]: self._connections.run(self.expiration_interval) 2024-09-12T23:23:56.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:56 smithi136 bash[22852]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-12T23:23:56.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:56 smithi136 bash[22852]: self._run(expiration_interval) 2024-09-12T23:23:56.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:56 smithi136 bash[22852]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-12T23:23:56.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:56 smithi136 bash[22852]: new_conn = self._from_server_socket(self.server.socket) 2024-09-12T23:23:56.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:56 smithi136 bash[22852]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-12T23:23:56.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:56 smithi136 bash[22852]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-12T23:23:56.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:56 smithi136 bash[22852]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-12T23:23:56.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:56 smithi136 bash[22852]: s = self.context.wrap_socket( 2024-09-12T23:23:56.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:56 smithi136 bash[22852]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-12T23:23:56.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:56 smithi136 bash[22852]: return self.sslsocket_class._create( 2024-09-12T23:23:56.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:56 smithi136 bash[22852]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-12T23:23:56.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:56 smithi136 bash[22852]: self.do_handshake() 2024-09-12T23:23:56.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:56 smithi136 bash[22852]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-12T23:23:56.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:56 smithi136 bash[22852]: self._sslobj.do_handshake() 2024-09-12T23:23:56.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:23:56 smithi136 bash[22852]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-12T23:23:57.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:57 smithi136 bash[22601]: audit 2024-09-12T23:23:55.856859+0000 mgr.y (mgr.14118) 2 : audit [DBG] from='client.14122 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-12T23:23:57.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:57 smithi136 bash[22601]: audit 2024-09-12T23:23:55.869193+0000 mgr.y (mgr.14118) 3 : audit [DBG] from='client.14122 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-12T23:23:57.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:57 smithi136 bash[22601]: cephadm 2024-09-12T23:23:56.186327+0000 mgr.y (mgr.14118) 4 : cephadm [INF] [12/Sep/2024:23:23:56] ENGINE Bus STARTING 2024-09-12T23:23:57.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:57 smithi136 bash[22601]: cephadm 2024-09-12T23:23:56.306564+0000 mgr.y (mgr.14118) 5 : cephadm [INF] [12/Sep/2024:23:23:56] ENGINE Serving on https://172.21.15.136:7150 2024-09-12T23:23:57.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:57 smithi136 bash[22601]: cephadm 2024-09-12T23:23:56.306663+0000 mgr.y (mgr.14118) 6 : cephadm [INF] [12/Sep/2024:23:23:56] ENGINE Bus STARTED 2024-09-12T23:23:57.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:57 smithi136 bash[22601]: cephadm 2024-09-12T23:23:56.308918+0000 mgr.y (mgr.14118) 7 : cephadm [ERR] [12/Sep/2024:23:23:56] ENGINE Error in HTTPServer.serve 2024-09-12T23:23:57.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:57 smithi136 bash[22601]: Traceback (most recent call last): 2024-09-12T23:23:57.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:57 smithi136 bash[22601]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-12T23:23:57.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:57 smithi136 bash[22601]: self._connections.run(self.expiration_interval) 2024-09-12T23:23:57.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:57 smithi136 bash[22601]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-12T23:23:57.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:57 smithi136 bash[22601]: self._run(expiration_interval) 2024-09-12T23:23:57.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:57 smithi136 bash[22601]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-12T23:23:57.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:57 smithi136 bash[22601]: new_conn = self._from_server_socket(self.server.socket) 2024-09-12T23:23:57.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:57 smithi136 bash[22601]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-12T23:23:57.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:57 smithi136 bash[22601]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-12T23:23:57.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:57 smithi136 bash[22601]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-12T23:23:57.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:57 smithi136 bash[22601]: s = self.context.wrap_socket( 2024-09-12T23:23:57.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:57 smithi136 bash[22601]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-12T23:23:57.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:57 smithi136 bash[22601]: return self.sslsocket_class._create( 2024-09-12T23:23:57.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:57 smithi136 bash[22601]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-12T23:23:57.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:57 smithi136 bash[22601]: self.do_handshake() 2024-09-12T23:23:57.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:57 smithi136 bash[22601]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-12T23:23:57.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:57 smithi136 bash[22601]: self._sslobj.do_handshake() 2024-09-12T23:23:57.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:57 smithi136 bash[22601]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-12T23:23:57.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:57 smithi136 bash[22601]: audit 2024-09-12T23:23:56.313405+0000 mon.a (mon.0) 50 : audit [INF] from='mgr.14118 172.21.15.136:0/2117272645' entity='mgr.y' 2024-09-12T23:23:57.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:57 smithi136 bash[22601]: audit 2024-09-12T23:23:56.314635+0000 mon.a (mon.0) 51 : audit [DBG] from='mgr.14118 172.21.15.136:0/2117272645' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:23:58.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:58 smithi136 bash[22601]: cluster 2024-09-12T23:23:57.322843+0000 mon.a (mon.0) 52 : cluster [DBG] mgrmap e8: y(active, since 2s) 2024-09-12T23:23:58.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:58 smithi136 bash[22601]: audit 2024-09-12T23:23:57.840941+0000 mon.a (mon.0) 53 : audit [INF] from='mgr.14118 172.21.15.136:0/2117272645' entity='mgr.y' 2024-09-12T23:23:58.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:58 smithi136 bash[22601]: audit 2024-09-12T23:23:57.853290+0000 mon.a (mon.0) 54 : audit [DBG] from='mgr.14118 172.21.15.136:0/2117272645' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:23:59.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:23:59 smithi136 bash[22601]: audit 2024-09-12T23:23:57.831185+0000 mgr.y (mgr.14118) 8 : audit [DBG] from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:24:00.237 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout value unchanged 2024-09-12T23:24:00.237 INFO:teuthology.orchestra.run.smithi136.stdout:Generating ssh key... 2024-09-12T23:24:01.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:00 smithi136 bash[22601]: audit 2024-09-12T23:23:59.686867+0000 mgr.y (mgr.14118) 9 : audit [DBG] from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:24:01.952 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:01 smithi136 bash[22601]: audit 2024-09-12T23:24:01.509551+0000 mgr.y (mgr.14118) 10 : audit [DBG] from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:24:01.953 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:01 smithi136 bash[22601]: cephadm 2024-09-12T23:24:01.509943+0000 mgr.y (mgr.14118) 11 : cephadm [INF] Generating ssh key... 2024-09-12T23:24:02.258 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:01 smithi136 bash[22852]: Generating public/private rsa key pair. 2024-09-12T23:24:02.258 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:01 smithi136 bash[22852]: Your identification has been saved in /tmp/tmpkbf_b859/key 2024-09-12T23:24:02.259 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:01 smithi136 bash[22852]: Your public key has been saved in /tmp/tmpkbf_b859/key.pub 2024-09-12T23:24:02.259 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:01 smithi136 bash[22852]: The key fingerprint is: 2024-09-12T23:24:02.259 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:01 smithi136 bash[22852]: SHA256:wuLp/BNgJ+x6qRjsa0wPFN5W1C9a9Zv/6+70r9VHnpY ceph-da58a47c-715d-11ef-bceb-c7b262605968 2024-09-12T23:24:02.259 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:01 smithi136 bash[22852]: The key's randomart image is: 2024-09-12T23:24:02.259 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:01 smithi136 bash[22852]: +---[RSA 3072]----+ 2024-09-12T23:24:02.259 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:01 smithi136 bash[22852]: | ... | 2024-09-12T23:24:02.259 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:01 smithi136 bash[22852]: | . . . . | 2024-09-12T23:24:02.259 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:01 smithi136 bash[22852]: |. o.. o . | 2024-09-12T23:24:02.259 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:01 smithi136 bash[22852]: | o o=..o . . | 2024-09-12T23:24:02.259 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:01 smithi136 bash[22852]: |. .o.+= S o .| 2024-09-12T23:24:02.259 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:01 smithi136 bash[22852]: |.o ..+.. o o+| 2024-09-12T23:24:02.259 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:01 smithi136 bash[22852]: |+.o.o. . . E=| 2024-09-12T23:24:02.259 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:01 smithi136 bash[22852]: |.=.+o . .o.o| 2024-09-12T23:24:02.259 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:01 smithi136 bash[22852]: |ooooo... *B=| 2024-09-12T23:24:02.259 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:01 smithi136 bash[22852]: +----[SHA256]-----+ 2024-09-12T23:24:03.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:02 smithi136 bash[22601]: audit 2024-09-12T23:24:01.960936+0000 mon.a (mon.0) 55 : audit [INF] from='mgr.14118 172.21.15.136:0/2117272645' entity='mgr.y' 2024-09-12T23:24:03.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:02 smithi136 bash[22601]: audit 2024-09-12T23:24:01.965374+0000 mon.a (mon.0) 56 : audit [INF] from='mgr.14118 172.21.15.136:0/2117272645' entity='mgr.y' 2024-09-12T23:24:04.368 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCtnxKsTN2Ggr6cNaqM1xP3Ucs8yiiWEXpZ6YnFcSQJHU6x0qZX6r87nSJ66yVsPKr5DsFRuwAcXMItQNZND6xZlG4nbUUeJs6Sf95gzdbZ0YnPrOSz7XO8YGQotzhU5gxfS0O1uDQ+PHQNr0YKtXytAtbDIiiYCY+YIr++6E/zSeJt5y30LDoS+jkzzIsuQ+g1oKLrd7yW8YON3SJPDpUC3tvspdIfzt2I0A0BUtsUztMMXCjc9up0XRiOkxd/PU2StVlwWm6xJVzaOHR/W/5DSXAKSHrRLXiNJGMTJjDS7TiDZCLZXVtM51+DB080WQqxQ+3dKDpTHfZ1Xt//igvtGlqTQgJGzSRN81kr9lwZ2Jfkwf+PrAIPuzX3S9WBG5vTYLKlRYFPM2XovObUj1UxP5z0qTdhJRKmQoyGXTFvIGXisE8b3LaiWoXbMZ7vwf885Ng2oc1fIAHTe/X66VRECz5qjBTkkeY1WzL2rNJa2NCn03JFbFbQc0ibJdw5Ty0= ceph-da58a47c-715d-11ef-bceb-c7b262605968 2024-09-12T23:24:04.368 INFO:teuthology.orchestra.run.smithi136.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-09-12T23:24:04.368 INFO:teuthology.orchestra.run.smithi136.stdout:Adding key to root@localhost authorized_keys... 2024-09-12T23:24:04.368 INFO:teuthology.orchestra.run.smithi136.stdout:Adding host smithi136... 2024-09-12T23:24:04.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:04 smithi136 bash[22601]: audit 2024-09-12T23:24:03.797808+0000 mgr.y (mgr.14118) 12 : audit [DBG] from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:24:06.743 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:06 smithi136 bash[22601]: audit 2024-09-12T23:24:05.669609+0000 mgr.y (mgr.14118) 13 : audit [DBG] from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi136", "addr": "172.21.15.136", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:24:06.743 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:06 smithi136 bash[22601]: cephadm 2024-09-12T23:24:06.630942+0000 mgr.y (mgr.14118) 14 : cephadm [INF] Deploying cephadm binary to smithi136 2024-09-12T23:24:09.437 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout Added host 'smithi136' with addr '172.21.15.136' 2024-09-12T23:24:09.438 INFO:teuthology.orchestra.run.smithi136.stdout:Deploying unmanaged mon service... 2024-09-12T23:24:10.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:09 smithi136 bash[22601]: audit 2024-09-12T23:24:08.722159+0000 mon.a (mon.0) 57 : audit [INF] from='mgr.14118 172.21.15.136:0/2117272645' entity='mgr.y' 2024-09-12T23:24:10.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:09 smithi136 bash[22601]: cephadm 2024-09-12T23:24:08.722673+0000 mgr.y (mgr.14118) 15 : cephadm [INF] Added host smithi136 2024-09-12T23:24:10.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:09 smithi136 bash[22601]: audit 2024-09-12T23:24:08.723169+0000 mon.a (mon.0) 58 : audit [DBG] from='mgr.14118 172.21.15.136:0/2117272645' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:24:12.044 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2024-09-12T23:24:12.044 INFO:teuthology.orchestra.run.smithi136.stdout:Deploying unmanaged mgr service... 2024-09-12T23:24:12.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:12 smithi136 bash[22601]: audit 2024-09-12T23:24:11.314527+0000 mgr.y (mgr.14118) 16 : audit [DBG] from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:24:12.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:12 smithi136 bash[22601]: cephadm 2024-09-12T23:24:11.317663+0000 mgr.y (mgr.14118) 17 : cephadm [INF] Saving service mon spec with placement count:5 2024-09-12T23:24:12.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:12 smithi136 bash[22601]: audit 2024-09-12T23:24:11.320603+0000 mon.a (mon.0) 59 : audit [INF] from='mgr.14118 172.21.15.136:0/2117272645' entity='mgr.y' 2024-09-12T23:24:12.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:12 smithi136 bash[22601]: audit 2024-09-12T23:24:11.758805+0000 mon.a (mon.0) 60 : audit [INF] from='mgr.14118 172.21.15.136:0/2117272645' entity='mgr.y' 2024-09-12T23:24:12.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:12 smithi136 bash[22601]: audit 2024-09-12T23:24:12.049838+0000 mon.a (mon.0) 61 : audit [INF] from='mgr.14118 172.21.15.136:0/2117272645' entity='mgr.y' 2024-09-12T23:24:13.946 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2024-09-12T23:24:14.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:14 smithi136 bash[22601]: audit 2024-09-12T23:24:13.429738+0000 mgr.y (mgr.14118) 18 : audit [DBG] from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:24:14.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:14 smithi136 bash[22601]: cephadm 2024-09-12T23:24:13.431771+0000 mgr.y (mgr.14118) 19 : cephadm [INF] Saving service mgr spec with placement count:2 2024-09-12T23:24:14.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:14 smithi136 bash[22601]: audit 2024-09-12T23:24:13.436471+0000 mon.a (mon.0) 62 : audit [INF] from='mgr.14118 172.21.15.136:0/2117272645' entity='mgr.y' 2024-09-12T23:24:16.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:16 smithi136 bash[22601]: cluster 2024-09-12T23:24:14.860304+0000 mgr.y (mgr.14118) 20 : cluster [DBG] pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:24:16.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:16 smithi136 bash[22601]: audit 2024-09-12T23:24:15.240127+0000 mon.a (mon.0) 63 : audit [INF] from='client.? 172.21.15.136:0/2311308156' entity='client.admin' 2024-09-12T23:24:17.585 INFO:teuthology.orchestra.run.smithi136.stdout:Enabling the dashboard module... 2024-09-12T23:24:18.408 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:18 smithi136 bash[22601]: cluster 2024-09-12T23:24:16.860640+0000 mgr.y (mgr.14118) 21 : cluster [DBG] pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:24:18.408 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:18 smithi136 bash[22601]: audit 2024-09-12T23:24:17.081128+0000 mon.a (mon.0) 64 : audit [INF] from='client.? 172.21.15.136:0/2124692369' entity='client.admin' 2024-09-12T23:24:19.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:19 smithi136 bash[22601]: audit 2024-09-12T23:24:18.241693+0000 mon.a (mon.0) 65 : audit [INF] from='mgr.14118 172.21.15.136:0/2117272645' entity='mgr.y' 2024-09-12T23:24:19.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:19 smithi136 bash[22601]: audit 2024-09-12T23:24:18.805499+0000 mon.a (mon.0) 66 : audit [INF] from='mgr.14118 172.21.15.136:0/2117272645' entity='mgr.y' 2024-09-12T23:24:19.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:19 smithi136 bash[22601]: audit 2024-09-12T23:24:18.955965+0000 mon.a (mon.0) 67 : audit [INF] from='client.? 172.21.15.136:0/439109260' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-09-12T23:24:20.245 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:19 smithi136 bash[22852]: ignoring --setuser ceph since I am not root 2024-09-12T23:24:20.245 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:19 smithi136 bash[22852]: ignoring --setgroup ceph since I am not root 2024-09-12T23:24:20.245 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:19 smithi136 bash[22852]: debug 2024-09-12T23:24:19.967+0000 7fdebe657ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-12T23:24:20.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:20 smithi136 bash[22601]: cluster 2024-09-12T23:24:18.861082+0000 mgr.y (mgr.14118) 22 : cluster [DBG] pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:24:20.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:20 smithi136 bash[22601]: audit 2024-09-12T23:24:19.815040+0000 mon.a (mon.0) 68 : audit [INF] from='client.? 172.21.15.136:0/439109260' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-09-12T23:24:20.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:20 smithi136 bash[22601]: cluster 2024-09-12T23:24:19.815128+0000 mon.a (mon.0) 69 : cluster [DBG] mgrmap e9: y(active, since 24s) 2024-09-12T23:24:21.444 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:21 smithi136 bash[22852]: debug 2024-09-12T23:24:21.191+0000 7fdebe657ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-12T23:24:21.444 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:21 smithi136 bash[22852]: debug 2024-09-12T23:24:21.255+0000 7fdebe657ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-12T23:24:21.444 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:21 smithi136 bash[22852]: debug 2024-09-12T23:24:21.415+0000 7fdebe657ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-12T23:24:22.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:22 smithi136 bash[22601]: audit 2024-09-12T23:24:21.973548+0000 mon.a (mon.0) 70 : audit [DBG] from='client.? 172.21.15.136:0/1656122436' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-12T23:24:22.259 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:22 smithi136 bash[22852]: debug 2024-09-12T23:24:22.083+0000 7fdebe657ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-12T23:24:22.259 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:22 smithi136 bash[22852]: debug 2024-09-12T23:24:22.155+0000 7fdebe657ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-12T23:24:22.259 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:22 smithi136 bash[22852]: debug 2024-09-12T23:24:22.227+0000 7fdebe657ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-12T23:24:22.524 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout { 2024-09-12T23:24:22.525 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "epoch": 9, 2024-09-12T23:24:22.525 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "available": true, 2024-09-12T23:24:22.525 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-09-12T23:24:22.525 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-09-12T23:24:22.525 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout } 2024-09-12T23:24:22.525 INFO:teuthology.orchestra.run.smithi136.stdout:Waiting for the mgr to restart... 2024-09-12T23:24:22.525 INFO:teuthology.orchestra.run.smithi136.stdout:Waiting for mgr epoch 9... 2024-09-12T23:24:22.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:22 smithi136 bash[22852]: debug 2024-09-12T23:24:22.387+0000 7fdebe657ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-12T23:24:22.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:22 smithi136 bash[22852]: debug 2024-09-12T23:24:22.467+0000 7fdebe657ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-12T23:24:22.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:22 smithi136 bash[22852]: debug 2024-09-12T23:24:22.535+0000 7fdebe657ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-12T23:24:23.096 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:22 smithi136 bash[22852]: debug 2024-09-12T23:24:22.775+0000 7fdebe657ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-12T23:24:23.096 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:22 smithi136 bash[22852]: debug 2024-09-12T23:24:22.843+0000 7fdebe657ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-12T23:24:23.097 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:23 smithi136 bash[22852]: debug 2024-09-12T23:24:23.015+0000 7fdebe657ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-12T23:24:23.097 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:23 smithi136 bash[22852]: debug 2024-09-12T23:24:23.095+0000 7fdebe657ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-12T23:24:23.434 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:23 smithi136 bash[22852]: debug 2024-09-12T23:24:23.335+0000 7fdebe657ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-12T23:24:23.750 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:23 smithi136 bash[22852]: debug 2024-09-12T23:24:23.507+0000 7fdebe657ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-12T23:24:23.750 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:23 smithi136 bash[22852]: debug 2024-09-12T23:24:23.591+0000 7fdebe657ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-12T23:24:23.750 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:23 smithi136 bash[22852]: debug 2024-09-12T23:24:23.747+0000 7fdebe657ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-12T23:24:24.831 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:24 smithi136 bash[22852]: debug 2024-09-12T23:24:24.547+0000 7fdebe657ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-12T23:24:24.831 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:24 smithi136 bash[22852]: debug 2024-09-12T23:24:24.623+0000 7fdebe657ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-12T23:24:24.832 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:24 smithi136 bash[22852]: debug 2024-09-12T23:24:24.827+0000 7fdebe657ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-12T23:24:25.217 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:24 smithi136 bash[22852]: debug 2024-09-12T23:24:24.907+0000 7fdebe657ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-12T23:24:25.509 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:25 smithi136 bash[22852]: debug 2024-09-12T23:24:25.243+0000 7fdebe657ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-12T23:24:25.509 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:25 smithi136 bash[22852]: debug 2024-09-12T23:24:25.319+0000 7fdebe657ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-12T23:24:25.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:25 smithi136 bash[22601]: cluster 2024-09-12T23:24:25.346792+0000 mon.a (mon.0) 71 : cluster [INF] Active manager daemon y restarted 2024-09-12T23:24:25.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:25 smithi136 bash[22601]: cluster 2024-09-12T23:24:25.347248+0000 mon.a (mon.0) 72 : cluster [INF] Activating manager daemon y 2024-09-12T23:24:25.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:25 smithi136 bash[22601]: cluster 2024-09-12T23:24:25.354619+0000 mon.a (mon.0) 73 : cluster [DBG] osdmap e3: 0 total, 0 up, 0 in 2024-09-12T23:24:25.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:25 smithi136 bash[22601]: cluster 2024-09-12T23:24:25.354855+0000 mon.a (mon.0) 74 : cluster [DBG] mgrmap e10: y(active, starting, since 0.00779294s) 2024-09-12T23:24:25.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:25 smithi136 bash[22601]: audit 2024-09-12T23:24:25.356307+0000 mon.a (mon.0) 75 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:24:25.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:25 smithi136 bash[22601]: audit 2024-09-12T23:24:25.358732+0000 mon.a (mon.0) 76 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-12T23:24:25.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:25 smithi136 bash[22601]: audit 2024-09-12T23:24:25.360068+0000 mon.a (mon.0) 77 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-12T23:24:25.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:25 smithi136 bash[22601]: audit 2024-09-12T23:24:25.360215+0000 mon.a (mon.0) 78 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-12T23:24:25.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:25 smithi136 bash[22601]: audit 2024-09-12T23:24:25.360399+0000 mon.a (mon.0) 79 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-12T23:24:25.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:25 smithi136 bash[22601]: cluster 2024-09-12T23:24:25.390895+0000 mon.a (mon.0) 80 : cluster [INF] Manager daemon y is now available 2024-09-12T23:24:26.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:26 smithi136 bash[22601]: audit 2024-09-12T23:24:25.406819+0000 mon.a (mon.0) 81 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:24:26.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:26 smithi136 bash[22601]: audit 2024-09-12T23:24:25.421770+0000 mon.a (mon.0) 82 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-12T23:24:26.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:26 smithi136 bash[22601]: audit 2024-09-12T23:24:25.426588+0000 mon.a (mon.0) 83 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-12T23:24:26.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:26 smithi136 bash[22601]: cluster 2024-09-12T23:24:26.356757+0000 mon.a (mon.0) 84 : cluster [DBG] mgrmap e11: y(active, since 1.0097s) 2024-09-12T23:24:26.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:26 smithi136 bash[22852]: debug 2024-09-12T23:24:26.403+0000 7fde6f1a4640 -1 log_channel(cephadm) log [ERR] : [12/Sep/2024:23:24:26] ENGINE Error in HTTPServer.serve 2024-09-12T23:24:26.761 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:26 smithi136 bash[22852]: Traceback (most recent call last): 2024-09-12T23:24:26.761 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:26 smithi136 bash[22852]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-12T23:24:26.761 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:26 smithi136 bash[22852]: self._connections.run(self.expiration_interval) 2024-09-12T23:24:26.761 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:26 smithi136 bash[22852]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-12T23:24:26.761 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:26 smithi136 bash[22852]: self._run(expiration_interval) 2024-09-12T23:24:26.761 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:26 smithi136 bash[22852]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-12T23:24:26.761 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:26 smithi136 bash[22852]: new_conn = self._from_server_socket(self.server.socket) 2024-09-12T23:24:26.761 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:26 smithi136 bash[22852]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-12T23:24:26.762 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:26 smithi136 bash[22852]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-12T23:24:26.762 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:26 smithi136 bash[22852]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-12T23:24:26.762 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:26 smithi136 bash[22852]: s = self.context.wrap_socket( 2024-09-12T23:24:26.762 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:26 smithi136 bash[22852]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-12T23:24:26.762 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:26 smithi136 bash[22852]: return self.sslsocket_class._create( 2024-09-12T23:24:26.762 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:26 smithi136 bash[22852]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-12T23:24:26.762 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:26 smithi136 bash[22852]: self.do_handshake() 2024-09-12T23:24:26.762 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:26 smithi136 bash[22852]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-12T23:24:26.763 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:26 smithi136 bash[22852]: self._sslobj.do_handshake() 2024-09-12T23:24:26.763 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:24:26 smithi136 bash[22852]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-12T23:24:26.957 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout { 2024-09-12T23:24:26.957 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2024-09-12T23:24:26.957 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout "initialized": true 2024-09-12T23:24:26.957 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout } 2024-09-12T23:24:26.957 INFO:teuthology.orchestra.run.smithi136.stdout:mgr epoch 9 is available 2024-09-12T23:24:26.957 INFO:teuthology.orchestra.run.smithi136.stdout:Generating a dashboard self-signed certificate... 2024-09-12T23:24:27.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:27 smithi136 bash[22601]: cephadm 2024-09-12T23:24:26.283623+0000 mgr.y (mgr.14150) 1 : cephadm [INF] [12/Sep/2024:23:24:26] ENGINE Bus STARTING 2024-09-12T23:24:27.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:27 smithi136 bash[22601]: audit 2024-09-12T23:24:26.355390+0000 mgr.y (mgr.14150) 2 : audit [DBG] from='client.14154 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-12T23:24:27.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:27 smithi136 bash[22601]: audit 2024-09-12T23:24:26.365861+0000 mgr.y (mgr.14150) 3 : audit [DBG] from='client.14154 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-12T23:24:27.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:27 smithi136 bash[22601]: cephadm 2024-09-12T23:24:26.405146+0000 mgr.y (mgr.14150) 4 : cephadm [INF] [12/Sep/2024:23:24:26] ENGINE Serving on https://172.21.15.136:7150 2024-09-12T23:24:27.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:27 smithi136 bash[22601]: cephadm 2024-09-12T23:24:26.405293+0000 mgr.y (mgr.14150) 5 : cephadm [INF] [12/Sep/2024:23:24:26] ENGINE Bus STARTED 2024-09-12T23:24:27.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:27 smithi136 bash[22601]: cephadm 2024-09-12T23:24:26.407052+0000 mgr.y (mgr.14150) 6 : cephadm [ERR] [12/Sep/2024:23:24:26] ENGINE Error in HTTPServer.serve 2024-09-12T23:24:27.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:27 smithi136 bash[22601]: Traceback (most recent call last): 2024-09-12T23:24:27.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:27 smithi136 bash[22601]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-12T23:24:27.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:27 smithi136 bash[22601]: self._connections.run(self.expiration_interval) 2024-09-12T23:24:27.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:27 smithi136 bash[22601]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-12T23:24:27.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:27 smithi136 bash[22601]: self._run(expiration_interval) 2024-09-12T23:24:27.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:27 smithi136 bash[22601]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-12T23:24:27.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:27 smithi136 bash[22601]: new_conn = self._from_server_socket(self.server.socket) 2024-09-12T23:24:27.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:27 smithi136 bash[22601]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-12T23:24:27.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:27 smithi136 bash[22601]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-12T23:24:27.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:27 smithi136 bash[22601]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-12T23:24:27.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:27 smithi136 bash[22601]: s = self.context.wrap_socket( 2024-09-12T23:24:27.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:27 smithi136 bash[22601]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-12T23:24:27.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:27 smithi136 bash[22601]: return self.sslsocket_class._create( 2024-09-12T23:24:27.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:27 smithi136 bash[22601]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-12T23:24:27.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:27 smithi136 bash[22601]: self.do_handshake() 2024-09-12T23:24:27.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:27 smithi136 bash[22601]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-12T23:24:27.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:27 smithi136 bash[22601]: self._sslobj.do_handshake() 2024-09-12T23:24:27.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:27 smithi136 bash[22601]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-12T23:24:27.762 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:27 smithi136 bash[22601]: audit 2024-09-12T23:24:26.408885+0000 mon.a (mon.0) 85 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:24:28.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:28 smithi136 bash[22601]: cluster 2024-09-12T23:24:27.422353+0000 mon.a (mon.0) 86 : cluster [DBG] mgrmap e12: y(active, since 2s) 2024-09-12T23:24:28.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:28 smithi136 bash[22601]: audit 2024-09-12T23:24:28.342112+0000 mon.a (mon.0) 87 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:24:28.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:28 smithi136 bash[22601]: audit 2024-09-12T23:24:28.348554+0000 mon.a (mon.0) 88 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:24:28.897 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2024-09-12T23:24:28.897 INFO:teuthology.orchestra.run.smithi136.stdout:Creating initial admin user... 2024-09-12T23:24:29.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:29 smithi136 bash[22601]: audit 2024-09-12T23:24:28.286996+0000 mgr.y (mgr.14150) 7 : audit [DBG] from='client.14162 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:24:31.129 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$Q3F1BE7B0SF9bgkXT3h3aOJwCYfNMfTEFHIlp2ne09Ro9/GK3ctHK", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1726183470, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-09-12T23:24:31.130 INFO:teuthology.orchestra.run.smithi136.stdout:Fetching dashboard port number... 2024-09-12T23:24:31.898 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:31 smithi136 bash[22601]: audit 2024-09-12T23:24:30.336792+0000 mgr.y (mgr.14150) 8 : audit [DBG] from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:24:31.898 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:31 smithi136 bash[22601]: audit 2024-09-12T23:24:30.559003+0000 mon.a (mon.0) 89 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:24:32.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:32 smithi136 bash[22601]: cluster 2024-09-12T23:24:31.564435+0000 mon.a (mon.0) 90 : cluster [DBG] mgrmap e13: y(active, since 6s) 2024-09-12T23:24:32.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:32 smithi136 bash[22601]: audit 2024-09-12T23:24:32.070432+0000 mon.a (mon.0) 91 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:24:32.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:32 smithi136 bash[22601]: audit 2024-09-12T23:24:32.360141+0000 mon.a (mon.0) 92 : audit [DBG] from='client.? 172.21.15.136:0/3769648579' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-09-12T23:24:32.918 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stdout 8443 2024-09-12T23:24:32.918 INFO:teuthology.orchestra.run.smithi136.stdout:firewalld does not appear to be present 2024-09-12T23:24:32.918 INFO:teuthology.orchestra.run.smithi136.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2024-09-12T23:24:32.920 INFO:teuthology.orchestra.run.smithi136.stdout:Ceph Dashboard is now available at: 2024-09-12T23:24:32.920 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-12T23:24:32.920 INFO:teuthology.orchestra.run.smithi136.stdout: URL: https://smithi136.front.sepia.ceph.com:8443/ 2024-09-12T23:24:32.920 INFO:teuthology.orchestra.run.smithi136.stdout: User: admin 2024-09-12T23:24:32.920 INFO:teuthology.orchestra.run.smithi136.stdout: Password: dymh4bei2r 2024-09-12T23:24:32.920 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-12T23:24:32.920 INFO:teuthology.orchestra.run.smithi136.stdout:Saving cluster configuration to /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config directory 2024-09-12T23:24:32.920 INFO:teuthology.orchestra.run.smithi136.stdout:Enabling autotune for osd_memory_target 2024-09-12T23:24:33.857 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:33 smithi136 bash[22601]: audit 2024-09-12T23:24:32.578849+0000 mon.a (mon.0) 93 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:24:36.743 INFO:teuthology.orchestra.run.smithi136.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2024-09-12T23:24:36.744 INFO:teuthology.orchestra.run.smithi136.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2024-09-12T23:24:36.744 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-12T23:24:36.744 INFO:teuthology.orchestra.run.smithi136.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid da58a47c-715d-11ef-bceb-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-09-12T23:24:36.744 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-12T23:24:36.744 INFO:teuthology.orchestra.run.smithi136.stdout:Or, if you are only running a single cluster on this host: 2024-09-12T23:24:36.744 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-12T23:24:36.744 INFO:teuthology.orchestra.run.smithi136.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2024-09-12T23:24:36.744 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-12T23:24:36.744 INFO:teuthology.orchestra.run.smithi136.stdout:Please consider enabling telemetry to help improve Ceph: 2024-09-12T23:24:36.744 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-12T23:24:36.744 INFO:teuthology.orchestra.run.smithi136.stdout: ceph telemetry on 2024-09-12T23:24:36.744 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-12T23:24:36.744 INFO:teuthology.orchestra.run.smithi136.stdout:For more information see: 2024-09-12T23:24:36.745 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-12T23:24:36.745 INFO:teuthology.orchestra.run.smithi136.stdout: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-09-12T23:24:36.745 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-12T23:24:36.745 INFO:teuthology.orchestra.run.smithi136.stdout:Bootstrap complete. 2024-09-12T23:24:36.787 INFO:tasks.cephadm:Fetching config... 2024-09-12T23:24:36.787 DEBUG:teuthology.orchestra.run.smithi136:> set -ex 2024-09-12T23:24:36.787 DEBUG:teuthology.orchestra.run.smithi136:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-09-12T23:24:36.793 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-09-12T23:24:36.793 DEBUG:teuthology.orchestra.run.smithi136:> set -ex 2024-09-12T23:24:36.793 DEBUG:teuthology.orchestra.run.smithi136:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-09-12T23:24:36.839 INFO:tasks.cephadm:Fetching mon keyring... 2024-09-12T23:24:36.840 DEBUG:teuthology.orchestra.run.smithi136:> set -ex 2024-09-12T23:24:36.840 DEBUG:teuthology.orchestra.run.smithi136:> sudo dd if=/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/keyring of=/dev/stdout 2024-09-12T23:24:36.898 INFO:tasks.cephadm:Fetching pub ssh key... 2024-09-12T23:24:36.898 DEBUG:teuthology.orchestra.run.smithi136:> set -ex 2024-09-12T23:24:36.898 DEBUG:teuthology.orchestra.run.smithi136:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-09-12T23:24:36.947 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-09-12T23:24:36.947 DEBUG:teuthology.orchestra.run.smithi136:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCtnxKsTN2Ggr6cNaqM1xP3Ucs8yiiWEXpZ6YnFcSQJHU6x0qZX6r87nSJ66yVsPKr5DsFRuwAcXMItQNZND6xZlG4nbUUeJs6Sf95gzdbZ0YnPrOSz7XO8YGQotzhU5gxfS0O1uDQ+PHQNr0YKtXytAtbDIiiYCY+YIr++6E/zSeJt5y30LDoS+jkzzIsuQ+g1oKLrd7yW8YON3SJPDpUC3tvspdIfzt2I0A0BUtsUztMMXCjc9up0XRiOkxd/PU2StVlwWm6xJVzaOHR/W/5DSXAKSHrRLXiNJGMTJjDS7TiDZCLZXVtM51+DB080WQqxQ+3dKDpTHfZ1Xt//igvtGlqTQgJGzSRN81kr9lwZ2Jfkwf+PrAIPuzX3S9WBG5vTYLKlRYFPM2XovObUj1UxP5z0qTdhJRKmQoyGXTFvIGXisE8b3LaiWoXbMZ7vwf885Ng2oc1fIAHTe/X66VRECz5qjBTkkeY1WzL2rNJa2NCn03JFbFbQc0ibJdw5Ty0= ceph-da58a47c-715d-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-12T23:24:37.011 INFO:teuthology.orchestra.run.smithi136.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCtnxKsTN2Ggr6cNaqM1xP3Ucs8yiiWEXpZ6YnFcSQJHU6x0qZX6r87nSJ66yVsPKr5DsFRuwAcXMItQNZND6xZlG4nbUUeJs6Sf95gzdbZ0YnPrOSz7XO8YGQotzhU5gxfS0O1uDQ+PHQNr0YKtXytAtbDIiiYCY+YIr++6E/zSeJt5y30LDoS+jkzzIsuQ+g1oKLrd7yW8YON3SJPDpUC3tvspdIfzt2I0A0BUtsUztMMXCjc9up0XRiOkxd/PU2StVlwWm6xJVzaOHR/W/5DSXAKSHrRLXiNJGMTJjDS7TiDZCLZXVtM51+DB080WQqxQ+3dKDpTHfZ1Xt//igvtGlqTQgJGzSRN81kr9lwZ2Jfkwf+PrAIPuzX3S9WBG5vTYLKlRYFPM2XovObUj1UxP5z0qTdhJRKmQoyGXTFvIGXisE8b3LaiWoXbMZ7vwf885Ng2oc1fIAHTe/X66VRECz5qjBTkkeY1WzL2rNJa2NCn03JFbFbQc0ibJdw5Ty0= ceph-da58a47c-715d-11ef-bceb-c7b262605968 2024-09-12T23:24:37.021 DEBUG:teuthology.orchestra.run.smithi181:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCtnxKsTN2Ggr6cNaqM1xP3Ucs8yiiWEXpZ6YnFcSQJHU6x0qZX6r87nSJ66yVsPKr5DsFRuwAcXMItQNZND6xZlG4nbUUeJs6Sf95gzdbZ0YnPrOSz7XO8YGQotzhU5gxfS0O1uDQ+PHQNr0YKtXytAtbDIiiYCY+YIr++6E/zSeJt5y30LDoS+jkzzIsuQ+g1oKLrd7yW8YON3SJPDpUC3tvspdIfzt2I0A0BUtsUztMMXCjc9up0XRiOkxd/PU2StVlwWm6xJVzaOHR/W/5DSXAKSHrRLXiNJGMTJjDS7TiDZCLZXVtM51+DB080WQqxQ+3dKDpTHfZ1Xt//igvtGlqTQgJGzSRN81kr9lwZ2Jfkwf+PrAIPuzX3S9WBG5vTYLKlRYFPM2XovObUj1UxP5z0qTdhJRKmQoyGXTFvIGXisE8b3LaiWoXbMZ7vwf885Ng2oc1fIAHTe/X66VRECz5qjBTkkeY1WzL2rNJa2NCn03JFbFbQc0ibJdw5Ty0= ceph-da58a47c-715d-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-12T23:24:37.043 INFO:teuthology.orchestra.run.smithi181.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCtnxKsTN2Ggr6cNaqM1xP3Ucs8yiiWEXpZ6YnFcSQJHU6x0qZX6r87nSJ66yVsPKr5DsFRuwAcXMItQNZND6xZlG4nbUUeJs6Sf95gzdbZ0YnPrOSz7XO8YGQotzhU5gxfS0O1uDQ+PHQNr0YKtXytAtbDIiiYCY+YIr++6E/zSeJt5y30LDoS+jkzzIsuQ+g1oKLrd7yW8YON3SJPDpUC3tvspdIfzt2I0A0BUtsUztMMXCjc9up0XRiOkxd/PU2StVlwWm6xJVzaOHR/W/5DSXAKSHrRLXiNJGMTJjDS7TiDZCLZXVtM51+DB080WQqxQ+3dKDpTHfZ1Xt//igvtGlqTQgJGzSRN81kr9lwZ2Jfkwf+PrAIPuzX3S9WBG5vTYLKlRYFPM2XovObUj1UxP5z0qTdhJRKmQoyGXTFvIGXisE8b3LaiWoXbMZ7vwf885Ng2oc1fIAHTe/X66VRECz5qjBTkkeY1WzL2rNJa2NCn03JFbFbQc0ibJdw5Ty0= ceph-da58a47c-715d-11ef-bceb-c7b262605968 2024-09-12T23:24:37.053 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-09-12T23:24:37.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:37 smithi136 bash[22601]: audit 2024-09-12T23:24:36.239693+0000 mon.a (mon.0) 94 : audit [INF] from='client.? 172.21.15.136:0/825725035' entity='client.admin' 2024-09-12T23:24:42.987 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:24:43.872 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:43 smithi136 bash[22601]: audit 2024-09-12T23:24:42.586901+0000 mon.a (mon.0) 95 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:24:43.872 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:43 smithi136 bash[22601]: audit 2024-09-12T23:24:42.592587+0000 mon.a (mon.0) 96 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:24:43.872 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:43 smithi136 bash[22601]: audit 2024-09-12T23:24:42.594113+0000 mon.a (mon.0) 97 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi136", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:24:43.872 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:43 smithi136 bash[22601]: audit 2024-09-12T23:24:42.600591+0000 mon.a (mon.0) 98 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:24:43.872 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:43 smithi136 bash[22601]: audit 2024-09-12T23:24:42.625469+0000 mon.a (mon.0) 99 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:24:43.872 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:43 smithi136 bash[22601]: audit 2024-09-12T23:24:42.633746+0000 mon.a (mon.0) 100 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:24:46.245 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-09-12T23:24:46.245 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-09-12T23:24:47.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:46 smithi136 bash[22601]: cluster 2024-09-12T23:24:45.361788+0000 mgr.y (mgr.14150) 9 : cluster [DBG] pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:24:47.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:46 smithi136 bash[22601]: audit 2024-09-12T23:24:45.742978+0000 mon.a (mon.0) 101 : audit [INF] from='client.? 172.21.15.136:0/996607354' entity='client.admin' 2024-09-12T23:24:47.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:46 smithi136 bash[22601]: audit 2024-09-12T23:24:45.754054+0000 mon.a (mon.0) 102 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:24:47.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:46 smithi136 bash[22601]: audit 2024-09-12T23:24:45.761008+0000 mon.a (mon.0) 103 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:24:49.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:48 smithi136 bash[22601]: cluster 2024-09-12T23:24:47.362200+0000 mgr.y (mgr.14150) 10 : cluster [DBG] pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:24:50.960 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:24:51.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:50 smithi136 bash[22601]: cluster 2024-09-12T23:24:49.362680+0000 mgr.y (mgr.14150) 11 : cluster [DBG] pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:24:53.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:52 smithi136 bash[22601]: cluster 2024-09-12T23:24:51.363089+0000 mgr.y (mgr.14150) 12 : cluster [DBG] pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:24:54.447 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi181 2024-09-12T23:24:54.447 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2024-09-12T23:24:54.447 DEBUG:teuthology.orchestra.run.smithi181:> dd of=/etc/ceph/ceph.conf 2024-09-12T23:24:54.453 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2024-09-12T23:24:54.453 DEBUG:teuthology.orchestra.run.smithi181:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-12T23:24:54.499 INFO:tasks.cephadm:Adding host smithi181 to orchestrator... 2024-09-12T23:24:54.499 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph orch host add smithi181 2024-09-12T23:24:55.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:54 smithi136 bash[22601]: cluster 2024-09-12T23:24:53.363539+0000 mgr.y (mgr.14150) 13 : cluster [DBG] pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:24:55.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:54 smithi136 bash[22601]: audit 2024-09-12T23:24:53.858578+0000 mgr.y (mgr.14150) 14 : audit [DBG] from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:24:55.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:54 smithi136 bash[22601]: audit 2024-09-12T23:24:53.863520+0000 mon.a (mon.0) 104 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:24:55.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:54 smithi136 bash[22601]: audit 2024-09-12T23:24:53.864593+0000 mon.a (mon.0) 105 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:24:55.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:54 smithi136 bash[22601]: audit 2024-09-12T23:24:53.867471+0000 mon.a (mon.0) 106 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:24:55.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:54 smithi136 bash[22601]: audit 2024-09-12T23:24:53.868376+0000 mon.a (mon.0) 107 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:24:55.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:54 smithi136 bash[22601]: cephadm 2024-09-12T23:24:53.869571+0000 mgr.y (mgr.14150) 15 : cephadm [INF] Updating smithi136:/etc/ceph/ceph.conf 2024-09-12T23:24:55.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:54 smithi136 bash[22601]: audit 2024-09-12T23:24:54.215961+0000 mon.a (mon.0) 108 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:24:55.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:54 smithi136 bash[22601]: audit 2024-09-12T23:24:54.218690+0000 mon.a (mon.0) 109 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:24:55.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:54 smithi136 bash[22601]: audit 2024-09-12T23:24:54.222023+0000 mon.a (mon.0) 110 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:24:56.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:55 smithi136 bash[22601]: cephadm 2024-09-12T23:24:53.959772+0000 mgr.y (mgr.14150) 16 : cephadm [INF] Updating smithi136:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-12T23:24:56.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:55 smithi136 bash[22601]: cephadm 2024-09-12T23:24:54.044409+0000 mgr.y (mgr.14150) 17 : cephadm [INF] Updating smithi136:/etc/ceph/ceph.client.admin.keyring 2024-09-12T23:24:56.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:55 smithi136 bash[22601]: cephadm 2024-09-12T23:24:54.127752+0000 mgr.y (mgr.14150) 18 : cephadm [INF] Updating smithi136:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-12T23:24:57.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:56 smithi136 bash[22601]: cluster 2024-09-12T23:24:55.363969+0000 mgr.y (mgr.14150) 19 : cluster [DBG] pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:24:59.198 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:24:59.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:24:58 smithi136 bash[22601]: cluster 2024-09-12T23:24:57.364413+0000 mgr.y (mgr.14150) 20 : cluster [DBG] pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:01.128 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:00 smithi136 bash[22601]: cluster 2024-09-12T23:24:59.364876+0000 mgr.y (mgr.14150) 21 : cluster [DBG] pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:03.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:02 smithi136 bash[22601]: cluster 2024-09-12T23:25:01.365296+0000 mgr.y (mgr.14150) 22 : cluster [DBG] pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:04.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:03 smithi136 bash[22601]: audit 2024-09-12T23:25:02.093282+0000 mgr.y (mgr.14150) 23 : audit [DBG] from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi181", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:25:05.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:04 smithi136 bash[22601]: cephadm 2024-09-12T23:25:03.121919+0000 mgr.y (mgr.14150) 24 : cephadm [INF] Deploying cephadm binary to smithi181 2024-09-12T23:25:05.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:04 smithi136 bash[22601]: cluster 2024-09-12T23:25:03.365676+0000 mgr.y (mgr.14150) 25 : cluster [DBG] pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:05.295 INFO:teuthology.orchestra.run.smithi136.stdout:Added host 'smithi181' with addr '172.21.15.181' 2024-09-12T23:25:05.948 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph orch host ls --format=json 2024-09-12T23:25:06.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:06 smithi136 bash[22601]: audit 2024-09-12T23:25:05.294523+0000 mon.a (mon.0) 111 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:06.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:06 smithi136 bash[22601]: cephadm 2024-09-12T23:25:05.295372+0000 mgr.y (mgr.14150) 26 : cephadm [INF] Added host smithi181 2024-09-12T23:25:06.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:06 smithi136 bash[22601]: audit 2024-09-12T23:25:05.296159+0000 mon.a (mon.0) 112 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:25:06.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:06 smithi136 bash[22601]: cluster 2024-09-12T23:25:05.366062+0000 mgr.y (mgr.14150) 27 : cluster [DBG] pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:06.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:06 smithi136 bash[22601]: audit 2024-09-12T23:25:05.556552+0000 mon.a (mon.0) 113 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:08.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:07 smithi136 bash[22601]: audit 2024-09-12T23:25:06.835138+0000 mon.a (mon.0) 114 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:08.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:07 smithi136 bash[22601]: audit 2024-09-12T23:25:07.507732+0000 mon.a (mon.0) 115 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:09.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:08 smithi136 bash[22601]: cluster 2024-09-12T23:25:07.366453+0000 mgr.y (mgr.14150) 28 : cluster [DBG] pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:10.660 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:25:11.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:10 smithi136 bash[22601]: cluster 2024-09-12T23:25:09.366892+0000 mgr.y (mgr.14150) 29 : cluster [DBG] pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:13.175 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:12 smithi136 bash[22601]: cluster 2024-09-12T23:25:11.367292+0000 mgr.y (mgr.14150) 30 : cluster [DBG] pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:13.176 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:12 smithi136 bash[22601]: audit 2024-09-12T23:25:12.635824+0000 mon.a (mon.0) 116 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:13.176 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:12 smithi136 bash[22601]: audit 2024-09-12T23:25:12.641571+0000 mon.a (mon.0) 117 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:13.176 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:12 smithi136 bash[22601]: audit 2024-09-12T23:25:12.648312+0000 mon.a (mon.0) 118 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:13.176 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:12 smithi136 bash[22601]: audit 2024-09-12T23:25:12.654059+0000 mon.a (mon.0) 119 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:13.176 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:12 smithi136 bash[22601]: audit 2024-09-12T23:25:12.655624+0000 mon.a (mon.0) 120 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi181", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:25:13.176 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:12 smithi136 bash[22601]: audit 2024-09-12T23:25:12.658035+0000 mon.a (mon.0) 121 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:25:13.176 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:12 smithi136 bash[22601]: audit 2024-09-12T23:25:12.659761+0000 mon.a (mon.0) 122 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:25:13.513 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-12T23:25:13.514 INFO:teuthology.orchestra.run.smithi136.stdout:[{"addr": "172.21.15.136", "hostname": "smithi136", "labels": [], "status": ""}, {"addr": "172.21.15.181", "hostname": "smithi181", "labels": [], "status": ""}] 2024-09-12T23:25:14.063 INFO:tasks.cephadm:Setting crush tunables to default 2024-09-12T23:25:14.063 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph osd crush tunables default 2024-09-12T23:25:14.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:13 smithi136 bash[22601]: cephadm 2024-09-12T23:25:12.662155+0000 mgr.y (mgr.14150) 31 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.conf 2024-09-12T23:25:14.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:13 smithi136 bash[22601]: cephadm 2024-09-12T23:25:12.836796+0000 mgr.y (mgr.14150) 32 : cephadm [INF] Updating smithi181:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-12T23:25:14.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:13 smithi136 bash[22601]: audit 2024-09-12T23:25:13.249492+0000 mon.a (mon.0) 123 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:14.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:13 smithi136 bash[22601]: audit 2024-09-12T23:25:13.254461+0000 mon.a (mon.0) 124 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:14.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:13 smithi136 bash[22601]: audit 2024-09-12T23:25:13.259954+0000 mon.a (mon.0) 125 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:15.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:14 smithi136 bash[22601]: cephadm 2024-09-12T23:25:12.983102+0000 mgr.y (mgr.14150) 33 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2024-09-12T23:25:15.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:14 smithi136 bash[22601]: cephadm 2024-09-12T23:25:13.119506+0000 mgr.y (mgr.14150) 34 : cephadm [INF] Updating smithi181:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-12T23:25:15.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:14 smithi136 bash[22601]: cluster 2024-09-12T23:25:13.367657+0000 mgr.y (mgr.14150) 35 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:15.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:14 smithi136 bash[22601]: audit 2024-09-12T23:25:13.513168+0000 mgr.y (mgr.14150) 36 : audit [DBG] from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-12T23:25:17.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:16 smithi136 bash[22601]: cluster 2024-09-12T23:25:15.368047+0000 mgr.y (mgr.14150) 37 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:18.774 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:25:19.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:18 smithi136 bash[22601]: cluster 2024-09-12T23:25:17.368445+0000 mgr.y (mgr.14150) 38 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:21.198 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:20 smithi136 bash[22601]: cluster 2024-09-12T23:25:19.368877+0000 mgr.y (mgr.14150) 39 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:21.864 INFO:teuthology.orchestra.run.smithi136.stderr:adjusted tunables profile to default 2024-09-12T23:25:22.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:21 smithi136 bash[22601]: audit 2024-09-12T23:25:21.528875+0000 mon.a (mon.0) 126 : audit [INF] from='client.? 172.21.15.136:0/4065426733' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-12T23:25:22.453 INFO:tasks.cephadm:Adding mon.a on smithi136 2024-09-12T23:25:22.453 INFO:tasks.cephadm:Adding mon.c on smithi136 2024-09-12T23:25:22.453 INFO:tasks.cephadm:Adding mon.b on smithi181 2024-09-12T23:25:22.453 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph orch apply mon '3;smithi136:172.21.15.136=a;smithi136:[v2:172.21.15.136:3301,v1:172.21.15.136:6790]=c;smithi181:172.21.15.181=b' 2024-09-12T23:25:23.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:22 smithi136 bash[22601]: cluster 2024-09-12T23:25:21.369150+0000 mgr.y (mgr.14150) 40 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:23.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:22 smithi136 bash[22601]: audit 2024-09-12T23:25:21.864762+0000 mon.a (mon.0) 127 : audit [INF] from='client.? 172.21.15.136:0/4065426733' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-12T23:25:23.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:22 smithi136 bash[22601]: cluster 2024-09-12T23:25:21.864855+0000 mon.a (mon.0) 128 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-09-12T23:25:23.668 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-12T23:25:25.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:24 smithi136 bash[22601]: cluster 2024-09-12T23:25:23.369498+0000 mgr.y (mgr.14150) 41 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:26.316 INFO:teuthology.orchestra.run.smithi181.stdout:Scheduled mon update... 2024-09-12T23:25:27.106 DEBUG:teuthology.orchestra.run.smithi136:mon.c> sudo journalctl -f -n 0 -u ceph-da58a47c-715d-11ef-bceb-c7b262605968@mon.c.service 2024-09-12T23:25:27.109 DEBUG:teuthology.orchestra.run.smithi181:mon.b> sudo journalctl -f -n 0 -u ceph-da58a47c-715d-11ef-bceb-c7b262605968@mon.b.service 2024-09-12T23:25:27.113 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-12T23:25:27.113 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-12T23:25:27.118 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:26 smithi136 bash[22601]: cluster 2024-09-12T23:25:25.369873+0000 mgr.y (mgr.14150) 42 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:27.118 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:26 smithi136 bash[22601]: audit 2024-09-12T23:25:26.314317+0000 mon.a (mon.0) 129 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:27.118 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:26 smithi136 bash[22601]: audit 2024-09-12T23:25:26.315654+0000 mon.a (mon.0) 130 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:25:27.118 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:26 smithi136 bash[22601]: audit 2024-09-12T23:25:26.319375+0000 mon.a (mon.0) 131 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:25:27.118 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:26 smithi136 bash[22601]: audit 2024-09-12T23:25:26.321301+0000 mon.a (mon.0) 132 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:25:27.118 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:26 smithi136 bash[22601]: audit 2024-09-12T23:25:26.331025+0000 mon.a (mon.0) 133 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:27.118 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:26 smithi136 bash[22601]: audit 2024-09-12T23:25:26.334698+0000 mon.a (mon.0) 134 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-12T23:25:27.118 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:26 smithi136 bash[22601]: audit 2024-09-12T23:25:26.336497+0000 mon.a (mon.0) 135 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:25:28.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:27 smithi136 bash[22601]: audit 2024-09-12T23:25:26.301366+0000 mgr.y (mgr.14150) 43 : audit [DBG] from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi136:172.21.15.136=a;smithi136:[v2:172.21.15.136:3301,v1:172.21.15.136:6790]=c;smithi181:172.21.15.181=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:25:28.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:27 smithi136 bash[22601]: cephadm 2024-09-12T23:25:26.306863+0000 mgr.y (mgr.14150) 44 : cephadm [INF] Saving service mon spec with placement smithi136:172.21.15.136=a;smithi136:[v2:172.21.15.136:3301,v1:172.21.15.136:6790]=c;smithi181:172.21.15.181=b;count:3 2024-09-12T23:25:28.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:27 smithi136 bash[22601]: cephadm 2024-09-12T23:25:26.338317+0000 mgr.y (mgr.14150) 45 : cephadm [INF] Deploying daemon mon.b on smithi181 2024-09-12T23:25:28.353 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-12T23:25:29.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:28 smithi136 bash[22601]: cluster 2024-09-12T23:25:27.370354+0000 mgr.y (mgr.14150) 46 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:31.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:30 smithi136 bash[22601]: cluster 2024-09-12T23:25:29.370749+0000 mgr.y (mgr.14150) 47 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:31.822 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:31 smithi181 systemd[1]: Started Ceph mon.b for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:25:32.012 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-09-12T23:25:32.012 INFO:teuthology.orchestra.run.smithi181.stdout:{"epoch":1,"fsid":"da58a47c-715d-11ef-bceb-c7b262605968","modified":"2024-09-12T23:23:24.585690Z","created":"2024-09-12T23:23:24.585690Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:3300","nonce":0},{"type":"v1","addr":"172.21.15.136:6789","nonce":0}]},"addr":"172.21.15.136:6789/0","public_addr":"172.21.15.136:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-12T23:25:32.013 INFO:teuthology.orchestra.run.smithi181.stderr:dumped monmap epoch 1 2024-09-12T23:25:32.983 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:32 smithi136 bash[22601]: cluster 2024-09-12T23:25:31.371134+0000 mgr.y (mgr.14150) 48 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:32.983 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:32 smithi136 bash[22601]: audit 2024-09-12T23:25:31.707954+0000 mon.a (mon.0) 136 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:32.983 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:32 smithi136 bash[22601]: audit 2024-09-12T23:25:31.714265+0000 mon.a (mon.0) 137 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:32.983 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:32 smithi136 bash[22601]: audit 2024-09-12T23:25:31.720924+0000 mon.a (mon.0) 138 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:32.984 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:32 smithi136 bash[22601]: audit 2024-09-12T23:25:31.722081+0000 mon.a (mon.0) 139 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-12T23:25:32.984 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:32 smithi136 bash[22601]: audit 2024-09-12T23:25:31.723749+0000 mon.a (mon.0) 140 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:25:32.984 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:32 smithi136 bash[22601]: cephadm 2024-09-12T23:25:31.725609+0000 mgr.y (mgr.14150) 49 : cephadm [INF] Deploying daemon mon.c on smithi136 2024-09-12T23:25:32.984 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:32 smithi136 bash[22601]: audit 2024-09-12T23:25:32.012208+0000 mon.a (mon.0) 141 : audit [DBG] from='client.? 172.21.15.181:0/3941909899' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-12T23:25:33.148 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.107+0000 7f1adf56d900 0 set uid:gid to 167:167 (ceph:ceph) 2024-09-12T23:25:33.148 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.107+0000 7f1adf56d900 0 ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable), process ceph-mon, pid 7 2024-09-12T23:25:33.148 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.107+0000 7f1adf56d900 0 pidfile_write: ignore empty --pid-file 2024-09-12T23:25:33.148 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 0 load: jerasure load: lrc 2024-09-12T23:25:33.148 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: RocksDB version: 6.15.5 2024-09-12T23:25:33.149 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-12T23:25:33.149 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Compile date Sep 12 2024 2024-09-12T23:25:33.149 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: DB SUMMARY 2024-09-12T23:25:33.149 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: DB Session ID: C78CXG6IQOH7W5D7VLIL 2024-09-12T23:25:33.149 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: CURRENT file: CURRENT 2024-09-12T23:25:33.149 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: IDENTITY file: IDENTITY 2024-09-12T23:25:33.149 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-09-12T23:25:33.149 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2024-09-12T23:25:33.149 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000004.log size: 511 ; 2024-09-12T23:25:33.149 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.error_if_exists: 0 2024-09-12T23:25:33.149 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.create_if_missing: 0 2024-09-12T23:25:33.150 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.paranoid_checks: 1 2024-09-12T23:25:33.150 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-12T23:25:33.150 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.env: 0x55d65fb37440 2024-09-12T23:25:33.150 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.fs: Posix File System 2024-09-12T23:25:33.150 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.info_log: 0x55d661b30ee0 2024-09-12T23:25:33.150 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.max_file_opening_threads: 16 2024-09-12T23:25:33.150 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.statistics: (nil) 2024-09-12T23:25:33.150 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.use_fsync: 0 2024-09-12T23:25:33.150 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.max_log_file_size: 0 2024-09-12T23:25:33.150 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-12T23:25:33.150 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.log_file_time_to_roll: 0 2024-09-12T23:25:33.151 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.keep_log_file_num: 1000 2024-09-12T23:25:33.151 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.recycle_log_file_num: 0 2024-09-12T23:25:33.151 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.allow_fallocate: 1 2024-09-12T23:25:33.151 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.allow_mmap_reads: 0 2024-09-12T23:25:33.151 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.allow_mmap_writes: 0 2024-09-12T23:25:33.151 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.use_direct_reads: 0 2024-09-12T23:25:33.151 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-12T23:25:33.151 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.create_missing_column_families: 0 2024-09-12T23:25:33.151 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.db_log_dir: 2024-09-12T23:25:33.151 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2024-09-12T23:25:33.151 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.table_cache_numshardbits: 6 2024-09-12T23:25:33.151 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-09-12T23:25:33.152 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-09-12T23:25:33.152 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-12T23:25:33.152 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-12T23:25:33.152 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-09-12T23:25:33.152 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.advise_random_on_open: 1 2024-09-12T23:25:33.152 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.db_write_buffer_size: 0 2024-09-12T23:25:33.152 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.write_buffer_manager: 0x55d661bee780 2024-09-12T23:25:33.152 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-12T23:25:33.152 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-12T23:25:33.152 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-12T23:25:33.153 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.use_adaptive_mutex: 0 2024-09-12T23:25:33.153 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.rate_limiter: (nil) 2024-09-12T23:25:33.156 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-12T23:25:33.156 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.wal_recovery_mode: 2 2024-09-12T23:25:33.156 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.enable_thread_tracking: 0 2024-09-12T23:25:33.158 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.enable_pipelined_write: 0 2024-09-12T23:25:33.158 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.unordered_write: 0 2024-09-12T23:25:33.158 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-12T23:25:33.158 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-12T23:25:33.158 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-12T23:25:33.159 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-12T23:25:33.159 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.row_cache: None 2024-09-12T23:25:33.159 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.wal_filter: None 2024-09-12T23:25:33.159 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-12T23:25:33.159 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.allow_ingest_behind: 0 2024-09-12T23:25:33.159 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.preserve_deletes: 0 2024-09-12T23:25:33.159 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.two_write_queues: 0 2024-09-12T23:25:33.159 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.manual_wal_flush: 0 2024-09-12T23:25:33.159 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.atomic_flush: 0 2024-09-12T23:25:33.159 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-12T23:25:33.159 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.persist_stats_to_disk: 0 2024-09-12T23:25:33.159 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-09-12T23:25:33.159 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.log_readahead_size: 0 2024-09-12T23:25:33.159 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-12T23:25:33.160 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.best_efforts_recovery: 0 2024-09-12T23:25:33.160 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-12T23:25:33.160 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-12T23:25:33.160 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.allow_data_in_errors: 0 2024-09-12T23:25:33.160 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.db_host_id: __hostname__ 2024-09-12T23:25:33.160 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.max_background_jobs: 2 2024-09-12T23:25:33.160 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.max_background_compactions: -1 2024-09-12T23:25:33.160 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.max_subcompactions: 1 2024-09-12T23:25:33.160 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-12T23:25:33.160 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-12T23:25:33.160 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.delayed_write_rate : 16777216 2024-09-12T23:25:33.160 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.max_total_wal_size: 0 2024-09-12T23:25:33.160 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-12T23:25:33.161 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.stats_dump_period_sec: 600 2024-09-12T23:25:33.161 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.stats_persist_period_sec: 600 2024-09-12T23:25:33.161 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-12T23:25:33.161 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.max_open_files: -1 2024-09-12T23:25:33.161 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.bytes_per_sync: 0 2024-09-12T23:25:33.161 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-09-12T23:25:33.161 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-09-12T23:25:33.161 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.compaction_readahead_size: 0 2024-09-12T23:25:33.161 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.max_background_flushes: -1 2024-09-12T23:25:33.161 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Compression algorithms supported: 2024-09-12T23:25:33.161 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: kZSTD supported: 0 2024-09-12T23:25:33.161 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: kXpressCompression supported: 0 2024-09-12T23:25:33.161 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: kBZip2Compression supported: 0 2024-09-12T23:25:33.162 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-12T23:25:33.162 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: kLZ4Compression supported: 1 2024-09-12T23:25:33.162 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: kZlibCompression supported: 1 2024-09-12T23:25:33.162 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: kLZ4HCCompression supported: 1 2024-09-12T23:25:33.162 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: kSnappyCompression supported: 1 2024-09-12T23:25:33.162 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-09-12T23:25:33.162 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 2024-09-12T23:25:33.162 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-09-12T23:25:33.162 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-12T23:25:33.162 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.merge_operator: 2024-09-12T23:25:33.162 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.compaction_filter: None 2024-09-12T23:25:33.162 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.compaction_filter_factory: None 2024-09-12T23:25:33.163 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.sst_partitioner_factory: None 2024-09-12T23:25:33.163 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-09-12T23:25:33.163 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.table_factory: BlockBasedTable 2024-09-12T23:25:33.163 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55d661ad9a38) 2024-09-12T23:25:33.163 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: cache_index_and_filter_blocks: 1 2024-09-12T23:25:33.163 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-12T23:25:33.163 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-12T23:25:33.163 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: pin_top_level_index_and_filter: 1 2024-09-12T23:25:33.163 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: index_type: 0 2024-09-12T23:25:33.163 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: data_block_index_type: 0 2024-09-12T23:25:33.163 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: index_shortening: 1 2024-09-12T23:25:33.163 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: data_block_hash_table_util_ratio: 0.750000 2024-09-12T23:25:33.163 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: hash_index_allow_collision: 1 2024-09-12T23:25:33.164 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: checksum: 1 2024-09-12T23:25:33.164 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: no_block_cache: 0 2024-09-12T23:25:33.164 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: block_cache: 0x55d661b0d1f0 2024-09-12T23:25:33.164 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: block_cache_name: BinnedLRUCache 2024-09-12T23:25:33.164 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: block_cache_options: 2024-09-12T23:25:33.164 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: capacity : 536870912 2024-09-12T23:25:33.164 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: num_shard_bits : 4 2024-09-12T23:25:33.164 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: strict_capacity_limit : 0 2024-09-12T23:25:33.164 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: high_pri_pool_ratio: 0.000 2024-09-12T23:25:33.164 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: block_cache_compressed: (nil) 2024-09-12T23:25:33.164 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: persistent_cache: (nil) 2024-09-12T23:25:33.164 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: block_size: 4096 2024-09-12T23:25:33.164 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: block_size_deviation: 10 2024-09-12T23:25:33.165 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: block_restart_interval: 16 2024-09-12T23:25:33.165 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: index_block_restart_interval: 1 2024-09-12T23:25:33.165 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: metadata_block_size: 4096 2024-09-12T23:25:33.165 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: partition_filters: 0 2024-09-12T23:25:33.165 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: use_delta_encoding: 1 2024-09-12T23:25:33.165 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-12T23:25:33.165 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: whole_key_filtering: 1 2024-09-12T23:25:33.165 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: verify_compression: 0 2024-09-12T23:25:33.165 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: read_amp_bytes_per_bit: 0 2024-09-12T23:25:33.165 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: format_version: 4 2024-09-12T23:25:33.165 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: enable_index_compression: 1 2024-09-12T23:25:33.165 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: block_align: 0 2024-09-12T23:25:33.165 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.write_buffer_size: 33554432 2024-09-12T23:25:33.166 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.max_write_buffer_number: 2 2024-09-12T23:25:33.166 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.compression: NoCompression 2024-09-12T23:25:33.166 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.bottommost_compression: Disabled 2024-09-12T23:25:33.166 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.prefix_extractor: nullptr 2024-09-12T23:25:33.166 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-12T23:25:33.166 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.num_levels: 7 2024-09-12T23:25:33.166 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-12T23:25:33.166 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-12T23:25:33.166 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-12T23:25:33.166 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-12T23:25:33.166 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-12T23:25:33.166 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-12T23:25:33.167 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-12T23:25:33.167 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-12T23:25:33.167 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-12T23:25:33.167 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-12T23:25:33.167 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.compression_opts.window_bits: -14 2024-09-12T23:25:33.167 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.compression_opts.level: 32767 2024-09-12T23:25:33.167 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.compression_opts.strategy: 0 2024-09-12T23:25:33.167 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-12T23:25:33.167 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-12T23:25:33.167 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-12T23:25:33.167 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.compression_opts.enabled: false 2024-09-12T23:25:33.167 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-12T23:25:33.167 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-12T23:25:33.168 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-12T23:25:33.168 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.target_file_size_base: 67108864 2024-09-12T23:25:33.168 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.target_file_size_multiplier: 1 2024-09-12T23:25:33.168 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-12T23:25:33.168 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-12T23:25:33.168 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-12T23:25:33.168 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-12T23:25:33.168 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-12T23:25:33.168 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-12T23:25:33.168 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-12T23:25:33.168 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-12T23:25:33.168 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-12T23:25:33.168 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-12T23:25:33.169 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-12T23:25:33.169 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-12T23:25:33.169 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.arena_block_size: 4194304 2024-09-12T23:25:33.169 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-12T23:25:33.169 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-12T23:25:33.169 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-12T23:25:33.169 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.disable_auto_compactions: 0 2024-09-12T23:25:33.169 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-12T23:25:33.169 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-12T23:25:33.170 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-12T23:25:33.170 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-12T23:25:33.170 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-12T23:25:33.170 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-12T23:25:33.170 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-12T23:25:33.170 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-12T23:25:33.170 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-12T23:25:33.170 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-12T23:25:33.170 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-12T23:25:33.170 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.inplace_update_support: 0 2024-09-12T23:25:33.170 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-09-12T23:25:33.170 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-12T23:25:33.171 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-12T23:25:33.171 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.memtable_huge_page_size: 0 2024-09-12T23:25:33.171 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.bloom_locality: 0 2024-09-12T23:25:33.171 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.max_successive_merges: 0 2024-09-12T23:25:33.171 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-09-12T23:25:33.171 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.paranoid_file_checks: 0 2024-09-12T23:25:33.171 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.force_consistency_checks: 1 2024-09-12T23:25:33.171 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.report_bg_io_stats: 0 2024-09-12T23:25:33.171 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.ttl: 2592000 2024-09-12T23:25:33.171 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-09-12T23:25:33.171 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.enable_blob_files: false 2024-09-12T23:25:33.171 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.min_blob_size: 0 2024-09-12T23:25:33.171 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.blob_file_size: 268435456 2024-09-12T23:25:33.172 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.blob_compression_type: NoCompression 2024-09-12T23:25:33.172 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.enable_blob_garbage_collection: false 2024-09-12T23:25:33.172 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.111+0000 7f1adf56d900 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-12T23:25:33.172 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.115+0000 7f1adf56d900 4 rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-12T23:25:33.172 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.115+0000 7f1adf56d900 4 rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2024-09-12T23:25:33.172 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.115+0000 7f1adf56d900 4 rocksdb: [db/version_set.cc:4082] Creating manifest 7 2024-09-12T23:25:33.172 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.115+0000 7f1adf56d900 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726183533117056, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-09-12T23:25:33.172 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.115+0000 7f1adf56d900 4 rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2024-09-12T23:25:33.172 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.115+0000 7f1adf56d900 3 rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-12T23:25:33.172 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.115+0000 7f1adf56d900 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726183533117426, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1571, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726183533, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "ebd3e9c1-ca35-43b2-9a78-ef88d46dad0e", "db_session_id": "C78CXG6IQOH7W5D7VLIL"}} 2024-09-12T23:25:33.172 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.115+0000 7f1adf56d900 4 rocksdb: [db/version_set.cc:4082] Creating manifest 9 2024-09-12T23:25:33.172 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.115+0000 7f1adf56d900 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726183533118506, "job": 1, "event": "recovery_finished"} 2024-09-12T23:25:33.173 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.115+0000 7f1adf56d900 4 rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-12T23:25:33.173 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.115+0000 7f1adf56d900 4 rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x55d661b42700 2024-09-12T23:25:33.173 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.115+0000 7f1adf56d900 4 rocksdb: DB pointer 0x55d661b10000 2024-09-12T23:25:33.173 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.115+0000 7f1ad5339640 4 rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-09-12T23:25:33.173 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.115+0000 7f1ad5339640 4 rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-12T23:25:33.173 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: ** DB Stats ** 2024-09-12T23:25:33.173 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: Uptime(secs): 0.0 total, 0.0 interval 2024-09-12T23:25:33.173 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-12T23:25:33.173 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-12T23:25:33.173 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-12T23:25:33.173 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-12T23:25:33.173 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-12T23:25:33.174 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-12T23:25:33.174 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: ** Compaction Stats [default] ** 2024-09-12T23:25:33.174 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-12T23:25:33.174 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-12T23:25:33.174 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: L0 1/0 1.53 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.2 0.00 0.00 1 0.000 0 0 2024-09-12T23:25:33.174 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: Sum 1/0 1.53 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.2 0.00 0.00 1 0.000 0 0 2024-09-12T23:25:33.174 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.2 0.00 0.00 1 0.000 0 0 2024-09-12T23:25:33.174 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: ** Compaction Stats [default] ** 2024-09-12T23:25:33.174 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-12T23:25:33.174 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-12T23:25:33.174 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.2 0.00 0.00 1 0.000 0 0 2024-09-12T23:25:33.174 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: Uptime(secs): 0.0 total, 0.0 interval 2024-09-12T23:25:33.174 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: Flush(GB): cumulative 0.000, interval 0.000 2024-09-12T23:25:33.175 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-12T23:25:33.175 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: AddFile(Total Files): cumulative 0, interval 0 2024-09-12T23:25:33.175 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: AddFile(L0 Files): cumulative 0, interval 0 2024-09-12T23:25:33.175 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: AddFile(Keys): cumulative 0, interval 0 2024-09-12T23:25:33.175 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: Cumulative compaction: 0.00 GB write, 0.36 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-12T23:25:33.175 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: Interval compaction: 0.00 GB write, 0.36 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-12T23:25:33.175 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-12T23:25:33.175 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: ** File Read Latency Histogram By Level [default] ** 2024-09-12T23:25:33.175 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: ** Compaction Stats [default] ** 2024-09-12T23:25:33.175 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-12T23:25:33.175 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-12T23:25:33.175 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: L0 1/0 1.53 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.2 0.00 0.00 1 0.000 0 0 2024-09-12T23:25:33.176 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: Sum 1/0 1.53 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.2 0.00 0.00 1 0.000 0 0 2024-09-12T23:25:33.176 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-12T23:25:33.176 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: ** Compaction Stats [default] ** 2024-09-12T23:25:33.176 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-12T23:25:33.176 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-12T23:25:33.176 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.2 0.00 0.00 1 0.000 0 0 2024-09-12T23:25:33.176 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: Uptime(secs): 0.0 total, 0.0 interval 2024-09-12T23:25:33.176 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: Flush(GB): cumulative 0.000, interval 0.000 2024-09-12T23:25:33.176 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-12T23:25:33.176 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: AddFile(Total Files): cumulative 0, interval 0 2024-09-12T23:25:33.176 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: AddFile(L0 Files): cumulative 0, interval 0 2024-09-12T23:25:33.176 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: AddFile(Keys): cumulative 0, interval 0 2024-09-12T23:25:33.177 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: Cumulative compaction: 0.00 GB write, 0.36 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-12T23:25:33.177 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-12T23:25:33.177 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-12T23:25:33.177 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: ** File Read Latency Histogram By Level [default] ** 2024-09-12T23:25:33.177 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.119+0000 7f1adf56d900 0 mon.b does not exist in monmap, will attempt to join an existing cluster 2024-09-12T23:25:33.177 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.119+0000 7f1adf56d900 0 using public_addr v2:172.21.15.181:0/0 -> [v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0] 2024-09-12T23:25:33.177 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.119+0000 7f1adf56d900 0 starting mon.b rank -1 at public addrs [v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0] at bind addrs [v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid da58a47c-715d-11ef-bceb-c7b262605968 2024-09-12T23:25:33.177 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.119+0000 7f1adf56d900 1 mon.b@-1(???) e0 preinit fsid da58a47c-715d-11ef-bceb-c7b262605968 2024-09-12T23:25:33.177 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.143+0000 7f1ad833f640 0 mon.b@-1(synchronizing).mds e1 new map 2024-09-12T23:25:33.177 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.143+0000 7f1ad833f640 0 mon.b@-1(synchronizing).mds e1 print_map 2024-09-12T23:25:33.177 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: e1 2024-09-12T23:25:33.177 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: enable_multiple, ever_enabled_multiple: 1,1 2024-09-12T23:25:33.177 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-12T23:25:33.178 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: legacy client fscid: -1 2024-09-12T23:25:33.178 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: 2024-09-12T23:25:33.178 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: No filesystems configured 2024-09-12T23:25:33.178 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.143+0000 7f1ad833f640 1 mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-09-12T23:25:33.178 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.143+0000 7f1ad833f640 1 mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-09-12T23:25:33.178 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.143+0000 7f1ad833f640 1 mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-09-12T23:25:33.178 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.143+0000 7f1ad833f640 1 mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-09-12T23:25:33.178 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.143+0000 7f1ad833f640 1 mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-09-12T23:25:33.178 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.143+0000 7f1ad833f640 1 mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-09-12T23:25:33.178 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.143+0000 7f1ad833f640 0 mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-12T23:25:33.178 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.143+0000 7f1ad833f640 0 mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-12T23:25:33.178 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.143+0000 7f1ad833f640 0 mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-12T23:25:33.178 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.143+0000 7f1ad833f640 0 mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-12T23:25:33.179 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: cluster 2024-09-12T23:24:57.364413+0000 mgr.y (mgr.14150) 20 : cluster [DBG] pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:33.179 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: cluster 2024-09-12T23:24:59.364876+0000 mgr.y (mgr.14150) 21 : cluster [DBG] pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:33.179 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: cluster 2024-09-12T23:25:01.365296+0000 mgr.y (mgr.14150) 22 : cluster [DBG] pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:33.179 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: audit 2024-09-12T23:25:02.093282+0000 mgr.y (mgr.14150) 23 : audit [DBG] from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi181", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:25:33.179 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: cephadm 2024-09-12T23:25:03.121919+0000 mgr.y (mgr.14150) 24 : cephadm [INF] Deploying cephadm binary to smithi181 2024-09-12T23:25:33.179 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: cluster 2024-09-12T23:25:03.365676+0000 mgr.y (mgr.14150) 25 : cluster [DBG] pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:33.179 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: audit 2024-09-12T23:25:05.294523+0000 mon.a (mon.0) 111 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:33.179 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: cephadm 2024-09-12T23:25:05.295372+0000 mgr.y (mgr.14150) 26 : cephadm [INF] Added host smithi181 2024-09-12T23:25:33.179 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: audit 2024-09-12T23:25:05.296159+0000 mon.a (mon.0) 112 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:25:33.179 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: cluster 2024-09-12T23:25:05.366062+0000 mgr.y (mgr.14150) 27 : cluster [DBG] pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:33.179 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: audit 2024-09-12T23:25:05.556552+0000 mon.a (mon.0) 113 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:33.179 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: audit 2024-09-12T23:25:06.835138+0000 mon.a (mon.0) 114 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:33.179 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: audit 2024-09-12T23:25:07.507732+0000 mon.a (mon.0) 115 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:33.180 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: cluster 2024-09-12T23:25:07.366453+0000 mgr.y (mgr.14150) 28 : cluster [DBG] pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:33.180 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: cluster 2024-09-12T23:25:09.366892+0000 mgr.y (mgr.14150) 29 : cluster [DBG] pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:33.180 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: cluster 2024-09-12T23:25:11.367292+0000 mgr.y (mgr.14150) 30 : cluster [DBG] pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:33.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: audit 2024-09-12T23:25:12.635824+0000 mon.a (mon.0) 116 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:33.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: audit 2024-09-12T23:25:12.641571+0000 mon.a (mon.0) 117 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:33.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: audit 2024-09-12T23:25:12.648312+0000 mon.a (mon.0) 118 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:33.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: audit 2024-09-12T23:25:12.654059+0000 mon.a (mon.0) 119 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:33.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: audit 2024-09-12T23:25:12.655624+0000 mon.a (mon.0) 120 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi181", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:25:33.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: audit 2024-09-12T23:25:12.658035+0000 mon.a (mon.0) 121 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:25:33.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: audit 2024-09-12T23:25:12.659761+0000 mon.a (mon.0) 122 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:25:33.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: cephadm 2024-09-12T23:25:12.662155+0000 mgr.y (mgr.14150) 31 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.conf 2024-09-12T23:25:33.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: cephadm 2024-09-12T23:25:12.836796+0000 mgr.y (mgr.14150) 32 : cephadm [INF] Updating smithi181:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-12T23:25:33.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: audit 2024-09-12T23:25:13.249492+0000 mon.a (mon.0) 123 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:33.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: audit 2024-09-12T23:25:13.254461+0000 mon.a (mon.0) 124 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:33.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: audit 2024-09-12T23:25:13.259954+0000 mon.a (mon.0) 125 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:33.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: cephadm 2024-09-12T23:25:12.983102+0000 mgr.y (mgr.14150) 33 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2024-09-12T23:25:33.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: cephadm 2024-09-12T23:25:13.119506+0000 mgr.y (mgr.14150) 34 : cephadm [INF] Updating smithi181:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-12T23:25:33.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: cluster 2024-09-12T23:25:13.367657+0000 mgr.y (mgr.14150) 35 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:33.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: audit 2024-09-12T23:25:13.513168+0000 mgr.y (mgr.14150) 36 : audit [DBG] from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-12T23:25:33.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: cluster 2024-09-12T23:25:15.368047+0000 mgr.y (mgr.14150) 37 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:33.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: cluster 2024-09-12T23:25:17.368445+0000 mgr.y (mgr.14150) 38 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:33.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: cluster 2024-09-12T23:25:19.368877+0000 mgr.y (mgr.14150) 39 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:33.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: audit 2024-09-12T23:25:21.528875+0000 mon.a (mon.0) 126 : audit [INF] from='client.? 172.21.15.136:0/4065426733' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-12T23:25:33.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: cluster 2024-09-12T23:25:21.369150+0000 mgr.y (mgr.14150) 40 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:33.352 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: audit 2024-09-12T23:25:21.864762+0000 mon.a (mon.0) 127 : audit [INF] from='client.? 172.21.15.136:0/4065426733' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-12T23:25:33.352 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: cluster 2024-09-12T23:25:21.864855+0000 mon.a (mon.0) 128 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-09-12T23:25:33.352 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: cluster 2024-09-12T23:25:23.369498+0000 mgr.y (mgr.14150) 41 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:33.352 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: cluster 2024-09-12T23:25:25.369873+0000 mgr.y (mgr.14150) 42 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:33.352 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: audit 2024-09-12T23:25:26.314317+0000 mon.a (mon.0) 129 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:33.352 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: audit 2024-09-12T23:25:26.315654+0000 mon.a (mon.0) 130 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:25:33.352 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: audit 2024-09-12T23:25:26.319375+0000 mon.a (mon.0) 131 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:25:33.352 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: audit 2024-09-12T23:25:26.321301+0000 mon.a (mon.0) 132 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:25:33.352 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: audit 2024-09-12T23:25:26.331025+0000 mon.a (mon.0) 133 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:33.352 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: audit 2024-09-12T23:25:26.334698+0000 mon.a (mon.0) 134 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-12T23:25:33.353 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: audit 2024-09-12T23:25:26.336497+0000 mon.a (mon.0) 135 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:25:33.353 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: audit 2024-09-12T23:25:26.301366+0000 mgr.y (mgr.14150) 43 : audit [DBG] from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi136:172.21.15.136=a;smithi136:[v2:172.21.15.136:3301,v1:172.21.15.136:6790]=c;smithi181:172.21.15.181=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:25:33.353 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: cephadm 2024-09-12T23:25:26.306863+0000 mgr.y (mgr.14150) 44 : cephadm [INF] Saving service mon spec with placement smithi136:172.21.15.136=a;smithi136:[v2:172.21.15.136:3301,v1:172.21.15.136:6790]=c;smithi181:172.21.15.181=b;count:3 2024-09-12T23:25:33.353 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: cephadm 2024-09-12T23:25:26.338317+0000 mgr.y (mgr.14150) 45 : cephadm [INF] Deploying daemon mon.b on smithi181 2024-09-12T23:25:33.353 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: cluster 2024-09-12T23:25:27.370354+0000 mgr.y (mgr.14150) 46 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:33.353 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: cluster 2024-09-12T23:25:29.370749+0000 mgr.y (mgr.14150) 47 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:33.353 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: cluster 2024-09-12T23:25:31.371134+0000 mgr.y (mgr.14150) 48 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:33.353 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: audit 2024-09-12T23:25:31.707954+0000 mon.a (mon.0) 136 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:33.353 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: audit 2024-09-12T23:25:31.714265+0000 mon.a (mon.0) 137 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:33.353 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: audit 2024-09-12T23:25:31.720924+0000 mon.a (mon.0) 138 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:33.353 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: audit 2024-09-12T23:25:31.722081+0000 mon.a (mon.0) 139 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-12T23:25:33.353 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: audit 2024-09-12T23:25:31.723749+0000 mon.a (mon.0) 140 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:25:33.353 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: cephadm 2024-09-12T23:25:31.725609+0000 mgr.y (mgr.14150) 49 : cephadm [INF] Deploying daemon mon.c on smithi136 2024-09-12T23:25:33.354 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: audit 2024-09-12T23:25:32.012208+0000 mon.a (mon.0) 141 : audit [DBG] from='client.? 172.21.15.181:0/3941909899' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-12T23:25:33.354 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:33 smithi181 bash[24776]: debug 2024-09-12T23:25:33.147+0000 7f1ad833f640 1 mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-09-12T23:25:33.901 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-12T23:25:33.901 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-12T23:25:35.245 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:35 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:25:35.245 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:25:35 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:25:35.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:35 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:25:35.510 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:25:35 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:25:35.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:35 smithi136 systemd[1]: Started Ceph mon.c for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:25:36.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 0 set uid:gid to 167:167 (ceph:ceph) 2024-09-12T23:25:36.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 0 ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable), process ceph-mon, pid 7 2024-09-12T23:25:36.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 0 pidfile_write: ignore empty --pid-file 2024-09-12T23:25:36.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 0 load: jerasure load: lrc 2024-09-12T23:25:36.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: RocksDB version: 6.15.5 2024-09-12T23:25:36.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-12T23:25:36.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Compile date Sep 12 2024 2024-09-12T23:25:36.762 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: DB SUMMARY 2024-09-12T23:25:36.762 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: DB Session ID: B6609I289JC2YHJGV7M2 2024-09-12T23:25:36.762 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: CURRENT file: CURRENT 2024-09-12T23:25:36.762 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: IDENTITY file: IDENTITY 2024-09-12T23:25:36.762 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-09-12T23:25:36.762 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2024-09-12T23:25:36.762 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 511 ; 2024-09-12T23:25:36.762 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.error_if_exists: 0 2024-09-12T23:25:36.762 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.create_if_missing: 0 2024-09-12T23:25:36.762 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.paranoid_checks: 1 2024-09-12T23:25:36.762 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-12T23:25:36.762 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.env: 0x5582376e1440 2024-09-12T23:25:36.763 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.fs: Posix File System 2024-09-12T23:25:36.763 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.info_log: 0x5582396e2ee0 2024-09-12T23:25:36.763 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.max_file_opening_threads: 16 2024-09-12T23:25:36.763 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.statistics: (nil) 2024-09-12T23:25:36.763 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.use_fsync: 0 2024-09-12T23:25:36.763 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.max_log_file_size: 0 2024-09-12T23:25:36.763 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-12T23:25:36.763 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.log_file_time_to_roll: 0 2024-09-12T23:25:36.763 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.keep_log_file_num: 1000 2024-09-12T23:25:36.763 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.recycle_log_file_num: 0 2024-09-12T23:25:36.764 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.allow_fallocate: 1 2024-09-12T23:25:36.764 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.allow_mmap_reads: 0 2024-09-12T23:25:36.764 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.allow_mmap_writes: 0 2024-09-12T23:25:36.764 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.use_direct_reads: 0 2024-09-12T23:25:36.764 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-12T23:25:36.764 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.create_missing_column_families: 0 2024-09-12T23:25:36.764 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.db_log_dir: 2024-09-12T23:25:36.764 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2024-09-12T23:25:36.764 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.table_cache_numshardbits: 6 2024-09-12T23:25:36.764 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-09-12T23:25:36.764 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-09-12T23:25:36.765 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-12T23:25:36.765 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-12T23:25:36.765 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-09-12T23:25:36.765 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.advise_random_on_open: 1 2024-09-12T23:25:36.765 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.db_write_buffer_size: 0 2024-09-12T23:25:36.765 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.write_buffer_manager: 0x5582397a07b0 2024-09-12T23:25:36.765 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-12T23:25:36.765 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-12T23:25:36.765 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-12T23:25:36.765 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.use_adaptive_mutex: 0 2024-09-12T23:25:36.766 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.rate_limiter: (nil) 2024-09-12T23:25:36.766 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-12T23:25:36.766 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.wal_recovery_mode: 2 2024-09-12T23:25:36.766 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.enable_thread_tracking: 0 2024-09-12T23:25:36.766 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.enable_pipelined_write: 0 2024-09-12T23:25:36.766 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.unordered_write: 0 2024-09-12T23:25:36.766 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-12T23:25:36.766 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-12T23:25:36.766 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-12T23:25:36.766 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-12T23:25:36.766 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.row_cache: None 2024-09-12T23:25:36.766 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.wal_filter: None 2024-09-12T23:25:36.767 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-12T23:25:36.767 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.allow_ingest_behind: 0 2024-09-12T23:25:36.767 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.preserve_deletes: 0 2024-09-12T23:25:36.767 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.two_write_queues: 0 2024-09-12T23:25:36.767 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.manual_wal_flush: 0 2024-09-12T23:25:36.767 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.atomic_flush: 0 2024-09-12T23:25:36.767 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-12T23:25:36.767 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.persist_stats_to_disk: 0 2024-09-12T23:25:36.767 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-09-12T23:25:36.767 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.log_readahead_size: 0 2024-09-12T23:25:36.767 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-12T23:25:36.767 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.best_efforts_recovery: 0 2024-09-12T23:25:36.768 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-12T23:25:36.768 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-12T23:25:36.768 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.allow_data_in_errors: 0 2024-09-12T23:25:36.768 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.db_host_id: __hostname__ 2024-09-12T23:25:36.768 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.max_background_jobs: 2 2024-09-12T23:25:36.768 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.max_background_compactions: -1 2024-09-12T23:25:36.768 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.max_subcompactions: 1 2024-09-12T23:25:36.768 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-12T23:25:36.768 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-12T23:25:36.768 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.delayed_write_rate : 16777216 2024-09-12T23:25:36.768 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.max_total_wal_size: 0 2024-09-12T23:25:36.769 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-12T23:25:36.769 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.stats_dump_period_sec: 600 2024-09-12T23:25:36.769 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.stats_persist_period_sec: 600 2024-09-12T23:25:36.769 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-12T23:25:36.769 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.max_open_files: -1 2024-09-12T23:25:36.769 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.bytes_per_sync: 0 2024-09-12T23:25:36.769 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-09-12T23:25:36.769 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-09-12T23:25:36.769 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.compaction_readahead_size: 0 2024-09-12T23:25:36.769 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.max_background_flushes: -1 2024-09-12T23:25:36.769 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Compression algorithms supported: 2024-09-12T23:25:36.769 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: kZSTD supported: 0 2024-09-12T23:25:36.770 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: kXpressCompression supported: 0 2024-09-12T23:25:36.770 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: kBZip2Compression supported: 0 2024-09-12T23:25:36.770 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-12T23:25:36.770 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: kLZ4Compression supported: 1 2024-09-12T23:25:36.770 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: kZlibCompression supported: 1 2024-09-12T23:25:36.770 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: kLZ4HCCompression supported: 1 2024-09-12T23:25:36.770 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: kSnappyCompression supported: 1 2024-09-12T23:25:36.770 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-09-12T23:25:36.770 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 2024-09-12T23:25:36.770 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-09-12T23:25:36.770 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-12T23:25:36.771 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.merge_operator: 2024-09-12T23:25:36.772 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.compaction_filter: None 2024-09-12T23:25:36.772 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.compaction_filter_factory: None 2024-09-12T23:25:36.773 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.sst_partitioner_factory: None 2024-09-12T23:25:36.773 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-09-12T23:25:36.773 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.table_factory: BlockBasedTable 2024-09-12T23:25:36.773 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55823968ba38) 2024-09-12T23:25:36.773 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: cache_index_and_filter_blocks: 1 2024-09-12T23:25:36.773 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-12T23:25:36.773 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-12T23:25:36.773 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: pin_top_level_index_and_filter: 1 2024-09-12T23:25:36.773 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: index_type: 0 2024-09-12T23:25:36.773 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: data_block_index_type: 0 2024-09-12T23:25:36.774 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: index_shortening: 1 2024-09-12T23:25:36.774 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: data_block_hash_table_util_ratio: 0.750000 2024-09-12T23:25:36.774 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: hash_index_allow_collision: 1 2024-09-12T23:25:36.774 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: checksum: 1 2024-09-12T23:25:36.774 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: no_block_cache: 0 2024-09-12T23:25:36.774 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: block_cache: 0x5582396bf350 2024-09-12T23:25:36.774 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: block_cache_name: BinnedLRUCache 2024-09-12T23:25:36.774 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: block_cache_options: 2024-09-12T23:25:36.774 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: capacity : 536870912 2024-09-12T23:25:36.774 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: num_shard_bits : 4 2024-09-12T23:25:36.774 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: strict_capacity_limit : 0 2024-09-12T23:25:36.774 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: high_pri_pool_ratio: 0.000 2024-09-12T23:25:36.775 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: block_cache_compressed: (nil) 2024-09-12T23:25:36.775 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: persistent_cache: (nil) 2024-09-12T23:25:36.775 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: block_size: 4096 2024-09-12T23:25:36.775 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: block_size_deviation: 10 2024-09-12T23:25:36.775 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: block_restart_interval: 16 2024-09-12T23:25:36.775 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: index_block_restart_interval: 1 2024-09-12T23:25:36.775 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: metadata_block_size: 4096 2024-09-12T23:25:36.775 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: partition_filters: 0 2024-09-12T23:25:36.775 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: use_delta_encoding: 1 2024-09-12T23:25:36.775 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-12T23:25:36.775 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: whole_key_filtering: 1 2024-09-12T23:25:36.775 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: verify_compression: 0 2024-09-12T23:25:36.776 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: read_amp_bytes_per_bit: 0 2024-09-12T23:25:36.776 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: format_version: 4 2024-09-12T23:25:36.776 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: enable_index_compression: 1 2024-09-12T23:25:36.776 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: block_align: 0 2024-09-12T23:25:36.776 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.write_buffer_size: 33554432 2024-09-12T23:25:36.776 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.max_write_buffer_number: 2 2024-09-12T23:25:36.776 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.compression: NoCompression 2024-09-12T23:25:36.776 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.bottommost_compression: Disabled 2024-09-12T23:25:36.776 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.prefix_extractor: nullptr 2024-09-12T23:25:36.776 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-12T23:25:36.776 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.num_levels: 7 2024-09-12T23:25:36.777 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-12T23:25:36.777 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-12T23:25:36.777 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-12T23:25:36.777 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-12T23:25:36.777 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-12T23:25:36.777 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-12T23:25:36.777 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-12T23:25:36.777 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-12T23:25:36.777 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-12T23:25:36.777 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-12T23:25:36.778 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.compression_opts.window_bits: -14 2024-09-12T23:25:36.778 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.compression_opts.level: 32767 2024-09-12T23:25:36.778 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.compression_opts.strategy: 0 2024-09-12T23:25:36.778 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-12T23:25:36.778 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-12T23:25:36.778 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-12T23:25:36.778 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.compression_opts.enabled: false 2024-09-12T23:25:36.778 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-12T23:25:36.778 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-12T23:25:36.779 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-12T23:25:36.779 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.target_file_size_base: 67108864 2024-09-12T23:25:36.779 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.target_file_size_multiplier: 1 2024-09-12T23:25:36.779 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-12T23:25:36.779 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-12T23:25:36.779 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-12T23:25:36.779 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-12T23:25:36.779 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-12T23:25:36.780 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-12T23:25:36.780 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-12T23:25:36.780 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-12T23:25:36.780 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-12T23:25:36.780 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-12T23:25:36.780 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-12T23:25:36.780 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-12T23:25:36.780 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.arena_block_size: 4194304 2024-09-12T23:25:36.780 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-12T23:25:36.780 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-12T23:25:36.780 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-12T23:25:36.780 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.disable_auto_compactions: 0 2024-09-12T23:25:36.781 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-12T23:25:36.781 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-12T23:25:36.781 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-12T23:25:36.781 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-12T23:25:36.781 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-12T23:25:36.781 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-12T23:25:36.781 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-12T23:25:36.781 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-12T23:25:36.781 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-12T23:25:36.781 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-12T23:25:36.781 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-12T23:25:36.781 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.inplace_update_support: 0 2024-09-12T23:25:36.782 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-09-12T23:25:36.782 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-12T23:25:36.782 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-12T23:25:36.782 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.memtable_huge_page_size: 0 2024-09-12T23:25:36.782 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.bloom_locality: 0 2024-09-12T23:25:36.782 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.max_successive_merges: 0 2024-09-12T23:25:36.782 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-09-12T23:25:36.782 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.paranoid_file_checks: 0 2024-09-12T23:25:36.782 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.force_consistency_checks: 1 2024-09-12T23:25:36.782 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.report_bg_io_stats: 0 2024-09-12T23:25:36.782 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.ttl: 2592000 2024-09-12T23:25:36.782 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-09-12T23:25:36.783 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.enable_blob_files: false 2024-09-12T23:25:36.783 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.min_blob_size: 0 2024-09-12T23:25:36.783 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.blob_file_size: 268435456 2024-09-12T23:25:36.783 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.blob_compression_type: NoCompression 2024-09-12T23:25:36.783 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.enable_blob_garbage_collection: false 2024-09-12T23:25:36.783 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-12T23:25:36.783 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-12T23:25:36.783 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.528+0000 7f9c38a7a900 4 rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2024-09-12T23:25:36.783 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.532+0000 7f9c38a7a900 4 rocksdb: [db/version_set.cc:4082] Creating manifest 7 2024-09-12T23:25:36.783 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.532+0000 7f9c38a7a900 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726183536537194, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-09-12T23:25:36.783 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.532+0000 7f9c38a7a900 4 rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2024-09-12T23:25:36.783 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.532+0000 7f9c38a7a900 3 rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-12T23:25:36.784 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.532+0000 7f9c38a7a900 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726183536537590, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1571, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726183536, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "eaf9438c-dfa2-4677-b545-e6b896130802", "db_session_id": "B6609I289JC2YHJGV7M2"}} 2024-09-12T23:25:36.784 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.532+0000 7f9c38a7a900 4 rocksdb: [db/version_set.cc:4082] Creating manifest 9 2024-09-12T23:25:36.784 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.532+0000 7f9c38a7a900 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726183536538022, "job": 1, "event": "recovery_finished"} 2024-09-12T23:25:36.784 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.532+0000 7f9c38a7a900 4 rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-12T23:25:36.784 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.532+0000 7f9c38a7a900 4 rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x5582396f4700 2024-09-12T23:25:36.784 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.532+0000 7f9c38a7a900 4 rocksdb: DB pointer 0x5582396c2000 2024-09-12T23:25:36.784 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.532+0000 7f9c2e846640 4 rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-09-12T23:25:36.784 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.532+0000 7f9c2e846640 4 rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-12T23:25:36.784 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: ** DB Stats ** 2024-09-12T23:25:36.784 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: Uptime(secs): 0.0 total, 0.0 interval 2024-09-12T23:25:36.784 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-12T23:25:36.784 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-12T23:25:36.785 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-12T23:25:36.785 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-12T23:25:36.785 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-12T23:25:36.785 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-12T23:25:36.785 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: ** Compaction Stats [default] ** 2024-09-12T23:25:36.785 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-12T23:25:36.785 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-12T23:25:36.785 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: L0 1/0 1.53 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.0 0.00 0.00 1 0.000 0 0 2024-09-12T23:25:36.785 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: Sum 1/0 1.53 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.0 0.00 0.00 1 0.000 0 0 2024-09-12T23:25:36.785 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.0 0.00 0.00 1 0.000 0 0 2024-09-12T23:25:36.785 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: ** Compaction Stats [default] ** 2024-09-12T23:25:36.786 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-12T23:25:36.786 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-12T23:25:36.786 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.0 0.00 0.00 1 0.000 0 0 2024-09-12T23:25:36.786 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: Uptime(secs): 0.0 total, 0.0 interval 2024-09-12T23:25:36.786 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: Flush(GB): cumulative 0.000, interval 0.000 2024-09-12T23:25:36.786 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-12T23:25:36.786 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: AddFile(Total Files): cumulative 0, interval 0 2024-09-12T23:25:36.786 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: AddFile(L0 Files): cumulative 0, interval 0 2024-09-12T23:25:36.786 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: AddFile(Keys): cumulative 0, interval 0 2024-09-12T23:25:36.786 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: Cumulative compaction: 0.00 GB write, 0.43 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-12T23:25:36.786 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: Interval compaction: 0.00 GB write, 0.43 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-12T23:25:36.786 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-12T23:25:36.786 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: ** File Read Latency Histogram By Level [default] ** 2024-09-12T23:25:36.787 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: ** Compaction Stats [default] ** 2024-09-12T23:25:36.787 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-12T23:25:36.787 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-12T23:25:36.787 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: L0 1/0 1.53 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.0 0.00 0.00 1 0.000 0 0 2024-09-12T23:25:36.787 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: Sum 1/0 1.53 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.0 0.00 0.00 1 0.000 0 0 2024-09-12T23:25:36.787 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-12T23:25:36.787 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: ** Compaction Stats [default] ** 2024-09-12T23:25:36.787 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-12T23:25:36.787 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-12T23:25:36.787 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.0 0.00 0.00 1 0.000 0 0 2024-09-12T23:25:36.787 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: Uptime(secs): 0.0 total, 0.0 interval 2024-09-12T23:25:36.788 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: Flush(GB): cumulative 0.000, interval 0.000 2024-09-12T23:25:36.788 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-12T23:25:36.788 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: AddFile(Total Files): cumulative 0, interval 0 2024-09-12T23:25:36.788 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: AddFile(L0 Files): cumulative 0, interval 0 2024-09-12T23:25:36.788 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: AddFile(Keys): cumulative 0, interval 0 2024-09-12T23:25:36.788 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: Cumulative compaction: 0.00 GB write, 0.43 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-12T23:25:36.788 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-12T23:25:36.788 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-12T23:25:36.788 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: ** File Read Latency Histogram By Level [default] ** 2024-09-12T23:25:36.789 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.532+0000 7f9c38a7a900 0 mon.c does not exist in monmap, will attempt to join an existing cluster 2024-09-12T23:25:36.789 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.532+0000 7f9c38a7a900 0 using public_addrv [v2:172.21.15.136:3301/0,v1:172.21.15.136:6790/0] 2024-09-12T23:25:36.789 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.532+0000 7f9c38a7a900 0 starting mon.c rank -1 at public addrs [v2:172.21.15.136:3301/0,v1:172.21.15.136:6790/0] at bind addrs [v2:172.21.15.136:3301/0,v1:172.21.15.136:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid da58a47c-715d-11ef-bceb-c7b262605968 2024-09-12T23:25:36.789 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.536+0000 7f9c38a7a900 1 mon.c@-1(???) e0 preinit fsid da58a47c-715d-11ef-bceb-c7b262605968 2024-09-12T23:25:36.789 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.552+0000 7f9c3184c640 0 mon.c@-1(synchronizing).mds e1 new map 2024-09-12T23:25:36.789 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.552+0000 7f9c3184c640 0 mon.c@-1(synchronizing).mds e1 print_map 2024-09-12T23:25:36.789 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: e1 2024-09-12T23:25:36.789 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: enable_multiple, ever_enabled_multiple: 1,1 2024-09-12T23:25:36.789 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-12T23:25:36.790 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: legacy client fscid: -1 2024-09-12T23:25:36.790 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: 2024-09-12T23:25:36.790 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: No filesystems configured 2024-09-12T23:25:36.790 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.552+0000 7f9c3184c640 1 mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-09-12T23:25:36.790 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.552+0000 7f9c3184c640 1 mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-09-12T23:25:36.790 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.552+0000 7f9c3184c640 1 mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-09-12T23:25:36.790 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.552+0000 7f9c3184c640 1 mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-09-12T23:25:36.790 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.552+0000 7f9c3184c640 1 mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-09-12T23:25:36.790 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.552+0000 7f9c3184c640 1 mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-09-12T23:25:36.790 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.552+0000 7f9c3184c640 0 mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-12T23:25:36.790 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.552+0000 7f9c3184c640 0 mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-12T23:25:36.791 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.552+0000 7f9c3184c640 0 mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-12T23:25:36.791 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.552+0000 7f9c3184c640 0 mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-12T23:25:36.791 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: cluster 2024-09-12T23:24:57.364413+0000 mgr.y (mgr.14150) 20 : cluster [DBG] pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:36.791 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: cluster 2024-09-12T23:24:59.364876+0000 mgr.y (mgr.14150) 21 : cluster [DBG] pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:36.791 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: cluster 2024-09-12T23:25:01.365296+0000 mgr.y (mgr.14150) 22 : cluster [DBG] pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:36.791 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: audit 2024-09-12T23:25:02.093282+0000 mgr.y (mgr.14150) 23 : audit [DBG] from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi181", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:25:36.791 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: cephadm 2024-09-12T23:25:03.121919+0000 mgr.y (mgr.14150) 24 : cephadm [INF] Deploying cephadm binary to smithi181 2024-09-12T23:25:36.791 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: cluster 2024-09-12T23:25:03.365676+0000 mgr.y (mgr.14150) 25 : cluster [DBG] pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:36.791 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: audit 2024-09-12T23:25:05.294523+0000 mon.a (mon.0) 111 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:36.791 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: cephadm 2024-09-12T23:25:05.295372+0000 mgr.y (mgr.14150) 26 : cephadm [INF] Added host smithi181 2024-09-12T23:25:36.791 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: audit 2024-09-12T23:25:05.296159+0000 mon.a (mon.0) 112 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:25:36.791 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: cluster 2024-09-12T23:25:05.366062+0000 mgr.y (mgr.14150) 27 : cluster [DBG] pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:36.792 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: audit 2024-09-12T23:25:05.556552+0000 mon.a (mon.0) 113 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:36.792 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: audit 2024-09-12T23:25:06.835138+0000 mon.a (mon.0) 114 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:36.792 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: audit 2024-09-12T23:25:07.507732+0000 mon.a (mon.0) 115 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:36.792 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: cluster 2024-09-12T23:25:07.366453+0000 mgr.y (mgr.14150) 28 : cluster [DBG] pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:36.792 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: cluster 2024-09-12T23:25:09.366892+0000 mgr.y (mgr.14150) 29 : cluster [DBG] pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:36.792 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: cluster 2024-09-12T23:25:11.367292+0000 mgr.y (mgr.14150) 30 : cluster [DBG] pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:36.792 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: audit 2024-09-12T23:25:12.635824+0000 mon.a (mon.0) 116 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:36.792 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: audit 2024-09-12T23:25:12.641571+0000 mon.a (mon.0) 117 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:36.792 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: audit 2024-09-12T23:25:12.648312+0000 mon.a (mon.0) 118 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:36.792 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: audit 2024-09-12T23:25:12.654059+0000 mon.a (mon.0) 119 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:36.792 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: audit 2024-09-12T23:25:12.655624+0000 mon.a (mon.0) 120 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi181", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:25:36.792 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: audit 2024-09-12T23:25:12.658035+0000 mon.a (mon.0) 121 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:25:36.793 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: audit 2024-09-12T23:25:12.659761+0000 mon.a (mon.0) 122 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:25:36.793 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: cephadm 2024-09-12T23:25:12.662155+0000 mgr.y (mgr.14150) 31 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.conf 2024-09-12T23:25:36.793 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: cephadm 2024-09-12T23:25:12.836796+0000 mgr.y (mgr.14150) 32 : cephadm [INF] Updating smithi181:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-12T23:25:36.793 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: audit 2024-09-12T23:25:13.249492+0000 mon.a (mon.0) 123 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:36.793 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: audit 2024-09-12T23:25:13.254461+0000 mon.a (mon.0) 124 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:36.793 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: audit 2024-09-12T23:25:13.259954+0000 mon.a (mon.0) 125 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:36.793 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: cephadm 2024-09-12T23:25:12.983102+0000 mgr.y (mgr.14150) 33 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2024-09-12T23:25:36.793 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: cephadm 2024-09-12T23:25:13.119506+0000 mgr.y (mgr.14150) 34 : cephadm [INF] Updating smithi181:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-12T23:25:36.793 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: cluster 2024-09-12T23:25:13.367657+0000 mgr.y (mgr.14150) 35 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:36.793 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: audit 2024-09-12T23:25:13.513168+0000 mgr.y (mgr.14150) 36 : audit [DBG] from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-12T23:25:36.793 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: cluster 2024-09-12T23:25:15.368047+0000 mgr.y (mgr.14150) 37 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:36.794 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: cluster 2024-09-12T23:25:17.368445+0000 mgr.y (mgr.14150) 38 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:36.794 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: cluster 2024-09-12T23:25:19.368877+0000 mgr.y (mgr.14150) 39 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:36.794 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: audit 2024-09-12T23:25:21.528875+0000 mon.a (mon.0) 126 : audit [INF] from='client.? 172.21.15.136:0/4065426733' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-12T23:25:36.794 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: cluster 2024-09-12T23:25:21.369150+0000 mgr.y (mgr.14150) 40 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:36.794 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: audit 2024-09-12T23:25:21.864762+0000 mon.a (mon.0) 127 : audit [INF] from='client.? 172.21.15.136:0/4065426733' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-12T23:25:36.794 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: cluster 2024-09-12T23:25:21.864855+0000 mon.a (mon.0) 128 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-09-12T23:25:36.794 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: cluster 2024-09-12T23:25:23.369498+0000 mgr.y (mgr.14150) 41 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:36.794 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: cluster 2024-09-12T23:25:25.369873+0000 mgr.y (mgr.14150) 42 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:36.794 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: audit 2024-09-12T23:25:26.314317+0000 mon.a (mon.0) 129 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:36.794 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: audit 2024-09-12T23:25:26.315654+0000 mon.a (mon.0) 130 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:25:36.794 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: audit 2024-09-12T23:25:26.319375+0000 mon.a (mon.0) 131 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:25:36.794 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: audit 2024-09-12T23:25:26.321301+0000 mon.a (mon.0) 132 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:25:36.795 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: audit 2024-09-12T23:25:26.331025+0000 mon.a (mon.0) 133 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:36.795 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: audit 2024-09-12T23:25:26.334698+0000 mon.a (mon.0) 134 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-12T23:25:36.795 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: audit 2024-09-12T23:25:26.336497+0000 mon.a (mon.0) 135 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:25:36.795 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: audit 2024-09-12T23:25:26.301366+0000 mgr.y (mgr.14150) 43 : audit [DBG] from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi136:172.21.15.136=a;smithi136:[v2:172.21.15.136:3301,v1:172.21.15.136:6790]=c;smithi181:172.21.15.181=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:25:36.795 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: cephadm 2024-09-12T23:25:26.306863+0000 mgr.y (mgr.14150) 44 : cephadm [INF] Saving service mon spec with placement smithi136:172.21.15.136=a;smithi136:[v2:172.21.15.136:3301,v1:172.21.15.136:6790]=c;smithi181:172.21.15.181=b;count:3 2024-09-12T23:25:36.795 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: cephadm 2024-09-12T23:25:26.338317+0000 mgr.y (mgr.14150) 45 : cephadm [INF] Deploying daemon mon.b on smithi181 2024-09-12T23:25:36.795 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: cluster 2024-09-12T23:25:27.370354+0000 mgr.y (mgr.14150) 46 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:36.795 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: cluster 2024-09-12T23:25:29.370749+0000 mgr.y (mgr.14150) 47 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:36.795 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: cluster 2024-09-12T23:25:31.371134+0000 mgr.y (mgr.14150) 48 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:36.795 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: audit 2024-09-12T23:25:31.707954+0000 mon.a (mon.0) 136 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:36.795 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: audit 2024-09-12T23:25:31.714265+0000 mon.a (mon.0) 137 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:36.795 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: audit 2024-09-12T23:25:31.720924+0000 mon.a (mon.0) 138 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:36.796 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: audit 2024-09-12T23:25:31.722081+0000 mon.a (mon.0) 139 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-12T23:25:36.796 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: audit 2024-09-12T23:25:31.723749+0000 mon.a (mon.0) 140 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:25:36.796 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: cephadm 2024-09-12T23:25:31.725609+0000 mgr.y (mgr.14150) 49 : cephadm [INF] Deploying daemon mon.c on smithi136 2024-09-12T23:25:36.796 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: audit 2024-09-12T23:25:32.012208+0000 mon.a (mon.0) 141 : audit [DBG] from='client.? 172.21.15.181:0/3941909899' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-12T23:25:36.796 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:36 smithi136 bash[32189]: debug 2024-09-12T23:25:36.556+0000 7f9c3184c640 1 mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-09-12T23:25:37.609 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.b/config 2024-09-12T23:25:38.600 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:38 smithi181 bash[24776]: audit 2024-09-12T23:25:33.160793+0000 mon.a (mon.0) 143 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:25:38.600 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:38 smithi181 bash[24776]: cluster 2024-09-12T23:25:33.161152+0000 mon.a (mon.0) 144 : cluster [INF] mon.a calling monitor election 2024-09-12T23:25:38.600 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:38 smithi181 bash[24776]: audit 2024-09-12T23:25:33.181856+0000 mon.a (mon.0) 145 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:25:38.600 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:38 smithi181 bash[24776]: cluster 2024-09-12T23:25:33.371583+0000 mgr.y (mgr.14150) 50 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:38.600 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:38 smithi181 bash[24776]: audit 2024-09-12T23:25:34.155027+0000 mon.a (mon.0) 146 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:25:38.600 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:38 smithi181 bash[24776]: audit 2024-09-12T23:25:35.154441+0000 mon.a (mon.0) 147 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:25:38.600 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:38 smithi181 bash[24776]: cluster 2024-09-12T23:25:35.156422+0000 mon.b (mon.1) 1 : cluster [INF] mon.b calling monitor election 2024-09-12T23:25:38.600 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:38 smithi181 bash[24776]: cluster 2024-09-12T23:25:35.371918+0000 mgr.y (mgr.14150) 51 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:38.600 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:38 smithi181 bash[24776]: audit 2024-09-12T23:25:36.155225+0000 mon.a (mon.0) 148 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:25:38.600 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:38 smithi181 bash[24776]: audit 2024-09-12T23:25:36.564845+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:25:38.600 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:38 smithi181 bash[24776]: audit 2024-09-12T23:25:37.155433+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:25:38.601 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:38 smithi181 bash[24776]: cluster 2024-09-12T23:25:37.372284+0000 mgr.y (mgr.14150) 52 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:38.601 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:38 smithi181 bash[24776]: audit 2024-09-12T23:25:37.565668+0000 mon.a (mon.0) 151 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:25:38.601 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:38 smithi181 bash[24776]: audit 2024-09-12T23:25:38.155572+0000 mon.a (mon.0) 152 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:25:38.601 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:38 smithi181 bash[24776]: cluster 2024-09-12T23:25:38.202244+0000 mon.a (mon.0) 153 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-12T23:25:38.601 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:38 smithi181 bash[24776]: cluster 2024-09-12T23:25:38.212796+0000 mon.a (mon.0) 154 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.136:3300/0,v1:172.21.15.136:6789/0],b=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-12T23:25:38.601 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:38 smithi181 bash[24776]: cluster 2024-09-12T23:25:38.233550+0000 mon.a (mon.0) 155 : cluster [DBG] fsmap 2024-09-12T23:25:38.601 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:38 smithi181 bash[24776]: cluster 2024-09-12T23:25:38.233603+0000 mon.a (mon.0) 156 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-09-12T23:25:38.601 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:38 smithi181 bash[24776]: cluster 2024-09-12T23:25:38.233864+0000 mon.a (mon.0) 157 : cluster [DBG] mgrmap e13: y(active, since 72s) 2024-09-12T23:25:38.601 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:38 smithi181 bash[24776]: cluster 2024-09-12T23:25:38.234103+0000 mon.a (mon.0) 158 : cluster [INF] overall HEALTH_OK 2024-09-12T23:25:38.601 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:38 smithi181 bash[24776]: audit 2024-09-12T23:25:38.240457+0000 mon.a (mon.0) 159 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:38.601 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:38 smithi181 bash[24776]: audit 2024-09-12T23:25:38.247325+0000 mon.a (mon.0) 160 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:38.601 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:38 smithi181 bash[24776]: audit 2024-09-12T23:25:38.255217+0000 mon.a (mon.0) 161 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:38.601 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:38 smithi181 bash[24776]: audit 2024-09-12T23:25:38.262092+0000 mon.a (mon.0) 162 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:38.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:38 smithi136 bash[22601]: audit 2024-09-12T23:25:33.160793+0000 mon.a (mon.0) 143 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:25:38.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:38 smithi136 bash[22601]: cluster 2024-09-12T23:25:33.161152+0000 mon.a (mon.0) 144 : cluster [INF] mon.a calling monitor election 2024-09-12T23:25:38.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:38 smithi136 bash[22601]: audit 2024-09-12T23:25:33.181856+0000 mon.a (mon.0) 145 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:25:38.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:38 smithi136 bash[22601]: cluster 2024-09-12T23:25:33.371583+0000 mgr.y (mgr.14150) 50 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:38.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:38 smithi136 bash[22601]: audit 2024-09-12T23:25:34.155027+0000 mon.a (mon.0) 146 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:25:38.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:38 smithi136 bash[22601]: audit 2024-09-12T23:25:35.154441+0000 mon.a (mon.0) 147 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:25:38.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:38 smithi136 bash[22601]: cluster 2024-09-12T23:25:35.156422+0000 mon.b (mon.1) 1 : cluster [INF] mon.b calling monitor election 2024-09-12T23:25:38.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:38 smithi136 bash[22601]: cluster 2024-09-12T23:25:35.371918+0000 mgr.y (mgr.14150) 51 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:38.762 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:38 smithi136 bash[22601]: audit 2024-09-12T23:25:36.155225+0000 mon.a (mon.0) 148 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:25:38.762 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:38 smithi136 bash[22601]: audit 2024-09-12T23:25:36.564845+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:25:38.762 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:38 smithi136 bash[22601]: audit 2024-09-12T23:25:37.155433+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:25:38.762 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:38 smithi136 bash[22601]: cluster 2024-09-12T23:25:37.372284+0000 mgr.y (mgr.14150) 52 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:38.762 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:38 smithi136 bash[22601]: audit 2024-09-12T23:25:37.565668+0000 mon.a (mon.0) 151 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:25:38.762 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:38 smithi136 bash[22601]: audit 2024-09-12T23:25:38.155572+0000 mon.a (mon.0) 152 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:25:38.762 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:38 smithi136 bash[22601]: cluster 2024-09-12T23:25:38.202244+0000 mon.a (mon.0) 153 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-12T23:25:38.762 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:38 smithi136 bash[22601]: cluster 2024-09-12T23:25:38.212796+0000 mon.a (mon.0) 154 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.136:3300/0,v1:172.21.15.136:6789/0],b=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-12T23:25:38.762 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:38 smithi136 bash[22601]: cluster 2024-09-12T23:25:38.233550+0000 mon.a (mon.0) 155 : cluster [DBG] fsmap 2024-09-12T23:25:38.762 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:38 smithi136 bash[22601]: cluster 2024-09-12T23:25:38.233603+0000 mon.a (mon.0) 156 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-09-12T23:25:38.762 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:38 smithi136 bash[22601]: cluster 2024-09-12T23:25:38.233864+0000 mon.a (mon.0) 157 : cluster [DBG] mgrmap e13: y(active, since 72s) 2024-09-12T23:25:38.762 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:38 smithi136 bash[22601]: cluster 2024-09-12T23:25:38.234103+0000 mon.a (mon.0) 158 : cluster [INF] overall HEALTH_OK 2024-09-12T23:25:38.763 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:38 smithi136 bash[22601]: audit 2024-09-12T23:25:38.240457+0000 mon.a (mon.0) 159 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:38.763 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:38 smithi136 bash[22601]: audit 2024-09-12T23:25:38.247325+0000 mon.a (mon.0) 160 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:38.763 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:38 smithi136 bash[22601]: audit 2024-09-12T23:25:38.255217+0000 mon.a (mon.0) 161 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:38.763 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:38 smithi136 bash[22601]: audit 2024-09-12T23:25:38.262092+0000 mon.a (mon.0) 162 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:39.512 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:25:39 smithi136 bash[22852]: debug 2024-09-12T23:25:39.156+0000 7fdea2b8b640 -1 mgr.server handle_report got status from non-daemon mon.b 2024-09-12T23:25:44.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[22601]: audit 2024-09-12T23:25:38.570869+0000 mon.a (mon.0) 165 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:25:44.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[22601]: cluster 2024-09-12T23:25:38.571030+0000 mon.a (mon.0) 166 : cluster [INF] mon.a calling monitor election 2024-09-12T23:25:44.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[22601]: cluster 2024-09-12T23:25:38.571317+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2024-09-12T23:25:44.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[22601]: audit 2024-09-12T23:25:38.589385+0000 mon.a (mon.0) 167 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:25:44.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[22601]: audit 2024-09-12T23:25:38.589500+0000 mon.a (mon.0) 168 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:25:44.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[22601]: cluster 2024-09-12T23:25:39.372717+0000 mgr.y (mgr.14150) 53 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:44.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[22601]: audit 2024-09-12T23:25:39.565712+0000 mon.a (mon.0) 169 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:25:44.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[22601]: audit 2024-09-12T23:25:40.566398+0000 mon.a (mon.0) 170 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:25:44.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[22601]: cluster 2024-09-12T23:25:40.568724+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2024-09-12T23:25:44.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[22601]: cluster 2024-09-12T23:25:41.373119+0000 mgr.y (mgr.14150) 54 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:44.012 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[22601]: audit 2024-09-12T23:25:41.566269+0000 mon.a (mon.0) 171 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:25:44.012 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[22601]: audit 2024-09-12T23:25:42.566343+0000 mon.a (mon.0) 172 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:25:44.012 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[22601]: audit 2024-09-12T23:25:43.566512+0000 mon.a (mon.0) 173 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:25:44.012 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[22601]: cluster 2024-09-12T23:25:43.573700+0000 mon.a (mon.0) 174 : cluster [INF] mon.a calling monitor election 2024-09-12T23:25:44.012 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[22601]: cluster 2024-09-12T23:25:43.613967+0000 mon.a (mon.0) 175 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-12T23:25:44.012 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[22601]: cluster 2024-09-12T23:25:43.623768+0000 mon.a (mon.0) 176 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.136:3300/0,v1:172.21.15.136:6789/0],b=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0],c=[v2:172.21.15.136:3301/0,v1:172.21.15.136:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-12T23:25:44.012 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[22601]: cluster 2024-09-12T23:25:43.625366+0000 mon.a (mon.0) 177 : cluster [DBG] fsmap 2024-09-12T23:25:44.012 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[22601]: cluster 2024-09-12T23:25:43.625449+0000 mon.a (mon.0) 178 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-09-12T23:25:44.012 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[22601]: cluster 2024-09-12T23:25:43.625853+0000 mon.a (mon.0) 179 : cluster [DBG] mgrmap e13: y(active, since 78s) 2024-09-12T23:25:44.012 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[22601]: cluster 2024-09-12T23:25:43.626203+0000 mon.a (mon.0) 180 : cluster [INF] overall HEALTH_OK 2024-09-12T23:25:44.012 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[22601]: audit 2024-09-12T23:25:43.635099+0000 mon.a (mon.0) 181 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:44.012 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[22601]: audit 2024-09-12T23:25:43.642580+0000 mon.a (mon.0) 182 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:44.012 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[22601]: audit 2024-09-12T23:25:43.649758+0000 mon.a (mon.0) 183 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:44.013 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[22601]: audit 2024-09-12T23:25:43.659009+0000 mon.a (mon.0) 184 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:44.013 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[22601]: audit 2024-09-12T23:25:43.666846+0000 mon.a (mon.0) 185 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:44.013 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[32189]: audit 2024-09-12T23:25:33.160793+0000 mon.a (mon.0) 143 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:25:44.013 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[32189]: cluster 2024-09-12T23:25:33.161152+0000 mon.a (mon.0) 144 : cluster [INF] mon.a calling monitor election 2024-09-12T23:25:44.013 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[32189]: audit 2024-09-12T23:25:33.181856+0000 mon.a (mon.0) 145 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:25:44.013 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[32189]: cluster 2024-09-12T23:25:33.371583+0000 mgr.y (mgr.14150) 50 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:44.013 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[32189]: audit 2024-09-12T23:25:34.155027+0000 mon.a (mon.0) 146 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:25:44.014 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[32189]: audit 2024-09-12T23:25:35.154441+0000 mon.a (mon.0) 147 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:25:44.014 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[32189]: cluster 2024-09-12T23:25:35.156422+0000 mon.b (mon.1) 1 : cluster [INF] mon.b calling monitor election 2024-09-12T23:25:44.014 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[32189]: cluster 2024-09-12T23:25:35.371918+0000 mgr.y (mgr.14150) 51 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:44.014 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[32189]: audit 2024-09-12T23:25:36.155225+0000 mon.a (mon.0) 148 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:25:44.014 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[32189]: audit 2024-09-12T23:25:36.564845+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:25:44.014 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[32189]: audit 2024-09-12T23:25:37.155433+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:25:44.014 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[32189]: cluster 2024-09-12T23:25:37.372284+0000 mgr.y (mgr.14150) 52 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:44.014 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[32189]: audit 2024-09-12T23:25:37.565668+0000 mon.a (mon.0) 151 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:25:44.014 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[32189]: audit 2024-09-12T23:25:38.155572+0000 mon.a (mon.0) 152 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:25:44.014 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[32189]: cluster 2024-09-12T23:25:38.202244+0000 mon.a (mon.0) 153 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-12T23:25:44.014 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[32189]: cluster 2024-09-12T23:25:38.212796+0000 mon.a (mon.0) 154 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.136:3300/0,v1:172.21.15.136:6789/0],b=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-12T23:25:44.014 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[32189]: cluster 2024-09-12T23:25:38.233550+0000 mon.a (mon.0) 155 : cluster [DBG] fsmap 2024-09-12T23:25:44.014 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[32189]: cluster 2024-09-12T23:25:38.233603+0000 mon.a (mon.0) 156 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-09-12T23:25:44.015 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[32189]: cluster 2024-09-12T23:25:38.233864+0000 mon.a (mon.0) 157 : cluster [DBG] mgrmap e13: y(active, since 72s) 2024-09-12T23:25:44.015 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[32189]: cluster 2024-09-12T23:25:38.234103+0000 mon.a (mon.0) 158 : cluster [INF] overall HEALTH_OK 2024-09-12T23:25:44.015 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[32189]: audit 2024-09-12T23:25:38.240457+0000 mon.a (mon.0) 159 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:44.015 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[32189]: audit 2024-09-12T23:25:38.247325+0000 mon.a (mon.0) 160 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:44.015 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[32189]: audit 2024-09-12T23:25:38.255217+0000 mon.a (mon.0) 161 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:44.015 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[32189]: audit 2024-09-12T23:25:38.262092+0000 mon.a (mon.0) 162 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:44.015 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[32189]: audit 2024-09-12T23:25:38.570869+0000 mon.a (mon.0) 165 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:25:44.015 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[32189]: cluster 2024-09-12T23:25:38.571030+0000 mon.a (mon.0) 166 : cluster [INF] mon.a calling monitor election 2024-09-12T23:25:44.015 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[32189]: cluster 2024-09-12T23:25:38.571317+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2024-09-12T23:25:44.015 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[32189]: audit 2024-09-12T23:25:38.589385+0000 mon.a (mon.0) 167 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:25:44.015 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[32189]: audit 2024-09-12T23:25:38.589500+0000 mon.a (mon.0) 168 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:25:44.015 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[32189]: cluster 2024-09-12T23:25:39.372717+0000 mgr.y (mgr.14150) 53 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:44.015 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[32189]: audit 2024-09-12T23:25:39.565712+0000 mon.a (mon.0) 169 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:25:44.015 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[32189]: audit 2024-09-12T23:25:40.566398+0000 mon.a (mon.0) 170 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:25:44.016 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[32189]: cluster 2024-09-12T23:25:40.568724+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2024-09-12T23:25:44.016 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[32189]: cluster 2024-09-12T23:25:41.373119+0000 mgr.y (mgr.14150) 54 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:44.016 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[32189]: audit 2024-09-12T23:25:41.566269+0000 mon.a (mon.0) 171 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:25:44.016 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[32189]: audit 2024-09-12T23:25:42.566343+0000 mon.a (mon.0) 172 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:25:44.016 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[32189]: audit 2024-09-12T23:25:43.566512+0000 mon.a (mon.0) 173 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:25:44.016 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[32189]: cluster 2024-09-12T23:25:43.573700+0000 mon.a (mon.0) 174 : cluster [INF] mon.a calling monitor election 2024-09-12T23:25:44.016 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[32189]: cluster 2024-09-12T23:25:43.613967+0000 mon.a (mon.0) 175 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-12T23:25:44.016 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[32189]: cluster 2024-09-12T23:25:43.623768+0000 mon.a (mon.0) 176 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.136:3300/0,v1:172.21.15.136:6789/0],b=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0],c=[v2:172.21.15.136:3301/0,v1:172.21.15.136:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-12T23:25:44.016 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[32189]: cluster 2024-09-12T23:25:43.625366+0000 mon.a (mon.0) 177 : cluster [DBG] fsmap 2024-09-12T23:25:44.016 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[32189]: cluster 2024-09-12T23:25:43.625449+0000 mon.a (mon.0) 178 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-09-12T23:25:44.016 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[32189]: cluster 2024-09-12T23:25:43.625853+0000 mon.a (mon.0) 179 : cluster [DBG] mgrmap e13: y(active, since 78s) 2024-09-12T23:25:44.016 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[32189]: cluster 2024-09-12T23:25:43.626203+0000 mon.a (mon.0) 180 : cluster [INF] overall HEALTH_OK 2024-09-12T23:25:44.016 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[32189]: audit 2024-09-12T23:25:43.635099+0000 mon.a (mon.0) 181 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:44.017 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[32189]: audit 2024-09-12T23:25:43.642580+0000 mon.a (mon.0) 182 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:44.017 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[32189]: audit 2024-09-12T23:25:43.649758+0000 mon.a (mon.0) 183 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:44.017 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[32189]: audit 2024-09-12T23:25:43.659009+0000 mon.a (mon.0) 184 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:44.017 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:43 smithi136 bash[32189]: audit 2024-09-12T23:25:43.666846+0000 mon.a (mon.0) 185 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:44.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:43 smithi181 bash[24776]: audit 2024-09-12T23:25:38.570869+0000 mon.a (mon.0) 165 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:25:44.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:43 smithi181 bash[24776]: cluster 2024-09-12T23:25:38.571030+0000 mon.a (mon.0) 166 : cluster [INF] mon.a calling monitor election 2024-09-12T23:25:44.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:43 smithi181 bash[24776]: cluster 2024-09-12T23:25:38.571317+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2024-09-12T23:25:44.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:43 smithi181 bash[24776]: audit 2024-09-12T23:25:38.589385+0000 mon.a (mon.0) 167 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:25:44.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:43 smithi181 bash[24776]: audit 2024-09-12T23:25:38.589500+0000 mon.a (mon.0) 168 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:25:44.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:43 smithi181 bash[24776]: cluster 2024-09-12T23:25:39.372717+0000 mgr.y (mgr.14150) 53 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:44.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:43 smithi181 bash[24776]: audit 2024-09-12T23:25:39.565712+0000 mon.a (mon.0) 169 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:25:44.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:43 smithi181 bash[24776]: audit 2024-09-12T23:25:40.566398+0000 mon.a (mon.0) 170 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:25:44.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:43 smithi181 bash[24776]: cluster 2024-09-12T23:25:40.568724+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2024-09-12T23:25:44.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:43 smithi181 bash[24776]: cluster 2024-09-12T23:25:41.373119+0000 mgr.y (mgr.14150) 54 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:44.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:43 smithi181 bash[24776]: audit 2024-09-12T23:25:41.566269+0000 mon.a (mon.0) 171 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:25:44.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:43 smithi181 bash[24776]: audit 2024-09-12T23:25:42.566343+0000 mon.a (mon.0) 172 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:25:44.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:43 smithi181 bash[24776]: audit 2024-09-12T23:25:43.566512+0000 mon.a (mon.0) 173 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:25:44.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:43 smithi181 bash[24776]: cluster 2024-09-12T23:25:43.573700+0000 mon.a (mon.0) 174 : cluster [INF] mon.a calling monitor election 2024-09-12T23:25:44.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:43 smithi181 bash[24776]: cluster 2024-09-12T23:25:43.613967+0000 mon.a (mon.0) 175 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-12T23:25:44.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:43 smithi181 bash[24776]: cluster 2024-09-12T23:25:43.623768+0000 mon.a (mon.0) 176 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.136:3300/0,v1:172.21.15.136:6789/0],b=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0],c=[v2:172.21.15.136:3301/0,v1:172.21.15.136:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-12T23:25:44.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:43 smithi181 bash[24776]: cluster 2024-09-12T23:25:43.625366+0000 mon.a (mon.0) 177 : cluster [DBG] fsmap 2024-09-12T23:25:44.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:43 smithi181 bash[24776]: cluster 2024-09-12T23:25:43.625449+0000 mon.a (mon.0) 178 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-09-12T23:25:44.102 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:43 smithi181 bash[24776]: cluster 2024-09-12T23:25:43.625853+0000 mon.a (mon.0) 179 : cluster [DBG] mgrmap e13: y(active, since 78s) 2024-09-12T23:25:44.102 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:43 smithi181 bash[24776]: cluster 2024-09-12T23:25:43.626203+0000 mon.a (mon.0) 180 : cluster [INF] overall HEALTH_OK 2024-09-12T23:25:44.102 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:43 smithi181 bash[24776]: audit 2024-09-12T23:25:43.635099+0000 mon.a (mon.0) 181 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:44.102 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:43 smithi181 bash[24776]: audit 2024-09-12T23:25:43.642580+0000 mon.a (mon.0) 182 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:44.102 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:43 smithi181 bash[24776]: audit 2024-09-12T23:25:43.649758+0000 mon.a (mon.0) 183 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:44.102 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:43 smithi181 bash[24776]: audit 2024-09-12T23:25:43.659009+0000 mon.a (mon.0) 184 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:44.102 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:43 smithi181 bash[24776]: audit 2024-09-12T23:25:43.666846+0000 mon.a (mon.0) 185 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:44.233 INFO:teuthology.orchestra.run.smithi181.stderr:dumped monmap epoch 3 2024-09-12T23:25:44.234 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-09-12T23:25:44.234 INFO:teuthology.orchestra.run.smithi181.stdout:{"epoch":3,"fsid":"da58a47c-715d-11ef-bceb-c7b262605968","modified":"2024-09-12T23:25:38.565715Z","created":"2024-09-12T23:23:24.585690Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:3300","nonce":0},{"type":"v1","addr":"172.21.15.136:6789","nonce":0}]},"addr":"172.21.15.136:6789/0","public_addr":"172.21.15.136:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:3300","nonce":0},{"type":"v1","addr":"172.21.15.181:6789","nonce":0}]},"addr":"172.21.15.181:6789/0","public_addr":"172.21.15.181:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:3301","nonce":0},{"type":"v1","addr":"172.21.15.136:6790","nonce":0}]},"addr":"172.21.15.136:6790/0","public_addr":"172.21.15.136:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2024-09-12T23:25:44.854 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-09-12T23:25:44.854 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph config generate-minimal-conf 2024-09-12T23:25:45.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:44 smithi136 bash[22601]: cluster 2024-09-12T23:25:43.373451+0000 mgr.y (mgr.14150) 55 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:45.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:44 smithi136 bash[22601]: audit 2024-09-12T23:25:44.090807+0000 mon.a (mon.0) 186 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:25:45.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:44 smithi136 bash[22601]: audit 2024-09-12T23:25:44.092333+0000 mon.a (mon.0) 187 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:25:45.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:44 smithi136 bash[22601]: audit 2024-09-12T23:25:44.233672+0000 mon.a (mon.0) 188 : audit [DBG] from='client.? 172.21.15.181:0/2661915643' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-12T23:25:45.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:44 smithi136 bash[22601]: audit 2024-09-12T23:25:44.344898+0000 mon.a (mon.0) 189 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:45.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:44 smithi136 bash[22601]: audit 2024-09-12T23:25:44.351303+0000 mon.a (mon.0) 190 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:45.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:44 smithi136 bash[22601]: audit 2024-09-12T23:25:44.360468+0000 mon.a (mon.0) 191 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:45.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:44 smithi136 bash[22601]: audit 2024-09-12T23:25:44.366393+0000 mon.a (mon.0) 192 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:45.012 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:44 smithi136 bash[22601]: audit 2024-09-12T23:25:44.372391+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:45.012 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:44 smithi136 bash[22601]: audit 2024-09-12T23:25:44.403068+0000 mon.a (mon.0) 194 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-12T23:25:45.012 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:44 smithi136 bash[22601]: audit 2024-09-12T23:25:44.404304+0000 mon.a (mon.0) 195 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-12T23:25:45.012 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:44 smithi136 bash[22601]: audit 2024-09-12T23:25:44.405502+0000 mon.a (mon.0) 196 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:25:45.012 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:44 smithi136 bash[22601]: audit 2024-09-12T23:25:44.566486+0000 mon.a (mon.0) 197 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:25:45.012 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:44 smithi136 bash[32189]: cluster 2024-09-12T23:25:43.373451+0000 mgr.y (mgr.14150) 55 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:45.012 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:44 smithi136 bash[32189]: audit 2024-09-12T23:25:44.090807+0000 mon.a (mon.0) 186 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:25:45.012 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:44 smithi136 bash[32189]: audit 2024-09-12T23:25:44.092333+0000 mon.a (mon.0) 187 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:25:45.012 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:44 smithi136 bash[32189]: audit 2024-09-12T23:25:44.233672+0000 mon.a (mon.0) 188 : audit [DBG] from='client.? 172.21.15.181:0/2661915643' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-12T23:25:45.012 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:44 smithi136 bash[32189]: audit 2024-09-12T23:25:44.344898+0000 mon.a (mon.0) 189 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:45.012 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:44 smithi136 bash[32189]: audit 2024-09-12T23:25:44.351303+0000 mon.a (mon.0) 190 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:45.013 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:44 smithi136 bash[32189]: audit 2024-09-12T23:25:44.360468+0000 mon.a (mon.0) 191 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:45.013 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:44 smithi136 bash[32189]: audit 2024-09-12T23:25:44.366393+0000 mon.a (mon.0) 192 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:45.013 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:44 smithi136 bash[32189]: audit 2024-09-12T23:25:44.372391+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:45.013 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:44 smithi136 bash[32189]: audit 2024-09-12T23:25:44.403068+0000 mon.a (mon.0) 194 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-12T23:25:45.013 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:44 smithi136 bash[32189]: audit 2024-09-12T23:25:44.404304+0000 mon.a (mon.0) 195 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-12T23:25:45.013 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:44 smithi136 bash[32189]: audit 2024-09-12T23:25:44.405502+0000 mon.a (mon.0) 196 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:25:45.013 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:44 smithi136 bash[32189]: audit 2024-09-12T23:25:44.566486+0000 mon.a (mon.0) 197 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:25:45.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:44 smithi181 bash[24776]: cluster 2024-09-12T23:25:43.373451+0000 mgr.y (mgr.14150) 55 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:45.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:44 smithi181 bash[24776]: audit 2024-09-12T23:25:44.090807+0000 mon.a (mon.0) 186 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:25:45.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:44 smithi181 bash[24776]: audit 2024-09-12T23:25:44.092333+0000 mon.a (mon.0) 187 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:25:45.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:44 smithi181 bash[24776]: audit 2024-09-12T23:25:44.233672+0000 mon.a (mon.0) 188 : audit [DBG] from='client.? 172.21.15.181:0/2661915643' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-12T23:25:45.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:44 smithi181 bash[24776]: audit 2024-09-12T23:25:44.344898+0000 mon.a (mon.0) 189 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:45.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:44 smithi181 bash[24776]: audit 2024-09-12T23:25:44.351303+0000 mon.a (mon.0) 190 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:45.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:44 smithi181 bash[24776]: audit 2024-09-12T23:25:44.360468+0000 mon.a (mon.0) 191 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:45.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:44 smithi181 bash[24776]: audit 2024-09-12T23:25:44.366393+0000 mon.a (mon.0) 192 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:45.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:44 smithi181 bash[24776]: audit 2024-09-12T23:25:44.372391+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:45.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:44 smithi181 bash[24776]: audit 2024-09-12T23:25:44.403068+0000 mon.a (mon.0) 194 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-12T23:25:45.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:44 smithi181 bash[24776]: audit 2024-09-12T23:25:44.404304+0000 mon.a (mon.0) 195 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-12T23:25:45.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:44 smithi181 bash[24776]: audit 2024-09-12T23:25:44.405502+0000 mon.a (mon.0) 196 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:25:45.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:44 smithi181 bash[24776]: audit 2024-09-12T23:25:44.566486+0000 mon.a (mon.0) 197 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:25:45.925 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:45 smithi136 bash[22601]: cephadm 2024-09-12T23:25:44.094304+0000 mgr.y (mgr.14150) 56 : cephadm [INF] Updating smithi136:/etc/ceph/ceph.conf 2024-09-12T23:25:45.925 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:45 smithi136 bash[22601]: cephadm 2024-09-12T23:25:44.094695+0000 mgr.y (mgr.14150) 57 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.conf 2024-09-12T23:25:45.926 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:45 smithi136 bash[22601]: cephadm 2024-09-12T23:25:44.222434+0000 mgr.y (mgr.14150) 58 : cephadm [INF] Updating smithi181:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-12T23:25:45.926 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:45 smithi136 bash[22601]: cephadm 2024-09-12T23:25:44.226362+0000 mgr.y (mgr.14150) 59 : cephadm [INF] Updating smithi136:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-12T23:25:45.926 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:45 smithi136 bash[22601]: cephadm 2024-09-12T23:25:44.402666+0000 mgr.y (mgr.14150) 60 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2024-09-12T23:25:45.926 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:45 smithi136 bash[22601]: cephadm 2024-09-12T23:25:44.406619+0000 mgr.y (mgr.14150) 61 : cephadm [INF] Reconfiguring daemon mon.a on smithi136 2024-09-12T23:25:45.926 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:45 smithi136 bash[32189]: cephadm 2024-09-12T23:25:44.094304+0000 mgr.y (mgr.14150) 56 : cephadm [INF] Updating smithi136:/etc/ceph/ceph.conf 2024-09-12T23:25:45.926 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:45 smithi136 bash[32189]: cephadm 2024-09-12T23:25:44.094695+0000 mgr.y (mgr.14150) 57 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.conf 2024-09-12T23:25:45.926 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:45 smithi136 bash[32189]: cephadm 2024-09-12T23:25:44.222434+0000 mgr.y (mgr.14150) 58 : cephadm [INF] Updating smithi181:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-12T23:25:45.926 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:45 smithi136 bash[32189]: cephadm 2024-09-12T23:25:44.226362+0000 mgr.y (mgr.14150) 59 : cephadm [INF] Updating smithi136:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-12T23:25:45.926 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:45 smithi136 bash[32189]: cephadm 2024-09-12T23:25:44.402666+0000 mgr.y (mgr.14150) 60 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2024-09-12T23:25:45.927 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:45 smithi136 bash[32189]: cephadm 2024-09-12T23:25:44.406619+0000 mgr.y (mgr.14150) 61 : cephadm [INF] Reconfiguring daemon mon.a on smithi136 2024-09-12T23:25:46.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:45 smithi181 bash[24776]: cephadm 2024-09-12T23:25:44.094304+0000 mgr.y (mgr.14150) 56 : cephadm [INF] Updating smithi136:/etc/ceph/ceph.conf 2024-09-12T23:25:46.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:45 smithi181 bash[24776]: cephadm 2024-09-12T23:25:44.094695+0000 mgr.y (mgr.14150) 57 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.conf 2024-09-12T23:25:46.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:45 smithi181 bash[24776]: cephadm 2024-09-12T23:25:44.222434+0000 mgr.y (mgr.14150) 58 : cephadm [INF] Updating smithi181:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-12T23:25:46.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:45 smithi181 bash[24776]: cephadm 2024-09-12T23:25:44.226362+0000 mgr.y (mgr.14150) 59 : cephadm [INF] Updating smithi136:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-12T23:25:46.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:45 smithi181 bash[24776]: cephadm 2024-09-12T23:25:44.402666+0000 mgr.y (mgr.14150) 60 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2024-09-12T23:25:46.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:45 smithi181 bash[24776]: cephadm 2024-09-12T23:25:44.406619+0000 mgr.y (mgr.14150) 61 : cephadm [INF] Reconfiguring daemon mon.a on smithi136 2024-09-12T23:25:47.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:46 smithi136 bash[22601]: cluster 2024-09-12T23:25:45.373863+0000 mgr.y (mgr.14150) 62 : cluster [DBG] pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:47.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:46 smithi136 bash[22601]: audit 2024-09-12T23:25:46.348697+0000 mon.a (mon.0) 198 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:47.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:46 smithi136 bash[22601]: audit 2024-09-12T23:25:46.354007+0000 mon.a (mon.0) 199 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:47.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:46 smithi136 bash[22601]: audit 2024-09-12T23:25:46.355205+0000 mon.a (mon.0) 200 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-12T23:25:47.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:46 smithi136 bash[22601]: audit 2024-09-12T23:25:46.356056+0000 mon.a (mon.0) 201 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-12T23:25:47.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:46 smithi136 bash[22601]: audit 2024-09-12T23:25:46.356832+0000 mon.a (mon.0) 202 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:25:47.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:46 smithi136 bash[32189]: cluster 2024-09-12T23:25:45.373863+0000 mgr.y (mgr.14150) 62 : cluster [DBG] pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:47.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:46 smithi136 bash[32189]: audit 2024-09-12T23:25:46.348697+0000 mon.a (mon.0) 198 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:47.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:46 smithi136 bash[32189]: audit 2024-09-12T23:25:46.354007+0000 mon.a (mon.0) 199 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:47.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:46 smithi136 bash[32189]: audit 2024-09-12T23:25:46.355205+0000 mon.a (mon.0) 200 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-12T23:25:47.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:46 smithi136 bash[32189]: audit 2024-09-12T23:25:46.356056+0000 mon.a (mon.0) 201 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-12T23:25:47.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:46 smithi136 bash[32189]: audit 2024-09-12T23:25:46.356832+0000 mon.a (mon.0) 202 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:25:47.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:46 smithi181 bash[24776]: cluster 2024-09-12T23:25:45.373863+0000 mgr.y (mgr.14150) 62 : cluster [DBG] pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:47.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:46 smithi181 bash[24776]: audit 2024-09-12T23:25:46.348697+0000 mon.a (mon.0) 198 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:47.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:46 smithi181 bash[24776]: audit 2024-09-12T23:25:46.354007+0000 mon.a (mon.0) 199 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:47.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:46 smithi181 bash[24776]: audit 2024-09-12T23:25:46.355205+0000 mon.a (mon.0) 200 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-12T23:25:47.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:46 smithi181 bash[24776]: audit 2024-09-12T23:25:46.356056+0000 mon.a (mon.0) 201 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-12T23:25:47.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:46 smithi181 bash[24776]: audit 2024-09-12T23:25:46.356832+0000 mon.a (mon.0) 202 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:25:47.960 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:47 smithi136 bash[32189]: cephadm 2024-09-12T23:25:46.354909+0000 mgr.y (mgr.14150) 63 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2024-09-12T23:25:47.960 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:47 smithi136 bash[32189]: cephadm 2024-09-12T23:25:46.357778+0000 mgr.y (mgr.14150) 64 : cephadm [INF] Reconfiguring daemon mon.c on smithi136 2024-09-12T23:25:47.960 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:47 smithi136 bash[22601]: cephadm 2024-09-12T23:25:46.354909+0000 mgr.y (mgr.14150) 63 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2024-09-12T23:25:47.961 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:47 smithi136 bash[22601]: cephadm 2024-09-12T23:25:46.357778+0000 mgr.y (mgr.14150) 64 : cephadm [INF] Reconfiguring daemon mon.c on smithi136 2024-09-12T23:25:48.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:47 smithi181 bash[24776]: cephadm 2024-09-12T23:25:46.354909+0000 mgr.y (mgr.14150) 63 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2024-09-12T23:25:48.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:47 smithi181 bash[24776]: cephadm 2024-09-12T23:25:46.357778+0000 mgr.y (mgr.14150) 64 : cephadm [INF] Reconfiguring daemon mon.c on smithi136 2024-09-12T23:25:48.852 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:48 smithi181 bash[24776]: cluster 2024-09-12T23:25:47.374324+0000 mgr.y (mgr.14150) 65 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:48.852 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:48 smithi181 bash[24776]: audit 2024-09-12T23:25:48.220233+0000 mon.a (mon.0) 203 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:48.852 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:48 smithi181 bash[24776]: audit 2024-09-12T23:25:48.225186+0000 mon.a (mon.0) 204 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:48.852 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:48 smithi181 bash[24776]: audit 2024-09-12T23:25:48.226349+0000 mon.a (mon.0) 205 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-12T23:25:48.852 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:48 smithi181 bash[24776]: audit 2024-09-12T23:25:48.227167+0000 mon.a (mon.0) 206 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-12T23:25:48.852 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:48 smithi181 bash[24776]: audit 2024-09-12T23:25:48.227939+0000 mon.a (mon.0) 207 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:25:49.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:48 smithi136 bash[22601]: cluster 2024-09-12T23:25:47.374324+0000 mgr.y (mgr.14150) 65 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:49.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:48 smithi136 bash[22601]: audit 2024-09-12T23:25:48.220233+0000 mon.a (mon.0) 203 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:49.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:48 smithi136 bash[22601]: audit 2024-09-12T23:25:48.225186+0000 mon.a (mon.0) 204 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:49.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:48 smithi136 bash[22601]: audit 2024-09-12T23:25:48.226349+0000 mon.a (mon.0) 205 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-12T23:25:49.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:48 smithi136 bash[22601]: audit 2024-09-12T23:25:48.227167+0000 mon.a (mon.0) 206 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-12T23:25:49.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:48 smithi136 bash[22601]: audit 2024-09-12T23:25:48.227939+0000 mon.a (mon.0) 207 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:25:49.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:48 smithi136 bash[32189]: cluster 2024-09-12T23:25:47.374324+0000 mgr.y (mgr.14150) 65 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:49.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:48 smithi136 bash[32189]: audit 2024-09-12T23:25:48.220233+0000 mon.a (mon.0) 203 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:49.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:48 smithi136 bash[32189]: audit 2024-09-12T23:25:48.225186+0000 mon.a (mon.0) 204 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:49.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:48 smithi136 bash[32189]: audit 2024-09-12T23:25:48.226349+0000 mon.a (mon.0) 205 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-12T23:25:49.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:48 smithi136 bash[32189]: audit 2024-09-12T23:25:48.227167+0000 mon.a (mon.0) 206 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-12T23:25:49.012 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:48 smithi136 bash[32189]: audit 2024-09-12T23:25:48.227939+0000 mon.a (mon.0) 207 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:25:49.582 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:25:50.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:49 smithi136 bash[32189]: cephadm 2024-09-12T23:25:48.226052+0000 mgr.y (mgr.14150) 66 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2024-09-12T23:25:50.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:49 smithi136 bash[32189]: cephadm 2024-09-12T23:25:48.228916+0000 mgr.y (mgr.14150) 67 : cephadm [INF] Reconfiguring daemon mon.b on smithi181 2024-09-12T23:25:50.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:49 smithi136 bash[22601]: cephadm 2024-09-12T23:25:48.226052+0000 mgr.y (mgr.14150) 66 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2024-09-12T23:25:50.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:49 smithi136 bash[22601]: cephadm 2024-09-12T23:25:48.228916+0000 mgr.y (mgr.14150) 67 : cephadm [INF] Reconfiguring daemon mon.b on smithi181 2024-09-12T23:25:50.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:49 smithi181 bash[24776]: cephadm 2024-09-12T23:25:48.226052+0000 mgr.y (mgr.14150) 66 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2024-09-12T23:25:50.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:49 smithi181 bash[24776]: cephadm 2024-09-12T23:25:48.228916+0000 mgr.y (mgr.14150) 67 : cephadm [INF] Reconfiguring daemon mon.b on smithi181 2024-09-12T23:25:50.866 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:50 smithi136 bash[32189]: cluster 2024-09-12T23:25:49.374751+0000 mgr.y (mgr.14150) 68 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:50.866 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:50 smithi136 bash[32189]: audit 2024-09-12T23:25:50.090924+0000 mon.a (mon.0) 208 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:50.866 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:50 smithi136 bash[32189]: audit 2024-09-12T23:25:50.097408+0000 mon.a (mon.0) 209 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:50.866 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:50 smithi136 bash[32189]: audit 2024-09-12T23:25:50.099003+0000 mon.a (mon.0) 210 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:25:50.866 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:50 smithi136 bash[32189]: audit 2024-09-12T23:25:50.101170+0000 mon.a (mon.0) 211 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:25:50.866 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:50 smithi136 bash[32189]: audit 2024-09-12T23:25:50.102431+0000 mon.a (mon.0) 212 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:25:50.866 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:50 smithi136 bash[32189]: audit 2024-09-12T23:25:50.109561+0000 mon.a (mon.0) 213 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:50.866 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:50 smithi136 bash[22601]: cluster 2024-09-12T23:25:49.374751+0000 mgr.y (mgr.14150) 68 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:50.867 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:50 smithi136 bash[22601]: audit 2024-09-12T23:25:50.090924+0000 mon.a (mon.0) 208 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:50.867 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:50 smithi136 bash[22601]: audit 2024-09-12T23:25:50.097408+0000 mon.a (mon.0) 209 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:50.867 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:50 smithi136 bash[22601]: audit 2024-09-12T23:25:50.099003+0000 mon.a (mon.0) 210 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:25:50.867 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:50 smithi136 bash[22601]: audit 2024-09-12T23:25:50.101170+0000 mon.a (mon.0) 211 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:25:50.867 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:50 smithi136 bash[22601]: audit 2024-09-12T23:25:50.102431+0000 mon.a (mon.0) 212 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:25:50.867 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:50 smithi136 bash[22601]: audit 2024-09-12T23:25:50.109561+0000 mon.a (mon.0) 213 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:51.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:50 smithi181 bash[24776]: cluster 2024-09-12T23:25:49.374751+0000 mgr.y (mgr.14150) 68 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:51.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:50 smithi181 bash[24776]: audit 2024-09-12T23:25:50.090924+0000 mon.a (mon.0) 208 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:51.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:50 smithi181 bash[24776]: audit 2024-09-12T23:25:50.097408+0000 mon.a (mon.0) 209 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:51.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:50 smithi181 bash[24776]: audit 2024-09-12T23:25:50.099003+0000 mon.a (mon.0) 210 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:25:51.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:50 smithi181 bash[24776]: audit 2024-09-12T23:25:50.101170+0000 mon.a (mon.0) 211 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:25:51.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:50 smithi181 bash[24776]: audit 2024-09-12T23:25:50.102431+0000 mon.a (mon.0) 212 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:25:51.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:50 smithi181 bash[24776]: audit 2024-09-12T23:25:50.109561+0000 mon.a (mon.0) 213 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:25:52.526 INFO:teuthology.orchestra.run.smithi136.stdout:# minimal ceph.conf for da58a47c-715d-11ef-bceb-c7b262605968 2024-09-12T23:25:52.527 INFO:teuthology.orchestra.run.smithi136.stdout:[global] 2024-09-12T23:25:52.527 INFO:teuthology.orchestra.run.smithi136.stdout: fsid = da58a47c-715d-11ef-bceb-c7b262605968 2024-09-12T23:25:52.527 INFO:teuthology.orchestra.run.smithi136.stdout: mon_host = [v2:172.21.15.136:3300/0,v1:172.21.15.136:6789/0] [v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0] [v2:172.21.15.136:3301/0,v1:172.21.15.136:6790/0] 2024-09-12T23:25:53.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:52 smithi136 bash[32189]: cluster 2024-09-12T23:25:51.375120+0000 mgr.y (mgr.14150) 69 : cluster [DBG] pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:53.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:52 smithi136 bash[32189]: audit 2024-09-12T23:25:52.526634+0000 mon.a (mon.0) 214 : audit [DBG] from='client.? 172.21.15.136:0/1167086675' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:25:53.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:52 smithi136 bash[22601]: cluster 2024-09-12T23:25:51.375120+0000 mgr.y (mgr.14150) 69 : cluster [DBG] pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:53.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:52 smithi136 bash[22601]: audit 2024-09-12T23:25:52.526634+0000 mon.a (mon.0) 214 : audit [DBG] from='client.? 172.21.15.136:0/1167086675' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:25:53.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:52 smithi181 bash[24776]: cluster 2024-09-12T23:25:51.375120+0000 mgr.y (mgr.14150) 69 : cluster [DBG] pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:53.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:52 smithi181 bash[24776]: audit 2024-09-12T23:25:52.526634+0000 mon.a (mon.0) 214 : audit [DBG] from='client.? 172.21.15.136:0/1167086675' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:25:53.180 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-09-12T23:25:53.181 DEBUG:teuthology.orchestra.run.smithi136:> set -ex 2024-09-12T23:25:53.181 DEBUG:teuthology.orchestra.run.smithi136:> sudo dd of=/etc/ceph/ceph.conf 2024-09-12T23:25:53.195 DEBUG:teuthology.orchestra.run.smithi136:> set -ex 2024-09-12T23:25:53.196 DEBUG:teuthology.orchestra.run.smithi136:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-12T23:25:53.251 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2024-09-12T23:25:53.252 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd of=/etc/ceph/ceph.conf 2024-09-12T23:25:53.266 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2024-09-12T23:25:53.266 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-12T23:25:53.320 INFO:tasks.cephadm:Adding mgr.y on smithi136 2024-09-12T23:25:53.320 INFO:tasks.cephadm:Adding mgr.x on smithi181 2024-09-12T23:25:53.320 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph orch apply mgr '2;smithi136=y;smithi181=x' 2024-09-12T23:25:55.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:54 smithi136 bash[22601]: cluster 2024-09-12T23:25:53.375467+0000 mgr.y (mgr.14150) 70 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:55.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:54 smithi136 bash[32189]: cluster 2024-09-12T23:25:53.375467+0000 mgr.y (mgr.14150) 70 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:55.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:54 smithi181 bash[24776]: cluster 2024-09-12T23:25:53.375467+0000 mgr.y (mgr.14150) 70 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:57.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:56 smithi136 bash[22601]: cluster 2024-09-12T23:25:55.375893+0000 mgr.y (mgr.14150) 71 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:57.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:56 smithi136 bash[32189]: cluster 2024-09-12T23:25:55.375893+0000 mgr.y (mgr.14150) 71 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:57.063 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.b/config 2024-09-12T23:25:57.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:56 smithi181 bash[24776]: cluster 2024-09-12T23:25:55.375893+0000 mgr.y (mgr.14150) 71 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:59.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:25:58 smithi136 bash[22601]: cluster 2024-09-12T23:25:57.376245+0000 mgr.y (mgr.14150) 72 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:59.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:25:58 smithi136 bash[32189]: cluster 2024-09-12T23:25:57.376245+0000 mgr.y (mgr.14150) 72 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:59.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:25:58 smithi181 bash[24776]: cluster 2024-09-12T23:25:57.376245+0000 mgr.y (mgr.14150) 72 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:25:59.878 INFO:teuthology.orchestra.run.smithi181.stdout:Scheduled mgr update... 2024-09-12T23:26:00.764 DEBUG:teuthology.orchestra.run.smithi181:mgr.x> sudo journalctl -f -n 0 -u ceph-da58a47c-715d-11ef-bceb-c7b262605968@mgr.x.service 2024-09-12T23:26:00.767 INFO:tasks.cephadm:Deploying OSDs... 2024-09-12T23:26:00.767 DEBUG:teuthology.orchestra.run.smithi136:> set -ex 2024-09-12T23:26:00.767 DEBUG:teuthology.orchestra.run.smithi136:> dd if=/scratch_devs of=/dev/stdout 2024-09-12T23:26:00.773 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-12T23:26:00.774 DEBUG:teuthology.orchestra.run.smithi136:> stat /dev/vg_nvme/lv_1 2024-09-12T23:26:00.821 INFO:teuthology.orchestra.run.smithi136.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-12T23:26:00.822 INFO:teuthology.orchestra.run.smithi136.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-12T23:26:00.822 INFO:teuthology.orchestra.run.smithi136.stdout:Device: 5h/5d Inode: 877 Links: 1 2024-09-12T23:26:00.822 INFO:teuthology.orchestra.run.smithi136.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-12T23:26:00.822 INFO:teuthology.orchestra.run.smithi136.stdout:Access: 2024-09-12 23:17:04.196878833 +0000 2024-09-12T23:26:00.822 INFO:teuthology.orchestra.run.smithi136.stdout:Modify: 2024-09-12 23:17:03.936881141 +0000 2024-09-12T23:26:00.822 INFO:teuthology.orchestra.run.smithi136.stdout:Change: 2024-09-12 23:17:03.936881141 +0000 2024-09-12T23:26:00.822 INFO:teuthology.orchestra.run.smithi136.stdout: Birth: - 2024-09-12T23:26:00.823 DEBUG:teuthology.orchestra.run.smithi136:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-12T23:26:00.876 INFO:teuthology.orchestra.run.smithi136.stderr:1+0 records in 2024-09-12T23:26:00.876 INFO:teuthology.orchestra.run.smithi136.stderr:1+0 records out 2024-09-12T23:26:00.876 INFO:teuthology.orchestra.run.smithi136.stderr:512 bytes copied, 0.000633698 s, 808 kB/s 2024-09-12T23:26:00.877 DEBUG:teuthology.orchestra.run.smithi136:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-12T23:26:00.926 DEBUG:teuthology.orchestra.run.smithi136:> stat /dev/vg_nvme/lv_2 2024-09-12T23:26:00.973 INFO:teuthology.orchestra.run.smithi136.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-12T23:26:00.973 INFO:teuthology.orchestra.run.smithi136.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-12T23:26:00.973 INFO:teuthology.orchestra.run.smithi136.stdout:Device: 5h/5d Inode: 896 Links: 1 2024-09-12T23:26:00.973 INFO:teuthology.orchestra.run.smithi136.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-12T23:26:00.973 INFO:teuthology.orchestra.run.smithi136.stdout:Access: 2024-09-12 23:17:04.712874252 +0000 2024-09-12T23:26:00.974 INFO:teuthology.orchestra.run.smithi136.stdout:Modify: 2024-09-12 23:17:04.460876489 +0000 2024-09-12T23:26:00.974 INFO:teuthology.orchestra.run.smithi136.stdout:Change: 2024-09-12 23:17:04.460876489 +0000 2024-09-12T23:26:00.974 INFO:teuthology.orchestra.run.smithi136.stdout: Birth: - 2024-09-12T23:26:00.974 DEBUG:teuthology.orchestra.run.smithi136:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-12T23:26:01.027 INFO:teuthology.orchestra.run.smithi136.stderr:1+0 records in 2024-09-12T23:26:01.027 INFO:teuthology.orchestra.run.smithi136.stderr:1+0 records out 2024-09-12T23:26:01.027 INFO:teuthology.orchestra.run.smithi136.stderr:512 bytes copied, 0.000422 s, 1.2 MB/s 2024-09-12T23:26:01.028 DEBUG:teuthology.orchestra.run.smithi136:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-12T23:26:01.078 DEBUG:teuthology.orchestra.run.smithi136:> stat /dev/vg_nvme/lv_3 2024-09-12T23:26:01.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:00 smithi181 bash[24776]: cluster 2024-09-12T23:25:59.376688+0000 mgr.y (mgr.14150) 73 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:01.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:00 smithi181 bash[24776]: audit 2024-09-12T23:25:59.866690+0000 mgr.y (mgr.14150) 74 : audit [DBG] from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi136=y;smithi181=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:26:01.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:00 smithi181 bash[24776]: cephadm 2024-09-12T23:25:59.869709+0000 mgr.y (mgr.14150) 75 : cephadm [INF] Saving service mgr spec with placement smithi136=y;smithi181=x;count:2 2024-09-12T23:26:01.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:00 smithi181 bash[24776]: audit 2024-09-12T23:25:59.877207+0000 mon.a (mon.0) 215 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:01.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:00 smithi181 bash[24776]: audit 2024-09-12T23:25:59.878907+0000 mon.a (mon.0) 216 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:26:01.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:00 smithi181 bash[24776]: audit 2024-09-12T23:25:59.882238+0000 mon.a (mon.0) 217 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:26:01.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:00 smithi181 bash[24776]: audit 2024-09-12T23:25:59.884008+0000 mon.a (mon.0) 218 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:26:01.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:00 smithi181 bash[24776]: audit 2024-09-12T23:25:59.894201+0000 mon.a (mon.0) 219 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:01.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:00 smithi181 bash[24776]: audit 2024-09-12T23:25:59.897719+0000 mon.a (mon.0) 220 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-12T23:26:01.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:00 smithi181 bash[24776]: audit 2024-09-12T23:25:59.904527+0000 mon.a (mon.0) 221 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-12T23:26:01.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:00 smithi181 bash[24776]: audit 2024-09-12T23:25:59.906304+0000 mon.a (mon.0) 222 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-12T23:26:01.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:00 smithi181 bash[24776]: audit 2024-09-12T23:25:59.908181+0000 mon.a (mon.0) 223 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:26:01.124 INFO:teuthology.orchestra.run.smithi136.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-12T23:26:01.125 INFO:teuthology.orchestra.run.smithi136.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-12T23:26:01.125 INFO:teuthology.orchestra.run.smithi136.stdout:Device: 5h/5d Inode: 909 Links: 1 2024-09-12T23:26:01.125 INFO:teuthology.orchestra.run.smithi136.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-12T23:26:01.125 INFO:teuthology.orchestra.run.smithi136.stdout:Access: 2024-09-12 23:17:05.208869848 +0000 2024-09-12T23:26:01.125 INFO:teuthology.orchestra.run.smithi136.stdout:Modify: 2024-09-12 23:17:04.964872014 +0000 2024-09-12T23:26:01.125 INFO:teuthology.orchestra.run.smithi136.stdout:Change: 2024-09-12 23:17:04.964872014 +0000 2024-09-12T23:26:01.125 INFO:teuthology.orchestra.run.smithi136.stdout: Birth: - 2024-09-12T23:26:01.125 DEBUG:teuthology.orchestra.run.smithi136:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-12T23:26:01.177 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:00 smithi136 bash[32189]: cluster 2024-09-12T23:25:59.376688+0000 mgr.y (mgr.14150) 73 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:01.177 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:00 smithi136 bash[32189]: audit 2024-09-12T23:25:59.866690+0000 mgr.y (mgr.14150) 74 : audit [DBG] from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi136=y;smithi181=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:26:01.177 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:00 smithi136 bash[32189]: cephadm 2024-09-12T23:25:59.869709+0000 mgr.y (mgr.14150) 75 : cephadm [INF] Saving service mgr spec with placement smithi136=y;smithi181=x;count:2 2024-09-12T23:26:01.178 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:00 smithi136 bash[32189]: audit 2024-09-12T23:25:59.877207+0000 mon.a (mon.0) 215 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:01.178 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:00 smithi136 bash[32189]: audit 2024-09-12T23:25:59.878907+0000 mon.a (mon.0) 216 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:26:01.178 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:00 smithi136 bash[32189]: audit 2024-09-12T23:25:59.882238+0000 mon.a (mon.0) 217 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:26:01.178 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:00 smithi136 bash[32189]: audit 2024-09-12T23:25:59.884008+0000 mon.a (mon.0) 218 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:26:01.178 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:00 smithi136 bash[32189]: audit 2024-09-12T23:25:59.894201+0000 mon.a (mon.0) 219 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:01.178 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:00 smithi136 bash[32189]: audit 2024-09-12T23:25:59.897719+0000 mon.a (mon.0) 220 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-12T23:26:01.178 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:00 smithi136 bash[32189]: audit 2024-09-12T23:25:59.904527+0000 mon.a (mon.0) 221 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-12T23:26:01.178 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:00 smithi136 bash[32189]: audit 2024-09-12T23:25:59.906304+0000 mon.a (mon.0) 222 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-12T23:26:01.178 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:00 smithi136 bash[32189]: audit 2024-09-12T23:25:59.908181+0000 mon.a (mon.0) 223 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:26:01.178 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:00 smithi136 bash[22601]: cluster 2024-09-12T23:25:59.376688+0000 mgr.y (mgr.14150) 73 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:01.179 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:00 smithi136 bash[22601]: audit 2024-09-12T23:25:59.866690+0000 mgr.y (mgr.14150) 74 : audit [DBG] from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi136=y;smithi181=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:26:01.179 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:00 smithi136 bash[22601]: cephadm 2024-09-12T23:25:59.869709+0000 mgr.y (mgr.14150) 75 : cephadm [INF] Saving service mgr spec with placement smithi136=y;smithi181=x;count:2 2024-09-12T23:26:01.179 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:00 smithi136 bash[22601]: audit 2024-09-12T23:25:59.877207+0000 mon.a (mon.0) 215 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:01.179 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:00 smithi136 bash[22601]: audit 2024-09-12T23:25:59.878907+0000 mon.a (mon.0) 216 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:26:01.179 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:00 smithi136 bash[22601]: audit 2024-09-12T23:25:59.882238+0000 mon.a (mon.0) 217 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:26:01.179 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:00 smithi136 bash[22601]: audit 2024-09-12T23:25:59.884008+0000 mon.a (mon.0) 218 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:26:01.179 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:00 smithi136 bash[22601]: audit 2024-09-12T23:25:59.894201+0000 mon.a (mon.0) 219 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:01.179 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:00 smithi136 bash[22601]: audit 2024-09-12T23:25:59.897719+0000 mon.a (mon.0) 220 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-12T23:26:01.179 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:00 smithi136 bash[22601]: audit 2024-09-12T23:25:59.904527+0000 mon.a (mon.0) 221 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-12T23:26:01.179 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:00 smithi136 bash[22601]: audit 2024-09-12T23:25:59.906304+0000 mon.a (mon.0) 222 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-12T23:26:01.179 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:00 smithi136 bash[22601]: audit 2024-09-12T23:25:59.908181+0000 mon.a (mon.0) 223 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:26:01.180 INFO:teuthology.orchestra.run.smithi136.stderr:1+0 records in 2024-09-12T23:26:01.180 INFO:teuthology.orchestra.run.smithi136.stderr:1+0 records out 2024-09-12T23:26:01.180 INFO:teuthology.orchestra.run.smithi136.stderr:512 bytes copied, 0.000317175 s, 1.6 MB/s 2024-09-12T23:26:01.181 DEBUG:teuthology.orchestra.run.smithi136:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-12T23:26:01.230 DEBUG:teuthology.orchestra.run.smithi136:> stat /dev/vg_nvme/lv_4 2024-09-12T23:26:01.276 INFO:teuthology.orchestra.run.smithi136.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-12T23:26:01.277 INFO:teuthology.orchestra.run.smithi136.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-12T23:26:01.277 INFO:teuthology.orchestra.run.smithi136.stdout:Device: 5h/5d Inode: 931 Links: 1 2024-09-12T23:26:01.277 INFO:teuthology.orchestra.run.smithi136.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-12T23:26:01.277 INFO:teuthology.orchestra.run.smithi136.stdout:Access: 2024-09-12 23:17:05.720865303 +0000 2024-09-12T23:26:01.277 INFO:teuthology.orchestra.run.smithi136.stdout:Modify: 2024-09-12 23:17:05.456867647 +0000 2024-09-12T23:26:01.277 INFO:teuthology.orchestra.run.smithi136.stdout:Change: 2024-09-12 23:17:05.456867647 +0000 2024-09-12T23:26:01.277 INFO:teuthology.orchestra.run.smithi136.stdout: Birth: - 2024-09-12T23:26:01.277 DEBUG:teuthology.orchestra.run.smithi136:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-12T23:26:01.330 INFO:teuthology.orchestra.run.smithi136.stderr:1+0 records in 2024-09-12T23:26:01.330 INFO:teuthology.orchestra.run.smithi136.stderr:1+0 records out 2024-09-12T23:26:01.330 INFO:teuthology.orchestra.run.smithi136.stderr:512 bytes copied, 0.000314853 s, 1.6 MB/s 2024-09-12T23:26:01.331 DEBUG:teuthology.orchestra.run.smithi136:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-12T23:26:01.382 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2024-09-12T23:26:01.382 DEBUG:teuthology.orchestra.run.smithi181:> dd if=/scratch_devs of=/dev/stdout 2024-09-12T23:26:01.388 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-12T23:26:01.388 DEBUG:teuthology.orchestra.run.smithi181:> stat /dev/vg_nvme/lv_1 2024-09-12T23:26:01.436 INFO:teuthology.orchestra.run.smithi181.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-12T23:26:01.436 INFO:teuthology.orchestra.run.smithi181.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-12T23:26:01.437 INFO:teuthology.orchestra.run.smithi181.stdout:Device: 5h/5d Inode: 873 Links: 1 2024-09-12T23:26:01.437 INFO:teuthology.orchestra.run.smithi181.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-12T23:26:01.437 INFO:teuthology.orchestra.run.smithi181.stdout:Access: 2024-09-12 23:17:04.173991272 +0000 2024-09-12T23:26:01.437 INFO:teuthology.orchestra.run.smithi181.stdout:Modify: 2024-09-12 23:17:04.169991306 +0000 2024-09-12T23:26:01.437 INFO:teuthology.orchestra.run.smithi181.stdout:Change: 2024-09-12 23:17:04.169991306 +0000 2024-09-12T23:26:01.437 INFO:teuthology.orchestra.run.smithi181.stdout: Birth: - 2024-09-12T23:26:01.437 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-12T23:26:01.491 INFO:teuthology.orchestra.run.smithi181.stderr:1+0 records in 2024-09-12T23:26:01.491 INFO:teuthology.orchestra.run.smithi181.stderr:1+0 records out 2024-09-12T23:26:01.492 INFO:teuthology.orchestra.run.smithi181.stderr:512 bytes copied, 0.000421566 s, 1.2 MB/s 2024-09-12T23:26:01.493 DEBUG:teuthology.orchestra.run.smithi181:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-12T23:26:01.540 DEBUG:teuthology.orchestra.run.smithi181:> stat /dev/vg_nvme/lv_2 2024-09-12T23:26:01.587 INFO:teuthology.orchestra.run.smithi181.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-12T23:26:01.588 INFO:teuthology.orchestra.run.smithi181.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-12T23:26:01.588 INFO:teuthology.orchestra.run.smithi181.stdout:Device: 5h/5d Inode: 891 Links: 1 2024-09-12T23:26:01.588 INFO:teuthology.orchestra.run.smithi181.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-12T23:26:01.588 INFO:teuthology.orchestra.run.smithi181.stdout:Access: 2024-09-12 23:17:04.929984518 +0000 2024-09-12T23:26:01.588 INFO:teuthology.orchestra.run.smithi181.stdout:Modify: 2024-09-12 23:17:04.649987019 +0000 2024-09-12T23:26:01.588 INFO:teuthology.orchestra.run.smithi181.stdout:Change: 2024-09-12 23:17:04.649987019 +0000 2024-09-12T23:26:01.588 INFO:teuthology.orchestra.run.smithi181.stdout: Birth: - 2024-09-12T23:26:01.588 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-12T23:26:01.641 INFO:teuthology.orchestra.run.smithi181.stderr:1+0 records in 2024-09-12T23:26:01.641 INFO:teuthology.orchestra.run.smithi181.stderr:1+0 records out 2024-09-12T23:26:01.642 INFO:teuthology.orchestra.run.smithi181.stderr:512 bytes copied, 0.000405402 s, 1.3 MB/s 2024-09-12T23:26:01.643 DEBUG:teuthology.orchestra.run.smithi181:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-12T23:26:01.689 DEBUG:teuthology.orchestra.run.smithi181:> stat /dev/vg_nvme/lv_3 2024-09-12T23:26:01.740 INFO:teuthology.orchestra.run.smithi181.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-12T23:26:01.740 INFO:teuthology.orchestra.run.smithi181.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-12T23:26:01.740 INFO:teuthology.orchestra.run.smithi181.stdout:Device: 5h/5d Inode: 903 Links: 1 2024-09-12T23:26:01.741 INFO:teuthology.orchestra.run.smithi181.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-12T23:26:01.741 INFO:teuthology.orchestra.run.smithi181.stdout:Access: 2024-09-12 23:17:05.405980267 +0000 2024-09-12T23:26:01.741 INFO:teuthology.orchestra.run.smithi181.stdout:Modify: 2024-09-12 23:17:05.157982482 +0000 2024-09-12T23:26:01.741 INFO:teuthology.orchestra.run.smithi181.stdout:Change: 2024-09-12 23:17:05.157982482 +0000 2024-09-12T23:26:01.741 INFO:teuthology.orchestra.run.smithi181.stdout: Birth: - 2024-09-12T23:26:01.741 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-12T23:26:01.794 INFO:teuthology.orchestra.run.smithi181.stderr:1+0 records in 2024-09-12T23:26:01.795 INFO:teuthology.orchestra.run.smithi181.stderr:1+0 records out 2024-09-12T23:26:01.795 INFO:teuthology.orchestra.run.smithi181.stderr:512 bytes copied, 0.000443114 s, 1.2 MB/s 2024-09-12T23:26:01.796 DEBUG:teuthology.orchestra.run.smithi181:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-12T23:26:01.845 DEBUG:teuthology.orchestra.run.smithi181:> stat /dev/vg_nvme/lv_4 2024-09-12T23:26:01.891 INFO:teuthology.orchestra.run.smithi181.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-12T23:26:01.892 INFO:teuthology.orchestra.run.smithi181.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-12T23:26:01.892 INFO:teuthology.orchestra.run.smithi181.stdout:Device: 5h/5d Inode: 920 Links: 1 2024-09-12T23:26:01.892 INFO:teuthology.orchestra.run.smithi181.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-12T23:26:01.892 INFO:teuthology.orchestra.run.smithi181.stdout:Access: 2024-09-12 23:17:05.905975800 +0000 2024-09-12T23:26:01.892 INFO:teuthology.orchestra.run.smithi181.stdout:Modify: 2024-09-12 23:17:05.629978265 +0000 2024-09-12T23:26:01.892 INFO:teuthology.orchestra.run.smithi181.stdout:Change: 2024-09-12 23:17:05.629978265 +0000 2024-09-12T23:26:01.892 INFO:teuthology.orchestra.run.smithi181.stdout: Birth: - 2024-09-12T23:26:01.892 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-12T23:26:01.950 INFO:teuthology.orchestra.run.smithi181.stderr:1+0 records in 2024-09-12T23:26:01.950 INFO:teuthology.orchestra.run.smithi181.stderr:1+0 records out 2024-09-12T23:26:01.950 INFO:teuthology.orchestra.run.smithi181.stderr:512 bytes copied, 0.000397232 s, 1.3 MB/s 2024-09-12T23:26:01.951 DEBUG:teuthology.orchestra.run.smithi181:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-12T23:26:02.001 INFO:tasks.cephadm:Deploying osd.0 on smithi136 with /dev/vg_nvme/lv_4... 2024-09-12T23:26:02.001 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-09-12T23:26:02.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:01 smithi181 bash[24776]: cephadm 2024-09-12T23:25:59.909972+0000 mgr.y (mgr.14150) 76 : cephadm [INF] Deploying daemon mgr.x on smithi181 2024-09-12T23:26:02.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:01 smithi136 bash[22601]: cephadm 2024-09-12T23:25:59.909972+0000 mgr.y (mgr.14150) 76 : cephadm [INF] Deploying daemon mgr.x on smithi181 2024-09-12T23:26:02.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:01 smithi136 bash[32189]: cephadm 2024-09-12T23:25:59.909972+0000 mgr.y (mgr.14150) 76 : cephadm [INF] Deploying daemon mgr.x on smithi181 2024-09-12T23:26:02.489 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:02 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:26:02.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:02 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:26:02.820 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:26:02 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:26:02.820 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:26:02 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:26:02.820 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:26:02 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:26:02.820 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:26:02 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:26:02.820 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:26:02 smithi181 systemd[1]: Started Ceph mgr.x for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:26:03.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:02 smithi181 bash[24776]: cluster 2024-09-12T23:26:01.376998+0000 mgr.y (mgr.14150) 77 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:03.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:02 smithi181 bash[24776]: audit 2024-09-12T23:26:02.751126+0000 mon.a (mon.0) 224 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:03.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:02 smithi181 bash[24776]: audit 2024-09-12T23:26:02.758404+0000 mon.a (mon.0) 225 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:03.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:02 smithi181 bash[24776]: audit 2024-09-12T23:26:02.765458+0000 mon.a (mon.0) 226 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:03.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:02 smithi181 bash[24776]: audit 2024-09-12T23:26:02.772455+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:03.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:02 smithi181 bash[24776]: audit 2024-09-12T23:26:02.831976+0000 mon.a (mon.0) 228 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:26:03.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:02 smithi136 bash[22601]: cluster 2024-09-12T23:26:01.376998+0000 mgr.y (mgr.14150) 77 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:03.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:02 smithi136 bash[22601]: audit 2024-09-12T23:26:02.751126+0000 mon.a (mon.0) 224 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:03.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:02 smithi136 bash[22601]: audit 2024-09-12T23:26:02.758404+0000 mon.a (mon.0) 225 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:03.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:02 smithi136 bash[22601]: audit 2024-09-12T23:26:02.765458+0000 mon.a (mon.0) 226 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:03.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:02 smithi136 bash[22601]: audit 2024-09-12T23:26:02.772455+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:03.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:02 smithi136 bash[22601]: audit 2024-09-12T23:26:02.831976+0000 mon.a (mon.0) 228 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:26:03.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:02 smithi136 bash[32189]: cluster 2024-09-12T23:26:01.376998+0000 mgr.y (mgr.14150) 77 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:03.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:02 smithi136 bash[32189]: audit 2024-09-12T23:26:02.751126+0000 mon.a (mon.0) 224 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:03.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:02 smithi136 bash[32189]: audit 2024-09-12T23:26:02.758404+0000 mon.a (mon.0) 225 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:03.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:02 smithi136 bash[32189]: audit 2024-09-12T23:26:02.765458+0000 mon.a (mon.0) 226 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:03.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:02 smithi136 bash[32189]: audit 2024-09-12T23:26:02.772455+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:03.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:02 smithi136 bash[32189]: audit 2024-09-12T23:26:02.831976+0000 mon.a (mon.0) 228 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:26:04.099 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:26:03 smithi181 bash[25717]: debug 2024-09-12T23:26:03.842+0000 7f21b414a640 1 -- 172.21.15.181:0/3314840631 <== mon.1 v2:172.21.15.181:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x558af3ba4340 con 0x558af3bb0800 2024-09-12T23:26:04.099 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:26:03 smithi181 bash[25717]: debug 2024-09-12T23:26:03.954+0000 7f21b69b2ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-12T23:26:05.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:04 smithi136 bash[22601]: cluster 2024-09-12T23:26:03.377416+0000 mgr.y (mgr.14150) 78 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:05.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:04 smithi136 bash[22601]: audit 2024-09-12T23:26:03.643500+0000 mon.a (mon.0) 229 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:05.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:04 smithi136 bash[32189]: cluster 2024-09-12T23:26:03.377416+0000 mgr.y (mgr.14150) 78 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:05.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:04 smithi136 bash[32189]: audit 2024-09-12T23:26:03.643500+0000 mon.a (mon.0) 229 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:05.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:04 smithi181 bash[24776]: cluster 2024-09-12T23:26:03.377416+0000 mgr.y (mgr.14150) 78 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:05.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:04 smithi181 bash[24776]: audit 2024-09-12T23:26:03.643500+0000 mon.a (mon.0) 229 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:05.599 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:26:05 smithi181 bash[25717]: debug 2024-09-12T23:26:05.130+0000 7f21b69b2ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-12T23:26:05.599 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:26:05 smithi181 bash[25717]: debug 2024-09-12T23:26:05.190+0000 7f21b69b2ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-12T23:26:05.599 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:26:05 smithi181 bash[25717]: debug 2024-09-12T23:26:05.338+0000 7f21b69b2ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-12T23:26:06.255 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:26:05 smithi181 bash[25717]: debug 2024-09-12T23:26:05.962+0000 7f21b69b2ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-12T23:26:06.255 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:26:06 smithi181 bash[25717]: debug 2024-09-12T23:26:06.030+0000 7f21b69b2ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-12T23:26:06.255 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:26:06 smithi181 bash[25717]: debug 2024-09-12T23:26:06.098+0000 7f21b69b2ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-12T23:26:06.599 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:26:06 smithi181 bash[25717]: debug 2024-09-12T23:26:06.250+0000 7f21b69b2ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-12T23:26:06.599 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:26:06 smithi181 bash[25717]: debug 2024-09-12T23:26:06.326+0000 7f21b69b2ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-12T23:26:06.599 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:26:06 smithi181 bash[25717]: debug 2024-09-12T23:26:06.390+0000 7f21b69b2ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-12T23:26:06.731 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:26:06.923 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:06 smithi181 bash[24776]: cluster 2024-09-12T23:26:05.377825+0000 mgr.y (mgr.14150) 79 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:06.923 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:26:06 smithi181 bash[25717]: debug 2024-09-12T23:26:06.614+0000 7f21b69b2ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-12T23:26:06.923 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:26:06 smithi181 bash[25717]: debug 2024-09-12T23:26:06.682+0000 7f21b69b2ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-12T23:26:06.923 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:26:06 smithi181 bash[25717]: debug 2024-09-12T23:26:06.846+0000 7f21b69b2ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-12T23:26:06.923 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:26:06 smithi181 bash[25717]: debug 2024-09-12T23:26:06.918+0000 7f21b69b2ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-12T23:26:07.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:06 smithi136 bash[22601]: cluster 2024-09-12T23:26:05.377825+0000 mgr.y (mgr.14150) 79 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:07.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:06 smithi136 bash[32189]: cluster 2024-09-12T23:26:05.377825+0000 mgr.y (mgr.14150) 79 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:07.309 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:26:07 smithi181 bash[25717]: debug 2024-09-12T23:26:07.150+0000 7f21b69b2ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-12T23:26:07.599 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:26:07 smithi181 bash[25717]: debug 2024-09-12T23:26:07.306+0000 7f21b69b2ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-12T23:26:07.599 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:26:07 smithi181 bash[25717]: debug 2024-09-12T23:26:07.382+0000 7f21b69b2ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-12T23:26:07.599 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:26:07 smithi181 bash[25717]: debug 2024-09-12T23:26:07.530+0000 7f21b69b2ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-12T23:26:08.595 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:26:08 smithi181 bash[25717]: debug 2024-09-12T23:26:08.322+0000 7f21b69b2ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-12T23:26:08.595 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:26:08 smithi181 bash[25717]: debug 2024-09-12T23:26:08.398+0000 7f21b69b2ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-12T23:26:08.849 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:26:08 smithi181 bash[25717]: debug 2024-09-12T23:26:08.590+0000 7f21b69b2ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-12T23:26:08.849 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:26:08 smithi181 bash[25717]: debug 2024-09-12T23:26:08.662+0000 7f21b69b2ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-12T23:26:08.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:08 smithi181 bash[24776]: cluster 2024-09-12T23:26:07.378246+0000 mgr.y (mgr.14150) 80 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:08.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:08 smithi181 bash[24776]: audit 2024-09-12T23:26:07.788741+0000 mon.a (mon.0) 230 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:08.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:08 smithi181 bash[24776]: audit 2024-09-12T23:26:07.794359+0000 mon.a (mon.0) 231 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:08.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:08 smithi181 bash[24776]: audit 2024-09-12T23:26:08.171942+0000 mon.a (mon.0) 232 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:26:08.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:08 smithi181 bash[24776]: audit 2024-09-12T23:26:08.173780+0000 mon.a (mon.0) 233 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:26:08.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:08 smithi181 bash[24776]: audit 2024-09-12T23:26:08.184736+0000 mon.a (mon.0) 234 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:08.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:08 smithi181 bash[24776]: audit 2024-09-12T23:26:08.226641+0000 mon.a (mon.0) 235 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-12T23:26:08.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:08 smithi181 bash[24776]: audit 2024-09-12T23:26:08.228333+0000 mon.a (mon.0) 236 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-12T23:26:08.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:08 smithi181 bash[24776]: audit 2024-09-12T23:26:08.229856+0000 mon.a (mon.0) 237 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:26:09.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:08 smithi136 bash[22601]: cluster 2024-09-12T23:26:07.378246+0000 mgr.y (mgr.14150) 80 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:09.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:08 smithi136 bash[22601]: audit 2024-09-12T23:26:07.788741+0000 mon.a (mon.0) 230 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:09.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:08 smithi136 bash[22601]: audit 2024-09-12T23:26:07.794359+0000 mon.a (mon.0) 231 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:09.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:08 smithi136 bash[22601]: audit 2024-09-12T23:26:08.171942+0000 mon.a (mon.0) 232 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:26:09.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:08 smithi136 bash[22601]: audit 2024-09-12T23:26:08.173780+0000 mon.a (mon.0) 233 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:26:09.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:08 smithi136 bash[22601]: audit 2024-09-12T23:26:08.184736+0000 mon.a (mon.0) 234 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:09.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:08 smithi136 bash[22601]: audit 2024-09-12T23:26:08.226641+0000 mon.a (mon.0) 235 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-12T23:26:09.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:08 smithi136 bash[22601]: audit 2024-09-12T23:26:08.228333+0000 mon.a (mon.0) 236 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-12T23:26:09.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:08 smithi136 bash[22601]: audit 2024-09-12T23:26:08.229856+0000 mon.a (mon.0) 237 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:26:09.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:08 smithi136 bash[32189]: cluster 2024-09-12T23:26:07.378246+0000 mgr.y (mgr.14150) 80 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:09.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:08 smithi136 bash[32189]: audit 2024-09-12T23:26:07.788741+0000 mon.a (mon.0) 230 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:09.262 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:08 smithi136 bash[32189]: audit 2024-09-12T23:26:07.794359+0000 mon.a (mon.0) 231 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:09.262 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:08 smithi136 bash[32189]: audit 2024-09-12T23:26:08.171942+0000 mon.a (mon.0) 232 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:26:09.262 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:08 smithi136 bash[32189]: audit 2024-09-12T23:26:08.173780+0000 mon.a (mon.0) 233 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:26:09.262 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:08 smithi136 bash[32189]: audit 2024-09-12T23:26:08.184736+0000 mon.a (mon.0) 234 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:09.262 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:08 smithi136 bash[32189]: audit 2024-09-12T23:26:08.226641+0000 mon.a (mon.0) 235 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-12T23:26:09.262 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:08 smithi136 bash[32189]: audit 2024-09-12T23:26:08.228333+0000 mon.a (mon.0) 236 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-12T23:26:09.262 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:08 smithi136 bash[32189]: audit 2024-09-12T23:26:08.229856+0000 mon.a (mon.0) 237 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:26:09.349 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:26:08 smithi181 bash[25717]: debug 2024-09-12T23:26:08.978+0000 7f21b69b2ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-12T23:26:09.349 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:26:09 smithi181 bash[25717]: debug 2024-09-12T23:26:09.050+0000 7f21b69b2ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-12T23:26:10.070 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:09 smithi136 bash[22601]: cephadm 2024-09-12T23:26:08.226028+0000 mgr.y (mgr.14150) 81 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2024-09-12T23:26:10.070 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:09 smithi136 bash[22601]: cephadm 2024-09-12T23:26:08.231387+0000 mgr.y (mgr.14150) 82 : cephadm [INF] Reconfiguring daemon mgr.y on smithi136 2024-09-12T23:26:10.070 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:09 smithi136 bash[22601]: cluster 2024-09-12T23:26:09.076196+0000 mon.a (mon.0) 238 : cluster [DBG] Standby manager daemon x started 2024-09-12T23:26:10.070 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:09 smithi136 bash[22601]: audit 2024-09-12T23:26:09.076870+0000 mon.b (mon.1) 3 : audit [DBG] from='mgr.? 172.21.15.181:0/1503847816' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-12T23:26:10.071 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:09 smithi136 bash[22601]: audit 2024-09-12T23:26:09.077799+0000 mon.b (mon.1) 4 : audit [DBG] from='mgr.? 172.21.15.181:0/1503847816' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-12T23:26:10.071 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:09 smithi136 bash[22601]: audit 2024-09-12T23:26:09.079037+0000 mon.b (mon.1) 5 : audit [DBG] from='mgr.? 172.21.15.181:0/1503847816' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-12T23:26:10.071 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:09 smithi136 bash[22601]: audit 2024-09-12T23:26:09.079723+0000 mon.b (mon.1) 6 : audit [DBG] from='mgr.? 172.21.15.181:0/1503847816' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-12T23:26:10.071 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:09 smithi136 bash[32189]: cephadm 2024-09-12T23:26:08.226028+0000 mgr.y (mgr.14150) 81 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2024-09-12T23:26:10.071 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:09 smithi136 bash[32189]: cephadm 2024-09-12T23:26:08.231387+0000 mgr.y (mgr.14150) 82 : cephadm [INF] Reconfiguring daemon mgr.y on smithi136 2024-09-12T23:26:10.071 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:09 smithi136 bash[32189]: cluster 2024-09-12T23:26:09.076196+0000 mon.a (mon.0) 238 : cluster [DBG] Standby manager daemon x started 2024-09-12T23:26:10.071 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:09 smithi136 bash[32189]: audit 2024-09-12T23:26:09.076870+0000 mon.b (mon.1) 3 : audit [DBG] from='mgr.? 172.21.15.181:0/1503847816' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-12T23:26:10.071 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:09 smithi136 bash[32189]: audit 2024-09-12T23:26:09.077799+0000 mon.b (mon.1) 4 : audit [DBG] from='mgr.? 172.21.15.181:0/1503847816' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-12T23:26:10.071 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:09 smithi136 bash[32189]: audit 2024-09-12T23:26:09.079037+0000 mon.b (mon.1) 5 : audit [DBG] from='mgr.? 172.21.15.181:0/1503847816' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-12T23:26:10.071 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:09 smithi136 bash[32189]: audit 2024-09-12T23:26:09.079723+0000 mon.b (mon.1) 6 : audit [DBG] from='mgr.? 172.21.15.181:0/1503847816' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-12T23:26:10.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:09 smithi181 bash[24776]: cephadm 2024-09-12T23:26:08.226028+0000 mgr.y (mgr.14150) 81 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2024-09-12T23:26:10.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:09 smithi181 bash[24776]: cephadm 2024-09-12T23:26:08.231387+0000 mgr.y (mgr.14150) 82 : cephadm [INF] Reconfiguring daemon mgr.y on smithi136 2024-09-12T23:26:10.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:09 smithi181 bash[24776]: cluster 2024-09-12T23:26:09.076196+0000 mon.a (mon.0) 238 : cluster [DBG] Standby manager daemon x started 2024-09-12T23:26:10.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:09 smithi181 bash[24776]: audit 2024-09-12T23:26:09.076870+0000 mon.b (mon.1) 3 : audit [DBG] from='mgr.? 172.21.15.181:0/1503847816' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-12T23:26:10.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:09 smithi181 bash[24776]: audit 2024-09-12T23:26:09.077799+0000 mon.b (mon.1) 4 : audit [DBG] from='mgr.? 172.21.15.181:0/1503847816' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-12T23:26:10.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:09 smithi181 bash[24776]: audit 2024-09-12T23:26:09.079037+0000 mon.b (mon.1) 5 : audit [DBG] from='mgr.? 172.21.15.181:0/1503847816' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-12T23:26:10.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:09 smithi181 bash[24776]: audit 2024-09-12T23:26:09.079723+0000 mon.b (mon.1) 6 : audit [DBG] from='mgr.? 172.21.15.181:0/1503847816' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-12T23:26:10.324 INFO:teuthology.orchestra.run.smithi136.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-09-12T23:26:10.484 INFO:teuthology.orchestra.run.smithi136.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-09-12T23:26:10.497 INFO:teuthology.orchestra.run.smithi136.stderr: stderr: 10+0 records in 2024-09-12T23:26:10.497 INFO:teuthology.orchestra.run.smithi136.stderr:10+0 records out 2024-09-12T23:26:10.497 INFO:teuthology.orchestra.run.smithi136.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0110993 s, 945 MB/s 2024-09-12T23:26:10.501 INFO:teuthology.orchestra.run.smithi136.stderr:--> Zapping successful for: 2024-09-12T23:26:11.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:10 smithi181 bash[24776]: cluster 2024-09-12T23:26:09.378651+0000 mgr.y (mgr.14150) 83 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:11.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:10 smithi181 bash[24776]: cluster 2024-09-12T23:26:09.814120+0000 mon.a (mon.0) 239 : cluster [DBG] mgrmap e14: y(active, since 104s), standbys: x 2024-09-12T23:26:11.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:10 smithi181 bash[24776]: audit 2024-09-12T23:26:09.814630+0000 mon.a (mon.0) 240 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-12T23:26:11.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:10 smithi136 bash[22601]: cluster 2024-09-12T23:26:09.378651+0000 mgr.y (mgr.14150) 83 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:11.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:10 smithi136 bash[22601]: cluster 2024-09-12T23:26:09.814120+0000 mon.a (mon.0) 239 : cluster [DBG] mgrmap e14: y(active, since 104s), standbys: x 2024-09-12T23:26:11.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:10 smithi136 bash[22601]: audit 2024-09-12T23:26:09.814630+0000 mon.a (mon.0) 240 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-12T23:26:11.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:10 smithi136 bash[32189]: cluster 2024-09-12T23:26:09.378651+0000 mgr.y (mgr.14150) 83 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:11.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:10 smithi136 bash[32189]: cluster 2024-09-12T23:26:09.814120+0000 mon.a (mon.0) 239 : cluster [DBG] mgrmap e14: y(active, since 104s), standbys: x 2024-09-12T23:26:11.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:10 smithi136 bash[32189]: audit 2024-09-12T23:26:09.814630+0000 mon.a (mon.0) 240 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-12T23:26:11.267 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi136:vg_nvme/lv_4 2024-09-12T23:26:12.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:11 smithi181 bash[24776]: audit 2024-09-12T23:26:10.813689+0000 mon.a (mon.0) 241 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:12.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:11 smithi181 bash[24776]: audit 2024-09-12T23:26:10.822827+0000 mon.a (mon.0) 242 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:12.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:11 smithi181 bash[24776]: audit 2024-09-12T23:26:10.824518+0000 mon.a (mon.0) 243 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:26:12.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:11 smithi181 bash[24776]: audit 2024-09-12T23:26:10.826342+0000 mon.a (mon.0) 244 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:26:12.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:11 smithi181 bash[24776]: audit 2024-09-12T23:26:10.827164+0000 mon.a (mon.0) 245 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:26:12.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:11 smithi181 bash[24776]: audit 2024-09-12T23:26:10.836371+0000 mon.a (mon.0) 246 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:12.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:11 smithi136 bash[22601]: audit 2024-09-12T23:26:10.813689+0000 mon.a (mon.0) 241 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:12.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:11 smithi136 bash[22601]: audit 2024-09-12T23:26:10.822827+0000 mon.a (mon.0) 242 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:12.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:11 smithi136 bash[22601]: audit 2024-09-12T23:26:10.824518+0000 mon.a (mon.0) 243 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:26:12.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:11 smithi136 bash[22601]: audit 2024-09-12T23:26:10.826342+0000 mon.a (mon.0) 244 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:26:12.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:11 smithi136 bash[22601]: audit 2024-09-12T23:26:10.827164+0000 mon.a (mon.0) 245 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:26:12.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:11 smithi136 bash[22601]: audit 2024-09-12T23:26:10.836371+0000 mon.a (mon.0) 246 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:12.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:11 smithi136 bash[32189]: audit 2024-09-12T23:26:10.813689+0000 mon.a (mon.0) 241 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:12.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:11 smithi136 bash[32189]: audit 2024-09-12T23:26:10.822827+0000 mon.a (mon.0) 242 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:12.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:11 smithi136 bash[32189]: audit 2024-09-12T23:26:10.824518+0000 mon.a (mon.0) 243 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:26:12.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:11 smithi136 bash[32189]: audit 2024-09-12T23:26:10.826342+0000 mon.a (mon.0) 244 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:26:12.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:11 smithi136 bash[32189]: audit 2024-09-12T23:26:10.827164+0000 mon.a (mon.0) 245 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:26:12.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:11 smithi136 bash[32189]: audit 2024-09-12T23:26:10.836371+0000 mon.a (mon.0) 246 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:13.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:12 smithi181 bash[24776]: cluster 2024-09-12T23:26:11.379049+0000 mgr.y (mgr.14150) 84 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:13.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:12 smithi136 bash[22601]: cluster 2024-09-12T23:26:11.379049+0000 mgr.y (mgr.14150) 84 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:13.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:12 smithi136 bash[32189]: cluster 2024-09-12T23:26:11.379049+0000 mgr.y (mgr.14150) 84 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:15.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:14 smithi181 bash[24776]: cluster 2024-09-12T23:26:13.379497+0000 mgr.y (mgr.14150) 85 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:15.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:14 smithi136 bash[22601]: cluster 2024-09-12T23:26:13.379497+0000 mgr.y (mgr.14150) 85 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:15.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:14 smithi136 bash[32189]: cluster 2024-09-12T23:26:13.379497+0000 mgr.y (mgr.14150) 85 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:15.988 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:26:17.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:16 smithi181 bash[24776]: cluster 2024-09-12T23:26:15.379906+0000 mgr.y (mgr.14150) 86 : cluster [DBG] pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:17.209 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:16 smithi136 bash[22601]: cluster 2024-09-12T23:26:15.379906+0000 mgr.y (mgr.14150) 86 : cluster [DBG] pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:17.210 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:16 smithi136 bash[32189]: cluster 2024-09-12T23:26:15.379906+0000 mgr.y (mgr.14150) 86 : cluster [DBG] pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:19.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:18 smithi181 bash[24776]: cluster 2024-09-12T23:26:17.380243+0000 mgr.y (mgr.14150) 87 : cluster [DBG] pgmap v50: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:19.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:18 smithi181 bash[24776]: audit 2024-09-12T23:26:18.831505+0000 mon.a (mon.0) 247 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-12T23:26:19.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:18 smithi181 bash[24776]: audit 2024-09-12T23:26:18.837067+0000 mon.a (mon.0) 248 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-12T23:26:19.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:18 smithi181 bash[24776]: audit 2024-09-12T23:26:18.838434+0000 mon.a (mon.0) 249 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:26:19.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:18 smithi136 bash[22601]: cluster 2024-09-12T23:26:17.380243+0000 mgr.y (mgr.14150) 87 : cluster [DBG] pgmap v50: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:19.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:18 smithi136 bash[22601]: audit 2024-09-12T23:26:18.831505+0000 mon.a (mon.0) 247 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-12T23:26:19.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:18 smithi136 bash[22601]: audit 2024-09-12T23:26:18.837067+0000 mon.a (mon.0) 248 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-12T23:26:19.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:18 smithi136 bash[22601]: audit 2024-09-12T23:26:18.838434+0000 mon.a (mon.0) 249 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:26:19.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:18 smithi136 bash[32189]: cluster 2024-09-12T23:26:17.380243+0000 mgr.y (mgr.14150) 87 : cluster [DBG] pgmap v50: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:19.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:18 smithi136 bash[32189]: audit 2024-09-12T23:26:18.831505+0000 mon.a (mon.0) 247 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-12T23:26:19.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:18 smithi136 bash[32189]: audit 2024-09-12T23:26:18.837067+0000 mon.a (mon.0) 248 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-12T23:26:19.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:18 smithi136 bash[32189]: audit 2024-09-12T23:26:18.838434+0000 mon.a (mon.0) 249 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:26:20.267 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:19 smithi136 bash[22601]: audit 2024-09-12T23:26:18.828483+0000 mgr.y (mgr.14150) 88 : audit [DBG] from='client.14205 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi136:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:26:20.268 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:19 smithi136 bash[32189]: audit 2024-09-12T23:26:18.828483+0000 mgr.y (mgr.14150) 88 : audit [DBG] from='client.14205 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi136:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:26:20.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:19 smithi181 bash[24776]: audit 2024-09-12T23:26:18.828483+0000 mgr.y (mgr.14150) 88 : audit [DBG] from='client.14205 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi136:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:26:21.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:20 smithi136 bash[22601]: cluster 2024-09-12T23:26:19.380626+0000 mgr.y (mgr.14150) 89 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:21.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:20 smithi136 bash[32189]: cluster 2024-09-12T23:26:19.380626+0000 mgr.y (mgr.14150) 89 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:21.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:20 smithi181 bash[24776]: cluster 2024-09-12T23:26:19.380626+0000 mgr.y (mgr.14150) 89 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:23.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:22 smithi136 bash[22601]: cluster 2024-09-12T23:26:21.381072+0000 mgr.y (mgr.14150) 90 : cluster [DBG] pgmap v52: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:23.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:22 smithi136 bash[32189]: cluster 2024-09-12T23:26:21.381072+0000 mgr.y (mgr.14150) 90 : cluster [DBG] pgmap v52: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:23.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:22 smithi181 bash[24776]: cluster 2024-09-12T23:26:21.381072+0000 mgr.y (mgr.14150) 90 : cluster [DBG] pgmap v52: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:25.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:24 smithi136 bash[22601]: cluster 2024-09-12T23:26:23.381486+0000 mgr.y (mgr.14150) 91 : cluster [DBG] pgmap v53: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:25.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:24 smithi136 bash[32189]: cluster 2024-09-12T23:26:23.381486+0000 mgr.y (mgr.14150) 91 : cluster [DBG] pgmap v53: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:25.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:24 smithi181 bash[24776]: cluster 2024-09-12T23:26:23.381486+0000 mgr.y (mgr.14150) 91 : cluster [DBG] pgmap v53: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:27.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:26 smithi136 bash[22601]: cluster 2024-09-12T23:26:25.381794+0000 mgr.y (mgr.14150) 92 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:27.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:26 smithi136 bash[32189]: cluster 2024-09-12T23:26:25.381794+0000 mgr.y (mgr.14150) 92 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:27.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:26 smithi181 bash[24776]: cluster 2024-09-12T23:26:25.381794+0000 mgr.y (mgr.14150) 92 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:28.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:27 smithi136 bash[22601]: audit 2024-09-12T23:26:26.950077+0000 mon.c (mon.2) 2 : audit [INF] from='client.? 172.21.15.136:0/2402066045' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "87aedf04-695e-4a8f-80c3-ffd18883a4d5"}]: dispatch 2024-09-12T23:26:28.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:27 smithi136 bash[22601]: audit 2024-09-12T23:26:26.950785+0000 mon.a (mon.0) 250 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "87aedf04-695e-4a8f-80c3-ffd18883a4d5"}]: dispatch 2024-09-12T23:26:28.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:27 smithi136 bash[22601]: audit 2024-09-12T23:26:26.961606+0000 mon.a (mon.0) 251 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "87aedf04-695e-4a8f-80c3-ffd18883a4d5"}]': finished 2024-09-12T23:26:28.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:27 smithi136 bash[22601]: cluster 2024-09-12T23:26:26.961718+0000 mon.a (mon.0) 252 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2024-09-12T23:26:28.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:27 smithi136 bash[22601]: audit 2024-09-12T23:26:26.961846+0000 mon.a (mon.0) 253 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-12T23:26:28.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:27 smithi136 bash[22601]: audit 2024-09-12T23:26:27.586796+0000 mon.a (mon.0) 254 : audit [DBG] from='client.? 172.21.15.136:0/2336966765' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-12T23:26:28.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:27 smithi136 bash[32189]: audit 2024-09-12T23:26:26.950077+0000 mon.c (mon.2) 2 : audit [INF] from='client.? 172.21.15.136:0/2402066045' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "87aedf04-695e-4a8f-80c3-ffd18883a4d5"}]: dispatch 2024-09-12T23:26:28.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:27 smithi136 bash[32189]: audit 2024-09-12T23:26:26.950785+0000 mon.a (mon.0) 250 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "87aedf04-695e-4a8f-80c3-ffd18883a4d5"}]: dispatch 2024-09-12T23:26:28.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:27 smithi136 bash[32189]: audit 2024-09-12T23:26:26.961606+0000 mon.a (mon.0) 251 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "87aedf04-695e-4a8f-80c3-ffd18883a4d5"}]': finished 2024-09-12T23:26:28.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:27 smithi136 bash[32189]: cluster 2024-09-12T23:26:26.961718+0000 mon.a (mon.0) 252 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2024-09-12T23:26:28.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:27 smithi136 bash[32189]: audit 2024-09-12T23:26:26.961846+0000 mon.a (mon.0) 253 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-12T23:26:28.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:27 smithi136 bash[32189]: audit 2024-09-12T23:26:27.586796+0000 mon.a (mon.0) 254 : audit [DBG] from='client.? 172.21.15.136:0/2336966765' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-12T23:26:28.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:27 smithi181 bash[24776]: audit 2024-09-12T23:26:26.950077+0000 mon.c (mon.2) 2 : audit [INF] from='client.? 172.21.15.136:0/2402066045' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "87aedf04-695e-4a8f-80c3-ffd18883a4d5"}]: dispatch 2024-09-12T23:26:28.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:27 smithi181 bash[24776]: audit 2024-09-12T23:26:26.950785+0000 mon.a (mon.0) 250 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "87aedf04-695e-4a8f-80c3-ffd18883a4d5"}]: dispatch 2024-09-12T23:26:28.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:27 smithi181 bash[24776]: audit 2024-09-12T23:26:26.961606+0000 mon.a (mon.0) 251 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "87aedf04-695e-4a8f-80c3-ffd18883a4d5"}]': finished 2024-09-12T23:26:28.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:27 smithi181 bash[24776]: cluster 2024-09-12T23:26:26.961718+0000 mon.a (mon.0) 252 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2024-09-12T23:26:28.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:27 smithi181 bash[24776]: audit 2024-09-12T23:26:26.961846+0000 mon.a (mon.0) 253 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-12T23:26:28.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:27 smithi181 bash[24776]: audit 2024-09-12T23:26:27.586796+0000 mon.a (mon.0) 254 : audit [DBG] from='client.? 172.21.15.136:0/2336966765' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-12T23:26:29.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:28 smithi136 bash[22601]: cluster 2024-09-12T23:26:27.382054+0000 mgr.y (mgr.14150) 93 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:29.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:28 smithi136 bash[32189]: cluster 2024-09-12T23:26:27.382054+0000 mgr.y (mgr.14150) 93 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:29.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:28 smithi181 bash[24776]: cluster 2024-09-12T23:26:27.382054+0000 mgr.y (mgr.14150) 93 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:31.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:30 smithi136 bash[22601]: cluster 2024-09-12T23:26:29.382453+0000 mgr.y (mgr.14150) 94 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:31.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:30 smithi136 bash[32189]: cluster 2024-09-12T23:26:29.382453+0000 mgr.y (mgr.14150) 94 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:31.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:30 smithi181 bash[24776]: cluster 2024-09-12T23:26:29.382453+0000 mgr.y (mgr.14150) 94 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:33.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:32 smithi136 bash[22601]: cluster 2024-09-12T23:26:31.382916+0000 mgr.y (mgr.14150) 95 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:33.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:32 smithi136 bash[32189]: cluster 2024-09-12T23:26:31.382916+0000 mgr.y (mgr.14150) 95 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:33.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:32 smithi181 bash[24776]: cluster 2024-09-12T23:26:31.382916+0000 mgr.y (mgr.14150) 95 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:35.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:34 smithi136 bash[22601]: cluster 2024-09-12T23:26:33.383344+0000 mgr.y (mgr.14150) 96 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:35.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:34 smithi136 bash[32189]: cluster 2024-09-12T23:26:33.383344+0000 mgr.y (mgr.14150) 96 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:35.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:34 smithi181 bash[24776]: cluster 2024-09-12T23:26:33.383344+0000 mgr.y (mgr.14150) 96 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:37.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:36 smithi136 bash[22601]: cluster 2024-09-12T23:26:35.383704+0000 mgr.y (mgr.14150) 97 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:37.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:36 smithi136 bash[32189]: cluster 2024-09-12T23:26:35.383704+0000 mgr.y (mgr.14150) 97 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:37.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:36 smithi181 bash[24776]: cluster 2024-09-12T23:26:35.383704+0000 mgr.y (mgr.14150) 97 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:39.244 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:38 smithi136 bash[32189]: cluster 2024-09-12T23:26:37.384204+0000 mgr.y (mgr.14150) 98 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:39.245 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:38 smithi136 bash[22601]: cluster 2024-09-12T23:26:37.384204+0000 mgr.y (mgr.14150) 98 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:39.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:38 smithi181 bash[24776]: cluster 2024-09-12T23:26:37.384204+0000 mgr.y (mgr.14150) 98 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:40.165 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:39 smithi136 bash[32189]: audit 2024-09-12T23:26:38.930792+0000 mon.a (mon.0) 255 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-12T23:26:40.165 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:39 smithi136 bash[32189]: audit 2024-09-12T23:26:38.932172+0000 mon.a (mon.0) 256 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:26:40.165 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:39 smithi136 bash[32189]: cephadm 2024-09-12T23:26:38.933267+0000 mgr.y (mgr.14150) 99 : cephadm [INF] Deploying daemon osd.0 on smithi136 2024-09-12T23:26:40.165 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:39 smithi136 bash[32189]: cluster 2024-09-12T23:26:39.384780+0000 mgr.y (mgr.14150) 100 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:40.165 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:39 smithi136 bash[22601]: audit 2024-09-12T23:26:38.930792+0000 mon.a (mon.0) 255 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-12T23:26:40.165 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:39 smithi136 bash[22601]: audit 2024-09-12T23:26:38.932172+0000 mon.a (mon.0) 256 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:26:40.166 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:39 smithi136 bash[22601]: cephadm 2024-09-12T23:26:38.933267+0000 mgr.y (mgr.14150) 99 : cephadm [INF] Deploying daemon osd.0 on smithi136 2024-09-12T23:26:40.166 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:39 smithi136 bash[22601]: cluster 2024-09-12T23:26:39.384780+0000 mgr.y (mgr.14150) 100 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:40.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:39 smithi181 bash[24776]: audit 2024-09-12T23:26:38.930792+0000 mon.a (mon.0) 255 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-12T23:26:40.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:39 smithi181 bash[24776]: audit 2024-09-12T23:26:38.932172+0000 mon.a (mon.0) 256 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:26:40.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:39 smithi181 bash[24776]: cephadm 2024-09-12T23:26:38.933267+0000 mgr.y (mgr.14150) 99 : cephadm [INF] Deploying daemon osd.0 on smithi136 2024-09-12T23:26:40.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:39 smithi181 bash[24776]: cluster 2024-09-12T23:26:39.384780+0000 mgr.y (mgr.14150) 100 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:42.762 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:42 smithi136 bash[22601]: cluster 2024-09-12T23:26:41.385165+0000 mgr.y (mgr.14150) 101 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:42.762 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:42 smithi136 bash[32189]: cluster 2024-09-12T23:26:41.385165+0000 mgr.y (mgr.14150) 101 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:42.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:42 smithi181 bash[24776]: cluster 2024-09-12T23:26:41.385165+0000 mgr.y (mgr.14150) 101 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:43.124 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:42 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:26:43.124 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:43 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:26:43.124 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:26:42 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:26:43.125 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:26:43 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:26:43.125 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:42 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:26:43.125 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:43 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:26:43.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:43 smithi136 bash[32189]: audit 2024-09-12T23:26:43.160083+0000 mon.a (mon.0) 257 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:26:43.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:43 smithi136 bash[32189]: audit 2024-09-12T23:26:43.164912+0000 mon.a (mon.0) 258 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:43.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:43 smithi136 bash[32189]: audit 2024-09-12T23:26:43.171323+0000 mon.a (mon.0) 259 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:43.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:43 smithi136 bash[22601]: audit 2024-09-12T23:26:43.160083+0000 mon.a (mon.0) 257 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:26:43.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:43 smithi136 bash[22601]: audit 2024-09-12T23:26:43.164912+0000 mon.a (mon.0) 258 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:43.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:43 smithi136 bash[22601]: audit 2024-09-12T23:26:43.171323+0000 mon.a (mon.0) 259 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:43.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:43 smithi181 bash[24776]: audit 2024-09-12T23:26:43.160083+0000 mon.a (mon.0) 257 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:26:43.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:43 smithi181 bash[24776]: audit 2024-09-12T23:26:43.164912+0000 mon.a (mon.0) 258 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:43.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:43 smithi181 bash[24776]: audit 2024-09-12T23:26:43.171323+0000 mon.a (mon.0) 259 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:44.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:44 smithi136 bash[32189]: cluster 2024-09-12T23:26:43.385416+0000 mgr.y (mgr.14150) 102 : cluster [DBG] pgmap v64: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:44.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:44 smithi136 bash[22601]: cluster 2024-09-12T23:26:43.385416+0000 mgr.y (mgr.14150) 102 : cluster [DBG] pgmap v64: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:44.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:44 smithi181 bash[24776]: cluster 2024-09-12T23:26:43.385416+0000 mgr.y (mgr.14150) 102 : cluster [DBG] pgmap v64: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:46.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:46 smithi136 bash[32189]: cluster 2024-09-12T23:26:45.385787+0000 mgr.y (mgr.14150) 103 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:46.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:46 smithi136 bash[22601]: cluster 2024-09-12T23:26:45.385787+0000 mgr.y (mgr.14150) 103 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:46.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:46 smithi181 bash[24776]: cluster 2024-09-12T23:26:45.385787+0000 mgr.y (mgr.14150) 103 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:48.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:48 smithi136 bash[22601]: cluster 2024-09-12T23:26:47.386247+0000 mgr.y (mgr.14150) 104 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:48.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:48 smithi136 bash[22601]: audit 2024-09-12T23:26:48.185224+0000 mon.a (mon.0) 260 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:48.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:48 smithi136 bash[22601]: audit 2024-09-12T23:26:48.189138+0000 mon.a (mon.0) 261 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:48.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:48 smithi136 bash[32189]: cluster 2024-09-12T23:26:47.386247+0000 mgr.y (mgr.14150) 104 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:48.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:48 smithi136 bash[32189]: audit 2024-09-12T23:26:48.185224+0000 mon.a (mon.0) 260 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:48.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:48 smithi136 bash[32189]: audit 2024-09-12T23:26:48.189138+0000 mon.a (mon.0) 261 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:48.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:48 smithi181 bash[24776]: cluster 2024-09-12T23:26:47.386247+0000 mgr.y (mgr.14150) 104 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:48.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:48 smithi181 bash[24776]: audit 2024-09-12T23:26:48.185224+0000 mon.a (mon.0) 260 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:48.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:48 smithi181 bash[24776]: audit 2024-09-12T23:26:48.189138+0000 mon.a (mon.0) 261 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:49.454 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:49 smithi136 bash[22601]: audit 2024-09-12T23:26:48.560117+0000 mon.a (mon.0) 262 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:26:49.454 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:49 smithi136 bash[22601]: audit 2024-09-12T23:26:48.561416+0000 mon.a (mon.0) 263 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:26:49.454 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:49 smithi136 bash[22601]: audit 2024-09-12T23:26:48.568039+0000 mon.a (mon.0) 264 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:49.728 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:49 smithi136 bash[32189]: audit 2024-09-12T23:26:48.560117+0000 mon.a (mon.0) 262 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:26:49.729 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:49 smithi136 bash[32189]: audit 2024-09-12T23:26:48.561416+0000 mon.a (mon.0) 263 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:26:49.729 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:49 smithi136 bash[32189]: audit 2024-09-12T23:26:48.568039+0000 mon.a (mon.0) 264 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:49.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:49 smithi181 bash[24776]: audit 2024-09-12T23:26:48.560117+0000 mon.a (mon.0) 262 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:26:49.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:49 smithi181 bash[24776]: audit 2024-09-12T23:26:48.561416+0000 mon.a (mon.0) 263 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:26:49.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:49 smithi181 bash[24776]: audit 2024-09-12T23:26:48.568039+0000 mon.a (mon.0) 264 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:50.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:50 smithi181 bash[24776]: cluster 2024-09-12T23:26:49.386668+0000 mgr.y (mgr.14150) 105 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:50.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:50 smithi181 bash[24776]: audit 2024-09-12T23:26:49.617331+0000 mon.c (mon.2) 3 : audit [INF] from='osd.0 [v2:172.21.15.136:6802/4251571952,v1:172.21.15.136:6803/4251571952]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-12T23:26:50.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:50 smithi181 bash[24776]: audit 2024-09-12T23:26:49.617708+0000 mon.a (mon.0) 265 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-12T23:26:50.974 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:50 smithi136 bash[22601]: cluster 2024-09-12T23:26:49.386668+0000 mgr.y (mgr.14150) 105 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:50.974 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:50 smithi136 bash[22601]: audit 2024-09-12T23:26:49.617331+0000 mon.c (mon.2) 3 : audit [INF] from='osd.0 [v2:172.21.15.136:6802/4251571952,v1:172.21.15.136:6803/4251571952]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-12T23:26:50.974 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:50 smithi136 bash[22601]: audit 2024-09-12T23:26:49.617708+0000 mon.a (mon.0) 265 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-12T23:26:50.975 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:50 smithi136 bash[32189]: cluster 2024-09-12T23:26:49.386668+0000 mgr.y (mgr.14150) 105 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:50.975 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:50 smithi136 bash[32189]: audit 2024-09-12T23:26:49.617331+0000 mon.c (mon.2) 3 : audit [INF] from='osd.0 [v2:172.21.15.136:6802/4251571952,v1:172.21.15.136:6803/4251571952]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-12T23:26:50.975 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:50 smithi136 bash[32189]: audit 2024-09-12T23:26:49.617708+0000 mon.a (mon.0) 265 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-12T23:26:51.604 INFO:teuthology.orchestra.run.smithi136.stdout:Created osd(s) 0 on host 'smithi136' 2024-09-12T23:26:51.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:51 smithi181 bash[24776]: audit 2024-09-12T23:26:50.580206+0000 mon.a (mon.0) 266 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-12T23:26:51.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:51 smithi181 bash[24776]: cluster 2024-09-12T23:26:50.580266+0000 mon.a (mon.0) 267 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2024-09-12T23:26:51.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:51 smithi181 bash[24776]: audit 2024-09-12T23:26:50.580411+0000 mon.a (mon.0) 268 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-12T23:26:51.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:51 smithi181 bash[24776]: audit 2024-09-12T23:26:50.581701+0000 mon.c (mon.2) 4 : audit [INF] from='osd.0 [v2:172.21.15.136:6802/4251571952,v1:172.21.15.136:6803/4251571952]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-12T23:26:51.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:51 smithi181 bash[24776]: audit 2024-09-12T23:26:50.582086+0000 mon.a (mon.0) 269 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-12T23:26:52.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:51 smithi136 bash[22601]: audit 2024-09-12T23:26:50.580206+0000 mon.a (mon.0) 266 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-12T23:26:52.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:51 smithi136 bash[22601]: cluster 2024-09-12T23:26:50.580266+0000 mon.a (mon.0) 267 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2024-09-12T23:26:52.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:51 smithi136 bash[22601]: audit 2024-09-12T23:26:50.580411+0000 mon.a (mon.0) 268 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-12T23:26:52.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:51 smithi136 bash[22601]: audit 2024-09-12T23:26:50.581701+0000 mon.c (mon.2) 4 : audit [INF] from='osd.0 [v2:172.21.15.136:6802/4251571952,v1:172.21.15.136:6803/4251571952]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-12T23:26:52.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:51 smithi136 bash[22601]: audit 2024-09-12T23:26:50.582086+0000 mon.a (mon.0) 269 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-12T23:26:52.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:51 smithi136 bash[32189]: audit 2024-09-12T23:26:50.580206+0000 mon.a (mon.0) 266 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-12T23:26:52.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:51 smithi136 bash[32189]: cluster 2024-09-12T23:26:50.580266+0000 mon.a (mon.0) 267 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2024-09-12T23:26:52.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:51 smithi136 bash[32189]: audit 2024-09-12T23:26:50.580411+0000 mon.a (mon.0) 268 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-12T23:26:52.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:51 smithi136 bash[32189]: audit 2024-09-12T23:26:50.581701+0000 mon.c (mon.2) 4 : audit [INF] from='osd.0 [v2:172.21.15.136:6802/4251571952,v1:172.21.15.136:6803/4251571952]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-12T23:26:52.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:51 smithi136 bash[32189]: audit 2024-09-12T23:26:50.582086+0000 mon.a (mon.0) 269 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-12T23:26:52.187 DEBUG:teuthology.orchestra.run.smithi136:osd.0> sudo journalctl -f -n 0 -u ceph-da58a47c-715d-11ef-bceb-c7b262605968@osd.0.service 2024-09-12T23:26:52.190 INFO:tasks.cephadm:Deploying osd.1 on smithi136 with /dev/vg_nvme/lv_3... 2024-09-12T23:26:52.190 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-09-12T23:26:52.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:52 smithi181 bash[24776]: cluster 2024-09-12T23:26:51.387106+0000 mgr.y (mgr.14150) 106 : cluster [DBG] pgmap v69: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:52.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:52 smithi181 bash[24776]: audit 2024-09-12T23:26:51.576973+0000 mon.a (mon.0) 270 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:26:52.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:52 smithi181 bash[24776]: audit 2024-09-12T23:26:51.586410+0000 mon.a (mon.0) 271 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]': finished 2024-09-12T23:26:52.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:52 smithi181 bash[24776]: cluster 2024-09-12T23:26:51.586602+0000 mon.a (mon.0) 272 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2024-09-12T23:26:52.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:52 smithi181 bash[24776]: audit 2024-09-12T23:26:51.587347+0000 mon.a (mon.0) 273 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-12T23:26:52.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:52 smithi181 bash[24776]: audit 2024-09-12T23:26:51.593841+0000 mon.a (mon.0) 274 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:52.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:52 smithi181 bash[24776]: audit 2024-09-12T23:26:51.594959+0000 mon.a (mon.0) 275 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-12T23:26:52.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:52 smithi181 bash[24776]: audit 2024-09-12T23:26:51.600882+0000 mon.a (mon.0) 276 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:53.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:52 smithi136 bash[22601]: cluster 2024-09-12T23:26:51.387106+0000 mgr.y (mgr.14150) 106 : cluster [DBG] pgmap v69: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:53.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:52 smithi136 bash[22601]: audit 2024-09-12T23:26:51.576973+0000 mon.a (mon.0) 270 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:26:53.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:52 smithi136 bash[22601]: audit 2024-09-12T23:26:51.586410+0000 mon.a (mon.0) 271 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]': finished 2024-09-12T23:26:53.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:52 smithi136 bash[22601]: cluster 2024-09-12T23:26:51.586602+0000 mon.a (mon.0) 272 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2024-09-12T23:26:53.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:52 smithi136 bash[22601]: audit 2024-09-12T23:26:51.587347+0000 mon.a (mon.0) 273 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-12T23:26:53.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:52 smithi136 bash[22601]: audit 2024-09-12T23:26:51.593841+0000 mon.a (mon.0) 274 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:53.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:52 smithi136 bash[22601]: audit 2024-09-12T23:26:51.594959+0000 mon.a (mon.0) 275 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-12T23:26:53.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:52 smithi136 bash[22601]: audit 2024-09-12T23:26:51.600882+0000 mon.a (mon.0) 276 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:53.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:52 smithi136 bash[32189]: cluster 2024-09-12T23:26:51.387106+0000 mgr.y (mgr.14150) 106 : cluster [DBG] pgmap v69: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:53.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:52 smithi136 bash[32189]: audit 2024-09-12T23:26:51.576973+0000 mon.a (mon.0) 270 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:26:53.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:52 smithi136 bash[32189]: audit 2024-09-12T23:26:51.586410+0000 mon.a (mon.0) 271 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]': finished 2024-09-12T23:26:53.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:52 smithi136 bash[32189]: cluster 2024-09-12T23:26:51.586602+0000 mon.a (mon.0) 272 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2024-09-12T23:26:53.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:52 smithi136 bash[32189]: audit 2024-09-12T23:26:51.587347+0000 mon.a (mon.0) 273 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-12T23:26:53.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:52 smithi136 bash[32189]: audit 2024-09-12T23:26:51.593841+0000 mon.a (mon.0) 274 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:53.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:52 smithi136 bash[32189]: audit 2024-09-12T23:26:51.594959+0000 mon.a (mon.0) 275 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-12T23:26:53.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:52 smithi136 bash[32189]: audit 2024-09-12T23:26:51.600882+0000 mon.a (mon.0) 276 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:26:53.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:53 smithi181 bash[24776]: cluster 2024-09-12T23:26:50.657503+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-12T23:26:53.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:53 smithi181 bash[24776]: cluster 2024-09-12T23:26:50.657577+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-12T23:26:53.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:53 smithi181 bash[24776]: audit 2024-09-12T23:26:52.598530+0000 mon.a (mon.0) 277 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-12T23:26:53.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:53 smithi181 bash[24776]: cluster 2024-09-12T23:26:52.609173+0000 mon.a (mon.0) 278 : cluster [INF] osd.0 [v2:172.21.15.136:6802/4251571952,v1:172.21.15.136:6803/4251571952] boot 2024-09-12T23:26:53.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:53 smithi181 bash[24776]: cluster 2024-09-12T23:26:52.609279+0000 mon.a (mon.0) 279 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2024-09-12T23:26:53.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:53 smithi181 bash[24776]: audit 2024-09-12T23:26:52.609526+0000 mon.a (mon.0) 280 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-12T23:26:54.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:53 smithi136 bash[22601]: cluster 2024-09-12T23:26:50.657503+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-12T23:26:54.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:53 smithi136 bash[22601]: cluster 2024-09-12T23:26:50.657577+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-12T23:26:54.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:53 smithi136 bash[22601]: audit 2024-09-12T23:26:52.598530+0000 mon.a (mon.0) 277 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-12T23:26:54.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:53 smithi136 bash[22601]: cluster 2024-09-12T23:26:52.609173+0000 mon.a (mon.0) 278 : cluster [INF] osd.0 [v2:172.21.15.136:6802/4251571952,v1:172.21.15.136:6803/4251571952] boot 2024-09-12T23:26:54.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:53 smithi136 bash[22601]: cluster 2024-09-12T23:26:52.609279+0000 mon.a (mon.0) 279 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2024-09-12T23:26:54.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:53 smithi136 bash[22601]: audit 2024-09-12T23:26:52.609526+0000 mon.a (mon.0) 280 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-12T23:26:54.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:53 smithi136 bash[32189]: cluster 2024-09-12T23:26:50.657503+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-12T23:26:54.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:53 smithi136 bash[32189]: cluster 2024-09-12T23:26:50.657577+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-12T23:26:54.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:53 smithi136 bash[32189]: audit 2024-09-12T23:26:52.598530+0000 mon.a (mon.0) 277 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-12T23:26:54.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:53 smithi136 bash[32189]: cluster 2024-09-12T23:26:52.609173+0000 mon.a (mon.0) 278 : cluster [INF] osd.0 [v2:172.21.15.136:6802/4251571952,v1:172.21.15.136:6803/4251571952] boot 2024-09-12T23:26:54.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:53 smithi136 bash[32189]: cluster 2024-09-12T23:26:52.609279+0000 mon.a (mon.0) 279 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2024-09-12T23:26:54.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:53 smithi136 bash[32189]: audit 2024-09-12T23:26:52.609526+0000 mon.a (mon.0) 280 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-12T23:26:55.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:54 smithi136 bash[22601]: cluster 2024-09-12T23:26:53.387564+0000 mgr.y (mgr.14150) 107 : cluster [DBG] pgmap v72: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:55.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:54 smithi136 bash[22601]: cluster 2024-09-12T23:26:53.621897+0000 mon.a (mon.0) 281 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2024-09-12T23:26:55.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:54 smithi136 bash[32189]: cluster 2024-09-12T23:26:53.387564+0000 mgr.y (mgr.14150) 107 : cluster [DBG] pgmap v72: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:55.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:54 smithi136 bash[32189]: cluster 2024-09-12T23:26:53.621897+0000 mon.a (mon.0) 281 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2024-09-12T23:26:55.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:54 smithi181 bash[24776]: cluster 2024-09-12T23:26:53.387564+0000 mgr.y (mgr.14150) 107 : cluster [DBG] pgmap v72: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-12T23:26:55.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:54 smithi181 bash[24776]: cluster 2024-09-12T23:26:53.621897+0000 mon.a (mon.0) 281 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2024-09-12T23:26:56.350 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:26:56.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:56 smithi136 bash[22601]: cluster 2024-09-12T23:26:55.388062+0000 mgr.y (mgr.14150) 108 : cluster [DBG] pgmap v74: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:26:56.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:56 smithi136 bash[32189]: cluster 2024-09-12T23:26:55.388062+0000 mgr.y (mgr.14150) 108 : cluster [DBG] pgmap v74: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:26:57.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:56 smithi181 bash[24776]: cluster 2024-09-12T23:26:55.388062+0000 mgr.y (mgr.14150) 108 : cluster [DBG] pgmap v74: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:26:58.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:26:58 smithi136 bash[22601]: cluster 2024-09-12T23:26:57.388526+0000 mgr.y (mgr.14150) 109 : cluster [DBG] pgmap v75: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:26:58.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:26:58 smithi136 bash[32189]: cluster 2024-09-12T23:26:57.388526+0000 mgr.y (mgr.14150) 109 : cluster [DBG] pgmap v75: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:26:59.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:26:58 smithi181 bash[24776]: cluster 2024-09-12T23:26:57.388526+0000 mgr.y (mgr.14150) 109 : cluster [DBG] pgmap v75: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:00.477 INFO:teuthology.orchestra.run.smithi136.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-09-12T23:27:00.647 INFO:teuthology.orchestra.run.smithi136.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-09-12T23:27:00.660 INFO:teuthology.orchestra.run.smithi136.stderr: stderr: 10+0 records in 2024-09-12T23:27:00.660 INFO:teuthology.orchestra.run.smithi136.stderr:10+0 records out 2024-09-12T23:27:00.660 INFO:teuthology.orchestra.run.smithi136.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0115214 s, 910 MB/s 2024-09-12T23:27:00.660 INFO:teuthology.orchestra.run.smithi136.stderr:--> Zapping successful for: 2024-09-12T23:27:00.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:00 smithi136 bash[32189]: cluster 2024-09-12T23:26:59.389009+0000 mgr.y (mgr.14150) 110 : cluster [DBG] pgmap v76: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:00.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:00 smithi136 bash[22601]: cluster 2024-09-12T23:26:59.389009+0000 mgr.y (mgr.14150) 110 : cluster [DBG] pgmap v76: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:01.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:00 smithi181 bash[24776]: cluster 2024-09-12T23:26:59.389009+0000 mgr.y (mgr.14150) 110 : cluster [DBG] pgmap v76: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:01.309 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi136:vg_nvme/lv_3 2024-09-12T23:27:03.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:02 smithi136 bash[22601]: cluster 2024-09-12T23:27:01.389322+0000 mgr.y (mgr.14150) 111 : cluster [DBG] pgmap v77: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:03.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:02 smithi136 bash[22601]: cephadm 2024-09-12T23:27:01.690942+0000 mgr.y (mgr.14150) 112 : cephadm [INF] Detected new or changed devices on smithi136 2024-09-12T23:27:03.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:02 smithi136 bash[22601]: audit 2024-09-12T23:27:01.699603+0000 mon.a (mon.0) 282 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:27:03.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:02 smithi136 bash[22601]: audit 2024-09-12T23:27:01.707986+0000 mon.a (mon.0) 283 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:27:03.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:02 smithi136 bash[22601]: audit 2024-09-12T23:27:01.710006+0000 mon.a (mon.0) 284 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:27:03.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:02 smithi136 bash[22601]: cephadm 2024-09-12T23:27:01.711376+0000 mgr.y (mgr.14150) 113 : cephadm [INF] Adjusting osd_memory_target on smithi136 to 16241M 2024-09-12T23:27:03.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:02 smithi136 bash[22601]: audit 2024-09-12T23:27:01.719632+0000 mon.a (mon.0) 285 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:27:03.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:02 smithi136 bash[22601]: audit 2024-09-12T23:27:01.721753+0000 mon.a (mon.0) 286 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:27:03.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:02 smithi136 bash[22601]: audit 2024-09-12T23:27:01.723343+0000 mon.a (mon.0) 287 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:27:03.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:02 smithi136 bash[22601]: audit 2024-09-12T23:27:01.732894+0000 mon.a (mon.0) 288 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:27:03.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:02 smithi136 bash[32189]: cluster 2024-09-12T23:27:01.389322+0000 mgr.y (mgr.14150) 111 : cluster [DBG] pgmap v77: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:03.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:02 smithi136 bash[32189]: cephadm 2024-09-12T23:27:01.690942+0000 mgr.y (mgr.14150) 112 : cephadm [INF] Detected new or changed devices on smithi136 2024-09-12T23:27:03.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:02 smithi136 bash[32189]: audit 2024-09-12T23:27:01.699603+0000 mon.a (mon.0) 282 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:27:03.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:02 smithi136 bash[32189]: audit 2024-09-12T23:27:01.707986+0000 mon.a (mon.0) 283 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:27:03.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:02 smithi136 bash[32189]: audit 2024-09-12T23:27:01.710006+0000 mon.a (mon.0) 284 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:27:03.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:02 smithi136 bash[32189]: cephadm 2024-09-12T23:27:01.711376+0000 mgr.y (mgr.14150) 113 : cephadm [INF] Adjusting osd_memory_target on smithi136 to 16241M 2024-09-12T23:27:03.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:02 smithi136 bash[32189]: audit 2024-09-12T23:27:01.719632+0000 mon.a (mon.0) 285 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:27:03.012 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:02 smithi136 bash[32189]: audit 2024-09-12T23:27:01.721753+0000 mon.a (mon.0) 286 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:27:03.012 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:02 smithi136 bash[32189]: audit 2024-09-12T23:27:01.723343+0000 mon.a (mon.0) 287 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:27:03.012 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:02 smithi136 bash[32189]: audit 2024-09-12T23:27:01.732894+0000 mon.a (mon.0) 288 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:27:03.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:02 smithi181 bash[24776]: cluster 2024-09-12T23:27:01.389322+0000 mgr.y (mgr.14150) 111 : cluster [DBG] pgmap v77: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:03.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:02 smithi181 bash[24776]: cephadm 2024-09-12T23:27:01.690942+0000 mgr.y (mgr.14150) 112 : cephadm [INF] Detected new or changed devices on smithi136 2024-09-12T23:27:03.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:02 smithi181 bash[24776]: audit 2024-09-12T23:27:01.699603+0000 mon.a (mon.0) 282 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:27:03.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:02 smithi181 bash[24776]: audit 2024-09-12T23:27:01.707986+0000 mon.a (mon.0) 283 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:27:03.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:02 smithi181 bash[24776]: audit 2024-09-12T23:27:01.710006+0000 mon.a (mon.0) 284 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:27:03.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:02 smithi181 bash[24776]: cephadm 2024-09-12T23:27:01.711376+0000 mgr.y (mgr.14150) 113 : cephadm [INF] Adjusting osd_memory_target on smithi136 to 16241M 2024-09-12T23:27:03.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:02 smithi181 bash[24776]: audit 2024-09-12T23:27:01.719632+0000 mon.a (mon.0) 285 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:27:03.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:02 smithi181 bash[24776]: audit 2024-09-12T23:27:01.721753+0000 mon.a (mon.0) 286 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:27:03.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:02 smithi181 bash[24776]: audit 2024-09-12T23:27:01.723343+0000 mon.a (mon.0) 287 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:27:03.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:02 smithi181 bash[24776]: audit 2024-09-12T23:27:01.732894+0000 mon.a (mon.0) 288 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:27:05.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:04 smithi136 bash[22601]: cluster 2024-09-12T23:27:03.389744+0000 mgr.y (mgr.14150) 114 : cluster [DBG] pgmap v78: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:05.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:04 smithi136 bash[32189]: cluster 2024-09-12T23:27:03.389744+0000 mgr.y (mgr.14150) 114 : cluster [DBG] pgmap v78: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:05.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:04 smithi181 bash[24776]: cluster 2024-09-12T23:27:03.389744+0000 mgr.y (mgr.14150) 114 : cluster [DBG] pgmap v78: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:06.042 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:27:06.990 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:06 smithi136 bash[22601]: cluster 2024-09-12T23:27:05.390266+0000 mgr.y (mgr.14150) 115 : cluster [DBG] pgmap v79: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:06.990 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:06 smithi136 bash[32189]: cluster 2024-09-12T23:27:05.390266+0000 mgr.y (mgr.14150) 115 : cluster [DBG] pgmap v79: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:07.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:06 smithi181 bash[24776]: cluster 2024-09-12T23:27:05.390266+0000 mgr.y (mgr.14150) 115 : cluster [DBG] pgmap v79: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:09.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:08 smithi136 bash[22601]: cluster 2024-09-12T23:27:07.390846+0000 mgr.y (mgr.14150) 116 : cluster [DBG] pgmap v80: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:09.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:08 smithi136 bash[32189]: cluster 2024-09-12T23:27:07.390846+0000 mgr.y (mgr.14150) 116 : cluster [DBG] pgmap v80: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:09.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:08 smithi181 bash[24776]: cluster 2024-09-12T23:27:07.390846+0000 mgr.y (mgr.14150) 116 : cluster [DBG] pgmap v80: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:10.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:09 smithi136 bash[22601]: audit 2024-09-12T23:27:08.868630+0000 mgr.y (mgr.14150) 117 : audit [DBG] from='client.14232 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi136:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:27:10.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:09 smithi136 bash[22601]: audit 2024-09-12T23:27:08.871246+0000 mon.a (mon.0) 289 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-12T23:27:10.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:09 smithi136 bash[22601]: audit 2024-09-12T23:27:08.876637+0000 mon.a (mon.0) 290 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-12T23:27:10.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:09 smithi136 bash[22601]: audit 2024-09-12T23:27:08.877898+0000 mon.a (mon.0) 291 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:27:10.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:09 smithi136 bash[32189]: audit 2024-09-12T23:27:08.868630+0000 mgr.y (mgr.14150) 117 : audit [DBG] from='client.14232 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi136:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:27:10.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:09 smithi136 bash[32189]: audit 2024-09-12T23:27:08.871246+0000 mon.a (mon.0) 289 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-12T23:27:10.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:09 smithi136 bash[32189]: audit 2024-09-12T23:27:08.876637+0000 mon.a (mon.0) 290 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-12T23:27:10.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:09 smithi136 bash[32189]: audit 2024-09-12T23:27:08.877898+0000 mon.a (mon.0) 291 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:27:10.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:09 smithi181 bash[24776]: audit 2024-09-12T23:27:08.868630+0000 mgr.y (mgr.14150) 117 : audit [DBG] from='client.14232 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi136:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:27:10.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:09 smithi181 bash[24776]: audit 2024-09-12T23:27:08.871246+0000 mon.a (mon.0) 289 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-12T23:27:10.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:09 smithi181 bash[24776]: audit 2024-09-12T23:27:08.876637+0000 mon.a (mon.0) 290 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-12T23:27:10.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:09 smithi181 bash[24776]: audit 2024-09-12T23:27:08.877898+0000 mon.a (mon.0) 291 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:27:11.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:10 smithi136 bash[22601]: cluster 2024-09-12T23:27:09.391410+0000 mgr.y (mgr.14150) 118 : cluster [DBG] pgmap v81: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:11.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:10 smithi136 bash[32189]: cluster 2024-09-12T23:27:09.391410+0000 mgr.y (mgr.14150) 118 : cluster [DBG] pgmap v81: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:11.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:10 smithi181 bash[24776]: cluster 2024-09-12T23:27:09.391410+0000 mgr.y (mgr.14150) 118 : cluster [DBG] pgmap v81: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:13.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:12 smithi136 bash[22601]: cluster 2024-09-12T23:27:11.391984+0000 mgr.y (mgr.14150) 119 : cluster [DBG] pgmap v82: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:13.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:12 smithi136 bash[32189]: cluster 2024-09-12T23:27:11.391984+0000 mgr.y (mgr.14150) 119 : cluster [DBG] pgmap v82: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:13.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:12 smithi181 bash[24776]: cluster 2024-09-12T23:27:11.391984+0000 mgr.y (mgr.14150) 119 : cluster [DBG] pgmap v82: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:14.914 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:14 smithi136 bash[22601]: cluster 2024-09-12T23:27:13.392485+0000 mgr.y (mgr.14150) 120 : cluster [DBG] pgmap v83: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:14.914 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:14 smithi136 bash[32189]: cluster 2024-09-12T23:27:13.392485+0000 mgr.y (mgr.14150) 120 : cluster [DBG] pgmap v83: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:15.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:14 smithi181 bash[24776]: cluster 2024-09-12T23:27:13.392485+0000 mgr.y (mgr.14150) 120 : cluster [DBG] pgmap v83: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:17.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:16 smithi136 bash[32189]: cluster 2024-09-12T23:27:15.392842+0000 mgr.y (mgr.14150) 121 : cluster [DBG] pgmap v84: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:17.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:16 smithi136 bash[22601]: cluster 2024-09-12T23:27:15.392842+0000 mgr.y (mgr.14150) 121 : cluster [DBG] pgmap v84: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:17.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:16 smithi181 bash[24776]: cluster 2024-09-12T23:27:15.392842+0000 mgr.y (mgr.14150) 121 : cluster [DBG] pgmap v84: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:18.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:17 smithi136 bash[22601]: audit 2024-09-12T23:27:16.962080+0000 mon.c (mon.2) 5 : audit [INF] from='client.? 172.21.15.136:0/2209341053' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "850d4f98-0640-4151-bd45-a351ca41d54e"}]: dispatch 2024-09-12T23:27:18.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:17 smithi136 bash[22601]: audit 2024-09-12T23:27:16.962621+0000 mon.a (mon.0) 292 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "850d4f98-0640-4151-bd45-a351ca41d54e"}]: dispatch 2024-09-12T23:27:18.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:17 smithi136 bash[22601]: audit 2024-09-12T23:27:16.967844+0000 mon.a (mon.0) 293 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "850d4f98-0640-4151-bd45-a351ca41d54e"}]': finished 2024-09-12T23:27:18.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:17 smithi136 bash[22601]: cluster 2024-09-12T23:27:16.967904+0000 mon.a (mon.0) 294 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2024-09-12T23:27:18.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:17 smithi136 bash[22601]: audit 2024-09-12T23:27:16.967987+0000 mon.a (mon.0) 295 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-12T23:27:18.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:17 smithi136 bash[22601]: audit 2024-09-12T23:27:17.591071+0000 mon.a (mon.0) 296 : audit [DBG] from='client.? 172.21.15.136:0/2254683302' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-12T23:27:18.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:17 smithi136 bash[32189]: audit 2024-09-12T23:27:16.962080+0000 mon.c (mon.2) 5 : audit [INF] from='client.? 172.21.15.136:0/2209341053' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "850d4f98-0640-4151-bd45-a351ca41d54e"}]: dispatch 2024-09-12T23:27:18.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:17 smithi136 bash[32189]: audit 2024-09-12T23:27:16.962621+0000 mon.a (mon.0) 292 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "850d4f98-0640-4151-bd45-a351ca41d54e"}]: dispatch 2024-09-12T23:27:18.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:17 smithi136 bash[32189]: audit 2024-09-12T23:27:16.967844+0000 mon.a (mon.0) 293 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "850d4f98-0640-4151-bd45-a351ca41d54e"}]': finished 2024-09-12T23:27:18.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:17 smithi136 bash[32189]: cluster 2024-09-12T23:27:16.967904+0000 mon.a (mon.0) 294 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2024-09-12T23:27:18.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:17 smithi136 bash[32189]: audit 2024-09-12T23:27:16.967987+0000 mon.a (mon.0) 295 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-12T23:27:18.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:17 smithi136 bash[32189]: audit 2024-09-12T23:27:17.591071+0000 mon.a (mon.0) 296 : audit [DBG] from='client.? 172.21.15.136:0/2254683302' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-12T23:27:18.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:17 smithi181 bash[24776]: audit 2024-09-12T23:27:16.962080+0000 mon.c (mon.2) 5 : audit [INF] from='client.? 172.21.15.136:0/2209341053' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "850d4f98-0640-4151-bd45-a351ca41d54e"}]: dispatch 2024-09-12T23:27:18.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:17 smithi181 bash[24776]: audit 2024-09-12T23:27:16.962621+0000 mon.a (mon.0) 292 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "850d4f98-0640-4151-bd45-a351ca41d54e"}]: dispatch 2024-09-12T23:27:18.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:17 smithi181 bash[24776]: audit 2024-09-12T23:27:16.967844+0000 mon.a (mon.0) 293 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "850d4f98-0640-4151-bd45-a351ca41d54e"}]': finished 2024-09-12T23:27:18.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:17 smithi181 bash[24776]: cluster 2024-09-12T23:27:16.967904+0000 mon.a (mon.0) 294 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2024-09-12T23:27:18.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:17 smithi181 bash[24776]: audit 2024-09-12T23:27:16.967987+0000 mon.a (mon.0) 295 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-12T23:27:18.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:17 smithi181 bash[24776]: audit 2024-09-12T23:27:17.591071+0000 mon.a (mon.0) 296 : audit [DBG] from='client.? 172.21.15.136:0/2254683302' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-12T23:27:19.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:18 smithi136 bash[22601]: cluster 2024-09-12T23:27:17.393113+0000 mgr.y (mgr.14150) 122 : cluster [DBG] pgmap v86: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:19.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:18 smithi136 bash[32189]: cluster 2024-09-12T23:27:17.393113+0000 mgr.y (mgr.14150) 122 : cluster [DBG] pgmap v86: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:19.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:18 smithi181 bash[24776]: cluster 2024-09-12T23:27:17.393113+0000 mgr.y (mgr.14150) 122 : cluster [DBG] pgmap v86: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:21.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:20 smithi136 bash[22601]: cluster 2024-09-12T23:27:19.393520+0000 mgr.y (mgr.14150) 123 : cluster [DBG] pgmap v87: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:21.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:20 smithi136 bash[32189]: cluster 2024-09-12T23:27:19.393520+0000 mgr.y (mgr.14150) 123 : cluster [DBG] pgmap v87: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:21.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:20 smithi181 bash[24776]: cluster 2024-09-12T23:27:19.393520+0000 mgr.y (mgr.14150) 123 : cluster [DBG] pgmap v87: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:23.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:22 smithi136 bash[22601]: cluster 2024-09-12T23:27:21.393971+0000 mgr.y (mgr.14150) 124 : cluster [DBG] pgmap v88: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:23.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:22 smithi136 bash[32189]: cluster 2024-09-12T23:27:21.393971+0000 mgr.y (mgr.14150) 124 : cluster [DBG] pgmap v88: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:23.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:22 smithi181 bash[24776]: cluster 2024-09-12T23:27:21.393971+0000 mgr.y (mgr.14150) 124 : cluster [DBG] pgmap v88: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:25.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:24 smithi136 bash[22601]: cluster 2024-09-12T23:27:23.394440+0000 mgr.y (mgr.14150) 125 : cluster [DBG] pgmap v89: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:25.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:24 smithi136 bash[32189]: cluster 2024-09-12T23:27:23.394440+0000 mgr.y (mgr.14150) 125 : cluster [DBG] pgmap v89: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:25.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:24 smithi181 bash[24776]: cluster 2024-09-12T23:27:23.394440+0000 mgr.y (mgr.14150) 125 : cluster [DBG] pgmap v89: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:27.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:26 smithi136 bash[22601]: cluster 2024-09-12T23:27:25.394874+0000 mgr.y (mgr.14150) 126 : cluster [DBG] pgmap v90: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:27.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:26 smithi136 bash[32189]: cluster 2024-09-12T23:27:25.394874+0000 mgr.y (mgr.14150) 126 : cluster [DBG] pgmap v90: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:27.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:26 smithi181 bash[24776]: cluster 2024-09-12T23:27:25.394874+0000 mgr.y (mgr.14150) 126 : cluster [DBG] pgmap v90: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:29.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:28 smithi136 bash[22601]: cluster 2024-09-12T23:27:27.395327+0000 mgr.y (mgr.14150) 127 : cluster [DBG] pgmap v91: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:29.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:28 smithi136 bash[32189]: cluster 2024-09-12T23:27:27.395327+0000 mgr.y (mgr.14150) 127 : cluster [DBG] pgmap v91: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:29.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:28 smithi181 bash[24776]: cluster 2024-09-12T23:27:27.395327+0000 mgr.y (mgr.14150) 127 : cluster [DBG] pgmap v91: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:30.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:29 smithi181 bash[24776]: audit 2024-09-12T23:27:28.977520+0000 mon.a (mon.0) 297 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-12T23:27:30.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:29 smithi181 bash[24776]: audit 2024-09-12T23:27:28.979012+0000 mon.a (mon.0) 298 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:27:30.171 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:29 smithi136 bash[22601]: audit 2024-09-12T23:27:28.977520+0000 mon.a (mon.0) 297 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-12T23:27:30.171 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:29 smithi136 bash[22601]: audit 2024-09-12T23:27:28.979012+0000 mon.a (mon.0) 298 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:27:30.171 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:29 smithi136 bash[32189]: audit 2024-09-12T23:27:28.977520+0000 mon.a (mon.0) 297 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-12T23:27:30.171 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:29 smithi136 bash[32189]: audit 2024-09-12T23:27:28.979012+0000 mon.a (mon.0) 298 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:27:31.034 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:30 smithi136 bash[32189]: cephadm 2024-09-12T23:27:28.980118+0000 mgr.y (mgr.14150) 128 : cephadm [INF] Deploying daemon osd.1 on smithi136 2024-09-12T23:27:31.034 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:30 smithi136 bash[32189]: cluster 2024-09-12T23:27:29.395810+0000 mgr.y (mgr.14150) 129 : cluster [DBG] pgmap v92: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:31.034 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:30 smithi136 bash[22601]: cephadm 2024-09-12T23:27:28.980118+0000 mgr.y (mgr.14150) 128 : cephadm [INF] Deploying daemon osd.1 on smithi136 2024-09-12T23:27:31.035 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:30 smithi136 bash[22601]: cluster 2024-09-12T23:27:29.395810+0000 mgr.y (mgr.14150) 129 : cluster [DBG] pgmap v92: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:31.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:30 smithi181 bash[24776]: cephadm 2024-09-12T23:27:28.980118+0000 mgr.y (mgr.14150) 128 : cephadm [INF] Deploying daemon osd.1 on smithi136 2024-09-12T23:27:31.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:30 smithi181 bash[24776]: cluster 2024-09-12T23:27:29.395810+0000 mgr.y (mgr.14150) 129 : cluster [DBG] pgmap v92: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:33.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:32 smithi136 bash[22601]: cluster 2024-09-12T23:27:31.396286+0000 mgr.y (mgr.14150) 130 : cluster [DBG] pgmap v93: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:33.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:32 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:27:33.010 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:27:32 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:27:33.010 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 12 23:27:32 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:27:33.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:32 smithi136 bash[32189]: cluster 2024-09-12T23:27:31.396286+0000 mgr.y (mgr.14150) 130 : cluster [DBG] pgmap v93: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:33.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:32 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:27:33.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:32 smithi181 bash[24776]: cluster 2024-09-12T23:27:31.396286+0000 mgr.y (mgr.14150) 130 : cluster [DBG] pgmap v93: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:33.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:33 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:27:33.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:33 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:27:33.510 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:27:33 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:27:33.511 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 12 23:27:33 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:27:34.094 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:33 smithi136 bash[22601]: audit 2024-09-12T23:27:33.171678+0000 mon.a (mon.0) 299 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:27:34.094 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:33 smithi136 bash[22601]: audit 2024-09-12T23:27:33.176505+0000 mon.a (mon.0) 300 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:27:34.094 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:33 smithi136 bash[22601]: audit 2024-09-12T23:27:33.180926+0000 mon.a (mon.0) 301 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:27:34.094 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:33 smithi136 bash[32189]: audit 2024-09-12T23:27:33.171678+0000 mon.a (mon.0) 299 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:27:34.094 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:33 smithi136 bash[32189]: audit 2024-09-12T23:27:33.176505+0000 mon.a (mon.0) 300 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:27:34.094 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:33 smithi136 bash[32189]: audit 2024-09-12T23:27:33.180926+0000 mon.a (mon.0) 301 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:27:34.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:33 smithi181 bash[24776]: audit 2024-09-12T23:27:33.171678+0000 mon.a (mon.0) 299 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:27:34.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:33 smithi181 bash[24776]: audit 2024-09-12T23:27:33.176505+0000 mon.a (mon.0) 300 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:27:34.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:33 smithi181 bash[24776]: audit 2024-09-12T23:27:33.180926+0000 mon.a (mon.0) 301 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:27:35.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:34 smithi181 bash[24776]: cluster 2024-09-12T23:27:33.396595+0000 mgr.y (mgr.14150) 131 : cluster [DBG] pgmap v94: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:35.156 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:34 smithi136 bash[22601]: cluster 2024-09-12T23:27:33.396595+0000 mgr.y (mgr.14150) 131 : cluster [DBG] pgmap v94: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:35.156 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:34 smithi136 bash[32189]: cluster 2024-09-12T23:27:33.396595+0000 mgr.y (mgr.14150) 131 : cluster [DBG] pgmap v94: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:37.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:36 smithi136 bash[22601]: cluster 2024-09-12T23:27:35.397098+0000 mgr.y (mgr.14150) 132 : cluster [DBG] pgmap v95: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:37.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:36 smithi136 bash[32189]: cluster 2024-09-12T23:27:35.397098+0000 mgr.y (mgr.14150) 132 : cluster [DBG] pgmap v95: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:37.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:36 smithi181 bash[24776]: cluster 2024-09-12T23:27:35.397098+0000 mgr.y (mgr.14150) 132 : cluster [DBG] pgmap v95: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:39.063 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:38 smithi136 bash[22601]: cluster 2024-09-12T23:27:37.397686+0000 mgr.y (mgr.14150) 133 : cluster [DBG] pgmap v96: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:39.063 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:38 smithi136 bash[22601]: audit 2024-09-12T23:27:38.229493+0000 mon.a (mon.0) 302 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:27:39.064 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:38 smithi136 bash[22601]: audit 2024-09-12T23:27:38.234363+0000 mon.a (mon.0) 303 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:27:39.064 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:38 smithi136 bash[22601]: audit 2024-09-12T23:27:38.235174+0000 mon.a (mon.0) 304 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:27:39.064 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:38 smithi136 bash[22601]: audit 2024-09-12T23:27:38.235855+0000 mon.a (mon.0) 305 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:27:39.064 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:38 smithi136 bash[22601]: audit 2024-09-12T23:27:38.241031+0000 mon.a (mon.0) 306 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:27:39.064 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:38 smithi136 bash[32189]: cluster 2024-09-12T23:27:37.397686+0000 mgr.y (mgr.14150) 133 : cluster [DBG] pgmap v96: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:39.064 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:38 smithi136 bash[32189]: audit 2024-09-12T23:27:38.229493+0000 mon.a (mon.0) 302 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:27:39.064 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:38 smithi136 bash[32189]: audit 2024-09-12T23:27:38.234363+0000 mon.a (mon.0) 303 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:27:39.064 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:38 smithi136 bash[32189]: audit 2024-09-12T23:27:38.235174+0000 mon.a (mon.0) 304 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:27:39.064 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:38 smithi136 bash[32189]: audit 2024-09-12T23:27:38.235855+0000 mon.a (mon.0) 305 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:27:39.064 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:38 smithi136 bash[32189]: audit 2024-09-12T23:27:38.241031+0000 mon.a (mon.0) 306 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:27:39.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:38 smithi181 bash[24776]: cluster 2024-09-12T23:27:37.397686+0000 mgr.y (mgr.14150) 133 : cluster [DBG] pgmap v96: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:39.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:38 smithi181 bash[24776]: audit 2024-09-12T23:27:38.229493+0000 mon.a (mon.0) 302 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:27:39.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:38 smithi181 bash[24776]: audit 2024-09-12T23:27:38.234363+0000 mon.a (mon.0) 303 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:27:39.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:38 smithi181 bash[24776]: audit 2024-09-12T23:27:38.235174+0000 mon.a (mon.0) 304 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:27:39.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:38 smithi181 bash[24776]: audit 2024-09-12T23:27:38.235855+0000 mon.a (mon.0) 305 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:27:39.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:38 smithi181 bash[24776]: audit 2024-09-12T23:27:38.241031+0000 mon.a (mon.0) 306 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:27:40.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:39 smithi181 bash[24776]: audit 2024-09-12T23:27:39.507317+0000 mon.c (mon.2) 6 : audit [INF] from='osd.1 [v2:172.21.15.136:6810/3482177958,v1:172.21.15.136:6811/3482177958]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-12T23:27:40.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:39 smithi181 bash[24776]: audit 2024-09-12T23:27:39.507677+0000 mon.a (mon.0) 307 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-12T23:27:40.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:39 smithi136 bash[22601]: audit 2024-09-12T23:27:39.507317+0000 mon.c (mon.2) 6 : audit [INF] from='osd.1 [v2:172.21.15.136:6810/3482177958,v1:172.21.15.136:6811/3482177958]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-12T23:27:40.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:39 smithi136 bash[22601]: audit 2024-09-12T23:27:39.507677+0000 mon.a (mon.0) 307 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-12T23:27:40.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:39 smithi136 bash[32189]: audit 2024-09-12T23:27:39.507317+0000 mon.c (mon.2) 6 : audit [INF] from='osd.1 [v2:172.21.15.136:6810/3482177958,v1:172.21.15.136:6811/3482177958]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-12T23:27:40.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:39 smithi136 bash[32189]: audit 2024-09-12T23:27:39.507677+0000 mon.a (mon.0) 307 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-12T23:27:41.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:40 smithi136 bash[32189]: cluster 2024-09-12T23:27:39.398157+0000 mgr.y (mgr.14150) 134 : cluster [DBG] pgmap v97: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:41.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:40 smithi136 bash[32189]: audit 2024-09-12T23:27:39.804129+0000 mon.a (mon.0) 308 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-12T23:27:41.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:40 smithi136 bash[32189]: cluster 2024-09-12T23:27:39.804256+0000 mon.a (mon.0) 309 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2024-09-12T23:27:41.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:40 smithi136 bash[32189]: audit 2024-09-12T23:27:39.804552+0000 mon.a (mon.0) 310 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-12T23:27:41.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:40 smithi136 bash[32189]: audit 2024-09-12T23:27:39.805448+0000 mon.c (mon.2) 7 : audit [INF] from='osd.1 [v2:172.21.15.136:6810/3482177958,v1:172.21.15.136:6811/3482177958]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-12T23:27:41.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:40 smithi136 bash[32189]: audit 2024-09-12T23:27:39.806105+0000 mon.a (mon.0) 311 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-12T23:27:41.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:40 smithi136 bash[22601]: cluster 2024-09-12T23:27:39.398157+0000 mgr.y (mgr.14150) 134 : cluster [DBG] pgmap v97: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:41.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:40 smithi136 bash[22601]: audit 2024-09-12T23:27:39.804129+0000 mon.a (mon.0) 308 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-12T23:27:41.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:40 smithi136 bash[22601]: cluster 2024-09-12T23:27:39.804256+0000 mon.a (mon.0) 309 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2024-09-12T23:27:41.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:40 smithi136 bash[22601]: audit 2024-09-12T23:27:39.804552+0000 mon.a (mon.0) 310 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-12T23:27:41.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:40 smithi136 bash[22601]: audit 2024-09-12T23:27:39.805448+0000 mon.c (mon.2) 7 : audit [INF] from='osd.1 [v2:172.21.15.136:6810/3482177958,v1:172.21.15.136:6811/3482177958]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-12T23:27:41.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:40 smithi136 bash[22601]: audit 2024-09-12T23:27:39.806105+0000 mon.a (mon.0) 311 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-12T23:27:41.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:40 smithi181 bash[24776]: cluster 2024-09-12T23:27:39.398157+0000 mgr.y (mgr.14150) 134 : cluster [DBG] pgmap v97: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:41.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:40 smithi181 bash[24776]: audit 2024-09-12T23:27:39.804129+0000 mon.a (mon.0) 308 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-12T23:27:41.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:40 smithi181 bash[24776]: cluster 2024-09-12T23:27:39.804256+0000 mon.a (mon.0) 309 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2024-09-12T23:27:41.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:40 smithi181 bash[24776]: audit 2024-09-12T23:27:39.804552+0000 mon.a (mon.0) 310 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-12T23:27:41.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:40 smithi181 bash[24776]: audit 2024-09-12T23:27:39.805448+0000 mon.c (mon.2) 7 : audit [INF] from='osd.1 [v2:172.21.15.136:6810/3482177958,v1:172.21.15.136:6811/3482177958]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-12T23:27:41.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:40 smithi181 bash[24776]: audit 2024-09-12T23:27:39.806105+0000 mon.a (mon.0) 311 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-12T23:27:41.656 INFO:teuthology.orchestra.run.smithi136.stdout:Created osd(s) 1 on host 'smithi136' 2024-09-12T23:27:42.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:41 smithi136 bash[22601]: audit 2024-09-12T23:27:40.803938+0000 mon.a (mon.0) 312 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]': finished 2024-09-12T23:27:42.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:41 smithi136 bash[22601]: cluster 2024-09-12T23:27:40.804139+0000 mon.a (mon.0) 313 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2024-09-12T23:27:42.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:41 smithi136 bash[22601]: audit 2024-09-12T23:27:40.804732+0000 mon.a (mon.0) 314 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-12T23:27:42.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:41 smithi136 bash[22601]: audit 2024-09-12T23:27:40.813806+0000 mon.a (mon.0) 315 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-12T23:27:42.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:41 smithi136 bash[22601]: audit 2024-09-12T23:27:41.638382+0000 mon.a (mon.0) 316 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:27:42.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:41 smithi136 bash[22601]: audit 2024-09-12T23:27:41.644924+0000 mon.a (mon.0) 317 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:27:42.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:41 smithi136 bash[22601]: audit 2024-09-12T23:27:41.652430+0000 mon.a (mon.0) 318 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:27:42.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:41 smithi136 bash[32189]: audit 2024-09-12T23:27:40.803938+0000 mon.a (mon.0) 312 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]': finished 2024-09-12T23:27:42.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:41 smithi136 bash[32189]: cluster 2024-09-12T23:27:40.804139+0000 mon.a (mon.0) 313 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2024-09-12T23:27:42.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:41 smithi136 bash[32189]: audit 2024-09-12T23:27:40.804732+0000 mon.a (mon.0) 314 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-12T23:27:42.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:41 smithi136 bash[32189]: audit 2024-09-12T23:27:40.813806+0000 mon.a (mon.0) 315 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-12T23:27:42.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:41 smithi136 bash[32189]: audit 2024-09-12T23:27:41.638382+0000 mon.a (mon.0) 316 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:27:42.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:41 smithi136 bash[32189]: audit 2024-09-12T23:27:41.644924+0000 mon.a (mon.0) 317 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:27:42.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:41 smithi136 bash[32189]: audit 2024-09-12T23:27:41.652430+0000 mon.a (mon.0) 318 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:27:42.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:41 smithi181 bash[24776]: audit 2024-09-12T23:27:40.803938+0000 mon.a (mon.0) 312 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]': finished 2024-09-12T23:27:42.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:41 smithi181 bash[24776]: cluster 2024-09-12T23:27:40.804139+0000 mon.a (mon.0) 313 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2024-09-12T23:27:42.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:41 smithi181 bash[24776]: audit 2024-09-12T23:27:40.804732+0000 mon.a (mon.0) 314 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-12T23:27:42.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:41 smithi181 bash[24776]: audit 2024-09-12T23:27:40.813806+0000 mon.a (mon.0) 315 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-12T23:27:42.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:41 smithi181 bash[24776]: audit 2024-09-12T23:27:41.638382+0000 mon.a (mon.0) 316 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:27:42.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:41 smithi181 bash[24776]: audit 2024-09-12T23:27:41.644924+0000 mon.a (mon.0) 317 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:27:42.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:41 smithi181 bash[24776]: audit 2024-09-12T23:27:41.652430+0000 mon.a (mon.0) 318 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:27:42.254 DEBUG:teuthology.orchestra.run.smithi136:osd.1> sudo journalctl -f -n 0 -u ceph-da58a47c-715d-11ef-bceb-c7b262605968@osd.1.service 2024-09-12T23:27:42.257 INFO:tasks.cephadm:Deploying osd.2 on smithi136 with /dev/vg_nvme/lv_2... 2024-09-12T23:27:42.257 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-09-12T23:27:43.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:42 smithi181 bash[24776]: cluster 2024-09-12T23:27:40.536381+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-12T23:27:43.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:42 smithi181 bash[24776]: cluster 2024-09-12T23:27:40.536470+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-12T23:27:43.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:42 smithi181 bash[24776]: cluster 2024-09-12T23:27:41.398604+0000 mgr.y (mgr.14150) 135 : cluster [DBG] pgmap v100: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:43.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:42 smithi181 bash[24776]: cluster 2024-09-12T23:27:41.813234+0000 mon.a (mon.0) 319 : cluster [INF] osd.1 [v2:172.21.15.136:6810/3482177958,v1:172.21.15.136:6811/3482177958] boot 2024-09-12T23:27:43.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:42 smithi181 bash[24776]: cluster 2024-09-12T23:27:41.813345+0000 mon.a (mon.0) 320 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2024-09-12T23:27:43.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:42 smithi181 bash[24776]: audit 2024-09-12T23:27:41.813880+0000 mon.a (mon.0) 321 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-12T23:27:43.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:42 smithi136 bash[22601]: cluster 2024-09-12T23:27:40.536381+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-12T23:27:43.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:42 smithi136 bash[22601]: cluster 2024-09-12T23:27:40.536470+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-12T23:27:43.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:42 smithi136 bash[22601]: cluster 2024-09-12T23:27:41.398604+0000 mgr.y (mgr.14150) 135 : cluster [DBG] pgmap v100: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:43.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:42 smithi136 bash[22601]: cluster 2024-09-12T23:27:41.813234+0000 mon.a (mon.0) 319 : cluster [INF] osd.1 [v2:172.21.15.136:6810/3482177958,v1:172.21.15.136:6811/3482177958] boot 2024-09-12T23:27:43.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:42 smithi136 bash[22601]: cluster 2024-09-12T23:27:41.813345+0000 mon.a (mon.0) 320 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2024-09-12T23:27:43.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:42 smithi136 bash[22601]: audit 2024-09-12T23:27:41.813880+0000 mon.a (mon.0) 321 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-12T23:27:43.262 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:42 smithi136 bash[32189]: cluster 2024-09-12T23:27:40.536381+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-12T23:27:43.262 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:42 smithi136 bash[32189]: cluster 2024-09-12T23:27:40.536470+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-12T23:27:43.262 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:42 smithi136 bash[32189]: cluster 2024-09-12T23:27:41.398604+0000 mgr.y (mgr.14150) 135 : cluster [DBG] pgmap v100: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-12T23:27:43.262 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:42 smithi136 bash[32189]: cluster 2024-09-12T23:27:41.813234+0000 mon.a (mon.0) 319 : cluster [INF] osd.1 [v2:172.21.15.136:6810/3482177958,v1:172.21.15.136:6811/3482177958] boot 2024-09-12T23:27:43.262 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:42 smithi136 bash[32189]: cluster 2024-09-12T23:27:41.813345+0000 mon.a (mon.0) 320 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2024-09-12T23:27:43.262 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:42 smithi136 bash[32189]: audit 2024-09-12T23:27:41.813880+0000 mon.a (mon.0) 321 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-12T23:27:44.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:43 smithi181 bash[24776]: cluster 2024-09-12T23:27:42.831681+0000 mon.a (mon.0) 322 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2024-09-12T23:27:44.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:43 smithi136 bash[22601]: cluster 2024-09-12T23:27:42.831681+0000 mon.a (mon.0) 322 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2024-09-12T23:27:44.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:43 smithi136 bash[32189]: cluster 2024-09-12T23:27:42.831681+0000 mon.a (mon.0) 322 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2024-09-12T23:27:45.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:44 smithi181 bash[24776]: cluster 2024-09-12T23:27:43.399046+0000 mgr.y (mgr.14150) 136 : cluster [DBG] pgmap v103: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:27:45.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:44 smithi136 bash[22601]: cluster 2024-09-12T23:27:43.399046+0000 mgr.y (mgr.14150) 136 : cluster [DBG] pgmap v103: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:27:45.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:44 smithi136 bash[32189]: cluster 2024-09-12T23:27:43.399046+0000 mgr.y (mgr.14150) 136 : cluster [DBG] pgmap v103: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:27:46.418 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:27:47.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:46 smithi181 bash[24776]: cluster 2024-09-12T23:27:45.399471+0000 mgr.y (mgr.14150) 137 : cluster [DBG] pgmap v104: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:27:47.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:46 smithi136 bash[22601]: cluster 2024-09-12T23:27:45.399471+0000 mgr.y (mgr.14150) 137 : cluster [DBG] pgmap v104: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:27:47.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:46 smithi136 bash[32189]: cluster 2024-09-12T23:27:45.399471+0000 mgr.y (mgr.14150) 137 : cluster [DBG] pgmap v104: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:27:49.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:48 smithi181 bash[24776]: cluster 2024-09-12T23:27:47.399926+0000 mgr.y (mgr.14150) 138 : cluster [DBG] pgmap v105: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:27:49.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:48 smithi136 bash[22601]: cluster 2024-09-12T23:27:47.399926+0000 mgr.y (mgr.14150) 138 : cluster [DBG] pgmap v105: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:27:49.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:48 smithi136 bash[32189]: cluster 2024-09-12T23:27:47.399926+0000 mgr.y (mgr.14150) 138 : cluster [DBG] pgmap v105: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:27:50.887 INFO:teuthology.orchestra.run.smithi136.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-09-12T23:27:51.085 INFO:teuthology.orchestra.run.smithi136.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-09-12T23:27:51.098 INFO:teuthology.orchestra.run.smithi136.stderr: stderr: 10+0 records in 2024-09-12T23:27:51.098 INFO:teuthology.orchestra.run.smithi136.stderr:10+0 records out 2024-09-12T23:27:51.099 INFO:teuthology.orchestra.run.smithi136.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.011268 s, 931 MB/s 2024-09-12T23:27:51.099 INFO:teuthology.orchestra.run.smithi136.stderr:--> Zapping successful for: 2024-09-12T23:27:51.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:50 smithi181 bash[24776]: cluster 2024-09-12T23:27:49.400399+0000 mgr.y (mgr.14150) 139 : cluster [DBG] pgmap v106: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:27:51.118 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:50 smithi136 bash[32189]: cluster 2024-09-12T23:27:49.400399+0000 mgr.y (mgr.14150) 139 : cluster [DBG] pgmap v106: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:27:51.118 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:50 smithi136 bash[22601]: cluster 2024-09-12T23:27:49.400399+0000 mgr.y (mgr.14150) 139 : cluster [DBG] pgmap v106: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:27:52.003 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi136:vg_nvme/lv_2 2024-09-12T23:27:53.262 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:52 smithi136 bash[22601]: cluster 2024-09-12T23:27:51.400669+0000 mgr.y (mgr.14150) 140 : cluster [DBG] pgmap v107: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:27:53.263 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:52 smithi136 bash[22601]: audit 2024-09-12T23:27:52.077498+0000 mon.a (mon.0) 323 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:27:53.263 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:52 smithi136 bash[22601]: audit 2024-09-12T23:27:52.084911+0000 mon.a (mon.0) 324 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:27:53.263 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:52 smithi136 bash[22601]: audit 2024-09-12T23:27:52.086182+0000 mon.a (mon.0) 325 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:27:53.263 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:52 smithi136 bash[22601]: audit 2024-09-12T23:27:52.087052+0000 mon.a (mon.0) 326 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:27:53.263 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:52 smithi136 bash[22601]: audit 2024-09-12T23:27:52.093572+0000 mon.a (mon.0) 327 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:27:53.263 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:52 smithi136 bash[22601]: audit 2024-09-12T23:27:52.095759+0000 mon.a (mon.0) 328 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:27:53.263 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:52 smithi136 bash[22601]: audit 2024-09-12T23:27:52.096732+0000 mon.a (mon.0) 329 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:27:53.263 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:52 smithi136 bash[22601]: audit 2024-09-12T23:27:52.103920+0000 mon.a (mon.0) 330 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:27:53.263 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:52 smithi136 bash[32189]: cluster 2024-09-12T23:27:51.400669+0000 mgr.y (mgr.14150) 140 : cluster [DBG] pgmap v107: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:27:53.263 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:52 smithi136 bash[32189]: audit 2024-09-12T23:27:52.077498+0000 mon.a (mon.0) 323 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:27:53.263 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:52 smithi136 bash[32189]: audit 2024-09-12T23:27:52.084911+0000 mon.a (mon.0) 324 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:27:53.263 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:52 smithi136 bash[32189]: audit 2024-09-12T23:27:52.086182+0000 mon.a (mon.0) 325 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:27:53.264 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:52 smithi136 bash[32189]: audit 2024-09-12T23:27:52.087052+0000 mon.a (mon.0) 326 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:27:53.264 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:52 smithi136 bash[32189]: audit 2024-09-12T23:27:52.093572+0000 mon.a (mon.0) 327 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:27:53.264 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:52 smithi136 bash[32189]: audit 2024-09-12T23:27:52.095759+0000 mon.a (mon.0) 328 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:27:53.264 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:52 smithi136 bash[32189]: audit 2024-09-12T23:27:52.096732+0000 mon.a (mon.0) 329 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:27:53.264 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:52 smithi136 bash[32189]: audit 2024-09-12T23:27:52.103920+0000 mon.a (mon.0) 330 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:27:53.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:52 smithi181 bash[24776]: cluster 2024-09-12T23:27:51.400669+0000 mgr.y (mgr.14150) 140 : cluster [DBG] pgmap v107: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:27:53.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:52 smithi181 bash[24776]: audit 2024-09-12T23:27:52.077498+0000 mon.a (mon.0) 323 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:27:53.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:52 smithi181 bash[24776]: audit 2024-09-12T23:27:52.084911+0000 mon.a (mon.0) 324 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:27:53.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:52 smithi181 bash[24776]: audit 2024-09-12T23:27:52.086182+0000 mon.a (mon.0) 325 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:27:53.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:52 smithi181 bash[24776]: audit 2024-09-12T23:27:52.087052+0000 mon.a (mon.0) 326 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:27:53.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:52 smithi181 bash[24776]: audit 2024-09-12T23:27:52.093572+0000 mon.a (mon.0) 327 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:27:53.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:52 smithi181 bash[24776]: audit 2024-09-12T23:27:52.095759+0000 mon.a (mon.0) 328 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:27:53.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:52 smithi181 bash[24776]: audit 2024-09-12T23:27:52.096732+0000 mon.a (mon.0) 329 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:27:53.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:52 smithi181 bash[24776]: audit 2024-09-12T23:27:52.103920+0000 mon.a (mon.0) 330 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:27:54.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:53 smithi136 bash[22601]: cephadm 2024-09-12T23:27:52.070108+0000 mgr.y (mgr.14150) 141 : cephadm [INF] Detected new or changed devices on smithi136 2024-09-12T23:27:54.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:53 smithi136 bash[22601]: cephadm 2024-09-12T23:27:52.087690+0000 mgr.y (mgr.14150) 142 : cephadm [INF] Adjusting osd_memory_target on smithi136 to 8120M 2024-09-12T23:27:54.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:53 smithi136 bash[32189]: cephadm 2024-09-12T23:27:52.070108+0000 mgr.y (mgr.14150) 141 : cephadm [INF] Detected new or changed devices on smithi136 2024-09-12T23:27:54.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:53 smithi136 bash[32189]: cephadm 2024-09-12T23:27:52.087690+0000 mgr.y (mgr.14150) 142 : cephadm [INF] Adjusting osd_memory_target on smithi136 to 8120M 2024-09-12T23:27:54.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:53 smithi181 bash[24776]: cephadm 2024-09-12T23:27:52.070108+0000 mgr.y (mgr.14150) 141 : cephadm [INF] Detected new or changed devices on smithi136 2024-09-12T23:27:54.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:53 smithi181 bash[24776]: cephadm 2024-09-12T23:27:52.087690+0000 mgr.y (mgr.14150) 142 : cephadm [INF] Adjusting osd_memory_target on smithi136 to 8120M 2024-09-12T23:27:55.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:54 smithi136 bash[22601]: cluster 2024-09-12T23:27:53.401089+0000 mgr.y (mgr.14150) 143 : cluster [DBG] pgmap v108: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:27:55.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:54 smithi136 bash[32189]: cluster 2024-09-12T23:27:53.401089+0000 mgr.y (mgr.14150) 143 : cluster [DBG] pgmap v108: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:27:55.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:54 smithi181 bash[24776]: cluster 2024-09-12T23:27:53.401089+0000 mgr.y (mgr.14150) 143 : cluster [DBG] pgmap v108: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:27:56.770 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:27:57.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:56 smithi136 bash[22601]: cluster 2024-09-12T23:27:55.401566+0000 mgr.y (mgr.14150) 144 : cluster [DBG] pgmap v109: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:27:57.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:56 smithi136 bash[32189]: cluster 2024-09-12T23:27:55.401566+0000 mgr.y (mgr.14150) 144 : cluster [DBG] pgmap v109: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:27:57.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:56 smithi181 bash[24776]: cluster 2024-09-12T23:27:55.401566+0000 mgr.y (mgr.14150) 144 : cluster [DBG] pgmap v109: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:27:59.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:58 smithi136 bash[32189]: cluster 2024-09-12T23:27:57.401973+0000 mgr.y (mgr.14150) 145 : cluster [DBG] pgmap v110: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:27:59.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:58 smithi136 bash[22601]: cluster 2024-09-12T23:27:57.401973+0000 mgr.y (mgr.14150) 145 : cluster [DBG] pgmap v110: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:27:59.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:58 smithi181 bash[24776]: cluster 2024-09-12T23:27:57.401973+0000 mgr.y (mgr.14150) 145 : cluster [DBG] pgmap v110: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:00.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:59 smithi136 bash[22601]: audit 2024-09-12T23:27:59.673405+0000 mon.a (mon.0) 331 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-12T23:28:00.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:59 smithi136 bash[22601]: audit 2024-09-12T23:27:59.679021+0000 mon.a (mon.0) 332 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-12T23:28:00.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:27:59 smithi136 bash[22601]: audit 2024-09-12T23:27:59.680300+0000 mon.a (mon.0) 333 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:28:00.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:59 smithi136 bash[32189]: audit 2024-09-12T23:27:59.673405+0000 mon.a (mon.0) 331 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-12T23:28:00.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:59 smithi136 bash[32189]: audit 2024-09-12T23:27:59.679021+0000 mon.a (mon.0) 332 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-12T23:28:00.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:27:59 smithi136 bash[32189]: audit 2024-09-12T23:27:59.680300+0000 mon.a (mon.0) 333 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:28:00.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:59 smithi181 bash[24776]: audit 2024-09-12T23:27:59.673405+0000 mon.a (mon.0) 331 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-12T23:28:00.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:59 smithi181 bash[24776]: audit 2024-09-12T23:27:59.679021+0000 mon.a (mon.0) 332 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-12T23:28:00.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:27:59 smithi181 bash[24776]: audit 2024-09-12T23:27:59.680300+0000 mon.a (mon.0) 333 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:28:01.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:00 smithi136 bash[22601]: cluster 2024-09-12T23:27:59.402315+0000 mgr.y (mgr.14150) 146 : cluster [DBG] pgmap v111: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:01.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:00 smithi136 bash[22601]: audit 2024-09-12T23:27:59.667198+0000 mgr.y (mgr.14150) 147 : audit [DBG] from='client.24170 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi136:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:28:01.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:00 smithi136 bash[32189]: cluster 2024-09-12T23:27:59.402315+0000 mgr.y (mgr.14150) 146 : cluster [DBG] pgmap v111: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:01.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:00 smithi136 bash[32189]: audit 2024-09-12T23:27:59.667198+0000 mgr.y (mgr.14150) 147 : audit [DBG] from='client.24170 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi136:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:28:01.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:00 smithi181 bash[24776]: cluster 2024-09-12T23:27:59.402315+0000 mgr.y (mgr.14150) 146 : cluster [DBG] pgmap v111: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:01.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:00 smithi181 bash[24776]: audit 2024-09-12T23:27:59.667198+0000 mgr.y (mgr.14150) 147 : audit [DBG] from='client.24170 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi136:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:28:03.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:02 smithi136 bash[22601]: cluster 2024-09-12T23:28:01.402732+0000 mgr.y (mgr.14150) 148 : cluster [DBG] pgmap v112: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:03.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:02 smithi136 bash[32189]: cluster 2024-09-12T23:28:01.402732+0000 mgr.y (mgr.14150) 148 : cluster [DBG] pgmap v112: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:03.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:02 smithi181 bash[24776]: cluster 2024-09-12T23:28:01.402732+0000 mgr.y (mgr.14150) 148 : cluster [DBG] pgmap v112: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:05.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:04 smithi136 bash[22601]: cluster 2024-09-12T23:28:03.403226+0000 mgr.y (mgr.14150) 149 : cluster [DBG] pgmap v113: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:05.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:04 smithi136 bash[32189]: cluster 2024-09-12T23:28:03.403226+0000 mgr.y (mgr.14150) 149 : cluster [DBG] pgmap v113: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:05.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:04 smithi181 bash[24776]: cluster 2024-09-12T23:28:03.403226+0000 mgr.y (mgr.14150) 149 : cluster [DBG] pgmap v113: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:07.165 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:06 smithi136 bash[22601]: cluster 2024-09-12T23:28:05.403724+0000 mgr.y (mgr.14150) 150 : cluster [DBG] pgmap v114: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:07.165 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:06 smithi136 bash[32189]: cluster 2024-09-12T23:28:05.403724+0000 mgr.y (mgr.14150) 150 : cluster [DBG] pgmap v114: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:07.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:06 smithi181 bash[24776]: cluster 2024-09-12T23:28:05.403724+0000 mgr.y (mgr.14150) 150 : cluster [DBG] pgmap v114: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:08.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:07 smithi136 bash[32189]: audit 2024-09-12T23:28:07.819964+0000 mon.c (mon.2) 8 : audit [INF] from='client.? 172.21.15.136:0/3934250458' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bde707d6-e027-4b63-9853-fd59212eea1e"}]: dispatch 2024-09-12T23:28:08.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:07 smithi136 bash[32189]: audit 2024-09-12T23:28:07.820437+0000 mon.a (mon.0) 334 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bde707d6-e027-4b63-9853-fd59212eea1e"}]: dispatch 2024-09-12T23:28:08.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:07 smithi136 bash[32189]: audit 2024-09-12T23:28:07.828395+0000 mon.a (mon.0) 335 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bde707d6-e027-4b63-9853-fd59212eea1e"}]': finished 2024-09-12T23:28:08.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:07 smithi136 bash[32189]: cluster 2024-09-12T23:28:07.828564+0000 mon.a (mon.0) 336 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2024-09-12T23:28:08.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:07 smithi136 bash[32189]: audit 2024-09-12T23:28:07.828811+0000 mon.a (mon.0) 337 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-12T23:28:08.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:07 smithi136 bash[22601]: audit 2024-09-12T23:28:07.819964+0000 mon.c (mon.2) 8 : audit [INF] from='client.? 172.21.15.136:0/3934250458' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bde707d6-e027-4b63-9853-fd59212eea1e"}]: dispatch 2024-09-12T23:28:08.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:07 smithi136 bash[22601]: audit 2024-09-12T23:28:07.820437+0000 mon.a (mon.0) 334 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bde707d6-e027-4b63-9853-fd59212eea1e"}]: dispatch 2024-09-12T23:28:08.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:07 smithi136 bash[22601]: audit 2024-09-12T23:28:07.828395+0000 mon.a (mon.0) 335 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bde707d6-e027-4b63-9853-fd59212eea1e"}]': finished 2024-09-12T23:28:08.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:07 smithi136 bash[22601]: cluster 2024-09-12T23:28:07.828564+0000 mon.a (mon.0) 336 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2024-09-12T23:28:08.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:07 smithi136 bash[22601]: audit 2024-09-12T23:28:07.828811+0000 mon.a (mon.0) 337 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-12T23:28:08.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:07 smithi181 bash[24776]: audit 2024-09-12T23:28:07.819964+0000 mon.c (mon.2) 8 : audit [INF] from='client.? 172.21.15.136:0/3934250458' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bde707d6-e027-4b63-9853-fd59212eea1e"}]: dispatch 2024-09-12T23:28:08.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:07 smithi181 bash[24776]: audit 2024-09-12T23:28:07.820437+0000 mon.a (mon.0) 334 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bde707d6-e027-4b63-9853-fd59212eea1e"}]: dispatch 2024-09-12T23:28:08.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:07 smithi181 bash[24776]: audit 2024-09-12T23:28:07.828395+0000 mon.a (mon.0) 335 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bde707d6-e027-4b63-9853-fd59212eea1e"}]': finished 2024-09-12T23:28:08.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:07 smithi181 bash[24776]: cluster 2024-09-12T23:28:07.828564+0000 mon.a (mon.0) 336 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2024-09-12T23:28:08.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:07 smithi181 bash[24776]: audit 2024-09-12T23:28:07.828811+0000 mon.a (mon.0) 337 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-12T23:28:09.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:08 smithi136 bash[22601]: cluster 2024-09-12T23:28:07.404139+0000 mgr.y (mgr.14150) 151 : cluster [DBG] pgmap v115: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:09.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:08 smithi136 bash[22601]: audit 2024-09-12T23:28:08.444013+0000 mon.a (mon.0) 338 : audit [DBG] from='client.? 172.21.15.136:0/3279425364' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-12T23:28:09.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:08 smithi136 bash[32189]: cluster 2024-09-12T23:28:07.404139+0000 mgr.y (mgr.14150) 151 : cluster [DBG] pgmap v115: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:09.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:08 smithi136 bash[32189]: audit 2024-09-12T23:28:08.444013+0000 mon.a (mon.0) 338 : audit [DBG] from='client.? 172.21.15.136:0/3279425364' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-12T23:28:09.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:08 smithi181 bash[24776]: cluster 2024-09-12T23:28:07.404139+0000 mgr.y (mgr.14150) 151 : cluster [DBG] pgmap v115: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:09.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:08 smithi181 bash[24776]: audit 2024-09-12T23:28:08.444013+0000 mon.a (mon.0) 338 : audit [DBG] from='client.? 172.21.15.136:0/3279425364' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-12T23:28:11.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:10 smithi136 bash[22601]: cluster 2024-09-12T23:28:09.404495+0000 mgr.y (mgr.14150) 152 : cluster [DBG] pgmap v117: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:11.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:10 smithi136 bash[32189]: cluster 2024-09-12T23:28:09.404495+0000 mgr.y (mgr.14150) 152 : cluster [DBG] pgmap v117: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:11.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:10 smithi181 bash[24776]: cluster 2024-09-12T23:28:09.404495+0000 mgr.y (mgr.14150) 152 : cluster [DBG] pgmap v117: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:13.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:12 smithi136 bash[32189]: cluster 2024-09-12T23:28:11.404922+0000 mgr.y (mgr.14150) 153 : cluster [DBG] pgmap v118: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:13.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:12 smithi136 bash[22601]: cluster 2024-09-12T23:28:11.404922+0000 mgr.y (mgr.14150) 153 : cluster [DBG] pgmap v118: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:13.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:12 smithi181 bash[24776]: cluster 2024-09-12T23:28:11.404922+0000 mgr.y (mgr.14150) 153 : cluster [DBG] pgmap v118: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:15.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:14 smithi136 bash[22601]: cluster 2024-09-12T23:28:13.405379+0000 mgr.y (mgr.14150) 154 : cluster [DBG] pgmap v119: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:15.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:14 smithi136 bash[32189]: cluster 2024-09-12T23:28:13.405379+0000 mgr.y (mgr.14150) 154 : cluster [DBG] pgmap v119: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:15.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:14 smithi181 bash[24776]: cluster 2024-09-12T23:28:13.405379+0000 mgr.y (mgr.14150) 154 : cluster [DBG] pgmap v119: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:16.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:15 smithi136 bash[32189]: cluster 2024-09-12T23:28:15.405872+0000 mgr.y (mgr.14150) 155 : cluster [DBG] pgmap v120: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:16.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:15 smithi136 bash[22601]: cluster 2024-09-12T23:28:15.405872+0000 mgr.y (mgr.14150) 155 : cluster [DBG] pgmap v120: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:16.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:15 smithi181 bash[24776]: cluster 2024-09-12T23:28:15.405872+0000 mgr.y (mgr.14150) 155 : cluster [DBG] pgmap v120: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:18.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:18 smithi136 bash[22601]: cluster 2024-09-12T23:28:17.406193+0000 mgr.y (mgr.14150) 156 : cluster [DBG] pgmap v121: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:18.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:18 smithi136 bash[32189]: cluster 2024-09-12T23:28:17.406193+0000 mgr.y (mgr.14150) 156 : cluster [DBG] pgmap v121: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:18.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:18 smithi181 bash[24776]: cluster 2024-09-12T23:28:17.406193+0000 mgr.y (mgr.14150) 156 : cluster [DBG] pgmap v121: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:20.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:20 smithi136 bash[32189]: cluster 2024-09-12T23:28:19.406654+0000 mgr.y (mgr.14150) 157 : cluster [DBG] pgmap v122: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:20.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:20 smithi136 bash[32189]: audit 2024-09-12T23:28:19.923070+0000 mon.a (mon.0) 339 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-12T23:28:20.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:20 smithi136 bash[32189]: audit 2024-09-12T23:28:19.924472+0000 mon.a (mon.0) 340 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:28:20.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:20 smithi136 bash[22601]: cluster 2024-09-12T23:28:19.406654+0000 mgr.y (mgr.14150) 157 : cluster [DBG] pgmap v122: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:20.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:20 smithi136 bash[22601]: audit 2024-09-12T23:28:19.923070+0000 mon.a (mon.0) 339 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-12T23:28:20.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:20 smithi136 bash[22601]: audit 2024-09-12T23:28:19.924472+0000 mon.a (mon.0) 340 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:28:20.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:20 smithi181 bash[24776]: cluster 2024-09-12T23:28:19.406654+0000 mgr.y (mgr.14150) 157 : cluster [DBG] pgmap v122: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:20.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:20 smithi181 bash[24776]: audit 2024-09-12T23:28:19.923070+0000 mon.a (mon.0) 339 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-12T23:28:20.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:20 smithi181 bash[24776]: audit 2024-09-12T23:28:19.924472+0000 mon.a (mon.0) 340 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:28:21.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:21 smithi136 bash[32189]: cephadm 2024-09-12T23:28:19.925699+0000 mgr.y (mgr.14150) 158 : cephadm [INF] Deploying daemon osd.2 on smithi136 2024-09-12T23:28:21.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:21 smithi136 bash[22601]: cephadm 2024-09-12T23:28:19.925699+0000 mgr.y (mgr.14150) 158 : cephadm [INF] Deploying daemon osd.2 on smithi136 2024-09-12T23:28:21.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:21 smithi181 bash[24776]: cephadm 2024-09-12T23:28:19.925699+0000 mgr.y (mgr.14150) 158 : cephadm [INF] Deploying daemon osd.2 on smithi136 2024-09-12T23:28:22.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:22 smithi136 bash[32189]: cluster 2024-09-12T23:28:21.407055+0000 mgr.y (mgr.14150) 159 : cluster [DBG] pgmap v123: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:22.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:22 smithi136 bash[22601]: cluster 2024-09-12T23:28:21.407055+0000 mgr.y (mgr.14150) 159 : cluster [DBG] pgmap v123: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:22.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:22 smithi181 bash[24776]: cluster 2024-09-12T23:28:21.407055+0000 mgr.y (mgr.14150) 159 : cluster [DBG] pgmap v123: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:23.930 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:28:23 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:28:23.931 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 12 23:28:23 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:28:23.931 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 12 23:28:23 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:28:23.931 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:23 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:28:23.932 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:23 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:28:24.182 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:24 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:28:24.183 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:24 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:28:24.183 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 12 23:28:24 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:28:24.183 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 12 23:28:24 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:28:24.183 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:28:24 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:28:24.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:24 smithi136 bash[22601]: cluster 2024-09-12T23:28:23.407555+0000 mgr.y (mgr.14150) 160 : cluster [DBG] pgmap v124: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:24.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:24 smithi136 bash[22601]: audit 2024-09-12T23:28:24.215035+0000 mon.a (mon.0) 341 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:28:24.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:24 smithi136 bash[22601]: audit 2024-09-12T23:28:24.219991+0000 mon.a (mon.0) 342 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:28:24.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:24 smithi136 bash[22601]: audit 2024-09-12T23:28:24.226655+0000 mon.a (mon.0) 343 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:28:24.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:24 smithi136 bash[32189]: cluster 2024-09-12T23:28:23.407555+0000 mgr.y (mgr.14150) 160 : cluster [DBG] pgmap v124: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:24.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:24 smithi136 bash[32189]: audit 2024-09-12T23:28:24.215035+0000 mon.a (mon.0) 341 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:28:24.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:24 smithi136 bash[32189]: audit 2024-09-12T23:28:24.219991+0000 mon.a (mon.0) 342 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:28:24.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:24 smithi136 bash[32189]: audit 2024-09-12T23:28:24.226655+0000 mon.a (mon.0) 343 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:28:24.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:24 smithi181 bash[24776]: cluster 2024-09-12T23:28:23.407555+0000 mgr.y (mgr.14150) 160 : cluster [DBG] pgmap v124: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:24.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:24 smithi181 bash[24776]: audit 2024-09-12T23:28:24.215035+0000 mon.a (mon.0) 341 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:28:24.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:24 smithi181 bash[24776]: audit 2024-09-12T23:28:24.219991+0000 mon.a (mon.0) 342 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:28:24.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:24 smithi181 bash[24776]: audit 2024-09-12T23:28:24.226655+0000 mon.a (mon.0) 343 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:28:26.762 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:26 smithi136 bash[22601]: cluster 2024-09-12T23:28:25.407897+0000 mgr.y (mgr.14150) 161 : cluster [DBG] pgmap v125: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:26.762 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:26 smithi136 bash[32189]: cluster 2024-09-12T23:28:25.407897+0000 mgr.y (mgr.14150) 161 : cluster [DBG] pgmap v125: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:26.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:26 smithi181 bash[24776]: cluster 2024-09-12T23:28:25.407897+0000 mgr.y (mgr.14150) 161 : cluster [DBG] pgmap v125: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:28.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:28 smithi136 bash[22601]: cluster 2024-09-12T23:28:27.408372+0000 mgr.y (mgr.14150) 162 : cluster [DBG] pgmap v126: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:28.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:28 smithi136 bash[32189]: cluster 2024-09-12T23:28:27.408372+0000 mgr.y (mgr.14150) 162 : cluster [DBG] pgmap v126: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:28.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:28 smithi181 bash[24776]: cluster 2024-09-12T23:28:27.408372+0000 mgr.y (mgr.14150) 162 : cluster [DBG] pgmap v126: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:30.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:30 smithi181 bash[24776]: audit 2024-09-12T23:28:29.283007+0000 mon.a (mon.0) 344 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:28:30.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:30 smithi181 bash[24776]: audit 2024-09-12T23:28:29.289505+0000 mon.a (mon.0) 345 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:28:30.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:30 smithi181 bash[24776]: cluster 2024-09-12T23:28:29.408926+0000 mgr.y (mgr.14150) 163 : cluster [DBG] pgmap v127: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:30.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:30 smithi181 bash[24776]: audit 2024-09-12T23:28:29.637460+0000 mon.a (mon.0) 346 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:28:30.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:30 smithi181 bash[24776]: audit 2024-09-12T23:28:29.638173+0000 mon.a (mon.0) 347 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:28:30.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:30 smithi181 bash[24776]: audit 2024-09-12T23:28:29.645456+0000 mon.a (mon.0) 348 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:28:30.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:30 smithi136 bash[32189]: audit 2024-09-12T23:28:29.283007+0000 mon.a (mon.0) 344 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:28:30.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:30 smithi136 bash[32189]: audit 2024-09-12T23:28:29.289505+0000 mon.a (mon.0) 345 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:28:30.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:30 smithi136 bash[32189]: cluster 2024-09-12T23:28:29.408926+0000 mgr.y (mgr.14150) 163 : cluster [DBG] pgmap v127: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:30.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:30 smithi136 bash[32189]: audit 2024-09-12T23:28:29.637460+0000 mon.a (mon.0) 346 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:28:30.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:30 smithi136 bash[32189]: audit 2024-09-12T23:28:29.638173+0000 mon.a (mon.0) 347 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:28:30.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:30 smithi136 bash[32189]: audit 2024-09-12T23:28:29.645456+0000 mon.a (mon.0) 348 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:28:30.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:30 smithi136 bash[22601]: audit 2024-09-12T23:28:29.283007+0000 mon.a (mon.0) 344 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:28:30.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:30 smithi136 bash[22601]: audit 2024-09-12T23:28:29.289505+0000 mon.a (mon.0) 345 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:28:30.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:30 smithi136 bash[22601]: cluster 2024-09-12T23:28:29.408926+0000 mgr.y (mgr.14150) 163 : cluster [DBG] pgmap v127: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:30.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:30 smithi136 bash[22601]: audit 2024-09-12T23:28:29.637460+0000 mon.a (mon.0) 346 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:28:30.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:30 smithi136 bash[22601]: audit 2024-09-12T23:28:29.638173+0000 mon.a (mon.0) 347 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:28:30.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:30 smithi136 bash[22601]: audit 2024-09-12T23:28:29.645456+0000 mon.a (mon.0) 348 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:28:31.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:31 smithi181 bash[24776]: audit 2024-09-12T23:28:30.800745+0000 mon.c (mon.2) 9 : audit [INF] from='osd.2 [v2:172.21.15.136:6818/308477304,v1:172.21.15.136:6819/308477304]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-12T23:28:31.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:31 smithi181 bash[24776]: audit 2024-09-12T23:28:30.801056+0000 mon.a (mon.0) 349 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-12T23:28:31.748 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:31 smithi136 bash[22601]: audit 2024-09-12T23:28:30.800745+0000 mon.c (mon.2) 9 : audit [INF] from='osd.2 [v2:172.21.15.136:6818/308477304,v1:172.21.15.136:6819/308477304]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-12T23:28:31.748 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:31 smithi136 bash[22601]: audit 2024-09-12T23:28:30.801056+0000 mon.a (mon.0) 349 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-12T23:28:31.748 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:31 smithi136 bash[32189]: audit 2024-09-12T23:28:30.800745+0000 mon.c (mon.2) 9 : audit [INF] from='osd.2 [v2:172.21.15.136:6818/308477304,v1:172.21.15.136:6819/308477304]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-12T23:28:31.748 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:31 smithi136 bash[32189]: audit 2024-09-12T23:28:30.801056+0000 mon.a (mon.0) 349 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-12T23:28:32.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:32 smithi136 bash[32189]: audit 2024-09-12T23:28:31.305655+0000 mon.a (mon.0) 350 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-12T23:28:32.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:32 smithi136 bash[32189]: cluster 2024-09-12T23:28:31.305826+0000 mon.a (mon.0) 351 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2024-09-12T23:28:32.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:32 smithi136 bash[32189]: audit 2024-09-12T23:28:31.306180+0000 mon.a (mon.0) 352 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-12T23:28:32.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:32 smithi136 bash[32189]: audit 2024-09-12T23:28:31.307659+0000 mon.c (mon.2) 10 : audit [INF] from='osd.2 [v2:172.21.15.136:6818/308477304,v1:172.21.15.136:6819/308477304]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-12T23:28:32.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:32 smithi136 bash[32189]: audit 2024-09-12T23:28:31.308473+0000 mon.a (mon.0) 353 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-12T23:28:32.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:32 smithi136 bash[32189]: cluster 2024-09-12T23:28:31.409444+0000 mgr.y (mgr.14150) 164 : cluster [DBG] pgmap v129: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:32.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:32 smithi136 bash[22601]: audit 2024-09-12T23:28:31.305655+0000 mon.a (mon.0) 350 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-12T23:28:32.511 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:32 smithi136 bash[22601]: cluster 2024-09-12T23:28:31.305826+0000 mon.a (mon.0) 351 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2024-09-12T23:28:32.511 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:32 smithi136 bash[22601]: audit 2024-09-12T23:28:31.306180+0000 mon.a (mon.0) 352 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-12T23:28:32.511 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:32 smithi136 bash[22601]: audit 2024-09-12T23:28:31.307659+0000 mon.c (mon.2) 10 : audit [INF] from='osd.2 [v2:172.21.15.136:6818/308477304,v1:172.21.15.136:6819/308477304]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-12T23:28:32.511 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:32 smithi136 bash[22601]: audit 2024-09-12T23:28:31.308473+0000 mon.a (mon.0) 353 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-12T23:28:32.511 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:32 smithi136 bash[22601]: cluster 2024-09-12T23:28:31.409444+0000 mgr.y (mgr.14150) 164 : cluster [DBG] pgmap v129: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:32.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:32 smithi181 bash[24776]: audit 2024-09-12T23:28:31.305655+0000 mon.a (mon.0) 350 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-12T23:28:32.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:32 smithi181 bash[24776]: cluster 2024-09-12T23:28:31.305826+0000 mon.a (mon.0) 351 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2024-09-12T23:28:32.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:32 smithi181 bash[24776]: audit 2024-09-12T23:28:31.306180+0000 mon.a (mon.0) 352 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-12T23:28:32.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:32 smithi181 bash[24776]: audit 2024-09-12T23:28:31.307659+0000 mon.c (mon.2) 10 : audit [INF] from='osd.2 [v2:172.21.15.136:6818/308477304,v1:172.21.15.136:6819/308477304]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-12T23:28:32.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:32 smithi181 bash[24776]: audit 2024-09-12T23:28:31.308473+0000 mon.a (mon.0) 353 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-12T23:28:32.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:32 smithi181 bash[24776]: cluster 2024-09-12T23:28:31.409444+0000 mgr.y (mgr.14150) 164 : cluster [DBG] pgmap v129: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:32.780 INFO:teuthology.orchestra.run.smithi136.stdout:Created osd(s) 2 on host 'smithi136' 2024-09-12T23:28:33.367 DEBUG:teuthology.orchestra.run.smithi136:osd.2> sudo journalctl -f -n 0 -u ceph-da58a47c-715d-11ef-bceb-c7b262605968@osd.2.service 2024-09-12T23:28:33.369 INFO:tasks.cephadm:Deploying osd.3 on smithi136 with /dev/vg_nvme/lv_1... 2024-09-12T23:28:33.370 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-09-12T23:28:33.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:33 smithi181 bash[24776]: audit 2024-09-12T23:28:32.308120+0000 mon.a (mon.0) 354 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]': finished 2024-09-12T23:28:33.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:33 smithi181 bash[24776]: cluster 2024-09-12T23:28:32.308351+0000 mon.a (mon.0) 355 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2024-09-12T23:28:33.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:33 smithi181 bash[24776]: audit 2024-09-12T23:28:32.309163+0000 mon.a (mon.0) 356 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-12T23:28:33.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:33 smithi181 bash[24776]: audit 2024-09-12T23:28:32.317713+0000 mon.a (mon.0) 357 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-12T23:28:33.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:33 smithi181 bash[24776]: audit 2024-09-12T23:28:32.765208+0000 mon.a (mon.0) 358 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:28:33.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:33 smithi181 bash[24776]: audit 2024-09-12T23:28:32.770363+0000 mon.a (mon.0) 359 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:28:33.600 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:33 smithi181 bash[24776]: audit 2024-09-12T23:28:32.777737+0000 mon.a (mon.0) 360 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:28:33.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:33 smithi136 bash[22601]: audit 2024-09-12T23:28:32.308120+0000 mon.a (mon.0) 354 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]': finished 2024-09-12T23:28:33.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:33 smithi136 bash[22601]: cluster 2024-09-12T23:28:32.308351+0000 mon.a (mon.0) 355 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2024-09-12T23:28:33.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:33 smithi136 bash[22601]: audit 2024-09-12T23:28:32.309163+0000 mon.a (mon.0) 356 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-12T23:28:33.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:33 smithi136 bash[22601]: audit 2024-09-12T23:28:32.317713+0000 mon.a (mon.0) 357 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-12T23:28:33.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:33 smithi136 bash[22601]: audit 2024-09-12T23:28:32.765208+0000 mon.a (mon.0) 358 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:28:33.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:33 smithi136 bash[22601]: audit 2024-09-12T23:28:32.770363+0000 mon.a (mon.0) 359 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:28:33.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:33 smithi136 bash[22601]: audit 2024-09-12T23:28:32.777737+0000 mon.a (mon.0) 360 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:28:33.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:33 smithi136 bash[32189]: audit 2024-09-12T23:28:32.308120+0000 mon.a (mon.0) 354 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]': finished 2024-09-12T23:28:33.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:33 smithi136 bash[32189]: cluster 2024-09-12T23:28:32.308351+0000 mon.a (mon.0) 355 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2024-09-12T23:28:33.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:33 smithi136 bash[32189]: audit 2024-09-12T23:28:32.309163+0000 mon.a (mon.0) 356 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-12T23:28:33.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:33 smithi136 bash[32189]: audit 2024-09-12T23:28:32.317713+0000 mon.a (mon.0) 357 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-12T23:28:33.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:33 smithi136 bash[32189]: audit 2024-09-12T23:28:32.765208+0000 mon.a (mon.0) 358 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:28:33.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:33 smithi136 bash[32189]: audit 2024-09-12T23:28:32.770363+0000 mon.a (mon.0) 359 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:28:33.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:33 smithi136 bash[32189]: audit 2024-09-12T23:28:32.777737+0000 mon.a (mon.0) 360 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:28:34.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:34 smithi181 bash[24776]: cluster 2024-09-12T23:28:31.754922+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-12T23:28:34.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:34 smithi181 bash[24776]: cluster 2024-09-12T23:28:31.755008+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-12T23:28:34.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:34 smithi181 bash[24776]: audit 2024-09-12T23:28:33.321083+0000 mon.a (mon.0) 361 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-12T23:28:34.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:34 smithi181 bash[24776]: cluster 2024-09-12T23:28:33.328201+0000 mon.a (mon.0) 362 : cluster [INF] osd.2 [v2:172.21.15.136:6818/308477304,v1:172.21.15.136:6819/308477304] boot 2024-09-12T23:28:34.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:34 smithi181 bash[24776]: cluster 2024-09-12T23:28:33.328310+0000 mon.a (mon.0) 363 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2024-09-12T23:28:34.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:34 smithi181 bash[24776]: audit 2024-09-12T23:28:33.328581+0000 mon.a (mon.0) 364 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-12T23:28:34.600 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:34 smithi181 bash[24776]: cluster 2024-09-12T23:28:33.409890+0000 mgr.y (mgr.14150) 165 : cluster [DBG] pgmap v132: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:34.600 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:34 smithi181 bash[24776]: audit 2024-09-12T23:28:33.521045+0000 mon.a (mon.0) 365 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-12T23:28:34.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:34 smithi136 bash[22601]: cluster 2024-09-12T23:28:31.754922+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-12T23:28:34.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:34 smithi136 bash[22601]: cluster 2024-09-12T23:28:31.755008+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-12T23:28:34.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:34 smithi136 bash[22601]: audit 2024-09-12T23:28:33.321083+0000 mon.a (mon.0) 361 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-12T23:28:34.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:34 smithi136 bash[22601]: cluster 2024-09-12T23:28:33.328201+0000 mon.a (mon.0) 362 : cluster [INF] osd.2 [v2:172.21.15.136:6818/308477304,v1:172.21.15.136:6819/308477304] boot 2024-09-12T23:28:34.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:34 smithi136 bash[22601]: cluster 2024-09-12T23:28:33.328310+0000 mon.a (mon.0) 363 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2024-09-12T23:28:34.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:34 smithi136 bash[22601]: audit 2024-09-12T23:28:33.328581+0000 mon.a (mon.0) 364 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-12T23:28:34.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:34 smithi136 bash[22601]: cluster 2024-09-12T23:28:33.409890+0000 mgr.y (mgr.14150) 165 : cluster [DBG] pgmap v132: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:34.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:34 smithi136 bash[22601]: audit 2024-09-12T23:28:33.521045+0000 mon.a (mon.0) 365 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-12T23:28:34.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:34 smithi136 bash[32189]: cluster 2024-09-12T23:28:31.754922+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-12T23:28:34.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:34 smithi136 bash[32189]: cluster 2024-09-12T23:28:31.755008+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-12T23:28:34.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:34 smithi136 bash[32189]: audit 2024-09-12T23:28:33.321083+0000 mon.a (mon.0) 361 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-12T23:28:34.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:34 smithi136 bash[32189]: cluster 2024-09-12T23:28:33.328201+0000 mon.a (mon.0) 362 : cluster [INF] osd.2 [v2:172.21.15.136:6818/308477304,v1:172.21.15.136:6819/308477304] boot 2024-09-12T23:28:34.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:34 smithi136 bash[32189]: cluster 2024-09-12T23:28:33.328310+0000 mon.a (mon.0) 363 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2024-09-12T23:28:34.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:34 smithi136 bash[32189]: audit 2024-09-12T23:28:33.328581+0000 mon.a (mon.0) 364 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-12T23:28:34.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:34 smithi136 bash[32189]: cluster 2024-09-12T23:28:33.409890+0000 mgr.y (mgr.14150) 165 : cluster [DBG] pgmap v132: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-12T23:28:34.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:34 smithi136 bash[32189]: audit 2024-09-12T23:28:33.521045+0000 mon.a (mon.0) 365 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-12T23:28:35.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:35 smithi181 bash[24776]: audit 2024-09-12T23:28:34.336567+0000 mon.a (mon.0) 366 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-12T23:28:35.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:35 smithi181 bash[24776]: cluster 2024-09-12T23:28:34.336772+0000 mon.a (mon.0) 367 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2024-09-12T23:28:35.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:35 smithi181 bash[24776]: audit 2024-09-12T23:28:34.338602+0000 mon.a (mon.0) 368 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:28:35.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:35 smithi136 bash[22601]: audit 2024-09-12T23:28:34.336567+0000 mon.a (mon.0) 366 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-12T23:28:35.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:35 smithi136 bash[22601]: cluster 2024-09-12T23:28:34.336772+0000 mon.a (mon.0) 367 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2024-09-12T23:28:35.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:35 smithi136 bash[22601]: audit 2024-09-12T23:28:34.338602+0000 mon.a (mon.0) 368 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:28:35.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:35 smithi136 bash[32189]: audit 2024-09-12T23:28:34.336567+0000 mon.a (mon.0) 366 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-12T23:28:35.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:35 smithi136 bash[32189]: cluster 2024-09-12T23:28:34.336772+0000 mon.a (mon.0) 367 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2024-09-12T23:28:35.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:35 smithi136 bash[32189]: audit 2024-09-12T23:28:34.338602+0000 mon.a (mon.0) 368 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:28:36.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:36 smithi136 bash[22601]: audit 2024-09-12T23:28:35.338487+0000 mon.a (mon.0) 369 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-12T23:28:36.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:36 smithi136 bash[22601]: cluster 2024-09-12T23:28:35.338623+0000 mon.a (mon.0) 370 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2024-09-12T23:28:36.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:36 smithi136 bash[22601]: cluster 2024-09-12T23:28:35.410427+0000 mgr.y (mgr.14150) 166 : cluster [DBG] pgmap v135: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:28:36.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:36 smithi136 bash[32189]: audit 2024-09-12T23:28:35.338487+0000 mon.a (mon.0) 369 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-12T23:28:36.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:36 smithi136 bash[32189]: cluster 2024-09-12T23:28:35.338623+0000 mon.a (mon.0) 370 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2024-09-12T23:28:36.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:36 smithi136 bash[32189]: cluster 2024-09-12T23:28:35.410427+0000 mgr.y (mgr.14150) 166 : cluster [DBG] pgmap v135: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:28:36.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:36 smithi181 bash[24776]: audit 2024-09-12T23:28:35.338487+0000 mon.a (mon.0) 369 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-12T23:28:36.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:36 smithi181 bash[24776]: cluster 2024-09-12T23:28:35.338623+0000 mon.a (mon.0) 370 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2024-09-12T23:28:36.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:36 smithi181 bash[24776]: cluster 2024-09-12T23:28:35.410427+0000 mgr.y (mgr.14150) 166 : cluster [DBG] pgmap v135: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:28:37.542 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:28:37.689 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:37 smithi136 bash[32189]: cluster 2024-09-12T23:28:36.354080+0000 mon.a (mon.0) 371 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2024-09-12T23:28:37.689 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:37 smithi136 bash[32189]: audit 2024-09-12T23:28:36.875104+0000 mon.a (mon.0) 372 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:28:37.689 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:37 smithi136 bash[22601]: cluster 2024-09-12T23:28:36.354080+0000 mon.a (mon.0) 371 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2024-09-12T23:28:37.690 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:37 smithi136 bash[22601]: audit 2024-09-12T23:28:36.875104+0000 mon.a (mon.0) 372 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:28:37.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:37 smithi181 bash[24776]: cluster 2024-09-12T23:28:36.354080+0000 mon.a (mon.0) 371 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2024-09-12T23:28:37.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:37 smithi181 bash[24776]: audit 2024-09-12T23:28:36.875104+0000 mon.a (mon.0) 372 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:28:38.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:38 smithi136 bash[22601]: cluster 2024-09-12T23:28:37.410927+0000 mgr.y (mgr.14150) 167 : cluster [DBG] pgmap v137: 1 pgs: 1 active+clean; 320 KiB data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:28:38.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:38 smithi136 bash[32189]: cluster 2024-09-12T23:28:37.410927+0000 mgr.y (mgr.14150) 167 : cluster [DBG] pgmap v137: 1 pgs: 1 active+clean; 320 KiB data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:28:38.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:38 smithi181 bash[24776]: cluster 2024-09-12T23:28:37.410927+0000 mgr.y (mgr.14150) 167 : cluster [DBG] pgmap v137: 1 pgs: 1 active+clean; 320 KiB data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:28:39.840 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:39 smithi136 bash[22601]: cluster 2024-09-12T23:28:38.374986+0000 mon.a (mon.0) 373 : cluster [DBG] mgrmap e15: y(active, since 4m), standbys: x 2024-09-12T23:28:39.841 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:39 smithi136 bash[32189]: cluster 2024-09-12T23:28:38.374986+0000 mon.a (mon.0) 373 : cluster [DBG] mgrmap e15: y(active, since 4m), standbys: x 2024-09-12T23:28:39.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:39 smithi181 bash[24776]: cluster 2024-09-12T23:28:38.374986+0000 mon.a (mon.0) 373 : cluster [DBG] mgrmap e15: y(active, since 4m), standbys: x 2024-09-12T23:28:41.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:41 smithi136 bash[22601]: audit 2024-09-12T23:28:38.671442+0000 mon.a (mon.0) 374 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-12T23:28:41.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:41 smithi136 bash[22601]: cluster 2024-09-12T23:28:39.411270+0000 mgr.y (mgr.14150) 168 : cluster [DBG] pgmap v138: 1 pgs: 1 active+clean; 320 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:28:41.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:41 smithi136 bash[22601]: audit 2024-09-12T23:28:39.556195+0000 mon.a (mon.0) 375 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-12T23:28:41.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:41 smithi136 bash[22601]: audit 2024-09-12T23:28:39.556426+0000 mon.a (mon.0) 376 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:28:41.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:41 smithi136 bash[22601]: audit 2024-09-12T23:28:39.557527+0000 mon.a (mon.0) 377 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:28:41.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:41 smithi136 bash[22601]: audit 2024-09-12T23:28:39.561429+0000 mon.a (mon.0) 378 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:28:41.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:41 smithi136 bash[22601]: audit 2024-09-12T23:28:39.563811+0000 mon.b (mon.1) 7 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-12T23:28:41.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:41 smithi136 bash[22601]: audit 2024-09-12T23:28:39.564040+0000 mon.a (mon.0) 379 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:28:41.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:41 smithi136 bash[22601]: audit 2024-09-12T23:28:39.564228+0000 mon.a (mon.0) 380 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:28:41.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:41 smithi136 bash[22601]: audit 2024-09-12T23:28:39.564358+0000 mon.a (mon.0) 381 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:28:41.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:41 smithi136 bash[22601]: audit 2024-09-12T23:28:40.462987+0000 mon.b (mon.1) 8 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-12T23:28:41.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:41 smithi136 bash[22601]: audit 2024-09-12T23:28:40.511765+0000 mon.a (mon.0) 382 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:28:41.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:41 smithi136 bash[22601]: audit 2024-09-12T23:28:40.514710+0000 mon.a (mon.0) 383 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:28:41.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:41 smithi136 bash[22601]: audit 2024-09-12T23:28:40.515032+0000 mon.a (mon.0) 384 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:28:41.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:41 smithi136 bash[22601]: audit 2024-09-12T23:28:40.515319+0000 mon.a (mon.0) 385 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:28:41.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:41 smithi136 bash[32189]: audit 2024-09-12T23:28:38.671442+0000 mon.a (mon.0) 374 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-12T23:28:41.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:41 smithi136 bash[32189]: cluster 2024-09-12T23:28:39.411270+0000 mgr.y (mgr.14150) 168 : cluster [DBG] pgmap v138: 1 pgs: 1 active+clean; 320 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:28:41.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:41 smithi136 bash[32189]: audit 2024-09-12T23:28:39.556195+0000 mon.a (mon.0) 375 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-12T23:28:41.762 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:41 smithi136 bash[32189]: audit 2024-09-12T23:28:39.556426+0000 mon.a (mon.0) 376 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:28:41.762 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:41 smithi136 bash[32189]: audit 2024-09-12T23:28:39.557527+0000 mon.a (mon.0) 377 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:28:41.762 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:41 smithi136 bash[32189]: audit 2024-09-12T23:28:39.561429+0000 mon.a (mon.0) 378 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:28:41.762 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:41 smithi136 bash[32189]: audit 2024-09-12T23:28:39.563811+0000 mon.b (mon.1) 7 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-12T23:28:41.762 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:41 smithi136 bash[32189]: audit 2024-09-12T23:28:39.564040+0000 mon.a (mon.0) 379 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:28:41.762 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:41 smithi136 bash[32189]: audit 2024-09-12T23:28:39.564228+0000 mon.a (mon.0) 380 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:28:41.762 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:41 smithi136 bash[32189]: audit 2024-09-12T23:28:39.564358+0000 mon.a (mon.0) 381 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:28:41.762 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:41 smithi136 bash[32189]: audit 2024-09-12T23:28:40.462987+0000 mon.b (mon.1) 8 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-12T23:28:41.762 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:41 smithi136 bash[32189]: audit 2024-09-12T23:28:40.511765+0000 mon.a (mon.0) 382 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:28:41.762 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:41 smithi136 bash[32189]: audit 2024-09-12T23:28:40.514710+0000 mon.a (mon.0) 383 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:28:41.762 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:41 smithi136 bash[32189]: audit 2024-09-12T23:28:40.515032+0000 mon.a (mon.0) 384 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:28:41.762 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:41 smithi136 bash[32189]: audit 2024-09-12T23:28:40.515319+0000 mon.a (mon.0) 385 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:28:41.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:41 smithi181 bash[24776]: audit 2024-09-12T23:28:38.671442+0000 mon.a (mon.0) 374 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-12T23:28:41.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:41 smithi181 bash[24776]: cluster 2024-09-12T23:28:39.411270+0000 mgr.y (mgr.14150) 168 : cluster [DBG] pgmap v138: 1 pgs: 1 active+clean; 320 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:28:41.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:41 smithi181 bash[24776]: audit 2024-09-12T23:28:39.556195+0000 mon.a (mon.0) 375 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-12T23:28:41.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:41 smithi181 bash[24776]: audit 2024-09-12T23:28:39.556426+0000 mon.a (mon.0) 376 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:28:41.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:41 smithi181 bash[24776]: audit 2024-09-12T23:28:39.557527+0000 mon.a (mon.0) 377 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:28:41.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:41 smithi181 bash[24776]: audit 2024-09-12T23:28:39.561429+0000 mon.a (mon.0) 378 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:28:41.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:41 smithi181 bash[24776]: audit 2024-09-12T23:28:39.563811+0000 mon.b (mon.1) 7 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-12T23:28:41.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:41 smithi181 bash[24776]: audit 2024-09-12T23:28:39.564040+0000 mon.a (mon.0) 379 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:28:41.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:41 smithi181 bash[24776]: audit 2024-09-12T23:28:39.564228+0000 mon.a (mon.0) 380 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:28:41.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:41 smithi181 bash[24776]: audit 2024-09-12T23:28:39.564358+0000 mon.a (mon.0) 381 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:28:41.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:41 smithi181 bash[24776]: audit 2024-09-12T23:28:40.462987+0000 mon.b (mon.1) 8 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-12T23:28:41.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:41 smithi181 bash[24776]: audit 2024-09-12T23:28:40.511765+0000 mon.a (mon.0) 382 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:28:41.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:41 smithi181 bash[24776]: audit 2024-09-12T23:28:40.514710+0000 mon.a (mon.0) 383 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:28:41.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:41 smithi181 bash[24776]: audit 2024-09-12T23:28:40.515032+0000 mon.a (mon.0) 384 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:28:41.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:41 smithi181 bash[24776]: audit 2024-09-12T23:28:40.515319+0000 mon.a (mon.0) 385 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:28:42.190 INFO:teuthology.orchestra.run.smithi136.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-09-12T23:28:42.379 INFO:teuthology.orchestra.run.smithi136.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-09-12T23:28:42.394 INFO:teuthology.orchestra.run.smithi136.stderr: stderr: 10+0 records in 2024-09-12T23:28:42.394 INFO:teuthology.orchestra.run.smithi136.stderr:10+0 records out 2024-09-12T23:28:42.394 INFO:teuthology.orchestra.run.smithi136.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0133904 s, 783 MB/s 2024-09-12T23:28:42.395 INFO:teuthology.orchestra.run.smithi136.stderr:--> Zapping successful for: 2024-09-12T23:28:42.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:42 smithi136 bash[32189]: audit 2024-09-12T23:28:40.514290+0000 mon.c (mon.2) 11 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-12T23:28:42.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:42 smithi136 bash[32189]: cluster 2024-09-12T23:28:41.411734+0000 mgr.y (mgr.14150) 169 : cluster [DBG] pgmap v139: 1 pgs: 1 active+clean; 320 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:28:42.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:42 smithi136 bash[32189]: audit 2024-09-12T23:28:41.425079+0000 mon.c (mon.2) 12 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-12T23:28:42.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:42 smithi136 bash[22601]: audit 2024-09-12T23:28:40.514290+0000 mon.c (mon.2) 11 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-12T23:28:42.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:42 smithi136 bash[22601]: cluster 2024-09-12T23:28:41.411734+0000 mgr.y (mgr.14150) 169 : cluster [DBG] pgmap v139: 1 pgs: 1 active+clean; 320 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:28:42.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:42 smithi136 bash[22601]: audit 2024-09-12T23:28:41.425079+0000 mon.c (mon.2) 12 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-12T23:28:42.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:42 smithi181 bash[24776]: audit 2024-09-12T23:28:40.514290+0000 mon.c (mon.2) 11 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-12T23:28:42.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:42 smithi181 bash[24776]: cluster 2024-09-12T23:28:41.411734+0000 mgr.y (mgr.14150) 169 : cluster [DBG] pgmap v139: 1 pgs: 1 active+clean; 320 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:28:42.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:42 smithi181 bash[24776]: audit 2024-09-12T23:28:41.425079+0000 mon.c (mon.2) 12 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-12T23:28:43.329 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi136:vg_nvme/lv_1 2024-09-12T23:28:44.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:44 smithi136 bash[22601]: cephadm 2024-09-12T23:28:43.372023+0000 mgr.y (mgr.14150) 170 : cephadm [INF] Detected new or changed devices on smithi136 2024-09-12T23:28:44.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:44 smithi136 bash[22601]: audit 2024-09-12T23:28:43.380332+0000 mon.a (mon.0) 386 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:28:44.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:44 smithi136 bash[22601]: audit 2024-09-12T23:28:43.390707+0000 mon.a (mon.0) 387 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:28:44.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:44 smithi136 bash[22601]: audit 2024-09-12T23:28:43.392640+0000 mon.a (mon.0) 388 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:28:44.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:44 smithi136 bash[22601]: audit 2024-09-12T23:28:43.393850+0000 mon.a (mon.0) 389 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:28:44.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:44 smithi136 bash[22601]: audit 2024-09-12T23:28:43.394995+0000 mon.a (mon.0) 390 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:28:44.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:44 smithi136 bash[22601]: cephadm 2024-09-12T23:28:43.395788+0000 mgr.y (mgr.14150) 171 : cephadm [INF] Adjusting osd_memory_target on smithi136 to 5413M 2024-09-12T23:28:44.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:44 smithi136 bash[22601]: audit 2024-09-12T23:28:43.405945+0000 mon.a (mon.0) 391 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:28:44.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:44 smithi136 bash[22601]: audit 2024-09-12T23:28:43.407388+0000 mon.a (mon.0) 392 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:28:44.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:44 smithi136 bash[22601]: audit 2024-09-12T23:28:43.408337+0000 mon.a (mon.0) 393 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:28:44.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:44 smithi136 bash[22601]: cluster 2024-09-12T23:28:43.412096+0000 mgr.y (mgr.14150) 172 : cluster [DBG] pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 4.5 KiB/s rd, 218 KiB/s wr, 14 op/s 2024-09-12T23:28:44.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:44 smithi136 bash[22601]: audit 2024-09-12T23:28:43.418203+0000 mon.a (mon.0) 394 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:28:44.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:44 smithi136 bash[32189]: cephadm 2024-09-12T23:28:43.372023+0000 mgr.y (mgr.14150) 170 : cephadm [INF] Detected new or changed devices on smithi136 2024-09-12T23:28:44.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:44 smithi136 bash[32189]: audit 2024-09-12T23:28:43.380332+0000 mon.a (mon.0) 386 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:28:44.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:44 smithi136 bash[32189]: audit 2024-09-12T23:28:43.390707+0000 mon.a (mon.0) 387 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:28:44.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:44 smithi136 bash[32189]: audit 2024-09-12T23:28:43.392640+0000 mon.a (mon.0) 388 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:28:44.762 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:44 smithi136 bash[32189]: audit 2024-09-12T23:28:43.393850+0000 mon.a (mon.0) 389 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:28:44.762 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:44 smithi136 bash[32189]: audit 2024-09-12T23:28:43.394995+0000 mon.a (mon.0) 390 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:28:44.762 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:44 smithi136 bash[32189]: cephadm 2024-09-12T23:28:43.395788+0000 mgr.y (mgr.14150) 171 : cephadm [INF] Adjusting osd_memory_target on smithi136 to 5413M 2024-09-12T23:28:44.762 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:44 smithi136 bash[32189]: audit 2024-09-12T23:28:43.405945+0000 mon.a (mon.0) 391 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:28:44.762 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:44 smithi136 bash[32189]: audit 2024-09-12T23:28:43.407388+0000 mon.a (mon.0) 392 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:28:44.762 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:44 smithi136 bash[32189]: audit 2024-09-12T23:28:43.408337+0000 mon.a (mon.0) 393 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:28:44.762 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:44 smithi136 bash[32189]: cluster 2024-09-12T23:28:43.412096+0000 mgr.y (mgr.14150) 172 : cluster [DBG] pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 4.5 KiB/s rd, 218 KiB/s wr, 14 op/s 2024-09-12T23:28:44.762 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:44 smithi136 bash[32189]: audit 2024-09-12T23:28:43.418203+0000 mon.a (mon.0) 394 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:28:44.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:44 smithi181 bash[24776]: cephadm 2024-09-12T23:28:43.372023+0000 mgr.y (mgr.14150) 170 : cephadm [INF] Detected new or changed devices on smithi136 2024-09-12T23:28:44.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:44 smithi181 bash[24776]: audit 2024-09-12T23:28:43.380332+0000 mon.a (mon.0) 386 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:28:44.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:44 smithi181 bash[24776]: audit 2024-09-12T23:28:43.390707+0000 mon.a (mon.0) 387 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:28:44.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:44 smithi181 bash[24776]: audit 2024-09-12T23:28:43.392640+0000 mon.a (mon.0) 388 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:28:44.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:44 smithi181 bash[24776]: audit 2024-09-12T23:28:43.393850+0000 mon.a (mon.0) 389 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:28:44.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:44 smithi181 bash[24776]: audit 2024-09-12T23:28:43.394995+0000 mon.a (mon.0) 390 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:28:44.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:44 smithi181 bash[24776]: cephadm 2024-09-12T23:28:43.395788+0000 mgr.y (mgr.14150) 171 : cephadm [INF] Adjusting osd_memory_target on smithi136 to 5413M 2024-09-12T23:28:44.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:44 smithi181 bash[24776]: audit 2024-09-12T23:28:43.405945+0000 mon.a (mon.0) 391 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:28:44.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:44 smithi181 bash[24776]: audit 2024-09-12T23:28:43.407388+0000 mon.a (mon.0) 392 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:28:44.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:44 smithi181 bash[24776]: audit 2024-09-12T23:28:43.408337+0000 mon.a (mon.0) 393 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:28:44.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:44 smithi181 bash[24776]: cluster 2024-09-12T23:28:43.412096+0000 mgr.y (mgr.14150) 172 : cluster [DBG] pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 4.5 KiB/s rd, 218 KiB/s wr, 14 op/s 2024-09-12T23:28:44.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:44 smithi181 bash[24776]: audit 2024-09-12T23:28:43.418203+0000 mon.a (mon.0) 394 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:28:46.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:46 smithi136 bash[32189]: cluster 2024-09-12T23:28:45.412582+0000 mgr.y (mgr.14150) 173 : cluster [DBG] pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.4 KiB/s rd, 163 KiB/s wr, 11 op/s 2024-09-12T23:28:46.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:46 smithi136 bash[22601]: cluster 2024-09-12T23:28:45.412582+0000 mgr.y (mgr.14150) 173 : cluster [DBG] pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.4 KiB/s rd, 163 KiB/s wr, 11 op/s 2024-09-12T23:28:46.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:46 smithi181 bash[24776]: cluster 2024-09-12T23:28:45.412582+0000 mgr.y (mgr.14150) 173 : cluster [DBG] pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.4 KiB/s rd, 163 KiB/s wr, 11 op/s 2024-09-12T23:28:48.087 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:28:48.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:48 smithi136 bash[22601]: cluster 2024-09-12T23:28:47.413145+0000 mgr.y (mgr.14150) 174 : cluster [DBG] pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.7 KiB/s rd, 131 KiB/s wr, 8 op/s 2024-09-12T23:28:48.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:48 smithi136 bash[32189]: cluster 2024-09-12T23:28:47.413145+0000 mgr.y (mgr.14150) 174 : cluster [DBG] pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.7 KiB/s rd, 131 KiB/s wr, 8 op/s 2024-09-12T23:28:48.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:48 smithi181 bash[24776]: cluster 2024-09-12T23:28:47.413145+0000 mgr.y (mgr.14150) 174 : cluster [DBG] pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.7 KiB/s rd, 131 KiB/s wr, 8 op/s 2024-09-12T23:28:50.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:50 smithi136 bash[32189]: cluster 2024-09-12T23:28:49.413626+0000 mgr.y (mgr.14150) 175 : cluster [DBG] pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 109 KiB/s wr, 7 op/s 2024-09-12T23:28:50.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:50 smithi136 bash[22601]: cluster 2024-09-12T23:28:49.413626+0000 mgr.y (mgr.14150) 175 : cluster [DBG] pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 109 KiB/s wr, 7 op/s 2024-09-12T23:28:50.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:50 smithi181 bash[24776]: cluster 2024-09-12T23:28:49.413626+0000 mgr.y (mgr.14150) 175 : cluster [DBG] pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 109 KiB/s wr, 7 op/s 2024-09-12T23:28:51.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:51 smithi136 bash[22601]: audit 2024-09-12T23:28:51.129736+0000 mon.a (mon.0) 395 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-12T23:28:51.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:51 smithi136 bash[22601]: audit 2024-09-12T23:28:51.135242+0000 mon.a (mon.0) 396 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-12T23:28:51.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:51 smithi136 bash[22601]: audit 2024-09-12T23:28:51.136491+0000 mon.a (mon.0) 397 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:28:51.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:51 smithi136 bash[32189]: audit 2024-09-12T23:28:51.129736+0000 mon.a (mon.0) 395 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-12T23:28:51.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:51 smithi136 bash[32189]: audit 2024-09-12T23:28:51.135242+0000 mon.a (mon.0) 396 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-12T23:28:51.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:51 smithi136 bash[32189]: audit 2024-09-12T23:28:51.136491+0000 mon.a (mon.0) 397 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:28:51.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:51 smithi181 bash[24776]: audit 2024-09-12T23:28:51.129736+0000 mon.a (mon.0) 395 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-12T23:28:51.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:51 smithi181 bash[24776]: audit 2024-09-12T23:28:51.135242+0000 mon.a (mon.0) 396 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-12T23:28:51.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:51 smithi181 bash[24776]: audit 2024-09-12T23:28:51.136491+0000 mon.a (mon.0) 397 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:28:52.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:52 smithi136 bash[22601]: audit 2024-09-12T23:28:51.123574+0000 mgr.y (mgr.14150) 176 : audit [DBG] from='client.14295 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi136:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:28:52.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:52 smithi136 bash[22601]: cluster 2024-09-12T23:28:51.414018+0000 mgr.y (mgr.14150) 177 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 109 KiB/s wr, 7 op/s 2024-09-12T23:28:52.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:52 smithi136 bash[32189]: audit 2024-09-12T23:28:51.123574+0000 mgr.y (mgr.14150) 176 : audit [DBG] from='client.14295 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi136:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:28:52.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:52 smithi136 bash[32189]: cluster 2024-09-12T23:28:51.414018+0000 mgr.y (mgr.14150) 177 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 109 KiB/s wr, 7 op/s 2024-09-12T23:28:52.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:52 smithi181 bash[24776]: audit 2024-09-12T23:28:51.123574+0000 mgr.y (mgr.14150) 176 : audit [DBG] from='client.14295 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi136:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:28:52.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:52 smithi181 bash[24776]: cluster 2024-09-12T23:28:51.414018+0000 mgr.y (mgr.14150) 177 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 109 KiB/s wr, 7 op/s 2024-09-12T23:28:54.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:54 smithi136 bash[22601]: cluster 2024-09-12T23:28:53.414526+0000 mgr.y (mgr.14150) 178 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 109 KiB/s wr, 7 op/s 2024-09-12T23:28:54.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:54 smithi136 bash[32189]: cluster 2024-09-12T23:28:53.414526+0000 mgr.y (mgr.14150) 178 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 109 KiB/s wr, 7 op/s 2024-09-12T23:28:54.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:54 smithi181 bash[24776]: cluster 2024-09-12T23:28:53.414526+0000 mgr.y (mgr.14150) 178 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 109 KiB/s wr, 7 op/s 2024-09-12T23:28:56.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:56 smithi136 bash[22601]: cluster 2024-09-12T23:28:55.415043+0000 mgr.y (mgr.14150) 179 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:28:56.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:56 smithi136 bash[32189]: cluster 2024-09-12T23:28:55.415043+0000 mgr.y (mgr.14150) 179 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:28:56.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:56 smithi181 bash[24776]: cluster 2024-09-12T23:28:55.415043+0000 mgr.y (mgr.14150) 179 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:28:58.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:58 smithi136 bash[22601]: cluster 2024-09-12T23:28:57.415587+0000 mgr.y (mgr.14150) 180 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:28:58.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:58 smithi136 bash[32189]: cluster 2024-09-12T23:28:57.415587+0000 mgr.y (mgr.14150) 180 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:28:58.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:58 smithi181 bash[24776]: cluster 2024-09-12T23:28:57.415587+0000 mgr.y (mgr.14150) 180 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:28:59.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:59 smithi136 bash[22601]: audit 2024-09-12T23:28:59.356819+0000 mon.c (mon.2) 13 : audit [INF] from='client.? 172.21.15.136:0/787015978' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d61f8357-b89b-4c05-b366-65a9f788064d"}]: dispatch 2024-09-12T23:28:59.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:59 smithi136 bash[22601]: audit 2024-09-12T23:28:59.357329+0000 mon.a (mon.0) 398 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d61f8357-b89b-4c05-b366-65a9f788064d"}]: dispatch 2024-09-12T23:28:59.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:59 smithi136 bash[22601]: audit 2024-09-12T23:28:59.365342+0000 mon.a (mon.0) 399 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d61f8357-b89b-4c05-b366-65a9f788064d"}]': finished 2024-09-12T23:28:59.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:59 smithi136 bash[22601]: cluster 2024-09-12T23:28:59.365398+0000 mon.a (mon.0) 400 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2024-09-12T23:28:59.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:28:59 smithi136 bash[22601]: audit 2024-09-12T23:28:59.365575+0000 mon.a (mon.0) 401 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-12T23:28:59.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:59 smithi136 bash[32189]: audit 2024-09-12T23:28:59.356819+0000 mon.c (mon.2) 13 : audit [INF] from='client.? 172.21.15.136:0/787015978' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d61f8357-b89b-4c05-b366-65a9f788064d"}]: dispatch 2024-09-12T23:28:59.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:59 smithi136 bash[32189]: audit 2024-09-12T23:28:59.357329+0000 mon.a (mon.0) 398 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d61f8357-b89b-4c05-b366-65a9f788064d"}]: dispatch 2024-09-12T23:28:59.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:59 smithi136 bash[32189]: audit 2024-09-12T23:28:59.365342+0000 mon.a (mon.0) 399 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d61f8357-b89b-4c05-b366-65a9f788064d"}]': finished 2024-09-12T23:28:59.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:59 smithi136 bash[32189]: cluster 2024-09-12T23:28:59.365398+0000 mon.a (mon.0) 400 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2024-09-12T23:28:59.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:28:59 smithi136 bash[32189]: audit 2024-09-12T23:28:59.365575+0000 mon.a (mon.0) 401 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-12T23:28:59.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:59 smithi181 bash[24776]: audit 2024-09-12T23:28:59.356819+0000 mon.c (mon.2) 13 : audit [INF] from='client.? 172.21.15.136:0/787015978' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d61f8357-b89b-4c05-b366-65a9f788064d"}]: dispatch 2024-09-12T23:28:59.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:59 smithi181 bash[24776]: audit 2024-09-12T23:28:59.357329+0000 mon.a (mon.0) 398 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d61f8357-b89b-4c05-b366-65a9f788064d"}]: dispatch 2024-09-12T23:28:59.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:59 smithi181 bash[24776]: audit 2024-09-12T23:28:59.365342+0000 mon.a (mon.0) 399 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d61f8357-b89b-4c05-b366-65a9f788064d"}]': finished 2024-09-12T23:28:59.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:59 smithi181 bash[24776]: cluster 2024-09-12T23:28:59.365398+0000 mon.a (mon.0) 400 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2024-09-12T23:28:59.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:28:59 smithi181 bash[24776]: audit 2024-09-12T23:28:59.365575+0000 mon.a (mon.0) 401 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-12T23:29:00.763 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:00 smithi136 bash[22601]: cluster 2024-09-12T23:28:59.416001+0000 mgr.y (mgr.14150) 181 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:29:00.763 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:00 smithi136 bash[22601]: audit 2024-09-12T23:28:59.991607+0000 mon.c (mon.2) 14 : audit [DBG] from='client.? 172.21.15.136:0/3448887303' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-12T23:29:00.764 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:00 smithi136 bash[32189]: cluster 2024-09-12T23:28:59.416001+0000 mgr.y (mgr.14150) 181 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:29:00.765 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:00 smithi136 bash[32189]: audit 2024-09-12T23:28:59.991607+0000 mon.c (mon.2) 14 : audit [DBG] from='client.? 172.21.15.136:0/3448887303' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-12T23:29:00.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:00 smithi181 bash[24776]: cluster 2024-09-12T23:28:59.416001+0000 mgr.y (mgr.14150) 181 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:29:00.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:00 smithi181 bash[24776]: audit 2024-09-12T23:28:59.991607+0000 mon.c (mon.2) 14 : audit [DBG] from='client.? 172.21.15.136:0/3448887303' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-12T23:29:02.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:02 smithi136 bash[32189]: cluster 2024-09-12T23:29:01.416430+0000 mgr.y (mgr.14150) 182 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:29:02.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:02 smithi136 bash[22601]: cluster 2024-09-12T23:29:01.416430+0000 mgr.y (mgr.14150) 182 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:29:02.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:02 smithi181 bash[24776]: cluster 2024-09-12T23:29:01.416430+0000 mgr.y (mgr.14150) 182 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:29:04.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:04 smithi136 bash[32189]: cluster 2024-09-12T23:29:03.416800+0000 mgr.y (mgr.14150) 183 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:29:04.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:04 smithi136 bash[22601]: cluster 2024-09-12T23:29:03.416800+0000 mgr.y (mgr.14150) 183 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:29:04.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:04 smithi181 bash[24776]: cluster 2024-09-12T23:29:03.416800+0000 mgr.y (mgr.14150) 183 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:29:06.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:06 smithi181 bash[24776]: cluster 2024-09-12T23:29:05.417122+0000 mgr.y (mgr.14150) 184 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:29:07.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:06 smithi136 bash[32189]: cluster 2024-09-12T23:29:05.417122+0000 mgr.y (mgr.14150) 184 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:29:07.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:06 smithi136 bash[22601]: cluster 2024-09-12T23:29:05.417122+0000 mgr.y (mgr.14150) 184 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:29:08.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:08 smithi181 bash[24776]: cluster 2024-09-12T23:29:07.417607+0000 mgr.y (mgr.14150) 185 : cluster [DBG] pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:29:08.888 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:08 smithi136 bash[22601]: cluster 2024-09-12T23:29:07.417607+0000 mgr.y (mgr.14150) 185 : cluster [DBG] pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:29:08.888 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:08 smithi136 bash[32189]: cluster 2024-09-12T23:29:07.417607+0000 mgr.y (mgr.14150) 185 : cluster [DBG] pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:29:10.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:10 smithi181 bash[24776]: cluster 2024-09-12T23:29:09.418114+0000 mgr.y (mgr.14150) 186 : cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:29:10.948 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:10 smithi136 bash[22601]: cluster 2024-09-12T23:29:09.418114+0000 mgr.y (mgr.14150) 186 : cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:29:10.948 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:10 smithi136 bash[32189]: cluster 2024-09-12T23:29:09.418114+0000 mgr.y (mgr.14150) 186 : cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:29:12.787 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:12 smithi136 bash[22601]: cluster 2024-09-12T23:29:11.418589+0000 mgr.y (mgr.14150) 187 : cluster [DBG] pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:29:12.787 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:12 smithi136 bash[22601]: audit 2024-09-12T23:29:11.587409+0000 mon.a (mon.0) 402 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-12T23:29:12.788 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:12 smithi136 bash[22601]: audit 2024-09-12T23:29:11.589086+0000 mon.a (mon.0) 403 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:29:12.788 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:12 smithi136 bash[22601]: cephadm 2024-09-12T23:29:11.590522+0000 mgr.y (mgr.14150) 188 : cephadm [INF] Deploying daemon osd.3 on smithi136 2024-09-12T23:29:12.788 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:12 smithi136 bash[32189]: cluster 2024-09-12T23:29:11.418589+0000 mgr.y (mgr.14150) 187 : cluster [DBG] pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:29:12.788 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:12 smithi136 bash[32189]: audit 2024-09-12T23:29:11.587409+0000 mon.a (mon.0) 402 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-12T23:29:12.788 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:12 smithi136 bash[32189]: audit 2024-09-12T23:29:11.589086+0000 mon.a (mon.0) 403 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:29:12.788 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:12 smithi136 bash[32189]: cephadm 2024-09-12T23:29:11.590522+0000 mgr.y (mgr.14150) 188 : cephadm [INF] Deploying daemon osd.3 on smithi136 2024-09-12T23:29:12.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:12 smithi181 bash[24776]: cluster 2024-09-12T23:29:11.418589+0000 mgr.y (mgr.14150) 187 : cluster [DBG] pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:29:12.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:12 smithi181 bash[24776]: audit 2024-09-12T23:29:11.587409+0000 mon.a (mon.0) 402 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-12T23:29:12.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:12 smithi181 bash[24776]: audit 2024-09-12T23:29:11.589086+0000 mon.a (mon.0) 403 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:29:12.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:12 smithi181 bash[24776]: cephadm 2024-09-12T23:29:11.590522+0000 mgr.y (mgr.14150) 188 : cephadm [INF] Deploying daemon osd.3 on smithi136 2024-09-12T23:29:14.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:14 smithi136 bash[22601]: cluster 2024-09-12T23:29:13.419199+0000 mgr.y (mgr.14150) 189 : cluster [DBG] pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:29:14.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:14 smithi136 bash[32189]: cluster 2024-09-12T23:29:13.419199+0000 mgr.y (mgr.14150) 189 : cluster [DBG] pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:29:14.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:14 smithi181 bash[24776]: cluster 2024-09-12T23:29:13.419199+0000 mgr.y (mgr.14150) 189 : cluster [DBG] pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:29:15.645 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:15 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:29:15.646 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 12 23:29:15 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:29:15.646 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:29:15 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:29:15.646 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:15 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:29:15.647 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 12 23:29:15 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:29:15.647 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 12 23:29:15 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:29:15.989 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:29:15 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:29:15.989 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:15 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:29:15.989 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 12 23:29:15 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:29:15.990 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 12 23:29:15 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:29:15.990 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 12 23:29:15 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:29:15.990 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:15 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:29:16.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:16 smithi181 bash[24776]: cluster 2024-09-12T23:29:15.419594+0000 mgr.y (mgr.14150) 190 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:29:16.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:16 smithi181 bash[24776]: audit 2024-09-12T23:29:15.907008+0000 mon.a (mon.0) 404 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:29:16.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:16 smithi181 bash[24776]: audit 2024-09-12T23:29:15.911054+0000 mon.a (mon.0) 405 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:29:16.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:16 smithi181 bash[24776]: audit 2024-09-12T23:29:15.915728+0000 mon.a (mon.0) 406 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:29:16.929 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:16 smithi136 bash[32189]: cluster 2024-09-12T23:29:15.419594+0000 mgr.y (mgr.14150) 190 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:29:16.929 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:16 smithi136 bash[32189]: audit 2024-09-12T23:29:15.907008+0000 mon.a (mon.0) 404 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:29:16.929 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:16 smithi136 bash[32189]: audit 2024-09-12T23:29:15.911054+0000 mon.a (mon.0) 405 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:29:16.929 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:16 smithi136 bash[32189]: audit 2024-09-12T23:29:15.915728+0000 mon.a (mon.0) 406 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:29:16.929 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:16 smithi136 bash[22601]: cluster 2024-09-12T23:29:15.419594+0000 mgr.y (mgr.14150) 190 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:29:16.929 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:16 smithi136 bash[22601]: audit 2024-09-12T23:29:15.907008+0000 mon.a (mon.0) 404 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:29:16.930 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:16 smithi136 bash[22601]: audit 2024-09-12T23:29:15.911054+0000 mon.a (mon.0) 405 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:29:16.930 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:16 smithi136 bash[22601]: audit 2024-09-12T23:29:15.915728+0000 mon.a (mon.0) 406 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:29:18.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:18 smithi181 bash[24776]: cluster 2024-09-12T23:29:17.419988+0000 mgr.y (mgr.14150) 191 : cluster [DBG] pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:29:18.971 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:18 smithi136 bash[22601]: cluster 2024-09-12T23:29:17.419988+0000 mgr.y (mgr.14150) 191 : cluster [DBG] pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:29:18.971 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:18 smithi136 bash[32189]: cluster 2024-09-12T23:29:17.419988+0000 mgr.y (mgr.14150) 191 : cluster [DBG] pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:29:20.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:20 smithi181 bash[24776]: cluster 2024-09-12T23:29:19.420493+0000 mgr.y (mgr.14150) 192 : cluster [DBG] pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:29:21.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:20 smithi136 bash[22601]: cluster 2024-09-12T23:29:19.420493+0000 mgr.y (mgr.14150) 192 : cluster [DBG] pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:29:21.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:20 smithi136 bash[32189]: cluster 2024-09-12T23:29:19.420493+0000 mgr.y (mgr.14150) 192 : cluster [DBG] pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:29:22.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:21 smithi136 bash[22601]: audit 2024-09-12T23:29:20.985404+0000 mon.a (mon.0) 407 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:29:22.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:21 smithi136 bash[22601]: audit 2024-09-12T23:29:20.989926+0000 mon.a (mon.0) 408 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:29:22.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:21 smithi136 bash[22601]: audit 2024-09-12T23:29:20.990767+0000 mon.a (mon.0) 409 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:29:22.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:21 smithi136 bash[22601]: audit 2024-09-12T23:29:20.991492+0000 mon.a (mon.0) 410 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:29:22.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:21 smithi136 bash[22601]: audit 2024-09-12T23:29:20.996186+0000 mon.a (mon.0) 411 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:29:22.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:21 smithi136 bash[22601]: cluster 2024-09-12T23:29:21.421029+0000 mgr.y (mgr.14150) 193 : cluster [DBG] pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:29:22.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:21 smithi136 bash[32189]: audit 2024-09-12T23:29:20.985404+0000 mon.a (mon.0) 407 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:29:22.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:21 smithi136 bash[32189]: audit 2024-09-12T23:29:20.989926+0000 mon.a (mon.0) 408 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:29:22.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:21 smithi136 bash[32189]: audit 2024-09-12T23:29:20.990767+0000 mon.a (mon.0) 409 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:29:22.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:21 smithi136 bash[32189]: audit 2024-09-12T23:29:20.991492+0000 mon.a (mon.0) 410 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:29:22.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:21 smithi136 bash[32189]: audit 2024-09-12T23:29:20.996186+0000 mon.a (mon.0) 411 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:29:22.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:21 smithi136 bash[32189]: cluster 2024-09-12T23:29:21.421029+0000 mgr.y (mgr.14150) 193 : cluster [DBG] pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:29:22.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:21 smithi181 bash[24776]: audit 2024-09-12T23:29:20.985404+0000 mon.a (mon.0) 407 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:29:22.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:21 smithi181 bash[24776]: audit 2024-09-12T23:29:20.989926+0000 mon.a (mon.0) 408 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:29:22.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:21 smithi181 bash[24776]: audit 2024-09-12T23:29:20.990767+0000 mon.a (mon.0) 409 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:29:22.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:21 smithi181 bash[24776]: audit 2024-09-12T23:29:20.991492+0000 mon.a (mon.0) 410 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:29:22.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:21 smithi181 bash[24776]: audit 2024-09-12T23:29:20.996186+0000 mon.a (mon.0) 411 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:29:22.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:21 smithi181 bash[24776]: cluster 2024-09-12T23:29:21.421029+0000 mgr.y (mgr.14150) 193 : cluster [DBG] pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:29:23.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:22 smithi136 bash[22601]: audit 2024-09-12T23:29:22.627662+0000 mon.c (mon.2) 15 : audit [INF] from='osd.3 [v2:172.21.15.136:6826/201948269,v1:172.21.15.136:6827/201948269]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-12T23:29:23.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:23 smithi136 bash[22601]: audit 2024-09-12T23:29:22.628020+0000 mon.a (mon.0) 412 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-12T23:29:23.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:23 smithi136 bash[32189]: audit 2024-09-12T23:29:22.627662+0000 mon.c (mon.2) 15 : audit [INF] from='osd.3 [v2:172.21.15.136:6826/201948269,v1:172.21.15.136:6827/201948269]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-12T23:29:23.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:23 smithi136 bash[32189]: audit 2024-09-12T23:29:22.628020+0000 mon.a (mon.0) 412 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-12T23:29:23.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:23 smithi181 bash[24776]: audit 2024-09-12T23:29:22.627662+0000 mon.c (mon.2) 15 : audit [INF] from='osd.3 [v2:172.21.15.136:6826/201948269,v1:172.21.15.136:6827/201948269]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-12T23:29:23.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:23 smithi181 bash[24776]: audit 2024-09-12T23:29:22.628020+0000 mon.a (mon.0) 412 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-12T23:29:24.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:24 smithi136 bash[22601]: audit 2024-09-12T23:29:23.020009+0000 mon.a (mon.0) 413 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-12T23:29:24.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:24 smithi136 bash[22601]: cluster 2024-09-12T23:29:23.020184+0000 mon.a (mon.0) 414 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2024-09-12T23:29:24.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:24 smithi136 bash[22601]: audit 2024-09-12T23:29:23.020429+0000 mon.a (mon.0) 415 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-12T23:29:24.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:24 smithi136 bash[22601]: audit 2024-09-12T23:29:23.021503+0000 mon.c (mon.2) 16 : audit [INF] from='osd.3 [v2:172.21.15.136:6826/201948269,v1:172.21.15.136:6827/201948269]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-12T23:29:24.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:24 smithi136 bash[22601]: audit 2024-09-12T23:29:23.022351+0000 mon.a (mon.0) 416 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-12T23:29:24.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:24 smithi136 bash[22601]: cluster 2024-09-12T23:29:23.421454+0000 mgr.y (mgr.14150) 194 : cluster [DBG] pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:29:24.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:24 smithi136 bash[32189]: audit 2024-09-12T23:29:23.020009+0000 mon.a (mon.0) 413 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-12T23:29:24.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:24 smithi136 bash[32189]: cluster 2024-09-12T23:29:23.020184+0000 mon.a (mon.0) 414 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2024-09-12T23:29:24.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:24 smithi136 bash[32189]: audit 2024-09-12T23:29:23.020429+0000 mon.a (mon.0) 415 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-12T23:29:24.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:24 smithi136 bash[32189]: audit 2024-09-12T23:29:23.021503+0000 mon.c (mon.2) 16 : audit [INF] from='osd.3 [v2:172.21.15.136:6826/201948269,v1:172.21.15.136:6827/201948269]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-12T23:29:24.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:24 smithi136 bash[32189]: audit 2024-09-12T23:29:23.022351+0000 mon.a (mon.0) 416 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-12T23:29:24.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:24 smithi136 bash[32189]: cluster 2024-09-12T23:29:23.421454+0000 mgr.y (mgr.14150) 194 : cluster [DBG] pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:29:24.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:24 smithi181 bash[24776]: audit 2024-09-12T23:29:23.020009+0000 mon.a (mon.0) 413 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-12T23:29:24.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:24 smithi181 bash[24776]: cluster 2024-09-12T23:29:23.020184+0000 mon.a (mon.0) 414 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2024-09-12T23:29:24.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:24 smithi181 bash[24776]: audit 2024-09-12T23:29:23.020429+0000 mon.a (mon.0) 415 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-12T23:29:24.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:24 smithi181 bash[24776]: audit 2024-09-12T23:29:23.021503+0000 mon.c (mon.2) 16 : audit [INF] from='osd.3 [v2:172.21.15.136:6826/201948269,v1:172.21.15.136:6827/201948269]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-12T23:29:24.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:24 smithi181 bash[24776]: audit 2024-09-12T23:29:23.022351+0000 mon.a (mon.0) 416 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-12T23:29:24.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:24 smithi181 bash[24776]: cluster 2024-09-12T23:29:23.421454+0000 mgr.y (mgr.14150) 194 : cluster [DBG] pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:29:24.521 INFO:teuthology.orchestra.run.smithi136.stdout:Created osd(s) 3 on host 'smithi136' 2024-09-12T23:29:25.123 DEBUG:teuthology.orchestra.run.smithi136:osd.3> sudo journalctl -f -n 0 -u ceph-da58a47c-715d-11ef-bceb-c7b262605968@osd.3.service 2024-09-12T23:29:25.126 INFO:tasks.cephadm:Deploying osd.4 on smithi181 with /dev/vg_nvme/lv_4... 2024-09-12T23:29:25.126 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-09-12T23:29:25.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:25 smithi181 bash[24776]: audit 2024-09-12T23:29:24.019691+0000 mon.a (mon.0) 417 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]': finished 2024-09-12T23:29:25.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:25 smithi181 bash[24776]: cluster 2024-09-12T23:29:24.019892+0000 mon.a (mon.0) 418 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2024-09-12T23:29:25.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:25 smithi181 bash[24776]: audit 2024-09-12T23:29:24.020668+0000 mon.a (mon.0) 419 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-12T23:29:25.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:25 smithi181 bash[24776]: audit 2024-09-12T23:29:24.031982+0000 mon.a (mon.0) 420 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-12T23:29:25.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:25 smithi181 bash[24776]: audit 2024-09-12T23:29:24.501138+0000 mon.a (mon.0) 421 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:29:25.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:25 smithi181 bash[24776]: audit 2024-09-12T23:29:24.507649+0000 mon.a (mon.0) 422 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:29:25.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:25 smithi181 bash[24776]: audit 2024-09-12T23:29:24.517218+0000 mon.a (mon.0) 423 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:29:25.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:25 smithi136 bash[32189]: audit 2024-09-12T23:29:24.019691+0000 mon.a (mon.0) 417 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]': finished 2024-09-12T23:29:25.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:25 smithi136 bash[32189]: cluster 2024-09-12T23:29:24.019892+0000 mon.a (mon.0) 418 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2024-09-12T23:29:25.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:25 smithi136 bash[32189]: audit 2024-09-12T23:29:24.020668+0000 mon.a (mon.0) 419 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-12T23:29:25.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:25 smithi136 bash[32189]: audit 2024-09-12T23:29:24.031982+0000 mon.a (mon.0) 420 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-12T23:29:25.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:25 smithi136 bash[32189]: audit 2024-09-12T23:29:24.501138+0000 mon.a (mon.0) 421 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:29:25.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:25 smithi136 bash[32189]: audit 2024-09-12T23:29:24.507649+0000 mon.a (mon.0) 422 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:29:25.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:25 smithi136 bash[32189]: audit 2024-09-12T23:29:24.517218+0000 mon.a (mon.0) 423 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:29:25.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:25 smithi136 bash[22601]: audit 2024-09-12T23:29:24.019691+0000 mon.a (mon.0) 417 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]': finished 2024-09-12T23:29:25.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:25 smithi136 bash[22601]: cluster 2024-09-12T23:29:24.019892+0000 mon.a (mon.0) 418 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2024-09-12T23:29:25.511 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:25 smithi136 bash[22601]: audit 2024-09-12T23:29:24.020668+0000 mon.a (mon.0) 419 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-12T23:29:25.511 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:25 smithi136 bash[22601]: audit 2024-09-12T23:29:24.031982+0000 mon.a (mon.0) 420 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-12T23:29:25.511 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:25 smithi136 bash[22601]: audit 2024-09-12T23:29:24.501138+0000 mon.a (mon.0) 421 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:29:25.511 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:25 smithi136 bash[22601]: audit 2024-09-12T23:29:24.507649+0000 mon.a (mon.0) 422 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:29:25.511 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:25 smithi136 bash[22601]: audit 2024-09-12T23:29:24.517218+0000 mon.a (mon.0) 423 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:29:26.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:26 smithi181 bash[24776]: cluster 2024-09-12T23:29:23.585334+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-12T23:29:26.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:26 smithi181 bash[24776]: cluster 2024-09-12T23:29:23.585447+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-12T23:29:26.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:26 smithi181 bash[24776]: cluster 2024-09-12T23:29:25.031931+0000 mon.a (mon.0) 424 : cluster [INF] osd.3 [v2:172.21.15.136:6826/201948269,v1:172.21.15.136:6827/201948269] boot 2024-09-12T23:29:26.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:26 smithi181 bash[24776]: cluster 2024-09-12T23:29:25.032017+0000 mon.a (mon.0) 425 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2024-09-12T23:29:26.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:26 smithi181 bash[24776]: audit 2024-09-12T23:29:25.032674+0000 mon.a (mon.0) 426 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-12T23:29:26.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:26 smithi181 bash[24776]: cluster 2024-09-12T23:29:25.421984+0000 mgr.y (mgr.14150) 195 : cluster [DBG] pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:29:26.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:26 smithi136 bash[32189]: cluster 2024-09-12T23:29:23.585334+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-12T23:29:26.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:26 smithi136 bash[32189]: cluster 2024-09-12T23:29:23.585447+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-12T23:29:26.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:26 smithi136 bash[32189]: cluster 2024-09-12T23:29:25.031931+0000 mon.a (mon.0) 424 : cluster [INF] osd.3 [v2:172.21.15.136:6826/201948269,v1:172.21.15.136:6827/201948269] boot 2024-09-12T23:29:26.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:26 smithi136 bash[32189]: cluster 2024-09-12T23:29:25.032017+0000 mon.a (mon.0) 425 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2024-09-12T23:29:26.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:26 smithi136 bash[32189]: audit 2024-09-12T23:29:25.032674+0000 mon.a (mon.0) 426 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-12T23:29:26.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:26 smithi136 bash[32189]: cluster 2024-09-12T23:29:25.421984+0000 mgr.y (mgr.14150) 195 : cluster [DBG] pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:29:26.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:26 smithi136 bash[22601]: cluster 2024-09-12T23:29:23.585334+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-12T23:29:26.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:26 smithi136 bash[22601]: cluster 2024-09-12T23:29:23.585447+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-12T23:29:26.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:26 smithi136 bash[22601]: cluster 2024-09-12T23:29:25.031931+0000 mon.a (mon.0) 424 : cluster [INF] osd.3 [v2:172.21.15.136:6826/201948269,v1:172.21.15.136:6827/201948269] boot 2024-09-12T23:29:26.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:26 smithi136 bash[22601]: cluster 2024-09-12T23:29:25.032017+0000 mon.a (mon.0) 425 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2024-09-12T23:29:26.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:26 smithi136 bash[22601]: audit 2024-09-12T23:29:25.032674+0000 mon.a (mon.0) 426 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-12T23:29:26.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:26 smithi136 bash[22601]: cluster 2024-09-12T23:29:25.421984+0000 mgr.y (mgr.14150) 195 : cluster [DBG] pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-12T23:29:27.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:27 smithi181 bash[24776]: cluster 2024-09-12T23:29:26.050414+0000 mon.a (mon.0) 427 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2024-09-12T23:29:27.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:27 smithi136 bash[32189]: cluster 2024-09-12T23:29:26.050414+0000 mon.a (mon.0) 427 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2024-09-12T23:29:27.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:27 smithi136 bash[22601]: cluster 2024-09-12T23:29:26.050414+0000 mon.a (mon.0) 427 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2024-09-12T23:29:28.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:28 smithi181 bash[24776]: cluster 2024-09-12T23:29:27.422551+0000 mgr.y (mgr.14150) 196 : cluster [DBG] pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:29:28.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:28 smithi136 bash[22601]: cluster 2024-09-12T23:29:27.422551+0000 mgr.y (mgr.14150) 196 : cluster [DBG] pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:29:28.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:28 smithi136 bash[32189]: cluster 2024-09-12T23:29:27.422551+0000 mgr.y (mgr.14150) 196 : cluster [DBG] pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:29:29.836 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.b/config 2024-09-12T23:29:30.625 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:30 smithi136 bash[32189]: cluster 2024-09-12T23:29:29.423083+0000 mgr.y (mgr.14150) 197 : cluster [DBG] pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:29:30.626 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:30 smithi136 bash[22601]: cluster 2024-09-12T23:29:29.423083+0000 mgr.y (mgr.14150) 197 : cluster [DBG] pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:29:30.755 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:30 smithi181 bash[24776]: cluster 2024-09-12T23:29:29.423083+0000 mgr.y (mgr.14150) 197 : cluster [DBG] pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:29:32.507 INFO:teuthology.orchestra.run.smithi181.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-09-12T23:29:32.700 INFO:teuthology.orchestra.run.smithi181.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-09-12T23:29:32.717 INFO:teuthology.orchestra.run.smithi181.stderr: stderr: 10+0 records in 2024-09-12T23:29:32.717 INFO:teuthology.orchestra.run.smithi181.stderr:10+0 records out 2024-09-12T23:29:32.718 INFO:teuthology.orchestra.run.smithi181.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0151529 s, 692 MB/s 2024-09-12T23:29:32.718 INFO:teuthology.orchestra.run.smithi181.stderr:--> Zapping successful for: 2024-09-12T23:29:32.756 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:32 smithi181 bash[24776]: cluster 2024-09-12T23:29:31.423593+0000 mgr.y (mgr.14150) 198 : cluster [DBG] pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:29:32.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:32 smithi136 bash[22601]: cluster 2024-09-12T23:29:31.423593+0000 mgr.y (mgr.14150) 198 : cluster [DBG] pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:29:32.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:32 smithi136 bash[32189]: cluster 2024-09-12T23:29:31.423593+0000 mgr.y (mgr.14150) 198 : cluster [DBG] pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:29:33.282 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi181:vg_nvme/lv_4 2024-09-12T23:29:35.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:34 smithi136 bash[22601]: cluster 2024-09-12T23:29:33.424129+0000 mgr.y (mgr.14150) 199 : cluster [DBG] pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:29:35.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:34 smithi136 bash[22601]: cephadm 2024-09-12T23:29:33.656918+0000 mgr.y (mgr.14150) 200 : cephadm [INF] Detected new or changed devices on smithi136 2024-09-12T23:29:35.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:34 smithi136 bash[22601]: audit 2024-09-12T23:29:33.665851+0000 mon.a (mon.0) 428 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:29:35.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:34 smithi136 bash[22601]: audit 2024-09-12T23:29:33.675385+0000 mon.a (mon.0) 429 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:29:35.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:34 smithi136 bash[22601]: audit 2024-09-12T23:29:33.677687+0000 mon.a (mon.0) 430 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:29:35.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:34 smithi136 bash[22601]: audit 2024-09-12T23:29:33.679363+0000 mon.a (mon.0) 431 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:29:35.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:34 smithi136 bash[22601]: audit 2024-09-12T23:29:33.680889+0000 mon.a (mon.0) 432 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:29:35.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:34 smithi136 bash[22601]: audit 2024-09-12T23:29:33.682471+0000 mon.a (mon.0) 433 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:29:35.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:34 smithi136 bash[22601]: cephadm 2024-09-12T23:29:33.683664+0000 mgr.y (mgr.14150) 201 : cephadm [INF] Adjusting osd_memory_target on smithi136 to 4060M 2024-09-12T23:29:35.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:34 smithi136 bash[22601]: audit 2024-09-12T23:29:33.691954+0000 mon.a (mon.0) 434 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:29:35.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:34 smithi136 bash[22601]: audit 2024-09-12T23:29:33.694244+0000 mon.a (mon.0) 435 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:29:35.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:34 smithi136 bash[22601]: audit 2024-09-12T23:29:33.695744+0000 mon.a (mon.0) 436 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:29:35.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:34 smithi136 bash[22601]: audit 2024-09-12T23:29:33.704044+0000 mon.a (mon.0) 437 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:29:35.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:34 smithi136 bash[32189]: cluster 2024-09-12T23:29:33.424129+0000 mgr.y (mgr.14150) 199 : cluster [DBG] pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:29:35.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:34 smithi136 bash[32189]: cephadm 2024-09-12T23:29:33.656918+0000 mgr.y (mgr.14150) 200 : cephadm [INF] Detected new or changed devices on smithi136 2024-09-12T23:29:35.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:34 smithi136 bash[32189]: audit 2024-09-12T23:29:33.665851+0000 mon.a (mon.0) 428 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:29:35.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:34 smithi136 bash[32189]: audit 2024-09-12T23:29:33.675385+0000 mon.a (mon.0) 429 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:29:35.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:34 smithi136 bash[32189]: audit 2024-09-12T23:29:33.677687+0000 mon.a (mon.0) 430 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:29:35.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:34 smithi136 bash[32189]: audit 2024-09-12T23:29:33.679363+0000 mon.a (mon.0) 431 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:29:35.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:34 smithi136 bash[32189]: audit 2024-09-12T23:29:33.680889+0000 mon.a (mon.0) 432 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:29:35.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:34 smithi136 bash[32189]: audit 2024-09-12T23:29:33.682471+0000 mon.a (mon.0) 433 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:29:35.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:34 smithi136 bash[32189]: cephadm 2024-09-12T23:29:33.683664+0000 mgr.y (mgr.14150) 201 : cephadm [INF] Adjusting osd_memory_target on smithi136 to 4060M 2024-09-12T23:29:35.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:34 smithi136 bash[32189]: audit 2024-09-12T23:29:33.691954+0000 mon.a (mon.0) 434 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:29:35.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:34 smithi136 bash[32189]: audit 2024-09-12T23:29:33.694244+0000 mon.a (mon.0) 435 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:29:35.012 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:34 smithi136 bash[32189]: audit 2024-09-12T23:29:33.695744+0000 mon.a (mon.0) 436 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:29:35.012 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:34 smithi136 bash[32189]: audit 2024-09-12T23:29:33.704044+0000 mon.a (mon.0) 437 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:29:35.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:34 smithi181 bash[24776]: cluster 2024-09-12T23:29:33.424129+0000 mgr.y (mgr.14150) 199 : cluster [DBG] pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:29:35.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:34 smithi181 bash[24776]: cephadm 2024-09-12T23:29:33.656918+0000 mgr.y (mgr.14150) 200 : cephadm [INF] Detected new or changed devices on smithi136 2024-09-12T23:29:35.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:34 smithi181 bash[24776]: audit 2024-09-12T23:29:33.665851+0000 mon.a (mon.0) 428 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:29:35.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:34 smithi181 bash[24776]: audit 2024-09-12T23:29:33.675385+0000 mon.a (mon.0) 429 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:29:35.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:34 smithi181 bash[24776]: audit 2024-09-12T23:29:33.677687+0000 mon.a (mon.0) 430 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:29:35.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:34 smithi181 bash[24776]: audit 2024-09-12T23:29:33.679363+0000 mon.a (mon.0) 431 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:29:35.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:34 smithi181 bash[24776]: audit 2024-09-12T23:29:33.680889+0000 mon.a (mon.0) 432 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:29:35.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:34 smithi181 bash[24776]: audit 2024-09-12T23:29:33.682471+0000 mon.a (mon.0) 433 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:29:35.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:34 smithi181 bash[24776]: cephadm 2024-09-12T23:29:33.683664+0000 mgr.y (mgr.14150) 201 : cephadm [INF] Adjusting osd_memory_target on smithi136 to 4060M 2024-09-12T23:29:35.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:34 smithi181 bash[24776]: audit 2024-09-12T23:29:33.691954+0000 mon.a (mon.0) 434 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:29:35.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:34 smithi181 bash[24776]: audit 2024-09-12T23:29:33.694244+0000 mon.a (mon.0) 435 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:29:35.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:34 smithi181 bash[24776]: audit 2024-09-12T23:29:33.695744+0000 mon.a (mon.0) 436 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:29:35.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:34 smithi181 bash[24776]: audit 2024-09-12T23:29:33.704044+0000 mon.a (mon.0) 437 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:29:37.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:36 smithi136 bash[32189]: cluster 2024-09-12T23:29:35.424647+0000 mgr.y (mgr.14150) 202 : cluster [DBG] pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:29:37.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:36 smithi136 bash[22601]: cluster 2024-09-12T23:29:35.424647+0000 mgr.y (mgr.14150) 202 : cluster [DBG] pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:29:37.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:36 smithi181 bash[24776]: cluster 2024-09-12T23:29:35.424647+0000 mgr.y (mgr.14150) 202 : cluster [DBG] pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:29:37.994 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.b/config 2024-09-12T23:29:38.967 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:38 smithi181 bash[24776]: cluster 2024-09-12T23:29:37.425206+0000 mgr.y (mgr.14150) 203 : cluster [DBG] pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:29:39.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:38 smithi136 bash[32189]: cluster 2024-09-12T23:29:37.425206+0000 mgr.y (mgr.14150) 203 : cluster [DBG] pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:29:39.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:38 smithi136 bash[22601]: cluster 2024-09-12T23:29:37.425206+0000 mgr.y (mgr.14150) 203 : cluster [DBG] pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:29:40.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:40 smithi181 bash[24776]: cluster 2024-09-12T23:29:39.425729+0000 mgr.y (mgr.14150) 204 : cluster [DBG] pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:29:41.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:40 smithi136 bash[22601]: cluster 2024-09-12T23:29:39.425729+0000 mgr.y (mgr.14150) 204 : cluster [DBG] pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:29:41.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:40 smithi136 bash[32189]: cluster 2024-09-12T23:29:39.425729+0000 mgr.y (mgr.14150) 204 : cluster [DBG] pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:29:42.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:41 smithi136 bash[22601]: audit 2024-09-12T23:29:40.787602+0000 mgr.y (mgr.14150) 205 : audit [DBG] from='client.24169 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:29:42.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:41 smithi136 bash[22601]: audit 2024-09-12T23:29:40.794398+0000 mon.a (mon.0) 438 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-12T23:29:42.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:41 smithi136 bash[22601]: audit 2024-09-12T23:29:40.800769+0000 mon.a (mon.0) 439 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-12T23:29:42.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:41 smithi136 bash[22601]: audit 2024-09-12T23:29:40.802532+0000 mon.a (mon.0) 440 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:29:42.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:41 smithi136 bash[32189]: audit 2024-09-12T23:29:40.787602+0000 mgr.y (mgr.14150) 205 : audit [DBG] from='client.24169 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:29:42.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:41 smithi136 bash[32189]: audit 2024-09-12T23:29:40.794398+0000 mon.a (mon.0) 438 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-12T23:29:42.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:41 smithi136 bash[32189]: audit 2024-09-12T23:29:40.800769+0000 mon.a (mon.0) 439 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-12T23:29:42.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:41 smithi136 bash[32189]: audit 2024-09-12T23:29:40.802532+0000 mon.a (mon.0) 440 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:29:42.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:41 smithi181 bash[24776]: audit 2024-09-12T23:29:40.787602+0000 mgr.y (mgr.14150) 205 : audit [DBG] from='client.24169 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:29:42.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:41 smithi181 bash[24776]: audit 2024-09-12T23:29:40.794398+0000 mon.a (mon.0) 438 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-12T23:29:42.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:41 smithi181 bash[24776]: audit 2024-09-12T23:29:40.800769+0000 mon.a (mon.0) 439 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-12T23:29:42.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:41 smithi181 bash[24776]: audit 2024-09-12T23:29:40.802532+0000 mon.a (mon.0) 440 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:29:43.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:42 smithi136 bash[22601]: cluster 2024-09-12T23:29:41.426233+0000 mgr.y (mgr.14150) 206 : cluster [DBG] pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:29:43.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:42 smithi136 bash[32189]: cluster 2024-09-12T23:29:41.426233+0000 mgr.y (mgr.14150) 206 : cluster [DBG] pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:29:43.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:42 smithi181 bash[24776]: cluster 2024-09-12T23:29:41.426233+0000 mgr.y (mgr.14150) 206 : cluster [DBG] pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:29:45.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:44 smithi136 bash[22601]: cluster 2024-09-12T23:29:43.426754+0000 mgr.y (mgr.14150) 207 : cluster [DBG] pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:29:45.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:44 smithi136 bash[32189]: cluster 2024-09-12T23:29:43.426754+0000 mgr.y (mgr.14150) 207 : cluster [DBG] pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:29:45.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:44 smithi181 bash[24776]: cluster 2024-09-12T23:29:43.426754+0000 mgr.y (mgr.14150) 207 : cluster [DBG] pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:29:46.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:46 smithi181 bash[24776]: cluster 2024-09-12T23:29:45.427277+0000 mgr.y (mgr.14150) 208 : cluster [DBG] pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:29:47.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:46 smithi136 bash[22601]: cluster 2024-09-12T23:29:45.427277+0000 mgr.y (mgr.14150) 208 : cluster [DBG] pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:29:47.768 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:46 smithi136 bash[32189]: cluster 2024-09-12T23:29:45.427277+0000 mgr.y (mgr.14150) 208 : cluster [DBG] pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:29:49.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:48 smithi136 bash[22601]: cluster 2024-09-12T23:29:47.427804+0000 mgr.y (mgr.14150) 209 : cluster [DBG] pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:29:49.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:48 smithi136 bash[32189]: cluster 2024-09-12T23:29:47.427804+0000 mgr.y (mgr.14150) 209 : cluster [DBG] pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:29:49.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:48 smithi181 bash[24776]: cluster 2024-09-12T23:29:47.427804+0000 mgr.y (mgr.14150) 209 : cluster [DBG] pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:29:50.013 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:49 smithi136 bash[32189]: audit 2024-09-12T23:29:48.762541+0000 mon.b (mon.1) 9 : audit [INF] from='client.? 172.21.15.181:0/1793996787' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "98b0e81c-824e-4dbc-9faa-2de9f199af1b"}]: dispatch 2024-09-12T23:29:50.013 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:49 smithi136 bash[32189]: audit 2024-09-12T23:29:48.763992+0000 mon.a (mon.0) 441 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "98b0e81c-824e-4dbc-9faa-2de9f199af1b"}]: dispatch 2024-09-12T23:29:50.013 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:49 smithi136 bash[32189]: audit 2024-09-12T23:29:48.772869+0000 mon.a (mon.0) 442 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "98b0e81c-824e-4dbc-9faa-2de9f199af1b"}]': finished 2024-09-12T23:29:50.013 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:49 smithi136 bash[32189]: cluster 2024-09-12T23:29:48.773043+0000 mon.a (mon.0) 443 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2024-09-12T23:29:50.013 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:49 smithi136 bash[32189]: audit 2024-09-12T23:29:48.773278+0000 mon.a (mon.0) 444 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-12T23:29:50.013 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:49 smithi136 bash[32189]: audit 2024-09-12T23:29:49.375621+0000 mon.b (mon.1) 10 : audit [DBG] from='client.? 172.21.15.181:0/2037039073' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-12T23:29:50.013 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:49 smithi136 bash[22601]: audit 2024-09-12T23:29:48.762541+0000 mon.b (mon.1) 9 : audit [INF] from='client.? 172.21.15.181:0/1793996787' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "98b0e81c-824e-4dbc-9faa-2de9f199af1b"}]: dispatch 2024-09-12T23:29:50.013 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:49 smithi136 bash[22601]: audit 2024-09-12T23:29:48.763992+0000 mon.a (mon.0) 441 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "98b0e81c-824e-4dbc-9faa-2de9f199af1b"}]: dispatch 2024-09-12T23:29:50.014 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:49 smithi136 bash[22601]: audit 2024-09-12T23:29:48.772869+0000 mon.a (mon.0) 442 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "98b0e81c-824e-4dbc-9faa-2de9f199af1b"}]': finished 2024-09-12T23:29:50.014 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:49 smithi136 bash[22601]: cluster 2024-09-12T23:29:48.773043+0000 mon.a (mon.0) 443 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2024-09-12T23:29:50.014 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:49 smithi136 bash[22601]: audit 2024-09-12T23:29:48.773278+0000 mon.a (mon.0) 444 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-12T23:29:50.014 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:49 smithi136 bash[22601]: audit 2024-09-12T23:29:49.375621+0000 mon.b (mon.1) 10 : audit [DBG] from='client.? 172.21.15.181:0/2037039073' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-12T23:29:50.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:49 smithi181 bash[24776]: audit 2024-09-12T23:29:48.762541+0000 mon.b (mon.1) 9 : audit [INF] from='client.? 172.21.15.181:0/1793996787' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "98b0e81c-824e-4dbc-9faa-2de9f199af1b"}]: dispatch 2024-09-12T23:29:50.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:49 smithi181 bash[24776]: audit 2024-09-12T23:29:48.763992+0000 mon.a (mon.0) 441 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "98b0e81c-824e-4dbc-9faa-2de9f199af1b"}]: dispatch 2024-09-12T23:29:50.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:49 smithi181 bash[24776]: audit 2024-09-12T23:29:48.772869+0000 mon.a (mon.0) 442 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "98b0e81c-824e-4dbc-9faa-2de9f199af1b"}]': finished 2024-09-12T23:29:50.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:49 smithi181 bash[24776]: cluster 2024-09-12T23:29:48.773043+0000 mon.a (mon.0) 443 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2024-09-12T23:29:50.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:49 smithi181 bash[24776]: audit 2024-09-12T23:29:48.773278+0000 mon.a (mon.0) 444 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-12T23:29:50.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:49 smithi181 bash[24776]: audit 2024-09-12T23:29:49.375621+0000 mon.b (mon.1) 10 : audit [DBG] from='client.? 172.21.15.181:0/2037039073' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-12T23:29:51.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:50 smithi181 bash[24776]: cluster 2024-09-12T23:29:49.428324+0000 mgr.y (mgr.14150) 210 : cluster [DBG] pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:29:51.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:50 smithi136 bash[22601]: cluster 2024-09-12T23:29:49.428324+0000 mgr.y (mgr.14150) 210 : cluster [DBG] pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:29:51.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:50 smithi136 bash[32189]: cluster 2024-09-12T23:29:49.428324+0000 mgr.y (mgr.14150) 210 : cluster [DBG] pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:29:53.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:52 smithi181 bash[24776]: cluster 2024-09-12T23:29:51.428870+0000 mgr.y (mgr.14150) 211 : cluster [DBG] pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:29:53.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:52 smithi136 bash[22601]: cluster 2024-09-12T23:29:51.428870+0000 mgr.y (mgr.14150) 211 : cluster [DBG] pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:29:53.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:52 smithi136 bash[32189]: cluster 2024-09-12T23:29:51.428870+0000 mgr.y (mgr.14150) 211 : cluster [DBG] pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:29:55.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:54 smithi181 bash[24776]: cluster 2024-09-12T23:29:53.429407+0000 mgr.y (mgr.14150) 212 : cluster [DBG] pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:29:55.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:54 smithi136 bash[22601]: cluster 2024-09-12T23:29:53.429407+0000 mgr.y (mgr.14150) 212 : cluster [DBG] pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:29:55.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:54 smithi136 bash[32189]: cluster 2024-09-12T23:29:53.429407+0000 mgr.y (mgr.14150) 212 : cluster [DBG] pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:29:57.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:56 smithi181 bash[24776]: cluster 2024-09-12T23:29:55.429927+0000 mgr.y (mgr.14150) 213 : cluster [DBG] pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:29:57.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:56 smithi136 bash[32189]: cluster 2024-09-12T23:29:55.429927+0000 mgr.y (mgr.14150) 213 : cluster [DBG] pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:29:57.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:56 smithi136 bash[22601]: cluster 2024-09-12T23:29:55.429927+0000 mgr.y (mgr.14150) 213 : cluster [DBG] pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:29:59.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:29:58 smithi181 bash[24776]: cluster 2024-09-12T23:29:57.430457+0000 mgr.y (mgr.14150) 214 : cluster [DBG] pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:29:59.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:29:58 smithi136 bash[22601]: cluster 2024-09-12T23:29:57.430457+0000 mgr.y (mgr.14150) 214 : cluster [DBG] pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:29:59.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:29:58 smithi136 bash[32189]: cluster 2024-09-12T23:29:57.430457+0000 mgr.y (mgr.14150) 214 : cluster [DBG] pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:30:01.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:00 smithi181 bash[24776]: cluster 2024-09-12T23:29:59.430961+0000 mgr.y (mgr.14150) 215 : cluster [DBG] pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:30:01.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:00 smithi181 bash[24776]: cluster 2024-09-12T23:30:00.000167+0000 mon.a (mon.0) 445 : cluster [INF] overall HEALTH_OK 2024-09-12T23:30:01.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:00 smithi181 bash[24776]: audit 2024-09-12T23:30:00.614797+0000 mon.a (mon.0) 446 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-12T23:30:01.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:00 smithi181 bash[24776]: audit 2024-09-12T23:30:00.616477+0000 mon.a (mon.0) 447 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:30:01.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:00 smithi136 bash[22601]: cluster 2024-09-12T23:29:59.430961+0000 mgr.y (mgr.14150) 215 : cluster [DBG] pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:30:01.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:00 smithi136 bash[22601]: cluster 2024-09-12T23:30:00.000167+0000 mon.a (mon.0) 445 : cluster [INF] overall HEALTH_OK 2024-09-12T23:30:01.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:00 smithi136 bash[22601]: audit 2024-09-12T23:30:00.614797+0000 mon.a (mon.0) 446 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-12T23:30:01.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:00 smithi136 bash[22601]: audit 2024-09-12T23:30:00.616477+0000 mon.a (mon.0) 447 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:30:01.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:00 smithi136 bash[32189]: cluster 2024-09-12T23:29:59.430961+0000 mgr.y (mgr.14150) 215 : cluster [DBG] pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:30:01.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:00 smithi136 bash[32189]: cluster 2024-09-12T23:30:00.000167+0000 mon.a (mon.0) 445 : cluster [INF] overall HEALTH_OK 2024-09-12T23:30:01.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:00 smithi136 bash[32189]: audit 2024-09-12T23:30:00.614797+0000 mon.a (mon.0) 446 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-12T23:30:01.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:00 smithi136 bash[32189]: audit 2024-09-12T23:30:00.616477+0000 mon.a (mon.0) 447 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:30:02.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:01 smithi181 bash[24776]: cephadm 2024-09-12T23:30:00.618033+0000 mgr.y (mgr.14150) 216 : cephadm [INF] Deploying daemon osd.4 on smithi181 2024-09-12T23:30:02.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:01 smithi136 bash[22601]: cephadm 2024-09-12T23:30:00.618033+0000 mgr.y (mgr.14150) 216 : cephadm [INF] Deploying daemon osd.4 on smithi181 2024-09-12T23:30:02.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:01 smithi136 bash[32189]: cephadm 2024-09-12T23:30:00.618033+0000 mgr.y (mgr.14150) 216 : cephadm [INF] Deploying daemon osd.4 on smithi181 2024-09-12T23:30:03.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:02 smithi181 bash[24776]: cluster 2024-09-12T23:30:01.431465+0000 mgr.y (mgr.14150) 217 : cluster [DBG] pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:30:03.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:02 smithi136 bash[22601]: cluster 2024-09-12T23:30:01.431465+0000 mgr.y (mgr.14150) 217 : cluster [DBG] pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:30:03.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:02 smithi136 bash[32189]: cluster 2024-09-12T23:30:01.431465+0000 mgr.y (mgr.14150) 217 : cluster [DBG] pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:30:04.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:04 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:30:04.349 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:30:04 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:30:04.804 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:04 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:30:04.805 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:30:04 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:30:05.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:04 smithi181 bash[24776]: cluster 2024-09-12T23:30:03.431985+0000 mgr.y (mgr.14150) 218 : cluster [DBG] pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:30:05.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:04 smithi181 bash[24776]: audit 2024-09-12T23:30:04.522028+0000 mon.a (mon.0) 448 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:30:05.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:04 smithi181 bash[24776]: audit 2024-09-12T23:30:04.529974+0000 mon.a (mon.0) 449 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:30:05.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:04 smithi181 bash[24776]: audit 2024-09-12T23:30:04.538271+0000 mon.a (mon.0) 450 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:30:05.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:04 smithi136 bash[22601]: cluster 2024-09-12T23:30:03.431985+0000 mgr.y (mgr.14150) 218 : cluster [DBG] pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:30:05.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:04 smithi136 bash[22601]: audit 2024-09-12T23:30:04.522028+0000 mon.a (mon.0) 448 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:30:05.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:04 smithi136 bash[22601]: audit 2024-09-12T23:30:04.529974+0000 mon.a (mon.0) 449 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:30:05.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:04 smithi136 bash[22601]: audit 2024-09-12T23:30:04.538271+0000 mon.a (mon.0) 450 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:30:05.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:04 smithi136 bash[32189]: cluster 2024-09-12T23:30:03.431985+0000 mgr.y (mgr.14150) 218 : cluster [DBG] pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:30:05.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:04 smithi136 bash[32189]: audit 2024-09-12T23:30:04.522028+0000 mon.a (mon.0) 448 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:30:05.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:04 smithi136 bash[32189]: audit 2024-09-12T23:30:04.529974+0000 mon.a (mon.0) 449 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:30:05.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:04 smithi136 bash[32189]: audit 2024-09-12T23:30:04.538271+0000 mon.a (mon.0) 450 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:30:06.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:06 smithi181 bash[24776]: cluster 2024-09-12T23:30:05.432484+0000 mgr.y (mgr.14150) 219 : cluster [DBG] pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:30:07.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:06 smithi136 bash[22601]: cluster 2024-09-12T23:30:05.432484+0000 mgr.y (mgr.14150) 219 : cluster [DBG] pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:30:07.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:06 smithi136 bash[32189]: cluster 2024-09-12T23:30:05.432484+0000 mgr.y (mgr.14150) 219 : cluster [DBG] pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:30:09.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:08 smithi181 bash[24776]: cluster 2024-09-12T23:30:07.432997+0000 mgr.y (mgr.14150) 220 : cluster [DBG] pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:30:09.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:08 smithi136 bash[22601]: cluster 2024-09-12T23:30:07.432997+0000 mgr.y (mgr.14150) 220 : cluster [DBG] pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:30:09.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:08 smithi136 bash[32189]: cluster 2024-09-12T23:30:07.432997+0000 mgr.y (mgr.14150) 220 : cluster [DBG] pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:30:10.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:10 smithi181 bash[24776]: cluster 2024-09-12T23:30:09.433526+0000 mgr.y (mgr.14150) 221 : cluster [DBG] pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:30:10.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:10 smithi181 bash[24776]: audit 2024-09-12T23:30:09.577933+0000 mon.a (mon.0) 451 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:30:10.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:10 smithi181 bash[24776]: audit 2024-09-12T23:30:09.588177+0000 mon.a (mon.0) 452 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:30:10.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:10 smithi181 bash[24776]: audit 2024-09-12T23:30:09.590059+0000 mon.a (mon.0) 453 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:30:10.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:10 smithi181 bash[24776]: audit 2024-09-12T23:30:09.591950+0000 mon.a (mon.0) 454 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:30:10.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:10 smithi181 bash[24776]: audit 2024-09-12T23:30:09.603382+0000 mon.a (mon.0) 455 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:30:11.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:10 smithi136 bash[22601]: cluster 2024-09-12T23:30:09.433526+0000 mgr.y (mgr.14150) 221 : cluster [DBG] pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:30:11.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:10 smithi136 bash[22601]: audit 2024-09-12T23:30:09.577933+0000 mon.a (mon.0) 451 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:30:11.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:10 smithi136 bash[22601]: audit 2024-09-12T23:30:09.588177+0000 mon.a (mon.0) 452 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:30:11.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:10 smithi136 bash[22601]: audit 2024-09-12T23:30:09.590059+0000 mon.a (mon.0) 453 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:30:11.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:10 smithi136 bash[22601]: audit 2024-09-12T23:30:09.591950+0000 mon.a (mon.0) 454 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:30:11.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:10 smithi136 bash[22601]: audit 2024-09-12T23:30:09.603382+0000 mon.a (mon.0) 455 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:30:11.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:10 smithi136 bash[32189]: cluster 2024-09-12T23:30:09.433526+0000 mgr.y (mgr.14150) 221 : cluster [DBG] pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:30:11.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:10 smithi136 bash[32189]: audit 2024-09-12T23:30:09.577933+0000 mon.a (mon.0) 451 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:30:11.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:10 smithi136 bash[32189]: audit 2024-09-12T23:30:09.588177+0000 mon.a (mon.0) 452 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:30:11.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:10 smithi136 bash[32189]: audit 2024-09-12T23:30:09.590059+0000 mon.a (mon.0) 453 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:30:11.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:10 smithi136 bash[32189]: audit 2024-09-12T23:30:09.591950+0000 mon.a (mon.0) 454 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:30:11.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:10 smithi136 bash[32189]: audit 2024-09-12T23:30:09.603382+0000 mon.a (mon.0) 455 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:30:11.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:11 smithi181 bash[24776]: audit 2024-09-12T23:30:10.887263+0000 mon.b (mon.1) 11 : audit [INF] from='osd.4 [v2:172.21.15.181:6800/775251087,v1:172.21.15.181:6801/775251087]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-12T23:30:11.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:11 smithi181 bash[24776]: audit 2024-09-12T23:30:10.888374+0000 mon.a (mon.0) 456 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-12T23:30:12.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:11 smithi136 bash[22601]: audit 2024-09-12T23:30:10.887263+0000 mon.b (mon.1) 11 : audit [INF] from='osd.4 [v2:172.21.15.181:6800/775251087,v1:172.21.15.181:6801/775251087]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-12T23:30:12.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:11 smithi136 bash[22601]: audit 2024-09-12T23:30:10.888374+0000 mon.a (mon.0) 456 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-12T23:30:12.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:11 smithi136 bash[32189]: audit 2024-09-12T23:30:10.887263+0000 mon.b (mon.1) 11 : audit [INF] from='osd.4 [v2:172.21.15.181:6800/775251087,v1:172.21.15.181:6801/775251087]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-12T23:30:12.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:11 smithi136 bash[32189]: audit 2024-09-12T23:30:10.888374+0000 mon.a (mon.0) 456 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-12T23:30:12.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:12 smithi181 bash[24776]: cluster 2024-09-12T23:30:11.434105+0000 mgr.y (mgr.14150) 222 : cluster [DBG] pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:30:12.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:12 smithi181 bash[24776]: audit 2024-09-12T23:30:11.594964+0000 mon.a (mon.0) 457 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-12T23:30:12.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:12 smithi181 bash[24776]: cluster 2024-09-12T23:30:11.595115+0000 mon.a (mon.0) 458 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2024-09-12T23:30:12.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:12 smithi181 bash[24776]: audit 2024-09-12T23:30:11.595444+0000 mon.a (mon.0) 459 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-12T23:30:12.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:12 smithi181 bash[24776]: audit 2024-09-12T23:30:11.596044+0000 mon.b (mon.1) 12 : audit [INF] from='osd.4 [v2:172.21.15.181:6800/775251087,v1:172.21.15.181:6801/775251087]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-12T23:30:12.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:12 smithi181 bash[24776]: audit 2024-09-12T23:30:11.597442+0000 mon.a (mon.0) 460 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-12T23:30:12.886 INFO:teuthology.orchestra.run.smithi181.stdout:Created osd(s) 4 on host 'smithi181' 2024-09-12T23:30:13.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:12 smithi136 bash[22601]: cluster 2024-09-12T23:30:11.434105+0000 mgr.y (mgr.14150) 222 : cluster [DBG] pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:30:13.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:12 smithi136 bash[22601]: audit 2024-09-12T23:30:11.594964+0000 mon.a (mon.0) 457 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-12T23:30:13.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:12 smithi136 bash[22601]: cluster 2024-09-12T23:30:11.595115+0000 mon.a (mon.0) 458 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2024-09-12T23:30:13.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:12 smithi136 bash[22601]: audit 2024-09-12T23:30:11.595444+0000 mon.a (mon.0) 459 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-12T23:30:13.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:12 smithi136 bash[22601]: audit 2024-09-12T23:30:11.596044+0000 mon.b (mon.1) 12 : audit [INF] from='osd.4 [v2:172.21.15.181:6800/775251087,v1:172.21.15.181:6801/775251087]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-12T23:30:13.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:12 smithi136 bash[22601]: audit 2024-09-12T23:30:11.597442+0000 mon.a (mon.0) 460 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-12T23:30:13.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:12 smithi136 bash[32189]: cluster 2024-09-12T23:30:11.434105+0000 mgr.y (mgr.14150) 222 : cluster [DBG] pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:30:13.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:12 smithi136 bash[32189]: audit 2024-09-12T23:30:11.594964+0000 mon.a (mon.0) 457 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-12T23:30:13.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:12 smithi136 bash[32189]: cluster 2024-09-12T23:30:11.595115+0000 mon.a (mon.0) 458 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2024-09-12T23:30:13.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:12 smithi136 bash[32189]: audit 2024-09-12T23:30:11.595444+0000 mon.a (mon.0) 459 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-12T23:30:13.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:12 smithi136 bash[32189]: audit 2024-09-12T23:30:11.596044+0000 mon.b (mon.1) 12 : audit [INF] from='osd.4 [v2:172.21.15.181:6800/775251087,v1:172.21.15.181:6801/775251087]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-12T23:30:13.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:12 smithi136 bash[32189]: audit 2024-09-12T23:30:11.597442+0000 mon.a (mon.0) 460 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-12T23:30:13.396 DEBUG:teuthology.orchestra.run.smithi181:osd.4> sudo journalctl -f -n 0 -u ceph-da58a47c-715d-11ef-bceb-c7b262605968@osd.4.service 2024-09-12T23:30:13.398 INFO:tasks.cephadm:Deploying osd.5 on smithi181 with /dev/vg_nvme/lv_3... 2024-09-12T23:30:13.399 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-09-12T23:30:13.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:13 smithi181 bash[24776]: audit 2024-09-12T23:30:12.593434+0000 mon.a (mon.0) 461 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2024-09-12T23:30:13.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:13 smithi181 bash[24776]: cluster 2024-09-12T23:30:12.593652+0000 mon.a (mon.0) 462 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2024-09-12T23:30:13.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:13 smithi181 bash[24776]: audit 2024-09-12T23:30:12.594257+0000 mon.a (mon.0) 463 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-12T23:30:13.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:13 smithi181 bash[24776]: audit 2024-09-12T23:30:12.603143+0000 mon.a (mon.0) 464 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-12T23:30:13.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:13 smithi181 bash[24776]: audit 2024-09-12T23:30:12.864510+0000 mon.a (mon.0) 465 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:30:13.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:13 smithi181 bash[24776]: audit 2024-09-12T23:30:12.872065+0000 mon.a (mon.0) 466 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:30:13.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:13 smithi181 bash[24776]: audit 2024-09-12T23:30:12.882923+0000 mon.a (mon.0) 467 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:30:14.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:13 smithi136 bash[22601]: audit 2024-09-12T23:30:12.593434+0000 mon.a (mon.0) 461 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2024-09-12T23:30:14.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:13 smithi136 bash[22601]: cluster 2024-09-12T23:30:12.593652+0000 mon.a (mon.0) 462 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2024-09-12T23:30:14.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:13 smithi136 bash[22601]: audit 2024-09-12T23:30:12.594257+0000 mon.a (mon.0) 463 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-12T23:30:14.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:13 smithi136 bash[22601]: audit 2024-09-12T23:30:12.603143+0000 mon.a (mon.0) 464 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-12T23:30:14.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:13 smithi136 bash[22601]: audit 2024-09-12T23:30:12.864510+0000 mon.a (mon.0) 465 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:30:14.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:13 smithi136 bash[22601]: audit 2024-09-12T23:30:12.872065+0000 mon.a (mon.0) 466 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:30:14.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:13 smithi136 bash[22601]: audit 2024-09-12T23:30:12.882923+0000 mon.a (mon.0) 467 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:30:14.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:13 smithi136 bash[32189]: audit 2024-09-12T23:30:12.593434+0000 mon.a (mon.0) 461 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2024-09-12T23:30:14.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:13 smithi136 bash[32189]: cluster 2024-09-12T23:30:12.593652+0000 mon.a (mon.0) 462 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2024-09-12T23:30:14.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:13 smithi136 bash[32189]: audit 2024-09-12T23:30:12.594257+0000 mon.a (mon.0) 463 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-12T23:30:14.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:13 smithi136 bash[32189]: audit 2024-09-12T23:30:12.603143+0000 mon.a (mon.0) 464 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-12T23:30:14.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:13 smithi136 bash[32189]: audit 2024-09-12T23:30:12.864510+0000 mon.a (mon.0) 465 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:30:14.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:13 smithi136 bash[32189]: audit 2024-09-12T23:30:12.872065+0000 mon.a (mon.0) 466 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:30:14.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:13 smithi136 bash[32189]: audit 2024-09-12T23:30:12.882923+0000 mon.a (mon.0) 467 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:30:15.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:14 smithi136 bash[22601]: cluster 2024-09-12T23:30:11.888164+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-12T23:30:15.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:14 smithi136 bash[22601]: cluster 2024-09-12T23:30:11.888250+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-12T23:30:15.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:14 smithi136 bash[22601]: cluster 2024-09-12T23:30:13.434627+0000 mgr.y (mgr.14150) 223 : cluster [DBG] pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:30:15.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:14 smithi136 bash[22601]: cluster 2024-09-12T23:30:13.607656+0000 mon.a (mon.0) 468 : cluster [INF] osd.4 [v2:172.21.15.181:6800/775251087,v1:172.21.15.181:6801/775251087] boot 2024-09-12T23:30:15.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:14 smithi136 bash[22601]: cluster 2024-09-12T23:30:13.607835+0000 mon.a (mon.0) 469 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2024-09-12T23:30:15.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:14 smithi136 bash[22601]: audit 2024-09-12T23:30:13.608858+0000 mon.a (mon.0) 470 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-12T23:30:15.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:14 smithi136 bash[22601]: cluster 2024-09-12T23:30:14.609301+0000 mon.a (mon.0) 471 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2024-09-12T23:30:15.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:14 smithi136 bash[32189]: cluster 2024-09-12T23:30:11.888164+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-12T23:30:15.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:14 smithi136 bash[32189]: cluster 2024-09-12T23:30:11.888250+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-12T23:30:15.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:14 smithi136 bash[32189]: cluster 2024-09-12T23:30:13.434627+0000 mgr.y (mgr.14150) 223 : cluster [DBG] pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:30:15.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:14 smithi136 bash[32189]: cluster 2024-09-12T23:30:13.607656+0000 mon.a (mon.0) 468 : cluster [INF] osd.4 [v2:172.21.15.181:6800/775251087,v1:172.21.15.181:6801/775251087] boot 2024-09-12T23:30:15.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:14 smithi136 bash[32189]: cluster 2024-09-12T23:30:13.607835+0000 mon.a (mon.0) 469 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2024-09-12T23:30:15.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:14 smithi136 bash[32189]: audit 2024-09-12T23:30:13.608858+0000 mon.a (mon.0) 470 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-12T23:30:15.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:14 smithi136 bash[32189]: cluster 2024-09-12T23:30:14.609301+0000 mon.a (mon.0) 471 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2024-09-12T23:30:15.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:14 smithi181 bash[24776]: cluster 2024-09-12T23:30:11.888164+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-12T23:30:15.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:14 smithi181 bash[24776]: cluster 2024-09-12T23:30:11.888250+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-12T23:30:15.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:14 smithi181 bash[24776]: cluster 2024-09-12T23:30:13.434627+0000 mgr.y (mgr.14150) 223 : cluster [DBG] pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-12T23:30:15.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:14 smithi181 bash[24776]: cluster 2024-09-12T23:30:13.607656+0000 mon.a (mon.0) 468 : cluster [INF] osd.4 [v2:172.21.15.181:6800/775251087,v1:172.21.15.181:6801/775251087] boot 2024-09-12T23:30:15.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:14 smithi181 bash[24776]: cluster 2024-09-12T23:30:13.607835+0000 mon.a (mon.0) 469 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2024-09-12T23:30:15.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:14 smithi181 bash[24776]: audit 2024-09-12T23:30:13.608858+0000 mon.a (mon.0) 470 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-12T23:30:15.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:14 smithi181 bash[24776]: cluster 2024-09-12T23:30:14.609301+0000 mon.a (mon.0) 471 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2024-09-12T23:30:17.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:16 smithi136 bash[22601]: cluster 2024-09-12T23:30:15.435126+0000 mgr.y (mgr.14150) 224 : cluster [DBG] pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:30:17.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:16 smithi136 bash[22601]: cluster 2024-09-12T23:30:15.633046+0000 mon.a (mon.0) 472 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2024-09-12T23:30:17.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:16 smithi136 bash[32189]: cluster 2024-09-12T23:30:15.435126+0000 mgr.y (mgr.14150) 224 : cluster [DBG] pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:30:17.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:16 smithi136 bash[32189]: cluster 2024-09-12T23:30:15.633046+0000 mon.a (mon.0) 472 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2024-09-12T23:30:17.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:16 smithi181 bash[24776]: cluster 2024-09-12T23:30:15.435126+0000 mgr.y (mgr.14150) 224 : cluster [DBG] pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:30:17.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:16 smithi181 bash[24776]: cluster 2024-09-12T23:30:15.633046+0000 mon.a (mon.0) 472 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2024-09-12T23:30:17.629 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.b/config 2024-09-12T23:30:19.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:18 smithi136 bash[22601]: cluster 2024-09-12T23:30:17.435679+0000 mgr.y (mgr.14150) 225 : cluster [DBG] pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 119 KiB/s, 0 objects/s recovering 2024-09-12T23:30:19.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:18 smithi136 bash[32189]: cluster 2024-09-12T23:30:17.435679+0000 mgr.y (mgr.14150) 225 : cluster [DBG] pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 119 KiB/s, 0 objects/s recovering 2024-09-12T23:30:19.077 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:18 smithi181 bash[24776]: cluster 2024-09-12T23:30:17.435679+0000 mgr.y (mgr.14150) 225 : cluster [DBG] pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 119 KiB/s, 0 objects/s recovering 2024-09-12T23:30:20.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:20 smithi181 bash[24776]: cluster 2024-09-12T23:30:19.436256+0000 mgr.y (mgr.14150) 226 : cluster [DBG] pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-12T23:30:21.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:20 smithi136 bash[22601]: cluster 2024-09-12T23:30:19.436256+0000 mgr.y (mgr.14150) 226 : cluster [DBG] pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-12T23:30:21.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:20 smithi136 bash[32189]: cluster 2024-09-12T23:30:19.436256+0000 mgr.y (mgr.14150) 226 : cluster [DBG] pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-12T23:30:21.790 INFO:teuthology.orchestra.run.smithi181.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-09-12T23:30:22.018 INFO:teuthology.orchestra.run.smithi181.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-09-12T23:30:22.036 INFO:teuthology.orchestra.run.smithi181.stderr: stderr: 10+0 records in 2024-09-12T23:30:22.036 INFO:teuthology.orchestra.run.smithi181.stderr:10+0 records out 2024-09-12T23:30:22.036 INFO:teuthology.orchestra.run.smithi181.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0161277 s, 650 MB/s 2024-09-12T23:30:22.036 INFO:teuthology.orchestra.run.smithi181.stderr:--> Zapping successful for: 2024-09-12T23:30:22.751 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi181:vg_nvme/lv_3 2024-09-12T23:30:23.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:22 smithi136 bash[22601]: cluster 2024-09-12T23:30:21.436830+0000 mgr.y (mgr.14150) 227 : cluster [DBG] pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-09-12T23:30:23.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:22 smithi136 bash[32189]: cluster 2024-09-12T23:30:21.436830+0000 mgr.y (mgr.14150) 227 : cluster [DBG] pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-09-12T23:30:23.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:22 smithi181 bash[24776]: cluster 2024-09-12T23:30:21.436830+0000 mgr.y (mgr.14150) 227 : cluster [DBG] pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-09-12T23:30:24.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:24 smithi181 bash[24776]: cephadm 2024-09-12T23:30:23.068002+0000 mgr.y (mgr.14150) 228 : cephadm [INF] Detected new or changed devices on smithi181 2024-09-12T23:30:24.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:24 smithi181 bash[24776]: audit 2024-09-12T23:30:23.076793+0000 mon.a (mon.0) 473 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:30:24.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:24 smithi181 bash[24776]: audit 2024-09-12T23:30:23.084828+0000 mon.a (mon.0) 474 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:30:24.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:24 smithi181 bash[24776]: audit 2024-09-12T23:30:23.086746+0000 mon.a (mon.0) 475 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:30:24.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:24 smithi181 bash[24776]: cephadm 2024-09-12T23:30:23.088277+0000 mgr.y (mgr.14150) 229 : cephadm [INF] Adjusting osd_memory_target on smithi181 to 17265M 2024-09-12T23:30:24.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:24 smithi181 bash[24776]: audit 2024-09-12T23:30:23.096397+0000 mon.a (mon.0) 476 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:30:24.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:24 smithi181 bash[24776]: audit 2024-09-12T23:30:23.098640+0000 mon.a (mon.0) 477 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:30:24.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:24 smithi181 bash[24776]: audit 2024-09-12T23:30:23.100352+0000 mon.a (mon.0) 478 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:30:24.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:24 smithi181 bash[24776]: audit 2024-09-12T23:30:23.110312+0000 mon.a (mon.0) 479 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:30:24.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:24 smithi181 bash[24776]: cluster 2024-09-12T23:30:23.437333+0000 mgr.y (mgr.14150) 230 : cluster [DBG] pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-12T23:30:24.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:24 smithi136 bash[22601]: cephadm 2024-09-12T23:30:23.068002+0000 mgr.y (mgr.14150) 228 : cephadm [INF] Detected new or changed devices on smithi181 2024-09-12T23:30:24.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:24 smithi136 bash[22601]: audit 2024-09-12T23:30:23.076793+0000 mon.a (mon.0) 473 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:30:24.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:24 smithi136 bash[22601]: audit 2024-09-12T23:30:23.084828+0000 mon.a (mon.0) 474 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:30:24.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:24 smithi136 bash[22601]: audit 2024-09-12T23:30:23.086746+0000 mon.a (mon.0) 475 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:30:24.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:24 smithi136 bash[22601]: cephadm 2024-09-12T23:30:23.088277+0000 mgr.y (mgr.14150) 229 : cephadm [INF] Adjusting osd_memory_target on smithi181 to 17265M 2024-09-12T23:30:24.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:24 smithi136 bash[22601]: audit 2024-09-12T23:30:23.096397+0000 mon.a (mon.0) 476 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:30:24.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:24 smithi136 bash[22601]: audit 2024-09-12T23:30:23.098640+0000 mon.a (mon.0) 477 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:30:24.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:24 smithi136 bash[22601]: audit 2024-09-12T23:30:23.100352+0000 mon.a (mon.0) 478 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:30:24.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:24 smithi136 bash[22601]: audit 2024-09-12T23:30:23.110312+0000 mon.a (mon.0) 479 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:30:24.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:24 smithi136 bash[22601]: cluster 2024-09-12T23:30:23.437333+0000 mgr.y (mgr.14150) 230 : cluster [DBG] pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-12T23:30:24.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:24 smithi136 bash[32189]: cephadm 2024-09-12T23:30:23.068002+0000 mgr.y (mgr.14150) 228 : cephadm [INF] Detected new or changed devices on smithi181 2024-09-12T23:30:24.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:24 smithi136 bash[32189]: audit 2024-09-12T23:30:23.076793+0000 mon.a (mon.0) 473 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:30:24.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:24 smithi136 bash[32189]: audit 2024-09-12T23:30:23.084828+0000 mon.a (mon.0) 474 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:30:24.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:24 smithi136 bash[32189]: audit 2024-09-12T23:30:23.086746+0000 mon.a (mon.0) 475 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:30:24.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:24 smithi136 bash[32189]: cephadm 2024-09-12T23:30:23.088277+0000 mgr.y (mgr.14150) 229 : cephadm [INF] Adjusting osd_memory_target on smithi181 to 17265M 2024-09-12T23:30:24.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:24 smithi136 bash[32189]: audit 2024-09-12T23:30:23.096397+0000 mon.a (mon.0) 476 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:30:24.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:24 smithi136 bash[32189]: audit 2024-09-12T23:30:23.098640+0000 mon.a (mon.0) 477 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:30:24.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:24 smithi136 bash[32189]: audit 2024-09-12T23:30:23.100352+0000 mon.a (mon.0) 478 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:30:24.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:24 smithi136 bash[32189]: audit 2024-09-12T23:30:23.110312+0000 mon.a (mon.0) 479 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:30:24.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:24 smithi136 bash[32189]: cluster 2024-09-12T23:30:23.437333+0000 mgr.y (mgr.14150) 230 : cluster [DBG] pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-12T23:30:26.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:26 smithi136 bash[22601]: cluster 2024-09-12T23:30:25.437900+0000 mgr.y (mgr.14150) 231 : cluster [DBG] pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-12T23:30:26.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:26 smithi136 bash[32189]: cluster 2024-09-12T23:30:25.437900+0000 mgr.y (mgr.14150) 231 : cluster [DBG] pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-12T23:30:26.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:26 smithi181 bash[24776]: cluster 2024-09-12T23:30:25.437900+0000 mgr.y (mgr.14150) 231 : cluster [DBG] pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-12T23:30:27.473 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.b/config 2024-09-12T23:30:28.650 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:28 smithi181 bash[24776]: cluster 2024-09-12T23:30:27.438500+0000 mgr.y (mgr.14150) 232 : cluster [DBG] pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 49 KiB/s, 0 objects/s recovering 2024-09-12T23:30:28.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:28 smithi136 bash[22601]: cluster 2024-09-12T23:30:27.438500+0000 mgr.y (mgr.14150) 232 : cluster [DBG] pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 49 KiB/s, 0 objects/s recovering 2024-09-12T23:30:28.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:28 smithi136 bash[32189]: cluster 2024-09-12T23:30:27.438500+0000 mgr.y (mgr.14150) 232 : cluster [DBG] pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 49 KiB/s, 0 objects/s recovering 2024-09-12T23:30:30.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:30 smithi136 bash[22601]: cluster 2024-09-12T23:30:29.439046+0000 mgr.y (mgr.14150) 233 : cluster [DBG] pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:30:30.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:30 smithi136 bash[22601]: audit 2024-09-12T23:30:30.177080+0000 mon.a (mon.0) 480 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-12T23:30:30.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:30 smithi136 bash[22601]: audit 2024-09-12T23:30:30.183124+0000 mon.a (mon.0) 481 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-12T23:30:30.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:30 smithi136 bash[22601]: audit 2024-09-12T23:30:30.184870+0000 mon.a (mon.0) 482 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:30:30.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:30 smithi136 bash[32189]: cluster 2024-09-12T23:30:29.439046+0000 mgr.y (mgr.14150) 233 : cluster [DBG] pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:30:30.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:30 smithi136 bash[32189]: audit 2024-09-12T23:30:30.177080+0000 mon.a (mon.0) 480 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-12T23:30:30.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:30 smithi136 bash[32189]: audit 2024-09-12T23:30:30.183124+0000 mon.a (mon.0) 481 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-12T23:30:30.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:30 smithi136 bash[32189]: audit 2024-09-12T23:30:30.184870+0000 mon.a (mon.0) 482 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:30:30.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:30 smithi181 bash[24776]: cluster 2024-09-12T23:30:29.439046+0000 mgr.y (mgr.14150) 233 : cluster [DBG] pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:30:30.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:30 smithi181 bash[24776]: audit 2024-09-12T23:30:30.177080+0000 mon.a (mon.0) 480 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-12T23:30:30.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:30 smithi181 bash[24776]: audit 2024-09-12T23:30:30.183124+0000 mon.a (mon.0) 481 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-12T23:30:30.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:30 smithi181 bash[24776]: audit 2024-09-12T23:30:30.184870+0000 mon.a (mon.0) 482 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:30:31.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:31 smithi136 bash[22601]: audit 2024-09-12T23:30:30.170671+0000 mgr.y (mgr.14150) 234 : audit [DBG] from='client.24196 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:30:31.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:31 smithi136 bash[32189]: audit 2024-09-12T23:30:30.170671+0000 mgr.y (mgr.14150) 234 : audit [DBG] from='client.24196 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:30:31.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:31 smithi181 bash[24776]: audit 2024-09-12T23:30:30.170671+0000 mgr.y (mgr.14150) 234 : audit [DBG] from='client.24196 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:30:32.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:32 smithi181 bash[24776]: cluster 2024-09-12T23:30:31.439577+0000 mgr.y (mgr.14150) 235 : cluster [DBG] pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:30:33.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:32 smithi136 bash[22601]: cluster 2024-09-12T23:30:31.439577+0000 mgr.y (mgr.14150) 235 : cluster [DBG] pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:30:33.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:32 smithi136 bash[32189]: cluster 2024-09-12T23:30:31.439577+0000 mgr.y (mgr.14150) 235 : cluster [DBG] pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:30:34.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:34 smithi181 bash[24776]: cluster 2024-09-12T23:30:33.440124+0000 mgr.y (mgr.14150) 236 : cluster [DBG] pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:30:35.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:34 smithi136 bash[22601]: cluster 2024-09-12T23:30:33.440124+0000 mgr.y (mgr.14150) 236 : cluster [DBG] pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:30:35.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:34 smithi136 bash[32189]: cluster 2024-09-12T23:30:33.440124+0000 mgr.y (mgr.14150) 236 : cluster [DBG] pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:30:36.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:36 smithi181 bash[24776]: cluster 2024-09-12T23:30:35.440746+0000 mgr.y (mgr.14150) 237 : cluster [DBG] pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:30:37.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:36 smithi136 bash[22601]: cluster 2024-09-12T23:30:35.440746+0000 mgr.y (mgr.14150) 237 : cluster [DBG] pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:30:37.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:36 smithi136 bash[32189]: cluster 2024-09-12T23:30:35.440746+0000 mgr.y (mgr.14150) 237 : cluster [DBG] pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:30:38.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:38 smithi181 bash[24776]: cluster 2024-09-12T23:30:37.441321+0000 mgr.y (mgr.14150) 238 : cluster [DBG] pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:30:38.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:38 smithi181 bash[24776]: audit 2024-09-12T23:30:38.125239+0000 mon.b (mon.1) 13 : audit [INF] from='client.? 172.21.15.181:0/1906747639' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5c6d2f69-7f9c-4689-95b0-bd4eb858956d"}]: dispatch 2024-09-12T23:30:38.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:38 smithi181 bash[24776]: audit 2024-09-12T23:30:38.126462+0000 mon.a (mon.0) 483 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5c6d2f69-7f9c-4689-95b0-bd4eb858956d"}]: dispatch 2024-09-12T23:30:38.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:38 smithi181 bash[24776]: audit 2024-09-12T23:30:38.134860+0000 mon.a (mon.0) 484 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5c6d2f69-7f9c-4689-95b0-bd4eb858956d"}]': finished 2024-09-12T23:30:38.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:38 smithi181 bash[24776]: cluster 2024-09-12T23:30:38.134972+0000 mon.a (mon.0) 485 : cluster [DBG] osdmap e33: 6 total, 5 up, 6 in 2024-09-12T23:30:38.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:38 smithi181 bash[24776]: audit 2024-09-12T23:30:38.135252+0000 mon.a (mon.0) 486 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-12T23:30:39.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:38 smithi136 bash[22601]: cluster 2024-09-12T23:30:37.441321+0000 mgr.y (mgr.14150) 238 : cluster [DBG] pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:30:39.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:38 smithi136 bash[22601]: audit 2024-09-12T23:30:38.125239+0000 mon.b (mon.1) 13 : audit [INF] from='client.? 172.21.15.181:0/1906747639' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5c6d2f69-7f9c-4689-95b0-bd4eb858956d"}]: dispatch 2024-09-12T23:30:39.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:38 smithi136 bash[22601]: audit 2024-09-12T23:30:38.126462+0000 mon.a (mon.0) 483 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5c6d2f69-7f9c-4689-95b0-bd4eb858956d"}]: dispatch 2024-09-12T23:30:39.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:38 smithi136 bash[22601]: audit 2024-09-12T23:30:38.134860+0000 mon.a (mon.0) 484 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5c6d2f69-7f9c-4689-95b0-bd4eb858956d"}]': finished 2024-09-12T23:30:39.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:38 smithi136 bash[22601]: cluster 2024-09-12T23:30:38.134972+0000 mon.a (mon.0) 485 : cluster [DBG] osdmap e33: 6 total, 5 up, 6 in 2024-09-12T23:30:39.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:38 smithi136 bash[22601]: audit 2024-09-12T23:30:38.135252+0000 mon.a (mon.0) 486 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-12T23:30:39.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:38 smithi136 bash[32189]: cluster 2024-09-12T23:30:37.441321+0000 mgr.y (mgr.14150) 238 : cluster [DBG] pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:30:39.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:38 smithi136 bash[32189]: audit 2024-09-12T23:30:38.125239+0000 mon.b (mon.1) 13 : audit [INF] from='client.? 172.21.15.181:0/1906747639' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5c6d2f69-7f9c-4689-95b0-bd4eb858956d"}]: dispatch 2024-09-12T23:30:39.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:38 smithi136 bash[32189]: audit 2024-09-12T23:30:38.126462+0000 mon.a (mon.0) 483 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5c6d2f69-7f9c-4689-95b0-bd4eb858956d"}]: dispatch 2024-09-12T23:30:39.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:38 smithi136 bash[32189]: audit 2024-09-12T23:30:38.134860+0000 mon.a (mon.0) 484 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5c6d2f69-7f9c-4689-95b0-bd4eb858956d"}]': finished 2024-09-12T23:30:39.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:38 smithi136 bash[32189]: cluster 2024-09-12T23:30:38.134972+0000 mon.a (mon.0) 485 : cluster [DBG] osdmap e33: 6 total, 5 up, 6 in 2024-09-12T23:30:39.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:38 smithi136 bash[32189]: audit 2024-09-12T23:30:38.135252+0000 mon.a (mon.0) 486 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-12T23:30:39.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:39 smithi181 bash[24776]: audit 2024-09-12T23:30:38.795512+0000 mon.b (mon.1) 14 : audit [DBG] from='client.? 172.21.15.181:0/1372031940' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-12T23:30:40.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:39 smithi136 bash[22601]: audit 2024-09-12T23:30:38.795512+0000 mon.b (mon.1) 14 : audit [DBG] from='client.? 172.21.15.181:0/1372031940' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-12T23:30:40.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:39 smithi136 bash[32189]: audit 2024-09-12T23:30:38.795512+0000 mon.b (mon.1) 14 : audit [DBG] from='client.? 172.21.15.181:0/1372031940' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-12T23:30:40.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:40 smithi181 bash[24776]: cluster 2024-09-12T23:30:39.441836+0000 mgr.y (mgr.14150) 239 : cluster [DBG] pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:30:41.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:40 smithi136 bash[22601]: cluster 2024-09-12T23:30:39.441836+0000 mgr.y (mgr.14150) 239 : cluster [DBG] pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:30:41.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:40 smithi136 bash[32189]: cluster 2024-09-12T23:30:39.441836+0000 mgr.y (mgr.14150) 239 : cluster [DBG] pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:30:42.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:42 smithi181 bash[24776]: cluster 2024-09-12T23:30:41.442378+0000 mgr.y (mgr.14150) 240 : cluster [DBG] pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:30:43.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:42 smithi136 bash[22601]: cluster 2024-09-12T23:30:41.442378+0000 mgr.y (mgr.14150) 240 : cluster [DBG] pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:30:43.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:42 smithi136 bash[32189]: cluster 2024-09-12T23:30:41.442378+0000 mgr.y (mgr.14150) 240 : cluster [DBG] pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:30:44.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:44 smithi181 bash[24776]: cluster 2024-09-12T23:30:43.442895+0000 mgr.y (mgr.14150) 241 : cluster [DBG] pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:30:45.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:44 smithi136 bash[22601]: cluster 2024-09-12T23:30:43.442895+0000 mgr.y (mgr.14150) 241 : cluster [DBG] pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:30:45.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:44 smithi136 bash[32189]: cluster 2024-09-12T23:30:43.442895+0000 mgr.y (mgr.14150) 241 : cluster [DBG] pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:30:46.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:46 smithi181 bash[24776]: cluster 2024-09-12T23:30:45.443409+0000 mgr.y (mgr.14150) 242 : cluster [DBG] pgmap v213: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:30:47.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:46 smithi136 bash[22601]: cluster 2024-09-12T23:30:45.443409+0000 mgr.y (mgr.14150) 242 : cluster [DBG] pgmap v213: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:30:47.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:46 smithi136 bash[32189]: cluster 2024-09-12T23:30:45.443409+0000 mgr.y (mgr.14150) 242 : cluster [DBG] pgmap v213: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:30:48.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:48 smithi181 bash[24776]: cluster 2024-09-12T23:30:47.443933+0000 mgr.y (mgr.14150) 243 : cluster [DBG] pgmap v214: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:30:49.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:48 smithi136 bash[22601]: cluster 2024-09-12T23:30:47.443933+0000 mgr.y (mgr.14150) 243 : cluster [DBG] pgmap v214: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:30:49.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:48 smithi136 bash[32189]: cluster 2024-09-12T23:30:47.443933+0000 mgr.y (mgr.14150) 243 : cluster [DBG] pgmap v214: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:30:50.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:50 smithi181 bash[24776]: cluster 2024-09-12T23:30:49.444440+0000 mgr.y (mgr.14150) 244 : cluster [DBG] pgmap v215: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:30:50.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:50 smithi181 bash[24776]: audit 2024-09-12T23:30:50.194244+0000 mon.a (mon.0) 487 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-12T23:30:50.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:50 smithi181 bash[24776]: audit 2024-09-12T23:30:50.195888+0000 mon.a (mon.0) 488 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:30:51.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:50 smithi136 bash[22601]: cluster 2024-09-12T23:30:49.444440+0000 mgr.y (mgr.14150) 244 : cluster [DBG] pgmap v215: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:30:51.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:50 smithi136 bash[22601]: audit 2024-09-12T23:30:50.194244+0000 mon.a (mon.0) 487 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-12T23:30:51.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:50 smithi136 bash[22601]: audit 2024-09-12T23:30:50.195888+0000 mon.a (mon.0) 488 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:30:51.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:50 smithi136 bash[32189]: cluster 2024-09-12T23:30:49.444440+0000 mgr.y (mgr.14150) 244 : cluster [DBG] pgmap v215: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:30:51.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:50 smithi136 bash[32189]: audit 2024-09-12T23:30:50.194244+0000 mon.a (mon.0) 487 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-12T23:30:51.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:50 smithi136 bash[32189]: audit 2024-09-12T23:30:50.195888+0000 mon.a (mon.0) 488 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:30:51.730 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:51 smithi181 bash[24776]: cephadm 2024-09-12T23:30:50.197372+0000 mgr.y (mgr.14150) 245 : cephadm [INF] Deploying daemon osd.5 on smithi181 2024-09-12T23:30:52.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:51 smithi136 bash[22601]: cephadm 2024-09-12T23:30:50.197372+0000 mgr.y (mgr.14150) 245 : cephadm [INF] Deploying daemon osd.5 on smithi181 2024-09-12T23:30:52.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:51 smithi136 bash[32189]: cephadm 2024-09-12T23:30:50.197372+0000 mgr.y (mgr.14150) 245 : cephadm [INF] Deploying daemon osd.5 on smithi181 2024-09-12T23:30:52.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:52 smithi181 bash[24776]: cluster 2024-09-12T23:30:51.444978+0000 mgr.y (mgr.14150) 246 : cluster [DBG] pgmap v216: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:30:53.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:52 smithi136 bash[32189]: cluster 2024-09-12T23:30:51.444978+0000 mgr.y (mgr.14150) 246 : cluster [DBG] pgmap v216: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:30:53.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:52 smithi136 bash[22601]: cluster 2024-09-12T23:30:51.444978+0000 mgr.y (mgr.14150) 246 : cluster [DBG] pgmap v216: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:30:54.130 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:54 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:30:54.130 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:30:54 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:30:54.131 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:30:54 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:30:54.488 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:54 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:30:54.488 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:30:54 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:30:54.489 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:30:54 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:30:54.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:54 smithi181 bash[24776]: cluster 2024-09-12T23:30:53.445529+0000 mgr.y (mgr.14150) 247 : cluster [DBG] pgmap v217: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:30:54.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:54 smithi181 bash[24776]: audit 2024-09-12T23:30:54.409954+0000 mon.a (mon.0) 489 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:30:54.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:54 smithi181 bash[24776]: audit 2024-09-12T23:30:54.417679+0000 mon.a (mon.0) 490 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:30:54.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:54 smithi181 bash[24776]: audit 2024-09-12T23:30:54.426318+0000 mon.a (mon.0) 491 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:30:55.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:54 smithi136 bash[22601]: cluster 2024-09-12T23:30:53.445529+0000 mgr.y (mgr.14150) 247 : cluster [DBG] pgmap v217: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:30:55.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:54 smithi136 bash[22601]: audit 2024-09-12T23:30:54.409954+0000 mon.a (mon.0) 489 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:30:55.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:54 smithi136 bash[22601]: audit 2024-09-12T23:30:54.417679+0000 mon.a (mon.0) 490 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:30:55.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:54 smithi136 bash[22601]: audit 2024-09-12T23:30:54.426318+0000 mon.a (mon.0) 491 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:30:55.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:54 smithi136 bash[32189]: cluster 2024-09-12T23:30:53.445529+0000 mgr.y (mgr.14150) 247 : cluster [DBG] pgmap v217: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:30:55.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:54 smithi136 bash[32189]: audit 2024-09-12T23:30:54.409954+0000 mon.a (mon.0) 489 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:30:55.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:54 smithi136 bash[32189]: audit 2024-09-12T23:30:54.417679+0000 mon.a (mon.0) 490 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:30:55.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:54 smithi136 bash[32189]: audit 2024-09-12T23:30:54.426318+0000 mon.a (mon.0) 491 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:30:56.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:56 smithi181 bash[24776]: cluster 2024-09-12T23:30:55.446048+0000 mgr.y (mgr.14150) 248 : cluster [DBG] pgmap v218: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:30:57.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:56 smithi136 bash[22601]: cluster 2024-09-12T23:30:55.446048+0000 mgr.y (mgr.14150) 248 : cluster [DBG] pgmap v218: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:30:57.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:56 smithi136 bash[32189]: cluster 2024-09-12T23:30:55.446048+0000 mgr.y (mgr.14150) 248 : cluster [DBG] pgmap v218: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:30:58.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:30:58 smithi181 bash[24776]: cluster 2024-09-12T23:30:57.446609+0000 mgr.y (mgr.14150) 249 : cluster [DBG] pgmap v219: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:30:59.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:30:58 smithi136 bash[22601]: cluster 2024-09-12T23:30:57.446609+0000 mgr.y (mgr.14150) 249 : cluster [DBG] pgmap v219: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:30:59.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:30:58 smithi136 bash[32189]: cluster 2024-09-12T23:30:57.446609+0000 mgr.y (mgr.14150) 249 : cluster [DBG] pgmap v219: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:31:00.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:00 smithi136 bash[22601]: cluster 2024-09-12T23:30:59.447126+0000 mgr.y (mgr.14150) 250 : cluster [DBG] pgmap v220: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:31:00.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:00 smithi136 bash[22601]: audit 2024-09-12T23:30:59.470338+0000 mon.a (mon.0) 492 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:31:00.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:00 smithi136 bash[22601]: audit 2024-09-12T23:30:59.478942+0000 mon.a (mon.0) 493 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:31:00.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:00 smithi136 bash[22601]: audit 2024-09-12T23:30:59.911067+0000 mon.a (mon.0) 494 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:31:00.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:00 smithi136 bash[22601]: audit 2024-09-12T23:30:59.912829+0000 mon.a (mon.0) 495 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:31:00.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:00 smithi136 bash[22601]: audit 2024-09-12T23:30:59.922033+0000 mon.a (mon.0) 496 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:31:00.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:00 smithi136 bash[32189]: cluster 2024-09-12T23:30:59.447126+0000 mgr.y (mgr.14150) 250 : cluster [DBG] pgmap v220: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:31:00.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:00 smithi136 bash[32189]: audit 2024-09-12T23:30:59.470338+0000 mon.a (mon.0) 492 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:31:00.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:00 smithi136 bash[32189]: audit 2024-09-12T23:30:59.478942+0000 mon.a (mon.0) 493 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:31:00.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:00 smithi136 bash[32189]: audit 2024-09-12T23:30:59.911067+0000 mon.a (mon.0) 494 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:31:00.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:00 smithi136 bash[32189]: audit 2024-09-12T23:30:59.912829+0000 mon.a (mon.0) 495 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:31:00.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:00 smithi136 bash[32189]: audit 2024-09-12T23:30:59.922033+0000 mon.a (mon.0) 496 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:31:00.792 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:00 smithi181 bash[24776]: cluster 2024-09-12T23:30:59.447126+0000 mgr.y (mgr.14150) 250 : cluster [DBG] pgmap v220: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:31:00.793 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:00 smithi181 bash[24776]: audit 2024-09-12T23:30:59.470338+0000 mon.a (mon.0) 492 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:31:00.793 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:00 smithi181 bash[24776]: audit 2024-09-12T23:30:59.478942+0000 mon.a (mon.0) 493 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:31:00.793 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:00 smithi181 bash[24776]: audit 2024-09-12T23:30:59.911067+0000 mon.a (mon.0) 494 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:31:00.793 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:00 smithi181 bash[24776]: audit 2024-09-12T23:30:59.912829+0000 mon.a (mon.0) 495 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:31:00.793 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:00 smithi181 bash[24776]: audit 2024-09-12T23:30:59.922033+0000 mon.a (mon.0) 496 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:31:01.735 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:01 smithi181 bash[24776]: audit 2024-09-12T23:31:00.798275+0000 mon.b (mon.1) 15 : audit [INF] from='osd.5 [v2:172.21.15.181:6808/2109931558,v1:172.21.15.181:6809/2109931558]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-12T23:31:01.735 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:01 smithi181 bash[24776]: audit 2024-09-12T23:31:00.799620+0000 mon.a (mon.0) 497 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-12T23:31:01.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:01 smithi136 bash[22601]: audit 2024-09-12T23:31:00.798275+0000 mon.b (mon.1) 15 : audit [INF] from='osd.5 [v2:172.21.15.181:6808/2109931558,v1:172.21.15.181:6809/2109931558]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-12T23:31:01.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:01 smithi136 bash[22601]: audit 2024-09-12T23:31:00.799620+0000 mon.a (mon.0) 497 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-12T23:31:01.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:01 smithi136 bash[32189]: audit 2024-09-12T23:31:00.798275+0000 mon.b (mon.1) 15 : audit [INF] from='osd.5 [v2:172.21.15.181:6808/2109931558,v1:172.21.15.181:6809/2109931558]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-12T23:31:01.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:01 smithi136 bash[32189]: audit 2024-09-12T23:31:00.799620+0000 mon.a (mon.0) 497 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-12T23:31:02.719 INFO:teuthology.orchestra.run.smithi181.stdout:Created osd(s) 5 on host 'smithi181' 2024-09-12T23:31:02.736 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:02 smithi181 bash[24776]: cluster 2024-09-12T23:31:01.447636+0000 mgr.y (mgr.14150) 251 : cluster [DBG] pgmap v221: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:31:02.736 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:02 smithi181 bash[24776]: audit 2024-09-12T23:31:01.483310+0000 mon.a (mon.0) 498 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-12T23:31:02.736 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:02 smithi181 bash[24776]: cluster 2024-09-12T23:31:01.483435+0000 mon.a (mon.0) 499 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2024-09-12T23:31:02.736 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:02 smithi181 bash[24776]: audit 2024-09-12T23:31:01.483723+0000 mon.a (mon.0) 500 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-12T23:31:02.736 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:02 smithi181 bash[24776]: audit 2024-09-12T23:31:01.483816+0000 mon.b (mon.1) 16 : audit [INF] from='osd.5 [v2:172.21.15.181:6808/2109931558,v1:172.21.15.181:6809/2109931558]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-12T23:31:02.736 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:02 smithi181 bash[24776]: audit 2024-09-12T23:31:01.485017+0000 mon.a (mon.0) 501 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-12T23:31:02.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:02 smithi136 bash[22601]: cluster 2024-09-12T23:31:01.447636+0000 mgr.y (mgr.14150) 251 : cluster [DBG] pgmap v221: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:31:02.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:02 smithi136 bash[22601]: audit 2024-09-12T23:31:01.483310+0000 mon.a (mon.0) 498 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-12T23:31:02.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:02 smithi136 bash[22601]: cluster 2024-09-12T23:31:01.483435+0000 mon.a (mon.0) 499 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2024-09-12T23:31:02.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:02 smithi136 bash[22601]: audit 2024-09-12T23:31:01.483723+0000 mon.a (mon.0) 500 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-12T23:31:02.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:02 smithi136 bash[22601]: audit 2024-09-12T23:31:01.483816+0000 mon.b (mon.1) 16 : audit [INF] from='osd.5 [v2:172.21.15.181:6808/2109931558,v1:172.21.15.181:6809/2109931558]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-12T23:31:02.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:02 smithi136 bash[22601]: audit 2024-09-12T23:31:01.485017+0000 mon.a (mon.0) 501 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-12T23:31:02.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:02 smithi136 bash[32189]: cluster 2024-09-12T23:31:01.447636+0000 mgr.y (mgr.14150) 251 : cluster [DBG] pgmap v221: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:31:02.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:02 smithi136 bash[32189]: audit 2024-09-12T23:31:01.483310+0000 mon.a (mon.0) 498 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-12T23:31:02.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:02 smithi136 bash[32189]: cluster 2024-09-12T23:31:01.483435+0000 mon.a (mon.0) 499 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2024-09-12T23:31:02.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:02 smithi136 bash[32189]: audit 2024-09-12T23:31:01.483723+0000 mon.a (mon.0) 500 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-12T23:31:02.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:02 smithi136 bash[32189]: audit 2024-09-12T23:31:01.483816+0000 mon.b (mon.1) 16 : audit [INF] from='osd.5 [v2:172.21.15.181:6808/2109931558,v1:172.21.15.181:6809/2109931558]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-12T23:31:02.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:02 smithi136 bash[32189]: audit 2024-09-12T23:31:01.485017+0000 mon.a (mon.0) 501 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-12T23:31:03.282 DEBUG:teuthology.orchestra.run.smithi181:osd.5> sudo journalctl -f -n 0 -u ceph-da58a47c-715d-11ef-bceb-c7b262605968@osd.5.service 2024-09-12T23:31:03.285 INFO:tasks.cephadm:Deploying osd.6 on smithi181 with /dev/vg_nvme/lv_2... 2024-09-12T23:31:03.285 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-09-12T23:31:03.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:03 smithi181 bash[24776]: audit 2024-09-12T23:31:02.483853+0000 mon.a (mon.0) 502 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2024-09-12T23:31:03.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:03 smithi181 bash[24776]: cluster 2024-09-12T23:31:02.484038+0000 mon.a (mon.0) 503 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2024-09-12T23:31:03.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:03 smithi181 bash[24776]: audit 2024-09-12T23:31:02.484903+0000 mon.a (mon.0) 504 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-12T23:31:03.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:03 smithi181 bash[24776]: audit 2024-09-12T23:31:02.495224+0000 mon.a (mon.0) 505 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-12T23:31:03.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:03 smithi181 bash[24776]: audit 2024-09-12T23:31:02.694545+0000 mon.a (mon.0) 506 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:31:03.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:03 smithi181 bash[24776]: audit 2024-09-12T23:31:02.704195+0000 mon.a (mon.0) 507 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:31:03.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:03 smithi181 bash[24776]: audit 2024-09-12T23:31:02.716533+0000 mon.a (mon.0) 508 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:31:03.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:03 smithi136 bash[32189]: audit 2024-09-12T23:31:02.483853+0000 mon.a (mon.0) 502 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2024-09-12T23:31:03.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:03 smithi136 bash[32189]: cluster 2024-09-12T23:31:02.484038+0000 mon.a (mon.0) 503 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2024-09-12T23:31:03.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:03 smithi136 bash[32189]: audit 2024-09-12T23:31:02.484903+0000 mon.a (mon.0) 504 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-12T23:31:03.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:03 smithi136 bash[32189]: audit 2024-09-12T23:31:02.495224+0000 mon.a (mon.0) 505 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-12T23:31:03.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:03 smithi136 bash[32189]: audit 2024-09-12T23:31:02.694545+0000 mon.a (mon.0) 506 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:31:03.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:03 smithi136 bash[32189]: audit 2024-09-12T23:31:02.704195+0000 mon.a (mon.0) 507 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:31:03.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:03 smithi136 bash[32189]: audit 2024-09-12T23:31:02.716533+0000 mon.a (mon.0) 508 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:31:03.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:03 smithi136 bash[22601]: audit 2024-09-12T23:31:02.483853+0000 mon.a (mon.0) 502 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2024-09-12T23:31:03.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:03 smithi136 bash[22601]: cluster 2024-09-12T23:31:02.484038+0000 mon.a (mon.0) 503 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2024-09-12T23:31:03.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:03 smithi136 bash[22601]: audit 2024-09-12T23:31:02.484903+0000 mon.a (mon.0) 504 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-12T23:31:03.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:03 smithi136 bash[22601]: audit 2024-09-12T23:31:02.495224+0000 mon.a (mon.0) 505 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-12T23:31:03.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:03 smithi136 bash[22601]: audit 2024-09-12T23:31:02.694545+0000 mon.a (mon.0) 506 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:31:03.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:03 smithi136 bash[22601]: audit 2024-09-12T23:31:02.704195+0000 mon.a (mon.0) 507 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:31:03.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:03 smithi136 bash[22601]: audit 2024-09-12T23:31:02.716533+0000 mon.a (mon.0) 508 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:31:04.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:04 smithi181 bash[24776]: cluster 2024-09-12T23:31:01.812703+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-12T23:31:04.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:04 smithi181 bash[24776]: cluster 2024-09-12T23:31:01.812763+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-12T23:31:04.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:04 smithi181 bash[24776]: cluster 2024-09-12T23:31:03.448170+0000 mgr.y (mgr.14150) 252 : cluster [DBG] pgmap v224: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:31:04.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:04 smithi181 bash[24776]: cluster 2024-09-12T23:31:03.495737+0000 mon.a (mon.0) 509 : cluster [INF] osd.5 [v2:172.21.15.181:6808/2109931558,v1:172.21.15.181:6809/2109931558] boot 2024-09-12T23:31:04.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:04 smithi181 bash[24776]: cluster 2024-09-12T23:31:03.495860+0000 mon.a (mon.0) 510 : cluster [DBG] osdmap e36: 6 total, 6 up, 6 in 2024-09-12T23:31:04.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:04 smithi181 bash[24776]: audit 2024-09-12T23:31:03.496826+0000 mon.a (mon.0) 511 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-12T23:31:05.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:04 smithi136 bash[32189]: cluster 2024-09-12T23:31:01.812703+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-12T23:31:05.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:04 smithi136 bash[32189]: cluster 2024-09-12T23:31:01.812763+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-12T23:31:05.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:04 smithi136 bash[32189]: cluster 2024-09-12T23:31:03.448170+0000 mgr.y (mgr.14150) 252 : cluster [DBG] pgmap v224: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:31:05.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:04 smithi136 bash[32189]: cluster 2024-09-12T23:31:03.495737+0000 mon.a (mon.0) 509 : cluster [INF] osd.5 [v2:172.21.15.181:6808/2109931558,v1:172.21.15.181:6809/2109931558] boot 2024-09-12T23:31:05.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:04 smithi136 bash[32189]: cluster 2024-09-12T23:31:03.495860+0000 mon.a (mon.0) 510 : cluster [DBG] osdmap e36: 6 total, 6 up, 6 in 2024-09-12T23:31:05.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:04 smithi136 bash[32189]: audit 2024-09-12T23:31:03.496826+0000 mon.a (mon.0) 511 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-12T23:31:05.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:04 smithi136 bash[22601]: cluster 2024-09-12T23:31:01.812703+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-12T23:31:05.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:04 smithi136 bash[22601]: cluster 2024-09-12T23:31:01.812763+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-12T23:31:05.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:04 smithi136 bash[22601]: cluster 2024-09-12T23:31:03.448170+0000 mgr.y (mgr.14150) 252 : cluster [DBG] pgmap v224: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-12T23:31:05.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:04 smithi136 bash[22601]: cluster 2024-09-12T23:31:03.495737+0000 mon.a (mon.0) 509 : cluster [INF] osd.5 [v2:172.21.15.181:6808/2109931558,v1:172.21.15.181:6809/2109931558] boot 2024-09-12T23:31:05.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:04 smithi136 bash[22601]: cluster 2024-09-12T23:31:03.495860+0000 mon.a (mon.0) 510 : cluster [DBG] osdmap e36: 6 total, 6 up, 6 in 2024-09-12T23:31:05.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:04 smithi136 bash[22601]: audit 2024-09-12T23:31:03.496826+0000 mon.a (mon.0) 511 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-12T23:31:05.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:05 smithi181 bash[24776]: cluster 2024-09-12T23:31:04.510994+0000 mon.a (mon.0) 512 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2024-09-12T23:31:06.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:05 smithi136 bash[22601]: cluster 2024-09-12T23:31:04.510994+0000 mon.a (mon.0) 512 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2024-09-12T23:31:06.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:05 smithi136 bash[32189]: cluster 2024-09-12T23:31:04.510994+0000 mon.a (mon.0) 512 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2024-09-12T23:31:06.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:06 smithi181 bash[24776]: cluster 2024-09-12T23:31:05.448832+0000 mgr.y (mgr.14150) 253 : cluster [DBG] pgmap v227: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:06.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:06 smithi181 bash[24776]: cluster 2024-09-12T23:31:05.512406+0000 mon.a (mon.0) 513 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2024-09-12T23:31:06.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:06 smithi181 bash[24776]: cluster 2024-09-12T23:31:05.512930+0000 mon.a (mon.0) 514 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-12T23:31:07.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:06 smithi136 bash[22601]: cluster 2024-09-12T23:31:05.448832+0000 mgr.y (mgr.14150) 253 : cluster [DBG] pgmap v227: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:07.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:06 smithi136 bash[22601]: cluster 2024-09-12T23:31:05.512406+0000 mon.a (mon.0) 513 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2024-09-12T23:31:07.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:06 smithi136 bash[22601]: cluster 2024-09-12T23:31:05.512930+0000 mon.a (mon.0) 514 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-12T23:31:07.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:06 smithi136 bash[32189]: cluster 2024-09-12T23:31:05.448832+0000 mgr.y (mgr.14150) 253 : cluster [DBG] pgmap v227: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:07.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:06 smithi136 bash[32189]: cluster 2024-09-12T23:31:05.512406+0000 mon.a (mon.0) 513 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2024-09-12T23:31:07.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:06 smithi136 bash[32189]: cluster 2024-09-12T23:31:05.512930+0000 mon.a (mon.0) 514 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-12T23:31:07.481 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.b/config 2024-09-12T23:31:08.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:08 smithi181 bash[24776]: cluster 2024-09-12T23:31:07.449525+0000 mgr.y (mgr.14150) 254 : cluster [DBG] pgmap v229: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:09.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:08 smithi136 bash[22601]: cluster 2024-09-12T23:31:07.449525+0000 mgr.y (mgr.14150) 254 : cluster [DBG] pgmap v229: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:09.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:08 smithi136 bash[32189]: cluster 2024-09-12T23:31:07.449525+0000 mgr.y (mgr.14150) 254 : cluster [DBG] pgmap v229: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:10.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:10 smithi181 bash[24776]: cluster 2024-09-12T23:31:09.450223+0000 mgr.y (mgr.14150) 255 : cluster [DBG] pgmap v230: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:11.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:10 smithi136 bash[22601]: cluster 2024-09-12T23:31:09.450223+0000 mgr.y (mgr.14150) 255 : cluster [DBG] pgmap v230: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:11.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:10 smithi136 bash[32189]: cluster 2024-09-12T23:31:09.450223+0000 mgr.y (mgr.14150) 255 : cluster [DBG] pgmap v230: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:11.708 INFO:teuthology.orchestra.run.smithi181.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-09-12T23:31:11.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:11 smithi181 bash[24776]: cluster 2024-09-12T23:31:11.532465+0000 mon.a (mon.0) 515 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-12T23:31:11.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:11 smithi181 bash[24776]: cluster 2024-09-12T23:31:11.532513+0000 mon.a (mon.0) 516 : cluster [INF] Cluster is now healthy 2024-09-12T23:31:11.890 INFO:teuthology.orchestra.run.smithi181.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-09-12T23:31:11.908 INFO:teuthology.orchestra.run.smithi181.stderr: stderr: 10+0 records in 2024-09-12T23:31:11.908 INFO:teuthology.orchestra.run.smithi181.stderr:10+0 records out 2024-09-12T23:31:11.908 INFO:teuthology.orchestra.run.smithi181.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0160022 s, 655 MB/s 2024-09-12T23:31:11.908 INFO:teuthology.orchestra.run.smithi181.stderr:--> Zapping successful for: 2024-09-12T23:31:12.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:11 smithi136 bash[22601]: cluster 2024-09-12T23:31:11.532465+0000 mon.a (mon.0) 515 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-12T23:31:12.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:11 smithi136 bash[22601]: cluster 2024-09-12T23:31:11.532513+0000 mon.a (mon.0) 516 : cluster [INF] Cluster is now healthy 2024-09-12T23:31:12.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:11 smithi136 bash[32189]: cluster 2024-09-12T23:31:11.532465+0000 mon.a (mon.0) 515 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-12T23:31:12.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:11 smithi136 bash[32189]: cluster 2024-09-12T23:31:11.532513+0000 mon.a (mon.0) 516 : cluster [INF] Cluster is now healthy 2024-09-12T23:31:12.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:12 smithi181 bash[24776]: cluster 2024-09-12T23:31:11.450872+0000 mgr.y (mgr.14150) 256 : cluster [DBG] pgmap v231: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:12.923 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi181:vg_nvme/lv_2 2024-09-12T23:31:13.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:12 smithi136 bash[22601]: cluster 2024-09-12T23:31:11.450872+0000 mgr.y (mgr.14150) 256 : cluster [DBG] pgmap v231: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:13.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:12 smithi136 bash[32189]: cluster 2024-09-12T23:31:11.450872+0000 mgr.y (mgr.14150) 256 : cluster [DBG] pgmap v231: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:14.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:14 smithi181 bash[24776]: cephadm 2024-09-12T23:31:13.084602+0000 mgr.y (mgr.14150) 257 : cephadm [INF] Detected new or changed devices on smithi181 2024-09-12T23:31:14.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:14 smithi181 bash[24776]: audit 2024-09-12T23:31:13.095389+0000 mon.a (mon.0) 517 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:31:14.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:14 smithi181 bash[24776]: audit 2024-09-12T23:31:13.106185+0000 mon.a (mon.0) 518 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:31:14.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:14 smithi181 bash[24776]: audit 2024-09-12T23:31:13.108225+0000 mon.a (mon.0) 519 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:31:14.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:14 smithi181 bash[24776]: audit 2024-09-12T23:31:13.110136+0000 mon.a (mon.0) 520 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:31:14.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:14 smithi181 bash[24776]: cephadm 2024-09-12T23:31:13.111435+0000 mgr.y (mgr.14150) 258 : cephadm [INF] Adjusting osd_memory_target on smithi181 to 8632M 2024-09-12T23:31:14.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:14 smithi181 bash[24776]: audit 2024-09-12T23:31:13.121813+0000 mon.a (mon.0) 521 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:31:14.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:14 smithi181 bash[24776]: audit 2024-09-12T23:31:13.124010+0000 mon.a (mon.0) 522 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:31:14.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:14 smithi181 bash[24776]: audit 2024-09-12T23:31:13.125755+0000 mon.a (mon.0) 523 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:31:14.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:14 smithi181 bash[24776]: audit 2024-09-12T23:31:13.137415+0000 mon.a (mon.0) 524 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:31:14.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:14 smithi181 bash[24776]: cluster 2024-09-12T23:31:13.451447+0000 mgr.y (mgr.14150) 259 : cluster [DBG] pgmap v232: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-09-12T23:31:14.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:14 smithi136 bash[22601]: cephadm 2024-09-12T23:31:13.084602+0000 mgr.y (mgr.14150) 257 : cephadm [INF] Detected new or changed devices on smithi181 2024-09-12T23:31:14.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:14 smithi136 bash[22601]: audit 2024-09-12T23:31:13.095389+0000 mon.a (mon.0) 517 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:31:14.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:14 smithi136 bash[22601]: audit 2024-09-12T23:31:13.106185+0000 mon.a (mon.0) 518 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:31:14.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:14 smithi136 bash[22601]: audit 2024-09-12T23:31:13.108225+0000 mon.a (mon.0) 519 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:31:14.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:14 smithi136 bash[22601]: audit 2024-09-12T23:31:13.110136+0000 mon.a (mon.0) 520 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:31:14.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:14 smithi136 bash[22601]: cephadm 2024-09-12T23:31:13.111435+0000 mgr.y (mgr.14150) 258 : cephadm [INF] Adjusting osd_memory_target on smithi181 to 8632M 2024-09-12T23:31:14.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:14 smithi136 bash[22601]: audit 2024-09-12T23:31:13.121813+0000 mon.a (mon.0) 521 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:31:14.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:14 smithi136 bash[22601]: audit 2024-09-12T23:31:13.124010+0000 mon.a (mon.0) 522 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:31:14.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:14 smithi136 bash[22601]: audit 2024-09-12T23:31:13.125755+0000 mon.a (mon.0) 523 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:31:14.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:14 smithi136 bash[22601]: audit 2024-09-12T23:31:13.137415+0000 mon.a (mon.0) 524 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:31:14.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:14 smithi136 bash[22601]: cluster 2024-09-12T23:31:13.451447+0000 mgr.y (mgr.14150) 259 : cluster [DBG] pgmap v232: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-09-12T23:31:14.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:14 smithi136 bash[32189]: cephadm 2024-09-12T23:31:13.084602+0000 mgr.y (mgr.14150) 257 : cephadm [INF] Detected new or changed devices on smithi181 2024-09-12T23:31:14.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:14 smithi136 bash[32189]: audit 2024-09-12T23:31:13.095389+0000 mon.a (mon.0) 517 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:31:14.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:14 smithi136 bash[32189]: audit 2024-09-12T23:31:13.106185+0000 mon.a (mon.0) 518 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:31:14.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:14 smithi136 bash[32189]: audit 2024-09-12T23:31:13.108225+0000 mon.a (mon.0) 519 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:31:14.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:14 smithi136 bash[32189]: audit 2024-09-12T23:31:13.110136+0000 mon.a (mon.0) 520 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:31:14.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:14 smithi136 bash[32189]: cephadm 2024-09-12T23:31:13.111435+0000 mgr.y (mgr.14150) 258 : cephadm [INF] Adjusting osd_memory_target on smithi181 to 8632M 2024-09-12T23:31:14.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:14 smithi136 bash[32189]: audit 2024-09-12T23:31:13.121813+0000 mon.a (mon.0) 521 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:31:14.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:14 smithi136 bash[32189]: audit 2024-09-12T23:31:13.124010+0000 mon.a (mon.0) 522 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:31:14.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:14 smithi136 bash[32189]: audit 2024-09-12T23:31:13.125755+0000 mon.a (mon.0) 523 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:31:14.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:14 smithi136 bash[32189]: audit 2024-09-12T23:31:13.137415+0000 mon.a (mon.0) 524 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:31:14.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:14 smithi136 bash[32189]: cluster 2024-09-12T23:31:13.451447+0000 mgr.y (mgr.14150) 259 : cluster [DBG] pgmap v232: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-09-12T23:31:16.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:16 smithi136 bash[22601]: cluster 2024-09-12T23:31:15.452058+0000 mgr.y (mgr.14150) 260 : cluster [DBG] pgmap v233: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-12T23:31:16.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:16 smithi136 bash[32189]: cluster 2024-09-12T23:31:15.452058+0000 mgr.y (mgr.14150) 260 : cluster [DBG] pgmap v233: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-12T23:31:16.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:16 smithi181 bash[24776]: cluster 2024-09-12T23:31:15.452058+0000 mgr.y (mgr.14150) 260 : cluster [DBG] pgmap v233: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-12T23:31:17.638 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.b/config 2024-09-12T23:31:18.766 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:18 smithi181 bash[24776]: cluster 2024-09-12T23:31:17.452613+0000 mgr.y (mgr.14150) 261 : cluster [DBG] pgmap v234: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-12T23:31:19.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:18 smithi136 bash[22601]: cluster 2024-09-12T23:31:17.452613+0000 mgr.y (mgr.14150) 261 : cluster [DBG] pgmap v234: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-12T23:31:19.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:18 smithi136 bash[32189]: cluster 2024-09-12T23:31:17.452613+0000 mgr.y (mgr.14150) 261 : cluster [DBG] pgmap v234: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-12T23:31:20.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:20 smithi181 bash[24776]: cluster 2024-09-12T23:31:19.453154+0000 mgr.y (mgr.14150) 262 : cluster [DBG] pgmap v235: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-12T23:31:21.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:20 smithi136 bash[22601]: cluster 2024-09-12T23:31:19.453154+0000 mgr.y (mgr.14150) 262 : cluster [DBG] pgmap v235: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-12T23:31:21.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:20 smithi136 bash[32189]: cluster 2024-09-12T23:31:19.453154+0000 mgr.y (mgr.14150) 262 : cluster [DBG] pgmap v235: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-12T23:31:21.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:21 smithi181 bash[24776]: audit 2024-09-12T23:31:20.547753+0000 mgr.y (mgr.14150) 263 : audit [DBG] from='client.24223 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:31:21.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:21 smithi181 bash[24776]: audit 2024-09-12T23:31:20.554190+0000 mon.a (mon.0) 525 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-12T23:31:21.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:21 smithi181 bash[24776]: audit 2024-09-12T23:31:20.560258+0000 mon.a (mon.0) 526 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-12T23:31:21.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:21 smithi181 bash[24776]: audit 2024-09-12T23:31:20.561845+0000 mon.a (mon.0) 527 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:31:22.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:21 smithi136 bash[22601]: audit 2024-09-12T23:31:20.547753+0000 mgr.y (mgr.14150) 263 : audit [DBG] from='client.24223 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:31:22.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:21 smithi136 bash[22601]: audit 2024-09-12T23:31:20.554190+0000 mon.a (mon.0) 525 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-12T23:31:22.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:21 smithi136 bash[22601]: audit 2024-09-12T23:31:20.560258+0000 mon.a (mon.0) 526 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-12T23:31:22.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:21 smithi136 bash[22601]: audit 2024-09-12T23:31:20.561845+0000 mon.a (mon.0) 527 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:31:22.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:21 smithi136 bash[32189]: audit 2024-09-12T23:31:20.547753+0000 mgr.y (mgr.14150) 263 : audit [DBG] from='client.24223 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:31:22.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:21 smithi136 bash[32189]: audit 2024-09-12T23:31:20.554190+0000 mon.a (mon.0) 525 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-12T23:31:22.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:21 smithi136 bash[32189]: audit 2024-09-12T23:31:20.560258+0000 mon.a (mon.0) 526 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-12T23:31:22.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:21 smithi136 bash[32189]: audit 2024-09-12T23:31:20.561845+0000 mon.a (mon.0) 527 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:31:22.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:22 smithi181 bash[24776]: cluster 2024-09-12T23:31:21.453680+0000 mgr.y (mgr.14150) 264 : cluster [DBG] pgmap v236: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-12T23:31:23.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:22 smithi136 bash[22601]: cluster 2024-09-12T23:31:21.453680+0000 mgr.y (mgr.14150) 264 : cluster [DBG] pgmap v236: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-12T23:31:23.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:22 smithi136 bash[32189]: cluster 2024-09-12T23:31:21.453680+0000 mgr.y (mgr.14150) 264 : cluster [DBG] pgmap v236: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-12T23:31:24.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:24 smithi181 bash[24776]: cluster 2024-09-12T23:31:23.454209+0000 mgr.y (mgr.14150) 265 : cluster [DBG] pgmap v237: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:25.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:24 smithi136 bash[22601]: cluster 2024-09-12T23:31:23.454209+0000 mgr.y (mgr.14150) 265 : cluster [DBG] pgmap v237: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:25.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:24 smithi136 bash[32189]: cluster 2024-09-12T23:31:23.454209+0000 mgr.y (mgr.14150) 265 : cluster [DBG] pgmap v237: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:26.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:26 smithi181 bash[24776]: cluster 2024-09-12T23:31:25.454764+0000 mgr.y (mgr.14150) 266 : cluster [DBG] pgmap v238: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:27.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:26 smithi136 bash[22601]: cluster 2024-09-12T23:31:25.454764+0000 mgr.y (mgr.14150) 266 : cluster [DBG] pgmap v238: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:27.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:26 smithi136 bash[32189]: cluster 2024-09-12T23:31:25.454764+0000 mgr.y (mgr.14150) 266 : cluster [DBG] pgmap v238: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:28.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:28 smithi181 bash[24776]: cluster 2024-09-12T23:31:27.455285+0000 mgr.y (mgr.14150) 267 : cluster [DBG] pgmap v239: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:28.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:28 smithi181 bash[24776]: audit 2024-09-12T23:31:28.525229+0000 mon.b (mon.1) 17 : audit [INF] from='client.? 172.21.15.181:0/2825916951' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "532ee052-af15-433a-8e93-b0b642aaf85b"}]: dispatch 2024-09-12T23:31:28.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:28 smithi181 bash[24776]: audit 2024-09-12T23:31:28.526585+0000 mon.a (mon.0) 528 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "532ee052-af15-433a-8e93-b0b642aaf85b"}]: dispatch 2024-09-12T23:31:28.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:28 smithi181 bash[24776]: audit 2024-09-12T23:31:28.536418+0000 mon.a (mon.0) 529 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "532ee052-af15-433a-8e93-b0b642aaf85b"}]': finished 2024-09-12T23:31:29.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:28 smithi136 bash[22601]: cluster 2024-09-12T23:31:27.455285+0000 mgr.y (mgr.14150) 267 : cluster [DBG] pgmap v239: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:29.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:28 smithi136 bash[22601]: audit 2024-09-12T23:31:28.525229+0000 mon.b (mon.1) 17 : audit [INF] from='client.? 172.21.15.181:0/2825916951' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "532ee052-af15-433a-8e93-b0b642aaf85b"}]: dispatch 2024-09-12T23:31:29.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:28 smithi136 bash[22601]: audit 2024-09-12T23:31:28.526585+0000 mon.a (mon.0) 528 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "532ee052-af15-433a-8e93-b0b642aaf85b"}]: dispatch 2024-09-12T23:31:29.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:28 smithi136 bash[22601]: audit 2024-09-12T23:31:28.536418+0000 mon.a (mon.0) 529 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "532ee052-af15-433a-8e93-b0b642aaf85b"}]': finished 2024-09-12T23:31:29.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:28 smithi136 bash[32189]: cluster 2024-09-12T23:31:27.455285+0000 mgr.y (mgr.14150) 267 : cluster [DBG] pgmap v239: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:29.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:28 smithi136 bash[32189]: audit 2024-09-12T23:31:28.525229+0000 mon.b (mon.1) 17 : audit [INF] from='client.? 172.21.15.181:0/2825916951' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "532ee052-af15-433a-8e93-b0b642aaf85b"}]: dispatch 2024-09-12T23:31:29.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:28 smithi136 bash[32189]: audit 2024-09-12T23:31:28.526585+0000 mon.a (mon.0) 528 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "532ee052-af15-433a-8e93-b0b642aaf85b"}]: dispatch 2024-09-12T23:31:29.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:28 smithi136 bash[32189]: audit 2024-09-12T23:31:28.536418+0000 mon.a (mon.0) 529 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "532ee052-af15-433a-8e93-b0b642aaf85b"}]': finished 2024-09-12T23:31:29.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:29 smithi181 bash[24776]: cluster 2024-09-12T23:31:28.536562+0000 mon.a (mon.0) 530 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2024-09-12T23:31:29.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:29 smithi181 bash[24776]: audit 2024-09-12T23:31:28.536875+0000 mon.a (mon.0) 531 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-12T23:31:29.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:29 smithi181 bash[24776]: audit 2024-09-12T23:31:29.188249+0000 mon.b (mon.1) 18 : audit [DBG] from='client.? 172.21.15.181:0/2551777937' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-12T23:31:30.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:29 smithi136 bash[22601]: cluster 2024-09-12T23:31:28.536562+0000 mon.a (mon.0) 530 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2024-09-12T23:31:30.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:29 smithi136 bash[22601]: audit 2024-09-12T23:31:28.536875+0000 mon.a (mon.0) 531 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-12T23:31:30.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:29 smithi136 bash[22601]: audit 2024-09-12T23:31:29.188249+0000 mon.b (mon.1) 18 : audit [DBG] from='client.? 172.21.15.181:0/2551777937' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-12T23:31:30.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:29 smithi136 bash[32189]: cluster 2024-09-12T23:31:28.536562+0000 mon.a (mon.0) 530 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2024-09-12T23:31:30.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:29 smithi136 bash[32189]: audit 2024-09-12T23:31:28.536875+0000 mon.a (mon.0) 531 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-12T23:31:30.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:29 smithi136 bash[32189]: audit 2024-09-12T23:31:29.188249+0000 mon.b (mon.1) 18 : audit [DBG] from='client.? 172.21.15.181:0/2551777937' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-12T23:31:30.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:30 smithi181 bash[24776]: cluster 2024-09-12T23:31:29.455818+0000 mgr.y (mgr.14150) 268 : cluster [DBG] pgmap v241: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:31.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:30 smithi136 bash[22601]: cluster 2024-09-12T23:31:29.455818+0000 mgr.y (mgr.14150) 268 : cluster [DBG] pgmap v241: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:31.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:30 smithi136 bash[32189]: cluster 2024-09-12T23:31:29.455818+0000 mgr.y (mgr.14150) 268 : cluster [DBG] pgmap v241: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:32.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:32 smithi181 bash[24776]: cluster 2024-09-12T23:31:31.456349+0000 mgr.y (mgr.14150) 269 : cluster [DBG] pgmap v242: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:33.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:32 smithi136 bash[22601]: cluster 2024-09-12T23:31:31.456349+0000 mgr.y (mgr.14150) 269 : cluster [DBG] pgmap v242: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:33.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:32 smithi136 bash[32189]: cluster 2024-09-12T23:31:31.456349+0000 mgr.y (mgr.14150) 269 : cluster [DBG] pgmap v242: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:34.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:34 smithi181 bash[24776]: cluster 2024-09-12T23:31:33.456908+0000 mgr.y (mgr.14150) 270 : cluster [DBG] pgmap v243: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:35.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:34 smithi136 bash[22601]: cluster 2024-09-12T23:31:33.456908+0000 mgr.y (mgr.14150) 270 : cluster [DBG] pgmap v243: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:35.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:34 smithi136 bash[32189]: cluster 2024-09-12T23:31:33.456908+0000 mgr.y (mgr.14150) 270 : cluster [DBG] pgmap v243: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:36.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:36 smithi181 bash[24776]: cluster 2024-09-12T23:31:35.457467+0000 mgr.y (mgr.14150) 271 : cluster [DBG] pgmap v244: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:37.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:36 smithi136 bash[22601]: cluster 2024-09-12T23:31:35.457467+0000 mgr.y (mgr.14150) 271 : cluster [DBG] pgmap v244: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:37.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:36 smithi136 bash[32189]: cluster 2024-09-12T23:31:35.457467+0000 mgr.y (mgr.14150) 271 : cluster [DBG] pgmap v244: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:38.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:38 smithi181 bash[24776]: cluster 2024-09-12T23:31:37.457974+0000 mgr.y (mgr.14150) 272 : cluster [DBG] pgmap v245: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:39.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:38 smithi136 bash[22601]: cluster 2024-09-12T23:31:37.457974+0000 mgr.y (mgr.14150) 272 : cluster [DBG] pgmap v245: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:39.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:38 smithi136 bash[32189]: cluster 2024-09-12T23:31:37.457974+0000 mgr.y (mgr.14150) 272 : cluster [DBG] pgmap v245: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:40.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:40 smithi181 bash[24776]: cluster 2024-09-12T23:31:39.458489+0000 mgr.y (mgr.14150) 273 : cluster [DBG] pgmap v246: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:40.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:40 smithi181 bash[24776]: audit 2024-09-12T23:31:40.529579+0000 mon.a (mon.0) 532 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-12T23:31:40.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:40 smithi181 bash[24776]: audit 2024-09-12T23:31:40.531346+0000 mon.a (mon.0) 533 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:31:41.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:40 smithi136 bash[22601]: cluster 2024-09-12T23:31:39.458489+0000 mgr.y (mgr.14150) 273 : cluster [DBG] pgmap v246: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:41.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:40 smithi136 bash[22601]: audit 2024-09-12T23:31:40.529579+0000 mon.a (mon.0) 532 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-12T23:31:41.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:40 smithi136 bash[22601]: audit 2024-09-12T23:31:40.531346+0000 mon.a (mon.0) 533 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:31:41.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:40 smithi136 bash[32189]: cluster 2024-09-12T23:31:39.458489+0000 mgr.y (mgr.14150) 273 : cluster [DBG] pgmap v246: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:41.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:40 smithi136 bash[32189]: audit 2024-09-12T23:31:40.529579+0000 mon.a (mon.0) 532 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-12T23:31:41.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:40 smithi136 bash[32189]: audit 2024-09-12T23:31:40.531346+0000 mon.a (mon.0) 533 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:31:41.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:41 smithi181 bash[24776]: cephadm 2024-09-12T23:31:40.532642+0000 mgr.y (mgr.14150) 274 : cephadm [INF] Deploying daemon osd.6 on smithi181 2024-09-12T23:31:42.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:41 smithi136 bash[22601]: cephadm 2024-09-12T23:31:40.532642+0000 mgr.y (mgr.14150) 274 : cephadm [INF] Deploying daemon osd.6 on smithi181 2024-09-12T23:31:42.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:41 smithi136 bash[32189]: cephadm 2024-09-12T23:31:40.532642+0000 mgr.y (mgr.14150) 274 : cephadm [INF] Deploying daemon osd.6 on smithi181 2024-09-12T23:31:42.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:42 smithi181 bash[24776]: cluster 2024-09-12T23:31:41.459033+0000 mgr.y (mgr.14150) 275 : cluster [DBG] pgmap v247: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:43.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:42 smithi136 bash[22601]: cluster 2024-09-12T23:31:41.459033+0000 mgr.y (mgr.14150) 275 : cluster [DBG] pgmap v247: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:43.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:42 smithi136 bash[32189]: cluster 2024-09-12T23:31:41.459033+0000 mgr.y (mgr.14150) 275 : cluster [DBG] pgmap v247: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:44.554 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:44 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:31:44.554 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:31:44 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:31:44.554 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:31:44 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:31:44.555 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:31:44 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:31:44.847 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:31:44 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:31:44.848 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:31:44 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:31:44.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:44 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:31:44.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:44 smithi181 bash[24776]: cluster 2024-09-12T23:31:43.459538+0000 mgr.y (mgr.14150) 276 : cluster [DBG] pgmap v248: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:44.848 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:31:44 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:31:45.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:44 smithi136 bash[22601]: cluster 2024-09-12T23:31:43.459538+0000 mgr.y (mgr.14150) 276 : cluster [DBG] pgmap v248: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:45.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:44 smithi136 bash[32189]: cluster 2024-09-12T23:31:43.459538+0000 mgr.y (mgr.14150) 276 : cluster [DBG] pgmap v248: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:45.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:45 smithi181 bash[24776]: audit 2024-09-12T23:31:44.693274+0000 mon.a (mon.0) 534 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:31:45.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:45 smithi181 bash[24776]: audit 2024-09-12T23:31:44.700645+0000 mon.a (mon.0) 535 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:31:45.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:45 smithi181 bash[24776]: audit 2024-09-12T23:31:44.709765+0000 mon.a (mon.0) 536 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:31:46.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:45 smithi136 bash[22601]: audit 2024-09-12T23:31:44.693274+0000 mon.a (mon.0) 534 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:31:46.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:45 smithi136 bash[22601]: audit 2024-09-12T23:31:44.700645+0000 mon.a (mon.0) 535 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:31:46.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:45 smithi136 bash[22601]: audit 2024-09-12T23:31:44.709765+0000 mon.a (mon.0) 536 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:31:46.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:45 smithi136 bash[32189]: audit 2024-09-12T23:31:44.693274+0000 mon.a (mon.0) 534 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:31:46.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:45 smithi136 bash[32189]: audit 2024-09-12T23:31:44.700645+0000 mon.a (mon.0) 535 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:31:46.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:45 smithi136 bash[32189]: audit 2024-09-12T23:31:44.709765+0000 mon.a (mon.0) 536 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:31:47.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:46 smithi136 bash[22601]: cluster 2024-09-12T23:31:45.460064+0000 mgr.y (mgr.14150) 277 : cluster [DBG] pgmap v249: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:47.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:46 smithi136 bash[32189]: cluster 2024-09-12T23:31:45.460064+0000 mgr.y (mgr.14150) 277 : cluster [DBG] pgmap v249: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:47.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:46 smithi181 bash[24776]: cluster 2024-09-12T23:31:45.460064+0000 mgr.y (mgr.14150) 277 : cluster [DBG] pgmap v249: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:49.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:48 smithi136 bash[22601]: cluster 2024-09-12T23:31:47.460713+0000 mgr.y (mgr.14150) 278 : cluster [DBG] pgmap v250: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:49.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:48 smithi136 bash[32189]: cluster 2024-09-12T23:31:47.460713+0000 mgr.y (mgr.14150) 278 : cluster [DBG] pgmap v250: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:49.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:48 smithi181 bash[24776]: cluster 2024-09-12T23:31:47.460713+0000 mgr.y (mgr.14150) 278 : cluster [DBG] pgmap v250: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:51.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:50 smithi181 bash[24776]: cluster 2024-09-12T23:31:49.461311+0000 mgr.y (mgr.14150) 279 : cluster [DBG] pgmap v251: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:51.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:50 smithi181 bash[24776]: audit 2024-09-12T23:31:49.800337+0000 mon.a (mon.0) 537 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:31:51.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:50 smithi181 bash[24776]: audit 2024-09-12T23:31:49.809966+0000 mon.a (mon.0) 538 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:31:51.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:50 smithi181 bash[24776]: audit 2024-09-12T23:31:49.811768+0000 mon.a (mon.0) 539 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:31:51.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:50 smithi181 bash[24776]: audit 2024-09-12T23:31:49.813437+0000 mon.a (mon.0) 540 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:31:51.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:50 smithi181 bash[24776]: audit 2024-09-12T23:31:49.822994+0000 mon.a (mon.0) 541 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:31:51.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:50 smithi136 bash[22601]: cluster 2024-09-12T23:31:49.461311+0000 mgr.y (mgr.14150) 279 : cluster [DBG] pgmap v251: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:51.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:50 smithi136 bash[22601]: audit 2024-09-12T23:31:49.800337+0000 mon.a (mon.0) 537 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:31:51.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:50 smithi136 bash[22601]: audit 2024-09-12T23:31:49.809966+0000 mon.a (mon.0) 538 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:31:51.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:50 smithi136 bash[22601]: audit 2024-09-12T23:31:49.811768+0000 mon.a (mon.0) 539 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:31:51.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:50 smithi136 bash[22601]: audit 2024-09-12T23:31:49.813437+0000 mon.a (mon.0) 540 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:31:51.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:50 smithi136 bash[22601]: audit 2024-09-12T23:31:49.822994+0000 mon.a (mon.0) 541 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:31:51.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:50 smithi136 bash[32189]: cluster 2024-09-12T23:31:49.461311+0000 mgr.y (mgr.14150) 279 : cluster [DBG] pgmap v251: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:51.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:50 smithi136 bash[32189]: audit 2024-09-12T23:31:49.800337+0000 mon.a (mon.0) 537 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:31:51.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:50 smithi136 bash[32189]: audit 2024-09-12T23:31:49.809966+0000 mon.a (mon.0) 538 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:31:51.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:50 smithi136 bash[32189]: audit 2024-09-12T23:31:49.811768+0000 mon.a (mon.0) 539 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:31:51.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:50 smithi136 bash[32189]: audit 2024-09-12T23:31:49.813437+0000 mon.a (mon.0) 540 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:31:51.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:50 smithi136 bash[32189]: audit 2024-09-12T23:31:49.822994+0000 mon.a (mon.0) 541 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:31:52.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:51 smithi181 bash[24776]: audit 2024-09-12T23:31:51.165473+0000 mon.b (mon.1) 19 : audit [INF] from='osd.6 [v2:172.21.15.181:6816/4099148413,v1:172.21.15.181:6817/4099148413]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-12T23:31:52.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:51 smithi181 bash[24776]: audit 2024-09-12T23:31:51.166704+0000 mon.a (mon.0) 542 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-12T23:31:52.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:51 smithi136 bash[22601]: audit 2024-09-12T23:31:51.165473+0000 mon.b (mon.1) 19 : audit [INF] from='osd.6 [v2:172.21.15.181:6816/4099148413,v1:172.21.15.181:6817/4099148413]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-12T23:31:52.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:51 smithi136 bash[22601]: audit 2024-09-12T23:31:51.166704+0000 mon.a (mon.0) 542 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-12T23:31:52.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:51 smithi136 bash[32189]: audit 2024-09-12T23:31:51.165473+0000 mon.b (mon.1) 19 : audit [INF] from='osd.6 [v2:172.21.15.181:6816/4099148413,v1:172.21.15.181:6817/4099148413]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-12T23:31:52.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:51 smithi136 bash[32189]: audit 2024-09-12T23:31:51.166704+0000 mon.a (mon.0) 542 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-12T23:31:53.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:52 smithi181 bash[24776]: cluster 2024-09-12T23:31:51.461880+0000 mgr.y (mgr.14150) 280 : cluster [DBG] pgmap v252: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:53.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:52 smithi181 bash[24776]: audit 2024-09-12T23:31:51.814700+0000 mon.a (mon.0) 543 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-12T23:31:53.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:52 smithi181 bash[24776]: cluster 2024-09-12T23:31:51.814823+0000 mon.a (mon.0) 544 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2024-09-12T23:31:53.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:52 smithi181 bash[24776]: audit 2024-09-12T23:31:51.815062+0000 mon.a (mon.0) 545 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-12T23:31:53.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:52 smithi181 bash[24776]: audit 2024-09-12T23:31:51.815475+0000 mon.b (mon.1) 20 : audit [INF] from='osd.6 [v2:172.21.15.181:6816/4099148413,v1:172.21.15.181:6817/4099148413]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-12T23:31:53.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:52 smithi181 bash[24776]: audit 2024-09-12T23:31:51.816694+0000 mon.a (mon.0) 546 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-12T23:31:53.161 INFO:teuthology.orchestra.run.smithi181.stdout:Created osd(s) 6 on host 'smithi181' 2024-09-12T23:31:53.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:52 smithi136 bash[22601]: cluster 2024-09-12T23:31:51.461880+0000 mgr.y (mgr.14150) 280 : cluster [DBG] pgmap v252: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:53.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:52 smithi136 bash[22601]: audit 2024-09-12T23:31:51.814700+0000 mon.a (mon.0) 543 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-12T23:31:53.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:52 smithi136 bash[22601]: cluster 2024-09-12T23:31:51.814823+0000 mon.a (mon.0) 544 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2024-09-12T23:31:53.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:52 smithi136 bash[22601]: audit 2024-09-12T23:31:51.815062+0000 mon.a (mon.0) 545 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-12T23:31:53.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:52 smithi136 bash[22601]: audit 2024-09-12T23:31:51.815475+0000 mon.b (mon.1) 20 : audit [INF] from='osd.6 [v2:172.21.15.181:6816/4099148413,v1:172.21.15.181:6817/4099148413]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-12T23:31:53.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:52 smithi136 bash[22601]: audit 2024-09-12T23:31:51.816694+0000 mon.a (mon.0) 546 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-12T23:31:53.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:52 smithi136 bash[32189]: cluster 2024-09-12T23:31:51.461880+0000 mgr.y (mgr.14150) 280 : cluster [DBG] pgmap v252: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:53.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:52 smithi136 bash[32189]: audit 2024-09-12T23:31:51.814700+0000 mon.a (mon.0) 543 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-12T23:31:53.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:52 smithi136 bash[32189]: cluster 2024-09-12T23:31:51.814823+0000 mon.a (mon.0) 544 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2024-09-12T23:31:53.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:52 smithi136 bash[32189]: audit 2024-09-12T23:31:51.815062+0000 mon.a (mon.0) 545 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-12T23:31:53.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:52 smithi136 bash[32189]: audit 2024-09-12T23:31:51.815475+0000 mon.b (mon.1) 20 : audit [INF] from='osd.6 [v2:172.21.15.181:6816/4099148413,v1:172.21.15.181:6817/4099148413]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-12T23:31:53.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:52 smithi136 bash[32189]: audit 2024-09-12T23:31:51.816694+0000 mon.a (mon.0) 546 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-12T23:31:53.710 DEBUG:teuthology.orchestra.run.smithi181:osd.6> sudo journalctl -f -n 0 -u ceph-da58a47c-715d-11ef-bceb-c7b262605968@osd.6.service 2024-09-12T23:31:53.713 INFO:tasks.cephadm:Deploying osd.7 on smithi181 with /dev/vg_nvme/lv_1... 2024-09-12T23:31:53.713 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-09-12T23:31:54.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:53 smithi181 bash[24776]: audit 2024-09-12T23:31:52.816514+0000 mon.a (mon.0) 547 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2024-09-12T23:31:54.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:53 smithi181 bash[24776]: cluster 2024-09-12T23:31:52.816855+0000 mon.a (mon.0) 548 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2024-09-12T23:31:54.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:53 smithi181 bash[24776]: audit 2024-09-12T23:31:52.817560+0000 mon.a (mon.0) 549 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-12T23:31:54.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:53 smithi181 bash[24776]: audit 2024-09-12T23:31:52.827059+0000 mon.a (mon.0) 550 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-12T23:31:54.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:53 smithi181 bash[24776]: audit 2024-09-12T23:31:53.138529+0000 mon.a (mon.0) 551 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:31:54.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:53 smithi181 bash[24776]: audit 2024-09-12T23:31:53.148216+0000 mon.a (mon.0) 552 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:31:54.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:53 smithi181 bash[24776]: audit 2024-09-12T23:31:53.158905+0000 mon.a (mon.0) 553 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:31:54.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:53 smithi136 bash[22601]: audit 2024-09-12T23:31:52.816514+0000 mon.a (mon.0) 547 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2024-09-12T23:31:54.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:53 smithi136 bash[22601]: cluster 2024-09-12T23:31:52.816855+0000 mon.a (mon.0) 548 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2024-09-12T23:31:54.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:53 smithi136 bash[22601]: audit 2024-09-12T23:31:52.817560+0000 mon.a (mon.0) 549 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-12T23:31:54.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:53 smithi136 bash[22601]: audit 2024-09-12T23:31:52.827059+0000 mon.a (mon.0) 550 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-12T23:31:54.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:53 smithi136 bash[22601]: audit 2024-09-12T23:31:53.138529+0000 mon.a (mon.0) 551 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:31:54.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:53 smithi136 bash[22601]: audit 2024-09-12T23:31:53.148216+0000 mon.a (mon.0) 552 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:31:54.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:53 smithi136 bash[22601]: audit 2024-09-12T23:31:53.158905+0000 mon.a (mon.0) 553 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:31:54.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:53 smithi136 bash[32189]: audit 2024-09-12T23:31:52.816514+0000 mon.a (mon.0) 547 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2024-09-12T23:31:54.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:53 smithi136 bash[32189]: cluster 2024-09-12T23:31:52.816855+0000 mon.a (mon.0) 548 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2024-09-12T23:31:54.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:53 smithi136 bash[32189]: audit 2024-09-12T23:31:52.817560+0000 mon.a (mon.0) 549 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-12T23:31:54.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:53 smithi136 bash[32189]: audit 2024-09-12T23:31:52.827059+0000 mon.a (mon.0) 550 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-12T23:31:54.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:53 smithi136 bash[32189]: audit 2024-09-12T23:31:53.138529+0000 mon.a (mon.0) 551 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:31:54.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:53 smithi136 bash[32189]: audit 2024-09-12T23:31:53.148216+0000 mon.a (mon.0) 552 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:31:54.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:53 smithi136 bash[32189]: audit 2024-09-12T23:31:53.158905+0000 mon.a (mon.0) 553 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:31:55.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:54 smithi181 bash[24776]: cluster 2024-09-12T23:31:52.166590+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-12T23:31:55.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:54 smithi181 bash[24776]: cluster 2024-09-12T23:31:52.166691+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-12T23:31:55.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:54 smithi181 bash[24776]: cluster 2024-09-12T23:31:53.462369+0000 mgr.y (mgr.14150) 281 : cluster [DBG] pgmap v255: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:55.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:54 smithi181 bash[24776]: cluster 2024-09-12T23:31:53.827089+0000 mon.a (mon.0) 554 : cluster [INF] osd.6 [v2:172.21.15.181:6816/4099148413,v1:172.21.15.181:6817/4099148413] boot 2024-09-12T23:31:55.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:54 smithi181 bash[24776]: cluster 2024-09-12T23:31:53.827172+0000 mon.a (mon.0) 555 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2024-09-12T23:31:55.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:54 smithi181 bash[24776]: audit 2024-09-12T23:31:53.827898+0000 mon.a (mon.0) 556 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-12T23:31:55.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:54 smithi136 bash[22601]: cluster 2024-09-12T23:31:52.166590+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-12T23:31:55.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:54 smithi136 bash[22601]: cluster 2024-09-12T23:31:52.166691+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-12T23:31:55.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:54 smithi136 bash[22601]: cluster 2024-09-12T23:31:53.462369+0000 mgr.y (mgr.14150) 281 : cluster [DBG] pgmap v255: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:55.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:54 smithi136 bash[22601]: cluster 2024-09-12T23:31:53.827089+0000 mon.a (mon.0) 554 : cluster [INF] osd.6 [v2:172.21.15.181:6816/4099148413,v1:172.21.15.181:6817/4099148413] boot 2024-09-12T23:31:55.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:54 smithi136 bash[22601]: cluster 2024-09-12T23:31:53.827172+0000 mon.a (mon.0) 555 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2024-09-12T23:31:55.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:54 smithi136 bash[22601]: audit 2024-09-12T23:31:53.827898+0000 mon.a (mon.0) 556 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-12T23:31:55.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:54 smithi136 bash[32189]: cluster 2024-09-12T23:31:52.166590+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-12T23:31:55.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:54 smithi136 bash[32189]: cluster 2024-09-12T23:31:52.166691+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-12T23:31:55.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:54 smithi136 bash[32189]: cluster 2024-09-12T23:31:53.462369+0000 mgr.y (mgr.14150) 281 : cluster [DBG] pgmap v255: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-12T23:31:55.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:54 smithi136 bash[32189]: cluster 2024-09-12T23:31:53.827089+0000 mon.a (mon.0) 554 : cluster [INF] osd.6 [v2:172.21.15.181:6816/4099148413,v1:172.21.15.181:6817/4099148413] boot 2024-09-12T23:31:55.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:54 smithi136 bash[32189]: cluster 2024-09-12T23:31:53.827172+0000 mon.a (mon.0) 555 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2024-09-12T23:31:55.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:54 smithi136 bash[32189]: audit 2024-09-12T23:31:53.827898+0000 mon.a (mon.0) 556 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-12T23:31:56.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:55 smithi181 bash[24776]: cluster 2024-09-12T23:31:54.836196+0000 mon.a (mon.0) 557 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2024-09-12T23:31:56.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:55 smithi136 bash[22601]: cluster 2024-09-12T23:31:54.836196+0000 mon.a (mon.0) 557 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2024-09-12T23:31:56.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:55 smithi136 bash[32189]: cluster 2024-09-12T23:31:54.836196+0000 mon.a (mon.0) 557 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2024-09-12T23:31:57.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:56 smithi136 bash[22601]: cluster 2024-09-12T23:31:55.462957+0000 mgr.y (mgr.14150) 282 : cluster [DBG] pgmap v258: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:31:57.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:56 smithi136 bash[22601]: cluster 2024-09-12T23:31:55.838149+0000 mon.a (mon.0) 558 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-12T23:31:57.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:56 smithi136 bash[22601]: cluster 2024-09-12T23:31:55.853478+0000 mon.a (mon.0) 559 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2024-09-12T23:31:57.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:56 smithi136 bash[32189]: cluster 2024-09-12T23:31:55.462957+0000 mgr.y (mgr.14150) 282 : cluster [DBG] pgmap v258: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:31:57.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:56 smithi136 bash[32189]: cluster 2024-09-12T23:31:55.838149+0000 mon.a (mon.0) 558 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-12T23:31:57.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:56 smithi136 bash[32189]: cluster 2024-09-12T23:31:55.853478+0000 mon.a (mon.0) 559 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2024-09-12T23:31:57.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:56 smithi181 bash[24776]: cluster 2024-09-12T23:31:55.462957+0000 mgr.y (mgr.14150) 282 : cluster [DBG] pgmap v258: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:31:57.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:56 smithi181 bash[24776]: cluster 2024-09-12T23:31:55.838149+0000 mon.a (mon.0) 558 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-12T23:31:57.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:56 smithi181 bash[24776]: cluster 2024-09-12T23:31:55.853478+0000 mon.a (mon.0) 559 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2024-09-12T23:31:57.915 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.b/config 2024-09-12T23:31:59.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:58 smithi136 bash[32189]: cluster 2024-09-12T23:31:57.463514+0000 mgr.y (mgr.14150) 283 : cluster [DBG] pgmap v260: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:31:59.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:58 smithi136 bash[22601]: cluster 2024-09-12T23:31:57.463514+0000 mgr.y (mgr.14150) 283 : cluster [DBG] pgmap v260: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:31:59.344 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:58 smithi181 bash[24776]: cluster 2024-09-12T23:31:57.463514+0000 mgr.y (mgr.14150) 283 : cluster [DBG] pgmap v260: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:00.048 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:59 smithi181 bash[24776]: cluster 2024-09-12T23:31:59.852820+0000 mon.a (mon.0) 560 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-12T23:32:00.049 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:31:59 smithi181 bash[24776]: cluster 2024-09-12T23:31:59.852891+0000 mon.a (mon.0) 561 : cluster [INF] Cluster is now healthy 2024-09-12T23:32:00.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:59 smithi136 bash[22601]: cluster 2024-09-12T23:31:59.852820+0000 mon.a (mon.0) 560 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-12T23:32:00.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:31:59 smithi136 bash[22601]: cluster 2024-09-12T23:31:59.852891+0000 mon.a (mon.0) 561 : cluster [INF] Cluster is now healthy 2024-09-12T23:32:00.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:59 smithi136 bash[32189]: cluster 2024-09-12T23:31:59.852820+0000 mon.a (mon.0) 560 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-12T23:32:00.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:31:59 smithi136 bash[32189]: cluster 2024-09-12T23:31:59.852891+0000 mon.a (mon.0) 561 : cluster [INF] Cluster is now healthy 2024-09-12T23:32:01.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:00 smithi181 bash[24776]: cluster 2024-09-12T23:31:59.464018+0000 mgr.y (mgr.14150) 284 : cluster [DBG] pgmap v261: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:01.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:00 smithi136 bash[22601]: cluster 2024-09-12T23:31:59.464018+0000 mgr.y (mgr.14150) 284 : cluster [DBG] pgmap v261: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:01.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:00 smithi136 bash[32189]: cluster 2024-09-12T23:31:59.464018+0000 mgr.y (mgr.14150) 284 : cluster [DBG] pgmap v261: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:02.461 INFO:teuthology.orchestra.run.smithi181.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-09-12T23:32:02.658 INFO:teuthology.orchestra.run.smithi181.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-09-12T23:32:02.674 INFO:teuthology.orchestra.run.smithi181.stderr: stderr: 10+0 records in 2024-09-12T23:32:02.674 INFO:teuthology.orchestra.run.smithi181.stderr:10+0 records out 2024-09-12T23:32:02.674 INFO:teuthology.orchestra.run.smithi181.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0137259 s, 764 MB/s 2024-09-12T23:32:02.675 INFO:teuthology.orchestra.run.smithi181.stderr:--> Zapping successful for: 2024-09-12T23:32:03.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:02 smithi181 bash[24776]: cluster 2024-09-12T23:32:01.464589+0000 mgr.y (mgr.14150) 285 : cluster [DBG] pgmap v262: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-09-12T23:32:03.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:02 smithi136 bash[32189]: cluster 2024-09-12T23:32:01.464589+0000 mgr.y (mgr.14150) 285 : cluster [DBG] pgmap v262: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-09-12T23:32:03.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:02 smithi136 bash[22601]: cluster 2024-09-12T23:32:01.464589+0000 mgr.y (mgr.14150) 285 : cluster [DBG] pgmap v262: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-09-12T23:32:03.382 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi181:vg_nvme/lv_1 2024-09-12T23:32:05.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:04 smithi136 bash[32189]: cluster 2024-09-12T23:32:03.465128+0000 mgr.y (mgr.14150) 286 : cluster [DBG] pgmap v263: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 67 KiB/s, 0 objects/s recovering 2024-09-12T23:32:05.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:04 smithi136 bash[32189]: cephadm 2024-09-12T23:32:03.738588+0000 mgr.y (mgr.14150) 287 : cephadm [INF] Detected new or changed devices on smithi181 2024-09-12T23:32:05.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:04 smithi136 bash[32189]: audit 2024-09-12T23:32:03.747430+0000 mon.a (mon.0) 562 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:32:05.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:04 smithi136 bash[32189]: audit 2024-09-12T23:32:03.756723+0000 mon.a (mon.0) 563 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:32:05.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:04 smithi136 bash[32189]: audit 2024-09-12T23:32:03.759021+0000 mon.a (mon.0) 564 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:32:05.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:04 smithi136 bash[32189]: audit 2024-09-12T23:32:03.760815+0000 mon.a (mon.0) 565 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:32:05.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:04 smithi136 bash[32189]: audit 2024-09-12T23:32:03.762573+0000 mon.a (mon.0) 566 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:32:05.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:04 smithi136 bash[32189]: cephadm 2024-09-12T23:32:03.763910+0000 mgr.y (mgr.14150) 288 : cephadm [INF] Adjusting osd_memory_target on smithi181 to 5755M 2024-09-12T23:32:05.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:04 smithi136 bash[32189]: audit 2024-09-12T23:32:03.772513+0000 mon.a (mon.0) 567 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:32:05.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:04 smithi136 bash[32189]: audit 2024-09-12T23:32:03.776068+0000 mon.a (mon.0) 568 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:32:05.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:04 smithi136 bash[32189]: audit 2024-09-12T23:32:03.781678+0000 mon.a (mon.0) 569 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:32:05.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:04 smithi136 bash[32189]: audit 2024-09-12T23:32:03.790198+0000 mon.a (mon.0) 570 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:32:05.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:04 smithi136 bash[22601]: cluster 2024-09-12T23:32:03.465128+0000 mgr.y (mgr.14150) 286 : cluster [DBG] pgmap v263: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 67 KiB/s, 0 objects/s recovering 2024-09-12T23:32:05.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:04 smithi136 bash[22601]: cephadm 2024-09-12T23:32:03.738588+0000 mgr.y (mgr.14150) 287 : cephadm [INF] Detected new or changed devices on smithi181 2024-09-12T23:32:05.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:04 smithi136 bash[22601]: audit 2024-09-12T23:32:03.747430+0000 mon.a (mon.0) 562 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:32:05.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:04 smithi136 bash[22601]: audit 2024-09-12T23:32:03.756723+0000 mon.a (mon.0) 563 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:32:05.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:04 smithi136 bash[22601]: audit 2024-09-12T23:32:03.759021+0000 mon.a (mon.0) 564 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:32:05.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:04 smithi136 bash[22601]: audit 2024-09-12T23:32:03.760815+0000 mon.a (mon.0) 565 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:32:05.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:04 smithi136 bash[22601]: audit 2024-09-12T23:32:03.762573+0000 mon.a (mon.0) 566 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:32:05.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:04 smithi136 bash[22601]: cephadm 2024-09-12T23:32:03.763910+0000 mgr.y (mgr.14150) 288 : cephadm [INF] Adjusting osd_memory_target on smithi181 to 5755M 2024-09-12T23:32:05.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:04 smithi136 bash[22601]: audit 2024-09-12T23:32:03.772513+0000 mon.a (mon.0) 567 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:32:05.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:04 smithi136 bash[22601]: audit 2024-09-12T23:32:03.776068+0000 mon.a (mon.0) 568 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:32:05.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:04 smithi136 bash[22601]: audit 2024-09-12T23:32:03.781678+0000 mon.a (mon.0) 569 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:32:05.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:04 smithi136 bash[22601]: audit 2024-09-12T23:32:03.790198+0000 mon.a (mon.0) 570 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:32:05.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:04 smithi181 bash[24776]: cluster 2024-09-12T23:32:03.465128+0000 mgr.y (mgr.14150) 286 : cluster [DBG] pgmap v263: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 67 KiB/s, 0 objects/s recovering 2024-09-12T23:32:05.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:04 smithi181 bash[24776]: cephadm 2024-09-12T23:32:03.738588+0000 mgr.y (mgr.14150) 287 : cephadm [INF] Detected new or changed devices on smithi181 2024-09-12T23:32:05.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:04 smithi181 bash[24776]: audit 2024-09-12T23:32:03.747430+0000 mon.a (mon.0) 562 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:32:05.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:04 smithi181 bash[24776]: audit 2024-09-12T23:32:03.756723+0000 mon.a (mon.0) 563 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:32:05.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:04 smithi181 bash[24776]: audit 2024-09-12T23:32:03.759021+0000 mon.a (mon.0) 564 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:32:05.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:04 smithi181 bash[24776]: audit 2024-09-12T23:32:03.760815+0000 mon.a (mon.0) 565 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:32:05.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:04 smithi181 bash[24776]: audit 2024-09-12T23:32:03.762573+0000 mon.a (mon.0) 566 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:32:05.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:04 smithi181 bash[24776]: cephadm 2024-09-12T23:32:03.763910+0000 mgr.y (mgr.14150) 288 : cephadm [INF] Adjusting osd_memory_target on smithi181 to 5755M 2024-09-12T23:32:05.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:04 smithi181 bash[24776]: audit 2024-09-12T23:32:03.772513+0000 mon.a (mon.0) 567 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:32:05.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:04 smithi181 bash[24776]: audit 2024-09-12T23:32:03.776068+0000 mon.a (mon.0) 568 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:32:05.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:04 smithi181 bash[24776]: audit 2024-09-12T23:32:03.781678+0000 mon.a (mon.0) 569 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:32:05.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:04 smithi181 bash[24776]: audit 2024-09-12T23:32:03.790198+0000 mon.a (mon.0) 570 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:32:07.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:06 smithi136 bash[22601]: cluster 2024-09-12T23:32:05.465677+0000 mgr.y (mgr.14150) 289 : cluster [DBG] pgmap v264: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-12T23:32:07.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:06 smithi136 bash[32189]: cluster 2024-09-12T23:32:05.465677+0000 mgr.y (mgr.14150) 289 : cluster [DBG] pgmap v264: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-12T23:32:07.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:06 smithi181 bash[24776]: cluster 2024-09-12T23:32:05.465677+0000 mgr.y (mgr.14150) 289 : cluster [DBG] pgmap v264: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-12T23:32:08.118 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.b/config 2024-09-12T23:32:09.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:08 smithi136 bash[22601]: cluster 2024-09-12T23:32:07.466181+0000 mgr.y (mgr.14150) 290 : cluster [DBG] pgmap v265: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 50 KiB/s, 0 objects/s recovering 2024-09-12T23:32:09.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:08 smithi136 bash[32189]: cluster 2024-09-12T23:32:07.466181+0000 mgr.y (mgr.14150) 290 : cluster [DBG] pgmap v265: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 50 KiB/s, 0 objects/s recovering 2024-09-12T23:32:09.026 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:08 smithi181 bash[24776]: cluster 2024-09-12T23:32:07.466181+0000 mgr.y (mgr.14150) 290 : cluster [DBG] pgmap v265: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 50 KiB/s, 0 objects/s recovering 2024-09-12T23:32:10.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:10 smithi181 bash[24776]: cluster 2024-09-12T23:32:09.466740+0000 mgr.y (mgr.14150) 291 : cluster [DBG] pgmap v266: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-12T23:32:11.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:10 smithi136 bash[22601]: cluster 2024-09-12T23:32:09.466740+0000 mgr.y (mgr.14150) 291 : cluster [DBG] pgmap v266: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-12T23:32:11.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:10 smithi136 bash[32189]: cluster 2024-09-12T23:32:09.466740+0000 mgr.y (mgr.14150) 291 : cluster [DBG] pgmap v266: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-12T23:32:12.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:11 smithi181 bash[24776]: audit 2024-09-12T23:32:10.890341+0000 mgr.y (mgr.14150) 292 : audit [DBG] from='client.24250 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:32:12.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:11 smithi181 bash[24776]: audit 2024-09-12T23:32:10.896721+0000 mon.a (mon.0) 571 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-12T23:32:12.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:11 smithi181 bash[24776]: audit 2024-09-12T23:32:10.903492+0000 mon.a (mon.0) 572 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-12T23:32:12.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:11 smithi181 bash[24776]: audit 2024-09-12T23:32:10.905185+0000 mon.a (mon.0) 573 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:32:12.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:11 smithi136 bash[22601]: audit 2024-09-12T23:32:10.890341+0000 mgr.y (mgr.14150) 292 : audit [DBG] from='client.24250 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:32:12.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:11 smithi136 bash[22601]: audit 2024-09-12T23:32:10.896721+0000 mon.a (mon.0) 571 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-12T23:32:12.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:11 smithi136 bash[22601]: audit 2024-09-12T23:32:10.903492+0000 mon.a (mon.0) 572 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-12T23:32:12.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:11 smithi136 bash[22601]: audit 2024-09-12T23:32:10.905185+0000 mon.a (mon.0) 573 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:32:12.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:11 smithi136 bash[32189]: audit 2024-09-12T23:32:10.890341+0000 mgr.y (mgr.14150) 292 : audit [DBG] from='client.24250 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:32:12.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:11 smithi136 bash[32189]: audit 2024-09-12T23:32:10.896721+0000 mon.a (mon.0) 571 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-12T23:32:12.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:11 smithi136 bash[32189]: audit 2024-09-12T23:32:10.903492+0000 mon.a (mon.0) 572 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-12T23:32:12.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:11 smithi136 bash[32189]: audit 2024-09-12T23:32:10.905185+0000 mon.a (mon.0) 573 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:32:13.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:12 smithi181 bash[24776]: cluster 2024-09-12T23:32:11.467254+0000 mgr.y (mgr.14150) 293 : cluster [DBG] pgmap v267: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:13.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:12 smithi136 bash[22601]: cluster 2024-09-12T23:32:11.467254+0000 mgr.y (mgr.14150) 293 : cluster [DBG] pgmap v267: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:13.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:12 smithi136 bash[32189]: cluster 2024-09-12T23:32:11.467254+0000 mgr.y (mgr.14150) 293 : cluster [DBG] pgmap v267: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:15.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:14 smithi181 bash[24776]: cluster 2024-09-12T23:32:13.467796+0000 mgr.y (mgr.14150) 294 : cluster [DBG] pgmap v268: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:15.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:14 smithi136 bash[22601]: cluster 2024-09-12T23:32:13.467796+0000 mgr.y (mgr.14150) 294 : cluster [DBG] pgmap v268: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:15.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:14 smithi136 bash[32189]: cluster 2024-09-12T23:32:13.467796+0000 mgr.y (mgr.14150) 294 : cluster [DBG] pgmap v268: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:16.925 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:16 smithi181 bash[24776]: cluster 2024-09-12T23:32:15.468441+0000 mgr.y (mgr.14150) 295 : cluster [DBG] pgmap v269: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:17.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:16 smithi136 bash[22601]: cluster 2024-09-12T23:32:15.468441+0000 mgr.y (mgr.14150) 295 : cluster [DBG] pgmap v269: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:17.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:16 smithi136 bash[32189]: cluster 2024-09-12T23:32:15.468441+0000 mgr.y (mgr.14150) 295 : cluster [DBG] pgmap v269: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:19.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:18 smithi181 bash[24776]: cluster 2024-09-12T23:32:17.469013+0000 mgr.y (mgr.14150) 296 : cluster [DBG] pgmap v270: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:19.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:18 smithi136 bash[22601]: cluster 2024-09-12T23:32:17.469013+0000 mgr.y (mgr.14150) 296 : cluster [DBG] pgmap v270: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:19.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:18 smithi136 bash[32189]: cluster 2024-09-12T23:32:17.469013+0000 mgr.y (mgr.14150) 296 : cluster [DBG] pgmap v270: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:20.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:19 smithi181 bash[24776]: audit 2024-09-12T23:32:18.992874+0000 mon.b (mon.1) 21 : audit [INF] from='client.? 172.21.15.181:0/2122929497' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d2ff80f0-6ef0-4098-877c-a891d48c687d"}]: dispatch 2024-09-12T23:32:20.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:19 smithi181 bash[24776]: audit 2024-09-12T23:32:18.994355+0000 mon.a (mon.0) 574 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d2ff80f0-6ef0-4098-877c-a891d48c687d"}]: dispatch 2024-09-12T23:32:20.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:19 smithi181 bash[24776]: audit 2024-09-12T23:32:19.002850+0000 mon.a (mon.0) 575 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d2ff80f0-6ef0-4098-877c-a891d48c687d"}]': finished 2024-09-12T23:32:20.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:19 smithi181 bash[24776]: cluster 2024-09-12T23:32:19.003022+0000 mon.a (mon.0) 576 : cluster [DBG] osdmap e45: 8 total, 7 up, 8 in 2024-09-12T23:32:20.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:19 smithi181 bash[24776]: audit 2024-09-12T23:32:19.003250+0000 mon.a (mon.0) 577 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-12T23:32:20.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:19 smithi181 bash[24776]: audit 2024-09-12T23:32:19.601170+0000 mon.b (mon.1) 22 : audit [DBG] from='client.? 172.21.15.181:0/2225672065' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-12T23:32:20.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:19 smithi136 bash[22601]: audit 2024-09-12T23:32:18.992874+0000 mon.b (mon.1) 21 : audit [INF] from='client.? 172.21.15.181:0/2122929497' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d2ff80f0-6ef0-4098-877c-a891d48c687d"}]: dispatch 2024-09-12T23:32:20.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:19 smithi136 bash[22601]: audit 2024-09-12T23:32:18.994355+0000 mon.a (mon.0) 574 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d2ff80f0-6ef0-4098-877c-a891d48c687d"}]: dispatch 2024-09-12T23:32:20.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:19 smithi136 bash[22601]: audit 2024-09-12T23:32:19.002850+0000 mon.a (mon.0) 575 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d2ff80f0-6ef0-4098-877c-a891d48c687d"}]': finished 2024-09-12T23:32:20.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:19 smithi136 bash[22601]: cluster 2024-09-12T23:32:19.003022+0000 mon.a (mon.0) 576 : cluster [DBG] osdmap e45: 8 total, 7 up, 8 in 2024-09-12T23:32:20.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:19 smithi136 bash[22601]: audit 2024-09-12T23:32:19.003250+0000 mon.a (mon.0) 577 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-12T23:32:20.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:19 smithi136 bash[22601]: audit 2024-09-12T23:32:19.601170+0000 mon.b (mon.1) 22 : audit [DBG] from='client.? 172.21.15.181:0/2225672065' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-12T23:32:20.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:19 smithi136 bash[32189]: audit 2024-09-12T23:32:18.992874+0000 mon.b (mon.1) 21 : audit [INF] from='client.? 172.21.15.181:0/2122929497' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d2ff80f0-6ef0-4098-877c-a891d48c687d"}]: dispatch 2024-09-12T23:32:20.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:19 smithi136 bash[32189]: audit 2024-09-12T23:32:18.994355+0000 mon.a (mon.0) 574 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d2ff80f0-6ef0-4098-877c-a891d48c687d"}]: dispatch 2024-09-12T23:32:20.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:19 smithi136 bash[32189]: audit 2024-09-12T23:32:19.002850+0000 mon.a (mon.0) 575 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d2ff80f0-6ef0-4098-877c-a891d48c687d"}]': finished 2024-09-12T23:32:20.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:19 smithi136 bash[32189]: cluster 2024-09-12T23:32:19.003022+0000 mon.a (mon.0) 576 : cluster [DBG] osdmap e45: 8 total, 7 up, 8 in 2024-09-12T23:32:20.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:19 smithi136 bash[32189]: audit 2024-09-12T23:32:19.003250+0000 mon.a (mon.0) 577 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-12T23:32:20.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:19 smithi136 bash[32189]: audit 2024-09-12T23:32:19.601170+0000 mon.b (mon.1) 22 : audit [DBG] from='client.? 172.21.15.181:0/2225672065' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-12T23:32:21.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:20 smithi181 bash[24776]: cluster 2024-09-12T23:32:19.469495+0000 mgr.y (mgr.14150) 297 : cluster [DBG] pgmap v272: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:21.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:20 smithi136 bash[22601]: cluster 2024-09-12T23:32:19.469495+0000 mgr.y (mgr.14150) 297 : cluster [DBG] pgmap v272: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:21.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:20 smithi136 bash[32189]: cluster 2024-09-12T23:32:19.469495+0000 mgr.y (mgr.14150) 297 : cluster [DBG] pgmap v272: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:23.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:22 smithi181 bash[24776]: cluster 2024-09-12T23:32:21.470014+0000 mgr.y (mgr.14150) 298 : cluster [DBG] pgmap v273: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:23.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:22 smithi136 bash[22601]: cluster 2024-09-12T23:32:21.470014+0000 mgr.y (mgr.14150) 298 : cluster [DBG] pgmap v273: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:23.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:22 smithi136 bash[32189]: cluster 2024-09-12T23:32:21.470014+0000 mgr.y (mgr.14150) 298 : cluster [DBG] pgmap v273: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:25.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:24 smithi181 bash[24776]: cluster 2024-09-12T23:32:23.470581+0000 mgr.y (mgr.14150) 299 : cluster [DBG] pgmap v274: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:25.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:24 smithi136 bash[22601]: cluster 2024-09-12T23:32:23.470581+0000 mgr.y (mgr.14150) 299 : cluster [DBG] pgmap v274: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:25.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:24 smithi136 bash[32189]: cluster 2024-09-12T23:32:23.470581+0000 mgr.y (mgr.14150) 299 : cluster [DBG] pgmap v274: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:27.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:26 smithi181 bash[24776]: cluster 2024-09-12T23:32:25.471188+0000 mgr.y (mgr.14150) 300 : cluster [DBG] pgmap v275: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:27.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:26 smithi136 bash[22601]: cluster 2024-09-12T23:32:25.471188+0000 mgr.y (mgr.14150) 300 : cluster [DBG] pgmap v275: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:27.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:26 smithi136 bash[32189]: cluster 2024-09-12T23:32:25.471188+0000 mgr.y (mgr.14150) 300 : cluster [DBG] pgmap v275: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:29.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:28 smithi181 bash[24776]: cluster 2024-09-12T23:32:27.471772+0000 mgr.y (mgr.14150) 301 : cluster [DBG] pgmap v276: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:29.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:28 smithi136 bash[22601]: cluster 2024-09-12T23:32:27.471772+0000 mgr.y (mgr.14150) 301 : cluster [DBG] pgmap v276: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:29.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:28 smithi136 bash[32189]: cluster 2024-09-12T23:32:27.471772+0000 mgr.y (mgr.14150) 301 : cluster [DBG] pgmap v276: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:31.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:30 smithi181 bash[24776]: cluster 2024-09-12T23:32:29.472312+0000 mgr.y (mgr.14150) 302 : cluster [DBG] pgmap v277: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:31.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:30 smithi136 bash[22601]: cluster 2024-09-12T23:32:29.472312+0000 mgr.y (mgr.14150) 302 : cluster [DBG] pgmap v277: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:31.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:30 smithi136 bash[32189]: cluster 2024-09-12T23:32:29.472312+0000 mgr.y (mgr.14150) 302 : cluster [DBG] pgmap v277: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:32.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:31 smithi181 bash[24776]: audit 2024-09-12T23:32:30.961953+0000 mon.a (mon.0) 578 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-12T23:32:32.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:31 smithi181 bash[24776]: audit 2024-09-12T23:32:30.963855+0000 mon.a (mon.0) 579 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:32:32.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:31 smithi136 bash[22601]: audit 2024-09-12T23:32:30.961953+0000 mon.a (mon.0) 578 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-12T23:32:32.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:31 smithi136 bash[22601]: audit 2024-09-12T23:32:30.963855+0000 mon.a (mon.0) 579 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:32:32.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:31 smithi136 bash[32189]: audit 2024-09-12T23:32:30.961953+0000 mon.a (mon.0) 578 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-12T23:32:32.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:31 smithi136 bash[32189]: audit 2024-09-12T23:32:30.963855+0000 mon.a (mon.0) 579 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:32:33.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:32 smithi181 bash[24776]: cephadm 2024-09-12T23:32:30.965503+0000 mgr.y (mgr.14150) 303 : cephadm [INF] Deploying daemon osd.7 on smithi181 2024-09-12T23:32:33.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:32 smithi181 bash[24776]: cluster 2024-09-12T23:32:31.472891+0000 mgr.y (mgr.14150) 304 : cluster [DBG] pgmap v278: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:33.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:32 smithi136 bash[22601]: cephadm 2024-09-12T23:32:30.965503+0000 mgr.y (mgr.14150) 303 : cephadm [INF] Deploying daemon osd.7 on smithi181 2024-09-12T23:32:33.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:32 smithi136 bash[22601]: cluster 2024-09-12T23:32:31.472891+0000 mgr.y (mgr.14150) 304 : cluster [DBG] pgmap v278: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:33.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:32 smithi136 bash[32189]: cephadm 2024-09-12T23:32:30.965503+0000 mgr.y (mgr.14150) 303 : cephadm [INF] Deploying daemon osd.7 on smithi181 2024-09-12T23:32:33.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:32 smithi136 bash[32189]: cluster 2024-09-12T23:32:31.472891+0000 mgr.y (mgr.14150) 304 : cluster [DBG] pgmap v278: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:34.907 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:32:34 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:32:34.907 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:32:34 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:32:34.908 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:32:34 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:32:34.908 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 12 23:32:34 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:32:34.908 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:34 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:32:34.908 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:34 smithi181 bash[24776]: cluster 2024-09-12T23:32:33.473374+0000 mgr.y (mgr.14150) 305 : cluster [DBG] pgmap v279: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:35.245 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:35 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:32:35.246 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:32:35 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:32:35.246 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:32:35 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:32:35.246 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 12 23:32:35 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:32:35.246 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:32:35 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:32:35.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:34 smithi136 bash[22601]: cluster 2024-09-12T23:32:33.473374+0000 mgr.y (mgr.14150) 305 : cluster [DBG] pgmap v279: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:35.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:34 smithi136 bash[32189]: cluster 2024-09-12T23:32:33.473374+0000 mgr.y (mgr.14150) 305 : cluster [DBG] pgmap v279: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:36.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:35 smithi181 bash[24776]: audit 2024-09-12T23:32:35.185324+0000 mon.a (mon.0) 580 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:32:36.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:35 smithi181 bash[24776]: audit 2024-09-12T23:32:35.192555+0000 mon.a (mon.0) 581 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:32:36.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:35 smithi181 bash[24776]: audit 2024-09-12T23:32:35.201610+0000 mon.a (mon.0) 582 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:32:36.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:35 smithi136 bash[22601]: audit 2024-09-12T23:32:35.185324+0000 mon.a (mon.0) 580 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:32:36.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:35 smithi136 bash[22601]: audit 2024-09-12T23:32:35.192555+0000 mon.a (mon.0) 581 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:32:36.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:35 smithi136 bash[22601]: audit 2024-09-12T23:32:35.201610+0000 mon.a (mon.0) 582 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:32:36.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:35 smithi136 bash[32189]: audit 2024-09-12T23:32:35.185324+0000 mon.a (mon.0) 580 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:32:36.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:35 smithi136 bash[32189]: audit 2024-09-12T23:32:35.192555+0000 mon.a (mon.0) 581 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:32:36.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:35 smithi136 bash[32189]: audit 2024-09-12T23:32:35.201610+0000 mon.a (mon.0) 582 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:32:37.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:36 smithi181 bash[24776]: cluster 2024-09-12T23:32:35.473918+0000 mgr.y (mgr.14150) 306 : cluster [DBG] pgmap v280: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:37.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:36 smithi136 bash[22601]: cluster 2024-09-12T23:32:35.473918+0000 mgr.y (mgr.14150) 306 : cluster [DBG] pgmap v280: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:37.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:36 smithi136 bash[32189]: cluster 2024-09-12T23:32:35.473918+0000 mgr.y (mgr.14150) 306 : cluster [DBG] pgmap v280: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:39.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:38 smithi181 bash[24776]: cluster 2024-09-12T23:32:37.474459+0000 mgr.y (mgr.14150) 307 : cluster [DBG] pgmap v281: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:39.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:38 smithi136 bash[22601]: cluster 2024-09-12T23:32:37.474459+0000 mgr.y (mgr.14150) 307 : cluster [DBG] pgmap v281: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:39.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:38 smithi136 bash[32189]: cluster 2024-09-12T23:32:37.474459+0000 mgr.y (mgr.14150) 307 : cluster [DBG] pgmap v281: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:41.089 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:40 smithi181 bash[24776]: cluster 2024-09-12T23:32:39.475007+0000 mgr.y (mgr.14150) 308 : cluster [DBG] pgmap v282: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:41.090 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:40 smithi181 bash[24776]: audit 2024-09-12T23:32:40.296758+0000 mon.a (mon.0) 583 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:32:41.090 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:40 smithi181 bash[24776]: audit 2024-09-12T23:32:40.307916+0000 mon.a (mon.0) 584 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:32:41.090 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:40 smithi181 bash[24776]: audit 2024-09-12T23:32:40.730364+0000 mon.a (mon.0) 585 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:32:41.090 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:40 smithi181 bash[24776]: audit 2024-09-12T23:32:40.731574+0000 mon.a (mon.0) 586 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:32:41.090 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:40 smithi181 bash[24776]: audit 2024-09-12T23:32:40.742745+0000 mon.a (mon.0) 587 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:32:41.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:40 smithi136 bash[22601]: cluster 2024-09-12T23:32:39.475007+0000 mgr.y (mgr.14150) 308 : cluster [DBG] pgmap v282: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:41.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:40 smithi136 bash[22601]: audit 2024-09-12T23:32:40.296758+0000 mon.a (mon.0) 583 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:32:41.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:40 smithi136 bash[22601]: audit 2024-09-12T23:32:40.307916+0000 mon.a (mon.0) 584 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:32:41.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:40 smithi136 bash[22601]: audit 2024-09-12T23:32:40.730364+0000 mon.a (mon.0) 585 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:32:41.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:40 smithi136 bash[22601]: audit 2024-09-12T23:32:40.731574+0000 mon.a (mon.0) 586 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:32:41.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:40 smithi136 bash[22601]: audit 2024-09-12T23:32:40.742745+0000 mon.a (mon.0) 587 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:32:41.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:40 smithi136 bash[32189]: cluster 2024-09-12T23:32:39.475007+0000 mgr.y (mgr.14150) 308 : cluster [DBG] pgmap v282: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:41.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:40 smithi136 bash[32189]: audit 2024-09-12T23:32:40.296758+0000 mon.a (mon.0) 583 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:32:41.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:40 smithi136 bash[32189]: audit 2024-09-12T23:32:40.307916+0000 mon.a (mon.0) 584 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:32:41.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:40 smithi136 bash[32189]: audit 2024-09-12T23:32:40.730364+0000 mon.a (mon.0) 585 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:32:41.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:40 smithi136 bash[32189]: audit 2024-09-12T23:32:40.731574+0000 mon.a (mon.0) 586 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:32:41.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:40 smithi136 bash[32189]: audit 2024-09-12T23:32:40.742745+0000 mon.a (mon.0) 587 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:32:42.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:41 smithi181 bash[24776]: audit 2024-09-12T23:32:41.708707+0000 mon.b (mon.1) 23 : audit [INF] from='osd.7 [v2:172.21.15.181:6824/783620875,v1:172.21.15.181:6825/783620875]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-12T23:32:42.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:41 smithi181 bash[24776]: audit 2024-09-12T23:32:41.709905+0000 mon.a (mon.0) 588 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-12T23:32:42.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:41 smithi136 bash[22601]: audit 2024-09-12T23:32:41.708707+0000 mon.b (mon.1) 23 : audit [INF] from='osd.7 [v2:172.21.15.181:6824/783620875,v1:172.21.15.181:6825/783620875]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-12T23:32:42.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:41 smithi136 bash[22601]: audit 2024-09-12T23:32:41.709905+0000 mon.a (mon.0) 588 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-12T23:32:42.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:41 smithi136 bash[32189]: audit 2024-09-12T23:32:41.708707+0000 mon.b (mon.1) 23 : audit [INF] from='osd.7 [v2:172.21.15.181:6824/783620875,v1:172.21.15.181:6825/783620875]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-12T23:32:42.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:41 smithi136 bash[32189]: audit 2024-09-12T23:32:41.709905+0000 mon.a (mon.0) 588 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-12T23:32:43.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:42 smithi181 bash[24776]: cluster 2024-09-12T23:32:41.475523+0000 mgr.y (mgr.14150) 309 : cluster [DBG] pgmap v283: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:43.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:42 smithi181 bash[24776]: audit 2024-09-12T23:32:41.851662+0000 mon.a (mon.0) 589 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-12T23:32:43.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:42 smithi181 bash[24776]: cluster 2024-09-12T23:32:41.851861+0000 mon.a (mon.0) 590 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2024-09-12T23:32:43.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:42 smithi181 bash[24776]: audit 2024-09-12T23:32:41.853977+0000 mon.b (mon.1) 24 : audit [INF] from='osd.7 [v2:172.21.15.181:6824/783620875,v1:172.21.15.181:6825/783620875]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-12T23:32:43.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:42 smithi181 bash[24776]: audit 2024-09-12T23:32:41.855474+0000 mon.a (mon.0) 591 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-12T23:32:43.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:42 smithi181 bash[24776]: audit 2024-09-12T23:32:41.862972+0000 mon.a (mon.0) 592 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-12T23:32:43.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:42 smithi136 bash[22601]: cluster 2024-09-12T23:32:41.475523+0000 mgr.y (mgr.14150) 309 : cluster [DBG] pgmap v283: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:43.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:42 smithi136 bash[22601]: audit 2024-09-12T23:32:41.851662+0000 mon.a (mon.0) 589 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-12T23:32:43.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:42 smithi136 bash[22601]: cluster 2024-09-12T23:32:41.851861+0000 mon.a (mon.0) 590 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2024-09-12T23:32:43.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:42 smithi136 bash[22601]: audit 2024-09-12T23:32:41.853977+0000 mon.b (mon.1) 24 : audit [INF] from='osd.7 [v2:172.21.15.181:6824/783620875,v1:172.21.15.181:6825/783620875]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-12T23:32:43.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:42 smithi136 bash[22601]: audit 2024-09-12T23:32:41.855474+0000 mon.a (mon.0) 591 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-12T23:32:43.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:42 smithi136 bash[22601]: audit 2024-09-12T23:32:41.862972+0000 mon.a (mon.0) 592 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-12T23:32:43.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:42 smithi136 bash[32189]: cluster 2024-09-12T23:32:41.475523+0000 mgr.y (mgr.14150) 309 : cluster [DBG] pgmap v283: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:43.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:42 smithi136 bash[32189]: audit 2024-09-12T23:32:41.851662+0000 mon.a (mon.0) 589 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-12T23:32:43.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:42 smithi136 bash[32189]: cluster 2024-09-12T23:32:41.851861+0000 mon.a (mon.0) 590 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2024-09-12T23:32:43.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:42 smithi136 bash[32189]: audit 2024-09-12T23:32:41.853977+0000 mon.b (mon.1) 24 : audit [INF] from='osd.7 [v2:172.21.15.181:6824/783620875,v1:172.21.15.181:6825/783620875]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-12T23:32:43.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:42 smithi136 bash[32189]: audit 2024-09-12T23:32:41.855474+0000 mon.a (mon.0) 591 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-12T23:32:43.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:42 smithi136 bash[32189]: audit 2024-09-12T23:32:41.862972+0000 mon.a (mon.0) 592 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-12T23:32:43.764 INFO:teuthology.orchestra.run.smithi181.stdout:Created osd(s) 7 on host 'smithi181' 2024-09-12T23:32:44.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:43 smithi181 bash[24776]: audit 2024-09-12T23:32:42.853229+0000 mon.a (mon.0) 593 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2024-09-12T23:32:44.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:43 smithi181 bash[24776]: cluster 2024-09-12T23:32:42.853443+0000 mon.a (mon.0) 594 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2024-09-12T23:32:44.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:43 smithi181 bash[24776]: audit 2024-09-12T23:32:42.854285+0000 mon.a (mon.0) 595 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-12T23:32:44.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:43 smithi181 bash[24776]: audit 2024-09-12T23:32:42.866868+0000 mon.a (mon.0) 596 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-12T23:32:44.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:43 smithi181 bash[24776]: audit 2024-09-12T23:32:43.745182+0000 mon.a (mon.0) 597 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:32:44.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:43 smithi181 bash[24776]: audit 2024-09-12T23:32:43.752857+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:32:44.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:43 smithi181 bash[24776]: audit 2024-09-12T23:32:43.761780+0000 mon.a (mon.0) 599 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:32:44.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:43 smithi136 bash[22601]: audit 2024-09-12T23:32:42.853229+0000 mon.a (mon.0) 593 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2024-09-12T23:32:44.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:43 smithi136 bash[22601]: cluster 2024-09-12T23:32:42.853443+0000 mon.a (mon.0) 594 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2024-09-12T23:32:44.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:43 smithi136 bash[22601]: audit 2024-09-12T23:32:42.854285+0000 mon.a (mon.0) 595 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-12T23:32:44.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:43 smithi136 bash[22601]: audit 2024-09-12T23:32:42.866868+0000 mon.a (mon.0) 596 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-12T23:32:44.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:43 smithi136 bash[22601]: audit 2024-09-12T23:32:43.745182+0000 mon.a (mon.0) 597 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:32:44.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:43 smithi136 bash[22601]: audit 2024-09-12T23:32:43.752857+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:32:44.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:43 smithi136 bash[22601]: audit 2024-09-12T23:32:43.761780+0000 mon.a (mon.0) 599 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:32:44.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:43 smithi136 bash[32189]: audit 2024-09-12T23:32:42.853229+0000 mon.a (mon.0) 593 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2024-09-12T23:32:44.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:43 smithi136 bash[32189]: cluster 2024-09-12T23:32:42.853443+0000 mon.a (mon.0) 594 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2024-09-12T23:32:44.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:43 smithi136 bash[32189]: audit 2024-09-12T23:32:42.854285+0000 mon.a (mon.0) 595 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-12T23:32:44.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:43 smithi136 bash[32189]: audit 2024-09-12T23:32:42.866868+0000 mon.a (mon.0) 596 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-12T23:32:44.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:43 smithi136 bash[32189]: audit 2024-09-12T23:32:43.745182+0000 mon.a (mon.0) 597 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:32:44.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:43 smithi136 bash[32189]: audit 2024-09-12T23:32:43.752857+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:32:44.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:43 smithi136 bash[32189]: audit 2024-09-12T23:32:43.761780+0000 mon.a (mon.0) 599 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:32:44.334 DEBUG:teuthology.orchestra.run.smithi181:osd.7> sudo journalctl -f -n 0 -u ceph-da58a47c-715d-11ef-bceb-c7b262605968@osd.7.service 2024-09-12T23:32:44.336 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-09-12T23:32:44.336 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph osd stat -f json 2024-09-12T23:32:45.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:44 smithi136 bash[22601]: cluster 2024-09-12T23:32:42.696422+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-12T23:32:45.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:44 smithi136 bash[22601]: cluster 2024-09-12T23:32:42.696518+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-12T23:32:45.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:44 smithi136 bash[22601]: cluster 2024-09-12T23:32:43.476057+0000 mgr.y (mgr.14150) 310 : cluster [DBG] pgmap v286: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:45.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:44 smithi136 bash[22601]: audit 2024-09-12T23:32:43.865745+0000 mon.a (mon.0) 600 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-12T23:32:45.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:44 smithi136 bash[22601]: cluster 2024-09-12T23:32:43.880116+0000 mon.a (mon.0) 601 : cluster [INF] osd.7 [v2:172.21.15.181:6824/783620875,v1:172.21.15.181:6825/783620875] boot 2024-09-12T23:32:45.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:44 smithi136 bash[22601]: cluster 2024-09-12T23:32:43.880290+0000 mon.a (mon.0) 602 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2024-09-12T23:32:45.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:44 smithi136 bash[22601]: audit 2024-09-12T23:32:43.886906+0000 mon.a (mon.0) 603 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-12T23:32:45.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:44 smithi136 bash[32189]: cluster 2024-09-12T23:32:42.696422+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-12T23:32:45.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:44 smithi136 bash[32189]: cluster 2024-09-12T23:32:42.696518+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-12T23:32:45.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:44 smithi136 bash[32189]: cluster 2024-09-12T23:32:43.476057+0000 mgr.y (mgr.14150) 310 : cluster [DBG] pgmap v286: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:45.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:44 smithi136 bash[32189]: audit 2024-09-12T23:32:43.865745+0000 mon.a (mon.0) 600 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-12T23:32:45.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:44 smithi136 bash[32189]: cluster 2024-09-12T23:32:43.880116+0000 mon.a (mon.0) 601 : cluster [INF] osd.7 [v2:172.21.15.181:6824/783620875,v1:172.21.15.181:6825/783620875] boot 2024-09-12T23:32:45.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:44 smithi136 bash[32189]: cluster 2024-09-12T23:32:43.880290+0000 mon.a (mon.0) 602 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2024-09-12T23:32:45.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:44 smithi136 bash[32189]: audit 2024-09-12T23:32:43.886906+0000 mon.a (mon.0) 603 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-12T23:32:45.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:44 smithi181 bash[24776]: cluster 2024-09-12T23:32:42.696422+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-12T23:32:45.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:44 smithi181 bash[24776]: cluster 2024-09-12T23:32:42.696518+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-12T23:32:45.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:44 smithi181 bash[24776]: cluster 2024-09-12T23:32:43.476057+0000 mgr.y (mgr.14150) 310 : cluster [DBG] pgmap v286: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-12T23:32:45.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:44 smithi181 bash[24776]: audit 2024-09-12T23:32:43.865745+0000 mon.a (mon.0) 600 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-12T23:32:45.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:44 smithi181 bash[24776]: cluster 2024-09-12T23:32:43.880116+0000 mon.a (mon.0) 601 : cluster [INF] osd.7 [v2:172.21.15.181:6824/783620875,v1:172.21.15.181:6825/783620875] boot 2024-09-12T23:32:45.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:44 smithi181 bash[24776]: cluster 2024-09-12T23:32:43.880290+0000 mon.a (mon.0) 602 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2024-09-12T23:32:45.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:44 smithi181 bash[24776]: audit 2024-09-12T23:32:43.886906+0000 mon.a (mon.0) 603 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-12T23:32:46.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:45 smithi136 bash[32189]: cluster 2024-09-12T23:32:44.881844+0000 mon.a (mon.0) 604 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2024-09-12T23:32:46.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:45 smithi136 bash[32189]: cluster 2024-09-12T23:32:45.880933+0000 mon.a (mon.0) 605 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2024-09-12T23:32:46.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:45 smithi136 bash[22601]: cluster 2024-09-12T23:32:44.881844+0000 mon.a (mon.0) 604 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2024-09-12T23:32:46.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:45 smithi136 bash[22601]: cluster 2024-09-12T23:32:45.880933+0000 mon.a (mon.0) 605 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2024-09-12T23:32:46.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:45 smithi181 bash[24776]: cluster 2024-09-12T23:32:44.881844+0000 mon.a (mon.0) 604 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2024-09-12T23:32:46.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:45 smithi181 bash[24776]: cluster 2024-09-12T23:32:45.880933+0000 mon.a (mon.0) 605 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2024-09-12T23:32:47.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:46 smithi136 bash[22601]: cluster 2024-09-12T23:32:45.476788+0000 mgr.y (mgr.14150) 311 : cluster [DBG] pgmap v289: 1 pgs: 1 peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:32:47.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:46 smithi136 bash[22601]: cluster 2024-09-12T23:32:45.885519+0000 mon.a (mon.0) 606 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-12T23:32:47.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:46 smithi136 bash[32189]: cluster 2024-09-12T23:32:45.476788+0000 mgr.y (mgr.14150) 311 : cluster [DBG] pgmap v289: 1 pgs: 1 peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:32:47.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:46 smithi136 bash[32189]: cluster 2024-09-12T23:32:45.885519+0000 mon.a (mon.0) 606 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-12T23:32:47.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:46 smithi181 bash[24776]: cluster 2024-09-12T23:32:45.476788+0000 mgr.y (mgr.14150) 311 : cluster [DBG] pgmap v289: 1 pgs: 1 peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:32:47.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:46 smithi181 bash[24776]: cluster 2024-09-12T23:32:45.885519+0000 mon.a (mon.0) 606 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-12T23:32:49.098 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:32:49.185 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:48 smithi136 bash[32189]: cluster 2024-09-12T23:32:47.477393+0000 mgr.y (mgr.14150) 312 : cluster [DBG] pgmap v291: 1 pgs: 1 peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:32:49.185 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:48 smithi136 bash[22601]: cluster 2024-09-12T23:32:47.477393+0000 mgr.y (mgr.14150) 312 : cluster [DBG] pgmap v291: 1 pgs: 1 peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:32:49.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:48 smithi181 bash[24776]: cluster 2024-09-12T23:32:47.477393+0000 mgr.y (mgr.14150) 312 : cluster [DBG] pgmap v291: 1 pgs: 1 peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:32:51.170 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:50 smithi181 bash[24776]: cluster 2024-09-12T23:32:49.477987+0000 mgr.y (mgr.14150) 313 : cluster [DBG] pgmap v292: 1 pgs: 1 peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:32:51.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:50 smithi136 bash[22601]: cluster 2024-09-12T23:32:49.477987+0000 mgr.y (mgr.14150) 313 : cluster [DBG] pgmap v292: 1 pgs: 1 peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:32:51.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:50 smithi136 bash[32189]: cluster 2024-09-12T23:32:49.477987+0000 mgr.y (mgr.14150) 313 : cluster [DBG] pgmap v292: 1 pgs: 1 peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:32:52.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:51 smithi136 bash[32189]: cluster 2024-09-12T23:32:51.903783+0000 mon.a (mon.0) 607 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-12T23:32:52.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:51 smithi136 bash[32189]: cluster 2024-09-12T23:32:51.903804+0000 mon.a (mon.0) 608 : cluster [INF] Cluster is now healthy 2024-09-12T23:32:52.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:51 smithi136 bash[22601]: cluster 2024-09-12T23:32:51.903783+0000 mon.a (mon.0) 607 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-12T23:32:52.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:51 smithi136 bash[22601]: cluster 2024-09-12T23:32:51.903804+0000 mon.a (mon.0) 608 : cluster [INF] Cluster is now healthy 2024-09-12T23:32:52.069 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-12T23:32:52.225 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:51 smithi181 bash[24776]: cluster 2024-09-12T23:32:51.903783+0000 mon.a (mon.0) 607 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-12T23:32:52.225 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:51 smithi181 bash[24776]: cluster 2024-09-12T23:32:51.903804+0000 mon.a (mon.0) 608 : cluster [INF] Cluster is now healthy 2024-09-12T23:32:52.699 INFO:teuthology.orchestra.run.smithi136.stdout:{"epoch":50,"num_osds":8,"num_up_osds":8,"osd_up_since":1726183963,"num_in_osds":8,"osd_in_since":1726183938,"num_remapped_pgs":0} 2024-09-12T23:32:52.700 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-12T23:32:53.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:52 smithi136 bash[22601]: cluster 2024-09-12T23:32:51.478605+0000 mgr.y (mgr.14150) 314 : cluster [DBG] pgmap v293: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 76 KiB/s, 0 objects/s recovering 2024-09-12T23:32:53.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:52 smithi136 bash[22601]: audit 2024-09-12T23:32:52.072246+0000 mon.a (mon.0) 609 : audit [DBG] from='client.? 172.21.15.136:0/3790556614' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-12T23:32:53.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:52 smithi136 bash[22601]: audit 2024-09-12T23:32:52.851968+0000 mon.a (mon.0) 610 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:32:53.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:52 smithi136 bash[22601]: audit 2024-09-12T23:32:52.860576+0000 mon.a (mon.0) 611 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:32:53.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:52 smithi136 bash[22601]: audit 2024-09-12T23:32:52.862757+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:32:53.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:52 smithi136 bash[22601]: audit 2024-09-12T23:32:52.864328+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:32:53.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:52 smithi136 bash[22601]: audit 2024-09-12T23:32:52.865850+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:32:53.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:52 smithi136 bash[22601]: audit 2024-09-12T23:32:52.867935+0000 mon.a (mon.0) 615 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:32:53.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:52 smithi136 bash[22601]: audit 2024-09-12T23:32:52.879179+0000 mon.a (mon.0) 616 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:32:53.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:52 smithi136 bash[22601]: audit 2024-09-12T23:32:52.880853+0000 mon.a (mon.0) 617 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:32:53.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:52 smithi136 bash[22601]: audit 2024-09-12T23:32:52.881526+0000 mon.a (mon.0) 618 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:32:53.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:52 smithi136 bash[22601]: audit 2024-09-12T23:32:52.889548+0000 mon.a (mon.0) 619 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:32:53.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:52 smithi136 bash[32189]: cluster 2024-09-12T23:32:51.478605+0000 mgr.y (mgr.14150) 314 : cluster [DBG] pgmap v293: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 76 KiB/s, 0 objects/s recovering 2024-09-12T23:32:53.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:52 smithi136 bash[32189]: audit 2024-09-12T23:32:52.072246+0000 mon.a (mon.0) 609 : audit [DBG] from='client.? 172.21.15.136:0/3790556614' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-12T23:32:53.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:52 smithi136 bash[32189]: audit 2024-09-12T23:32:52.851968+0000 mon.a (mon.0) 610 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:32:53.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:52 smithi136 bash[32189]: audit 2024-09-12T23:32:52.860576+0000 mon.a (mon.0) 611 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:32:53.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:52 smithi136 bash[32189]: audit 2024-09-12T23:32:52.862757+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:32:53.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:52 smithi136 bash[32189]: audit 2024-09-12T23:32:52.864328+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:32:53.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:52 smithi136 bash[32189]: audit 2024-09-12T23:32:52.865850+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:32:53.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:52 smithi136 bash[32189]: audit 2024-09-12T23:32:52.867935+0000 mon.a (mon.0) 615 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:32:53.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:52 smithi136 bash[32189]: audit 2024-09-12T23:32:52.879179+0000 mon.a (mon.0) 616 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:32:53.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:52 smithi136 bash[32189]: audit 2024-09-12T23:32:52.880853+0000 mon.a (mon.0) 617 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:32:53.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:52 smithi136 bash[32189]: audit 2024-09-12T23:32:52.881526+0000 mon.a (mon.0) 618 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:32:53.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:52 smithi136 bash[32189]: audit 2024-09-12T23:32:52.889548+0000 mon.a (mon.0) 619 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:32:53.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:52 smithi181 bash[24776]: cluster 2024-09-12T23:32:51.478605+0000 mgr.y (mgr.14150) 314 : cluster [DBG] pgmap v293: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 76 KiB/s, 0 objects/s recovering 2024-09-12T23:32:53.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:52 smithi181 bash[24776]: audit 2024-09-12T23:32:52.072246+0000 mon.a (mon.0) 609 : audit [DBG] from='client.? 172.21.15.136:0/3790556614' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-12T23:32:53.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:52 smithi181 bash[24776]: audit 2024-09-12T23:32:52.851968+0000 mon.a (mon.0) 610 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:32:53.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:52 smithi181 bash[24776]: audit 2024-09-12T23:32:52.860576+0000 mon.a (mon.0) 611 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:32:53.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:52 smithi181 bash[24776]: audit 2024-09-12T23:32:52.862757+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:32:53.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:52 smithi181 bash[24776]: audit 2024-09-12T23:32:52.864328+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:32:53.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:52 smithi181 bash[24776]: audit 2024-09-12T23:32:52.865850+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:32:53.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:52 smithi181 bash[24776]: audit 2024-09-12T23:32:52.867935+0000 mon.a (mon.0) 615 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:32:53.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:52 smithi181 bash[24776]: audit 2024-09-12T23:32:52.879179+0000 mon.a (mon.0) 616 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:32:53.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:52 smithi181 bash[24776]: audit 2024-09-12T23:32:52.880853+0000 mon.a (mon.0) 617 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:32:53.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:52 smithi181 bash[24776]: audit 2024-09-12T23:32:52.881526+0000 mon.a (mon.0) 618 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:32:53.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:52 smithi181 bash[24776]: audit 2024-09-12T23:32:52.889548+0000 mon.a (mon.0) 619 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:32:54.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:53 smithi136 bash[22601]: cephadm 2024-09-12T23:32:52.844588+0000 mgr.y (mgr.14150) 315 : cephadm [INF] Detected new or changed devices on smithi181 2024-09-12T23:32:54.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:53 smithi136 bash[22601]: cephadm 2024-09-12T23:32:52.869056+0000 mgr.y (mgr.14150) 316 : cephadm [INF] Adjusting osd_memory_target on smithi181 to 4316M 2024-09-12T23:32:54.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:53 smithi136 bash[22601]: cluster 2024-09-12T23:32:53.479172+0000 mgr.y (mgr.14150) 317 : cluster [DBG] pgmap v294: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 67 KiB/s, 0 objects/s recovering 2024-09-12T23:32:54.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:53 smithi136 bash[32189]: cephadm 2024-09-12T23:32:52.844588+0000 mgr.y (mgr.14150) 315 : cephadm [INF] Detected new or changed devices on smithi181 2024-09-12T23:32:54.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:53 smithi136 bash[32189]: cephadm 2024-09-12T23:32:52.869056+0000 mgr.y (mgr.14150) 316 : cephadm [INF] Adjusting osd_memory_target on smithi181 to 4316M 2024-09-12T23:32:54.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:53 smithi136 bash[32189]: cluster 2024-09-12T23:32:53.479172+0000 mgr.y (mgr.14150) 317 : cluster [DBG] pgmap v294: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 67 KiB/s, 0 objects/s recovering 2024-09-12T23:32:54.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:53 smithi181 bash[24776]: cephadm 2024-09-12T23:32:52.844588+0000 mgr.y (mgr.14150) 315 : cephadm [INF] Detected new or changed devices on smithi181 2024-09-12T23:32:54.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:53 smithi181 bash[24776]: cephadm 2024-09-12T23:32:52.869056+0000 mgr.y (mgr.14150) 316 : cephadm [INF] Adjusting osd_memory_target on smithi181 to 4316M 2024-09-12T23:32:54.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:53 smithi181 bash[24776]: cluster 2024-09-12T23:32:53.479172+0000 mgr.y (mgr.14150) 317 : cluster [DBG] pgmap v294: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 67 KiB/s, 0 objects/s recovering 2024-09-12T23:32:56.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:56 smithi181 bash[24776]: cluster 2024-09-12T23:32:55.479630+0000 mgr.y (mgr.14150) 318 : cluster [DBG] pgmap v295: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-12T23:32:57.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:56 smithi136 bash[22601]: cluster 2024-09-12T23:32:55.479630+0000 mgr.y (mgr.14150) 318 : cluster [DBG] pgmap v295: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-12T23:32:57.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:56 smithi136 bash[32189]: cluster 2024-09-12T23:32:55.479630+0000 mgr.y (mgr.14150) 318 : cluster [DBG] pgmap v295: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-12T23:32:57.463 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:32:58.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:32:58 smithi181 bash[24776]: cluster 2024-09-12T23:32:57.479964+0000 mgr.y (mgr.14150) 319 : cluster [DBG] pgmap v296: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 50 KiB/s, 0 objects/s recovering 2024-09-12T23:32:58.905 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:32:58 smithi136 bash[22601]: cluster 2024-09-12T23:32:57.479964+0000 mgr.y (mgr.14150) 319 : cluster [DBG] pgmap v296: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 50 KiB/s, 0 objects/s recovering 2024-09-12T23:32:58.905 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:32:58 smithi136 bash[32189]: cluster 2024-09-12T23:32:57.479964+0000 mgr.y (mgr.14150) 319 : cluster [DBG] pgmap v296: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 50 KiB/s, 0 objects/s recovering 2024-09-12T23:33:00.592 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-12T23:33:00.592 INFO:teuthology.orchestra.run.smithi136.stdout:{"epoch":50,"fsid":"da58a47c-715d-11ef-bceb-c7b262605968","created":"2024-09-12T23:23:29.000926+0000","modified":"2024-09-12T23:32:45.871955+0000","last_up_change":"2024-09-12T23:32:43.859472+0000","last_in_change":"2024-09-12T23:32:18.995528+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-12T23:28:33.956985+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"87aedf04-695e-4a8f-80c3-ffd18883a4d5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6802","nonce":4251571952},{"type":"v1","addr":"172.21.15.136:6803","nonce":4251571952}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6804","nonce":4251571952},{"type":"v1","addr":"172.21.15.136:6805","nonce":4251571952}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6808","nonce":4251571952},{"type":"v1","addr":"172.21.15.136:6809","nonce":4251571952}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6806","nonce":4251571952},{"type":"v1","addr":"172.21.15.136:6807","nonce":4251571952}]},"public_addr":"172.21.15.136:6803/4251571952","cluster_addr":"172.21.15.136:6805/4251571952","heartbeat_back_addr":"172.21.15.136:6809/4251571952","heartbeat_front_addr":"172.21.15.136:6807/4251571952","state":["exists","up"]},{"osd":1,"uuid":"850d4f98-0640-4151-bd45-a351ca41d54e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6810","nonce":3482177958},{"type":"v1","addr":"172.21.15.136:6811","nonce":3482177958}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6812","nonce":3482177958},{"type":"v1","addr":"172.21.15.136:6813","nonce":3482177958}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6816","nonce":3482177958},{"type":"v1","addr":"172.21.15.136:6817","nonce":3482177958}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6814","nonce":3482177958},{"type":"v1","addr":"172.21.15.136:6815","nonce":3482177958}]},"public_addr":"172.21.15.136:6811/3482177958","cluster_addr":"172.21.15.136:6813/3482177958","heartbeat_back_addr":"172.21.15.136:6817/3482177958","heartbeat_front_addr":"172.21.15.136:6815/3482177958","state":["exists","up"]},{"osd":2,"uuid":"bde707d6-e027-4b63-9853-fd59212eea1e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6818","nonce":308477304},{"type":"v1","addr":"172.21.15.136:6819","nonce":308477304}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6820","nonce":308477304},{"type":"v1","addr":"172.21.15.136:6821","nonce":308477304}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6824","nonce":308477304},{"type":"v1","addr":"172.21.15.136:6825","nonce":308477304}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6822","nonce":308477304},{"type":"v1","addr":"172.21.15.136:6823","nonce":308477304}]},"public_addr":"172.21.15.136:6819/308477304","cluster_addr":"172.21.15.136:6821/308477304","heartbeat_back_addr":"172.21.15.136:6825/308477304","heartbeat_front_addr":"172.21.15.136:6823/308477304","state":["exists","up"]},{"osd":3,"uuid":"d61f8357-b89b-4c05-b366-65a9f788064d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6826","nonce":201948269},{"type":"v1","addr":"172.21.15.136:6827","nonce":201948269}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6828","nonce":201948269},{"type":"v1","addr":"172.21.15.136:6829","nonce":201948269}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6832","nonce":201948269},{"type":"v1","addr":"172.21.15.136:6833","nonce":201948269}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6830","nonce":201948269},{"type":"v1","addr":"172.21.15.136:6831","nonce":201948269}]},"public_addr":"172.21.15.136:6827/201948269","cluster_addr":"172.21.15.136:6829/201948269","heartbeat_back_addr":"172.21.15.136:6833/201948269","heartbeat_front_addr":"172.21.15.136:6831/201948269","state":["exists","up"]},{"osd":4,"uuid":"98b0e81c-824e-4dbc-9faa-2de9f199af1b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6800","nonce":775251087},{"type":"v1","addr":"172.21.15.181:6801","nonce":775251087}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6802","nonce":775251087},{"type":"v1","addr":"172.21.15.181:6803","nonce":775251087}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6806","nonce":775251087},{"type":"v1","addr":"172.21.15.181:6807","nonce":775251087}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6804","nonce":775251087},{"type":"v1","addr":"172.21.15.181:6805","nonce":775251087}]},"public_addr":"172.21.15.181:6801/775251087","cluster_addr":"172.21.15.181:6803/775251087","heartbeat_back_addr":"172.21.15.181:6807/775251087","heartbeat_front_addr":"172.21.15.181:6805/775251087","state":["exists","up"]},{"osd":5,"uuid":"5c6d2f69-7f9c-4689-95b0-bd4eb858956d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":37,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6808","nonce":2109931558},{"type":"v1","addr":"172.21.15.181:6809","nonce":2109931558}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6810","nonce":2109931558},{"type":"v1","addr":"172.21.15.181:6811","nonce":2109931558}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6814","nonce":2109931558},{"type":"v1","addr":"172.21.15.181:6815","nonce":2109931558}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6812","nonce":2109931558},{"type":"v1","addr":"172.21.15.181:6813","nonce":2109931558}]},"public_addr":"172.21.15.181:6809/2109931558","cluster_addr":"172.21.15.181:6811/2109931558","heartbeat_back_addr":"172.21.15.181:6815/2109931558","heartbeat_front_addr":"172.21.15.181:6813/2109931558","state":["exists","up"]},{"osd":6,"uuid":"532ee052-af15-433a-8e93-b0b642aaf85b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6816","nonce":4099148413},{"type":"v1","addr":"172.21.15.181:6817","nonce":4099148413}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6818","nonce":4099148413},{"type":"v1","addr":"172.21.15.181:6819","nonce":4099148413}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6822","nonce":4099148413},{"type":"v1","addr":"172.21.15.181:6823","nonce":4099148413}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6820","nonce":4099148413},{"type":"v1","addr":"172.21.15.181:6821","nonce":4099148413}]},"public_addr":"172.21.15.181:6817/4099148413","cluster_addr":"172.21.15.181:6819/4099148413","heartbeat_back_addr":"172.21.15.181:6823/4099148413","heartbeat_front_addr":"172.21.15.181:6821/4099148413","state":["exists","up"]},{"osd":7,"uuid":"d2ff80f0-6ef0-4098-877c-a891d48c687d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6824","nonce":783620875},{"type":"v1","addr":"172.21.15.181:6825","nonce":783620875}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6826","nonce":783620875},{"type":"v1","addr":"172.21.15.181:6827","nonce":783620875}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6830","nonce":783620875},{"type":"v1","addr":"172.21.15.181:6831","nonce":783620875}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6828","nonce":783620875},{"type":"v1","addr":"172.21.15.181:6829","nonce":783620875}]},"public_addr":"172.21.15.181:6825/783620875","cluster_addr":"172.21.15.181:6827/783620875","heartbeat_back_addr":"172.21.15.181:6831/783620875","heartbeat_front_addr":"172.21.15.181:6829/783620875","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:26:50.657580+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:27:40.536476+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:28:31.755012+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:29:23.585451+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:30:11.888258+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:31:01.812765+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:31:52.166696+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:32:42.696525+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.136:0/4260962972":"2024-09-13T23:24:25.346823+0000","172.21.15.136:6801/3594272622":"2024-09-13T23:24:25.346823+0000","172.21.15.136:0/209539205":"2024-09-13T23:24:25.346823+0000","172.21.15.136:6800/3594272622":"2024-09-13T23:24:25.346823+0000","172.21.15.136:0/1549678324":"2024-09-13T23:23:54.847045+0000","172.21.15.136:0/3376519841":"2024-09-13T23:23:54.847045+0000","172.21.15.136:0/3910468981":"2024-09-13T23:23:54.847045+0000","172.21.15.136:6801/2894181186":"2024-09-13T23:23:54.847045+0000","172.21.15.136:0/1964283041":"2024-09-13T23:24:25.346823+0000","172.21.15.136:6800/2894181186":"2024-09-13T23:23:54.847045+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-12T23:33:00.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:00 smithi181 bash[24776]: cluster 2024-09-12T23:32:59.480501+0000 mgr.y (mgr.14150) 320 : cluster [DBG] pgmap v297: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-12T23:33:01.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:00 smithi136 bash[22601]: cluster 2024-09-12T23:32:59.480501+0000 mgr.y (mgr.14150) 320 : cluster [DBG] pgmap v297: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-12T23:33:01.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:00 smithi136 bash[32189]: cluster 2024-09-12T23:32:59.480501+0000 mgr.y (mgr.14150) 320 : cluster [DBG] pgmap v297: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-12T23:33:01.223 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-09-12T23:28:33.956985+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-09-12T23:33:01.223 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-09-12T23:33:01.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:01 smithi181 bash[24776]: audit 2024-09-12T23:33:00.593706+0000 mon.c (mon.2) 17 : audit [DBG] from='client.? 172.21.15.136:0/605463896' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-12T23:33:02.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:01 smithi136 bash[22601]: audit 2024-09-12T23:33:00.593706+0000 mon.c (mon.2) 17 : audit [DBG] from='client.? 172.21.15.136:0/605463896' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-12T23:33:02.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:01 smithi136 bash[32189]: audit 2024-09-12T23:33:00.593706+0000 mon.c (mon.2) 17 : audit [DBG] from='client.? 172.21.15.136:0/605463896' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-12T23:33:02.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:02 smithi181 bash[24776]: cluster 2024-09-12T23:33:01.480913+0000 mgr.y (mgr.14150) 321 : cluster [DBG] pgmap v298: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-12T23:33:03.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:02 smithi136 bash[22601]: cluster 2024-09-12T23:33:01.480913+0000 mgr.y (mgr.14150) 321 : cluster [DBG] pgmap v298: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-12T23:33:03.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:02 smithi136 bash[32189]: cluster 2024-09-12T23:33:01.480913+0000 mgr.y (mgr.14150) 321 : cluster [DBG] pgmap v298: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-12T23:33:04.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:04 smithi181 bash[24776]: cluster 2024-09-12T23:33:03.481366+0000 mgr.y (mgr.14150) 322 : cluster [DBG] pgmap v299: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:05.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:04 smithi136 bash[22601]: cluster 2024-09-12T23:33:03.481366+0000 mgr.y (mgr.14150) 322 : cluster [DBG] pgmap v299: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:05.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:04 smithi136 bash[32189]: cluster 2024-09-12T23:33:03.481366+0000 mgr.y (mgr.14150) 322 : cluster [DBG] pgmap v299: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:05.985 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:33:06.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:06 smithi181 bash[24776]: cluster 2024-09-12T23:33:05.481782+0000 mgr.y (mgr.14150) 323 : cluster [DBG] pgmap v300: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:07.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:06 smithi136 bash[22601]: cluster 2024-09-12T23:33:05.481782+0000 mgr.y (mgr.14150) 323 : cluster [DBG] pgmap v300: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:07.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:06 smithi136 bash[32189]: cluster 2024-09-12T23:33:05.481782+0000 mgr.y (mgr.14150) 323 : cluster [DBG] pgmap v300: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:08.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:08 smithi181 bash[24776]: cluster 2024-09-12T23:33:07.482333+0000 mgr.y (mgr.14150) 324 : cluster [DBG] pgmap v301: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:09.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:08 smithi136 bash[32189]: cluster 2024-09-12T23:33:07.482333+0000 mgr.y (mgr.14150) 324 : cluster [DBG] pgmap v301: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:09.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:08 smithi136 bash[22601]: cluster 2024-09-12T23:33:07.482333+0000 mgr.y (mgr.14150) 324 : cluster [DBG] pgmap v301: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:09.092 INFO:teuthology.orchestra.run.smithi136.stdout:pg_num: 1 2024-09-12T23:33:09.773 INFO:tasks.cephadm:Adding mds.a on smithi136 2024-09-12T23:33:09.773 INFO:tasks.cephadm:Adding mds.b on smithi181 2024-09-12T23:33:09.774 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph orch apply mds all '2;smithi136=a;smithi181=b' 2024-09-12T23:33:09.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:09 smithi181 bash[24776]: audit 2024-09-12T23:33:09.094926+0000 mon.a (mon.0) 620 : audit [DBG] from='client.? 172.21.15.136:0/298841089' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-12T23:33:10.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:09 smithi136 bash[32189]: audit 2024-09-12T23:33:09.094926+0000 mon.a (mon.0) 620 : audit [DBG] from='client.? 172.21.15.136:0/298841089' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-12T23:33:10.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:09 smithi136 bash[22601]: audit 2024-09-12T23:33:09.094926+0000 mon.a (mon.0) 620 : audit [DBG] from='client.? 172.21.15.136:0/298841089' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-12T23:33:10.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:10 smithi181 bash[24776]: cluster 2024-09-12T23:33:09.482835+0000 mgr.y (mgr.14150) 325 : cluster [DBG] pgmap v302: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:11.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:10 smithi136 bash[22601]: cluster 2024-09-12T23:33:09.482835+0000 mgr.y (mgr.14150) 325 : cluster [DBG] pgmap v302: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:11.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:10 smithi136 bash[32189]: cluster 2024-09-12T23:33:09.482835+0000 mgr.y (mgr.14150) 325 : cluster [DBG] pgmap v302: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:12.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:12 smithi181 bash[24776]: cluster 2024-09-12T23:33:11.483383+0000 mgr.y (mgr.14150) 326 : cluster [DBG] pgmap v303: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:13.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:12 smithi136 bash[22601]: cluster 2024-09-12T23:33:11.483383+0000 mgr.y (mgr.14150) 326 : cluster [DBG] pgmap v303: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:13.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:12 smithi136 bash[32189]: cluster 2024-09-12T23:33:11.483383+0000 mgr.y (mgr.14150) 326 : cluster [DBG] pgmap v303: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:14.529 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.b/config 2024-09-12T23:33:14.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:14 smithi181 bash[24776]: cluster 2024-09-12T23:33:13.483922+0000 mgr.y (mgr.14150) 327 : cluster [DBG] pgmap v304: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:15.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:14 smithi136 bash[22601]: cluster 2024-09-12T23:33:13.483922+0000 mgr.y (mgr.14150) 327 : cluster [DBG] pgmap v304: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:15.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:14 smithi136 bash[32189]: cluster 2024-09-12T23:33:13.483922+0000 mgr.y (mgr.14150) 327 : cluster [DBG] pgmap v304: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:16.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:16 smithi181 bash[24776]: cluster 2024-09-12T23:33:15.484485+0000 mgr.y (mgr.14150) 328 : cluster [DBG] pgmap v305: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:17.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:16 smithi136 bash[22601]: cluster 2024-09-12T23:33:15.484485+0000 mgr.y (mgr.14150) 328 : cluster [DBG] pgmap v305: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:17.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:16 smithi136 bash[32189]: cluster 2024-09-12T23:33:15.484485+0000 mgr.y (mgr.14150) 328 : cluster [DBG] pgmap v305: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:17.237 INFO:teuthology.orchestra.run.smithi181.stdout:Scheduled mds.all update... 2024-09-12T23:33:17.841 DEBUG:teuthology.orchestra.run.smithi136:mds.a> sudo journalctl -f -n 0 -u ceph-da58a47c-715d-11ef-bceb-c7b262605968@mds.a.service 2024-09-12T23:33:17.843 DEBUG:teuthology.orchestra.run.smithi181:mds.b> sudo journalctl -f -n 0 -u ceph-da58a47c-715d-11ef-bceb-c7b262605968@mds.b.service 2024-09-12T23:33:17.846 INFO:tasks.cephadm:Setting up CephFS filesystem(s)... 2024-09-12T23:33:17.847 DEBUG:teuthology.orchestra.run.smithi136:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-12T23:33:18.178 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-12T23:33:18.179 INFO:teuthology.orchestra.run.smithi136.stdout:{"epoch":50,"fsid":"da58a47c-715d-11ef-bceb-c7b262605968","created":"2024-09-12T23:23:29.000926+0000","modified":"2024-09-12T23:32:45.871955+0000","last_up_change":"2024-09-12T23:32:43.859472+0000","last_in_change":"2024-09-12T23:32:18.995528+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-12T23:28:33.956985+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"87aedf04-695e-4a8f-80c3-ffd18883a4d5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6802","nonce":4251571952},{"type":"v1","addr":"172.21.15.136:6803","nonce":4251571952}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6804","nonce":4251571952},{"type":"v1","addr":"172.21.15.136:6805","nonce":4251571952}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6808","nonce":4251571952},{"type":"v1","addr":"172.21.15.136:6809","nonce":4251571952}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6806","nonce":4251571952},{"type":"v1","addr":"172.21.15.136:6807","nonce":4251571952}]},"public_addr":"172.21.15.136:6803/4251571952","cluster_addr":"172.21.15.136:6805/4251571952","heartbeat_back_addr":"172.21.15.136:6809/4251571952","heartbeat_front_addr":"172.21.15.136:6807/4251571952","state":["exists","up"]},{"osd":1,"uuid":"850d4f98-0640-4151-bd45-a351ca41d54e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6810","nonce":3482177958},{"type":"v1","addr":"172.21.15.136:6811","nonce":3482177958}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6812","nonce":3482177958},{"type":"v1","addr":"172.21.15.136:6813","nonce":3482177958}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6816","nonce":3482177958},{"type":"v1","addr":"172.21.15.136:6817","nonce":3482177958}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6814","nonce":3482177958},{"type":"v1","addr":"172.21.15.136:6815","nonce":3482177958}]},"public_addr":"172.21.15.136:6811/3482177958","cluster_addr":"172.21.15.136:6813/3482177958","heartbeat_back_addr":"172.21.15.136:6817/3482177958","heartbeat_front_addr":"172.21.15.136:6815/3482177958","state":["exists","up"]},{"osd":2,"uuid":"bde707d6-e027-4b63-9853-fd59212eea1e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6818","nonce":308477304},{"type":"v1","addr":"172.21.15.136:6819","nonce":308477304}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6820","nonce":308477304},{"type":"v1","addr":"172.21.15.136:6821","nonce":308477304}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6824","nonce":308477304},{"type":"v1","addr":"172.21.15.136:6825","nonce":308477304}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6822","nonce":308477304},{"type":"v1","addr":"172.21.15.136:6823","nonce":308477304}]},"public_addr":"172.21.15.136:6819/308477304","cluster_addr":"172.21.15.136:6821/308477304","heartbeat_back_addr":"172.21.15.136:6825/308477304","heartbeat_front_addr":"172.21.15.136:6823/308477304","state":["exists","up"]},{"osd":3,"uuid":"d61f8357-b89b-4c05-b366-65a9f788064d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6826","nonce":201948269},{"type":"v1","addr":"172.21.15.136:6827","nonce":201948269}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6828","nonce":201948269},{"type":"v1","addr":"172.21.15.136:6829","nonce":201948269}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6832","nonce":201948269},{"type":"v1","addr":"172.21.15.136:6833","nonce":201948269}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6830","nonce":201948269},{"type":"v1","addr":"172.21.15.136:6831","nonce":201948269}]},"public_addr":"172.21.15.136:6827/201948269","cluster_addr":"172.21.15.136:6829/201948269","heartbeat_back_addr":"172.21.15.136:6833/201948269","heartbeat_front_addr":"172.21.15.136:6831/201948269","state":["exists","up"]},{"osd":4,"uuid":"98b0e81c-824e-4dbc-9faa-2de9f199af1b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6800","nonce":775251087},{"type":"v1","addr":"172.21.15.181:6801","nonce":775251087}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6802","nonce":775251087},{"type":"v1","addr":"172.21.15.181:6803","nonce":775251087}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6806","nonce":775251087},{"type":"v1","addr":"172.21.15.181:6807","nonce":775251087}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6804","nonce":775251087},{"type":"v1","addr":"172.21.15.181:6805","nonce":775251087}]},"public_addr":"172.21.15.181:6801/775251087","cluster_addr":"172.21.15.181:6803/775251087","heartbeat_back_addr":"172.21.15.181:6807/775251087","heartbeat_front_addr":"172.21.15.181:6805/775251087","state":["exists","up"]},{"osd":5,"uuid":"5c6d2f69-7f9c-4689-95b0-bd4eb858956d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":37,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6808","nonce":2109931558},{"type":"v1","addr":"172.21.15.181:6809","nonce":2109931558}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6810","nonce":2109931558},{"type":"v1","addr":"172.21.15.181:6811","nonce":2109931558}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6814","nonce":2109931558},{"type":"v1","addr":"172.21.15.181:6815","nonce":2109931558}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6812","nonce":2109931558},{"type":"v1","addr":"172.21.15.181:6813","nonce":2109931558}]},"public_addr":"172.21.15.181:6809/2109931558","cluster_addr":"172.21.15.181:6811/2109931558","heartbeat_back_addr":"172.21.15.181:6815/2109931558","heartbeat_front_addr":"172.21.15.181:6813/2109931558","state":["exists","up"]},{"osd":6,"uuid":"532ee052-af15-433a-8e93-b0b642aaf85b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6816","nonce":4099148413},{"type":"v1","addr":"172.21.15.181:6817","nonce":4099148413}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6818","nonce":4099148413},{"type":"v1","addr":"172.21.15.181:6819","nonce":4099148413}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6822","nonce":4099148413},{"type":"v1","addr":"172.21.15.181:6823","nonce":4099148413}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6820","nonce":4099148413},{"type":"v1","addr":"172.21.15.181:6821","nonce":4099148413}]},"public_addr":"172.21.15.181:6817/4099148413","cluster_addr":"172.21.15.181:6819/4099148413","heartbeat_back_addr":"172.21.15.181:6823/4099148413","heartbeat_front_addr":"172.21.15.181:6821/4099148413","state":["exists","up"]},{"osd":7,"uuid":"d2ff80f0-6ef0-4098-877c-a891d48c687d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6824","nonce":783620875},{"type":"v1","addr":"172.21.15.181:6825","nonce":783620875}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6826","nonce":783620875},{"type":"v1","addr":"172.21.15.181:6827","nonce":783620875}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6830","nonce":783620875},{"type":"v1","addr":"172.21.15.181:6831","nonce":783620875}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6828","nonce":783620875},{"type":"v1","addr":"172.21.15.181:6829","nonce":783620875}]},"public_addr":"172.21.15.181:6825/783620875","cluster_addr":"172.21.15.181:6827/783620875","heartbeat_back_addr":"172.21.15.181:6831/783620875","heartbeat_front_addr":"172.21.15.181:6829/783620875","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:26:50.657580+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:27:40.536476+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:28:31.755012+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:29:23.585451+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:30:11.888258+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:31:01.812765+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:31:52.166696+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:32:42.696525+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.136:0/4260962972":"2024-09-13T23:24:25.346823+0000","172.21.15.136:6801/3594272622":"2024-09-13T23:24:25.346823+0000","172.21.15.136:0/209539205":"2024-09-13T23:24:25.346823+0000","172.21.15.136:6800/3594272622":"2024-09-13T23:24:25.346823+0000","172.21.15.136:0/1549678324":"2024-09-13T23:23:54.847045+0000","172.21.15.136:0/3376519841":"2024-09-13T23:23:54.847045+0000","172.21.15.136:0/3910468981":"2024-09-13T23:23:54.847045+0000","172.21.15.136:6801/2894181186":"2024-09-13T23:23:54.847045+0000","172.21.15.136:0/1964283041":"2024-09-13T23:24:25.346823+0000","172.21.15.136:6800/2894181186":"2024-09-13T23:23:54.847045+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-12T23:33:18.196 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-09-12T23:28:33.956985+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-09-12T23:33:18.196 DEBUG:teuthology.orchestra.run.smithi136:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-09-12T23:33:18.482 INFO:teuthology.orchestra.run.smithi136.stdout:pg_num: 1 2024-09-12T23:33:18.500 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:18 smithi136 bash[22601]: audit 2024-09-12T23:33:17.228774+0000 mgr.y (mgr.14150) 329 : audit [DBG] from='client.24289 -' entity='client.admin' cmd=[{"prefix": "orch apply mds", "fs_name": "all", "placement": "2;smithi136=a;smithi181=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:33:18.500 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:18 smithi136 bash[22601]: cephadm 2024-09-12T23:33:17.232357+0000 mgr.y (mgr.14150) 330 : cephadm [INF] Saving service mds.all spec with placement smithi136=a;smithi181=b;count:2 2024-09-12T23:33:18.501 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:18 smithi136 bash[22601]: audit 2024-09-12T23:33:17.239578+0000 mon.a (mon.0) 621 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:18.501 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:18 smithi136 bash[22601]: audit 2024-09-12T23:33:17.241403+0000 mon.a (mon.0) 622 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:33:18.501 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:18 smithi136 bash[22601]: cluster 2024-09-12T23:33:17.485018+0000 mgr.y (mgr.14150) 331 : cluster [DBG] pgmap v306: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:18.501 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:18 smithi136 bash[22601]: audit 2024-09-12T23:33:17.641354+0000 mon.a (mon.0) 623 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:33:18.501 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:18 smithi136 bash[22601]: audit 2024-09-12T23:33:17.642844+0000 mon.a (mon.0) 624 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:33:18.501 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:18 smithi136 bash[22601]: audit 2024-09-12T23:33:17.652006+0000 mon.a (mon.0) 625 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:18.501 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:18 smithi136 bash[22601]: audit 2024-09-12T23:33:17.657090+0000 mon.a (mon.0) 626 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-12T23:33:18.501 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:18 smithi136 bash[22601]: audit 2024-09-12T23:33:17.663729+0000 mon.a (mon.0) 627 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-12T23:33:18.501 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:18 smithi136 bash[22601]: audit 2024-09-12T23:33:17.665351+0000 mon.a (mon.0) 628 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:33:18.501 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:18 smithi136 bash[22601]: cephadm 2024-09-12T23:33:17.666818+0000 mgr.y (mgr.14150) 332 : cephadm [INF] Deploying daemon mds.a on smithi136 2024-09-12T23:33:18.501 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:18 smithi136 bash[22601]: audit 2024-09-12T23:33:18.180640+0000 mon.a (mon.0) 629 : audit [DBG] from='client.? 172.21.15.136:0/1409434709' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-12T23:33:18.502 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:18 smithi136 bash[32189]: audit 2024-09-12T23:33:17.228774+0000 mgr.y (mgr.14150) 329 : audit [DBG] from='client.24289 -' entity='client.admin' cmd=[{"prefix": "orch apply mds", "fs_name": "all", "placement": "2;smithi136=a;smithi181=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:33:18.502 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:18 smithi136 bash[32189]: cephadm 2024-09-12T23:33:17.232357+0000 mgr.y (mgr.14150) 330 : cephadm [INF] Saving service mds.all spec with placement smithi136=a;smithi181=b;count:2 2024-09-12T23:33:18.502 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:18 smithi136 bash[32189]: audit 2024-09-12T23:33:17.239578+0000 mon.a (mon.0) 621 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:18.502 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:18 smithi136 bash[32189]: audit 2024-09-12T23:33:17.241403+0000 mon.a (mon.0) 622 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:33:18.502 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:18 smithi136 bash[32189]: cluster 2024-09-12T23:33:17.485018+0000 mgr.y (mgr.14150) 331 : cluster [DBG] pgmap v306: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:18.502 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:18 smithi136 bash[32189]: audit 2024-09-12T23:33:17.641354+0000 mon.a (mon.0) 623 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:33:18.502 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:18 smithi136 bash[32189]: audit 2024-09-12T23:33:17.642844+0000 mon.a (mon.0) 624 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:33:18.502 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:18 smithi136 bash[32189]: audit 2024-09-12T23:33:17.652006+0000 mon.a (mon.0) 625 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:18.502 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:18 smithi136 bash[32189]: audit 2024-09-12T23:33:17.657090+0000 mon.a (mon.0) 626 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-12T23:33:18.502 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:18 smithi136 bash[32189]: audit 2024-09-12T23:33:17.663729+0000 mon.a (mon.0) 627 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-12T23:33:18.502 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:18 smithi136 bash[32189]: audit 2024-09-12T23:33:17.665351+0000 mon.a (mon.0) 628 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:33:18.503 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:18 smithi136 bash[32189]: cephadm 2024-09-12T23:33:17.666818+0000 mgr.y (mgr.14150) 332 : cephadm [INF] Deploying daemon mds.a on smithi136 2024-09-12T23:33:18.503 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:18 smithi136 bash[32189]: audit 2024-09-12T23:33:18.180640+0000 mon.a (mon.0) 629 : audit [DBG] from='client.? 172.21.15.136:0/1409434709' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-12T23:33:18.503 DEBUG:teuthology.orchestra.run.smithi136:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-09-12T23:33:18.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:18 smithi181 bash[24776]: audit 2024-09-12T23:33:17.228774+0000 mgr.y (mgr.14150) 329 : audit [DBG] from='client.24289 -' entity='client.admin' cmd=[{"prefix": "orch apply mds", "fs_name": "all", "placement": "2;smithi136=a;smithi181=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:33:18.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:18 smithi181 bash[24776]: cephadm 2024-09-12T23:33:17.232357+0000 mgr.y (mgr.14150) 330 : cephadm [INF] Saving service mds.all spec with placement smithi136=a;smithi181=b;count:2 2024-09-12T23:33:18.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:18 smithi181 bash[24776]: audit 2024-09-12T23:33:17.239578+0000 mon.a (mon.0) 621 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:18.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:18 smithi181 bash[24776]: audit 2024-09-12T23:33:17.241403+0000 mon.a (mon.0) 622 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:33:18.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:18 smithi181 bash[24776]: cluster 2024-09-12T23:33:17.485018+0000 mgr.y (mgr.14150) 331 : cluster [DBG] pgmap v306: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:18.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:18 smithi181 bash[24776]: audit 2024-09-12T23:33:17.641354+0000 mon.a (mon.0) 623 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:33:18.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:18 smithi181 bash[24776]: audit 2024-09-12T23:33:17.642844+0000 mon.a (mon.0) 624 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:33:18.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:18 smithi181 bash[24776]: audit 2024-09-12T23:33:17.652006+0000 mon.a (mon.0) 625 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:18.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:18 smithi181 bash[24776]: audit 2024-09-12T23:33:17.657090+0000 mon.a (mon.0) 626 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-12T23:33:18.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:18 smithi181 bash[24776]: audit 2024-09-12T23:33:17.663729+0000 mon.a (mon.0) 627 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-12T23:33:18.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:18 smithi181 bash[24776]: audit 2024-09-12T23:33:17.665351+0000 mon.a (mon.0) 628 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:33:18.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:18 smithi181 bash[24776]: cephadm 2024-09-12T23:33:17.666818+0000 mgr.y (mgr.14150) 332 : cephadm [INF] Deploying daemon mds.a on smithi136 2024-09-12T23:33:18.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:18 smithi181 bash[24776]: audit 2024-09-12T23:33:18.180640+0000 mon.a (mon.0) 629 : audit [DBG] from='client.? 172.21.15.136:0/1409434709' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-12T23:33:18.817 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-12T23:33:18.818 INFO:teuthology.orchestra.run.smithi136.stdout:{"epoch":1,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[]} 2024-09-12T23:33:18.818 INFO:teuthology.orchestra.run.smithi136.stderr:dumped fsmap epoch 1 2024-09-12T23:33:19.451 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:19 smithi136 bash[32189]: audit 2024-09-12T23:33:18.485142+0000 mon.c (mon.2) 18 : audit [DBG] from='client.? 172.21.15.136:0/3023171657' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-12T23:33:19.451 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:19 smithi136 bash[32189]: audit 2024-09-12T23:33:18.821053+0000 mon.a (mon.0) 630 : audit [DBG] from='client.? 172.21.15.136:0/3331786243' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-12T23:33:19.451 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:19 smithi136 bash[22601]: audit 2024-09-12T23:33:18.485142+0000 mon.c (mon.2) 18 : audit [DBG] from='client.? 172.21.15.136:0/3023171657' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-12T23:33:19.451 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:19 smithi136 bash[22601]: audit 2024-09-12T23:33:18.821053+0000 mon.a (mon.0) 630 : audit [DBG] from='client.? 172.21.15.136:0/3331786243' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-12T23:33:19.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:19 smithi181 bash[24776]: audit 2024-09-12T23:33:18.485142+0000 mon.c (mon.2) 18 : audit [DBG] from='client.? 172.21.15.136:0/3023171657' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-12T23:33:19.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:19 smithi181 bash[24776]: audit 2024-09-12T23:33:18.821053+0000 mon.a (mon.0) 630 : audit [DBG] from='client.? 172.21.15.136:0/3331786243' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-12T23:33:20.222 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:19 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:33:20.223 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:33:19 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:33:20.223 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:19 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:33:20.223 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 12 23:33:19 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:33:20.224 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 12 23:33:19 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:33:20.224 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 12 23:33:19 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:33:20.224 INFO:journalctl@ceph.mds.a.smithi136.stdout:Sep 12 23:33:20 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:33:20.224 INFO:journalctl@ceph.mds.a.smithi136.stdout:Sep 12 23:33:20 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:33:20.224 INFO:journalctl@ceph.mds.a.smithi136.stdout:Sep 12 23:33:20 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:33:20.225 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 12 23:33:19 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:33:20.507 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:33:20 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:33:20.507 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 12 23:33:20 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:33:20.507 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 12 23:33:20 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:33:20.507 INFO:journalctl@ceph.mds.a.smithi136.stdout:Sep 12 23:33:20 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:33:20.508 INFO:journalctl@ceph.mds.a.smithi136.stdout:Sep 12 23:33:20 smithi136 systemd[1]: Started Ceph mds.a for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:33:20.508 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 12 23:33:20 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:33:20.508 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 12 23:33:20 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:33:20.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:20 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:33:20.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:20 smithi136 bash[22601]: cluster 2024-09-12T23:33:19.485564+0000 mgr.y (mgr.14150) 333 : cluster [DBG] pgmap v307: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:20.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:20 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:33:20.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:20 smithi136 bash[32189]: cluster 2024-09-12T23:33:19.485564+0000 mgr.y (mgr.14150) 333 : cluster [DBG] pgmap v307: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:20.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:20 smithi181 bash[24776]: cluster 2024-09-12T23:33:19.485564+0000 mgr.y (mgr.14150) 333 : cluster [DBG] pgmap v307: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:20.835 DEBUG:teuthology.orchestra.run.smithi136:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-09-12T23:33:21.170 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-12T23:33:21.170 INFO:teuthology.orchestra.run.smithi136.stdout:{"epoch":1,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[]} 2024-09-12T23:33:21.170 INFO:teuthology.orchestra.run.smithi136.stderr:dumped fsmap epoch 1 2024-09-12T23:33:21.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:21 smithi136 bash[22601]: audit 2024-09-12T23:33:20.375641+0000 mon.a (mon.0) 631 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:21.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:21 smithi136 bash[22601]: audit 2024-09-12T23:33:20.385623+0000 mon.a (mon.0) 632 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:21.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:21 smithi136 bash[22601]: audit 2024-09-12T23:33:20.393243+0000 mon.a (mon.0) 633 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:21.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:21 smithi136 bash[22601]: audit 2024-09-12T23:33:20.393826+0000 mon.a (mon.0) 634 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-12T23:33:21.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:21 smithi136 bash[22601]: audit 2024-09-12T23:33:20.397500+0000 mon.a (mon.0) 635 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-12T23:33:21.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:21 smithi136 bash[22601]: audit 2024-09-12T23:33:20.398142+0000 mon.a (mon.0) 636 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:33:21.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:21 smithi136 bash[22601]: cephadm 2024-09-12T23:33:20.398848+0000 mgr.y (mgr.14150) 334 : cephadm [INF] Deploying daemon mds.b on smithi181 2024-09-12T23:33:21.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:21 smithi136 bash[22601]: audit 2024-09-12T23:33:21.173239+0000 mon.a (mon.0) 637 : audit [DBG] from='client.? 172.21.15.136:0/3868439501' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-12T23:33:21.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:21 smithi136 bash[32189]: audit 2024-09-12T23:33:20.375641+0000 mon.a (mon.0) 631 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:21.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:21 smithi136 bash[32189]: audit 2024-09-12T23:33:20.385623+0000 mon.a (mon.0) 632 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:21.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:21 smithi136 bash[32189]: audit 2024-09-12T23:33:20.393243+0000 mon.a (mon.0) 633 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:21.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:21 smithi136 bash[32189]: audit 2024-09-12T23:33:20.393826+0000 mon.a (mon.0) 634 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-12T23:33:21.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:21 smithi136 bash[32189]: audit 2024-09-12T23:33:20.397500+0000 mon.a (mon.0) 635 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-12T23:33:21.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:21 smithi136 bash[32189]: audit 2024-09-12T23:33:20.398142+0000 mon.a (mon.0) 636 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:33:21.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:21 smithi136 bash[32189]: cephadm 2024-09-12T23:33:20.398848+0000 mgr.y (mgr.14150) 334 : cephadm [INF] Deploying daemon mds.b on smithi181 2024-09-12T23:33:21.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:21 smithi136 bash[32189]: audit 2024-09-12T23:33:21.173239+0000 mon.a (mon.0) 637 : audit [DBG] from='client.? 172.21.15.136:0/3868439501' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-12T23:33:21.653 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:21 smithi181 bash[24776]: audit 2024-09-12T23:33:20.375641+0000 mon.a (mon.0) 631 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:21.653 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:21 smithi181 bash[24776]: audit 2024-09-12T23:33:20.385623+0000 mon.a (mon.0) 632 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:21.654 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:21 smithi181 bash[24776]: audit 2024-09-12T23:33:20.393243+0000 mon.a (mon.0) 633 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:21.654 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:21 smithi181 bash[24776]: audit 2024-09-12T23:33:20.393826+0000 mon.a (mon.0) 634 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-12T23:33:21.654 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:21 smithi181 bash[24776]: audit 2024-09-12T23:33:20.397500+0000 mon.a (mon.0) 635 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-12T23:33:21.654 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:21 smithi181 bash[24776]: audit 2024-09-12T23:33:20.398142+0000 mon.a (mon.0) 636 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:33:21.654 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:21 smithi181 bash[24776]: cephadm 2024-09-12T23:33:20.398848+0000 mgr.y (mgr.14150) 334 : cephadm [INF] Deploying daemon mds.b on smithi181 2024-09-12T23:33:21.654 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:21 smithi181 bash[24776]: audit 2024-09-12T23:33:21.173239+0000 mon.a (mon.0) 637 : audit [DBG] from='client.? 172.21.15.136:0/3868439501' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-12T23:33:22.007 INFO:journalctl@ceph.mds.a.smithi136.stdout:Sep 12 23:33:21 smithi136 bash[56511]: starting mds.a at 2024-09-12T23:33:22.499 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:22 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:33:22.500 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:33:22 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:33:22.500 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 12 23:33:22 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:33:22.500 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 12 23:33:22 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:33:22.501 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:33:22 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:33:22.501 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:33:22 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:33:22.846 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:33:22 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:33:22.847 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 12 23:33:22 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:33:22.847 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 12 23:33:22 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:33:22.848 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:33:22 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:33:22.848 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:33:22 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:33:22.848 INFO:journalctl@ceph.mds.b.smithi181.stdout:Sep 12 23:33:22 smithi181 systemd[1]: Started Ceph mds.b for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:33:22.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:22 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:33:22.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:22 smithi181 bash[24776]: cluster 2024-09-12T23:33:21.486115+0000 mgr.y (mgr.14150) 335 : cluster [DBG] pgmap v308: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:22.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:22 smithi181 bash[24776]: cluster 2024-09-12T23:33:21.652629+0000 mon.a (mon.0) 638 : cluster [DBG] mds.? [v2:172.21.15.136:6834/3940972569,v1:172.21.15.136:6835/3940972569] up:boot 2024-09-12T23:33:22.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:22 smithi181 bash[24776]: cluster 2024-09-12T23:33:21.652693+0000 mon.a (mon.0) 639 : cluster [DBG] fsmap 1 up:standby 2024-09-12T23:33:22.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:22 smithi181 bash[24776]: audit 2024-09-12T23:33:21.653286+0000 mon.a (mon.0) 640 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-12T23:33:23.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:22 smithi136 bash[22601]: cluster 2024-09-12T23:33:21.486115+0000 mgr.y (mgr.14150) 335 : cluster [DBG] pgmap v308: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:23.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:22 smithi136 bash[22601]: cluster 2024-09-12T23:33:21.652629+0000 mon.a (mon.0) 638 : cluster [DBG] mds.? [v2:172.21.15.136:6834/3940972569,v1:172.21.15.136:6835/3940972569] up:boot 2024-09-12T23:33:23.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:22 smithi136 bash[22601]: cluster 2024-09-12T23:33:21.652693+0000 mon.a (mon.0) 639 : cluster [DBG] fsmap 1 up:standby 2024-09-12T23:33:23.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:22 smithi136 bash[22601]: audit 2024-09-12T23:33:21.653286+0000 mon.a (mon.0) 640 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-12T23:33:23.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:22 smithi136 bash[32189]: cluster 2024-09-12T23:33:21.486115+0000 mgr.y (mgr.14150) 335 : cluster [DBG] pgmap v308: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:23.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:22 smithi136 bash[32189]: cluster 2024-09-12T23:33:21.652629+0000 mon.a (mon.0) 638 : cluster [DBG] mds.? [v2:172.21.15.136:6834/3940972569,v1:172.21.15.136:6835/3940972569] up:boot 2024-09-12T23:33:23.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:22 smithi136 bash[32189]: cluster 2024-09-12T23:33:21.652693+0000 mon.a (mon.0) 639 : cluster [DBG] fsmap 1 up:standby 2024-09-12T23:33:23.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:22 smithi136 bash[32189]: audit 2024-09-12T23:33:21.653286+0000 mon.a (mon.0) 640 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-12T23:33:23.189 DEBUG:teuthology.orchestra.run.smithi136:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-09-12T23:33:23.517 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-12T23:33:23.517 INFO:teuthology.orchestra.run.smithi136.stdout:{"epoch":2,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14391,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.136:6835/3940972569","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6834","nonce":3940972569},{"type":"v1","addr":"172.21.15.136:6835","nonce":3940972569}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[]} 2024-09-12T23:33:23.517 INFO:teuthology.orchestra.run.smithi136.stderr:dumped fsmap epoch 2 2024-09-12T23:33:24.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:23 smithi136 bash[22601]: audit 2024-09-12T23:33:22.786793+0000 mon.a (mon.0) 641 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:24.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:23 smithi136 bash[22601]: audit 2024-09-12T23:33:22.797671+0000 mon.a (mon.0) 642 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:24.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:23 smithi136 bash[22601]: audit 2024-09-12T23:33:22.805908+0000 mon.a (mon.0) 643 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:24.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:23 smithi136 bash[22601]: audit 2024-09-12T23:33:22.814713+0000 mon.a (mon.0) 644 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:24.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:23 smithi136 bash[22601]: audit 2024-09-12T23:33:22.822601+0000 mon.a (mon.0) 645 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:24.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:23 smithi136 bash[22601]: audit 2024-09-12T23:33:22.879333+0000 mon.a (mon.0) 646 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:33:24.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:23 smithi136 bash[22601]: audit 2024-09-12T23:33:23.520198+0000 mon.c (mon.2) 19 : audit [DBG] from='client.? 172.21.15.136:0/1239627786' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-12T23:33:24.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:23 smithi136 bash[22601]: audit 2024-09-12T23:33:23.667762+0000 mon.a (mon.0) 647 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:24.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:23 smithi136 bash[32189]: audit 2024-09-12T23:33:22.786793+0000 mon.a (mon.0) 641 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:24.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:23 smithi136 bash[32189]: audit 2024-09-12T23:33:22.797671+0000 mon.a (mon.0) 642 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:24.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:23 smithi136 bash[32189]: audit 2024-09-12T23:33:22.805908+0000 mon.a (mon.0) 643 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:24.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:23 smithi136 bash[32189]: audit 2024-09-12T23:33:22.814713+0000 mon.a (mon.0) 644 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:24.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:23 smithi136 bash[32189]: audit 2024-09-12T23:33:22.822601+0000 mon.a (mon.0) 645 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:24.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:23 smithi136 bash[32189]: audit 2024-09-12T23:33:22.879333+0000 mon.a (mon.0) 646 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:33:24.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:23 smithi136 bash[32189]: audit 2024-09-12T23:33:23.520198+0000 mon.c (mon.2) 19 : audit [DBG] from='client.? 172.21.15.136:0/1239627786' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-12T23:33:24.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:23 smithi136 bash[32189]: audit 2024-09-12T23:33:23.667762+0000 mon.a (mon.0) 647 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:24.097 INFO:journalctl@ceph.mds.b.smithi181.stdout:Sep 12 23:33:23 smithi181 bash[48042]: starting mds.b at 2024-09-12T23:33:24.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:23 smithi181 bash[24776]: audit 2024-09-12T23:33:22.786793+0000 mon.a (mon.0) 641 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:24.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:23 smithi181 bash[24776]: audit 2024-09-12T23:33:22.797671+0000 mon.a (mon.0) 642 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:24.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:23 smithi181 bash[24776]: audit 2024-09-12T23:33:22.805908+0000 mon.a (mon.0) 643 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:24.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:23 smithi181 bash[24776]: audit 2024-09-12T23:33:22.814713+0000 mon.a (mon.0) 644 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:24.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:23 smithi181 bash[24776]: audit 2024-09-12T23:33:22.822601+0000 mon.a (mon.0) 645 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:24.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:23 smithi181 bash[24776]: audit 2024-09-12T23:33:22.879333+0000 mon.a (mon.0) 646 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:33:24.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:23 smithi181 bash[24776]: audit 2024-09-12T23:33:23.520198+0000 mon.c (mon.2) 19 : audit [DBG] from='client.? 172.21.15.136:0/1239627786' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-12T23:33:24.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:23 smithi181 bash[24776]: audit 2024-09-12T23:33:23.667762+0000 mon.a (mon.0) 647 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:25.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:24 smithi181 bash[24776]: cluster 2024-09-12T23:33:23.486629+0000 mgr.y (mgr.14150) 336 : cluster [DBG] pgmap v309: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:25.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:24 smithi181 bash[24776]: cluster 2024-09-12T23:33:24.672087+0000 mon.a (mon.0) 648 : cluster [DBG] mds.? [v2:172.21.15.181:6832/2077070284,v1:172.21.15.181:6833/2077070284] up:boot 2024-09-12T23:33:25.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:24 smithi181 bash[24776]: cluster 2024-09-12T23:33:24.672181+0000 mon.a (mon.0) 649 : cluster [DBG] fsmap 2 up:standby 2024-09-12T23:33:25.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:24 smithi181 bash[24776]: audit 2024-09-12T23:33:24.672412+0000 mon.a (mon.0) 650 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-12T23:33:25.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:24 smithi136 bash[22601]: cluster 2024-09-12T23:33:23.486629+0000 mgr.y (mgr.14150) 336 : cluster [DBG] pgmap v309: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:25.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:24 smithi136 bash[22601]: cluster 2024-09-12T23:33:24.672087+0000 mon.a (mon.0) 648 : cluster [DBG] mds.? [v2:172.21.15.181:6832/2077070284,v1:172.21.15.181:6833/2077070284] up:boot 2024-09-12T23:33:25.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:24 smithi136 bash[22601]: cluster 2024-09-12T23:33:24.672181+0000 mon.a (mon.0) 649 : cluster [DBG] fsmap 2 up:standby 2024-09-12T23:33:25.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:24 smithi136 bash[22601]: audit 2024-09-12T23:33:24.672412+0000 mon.a (mon.0) 650 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-12T23:33:25.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:24 smithi136 bash[32189]: cluster 2024-09-12T23:33:23.486629+0000 mgr.y (mgr.14150) 336 : cluster [DBG] pgmap v309: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:25.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:24 smithi136 bash[32189]: cluster 2024-09-12T23:33:24.672087+0000 mon.a (mon.0) 648 : cluster [DBG] mds.? [v2:172.21.15.181:6832/2077070284,v1:172.21.15.181:6833/2077070284] up:boot 2024-09-12T23:33:25.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:24 smithi136 bash[32189]: cluster 2024-09-12T23:33:24.672181+0000 mon.a (mon.0) 649 : cluster [DBG] fsmap 2 up:standby 2024-09-12T23:33:25.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:24 smithi136 bash[32189]: audit 2024-09-12T23:33:24.672412+0000 mon.a (mon.0) 650 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-12T23:33:25.535 DEBUG:teuthology.orchestra.run.smithi136:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-09-12T23:33:25.868 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-12T23:33:25.868 INFO:teuthology.orchestra.run.smithi136.stdout:{"epoch":3,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14391,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.136:6835/3940972569","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6834","nonce":3940972569},{"type":"v1","addr":"172.21.15.136:6835","nonce":3940972569}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2},{"gid":24316,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.181:6833/2077070284","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6832","nonce":2077070284},{"type":"v1","addr":"172.21.15.181:6833","nonce":2077070284}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3}],"filesystems":[]} 2024-09-12T23:33:25.868 INFO:teuthology.orchestra.run.smithi136.stderr:dumped fsmap epoch 3 2024-09-12T23:33:25.885 DEBUG:teuthology.orchestra.run.smithi136:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-12T23:33:26.178 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-12T23:33:26.178 INFO:teuthology.orchestra.run.smithi136.stdout:{"epoch":50,"fsid":"da58a47c-715d-11ef-bceb-c7b262605968","created":"2024-09-12T23:23:29.000926+0000","modified":"2024-09-12T23:32:45.871955+0000","last_up_change":"2024-09-12T23:32:43.859472+0000","last_in_change":"2024-09-12T23:32:18.995528+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-12T23:28:33.956985+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"87aedf04-695e-4a8f-80c3-ffd18883a4d5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6802","nonce":4251571952},{"type":"v1","addr":"172.21.15.136:6803","nonce":4251571952}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6804","nonce":4251571952},{"type":"v1","addr":"172.21.15.136:6805","nonce":4251571952}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6808","nonce":4251571952},{"type":"v1","addr":"172.21.15.136:6809","nonce":4251571952}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6806","nonce":4251571952},{"type":"v1","addr":"172.21.15.136:6807","nonce":4251571952}]},"public_addr":"172.21.15.136:6803/4251571952","cluster_addr":"172.21.15.136:6805/4251571952","heartbeat_back_addr":"172.21.15.136:6809/4251571952","heartbeat_front_addr":"172.21.15.136:6807/4251571952","state":["exists","up"]},{"osd":1,"uuid":"850d4f98-0640-4151-bd45-a351ca41d54e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6810","nonce":3482177958},{"type":"v1","addr":"172.21.15.136:6811","nonce":3482177958}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6812","nonce":3482177958},{"type":"v1","addr":"172.21.15.136:6813","nonce":3482177958}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6816","nonce":3482177958},{"type":"v1","addr":"172.21.15.136:6817","nonce":3482177958}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6814","nonce":3482177958},{"type":"v1","addr":"172.21.15.136:6815","nonce":3482177958}]},"public_addr":"172.21.15.136:6811/3482177958","cluster_addr":"172.21.15.136:6813/3482177958","heartbeat_back_addr":"172.21.15.136:6817/3482177958","heartbeat_front_addr":"172.21.15.136:6815/3482177958","state":["exists","up"]},{"osd":2,"uuid":"bde707d6-e027-4b63-9853-fd59212eea1e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6818","nonce":308477304},{"type":"v1","addr":"172.21.15.136:6819","nonce":308477304}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6820","nonce":308477304},{"type":"v1","addr":"172.21.15.136:6821","nonce":308477304}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6824","nonce":308477304},{"type":"v1","addr":"172.21.15.136:6825","nonce":308477304}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6822","nonce":308477304},{"type":"v1","addr":"172.21.15.136:6823","nonce":308477304}]},"public_addr":"172.21.15.136:6819/308477304","cluster_addr":"172.21.15.136:6821/308477304","heartbeat_back_addr":"172.21.15.136:6825/308477304","heartbeat_front_addr":"172.21.15.136:6823/308477304","state":["exists","up"]},{"osd":3,"uuid":"d61f8357-b89b-4c05-b366-65a9f788064d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6826","nonce":201948269},{"type":"v1","addr":"172.21.15.136:6827","nonce":201948269}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6828","nonce":201948269},{"type":"v1","addr":"172.21.15.136:6829","nonce":201948269}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6832","nonce":201948269},{"type":"v1","addr":"172.21.15.136:6833","nonce":201948269}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6830","nonce":201948269},{"type":"v1","addr":"172.21.15.136:6831","nonce":201948269}]},"public_addr":"172.21.15.136:6827/201948269","cluster_addr":"172.21.15.136:6829/201948269","heartbeat_back_addr":"172.21.15.136:6833/201948269","heartbeat_front_addr":"172.21.15.136:6831/201948269","state":["exists","up"]},{"osd":4,"uuid":"98b0e81c-824e-4dbc-9faa-2de9f199af1b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6800","nonce":775251087},{"type":"v1","addr":"172.21.15.181:6801","nonce":775251087}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6802","nonce":775251087},{"type":"v1","addr":"172.21.15.181:6803","nonce":775251087}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6806","nonce":775251087},{"type":"v1","addr":"172.21.15.181:6807","nonce":775251087}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6804","nonce":775251087},{"type":"v1","addr":"172.21.15.181:6805","nonce":775251087}]},"public_addr":"172.21.15.181:6801/775251087","cluster_addr":"172.21.15.181:6803/775251087","heartbeat_back_addr":"172.21.15.181:6807/775251087","heartbeat_front_addr":"172.21.15.181:6805/775251087","state":["exists","up"]},{"osd":5,"uuid":"5c6d2f69-7f9c-4689-95b0-bd4eb858956d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":37,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6808","nonce":2109931558},{"type":"v1","addr":"172.21.15.181:6809","nonce":2109931558}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6810","nonce":2109931558},{"type":"v1","addr":"172.21.15.181:6811","nonce":2109931558}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6814","nonce":2109931558},{"type":"v1","addr":"172.21.15.181:6815","nonce":2109931558}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6812","nonce":2109931558},{"type":"v1","addr":"172.21.15.181:6813","nonce":2109931558}]},"public_addr":"172.21.15.181:6809/2109931558","cluster_addr":"172.21.15.181:6811/2109931558","heartbeat_back_addr":"172.21.15.181:6815/2109931558","heartbeat_front_addr":"172.21.15.181:6813/2109931558","state":["exists","up"]},{"osd":6,"uuid":"532ee052-af15-433a-8e93-b0b642aaf85b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6816","nonce":4099148413},{"type":"v1","addr":"172.21.15.181:6817","nonce":4099148413}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6818","nonce":4099148413},{"type":"v1","addr":"172.21.15.181:6819","nonce":4099148413}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6822","nonce":4099148413},{"type":"v1","addr":"172.21.15.181:6823","nonce":4099148413}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6820","nonce":4099148413},{"type":"v1","addr":"172.21.15.181:6821","nonce":4099148413}]},"public_addr":"172.21.15.181:6817/4099148413","cluster_addr":"172.21.15.181:6819/4099148413","heartbeat_back_addr":"172.21.15.181:6823/4099148413","heartbeat_front_addr":"172.21.15.181:6821/4099148413","state":["exists","up"]},{"osd":7,"uuid":"d2ff80f0-6ef0-4098-877c-a891d48c687d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6824","nonce":783620875},{"type":"v1","addr":"172.21.15.181:6825","nonce":783620875}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6826","nonce":783620875},{"type":"v1","addr":"172.21.15.181:6827","nonce":783620875}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6830","nonce":783620875},{"type":"v1","addr":"172.21.15.181:6831","nonce":783620875}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6828","nonce":783620875},{"type":"v1","addr":"172.21.15.181:6829","nonce":783620875}]},"public_addr":"172.21.15.181:6825/783620875","cluster_addr":"172.21.15.181:6827/783620875","heartbeat_back_addr":"172.21.15.181:6831/783620875","heartbeat_front_addr":"172.21.15.181:6829/783620875","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:26:50.657580+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:27:40.536476+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:28:31.755012+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:29:23.585451+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:30:11.888258+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:31:01.812765+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:31:52.166696+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:32:42.696525+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.136:0/4260962972":"2024-09-13T23:24:25.346823+0000","172.21.15.136:6801/3594272622":"2024-09-13T23:24:25.346823+0000","172.21.15.136:0/209539205":"2024-09-13T23:24:25.346823+0000","172.21.15.136:6800/3594272622":"2024-09-13T23:24:25.346823+0000","172.21.15.136:0/1549678324":"2024-09-13T23:23:54.847045+0000","172.21.15.136:0/3376519841":"2024-09-13T23:23:54.847045+0000","172.21.15.136:0/3910468981":"2024-09-13T23:23:54.847045+0000","172.21.15.136:6801/2894181186":"2024-09-13T23:23:54.847045+0000","172.21.15.136:0/1964283041":"2024-09-13T23:24:25.346823+0000","172.21.15.136:6800/2894181186":"2024-09-13T23:23:54.847045+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-12T23:33:26.196 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-09-12T23:28:33.956985+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-09-12T23:33:26.196 DEBUG:teuthology.orchestra.run.smithi136:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-09-12T23:33:26.475 INFO:teuthology.orchestra.run.smithi136.stdout:pg_num: 1 2024-09-12T23:33:26.492 DEBUG:teuthology.orchestra.run.smithi136:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --format=json-pretty osd lspools 2024-09-12T23:33:26.776 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-12T23:33:26.776 INFO:teuthology.orchestra.run.smithi136.stdout:[ 2024-09-12T23:33:26.776 INFO:teuthology.orchestra.run.smithi136.stdout: { 2024-09-12T23:33:26.776 INFO:teuthology.orchestra.run.smithi136.stdout: "poolnum": 1, 2024-09-12T23:33:26.776 INFO:teuthology.orchestra.run.smithi136.stdout: "poolname": ".mgr" 2024-09-12T23:33:26.776 INFO:teuthology.orchestra.run.smithi136.stdout: } 2024-09-12T23:33:26.777 INFO:teuthology.orchestra.run.smithi136.stdout:] 2024-09-12T23:33:26.794 DEBUG:tasks.cephfs.filesystem:Creating filesystem 'cephfs' 2024-09-12T23:33:26.794 DEBUG:teuthology.orchestra.run.smithi136:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_metadata --pg_num_min 64 2024-09-12T23:33:27.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:26 smithi181 bash[24776]: cluster 2024-09-12T23:33:25.487186+0000 mgr.y (mgr.14150) 337 : cluster [DBG] pgmap v310: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:27.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:26 smithi181 bash[24776]: audit 2024-09-12T23:33:25.871207+0000 mon.a (mon.0) 651 : audit [DBG] from='client.? 172.21.15.136:0/2082478619' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-12T23:33:27.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:26 smithi181 bash[24776]: audit 2024-09-12T23:33:26.181305+0000 mon.a (mon.0) 652 : audit [DBG] from='client.? 172.21.15.136:0/2928152715' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-12T23:33:27.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:26 smithi181 bash[24776]: audit 2024-09-12T23:33:26.478733+0000 mon.a (mon.0) 653 : audit [DBG] from='client.? 172.21.15.136:0/3489433618' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-12T23:33:27.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:26 smithi181 bash[24776]: audit 2024-09-12T23:33:26.779723+0000 mon.a (mon.0) 654 : audit [DBG] from='client.? 172.21.15.136:0/3745527403' entity='client.admin' cmd=[{"prefix": "osd lspools", "format": "json-pretty"}]: dispatch 2024-09-12T23:33:27.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:26 smithi136 bash[22601]: cluster 2024-09-12T23:33:25.487186+0000 mgr.y (mgr.14150) 337 : cluster [DBG] pgmap v310: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:27.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:26 smithi136 bash[22601]: audit 2024-09-12T23:33:25.871207+0000 mon.a (mon.0) 651 : audit [DBG] from='client.? 172.21.15.136:0/2082478619' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-12T23:33:27.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:26 smithi136 bash[22601]: audit 2024-09-12T23:33:26.181305+0000 mon.a (mon.0) 652 : audit [DBG] from='client.? 172.21.15.136:0/2928152715' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-12T23:33:27.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:26 smithi136 bash[22601]: audit 2024-09-12T23:33:26.478733+0000 mon.a (mon.0) 653 : audit [DBG] from='client.? 172.21.15.136:0/3489433618' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-12T23:33:27.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:26 smithi136 bash[22601]: audit 2024-09-12T23:33:26.779723+0000 mon.a (mon.0) 654 : audit [DBG] from='client.? 172.21.15.136:0/3745527403' entity='client.admin' cmd=[{"prefix": "osd lspools", "format": "json-pretty"}]: dispatch 2024-09-12T23:33:27.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:26 smithi136 bash[32189]: cluster 2024-09-12T23:33:25.487186+0000 mgr.y (mgr.14150) 337 : cluster [DBG] pgmap v310: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:27.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:26 smithi136 bash[32189]: audit 2024-09-12T23:33:25.871207+0000 mon.a (mon.0) 651 : audit [DBG] from='client.? 172.21.15.136:0/2082478619' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-12T23:33:27.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:26 smithi136 bash[32189]: audit 2024-09-12T23:33:26.181305+0000 mon.a (mon.0) 652 : audit [DBG] from='client.? 172.21.15.136:0/2928152715' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-12T23:33:27.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:26 smithi136 bash[32189]: audit 2024-09-12T23:33:26.478733+0000 mon.a (mon.0) 653 : audit [DBG] from='client.? 172.21.15.136:0/3489433618' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-12T23:33:27.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:26 smithi136 bash[32189]: audit 2024-09-12T23:33:26.779723+0000 mon.a (mon.0) 654 : audit [DBG] from='client.? 172.21.15.136:0/3745527403' entity='client.admin' cmd=[{"prefix": "osd lspools", "format": "json-pretty"}]: dispatch 2024-09-12T23:33:27.834 INFO:teuthology.orchestra.run.smithi136.stderr:pool 'cephfs_metadata' created 2024-09-12T23:33:27.857 DEBUG:teuthology.orchestra.run.smithi136:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_data 64 --pg_num_min 64 --target_size_ratio 0.9 2024-09-12T23:33:28.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:27 smithi181 bash[24776]: audit 2024-09-12T23:33:27.074437+0000 mon.a (mon.0) 655 : audit [INF] from='client.? 172.21.15.136:0/561597242' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2024-09-12T23:33:28.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:27 smithi136 bash[22601]: audit 2024-09-12T23:33:27.074437+0000 mon.a (mon.0) 655 : audit [INF] from='client.? 172.21.15.136:0/561597242' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2024-09-12T23:33:28.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:27 smithi136 bash[32189]: audit 2024-09-12T23:33:27.074437+0000 mon.a (mon.0) 655 : audit [INF] from='client.? 172.21.15.136:0/561597242' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2024-09-12T23:33:28.849 INFO:teuthology.orchestra.run.smithi136.stderr:pool 'cephfs_data' created 2024-09-12T23:33:28.885 DEBUG:teuthology.orchestra.run.smithi136:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs new cephfs cephfs_metadata cephfs_data 2024-09-12T23:33:29.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:28 smithi136 bash[22601]: cluster 2024-09-12T23:33:27.487705+0000 mgr.y (mgr.14150) 338 : cluster [DBG] pgmap v311: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:29.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:28 smithi136 bash[22601]: audit 2024-09-12T23:33:27.838038+0000 mon.a (mon.0) 656 : audit [INF] from='client.? 172.21.15.136:0/561597242' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]': finished 2024-09-12T23:33:29.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:28 smithi136 bash[22601]: cluster 2024-09-12T23:33:27.838114+0000 mon.a (mon.0) 657 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2024-09-12T23:33:29.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:28 smithi136 bash[22601]: audit 2024-09-12T23:33:28.026292+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:29.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:28 smithi136 bash[22601]: audit 2024-09-12T23:33:28.031594+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:29.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:28 smithi136 bash[22601]: audit 2024-09-12T23:33:28.036603+0000 mon.a (mon.0) 660 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:29.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:28 smithi136 bash[22601]: audit 2024-09-12T23:33:28.041136+0000 mon.a (mon.0) 661 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:29.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:28 smithi136 bash[22601]: audit 2024-09-12T23:33:28.042009+0000 mon.a (mon.0) 662 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:33:29.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:28 smithi136 bash[22601]: audit 2024-09-12T23:33:28.042711+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:33:29.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:28 smithi136 bash[22601]: audit 2024-09-12T23:33:28.047417+0000 mon.a (mon.0) 664 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:29.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:28 smithi136 bash[22601]: audit 2024-09-12T23:33:28.076648+0000 mon.a (mon.0) 665 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:33:29.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:28 smithi136 bash[22601]: audit 2024-09-12T23:33:28.077871+0000 mon.a (mon.0) 666 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:33:29.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:28 smithi136 bash[22601]: audit 2024-09-12T23:33:28.078505+0000 mon.a (mon.0) 667 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:33:29.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:28 smithi136 bash[22601]: audit 2024-09-12T23:33:28.084082+0000 mon.a (mon.0) 668 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:29.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:28 smithi136 bash[22601]: audit 2024-09-12T23:33:28.147999+0000 mon.c (mon.2) 20 : audit [INF] from='client.? 172.21.15.136:0/330359329' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2024-09-12T23:33:29.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:28 smithi136 bash[22601]: audit 2024-09-12T23:33:28.148431+0000 mon.a (mon.0) 669 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2024-09-12T23:33:29.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:28 smithi136 bash[32189]: cluster 2024-09-12T23:33:27.487705+0000 mgr.y (mgr.14150) 338 : cluster [DBG] pgmap v311: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:29.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:28 smithi136 bash[32189]: audit 2024-09-12T23:33:27.838038+0000 mon.a (mon.0) 656 : audit [INF] from='client.? 172.21.15.136:0/561597242' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]': finished 2024-09-12T23:33:29.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:28 smithi136 bash[32189]: cluster 2024-09-12T23:33:27.838114+0000 mon.a (mon.0) 657 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2024-09-12T23:33:29.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:28 smithi136 bash[32189]: audit 2024-09-12T23:33:28.026292+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:29.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:28 smithi136 bash[32189]: audit 2024-09-12T23:33:28.031594+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:29.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:28 smithi136 bash[32189]: audit 2024-09-12T23:33:28.036603+0000 mon.a (mon.0) 660 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:29.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:28 smithi136 bash[32189]: audit 2024-09-12T23:33:28.041136+0000 mon.a (mon.0) 661 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:29.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:28 smithi136 bash[32189]: audit 2024-09-12T23:33:28.042009+0000 mon.a (mon.0) 662 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:33:29.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:28 smithi136 bash[32189]: audit 2024-09-12T23:33:28.042711+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:33:29.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:28 smithi136 bash[32189]: audit 2024-09-12T23:33:28.047417+0000 mon.a (mon.0) 664 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:29.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:28 smithi136 bash[32189]: audit 2024-09-12T23:33:28.076648+0000 mon.a (mon.0) 665 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:33:29.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:28 smithi136 bash[32189]: audit 2024-09-12T23:33:28.077871+0000 mon.a (mon.0) 666 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:33:29.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:28 smithi136 bash[32189]: audit 2024-09-12T23:33:28.078505+0000 mon.a (mon.0) 667 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:33:29.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:28 smithi136 bash[32189]: audit 2024-09-12T23:33:28.084082+0000 mon.a (mon.0) 668 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:29.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:28 smithi136 bash[32189]: audit 2024-09-12T23:33:28.147999+0000 mon.c (mon.2) 20 : audit [INF] from='client.? 172.21.15.136:0/330359329' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2024-09-12T23:33:29.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:28 smithi136 bash[32189]: audit 2024-09-12T23:33:28.148431+0000 mon.a (mon.0) 669 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2024-09-12T23:33:29.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:28 smithi181 bash[24776]: cluster 2024-09-12T23:33:27.487705+0000 mgr.y (mgr.14150) 338 : cluster [DBG] pgmap v311: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:29.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:28 smithi181 bash[24776]: audit 2024-09-12T23:33:27.838038+0000 mon.a (mon.0) 656 : audit [INF] from='client.? 172.21.15.136:0/561597242' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]': finished 2024-09-12T23:33:29.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:28 smithi181 bash[24776]: cluster 2024-09-12T23:33:27.838114+0000 mon.a (mon.0) 657 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2024-09-12T23:33:29.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:28 smithi181 bash[24776]: audit 2024-09-12T23:33:28.026292+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:29.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:28 smithi181 bash[24776]: audit 2024-09-12T23:33:28.031594+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:29.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:28 smithi181 bash[24776]: audit 2024-09-12T23:33:28.036603+0000 mon.a (mon.0) 660 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:29.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:28 smithi181 bash[24776]: audit 2024-09-12T23:33:28.041136+0000 mon.a (mon.0) 661 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:29.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:28 smithi181 bash[24776]: audit 2024-09-12T23:33:28.042009+0000 mon.a (mon.0) 662 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:33:29.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:28 smithi181 bash[24776]: audit 2024-09-12T23:33:28.042711+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:33:29.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:28 smithi181 bash[24776]: audit 2024-09-12T23:33:28.047417+0000 mon.a (mon.0) 664 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:29.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:28 smithi181 bash[24776]: audit 2024-09-12T23:33:28.076648+0000 mon.a (mon.0) 665 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:33:29.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:28 smithi181 bash[24776]: audit 2024-09-12T23:33:28.077871+0000 mon.a (mon.0) 666 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:33:29.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:28 smithi181 bash[24776]: audit 2024-09-12T23:33:28.078505+0000 mon.a (mon.0) 667 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:33:29.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:28 smithi181 bash[24776]: audit 2024-09-12T23:33:28.084082+0000 mon.a (mon.0) 668 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:29.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:28 smithi181 bash[24776]: audit 2024-09-12T23:33:28.147999+0000 mon.c (mon.2) 20 : audit [INF] from='client.? 172.21.15.136:0/330359329' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2024-09-12T23:33:29.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:28 smithi181 bash[24776]: audit 2024-09-12T23:33:28.148431+0000 mon.a (mon.0) 669 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2024-09-12T23:33:29.885 INFO:teuthology.orchestra.run.smithi136.stderr:new fs with metadata pool 2 and data pool 3 2024-09-12T23:33:29.926 DEBUG:teuthology.orchestra.run.smithi136:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-12T23:33:30.187 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-12T23:33:30.187 INFO:teuthology.orchestra.run.smithi136.stdout:{"epoch":53,"fsid":"da58a47c-715d-11ef-bceb-c7b262605968","created":"2024-09-12T23:23:29.000926+0000","modified":"2024-09-12T23:33:29.191548+0000","last_up_change":"2024-09-12T23:32:43.859472+0000","last_in_change":"2024-09-12T23:32:18.995528+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-12T23:28:33.956985+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-09-12T23:33:27.478434+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-09-12T23:33:28.566349+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"87aedf04-695e-4a8f-80c3-ffd18883a4d5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6802","nonce":4251571952},{"type":"v1","addr":"172.21.15.136:6803","nonce":4251571952}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6804","nonce":4251571952},{"type":"v1","addr":"172.21.15.136:6805","nonce":4251571952}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6808","nonce":4251571952},{"type":"v1","addr":"172.21.15.136:6809","nonce":4251571952}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6806","nonce":4251571952},{"type":"v1","addr":"172.21.15.136:6807","nonce":4251571952}]},"public_addr":"172.21.15.136:6803/4251571952","cluster_addr":"172.21.15.136:6805/4251571952","heartbeat_back_addr":"172.21.15.136:6809/4251571952","heartbeat_front_addr":"172.21.15.136:6807/4251571952","state":["exists","up"]},{"osd":1,"uuid":"850d4f98-0640-4151-bd45-a351ca41d54e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6810","nonce":3482177958},{"type":"v1","addr":"172.21.15.136:6811","nonce":3482177958}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6812","nonce":3482177958},{"type":"v1","addr":"172.21.15.136:6813","nonce":3482177958}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6816","nonce":3482177958},{"type":"v1","addr":"172.21.15.136:6817","nonce":3482177958}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6814","nonce":3482177958},{"type":"v1","addr":"172.21.15.136:6815","nonce":3482177958}]},"public_addr":"172.21.15.136:6811/3482177958","cluster_addr":"172.21.15.136:6813/3482177958","heartbeat_back_addr":"172.21.15.136:6817/3482177958","heartbeat_front_addr":"172.21.15.136:6815/3482177958","state":["exists","up"]},{"osd":2,"uuid":"bde707d6-e027-4b63-9853-fd59212eea1e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6818","nonce":308477304},{"type":"v1","addr":"172.21.15.136:6819","nonce":308477304}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6820","nonce":308477304},{"type":"v1","addr":"172.21.15.136:6821","nonce":308477304}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6824","nonce":308477304},{"type":"v1","addr":"172.21.15.136:6825","nonce":308477304}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6822","nonce":308477304},{"type":"v1","addr":"172.21.15.136:6823","nonce":308477304}]},"public_addr":"172.21.15.136:6819/308477304","cluster_addr":"172.21.15.136:6821/308477304","heartbeat_back_addr":"172.21.15.136:6825/308477304","heartbeat_front_addr":"172.21.15.136:6823/308477304","state":["exists","up"]},{"osd":3,"uuid":"d61f8357-b89b-4c05-b366-65a9f788064d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6826","nonce":201948269},{"type":"v1","addr":"172.21.15.136:6827","nonce":201948269}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6828","nonce":201948269},{"type":"v1","addr":"172.21.15.136:6829","nonce":201948269}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6832","nonce":201948269},{"type":"v1","addr":"172.21.15.136:6833","nonce":201948269}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6830","nonce":201948269},{"type":"v1","addr":"172.21.15.136:6831","nonce":201948269}]},"public_addr":"172.21.15.136:6827/201948269","cluster_addr":"172.21.15.136:6829/201948269","heartbeat_back_addr":"172.21.15.136:6833/201948269","heartbeat_front_addr":"172.21.15.136:6831/201948269","state":["exists","up"]},{"osd":4,"uuid":"98b0e81c-824e-4dbc-9faa-2de9f199af1b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6800","nonce":775251087},{"type":"v1","addr":"172.21.15.181:6801","nonce":775251087}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6802","nonce":775251087},{"type":"v1","addr":"172.21.15.181:6803","nonce":775251087}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6806","nonce":775251087},{"type":"v1","addr":"172.21.15.181:6807","nonce":775251087}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6804","nonce":775251087},{"type":"v1","addr":"172.21.15.181:6805","nonce":775251087}]},"public_addr":"172.21.15.181:6801/775251087","cluster_addr":"172.21.15.181:6803/775251087","heartbeat_back_addr":"172.21.15.181:6807/775251087","heartbeat_front_addr":"172.21.15.181:6805/775251087","state":["exists","up"]},{"osd":5,"uuid":"5c6d2f69-7f9c-4689-95b0-bd4eb858956d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6808","nonce":2109931558},{"type":"v1","addr":"172.21.15.181:6809","nonce":2109931558}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6810","nonce":2109931558},{"type":"v1","addr":"172.21.15.181:6811","nonce":2109931558}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6814","nonce":2109931558},{"type":"v1","addr":"172.21.15.181:6815","nonce":2109931558}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6812","nonce":2109931558},{"type":"v1","addr":"172.21.15.181:6813","nonce":2109931558}]},"public_addr":"172.21.15.181:6809/2109931558","cluster_addr":"172.21.15.181:6811/2109931558","heartbeat_back_addr":"172.21.15.181:6815/2109931558","heartbeat_front_addr":"172.21.15.181:6813/2109931558","state":["exists","up"]},{"osd":6,"uuid":"532ee052-af15-433a-8e93-b0b642aaf85b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6816","nonce":4099148413},{"type":"v1","addr":"172.21.15.181:6817","nonce":4099148413}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6818","nonce":4099148413},{"type":"v1","addr":"172.21.15.181:6819","nonce":4099148413}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6822","nonce":4099148413},{"type":"v1","addr":"172.21.15.181:6823","nonce":4099148413}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6820","nonce":4099148413},{"type":"v1","addr":"172.21.15.181:6821","nonce":4099148413}]},"public_addr":"172.21.15.181:6817/4099148413","cluster_addr":"172.21.15.181:6819/4099148413","heartbeat_back_addr":"172.21.15.181:6823/4099148413","heartbeat_front_addr":"172.21.15.181:6821/4099148413","state":["exists","up"]},{"osd":7,"uuid":"d2ff80f0-6ef0-4098-877c-a891d48c687d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6824","nonce":783620875},{"type":"v1","addr":"172.21.15.181:6825","nonce":783620875}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6826","nonce":783620875},{"type":"v1","addr":"172.21.15.181:6827","nonce":783620875}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6830","nonce":783620875},{"type":"v1","addr":"172.21.15.181:6831","nonce":783620875}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6828","nonce":783620875},{"type":"v1","addr":"172.21.15.181:6829","nonce":783620875}]},"public_addr":"172.21.15.181:6825/783620875","cluster_addr":"172.21.15.181:6827/783620875","heartbeat_back_addr":"172.21.15.181:6831/783620875","heartbeat_front_addr":"172.21.15.181:6829/783620875","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:26:50.657580+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:27:40.536476+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:28:31.755012+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:29:23.585451+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:30:11.888258+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:31:01.812765+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:31:52.166696+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:32:42.696525+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.136:0/4260962972":"2024-09-13T23:24:25.346823+0000","172.21.15.136:6801/3594272622":"2024-09-13T23:24:25.346823+0000","172.21.15.136:0/209539205":"2024-09-13T23:24:25.346823+0000","172.21.15.136:6800/3594272622":"2024-09-13T23:24:25.346823+0000","172.21.15.136:0/1549678324":"2024-09-13T23:23:54.847045+0000","172.21.15.136:0/3376519841":"2024-09-13T23:23:54.847045+0000","172.21.15.136:0/3910468981":"2024-09-13T23:23:54.847045+0000","172.21.15.136:6801/2894181186":"2024-09-13T23:23:54.847045+0000","172.21.15.136:0/1964283041":"2024-09-13T23:24:25.346823+0000","172.21.15.136:6800/2894181186":"2024-09-13T23:23:54.847045+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-12T23:33:30.204 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:29 smithi136 bash[32189]: audit 2024-09-12T23:33:28.846868+0000 mon.a (mon.0) 670 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]': finished 2024-09-12T23:33:30.204 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:29 smithi136 bash[32189]: cluster 2024-09-12T23:33:28.847565+0000 mon.a (mon.0) 671 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2024-09-12T23:33:30.204 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:29 smithi136 bash[32189]: audit 2024-09-12T23:33:29.190652+0000 mon.c (mon.2) 21 : audit [INF] from='client.? 172.21.15.136:0/1696354757' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2024-09-12T23:33:30.204 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:29 smithi136 bash[32189]: audit 2024-09-12T23:33:29.191345+0000 mon.a (mon.0) 672 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2024-09-12T23:33:30.204 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:29 smithi136 bash[32189]: cluster 2024-09-12T23:33:29.192410+0000 mon.a (mon.0) 673 : cluster [INF] daemon mds.b assigned to filesystem cephfs as rank 0 2024-09-12T23:33:30.205 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:29 smithi136 bash[22601]: audit 2024-09-12T23:33:28.846868+0000 mon.a (mon.0) 670 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]': finished 2024-09-12T23:33:30.205 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:29 smithi136 bash[22601]: cluster 2024-09-12T23:33:28.847565+0000 mon.a (mon.0) 671 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2024-09-12T23:33:30.205 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:29 smithi136 bash[22601]: audit 2024-09-12T23:33:29.190652+0000 mon.c (mon.2) 21 : audit [INF] from='client.? 172.21.15.136:0/1696354757' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2024-09-12T23:33:30.205 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:29 smithi136 bash[22601]: audit 2024-09-12T23:33:29.191345+0000 mon.a (mon.0) 672 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2024-09-12T23:33:30.205 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:29 smithi136 bash[22601]: cluster 2024-09-12T23:33:29.192410+0000 mon.a (mon.0) 673 : cluster [INF] daemon mds.b assigned to filesystem cephfs as rank 0 2024-09-12T23:33:30.206 DEBUG:teuthology.orchestra.run.smithi136:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-12T23:33:30.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:29 smithi181 bash[24776]: audit 2024-09-12T23:33:28.846868+0000 mon.a (mon.0) 670 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]': finished 2024-09-12T23:33:30.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:29 smithi181 bash[24776]: cluster 2024-09-12T23:33:28.847565+0000 mon.a (mon.0) 671 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2024-09-12T23:33:30.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:29 smithi181 bash[24776]: audit 2024-09-12T23:33:29.190652+0000 mon.c (mon.2) 21 : audit [INF] from='client.? 172.21.15.136:0/1696354757' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2024-09-12T23:33:30.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:29 smithi181 bash[24776]: audit 2024-09-12T23:33:29.191345+0000 mon.a (mon.0) 672 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2024-09-12T23:33:30.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:29 smithi181 bash[24776]: cluster 2024-09-12T23:33:29.192410+0000 mon.a (mon.0) 673 : cluster [INF] daemon mds.b assigned to filesystem cephfs as rank 0 2024-09-12T23:33:30.484 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-12T23:33:30.485 INFO:teuthology.orchestra.run.smithi136.stdout:{"epoch":53,"fsid":"da58a47c-715d-11ef-bceb-c7b262605968","created":"2024-09-12T23:23:29.000926+0000","modified":"2024-09-12T23:33:29.191548+0000","last_up_change":"2024-09-12T23:32:43.859472+0000","last_in_change":"2024-09-12T23:32:18.995528+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-12T23:28:33.956985+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-09-12T23:33:27.478434+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-09-12T23:33:28.566349+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"87aedf04-695e-4a8f-80c3-ffd18883a4d5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6802","nonce":4251571952},{"type":"v1","addr":"172.21.15.136:6803","nonce":4251571952}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6804","nonce":4251571952},{"type":"v1","addr":"172.21.15.136:6805","nonce":4251571952}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6808","nonce":4251571952},{"type":"v1","addr":"172.21.15.136:6809","nonce":4251571952}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6806","nonce":4251571952},{"type":"v1","addr":"172.21.15.136:6807","nonce":4251571952}]},"public_addr":"172.21.15.136:6803/4251571952","cluster_addr":"172.21.15.136:6805/4251571952","heartbeat_back_addr":"172.21.15.136:6809/4251571952","heartbeat_front_addr":"172.21.15.136:6807/4251571952","state":["exists","up"]},{"osd":1,"uuid":"850d4f98-0640-4151-bd45-a351ca41d54e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6810","nonce":3482177958},{"type":"v1","addr":"172.21.15.136:6811","nonce":3482177958}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6812","nonce":3482177958},{"type":"v1","addr":"172.21.15.136:6813","nonce":3482177958}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6816","nonce":3482177958},{"type":"v1","addr":"172.21.15.136:6817","nonce":3482177958}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6814","nonce":3482177958},{"type":"v1","addr":"172.21.15.136:6815","nonce":3482177958}]},"public_addr":"172.21.15.136:6811/3482177958","cluster_addr":"172.21.15.136:6813/3482177958","heartbeat_back_addr":"172.21.15.136:6817/3482177958","heartbeat_front_addr":"172.21.15.136:6815/3482177958","state":["exists","up"]},{"osd":2,"uuid":"bde707d6-e027-4b63-9853-fd59212eea1e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6818","nonce":308477304},{"type":"v1","addr":"172.21.15.136:6819","nonce":308477304}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6820","nonce":308477304},{"type":"v1","addr":"172.21.15.136:6821","nonce":308477304}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6824","nonce":308477304},{"type":"v1","addr":"172.21.15.136:6825","nonce":308477304}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6822","nonce":308477304},{"type":"v1","addr":"172.21.15.136:6823","nonce":308477304}]},"public_addr":"172.21.15.136:6819/308477304","cluster_addr":"172.21.15.136:6821/308477304","heartbeat_back_addr":"172.21.15.136:6825/308477304","heartbeat_front_addr":"172.21.15.136:6823/308477304","state":["exists","up"]},{"osd":3,"uuid":"d61f8357-b89b-4c05-b366-65a9f788064d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6826","nonce":201948269},{"type":"v1","addr":"172.21.15.136:6827","nonce":201948269}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6828","nonce":201948269},{"type":"v1","addr":"172.21.15.136:6829","nonce":201948269}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6832","nonce":201948269},{"type":"v1","addr":"172.21.15.136:6833","nonce":201948269}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6830","nonce":201948269},{"type":"v1","addr":"172.21.15.136:6831","nonce":201948269}]},"public_addr":"172.21.15.136:6827/201948269","cluster_addr":"172.21.15.136:6829/201948269","heartbeat_back_addr":"172.21.15.136:6833/201948269","heartbeat_front_addr":"172.21.15.136:6831/201948269","state":["exists","up"]},{"osd":4,"uuid":"98b0e81c-824e-4dbc-9faa-2de9f199af1b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6800","nonce":775251087},{"type":"v1","addr":"172.21.15.181:6801","nonce":775251087}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6802","nonce":775251087},{"type":"v1","addr":"172.21.15.181:6803","nonce":775251087}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6806","nonce":775251087},{"type":"v1","addr":"172.21.15.181:6807","nonce":775251087}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6804","nonce":775251087},{"type":"v1","addr":"172.21.15.181:6805","nonce":775251087}]},"public_addr":"172.21.15.181:6801/775251087","cluster_addr":"172.21.15.181:6803/775251087","heartbeat_back_addr":"172.21.15.181:6807/775251087","heartbeat_front_addr":"172.21.15.181:6805/775251087","state":["exists","up"]},{"osd":5,"uuid":"5c6d2f69-7f9c-4689-95b0-bd4eb858956d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6808","nonce":2109931558},{"type":"v1","addr":"172.21.15.181:6809","nonce":2109931558}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6810","nonce":2109931558},{"type":"v1","addr":"172.21.15.181:6811","nonce":2109931558}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6814","nonce":2109931558},{"type":"v1","addr":"172.21.15.181:6815","nonce":2109931558}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6812","nonce":2109931558},{"type":"v1","addr":"172.21.15.181:6813","nonce":2109931558}]},"public_addr":"172.21.15.181:6809/2109931558","cluster_addr":"172.21.15.181:6811/2109931558","heartbeat_back_addr":"172.21.15.181:6815/2109931558","heartbeat_front_addr":"172.21.15.181:6813/2109931558","state":["exists","up"]},{"osd":6,"uuid":"532ee052-af15-433a-8e93-b0b642aaf85b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6816","nonce":4099148413},{"type":"v1","addr":"172.21.15.181:6817","nonce":4099148413}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6818","nonce":4099148413},{"type":"v1","addr":"172.21.15.181:6819","nonce":4099148413}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6822","nonce":4099148413},{"type":"v1","addr":"172.21.15.181:6823","nonce":4099148413}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6820","nonce":4099148413},{"type":"v1","addr":"172.21.15.181:6821","nonce":4099148413}]},"public_addr":"172.21.15.181:6817/4099148413","cluster_addr":"172.21.15.181:6819/4099148413","heartbeat_back_addr":"172.21.15.181:6823/4099148413","heartbeat_front_addr":"172.21.15.181:6821/4099148413","state":["exists","up"]},{"osd":7,"uuid":"d2ff80f0-6ef0-4098-877c-a891d48c687d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6824","nonce":783620875},{"type":"v1","addr":"172.21.15.181:6825","nonce":783620875}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6826","nonce":783620875},{"type":"v1","addr":"172.21.15.181:6827","nonce":783620875}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6830","nonce":783620875},{"type":"v1","addr":"172.21.15.181:6831","nonce":783620875}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6828","nonce":783620875},{"type":"v1","addr":"172.21.15.181:6829","nonce":783620875}]},"public_addr":"172.21.15.181:6825/783620875","cluster_addr":"172.21.15.181:6827/783620875","heartbeat_back_addr":"172.21.15.181:6831/783620875","heartbeat_front_addr":"172.21.15.181:6829/783620875","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:26:50.657580+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:27:40.536476+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:28:31.755012+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:29:23.585451+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:30:11.888258+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:31:01.812765+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:31:52.166696+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:32:42.696525+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.136:0/4260962972":"2024-09-13T23:24:25.346823+0000","172.21.15.136:6801/3594272622":"2024-09-13T23:24:25.346823+0000","172.21.15.136:0/209539205":"2024-09-13T23:24:25.346823+0000","172.21.15.136:6800/3594272622":"2024-09-13T23:24:25.346823+0000","172.21.15.136:0/1549678324":"2024-09-13T23:23:54.847045+0000","172.21.15.136:0/3376519841":"2024-09-13T23:23:54.847045+0000","172.21.15.136:0/3910468981":"2024-09-13T23:23:54.847045+0000","172.21.15.136:6801/2894181186":"2024-09-13T23:23:54.847045+0000","172.21.15.136:0/1964283041":"2024-09-13T23:24:25.346823+0000","172.21.15.136:6800/2894181186":"2024-09-13T23:23:54.847045+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-12T23:33:30.503 DEBUG:teuthology.orchestra.run.smithi136:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs standby_count_wanted 0 2024-09-12T23:33:31.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:30 smithi136 bash[22601]: cluster 2024-09-12T23:33:29.488468+0000 mgr.y (mgr.14150) 339 : cluster [DBG] pgmap v314: 97 pgs: 5 creating+activating, 12 creating+peering, 75 unknown, 5 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:31.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:30 smithi136 bash[22601]: cluster 2024-09-12T23:33:29.854778+0000 mon.a (mon.0) 674 : cluster [WRN] Health check failed: 2 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:33:31.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:30 smithi136 bash[22601]: cluster 2024-09-12T23:33:29.868278+0000 mon.a (mon.0) 675 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2024-09-12T23:33:31.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:30 smithi136 bash[22601]: audit 2024-09-12T23:33:29.883992+0000 mon.a (mon.0) 676 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]': finished 2024-09-12T23:33:31.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:30 smithi136 bash[22601]: cluster 2024-09-12T23:33:29.884216+0000 mon.a (mon.0) 677 : cluster [DBG] fsmap cephfs:1 {0=b=up:creating} 1 up:standby 2024-09-12T23:33:31.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:30 smithi136 bash[22601]: cluster 2024-09-12T23:33:29.925726+0000 mon.a (mon.0) 678 : cluster [INF] daemon mds.b is now active in filesystem cephfs as rank 0 2024-09-12T23:33:31.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:30 smithi136 bash[22601]: audit 2024-09-12T23:33:30.190243+0000 mon.c (mon.2) 22 : audit [DBG] from='client.? 172.21.15.136:0/2138505377' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-12T23:33:31.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:30 smithi136 bash[22601]: cluster 2024-09-12T23:33:30.367566+0000 mon.a (mon.0) 679 : cluster [DBG] mds.? [v2:172.21.15.181:6832/2077070284,v1:172.21.15.181:6833/2077070284] up:active 2024-09-12T23:33:31.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:30 smithi136 bash[22601]: cluster 2024-09-12T23:33:30.367612+0000 mon.a (mon.0) 680 : cluster [DBG] fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-12T23:33:31.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:30 smithi136 bash[22601]: audit 2024-09-12T23:33:30.486529+0000 mon.c (mon.2) 23 : audit [DBG] from='client.? 172.21.15.136:0/3858510696' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-12T23:33:31.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:30 smithi136 bash[22601]: audit 2024-09-12T23:33:30.833906+0000 mon.a (mon.0) 681 : audit [INF] from='client.? 172.21.15.136:0/4147334151' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]: dispatch 2024-09-12T23:33:31.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:30 smithi136 bash[32189]: cluster 2024-09-12T23:33:29.488468+0000 mgr.y (mgr.14150) 339 : cluster [DBG] pgmap v314: 97 pgs: 5 creating+activating, 12 creating+peering, 75 unknown, 5 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:31.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:30 smithi136 bash[32189]: cluster 2024-09-12T23:33:29.854778+0000 mon.a (mon.0) 674 : cluster [WRN] Health check failed: 2 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:33:31.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:30 smithi136 bash[32189]: cluster 2024-09-12T23:33:29.868278+0000 mon.a (mon.0) 675 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2024-09-12T23:33:31.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:30 smithi136 bash[32189]: audit 2024-09-12T23:33:29.883992+0000 mon.a (mon.0) 676 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]': finished 2024-09-12T23:33:31.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:30 smithi136 bash[32189]: cluster 2024-09-12T23:33:29.884216+0000 mon.a (mon.0) 677 : cluster [DBG] fsmap cephfs:1 {0=b=up:creating} 1 up:standby 2024-09-12T23:33:31.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:30 smithi136 bash[32189]: cluster 2024-09-12T23:33:29.925726+0000 mon.a (mon.0) 678 : cluster [INF] daemon mds.b is now active in filesystem cephfs as rank 0 2024-09-12T23:33:31.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:30 smithi136 bash[32189]: audit 2024-09-12T23:33:30.190243+0000 mon.c (mon.2) 22 : audit [DBG] from='client.? 172.21.15.136:0/2138505377' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-12T23:33:31.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:30 smithi136 bash[32189]: cluster 2024-09-12T23:33:30.367566+0000 mon.a (mon.0) 679 : cluster [DBG] mds.? [v2:172.21.15.181:6832/2077070284,v1:172.21.15.181:6833/2077070284] up:active 2024-09-12T23:33:31.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:30 smithi136 bash[32189]: cluster 2024-09-12T23:33:30.367612+0000 mon.a (mon.0) 680 : cluster [DBG] fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-12T23:33:31.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:30 smithi136 bash[32189]: audit 2024-09-12T23:33:30.486529+0000 mon.c (mon.2) 23 : audit [DBG] from='client.? 172.21.15.136:0/3858510696' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-12T23:33:31.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:30 smithi136 bash[32189]: audit 2024-09-12T23:33:30.833906+0000 mon.a (mon.0) 681 : audit [INF] from='client.? 172.21.15.136:0/4147334151' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]: dispatch 2024-09-12T23:33:31.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:30 smithi181 bash[24776]: cluster 2024-09-12T23:33:29.488468+0000 mgr.y (mgr.14150) 339 : cluster [DBG] pgmap v314: 97 pgs: 5 creating+activating, 12 creating+peering, 75 unknown, 5 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:31.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:30 smithi181 bash[24776]: cluster 2024-09-12T23:33:29.854778+0000 mon.a (mon.0) 674 : cluster [WRN] Health check failed: 2 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:33:31.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:30 smithi181 bash[24776]: cluster 2024-09-12T23:33:29.868278+0000 mon.a (mon.0) 675 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2024-09-12T23:33:31.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:30 smithi181 bash[24776]: audit 2024-09-12T23:33:29.883992+0000 mon.a (mon.0) 676 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]': finished 2024-09-12T23:33:31.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:30 smithi181 bash[24776]: cluster 2024-09-12T23:33:29.884216+0000 mon.a (mon.0) 677 : cluster [DBG] fsmap cephfs:1 {0=b=up:creating} 1 up:standby 2024-09-12T23:33:31.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:30 smithi181 bash[24776]: cluster 2024-09-12T23:33:29.925726+0000 mon.a (mon.0) 678 : cluster [INF] daemon mds.b is now active in filesystem cephfs as rank 0 2024-09-12T23:33:31.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:30 smithi181 bash[24776]: audit 2024-09-12T23:33:30.190243+0000 mon.c (mon.2) 22 : audit [DBG] from='client.? 172.21.15.136:0/2138505377' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-12T23:33:31.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:30 smithi181 bash[24776]: cluster 2024-09-12T23:33:30.367566+0000 mon.a (mon.0) 679 : cluster [DBG] mds.? [v2:172.21.15.181:6832/2077070284,v1:172.21.15.181:6833/2077070284] up:active 2024-09-12T23:33:31.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:30 smithi181 bash[24776]: cluster 2024-09-12T23:33:30.367612+0000 mon.a (mon.0) 680 : cluster [DBG] fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-12T23:33:31.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:30 smithi181 bash[24776]: audit 2024-09-12T23:33:30.486529+0000 mon.c (mon.2) 23 : audit [DBG] from='client.? 172.21.15.136:0/3858510696' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-12T23:33:31.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:30 smithi181 bash[24776]: audit 2024-09-12T23:33:30.833906+0000 mon.a (mon.0) 681 : audit [INF] from='client.? 172.21.15.136:0/4147334151' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]: dispatch 2024-09-12T23:33:31.392 DEBUG:tasks.cephfs.filesystem:fs_config: {} 2024-09-12T23:33:31.392 DEBUG:teuthology.orchestra.run.smithi136:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs allow_standby_replay false 2024-09-12T23:33:32.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:31 smithi136 bash[22601]: cluster 2024-09-12T23:33:30.885457+0000 mon.a (mon.0) 682 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2024-09-12T23:33:32.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:31 smithi136 bash[22601]: audit 2024-09-12T23:33:31.372856+0000 mon.a (mon.0) 683 : audit [INF] from='client.? 172.21.15.136:0/4147334151' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]': finished 2024-09-12T23:33:32.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:31 smithi136 bash[22601]: cluster 2024-09-12T23:33:31.373058+0000 mon.a (mon.0) 684 : cluster [DBG] fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-12T23:33:32.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:31 smithi136 bash[22601]: audit 2024-09-12T23:33:31.722556+0000 mon.a (mon.0) 685 : audit [INF] from='client.? 172.21.15.136:0/1259745480' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]: dispatch 2024-09-12T23:33:32.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:31 smithi136 bash[32189]: cluster 2024-09-12T23:33:30.885457+0000 mon.a (mon.0) 682 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2024-09-12T23:33:32.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:31 smithi136 bash[32189]: audit 2024-09-12T23:33:31.372856+0000 mon.a (mon.0) 683 : audit [INF] from='client.? 172.21.15.136:0/4147334151' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]': finished 2024-09-12T23:33:32.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:31 smithi136 bash[32189]: cluster 2024-09-12T23:33:31.373058+0000 mon.a (mon.0) 684 : cluster [DBG] fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-12T23:33:32.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:31 smithi136 bash[32189]: audit 2024-09-12T23:33:31.722556+0000 mon.a (mon.0) 685 : audit [INF] from='client.? 172.21.15.136:0/1259745480' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]: dispatch 2024-09-12T23:33:32.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:31 smithi181 bash[24776]: cluster 2024-09-12T23:33:30.885457+0000 mon.a (mon.0) 682 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2024-09-12T23:33:32.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:31 smithi181 bash[24776]: audit 2024-09-12T23:33:31.372856+0000 mon.a (mon.0) 683 : audit [INF] from='client.? 172.21.15.136:0/4147334151' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]': finished 2024-09-12T23:33:32.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:31 smithi181 bash[24776]: cluster 2024-09-12T23:33:31.373058+0000 mon.a (mon.0) 684 : cluster [DBG] fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-12T23:33:32.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:31 smithi181 bash[24776]: audit 2024-09-12T23:33:31.722556+0000 mon.a (mon.0) 685 : audit [INF] from='client.? 172.21.15.136:0/1259745480' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]: dispatch 2024-09-12T23:33:32.399 DEBUG:tasks.cephfs.filesystem:Not Creating any subvols for filesystem 'cephfs' 2024-09-12T23:33:32.399 DEBUG:teuthology.orchestra.run.smithi136:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-09-12T23:33:32.717 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-12T23:33:32.717 INFO:teuthology.orchestra.run.smithi136.stdout:{"epoch":7,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14391,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.136:6835/3940972569","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6834","nonce":3940972569},{"type":"v1","addr":"172.21.15.136:6835","nonce":3940972569}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":7,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-09-12T23:33:29.192396+0000","modified":"2024-09-12T23:33:32.368306+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":1,"in":[0],"up":{"mds_0":24316},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24316":{"gid":24316,"name":"b","rank":0,"incarnation":4,"state":"up:active","state_seq":3,"addr":"172.21.15.181:6833/2077070284","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6832","nonce":2077070284},{"type":"v1","addr":"172.21.15.181:6833","nonce":2077070284}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","standby_count_wanted":0},"id":1}]} 2024-09-12T23:33:32.717 INFO:teuthology.orchestra.run.smithi136.stderr:dumped fsmap epoch 7 2024-09-12T23:33:32.734 DEBUG:teuthology.orchestra.run.smithi136:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-12T23:33:33.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:32 smithi136 bash[22601]: cluster 2024-09-12T23:33:31.489162+0000 mgr.y (mgr.14150) 340 : cluster [DBG] pgmap v317: 97 pgs: 5 creating+activating, 35 creating+peering, 27 unknown, 30 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:33.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:32 smithi136 bash[22601]: cluster 2024-09-12T23:33:32.367695+0000 mon.a (mon.0) 686 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 2 pool(s) do not have an application enabled) 2024-09-12T23:33:33.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:32 smithi136 bash[22601]: cluster 2024-09-12T23:33:32.367744+0000 mon.a (mon.0) 687 : cluster [INF] Cluster is now healthy 2024-09-12T23:33:33.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:32 smithi136 bash[22601]: audit 2024-09-12T23:33:32.380673+0000 mon.a (mon.0) 688 : audit [INF] from='client.? 172.21.15.136:0/1259745480' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]': finished 2024-09-12T23:33:33.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:32 smithi136 bash[22601]: cluster 2024-09-12T23:33:32.380856+0000 mon.a (mon.0) 689 : cluster [DBG] fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-12T23:33:33.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:32 smithi136 bash[22601]: audit 2024-09-12T23:33:32.720015+0000 mon.a (mon.0) 690 : audit [DBG] from='client.? 172.21.15.136:0/783031366' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-12T23:33:33.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:32 smithi136 bash[32189]: cluster 2024-09-12T23:33:31.489162+0000 mgr.y (mgr.14150) 340 : cluster [DBG] pgmap v317: 97 pgs: 5 creating+activating, 35 creating+peering, 27 unknown, 30 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:33.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:32 smithi136 bash[32189]: cluster 2024-09-12T23:33:32.367695+0000 mon.a (mon.0) 686 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 2 pool(s) do not have an application enabled) 2024-09-12T23:33:33.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:32 smithi136 bash[32189]: cluster 2024-09-12T23:33:32.367744+0000 mon.a (mon.0) 687 : cluster [INF] Cluster is now healthy 2024-09-12T23:33:33.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:32 smithi136 bash[32189]: audit 2024-09-12T23:33:32.380673+0000 mon.a (mon.0) 688 : audit [INF] from='client.? 172.21.15.136:0/1259745480' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]': finished 2024-09-12T23:33:33.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:32 smithi136 bash[32189]: cluster 2024-09-12T23:33:32.380856+0000 mon.a (mon.0) 689 : cluster [DBG] fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-12T23:33:33.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:32 smithi136 bash[32189]: audit 2024-09-12T23:33:32.720015+0000 mon.a (mon.0) 690 : audit [DBG] from='client.? 172.21.15.136:0/783031366' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-12T23:33:33.014 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-12T23:33:33.015 INFO:teuthology.orchestra.run.smithi136.stdout:{"epoch":54,"fsid":"da58a47c-715d-11ef-bceb-c7b262605968","created":"2024-09-12T23:23:29.000926+0000","modified":"2024-09-12T23:33:30.875962+0000","last_up_change":"2024-09-12T23:32:43.859472+0000","last_in_change":"2024-09-12T23:32:18.995528+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-12T23:28:33.956985+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-09-12T23:33:27.478434+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-09-12T23:33:28.566349+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"54","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"87aedf04-695e-4a8f-80c3-ffd18883a4d5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6802","nonce":4251571952},{"type":"v1","addr":"172.21.15.136:6803","nonce":4251571952}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6804","nonce":4251571952},{"type":"v1","addr":"172.21.15.136:6805","nonce":4251571952}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6808","nonce":4251571952},{"type":"v1","addr":"172.21.15.136:6809","nonce":4251571952}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6806","nonce":4251571952},{"type":"v1","addr":"172.21.15.136:6807","nonce":4251571952}]},"public_addr":"172.21.15.136:6803/4251571952","cluster_addr":"172.21.15.136:6805/4251571952","heartbeat_back_addr":"172.21.15.136:6809/4251571952","heartbeat_front_addr":"172.21.15.136:6807/4251571952","state":["exists","up"]},{"osd":1,"uuid":"850d4f98-0640-4151-bd45-a351ca41d54e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6810","nonce":3482177958},{"type":"v1","addr":"172.21.15.136:6811","nonce":3482177958}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6812","nonce":3482177958},{"type":"v1","addr":"172.21.15.136:6813","nonce":3482177958}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6816","nonce":3482177958},{"type":"v1","addr":"172.21.15.136:6817","nonce":3482177958}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6814","nonce":3482177958},{"type":"v1","addr":"172.21.15.136:6815","nonce":3482177958}]},"public_addr":"172.21.15.136:6811/3482177958","cluster_addr":"172.21.15.136:6813/3482177958","heartbeat_back_addr":"172.21.15.136:6817/3482177958","heartbeat_front_addr":"172.21.15.136:6815/3482177958","state":["exists","up"]},{"osd":2,"uuid":"bde707d6-e027-4b63-9853-fd59212eea1e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6818","nonce":308477304},{"type":"v1","addr":"172.21.15.136:6819","nonce":308477304}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6820","nonce":308477304},{"type":"v1","addr":"172.21.15.136:6821","nonce":308477304}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6824","nonce":308477304},{"type":"v1","addr":"172.21.15.136:6825","nonce":308477304}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6822","nonce":308477304},{"type":"v1","addr":"172.21.15.136:6823","nonce":308477304}]},"public_addr":"172.21.15.136:6819/308477304","cluster_addr":"172.21.15.136:6821/308477304","heartbeat_back_addr":"172.21.15.136:6825/308477304","heartbeat_front_addr":"172.21.15.136:6823/308477304","state":["exists","up"]},{"osd":3,"uuid":"d61f8357-b89b-4c05-b366-65a9f788064d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6826","nonce":201948269},{"type":"v1","addr":"172.21.15.136:6827","nonce":201948269}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6828","nonce":201948269},{"type":"v1","addr":"172.21.15.136:6829","nonce":201948269}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6832","nonce":201948269},{"type":"v1","addr":"172.21.15.136:6833","nonce":201948269}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6830","nonce":201948269},{"type":"v1","addr":"172.21.15.136:6831","nonce":201948269}]},"public_addr":"172.21.15.136:6827/201948269","cluster_addr":"172.21.15.136:6829/201948269","heartbeat_back_addr":"172.21.15.136:6833/201948269","heartbeat_front_addr":"172.21.15.136:6831/201948269","state":["exists","up"]},{"osd":4,"uuid":"98b0e81c-824e-4dbc-9faa-2de9f199af1b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6800","nonce":775251087},{"type":"v1","addr":"172.21.15.181:6801","nonce":775251087}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6802","nonce":775251087},{"type":"v1","addr":"172.21.15.181:6803","nonce":775251087}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6806","nonce":775251087},{"type":"v1","addr":"172.21.15.181:6807","nonce":775251087}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6804","nonce":775251087},{"type":"v1","addr":"172.21.15.181:6805","nonce":775251087}]},"public_addr":"172.21.15.181:6801/775251087","cluster_addr":"172.21.15.181:6803/775251087","heartbeat_back_addr":"172.21.15.181:6807/775251087","heartbeat_front_addr":"172.21.15.181:6805/775251087","state":["exists","up"]},{"osd":5,"uuid":"5c6d2f69-7f9c-4689-95b0-bd4eb858956d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6808","nonce":2109931558},{"type":"v1","addr":"172.21.15.181:6809","nonce":2109931558}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6810","nonce":2109931558},{"type":"v1","addr":"172.21.15.181:6811","nonce":2109931558}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6814","nonce":2109931558},{"type":"v1","addr":"172.21.15.181:6815","nonce":2109931558}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6812","nonce":2109931558},{"type":"v1","addr":"172.21.15.181:6813","nonce":2109931558}]},"public_addr":"172.21.15.181:6809/2109931558","cluster_addr":"172.21.15.181:6811/2109931558","heartbeat_back_addr":"172.21.15.181:6815/2109931558","heartbeat_front_addr":"172.21.15.181:6813/2109931558","state":["exists","up"]},{"osd":6,"uuid":"532ee052-af15-433a-8e93-b0b642aaf85b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6816","nonce":4099148413},{"type":"v1","addr":"172.21.15.181:6817","nonce":4099148413}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6818","nonce":4099148413},{"type":"v1","addr":"172.21.15.181:6819","nonce":4099148413}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6822","nonce":4099148413},{"type":"v1","addr":"172.21.15.181:6823","nonce":4099148413}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6820","nonce":4099148413},{"type":"v1","addr":"172.21.15.181:6821","nonce":4099148413}]},"public_addr":"172.21.15.181:6817/4099148413","cluster_addr":"172.21.15.181:6819/4099148413","heartbeat_back_addr":"172.21.15.181:6823/4099148413","heartbeat_front_addr":"172.21.15.181:6821/4099148413","state":["exists","up"]},{"osd":7,"uuid":"d2ff80f0-6ef0-4098-877c-a891d48c687d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6824","nonce":783620875},{"type":"v1","addr":"172.21.15.181:6825","nonce":783620875}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6826","nonce":783620875},{"type":"v1","addr":"172.21.15.181:6827","nonce":783620875}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6830","nonce":783620875},{"type":"v1","addr":"172.21.15.181:6831","nonce":783620875}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6828","nonce":783620875},{"type":"v1","addr":"172.21.15.181:6829","nonce":783620875}]},"public_addr":"172.21.15.181:6825/783620875","cluster_addr":"172.21.15.181:6827/783620875","heartbeat_back_addr":"172.21.15.181:6831/783620875","heartbeat_front_addr":"172.21.15.181:6829/783620875","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:26:50.657580+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:27:40.536476+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:28:31.755012+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:29:23.585451+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:30:11.888258+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:31:01.812765+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:31:52.166696+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:32:42.696525+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.136:0/4260962972":"2024-09-13T23:24:25.346823+0000","172.21.15.136:6801/3594272622":"2024-09-13T23:24:25.346823+0000","172.21.15.136:0/209539205":"2024-09-13T23:24:25.346823+0000","172.21.15.136:6800/3594272622":"2024-09-13T23:24:25.346823+0000","172.21.15.136:0/1549678324":"2024-09-13T23:23:54.847045+0000","172.21.15.136:0/3376519841":"2024-09-13T23:23:54.847045+0000","172.21.15.136:0/3910468981":"2024-09-13T23:23:54.847045+0000","172.21.15.136:6801/2894181186":"2024-09-13T23:23:54.847045+0000","172.21.15.136:0/1964283041":"2024-09-13T23:24:25.346823+0000","172.21.15.136:6800/2894181186":"2024-09-13T23:23:54.847045+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-12T23:33:33.032 INFO:tasks.cephfs.filesystem.ceph_manager:waiting for clean 2024-09-12T23:33:33.032 DEBUG:teuthology.orchestra.run.smithi136:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-09-12T23:33:33.327 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-12T23:33:33.329 INFO:teuthology.orchestra.run.smithi136.stderr:dumped all 2024-09-12T23:33:33.346 INFO:teuthology.orchestra.run.smithi136.stdout:{"pg_ready":true,"pg_map":{"version":317,"stamp":"2024-09-12T23:33:31.488747+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":74,"ondisk_log_size":74,"up":210,"acting":210,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":59,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":7,"kb":749961216,"kb_used":2384460,"kb_used_data":5608,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747576756,"statfs":{"total":767960285184,"available":765518598144,"internally_reserved":0,"allocated":5742592,"data_stored":3378332,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.001284"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.861074+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:54:27.307253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,6],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.876654+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:18:46.573142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[3,7],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.859085+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:07:03.294377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[2,4],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.873576+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:07:11.163630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,1],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.850034+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:17:57.421440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,4],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.902033+0000","last_change":"2024-09-12T23:33:29.902033+0000","last_active":"2024-09-12T23:33:29.902033+0000","last_peered":"2024-09-12T23:33:29.902033+0000","last_clean":"2024-09-12T23:33:29.902033+0000","last_became_active":"2024-09-12T23:33:29.901611+0000","last_became_peered":"2024-09-12T23:33:29.901611+0000","last_unstale":"2024-09-12T23:33:29.902033+0000","last_undegraded":"2024-09-12T23:33:29.902033+0000","last_fullsized":"2024-09-12T23:33:29.902033+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:57:58.671558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.856686+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:41:19.476191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,5],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.852231+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:56:38.506429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,5],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.855386+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:08:21.774244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,5],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.907459+0000","last_change":"2024-09-12T23:33:29.907459+0000","last_active":"2024-09-12T23:33:29.907459+0000","last_peered":"2024-09-12T23:33:29.907459+0000","last_clean":"2024-09-12T23:33:29.907459+0000","last_became_active":"2024-09-12T23:33:29.906577+0000","last_became_peered":"2024-09-12T23:33:29.906577+0000","last_unstale":"2024-09-12T23:33:29.907459+0000","last_undegraded":"2024-09-12T23:33:29.907459+0000","last_fullsized":"2024-09-12T23:33:29.907459+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:21:41.131616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.867621+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:53:38.724701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,7],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.866959+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-13T23:33:43.071267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[4,7],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.849688+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:08:59.758427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[6,7],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.849822+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:08:17.677941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[2,6],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.849372+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:43:43.192749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,4],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.870650+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:34:34.650312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[6,7],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.906087+0000","last_change":"2024-09-12T23:33:29.906087+0000","last_active":"2024-09-12T23:33:29.906087+0000","last_peered":"2024-09-12T23:33:29.906087+0000","last_clean":"2024-09-12T23:33:29.906087+0000","last_became_active":"2024-09-12T23:33:29.905780+0000","last_became_peered":"2024-09-12T23:33:29.905780+0000","last_unstale":"2024-09-12T23:33:29.906087+0000","last_undegraded":"2024-09-12T23:33:29.906087+0000","last_fullsized":"2024-09-12T23:33:29.906087+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:00:40.777941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.855239+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:05:56.032960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[2,3],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.903381+0000","last_change":"2024-09-12T23:33:29.903381+0000","last_active":"2024-09-12T23:33:29.903381+0000","last_peered":"2024-09-12T23:33:29.903381+0000","last_clean":"2024-09-12T23:33:29.903381+0000","last_became_active":"2024-09-12T23:33:29.903081+0000","last_became_peered":"2024-09-12T23:33:29.903081+0000","last_unstale":"2024-09-12T23:33:29.903381+0000","last_undegraded":"2024-09-12T23:33:29.903381+0000","last_fullsized":"2024-09-12T23:33:29.903381+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:43:01.175691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.850972+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:39:14.674234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[4,7],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.913907+0000","last_change":"2024-09-12T23:33:29.913907+0000","last_active":"2024-09-12T23:33:29.913907+0000","last_peered":"2024-09-12T23:33:29.913907+0000","last_clean":"2024-09-12T23:33:29.913907+0000","last_became_active":"2024-09-12T23:33:29.913193+0000","last_became_peered":"2024-09-12T23:33:29.913193+0000","last_unstale":"2024-09-12T23:33:29.913907+0000","last_undegraded":"2024-09-12T23:33:29.913907+0000","last_fullsized":"2024-09-12T23:33:29.913907+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:32:45.117056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.869503+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:25:33.906182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,3],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.850701+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:38:56.219241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[5,6],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.852156+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:09:34.637115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,5],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.1b","version":"53'1","reported_seq":11,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.878981+0000","last_change":"2024-09-12T23:33:28.883222+0000","last_active":"2024-09-12T23:33:29.878981+0000","last_peered":"2024-09-12T23:33:29.878981+0000","last_clean":"2024-09-12T23:33:29.878981+0000","last_became_active":"2024-09-12T23:33:28.882848+0000","last_became_peered":"2024-09-12T23:33:28.882848+0000","last_unstale":"2024-09-12T23:33:29.878981+0000","last_undegraded":"2024-09-12T23:33:29.878981+0000","last_fullsized":"2024-09-12T23:33:29.878981+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:57:33.905996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.1a","version":"0'0","reported_seq":2,"reported_epoch":52,"state":"creating+activating","last_fresh":"2024-09-12T23:33:28.846682+0000","last_change":"2024-09-12T23:33:28.847145+0000","last_active":"2024-09-12T23:33:27.830370+0000","last_peered":"2024-09-12T23:33:27.830370+0000","last_clean":"2024-09-12T23:33:27.830370+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.846682+0000","last_undegraded":"2024-09-12T23:33:28.846682+0000","last_fullsized":"2024-09-12T23:33:28.846682+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:57:50.175287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":["6","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":10,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.873141+0000","last_change":"2024-09-12T23:33:28.872822+0000","last_active":"2024-09-12T23:33:28.873141+0000","last_peered":"2024-09-12T23:33:28.873141+0000","last_clean":"2024-09-12T23:33:28.873141+0000","last_became_active":"2024-09-12T23:33:28.872404+0000","last_became_peered":"2024-09-12T23:33:28.872404+0000","last_unstale":"2024-09-12T23:33:28.873141+0000","last_undegraded":"2024-09-12T23:33:28.873141+0000","last_fullsized":"2024-09-12T23:33:28.873141+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-13T23:47:42.317408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.907360+0000","last_change":"2024-09-12T23:33:29.907360+0000","last_active":"2024-09-12T23:33:29.907360+0000","last_peered":"2024-09-12T23:33:29.907360+0000","last_clean":"2024-09-12T23:33:29.907360+0000","last_became_active":"2024-09-12T23:33:29.907067+0000","last_became_peered":"2024-09-12T23:33:29.907067+0000","last_unstale":"2024-09-12T23:33:29.907360+0000","last_undegraded":"2024-09-12T23:33:29.907360+0000","last_fullsized":"2024-09-12T23:33:29.907360+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:26:38.084938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.896704+0000","last_change":"2024-09-12T23:33:28.896704+0000","last_active":"2024-09-12T23:33:28.896704+0000","last_peered":"2024-09-12T23:33:28.896704+0000","last_clean":"2024-09-12T23:33:28.896704+0000","last_became_active":"2024-09-12T23:33:28.896167+0000","last_became_peered":"2024-09-12T23:33:28.896167+0000","last_unstale":"2024-09-12T23:33:28.896704+0000","last_undegraded":"2024-09-12T23:33:28.896704+0000","last_fullsized":"2024-09-12T23:33:28.896704+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:14:47.959501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":3,"reported_epoch":52,"state":"creating+activating","last_fresh":"2024-09-12T23:33:28.848624+0000","last_change":"2024-09-12T23:33:28.848624+0000","last_active":"2024-09-12T23:33:27.830370+0000","last_peered":"2024-09-12T23:33:27.830370+0000","last_clean":"2024-09-12T23:33:27.830370+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.848624+0000","last_undegraded":"2024-09-12T23:33:28.848624+0000","last_fullsized":"2024-09-12T23:33:28.848624+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:20:27.774655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":["6","2","5"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.866433+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-13T23:52:12.475067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,7],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.890012+0000","last_change":"2024-09-12T23:33:28.890012+0000","last_active":"2024-09-12T23:33:28.890012+0000","last_peered":"2024-09-12T23:33:28.890012+0000","last_clean":"2024-09-12T23:33:28.890012+0000","last_became_active":"2024-09-12T23:33:28.887477+0000","last_became_peered":"2024-09-12T23:33:28.887477+0000","last_unstale":"2024-09-12T23:33:28.890012+0000","last_undegraded":"2024-09-12T23:33:28.890012+0000","last_fullsized":"2024-09-12T23:33:28.890012+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:15:02.988389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.15","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:27.830370+0000","last_change":"2024-09-12T23:33:27.830370+0000","last_active":"2024-09-12T23:33:27.830370+0000","last_peered":"2024-09-12T23:33:27.830370+0000","last_clean":"2024-09-12T23:33:27.830370+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:27.830370+0000","last_undegraded":"2024-09-12T23:33:27.830370+0000","last_fullsized":"2024-09-12T23:33:27.830370+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.14","version":"0'0","reported_seq":2,"reported_epoch":52,"state":"creating+activating","last_fresh":"2024-09-12T23:33:28.847672+0000","last_change":"2024-09-12T23:33:28.848228+0000","last_active":"2024-09-12T23:33:27.830370+0000","last_peered":"2024-09-12T23:33:27.830370+0000","last_clean":"2024-09-12T23:33:27.830370+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.847672+0000","last_undegraded":"2024-09-12T23:33:28.847672+0000","last_fullsized":"2024-09-12T23:33:28.847672+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:50:58.477301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":["6","3","5"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.858409+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:06:00.373844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[3,4],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.13","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:27.830370+0000","last_change":"2024-09-12T23:33:27.837333+0000","last_active":"2024-09-12T23:33:27.830370+0000","last_peered":"2024-09-12T23:33:27.830370+0000","last_clean":"2024-09-12T23:33:27.830370+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:27.830370+0000","last_undegraded":"2024-09-12T23:33:27.830370+0000","last_fullsized":"2024-09-12T23:33:27.830370+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:00:55.966645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[3,4],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.899424+0000","last_change":"2024-09-12T23:33:28.899424+0000","last_active":"2024-09-12T23:33:28.899424+0000","last_peered":"2024-09-12T23:33:28.899424+0000","last_clean":"2024-09-12T23:33:28.899424+0000","last_became_active":"2024-09-12T23:33:28.898942+0000","last_became_peered":"2024-09-12T23:33:28.898942+0000","last_unstale":"2024-09-12T23:33:28.899424+0000","last_undegraded":"2024-09-12T23:33:28.899424+0000","last_fullsized":"2024-09-12T23:33:28.899424+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:14:47.069896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.847369+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:28:55.241684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[2,4],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":3,"reported_epoch":52,"state":"creating+activating","last_fresh":"2024-09-12T23:33:28.846921+0000","last_change":"2024-09-12T23:33:28.846921+0000","last_active":"2024-09-12T23:33:27.830370+0000","last_peered":"2024-09-12T23:33:27.830370+0000","last_clean":"2024-09-12T23:33:27.830370+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.846921+0000","last_undegraded":"2024-09-12T23:33:28.846921+0000","last_fullsized":"2024-09-12T23:33:28.846921+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:27:43.449941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":["6","1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.847037+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:29:56.772001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,5],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.863788+0000","last_change":"2024-09-12T23:33:28.863788+0000","last_active":"2024-09-12T23:33:28.863788+0000","last_peered":"2024-09-12T23:33:28.863788+0000","last_clean":"2024-09-12T23:33:28.863788+0000","last_became_active":"2024-09-12T23:33:28.863479+0000","last_became_peered":"2024-09-12T23:33:28.863479+0000","last_unstale":"2024-09-12T23:33:28.863788+0000","last_undegraded":"2024-09-12T23:33:28.863788+0000","last_fullsized":"2024-09-12T23:33:28.863788+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:26:09.204038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.847091+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:57:03.461858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[4,6],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:27.830370+0000","last_change":"2024-09-12T23:33:27.840289+0000","last_active":"2024-09-12T23:33:27.830370+0000","last_peered":"2024-09-12T23:33:27.830370+0000","last_clean":"2024-09-12T23:33:27.830370+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:27.830370+0000","last_undegraded":"2024-09-12T23:33:27.830370+0000","last_fullsized":"2024-09-12T23:33:27.830370+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:54:14.860070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,7],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.849355+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:45:39.430174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,4],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.887441+0000","last_change":"2024-09-12T23:33:28.887441+0000","last_active":"2024-09-12T23:33:28.887441+0000","last_peered":"2024-09-12T23:33:28.887441+0000","last_clean":"2024-09-12T23:33:28.887441+0000","last_became_active":"2024-09-12T23:33:28.887109+0000","last_became_peered":"2024-09-12T23:33:28.887109+0000","last_unstale":"2024-09-12T23:33:28.887441+0000","last_undegraded":"2024-09-12T23:33:28.887441+0000","last_fullsized":"2024-09-12T23:33:28.887441+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:35:14.071342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.859939+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:06:23.076336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,4],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:27.830370+0000","last_change":"2024-09-12T23:33:27.830370+0000","last_active":"2024-09-12T23:33:27.830370+0000","last_peered":"2024-09-12T23:33:27.830370+0000","last_clean":"2024-09-12T23:33:27.830370+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:27.830370+0000","last_undegraded":"2024-09-12T23:33:27.830370+0000","last_fullsized":"2024-09-12T23:33:27.830370+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.874292+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:42:32.768139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[3,6],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.c","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.882138+0000","last_change":"2024-09-12T23:33:28.882138+0000","last_active":"2024-09-12T23:33:28.882138+0000","last_peered":"2024-09-12T23:33:28.882138+0000","last_clean":"2024-09-12T23:33:28.882138+0000","last_became_active":"2024-09-12T23:33:28.881774+0000","last_became_peered":"2024-09-12T23:33:28.881774+0000","last_unstale":"2024-09-12T23:33:28.882138+0000","last_undegraded":"2024-09-12T23:33:28.882138+0000","last_fullsized":"2024-09-12T23:33:28.882138+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:07:36.075590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.857274+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:24:29.251601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[5,6],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"21'72","reported_seq":18,"reported_epoch":51,"state":"active+clean","last_fresh":"2024-09-12T23:33:27.838959+0000","last_change":"2024-09-12T23:32:45.902460+0000","last_active":"2024-09-12T23:33:27.838959+0000","last_peered":"2024-09-12T23:33:27.838959+0000","last_clean":"2024-09-12T23:33:27.838959+0000","last_became_active":"2024-09-12T23:32:45.887205+0000","last_became_peered":"2024-09-12T23:32:45.887205+0000","last_unstale":"2024-09-12T23:33:27.838959+0000","last_undegraded":"2024-09-12T23:33:27.838959+0000","last_fullsized":"2024-09-12T23:33:27.838959+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:28:34.324093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:28:34.324093+0000","last_clean_scrub_stamp":"2024-09-12T23:28:34.324093+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:37:37.825919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.898796+0000","last_change":"2024-09-12T23:33:28.898796+0000","last_active":"2024-09-12T23:33:28.898796+0000","last_peered":"2024-09-12T23:33:28.898796+0000","last_clean":"2024-09-12T23:33:28.898796+0000","last_became_active":"2024-09-12T23:33:28.898329+0000","last_became_peered":"2024-09-12T23:33:28.898329+0000","last_unstale":"2024-09-12T23:33:28.898796+0000","last_undegraded":"2024-09-12T23:33:28.898796+0000","last_fullsized":"2024-09-12T23:33:28.898796+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:54:48.474286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.905028+0000","last_change":"2024-09-12T23:33:29.905028+0000","last_active":"2024-09-12T23:33:29.905028+0000","last_peered":"2024-09-12T23:33:29.905028+0000","last_clean":"2024-09-12T23:33:29.905028+0000","last_became_active":"2024-09-12T23:33:29.904695+0000","last_became_peered":"2024-09-12T23:33:29.904695+0000","last_unstale":"2024-09-12T23:33:29.905028+0000","last_undegraded":"2024-09-12T23:33:29.905028+0000","last_fullsized":"2024-09-12T23:33:29.905028+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:06:47.004852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.895135+0000","last_change":"2024-09-12T23:33:28.895135+0000","last_active":"2024-09-12T23:33:28.895135+0000","last_peered":"2024-09-12T23:33:28.895135+0000","last_clean":"2024-09-12T23:33:28.895135+0000","last_became_active":"2024-09-12T23:33:28.894687+0000","last_became_peered":"2024-09-12T23:33:28.894687+0000","last_unstale":"2024-09-12T23:33:28.895135+0000","last_undegraded":"2024-09-12T23:33:28.895135+0000","last_fullsized":"2024-09-12T23:33:28.895135+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:57:06.646238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.868957+0000","last_change":"2024-09-12T23:33:28.868957+0000","last_active":"2024-09-12T23:33:28.868957+0000","last_peered":"2024-09-12T23:33:28.868957+0000","last_clean":"2024-09-12T23:33:28.868957+0000","last_became_active":"2024-09-12T23:33:28.868587+0000","last_became_peered":"2024-09-12T23:33:28.868587+0000","last_unstale":"2024-09-12T23:33:28.868957+0000","last_undegraded":"2024-09-12T23:33:28.868957+0000","last_fullsized":"2024-09-12T23:33:28.868957+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:45:32.284209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.899931+0000","last_change":"2024-09-12T23:33:28.899931+0000","last_active":"2024-09-12T23:33:28.899931+0000","last_peered":"2024-09-12T23:33:28.899931+0000","last_clean":"2024-09-12T23:33:28.899931+0000","last_became_active":"2024-09-12T23:33:28.899410+0000","last_became_peered":"2024-09-12T23:33:28.899410+0000","last_unstale":"2024-09-12T23:33:28.899931+0000","last_undegraded":"2024-09-12T23:33:28.899931+0000","last_fullsized":"2024-09-12T23:33:28.899931+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:05:26.671172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:27.830370+0000","last_change":"2024-09-12T23:33:27.830370+0000","last_active":"2024-09-12T23:33:27.830370+0000","last_peered":"2024-09-12T23:33:27.830370+0000","last_clean":"2024-09-12T23:33:27.830370+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:27.830370+0000","last_undegraded":"2024-09-12T23:33:27.830370+0000","last_fullsized":"2024-09-12T23:33:27.830370+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.865496+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:38:00.819958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[2,3],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.895077+0000","last_change":"2024-09-12T23:33:28.895077+0000","last_active":"2024-09-12T23:33:28.895077+0000","last_peered":"2024-09-12T23:33:28.895077+0000","last_clean":"2024-09-12T23:33:28.895077+0000","last_became_active":"2024-09-12T23:33:28.892268+0000","last_became_peered":"2024-09-12T23:33:28.892268+0000","last_unstale":"2024-09-12T23:33:28.895077+0000","last_undegraded":"2024-09-12T23:33:28.895077+0000","last_fullsized":"2024-09-12T23:33:28.895077+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:26:19.125877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:27.830370+0000","last_change":"2024-09-12T23:33:27.830370+0000","last_active":"2024-09-12T23:33:27.830370+0000","last_peered":"2024-09-12T23:33:27.830370+0000","last_clean":"2024-09-12T23:33:27.830370+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:27.830370+0000","last_undegraded":"2024-09-12T23:33:27.830370+0000","last_fullsized":"2024-09-12T23:33:27.830370+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.915275+0000","last_change":"2024-09-12T23:33:29.915275+0000","last_active":"2024-09-12T23:33:29.915275+0000","last_peered":"2024-09-12T23:33:29.915275+0000","last_clean":"2024-09-12T23:33:29.915275+0000","last_became_active":"2024-09-12T23:33:29.914934+0000","last_became_peered":"2024-09-12T23:33:29.914934+0000","last_unstale":"2024-09-12T23:33:29.915275+0000","last_undegraded":"2024-09-12T23:33:29.915275+0000","last_fullsized":"2024-09-12T23:33:29.915275+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:43:27.562543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":3,"reported_epoch":52,"state":"creating+activating","last_fresh":"2024-09-12T23:33:28.847239+0000","last_change":"2024-09-12T23:33:28.847239+0000","last_active":"2024-09-12T23:33:27.830370+0000","last_peered":"2024-09-12T23:33:27.830370+0000","last_clean":"2024-09-12T23:33:27.830370+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.847239+0000","last_undegraded":"2024-09-12T23:33:28.847239+0000","last_fullsized":"2024-09-12T23:33:28.847239+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:58:40.820895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":["6","2","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.8","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.894286+0000","last_change":"2024-09-12T23:33:28.894286+0000","last_active":"2024-09-12T23:33:28.894286+0000","last_peered":"2024-09-12T23:33:28.894286+0000","last_clean":"2024-09-12T23:33:28.894286+0000","last_became_active":"2024-09-12T23:33:28.893861+0000","last_became_peered":"2024-09-12T23:33:28.893861+0000","last_unstale":"2024-09-12T23:33:28.894286+0000","last_undegraded":"2024-09-12T23:33:28.894286+0000","last_fullsized":"2024-09-12T23:33:28.894286+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:20:01.806723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:27.830370+0000","last_change":"2024-09-12T23:33:27.830370+0000","last_active":"2024-09-12T23:33:27.830370+0000","last_peered":"2024-09-12T23:33:27.830370+0000","last_clean":"2024-09-12T23:33:27.830370+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:27.830370+0000","last_undegraded":"2024-09-12T23:33:27.830370+0000","last_fullsized":"2024-09-12T23:33:27.830370+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.915243+0000","last_change":"2024-09-12T23:33:29.915243+0000","last_active":"2024-09-12T23:33:29.915243+0000","last_peered":"2024-09-12T23:33:29.915243+0000","last_clean":"2024-09-12T23:33:29.915243+0000","last_became_active":"2024-09-12T23:33:29.914543+0000","last_became_peered":"2024-09-12T23:33:29.914543+0000","last_unstale":"2024-09-12T23:33:29.915243+0000","last_undegraded":"2024-09-12T23:33:29.915243+0000","last_fullsized":"2024-09-12T23:33:29.915243+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-13T23:48:16.023985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:27.830370+0000","last_change":"2024-09-12T23:33:27.830370+0000","last_active":"2024-09-12T23:33:27.830370+0000","last_peered":"2024-09-12T23:33:27.830370+0000","last_clean":"2024-09-12T23:33:27.830370+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:27.830370+0000","last_undegraded":"2024-09-12T23:33:27.830370+0000","last_fullsized":"2024-09-12T23:33:27.830370+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.908185+0000","last_change":"2024-09-12T23:33:29.908185+0000","last_active":"2024-09-12T23:33:29.908185+0000","last_peered":"2024-09-12T23:33:29.908185+0000","last_clean":"2024-09-12T23:33:29.908185+0000","last_became_active":"2024-09-12T23:33:29.907814+0000","last_became_peered":"2024-09-12T23:33:29.907814+0000","last_unstale":"2024-09-12T23:33:29.908185+0000","last_undegraded":"2024-09-12T23:33:29.908185+0000","last_fullsized":"2024-09-12T23:33:29.908185+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:45:47.207242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.902988+0000","last_change":"2024-09-12T23:33:28.902988+0000","last_active":"2024-09-12T23:33:28.902988+0000","last_peered":"2024-09-12T23:33:28.902988+0000","last_clean":"2024-09-12T23:33:28.902988+0000","last_became_active":"2024-09-12T23:33:28.902514+0000","last_became_peered":"2024-09-12T23:33:28.902514+0000","last_unstale":"2024-09-12T23:33:28.902988+0000","last_undegraded":"2024-09-12T23:33:28.902988+0000","last_fullsized":"2024-09-12T23:33:28.902988+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:29:42.595057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.848452+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:12:37.049274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,4],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.866904+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:49:53.725680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[4,6],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.902224+0000","last_change":"2024-09-12T23:33:28.902224+0000","last_active":"2024-09-12T23:33:28.902224+0000","last_peered":"2024-09-12T23:33:28.902224+0000","last_clean":"2024-09-12T23:33:28.902224+0000","last_became_active":"2024-09-12T23:33:28.901755+0000","last_became_peered":"2024-09-12T23:33:28.901755+0000","last_unstale":"2024-09-12T23:33:28.902224+0000","last_undegraded":"2024-09-12T23:33:28.902224+0000","last_fullsized":"2024-09-12T23:33:28.902224+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:06:15.504175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.875336+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:20:24.609184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,4],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1d","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.897466+0000","last_change":"2024-09-12T23:33:28.897466+0000","last_active":"2024-09-12T23:33:28.897466+0000","last_peered":"2024-09-12T23:33:28.897466+0000","last_clean":"2024-09-12T23:33:28.897466+0000","last_became_active":"2024-09-12T23:33:28.896936+0000","last_became_peered":"2024-09-12T23:33:28.896936+0000","last_unstale":"2024-09-12T23:33:28.897466+0000","last_undegraded":"2024-09-12T23:33:28.897466+0000","last_fullsized":"2024-09-12T23:33:28.897466+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:29:38.732168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.1e","version":"53'1","reported_seq":11,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.875384+0000","last_change":"2024-09-12T23:33:28.894353+0000","last_active":"2024-09-12T23:33:29.875384+0000","last_peered":"2024-09-12T23:33:29.875384+0000","last_clean":"2024-09-12T23:33:29.875384+0000","last_became_active":"2024-09-12T23:33:28.893992+0000","last_became_peered":"2024-09-12T23:33:28.893992+0000","last_unstale":"2024-09-12T23:33:29.875384+0000","last_undegraded":"2024-09-12T23:33:29.875384+0000","last_fullsized":"2024-09-12T23:33:29.875384+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:43:43.889025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.891227+0000","last_change":"2024-09-12T23:33:29.891227+0000","last_active":"2024-09-12T23:33:29.891227+0000","last_peered":"2024-09-12T23:33:29.891227+0000","last_clean":"2024-09-12T23:33:29.891227+0000","last_became_active":"2024-09-12T23:33:29.890368+0000","last_became_peered":"2024-09-12T23:33:29.890368+0000","last_unstale":"2024-09-12T23:33:29.891227+0000","last_undegraded":"2024-09-12T23:33:29.891227+0000","last_fullsized":"2024-09-12T23:33:29.891227+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:38:37.526610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:27.830370+0000","last_change":"2024-09-12T23:33:27.837410+0000","last_active":"2024-09-12T23:33:27.830370+0000","last_peered":"2024-09-12T23:33:27.830370+0000","last_clean":"2024-09-12T23:33:27.830370+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:27.830370+0000","last_undegraded":"2024-09-12T23:33:27.830370+0000","last_fullsized":"2024-09-12T23:33:27.830370+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:42:19.518598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[4,7],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":129,"acting":129,"num_store_stats":5},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":1656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2,"ondisk_log_size":2,"up":78,"acting":78,"num_store_stats":7},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3563520,"data_stored":3542208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430219,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298432,"kb_used_data":1072,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446720,"statfs":{"total":95995035648,"available":95689441280,"internally_reserved":0,"allocated":1097728,"data_stored":796122,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.74299999999999999}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.69799999999999995}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47499999999999998}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.82099999999999995}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64000000000000001}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.90200000000000002}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.83899999999999997}]}]},{"osd":6,"up_from":42,"seq":180388626453,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298412,"kb_used_data":1056,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446740,"statfs":{"total":95995035648,"available":95689461760,"internally_reserved":0,"allocated":1081344,"data_stored":789114,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 12 23:32:56 2024","interfaces":[{"interface":"back","average":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"min":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":1.2450000000000001,"5min":1.2450000000000001,"15min":1.2450000000000001},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.874,"5min":0.874,"15min":0.874},"min":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"max":{"1min":1.222,"5min":1.222,"15min":1.222},"last":0.82099999999999995}]},{"osd":1,"last update":"Thu Sep 12 23:32:56 2024","interfaces":[{"interface":"back","average":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"min":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":1.153,"5min":1.153,"15min":1.153},"last":0.89800000000000002},{"interface":"front","average":{"1min":0.871,"5min":0.871,"15min":0.871},"min":{"1min":0.62,"5min":0.62,"15min":0.62},"max":{"1min":1.306,"5min":1.306,"15min":1.306},"last":1.1499999999999999}]},{"osd":2,"last update":"Thu Sep 12 23:32:56 2024","interfaces":[{"interface":"back","average":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"min":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"max":{"1min":1.4359999999999999,"5min":1.4359999999999999,"15min":1.4359999999999999},"last":0.76900000000000002},{"interface":"front","average":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"min":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"max":{"1min":1.131,"5min":1.131,"15min":1.131},"last":0.49199999999999999}]},{"osd":3,"last update":"Thu Sep 12 23:32:56 2024","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"min":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"max":{"1min":1.2330000000000001,"5min":1.2330000000000001,"15min":1.2330000000000001},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"min":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"max":{"1min":1.2290000000000001,"5min":1.2290000000000001,"15min":1.2290000000000001},"last":1.018}]},{"osd":4,"last update":"Thu Sep 12 23:32:56 2024","interfaces":[{"interface":"back","average":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"min":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":1.131,"5min":1.131,"15min":1.131},"last":0.95399999999999996},{"interface":"front","average":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":1.389,"5min":1.389,"15min":1.389},"last":0.97999999999999998}]},{"osd":5,"last update":"Thu Sep 12 23:32:56 2024","interfaces":[{"interface":"back","average":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"min":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"max":{"1min":1.228,"5min":1.228,"15min":1.228},"last":1.0780000000000001},{"interface":"front","average":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"min":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":1.3109999999999999,"5min":1.3109999999999999,"15min":1.3109999999999999},"last":1.105}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.139}]}]},{"osd":5,"up_from":36,"seq":154618822688,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297828,"kb_used_data":472,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447324,"statfs":{"total":95995035648,"available":95690059776,"internally_reserved":0,"allocated":483328,"data_stored":198431,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 12 23:33:07 2024","interfaces":[{"interface":"back","average":{"1min":0.81399999999999995,"5min":0.78700000000000003,"15min":0.78300000000000003},"min":{"1min":0.56200000000000006,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":1.0740000000000001,"5min":1.6100000000000001,"15min":1.6100000000000001},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.872,"5min":0.83399999999999996,"15min":0.82799999999999996},"min":{"1min":0.68999999999999995,"5min":0.58199999999999996,"15min":0.58199999999999996},"max":{"1min":1.0469999999999999,"5min":1.0840000000000001,"15min":1.0840000000000001},"last":0.56799999999999995}]},{"osd":1,"last update":"Thu Sep 12 23:33:07 2024","interfaces":[{"interface":"back","average":{"1min":0.88100000000000001,"5min":0.85199999999999998,"15min":0.84699999999999998},"min":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"max":{"1min":1.218,"5min":1.4470000000000001,"15min":1.4470000000000001},"last":0.81799999999999995},{"interface":"front","average":{"1min":0.89000000000000001,"5min":0.81200000000000006,"15min":0.79900000000000004},"min":{"1min":0.66300000000000003,"5min":0.502,"15min":0.502},"max":{"1min":1.274,"5min":1.274,"15min":1.274},"last":0.52800000000000002}]},{"osd":2,"last update":"Thu Sep 12 23:33:07 2024","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":0.878,"15min":0.87},"min":{"1min":0.73999999999999999,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":1.2509999999999999,"5min":1.2509999999999999,"15min":1.2509999999999999},"last":0.75},{"interface":"front","average":{"1min":0.876,"5min":0.90400000000000003,"15min":0.90900000000000003},"min":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"max":{"1min":1.0980000000000001,"5min":1.141,"15min":1.141},"last":0.77300000000000002}]},{"osd":3,"last update":"Thu Sep 12 23:33:07 2024","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":0.91300000000000003,"15min":0.91100000000000003},"min":{"1min":0.68100000000000005,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.2010000000000001,"5min":1.3280000000000001,"15min":1.3280000000000001},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.92000000000000004,"5min":0.91500000000000004,"15min":0.91400000000000003},"min":{"1min":0.67400000000000004,"5min":0.57499999999999996,"15min":0.57499999999999996},"max":{"1min":1.2609999999999999,"5min":1.536,"15min":1.536},"last":0.73399999999999999}]},{"osd":4,"last update":"Thu Sep 12 23:33:07 2024","interfaces":[{"interface":"back","average":{"1min":0.747,"5min":0.74299999999999999,"15min":0.74199999999999999},"min":{"1min":0.47199999999999998,"5min":0.308,"15min":0.308},"max":{"1min":1.1910000000000001,"5min":1.1910000000000001,"15min":1.1910000000000001},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.79500000000000004,"5min":0.77700000000000002,"15min":0.77400000000000002},"min":{"1min":0.504,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":1.1679999999999999,"5min":1.1679999999999999,"15min":1.1679999999999999},"last":0.67700000000000005}]},{"osd":6,"last update":"Thu Sep 12 23:32:54 2024","interfaces":[{"interface":"back","average":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":1.1220000000000001,"5min":1.1220000000000001,"15min":1.1220000000000001},"last":0.79700000000000004},{"interface":"front","average":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"min":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":1.0840000000000001,"5min":1.0840000000000001,"15min":1.0840000000000001},"last":0.65500000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.72099999999999997}]}]},{"osd":4,"up_from":30,"seq":128849018921,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297836,"kb_used_data":480,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447316,"statfs":{"total":95995035648,"available":95690051584,"internally_reserved":0,"allocated":491520,"data_stored":199061,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 12 23:33:17 2024","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":0.83599999999999997,"15min":0.83399999999999996},"min":{"1min":0.63200000000000001,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.375,"5min":1.375,"15min":1.375},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.89500000000000002,"5min":0.84699999999999998,"15min":0.84299999999999997},"min":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"max":{"1min":1.1479999999999999,"5min":1.1479999999999999,"15min":1.1479999999999999},"last":0.85999999999999999}]},{"osd":1,"last update":"Thu Sep 12 23:33:17 2024","interfaces":[{"interface":"back","average":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82399999999999995},"min":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":1.194,"5min":1.194,"15min":1.194},"last":1.01},{"interface":"front","average":{"1min":0.86899999999999999,"5min":0.85899999999999999,"15min":0.85699999999999998},"min":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.218,"5min":1.218,"15min":1.218},"last":0.80600000000000005}]},{"osd":2,"last update":"Thu Sep 12 23:33:17 2024","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":0.84599999999999997,"15min":0.85599999999999998},"min":{"1min":0.60199999999999998,"5min":0.55700000000000005,"15min":0.55700000000000005},"max":{"1min":1.175,"5min":1.175,"15min":1.175},"last":0.98599999999999999},{"interface":"front","average":{"1min":0.89500000000000002,"5min":0.85699999999999998,"15min":0.86299999999999999},"min":{"1min":0.66000000000000003,"5min":0.63500000000000001,"15min":0.63500000000000001},"max":{"1min":1.2909999999999999,"5min":1.2909999999999999,"15min":1.2909999999999999},"last":1.079}]},{"osd":3,"last update":"Thu Sep 12 23:33:17 2024","interfaces":[{"interface":"back","average":{"1min":0.89700000000000002,"5min":0.85199999999999998,"15min":0.84999999999999998},"min":{"1min":0.65200000000000002,"5min":0.60099999999999998,"15min":0.60099999999999998},"max":{"1min":1.5129999999999999,"5min":1.5129999999999999,"15min":1.5129999999999999},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.97899999999999998,"5min":0.90200000000000002,"15min":0.89000000000000001},"min":{"1min":0.57099999999999995,"5min":0.55900000000000005,"15min":0.55900000000000005},"max":{"1min":1.595,"5min":1.595,"15min":1.595},"last":1.214}]},{"osd":5,"last update":"Thu Sep 12 23:33:09 2024","interfaces":[{"interface":"back","average":{"1min":0.80500000000000005,"5min":0.78400000000000003,"15min":0.78000000000000003},"min":{"1min":0.49199999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":1.7430000000000001,"5min":1.7430000000000001,"15min":1.7430000000000001},"last":0.91200000000000003},{"interface":"front","average":{"1min":0.90300000000000002,"5min":0.82199999999999995,"15min":0.80800000000000005},"min":{"1min":0.375,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.716,"5min":1.716,"15min":1.716},"last":1.0489999999999999}]},{"osd":6,"last update":"Thu Sep 12 23:32:56 2024","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":0.91300000000000003,"15min":0.91300000000000003},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":1.7629999999999999,"5min":1.7629999999999999,"15min":1.7629999999999999},"last":1.1140000000000001},{"interface":"front","average":{"1min":0.96299999999999997,"5min":0.96299999999999997,"15min":0.96299999999999997},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":1.7010000000000001,"5min":1.7010000000000001,"15min":1.7010000000000001},"last":1.2849999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.1379999999999999}]}]},{"osd":3,"up_from":25,"seq":107374182451,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297880,"kb_used_data":524,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447272,"statfs":{"total":95995035648,"available":95690006528,"internally_reserved":0,"allocated":536576,"data_stored":214603,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 12 23:32:29 2024","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.62,"15min":0.60199999999999998},"min":{"1min":0.53400000000000003,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.995,"5min":0.999,"15min":0.999},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.78300000000000003,"5min":0.70499999999999996,"15min":0.69299999999999995},"min":{"1min":0.58299999999999996,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":1.3120000000000001,"5min":1.3120000000000001,"15min":1.3120000000000001},"last":0.46600000000000003}]},{"osd":1,"last update":"Thu Sep 12 23:32:29 2024","interfaces":[{"interface":"back","average":{"1min":0.755,"5min":0.67200000000000004,"15min":0.65600000000000003},"min":{"1min":0.54200000000000004,"5min":0.52200000000000002,"15min":0.52200000000000002},"max":{"1min":0.96999999999999997,"5min":0.96999999999999997,"15min":0.96999999999999997},"last":0.88600000000000001},{"interface":"front","average":{"1min":0.746,"5min":0.75,"15min":0.74199999999999999},"min":{"1min":0.57199999999999995,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":1.095,"5min":1.171,"15min":1.171},"last":0.61899999999999999}]},{"osd":2,"last update":"Thu Sep 12 23:32:29 2024","interfaces":[{"interface":"back","average":{"1min":0.81999999999999995,"5min":0.78900000000000003,"15min":0.77700000000000002},"min":{"1min":0.66100000000000003,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":0.96799999999999997,"5min":1.1140000000000001,"15min":1.1140000000000001},"last":0.78800000000000003},{"interface":"front","average":{"1min":0.85299999999999998,"5min":0.75800000000000001,"15min":0.73399999999999999},"min":{"1min":0.63800000000000001,"5min":0.54900000000000004,"15min":0.54900000000000004},"max":{"1min":1.238,"5min":1.238,"15min":1.238},"last":0.56299999999999994}]},{"osd":4,"last update":"Thu Sep 12 23:33:24 2024","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":0.90300000000000002,"15min":0.88200000000000001},"min":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.4950000000000001,"5min":1.4950000000000001,"15min":1.4950000000000001},"last":0.86099999999999999},{"interface":"front","average":{"1min":0.90300000000000002,"5min":0.87,"15min":0.84899999999999998},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":1.3640000000000001,"5min":1.3640000000000001,"15min":1.3640000000000001},"last":0.98299999999999998}]},{"osd":5,"last update":"Thu Sep 12 23:33:05 2024","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":0.98399999999999999,"15min":0.98399999999999999},"min":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"max":{"1min":1.321,"5min":1.371,"15min":1.371},"last":0.72499999999999998},{"interface":"front","average":{"1min":1.05,"5min":0.98199999999999998,"15min":0.97099999999999997},"min":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":1.423,"5min":1.423,"15min":1.423},"last":0.65600000000000003}]},{"osd":6,"last update":"Thu Sep 12 23:32:54 2024","interfaces":[{"interface":"back","average":{"1min":1.052,"5min":1.052,"15min":1.052},"min":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"max":{"1min":1.3540000000000001,"5min":1.3540000000000001,"15min":1.3540000000000001},"last":0.94499999999999995},{"interface":"front","average":{"1min":1.115,"5min":1.115,"15min":1.115},"min":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"max":{"1min":1.718,"5min":1.718,"15min":1.718},"last":0.96299999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.90800000000000003}]}]},{"osd":2,"up_from":18,"seq":77309411389,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297836,"kb_used_data":480,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447316,"statfs":{"total":95995035648,"available":95690051584,"internally_reserved":0,"allocated":491520,"data_stored":199061,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 12 23:32:41 2024","interfaces":[{"interface":"back","average":{"1min":0.72999999999999998,"5min":0.626,"15min":0.58799999999999997},"min":{"1min":0.53800000000000003,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":1.0429999999999999,"5min":1.23,"15min":1.23},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.745,"5min":0.66400000000000003,"15min":0.61299999999999999},"min":{"1min":0.50800000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":1.0940000000000001,"5min":1.0940000000000001,"15min":1.0940000000000001},"last":0.38300000000000001}]},{"osd":1,"last update":"Thu Sep 12 23:32:41 2024","interfaces":[{"interface":"back","average":{"1min":0.76400000000000001,"5min":0.67700000000000005,"15min":0.628},"min":{"1min":0.5,"5min":0.316,"15min":0.316},"max":{"1min":1.014,"5min":1.5329999999999999,"15min":1.5329999999999999},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.74199999999999999,"5min":0.68400000000000005,"15min":0.63700000000000001},"min":{"1min":0.60999999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.95099999999999996,"5min":1.0860000000000001,"15min":1.0860000000000001},"last":0.44}]},{"osd":3,"last update":"Thu Sep 12 23:32:33 2024","interfaces":[{"interface":"back","average":{"1min":0.80000000000000004,"5min":0.75800000000000001,"15min":0.74399999999999999},"min":{"1min":0.627,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":1.2270000000000001,"5min":1.2270000000000001,"15min":1.2270000000000001},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.83299999999999996,"5min":0.79800000000000004,"15min":0.78000000000000003},"min":{"1min":0.623,"5min":0.433,"15min":0.433},"max":{"1min":1.0049999999999999,"5min":1.6619999999999999,"15min":1.6619999999999999},"last":0.36599999999999999}]},{"osd":4,"last update":"Thu Sep 12 23:33:19 2024","interfaces":[{"interface":"back","average":{"1min":0.88300000000000001,"5min":0.873,"15min":0.86499999999999999},"min":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.107,"5min":1.1639999999999999,"15min":1.1639999999999999},"last":0.82499999999999996},{"interface":"front","average":{"1min":0.96899999999999997,"5min":0.95299999999999996,"15min":0.93500000000000005},"min":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"max":{"1min":1.23,"5min":1.363,"15min":1.363},"last":0.79000000000000004}]},{"osd":5,"last update":"Thu Sep 12 23:33:10 2024","interfaces":[{"interface":"back","average":{"1min":0.95599999999999996,"5min":1.022,"15min":1.0329999999999999},"min":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"max":{"1min":1.282,"5min":1.4830000000000001,"15min":1.4830000000000001},"last":0.89700000000000002},{"interface":"front","average":{"1min":0.90700000000000003,"5min":1.006,"15min":1.022},"min":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"max":{"1min":1.101,"5min":1.3129999999999999,"15min":1.3129999999999999},"last":0.63900000000000001}]},{"osd":6,"last update":"Thu Sep 12 23:32:56 2024","interfaces":[{"interface":"back","average":{"1min":0.94499999999999995,"5min":0.94499999999999995,"15min":0.94499999999999995},"min":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"max":{"1min":1.2330000000000001,"5min":1.2330000000000001,"15min":1.2330000000000001},"last":0.98099999999999998},{"interface":"front","average":{"1min":1.0880000000000001,"5min":1.0880000000000001,"15min":1.0880000000000001},"min":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"max":{"1min":1.498,"5min":1.498,"15min":1.498},"last":0.67000000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67800000000000005}]}]},{"osd":1,"up_from":13,"seq":55834574919,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":297824,"kb_used_data":468,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447328,"statfs":{"total":95995035648,"available":95690063872,"internally_reserved":0,"allocated":479232,"data_stored":192826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 12 23:32:55 2024","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":0.59899999999999998,"15min":0.53000000000000003},"min":{"1min":0.51400000000000001,"5min":0.255,"15min":0.255},"max":{"1min":1.1579999999999999,"5min":1.1579999999999999,"15min":1.1579999999999999},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.77400000000000002,"5min":0.66000000000000003,"15min":0.58199999999999996},"min":{"1min":0.501,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":1.1830000000000001,"5min":1.1830000000000001,"15min":1.1830000000000001},"last":1.1080000000000001}]},{"osd":2,"last update":"Thu Sep 12 23:32:40 2024","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":0.64600000000000002,"15min":0.58799999999999997},"min":{"1min":0.54700000000000004,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.95899999999999996,"5min":1.046,"15min":1.046},"last":1.238},{"interface":"front","average":{"1min":0.82899999999999996,"5min":0.69999999999999996,"15min":0.629},"min":{"1min":0.53200000000000003,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":1.3049999999999999,"5min":1.3049999999999999,"15min":1.3049999999999999},"last":0.94799999999999995}]},{"osd":3,"last update":"Thu Sep 12 23:32:34 2024","interfaces":[{"interface":"back","average":{"1min":0.81999999999999995,"5min":0.79000000000000004,"15min":0.78100000000000003},"min":{"1min":0.56999999999999995,"5min":0.54800000000000004,"15min":0.54800000000000004},"max":{"1min":1.0249999999999999,"5min":1.1479999999999999,"15min":1.1479999999999999},"last":1.198},{"interface":"front","average":{"1min":0.81299999999999994,"5min":0.77100000000000002,"15min":0.76200000000000001},"min":{"1min":0.61499999999999999,"5min":0.57799999999999996,"15min":0.57799999999999996},"max":{"1min":1.109,"5min":1.109,"15min":1.109},"last":0.83899999999999997}]},{"osd":4,"last update":"Thu Sep 12 23:33:16 2024","interfaces":[{"interface":"back","average":{"1min":0.95999999999999996,"5min":0.92100000000000004,"15min":0.91700000000000004},"min":{"1min":0.75700000000000001,"5min":0.65100000000000002,"15min":0.65100000000000002},"max":{"1min":1.23,"5min":1.585,"15min":1.585},"last":0.78900000000000003},{"interface":"front","average":{"1min":1.006,"5min":0.93700000000000006,"15min":0.92700000000000005},"min":{"1min":0.76000000000000001,"5min":0.73499999999999999,"15min":0.73499999999999999},"max":{"1min":1.389,"5min":1.4279999999999999,"15min":1.4279999999999999},"last":0.68999999999999995}]},{"osd":5,"last update":"Thu Sep 12 23:33:05 2024","interfaces":[{"interface":"back","average":{"1min":0.95399999999999996,"5min":0.92100000000000004,"15min":0.91500000000000004},"min":{"1min":0.66000000000000003,"5min":0.65200000000000002,"15min":0.65200000000000002},"max":{"1min":1.484,"5min":1.6819999999999999,"15min":1.6819999999999999},"last":0.63400000000000001},{"interface":"front","average":{"1min":1.0289999999999999,"5min":1.006,"15min":1.002},"min":{"1min":0.81000000000000005,"5min":0.66200000000000003,"15min":0.66200000000000003},"max":{"1min":1.5069999999999999,"5min":1.5069999999999999,"15min":1.5069999999999999},"last":0.72799999999999998}]},{"osd":6,"last update":"Thu Sep 12 23:32:55 2024","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":1.0529999999999999,"15min":1.0529999999999999},"min":{"1min":0.746,"5min":0.746,"15min":0.746},"max":{"1min":1.5429999999999999,"5min":1.5429999999999999,"15min":1.5429999999999999},"last":0.97899999999999998},{"interface":"front","average":{"1min":1.101,"5min":1.101,"15min":1.101},"min":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"max":{"1min":1.5649999999999999,"5min":1.5649999999999999,"15min":1.5649999999999999},"last":0.86899999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.0720000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738448,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298412,"kb_used_data":1056,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446740,"statfs":{"total":95995035648,"available":95689461760,"internally_reserved":0,"allocated":1081344,"data_stored":789114,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Sep 12 23:32:49 2024","interfaces":[{"interface":"back","average":{"1min":0.69599999999999995,"5min":0.59899999999999998,"15min":0.51100000000000001},"min":{"1min":0.48499999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":1.0509999999999999,"5min":1.0509999999999999,"15min":1.0509999999999999},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.74399999999999999,"5min":0.629,"15min":0.53300000000000003},"min":{"1min":0.55400000000000005,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.0049999999999999,"5min":1.028,"15min":1.028},"last":0.48699999999999999}]},{"osd":2,"last update":"Thu Sep 12 23:32:43 2024","interfaces":[{"interface":"back","average":{"1min":0.73299999999999998,"5min":0.66600000000000004,"15min":0.61899999999999999},"min":{"1min":0.52800000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":1.081,"5min":1.081,"15min":1.081},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.82699999999999996,"5min":0.69599999999999995,"15min":0.63900000000000001},"min":{"1min":0.56000000000000005,"5min":0.317,"15min":0.317},"max":{"1min":1.151,"5min":1.151,"15min":1.151},"last":0.81000000000000005}]},{"osd":3,"last update":"Thu Sep 12 23:32:27 2024","interfaces":[{"interface":"back","average":{"1min":0.79900000000000004,"5min":0.76800000000000002,"15min":0.76400000000000001},"min":{"1min":0.60999999999999999,"5min":0.60599999999999998,"15min":0.60599999999999998},"max":{"1min":1.177,"5min":1.177,"15min":1.177},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.83599999999999997,"5min":0.79600000000000004,"15min":0.79200000000000004},"min":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"max":{"1min":1.0009999999999999,"5min":1.1950000000000001,"15min":1.1950000000000001},"last":0.68400000000000005}]},{"osd":4,"last update":"Thu Sep 12 23:33:23 2024","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":0.93100000000000005,"15min":0.90200000000000002},"min":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"max":{"1min":1.304,"5min":1.304,"15min":1.304},"last":0.79000000000000004},{"interface":"front","average":{"1min":0.95699999999999996,"5min":0.876,"15min":0.84999999999999998},"min":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"max":{"1min":1.266,"5min":1.266,"15min":1.266},"last":0.67200000000000004}]},{"osd":5,"last update":"Thu Sep 12 23:33:10 2024","interfaces":[{"interface":"back","average":{"1min":0.94499999999999995,"5min":0.94299999999999995,"15min":0.94299999999999995},"min":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"max":{"1min":1.2330000000000001,"5min":1.2330000000000001,"15min":1.2330000000000001},"last":0.64200000000000002},{"interface":"front","average":{"1min":1.002,"5min":0.95899999999999996,"15min":0.95199999999999996},"min":{"1min":0.75,"5min":0.70099999999999996,"15min":0.70099999999999996},"max":{"1min":1.1579999999999999,"5min":1.1579999999999999,"15min":1.1579999999999999},"last":0.749}]},{"osd":6,"last update":"Thu Sep 12 23:32:55 2024","interfaces":[{"interface":"back","average":{"1min":1.1120000000000001,"5min":1.1120000000000001,"15min":1.1120000000000001},"min":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"max":{"1min":1.492,"5min":1.492,"15min":1.492},"last":0.98199999999999998},{"interface":"front","average":{"1min":1.0509999999999999,"5min":1.0509999999999999,"15min":1.0509999999999999},"min":{"1min":0.754,"5min":0.754,"15min":0.754},"max":{"1min":1.377,"5min":1.377,"15min":1.377},"last":0.90700000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.96699999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":1656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-12T23:33:33.351 DEBUG:teuthology.orchestra.run.smithi136:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-09-12T23:33:33.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:32 smithi181 bash[24776]: cluster 2024-09-12T23:33:31.489162+0000 mgr.y (mgr.14150) 340 : cluster [DBG] pgmap v317: 97 pgs: 5 creating+activating, 35 creating+peering, 27 unknown, 30 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:33.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:32 smithi181 bash[24776]: cluster 2024-09-12T23:33:32.367695+0000 mon.a (mon.0) 686 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 2 pool(s) do not have an application enabled) 2024-09-12T23:33:33.352 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:32 smithi181 bash[24776]: cluster 2024-09-12T23:33:32.367744+0000 mon.a (mon.0) 687 : cluster [INF] Cluster is now healthy 2024-09-12T23:33:33.352 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:32 smithi181 bash[24776]: audit 2024-09-12T23:33:32.380673+0000 mon.a (mon.0) 688 : audit [INF] from='client.? 172.21.15.136:0/1259745480' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]': finished 2024-09-12T23:33:33.352 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:32 smithi181 bash[24776]: cluster 2024-09-12T23:33:32.380856+0000 mon.a (mon.0) 689 : cluster [DBG] fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-12T23:33:33.352 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:32 smithi181 bash[24776]: audit 2024-09-12T23:33:32.720015+0000 mon.a (mon.0) 690 : audit [DBG] from='client.? 172.21.15.136:0/783031366' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-12T23:33:33.691 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-12T23:33:33.693 INFO:teuthology.orchestra.run.smithi136.stderr:dumped all 2024-09-12T23:33:33.708 INFO:teuthology.orchestra.run.smithi136.stdout:{"pg_ready":true,"pg_map":{"version":318,"stamp":"2024-09-12T23:33:33.489398+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590528,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":106,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":78,"ondisk_log_size":78,"up":246,"acting":246,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":59,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2384512,"kb_used_data":5652,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747576704,"statfs":{"total":767960285184,"available":765518544896,"internally_reserved":0,"allocated":5787648,"data_stored":3398613,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":160,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.654574"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.861074+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:54:27.307253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,6],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.916927+0000","last_change":"2024-09-12T23:33:29.907919+0000","last_active":"2024-09-12T23:33:29.916927+0000","last_peered":"2024-09-12T23:33:29.916927+0000","last_clean":"2024-09-12T23:33:29.916927+0000","last_became_active":"2024-09-12T23:33:29.906765+0000","last_became_peered":"2024-09-12T23:33:29.906765+0000","last_unstale":"2024-09-12T23:33:29.916927+0000","last_undegraded":"2024-09-12T23:33:29.916927+0000","last_fullsized":"2024-09-12T23:33:29.916927+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:39:49.787701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.876654+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:18:46.573142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[3,7],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.859085+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:07:03.294377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[2,4],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.873576+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:07:11.163630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,1],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.850034+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:17:57.421440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,4],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.902033+0000","last_change":"2024-09-12T23:33:29.902033+0000","last_active":"2024-09-12T23:33:29.902033+0000","last_peered":"2024-09-12T23:33:29.902033+0000","last_clean":"2024-09-12T23:33:29.902033+0000","last_became_active":"2024-09-12T23:33:29.901611+0000","last_became_peered":"2024-09-12T23:33:29.901611+0000","last_unstale":"2024-09-12T23:33:29.902033+0000","last_undegraded":"2024-09-12T23:33:29.902033+0000","last_fullsized":"2024-09-12T23:33:29.902033+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:57:58.671558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.856686+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:41:19.476191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,5],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.852231+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:56:38.506429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,5],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.855386+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:08:21.774244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,5],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.907459+0000","last_change":"2024-09-12T23:33:29.907459+0000","last_active":"2024-09-12T23:33:29.907459+0000","last_peered":"2024-09-12T23:33:29.907459+0000","last_clean":"2024-09-12T23:33:29.907459+0000","last_became_active":"2024-09-12T23:33:29.906577+0000","last_became_peered":"2024-09-12T23:33:29.906577+0000","last_unstale":"2024-09-12T23:33:29.907459+0000","last_undegraded":"2024-09-12T23:33:29.907459+0000","last_fullsized":"2024-09-12T23:33:29.907459+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:21:41.131616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.898912+0000","last_change":"2024-09-12T23:33:29.894796+0000","last_active":"2024-09-12T23:33:29.898912+0000","last_peered":"2024-09-12T23:33:29.898912+0000","last_clean":"2024-09-12T23:33:29.898912+0000","last_became_active":"2024-09-12T23:33:29.894498+0000","last_became_peered":"2024-09-12T23:33:29.894498+0000","last_unstale":"2024-09-12T23:33:29.898912+0000","last_undegraded":"2024-09-12T23:33:29.898912+0000","last_fullsized":"2024-09-12T23:33:29.898912+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:12:10.219015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.867621+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:53:38.724701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,7],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.866959+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-13T23:33:43.071267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[4,7],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.849688+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:08:59.758427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[6,7],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.849822+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:08:17.677941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[2,6],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.849372+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:43:43.192749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,4],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.870650+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:34:34.650312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[6,7],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.906087+0000","last_change":"2024-09-12T23:33:29.906087+0000","last_active":"2024-09-12T23:33:29.906087+0000","last_peered":"2024-09-12T23:33:29.906087+0000","last_clean":"2024-09-12T23:33:29.906087+0000","last_became_active":"2024-09-12T23:33:29.905780+0000","last_became_peered":"2024-09-12T23:33:29.905780+0000","last_unstale":"2024-09-12T23:33:29.906087+0000","last_undegraded":"2024-09-12T23:33:29.906087+0000","last_fullsized":"2024-09-12T23:33:29.906087+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:00:40.777941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.855239+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:05:56.032960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[2,3],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.903381+0000","last_change":"2024-09-12T23:33:29.903381+0000","last_active":"2024-09-12T23:33:29.903381+0000","last_peered":"2024-09-12T23:33:29.903381+0000","last_clean":"2024-09-12T23:33:29.903381+0000","last_became_active":"2024-09-12T23:33:29.903081+0000","last_became_peered":"2024-09-12T23:33:29.903081+0000","last_unstale":"2024-09-12T23:33:29.903381+0000","last_undegraded":"2024-09-12T23:33:29.903381+0000","last_fullsized":"2024-09-12T23:33:29.903381+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:43:01.175691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.850972+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:39:14.674234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[4,7],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.920779+0000","last_change":"2024-09-12T23:33:29.919500+0000","last_active":"2024-09-12T23:33:29.920779+0000","last_peered":"2024-09-12T23:33:29.920779+0000","last_clean":"2024-09-12T23:33:29.920779+0000","last_became_active":"2024-09-12T23:33:29.919185+0000","last_became_peered":"2024-09-12T23:33:29.919185+0000","last_unstale":"2024-09-12T23:33:29.920779+0000","last_undegraded":"2024-09-12T23:33:29.920779+0000","last_fullsized":"2024-09-12T23:33:29.920779+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:56:52.566524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.913907+0000","last_change":"2024-09-12T23:33:29.913907+0000","last_active":"2024-09-12T23:33:29.913907+0000","last_peered":"2024-09-12T23:33:29.913907+0000","last_clean":"2024-09-12T23:33:29.913907+0000","last_became_active":"2024-09-12T23:33:29.913193+0000","last_became_peered":"2024-09-12T23:33:29.913193+0000","last_unstale":"2024-09-12T23:33:29.913907+0000","last_undegraded":"2024-09-12T23:33:29.913907+0000","last_fullsized":"2024-09-12T23:33:29.913907+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:32:45.117056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.869503+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:25:33.906182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,3],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.850701+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:38:56.219241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[5,6],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.852156+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:09:34.637115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,5],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.1b","version":"53'1","reported_seq":11,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.878981+0000","last_change":"2024-09-12T23:33:28.883222+0000","last_active":"2024-09-12T23:33:29.878981+0000","last_peered":"2024-09-12T23:33:29.878981+0000","last_clean":"2024-09-12T23:33:29.878981+0000","last_became_active":"2024-09-12T23:33:28.882848+0000","last_became_peered":"2024-09-12T23:33:28.882848+0000","last_unstale":"2024-09-12T23:33:29.878981+0000","last_undegraded":"2024-09-12T23:33:29.878981+0000","last_fullsized":"2024-09-12T23:33:29.878981+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:57:33.905996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.1a","version":"0'0","reported_seq":2,"reported_epoch":52,"state":"creating+activating","last_fresh":"2024-09-12T23:33:28.846682+0000","last_change":"2024-09-12T23:33:28.847145+0000","last_active":"2024-09-12T23:33:27.830370+0000","last_peered":"2024-09-12T23:33:27.830370+0000","last_clean":"2024-09-12T23:33:27.830370+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.846682+0000","last_undegraded":"2024-09-12T23:33:28.846682+0000","last_fullsized":"2024-09-12T23:33:28.846682+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:57:50.175287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":["6","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":10,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.873141+0000","last_change":"2024-09-12T23:33:28.872822+0000","last_active":"2024-09-12T23:33:28.873141+0000","last_peered":"2024-09-12T23:33:28.873141+0000","last_clean":"2024-09-12T23:33:28.873141+0000","last_became_active":"2024-09-12T23:33:28.872404+0000","last_became_peered":"2024-09-12T23:33:28.872404+0000","last_unstale":"2024-09-12T23:33:28.873141+0000","last_undegraded":"2024-09-12T23:33:28.873141+0000","last_fullsized":"2024-09-12T23:33:28.873141+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-13T23:47:42.317408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.907360+0000","last_change":"2024-09-12T23:33:29.907360+0000","last_active":"2024-09-12T23:33:29.907360+0000","last_peered":"2024-09-12T23:33:29.907360+0000","last_clean":"2024-09-12T23:33:29.907360+0000","last_became_active":"2024-09-12T23:33:29.907067+0000","last_became_peered":"2024-09-12T23:33:29.907067+0000","last_unstale":"2024-09-12T23:33:29.907360+0000","last_undegraded":"2024-09-12T23:33:29.907360+0000","last_fullsized":"2024-09-12T23:33:29.907360+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:26:38.084938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.896704+0000","last_change":"2024-09-12T23:33:28.896704+0000","last_active":"2024-09-12T23:33:28.896704+0000","last_peered":"2024-09-12T23:33:28.896704+0000","last_clean":"2024-09-12T23:33:28.896704+0000","last_became_active":"2024-09-12T23:33:28.896167+0000","last_became_peered":"2024-09-12T23:33:28.896167+0000","last_unstale":"2024-09-12T23:33:28.896704+0000","last_undegraded":"2024-09-12T23:33:28.896704+0000","last_fullsized":"2024-09-12T23:33:28.896704+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:14:47.959501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.894866+0000","last_change":"2024-09-12T23:33:29.893401+0000","last_active":"2024-09-12T23:33:29.894866+0000","last_peered":"2024-09-12T23:33:29.894866+0000","last_clean":"2024-09-12T23:33:29.894866+0000","last_became_active":"2024-09-12T23:33:29.892988+0000","last_became_peered":"2024-09-12T23:33:29.892988+0000","last_unstale":"2024-09-12T23:33:29.894866+0000","last_undegraded":"2024-09-12T23:33:29.894866+0000","last_fullsized":"2024-09-12T23:33:29.894866+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:30:04.271132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":3,"reported_epoch":52,"state":"creating+activating","last_fresh":"2024-09-12T23:33:28.848624+0000","last_change":"2024-09-12T23:33:28.848624+0000","last_active":"2024-09-12T23:33:27.830370+0000","last_peered":"2024-09-12T23:33:27.830370+0000","last_clean":"2024-09-12T23:33:27.830370+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.848624+0000","last_undegraded":"2024-09-12T23:33:28.848624+0000","last_fullsized":"2024-09-12T23:33:28.848624+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:20:27.774655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":["6","2","5"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.866433+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-13T23:52:12.475067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,7],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.890012+0000","last_change":"2024-09-12T23:33:28.890012+0000","last_active":"2024-09-12T23:33:28.890012+0000","last_peered":"2024-09-12T23:33:28.890012+0000","last_clean":"2024-09-12T23:33:28.890012+0000","last_became_active":"2024-09-12T23:33:28.887477+0000","last_became_peered":"2024-09-12T23:33:28.887477+0000","last_unstale":"2024-09-12T23:33:28.890012+0000","last_undegraded":"2024-09-12T23:33:28.890012+0000","last_fullsized":"2024-09-12T23:33:28.890012+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:15:02.988389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.15","version":"53'3","reported_seq":14,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.917038+0000","last_change":"2024-09-12T23:33:28.879732+0000","last_active":"2024-09-12T23:33:29.917038+0000","last_peered":"2024-09-12T23:33:29.917038+0000","last_clean":"2024-09-12T23:33:29.917038+0000","last_became_active":"2024-09-12T23:33:28.879399+0000","last_became_peered":"2024-09-12T23:33:28.879399+0000","last_unstale":"2024-09-12T23:33:29.917038+0000","last_undegraded":"2024-09-12T23:33:29.917038+0000","last_fullsized":"2024-09-12T23:33:29.917038+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:00:06.109352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.14","version":"0'0","reported_seq":2,"reported_epoch":52,"state":"creating+activating","last_fresh":"2024-09-12T23:33:28.847672+0000","last_change":"2024-09-12T23:33:28.848228+0000","last_active":"2024-09-12T23:33:27.830370+0000","last_peered":"2024-09-12T23:33:27.830370+0000","last_clean":"2024-09-12T23:33:27.830370+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.847672+0000","last_undegraded":"2024-09-12T23:33:28.847672+0000","last_fullsized":"2024-09-12T23:33:28.847672+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:50:58.477301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":["6","3","5"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.858409+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:06:00.373844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[3,4],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.13","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:27.830370+0000","last_change":"2024-09-12T23:33:27.837333+0000","last_active":"2024-09-12T23:33:27.830370+0000","last_peered":"2024-09-12T23:33:27.830370+0000","last_clean":"2024-09-12T23:33:27.830370+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:27.830370+0000","last_undegraded":"2024-09-12T23:33:27.830370+0000","last_fullsized":"2024-09-12T23:33:27.830370+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:00:55.966645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[3,4],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.899424+0000","last_change":"2024-09-12T23:33:28.899424+0000","last_active":"2024-09-12T23:33:28.899424+0000","last_peered":"2024-09-12T23:33:28.899424+0000","last_clean":"2024-09-12T23:33:28.899424+0000","last_became_active":"2024-09-12T23:33:28.898942+0000","last_became_peered":"2024-09-12T23:33:28.898942+0000","last_unstale":"2024-09-12T23:33:28.899424+0000","last_undegraded":"2024-09-12T23:33:28.899424+0000","last_fullsized":"2024-09-12T23:33:28.899424+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:14:47.069896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.847369+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:28:55.241684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[2,4],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":3,"reported_epoch":52,"state":"creating+activating","last_fresh":"2024-09-12T23:33:28.846921+0000","last_change":"2024-09-12T23:33:28.846921+0000","last_active":"2024-09-12T23:33:27.830370+0000","last_peered":"2024-09-12T23:33:27.830370+0000","last_clean":"2024-09-12T23:33:27.830370+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.846921+0000","last_undegraded":"2024-09-12T23:33:28.846921+0000","last_fullsized":"2024-09-12T23:33:28.846921+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:27:43.449941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":["6","1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.847037+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:29:56.772001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,5],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.863788+0000","last_change":"2024-09-12T23:33:28.863788+0000","last_active":"2024-09-12T23:33:28.863788+0000","last_peered":"2024-09-12T23:33:28.863788+0000","last_clean":"2024-09-12T23:33:28.863788+0000","last_became_active":"2024-09-12T23:33:28.863479+0000","last_became_peered":"2024-09-12T23:33:28.863479+0000","last_unstale":"2024-09-12T23:33:28.863788+0000","last_undegraded":"2024-09-12T23:33:28.863788+0000","last_fullsized":"2024-09-12T23:33:28.863788+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:26:09.204038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.847091+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:57:03.461858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[4,6],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:27.830370+0000","last_change":"2024-09-12T23:33:27.840289+0000","last_active":"2024-09-12T23:33:27.830370+0000","last_peered":"2024-09-12T23:33:27.830370+0000","last_clean":"2024-09-12T23:33:27.830370+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:27.830370+0000","last_undegraded":"2024-09-12T23:33:27.830370+0000","last_fullsized":"2024-09-12T23:33:27.830370+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:54:14.860070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,7],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.849355+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:45:39.430174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,4],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.887441+0000","last_change":"2024-09-12T23:33:28.887441+0000","last_active":"2024-09-12T23:33:28.887441+0000","last_peered":"2024-09-12T23:33:28.887441+0000","last_clean":"2024-09-12T23:33:28.887441+0000","last_became_active":"2024-09-12T23:33:28.887109+0000","last_became_peered":"2024-09-12T23:33:28.887109+0000","last_unstale":"2024-09-12T23:33:28.887441+0000","last_undegraded":"2024-09-12T23:33:28.887441+0000","last_fullsized":"2024-09-12T23:33:28.887441+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:35:14.071342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.859939+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:06:23.076336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,4],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"53'1","reported_seq":12,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.920461+0000","last_change":"2024-09-12T23:33:28.871944+0000","last_active":"2024-09-12T23:33:29.920461+0000","last_peered":"2024-09-12T23:33:29.920461+0000","last_clean":"2024-09-12T23:33:29.920461+0000","last_became_active":"2024-09-12T23:33:28.871628+0000","last_became_peered":"2024-09-12T23:33:28.871628+0000","last_unstale":"2024-09-12T23:33:29.920461+0000","last_undegraded":"2024-09-12T23:33:29.920461+0000","last_fullsized":"2024-09-12T23:33:29.920461+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:58:29.645624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.874292+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:42:32.768139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[3,6],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.c","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.882138+0000","last_change":"2024-09-12T23:33:28.882138+0000","last_active":"2024-09-12T23:33:28.882138+0000","last_peered":"2024-09-12T23:33:28.882138+0000","last_clean":"2024-09-12T23:33:28.882138+0000","last_became_active":"2024-09-12T23:33:28.881774+0000","last_became_peered":"2024-09-12T23:33:28.881774+0000","last_unstale":"2024-09-12T23:33:28.882138+0000","last_undegraded":"2024-09-12T23:33:28.882138+0000","last_fullsized":"2024-09-12T23:33:28.882138+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:07:36.075590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.857274+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:24:29.251601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[5,6],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"21'72","reported_seq":18,"reported_epoch":51,"state":"active+clean","last_fresh":"2024-09-12T23:33:27.838959+0000","last_change":"2024-09-12T23:32:45.902460+0000","last_active":"2024-09-12T23:33:27.838959+0000","last_peered":"2024-09-12T23:33:27.838959+0000","last_clean":"2024-09-12T23:33:27.838959+0000","last_became_active":"2024-09-12T23:32:45.887205+0000","last_became_peered":"2024-09-12T23:32:45.887205+0000","last_unstale":"2024-09-12T23:33:27.838959+0000","last_undegraded":"2024-09-12T23:33:27.838959+0000","last_fullsized":"2024-09-12T23:33:27.838959+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:28:34.324093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:28:34.324093+0000","last_clean_scrub_stamp":"2024-09-12T23:28:34.324093+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:37:37.825919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.898796+0000","last_change":"2024-09-12T23:33:28.898796+0000","last_active":"2024-09-12T23:33:28.898796+0000","last_peered":"2024-09-12T23:33:28.898796+0000","last_clean":"2024-09-12T23:33:28.898796+0000","last_became_active":"2024-09-12T23:33:28.898329+0000","last_became_peered":"2024-09-12T23:33:28.898329+0000","last_unstale":"2024-09-12T23:33:28.898796+0000","last_undegraded":"2024-09-12T23:33:28.898796+0000","last_fullsized":"2024-09-12T23:33:28.898796+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:54:48.474286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.905028+0000","last_change":"2024-09-12T23:33:29.905028+0000","last_active":"2024-09-12T23:33:29.905028+0000","last_peered":"2024-09-12T23:33:29.905028+0000","last_clean":"2024-09-12T23:33:29.905028+0000","last_became_active":"2024-09-12T23:33:29.904695+0000","last_became_peered":"2024-09-12T23:33:29.904695+0000","last_unstale":"2024-09-12T23:33:29.905028+0000","last_undegraded":"2024-09-12T23:33:29.905028+0000","last_fullsized":"2024-09-12T23:33:29.905028+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:06:47.004852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.895135+0000","last_change":"2024-09-12T23:33:28.895135+0000","last_active":"2024-09-12T23:33:28.895135+0000","last_peered":"2024-09-12T23:33:28.895135+0000","last_clean":"2024-09-12T23:33:28.895135+0000","last_became_active":"2024-09-12T23:33:28.894687+0000","last_became_peered":"2024-09-12T23:33:28.894687+0000","last_unstale":"2024-09-12T23:33:28.895135+0000","last_undegraded":"2024-09-12T23:33:28.895135+0000","last_fullsized":"2024-09-12T23:33:28.895135+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:57:06.646238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.868957+0000","last_change":"2024-09-12T23:33:28.868957+0000","last_active":"2024-09-12T23:33:28.868957+0000","last_peered":"2024-09-12T23:33:28.868957+0000","last_clean":"2024-09-12T23:33:28.868957+0000","last_became_active":"2024-09-12T23:33:28.868587+0000","last_became_peered":"2024-09-12T23:33:28.868587+0000","last_unstale":"2024-09-12T23:33:28.868957+0000","last_undegraded":"2024-09-12T23:33:28.868957+0000","last_fullsized":"2024-09-12T23:33:28.868957+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:45:32.284209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.892338+0000","last_change":"2024-09-12T23:33:29.890360+0000","last_active":"2024-09-12T23:33:29.892338+0000","last_peered":"2024-09-12T23:33:29.892338+0000","last_clean":"2024-09-12T23:33:29.892338+0000","last_became_active":"2024-09-12T23:33:29.889575+0000","last_became_peered":"2024-09-12T23:33:29.889575+0000","last_unstale":"2024-09-12T23:33:29.892338+0000","last_undegraded":"2024-09-12T23:33:29.892338+0000","last_fullsized":"2024-09-12T23:33:29.892338+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:36:34.938054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.899931+0000","last_change":"2024-09-12T23:33:28.899931+0000","last_active":"2024-09-12T23:33:28.899931+0000","last_peered":"2024-09-12T23:33:28.899931+0000","last_clean":"2024-09-12T23:33:28.899931+0000","last_became_active":"2024-09-12T23:33:28.899410+0000","last_became_peered":"2024-09-12T23:33:28.899410+0000","last_unstale":"2024-09-12T23:33:28.899931+0000","last_undegraded":"2024-09-12T23:33:28.899931+0000","last_fullsized":"2024-09-12T23:33:28.899931+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:05:26.671172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.878775+0000","last_change":"2024-09-12T23:33:28.867304+0000","last_active":"2024-09-12T23:33:29.878775+0000","last_peered":"2024-09-12T23:33:29.878775+0000","last_clean":"2024-09-12T23:33:29.878775+0000","last_became_active":"2024-09-12T23:33:28.867020+0000","last_became_peered":"2024-09-12T23:33:28.867020+0000","last_unstale":"2024-09-12T23:33:29.878775+0000","last_undegraded":"2024-09-12T23:33:29.878775+0000","last_fullsized":"2024-09-12T23:33:29.878775+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:51:22.908188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.865496+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:38:00.819958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[2,3],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.895077+0000","last_change":"2024-09-12T23:33:28.895077+0000","last_active":"2024-09-12T23:33:28.895077+0000","last_peered":"2024-09-12T23:33:28.895077+0000","last_clean":"2024-09-12T23:33:28.895077+0000","last_became_active":"2024-09-12T23:33:28.892268+0000","last_became_peered":"2024-09-12T23:33:28.892268+0000","last_unstale":"2024-09-12T23:33:28.895077+0000","last_undegraded":"2024-09-12T23:33:28.895077+0000","last_fullsized":"2024-09-12T23:33:28.895077+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:26:19.125877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.913453+0000","last_change":"2024-09-12T23:33:29.899700+0000","last_active":"2024-09-12T23:33:29.913453+0000","last_peered":"2024-09-12T23:33:29.913453+0000","last_clean":"2024-09-12T23:33:29.913453+0000","last_became_active":"2024-09-12T23:33:29.899381+0000","last_became_peered":"2024-09-12T23:33:29.899381+0000","last_unstale":"2024-09-12T23:33:29.913453+0000","last_undegraded":"2024-09-12T23:33:29.913453+0000","last_fullsized":"2024-09-12T23:33:29.913453+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:30:20.590373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.875764+0000","last_change":"2024-09-12T23:33:28.866367+0000","last_active":"2024-09-12T23:33:29.875764+0000","last_peered":"2024-09-12T23:33:29.875764+0000","last_clean":"2024-09-12T23:33:29.875764+0000","last_became_active":"2024-09-12T23:33:28.866005+0000","last_became_peered":"2024-09-12T23:33:28.866005+0000","last_unstale":"2024-09-12T23:33:29.875764+0000","last_undegraded":"2024-09-12T23:33:29.875764+0000","last_fullsized":"2024-09-12T23:33:29.875764+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:49:27.803045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.915275+0000","last_change":"2024-09-12T23:33:29.915275+0000","last_active":"2024-09-12T23:33:29.915275+0000","last_peered":"2024-09-12T23:33:29.915275+0000","last_clean":"2024-09-12T23:33:29.915275+0000","last_became_active":"2024-09-12T23:33:29.914934+0000","last_became_peered":"2024-09-12T23:33:29.914934+0000","last_unstale":"2024-09-12T23:33:29.915275+0000","last_undegraded":"2024-09-12T23:33:29.915275+0000","last_fullsized":"2024-09-12T23:33:29.915275+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:43:27.562543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":3,"reported_epoch":52,"state":"creating+activating","last_fresh":"2024-09-12T23:33:28.847239+0000","last_change":"2024-09-12T23:33:28.847239+0000","last_active":"2024-09-12T23:33:27.830370+0000","last_peered":"2024-09-12T23:33:27.830370+0000","last_clean":"2024-09-12T23:33:27.830370+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.847239+0000","last_undegraded":"2024-09-12T23:33:28.847239+0000","last_fullsized":"2024-09-12T23:33:28.847239+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:58:40.820895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":["6","2","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.8","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.894286+0000","last_change":"2024-09-12T23:33:28.894286+0000","last_active":"2024-09-12T23:33:28.894286+0000","last_peered":"2024-09-12T23:33:28.894286+0000","last_clean":"2024-09-12T23:33:28.894286+0000","last_became_active":"2024-09-12T23:33:28.893861+0000","last_became_peered":"2024-09-12T23:33:28.893861+0000","last_unstale":"2024-09-12T23:33:28.894286+0000","last_undegraded":"2024-09-12T23:33:28.894286+0000","last_fullsized":"2024-09-12T23:33:28.894286+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:20:01.806723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":11,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.874494+0000","last_change":"2024-09-12T23:33:28.884360+0000","last_active":"2024-09-12T23:33:29.874494+0000","last_peered":"2024-09-12T23:33:29.874494+0000","last_clean":"2024-09-12T23:33:29.874494+0000","last_became_active":"2024-09-12T23:33:28.884027+0000","last_became_peered":"2024-09-12T23:33:28.884027+0000","last_unstale":"2024-09-12T23:33:29.874494+0000","last_undegraded":"2024-09-12T23:33:29.874494+0000","last_fullsized":"2024-09-12T23:33:29.874494+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:11:57.567074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.915243+0000","last_change":"2024-09-12T23:33:29.915243+0000","last_active":"2024-09-12T23:33:29.915243+0000","last_peered":"2024-09-12T23:33:29.915243+0000","last_clean":"2024-09-12T23:33:29.915243+0000","last_became_active":"2024-09-12T23:33:29.914543+0000","last_became_peered":"2024-09-12T23:33:29.914543+0000","last_unstale":"2024-09-12T23:33:29.915243+0000","last_undegraded":"2024-09-12T23:33:29.915243+0000","last_fullsized":"2024-09-12T23:33:29.915243+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-13T23:48:16.023985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":11,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.879749+0000","last_change":"2024-09-12T23:33:28.883601+0000","last_active":"2024-09-12T23:33:29.879749+0000","last_peered":"2024-09-12T23:33:29.879749+0000","last_clean":"2024-09-12T23:33:29.879749+0000","last_became_active":"2024-09-12T23:33:28.883276+0000","last_became_peered":"2024-09-12T23:33:28.883276+0000","last_unstale":"2024-09-12T23:33:29.879749+0000","last_undegraded":"2024-09-12T23:33:29.879749+0000","last_fullsized":"2024-09-12T23:33:29.879749+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:26:21.733615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.908185+0000","last_change":"2024-09-12T23:33:29.908185+0000","last_active":"2024-09-12T23:33:29.908185+0000","last_peered":"2024-09-12T23:33:29.908185+0000","last_clean":"2024-09-12T23:33:29.908185+0000","last_became_active":"2024-09-12T23:33:29.907814+0000","last_became_peered":"2024-09-12T23:33:29.907814+0000","last_unstale":"2024-09-12T23:33:29.908185+0000","last_undegraded":"2024-09-12T23:33:29.908185+0000","last_fullsized":"2024-09-12T23:33:29.908185+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:45:47.207242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.902988+0000","last_change":"2024-09-12T23:33:28.902988+0000","last_active":"2024-09-12T23:33:28.902988+0000","last_peered":"2024-09-12T23:33:28.902988+0000","last_clean":"2024-09-12T23:33:28.902988+0000","last_became_active":"2024-09-12T23:33:28.902514+0000","last_became_peered":"2024-09-12T23:33:28.902514+0000","last_unstale":"2024-09-12T23:33:28.902988+0000","last_undegraded":"2024-09-12T23:33:28.902988+0000","last_fullsized":"2024-09-12T23:33:28.902988+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:29:42.595057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.848452+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:12:37.049274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,4],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.866904+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:49:53.725680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[4,6],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.902224+0000","last_change":"2024-09-12T23:33:28.902224+0000","last_active":"2024-09-12T23:33:28.902224+0000","last_peered":"2024-09-12T23:33:28.902224+0000","last_clean":"2024-09-12T23:33:28.902224+0000","last_became_active":"2024-09-12T23:33:28.901755+0000","last_became_peered":"2024-09-12T23:33:28.901755+0000","last_unstale":"2024-09-12T23:33:28.902224+0000","last_undegraded":"2024-09-12T23:33:28.902224+0000","last_fullsized":"2024-09-12T23:33:28.902224+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:06:15.504175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.875336+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:20:24.609184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,4],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1d","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.897466+0000","last_change":"2024-09-12T23:33:28.897466+0000","last_active":"2024-09-12T23:33:28.897466+0000","last_peered":"2024-09-12T23:33:28.897466+0000","last_clean":"2024-09-12T23:33:28.897466+0000","last_became_active":"2024-09-12T23:33:28.896936+0000","last_became_peered":"2024-09-12T23:33:28.896936+0000","last_unstale":"2024-09-12T23:33:28.897466+0000","last_undegraded":"2024-09-12T23:33:28.897466+0000","last_fullsized":"2024-09-12T23:33:28.897466+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:29:38.732168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.1e","version":"53'1","reported_seq":11,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.875384+0000","last_change":"2024-09-12T23:33:28.894353+0000","last_active":"2024-09-12T23:33:29.875384+0000","last_peered":"2024-09-12T23:33:29.875384+0000","last_clean":"2024-09-12T23:33:29.875384+0000","last_became_active":"2024-09-12T23:33:28.893992+0000","last_became_peered":"2024-09-12T23:33:28.893992+0000","last_unstale":"2024-09-12T23:33:29.875384+0000","last_undegraded":"2024-09-12T23:33:29.875384+0000","last_fullsized":"2024-09-12T23:33:29.875384+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:43:43.889025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.891227+0000","last_change":"2024-09-12T23:33:29.891227+0000","last_active":"2024-09-12T23:33:29.891227+0000","last_peered":"2024-09-12T23:33:29.891227+0000","last_clean":"2024-09-12T23:33:29.891227+0000","last_became_active":"2024-09-12T23:33:29.890368+0000","last_became_peered":"2024-09-12T23:33:29.890368+0000","last_unstale":"2024-09-12T23:33:29.891227+0000","last_undegraded":"2024-09-12T23:33:29.891227+0000","last_fullsized":"2024-09-12T23:33:29.891227+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:38:37.526610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:27.830370+0000","last_change":"2024-09-12T23:33:27.837410+0000","last_active":"2024-09-12T23:33:27.830370+0000","last_peered":"2024-09-12T23:33:27.830370+0000","last_clean":"2024-09-12T23:33:27.830370+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:27.830370+0000","last_undegraded":"2024-09-12T23:33:27.830370+0000","last_fullsized":"2024-09-12T23:33:27.830370+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:42:19.518598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[4,7],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":147,"acting":147,"num_store_stats":6},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":160,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3563520,"data_stored":3542208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430219,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298432,"kb_used_data":1072,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446720,"statfs":{"total":95995035648,"available":95689441280,"internally_reserved":0,"allocated":1097728,"data_stored":796122,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.74299999999999999}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.69799999999999995}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47499999999999998}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.82099999999999995}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64000000000000001}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.90200000000000002}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.83899999999999997}]}]},{"osd":6,"up_from":42,"seq":180388626453,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298412,"kb_used_data":1056,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446740,"statfs":{"total":95995035648,"available":95689461760,"internally_reserved":0,"allocated":1081344,"data_stored":789114,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 12 23:32:56 2024","interfaces":[{"interface":"back","average":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"min":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":1.2450000000000001,"5min":1.2450000000000001,"15min":1.2450000000000001},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.874,"5min":0.874,"15min":0.874},"min":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"max":{"1min":1.222,"5min":1.222,"15min":1.222},"last":0.82099999999999995}]},{"osd":1,"last update":"Thu Sep 12 23:32:56 2024","interfaces":[{"interface":"back","average":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"min":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":1.153,"5min":1.153,"15min":1.153},"last":0.89800000000000002},{"interface":"front","average":{"1min":0.871,"5min":0.871,"15min":0.871},"min":{"1min":0.62,"5min":0.62,"15min":0.62},"max":{"1min":1.306,"5min":1.306,"15min":1.306},"last":1.1499999999999999}]},{"osd":2,"last update":"Thu Sep 12 23:32:56 2024","interfaces":[{"interface":"back","average":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"min":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"max":{"1min":1.4359999999999999,"5min":1.4359999999999999,"15min":1.4359999999999999},"last":0.76900000000000002},{"interface":"front","average":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"min":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"max":{"1min":1.131,"5min":1.131,"15min":1.131},"last":0.49199999999999999}]},{"osd":3,"last update":"Thu Sep 12 23:32:56 2024","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"min":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"max":{"1min":1.2330000000000001,"5min":1.2330000000000001,"15min":1.2330000000000001},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"min":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"max":{"1min":1.2290000000000001,"5min":1.2290000000000001,"15min":1.2290000000000001},"last":1.018}]},{"osd":4,"last update":"Thu Sep 12 23:32:56 2024","interfaces":[{"interface":"back","average":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"min":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":1.131,"5min":1.131,"15min":1.131},"last":0.95399999999999996},{"interface":"front","average":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":1.389,"5min":1.389,"15min":1.389},"last":0.97999999999999998}]},{"osd":5,"last update":"Thu Sep 12 23:32:56 2024","interfaces":[{"interface":"back","average":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"min":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"max":{"1min":1.228,"5min":1.228,"15min":1.228},"last":1.0780000000000001},{"interface":"front","average":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"min":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":1.3109999999999999,"5min":1.3109999999999999,"15min":1.3109999999999999},"last":1.105}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.139}]}]},{"osd":5,"up_from":36,"seq":154618822688,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297828,"kb_used_data":472,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447324,"statfs":{"total":95995035648,"available":95690059776,"internally_reserved":0,"allocated":483328,"data_stored":198431,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 12 23:33:07 2024","interfaces":[{"interface":"back","average":{"1min":0.81399999999999995,"5min":0.78700000000000003,"15min":0.78300000000000003},"min":{"1min":0.56200000000000006,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":1.0740000000000001,"5min":1.6100000000000001,"15min":1.6100000000000001},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.872,"5min":0.83399999999999996,"15min":0.82799999999999996},"min":{"1min":0.68999999999999995,"5min":0.58199999999999996,"15min":0.58199999999999996},"max":{"1min":1.0469999999999999,"5min":1.0840000000000001,"15min":1.0840000000000001},"last":0.56799999999999995}]},{"osd":1,"last update":"Thu Sep 12 23:33:07 2024","interfaces":[{"interface":"back","average":{"1min":0.88100000000000001,"5min":0.85199999999999998,"15min":0.84699999999999998},"min":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"max":{"1min":1.218,"5min":1.4470000000000001,"15min":1.4470000000000001},"last":0.81799999999999995},{"interface":"front","average":{"1min":0.89000000000000001,"5min":0.81200000000000006,"15min":0.79900000000000004},"min":{"1min":0.66300000000000003,"5min":0.502,"15min":0.502},"max":{"1min":1.274,"5min":1.274,"15min":1.274},"last":0.52800000000000002}]},{"osd":2,"last update":"Thu Sep 12 23:33:07 2024","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":0.878,"15min":0.87},"min":{"1min":0.73999999999999999,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":1.2509999999999999,"5min":1.2509999999999999,"15min":1.2509999999999999},"last":0.75},{"interface":"front","average":{"1min":0.876,"5min":0.90400000000000003,"15min":0.90900000000000003},"min":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"max":{"1min":1.0980000000000001,"5min":1.141,"15min":1.141},"last":0.77300000000000002}]},{"osd":3,"last update":"Thu Sep 12 23:33:07 2024","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":0.91300000000000003,"15min":0.91100000000000003},"min":{"1min":0.68100000000000005,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.2010000000000001,"5min":1.3280000000000001,"15min":1.3280000000000001},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.92000000000000004,"5min":0.91500000000000004,"15min":0.91400000000000003},"min":{"1min":0.67400000000000004,"5min":0.57499999999999996,"15min":0.57499999999999996},"max":{"1min":1.2609999999999999,"5min":1.536,"15min":1.536},"last":0.73399999999999999}]},{"osd":4,"last update":"Thu Sep 12 23:33:07 2024","interfaces":[{"interface":"back","average":{"1min":0.747,"5min":0.74299999999999999,"15min":0.74199999999999999},"min":{"1min":0.47199999999999998,"5min":0.308,"15min":0.308},"max":{"1min":1.1910000000000001,"5min":1.1910000000000001,"15min":1.1910000000000001},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.79500000000000004,"5min":0.77700000000000002,"15min":0.77400000000000002},"min":{"1min":0.504,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":1.1679999999999999,"5min":1.1679999999999999,"15min":1.1679999999999999},"last":0.67700000000000005}]},{"osd":6,"last update":"Thu Sep 12 23:32:54 2024","interfaces":[{"interface":"back","average":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":1.1220000000000001,"5min":1.1220000000000001,"15min":1.1220000000000001},"last":0.79700000000000004},{"interface":"front","average":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"min":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":1.0840000000000001,"5min":1.0840000000000001,"15min":1.0840000000000001},"last":0.65500000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.72099999999999997}]}]},{"osd":4,"up_from":30,"seq":128849018921,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297836,"kb_used_data":480,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447316,"statfs":{"total":95995035648,"available":95690051584,"internally_reserved":0,"allocated":491520,"data_stored":199061,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 12 23:33:17 2024","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":0.83599999999999997,"15min":0.83399999999999996},"min":{"1min":0.63200000000000001,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.375,"5min":1.375,"15min":1.375},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.89500000000000002,"5min":0.84699999999999998,"15min":0.84299999999999997},"min":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"max":{"1min":1.1479999999999999,"5min":1.1479999999999999,"15min":1.1479999999999999},"last":0.85999999999999999}]},{"osd":1,"last update":"Thu Sep 12 23:33:17 2024","interfaces":[{"interface":"back","average":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82399999999999995},"min":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":1.194,"5min":1.194,"15min":1.194},"last":1.01},{"interface":"front","average":{"1min":0.86899999999999999,"5min":0.85899999999999999,"15min":0.85699999999999998},"min":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.218,"5min":1.218,"15min":1.218},"last":0.80600000000000005}]},{"osd":2,"last update":"Thu Sep 12 23:33:17 2024","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":0.84599999999999997,"15min":0.85599999999999998},"min":{"1min":0.60199999999999998,"5min":0.55700000000000005,"15min":0.55700000000000005},"max":{"1min":1.175,"5min":1.175,"15min":1.175},"last":0.98599999999999999},{"interface":"front","average":{"1min":0.89500000000000002,"5min":0.85699999999999998,"15min":0.86299999999999999},"min":{"1min":0.66000000000000003,"5min":0.63500000000000001,"15min":0.63500000000000001},"max":{"1min":1.2909999999999999,"5min":1.2909999999999999,"15min":1.2909999999999999},"last":1.079}]},{"osd":3,"last update":"Thu Sep 12 23:33:17 2024","interfaces":[{"interface":"back","average":{"1min":0.89700000000000002,"5min":0.85199999999999998,"15min":0.84999999999999998},"min":{"1min":0.65200000000000002,"5min":0.60099999999999998,"15min":0.60099999999999998},"max":{"1min":1.5129999999999999,"5min":1.5129999999999999,"15min":1.5129999999999999},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.97899999999999998,"5min":0.90200000000000002,"15min":0.89000000000000001},"min":{"1min":0.57099999999999995,"5min":0.55900000000000005,"15min":0.55900000000000005},"max":{"1min":1.595,"5min":1.595,"15min":1.595},"last":1.214}]},{"osd":5,"last update":"Thu Sep 12 23:33:09 2024","interfaces":[{"interface":"back","average":{"1min":0.80500000000000005,"5min":0.78400000000000003,"15min":0.78000000000000003},"min":{"1min":0.49199999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":1.7430000000000001,"5min":1.7430000000000001,"15min":1.7430000000000001},"last":0.91200000000000003},{"interface":"front","average":{"1min":0.90300000000000002,"5min":0.82199999999999995,"15min":0.80800000000000005},"min":{"1min":0.375,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.716,"5min":1.716,"15min":1.716},"last":1.0489999999999999}]},{"osd":6,"last update":"Thu Sep 12 23:32:56 2024","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":0.91300000000000003,"15min":0.91300000000000003},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":1.7629999999999999,"5min":1.7629999999999999,"15min":1.7629999999999999},"last":1.1140000000000001},{"interface":"front","average":{"1min":0.96299999999999997,"5min":0.96299999999999997,"15min":0.96299999999999997},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":1.7010000000000001,"5min":1.7010000000000001,"15min":1.7010000000000001},"last":1.2849999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.1379999999999999}]}]},{"osd":3,"up_from":25,"seq":107374182451,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297880,"kb_used_data":524,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447272,"statfs":{"total":95995035648,"available":95690006528,"internally_reserved":0,"allocated":536576,"data_stored":214603,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 12 23:32:29 2024","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.62,"15min":0.60199999999999998},"min":{"1min":0.53400000000000003,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.995,"5min":0.999,"15min":0.999},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.78300000000000003,"5min":0.70499999999999996,"15min":0.69299999999999995},"min":{"1min":0.58299999999999996,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":1.3120000000000001,"5min":1.3120000000000001,"15min":1.3120000000000001},"last":0.46600000000000003}]},{"osd":1,"last update":"Thu Sep 12 23:32:29 2024","interfaces":[{"interface":"back","average":{"1min":0.755,"5min":0.67200000000000004,"15min":0.65600000000000003},"min":{"1min":0.54200000000000004,"5min":0.52200000000000002,"15min":0.52200000000000002},"max":{"1min":0.96999999999999997,"5min":0.96999999999999997,"15min":0.96999999999999997},"last":0.88600000000000001},{"interface":"front","average":{"1min":0.746,"5min":0.75,"15min":0.74199999999999999},"min":{"1min":0.57199999999999995,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":1.095,"5min":1.171,"15min":1.171},"last":0.61899999999999999}]},{"osd":2,"last update":"Thu Sep 12 23:32:29 2024","interfaces":[{"interface":"back","average":{"1min":0.81999999999999995,"5min":0.78900000000000003,"15min":0.77700000000000002},"min":{"1min":0.66100000000000003,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":0.96799999999999997,"5min":1.1140000000000001,"15min":1.1140000000000001},"last":0.78800000000000003},{"interface":"front","average":{"1min":0.85299999999999998,"5min":0.75800000000000001,"15min":0.73399999999999999},"min":{"1min":0.63800000000000001,"5min":0.54900000000000004,"15min":0.54900000000000004},"max":{"1min":1.238,"5min":1.238,"15min":1.238},"last":0.56299999999999994}]},{"osd":4,"last update":"Thu Sep 12 23:33:24 2024","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":0.90300000000000002,"15min":0.88200000000000001},"min":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.4950000000000001,"5min":1.4950000000000001,"15min":1.4950000000000001},"last":0.86099999999999999},{"interface":"front","average":{"1min":0.90300000000000002,"5min":0.87,"15min":0.84899999999999998},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":1.3640000000000001,"5min":1.3640000000000001,"15min":1.3640000000000001},"last":0.98299999999999998}]},{"osd":5,"last update":"Thu Sep 12 23:33:05 2024","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":0.98399999999999999,"15min":0.98399999999999999},"min":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"max":{"1min":1.321,"5min":1.371,"15min":1.371},"last":0.72499999999999998},{"interface":"front","average":{"1min":1.05,"5min":0.98199999999999998,"15min":0.97099999999999997},"min":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":1.423,"5min":1.423,"15min":1.423},"last":0.65600000000000003}]},{"osd":6,"last update":"Thu Sep 12 23:32:54 2024","interfaces":[{"interface":"back","average":{"1min":1.052,"5min":1.052,"15min":1.052},"min":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"max":{"1min":1.3540000000000001,"5min":1.3540000000000001,"15min":1.3540000000000001},"last":0.94499999999999995},{"interface":"front","average":{"1min":1.115,"5min":1.115,"15min":1.115},"min":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"max":{"1min":1.718,"5min":1.718,"15min":1.718},"last":0.96299999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.90800000000000003}]}]},{"osd":2,"up_from":18,"seq":77309411389,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297836,"kb_used_data":480,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447316,"statfs":{"total":95995035648,"available":95690051584,"internally_reserved":0,"allocated":491520,"data_stored":199061,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 12 23:32:41 2024","interfaces":[{"interface":"back","average":{"1min":0.72999999999999998,"5min":0.626,"15min":0.58799999999999997},"min":{"1min":0.53800000000000003,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":1.0429999999999999,"5min":1.23,"15min":1.23},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.745,"5min":0.66400000000000003,"15min":0.61299999999999999},"min":{"1min":0.50800000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":1.0940000000000001,"5min":1.0940000000000001,"15min":1.0940000000000001},"last":0.38300000000000001}]},{"osd":1,"last update":"Thu Sep 12 23:32:41 2024","interfaces":[{"interface":"back","average":{"1min":0.76400000000000001,"5min":0.67700000000000005,"15min":0.628},"min":{"1min":0.5,"5min":0.316,"15min":0.316},"max":{"1min":1.014,"5min":1.5329999999999999,"15min":1.5329999999999999},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.74199999999999999,"5min":0.68400000000000005,"15min":0.63700000000000001},"min":{"1min":0.60999999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.95099999999999996,"5min":1.0860000000000001,"15min":1.0860000000000001},"last":0.44}]},{"osd":3,"last update":"Thu Sep 12 23:32:33 2024","interfaces":[{"interface":"back","average":{"1min":0.80000000000000004,"5min":0.75800000000000001,"15min":0.74399999999999999},"min":{"1min":0.627,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":1.2270000000000001,"5min":1.2270000000000001,"15min":1.2270000000000001},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.83299999999999996,"5min":0.79800000000000004,"15min":0.78000000000000003},"min":{"1min":0.623,"5min":0.433,"15min":0.433},"max":{"1min":1.0049999999999999,"5min":1.6619999999999999,"15min":1.6619999999999999},"last":0.36599999999999999}]},{"osd":4,"last update":"Thu Sep 12 23:33:19 2024","interfaces":[{"interface":"back","average":{"1min":0.88300000000000001,"5min":0.873,"15min":0.86499999999999999},"min":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.107,"5min":1.1639999999999999,"15min":1.1639999999999999},"last":0.82499999999999996},{"interface":"front","average":{"1min":0.96899999999999997,"5min":0.95299999999999996,"15min":0.93500000000000005},"min":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"max":{"1min":1.23,"5min":1.363,"15min":1.363},"last":0.79000000000000004}]},{"osd":5,"last update":"Thu Sep 12 23:33:10 2024","interfaces":[{"interface":"back","average":{"1min":0.95599999999999996,"5min":1.022,"15min":1.0329999999999999},"min":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"max":{"1min":1.282,"5min":1.4830000000000001,"15min":1.4830000000000001},"last":0.89700000000000002},{"interface":"front","average":{"1min":0.90700000000000003,"5min":1.006,"15min":1.022},"min":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"max":{"1min":1.101,"5min":1.3129999999999999,"15min":1.3129999999999999},"last":0.63900000000000001}]},{"osd":6,"last update":"Thu Sep 12 23:32:56 2024","interfaces":[{"interface":"back","average":{"1min":0.94499999999999995,"5min":0.94499999999999995,"15min":0.94499999999999995},"min":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"max":{"1min":1.2330000000000001,"5min":1.2330000000000001,"15min":1.2330000000000001},"last":0.98099999999999998},{"interface":"front","average":{"1min":1.0880000000000001,"5min":1.0880000000000001,"15min":1.0880000000000001},"min":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"max":{"1min":1.498,"5min":1.498,"15min":1.498},"last":0.67000000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67800000000000005}]}]},{"osd":1,"up_from":13,"seq":55834574920,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297876,"kb_used_data":512,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447276,"statfs":{"total":95995035648,"available":95690010624,"internally_reserved":0,"allocated":524288,"data_stored":213107,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 12 23:32:55 2024","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":0.59899999999999998,"15min":0.53000000000000003},"min":{"1min":0.51400000000000001,"5min":0.255,"15min":0.255},"max":{"1min":1.1579999999999999,"5min":1.1579999999999999,"15min":1.1579999999999999},"last":0.69199999999999995},{"interface":"front","average":{"1min":0.77400000000000002,"5min":0.66000000000000003,"15min":0.58199999999999996},"min":{"1min":0.501,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":1.1830000000000001,"5min":1.1830000000000001,"15min":1.1830000000000001},"last":0.91300000000000003}]},{"osd":2,"last update":"Thu Sep 12 23:32:40 2024","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":0.64600000000000002,"15min":0.58799999999999997},"min":{"1min":0.54700000000000004,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.95899999999999996,"5min":1.046,"15min":1.046},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.82899999999999996,"5min":0.69999999999999996,"15min":0.629},"min":{"1min":0.53200000000000003,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":1.3049999999999999,"5min":1.3049999999999999,"15min":1.3049999999999999},"last":0.59399999999999997}]},{"osd":3,"last update":"Thu Sep 12 23:32:34 2024","interfaces":[{"interface":"back","average":{"1min":0.81999999999999995,"5min":0.79000000000000004,"15min":0.78100000000000003},"min":{"1min":0.56999999999999995,"5min":0.54800000000000004,"15min":0.54800000000000004},"max":{"1min":1.0249999999999999,"5min":1.1479999999999999,"15min":1.1479999999999999},"last":0.78100000000000003},{"interface":"front","average":{"1min":0.81299999999999994,"5min":0.77100000000000002,"15min":0.76200000000000001},"min":{"1min":0.61499999999999999,"5min":0.57799999999999996,"15min":0.57799999999999996},"max":{"1min":1.109,"5min":1.109,"15min":1.109},"last":0.84199999999999997}]},{"osd":4,"last update":"Thu Sep 12 23:33:16 2024","interfaces":[{"interface":"back","average":{"1min":0.95999999999999996,"5min":0.92100000000000004,"15min":0.91700000000000004},"min":{"1min":0.75700000000000001,"5min":0.65100000000000002,"15min":0.65100000000000002},"max":{"1min":1.23,"5min":1.585,"15min":1.585},"last":1.0620000000000001},{"interface":"front","average":{"1min":1.006,"5min":0.93700000000000006,"15min":0.92700000000000005},"min":{"1min":0.76000000000000001,"5min":0.73499999999999999,"15min":0.73499999999999999},"max":{"1min":1.389,"5min":1.4279999999999999,"15min":1.4279999999999999},"last":0.995}]},{"osd":5,"last update":"Thu Sep 12 23:33:05 2024","interfaces":[{"interface":"back","average":{"1min":0.95399999999999996,"5min":0.92100000000000004,"15min":0.91500000000000004},"min":{"1min":0.66000000000000003,"5min":0.65200000000000002,"15min":0.65200000000000002},"max":{"1min":1.484,"5min":1.6819999999999999,"15min":1.6819999999999999},"last":1.014},{"interface":"front","average":{"1min":1.0289999999999999,"5min":1.006,"15min":1.002},"min":{"1min":0.81000000000000005,"5min":0.66200000000000003,"15min":0.66200000000000003},"max":{"1min":1.5069999999999999,"5min":1.5069999999999999,"15min":1.5069999999999999},"last":0.879}]},{"osd":6,"last update":"Thu Sep 12 23:32:55 2024","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":1.0529999999999999,"15min":1.0529999999999999},"min":{"1min":0.746,"5min":0.746,"15min":0.746},"max":{"1min":1.5429999999999999,"5min":1.5429999999999999,"15min":1.5429999999999999},"last":0.93300000000000005},{"interface":"front","average":{"1min":1.101,"5min":1.101,"15min":1.101},"min":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"max":{"1min":1.5649999999999999,"5min":1.5649999999999999,"15min":1.5649999999999999},"last":0.96799999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.81200000000000006}]}]},{"osd":0,"up_from":8,"seq":34359738448,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298412,"kb_used_data":1056,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446740,"statfs":{"total":95995035648,"available":95689461760,"internally_reserved":0,"allocated":1081344,"data_stored":789114,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Sep 12 23:32:49 2024","interfaces":[{"interface":"back","average":{"1min":0.69599999999999995,"5min":0.59899999999999998,"15min":0.51100000000000001},"min":{"1min":0.48499999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":1.0509999999999999,"5min":1.0509999999999999,"15min":1.0509999999999999},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.74399999999999999,"5min":0.629,"15min":0.53300000000000003},"min":{"1min":0.55400000000000005,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.0049999999999999,"5min":1.028,"15min":1.028},"last":0.48699999999999999}]},{"osd":2,"last update":"Thu Sep 12 23:32:43 2024","interfaces":[{"interface":"back","average":{"1min":0.73299999999999998,"5min":0.66600000000000004,"15min":0.61899999999999999},"min":{"1min":0.52800000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":1.081,"5min":1.081,"15min":1.081},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.82699999999999996,"5min":0.69599999999999995,"15min":0.63900000000000001},"min":{"1min":0.56000000000000005,"5min":0.317,"15min":0.317},"max":{"1min":1.151,"5min":1.151,"15min":1.151},"last":0.81000000000000005}]},{"osd":3,"last update":"Thu Sep 12 23:32:27 2024","interfaces":[{"interface":"back","average":{"1min":0.79900000000000004,"5min":0.76800000000000002,"15min":0.76400000000000001},"min":{"1min":0.60999999999999999,"5min":0.60599999999999998,"15min":0.60599999999999998},"max":{"1min":1.177,"5min":1.177,"15min":1.177},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.83599999999999997,"5min":0.79600000000000004,"15min":0.79200000000000004},"min":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"max":{"1min":1.0009999999999999,"5min":1.1950000000000001,"15min":1.1950000000000001},"last":0.68400000000000005}]},{"osd":4,"last update":"Thu Sep 12 23:33:23 2024","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":0.93100000000000005,"15min":0.90200000000000002},"min":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"max":{"1min":1.304,"5min":1.304,"15min":1.304},"last":0.79000000000000004},{"interface":"front","average":{"1min":0.95699999999999996,"5min":0.876,"15min":0.84999999999999998},"min":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"max":{"1min":1.266,"5min":1.266,"15min":1.266},"last":0.67200000000000004}]},{"osd":5,"last update":"Thu Sep 12 23:33:10 2024","interfaces":[{"interface":"back","average":{"1min":0.94499999999999995,"5min":0.94299999999999995,"15min":0.94299999999999995},"min":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"max":{"1min":1.2330000000000001,"5min":1.2330000000000001,"15min":1.2330000000000001},"last":0.64200000000000002},{"interface":"front","average":{"1min":1.002,"5min":0.95899999999999996,"15min":0.95199999999999996},"min":{"1min":0.75,"5min":0.70099999999999996,"15min":0.70099999999999996},"max":{"1min":1.1579999999999999,"5min":1.1579999999999999,"15min":1.1579999999999999},"last":0.749}]},{"osd":6,"last update":"Thu Sep 12 23:32:55 2024","interfaces":[{"interface":"back","average":{"1min":1.1120000000000001,"5min":1.1120000000000001,"15min":1.1120000000000001},"min":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"max":{"1min":1.492,"5min":1.492,"15min":1.492},"last":0.98199999999999998},{"interface":"front","average":{"1min":1.0509999999999999,"5min":1.0509999999999999,"15min":1.0509999999999999},"min":{"1min":0.754,"5min":0.754,"15min":0.754},"max":{"1min":1.377,"5min":1.377,"15min":1.377},"last":0.90700000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.96699999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":1656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-12T23:33:33.715 DEBUG:teuthology.orchestra.run.smithi136:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-09-12T23:33:34.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:33 smithi136 bash[22601]: audit 2024-09-12T23:33:33.016968+0000 mon.c (mon.2) 24 : audit [DBG] from='client.? 172.21.15.136:0/1918650490' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-12T23:33:34.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:33 smithi136 bash[32189]: audit 2024-09-12T23:33:33.016968+0000 mon.c (mon.2) 24 : audit [DBG] from='client.? 172.21.15.136:0/1918650490' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-12T23:33:34.052 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-12T23:33:34.054 INFO:teuthology.orchestra.run.smithi136.stderr:dumped all 2024-09-12T23:33:34.069 INFO:teuthology.orchestra.run.smithi136.stdout:{"pg_ready":true,"pg_map":{"version":318,"stamp":"2024-09-12T23:33:33.489398+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590528,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":106,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":78,"ondisk_log_size":78,"up":246,"acting":246,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":59,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2384512,"kb_used_data":5652,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747576704,"statfs":{"total":767960285184,"available":765518544896,"internally_reserved":0,"allocated":5787648,"data_stored":3398613,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":160,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.654574"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.861074+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:54:27.307253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,6],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.916927+0000","last_change":"2024-09-12T23:33:29.907919+0000","last_active":"2024-09-12T23:33:29.916927+0000","last_peered":"2024-09-12T23:33:29.916927+0000","last_clean":"2024-09-12T23:33:29.916927+0000","last_became_active":"2024-09-12T23:33:29.906765+0000","last_became_peered":"2024-09-12T23:33:29.906765+0000","last_unstale":"2024-09-12T23:33:29.916927+0000","last_undegraded":"2024-09-12T23:33:29.916927+0000","last_fullsized":"2024-09-12T23:33:29.916927+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:39:49.787701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.876654+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:18:46.573142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[3,7],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.859085+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:07:03.294377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[2,4],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.873576+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:07:11.163630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,1],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.850034+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:17:57.421440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,4],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.902033+0000","last_change":"2024-09-12T23:33:29.902033+0000","last_active":"2024-09-12T23:33:29.902033+0000","last_peered":"2024-09-12T23:33:29.902033+0000","last_clean":"2024-09-12T23:33:29.902033+0000","last_became_active":"2024-09-12T23:33:29.901611+0000","last_became_peered":"2024-09-12T23:33:29.901611+0000","last_unstale":"2024-09-12T23:33:29.902033+0000","last_undegraded":"2024-09-12T23:33:29.902033+0000","last_fullsized":"2024-09-12T23:33:29.902033+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:57:58.671558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.856686+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:41:19.476191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,5],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.852231+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:56:38.506429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,5],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.855386+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:08:21.774244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,5],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.907459+0000","last_change":"2024-09-12T23:33:29.907459+0000","last_active":"2024-09-12T23:33:29.907459+0000","last_peered":"2024-09-12T23:33:29.907459+0000","last_clean":"2024-09-12T23:33:29.907459+0000","last_became_active":"2024-09-12T23:33:29.906577+0000","last_became_peered":"2024-09-12T23:33:29.906577+0000","last_unstale":"2024-09-12T23:33:29.907459+0000","last_undegraded":"2024-09-12T23:33:29.907459+0000","last_fullsized":"2024-09-12T23:33:29.907459+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:21:41.131616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.898912+0000","last_change":"2024-09-12T23:33:29.894796+0000","last_active":"2024-09-12T23:33:29.898912+0000","last_peered":"2024-09-12T23:33:29.898912+0000","last_clean":"2024-09-12T23:33:29.898912+0000","last_became_active":"2024-09-12T23:33:29.894498+0000","last_became_peered":"2024-09-12T23:33:29.894498+0000","last_unstale":"2024-09-12T23:33:29.898912+0000","last_undegraded":"2024-09-12T23:33:29.898912+0000","last_fullsized":"2024-09-12T23:33:29.898912+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:12:10.219015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.867621+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:53:38.724701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,7],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.866959+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-13T23:33:43.071267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[4,7],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.849688+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:08:59.758427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[6,7],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.849822+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:08:17.677941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[2,6],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.849372+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:43:43.192749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,4],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.870650+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:34:34.650312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[6,7],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.906087+0000","last_change":"2024-09-12T23:33:29.906087+0000","last_active":"2024-09-12T23:33:29.906087+0000","last_peered":"2024-09-12T23:33:29.906087+0000","last_clean":"2024-09-12T23:33:29.906087+0000","last_became_active":"2024-09-12T23:33:29.905780+0000","last_became_peered":"2024-09-12T23:33:29.905780+0000","last_unstale":"2024-09-12T23:33:29.906087+0000","last_undegraded":"2024-09-12T23:33:29.906087+0000","last_fullsized":"2024-09-12T23:33:29.906087+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:00:40.777941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.855239+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:05:56.032960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[2,3],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.903381+0000","last_change":"2024-09-12T23:33:29.903381+0000","last_active":"2024-09-12T23:33:29.903381+0000","last_peered":"2024-09-12T23:33:29.903381+0000","last_clean":"2024-09-12T23:33:29.903381+0000","last_became_active":"2024-09-12T23:33:29.903081+0000","last_became_peered":"2024-09-12T23:33:29.903081+0000","last_unstale":"2024-09-12T23:33:29.903381+0000","last_undegraded":"2024-09-12T23:33:29.903381+0000","last_fullsized":"2024-09-12T23:33:29.903381+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:43:01.175691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.850972+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:39:14.674234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[4,7],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.920779+0000","last_change":"2024-09-12T23:33:29.919500+0000","last_active":"2024-09-12T23:33:29.920779+0000","last_peered":"2024-09-12T23:33:29.920779+0000","last_clean":"2024-09-12T23:33:29.920779+0000","last_became_active":"2024-09-12T23:33:29.919185+0000","last_became_peered":"2024-09-12T23:33:29.919185+0000","last_unstale":"2024-09-12T23:33:29.920779+0000","last_undegraded":"2024-09-12T23:33:29.920779+0000","last_fullsized":"2024-09-12T23:33:29.920779+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:56:52.566524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.913907+0000","last_change":"2024-09-12T23:33:29.913907+0000","last_active":"2024-09-12T23:33:29.913907+0000","last_peered":"2024-09-12T23:33:29.913907+0000","last_clean":"2024-09-12T23:33:29.913907+0000","last_became_active":"2024-09-12T23:33:29.913193+0000","last_became_peered":"2024-09-12T23:33:29.913193+0000","last_unstale":"2024-09-12T23:33:29.913907+0000","last_undegraded":"2024-09-12T23:33:29.913907+0000","last_fullsized":"2024-09-12T23:33:29.913907+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:32:45.117056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.869503+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:25:33.906182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,3],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.850701+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:38:56.219241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[5,6],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.852156+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:09:34.637115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,5],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.1b","version":"53'1","reported_seq":11,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.878981+0000","last_change":"2024-09-12T23:33:28.883222+0000","last_active":"2024-09-12T23:33:29.878981+0000","last_peered":"2024-09-12T23:33:29.878981+0000","last_clean":"2024-09-12T23:33:29.878981+0000","last_became_active":"2024-09-12T23:33:28.882848+0000","last_became_peered":"2024-09-12T23:33:28.882848+0000","last_unstale":"2024-09-12T23:33:29.878981+0000","last_undegraded":"2024-09-12T23:33:29.878981+0000","last_fullsized":"2024-09-12T23:33:29.878981+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:57:33.905996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.1a","version":"0'0","reported_seq":2,"reported_epoch":52,"state":"creating+activating","last_fresh":"2024-09-12T23:33:28.846682+0000","last_change":"2024-09-12T23:33:28.847145+0000","last_active":"2024-09-12T23:33:27.830370+0000","last_peered":"2024-09-12T23:33:27.830370+0000","last_clean":"2024-09-12T23:33:27.830370+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.846682+0000","last_undegraded":"2024-09-12T23:33:28.846682+0000","last_fullsized":"2024-09-12T23:33:28.846682+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:57:50.175287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":["6","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":10,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.873141+0000","last_change":"2024-09-12T23:33:28.872822+0000","last_active":"2024-09-12T23:33:28.873141+0000","last_peered":"2024-09-12T23:33:28.873141+0000","last_clean":"2024-09-12T23:33:28.873141+0000","last_became_active":"2024-09-12T23:33:28.872404+0000","last_became_peered":"2024-09-12T23:33:28.872404+0000","last_unstale":"2024-09-12T23:33:28.873141+0000","last_undegraded":"2024-09-12T23:33:28.873141+0000","last_fullsized":"2024-09-12T23:33:28.873141+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-13T23:47:42.317408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.907360+0000","last_change":"2024-09-12T23:33:29.907360+0000","last_active":"2024-09-12T23:33:29.907360+0000","last_peered":"2024-09-12T23:33:29.907360+0000","last_clean":"2024-09-12T23:33:29.907360+0000","last_became_active":"2024-09-12T23:33:29.907067+0000","last_became_peered":"2024-09-12T23:33:29.907067+0000","last_unstale":"2024-09-12T23:33:29.907360+0000","last_undegraded":"2024-09-12T23:33:29.907360+0000","last_fullsized":"2024-09-12T23:33:29.907360+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:26:38.084938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.896704+0000","last_change":"2024-09-12T23:33:28.896704+0000","last_active":"2024-09-12T23:33:28.896704+0000","last_peered":"2024-09-12T23:33:28.896704+0000","last_clean":"2024-09-12T23:33:28.896704+0000","last_became_active":"2024-09-12T23:33:28.896167+0000","last_became_peered":"2024-09-12T23:33:28.896167+0000","last_unstale":"2024-09-12T23:33:28.896704+0000","last_undegraded":"2024-09-12T23:33:28.896704+0000","last_fullsized":"2024-09-12T23:33:28.896704+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:14:47.959501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.894866+0000","last_change":"2024-09-12T23:33:29.893401+0000","last_active":"2024-09-12T23:33:29.894866+0000","last_peered":"2024-09-12T23:33:29.894866+0000","last_clean":"2024-09-12T23:33:29.894866+0000","last_became_active":"2024-09-12T23:33:29.892988+0000","last_became_peered":"2024-09-12T23:33:29.892988+0000","last_unstale":"2024-09-12T23:33:29.894866+0000","last_undegraded":"2024-09-12T23:33:29.894866+0000","last_fullsized":"2024-09-12T23:33:29.894866+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:30:04.271132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":3,"reported_epoch":52,"state":"creating+activating","last_fresh":"2024-09-12T23:33:28.848624+0000","last_change":"2024-09-12T23:33:28.848624+0000","last_active":"2024-09-12T23:33:27.830370+0000","last_peered":"2024-09-12T23:33:27.830370+0000","last_clean":"2024-09-12T23:33:27.830370+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.848624+0000","last_undegraded":"2024-09-12T23:33:28.848624+0000","last_fullsized":"2024-09-12T23:33:28.848624+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:20:27.774655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":["6","2","5"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.866433+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-13T23:52:12.475067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,7],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.890012+0000","last_change":"2024-09-12T23:33:28.890012+0000","last_active":"2024-09-12T23:33:28.890012+0000","last_peered":"2024-09-12T23:33:28.890012+0000","last_clean":"2024-09-12T23:33:28.890012+0000","last_became_active":"2024-09-12T23:33:28.887477+0000","last_became_peered":"2024-09-12T23:33:28.887477+0000","last_unstale":"2024-09-12T23:33:28.890012+0000","last_undegraded":"2024-09-12T23:33:28.890012+0000","last_fullsized":"2024-09-12T23:33:28.890012+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:15:02.988389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.15","version":"53'3","reported_seq":14,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.917038+0000","last_change":"2024-09-12T23:33:28.879732+0000","last_active":"2024-09-12T23:33:29.917038+0000","last_peered":"2024-09-12T23:33:29.917038+0000","last_clean":"2024-09-12T23:33:29.917038+0000","last_became_active":"2024-09-12T23:33:28.879399+0000","last_became_peered":"2024-09-12T23:33:28.879399+0000","last_unstale":"2024-09-12T23:33:29.917038+0000","last_undegraded":"2024-09-12T23:33:29.917038+0000","last_fullsized":"2024-09-12T23:33:29.917038+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:00:06.109352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.14","version":"0'0","reported_seq":2,"reported_epoch":52,"state":"creating+activating","last_fresh":"2024-09-12T23:33:28.847672+0000","last_change":"2024-09-12T23:33:28.848228+0000","last_active":"2024-09-12T23:33:27.830370+0000","last_peered":"2024-09-12T23:33:27.830370+0000","last_clean":"2024-09-12T23:33:27.830370+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.847672+0000","last_undegraded":"2024-09-12T23:33:28.847672+0000","last_fullsized":"2024-09-12T23:33:28.847672+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:50:58.477301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":["6","3","5"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.858409+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:06:00.373844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[3,4],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.13","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:27.830370+0000","last_change":"2024-09-12T23:33:27.837333+0000","last_active":"2024-09-12T23:33:27.830370+0000","last_peered":"2024-09-12T23:33:27.830370+0000","last_clean":"2024-09-12T23:33:27.830370+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:27.830370+0000","last_undegraded":"2024-09-12T23:33:27.830370+0000","last_fullsized":"2024-09-12T23:33:27.830370+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:00:55.966645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[3,4],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.899424+0000","last_change":"2024-09-12T23:33:28.899424+0000","last_active":"2024-09-12T23:33:28.899424+0000","last_peered":"2024-09-12T23:33:28.899424+0000","last_clean":"2024-09-12T23:33:28.899424+0000","last_became_active":"2024-09-12T23:33:28.898942+0000","last_became_peered":"2024-09-12T23:33:28.898942+0000","last_unstale":"2024-09-12T23:33:28.899424+0000","last_undegraded":"2024-09-12T23:33:28.899424+0000","last_fullsized":"2024-09-12T23:33:28.899424+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:14:47.069896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.847369+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:28:55.241684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[2,4],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":3,"reported_epoch":52,"state":"creating+activating","last_fresh":"2024-09-12T23:33:28.846921+0000","last_change":"2024-09-12T23:33:28.846921+0000","last_active":"2024-09-12T23:33:27.830370+0000","last_peered":"2024-09-12T23:33:27.830370+0000","last_clean":"2024-09-12T23:33:27.830370+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.846921+0000","last_undegraded":"2024-09-12T23:33:28.846921+0000","last_fullsized":"2024-09-12T23:33:28.846921+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:27:43.449941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":["6","1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.847037+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:29:56.772001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,5],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.863788+0000","last_change":"2024-09-12T23:33:28.863788+0000","last_active":"2024-09-12T23:33:28.863788+0000","last_peered":"2024-09-12T23:33:28.863788+0000","last_clean":"2024-09-12T23:33:28.863788+0000","last_became_active":"2024-09-12T23:33:28.863479+0000","last_became_peered":"2024-09-12T23:33:28.863479+0000","last_unstale":"2024-09-12T23:33:28.863788+0000","last_undegraded":"2024-09-12T23:33:28.863788+0000","last_fullsized":"2024-09-12T23:33:28.863788+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:26:09.204038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.847091+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:57:03.461858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[4,6],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:27.830370+0000","last_change":"2024-09-12T23:33:27.840289+0000","last_active":"2024-09-12T23:33:27.830370+0000","last_peered":"2024-09-12T23:33:27.830370+0000","last_clean":"2024-09-12T23:33:27.830370+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:27.830370+0000","last_undegraded":"2024-09-12T23:33:27.830370+0000","last_fullsized":"2024-09-12T23:33:27.830370+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:54:14.860070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,7],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.849355+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:45:39.430174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,4],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.887441+0000","last_change":"2024-09-12T23:33:28.887441+0000","last_active":"2024-09-12T23:33:28.887441+0000","last_peered":"2024-09-12T23:33:28.887441+0000","last_clean":"2024-09-12T23:33:28.887441+0000","last_became_active":"2024-09-12T23:33:28.887109+0000","last_became_peered":"2024-09-12T23:33:28.887109+0000","last_unstale":"2024-09-12T23:33:28.887441+0000","last_undegraded":"2024-09-12T23:33:28.887441+0000","last_fullsized":"2024-09-12T23:33:28.887441+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:35:14.071342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.859939+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:06:23.076336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,4],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"53'1","reported_seq":12,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.920461+0000","last_change":"2024-09-12T23:33:28.871944+0000","last_active":"2024-09-12T23:33:29.920461+0000","last_peered":"2024-09-12T23:33:29.920461+0000","last_clean":"2024-09-12T23:33:29.920461+0000","last_became_active":"2024-09-12T23:33:28.871628+0000","last_became_peered":"2024-09-12T23:33:28.871628+0000","last_unstale":"2024-09-12T23:33:29.920461+0000","last_undegraded":"2024-09-12T23:33:29.920461+0000","last_fullsized":"2024-09-12T23:33:29.920461+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:58:29.645624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.874292+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:42:32.768139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[3,6],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.c","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.882138+0000","last_change":"2024-09-12T23:33:28.882138+0000","last_active":"2024-09-12T23:33:28.882138+0000","last_peered":"2024-09-12T23:33:28.882138+0000","last_clean":"2024-09-12T23:33:28.882138+0000","last_became_active":"2024-09-12T23:33:28.881774+0000","last_became_peered":"2024-09-12T23:33:28.881774+0000","last_unstale":"2024-09-12T23:33:28.882138+0000","last_undegraded":"2024-09-12T23:33:28.882138+0000","last_fullsized":"2024-09-12T23:33:28.882138+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:07:36.075590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.857274+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:24:29.251601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[5,6],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"21'72","reported_seq":18,"reported_epoch":51,"state":"active+clean","last_fresh":"2024-09-12T23:33:27.838959+0000","last_change":"2024-09-12T23:32:45.902460+0000","last_active":"2024-09-12T23:33:27.838959+0000","last_peered":"2024-09-12T23:33:27.838959+0000","last_clean":"2024-09-12T23:33:27.838959+0000","last_became_active":"2024-09-12T23:32:45.887205+0000","last_became_peered":"2024-09-12T23:32:45.887205+0000","last_unstale":"2024-09-12T23:33:27.838959+0000","last_undegraded":"2024-09-12T23:33:27.838959+0000","last_fullsized":"2024-09-12T23:33:27.838959+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:28:34.324093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:28:34.324093+0000","last_clean_scrub_stamp":"2024-09-12T23:28:34.324093+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:37:37.825919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.898796+0000","last_change":"2024-09-12T23:33:28.898796+0000","last_active":"2024-09-12T23:33:28.898796+0000","last_peered":"2024-09-12T23:33:28.898796+0000","last_clean":"2024-09-12T23:33:28.898796+0000","last_became_active":"2024-09-12T23:33:28.898329+0000","last_became_peered":"2024-09-12T23:33:28.898329+0000","last_unstale":"2024-09-12T23:33:28.898796+0000","last_undegraded":"2024-09-12T23:33:28.898796+0000","last_fullsized":"2024-09-12T23:33:28.898796+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:54:48.474286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.905028+0000","last_change":"2024-09-12T23:33:29.905028+0000","last_active":"2024-09-12T23:33:29.905028+0000","last_peered":"2024-09-12T23:33:29.905028+0000","last_clean":"2024-09-12T23:33:29.905028+0000","last_became_active":"2024-09-12T23:33:29.904695+0000","last_became_peered":"2024-09-12T23:33:29.904695+0000","last_unstale":"2024-09-12T23:33:29.905028+0000","last_undegraded":"2024-09-12T23:33:29.905028+0000","last_fullsized":"2024-09-12T23:33:29.905028+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:06:47.004852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.895135+0000","last_change":"2024-09-12T23:33:28.895135+0000","last_active":"2024-09-12T23:33:28.895135+0000","last_peered":"2024-09-12T23:33:28.895135+0000","last_clean":"2024-09-12T23:33:28.895135+0000","last_became_active":"2024-09-12T23:33:28.894687+0000","last_became_peered":"2024-09-12T23:33:28.894687+0000","last_unstale":"2024-09-12T23:33:28.895135+0000","last_undegraded":"2024-09-12T23:33:28.895135+0000","last_fullsized":"2024-09-12T23:33:28.895135+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:57:06.646238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.868957+0000","last_change":"2024-09-12T23:33:28.868957+0000","last_active":"2024-09-12T23:33:28.868957+0000","last_peered":"2024-09-12T23:33:28.868957+0000","last_clean":"2024-09-12T23:33:28.868957+0000","last_became_active":"2024-09-12T23:33:28.868587+0000","last_became_peered":"2024-09-12T23:33:28.868587+0000","last_unstale":"2024-09-12T23:33:28.868957+0000","last_undegraded":"2024-09-12T23:33:28.868957+0000","last_fullsized":"2024-09-12T23:33:28.868957+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:45:32.284209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.892338+0000","last_change":"2024-09-12T23:33:29.890360+0000","last_active":"2024-09-12T23:33:29.892338+0000","last_peered":"2024-09-12T23:33:29.892338+0000","last_clean":"2024-09-12T23:33:29.892338+0000","last_became_active":"2024-09-12T23:33:29.889575+0000","last_became_peered":"2024-09-12T23:33:29.889575+0000","last_unstale":"2024-09-12T23:33:29.892338+0000","last_undegraded":"2024-09-12T23:33:29.892338+0000","last_fullsized":"2024-09-12T23:33:29.892338+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:36:34.938054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.899931+0000","last_change":"2024-09-12T23:33:28.899931+0000","last_active":"2024-09-12T23:33:28.899931+0000","last_peered":"2024-09-12T23:33:28.899931+0000","last_clean":"2024-09-12T23:33:28.899931+0000","last_became_active":"2024-09-12T23:33:28.899410+0000","last_became_peered":"2024-09-12T23:33:28.899410+0000","last_unstale":"2024-09-12T23:33:28.899931+0000","last_undegraded":"2024-09-12T23:33:28.899931+0000","last_fullsized":"2024-09-12T23:33:28.899931+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:05:26.671172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.878775+0000","last_change":"2024-09-12T23:33:28.867304+0000","last_active":"2024-09-12T23:33:29.878775+0000","last_peered":"2024-09-12T23:33:29.878775+0000","last_clean":"2024-09-12T23:33:29.878775+0000","last_became_active":"2024-09-12T23:33:28.867020+0000","last_became_peered":"2024-09-12T23:33:28.867020+0000","last_unstale":"2024-09-12T23:33:29.878775+0000","last_undegraded":"2024-09-12T23:33:29.878775+0000","last_fullsized":"2024-09-12T23:33:29.878775+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:51:22.908188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.865496+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:38:00.819958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[2,3],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.895077+0000","last_change":"2024-09-12T23:33:28.895077+0000","last_active":"2024-09-12T23:33:28.895077+0000","last_peered":"2024-09-12T23:33:28.895077+0000","last_clean":"2024-09-12T23:33:28.895077+0000","last_became_active":"2024-09-12T23:33:28.892268+0000","last_became_peered":"2024-09-12T23:33:28.892268+0000","last_unstale":"2024-09-12T23:33:28.895077+0000","last_undegraded":"2024-09-12T23:33:28.895077+0000","last_fullsized":"2024-09-12T23:33:28.895077+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:26:19.125877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.913453+0000","last_change":"2024-09-12T23:33:29.899700+0000","last_active":"2024-09-12T23:33:29.913453+0000","last_peered":"2024-09-12T23:33:29.913453+0000","last_clean":"2024-09-12T23:33:29.913453+0000","last_became_active":"2024-09-12T23:33:29.899381+0000","last_became_peered":"2024-09-12T23:33:29.899381+0000","last_unstale":"2024-09-12T23:33:29.913453+0000","last_undegraded":"2024-09-12T23:33:29.913453+0000","last_fullsized":"2024-09-12T23:33:29.913453+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:30:20.590373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.875764+0000","last_change":"2024-09-12T23:33:28.866367+0000","last_active":"2024-09-12T23:33:29.875764+0000","last_peered":"2024-09-12T23:33:29.875764+0000","last_clean":"2024-09-12T23:33:29.875764+0000","last_became_active":"2024-09-12T23:33:28.866005+0000","last_became_peered":"2024-09-12T23:33:28.866005+0000","last_unstale":"2024-09-12T23:33:29.875764+0000","last_undegraded":"2024-09-12T23:33:29.875764+0000","last_fullsized":"2024-09-12T23:33:29.875764+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:49:27.803045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.915275+0000","last_change":"2024-09-12T23:33:29.915275+0000","last_active":"2024-09-12T23:33:29.915275+0000","last_peered":"2024-09-12T23:33:29.915275+0000","last_clean":"2024-09-12T23:33:29.915275+0000","last_became_active":"2024-09-12T23:33:29.914934+0000","last_became_peered":"2024-09-12T23:33:29.914934+0000","last_unstale":"2024-09-12T23:33:29.915275+0000","last_undegraded":"2024-09-12T23:33:29.915275+0000","last_fullsized":"2024-09-12T23:33:29.915275+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:43:27.562543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":3,"reported_epoch":52,"state":"creating+activating","last_fresh":"2024-09-12T23:33:28.847239+0000","last_change":"2024-09-12T23:33:28.847239+0000","last_active":"2024-09-12T23:33:27.830370+0000","last_peered":"2024-09-12T23:33:27.830370+0000","last_clean":"2024-09-12T23:33:27.830370+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.847239+0000","last_undegraded":"2024-09-12T23:33:28.847239+0000","last_fullsized":"2024-09-12T23:33:28.847239+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:58:40.820895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":["6","2","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.8","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.894286+0000","last_change":"2024-09-12T23:33:28.894286+0000","last_active":"2024-09-12T23:33:28.894286+0000","last_peered":"2024-09-12T23:33:28.894286+0000","last_clean":"2024-09-12T23:33:28.894286+0000","last_became_active":"2024-09-12T23:33:28.893861+0000","last_became_peered":"2024-09-12T23:33:28.893861+0000","last_unstale":"2024-09-12T23:33:28.894286+0000","last_undegraded":"2024-09-12T23:33:28.894286+0000","last_fullsized":"2024-09-12T23:33:28.894286+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:20:01.806723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":11,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.874494+0000","last_change":"2024-09-12T23:33:28.884360+0000","last_active":"2024-09-12T23:33:29.874494+0000","last_peered":"2024-09-12T23:33:29.874494+0000","last_clean":"2024-09-12T23:33:29.874494+0000","last_became_active":"2024-09-12T23:33:28.884027+0000","last_became_peered":"2024-09-12T23:33:28.884027+0000","last_unstale":"2024-09-12T23:33:29.874494+0000","last_undegraded":"2024-09-12T23:33:29.874494+0000","last_fullsized":"2024-09-12T23:33:29.874494+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:11:57.567074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.915243+0000","last_change":"2024-09-12T23:33:29.915243+0000","last_active":"2024-09-12T23:33:29.915243+0000","last_peered":"2024-09-12T23:33:29.915243+0000","last_clean":"2024-09-12T23:33:29.915243+0000","last_became_active":"2024-09-12T23:33:29.914543+0000","last_became_peered":"2024-09-12T23:33:29.914543+0000","last_unstale":"2024-09-12T23:33:29.915243+0000","last_undegraded":"2024-09-12T23:33:29.915243+0000","last_fullsized":"2024-09-12T23:33:29.915243+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-13T23:48:16.023985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":11,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.879749+0000","last_change":"2024-09-12T23:33:28.883601+0000","last_active":"2024-09-12T23:33:29.879749+0000","last_peered":"2024-09-12T23:33:29.879749+0000","last_clean":"2024-09-12T23:33:29.879749+0000","last_became_active":"2024-09-12T23:33:28.883276+0000","last_became_peered":"2024-09-12T23:33:28.883276+0000","last_unstale":"2024-09-12T23:33:29.879749+0000","last_undegraded":"2024-09-12T23:33:29.879749+0000","last_fullsized":"2024-09-12T23:33:29.879749+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:26:21.733615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.908185+0000","last_change":"2024-09-12T23:33:29.908185+0000","last_active":"2024-09-12T23:33:29.908185+0000","last_peered":"2024-09-12T23:33:29.908185+0000","last_clean":"2024-09-12T23:33:29.908185+0000","last_became_active":"2024-09-12T23:33:29.907814+0000","last_became_peered":"2024-09-12T23:33:29.907814+0000","last_unstale":"2024-09-12T23:33:29.908185+0000","last_undegraded":"2024-09-12T23:33:29.908185+0000","last_fullsized":"2024-09-12T23:33:29.908185+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:45:47.207242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.902988+0000","last_change":"2024-09-12T23:33:28.902988+0000","last_active":"2024-09-12T23:33:28.902988+0000","last_peered":"2024-09-12T23:33:28.902988+0000","last_clean":"2024-09-12T23:33:28.902988+0000","last_became_active":"2024-09-12T23:33:28.902514+0000","last_became_peered":"2024-09-12T23:33:28.902514+0000","last_unstale":"2024-09-12T23:33:28.902988+0000","last_undegraded":"2024-09-12T23:33:28.902988+0000","last_fullsized":"2024-09-12T23:33:28.902988+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:29:42.595057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.848452+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:12:37.049274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,4],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.866904+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:49:53.725680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[4,6],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.902224+0000","last_change":"2024-09-12T23:33:28.902224+0000","last_active":"2024-09-12T23:33:28.902224+0000","last_peered":"2024-09-12T23:33:28.902224+0000","last_clean":"2024-09-12T23:33:28.902224+0000","last_became_active":"2024-09-12T23:33:28.901755+0000","last_became_peered":"2024-09-12T23:33:28.901755+0000","last_unstale":"2024-09-12T23:33:28.902224+0000","last_undegraded":"2024-09-12T23:33:28.902224+0000","last_fullsized":"2024-09-12T23:33:28.902224+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:06:15.504175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.875336+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:20:24.609184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,4],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1d","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.897466+0000","last_change":"2024-09-12T23:33:28.897466+0000","last_active":"2024-09-12T23:33:28.897466+0000","last_peered":"2024-09-12T23:33:28.897466+0000","last_clean":"2024-09-12T23:33:28.897466+0000","last_became_active":"2024-09-12T23:33:28.896936+0000","last_became_peered":"2024-09-12T23:33:28.896936+0000","last_unstale":"2024-09-12T23:33:28.897466+0000","last_undegraded":"2024-09-12T23:33:28.897466+0000","last_fullsized":"2024-09-12T23:33:28.897466+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:29:38.732168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.1e","version":"53'1","reported_seq":11,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.875384+0000","last_change":"2024-09-12T23:33:28.894353+0000","last_active":"2024-09-12T23:33:29.875384+0000","last_peered":"2024-09-12T23:33:29.875384+0000","last_clean":"2024-09-12T23:33:29.875384+0000","last_became_active":"2024-09-12T23:33:28.893992+0000","last_became_peered":"2024-09-12T23:33:28.893992+0000","last_unstale":"2024-09-12T23:33:29.875384+0000","last_undegraded":"2024-09-12T23:33:29.875384+0000","last_fullsized":"2024-09-12T23:33:29.875384+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:43:43.889025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.891227+0000","last_change":"2024-09-12T23:33:29.891227+0000","last_active":"2024-09-12T23:33:29.891227+0000","last_peered":"2024-09-12T23:33:29.891227+0000","last_clean":"2024-09-12T23:33:29.891227+0000","last_became_active":"2024-09-12T23:33:29.890368+0000","last_became_peered":"2024-09-12T23:33:29.890368+0000","last_unstale":"2024-09-12T23:33:29.891227+0000","last_undegraded":"2024-09-12T23:33:29.891227+0000","last_fullsized":"2024-09-12T23:33:29.891227+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:38:37.526610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:27.830370+0000","last_change":"2024-09-12T23:33:27.837410+0000","last_active":"2024-09-12T23:33:27.830370+0000","last_peered":"2024-09-12T23:33:27.830370+0000","last_clean":"2024-09-12T23:33:27.830370+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:27.830370+0000","last_undegraded":"2024-09-12T23:33:27.830370+0000","last_fullsized":"2024-09-12T23:33:27.830370+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:42:19.518598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[4,7],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":147,"acting":147,"num_store_stats":6},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":160,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3563520,"data_stored":3542208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430219,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298432,"kb_used_data":1072,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446720,"statfs":{"total":95995035648,"available":95689441280,"internally_reserved":0,"allocated":1097728,"data_stored":796122,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.74299999999999999}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.69799999999999995}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47499999999999998}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.82099999999999995}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64000000000000001}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.90200000000000002}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.83899999999999997}]}]},{"osd":6,"up_from":42,"seq":180388626453,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298412,"kb_used_data":1056,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446740,"statfs":{"total":95995035648,"available":95689461760,"internally_reserved":0,"allocated":1081344,"data_stored":789114,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 12 23:32:56 2024","interfaces":[{"interface":"back","average":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"min":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":1.2450000000000001,"5min":1.2450000000000001,"15min":1.2450000000000001},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.874,"5min":0.874,"15min":0.874},"min":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"max":{"1min":1.222,"5min":1.222,"15min":1.222},"last":0.82099999999999995}]},{"osd":1,"last update":"Thu Sep 12 23:32:56 2024","interfaces":[{"interface":"back","average":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"min":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":1.153,"5min":1.153,"15min":1.153},"last":0.89800000000000002},{"interface":"front","average":{"1min":0.871,"5min":0.871,"15min":0.871},"min":{"1min":0.62,"5min":0.62,"15min":0.62},"max":{"1min":1.306,"5min":1.306,"15min":1.306},"last":1.1499999999999999}]},{"osd":2,"last update":"Thu Sep 12 23:32:56 2024","interfaces":[{"interface":"back","average":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"min":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"max":{"1min":1.4359999999999999,"5min":1.4359999999999999,"15min":1.4359999999999999},"last":0.76900000000000002},{"interface":"front","average":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"min":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"max":{"1min":1.131,"5min":1.131,"15min":1.131},"last":0.49199999999999999}]},{"osd":3,"last update":"Thu Sep 12 23:32:56 2024","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"min":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"max":{"1min":1.2330000000000001,"5min":1.2330000000000001,"15min":1.2330000000000001},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"min":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"max":{"1min":1.2290000000000001,"5min":1.2290000000000001,"15min":1.2290000000000001},"last":1.018}]},{"osd":4,"last update":"Thu Sep 12 23:32:56 2024","interfaces":[{"interface":"back","average":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"min":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":1.131,"5min":1.131,"15min":1.131},"last":0.95399999999999996},{"interface":"front","average":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":1.389,"5min":1.389,"15min":1.389},"last":0.97999999999999998}]},{"osd":5,"last update":"Thu Sep 12 23:32:56 2024","interfaces":[{"interface":"back","average":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"min":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"max":{"1min":1.228,"5min":1.228,"15min":1.228},"last":1.0780000000000001},{"interface":"front","average":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"min":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":1.3109999999999999,"5min":1.3109999999999999,"15min":1.3109999999999999},"last":1.105}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.139}]}]},{"osd":5,"up_from":36,"seq":154618822688,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297828,"kb_used_data":472,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447324,"statfs":{"total":95995035648,"available":95690059776,"internally_reserved":0,"allocated":483328,"data_stored":198431,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 12 23:33:07 2024","interfaces":[{"interface":"back","average":{"1min":0.81399999999999995,"5min":0.78700000000000003,"15min":0.78300000000000003},"min":{"1min":0.56200000000000006,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":1.0740000000000001,"5min":1.6100000000000001,"15min":1.6100000000000001},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.872,"5min":0.83399999999999996,"15min":0.82799999999999996},"min":{"1min":0.68999999999999995,"5min":0.58199999999999996,"15min":0.58199999999999996},"max":{"1min":1.0469999999999999,"5min":1.0840000000000001,"15min":1.0840000000000001},"last":0.56799999999999995}]},{"osd":1,"last update":"Thu Sep 12 23:33:07 2024","interfaces":[{"interface":"back","average":{"1min":0.88100000000000001,"5min":0.85199999999999998,"15min":0.84699999999999998},"min":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"max":{"1min":1.218,"5min":1.4470000000000001,"15min":1.4470000000000001},"last":0.81799999999999995},{"interface":"front","average":{"1min":0.89000000000000001,"5min":0.81200000000000006,"15min":0.79900000000000004},"min":{"1min":0.66300000000000003,"5min":0.502,"15min":0.502},"max":{"1min":1.274,"5min":1.274,"15min":1.274},"last":0.52800000000000002}]},{"osd":2,"last update":"Thu Sep 12 23:33:07 2024","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":0.878,"15min":0.87},"min":{"1min":0.73999999999999999,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":1.2509999999999999,"5min":1.2509999999999999,"15min":1.2509999999999999},"last":0.75},{"interface":"front","average":{"1min":0.876,"5min":0.90400000000000003,"15min":0.90900000000000003},"min":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"max":{"1min":1.0980000000000001,"5min":1.141,"15min":1.141},"last":0.77300000000000002}]},{"osd":3,"last update":"Thu Sep 12 23:33:07 2024","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":0.91300000000000003,"15min":0.91100000000000003},"min":{"1min":0.68100000000000005,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.2010000000000001,"5min":1.3280000000000001,"15min":1.3280000000000001},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.92000000000000004,"5min":0.91500000000000004,"15min":0.91400000000000003},"min":{"1min":0.67400000000000004,"5min":0.57499999999999996,"15min":0.57499999999999996},"max":{"1min":1.2609999999999999,"5min":1.536,"15min":1.536},"last":0.73399999999999999}]},{"osd":4,"last update":"Thu Sep 12 23:33:07 2024","interfaces":[{"interface":"back","average":{"1min":0.747,"5min":0.74299999999999999,"15min":0.74199999999999999},"min":{"1min":0.47199999999999998,"5min":0.308,"15min":0.308},"max":{"1min":1.1910000000000001,"5min":1.1910000000000001,"15min":1.1910000000000001},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.79500000000000004,"5min":0.77700000000000002,"15min":0.77400000000000002},"min":{"1min":0.504,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":1.1679999999999999,"5min":1.1679999999999999,"15min":1.1679999999999999},"last":0.67700000000000005}]},{"osd":6,"last update":"Thu Sep 12 23:32:54 2024","interfaces":[{"interface":"back","average":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":1.1220000000000001,"5min":1.1220000000000001,"15min":1.1220000000000001},"last":0.79700000000000004},{"interface":"front","average":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"min":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":1.0840000000000001,"5min":1.0840000000000001,"15min":1.0840000000000001},"last":0.65500000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.72099999999999997}]}]},{"osd":4,"up_from":30,"seq":128849018921,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297836,"kb_used_data":480,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447316,"statfs":{"total":95995035648,"available":95690051584,"internally_reserved":0,"allocated":491520,"data_stored":199061,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 12 23:33:17 2024","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":0.83599999999999997,"15min":0.83399999999999996},"min":{"1min":0.63200000000000001,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.375,"5min":1.375,"15min":1.375},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.89500000000000002,"5min":0.84699999999999998,"15min":0.84299999999999997},"min":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"max":{"1min":1.1479999999999999,"5min":1.1479999999999999,"15min":1.1479999999999999},"last":0.85999999999999999}]},{"osd":1,"last update":"Thu Sep 12 23:33:17 2024","interfaces":[{"interface":"back","average":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82399999999999995},"min":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":1.194,"5min":1.194,"15min":1.194},"last":1.01},{"interface":"front","average":{"1min":0.86899999999999999,"5min":0.85899999999999999,"15min":0.85699999999999998},"min":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.218,"5min":1.218,"15min":1.218},"last":0.80600000000000005}]},{"osd":2,"last update":"Thu Sep 12 23:33:17 2024","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":0.84599999999999997,"15min":0.85599999999999998},"min":{"1min":0.60199999999999998,"5min":0.55700000000000005,"15min":0.55700000000000005},"max":{"1min":1.175,"5min":1.175,"15min":1.175},"last":0.98599999999999999},{"interface":"front","average":{"1min":0.89500000000000002,"5min":0.85699999999999998,"15min":0.86299999999999999},"min":{"1min":0.66000000000000003,"5min":0.63500000000000001,"15min":0.63500000000000001},"max":{"1min":1.2909999999999999,"5min":1.2909999999999999,"15min":1.2909999999999999},"last":1.079}]},{"osd":3,"last update":"Thu Sep 12 23:33:17 2024","interfaces":[{"interface":"back","average":{"1min":0.89700000000000002,"5min":0.85199999999999998,"15min":0.84999999999999998},"min":{"1min":0.65200000000000002,"5min":0.60099999999999998,"15min":0.60099999999999998},"max":{"1min":1.5129999999999999,"5min":1.5129999999999999,"15min":1.5129999999999999},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.97899999999999998,"5min":0.90200000000000002,"15min":0.89000000000000001},"min":{"1min":0.57099999999999995,"5min":0.55900000000000005,"15min":0.55900000000000005},"max":{"1min":1.595,"5min":1.595,"15min":1.595},"last":1.214}]},{"osd":5,"last update":"Thu Sep 12 23:33:09 2024","interfaces":[{"interface":"back","average":{"1min":0.80500000000000005,"5min":0.78400000000000003,"15min":0.78000000000000003},"min":{"1min":0.49199999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":1.7430000000000001,"5min":1.7430000000000001,"15min":1.7430000000000001},"last":0.91200000000000003},{"interface":"front","average":{"1min":0.90300000000000002,"5min":0.82199999999999995,"15min":0.80800000000000005},"min":{"1min":0.375,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.716,"5min":1.716,"15min":1.716},"last":1.0489999999999999}]},{"osd":6,"last update":"Thu Sep 12 23:32:56 2024","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":0.91300000000000003,"15min":0.91300000000000003},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":1.7629999999999999,"5min":1.7629999999999999,"15min":1.7629999999999999},"last":1.1140000000000001},{"interface":"front","average":{"1min":0.96299999999999997,"5min":0.96299999999999997,"15min":0.96299999999999997},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":1.7010000000000001,"5min":1.7010000000000001,"15min":1.7010000000000001},"last":1.2849999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.1379999999999999}]}]},{"osd":3,"up_from":25,"seq":107374182451,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297880,"kb_used_data":524,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447272,"statfs":{"total":95995035648,"available":95690006528,"internally_reserved":0,"allocated":536576,"data_stored":214603,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 12 23:32:29 2024","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.62,"15min":0.60199999999999998},"min":{"1min":0.53400000000000003,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.995,"5min":0.999,"15min":0.999},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.78300000000000003,"5min":0.70499999999999996,"15min":0.69299999999999995},"min":{"1min":0.58299999999999996,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":1.3120000000000001,"5min":1.3120000000000001,"15min":1.3120000000000001},"last":0.46600000000000003}]},{"osd":1,"last update":"Thu Sep 12 23:32:29 2024","interfaces":[{"interface":"back","average":{"1min":0.755,"5min":0.67200000000000004,"15min":0.65600000000000003},"min":{"1min":0.54200000000000004,"5min":0.52200000000000002,"15min":0.52200000000000002},"max":{"1min":0.96999999999999997,"5min":0.96999999999999997,"15min":0.96999999999999997},"last":0.88600000000000001},{"interface":"front","average":{"1min":0.746,"5min":0.75,"15min":0.74199999999999999},"min":{"1min":0.57199999999999995,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":1.095,"5min":1.171,"15min":1.171},"last":0.61899999999999999}]},{"osd":2,"last update":"Thu Sep 12 23:32:29 2024","interfaces":[{"interface":"back","average":{"1min":0.81999999999999995,"5min":0.78900000000000003,"15min":0.77700000000000002},"min":{"1min":0.66100000000000003,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":0.96799999999999997,"5min":1.1140000000000001,"15min":1.1140000000000001},"last":0.78800000000000003},{"interface":"front","average":{"1min":0.85299999999999998,"5min":0.75800000000000001,"15min":0.73399999999999999},"min":{"1min":0.63800000000000001,"5min":0.54900000000000004,"15min":0.54900000000000004},"max":{"1min":1.238,"5min":1.238,"15min":1.238},"last":0.56299999999999994}]},{"osd":4,"last update":"Thu Sep 12 23:33:24 2024","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":0.90300000000000002,"15min":0.88200000000000001},"min":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.4950000000000001,"5min":1.4950000000000001,"15min":1.4950000000000001},"last":0.86099999999999999},{"interface":"front","average":{"1min":0.90300000000000002,"5min":0.87,"15min":0.84899999999999998},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":1.3640000000000001,"5min":1.3640000000000001,"15min":1.3640000000000001},"last":0.98299999999999998}]},{"osd":5,"last update":"Thu Sep 12 23:33:05 2024","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":0.98399999999999999,"15min":0.98399999999999999},"min":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"max":{"1min":1.321,"5min":1.371,"15min":1.371},"last":0.72499999999999998},{"interface":"front","average":{"1min":1.05,"5min":0.98199999999999998,"15min":0.97099999999999997},"min":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":1.423,"5min":1.423,"15min":1.423},"last":0.65600000000000003}]},{"osd":6,"last update":"Thu Sep 12 23:32:54 2024","interfaces":[{"interface":"back","average":{"1min":1.052,"5min":1.052,"15min":1.052},"min":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"max":{"1min":1.3540000000000001,"5min":1.3540000000000001,"15min":1.3540000000000001},"last":0.94499999999999995},{"interface":"front","average":{"1min":1.115,"5min":1.115,"15min":1.115},"min":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"max":{"1min":1.718,"5min":1.718,"15min":1.718},"last":0.96299999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.90800000000000003}]}]},{"osd":2,"up_from":18,"seq":77309411389,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297836,"kb_used_data":480,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447316,"statfs":{"total":95995035648,"available":95690051584,"internally_reserved":0,"allocated":491520,"data_stored":199061,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 12 23:32:41 2024","interfaces":[{"interface":"back","average":{"1min":0.72999999999999998,"5min":0.626,"15min":0.58799999999999997},"min":{"1min":0.53800000000000003,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":1.0429999999999999,"5min":1.23,"15min":1.23},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.745,"5min":0.66400000000000003,"15min":0.61299999999999999},"min":{"1min":0.50800000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":1.0940000000000001,"5min":1.0940000000000001,"15min":1.0940000000000001},"last":0.38300000000000001}]},{"osd":1,"last update":"Thu Sep 12 23:32:41 2024","interfaces":[{"interface":"back","average":{"1min":0.76400000000000001,"5min":0.67700000000000005,"15min":0.628},"min":{"1min":0.5,"5min":0.316,"15min":0.316},"max":{"1min":1.014,"5min":1.5329999999999999,"15min":1.5329999999999999},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.74199999999999999,"5min":0.68400000000000005,"15min":0.63700000000000001},"min":{"1min":0.60999999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.95099999999999996,"5min":1.0860000000000001,"15min":1.0860000000000001},"last":0.44}]},{"osd":3,"last update":"Thu Sep 12 23:32:33 2024","interfaces":[{"interface":"back","average":{"1min":0.80000000000000004,"5min":0.75800000000000001,"15min":0.74399999999999999},"min":{"1min":0.627,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":1.2270000000000001,"5min":1.2270000000000001,"15min":1.2270000000000001},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.83299999999999996,"5min":0.79800000000000004,"15min":0.78000000000000003},"min":{"1min":0.623,"5min":0.433,"15min":0.433},"max":{"1min":1.0049999999999999,"5min":1.6619999999999999,"15min":1.6619999999999999},"last":0.36599999999999999}]},{"osd":4,"last update":"Thu Sep 12 23:33:19 2024","interfaces":[{"interface":"back","average":{"1min":0.88300000000000001,"5min":0.873,"15min":0.86499999999999999},"min":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.107,"5min":1.1639999999999999,"15min":1.1639999999999999},"last":0.82499999999999996},{"interface":"front","average":{"1min":0.96899999999999997,"5min":0.95299999999999996,"15min":0.93500000000000005},"min":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"max":{"1min":1.23,"5min":1.363,"15min":1.363},"last":0.79000000000000004}]},{"osd":5,"last update":"Thu Sep 12 23:33:10 2024","interfaces":[{"interface":"back","average":{"1min":0.95599999999999996,"5min":1.022,"15min":1.0329999999999999},"min":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"max":{"1min":1.282,"5min":1.4830000000000001,"15min":1.4830000000000001},"last":0.89700000000000002},{"interface":"front","average":{"1min":0.90700000000000003,"5min":1.006,"15min":1.022},"min":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"max":{"1min":1.101,"5min":1.3129999999999999,"15min":1.3129999999999999},"last":0.63900000000000001}]},{"osd":6,"last update":"Thu Sep 12 23:32:56 2024","interfaces":[{"interface":"back","average":{"1min":0.94499999999999995,"5min":0.94499999999999995,"15min":0.94499999999999995},"min":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"max":{"1min":1.2330000000000001,"5min":1.2330000000000001,"15min":1.2330000000000001},"last":0.98099999999999998},{"interface":"front","average":{"1min":1.0880000000000001,"5min":1.0880000000000001,"15min":1.0880000000000001},"min":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"max":{"1min":1.498,"5min":1.498,"15min":1.498},"last":0.67000000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67800000000000005}]}]},{"osd":1,"up_from":13,"seq":55834574920,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297876,"kb_used_data":512,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447276,"statfs":{"total":95995035648,"available":95690010624,"internally_reserved":0,"allocated":524288,"data_stored":213107,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 12 23:32:55 2024","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":0.59899999999999998,"15min":0.53000000000000003},"min":{"1min":0.51400000000000001,"5min":0.255,"15min":0.255},"max":{"1min":1.1579999999999999,"5min":1.1579999999999999,"15min":1.1579999999999999},"last":0.69199999999999995},{"interface":"front","average":{"1min":0.77400000000000002,"5min":0.66000000000000003,"15min":0.58199999999999996},"min":{"1min":0.501,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":1.1830000000000001,"5min":1.1830000000000001,"15min":1.1830000000000001},"last":0.91300000000000003}]},{"osd":2,"last update":"Thu Sep 12 23:32:40 2024","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":0.64600000000000002,"15min":0.58799999999999997},"min":{"1min":0.54700000000000004,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.95899999999999996,"5min":1.046,"15min":1.046},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.82899999999999996,"5min":0.69999999999999996,"15min":0.629},"min":{"1min":0.53200000000000003,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":1.3049999999999999,"5min":1.3049999999999999,"15min":1.3049999999999999},"last":0.59399999999999997}]},{"osd":3,"last update":"Thu Sep 12 23:32:34 2024","interfaces":[{"interface":"back","average":{"1min":0.81999999999999995,"5min":0.79000000000000004,"15min":0.78100000000000003},"min":{"1min":0.56999999999999995,"5min":0.54800000000000004,"15min":0.54800000000000004},"max":{"1min":1.0249999999999999,"5min":1.1479999999999999,"15min":1.1479999999999999},"last":0.78100000000000003},{"interface":"front","average":{"1min":0.81299999999999994,"5min":0.77100000000000002,"15min":0.76200000000000001},"min":{"1min":0.61499999999999999,"5min":0.57799999999999996,"15min":0.57799999999999996},"max":{"1min":1.109,"5min":1.109,"15min":1.109},"last":0.84199999999999997}]},{"osd":4,"last update":"Thu Sep 12 23:33:16 2024","interfaces":[{"interface":"back","average":{"1min":0.95999999999999996,"5min":0.92100000000000004,"15min":0.91700000000000004},"min":{"1min":0.75700000000000001,"5min":0.65100000000000002,"15min":0.65100000000000002},"max":{"1min":1.23,"5min":1.585,"15min":1.585},"last":1.0620000000000001},{"interface":"front","average":{"1min":1.006,"5min":0.93700000000000006,"15min":0.92700000000000005},"min":{"1min":0.76000000000000001,"5min":0.73499999999999999,"15min":0.73499999999999999},"max":{"1min":1.389,"5min":1.4279999999999999,"15min":1.4279999999999999},"last":0.995}]},{"osd":5,"last update":"Thu Sep 12 23:33:05 2024","interfaces":[{"interface":"back","average":{"1min":0.95399999999999996,"5min":0.92100000000000004,"15min":0.91500000000000004},"min":{"1min":0.66000000000000003,"5min":0.65200000000000002,"15min":0.65200000000000002},"max":{"1min":1.484,"5min":1.6819999999999999,"15min":1.6819999999999999},"last":1.014},{"interface":"front","average":{"1min":1.0289999999999999,"5min":1.006,"15min":1.002},"min":{"1min":0.81000000000000005,"5min":0.66200000000000003,"15min":0.66200000000000003},"max":{"1min":1.5069999999999999,"5min":1.5069999999999999,"15min":1.5069999999999999},"last":0.879}]},{"osd":6,"last update":"Thu Sep 12 23:32:55 2024","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":1.0529999999999999,"15min":1.0529999999999999},"min":{"1min":0.746,"5min":0.746,"15min":0.746},"max":{"1min":1.5429999999999999,"5min":1.5429999999999999,"15min":1.5429999999999999},"last":0.93300000000000005},{"interface":"front","average":{"1min":1.101,"5min":1.101,"15min":1.101},"min":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"max":{"1min":1.5649999999999999,"5min":1.5649999999999999,"15min":1.5649999999999999},"last":0.96799999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.81200000000000006}]}]},{"osd":0,"up_from":8,"seq":34359738448,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298412,"kb_used_data":1056,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446740,"statfs":{"total":95995035648,"available":95689461760,"internally_reserved":0,"allocated":1081344,"data_stored":789114,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Sep 12 23:32:49 2024","interfaces":[{"interface":"back","average":{"1min":0.69599999999999995,"5min":0.59899999999999998,"15min":0.51100000000000001},"min":{"1min":0.48499999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":1.0509999999999999,"5min":1.0509999999999999,"15min":1.0509999999999999},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.74399999999999999,"5min":0.629,"15min":0.53300000000000003},"min":{"1min":0.55400000000000005,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.0049999999999999,"5min":1.028,"15min":1.028},"last":0.48699999999999999}]},{"osd":2,"last update":"Thu Sep 12 23:32:43 2024","interfaces":[{"interface":"back","average":{"1min":0.73299999999999998,"5min":0.66600000000000004,"15min":0.61899999999999999},"min":{"1min":0.52800000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":1.081,"5min":1.081,"15min":1.081},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.82699999999999996,"5min":0.69599999999999995,"15min":0.63900000000000001},"min":{"1min":0.56000000000000005,"5min":0.317,"15min":0.317},"max":{"1min":1.151,"5min":1.151,"15min":1.151},"last":0.81000000000000005}]},{"osd":3,"last update":"Thu Sep 12 23:32:27 2024","interfaces":[{"interface":"back","average":{"1min":0.79900000000000004,"5min":0.76800000000000002,"15min":0.76400000000000001},"min":{"1min":0.60999999999999999,"5min":0.60599999999999998,"15min":0.60599999999999998},"max":{"1min":1.177,"5min":1.177,"15min":1.177},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.83599999999999997,"5min":0.79600000000000004,"15min":0.79200000000000004},"min":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"max":{"1min":1.0009999999999999,"5min":1.1950000000000001,"15min":1.1950000000000001},"last":0.68400000000000005}]},{"osd":4,"last update":"Thu Sep 12 23:33:23 2024","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":0.93100000000000005,"15min":0.90200000000000002},"min":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"max":{"1min":1.304,"5min":1.304,"15min":1.304},"last":0.79000000000000004},{"interface":"front","average":{"1min":0.95699999999999996,"5min":0.876,"15min":0.84999999999999998},"min":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"max":{"1min":1.266,"5min":1.266,"15min":1.266},"last":0.67200000000000004}]},{"osd":5,"last update":"Thu Sep 12 23:33:10 2024","interfaces":[{"interface":"back","average":{"1min":0.94499999999999995,"5min":0.94299999999999995,"15min":0.94299999999999995},"min":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"max":{"1min":1.2330000000000001,"5min":1.2330000000000001,"15min":1.2330000000000001},"last":0.64200000000000002},{"interface":"front","average":{"1min":1.002,"5min":0.95899999999999996,"15min":0.95199999999999996},"min":{"1min":0.75,"5min":0.70099999999999996,"15min":0.70099999999999996},"max":{"1min":1.1579999999999999,"5min":1.1579999999999999,"15min":1.1579999999999999},"last":0.749}]},{"osd":6,"last update":"Thu Sep 12 23:32:55 2024","interfaces":[{"interface":"back","average":{"1min":1.1120000000000001,"5min":1.1120000000000001,"15min":1.1120000000000001},"min":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"max":{"1min":1.492,"5min":1.492,"15min":1.492},"last":0.98199999999999998},{"interface":"front","average":{"1min":1.0509999999999999,"5min":1.0509999999999999,"15min":1.0509999999999999},"min":{"1min":0.754,"5min":0.754,"15min":0.754},"max":{"1min":1.377,"5min":1.377,"15min":1.377},"last":0.90700000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.96699999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":1656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-12T23:33:34.074 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.3f is not active+clean 2024-09-12T23:33:34.074 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.3f', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'creating+peering', 'last_fresh': '2024-09-12T23:33:28.834450+0000', 'last_change': '2024-09-12T23:33:28.861074+0000', 'last_active': '2024-09-12T23:33:28.834450+0000', 'last_peered': '2024-09-12T23:33:28.834450+0000', 'last_clean': '2024-09-12T23:33:28.834450+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:28.834450+0000', 'last_undegraded': '2024-09-12T23:33:28.834450+0000', 'last_fullsized': '2024-09-12T23:33:28.834450+0000', 'mapping_epoch': 52, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 52, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2024-09-14T07:54:27.307253+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [7, 6, 0], 'acting': [7, 6, 0], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [0, 6], 'up_primary': 7, 'acting_primary': 7, 'purged_snaps': []} 2024-09-12T23:33:34.074 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.3d is not active+clean 2024-09-12T23:33:34.074 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.3d', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-09-12T23:33:28.834450+0000', 'last_change': '2024-09-12T23:33:28.834450+0000', 'last_active': '2024-09-12T23:33:28.834450+0000', 'last_peered': '2024-09-12T23:33:28.834450+0000', 'last_clean': '2024-09-12T23:33:28.834450+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:28.834450+0000', 'last_undegraded': '2024-09-12T23:33:28.834450+0000', 'last_fullsized': '2024-09-12T23:33:28.834450+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-09-12T23:33:34.074 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.3c is not active+clean 2024-09-12T23:33:34.074 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.3c', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'creating+peering', 'last_fresh': '2024-09-12T23:33:28.834450+0000', 'last_change': '2024-09-12T23:33:28.876654+0000', 'last_active': '2024-09-12T23:33:28.834450+0000', 'last_peered': '2024-09-12T23:33:28.834450+0000', 'last_clean': '2024-09-12T23:33:28.834450+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:28.834450+0000', 'last_undegraded': '2024-09-12T23:33:28.834450+0000', 'last_fullsized': '2024-09-12T23:33:28.834450+0000', 'mapping_epoch': 52, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 52, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2024-09-14T05:18:46.573142+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [5, 3, 7], 'acting': [5, 3, 7], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [3, 7], 'up_primary': 5, 'acting_primary': 5, 'purged_snaps': []} 2024-09-12T23:33:34.074 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.3b is not active+clean 2024-09-12T23:33:34.074 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.3b', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'creating+peering', 'last_fresh': '2024-09-12T23:33:28.834450+0000', 'last_change': '2024-09-12T23:33:28.859085+0000', 'last_active': '2024-09-12T23:33:28.834450+0000', 'last_peered': '2024-09-12T23:33:28.834450+0000', 'last_clean': '2024-09-12T23:33:28.834450+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:28.834450+0000', 'last_undegraded': '2024-09-12T23:33:28.834450+0000', 'last_fullsized': '2024-09-12T23:33:28.834450+0000', 'mapping_epoch': 52, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 52, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2024-09-14T05:07:03.294377+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [7, 2, 4], 'acting': [7, 2, 4], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [2, 4], 'up_primary': 7, 'acting_primary': 7, 'purged_snaps': []} 2024-09-12T23:33:34.075 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.3a is not active+clean 2024-09-12T23:33:34.075 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.3a', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'creating+peering', 'last_fresh': '2024-09-12T23:33:28.834450+0000', 'last_change': '2024-09-12T23:33:28.873576+0000', 'last_active': '2024-09-12T23:33:28.834450+0000', 'last_peered': '2024-09-12T23:33:28.834450+0000', 'last_clean': '2024-09-12T23:33:28.834450+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:28.834450+0000', 'last_undegraded': '2024-09-12T23:33:28.834450+0000', 'last_fullsized': '2024-09-12T23:33:28.834450+0000', 'mapping_epoch': 52, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 52, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2024-09-14T09:07:11.163630+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [5, 0, 1], 'acting': [5, 0, 1], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [0, 1], 'up_primary': 5, 'acting_primary': 5, 'purged_snaps': []} 2024-09-12T23:33:34.075 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.39 is not active+clean 2024-09-12T23:33:34.075 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.39', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'creating+peering', 'last_fresh': '2024-09-12T23:33:28.834450+0000', 'last_change': '2024-09-12T23:33:28.850034+0000', 'last_active': '2024-09-12T23:33:28.834450+0000', 'last_peered': '2024-09-12T23:33:28.834450+0000', 'last_clean': '2024-09-12T23:33:28.834450+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:28.834450+0000', 'last_undegraded': '2024-09-12T23:33:28.834450+0000', 'last_fullsized': '2024-09-12T23:33:28.834450+0000', 'mapping_epoch': 52, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 52, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2024-09-14T09:17:57.421440+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [7, 0, 4], 'acting': [7, 0, 4], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [0, 4], 'up_primary': 7, 'acting_primary': 7, 'purged_snaps': []} 2024-09-12T23:33:34.075 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.37 is not active+clean 2024-09-12T23:33:34.075 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.37', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'creating+peering', 'last_fresh': '2024-09-12T23:33:28.834450+0000', 'last_change': '2024-09-12T23:33:28.856686+0000', 'last_active': '2024-09-12T23:33:28.834450+0000', 'last_peered': '2024-09-12T23:33:28.834450+0000', 'last_clean': '2024-09-12T23:33:28.834450+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:28.834450+0000', 'last_undegraded': '2024-09-12T23:33:28.834450+0000', 'last_fullsized': '2024-09-12T23:33:28.834450+0000', 'mapping_epoch': 52, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 52, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2024-09-14T01:41:19.476191+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [7, 5, 0], 'acting': [7, 5, 0], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [0, 5], 'up_primary': 7, 'acting_primary': 7, 'purged_snaps': []} 2024-09-12T23:33:34.075 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.36 is not active+clean 2024-09-12T23:33:34.075 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.36', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'creating+peering', 'last_fresh': '2024-09-12T23:33:28.834450+0000', 'last_change': '2024-09-12T23:33:28.852231+0000', 'last_active': '2024-09-12T23:33:28.834450+0000', 'last_peered': '2024-09-12T23:33:28.834450+0000', 'last_clean': '2024-09-12T23:33:28.834450+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:28.834450+0000', 'last_undegraded': '2024-09-12T23:33:28.834450+0000', 'last_fullsized': '2024-09-12T23:33:28.834450+0000', 'mapping_epoch': 52, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 52, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2024-09-14T08:56:38.506429+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [6, 5, 1], 'acting': [6, 5, 1], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [1, 5], 'up_primary': 6, 'acting_primary': 6, 'purged_snaps': []} 2024-09-12T23:33:34.075 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.35 is not active+clean 2024-09-12T23:33:34.075 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.35', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'creating+peering', 'last_fresh': '2024-09-12T23:33:28.834450+0000', 'last_change': '2024-09-12T23:33:28.855386+0000', 'last_active': '2024-09-12T23:33:28.834450+0000', 'last_peered': '2024-09-12T23:33:28.834450+0000', 'last_clean': '2024-09-12T23:33:28.834450+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:28.834450+0000', 'last_undegraded': '2024-09-12T23:33:28.834450+0000', 'last_fullsized': '2024-09-12T23:33:28.834450+0000', 'mapping_epoch': 52, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 52, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2024-09-14T01:08:21.774244+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [6, 5, 0], 'acting': [6, 5, 0], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [0, 5], 'up_primary': 6, 'acting_primary': 6, 'purged_snaps': []} 2024-09-12T23:33:34.075 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.34 is not active+clean 2024-09-12T23:33:34.075 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.34', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-09-12T23:33:28.834450+0000', 'last_change': '2024-09-12T23:33:28.834450+0000', 'last_active': '2024-09-12T23:33:28.834450+0000', 'last_peered': '2024-09-12T23:33:28.834450+0000', 'last_clean': '2024-09-12T23:33:28.834450+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:28.834450+0000', 'last_undegraded': '2024-09-12T23:33:28.834450+0000', 'last_fullsized': '2024-09-12T23:33:28.834450+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-09-12T23:33:34.076 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.32 is not active+clean 2024-09-12T23:33:34.076 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.32', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-09-12T23:33:28.834450+0000', 'last_change': '2024-09-12T23:33:28.834450+0000', 'last_active': '2024-09-12T23:33:28.834450+0000', 'last_peered': '2024-09-12T23:33:28.834450+0000', 'last_clean': '2024-09-12T23:33:28.834450+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:28.834450+0000', 'last_undegraded': '2024-09-12T23:33:28.834450+0000', 'last_fullsized': '2024-09-12T23:33:28.834450+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-09-12T23:33:34.076 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.30 is not active+clean 2024-09-12T23:33:34.076 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.30', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'creating+peering', 'last_fresh': '2024-09-12T23:33:28.834450+0000', 'last_change': '2024-09-12T23:33:28.867621+0000', 'last_active': '2024-09-12T23:33:28.834450+0000', 'last_peered': '2024-09-12T23:33:28.834450+0000', 'last_clean': '2024-09-12T23:33:28.834450+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:28.834450+0000', 'last_undegraded': '2024-09-12T23:33:28.834450+0000', 'last_fullsized': '2024-09-12T23:33:28.834450+0000', 'mapping_epoch': 52, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 52, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2024-09-14T10:53:38.724701+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [5, 0, 7], 'acting': [5, 0, 7], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [0, 7], 'up_primary': 5, 'acting_primary': 5, 'purged_snaps': []} 2024-09-12T23:33:34.076 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.2f is not active+clean 2024-09-12T23:33:34.076 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.2f', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'creating+peering', 'last_fresh': '2024-09-12T23:33:28.834450+0000', 'last_change': '2024-09-12T23:33:28.866959+0000', 'last_active': '2024-09-12T23:33:28.834450+0000', 'last_peered': '2024-09-12T23:33:28.834450+0000', 'last_clean': '2024-09-12T23:33:28.834450+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:28.834450+0000', 'last_undegraded': '2024-09-12T23:33:28.834450+0000', 'last_fullsized': '2024-09-12T23:33:28.834450+0000', 'mapping_epoch': 52, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 52, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2024-09-13T23:33:43.071267+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [5, 7, 4], 'acting': [5, 7, 4], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [4, 7], 'up_primary': 5, 'acting_primary': 5, 'purged_snaps': []} 2024-09-12T23:33:34.076 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.2e is not active+clean 2024-09-12T23:33:34.076 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.2e', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'creating+peering', 'last_fresh': '2024-09-12T23:33:28.834450+0000', 'last_change': '2024-09-12T23:33:28.849688+0000', 'last_active': '2024-09-12T23:33:28.834450+0000', 'last_peered': '2024-09-12T23:33:28.834450+0000', 'last_clean': '2024-09-12T23:33:28.834450+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:28.834450+0000', 'last_undegraded': '2024-09-12T23:33:28.834450+0000', 'last_fullsized': '2024-09-12T23:33:28.834450+0000', 'mapping_epoch': 52, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 52, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2024-09-14T01:08:59.758427+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 6, 7], 'acting': [2, 6, 7], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [6, 7], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} 2024-09-12T23:33:34.077 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.2d is not active+clean 2024-09-12T23:33:34.077 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.2d', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'creating+peering', 'last_fresh': '2024-09-12T23:33:28.834450+0000', 'last_change': '2024-09-12T23:33:28.849822+0000', 'last_active': '2024-09-12T23:33:28.834450+0000', 'last_peered': '2024-09-12T23:33:28.834450+0000', 'last_clean': '2024-09-12T23:33:28.834450+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:28.834450+0000', 'last_undegraded': '2024-09-12T23:33:28.834450+0000', 'last_fullsized': '2024-09-12T23:33:28.834450+0000', 'mapping_epoch': 52, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 52, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2024-09-14T05:08:17.677941+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [7, 2, 6], 'acting': [7, 2, 6], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [2, 6], 'up_primary': 7, 'acting_primary': 7, 'purged_snaps': []} 2024-09-12T23:33:34.077 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.2c is not active+clean 2024-09-12T23:33:34.077 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.2c', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'creating+peering', 'last_fresh': '2024-09-12T23:33:28.834450+0000', 'last_change': '2024-09-12T23:33:28.849372+0000', 'last_active': '2024-09-12T23:33:28.834450+0000', 'last_peered': '2024-09-12T23:33:28.834450+0000', 'last_clean': '2024-09-12T23:33:28.834450+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:28.834450+0000', 'last_undegraded': '2024-09-12T23:33:28.834450+0000', 'last_fullsized': '2024-09-12T23:33:28.834450+0000', 'mapping_epoch': 52, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 52, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2024-09-14T03:43:43.192749+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [6, 4, 0], 'acting': [6, 4, 0], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [0, 4], 'up_primary': 6, 'acting_primary': 6, 'purged_snaps': []} 2024-09-12T23:33:34.077 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.2b is not active+clean 2024-09-12T23:33:34.077 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.2b', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-09-12T23:33:28.834450+0000', 'last_change': '2024-09-12T23:33:28.834450+0000', 'last_active': '2024-09-12T23:33:28.834450+0000', 'last_peered': '2024-09-12T23:33:28.834450+0000', 'last_clean': '2024-09-12T23:33:28.834450+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:28.834450+0000', 'last_undegraded': '2024-09-12T23:33:28.834450+0000', 'last_fullsized': '2024-09-12T23:33:28.834450+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-09-12T23:33:34.077 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.2a is not active+clean 2024-09-12T23:33:34.077 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.2a', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'creating+peering', 'last_fresh': '2024-09-12T23:33:28.834450+0000', 'last_change': '2024-09-12T23:33:28.870650+0000', 'last_active': '2024-09-12T23:33:28.834450+0000', 'last_peered': '2024-09-12T23:33:28.834450+0000', 'last_clean': '2024-09-12T23:33:28.834450+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:28.834450+0000', 'last_undegraded': '2024-09-12T23:33:28.834450+0000', 'last_fullsized': '2024-09-12T23:33:28.834450+0000', 'mapping_epoch': 52, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 52, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2024-09-14T02:34:34.650312+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [5, 7, 6], 'acting': [5, 7, 6], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [6, 7], 'up_primary': 5, 'acting_primary': 5, 'purged_snaps': []} 2024-09-12T23:33:34.077 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.28 is not active+clean 2024-09-12T23:33:34.077 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.28', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'creating+peering', 'last_fresh': '2024-09-12T23:33:28.834450+0000', 'last_change': '2024-09-12T23:33:28.855239+0000', 'last_active': '2024-09-12T23:33:28.834450+0000', 'last_peered': '2024-09-12T23:33:28.834450+0000', 'last_clean': '2024-09-12T23:33:28.834450+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:28.834450+0000', 'last_undegraded': '2024-09-12T23:33:28.834450+0000', 'last_fullsized': '2024-09-12T23:33:28.834450+0000', 'mapping_epoch': 52, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 52, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2024-09-14T07:05:56.032960+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [5, 3, 2], 'acting': [5, 3, 2], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [2, 3], 'up_primary': 5, 'acting_primary': 5, 'purged_snaps': []} 2024-09-12T23:33:34.077 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.26 is not active+clean 2024-09-12T23:33:34.078 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.26', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'creating+peering', 'last_fresh': '2024-09-12T23:33:28.834450+0000', 'last_change': '2024-09-12T23:33:28.850972+0000', 'last_active': '2024-09-12T23:33:28.834450+0000', 'last_peered': '2024-09-12T23:33:28.834450+0000', 'last_clean': '2024-09-12T23:33:28.834450+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:28.834450+0000', 'last_undegraded': '2024-09-12T23:33:28.834450+0000', 'last_fullsized': '2024-09-12T23:33:28.834450+0000', 'mapping_epoch': 52, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 52, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2024-09-14T06:39:14.674234+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [6, 7, 4], 'acting': [6, 7, 4], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [4, 7], 'up_primary': 6, 'acting_primary': 6, 'purged_snaps': []} 2024-09-12T23:33:34.078 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.23 is not active+clean 2024-09-12T23:33:34.078 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.23', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-09-12T23:33:28.834450+0000', 'last_change': '2024-09-12T23:33:28.834450+0000', 'last_active': '2024-09-12T23:33:28.834450+0000', 'last_peered': '2024-09-12T23:33:28.834450+0000', 'last_clean': '2024-09-12T23:33:28.834450+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:28.834450+0000', 'last_undegraded': '2024-09-12T23:33:28.834450+0000', 'last_fullsized': '2024-09-12T23:33:28.834450+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-09-12T23:33:34.078 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.22 is not active+clean 2024-09-12T23:33:34.078 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.22', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'creating+peering', 'last_fresh': '2024-09-12T23:33:28.834450+0000', 'last_change': '2024-09-12T23:33:28.869503+0000', 'last_active': '2024-09-12T23:33:28.834450+0000', 'last_peered': '2024-09-12T23:33:28.834450+0000', 'last_clean': '2024-09-12T23:33:28.834450+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:28.834450+0000', 'last_undegraded': '2024-09-12T23:33:28.834450+0000', 'last_fullsized': '2024-09-12T23:33:28.834450+0000', 'mapping_epoch': 52, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 52, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2024-09-14T08:25:33.906182+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [5, 3, 0], 'acting': [5, 3, 0], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [0, 3], 'up_primary': 5, 'acting_primary': 5, 'purged_snaps': []} 2024-09-12T23:33:34.078 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.21 is not active+clean 2024-09-12T23:33:34.078 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.21', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'creating+peering', 'last_fresh': '2024-09-12T23:33:28.834450+0000', 'last_change': '2024-09-12T23:33:28.850701+0000', 'last_active': '2024-09-12T23:33:28.834450+0000', 'last_peered': '2024-09-12T23:33:28.834450+0000', 'last_clean': '2024-09-12T23:33:28.834450+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:28.834450+0000', 'last_undegraded': '2024-09-12T23:33:28.834450+0000', 'last_fullsized': '2024-09-12T23:33:28.834450+0000', 'mapping_epoch': 52, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 52, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2024-09-14T02:38:56.219241+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 5, 6], 'acting': [2, 5, 6], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [5, 6], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} 2024-09-12T23:33:34.078 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.20 is not active+clean 2024-09-12T23:33:34.078 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.20', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'creating+peering', 'last_fresh': '2024-09-12T23:33:28.834450+0000', 'last_change': '2024-09-12T23:33:28.852156+0000', 'last_active': '2024-09-12T23:33:28.834450+0000', 'last_peered': '2024-09-12T23:33:28.834450+0000', 'last_clean': '2024-09-12T23:33:28.834450+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:28.834450+0000', 'last_undegraded': '2024-09-12T23:33:28.834450+0000', 'last_fullsized': '2024-09-12T23:33:28.834450+0000', 'mapping_epoch': 52, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 52, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2024-09-14T01:09:34.637115+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [6, 0, 5], 'acting': [6, 0, 5], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [0, 5], 'up_primary': 6, 'acting_primary': 6, 'purged_snaps': []} 2024-09-12T23:33:34.078 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.1a is not active+clean 2024-09-12T23:33:34.078 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.1a', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-09-12T23:33:28.834450+0000', 'last_change': '2024-09-12T23:33:28.834450+0000', 'last_active': '2024-09-12T23:33:28.834450+0000', 'last_peered': '2024-09-12T23:33:28.834450+0000', 'last_clean': '2024-09-12T23:33:28.834450+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:28.834450+0000', 'last_undegraded': '2024-09-12T23:33:28.834450+0000', 'last_fullsized': '2024-09-12T23:33:28.834450+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-09-12T23:33:34.078 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.1b is not active+clean 2024-09-12T23:33:34.078 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.1b', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-09-12T23:33:28.834450+0000', 'last_change': '2024-09-12T23:33:28.834450+0000', 'last_active': '2024-09-12T23:33:28.834450+0000', 'last_peered': '2024-09-12T23:33:28.834450+0000', 'last_clean': '2024-09-12T23:33:28.834450+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:28.834450+0000', 'last_undegraded': '2024-09-12T23:33:28.834450+0000', 'last_fullsized': '2024-09-12T23:33:28.834450+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-09-12T23:33:34.079 INFO:tasks.cephfs.filesystem.ceph_manager:PG 2.1a is not active+clean 2024-09-12T23:33:34.079 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '2.1a', 'version': "0'0", 'reported_seq': 2, 'reported_epoch': 52, 'state': 'creating+activating', 'last_fresh': '2024-09-12T23:33:28.846682+0000', 'last_change': '2024-09-12T23:33:28.847145+0000', 'last_active': '2024-09-12T23:33:27.830370+0000', 'last_peered': '2024-09-12T23:33:27.830370+0000', 'last_clean': '2024-09-12T23:33:27.830370+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:28.846682+0000', 'last_undegraded': '2024-09-12T23:33:28.846682+0000', 'last_fullsized': '2024-09-12T23:33:28.846682+0000', 'mapping_epoch': 51, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 51, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:27.830370+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:27.830370+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:27.830370+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2024-09-14T04:57:50.175287+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [6, 4, 7], 'acting': [6, 4, 7], 'avail_no_missing': ['6', '4', '7'], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 6, 'acting_primary': 6, 'purged_snaps': []} 2024-09-12T23:33:34.079 INFO:tasks.cephfs.filesystem.ceph_manager:PG 2.17 is not active+clean 2024-09-12T23:33:34.079 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '2.17', 'version': "0'0", 'reported_seq': 3, 'reported_epoch': 52, 'state': 'creating+activating', 'last_fresh': '2024-09-12T23:33:28.848624+0000', 'last_change': '2024-09-12T23:33:28.848624+0000', 'last_active': '2024-09-12T23:33:27.830370+0000', 'last_peered': '2024-09-12T23:33:27.830370+0000', 'last_clean': '2024-09-12T23:33:27.830370+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:28.848624+0000', 'last_undegraded': '2024-09-12T23:33:28.848624+0000', 'last_fullsized': '2024-09-12T23:33:28.848624+0000', 'mapping_epoch': 51, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 51, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:27.830370+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:27.830370+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:27.830370+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2024-09-14T04:20:27.774655+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [6, 5, 2], 'acting': [6, 5, 2], 'avail_no_missing': ['6', '2', '5'], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 6, 'acting_primary': 6, 'purged_snaps': []} 2024-09-12T23:33:34.079 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.16 is not active+clean 2024-09-12T23:33:34.079 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.16', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'creating+peering', 'last_fresh': '2024-09-12T23:33:28.834450+0000', 'last_change': '2024-09-12T23:33:28.866433+0000', 'last_active': '2024-09-12T23:33:28.834450+0000', 'last_peered': '2024-09-12T23:33:28.834450+0000', 'last_clean': '2024-09-12T23:33:28.834450+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:28.834450+0000', 'last_undegraded': '2024-09-12T23:33:28.834450+0000', 'last_fullsized': '2024-09-12T23:33:28.834450+0000', 'mapping_epoch': 52, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 52, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2024-09-13T23:52:12.475067+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [5, 7, 1], 'acting': [5, 7, 1], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [1, 7], 'up_primary': 5, 'acting_primary': 5, 'purged_snaps': []} 2024-09-12T23:33:34.079 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.17 is not active+clean 2024-09-12T23:33:34.079 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.17', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-09-12T23:33:28.834450+0000', 'last_change': '2024-09-12T23:33:28.834450+0000', 'last_active': '2024-09-12T23:33:28.834450+0000', 'last_peered': '2024-09-12T23:33:28.834450+0000', 'last_clean': '2024-09-12T23:33:28.834450+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:28.834450+0000', 'last_undegraded': '2024-09-12T23:33:28.834450+0000', 'last_fullsized': '2024-09-12T23:33:28.834450+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-09-12T23:33:34.079 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.14 is not active+clean 2024-09-12T23:33:34.079 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.14', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-09-12T23:33:28.834450+0000', 'last_change': '2024-09-12T23:33:28.834450+0000', 'last_active': '2024-09-12T23:33:28.834450+0000', 'last_peered': '2024-09-12T23:33:28.834450+0000', 'last_clean': '2024-09-12T23:33:28.834450+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:28.834450+0000', 'last_undegraded': '2024-09-12T23:33:28.834450+0000', 'last_fullsized': '2024-09-12T23:33:28.834450+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-09-12T23:33:34.079 INFO:tasks.cephfs.filesystem.ceph_manager:PG 2.14 is not active+clean 2024-09-12T23:33:34.079 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '2.14', 'version': "0'0", 'reported_seq': 2, 'reported_epoch': 52, 'state': 'creating+activating', 'last_fresh': '2024-09-12T23:33:28.847672+0000', 'last_change': '2024-09-12T23:33:28.848228+0000', 'last_active': '2024-09-12T23:33:27.830370+0000', 'last_peered': '2024-09-12T23:33:27.830370+0000', 'last_clean': '2024-09-12T23:33:27.830370+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:28.847672+0000', 'last_undegraded': '2024-09-12T23:33:28.847672+0000', 'last_fullsized': '2024-09-12T23:33:28.847672+0000', 'mapping_epoch': 51, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 51, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:27.830370+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:27.830370+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:27.830370+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2024-09-14T09:50:58.477301+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [6, 3, 5], 'acting': [6, 3, 5], 'avail_no_missing': ['6', '3', '5'], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 6, 'acting_primary': 6, 'purged_snaps': []} 2024-09-12T23:33:34.080 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.15 is not active+clean 2024-09-12T23:33:34.080 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.15', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'creating+peering', 'last_fresh': '2024-09-12T23:33:28.834450+0000', 'last_change': '2024-09-12T23:33:28.858409+0000', 'last_active': '2024-09-12T23:33:28.834450+0000', 'last_peered': '2024-09-12T23:33:28.834450+0000', 'last_clean': '2024-09-12T23:33:28.834450+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:28.834450+0000', 'last_undegraded': '2024-09-12T23:33:28.834450+0000', 'last_fullsized': '2024-09-12T23:33:28.834450+0000', 'mapping_epoch': 52, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 52, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2024-09-14T01:06:00.373844+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [7, 3, 4], 'acting': [7, 3, 4], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [3, 4], 'up_primary': 7, 'acting_primary': 7, 'purged_snaps': []} 2024-09-12T23:33:34.080 INFO:tasks.cephfs.filesystem.ceph_manager:PG 2.13 is not active+clean 2024-09-12T23:33:34.080 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '2.13', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'creating+peering', 'last_fresh': '2024-09-12T23:33:27.830370+0000', 'last_change': '2024-09-12T23:33:27.837333+0000', 'last_active': '2024-09-12T23:33:27.830370+0000', 'last_peered': '2024-09-12T23:33:27.830370+0000', 'last_clean': '2024-09-12T23:33:27.830370+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:27.830370+0000', 'last_undegraded': '2024-09-12T23:33:27.830370+0000', 'last_fullsized': '2024-09-12T23:33:27.830370+0000', 'mapping_epoch': 51, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 51, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:27.830370+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:27.830370+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:27.830370+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2024-09-14T11:00:55.966645+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [0, 4, 3], 'acting': [0, 4, 3], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [3, 4], 'up_primary': 0, 'acting_primary': 0, 'purged_snaps': []} 2024-09-12T23:33:34.080 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.12 is not active+clean 2024-09-12T23:33:34.080 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.12', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-09-12T23:33:28.834450+0000', 'last_change': '2024-09-12T23:33:28.834450+0000', 'last_active': '2024-09-12T23:33:28.834450+0000', 'last_peered': '2024-09-12T23:33:28.834450+0000', 'last_clean': '2024-09-12T23:33:28.834450+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:28.834450+0000', 'last_undegraded': '2024-09-12T23:33:28.834450+0000', 'last_fullsized': '2024-09-12T23:33:28.834450+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-09-12T23:33:34.080 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.13 is not active+clean 2024-09-12T23:33:34.080 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.13', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'creating+peering', 'last_fresh': '2024-09-12T23:33:28.834450+0000', 'last_change': '2024-09-12T23:33:28.847369+0000', 'last_active': '2024-09-12T23:33:28.834450+0000', 'last_peered': '2024-09-12T23:33:28.834450+0000', 'last_clean': '2024-09-12T23:33:28.834450+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:28.834450+0000', 'last_undegraded': '2024-09-12T23:33:28.834450+0000', 'last_fullsized': '2024-09-12T23:33:28.834450+0000', 'mapping_epoch': 52, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 52, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2024-09-14T01:28:55.241684+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [7, 4, 2], 'acting': [7, 4, 2], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [2, 4], 'up_primary': 7, 'acting_primary': 7, 'purged_snaps': []} 2024-09-12T23:33:34.080 INFO:tasks.cephfs.filesystem.ceph_manager:PG 2.11 is not active+clean 2024-09-12T23:33:34.080 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '2.11', 'version': "0'0", 'reported_seq': 3, 'reported_epoch': 52, 'state': 'creating+activating', 'last_fresh': '2024-09-12T23:33:28.846921+0000', 'last_change': '2024-09-12T23:33:28.846921+0000', 'last_active': '2024-09-12T23:33:27.830370+0000', 'last_peered': '2024-09-12T23:33:27.830370+0000', 'last_clean': '2024-09-12T23:33:27.830370+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:28.846921+0000', 'last_undegraded': '2024-09-12T23:33:28.846921+0000', 'last_fullsized': '2024-09-12T23:33:28.846921+0000', 'mapping_epoch': 51, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 51, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:27.830370+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:27.830370+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:27.830370+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2024-09-14T11:27:43.449941+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [6, 4, 1], 'acting': [6, 4, 1], 'avail_no_missing': ['6', '1', '4'], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 6, 'acting_primary': 6, 'purged_snaps': []} 2024-09-12T23:33:34.080 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.10 is not active+clean 2024-09-12T23:33:34.081 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.10', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'creating+peering', 'last_fresh': '2024-09-12T23:33:28.834450+0000', 'last_change': '2024-09-12T23:33:28.847037+0000', 'last_active': '2024-09-12T23:33:28.834450+0000', 'last_peered': '2024-09-12T23:33:28.834450+0000', 'last_clean': '2024-09-12T23:33:28.834450+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:28.834450+0000', 'last_undegraded': '2024-09-12T23:33:28.834450+0000', 'last_fullsized': '2024-09-12T23:33:28.834450+0000', 'mapping_epoch': 52, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 52, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2024-09-14T05:29:56.772001+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [6, 0, 5], 'acting': [6, 0, 5], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [0, 5], 'up_primary': 6, 'acting_primary': 6, 'purged_snaps': []} 2024-09-12T23:33:34.081 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.11 is not active+clean 2024-09-12T23:33:34.081 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.11', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'creating+peering', 'last_fresh': '2024-09-12T23:33:28.834450+0000', 'last_change': '2024-09-12T23:33:28.847091+0000', 'last_active': '2024-09-12T23:33:28.834450+0000', 'last_peered': '2024-09-12T23:33:28.834450+0000', 'last_clean': '2024-09-12T23:33:28.834450+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:28.834450+0000', 'last_undegraded': '2024-09-12T23:33:28.834450+0000', 'last_fullsized': '2024-09-12T23:33:28.834450+0000', 'mapping_epoch': 52, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 52, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2024-09-14T01:57:03.461858+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [7, 4, 6], 'acting': [7, 4, 6], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [4, 6], 'up_primary': 7, 'acting_primary': 7, 'purged_snaps': []} 2024-09-12T23:33:34.081 INFO:tasks.cephfs.filesystem.ceph_manager:PG 2.f is not active+clean 2024-09-12T23:33:34.081 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '2.f', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'creating+peering', 'last_fresh': '2024-09-12T23:33:27.830370+0000', 'last_change': '2024-09-12T23:33:27.840289+0000', 'last_active': '2024-09-12T23:33:27.830370+0000', 'last_peered': '2024-09-12T23:33:27.830370+0000', 'last_clean': '2024-09-12T23:33:27.830370+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:27.830370+0000', 'last_undegraded': '2024-09-12T23:33:27.830370+0000', 'last_fullsized': '2024-09-12T23:33:27.830370+0000', 'mapping_epoch': 51, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 51, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:27.830370+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:27.830370+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:27.830370+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2024-09-14T10:54:14.860070+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [4, 0, 7], 'acting': [4, 0, 7], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [0, 7], 'up_primary': 4, 'acting_primary': 4, 'purged_snaps': []} 2024-09-12T23:33:34.081 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.e is not active+clean 2024-09-12T23:33:34.081 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.e', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'creating+peering', 'last_fresh': '2024-09-12T23:33:28.834450+0000', 'last_change': '2024-09-12T23:33:28.849355+0000', 'last_active': '2024-09-12T23:33:28.834450+0000', 'last_peered': '2024-09-12T23:33:28.834450+0000', 'last_clean': '2024-09-12T23:33:28.834450+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:28.834450+0000', 'last_undegraded': '2024-09-12T23:33:28.834450+0000', 'last_fullsized': '2024-09-12T23:33:28.834450+0000', 'mapping_epoch': 52, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 52, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2024-09-14T00:45:39.430174+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [7, 4, 1], 'acting': [7, 4, 1], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [1, 4], 'up_primary': 7, 'acting_primary': 7, 'purged_snaps': []} 2024-09-12T23:33:34.081 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.f is not active+clean 2024-09-12T23:33:34.081 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.f', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'creating+peering', 'last_fresh': '2024-09-12T23:33:28.834450+0000', 'last_change': '2024-09-12T23:33:28.859939+0000', 'last_active': '2024-09-12T23:33:28.834450+0000', 'last_peered': '2024-09-12T23:33:28.834450+0000', 'last_clean': '2024-09-12T23:33:28.834450+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:28.834450+0000', 'last_undegraded': '2024-09-12T23:33:28.834450+0000', 'last_fullsized': '2024-09-12T23:33:28.834450+0000', 'mapping_epoch': 52, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 52, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2024-09-14T01:06:23.076336+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [7, 4, 0], 'acting': [7, 4, 0], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [0, 4], 'up_primary': 7, 'acting_primary': 7, 'purged_snaps': []} 2024-09-12T23:33:34.081 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.c is not active+clean 2024-09-12T23:33:34.081 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.c', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'creating+peering', 'last_fresh': '2024-09-12T23:33:28.834450+0000', 'last_change': '2024-09-12T23:33:28.874292+0000', 'last_active': '2024-09-12T23:33:28.834450+0000', 'last_peered': '2024-09-12T23:33:28.834450+0000', 'last_clean': '2024-09-12T23:33:28.834450+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:28.834450+0000', 'last_undegraded': '2024-09-12T23:33:28.834450+0000', 'last_fullsized': '2024-09-12T23:33:28.834450+0000', 'mapping_epoch': 52, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 52, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2024-09-14T07:42:32.768139+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [5, 3, 6], 'acting': [5, 3, 6], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [3, 6], 'up_primary': 5, 'acting_primary': 5, 'purged_snaps': []} 2024-09-12T23:33:34.082 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.d is not active+clean 2024-09-12T23:33:34.082 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.d', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'creating+peering', 'last_fresh': '2024-09-12T23:33:28.834450+0000', 'last_change': '2024-09-12T23:33:28.857274+0000', 'last_active': '2024-09-12T23:33:28.834450+0000', 'last_peered': '2024-09-12T23:33:28.834450+0000', 'last_clean': '2024-09-12T23:33:28.834450+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:28.834450+0000', 'last_undegraded': '2024-09-12T23:33:28.834450+0000', 'last_fullsized': '2024-09-12T23:33:28.834450+0000', 'mapping_epoch': 52, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 52, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2024-09-14T04:24:29.251601+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [7, 5, 6], 'acting': [7, 5, 6], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [5, 6], 'up_primary': 7, 'acting_primary': 7, 'purged_snaps': []} 2024-09-12T23:33:34.082 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.1 is not active+clean 2024-09-12T23:33:34.082 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.1', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-09-12T23:33:28.834450+0000', 'last_change': '2024-09-12T23:33:28.834450+0000', 'last_active': '2024-09-12T23:33:28.834450+0000', 'last_peered': '2024-09-12T23:33:28.834450+0000', 'last_clean': '2024-09-12T23:33:28.834450+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:28.834450+0000', 'last_undegraded': '2024-09-12T23:33:28.834450+0000', 'last_fullsized': '2024-09-12T23:33:28.834450+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-09-12T23:33:34.082 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.3 is not active+clean 2024-09-12T23:33:34.082 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.3', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-09-12T23:33:28.834450+0000', 'last_change': '2024-09-12T23:33:28.834450+0000', 'last_active': '2024-09-12T23:33:28.834450+0000', 'last_peered': '2024-09-12T23:33:28.834450+0000', 'last_clean': '2024-09-12T23:33:28.834450+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:28.834450+0000', 'last_undegraded': '2024-09-12T23:33:28.834450+0000', 'last_fullsized': '2024-09-12T23:33:28.834450+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-09-12T23:33:34.082 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.5 is not active+clean 2024-09-12T23:33:34.082 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.5', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'creating+peering', 'last_fresh': '2024-09-12T23:33:28.834450+0000', 'last_change': '2024-09-12T23:33:28.865496+0000', 'last_active': '2024-09-12T23:33:28.834450+0000', 'last_peered': '2024-09-12T23:33:28.834450+0000', 'last_clean': '2024-09-12T23:33:28.834450+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:28.834450+0000', 'last_undegraded': '2024-09-12T23:33:28.834450+0000', 'last_fullsized': '2024-09-12T23:33:28.834450+0000', 'mapping_epoch': 52, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 52, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2024-09-14T06:38:00.819958+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [5, 3, 2], 'acting': [5, 3, 2], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [2, 3], 'up_primary': 5, 'acting_primary': 5, 'purged_snaps': []} 2024-09-12T23:33:34.082 INFO:tasks.cephfs.filesystem.ceph_manager:PG 2.7 is not active+clean 2024-09-12T23:33:34.082 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '2.7', 'version': "0'0", 'reported_seq': 3, 'reported_epoch': 52, 'state': 'creating+activating', 'last_fresh': '2024-09-12T23:33:28.847239+0000', 'last_change': '2024-09-12T23:33:28.847239+0000', 'last_active': '2024-09-12T23:33:27.830370+0000', 'last_peered': '2024-09-12T23:33:27.830370+0000', 'last_clean': '2024-09-12T23:33:27.830370+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:28.847239+0000', 'last_undegraded': '2024-09-12T23:33:28.847239+0000', 'last_fullsized': '2024-09-12T23:33:28.847239+0000', 'mapping_epoch': 51, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 51, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:27.830370+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:27.830370+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:27.830370+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2024-09-14T06:58:40.820895+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [6, 7, 2], 'acting': [6, 7, 2], 'avail_no_missing': ['6', '2', '7'], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 6, 'acting_primary': 6, 'purged_snaps': []} 2024-09-12T23:33:34.082 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.6 is not active+clean 2024-09-12T23:33:34.082 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.6', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-09-12T23:33:28.834450+0000', 'last_change': '2024-09-12T23:33:28.834450+0000', 'last_active': '2024-09-12T23:33:28.834450+0000', 'last_peered': '2024-09-12T23:33:28.834450+0000', 'last_clean': '2024-09-12T23:33:28.834450+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:28.834450+0000', 'last_undegraded': '2024-09-12T23:33:28.834450+0000', 'last_fullsized': '2024-09-12T23:33:28.834450+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-09-12T23:33:34.083 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.9 is not active+clean 2024-09-12T23:33:34.083 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.9', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-09-12T23:33:28.834450+0000', 'last_change': '2024-09-12T23:33:28.834450+0000', 'last_active': '2024-09-12T23:33:28.834450+0000', 'last_peered': '2024-09-12T23:33:28.834450+0000', 'last_clean': '2024-09-12T23:33:28.834450+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:28.834450+0000', 'last_undegraded': '2024-09-12T23:33:28.834450+0000', 'last_fullsized': '2024-09-12T23:33:28.834450+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-09-12T23:33:34.083 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.a is not active+clean 2024-09-12T23:33:34.083 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.a', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'creating+peering', 'last_fresh': '2024-09-12T23:33:28.834450+0000', 'last_change': '2024-09-12T23:33:28.848452+0000', 'last_active': '2024-09-12T23:33:28.834450+0000', 'last_peered': '2024-09-12T23:33:28.834450+0000', 'last_clean': '2024-09-12T23:33:28.834450+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:28.834450+0000', 'last_undegraded': '2024-09-12T23:33:28.834450+0000', 'last_fullsized': '2024-09-12T23:33:28.834450+0000', 'mapping_epoch': 52, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 52, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2024-09-14T00:12:37.049274+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [6, 4, 1], 'acting': [6, 4, 1], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [1, 4], 'up_primary': 6, 'acting_primary': 6, 'purged_snaps': []} 2024-09-12T23:33:34.083 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.1d is not active+clean 2024-09-12T23:33:34.083 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.1d', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'creating+peering', 'last_fresh': '2024-09-12T23:33:28.834450+0000', 'last_change': '2024-09-12T23:33:28.866904+0000', 'last_active': '2024-09-12T23:33:28.834450+0000', 'last_peered': '2024-09-12T23:33:28.834450+0000', 'last_clean': '2024-09-12T23:33:28.834450+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:28.834450+0000', 'last_undegraded': '2024-09-12T23:33:28.834450+0000', 'last_fullsized': '2024-09-12T23:33:28.834450+0000', 'mapping_epoch': 52, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 52, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2024-09-14T10:49:53.725680+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [5, 4, 6], 'acting': [5, 4, 6], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [4, 6], 'up_primary': 5, 'acting_primary': 5, 'purged_snaps': []} 2024-09-12T23:33:34.083 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.1c is not active+clean 2024-09-12T23:33:34.083 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.1c', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'creating+peering', 'last_fresh': '2024-09-12T23:33:28.834450+0000', 'last_change': '2024-09-12T23:33:28.875336+0000', 'last_active': '2024-09-12T23:33:28.834450+0000', 'last_peered': '2024-09-12T23:33:28.834450+0000', 'last_clean': '2024-09-12T23:33:28.834450+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:28.834450+0000', 'last_undegraded': '2024-09-12T23:33:28.834450+0000', 'last_fullsized': '2024-09-12T23:33:28.834450+0000', 'mapping_epoch': 52, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 52, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2024-09-14T10:20:24.609184+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [5, 4, 1], 'acting': [5, 4, 1], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [1, 4], 'up_primary': 5, 'acting_primary': 5, 'purged_snaps': []} 2024-09-12T23:33:34.083 INFO:tasks.cephfs.filesystem.ceph_manager:PG 3.1f is not active+clean 2024-09-12T23:33:34.083 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '3.1f', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-09-12T23:33:28.834450+0000', 'last_change': '2024-09-12T23:33:28.834450+0000', 'last_active': '2024-09-12T23:33:28.834450+0000', 'last_peered': '2024-09-12T23:33:28.834450+0000', 'last_clean': '2024-09-12T23:33:28.834450+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:28.834450+0000', 'last_undegraded': '2024-09-12T23:33:28.834450+0000', 'last_fullsized': '2024-09-12T23:33:28.834450+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:28.834450+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-09-12T23:33:34.083 INFO:tasks.cephfs.filesystem.ceph_manager:PG 2.1f is not active+clean 2024-09-12T23:33:34.084 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '2.1f', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'creating+peering', 'last_fresh': '2024-09-12T23:33:27.830370+0000', 'last_change': '2024-09-12T23:33:27.837410+0000', 'last_active': '2024-09-12T23:33:27.830370+0000', 'last_peered': '2024-09-12T23:33:27.830370+0000', 'last_clean': '2024-09-12T23:33:27.830370+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-09-12T23:33:27.830370+0000', 'last_undegraded': '2024-09-12T23:33:27.830370+0000', 'last_fullsized': '2024-09-12T23:33:27.830370+0000', 'mapping_epoch': 51, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 51, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-12T23:33:27.830370+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-12T23:33:27.830370+0000', 'last_clean_scrub_stamp': '2024-09-12T23:33:27.830370+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2024-09-14T09:42:19.518598+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [0, 7, 4], 'acting': [0, 7, 4], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [4, 7], 'up_primary': 0, 'acting_primary': 0, 'purged_snaps': []} 2024-09-12T23:33:34.084 DEBUG:teuthology.orchestra.run.smithi136:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json 2024-09-12T23:33:34.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:33 smithi181 bash[24776]: audit 2024-09-12T23:33:33.016968+0000 mon.c (mon.2) 24 : audit [DBG] from='client.? 172.21.15.136:0/1918650490' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-12T23:33:34.444 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-12T23:33:34.445 INFO:teuthology.orchestra.run.smithi136.stdout:{"fsid":"da58a47c-715d-11ef-bceb-c7b262605968","health":{"status":"HEALTH_OK","checks":{},"mutes":[]},"election_epoch":16,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":470,"monmap":{"epoch":3,"min_mon_release_name":"quincy","num_mons":3},"osdmap":{"epoch":54,"num_osds":8,"num_up_osds":8,"osd_up_since":1726183963,"num_in_osds":8,"osd_in_since":1726183938,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":42},{"state_name":"creating+peering","count":35},{"state_name":"unknown","count":15},{"state_name":"creating+activating","count":5}],"num_pgs":97,"num_pools":3,"num_objects":5,"data_bytes":590528,"bytes_used":2441740288,"bytes_avail":765518544896,"bytes_total":767960285184,"unknown_pgs_ratio":0.15463916957378387,"inactive_pgs_ratio":0.41237112879753113,"write_bytes_sec":362,"read_op_per_sec":0,"write_op_per_sec":0},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"b","status":"up:active","gid":24316}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","restful"],"services":{"dashboard":"https://172.21.15.136:8443/"}},"servicemap":{"epoch":12,"modified":"2024-09-12T23:32:55.479508+0000","services":{"osd":{"daemons":{"summary":"","7":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}} 2024-09-12T23:33:34.462 INFO:tasks.cephfs.filesystem.ceph_manager:no progress seen, keeping timeout for now 2024-09-12T23:33:34.462 DEBUG:teuthology.orchestra.run.smithi136:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-09-12T23:33:34.767 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-12T23:33:34.768 INFO:teuthology.orchestra.run.smithi136.stderr:dumped all 2024-09-12T23:33:34.783 INFO:teuthology.orchestra.run.smithi136.stdout:{"pg_ready":true,"pg_map":{"version":318,"stamp":"2024-09-12T23:33:33.489398+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590528,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":106,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":78,"ondisk_log_size":78,"up":246,"acting":246,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":59,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2384512,"kb_used_data":5652,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747576704,"statfs":{"total":767960285184,"available":765518544896,"internally_reserved":0,"allocated":5787648,"data_stored":3398613,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":160,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.654574"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.861074+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:54:27.307253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,6],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.916927+0000","last_change":"2024-09-12T23:33:29.907919+0000","last_active":"2024-09-12T23:33:29.916927+0000","last_peered":"2024-09-12T23:33:29.916927+0000","last_clean":"2024-09-12T23:33:29.916927+0000","last_became_active":"2024-09-12T23:33:29.906765+0000","last_became_peered":"2024-09-12T23:33:29.906765+0000","last_unstale":"2024-09-12T23:33:29.916927+0000","last_undegraded":"2024-09-12T23:33:29.916927+0000","last_fullsized":"2024-09-12T23:33:29.916927+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:39:49.787701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.876654+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:18:46.573142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[3,7],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.859085+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:07:03.294377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[2,4],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.873576+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:07:11.163630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,1],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.850034+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:17:57.421440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,4],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.902033+0000","last_change":"2024-09-12T23:33:29.902033+0000","last_active":"2024-09-12T23:33:29.902033+0000","last_peered":"2024-09-12T23:33:29.902033+0000","last_clean":"2024-09-12T23:33:29.902033+0000","last_became_active":"2024-09-12T23:33:29.901611+0000","last_became_peered":"2024-09-12T23:33:29.901611+0000","last_unstale":"2024-09-12T23:33:29.902033+0000","last_undegraded":"2024-09-12T23:33:29.902033+0000","last_fullsized":"2024-09-12T23:33:29.902033+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:57:58.671558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.856686+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:41:19.476191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,5],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.852231+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:56:38.506429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,5],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.855386+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:08:21.774244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,5],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.907459+0000","last_change":"2024-09-12T23:33:29.907459+0000","last_active":"2024-09-12T23:33:29.907459+0000","last_peered":"2024-09-12T23:33:29.907459+0000","last_clean":"2024-09-12T23:33:29.907459+0000","last_became_active":"2024-09-12T23:33:29.906577+0000","last_became_peered":"2024-09-12T23:33:29.906577+0000","last_unstale":"2024-09-12T23:33:29.907459+0000","last_undegraded":"2024-09-12T23:33:29.907459+0000","last_fullsized":"2024-09-12T23:33:29.907459+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:21:41.131616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.898912+0000","last_change":"2024-09-12T23:33:29.894796+0000","last_active":"2024-09-12T23:33:29.898912+0000","last_peered":"2024-09-12T23:33:29.898912+0000","last_clean":"2024-09-12T23:33:29.898912+0000","last_became_active":"2024-09-12T23:33:29.894498+0000","last_became_peered":"2024-09-12T23:33:29.894498+0000","last_unstale":"2024-09-12T23:33:29.898912+0000","last_undegraded":"2024-09-12T23:33:29.898912+0000","last_fullsized":"2024-09-12T23:33:29.898912+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:12:10.219015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.867621+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:53:38.724701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,7],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.866959+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-13T23:33:43.071267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[4,7],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.849688+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:08:59.758427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[6,7],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.849822+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:08:17.677941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[2,6],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.849372+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:43:43.192749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,4],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.870650+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:34:34.650312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[6,7],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.906087+0000","last_change":"2024-09-12T23:33:29.906087+0000","last_active":"2024-09-12T23:33:29.906087+0000","last_peered":"2024-09-12T23:33:29.906087+0000","last_clean":"2024-09-12T23:33:29.906087+0000","last_became_active":"2024-09-12T23:33:29.905780+0000","last_became_peered":"2024-09-12T23:33:29.905780+0000","last_unstale":"2024-09-12T23:33:29.906087+0000","last_undegraded":"2024-09-12T23:33:29.906087+0000","last_fullsized":"2024-09-12T23:33:29.906087+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:00:40.777941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.855239+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:05:56.032960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[2,3],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.903381+0000","last_change":"2024-09-12T23:33:29.903381+0000","last_active":"2024-09-12T23:33:29.903381+0000","last_peered":"2024-09-12T23:33:29.903381+0000","last_clean":"2024-09-12T23:33:29.903381+0000","last_became_active":"2024-09-12T23:33:29.903081+0000","last_became_peered":"2024-09-12T23:33:29.903081+0000","last_unstale":"2024-09-12T23:33:29.903381+0000","last_undegraded":"2024-09-12T23:33:29.903381+0000","last_fullsized":"2024-09-12T23:33:29.903381+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:43:01.175691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.850972+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:39:14.674234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[4,7],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.920779+0000","last_change":"2024-09-12T23:33:29.919500+0000","last_active":"2024-09-12T23:33:29.920779+0000","last_peered":"2024-09-12T23:33:29.920779+0000","last_clean":"2024-09-12T23:33:29.920779+0000","last_became_active":"2024-09-12T23:33:29.919185+0000","last_became_peered":"2024-09-12T23:33:29.919185+0000","last_unstale":"2024-09-12T23:33:29.920779+0000","last_undegraded":"2024-09-12T23:33:29.920779+0000","last_fullsized":"2024-09-12T23:33:29.920779+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:56:52.566524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.913907+0000","last_change":"2024-09-12T23:33:29.913907+0000","last_active":"2024-09-12T23:33:29.913907+0000","last_peered":"2024-09-12T23:33:29.913907+0000","last_clean":"2024-09-12T23:33:29.913907+0000","last_became_active":"2024-09-12T23:33:29.913193+0000","last_became_peered":"2024-09-12T23:33:29.913193+0000","last_unstale":"2024-09-12T23:33:29.913907+0000","last_undegraded":"2024-09-12T23:33:29.913907+0000","last_fullsized":"2024-09-12T23:33:29.913907+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:32:45.117056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.869503+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:25:33.906182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,3],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.850701+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:38:56.219241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[5,6],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.852156+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:09:34.637115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,5],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.1b","version":"53'1","reported_seq":11,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.878981+0000","last_change":"2024-09-12T23:33:28.883222+0000","last_active":"2024-09-12T23:33:29.878981+0000","last_peered":"2024-09-12T23:33:29.878981+0000","last_clean":"2024-09-12T23:33:29.878981+0000","last_became_active":"2024-09-12T23:33:28.882848+0000","last_became_peered":"2024-09-12T23:33:28.882848+0000","last_unstale":"2024-09-12T23:33:29.878981+0000","last_undegraded":"2024-09-12T23:33:29.878981+0000","last_fullsized":"2024-09-12T23:33:29.878981+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:57:33.905996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.1a","version":"0'0","reported_seq":2,"reported_epoch":52,"state":"creating+activating","last_fresh":"2024-09-12T23:33:28.846682+0000","last_change":"2024-09-12T23:33:28.847145+0000","last_active":"2024-09-12T23:33:27.830370+0000","last_peered":"2024-09-12T23:33:27.830370+0000","last_clean":"2024-09-12T23:33:27.830370+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.846682+0000","last_undegraded":"2024-09-12T23:33:28.846682+0000","last_fullsized":"2024-09-12T23:33:28.846682+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:57:50.175287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":["6","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":10,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.873141+0000","last_change":"2024-09-12T23:33:28.872822+0000","last_active":"2024-09-12T23:33:28.873141+0000","last_peered":"2024-09-12T23:33:28.873141+0000","last_clean":"2024-09-12T23:33:28.873141+0000","last_became_active":"2024-09-12T23:33:28.872404+0000","last_became_peered":"2024-09-12T23:33:28.872404+0000","last_unstale":"2024-09-12T23:33:28.873141+0000","last_undegraded":"2024-09-12T23:33:28.873141+0000","last_fullsized":"2024-09-12T23:33:28.873141+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-13T23:47:42.317408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.907360+0000","last_change":"2024-09-12T23:33:29.907360+0000","last_active":"2024-09-12T23:33:29.907360+0000","last_peered":"2024-09-12T23:33:29.907360+0000","last_clean":"2024-09-12T23:33:29.907360+0000","last_became_active":"2024-09-12T23:33:29.907067+0000","last_became_peered":"2024-09-12T23:33:29.907067+0000","last_unstale":"2024-09-12T23:33:29.907360+0000","last_undegraded":"2024-09-12T23:33:29.907360+0000","last_fullsized":"2024-09-12T23:33:29.907360+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:26:38.084938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.896704+0000","last_change":"2024-09-12T23:33:28.896704+0000","last_active":"2024-09-12T23:33:28.896704+0000","last_peered":"2024-09-12T23:33:28.896704+0000","last_clean":"2024-09-12T23:33:28.896704+0000","last_became_active":"2024-09-12T23:33:28.896167+0000","last_became_peered":"2024-09-12T23:33:28.896167+0000","last_unstale":"2024-09-12T23:33:28.896704+0000","last_undegraded":"2024-09-12T23:33:28.896704+0000","last_fullsized":"2024-09-12T23:33:28.896704+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:14:47.959501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.894866+0000","last_change":"2024-09-12T23:33:29.893401+0000","last_active":"2024-09-12T23:33:29.894866+0000","last_peered":"2024-09-12T23:33:29.894866+0000","last_clean":"2024-09-12T23:33:29.894866+0000","last_became_active":"2024-09-12T23:33:29.892988+0000","last_became_peered":"2024-09-12T23:33:29.892988+0000","last_unstale":"2024-09-12T23:33:29.894866+0000","last_undegraded":"2024-09-12T23:33:29.894866+0000","last_fullsized":"2024-09-12T23:33:29.894866+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:30:04.271132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":3,"reported_epoch":52,"state":"creating+activating","last_fresh":"2024-09-12T23:33:28.848624+0000","last_change":"2024-09-12T23:33:28.848624+0000","last_active":"2024-09-12T23:33:27.830370+0000","last_peered":"2024-09-12T23:33:27.830370+0000","last_clean":"2024-09-12T23:33:27.830370+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.848624+0000","last_undegraded":"2024-09-12T23:33:28.848624+0000","last_fullsized":"2024-09-12T23:33:28.848624+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:20:27.774655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":["6","2","5"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.866433+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-13T23:52:12.475067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,7],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.890012+0000","last_change":"2024-09-12T23:33:28.890012+0000","last_active":"2024-09-12T23:33:28.890012+0000","last_peered":"2024-09-12T23:33:28.890012+0000","last_clean":"2024-09-12T23:33:28.890012+0000","last_became_active":"2024-09-12T23:33:28.887477+0000","last_became_peered":"2024-09-12T23:33:28.887477+0000","last_unstale":"2024-09-12T23:33:28.890012+0000","last_undegraded":"2024-09-12T23:33:28.890012+0000","last_fullsized":"2024-09-12T23:33:28.890012+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:15:02.988389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.15","version":"53'3","reported_seq":14,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.917038+0000","last_change":"2024-09-12T23:33:28.879732+0000","last_active":"2024-09-12T23:33:29.917038+0000","last_peered":"2024-09-12T23:33:29.917038+0000","last_clean":"2024-09-12T23:33:29.917038+0000","last_became_active":"2024-09-12T23:33:28.879399+0000","last_became_peered":"2024-09-12T23:33:28.879399+0000","last_unstale":"2024-09-12T23:33:29.917038+0000","last_undegraded":"2024-09-12T23:33:29.917038+0000","last_fullsized":"2024-09-12T23:33:29.917038+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:00:06.109352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.14","version":"0'0","reported_seq":2,"reported_epoch":52,"state":"creating+activating","last_fresh":"2024-09-12T23:33:28.847672+0000","last_change":"2024-09-12T23:33:28.848228+0000","last_active":"2024-09-12T23:33:27.830370+0000","last_peered":"2024-09-12T23:33:27.830370+0000","last_clean":"2024-09-12T23:33:27.830370+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.847672+0000","last_undegraded":"2024-09-12T23:33:28.847672+0000","last_fullsized":"2024-09-12T23:33:28.847672+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:50:58.477301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":["6","3","5"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.858409+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:06:00.373844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[3,4],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.13","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:27.830370+0000","last_change":"2024-09-12T23:33:27.837333+0000","last_active":"2024-09-12T23:33:27.830370+0000","last_peered":"2024-09-12T23:33:27.830370+0000","last_clean":"2024-09-12T23:33:27.830370+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:27.830370+0000","last_undegraded":"2024-09-12T23:33:27.830370+0000","last_fullsized":"2024-09-12T23:33:27.830370+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:00:55.966645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[3,4],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.899424+0000","last_change":"2024-09-12T23:33:28.899424+0000","last_active":"2024-09-12T23:33:28.899424+0000","last_peered":"2024-09-12T23:33:28.899424+0000","last_clean":"2024-09-12T23:33:28.899424+0000","last_became_active":"2024-09-12T23:33:28.898942+0000","last_became_peered":"2024-09-12T23:33:28.898942+0000","last_unstale":"2024-09-12T23:33:28.899424+0000","last_undegraded":"2024-09-12T23:33:28.899424+0000","last_fullsized":"2024-09-12T23:33:28.899424+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:14:47.069896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.847369+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:28:55.241684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[2,4],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":3,"reported_epoch":52,"state":"creating+activating","last_fresh":"2024-09-12T23:33:28.846921+0000","last_change":"2024-09-12T23:33:28.846921+0000","last_active":"2024-09-12T23:33:27.830370+0000","last_peered":"2024-09-12T23:33:27.830370+0000","last_clean":"2024-09-12T23:33:27.830370+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.846921+0000","last_undegraded":"2024-09-12T23:33:28.846921+0000","last_fullsized":"2024-09-12T23:33:28.846921+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:27:43.449941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":["6","1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.847037+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:29:56.772001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,5],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.863788+0000","last_change":"2024-09-12T23:33:28.863788+0000","last_active":"2024-09-12T23:33:28.863788+0000","last_peered":"2024-09-12T23:33:28.863788+0000","last_clean":"2024-09-12T23:33:28.863788+0000","last_became_active":"2024-09-12T23:33:28.863479+0000","last_became_peered":"2024-09-12T23:33:28.863479+0000","last_unstale":"2024-09-12T23:33:28.863788+0000","last_undegraded":"2024-09-12T23:33:28.863788+0000","last_fullsized":"2024-09-12T23:33:28.863788+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:26:09.204038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.847091+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:57:03.461858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[4,6],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:27.830370+0000","last_change":"2024-09-12T23:33:27.840289+0000","last_active":"2024-09-12T23:33:27.830370+0000","last_peered":"2024-09-12T23:33:27.830370+0000","last_clean":"2024-09-12T23:33:27.830370+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:27.830370+0000","last_undegraded":"2024-09-12T23:33:27.830370+0000","last_fullsized":"2024-09-12T23:33:27.830370+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:54:14.860070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,7],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.849355+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:45:39.430174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,4],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.887441+0000","last_change":"2024-09-12T23:33:28.887441+0000","last_active":"2024-09-12T23:33:28.887441+0000","last_peered":"2024-09-12T23:33:28.887441+0000","last_clean":"2024-09-12T23:33:28.887441+0000","last_became_active":"2024-09-12T23:33:28.887109+0000","last_became_peered":"2024-09-12T23:33:28.887109+0000","last_unstale":"2024-09-12T23:33:28.887441+0000","last_undegraded":"2024-09-12T23:33:28.887441+0000","last_fullsized":"2024-09-12T23:33:28.887441+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:35:14.071342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.859939+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:06:23.076336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,4],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"53'1","reported_seq":12,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.920461+0000","last_change":"2024-09-12T23:33:28.871944+0000","last_active":"2024-09-12T23:33:29.920461+0000","last_peered":"2024-09-12T23:33:29.920461+0000","last_clean":"2024-09-12T23:33:29.920461+0000","last_became_active":"2024-09-12T23:33:28.871628+0000","last_became_peered":"2024-09-12T23:33:28.871628+0000","last_unstale":"2024-09-12T23:33:29.920461+0000","last_undegraded":"2024-09-12T23:33:29.920461+0000","last_fullsized":"2024-09-12T23:33:29.920461+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:58:29.645624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.874292+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:42:32.768139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[3,6],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.c","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.882138+0000","last_change":"2024-09-12T23:33:28.882138+0000","last_active":"2024-09-12T23:33:28.882138+0000","last_peered":"2024-09-12T23:33:28.882138+0000","last_clean":"2024-09-12T23:33:28.882138+0000","last_became_active":"2024-09-12T23:33:28.881774+0000","last_became_peered":"2024-09-12T23:33:28.881774+0000","last_unstale":"2024-09-12T23:33:28.882138+0000","last_undegraded":"2024-09-12T23:33:28.882138+0000","last_fullsized":"2024-09-12T23:33:28.882138+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:07:36.075590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.857274+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:24:29.251601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[5,6],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"21'72","reported_seq":18,"reported_epoch":51,"state":"active+clean","last_fresh":"2024-09-12T23:33:27.838959+0000","last_change":"2024-09-12T23:32:45.902460+0000","last_active":"2024-09-12T23:33:27.838959+0000","last_peered":"2024-09-12T23:33:27.838959+0000","last_clean":"2024-09-12T23:33:27.838959+0000","last_became_active":"2024-09-12T23:32:45.887205+0000","last_became_peered":"2024-09-12T23:32:45.887205+0000","last_unstale":"2024-09-12T23:33:27.838959+0000","last_undegraded":"2024-09-12T23:33:27.838959+0000","last_fullsized":"2024-09-12T23:33:27.838959+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:28:34.324093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:28:34.324093+0000","last_clean_scrub_stamp":"2024-09-12T23:28:34.324093+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:37:37.825919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.898796+0000","last_change":"2024-09-12T23:33:28.898796+0000","last_active":"2024-09-12T23:33:28.898796+0000","last_peered":"2024-09-12T23:33:28.898796+0000","last_clean":"2024-09-12T23:33:28.898796+0000","last_became_active":"2024-09-12T23:33:28.898329+0000","last_became_peered":"2024-09-12T23:33:28.898329+0000","last_unstale":"2024-09-12T23:33:28.898796+0000","last_undegraded":"2024-09-12T23:33:28.898796+0000","last_fullsized":"2024-09-12T23:33:28.898796+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:54:48.474286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.905028+0000","last_change":"2024-09-12T23:33:29.905028+0000","last_active":"2024-09-12T23:33:29.905028+0000","last_peered":"2024-09-12T23:33:29.905028+0000","last_clean":"2024-09-12T23:33:29.905028+0000","last_became_active":"2024-09-12T23:33:29.904695+0000","last_became_peered":"2024-09-12T23:33:29.904695+0000","last_unstale":"2024-09-12T23:33:29.905028+0000","last_undegraded":"2024-09-12T23:33:29.905028+0000","last_fullsized":"2024-09-12T23:33:29.905028+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:06:47.004852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.895135+0000","last_change":"2024-09-12T23:33:28.895135+0000","last_active":"2024-09-12T23:33:28.895135+0000","last_peered":"2024-09-12T23:33:28.895135+0000","last_clean":"2024-09-12T23:33:28.895135+0000","last_became_active":"2024-09-12T23:33:28.894687+0000","last_became_peered":"2024-09-12T23:33:28.894687+0000","last_unstale":"2024-09-12T23:33:28.895135+0000","last_undegraded":"2024-09-12T23:33:28.895135+0000","last_fullsized":"2024-09-12T23:33:28.895135+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:57:06.646238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.868957+0000","last_change":"2024-09-12T23:33:28.868957+0000","last_active":"2024-09-12T23:33:28.868957+0000","last_peered":"2024-09-12T23:33:28.868957+0000","last_clean":"2024-09-12T23:33:28.868957+0000","last_became_active":"2024-09-12T23:33:28.868587+0000","last_became_peered":"2024-09-12T23:33:28.868587+0000","last_unstale":"2024-09-12T23:33:28.868957+0000","last_undegraded":"2024-09-12T23:33:28.868957+0000","last_fullsized":"2024-09-12T23:33:28.868957+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:45:32.284209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.892338+0000","last_change":"2024-09-12T23:33:29.890360+0000","last_active":"2024-09-12T23:33:29.892338+0000","last_peered":"2024-09-12T23:33:29.892338+0000","last_clean":"2024-09-12T23:33:29.892338+0000","last_became_active":"2024-09-12T23:33:29.889575+0000","last_became_peered":"2024-09-12T23:33:29.889575+0000","last_unstale":"2024-09-12T23:33:29.892338+0000","last_undegraded":"2024-09-12T23:33:29.892338+0000","last_fullsized":"2024-09-12T23:33:29.892338+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:36:34.938054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.899931+0000","last_change":"2024-09-12T23:33:28.899931+0000","last_active":"2024-09-12T23:33:28.899931+0000","last_peered":"2024-09-12T23:33:28.899931+0000","last_clean":"2024-09-12T23:33:28.899931+0000","last_became_active":"2024-09-12T23:33:28.899410+0000","last_became_peered":"2024-09-12T23:33:28.899410+0000","last_unstale":"2024-09-12T23:33:28.899931+0000","last_undegraded":"2024-09-12T23:33:28.899931+0000","last_fullsized":"2024-09-12T23:33:28.899931+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:05:26.671172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.878775+0000","last_change":"2024-09-12T23:33:28.867304+0000","last_active":"2024-09-12T23:33:29.878775+0000","last_peered":"2024-09-12T23:33:29.878775+0000","last_clean":"2024-09-12T23:33:29.878775+0000","last_became_active":"2024-09-12T23:33:28.867020+0000","last_became_peered":"2024-09-12T23:33:28.867020+0000","last_unstale":"2024-09-12T23:33:29.878775+0000","last_undegraded":"2024-09-12T23:33:29.878775+0000","last_fullsized":"2024-09-12T23:33:29.878775+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:51:22.908188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.865496+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:38:00.819958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[2,3],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.895077+0000","last_change":"2024-09-12T23:33:28.895077+0000","last_active":"2024-09-12T23:33:28.895077+0000","last_peered":"2024-09-12T23:33:28.895077+0000","last_clean":"2024-09-12T23:33:28.895077+0000","last_became_active":"2024-09-12T23:33:28.892268+0000","last_became_peered":"2024-09-12T23:33:28.892268+0000","last_unstale":"2024-09-12T23:33:28.895077+0000","last_undegraded":"2024-09-12T23:33:28.895077+0000","last_fullsized":"2024-09-12T23:33:28.895077+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:26:19.125877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.913453+0000","last_change":"2024-09-12T23:33:29.899700+0000","last_active":"2024-09-12T23:33:29.913453+0000","last_peered":"2024-09-12T23:33:29.913453+0000","last_clean":"2024-09-12T23:33:29.913453+0000","last_became_active":"2024-09-12T23:33:29.899381+0000","last_became_peered":"2024-09-12T23:33:29.899381+0000","last_unstale":"2024-09-12T23:33:29.913453+0000","last_undegraded":"2024-09-12T23:33:29.913453+0000","last_fullsized":"2024-09-12T23:33:29.913453+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:30:20.590373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.875764+0000","last_change":"2024-09-12T23:33:28.866367+0000","last_active":"2024-09-12T23:33:29.875764+0000","last_peered":"2024-09-12T23:33:29.875764+0000","last_clean":"2024-09-12T23:33:29.875764+0000","last_became_active":"2024-09-12T23:33:28.866005+0000","last_became_peered":"2024-09-12T23:33:28.866005+0000","last_unstale":"2024-09-12T23:33:29.875764+0000","last_undegraded":"2024-09-12T23:33:29.875764+0000","last_fullsized":"2024-09-12T23:33:29.875764+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:49:27.803045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.915275+0000","last_change":"2024-09-12T23:33:29.915275+0000","last_active":"2024-09-12T23:33:29.915275+0000","last_peered":"2024-09-12T23:33:29.915275+0000","last_clean":"2024-09-12T23:33:29.915275+0000","last_became_active":"2024-09-12T23:33:29.914934+0000","last_became_peered":"2024-09-12T23:33:29.914934+0000","last_unstale":"2024-09-12T23:33:29.915275+0000","last_undegraded":"2024-09-12T23:33:29.915275+0000","last_fullsized":"2024-09-12T23:33:29.915275+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:43:27.562543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":3,"reported_epoch":52,"state":"creating+activating","last_fresh":"2024-09-12T23:33:28.847239+0000","last_change":"2024-09-12T23:33:28.847239+0000","last_active":"2024-09-12T23:33:27.830370+0000","last_peered":"2024-09-12T23:33:27.830370+0000","last_clean":"2024-09-12T23:33:27.830370+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.847239+0000","last_undegraded":"2024-09-12T23:33:28.847239+0000","last_fullsized":"2024-09-12T23:33:28.847239+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:58:40.820895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":["6","2","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.8","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.894286+0000","last_change":"2024-09-12T23:33:28.894286+0000","last_active":"2024-09-12T23:33:28.894286+0000","last_peered":"2024-09-12T23:33:28.894286+0000","last_clean":"2024-09-12T23:33:28.894286+0000","last_became_active":"2024-09-12T23:33:28.893861+0000","last_became_peered":"2024-09-12T23:33:28.893861+0000","last_unstale":"2024-09-12T23:33:28.894286+0000","last_undegraded":"2024-09-12T23:33:28.894286+0000","last_fullsized":"2024-09-12T23:33:28.894286+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:20:01.806723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":11,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.874494+0000","last_change":"2024-09-12T23:33:28.884360+0000","last_active":"2024-09-12T23:33:29.874494+0000","last_peered":"2024-09-12T23:33:29.874494+0000","last_clean":"2024-09-12T23:33:29.874494+0000","last_became_active":"2024-09-12T23:33:28.884027+0000","last_became_peered":"2024-09-12T23:33:28.884027+0000","last_unstale":"2024-09-12T23:33:29.874494+0000","last_undegraded":"2024-09-12T23:33:29.874494+0000","last_fullsized":"2024-09-12T23:33:29.874494+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:11:57.567074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.915243+0000","last_change":"2024-09-12T23:33:29.915243+0000","last_active":"2024-09-12T23:33:29.915243+0000","last_peered":"2024-09-12T23:33:29.915243+0000","last_clean":"2024-09-12T23:33:29.915243+0000","last_became_active":"2024-09-12T23:33:29.914543+0000","last_became_peered":"2024-09-12T23:33:29.914543+0000","last_unstale":"2024-09-12T23:33:29.915243+0000","last_undegraded":"2024-09-12T23:33:29.915243+0000","last_fullsized":"2024-09-12T23:33:29.915243+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-13T23:48:16.023985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":11,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.879749+0000","last_change":"2024-09-12T23:33:28.883601+0000","last_active":"2024-09-12T23:33:29.879749+0000","last_peered":"2024-09-12T23:33:29.879749+0000","last_clean":"2024-09-12T23:33:29.879749+0000","last_became_active":"2024-09-12T23:33:28.883276+0000","last_became_peered":"2024-09-12T23:33:28.883276+0000","last_unstale":"2024-09-12T23:33:29.879749+0000","last_undegraded":"2024-09-12T23:33:29.879749+0000","last_fullsized":"2024-09-12T23:33:29.879749+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:26:21.733615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.908185+0000","last_change":"2024-09-12T23:33:29.908185+0000","last_active":"2024-09-12T23:33:29.908185+0000","last_peered":"2024-09-12T23:33:29.908185+0000","last_clean":"2024-09-12T23:33:29.908185+0000","last_became_active":"2024-09-12T23:33:29.907814+0000","last_became_peered":"2024-09-12T23:33:29.907814+0000","last_unstale":"2024-09-12T23:33:29.908185+0000","last_undegraded":"2024-09-12T23:33:29.908185+0000","last_fullsized":"2024-09-12T23:33:29.908185+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:45:47.207242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.902988+0000","last_change":"2024-09-12T23:33:28.902988+0000","last_active":"2024-09-12T23:33:28.902988+0000","last_peered":"2024-09-12T23:33:28.902988+0000","last_clean":"2024-09-12T23:33:28.902988+0000","last_became_active":"2024-09-12T23:33:28.902514+0000","last_became_peered":"2024-09-12T23:33:28.902514+0000","last_unstale":"2024-09-12T23:33:28.902988+0000","last_undegraded":"2024-09-12T23:33:28.902988+0000","last_fullsized":"2024-09-12T23:33:28.902988+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:29:42.595057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.848452+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:12:37.049274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,4],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.866904+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:49:53.725680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[4,6],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.902224+0000","last_change":"2024-09-12T23:33:28.902224+0000","last_active":"2024-09-12T23:33:28.902224+0000","last_peered":"2024-09-12T23:33:28.902224+0000","last_clean":"2024-09-12T23:33:28.902224+0000","last_became_active":"2024-09-12T23:33:28.901755+0000","last_became_peered":"2024-09-12T23:33:28.901755+0000","last_unstale":"2024-09-12T23:33:28.902224+0000","last_undegraded":"2024-09-12T23:33:28.902224+0000","last_fullsized":"2024-09-12T23:33:28.902224+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:06:15.504175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.875336+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:20:24.609184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,4],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1d","version":"0'0","reported_seq":9,"reported_epoch":52,"state":"active+clean","last_fresh":"2024-09-12T23:33:28.897466+0000","last_change":"2024-09-12T23:33:28.897466+0000","last_active":"2024-09-12T23:33:28.897466+0000","last_peered":"2024-09-12T23:33:28.897466+0000","last_clean":"2024-09-12T23:33:28.897466+0000","last_became_active":"2024-09-12T23:33:28.896936+0000","last_became_peered":"2024-09-12T23:33:28.896936+0000","last_unstale":"2024-09-12T23:33:28.897466+0000","last_undegraded":"2024-09-12T23:33:28.897466+0000","last_fullsized":"2024-09-12T23:33:28.897466+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:29:38.732168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-12T23:33:28.834450+0000","last_change":"2024-09-12T23:33:28.834450+0000","last_active":"2024-09-12T23:33:28.834450+0000","last_peered":"2024-09-12T23:33:28.834450+0000","last_clean":"2024-09-12T23:33:28.834450+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:28.834450+0000","last_undegraded":"2024-09-12T23:33:28.834450+0000","last_fullsized":"2024-09-12T23:33:28.834450+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.1e","version":"53'1","reported_seq":11,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.875384+0000","last_change":"2024-09-12T23:33:28.894353+0000","last_active":"2024-09-12T23:33:29.875384+0000","last_peered":"2024-09-12T23:33:29.875384+0000","last_clean":"2024-09-12T23:33:29.875384+0000","last_became_active":"2024-09-12T23:33:28.893992+0000","last_became_peered":"2024-09-12T23:33:28.893992+0000","last_unstale":"2024-09-12T23:33:29.875384+0000","last_undegraded":"2024-09-12T23:33:29.875384+0000","last_fullsized":"2024-09-12T23:33:29.875384+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:43:43.889025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.891227+0000","last_change":"2024-09-12T23:33:29.891227+0000","last_active":"2024-09-12T23:33:29.891227+0000","last_peered":"2024-09-12T23:33:29.891227+0000","last_clean":"2024-09-12T23:33:29.891227+0000","last_became_active":"2024-09-12T23:33:29.890368+0000","last_became_peered":"2024-09-12T23:33:29.890368+0000","last_unstale":"2024-09-12T23:33:29.891227+0000","last_undegraded":"2024-09-12T23:33:29.891227+0000","last_fullsized":"2024-09-12T23:33:29.891227+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:38:37.526610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2024-09-12T23:33:27.830370+0000","last_change":"2024-09-12T23:33:27.837410+0000","last_active":"2024-09-12T23:33:27.830370+0000","last_peered":"2024-09-12T23:33:27.830370+0000","last_clean":"2024-09-12T23:33:27.830370+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-12T23:33:27.830370+0000","last_undegraded":"2024-09-12T23:33:27.830370+0000","last_fullsized":"2024-09-12T23:33:27.830370+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:42:19.518598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[4,7],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":147,"acting":147,"num_store_stats":6},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":160,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3563520,"data_stored":3542208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430219,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298432,"kb_used_data":1072,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446720,"statfs":{"total":95995035648,"available":95689441280,"internally_reserved":0,"allocated":1097728,"data_stored":796122,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.74299999999999999}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.69799999999999995}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47499999999999998}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.82099999999999995}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64000000000000001}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.90200000000000002}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.83899999999999997}]}]},{"osd":6,"up_from":42,"seq":180388626453,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298412,"kb_used_data":1056,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446740,"statfs":{"total":95995035648,"available":95689461760,"internally_reserved":0,"allocated":1081344,"data_stored":789114,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 12 23:32:56 2024","interfaces":[{"interface":"back","average":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"min":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":1.2450000000000001,"5min":1.2450000000000001,"15min":1.2450000000000001},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.874,"5min":0.874,"15min":0.874},"min":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"max":{"1min":1.222,"5min":1.222,"15min":1.222},"last":0.82099999999999995}]},{"osd":1,"last update":"Thu Sep 12 23:32:56 2024","interfaces":[{"interface":"back","average":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"min":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":1.153,"5min":1.153,"15min":1.153},"last":0.89800000000000002},{"interface":"front","average":{"1min":0.871,"5min":0.871,"15min":0.871},"min":{"1min":0.62,"5min":0.62,"15min":0.62},"max":{"1min":1.306,"5min":1.306,"15min":1.306},"last":1.1499999999999999}]},{"osd":2,"last update":"Thu Sep 12 23:32:56 2024","interfaces":[{"interface":"back","average":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"min":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"max":{"1min":1.4359999999999999,"5min":1.4359999999999999,"15min":1.4359999999999999},"last":0.76900000000000002},{"interface":"front","average":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"min":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"max":{"1min":1.131,"5min":1.131,"15min":1.131},"last":0.49199999999999999}]},{"osd":3,"last update":"Thu Sep 12 23:32:56 2024","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"min":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"max":{"1min":1.2330000000000001,"5min":1.2330000000000001,"15min":1.2330000000000001},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"min":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"max":{"1min":1.2290000000000001,"5min":1.2290000000000001,"15min":1.2290000000000001},"last":1.018}]},{"osd":4,"last update":"Thu Sep 12 23:32:56 2024","interfaces":[{"interface":"back","average":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"min":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":1.131,"5min":1.131,"15min":1.131},"last":0.95399999999999996},{"interface":"front","average":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":1.389,"5min":1.389,"15min":1.389},"last":0.97999999999999998}]},{"osd":5,"last update":"Thu Sep 12 23:32:56 2024","interfaces":[{"interface":"back","average":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"min":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"max":{"1min":1.228,"5min":1.228,"15min":1.228},"last":1.0780000000000001},{"interface":"front","average":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"min":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":1.3109999999999999,"5min":1.3109999999999999,"15min":1.3109999999999999},"last":1.105}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.139}]}]},{"osd":5,"up_from":36,"seq":154618822688,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297828,"kb_used_data":472,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447324,"statfs":{"total":95995035648,"available":95690059776,"internally_reserved":0,"allocated":483328,"data_stored":198431,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 12 23:33:07 2024","interfaces":[{"interface":"back","average":{"1min":0.81399999999999995,"5min":0.78700000000000003,"15min":0.78300000000000003},"min":{"1min":0.56200000000000006,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":1.0740000000000001,"5min":1.6100000000000001,"15min":1.6100000000000001},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.872,"5min":0.83399999999999996,"15min":0.82799999999999996},"min":{"1min":0.68999999999999995,"5min":0.58199999999999996,"15min":0.58199999999999996},"max":{"1min":1.0469999999999999,"5min":1.0840000000000001,"15min":1.0840000000000001},"last":0.56799999999999995}]},{"osd":1,"last update":"Thu Sep 12 23:33:07 2024","interfaces":[{"interface":"back","average":{"1min":0.88100000000000001,"5min":0.85199999999999998,"15min":0.84699999999999998},"min":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"max":{"1min":1.218,"5min":1.4470000000000001,"15min":1.4470000000000001},"last":0.81799999999999995},{"interface":"front","average":{"1min":0.89000000000000001,"5min":0.81200000000000006,"15min":0.79900000000000004},"min":{"1min":0.66300000000000003,"5min":0.502,"15min":0.502},"max":{"1min":1.274,"5min":1.274,"15min":1.274},"last":0.52800000000000002}]},{"osd":2,"last update":"Thu Sep 12 23:33:07 2024","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":0.878,"15min":0.87},"min":{"1min":0.73999999999999999,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":1.2509999999999999,"5min":1.2509999999999999,"15min":1.2509999999999999},"last":0.75},{"interface":"front","average":{"1min":0.876,"5min":0.90400000000000003,"15min":0.90900000000000003},"min":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"max":{"1min":1.0980000000000001,"5min":1.141,"15min":1.141},"last":0.77300000000000002}]},{"osd":3,"last update":"Thu Sep 12 23:33:07 2024","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":0.91300000000000003,"15min":0.91100000000000003},"min":{"1min":0.68100000000000005,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.2010000000000001,"5min":1.3280000000000001,"15min":1.3280000000000001},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.92000000000000004,"5min":0.91500000000000004,"15min":0.91400000000000003},"min":{"1min":0.67400000000000004,"5min":0.57499999999999996,"15min":0.57499999999999996},"max":{"1min":1.2609999999999999,"5min":1.536,"15min":1.536},"last":0.73399999999999999}]},{"osd":4,"last update":"Thu Sep 12 23:33:07 2024","interfaces":[{"interface":"back","average":{"1min":0.747,"5min":0.74299999999999999,"15min":0.74199999999999999},"min":{"1min":0.47199999999999998,"5min":0.308,"15min":0.308},"max":{"1min":1.1910000000000001,"5min":1.1910000000000001,"15min":1.1910000000000001},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.79500000000000004,"5min":0.77700000000000002,"15min":0.77400000000000002},"min":{"1min":0.504,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":1.1679999999999999,"5min":1.1679999999999999,"15min":1.1679999999999999},"last":0.67700000000000005}]},{"osd":6,"last update":"Thu Sep 12 23:32:54 2024","interfaces":[{"interface":"back","average":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":1.1220000000000001,"5min":1.1220000000000001,"15min":1.1220000000000001},"last":0.79700000000000004},{"interface":"front","average":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"min":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":1.0840000000000001,"5min":1.0840000000000001,"15min":1.0840000000000001},"last":0.65500000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.72099999999999997}]}]},{"osd":4,"up_from":30,"seq":128849018921,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297836,"kb_used_data":480,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447316,"statfs":{"total":95995035648,"available":95690051584,"internally_reserved":0,"allocated":491520,"data_stored":199061,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 12 23:33:17 2024","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":0.83599999999999997,"15min":0.83399999999999996},"min":{"1min":0.63200000000000001,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.375,"5min":1.375,"15min":1.375},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.89500000000000002,"5min":0.84699999999999998,"15min":0.84299999999999997},"min":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"max":{"1min":1.1479999999999999,"5min":1.1479999999999999,"15min":1.1479999999999999},"last":0.85999999999999999}]},{"osd":1,"last update":"Thu Sep 12 23:33:17 2024","interfaces":[{"interface":"back","average":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82399999999999995},"min":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":1.194,"5min":1.194,"15min":1.194},"last":1.01},{"interface":"front","average":{"1min":0.86899999999999999,"5min":0.85899999999999999,"15min":0.85699999999999998},"min":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.218,"5min":1.218,"15min":1.218},"last":0.80600000000000005}]},{"osd":2,"last update":"Thu Sep 12 23:33:17 2024","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":0.84599999999999997,"15min":0.85599999999999998},"min":{"1min":0.60199999999999998,"5min":0.55700000000000005,"15min":0.55700000000000005},"max":{"1min":1.175,"5min":1.175,"15min":1.175},"last":0.98599999999999999},{"interface":"front","average":{"1min":0.89500000000000002,"5min":0.85699999999999998,"15min":0.86299999999999999},"min":{"1min":0.66000000000000003,"5min":0.63500000000000001,"15min":0.63500000000000001},"max":{"1min":1.2909999999999999,"5min":1.2909999999999999,"15min":1.2909999999999999},"last":1.079}]},{"osd":3,"last update":"Thu Sep 12 23:33:17 2024","interfaces":[{"interface":"back","average":{"1min":0.89700000000000002,"5min":0.85199999999999998,"15min":0.84999999999999998},"min":{"1min":0.65200000000000002,"5min":0.60099999999999998,"15min":0.60099999999999998},"max":{"1min":1.5129999999999999,"5min":1.5129999999999999,"15min":1.5129999999999999},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.97899999999999998,"5min":0.90200000000000002,"15min":0.89000000000000001},"min":{"1min":0.57099999999999995,"5min":0.55900000000000005,"15min":0.55900000000000005},"max":{"1min":1.595,"5min":1.595,"15min":1.595},"last":1.214}]},{"osd":5,"last update":"Thu Sep 12 23:33:09 2024","interfaces":[{"interface":"back","average":{"1min":0.80500000000000005,"5min":0.78400000000000003,"15min":0.78000000000000003},"min":{"1min":0.49199999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":1.7430000000000001,"5min":1.7430000000000001,"15min":1.7430000000000001},"last":0.91200000000000003},{"interface":"front","average":{"1min":0.90300000000000002,"5min":0.82199999999999995,"15min":0.80800000000000005},"min":{"1min":0.375,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.716,"5min":1.716,"15min":1.716},"last":1.0489999999999999}]},{"osd":6,"last update":"Thu Sep 12 23:32:56 2024","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":0.91300000000000003,"15min":0.91300000000000003},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":1.7629999999999999,"5min":1.7629999999999999,"15min":1.7629999999999999},"last":1.1140000000000001},{"interface":"front","average":{"1min":0.96299999999999997,"5min":0.96299999999999997,"15min":0.96299999999999997},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":1.7010000000000001,"5min":1.7010000000000001,"15min":1.7010000000000001},"last":1.2849999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.1379999999999999}]}]},{"osd":3,"up_from":25,"seq":107374182451,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297880,"kb_used_data":524,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447272,"statfs":{"total":95995035648,"available":95690006528,"internally_reserved":0,"allocated":536576,"data_stored":214603,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 12 23:32:29 2024","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.62,"15min":0.60199999999999998},"min":{"1min":0.53400000000000003,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.995,"5min":0.999,"15min":0.999},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.78300000000000003,"5min":0.70499999999999996,"15min":0.69299999999999995},"min":{"1min":0.58299999999999996,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":1.3120000000000001,"5min":1.3120000000000001,"15min":1.3120000000000001},"last":0.46600000000000003}]},{"osd":1,"last update":"Thu Sep 12 23:32:29 2024","interfaces":[{"interface":"back","average":{"1min":0.755,"5min":0.67200000000000004,"15min":0.65600000000000003},"min":{"1min":0.54200000000000004,"5min":0.52200000000000002,"15min":0.52200000000000002},"max":{"1min":0.96999999999999997,"5min":0.96999999999999997,"15min":0.96999999999999997},"last":0.88600000000000001},{"interface":"front","average":{"1min":0.746,"5min":0.75,"15min":0.74199999999999999},"min":{"1min":0.57199999999999995,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":1.095,"5min":1.171,"15min":1.171},"last":0.61899999999999999}]},{"osd":2,"last update":"Thu Sep 12 23:32:29 2024","interfaces":[{"interface":"back","average":{"1min":0.81999999999999995,"5min":0.78900000000000003,"15min":0.77700000000000002},"min":{"1min":0.66100000000000003,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":0.96799999999999997,"5min":1.1140000000000001,"15min":1.1140000000000001},"last":0.78800000000000003},{"interface":"front","average":{"1min":0.85299999999999998,"5min":0.75800000000000001,"15min":0.73399999999999999},"min":{"1min":0.63800000000000001,"5min":0.54900000000000004,"15min":0.54900000000000004},"max":{"1min":1.238,"5min":1.238,"15min":1.238},"last":0.56299999999999994}]},{"osd":4,"last update":"Thu Sep 12 23:33:24 2024","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":0.90300000000000002,"15min":0.88200000000000001},"min":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.4950000000000001,"5min":1.4950000000000001,"15min":1.4950000000000001},"last":0.86099999999999999},{"interface":"front","average":{"1min":0.90300000000000002,"5min":0.87,"15min":0.84899999999999998},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":1.3640000000000001,"5min":1.3640000000000001,"15min":1.3640000000000001},"last":0.98299999999999998}]},{"osd":5,"last update":"Thu Sep 12 23:33:05 2024","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":0.98399999999999999,"15min":0.98399999999999999},"min":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"max":{"1min":1.321,"5min":1.371,"15min":1.371},"last":0.72499999999999998},{"interface":"front","average":{"1min":1.05,"5min":0.98199999999999998,"15min":0.97099999999999997},"min":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":1.423,"5min":1.423,"15min":1.423},"last":0.65600000000000003}]},{"osd":6,"last update":"Thu Sep 12 23:32:54 2024","interfaces":[{"interface":"back","average":{"1min":1.052,"5min":1.052,"15min":1.052},"min":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"max":{"1min":1.3540000000000001,"5min":1.3540000000000001,"15min":1.3540000000000001},"last":0.94499999999999995},{"interface":"front","average":{"1min":1.115,"5min":1.115,"15min":1.115},"min":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"max":{"1min":1.718,"5min":1.718,"15min":1.718},"last":0.96299999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.90800000000000003}]}]},{"osd":2,"up_from":18,"seq":77309411389,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297836,"kb_used_data":480,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447316,"statfs":{"total":95995035648,"available":95690051584,"internally_reserved":0,"allocated":491520,"data_stored":199061,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 12 23:32:41 2024","interfaces":[{"interface":"back","average":{"1min":0.72999999999999998,"5min":0.626,"15min":0.58799999999999997},"min":{"1min":0.53800000000000003,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":1.0429999999999999,"5min":1.23,"15min":1.23},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.745,"5min":0.66400000000000003,"15min":0.61299999999999999},"min":{"1min":0.50800000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":1.0940000000000001,"5min":1.0940000000000001,"15min":1.0940000000000001},"last":0.38300000000000001}]},{"osd":1,"last update":"Thu Sep 12 23:32:41 2024","interfaces":[{"interface":"back","average":{"1min":0.76400000000000001,"5min":0.67700000000000005,"15min":0.628},"min":{"1min":0.5,"5min":0.316,"15min":0.316},"max":{"1min":1.014,"5min":1.5329999999999999,"15min":1.5329999999999999},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.74199999999999999,"5min":0.68400000000000005,"15min":0.63700000000000001},"min":{"1min":0.60999999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.95099999999999996,"5min":1.0860000000000001,"15min":1.0860000000000001},"last":0.44}]},{"osd":3,"last update":"Thu Sep 12 23:32:33 2024","interfaces":[{"interface":"back","average":{"1min":0.80000000000000004,"5min":0.75800000000000001,"15min":0.74399999999999999},"min":{"1min":0.627,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":1.2270000000000001,"5min":1.2270000000000001,"15min":1.2270000000000001},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.83299999999999996,"5min":0.79800000000000004,"15min":0.78000000000000003},"min":{"1min":0.623,"5min":0.433,"15min":0.433},"max":{"1min":1.0049999999999999,"5min":1.6619999999999999,"15min":1.6619999999999999},"last":0.36599999999999999}]},{"osd":4,"last update":"Thu Sep 12 23:33:19 2024","interfaces":[{"interface":"back","average":{"1min":0.88300000000000001,"5min":0.873,"15min":0.86499999999999999},"min":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.107,"5min":1.1639999999999999,"15min":1.1639999999999999},"last":0.82499999999999996},{"interface":"front","average":{"1min":0.96899999999999997,"5min":0.95299999999999996,"15min":0.93500000000000005},"min":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"max":{"1min":1.23,"5min":1.363,"15min":1.363},"last":0.79000000000000004}]},{"osd":5,"last update":"Thu Sep 12 23:33:10 2024","interfaces":[{"interface":"back","average":{"1min":0.95599999999999996,"5min":1.022,"15min":1.0329999999999999},"min":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"max":{"1min":1.282,"5min":1.4830000000000001,"15min":1.4830000000000001},"last":0.89700000000000002},{"interface":"front","average":{"1min":0.90700000000000003,"5min":1.006,"15min":1.022},"min":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"max":{"1min":1.101,"5min":1.3129999999999999,"15min":1.3129999999999999},"last":0.63900000000000001}]},{"osd":6,"last update":"Thu Sep 12 23:32:56 2024","interfaces":[{"interface":"back","average":{"1min":0.94499999999999995,"5min":0.94499999999999995,"15min":0.94499999999999995},"min":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"max":{"1min":1.2330000000000001,"5min":1.2330000000000001,"15min":1.2330000000000001},"last":0.98099999999999998},{"interface":"front","average":{"1min":1.0880000000000001,"5min":1.0880000000000001,"15min":1.0880000000000001},"min":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"max":{"1min":1.498,"5min":1.498,"15min":1.498},"last":0.67000000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67800000000000005}]}]},{"osd":1,"up_from":13,"seq":55834574920,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297876,"kb_used_data":512,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447276,"statfs":{"total":95995035648,"available":95690010624,"internally_reserved":0,"allocated":524288,"data_stored":213107,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 12 23:32:55 2024","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":0.59899999999999998,"15min":0.53000000000000003},"min":{"1min":0.51400000000000001,"5min":0.255,"15min":0.255},"max":{"1min":1.1579999999999999,"5min":1.1579999999999999,"15min":1.1579999999999999},"last":0.69199999999999995},{"interface":"front","average":{"1min":0.77400000000000002,"5min":0.66000000000000003,"15min":0.58199999999999996},"min":{"1min":0.501,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":1.1830000000000001,"5min":1.1830000000000001,"15min":1.1830000000000001},"last":0.91300000000000003}]},{"osd":2,"last update":"Thu Sep 12 23:32:40 2024","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":0.64600000000000002,"15min":0.58799999999999997},"min":{"1min":0.54700000000000004,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.95899999999999996,"5min":1.046,"15min":1.046},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.82899999999999996,"5min":0.69999999999999996,"15min":0.629},"min":{"1min":0.53200000000000003,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":1.3049999999999999,"5min":1.3049999999999999,"15min":1.3049999999999999},"last":0.59399999999999997}]},{"osd":3,"last update":"Thu Sep 12 23:32:34 2024","interfaces":[{"interface":"back","average":{"1min":0.81999999999999995,"5min":0.79000000000000004,"15min":0.78100000000000003},"min":{"1min":0.56999999999999995,"5min":0.54800000000000004,"15min":0.54800000000000004},"max":{"1min":1.0249999999999999,"5min":1.1479999999999999,"15min":1.1479999999999999},"last":0.78100000000000003},{"interface":"front","average":{"1min":0.81299999999999994,"5min":0.77100000000000002,"15min":0.76200000000000001},"min":{"1min":0.61499999999999999,"5min":0.57799999999999996,"15min":0.57799999999999996},"max":{"1min":1.109,"5min":1.109,"15min":1.109},"last":0.84199999999999997}]},{"osd":4,"last update":"Thu Sep 12 23:33:16 2024","interfaces":[{"interface":"back","average":{"1min":0.95999999999999996,"5min":0.92100000000000004,"15min":0.91700000000000004},"min":{"1min":0.75700000000000001,"5min":0.65100000000000002,"15min":0.65100000000000002},"max":{"1min":1.23,"5min":1.585,"15min":1.585},"last":1.0620000000000001},{"interface":"front","average":{"1min":1.006,"5min":0.93700000000000006,"15min":0.92700000000000005},"min":{"1min":0.76000000000000001,"5min":0.73499999999999999,"15min":0.73499999999999999},"max":{"1min":1.389,"5min":1.4279999999999999,"15min":1.4279999999999999},"last":0.995}]},{"osd":5,"last update":"Thu Sep 12 23:33:05 2024","interfaces":[{"interface":"back","average":{"1min":0.95399999999999996,"5min":0.92100000000000004,"15min":0.91500000000000004},"min":{"1min":0.66000000000000003,"5min":0.65200000000000002,"15min":0.65200000000000002},"max":{"1min":1.484,"5min":1.6819999999999999,"15min":1.6819999999999999},"last":1.014},{"interface":"front","average":{"1min":1.0289999999999999,"5min":1.006,"15min":1.002},"min":{"1min":0.81000000000000005,"5min":0.66200000000000003,"15min":0.66200000000000003},"max":{"1min":1.5069999999999999,"5min":1.5069999999999999,"15min":1.5069999999999999},"last":0.879}]},{"osd":6,"last update":"Thu Sep 12 23:32:55 2024","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":1.0529999999999999,"15min":1.0529999999999999},"min":{"1min":0.746,"5min":0.746,"15min":0.746},"max":{"1min":1.5429999999999999,"5min":1.5429999999999999,"15min":1.5429999999999999},"last":0.93300000000000005},{"interface":"front","average":{"1min":1.101,"5min":1.101,"15min":1.101},"min":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"max":{"1min":1.5649999999999999,"5min":1.5649999999999999,"15min":1.5649999999999999},"last":0.96799999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.81200000000000006}]}]},{"osd":0,"up_from":8,"seq":34359738448,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298412,"kb_used_data":1056,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446740,"statfs":{"total":95995035648,"available":95689461760,"internally_reserved":0,"allocated":1081344,"data_stored":789114,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Sep 12 23:32:49 2024","interfaces":[{"interface":"back","average":{"1min":0.69599999999999995,"5min":0.59899999999999998,"15min":0.51100000000000001},"min":{"1min":0.48499999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":1.0509999999999999,"5min":1.0509999999999999,"15min":1.0509999999999999},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.74399999999999999,"5min":0.629,"15min":0.53300000000000003},"min":{"1min":0.55400000000000005,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.0049999999999999,"5min":1.028,"15min":1.028},"last":0.48699999999999999}]},{"osd":2,"last update":"Thu Sep 12 23:32:43 2024","interfaces":[{"interface":"back","average":{"1min":0.73299999999999998,"5min":0.66600000000000004,"15min":0.61899999999999999},"min":{"1min":0.52800000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":1.081,"5min":1.081,"15min":1.081},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.82699999999999996,"5min":0.69599999999999995,"15min":0.63900000000000001},"min":{"1min":0.56000000000000005,"5min":0.317,"15min":0.317},"max":{"1min":1.151,"5min":1.151,"15min":1.151},"last":0.81000000000000005}]},{"osd":3,"last update":"Thu Sep 12 23:32:27 2024","interfaces":[{"interface":"back","average":{"1min":0.79900000000000004,"5min":0.76800000000000002,"15min":0.76400000000000001},"min":{"1min":0.60999999999999999,"5min":0.60599999999999998,"15min":0.60599999999999998},"max":{"1min":1.177,"5min":1.177,"15min":1.177},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.83599999999999997,"5min":0.79600000000000004,"15min":0.79200000000000004},"min":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"max":{"1min":1.0009999999999999,"5min":1.1950000000000001,"15min":1.1950000000000001},"last":0.68400000000000005}]},{"osd":4,"last update":"Thu Sep 12 23:33:23 2024","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":0.93100000000000005,"15min":0.90200000000000002},"min":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"max":{"1min":1.304,"5min":1.304,"15min":1.304},"last":0.79000000000000004},{"interface":"front","average":{"1min":0.95699999999999996,"5min":0.876,"15min":0.84999999999999998},"min":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"max":{"1min":1.266,"5min":1.266,"15min":1.266},"last":0.67200000000000004}]},{"osd":5,"last update":"Thu Sep 12 23:33:10 2024","interfaces":[{"interface":"back","average":{"1min":0.94499999999999995,"5min":0.94299999999999995,"15min":0.94299999999999995},"min":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"max":{"1min":1.2330000000000001,"5min":1.2330000000000001,"15min":1.2330000000000001},"last":0.64200000000000002},{"interface":"front","average":{"1min":1.002,"5min":0.95899999999999996,"15min":0.95199999999999996},"min":{"1min":0.75,"5min":0.70099999999999996,"15min":0.70099999999999996},"max":{"1min":1.1579999999999999,"5min":1.1579999999999999,"15min":1.1579999999999999},"last":0.749}]},{"osd":6,"last update":"Thu Sep 12 23:32:55 2024","interfaces":[{"interface":"back","average":{"1min":1.1120000000000001,"5min":1.1120000000000001,"15min":1.1120000000000001},"min":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"max":{"1min":1.492,"5min":1.492,"15min":1.492},"last":0.98199999999999998},{"interface":"front","average":{"1min":1.0509999999999999,"5min":1.0509999999999999,"15min":1.0509999999999999},"min":{"1min":0.754,"5min":0.754,"15min":0.754},"max":{"1min":1.377,"5min":1.377,"15min":1.377},"last":0.90700000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.96699999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":1656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-12T23:33:35.097 INFO:journalctl@ceph.mds.b.smithi181.stdout:Sep 12 23:33:34 smithi181 bash[48042]: debug 2024-09-12T23:33:34.676+0000 7fd71b57a640 -1 mds.pinger is_rank_lagging: rank=0 was never sent ping request. 2024-09-12T23:33:35.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:34 smithi181 bash[24776]: audit 2024-09-12T23:33:33.301443+0000 mgr.y (mgr.14150) 341 : audit [DBG] from='client.24404 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-12T23:33:35.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:34 smithi181 bash[24776]: cluster 2024-09-12T23:33:33.489810+0000 mgr.y (mgr.14150) 342 : cluster [DBG] pgmap v318: 97 pgs: 5 creating+activating, 35 creating+peering, 15 unknown, 42 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 362 B/s wr, 0 op/s 2024-09-12T23:33:35.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:34 smithi181 bash[24776]: audit 2024-09-12T23:33:33.669120+0000 mgr.y (mgr.14150) 343 : audit [DBG] from='client.14490 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-12T23:33:35.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:34 smithi181 bash[24776]: audit 2024-09-12T23:33:34.448081+0000 mon.a (mon.0) 691 : audit [DBG] from='client.? 172.21.15.136:0/2447936047' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2024-09-12T23:33:35.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:34 smithi136 bash[32189]: audit 2024-09-12T23:33:33.301443+0000 mgr.y (mgr.14150) 341 : audit [DBG] from='client.24404 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-12T23:33:35.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:34 smithi136 bash[32189]: cluster 2024-09-12T23:33:33.489810+0000 mgr.y (mgr.14150) 342 : cluster [DBG] pgmap v318: 97 pgs: 5 creating+activating, 35 creating+peering, 15 unknown, 42 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 362 B/s wr, 0 op/s 2024-09-12T23:33:35.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:34 smithi136 bash[32189]: audit 2024-09-12T23:33:33.669120+0000 mgr.y (mgr.14150) 343 : audit [DBG] from='client.14490 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-12T23:33:35.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:34 smithi136 bash[32189]: audit 2024-09-12T23:33:34.448081+0000 mon.a (mon.0) 691 : audit [DBG] from='client.? 172.21.15.136:0/2447936047' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2024-09-12T23:33:35.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:34 smithi136 bash[22601]: audit 2024-09-12T23:33:33.301443+0000 mgr.y (mgr.14150) 341 : audit [DBG] from='client.24404 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-12T23:33:35.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:34 smithi136 bash[22601]: cluster 2024-09-12T23:33:33.489810+0000 mgr.y (mgr.14150) 342 : cluster [DBG] pgmap v318: 97 pgs: 5 creating+activating, 35 creating+peering, 15 unknown, 42 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 362 B/s wr, 0 op/s 2024-09-12T23:33:35.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:34 smithi136 bash[22601]: audit 2024-09-12T23:33:33.669120+0000 mgr.y (mgr.14150) 343 : audit [DBG] from='client.14490 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-12T23:33:35.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:34 smithi136 bash[22601]: audit 2024-09-12T23:33:34.448081+0000 mon.a (mon.0) 691 : audit [DBG] from='client.? 172.21.15.136:0/2447936047' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2024-09-12T23:33:36.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:35 smithi136 bash[22601]: audit 2024-09-12T23:33:34.040527+0000 mgr.y (mgr.14150) 344 : audit [DBG] from='client.24416 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-12T23:33:36.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:35 smithi136 bash[22601]: audit 2024-09-12T23:33:34.744765+0000 mgr.y (mgr.14150) 345 : audit [DBG] from='client.14508 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-12T23:33:36.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:35 smithi136 bash[32189]: audit 2024-09-12T23:33:34.040527+0000 mgr.y (mgr.14150) 344 : audit [DBG] from='client.24416 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-12T23:33:36.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:35 smithi136 bash[32189]: audit 2024-09-12T23:33:34.744765+0000 mgr.y (mgr.14150) 345 : audit [DBG] from='client.14508 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-12T23:33:36.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:35 smithi181 bash[24776]: audit 2024-09-12T23:33:34.040527+0000 mgr.y (mgr.14150) 344 : audit [DBG] from='client.24416 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-12T23:33:36.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:35 smithi181 bash[24776]: audit 2024-09-12T23:33:34.744765+0000 mgr.y (mgr.14150) 345 : audit [DBG] from='client.14508 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-12T23:33:37.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:36 smithi136 bash[22601]: cluster 2024-09-12T23:33:35.490898+0000 mgr.y (mgr.14150) 346 : cluster [DBG] pgmap v319: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.0 KiB/s wr, 6 op/s 2024-09-12T23:33:37.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:36 smithi136 bash[32189]: cluster 2024-09-12T23:33:35.490898+0000 mgr.y (mgr.14150) 346 : cluster [DBG] pgmap v319: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.0 KiB/s wr, 6 op/s 2024-09-12T23:33:37.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:36 smithi181 bash[24776]: cluster 2024-09-12T23:33:35.490898+0000 mgr.y (mgr.14150) 346 : cluster [DBG] pgmap v319: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.0 KiB/s wr, 6 op/s 2024-09-12T23:33:37.789 DEBUG:teuthology.orchestra.run.smithi136:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-09-12T23:33:38.117 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-12T23:33:38.119 INFO:teuthology.orchestra.run.smithi136.stderr:dumped all 2024-09-12T23:33:38.135 INFO:teuthology.orchestra.run.smithi136.stdout:{"pg_ready":true,"pg_map":{"version":320,"stamp":"2024-09-12T23:33:37.491197+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592706,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":146,"num_write_kb":1641,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":111,"ondisk_log_size":111,"up":291,"acting":291,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":237,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2384808,"kb_used_data":5948,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747576408,"statfs":{"total":767960285184,"available":765518241792,"internally_reserved":0,"allocated":6090752,"data_stored":3534178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.003190"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.909241+0000","last_change":"2024-09-12T23:33:29.908602+0000","last_active":"2024-09-12T23:33:29.909241+0000","last_peered":"2024-09-12T23:33:29.909241+0000","last_clean":"2024-09-12T23:33:29.909241+0000","last_became_active":"2024-09-12T23:33:29.908161+0000","last_became_peered":"2024-09-12T23:33:29.908161+0000","last_unstale":"2024-09-12T23:33:29.909241+0000","last_undegraded":"2024-09-12T23:33:29.909241+0000","last_fullsized":"2024-09-12T23:33:29.909241+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:18:25.662166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.916927+0000","last_change":"2024-09-12T23:33:29.907919+0000","last_active":"2024-09-12T23:33:29.916927+0000","last_peered":"2024-09-12T23:33:29.916927+0000","last_clean":"2024-09-12T23:33:29.916927+0000","last_became_active":"2024-09-12T23:33:29.906765+0000","last_became_peered":"2024-09-12T23:33:29.906765+0000","last_unstale":"2024-09-12T23:33:29.916927+0000","last_undegraded":"2024-09-12T23:33:29.916927+0000","last_fullsized":"2024-09-12T23:33:29.916927+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:39:49.787701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.925520+0000","last_change":"2024-09-12T23:33:29.924626+0000","last_active":"2024-09-12T23:33:29.925520+0000","last_peered":"2024-09-12T23:33:29.925520+0000","last_clean":"2024-09-12T23:33:29.925520+0000","last_became_active":"2024-09-12T23:33:29.924268+0000","last_became_peered":"2024-09-12T23:33:29.924268+0000","last_unstale":"2024-09-12T23:33:29.925520+0000","last_undegraded":"2024-09-12T23:33:29.925520+0000","last_fullsized":"2024-09-12T23:33:29.925520+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:41:31.853826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.915698+0000","last_change":"2024-09-12T23:33:29.913990+0000","last_active":"2024-09-12T23:33:29.915698+0000","last_peered":"2024-09-12T23:33:29.915698+0000","last_clean":"2024-09-12T23:33:29.915698+0000","last_became_active":"2024-09-12T23:33:29.913672+0000","last_became_peered":"2024-09-12T23:33:29.913672+0000","last_unstale":"2024-09-12T23:33:29.915698+0000","last_undegraded":"2024-09-12T23:33:29.915698+0000","last_fullsized":"2024-09-12T23:33:29.915698+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:57:54.241724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.905810+0000","last_change":"2024-09-12T23:33:29.904505+0000","last_active":"2024-09-12T23:33:29.905810+0000","last_peered":"2024-09-12T23:33:29.905810+0000","last_clean":"2024-09-12T23:33:29.905810+0000","last_became_active":"2024-09-12T23:33:29.904192+0000","last_became_peered":"2024-09-12T23:33:29.904192+0000","last_unstale":"2024-09-12T23:33:29.905810+0000","last_undegraded":"2024-09-12T23:33:29.905810+0000","last_fullsized":"2024-09-12T23:33:29.905810+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:49:27.086584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.907235+0000","last_change":"2024-09-12T23:33:29.905467+0000","last_active":"2024-09-12T23:33:29.907235+0000","last_peered":"2024-09-12T23:33:29.907235+0000","last_clean":"2024-09-12T23:33:29.907235+0000","last_became_active":"2024-09-12T23:33:29.905154+0000","last_became_peered":"2024-09-12T23:33:29.905154+0000","last_unstale":"2024-09-12T23:33:29.907235+0000","last_undegraded":"2024-09-12T23:33:29.907235+0000","last_fullsized":"2024-09-12T23:33:29.907235+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:49:00.596502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.910447+0000","last_change":"2024-09-12T23:33:29.908762+0000","last_active":"2024-09-12T23:33:29.910447+0000","last_peered":"2024-09-12T23:33:29.910447+0000","last_clean":"2024-09-12T23:33:29.910447+0000","last_became_active":"2024-09-12T23:33:29.908427+0000","last_became_peered":"2024-09-12T23:33:29.908427+0000","last_unstale":"2024-09-12T23:33:29.910447+0000","last_undegraded":"2024-09-12T23:33:29.910447+0000","last_fullsized":"2024-09-12T23:33:29.910447+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:44:26.357691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.904330+0000","last_change":"2024-09-12T23:33:29.902033+0000","last_active":"2024-09-12T23:33:29.904330+0000","last_peered":"2024-09-12T23:33:29.904330+0000","last_clean":"2024-09-12T23:33:29.904330+0000","last_became_active":"2024-09-12T23:33:29.901611+0000","last_became_peered":"2024-09-12T23:33:29.901611+0000","last_unstale":"2024-09-12T23:33:29.904330+0000","last_undegraded":"2024-09-12T23:33:29.904330+0000","last_fullsized":"2024-09-12T23:33:29.904330+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:57:58.671558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.913331+0000","last_change":"2024-09-12T23:33:29.912679+0000","last_active":"2024-09-12T23:33:29.913331+0000","last_peered":"2024-09-12T23:33:29.913331+0000","last_clean":"2024-09-12T23:33:29.913331+0000","last_became_active":"2024-09-12T23:33:29.912390+0000","last_became_peered":"2024-09-12T23:33:29.912390+0000","last_unstale":"2024-09-12T23:33:29.913331+0000","last_undegraded":"2024-09-12T23:33:29.913331+0000","last_fullsized":"2024-09-12T23:33:29.913331+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:42:00.289573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.893768+0000","last_change":"2024-09-12T23:33:29.893241+0000","last_active":"2024-09-12T23:33:29.893768+0000","last_peered":"2024-09-12T23:33:29.893768+0000","last_clean":"2024-09-12T23:33:29.893768+0000","last_became_active":"2024-09-12T23:33:29.892815+0000","last_became_peered":"2024-09-12T23:33:29.892815+0000","last_unstale":"2024-09-12T23:33:29.893768+0000","last_undegraded":"2024-09-12T23:33:29.893768+0000","last_fullsized":"2024-09-12T23:33:29.893768+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:46:21.539771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.907199+0000","last_change":"2024-09-12T23:33:29.906810+0000","last_active":"2024-09-12T23:33:29.907199+0000","last_peered":"2024-09-12T23:33:29.907199+0000","last_clean":"2024-09-12T23:33:29.907199+0000","last_became_active":"2024-09-12T23:33:29.906493+0000","last_became_peered":"2024-09-12T23:33:29.906493+0000","last_unstale":"2024-09-12T23:33:29.907199+0000","last_undegraded":"2024-09-12T23:33:29.907199+0000","last_fullsized":"2024-09-12T23:33:29.907199+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:56:07.145258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.903433+0000","last_change":"2024-09-12T23:33:29.901964+0000","last_active":"2024-09-12T23:33:29.903433+0000","last_peered":"2024-09-12T23:33:29.903433+0000","last_clean":"2024-09-12T23:33:29.903433+0000","last_became_active":"2024-09-12T23:33:29.901579+0000","last_became_peered":"2024-09-12T23:33:29.901579+0000","last_unstale":"2024-09-12T23:33:29.903433+0000","last_undegraded":"2024-09-12T23:33:29.903433+0000","last_fullsized":"2024-09-12T23:33:29.903433+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:06:56.553469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.908094+0000","last_change":"2024-09-12T23:33:29.907459+0000","last_active":"2024-09-12T23:33:29.908094+0000","last_peered":"2024-09-12T23:33:29.908094+0000","last_clean":"2024-09-12T23:33:29.908094+0000","last_became_active":"2024-09-12T23:33:29.906577+0000","last_became_peered":"2024-09-12T23:33:29.906577+0000","last_unstale":"2024-09-12T23:33:29.908094+0000","last_undegraded":"2024-09-12T23:33:29.908094+0000","last_fullsized":"2024-09-12T23:33:29.908094+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:21:41.131616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.922852+0000","last_change":"2024-09-12T23:33:29.920721+0000","last_active":"2024-09-12T23:33:29.922852+0000","last_peered":"2024-09-12T23:33:29.922852+0000","last_clean":"2024-09-12T23:33:29.922852+0000","last_became_active":"2024-09-12T23:33:29.918051+0000","last_became_peered":"2024-09-12T23:33:29.918051+0000","last_unstale":"2024-09-12T23:33:29.922852+0000","last_undegraded":"2024-09-12T23:33:29.922852+0000","last_fullsized":"2024-09-12T23:33:29.922852+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:32:27.077464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.898912+0000","last_change":"2024-09-12T23:33:29.894796+0000","last_active":"2024-09-12T23:33:29.898912+0000","last_peered":"2024-09-12T23:33:29.898912+0000","last_clean":"2024-09-12T23:33:29.898912+0000","last_became_active":"2024-09-12T23:33:29.894498+0000","last_became_peered":"2024-09-12T23:33:29.894498+0000","last_unstale":"2024-09-12T23:33:29.898912+0000","last_undegraded":"2024-09-12T23:33:29.898912+0000","last_fullsized":"2024-09-12T23:33:29.898912+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:12:10.219015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.911583+0000","last_change":"2024-09-12T23:33:29.910516+0000","last_active":"2024-09-12T23:33:29.911583+0000","last_peered":"2024-09-12T23:33:29.911583+0000","last_clean":"2024-09-12T23:33:29.911583+0000","last_became_active":"2024-09-12T23:33:29.910234+0000","last_became_peered":"2024-09-12T23:33:29.910234+0000","last_unstale":"2024-09-12T23:33:29.911583+0000","last_undegraded":"2024-09-12T23:33:29.911583+0000","last_fullsized":"2024-09-12T23:33:29.911583+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:39:36.292708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.914158+0000","last_change":"2024-09-12T23:33:29.913684+0000","last_active":"2024-09-12T23:33:29.914158+0000","last_peered":"2024-09-12T23:33:29.914158+0000","last_clean":"2024-09-12T23:33:29.914158+0000","last_became_active":"2024-09-12T23:33:29.911877+0000","last_became_peered":"2024-09-12T23:33:29.911877+0000","last_unstale":"2024-09-12T23:33:29.914158+0000","last_undegraded":"2024-09-12T23:33:29.914158+0000","last_fullsized":"2024-09-12T23:33:29.914158+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:39:10.601087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.883884+0000","last_change":"2024-09-12T23:33:29.882889+0000","last_active":"2024-09-12T23:33:29.883884+0000","last_peered":"2024-09-12T23:33:29.883884+0000","last_clean":"2024-09-12T23:33:29.883884+0000","last_became_active":"2024-09-12T23:33:29.882148+0000","last_became_peered":"2024-09-12T23:33:29.882148+0000","last_unstale":"2024-09-12T23:33:29.883884+0000","last_undegraded":"2024-09-12T23:33:29.883884+0000","last_fullsized":"2024-09-12T23:33:29.883884+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:59:38.545795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.905811+0000","last_change":"2024-09-12T23:33:29.904552+0000","last_active":"2024-09-12T23:33:29.905811+0000","last_peered":"2024-09-12T23:33:29.905811+0000","last_clean":"2024-09-12T23:33:29.905811+0000","last_became_active":"2024-09-12T23:33:29.904247+0000","last_became_peered":"2024-09-12T23:33:29.904247+0000","last_unstale":"2024-09-12T23:33:29.905811+0000","last_undegraded":"2024-09-12T23:33:29.905811+0000","last_fullsized":"2024-09-12T23:33:29.905811+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:30:06.600814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.906398+0000","last_change":"2024-09-12T23:33:29.905965+0000","last_active":"2024-09-12T23:33:29.906398+0000","last_peered":"2024-09-12T23:33:29.906398+0000","last_clean":"2024-09-12T23:33:29.906398+0000","last_became_active":"2024-09-12T23:33:29.905690+0000","last_became_peered":"2024-09-12T23:33:29.905690+0000","last_unstale":"2024-09-12T23:33:29.906398+0000","last_undegraded":"2024-09-12T23:33:29.906398+0000","last_fullsized":"2024-09-12T23:33:29.906398+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:58:29.597052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.919114+0000","last_change":"2024-09-12T23:33:29.918231+0000","last_active":"2024-09-12T23:33:29.919114+0000","last_peered":"2024-09-12T23:33:29.919114+0000","last_clean":"2024-09-12T23:33:29.919114+0000","last_became_active":"2024-09-12T23:33:29.917878+0000","last_became_peered":"2024-09-12T23:33:29.917878+0000","last_unstale":"2024-09-12T23:33:29.919114+0000","last_undegraded":"2024-09-12T23:33:29.919114+0000","last_fullsized":"2024-09-12T23:33:29.919114+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:22:46.498222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.910382+0000","last_change":"2024-09-12T23:33:29.909780+0000","last_active":"2024-09-12T23:33:29.910382+0000","last_peered":"2024-09-12T23:33:29.910382+0000","last_clean":"2024-09-12T23:33:29.910382+0000","last_became_active":"2024-09-12T23:33:29.909507+0000","last_became_peered":"2024-09-12T23:33:29.909507+0000","last_unstale":"2024-09-12T23:33:29.910382+0000","last_undegraded":"2024-09-12T23:33:29.910382+0000","last_fullsized":"2024-09-12T23:33:29.910382+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:39:50.529545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.907183+0000","last_change":"2024-09-12T23:33:29.906087+0000","last_active":"2024-09-12T23:33:29.907183+0000","last_peered":"2024-09-12T23:33:29.907183+0000","last_clean":"2024-09-12T23:33:29.907183+0000","last_became_active":"2024-09-12T23:33:29.905780+0000","last_became_peered":"2024-09-12T23:33:29.905780+0000","last_unstale":"2024-09-12T23:33:29.907183+0000","last_undegraded":"2024-09-12T23:33:29.907183+0000","last_fullsized":"2024-09-12T23:33:29.907183+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:00:40.777941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.918786+0000","last_change":"2024-09-12T23:33:29.918286+0000","last_active":"2024-09-12T23:33:29.918786+0000","last_peered":"2024-09-12T23:33:29.918786+0000","last_clean":"2024-09-12T23:33:29.918786+0000","last_became_active":"2024-09-12T23:33:29.917757+0000","last_became_peered":"2024-09-12T23:33:29.917757+0000","last_unstale":"2024-09-12T23:33:29.918786+0000","last_undegraded":"2024-09-12T23:33:29.918786+0000","last_fullsized":"2024-09-12T23:33:29.918786+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:49:26.856852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.904345+0000","last_change":"2024-09-12T23:33:29.903381+0000","last_active":"2024-09-12T23:33:29.904345+0000","last_peered":"2024-09-12T23:33:29.904345+0000","last_clean":"2024-09-12T23:33:29.904345+0000","last_became_active":"2024-09-12T23:33:29.903081+0000","last_became_peered":"2024-09-12T23:33:29.903081+0000","last_unstale":"2024-09-12T23:33:29.904345+0000","last_undegraded":"2024-09-12T23:33:29.904345+0000","last_fullsized":"2024-09-12T23:33:29.904345+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:43:01.175691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.897856+0000","last_change":"2024-09-12T23:33:29.897170+0000","last_active":"2024-09-12T23:33:29.897856+0000","last_peered":"2024-09-12T23:33:29.897856+0000","last_clean":"2024-09-12T23:33:29.897856+0000","last_became_active":"2024-09-12T23:33:29.896834+0000","last_became_peered":"2024-09-12T23:33:29.896834+0000","last_unstale":"2024-09-12T23:33:29.897856+0000","last_undegraded":"2024-09-12T23:33:29.897856+0000","last_fullsized":"2024-09-12T23:33:29.897856+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:47:43.534530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.920779+0000","last_change":"2024-09-12T23:33:29.919500+0000","last_active":"2024-09-12T23:33:29.920779+0000","last_peered":"2024-09-12T23:33:29.920779+0000","last_clean":"2024-09-12T23:33:29.920779+0000","last_became_active":"2024-09-12T23:33:29.919185+0000","last_became_peered":"2024-09-12T23:33:29.919185+0000","last_unstale":"2024-09-12T23:33:29.920779+0000","last_undegraded":"2024-09-12T23:33:29.920779+0000","last_fullsized":"2024-09-12T23:33:29.920779+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:56:52.566524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.915690+0000","last_change":"2024-09-12T23:33:29.913907+0000","last_active":"2024-09-12T23:33:29.915690+0000","last_peered":"2024-09-12T23:33:29.915690+0000","last_clean":"2024-09-12T23:33:29.915690+0000","last_became_active":"2024-09-12T23:33:29.913193+0000","last_became_peered":"2024-09-12T23:33:29.913193+0000","last_unstale":"2024-09-12T23:33:29.915690+0000","last_undegraded":"2024-09-12T23:33:29.915690+0000","last_fullsized":"2024-09-12T23:33:29.915690+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:32:45.117056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.912356+0000","last_change":"2024-09-12T23:33:29.911279+0000","last_active":"2024-09-12T23:33:29.912356+0000","last_peered":"2024-09-12T23:33:29.912356+0000","last_clean":"2024-09-12T23:33:29.912356+0000","last_became_active":"2024-09-12T23:33:29.911007+0000","last_became_peered":"2024-09-12T23:33:29.911007+0000","last_unstale":"2024-09-12T23:33:29.912356+0000","last_undegraded":"2024-09-12T23:33:29.912356+0000","last_fullsized":"2024-09-12T23:33:29.912356+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:26:51.842858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.915725+0000","last_change":"2024-09-12T23:33:29.914015+0000","last_active":"2024-09-12T23:33:29.915725+0000","last_peered":"2024-09-12T23:33:29.915725+0000","last_clean":"2024-09-12T23:33:29.915725+0000","last_became_active":"2024-09-12T23:33:29.913659+0000","last_became_peered":"2024-09-12T23:33:29.913659+0000","last_unstale":"2024-09-12T23:33:29.915725+0000","last_undegraded":"2024-09-12T23:33:29.915725+0000","last_fullsized":"2024-09-12T23:33:29.915725+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:37:02.989629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.903586+0000","last_change":"2024-09-12T23:33:29.889782+0000","last_active":"2024-09-12T23:33:29.903586+0000","last_peered":"2024-09-12T23:33:29.903586+0000","last_clean":"2024-09-12T23:33:29.903586+0000","last_became_active":"2024-09-12T23:33:29.889548+0000","last_became_peered":"2024-09-12T23:33:29.889548+0000","last_unstale":"2024-09-12T23:33:29.903586+0000","last_undegraded":"2024-09-12T23:33:29.903586+0000","last_fullsized":"2024-09-12T23:33:29.903586+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-13T23:54:53.906619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.897892+0000","last_change":"2024-09-12T23:33:29.896515+0000","last_active":"2024-09-12T23:33:29.897892+0000","last_peered":"2024-09-12T23:33:29.897892+0000","last_clean":"2024-09-12T23:33:29.897892+0000","last_became_active":"2024-09-12T23:33:29.895919+0000","last_became_peered":"2024-09-12T23:33:29.895919+0000","last_unstale":"2024-09-12T23:33:29.897892+0000","last_undegraded":"2024-09-12T23:33:29.897892+0000","last_fullsized":"2024-09-12T23:33:29.897892+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:17:53.675501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.893204+0000","last_change":"2024-09-12T23:33:29.891596+0000","last_active":"2024-09-12T23:33:29.893204+0000","last_peered":"2024-09-12T23:33:29.893204+0000","last_clean":"2024-09-12T23:33:29.893204+0000","last_became_active":"2024-09-12T23:33:29.890839+0000","last_became_peered":"2024-09-12T23:33:29.890839+0000","last_unstale":"2024-09-12T23:33:29.893204+0000","last_undegraded":"2024-09-12T23:33:29.893204+0000","last_fullsized":"2024-09-12T23:33:29.893204+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:29:49.995950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1b","version":"53'1","reported_seq":12,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.922567+0000","last_change":"2024-09-12T23:33:28.883222+0000","last_active":"2024-09-12T23:33:29.922567+0000","last_peered":"2024-09-12T23:33:29.922567+0000","last_clean":"2024-09-12T23:33:29.922567+0000","last_became_active":"2024-09-12T23:33:28.882848+0000","last_became_peered":"2024-09-12T23:33:28.882848+0000","last_unstale":"2024-09-12T23:33:29.922567+0000","last_undegraded":"2024-09-12T23:33:29.922567+0000","last_fullsized":"2024-09-12T23:33:29.922567+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:57:33.905996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.914145+0000","last_change":"2024-09-12T23:33:29.912763+0000","last_active":"2024-09-12T23:33:29.914145+0000","last_peered":"2024-09-12T23:33:29.914145+0000","last_clean":"2024-09-12T23:33:29.914145+0000","last_became_active":"2024-09-12T23:33:29.912406+0000","last_became_peered":"2024-09-12T23:33:29.912406+0000","last_unstale":"2024-09-12T23:33:29.914145+0000","last_undegraded":"2024-09-12T23:33:29.914145+0000","last_fullsized":"2024-09-12T23:33:29.914145+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:49:06.828514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1a","version":"53'1","reported_seq":12,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.901717+0000","last_change":"2024-09-12T23:33:28.884599+0000","last_active":"2024-09-12T23:33:29.901717+0000","last_peered":"2024-09-12T23:33:29.901717+0000","last_clean":"2024-09-12T23:33:29.901717+0000","last_became_active":"2024-09-12T23:33:28.884139+0000","last_became_peered":"2024-09-12T23:33:28.884139+0000","last_unstale":"2024-09-12T23:33:29.901717+0000","last_undegraded":"2024-09-12T23:33:29.901717+0000","last_fullsized":"2024-09-12T23:33:29.901717+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:57:50.175287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":11,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.876799+0000","last_change":"2024-09-12T23:33:28.872822+0000","last_active":"2024-09-12T23:33:29.876799+0000","last_peered":"2024-09-12T23:33:29.876799+0000","last_clean":"2024-09-12T23:33:29.876799+0000","last_became_active":"2024-09-12T23:33:28.872404+0000","last_became_peered":"2024-09-12T23:33:28.872404+0000","last_unstale":"2024-09-12T23:33:29.876799+0000","last_undegraded":"2024-09-12T23:33:29.876799+0000","last_fullsized":"2024-09-12T23:33:29.876799+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-13T23:47:42.317408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.908063+0000","last_change":"2024-09-12T23:33:29.907360+0000","last_active":"2024-09-12T23:33:29.908063+0000","last_peered":"2024-09-12T23:33:29.908063+0000","last_clean":"2024-09-12T23:33:29.908063+0000","last_became_active":"2024-09-12T23:33:29.907067+0000","last_became_peered":"2024-09-12T23:33:29.907067+0000","last_unstale":"2024-09-12T23:33:29.908063+0000","last_undegraded":"2024-09-12T23:33:29.908063+0000","last_fullsized":"2024-09-12T23:33:29.908063+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:26:38.084938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":11,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.883761+0000","last_change":"2024-09-12T23:33:28.896704+0000","last_active":"2024-09-12T23:33:29.883761+0000","last_peered":"2024-09-12T23:33:29.883761+0000","last_clean":"2024-09-12T23:33:29.883761+0000","last_became_active":"2024-09-12T23:33:28.896167+0000","last_became_peered":"2024-09-12T23:33:28.896167+0000","last_unstale":"2024-09-12T23:33:29.883761+0000","last_undegraded":"2024-09-12T23:33:29.883761+0000","last_fullsized":"2024-09-12T23:33:29.883761+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:14:47.959501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.894866+0000","last_change":"2024-09-12T23:33:29.893401+0000","last_active":"2024-09-12T23:33:29.894866+0000","last_peered":"2024-09-12T23:33:29.894866+0000","last_clean":"2024-09-12T23:33:29.894866+0000","last_became_active":"2024-09-12T23:33:29.892988+0000","last_became_peered":"2024-09-12T23:33:29.892988+0000","last_unstale":"2024-09-12T23:33:29.894866+0000","last_undegraded":"2024-09-12T23:33:29.894866+0000","last_fullsized":"2024-09-12T23:33:29.894866+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:30:04.271132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":11,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.863299+0000","last_change":"2024-09-12T23:33:28.884106+0000","last_active":"2024-09-12T23:33:29.863299+0000","last_peered":"2024-09-12T23:33:29.863299+0000","last_clean":"2024-09-12T23:33:29.863299+0000","last_became_active":"2024-09-12T23:33:28.883028+0000","last_became_peered":"2024-09-12T23:33:28.883028+0000","last_unstale":"2024-09-12T23:33:29.863299+0000","last_undegraded":"2024-09-12T23:33:29.863299+0000","last_fullsized":"2024-09-12T23:33:29.863299+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:20:27.774655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.910043+0000","last_change":"2024-09-12T23:33:29.909613+0000","last_active":"2024-09-12T23:33:29.910043+0000","last_peered":"2024-09-12T23:33:29.910043+0000","last_clean":"2024-09-12T23:33:29.910043+0000","last_became_active":"2024-09-12T23:33:29.909325+0000","last_became_peered":"2024-09-12T23:33:29.909325+0000","last_unstale":"2024-09-12T23:33:29.910043+0000","last_undegraded":"2024-09-12T23:33:29.910043+0000","last_fullsized":"2024-09-12T23:33:29.910043+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:50:21.069936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":11,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.889594+0000","last_change":"2024-09-12T23:33:28.890012+0000","last_active":"2024-09-12T23:33:29.889594+0000","last_peered":"2024-09-12T23:33:29.889594+0000","last_clean":"2024-09-12T23:33:29.889594+0000","last_became_active":"2024-09-12T23:33:28.887477+0000","last_became_peered":"2024-09-12T23:33:28.887477+0000","last_unstale":"2024-09-12T23:33:29.889594+0000","last_undegraded":"2024-09-12T23:33:29.889594+0000","last_fullsized":"2024-09-12T23:33:29.889594+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:15:02.988389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.918032+0000","last_change":"2024-09-12T23:33:29.917549+0000","last_active":"2024-09-12T23:33:29.918032+0000","last_peered":"2024-09-12T23:33:29.918032+0000","last_clean":"2024-09-12T23:33:29.918032+0000","last_became_active":"2024-09-12T23:33:29.916648+0000","last_became_peered":"2024-09-12T23:33:29.916648+0000","last_unstale":"2024-09-12T23:33:29.918032+0000","last_undegraded":"2024-09-12T23:33:29.918032+0000","last_fullsized":"2024-09-12T23:33:29.918032+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:37:55.708866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"53'3","reported_seq":14,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.917038+0000","last_change":"2024-09-12T23:33:28.879732+0000","last_active":"2024-09-12T23:33:29.917038+0000","last_peered":"2024-09-12T23:33:29.917038+0000","last_clean":"2024-09-12T23:33:29.917038+0000","last_became_active":"2024-09-12T23:33:28.879399+0000","last_became_peered":"2024-09-12T23:33:28.879399+0000","last_unstale":"2024-09-12T23:33:29.917038+0000","last_undegraded":"2024-09-12T23:33:29.917038+0000","last_fullsized":"2024-09-12T23:33:29.917038+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:00:06.109352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.896416+0000","last_change":"2024-09-12T23:33:29.896011+0000","last_active":"2024-09-12T23:33:29.896416+0000","last_peered":"2024-09-12T23:33:29.896416+0000","last_clean":"2024-09-12T23:33:29.896416+0000","last_became_active":"2024-09-12T23:33:29.895657+0000","last_became_peered":"2024-09-12T23:33:29.895657+0000","last_unstale":"2024-09-12T23:33:29.896416+0000","last_undegraded":"2024-09-12T23:33:29.896416+0000","last_fullsized":"2024-09-12T23:33:29.896416+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:24:01.074782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.14","version":"53'6","reported_seq":17,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.924341+0000","last_change":"2024-09-12T23:33:28.886930+0000","last_active":"2024-09-12T23:33:29.924341+0000","last_peered":"2024-09-12T23:33:29.924341+0000","last_clean":"2024-09-12T23:33:29.924341+0000","last_became_active":"2024-09-12T23:33:28.885991+0000","last_became_peered":"2024-09-12T23:33:28.885991+0000","last_unstale":"2024-09-12T23:33:29.924341+0000","last_undegraded":"2024-09-12T23:33:29.924341+0000","last_fullsized":"2024-09-12T23:33:29.924341+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:50:58.477301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.917401+0000","last_change":"2024-09-12T23:33:29.916929+0000","last_active":"2024-09-12T23:33:29.917401+0000","last_peered":"2024-09-12T23:33:29.917401+0000","last_clean":"2024-09-12T23:33:29.917401+0000","last_became_active":"2024-09-12T23:33:29.916622+0000","last_became_peered":"2024-09-12T23:33:29.916622+0000","last_unstale":"2024-09-12T23:33:29.917401+0000","last_undegraded":"2024-09-12T23:33:29.917401+0000","last_fullsized":"2024-09-12T23:33:29.917401+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:40:09.433278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.13","version":"53'2","reported_seq":13,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.923503+0000","last_change":"2024-09-12T23:33:28.880385+0000","last_active":"2024-09-12T23:33:29.923503+0000","last_peered":"2024-09-12T23:33:29.923503+0000","last_clean":"2024-09-12T23:33:29.923503+0000","last_became_active":"2024-09-12T23:33:28.879825+0000","last_became_peered":"2024-09-12T23:33:28.879825+0000","last_unstale":"2024-09-12T23:33:29.923503+0000","last_undegraded":"2024-09-12T23:33:29.923503+0000","last_fullsized":"2024-09-12T23:33:29.923503+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:40:33.073593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.919982+0000","last_change":"2024-09-12T23:33:29.919045+0000","last_active":"2024-09-12T23:33:29.919982+0000","last_peered":"2024-09-12T23:33:29.919982+0000","last_clean":"2024-09-12T23:33:29.919982+0000","last_became_active":"2024-09-12T23:33:29.918580+0000","last_became_peered":"2024-09-12T23:33:29.918580+0000","last_unstale":"2024-09-12T23:33:29.919982+0000","last_undegraded":"2024-09-12T23:33:29.919982+0000","last_fullsized":"2024-09-12T23:33:29.919982+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:39:32.320667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":11,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.884749+0000","last_change":"2024-09-12T23:33:28.899424+0000","last_active":"2024-09-12T23:33:29.884749+0000","last_peered":"2024-09-12T23:33:29.884749+0000","last_clean":"2024-09-12T23:33:29.884749+0000","last_became_active":"2024-09-12T23:33:28.898942+0000","last_became_peered":"2024-09-12T23:33:28.898942+0000","last_unstale":"2024-09-12T23:33:29.884749+0000","last_undegraded":"2024-09-12T23:33:29.884749+0000","last_fullsized":"2024-09-12T23:33:29.884749+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:14:47.069896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.902807+0000","last_change":"2024-09-12T23:33:29.901397+0000","last_active":"2024-09-12T23:33:29.902807+0000","last_peered":"2024-09-12T23:33:29.902807+0000","last_clean":"2024-09-12T23:33:29.902807+0000","last_became_active":"2024-09-12T23:33:29.901060+0000","last_became_peered":"2024-09-12T23:33:29.901060+0000","last_unstale":"2024-09-12T23:33:29.902807+0000","last_undegraded":"2024-09-12T23:33:29.902807+0000","last_fullsized":"2024-09-12T23:33:29.902807+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:21:23.285498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":11,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.862637+0000","last_change":"2024-09-12T23:33:28.871430+0000","last_active":"2024-09-12T23:33:29.862637+0000","last_peered":"2024-09-12T23:33:29.862637+0000","last_clean":"2024-09-12T23:33:29.862637+0000","last_became_active":"2024-09-12T23:33:28.869528+0000","last_became_peered":"2024-09-12T23:33:28.869528+0000","last_unstale":"2024-09-12T23:33:29.862637+0000","last_undegraded":"2024-09-12T23:33:29.862637+0000","last_fullsized":"2024-09-12T23:33:29.862637+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:27:43.449941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.897453+0000","last_change":"2024-09-12T23:33:29.896291+0000","last_active":"2024-09-12T23:33:29.897453+0000","last_peered":"2024-09-12T23:33:29.897453+0000","last_clean":"2024-09-12T23:33:29.897453+0000","last_became_active":"2024-09-12T23:33:29.895397+0000","last_became_peered":"2024-09-12T23:33:29.895397+0000","last_unstale":"2024-09-12T23:33:29.897453+0000","last_undegraded":"2024-09-12T23:33:29.897453+0000","last_fullsized":"2024-09-12T23:33:29.897453+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:20:05.947678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"53'8","reported_seq":19,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.924285+0000","last_change":"2024-09-12T23:33:28.863788+0000","last_active":"2024-09-12T23:33:29.924285+0000","last_peered":"2024-09-12T23:33:29.924285+0000","last_clean":"2024-09-12T23:33:29.924285+0000","last_became_active":"2024-09-12T23:33:28.863479+0000","last_became_peered":"2024-09-12T23:33:28.863479+0000","last_unstale":"2024-09-12T23:33:29.924285+0000","last_undegraded":"2024-09-12T23:33:29.924285+0000","last_fullsized":"2024-09-12T23:33:29.924285+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:26:09.204038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.903132+0000","last_change":"2024-09-12T23:33:29.902101+0000","last_active":"2024-09-12T23:33:29.903132+0000","last_peered":"2024-09-12T23:33:29.903132+0000","last_clean":"2024-09-12T23:33:29.903132+0000","last_became_active":"2024-09-12T23:33:29.901803+0000","last_became_peered":"2024-09-12T23:33:29.901803+0000","last_unstale":"2024-09-12T23:33:29.903132+0000","last_undegraded":"2024-09-12T23:33:29.903132+0000","last_fullsized":"2024-09-12T23:33:29.903132+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:00:48.557318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.f","version":"53'1","reported_seq":12,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.909813+0000","last_change":"2024-09-12T23:33:28.879318+0000","last_active":"2024-09-12T23:33:29.909813+0000","last_peered":"2024-09-12T23:33:29.909813+0000","last_clean":"2024-09-12T23:33:29.909813+0000","last_became_active":"2024-09-12T23:33:28.877913+0000","last_became_peered":"2024-09-12T23:33:28.877913+0000","last_unstale":"2024-09-12T23:33:29.909813+0000","last_undegraded":"2024-09-12T23:33:29.909813+0000","last_fullsized":"2024-09-12T23:33:29.909813+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:13:19.574352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.901931+0000","last_change":"2024-09-12T23:33:29.901391+0000","last_active":"2024-09-12T23:33:29.901931+0000","last_peered":"2024-09-12T23:33:29.901931+0000","last_clean":"2024-09-12T23:33:29.901931+0000","last_became_active":"2024-09-12T23:33:29.901052+0000","last_became_peered":"2024-09-12T23:33:29.901052+0000","last_unstale":"2024-09-12T23:33:29.901931+0000","last_undegraded":"2024-09-12T23:33:29.901931+0000","last_fullsized":"2024-09-12T23:33:29.901931+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:54:04.075006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"53'2","reported_seq":13,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.909722+0000","last_change":"2024-09-12T23:33:28.887441+0000","last_active":"2024-09-12T23:33:29.909722+0000","last_peered":"2024-09-12T23:33:29.909722+0000","last_clean":"2024-09-12T23:33:29.909722+0000","last_became_active":"2024-09-12T23:33:28.887109+0000","last_became_peered":"2024-09-12T23:33:28.887109+0000","last_unstale":"2024-09-12T23:33:29.909722+0000","last_undegraded":"2024-09-12T23:33:29.909722+0000","last_fullsized":"2024-09-12T23:33:29.909722+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:35:14.071342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.908424+0000","last_change":"2024-09-12T23:33:29.907594+0000","last_active":"2024-09-12T23:33:29.908424+0000","last_peered":"2024-09-12T23:33:29.908424+0000","last_clean":"2024-09-12T23:33:29.908424+0000","last_became_active":"2024-09-12T23:33:29.907302+0000","last_became_peered":"2024-09-12T23:33:29.907302+0000","last_unstale":"2024-09-12T23:33:29.908424+0000","last_undegraded":"2024-09-12T23:33:29.908424+0000","last_fullsized":"2024-09-12T23:33:29.908424+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:45:37.988381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"53'1","reported_seq":12,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.920461+0000","last_change":"2024-09-12T23:33:28.871944+0000","last_active":"2024-09-12T23:33:29.920461+0000","last_peered":"2024-09-12T23:33:29.920461+0000","last_clean":"2024-09-12T23:33:29.920461+0000","last_became_active":"2024-09-12T23:33:28.871628+0000","last_became_peered":"2024-09-12T23:33:28.871628+0000","last_unstale":"2024-09-12T23:33:29.920461+0000","last_undegraded":"2024-09-12T23:33:29.920461+0000","last_fullsized":"2024-09-12T23:33:29.920461+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:58:29.645624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.915974+0000","last_change":"2024-09-12T23:33:29.914833+0000","last_active":"2024-09-12T23:33:29.915974+0000","last_peered":"2024-09-12T23:33:29.915974+0000","last_clean":"2024-09-12T23:33:29.915974+0000","last_became_active":"2024-09-12T23:33:29.914416+0000","last_became_peered":"2024-09-12T23:33:29.914416+0000","last_unstale":"2024-09-12T23:33:29.915974+0000","last_undegraded":"2024-09-12T23:33:29.915974+0000","last_fullsized":"2024-09-12T23:33:29.915974+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:27:20.253954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.c","version":"0'0","reported_seq":11,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.865877+0000","last_change":"2024-09-12T23:33:28.882138+0000","last_active":"2024-09-12T23:33:29.865877+0000","last_peered":"2024-09-12T23:33:29.865877+0000","last_clean":"2024-09-12T23:33:29.865877+0000","last_became_active":"2024-09-12T23:33:28.881774+0000","last_became_peered":"2024-09-12T23:33:28.881774+0000","last_unstale":"2024-09-12T23:33:29.865877+0000","last_undegraded":"2024-09-12T23:33:29.865877+0000","last_fullsized":"2024-09-12T23:33:29.865877+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:07:36.075590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.912612+0000","last_change":"2024-09-12T23:33:29.911549+0000","last_active":"2024-09-12T23:33:29.912612+0000","last_peered":"2024-09-12T23:33:29.912612+0000","last_clean":"2024-09-12T23:33:29.912612+0000","last_became_active":"2024-09-12T23:33:29.911244+0000","last_became_peered":"2024-09-12T23:33:29.911244+0000","last_unstale":"2024-09-12T23:33:29.912612+0000","last_undegraded":"2024-09-12T23:33:29.912612+0000","last_fullsized":"2024-09-12T23:33:29.912612+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:22:20.405567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"21'72","reported_seq":20,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.874200+0000","last_change":"2024-09-12T23:32:45.902460+0000","last_active":"2024-09-12T23:33:29.874200+0000","last_peered":"2024-09-12T23:33:29.874200+0000","last_clean":"2024-09-12T23:33:29.874200+0000","last_became_active":"2024-09-12T23:32:45.887205+0000","last_became_peered":"2024-09-12T23:32:45.887205+0000","last_unstale":"2024-09-12T23:33:29.874200+0000","last_undegraded":"2024-09-12T23:33:29.874200+0000","last_fullsized":"2024-09-12T23:33:29.874200+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:28:34.324093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:28:34.324093+0000","last_clean_scrub_stamp":"2024-09-12T23:28:34.324093+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:37:37.825919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.871461+0000","last_change":"2024-09-12T23:33:28.898796+0000","last_active":"2024-09-12T23:33:29.871461+0000","last_peered":"2024-09-12T23:33:29.871461+0000","last_clean":"2024-09-12T23:33:29.871461+0000","last_became_active":"2024-09-12T23:33:28.898329+0000","last_became_peered":"2024-09-12T23:33:28.898329+0000","last_unstale":"2024-09-12T23:33:29.871461+0000","last_undegraded":"2024-09-12T23:33:29.871461+0000","last_fullsized":"2024-09-12T23:33:29.871461+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:54:48.474286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.905325+0000","last_change":"2024-09-12T23:33:29.905028+0000","last_active":"2024-09-12T23:33:29.905325+0000","last_peered":"2024-09-12T23:33:29.905325+0000","last_clean":"2024-09-12T23:33:29.905325+0000","last_became_active":"2024-09-12T23:33:29.904695+0000","last_became_peered":"2024-09-12T23:33:29.904695+0000","last_unstale":"2024-09-12T23:33:29.905325+0000","last_undegraded":"2024-09-12T23:33:29.905325+0000","last_fullsized":"2024-09-12T23:33:29.905325+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:06:47.004852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.873987+0000","last_change":"2024-09-12T23:33:28.895135+0000","last_active":"2024-09-12T23:33:29.873987+0000","last_peered":"2024-09-12T23:33:29.873987+0000","last_clean":"2024-09-12T23:33:29.873987+0000","last_became_active":"2024-09-12T23:33:28.894687+0000","last_became_peered":"2024-09-12T23:33:28.894687+0000","last_unstale":"2024-09-12T23:33:29.873987+0000","last_undegraded":"2024-09-12T23:33:29.873987+0000","last_fullsized":"2024-09-12T23:33:29.873987+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:57:06.646238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.910373+0000","last_change":"2024-09-12T23:33:29.910025+0000","last_active":"2024-09-12T23:33:29.910373+0000","last_peered":"2024-09-12T23:33:29.910373+0000","last_clean":"2024-09-12T23:33:29.910373+0000","last_became_active":"2024-09-12T23:33:29.909765+0000","last_became_peered":"2024-09-12T23:33:29.909765+0000","last_unstale":"2024-09-12T23:33:29.910373+0000","last_undegraded":"2024-09-12T23:33:29.910373+0000","last_fullsized":"2024-09-12T23:33:29.910373+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:23:35.389216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.865860+0000","last_change":"2024-09-12T23:33:28.868957+0000","last_active":"2024-09-12T23:33:29.865860+0000","last_peered":"2024-09-12T23:33:29.865860+0000","last_clean":"2024-09-12T23:33:29.865860+0000","last_became_active":"2024-09-12T23:33:28.868587+0000","last_became_peered":"2024-09-12T23:33:28.868587+0000","last_unstale":"2024-09-12T23:33:29.865860+0000","last_undegraded":"2024-09-12T23:33:29.865860+0000","last_fullsized":"2024-09-12T23:33:29.865860+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:45:32.284209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.892338+0000","last_change":"2024-09-12T23:33:29.890360+0000","last_active":"2024-09-12T23:33:29.892338+0000","last_peered":"2024-09-12T23:33:29.892338+0000","last_clean":"2024-09-12T23:33:29.892338+0000","last_became_active":"2024-09-12T23:33:29.889575+0000","last_became_peered":"2024-09-12T23:33:29.889575+0000","last_unstale":"2024-09-12T23:33:29.892338+0000","last_undegraded":"2024-09-12T23:33:29.892338+0000","last_fullsized":"2024-09-12T23:33:29.892338+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:36:34.938054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.884900+0000","last_change":"2024-09-12T23:33:28.899931+0000","last_active":"2024-09-12T23:33:29.884900+0000","last_peered":"2024-09-12T23:33:29.884900+0000","last_clean":"2024-09-12T23:33:29.884900+0000","last_became_active":"2024-09-12T23:33:28.899410+0000","last_became_peered":"2024-09-12T23:33:28.899410+0000","last_unstale":"2024-09-12T23:33:29.884900+0000","last_undegraded":"2024-09-12T23:33:29.884900+0000","last_fullsized":"2024-09-12T23:33:29.884900+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:05:26.671172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.903500+0000","last_change":"2024-09-12T23:33:29.902878+0000","last_active":"2024-09-12T23:33:29.903500+0000","last_peered":"2024-09-12T23:33:29.903500+0000","last_clean":"2024-09-12T23:33:29.903500+0000","last_became_active":"2024-09-12T23:33:29.902561+0000","last_became_peered":"2024-09-12T23:33:29.902561+0000","last_unstale":"2024-09-12T23:33:29.903500+0000","last_undegraded":"2024-09-12T23:33:29.903500+0000","last_fullsized":"2024-09-12T23:33:29.903500+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:12:55.352139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.878775+0000","last_change":"2024-09-12T23:33:28.867304+0000","last_active":"2024-09-12T23:33:29.878775+0000","last_peered":"2024-09-12T23:33:29.878775+0000","last_clean":"2024-09-12T23:33:29.878775+0000","last_became_active":"2024-09-12T23:33:28.867020+0000","last_became_peered":"2024-09-12T23:33:28.867020+0000","last_unstale":"2024-09-12T23:33:29.878775+0000","last_undegraded":"2024-09-12T23:33:29.878775+0000","last_fullsized":"2024-09-12T23:33:29.878775+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:51:22.908188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.915943+0000","last_change":"2024-09-12T23:33:29.914728+0000","last_active":"2024-09-12T23:33:29.915943+0000","last_peered":"2024-09-12T23:33:29.915943+0000","last_clean":"2024-09-12T23:33:29.915943+0000","last_became_active":"2024-09-12T23:33:29.914385+0000","last_became_peered":"2024-09-12T23:33:29.914385+0000","last_unstale":"2024-09-12T23:33:29.915943+0000","last_undegraded":"2024-09-12T23:33:29.915943+0000","last_fullsized":"2024-09-12T23:33:29.915943+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:45:02.815733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.884021+0000","last_change":"2024-09-12T23:33:28.895077+0000","last_active":"2024-09-12T23:33:29.884021+0000","last_peered":"2024-09-12T23:33:29.884021+0000","last_clean":"2024-09-12T23:33:29.884021+0000","last_became_active":"2024-09-12T23:33:28.892268+0000","last_became_peered":"2024-09-12T23:33:28.892268+0000","last_unstale":"2024-09-12T23:33:29.884021+0000","last_undegraded":"2024-09-12T23:33:29.884021+0000","last_fullsized":"2024-09-12T23:33:29.884021+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:26:19.125877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.913453+0000","last_change":"2024-09-12T23:33:29.899700+0000","last_active":"2024-09-12T23:33:29.913453+0000","last_peered":"2024-09-12T23:33:29.913453+0000","last_clean":"2024-09-12T23:33:29.913453+0000","last_became_active":"2024-09-12T23:33:29.899381+0000","last_became_peered":"2024-09-12T23:33:29.899381+0000","last_unstale":"2024-09-12T23:33:29.913453+0000","last_undegraded":"2024-09-12T23:33:29.913453+0000","last_fullsized":"2024-09-12T23:33:29.913453+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:30:20.590373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"54'1","reported_seq":12,"reported_epoch":54,"state":"active+clean","last_fresh":"2024-09-12T23:33:31.824574+0000","last_change":"2024-09-12T23:33:28.866367+0000","last_active":"2024-09-12T23:33:31.824574+0000","last_peered":"2024-09-12T23:33:31.824574+0000","last_clean":"2024-09-12T23:33:31.824574+0000","last_became_active":"2024-09-12T23:33:28.866005+0000","last_became_peered":"2024-09-12T23:33:28.866005+0000","last_unstale":"2024-09-12T23:33:31.824574+0000","last_undegraded":"2024-09-12T23:33:31.824574+0000","last_fullsized":"2024-09-12T23:33:31.824574+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:49:27.803045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.917553+0000","last_change":"2024-09-12T23:33:29.915275+0000","last_active":"2024-09-12T23:33:29.917553+0000","last_peered":"2024-09-12T23:33:29.917553+0000","last_clean":"2024-09-12T23:33:29.917553+0000","last_became_active":"2024-09-12T23:33:29.914934+0000","last_became_peered":"2024-09-12T23:33:29.914934+0000","last_unstale":"2024-09-12T23:33:29.917553+0000","last_undegraded":"2024-09-12T23:33:29.917553+0000","last_fullsized":"2024-09-12T23:33:29.917553+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:43:27.562543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"53'2","reported_seq":13,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.904164+0000","last_change":"2024-09-12T23:33:28.887351+0000","last_active":"2024-09-12T23:33:29.904164+0000","last_peered":"2024-09-12T23:33:29.904164+0000","last_clean":"2024-09-12T23:33:29.904164+0000","last_became_active":"2024-09-12T23:33:28.886573+0000","last_became_peered":"2024-09-12T23:33:28.886573+0000","last_unstale":"2024-09-12T23:33:29.904164+0000","last_undegraded":"2024-09-12T23:33:29.904164+0000","last_fullsized":"2024-09-12T23:33:29.904164+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:58:40.820895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.909692+0000","last_change":"2024-09-12T23:33:29.908474+0000","last_active":"2024-09-12T23:33:29.909692+0000","last_peered":"2024-09-12T23:33:29.909692+0000","last_clean":"2024-09-12T23:33:29.909692+0000","last_became_active":"2024-09-12T23:33:29.906278+0000","last_became_peered":"2024-09-12T23:33:29.906278+0000","last_unstale":"2024-09-12T23:33:29.909692+0000","last_undegraded":"2024-09-12T23:33:29.909692+0000","last_fullsized":"2024-09-12T23:33:29.909692+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:27:51.832430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"0'0","reported_seq":11,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.873934+0000","last_change":"2024-09-12T23:33:28.894286+0000","last_active":"2024-09-12T23:33:29.873934+0000","last_peered":"2024-09-12T23:33:29.873934+0000","last_clean":"2024-09-12T23:33:29.873934+0000","last_became_active":"2024-09-12T23:33:28.893861+0000","last_became_peered":"2024-09-12T23:33:28.893861+0000","last_unstale":"2024-09-12T23:33:29.873934+0000","last_undegraded":"2024-09-12T23:33:29.873934+0000","last_fullsized":"2024-09-12T23:33:29.873934+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:20:01.806723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.890411+0000","last_change":"2024-09-12T23:33:29.888659+0000","last_active":"2024-09-12T23:33:29.890411+0000","last_peered":"2024-09-12T23:33:29.890411+0000","last_clean":"2024-09-12T23:33:29.890411+0000","last_became_active":"2024-09-12T23:33:29.888306+0000","last_became_peered":"2024-09-12T23:33:29.888306+0000","last_unstale":"2024-09-12T23:33:29.890411+0000","last_undegraded":"2024-09-12T23:33:29.890411+0000","last_fullsized":"2024-09-12T23:33:29.890411+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:09:39.250781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":11,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.874494+0000","last_change":"2024-09-12T23:33:28.884360+0000","last_active":"2024-09-12T23:33:29.874494+0000","last_peered":"2024-09-12T23:33:29.874494+0000","last_clean":"2024-09-12T23:33:29.874494+0000","last_became_active":"2024-09-12T23:33:28.884027+0000","last_became_peered":"2024-09-12T23:33:28.884027+0000","last_unstale":"2024-09-12T23:33:29.874494+0000","last_undegraded":"2024-09-12T23:33:29.874494+0000","last_fullsized":"2024-09-12T23:33:29.874494+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:11:57.567074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.917381+0000","last_change":"2024-09-12T23:33:29.915243+0000","last_active":"2024-09-12T23:33:29.917381+0000","last_peered":"2024-09-12T23:33:29.917381+0000","last_clean":"2024-09-12T23:33:29.917381+0000","last_became_active":"2024-09-12T23:33:29.914543+0000","last_became_peered":"2024-09-12T23:33:29.914543+0000","last_unstale":"2024-09-12T23:33:29.917381+0000","last_undegraded":"2024-09-12T23:33:29.917381+0000","last_fullsized":"2024-09-12T23:33:29.917381+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-13T23:48:16.023985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":11,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.879749+0000","last_change":"2024-09-12T23:33:28.883601+0000","last_active":"2024-09-12T23:33:29.879749+0000","last_peered":"2024-09-12T23:33:29.879749+0000","last_clean":"2024-09-12T23:33:29.879749+0000","last_became_active":"2024-09-12T23:33:28.883276+0000","last_became_peered":"2024-09-12T23:33:28.883276+0000","last_unstale":"2024-09-12T23:33:29.879749+0000","last_undegraded":"2024-09-12T23:33:29.879749+0000","last_fullsized":"2024-09-12T23:33:29.879749+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:26:21.733615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.913043+0000","last_change":"2024-09-12T23:33:29.908185+0000","last_active":"2024-09-12T23:33:29.913043+0000","last_peered":"2024-09-12T23:33:29.913043+0000","last_clean":"2024-09-12T23:33:29.913043+0000","last_became_active":"2024-09-12T23:33:29.907814+0000","last_became_peered":"2024-09-12T23:33:29.907814+0000","last_unstale":"2024-09-12T23:33:29.913043+0000","last_undegraded":"2024-09-12T23:33:29.913043+0000","last_fullsized":"2024-09-12T23:33:29.913043+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:45:47.207242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"53'2","reported_seq":13,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.917978+0000","last_change":"2024-09-12T23:33:28.902988+0000","last_active":"2024-09-12T23:33:29.917978+0000","last_peered":"2024-09-12T23:33:29.917978+0000","last_clean":"2024-09-12T23:33:29.917978+0000","last_became_active":"2024-09-12T23:33:28.902514+0000","last_became_peered":"2024-09-12T23:33:28.902514+0000","last_unstale":"2024-09-12T23:33:29.917978+0000","last_undegraded":"2024-09-12T23:33:29.917978+0000","last_fullsized":"2024-09-12T23:33:29.917978+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:29:42.595057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.897849+0000","last_change":"2024-09-12T23:33:29.897132+0000","last_active":"2024-09-12T23:33:29.897849+0000","last_peered":"2024-09-12T23:33:29.897849+0000","last_clean":"2024-09-12T23:33:29.897849+0000","last_became_active":"2024-09-12T23:33:29.896761+0000","last_became_peered":"2024-09-12T23:33:29.896761+0000","last_unstale":"2024-09-12T23:33:29.897849+0000","last_undegraded":"2024-09-12T23:33:29.897849+0000","last_fullsized":"2024-09-12T23:33:29.897849+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:49:56.554393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.911801+0000","last_change":"2024-09-12T23:33:29.911440+0000","last_active":"2024-09-12T23:33:29.911801+0000","last_peered":"2024-09-12T23:33:29.911801+0000","last_clean":"2024-09-12T23:33:29.911801+0000","last_became_active":"2024-09-12T23:33:29.911157+0000","last_became_peered":"2024-09-12T23:33:29.911157+0000","last_unstale":"2024-09-12T23:33:29.911801+0000","last_undegraded":"2024-09-12T23:33:29.911801+0000","last_fullsized":"2024-09-12T23:33:29.911801+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:49:59.313436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"53'2","reported_seq":13,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.917448+0000","last_change":"2024-09-12T23:33:28.902224+0000","last_active":"2024-09-12T23:33:29.917448+0000","last_peered":"2024-09-12T23:33:29.917448+0000","last_clean":"2024-09-12T23:33:29.917448+0000","last_became_active":"2024-09-12T23:33:28.901755+0000","last_became_peered":"2024-09-12T23:33:28.901755+0000","last_unstale":"2024-09-12T23:33:29.917448+0000","last_undegraded":"2024-09-12T23:33:29.917448+0000","last_fullsized":"2024-09-12T23:33:29.917448+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:06:15.504175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.915680+0000","last_change":"2024-09-12T23:33:29.913814+0000","last_active":"2024-09-12T23:33:29.915680+0000","last_peered":"2024-09-12T23:33:29.915680+0000","last_clean":"2024-09-12T23:33:29.915680+0000","last_became_active":"2024-09-12T23:33:29.913521+0000","last_became_peered":"2024-09-12T23:33:29.913521+0000","last_unstale":"2024-09-12T23:33:29.915680+0000","last_undegraded":"2024-09-12T23:33:29.915680+0000","last_fullsized":"2024-09-12T23:33:29.915680+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:30:35.754521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1d","version":"53'2","reported_seq":13,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.908778+0000","last_change":"2024-09-12T23:33:28.897466+0000","last_active":"2024-09-12T23:33:29.908778+0000","last_peered":"2024-09-12T23:33:29.908778+0000","last_clean":"2024-09-12T23:33:29.908778+0000","last_became_active":"2024-09-12T23:33:28.896936+0000","last_became_peered":"2024-09-12T23:33:28.896936+0000","last_unstale":"2024-09-12T23:33:29.908778+0000","last_undegraded":"2024-09-12T23:33:29.908778+0000","last_fullsized":"2024-09-12T23:33:29.908778+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:29:38.732168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.924987+0000","last_change":"2024-09-12T23:33:29.924610+0000","last_active":"2024-09-12T23:33:29.924987+0000","last_peered":"2024-09-12T23:33:29.924987+0000","last_clean":"2024-09-12T23:33:29.924987+0000","last_became_active":"2024-09-12T23:33:29.924320+0000","last_became_peered":"2024-09-12T23:33:29.924320+0000","last_unstale":"2024-09-12T23:33:29.924987+0000","last_undegraded":"2024-09-12T23:33:29.924987+0000","last_fullsized":"2024-09-12T23:33:29.924987+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:23:22.797301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1e","version":"53'1","reported_seq":12,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.921716+0000","last_change":"2024-09-12T23:33:28.894353+0000","last_active":"2024-09-12T23:33:29.921716+0000","last_peered":"2024-09-12T23:33:29.921716+0000","last_clean":"2024-09-12T23:33:29.921716+0000","last_became_active":"2024-09-12T23:33:28.893992+0000","last_became_peered":"2024-09-12T23:33:28.893992+0000","last_unstale":"2024-09-12T23:33:29.921716+0000","last_undegraded":"2024-09-12T23:33:29.921716+0000","last_fullsized":"2024-09-12T23:33:29.921716+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:43:43.889025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.900863+0000","last_change":"2024-09-12T23:33:29.891227+0000","last_active":"2024-09-12T23:33:29.900863+0000","last_peered":"2024-09-12T23:33:29.900863+0000","last_clean":"2024-09-12T23:33:29.900863+0000","last_became_active":"2024-09-12T23:33:29.890368+0000","last_became_peered":"2024-09-12T23:33:29.890368+0000","last_unstale":"2024-09-12T23:33:29.900863+0000","last_undegraded":"2024-09-12T23:33:29.900863+0000","last_fullsized":"2024-09-12T23:33:29.900863+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:38:37.526610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1f","version":"53'4","reported_seq":15,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-09-12T23:33:29.923427+0000","last_change":"2024-09-12T23:33:28.886016+0000","last_active":"2024-09-12T23:33:29.923427+0000","last_peered":"2024-09-12T23:33:29.923427+0000","last_clean":"2024-09-12T23:33:29.923427+0000","last_became_active":"2024-09-12T23:33:28.885720+0000","last_became_peered":"2024-09-12T23:33:28.885720+0000","last_unstale":"2024-09-12T23:33:29.923427+0000","last_undegraded":"2024-09-12T23:33:29.923427+0000","last_fullsized":"2024-09-12T23:33:29.923427+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:44:38.950039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":192,"acting":192,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3563520,"data_stored":3542208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430220,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298464,"kb_used_data":1104,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446688,"statfs":{"total":95995035648,"available":95689408512,"internally_reserved":0,"allocated":1130496,"data_stored":810625,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.76900000000000002}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.88600000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68200000000000005}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.0640000000000001}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64600000000000002}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.157}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.0309999999999999}]}]},{"osd":6,"up_from":42,"seq":180388626454,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298460,"kb_used_data":1104,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446692,"statfs":{"total":95995035648,"available":95689412608,"internally_reserved":0,"allocated":1130496,"data_stored":811270,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 12 23:32:56 2024","interfaces":[{"interface":"back","average":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"min":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":1.2450000000000001,"5min":1.2450000000000001,"15min":1.2450000000000001},"last":0.84599999999999997},{"interface":"front","average":{"1min":0.874,"5min":0.874,"15min":0.874},"min":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"max":{"1min":1.222,"5min":1.222,"15min":1.222},"last":0.93799999999999994}]},{"osd":1,"last update":"Thu Sep 12 23:32:56 2024","interfaces":[{"interface":"back","average":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"min":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":1.153,"5min":1.153,"15min":1.153},"last":1.1240000000000001},{"interface":"front","average":{"1min":0.871,"5min":0.871,"15min":0.871},"min":{"1min":0.62,"5min":0.62,"15min":0.62},"max":{"1min":1.306,"5min":1.306,"15min":1.306},"last":0.72999999999999998}]},{"osd":2,"last update":"Thu Sep 12 23:32:56 2024","interfaces":[{"interface":"back","average":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"min":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"max":{"1min":1.4359999999999999,"5min":1.4359999999999999,"15min":1.4359999999999999},"last":1.024},{"interface":"front","average":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"min":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"max":{"1min":1.131,"5min":1.131,"15min":1.131},"last":0.63900000000000001}]},{"osd":3,"last update":"Thu Sep 12 23:32:56 2024","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"min":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"max":{"1min":1.2330000000000001,"5min":1.2330000000000001,"15min":1.2330000000000001},"last":0.99099999999999999},{"interface":"front","average":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"min":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"max":{"1min":1.2290000000000001,"5min":1.2290000000000001,"15min":1.2290000000000001},"last":0.90500000000000003}]},{"osd":4,"last update":"Thu Sep 12 23:32:56 2024","interfaces":[{"interface":"back","average":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"min":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":1.131,"5min":1.131,"15min":1.131},"last":1.1020000000000001},{"interface":"front","average":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":1.389,"5min":1.389,"15min":1.389},"last":1.0780000000000001}]},{"osd":5,"last update":"Thu Sep 12 23:32:56 2024","interfaces":[{"interface":"back","average":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"min":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"max":{"1min":1.228,"5min":1.228,"15min":1.228},"last":0.77600000000000002},{"interface":"front","average":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"min":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":1.3109999999999999,"5min":1.3109999999999999,"15min":1.3109999999999999},"last":1.22}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.052}]}]},{"osd":5,"up_from":36,"seq":154618822689,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297880,"kb_used_data":524,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447272,"statfs":{"total":95995035648,"available":95690006528,"internally_reserved":0,"allocated":536576,"data_stored":220677,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 12 23:33:07 2024","interfaces":[{"interface":"back","average":{"1min":0.81399999999999995,"5min":0.78700000000000003,"15min":0.78300000000000003},"min":{"1min":0.56200000000000006,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":1.0740000000000001,"5min":1.6100000000000001,"15min":1.6100000000000001},"last":0.85699999999999998},{"interface":"front","average":{"1min":0.872,"5min":0.83399999999999996,"15min":0.82799999999999996},"min":{"1min":0.68999999999999995,"5min":0.58199999999999996,"15min":0.58199999999999996},"max":{"1min":1.0469999999999999,"5min":1.0840000000000001,"15min":1.0840000000000001},"last":0.79400000000000004}]},{"osd":1,"last update":"Thu Sep 12 23:33:07 2024","interfaces":[{"interface":"back","average":{"1min":0.88100000000000001,"5min":0.85199999999999998,"15min":0.84699999999999998},"min":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"max":{"1min":1.218,"5min":1.4470000000000001,"15min":1.4470000000000001},"last":0.76000000000000001},{"interface":"front","average":{"1min":0.89000000000000001,"5min":0.81200000000000006,"15min":0.79900000000000004},"min":{"1min":0.66300000000000003,"5min":0.502,"15min":0.502},"max":{"1min":1.274,"5min":1.274,"15min":1.274},"last":1.0980000000000001}]},{"osd":2,"last update":"Thu Sep 12 23:33:07 2024","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":0.878,"15min":0.87},"min":{"1min":0.73999999999999999,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":1.2509999999999999,"5min":1.2509999999999999,"15min":1.2509999999999999},"last":0.72999999999999998},{"interface":"front","average":{"1min":0.876,"5min":0.90400000000000003,"15min":0.90900000000000003},"min":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"max":{"1min":1.0980000000000001,"5min":1.141,"15min":1.141},"last":1.0649999999999999}]},{"osd":3,"last update":"Thu Sep 12 23:33:07 2024","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":0.91300000000000003,"15min":0.91100000000000003},"min":{"1min":0.68100000000000005,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.2010000000000001,"5min":1.3280000000000001,"15min":1.3280000000000001},"last":0.83999999999999997},{"interface":"front","average":{"1min":0.92000000000000004,"5min":0.91500000000000004,"15min":0.91400000000000003},"min":{"1min":0.67400000000000004,"5min":0.57499999999999996,"15min":0.57499999999999996},"max":{"1min":1.2609999999999999,"5min":1.536,"15min":1.536},"last":0.97799999999999998}]},{"osd":4,"last update":"Thu Sep 12 23:33:07 2024","interfaces":[{"interface":"back","average":{"1min":0.747,"5min":0.74299999999999999,"15min":0.74199999999999999},"min":{"1min":0.47199999999999998,"5min":0.308,"15min":0.308},"max":{"1min":1.1910000000000001,"5min":1.1910000000000001,"15min":1.1910000000000001},"last":0.91500000000000004},{"interface":"front","average":{"1min":0.79500000000000004,"5min":0.77700000000000002,"15min":0.77400000000000002},"min":{"1min":0.504,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":1.1679999999999999,"5min":1.1679999999999999,"15min":1.1679999999999999},"last":0.89300000000000002}]},{"osd":6,"last update":"Thu Sep 12 23:32:54 2024","interfaces":[{"interface":"back","average":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":1.1220000000000001,"5min":1.1220000000000001,"15min":1.1220000000000001},"last":0.95599999999999996},{"interface":"front","average":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"min":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":1.0840000000000001,"5min":1.0840000000000001,"15min":1.0840000000000001},"last":1.1359999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.194}]}]},{"osd":4,"up_from":30,"seq":128849018922,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297892,"kb_used_data":532,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447260,"statfs":{"total":95995035648,"available":95689994240,"internally_reserved":0,"allocated":544768,"data_stored":220361,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 12 23:33:17 2024","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":0.83599999999999997,"15min":0.83399999999999996},"min":{"1min":0.63200000000000001,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.375,"5min":1.375,"15min":1.375},"last":1.3720000000000001},{"interface":"front","average":{"1min":0.89500000000000002,"5min":0.84699999999999998,"15min":0.84299999999999997},"min":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"max":{"1min":1.1479999999999999,"5min":1.1479999999999999,"15min":1.1479999999999999},"last":1.2310000000000001}]},{"osd":1,"last update":"Thu Sep 12 23:33:17 2024","interfaces":[{"interface":"back","average":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82399999999999995},"min":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":1.194,"5min":1.194,"15min":1.194},"last":1.196},{"interface":"front","average":{"1min":0.86899999999999999,"5min":0.85899999999999999,"15min":0.85699999999999998},"min":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.218,"5min":1.218,"15min":1.218},"last":0.93500000000000005}]},{"osd":2,"last update":"Thu Sep 12 23:33:17 2024","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":0.84599999999999997,"15min":0.85599999999999998},"min":{"1min":0.60199999999999998,"5min":0.55700000000000005,"15min":0.55700000000000005},"max":{"1min":1.175,"5min":1.175,"15min":1.175},"last":1.119},{"interface":"front","average":{"1min":0.89500000000000002,"5min":0.85699999999999998,"15min":0.86299999999999999},"min":{"1min":0.66000000000000003,"5min":0.63500000000000001,"15min":0.63500000000000001},"max":{"1min":1.2909999999999999,"5min":1.2909999999999999,"15min":1.2909999999999999},"last":1.0109999999999999}]},{"osd":3,"last update":"Thu Sep 12 23:33:17 2024","interfaces":[{"interface":"back","average":{"1min":0.89700000000000002,"5min":0.85199999999999998,"15min":0.84999999999999998},"min":{"1min":0.65200000000000002,"5min":0.60099999999999998,"15min":0.60099999999999998},"max":{"1min":1.5129999999999999,"5min":1.5129999999999999,"15min":1.5129999999999999},"last":1.161},{"interface":"front","average":{"1min":0.97899999999999998,"5min":0.90200000000000002,"15min":0.89000000000000001},"min":{"1min":0.57099999999999995,"5min":0.55900000000000005,"15min":0.55900000000000005},"max":{"1min":1.595,"5min":1.595,"15min":1.595},"last":1.0529999999999999}]},{"osd":5,"last update":"Thu Sep 12 23:33:09 2024","interfaces":[{"interface":"back","average":{"1min":0.80500000000000005,"5min":0.78400000000000003,"15min":0.78000000000000003},"min":{"1min":0.49199999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":1.7430000000000001,"5min":1.7430000000000001,"15min":1.7430000000000001},"last":0.745},{"interface":"front","average":{"1min":0.90300000000000002,"5min":0.82199999999999995,"15min":0.80800000000000005},"min":{"1min":0.375,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.716,"5min":1.716,"15min":1.716},"last":1.2769999999999999}]},{"osd":6,"last update":"Thu Sep 12 23:32:56 2024","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":0.91300000000000003,"15min":0.91300000000000003},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":1.7629999999999999,"5min":1.7629999999999999,"15min":1.7629999999999999},"last":1.4039999999999999},{"interface":"front","average":{"1min":0.96299999999999997,"5min":0.96299999999999997,"15min":0.96299999999999997},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":1.7010000000000001,"5min":1.7010000000000001,"15min":1.7010000000000001},"last":1.4610000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.254}]}]},{"osd":3,"up_from":25,"seq":107374182452,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297896,"kb_used_data":536,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447256,"statfs":{"total":95995035648,"available":95689990144,"internally_reserved":0,"allocated":548864,"data_stored":221321,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 12 23:33:31 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.625,"15min":0.60299999999999998},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":1.04,"5min":1.04,"15min":1.04},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.629,"5min":0.69299999999999995,"15min":0.68899999999999995},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.89500000000000002,"5min":1.3120000000000001,"15min":1.3120000000000001},"last":0.71799999999999997}]},{"osd":1,"last update":"Thu Sep 12 23:33:31 2024","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":0.67500000000000004,"15min":0.65700000000000003},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":1.006,"5min":1.006,"15min":1.006},"last":0.67600000000000005},{"interface":"front","average":{"1min":0.71799999999999997,"5min":0.746,"15min":0.74099999999999999},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":1.401,"5min":1.401,"15min":1.401},"last":0.77900000000000003}]},{"osd":2,"last update":"Thu Sep 12 23:33:31 2024","interfaces":[{"interface":"back","average":{"1min":0.77100000000000002,"5min":0.78900000000000003,"15min":0.77700000000000002},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":1.2709999999999999,"5min":1.2709999999999999,"15min":1.2709999999999999},"last":0.96599999999999997},{"interface":"front","average":{"1min":0.73799999999999999,"5min":0.76100000000000001,"15min":0.73499999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":1.458,"5min":1.458,"15min":1.458},"last":0.996}]},{"osd":4,"last update":"Thu Sep 12 23:33:24 2024","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":0.90300000000000002,"15min":0.88200000000000001},"min":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.4950000000000001,"5min":1.4950000000000001,"15min":1.4950000000000001},"last":0.91300000000000003},{"interface":"front","average":{"1min":0.90300000000000002,"5min":0.87,"15min":0.84899999999999998},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":1.3640000000000001,"5min":1.3640000000000001,"15min":1.3640000000000001},"last":0.93999999999999995}]},{"osd":5,"last update":"Thu Sep 12 23:33:05 2024","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":0.98399999999999999,"15min":0.98399999999999999},"min":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"max":{"1min":1.321,"5min":1.371,"15min":1.371},"last":1.054},{"interface":"front","average":{"1min":1.05,"5min":0.98199999999999998,"15min":0.97099999999999997},"min":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":1.423,"5min":1.423,"15min":1.423},"last":0.81399999999999995}]},{"osd":6,"last update":"Thu Sep 12 23:32:54 2024","interfaces":[{"interface":"back","average":{"1min":1.052,"5min":1.052,"15min":1.052},"min":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"max":{"1min":1.3540000000000001,"5min":1.3540000000000001,"15min":1.3540000000000001},"last":0.86799999999999999},{"interface":"front","average":{"1min":1.115,"5min":1.115,"15min":1.115},"min":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"max":{"1min":1.718,"5min":1.718,"15min":1.718},"last":1.1060000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.153}]}]},{"osd":2,"up_from":18,"seq":77309411390,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297872,"kb_used_data":516,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447280,"statfs":{"total":95995035648,"available":95690014720,"internally_reserved":0,"allocated":528384,"data_stored":219665,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 12 23:32:41 2024","interfaces":[{"interface":"back","average":{"1min":0.72999999999999998,"5min":0.626,"15min":0.58799999999999997},"min":{"1min":0.53800000000000003,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":1.0429999999999999,"5min":1.23,"15min":1.23},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.745,"5min":0.66400000000000003,"15min":0.61299999999999999},"min":{"1min":0.50800000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":1.0940000000000001,"5min":1.0940000000000001,"15min":1.0940000000000001},"last":0.46100000000000002}]},{"osd":1,"last update":"Thu Sep 12 23:32:41 2024","interfaces":[{"interface":"back","average":{"1min":0.76400000000000001,"5min":0.67700000000000005,"15min":0.628},"min":{"1min":0.5,"5min":0.316,"15min":0.316},"max":{"1min":1.014,"5min":1.5329999999999999,"15min":1.5329999999999999},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.74199999999999999,"5min":0.68400000000000005,"15min":0.63700000000000001},"min":{"1min":0.60999999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.95099999999999996,"5min":1.0860000000000001,"15min":1.0860000000000001},"last":0.626}]},{"osd":3,"last update":"Thu Sep 12 23:33:33 2024","interfaces":[{"interface":"back","average":{"1min":0.69099999999999995,"5min":0.749,"15min":0.74099999999999999},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":1.163,"5min":1.2270000000000001,"15min":1.2270000000000001},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.69099999999999995,"5min":0.78200000000000003,"15min":0.77400000000000002},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.97999999999999998,"5min":1.6619999999999999,"15min":1.6619999999999999},"last":0.66200000000000003}]},{"osd":4,"last update":"Thu Sep 12 23:33:19 2024","interfaces":[{"interface":"back","average":{"1min":0.88300000000000001,"5min":0.873,"15min":0.86499999999999999},"min":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.107,"5min":1.1639999999999999,"15min":1.1639999999999999},"last":1.0409999999999999},{"interface":"front","average":{"1min":0.96899999999999997,"5min":0.95299999999999996,"15min":0.93500000000000005},"min":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"max":{"1min":1.23,"5min":1.363,"15min":1.363},"last":0.875}]},{"osd":5,"last update":"Thu Sep 12 23:33:10 2024","interfaces":[{"interface":"back","average":{"1min":0.95599999999999996,"5min":1.022,"15min":1.0329999999999999},"min":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"max":{"1min":1.282,"5min":1.4830000000000001,"15min":1.4830000000000001},"last":0.91500000000000004},{"interface":"front","average":{"1min":0.90700000000000003,"5min":1.006,"15min":1.022},"min":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"max":{"1min":1.101,"5min":1.3129999999999999,"15min":1.3129999999999999},"last":1.1890000000000001}]},{"osd":6,"last update":"Thu Sep 12 23:32:56 2024","interfaces":[{"interface":"back","average":{"1min":0.94499999999999995,"5min":0.94499999999999995,"15min":0.94499999999999995},"min":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"max":{"1min":1.2330000000000001,"5min":1.2330000000000001,"15min":1.2330000000000001},"last":0.94099999999999995},{"interface":"front","average":{"1min":1.0880000000000001,"5min":1.0880000000000001,"15min":1.0880000000000001},"min":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"max":{"1min":1.498,"5min":1.498,"15min":1.498},"last":0.96699999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.141}]}]},{"osd":1,"up_from":13,"seq":55834574921,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297880,"kb_used_data":524,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447272,"statfs":{"total":95995035648,"available":95690006528,"internally_reserved":0,"allocated":536576,"data_stored":219825,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 12 23:32:55 2024","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":0.59899999999999998,"15min":0.53000000000000003},"min":{"1min":0.51400000000000001,"5min":0.255,"15min":0.255},"max":{"1min":1.1579999999999999,"5min":1.1579999999999999,"15min":1.1579999999999999},"last":0.878},{"interface":"front","average":{"1min":0.77400000000000002,"5min":0.66000000000000003,"15min":0.58199999999999996},"min":{"1min":0.501,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":1.1830000000000001,"5min":1.1830000000000001,"15min":1.1830000000000001},"last":0.61799999999999999}]},{"osd":2,"last update":"Thu Sep 12 23:32:40 2024","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":0.64600000000000002,"15min":0.58799999999999997},"min":{"1min":0.54700000000000004,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.95899999999999996,"5min":1.046,"15min":1.046},"last":0.68999999999999995},{"interface":"front","average":{"1min":0.82899999999999996,"5min":0.69999999999999996,"15min":0.629},"min":{"1min":0.53200000000000003,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":1.3049999999999999,"5min":1.3049999999999999,"15min":1.3049999999999999},"last":0.79300000000000004}]},{"osd":3,"last update":"Thu Sep 12 23:33:34 2024","interfaces":[{"interface":"back","average":{"1min":0.89300000000000002,"5min":0.81399999999999995,"15min":0.78900000000000003},"min":{"1min":0.69499999999999995,"5min":0.54800000000000004,"15min":0.54800000000000004},"max":{"1min":1.198,"5min":1.198,"15min":1.198},"last":0.93700000000000006},{"interface":"front","average":{"1min":0.81899999999999995,"5min":0.78300000000000003,"15min":0.76600000000000001},"min":{"1min":0.63100000000000001,"5min":0.57799999999999996,"15min":0.57799999999999996},"max":{"1min":1.194,"5min":1.194,"15min":1.194},"last":0.76000000000000001}]},{"osd":4,"last update":"Thu Sep 12 23:33:16 2024","interfaces":[{"interface":"back","average":{"1min":0.95999999999999996,"5min":0.92100000000000004,"15min":0.91700000000000004},"min":{"1min":0.75700000000000001,"5min":0.65100000000000002,"15min":0.65100000000000002},"max":{"1min":1.23,"5min":1.585,"15min":1.585},"last":1.071},{"interface":"front","average":{"1min":1.006,"5min":0.93700000000000006,"15min":0.92700000000000005},"min":{"1min":0.76000000000000001,"5min":0.73499999999999999,"15min":0.73499999999999999},"max":{"1min":1.389,"5min":1.4279999999999999,"15min":1.4279999999999999},"last":1.03}]},{"osd":5,"last update":"Thu Sep 12 23:33:05 2024","interfaces":[{"interface":"back","average":{"1min":0.95399999999999996,"5min":0.92100000000000004,"15min":0.91500000000000004},"min":{"1min":0.66000000000000003,"5min":0.65200000000000002,"15min":0.65200000000000002},"max":{"1min":1.484,"5min":1.6819999999999999,"15min":1.6819999999999999},"last":0.89400000000000002},{"interface":"front","average":{"1min":1.0289999999999999,"5min":1.006,"15min":1.002},"min":{"1min":0.81000000000000005,"5min":0.66200000000000003,"15min":0.66200000000000003},"max":{"1min":1.5069999999999999,"5min":1.5069999999999999,"15min":1.5069999999999999},"last":1.0529999999999999}]},{"osd":6,"last update":"Thu Sep 12 23:32:55 2024","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":1.0529999999999999,"15min":1.0529999999999999},"min":{"1min":0.746,"5min":0.746,"15min":0.746},"max":{"1min":1.5429999999999999,"5min":1.5429999999999999,"15min":1.5429999999999999},"last":0.82999999999999996},{"interface":"front","average":{"1min":1.101,"5min":1.101,"15min":1.101},"min":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"max":{"1min":1.5649999999999999,"5min":1.5649999999999999,"15min":1.5649999999999999},"last":1.008}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.96799999999999997}]}]},{"osd":0,"up_from":8,"seq":34359738449,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298464,"kb_used_data":1108,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446688,"statfs":{"total":95995035648,"available":95689408512,"internally_reserved":0,"allocated":1134592,"data_stored":810434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Sep 12 23:32:49 2024","interfaces":[{"interface":"back","average":{"1min":0.69599999999999995,"5min":0.59899999999999998,"15min":0.51100000000000001},"min":{"1min":0.48499999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":1.0509999999999999,"5min":1.0509999999999999,"15min":1.0509999999999999},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.74399999999999999,"5min":0.629,"15min":0.53300000000000003},"min":{"1min":0.55400000000000005,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.0049999999999999,"5min":1.028,"15min":1.028},"last":0.69199999999999995}]},{"osd":2,"last update":"Thu Sep 12 23:32:43 2024","interfaces":[{"interface":"back","average":{"1min":0.73299999999999998,"5min":0.66600000000000004,"15min":0.61899999999999999},"min":{"1min":0.52800000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":1.081,"5min":1.081,"15min":1.081},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.82699999999999996,"5min":0.69599999999999995,"15min":0.63900000000000001},"min":{"1min":0.56000000000000005,"5min":0.317,"15min":0.317},"max":{"1min":1.151,"5min":1.151,"15min":1.151},"last":0.53700000000000003}]},{"osd":3,"last update":"Thu Sep 12 23:33:28 2024","interfaces":[{"interface":"back","average":{"1min":0.71299999999999997,"5min":0.75800000000000001,"15min":0.76000000000000001},"min":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":0.98799999999999999,"5min":1.177,"15min":1.177},"last":0.79200000000000004},{"interface":"front","average":{"1min":0.875,"5min":0.81299999999999994,"15min":0.79700000000000004},"min":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"max":{"1min":1.3140000000000001,"5min":1.3140000000000001,"15min":1.3140000000000001},"last":0.81100000000000005}]},{"osd":4,"last update":"Thu Sep 12 23:33:23 2024","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":0.93100000000000005,"15min":0.90200000000000002},"min":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"max":{"1min":1.304,"5min":1.304,"15min":1.304},"last":0.85799999999999998},{"interface":"front","average":{"1min":0.95699999999999996,"5min":0.876,"15min":0.84999999999999998},"min":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"max":{"1min":1.266,"5min":1.266,"15min":1.266},"last":1.0780000000000001}]},{"osd":5,"last update":"Thu Sep 12 23:33:10 2024","interfaces":[{"interface":"back","average":{"1min":0.94499999999999995,"5min":0.94299999999999995,"15min":0.94299999999999995},"min":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"max":{"1min":1.2330000000000001,"5min":1.2330000000000001,"15min":1.2330000000000001},"last":0.94199999999999995},{"interface":"front","average":{"1min":1.002,"5min":0.95899999999999996,"15min":0.95199999999999996},"min":{"1min":0.75,"5min":0.70099999999999996,"15min":0.70099999999999996},"max":{"1min":1.1579999999999999,"5min":1.1579999999999999,"15min":1.1579999999999999},"last":0.83999999999999997}]},{"osd":6,"last update":"Thu Sep 12 23:32:55 2024","interfaces":[{"interface":"back","average":{"1min":1.1120000000000001,"5min":1.1120000000000001,"15min":1.1120000000000001},"min":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"max":{"1min":1.492,"5min":1.492,"15min":1.492},"last":1.0629999999999999},{"interface":"front","average":{"1min":1.0509999999999999,"5min":1.0509999999999999,"15min":1.0509999999999999},"min":{"1min":0.754,"5min":0.754,"15min":0.754},"max":{"1min":1.377,"5min":1.377,"15min":1.377},"last":0.86899999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.98899999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":1656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1642,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-12T23:33:38.140 INFO:tasks.cephfs.filesystem.ceph_manager:clean! 2024-09-12T23:33:38.140 INFO:tasks.cephadm:Adding prometheus.a on smithi181 2024-09-12T23:33:38.140 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph orch apply prometheus '1;smithi181=a' 2024-09-12T23:33:39.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:38 smithi136 bash[22601]: cluster 2024-09-12T23:33:37.491710+0000 mgr.y (mgr.14150) 347 : cluster [DBG] pgmap v320: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2024-09-12T23:33:39.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:38 smithi136 bash[32189]: cluster 2024-09-12T23:33:37.491710+0000 mgr.y (mgr.14150) 347 : cluster [DBG] pgmap v320: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2024-09-12T23:33:39.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:38 smithi181 bash[24776]: cluster 2024-09-12T23:33:37.491710+0000 mgr.y (mgr.14150) 347 : cluster [DBG] pgmap v320: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2024-09-12T23:33:40.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:39 smithi136 bash[22601]: audit 2024-09-12T23:33:38.094205+0000 mgr.y (mgr.14150) 348 : audit [DBG] from='client.14514 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-12T23:33:40.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:39 smithi136 bash[32189]: audit 2024-09-12T23:33:38.094205+0000 mgr.y (mgr.14150) 348 : audit [DBG] from='client.14514 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-12T23:33:40.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:39 smithi181 bash[24776]: audit 2024-09-12T23:33:38.094205+0000 mgr.y (mgr.14150) 348 : audit [DBG] from='client.14514 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-12T23:33:41.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:40 smithi136 bash[22601]: cluster 2024-09-12T23:33:39.492563+0000 mgr.y (mgr.14150) 349 : cluster [DBG] pgmap v321: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-12T23:33:41.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:40 smithi136 bash[32189]: cluster 2024-09-12T23:33:39.492563+0000 mgr.y (mgr.14150) 349 : cluster [DBG] pgmap v321: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-12T23:33:41.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:40 smithi181 bash[24776]: cluster 2024-09-12T23:33:39.492563+0000 mgr.y (mgr.14150) 349 : cluster [DBG] pgmap v321: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-12T23:33:42.903 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.b/config 2024-09-12T23:33:43.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:42 smithi136 bash[22601]: cluster 2024-09-12T23:33:41.493523+0000 mgr.y (mgr.14150) 350 : cluster [DBG] pgmap v322: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2024-09-12T23:33:43.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:42 smithi136 bash[32189]: cluster 2024-09-12T23:33:41.493523+0000 mgr.y (mgr.14150) 350 : cluster [DBG] pgmap v322: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2024-09-12T23:33:43.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:42 smithi181 bash[24776]: cluster 2024-09-12T23:33:41.493523+0000 mgr.y (mgr.14150) 350 : cluster [DBG] pgmap v322: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2024-09-12T23:33:45.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:44 smithi136 bash[22601]: cluster 2024-09-12T23:33:43.494249+0000 mgr.y (mgr.14150) 351 : cluster [DBG] pgmap v323: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-12T23:33:45.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:44 smithi136 bash[32189]: cluster 2024-09-12T23:33:43.494249+0000 mgr.y (mgr.14150) 351 : cluster [DBG] pgmap v323: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-12T23:33:45.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:44 smithi181 bash[24776]: cluster 2024-09-12T23:33:43.494249+0000 mgr.y (mgr.14150) 351 : cluster [DBG] pgmap v323: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-12T23:33:45.750 INFO:teuthology.orchestra.run.smithi181.stdout:Scheduled prometheus update... 2024-09-12T23:33:46.371 DEBUG:teuthology.orchestra.run.smithi181:prometheus.a> sudo journalctl -f -n 0 -u ceph-da58a47c-715d-11ef-bceb-c7b262605968@prometheus.a.service 2024-09-12T23:33:46.374 INFO:tasks.cephadm:Adding node-exporter.a on smithi136 2024-09-12T23:33:46.374 INFO:tasks.cephadm:Adding node-exporter.b on smithi181 2024-09-12T23:33:46.374 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph orch apply node-exporter '2;smithi136=a;smithi181=b' 2024-09-12T23:33:46.748 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:33:46 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:33:47.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:46 smithi136 bash[22601]: cluster 2024-09-12T23:33:45.495439+0000 mgr.y (mgr.14150) 352 : cluster [DBG] pgmap v324: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 938 B/s wr, 3 op/s 2024-09-12T23:33:47.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:46 smithi136 bash[22601]: audit 2024-09-12T23:33:45.742335+0000 mgr.y (mgr.14150) 353 : audit [DBG] from='client.24358 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi181=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:33:47.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:46 smithi136 bash[22601]: cephadm 2024-09-12T23:33:45.745364+0000 mgr.y (mgr.14150) 354 : cephadm [INF] Saving service prometheus spec with placement smithi181=a;count:1 2024-09-12T23:33:47.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:46 smithi136 bash[22601]: audit 2024-09-12T23:33:45.753113+0000 mon.a (mon.0) 692 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:47.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:46 smithi136 bash[22601]: audit 2024-09-12T23:33:45.754880+0000 mon.a (mon.0) 693 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:33:47.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:46 smithi136 bash[22601]: audit 2024-09-12T23:33:46.174190+0000 mon.a (mon.0) 694 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:33:47.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:46 smithi136 bash[22601]: audit 2024-09-12T23:33:46.175841+0000 mon.a (mon.0) 695 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:33:47.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:46 smithi136 bash[22601]: audit 2024-09-12T23:33:46.187856+0000 mon.a (mon.0) 696 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:47.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:46 smithi136 bash[32189]: cluster 2024-09-12T23:33:45.495439+0000 mgr.y (mgr.14150) 352 : cluster [DBG] pgmap v324: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 938 B/s wr, 3 op/s 2024-09-12T23:33:47.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:46 smithi136 bash[32189]: audit 2024-09-12T23:33:45.742335+0000 mgr.y (mgr.14150) 353 : audit [DBG] from='client.24358 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi181=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:33:47.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:46 smithi136 bash[32189]: cephadm 2024-09-12T23:33:45.745364+0000 mgr.y (mgr.14150) 354 : cephadm [INF] Saving service prometheus spec with placement smithi181=a;count:1 2024-09-12T23:33:47.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:46 smithi136 bash[32189]: audit 2024-09-12T23:33:45.753113+0000 mon.a (mon.0) 692 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:47.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:46 smithi136 bash[32189]: audit 2024-09-12T23:33:45.754880+0000 mon.a (mon.0) 693 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:33:47.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:46 smithi136 bash[32189]: audit 2024-09-12T23:33:46.174190+0000 mon.a (mon.0) 694 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:33:47.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:46 smithi136 bash[32189]: audit 2024-09-12T23:33:46.175841+0000 mon.a (mon.0) 695 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:33:47.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:46 smithi136 bash[32189]: audit 2024-09-12T23:33:46.187856+0000 mon.a (mon.0) 696 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:47.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:46 smithi181 bash[24776]: cluster 2024-09-12T23:33:45.495439+0000 mgr.y (mgr.14150) 352 : cluster [DBG] pgmap v324: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 938 B/s wr, 3 op/s 2024-09-12T23:33:47.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:46 smithi181 bash[24776]: audit 2024-09-12T23:33:45.742335+0000 mgr.y (mgr.14150) 353 : audit [DBG] from='client.24358 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi181=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:33:47.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:46 smithi181 bash[24776]: cephadm 2024-09-12T23:33:45.745364+0000 mgr.y (mgr.14150) 354 : cephadm [INF] Saving service prometheus spec with placement smithi181=a;count:1 2024-09-12T23:33:47.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:46 smithi181 bash[24776]: audit 2024-09-12T23:33:45.753113+0000 mon.a (mon.0) 692 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:47.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:46 smithi181 bash[24776]: audit 2024-09-12T23:33:45.754880+0000 mon.a (mon.0) 693 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:33:47.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:46 smithi181 bash[24776]: audit 2024-09-12T23:33:46.174190+0000 mon.a (mon.0) 694 : audit [DBG] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:33:47.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:46 smithi181 bash[24776]: audit 2024-09-12T23:33:46.175841+0000 mon.a (mon.0) 695 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:33:47.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:46 smithi181 bash[24776]: audit 2024-09-12T23:33:46.187856+0000 mon.a (mon.0) 696 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:48.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:47 smithi136 bash[22601]: cephadm 2024-09-12T23:33:46.220122+0000 mgr.y (mgr.14150) 355 : cephadm [INF] Deploying daemon prometheus.a on smithi181 2024-09-12T23:33:48.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:47 smithi136 bash[32189]: cephadm 2024-09-12T23:33:46.220122+0000 mgr.y (mgr.14150) 355 : cephadm [INF] Deploying daemon prometheus.a on smithi181 2024-09-12T23:33:48.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:47 smithi181 bash[24776]: cephadm 2024-09-12T23:33:46.220122+0000 mgr.y (mgr.14150) 355 : cephadm [INF] Deploying daemon prometheus.a on smithi181 2024-09-12T23:33:49.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:48 smithi181 bash[24776]: cluster 2024-09-12T23:33:47.496364+0000 mgr.y (mgr.14150) 356 : cluster [DBG] pgmap v325: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-12T23:33:49.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:48 smithi136 bash[22601]: cluster 2024-09-12T23:33:47.496364+0000 mgr.y (mgr.14150) 356 : cluster [DBG] pgmap v325: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-12T23:33:49.295 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:48 smithi136 bash[32189]: cluster 2024-09-12T23:33:47.496364+0000 mgr.y (mgr.14150) 356 : cluster [DBG] pgmap v325: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-12T23:33:51.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:50 smithi181 bash[24776]: cluster 2024-09-12T23:33:49.497359+0000 mgr.y (mgr.14150) 357 : cluster [DBG] pgmap v326: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:51.157 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.b/config 2024-09-12T23:33:51.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:50 smithi136 bash[22601]: cluster 2024-09-12T23:33:49.497359+0000 mgr.y (mgr.14150) 357 : cluster [DBG] pgmap v326: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:51.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:50 smithi136 bash[32189]: cluster 2024-09-12T23:33:49.497359+0000 mgr.y (mgr.14150) 357 : cluster [DBG] pgmap v326: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:53.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:52 smithi181 bash[24776]: cluster 2024-09-12T23:33:51.498417+0000 mgr.y (mgr.14150) 358 : cluster [DBG] pgmap v327: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:53.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:52 smithi136 bash[22601]: cluster 2024-09-12T23:33:51.498417+0000 mgr.y (mgr.14150) 358 : cluster [DBG] pgmap v327: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:53.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:52 smithi136 bash[32189]: cluster 2024-09-12T23:33:51.498417+0000 mgr.y (mgr.14150) 358 : cluster [DBG] pgmap v327: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:55.178 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:54 smithi181 bash[24776]: cluster 2024-09-12T23:33:53.499242+0000 mgr.y (mgr.14150) 359 : cluster [DBG] pgmap v328: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:55.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:54 smithi136 bash[22601]: cluster 2024-09-12T23:33:53.499242+0000 mgr.y (mgr.14150) 359 : cluster [DBG] pgmap v328: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:55.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:54 smithi136 bash[32189]: cluster 2024-09-12T23:33:53.499242+0000 mgr.y (mgr.14150) 359 : cluster [DBG] pgmap v328: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:57.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:56 smithi181 bash[24776]: cluster 2024-09-12T23:33:55.500345+0000 mgr.y (mgr.14150) 360 : cluster [DBG] pgmap v329: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:57.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:56 smithi136 bash[22601]: cluster 2024-09-12T23:33:55.500345+0000 mgr.y (mgr.14150) 360 : cluster [DBG] pgmap v329: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:57.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:56 smithi136 bash[32189]: cluster 2024-09-12T23:33:55.500345+0000 mgr.y (mgr.14150) 360 : cluster [DBG] pgmap v329: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:57.264 INFO:teuthology.orchestra.run.smithi181.stdout:Scheduled node-exporter update... 2024-09-12T23:33:58.159 DEBUG:teuthology.orchestra.run.smithi136:node-exporter.a> sudo journalctl -f -n 0 -u ceph-da58a47c-715d-11ef-bceb-c7b262605968@node-exporter.a.service 2024-09-12T23:33:58.164 DEBUG:teuthology.orchestra.run.smithi181:node-exporter.b> sudo journalctl -f -n 0 -u ceph-da58a47c-715d-11ef-bceb-c7b262605968@node-exporter.b.service 2024-09-12T23:33:58.167 INFO:tasks.cephadm:Adding alertmanager.a on smithi136 2024-09-12T23:33:58.167 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph orch apply alertmanager '1;smithi136=a' 2024-09-12T23:33:58.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:58 smithi136 bash[22601]: audit 2024-09-12T23:33:57.254906+0000 mgr.y (mgr.14150) 361 : audit [DBG] from='client.24364 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi136=a;smithi181=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:33:58.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:58 smithi136 bash[22601]: cephadm 2024-09-12T23:33:57.258006+0000 mgr.y (mgr.14150) 362 : cephadm [INF] Saving service node-exporter spec with placement smithi136=a;smithi181=b;count:2 2024-09-12T23:33:58.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:58 smithi136 bash[22601]: audit 2024-09-12T23:33:57.267219+0000 mon.a (mon.0) 697 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:58.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:33:58 smithi136 bash[22601]: cluster 2024-09-12T23:33:57.501164+0000 mgr.y (mgr.14150) 363 : cluster [DBG] pgmap v330: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:58.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:58 smithi136 bash[32189]: audit 2024-09-12T23:33:57.254906+0000 mgr.y (mgr.14150) 361 : audit [DBG] from='client.24364 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi136=a;smithi181=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:33:58.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:58 smithi136 bash[32189]: cephadm 2024-09-12T23:33:57.258006+0000 mgr.y (mgr.14150) 362 : cephadm [INF] Saving service node-exporter spec with placement smithi136=a;smithi181=b;count:2 2024-09-12T23:33:58.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:58 smithi136 bash[32189]: audit 2024-09-12T23:33:57.267219+0000 mon.a (mon.0) 697 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:58.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:33:58 smithi136 bash[32189]: cluster 2024-09-12T23:33:57.501164+0000 mgr.y (mgr.14150) 363 : cluster [DBG] pgmap v330: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:33:58.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:58 smithi181 bash[24776]: audit 2024-09-12T23:33:57.254906+0000 mgr.y (mgr.14150) 361 : audit [DBG] from='client.24364 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi136=a;smithi181=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:33:58.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:58 smithi181 bash[24776]: cephadm 2024-09-12T23:33:57.258006+0000 mgr.y (mgr.14150) 362 : cephadm [INF] Saving service node-exporter spec with placement smithi136=a;smithi181=b;count:2 2024-09-12T23:33:58.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:58 smithi181 bash[24776]: audit 2024-09-12T23:33:57.267219+0000 mon.a (mon.0) 697 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:33:58.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:33:58 smithi181 bash[24776]: cluster 2024-09-12T23:33:57.501164+0000 mgr.y (mgr.14150) 363 : cluster [DBG] pgmap v330: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:00.668 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:00 smithi181 bash[24776]: cluster 2024-09-12T23:33:59.502099+0000 mgr.y (mgr.14150) 364 : cluster [DBG] pgmap v331: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:01.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:00 smithi136 bash[22601]: cluster 2024-09-12T23:33:59.502099+0000 mgr.y (mgr.14150) 364 : cluster [DBG] pgmap v331: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:01.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:00 smithi136 bash[32189]: cluster 2024-09-12T23:33:59.502099+0000 mgr.y (mgr.14150) 364 : cluster [DBG] pgmap v331: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:01.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:01 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:01.347 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:34:01 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:01.347 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:34:01 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:01.347 INFO:journalctl@ceph.mds.b.smithi181.stdout:Sep 12 23:34:01 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:01.348 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:34:01 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:01.348 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 12 23:34:01 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:01.349 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 12 23:34:01 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:01.349 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:34:01 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:01.349 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:34:01 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:01.349 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:34:01 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:01.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:01 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:01.847 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:34:01 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:01.847 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:34:01 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:01.848 INFO:journalctl@ceph.mds.b.smithi181.stdout:Sep 12 23:34:01 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:01.848 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:34:01 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:01.848 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 12 23:34:01 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:01.848 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 12 23:34:01 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:01.849 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:34:01 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:01.849 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:34:01 smithi181 systemd[1]: Started Ceph prometheus.a for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:34:02.754 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:34:02 smithi181 bash[25717]: ignoring --setuser ceph since I am not root 2024-09-12T23:34:02.754 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:34:02 smithi181 bash[25717]: ignoring --setgroup ceph since I am not root 2024-09-12T23:34:02.755 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:02 smithi181 bash[24776]: cluster 2024-09-12T23:34:01.503117+0000 mgr.y (mgr.14150) 365 : cluster [DBG] pgmap v332: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:02.755 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:02 smithi181 bash[24776]: audit 2024-09-12T23:34:01.580444+0000 mon.a (mon.0) 698 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:34:02.755 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:02 smithi181 bash[24776]: audit 2024-09-12T23:34:01.589579+0000 mon.a (mon.0) 699 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:34:02.755 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:02 smithi181 bash[24776]: audit 2024-09-12T23:34:01.597923+0000 mon.a (mon.0) 700 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:34:02.755 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:02 smithi181 bash[24776]: audit 2024-09-12T23:34:01.605738+0000 mon.a (mon.0) 701 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-12T23:34:02.755 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:34:02 smithi181 bash[49069]: ts=2024-09-12T23:34:02.558Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-12T23:34:02.755 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:34:02 smithi181 bash[49069]: ts=2024-09-12T23:34:02.558Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-12T23:34:02.755 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:34:02 smithi181 bash[49069]: ts=2024-09-12T23:34:02.558Z caller=main.go:570 level=info host_details="(Linux 5.15.0-121-generic #131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024 x86_64 smithi181 (none))" 2024-09-12T23:34:02.755 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:34:02 smithi181 bash[49069]: ts=2024-09-12T23:34:02.558Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-12T23:34:02.755 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:34:02 smithi181 bash[49069]: ts=2024-09-12T23:34:02.558Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-12T23:34:02.755 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:34:02 smithi181 bash[49069]: ts=2024-09-12T23:34:02.559Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-12T23:34:02.756 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:34:02 smithi181 bash[49069]: ts=2024-09-12T23:34:02.560Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-12T23:34:02.756 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:34:02 smithi181 bash[49069]: ts=2024-09-12T23:34:02.561Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-12T23:34:02.756 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:34:02 smithi181 bash[49069]: ts=2024-09-12T23:34:02.561Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-12T23:34:02.756 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:34:02 smithi181 bash[49069]: ts=2024-09-12T23:34:02.563Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-12T23:34:02.756 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:34:02 smithi181 bash[49069]: ts=2024-09-12T23:34:02.563Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.267µs 2024-09-12T23:34:02.756 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:34:02 smithi181 bash[49069]: ts=2024-09-12T23:34:02.563Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-12T23:34:02.756 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:34:02 smithi181 bash[49069]: ts=2024-09-12T23:34:02.564Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-09-12T23:34:02.756 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:34:02 smithi181 bash[49069]: ts=2024-09-12T23:34:02.564Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=48.558µs wal_replay_duration=352.881µs wbl_replay_duration=127ns total_replay_duration=422.889µs 2024-09-12T23:34:02.756 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:34:02 smithi181 bash[49069]: ts=2024-09-12T23:34:02.565Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-12T23:34:02.756 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:34:02 smithi181 bash[49069]: ts=2024-09-12T23:34:02.565Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-12T23:34:02.756 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:34:02 smithi181 bash[49069]: ts=2024-09-12T23:34:02.565Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-12T23:34:02.756 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:34:02 smithi181 bash[49069]: ts=2024-09-12T23:34:02.582Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=17.433154ms db_storage=1.332µs remote_storage=1.429µs web_handler=576ns query_engine=792ns scrape=270.89µs scrape_sd=35.482µs notify=739ns notify_sd=1.62µs rules=16.844796ms tracing=5.748µs 2024-09-12T23:34:02.757 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:34:02 smithi181 bash[49069]: ts=2024-09-12T23:34:02.582Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-12T23:34:02.757 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:34:02 smithi181 bash[49069]: ts=2024-09-12T23:34:02.582Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-12T23:34:02.947 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.b/config 2024-09-12T23:34:03.006 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:02 smithi136 bash[22852]: ignoring --setuser ceph since I am not root 2024-09-12T23:34:03.006 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:02 smithi136 bash[22852]: ignoring --setgroup ceph since I am not root 2024-09-12T23:34:03.006 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:02 smithi136 bash[22852]: debug 2024-09-12T23:34:02.656+0000 7f491752d640 1 -- 172.21.15.136:0/3178082271 <== mon.0 v2:172.21.15.136:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x560554f96340 con 0x560554fa3000 2024-09-12T23:34:03.006 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:02 smithi136 bash[22852]: debug 2024-09-12T23:34:02.764+0000 7f4919d87ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-12T23:34:03.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:02 smithi136 bash[22601]: cluster 2024-09-12T23:34:01.503117+0000 mgr.y (mgr.14150) 365 : cluster [DBG] pgmap v332: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:03.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:02 smithi136 bash[22601]: audit 2024-09-12T23:34:01.580444+0000 mon.a (mon.0) 698 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:34:03.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:02 smithi136 bash[22601]: audit 2024-09-12T23:34:01.589579+0000 mon.a (mon.0) 699 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:34:03.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:02 smithi136 bash[22601]: audit 2024-09-12T23:34:01.597923+0000 mon.a (mon.0) 700 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:34:03.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:02 smithi136 bash[22601]: audit 2024-09-12T23:34:01.605738+0000 mon.a (mon.0) 701 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-12T23:34:03.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:02 smithi136 bash[32189]: cluster 2024-09-12T23:34:01.503117+0000 mgr.y (mgr.14150) 365 : cluster [DBG] pgmap v332: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:03.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:02 smithi136 bash[32189]: audit 2024-09-12T23:34:01.580444+0000 mon.a (mon.0) 698 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:34:03.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:02 smithi136 bash[32189]: audit 2024-09-12T23:34:01.589579+0000 mon.a (mon.0) 699 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:34:03.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:02 smithi136 bash[32189]: audit 2024-09-12T23:34:01.597923+0000 mon.a (mon.0) 700 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' 2024-09-12T23:34:03.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:02 smithi136 bash[32189]: audit 2024-09-12T23:34:01.605738+0000 mon.a (mon.0) 701 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-12T23:34:03.096 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:34:02 smithi181 bash[25717]: debug 2024-09-12T23:34:02.752+0000 7f7302c4eec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-12T23:34:03.946 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:03 smithi136 bash[32189]: audit 2024-09-12T23:34:02.609684+0000 mon.a (mon.0) 702 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-12T23:34:03.947 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:03 smithi136 bash[32189]: cluster 2024-09-12T23:34:02.609764+0000 mon.a (mon.0) 703 : cluster [DBG] mgrmap e16: y(active, since 9m), standbys: x 2024-09-12T23:34:03.947 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:03 smithi136 bash[22601]: audit 2024-09-12T23:34:02.609684+0000 mon.a (mon.0) 702 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-12T23:34:03.947 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:03 smithi136 bash[22601]: cluster 2024-09-12T23:34:02.609764+0000 mon.a (mon.0) 703 : cluster [DBG] mgrmap e16: y(active, since 9m), standbys: x 2024-09-12T23:34:03.954 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:03 smithi181 bash[24776]: audit 2024-09-12T23:34:02.609684+0000 mon.a (mon.0) 702 : audit [INF] from='mgr.14150 172.21.15.136:0/3771271889' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-12T23:34:03.954 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:03 smithi181 bash[24776]: cluster 2024-09-12T23:34:02.609764+0000 mon.a (mon.0) 703 : cluster [DBG] mgrmap e16: y(active, since 9m), standbys: x 2024-09-12T23:34:04.256 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:03 smithi136 bash[22852]: debug 2024-09-12T23:34:03.948+0000 7f4919d87ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-12T23:34:04.256 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:04 smithi136 bash[22852]: debug 2024-09-12T23:34:04.008+0000 7f4919d87ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-12T23:34:04.257 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:04 smithi136 bash[22852]: debug 2024-09-12T23:34:04.160+0000 7f4919d87ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-12T23:34:04.266 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:34:03 smithi181 bash[25717]: debug 2024-09-12T23:34:03.952+0000 7f7302c4eec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-12T23:34:04.266 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:34:04 smithi181 bash[25717]: debug 2024-09-12T23:34:04.020+0000 7f7302c4eec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-12T23:34:04.266 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:34:04 smithi181 bash[25717]: debug 2024-09-12T23:34:04.172+0000 7f7302c4eec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-12T23:34:05.070 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:04 smithi136 bash[22852]: debug 2024-09-12T23:34:04.780+0000 7f4919d87ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-12T23:34:05.070 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:04 smithi136 bash[22852]: debug 2024-09-12T23:34:04.848+0000 7f4919d87ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-12T23:34:05.071 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:04 smithi136 bash[22852]: debug 2024-09-12T23:34:04.916+0000 7f4919d87ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-12T23:34:05.096 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:34:04 smithi181 bash[25717]: debug 2024-09-12T23:34:04.828+0000 7f7302c4eec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-12T23:34:05.096 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:34:04 smithi181 bash[25717]: debug 2024-09-12T23:34:04.900+0000 7f7302c4eec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-12T23:34:05.096 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:34:04 smithi181 bash[25717]: debug 2024-09-12T23:34:04.972+0000 7f7302c4eec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-12T23:34:05.437 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:05 smithi136 bash[22852]: debug 2024-09-12T23:34:05.072+0000 7f4919d87ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-12T23:34:05.437 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:05 smithi136 bash[22852]: debug 2024-09-12T23:34:05.148+0000 7f4919d87ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-12T23:34:05.437 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:05 smithi136 bash[22852]: debug 2024-09-12T23:34:05.212+0000 7f4919d87ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-12T23:34:05.507 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:34:05 smithi181 bash[25717]: debug 2024-09-12T23:34:05.132+0000 7f7302c4eec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-12T23:34:05.507 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:34:05 smithi181 bash[25717]: debug 2024-09-12T23:34:05.208+0000 7f7302c4eec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-12T23:34:05.507 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:34:05 smithi181 bash[25717]: debug 2024-09-12T23:34:05.276+0000 7f7302c4eec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-12T23:34:05.507 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:34:05 smithi181 bash[25717]: debug 2024-09-12T23:34:05.504+0000 7f7302c4eec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-12T23:34:05.745 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:05 smithi136 bash[22852]: debug 2024-09-12T23:34:05.436+0000 7f4919d87ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-12T23:34:05.745 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:05 smithi136 bash[22852]: debug 2024-09-12T23:34:05.504+0000 7f4919d87ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-12T23:34:05.745 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:05 smithi136 bash[22852]: debug 2024-09-12T23:34:05.672+0000 7f4919d87ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-12T23:34:05.828 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:34:05 smithi181 bash[25717]: debug 2024-09-12T23:34:05.572+0000 7f7302c4eec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-12T23:34:05.828 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:34:05 smithi181 bash[25717]: debug 2024-09-12T23:34:05.748+0000 7f7302c4eec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-12T23:34:06.006 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:05 smithi136 bash[22852]: debug 2024-09-12T23:34:05.748+0000 7f4919d87ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-12T23:34:06.006 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:05 smithi136 bash[22852]: debug 2024-09-12T23:34:05.976+0000 7f4919d87ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-12T23:34:06.096 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:34:05 smithi181 bash[25717]: debug 2024-09-12T23:34:05.824+0000 7f7302c4eec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-12T23:34:06.096 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:34:06 smithi181 bash[25717]: debug 2024-09-12T23:34:06.060+0000 7f7302c4eec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-12T23:34:06.506 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:06 smithi136 bash[22852]: debug 2024-09-12T23:34:06.136+0000 7f4919d87ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-12T23:34:06.506 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:06 smithi136 bash[22852]: debug 2024-09-12T23:34:06.212+0000 7f4919d87ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-12T23:34:06.506 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:06 smithi136 bash[22852]: debug 2024-09-12T23:34:06.364+0000 7f4919d87ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-12T23:34:06.596 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:34:06 smithi181 bash[25717]: debug 2024-09-12T23:34:06.216+0000 7f7302c4eec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-12T23:34:06.597 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:34:06 smithi181 bash[25717]: debug 2024-09-12T23:34:06.292+0000 7f7302c4eec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-12T23:34:06.597 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:34:06 smithi181 bash[25717]: debug 2024-09-12T23:34:06.444+0000 7f7302c4eec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-12T23:34:07.403 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[22852]: debug 2024-09-12T23:34:07.136+0000 7f4919d87ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-12T23:34:07.403 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[22852]: debug 2024-09-12T23:34:07.208+0000 7f4919d87ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-12T23:34:07.480 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:34:07 smithi181 bash[25717]: debug 2024-09-12T23:34:07.207+0000 7f7302c4eec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-12T23:34:07.480 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:34:07 smithi181 bash[25717]: debug 2024-09-12T23:34:07.279+0000 7f7302c4eec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-12T23:34:07.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[22852]: debug 2024-09-12T23:34:07.404+0000 7f4919d87ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-12T23:34:07.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[22852]: debug 2024-09-12T23:34:07.476+0000 7f4919d87ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-12T23:34:07.846 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:34:07 smithi181 bash[25717]: debug 2024-09-12T23:34:07.475+0000 7f7302c4eec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-12T23:34:07.847 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:34:07 smithi181 bash[25717]: debug 2024-09-12T23:34:07.547+0000 7f7302c4eec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-12T23:34:08.107 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[22601]: cluster 2024-09-12T23:34:07.892213+0000 mon.a (mon.0) 704 : cluster [INF] Active manager daemon y restarted 2024-09-12T23:34:08.107 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[22601]: cluster 2024-09-12T23:34:07.893135+0000 mon.a (mon.0) 705 : cluster [INF] Activating manager daemon y 2024-09-12T23:34:08.107 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[22601]: cluster 2024-09-12T23:34:07.903947+0000 mon.a (mon.0) 706 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2024-09-12T23:34:08.107 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[22601]: cluster 2024-09-12T23:34:07.905466+0000 mon.a (mon.0) 707 : cluster [DBG] mgrmap e17: y(active, starting, since 0.0126415s), standbys: x 2024-09-12T23:34:08.107 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[22601]: audit 2024-09-12T23:34:07.911762+0000 mon.a (mon.0) 708 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:34:08.108 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[22601]: audit 2024-09-12T23:34:07.911925+0000 mon.a (mon.0) 709 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:34:08.108 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[22601]: audit 2024-09-12T23:34:07.912173+0000 mon.a (mon.0) 710 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:34:08.108 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[22601]: audit 2024-09-12T23:34:07.912460+0000 mon.a (mon.0) 711 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-12T23:34:08.108 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[22601]: audit 2024-09-12T23:34:07.913241+0000 mon.a (mon.0) 712 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-12T23:34:08.108 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[22601]: audit 2024-09-12T23:34:07.913892+0000 mon.a (mon.0) 713 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-12T23:34:08.108 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[22601]: audit 2024-09-12T23:34:07.914222+0000 mon.a (mon.0) 714 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-12T23:34:08.108 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[22601]: audit 2024-09-12T23:34:07.914565+0000 mon.a (mon.0) 715 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-12T23:34:08.108 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[22601]: audit 2024-09-12T23:34:07.914855+0000 mon.a (mon.0) 716 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-12T23:34:08.108 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[22601]: audit 2024-09-12T23:34:07.915117+0000 mon.a (mon.0) 717 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-12T23:34:08.108 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[22601]: audit 2024-09-12T23:34:07.915347+0000 mon.a (mon.0) 718 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-12T23:34:08.108 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[22601]: audit 2024-09-12T23:34:07.915700+0000 mon.a (mon.0) 719 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-12T23:34:08.108 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[22601]: audit 2024-09-12T23:34:07.915964+0000 mon.a (mon.0) 720 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-12T23:34:08.108 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[22601]: audit 2024-09-12T23:34:07.916195+0000 mon.a (mon.0) 721 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-12T23:34:08.109 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[22601]: audit 2024-09-12T23:34:07.916428+0000 mon.a (mon.0) 722 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-12T23:34:08.109 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[22601]: audit 2024-09-12T23:34:07.916691+0000 mon.a (mon.0) 723 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-12T23:34:08.109 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[22601]: audit 2024-09-12T23:34:07.916964+0000 mon.a (mon.0) 724 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-12T23:34:08.109 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[22601]: audit 2024-09-12T23:34:07.917533+0000 mon.a (mon.0) 725 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-12T23:34:08.109 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[22601]: cluster 2024-09-12T23:34:07.942904+0000 mon.a (mon.0) 726 : cluster [INF] Manager daemon y is now available 2024-09-12T23:34:08.109 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[22852]: debug 2024-09-12T23:34:07.792+0000 7f4919d87ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-12T23:34:08.109 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[22852]: debug 2024-09-12T23:34:07.868+0000 7f4919d87ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-12T23:34:08.109 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[22852]: [12/Sep/2024:23:34:07] ENGINE Bus STARTING 2024-09-12T23:34:08.109 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[22852]: CherryPy Checker: 2024-09-12T23:34:08.109 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[22852]: The Application mounted at '' has an empty config. 2024-09-12T23:34:08.110 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:08 smithi136 bash[22852]: debug 2024-09-12T23:34:08.036+0000 7f48c96d2640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-12T23:34:08.110 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:08 smithi136 bash[22852]: debug 2024-09-12T23:34:08.036+0000 7f48c96d2640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-12T23:34:08.110 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:08 smithi136 bash[22852]: debug 2024-09-12T23:34:08.036+0000 7f48c96d2640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-12T23:34:08.110 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:08 smithi136 bash[22852]: debug 2024-09-12T23:34:08.036+0000 7f48c96d2640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-12T23:34:08.110 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:08 smithi136 bash[22852]: debug 2024-09-12T23:34:08.036+0000 7f48c96d2640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-12T23:34:08.110 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:08 smithi136 bash[22852]: debug 2024-09-12T23:34:08.036+0000 7f48c66cc640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-12T23:34:08.110 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:08 smithi136 bash[22852]: debug 2024-09-12T23:34:08.036+0000 7f48c66cc640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-12T23:34:08.110 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:08 smithi136 bash[22852]: debug 2024-09-12T23:34:08.036+0000 7f48c66cc640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-12T23:34:08.110 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:08 smithi136 bash[22852]: debug 2024-09-12T23:34:08.036+0000 7f48c66cc640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-12T23:34:08.110 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:08 smithi136 bash[22852]: debug 2024-09-12T23:34:08.036+0000 7f48c66cc640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-12T23:34:08.110 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:08 smithi136 bash[22852]: [12/Sep/2024:23:34:08] ENGINE Serving on http://:::9283 2024-09-12T23:34:08.111 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[32189]: cluster 2024-09-12T23:34:07.892213+0000 mon.a (mon.0) 704 : cluster [INF] Active manager daemon y restarted 2024-09-12T23:34:08.111 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[32189]: cluster 2024-09-12T23:34:07.893135+0000 mon.a (mon.0) 705 : cluster [INF] Activating manager daemon y 2024-09-12T23:34:08.111 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[32189]: cluster 2024-09-12T23:34:07.903947+0000 mon.a (mon.0) 706 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2024-09-12T23:34:08.111 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[32189]: cluster 2024-09-12T23:34:07.905466+0000 mon.a (mon.0) 707 : cluster [DBG] mgrmap e17: y(active, starting, since 0.0126415s), standbys: x 2024-09-12T23:34:08.111 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[32189]: audit 2024-09-12T23:34:07.911762+0000 mon.a (mon.0) 708 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:34:08.111 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[32189]: audit 2024-09-12T23:34:07.911925+0000 mon.a (mon.0) 709 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:34:08.111 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[32189]: audit 2024-09-12T23:34:07.912173+0000 mon.a (mon.0) 710 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:34:08.111 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[32189]: audit 2024-09-12T23:34:07.912460+0000 mon.a (mon.0) 711 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-12T23:34:08.111 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[32189]: audit 2024-09-12T23:34:07.913241+0000 mon.a (mon.0) 712 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-12T23:34:08.111 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[32189]: audit 2024-09-12T23:34:07.913892+0000 mon.a (mon.0) 713 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-12T23:34:08.111 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[32189]: audit 2024-09-12T23:34:07.914222+0000 mon.a (mon.0) 714 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-12T23:34:08.111 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[32189]: audit 2024-09-12T23:34:07.914565+0000 mon.a (mon.0) 715 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-12T23:34:08.112 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[32189]: audit 2024-09-12T23:34:07.914855+0000 mon.a (mon.0) 716 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-12T23:34:08.112 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[32189]: audit 2024-09-12T23:34:07.915117+0000 mon.a (mon.0) 717 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-12T23:34:08.112 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[32189]: audit 2024-09-12T23:34:07.915347+0000 mon.a (mon.0) 718 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-12T23:34:08.112 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[32189]: audit 2024-09-12T23:34:07.915700+0000 mon.a (mon.0) 719 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-12T23:34:08.112 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[32189]: audit 2024-09-12T23:34:07.915964+0000 mon.a (mon.0) 720 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-12T23:34:08.112 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[32189]: audit 2024-09-12T23:34:07.916195+0000 mon.a (mon.0) 721 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-12T23:34:08.112 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[32189]: audit 2024-09-12T23:34:07.916428+0000 mon.a (mon.0) 722 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-12T23:34:08.112 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[32189]: audit 2024-09-12T23:34:07.916691+0000 mon.a (mon.0) 723 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-12T23:34:08.112 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[32189]: audit 2024-09-12T23:34:07.916964+0000 mon.a (mon.0) 724 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-12T23:34:08.112 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[32189]: audit 2024-09-12T23:34:07.917533+0000 mon.a (mon.0) 725 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-12T23:34:08.112 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:07 smithi136 bash[32189]: cluster 2024-09-12T23:34:07.942904+0000 mon.a (mon.0) 726 : cluster [INF] Manager daemon y is now available 2024-09-12T23:34:08.265 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:34:07 smithi181 bash[25717]: debug 2024-09-12T23:34:07.863+0000 7f7302c4eec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-12T23:34:08.265 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:34:07 smithi181 bash[25717]: debug 2024-09-12T23:34:07.943+0000 7f7302c4eec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-12T23:34:08.265 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:34:07 smithi181 bash[25717]: [12/Sep/2024:23:34:07] ENGINE Bus STARTING 2024-09-12T23:34:08.265 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:34:07 smithi181 bash[25717]: CherryPy Checker: 2024-09-12T23:34:08.265 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:34:07 smithi181 bash[25717]: The Application mounted at '' has an empty config. 2024-09-12T23:34:08.265 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:34:08 smithi181 bash[25717]: [12/Sep/2024:23:34:08] ENGINE Serving on http://:::9283 2024-09-12T23:34:08.265 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:34:08 smithi181 bash[25717]: [12/Sep/2024:23:34:08] ENGINE Bus STARTED 2024-09-12T23:34:08.266 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:07 smithi181 bash[24776]: cluster 2024-09-12T23:34:07.892213+0000 mon.a (mon.0) 704 : cluster [INF] Active manager daemon y restarted 2024-09-12T23:34:08.266 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:07 smithi181 bash[24776]: cluster 2024-09-12T23:34:07.893135+0000 mon.a (mon.0) 705 : cluster [INF] Activating manager daemon y 2024-09-12T23:34:08.266 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:07 smithi181 bash[24776]: cluster 2024-09-12T23:34:07.903947+0000 mon.a (mon.0) 706 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2024-09-12T23:34:08.266 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:07 smithi181 bash[24776]: cluster 2024-09-12T23:34:07.905466+0000 mon.a (mon.0) 707 : cluster [DBG] mgrmap e17: y(active, starting, since 0.0126415s), standbys: x 2024-09-12T23:34:08.266 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:07 smithi181 bash[24776]: audit 2024-09-12T23:34:07.911762+0000 mon.a (mon.0) 708 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:34:08.266 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:07 smithi181 bash[24776]: audit 2024-09-12T23:34:07.911925+0000 mon.a (mon.0) 709 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:34:08.266 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:07 smithi181 bash[24776]: audit 2024-09-12T23:34:07.912173+0000 mon.a (mon.0) 710 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:34:08.266 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:07 smithi181 bash[24776]: audit 2024-09-12T23:34:07.912460+0000 mon.a (mon.0) 711 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-12T23:34:08.267 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:07 smithi181 bash[24776]: audit 2024-09-12T23:34:07.913241+0000 mon.a (mon.0) 712 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-12T23:34:08.267 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:07 smithi181 bash[24776]: audit 2024-09-12T23:34:07.913892+0000 mon.a (mon.0) 713 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-12T23:34:08.267 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:07 smithi181 bash[24776]: audit 2024-09-12T23:34:07.914222+0000 mon.a (mon.0) 714 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-12T23:34:08.267 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:07 smithi181 bash[24776]: audit 2024-09-12T23:34:07.914565+0000 mon.a (mon.0) 715 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-12T23:34:08.267 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:07 smithi181 bash[24776]: audit 2024-09-12T23:34:07.914855+0000 mon.a (mon.0) 716 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-12T23:34:08.267 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:07 smithi181 bash[24776]: audit 2024-09-12T23:34:07.915117+0000 mon.a (mon.0) 717 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-12T23:34:08.267 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:07 smithi181 bash[24776]: audit 2024-09-12T23:34:07.915347+0000 mon.a (mon.0) 718 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-12T23:34:08.267 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:07 smithi181 bash[24776]: audit 2024-09-12T23:34:07.915700+0000 mon.a (mon.0) 719 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-12T23:34:08.267 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:07 smithi181 bash[24776]: audit 2024-09-12T23:34:07.915964+0000 mon.a (mon.0) 720 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-12T23:34:08.267 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:07 smithi181 bash[24776]: audit 2024-09-12T23:34:07.916195+0000 mon.a (mon.0) 721 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-12T23:34:08.267 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:07 smithi181 bash[24776]: audit 2024-09-12T23:34:07.916428+0000 mon.a (mon.0) 722 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-12T23:34:08.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:07 smithi181 bash[24776]: audit 2024-09-12T23:34:07.916691+0000 mon.a (mon.0) 723 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-12T23:34:08.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:07 smithi181 bash[24776]: audit 2024-09-12T23:34:07.916964+0000 mon.a (mon.0) 724 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-12T23:34:08.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:07 smithi181 bash[24776]: audit 2024-09-12T23:34:07.917533+0000 mon.a (mon.0) 725 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-12T23:34:08.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:07 smithi181 bash[24776]: cluster 2024-09-12T23:34:07.942904+0000 mon.a (mon.0) 726 : cluster [INF] Manager daemon y is now available 2024-09-12T23:34:08.506 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:08 smithi136 bash[22852]: [12/Sep/2024:23:34:08] ENGINE Bus STARTED 2024-09-12T23:34:08.937 INFO:teuthology.orchestra.run.smithi181.stdout:Scheduled alertmanager update... 2024-09-12T23:34:09.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:08 smithi136 bash[22601]: audit 2024-09-12T23:34:07.963951+0000 mon.a (mon.0) 727 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:34:09.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:08 smithi136 bash[22601]: cluster 2024-09-12T23:34:07.974759+0000 mon.a (mon.0) 728 : cluster [DBG] Standby manager daemon x restarted 2024-09-12T23:34:09.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:08 smithi136 bash[22601]: cluster 2024-09-12T23:34:07.974869+0000 mon.a (mon.0) 729 : cluster [DBG] Standby manager daemon x started 2024-09-12T23:34:09.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:08 smithi136 bash[22601]: audit 2024-09-12T23:34:07.979302+0000 mon.c (mon.2) 25 : audit [DBG] from='mgr.? 172.21.15.181:0/2296498307' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-12T23:34:09.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:08 smithi136 bash[22601]: audit 2024-09-12T23:34:07.982027+0000 mon.a (mon.0) 730 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:09.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:08 smithi136 bash[22601]: audit 2024-09-12T23:34:07.984049+0000 mon.c (mon.2) 26 : audit [DBG] from='mgr.? 172.21.15.181:0/2296498307' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-12T23:34:09.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:08 smithi136 bash[22601]: audit 2024-09-12T23:34:07.986141+0000 mon.c (mon.2) 27 : audit [DBG] from='mgr.? 172.21.15.181:0/2296498307' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-12T23:34:09.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:08 smithi136 bash[22601]: audit 2024-09-12T23:34:07.986578+0000 mon.c (mon.2) 28 : audit [DBG] from='mgr.? 172.21.15.181:0/2296498307' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-12T23:34:09.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:08 smithi136 bash[22601]: audit 2024-09-12T23:34:07.987461+0000 mon.a (mon.0) 731 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:34:09.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:08 smithi136 bash[22601]: audit 2024-09-12T23:34:07.998604+0000 mon.a (mon.0) 732 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-12T23:34:09.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:08 smithi136 bash[22601]: audit 2024-09-12T23:34:08.005209+0000 mon.a (mon.0) 733 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-12T23:34:09.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:08 smithi136 bash[22601]: cluster 2024-09-12T23:34:08.915216+0000 mon.a (mon.0) 734 : cluster [DBG] mgrmap e18: y(active, since 1.0224s), standbys: x 2024-09-12T23:34:09.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:08 smithi136 bash[22601]: audit 2024-09-12T23:34:08.939665+0000 mon.a (mon.0) 735 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:09.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:08 smithi136 bash[32189]: audit 2024-09-12T23:34:07.963951+0000 mon.a (mon.0) 727 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:34:09.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:08 smithi136 bash[32189]: cluster 2024-09-12T23:34:07.974759+0000 mon.a (mon.0) 728 : cluster [DBG] Standby manager daemon x restarted 2024-09-12T23:34:09.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:08 smithi136 bash[32189]: cluster 2024-09-12T23:34:07.974869+0000 mon.a (mon.0) 729 : cluster [DBG] Standby manager daemon x started 2024-09-12T23:34:09.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:08 smithi136 bash[32189]: audit 2024-09-12T23:34:07.979302+0000 mon.c (mon.2) 25 : audit [DBG] from='mgr.? 172.21.15.181:0/2296498307' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-12T23:34:09.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:08 smithi136 bash[32189]: audit 2024-09-12T23:34:07.982027+0000 mon.a (mon.0) 730 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:09.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:08 smithi136 bash[32189]: audit 2024-09-12T23:34:07.984049+0000 mon.c (mon.2) 26 : audit [DBG] from='mgr.? 172.21.15.181:0/2296498307' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-12T23:34:09.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:08 smithi136 bash[32189]: audit 2024-09-12T23:34:07.986141+0000 mon.c (mon.2) 27 : audit [DBG] from='mgr.? 172.21.15.181:0/2296498307' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-12T23:34:09.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:08 smithi136 bash[32189]: audit 2024-09-12T23:34:07.986578+0000 mon.c (mon.2) 28 : audit [DBG] from='mgr.? 172.21.15.181:0/2296498307' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-12T23:34:09.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:08 smithi136 bash[32189]: audit 2024-09-12T23:34:07.987461+0000 mon.a (mon.0) 731 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:34:09.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:08 smithi136 bash[32189]: audit 2024-09-12T23:34:07.998604+0000 mon.a (mon.0) 732 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-12T23:34:09.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:08 smithi136 bash[32189]: audit 2024-09-12T23:34:08.005209+0000 mon.a (mon.0) 733 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-12T23:34:09.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:08 smithi136 bash[32189]: cluster 2024-09-12T23:34:08.915216+0000 mon.a (mon.0) 734 : cluster [DBG] mgrmap e18: y(active, since 1.0224s), standbys: x 2024-09-12T23:34:09.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:08 smithi136 bash[32189]: audit 2024-09-12T23:34:08.939665+0000 mon.a (mon.0) 735 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:09.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:08 smithi181 bash[24776]: audit 2024-09-12T23:34:07.963951+0000 mon.a (mon.0) 727 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:34:09.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:08 smithi181 bash[24776]: cluster 2024-09-12T23:34:07.974759+0000 mon.a (mon.0) 728 : cluster [DBG] Standby manager daemon x restarted 2024-09-12T23:34:09.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:08 smithi181 bash[24776]: cluster 2024-09-12T23:34:07.974869+0000 mon.a (mon.0) 729 : cluster [DBG] Standby manager daemon x started 2024-09-12T23:34:09.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:08 smithi181 bash[24776]: audit 2024-09-12T23:34:07.979302+0000 mon.c (mon.2) 25 : audit [DBG] from='mgr.? 172.21.15.181:0/2296498307' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-12T23:34:09.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:08 smithi181 bash[24776]: audit 2024-09-12T23:34:07.982027+0000 mon.a (mon.0) 730 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:09.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:08 smithi181 bash[24776]: audit 2024-09-12T23:34:07.984049+0000 mon.c (mon.2) 26 : audit [DBG] from='mgr.? 172.21.15.181:0/2296498307' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-12T23:34:09.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:08 smithi181 bash[24776]: audit 2024-09-12T23:34:07.986141+0000 mon.c (mon.2) 27 : audit [DBG] from='mgr.? 172.21.15.181:0/2296498307' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-12T23:34:09.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:08 smithi181 bash[24776]: audit 2024-09-12T23:34:07.986578+0000 mon.c (mon.2) 28 : audit [DBG] from='mgr.? 172.21.15.181:0/2296498307' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-12T23:34:09.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:08 smithi181 bash[24776]: audit 2024-09-12T23:34:07.987461+0000 mon.a (mon.0) 731 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:34:09.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:08 smithi181 bash[24776]: audit 2024-09-12T23:34:07.998604+0000 mon.a (mon.0) 732 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-12T23:34:09.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:08 smithi181 bash[24776]: audit 2024-09-12T23:34:08.005209+0000 mon.a (mon.0) 733 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-12T23:34:09.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:08 smithi181 bash[24776]: cluster 2024-09-12T23:34:08.915216+0000 mon.a (mon.0) 734 : cluster [DBG] mgrmap e18: y(active, since 1.0224s), standbys: x 2024-09-12T23:34:09.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:08 smithi181 bash[24776]: audit 2024-09-12T23:34:08.939665+0000 mon.a (mon.0) 735 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:09.648 DEBUG:teuthology.orchestra.run.smithi136:alertmanager.a> sudo journalctl -f -n 0 -u ceph-da58a47c-715d-11ef-bceb-c7b262605968@alertmanager.a.service 2024-09-12T23:34:09.651 INFO:tasks.cephadm:Adding grafana.a on smithi181 2024-09-12T23:34:09.651 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph orch apply grafana '1;smithi181=a' 2024-09-12T23:34:10.006 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:09 smithi136 bash[22852]: debug 2024-09-12T23:34:09.588+0000 7f48b2ae5640 -1 log_channel(cephadm) log [ERR] : [12/Sep/2024:23:34:09] ENGINE Error in HTTPServer.serve 2024-09-12T23:34:10.006 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:09 smithi136 bash[22852]: Traceback (most recent call last): 2024-09-12T23:34:10.006 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:09 smithi136 bash[22852]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-12T23:34:10.007 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:09 smithi136 bash[22852]: self._connections.run(self.expiration_interval) 2024-09-12T23:34:10.007 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:09 smithi136 bash[22852]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-12T23:34:10.007 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:09 smithi136 bash[22852]: self._run(expiration_interval) 2024-09-12T23:34:10.007 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:09 smithi136 bash[22852]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-12T23:34:10.007 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:09 smithi136 bash[22852]: new_conn = self._from_server_socket(self.server.socket) 2024-09-12T23:34:10.007 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:09 smithi136 bash[22852]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-12T23:34:10.007 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:09 smithi136 bash[22852]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-12T23:34:10.007 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:09 smithi136 bash[22852]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-12T23:34:10.007 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:09 smithi136 bash[22852]: s = self.context.wrap_socket( 2024-09-12T23:34:10.007 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:09 smithi136 bash[22852]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-12T23:34:10.007 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:09 smithi136 bash[22852]: return self.sslsocket_class._create( 2024-09-12T23:34:10.007 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:09 smithi136 bash[22852]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-12T23:34:10.007 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:09 smithi136 bash[22852]: self.do_handshake() 2024-09-12T23:34:10.007 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:09 smithi136 bash[22852]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-12T23:34:10.008 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:09 smithi136 bash[22852]: self._sslobj.do_handshake() 2024-09-12T23:34:10.008 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:09 smithi136 bash[22852]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-12T23:34:10.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:10 smithi181 bash[24776]: cephadm 2024-09-12T23:34:08.934707+0000 mgr.y (mgr.24446) 3 : cephadm [INF] Saving service alertmanager spec with placement smithi136=a;count:1 2024-09-12T23:34:10.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:10 smithi181 bash[24776]: cephadm 2024-09-12T23:34:09.469520+0000 mgr.y (mgr.24446) 4 : cephadm [INF] [12/Sep/2024:23:34:09] ENGINE Bus STARTING 2024-09-12T23:34:10.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:10 smithi181 bash[24776]: cephadm 2024-09-12T23:34:09.590030+0000 mgr.y (mgr.24446) 5 : cephadm [INF] [12/Sep/2024:23:34:09] ENGINE Serving on https://172.21.15.136:7150 2024-09-12T23:34:10.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:10 smithi181 bash[24776]: cephadm 2024-09-12T23:34:09.590117+0000 mgr.y (mgr.24446) 6 : cephadm [INF] [12/Sep/2024:23:34:09] ENGINE Bus STARTED 2024-09-12T23:34:10.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:10 smithi181 bash[24776]: cephadm 2024-09-12T23:34:09.591995+0000 mgr.y (mgr.24446) 7 : cephadm [ERR] [12/Sep/2024:23:34:09] ENGINE Error in HTTPServer.serve 2024-09-12T23:34:10.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:10 smithi181 bash[24776]: Traceback (most recent call last): 2024-09-12T23:34:10.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:10 smithi181 bash[24776]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-12T23:34:10.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:10 smithi181 bash[24776]: self._connections.run(self.expiration_interval) 2024-09-12T23:34:10.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:10 smithi181 bash[24776]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-12T23:34:10.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:10 smithi181 bash[24776]: self._run(expiration_interval) 2024-09-12T23:34:10.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:10 smithi181 bash[24776]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-12T23:34:10.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:10 smithi181 bash[24776]: new_conn = self._from_server_socket(self.server.socket) 2024-09-12T23:34:10.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:10 smithi181 bash[24776]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-12T23:34:10.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:10 smithi181 bash[24776]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-12T23:34:10.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:10 smithi181 bash[24776]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-12T23:34:10.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:10 smithi181 bash[24776]: s = self.context.wrap_socket( 2024-09-12T23:34:10.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:10 smithi181 bash[24776]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-12T23:34:10.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:10 smithi181 bash[24776]: return self.sslsocket_class._create( 2024-09-12T23:34:10.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:10 smithi181 bash[24776]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-12T23:34:10.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:10 smithi181 bash[24776]: self.do_handshake() 2024-09-12T23:34:10.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:10 smithi181 bash[24776]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-12T23:34:10.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:10 smithi181 bash[24776]: self._sslobj.do_handshake() 2024-09-12T23:34:10.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:10 smithi181 bash[24776]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-12T23:34:10.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:10 smithi181 bash[24776]: audit 2024-09-12T23:34:09.595947+0000 mon.a (mon.0) 736 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:11.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:10 smithi136 bash[22601]: cephadm 2024-09-12T23:34:08.934707+0000 mgr.y (mgr.24446) 3 : cephadm [INF] Saving service alertmanager spec with placement smithi136=a;count:1 2024-09-12T23:34:11.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:10 smithi136 bash[22601]: cephadm 2024-09-12T23:34:09.469520+0000 mgr.y (mgr.24446) 4 : cephadm [INF] [12/Sep/2024:23:34:09] ENGINE Bus STARTING 2024-09-12T23:34:11.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:10 smithi136 bash[22601]: cephadm 2024-09-12T23:34:09.590030+0000 mgr.y (mgr.24446) 5 : cephadm [INF] [12/Sep/2024:23:34:09] ENGINE Serving on https://172.21.15.136:7150 2024-09-12T23:34:11.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:10 smithi136 bash[22601]: cephadm 2024-09-12T23:34:09.590117+0000 mgr.y (mgr.24446) 6 : cephadm [INF] [12/Sep/2024:23:34:09] ENGINE Bus STARTED 2024-09-12T23:34:11.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:10 smithi136 bash[22601]: cephadm 2024-09-12T23:34:09.591995+0000 mgr.y (mgr.24446) 7 : cephadm [ERR] [12/Sep/2024:23:34:09] ENGINE Error in HTTPServer.serve 2024-09-12T23:34:11.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:10 smithi136 bash[22601]: Traceback (most recent call last): 2024-09-12T23:34:11.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:10 smithi136 bash[22601]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-12T23:34:11.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:10 smithi136 bash[22601]: self._connections.run(self.expiration_interval) 2024-09-12T23:34:11.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:10 smithi136 bash[22601]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-12T23:34:11.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:10 smithi136 bash[22601]: self._run(expiration_interval) 2024-09-12T23:34:11.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:10 smithi136 bash[22601]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-12T23:34:11.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:10 smithi136 bash[22601]: new_conn = self._from_server_socket(self.server.socket) 2024-09-12T23:34:11.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:10 smithi136 bash[22601]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-12T23:34:11.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:10 smithi136 bash[22601]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-12T23:34:11.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:10 smithi136 bash[22601]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-12T23:34:11.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:10 smithi136 bash[22601]: s = self.context.wrap_socket( 2024-09-12T23:34:11.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:10 smithi136 bash[22601]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-12T23:34:11.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:10 smithi136 bash[22601]: return self.sslsocket_class._create( 2024-09-12T23:34:11.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:10 smithi136 bash[22601]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-12T23:34:11.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:10 smithi136 bash[22601]: self.do_handshake() 2024-09-12T23:34:11.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:10 smithi136 bash[22601]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-12T23:34:11.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:10 smithi136 bash[22601]: self._sslobj.do_handshake() 2024-09-12T23:34:11.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:10 smithi136 bash[22601]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-12T23:34:11.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:10 smithi136 bash[22601]: audit 2024-09-12T23:34:09.595947+0000 mon.a (mon.0) 736 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:11.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:10 smithi136 bash[32189]: cephadm 2024-09-12T23:34:08.934707+0000 mgr.y (mgr.24446) 3 : cephadm [INF] Saving service alertmanager spec with placement smithi136=a;count:1 2024-09-12T23:34:11.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:10 smithi136 bash[32189]: cephadm 2024-09-12T23:34:09.469520+0000 mgr.y (mgr.24446) 4 : cephadm [INF] [12/Sep/2024:23:34:09] ENGINE Bus STARTING 2024-09-12T23:34:11.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:10 smithi136 bash[32189]: cephadm 2024-09-12T23:34:09.590030+0000 mgr.y (mgr.24446) 5 : cephadm [INF] [12/Sep/2024:23:34:09] ENGINE Serving on https://172.21.15.136:7150 2024-09-12T23:34:11.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:10 smithi136 bash[32189]: cephadm 2024-09-12T23:34:09.590117+0000 mgr.y (mgr.24446) 6 : cephadm [INF] [12/Sep/2024:23:34:09] ENGINE Bus STARTED 2024-09-12T23:34:11.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:10 smithi136 bash[32189]: cephadm 2024-09-12T23:34:09.591995+0000 mgr.y (mgr.24446) 7 : cephadm [ERR] [12/Sep/2024:23:34:09] ENGINE Error in HTTPServer.serve 2024-09-12T23:34:11.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:10 smithi136 bash[32189]: Traceback (most recent call last): 2024-09-12T23:34:11.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:10 smithi136 bash[32189]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-12T23:34:11.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:10 smithi136 bash[32189]: self._connections.run(self.expiration_interval) 2024-09-12T23:34:11.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:10 smithi136 bash[32189]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-12T23:34:11.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:10 smithi136 bash[32189]: self._run(expiration_interval) 2024-09-12T23:34:11.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:10 smithi136 bash[32189]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-12T23:34:11.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:10 smithi136 bash[32189]: new_conn = self._from_server_socket(self.server.socket) 2024-09-12T23:34:11.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:10 smithi136 bash[32189]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-12T23:34:11.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:10 smithi136 bash[32189]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-12T23:34:11.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:10 smithi136 bash[32189]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-12T23:34:11.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:10 smithi136 bash[32189]: s = self.context.wrap_socket( 2024-09-12T23:34:11.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:10 smithi136 bash[32189]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-12T23:34:11.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:10 smithi136 bash[32189]: return self.sslsocket_class._create( 2024-09-12T23:34:11.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:10 smithi136 bash[32189]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-12T23:34:11.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:10 smithi136 bash[32189]: self.do_handshake() 2024-09-12T23:34:11.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:10 smithi136 bash[32189]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-12T23:34:11.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:10 smithi136 bash[32189]: self._sslobj.do_handshake() 2024-09-12T23:34:11.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:10 smithi136 bash[32189]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-12T23:34:11.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:10 smithi136 bash[32189]: audit 2024-09-12T23:34:09.595947+0000 mon.a (mon.0) 736 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:11.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:11 smithi181 bash[24776]: cluster 2024-09-12T23:34:09.916082+0000 mgr.y (mgr.24446) 8 : cluster [DBG] pgmap v4: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:11.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:11 smithi181 bash[24776]: cluster 2024-09-12T23:34:10.932885+0000 mon.a (mon.0) 737 : cluster [DBG] mgrmap e19: y(active, since 3s), standbys: x 2024-09-12T23:34:12.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:11 smithi136 bash[22601]: cluster 2024-09-12T23:34:09.916082+0000 mgr.y (mgr.24446) 8 : cluster [DBG] pgmap v4: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:12.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:11 smithi136 bash[22601]: cluster 2024-09-12T23:34:10.932885+0000 mon.a (mon.0) 737 : cluster [DBG] mgrmap e19: y(active, since 3s), standbys: x 2024-09-12T23:34:12.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:11 smithi136 bash[32189]: cluster 2024-09-12T23:34:09.916082+0000 mgr.y (mgr.24446) 8 : cluster [DBG] pgmap v4: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:12.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:11 smithi136 bash[32189]: cluster 2024-09-12T23:34:10.932885+0000 mon.a (mon.0) 737 : cluster [DBG] mgrmap e19: y(active, since 3s), standbys: x 2024-09-12T23:34:14.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:13 smithi136 bash[22601]: cluster 2024-09-12T23:34:11.916814+0000 mgr.y (mgr.24446) 9 : cluster [DBG] pgmap v5: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:14.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:13 smithi136 bash[22601]: cluster 2024-09-12T23:34:12.619140+0000 mon.a (mon.0) 738 : cluster [DBG] mgrmap e20: y(active, since 4s), standbys: x 2024-09-12T23:34:14.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:13 smithi136 bash[22601]: audit 2024-09-12T23:34:13.480298+0000 mon.a (mon.0) 739 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:14.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:13 smithi136 bash[22601]: audit 2024-09-12T23:34:13.485266+0000 mon.a (mon.0) 740 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:14.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:13 smithi136 bash[32189]: cluster 2024-09-12T23:34:11.916814+0000 mgr.y (mgr.24446) 9 : cluster [DBG] pgmap v5: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:14.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:13 smithi136 bash[32189]: cluster 2024-09-12T23:34:12.619140+0000 mon.a (mon.0) 738 : cluster [DBG] mgrmap e20: y(active, since 4s), standbys: x 2024-09-12T23:34:14.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:13 smithi136 bash[32189]: audit 2024-09-12T23:34:13.480298+0000 mon.a (mon.0) 739 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:14.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:13 smithi136 bash[32189]: audit 2024-09-12T23:34:13.485266+0000 mon.a (mon.0) 740 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:14.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:13 smithi181 bash[24776]: cluster 2024-09-12T23:34:11.916814+0000 mgr.y (mgr.24446) 9 : cluster [DBG] pgmap v5: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:14.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:13 smithi181 bash[24776]: cluster 2024-09-12T23:34:12.619140+0000 mon.a (mon.0) 738 : cluster [DBG] mgrmap e20: y(active, since 4s), standbys: x 2024-09-12T23:34:14.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:13 smithi181 bash[24776]: audit 2024-09-12T23:34:13.480298+0000 mon.a (mon.0) 739 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:14.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:13 smithi181 bash[24776]: audit 2024-09-12T23:34:13.485266+0000 mon.a (mon.0) 740 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:14.195 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.b/config 2024-09-12T23:34:15.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:14 smithi136 bash[22601]: audit 2024-09-12T23:34:13.661839+0000 mon.a (mon.0) 741 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:15.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:14 smithi136 bash[22601]: audit 2024-09-12T23:34:13.668420+0000 mon.a (mon.0) 742 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:15.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:14 smithi136 bash[22601]: audit 2024-09-12T23:34:14.086506+0000 mon.a (mon.0) 743 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:15.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:14 smithi136 bash[22601]: audit 2024-09-12T23:34:14.092260+0000 mon.a (mon.0) 744 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:15.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:14 smithi136 bash[22601]: audit 2024-09-12T23:34:14.093669+0000 mon.a (mon.0) 745 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:34:15.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:14 smithi136 bash[22601]: audit 2024-09-12T23:34:14.094510+0000 mon.a (mon.0) 746 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:34:15.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:14 smithi136 bash[22601]: audit 2024-09-12T23:34:14.095271+0000 mon.a (mon.0) 747 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:34:15.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:14 smithi136 bash[22601]: audit 2024-09-12T23:34:14.096029+0000 mon.a (mon.0) 748 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:34:15.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:14 smithi136 bash[22601]: audit 2024-09-12T23:34:14.102583+0000 mon.a (mon.0) 749 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:15.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:14 smithi136 bash[22601]: audit 2024-09-12T23:34:14.247964+0000 mon.a (mon.0) 750 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:15.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:14 smithi136 bash[22601]: audit 2024-09-12T23:34:14.257038+0000 mon.a (mon.0) 751 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:15.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:14 smithi136 bash[22601]: audit 2024-09-12T23:34:14.259646+0000 mon.a (mon.0) 752 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:34:15.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:14 smithi136 bash[22601]: audit 2024-09-12T23:34:14.261447+0000 mon.a (mon.0) 753 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:34:15.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:14 smithi136 bash[22601]: audit 2024-09-12T23:34:14.263109+0000 mon.a (mon.0) 754 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:34:15.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:14 smithi136 bash[22601]: audit 2024-09-12T23:34:14.264799+0000 mon.a (mon.0) 755 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:34:15.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:14 smithi136 bash[22601]: audit 2024-09-12T23:34:14.274057+0000 mon.a (mon.0) 756 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:15.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:14 smithi136 bash[22601]: audit 2024-09-12T23:34:14.277020+0000 mon.a (mon.0) 757 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:34:15.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:14 smithi136 bash[22601]: audit 2024-09-12T23:34:14.278561+0000 mon.a (mon.0) 758 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:34:15.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:14 smithi136 bash[32189]: audit 2024-09-12T23:34:13.661839+0000 mon.a (mon.0) 741 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:15.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:14 smithi136 bash[32189]: audit 2024-09-12T23:34:13.668420+0000 mon.a (mon.0) 742 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:15.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:14 smithi136 bash[32189]: audit 2024-09-12T23:34:14.086506+0000 mon.a (mon.0) 743 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:15.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:14 smithi136 bash[32189]: audit 2024-09-12T23:34:14.092260+0000 mon.a (mon.0) 744 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:15.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:14 smithi136 bash[32189]: audit 2024-09-12T23:34:14.093669+0000 mon.a (mon.0) 745 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:34:15.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:14 smithi136 bash[32189]: audit 2024-09-12T23:34:14.094510+0000 mon.a (mon.0) 746 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:34:15.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:14 smithi136 bash[32189]: audit 2024-09-12T23:34:14.095271+0000 mon.a (mon.0) 747 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:34:15.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:14 smithi136 bash[32189]: audit 2024-09-12T23:34:14.096029+0000 mon.a (mon.0) 748 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:34:15.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:14 smithi136 bash[32189]: audit 2024-09-12T23:34:14.102583+0000 mon.a (mon.0) 749 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:15.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:14 smithi136 bash[32189]: audit 2024-09-12T23:34:14.247964+0000 mon.a (mon.0) 750 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:15.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:14 smithi136 bash[32189]: audit 2024-09-12T23:34:14.257038+0000 mon.a (mon.0) 751 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:15.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:14 smithi136 bash[32189]: audit 2024-09-12T23:34:14.259646+0000 mon.a (mon.0) 752 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:34:15.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:14 smithi136 bash[32189]: audit 2024-09-12T23:34:14.261447+0000 mon.a (mon.0) 753 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:34:15.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:14 smithi136 bash[32189]: audit 2024-09-12T23:34:14.263109+0000 mon.a (mon.0) 754 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:34:15.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:14 smithi136 bash[32189]: audit 2024-09-12T23:34:14.264799+0000 mon.a (mon.0) 755 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:34:15.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:14 smithi136 bash[32189]: audit 2024-09-12T23:34:14.274057+0000 mon.a (mon.0) 756 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:15.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:14 smithi136 bash[32189]: audit 2024-09-12T23:34:14.277020+0000 mon.a (mon.0) 757 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:34:15.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:14 smithi136 bash[32189]: audit 2024-09-12T23:34:14.278561+0000 mon.a (mon.0) 758 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:34:15.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:14 smithi181 bash[24776]: audit 2024-09-12T23:34:13.661839+0000 mon.a (mon.0) 741 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:15.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:14 smithi181 bash[24776]: audit 2024-09-12T23:34:13.668420+0000 mon.a (mon.0) 742 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:15.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:14 smithi181 bash[24776]: audit 2024-09-12T23:34:14.086506+0000 mon.a (mon.0) 743 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:15.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:14 smithi181 bash[24776]: audit 2024-09-12T23:34:14.092260+0000 mon.a (mon.0) 744 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:15.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:14 smithi181 bash[24776]: audit 2024-09-12T23:34:14.093669+0000 mon.a (mon.0) 745 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:34:15.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:14 smithi181 bash[24776]: audit 2024-09-12T23:34:14.094510+0000 mon.a (mon.0) 746 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:34:15.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:14 smithi181 bash[24776]: audit 2024-09-12T23:34:14.095271+0000 mon.a (mon.0) 747 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:34:15.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:14 smithi181 bash[24776]: audit 2024-09-12T23:34:14.096029+0000 mon.a (mon.0) 748 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:34:15.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:14 smithi181 bash[24776]: audit 2024-09-12T23:34:14.102583+0000 mon.a (mon.0) 749 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:15.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:14 smithi181 bash[24776]: audit 2024-09-12T23:34:14.247964+0000 mon.a (mon.0) 750 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:15.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:14 smithi181 bash[24776]: audit 2024-09-12T23:34:14.257038+0000 mon.a (mon.0) 751 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:15.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:14 smithi181 bash[24776]: audit 2024-09-12T23:34:14.259646+0000 mon.a (mon.0) 752 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:34:15.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:14 smithi181 bash[24776]: audit 2024-09-12T23:34:14.261447+0000 mon.a (mon.0) 753 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:34:15.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:14 smithi181 bash[24776]: audit 2024-09-12T23:34:14.263109+0000 mon.a (mon.0) 754 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:34:15.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:14 smithi181 bash[24776]: audit 2024-09-12T23:34:14.264799+0000 mon.a (mon.0) 755 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:34:15.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:14 smithi181 bash[24776]: audit 2024-09-12T23:34:14.274057+0000 mon.a (mon.0) 756 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:15.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:14 smithi181 bash[24776]: audit 2024-09-12T23:34:14.277020+0000 mon.a (mon.0) 757 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:34:15.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:14 smithi181 bash[24776]: audit 2024-09-12T23:34:14.278561+0000 mon.a (mon.0) 758 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:34:15.394 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:15 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:15.395 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 12 23:34:15 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:15.395 INFO:journalctl@ceph.mds.a.smithi136.stdout:Sep 12 23:34:15 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:15.395 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:15 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:15.396 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:15 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:15.396 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 12 23:34:15 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:15.396 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 12 23:34:15 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:15.397 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 12 23:34:15 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:15.402 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:34:15 smithi181 bash[25717]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:34:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-12T23:34:15.653 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:15 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:15.654 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:15 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:15.654 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 12 23:34:15 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:15.654 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 12 23:34:15 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:15.654 INFO:journalctl@ceph.mds.a.smithi136.stdout:Sep 12 23:34:15 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:15.655 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:15 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:15.655 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 12 23:34:15 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:15.655 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 12 23:34:15 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:15.656 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:15 smithi136 systemd[1]: Started Ceph node-exporter.a for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:34:16.006 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:15 smithi136 bash[57913]: Unable to find image 'quay.io/prometheus/node-exporter:v1.5.0' locally 2024-09-12T23:34:16.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:15 smithi136 bash[22601]: cluster 2024-09-12T23:34:13.917391+0000 mgr.y (mgr.24446) 10 : cluster [DBG] pgmap v6: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:16.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:15 smithi136 bash[22601]: cephadm 2024-09-12T23:34:14.096664+0000 mgr.y (mgr.24446) 11 : cephadm [INF] Adjusting osd_memory_target on smithi136 to 3036M 2024-09-12T23:34:16.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:15 smithi136 bash[22601]: cephadm 2024-09-12T23:34:14.266072+0000 mgr.y (mgr.24446) 12 : cephadm [INF] Adjusting osd_memory_target on smithi181 to 3036M 2024-09-12T23:34:16.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:15 smithi136 bash[22601]: cephadm 2024-09-12T23:34:14.280757+0000 mgr.y (mgr.24446) 13 : cephadm [INF] Updating smithi136:/etc/ceph/ceph.conf 2024-09-12T23:34:16.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:15 smithi136 bash[22601]: cephadm 2024-09-12T23:34:14.281245+0000 mgr.y (mgr.24446) 14 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.conf 2024-09-12T23:34:16.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:15 smithi136 bash[22601]: cephadm 2024-09-12T23:34:14.445792+0000 mgr.y (mgr.24446) 15 : cephadm [INF] Updating smithi136:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-12T23:34:16.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:15 smithi136 bash[22601]: cephadm 2024-09-12T23:34:14.542749+0000 mgr.y (mgr.24446) 16 : cephadm [INF] Updating smithi181:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-12T23:34:16.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:15 smithi136 bash[22601]: cephadm 2024-09-12T23:34:14.546670+0000 mgr.y (mgr.24446) 17 : cephadm [INF] Updating smithi136:/etc/ceph/ceph.client.admin.keyring 2024-09-12T23:34:16.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:15 smithi136 bash[22601]: audit 2024-09-12T23:34:14.814046+0000 mon.a (mon.0) 759 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:16.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:15 smithi136 bash[22601]: audit 2024-09-12T23:34:14.821978+0000 mon.a (mon.0) 760 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:16.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:15 smithi136 bash[22601]: audit 2024-09-12T23:34:14.916479+0000 mon.a (mon.0) 761 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:16.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:15 smithi136 bash[22601]: audit 2024-09-12T23:34:14.925005+0000 mon.a (mon.0) 762 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:16.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:15 smithi136 bash[22601]: audit 2024-09-12T23:34:14.934071+0000 mon.a (mon.0) 763 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:16.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:15 smithi136 bash[22601]: audit 2024-09-12T23:34:15.650551+0000 mon.a (mon.0) 764 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:16.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:15 smithi136 bash[22601]: audit 2024-09-12T23:34:15.656777+0000 mon.a (mon.0) 765 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:16.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:15 smithi136 bash[32189]: cluster 2024-09-12T23:34:13.917391+0000 mgr.y (mgr.24446) 10 : cluster [DBG] pgmap v6: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:16.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:15 smithi136 bash[32189]: cephadm 2024-09-12T23:34:14.096664+0000 mgr.y (mgr.24446) 11 : cephadm [INF] Adjusting osd_memory_target on smithi136 to 3036M 2024-09-12T23:34:16.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:15 smithi136 bash[32189]: cephadm 2024-09-12T23:34:14.266072+0000 mgr.y (mgr.24446) 12 : cephadm [INF] Adjusting osd_memory_target on smithi181 to 3036M 2024-09-12T23:34:16.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:15 smithi136 bash[32189]: cephadm 2024-09-12T23:34:14.280757+0000 mgr.y (mgr.24446) 13 : cephadm [INF] Updating smithi136:/etc/ceph/ceph.conf 2024-09-12T23:34:16.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:15 smithi136 bash[32189]: cephadm 2024-09-12T23:34:14.281245+0000 mgr.y (mgr.24446) 14 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.conf 2024-09-12T23:34:16.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:15 smithi136 bash[32189]: cephadm 2024-09-12T23:34:14.445792+0000 mgr.y (mgr.24446) 15 : cephadm [INF] Updating smithi136:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-12T23:34:16.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:15 smithi136 bash[32189]: cephadm 2024-09-12T23:34:14.542749+0000 mgr.y (mgr.24446) 16 : cephadm [INF] Updating smithi181:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-12T23:34:16.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:15 smithi136 bash[32189]: cephadm 2024-09-12T23:34:14.546670+0000 mgr.y (mgr.24446) 17 : cephadm [INF] Updating smithi136:/etc/ceph/ceph.client.admin.keyring 2024-09-12T23:34:16.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:15 smithi136 bash[32189]: audit 2024-09-12T23:34:14.814046+0000 mon.a (mon.0) 759 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:16.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:15 smithi136 bash[32189]: audit 2024-09-12T23:34:14.821978+0000 mon.a (mon.0) 760 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:16.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:15 smithi136 bash[32189]: audit 2024-09-12T23:34:14.916479+0000 mon.a (mon.0) 761 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:16.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:15 smithi136 bash[32189]: audit 2024-09-12T23:34:14.925005+0000 mon.a (mon.0) 762 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:16.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:15 smithi136 bash[32189]: audit 2024-09-12T23:34:14.934071+0000 mon.a (mon.0) 763 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:16.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:15 smithi136 bash[32189]: audit 2024-09-12T23:34:15.650551+0000 mon.a (mon.0) 764 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:16.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:15 smithi136 bash[32189]: audit 2024-09-12T23:34:15.656777+0000 mon.a (mon.0) 765 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:16.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:15 smithi181 bash[24776]: cluster 2024-09-12T23:34:13.917391+0000 mgr.y (mgr.24446) 10 : cluster [DBG] pgmap v6: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:16.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:15 smithi181 bash[24776]: cephadm 2024-09-12T23:34:14.096664+0000 mgr.y (mgr.24446) 11 : cephadm [INF] Adjusting osd_memory_target on smithi136 to 3036M 2024-09-12T23:34:16.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:15 smithi181 bash[24776]: cephadm 2024-09-12T23:34:14.266072+0000 mgr.y (mgr.24446) 12 : cephadm [INF] Adjusting osd_memory_target on smithi181 to 3036M 2024-09-12T23:34:16.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:15 smithi181 bash[24776]: cephadm 2024-09-12T23:34:14.280757+0000 mgr.y (mgr.24446) 13 : cephadm [INF] Updating smithi136:/etc/ceph/ceph.conf 2024-09-12T23:34:16.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:15 smithi181 bash[24776]: cephadm 2024-09-12T23:34:14.281245+0000 mgr.y (mgr.24446) 14 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.conf 2024-09-12T23:34:16.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:15 smithi181 bash[24776]: cephadm 2024-09-12T23:34:14.445792+0000 mgr.y (mgr.24446) 15 : cephadm [INF] Updating smithi136:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-12T23:34:16.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:15 smithi181 bash[24776]: cephadm 2024-09-12T23:34:14.542749+0000 mgr.y (mgr.24446) 16 : cephadm [INF] Updating smithi181:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-12T23:34:16.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:15 smithi181 bash[24776]: cephadm 2024-09-12T23:34:14.546670+0000 mgr.y (mgr.24446) 17 : cephadm [INF] Updating smithi136:/etc/ceph/ceph.client.admin.keyring 2024-09-12T23:34:16.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:15 smithi181 bash[24776]: audit 2024-09-12T23:34:14.814046+0000 mon.a (mon.0) 759 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:16.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:15 smithi181 bash[24776]: audit 2024-09-12T23:34:14.821978+0000 mon.a (mon.0) 760 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:16.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:15 smithi181 bash[24776]: audit 2024-09-12T23:34:14.916479+0000 mon.a (mon.0) 761 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:16.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:15 smithi181 bash[24776]: audit 2024-09-12T23:34:14.925005+0000 mon.a (mon.0) 762 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:16.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:15 smithi181 bash[24776]: audit 2024-09-12T23:34:14.934071+0000 mon.a (mon.0) 763 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:16.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:15 smithi181 bash[24776]: audit 2024-09-12T23:34:15.650551+0000 mon.a (mon.0) 764 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:16.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:15 smithi181 bash[24776]: audit 2024-09-12T23:34:15.656777+0000 mon.a (mon.0) 765 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:16.346 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:34:16 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:16.346 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:34:16 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:16.347 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:34:16 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:16.347 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:34:16 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:16.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:16 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:16.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:16 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:16.347 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 12 23:34:16 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:16.347 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 12 23:34:16 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:16.348 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 12 23:34:16 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:16.348 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 12 23:34:16 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:16.348 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:34:16 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:16.348 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:34:16 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:16.349 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:34:16 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:16.349 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:34:16 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:16.349 INFO:journalctl@ceph.mds.b.smithi181.stdout:Sep 12 23:34:16 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:16.349 INFO:journalctl@ceph.mds.b.smithi181.stdout:Sep 12 23:34:16 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:16.506 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:16 smithi136 bash[57913]: v1.5.0: Pulling from prometheus/node-exporter 2024-09-12T23:34:16.506 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:16 smithi136 bash[57913]: 22b70bddd3ac: Pulling fs layer 2024-09-12T23:34:16.506 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:16 smithi136 bash[57913]: 5c12815fee55: Pulling fs layer 2024-09-12T23:34:16.506 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:16 smithi136 bash[57913]: c0e87333d380: Pulling fs layer 2024-09-12T23:34:16.662 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:16 smithi181 systemd[1]: Started Ceph node-exporter.b for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:34:16.663 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:16 smithi181 bash[49903]: Unable to find image 'quay.io/prometheus/node-exporter:v1.5.0' locally 2024-09-12T23:34:17.006 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:16 smithi136 bash[57913]: 5c12815fee55: Verifying Checksum 2024-09-12T23:34:17.006 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:16 smithi136 bash[57913]: 22b70bddd3ac: Verifying Checksum 2024-09-12T23:34:17.006 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:16 smithi136 bash[57913]: 22b70bddd3ac: Download complete 2024-09-12T23:34:17.006 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:16 smithi136 bash[57913]: c0e87333d380: Verifying Checksum 2024-09-12T23:34:17.006 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:16 smithi136 bash[57913]: c0e87333d380: Download complete 2024-09-12T23:34:17.007 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:16 smithi136 bash[57913]: 22b70bddd3ac: Pull complete 2024-09-12T23:34:17.007 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:34:16 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:17.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:16 smithi136 bash[22601]: cephadm 2024-09-12T23:34:14.679000+0000 mgr.y (mgr.24446) 18 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2024-09-12T23:34:17.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:16 smithi136 bash[22601]: cephadm 2024-09-12T23:34:14.679136+0000 mgr.y (mgr.24446) 19 : cephadm [INF] Updating smithi136:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-12T23:34:17.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:16 smithi136 bash[22601]: cephadm 2024-09-12T23:34:14.807226+0000 mgr.y (mgr.24446) 20 : cephadm [INF] Updating smithi181:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-12T23:34:17.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:16 smithi136 bash[22601]: cephadm 2024-09-12T23:34:14.940042+0000 mgr.y (mgr.24446) 21 : cephadm [INF] Deploying daemon node-exporter.a on smithi136 2024-09-12T23:34:17.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:16 smithi136 bash[22601]: audit 2024-09-12T23:34:15.661102+0000 mon.a (mon.0) 766 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:17.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:16 smithi136 bash[22601]: cephadm 2024-09-12T23:34:15.661852+0000 mgr.y (mgr.24446) 22 : cephadm [INF] Deploying daemon node-exporter.b on smithi181 2024-09-12T23:34:17.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:16 smithi136 bash[22601]: cluster 2024-09-12T23:34:15.918540+0000 mgr.y (mgr.24446) 23 : cluster [DBG] pgmap v7: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-12T23:34:17.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:16 smithi136 bash[22601]: audit 2024-09-12T23:34:16.396615+0000 mon.a (mon.0) 767 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:17.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:16 smithi136 bash[22601]: audit 2024-09-12T23:34:16.404309+0000 mon.a (mon.0) 768 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:17.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:16 smithi136 bash[22601]: audit 2024-09-12T23:34:16.413208+0000 mon.a (mon.0) 769 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:17.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:16 smithi136 bash[22601]: audit 2024-09-12T23:34:16.420939+0000 mon.a (mon.0) 770 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:17.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:16 smithi136 bash[22601]: audit 2024-09-12T23:34:16.435573+0000 mon.a (mon.0) 771 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:17.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:16 smithi136 bash[22601]: cephadm 2024-09-12T23:34:16.461234+0000 mgr.y (mgr.24446) 24 : cephadm [INF] Deploying daemon alertmanager.a on smithi136 2024-09-12T23:34:17.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:16 smithi136 bash[32189]: cephadm 2024-09-12T23:34:14.679000+0000 mgr.y (mgr.24446) 18 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2024-09-12T23:34:17.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:16 smithi136 bash[32189]: cephadm 2024-09-12T23:34:14.679136+0000 mgr.y (mgr.24446) 19 : cephadm [INF] Updating smithi136:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-12T23:34:17.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:16 smithi136 bash[32189]: cephadm 2024-09-12T23:34:14.807226+0000 mgr.y (mgr.24446) 20 : cephadm [INF] Updating smithi181:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-12T23:34:17.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:16 smithi136 bash[32189]: cephadm 2024-09-12T23:34:14.940042+0000 mgr.y (mgr.24446) 21 : cephadm [INF] Deploying daemon node-exporter.a on smithi136 2024-09-12T23:34:17.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:16 smithi136 bash[32189]: audit 2024-09-12T23:34:15.661102+0000 mon.a (mon.0) 766 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:17.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:16 smithi136 bash[32189]: cephadm 2024-09-12T23:34:15.661852+0000 mgr.y (mgr.24446) 22 : cephadm [INF] Deploying daemon node-exporter.b on smithi181 2024-09-12T23:34:17.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:16 smithi136 bash[32189]: cluster 2024-09-12T23:34:15.918540+0000 mgr.y (mgr.24446) 23 : cluster [DBG] pgmap v7: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-12T23:34:17.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:16 smithi136 bash[32189]: audit 2024-09-12T23:34:16.396615+0000 mon.a (mon.0) 767 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:17.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:16 smithi136 bash[32189]: audit 2024-09-12T23:34:16.404309+0000 mon.a (mon.0) 768 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:17.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:16 smithi136 bash[32189]: audit 2024-09-12T23:34:16.413208+0000 mon.a (mon.0) 769 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:17.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:16 smithi136 bash[32189]: audit 2024-09-12T23:34:16.420939+0000 mon.a (mon.0) 770 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:17.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:16 smithi136 bash[32189]: audit 2024-09-12T23:34:16.435573+0000 mon.a (mon.0) 771 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:17.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:16 smithi136 bash[32189]: cephadm 2024-09-12T23:34:16.461234+0000 mgr.y (mgr.24446) 24 : cephadm [INF] Deploying daemon alertmanager.a on smithi136 2024-09-12T23:34:17.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:16 smithi181 bash[24776]: cephadm 2024-09-12T23:34:14.679000+0000 mgr.y (mgr.24446) 18 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2024-09-12T23:34:17.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:16 smithi181 bash[24776]: cephadm 2024-09-12T23:34:14.679136+0000 mgr.y (mgr.24446) 19 : cephadm [INF] Updating smithi136:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-12T23:34:17.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:16 smithi181 bash[24776]: cephadm 2024-09-12T23:34:14.807226+0000 mgr.y (mgr.24446) 20 : cephadm [INF] Updating smithi181:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-12T23:34:17.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:16 smithi181 bash[24776]: cephadm 2024-09-12T23:34:14.940042+0000 mgr.y (mgr.24446) 21 : cephadm [INF] Deploying daemon node-exporter.a on smithi136 2024-09-12T23:34:17.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:16 smithi181 bash[24776]: audit 2024-09-12T23:34:15.661102+0000 mon.a (mon.0) 766 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:17.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:16 smithi181 bash[24776]: cephadm 2024-09-12T23:34:15.661852+0000 mgr.y (mgr.24446) 22 : cephadm [INF] Deploying daemon node-exporter.b on smithi181 2024-09-12T23:34:17.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:16 smithi181 bash[24776]: cluster 2024-09-12T23:34:15.918540+0000 mgr.y (mgr.24446) 23 : cluster [DBG] pgmap v7: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-12T23:34:17.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:16 smithi181 bash[24776]: audit 2024-09-12T23:34:16.396615+0000 mon.a (mon.0) 767 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:17.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:16 smithi181 bash[24776]: audit 2024-09-12T23:34:16.404309+0000 mon.a (mon.0) 768 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:17.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:16 smithi181 bash[24776]: audit 2024-09-12T23:34:16.413208+0000 mon.a (mon.0) 769 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:17.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:16 smithi181 bash[24776]: audit 2024-09-12T23:34:16.420939+0000 mon.a (mon.0) 770 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:17.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:16 smithi181 bash[24776]: audit 2024-09-12T23:34:16.435573+0000 mon.a (mon.0) 771 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:17.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:16 smithi181 bash[24776]: cephadm 2024-09-12T23:34:16.461234+0000 mgr.y (mgr.24446) 24 : cephadm [INF] Deploying daemon alertmanager.a on smithi136 2024-09-12T23:34:17.596 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:17 smithi181 bash[49903]: v1.5.0: Pulling from prometheus/node-exporter 2024-09-12T23:34:17.596 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:17 smithi181 bash[49903]: 22b70bddd3ac: Pulling fs layer 2024-09-12T23:34:17.596 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:17 smithi181 bash[49903]: 5c12815fee55: Pulling fs layer 2024-09-12T23:34:17.597 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:17 smithi181 bash[49903]: c0e87333d380: Pulling fs layer 2024-09-12T23:34:17.823 INFO:teuthology.orchestra.run.smithi181.stdout:Scheduled grafana update... 2024-09-12T23:34:18.006 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:17 smithi136 bash[57913]: 5c12815fee55: Pull complete 2024-09-12T23:34:18.047 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:17 smithi181 bash[49903]: 22b70bddd3ac: Verifying Checksum 2024-09-12T23:34:18.047 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:17 smithi181 bash[49903]: 22b70bddd3ac: Download complete 2024-09-12T23:34:18.047 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:17 smithi181 bash[49903]: 5c12815fee55: Verifying Checksum 2024-09-12T23:34:18.047 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:17 smithi181 bash[49903]: 5c12815fee55: Download complete 2024-09-12T23:34:18.047 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:17 smithi181 bash[49903]: c0e87333d380: Verifying Checksum 2024-09-12T23:34:18.048 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:17 smithi181 bash[49903]: c0e87333d380: Download complete 2024-09-12T23:34:18.048 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:17 smithi181 bash[49903]: 22b70bddd3ac: Pull complete 2024-09-12T23:34:18.411 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:18 smithi136 bash[57913]: c0e87333d380: Pull complete 2024-09-12T23:34:18.756 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:18 smithi136 bash[57913]: Digest: sha256:39c642b2b337e38c18e80266fb14383754178202f40103646337722a594d984c 2024-09-12T23:34:18.756 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:18 smithi136 bash[57913]: Status: Downloaded newer image for quay.io/prometheus/node-exporter:v1.5.0 2024-09-12T23:34:18.798 DEBUG:teuthology.orchestra.run.smithi181:grafana.a> sudo journalctl -f -n 0 -u ceph-da58a47c-715d-11ef-bceb-c7b262605968@grafana.a.service 2024-09-12T23:34:18.800 INFO:tasks.cephadm:Setting up client nodes... 2024-09-12T23:34:18.800 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-12T23:34:19.096 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:18 smithi181 bash[49903]: 5c12815fee55: Pull complete 2024-09-12T23:34:19.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:18 smithi181 bash[24776]: audit 2024-09-12T23:34:17.816779+0000 mgr.y (mgr.24446) 25 : audit [DBG] from='client.24415 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi181=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:34:19.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:18 smithi181 bash[24776]: cephadm 2024-09-12T23:34:17.819577+0000 mgr.y (mgr.24446) 26 : cephadm [INF] Saving service grafana spec with placement smithi181=a;count:1 2024-09-12T23:34:19.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:18 smithi181 bash[24776]: audit 2024-09-12T23:34:17.826142+0000 mon.a (mon.0) 772 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:19.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:18 smithi181 bash[24776]: cluster 2024-09-12T23:34:17.918996+0000 mgr.y (mgr.24446) 27 : cluster [DBG] pgmap v8: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-09-12T23:34:19.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:18 smithi181 bash[24776]: audit 2024-09-12T23:34:17.990041+0000 mon.a (mon.0) 773 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:19.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:18 smithi136 bash[22601]: audit 2024-09-12T23:34:17.816779+0000 mgr.y (mgr.24446) 25 : audit [DBG] from='client.24415 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi181=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:34:19.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:18 smithi136 bash[22601]: cephadm 2024-09-12T23:34:17.819577+0000 mgr.y (mgr.24446) 26 : cephadm [INF] Saving service grafana spec with placement smithi181=a;count:1 2024-09-12T23:34:19.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:18 smithi136 bash[22601]: audit 2024-09-12T23:34:17.826142+0000 mon.a (mon.0) 772 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:19.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:18 smithi136 bash[22601]: cluster 2024-09-12T23:34:17.918996+0000 mgr.y (mgr.24446) 27 : cluster [DBG] pgmap v8: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-09-12T23:34:19.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:18 smithi136 bash[22601]: audit 2024-09-12T23:34:17.990041+0000 mon.a (mon.0) 773 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:19.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:18 smithi136 bash[32189]: audit 2024-09-12T23:34:17.816779+0000 mgr.y (mgr.24446) 25 : audit [DBG] from='client.24415 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi181=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:34:19.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:18 smithi136 bash[32189]: cephadm 2024-09-12T23:34:17.819577+0000 mgr.y (mgr.24446) 26 : cephadm [INF] Saving service grafana spec with placement smithi181=a;count:1 2024-09-12T23:34:19.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:18 smithi136 bash[32189]: audit 2024-09-12T23:34:17.826142+0000 mon.a (mon.0) 772 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:19.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:18 smithi136 bash[32189]: cluster 2024-09-12T23:34:17.918996+0000 mgr.y (mgr.24446) 27 : cluster [DBG] pgmap v8: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-09-12T23:34:19.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:18 smithi136 bash[32189]: audit 2024-09-12T23:34:17.990041+0000 mon.a (mon.0) 773 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:19.596 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:19 smithi181 bash[49903]: c0e87333d380: Pull complete 2024-09-12T23:34:19.597 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:19 smithi181 bash[49903]: Digest: sha256:39c642b2b337e38c18e80266fb14383754178202f40103646337722a594d984c 2024-09-12T23:34:19.597 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:19 smithi181 bash[49903]: Status: Downloaded newer image for quay.io/prometheus/node-exporter:v1.5.0 2024-09-12T23:34:20.973 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[57913]: ts=2024-09-12T23:34:20.675Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-09-12T23:34:20.973 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[57913]: ts=2024-09-12T23:34:20.675Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-09-12T23:34:20.973 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[57913]: ts=2024-09-12T23:34:20.676Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-12T23:34:20.973 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[57913]: ts=2024-09-12T23:34:20.676Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-12T23:34:20.974 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[57913]: ts=2024-09-12T23:34:20.676Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-12T23:34:20.974 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[57913]: ts=2024-09-12T23:34:20.676Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-12T23:34:20.974 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[57913]: ts=2024-09-12T23:34:20.677Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-12T23:34:20.974 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[57913]: ts=2024-09-12T23:34:20.677Z caller=node_exporter.go:117 level=info collector=arp 2024-09-12T23:34:20.974 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[57913]: ts=2024-09-12T23:34:20.677Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-12T23:34:20.974 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[57913]: ts=2024-09-12T23:34:20.677Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-12T23:34:20.974 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[57913]: ts=2024-09-12T23:34:20.677Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-12T23:34:20.974 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[57913]: ts=2024-09-12T23:34:20.677Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-12T23:34:20.974 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[57913]: ts=2024-09-12T23:34:20.677Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-12T23:34:20.974 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[57913]: ts=2024-09-12T23:34:20.677Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-12T23:34:20.974 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[57913]: ts=2024-09-12T23:34:20.677Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-12T23:34:20.974 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[57913]: ts=2024-09-12T23:34:20.677Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-12T23:34:20.974 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[57913]: ts=2024-09-12T23:34:20.677Z caller=node_exporter.go:117 level=info collector=edac 2024-09-12T23:34:20.974 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[57913]: ts=2024-09-12T23:34:20.677Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-12T23:34:20.975 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[57913]: ts=2024-09-12T23:34:20.677Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-12T23:34:20.975 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[57913]: ts=2024-09-12T23:34:20.677Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-12T23:34:20.975 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[57913]: ts=2024-09-12T23:34:20.677Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-12T23:34:20.975 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[57913]: ts=2024-09-12T23:34:20.677Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-12T23:34:20.975 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[57913]: ts=2024-09-12T23:34:20.677Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-12T23:34:20.975 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[57913]: ts=2024-09-12T23:34:20.677Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-12T23:34:20.975 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[57913]: ts=2024-09-12T23:34:20.677Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-12T23:34:20.975 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[57913]: ts=2024-09-12T23:34:20.677Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-12T23:34:20.976 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[57913]: ts=2024-09-12T23:34:20.677Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-12T23:34:20.976 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[57913]: ts=2024-09-12T23:34:20.677Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-12T23:34:20.976 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[57913]: ts=2024-09-12T23:34:20.677Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-12T23:34:20.976 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[57913]: ts=2024-09-12T23:34:20.677Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-12T23:34:20.976 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[57913]: ts=2024-09-12T23:34:20.677Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-12T23:34:20.976 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[57913]: ts=2024-09-12T23:34:20.677Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-12T23:34:20.976 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[57913]: ts=2024-09-12T23:34:20.677Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-12T23:34:20.976 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[57913]: ts=2024-09-12T23:34:20.677Z caller=node_exporter.go:117 level=info collector=os 2024-09-12T23:34:20.977 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[57913]: ts=2024-09-12T23:34:20.677Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-12T23:34:20.977 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[57913]: ts=2024-09-12T23:34:20.677Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-12T23:34:20.977 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[57913]: ts=2024-09-12T23:34:20.677Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-12T23:34:20.977 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[57913]: ts=2024-09-12T23:34:20.677Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-12T23:34:20.977 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[57913]: ts=2024-09-12T23:34:20.677Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-12T23:34:20.977 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[57913]: ts=2024-09-12T23:34:20.677Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-12T23:34:20.977 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[57913]: ts=2024-09-12T23:34:20.677Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-12T23:34:20.977 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[57913]: ts=2024-09-12T23:34:20.677Z caller=node_exporter.go:117 level=info collector=stat 2024-09-12T23:34:20.977 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[57913]: ts=2024-09-12T23:34:20.677Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-12T23:34:20.977 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[57913]: ts=2024-09-12T23:34:20.677Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-12T23:34:20.977 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[57913]: ts=2024-09-12T23:34:20.677Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-12T23:34:20.978 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[57913]: ts=2024-09-12T23:34:20.677Z caller=node_exporter.go:117 level=info collector=time 2024-09-12T23:34:20.978 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[57913]: ts=2024-09-12T23:34:20.677Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-12T23:34:20.978 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[57913]: ts=2024-09-12T23:34:20.677Z caller=node_exporter.go:117 level=info collector=uname 2024-09-12T23:34:20.978 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[57913]: ts=2024-09-12T23:34:20.677Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-12T23:34:20.978 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[57913]: ts=2024-09-12T23:34:20.677Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-12T23:34:20.978 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[57913]: ts=2024-09-12T23:34:20.677Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-12T23:34:20.978 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[57913]: ts=2024-09-12T23:34:20.677Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-09-12T23:34:20.978 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[57913]: ts=2024-09-12T23:34:20.677Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-12T23:34:20.978 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[49903]: ts=2024-09-12T23:34:20.794Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-09-12T23:34:20.978 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[49903]: ts=2024-09-12T23:34:20.794Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-09-12T23:34:20.978 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[49903]: ts=2024-09-12T23:34:20.794Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-12T23:34:20.978 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[49903]: ts=2024-09-12T23:34:20.795Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-12T23:34:20.979 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[49903]: ts=2024-09-12T23:34:20.795Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-12T23:34:20.979 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[49903]: ts=2024-09-12T23:34:20.795Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-12T23:34:20.979 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[49903]: ts=2024-09-12T23:34:20.796Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-12T23:34:20.979 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[49903]: ts=2024-09-12T23:34:20.796Z caller=node_exporter.go:117 level=info collector=arp 2024-09-12T23:34:20.979 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[49903]: ts=2024-09-12T23:34:20.796Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-12T23:34:20.979 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[49903]: ts=2024-09-12T23:34:20.796Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-12T23:34:20.979 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[49903]: ts=2024-09-12T23:34:20.796Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-12T23:34:20.979 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[49903]: ts=2024-09-12T23:34:20.796Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-12T23:34:20.979 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[49903]: ts=2024-09-12T23:34:20.796Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-12T23:34:20.979 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[49903]: ts=2024-09-12T23:34:20.796Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-12T23:34:20.979 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[49903]: ts=2024-09-12T23:34:20.796Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-12T23:34:20.980 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[49903]: ts=2024-09-12T23:34:20.796Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-12T23:34:20.980 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[49903]: ts=2024-09-12T23:34:20.796Z caller=node_exporter.go:117 level=info collector=edac 2024-09-12T23:34:20.980 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[49903]: ts=2024-09-12T23:34:20.796Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-12T23:34:20.980 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[49903]: ts=2024-09-12T23:34:20.796Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-12T23:34:20.980 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[49903]: ts=2024-09-12T23:34:20.796Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-12T23:34:20.980 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[49903]: ts=2024-09-12T23:34:20.796Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-12T23:34:20.980 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[49903]: ts=2024-09-12T23:34:20.796Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-12T23:34:20.980 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[49903]: ts=2024-09-12T23:34:20.796Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-12T23:34:20.980 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[49903]: ts=2024-09-12T23:34:20.796Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-12T23:34:20.980 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[49903]: ts=2024-09-12T23:34:20.796Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-12T23:34:20.980 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[49903]: ts=2024-09-12T23:34:20.796Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-12T23:34:20.980 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[49903]: ts=2024-09-12T23:34:20.796Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-12T23:34:20.981 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[49903]: ts=2024-09-12T23:34:20.796Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-12T23:34:20.981 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[49903]: ts=2024-09-12T23:34:20.796Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-12T23:34:20.981 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[49903]: ts=2024-09-12T23:34:20.796Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-12T23:34:20.981 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[49903]: ts=2024-09-12T23:34:20.796Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-12T23:34:20.981 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[49903]: ts=2024-09-12T23:34:20.796Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-12T23:34:20.981 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[49903]: ts=2024-09-12T23:34:20.796Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-12T23:34:20.981 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[49903]: ts=2024-09-12T23:34:20.796Z caller=node_exporter.go:117 level=info collector=os 2024-09-12T23:34:20.981 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[49903]: ts=2024-09-12T23:34:20.796Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-12T23:34:20.981 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[49903]: ts=2024-09-12T23:34:20.796Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-12T23:34:20.981 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[49903]: ts=2024-09-12T23:34:20.796Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-12T23:34:20.981 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[49903]: ts=2024-09-12T23:34:20.796Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-12T23:34:20.981 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[49903]: ts=2024-09-12T23:34:20.796Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-12T23:34:20.982 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[49903]: ts=2024-09-12T23:34:20.796Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-12T23:34:20.982 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[49903]: ts=2024-09-12T23:34:20.796Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-12T23:34:20.982 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[49903]: ts=2024-09-12T23:34:20.796Z caller=node_exporter.go:117 level=info collector=stat 2024-09-12T23:34:20.982 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[49903]: ts=2024-09-12T23:34:20.796Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-12T23:34:20.982 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[49903]: ts=2024-09-12T23:34:20.796Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-12T23:34:20.982 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[49903]: ts=2024-09-12T23:34:20.796Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-12T23:34:20.982 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[49903]: ts=2024-09-12T23:34:20.796Z caller=node_exporter.go:117 level=info collector=time 2024-09-12T23:34:20.982 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[49903]: ts=2024-09-12T23:34:20.796Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-12T23:34:20.982 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[49903]: ts=2024-09-12T23:34:20.796Z caller=node_exporter.go:117 level=info collector=uname 2024-09-12T23:34:20.982 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[49903]: ts=2024-09-12T23:34:20.796Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-12T23:34:20.982 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[49903]: ts=2024-09-12T23:34:20.796Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-12T23:34:20.982 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[49903]: ts=2024-09-12T23:34:20.796Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-12T23:34:20.983 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[49903]: ts=2024-09-12T23:34:20.796Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-09-12T23:34:20.983 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[49903]: ts=2024-09-12T23:34:20.796Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-12T23:34:21.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[22601]: cluster 2024-09-12T23:34:19.920269+0000 mgr.y (mgr.24446) 28 : cluster [DBG] pgmap v9: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 6 op/s 2024-09-12T23:34:21.256 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:20 smithi136 bash[32189]: cluster 2024-09-12T23:34:19.920269+0000 mgr.y (mgr.24446) 28 : cluster [DBG] pgmap v9: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 6 op/s 2024-09-12T23:34:21.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:20 smithi181 bash[24776]: cluster 2024-09-12T23:34:19.920269+0000 mgr.y (mgr.24446) 28 : cluster [DBG] pgmap v9: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 6 op/s 2024-09-12T23:34:23.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:22 smithi136 bash[22601]: cluster 2024-09-12T23:34:21.920911+0000 mgr.y (mgr.24446) 29 : cluster [DBG] pgmap v10: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-12T23:34:23.256 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:22 smithi136 bash[32189]: cluster 2024-09-12T23:34:21.920911+0000 mgr.y (mgr.24446) 29 : cluster [DBG] pgmap v10: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-12T23:34:23.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:22 smithi181 bash[24776]: cluster 2024-09-12T23:34:21.920911+0000 mgr.y (mgr.24446) 29 : cluster [DBG] pgmap v10: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-12T23:34:24.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:23 smithi136 bash[22601]: audit 2024-09-12T23:34:22.988361+0000 mon.a (mon.0) 774 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:34:24.256 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:23 smithi136 bash[32189]: audit 2024-09-12T23:34:22.988361+0000 mon.a (mon.0) 774 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:34:24.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:23 smithi181 bash[24776]: audit 2024-09-12T23:34:22.988361+0000 mon.a (mon.0) 774 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:34:24.850 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:34:25.256 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:24 smithi136 bash[32189]: cluster 2024-09-12T23:34:23.921485+0000 mgr.y (mgr.24446) 30 : cluster [DBG] pgmap v11: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-12T23:34:25.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:24 smithi136 bash[22601]: cluster 2024-09-12T23:34:23.921485+0000 mgr.y (mgr.24446) 30 : cluster [DBG] pgmap v11: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-12T23:34:25.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:24 smithi181 bash[24776]: cluster 2024-09-12T23:34:23.921485+0000 mgr.y (mgr.24446) 30 : cluster [DBG] pgmap v11: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-12T23:34:25.347 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:34:25 smithi181 bash[25717]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:34:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-12T23:34:26.982 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 12 23:34:26 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:26.982 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 12 23:34:26 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:26.982 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 12 23:34:26 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:27.256 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 12 23:34:26 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:27.256 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 12 23:34:27 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:27.256 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:26 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:27.256 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:26 smithi136 bash[32189]: cluster 2024-09-12T23:34:25.922145+0000 mgr.y (mgr.24446) 31 : cluster [DBG] pgmap v12: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-12T23:34:27.256 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:27 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:27.257 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 12 23:34:27 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:27.257 INFO:journalctl@ceph.mds.a.smithi136.stdout:Sep 12 23:34:26 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:27.257 INFO:journalctl@ceph.mds.a.smithi136.stdout:Sep 12 23:34:27 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:27.257 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:26 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:27.257 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:27 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:27.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:26 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:27.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:26 smithi136 bash[22601]: cluster 2024-09-12T23:34:25.922145+0000 mgr.y (mgr.24446) 31 : cluster [DBG] pgmap v12: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-12T23:34:27.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:27 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:27.258 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 12 23:34:27 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:27.258 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:34:26 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:27.259 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:34:26 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:27.259 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:34:26 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:27.259 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:34:27 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:27.259 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:34:27 smithi136 systemd[1]: Started Ceph alertmanager.a for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:34:27.259 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:26 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:27.259 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:34:27 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:27.259 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 12 23:34:27 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:27.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:26 smithi181 bash[24776]: cluster 2024-09-12T23:34:25.922145+0000 mgr.y (mgr.24446) 31 : cluster [DBG] pgmap v12: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-12T23:34:28.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:28 smithi136 bash[22601]: audit 2024-09-12T23:34:27.230763+0000 mon.a (mon.0) 775 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:28.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:28 smithi136 bash[22601]: audit 2024-09-12T23:34:27.237878+0000 mon.a (mon.0) 776 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:28.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:28 smithi136 bash[22601]: audit 2024-09-12T23:34:27.244957+0000 mon.a (mon.0) 777 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:28.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:28 smithi136 bash[22601]: audit 2024-09-12T23:34:27.263771+0000 mon.a (mon.0) 778 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:28.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:28 smithi136 bash[22601]: audit 2024-09-12T23:34:27.295585+0000 mon.a (mon.0) 779 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:34:28.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:28 smithi136 bash[22601]: audit 2024-09-12T23:34:27.539332+0000 mon.a (mon.0) 780 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:28.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:28 smithi136 bash[22601]: audit 2024-09-12T23:34:27.545300+0000 mon.a (mon.0) 781 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:28.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:28 smithi136 bash[22601]: audit 2024-09-12T23:34:28.001804+0000 mon.a (mon.0) 782 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:28.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:28 smithi136 bash[32189]: audit 2024-09-12T23:34:27.230763+0000 mon.a (mon.0) 775 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:28.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:28 smithi136 bash[32189]: audit 2024-09-12T23:34:27.237878+0000 mon.a (mon.0) 776 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:28.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:28 smithi136 bash[32189]: audit 2024-09-12T23:34:27.244957+0000 mon.a (mon.0) 777 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:28.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:28 smithi136 bash[32189]: audit 2024-09-12T23:34:27.263771+0000 mon.a (mon.0) 778 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:28.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:28 smithi136 bash[32189]: audit 2024-09-12T23:34:27.295585+0000 mon.a (mon.0) 779 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:34:28.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:28 smithi136 bash[32189]: audit 2024-09-12T23:34:27.539332+0000 mon.a (mon.0) 780 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:28.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:28 smithi136 bash[32189]: audit 2024-09-12T23:34:27.545300+0000 mon.a (mon.0) 781 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:28.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:28 smithi136 bash[32189]: audit 2024-09-12T23:34:28.001804+0000 mon.a (mon.0) 782 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:28.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:28 smithi181 bash[24776]: audit 2024-09-12T23:34:27.230763+0000 mon.a (mon.0) 775 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:28.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:28 smithi181 bash[24776]: audit 2024-09-12T23:34:27.237878+0000 mon.a (mon.0) 776 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:28.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:28 smithi181 bash[24776]: audit 2024-09-12T23:34:27.244957+0000 mon.a (mon.0) 777 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:28.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:28 smithi181 bash[24776]: audit 2024-09-12T23:34:27.263771+0000 mon.a (mon.0) 778 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:28.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:28 smithi181 bash[24776]: audit 2024-09-12T23:34:27.295585+0000 mon.a (mon.0) 779 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:34:28.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:28 smithi181 bash[24776]: audit 2024-09-12T23:34:27.539332+0000 mon.a (mon.0) 780 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:28.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:28 smithi181 bash[24776]: audit 2024-09-12T23:34:27.545300+0000 mon.a (mon.0) 781 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:28.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:28 smithi181 bash[24776]: audit 2024-09-12T23:34:28.001804+0000 mon.a (mon.0) 782 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:29.211 INFO:teuthology.orchestra.run.smithi136.stdout:[client.0] 2024-09-12T23:34:29.211 INFO:teuthology.orchestra.run.smithi136.stdout: key = AQCFeuNmYbpRDBAAI3byT91FoYSmQ6/CBtzxig== 2024-09-12T23:34:29.223 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:34:28 smithi136 bash[58363]: ts=2024-09-12T23:34:28.964Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-09-12T23:34:29.223 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:34:28 smithi136 bash[58363]: ts=2024-09-12T23:34:28.964Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-09-12T23:34:29.223 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:34:28 smithi136 bash[58363]: ts=2024-09-12T23:34:28.967Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.136 port=9094 2024-09-12T23:34:29.223 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:34:28 smithi136 bash[58363]: ts=2024-09-12T23:34:28.968Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-12T23:34:29.223 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:34:28 smithi136 bash[58363]: ts=2024-09-12T23:34:28.999Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-12T23:34:29.223 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:34:28 smithi136 bash[58363]: ts=2024-09-12T23:34:28.999Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-12T23:34:29.223 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:34:29 smithi136 bash[58363]: ts=2024-09-12T23:34:29.003Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-09-12T23:34:29.223 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:34:29 smithi136 bash[58363]: ts=2024-09-12T23:34:29.003Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-09-12T23:34:29.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:29 smithi181 bash[24776]: cluster 2024-09-12T23:34:27.922769+0000 mgr.y (mgr.24446) 32 : cluster [DBG] pgmap v13: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:29.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:29 smithi181 bash[24776]: audit 2024-09-12T23:34:29.205704+0000 mon.c (mon.2) 29 : audit [INF] from='client.? 172.21.15.136:0/762945620' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-12T23:34:29.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:29 smithi181 bash[24776]: audit 2024-09-12T23:34:29.206508+0000 mon.a (mon.0) 783 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-12T23:34:29.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:29 smithi181 bash[24776]: audit 2024-09-12T23:34:29.210633+0000 mon.a (mon.0) 784 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-12T23:34:29.899 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:29 smithi136 bash[22601]: cluster 2024-09-12T23:34:27.922769+0000 mgr.y (mgr.24446) 32 : cluster [DBG] pgmap v13: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:29.899 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:29 smithi136 bash[22601]: audit 2024-09-12T23:34:29.205704+0000 mon.c (mon.2) 29 : audit [INF] from='client.? 172.21.15.136:0/762945620' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-12T23:34:29.900 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:29 smithi136 bash[22601]: audit 2024-09-12T23:34:29.206508+0000 mon.a (mon.0) 783 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-12T23:34:29.900 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:29 smithi136 bash[22601]: audit 2024-09-12T23:34:29.210633+0000 mon.a (mon.0) 784 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-12T23:34:29.900 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:29 smithi136 bash[32189]: cluster 2024-09-12T23:34:27.922769+0000 mgr.y (mgr.24446) 32 : cluster [DBG] pgmap v13: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:29.900 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:29 smithi136 bash[32189]: audit 2024-09-12T23:34:29.205704+0000 mon.c (mon.2) 29 : audit [INF] from='client.? 172.21.15.136:0/762945620' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-12T23:34:29.900 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:29 smithi136 bash[32189]: audit 2024-09-12T23:34:29.206508+0000 mon.a (mon.0) 783 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-12T23:34:29.900 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:29 smithi136 bash[32189]: audit 2024-09-12T23:34:29.210633+0000 mon.a (mon.0) 784 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-12T23:34:29.901 DEBUG:teuthology.orchestra.run.smithi136:> set -ex 2024-09-12T23:34:29.901 DEBUG:teuthology.orchestra.run.smithi136:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-09-12T23:34:29.901 DEBUG:teuthology.orchestra.run.smithi136:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-09-12T23:34:29.924 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-12T23:34:31.256 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:34:30 smithi136 bash[58363]: ts=2024-09-12T23:34:30.968Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000018378s 2024-09-12T23:34:31.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:31 smithi181 bash[24776]: cluster 2024-09-12T23:34:29.923669+0000 mgr.y (mgr.24446) 33 : cluster [DBG] pgmap v14: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:32.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:31 smithi136 bash[22601]: cluster 2024-09-12T23:34:29.923669+0000 mgr.y (mgr.24446) 33 : cluster [DBG] pgmap v14: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:32.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:31 smithi136 bash[32189]: cluster 2024-09-12T23:34:29.923669+0000 mgr.y (mgr.24446) 33 : cluster [DBG] pgmap v14: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:34.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:33 smithi136 bash[22601]: cluster 2024-09-12T23:34:31.924242+0000 mgr.y (mgr.24446) 34 : cluster [DBG] pgmap v15: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:34.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:33 smithi136 bash[22601]: audit 2024-09-12T23:34:32.645401+0000 mon.a (mon.0) 785 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:34.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:33 smithi136 bash[22601]: audit 2024-09-12T23:34:32.656988+0000 mon.a (mon.0) 786 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:34.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:33 smithi136 bash[32189]: cluster 2024-09-12T23:34:31.924242+0000 mgr.y (mgr.24446) 34 : cluster [DBG] pgmap v15: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:34.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:33 smithi136 bash[32189]: audit 2024-09-12T23:34:32.645401+0000 mon.a (mon.0) 785 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:34.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:33 smithi136 bash[32189]: audit 2024-09-12T23:34:32.656988+0000 mon.a (mon.0) 786 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:34.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:33 smithi181 bash[24776]: cluster 2024-09-12T23:34:31.924242+0000 mgr.y (mgr.24446) 34 : cluster [DBG] pgmap v15: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:34.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:33 smithi181 bash[24776]: audit 2024-09-12T23:34:32.645401+0000 mon.a (mon.0) 785 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:34.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:33 smithi181 bash[24776]: audit 2024-09-12T23:34:32.656988+0000 mon.a (mon.0) 786 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:34.128 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.b/config 2024-09-12T23:34:34.506 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:34 smithi136 bash[22852]: [12/Sep/2024:23:34:34] ENGINE Bus STOPPING 2024-09-12T23:34:34.506 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:34 smithi136 bash[22852]: [12/Sep/2024:23:34:34] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-12T23:34:34.506 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:34 smithi136 bash[22852]: [12/Sep/2024:23:34:34] ENGINE Bus STOPPED 2024-09-12T23:34:34.506 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:34 smithi136 bash[22852]: [12/Sep/2024:23:34:34] ENGINE Bus STARTING 2024-09-12T23:34:34.506 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:34 smithi136 bash[22852]: [12/Sep/2024:23:34:34] ENGINE Serving on http://:::9283 2024-09-12T23:34:34.506 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:34:34 smithi136 bash[22852]: [12/Sep/2024:23:34:34] ENGINE Bus STARTED 2024-09-12T23:34:34.597 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:34:34 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:34:35.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:34 smithi136 bash[22601]: cluster 2024-09-12T23:34:33.924762+0000 mgr.y (mgr.24446) 35 : cluster [DBG] pgmap v16: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:35.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:34 smithi136 bash[22601]: audit 2024-09-12T23:34:33.976702+0000 mon.a (mon.0) 787 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:35.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:34 smithi136 bash[22601]: audit 2024-09-12T23:34:33.982096+0000 mon.a (mon.0) 788 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:35.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:34 smithi136 bash[22601]: audit 2024-09-12T23:34:33.982935+0000 mon.a (mon.0) 789 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:34:35.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:34 smithi136 bash[22601]: audit 2024-09-12T23:34:33.983536+0000 mon.a (mon.0) 790 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:34:35.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:34 smithi136 bash[22601]: audit 2024-09-12T23:34:33.987404+0000 mon.a (mon.0) 791 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:35.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:34 smithi136 bash[22601]: audit 2024-09-12T23:34:34.074542+0000 mon.a (mon.0) 792 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:35.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:34 smithi136 bash[22601]: audit 2024-09-12T23:34:34.081059+0000 mon.a (mon.0) 793 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:35.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:34 smithi136 bash[22601]: audit 2024-09-12T23:34:34.084283+0000 mon.a (mon.0) 794 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-12T23:34:35.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:34 smithi136 bash[22601]: audit 2024-09-12T23:34:34.084812+0000 mgr.y (mgr.24446) 36 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-12T23:34:35.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:34 smithi136 bash[22601]: audit 2024-09-12T23:34:34.093703+0000 mon.a (mon.0) 795 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:35.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:34 smithi136 bash[22601]: cephadm 2024-09-12T23:34:34.120403+0000 mgr.y (mgr.24446) 37 : cephadm [INF] Deploying daemon grafana.a on smithi181 2024-09-12T23:34:35.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:34 smithi136 bash[32189]: cluster 2024-09-12T23:34:33.924762+0000 mgr.y (mgr.24446) 35 : cluster [DBG] pgmap v16: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:35.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:34 smithi136 bash[32189]: audit 2024-09-12T23:34:33.976702+0000 mon.a (mon.0) 787 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:35.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:34 smithi136 bash[32189]: audit 2024-09-12T23:34:33.982096+0000 mon.a (mon.0) 788 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:35.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:34 smithi136 bash[32189]: audit 2024-09-12T23:34:33.982935+0000 mon.a (mon.0) 789 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:34:35.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:34 smithi136 bash[32189]: audit 2024-09-12T23:34:33.983536+0000 mon.a (mon.0) 790 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:34:35.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:34 smithi136 bash[32189]: audit 2024-09-12T23:34:33.987404+0000 mon.a (mon.0) 791 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:35.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:34 smithi136 bash[32189]: audit 2024-09-12T23:34:34.074542+0000 mon.a (mon.0) 792 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:35.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:34 smithi136 bash[32189]: audit 2024-09-12T23:34:34.081059+0000 mon.a (mon.0) 793 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:35.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:34 smithi136 bash[32189]: audit 2024-09-12T23:34:34.084283+0000 mon.a (mon.0) 794 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-12T23:34:35.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:34 smithi136 bash[32189]: audit 2024-09-12T23:34:34.084812+0000 mgr.y (mgr.24446) 36 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-12T23:34:35.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:34 smithi136 bash[32189]: audit 2024-09-12T23:34:34.093703+0000 mon.a (mon.0) 795 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:35.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:34 smithi136 bash[32189]: cephadm 2024-09-12T23:34:34.120403+0000 mgr.y (mgr.24446) 37 : cephadm [INF] Deploying daemon grafana.a on smithi181 2024-09-12T23:34:35.346 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:34:35 smithi181 bash[25717]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:34:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-12T23:34:35.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:34 smithi181 bash[24776]: cluster 2024-09-12T23:34:33.924762+0000 mgr.y (mgr.24446) 35 : cluster [DBG] pgmap v16: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:35.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:34 smithi181 bash[24776]: audit 2024-09-12T23:34:33.976702+0000 mon.a (mon.0) 787 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:35.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:34 smithi181 bash[24776]: audit 2024-09-12T23:34:33.982096+0000 mon.a (mon.0) 788 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:35.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:34 smithi181 bash[24776]: audit 2024-09-12T23:34:33.982935+0000 mon.a (mon.0) 789 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:34:35.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:34 smithi181 bash[24776]: audit 2024-09-12T23:34:33.983536+0000 mon.a (mon.0) 790 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:34:35.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:34 smithi181 bash[24776]: audit 2024-09-12T23:34:33.987404+0000 mon.a (mon.0) 791 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:35.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:34 smithi181 bash[24776]: audit 2024-09-12T23:34:34.074542+0000 mon.a (mon.0) 792 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:35.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:34 smithi181 bash[24776]: audit 2024-09-12T23:34:34.081059+0000 mon.a (mon.0) 793 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:35.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:34 smithi181 bash[24776]: audit 2024-09-12T23:34:34.084283+0000 mon.a (mon.0) 794 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-12T23:34:35.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:34 smithi181 bash[24776]: audit 2024-09-12T23:34:34.084812+0000 mgr.y (mgr.24446) 36 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-12T23:34:35.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:34 smithi181 bash[24776]: audit 2024-09-12T23:34:34.093703+0000 mon.a (mon.0) 795 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:35.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:34 smithi181 bash[24776]: cephadm 2024-09-12T23:34:34.120403+0000 mgr.y (mgr.24446) 37 : cephadm [INF] Deploying daemon grafana.a on smithi181 2024-09-12T23:34:37.256 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:36 smithi136 bash[32189]: cluster 2024-09-12T23:34:35.925772+0000 mgr.y (mgr.24446) 38 : cluster [DBG] pgmap v17: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:37.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:36 smithi136 bash[22601]: cluster 2024-09-12T23:34:35.925772+0000 mgr.y (mgr.24446) 38 : cluster [DBG] pgmap v17: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:37.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:36 smithi181 bash[24776]: cluster 2024-09-12T23:34:35.925772+0000 mgr.y (mgr.24446) 38 : cluster [DBG] pgmap v17: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:38.469 INFO:teuthology.orchestra.run.smithi181.stdout:[client.1] 2024-09-12T23:34:38.470 INFO:teuthology.orchestra.run.smithi181.stdout: key = AQCOeuNmRVCmGxAAe933N+ygav6NpAFdH4Orqg== 2024-09-12T23:34:39.256 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:34:38 smithi136 bash[58363]: ts=2024-09-12T23:34:38.970Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002652878s 2024-09-12T23:34:39.256 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:38 smithi136 bash[32189]: cluster 2024-09-12T23:34:37.926397+0000 mgr.y (mgr.24446) 39 : cluster [DBG] pgmap v18: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:39.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:38 smithi136 bash[32189]: audit 2024-09-12T23:34:37.988910+0000 mon.a (mon.0) 796 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:34:39.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:38 smithi136 bash[32189]: audit 2024-09-12T23:34:38.461339+0000 mon.b (mon.1) 25 : audit [INF] from='client.? 172.21.15.181:0/2248562504' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-12T23:34:39.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:38 smithi136 bash[32189]: audit 2024-09-12T23:34:38.463539+0000 mon.a (mon.0) 797 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-12T23:34:39.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:38 smithi136 bash[32189]: audit 2024-09-12T23:34:38.473700+0000 mon.a (mon.0) 798 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-12T23:34:39.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:38 smithi136 bash[22601]: cluster 2024-09-12T23:34:37.926397+0000 mgr.y (mgr.24446) 39 : cluster [DBG] pgmap v18: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:39.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:38 smithi136 bash[22601]: audit 2024-09-12T23:34:37.988910+0000 mon.a (mon.0) 796 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:34:39.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:38 smithi136 bash[22601]: audit 2024-09-12T23:34:38.461339+0000 mon.b (mon.1) 25 : audit [INF] from='client.? 172.21.15.181:0/2248562504' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-12T23:34:39.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:38 smithi136 bash[22601]: audit 2024-09-12T23:34:38.463539+0000 mon.a (mon.0) 797 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-12T23:34:39.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:38 smithi136 bash[22601]: audit 2024-09-12T23:34:38.473700+0000 mon.a (mon.0) 798 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-12T23:34:39.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:38 smithi181 bash[24776]: cluster 2024-09-12T23:34:37.926397+0000 mgr.y (mgr.24446) 39 : cluster [DBG] pgmap v18: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:39.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:38 smithi181 bash[24776]: audit 2024-09-12T23:34:37.988910+0000 mon.a (mon.0) 796 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:34:39.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:38 smithi181 bash[24776]: audit 2024-09-12T23:34:38.461339+0000 mon.b (mon.1) 25 : audit [INF] from='client.? 172.21.15.181:0/2248562504' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-12T23:34:39.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:38 smithi181 bash[24776]: audit 2024-09-12T23:34:38.463539+0000 mon.a (mon.0) 797 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-12T23:34:39.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:38 smithi181 bash[24776]: audit 2024-09-12T23:34:38.473700+0000 mon.a (mon.0) 798 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-12T23:34:40.031 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2024-09-12T23:34:40.031 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-09-12T23:34:40.031 DEBUG:teuthology.orchestra.run.smithi181:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-09-12T23:34:40.055 INFO:tasks.cephadm:Creating RBD pool 2024-09-12T23:34:40.056 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph osd pool create rbd 8 2024-09-12T23:34:41.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:40 smithi136 bash[22601]: cluster 2024-09-12T23:34:39.927590+0000 mgr.y (mgr.24446) 40 : cluster [DBG] pgmap v19: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:41.256 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:40 smithi136 bash[32189]: cluster 2024-09-12T23:34:39.927590+0000 mgr.y (mgr.24446) 40 : cluster [DBG] pgmap v19: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:41.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:40 smithi181 bash[24776]: cluster 2024-09-12T23:34:39.927590+0000 mgr.y (mgr.24446) 40 : cluster [DBG] pgmap v19: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:43.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:42 smithi136 bash[22601]: cluster 2024-09-12T23:34:41.928181+0000 mgr.y (mgr.24446) 41 : cluster [DBG] pgmap v20: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:43.256 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:42 smithi136 bash[32189]: cluster 2024-09-12T23:34:41.928181+0000 mgr.y (mgr.24446) 41 : cluster [DBG] pgmap v20: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:43.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:42 smithi181 bash[24776]: cluster 2024-09-12T23:34:41.928181+0000 mgr.y (mgr.24446) 41 : cluster [DBG] pgmap v20: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:44.843 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:34:45.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:45 smithi136 bash[22601]: cluster 2024-09-12T23:34:43.929137+0000 mgr.y (mgr.24446) 42 : cluster [DBG] pgmap v21: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:45.256 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:45 smithi136 bash[32189]: cluster 2024-09-12T23:34:43.929137+0000 mgr.y (mgr.24446) 42 : cluster [DBG] pgmap v21: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:45.346 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:34:45 smithi181 bash[25717]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:34:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-12T23:34:45.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:44 smithi181 bash[24776]: cluster 2024-09-12T23:34:43.929137+0000 mgr.y (mgr.24446) 42 : cluster [DBG] pgmap v21: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:47.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:47 smithi136 bash[22601]: cluster 2024-09-12T23:34:45.930182+0000 mgr.y (mgr.24446) 43 : cluster [DBG] pgmap v22: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:47.256 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:47 smithi136 bash[32189]: cluster 2024-09-12T23:34:45.930182+0000 mgr.y (mgr.24446) 43 : cluster [DBG] pgmap v22: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:47.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:47 smithi181 bash[24776]: cluster 2024-09-12T23:34:45.930182+0000 mgr.y (mgr.24446) 43 : cluster [DBG] pgmap v22: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:48.464 INFO:teuthology.orchestra.run.smithi136.stderr:pool 'rbd' created 2024-09-12T23:34:48.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:48 smithi136 bash[22601]: audit 2024-09-12T23:34:47.971489+0000 mon.a (mon.0) 799 : audit [INF] from='client.? 172.21.15.136:0/1381242693' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-12T23:34:48.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:48 smithi136 bash[32189]: audit 2024-09-12T23:34:47.971489+0000 mon.a (mon.0) 799 : audit [INF] from='client.? 172.21.15.136:0/1381242693' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-12T23:34:48.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:48 smithi181 bash[24776]: audit 2024-09-12T23:34:47.971489+0000 mon.a (mon.0) 799 : audit [INF] from='client.? 172.21.15.136:0/1381242693' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-12T23:34:49.109 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- rbd pool init rbd 2024-09-12T23:34:49.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:49 smithi136 bash[22601]: cluster 2024-09-12T23:34:47.930766+0000 mgr.y (mgr.24446) 44 : cluster [DBG] pgmap v23: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:49.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:49 smithi136 bash[22601]: audit 2024-09-12T23:34:48.468058+0000 mon.a (mon.0) 800 : audit [INF] from='client.? 172.21.15.136:0/1381242693' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-09-12T23:34:49.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:49 smithi136 bash[22601]: cluster 2024-09-12T23:34:48.468167+0000 mon.a (mon.0) 801 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2024-09-12T23:34:49.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:49 smithi136 bash[32189]: cluster 2024-09-12T23:34:47.930766+0000 mgr.y (mgr.24446) 44 : cluster [DBG] pgmap v23: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:49.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:49 smithi136 bash[32189]: audit 2024-09-12T23:34:48.468058+0000 mon.a (mon.0) 800 : audit [INF] from='client.? 172.21.15.136:0/1381242693' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-09-12T23:34:49.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:49 smithi136 bash[32189]: cluster 2024-09-12T23:34:48.468167+0000 mon.a (mon.0) 801 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2024-09-12T23:34:49.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:49 smithi181 bash[24776]: cluster 2024-09-12T23:34:47.930766+0000 mgr.y (mgr.24446) 44 : cluster [DBG] pgmap v23: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:49.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:49 smithi181 bash[24776]: audit 2024-09-12T23:34:48.468058+0000 mon.a (mon.0) 800 : audit [INF] from='client.? 172.21.15.136:0/1381242693' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-09-12T23:34:49.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:49 smithi181 bash[24776]: cluster 2024-09-12T23:34:48.468167+0000 mon.a (mon.0) 801 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2024-09-12T23:34:50.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:50 smithi136 bash[32189]: cluster 2024-09-12T23:34:49.487503+0000 mon.a (mon.0) 802 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2024-09-12T23:34:50.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:50 smithi136 bash[22601]: cluster 2024-09-12T23:34:49.487503+0000 mon.a (mon.0) 802 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2024-09-12T23:34:50.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:50 smithi181 bash[24776]: cluster 2024-09-12T23:34:49.487503+0000 mon.a (mon.0) 802 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2024-09-12T23:34:51.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:51 smithi136 bash[22601]: cluster 2024-09-12T23:34:49.931520+0000 mgr.y (mgr.24446) 45 : cluster [DBG] pgmap v26: 105 pgs: 8 creating+peering, 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:51.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:51 smithi136 bash[22601]: cluster 2024-09-12T23:34:50.482638+0000 mon.a (mon.0) 803 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:34:51.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:51 smithi136 bash[22601]: cluster 2024-09-12T23:34:50.500102+0000 mon.a (mon.0) 804 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2024-09-12T23:34:51.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:51 smithi136 bash[32189]: cluster 2024-09-12T23:34:49.931520+0000 mgr.y (mgr.24446) 45 : cluster [DBG] pgmap v26: 105 pgs: 8 creating+peering, 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:51.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:51 smithi136 bash[32189]: cluster 2024-09-12T23:34:50.482638+0000 mon.a (mon.0) 803 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:34:51.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:51 smithi136 bash[32189]: cluster 2024-09-12T23:34:50.500102+0000 mon.a (mon.0) 804 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2024-09-12T23:34:51.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:51 smithi181 bash[24776]: cluster 2024-09-12T23:34:49.931520+0000 mgr.y (mgr.24446) 45 : cluster [DBG] pgmap v26: 105 pgs: 8 creating+peering, 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:51.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:51 smithi181 bash[24776]: cluster 2024-09-12T23:34:50.482638+0000 mon.a (mon.0) 803 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:34:51.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:51 smithi181 bash[24776]: cluster 2024-09-12T23:34:50.500102+0000 mon.a (mon.0) 804 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2024-09-12T23:34:53.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:53 smithi136 bash[22601]: cluster 2024-09-12T23:34:51.932279+0000 mgr.y (mgr.24446) 46 : cluster [DBG] pgmap v28: 105 pgs: 8 creating+peering, 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:53.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:53 smithi136 bash[22601]: audit 2024-09-12T23:34:52.997712+0000 mon.a (mon.0) 805 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:53.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:53 smithi136 bash[22601]: audit 2024-09-12T23:34:52.999356+0000 mon.a (mon.0) 806 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:34:53.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:53 smithi136 bash[32189]: cluster 2024-09-12T23:34:51.932279+0000 mgr.y (mgr.24446) 46 : cluster [DBG] pgmap v28: 105 pgs: 8 creating+peering, 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:53.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:53 smithi136 bash[32189]: audit 2024-09-12T23:34:52.997712+0000 mon.a (mon.0) 805 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:53.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:53 smithi136 bash[32189]: audit 2024-09-12T23:34:52.999356+0000 mon.a (mon.0) 806 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:34:53.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:53 smithi181 bash[24776]: cluster 2024-09-12T23:34:51.932279+0000 mgr.y (mgr.24446) 46 : cluster [DBG] pgmap v28: 105 pgs: 8 creating+peering, 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:53.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:53 smithi181 bash[24776]: audit 2024-09-12T23:34:52.997712+0000 mon.a (mon.0) 805 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:34:53.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:53 smithi181 bash[24776]: audit 2024-09-12T23:34:52.999356+0000 mon.a (mon.0) 806 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:34:53.897 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:34:55.499 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:34:55 smithi181 bash[25717]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:34:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-12T23:34:55.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:55 smithi136 bash[22601]: cluster 2024-09-12T23:34:53.932927+0000 mgr.y (mgr.24446) 47 : cluster [DBG] pgmap v29: 105 pgs: 2 creating+peering, 103 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:55.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:55 smithi136 bash[32189]: cluster 2024-09-12T23:34:53.932927+0000 mgr.y (mgr.24446) 47 : cluster [DBG] pgmap v29: 105 pgs: 2 creating+peering, 103 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:55.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:55 smithi181 bash[24776]: cluster 2024-09-12T23:34:53.932927+0000 mgr.y (mgr.24446) 47 : cluster [DBG] pgmap v29: 105 pgs: 2 creating+peering, 103 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:57.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:57 smithi181 bash[24776]: cluster 2024-09-12T23:34:55.933657+0000 mgr.y (mgr.24446) 48 : cluster [DBG] pgmap v30: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:57.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:57 smithi181 bash[24776]: audit 2024-09-12T23:34:56.876556+0000 mon.c (mon.2) 30 : audit [INF] from='client.? 172.21.15.136:0/3788706494' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-12T23:34:57.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:57 smithi181 bash[24776]: audit 2024-09-12T23:34:56.876938+0000 mon.a (mon.0) 807 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-12T23:34:58.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:57 smithi136 bash[22601]: cluster 2024-09-12T23:34:55.933657+0000 mgr.y (mgr.24446) 48 : cluster [DBG] pgmap v30: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:58.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:57 smithi136 bash[22601]: audit 2024-09-12T23:34:56.876556+0000 mon.c (mon.2) 30 : audit [INF] from='client.? 172.21.15.136:0/3788706494' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-12T23:34:58.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:57 smithi136 bash[22601]: audit 2024-09-12T23:34:56.876938+0000 mon.a (mon.0) 807 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-12T23:34:58.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:57 smithi136 bash[32189]: cluster 2024-09-12T23:34:55.933657+0000 mgr.y (mgr.24446) 48 : cluster [DBG] pgmap v30: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:58.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:57 smithi136 bash[32189]: audit 2024-09-12T23:34:56.876556+0000 mon.c (mon.2) 30 : audit [INF] from='client.? 172.21.15.136:0/3788706494' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-12T23:34:58.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:57 smithi136 bash[32189]: audit 2024-09-12T23:34:56.876938+0000 mon.a (mon.0) 807 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-12T23:34:58.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:58 smithi181 bash[24776]: audit 2024-09-12T23:34:57.514427+0000 mon.a (mon.0) 808 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-09-12T23:34:58.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:58 smithi181 bash[24776]: cluster 2024-09-12T23:34:57.514608+0000 mon.a (mon.0) 809 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2024-09-12T23:34:59.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:58 smithi136 bash[22601]: audit 2024-09-12T23:34:57.514427+0000 mon.a (mon.0) 808 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-09-12T23:34:59.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:58 smithi136 bash[22601]: cluster 2024-09-12T23:34:57.514608+0000 mon.a (mon.0) 809 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2024-09-12T23:34:59.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:58 smithi136 bash[32189]: audit 2024-09-12T23:34:57.514427+0000 mon.a (mon.0) 808 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-09-12T23:34:59.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:58 smithi136 bash[32189]: cluster 2024-09-12T23:34:57.514608+0000 mon.a (mon.0) 809 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2024-09-12T23:34:59.778 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:59 smithi136 bash[32189]: cluster 2024-09-12T23:34:57.934313+0000 mgr.y (mgr.24446) 49 : cluster [DBG] pgmap v32: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:59.778 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:59 smithi136 bash[32189]: cluster 2024-09-12T23:34:58.516787+0000 mon.a (mon.0) 810 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:34:59.779 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:59 smithi136 bash[32189]: cluster 2024-09-12T23:34:58.516849+0000 mon.a (mon.0) 811 : cluster [INF] Cluster is now healthy 2024-09-12T23:34:59.779 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:34:59 smithi136 bash[32189]: cluster 2024-09-12T23:34:58.532397+0000 mon.a (mon.0) 812 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2024-09-12T23:34:59.779 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:59 smithi136 bash[22601]: cluster 2024-09-12T23:34:57.934313+0000 mgr.y (mgr.24446) 49 : cluster [DBG] pgmap v32: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:59.779 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:59 smithi136 bash[22601]: cluster 2024-09-12T23:34:58.516787+0000 mon.a (mon.0) 810 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:34:59.779 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:59 smithi136 bash[22601]: cluster 2024-09-12T23:34:58.516849+0000 mon.a (mon.0) 811 : cluster [INF] Cluster is now healthy 2024-09-12T23:34:59.779 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:34:59 smithi136 bash[22601]: cluster 2024-09-12T23:34:58.532397+0000 mon.a (mon.0) 812 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2024-09-12T23:34:59.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:59 smithi181 bash[24776]: cluster 2024-09-12T23:34:57.934313+0000 mgr.y (mgr.24446) 49 : cluster [DBG] pgmap v32: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:34:59.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:59 smithi181 bash[24776]: cluster 2024-09-12T23:34:58.516787+0000 mon.a (mon.0) 810 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:34:59.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:59 smithi181 bash[24776]: cluster 2024-09-12T23:34:58.516849+0000 mon.a (mon.0) 811 : cluster [INF] Cluster is now healthy 2024-09-12T23:34:59.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:34:59 smithi181 bash[24776]: cluster 2024-09-12T23:34:58.532397+0000 mon.a (mon.0) 812 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2024-09-12T23:35:00.257 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-12T23:35:00.257 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-12T23:35:00.258 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph mgr dump --format=json 2024-09-12T23:35:00.822 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:00 smithi181 bash[24776]: cluster 2024-09-12T23:34:59.541740+0000 mon.a (mon.0) 813 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2024-09-12T23:35:01.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:00 smithi136 bash[22601]: cluster 2024-09-12T23:34:59.541740+0000 mon.a (mon.0) 813 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2024-09-12T23:35:01.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:00 smithi136 bash[32189]: cluster 2024-09-12T23:34:59.541740+0000 mon.a (mon.0) 813 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2024-09-12T23:35:01.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:01 smithi181 bash[24776]: cluster 2024-09-12T23:34:59.934935+0000 mgr.y (mgr.24446) 50 : cluster [DBG] pgmap v35: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s wr, 0 op/s 2024-09-12T23:35:02.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:01 smithi136 bash[32189]: cluster 2024-09-12T23:34:59.934935+0000 mgr.y (mgr.24446) 50 : cluster [DBG] pgmap v35: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s wr, 0 op/s 2024-09-12T23:35:02.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:01 smithi136 bash[22601]: cluster 2024-09-12T23:34:59.934935+0000 mgr.y (mgr.24446) 50 : cluster [DBG] pgmap v35: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s wr, 0 op/s 2024-09-12T23:35:03.329 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:35:03 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:35:03.329 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 12 23:35:03 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:35:03.329 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:35:03 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:35:03.330 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 12 23:35:03 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:35:03.330 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:35:03 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:35:03.330 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:03 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:35:03.331 INFO:journalctl@ceph.mds.b.smithi181.stdout:Sep 12 23:35:03 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:35:03.331 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:35:03 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:35:03.331 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:35:03 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:35:03.596 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:35:03 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:35:03.596 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:35:03 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:35:03.597 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 12 23:35:03 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:35:03.597 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:35:03 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:35:03.597 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:35:03 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:35:03.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:03 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:35:03.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:03 smithi181 bash[24776]: cluster 2024-09-12T23:35:01.935690+0000 mgr.y (mgr.24446) 51 : cluster [DBG] pgmap v36: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s wr, 0 op/s 2024-09-12T23:35:03.598 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 12 23:35:03 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:35:03.598 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:35:03 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:35:03.598 INFO:journalctl@ceph.mds.b.smithi181.stdout:Sep 12 23:35:03 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:35:03.599 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:03 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:35:03.599 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:03 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:35:03.599 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:03 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:35:03.599 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:03 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:35:03.599 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:03 smithi181 systemd[1]: Started Ceph grafana.a for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:35:04.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:03 smithi136 bash[22601]: cluster 2024-09-12T23:35:01.935690+0000 mgr.y (mgr.24446) 51 : cluster [DBG] pgmap v36: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s wr, 0 op/s 2024-09-12T23:35:04.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:03 smithi136 bash[32189]: cluster 2024-09-12T23:35:01.935690+0000 mgr.y (mgr.24446) 51 : cluster [DBG] pgmap v36: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s wr, 0 op/s 2024-09-12T23:35:04.963 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:04 smithi181 bash[24776]: audit 2024-09-12T23:35:03.618903+0000 mon.a (mon.0) 814 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:04.963 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:04 smithi181 bash[24776]: audit 2024-09-12T23:35:03.628957+0000 mon.a (mon.0) 815 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:04.963 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:04 smithi181 bash[24776]: audit 2024-09-12T23:35:03.635467+0000 mon.a (mon.0) 816 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:04.963 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:04 smithi181 bash[24776]: audit 2024-09-12T23:35:03.642940+0000 mon.a (mon.0) 817 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:04.963 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:04 smithi181 bash[24776]: audit 2024-09-12T23:35:03.696269+0000 mon.a (mon.0) 818 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:35:05.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:04 smithi136 bash[22601]: audit 2024-09-12T23:35:03.618903+0000 mon.a (mon.0) 814 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:05.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:04 smithi136 bash[22601]: audit 2024-09-12T23:35:03.628957+0000 mon.a (mon.0) 815 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:05.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:04 smithi136 bash[22601]: audit 2024-09-12T23:35:03.635467+0000 mon.a (mon.0) 816 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:05.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:04 smithi136 bash[22601]: audit 2024-09-12T23:35:03.642940+0000 mon.a (mon.0) 817 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:05.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:04 smithi136 bash[22601]: audit 2024-09-12T23:35:03.696269+0000 mon.a (mon.0) 818 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:35:05.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:04 smithi136 bash[32189]: audit 2024-09-12T23:35:03.618903+0000 mon.a (mon.0) 814 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:05.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:04 smithi136 bash[32189]: audit 2024-09-12T23:35:03.628957+0000 mon.a (mon.0) 815 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:05.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:04 smithi136 bash[32189]: audit 2024-09-12T23:35:03.635467+0000 mon.a (mon.0) 816 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:05.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:04 smithi136 bash[32189]: audit 2024-09-12T23:35:03.642940+0000 mon.a (mon.0) 817 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:05.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:04 smithi136 bash[32189]: audit 2024-09-12T23:35:03.696269+0000 mon.a (mon.0) 818 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:35:05.041 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:35:05.233 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:35:05 smithi181 bash[25717]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:35:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-12T23:35:05.233 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:04 smithi181 bash[50806]: logger=settings t=2024-09-12T23:35:04.965871402Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-09-12T23:35:05.234 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:04 smithi181 bash[50806]: logger=settings t=2024-09-12T23:35:04.966065661Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-09-12T23:35:05.234 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:04 smithi181 bash[50806]: logger=settings t=2024-09-12T23:35:04.966086352Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-09-12T23:35:05.234 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:04 smithi181 bash[50806]: logger=settings t=2024-09-12T23:35:04.966104547Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-09-12T23:35:05.234 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:04 smithi181 bash[50806]: logger=settings t=2024-09-12T23:35:04.966118621Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-09-12T23:35:05.234 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:04 smithi181 bash[50806]: logger=settings t=2024-09-12T23:35:04.966130528Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-09-12T23:35:05.234 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:04 smithi181 bash[50806]: logger=settings t=2024-09-12T23:35:04.966142061Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-09-12T23:35:05.234 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:04 smithi181 bash[50806]: logger=settings t=2024-09-12T23:35:04.96615331Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-09-12T23:35:05.234 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:04 smithi181 bash[50806]: logger=settings t=2024-09-12T23:35:04.966170749Z level=info msg="Path Home" path=/usr/share/grafana 2024-09-12T23:35:05.234 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:04 smithi181 bash[50806]: logger=settings t=2024-09-12T23:35:04.966182858Z level=info msg="Path Data" path=/var/lib/grafana 2024-09-12T23:35:05.234 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:04 smithi181 bash[50806]: logger=settings t=2024-09-12T23:35:04.966193706Z level=info msg="Path Logs" path=/var/log/grafana 2024-09-12T23:35:05.234 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:04 smithi181 bash[50806]: logger=settings t=2024-09-12T23:35:04.966203602Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-09-12T23:35:05.234 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:04 smithi181 bash[50806]: logger=settings t=2024-09-12T23:35:04.966213598Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-09-12T23:35:05.234 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:04 smithi181 bash[50806]: logger=settings t=2024-09-12T23:35:04.966223999Z level=info msg="App mode production" 2024-09-12T23:35:05.235 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:04 smithi181 bash[50806]: logger=sqlstore t=2024-09-12T23:35:04.966309815Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-09-12T23:35:05.235 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:04 smithi181 bash[50806]: logger=sqlstore t=2024-09-12T23:35:04.966342516Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-09-12T23:35:05.235 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:04 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:04.967659069Z level=info msg="Starting DB migrations" 2024-09-12T23:35:05.235 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:04 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:04.967740228Z level=info msg="Executing migration" id="create migration_log table" 2024-09-12T23:35:05.235 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:05 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:05.016728371Z level=info msg="Executing migration" id="create user table" 2024-09-12T23:35:05.235 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:05 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:05.041830267Z level=info msg="Executing migration" id="add unique index user.login" 2024-09-12T23:35:05.235 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:05 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:05.100540554Z level=info msg="Executing migration" id="add unique index user.email" 2024-09-12T23:35:05.235 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:05 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:05.126071574Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2024-09-12T23:35:05.235 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:05 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:05.168181165Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2024-09-12T23:35:05.235 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:05 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:05.210115465Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2024-09-12T23:35:05.235 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:05 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:05.235472615Z level=info msg="Executing migration" id="create user table v2" 2024-09-12T23:35:05.493 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:05 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:05.277483995Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2024-09-12T23:35:05.494 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:05 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:05.302768024Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2024-09-12T23:35:05.494 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:05 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:05.353154859Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2024-09-12T23:35:05.494 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:05 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:05.378378796Z level=info msg="Executing migration" id="Drop old table user_v1" 2024-09-12T23:35:05.494 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:05 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:05.420392636Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2024-09-12T23:35:05.494 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:05 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:05.445618397Z level=info msg="Executing migration" id="Update user table charset" 2024-09-12T23:35:05.494 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:05 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:05.471009379Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2024-09-12T23:35:05.494 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:05 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:05.496266675Z level=info msg="Executing migration" id="Add missing user data" 2024-09-12T23:35:05.762 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:05 smithi181 bash[24776]: cluster 2024-09-12T23:35:03.936304+0000 mgr.y (mgr.24446) 52 : cluster [DBG] pgmap v37: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 318 B/s wr, 0 op/s 2024-09-12T23:35:05.763 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:05 smithi181 bash[24776]: cluster 2024-09-12T23:35:05.394358+0000 mon.a (mon.0) 819 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2024-09-12T23:35:05.763 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:05 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:05.53818385Z level=info msg="Executing migration" id="Add is_disabled column to user" 2024-09-12T23:35:05.763 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:05 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:05.580223083Z level=info msg="Executing migration" id="Add index user.login/user.email" 2024-09-12T23:35:05.763 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:05 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:05.605524605Z level=info msg="Executing migration" id="Add is_service_account column to user" 2024-09-12T23:35:05.763 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:05 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:05.647435383Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-09-12T23:35:05.763 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:05 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:05.672930514Z level=info msg="Executing migration" id="create temp user table v1-7" 2024-09-12T23:35:05.763 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:05 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:05.69817859Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2024-09-12T23:35:05.763 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:05 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:05.740161371Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2024-09-12T23:35:05.763 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:05 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:05.765506026Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2024-09-12T23:35:06.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:05 smithi136 bash[22601]: cluster 2024-09-12T23:35:03.936304+0000 mgr.y (mgr.24446) 52 : cluster [DBG] pgmap v37: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 318 B/s wr, 0 op/s 2024-09-12T23:35:06.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:05 smithi136 bash[22601]: cluster 2024-09-12T23:35:05.394358+0000 mon.a (mon.0) 819 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2024-09-12T23:35:06.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:05 smithi136 bash[32189]: cluster 2024-09-12T23:35:03.936304+0000 mgr.y (mgr.24446) 52 : cluster [DBG] pgmap v37: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 318 B/s wr, 0 op/s 2024-09-12T23:35:06.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:05 smithi136 bash[32189]: cluster 2024-09-12T23:35:05.394358+0000 mon.a (mon.0) 819 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2024-09-12T23:35:06.032 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:05 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:05.807906728Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2024-09-12T23:35:06.032 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:05 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:05.833133795Z level=info msg="Executing migration" id="Update temp_user table charset" 2024-09-12T23:35:06.033 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:05 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:05.858348509Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2024-09-12T23:35:06.033 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:05 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:05.883616401Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2024-09-12T23:35:06.033 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:05 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:05.925589689Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2024-09-12T23:35:06.033 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:05 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:05.967810426Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2024-09-12T23:35:06.033 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:05 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:05.99313345Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-09-12T23:35:06.033 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:06 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:06.035215556Z level=info msg="Executing migration" id="create temp_user v2" 2024-09-12T23:35:06.293 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:06 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:06.060376361Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2024-09-12T23:35:06.294 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:06 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:06.102488463Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2024-09-12T23:35:06.294 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:06 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:06.127796115Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2024-09-12T23:35:06.294 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:06 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:06.153023748Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2024-09-12T23:35:06.294 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:06 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:06.178350202Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2024-09-12T23:35:06.294 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:06 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:06.20358947Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2024-09-12T23:35:06.294 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:06 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:06.245473263Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2024-09-12T23:35:06.294 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:06 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:06.270926358Z level=info msg="Executing migration" id="create star table" 2024-09-12T23:35:06.294 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:06 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:06.296208357Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2024-09-12T23:35:06.554 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:06 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:06.338259125Z level=info msg="Executing migration" id="create org table v1" 2024-09-12T23:35:06.554 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:06 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:06.363524275Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2024-09-12T23:35:06.554 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:06 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:06.388760009Z level=info msg="Executing migration" id="create org_user table v1" 2024-09-12T23:35:06.555 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:06 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:06.414000304Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2024-09-12T23:35:06.555 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:06 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:06.439295043Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2024-09-12T23:35:06.555 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:06 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:06.464587277Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2024-09-12T23:35:06.555 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:06 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:06.481430367Z level=info msg="Executing migration" id="Update org table charset" 2024-09-12T23:35:06.555 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:06 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:06.498271663Z level=info msg="Executing migration" id="Update org_user table charset" 2024-09-12T23:35:06.555 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:06 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:06.515212163Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2024-09-12T23:35:06.555 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:06 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:06.557459528Z level=info msg="Executing migration" id="create dashboard table" 2024-09-12T23:35:06.824 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:06 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:06.582806998Z level=info msg="Executing migration" id="add index dashboard.account_id" 2024-09-12T23:35:06.824 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:06 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:06.624830104Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2024-09-12T23:35:06.824 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:06 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:06.666998975Z level=info msg="Executing migration" id="create dashboard_tag table" 2024-09-12T23:35:06.824 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:06 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:06.692275804Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2024-09-12T23:35:06.824 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:06 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:06.717514356Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-09-12T23:35:06.824 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:06 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:06.75939054Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2024-09-12T23:35:06.824 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:06 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:06.801559833Z level=info msg="Executing migration" id="create dashboard v2" 2024-09-12T23:35:06.824 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:06 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:06.826776795Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2024-09-12T23:35:07.096 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:06 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:06.902264811Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2024-09-12T23:35:07.097 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:06 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:06.927552567Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2024-09-12T23:35:07.097 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:06 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:06.952702213Z level=info msg="Executing migration" id="drop table dashboard_v1" 2024-09-12T23:35:07.097 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:06 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:06.978063168Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2024-09-12T23:35:07.097 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:07 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:07.020073846Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2024-09-12T23:35:07.097 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:07 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:07.045357366Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2024-09-12T23:35:07.097 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:07 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:07.070620874Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2024-09-12T23:35:07.371 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:07 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:07.11256929Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2024-09-12T23:35:07.371 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:07 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:07.137970445Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2024-09-12T23:35:07.371 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:07 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:07.163218444Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2024-09-12T23:35:07.371 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:07 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:07.205264921Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2024-09-12T23:35:07.371 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:07 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:07.230573625Z level=info msg="Executing migration" id="Update dashboard table charset" 2024-09-12T23:35:07.371 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:07 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:07.25583358Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2024-09-12T23:35:07.371 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:07 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:07.28109031Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2024-09-12T23:35:07.372 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:07 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:07.306382807Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2024-09-12T23:35:07.372 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:07 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:07.331652592Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2024-09-12T23:35:07.372 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:07 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:07.373610694Z level=info msg="Executing migration" id="Add column uid in dashboard" 2024-09-12T23:35:07.640 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:07 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:07.398814812Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2024-09-12T23:35:07.640 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:07 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:07.424182969Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2024-09-12T23:35:07.640 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:07 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:07.458059121Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2024-09-12T23:35:07.641 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:07 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:07.483322895Z level=info msg="Executing migration" id="Update dashboard title length" 2024-09-12T23:35:07.641 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:07 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:07.508475345Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2024-09-12T23:35:07.641 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:07 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:07.550550616Z level=info msg="Executing migration" id="create dashboard_provisioning" 2024-09-12T23:35:07.641 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:07 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:07.575883223Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-09-12T23:35:07.641 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:07 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:07.601083247Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2024-09-12T23:35:07.641 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:07 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:07.643087579Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-09-12T23:35:07.893 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:07 smithi181 bash[24776]: cluster 2024-09-12T23:35:05.937013+0000 mgr.y (mgr.24446) 53 : cluster [DBG] pgmap v39: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 276 B/s wr, 0 op/s 2024-09-12T23:35:07.894 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:07 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:07.685219992Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-09-12T23:35:07.894 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:07 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:07.702063765Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2024-09-12T23:35:07.894 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:07 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:07.718987093Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2024-09-12T23:35:07.894 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:07 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:07.735987555Z level=info msg="Executing migration" id="Add check_sum column" 2024-09-12T23:35:07.894 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:07 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:07.752863857Z level=info msg="Executing migration" id="Add index for dashboard_title" 2024-09-12T23:35:07.894 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:07 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:07.78651033Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2024-09-12T23:35:07.894 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:07 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:07.803453897Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2024-09-12T23:35:07.894 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:07 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:07.820337486Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2024-09-12T23:35:07.895 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:07 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:07.837400273Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-09-12T23:35:07.895 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:07 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:07.854351781Z level=info msg="Executing migration" id="create data_source table" 2024-09-12T23:35:07.895 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:07 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:07.87124263Z level=info msg="Executing migration" id="add index data_source.account_id" 2024-09-12T23:35:07.895 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:07 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:07.89661641Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2024-09-12T23:35:08.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:07 smithi136 bash[22601]: cluster 2024-09-12T23:35:05.937013+0000 mgr.y (mgr.24446) 53 : cluster [DBG] pgmap v39: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 276 B/s wr, 0 op/s 2024-09-12T23:35:08.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:07 smithi136 bash[32189]: cluster 2024-09-12T23:35:05.937013+0000 mgr.y (mgr.24446) 53 : cluster [DBG] pgmap v39: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 276 B/s wr, 0 op/s 2024-09-12T23:35:08.155 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:07 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:07.92189079Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2024-09-12T23:35:08.155 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:07 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:07.947118235Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2024-09-12T23:35:08.155 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:07 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:07.972338246Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2024-09-12T23:35:08.155 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:08 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:08.01444303Z level=info msg="Executing migration" id="create data_source table v2" 2024-09-12T23:35:08.155 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:08 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:08.039746041Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2024-09-12T23:35:08.155 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:08 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:08.081769093Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2024-09-12T23:35:08.155 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:08 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:08.107032337Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2024-09-12T23:35:08.155 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:08 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:08.132325699Z level=info msg="Executing migration" id="Add column with_credentials" 2024-09-12T23:35:08.156 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:08 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:08.157534004Z level=info msg="Executing migration" id="Add secure json data column" 2024-09-12T23:35:08.407 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:08 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:08.199762743Z level=info msg="Executing migration" id="Update data_source table charset" 2024-09-12T23:35:08.407 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:08 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:08.241735187Z level=info msg="Executing migration" id="Update initial version to 1" 2024-09-12T23:35:08.407 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:08 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:08.283790591Z level=info msg="Executing migration" id="Add read_only data column" 2024-09-12T23:35:08.408 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:08 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:08.309083966Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2024-09-12T23:35:08.408 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:08 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:08.350869818Z level=info msg="Executing migration" id="Update json_data with nulls" 2024-09-12T23:35:08.408 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:08 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:08.376390813Z level=info msg="Executing migration" id="Add uid column" 2024-09-12T23:35:08.408 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:08 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:08.393310115Z level=info msg="Executing migration" id="Update uid value" 2024-09-12T23:35:08.408 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:08 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:08.410153184Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2024-09-12T23:35:08.481 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-12T23:35:08.663 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:08 smithi181 bash[24776]: audit 2024-09-12T23:35:07.949816+0000 mon.a (mon.0) 820 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.b", "id": [7, 2]}]: dispatch 2024-09-12T23:35:08.663 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:08 smithi181 bash[24776]: audit 2024-09-12T23:35:07.950253+0000 mon.a (mon.0) 821 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.9", "id": [4, 1]}]: dispatch 2024-09-12T23:35:08.663 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:08 smithi181 bash[24776]: audit 2024-09-12T23:35:07.950628+0000 mon.a (mon.0) 822 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.10", "id": [0, 1]}]: dispatch 2024-09-12T23:35:08.663 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:08 smithi181 bash[24776]: audit 2024-09-12T23:35:07.951001+0000 mon.a (mon.0) 823 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 2]}]: dispatch 2024-09-12T23:35:08.664 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:08 smithi181 bash[24776]: audit 2024-09-12T23:35:07.951367+0000 mon.a (mon.0) 824 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.30", "id": [5, 2]}]: dispatch 2024-09-12T23:35:08.664 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:08 smithi181 bash[24776]: audit 2024-09-12T23:35:07.951736+0000 mon.a (mon.0) 825 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.31", "id": [4, 2]}]: dispatch 2024-09-12T23:35:08.664 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:08 smithi181 bash[24776]: audit 2024-09-12T23:35:07.952108+0000 mon.a (mon.0) 826 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.34", "id": [0, 3]}]: dispatch 2024-09-12T23:35:08.664 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:08 smithi181 bash[24776]: audit 2024-09-12T23:35:07.952484+0000 mon.a (mon.0) 827 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.37", "id": [5, 1]}]: dispatch 2024-09-12T23:35:08.664 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:08 smithi181 bash[24776]: audit 2024-09-12T23:35:08.003136+0000 mon.a (mon.0) 828 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:08.664 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:08 smithi181 bash[24776]: audit 2024-09-12T23:35:08.005655+0000 mon.a (mon.0) 829 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:35:08.664 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:08 smithi181 bash[24776]: audit 2024-09-12T23:35:08.047105+0000 mon.a (mon.0) 830 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:08.664 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:08 smithi181 bash[24776]: audit 2024-09-12T23:35:08.479649+0000 mon.a (mon.0) 831 : audit [DBG] from='client.? 172.21.15.136:0/4185931315' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-12T23:35:08.664 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:08 smithi181 bash[24776]: audit 2024-09-12T23:35:08.604681+0000 mon.a (mon.0) 832 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:08.664 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:08 smithi181 bash[24776]: audit 2024-09-12T23:35:08.608869+0000 mon.a (mon.0) 833 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:08.665 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:08 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:08.435609804Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2024-09-12T23:35:08.665 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:08 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:08.460914234Z level=info msg="Executing migration" id="create api_key table" 2024-09-12T23:35:08.665 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:08 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:08.478095336Z level=info msg="Executing migration" id="add index api_key.account_id" 2024-09-12T23:35:08.665 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:08 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:08.494699172Z level=info msg="Executing migration" id="add index api_key.key" 2024-09-12T23:35:08.665 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:08 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:08.511608638Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2024-09-12T23:35:08.665 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:08 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:08.545157428Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2024-09-12T23:35:08.665 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:08 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:08.578840917Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2024-09-12T23:35:08.665 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:08 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:08.620865092Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2024-09-12T23:35:08.665 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:08 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:08.663011689Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2024-09-12T23:35:08.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:08 smithi136 bash[22601]: audit 2024-09-12T23:35:07.949816+0000 mon.a (mon.0) 820 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.b", "id": [7, 2]}]: dispatch 2024-09-12T23:35:08.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:08 smithi136 bash[22601]: audit 2024-09-12T23:35:07.950253+0000 mon.a (mon.0) 821 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.9", "id": [4, 1]}]: dispatch 2024-09-12T23:35:08.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:08 smithi136 bash[22601]: audit 2024-09-12T23:35:07.950628+0000 mon.a (mon.0) 822 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.10", "id": [0, 1]}]: dispatch 2024-09-12T23:35:08.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:08 smithi136 bash[22601]: audit 2024-09-12T23:35:07.951001+0000 mon.a (mon.0) 823 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 2]}]: dispatch 2024-09-12T23:35:08.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:08 smithi136 bash[22601]: audit 2024-09-12T23:35:07.951367+0000 mon.a (mon.0) 824 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.30", "id": [5, 2]}]: dispatch 2024-09-12T23:35:08.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:08 smithi136 bash[22601]: audit 2024-09-12T23:35:07.951736+0000 mon.a (mon.0) 825 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.31", "id": [4, 2]}]: dispatch 2024-09-12T23:35:08.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:08 smithi136 bash[22601]: audit 2024-09-12T23:35:07.952108+0000 mon.a (mon.0) 826 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.34", "id": [0, 3]}]: dispatch 2024-09-12T23:35:08.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:08 smithi136 bash[22601]: audit 2024-09-12T23:35:07.952484+0000 mon.a (mon.0) 827 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.37", "id": [5, 1]}]: dispatch 2024-09-12T23:35:08.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:08 smithi136 bash[22601]: audit 2024-09-12T23:35:08.003136+0000 mon.a (mon.0) 828 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:08.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:08 smithi136 bash[22601]: audit 2024-09-12T23:35:08.005655+0000 mon.a (mon.0) 829 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:35:08.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:08 smithi136 bash[22601]: audit 2024-09-12T23:35:08.047105+0000 mon.a (mon.0) 830 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:08.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:08 smithi136 bash[22601]: audit 2024-09-12T23:35:08.479649+0000 mon.a (mon.0) 831 : audit [DBG] from='client.? 172.21.15.136:0/4185931315' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-12T23:35:08.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:08 smithi136 bash[22601]: audit 2024-09-12T23:35:08.604681+0000 mon.a (mon.0) 832 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:08.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:08 smithi136 bash[22601]: audit 2024-09-12T23:35:08.608869+0000 mon.a (mon.0) 833 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:08.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:08 smithi136 bash[32189]: audit 2024-09-12T23:35:07.949816+0000 mon.a (mon.0) 820 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.b", "id": [7, 2]}]: dispatch 2024-09-12T23:35:08.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:08 smithi136 bash[32189]: audit 2024-09-12T23:35:07.950253+0000 mon.a (mon.0) 821 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.9", "id": [4, 1]}]: dispatch 2024-09-12T23:35:08.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:08 smithi136 bash[32189]: audit 2024-09-12T23:35:07.950628+0000 mon.a (mon.0) 822 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.10", "id": [0, 1]}]: dispatch 2024-09-12T23:35:08.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:08 smithi136 bash[32189]: audit 2024-09-12T23:35:07.951001+0000 mon.a (mon.0) 823 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 2]}]: dispatch 2024-09-12T23:35:08.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:08 smithi136 bash[32189]: audit 2024-09-12T23:35:07.951367+0000 mon.a (mon.0) 824 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.30", "id": [5, 2]}]: dispatch 2024-09-12T23:35:08.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:08 smithi136 bash[32189]: audit 2024-09-12T23:35:07.951736+0000 mon.a (mon.0) 825 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.31", "id": [4, 2]}]: dispatch 2024-09-12T23:35:08.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:08 smithi136 bash[32189]: audit 2024-09-12T23:35:07.952108+0000 mon.a (mon.0) 826 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.34", "id": [0, 3]}]: dispatch 2024-09-12T23:35:08.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:08 smithi136 bash[32189]: audit 2024-09-12T23:35:07.952484+0000 mon.a (mon.0) 827 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.37", "id": [5, 1]}]: dispatch 2024-09-12T23:35:08.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:08 smithi136 bash[32189]: audit 2024-09-12T23:35:08.003136+0000 mon.a (mon.0) 828 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:08.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:08 smithi136 bash[32189]: audit 2024-09-12T23:35:08.005655+0000 mon.a (mon.0) 829 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:35:08.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:08 smithi136 bash[32189]: audit 2024-09-12T23:35:08.047105+0000 mon.a (mon.0) 830 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:08.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:08 smithi136 bash[32189]: audit 2024-09-12T23:35:08.479649+0000 mon.a (mon.0) 831 : audit [DBG] from='client.? 172.21.15.136:0/4185931315' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-12T23:35:08.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:08 smithi136 bash[32189]: audit 2024-09-12T23:35:08.604681+0000 mon.a (mon.0) 832 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:08.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:08 smithi136 bash[32189]: audit 2024-09-12T23:35:08.608869+0000 mon.a (mon.0) 833 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:08.948 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:08 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:08.705189998Z level=info msg="Executing migration" id="create api_key table v2" 2024-09-12T23:35:08.948 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:08 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:08.730506512Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2024-09-12T23:35:08.949 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:08 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:08.772663948Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2024-09-12T23:35:08.949 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:08 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:08.814647609Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2024-09-12T23:35:08.949 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:08 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:08.83997291Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2024-09-12T23:35:08.949 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:08 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:08.865226866Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2024-09-12T23:35:08.949 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:08 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:08.907403446Z level=info msg="Executing migration" id="Update api_key table charset" 2024-09-12T23:35:09.164 INFO:teuthology.orchestra.run.smithi136.stdout:{"epoch":20,"flags":0,"active_gid":24446,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6800","nonce":3939504613},{"type":"v1","addr":"172.21.15.136:6801","nonce":3939504613}]},"active_addr":"172.21.15.136:6801/3939504613","active_change":"2024-09-12T23:34:07.892796+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24440,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.136:8443/","prometheus":"http://172.21.15.136:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":55,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.136:0","nonce":1913282058}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.136:0","nonce":3346741202}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.136:0","nonce":1675562321}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.136:0","nonce":4113402351}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.136:0","nonce":3724151703}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.136:0","nonce":3335730259}]}]} 2024-09-12T23:35:09.169 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-12T23:35:09.169 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-12T23:35:09.169 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-12T23:35:09.209 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:08 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:08.951629047Z level=info msg="Executing migration" id="Add expires to api_key table" 2024-09-12T23:35:09.209 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:08 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:08.993935194Z level=info msg="Executing migration" id="Add service account foreign key" 2024-09-12T23:35:09.209 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:09 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:09.052541022Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-09-12T23:35:09.209 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:09 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:09.094642065Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-09-12T23:35:09.209 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:09 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:09.136753975Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-09-12T23:35:09.210 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:09 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:09.162042059Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2024-09-12T23:35:09.210 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:09 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:09.187334095Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2024-09-12T23:35:09.461 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:09 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:09.21253124Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2024-09-12T23:35:09.461 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:09 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:09.254576782Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2024-09-12T23:35:09.461 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:09 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:09.279886432Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-09-12T23:35:09.461 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:09 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:09.321717891Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2024-09-12T23:35:09.461 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:09 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:09.363744908Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2024-09-12T23:35:09.462 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:09 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:09.388923774Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2024-09-12T23:35:09.462 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:09 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:09.439291332Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2024-09-12T23:35:09.462 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:09 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:09.464513677Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2024-09-12T23:35:09.734 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:09 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:09.50662636Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-09-12T23:35:09.734 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:09 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:09.548709783Z level=info msg="Executing migration" id="create quota table v1" 2024-09-12T23:35:09.735 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:09 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:09.574045107Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2024-09-12T23:35:09.735 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:09 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:09.59928735Z level=info msg="Executing migration" id="Update quota table charset" 2024-09-12T23:35:09.735 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:09 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:09.641175364Z level=info msg="Executing migration" id="create plugin_setting table" 2024-09-12T23:35:09.735 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:09 smithi181 bash[24776]: cluster 2024-09-12T23:35:07.937800+0000 mgr.y (mgr.24446) 54 : cluster [DBG] pgmap v40: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 121 B/s wr, 0 op/s 2024-09-12T23:35:09.735 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:09 smithi181 bash[24776]: audit 2024-09-12T23:35:08.665457+0000 mon.a (mon.0) 834 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.b", "id": [7, 2]}]': finished 2024-09-12T23:35:09.735 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:09 smithi181 bash[24776]: audit 2024-09-12T23:35:08.665899+0000 mon.a (mon.0) 835 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.9", "id": [4, 1]}]': finished 2024-09-12T23:35:09.735 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:09 smithi181 bash[24776]: audit 2024-09-12T23:35:08.666297+0000 mon.a (mon.0) 836 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.10", "id": [0, 1]}]': finished 2024-09-12T23:35:09.735 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:09 smithi181 bash[24776]: audit 2024-09-12T23:35:08.666810+0000 mon.a (mon.0) 837 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 2]}]': finished 2024-09-12T23:35:09.736 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:09 smithi181 bash[24776]: audit 2024-09-12T23:35:08.667058+0000 mon.a (mon.0) 838 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.30", "id": [5, 2]}]': finished 2024-09-12T23:35:09.736 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:09 smithi181 bash[24776]: audit 2024-09-12T23:35:08.667282+0000 mon.a (mon.0) 839 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.31", "id": [4, 2]}]': finished 2024-09-12T23:35:09.736 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:09 smithi181 bash[24776]: audit 2024-09-12T23:35:08.667507+0000 mon.a (mon.0) 840 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.34", "id": [0, 3]}]': finished 2024-09-12T23:35:09.736 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:09 smithi181 bash[24776]: audit 2024-09-12T23:35:08.667686+0000 mon.a (mon.0) 841 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.37", "id": [5, 1]}]': finished 2024-09-12T23:35:09.736 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:09 smithi181 bash[24776]: cluster 2024-09-12T23:35:08.667940+0000 mon.a (mon.0) 842 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2024-09-12T23:35:09.736 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:09 smithi181 bash[24776]: audit 2024-09-12T23:35:09.325968+0000 mon.a (mon.0) 843 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:09.736 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:09 smithi181 bash[24776]: audit 2024-09-12T23:35:09.337615+0000 mon.a (mon.0) 844 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:09.736 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:09 smithi181 bash[24776]: audit 2024-09-12T23:35:09.339528+0000 mon.a (mon.0) 845 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:35:09.736 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:09 smithi181 bash[24776]: audit 2024-09-12T23:35:09.341169+0000 mon.a (mon.0) 846 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:35:09.736 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:09 smithi181 bash[24776]: audit 2024-09-12T23:35:09.352205+0000 mon.a (mon.0) 847 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:10.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:09 smithi136 bash[22601]: cluster 2024-09-12T23:35:07.937800+0000 mgr.y (mgr.24446) 54 : cluster [DBG] pgmap v40: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 121 B/s wr, 0 op/s 2024-09-12T23:35:10.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:09 smithi136 bash[22601]: audit 2024-09-12T23:35:08.665457+0000 mon.a (mon.0) 834 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.b", "id": [7, 2]}]': finished 2024-09-12T23:35:10.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:09 smithi136 bash[22601]: audit 2024-09-12T23:35:08.665899+0000 mon.a (mon.0) 835 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.9", "id": [4, 1]}]': finished 2024-09-12T23:35:10.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:09 smithi136 bash[22601]: audit 2024-09-12T23:35:08.666297+0000 mon.a (mon.0) 836 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.10", "id": [0, 1]}]': finished 2024-09-12T23:35:10.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:09 smithi136 bash[22601]: audit 2024-09-12T23:35:08.666810+0000 mon.a (mon.0) 837 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 2]}]': finished 2024-09-12T23:35:10.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:09 smithi136 bash[22601]: audit 2024-09-12T23:35:08.667058+0000 mon.a (mon.0) 838 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.30", "id": [5, 2]}]': finished 2024-09-12T23:35:10.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:09 smithi136 bash[22601]: audit 2024-09-12T23:35:08.667282+0000 mon.a (mon.0) 839 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.31", "id": [4, 2]}]': finished 2024-09-12T23:35:10.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:09 smithi136 bash[22601]: audit 2024-09-12T23:35:08.667507+0000 mon.a (mon.0) 840 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.34", "id": [0, 3]}]': finished 2024-09-12T23:35:10.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:09 smithi136 bash[22601]: audit 2024-09-12T23:35:08.667686+0000 mon.a (mon.0) 841 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.37", "id": [5, 1]}]': finished 2024-09-12T23:35:10.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:09 smithi136 bash[22601]: cluster 2024-09-12T23:35:08.667940+0000 mon.a (mon.0) 842 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2024-09-12T23:35:10.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:09 smithi136 bash[22601]: audit 2024-09-12T23:35:09.325968+0000 mon.a (mon.0) 843 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:10.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:09 smithi136 bash[22601]: audit 2024-09-12T23:35:09.337615+0000 mon.a (mon.0) 844 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:10.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:09 smithi136 bash[22601]: audit 2024-09-12T23:35:09.339528+0000 mon.a (mon.0) 845 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:35:10.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:09 smithi136 bash[22601]: audit 2024-09-12T23:35:09.341169+0000 mon.a (mon.0) 846 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:35:10.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:09 smithi136 bash[22601]: audit 2024-09-12T23:35:09.352205+0000 mon.a (mon.0) 847 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:10.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:09 smithi136 bash[32189]: cluster 2024-09-12T23:35:07.937800+0000 mgr.y (mgr.24446) 54 : cluster [DBG] pgmap v40: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 121 B/s wr, 0 op/s 2024-09-12T23:35:10.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:09 smithi136 bash[32189]: audit 2024-09-12T23:35:08.665457+0000 mon.a (mon.0) 834 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.b", "id": [7, 2]}]': finished 2024-09-12T23:35:10.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:09 smithi136 bash[32189]: audit 2024-09-12T23:35:08.665899+0000 mon.a (mon.0) 835 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.9", "id": [4, 1]}]': finished 2024-09-12T23:35:10.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:09 smithi136 bash[32189]: audit 2024-09-12T23:35:08.666297+0000 mon.a (mon.0) 836 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.10", "id": [0, 1]}]': finished 2024-09-12T23:35:10.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:09 smithi136 bash[32189]: audit 2024-09-12T23:35:08.666810+0000 mon.a (mon.0) 837 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 2]}]': finished 2024-09-12T23:35:10.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:09 smithi136 bash[32189]: audit 2024-09-12T23:35:08.667058+0000 mon.a (mon.0) 838 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.30", "id": [5, 2]}]': finished 2024-09-12T23:35:10.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:09 smithi136 bash[32189]: audit 2024-09-12T23:35:08.667282+0000 mon.a (mon.0) 839 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.31", "id": [4, 2]}]': finished 2024-09-12T23:35:10.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:09 smithi136 bash[32189]: audit 2024-09-12T23:35:08.667507+0000 mon.a (mon.0) 840 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.34", "id": [0, 3]}]': finished 2024-09-12T23:35:10.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:09 smithi136 bash[32189]: audit 2024-09-12T23:35:08.667686+0000 mon.a (mon.0) 841 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.37", "id": [5, 1]}]': finished 2024-09-12T23:35:10.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:09 smithi136 bash[32189]: cluster 2024-09-12T23:35:08.667940+0000 mon.a (mon.0) 842 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2024-09-12T23:35:10.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:09 smithi136 bash[32189]: audit 2024-09-12T23:35:09.325968+0000 mon.a (mon.0) 843 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:10.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:09 smithi136 bash[32189]: audit 2024-09-12T23:35:09.337615+0000 mon.a (mon.0) 844 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:10.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:09 smithi136 bash[32189]: audit 2024-09-12T23:35:09.339528+0000 mon.a (mon.0) 845 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:35:10.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:09 smithi136 bash[32189]: audit 2024-09-12T23:35:09.341169+0000 mon.a (mon.0) 846 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:35:10.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:09 smithi136 bash[32189]: audit 2024-09-12T23:35:09.352205+0000 mon.a (mon.0) 847 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:10.014 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:09 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:09.737329177Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-09-12T23:35:10.015 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:09 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:09.830656493Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2024-09-12T23:35:10.015 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:09 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:09.97148415Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2024-09-12T23:35:10.293 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:10 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:10.017908814Z level=info msg="Executing migration" id="create session table" 2024-09-12T23:35:10.293 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:10 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:10.043671347Z level=info msg="Executing migration" id="Drop old table playlist table" 2024-09-12T23:35:10.293 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:10 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:10.068686526Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2024-09-12T23:35:10.293 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:10 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:10.110708957Z level=info msg="Executing migration" id="create playlist table v2" 2024-09-12T23:35:10.293 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:10 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:10.136007676Z level=info msg="Executing migration" id="create playlist item table v2" 2024-09-12T23:35:10.293 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:10 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:10.161224125Z level=info msg="Executing migration" id="Update playlist table charset" 2024-09-12T23:35:10.293 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:10 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:10.186427648Z level=info msg="Executing migration" id="Update playlist_item table charset" 2024-09-12T23:35:10.293 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:10 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:10.211661031Z level=info msg="Executing migration" id="drop preferences table v2" 2024-09-12T23:35:10.294 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:10 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:10.228520927Z level=info msg="Executing migration" id="drop preferences table v3" 2024-09-12T23:35:10.294 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:10 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:10.262214525Z level=info msg="Executing migration" id="create preferences table v3" 2024-09-12T23:35:10.294 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:10 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:10.295781324Z level=info msg="Executing migration" id="Update preferences table charset" 2024-09-12T23:35:10.579 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:10 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:10.329456098Z level=info msg="Executing migration" id="Add column team_id in preferences" 2024-09-12T23:35:10.580 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:10 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:10.346391724Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2024-09-12T23:35:10.580 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:10 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:10.371639093Z level=info msg="Executing migration" id="Add column week_start in preferences" 2024-09-12T23:35:10.580 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:10 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:10.396863553Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-09-12T23:35:10.580 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:10 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:10.422094699Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-09-12T23:35:10.580 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:10 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:10.447370238Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-09-12T23:35:10.580 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:10 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:10.472590611Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-09-12T23:35:10.580 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:10 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:10.497947932Z level=info msg="Executing migration" id="create alert table v1" 2024-09-12T23:35:10.580 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:10 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:10.539788691Z level=info msg="Executing migration" id="add index alert org_id & id " 2024-09-12T23:35:10.580 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:10 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:10.58220951Z level=info msg="Executing migration" id="add index alert state" 2024-09-12T23:35:10.831 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:10 smithi181 bash[24776]: cephadm 2024-09-12T23:35:09.383848+0000 mgr.y (mgr.24446) 55 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2024-09-12T23:35:10.831 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:10 smithi181 bash[24776]: cephadm 2024-09-12T23:35:09.387317+0000 mgr.y (mgr.24446) 56 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi136 2024-09-12T23:35:10.831 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:10 smithi181 bash[24776]: cluster 2024-09-12T23:35:09.665996+0000 mon.a (mon.0) 848 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2024-09-12T23:35:10.832 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:10 smithi181 bash[24776]: cluster 2024-09-12T23:35:09.938576+0000 mgr.y (mgr.24446) 57 : cluster [DBG] pgmap v43: 105 pgs: 8 peering, 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:35:10.832 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:10 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:10.615582378Z level=info msg="Executing migration" id="add index alert dashboard_id" 2024-09-12T23:35:10.832 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:10 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:10.632481625Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2024-09-12T23:35:10.832 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:10 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:10.666186885Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-09-12T23:35:10.832 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:10 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:10.724956584Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-09-12T23:35:10.832 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:10 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:10.74183957Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-09-12T23:35:10.832 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:10 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:10.783778166Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2024-09-12T23:35:10.832 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:10 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:10.809059641Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-09-12T23:35:10.832 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:10 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:10.834320623Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2024-09-12T23:35:10.992 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:10 smithi136 bash[32189]: cephadm 2024-09-12T23:35:09.383848+0000 mgr.y (mgr.24446) 55 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2024-09-12T23:35:10.993 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:10 smithi136 bash[32189]: cephadm 2024-09-12T23:35:09.387317+0000 mgr.y (mgr.24446) 56 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi136 2024-09-12T23:35:10.993 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:10 smithi136 bash[32189]: cluster 2024-09-12T23:35:09.665996+0000 mon.a (mon.0) 848 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2024-09-12T23:35:10.993 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:10 smithi136 bash[32189]: cluster 2024-09-12T23:35:09.938576+0000 mgr.y (mgr.24446) 57 : cluster [DBG] pgmap v43: 105 pgs: 8 peering, 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:35:10.993 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:10 smithi136 bash[22601]: cephadm 2024-09-12T23:35:09.383848+0000 mgr.y (mgr.24446) 55 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2024-09-12T23:35:10.993 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:10 smithi136 bash[22601]: cephadm 2024-09-12T23:35:09.387317+0000 mgr.y (mgr.24446) 56 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi136 2024-09-12T23:35:10.993 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:10 smithi136 bash[22601]: cluster 2024-09-12T23:35:09.665996+0000 mon.a (mon.0) 848 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2024-09-12T23:35:10.993 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:10 smithi136 bash[22601]: cluster 2024-09-12T23:35:09.938576+0000 mgr.y (mgr.24446) 57 : cluster [DBG] pgmap v43: 105 pgs: 8 peering, 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:35:11.096 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:10 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:10.85951949Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2024-09-12T23:35:11.096 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:10 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:10.901918095Z level=info msg="Executing migration" id="create alert_notification table v1" 2024-09-12T23:35:11.096 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:10 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:10.927136558Z level=info msg="Executing migration" id="Add column is_default" 2024-09-12T23:35:11.096 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:10 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:10.952356914Z level=info msg="Executing migration" id="Add column frequency" 2024-09-12T23:35:11.097 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:10 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:10.977622283Z level=info msg="Executing migration" id="Add column send_reminder" 2024-09-12T23:35:11.097 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:11 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:11.002830943Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2024-09-12T23:35:11.097 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:11 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:11.044760975Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2024-09-12T23:35:11.097 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:11 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:11.07010634Z level=info msg="Executing migration" id="Update alert table charset" 2024-09-12T23:35:11.378 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:11 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:11.11196388Z level=info msg="Executing migration" id="Update alert_notification table charset" 2024-09-12T23:35:11.378 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:11 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:11.137220817Z level=info msg="Executing migration" id="create notification_journal table v1" 2024-09-12T23:35:11.379 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:11 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:11.162519699Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2024-09-12T23:35:11.379 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:11 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:11.187773692Z level=info msg="Executing migration" id="drop alert_notification_journal" 2024-09-12T23:35:11.379 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:11 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:11.212941329Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2024-09-12T23:35:11.379 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:11 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:11.238183962Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-09-12T23:35:11.379 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:11 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:11.280146671Z level=info msg="Executing migration" id="Add for to alert table" 2024-09-12T23:35:11.379 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:11 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:11.305433607Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2024-09-12T23:35:11.379 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:11 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:11.33067543Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2024-09-12T23:35:11.379 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:11 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:11.355919726Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2024-09-12T23:35:11.379 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:11 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:11.381220995Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2024-09-12T23:35:11.656 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:11 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:11.40638233Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2024-09-12T23:35:11.656 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:11 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:11.431648632Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2024-09-12T23:35:11.656 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:11 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:11.456856983Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2024-09-12T23:35:11.656 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:11 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:11.482138844Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2024-09-12T23:35:11.656 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:11 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:11.507410678Z level=info msg="Executing migration" id="Drop old annotation table v4" 2024-09-12T23:35:11.656 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:11 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:11.549379133Z level=info msg="Executing migration" id="create annotation table v5" 2024-09-12T23:35:11.656 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:11 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:11.591417538Z level=info msg="Executing migration" id="add index annotation 0 v3" 2024-09-12T23:35:11.656 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:11 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:11.616522638Z level=info msg="Executing migration" id="add index annotation 1 v3" 2024-09-12T23:35:11.656 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:11 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:11.658569297Z level=info msg="Executing migration" id="add index annotation 2 v3" 2024-09-12T23:35:11.908 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:11 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:11.683931058Z level=info msg="Executing migration" id="add index annotation 3 v3" 2024-09-12T23:35:11.908 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:11 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:11.709143925Z level=info msg="Executing migration" id="add index annotation 4 v3" 2024-09-12T23:35:11.908 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:11 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:11.7510836Z level=info msg="Executing migration" id="Update annotation table charset" 2024-09-12T23:35:11.908 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:11 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:11.776336767Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2024-09-12T23:35:11.908 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:11 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:11.801550067Z level=info msg="Executing migration" id="Drop category_id index" 2024-09-12T23:35:11.908 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:11 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:11.835457752Z level=info msg="Executing migration" id="Add column tags to annotation table" 2024-09-12T23:35:11.908 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:11 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:11.86057634Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2024-09-12T23:35:11.909 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:11 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:11.885936762Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2024-09-12T23:35:12.177 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:11 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:11.9111116Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-09-12T23:35:12.178 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:11 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:11.936365825Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-09-12T23:35:12.178 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:11 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:11.969945732Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2024-09-12T23:35:12.178 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:12 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:12.011935374Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-09-12T23:35:12.178 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:12 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:12.028843617Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2024-09-12T23:35:12.178 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:12 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:12.062410921Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2024-09-12T23:35:12.178 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:12 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:12.079282345Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2024-09-12T23:35:12.178 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:12 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:12.113000564Z level=info msg="Executing migration" id="Add created time to annotation table" 2024-09-12T23:35:12.178 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:12 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:12.130007153Z level=info msg="Executing migration" id="Add updated time to annotation table" 2024-09-12T23:35:12.178 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:12 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:12.155230686Z level=info msg="Executing migration" id="Add index for created in annotation table" 2024-09-12T23:35:12.178 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:12 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:12.180417187Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2024-09-12T23:35:12.438 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:12 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:12.222301706Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2024-09-12T23:35:12.438 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:12 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:12.26426484Z level=info msg="Executing migration" id="Add epoch_end column" 2024-09-12T23:35:12.438 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:12 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:12.297946319Z level=info msg="Executing migration" id="Add index for epoch_end" 2024-09-12T23:35:12.438 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:12 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:12.339923885Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2024-09-12T23:35:12.438 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:12 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:12.365135806Z level=info msg="Executing migration" id="Move region to single row" 2024-09-12T23:35:12.438 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:12 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:12.390441968Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2024-09-12T23:35:12.438 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:12 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:12.415646586Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-09-12T23:35:12.439 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:12 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:12.44085Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-09-12T23:35:12.715 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:12 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:12.46611447Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2024-09-12T23:35:12.715 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:12 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:12.49141415Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2024-09-12T23:35:12.716 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:12 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:12.51665059Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2024-09-12T23:35:12.716 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:12 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:12.541900263Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-09-12T23:35:12.716 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:12 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:12.567187932Z level=info msg="Executing migration" id="create test_data table" 2024-09-12T23:35:12.716 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:12 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:12.609090507Z level=info msg="Executing migration" id="create dashboard_version table v1" 2024-09-12T23:35:12.716 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:12 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:12.651067906Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2024-09-12T23:35:12.716 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:12 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:12.676355739Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-09-12T23:35:12.716 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:12 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:12.718301478Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2024-09-12T23:35:12.985 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:12 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:12.743496299Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2024-09-12T23:35:12.985 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:12 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:12.777350866Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2024-09-12T23:35:12.985 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:12 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:12.802616123Z level=info msg="Executing migration" id="create team table" 2024-09-12T23:35:12.985 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:12 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:12.827876102Z level=info msg="Executing migration" id="add index team.org_id" 2024-09-12T23:35:12.985 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:12 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:12.853126452Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2024-09-12T23:35:12.985 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:12 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:12.894961881Z level=info msg="Executing migration" id="create team member table" 2024-09-12T23:35:12.986 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:12 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:12.937175313Z level=info msg="Executing migration" id="add index team_member.org_id" 2024-09-12T23:35:12.986 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:12 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:12.96244914Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2024-09-12T23:35:12.986 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:12 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:12.987732546Z level=info msg="Executing migration" id="add index team_member.team_id" 2024-09-12T23:35:13.237 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:12 smithi181 bash[24776]: cluster 2024-09-12T23:35:11.939028+0000 mgr.y (mgr.24446) 58 : cluster [DBG] pgmap v44: 105 pgs: 8 peering, 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:35:13.237 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:13 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:13.029830794Z level=info msg="Executing migration" id="Add column email to team table" 2024-09-12T23:35:13.237 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:13 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:13.054981543Z level=info msg="Executing migration" id="Add column external to team_member table" 2024-09-12T23:35:13.237 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:13 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:13.080310069Z level=info msg="Executing migration" id="Add column permission to team_member table" 2024-09-12T23:35:13.237 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:13 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:13.105556133Z level=info msg="Executing migration" id="create dashboard acl table" 2024-09-12T23:35:13.237 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:13 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:13.130863683Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2024-09-12T23:35:13.238 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:13 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:13.156130337Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2024-09-12T23:35:13.238 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:13 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:13.19809691Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2024-09-12T23:35:13.238 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:13 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:13.240004385Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2024-09-12T23:35:13.255 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:12 smithi136 bash[22601]: cluster 2024-09-12T23:35:11.939028+0000 mgr.y (mgr.24446) 58 : cluster [DBG] pgmap v44: 105 pgs: 8 peering, 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:35:13.256 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:12 smithi136 bash[32189]: cluster 2024-09-12T23:35:11.939028+0000 mgr.y (mgr.24446) 58 : cluster [DBG] pgmap v44: 105 pgs: 8 peering, 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:35:13.256 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:35:12 smithi136 systemd[1]: Stopping Ceph alertmanager.a for da58a47c-715d-11ef-bceb-c7b262605968... 2024-09-12T23:35:13.256 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:35:13 smithi136 bash[58363]: ts=2024-09-12T23:35:13.118Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-12T23:35:13.498 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:13 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:13.265310115Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2024-09-12T23:35:13.498 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:13 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:13.307136713Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2024-09-12T23:35:13.498 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:13 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:13.33245792Z level=info msg="Executing migration" id="add index dashboard_permission" 2024-09-12T23:35:13.498 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:13 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:13.357655587Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2024-09-12T23:35:13.498 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:13 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:13.382997106Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2024-09-12T23:35:13.498 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:13 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:13.408132532Z level=info msg="Executing migration" id="create tag table" 2024-09-12T23:35:13.498 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:13 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:13.433448092Z level=info msg="Executing migration" id="add index tag.key_value" 2024-09-12T23:35:13.498 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:13 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:13.458678083Z level=info msg="Executing migration" id="create login attempt table" 2024-09-12T23:35:13.498 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:13 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:13.500624376Z level=info msg="Executing migration" id="add index login_attempt.username" 2024-09-12T23:35:13.707 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:35:13 smithi136 bash[59999]: ceph-da58a47c-715d-11ef-bceb-c7b262605968-alertmanager-a 2024-09-12T23:35:13.750 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:13 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:13.542650536Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2024-09-12T23:35:13.750 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:13 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:13.567927119Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-09-12T23:35:13.750 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:13 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:13.601465721Z level=info msg="Executing migration" id="create login_attempt v2" 2024-09-12T23:35:13.750 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:13 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:13.643420214Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2024-09-12T23:35:13.750 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:13 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:13.685754387Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2024-09-12T23:35:13.750 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:13 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:13.710942991Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2024-09-12T23:35:14.006 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:35:13 smithi136 bash[60058]: Error response from daemon: No such container: ceph-da58a47c-715d-11ef-bceb-c7b262605968-alertmanager-a 2024-09-12T23:35:14.006 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:35:13 smithi136 systemd[1]: ceph-da58a47c-715d-11ef-bceb-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-09-12T23:35:14.006 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:35:13 smithi136 systemd[1]: Stopped Ceph alertmanager.a for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:35:14.006 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:35:13 smithi136 systemd[1]: Started Ceph alertmanager.a for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:35:14.061 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:13 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:13.752987074Z level=info msg="Executing migration" id="create user auth table" 2024-09-12T23:35:14.061 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:13 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:13.794962697Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-09-12T23:35:14.061 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:13 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:13.820227936Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2024-09-12T23:35:14.062 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:13 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:13.845540637Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2024-09-12T23:35:14.062 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:13 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:13.870866664Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2024-09-12T23:35:14.062 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:13 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:13.912927932Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2024-09-12T23:35:14.062 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:13 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:13.954935479Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2024-09-12T23:35:14.062 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:13 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:13.996668316Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2024-09-12T23:35:14.062 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:14 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:14.064386072Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-09-12T23:35:14.144 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:35:14.351 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:14 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:14.312329162Z level=info msg="Executing migration" id="create server_lock table" 2024-09-12T23:35:14.724 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:14 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:14.41027055Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2024-09-12T23:35:14.982 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:14 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:14.727010767Z level=info msg="Executing migration" id="create user auth token table" 2024-09-12T23:35:14.982 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:14 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:14.924835191Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2024-09-12T23:35:14.982 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:14 smithi181 bash[24776]: audit 2024-09-12T23:35:13.757734+0000 mon.a (mon.0) 849 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:14.983 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:14 smithi181 bash[24776]: audit 2024-09-12T23:35:13.763565+0000 mon.a (mon.0) 850 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:14.983 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:14 smithi181 bash[24776]: cephadm 2024-09-12T23:35:13.765058+0000 mgr.y (mgr.24446) 59 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-09-12T23:35:14.983 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:14 smithi181 bash[24776]: cephadm 2024-09-12T23:35:13.773579+0000 mgr.y (mgr.24446) 60 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi181 2024-09-12T23:35:14.983 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:14 smithi181 bash[24776]: cluster 2024-09-12T23:35:13.940150+0000 mgr.y (mgr.24446) 61 : cluster [DBG] pgmap v45: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-12T23:35:15.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:14 smithi136 bash[22601]: audit 2024-09-12T23:35:13.757734+0000 mon.a (mon.0) 849 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:15.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:14 smithi136 bash[22601]: audit 2024-09-12T23:35:13.763565+0000 mon.a (mon.0) 850 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:15.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:14 smithi136 bash[22601]: cephadm 2024-09-12T23:35:13.765058+0000 mgr.y (mgr.24446) 59 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-09-12T23:35:15.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:14 smithi136 bash[22601]: cephadm 2024-09-12T23:35:13.773579+0000 mgr.y (mgr.24446) 60 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi181 2024-09-12T23:35:15.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:14 smithi136 bash[22601]: cluster 2024-09-12T23:35:13.940150+0000 mgr.y (mgr.24446) 61 : cluster [DBG] pgmap v45: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-12T23:35:15.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:14 smithi136 bash[32189]: audit 2024-09-12T23:35:13.757734+0000 mon.a (mon.0) 849 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:15.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:14 smithi136 bash[32189]: audit 2024-09-12T23:35:13.763565+0000 mon.a (mon.0) 850 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:15.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:14 smithi136 bash[32189]: cephadm 2024-09-12T23:35:13.765058+0000 mgr.y (mgr.24446) 59 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-09-12T23:35:15.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:14 smithi136 bash[32189]: cephadm 2024-09-12T23:35:13.773579+0000 mgr.y (mgr.24446) 60 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi181 2024-09-12T23:35:15.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:14 smithi136 bash[32189]: cluster 2024-09-12T23:35:13.940150+0000 mgr.y (mgr.24446) 61 : cluster [DBG] pgmap v45: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-12T23:35:15.346 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:35:15 smithi181 bash[25717]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:35:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-12T23:35:15.347 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:14 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:14.98491575Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2024-09-12T23:35:15.347 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:15 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:15.069797282Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2024-09-12T23:35:15.642 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:35:15 smithi136 bash[60087]: ts=2024-09-12T23:35:15.494Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-09-12T23:35:15.642 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:35:15 smithi136 bash[60087]: ts=2024-09-12T23:35:15.494Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-09-12T23:35:15.643 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:35:15 smithi136 bash[60087]: ts=2024-09-12T23:35:15.496Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.136 port=9094 2024-09-12T23:35:15.643 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:35:15 smithi136 bash[60087]: ts=2024-09-12T23:35:15.497Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-12T23:35:15.643 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:35:15 smithi136 bash[60087]: ts=2024-09-12T23:35:15.528Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-12T23:35:15.643 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:35:15 smithi136 bash[60087]: ts=2024-09-12T23:35:15.529Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-12T23:35:15.643 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:35:15 smithi136 bash[60087]: ts=2024-09-12T23:35:15.532Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-09-12T23:35:15.643 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:35:15 smithi136 bash[60087]: ts=2024-09-12T23:35:15.532Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-09-12T23:35:15.804 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:15 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:15.471629354Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2024-09-12T23:35:15.804 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:15 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:15.621443392Z level=info msg="Executing migration" id="create cache_data table" 2024-09-12T23:35:15.804 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:15 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:15.714010962Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2024-09-12T23:35:16.088 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:15 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:15.806819464Z level=info msg="Executing migration" id="create short_url table v1" 2024-09-12T23:35:16.088 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:15 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:15.898431978Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2024-09-12T23:35:16.088 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:15 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:15.948387622Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-09-12T23:35:16.088 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:16 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:16.007319305Z level=info msg="Executing migration" id="delete alert_definition table" 2024-09-12T23:35:16.346 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:16 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:16.091180854Z level=info msg="Executing migration" id="recreate alert_definition table" 2024-09-12T23:35:16.346 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:16 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:16.141577105Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2024-09-12T23:35:16.346 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:16 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:16.183813503Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2024-09-12T23:35:16.346 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:16 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:16.234239446Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2024-09-12T23:35:16.347 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:16 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:16.301617997Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2024-09-12T23:35:16.683 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:16 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:16.352365859Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2024-09-12T23:35:16.683 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:16 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:16.428059936Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2024-09-12T23:35:16.683 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:16 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:16.520005613Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2024-09-12T23:35:16.951 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:16 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:16.686204577Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2024-09-12T23:35:17.213 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:16 smithi181 bash[24776]: cluster 2024-09-12T23:35:15.940893+0000 mgr.y (mgr.24446) 62 : cluster [DBG] pgmap v46: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-12T23:35:17.213 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:16 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:16.954748624Z level=info msg="Executing migration" id="drop alert_definition table" 2024-09-12T23:35:17.213 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:17 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:17.04731432Z level=info msg="Executing migration" id="delete alert_definition_version table" 2024-09-12T23:35:17.213 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:17 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:17.114643248Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2024-09-12T23:35:17.213 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:17 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:17.215659703Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-09-12T23:35:17.244 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:16 smithi136 bash[22601]: cluster 2024-09-12T23:35:15.940893+0000 mgr.y (mgr.24446) 62 : cluster [DBG] pgmap v46: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-12T23:35:17.245 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:16 smithi136 bash[32189]: cluster 2024-09-12T23:35:15.940893+0000 mgr.y (mgr.24446) 62 : cluster [DBG] pgmap v46: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-12T23:35:17.506 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:35:17 smithi136 bash[60087]: ts=2024-09-12T23:35:17.497Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.0001713s 2024-09-12T23:35:17.544 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-12T23:35:17.544 INFO:teuthology.orchestra.run.smithi136.stdout:{"epoch":64,"fsid":"da58a47c-715d-11ef-bceb-c7b262605968","created":"2024-09-12T23:23:29.000926+0000","modified":"2024-09-12T23:35:09.657947+0000","last_up_change":"2024-09-12T23:32:43.859472+0000","last_in_change":"2024-09-12T23:32:18.995528+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-12T23:28:33.956985+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-09-12T23:33:27.478434+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-09-12T23:33:28.566349+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"54","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}},{"pool":4,"pool_name":"rbd","create_time":"2024-09-12T23:34:48.404257+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"61","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":61,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"87aedf04-695e-4a8f-80c3-ffd18883a4d5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6802","nonce":4251571952},{"type":"v1","addr":"172.21.15.136:6803","nonce":4251571952}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6804","nonce":4251571952},{"type":"v1","addr":"172.21.15.136:6805","nonce":4251571952}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6808","nonce":4251571952},{"type":"v1","addr":"172.21.15.136:6809","nonce":4251571952}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6806","nonce":4251571952},{"type":"v1","addr":"172.21.15.136:6807","nonce":4251571952}]},"public_addr":"172.21.15.136:6803/4251571952","cluster_addr":"172.21.15.136:6805/4251571952","heartbeat_back_addr":"172.21.15.136:6809/4251571952","heartbeat_front_addr":"172.21.15.136:6807/4251571952","state":["exists","up"]},{"osd":1,"uuid":"850d4f98-0640-4151-bd45-a351ca41d54e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":63,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6810","nonce":3482177958},{"type":"v1","addr":"172.21.15.136:6811","nonce":3482177958}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6812","nonce":3482177958},{"type":"v1","addr":"172.21.15.136:6813","nonce":3482177958}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6816","nonce":3482177958},{"type":"v1","addr":"172.21.15.136:6817","nonce":3482177958}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6814","nonce":3482177958},{"type":"v1","addr":"172.21.15.136:6815","nonce":3482177958}]},"public_addr":"172.21.15.136:6811/3482177958","cluster_addr":"172.21.15.136:6813/3482177958","heartbeat_back_addr":"172.21.15.136:6817/3482177958","heartbeat_front_addr":"172.21.15.136:6815/3482177958","state":["exists","up"]},{"osd":2,"uuid":"bde707d6-e027-4b63-9853-fd59212eea1e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":63,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6818","nonce":308477304},{"type":"v1","addr":"172.21.15.136:6819","nonce":308477304}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6820","nonce":308477304},{"type":"v1","addr":"172.21.15.136:6821","nonce":308477304}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6824","nonce":308477304},{"type":"v1","addr":"172.21.15.136:6825","nonce":308477304}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6822","nonce":308477304},{"type":"v1","addr":"172.21.15.136:6823","nonce":308477304}]},"public_addr":"172.21.15.136:6819/308477304","cluster_addr":"172.21.15.136:6821/308477304","heartbeat_back_addr":"172.21.15.136:6825/308477304","heartbeat_front_addr":"172.21.15.136:6823/308477304","state":["exists","up"]},{"osd":3,"uuid":"d61f8357-b89b-4c05-b366-65a9f788064d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6826","nonce":201948269},{"type":"v1","addr":"172.21.15.136:6827","nonce":201948269}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6828","nonce":201948269},{"type":"v1","addr":"172.21.15.136:6829","nonce":201948269}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6832","nonce":201948269},{"type":"v1","addr":"172.21.15.136:6833","nonce":201948269}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6830","nonce":201948269},{"type":"v1","addr":"172.21.15.136:6831","nonce":201948269}]},"public_addr":"172.21.15.136:6827/201948269","cluster_addr":"172.21.15.136:6829/201948269","heartbeat_back_addr":"172.21.15.136:6833/201948269","heartbeat_front_addr":"172.21.15.136:6831/201948269","state":["exists","up"]},{"osd":4,"uuid":"98b0e81c-824e-4dbc-9faa-2de9f199af1b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":63,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6800","nonce":775251087},{"type":"v1","addr":"172.21.15.181:6801","nonce":775251087}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6802","nonce":775251087},{"type":"v1","addr":"172.21.15.181:6803","nonce":775251087}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6806","nonce":775251087},{"type":"v1","addr":"172.21.15.181:6807","nonce":775251087}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6804","nonce":775251087},{"type":"v1","addr":"172.21.15.181:6805","nonce":775251087}]},"public_addr":"172.21.15.181:6801/775251087","cluster_addr":"172.21.15.181:6803/775251087","heartbeat_back_addr":"172.21.15.181:6807/775251087","heartbeat_front_addr":"172.21.15.181:6805/775251087","state":["exists","up"]},{"osd":5,"uuid":"5c6d2f69-7f9c-4689-95b0-bd4eb858956d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6808","nonce":2109931558},{"type":"v1","addr":"172.21.15.181:6809","nonce":2109931558}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6810","nonce":2109931558},{"type":"v1","addr":"172.21.15.181:6811","nonce":2109931558}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6814","nonce":2109931558},{"type":"v1","addr":"172.21.15.181:6815","nonce":2109931558}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6812","nonce":2109931558},{"type":"v1","addr":"172.21.15.181:6813","nonce":2109931558}]},"public_addr":"172.21.15.181:6809/2109931558","cluster_addr":"172.21.15.181:6811/2109931558","heartbeat_back_addr":"172.21.15.181:6815/2109931558","heartbeat_front_addr":"172.21.15.181:6813/2109931558","state":["exists","up"]},{"osd":6,"uuid":"532ee052-af15-433a-8e93-b0b642aaf85b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":63,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6816","nonce":4099148413},{"type":"v1","addr":"172.21.15.181:6817","nonce":4099148413}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6818","nonce":4099148413},{"type":"v1","addr":"172.21.15.181:6819","nonce":4099148413}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6822","nonce":4099148413},{"type":"v1","addr":"172.21.15.181:6823","nonce":4099148413}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6820","nonce":4099148413},{"type":"v1","addr":"172.21.15.181:6821","nonce":4099148413}]},"public_addr":"172.21.15.181:6817/4099148413","cluster_addr":"172.21.15.181:6819/4099148413","heartbeat_back_addr":"172.21.15.181:6823/4099148413","heartbeat_front_addr":"172.21.15.181:6821/4099148413","state":["exists","up"]},{"osd":7,"uuid":"d2ff80f0-6ef0-4098-877c-a891d48c687d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":63,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6824","nonce":783620875},{"type":"v1","addr":"172.21.15.181:6825","nonce":783620875}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6826","nonce":783620875},{"type":"v1","addr":"172.21.15.181:6827","nonce":783620875}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6830","nonce":783620875},{"type":"v1","addr":"172.21.15.181:6831","nonce":783620875}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6828","nonce":783620875},{"type":"v1","addr":"172.21.15.181:6829","nonce":783620875}]},"public_addr":"172.21.15.181:6825/783620875","cluster_addr":"172.21.15.181:6827/783620875","heartbeat_back_addr":"172.21.15.181:6831/783620875","heartbeat_front_addr":"172.21.15.181:6829/783620875","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:26:50.657580+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:27:40.536476+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:28:31.755012+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:29:23.585451+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:30:11.888258+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:31:01.812765+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:31:52.166696+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:32:42.696525+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.b","mappings":[{"from":7,"to":2}]},{"pgid":"3.9","mappings":[{"from":4,"to":1}]},{"pgid":"3.10","mappings":[{"from":0,"to":1}]},{"pgid":"3.14","mappings":[{"from":4,"to":2}]},{"pgid":"3.30","mappings":[{"from":5,"to":2}]},{"pgid":"3.31","mappings":[{"from":4,"to":2}]},{"pgid":"3.34","mappings":[{"from":0,"to":3}]},{"pgid":"3.37","mappings":[{"from":5,"to":1}]}],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.136:0/129641967":"2024-09-13T23:34:07.892265+0000","172.21.15.136:6801/1810367175":"2024-09-13T23:34:07.892265+0000","172.21.15.136:6800/1810367175":"2024-09-13T23:34:07.892265+0000","172.21.15.136:6800/2894181186":"2024-09-13T23:23:54.847045+0000","172.21.15.136:0/1964283041":"2024-09-13T23:24:25.346823+0000","172.21.15.136:0/3924275900":"2024-09-13T23:34:07.892265+0000","172.21.15.136:6801/2894181186":"2024-09-13T23:23:54.847045+0000","172.21.15.136:0/513809002":"2024-09-13T23:34:07.892265+0000","172.21.15.136:0/3910468981":"2024-09-13T23:23:54.847045+0000","172.21.15.136:0/1549678324":"2024-09-13T23:23:54.847045+0000","172.21.15.136:6800/3594272622":"2024-09-13T23:24:25.346823+0000","172.21.15.136:0/209539205":"2024-09-13T23:24:25.346823+0000","172.21.15.136:6801/3594272622":"2024-09-13T23:24:25.346823+0000","172.21.15.136:0/4260962972":"2024-09-13T23:24:25.346823+0000","172.21.15.136:0/3376519841":"2024-09-13T23:23:54.847045+0000","172.21.15.136:0/3642432358":"2024-09-13T23:34:07.892265+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-12T23:35:17.565 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:17 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:17.275076959Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-09-12T23:35:17.565 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:17 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:17.34186579Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2024-09-12T23:35:17.846 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:17 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:17.568057782Z level=info msg="Executing migration" id="drop alert_definition_version table" 2024-09-12T23:35:17.846 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:17 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:17.798080307Z level=info msg="Executing migration" id="create alert_instance table" 2024-09-12T23:35:18.133 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-12T23:35:18.134 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-12T23:35:18.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:17 smithi136 bash[22601]: audit 2024-09-12T23:35:17.546913+0000 mon.c (mon.2) 31 : audit [DBG] from='client.? 172.21.15.136:0/2525546215' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-12T23:35:18.256 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:17 smithi136 bash[32189]: audit 2024-09-12T23:35:17.546913+0000 mon.c (mon.2) 31 : audit [DBG] from='client.? 172.21.15.136:0/2525546215' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-12T23:35:18.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:17 smithi181 bash[24776]: audit 2024-09-12T23:35:17.546913+0000 mon.c (mon.2) 31 : audit [DBG] from='client.? 172.21.15.136:0/2525546215' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-12T23:35:18.346 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:17 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:17.961738715Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-09-12T23:35:18.346 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:18 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:18.046224202Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2024-09-12T23:35:18.696 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:18 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:18.358240823Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2024-09-12T23:35:18.696 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:18 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:18.505923534Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-09-12T23:35:18.696 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:18 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:18.597969712Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2024-09-12T23:35:18.963 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:18 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:18.69900923Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2024-09-12T23:35:18.964 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:18 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:18.808090956Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2024-09-12T23:35:18.964 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:18 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:18.883658887Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-09-12T23:35:18.964 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:18 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:18.949214456Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2024-09-12T23:35:19.245 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:19 smithi181 bash[24776]: cluster 2024-09-12T23:35:17.941600+0000 mgr.y (mgr.24446) 63 : cluster [DBG] pgmap v47: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-12T23:35:19.245 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:19 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:19.001174046Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-09-12T23:35:19.245 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:19 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:19.052474155Z level=info msg="Executing migration" id="create alert_rule table" 2024-09-12T23:35:19.245 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:19 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:19.128129931Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2024-09-12T23:35:19.246 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:19 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:19.169988228Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2024-09-12T23:35:19.246 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:19 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:19.248152625Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-09-12T23:35:19.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:19 smithi136 bash[22601]: cluster 2024-09-12T23:35:17.941600+0000 mgr.y (mgr.24446) 63 : cluster [DBG] pgmap v47: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-12T23:35:19.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:19 smithi136 bash[32189]: cluster 2024-09-12T23:35:17.941600+0000 mgr.y (mgr.24446) 63 : cluster [DBG] pgmap v47: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-12T23:35:19.596 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:19 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:19.331207096Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2024-09-12T23:35:19.596 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:19 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:19.431123457Z level=info msg="Executing migration" id="add column for to alert_rule" 2024-09-12T23:35:19.953 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:19 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:19.69927709Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2024-09-12T23:35:20.210 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:19 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:19.956297918Z level=info msg="Executing migration" id="add column labels to alert_rule" 2024-09-12T23:35:20.210 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:20 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:20.040494285Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2024-09-12T23:35:20.210 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:20 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:20.116254482Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-09-12T23:35:20.210 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:20 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:20.192259764Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2024-09-12T23:35:20.479 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:20 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:20.23421587Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2024-09-12T23:35:20.479 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:20 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:20.286927898Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-09-12T23:35:20.479 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:20 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:20.352230763Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-09-12T23:35:20.479 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:20 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:20.394561298Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-09-12T23:35:20.479 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:20 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:20.43644001Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-09-12T23:35:20.480 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:35:20 smithi181 systemd[1]: Stopping Ceph prometheus.a for da58a47c-715d-11ef-bceb-c7b262605968... 2024-09-12T23:35:20.480 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:35:20 smithi181 bash[49069]: ts=2024-09-12T23:35:20.378Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-12T23:35:20.480 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:35:20 smithi181 bash[49069]: ts=2024-09-12T23:35:20.378Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-12T23:35:20.480 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:35:20 smithi181 bash[49069]: ts=2024-09-12T23:35:20.378Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-12T23:35:20.480 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:35:20 smithi181 bash[49069]: ts=2024-09-12T23:35:20.378Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-12T23:35:20.480 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:35:20 smithi181 bash[49069]: ts=2024-09-12T23:35:20.378Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-12T23:35:20.480 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:35:20 smithi181 bash[49069]: ts=2024-09-12T23:35:20.378Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-12T23:35:20.480 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:35:20 smithi181 bash[49069]: ts=2024-09-12T23:35:20.378Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-12T23:35:20.480 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:35:20 smithi181 bash[49069]: ts=2024-09-12T23:35:20.378Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-12T23:35:20.481 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:35:20 smithi181 bash[49069]: ts=2024-09-12T23:35:20.378Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-12T23:35:20.481 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:35:20 smithi181 bash[49069]: ts=2024-09-12T23:35:20.379Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-12T23:35:20.481 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:35:20 smithi181 bash[49069]: ts=2024-09-12T23:35:20.379Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-12T23:35:20.481 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:35:20 smithi181 bash[49069]: ts=2024-09-12T23:35:20.379Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-12T23:35:20.846 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:20 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:20.482421503Z level=info msg="Executing migration" id="create alert_rule_version table" 2024-09-12T23:35:20.846 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:20 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:20.563821894Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-09-12T23:35:20.846 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:20 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:20.655495962Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-09-12T23:35:21.203 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:21 smithi181 bash[24776]: cluster 2024-09-12T23:35:19.942926+0000 mgr.y (mgr.24446) 64 : cluster [DBG] pgmap v48: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2024-09-12T23:35:21.204 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:20 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:20.880055024Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2024-09-12T23:35:21.464 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:35:21 smithi181 bash[51400]: ceph-da58a47c-715d-11ef-bceb-c7b262605968-prometheus-a 2024-09-12T23:35:21.464 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:21 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:21.206289057Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2024-09-12T23:35:21.464 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:21 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:21.357564689Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2024-09-12T23:35:21.464 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:21 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:21.424753663Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2024-09-12T23:35:21.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:21 smithi136 bash[22601]: cluster 2024-09-12T23:35:19.942926+0000 mgr.y (mgr.24446) 64 : cluster [DBG] pgmap v48: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2024-09-12T23:35:21.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:21 smithi136 bash[32189]: cluster 2024-09-12T23:35:19.942926+0000 mgr.y (mgr.24446) 64 : cluster [DBG] pgmap v48: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2024-09-12T23:35:21.785 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:21 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:21.484264216Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-09-12T23:35:21.786 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:21 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:21.517815425Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-09-12T23:35:21.786 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:21 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:21.577620714Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-09-12T23:35:21.786 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:35:21 smithi181 bash[51459]: Error response from daemon: No such container: ceph-da58a47c-715d-11ef-bceb-c7b262605968-prometheus-a 2024-09-12T23:35:21.786 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:35:21 smithi181 systemd[1]: ceph-da58a47c-715d-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-12T23:35:21.786 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:35:21 smithi181 systemd[1]: Stopped Ceph prometheus.a for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:35:21.786 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:35:21 smithi181 systemd[1]: Started Ceph prometheus.a for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:35:21.844 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:35:21 smithi136 bash[22852]: [12/Sep/2024:23:35:21] ENGINE Bus STOPPING 2024-09-12T23:35:22.092 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:21 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:21.788393019Z level=info msg="Executing migration" id=create_alert_configuration_table 2024-09-12T23:35:22.256 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:35:21 smithi136 bash[22852]: [12/Sep/2024:23:35:21] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-12T23:35:22.256 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:35:21 smithi136 bash[22852]: [12/Sep/2024:23:35:21] ENGINE Bus STOPPED 2024-09-12T23:35:22.256 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:35:21 smithi136 bash[22852]: [12/Sep/2024:23:35:21] ENGINE Bus STARTING 2024-09-12T23:35:22.256 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:35:21 smithi136 bash[22852]: [12/Sep/2024:23:35:21] ENGINE Serving on http://:::9283 2024-09-12T23:35:22.256 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:35:21 smithi136 bash[22852]: [12/Sep/2024:23:35:21] ENGINE Bus STARTED 2024-09-12T23:35:22.256 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:35:21 smithi136 bash[22852]: [12/Sep/2024:23:35:21] ENGINE Bus STOPPING 2024-09-12T23:35:22.346 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:22 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:22.094882841Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2024-09-12T23:35:22.346 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:22 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:22.239859188Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-09-12T23:35:22.346 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:22 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:22.298409793Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2024-09-12T23:35:22.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:35:22 smithi136 bash[22852]: [12/Sep/2024:23:35:22] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-12T23:35:22.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:35:22 smithi136 bash[22852]: [12/Sep/2024:23:35:22] ENGINE Bus STOPPED 2024-09-12T23:35:22.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:35:22 smithi136 bash[22852]: [12/Sep/2024:23:35:22] ENGINE Bus STARTING 2024-09-12T23:35:22.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:35:22 smithi136 bash[22852]: [12/Sep/2024:23:35:22] ENGINE Serving on http://:::9283 2024-09-12T23:35:22.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:35:22 smithi136 bash[22852]: [12/Sep/2024:23:35:22] ENGINE Bus STARTED 2024-09-12T23:35:22.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:35:22 smithi136 bash[22852]: [12/Sep/2024:23:35:22] ENGINE Bus STOPPING 2024-09-12T23:35:22.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:22 smithi136 bash[22601]: audit 2024-09-12T23:35:21.515096+0000 mon.a (mon.0) 851 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:22.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:22 smithi136 bash[22601]: audit 2024-09-12T23:35:21.525440+0000 mon.a (mon.0) 852 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:22.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:22 smithi136 bash[22601]: audit 2024-09-12T23:35:21.533957+0000 mon.a (mon.0) 853 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-12T23:35:22.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:22 smithi136 bash[22601]: audit 2024-09-12T23:35:21.534871+0000 mgr.y (mgr.24446) 65 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-12T23:35:22.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:22 smithi136 bash[22601]: audit 2024-09-12T23:35:21.537151+0000 mon.a (mon.0) 854 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi136.front.sepia.ceph.com:9093"}]: dispatch 2024-09-12T23:35:22.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:22 smithi136 bash[22601]: audit 2024-09-12T23:35:21.537883+0000 mgr.y (mgr.24446) 66 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi136.front.sepia.ceph.com:9093"}]: dispatch 2024-09-12T23:35:22.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:22 smithi136 bash[22601]: audit 2024-09-12T23:35:21.549186+0000 mon.a (mon.0) 855 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:22.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:22 smithi136 bash[22601]: audit 2024-09-12T23:35:21.571495+0000 mon.a (mon.0) 856 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-12T23:35:22.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:22 smithi136 bash[22601]: audit 2024-09-12T23:35:21.572453+0000 mgr.y (mgr.24446) 67 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-12T23:35:22.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:22 smithi136 bash[22601]: audit 2024-09-12T23:35:21.576782+0000 mon.a (mon.0) 857 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi181.front.sepia.ceph.com:9095"}]: dispatch 2024-09-12T23:35:22.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:22 smithi136 bash[22601]: audit 2024-09-12T23:35:21.577652+0000 mgr.y (mgr.24446) 68 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi181.front.sepia.ceph.com:9095"}]: dispatch 2024-09-12T23:35:22.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:22 smithi136 bash[22601]: audit 2024-09-12T23:35:21.588122+0000 mon.a (mon.0) 858 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:22.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:22 smithi136 bash[22601]: audit 2024-09-12T23:35:21.607885+0000 mon.a (mon.0) 859 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-12T23:35:22.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:22 smithi136 bash[22601]: audit 2024-09-12T23:35:21.608695+0000 mgr.y (mgr.24446) 69 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-12T23:35:22.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:22 smithi136 bash[22601]: audit 2024-09-12T23:35:21.613877+0000 mon.a (mon.0) 860 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi181.front.sepia.ceph.com:3000"}]: dispatch 2024-09-12T23:35:22.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:22 smithi136 bash[22601]: audit 2024-09-12T23:35:21.614735+0000 mgr.y (mgr.24446) 70 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi181.front.sepia.ceph.com:3000"}]: dispatch 2024-09-12T23:35:22.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:22 smithi136 bash[22601]: audit 2024-09-12T23:35:21.626593+0000 mon.a (mon.0) 861 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:22.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:22 smithi136 bash[22601]: audit 2024-09-12T23:35:21.643964+0000 mon.a (mon.0) 862 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:35:22.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:22 smithi136 bash[22601]: audit 2024-09-12T23:35:21.967307+0000 mon.a (mon.0) 863 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:22.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:22 smithi136 bash[22601]: audit 2024-09-12T23:35:21.974248+0000 mon.a (mon.0) 864 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:22.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:22 smithi136 bash[32189]: audit 2024-09-12T23:35:21.515096+0000 mon.a (mon.0) 851 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:22.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:22 smithi136 bash[32189]: audit 2024-09-12T23:35:21.525440+0000 mon.a (mon.0) 852 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:22.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:22 smithi136 bash[32189]: audit 2024-09-12T23:35:21.533957+0000 mon.a (mon.0) 853 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-12T23:35:22.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:22 smithi136 bash[32189]: audit 2024-09-12T23:35:21.534871+0000 mgr.y (mgr.24446) 65 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-12T23:35:22.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:22 smithi136 bash[32189]: audit 2024-09-12T23:35:21.537151+0000 mon.a (mon.0) 854 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi136.front.sepia.ceph.com:9093"}]: dispatch 2024-09-12T23:35:22.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:22 smithi136 bash[32189]: audit 2024-09-12T23:35:21.537883+0000 mgr.y (mgr.24446) 66 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi136.front.sepia.ceph.com:9093"}]: dispatch 2024-09-12T23:35:22.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:22 smithi136 bash[32189]: audit 2024-09-12T23:35:21.549186+0000 mon.a (mon.0) 855 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:22.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:22 smithi136 bash[32189]: audit 2024-09-12T23:35:21.571495+0000 mon.a (mon.0) 856 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-12T23:35:22.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:22 smithi136 bash[32189]: audit 2024-09-12T23:35:21.572453+0000 mgr.y (mgr.24446) 67 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-12T23:35:22.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:22 smithi136 bash[32189]: audit 2024-09-12T23:35:21.576782+0000 mon.a (mon.0) 857 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi181.front.sepia.ceph.com:9095"}]: dispatch 2024-09-12T23:35:22.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:22 smithi136 bash[32189]: audit 2024-09-12T23:35:21.577652+0000 mgr.y (mgr.24446) 68 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi181.front.sepia.ceph.com:9095"}]: dispatch 2024-09-12T23:35:22.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:22 smithi136 bash[32189]: audit 2024-09-12T23:35:21.588122+0000 mon.a (mon.0) 858 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:22.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:22 smithi136 bash[32189]: audit 2024-09-12T23:35:21.607885+0000 mon.a (mon.0) 859 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-12T23:35:22.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:22 smithi136 bash[32189]: audit 2024-09-12T23:35:21.608695+0000 mgr.y (mgr.24446) 69 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-12T23:35:22.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:22 smithi136 bash[32189]: audit 2024-09-12T23:35:21.613877+0000 mon.a (mon.0) 860 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi181.front.sepia.ceph.com:3000"}]: dispatch 2024-09-12T23:35:22.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:22 smithi136 bash[32189]: audit 2024-09-12T23:35:21.614735+0000 mgr.y (mgr.24446) 70 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi181.front.sepia.ceph.com:3000"}]: dispatch 2024-09-12T23:35:22.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:22 smithi136 bash[32189]: audit 2024-09-12T23:35:21.626593+0000 mon.a (mon.0) 861 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:22.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:22 smithi136 bash[32189]: audit 2024-09-12T23:35:21.643964+0000 mon.a (mon.0) 862 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:35:22.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:22 smithi136 bash[32189]: audit 2024-09-12T23:35:21.967307+0000 mon.a (mon.0) 863 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:22.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:22 smithi136 bash[32189]: audit 2024-09-12T23:35:21.974248+0000 mon.a (mon.0) 864 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:22.832 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:22 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:22.612152012Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2024-09-12T23:35:22.833 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:22 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:22.691971092Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-09-12T23:35:22.833 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:22 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:22.750819824Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2024-09-12T23:35:22.834 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:22 smithi181 bash[24776]: audit 2024-09-12T23:35:21.515096+0000 mon.a (mon.0) 851 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:22.834 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:22 smithi181 bash[24776]: audit 2024-09-12T23:35:21.525440+0000 mon.a (mon.0) 852 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:22.834 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:22 smithi181 bash[24776]: audit 2024-09-12T23:35:21.533957+0000 mon.a (mon.0) 853 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-12T23:35:22.834 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:22 smithi181 bash[24776]: audit 2024-09-12T23:35:21.534871+0000 mgr.y (mgr.24446) 65 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-12T23:35:22.834 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:22 smithi181 bash[24776]: audit 2024-09-12T23:35:21.537151+0000 mon.a (mon.0) 854 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi136.front.sepia.ceph.com:9093"}]: dispatch 2024-09-12T23:35:22.834 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:22 smithi181 bash[24776]: audit 2024-09-12T23:35:21.537883+0000 mgr.y (mgr.24446) 66 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi136.front.sepia.ceph.com:9093"}]: dispatch 2024-09-12T23:35:22.834 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:22 smithi181 bash[24776]: audit 2024-09-12T23:35:21.549186+0000 mon.a (mon.0) 855 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:22.834 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:22 smithi181 bash[24776]: audit 2024-09-12T23:35:21.571495+0000 mon.a (mon.0) 856 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-12T23:35:22.835 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:22 smithi181 bash[24776]: audit 2024-09-12T23:35:21.572453+0000 mgr.y (mgr.24446) 67 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-12T23:35:22.835 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:22 smithi181 bash[24776]: audit 2024-09-12T23:35:21.576782+0000 mon.a (mon.0) 857 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi181.front.sepia.ceph.com:9095"}]: dispatch 2024-09-12T23:35:22.835 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:22 smithi181 bash[24776]: audit 2024-09-12T23:35:21.577652+0000 mgr.y (mgr.24446) 68 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi181.front.sepia.ceph.com:9095"}]: dispatch 2024-09-12T23:35:22.835 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:22 smithi181 bash[24776]: audit 2024-09-12T23:35:21.588122+0000 mon.a (mon.0) 858 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:22.835 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:22 smithi181 bash[24776]: audit 2024-09-12T23:35:21.607885+0000 mon.a (mon.0) 859 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-12T23:35:22.835 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:22 smithi181 bash[24776]: audit 2024-09-12T23:35:21.608695+0000 mgr.y (mgr.24446) 69 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-12T23:35:22.835 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:22 smithi181 bash[24776]: audit 2024-09-12T23:35:21.613877+0000 mon.a (mon.0) 860 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi181.front.sepia.ceph.com:3000"}]: dispatch 2024-09-12T23:35:22.835 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:22 smithi181 bash[24776]: audit 2024-09-12T23:35:21.614735+0000 mgr.y (mgr.24446) 70 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi181.front.sepia.ceph.com:3000"}]: dispatch 2024-09-12T23:35:22.835 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:22 smithi181 bash[24776]: audit 2024-09-12T23:35:21.626593+0000 mon.a (mon.0) 861 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:22.835 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:22 smithi181 bash[24776]: audit 2024-09-12T23:35:21.643964+0000 mon.a (mon.0) 862 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:35:22.835 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:22 smithi181 bash[24776]: audit 2024-09-12T23:35:21.967307+0000 mon.a (mon.0) 863 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:22.835 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:22 smithi181 bash[24776]: audit 2024-09-12T23:35:21.974248+0000 mon.a (mon.0) 864 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:22.926 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:35:23.090 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:22 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:22.834583274Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2024-09-12T23:35:23.090 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:22 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:22.918251812Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-09-12T23:35:23.090 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:22 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:22.985587672Z level=info msg="Executing migration" id="create provenance_type table" 2024-09-12T23:35:23.090 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:23.048371076Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-09-12T23:35:23.345 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:23.094959644Z level=info msg="Executing migration" id="create alert_image table" 2024-09-12T23:35:23.346 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:23.137207064Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-09-12T23:35:23.346 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:23.187556019Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-09-12T23:35:23.346 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:23.255299663Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-09-12T23:35:23.346 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:23.27223751Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-09-12T23:35:23.346 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:23.297597153Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-09-12T23:35:23.346 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:23.297891113Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-09-12T23:35:23.346 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:23.322803209Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-09-12T23:35:23.346 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:23.34808837Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-09-12T23:35:23.346 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[51487]: ts=2024-09-12T23:35:23.178Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-12T23:35:23.346 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[51487]: ts=2024-09-12T23:35:23.178Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-12T23:35:23.347 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[51487]: ts=2024-09-12T23:35:23.178Z caller=main.go:570 level=info host_details="(Linux 5.15.0-121-generic #131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024 x86_64 smithi181 (none))" 2024-09-12T23:35:23.347 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[51487]: ts=2024-09-12T23:35:23.178Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-12T23:35:23.347 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[51487]: ts=2024-09-12T23:35:23.178Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-12T23:35:23.347 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[51487]: ts=2024-09-12T23:35:23.180Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-12T23:35:23.347 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[51487]: ts=2024-09-12T23:35:23.181Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-12T23:35:23.347 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[51487]: ts=2024-09-12T23:35:23.182Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-12T23:35:23.347 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[51487]: ts=2024-09-12T23:35:23.182Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-12T23:35:23.347 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[51487]: ts=2024-09-12T23:35:23.185Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-12T23:35:23.347 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[51487]: ts=2024-09-12T23:35:23.185Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.882µs 2024-09-12T23:35:23.347 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[51487]: ts=2024-09-12T23:35:23.185Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-12T23:35:23.347 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[51487]: ts=2024-09-12T23:35:23.186Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-09-12T23:35:23.347 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[51487]: ts=2024-09-12T23:35:23.186Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-09-12T23:35:23.347 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[51487]: ts=2024-09-12T23:35:23.186Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=35.332µs wal_replay_duration=672.654µs wbl_replay_duration=126ns total_replay_duration=731.498µs 2024-09-12T23:35:23.348 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[51487]: ts=2024-09-12T23:35:23.187Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-12T23:35:23.348 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[51487]: ts=2024-09-12T23:35:23.187Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-12T23:35:23.348 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[51487]: ts=2024-09-12T23:35:23.187Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-12T23:35:23.348 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[51487]: ts=2024-09-12T23:35:23.207Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=19.673255ms db_storage=1.062µs remote_storage=1.271µs web_handler=424ns query_engine=726ns scrape=321.88µs scrape_sd=31.381µs notify=32.449µs notify_sd=11.575µs rules=18.923027ms tracing=7.475µs 2024-09-12T23:35:23.348 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[51487]: ts=2024-09-12T23:35:23.207Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-12T23:35:23.348 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[51487]: ts=2024-09-12T23:35:23.207Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-12T23:35:23.506 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:35:23 smithi136 bash[22852]: [12/Sep/2024:23:35:23] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-12T23:35:23.507 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:35:23 smithi136 bash[22852]: [12/Sep/2024:23:35:23] ENGINE Bus STOPPED 2024-09-12T23:35:23.507 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:35:23 smithi136 bash[22852]: [12/Sep/2024:23:35:23] ENGINE Bus STARTING 2024-09-12T23:35:23.507 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:35:23 smithi136 bash[22852]: [12/Sep/2024:23:35:23] ENGINE Serving on http://:::9283 2024-09-12T23:35:23.507 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:35:23 smithi136 bash[22852]: [12/Sep/2024:23:35:23] ENGINE Bus STARTED 2024-09-12T23:35:23.596 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:23.373314866Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2024-09-12T23:35:23.596 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:23.37381693Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2024-09-12T23:35:23.597 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:23.374223278Z level=info msg="alerts found to migrate" alerts=0 2024-09-12T23:35:23.597 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:23.398637687Z level=info msg="Executing migration" id="create library_element table v1" 2024-09-12T23:35:23.597 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:23.423977134Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2024-09-12T23:35:23.597 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:23.449211236Z level=info msg="Executing migration" id="create library_element_connection table v1" 2024-09-12T23:35:23.597 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:23.474497325Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2024-09-12T23:35:23.597 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:23.4997398Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2024-09-12T23:35:23.597 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:23.541704039Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-09-12T23:35:23.597 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:23.583538218Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2024-09-12T23:35:23.884 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[24776]: cluster 2024-09-12T23:35:21.943392+0000 mgr.y (mgr.24446) 71 : cluster [DBG] pgmap v49: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2024-09-12T23:35:23.884 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[24776]: audit 2024-09-12T23:35:22.994528+0000 mon.a (mon.0) 865 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:35:23.884 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:23.625679663Z level=info msg="Executing migration" id="create data_keys table" 2024-09-12T23:35:23.884 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:23.667934441Z level=info msg="Executing migration" id="create secrets table" 2024-09-12T23:35:23.884 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:23.693113682Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-09-12T23:35:23.884 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:23.735174993Z level=info msg="Executing migration" id="add name column into data_keys" 2024-09-12T23:35:23.884 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:23.777072281Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-09-12T23:35:23.884 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:23.802317917Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-09-12T23:35:23.885 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:23.844466249Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-09-12T23:35:23.885 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:23.869811435Z level=info msg="Executing migration" id="create kv_store table v1" 2024-09-12T23:35:23.885 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:23.886742979Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2024-09-12T23:35:24.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:23 smithi136 bash[22601]: cluster 2024-09-12T23:35:21.943392+0000 mgr.y (mgr.24446) 71 : cluster [DBG] pgmap v49: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2024-09-12T23:35:24.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:23 smithi136 bash[22601]: audit 2024-09-12T23:35:22.994528+0000 mon.a (mon.0) 865 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:35:24.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:23 smithi136 bash[32189]: cluster 2024-09-12T23:35:21.943392+0000 mgr.y (mgr.24446) 71 : cluster [DBG] pgmap v49: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2024-09-12T23:35:24.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:23 smithi136 bash[32189]: audit 2024-09-12T23:35:22.994528+0000 mon.a (mon.0) 865 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:35:24.170 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:23.903707176Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2024-09-12T23:35:24.170 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:23.92056936Z level=info msg="Executing migration" id="create permission table" 2024-09-12T23:35:24.170 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:23 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:23.954389917Z level=info msg="Executing migration" id="add unique index permission.role_id" 2024-09-12T23:35:24.170 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:24 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:24.004667261Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2024-09-12T23:35:24.170 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:24 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:24.029968543Z level=info msg="Executing migration" id="create role table" 2024-09-12T23:35:24.170 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:24 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:24.055236229Z level=info msg="Executing migration" id="add column display_name" 2024-09-12T23:35:24.170 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:24 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:24.080379689Z level=info msg="Executing migration" id="add column group_name" 2024-09-12T23:35:24.170 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:24 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:24.105636244Z level=info msg="Executing migration" id="add index role.org_id" 2024-09-12T23:35:24.171 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:24 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:24.130917156Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2024-09-12T23:35:24.456 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:24 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:24.17300098Z level=info msg="Executing migration" id="add index role_org_id_uid" 2024-09-12T23:35:24.456 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:24 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:24.198342778Z level=info msg="Executing migration" id="create team role table" 2024-09-12T23:35:24.456 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:24 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:24.223550597Z level=info msg="Executing migration" id="add index team_role.org_id" 2024-09-12T23:35:24.457 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:24 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:24.265540636Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2024-09-12T23:35:24.457 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:24 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:24.290797284Z level=info msg="Executing migration" id="add index team_role.team_id" 2024-09-12T23:35:24.457 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:24 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:24.316055654Z level=info msg="Executing migration" id="create user role table" 2024-09-12T23:35:24.457 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:24 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:24.341283869Z level=info msg="Executing migration" id="add index user_role.org_id" 2024-09-12T23:35:24.457 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:24 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:24.366543558Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2024-09-12T23:35:24.457 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:24 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:24.391885642Z level=info msg="Executing migration" id="add index user_role.user_id" 2024-09-12T23:35:24.457 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:24 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:24.417086465Z level=info msg="Executing migration" id="create builtin role table" 2024-09-12T23:35:24.457 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:24 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:24.459053737Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2024-09-12T23:35:24.725 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:24 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:24.484383892Z level=info msg="Executing migration" id="add index builtin_role.name" 2024-09-12T23:35:24.725 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:24 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:24.509572152Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2024-09-12T23:35:24.726 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:24 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:24.551553377Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2024-09-12T23:35:24.726 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:24 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:24.576848338Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2024-09-12T23:35:24.726 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:24 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:24.618851673Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2024-09-12T23:35:24.726 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:24 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:24.660868721Z level=info msg="Executing migration" id="add unique index role.uid" 2024-09-12T23:35:24.726 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:24 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:24.703169336Z level=info msg="Executing migration" id="create seed assignment table" 2024-09-12T23:35:24.726 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:24 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:24.728430291Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2024-09-12T23:35:24.995 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:24 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:24.75373741Z level=info msg="Executing migration" id="add column hidden to role table" 2024-09-12T23:35:24.995 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:24 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:24.779030461Z level=info msg="Executing migration" id="create query_history table v1" 2024-09-12T23:35:24.995 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:24 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:24.821059722Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-09-12T23:35:24.995 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:24 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:24.846324981Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-09-12T23:35:24.995 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:24 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:24.871542224Z level=info msg="Executing migration" id="teams permissions migration" 2024-09-12T23:35:24.995 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:24 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:24.896821855Z level=info msg="Executing migration" id="dashboard permissions" 2024-09-12T23:35:24.995 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:24 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:24.922066668Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-09-12T23:35:24.996 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:24 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:24.947300754Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-09-12T23:35:24.996 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:24 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:24.972451584Z level=info msg="Executing migration" id="alerting notification permissions" 2024-09-12T23:35:24.996 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:24 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:24.997856274Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-09-12T23:35:25.248 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:25 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:25.039885043Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-09-12T23:35:25.248 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:25 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:25.065148942Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-09-12T23:35:25.248 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:25 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:25.090359045Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-09-12T23:35:25.248 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:25 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:25.115653965Z level=info msg="Executing migration" id="create correlation table v1" 2024-09-12T23:35:25.248 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:25 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:25.1409448Z level=info msg="Executing migration" id="add index correlations.uid" 2024-09-12T23:35:25.249 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:25 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:25.157836934Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-09-12T23:35:25.249 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:25 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:25.174765505Z level=info msg="Executing migration" id="add correlation config column" 2024-09-12T23:35:25.249 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:25 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:25.216574197Z level=info msg="Executing migration" id="create entity_events table" 2024-09-12T23:35:25.249 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:25 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:25.233739264Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-09-12T23:35:25.249 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:25 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:25.250621658Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-09-12T23:35:25.249 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:25 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:25.250935621Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-09-12T23:35:25.517 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:25 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:25.267542559Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-12T23:35:25.517 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:25 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:25.267900345Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-12T23:35:25.517 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:25 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:25.292744156Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-09-12T23:35:25.517 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:25 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:25.334838903Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-09-12T23:35:25.517 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:25 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:25.360139845Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-09-12T23:35:25.518 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:25 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:25.402093575Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-12T23:35:25.518 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:25 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:25.42740759Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-09-12T23:35:25.518 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:25 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:25.452525491Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-09-12T23:35:25.518 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:25 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:25.494878892Z level=info msg="Executing migration" id="Drop public config table" 2024-09-12T23:35:25.518 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:25 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:25.520198848Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-09-12T23:35:25.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:25 smithi136 bash[22601]: cluster 2024-09-12T23:35:23.944521+0000 mgr.y (mgr.24446) 72 : cluster [DBG] pgmap v50: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2024-09-12T23:35:25.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:25 smithi136 bash[32189]: cluster 2024-09-12T23:35:23.944521+0000 mgr.y (mgr.24446) 72 : cluster [DBG] pgmap v50: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2024-09-12T23:35:25.756 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:35:25 smithi136 bash[60087]: ts=2024-09-12T23:35:25.501Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.004285048s 2024-09-12T23:35:25.786 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:25 smithi181 bash[24776]: cluster 2024-09-12T23:35:23.944521+0000 mgr.y (mgr.24446) 72 : cluster [DBG] pgmap v50: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2024-09-12T23:35:25.786 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:25 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:25.56217259Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-09-12T23:35:25.786 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:25 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:25.587459825Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-09-12T23:35:25.786 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:25 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:25.612669515Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-09-12T23:35:25.787 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:25 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:25.654857872Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-09-12T23:35:25.787 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:25 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:25.696842635Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-09-12T23:35:25.787 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:25 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:25.722047093Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-09-12T23:35:25.787 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:25 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:25.764162112Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-09-12T23:35:25.787 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:25 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:25.789533283Z level=info msg="Executing migration" id="add share column" 2024-09-12T23:35:26.072 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:25 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:25.831481412Z level=info msg="Executing migration" id="create default alerting folders" 2024-09-12T23:35:26.072 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:25 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:25.85672819Z level=info msg="Executing migration" id="create file table" 2024-09-12T23:35:26.072 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:25 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:25.882508886Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-09-12T23:35:26.072 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:25 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:25.924069337Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-09-12T23:35:26.072 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:25 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:25.966110521Z level=info msg="Executing migration" id="create file_meta table" 2024-09-12T23:35:26.073 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:26 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:26.008108137Z level=info msg="Executing migration" id="file table idx: path key" 2024-09-12T23:35:26.073 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:26 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:26.07507667Z level=info msg="Executing migration" id="set path collation in file table" 2024-09-12T23:35:26.150 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-12T23:35:26.150 INFO:teuthology.orchestra.run.smithi136.stdout:{"epoch":64,"fsid":"da58a47c-715d-11ef-bceb-c7b262605968","created":"2024-09-12T23:23:29.000926+0000","modified":"2024-09-12T23:35:09.657947+0000","last_up_change":"2024-09-12T23:32:43.859472+0000","last_in_change":"2024-09-12T23:32:18.995528+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-12T23:28:33.956985+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-09-12T23:33:27.478434+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-09-12T23:33:28.566349+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"54","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}},{"pool":4,"pool_name":"rbd","create_time":"2024-09-12T23:34:48.404257+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"61","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":61,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"87aedf04-695e-4a8f-80c3-ffd18883a4d5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6802","nonce":4251571952},{"type":"v1","addr":"172.21.15.136:6803","nonce":4251571952}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6804","nonce":4251571952},{"type":"v1","addr":"172.21.15.136:6805","nonce":4251571952}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6808","nonce":4251571952},{"type":"v1","addr":"172.21.15.136:6809","nonce":4251571952}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6806","nonce":4251571952},{"type":"v1","addr":"172.21.15.136:6807","nonce":4251571952}]},"public_addr":"172.21.15.136:6803/4251571952","cluster_addr":"172.21.15.136:6805/4251571952","heartbeat_back_addr":"172.21.15.136:6809/4251571952","heartbeat_front_addr":"172.21.15.136:6807/4251571952","state":["exists","up"]},{"osd":1,"uuid":"850d4f98-0640-4151-bd45-a351ca41d54e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":63,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6810","nonce":3482177958},{"type":"v1","addr":"172.21.15.136:6811","nonce":3482177958}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6812","nonce":3482177958},{"type":"v1","addr":"172.21.15.136:6813","nonce":3482177958}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6816","nonce":3482177958},{"type":"v1","addr":"172.21.15.136:6817","nonce":3482177958}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6814","nonce":3482177958},{"type":"v1","addr":"172.21.15.136:6815","nonce":3482177958}]},"public_addr":"172.21.15.136:6811/3482177958","cluster_addr":"172.21.15.136:6813/3482177958","heartbeat_back_addr":"172.21.15.136:6817/3482177958","heartbeat_front_addr":"172.21.15.136:6815/3482177958","state":["exists","up"]},{"osd":2,"uuid":"bde707d6-e027-4b63-9853-fd59212eea1e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":63,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6818","nonce":308477304},{"type":"v1","addr":"172.21.15.136:6819","nonce":308477304}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6820","nonce":308477304},{"type":"v1","addr":"172.21.15.136:6821","nonce":308477304}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6824","nonce":308477304},{"type":"v1","addr":"172.21.15.136:6825","nonce":308477304}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6822","nonce":308477304},{"type":"v1","addr":"172.21.15.136:6823","nonce":308477304}]},"public_addr":"172.21.15.136:6819/308477304","cluster_addr":"172.21.15.136:6821/308477304","heartbeat_back_addr":"172.21.15.136:6825/308477304","heartbeat_front_addr":"172.21.15.136:6823/308477304","state":["exists","up"]},{"osd":3,"uuid":"d61f8357-b89b-4c05-b366-65a9f788064d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6826","nonce":201948269},{"type":"v1","addr":"172.21.15.136:6827","nonce":201948269}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6828","nonce":201948269},{"type":"v1","addr":"172.21.15.136:6829","nonce":201948269}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6832","nonce":201948269},{"type":"v1","addr":"172.21.15.136:6833","nonce":201948269}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6830","nonce":201948269},{"type":"v1","addr":"172.21.15.136:6831","nonce":201948269}]},"public_addr":"172.21.15.136:6827/201948269","cluster_addr":"172.21.15.136:6829/201948269","heartbeat_back_addr":"172.21.15.136:6833/201948269","heartbeat_front_addr":"172.21.15.136:6831/201948269","state":["exists","up"]},{"osd":4,"uuid":"98b0e81c-824e-4dbc-9faa-2de9f199af1b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":63,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6800","nonce":775251087},{"type":"v1","addr":"172.21.15.181:6801","nonce":775251087}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6802","nonce":775251087},{"type":"v1","addr":"172.21.15.181:6803","nonce":775251087}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6806","nonce":775251087},{"type":"v1","addr":"172.21.15.181:6807","nonce":775251087}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6804","nonce":775251087},{"type":"v1","addr":"172.21.15.181:6805","nonce":775251087}]},"public_addr":"172.21.15.181:6801/775251087","cluster_addr":"172.21.15.181:6803/775251087","heartbeat_back_addr":"172.21.15.181:6807/775251087","heartbeat_front_addr":"172.21.15.181:6805/775251087","state":["exists","up"]},{"osd":5,"uuid":"5c6d2f69-7f9c-4689-95b0-bd4eb858956d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6808","nonce":2109931558},{"type":"v1","addr":"172.21.15.181:6809","nonce":2109931558}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6810","nonce":2109931558},{"type":"v1","addr":"172.21.15.181:6811","nonce":2109931558}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6814","nonce":2109931558},{"type":"v1","addr":"172.21.15.181:6815","nonce":2109931558}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6812","nonce":2109931558},{"type":"v1","addr":"172.21.15.181:6813","nonce":2109931558}]},"public_addr":"172.21.15.181:6809/2109931558","cluster_addr":"172.21.15.181:6811/2109931558","heartbeat_back_addr":"172.21.15.181:6815/2109931558","heartbeat_front_addr":"172.21.15.181:6813/2109931558","state":["exists","up"]},{"osd":6,"uuid":"532ee052-af15-433a-8e93-b0b642aaf85b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":63,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6816","nonce":4099148413},{"type":"v1","addr":"172.21.15.181:6817","nonce":4099148413}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6818","nonce":4099148413},{"type":"v1","addr":"172.21.15.181:6819","nonce":4099148413}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6822","nonce":4099148413},{"type":"v1","addr":"172.21.15.181:6823","nonce":4099148413}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6820","nonce":4099148413},{"type":"v1","addr":"172.21.15.181:6821","nonce":4099148413}]},"public_addr":"172.21.15.181:6817/4099148413","cluster_addr":"172.21.15.181:6819/4099148413","heartbeat_back_addr":"172.21.15.181:6823/4099148413","heartbeat_front_addr":"172.21.15.181:6821/4099148413","state":["exists","up"]},{"osd":7,"uuid":"d2ff80f0-6ef0-4098-877c-a891d48c687d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":63,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6824","nonce":783620875},{"type":"v1","addr":"172.21.15.181:6825","nonce":783620875}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6826","nonce":783620875},{"type":"v1","addr":"172.21.15.181:6827","nonce":783620875}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6830","nonce":783620875},{"type":"v1","addr":"172.21.15.181:6831","nonce":783620875}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6828","nonce":783620875},{"type":"v1","addr":"172.21.15.181:6829","nonce":783620875}]},"public_addr":"172.21.15.181:6825/783620875","cluster_addr":"172.21.15.181:6827/783620875","heartbeat_back_addr":"172.21.15.181:6831/783620875","heartbeat_front_addr":"172.21.15.181:6829/783620875","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:26:50.657580+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:27:40.536476+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:28:31.755012+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:29:23.585451+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:30:11.888258+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:31:01.812765+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:31:52.166696+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:32:42.696525+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.b","mappings":[{"from":7,"to":2}]},{"pgid":"3.9","mappings":[{"from":4,"to":1}]},{"pgid":"3.10","mappings":[{"from":0,"to":1}]},{"pgid":"3.14","mappings":[{"from":4,"to":2}]},{"pgid":"3.30","mappings":[{"from":5,"to":2}]},{"pgid":"3.31","mappings":[{"from":4,"to":2}]},{"pgid":"3.34","mappings":[{"from":0,"to":3}]},{"pgid":"3.37","mappings":[{"from":5,"to":1}]}],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.136:0/129641967":"2024-09-13T23:34:07.892265+0000","172.21.15.136:6801/1810367175":"2024-09-13T23:34:07.892265+0000","172.21.15.136:6800/1810367175":"2024-09-13T23:34:07.892265+0000","172.21.15.136:6800/2894181186":"2024-09-13T23:23:54.847045+0000","172.21.15.136:0/1964283041":"2024-09-13T23:24:25.346823+0000","172.21.15.136:0/3924275900":"2024-09-13T23:34:07.892265+0000","172.21.15.136:6801/2894181186":"2024-09-13T23:23:54.847045+0000","172.21.15.136:0/513809002":"2024-09-13T23:34:07.892265+0000","172.21.15.136:0/3910468981":"2024-09-13T23:23:54.847045+0000","172.21.15.136:0/1549678324":"2024-09-13T23:23:54.847045+0000","172.21.15.136:6800/3594272622":"2024-09-13T23:24:25.346823+0000","172.21.15.136:0/209539205":"2024-09-13T23:24:25.346823+0000","172.21.15.136:6801/3594272622":"2024-09-13T23:24:25.346823+0000","172.21.15.136:0/4260962972":"2024-09-13T23:24:25.346823+0000","172.21.15.136:0/3376519841":"2024-09-13T23:23:54.847045+0000","172.21.15.136:0/3642432358":"2024-09-13T23:34:07.892265+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-12T23:35:26.333 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:26 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:26.100329206Z level=info msg="Executing migration" id="managed permissions migration" 2024-09-12T23:35:26.333 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:26 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:26.125550977Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-09-12T23:35:26.333 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:26 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:26.150854043Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-09-12T23:35:26.334 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:26 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:26.176119913Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-09-12T23:35:26.334 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:26 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:26.217990514Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-09-12T23:35:26.334 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:26 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:26.260150785Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-09-12T23:35:26.334 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:26 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:26.285432031Z level=info msg="Executing migration" id="update group index for alert rules" 2024-09-12T23:35:26.334 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:26 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:26.310646327Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-09-12T23:35:26.334 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:26 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:26.335929851Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-09-12T23:35:26.596 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:26 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:26.361187913Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-09-12T23:35:26.596 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:26 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:26.386748563Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-09-12T23:35:26.597 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:26 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:26.412024592Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-09-12T23:35:26.597 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:26 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:26.437229809Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-09-12T23:35:26.597 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:26 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:26.53757587Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-09-12T23:35:26.597 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:26 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:26.579604571Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-09-12T23:35:26.765 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-09-12T23:35:26.766 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-09-12T23:35:26.766 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-09-12T23:35:26.767 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-09-12T23:35:26.767 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-09-12T23:35:26.768 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-09-12T23:35:26.768 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-09-12T23:35:26.769 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-09-12T23:35:26.888 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:26 smithi181 bash[24776]: audit 2024-09-12T23:35:26.153322+0000 mon.a (mon.0) 866 : audit [DBG] from='client.? 172.21.15.136:0/1759185181' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-12T23:35:26.888 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:26 smithi181 bash[24776]: audit 2024-09-12T23:35:26.476601+0000 mon.a (mon.0) 867 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:26.888 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:26 smithi181 bash[24776]: audit 2024-09-12T23:35:26.482039+0000 mon.a (mon.0) 868 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:26.889 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:26 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:26.621628016Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-09-12T23:35:26.889 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:26 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:26.680368463Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-09-12T23:35:26.889 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:26 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:26.722418985Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-09-12T23:35:26.889 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:26 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:26.747679521Z level=info msg="Executing migration" id="create folder table" 2024-09-12T23:35:26.889 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:26 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:26.772969961Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-09-12T23:35:26.889 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:26 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:26.79827962Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-09-12T23:35:26.889 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:26 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:26.823498887Z level=info msg="Executing migration" id="Update folder title length" 2024-09-12T23:35:26.889 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:26 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:26.865475803Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-09-12T23:35:26.889 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:26 smithi181 bash[50806]: logger=migrator t=2024-09-12T23:35:26.890798623Z level=info msg="migrations completed" performed=485 skipped=0 duration=21.923064756s 2024-09-12T23:35:27.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:26 smithi136 bash[22601]: audit 2024-09-12T23:35:26.153322+0000 mon.a (mon.0) 866 : audit [DBG] from='client.? 172.21.15.136:0/1759185181' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-12T23:35:27.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:26 smithi136 bash[22601]: audit 2024-09-12T23:35:26.476601+0000 mon.a (mon.0) 867 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:27.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:26 smithi136 bash[22601]: audit 2024-09-12T23:35:26.482039+0000 mon.a (mon.0) 868 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:27.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:26 smithi136 bash[32189]: audit 2024-09-12T23:35:26.153322+0000 mon.a (mon.0) 866 : audit [DBG] from='client.? 172.21.15.136:0/1759185181' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-12T23:35:27.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:26 smithi136 bash[32189]: audit 2024-09-12T23:35:26.476601+0000 mon.a (mon.0) 867 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:27.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:26 smithi136 bash[32189]: audit 2024-09-12T23:35:26.482039+0000 mon.a (mon.0) 868 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:27.346 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:26 smithi181 bash[50806]: logger=sqlstore t=2024-09-12T23:35:26.892449238Z level=info msg="Created default organization" 2024-09-12T23:35:27.346 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:27 smithi181 bash[50806]: logger=plugin.loader t=2024-09-12T23:35:27.039813085Z level=info msg="Plugin registered" pluginID=input 2024-09-12T23:35:27.346 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:27 smithi181 bash[50806]: logger=plugin.loader t=2024-09-12T23:35:27.070198583Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-09-12T23:35:27.346 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:27 smithi181 bash[50806]: logger=plugin.loader t=2024-09-12T23:35:27.070219979Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-09-12T23:35:27.347 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:27 smithi181 bash[50806]: logger=secrets t=2024-09-12T23:35:27.070260888Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-09-12T23:35:27.347 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:27 smithi181 bash[50806]: logger=query_data t=2024-09-12T23:35:27.072373267Z level=info msg="Query Service initialization" 2024-09-12T23:35:27.347 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:27 smithi181 bash[50806]: logger=live.push_http t=2024-09-12T23:35:27.0772871Z level=info msg="Live Push Gateway initialization" 2024-09-12T23:35:28.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:27 smithi136 bash[22601]: cluster 2024-09-12T23:35:25.944922+0000 mgr.y (mgr.24446) 73 : cluster [DBG] pgmap v51: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:35:28.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:27 smithi136 bash[32189]: cluster 2024-09-12T23:35:25.944922+0000 mgr.y (mgr.24446) 73 : cluster [DBG] pgmap v51: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:35:28.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:27 smithi181 bash[24776]: cluster 2024-09-12T23:35:25.944922+0000 mgr.y (mgr.24446) 73 : cluster [DBG] pgmap v51: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:35:28.596 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:28 smithi181 bash[50806]: logger=infra.usagestats.collector t=2024-09-12T23:35:28.16280926Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-09-12T23:35:28.596 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:28 smithi181 bash[50806]: logger=provisioning.datasources t=2024-09-12T23:35:28.163218224Z level=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2024-09-12T23:35:28.596 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:28 smithi181 bash[50806]: logger=provisioning.datasources t=2024-09-12T23:35:28.196837487Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-09-12T23:35:28.596 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:28 smithi181 bash[50806]: logger=provisioning.datasources t=2024-09-12T23:35:28.268751568Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2024-09-12T23:35:28.596 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:28 smithi181 bash[50806]: logger=provisioning.alerting t=2024-09-12T23:35:28.334432738Z level=info msg="starting to provision alerting" 2024-09-12T23:35:28.597 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:28 smithi181 bash[50806]: logger=provisioning.alerting t=2024-09-12T23:35:28.334478357Z level=info msg="finished to provision alerting" 2024-09-12T23:35:28.597 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:28 smithi181 bash[50806]: logger=ngalert.state.manager t=2024-09-12T23:35:28.334611788Z level=info msg="Warming state cache for startup" 2024-09-12T23:35:28.597 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:28 smithi181 bash[50806]: logger=grafanaStorageLogger t=2024-09-12T23:35:28.334743268Z level=info msg="storage starting" 2024-09-12T23:35:28.597 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:28 smithi181 bash[50806]: logger=http.server t=2024-09-12T23:35:28.336909037Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-09-12T23:35:28.597 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:28 smithi181 bash[50806]: logger=sqlstore.transactions t=2024-09-12T23:35:28.345625634Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-12T23:35:28.597 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:28 smithi181 bash[50806]: logger=ngalert.state.manager t=2024-09-12T23:35:28.379100053Z level=info msg="State cache has been initialized" states=0 duration=44.485762ms 2024-09-12T23:35:28.597 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:28 smithi181 bash[50806]: logger=ticker t=2024-09-12T23:35:28.379165914Z level=info msg=starting first_tick=2024-09-12T23:35:30Z 2024-09-12T23:35:28.597 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:35:28 smithi181 bash[50806]: logger=ngalert.multiorg.alertmanager t=2024-09-12T23:35:28.379163753Z level=info msg="starting MultiOrg Alertmanager" 2024-09-12T23:35:29.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:29 smithi181 bash[24776]: cluster 2024-09-12T23:35:27.945498+0000 mgr.y (mgr.24446) 74 : cluster [DBG] pgmap v52: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:35:29.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:29 smithi181 bash[24776]: audit 2024-09-12T23:35:28.588758+0000 mon.a (mon.0) 869 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:29.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:29 smithi181 bash[24776]: audit 2024-09-12T23:35:28.599328+0000 mon.a (mon.0) 870 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:29.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:29 smithi181 bash[24776]: audit 2024-09-12T23:35:28.993250+0000 mon.a (mon.0) 871 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:35:29.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:29 smithi181 bash[24776]: audit 2024-09-12T23:35:28.994776+0000 mon.a (mon.0) 872 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:35:29.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:29 smithi181 bash[24776]: audit 2024-09-12T23:35:29.004850+0000 mon.a (mon.0) 873 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:29.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:29 smithi181 bash[24776]: audit 2024-09-12T23:35:29.049458+0000 mon.a (mon.0) 874 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:35:29.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:29 smithi181 bash[24776]: audit 2024-09-12T23:35:29.051955+0000 mon.a (mon.0) 875 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:35:29.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:29 smithi181 bash[24776]: audit 2024-09-12T23:35:29.053220+0000 mon.a (mon.0) 876 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:35:29.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:29 smithi181 bash[24776]: audit 2024-09-12T23:35:29.063358+0000 mon.a (mon.0) 877 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:30.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:29 smithi136 bash[22601]: cluster 2024-09-12T23:35:27.945498+0000 mgr.y (mgr.24446) 74 : cluster [DBG] pgmap v52: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:35:30.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:29 smithi136 bash[22601]: audit 2024-09-12T23:35:28.588758+0000 mon.a (mon.0) 869 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:30.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:29 smithi136 bash[22601]: audit 2024-09-12T23:35:28.599328+0000 mon.a (mon.0) 870 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:30.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:29 smithi136 bash[22601]: audit 2024-09-12T23:35:28.993250+0000 mon.a (mon.0) 871 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:35:30.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:29 smithi136 bash[22601]: audit 2024-09-12T23:35:28.994776+0000 mon.a (mon.0) 872 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:35:30.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:29 smithi136 bash[22601]: audit 2024-09-12T23:35:29.004850+0000 mon.a (mon.0) 873 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:30.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:29 smithi136 bash[22601]: audit 2024-09-12T23:35:29.049458+0000 mon.a (mon.0) 874 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:35:30.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:29 smithi136 bash[22601]: audit 2024-09-12T23:35:29.051955+0000 mon.a (mon.0) 875 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:35:30.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:29 smithi136 bash[22601]: audit 2024-09-12T23:35:29.053220+0000 mon.a (mon.0) 876 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:35:30.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:29 smithi136 bash[22601]: audit 2024-09-12T23:35:29.063358+0000 mon.a (mon.0) 877 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:30.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:29 smithi136 bash[32189]: cluster 2024-09-12T23:35:27.945498+0000 mgr.y (mgr.24446) 74 : cluster [DBG] pgmap v52: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:35:30.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:29 smithi136 bash[32189]: audit 2024-09-12T23:35:28.588758+0000 mon.a (mon.0) 869 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:30.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:29 smithi136 bash[32189]: audit 2024-09-12T23:35:28.599328+0000 mon.a (mon.0) 870 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:30.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:29 smithi136 bash[32189]: audit 2024-09-12T23:35:28.993250+0000 mon.a (mon.0) 871 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:35:30.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:29 smithi136 bash[32189]: audit 2024-09-12T23:35:28.994776+0000 mon.a (mon.0) 872 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:35:30.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:29 smithi136 bash[32189]: audit 2024-09-12T23:35:29.004850+0000 mon.a (mon.0) 873 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:30.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:29 smithi136 bash[32189]: audit 2024-09-12T23:35:29.049458+0000 mon.a (mon.0) 874 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:35:30.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:29 smithi136 bash[32189]: audit 2024-09-12T23:35:29.051955+0000 mon.a (mon.0) 875 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:35:30.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:29 smithi136 bash[32189]: audit 2024-09-12T23:35:29.053220+0000 mon.a (mon.0) 876 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:35:30.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:29 smithi136 bash[32189]: audit 2024-09-12T23:35:29.063358+0000 mon.a (mon.0) 877 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:35:31.664 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:35:31.665 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:35:31.666 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:35:31.666 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:35:31.666 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:35:31.667 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:35:31.667 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:35:31.670 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:35:31.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:31 smithi181 bash[24776]: cluster 2024-09-12T23:35:29.946663+0000 mgr.y (mgr.24446) 75 : cluster [DBG] pgmap v53: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:35:31.895 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:31 smithi136 bash[32189]: cluster 2024-09-12T23:35:29.946663+0000 mgr.y (mgr.24446) 75 : cluster [DBG] pgmap v53: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:35:31.895 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:31 smithi136 bash[22601]: cluster 2024-09-12T23:35:29.946663+0000 mgr.y (mgr.24446) 75 : cluster [DBG] pgmap v53: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:35:33.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:33 smithi181 bash[24776]: cluster 2024-09-12T23:35:31.947242+0000 mgr.y (mgr.24446) 76 : cluster [DBG] pgmap v54: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:35:33.846 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:33 smithi136 bash[22601]: cluster 2024-09-12T23:35:31.947242+0000 mgr.y (mgr.24446) 76 : cluster [DBG] pgmap v54: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:35:33.846 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:33 smithi136 bash[32189]: cluster 2024-09-12T23:35:31.947242+0000 mgr.y (mgr.24446) 76 : cluster [DBG] pgmap v54: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:35:35.594 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:35:35 smithi181 bash[25717]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:35:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-12T23:35:35.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:35 smithi181 bash[24776]: cluster 2024-09-12T23:35:33.947813+0000 mgr.y (mgr.24446) 77 : cluster [DBG] pgmap v55: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:35:36.000 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:35 smithi136 bash[22601]: cluster 2024-09-12T23:35:33.947813+0000 mgr.y (mgr.24446) 77 : cluster [DBG] pgmap v55: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:35:36.000 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:35 smithi136 bash[32189]: cluster 2024-09-12T23:35:33.947813+0000 mgr.y (mgr.24446) 77 : cluster [DBG] pgmap v55: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:35:36.294 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:35:36 smithi136 bash[22852]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:35:36] "GET /metrics HTTP/1.1" 200 236629 "" "Prometheus/2.43.0" 2024-09-12T23:35:37.691 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:37 smithi136 bash[22601]: cluster 2024-09-12T23:35:35.948205+0000 mgr.y (mgr.24446) 78 : cluster [DBG] pgmap v56: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:35:37.692 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:37 smithi136 bash[32189]: cluster 2024-09-12T23:35:35.948205+0000 mgr.y (mgr.24446) 78 : cluster [DBG] pgmap v56: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:35:38.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:37 smithi181 bash[24776]: cluster 2024-09-12T23:35:35.948205+0000 mgr.y (mgr.24446) 78 : cluster [DBG] pgmap v56: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:35:38.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:38 smithi136 bash[22601]: audit 2024-09-12T23:35:37.994443+0000 mon.a (mon.0) 878 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:35:38.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:38 smithi136 bash[32189]: audit 2024-09-12T23:35:37.994443+0000 mon.a (mon.0) 878 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:35:39.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:38 smithi181 bash[24776]: audit 2024-09-12T23:35:37.994443+0000 mon.a (mon.0) 878 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:35:40.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:39 smithi136 bash[22601]: cluster 2024-09-12T23:35:37.948609+0000 mgr.y (mgr.24446) 79 : cluster [DBG] pgmap v57: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:35:40.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:39 smithi136 bash[32189]: cluster 2024-09-12T23:35:37.948609+0000 mgr.y (mgr.24446) 79 : cluster [DBG] pgmap v57: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:35:40.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:39 smithi181 bash[24776]: cluster 2024-09-12T23:35:37.948609+0000 mgr.y (mgr.24446) 79 : cluster [DBG] pgmap v57: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:35:40.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:40 smithi136 bash[22601]: cluster 2024-09-12T23:35:39.633231+0000 mon.a (mon.0) 879 : cluster [DBG] mgrmap e21: y(active, since 91s), standbys: x 2024-09-12T23:35:40.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:40 smithi136 bash[32189]: cluster 2024-09-12T23:35:39.633231+0000 mon.a (mon.0) 879 : cluster [DBG] mgrmap e21: y(active, since 91s), standbys: x 2024-09-12T23:35:41.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:40 smithi181 bash[24776]: cluster 2024-09-12T23:35:39.633231+0000 mon.a (mon.0) 879 : cluster [DBG] mgrmap e21: y(active, since 91s), standbys: x 2024-09-12T23:35:41.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:41 smithi136 bash[22601]: cluster 2024-09-12T23:35:39.949746+0000 mgr.y (mgr.24446) 80 : cluster [DBG] pgmap v58: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-12T23:35:41.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:41 smithi136 bash[32189]: cluster 2024-09-12T23:35:39.949746+0000 mgr.y (mgr.24446) 80 : cluster [DBG] pgmap v58: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-12T23:35:42.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:41 smithi181 bash[24776]: cluster 2024-09-12T23:35:39.949746+0000 mgr.y (mgr.24446) 80 : cluster [DBG] pgmap v58: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-12T23:35:43.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:43 smithi136 bash[22601]: cluster 2024-09-12T23:35:41.950269+0000 mgr.y (mgr.24446) 81 : cluster [DBG] pgmap v59: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-12T23:35:43.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:43 smithi136 bash[32189]: cluster 2024-09-12T23:35:41.950269+0000 mgr.y (mgr.24446) 81 : cluster [DBG] pgmap v59: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-12T23:35:44.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:43 smithi181 bash[24776]: cluster 2024-09-12T23:35:41.950269+0000 mgr.y (mgr.24446) 81 : cluster [DBG] pgmap v59: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-12T23:35:44.437 INFO:teuthology.orchestra.run.smithi136.stdout:77309411415 2024-09-12T23:35:44.437 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-09-12T23:35:44.659 INFO:teuthology.orchestra.run.smithi136.stdout:206158430245 2024-09-12T23:35:44.660 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-09-12T23:35:45.596 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:35:45 smithi181 bash[25717]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:35:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-12T23:35:46.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:45 smithi136 bash[22601]: cluster 2024-09-12T23:35:43.951386+0000 mgr.y (mgr.24446) 82 : cluster [DBG] pgmap v60: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-12T23:35:46.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:45 smithi136 bash[32189]: cluster 2024-09-12T23:35:43.951386+0000 mgr.y (mgr.24446) 82 : cluster [DBG] pgmap v60: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-12T23:35:46.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:45 smithi181 bash[24776]: cluster 2024-09-12T23:35:43.951386+0000 mgr.y (mgr.24446) 82 : cluster [DBG] pgmap v60: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-12T23:35:46.506 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:35:46 smithi136 bash[22852]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:35:46] "GET /metrics HTTP/1.1" 200 236621 "" "Prometheus/2.43.0" 2024-09-12T23:35:46.675 INFO:teuthology.orchestra.run.smithi136.stdout:107374182478 2024-09-12T23:35:46.675 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-09-12T23:35:47.183 INFO:teuthology.orchestra.run.smithi136.stdout:55834574947 2024-09-12T23:35:47.184 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-09-12T23:35:47.665 INFO:teuthology.orchestra.run.smithi136.stdout:180388626480 2024-09-12T23:35:47.665 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-09-12T23:35:47.721 INFO:teuthology.orchestra.run.smithi136.stdout:154618822715 2024-09-12T23:35:47.722 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-09-12T23:35:48.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:47 smithi136 bash[22601]: cluster 2024-09-12T23:35:45.952411+0000 mgr.y (mgr.24446) 83 : cluster [DBG] pgmap v61: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-12T23:35:48.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:47 smithi136 bash[32189]: cluster 2024-09-12T23:35:45.952411+0000 mgr.y (mgr.24446) 83 : cluster [DBG] pgmap v61: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-12T23:35:48.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:47 smithi181 bash[24776]: cluster 2024-09-12T23:35:45.952411+0000 mgr.y (mgr.24446) 83 : cluster [DBG] pgmap v61: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-12T23:35:48.341 INFO:teuthology.orchestra.run.smithi136.stdout:128849018948 2024-09-12T23:35:48.341 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-09-12T23:35:48.355 INFO:teuthology.orchestra.run.smithi136.stdout:34359738475 2024-09-12T23:35:48.356 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-09-12T23:35:50.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:49 smithi136 bash[22601]: cluster 2024-09-12T23:35:47.952794+0000 mgr.y (mgr.24446) 84 : cluster [DBG] pgmap v62: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-12T23:35:50.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:49 smithi136 bash[32189]: cluster 2024-09-12T23:35:47.952794+0000 mgr.y (mgr.24446) 84 : cluster [DBG] pgmap v62: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-12T23:35:50.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:49 smithi181 bash[24776]: cluster 2024-09-12T23:35:47.952794+0000 mgr.y (mgr.24446) 84 : cluster [DBG] pgmap v62: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-12T23:35:51.219 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:35:51.220 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:35:51.220 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:35:51.220 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:35:52.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:51 smithi136 bash[22601]: cluster 2024-09-12T23:35:49.954052+0000 mgr.y (mgr.24446) 85 : cluster [DBG] pgmap v63: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-12T23:35:52.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:51 smithi136 bash[32189]: cluster 2024-09-12T23:35:49.954052+0000 mgr.y (mgr.24446) 85 : cluster [DBG] pgmap v63: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-12T23:35:52.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:51 smithi181 bash[24776]: cluster 2024-09-12T23:35:49.954052+0000 mgr.y (mgr.24446) 85 : cluster [DBG] pgmap v63: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-12T23:35:52.239 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:35:52.239 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:35:52.240 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:35:52.240 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:35:53.974 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:53 smithi136 bash[22601]: cluster 2024-09-12T23:35:51.954577+0000 mgr.y (mgr.24446) 86 : cluster [DBG] pgmap v64: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:35:53.974 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:53 smithi136 bash[22601]: audit 2024-09-12T23:35:52.995022+0000 mon.a (mon.0) 880 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:35:53.974 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:53 smithi136 bash[32189]: cluster 2024-09-12T23:35:51.954577+0000 mgr.y (mgr.24446) 86 : cluster [DBG] pgmap v64: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:35:53.974 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:53 smithi136 bash[32189]: audit 2024-09-12T23:35:52.995022+0000 mon.a (mon.0) 880 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:35:54.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:53 smithi181 bash[24776]: cluster 2024-09-12T23:35:51.954577+0000 mgr.y (mgr.24446) 86 : cluster [DBG] pgmap v64: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:35:54.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:53 smithi181 bash[24776]: audit 2024-09-12T23:35:52.995022+0000 mon.a (mon.0) 880 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:35:55.596 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:35:55 smithi181 bash[25717]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:35:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-12T23:35:56.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:55 smithi136 bash[22601]: cluster 2024-09-12T23:35:53.955712+0000 mgr.y (mgr.24446) 87 : cluster [DBG] pgmap v65: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:35:56.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:55 smithi136 bash[32189]: cluster 2024-09-12T23:35:53.955712+0000 mgr.y (mgr.24446) 87 : cluster [DBG] pgmap v65: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:35:56.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:55 smithi181 bash[24776]: cluster 2024-09-12T23:35:53.955712+0000 mgr.y (mgr.24446) 87 : cluster [DBG] pgmap v65: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:35:56.506 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:35:56 smithi136 bash[22852]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:35:56] "GET /metrics HTTP/1.1" 200 236610 "" "Prometheus/2.43.0" 2024-09-12T23:35:57.712 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:57 smithi136 bash[22601]: cluster 2024-09-12T23:35:55.956355+0000 mgr.y (mgr.24446) 88 : cluster [DBG] pgmap v66: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:35:57.712 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:57 smithi136 bash[32189]: cluster 2024-09-12T23:35:55.956355+0000 mgr.y (mgr.24446) 88 : cluster [DBG] pgmap v66: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:35:58.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:57 smithi181 bash[24776]: cluster 2024-09-12T23:35:55.956355+0000 mgr.y (mgr.24446) 88 : cluster [DBG] pgmap v66: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:00.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:35:59 smithi136 bash[22601]: cluster 2024-09-12T23:35:57.957039+0000 mgr.y (mgr.24446) 89 : cluster [DBG] pgmap v67: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:00.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:35:59 smithi136 bash[32189]: cluster 2024-09-12T23:35:57.957039+0000 mgr.y (mgr.24446) 89 : cluster [DBG] pgmap v67: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:00.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:35:59 smithi181 bash[24776]: cluster 2024-09-12T23:35:57.957039+0000 mgr.y (mgr.24446) 89 : cluster [DBG] pgmap v67: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:02.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:01 smithi136 bash[22601]: cluster 2024-09-12T23:35:59.958214+0000 mgr.y (mgr.24446) 90 : cluster [DBG] pgmap v68: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:02.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:01 smithi136 bash[32189]: cluster 2024-09-12T23:35:59.958214+0000 mgr.y (mgr.24446) 90 : cluster [DBG] pgmap v68: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:02.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:01 smithi181 bash[24776]: cluster 2024-09-12T23:35:59.958214+0000 mgr.y (mgr.24446) 90 : cluster [DBG] pgmap v68: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:02.711 INFO:teuthology.orchestra.run.smithi136.stdout:107374182481 2024-09-12T23:36:02.897 INFO:teuthology.orchestra.run.smithi136.stdout:55834574950 2024-09-12T23:36:03.215 INFO:teuthology.orchestra.run.smithi136.stdout:154618822718 2024-09-12T23:36:03.547 INFO:teuthology.orchestra.run.smithi136.stdout:206158430249 2024-09-12T23:36:04.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:03 smithi136 bash[22601]: cluster 2024-09-12T23:36:01.958861+0000 mgr.y (mgr.24446) 91 : cluster [DBG] pgmap v69: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:04.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:03 smithi136 bash[22601]: audit 2024-09-12T23:36:02.715886+0000 mon.c (mon.2) 32 : audit [DBG] from='client.? 172.21.15.136:0/1326828516' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-12T23:36:04.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:03 smithi136 bash[22601]: audit 2024-09-12T23:36:02.901538+0000 mon.c (mon.2) 33 : audit [DBG] from='client.? 172.21.15.136:0/942168149' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-12T23:36:04.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:03 smithi136 bash[22601]: audit 2024-09-12T23:36:03.219883+0000 mon.c (mon.2) 34 : audit [DBG] from='client.? 172.21.15.136:0/3543395715' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-12T23:36:04.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:03 smithi136 bash[22601]: audit 2024-09-12T23:36:03.551810+0000 mon.a (mon.0) 881 : audit [DBG] from='client.? 172.21.15.136:0/3909895701' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-12T23:36:04.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:03 smithi136 bash[32189]: cluster 2024-09-12T23:36:01.958861+0000 mgr.y (mgr.24446) 91 : cluster [DBG] pgmap v69: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:04.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:03 smithi136 bash[32189]: audit 2024-09-12T23:36:02.715886+0000 mon.c (mon.2) 32 : audit [DBG] from='client.? 172.21.15.136:0/1326828516' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-12T23:36:04.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:03 smithi136 bash[32189]: audit 2024-09-12T23:36:02.901538+0000 mon.c (mon.2) 33 : audit [DBG] from='client.? 172.21.15.136:0/942168149' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-12T23:36:04.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:03 smithi136 bash[32189]: audit 2024-09-12T23:36:03.219883+0000 mon.c (mon.2) 34 : audit [DBG] from='client.? 172.21.15.136:0/3543395715' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-12T23:36:04.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:03 smithi136 bash[32189]: audit 2024-09-12T23:36:03.551810+0000 mon.a (mon.0) 881 : audit [DBG] from='client.? 172.21.15.136:0/3909895701' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-12T23:36:04.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:03 smithi181 bash[24776]: cluster 2024-09-12T23:36:01.958861+0000 mgr.y (mgr.24446) 91 : cluster [DBG] pgmap v69: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:04.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:03 smithi181 bash[24776]: audit 2024-09-12T23:36:02.715886+0000 mon.c (mon.2) 32 : audit [DBG] from='client.? 172.21.15.136:0/1326828516' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-12T23:36:04.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:03 smithi181 bash[24776]: audit 2024-09-12T23:36:02.901538+0000 mon.c (mon.2) 33 : audit [DBG] from='client.? 172.21.15.136:0/942168149' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-12T23:36:04.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:03 smithi181 bash[24776]: audit 2024-09-12T23:36:03.219883+0000 mon.c (mon.2) 34 : audit [DBG] from='client.? 172.21.15.136:0/3543395715' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-12T23:36:04.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:03 smithi181 bash[24776]: audit 2024-09-12T23:36:03.551810+0000 mon.a (mon.0) 881 : audit [DBG] from='client.? 172.21.15.136:0/3909895701' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-12T23:36:04.268 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182478 got 107374182481 for osd.3 2024-09-12T23:36:04.269 DEBUG:teuthology.parallel:result is None 2024-09-12T23:36:04.334 INFO:teuthology.orchestra.run.smithi136.stdout:77309411419 2024-09-12T23:36:05.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:04 smithi136 bash[32189]: audit 2024-09-12T23:36:04.338655+0000 mon.a (mon.0) 882 : audit [DBG] from='client.? 172.21.15.136:0/1717308801' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-12T23:36:05.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:04 smithi136 bash[22601]: audit 2024-09-12T23:36:04.338655+0000 mon.a (mon.0) 882 : audit [DBG] from='client.? 172.21.15.136:0/1717308801' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-12T23:36:05.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:04 smithi181 bash[24776]: audit 2024-09-12T23:36:04.338655+0000 mon.a (mon.0) 882 : audit [DBG] from='client.? 172.21.15.136:0/1717308801' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-12T23:36:05.504 INFO:teuthology.orchestra.run.smithi136.stdout:180388626484 2024-09-12T23:36:05.596 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:36:05 smithi181 bash[25717]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:36:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-12T23:36:05.621 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574947 got 55834574950 for osd.1 2024-09-12T23:36:05.621 DEBUG:teuthology.parallel:result is None 2024-09-12T23:36:05.709 INFO:teuthology.orchestra.run.smithi136.stdout:128849018952 2024-09-12T23:36:05.937 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:05 smithi136 bash[22601]: cluster 2024-09-12T23:36:03.960006+0000 mgr.y (mgr.24446) 92 : cluster [DBG] pgmap v70: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:05.937 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:05 smithi136 bash[22601]: audit 2024-09-12T23:36:05.509172+0000 mon.a (mon.0) 883 : audit [DBG] from='client.? 172.21.15.136:0/395886168' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-12T23:36:05.937 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:05 smithi136 bash[32189]: cluster 2024-09-12T23:36:03.960006+0000 mgr.y (mgr.24446) 92 : cluster [DBG] pgmap v70: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:05.937 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:05 smithi136 bash[32189]: audit 2024-09-12T23:36:05.509172+0000 mon.a (mon.0) 883 : audit [DBG] from='client.? 172.21.15.136:0/395886168' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-12T23:36:06.031 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430245 got 206158430249 for osd.7 2024-09-12T23:36:06.031 DEBUG:teuthology.parallel:result is None 2024-09-12T23:36:06.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:05 smithi181 bash[24776]: cluster 2024-09-12T23:36:03.960006+0000 mgr.y (mgr.24446) 92 : cluster [DBG] pgmap v70: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:06.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:05 smithi181 bash[24776]: audit 2024-09-12T23:36:05.509172+0000 mon.a (mon.0) 883 : audit [DBG] from='client.? 172.21.15.136:0/395886168' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-12T23:36:06.256 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:36:06 smithi136 bash[22852]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:36:06] "GET /metrics HTTP/1.1" 200 236610 "" "Prometheus/2.43.0" 2024-09-12T23:36:06.430 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618822715 got 154618822718 for osd.5 2024-09-12T23:36:06.430 DEBUG:teuthology.parallel:result is None 2024-09-12T23:36:06.645 INFO:teuthology.orchestra.run.smithi136.stdout:34359738479 2024-09-12T23:36:07.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:06 smithi136 bash[32189]: audit 2024-09-12T23:36:05.713425+0000 mon.c (mon.2) 35 : audit [DBG] from='client.? 172.21.15.136:0/3531694988' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-12T23:36:07.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:06 smithi136 bash[32189]: cluster 2024-09-12T23:36:05.960613+0000 mgr.y (mgr.24446) 93 : cluster [DBG] pgmap v71: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:07.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:06 smithi136 bash[32189]: audit 2024-09-12T23:36:06.649777+0000 mon.c (mon.2) 36 : audit [DBG] from='client.? 172.21.15.136:0/4018068333' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-12T23:36:07.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:06 smithi136 bash[22601]: audit 2024-09-12T23:36:05.713425+0000 mon.c (mon.2) 35 : audit [DBG] from='client.? 172.21.15.136:0/3531694988' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-12T23:36:07.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:06 smithi136 bash[22601]: cluster 2024-09-12T23:36:05.960613+0000 mgr.y (mgr.24446) 93 : cluster [DBG] pgmap v71: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:07.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:06 smithi136 bash[22601]: audit 2024-09-12T23:36:06.649777+0000 mon.c (mon.2) 36 : audit [DBG] from='client.? 172.21.15.136:0/4018068333' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-12T23:36:07.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:06 smithi181 bash[24776]: audit 2024-09-12T23:36:05.713425+0000 mon.c (mon.2) 35 : audit [DBG] from='client.? 172.21.15.136:0/3531694988' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-12T23:36:07.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:06 smithi181 bash[24776]: cluster 2024-09-12T23:36:05.960613+0000 mgr.y (mgr.24446) 93 : cluster [DBG] pgmap v71: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:07.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:06 smithi181 bash[24776]: audit 2024-09-12T23:36:06.649777+0000 mon.c (mon.2) 36 : audit [DBG] from='client.? 172.21.15.136:0/4018068333' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-12T23:36:07.455 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411415 got 77309411419 for osd.2 2024-09-12T23:36:07.455 DEBUG:teuthology.parallel:result is None 2024-09-12T23:36:07.985 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388626480 got 180388626484 for osd.6 2024-09-12T23:36:07.985 DEBUG:teuthology.parallel:result is None 2024-09-12T23:36:08.014 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018948 got 128849018952 for osd.4 2024-09-12T23:36:08.015 DEBUG:teuthology.parallel:result is None 2024-09-12T23:36:08.149 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738475 got 34359738479 for osd.0 2024-09-12T23:36:08.149 DEBUG:teuthology.parallel:result is None 2024-09-12T23:36:08.149 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-12T23:36:08.149 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-12T23:36:08.256 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:08 smithi136 bash[32189]: audit 2024-09-12T23:36:07.998725+0000 mon.a (mon.0) 884 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:36:08.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:08 smithi136 bash[22601]: audit 2024-09-12T23:36:07.998725+0000 mon.a (mon.0) 884 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:36:08.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:08 smithi181 bash[24776]: audit 2024-09-12T23:36:07.998725+0000 mon.a (mon.0) 884 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:36:09.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:09 smithi181 bash[24776]: cluster 2024-09-12T23:36:07.961276+0000 mgr.y (mgr.24446) 94 : cluster [DBG] pgmap v72: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:09.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:09 smithi136 bash[22601]: cluster 2024-09-12T23:36:07.961276+0000 mgr.y (mgr.24446) 94 : cluster [DBG] pgmap v72: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:09.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:09 smithi136 bash[32189]: cluster 2024-09-12T23:36:07.961276+0000 mgr.y (mgr.24446) 94 : cluster [DBG] pgmap v72: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:11.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:11 smithi181 bash[24776]: cluster 2024-09-12T23:36:09.962551+0000 mgr.y (mgr.24446) 95 : cluster [DBG] pgmap v73: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:11.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:11 smithi136 bash[22601]: cluster 2024-09-12T23:36:09.962551+0000 mgr.y (mgr.24446) 95 : cluster [DBG] pgmap v73: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:11.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:11 smithi136 bash[32189]: cluster 2024-09-12T23:36:09.962551+0000 mgr.y (mgr.24446) 95 : cluster [DBG] pgmap v73: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:12.934 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:36:13.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:13 smithi181 bash[24776]: cluster 2024-09-12T23:36:11.963174+0000 mgr.y (mgr.24446) 96 : cluster [DBG] pgmap v74: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:13.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:13 smithi136 bash[22601]: cluster 2024-09-12T23:36:11.963174+0000 mgr.y (mgr.24446) 96 : cluster [DBG] pgmap v74: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:13.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:13 smithi136 bash[32189]: cluster 2024-09-12T23:36:11.963174+0000 mgr.y (mgr.24446) 96 : cluster [DBG] pgmap v74: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:15.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:15 smithi136 bash[22601]: cluster 2024-09-12T23:36:13.964327+0000 mgr.y (mgr.24446) 97 : cluster [DBG] pgmap v75: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:15.256 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:15 smithi136 bash[32189]: cluster 2024-09-12T23:36:13.964327+0000 mgr.y (mgr.24446) 97 : cluster [DBG] pgmap v75: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:15.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:15 smithi181 bash[24776]: cluster 2024-09-12T23:36:13.964327+0000 mgr.y (mgr.24446) 97 : cluster [DBG] pgmap v75: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:15.346 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:36:15 smithi181 bash[25717]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:36:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-12T23:36:16.104 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-12T23:36:16.107 INFO:teuthology.orchestra.run.smithi136.stderr:dumped all 2024-09-12T23:36:16.506 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:36:16 smithi136 bash[22852]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:36:16] "GET /metrics HTTP/1.1" 200 236600 "" "Prometheus/2.43.0" 2024-09-12T23:36:16.826 INFO:teuthology.orchestra.run.smithi136.stdout:{"pg_ready":true,"pg_map":{"version":76,"stamp":"2024-09-12T23:36:15.964545+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":593797,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":166,"num_write_kb":1649,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":127,"ondisk_log_size":127,"up":315,"acting":315,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":315,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385764,"kb_used_data":6920,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575452,"statfs":{"total":767960285184,"available":765517262848,"internally_reserved":0,"allocated":7086080,"data_stored":4112419,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.005403"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.665027+0000","last_change":"2024-09-12T23:33:29.908602+0000","last_active":"2024-09-12T23:35:08.665027+0000","last_peered":"2024-09-12T23:35:08.665027+0000","last_clean":"2024-09-12T23:35:08.665027+0000","last_became_active":"2024-09-12T23:33:29.908161+0000","last_became_peered":"2024-09-12T23:33:29.908161+0000","last_unstale":"2024-09-12T23:35:08.665027+0000","last_undegraded":"2024-09-12T23:35:08.665027+0000","last_fullsized":"2024-09-12T23:35:08.665027+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:18:25.662166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.664503+0000","last_change":"2024-09-12T23:33:29.907919+0000","last_active":"2024-09-12T23:35:08.664503+0000","last_peered":"2024-09-12T23:35:08.664503+0000","last_clean":"2024-09-12T23:35:08.664503+0000","last_became_active":"2024-09-12T23:33:29.906765+0000","last_became_peered":"2024-09-12T23:33:29.906765+0000","last_unstale":"2024-09-12T23:35:08.664503+0000","last_undegraded":"2024-09-12T23:35:08.664503+0000","last_fullsized":"2024-09-12T23:35:08.664503+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:39:49.787701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.671331+0000","last_change":"2024-09-12T23:33:29.924626+0000","last_active":"2024-09-12T23:35:08.671331+0000","last_peered":"2024-09-12T23:35:08.671331+0000","last_clean":"2024-09-12T23:35:08.671331+0000","last_became_active":"2024-09-12T23:33:29.924268+0000","last_became_peered":"2024-09-12T23:33:29.924268+0000","last_unstale":"2024-09-12T23:35:08.671331+0000","last_undegraded":"2024-09-12T23:35:08.671331+0000","last_fullsized":"2024-09-12T23:35:08.671331+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:41:31.853826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.670520+0000","last_change":"2024-09-12T23:33:29.913990+0000","last_active":"2024-09-12T23:35:08.670520+0000","last_peered":"2024-09-12T23:35:08.670520+0000","last_clean":"2024-09-12T23:35:08.670520+0000","last_became_active":"2024-09-12T23:33:29.913672+0000","last_became_peered":"2024-09-12T23:33:29.913672+0000","last_unstale":"2024-09-12T23:35:08.670520+0000","last_undegraded":"2024-09-12T23:35:08.670520+0000","last_fullsized":"2024-09-12T23:35:08.670520+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:57:54.241724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.663791+0000","last_change":"2024-09-12T23:33:29.904505+0000","last_active":"2024-09-12T23:35:08.663791+0000","last_peered":"2024-09-12T23:35:08.663791+0000","last_clean":"2024-09-12T23:35:08.663791+0000","last_became_active":"2024-09-12T23:33:29.904192+0000","last_became_peered":"2024-09-12T23:33:29.904192+0000","last_unstale":"2024-09-12T23:35:08.663791+0000","last_undegraded":"2024-09-12T23:35:08.663791+0000","last_fullsized":"2024-09-12T23:35:08.663791+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:49:27.086584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.671038+0000","last_change":"2024-09-12T23:33:29.905467+0000","last_active":"2024-09-12T23:35:08.671038+0000","last_peered":"2024-09-12T23:35:08.671038+0000","last_clean":"2024-09-12T23:35:08.671038+0000","last_became_active":"2024-09-12T23:33:29.905154+0000","last_became_peered":"2024-09-12T23:33:29.905154+0000","last_unstale":"2024-09-12T23:35:08.671038+0000","last_undegraded":"2024-09-12T23:35:08.671038+0000","last_fullsized":"2024-09-12T23:35:08.671038+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:49:00.596502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.664458+0000","last_change":"2024-09-12T23:33:29.908762+0000","last_active":"2024-09-12T23:35:08.664458+0000","last_peered":"2024-09-12T23:35:08.664458+0000","last_clean":"2024-09-12T23:35:08.664458+0000","last_became_active":"2024-09-12T23:33:29.908427+0000","last_became_peered":"2024-09-12T23:33:29.908427+0000","last_unstale":"2024-09-12T23:35:08.664458+0000","last_undegraded":"2024-09-12T23:35:08.664458+0000","last_fullsized":"2024-09-12T23:35:08.664458+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:44:26.357691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.666998+0000","last_change":"2024-09-12T23:33:29.902033+0000","last_active":"2024-09-12T23:35:08.666998+0000","last_peered":"2024-09-12T23:35:08.666998+0000","last_clean":"2024-09-12T23:35:08.666998+0000","last_became_active":"2024-09-12T23:33:29.901611+0000","last_became_peered":"2024-09-12T23:33:29.901611+0000","last_unstale":"2024-09-12T23:35:08.666998+0000","last_undegraded":"2024-09-12T23:35:08.666998+0000","last_fullsized":"2024-09-12T23:35:08.666998+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:57:58.671558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":30,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-12T23:35:09.803676+0000","last_change":"2024-09-12T23:35:09.803676+0000","last_active":"2024-09-12T23:35:09.803676+0000","last_peered":"2024-09-12T23:35:09.803676+0000","last_clean":"2024-09-12T23:35:09.803676+0000","last_became_active":"2024-09-12T23:35:09.802434+0000","last_became_peered":"2024-09-12T23:35:09.802434+0000","last_unstale":"2024-09-12T23:35:09.803676+0000","last_undegraded":"2024-09-12T23:35:09.803676+0000","last_fullsized":"2024-09-12T23:35:09.803676+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:01:37.156347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.666805+0000","last_change":"2024-09-12T23:33:29.893241+0000","last_active":"2024-09-12T23:35:08.666805+0000","last_peered":"2024-09-12T23:35:08.666805+0000","last_clean":"2024-09-12T23:35:08.666805+0000","last_became_active":"2024-09-12T23:33:29.892815+0000","last_became_peered":"2024-09-12T23:33:29.892815+0000","last_unstale":"2024-09-12T23:35:08.666805+0000","last_undegraded":"2024-09-12T23:35:08.666805+0000","last_fullsized":"2024-09-12T23:35:08.666805+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:46:21.539771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.665734+0000","last_change":"2024-09-12T23:33:29.906810+0000","last_active":"2024-09-12T23:35:08.665734+0000","last_peered":"2024-09-12T23:35:08.665734+0000","last_clean":"2024-09-12T23:35:08.665734+0000","last_became_active":"2024-09-12T23:33:29.906493+0000","last_became_peered":"2024-09-12T23:33:29.906493+0000","last_unstale":"2024-09-12T23:35:08.665734+0000","last_undegraded":"2024-09-12T23:35:08.665734+0000","last_fullsized":"2024-09-12T23:35:08.665734+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:56:07.145258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":30,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-12T23:35:09.802722+0000","last_change":"2024-09-12T23:35:09.802722+0000","last_active":"2024-09-12T23:35:09.802722+0000","last_peered":"2024-09-12T23:35:09.802722+0000","last_clean":"2024-09-12T23:35:09.802722+0000","last_became_active":"2024-09-12T23:35:09.801763+0000","last_became_peered":"2024-09-12T23:35:09.801763+0000","last_unstale":"2024-09-12T23:35:09.802722+0000","last_undegraded":"2024-09-12T23:35:09.802722+0000","last_fullsized":"2024-09-12T23:35:09.802722+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:33:24.574457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.668470+0000","last_change":"2024-09-12T23:33:29.907459+0000","last_active":"2024-09-12T23:35:08.668470+0000","last_peered":"2024-09-12T23:35:08.668470+0000","last_clean":"2024-09-12T23:35:08.668470+0000","last_became_active":"2024-09-12T23:33:29.906577+0000","last_became_peered":"2024-09-12T23:33:29.906577+0000","last_unstale":"2024-09-12T23:35:08.668470+0000","last_undegraded":"2024-09-12T23:35:08.668470+0000","last_fullsized":"2024-09-12T23:35:08.668470+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:21:41.131616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.670257+0000","last_change":"2024-09-12T23:33:29.920721+0000","last_active":"2024-09-12T23:35:08.670257+0000","last_peered":"2024-09-12T23:35:08.670257+0000","last_clean":"2024-09-12T23:35:08.670257+0000","last_became_active":"2024-09-12T23:33:29.918051+0000","last_became_peered":"2024-09-12T23:33:29.918051+0000","last_unstale":"2024-09-12T23:35:08.670257+0000","last_undegraded":"2024-09-12T23:35:08.670257+0000","last_fullsized":"2024-09-12T23:35:08.670257+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:32:27.077464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":30,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-12T23:35:09.683561+0000","last_change":"2024-09-12T23:35:09.683561+0000","last_active":"2024-09-12T23:35:09.683561+0000","last_peered":"2024-09-12T23:35:09.683561+0000","last_clean":"2024-09-12T23:35:09.683561+0000","last_became_active":"2024-09-12T23:35:09.682671+0000","last_became_peered":"2024-09-12T23:35:09.682671+0000","last_unstale":"2024-09-12T23:35:09.683561+0000","last_undegraded":"2024-09-12T23:35:09.683561+0000","last_fullsized":"2024-09-12T23:35:09.683561+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:45:39.393684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":10,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-12T23:35:09.804288+0000","last_change":"2024-09-12T23:35:09.804288+0000","last_active":"2024-09-12T23:35:09.804288+0000","last_peered":"2024-09-12T23:35:09.804288+0000","last_clean":"2024-09-12T23:35:09.804288+0000","last_became_active":"2024-09-12T23:35:09.803450+0000","last_became_peered":"2024-09-12T23:35:09.803450+0000","last_unstale":"2024-09-12T23:35:09.804288+0000","last_undegraded":"2024-09-12T23:35:09.804288+0000","last_fullsized":"2024-09-12T23:35:09.804288+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:31:12.292070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.670478+0000","last_change":"2024-09-12T23:33:29.913684+0000","last_active":"2024-09-12T23:35:08.670478+0000","last_peered":"2024-09-12T23:35:08.670478+0000","last_clean":"2024-09-12T23:35:08.670478+0000","last_became_active":"2024-09-12T23:33:29.911877+0000","last_became_peered":"2024-09-12T23:33:29.911877+0000","last_unstale":"2024-09-12T23:35:08.670478+0000","last_undegraded":"2024-09-12T23:35:08.670478+0000","last_fullsized":"2024-09-12T23:35:08.670478+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:39:10.601087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.661668+0000","last_change":"2024-09-12T23:33:29.882889+0000","last_active":"2024-09-12T23:35:08.661668+0000","last_peered":"2024-09-12T23:35:08.661668+0000","last_clean":"2024-09-12T23:35:08.661668+0000","last_became_active":"2024-09-12T23:33:29.882148+0000","last_became_peered":"2024-09-12T23:33:29.882148+0000","last_unstale":"2024-09-12T23:35:08.661668+0000","last_undegraded":"2024-09-12T23:35:08.661668+0000","last_fullsized":"2024-09-12T23:35:08.661668+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:59:38.545795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.666136+0000","last_change":"2024-09-12T23:33:29.904552+0000","last_active":"2024-09-12T23:35:08.666136+0000","last_peered":"2024-09-12T23:35:08.666136+0000","last_clean":"2024-09-12T23:35:08.666136+0000","last_became_active":"2024-09-12T23:33:29.904247+0000","last_became_peered":"2024-09-12T23:33:29.904247+0000","last_unstale":"2024-09-12T23:35:08.666136+0000","last_undegraded":"2024-09-12T23:35:08.666136+0000","last_fullsized":"2024-09-12T23:35:08.666136+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:30:06.600814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.665287+0000","last_change":"2024-09-12T23:33:29.905965+0000","last_active":"2024-09-12T23:35:08.665287+0000","last_peered":"2024-09-12T23:35:08.665287+0000","last_clean":"2024-09-12T23:35:08.665287+0000","last_became_active":"2024-09-12T23:33:29.905690+0000","last_became_peered":"2024-09-12T23:33:29.905690+0000","last_unstale":"2024-09-12T23:35:08.665287+0000","last_undegraded":"2024-09-12T23:35:08.665287+0000","last_fullsized":"2024-09-12T23:35:08.665287+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:58:29.597052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.669549+0000","last_change":"2024-09-12T23:33:29.918231+0000","last_active":"2024-09-12T23:35:08.669549+0000","last_peered":"2024-09-12T23:35:08.669549+0000","last_clean":"2024-09-12T23:35:08.669549+0000","last_became_active":"2024-09-12T23:33:29.917878+0000","last_became_peered":"2024-09-12T23:33:29.917878+0000","last_unstale":"2024-09-12T23:35:08.669549+0000","last_undegraded":"2024-09-12T23:35:08.669549+0000","last_fullsized":"2024-09-12T23:35:08.669549+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:22:46.498222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.671076+0000","last_change":"2024-09-12T23:33:29.909780+0000","last_active":"2024-09-12T23:35:08.671076+0000","last_peered":"2024-09-12T23:35:08.671076+0000","last_clean":"2024-09-12T23:35:08.671076+0000","last_became_active":"2024-09-12T23:33:29.909507+0000","last_became_peered":"2024-09-12T23:33:29.909507+0000","last_unstale":"2024-09-12T23:35:08.671076+0000","last_undegraded":"2024-09-12T23:35:08.671076+0000","last_fullsized":"2024-09-12T23:35:08.671076+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:39:50.529545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.667544+0000","last_change":"2024-09-12T23:33:29.906087+0000","last_active":"2024-09-12T23:35:08.667544+0000","last_peered":"2024-09-12T23:35:08.667544+0000","last_clean":"2024-09-12T23:35:08.667544+0000","last_became_active":"2024-09-12T23:33:29.905780+0000","last_became_peered":"2024-09-12T23:33:29.905780+0000","last_unstale":"2024-09-12T23:35:08.667544+0000","last_undegraded":"2024-09-12T23:35:08.667544+0000","last_fullsized":"2024-09-12T23:35:08.667544+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:00:40.777941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.670507+0000","last_change":"2024-09-12T23:33:29.918286+0000","last_active":"2024-09-12T23:35:08.670507+0000","last_peered":"2024-09-12T23:35:08.670507+0000","last_clean":"2024-09-12T23:35:08.670507+0000","last_became_active":"2024-09-12T23:33:29.917757+0000","last_became_peered":"2024-09-12T23:33:29.917757+0000","last_unstale":"2024-09-12T23:35:08.670507+0000","last_undegraded":"2024-09-12T23:35:08.670507+0000","last_fullsized":"2024-09-12T23:35:08.670507+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:49:26.856852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.673387+0000","last_change":"2024-09-12T23:33:29.903381+0000","last_active":"2024-09-12T23:35:08.673387+0000","last_peered":"2024-09-12T23:35:08.673387+0000","last_clean":"2024-09-12T23:35:08.673387+0000","last_became_active":"2024-09-12T23:33:29.903081+0000","last_became_peered":"2024-09-12T23:33:29.903081+0000","last_unstale":"2024-09-12T23:35:08.673387+0000","last_undegraded":"2024-09-12T23:35:08.673387+0000","last_fullsized":"2024-09-12T23:35:08.673387+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:43:01.175691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.666859+0000","last_change":"2024-09-12T23:33:29.897170+0000","last_active":"2024-09-12T23:35:08.666859+0000","last_peered":"2024-09-12T23:35:08.666859+0000","last_clean":"2024-09-12T23:35:08.666859+0000","last_became_active":"2024-09-12T23:33:29.896834+0000","last_became_peered":"2024-09-12T23:33:29.896834+0000","last_unstale":"2024-09-12T23:35:08.666859+0000","last_undegraded":"2024-09-12T23:35:08.666859+0000","last_fullsized":"2024-09-12T23:35:08.666859+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:47:43.534530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.663571+0000","last_change":"2024-09-12T23:33:29.919500+0000","last_active":"2024-09-12T23:35:08.663571+0000","last_peered":"2024-09-12T23:35:08.663571+0000","last_clean":"2024-09-12T23:35:08.663571+0000","last_became_active":"2024-09-12T23:33:29.919185+0000","last_became_peered":"2024-09-12T23:33:29.919185+0000","last_unstale":"2024-09-12T23:35:08.663571+0000","last_undegraded":"2024-09-12T23:35:08.663571+0000","last_fullsized":"2024-09-12T23:35:08.663571+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:56:52.566524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.672875+0000","last_change":"2024-09-12T23:33:29.913907+0000","last_active":"2024-09-12T23:35:08.672875+0000","last_peered":"2024-09-12T23:35:08.672875+0000","last_clean":"2024-09-12T23:35:08.672875+0000","last_became_active":"2024-09-12T23:33:29.913193+0000","last_became_peered":"2024-09-12T23:33:29.913193+0000","last_unstale":"2024-09-12T23:35:08.672875+0000","last_undegraded":"2024-09-12T23:35:08.672875+0000","last_fullsized":"2024-09-12T23:35:08.672875+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:32:45.117056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.669650+0000","last_change":"2024-09-12T23:33:29.911279+0000","last_active":"2024-09-12T23:35:08.669650+0000","last_peered":"2024-09-12T23:35:08.669650+0000","last_clean":"2024-09-12T23:35:08.669650+0000","last_became_active":"2024-09-12T23:33:29.911007+0000","last_became_peered":"2024-09-12T23:33:29.911007+0000","last_unstale":"2024-09-12T23:35:08.669650+0000","last_undegraded":"2024-09-12T23:35:08.669650+0000","last_fullsized":"2024-09-12T23:35:08.669650+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:26:51.842858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.671116+0000","last_change":"2024-09-12T23:33:29.914015+0000","last_active":"2024-09-12T23:35:08.671116+0000","last_peered":"2024-09-12T23:35:08.671116+0000","last_clean":"2024-09-12T23:35:08.671116+0000","last_became_active":"2024-09-12T23:33:29.913659+0000","last_became_peered":"2024-09-12T23:33:29.913659+0000","last_unstale":"2024-09-12T23:35:08.671116+0000","last_undegraded":"2024-09-12T23:35:08.671116+0000","last_fullsized":"2024-09-12T23:35:08.671116+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:37:02.989629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.661319+0000","last_change":"2024-09-12T23:33:29.889782+0000","last_active":"2024-09-12T23:35:08.661319+0000","last_peered":"2024-09-12T23:35:08.661319+0000","last_clean":"2024-09-12T23:35:08.661319+0000","last_became_active":"2024-09-12T23:33:29.889548+0000","last_became_peered":"2024-09-12T23:33:29.889548+0000","last_unstale":"2024-09-12T23:35:08.661319+0000","last_undegraded":"2024-09-12T23:35:08.661319+0000","last_fullsized":"2024-09-12T23:35:08.661319+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-13T23:54:53.906619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.665474+0000","last_change":"2024-09-12T23:33:29.896515+0000","last_active":"2024-09-12T23:35:08.665474+0000","last_peered":"2024-09-12T23:35:08.665474+0000","last_clean":"2024-09-12T23:35:08.665474+0000","last_became_active":"2024-09-12T23:33:29.895919+0000","last_became_peered":"2024-09-12T23:33:29.895919+0000","last_unstale":"2024-09-12T23:35:08.665474+0000","last_undegraded":"2024-09-12T23:35:08.665474+0000","last_fullsized":"2024-09-12T23:35:08.665474+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:17:53.675501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.660905+0000","last_change":"2024-09-12T23:33:29.891596+0000","last_active":"2024-09-12T23:35:08.660905+0000","last_peered":"2024-09-12T23:35:08.660905+0000","last_clean":"2024-09-12T23:35:08.660905+0000","last_became_active":"2024-09-12T23:33:29.890839+0000","last_became_peered":"2024-09-12T23:33:29.890839+0000","last_unstale":"2024-09-12T23:35:08.660905+0000","last_undegraded":"2024-09-12T23:35:08.660905+0000","last_fullsized":"2024-09-12T23:35:08.660905+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:29:49.995950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1b","version":"53'1","reported_seq":22,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.668622+0000","last_change":"2024-09-12T23:33:28.883222+0000","last_active":"2024-09-12T23:35:08.668622+0000","last_peered":"2024-09-12T23:35:08.668622+0000","last_clean":"2024-09-12T23:35:08.668622+0000","last_became_active":"2024-09-12T23:33:28.882848+0000","last_became_peered":"2024-09-12T23:33:28.882848+0000","last_unstale":"2024-09-12T23:35:08.668622+0000","last_undegraded":"2024-09-12T23:35:08.668622+0000","last_fullsized":"2024-09-12T23:35:08.668622+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:57:33.905996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.669749+0000","last_change":"2024-09-12T23:33:29.912763+0000","last_active":"2024-09-12T23:35:08.669749+0000","last_peered":"2024-09-12T23:35:08.669749+0000","last_clean":"2024-09-12T23:35:08.669749+0000","last_became_active":"2024-09-12T23:33:29.912406+0000","last_became_peered":"2024-09-12T23:33:29.912406+0000","last_unstale":"2024-09-12T23:35:08.669749+0000","last_undegraded":"2024-09-12T23:35:08.669749+0000","last_fullsized":"2024-09-12T23:35:08.669749+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:49:06.828514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1a","version":"53'1","reported_seq":22,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.667384+0000","last_change":"2024-09-12T23:33:28.884599+0000","last_active":"2024-09-12T23:35:08.667384+0000","last_peered":"2024-09-12T23:35:08.667384+0000","last_clean":"2024-09-12T23:35:08.667384+0000","last_became_active":"2024-09-12T23:33:28.884139+0000","last_became_peered":"2024-09-12T23:33:28.884139+0000","last_unstale":"2024-09-12T23:35:08.667384+0000","last_undegraded":"2024-09-12T23:35:08.667384+0000","last_fullsized":"2024-09-12T23:35:08.667384+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:57:50.175287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":21,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.668064+0000","last_change":"2024-09-12T23:33:28.872822+0000","last_active":"2024-09-12T23:35:08.668064+0000","last_peered":"2024-09-12T23:35:08.668064+0000","last_clean":"2024-09-12T23:35:08.668064+0000","last_became_active":"2024-09-12T23:33:28.872404+0000","last_became_peered":"2024-09-12T23:33:28.872404+0000","last_unstale":"2024-09-12T23:35:08.668064+0000","last_undegraded":"2024-09-12T23:35:08.668064+0000","last_fullsized":"2024-09-12T23:35:08.668064+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-13T23:47:42.317408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.667077+0000","last_change":"2024-09-12T23:33:29.907360+0000","last_active":"2024-09-12T23:35:08.667077+0000","last_peered":"2024-09-12T23:35:08.667077+0000","last_clean":"2024-09-12T23:35:08.667077+0000","last_became_active":"2024-09-12T23:33:29.907067+0000","last_became_peered":"2024-09-12T23:33:29.907067+0000","last_unstale":"2024-09-12T23:35:08.667077+0000","last_undegraded":"2024-09-12T23:35:08.667077+0000","last_fullsized":"2024-09-12T23:35:08.667077+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:26:38.084938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":21,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.670406+0000","last_change":"2024-09-12T23:33:28.896704+0000","last_active":"2024-09-12T23:35:08.670406+0000","last_peered":"2024-09-12T23:35:08.670406+0000","last_clean":"2024-09-12T23:35:08.670406+0000","last_became_active":"2024-09-12T23:33:28.896167+0000","last_became_peered":"2024-09-12T23:33:28.896167+0000","last_unstale":"2024-09-12T23:35:08.670406+0000","last_undegraded":"2024-09-12T23:35:08.670406+0000","last_fullsized":"2024-09-12T23:35:08.670406+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:14:47.959501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.663618+0000","last_change":"2024-09-12T23:33:29.893401+0000","last_active":"2024-09-12T23:35:08.663618+0000","last_peered":"2024-09-12T23:35:08.663618+0000","last_clean":"2024-09-12T23:35:08.663618+0000","last_became_active":"2024-09-12T23:33:29.892988+0000","last_became_peered":"2024-09-12T23:33:29.892988+0000","last_unstale":"2024-09-12T23:35:08.663618+0000","last_undegraded":"2024-09-12T23:35:08.663618+0000","last_fullsized":"2024-09-12T23:35:08.663618+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:30:04.271132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":21,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.666010+0000","last_change":"2024-09-12T23:33:28.884106+0000","last_active":"2024-09-12T23:35:08.666010+0000","last_peered":"2024-09-12T23:35:08.666010+0000","last_clean":"2024-09-12T23:35:08.666010+0000","last_became_active":"2024-09-12T23:33:28.883028+0000","last_became_peered":"2024-09-12T23:33:28.883028+0000","last_unstale":"2024-09-12T23:35:08.666010+0000","last_undegraded":"2024-09-12T23:35:08.666010+0000","last_fullsized":"2024-09-12T23:35:08.666010+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:20:27.774655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.670853+0000","last_change":"2024-09-12T23:33:29.909613+0000","last_active":"2024-09-12T23:35:08.670853+0000","last_peered":"2024-09-12T23:35:08.670853+0000","last_clean":"2024-09-12T23:35:08.670853+0000","last_became_active":"2024-09-12T23:33:29.909325+0000","last_became_peered":"2024-09-12T23:33:29.909325+0000","last_unstale":"2024-09-12T23:35:08.670853+0000","last_undegraded":"2024-09-12T23:35:08.670853+0000","last_fullsized":"2024-09-12T23:35:08.670853+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:50:21.069936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":21,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.670886+0000","last_change":"2024-09-12T23:33:28.890012+0000","last_active":"2024-09-12T23:35:08.670886+0000","last_peered":"2024-09-12T23:35:08.670886+0000","last_clean":"2024-09-12T23:35:08.670886+0000","last_became_active":"2024-09-12T23:33:28.887477+0000","last_became_peered":"2024-09-12T23:33:28.887477+0000","last_unstale":"2024-09-12T23:35:08.670886+0000","last_undegraded":"2024-09-12T23:35:08.670886+0000","last_fullsized":"2024-09-12T23:35:08.670886+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:15:02.988389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.670299+0000","last_change":"2024-09-12T23:33:29.917549+0000","last_active":"2024-09-12T23:35:08.670299+0000","last_peered":"2024-09-12T23:35:08.670299+0000","last_clean":"2024-09-12T23:35:08.670299+0000","last_became_active":"2024-09-12T23:33:29.916648+0000","last_became_peered":"2024-09-12T23:33:29.916648+0000","last_unstale":"2024-09-12T23:35:08.670299+0000","last_undegraded":"2024-09-12T23:35:08.670299+0000","last_fullsized":"2024-09-12T23:35:08.670299+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:37:55.708866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"53'3","reported_seq":24,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.663816+0000","last_change":"2024-09-12T23:33:28.879732+0000","last_active":"2024-09-12T23:35:08.663816+0000","last_peered":"2024-09-12T23:35:08.663816+0000","last_clean":"2024-09-12T23:35:08.663816+0000","last_became_active":"2024-09-12T23:33:28.879399+0000","last_became_peered":"2024-09-12T23:33:28.879399+0000","last_unstale":"2024-09-12T23:35:08.663816+0000","last_undegraded":"2024-09-12T23:35:08.663816+0000","last_fullsized":"2024-09-12T23:35:08.663816+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:00:06.109352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":10,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-12T23:35:09.684524+0000","last_change":"2024-09-12T23:35:09.684524+0000","last_active":"2024-09-12T23:35:09.684524+0000","last_peered":"2024-09-12T23:35:09.684524+0000","last_clean":"2024-09-12T23:35:09.684524+0000","last_became_active":"2024-09-12T23:35:09.683923+0000","last_became_peered":"2024-09-12T23:35:09.683923+0000","last_unstale":"2024-09-12T23:35:09.684524+0000","last_undegraded":"2024-09-12T23:35:09.684524+0000","last_fullsized":"2024-09-12T23:35:09.684524+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:30:40.963502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,6],"acting":[2,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.14","version":"64'12","reported_seq":33,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-12T23:35:38.071488+0000","last_change":"2024-09-12T23:33:28.886930+0000","last_active":"2024-09-12T23:35:38.071488+0000","last_peered":"2024-09-12T23:35:38.071488+0000","last_clean":"2024-09-12T23:35:38.071488+0000","last_became_active":"2024-09-12T23:33:28.885991+0000","last_became_peered":"2024-09-12T23:33:28.885991+0000","last_unstale":"2024-09-12T23:35:38.071488+0000","last_undegraded":"2024-09-12T23:35:38.071488+0000","last_fullsized":"2024-09-12T23:35:38.071488+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":12,"log_dups_size":0,"ondisk_log_size":12,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:50:58.477301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2624,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":9,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.665823+0000","last_change":"2024-09-12T23:33:29.916929+0000","last_active":"2024-09-12T23:35:08.665823+0000","last_peered":"2024-09-12T23:35:08.665823+0000","last_clean":"2024-09-12T23:35:08.665823+0000","last_became_active":"2024-09-12T23:33:29.916622+0000","last_became_peered":"2024-09-12T23:33:29.916622+0000","last_unstale":"2024-09-12T23:35:08.665823+0000","last_undegraded":"2024-09-12T23:35:08.665823+0000","last_fullsized":"2024-09-12T23:35:08.665823+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:40:09.433278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.13","version":"53'2","reported_seq":23,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.669915+0000","last_change":"2024-09-12T23:33:28.880385+0000","last_active":"2024-09-12T23:35:08.669915+0000","last_peered":"2024-09-12T23:35:08.669915+0000","last_clean":"2024-09-12T23:35:08.669915+0000","last_became_active":"2024-09-12T23:33:28.879825+0000","last_became_peered":"2024-09-12T23:33:28.879825+0000","last_unstale":"2024-09-12T23:35:08.669915+0000","last_undegraded":"2024-09-12T23:35:08.669915+0000","last_fullsized":"2024-09-12T23:35:08.669915+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:40:33.073593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.670434+0000","last_change":"2024-09-12T23:33:29.919045+0000","last_active":"2024-09-12T23:35:08.670434+0000","last_peered":"2024-09-12T23:35:08.670434+0000","last_clean":"2024-09-12T23:35:08.670434+0000","last_became_active":"2024-09-12T23:33:29.918580+0000","last_became_peered":"2024-09-12T23:33:29.918580+0000","last_unstale":"2024-09-12T23:35:08.670434+0000","last_undegraded":"2024-09-12T23:35:08.670434+0000","last_fullsized":"2024-09-12T23:35:08.670434+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:39:32.320667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":21,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.671158+0000","last_change":"2024-09-12T23:33:28.899424+0000","last_active":"2024-09-12T23:35:08.671158+0000","last_peered":"2024-09-12T23:35:08.671158+0000","last_clean":"2024-09-12T23:35:08.671158+0000","last_became_active":"2024-09-12T23:33:28.898942+0000","last_became_peered":"2024-09-12T23:33:28.898942+0000","last_unstale":"2024-09-12T23:35:08.671158+0000","last_undegraded":"2024-09-12T23:35:08.671158+0000","last_fullsized":"2024-09-12T23:35:08.671158+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:14:47.069896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.663898+0000","last_change":"2024-09-12T23:33:29.901397+0000","last_active":"2024-09-12T23:35:08.663898+0000","last_peered":"2024-09-12T23:35:08.663898+0000","last_clean":"2024-09-12T23:35:08.663898+0000","last_became_active":"2024-09-12T23:33:29.901060+0000","last_became_peered":"2024-09-12T23:33:29.901060+0000","last_unstale":"2024-09-12T23:35:08.663898+0000","last_undegraded":"2024-09-12T23:35:08.663898+0000","last_fullsized":"2024-09-12T23:35:08.663898+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:21:23.285498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":21,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.665621+0000","last_change":"2024-09-12T23:33:28.871430+0000","last_active":"2024-09-12T23:35:08.665621+0000","last_peered":"2024-09-12T23:35:08.665621+0000","last_clean":"2024-09-12T23:35:08.665621+0000","last_became_active":"2024-09-12T23:33:28.869528+0000","last_became_peered":"2024-09-12T23:33:28.869528+0000","last_unstale":"2024-09-12T23:35:08.665621+0000","last_undegraded":"2024-09-12T23:35:08.665621+0000","last_fullsized":"2024-09-12T23:35:08.665621+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:27:43.449941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":30,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-12T23:35:09.683003+0000","last_change":"2024-09-12T23:35:09.683003+0000","last_active":"2024-09-12T23:35:09.683003+0000","last_peered":"2024-09-12T23:35:09.683003+0000","last_clean":"2024-09-12T23:35:09.683003+0000","last_became_active":"2024-09-12T23:35:09.681812+0000","last_became_peered":"2024-09-12T23:35:09.681812+0000","last_unstale":"2024-09-12T23:35:09.683003+0000","last_undegraded":"2024-09-12T23:35:09.683003+0000","last_fullsized":"2024-09-12T23:35:09.683003+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-13T23:47:15.002852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"53'8","reported_seq":29,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.661383+0000","last_change":"2024-09-12T23:33:28.863788+0000","last_active":"2024-09-12T23:35:08.661383+0000","last_peered":"2024-09-12T23:35:08.661383+0000","last_clean":"2024-09-12T23:35:08.661383+0000","last_became_active":"2024-09-12T23:33:28.863479+0000","last_became_peered":"2024-09-12T23:33:28.863479+0000","last_unstale":"2024-09-12T23:35:08.661383+0000","last_undegraded":"2024-09-12T23:35:08.661383+0000","last_fullsized":"2024-09-12T23:35:08.661383+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:26:09.204038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.664571+0000","last_change":"2024-09-12T23:33:29.902101+0000","last_active":"2024-09-12T23:35:08.664571+0000","last_peered":"2024-09-12T23:35:08.664571+0000","last_clean":"2024-09-12T23:35:08.664571+0000","last_became_active":"2024-09-12T23:33:29.901803+0000","last_became_peered":"2024-09-12T23:33:29.901803+0000","last_unstale":"2024-09-12T23:35:08.664571+0000","last_undegraded":"2024-09-12T23:35:08.664571+0000","last_fullsized":"2024-09-12T23:35:08.664571+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:00:48.557318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.f","version":"53'1","reported_seq":22,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.662210+0000","last_change":"2024-09-12T23:33:28.879318+0000","last_active":"2024-09-12T23:35:08.662210+0000","last_peered":"2024-09-12T23:35:08.662210+0000","last_clean":"2024-09-12T23:35:08.662210+0000","last_became_active":"2024-09-12T23:33:28.877913+0000","last_became_peered":"2024-09-12T23:33:28.877913+0000","last_unstale":"2024-09-12T23:35:08.662210+0000","last_undegraded":"2024-09-12T23:35:08.662210+0000","last_fullsized":"2024-09-12T23:35:08.662210+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:13:19.574352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.664646+0000","last_change":"2024-09-12T23:33:29.901391+0000","last_active":"2024-09-12T23:35:08.664646+0000","last_peered":"2024-09-12T23:35:08.664646+0000","last_clean":"2024-09-12T23:35:08.664646+0000","last_became_active":"2024-09-12T23:33:29.901052+0000","last_became_peered":"2024-09-12T23:33:29.901052+0000","last_unstale":"2024-09-12T23:35:08.664646+0000","last_undegraded":"2024-09-12T23:35:08.664646+0000","last_fullsized":"2024-09-12T23:35:08.664646+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:54:04.075006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"53'2","reported_seq":23,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.661659+0000","last_change":"2024-09-12T23:33:28.887441+0000","last_active":"2024-09-12T23:35:08.661659+0000","last_peered":"2024-09-12T23:35:08.661659+0000","last_clean":"2024-09-12T23:35:08.661659+0000","last_became_active":"2024-09-12T23:33:28.887109+0000","last_became_peered":"2024-09-12T23:33:28.887109+0000","last_unstale":"2024-09-12T23:35:08.661659+0000","last_undegraded":"2024-09-12T23:35:08.661659+0000","last_fullsized":"2024-09-12T23:35:08.661659+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:35:14.071342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.665111+0000","last_change":"2024-09-12T23:33:29.907594+0000","last_active":"2024-09-12T23:35:08.665111+0000","last_peered":"2024-09-12T23:35:08.665111+0000","last_clean":"2024-09-12T23:35:08.665111+0000","last_became_active":"2024-09-12T23:33:29.907302+0000","last_became_peered":"2024-09-12T23:33:29.907302+0000","last_unstale":"2024-09-12T23:35:08.665111+0000","last_undegraded":"2024-09-12T23:35:08.665111+0000","last_fullsized":"2024-09-12T23:35:08.665111+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:45:37.988381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"53'1","reported_seq":22,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.663915+0000","last_change":"2024-09-12T23:33:28.871944+0000","last_active":"2024-09-12T23:35:08.663915+0000","last_peered":"2024-09-12T23:35:08.663915+0000","last_clean":"2024-09-12T23:35:08.663915+0000","last_became_active":"2024-09-12T23:33:28.871628+0000","last_became_peered":"2024-09-12T23:33:28.871628+0000","last_unstale":"2024-09-12T23:35:08.663915+0000","last_undegraded":"2024-09-12T23:35:08.663915+0000","last_fullsized":"2024-09-12T23:35:08.663915+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:58:29.645624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.670659+0000","last_change":"2024-09-12T23:33:29.914833+0000","last_active":"2024-09-12T23:35:08.670659+0000","last_peered":"2024-09-12T23:35:08.670659+0000","last_clean":"2024-09-12T23:35:08.670659+0000","last_became_active":"2024-09-12T23:33:29.914416+0000","last_became_peered":"2024-09-12T23:33:29.914416+0000","last_unstale":"2024-09-12T23:35:08.670659+0000","last_undegraded":"2024-09-12T23:35:08.670659+0000","last_fullsized":"2024-09-12T23:35:08.670659+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:27:20.253954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.c","version":"0'0","reported_seq":21,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.662019+0000","last_change":"2024-09-12T23:33:28.882138+0000","last_active":"2024-09-12T23:35:08.662019+0000","last_peered":"2024-09-12T23:35:08.662019+0000","last_clean":"2024-09-12T23:35:08.662019+0000","last_became_active":"2024-09-12T23:33:28.881774+0000","last_became_peered":"2024-09-12T23:33:28.881774+0000","last_unstale":"2024-09-12T23:35:08.662019+0000","last_undegraded":"2024-09-12T23:35:08.662019+0000","last_fullsized":"2024-09-12T23:35:08.662019+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:07:36.075590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.665947+0000","last_change":"2024-09-12T23:33:29.911549+0000","last_active":"2024-09-12T23:35:08.665947+0000","last_peered":"2024-09-12T23:35:08.665947+0000","last_clean":"2024-09-12T23:35:08.665947+0000","last_became_active":"2024-09-12T23:33:29.911244+0000","last_became_peered":"2024-09-12T23:33:29.911244+0000","last_unstale":"2024-09-12T23:35:08.665947+0000","last_undegraded":"2024-09-12T23:35:08.665947+0000","last_fullsized":"2024-09-12T23:35:08.665947+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:22:20.405567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":21,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.666935+0000","last_change":"2024-09-12T23:34:59.598602+0000","last_active":"2024-09-12T23:35:08.666935+0000","last_peered":"2024-09-12T23:35:08.666935+0000","last_clean":"2024-09-12T23:35:08.666935+0000","last_became_active":"2024-09-12T23:34:49.734724+0000","last_became_peered":"2024-09-12T23:34:49.734724+0000","last_unstale":"2024-09-12T23:35:08.666935+0000","last_undegraded":"2024-09-12T23:35:08.666935+0000","last_fullsized":"2024-09-12T23:35:08.666935+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:34:48.454503+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:34:48.454503+0000","last_clean_scrub_stamp":"2024-09-12T23:34:48.454503+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:27:05.298933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00079141900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"55'79","reported_seq":55,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.664101+0000","last_change":"2024-09-12T23:32:45.902460+0000","last_active":"2024-09-12T23:35:08.664101+0000","last_peered":"2024-09-12T23:35:08.664101+0000","last_clean":"2024-09-12T23:35:08.664101+0000","last_became_active":"2024-09-12T23:32:45.887205+0000","last_became_peered":"2024-09-12T23:32:45.887205+0000","last_unstale":"2024-09-12T23:35:08.664101+0000","last_undegraded":"2024-09-12T23:35:08.664101+0000","last_fullsized":"2024-09-12T23:35:08.664101+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:28:34.324093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:28:34.324093+0000","last_clean_scrub_stamp":"2024-09-12T23:28:34.324093+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:37:37.825919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":21,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.670187+0000","last_change":"2024-09-12T23:33:28.898796+0000","last_active":"2024-09-12T23:35:08.670187+0000","last_peered":"2024-09-12T23:35:08.670187+0000","last_clean":"2024-09-12T23:35:08.670187+0000","last_became_active":"2024-09-12T23:33:28.898329+0000","last_became_peered":"2024-09-12T23:33:28.898329+0000","last_unstale":"2024-09-12T23:35:08.670187+0000","last_undegraded":"2024-09-12T23:35:08.670187+0000","last_fullsized":"2024-09-12T23:35:08.670187+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:54:48.474286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.668367+0000","last_change":"2024-09-12T23:33:29.905028+0000","last_active":"2024-09-12T23:35:08.668367+0000","last_peered":"2024-09-12T23:35:08.668367+0000","last_clean":"2024-09-12T23:35:08.668367+0000","last_became_active":"2024-09-12T23:33:29.904695+0000","last_became_peered":"2024-09-12T23:33:29.904695+0000","last_unstale":"2024-09-12T23:35:08.668367+0000","last_undegraded":"2024-09-12T23:35:08.668367+0000","last_fullsized":"2024-09-12T23:35:08.668367+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:06:47.004852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":21,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.669471+0000","last_change":"2024-09-12T23:34:59.542653+0000","last_active":"2024-09-12T23:35:08.669471+0000","last_peered":"2024-09-12T23:35:08.669471+0000","last_clean":"2024-09-12T23:35:08.669471+0000","last_became_active":"2024-09-12T23:34:49.788254+0000","last_became_peered":"2024-09-12T23:34:49.788254+0000","last_unstale":"2024-09-12T23:35:08.669471+0000","last_undegraded":"2024-09-12T23:35:08.669471+0000","last_fullsized":"2024-09-12T23:35:08.669471+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:34:48.454503+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:34:48.454503+0000","last_clean_scrub_stamp":"2024-09-12T23:34:48.454503+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:47:44.627533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0012029090000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":21,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.663998+0000","last_change":"2024-09-12T23:33:28.895135+0000","last_active":"2024-09-12T23:35:08.663998+0000","last_peered":"2024-09-12T23:35:08.663998+0000","last_clean":"2024-09-12T23:35:08.663998+0000","last_became_active":"2024-09-12T23:33:28.894687+0000","last_became_peered":"2024-09-12T23:33:28.894687+0000","last_unstale":"2024-09-12T23:35:08.663998+0000","last_undegraded":"2024-09-12T23:35:08.663998+0000","last_fullsized":"2024-09-12T23:35:08.663998+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:57:06.646238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.669881+0000","last_change":"2024-09-12T23:33:29.910025+0000","last_active":"2024-09-12T23:35:08.669881+0000","last_peered":"2024-09-12T23:35:08.669881+0000","last_clean":"2024-09-12T23:35:08.669881+0000","last_became_active":"2024-09-12T23:33:29.909765+0000","last_became_peered":"2024-09-12T23:33:29.909765+0000","last_unstale":"2024-09-12T23:35:08.669881+0000","last_undegraded":"2024-09-12T23:35:08.669881+0000","last_fullsized":"2024-09-12T23:35:08.669881+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:23:35.389216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":21,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.663799+0000","last_change":"2024-09-12T23:34:59.549321+0000","last_active":"2024-09-12T23:35:08.663799+0000","last_peered":"2024-09-12T23:35:08.663799+0000","last_clean":"2024-09-12T23:35:08.663799+0000","last_became_active":"2024-09-12T23:34:49.788699+0000","last_became_peered":"2024-09-12T23:34:49.788699+0000","last_unstale":"2024-09-12T23:35:08.663799+0000","last_undegraded":"2024-09-12T23:35:08.663799+0000","last_fullsized":"2024-09-12T23:35:08.663799+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:34:48.454503+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:34:48.454503+0000","last_clean_scrub_stamp":"2024-09-12T23:34:48.454503+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:29:56.436091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0020296509999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":21,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.661420+0000","last_change":"2024-09-12T23:33:28.868957+0000","last_active":"2024-09-12T23:35:08.661420+0000","last_peered":"2024-09-12T23:35:08.661420+0000","last_clean":"2024-09-12T23:35:08.661420+0000","last_became_active":"2024-09-12T23:33:28.868587+0000","last_became_peered":"2024-09-12T23:33:28.868587+0000","last_unstale":"2024-09-12T23:35:08.661420+0000","last_undegraded":"2024-09-12T23:35:08.661420+0000","last_fullsized":"2024-09-12T23:35:08.661420+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:45:32.284209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.663738+0000","last_change":"2024-09-12T23:33:29.890360+0000","last_active":"2024-09-12T23:35:08.663738+0000","last_peered":"2024-09-12T23:35:08.663738+0000","last_clean":"2024-09-12T23:35:08.663738+0000","last_became_active":"2024-09-12T23:33:29.889575+0000","last_became_peered":"2024-09-12T23:33:29.889575+0000","last_unstale":"2024-09-12T23:35:08.663738+0000","last_undegraded":"2024-09-12T23:35:08.663738+0000","last_fullsized":"2024-09-12T23:35:08.663738+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:36:34.938054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":21,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.663553+0000","last_change":"2024-09-12T23:34:59.548775+0000","last_active":"2024-09-12T23:35:08.663553+0000","last_peered":"2024-09-12T23:35:08.663553+0000","last_clean":"2024-09-12T23:35:08.663553+0000","last_became_active":"2024-09-12T23:34:49.787490+0000","last_became_peered":"2024-09-12T23:34:49.787490+0000","last_unstale":"2024-09-12T23:35:08.663553+0000","last_undegraded":"2024-09-12T23:35:08.663553+0000","last_fullsized":"2024-09-12T23:35:08.663553+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:34:48.454503+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:34:48.454503+0000","last_clean_scrub_stamp":"2024-09-12T23:34:48.454503+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:19:10.283468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0020519940000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":21,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.671197+0000","last_change":"2024-09-12T23:33:28.899931+0000","last_active":"2024-09-12T23:35:08.671197+0000","last_peered":"2024-09-12T23:35:08.671197+0000","last_clean":"2024-09-12T23:35:08.671197+0000","last_became_active":"2024-09-12T23:33:28.899410+0000","last_became_peered":"2024-09-12T23:33:28.899410+0000","last_unstale":"2024-09-12T23:35:08.671197+0000","last_undegraded":"2024-09-12T23:35:08.671197+0000","last_fullsized":"2024-09-12T23:35:08.671197+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:05:26.671172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.661569+0000","last_change":"2024-09-12T23:33:29.902878+0000","last_active":"2024-09-12T23:35:08.661569+0000","last_peered":"2024-09-12T23:35:08.661569+0000","last_clean":"2024-09-12T23:35:08.661569+0000","last_became_active":"2024-09-12T23:33:29.902561+0000","last_became_peered":"2024-09-12T23:33:29.902561+0000","last_unstale":"2024-09-12T23:35:08.661569+0000","last_undegraded":"2024-09-12T23:35:08.661569+0000","last_fullsized":"2024-09-12T23:35:08.661569+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:12:55.352139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.2","version":"61'2","reported_seq":23,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.664284+0000","last_change":"2024-09-12T23:34:59.548734+0000","last_active":"2024-09-12T23:35:08.664284+0000","last_peered":"2024-09-12T23:35:08.664284+0000","last_clean":"2024-09-12T23:35:08.664284+0000","last_became_active":"2024-09-12T23:34:49.792105+0000","last_became_peered":"2024-09-12T23:34:49.792105+0000","last_unstale":"2024-09-12T23:35:08.664284+0000","last_undegraded":"2024-09-12T23:35:08.664284+0000","last_fullsized":"2024-09-12T23:35:08.664284+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:34:48.454503+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:34:48.454503+0000","last_clean_scrub_stamp":"2024-09-12T23:34:48.454503+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-13T23:37:40.496016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.002201358,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":21,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.664823+0000","last_change":"2024-09-12T23:33:28.867304+0000","last_active":"2024-09-12T23:35:08.664823+0000","last_peered":"2024-09-12T23:35:08.664823+0000","last_clean":"2024-09-12T23:35:08.664823+0000","last_became_active":"2024-09-12T23:33:28.867020+0000","last_became_peered":"2024-09-12T23:33:28.867020+0000","last_unstale":"2024-09-12T23:35:08.664823+0000","last_undegraded":"2024-09-12T23:35:08.664823+0000","last_fullsized":"2024-09-12T23:35:08.664823+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:51:22.908188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.670703+0000","last_change":"2024-09-12T23:33:29.914728+0000","last_active":"2024-09-12T23:35:08.670703+0000","last_peered":"2024-09-12T23:35:08.670703+0000","last_clean":"2024-09-12T23:35:08.670703+0000","last_became_active":"2024-09-12T23:33:29.914385+0000","last_became_peered":"2024-09-12T23:33:29.914385+0000","last_unstale":"2024-09-12T23:35:08.670703+0000","last_undegraded":"2024-09-12T23:35:08.670703+0000","last_fullsized":"2024-09-12T23:35:08.670703+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:45:02.815733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"59'1","reported_seq":22,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.669319+0000","last_change":"2024-09-12T23:34:59.541122+0000","last_active":"2024-09-12T23:35:08.669319+0000","last_peered":"2024-09-12T23:35:08.669319+0000","last_clean":"2024-09-12T23:35:08.669319+0000","last_became_active":"2024-09-12T23:34:49.787835+0000","last_became_peered":"2024-09-12T23:34:49.787835+0000","last_unstale":"2024-09-12T23:35:08.669319+0000","last_undegraded":"2024-09-12T23:35:08.669319+0000","last_fullsized":"2024-09-12T23:35:08.669319+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:34:48.454503+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:34:48.454503+0000","last_clean_scrub_stamp":"2024-09-12T23:34:48.454503+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:07:48.849192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099226499999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":21,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.666213+0000","last_change":"2024-09-12T23:33:28.895077+0000","last_active":"2024-09-12T23:35:08.666213+0000","last_peered":"2024-09-12T23:35:08.666213+0000","last_clean":"2024-09-12T23:35:08.666213+0000","last_became_active":"2024-09-12T23:33:28.892268+0000","last_became_peered":"2024-09-12T23:33:28.892268+0000","last_unstale":"2024-09-12T23:35:08.666213+0000","last_undegraded":"2024-09-12T23:35:08.666213+0000","last_fullsized":"2024-09-12T23:35:08.666213+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:26:19.125877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.664444+0000","last_change":"2024-09-12T23:33:29.899700+0000","last_active":"2024-09-12T23:35:08.664444+0000","last_peered":"2024-09-12T23:35:08.664444+0000","last_clean":"2024-09-12T23:35:08.664444+0000","last_became_active":"2024-09-12T23:33:29.899381+0000","last_became_peered":"2024-09-12T23:33:29.899381+0000","last_unstale":"2024-09-12T23:35:08.664444+0000","last_undegraded":"2024-09-12T23:35:08.664444+0000","last_fullsized":"2024-09-12T23:35:08.664444+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:30:20.590373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":21,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.666658+0000","last_change":"2024-09-12T23:34:59.899761+0000","last_active":"2024-09-12T23:35:08.666658+0000","last_peered":"2024-09-12T23:35:08.666658+0000","last_clean":"2024-09-12T23:35:08.666658+0000","last_became_active":"2024-09-12T23:34:49.736609+0000","last_became_peered":"2024-09-12T23:34:49.736609+0000","last_unstale":"2024-09-12T23:35:08.666658+0000","last_undegraded":"2024-09-12T23:35:08.666658+0000","last_fullsized":"2024-09-12T23:35:08.666658+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:34:48.454503+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:34:48.454503+0000","last_clean_scrub_stamp":"2024-09-12T23:34:48.454503+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:01:06.682284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001798914,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"54'1","reported_seq":22,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.665023+0000","last_change":"2024-09-12T23:33:28.866367+0000","last_active":"2024-09-12T23:35:08.665023+0000","last_peered":"2024-09-12T23:35:08.665023+0000","last_clean":"2024-09-12T23:35:08.665023+0000","last_became_active":"2024-09-12T23:33:28.866005+0000","last_became_peered":"2024-09-12T23:33:28.866005+0000","last_unstale":"2024-09-12T23:35:08.665023+0000","last_undegraded":"2024-09-12T23:35:08.665023+0000","last_fullsized":"2024-09-12T23:35:08.665023+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:49:27.803045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.673648+0000","last_change":"2024-09-12T23:33:29.915275+0000","last_active":"2024-09-12T23:35:08.673648+0000","last_peered":"2024-09-12T23:35:08.673648+0000","last_clean":"2024-09-12T23:35:08.673648+0000","last_became_active":"2024-09-12T23:33:29.914934+0000","last_became_peered":"2024-09-12T23:33:29.914934+0000","last_unstale":"2024-09-12T23:35:08.673648+0000","last_undegraded":"2024-09-12T23:35:08.673648+0000","last_fullsized":"2024-09-12T23:35:08.673648+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:43:27.562543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":21,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.661581+0000","last_change":"2024-09-12T23:34:59.550136+0000","last_active":"2024-09-12T23:35:08.661581+0000","last_peered":"2024-09-12T23:35:08.661581+0000","last_clean":"2024-09-12T23:35:08.661581+0000","last_became_active":"2024-09-12T23:34:49.785409+0000","last_became_peered":"2024-09-12T23:34:49.785409+0000","last_unstale":"2024-09-12T23:35:08.661581+0000","last_undegraded":"2024-09-12T23:35:08.661581+0000","last_fullsized":"2024-09-12T23:35:08.661581+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:34:48.454503+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:34:48.454503+0000","last_clean_scrub_stamp":"2024-09-12T23:34:48.454503+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:30:29.736480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010162299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"53'2","reported_seq":23,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.665837+0000","last_change":"2024-09-12T23:33:28.887351+0000","last_active":"2024-09-12T23:35:08.665837+0000","last_peered":"2024-09-12T23:35:08.665837+0000","last_clean":"2024-09-12T23:35:08.665837+0000","last_became_active":"2024-09-12T23:33:28.886573+0000","last_became_peered":"2024-09-12T23:33:28.886573+0000","last_unstale":"2024-09-12T23:35:08.665837+0000","last_undegraded":"2024-09-12T23:35:08.665837+0000","last_fullsized":"2024-09-12T23:35:08.665837+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:58:40.820895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.669756+0000","last_change":"2024-09-12T23:33:29.908474+0000","last_active":"2024-09-12T23:35:08.669756+0000","last_peered":"2024-09-12T23:35:08.669756+0000","last_clean":"2024-09-12T23:35:08.669756+0000","last_became_active":"2024-09-12T23:33:29.906278+0000","last_became_peered":"2024-09-12T23:33:29.906278+0000","last_unstale":"2024-09-12T23:35:08.669756+0000","last_undegraded":"2024-09-12T23:35:08.669756+0000","last_fullsized":"2024-09-12T23:35:08.669756+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:27:51.832430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"0'0","reported_seq":21,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.664547+0000","last_change":"2024-09-12T23:33:28.894286+0000","last_active":"2024-09-12T23:35:08.664547+0000","last_peered":"2024-09-12T23:35:08.664547+0000","last_clean":"2024-09-12T23:35:08.664547+0000","last_became_active":"2024-09-12T23:33:28.893861+0000","last_became_peered":"2024-09-12T23:33:28.893861+0000","last_unstale":"2024-09-12T23:35:08.664547+0000","last_undegraded":"2024-09-12T23:35:08.664547+0000","last_fullsized":"2024-09-12T23:35:08.664547+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:20:01.806723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":10,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-12T23:35:09.676563+0000","last_change":"2024-09-12T23:35:09.676563+0000","last_active":"2024-09-12T23:35:09.676563+0000","last_peered":"2024-09-12T23:35:09.676563+0000","last_clean":"2024-09-12T23:35:09.676563+0000","last_became_active":"2024-09-12T23:35:09.675322+0000","last_became_peered":"2024-09-12T23:35:09.675322+0000","last_unstale":"2024-09-12T23:35:09.676563+0000","last_undegraded":"2024-09-12T23:35:09.676563+0000","last_fullsized":"2024-09-12T23:35:09.676563+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:11:37.262719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":21,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.664006+0000","last_change":"2024-09-12T23:33:28.884360+0000","last_active":"2024-09-12T23:35:08.664006+0000","last_peered":"2024-09-12T23:35:08.664006+0000","last_clean":"2024-09-12T23:35:08.664006+0000","last_became_active":"2024-09-12T23:33:28.884027+0000","last_became_peered":"2024-09-12T23:33:28.884027+0000","last_unstale":"2024-09-12T23:35:08.664006+0000","last_undegraded":"2024-09-12T23:35:08.664006+0000","last_fullsized":"2024-09-12T23:35:08.664006+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:11:57.567074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.667167+0000","last_change":"2024-09-12T23:33:29.915243+0000","last_active":"2024-09-12T23:35:08.667167+0000","last_peered":"2024-09-12T23:35:08.667167+0000","last_clean":"2024-09-12T23:35:08.667167+0000","last_became_active":"2024-09-12T23:33:29.914543+0000","last_became_peered":"2024-09-12T23:33:29.914543+0000","last_unstale":"2024-09-12T23:35:08.667167+0000","last_undegraded":"2024-09-12T23:35:08.667167+0000","last_fullsized":"2024-09-12T23:35:08.667167+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-13T23:48:16.023985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":21,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.664752+0000","last_change":"2024-09-12T23:33:28.883601+0000","last_active":"2024-09-12T23:35:08.664752+0000","last_peered":"2024-09-12T23:35:08.664752+0000","last_clean":"2024-09-12T23:35:08.664752+0000","last_became_active":"2024-09-12T23:33:28.883276+0000","last_became_peered":"2024-09-12T23:33:28.883276+0000","last_unstale":"2024-09-12T23:35:08.664752+0000","last_undegraded":"2024-09-12T23:35:08.664752+0000","last_fullsized":"2024-09-12T23:35:08.664752+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:26:21.733615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.668547+0000","last_change":"2024-09-12T23:33:29.908185+0000","last_active":"2024-09-12T23:35:08.668547+0000","last_peered":"2024-09-12T23:35:08.668547+0000","last_clean":"2024-09-12T23:35:08.668547+0000","last_became_active":"2024-09-12T23:33:29.907814+0000","last_became_peered":"2024-09-12T23:33:29.907814+0000","last_unstale":"2024-09-12T23:35:08.668547+0000","last_undegraded":"2024-09-12T23:35:08.668547+0000","last_fullsized":"2024-09-12T23:35:08.668547+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:45:47.207242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"53'2","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-12T23:35:09.689819+0000","last_change":"2024-09-12T23:35:09.689819+0000","last_active":"2024-09-12T23:35:09.689819+0000","last_peered":"2024-09-12T23:35:09.689819+0000","last_clean":"2024-09-12T23:35:09.689819+0000","last_became_active":"2024-09-12T23:35:09.683737+0000","last_became_peered":"2024-09-12T23:35:09.683737+0000","last_unstale":"2024-09-12T23:35:09.689819+0000","last_undegraded":"2024-09-12T23:35:09.689819+0000","last_fullsized":"2024-09-12T23:35:09.689819+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:18:02.546637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.667383+0000","last_change":"2024-09-12T23:33:29.897132+0000","last_active":"2024-09-12T23:35:08.667383+0000","last_peered":"2024-09-12T23:35:08.667383+0000","last_clean":"2024-09-12T23:35:08.667383+0000","last_became_active":"2024-09-12T23:33:29.896761+0000","last_became_peered":"2024-09-12T23:33:29.896761+0000","last_unstale":"2024-09-12T23:35:08.667383+0000","last_undegraded":"2024-09-12T23:35:08.667383+0000","last_fullsized":"2024-09-12T23:35:08.667383+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:49:56.554393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.670622+0000","last_change":"2024-09-12T23:33:29.911440+0000","last_active":"2024-09-12T23:35:08.670622+0000","last_peered":"2024-09-12T23:35:08.670622+0000","last_clean":"2024-09-12T23:35:08.670622+0000","last_became_active":"2024-09-12T23:33:29.911157+0000","last_became_peered":"2024-09-12T23:33:29.911157+0000","last_unstale":"2024-09-12T23:35:08.670622+0000","last_undegraded":"2024-09-12T23:35:08.670622+0000","last_fullsized":"2024-09-12T23:35:08.670622+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:49:59.313436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"53'2","reported_seq":23,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.664158+0000","last_change":"2024-09-12T23:33:28.902224+0000","last_active":"2024-09-12T23:35:08.664158+0000","last_peered":"2024-09-12T23:35:08.664158+0000","last_clean":"2024-09-12T23:35:08.664158+0000","last_became_active":"2024-09-12T23:33:28.901755+0000","last_became_peered":"2024-09-12T23:33:28.901755+0000","last_unstale":"2024-09-12T23:35:08.664158+0000","last_undegraded":"2024-09-12T23:35:08.664158+0000","last_fullsized":"2024-09-12T23:35:08.664158+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:06:15.504175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.670603+0000","last_change":"2024-09-12T23:33:29.913814+0000","last_active":"2024-09-12T23:35:08.670603+0000","last_peered":"2024-09-12T23:35:08.670603+0000","last_clean":"2024-09-12T23:35:08.670603+0000","last_became_active":"2024-09-12T23:33:29.913521+0000","last_became_peered":"2024-09-12T23:33:29.913521+0000","last_unstale":"2024-09-12T23:35:08.670603+0000","last_undegraded":"2024-09-12T23:35:08.670603+0000","last_fullsized":"2024-09-12T23:35:08.670603+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:30:35.754521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1d","version":"53'2","reported_seq":23,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.666037+0000","last_change":"2024-09-12T23:33:28.897466+0000","last_active":"2024-09-12T23:35:08.666037+0000","last_peered":"2024-09-12T23:35:08.666037+0000","last_clean":"2024-09-12T23:35:08.666037+0000","last_became_active":"2024-09-12T23:33:28.896936+0000","last_became_peered":"2024-09-12T23:33:28.896936+0000","last_unstale":"2024-09-12T23:35:08.666037+0000","last_undegraded":"2024-09-12T23:35:08.666037+0000","last_fullsized":"2024-09-12T23:35:08.666037+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:29:38.732168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.670437+0000","last_change":"2024-09-12T23:33:29.924610+0000","last_active":"2024-09-12T23:35:08.670437+0000","last_peered":"2024-09-12T23:35:08.670437+0000","last_clean":"2024-09-12T23:35:08.670437+0000","last_became_active":"2024-09-12T23:33:29.924320+0000","last_became_peered":"2024-09-12T23:33:29.924320+0000","last_unstale":"2024-09-12T23:35:08.670437+0000","last_undegraded":"2024-09-12T23:35:08.670437+0000","last_fullsized":"2024-09-12T23:35:08.670437+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:23:22.797301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1e","version":"53'1","reported_seq":22,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.673262+0000","last_change":"2024-09-12T23:33:28.894353+0000","last_active":"2024-09-12T23:35:08.673262+0000","last_peered":"2024-09-12T23:35:08.673262+0000","last_clean":"2024-09-12T23:35:08.673262+0000","last_became_active":"2024-09-12T23:33:28.893992+0000","last_became_peered":"2024-09-12T23:33:28.893992+0000","last_unstale":"2024-09-12T23:35:08.673262+0000","last_undegraded":"2024-09-12T23:35:08.673262+0000","last_fullsized":"2024-09-12T23:35:08.673262+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:43:43.889025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.673097+0000","last_change":"2024-09-12T23:33:29.891227+0000","last_active":"2024-09-12T23:35:08.673097+0000","last_peered":"2024-09-12T23:35:08.673097+0000","last_clean":"2024-09-12T23:35:08.673097+0000","last_became_active":"2024-09-12T23:33:29.890368+0000","last_became_peered":"2024-09-12T23:33:29.890368+0000","last_unstale":"2024-09-12T23:35:08.673097+0000","last_undegraded":"2024-09-12T23:35:08.673097+0000","last_fullsized":"2024-09-12T23:35:08.673097+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:38:37.526610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1f","version":"53'4","reported_seq":25,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.670421+0000","last_change":"2024-09-12T23:33:28.886016+0000","last_active":"2024-09-12T23:35:08.670421+0000","last_peered":"2024-09-12T23:35:08.670421+0000","last_clean":"2024-09-12T23:35:08.670421+0000","last_became_active":"2024-09-12T23:33:28.885720+0000","last_became_peered":"2024-09-12T23:33:28.885720+0000","last_unstale":"2024-09-12T23:35:08.670421+0000","last_undegraded":"2024-09-12T23:35:08.670421+0000","last_fullsized":"2024-09-12T23:35:08.670421+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:44:38.950039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":192,"acting":192,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":3410,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":51,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":10230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":45,"ondisk_log_size":45,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430252,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298580,"kb_used_data":1224,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446572,"statfs":{"total":95995035648,"available":95689289728,"internally_reserved":0,"allocated":1253376,"data_stored":882496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 12 23:35:49 2024","interfaces":[{"interface":"back","average":{"1min":0.75700000000000001,"5min":0.86899999999999999,"15min":0.89800000000000002},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.256,"5min":1.3600000000000001,"15min":1.3600000000000001},"last":0.95999999999999996},{"interface":"front","average":{"1min":0.77500000000000002,"5min":0.89100000000000001,"15min":0.92900000000000005},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":1.0720000000000001,"5min":1.3740000000000001,"15min":1.3740000000000001},"last":1.125}]},{"osd":1,"last update":"Thu Sep 12 23:35:49 2024","interfaces":[{"interface":"back","average":{"1min":0.80700000000000005,"5min":0.90200000000000002,"15min":0.93400000000000005},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":1.179,"5min":1.3300000000000001,"15min":1.3300000000000001},"last":1.0249999999999999},{"interface":"front","average":{"1min":0.80800000000000005,"5min":0.872,"15min":0.88800000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":1.2270000000000001,"5min":1.474,"15min":1.474},"last":0.97999999999999998}]},{"osd":2,"last update":"Thu Sep 12 23:35:49 2024","interfaces":[{"interface":"back","average":{"1min":0.81100000000000005,"5min":0.89100000000000001,"15min":0.874},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":1.167,"5min":5.7919999999999998,"15min":5.7919999999999998},"last":1.081},{"interface":"front","average":{"1min":0.82199999999999995,"5min":0.93500000000000005,"15min":0.96199999999999997},"min":{"1min":0.47199999999999998,"5min":0.435,"15min":0.435},"max":{"1min":1.085,"5min":1.363,"15min":1.363},"last":0.76600000000000001}]},{"osd":3,"last update":"Thu Sep 12 23:35:49 2024","interfaces":[{"interface":"back","average":{"1min":0.83299999999999996,"5min":0.91500000000000004,"15min":0.95099999999999996},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":1.2170000000000001,"5min":1.284,"15min":1.284},"last":0.85299999999999998},{"interface":"front","average":{"1min":0.92000000000000004,"5min":1,"15min":1.0369999999999999},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":1.2030000000000001,"5min":1.407,"15min":1.407},"last":0.91500000000000004}]},{"osd":4,"last update":"Thu Sep 12 23:35:49 2024","interfaces":[{"interface":"back","average":{"1min":0.72599999999999998,"5min":0.79600000000000004,"15min":0.80300000000000005},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.0780000000000001,"5min":1.589,"15min":1.589},"last":0.70699999999999996},{"interface":"front","average":{"1min":0.80800000000000005,"5min":0.93799999999999994,"15min":0.995},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.179,"5min":1.3009999999999999,"15min":1.3009999999999999},"last":1.169}]},{"osd":5,"last update":"Thu Sep 12 23:35:49 2024","interfaces":[{"interface":"back","average":{"1min":0.80200000000000005,"5min":0.95799999999999996,"15min":1.016},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":1.1970000000000001,"5min":1.371,"15min":1.371},"last":1.1990000000000001},{"interface":"front","average":{"1min":0.84399999999999997,"5min":0.93999999999999995,"15min":0.97299999999999998},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":1.27,"5min":1.6339999999999999,"15min":1.6339999999999999},"last":1.153}]},{"osd":6,"last update":"Thu Sep 12 23:35:49 2024","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":1.0309999999999999,"15min":1.073},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":1.1990000000000001,"5min":1.5349999999999999,"15min":1.5349999999999999},"last":1.0580000000000001},{"interface":"front","average":{"1min":0.93700000000000006,"5min":1.0760000000000001,"15min":1.1259999999999999},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":1.2609999999999999,"5min":1.4530000000000001,"15min":1.4530000000000001},"last":1.2370000000000001}]}]},{"osd":6,"up_from":42,"seq":180388626486,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298576,"kb_used_data":1224,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446576,"statfs":{"total":95995035648,"available":95689293824,"internally_reserved":0,"allocated":1253376,"data_stored":884213,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 12 23:36:04 2024","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":0.85999999999999999,"15min":0.83999999999999997},"min":{"1min":0.52000000000000002,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":2.7069999999999999,"5min":2.7069999999999999,"15min":2.7069999999999999},"last":0.69999999999999996},{"interface":"front","average":{"1min":0.83899999999999997,"5min":0.874,"15min":0.874},"min":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":1.2190000000000001,"5min":1.2569999999999999,"15min":1.2569999999999999},"last":1.0529999999999999}]},{"osd":1,"last update":"Thu Sep 12 23:36:04 2024","interfaces":[{"interface":"back","average":{"1min":1.03,"5min":0.872,"15min":0.81999999999999995},"min":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"max":{"1min":2.73,"5min":2.73,"15min":2.73},"last":0.88300000000000001},{"interface":"front","average":{"1min":0.81299999999999994,"5min":0.85599999999999998,"15min":0.86599999999999999},"min":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"max":{"1min":1.056,"5min":1.306,"15min":1.306},"last":1.1479999999999999}]},{"osd":2,"last update":"Thu Sep 12 23:36:04 2024","interfaces":[{"interface":"back","average":{"1min":0.90300000000000002,"5min":0.875,"15min":0.86899999999999999},"min":{"1min":0.68200000000000005,"5min":0.61299999999999999,"15min":0.61299999999999999},"max":{"1min":1.157,"5min":1.4359999999999999,"15min":1.4359999999999999},"last":0.91800000000000004},{"interface":"front","average":{"1min":0.88700000000000001,"5min":0.86399999999999999,"15min":0.86299999999999999},"min":{"1min":0.55900000000000005,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":1.2310000000000001,"5min":1.2769999999999999,"15min":1.2769999999999999},"last":0.83199999999999996}]},{"osd":3,"last update":"Thu Sep 12 23:36:04 2024","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":0.90100000000000002,"15min":0.86099999999999999},"min":{"1min":0.60799999999999998,"5min":0.52400000000000002,"15min":0.52400000000000002},"max":{"1min":2.6880000000000002,"5min":2.6880000000000002,"15min":2.6880000000000002},"last":1.1020000000000001},{"interface":"front","average":{"1min":0.92700000000000005,"5min":0.92300000000000004,"15min":0.92900000000000005},"min":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":1.2310000000000001,"5min":1.349,"15min":1.349},"last":1.028}]},{"osd":4,"last update":"Thu Sep 12 23:36:04 2024","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":0.83099999999999996,"15min":0.78700000000000003},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":1.2470000000000001,"5min":1.2729999999999999,"15min":1.2729999999999999},"last":1.254},{"interface":"front","average":{"1min":1.022,"5min":0.93500000000000005,"15min":0.91200000000000003},"min":{"1min":0.39800000000000002,"5min":0.373,"15min":0.373},"max":{"1min":2.669,"5min":2.669,"15min":2.669},"last":1.2729999999999999}]},{"osd":5,"last update":"Thu Sep 12 23:36:04 2024","interfaces":[{"interface":"back","average":{"1min":0.998,"5min":0.90200000000000002,"15min":0.89300000000000002},"min":{"1min":0.48299999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.4810000000000001,"5min":1.4810000000000001,"15min":1.4810000000000001},"last":1.294},{"interface":"front","average":{"1min":1.1459999999999999,"5min":0.98199999999999998,"15min":0.93100000000000005},"min":{"1min":0.437,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":2.621,"5min":2.621,"15min":2.621},"last":1.1830000000000001}]},{"osd":7,"last update":"Thu Sep 12 23:35:48 2024","interfaces":[{"interface":"back","average":{"1min":1.0700000000000001,"5min":0.99199999999999999,"15min":0.99299999999999999},"min":{"1min":0.51500000000000001,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":1.3260000000000001,"5min":1.3460000000000001,"15min":1.3460000000000001},"last":1.2050000000000001},{"interface":"front","average":{"1min":1.22,"5min":1.097,"15min":1.0940000000000001},"min":{"1min":0.47099999999999997,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":2.6499999999999999,"5min":2.6499999999999999,"15min":2.6499999999999999},"last":1.2310000000000001}]}]},{"osd":5,"up_from":36,"seq":154618822721,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298000,"kb_used_data":648,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447152,"statfs":{"total":95995035648,"available":95689883648,"internally_reserved":0,"allocated":663552,"data_stored":293639,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 12 23:35:14 2024","interfaces":[{"interface":"back","average":{"1min":0.82399999999999995,"5min":0.81000000000000005,"15min":0.79000000000000004},"min":{"1min":0.50700000000000001,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":1.198,"5min":1.6100000000000001,"15min":1.6100000000000001},"last":1.008},{"interface":"front","average":{"1min":0.82699999999999996,"5min":0.85699999999999998,"15min":0.83499999999999996},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":1.282,"5min":1.3129999999999999,"15min":1.3129999999999999},"last":0.61499999999999999}]},{"osd":1,"last update":"Thu Sep 12 23:35:14 2024","interfaces":[{"interface":"back","average":{"1min":0.84999999999999998,"5min":0.876,"15min":0.85499999999999998},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":1.3680000000000001,"5min":1.4470000000000001,"15min":1.4470000000000001},"last":0.80900000000000005},{"interface":"front","average":{"1min":0.79300000000000004,"5min":0.83399999999999996,"15min":0.80600000000000005},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":1.22,"5min":1.321,"15min":1.321},"last":0.70399999999999996}]},{"osd":2,"last update":"Thu Sep 12 23:35:14 2024","interfaces":[{"interface":"back","average":{"1min":0.78100000000000003,"5min":0.86199999999999999,"15min":0.86499999999999999},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":1.2749999999999999,"5min":1.2749999999999999,"15min":1.2749999999999999},"last":0.98699999999999999},{"interface":"front","average":{"1min":0.90900000000000003,"5min":0.90700000000000003,"15min":0.91000000000000003},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":1.3620000000000001,"5min":1.3620000000000001,"15min":1.3620000000000001},"last":1.046}]},{"osd":3,"last update":"Thu Sep 12 23:35:14 2024","interfaces":[{"interface":"back","average":{"1min":0.88900000000000001,"5min":0.90600000000000003,"15min":0.90900000000000003},"min":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"max":{"1min":1.1850000000000001,"5min":1.3280000000000001,"15min":1.3280000000000001},"last":0.73599999999999999},{"interface":"front","average":{"1min":0.93300000000000005,"5min":0.93600000000000005,"15min":0.92100000000000004},"min":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":1.4730000000000001,"5min":1.536,"15min":1.536},"last":0.96299999999999997}]},{"osd":4,"last update":"Thu Sep 12 23:35:14 2024","interfaces":[{"interface":"back","average":{"1min":0.76000000000000001,"5min":0.77800000000000002,"15min":0.754},"min":{"1min":0.34899999999999998,"5min":0.308,"15min":0.308},"max":{"1min":1.347,"5min":1.391,"15min":1.391},"last":0.92600000000000005},{"interface":"front","average":{"1min":0.85299999999999998,"5min":0.82699999999999996,"15min":0.79100000000000004},"min":{"1min":0.40899999999999997,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":1.2649999999999999,"5min":1.2649999999999999,"15min":1.2649999999999999},"last":0.86399999999999999}]},{"osd":6,"last update":"Thu Sep 12 23:35:58 2024","interfaces":[{"interface":"back","average":{"1min":0.93100000000000005,"5min":0.89200000000000002,"15min":0.82899999999999996},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.393,"5min":1.393,"15min":1.393},"last":1.1259999999999999},{"interface":"front","average":{"1min":0.98999999999999999,"5min":0.92100000000000004,"15min":0.84099999999999997},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":1.548,"5min":1.548,"15min":1.548},"last":0.90900000000000003}]},{"osd":7,"last update":"Thu Sep 12 23:35:53 2024","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":1.0109999999999999,"15min":1.0369999999999999},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":1.3340000000000001,"5min":1.46,"15min":1.46},"last":1.0229999999999999},{"interface":"front","average":{"1min":1.0169999999999999,"5min":1.05,"15min":1.0640000000000001},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":1.5069999999999999,"5min":1.617,"15min":1.617},"last":1.0669999999999999}]}]},{"osd":4,"up_from":30,"seq":128849018954,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298016,"kb_used_data":656,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447136,"statfs":{"total":95995035648,"available":95689867264,"internally_reserved":0,"allocated":671744,"data_stored":292251,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 12 23:35:22 2024","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":0.86499999999999999,"15min":0.84399999999999997},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":1.5209999999999999,"5min":1.5209999999999999,"15min":1.5209999999999999},"last":0.96899999999999997},{"interface":"front","average":{"1min":0.91800000000000004,"5min":0.86599999999999999,"15min":0.84999999999999998},"min":{"1min":0.64300000000000002,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.599,"5min":1.599,"15min":1.599},"last":1.1599999999999999}]},{"osd":1,"last update":"Thu Sep 12 23:35:22 2024","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":0.86499999999999999,"15min":0.83599999999999997},"min":{"1min":0.54400000000000004,"5min":0.52000000000000002,"15min":0.52000000000000002},"max":{"1min":1.6890000000000001,"5min":1.6890000000000001,"15min":1.6890000000000001},"last":0.70799999999999996},{"interface":"front","average":{"1min":0.876,"5min":0.86599999999999999,"15min":0.85999999999999999},"min":{"1min":0.64100000000000001,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":1.3779999999999999,"5min":1.3779999999999999,"15min":1.3779999999999999},"last":0.68100000000000005}]},{"osd":2,"last update":"Thu Sep 12 23:35:22 2024","interfaces":[{"interface":"back","average":{"1min":0.93700000000000006,"5min":0.85799999999999998,"15min":0.85999999999999999},"min":{"1min":0.53900000000000003,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":1.466,"5min":1.466,"15min":1.466},"last":0.75900000000000001},{"interface":"front","average":{"1min":0.875,"5min":0.86699999999999999,"15min":0.86699999999999999},"min":{"1min":0.51900000000000002,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.1080000000000001,"5min":1.3120000000000001,"15min":1.3120000000000001},"last":0.90500000000000003}]},{"osd":3,"last update":"Thu Sep 12 23:35:22 2024","interfaces":[{"interface":"back","average":{"1min":1.014,"5min":0.89400000000000002,"15min":0.86399999999999999},"min":{"1min":0.56899999999999995,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":1.419,"5min":1.5129999999999999,"15min":1.5129999999999999},"last":1.0700000000000001},{"interface":"front","average":{"1min":1.044,"5min":0.94399999999999995,"15min":0.90400000000000003},"min":{"1min":0.72499999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":1.7869999999999999,"5min":1.7869999999999999,"15min":1.7869999999999999},"last":0.79600000000000004}]},{"osd":5,"last update":"Thu Sep 12 23:35:15 2024","interfaces":[{"interface":"back","average":{"1min":0.94699999999999995,"5min":0.82899999999999996,"15min":0.79500000000000004},"min":{"1min":0.38200000000000001,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":1.835,"5min":1.835,"15min":1.835},"last":0.85799999999999998},{"interface":"front","average":{"1min":1.081,"5min":0.90000000000000002,"15min":0.83399999999999996},"min":{"1min":0.505,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.639,"5min":1.716,"15min":1.716},"last":1.1200000000000001}]},{"osd":6,"last update":"Thu Sep 12 23:36:00 2024","interfaces":[{"interface":"back","average":{"1min":1.01,"5min":0.95199999999999996,"15min":0.92600000000000005},"min":{"1min":0.69799999999999995,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":1.776,"5min":1.776,"15min":1.776},"last":1.0069999999999999},{"interface":"front","average":{"1min":1.075,"5min":1.014,"15min":0.97999999999999998},"min":{"1min":0.61499999999999999,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":1.9119999999999999,"5min":1.9119999999999999,"15min":1.9119999999999999},"last":1.0940000000000001}]},{"osd":7,"last update":"Thu Sep 12 23:35:47 2024","interfaces":[{"interface":"back","average":{"1min":1.115,"5min":1.0600000000000001,"15min":1.0680000000000001},"min":{"1min":0.61399999999999999,"5min":0.373,"15min":0.373},"max":{"1min":1.8720000000000001,"5min":1.8720000000000001,"15min":1.8720000000000001},"last":1.046},{"interface":"front","average":{"1min":1.119,"5min":1.056,"15min":1.0680000000000001},"min":{"1min":0.81200000000000006,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.7410000000000001,"5min":1.7410000000000001,"15min":1.7410000000000001},"last":1.2130000000000001}]}]},{"osd":3,"up_from":25,"seq":107374182484,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298008,"kb_used_data":656,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447144,"statfs":{"total":95995035648,"available":95689875456,"internally_reserved":0,"allocated":671744,"data_stored":294264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 12 23:35:38 2024","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":0.64000000000000001,"15min":0.60799999999999998},"min":{"1min":0.318,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":1.0489999999999999,"5min":1.0489999999999999,"15min":1.0489999999999999},"last":0.79700000000000004},{"interface":"front","average":{"1min":0.66500000000000004,"5min":0.70699999999999996,"15min":0.69399999999999995},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":1.101,"5min":1.5009999999999999,"15min":1.5009999999999999},"last":0.95499999999999996}]},{"osd":1,"last update":"Thu Sep 12 23:35:38 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.67400000000000004,"15min":0.65600000000000003},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.88900000000000001,"5min":1.006,"15min":1.006},"last":0.74399999999999999},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.72899999999999998,"15min":0.73499999999999999},"min":{"1min":0.30499999999999999,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.88200000000000001,"5min":1.401,"15min":1.401},"last":0.626}]},{"osd":2,"last update":"Thu Sep 12 23:35:38 2024","interfaces":[{"interface":"back","average":{"1min":0.77500000000000002,"5min":0.79600000000000004,"15min":0.77900000000000003},"min":{"1min":0.38400000000000001,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":1.0609999999999999,"5min":1.2709999999999999,"15min":1.2709999999999999},"last":0.92400000000000004},{"interface":"front","average":{"1min":0.754,"5min":0.79500000000000004,"15min":0.747},"min":{"1min":0.34999999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":1.1699999999999999,"5min":1.458,"15min":1.458},"last":0.83799999999999997}]},{"osd":4,"last update":"Thu Sep 12 23:35:26 2024","interfaces":[{"interface":"back","average":{"1min":0.95099999999999996,"5min":0.92900000000000005,"15min":0.89100000000000001},"min":{"1min":0.65200000000000002,"5min":0.497,"15min":0.497},"max":{"1min":1.54,"5min":1.54,"15min":1.54},"last":1.137},{"interface":"front","average":{"1min":0.90600000000000003,"5min":0.90100000000000002,"15min":0.85899999999999999},"min":{"1min":0.57199999999999995,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":1.1539999999999999,"5min":1.3640000000000001,"15min":1.3640000000000001},"last":0.89000000000000001}]},{"osd":5,"last update":"Thu Sep 12 23:36:11 2024","interfaces":[{"interface":"back","average":{"1min":0.96599999999999997,"5min":0.97499999999999998,"15min":0.98099999999999998},"min":{"1min":0.67100000000000004,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":1.3620000000000001,"5min":1.6120000000000001,"15min":1.6120000000000001},"last":1.1950000000000001},{"interface":"front","average":{"1min":0.92100000000000004,"5min":0.96099999999999997,"15min":0.96399999999999997},"min":{"1min":0.56000000000000005,"5min":0.437,"15min":0.437},"max":{"1min":1.2110000000000001,"5min":1.5429999999999999,"15min":1.5429999999999999},"last":1.0640000000000001}]},{"osd":6,"last update":"Thu Sep 12 23:36:03 2024","interfaces":[{"interface":"back","average":{"1min":0.93300000000000005,"5min":0.999,"15min":1.034},"min":{"1min":0.59499999999999997,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":1.28,"5min":1.4930000000000001,"15min":1.4930000000000001},"last":0.99299999999999999},{"interface":"front","average":{"1min":1.0269999999999999,"5min":1.0820000000000001,"15min":1.1040000000000001},"min":{"1min":0.58399999999999996,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":2.0099999999999998,"5min":2.0099999999999998,"15min":2.0099999999999998},"last":1.091}]},{"osd":7,"last update":"Thu Sep 12 23:35:50 2024","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":1.0429999999999999,"15min":1.0309999999999999},"min":{"1min":0.58799999999999997,"5min":0.52300000000000002,"15min":0.52300000000000002},"max":{"1min":1.405,"5min":1.6879999999999999,"15min":1.6879999999999999},"last":1.026},{"interface":"front","average":{"1min":1.0129999999999999,"5min":1.0580000000000001,"15min":1.0649999999999999},"min":{"1min":0.61299999999999999,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":1.3169999999999999,"5min":1.5880000000000001,"15min":1.5880000000000001},"last":1.163}]}]},{"osd":2,"up_from":18,"seq":77309411422,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297996,"kb_used_data":636,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447156,"statfs":{"total":95995035648,"available":95689887744,"internally_reserved":0,"allocated":651264,"data_stored":291536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 12 23:35:47 2024","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":0.64700000000000002,"15min":0.59799999999999998},"min":{"1min":0.34599999999999997,"5min":0.28999999999999998,"15min":0.28799999999999998},"max":{"1min":1.018,"5min":1.0429999999999999,"15min":1.23},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.76300000000000001,"5min":0.72999999999999998,"15min":0.64100000000000001},"min":{"1min":0.34899999999999998,"5min":0.318,"15min":0.26200000000000001},"max":{"1min":1.0720000000000001,"5min":1.1870000000000001,"15min":1.1870000000000001},"last":0.59499999999999997}]},{"osd":1,"last update":"Thu Sep 12 23:35:47 2024","interfaces":[{"interface":"back","average":{"1min":0.68999999999999995,"5min":0.70999999999999996,"15min":0.64400000000000002},"min":{"1min":0.36799999999999999,"5min":0.33200000000000002,"15min":0.316},"max":{"1min":0.95099999999999996,"5min":1.5329999999999999,"15min":1.5329999999999999},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.81100000000000005,"5min":0.754,"15min":0.66500000000000004},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":1.1779999999999999,"5min":1.1779999999999999,"15min":1.1779999999999999},"last":0.53400000000000003}]},{"osd":3,"last update":"Thu Sep 12 23:35:34 2024","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":0.77100000000000002,"15min":0.749},"min":{"1min":0.40000000000000002,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.151,"5min":1.2270000000000001,"15min":1.2270000000000001},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.874,"5min":0.80300000000000005,"15min":0.78100000000000003},"min":{"1min":0.441,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.345,"5min":1.6619999999999999,"15min":1.6619999999999999},"last":0.61699999999999999}]},{"osd":4,"last update":"Thu Sep 12 23:35:20 2024","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":0.90800000000000003,"15min":0.876},"min":{"1min":0.66000000000000003,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.651,"5min":1.651,"15min":1.651},"last":0.86899999999999999},{"interface":"front","average":{"1min":1.0780000000000001,"5min":0.995,"15min":0.94899999999999995},"min":{"1min":0.70599999999999996,"5min":0.56899999999999995,"15min":0.56899999999999995},"max":{"1min":1.8660000000000001,"5min":1.8660000000000001,"15min":1.8660000000000001},"last":0.88600000000000001}]},{"osd":5,"last update":"Thu Sep 12 23:35:13 2024","interfaces":[{"interface":"back","average":{"1min":0.93500000000000005,"5min":0.97799999999999998,"15min":1.018},"min":{"1min":0.66200000000000003,"5min":0.65200000000000002,"15min":0.65200000000000002},"max":{"1min":1.681,"5min":1.681,"15min":1.681},"last":0.81999999999999995},{"interface":"front","average":{"1min":1.004,"5min":0.98499999999999999,"15min":1.0149999999999999},"min":{"1min":0.64700000000000002,"5min":0.61499999999999999,"15min":0.61499999999999999},"max":{"1min":1.296,"5min":1.3129999999999999,"15min":1.3129999999999999},"last":0.73399999999999999}]},{"osd":6,"last update":"Thu Sep 12 23:36:01 2024","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":0.96899999999999997,"15min":0.95299999999999996},"min":{"1min":0.76800000000000002,"5min":0.47799999999999998,"15min":0.47799999999999998},"max":{"1min":1.4279999999999999,"5min":1.429,"15min":1.429},"last":1.0640000000000001},{"interface":"front","average":{"1min":1.0569999999999999,"5min":1.056,"15min":1.077},"min":{"1min":0.66100000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":1.276,"5min":1.7909999999999999,"15min":1.7909999999999999},"last":0.92000000000000004}]},{"osd":7,"last update":"Thu Sep 12 23:35:47 2024","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":1.0069999999999999,"15min":0.97699999999999998},"min":{"1min":0.71199999999999997,"5min":0.57299999999999995,"15min":0.57299999999999995},"max":{"1min":1.373,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":0.81100000000000005},{"interface":"front","average":{"1min":1.0980000000000001,"5min":1.01,"15min":0.97799999999999998},"min":{"1min":0.60399999999999998,"5min":0.50900000000000001,"15min":0.50900000000000001},"max":{"1min":1.7609999999999999,"5min":1.7609999999999999,"15min":1.7609999999999999},"last":0.93300000000000005}]}]},{"osd":1,"up_from":13,"seq":55834574953,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298000,"kb_used_data":648,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447152,"statfs":{"total":95995035648,"available":95689883648,"internally_reserved":0,"allocated":663552,"data_stored":291715,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 12 23:36:02 2024","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":0.64900000000000002,"15min":0.55800000000000005},"min":{"1min":0.36199999999999999,"5min":0.317,"15min":0.255},"max":{"1min":1.0169999999999999,"5min":1.1579999999999999,"15min":1.1579999999999999},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.67300000000000004,"5min":0.71899999999999997,"15min":0.61399999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.25600000000000001},"max":{"1min":0.88200000000000001,"5min":1.1830000000000001,"15min":1.1830000000000001},"last":0.81200000000000006}]},{"osd":2,"last update":"Thu Sep 12 23:35:48 2024","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":0.70399999999999996,"15min":0.61499999999999999},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.26700000000000002},"max":{"1min":0.97599999999999998,"5min":1.238,"15min":1.238},"last":1.0760000000000001},{"interface":"front","average":{"1min":0.71299999999999997,"5min":0.751,"15min":0.65500000000000003},"min":{"1min":0.42399999999999999,"5min":0.41299999999999998,"15min":0.28999999999999998},"max":{"1min":1.147,"5min":1.3049999999999999,"15min":1.3049999999999999},"last":0.66300000000000003}]},{"osd":3,"last update":"Thu Sep 12 23:35:38 2024","interfaces":[{"interface":"back","average":{"1min":0.76800000000000002,"5min":0.81299999999999994,"15min":0.78900000000000003},"min":{"1min":0.49299999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.093,"5min":1.198,"15min":1.198},"last":0.97199999999999998},{"interface":"front","average":{"1min":0.746,"5min":0.78900000000000003,"15min":0.76800000000000002},"min":{"1min":0.39500000000000002,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.0329999999999999,"5min":1.194,"15min":1.194},"last":0.69299999999999995}]},{"osd":4,"last update":"Thu Sep 12 23:35:21 2024","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":0.93300000000000005,"15min":0.92100000000000004},"min":{"1min":0.72899999999999998,"5min":0.57499999999999996,"15min":0.57499999999999996},"max":{"1min":1.5369999999999999,"5min":1.585,"15min":1.585},"last":0.92700000000000005},{"interface":"front","average":{"1min":0.98399999999999999,"5min":0.94899999999999995,"15min":0.93100000000000005},"min":{"1min":0.75600000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":1.2669999999999999,"5min":1.4279999999999999,"15min":1.4279999999999999},"last":1.0349999999999999}]},{"osd":5,"last update":"Thu Sep 12 23:36:10 2024","interfaces":[{"interface":"back","average":{"1min":0.88100000000000001,"5min":0.91600000000000004,"15min":0.91400000000000003},"min":{"1min":0.61099999999999999,"5min":0.52600000000000002,"15min":0.52600000000000002},"max":{"1min":1.127,"5min":1.6819999999999999,"15min":1.6819999999999999},"last":1.214},{"interface":"front","average":{"1min":1.0129999999999999,"5min":1.0049999999999999,"15min":1.002},"min":{"1min":0.77900000000000003,"5min":0.63300000000000001,"15min":0.63300000000000001},"max":{"1min":1.2490000000000001,"5min":1.671,"15min":1.671},"last":1.127}]},{"osd":6,"last update":"Thu Sep 12 23:36:02 2024","interfaces":[{"interface":"back","average":{"1min":0.96099999999999997,"5min":1.0149999999999999,"15min":1.04},"min":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":1.2909999999999999,"5min":1.5429999999999999,"15min":1.5429999999999999},"last":1.4159999999999999},{"interface":"front","average":{"1min":1.026,"5min":1.0600000000000001,"15min":1.087},"min":{"1min":0.66600000000000004,"5min":0.63800000000000001,"15min":0.63800000000000001},"max":{"1min":1.325,"5min":1.579,"15min":1.579},"last":0.89800000000000002}]},{"osd":7,"last update":"Thu Sep 12 23:35:51 2024","interfaces":[{"interface":"back","average":{"1min":1.032,"5min":1.0489999999999999,"15min":1.0469999999999999},"min":{"1min":0.63,"5min":0.61399999999999999,"15min":0.61399999999999999},"max":{"1min":1.4930000000000001,"5min":1.4930000000000001,"15min":1.4930000000000001},"last":1.458},{"interface":"front","average":{"1min":1.0329999999999999,"5min":1.052,"15min":1.0700000000000001},"min":{"1min":0.66900000000000004,"5min":0.59799999999999998,"15min":0.59799999999999998},"max":{"1min":1.226,"5min":1.3600000000000001,"15min":1.3600000000000001},"last":1.1060000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738481,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298588,"kb_used_data":1228,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446564,"statfs":{"total":95995035648,"available":95689281536,"internally_reserved":0,"allocated":1257472,"data_stored":882305,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Sep 12 23:35:52 2024","interfaces":[{"interface":"back","average":{"1min":0.65900000000000003,"5min":0.65000000000000002,"15min":0.54800000000000004},"min":{"1min":0.48099999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.0680000000000001,"5min":1.085,"15min":1.085},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.71099999999999997,"5min":0.70399999999999996,"15min":0.57499999999999996},"min":{"1min":0.41899999999999998,"5min":0.216,"15min":0.216},"max":{"1min":1.087,"5min":1.087,"15min":1.087},"last":0.72799999999999998}]},{"osd":2,"last update":"Thu Sep 12 23:35:48 2024","interfaces":[{"interface":"back","average":{"1min":0.70299999999999996,"5min":0.70499999999999996,"15min":0.63900000000000001},"min":{"1min":0.45900000000000002,"5min":0.36699999999999999,"15min":0.27100000000000002},"max":{"1min":0.878,"5min":1.081,"15min":1.081},"last":1.0309999999999999},{"interface":"front","average":{"1min":0.81100000000000005,"5min":0.754,"15min":0.66800000000000004},"min":{"1min":0.52000000000000002,"5min":0.34300000000000003,"15min":0.317},"max":{"1min":1.206,"5min":1.206,"15min":1.206},"last":1.214}]},{"osd":3,"last update":"Thu Sep 12 23:35:34 2024","interfaces":[{"interface":"back","average":{"1min":0.78600000000000003,"5min":0.75800000000000001,"15min":0.76000000000000001},"min":{"1min":0.54600000000000004,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":1.016,"5min":1.177,"15min":1.177},"last":0.78600000000000003},{"interface":"front","average":{"1min":0.85999999999999999,"5min":0.81499999999999995,"15min":0.79800000000000004},"min":{"1min":0.69299999999999995,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.165,"5min":1.3140000000000001,"15min":1.3140000000000001},"last":0.95999999999999996}]},{"osd":4,"last update":"Thu Sep 12 23:35:27 2024","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":0.96899999999999997,"15min":0.91500000000000004},"min":{"1min":0.78800000000000003,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":1.2090000000000001,"5min":1.304,"15min":1.304},"last":1.1559999999999999},{"interface":"front","average":{"1min":0.98799999999999999,"5min":0.91800000000000004,"15min":0.86399999999999999},"min":{"1min":0.65200000000000002,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":1.2689999999999999,"5min":1.2689999999999999,"15min":1.2689999999999999},"last":0.999}]},{"osd":5,"last update":"Thu Sep 12 23:35:14 2024","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":0.92900000000000005,"15min":0.93799999999999994},"min":{"1min":0.65100000000000002,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.1890000000000001,"5min":1.3160000000000001,"15min":1.3160000000000001},"last":0.86099999999999999},{"interface":"front","average":{"1min":1.0089999999999999,"5min":0.97299999999999998,"15min":0.95699999999999996},"min":{"1min":0.73899999999999999,"5min":0.65500000000000003,"15min":0.65500000000000003},"max":{"1min":1.1799999999999999,"5min":1.218,"15min":1.218},"last":0.90500000000000003}]},{"osd":6,"last update":"Thu Sep 12 23:35:58 2024","interfaces":[{"interface":"back","average":{"1min":1.0369999999999999,"5min":1.071,"15min":1.0980000000000001},"min":{"1min":0.77100000000000002,"5min":0.56000000000000005,"15min":0.56000000000000005},"max":{"1min":1.3660000000000001,"5min":1.492,"15min":1.492},"last":1.05},{"interface":"front","average":{"1min":1.0049999999999999,"5min":1.0249999999999999,"15min":1.042},"min":{"1min":0.63200000000000001,"5min":0.621,"15min":0.621},"max":{"1min":1.351,"5min":1.4550000000000001,"15min":1.4550000000000001},"last":1.1220000000000001}]},{"osd":7,"last update":"Thu Sep 12 23:35:48 2024","interfaces":[{"interface":"back","average":{"1min":1.006,"5min":1.018,"15min":1.0169999999999999},"min":{"1min":0.72899999999999998,"5min":0.64500000000000002,"15min":0.64500000000000002},"max":{"1min":1.3200000000000001,"5min":1.403,"15min":1.403},"last":1.083},{"interface":"front","average":{"1min":1.0880000000000001,"5min":1.111,"15min":1.131},"min":{"1min":0.72299999999999998,"5min":0.55000000000000004,"15min":0.55000000000000004},"max":{"1min":1.341,"5min":1.569,"15min":1.569},"last":1.258}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":2728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":2714,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":2624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-12T23:36:16.832 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-12T23:36:17.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:17 smithi136 bash[22601]: cluster 2024-09-12T23:36:15.964875+0000 mgr.y (mgr.24446) 98 : cluster [DBG] pgmap v76: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:17.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:17 smithi136 bash[22601]: audit 2024-09-12T23:36:16.091610+0000 mgr.y (mgr.24446) 99 : audit [DBG] from='client.24620 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-12T23:36:17.256 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:17 smithi136 bash[32189]: cluster 2024-09-12T23:36:15.964875+0000 mgr.y (mgr.24446) 98 : cluster [DBG] pgmap v76: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:17.256 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:17 smithi136 bash[32189]: audit 2024-09-12T23:36:16.091610+0000 mgr.y (mgr.24446) 99 : audit [DBG] from='client.24620 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-12T23:36:17.345 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:17 smithi181 bash[24776]: cluster 2024-09-12T23:36:15.964875+0000 mgr.y (mgr.24446) 98 : cluster [DBG] pgmap v76: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:17.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:17 smithi181 bash[24776]: audit 2024-09-12T23:36:16.091610+0000 mgr.y (mgr.24446) 99 : audit [DBG] from='client.24620 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-12T23:36:19.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:19 smithi181 bash[24776]: cluster 2024-09-12T23:36:17.965524+0000 mgr.y (mgr.24446) 100 : cluster [DBG] pgmap v77: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:19.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:19 smithi136 bash[22601]: cluster 2024-09-12T23:36:17.965524+0000 mgr.y (mgr.24446) 100 : cluster [DBG] pgmap v77: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:19.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:19 smithi136 bash[32189]: cluster 2024-09-12T23:36:17.965524+0000 mgr.y (mgr.24446) 100 : cluster [DBG] pgmap v77: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:21.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:21 smithi181 bash[24776]: cluster 2024-09-12T23:36:19.966803+0000 mgr.y (mgr.24446) 101 : cluster [DBG] pgmap v78: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:21.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:21 smithi136 bash[22601]: cluster 2024-09-12T23:36:19.966803+0000 mgr.y (mgr.24446) 101 : cluster [DBG] pgmap v78: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:21.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:21 smithi136 bash[32189]: cluster 2024-09-12T23:36:19.966803+0000 mgr.y (mgr.24446) 101 : cluster [DBG] pgmap v78: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:21.621 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:36:23.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:23 smithi181 bash[24776]: cluster 2024-09-12T23:36:21.967533+0000 mgr.y (mgr.24446) 102 : cluster [DBG] pgmap v79: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:23.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:23 smithi181 bash[24776]: audit 2024-09-12T23:36:22.999270+0000 mon.a (mon.0) 885 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:36:23.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:23 smithi136 bash[32189]: cluster 2024-09-12T23:36:21.967533+0000 mgr.y (mgr.24446) 102 : cluster [DBG] pgmap v79: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:23.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:23 smithi136 bash[32189]: audit 2024-09-12T23:36:22.999270+0000 mon.a (mon.0) 885 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:36:23.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:23 smithi136 bash[22601]: cluster 2024-09-12T23:36:21.967533+0000 mgr.y (mgr.24446) 102 : cluster [DBG] pgmap v79: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:23.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:23 smithi136 bash[22601]: audit 2024-09-12T23:36:22.999270+0000 mon.a (mon.0) 885 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:36:24.739 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-12T23:36:24.740 INFO:teuthology.orchestra.run.smithi136.stderr:dumped all 2024-09-12T23:36:25.345 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:25 smithi181 bash[24776]: cluster 2024-09-12T23:36:23.968873+0000 mgr.y (mgr.24446) 103 : cluster [DBG] pgmap v80: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:25.346 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:36:25 smithi181 bash[25717]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:36:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-12T23:36:25.434 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:25 smithi136 bash[32189]: cluster 2024-09-12T23:36:23.968873+0000 mgr.y (mgr.24446) 103 : cluster [DBG] pgmap v80: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:25.434 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:25 smithi136 bash[22601]: cluster 2024-09-12T23:36:23.968873+0000 mgr.y (mgr.24446) 103 : cluster [DBG] pgmap v80: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:25.435 INFO:teuthology.orchestra.run.smithi136.stdout:{"pg_ready":true,"pg_map":{"version":80,"stamp":"2024-09-12T23:36:23.967908+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":593797,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":166,"num_write_kb":1649,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":127,"ondisk_log_size":127,"up":315,"acting":315,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":315,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385764,"kb_used_data":6920,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575452,"statfs":{"total":767960285184,"available":765517262848,"internally_reserved":0,"allocated":7086080,"data_stored":4112419,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.005062"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.665027+0000","last_change":"2024-09-12T23:33:29.908602+0000","last_active":"2024-09-12T23:35:08.665027+0000","last_peered":"2024-09-12T23:35:08.665027+0000","last_clean":"2024-09-12T23:35:08.665027+0000","last_became_active":"2024-09-12T23:33:29.908161+0000","last_became_peered":"2024-09-12T23:33:29.908161+0000","last_unstale":"2024-09-12T23:35:08.665027+0000","last_undegraded":"2024-09-12T23:35:08.665027+0000","last_fullsized":"2024-09-12T23:35:08.665027+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:18:25.662166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.664503+0000","last_change":"2024-09-12T23:33:29.907919+0000","last_active":"2024-09-12T23:35:08.664503+0000","last_peered":"2024-09-12T23:35:08.664503+0000","last_clean":"2024-09-12T23:35:08.664503+0000","last_became_active":"2024-09-12T23:33:29.906765+0000","last_became_peered":"2024-09-12T23:33:29.906765+0000","last_unstale":"2024-09-12T23:35:08.664503+0000","last_undegraded":"2024-09-12T23:35:08.664503+0000","last_fullsized":"2024-09-12T23:35:08.664503+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:39:49.787701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.671331+0000","last_change":"2024-09-12T23:33:29.924626+0000","last_active":"2024-09-12T23:35:08.671331+0000","last_peered":"2024-09-12T23:35:08.671331+0000","last_clean":"2024-09-12T23:35:08.671331+0000","last_became_active":"2024-09-12T23:33:29.924268+0000","last_became_peered":"2024-09-12T23:33:29.924268+0000","last_unstale":"2024-09-12T23:35:08.671331+0000","last_undegraded":"2024-09-12T23:35:08.671331+0000","last_fullsized":"2024-09-12T23:35:08.671331+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:41:31.853826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.670520+0000","last_change":"2024-09-12T23:33:29.913990+0000","last_active":"2024-09-12T23:35:08.670520+0000","last_peered":"2024-09-12T23:35:08.670520+0000","last_clean":"2024-09-12T23:35:08.670520+0000","last_became_active":"2024-09-12T23:33:29.913672+0000","last_became_peered":"2024-09-12T23:33:29.913672+0000","last_unstale":"2024-09-12T23:35:08.670520+0000","last_undegraded":"2024-09-12T23:35:08.670520+0000","last_fullsized":"2024-09-12T23:35:08.670520+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:57:54.241724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.663791+0000","last_change":"2024-09-12T23:33:29.904505+0000","last_active":"2024-09-12T23:35:08.663791+0000","last_peered":"2024-09-12T23:35:08.663791+0000","last_clean":"2024-09-12T23:35:08.663791+0000","last_became_active":"2024-09-12T23:33:29.904192+0000","last_became_peered":"2024-09-12T23:33:29.904192+0000","last_unstale":"2024-09-12T23:35:08.663791+0000","last_undegraded":"2024-09-12T23:35:08.663791+0000","last_fullsized":"2024-09-12T23:35:08.663791+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:49:27.086584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.671038+0000","last_change":"2024-09-12T23:33:29.905467+0000","last_active":"2024-09-12T23:35:08.671038+0000","last_peered":"2024-09-12T23:35:08.671038+0000","last_clean":"2024-09-12T23:35:08.671038+0000","last_became_active":"2024-09-12T23:33:29.905154+0000","last_became_peered":"2024-09-12T23:33:29.905154+0000","last_unstale":"2024-09-12T23:35:08.671038+0000","last_undegraded":"2024-09-12T23:35:08.671038+0000","last_fullsized":"2024-09-12T23:35:08.671038+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:49:00.596502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.664458+0000","last_change":"2024-09-12T23:33:29.908762+0000","last_active":"2024-09-12T23:35:08.664458+0000","last_peered":"2024-09-12T23:35:08.664458+0000","last_clean":"2024-09-12T23:35:08.664458+0000","last_became_active":"2024-09-12T23:33:29.908427+0000","last_became_peered":"2024-09-12T23:33:29.908427+0000","last_unstale":"2024-09-12T23:35:08.664458+0000","last_undegraded":"2024-09-12T23:35:08.664458+0000","last_fullsized":"2024-09-12T23:35:08.664458+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:44:26.357691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.666998+0000","last_change":"2024-09-12T23:33:29.902033+0000","last_active":"2024-09-12T23:35:08.666998+0000","last_peered":"2024-09-12T23:35:08.666998+0000","last_clean":"2024-09-12T23:35:08.666998+0000","last_became_active":"2024-09-12T23:33:29.901611+0000","last_became_peered":"2024-09-12T23:33:29.901611+0000","last_unstale":"2024-09-12T23:35:08.666998+0000","last_undegraded":"2024-09-12T23:35:08.666998+0000","last_fullsized":"2024-09-12T23:35:08.666998+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:57:58.671558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":30,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-12T23:35:09.803676+0000","last_change":"2024-09-12T23:35:09.803676+0000","last_active":"2024-09-12T23:35:09.803676+0000","last_peered":"2024-09-12T23:35:09.803676+0000","last_clean":"2024-09-12T23:35:09.803676+0000","last_became_active":"2024-09-12T23:35:09.802434+0000","last_became_peered":"2024-09-12T23:35:09.802434+0000","last_unstale":"2024-09-12T23:35:09.803676+0000","last_undegraded":"2024-09-12T23:35:09.803676+0000","last_fullsized":"2024-09-12T23:35:09.803676+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:01:37.156347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.666805+0000","last_change":"2024-09-12T23:33:29.893241+0000","last_active":"2024-09-12T23:35:08.666805+0000","last_peered":"2024-09-12T23:35:08.666805+0000","last_clean":"2024-09-12T23:35:08.666805+0000","last_became_active":"2024-09-12T23:33:29.892815+0000","last_became_peered":"2024-09-12T23:33:29.892815+0000","last_unstale":"2024-09-12T23:35:08.666805+0000","last_undegraded":"2024-09-12T23:35:08.666805+0000","last_fullsized":"2024-09-12T23:35:08.666805+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:46:21.539771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.665734+0000","last_change":"2024-09-12T23:33:29.906810+0000","last_active":"2024-09-12T23:35:08.665734+0000","last_peered":"2024-09-12T23:35:08.665734+0000","last_clean":"2024-09-12T23:35:08.665734+0000","last_became_active":"2024-09-12T23:33:29.906493+0000","last_became_peered":"2024-09-12T23:33:29.906493+0000","last_unstale":"2024-09-12T23:35:08.665734+0000","last_undegraded":"2024-09-12T23:35:08.665734+0000","last_fullsized":"2024-09-12T23:35:08.665734+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:56:07.145258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":30,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-12T23:35:09.802722+0000","last_change":"2024-09-12T23:35:09.802722+0000","last_active":"2024-09-12T23:35:09.802722+0000","last_peered":"2024-09-12T23:35:09.802722+0000","last_clean":"2024-09-12T23:35:09.802722+0000","last_became_active":"2024-09-12T23:35:09.801763+0000","last_became_peered":"2024-09-12T23:35:09.801763+0000","last_unstale":"2024-09-12T23:35:09.802722+0000","last_undegraded":"2024-09-12T23:35:09.802722+0000","last_fullsized":"2024-09-12T23:35:09.802722+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:33:24.574457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.668470+0000","last_change":"2024-09-12T23:33:29.907459+0000","last_active":"2024-09-12T23:35:08.668470+0000","last_peered":"2024-09-12T23:35:08.668470+0000","last_clean":"2024-09-12T23:35:08.668470+0000","last_became_active":"2024-09-12T23:33:29.906577+0000","last_became_peered":"2024-09-12T23:33:29.906577+0000","last_unstale":"2024-09-12T23:35:08.668470+0000","last_undegraded":"2024-09-12T23:35:08.668470+0000","last_fullsized":"2024-09-12T23:35:08.668470+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:21:41.131616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.670257+0000","last_change":"2024-09-12T23:33:29.920721+0000","last_active":"2024-09-12T23:35:08.670257+0000","last_peered":"2024-09-12T23:35:08.670257+0000","last_clean":"2024-09-12T23:35:08.670257+0000","last_became_active":"2024-09-12T23:33:29.918051+0000","last_became_peered":"2024-09-12T23:33:29.918051+0000","last_unstale":"2024-09-12T23:35:08.670257+0000","last_undegraded":"2024-09-12T23:35:08.670257+0000","last_fullsized":"2024-09-12T23:35:08.670257+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:32:27.077464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":30,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-12T23:35:09.683561+0000","last_change":"2024-09-12T23:35:09.683561+0000","last_active":"2024-09-12T23:35:09.683561+0000","last_peered":"2024-09-12T23:35:09.683561+0000","last_clean":"2024-09-12T23:35:09.683561+0000","last_became_active":"2024-09-12T23:35:09.682671+0000","last_became_peered":"2024-09-12T23:35:09.682671+0000","last_unstale":"2024-09-12T23:35:09.683561+0000","last_undegraded":"2024-09-12T23:35:09.683561+0000","last_fullsized":"2024-09-12T23:35:09.683561+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:45:39.393684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":10,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-12T23:35:09.804288+0000","last_change":"2024-09-12T23:35:09.804288+0000","last_active":"2024-09-12T23:35:09.804288+0000","last_peered":"2024-09-12T23:35:09.804288+0000","last_clean":"2024-09-12T23:35:09.804288+0000","last_became_active":"2024-09-12T23:35:09.803450+0000","last_became_peered":"2024-09-12T23:35:09.803450+0000","last_unstale":"2024-09-12T23:35:09.804288+0000","last_undegraded":"2024-09-12T23:35:09.804288+0000","last_fullsized":"2024-09-12T23:35:09.804288+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:31:12.292070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.670478+0000","last_change":"2024-09-12T23:33:29.913684+0000","last_active":"2024-09-12T23:35:08.670478+0000","last_peered":"2024-09-12T23:35:08.670478+0000","last_clean":"2024-09-12T23:35:08.670478+0000","last_became_active":"2024-09-12T23:33:29.911877+0000","last_became_peered":"2024-09-12T23:33:29.911877+0000","last_unstale":"2024-09-12T23:35:08.670478+0000","last_undegraded":"2024-09-12T23:35:08.670478+0000","last_fullsized":"2024-09-12T23:35:08.670478+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:39:10.601087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.661668+0000","last_change":"2024-09-12T23:33:29.882889+0000","last_active":"2024-09-12T23:35:08.661668+0000","last_peered":"2024-09-12T23:35:08.661668+0000","last_clean":"2024-09-12T23:35:08.661668+0000","last_became_active":"2024-09-12T23:33:29.882148+0000","last_became_peered":"2024-09-12T23:33:29.882148+0000","last_unstale":"2024-09-12T23:35:08.661668+0000","last_undegraded":"2024-09-12T23:35:08.661668+0000","last_fullsized":"2024-09-12T23:35:08.661668+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:59:38.545795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.666136+0000","last_change":"2024-09-12T23:33:29.904552+0000","last_active":"2024-09-12T23:35:08.666136+0000","last_peered":"2024-09-12T23:35:08.666136+0000","last_clean":"2024-09-12T23:35:08.666136+0000","last_became_active":"2024-09-12T23:33:29.904247+0000","last_became_peered":"2024-09-12T23:33:29.904247+0000","last_unstale":"2024-09-12T23:35:08.666136+0000","last_undegraded":"2024-09-12T23:35:08.666136+0000","last_fullsized":"2024-09-12T23:35:08.666136+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:30:06.600814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.665287+0000","last_change":"2024-09-12T23:33:29.905965+0000","last_active":"2024-09-12T23:35:08.665287+0000","last_peered":"2024-09-12T23:35:08.665287+0000","last_clean":"2024-09-12T23:35:08.665287+0000","last_became_active":"2024-09-12T23:33:29.905690+0000","last_became_peered":"2024-09-12T23:33:29.905690+0000","last_unstale":"2024-09-12T23:35:08.665287+0000","last_undegraded":"2024-09-12T23:35:08.665287+0000","last_fullsized":"2024-09-12T23:35:08.665287+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:58:29.597052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.669549+0000","last_change":"2024-09-12T23:33:29.918231+0000","last_active":"2024-09-12T23:35:08.669549+0000","last_peered":"2024-09-12T23:35:08.669549+0000","last_clean":"2024-09-12T23:35:08.669549+0000","last_became_active":"2024-09-12T23:33:29.917878+0000","last_became_peered":"2024-09-12T23:33:29.917878+0000","last_unstale":"2024-09-12T23:35:08.669549+0000","last_undegraded":"2024-09-12T23:35:08.669549+0000","last_fullsized":"2024-09-12T23:35:08.669549+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:22:46.498222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.671076+0000","last_change":"2024-09-12T23:33:29.909780+0000","last_active":"2024-09-12T23:35:08.671076+0000","last_peered":"2024-09-12T23:35:08.671076+0000","last_clean":"2024-09-12T23:35:08.671076+0000","last_became_active":"2024-09-12T23:33:29.909507+0000","last_became_peered":"2024-09-12T23:33:29.909507+0000","last_unstale":"2024-09-12T23:35:08.671076+0000","last_undegraded":"2024-09-12T23:35:08.671076+0000","last_fullsized":"2024-09-12T23:35:08.671076+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:39:50.529545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.667544+0000","last_change":"2024-09-12T23:33:29.906087+0000","last_active":"2024-09-12T23:35:08.667544+0000","last_peered":"2024-09-12T23:35:08.667544+0000","last_clean":"2024-09-12T23:35:08.667544+0000","last_became_active":"2024-09-12T23:33:29.905780+0000","last_became_peered":"2024-09-12T23:33:29.905780+0000","last_unstale":"2024-09-12T23:35:08.667544+0000","last_undegraded":"2024-09-12T23:35:08.667544+0000","last_fullsized":"2024-09-12T23:35:08.667544+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:00:40.777941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.670507+0000","last_change":"2024-09-12T23:33:29.918286+0000","last_active":"2024-09-12T23:35:08.670507+0000","last_peered":"2024-09-12T23:35:08.670507+0000","last_clean":"2024-09-12T23:35:08.670507+0000","last_became_active":"2024-09-12T23:33:29.917757+0000","last_became_peered":"2024-09-12T23:33:29.917757+0000","last_unstale":"2024-09-12T23:35:08.670507+0000","last_undegraded":"2024-09-12T23:35:08.670507+0000","last_fullsized":"2024-09-12T23:35:08.670507+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:49:26.856852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.673387+0000","last_change":"2024-09-12T23:33:29.903381+0000","last_active":"2024-09-12T23:35:08.673387+0000","last_peered":"2024-09-12T23:35:08.673387+0000","last_clean":"2024-09-12T23:35:08.673387+0000","last_became_active":"2024-09-12T23:33:29.903081+0000","last_became_peered":"2024-09-12T23:33:29.903081+0000","last_unstale":"2024-09-12T23:35:08.673387+0000","last_undegraded":"2024-09-12T23:35:08.673387+0000","last_fullsized":"2024-09-12T23:35:08.673387+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:43:01.175691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.666859+0000","last_change":"2024-09-12T23:33:29.897170+0000","last_active":"2024-09-12T23:35:08.666859+0000","last_peered":"2024-09-12T23:35:08.666859+0000","last_clean":"2024-09-12T23:35:08.666859+0000","last_became_active":"2024-09-12T23:33:29.896834+0000","last_became_peered":"2024-09-12T23:33:29.896834+0000","last_unstale":"2024-09-12T23:35:08.666859+0000","last_undegraded":"2024-09-12T23:35:08.666859+0000","last_fullsized":"2024-09-12T23:35:08.666859+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:47:43.534530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.663571+0000","last_change":"2024-09-12T23:33:29.919500+0000","last_active":"2024-09-12T23:35:08.663571+0000","last_peered":"2024-09-12T23:35:08.663571+0000","last_clean":"2024-09-12T23:35:08.663571+0000","last_became_active":"2024-09-12T23:33:29.919185+0000","last_became_peered":"2024-09-12T23:33:29.919185+0000","last_unstale":"2024-09-12T23:35:08.663571+0000","last_undegraded":"2024-09-12T23:35:08.663571+0000","last_fullsized":"2024-09-12T23:35:08.663571+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:56:52.566524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.672875+0000","last_change":"2024-09-12T23:33:29.913907+0000","last_active":"2024-09-12T23:35:08.672875+0000","last_peered":"2024-09-12T23:35:08.672875+0000","last_clean":"2024-09-12T23:35:08.672875+0000","last_became_active":"2024-09-12T23:33:29.913193+0000","last_became_peered":"2024-09-12T23:33:29.913193+0000","last_unstale":"2024-09-12T23:35:08.672875+0000","last_undegraded":"2024-09-12T23:35:08.672875+0000","last_fullsized":"2024-09-12T23:35:08.672875+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:32:45.117056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.669650+0000","last_change":"2024-09-12T23:33:29.911279+0000","last_active":"2024-09-12T23:35:08.669650+0000","last_peered":"2024-09-12T23:35:08.669650+0000","last_clean":"2024-09-12T23:35:08.669650+0000","last_became_active":"2024-09-12T23:33:29.911007+0000","last_became_peered":"2024-09-12T23:33:29.911007+0000","last_unstale":"2024-09-12T23:35:08.669650+0000","last_undegraded":"2024-09-12T23:35:08.669650+0000","last_fullsized":"2024-09-12T23:35:08.669650+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:26:51.842858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.671116+0000","last_change":"2024-09-12T23:33:29.914015+0000","last_active":"2024-09-12T23:35:08.671116+0000","last_peered":"2024-09-12T23:35:08.671116+0000","last_clean":"2024-09-12T23:35:08.671116+0000","last_became_active":"2024-09-12T23:33:29.913659+0000","last_became_peered":"2024-09-12T23:33:29.913659+0000","last_unstale":"2024-09-12T23:35:08.671116+0000","last_undegraded":"2024-09-12T23:35:08.671116+0000","last_fullsized":"2024-09-12T23:35:08.671116+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:37:02.989629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.661319+0000","last_change":"2024-09-12T23:33:29.889782+0000","last_active":"2024-09-12T23:35:08.661319+0000","last_peered":"2024-09-12T23:35:08.661319+0000","last_clean":"2024-09-12T23:35:08.661319+0000","last_became_active":"2024-09-12T23:33:29.889548+0000","last_became_peered":"2024-09-12T23:33:29.889548+0000","last_unstale":"2024-09-12T23:35:08.661319+0000","last_undegraded":"2024-09-12T23:35:08.661319+0000","last_fullsized":"2024-09-12T23:35:08.661319+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-13T23:54:53.906619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.665474+0000","last_change":"2024-09-12T23:33:29.896515+0000","last_active":"2024-09-12T23:35:08.665474+0000","last_peered":"2024-09-12T23:35:08.665474+0000","last_clean":"2024-09-12T23:35:08.665474+0000","last_became_active":"2024-09-12T23:33:29.895919+0000","last_became_peered":"2024-09-12T23:33:29.895919+0000","last_unstale":"2024-09-12T23:35:08.665474+0000","last_undegraded":"2024-09-12T23:35:08.665474+0000","last_fullsized":"2024-09-12T23:35:08.665474+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:17:53.675501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.660905+0000","last_change":"2024-09-12T23:33:29.891596+0000","last_active":"2024-09-12T23:35:08.660905+0000","last_peered":"2024-09-12T23:35:08.660905+0000","last_clean":"2024-09-12T23:35:08.660905+0000","last_became_active":"2024-09-12T23:33:29.890839+0000","last_became_peered":"2024-09-12T23:33:29.890839+0000","last_unstale":"2024-09-12T23:35:08.660905+0000","last_undegraded":"2024-09-12T23:35:08.660905+0000","last_fullsized":"2024-09-12T23:35:08.660905+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:29:49.995950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1b","version":"53'1","reported_seq":22,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.668622+0000","last_change":"2024-09-12T23:33:28.883222+0000","last_active":"2024-09-12T23:35:08.668622+0000","last_peered":"2024-09-12T23:35:08.668622+0000","last_clean":"2024-09-12T23:35:08.668622+0000","last_became_active":"2024-09-12T23:33:28.882848+0000","last_became_peered":"2024-09-12T23:33:28.882848+0000","last_unstale":"2024-09-12T23:35:08.668622+0000","last_undegraded":"2024-09-12T23:35:08.668622+0000","last_fullsized":"2024-09-12T23:35:08.668622+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:57:33.905996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.669749+0000","last_change":"2024-09-12T23:33:29.912763+0000","last_active":"2024-09-12T23:35:08.669749+0000","last_peered":"2024-09-12T23:35:08.669749+0000","last_clean":"2024-09-12T23:35:08.669749+0000","last_became_active":"2024-09-12T23:33:29.912406+0000","last_became_peered":"2024-09-12T23:33:29.912406+0000","last_unstale":"2024-09-12T23:35:08.669749+0000","last_undegraded":"2024-09-12T23:35:08.669749+0000","last_fullsized":"2024-09-12T23:35:08.669749+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:49:06.828514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1a","version":"53'1","reported_seq":22,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.667384+0000","last_change":"2024-09-12T23:33:28.884599+0000","last_active":"2024-09-12T23:35:08.667384+0000","last_peered":"2024-09-12T23:35:08.667384+0000","last_clean":"2024-09-12T23:35:08.667384+0000","last_became_active":"2024-09-12T23:33:28.884139+0000","last_became_peered":"2024-09-12T23:33:28.884139+0000","last_unstale":"2024-09-12T23:35:08.667384+0000","last_undegraded":"2024-09-12T23:35:08.667384+0000","last_fullsized":"2024-09-12T23:35:08.667384+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:57:50.175287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":21,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.668064+0000","last_change":"2024-09-12T23:33:28.872822+0000","last_active":"2024-09-12T23:35:08.668064+0000","last_peered":"2024-09-12T23:35:08.668064+0000","last_clean":"2024-09-12T23:35:08.668064+0000","last_became_active":"2024-09-12T23:33:28.872404+0000","last_became_peered":"2024-09-12T23:33:28.872404+0000","last_unstale":"2024-09-12T23:35:08.668064+0000","last_undegraded":"2024-09-12T23:35:08.668064+0000","last_fullsized":"2024-09-12T23:35:08.668064+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-13T23:47:42.317408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.667077+0000","last_change":"2024-09-12T23:33:29.907360+0000","last_active":"2024-09-12T23:35:08.667077+0000","last_peered":"2024-09-12T23:35:08.667077+0000","last_clean":"2024-09-12T23:35:08.667077+0000","last_became_active":"2024-09-12T23:33:29.907067+0000","last_became_peered":"2024-09-12T23:33:29.907067+0000","last_unstale":"2024-09-12T23:35:08.667077+0000","last_undegraded":"2024-09-12T23:35:08.667077+0000","last_fullsized":"2024-09-12T23:35:08.667077+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:26:38.084938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":21,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.670406+0000","last_change":"2024-09-12T23:33:28.896704+0000","last_active":"2024-09-12T23:35:08.670406+0000","last_peered":"2024-09-12T23:35:08.670406+0000","last_clean":"2024-09-12T23:35:08.670406+0000","last_became_active":"2024-09-12T23:33:28.896167+0000","last_became_peered":"2024-09-12T23:33:28.896167+0000","last_unstale":"2024-09-12T23:35:08.670406+0000","last_undegraded":"2024-09-12T23:35:08.670406+0000","last_fullsized":"2024-09-12T23:35:08.670406+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:14:47.959501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.663618+0000","last_change":"2024-09-12T23:33:29.893401+0000","last_active":"2024-09-12T23:35:08.663618+0000","last_peered":"2024-09-12T23:35:08.663618+0000","last_clean":"2024-09-12T23:35:08.663618+0000","last_became_active":"2024-09-12T23:33:29.892988+0000","last_became_peered":"2024-09-12T23:33:29.892988+0000","last_unstale":"2024-09-12T23:35:08.663618+0000","last_undegraded":"2024-09-12T23:35:08.663618+0000","last_fullsized":"2024-09-12T23:35:08.663618+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:30:04.271132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":21,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.666010+0000","last_change":"2024-09-12T23:33:28.884106+0000","last_active":"2024-09-12T23:35:08.666010+0000","last_peered":"2024-09-12T23:35:08.666010+0000","last_clean":"2024-09-12T23:35:08.666010+0000","last_became_active":"2024-09-12T23:33:28.883028+0000","last_became_peered":"2024-09-12T23:33:28.883028+0000","last_unstale":"2024-09-12T23:35:08.666010+0000","last_undegraded":"2024-09-12T23:35:08.666010+0000","last_fullsized":"2024-09-12T23:35:08.666010+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:20:27.774655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.670853+0000","last_change":"2024-09-12T23:33:29.909613+0000","last_active":"2024-09-12T23:35:08.670853+0000","last_peered":"2024-09-12T23:35:08.670853+0000","last_clean":"2024-09-12T23:35:08.670853+0000","last_became_active":"2024-09-12T23:33:29.909325+0000","last_became_peered":"2024-09-12T23:33:29.909325+0000","last_unstale":"2024-09-12T23:35:08.670853+0000","last_undegraded":"2024-09-12T23:35:08.670853+0000","last_fullsized":"2024-09-12T23:35:08.670853+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:50:21.069936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":21,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.670886+0000","last_change":"2024-09-12T23:33:28.890012+0000","last_active":"2024-09-12T23:35:08.670886+0000","last_peered":"2024-09-12T23:35:08.670886+0000","last_clean":"2024-09-12T23:35:08.670886+0000","last_became_active":"2024-09-12T23:33:28.887477+0000","last_became_peered":"2024-09-12T23:33:28.887477+0000","last_unstale":"2024-09-12T23:35:08.670886+0000","last_undegraded":"2024-09-12T23:35:08.670886+0000","last_fullsized":"2024-09-12T23:35:08.670886+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:15:02.988389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.670299+0000","last_change":"2024-09-12T23:33:29.917549+0000","last_active":"2024-09-12T23:35:08.670299+0000","last_peered":"2024-09-12T23:35:08.670299+0000","last_clean":"2024-09-12T23:35:08.670299+0000","last_became_active":"2024-09-12T23:33:29.916648+0000","last_became_peered":"2024-09-12T23:33:29.916648+0000","last_unstale":"2024-09-12T23:35:08.670299+0000","last_undegraded":"2024-09-12T23:35:08.670299+0000","last_fullsized":"2024-09-12T23:35:08.670299+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:37:55.708866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"53'3","reported_seq":24,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.663816+0000","last_change":"2024-09-12T23:33:28.879732+0000","last_active":"2024-09-12T23:35:08.663816+0000","last_peered":"2024-09-12T23:35:08.663816+0000","last_clean":"2024-09-12T23:35:08.663816+0000","last_became_active":"2024-09-12T23:33:28.879399+0000","last_became_peered":"2024-09-12T23:33:28.879399+0000","last_unstale":"2024-09-12T23:35:08.663816+0000","last_undegraded":"2024-09-12T23:35:08.663816+0000","last_fullsized":"2024-09-12T23:35:08.663816+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:00:06.109352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":10,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-12T23:35:09.684524+0000","last_change":"2024-09-12T23:35:09.684524+0000","last_active":"2024-09-12T23:35:09.684524+0000","last_peered":"2024-09-12T23:35:09.684524+0000","last_clean":"2024-09-12T23:35:09.684524+0000","last_became_active":"2024-09-12T23:35:09.683923+0000","last_became_peered":"2024-09-12T23:35:09.683923+0000","last_unstale":"2024-09-12T23:35:09.684524+0000","last_undegraded":"2024-09-12T23:35:09.684524+0000","last_fullsized":"2024-09-12T23:35:09.684524+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:30:40.963502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,6],"acting":[2,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.14","version":"64'12","reported_seq":33,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-12T23:35:38.071488+0000","last_change":"2024-09-12T23:33:28.886930+0000","last_active":"2024-09-12T23:35:38.071488+0000","last_peered":"2024-09-12T23:35:38.071488+0000","last_clean":"2024-09-12T23:35:38.071488+0000","last_became_active":"2024-09-12T23:33:28.885991+0000","last_became_peered":"2024-09-12T23:33:28.885991+0000","last_unstale":"2024-09-12T23:35:38.071488+0000","last_undegraded":"2024-09-12T23:35:38.071488+0000","last_fullsized":"2024-09-12T23:35:38.071488+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":12,"log_dups_size":0,"ondisk_log_size":12,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:50:58.477301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2624,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":9,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.665823+0000","last_change":"2024-09-12T23:33:29.916929+0000","last_active":"2024-09-12T23:35:08.665823+0000","last_peered":"2024-09-12T23:35:08.665823+0000","last_clean":"2024-09-12T23:35:08.665823+0000","last_became_active":"2024-09-12T23:33:29.916622+0000","last_became_peered":"2024-09-12T23:33:29.916622+0000","last_unstale":"2024-09-12T23:35:08.665823+0000","last_undegraded":"2024-09-12T23:35:08.665823+0000","last_fullsized":"2024-09-12T23:35:08.665823+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:40:09.433278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.13","version":"53'2","reported_seq":23,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.669915+0000","last_change":"2024-09-12T23:33:28.880385+0000","last_active":"2024-09-12T23:35:08.669915+0000","last_peered":"2024-09-12T23:35:08.669915+0000","last_clean":"2024-09-12T23:35:08.669915+0000","last_became_active":"2024-09-12T23:33:28.879825+0000","last_became_peered":"2024-09-12T23:33:28.879825+0000","last_unstale":"2024-09-12T23:35:08.669915+0000","last_undegraded":"2024-09-12T23:35:08.669915+0000","last_fullsized":"2024-09-12T23:35:08.669915+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:40:33.073593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.670434+0000","last_change":"2024-09-12T23:33:29.919045+0000","last_active":"2024-09-12T23:35:08.670434+0000","last_peered":"2024-09-12T23:35:08.670434+0000","last_clean":"2024-09-12T23:35:08.670434+0000","last_became_active":"2024-09-12T23:33:29.918580+0000","last_became_peered":"2024-09-12T23:33:29.918580+0000","last_unstale":"2024-09-12T23:35:08.670434+0000","last_undegraded":"2024-09-12T23:35:08.670434+0000","last_fullsized":"2024-09-12T23:35:08.670434+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:39:32.320667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":21,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.671158+0000","last_change":"2024-09-12T23:33:28.899424+0000","last_active":"2024-09-12T23:35:08.671158+0000","last_peered":"2024-09-12T23:35:08.671158+0000","last_clean":"2024-09-12T23:35:08.671158+0000","last_became_active":"2024-09-12T23:33:28.898942+0000","last_became_peered":"2024-09-12T23:33:28.898942+0000","last_unstale":"2024-09-12T23:35:08.671158+0000","last_undegraded":"2024-09-12T23:35:08.671158+0000","last_fullsized":"2024-09-12T23:35:08.671158+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:14:47.069896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.663898+0000","last_change":"2024-09-12T23:33:29.901397+0000","last_active":"2024-09-12T23:35:08.663898+0000","last_peered":"2024-09-12T23:35:08.663898+0000","last_clean":"2024-09-12T23:35:08.663898+0000","last_became_active":"2024-09-12T23:33:29.901060+0000","last_became_peered":"2024-09-12T23:33:29.901060+0000","last_unstale":"2024-09-12T23:35:08.663898+0000","last_undegraded":"2024-09-12T23:35:08.663898+0000","last_fullsized":"2024-09-12T23:35:08.663898+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:21:23.285498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":21,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.665621+0000","last_change":"2024-09-12T23:33:28.871430+0000","last_active":"2024-09-12T23:35:08.665621+0000","last_peered":"2024-09-12T23:35:08.665621+0000","last_clean":"2024-09-12T23:35:08.665621+0000","last_became_active":"2024-09-12T23:33:28.869528+0000","last_became_peered":"2024-09-12T23:33:28.869528+0000","last_unstale":"2024-09-12T23:35:08.665621+0000","last_undegraded":"2024-09-12T23:35:08.665621+0000","last_fullsized":"2024-09-12T23:35:08.665621+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:27:43.449941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":30,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-12T23:35:09.683003+0000","last_change":"2024-09-12T23:35:09.683003+0000","last_active":"2024-09-12T23:35:09.683003+0000","last_peered":"2024-09-12T23:35:09.683003+0000","last_clean":"2024-09-12T23:35:09.683003+0000","last_became_active":"2024-09-12T23:35:09.681812+0000","last_became_peered":"2024-09-12T23:35:09.681812+0000","last_unstale":"2024-09-12T23:35:09.683003+0000","last_undegraded":"2024-09-12T23:35:09.683003+0000","last_fullsized":"2024-09-12T23:35:09.683003+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-13T23:47:15.002852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"53'8","reported_seq":29,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.661383+0000","last_change":"2024-09-12T23:33:28.863788+0000","last_active":"2024-09-12T23:35:08.661383+0000","last_peered":"2024-09-12T23:35:08.661383+0000","last_clean":"2024-09-12T23:35:08.661383+0000","last_became_active":"2024-09-12T23:33:28.863479+0000","last_became_peered":"2024-09-12T23:33:28.863479+0000","last_unstale":"2024-09-12T23:35:08.661383+0000","last_undegraded":"2024-09-12T23:35:08.661383+0000","last_fullsized":"2024-09-12T23:35:08.661383+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:26:09.204038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.664571+0000","last_change":"2024-09-12T23:33:29.902101+0000","last_active":"2024-09-12T23:35:08.664571+0000","last_peered":"2024-09-12T23:35:08.664571+0000","last_clean":"2024-09-12T23:35:08.664571+0000","last_became_active":"2024-09-12T23:33:29.901803+0000","last_became_peered":"2024-09-12T23:33:29.901803+0000","last_unstale":"2024-09-12T23:35:08.664571+0000","last_undegraded":"2024-09-12T23:35:08.664571+0000","last_fullsized":"2024-09-12T23:35:08.664571+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:00:48.557318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.f","version":"53'1","reported_seq":22,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.662210+0000","last_change":"2024-09-12T23:33:28.879318+0000","last_active":"2024-09-12T23:35:08.662210+0000","last_peered":"2024-09-12T23:35:08.662210+0000","last_clean":"2024-09-12T23:35:08.662210+0000","last_became_active":"2024-09-12T23:33:28.877913+0000","last_became_peered":"2024-09-12T23:33:28.877913+0000","last_unstale":"2024-09-12T23:35:08.662210+0000","last_undegraded":"2024-09-12T23:35:08.662210+0000","last_fullsized":"2024-09-12T23:35:08.662210+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:13:19.574352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.664646+0000","last_change":"2024-09-12T23:33:29.901391+0000","last_active":"2024-09-12T23:35:08.664646+0000","last_peered":"2024-09-12T23:35:08.664646+0000","last_clean":"2024-09-12T23:35:08.664646+0000","last_became_active":"2024-09-12T23:33:29.901052+0000","last_became_peered":"2024-09-12T23:33:29.901052+0000","last_unstale":"2024-09-12T23:35:08.664646+0000","last_undegraded":"2024-09-12T23:35:08.664646+0000","last_fullsized":"2024-09-12T23:35:08.664646+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:54:04.075006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"53'2","reported_seq":23,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.661659+0000","last_change":"2024-09-12T23:33:28.887441+0000","last_active":"2024-09-12T23:35:08.661659+0000","last_peered":"2024-09-12T23:35:08.661659+0000","last_clean":"2024-09-12T23:35:08.661659+0000","last_became_active":"2024-09-12T23:33:28.887109+0000","last_became_peered":"2024-09-12T23:33:28.887109+0000","last_unstale":"2024-09-12T23:35:08.661659+0000","last_undegraded":"2024-09-12T23:35:08.661659+0000","last_fullsized":"2024-09-12T23:35:08.661659+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:35:14.071342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.665111+0000","last_change":"2024-09-12T23:33:29.907594+0000","last_active":"2024-09-12T23:35:08.665111+0000","last_peered":"2024-09-12T23:35:08.665111+0000","last_clean":"2024-09-12T23:35:08.665111+0000","last_became_active":"2024-09-12T23:33:29.907302+0000","last_became_peered":"2024-09-12T23:33:29.907302+0000","last_unstale":"2024-09-12T23:35:08.665111+0000","last_undegraded":"2024-09-12T23:35:08.665111+0000","last_fullsized":"2024-09-12T23:35:08.665111+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:45:37.988381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"53'1","reported_seq":22,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.663915+0000","last_change":"2024-09-12T23:33:28.871944+0000","last_active":"2024-09-12T23:35:08.663915+0000","last_peered":"2024-09-12T23:35:08.663915+0000","last_clean":"2024-09-12T23:35:08.663915+0000","last_became_active":"2024-09-12T23:33:28.871628+0000","last_became_peered":"2024-09-12T23:33:28.871628+0000","last_unstale":"2024-09-12T23:35:08.663915+0000","last_undegraded":"2024-09-12T23:35:08.663915+0000","last_fullsized":"2024-09-12T23:35:08.663915+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:58:29.645624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.670659+0000","last_change":"2024-09-12T23:33:29.914833+0000","last_active":"2024-09-12T23:35:08.670659+0000","last_peered":"2024-09-12T23:35:08.670659+0000","last_clean":"2024-09-12T23:35:08.670659+0000","last_became_active":"2024-09-12T23:33:29.914416+0000","last_became_peered":"2024-09-12T23:33:29.914416+0000","last_unstale":"2024-09-12T23:35:08.670659+0000","last_undegraded":"2024-09-12T23:35:08.670659+0000","last_fullsized":"2024-09-12T23:35:08.670659+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:27:20.253954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.c","version":"0'0","reported_seq":21,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.662019+0000","last_change":"2024-09-12T23:33:28.882138+0000","last_active":"2024-09-12T23:35:08.662019+0000","last_peered":"2024-09-12T23:35:08.662019+0000","last_clean":"2024-09-12T23:35:08.662019+0000","last_became_active":"2024-09-12T23:33:28.881774+0000","last_became_peered":"2024-09-12T23:33:28.881774+0000","last_unstale":"2024-09-12T23:35:08.662019+0000","last_undegraded":"2024-09-12T23:35:08.662019+0000","last_fullsized":"2024-09-12T23:35:08.662019+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:07:36.075590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.665947+0000","last_change":"2024-09-12T23:33:29.911549+0000","last_active":"2024-09-12T23:35:08.665947+0000","last_peered":"2024-09-12T23:35:08.665947+0000","last_clean":"2024-09-12T23:35:08.665947+0000","last_became_active":"2024-09-12T23:33:29.911244+0000","last_became_peered":"2024-09-12T23:33:29.911244+0000","last_unstale":"2024-09-12T23:35:08.665947+0000","last_undegraded":"2024-09-12T23:35:08.665947+0000","last_fullsized":"2024-09-12T23:35:08.665947+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:22:20.405567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":21,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.666935+0000","last_change":"2024-09-12T23:34:59.598602+0000","last_active":"2024-09-12T23:35:08.666935+0000","last_peered":"2024-09-12T23:35:08.666935+0000","last_clean":"2024-09-12T23:35:08.666935+0000","last_became_active":"2024-09-12T23:34:49.734724+0000","last_became_peered":"2024-09-12T23:34:49.734724+0000","last_unstale":"2024-09-12T23:35:08.666935+0000","last_undegraded":"2024-09-12T23:35:08.666935+0000","last_fullsized":"2024-09-12T23:35:08.666935+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:34:48.454503+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:34:48.454503+0000","last_clean_scrub_stamp":"2024-09-12T23:34:48.454503+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:27:05.298933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00079141900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"55'79","reported_seq":55,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.664101+0000","last_change":"2024-09-12T23:32:45.902460+0000","last_active":"2024-09-12T23:35:08.664101+0000","last_peered":"2024-09-12T23:35:08.664101+0000","last_clean":"2024-09-12T23:35:08.664101+0000","last_became_active":"2024-09-12T23:32:45.887205+0000","last_became_peered":"2024-09-12T23:32:45.887205+0000","last_unstale":"2024-09-12T23:35:08.664101+0000","last_undegraded":"2024-09-12T23:35:08.664101+0000","last_fullsized":"2024-09-12T23:35:08.664101+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:28:34.324093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:28:34.324093+0000","last_clean_scrub_stamp":"2024-09-12T23:28:34.324093+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:37:37.825919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":21,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.670187+0000","last_change":"2024-09-12T23:33:28.898796+0000","last_active":"2024-09-12T23:35:08.670187+0000","last_peered":"2024-09-12T23:35:08.670187+0000","last_clean":"2024-09-12T23:35:08.670187+0000","last_became_active":"2024-09-12T23:33:28.898329+0000","last_became_peered":"2024-09-12T23:33:28.898329+0000","last_unstale":"2024-09-12T23:35:08.670187+0000","last_undegraded":"2024-09-12T23:35:08.670187+0000","last_fullsized":"2024-09-12T23:35:08.670187+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:54:48.474286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.668367+0000","last_change":"2024-09-12T23:33:29.905028+0000","last_active":"2024-09-12T23:35:08.668367+0000","last_peered":"2024-09-12T23:35:08.668367+0000","last_clean":"2024-09-12T23:35:08.668367+0000","last_became_active":"2024-09-12T23:33:29.904695+0000","last_became_peered":"2024-09-12T23:33:29.904695+0000","last_unstale":"2024-09-12T23:35:08.668367+0000","last_undegraded":"2024-09-12T23:35:08.668367+0000","last_fullsized":"2024-09-12T23:35:08.668367+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:06:47.004852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":21,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.669471+0000","last_change":"2024-09-12T23:34:59.542653+0000","last_active":"2024-09-12T23:35:08.669471+0000","last_peered":"2024-09-12T23:35:08.669471+0000","last_clean":"2024-09-12T23:35:08.669471+0000","last_became_active":"2024-09-12T23:34:49.788254+0000","last_became_peered":"2024-09-12T23:34:49.788254+0000","last_unstale":"2024-09-12T23:35:08.669471+0000","last_undegraded":"2024-09-12T23:35:08.669471+0000","last_fullsized":"2024-09-12T23:35:08.669471+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:34:48.454503+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:34:48.454503+0000","last_clean_scrub_stamp":"2024-09-12T23:34:48.454503+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:47:44.627533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0012029090000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":21,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.663998+0000","last_change":"2024-09-12T23:33:28.895135+0000","last_active":"2024-09-12T23:35:08.663998+0000","last_peered":"2024-09-12T23:35:08.663998+0000","last_clean":"2024-09-12T23:35:08.663998+0000","last_became_active":"2024-09-12T23:33:28.894687+0000","last_became_peered":"2024-09-12T23:33:28.894687+0000","last_unstale":"2024-09-12T23:35:08.663998+0000","last_undegraded":"2024-09-12T23:35:08.663998+0000","last_fullsized":"2024-09-12T23:35:08.663998+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:57:06.646238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.669881+0000","last_change":"2024-09-12T23:33:29.910025+0000","last_active":"2024-09-12T23:35:08.669881+0000","last_peered":"2024-09-12T23:35:08.669881+0000","last_clean":"2024-09-12T23:35:08.669881+0000","last_became_active":"2024-09-12T23:33:29.909765+0000","last_became_peered":"2024-09-12T23:33:29.909765+0000","last_unstale":"2024-09-12T23:35:08.669881+0000","last_undegraded":"2024-09-12T23:35:08.669881+0000","last_fullsized":"2024-09-12T23:35:08.669881+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:23:35.389216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":21,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.663799+0000","last_change":"2024-09-12T23:34:59.549321+0000","last_active":"2024-09-12T23:35:08.663799+0000","last_peered":"2024-09-12T23:35:08.663799+0000","last_clean":"2024-09-12T23:35:08.663799+0000","last_became_active":"2024-09-12T23:34:49.788699+0000","last_became_peered":"2024-09-12T23:34:49.788699+0000","last_unstale":"2024-09-12T23:35:08.663799+0000","last_undegraded":"2024-09-12T23:35:08.663799+0000","last_fullsized":"2024-09-12T23:35:08.663799+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:34:48.454503+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:34:48.454503+0000","last_clean_scrub_stamp":"2024-09-12T23:34:48.454503+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:29:56.436091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0020296509999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":21,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.661420+0000","last_change":"2024-09-12T23:33:28.868957+0000","last_active":"2024-09-12T23:35:08.661420+0000","last_peered":"2024-09-12T23:35:08.661420+0000","last_clean":"2024-09-12T23:35:08.661420+0000","last_became_active":"2024-09-12T23:33:28.868587+0000","last_became_peered":"2024-09-12T23:33:28.868587+0000","last_unstale":"2024-09-12T23:35:08.661420+0000","last_undegraded":"2024-09-12T23:35:08.661420+0000","last_fullsized":"2024-09-12T23:35:08.661420+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:45:32.284209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.663738+0000","last_change":"2024-09-12T23:33:29.890360+0000","last_active":"2024-09-12T23:35:08.663738+0000","last_peered":"2024-09-12T23:35:08.663738+0000","last_clean":"2024-09-12T23:35:08.663738+0000","last_became_active":"2024-09-12T23:33:29.889575+0000","last_became_peered":"2024-09-12T23:33:29.889575+0000","last_unstale":"2024-09-12T23:35:08.663738+0000","last_undegraded":"2024-09-12T23:35:08.663738+0000","last_fullsized":"2024-09-12T23:35:08.663738+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:36:34.938054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":21,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.663553+0000","last_change":"2024-09-12T23:34:59.548775+0000","last_active":"2024-09-12T23:35:08.663553+0000","last_peered":"2024-09-12T23:35:08.663553+0000","last_clean":"2024-09-12T23:35:08.663553+0000","last_became_active":"2024-09-12T23:34:49.787490+0000","last_became_peered":"2024-09-12T23:34:49.787490+0000","last_unstale":"2024-09-12T23:35:08.663553+0000","last_undegraded":"2024-09-12T23:35:08.663553+0000","last_fullsized":"2024-09-12T23:35:08.663553+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:34:48.454503+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:34:48.454503+0000","last_clean_scrub_stamp":"2024-09-12T23:34:48.454503+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:19:10.283468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0020519940000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":21,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.671197+0000","last_change":"2024-09-12T23:33:28.899931+0000","last_active":"2024-09-12T23:35:08.671197+0000","last_peered":"2024-09-12T23:35:08.671197+0000","last_clean":"2024-09-12T23:35:08.671197+0000","last_became_active":"2024-09-12T23:33:28.899410+0000","last_became_peered":"2024-09-12T23:33:28.899410+0000","last_unstale":"2024-09-12T23:35:08.671197+0000","last_undegraded":"2024-09-12T23:35:08.671197+0000","last_fullsized":"2024-09-12T23:35:08.671197+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:05:26.671172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.661569+0000","last_change":"2024-09-12T23:33:29.902878+0000","last_active":"2024-09-12T23:35:08.661569+0000","last_peered":"2024-09-12T23:35:08.661569+0000","last_clean":"2024-09-12T23:35:08.661569+0000","last_became_active":"2024-09-12T23:33:29.902561+0000","last_became_peered":"2024-09-12T23:33:29.902561+0000","last_unstale":"2024-09-12T23:35:08.661569+0000","last_undegraded":"2024-09-12T23:35:08.661569+0000","last_fullsized":"2024-09-12T23:35:08.661569+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:12:55.352139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.2","version":"61'2","reported_seq":23,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.664284+0000","last_change":"2024-09-12T23:34:59.548734+0000","last_active":"2024-09-12T23:35:08.664284+0000","last_peered":"2024-09-12T23:35:08.664284+0000","last_clean":"2024-09-12T23:35:08.664284+0000","last_became_active":"2024-09-12T23:34:49.792105+0000","last_became_peered":"2024-09-12T23:34:49.792105+0000","last_unstale":"2024-09-12T23:35:08.664284+0000","last_undegraded":"2024-09-12T23:35:08.664284+0000","last_fullsized":"2024-09-12T23:35:08.664284+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:34:48.454503+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:34:48.454503+0000","last_clean_scrub_stamp":"2024-09-12T23:34:48.454503+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-13T23:37:40.496016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.002201358,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":21,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.664823+0000","last_change":"2024-09-12T23:33:28.867304+0000","last_active":"2024-09-12T23:35:08.664823+0000","last_peered":"2024-09-12T23:35:08.664823+0000","last_clean":"2024-09-12T23:35:08.664823+0000","last_became_active":"2024-09-12T23:33:28.867020+0000","last_became_peered":"2024-09-12T23:33:28.867020+0000","last_unstale":"2024-09-12T23:35:08.664823+0000","last_undegraded":"2024-09-12T23:35:08.664823+0000","last_fullsized":"2024-09-12T23:35:08.664823+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:51:22.908188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.670703+0000","last_change":"2024-09-12T23:33:29.914728+0000","last_active":"2024-09-12T23:35:08.670703+0000","last_peered":"2024-09-12T23:35:08.670703+0000","last_clean":"2024-09-12T23:35:08.670703+0000","last_became_active":"2024-09-12T23:33:29.914385+0000","last_became_peered":"2024-09-12T23:33:29.914385+0000","last_unstale":"2024-09-12T23:35:08.670703+0000","last_undegraded":"2024-09-12T23:35:08.670703+0000","last_fullsized":"2024-09-12T23:35:08.670703+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:45:02.815733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"59'1","reported_seq":22,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.669319+0000","last_change":"2024-09-12T23:34:59.541122+0000","last_active":"2024-09-12T23:35:08.669319+0000","last_peered":"2024-09-12T23:35:08.669319+0000","last_clean":"2024-09-12T23:35:08.669319+0000","last_became_active":"2024-09-12T23:34:49.787835+0000","last_became_peered":"2024-09-12T23:34:49.787835+0000","last_unstale":"2024-09-12T23:35:08.669319+0000","last_undegraded":"2024-09-12T23:35:08.669319+0000","last_fullsized":"2024-09-12T23:35:08.669319+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:34:48.454503+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:34:48.454503+0000","last_clean_scrub_stamp":"2024-09-12T23:34:48.454503+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:07:48.849192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099226499999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":21,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.666213+0000","last_change":"2024-09-12T23:33:28.895077+0000","last_active":"2024-09-12T23:35:08.666213+0000","last_peered":"2024-09-12T23:35:08.666213+0000","last_clean":"2024-09-12T23:35:08.666213+0000","last_became_active":"2024-09-12T23:33:28.892268+0000","last_became_peered":"2024-09-12T23:33:28.892268+0000","last_unstale":"2024-09-12T23:35:08.666213+0000","last_undegraded":"2024-09-12T23:35:08.666213+0000","last_fullsized":"2024-09-12T23:35:08.666213+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:26:19.125877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.664444+0000","last_change":"2024-09-12T23:33:29.899700+0000","last_active":"2024-09-12T23:35:08.664444+0000","last_peered":"2024-09-12T23:35:08.664444+0000","last_clean":"2024-09-12T23:35:08.664444+0000","last_became_active":"2024-09-12T23:33:29.899381+0000","last_became_peered":"2024-09-12T23:33:29.899381+0000","last_unstale":"2024-09-12T23:35:08.664444+0000","last_undegraded":"2024-09-12T23:35:08.664444+0000","last_fullsized":"2024-09-12T23:35:08.664444+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:30:20.590373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":21,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.666658+0000","last_change":"2024-09-12T23:34:59.899761+0000","last_active":"2024-09-12T23:35:08.666658+0000","last_peered":"2024-09-12T23:35:08.666658+0000","last_clean":"2024-09-12T23:35:08.666658+0000","last_became_active":"2024-09-12T23:34:49.736609+0000","last_became_peered":"2024-09-12T23:34:49.736609+0000","last_unstale":"2024-09-12T23:35:08.666658+0000","last_undegraded":"2024-09-12T23:35:08.666658+0000","last_fullsized":"2024-09-12T23:35:08.666658+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:34:48.454503+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:34:48.454503+0000","last_clean_scrub_stamp":"2024-09-12T23:34:48.454503+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:01:06.682284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001798914,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"54'1","reported_seq":22,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.665023+0000","last_change":"2024-09-12T23:33:28.866367+0000","last_active":"2024-09-12T23:35:08.665023+0000","last_peered":"2024-09-12T23:35:08.665023+0000","last_clean":"2024-09-12T23:35:08.665023+0000","last_became_active":"2024-09-12T23:33:28.866005+0000","last_became_peered":"2024-09-12T23:33:28.866005+0000","last_unstale":"2024-09-12T23:35:08.665023+0000","last_undegraded":"2024-09-12T23:35:08.665023+0000","last_fullsized":"2024-09-12T23:35:08.665023+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:49:27.803045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.673648+0000","last_change":"2024-09-12T23:33:29.915275+0000","last_active":"2024-09-12T23:35:08.673648+0000","last_peered":"2024-09-12T23:35:08.673648+0000","last_clean":"2024-09-12T23:35:08.673648+0000","last_became_active":"2024-09-12T23:33:29.914934+0000","last_became_peered":"2024-09-12T23:33:29.914934+0000","last_unstale":"2024-09-12T23:35:08.673648+0000","last_undegraded":"2024-09-12T23:35:08.673648+0000","last_fullsized":"2024-09-12T23:35:08.673648+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:43:27.562543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":21,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.661581+0000","last_change":"2024-09-12T23:34:59.550136+0000","last_active":"2024-09-12T23:35:08.661581+0000","last_peered":"2024-09-12T23:35:08.661581+0000","last_clean":"2024-09-12T23:35:08.661581+0000","last_became_active":"2024-09-12T23:34:49.785409+0000","last_became_peered":"2024-09-12T23:34:49.785409+0000","last_unstale":"2024-09-12T23:35:08.661581+0000","last_undegraded":"2024-09-12T23:35:08.661581+0000","last_fullsized":"2024-09-12T23:35:08.661581+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:34:48.454503+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:34:48.454503+0000","last_clean_scrub_stamp":"2024-09-12T23:34:48.454503+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:30:29.736480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010162299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"53'2","reported_seq":23,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.665837+0000","last_change":"2024-09-12T23:33:28.887351+0000","last_active":"2024-09-12T23:35:08.665837+0000","last_peered":"2024-09-12T23:35:08.665837+0000","last_clean":"2024-09-12T23:35:08.665837+0000","last_became_active":"2024-09-12T23:33:28.886573+0000","last_became_peered":"2024-09-12T23:33:28.886573+0000","last_unstale":"2024-09-12T23:35:08.665837+0000","last_undegraded":"2024-09-12T23:35:08.665837+0000","last_fullsized":"2024-09-12T23:35:08.665837+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:58:40.820895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.669756+0000","last_change":"2024-09-12T23:33:29.908474+0000","last_active":"2024-09-12T23:35:08.669756+0000","last_peered":"2024-09-12T23:35:08.669756+0000","last_clean":"2024-09-12T23:35:08.669756+0000","last_became_active":"2024-09-12T23:33:29.906278+0000","last_became_peered":"2024-09-12T23:33:29.906278+0000","last_unstale":"2024-09-12T23:35:08.669756+0000","last_undegraded":"2024-09-12T23:35:08.669756+0000","last_fullsized":"2024-09-12T23:35:08.669756+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:27:51.832430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"0'0","reported_seq":21,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.664547+0000","last_change":"2024-09-12T23:33:28.894286+0000","last_active":"2024-09-12T23:35:08.664547+0000","last_peered":"2024-09-12T23:35:08.664547+0000","last_clean":"2024-09-12T23:35:08.664547+0000","last_became_active":"2024-09-12T23:33:28.893861+0000","last_became_peered":"2024-09-12T23:33:28.893861+0000","last_unstale":"2024-09-12T23:35:08.664547+0000","last_undegraded":"2024-09-12T23:35:08.664547+0000","last_fullsized":"2024-09-12T23:35:08.664547+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:20:01.806723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":10,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-12T23:35:09.676563+0000","last_change":"2024-09-12T23:35:09.676563+0000","last_active":"2024-09-12T23:35:09.676563+0000","last_peered":"2024-09-12T23:35:09.676563+0000","last_clean":"2024-09-12T23:35:09.676563+0000","last_became_active":"2024-09-12T23:35:09.675322+0000","last_became_peered":"2024-09-12T23:35:09.675322+0000","last_unstale":"2024-09-12T23:35:09.676563+0000","last_undegraded":"2024-09-12T23:35:09.676563+0000","last_fullsized":"2024-09-12T23:35:09.676563+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:11:37.262719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":21,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.664006+0000","last_change":"2024-09-12T23:33:28.884360+0000","last_active":"2024-09-12T23:35:08.664006+0000","last_peered":"2024-09-12T23:35:08.664006+0000","last_clean":"2024-09-12T23:35:08.664006+0000","last_became_active":"2024-09-12T23:33:28.884027+0000","last_became_peered":"2024-09-12T23:33:28.884027+0000","last_unstale":"2024-09-12T23:35:08.664006+0000","last_undegraded":"2024-09-12T23:35:08.664006+0000","last_fullsized":"2024-09-12T23:35:08.664006+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:11:57.567074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.667167+0000","last_change":"2024-09-12T23:33:29.915243+0000","last_active":"2024-09-12T23:35:08.667167+0000","last_peered":"2024-09-12T23:35:08.667167+0000","last_clean":"2024-09-12T23:35:08.667167+0000","last_became_active":"2024-09-12T23:33:29.914543+0000","last_became_peered":"2024-09-12T23:33:29.914543+0000","last_unstale":"2024-09-12T23:35:08.667167+0000","last_undegraded":"2024-09-12T23:35:08.667167+0000","last_fullsized":"2024-09-12T23:35:08.667167+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-13T23:48:16.023985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":21,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.664752+0000","last_change":"2024-09-12T23:33:28.883601+0000","last_active":"2024-09-12T23:35:08.664752+0000","last_peered":"2024-09-12T23:35:08.664752+0000","last_clean":"2024-09-12T23:35:08.664752+0000","last_became_active":"2024-09-12T23:33:28.883276+0000","last_became_peered":"2024-09-12T23:33:28.883276+0000","last_unstale":"2024-09-12T23:35:08.664752+0000","last_undegraded":"2024-09-12T23:35:08.664752+0000","last_fullsized":"2024-09-12T23:35:08.664752+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:26:21.733615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.668547+0000","last_change":"2024-09-12T23:33:29.908185+0000","last_active":"2024-09-12T23:35:08.668547+0000","last_peered":"2024-09-12T23:35:08.668547+0000","last_clean":"2024-09-12T23:35:08.668547+0000","last_became_active":"2024-09-12T23:33:29.907814+0000","last_became_peered":"2024-09-12T23:33:29.907814+0000","last_unstale":"2024-09-12T23:35:08.668547+0000","last_undegraded":"2024-09-12T23:35:08.668547+0000","last_fullsized":"2024-09-12T23:35:08.668547+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:45:47.207242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"53'2","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-12T23:35:09.689819+0000","last_change":"2024-09-12T23:35:09.689819+0000","last_active":"2024-09-12T23:35:09.689819+0000","last_peered":"2024-09-12T23:35:09.689819+0000","last_clean":"2024-09-12T23:35:09.689819+0000","last_became_active":"2024-09-12T23:35:09.683737+0000","last_became_peered":"2024-09-12T23:35:09.683737+0000","last_unstale":"2024-09-12T23:35:09.689819+0000","last_undegraded":"2024-09-12T23:35:09.689819+0000","last_fullsized":"2024-09-12T23:35:09.689819+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:18:02.546637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.667383+0000","last_change":"2024-09-12T23:33:29.897132+0000","last_active":"2024-09-12T23:35:08.667383+0000","last_peered":"2024-09-12T23:35:08.667383+0000","last_clean":"2024-09-12T23:35:08.667383+0000","last_became_active":"2024-09-12T23:33:29.896761+0000","last_became_peered":"2024-09-12T23:33:29.896761+0000","last_unstale":"2024-09-12T23:35:08.667383+0000","last_undegraded":"2024-09-12T23:35:08.667383+0000","last_fullsized":"2024-09-12T23:35:08.667383+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:49:56.554393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.670622+0000","last_change":"2024-09-12T23:33:29.911440+0000","last_active":"2024-09-12T23:35:08.670622+0000","last_peered":"2024-09-12T23:35:08.670622+0000","last_clean":"2024-09-12T23:35:08.670622+0000","last_became_active":"2024-09-12T23:33:29.911157+0000","last_became_peered":"2024-09-12T23:33:29.911157+0000","last_unstale":"2024-09-12T23:35:08.670622+0000","last_undegraded":"2024-09-12T23:35:08.670622+0000","last_fullsized":"2024-09-12T23:35:08.670622+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:49:59.313436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"53'2","reported_seq":23,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.664158+0000","last_change":"2024-09-12T23:33:28.902224+0000","last_active":"2024-09-12T23:35:08.664158+0000","last_peered":"2024-09-12T23:35:08.664158+0000","last_clean":"2024-09-12T23:35:08.664158+0000","last_became_active":"2024-09-12T23:33:28.901755+0000","last_became_peered":"2024-09-12T23:33:28.901755+0000","last_unstale":"2024-09-12T23:35:08.664158+0000","last_undegraded":"2024-09-12T23:35:08.664158+0000","last_fullsized":"2024-09-12T23:35:08.664158+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:06:15.504175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.670603+0000","last_change":"2024-09-12T23:33:29.913814+0000","last_active":"2024-09-12T23:35:08.670603+0000","last_peered":"2024-09-12T23:35:08.670603+0000","last_clean":"2024-09-12T23:35:08.670603+0000","last_became_active":"2024-09-12T23:33:29.913521+0000","last_became_peered":"2024-09-12T23:33:29.913521+0000","last_unstale":"2024-09-12T23:35:08.670603+0000","last_undegraded":"2024-09-12T23:35:08.670603+0000","last_fullsized":"2024-09-12T23:35:08.670603+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:30:35.754521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1d","version":"53'2","reported_seq":23,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.666037+0000","last_change":"2024-09-12T23:33:28.897466+0000","last_active":"2024-09-12T23:35:08.666037+0000","last_peered":"2024-09-12T23:35:08.666037+0000","last_clean":"2024-09-12T23:35:08.666037+0000","last_became_active":"2024-09-12T23:33:28.896936+0000","last_became_peered":"2024-09-12T23:33:28.896936+0000","last_unstale":"2024-09-12T23:35:08.666037+0000","last_undegraded":"2024-09-12T23:35:08.666037+0000","last_fullsized":"2024-09-12T23:35:08.666037+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:29:38.732168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.670437+0000","last_change":"2024-09-12T23:33:29.924610+0000","last_active":"2024-09-12T23:35:08.670437+0000","last_peered":"2024-09-12T23:35:08.670437+0000","last_clean":"2024-09-12T23:35:08.670437+0000","last_became_active":"2024-09-12T23:33:29.924320+0000","last_became_peered":"2024-09-12T23:33:29.924320+0000","last_unstale":"2024-09-12T23:35:08.670437+0000","last_undegraded":"2024-09-12T23:35:08.670437+0000","last_fullsized":"2024-09-12T23:35:08.670437+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:23:22.797301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1e","version":"53'1","reported_seq":22,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.673262+0000","last_change":"2024-09-12T23:33:28.894353+0000","last_active":"2024-09-12T23:35:08.673262+0000","last_peered":"2024-09-12T23:35:08.673262+0000","last_clean":"2024-09-12T23:35:08.673262+0000","last_became_active":"2024-09-12T23:33:28.893992+0000","last_became_peered":"2024-09-12T23:33:28.893992+0000","last_unstale":"2024-09-12T23:35:08.673262+0000","last_undegraded":"2024-09-12T23:35:08.673262+0000","last_fullsized":"2024-09-12T23:35:08.673262+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:43:43.889025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":20,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.673097+0000","last_change":"2024-09-12T23:33:29.891227+0000","last_active":"2024-09-12T23:35:08.673097+0000","last_peered":"2024-09-12T23:35:08.673097+0000","last_clean":"2024-09-12T23:35:08.673097+0000","last_became_active":"2024-09-12T23:33:29.890368+0000","last_became_peered":"2024-09-12T23:33:29.890368+0000","last_unstale":"2024-09-12T23:35:08.673097+0000","last_undegraded":"2024-09-12T23:35:08.673097+0000","last_fullsized":"2024-09-12T23:35:08.673097+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:38:37.526610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1f","version":"53'4","reported_seq":25,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-09-12T23:35:08.670421+0000","last_change":"2024-09-12T23:33:28.886016+0000","last_active":"2024-09-12T23:35:08.670421+0000","last_peered":"2024-09-12T23:35:08.670421+0000","last_clean":"2024-09-12T23:35:08.670421+0000","last_became_active":"2024-09-12T23:33:28.885720+0000","last_became_peered":"2024-09-12T23:33:28.885720+0000","last_unstale":"2024-09-12T23:35:08.670421+0000","last_undegraded":"2024-09-12T23:35:08.670421+0000","last_fullsized":"2024-09-12T23:35:08.670421+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:44:38.950039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":192,"acting":192,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":3410,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":51,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":10230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":45,"ondisk_log_size":45,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430254,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298580,"kb_used_data":1224,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446572,"statfs":{"total":95995035648,"available":95689289728,"internally_reserved":0,"allocated":1253376,"data_stored":882496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 12 23:35:49 2024","interfaces":[{"interface":"back","average":{"1min":0.75700000000000001,"5min":0.86899999999999999,"15min":0.89800000000000002},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.256,"5min":1.3600000000000001,"15min":1.3600000000000001},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.77500000000000002,"5min":0.89100000000000001,"15min":0.92900000000000005},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":1.0720000000000001,"5min":1.3740000000000001,"15min":1.3740000000000001},"last":1.014}]},{"osd":1,"last update":"Thu Sep 12 23:35:49 2024","interfaces":[{"interface":"back","average":{"1min":0.80700000000000005,"5min":0.90200000000000002,"15min":0.93400000000000005},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":1.179,"5min":1.3300000000000001,"15min":1.3300000000000001},"last":0.95599999999999996},{"interface":"front","average":{"1min":0.80800000000000005,"5min":0.872,"15min":0.88800000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":1.2270000000000001,"5min":1.474,"15min":1.474},"last":0.78000000000000003}]},{"osd":2,"last update":"Thu Sep 12 23:35:49 2024","interfaces":[{"interface":"back","average":{"1min":0.81100000000000005,"5min":0.89100000000000001,"15min":0.874},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":1.167,"5min":5.7919999999999998,"15min":5.7919999999999998},"last":0.88200000000000001},{"interface":"front","average":{"1min":0.82199999999999995,"5min":0.93500000000000005,"15min":0.96199999999999997},"min":{"1min":0.47199999999999998,"5min":0.435,"15min":0.435},"max":{"1min":1.085,"5min":1.363,"15min":1.363},"last":0.83599999999999997}]},{"osd":3,"last update":"Thu Sep 12 23:35:49 2024","interfaces":[{"interface":"back","average":{"1min":0.83299999999999996,"5min":0.91500000000000004,"15min":0.95099999999999996},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":1.2170000000000001,"5min":1.284,"15min":1.284},"last":1.085},{"interface":"front","average":{"1min":0.92000000000000004,"5min":1,"15min":1.0369999999999999},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":1.2030000000000001,"5min":1.407,"15min":1.407},"last":1.0580000000000001}]},{"osd":4,"last update":"Thu Sep 12 23:35:49 2024","interfaces":[{"interface":"back","average":{"1min":0.72599999999999998,"5min":0.79600000000000004,"15min":0.80300000000000005},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.0780000000000001,"5min":1.589,"15min":1.589},"last":0.92700000000000005},{"interface":"front","average":{"1min":0.80800000000000005,"5min":0.93799999999999994,"15min":0.995},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.179,"5min":1.3009999999999999,"15min":1.3009999999999999},"last":0.97999999999999998}]},{"osd":5,"last update":"Thu Sep 12 23:35:49 2024","interfaces":[{"interface":"back","average":{"1min":0.80200000000000005,"5min":0.95799999999999996,"15min":1.016},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":1.1970000000000001,"5min":1.371,"15min":1.371},"last":1.121},{"interface":"front","average":{"1min":0.84399999999999997,"5min":0.93999999999999995,"15min":0.97299999999999998},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":1.27,"5min":1.6339999999999999,"15min":1.6339999999999999},"last":1.0369999999999999}]},{"osd":6,"last update":"Thu Sep 12 23:35:49 2024","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":1.0309999999999999,"15min":1.073},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":1.1990000000000001,"5min":1.5349999999999999,"15min":1.5349999999999999},"last":1.202},{"interface":"front","average":{"1min":0.93700000000000006,"5min":1.0760000000000001,"15min":1.1259999999999999},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":1.2609999999999999,"5min":1.4530000000000001,"15min":1.4530000000000001},"last":1.1719999999999999}]}]},{"osd":6,"up_from":42,"seq":180388626488,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298576,"kb_used_data":1224,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446576,"statfs":{"total":95995035648,"available":95689293824,"internally_reserved":0,"allocated":1253376,"data_stored":884213,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 12 23:36:04 2024","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":0.85999999999999999,"15min":0.83999999999999997},"min":{"1min":0.52000000000000002,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":2.7069999999999999,"5min":2.7069999999999999,"15min":2.7069999999999999},"last":1.248},{"interface":"front","average":{"1min":0.83899999999999997,"5min":0.874,"15min":0.874},"min":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":1.2190000000000001,"5min":1.2569999999999999,"15min":1.2569999999999999},"last":0.89400000000000002}]},{"osd":1,"last update":"Thu Sep 12 23:36:04 2024","interfaces":[{"interface":"back","average":{"1min":1.03,"5min":0.872,"15min":0.81999999999999995},"min":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"max":{"1min":2.73,"5min":2.73,"15min":2.73},"last":1.323},{"interface":"front","average":{"1min":0.81299999999999994,"5min":0.85599999999999998,"15min":0.86599999999999999},"min":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"max":{"1min":1.056,"5min":1.306,"15min":1.306},"last":0.85999999999999999}]},{"osd":2,"last update":"Thu Sep 12 23:36:04 2024","interfaces":[{"interface":"back","average":{"1min":0.90300000000000002,"5min":0.875,"15min":0.86899999999999999},"min":{"1min":0.68200000000000005,"5min":0.61299999999999999,"15min":0.61299999999999999},"max":{"1min":1.157,"5min":1.4359999999999999,"15min":1.4359999999999999},"last":0.92400000000000004},{"interface":"front","average":{"1min":0.88700000000000001,"5min":0.86399999999999999,"15min":0.86299999999999999},"min":{"1min":0.55900000000000005,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":1.2310000000000001,"5min":1.2769999999999999,"15min":1.2769999999999999},"last":0.72499999999999998}]},{"osd":3,"last update":"Thu Sep 12 23:36:04 2024","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":0.90100000000000002,"15min":0.86099999999999999},"min":{"1min":0.60799999999999998,"5min":0.52400000000000002,"15min":0.52400000000000002},"max":{"1min":2.6880000000000002,"5min":2.6880000000000002,"15min":2.6880000000000002},"last":1.0860000000000001},{"interface":"front","average":{"1min":0.92700000000000005,"5min":0.92300000000000004,"15min":0.92900000000000005},"min":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":1.2310000000000001,"5min":1.349,"15min":1.349},"last":0.97699999999999998}]},{"osd":4,"last update":"Thu Sep 12 23:36:04 2024","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":0.83099999999999996,"15min":0.78700000000000003},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":1.2470000000000001,"5min":1.2729999999999999,"15min":1.2729999999999999},"last":0.82399999999999995},{"interface":"front","average":{"1min":1.022,"5min":0.93500000000000005,"15min":0.91200000000000003},"min":{"1min":0.39800000000000002,"5min":0.373,"15min":0.373},"max":{"1min":2.669,"5min":2.669,"15min":2.669},"last":1.389}]},{"osd":5,"last update":"Thu Sep 12 23:36:04 2024","interfaces":[{"interface":"back","average":{"1min":0.998,"5min":0.90200000000000002,"15min":0.89300000000000002},"min":{"1min":0.48299999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.4810000000000001,"5min":1.4810000000000001,"15min":1.4810000000000001},"last":1.012},{"interface":"front","average":{"1min":1.1459999999999999,"5min":0.98199999999999998,"15min":0.93100000000000005},"min":{"1min":0.437,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":2.621,"5min":2.621,"15min":2.621},"last":1.452}]},{"osd":7,"last update":"Thu Sep 12 23:35:48 2024","interfaces":[{"interface":"back","average":{"1min":1.0700000000000001,"5min":0.99199999999999999,"15min":0.99299999999999999},"min":{"1min":0.51500000000000001,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":1.3260000000000001,"5min":1.3460000000000001,"15min":1.3460000000000001},"last":0.95599999999999996},{"interface":"front","average":{"1min":1.22,"5min":1.097,"15min":1.0940000000000001},"min":{"1min":0.47099999999999997,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":2.6499999999999999,"5min":2.6499999999999999,"15min":2.6499999999999999},"last":1.1819999999999999}]}]},{"osd":5,"up_from":36,"seq":154618822723,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298000,"kb_used_data":648,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447152,"statfs":{"total":95995035648,"available":95689883648,"internally_reserved":0,"allocated":663552,"data_stored":293639,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 12 23:36:14 2024","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":0.82699999999999996,"15min":0.79600000000000004},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":1.234,"5min":1.6100000000000001,"15min":1.6100000000000001},"last":1.1870000000000001},{"interface":"front","average":{"1min":0.82799999999999996,"5min":0.85799999999999998,"15min":0.83599999999999997},"min":{"1min":0.48499999999999999,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":1.1220000000000001,"5min":1.3129999999999999,"15min":1.3129999999999999},"last":0.85699999999999998}]},{"osd":1,"last update":"Thu Sep 12 23:36:14 2024","interfaces":[{"interface":"back","average":{"1min":0.81299999999999994,"5min":0.87,"15min":0.85299999999999998},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":1.2629999999999999,"5min":1.4470000000000001,"15min":1.4470000000000001},"last":0.91500000000000004},{"interface":"front","average":{"1min":0.86399999999999999,"5min":0.84799999999999998,"15min":0.81100000000000005},"min":{"1min":0.54500000000000004,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":1.2170000000000001,"5min":1.321,"15min":1.321},"last":0.77000000000000002}]},{"osd":2,"last update":"Thu Sep 12 23:36:14 2024","interfaces":[{"interface":"back","average":{"1min":0.872,"5min":0.86299999999999999,"15min":0.86499999999999999},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":1.1830000000000001,"5min":1.2749999999999999,"15min":1.2749999999999999},"last":0.89600000000000002},{"interface":"front","average":{"1min":0.88400000000000001,"5min":0.90200000000000002,"15min":0.90800000000000003},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":1.306,"5min":1.3620000000000001,"15min":1.3620000000000001},"last":1.0189999999999999}]},{"osd":3,"last update":"Thu Sep 12 23:36:14 2024","interfaces":[{"interface":"back","average":{"1min":0.93200000000000005,"5min":0.91000000000000003,"15min":0.91000000000000003},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":1.232,"5min":1.3280000000000001,"15min":1.3280000000000001},"last":0.71499999999999997},{"interface":"front","average":{"1min":0.96399999999999997,"5min":0.94599999999999995,"15min":0.92400000000000004},"min":{"1min":0.495,"5min":0.495,"15min":0.495},"max":{"1min":1.266,"5min":1.536,"15min":1.536},"last":1.085}]},{"osd":4,"last update":"Thu Sep 12 23:36:14 2024","interfaces":[{"interface":"back","average":{"1min":0.91700000000000004,"5min":0.81299999999999994,"15min":0.76500000000000001},"min":{"1min":0.46899999999999997,"5min":0.308,"15min":0.308},"max":{"1min":1.198,"5min":1.391,"15min":1.391},"last":0.96599999999999997},{"interface":"front","average":{"1min":0.93600000000000005,"5min":0.85999999999999999,"15min":0.80200000000000005},"min":{"1min":0.432,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":1.2809999999999999,"5min":1.2809999999999999,"15min":1.2809999999999999},"last":0.98899999999999999}]},{"osd":6,"last update":"Thu Sep 12 23:35:58 2024","interfaces":[{"interface":"back","average":{"1min":0.93100000000000005,"5min":0.89200000000000002,"15min":0.82899999999999996},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.393,"5min":1.393,"15min":1.393},"last":0.80300000000000005},{"interface":"front","average":{"1min":0.98999999999999999,"5min":0.92100000000000004,"15min":0.84099999999999997},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":1.548,"5min":1.548,"15min":1.548},"last":1.1399999999999999}]},{"osd":7,"last update":"Thu Sep 12 23:35:53 2024","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":1.0109999999999999,"15min":1.0369999999999999},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":1.3340000000000001,"5min":1.46,"15min":1.46},"last":1.0429999999999999},{"interface":"front","average":{"1min":1.0169999999999999,"5min":1.05,"15min":1.0640000000000001},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":1.5069999999999999,"5min":1.617,"15min":1.617},"last":0.95299999999999996}]}]},{"osd":4,"up_from":30,"seq":128849018956,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298016,"kb_used_data":656,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447136,"statfs":{"total":95995035648,"available":95689867264,"internally_reserved":0,"allocated":671744,"data_stored":292251,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 12 23:35:22 2024","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":0.86499999999999999,"15min":0.84399999999999997},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":1.5209999999999999,"5min":1.5209999999999999,"15min":1.5209999999999999},"last":1.054},{"interface":"front","average":{"1min":0.91800000000000004,"5min":0.86599999999999999,"15min":0.84999999999999998},"min":{"1min":0.64300000000000002,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.599,"5min":1.599,"15min":1.599},"last":1.085}]},{"osd":1,"last update":"Thu Sep 12 23:35:22 2024","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":0.86499999999999999,"15min":0.83599999999999997},"min":{"1min":0.54400000000000004,"5min":0.52000000000000002,"15min":0.52000000000000002},"max":{"1min":1.6890000000000001,"5min":1.6890000000000001,"15min":1.6890000000000001},"last":0.80400000000000005},{"interface":"front","average":{"1min":0.876,"5min":0.86599999999999999,"15min":0.85999999999999999},"min":{"1min":0.64100000000000001,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":1.3779999999999999,"5min":1.3779999999999999,"15min":1.3779999999999999},"last":1.107}]},{"osd":2,"last update":"Thu Sep 12 23:35:22 2024","interfaces":[{"interface":"back","average":{"1min":0.93700000000000006,"5min":0.85799999999999998,"15min":0.85999999999999999},"min":{"1min":0.53900000000000003,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":1.466,"5min":1.466,"15min":1.466},"last":0.95899999999999996},{"interface":"front","average":{"1min":0.875,"5min":0.86699999999999999,"15min":0.86699999999999999},"min":{"1min":0.51900000000000002,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.1080000000000001,"5min":1.3120000000000001,"15min":1.3120000000000001},"last":0.99199999999999999}]},{"osd":3,"last update":"Thu Sep 12 23:35:22 2024","interfaces":[{"interface":"back","average":{"1min":1.014,"5min":0.89400000000000002,"15min":0.86399999999999999},"min":{"1min":0.56899999999999995,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":1.419,"5min":1.5129999999999999,"15min":1.5129999999999999},"last":0.91500000000000004},{"interface":"front","average":{"1min":1.044,"5min":0.94399999999999995,"15min":0.90400000000000003},"min":{"1min":0.72499999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":1.7869999999999999,"5min":1.7869999999999999,"15min":1.7869999999999999},"last":1.228}]},{"osd":5,"last update":"Thu Sep 12 23:36:15 2024","interfaces":[{"interface":"back","average":{"1min":0.93899999999999995,"5min":0.86099999999999999,"15min":0.80600000000000005},"min":{"1min":0.57799999999999996,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":1.1419999999999999,"5min":1.835,"15min":1.835},"last":1.373},{"interface":"front","average":{"1min":0.98999999999999999,"5min":0.93700000000000006,"15min":0.84699999999999998},"min":{"1min":0.57899999999999996,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.131,"5min":1.716,"15min":1.716},"last":1.2589999999999999}]},{"osd":6,"last update":"Thu Sep 12 23:36:00 2024","interfaces":[{"interface":"back","average":{"1min":1.01,"5min":0.95199999999999996,"15min":0.92600000000000005},"min":{"1min":0.69799999999999995,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":1.776,"5min":1.776,"15min":1.776},"last":1.1879999999999999},{"interface":"front","average":{"1min":1.075,"5min":1.014,"15min":0.97999999999999998},"min":{"1min":0.61499999999999999,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":1.9119999999999999,"5min":1.9119999999999999,"15min":1.9119999999999999},"last":1.306}]},{"osd":7,"last update":"Thu Sep 12 23:35:47 2024","interfaces":[{"interface":"back","average":{"1min":1.115,"5min":1.0600000000000001,"15min":1.0680000000000001},"min":{"1min":0.61399999999999999,"5min":0.373,"15min":0.373},"max":{"1min":1.8720000000000001,"5min":1.8720000000000001,"15min":1.8720000000000001},"last":1.147},{"interface":"front","average":{"1min":1.119,"5min":1.056,"15min":1.0680000000000001},"min":{"1min":0.81200000000000006,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.7410000000000001,"5min":1.7410000000000001,"15min":1.7410000000000001},"last":1.1319999999999999}]}]},{"osd":3,"up_from":25,"seq":107374182486,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298008,"kb_used_data":656,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447144,"statfs":{"total":95995035648,"available":95689875456,"internally_reserved":0,"allocated":671744,"data_stored":294264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 12 23:35:38 2024","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":0.64000000000000001,"15min":0.60799999999999998},"min":{"1min":0.318,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":1.0489999999999999,"5min":1.0489999999999999,"15min":1.0489999999999999},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.66500000000000004,"5min":0.70699999999999996,"15min":0.69399999999999995},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":1.101,"5min":1.5009999999999999,"15min":1.5009999999999999},"last":0.78300000000000003}]},{"osd":1,"last update":"Thu Sep 12 23:35:38 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.67400000000000004,"15min":0.65600000000000003},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.88900000000000001,"5min":1.006,"15min":1.006},"last":0.70199999999999996},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.72899999999999998,"15min":0.73499999999999999},"min":{"1min":0.30499999999999999,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.88200000000000001,"5min":1.401,"15min":1.401},"last":0.57599999999999996}]},{"osd":2,"last update":"Thu Sep 12 23:35:38 2024","interfaces":[{"interface":"back","average":{"1min":0.77500000000000002,"5min":0.79600000000000004,"15min":0.77900000000000003},"min":{"1min":0.38400000000000001,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":1.0609999999999999,"5min":1.2709999999999999,"15min":1.2709999999999999},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.754,"5min":0.79500000000000004,"15min":0.747},"min":{"1min":0.34999999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":1.1699999999999999,"5min":1.458,"15min":1.458},"last":0.93899999999999995}]},{"osd":4,"last update":"Thu Sep 12 23:35:26 2024","interfaces":[{"interface":"back","average":{"1min":0.95099999999999996,"5min":0.92900000000000005,"15min":0.89100000000000001},"min":{"1min":0.65200000000000002,"5min":0.497,"15min":0.497},"max":{"1min":1.54,"5min":1.54,"15min":1.54},"last":1.1639999999999999},{"interface":"front","average":{"1min":0.90600000000000003,"5min":0.90100000000000002,"15min":0.85899999999999999},"min":{"1min":0.57199999999999995,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":1.1539999999999999,"5min":1.3640000000000001,"15min":1.3640000000000001},"last":1.081}]},{"osd":5,"last update":"Thu Sep 12 23:36:11 2024","interfaces":[{"interface":"back","average":{"1min":0.96599999999999997,"5min":0.97499999999999998,"15min":0.98099999999999998},"min":{"1min":0.67100000000000004,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":1.3620000000000001,"5min":1.6120000000000001,"15min":1.6120000000000001},"last":1.2410000000000001},{"interface":"front","average":{"1min":0.92100000000000004,"5min":0.96099999999999997,"15min":0.96399999999999997},"min":{"1min":0.56000000000000005,"5min":0.437,"15min":0.437},"max":{"1min":1.2110000000000001,"5min":1.5429999999999999,"15min":1.5429999999999999},"last":0.97999999999999998}]},{"osd":6,"last update":"Thu Sep 12 23:36:03 2024","interfaces":[{"interface":"back","average":{"1min":0.93300000000000005,"5min":0.999,"15min":1.034},"min":{"1min":0.59499999999999997,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":1.28,"5min":1.4930000000000001,"15min":1.4930000000000001},"last":1.0289999999999999},{"interface":"front","average":{"1min":1.0269999999999999,"5min":1.0820000000000001,"15min":1.1040000000000001},"min":{"1min":0.58399999999999996,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":2.0099999999999998,"5min":2.0099999999999998,"15min":2.0099999999999998},"last":1.393}]},{"osd":7,"last update":"Thu Sep 12 23:35:50 2024","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":1.0429999999999999,"15min":1.0309999999999999},"min":{"1min":0.58799999999999997,"5min":0.52300000000000002,"15min":0.52300000000000002},"max":{"1min":1.405,"5min":1.6879999999999999,"15min":1.6879999999999999},"last":1.4590000000000001},{"interface":"front","average":{"1min":1.0129999999999999,"5min":1.0580000000000001,"15min":1.0649999999999999},"min":{"1min":0.61299999999999999,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":1.3169999999999999,"5min":1.5880000000000001,"15min":1.5880000000000001},"last":1.127}]}]},{"osd":2,"up_from":18,"seq":77309411424,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297996,"kb_used_data":636,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447156,"statfs":{"total":95995035648,"available":95689887744,"internally_reserved":0,"allocated":651264,"data_stored":291536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 12 23:35:47 2024","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":0.64700000000000002,"15min":0.59799999999999998},"min":{"1min":0.34599999999999997,"5min":0.28999999999999998,"15min":0.28799999999999998},"max":{"1min":1.018,"5min":1.0429999999999999,"15min":1.23},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.76300000000000001,"5min":0.72999999999999998,"15min":0.64100000000000001},"min":{"1min":0.34899999999999998,"5min":0.318,"15min":0.26200000000000001},"max":{"1min":1.0720000000000001,"5min":1.1870000000000001,"15min":1.1870000000000001},"last":0.84199999999999997}]},{"osd":1,"last update":"Thu Sep 12 23:35:47 2024","interfaces":[{"interface":"back","average":{"1min":0.68999999999999995,"5min":0.70999999999999996,"15min":0.64400000000000002},"min":{"1min":0.36799999999999999,"5min":0.33200000000000002,"15min":0.316},"max":{"1min":0.95099999999999996,"5min":1.5329999999999999,"15min":1.5329999999999999},"last":0.76500000000000001},{"interface":"front","average":{"1min":0.81100000000000005,"5min":0.754,"15min":0.66500000000000004},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":1.1779999999999999,"5min":1.1779999999999999,"15min":1.1779999999999999},"last":0.97699999999999998}]},{"osd":3,"last update":"Thu Sep 12 23:35:34 2024","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":0.77100000000000002,"15min":0.749},"min":{"1min":0.40000000000000002,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.151,"5min":1.2270000000000001,"15min":1.2270000000000001},"last":0.70999999999999996},{"interface":"front","average":{"1min":0.874,"5min":0.80300000000000005,"15min":0.78100000000000003},"min":{"1min":0.441,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.345,"5min":1.6619999999999999,"15min":1.6619999999999999},"last":0.56999999999999995}]},{"osd":4,"last update":"Thu Sep 12 23:35:20 2024","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":0.90800000000000003,"15min":0.876},"min":{"1min":0.66000000000000003,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.651,"5min":1.651,"15min":1.651},"last":0.80000000000000004},{"interface":"front","average":{"1min":1.0780000000000001,"5min":0.995,"15min":0.94899999999999995},"min":{"1min":0.70599999999999996,"5min":0.56899999999999995,"15min":0.56899999999999995},"max":{"1min":1.8660000000000001,"5min":1.8660000000000001,"15min":1.8660000000000001},"last":1.034}]},{"osd":5,"last update":"Thu Sep 12 23:36:18 2024","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":0.97199999999999998,"15min":1.016},"min":{"1min":0.79300000000000004,"5min":0.65200000000000002,"15min":0.65200000000000002},"max":{"1min":1.232,"5min":1.681,"15min":1.681},"last":1.115},{"interface":"front","average":{"1min":0.94999999999999996,"5min":0.96899999999999997,"15min":1.01},"min":{"1min":0.67600000000000005,"5min":0.61499999999999999,"15min":0.61499999999999999},"max":{"1min":1.286,"5min":1.3129999999999999,"15min":1.3129999999999999},"last":0.95199999999999996}]},{"osd":6,"last update":"Thu Sep 12 23:36:01 2024","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":0.96899999999999997,"15min":0.95299999999999996},"min":{"1min":0.76800000000000002,"5min":0.47799999999999998,"15min":0.47799999999999998},"max":{"1min":1.4279999999999999,"5min":1.429,"15min":1.429},"last":1.1579999999999999},{"interface":"front","average":{"1min":1.0569999999999999,"5min":1.056,"15min":1.077},"min":{"1min":0.66100000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":1.276,"5min":1.7909999999999999,"15min":1.7909999999999999},"last":1.2070000000000001}]},{"osd":7,"last update":"Thu Sep 12 23:35:47 2024","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":1.0069999999999999,"15min":0.97699999999999998},"min":{"1min":0.71199999999999997,"5min":0.57299999999999995,"15min":0.57299999999999995},"max":{"1min":1.373,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":1.0629999999999999},{"interface":"front","average":{"1min":1.0980000000000001,"5min":1.01,"15min":0.97799999999999998},"min":{"1min":0.60399999999999998,"5min":0.50900000000000001,"15min":0.50900000000000001},"max":{"1min":1.7609999999999999,"5min":1.7609999999999999,"15min":1.7609999999999999},"last":1.085}]}]},{"osd":1,"up_from":13,"seq":55834574955,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298000,"kb_used_data":648,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447152,"statfs":{"total":95995035648,"available":95689883648,"internally_reserved":0,"allocated":663552,"data_stored":291715,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 12 23:36:02 2024","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":0.64900000000000002,"15min":0.55800000000000005},"min":{"1min":0.36199999999999999,"5min":0.317,"15min":0.255},"max":{"1min":1.0169999999999999,"5min":1.1579999999999999,"15min":1.1579999999999999},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.67300000000000004,"5min":0.71899999999999997,"15min":0.61399999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.25600000000000001},"max":{"1min":0.88200000000000001,"5min":1.1830000000000001,"15min":1.1830000000000001},"last":0.81999999999999995}]},{"osd":2,"last update":"Thu Sep 12 23:35:48 2024","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":0.70399999999999996,"15min":0.61499999999999999},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.26700000000000002},"max":{"1min":0.97599999999999998,"5min":1.238,"15min":1.238},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.71299999999999997,"5min":0.751,"15min":0.65500000000000003},"min":{"1min":0.42399999999999999,"5min":0.41299999999999998,"15min":0.28999999999999998},"max":{"1min":1.147,"5min":1.3049999999999999,"15min":1.3049999999999999},"last":0.749}]},{"osd":3,"last update":"Thu Sep 12 23:35:38 2024","interfaces":[{"interface":"back","average":{"1min":0.76800000000000002,"5min":0.81299999999999994,"15min":0.78900000000000003},"min":{"1min":0.49299999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.093,"5min":1.198,"15min":1.198},"last":0.85599999999999998},{"interface":"front","average":{"1min":0.746,"5min":0.78900000000000003,"15min":0.76800000000000002},"min":{"1min":0.39500000000000002,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.0329999999999999,"5min":1.194,"15min":1.194},"last":0.92000000000000004}]},{"osd":4,"last update":"Thu Sep 12 23:35:21 2024","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":0.93300000000000005,"15min":0.92100000000000004},"min":{"1min":0.72899999999999998,"5min":0.57499999999999996,"15min":0.57499999999999996},"max":{"1min":1.5369999999999999,"5min":1.585,"15min":1.585},"last":0.88800000000000001},{"interface":"front","average":{"1min":0.98399999999999999,"5min":0.94899999999999995,"15min":0.93100000000000005},"min":{"1min":0.75600000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":1.2669999999999999,"5min":1.4279999999999999,"15min":1.4279999999999999},"last":0.97399999999999998}]},{"osd":5,"last update":"Thu Sep 12 23:36:10 2024","interfaces":[{"interface":"back","average":{"1min":0.88100000000000001,"5min":0.91600000000000004,"15min":0.91400000000000003},"min":{"1min":0.61099999999999999,"5min":0.52600000000000002,"15min":0.52600000000000002},"max":{"1min":1.127,"5min":1.6819999999999999,"15min":1.6819999999999999},"last":1.1279999999999999},{"interface":"front","average":{"1min":1.0129999999999999,"5min":1.0049999999999999,"15min":1.002},"min":{"1min":0.77900000000000003,"5min":0.63300000000000001,"15min":0.63300000000000001},"max":{"1min":1.2490000000000001,"5min":1.671,"15min":1.671},"last":1.0249999999999999}]},{"osd":6,"last update":"Thu Sep 12 23:36:02 2024","interfaces":[{"interface":"back","average":{"1min":0.96099999999999997,"5min":1.0149999999999999,"15min":1.04},"min":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":1.2909999999999999,"5min":1.5429999999999999,"15min":1.5429999999999999},"last":1.0740000000000001},{"interface":"front","average":{"1min":1.026,"5min":1.0600000000000001,"15min":1.087},"min":{"1min":0.66600000000000004,"5min":0.63800000000000001,"15min":0.63800000000000001},"max":{"1min":1.325,"5min":1.579,"15min":1.579},"last":0.95099999999999996}]},{"osd":7,"last update":"Thu Sep 12 23:35:51 2024","interfaces":[{"interface":"back","average":{"1min":1.032,"5min":1.0489999999999999,"15min":1.0469999999999999},"min":{"1min":0.63,"5min":0.61399999999999999,"15min":0.61399999999999999},"max":{"1min":1.4930000000000001,"5min":1.4930000000000001,"15min":1.4930000000000001},"last":1.002},{"interface":"front","average":{"1min":1.0329999999999999,"5min":1.052,"15min":1.0700000000000001},"min":{"1min":0.66900000000000004,"5min":0.59799999999999998,"15min":0.59799999999999998},"max":{"1min":1.226,"5min":1.3600000000000001,"15min":1.3600000000000001},"last":1.0509999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738483,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298588,"kb_used_data":1228,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446564,"statfs":{"total":95995035648,"available":95689281536,"internally_reserved":0,"allocated":1257472,"data_stored":882305,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Sep 12 23:35:52 2024","interfaces":[{"interface":"back","average":{"1min":0.65900000000000003,"5min":0.65000000000000002,"15min":0.54800000000000004},"min":{"1min":0.48099999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.0680000000000001,"5min":1.085,"15min":1.085},"last":1.0449999999999999},{"interface":"front","average":{"1min":0.71099999999999997,"5min":0.70399999999999996,"15min":0.57499999999999996},"min":{"1min":0.41899999999999998,"5min":0.216,"15min":0.216},"max":{"1min":1.087,"5min":1.087,"15min":1.087},"last":0.70599999999999996}]},{"osd":2,"last update":"Thu Sep 12 23:35:48 2024","interfaces":[{"interface":"back","average":{"1min":0.70299999999999996,"5min":0.70499999999999996,"15min":0.63900000000000001},"min":{"1min":0.45900000000000002,"5min":0.36699999999999999,"15min":0.27100000000000002},"max":{"1min":0.878,"5min":1.081,"15min":1.081},"last":0.76700000000000002},{"interface":"front","average":{"1min":0.81100000000000005,"5min":0.754,"15min":0.66800000000000004},"min":{"1min":0.52000000000000002,"5min":0.34300000000000003,"15min":0.317},"max":{"1min":1.206,"5min":1.206,"15min":1.206},"last":0.66700000000000004}]},{"osd":3,"last update":"Thu Sep 12 23:35:34 2024","interfaces":[{"interface":"back","average":{"1min":0.78600000000000003,"5min":0.75800000000000001,"15min":0.76000000000000001},"min":{"1min":0.54600000000000004,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":1.016,"5min":1.177,"15min":1.177},"last":0.90400000000000003},{"interface":"front","average":{"1min":0.85999999999999999,"5min":0.81499999999999995,"15min":0.79800000000000004},"min":{"1min":0.69299999999999995,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.165,"5min":1.3140000000000001,"15min":1.3140000000000001},"last":0.871}]},{"osd":4,"last update":"Thu Sep 12 23:35:27 2024","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":0.96899999999999997,"15min":0.91500000000000004},"min":{"1min":0.78800000000000003,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":1.2090000000000001,"5min":1.304,"15min":1.304},"last":0.98899999999999999},{"interface":"front","average":{"1min":0.98799999999999999,"5min":0.91800000000000004,"15min":0.86399999999999999},"min":{"1min":0.65200000000000002,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":1.2689999999999999,"5min":1.2689999999999999,"15min":1.2689999999999999},"last":1.0800000000000001}]},{"osd":5,"last update":"Thu Sep 12 23:36:17 2024","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":0.92600000000000005,"15min":0.93700000000000006},"min":{"1min":0.68799999999999994,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.242,"5min":1.3160000000000001,"15min":1.3160000000000001},"last":0.95099999999999996},{"interface":"front","average":{"1min":0.98799999999999999,"5min":0.98099999999999998,"15min":0.95899999999999996},"min":{"1min":0.71199999999999997,"5min":0.65500000000000003,"15min":0.65500000000000003},"max":{"1min":1.177,"5min":1.218,"15min":1.218},"last":1.2490000000000001}]},{"osd":6,"last update":"Thu Sep 12 23:35:58 2024","interfaces":[{"interface":"back","average":{"1min":1.0369999999999999,"5min":1.071,"15min":1.0980000000000001},"min":{"1min":0.77100000000000002,"5min":0.56000000000000005,"15min":0.56000000000000005},"max":{"1min":1.3660000000000001,"5min":1.492,"15min":1.492},"last":1.1439999999999999},{"interface":"front","average":{"1min":1.0049999999999999,"5min":1.0249999999999999,"15min":1.042},"min":{"1min":0.63200000000000001,"5min":0.621,"15min":0.621},"max":{"1min":1.351,"5min":1.4550000000000001,"15min":1.4550000000000001},"last":1.1870000000000001}]},{"osd":7,"last update":"Thu Sep 12 23:35:48 2024","interfaces":[{"interface":"back","average":{"1min":1.006,"5min":1.018,"15min":1.0169999999999999},"min":{"1min":0.72899999999999998,"5min":0.64500000000000002,"15min":0.64500000000000002},"max":{"1min":1.3200000000000001,"5min":1.403,"15min":1.403},"last":1.3520000000000001},{"interface":"front","average":{"1min":1.0880000000000001,"5min":1.111,"15min":1.131},"min":{"1min":0.72299999999999998,"5min":0.55000000000000004,"15min":0.55000000000000004},"max":{"1min":1.341,"5min":1.569,"15min":1.569},"last":1.2230000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":2728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":2714,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":2624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-12T23:36:25.441 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-12T23:36:25.441 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-12T23:36:25.441 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-12T23:36:25.441 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-12T23:36:26.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:26 smithi181 bash[24776]: audit 2024-09-12T23:36:24.731743+0000 mgr.y (mgr.24446) 104 : audit [DBG] from='client.14712 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-12T23:36:26.506 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:36:26 smithi136 bash[22852]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:36:26] "GET /metrics HTTP/1.1" 200 236634 "" "Prometheus/2.43.0" 2024-09-12T23:36:26.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:26 smithi136 bash[22601]: audit 2024-09-12T23:36:24.731743+0000 mgr.y (mgr.24446) 104 : audit [DBG] from='client.14712 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-12T23:36:26.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:26 smithi136 bash[32189]: audit 2024-09-12T23:36:24.731743+0000 mgr.y (mgr.24446) 104 : audit [DBG] from='client.14712 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-12T23:36:27.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:27 smithi181 bash[24776]: cluster 2024-09-12T23:36:25.969483+0000 mgr.y (mgr.24446) 105 : cluster [DBG] pgmap v81: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:27.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:27 smithi136 bash[32189]: cluster 2024-09-12T23:36:25.969483+0000 mgr.y (mgr.24446) 105 : cluster [DBG] pgmap v81: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:27.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:27 smithi136 bash[22601]: cluster 2024-09-12T23:36:25.969483+0000 mgr.y (mgr.24446) 105 : cluster [DBG] pgmap v81: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:29.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:29 smithi136 bash[22601]: cluster 2024-09-12T23:36:27.970215+0000 mgr.y (mgr.24446) 106 : cluster [DBG] pgmap v82: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:29.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:29 smithi136 bash[32189]: cluster 2024-09-12T23:36:27.970215+0000 mgr.y (mgr.24446) 106 : cluster [DBG] pgmap v82: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:29.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:29 smithi181 bash[24776]: cluster 2024-09-12T23:36:27.970215+0000 mgr.y (mgr.24446) 106 : cluster [DBG] pgmap v82: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:30.234 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:36:30.354 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:30 smithi136 bash[32189]: audit 2024-09-12T23:36:29.117009+0000 mon.a (mon.0) 886 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:36:30.354 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:30 smithi136 bash[32189]: audit 2024-09-12T23:36:29.600324+0000 mon.a (mon.0) 887 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:36:30.354 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:30 smithi136 bash[32189]: audit 2024-09-12T23:36:29.602043+0000 mon.a (mon.0) 888 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:36:30.354 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:30 smithi136 bash[32189]: audit 2024-09-12T23:36:29.609485+0000 mon.a (mon.0) 889 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:36:30.355 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:30 smithi136 bash[22601]: audit 2024-09-12T23:36:29.117009+0000 mon.a (mon.0) 886 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:36:30.355 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:30 smithi136 bash[22601]: audit 2024-09-12T23:36:29.600324+0000 mon.a (mon.0) 887 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:36:30.355 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:30 smithi136 bash[22601]: audit 2024-09-12T23:36:29.602043+0000 mon.a (mon.0) 888 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:36:30.355 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:30 smithi136 bash[22601]: audit 2024-09-12T23:36:29.609485+0000 mon.a (mon.0) 889 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:36:30.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:30 smithi181 bash[24776]: audit 2024-09-12T23:36:29.117009+0000 mon.a (mon.0) 886 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:36:30.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:30 smithi181 bash[24776]: audit 2024-09-12T23:36:29.600324+0000 mon.a (mon.0) 887 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:36:30.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:30 smithi181 bash[24776]: audit 2024-09-12T23:36:29.602043+0000 mon.a (mon.0) 888 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:36:30.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:30 smithi181 bash[24776]: audit 2024-09-12T23:36:29.609485+0000 mon.a (mon.0) 889 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:36:32.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:31 smithi136 bash[22601]: cluster 2024-09-12T23:36:29.971478+0000 mgr.y (mgr.24446) 107 : cluster [DBG] pgmap v83: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:32.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:31 smithi136 bash[32189]: cluster 2024-09-12T23:36:29.971478+0000 mgr.y (mgr.24446) 107 : cluster [DBG] pgmap v83: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:32.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:31 smithi181 bash[24776]: cluster 2024-09-12T23:36:29.971478+0000 mgr.y (mgr.24446) 107 : cluster [DBG] pgmap v83: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:33.340 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-12T23:36:33.340 INFO:teuthology.orchestra.run.smithi136.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-12T23:36:33.938 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:33 smithi136 bash[32189]: cluster 2024-09-12T23:36:31.972114+0000 mgr.y (mgr.24446) 108 : cluster [DBG] pgmap v84: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:33.939 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:33 smithi136 bash[32189]: audit 2024-09-12T23:36:33.343921+0000 mon.a (mon.0) 890 : audit [DBG] from='client.? 172.21.15.136:0/3229988769' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-12T23:36:33.939 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:33 smithi136 bash[22601]: cluster 2024-09-12T23:36:31.972114+0000 mgr.y (mgr.24446) 108 : cluster [DBG] pgmap v84: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:33.939 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:33 smithi136 bash[22601]: audit 2024-09-12T23:36:33.343921+0000 mon.a (mon.0) 890 : audit [DBG] from='client.? 172.21.15.136:0/3229988769' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-12T23:36:33.940 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-12T23:36:33.940 DEBUG:teuthology.orchestra.run.smithi136:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-12T23:36:34.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:33 smithi181 bash[24776]: cluster 2024-09-12T23:36:31.972114+0000 mgr.y (mgr.24446) 108 : cluster [DBG] pgmap v84: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:34.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:33 smithi181 bash[24776]: audit 2024-09-12T23:36:33.343921+0000 mon.a (mon.0) 890 : audit [DBG] from='client.? 172.21.15.136:0/3229988769' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-12T23:36:34.236 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-12T23:36:34.236 INFO:teuthology.orchestra.run.smithi136.stdout:{"epoch":64,"fsid":"da58a47c-715d-11ef-bceb-c7b262605968","created":"2024-09-12T23:23:29.000926+0000","modified":"2024-09-12T23:35:09.657947+0000","last_up_change":"2024-09-12T23:32:43.859472+0000","last_in_change":"2024-09-12T23:32:18.995528+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-12T23:28:33.956985+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-09-12T23:33:27.478434+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-09-12T23:33:28.566349+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"54","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}},{"pool":4,"pool_name":"rbd","create_time":"2024-09-12T23:34:48.404257+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"61","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":61,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"87aedf04-695e-4a8f-80c3-ffd18883a4d5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6802","nonce":4251571952},{"type":"v1","addr":"172.21.15.136:6803","nonce":4251571952}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6804","nonce":4251571952},{"type":"v1","addr":"172.21.15.136:6805","nonce":4251571952}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6808","nonce":4251571952},{"type":"v1","addr":"172.21.15.136:6809","nonce":4251571952}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6806","nonce":4251571952},{"type":"v1","addr":"172.21.15.136:6807","nonce":4251571952}]},"public_addr":"172.21.15.136:6803/4251571952","cluster_addr":"172.21.15.136:6805/4251571952","heartbeat_back_addr":"172.21.15.136:6809/4251571952","heartbeat_front_addr":"172.21.15.136:6807/4251571952","state":["exists","up"]},{"osd":1,"uuid":"850d4f98-0640-4151-bd45-a351ca41d54e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":63,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6810","nonce":3482177958},{"type":"v1","addr":"172.21.15.136:6811","nonce":3482177958}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6812","nonce":3482177958},{"type":"v1","addr":"172.21.15.136:6813","nonce":3482177958}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6816","nonce":3482177958},{"type":"v1","addr":"172.21.15.136:6817","nonce":3482177958}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6814","nonce":3482177958},{"type":"v1","addr":"172.21.15.136:6815","nonce":3482177958}]},"public_addr":"172.21.15.136:6811/3482177958","cluster_addr":"172.21.15.136:6813/3482177958","heartbeat_back_addr":"172.21.15.136:6817/3482177958","heartbeat_front_addr":"172.21.15.136:6815/3482177958","state":["exists","up"]},{"osd":2,"uuid":"bde707d6-e027-4b63-9853-fd59212eea1e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":63,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6818","nonce":308477304},{"type":"v1","addr":"172.21.15.136:6819","nonce":308477304}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6820","nonce":308477304},{"type":"v1","addr":"172.21.15.136:6821","nonce":308477304}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6824","nonce":308477304},{"type":"v1","addr":"172.21.15.136:6825","nonce":308477304}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6822","nonce":308477304},{"type":"v1","addr":"172.21.15.136:6823","nonce":308477304}]},"public_addr":"172.21.15.136:6819/308477304","cluster_addr":"172.21.15.136:6821/308477304","heartbeat_back_addr":"172.21.15.136:6825/308477304","heartbeat_front_addr":"172.21.15.136:6823/308477304","state":["exists","up"]},{"osd":3,"uuid":"d61f8357-b89b-4c05-b366-65a9f788064d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6826","nonce":201948269},{"type":"v1","addr":"172.21.15.136:6827","nonce":201948269}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6828","nonce":201948269},{"type":"v1","addr":"172.21.15.136:6829","nonce":201948269}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6832","nonce":201948269},{"type":"v1","addr":"172.21.15.136:6833","nonce":201948269}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6830","nonce":201948269},{"type":"v1","addr":"172.21.15.136:6831","nonce":201948269}]},"public_addr":"172.21.15.136:6827/201948269","cluster_addr":"172.21.15.136:6829/201948269","heartbeat_back_addr":"172.21.15.136:6833/201948269","heartbeat_front_addr":"172.21.15.136:6831/201948269","state":["exists","up"]},{"osd":4,"uuid":"98b0e81c-824e-4dbc-9faa-2de9f199af1b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":63,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6800","nonce":775251087},{"type":"v1","addr":"172.21.15.181:6801","nonce":775251087}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6802","nonce":775251087},{"type":"v1","addr":"172.21.15.181:6803","nonce":775251087}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6806","nonce":775251087},{"type":"v1","addr":"172.21.15.181:6807","nonce":775251087}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6804","nonce":775251087},{"type":"v1","addr":"172.21.15.181:6805","nonce":775251087}]},"public_addr":"172.21.15.181:6801/775251087","cluster_addr":"172.21.15.181:6803/775251087","heartbeat_back_addr":"172.21.15.181:6807/775251087","heartbeat_front_addr":"172.21.15.181:6805/775251087","state":["exists","up"]},{"osd":5,"uuid":"5c6d2f69-7f9c-4689-95b0-bd4eb858956d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6808","nonce":2109931558},{"type":"v1","addr":"172.21.15.181:6809","nonce":2109931558}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6810","nonce":2109931558},{"type":"v1","addr":"172.21.15.181:6811","nonce":2109931558}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6814","nonce":2109931558},{"type":"v1","addr":"172.21.15.181:6815","nonce":2109931558}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6812","nonce":2109931558},{"type":"v1","addr":"172.21.15.181:6813","nonce":2109931558}]},"public_addr":"172.21.15.181:6809/2109931558","cluster_addr":"172.21.15.181:6811/2109931558","heartbeat_back_addr":"172.21.15.181:6815/2109931558","heartbeat_front_addr":"172.21.15.181:6813/2109931558","state":["exists","up"]},{"osd":6,"uuid":"532ee052-af15-433a-8e93-b0b642aaf85b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":63,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6816","nonce":4099148413},{"type":"v1","addr":"172.21.15.181:6817","nonce":4099148413}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6818","nonce":4099148413},{"type":"v1","addr":"172.21.15.181:6819","nonce":4099148413}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6822","nonce":4099148413},{"type":"v1","addr":"172.21.15.181:6823","nonce":4099148413}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6820","nonce":4099148413},{"type":"v1","addr":"172.21.15.181:6821","nonce":4099148413}]},"public_addr":"172.21.15.181:6817/4099148413","cluster_addr":"172.21.15.181:6819/4099148413","heartbeat_back_addr":"172.21.15.181:6823/4099148413","heartbeat_front_addr":"172.21.15.181:6821/4099148413","state":["exists","up"]},{"osd":7,"uuid":"d2ff80f0-6ef0-4098-877c-a891d48c687d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":63,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6824","nonce":783620875},{"type":"v1","addr":"172.21.15.181:6825","nonce":783620875}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6826","nonce":783620875},{"type":"v1","addr":"172.21.15.181:6827","nonce":783620875}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6830","nonce":783620875},{"type":"v1","addr":"172.21.15.181:6831","nonce":783620875}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6828","nonce":783620875},{"type":"v1","addr":"172.21.15.181:6829","nonce":783620875}]},"public_addr":"172.21.15.181:6825/783620875","cluster_addr":"172.21.15.181:6827/783620875","heartbeat_back_addr":"172.21.15.181:6831/783620875","heartbeat_front_addr":"172.21.15.181:6829/783620875","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:26:50.657580+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:27:40.536476+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:28:31.755012+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:29:23.585451+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:30:11.888258+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:31:01.812765+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:31:52.166696+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:32:42.696525+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.b","mappings":[{"from":7,"to":2}]},{"pgid":"3.9","mappings":[{"from":4,"to":1}]},{"pgid":"3.10","mappings":[{"from":0,"to":1}]},{"pgid":"3.14","mappings":[{"from":4,"to":2}]},{"pgid":"3.30","mappings":[{"from":5,"to":2}]},{"pgid":"3.31","mappings":[{"from":4,"to":2}]},{"pgid":"3.34","mappings":[{"from":0,"to":3}]},{"pgid":"3.37","mappings":[{"from":5,"to":1}]}],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.136:0/129641967":"2024-09-13T23:34:07.892265+0000","172.21.15.136:6801/1810367175":"2024-09-13T23:34:07.892265+0000","172.21.15.136:6800/1810367175":"2024-09-13T23:34:07.892265+0000","172.21.15.136:6800/2894181186":"2024-09-13T23:23:54.847045+0000","172.21.15.136:0/1964283041":"2024-09-13T23:24:25.346823+0000","172.21.15.136:0/3924275900":"2024-09-13T23:34:07.892265+0000","172.21.15.136:6801/2894181186":"2024-09-13T23:23:54.847045+0000","172.21.15.136:0/513809002":"2024-09-13T23:34:07.892265+0000","172.21.15.136:0/3910468981":"2024-09-13T23:23:54.847045+0000","172.21.15.136:0/1549678324":"2024-09-13T23:23:54.847045+0000","172.21.15.136:6800/3594272622":"2024-09-13T23:24:25.346823+0000","172.21.15.136:0/209539205":"2024-09-13T23:24:25.346823+0000","172.21.15.136:6801/3594272622":"2024-09-13T23:24:25.346823+0000","172.21.15.136:0/4260962972":"2024-09-13T23:24:25.346823+0000","172.21.15.136:0/3376519841":"2024-09-13T23:23:54.847045+0000","172.21.15.136:0/3642432358":"2024-09-13T23:34:07.892265+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-12T23:36:34.255 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-09-12T23:28:33.956985+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-09-12T23:33:27.478434+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '53', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-09-12T23:33:28.566349+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '54', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}}, {'pool': 4, 'pool_name': 'rbd', 'create_time': '2024-09-12T23:34:48.404257+0000', 'flags': 8193, 'flags_names': 'hashpspool,selfmanaged_snaps', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 8, 'pg_placement_num': 8, 'pg_placement_num_target': 8, 'pg_num_target': 8, 'pg_num_pending': 8, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '61', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 3, 'snap_epoch': 61, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {}, 'application_metadata': {'rbd': {}}}] 2024-09-12T23:36:34.256 DEBUG:teuthology.orchestra.run.smithi136:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-09-12T23:36:34.526 INFO:teuthology.orchestra.run.smithi136.stdout:pg_num: 1 2024-09-12T23:36:34.544 DEBUG:teuthology.orchestra.run.smithi136:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-09-12T23:36:34.810 INFO:teuthology.orchestra.run.smithi136.stdout:pg_num: 32 2024-09-12T23:36:34.827 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:34 smithi136 bash[32189]: audit 2024-09-12T23:36:34.239241+0000 mon.c (mon.2) 37 : audit [DBG] from='client.? 172.21.15.136:0/2795341561' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-12T23:36:34.827 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:34 smithi136 bash[32189]: audit 2024-09-12T23:36:34.530079+0000 mon.a (mon.0) 891 : audit [DBG] from='client.? 172.21.15.136:0/1107625358' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-12T23:36:34.827 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:34 smithi136 bash[22601]: audit 2024-09-12T23:36:34.239241+0000 mon.c (mon.2) 37 : audit [DBG] from='client.? 172.21.15.136:0/2795341561' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-12T23:36:34.827 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:34 smithi136 bash[22601]: audit 2024-09-12T23:36:34.530079+0000 mon.a (mon.0) 891 : audit [DBG] from='client.? 172.21.15.136:0/1107625358' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-12T23:36:34.828 DEBUG:teuthology.orchestra.run.smithi136:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-09-12T23:36:35.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:34 smithi181 bash[24776]: audit 2024-09-12T23:36:34.239241+0000 mon.c (mon.2) 37 : audit [DBG] from='client.? 172.21.15.136:0/2795341561' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-12T23:36:35.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:34 smithi181 bash[24776]: audit 2024-09-12T23:36:34.530079+0000 mon.a (mon.0) 891 : audit [DBG] from='client.? 172.21.15.136:0/1107625358' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-12T23:36:35.107 INFO:teuthology.orchestra.run.smithi136.stdout:pg_num: 64 2024-09-12T23:36:35.125 DEBUG:teuthology.orchestra.run.smithi136:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get rbd pg_num 2024-09-12T23:36:35.402 INFO:teuthology.orchestra.run.smithi136.stdout:pg_num: 8 2024-09-12T23:36:35.419 DEBUG:teuthology.orchestra.run.smithi136:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-09-12T23:36:35.595 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:36:35 smithi181 bash[25717]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:36:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-12T23:36:35.720 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-12T23:36:35.720 INFO:teuthology.orchestra.run.smithi136.stdout:{"epoch":7,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14391,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.136:6835/3940972569","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6834","nonce":3940972569},{"type":"v1","addr":"172.21.15.136:6835","nonce":3940972569}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":7,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-09-12T23:33:29.192396+0000","modified":"2024-09-12T23:33:32.368306+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":1,"in":[0],"up":{"mds_0":24316},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24316":{"gid":24316,"name":"b","rank":0,"incarnation":4,"state":"up:active","state_seq":3,"addr":"172.21.15.181:6833/2077070284","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6832","nonce":2077070284},{"type":"v1","addr":"172.21.15.181:6833","nonce":2077070284}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","standby_count_wanted":0},"id":1}]} 2024-09-12T23:36:35.720 INFO:teuthology.orchestra.run.smithi136.stderr:dumped fsmap epoch 7 2024-09-12T23:36:35.736 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:35 smithi136 bash[22601]: cluster 2024-09-12T23:36:33.972946+0000 mgr.y (mgr.24446) 109 : cluster [DBG] pgmap v85: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:35.736 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:35 smithi136 bash[22601]: audit 2024-09-12T23:36:34.814957+0000 mon.a (mon.0) 892 : audit [DBG] from='client.? 172.21.15.136:0/176749058' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-09-12T23:36:35.736 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:35 smithi136 bash[22601]: audit 2024-09-12T23:36:35.112087+0000 mon.c (mon.2) 38 : audit [DBG] from='client.? 172.21.15.136:0/1370371717' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-09-12T23:36:35.736 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:35 smithi136 bash[22601]: audit 2024-09-12T23:36:35.406340+0000 mon.c (mon.2) 39 : audit [DBG] from='client.? 172.21.15.136:0/1196597592' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "rbd", "var": "pg_num"}]: dispatch 2024-09-12T23:36:35.737 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:35 smithi136 bash[32189]: cluster 2024-09-12T23:36:33.972946+0000 mgr.y (mgr.24446) 109 : cluster [DBG] pgmap v85: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:35.737 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:35 smithi136 bash[32189]: audit 2024-09-12T23:36:34.814957+0000 mon.a (mon.0) 892 : audit [DBG] from='client.? 172.21.15.136:0/176749058' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-09-12T23:36:35.737 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:35 smithi136 bash[32189]: audit 2024-09-12T23:36:35.112087+0000 mon.c (mon.2) 38 : audit [DBG] from='client.? 172.21.15.136:0/1370371717' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-09-12T23:36:35.737 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:35 smithi136 bash[32189]: audit 2024-09-12T23:36:35.406340+0000 mon.c (mon.2) 39 : audit [DBG] from='client.? 172.21.15.136:0/1196597592' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "rbd", "var": "pg_num"}]: dispatch 2024-09-12T23:36:35.738 DEBUG:teuthology.orchestra.run.smithi136:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-12T23:36:36.004 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-12T23:36:36.005 INFO:teuthology.orchestra.run.smithi136.stdout:{"epoch":64,"fsid":"da58a47c-715d-11ef-bceb-c7b262605968","created":"2024-09-12T23:23:29.000926+0000","modified":"2024-09-12T23:35:09.657947+0000","last_up_change":"2024-09-12T23:32:43.859472+0000","last_in_change":"2024-09-12T23:32:18.995528+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-12T23:28:33.956985+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-09-12T23:33:27.478434+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-09-12T23:33:28.566349+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"54","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}},{"pool":4,"pool_name":"rbd","create_time":"2024-09-12T23:34:48.404257+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"61","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":61,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"87aedf04-695e-4a8f-80c3-ffd18883a4d5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6802","nonce":4251571952},{"type":"v1","addr":"172.21.15.136:6803","nonce":4251571952}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6804","nonce":4251571952},{"type":"v1","addr":"172.21.15.136:6805","nonce":4251571952}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6808","nonce":4251571952},{"type":"v1","addr":"172.21.15.136:6809","nonce":4251571952}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6806","nonce":4251571952},{"type":"v1","addr":"172.21.15.136:6807","nonce":4251571952}]},"public_addr":"172.21.15.136:6803/4251571952","cluster_addr":"172.21.15.136:6805/4251571952","heartbeat_back_addr":"172.21.15.136:6809/4251571952","heartbeat_front_addr":"172.21.15.136:6807/4251571952","state":["exists","up"]},{"osd":1,"uuid":"850d4f98-0640-4151-bd45-a351ca41d54e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":63,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6810","nonce":3482177958},{"type":"v1","addr":"172.21.15.136:6811","nonce":3482177958}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6812","nonce":3482177958},{"type":"v1","addr":"172.21.15.136:6813","nonce":3482177958}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6816","nonce":3482177958},{"type":"v1","addr":"172.21.15.136:6817","nonce":3482177958}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6814","nonce":3482177958},{"type":"v1","addr":"172.21.15.136:6815","nonce":3482177958}]},"public_addr":"172.21.15.136:6811/3482177958","cluster_addr":"172.21.15.136:6813/3482177958","heartbeat_back_addr":"172.21.15.136:6817/3482177958","heartbeat_front_addr":"172.21.15.136:6815/3482177958","state":["exists","up"]},{"osd":2,"uuid":"bde707d6-e027-4b63-9853-fd59212eea1e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":63,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6818","nonce":308477304},{"type":"v1","addr":"172.21.15.136:6819","nonce":308477304}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6820","nonce":308477304},{"type":"v1","addr":"172.21.15.136:6821","nonce":308477304}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6824","nonce":308477304},{"type":"v1","addr":"172.21.15.136:6825","nonce":308477304}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6822","nonce":308477304},{"type":"v1","addr":"172.21.15.136:6823","nonce":308477304}]},"public_addr":"172.21.15.136:6819/308477304","cluster_addr":"172.21.15.136:6821/308477304","heartbeat_back_addr":"172.21.15.136:6825/308477304","heartbeat_front_addr":"172.21.15.136:6823/308477304","state":["exists","up"]},{"osd":3,"uuid":"d61f8357-b89b-4c05-b366-65a9f788064d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6826","nonce":201948269},{"type":"v1","addr":"172.21.15.136:6827","nonce":201948269}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6828","nonce":201948269},{"type":"v1","addr":"172.21.15.136:6829","nonce":201948269}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6832","nonce":201948269},{"type":"v1","addr":"172.21.15.136:6833","nonce":201948269}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6830","nonce":201948269},{"type":"v1","addr":"172.21.15.136:6831","nonce":201948269}]},"public_addr":"172.21.15.136:6827/201948269","cluster_addr":"172.21.15.136:6829/201948269","heartbeat_back_addr":"172.21.15.136:6833/201948269","heartbeat_front_addr":"172.21.15.136:6831/201948269","state":["exists","up"]},{"osd":4,"uuid":"98b0e81c-824e-4dbc-9faa-2de9f199af1b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":63,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6800","nonce":775251087},{"type":"v1","addr":"172.21.15.181:6801","nonce":775251087}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6802","nonce":775251087},{"type":"v1","addr":"172.21.15.181:6803","nonce":775251087}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6806","nonce":775251087},{"type":"v1","addr":"172.21.15.181:6807","nonce":775251087}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6804","nonce":775251087},{"type":"v1","addr":"172.21.15.181:6805","nonce":775251087}]},"public_addr":"172.21.15.181:6801/775251087","cluster_addr":"172.21.15.181:6803/775251087","heartbeat_back_addr":"172.21.15.181:6807/775251087","heartbeat_front_addr":"172.21.15.181:6805/775251087","state":["exists","up"]},{"osd":5,"uuid":"5c6d2f69-7f9c-4689-95b0-bd4eb858956d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6808","nonce":2109931558},{"type":"v1","addr":"172.21.15.181:6809","nonce":2109931558}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6810","nonce":2109931558},{"type":"v1","addr":"172.21.15.181:6811","nonce":2109931558}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6814","nonce":2109931558},{"type":"v1","addr":"172.21.15.181:6815","nonce":2109931558}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6812","nonce":2109931558},{"type":"v1","addr":"172.21.15.181:6813","nonce":2109931558}]},"public_addr":"172.21.15.181:6809/2109931558","cluster_addr":"172.21.15.181:6811/2109931558","heartbeat_back_addr":"172.21.15.181:6815/2109931558","heartbeat_front_addr":"172.21.15.181:6813/2109931558","state":["exists","up"]},{"osd":6,"uuid":"532ee052-af15-433a-8e93-b0b642aaf85b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":63,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6816","nonce":4099148413},{"type":"v1","addr":"172.21.15.181:6817","nonce":4099148413}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6818","nonce":4099148413},{"type":"v1","addr":"172.21.15.181:6819","nonce":4099148413}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6822","nonce":4099148413},{"type":"v1","addr":"172.21.15.181:6823","nonce":4099148413}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6820","nonce":4099148413},{"type":"v1","addr":"172.21.15.181:6821","nonce":4099148413}]},"public_addr":"172.21.15.181:6817/4099148413","cluster_addr":"172.21.15.181:6819/4099148413","heartbeat_back_addr":"172.21.15.181:6823/4099148413","heartbeat_front_addr":"172.21.15.181:6821/4099148413","state":["exists","up"]},{"osd":7,"uuid":"d2ff80f0-6ef0-4098-877c-a891d48c687d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":63,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6824","nonce":783620875},{"type":"v1","addr":"172.21.15.181:6825","nonce":783620875}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6826","nonce":783620875},{"type":"v1","addr":"172.21.15.181:6827","nonce":783620875}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6830","nonce":783620875},{"type":"v1","addr":"172.21.15.181:6831","nonce":783620875}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6828","nonce":783620875},{"type":"v1","addr":"172.21.15.181:6829","nonce":783620875}]},"public_addr":"172.21.15.181:6825/783620875","cluster_addr":"172.21.15.181:6827/783620875","heartbeat_back_addr":"172.21.15.181:6831/783620875","heartbeat_front_addr":"172.21.15.181:6829/783620875","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:26:50.657580+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:27:40.536476+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:28:31.755012+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:29:23.585451+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:30:11.888258+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:31:01.812765+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:31:52.166696+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:32:42.696525+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.b","mappings":[{"from":7,"to":2}]},{"pgid":"3.9","mappings":[{"from":4,"to":1}]},{"pgid":"3.10","mappings":[{"from":0,"to":1}]},{"pgid":"3.14","mappings":[{"from":4,"to":2}]},{"pgid":"3.30","mappings":[{"from":5,"to":2}]},{"pgid":"3.31","mappings":[{"from":4,"to":2}]},{"pgid":"3.34","mappings":[{"from":0,"to":3}]},{"pgid":"3.37","mappings":[{"from":5,"to":1}]}],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.136:0/129641967":"2024-09-13T23:34:07.892265+0000","172.21.15.136:6801/1810367175":"2024-09-13T23:34:07.892265+0000","172.21.15.136:6800/1810367175":"2024-09-13T23:34:07.892265+0000","172.21.15.136:6800/2894181186":"2024-09-13T23:23:54.847045+0000","172.21.15.136:0/1964283041":"2024-09-13T23:24:25.346823+0000","172.21.15.136:0/3924275900":"2024-09-13T23:34:07.892265+0000","172.21.15.136:6801/2894181186":"2024-09-13T23:23:54.847045+0000","172.21.15.136:0/513809002":"2024-09-13T23:34:07.892265+0000","172.21.15.136:0/3910468981":"2024-09-13T23:23:54.847045+0000","172.21.15.136:0/1549678324":"2024-09-13T23:23:54.847045+0000","172.21.15.136:6800/3594272622":"2024-09-13T23:24:25.346823+0000","172.21.15.136:0/209539205":"2024-09-13T23:24:25.346823+0000","172.21.15.136:6801/3594272622":"2024-09-13T23:24:25.346823+0000","172.21.15.136:0/4260962972":"2024-09-13T23:24:25.346823+0000","172.21.15.136:0/3376519841":"2024-09-13T23:23:54.847045+0000","172.21.15.136:0/3642432358":"2024-09-13T23:34:07.892265+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-12T23:36:36.022 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-09-12T23:28:33.956985+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-09-12T23:33:27.478434+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '53', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-09-12T23:33:28.566349+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '54', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}}, {'pool': 4, 'pool_name': 'rbd', 'create_time': '2024-09-12T23:34:48.404257+0000', 'flags': 8193, 'flags_names': 'hashpspool,selfmanaged_snaps', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 8, 'pg_placement_num': 8, 'pg_placement_num_target': 8, 'pg_num_target': 8, 'pg_num_pending': 8, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '61', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 3, 'snap_epoch': 61, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {}, 'application_metadata': {'rbd': {}}}] 2024-09-12T23:36:36.023 DEBUG:teuthology.orchestra.run.smithi136:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-09-12T23:36:36.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:35 smithi181 bash[24776]: cluster 2024-09-12T23:36:33.972946+0000 mgr.y (mgr.24446) 109 : cluster [DBG] pgmap v85: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:36.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:35 smithi181 bash[24776]: audit 2024-09-12T23:36:34.814957+0000 mon.a (mon.0) 892 : audit [DBG] from='client.? 172.21.15.136:0/176749058' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-09-12T23:36:36.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:35 smithi181 bash[24776]: audit 2024-09-12T23:36:35.112087+0000 mon.c (mon.2) 38 : audit [DBG] from='client.? 172.21.15.136:0/1370371717' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-09-12T23:36:36.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:35 smithi181 bash[24776]: audit 2024-09-12T23:36:35.406340+0000 mon.c (mon.2) 39 : audit [DBG] from='client.? 172.21.15.136:0/1196597592' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "rbd", "var": "pg_num"}]: dispatch 2024-09-12T23:36:36.298 INFO:teuthology.orchestra.run.smithi136.stdout:pg_num: 1 2024-09-12T23:36:36.314 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:36:36 smithi136 bash[22852]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:36:36] "GET /metrics HTTP/1.1" 200 236634 "" "Prometheus/2.43.0" 2024-09-12T23:36:36.315 DEBUG:teuthology.orchestra.run.smithi136:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-09-12T23:36:36.581 INFO:teuthology.orchestra.run.smithi136.stdout:pg_num: 32 2024-09-12T23:36:36.599 DEBUG:teuthology.orchestra.run.smithi136:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-09-12T23:36:36.870 INFO:teuthology.orchestra.run.smithi136.stdout:pg_num: 64 2024-09-12T23:36:36.887 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:36 smithi136 bash[32189]: audit 2024-09-12T23:36:35.723789+0000 mon.c (mon.2) 40 : audit [DBG] from='client.? 172.21.15.136:0/929043933' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-12T23:36:36.887 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:36 smithi136 bash[32189]: audit 2024-09-12T23:36:36.007222+0000 mon.c (mon.2) 41 : audit [DBG] from='client.? 172.21.15.136:0/2307562835' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-12T23:36:36.887 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:36 smithi136 bash[32189]: audit 2024-09-12T23:36:36.302376+0000 mon.c (mon.2) 42 : audit [DBG] from='client.? 172.21.15.136:0/989253405' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-12T23:36:36.887 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:36 smithi136 bash[32189]: audit 2024-09-12T23:36:36.586058+0000 mon.c (mon.2) 43 : audit [DBG] from='client.? 172.21.15.136:0/1378793972' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-09-12T23:36:36.887 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:36 smithi136 bash[22601]: audit 2024-09-12T23:36:35.723789+0000 mon.c (mon.2) 40 : audit [DBG] from='client.? 172.21.15.136:0/929043933' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-12T23:36:36.888 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:36 smithi136 bash[22601]: audit 2024-09-12T23:36:36.007222+0000 mon.c (mon.2) 41 : audit [DBG] from='client.? 172.21.15.136:0/2307562835' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-12T23:36:36.888 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:36 smithi136 bash[22601]: audit 2024-09-12T23:36:36.302376+0000 mon.c (mon.2) 42 : audit [DBG] from='client.? 172.21.15.136:0/989253405' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-12T23:36:36.888 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:36 smithi136 bash[22601]: audit 2024-09-12T23:36:36.586058+0000 mon.c (mon.2) 43 : audit [DBG] from='client.? 172.21.15.136:0/1378793972' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-09-12T23:36:36.889 DEBUG:teuthology.orchestra.run.smithi136:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get rbd pg_num 2024-09-12T23:36:37.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:36 smithi181 bash[24776]: audit 2024-09-12T23:36:35.723789+0000 mon.c (mon.2) 40 : audit [DBG] from='client.? 172.21.15.136:0/929043933' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-12T23:36:37.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:36 smithi181 bash[24776]: audit 2024-09-12T23:36:36.007222+0000 mon.c (mon.2) 41 : audit [DBG] from='client.? 172.21.15.136:0/2307562835' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-12T23:36:37.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:36 smithi181 bash[24776]: audit 2024-09-12T23:36:36.302376+0000 mon.c (mon.2) 42 : audit [DBG] from='client.? 172.21.15.136:0/989253405' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-12T23:36:37.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:36 smithi181 bash[24776]: audit 2024-09-12T23:36:36.586058+0000 mon.c (mon.2) 43 : audit [DBG] from='client.? 172.21.15.136:0/1378793972' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-09-12T23:36:37.168 INFO:teuthology.orchestra.run.smithi136.stdout:pg_num: 8 2024-09-12T23:36:37.186 DEBUG:teuthology.orchestra.run.smithi136:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-09-12T23:36:37.504 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-12T23:36:37.504 INFO:teuthology.orchestra.run.smithi136.stdout:{"epoch":7,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14391,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.136:6835/3940972569","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6834","nonce":3940972569},{"type":"v1","addr":"172.21.15.136:6835","nonce":3940972569}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":7,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-09-12T23:33:29.192396+0000","modified":"2024-09-12T23:33:32.368306+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":1,"in":[0],"up":{"mds_0":24316},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24316":{"gid":24316,"name":"b","rank":0,"incarnation":4,"state":"up:active","state_seq":3,"addr":"172.21.15.181:6833/2077070284","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6832","nonce":2077070284},{"type":"v1","addr":"172.21.15.181:6833","nonce":2077070284}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","standby_count_wanted":0},"id":1}]} 2024-09-12T23:36:37.504 INFO:teuthology.orchestra.run.smithi136.stderr:dumped fsmap epoch 7 2024-09-12T23:36:37.523 DEBUG:teuthology.orchestra.run.smithi136:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-12T23:36:37.802 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-12T23:36:37.802 INFO:teuthology.orchestra.run.smithi136.stdout:{"epoch":64,"fsid":"da58a47c-715d-11ef-bceb-c7b262605968","created":"2024-09-12T23:23:29.000926+0000","modified":"2024-09-12T23:35:09.657947+0000","last_up_change":"2024-09-12T23:32:43.859472+0000","last_in_change":"2024-09-12T23:32:18.995528+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-12T23:28:33.956985+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-09-12T23:33:27.478434+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-09-12T23:33:28.566349+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"54","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}},{"pool":4,"pool_name":"rbd","create_time":"2024-09-12T23:34:48.404257+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"61","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":61,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"87aedf04-695e-4a8f-80c3-ffd18883a4d5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6802","nonce":4251571952},{"type":"v1","addr":"172.21.15.136:6803","nonce":4251571952}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6804","nonce":4251571952},{"type":"v1","addr":"172.21.15.136:6805","nonce":4251571952}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6808","nonce":4251571952},{"type":"v1","addr":"172.21.15.136:6809","nonce":4251571952}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6806","nonce":4251571952},{"type":"v1","addr":"172.21.15.136:6807","nonce":4251571952}]},"public_addr":"172.21.15.136:6803/4251571952","cluster_addr":"172.21.15.136:6805/4251571952","heartbeat_back_addr":"172.21.15.136:6809/4251571952","heartbeat_front_addr":"172.21.15.136:6807/4251571952","state":["exists","up"]},{"osd":1,"uuid":"850d4f98-0640-4151-bd45-a351ca41d54e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":63,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6810","nonce":3482177958},{"type":"v1","addr":"172.21.15.136:6811","nonce":3482177958}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6812","nonce":3482177958},{"type":"v1","addr":"172.21.15.136:6813","nonce":3482177958}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6816","nonce":3482177958},{"type":"v1","addr":"172.21.15.136:6817","nonce":3482177958}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6814","nonce":3482177958},{"type":"v1","addr":"172.21.15.136:6815","nonce":3482177958}]},"public_addr":"172.21.15.136:6811/3482177958","cluster_addr":"172.21.15.136:6813/3482177958","heartbeat_back_addr":"172.21.15.136:6817/3482177958","heartbeat_front_addr":"172.21.15.136:6815/3482177958","state":["exists","up"]},{"osd":2,"uuid":"bde707d6-e027-4b63-9853-fd59212eea1e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":63,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6818","nonce":308477304},{"type":"v1","addr":"172.21.15.136:6819","nonce":308477304}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6820","nonce":308477304},{"type":"v1","addr":"172.21.15.136:6821","nonce":308477304}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6824","nonce":308477304},{"type":"v1","addr":"172.21.15.136:6825","nonce":308477304}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6822","nonce":308477304},{"type":"v1","addr":"172.21.15.136:6823","nonce":308477304}]},"public_addr":"172.21.15.136:6819/308477304","cluster_addr":"172.21.15.136:6821/308477304","heartbeat_back_addr":"172.21.15.136:6825/308477304","heartbeat_front_addr":"172.21.15.136:6823/308477304","state":["exists","up"]},{"osd":3,"uuid":"d61f8357-b89b-4c05-b366-65a9f788064d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6826","nonce":201948269},{"type":"v1","addr":"172.21.15.136:6827","nonce":201948269}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6828","nonce":201948269},{"type":"v1","addr":"172.21.15.136:6829","nonce":201948269}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6832","nonce":201948269},{"type":"v1","addr":"172.21.15.136:6833","nonce":201948269}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6830","nonce":201948269},{"type":"v1","addr":"172.21.15.136:6831","nonce":201948269}]},"public_addr":"172.21.15.136:6827/201948269","cluster_addr":"172.21.15.136:6829/201948269","heartbeat_back_addr":"172.21.15.136:6833/201948269","heartbeat_front_addr":"172.21.15.136:6831/201948269","state":["exists","up"]},{"osd":4,"uuid":"98b0e81c-824e-4dbc-9faa-2de9f199af1b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":63,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6800","nonce":775251087},{"type":"v1","addr":"172.21.15.181:6801","nonce":775251087}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6802","nonce":775251087},{"type":"v1","addr":"172.21.15.181:6803","nonce":775251087}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6806","nonce":775251087},{"type":"v1","addr":"172.21.15.181:6807","nonce":775251087}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6804","nonce":775251087},{"type":"v1","addr":"172.21.15.181:6805","nonce":775251087}]},"public_addr":"172.21.15.181:6801/775251087","cluster_addr":"172.21.15.181:6803/775251087","heartbeat_back_addr":"172.21.15.181:6807/775251087","heartbeat_front_addr":"172.21.15.181:6805/775251087","state":["exists","up"]},{"osd":5,"uuid":"5c6d2f69-7f9c-4689-95b0-bd4eb858956d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6808","nonce":2109931558},{"type":"v1","addr":"172.21.15.181:6809","nonce":2109931558}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6810","nonce":2109931558},{"type":"v1","addr":"172.21.15.181:6811","nonce":2109931558}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6814","nonce":2109931558},{"type":"v1","addr":"172.21.15.181:6815","nonce":2109931558}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6812","nonce":2109931558},{"type":"v1","addr":"172.21.15.181:6813","nonce":2109931558}]},"public_addr":"172.21.15.181:6809/2109931558","cluster_addr":"172.21.15.181:6811/2109931558","heartbeat_back_addr":"172.21.15.181:6815/2109931558","heartbeat_front_addr":"172.21.15.181:6813/2109931558","state":["exists","up"]},{"osd":6,"uuid":"532ee052-af15-433a-8e93-b0b642aaf85b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":63,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6816","nonce":4099148413},{"type":"v1","addr":"172.21.15.181:6817","nonce":4099148413}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6818","nonce":4099148413},{"type":"v1","addr":"172.21.15.181:6819","nonce":4099148413}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6822","nonce":4099148413},{"type":"v1","addr":"172.21.15.181:6823","nonce":4099148413}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6820","nonce":4099148413},{"type":"v1","addr":"172.21.15.181:6821","nonce":4099148413}]},"public_addr":"172.21.15.181:6817/4099148413","cluster_addr":"172.21.15.181:6819/4099148413","heartbeat_back_addr":"172.21.15.181:6823/4099148413","heartbeat_front_addr":"172.21.15.181:6821/4099148413","state":["exists","up"]},{"osd":7,"uuid":"d2ff80f0-6ef0-4098-877c-a891d48c687d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":63,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6824","nonce":783620875},{"type":"v1","addr":"172.21.15.181:6825","nonce":783620875}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6826","nonce":783620875},{"type":"v1","addr":"172.21.15.181:6827","nonce":783620875}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6830","nonce":783620875},{"type":"v1","addr":"172.21.15.181:6831","nonce":783620875}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6828","nonce":783620875},{"type":"v1","addr":"172.21.15.181:6829","nonce":783620875}]},"public_addr":"172.21.15.181:6825/783620875","cluster_addr":"172.21.15.181:6827/783620875","heartbeat_back_addr":"172.21.15.181:6831/783620875","heartbeat_front_addr":"172.21.15.181:6829/783620875","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:26:50.657580+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:27:40.536476+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:28:31.755012+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:29:23.585451+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:30:11.888258+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:31:01.812765+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:31:52.166696+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:32:42.696525+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.b","mappings":[{"from":7,"to":2}]},{"pgid":"3.9","mappings":[{"from":4,"to":1}]},{"pgid":"3.10","mappings":[{"from":0,"to":1}]},{"pgid":"3.14","mappings":[{"from":4,"to":2}]},{"pgid":"3.30","mappings":[{"from":5,"to":2}]},{"pgid":"3.31","mappings":[{"from":4,"to":2}]},{"pgid":"3.34","mappings":[{"from":0,"to":3}]},{"pgid":"3.37","mappings":[{"from":5,"to":1}]}],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.136:0/129641967":"2024-09-13T23:34:07.892265+0000","172.21.15.136:6801/1810367175":"2024-09-13T23:34:07.892265+0000","172.21.15.136:6800/1810367175":"2024-09-13T23:34:07.892265+0000","172.21.15.136:6800/2894181186":"2024-09-13T23:23:54.847045+0000","172.21.15.136:0/1964283041":"2024-09-13T23:24:25.346823+0000","172.21.15.136:0/3924275900":"2024-09-13T23:34:07.892265+0000","172.21.15.136:6801/2894181186":"2024-09-13T23:23:54.847045+0000","172.21.15.136:0/513809002":"2024-09-13T23:34:07.892265+0000","172.21.15.136:0/3910468981":"2024-09-13T23:23:54.847045+0000","172.21.15.136:0/1549678324":"2024-09-13T23:23:54.847045+0000","172.21.15.136:6800/3594272622":"2024-09-13T23:24:25.346823+0000","172.21.15.136:0/209539205":"2024-09-13T23:24:25.346823+0000","172.21.15.136:6801/3594272622":"2024-09-13T23:24:25.346823+0000","172.21.15.136:0/4260962972":"2024-09-13T23:24:25.346823+0000","172.21.15.136:0/3376519841":"2024-09-13T23:23:54.847045+0000","172.21.15.136:0/3642432358":"2024-09-13T23:34:07.892265+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-12T23:36:37.818 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:37 smithi136 bash[22601]: cluster 2024-09-12T23:36:35.973516+0000 mgr.y (mgr.24446) 110 : cluster [DBG] pgmap v86: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:37.818 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:37 smithi136 bash[22601]: audit 2024-09-12T23:36:36.874513+0000 mon.c (mon.2) 44 : audit [DBG] from='client.? 172.21.15.136:0/331727834' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-09-12T23:36:37.818 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:37 smithi136 bash[22601]: audit 2024-09-12T23:36:37.172459+0000 mon.a (mon.0) 893 : audit [DBG] from='client.? 172.21.15.136:0/4182608756' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "rbd", "var": "pg_num"}]: dispatch 2024-09-12T23:36:37.818 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:37 smithi136 bash[22601]: audit 2024-09-12T23:36:37.508078+0000 mon.c (mon.2) 45 : audit [DBG] from='client.? 172.21.15.136:0/3444891921' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-12T23:36:37.819 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:37 smithi136 bash[32189]: cluster 2024-09-12T23:36:35.973516+0000 mgr.y (mgr.24446) 110 : cluster [DBG] pgmap v86: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:37.819 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:37 smithi136 bash[32189]: audit 2024-09-12T23:36:36.874513+0000 mon.c (mon.2) 44 : audit [DBG] from='client.? 172.21.15.136:0/331727834' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-09-12T23:36:37.819 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:37 smithi136 bash[32189]: audit 2024-09-12T23:36:37.172459+0000 mon.a (mon.0) 893 : audit [DBG] from='client.? 172.21.15.136:0/4182608756' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "rbd", "var": "pg_num"}]: dispatch 2024-09-12T23:36:37.819 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:37 smithi136 bash[32189]: audit 2024-09-12T23:36:37.508078+0000 mon.c (mon.2) 45 : audit [DBG] from='client.? 172.21.15.136:0/3444891921' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-12T23:36:37.820 DEBUG:teuthology.orchestra.run.smithi136:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-09-12T23:36:38.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:37 smithi181 bash[24776]: cluster 2024-09-12T23:36:35.973516+0000 mgr.y (mgr.24446) 110 : cluster [DBG] pgmap v86: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:38.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:37 smithi181 bash[24776]: audit 2024-09-12T23:36:36.874513+0000 mon.c (mon.2) 44 : audit [DBG] from='client.? 172.21.15.136:0/331727834' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-09-12T23:36:38.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:37 smithi181 bash[24776]: audit 2024-09-12T23:36:37.172459+0000 mon.a (mon.0) 893 : audit [DBG] from='client.? 172.21.15.136:0/4182608756' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "rbd", "var": "pg_num"}]: dispatch 2024-09-12T23:36:38.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:37 smithi181 bash[24776]: audit 2024-09-12T23:36:37.508078+0000 mon.c (mon.2) 45 : audit [DBG] from='client.? 172.21.15.136:0/3444891921' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-12T23:36:38.154 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-12T23:36:38.154 INFO:teuthology.orchestra.run.smithi136.stdout:{"epoch":7,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14391,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.136:6835/3940972569","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6834","nonce":3940972569},{"type":"v1","addr":"172.21.15.136:6835","nonce":3940972569}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":7,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-09-12T23:33:29.192396+0000","modified":"2024-09-12T23:33:32.368306+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":1,"in":[0],"up":{"mds_0":24316},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24316":{"gid":24316,"name":"b","rank":0,"incarnation":4,"state":"up:active","state_seq":3,"addr":"172.21.15.181:6833/2077070284","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6832","nonce":2077070284},{"type":"v1","addr":"172.21.15.181:6833","nonce":2077070284}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","standby_count_wanted":0},"id":1}]} 2024-09-12T23:36:38.154 INFO:teuthology.orchestra.run.smithi136.stderr:dumped fsmap epoch 7 2024-09-12T23:36:38.172 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 7, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-09-12T23:33:29.192396+0000', 'modified': '2024-09-12T23:33:32.368306+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}, 'max_mds': 1, 'in': [0], 'up': {'mds_0': 24316}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_24316': {'gid': 24316, 'name': 'b', 'rank': 0, 'incarnation': 4, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.181:6833/2077070284', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6832', 'nonce': 2077070284}, {'type': 'v1', 'addr': '172.21.15.181:6833', 'nonce': 2077070284}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540138320759226367, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}}}, 'data_pools': [3], 'metadata_pool': 2, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'standby_count_wanted': 0} 2024-09-12T23:36:38.172 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: 1/1 2024-09-12T23:36:38.172 DEBUG:teuthology.orchestra.run.smithi136:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.b status 2024-09-12T23:36:38.286 INFO:teuthology.orchestra.run.smithi136.stderr:2024-09-12T23:36:38.285+0000 7f8c5eff5640 0 client.24728 ms_handle_reset on v2:172.21.15.181:6832/2077070284 2024-09-12T23:36:38.304 INFO:teuthology.orchestra.run.smithi136.stderr:2024-09-12T23:36:38.305+0000 7f8c5eff5640 0 client.24734 ms_handle_reset on v2:172.21.15.181:6832/2077070284 2024-09-12T23:36:38.304 INFO:teuthology.orchestra.run.smithi136.stdout:{ 2024-09-12T23:36:38.305 INFO:teuthology.orchestra.run.smithi136.stdout: "cluster_fsid": "da58a47c-715d-11ef-bceb-c7b262605968", 2024-09-12T23:36:38.305 INFO:teuthology.orchestra.run.smithi136.stdout: "whoami": 0, 2024-09-12T23:36:38.305 INFO:teuthology.orchestra.run.smithi136.stdout: "id": 24316, 2024-09-12T23:36:38.305 INFO:teuthology.orchestra.run.smithi136.stdout: "want_state": "up:active", 2024-09-12T23:36:38.305 INFO:teuthology.orchestra.run.smithi136.stdout: "state": "up:active", 2024-09-12T23:36:38.305 INFO:teuthology.orchestra.run.smithi136.stdout: "fs_name": "cephfs", 2024-09-12T23:36:38.305 INFO:teuthology.orchestra.run.smithi136.stdout: "rank_uptime": 188.430658978, 2024-09-12T23:36:38.305 INFO:teuthology.orchestra.run.smithi136.stdout: "mdsmap_epoch": 7, 2024-09-12T23:36:38.305 INFO:teuthology.orchestra.run.smithi136.stdout: "osdmap_epoch": 64, 2024-09-12T23:36:38.305 INFO:teuthology.orchestra.run.smithi136.stdout: "osdmap_epoch_barrier": 54, 2024-09-12T23:36:38.305 INFO:teuthology.orchestra.run.smithi136.stdout: "uptime": 194.542731704 2024-09-12T23:36:38.305 INFO:teuthology.orchestra.run.smithi136.stdout:} 2024-09-12T23:36:38.319 INFO:tasks.cephadm:Setup complete, yielding 2024-09-12T23:36:38.319 INFO:teuthology.run_tasks:Running task print... 2024-09-12T23:36:38.330 INFO:teuthology.task.print:**** done end installing quincy cephadm ... 2024-09-12T23:36:38.330 INFO:teuthology.run_tasks:Running task print... 2024-09-12T23:36:38.363 INFO:teuthology.task.print:**** done start cephadm.shell ceph config set mgr... 2024-09-12T23:36:38.363 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-12T23:36:38.406 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi136.front.sepia.ceph.com 2024-09-12T23:36:38.407 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- bash -c 'ceph config set mgr mgr/cephadm/use_repo_digest true --force' 2024-09-12T23:36:39.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:38 smithi136 bash[22601]: audit 2024-09-12T23:36:37.805074+0000 mon.c (mon.2) 46 : audit [DBG] from='client.? 172.21.15.136:0/1217880861' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-12T23:36:39.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:38 smithi136 bash[22601]: audit 2024-09-12T23:36:37.999485+0000 mon.a (mon.0) 894 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:36:39.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:38 smithi136 bash[22601]: audit 2024-09-12T23:36:38.157856+0000 mon.a (mon.0) 895 : audit [DBG] from='client.? 172.21.15.136:0/1354970204' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-12T23:36:39.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:38 smithi136 bash[32189]: audit 2024-09-12T23:36:37.805074+0000 mon.c (mon.2) 46 : audit [DBG] from='client.? 172.21.15.136:0/1217880861' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-12T23:36:39.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:38 smithi136 bash[32189]: audit 2024-09-12T23:36:37.999485+0000 mon.a (mon.0) 894 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:36:39.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:38 smithi136 bash[32189]: audit 2024-09-12T23:36:38.157856+0000 mon.a (mon.0) 895 : audit [DBG] from='client.? 172.21.15.136:0/1354970204' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-12T23:36:39.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:38 smithi181 bash[24776]: audit 2024-09-12T23:36:37.805074+0000 mon.c (mon.2) 46 : audit [DBG] from='client.? 172.21.15.136:0/1217880861' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-12T23:36:39.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:38 smithi181 bash[24776]: audit 2024-09-12T23:36:37.999485+0000 mon.a (mon.0) 894 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:36:39.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:38 smithi181 bash[24776]: audit 2024-09-12T23:36:38.157856+0000 mon.a (mon.0) 895 : audit [DBG] from='client.? 172.21.15.136:0/1354970204' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-12T23:36:40.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:39 smithi136 bash[22601]: cluster 2024-09-12T23:36:37.973892+0000 mgr.y (mgr.24446) 111 : cluster [DBG] pgmap v87: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:40.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:39 smithi136 bash[32189]: cluster 2024-09-12T23:36:37.973892+0000 mgr.y (mgr.24446) 111 : cluster [DBG] pgmap v87: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:40.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:39 smithi181 bash[24776]: cluster 2024-09-12T23:36:37.973892+0000 mgr.y (mgr.24446) 111 : cluster [DBG] pgmap v87: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:42.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:41 smithi136 bash[22601]: cluster 2024-09-12T23:36:39.975045+0000 mgr.y (mgr.24446) 112 : cluster [DBG] pgmap v88: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:42.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:41 smithi136 bash[32189]: cluster 2024-09-12T23:36:39.975045+0000 mgr.y (mgr.24446) 112 : cluster [DBG] pgmap v88: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:42.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:41 smithi181 bash[24776]: cluster 2024-09-12T23:36:39.975045+0000 mgr.y (mgr.24446) 112 : cluster [DBG] pgmap v88: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:43.179 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:36:44.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:43 smithi136 bash[22601]: cluster 2024-09-12T23:36:41.975670+0000 mgr.y (mgr.24446) 113 : cluster [DBG] pgmap v89: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:44.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:43 smithi136 bash[32189]: cluster 2024-09-12T23:36:41.975670+0000 mgr.y (mgr.24446) 113 : cluster [DBG] pgmap v89: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:44.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:43 smithi181 bash[24776]: cluster 2024-09-12T23:36:41.975670+0000 mgr.y (mgr.24446) 113 : cluster [DBG] pgmap v89: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:45.595 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:36:45 smithi181 bash[25717]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:36:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-12T23:36:46.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:45 smithi136 bash[22601]: cluster 2024-09-12T23:36:43.976902+0000 mgr.y (mgr.24446) 114 : cluster [DBG] pgmap v90: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:46.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:45 smithi136 bash[32189]: cluster 2024-09-12T23:36:43.976902+0000 mgr.y (mgr.24446) 114 : cluster [DBG] pgmap v90: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:46.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:45 smithi181 bash[24776]: cluster 2024-09-12T23:36:43.976902+0000 mgr.y (mgr.24446) 114 : cluster [DBG] pgmap v90: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:46.343 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:36:46 smithi136 bash[22852]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:36:46] "GET /metrics HTTP/1.1" 200 236633 "" "Prometheus/2.43.0" 2024-09-12T23:36:46.639 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:36:46 smithi136 bash[22852]: [12/Sep/2024:23:36:46] ENGINE Bus STOPPING 2024-09-12T23:36:46.949 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:36:46 smithi136 bash[22852]: [12/Sep/2024:23:36:46] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-12T23:36:46.949 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:36:46 smithi136 bash[22852]: [12/Sep/2024:23:36:46] ENGINE Bus STOPPED 2024-09-12T23:36:46.949 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:36:46 smithi136 bash[22852]: [12/Sep/2024:23:36:46] ENGINE Bus STARTING 2024-09-12T23:36:46.949 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:36:46 smithi136 bash[22852]: [12/Sep/2024:23:36:46] ENGINE Serving on http://:::9283 2024-09-12T23:36:46.949 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:36:46 smithi136 bash[22852]: [12/Sep/2024:23:36:46] ENGINE Bus STARTED 2024-09-12T23:36:46.950 INFO:teuthology.run_tasks:Running task print... 2024-09-12T23:36:46.960 INFO:teuthology.task.print:**** done cephadm.shell ceph config set mgr... 2024-09-12T23:36:46.960 INFO:teuthology.run_tasks:Running task print... 2024-09-12T23:36:47.041 INFO:teuthology.task.print:**** done start telemetry quincy... 2024-09-12T23:36:47.041 INFO:teuthology.run_tasks:Running task workunit... 2024-09-12T23:36:47.085 INFO:tasks.workunit:Pulling workunits from ref 7a853b1706d076d58e0da7a5d3b9b9eb36276880 2024-09-12T23:36:47.085 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-09-12T23:36:47.085 DEBUG:teuthology.orchestra.run.smithi136:> stat -- /home/ubuntu/cephtest/mnt.0 2024-09-12T23:36:47.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-12T23:36:47.092 INFO:teuthology.orchestra.run.smithi136.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2024-09-12T23:36:47.092 DEBUG:teuthology.orchestra.run.smithi136:> mkdir -- /home/ubuntu/cephtest/mnt.0 2024-09-12T23:36:47.141 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2024-09-12T23:36:47.141 DEBUG:teuthology.orchestra.run.smithi136:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2024-09-12T23:36:47.188 INFO:tasks.workunit:timeout=3h 2024-09-12T23:36:47.188 INFO:tasks.workunit:cleanup=True 2024-09-12T23:36:47.189 DEBUG:teuthology.orchestra.run.smithi136:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone https://git.ceph.com/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 && git checkout 7a853b1706d076d58e0da7a5d3b9b9eb36276880 2024-09-12T23:36:47.238 INFO:tasks.workunit.client.0.smithi136.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-09-12T23:36:47.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:47 smithi181 bash[24776]: cluster 2024-09-12T23:36:45.977560+0000 mgr.y (mgr.24446) 115 : cluster [DBG] pgmap v91: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:47.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:47 smithi181 bash[24776]: audit 2024-09-12T23:36:46.341385+0000 mon.a (mon.0) 896 : audit [INF] from='client.? 172.21.15.136:0/2998872372' entity='client.admin' 2024-09-12T23:36:47.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:47 smithi181 bash[24776]: audit 2024-09-12T23:36:46.353943+0000 mon.a (mon.0) 897 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:36:47.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:47 smithi181 bash[24776]: audit 2024-09-12T23:36:46.355845+0000 mon.a (mon.0) 898 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:36:47.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:47 smithi181 bash[24776]: audit 2024-09-12T23:36:46.356697+0000 mon.a (mon.0) 899 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:36:47.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:47 smithi181 bash[24776]: audit 2024-09-12T23:36:46.361747+0000 mon.a (mon.0) 900 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:36:47.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:47 smithi136 bash[22601]: cluster 2024-09-12T23:36:45.977560+0000 mgr.y (mgr.24446) 115 : cluster [DBG] pgmap v91: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:47.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:47 smithi136 bash[22601]: audit 2024-09-12T23:36:46.341385+0000 mon.a (mon.0) 896 : audit [INF] from='client.? 172.21.15.136:0/2998872372' entity='client.admin' 2024-09-12T23:36:47.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:47 smithi136 bash[22601]: audit 2024-09-12T23:36:46.353943+0000 mon.a (mon.0) 897 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:36:47.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:47 smithi136 bash[22601]: audit 2024-09-12T23:36:46.355845+0000 mon.a (mon.0) 898 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:36:47.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:47 smithi136 bash[22601]: audit 2024-09-12T23:36:46.356697+0000 mon.a (mon.0) 899 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:36:47.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:47 smithi136 bash[22601]: audit 2024-09-12T23:36:46.361747+0000 mon.a (mon.0) 900 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:36:47.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:47 smithi136 bash[32189]: cluster 2024-09-12T23:36:45.977560+0000 mgr.y (mgr.24446) 115 : cluster [DBG] pgmap v91: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:47.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:47 smithi136 bash[32189]: audit 2024-09-12T23:36:46.341385+0000 mon.a (mon.0) 896 : audit [INF] from='client.? 172.21.15.136:0/2998872372' entity='client.admin' 2024-09-12T23:36:47.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:47 smithi136 bash[32189]: audit 2024-09-12T23:36:46.353943+0000 mon.a (mon.0) 897 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:36:47.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:47 smithi136 bash[32189]: audit 2024-09-12T23:36:46.355845+0000 mon.a (mon.0) 898 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:36:47.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:47 smithi136 bash[32189]: audit 2024-09-12T23:36:46.356697+0000 mon.a (mon.0) 899 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:36:47.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:47 smithi136 bash[32189]: audit 2024-09-12T23:36:46.361747+0000 mon.a (mon.0) 900 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:36:49.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:49 smithi181 bash[24776]: cluster 2024-09-12T23:36:47.978166+0000 mgr.y (mgr.24446) 116 : cluster [DBG] pgmap v92: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:49.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:49 smithi136 bash[22601]: cluster 2024-09-12T23:36:47.978166+0000 mgr.y (mgr.24446) 116 : cluster [DBG] pgmap v92: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:49.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:49 smithi136 bash[32189]: cluster 2024-09-12T23:36:47.978166+0000 mgr.y (mgr.24446) 116 : cluster [DBG] pgmap v92: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:51.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:51 smithi181 bash[24776]: cluster 2024-09-12T23:36:49.979364+0000 mgr.y (mgr.24446) 117 : cluster [DBG] pgmap v93: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:51.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:51 smithi136 bash[22601]: cluster 2024-09-12T23:36:49.979364+0000 mgr.y (mgr.24446) 117 : cluster [DBG] pgmap v93: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:51.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:51 smithi136 bash[32189]: cluster 2024-09-12T23:36:49.979364+0000 mgr.y (mgr.24446) 117 : cluster [DBG] pgmap v93: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:53.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:53 smithi136 bash[22601]: cluster 2024-09-12T23:36:51.979983+0000 mgr.y (mgr.24446) 118 : cluster [DBG] pgmap v94: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:53.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:53 smithi136 bash[22601]: audit 2024-09-12T23:36:53.000173+0000 mon.a (mon.0) 901 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:36:53.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:53 smithi136 bash[32189]: cluster 2024-09-12T23:36:51.979983+0000 mgr.y (mgr.24446) 118 : cluster [DBG] pgmap v94: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:53.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:53 smithi136 bash[32189]: audit 2024-09-12T23:36:53.000173+0000 mon.a (mon.0) 901 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:36:53.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:53 smithi181 bash[24776]: cluster 2024-09-12T23:36:51.979983+0000 mgr.y (mgr.24446) 118 : cluster [DBG] pgmap v94: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:53.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:53 smithi181 bash[24776]: audit 2024-09-12T23:36:53.000173+0000 mon.a (mon.0) 901 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:36:55.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:55 smithi181 bash[24776]: cluster 2024-09-12T23:36:53.981198+0000 mgr.y (mgr.24446) 119 : cluster [DBG] pgmap v95: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:55.596 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:36:55 smithi181 bash[25717]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:36:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-12T23:36:55.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:55 smithi136 bash[22601]: cluster 2024-09-12T23:36:53.981198+0000 mgr.y (mgr.24446) 119 : cluster [DBG] pgmap v95: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:55.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:55 smithi136 bash[32189]: cluster 2024-09-12T23:36:53.981198+0000 mgr.y (mgr.24446) 119 : cluster [DBG] pgmap v95: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:56.506 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:36:56 smithi136 bash[22852]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:36:56] "GET /metrics HTTP/1.1" 200 236633 "" "Prometheus/2.43.0" 2024-09-12T23:36:57.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:57 smithi136 bash[22601]: cluster 2024-09-12T23:36:55.981856+0000 mgr.y (mgr.24446) 120 : cluster [DBG] pgmap v96: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:57.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:57 smithi136 bash[32189]: cluster 2024-09-12T23:36:55.981856+0000 mgr.y (mgr.24446) 120 : cluster [DBG] pgmap v96: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:57.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:57 smithi181 bash[24776]: cluster 2024-09-12T23:36:55.981856+0000 mgr.y (mgr.24446) 120 : cluster [DBG] pgmap v96: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:59.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:36:59 smithi136 bash[22601]: cluster 2024-09-12T23:36:57.982515+0000 mgr.y (mgr.24446) 121 : cluster [DBG] pgmap v97: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:59.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:36:59 smithi136 bash[32189]: cluster 2024-09-12T23:36:57.982515+0000 mgr.y (mgr.24446) 121 : cluster [DBG] pgmap v97: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:36:59.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:36:59 smithi181 bash[24776]: cluster 2024-09-12T23:36:57.982515+0000 mgr.y (mgr.24446) 121 : cluster [DBG] pgmap v97: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:01.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:01 smithi136 bash[32189]: cluster 2024-09-12T23:36:59.983753+0000 mgr.y (mgr.24446) 122 : cluster [DBG] pgmap v98: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:01.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:01 smithi136 bash[22601]: cluster 2024-09-12T23:36:59.983753+0000 mgr.y (mgr.24446) 122 : cluster [DBG] pgmap v98: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:01.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:01 smithi181 bash[24776]: cluster 2024-09-12T23:36:59.983753+0000 mgr.y (mgr.24446) 122 : cluster [DBG] pgmap v98: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:03.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:03 smithi136 bash[22601]: cluster 2024-09-12T23:37:01.984405+0000 mgr.y (mgr.24446) 123 : cluster [DBG] pgmap v99: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:03.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:03 smithi136 bash[32189]: cluster 2024-09-12T23:37:01.984405+0000 mgr.y (mgr.24446) 123 : cluster [DBG] pgmap v99: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:03.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:03 smithi181 bash[24776]: cluster 2024-09-12T23:37:01.984405+0000 mgr.y (mgr.24446) 123 : cluster [DBG] pgmap v99: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:05.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:05 smithi181 bash[24776]: cluster 2024-09-12T23:37:03.985638+0000 mgr.y (mgr.24446) 124 : cluster [DBG] pgmap v100: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:05.596 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:37:05 smithi181 bash[25717]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:37:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-12T23:37:05.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:05 smithi136 bash[22601]: cluster 2024-09-12T23:37:03.985638+0000 mgr.y (mgr.24446) 124 : cluster [DBG] pgmap v100: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:05.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:05 smithi136 bash[32189]: cluster 2024-09-12T23:37:03.985638+0000 mgr.y (mgr.24446) 124 : cluster [DBG] pgmap v100: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:06.506 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:37:06 smithi136 bash[22852]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:37:06] "GET /metrics HTTP/1.1" 200 236633 "" "Prometheus/2.43.0" 2024-09-12T23:37:07.755 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:07 smithi136 bash[32189]: cluster 2024-09-12T23:37:05.986187+0000 mgr.y (mgr.24446) 125 : cluster [DBG] pgmap v101: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:07.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:07 smithi136 bash[22601]: cluster 2024-09-12T23:37:05.986187+0000 mgr.y (mgr.24446) 125 : cluster [DBG] pgmap v101: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:07.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:07 smithi181 bash[24776]: cluster 2024-09-12T23:37:05.986187+0000 mgr.y (mgr.24446) 125 : cluster [DBG] pgmap v101: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:08.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:08 smithi136 bash[22601]: audit 2024-09-12T23:37:08.011612+0000 mon.a (mon.0) 902 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:37:08.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:08 smithi136 bash[32189]: audit 2024-09-12T23:37:08.011612+0000 mon.a (mon.0) 902 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:37:08.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:08 smithi181 bash[24776]: audit 2024-09-12T23:37:08.011612+0000 mon.a (mon.0) 902 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:37:09.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:09 smithi136 bash[22601]: cluster 2024-09-12T23:37:07.986822+0000 mgr.y (mgr.24446) 126 : cluster [DBG] pgmap v102: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:09.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:09 smithi136 bash[32189]: cluster 2024-09-12T23:37:07.986822+0000 mgr.y (mgr.24446) 126 : cluster [DBG] pgmap v102: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:09.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:09 smithi181 bash[24776]: cluster 2024-09-12T23:37:07.986822+0000 mgr.y (mgr.24446) 126 : cluster [DBG] pgmap v102: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:11.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:11 smithi136 bash[22601]: cluster 2024-09-12T23:37:09.987951+0000 mgr.y (mgr.24446) 127 : cluster [DBG] pgmap v103: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:11.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:11 smithi136 bash[32189]: cluster 2024-09-12T23:37:09.987951+0000 mgr.y (mgr.24446) 127 : cluster [DBG] pgmap v103: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:11.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:11 smithi181 bash[24776]: cluster 2024-09-12T23:37:09.987951+0000 mgr.y (mgr.24446) 127 : cluster [DBG] pgmap v103: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:13.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:13 smithi136 bash[32189]: cluster 2024-09-12T23:37:11.988512+0000 mgr.y (mgr.24446) 128 : cluster [DBG] pgmap v104: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:13.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:13 smithi136 bash[22601]: cluster 2024-09-12T23:37:11.988512+0000 mgr.y (mgr.24446) 128 : cluster [DBG] pgmap v104: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:13.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:13 smithi181 bash[24776]: cluster 2024-09-12T23:37:11.988512+0000 mgr.y (mgr.24446) 128 : cluster [DBG] pgmap v104: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:15.395 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:37:15 smithi181 bash[25717]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:37:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-12T23:37:15.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:15 smithi136 bash[32189]: cluster 2024-09-12T23:37:13.989141+0000 mgr.y (mgr.24446) 129 : cluster [DBG] pgmap v105: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:15.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:15 smithi136 bash[22601]: cluster 2024-09-12T23:37:13.989141+0000 mgr.y (mgr.24446) 129 : cluster [DBG] pgmap v105: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:15.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:15 smithi181 bash[24776]: cluster 2024-09-12T23:37:13.989141+0000 mgr.y (mgr.24446) 129 : cluster [DBG] pgmap v105: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:16.506 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:37:16 smithi136 bash[22852]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:37:16] "GET /metrics HTTP/1.1" 200 236613 "" "Prometheus/2.43.0" 2024-09-12T23:37:17.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:17 smithi136 bash[22601]: cluster 2024-09-12T23:37:15.989503+0000 mgr.y (mgr.24446) 130 : cluster [DBG] pgmap v106: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:17.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:17 smithi136 bash[32189]: cluster 2024-09-12T23:37:15.989503+0000 mgr.y (mgr.24446) 130 : cluster [DBG] pgmap v106: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:17.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:17 smithi181 bash[24776]: cluster 2024-09-12T23:37:15.989503+0000 mgr.y (mgr.24446) 130 : cluster [DBG] pgmap v106: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:19.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:19 smithi136 bash[22601]: cluster 2024-09-12T23:37:17.989877+0000 mgr.y (mgr.24446) 131 : cluster [DBG] pgmap v107: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:19.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:19 smithi136 bash[32189]: cluster 2024-09-12T23:37:17.989877+0000 mgr.y (mgr.24446) 131 : cluster [DBG] pgmap v107: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:19.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:19 smithi181 bash[24776]: cluster 2024-09-12T23:37:17.989877+0000 mgr.y (mgr.24446) 131 : cluster [DBG] pgmap v107: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:21.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:21 smithi136 bash[22601]: cluster 2024-09-12T23:37:19.990562+0000 mgr.y (mgr.24446) 132 : cluster [DBG] pgmap v108: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:21.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:21 smithi136 bash[32189]: cluster 2024-09-12T23:37:19.990562+0000 mgr.y (mgr.24446) 132 : cluster [DBG] pgmap v108: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:21.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:21 smithi181 bash[24776]: cluster 2024-09-12T23:37:19.990562+0000 mgr.y (mgr.24446) 132 : cluster [DBG] pgmap v108: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:23.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:23 smithi136 bash[32189]: cluster 2024-09-12T23:37:21.990954+0000 mgr.y (mgr.24446) 133 : cluster [DBG] pgmap v109: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:23.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:23 smithi136 bash[32189]: audit 2024-09-12T23:37:23.011816+0000 mon.a (mon.0) 903 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:37:23.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:23 smithi136 bash[22601]: cluster 2024-09-12T23:37:21.990954+0000 mgr.y (mgr.24446) 133 : cluster [DBG] pgmap v109: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:23.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:23 smithi136 bash[22601]: audit 2024-09-12T23:37:23.011816+0000 mon.a (mon.0) 903 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:37:23.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:23 smithi181 bash[24776]: cluster 2024-09-12T23:37:21.990954+0000 mgr.y (mgr.24446) 133 : cluster [DBG] pgmap v109: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:23.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:23 smithi181 bash[24776]: audit 2024-09-12T23:37:23.011816+0000 mon.a (mon.0) 903 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:37:25.410 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:37:25 smithi181 bash[25717]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:37:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-12T23:37:25.755 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:25 smithi136 bash[32189]: cluster 2024-09-12T23:37:23.991592+0000 mgr.y (mgr.24446) 134 : cluster [DBG] pgmap v110: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:25.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:25 smithi136 bash[22601]: cluster 2024-09-12T23:37:23.991592+0000 mgr.y (mgr.24446) 134 : cluster [DBG] pgmap v110: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:25.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:25 smithi181 bash[24776]: cluster 2024-09-12T23:37:23.991592+0000 mgr.y (mgr.24446) 134 : cluster [DBG] pgmap v110: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:26.505 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:37:26 smithi136 bash[22852]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:37:26] "GET /metrics HTTP/1.1" 200 236589 "" "Prometheus/2.43.0" 2024-09-12T23:37:27.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:27 smithi136 bash[22601]: cluster 2024-09-12T23:37:25.991970+0000 mgr.y (mgr.24446) 135 : cluster [DBG] pgmap v111: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:27.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:27 smithi136 bash[32189]: cluster 2024-09-12T23:37:25.991970+0000 mgr.y (mgr.24446) 135 : cluster [DBG] pgmap v111: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:27.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:27 smithi181 bash[24776]: cluster 2024-09-12T23:37:25.991970+0000 mgr.y (mgr.24446) 135 : cluster [DBG] pgmap v111: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:29.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:29 smithi136 bash[22601]: cluster 2024-09-12T23:37:27.992353+0000 mgr.y (mgr.24446) 136 : cluster [DBG] pgmap v112: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:29.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:29 smithi136 bash[32189]: cluster 2024-09-12T23:37:27.992353+0000 mgr.y (mgr.24446) 136 : cluster [DBG] pgmap v112: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:29.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:29 smithi181 bash[24776]: cluster 2024-09-12T23:37:27.992353+0000 mgr.y (mgr.24446) 136 : cluster [DBG] pgmap v112: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:31.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:31 smithi136 bash[22601]: cluster 2024-09-12T23:37:29.993054+0000 mgr.y (mgr.24446) 137 : cluster [DBG] pgmap v113: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:31.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:31 smithi136 bash[32189]: cluster 2024-09-12T23:37:29.993054+0000 mgr.y (mgr.24446) 137 : cluster [DBG] pgmap v113: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:31.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:31 smithi181 bash[24776]: cluster 2024-09-12T23:37:29.993054+0000 mgr.y (mgr.24446) 137 : cluster [DBG] pgmap v113: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:33.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:33 smithi136 bash[22601]: cluster 2024-09-12T23:37:31.993448+0000 mgr.y (mgr.24446) 138 : cluster [DBG] pgmap v114: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:33.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:33 smithi136 bash[32189]: cluster 2024-09-12T23:37:31.993448+0000 mgr.y (mgr.24446) 138 : cluster [DBG] pgmap v114: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:33.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:33 smithi181 bash[24776]: cluster 2024-09-12T23:37:31.993448+0000 mgr.y (mgr.24446) 138 : cluster [DBG] pgmap v114: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:35.432 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:37:35 smithi181 bash[25717]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:37:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-12T23:37:35.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:35 smithi136 bash[22601]: cluster 2024-09-12T23:37:33.994119+0000 mgr.y (mgr.24446) 139 : cluster [DBG] pgmap v115: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:35.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:35 smithi136 bash[32189]: cluster 2024-09-12T23:37:33.994119+0000 mgr.y (mgr.24446) 139 : cluster [DBG] pgmap v115: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:35.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:35 smithi181 bash[24776]: cluster 2024-09-12T23:37:33.994119+0000 mgr.y (mgr.24446) 139 : cluster [DBG] pgmap v115: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:36.505 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:37:36 smithi136 bash[22852]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:37:36] "GET /metrics HTTP/1.1" 200 236589 "" "Prometheus/2.43.0" 2024-09-12T23:37:37.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:37 smithi136 bash[22601]: cluster 2024-09-12T23:37:35.994501+0000 mgr.y (mgr.24446) 140 : cluster [DBG] pgmap v116: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:37.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:37 smithi136 bash[32189]: cluster 2024-09-12T23:37:35.994501+0000 mgr.y (mgr.24446) 140 : cluster [DBG] pgmap v116: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:37.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:37 smithi181 bash[24776]: cluster 2024-09-12T23:37:35.994501+0000 mgr.y (mgr.24446) 140 : cluster [DBG] pgmap v116: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:38.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:38 smithi136 bash[32189]: audit 2024-09-12T23:37:38.012006+0000 mon.a (mon.0) 904 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:37:38.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:38 smithi136 bash[22601]: audit 2024-09-12T23:37:38.012006+0000 mon.a (mon.0) 904 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:37:38.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:38 smithi181 bash[24776]: audit 2024-09-12T23:37:38.012006+0000 mon.a (mon.0) 904 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:37:39.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:39 smithi136 bash[22601]: cluster 2024-09-12T23:37:37.994896+0000 mgr.y (mgr.24446) 141 : cluster [DBG] pgmap v117: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:39.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:39 smithi136 bash[32189]: cluster 2024-09-12T23:37:37.994896+0000 mgr.y (mgr.24446) 141 : cluster [DBG] pgmap v117: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:39.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:39 smithi181 bash[24776]: cluster 2024-09-12T23:37:37.994896+0000 mgr.y (mgr.24446) 141 : cluster [DBG] pgmap v117: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:41.547 INFO:tasks.workunit.client.0.smithi136.stderr:Note: switching to '7a853b1706d076d58e0da7a5d3b9b9eb36276880'. 2024-09-12T23:37:41.548 INFO:tasks.workunit.client.0.smithi136.stderr: 2024-09-12T23:37:41.548 INFO:tasks.workunit.client.0.smithi136.stderr:You are in 'detached HEAD' state. You can look around, make experimental 2024-09-12T23:37:41.548 INFO:tasks.workunit.client.0.smithi136.stderr:changes and commit them, and you can discard any commits you make in this 2024-09-12T23:37:41.548 INFO:tasks.workunit.client.0.smithi136.stderr:state without impacting any branches by switching back to a branch. 2024-09-12T23:37:41.548 INFO:tasks.workunit.client.0.smithi136.stderr: 2024-09-12T23:37:41.548 INFO:tasks.workunit.client.0.smithi136.stderr:If you want to create a new branch to retain commits you create, you may 2024-09-12T23:37:41.548 INFO:tasks.workunit.client.0.smithi136.stderr:do so (now or later) by using -c with the switch command. Example: 2024-09-12T23:37:41.548 INFO:tasks.workunit.client.0.smithi136.stderr: 2024-09-12T23:37:41.548 INFO:tasks.workunit.client.0.smithi136.stderr: git switch -c 2024-09-12T23:37:41.548 INFO:tasks.workunit.client.0.smithi136.stderr: 2024-09-12T23:37:41.548 INFO:tasks.workunit.client.0.smithi136.stderr:Or undo this operation with: 2024-09-12T23:37:41.549 INFO:tasks.workunit.client.0.smithi136.stderr: 2024-09-12T23:37:41.549 INFO:tasks.workunit.client.0.smithi136.stderr: git switch - 2024-09-12T23:37:41.549 INFO:tasks.workunit.client.0.smithi136.stderr: 2024-09-12T23:37:41.549 INFO:tasks.workunit.client.0.smithi136.stderr:Turn off this advice by setting config variable advice.detachedHead to false 2024-09-12T23:37:41.549 INFO:tasks.workunit.client.0.smithi136.stderr: 2024-09-12T23:37:41.549 INFO:tasks.workunit.client.0.smithi136.stderr:HEAD is now at 7a853b1706d Merge pull request #59683 from zdover23/wip-doc-2024-09-10-backport-59677-to-reef 2024-09-12T23:37:41.556 DEBUG:teuthology.orchestra.run.smithi136:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-09-12T23:37:41.605 INFO:tasks.workunit.client.0.smithi136.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-09-12T23:37:41.608 INFO:tasks.workunit.client.0.smithi136.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-12T23:37:41.608 INFO:tasks.workunit.client.0.smithi136.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-09-12T23:37:41.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:41 smithi136 bash[22601]: cluster 2024-09-12T23:37:39.996041+0000 mgr.y (mgr.24446) 142 : cluster [DBG] pgmap v118: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:41.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:41 smithi136 bash[32189]: cluster 2024-09-12T23:37:39.996041+0000 mgr.y (mgr.24446) 142 : cluster [DBG] pgmap v118: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:41.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:41 smithi181 bash[24776]: cluster 2024-09-12T23:37:39.996041+0000 mgr.y (mgr.24446) 142 : cluster [DBG] pgmap v118: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:42.407 INFO:tasks.workunit.client.0.smithi136.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-09-12T23:37:42.487 INFO:tasks.workunit.client.0.smithi136.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-09-12T23:37:42.541 INFO:tasks.workunit.client.0.smithi136.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-12T23:37:42.542 INFO:tasks.workunit.client.0.smithi136.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-12T23:37:42.542 INFO:tasks.workunit.client.0.smithi136.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-09-12T23:37:42.586 INFO:tasks.workunit.client.0.smithi136.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-12T23:37:42.589 DEBUG:teuthology.orchestra.run.smithi136:> set -ex 2024-09-12T23:37:42.589 DEBUG:teuthology.orchestra.run.smithi136:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-09-12T23:37:42.636 INFO:tasks.workunit:Running workunits matching test_telemetry_quincy.sh on client.0... 2024-09-12T23:37:42.638 INFO:tasks.workunit:Running workunit test_telemetry_quincy.sh... 2024-09-12T23:37:42.638 DEBUG:teuthology.orchestra.run.smithi136:workunit test test_telemetry_quincy.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=7a853b1706d076d58e0da7a5d3b9b9eb36276880 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/test_telemetry_quincy.sh 2024-09-12T23:37:42.688 INFO:tasks.workunit.client.0.smithi136.stderr:+ ceph config set mgr mgr/telemetry/channel_ident true 2024-09-12T23:37:43.110 INFO:tasks.workunit.client.0.smithi136.stderr:+ ceph config set mgr mgr/telemetry/organization ceph-qa 2024-09-12T23:37:43.255 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:37:43 smithi136 bash[22852]: [12/Sep/2024:23:37:43] ENGINE Bus STOPPING 2024-09-12T23:37:43.255 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:37:43 smithi136 bash[22852]: [12/Sep/2024:23:37:43] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-12T23:37:43.256 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:37:43 smithi136 bash[22852]: [12/Sep/2024:23:37:43] ENGINE Bus STOPPED 2024-09-12T23:37:43.256 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:37:43 smithi136 bash[22852]: [12/Sep/2024:23:37:43] ENGINE Bus STARTING 2024-09-12T23:37:43.256 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:37:43 smithi136 bash[22852]: [12/Sep/2024:23:37:43] ENGINE Serving on http://:::9283 2024-09-12T23:37:43.256 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:37:43 smithi136 bash[22852]: [12/Sep/2024:23:37:43] ENGINE Bus STARTED 2024-09-12T23:37:43.555 INFO:tasks.workunit.client.0.smithi136.stderr:+ ceph config set mgr mgr/telemetry/description 'upgrade test cluster' 2024-09-12T23:37:43.748 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:43 smithi136 bash[32189]: cluster 2024-09-12T23:37:41.996717+0000 mgr.y (mgr.24446) 143 : cluster [DBG] pgmap v119: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:43.749 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:43 smithi136 bash[32189]: audit 2024-09-12T23:37:42.992318+0000 mon.a (mon.0) 905 : audit [INF] from='client.? ' entity='client.admin' 2024-09-12T23:37:43.749 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:43 smithi136 bash[32189]: audit 2024-09-12T23:37:43.009395+0000 mon.a (mon.0) 906 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:37:43.749 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:43 smithi136 bash[32189]: audit 2024-09-12T23:37:43.433928+0000 mon.a (mon.0) 907 : audit [INF] from='client.? ' entity='client.admin' 2024-09-12T23:37:43.749 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:37:43 smithi136 bash[22852]: [12/Sep/2024:23:37:43] ENGINE Bus STOPPING 2024-09-12T23:37:43.749 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:43 smithi136 bash[22601]: cluster 2024-09-12T23:37:41.996717+0000 mgr.y (mgr.24446) 143 : cluster [DBG] pgmap v119: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:43.749 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:43 smithi136 bash[22601]: audit 2024-09-12T23:37:42.992318+0000 mon.a (mon.0) 905 : audit [INF] from='client.? ' entity='client.admin' 2024-09-12T23:37:43.750 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:43 smithi136 bash[22601]: audit 2024-09-12T23:37:43.009395+0000 mon.a (mon.0) 906 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:37:43.750 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:43 smithi136 bash[22601]: audit 2024-09-12T23:37:43.433928+0000 mon.a (mon.0) 907 : audit [INF] from='client.? ' entity='client.admin' 2024-09-12T23:37:43.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:43 smithi181 bash[24776]: cluster 2024-09-12T23:37:41.996717+0000 mgr.y (mgr.24446) 143 : cluster [DBG] pgmap v119: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:43.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:43 smithi181 bash[24776]: audit 2024-09-12T23:37:42.992318+0000 mon.a (mon.0) 905 : audit [INF] from='client.? ' entity='client.admin' 2024-09-12T23:37:43.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:43 smithi181 bash[24776]: audit 2024-09-12T23:37:43.009395+0000 mon.a (mon.0) 906 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:37:43.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:43 smithi181 bash[24776]: audit 2024-09-12T23:37:43.433928+0000 mon.a (mon.0) 907 : audit [INF] from='client.? ' entity='client.admin' 2024-09-12T23:37:43.952 INFO:tasks.workunit.client.0.smithi136.stderr:+ ceph telemetry preview 2024-09-12T23:37:44.005 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:37:43 smithi136 bash[22852]: [12/Sep/2024:23:37:43] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-12T23:37:44.006 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:37:43 smithi136 bash[22852]: [12/Sep/2024:23:37:43] ENGINE Bus STOPPED 2024-09-12T23:37:44.006 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:37:43 smithi136 bash[22852]: [12/Sep/2024:23:37:43] ENGINE Bus STARTING 2024-09-12T23:37:44.006 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:37:43 smithi136 bash[22852]: [12/Sep/2024:23:37:43] ENGINE Serving on http://:::9283 2024-09-12T23:37:44.006 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:37:43 smithi136 bash[22852]: [12/Sep/2024:23:37:43] ENGINE Bus STARTED 2024-09-12T23:37:44.006 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:37:43 smithi136 bash[22852]: [12/Sep/2024:23:37:43] ENGINE Bus STOPPING 2024-09-12T23:37:44.428 INFO:tasks.workunit.client.0.smithi136.stdout:{ 2024-09-12T23:37:44.428 INFO:tasks.workunit.client.0.smithi136.stdout: "balancer": { 2024-09-12T23:37:44.428 INFO:tasks.workunit.client.0.smithi136.stdout: "active": true, 2024-09-12T23:37:44.429 INFO:tasks.workunit.client.0.smithi136.stdout: "mode": "" 2024-09-12T23:37:44.429 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.429 INFO:tasks.workunit.client.0.smithi136.stdout: "channels": [ 2024-09-12T23:37:44.429 INFO:tasks.workunit.client.0.smithi136.stdout: "basic", 2024-09-12T23:37:44.429 INFO:tasks.workunit.client.0.smithi136.stdout: "crash", 2024-09-12T23:37:44.429 INFO:tasks.workunit.client.0.smithi136.stdout: "device", 2024-09-12T23:37:44.429 INFO:tasks.workunit.client.0.smithi136.stdout: "ident" 2024-09-12T23:37:44.429 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:44.429 INFO:tasks.workunit.client.0.smithi136.stdout: "channels_available": [ 2024-09-12T23:37:44.429 INFO:tasks.workunit.client.0.smithi136.stdout: "basic", 2024-09-12T23:37:44.430 INFO:tasks.workunit.client.0.smithi136.stdout: "ident", 2024-09-12T23:37:44.430 INFO:tasks.workunit.client.0.smithi136.stdout: "crash", 2024-09-12T23:37:44.430 INFO:tasks.workunit.client.0.smithi136.stdout: "device", 2024-09-12T23:37:44.430 INFO:tasks.workunit.client.0.smithi136.stdout: "perf" 2024-09-12T23:37:44.430 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:44.430 INFO:tasks.workunit.client.0.smithi136.stdout: "collections_available": [ 2024-09-12T23:37:44.431 INFO:tasks.workunit.client.0.smithi136.stdout: "basic_base", 2024-09-12T23:37:44.431 INFO:tasks.workunit.client.0.smithi136.stdout: "device_base", 2024-09-12T23:37:44.431 INFO:tasks.workunit.client.0.smithi136.stdout: "crash_base", 2024-09-12T23:37:44.431 INFO:tasks.workunit.client.0.smithi136.stdout: "ident_base", 2024-09-12T23:37:44.431 INFO:tasks.workunit.client.0.smithi136.stdout: "perf_perf", 2024-09-12T23:37:44.431 INFO:tasks.workunit.client.0.smithi136.stdout: "basic_mds_metadata", 2024-09-12T23:37:44.432 INFO:tasks.workunit.client.0.smithi136.stdout: "basic_pool_usage", 2024-09-12T23:37:44.432 INFO:tasks.workunit.client.0.smithi136.stdout: "basic_usage_by_class", 2024-09-12T23:37:44.432 INFO:tasks.workunit.client.0.smithi136.stdout: "basic_rook_v01", 2024-09-12T23:37:44.432 INFO:tasks.workunit.client.0.smithi136.stdout: "perf_memory_metrics", 2024-09-12T23:37:44.432 INFO:tasks.workunit.client.0.smithi136.stdout: "basic_pool_options_bluestore" 2024-09-12T23:37:44.432 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:44.432 INFO:tasks.workunit.client.0.smithi136.stdout: "collections_opted_in": [ 2024-09-12T23:37:44.432 INFO:tasks.workunit.client.0.smithi136.stdout: "basic_base", 2024-09-12T23:37:44.432 INFO:tasks.workunit.client.0.smithi136.stdout: "device_base", 2024-09-12T23:37:44.432 INFO:tasks.workunit.client.0.smithi136.stdout: "crash_base", 2024-09-12T23:37:44.432 INFO:tasks.workunit.client.0.smithi136.stdout: "ident_base", 2024-09-12T23:37:44.432 INFO:tasks.workunit.client.0.smithi136.stdout: "perf_perf", 2024-09-12T23:37:44.433 INFO:tasks.workunit.client.0.smithi136.stdout: "basic_mds_metadata", 2024-09-12T23:37:44.433 INFO:tasks.workunit.client.0.smithi136.stdout: "basic_pool_usage", 2024-09-12T23:37:44.433 INFO:tasks.workunit.client.0.smithi136.stdout: "basic_usage_by_class", 2024-09-12T23:37:44.433 INFO:tasks.workunit.client.0.smithi136.stdout: "basic_rook_v01", 2024-09-12T23:37:44.433 INFO:tasks.workunit.client.0.smithi136.stdout: "perf_memory_metrics", 2024-09-12T23:37:44.433 INFO:tasks.workunit.client.0.smithi136.stdout: "basic_pool_options_bluestore" 2024-09-12T23:37:44.433 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:44.433 INFO:tasks.workunit.client.0.smithi136.stdout: "config": { 2024-09-12T23:37:44.433 INFO:tasks.workunit.client.0.smithi136.stdout: "active_changed": [ 2024-09-12T23:37:44.433 INFO:tasks.workunit.client.0.smithi136.stdout: "auth_allow_insecure_global_id_reclaim", 2024-09-12T23:37:44.433 INFO:tasks.workunit.client.0.smithi136.stdout: "auth_debug", 2024-09-12T23:37:44.433 INFO:tasks.workunit.client.0.smithi136.stdout: "auth_mon_ticket_ttl", 2024-09-12T23:37:44.433 INFO:tasks.workunit.client.0.smithi136.stdout: "auth_service_ticket_ttl", 2024-09-12T23:37:44.433 INFO:tasks.workunit.client.0.smithi136.stdout: "bdev_debug_aio", 2024-09-12T23:37:44.434 INFO:tasks.workunit.client.0.smithi136.stdout: "client_check_pool_perm", 2024-09-12T23:37:44.434 INFO:tasks.workunit.client.0.smithi136.stdout: "client_mount_gid", 2024-09-12T23:37:44.434 INFO:tasks.workunit.client.0.smithi136.stdout: "client_mount_uid", 2024-09-12T23:37:44.434 INFO:tasks.workunit.client.0.smithi136.stdout: "client_quota", 2024-09-12T23:37:44.434 INFO:tasks.workunit.client.0.smithi136.stdout: "container_image", 2024-09-12T23:37:44.434 INFO:tasks.workunit.client.0.smithi136.stdout: "daemonize", 2024-09-12T23:37:44.434 INFO:tasks.workunit.client.0.smithi136.stdout: "debug_asserts_on_shutdown", 2024-09-12T23:37:44.434 INFO:tasks.workunit.client.0.smithi136.stdout: "debug_mgr", 2024-09-12T23:37:44.434 INFO:tasks.workunit.client.0.smithi136.stdout: "debug_mon", 2024-09-12T23:37:44.434 INFO:tasks.workunit.client.0.smithi136.stdout: "debug_ms", 2024-09-12T23:37:44.434 INFO:tasks.workunit.client.0.smithi136.stdout: "debug_osd", 2024-09-12T23:37:44.434 INFO:tasks.workunit.client.0.smithi136.stdout: "debug_paxos", 2024-09-12T23:37:44.434 INFO:tasks.workunit.client.0.smithi136.stdout: "keyring", 2024-09-12T23:37:44.434 INFO:tasks.workunit.client.0.smithi136.stdout: "leveldb_block_size", 2024-09-12T23:37:44.435 INFO:tasks.workunit.client.0.smithi136.stdout: "leveldb_cache_size", 2024-09-12T23:37:44.435 INFO:tasks.workunit.client.0.smithi136.stdout: "leveldb_compression", 2024-09-12T23:37:44.435 INFO:tasks.workunit.client.0.smithi136.stdout: "leveldb_log", 2024-09-12T23:37:44.435 INFO:tasks.workunit.client.0.smithi136.stdout: "leveldb_write_buffer_size", 2024-09-12T23:37:44.435 INFO:tasks.workunit.client.0.smithi136.stdout: "log_stderr_prefix", 2024-09-12T23:37:44.435 INFO:tasks.workunit.client.0.smithi136.stdout: "log_to_file", 2024-09-12T23:37:44.435 INFO:tasks.workunit.client.0.smithi136.stdout: "log_to_journald", 2024-09-12T23:37:44.435 INFO:tasks.workunit.client.0.smithi136.stdout: "log_to_stderr", 2024-09-12T23:37:44.435 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_allow_pool_delete", 2024-09-12T23:37:44.435 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_clock_drift_allowed", 2024-09-12T23:37:44.435 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_cluster_log_to_file", 2024-09-12T23:37:44.435 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_cluster_log_to_stderr", 2024-09-12T23:37:44.435 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_data_avail_warn", 2024-09-12T23:37:44.436 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_election_default_strategy", 2024-09-12T23:37:44.436 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_host", 2024-09-12T23:37:44.436 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_max_pg_per_osd", 2024-09-12T23:37:44.436 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_mgr_mkfs_grace", 2024-09-12T23:37:44.436 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_osd_prime_pg_temp", 2024-09-12T23:37:44.436 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_osd_reporter_subtree_level", 2024-09-12T23:37:44.436 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_pg_warn_max_object_skew", 2024-09-12T23:37:44.436 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_reweight_min_bytes_per_osd", 2024-09-12T23:37:44.436 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_reweight_min_pgs_per_osd", 2024-09-12T23:37:44.436 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_crush_straw_calc_version_zero", 2024-09-12T23:37:44.436 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_insecure_global_id_reclaim", 2024-09-12T23:37:44.436 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed", 2024-09-12T23:37:44.437 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_legacy_crush_tunables", 2024-09-12T23:37:44.437 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_osd_down_out_interval_zero", 2024-09-12T23:37:44.437 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_pool_pg_num_not_power_of_two", 2024-09-12T23:37:44.437 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_too_few_osds", 2024-09-12T23:37:44.437 INFO:tasks.workunit.client.0.smithi136.stdout: "ms_die_on_bug", 2024-09-12T23:37:44.437 INFO:tasks.workunit.client.0.smithi136.stdout: "ms_die_on_old_message", 2024-09-12T23:37:44.437 INFO:tasks.workunit.client.0.smithi136.stdout: "no_config_file", 2024-09-12T23:37:44.437 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_class_default_list", 2024-09-12T23:37:44.437 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_class_load_list", 2024-09-12T23:37:44.437 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_misdirected_ops", 2024-09-12T23:37:44.437 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_op_order", 2024-09-12T23:37:44.438 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_pg_log_writeout", 2024-09-12T23:37:44.438 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_shutdown", 2024-09-12T23:37:44.438 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_verify_cached_snaps", 2024-09-12T23:37:44.438 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_verify_missing_on_start", 2024-09-12T23:37:44.438 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_verify_stray_on_activate", 2024-09-12T23:37:44.438 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_deep_scrub_update_digest_min_age", 2024-09-12T23:37:44.438 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mclock_profile", 2024-09-12T23:37:44.438 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mclock_scheduler_background_best_effort_lim", 2024-09-12T23:37:44.438 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mclock_scheduler_background_best_effort_res", 2024-09-12T23:37:44.438 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mclock_scheduler_background_best_effort_wgt", 2024-09-12T23:37:44.438 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mclock_scheduler_background_recovery_lim", 2024-09-12T23:37:44.438 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mclock_scheduler_background_recovery_res", 2024-09-12T23:37:44.439 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mclock_scheduler_background_recovery_wgt", 2024-09-12T23:37:44.439 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mclock_scheduler_client_lim", 2024-09-12T23:37:44.439 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mclock_scheduler_client_res", 2024-09-12T23:37:44.439 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mclock_scheduler_client_wgt", 2024-09-12T23:37:44.439 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_memory_target", 2024-09-12T23:37:44.439 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_memory_target_autotune", 2024-09-12T23:37:44.439 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_op_queue", 2024-09-12T23:37:44.439 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_op_queue_cut_off", 2024-09-12T23:37:44.439 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_pool_default_erasure_code_profile", 2024-09-12T23:37:44.439 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_pool_default_pg_autoscale_mode", 2024-09-12T23:37:44.439 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_recover_clone_overlap", 2024-09-12T23:37:44.439 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_recovery_max_chunk", 2024-09-12T23:37:44.440 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_scrub_load_threshold", 2024-09-12T23:37:44.440 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_scrub_max_interval", 2024-09-12T23:37:44.440 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_shutdown_pgref_assert", 2024-09-12T23:37:44.440 INFO:tasks.workunit.client.0.smithi136.stdout: "public_addr", 2024-09-12T23:37:44.440 INFO:tasks.workunit.client.0.smithi136.stdout: "public_addrv", 2024-09-12T23:37:44.440 INFO:tasks.workunit.client.0.smithi136.stdout: "public_network", 2024-09-12T23:37:44.440 INFO:tasks.workunit.client.0.smithi136.stdout: "rbd_default_features", 2024-09-12T23:37:44.440 INFO:tasks.workunit.client.0.smithi136.stdout: "rbd_qos_exclude_ops", 2024-09-12T23:37:44.440 INFO:tasks.workunit.client.0.smithi136.stdout: "setgroup", 2024-09-12T23:37:44.440 INFO:tasks.workunit.client.0.smithi136.stdout: "setuser" 2024-09-12T23:37:44.441 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:44.441 INFO:tasks.workunit.client.0.smithi136.stdout: "cluster_changed": [ 2024-09-12T23:37:44.441 INFO:tasks.workunit.client.0.smithi136.stdout: "auth_allow_insecure_global_id_reclaim", 2024-09-12T23:37:44.441 INFO:tasks.workunit.client.0.smithi136.stdout: "auth_debug", 2024-09-12T23:37:44.441 INFO:tasks.workunit.client.0.smithi136.stdout: "auth_mon_ticket_ttl", 2024-09-12T23:37:44.441 INFO:tasks.workunit.client.0.smithi136.stdout: "auth_service_ticket_ttl", 2024-09-12T23:37:44.441 INFO:tasks.workunit.client.0.smithi136.stdout: "bdev_debug_aio", 2024-09-12T23:37:44.441 INFO:tasks.workunit.client.0.smithi136.stdout: "container_image", 2024-09-12T23:37:44.442 INFO:tasks.workunit.client.0.smithi136.stdout: "debug_asserts_on_shutdown", 2024-09-12T23:37:44.442 INFO:tasks.workunit.client.0.smithi136.stdout: "debug_mgr", 2024-09-12T23:37:44.442 INFO:tasks.workunit.client.0.smithi136.stdout: "debug_mon", 2024-09-12T23:37:44.442 INFO:tasks.workunit.client.0.smithi136.stdout: "debug_ms", 2024-09-12T23:37:44.442 INFO:tasks.workunit.client.0.smithi136.stdout: "debug_osd", 2024-09-12T23:37:44.442 INFO:tasks.workunit.client.0.smithi136.stdout: "debug_paxos", 2024-09-12T23:37:44.442 INFO:tasks.workunit.client.0.smithi136.stdout: "log_to_file", 2024-09-12T23:37:44.442 INFO:tasks.workunit.client.0.smithi136.stdout: "log_to_journald", 2024-09-12T23:37:44.442 INFO:tasks.workunit.client.0.smithi136.stdout: "log_to_stderr", 2024-09-12T23:37:44.442 INFO:tasks.workunit.client.0.smithi136.stdout: "mds_join_fs", 2024-09-12T23:37:44.442 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/cephadm/allow_ptrace", 2024-09-12T23:37:44.442 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/cephadm/container_init", 2024-09-12T23:37:44.442 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/cephadm/migration_current", 2024-09-12T23:37:44.443 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/cephadm/use_repo_digest", 2024-09-12T23:37:44.443 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/dashboard/ALERTMANAGER_API_HOST", 2024-09-12T23:37:44.443 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/dashboard/GRAFANA_API_SSL_VERIFY", 2024-09-12T23:37:44.443 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/dashboard/GRAFANA_API_URL", 2024-09-12T23:37:44.443 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/dashboard/PROMETHEUS_API_HOST", 2024-09-12T23:37:44.443 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/dashboard/ssl_server_port", 2024-09-12T23:37:44.443 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/orchestrator/orchestrator", 2024-09-12T23:37:44.443 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/telemetry/channel_ident", 2024-09-12T23:37:44.443 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/telemetry/description", 2024-09-12T23:37:44.443 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/telemetry/organization", 2024-09-12T23:37:44.443 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_allow_pool_delete", 2024-09-12T23:37:44.443 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_clock_drift_allowed", 2024-09-12T23:37:44.444 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_data_avail_warn", 2024-09-12T23:37:44.444 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_election_default_strategy", 2024-09-12T23:37:44.444 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_max_pg_per_osd", 2024-09-12T23:37:44.444 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_mgr_mkfs_grace", 2024-09-12T23:37:44.444 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_osd_prime_pg_temp", 2024-09-12T23:37:44.444 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_osd_reporter_subtree_level", 2024-09-12T23:37:44.444 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_pg_warn_max_object_skew", 2024-09-12T23:37:44.444 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_reweight_min_bytes_per_osd", 2024-09-12T23:37:44.444 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_reweight_min_pgs_per_osd", 2024-09-12T23:37:44.444 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_crush_straw_calc_version_zero", 2024-09-12T23:37:44.444 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_insecure_global_id_reclaim", 2024-09-12T23:37:44.444 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed", 2024-09-12T23:37:44.444 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_legacy_crush_tunables", 2024-09-12T23:37:44.445 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_osd_down_out_interval_zero", 2024-09-12T23:37:44.445 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_pool_pg_num_not_power_of_two", 2024-09-12T23:37:44.445 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_too_few_osds", 2024-09-12T23:37:44.445 INFO:tasks.workunit.client.0.smithi136.stdout: "ms_die_on_bug", 2024-09-12T23:37:44.445 INFO:tasks.workunit.client.0.smithi136.stdout: "ms_die_on_old_message", 2024-09-12T23:37:44.445 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_class_default_list", 2024-09-12T23:37:44.445 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_class_load_list", 2024-09-12T23:37:44.445 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_misdirected_ops", 2024-09-12T23:37:44.445 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_op_order", 2024-09-12T23:37:44.445 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_pg_log_writeout", 2024-09-12T23:37:44.445 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_shutdown", 2024-09-12T23:37:44.445 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_verify_cached_snaps", 2024-09-12T23:37:44.445 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_verify_missing_on_start", 2024-09-12T23:37:44.446 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_verify_stray_on_activate", 2024-09-12T23:37:44.446 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_deep_scrub_update_digest_min_age", 2024-09-12T23:37:44.446 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mclock_profile", 2024-09-12T23:37:44.446 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_memory_target", 2024-09-12T23:37:44.446 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_memory_target_autotune", 2024-09-12T23:37:44.446 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_op_queue", 2024-09-12T23:37:44.446 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_op_queue_cut_off", 2024-09-12T23:37:44.446 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_pool_default_erasure_code_profile", 2024-09-12T23:37:44.446 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_pool_default_pg_autoscale_mode", 2024-09-12T23:37:44.446 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_recover_clone_overlap", 2024-09-12T23:37:44.446 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_recovery_max_chunk", 2024-09-12T23:37:44.446 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_scrub_load_threshold", 2024-09-12T23:37:44.447 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_scrub_max_interval", 2024-09-12T23:37:44.447 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_shutdown_pgref_assert", 2024-09-12T23:37:44.447 INFO:tasks.workunit.client.0.smithi136.stdout: "public_network", 2024-09-12T23:37:44.447 INFO:tasks.workunit.client.0.smithi136.stdout: "rgw_cache_enabled", 2024-09-12T23:37:44.447 INFO:tasks.workunit.client.0.smithi136.stdout: "rgw_enable_ops_log", 2024-09-12T23:37:44.447 INFO:tasks.workunit.client.0.smithi136.stdout: "rgw_enable_usage_log" 2024-09-12T23:37:44.447 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:44.447 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.447 INFO:tasks.workunit.client.0.smithi136.stdout: "contact": null, 2024-09-12T23:37:44.447 INFO:tasks.workunit.client.0.smithi136.stdout: "crashes": [], 2024-09-12T23:37:44.447 INFO:tasks.workunit.client.0.smithi136.stdout: "created": "2024-09-12T23:23:24.585690Z", 2024-09-12T23:37:44.447 INFO:tasks.workunit.client.0.smithi136.stdout: "crush": { 2024-09-12T23:37:44.447 INFO:tasks.workunit.client.0.smithi136.stdout: "bucket_algs": { 2024-09-12T23:37:44.447 INFO:tasks.workunit.client.0.smithi136.stdout: "straw2": 3 2024-09-12T23:37:44.448 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.448 INFO:tasks.workunit.client.0.smithi136.stdout: "bucket_sizes": { 2024-09-12T23:37:44.448 INFO:tasks.workunit.client.0.smithi136.stdout: "2": 1, 2024-09-12T23:37:44.448 INFO:tasks.workunit.client.0.smithi136.stdout: "4": 2 2024-09-12T23:37:44.448 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.448 INFO:tasks.workunit.client.0.smithi136.stdout: "bucket_types": { 2024-09-12T23:37:44.448 INFO:tasks.workunit.client.0.smithi136.stdout: "1": 2, 2024-09-12T23:37:44.448 INFO:tasks.workunit.client.0.smithi136.stdout: "11": 1 2024-09-12T23:37:44.448 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.448 INFO:tasks.workunit.client.0.smithi136.stdout: "compat_weight_set": false, 2024-09-12T23:37:44.448 INFO:tasks.workunit.client.0.smithi136.stdout: "device_classes": [ 2024-09-12T23:37:44.448 INFO:tasks.workunit.client.0.smithi136.stdout: 8 2024-09-12T23:37:44.448 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:44.449 INFO:tasks.workunit.client.0.smithi136.stdout: "num_buckets": 6, 2024-09-12T23:37:44.449 INFO:tasks.workunit.client.0.smithi136.stdout: "num_devices": 8, 2024-09-12T23:37:44.449 INFO:tasks.workunit.client.0.smithi136.stdout: "num_rules": 1, 2024-09-12T23:37:44.449 INFO:tasks.workunit.client.0.smithi136.stdout: "num_types": 12, 2024-09-12T23:37:44.449 INFO:tasks.workunit.client.0.smithi136.stdout: "num_weight_sets": 0, 2024-09-12T23:37:44.449 INFO:tasks.workunit.client.0.smithi136.stdout: "tunables": { 2024-09-12T23:37:44.449 INFO:tasks.workunit.client.0.smithi136.stdout: "allowed_bucket_algs": 54, 2024-09-12T23:37:44.449 INFO:tasks.workunit.client.0.smithi136.stdout: "choose_local_fallback_tries": 0, 2024-09-12T23:37:44.449 INFO:tasks.workunit.client.0.smithi136.stdout: "choose_local_tries": 0, 2024-09-12T23:37:44.449 INFO:tasks.workunit.client.0.smithi136.stdout: "choose_total_tries": 50, 2024-09-12T23:37:44.449 INFO:tasks.workunit.client.0.smithi136.stdout: "chooseleaf_descend_once": 1, 2024-09-12T23:37:44.449 INFO:tasks.workunit.client.0.smithi136.stdout: "chooseleaf_stable": 1, 2024-09-12T23:37:44.450 INFO:tasks.workunit.client.0.smithi136.stdout: "chooseleaf_vary_r": 1, 2024-09-12T23:37:44.450 INFO:tasks.workunit.client.0.smithi136.stdout: "has_v2_rules": 0, 2024-09-12T23:37:44.450 INFO:tasks.workunit.client.0.smithi136.stdout: "has_v3_rules": 0, 2024-09-12T23:37:44.450 INFO:tasks.workunit.client.0.smithi136.stdout: "has_v4_buckets": 1, 2024-09-12T23:37:44.450 INFO:tasks.workunit.client.0.smithi136.stdout: "has_v5_rules": 0, 2024-09-12T23:37:44.450 INFO:tasks.workunit.client.0.smithi136.stdout: "legacy_tunables": 0, 2024-09-12T23:37:44.450 INFO:tasks.workunit.client.0.smithi136.stdout: "minimum_required_version": "jewel", 2024-09-12T23:37:44.450 INFO:tasks.workunit.client.0.smithi136.stdout: "optimal_tunables": 1, 2024-09-12T23:37:44.450 INFO:tasks.workunit.client.0.smithi136.stdout: "profile": "jewel", 2024-09-12T23:37:44.450 INFO:tasks.workunit.client.0.smithi136.stdout: "require_feature_tunables": 1, 2024-09-12T23:37:44.450 INFO:tasks.workunit.client.0.smithi136.stdout: "require_feature_tunables2": 1, 2024-09-12T23:37:44.450 INFO:tasks.workunit.client.0.smithi136.stdout: "require_feature_tunables3": 1, 2024-09-12T23:37:44.450 INFO:tasks.workunit.client.0.smithi136.stdout: "require_feature_tunables5": 1, 2024-09-12T23:37:44.451 INFO:tasks.workunit.client.0.smithi136.stdout: "straw_calc_version": 1 2024-09-12T23:37:44.451 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:44.451 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.451 INFO:tasks.workunit.client.0.smithi136.stdout: "description": "upgrade test cluster", 2024-09-12T23:37:44.451 INFO:tasks.workunit.client.0.smithi136.stdout: "fs": { 2024-09-12T23:37:44.451 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1, 2024-09-12T23:37:44.451 INFO:tasks.workunit.client.0.smithi136.stdout: "feature_flags": { 2024-09-12T23:37:44.451 INFO:tasks.workunit.client.0.smithi136.stdout: "enable_multiple": true, 2024-09-12T23:37:44.451 INFO:tasks.workunit.client.0.smithi136.stdout: "ever_enabled_multiple": true 2024-09-12T23:37:44.451 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.451 INFO:tasks.workunit.client.0.smithi136.stdout: "filesystems": [ 2024-09-12T23:37:44.451 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:44.451 INFO:tasks.workunit.client.0.smithi136.stdout: "approx_ctime": "2024-09", 2024-09-12T23:37:44.451 INFO:tasks.workunit.client.0.smithi136.stdout: "balancer_enabled": false, 2024-09-12T23:37:44.451 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:44.452 INFO:tasks.workunit.client.0.smithi136.stdout: "cached_caps": 0, 2024-09-12T23:37:44.452 INFO:tasks.workunit.client.0.smithi136.stdout: "cached_dns": 10, 2024-09-12T23:37:44.452 INFO:tasks.workunit.client.0.smithi136.stdout: "cached_inos": 13, 2024-09-12T23:37:44.452 INFO:tasks.workunit.client.0.smithi136.stdout: "cached_subtrees": 2, 2024-09-12T23:37:44.452 INFO:tasks.workunit.client.0.smithi136.stdout: "ever_allowed_features": 0, 2024-09-12T23:37:44.452 INFO:tasks.workunit.client.0.smithi136.stdout: "explicitly_allowed_features": 0, 2024-09-12T23:37:44.452 INFO:tasks.workunit.client.0.smithi136.stdout: "files": 0, 2024-09-12T23:37:44.452 INFO:tasks.workunit.client.0.smithi136.stdout: "max_mds": 1, 2024-09-12T23:37:44.452 INFO:tasks.workunit.client.0.smithi136.stdout: "num_data_pools": 1, 2024-09-12T23:37:44.452 INFO:tasks.workunit.client.0.smithi136.stdout: "num_in": 1, 2024-09-12T23:37:44.452 INFO:tasks.workunit.client.0.smithi136.stdout: "num_mds": 1, 2024-09-12T23:37:44.452 INFO:tasks.workunit.client.0.smithi136.stdout: "num_sessions": 0, 2024-09-12T23:37:44.453 INFO:tasks.workunit.client.0.smithi136.stdout: "num_standby_replay": 0, 2024-09-12T23:37:44.453 INFO:tasks.workunit.client.0.smithi136.stdout: "num_up": 1, 2024-09-12T23:37:44.453 INFO:tasks.workunit.client.0.smithi136.stdout: "snaps": 0, 2024-09-12T23:37:44.453 INFO:tasks.workunit.client.0.smithi136.stdout: "standby_count_wanted": 0 2024-09-12T23:37:44.453 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:44.453 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:44.453 INFO:tasks.workunit.client.0.smithi136.stdout: "num_standby_mds": 1, 2024-09-12T23:37:44.453 INFO:tasks.workunit.client.0.smithi136.stdout: "total_num_mds": 2 2024-09-12T23:37:44.453 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.453 INFO:tasks.workunit.client.0.smithi136.stdout: "hosts": { 2024-09-12T23:37:44.453 INFO:tasks.workunit.client.0.smithi136.stdout: "num": 2, 2024-09-12T23:37:44.453 INFO:tasks.workunit.client.0.smithi136.stdout: "num_with_mds": 2, 2024-09-12T23:37:44.454 INFO:tasks.workunit.client.0.smithi136.stdout: "num_with_mgr": 2, 2024-09-12T23:37:44.454 INFO:tasks.workunit.client.0.smithi136.stdout: "num_with_mon": 2, 2024-09-12T23:37:44.454 INFO:tasks.workunit.client.0.smithi136.stdout: "num_with_osd": 2 2024-09-12T23:37:44.454 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.454 INFO:tasks.workunit.client.0.smithi136.stdout: "leaderboard": false, 2024-09-12T23:37:44.454 INFO:tasks.workunit.client.0.smithi136.stdout: "license": "sharing-1-0", 2024-09-12T23:37:44.454 INFO:tasks.workunit.client.0.smithi136.stdout: "metadata": { 2024-09-12T23:37:44.454 INFO:tasks.workunit.client.0.smithi136.stdout: "mds": { 2024-09-12T23:37:44.454 INFO:tasks.workunit.client.0.smithi136.stdout: "arch": { 2024-09-12T23:37:44.454 INFO:tasks.workunit.client.0.smithi136.stdout: "x86_64": 2 2024-09-12T23:37:44.454 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.454 INFO:tasks.workunit.client.0.smithi136.stdout: "ceph_version": { 2024-09-12T23:37:44.454 INFO:tasks.workunit.client.0.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-12T23:37:44.454 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.455 INFO:tasks.workunit.client.0.smithi136.stdout: "cpu": { 2024-09-12T23:37:44.455 INFO:tasks.workunit.client.0.smithi136.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 2 2024-09-12T23:37:44.455 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.455 INFO:tasks.workunit.client.0.smithi136.stdout: "distro": { 2024-09-12T23:37:44.455 INFO:tasks.workunit.client.0.smithi136.stdout: "centos": 2 2024-09-12T23:37:44.455 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.455 INFO:tasks.workunit.client.0.smithi136.stdout: "distro_description": { 2024-09-12T23:37:44.455 INFO:tasks.workunit.client.0.smithi136.stdout: "CentOS Stream 9": 2 2024-09-12T23:37:44.455 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.455 INFO:tasks.workunit.client.0.smithi136.stdout: "kernel_description": { 2024-09-12T23:37:44.455 INFO:tasks.workunit.client.0.smithi136.stdout: "#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024": 2 2024-09-12T23:37:44.455 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.455 INFO:tasks.workunit.client.0.smithi136.stdout: "kernel_version": { 2024-09-12T23:37:44.455 INFO:tasks.workunit.client.0.smithi136.stdout: "5.15.0-121-generic": 2 2024-09-12T23:37:44.455 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.456 INFO:tasks.workunit.client.0.smithi136.stdout: "os": { 2024-09-12T23:37:44.456 INFO:tasks.workunit.client.0.smithi136.stdout: "Linux": 2 2024-09-12T23:37:44.456 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:44.456 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.456 INFO:tasks.workunit.client.0.smithi136.stdout: "mon": { 2024-09-12T23:37:44.456 INFO:tasks.workunit.client.0.smithi136.stdout: "arch": { 2024-09-12T23:37:44.457 INFO:tasks.workunit.client.0.smithi136.stdout: "x86_64": 3 2024-09-12T23:37:44.457 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.457 INFO:tasks.workunit.client.0.smithi136.stdout: "ceph_version": { 2024-09-12T23:37:44.457 INFO:tasks.workunit.client.0.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 3 2024-09-12T23:37:44.457 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.458 INFO:tasks.workunit.client.0.smithi136.stdout: "cpu": { 2024-09-12T23:37:44.458 INFO:tasks.workunit.client.0.smithi136.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 3 2024-09-12T23:37:44.458 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.458 INFO:tasks.workunit.client.0.smithi136.stdout: "distro": { 2024-09-12T23:37:44.458 INFO:tasks.workunit.client.0.smithi136.stdout: "centos": 3 2024-09-12T23:37:44.458 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.458 INFO:tasks.workunit.client.0.smithi136.stdout: "distro_description": { 2024-09-12T23:37:44.458 INFO:tasks.workunit.client.0.smithi136.stdout: "CentOS Stream 9": 3 2024-09-12T23:37:44.458 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.458 INFO:tasks.workunit.client.0.smithi136.stdout: "kernel_description": { 2024-09-12T23:37:44.458 INFO:tasks.workunit.client.0.smithi136.stdout: "#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024": 3 2024-09-12T23:37:44.459 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.459 INFO:tasks.workunit.client.0.smithi136.stdout: "kernel_version": { 2024-09-12T23:37:44.459 INFO:tasks.workunit.client.0.smithi136.stdout: "5.15.0-121-generic": 3 2024-09-12T23:37:44.459 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.459 INFO:tasks.workunit.client.0.smithi136.stdout: "os": { 2024-09-12T23:37:44.459 INFO:tasks.workunit.client.0.smithi136.stdout: "Linux": 3 2024-09-12T23:37:44.459 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:44.459 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.459 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:44.459 INFO:tasks.workunit.client.0.smithi136.stdout: "arch": { 2024-09-12T23:37:44.459 INFO:tasks.workunit.client.0.smithi136.stdout: "x86_64": 8 2024-09-12T23:37:44.459 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.459 INFO:tasks.workunit.client.0.smithi136.stdout: "ceph_version": { 2024-09-12T23:37:44.459 INFO:tasks.workunit.client.0.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 8 2024-09-12T23:37:44.460 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.460 INFO:tasks.workunit.client.0.smithi136.stdout: "cpu": { 2024-09-12T23:37:44.460 INFO:tasks.workunit.client.0.smithi136.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 8 2024-09-12T23:37:44.460 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.460 INFO:tasks.workunit.client.0.smithi136.stdout: "distro": { 2024-09-12T23:37:44.460 INFO:tasks.workunit.client.0.smithi136.stdout: "centos": 8 2024-09-12T23:37:44.460 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.460 INFO:tasks.workunit.client.0.smithi136.stdout: "distro_description": { 2024-09-12T23:37:44.460 INFO:tasks.workunit.client.0.smithi136.stdout: "CentOS Stream 9": 8 2024-09-12T23:37:44.460 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.460 INFO:tasks.workunit.client.0.smithi136.stdout: "kernel_description": { 2024-09-12T23:37:44.460 INFO:tasks.workunit.client.0.smithi136.stdout: "#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024": 8 2024-09-12T23:37:44.460 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.460 INFO:tasks.workunit.client.0.smithi136.stdout: "kernel_version": { 2024-09-12T23:37:44.461 INFO:tasks.workunit.client.0.smithi136.stdout: "5.15.0-121-generic": 8 2024-09-12T23:37:44.461 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.461 INFO:tasks.workunit.client.0.smithi136.stdout: "os": { 2024-09-12T23:37:44.461 INFO:tasks.workunit.client.0.smithi136.stdout: "Linux": 8 2024-09-12T23:37:44.461 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.461 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_objectstore": { 2024-09-12T23:37:44.461 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore": 8 2024-09-12T23:37:44.461 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.461 INFO:tasks.workunit.client.0.smithi136.stdout: "rotational": { 2024-09-12T23:37:44.461 INFO:tasks.workunit.client.0.smithi136.stdout: "0": 8 2024-09-12T23:37:44.461 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:44.461 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:44.461 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.461 INFO:tasks.workunit.client.0.smithi136.stdout: "mon": { 2024-09-12T23:37:44.462 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 3, 2024-09-12T23:37:44.462 INFO:tasks.workunit.client.0.smithi136.stdout: "features": { 2024-09-12T23:37:44.462 INFO:tasks.workunit.client.0.smithi136.stdout: "optional": [], 2024-09-12T23:37:44.462 INFO:tasks.workunit.client.0.smithi136.stdout: "persistent": [ 2024-09-12T23:37:44.462 INFO:tasks.workunit.client.0.smithi136.stdout: "kraken", 2024-09-12T23:37:44.462 INFO:tasks.workunit.client.0.smithi136.stdout: "luminous", 2024-09-12T23:37:44.462 INFO:tasks.workunit.client.0.smithi136.stdout: "mimic", 2024-09-12T23:37:44.462 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap-prune", 2024-09-12T23:37:44.462 INFO:tasks.workunit.client.0.smithi136.stdout: "nautilus", 2024-09-12T23:37:44.462 INFO:tasks.workunit.client.0.smithi136.stdout: "octopus", 2024-09-12T23:37:44.462 INFO:tasks.workunit.client.0.smithi136.stdout: "pacific", 2024-09-12T23:37:44.462 INFO:tasks.workunit.client.0.smithi136.stdout: "elector-pinging", 2024-09-12T23:37:44.462 INFO:tasks.workunit.client.0.smithi136.stdout: "quincy" 2024-09-12T23:37:44.462 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:44.462 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.463 INFO:tasks.workunit.client.0.smithi136.stdout: "ipv4_addr_mons": 6, 2024-09-12T23:37:44.463 INFO:tasks.workunit.client.0.smithi136.stdout: "ipv6_addr_mons": 0, 2024-09-12T23:37:44.463 INFO:tasks.workunit.client.0.smithi136.stdout: "min_mon_release": 17, 2024-09-12T23:37:44.463 INFO:tasks.workunit.client.0.smithi136.stdout: "v1_addr_mons": 3, 2024-09-12T23:37:44.463 INFO:tasks.workunit.client.0.smithi136.stdout: "v2_addr_mons": 3 2024-09-12T23:37:44.463 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.463 INFO:tasks.workunit.client.0.smithi136.stdout: "organization": "ceph-qa", 2024-09-12T23:37:44.463 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:44.463 INFO:tasks.workunit.client.0.smithi136.stdout: "cluster_network": false, 2024-09-12T23:37:44.463 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 8, 2024-09-12T23:37:44.463 INFO:tasks.workunit.client.0.smithi136.stdout: "require_min_compat_client": "luminous", 2024-09-12T23:37:44.464 INFO:tasks.workunit.client.0.smithi136.stdout: "require_osd_release": "quincy" 2024-09-12T23:37:44.464 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.464 INFO:tasks.workunit.client.0.smithi136.stdout: "pools": [ 2024-09-12T23:37:44.464 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:44.464 INFO:tasks.workunit.client.0.smithi136.stdout: "application": [ 2024-09-12T23:37:44.464 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr" 2024-09-12T23:37:44.464 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:44.464 INFO:tasks.workunit.client.0.smithi136.stdout: "cache_mode": "none", 2024-09-12T23:37:44.464 INFO:tasks.workunit.client.0.smithi136.stdout: "erasure_code_profile": {}, 2024-09-12T23:37:44.464 INFO:tasks.workunit.client.0.smithi136.stdout: "min_size": 2, 2024-09-12T23:37:44.464 INFO:tasks.workunit.client.0.smithi136.stdout: "options": {}, 2024-09-12T23:37:44.464 INFO:tasks.workunit.client.0.smithi136.stdout: "pg_autoscale_mode": "off", 2024-09-12T23:37:44.464 INFO:tasks.workunit.client.0.smithi136.stdout: "pg_num": 1, 2024-09-12T23:37:44.465 INFO:tasks.workunit.client.0.smithi136.stdout: "pgp_num": 1, 2024-09-12T23:37:44.465 INFO:tasks.workunit.client.0.smithi136.stdout: "pool": 1, 2024-09-12T23:37:44.465 INFO:tasks.workunit.client.0.smithi136.stdout: "size": 3, 2024-09-12T23:37:44.465 INFO:tasks.workunit.client.0.smithi136.stdout: "stats": { 2024-09-12T23:37:44.465 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_raw": 727116228874, 2024-09-12T23:37:44.465 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_used": 1781760, 2024-09-12T23:37:44.465 INFO:tasks.workunit.client.0.smithi136.stdout: "compress_bytes_used": 0, 2024-09-12T23:37:44.465 INFO:tasks.workunit.client.0.smithi136.stdout: "compress_under_bytes": 0, 2024-09-12T23:37:44.465 INFO:tasks.workunit.client.0.smithi136.stdout: "data_bytes_used": 1781760, 2024-09-12T23:37:44.465 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty": 0, 2024-09-12T23:37:44.465 INFO:tasks.workunit.client.0.smithi136.stdout: "max_avail": 242372067328, 2024-09-12T23:37:44.465 INFO:tasks.workunit.client.0.smithi136.stdout: "objects": 2, 2024-09-12T23:37:44.465 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_bytes_used": 0, 2024-09-12T23:37:44.465 INFO:tasks.workunit.client.0.smithi136.stdout: "percent_used": 2.4504413431714056e-06, 2024-09-12T23:37:44.466 INFO:tasks.workunit.client.0.smithi136.stdout: "quota_bytes": 0, 2024-09-12T23:37:44.466 INFO:tasks.workunit.client.0.smithi136.stdout: "quota_objects": 0, 2024-09-12T23:37:44.466 INFO:tasks.workunit.client.0.smithi136.stdout: "rd": 126, 2024-09-12T23:37:44.466 INFO:tasks.workunit.client.0.smithi136.stdout: "rd_bytes": 236544, 2024-09-12T23:37:44.466 INFO:tasks.workunit.client.0.smithi136.stdout: "stored": 590368, 2024-09-12T23:37:44.466 INFO:tasks.workunit.client.0.smithi136.stdout: "stored_data": 590368, 2024-09-12T23:37:44.466 INFO:tasks.workunit.client.0.smithi136.stdout: "stored_omap": 0, 2024-09-12T23:37:44.466 INFO:tasks.workunit.client.0.smithi136.stdout: "stored_raw": 1771104, 2024-09-12T23:37:44.466 INFO:tasks.workunit.client.0.smithi136.stdout: "wr": 113, 2024-09-12T23:37:44.466 INFO:tasks.workunit.client.0.smithi136.stdout: "wr_bytes": 1667072 2024-09-12T23:37:44.466 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.466 INFO:tasks.workunit.client.0.smithi136.stdout: "target_max_bytes": 0, 2024-09-12T23:37:44.466 INFO:tasks.workunit.client.0.smithi136.stdout: "target_max_objects": 0, 2024-09-12T23:37:44.467 INFO:tasks.workunit.client.0.smithi136.stdout: "type": "replicated" 2024-09-12T23:37:44.467 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.467 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:44.467 INFO:tasks.workunit.client.0.smithi136.stdout: "application": [ 2024-09-12T23:37:44.467 INFO:tasks.workunit.client.0.smithi136.stdout: "cephfs" 2024-09-12T23:37:44.467 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:44.467 INFO:tasks.workunit.client.0.smithi136.stdout: "cache_mode": "none", 2024-09-12T23:37:44.467 INFO:tasks.workunit.client.0.smithi136.stdout: "erasure_code_profile": {}, 2024-09-12T23:37:44.467 INFO:tasks.workunit.client.0.smithi136.stdout: "min_size": 2, 2024-09-12T23:37:44.467 INFO:tasks.workunit.client.0.smithi136.stdout: "options": {}, 2024-09-12T23:37:44.467 INFO:tasks.workunit.client.0.smithi136.stdout: "pg_autoscale_mode": "off", 2024-09-12T23:37:44.467 INFO:tasks.workunit.client.0.smithi136.stdout: "pg_num": 32, 2024-09-12T23:37:44.467 INFO:tasks.workunit.client.0.smithi136.stdout: "pgp_num": 32, 2024-09-12T23:37:44.467 INFO:tasks.workunit.client.0.smithi136.stdout: "pool": 2, 2024-09-12T23:37:44.467 INFO:tasks.workunit.client.0.smithi136.stdout: "size": 3, 2024-09-12T23:37:44.467 INFO:tasks.workunit.client.0.smithi136.stdout: "stats": { 2024-09-12T23:37:44.468 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_raw": 727116228874, 2024-09-12T23:37:44.468 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_used": 98304, 2024-09-12T23:37:44.468 INFO:tasks.workunit.client.0.smithi136.stdout: "compress_bytes_used": 0, 2024-09-12T23:37:44.468 INFO:tasks.workunit.client.0.smithi136.stdout: "compress_under_bytes": 0, 2024-09-12T23:37:44.468 INFO:tasks.workunit.client.0.smithi136.stdout: "data_bytes_used": 98304, 2024-09-12T23:37:44.468 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty": 0, 2024-09-12T23:37:44.468 INFO:tasks.workunit.client.0.smithi136.stdout: "max_avail": 242372067328, 2024-09-12T23:37:44.468 INFO:tasks.workunit.client.0.smithi136.stdout: "objects": 22, 2024-09-12T23:37:44.468 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_bytes_used": 0, 2024-09-12T23:37:44.468 INFO:tasks.workunit.client.0.smithi136.stdout: "percent_used": 1.3519708375042683e-07, 2024-09-12T23:37:44.468 INFO:tasks.workunit.client.0.smithi136.stdout: "quota_bytes": 0, 2024-09-12T23:37:44.469 INFO:tasks.workunit.client.0.smithi136.stdout: "quota_objects": 0, 2024-09-12T23:37:44.469 INFO:tasks.workunit.client.0.smithi136.stdout: "rd": 0, 2024-09-12T23:37:44.469 INFO:tasks.workunit.client.0.smithi136.stdout: "rd_bytes": 0, 2024-09-12T23:37:44.469 INFO:tasks.workunit.client.0.smithi136.stdout: "stored": 3410, 2024-09-12T23:37:44.469 INFO:tasks.workunit.client.0.smithi136.stdout: "stored_data": 3410, 2024-09-12T23:37:44.469 INFO:tasks.workunit.client.0.smithi136.stdout: "stored_omap": 0, 2024-09-12T23:37:44.469 INFO:tasks.workunit.client.0.smithi136.stdout: "stored_raw": 10230, 2024-09-12T23:37:44.469 INFO:tasks.workunit.client.0.smithi136.stdout: "wr": 51, 2024-09-12T23:37:44.469 INFO:tasks.workunit.client.0.smithi136.stdout: "wr_bytes": 19456 2024-09-12T23:37:44.469 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.469 INFO:tasks.workunit.client.0.smithi136.stdout: "target_max_bytes": 0, 2024-09-12T23:37:44.469 INFO:tasks.workunit.client.0.smithi136.stdout: "target_max_objects": 0, 2024-09-12T23:37:44.469 INFO:tasks.workunit.client.0.smithi136.stdout: "type": "replicated" 2024-09-12T23:37:44.470 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.470 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:44.470 INFO:tasks.workunit.client.0.smithi136.stdout: "application": [ 2024-09-12T23:37:44.470 INFO:tasks.workunit.client.0.smithi136.stdout: "cephfs" 2024-09-12T23:37:44.470 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:44.470 INFO:tasks.workunit.client.0.smithi136.stdout: "cache_mode": "none", 2024-09-12T23:37:44.470 INFO:tasks.workunit.client.0.smithi136.stdout: "erasure_code_profile": {}, 2024-09-12T23:37:44.470 INFO:tasks.workunit.client.0.smithi136.stdout: "min_size": 2, 2024-09-12T23:37:44.470 INFO:tasks.workunit.client.0.smithi136.stdout: "options": {}, 2024-09-12T23:37:44.470 INFO:tasks.workunit.client.0.smithi136.stdout: "pg_autoscale_mode": "off", 2024-09-12T23:37:44.470 INFO:tasks.workunit.client.0.smithi136.stdout: "pg_num": 64, 2024-09-12T23:37:44.470 INFO:tasks.workunit.client.0.smithi136.stdout: "pgp_num": 64, 2024-09-12T23:37:44.470 INFO:tasks.workunit.client.0.smithi136.stdout: "pool": 3, 2024-09-12T23:37:44.470 INFO:tasks.workunit.client.0.smithi136.stdout: "size": 3, 2024-09-12T23:37:44.471 INFO:tasks.workunit.client.0.smithi136.stdout: "stats": { 2024-09-12T23:37:44.471 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_raw": 727116228874, 2024-09-12T23:37:44.471 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_used": 0, 2024-09-12T23:37:44.471 INFO:tasks.workunit.client.0.smithi136.stdout: "compress_bytes_used": 0, 2024-09-12T23:37:44.471 INFO:tasks.workunit.client.0.smithi136.stdout: "compress_under_bytes": 0, 2024-09-12T23:37:44.471 INFO:tasks.workunit.client.0.smithi136.stdout: "data_bytes_used": 0, 2024-09-12T23:37:44.471 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty": 0, 2024-09-12T23:37:44.471 INFO:tasks.workunit.client.0.smithi136.stdout: "max_avail": 242372067328, 2024-09-12T23:37:44.471 INFO:tasks.workunit.client.0.smithi136.stdout: "objects": 0, 2024-09-12T23:37:44.471 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_bytes_used": 0, 2024-09-12T23:37:44.471 INFO:tasks.workunit.client.0.smithi136.stdout: "percent_used": 0.0, 2024-09-12T23:37:44.472 INFO:tasks.workunit.client.0.smithi136.stdout: "quota_bytes": 0, 2024-09-12T23:37:44.472 INFO:tasks.workunit.client.0.smithi136.stdout: "quota_objects": 0, 2024-09-12T23:37:44.472 INFO:tasks.workunit.client.0.smithi136.stdout: "rd": 0, 2024-09-12T23:37:44.472 INFO:tasks.workunit.client.0.smithi136.stdout: "rd_bytes": 0, 2024-09-12T23:37:44.472 INFO:tasks.workunit.client.0.smithi136.stdout: "stored": 0, 2024-09-12T23:37:44.472 INFO:tasks.workunit.client.0.smithi136.stdout: "stored_data": 0, 2024-09-12T23:37:44.472 INFO:tasks.workunit.client.0.smithi136.stdout: "stored_omap": 0, 2024-09-12T23:37:44.472 INFO:tasks.workunit.client.0.smithi136.stdout: "stored_raw": 0, 2024-09-12T23:37:44.472 INFO:tasks.workunit.client.0.smithi136.stdout: "wr": 0, 2024-09-12T23:37:44.472 INFO:tasks.workunit.client.0.smithi136.stdout: "wr_bytes": 0 2024-09-12T23:37:44.472 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.473 INFO:tasks.workunit.client.0.smithi136.stdout: "target_max_bytes": 0, 2024-09-12T23:37:44.473 INFO:tasks.workunit.client.0.smithi136.stdout: "target_max_objects": 0, 2024-09-12T23:37:44.473 INFO:tasks.workunit.client.0.smithi136.stdout: "type": "replicated" 2024-09-12T23:37:44.473 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.473 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:44.473 INFO:tasks.workunit.client.0.smithi136.stdout: "application": [ 2024-09-12T23:37:44.473 INFO:tasks.workunit.client.0.smithi136.stdout: "rbd" 2024-09-12T23:37:44.473 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:44.473 INFO:tasks.workunit.client.0.smithi136.stdout: "cache_mode": "none", 2024-09-12T23:37:44.473 INFO:tasks.workunit.client.0.smithi136.stdout: "erasure_code_profile": {}, 2024-09-12T23:37:44.473 INFO:tasks.workunit.client.0.smithi136.stdout: "min_size": 2, 2024-09-12T23:37:44.473 INFO:tasks.workunit.client.0.smithi136.stdout: "options": {}, 2024-09-12T23:37:44.473 INFO:tasks.workunit.client.0.smithi136.stdout: "pg_autoscale_mode": "off", 2024-09-12T23:37:44.473 INFO:tasks.workunit.client.0.smithi136.stdout: "pg_num": 8, 2024-09-12T23:37:44.474 INFO:tasks.workunit.client.0.smithi136.stdout: "pgp_num": 8, 2024-09-12T23:37:44.474 INFO:tasks.workunit.client.0.smithi136.stdout: "pool": 4, 2024-09-12T23:37:44.474 INFO:tasks.workunit.client.0.smithi136.stdout: "size": 3, 2024-09-12T23:37:44.474 INFO:tasks.workunit.client.0.smithi136.stdout: "stats": { 2024-09-12T23:37:44.474 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_raw": 727116228874, 2024-09-12T23:37:44.474 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_used": 12288, 2024-09-12T23:37:44.474 INFO:tasks.workunit.client.0.smithi136.stdout: "compress_bytes_used": 0, 2024-09-12T23:37:44.474 INFO:tasks.workunit.client.0.smithi136.stdout: "compress_under_bytes": 0, 2024-09-12T23:37:44.474 INFO:tasks.workunit.client.0.smithi136.stdout: "data_bytes_used": 12288, 2024-09-12T23:37:44.474 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty": 0, 2024-09-12T23:37:44.474 INFO:tasks.workunit.client.0.smithi136.stdout: "max_avail": 242372067328, 2024-09-12T23:37:44.474 INFO:tasks.workunit.client.0.smithi136.stdout: "objects": 2, 2024-09-12T23:37:44.474 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_bytes_used": 0, 2024-09-12T23:37:44.474 INFO:tasks.workunit.client.0.smithi136.stdout: "percent_used": 1.6899637245160193e-08, 2024-09-12T23:37:44.474 INFO:tasks.workunit.client.0.smithi136.stdout: "quota_bytes": 0, 2024-09-12T23:37:44.475 INFO:tasks.workunit.client.0.smithi136.stdout: "quota_objects": 0, 2024-09-12T23:37:44.475 INFO:tasks.workunit.client.0.smithi136.stdout: "rd": 0, 2024-09-12T23:37:44.475 INFO:tasks.workunit.client.0.smithi136.stdout: "rd_bytes": 0, 2024-09-12T23:37:44.475 INFO:tasks.workunit.client.0.smithi136.stdout: "stored": 19, 2024-09-12T23:37:44.475 INFO:tasks.workunit.client.0.smithi136.stdout: "stored_data": 19, 2024-09-12T23:37:44.475 INFO:tasks.workunit.client.0.smithi136.stdout: "stored_omap": 0, 2024-09-12T23:37:44.475 INFO:tasks.workunit.client.0.smithi136.stdout: "stored_raw": 57, 2024-09-12T23:37:44.475 INFO:tasks.workunit.client.0.smithi136.stdout: "wr": 2, 2024-09-12T23:37:44.475 INFO:tasks.workunit.client.0.smithi136.stdout: "wr_bytes": 2048 2024-09-12T23:37:44.475 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.475 INFO:tasks.workunit.client.0.smithi136.stdout: "target_max_bytes": 0, 2024-09-12T23:37:44.475 INFO:tasks.workunit.client.0.smithi136.stdout: "target_max_objects": 0, 2024-09-12T23:37:44.476 INFO:tasks.workunit.client.0.smithi136.stdout: "type": "replicated" 2024-09-12T23:37:44.476 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:44.476 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:44.476 INFO:tasks.workunit.client.0.smithi136.stdout: "rbd": { 2024-09-12T23:37:44.476 INFO:tasks.workunit.client.0.smithi136.stdout: "mirroring_by_pool": [ 2024-09-12T23:37:44.476 INFO:tasks.workunit.client.0.smithi136.stdout: false 2024-09-12T23:37:44.476 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:44.476 INFO:tasks.workunit.client.0.smithi136.stdout: "num_images_by_pool": [ 2024-09-12T23:37:44.476 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:44.476 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:44.476 INFO:tasks.workunit.client.0.smithi136.stdout: "num_pools": 1 2024-09-12T23:37:44.476 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.476 INFO:tasks.workunit.client.0.smithi136.stdout: "report_id": "679f567b-70cd-4d20-9c08-b6621403ecae", 2024-09-12T23:37:44.477 INFO:tasks.workunit.client.0.smithi136.stdout: "report_timestamp": "2024-09-12T23:37:44.422705", 2024-09-12T23:37:44.477 INFO:tasks.workunit.client.0.smithi136.stdout: "report_version": 1, 2024-09-12T23:37:44.477 INFO:tasks.workunit.client.0.smithi136.stdout: "rook": { 2024-09-12T23:37:44.477 INFO:tasks.workunit.client.0.smithi136.stdout: "cluster": { 2024-09-12T23:37:44.477 INFO:tasks.workunit.client.0.smithi136.stdout: "external-mode": null, 2024-09-12T23:37:44.477 INFO:tasks.workunit.client.0.smithi136.stdout: "mon": { 2024-09-12T23:37:44.477 INFO:tasks.workunit.client.0.smithi136.stdout: "allow-multiple-per-node": null, 2024-09-12T23:37:44.477 INFO:tasks.workunit.client.0.smithi136.stdout: "count": null, 2024-09-12T23:37:44.477 INFO:tasks.workunit.client.0.smithi136.stdout: "max-id": null, 2024-09-12T23:37:44.477 INFO:tasks.workunit.client.0.smithi136.stdout: "pvc": { 2024-09-12T23:37:44.477 INFO:tasks.workunit.client.0.smithi136.stdout: "enabled": null 2024-09-12T23:37:44.477 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.477 INFO:tasks.workunit.client.0.smithi136.stdout: "stretch": { 2024-09-12T23:37:44.477 INFO:tasks.workunit.client.0.smithi136.stdout: "enabled": null 2024-09-12T23:37:44.478 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:44.478 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.478 INFO:tasks.workunit.client.0.smithi136.stdout: "network": { 2024-09-12T23:37:44.478 INFO:tasks.workunit.client.0.smithi136.stdout: "provider": null 2024-09-12T23:37:44.478 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.478 INFO:tasks.workunit.client.0.smithi136.stdout: "storage": { 2024-09-12T23:37:44.478 INFO:tasks.workunit.client.0.smithi136.stdout: "device-set": { 2024-09-12T23:37:44.478 INFO:tasks.workunit.client.0.smithi136.stdout: "count": { 2024-09-12T23:37:44.478 INFO:tasks.workunit.client.0.smithi136.stdout: "non-portable": null, 2024-09-12T23:37:44.478 INFO:tasks.workunit.client.0.smithi136.stdout: "portable": null, 2024-09-12T23:37:44.478 INFO:tasks.workunit.client.0.smithi136.stdout: "total": null 2024-09-12T23:37:44.478 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:44.478 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:44.478 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:44.478 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.479 INFO:tasks.workunit.client.0.smithi136.stdout: "csi": { 2024-09-12T23:37:44.479 INFO:tasks.workunit.client.0.smithi136.stdout: "version": null 2024-09-12T23:37:44.479 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.479 INFO:tasks.workunit.client.0.smithi136.stdout: "kubernetes": { 2024-09-12T23:37:44.479 INFO:tasks.workunit.client.0.smithi136.stdout: "version": null 2024-09-12T23:37:44.479 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.479 INFO:tasks.workunit.client.0.smithi136.stdout: "node": { 2024-09-12T23:37:44.479 INFO:tasks.workunit.client.0.smithi136.stdout: "count": { 2024-09-12T23:37:44.479 INFO:tasks.workunit.client.0.smithi136.stdout: "kubernetes-total": null, 2024-09-12T23:37:44.479 INFO:tasks.workunit.client.0.smithi136.stdout: "with-ceph-daemons": null, 2024-09-12T23:37:44.479 INFO:tasks.workunit.client.0.smithi136.stdout: "with-csi-cephfs-plugin": null, 2024-09-12T23:37:44.479 INFO:tasks.workunit.client.0.smithi136.stdout: "with-csi-nfs-plugin": null, 2024-09-12T23:37:44.479 INFO:tasks.workunit.client.0.smithi136.stdout: "with-csi-rbd-plugin": null 2024-09-12T23:37:44.479 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:44.479 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.480 INFO:tasks.workunit.client.0.smithi136.stdout: "usage": { 2024-09-12T23:37:44.480 INFO:tasks.workunit.client.0.smithi136.stdout: "storage-class": { 2024-09-12T23:37:44.480 INFO:tasks.workunit.client.0.smithi136.stdout: "count": { 2024-09-12T23:37:44.480 INFO:tasks.workunit.client.0.smithi136.stdout: "bucket": null, 2024-09-12T23:37:44.480 INFO:tasks.workunit.client.0.smithi136.stdout: "cephfs": null, 2024-09-12T23:37:44.480 INFO:tasks.workunit.client.0.smithi136.stdout: "nfs": null, 2024-09-12T23:37:44.480 INFO:tasks.workunit.client.0.smithi136.stdout: "rbd": null, 2024-09-12T23:37:44.480 INFO:tasks.workunit.client.0.smithi136.stdout: "total": null 2024-09-12T23:37:44.480 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:44.480 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:44.480 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.480 INFO:tasks.workunit.client.0.smithi136.stdout: "version": null 2024-09-12T23:37:44.480 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.481 INFO:tasks.workunit.client.0.smithi136.stdout: "services": {}, 2024-09-12T23:37:44.481 INFO:tasks.workunit.client.0.smithi136.stdout: "usage": { 2024-09-12T23:37:44.481 INFO:tasks.workunit.client.0.smithi136.stdout: "pg_num": 105, 2024-09-12T23:37:44.481 INFO:tasks.workunit.client.0.smithi136.stdout: "pools": 4, 2024-09-12T23:37:44.481 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_by_class": { 2024-09-12T23:37:44.481 INFO:tasks.workunit.client.0.smithi136.stdout: "ssd": { 2024-09-12T23:37:44.481 INFO:tasks.workunit.client.0.smithi136.stdout: "total_avail_bytes": 765517262848, 2024-09-12T23:37:44.481 INFO:tasks.workunit.client.0.smithi136.stdout: "total_bytes": 767960285184, 2024-09-12T23:37:44.481 INFO:tasks.workunit.client.0.smithi136.stdout: "total_used_bytes": 2443022336, 2024-09-12T23:37:44.481 INFO:tasks.workunit.client.0.smithi136.stdout: "total_used_raw_bytes": 2443022336, 2024-09-12T23:37:44.481 INFO:tasks.workunit.client.0.smithi136.stdout: "total_used_raw_ratio": 0.0031811832450330257 2024-09-12T23:37:44.481 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:44.481 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.481 INFO:tasks.workunit.client.0.smithi136.stdout: "total_avail_bytes": 765517262848, 2024-09-12T23:37:44.482 INFO:tasks.workunit.client.0.smithi136.stdout: "total_bytes": 767960285184, 2024-09-12T23:37:44.482 INFO:tasks.workunit.client.0.smithi136.stdout: "total_used_bytes": 2443022336 2024-09-12T23:37:44.482 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:44.482 INFO:tasks.workunit.client.0.smithi136.stdout:} 2024-09-12T23:37:44.482 INFO:tasks.workunit.client.0.smithi136.stdout:Device report is generated separately. To see it run 'ceph telemetry preview-device'. 2024-09-12T23:37:44.483 INFO:tasks.workunit.client.0.smithi136.stderr:+ ceph telemetry preview-device 2024-09-12T23:37:44.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:37:44 smithi136 bash[22852]: [12/Sep/2024:23:37:44] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-12T23:37:44.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:37:44 smithi136 bash[22852]: [12/Sep/2024:23:37:44] ENGINE Bus STOPPED 2024-09-12T23:37:44.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:37:44 smithi136 bash[22852]: [12/Sep/2024:23:37:44] ENGINE Bus STARTING 2024-09-12T23:37:44.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:37:44 smithi136 bash[22852]: [12/Sep/2024:23:37:44] ENGINE Serving on http://:::9283 2024-09-12T23:37:44.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:37:44 smithi136 bash[22852]: [12/Sep/2024:23:37:44] ENGINE Bus STARTED 2024-09-12T23:37:44.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:44 smithi136 bash[22601]: audit 2024-09-12T23:37:43.505020+0000 mon.a (mon.0) 908 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:37:44.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:44 smithi136 bash[22601]: audit 2024-09-12T23:37:43.506407+0000 mon.a (mon.0) 909 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:37:44.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:44 smithi136 bash[22601]: audit 2024-09-12T23:37:43.515812+0000 mon.a (mon.0) 910 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:37:44.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:44 smithi136 bash[22601]: audit 2024-09-12T23:37:43.553419+0000 mon.a (mon.0) 911 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:37:44.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:44 smithi136 bash[22601]: audit 2024-09-12T23:37:43.554939+0000 mon.a (mon.0) 912 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:37:44.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:44 smithi136 bash[22601]: audit 2024-09-12T23:37:43.555610+0000 mon.a (mon.0) 913 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:37:44.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:44 smithi136 bash[22601]: audit 2024-09-12T23:37:43.560579+0000 mon.a (mon.0) 914 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:37:44.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:44 smithi136 bash[22601]: audit 2024-09-12T23:37:43.837283+0000 mon.a (mon.0) 915 : audit [INF] from='client.? 172.21.15.136:0/3858554721' entity='client.admin' 2024-09-12T23:37:44.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:44 smithi136 bash[22601]: audit 2024-09-12T23:37:43.858541+0000 mon.a (mon.0) 916 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:37:44.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:44 smithi136 bash[22601]: audit 2024-09-12T23:37:43.860298+0000 mon.a (mon.0) 917 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:37:44.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:44 smithi136 bash[22601]: audit 2024-09-12T23:37:43.861112+0000 mon.a (mon.0) 918 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:37:44.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:44 smithi136 bash[22601]: audit 2024-09-12T23:37:43.867285+0000 mon.a (mon.0) 919 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:37:44.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:44 smithi136 bash[22601]: audit 2024-09-12T23:37:44.272041+0000 mon.a (mon.0) 920 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:37:44.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:44 smithi136 bash[22601]: audit 2024-09-12T23:37:44.283381+0000 mon.a (mon.0) 921 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-12T23:37:44.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:44 smithi136 bash[22601]: audit 2024-09-12T23:37:44.423779+0000 mon.a (mon.0) 922 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:37:44.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:44 smithi136 bash[22601]: audit 2024-09-12T23:37:44.427938+0000 mon.a (mon.0) 923 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-12T23:37:44.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:44 smithi136 bash[32189]: audit 2024-09-12T23:37:43.505020+0000 mon.a (mon.0) 908 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:37:44.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:44 smithi136 bash[32189]: audit 2024-09-12T23:37:43.506407+0000 mon.a (mon.0) 909 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:37:44.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:44 smithi136 bash[32189]: audit 2024-09-12T23:37:43.515812+0000 mon.a (mon.0) 910 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:37:44.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:44 smithi136 bash[32189]: audit 2024-09-12T23:37:43.553419+0000 mon.a (mon.0) 911 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:37:44.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:44 smithi136 bash[32189]: audit 2024-09-12T23:37:43.554939+0000 mon.a (mon.0) 912 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:37:44.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:44 smithi136 bash[32189]: audit 2024-09-12T23:37:43.555610+0000 mon.a (mon.0) 913 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:37:44.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:44 smithi136 bash[32189]: audit 2024-09-12T23:37:43.560579+0000 mon.a (mon.0) 914 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:37:44.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:44 smithi136 bash[32189]: audit 2024-09-12T23:37:43.837283+0000 mon.a (mon.0) 915 : audit [INF] from='client.? 172.21.15.136:0/3858554721' entity='client.admin' 2024-09-12T23:37:44.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:44 smithi136 bash[32189]: audit 2024-09-12T23:37:43.858541+0000 mon.a (mon.0) 916 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:37:44.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:44 smithi136 bash[32189]: audit 2024-09-12T23:37:43.860298+0000 mon.a (mon.0) 917 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:37:44.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:44 smithi136 bash[32189]: audit 2024-09-12T23:37:43.861112+0000 mon.a (mon.0) 918 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:37:44.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:44 smithi136 bash[32189]: audit 2024-09-12T23:37:43.867285+0000 mon.a (mon.0) 919 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:37:44.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:44 smithi136 bash[32189]: audit 2024-09-12T23:37:44.272041+0000 mon.a (mon.0) 920 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:37:44.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:44 smithi136 bash[32189]: audit 2024-09-12T23:37:44.283381+0000 mon.a (mon.0) 921 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-12T23:37:44.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:44 smithi136 bash[32189]: audit 2024-09-12T23:37:44.423779+0000 mon.a (mon.0) 922 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:37:44.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:44 smithi136 bash[32189]: audit 2024-09-12T23:37:44.427938+0000 mon.a (mon.0) 923 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-12T23:37:44.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:44 smithi181 bash[24776]: audit 2024-09-12T23:37:43.505020+0000 mon.a (mon.0) 908 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:37:44.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:44 smithi181 bash[24776]: audit 2024-09-12T23:37:43.506407+0000 mon.a (mon.0) 909 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:37:44.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:44 smithi181 bash[24776]: audit 2024-09-12T23:37:43.515812+0000 mon.a (mon.0) 910 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:37:44.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:44 smithi181 bash[24776]: audit 2024-09-12T23:37:43.553419+0000 mon.a (mon.0) 911 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:37:44.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:44 smithi181 bash[24776]: audit 2024-09-12T23:37:43.554939+0000 mon.a (mon.0) 912 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:37:44.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:44 smithi181 bash[24776]: audit 2024-09-12T23:37:43.555610+0000 mon.a (mon.0) 913 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:37:44.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:44 smithi181 bash[24776]: audit 2024-09-12T23:37:43.560579+0000 mon.a (mon.0) 914 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:37:44.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:44 smithi181 bash[24776]: audit 2024-09-12T23:37:43.837283+0000 mon.a (mon.0) 915 : audit [INF] from='client.? 172.21.15.136:0/3858554721' entity='client.admin' 2024-09-12T23:37:44.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:44 smithi181 bash[24776]: audit 2024-09-12T23:37:43.858541+0000 mon.a (mon.0) 916 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:37:44.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:44 smithi181 bash[24776]: audit 2024-09-12T23:37:43.860298+0000 mon.a (mon.0) 917 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:37:44.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:44 smithi181 bash[24776]: audit 2024-09-12T23:37:43.861112+0000 mon.a (mon.0) 918 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:37:44.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:44 smithi181 bash[24776]: audit 2024-09-12T23:37:43.867285+0000 mon.a (mon.0) 919 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:37:44.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:44 smithi181 bash[24776]: audit 2024-09-12T23:37:44.272041+0000 mon.a (mon.0) 920 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:37:44.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:44 smithi181 bash[24776]: audit 2024-09-12T23:37:44.283381+0000 mon.a (mon.0) 921 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-12T23:37:44.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:44 smithi181 bash[24776]: audit 2024-09-12T23:37:44.423779+0000 mon.a (mon.0) 922 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:37:44.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:44 smithi181 bash[24776]: audit 2024-09-12T23:37:44.427938+0000 mon.a (mon.0) 923 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-12T23:37:44.973 INFO:tasks.workunit.client.0.smithi136.stdout:{ 2024-09-12T23:37:44.974 INFO:tasks.workunit.client.0.smithi136.stdout: "02b0b25a-7160-11ef-baee-0cc47a886e43": { 2024-09-12T23:37:44.974 INFO:tasks.workunit.client.0.smithi136.stdout: "INTEL_SSDPEDMD400G4_02b1c294-7160-11ef-baee-0cc47a886e43": { 2024-09-12T23:37:44.974 INFO:tasks.workunit.client.0.smithi136.stdout: "20240912-232840": { 2024-09-12T23:37:44.974 INFO:tasks.workunit.client.0.smithi136.stdout: "device": { 2024-09-12T23:37:44.974 INFO:tasks.workunit.client.0.smithi136.stdout: "info_name": "/dev/nvme0n1", 2024-09-12T23:37:44.974 INFO:tasks.workunit.client.0.smithi136.stdout: "name": "/dev/nvme0n1", 2024-09-12T23:37:44.974 INFO:tasks.workunit.client.0.smithi136.stdout: "protocol": "NVMe", 2024-09-12T23:37:44.974 INFO:tasks.workunit.client.0.smithi136.stdout: "type": "nvme" 2024-09-12T23:37:44.974 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.974 INFO:tasks.workunit.client.0.smithi136.stdout: "firmware_version": "8DV101H0", 2024-09-12T23:37:44.974 INFO:tasks.workunit.client.0.smithi136.stdout: "host_id": "02b0b25a-7160-11ef-baee-0cc47a886e43", 2024-09-12T23:37:44.975 INFO:tasks.workunit.client.0.smithi136.stdout: "json_format_version": [ 2024-09-12T23:37:44.975 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:44.975 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:44.975 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:44.975 INFO:tasks.workunit.client.0.smithi136.stdout: "local_time": { 2024-09-12T23:37:44.975 INFO:tasks.workunit.client.0.smithi136.stdout: "asctime": "Thu Sep 12 23:28:40 2024 UTC", 2024-09-12T23:37:44.975 INFO:tasks.workunit.client.0.smithi136.stdout: "time_t": 1726183720 2024-09-12T23:37:44.975 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.975 INFO:tasks.workunit.client.0.smithi136.stdout: "logical_block_size": 512, 2024-09-12T23:37:44.975 INFO:tasks.workunit.client.0.smithi136.stdout: "model_name": "INTEL SSDPEDMD400G4", 2024-09-12T23:37:44.976 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_controller_id": 0, 2024-09-12T23:37:44.976 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_ieee_oui_identifier": 6083300, 2024-09-12T23:37:44.976 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_namespaces": [ 2024-09-12T23:37:44.976 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:44.976 INFO:tasks.workunit.client.0.smithi136.stdout: "capacity": { 2024-09-12T23:37:44.976 INFO:tasks.workunit.client.0.smithi136.stdout: "blocks": 781422768, 2024-09-12T23:37:44.976 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 400088457216 2024-09-12T23:37:44.976 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.976 INFO:tasks.workunit.client.0.smithi136.stdout: "formatted_lba_size": 512, 2024-09-12T23:37:44.976 INFO:tasks.workunit.client.0.smithi136.stdout: "id": 1, 2024-09-12T23:37:44.976 INFO:tasks.workunit.client.0.smithi136.stdout: "size": { 2024-09-12T23:37:44.976 INFO:tasks.workunit.client.0.smithi136.stdout: "blocks": 781422768, 2024-09-12T23:37:44.976 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 400088457216 2024-09-12T23:37:44.977 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.977 INFO:tasks.workunit.client.0.smithi136.stdout: "utilization": { 2024-09-12T23:37:44.977 INFO:tasks.workunit.client.0.smithi136.stdout: "blocks": 781422768, 2024-09-12T23:37:44.977 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 400088457216 2024-09-12T23:37:44.977 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:44.977 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:44.977 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:44.977 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_number_of_namespaces": 1, 2024-09-12T23:37:44.977 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_pci_vendor": { 2024-09-12T23:37:44.977 INFO:tasks.workunit.client.0.smithi136.stdout: "id": 32902, 2024-09-12T23:37:44.977 INFO:tasks.workunit.client.0.smithi136.stdout: "subsystem_id": 32902 2024-09-12T23:37:44.977 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.977 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_smart_health_information_add_log": { 2024-09-12T23:37:44.978 INFO:tasks.workunit.client.0.smithi136.stdout: "Device stats": { 2024-09-12T23:37:44.978 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_fw_downgrades": { 2024-09-12T23:37:44.978 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 0, 2024-09-12T23:37:44.978 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:44.978 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.978 INFO:tasks.workunit.client.0.smithi136.stdout: "crc_error_count": { 2024-09-12T23:37:44.978 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:44.978 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:44.978 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.978 INFO:tasks.workunit.client.0.smithi136.stdout: "end_to_end_error_detection_count": { 2024-09-12T23:37:44.978 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:44.978 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:44.978 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.978 INFO:tasks.workunit.client.0.smithi136.stdout: "erase_fail_count": { 2024-09-12T23:37:44.978 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:44.979 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:44.979 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.979 INFO:tasks.workunit.client.0.smithi136.stdout: "host_bytes_written": { 2024-09-12T23:37:44.979 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:44.979 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 26605852 2024-09-12T23:37:44.979 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.979 INFO:tasks.workunit.client.0.smithi136.stdout: "host_ctx_wear_used": { 2024-09-12T23:37:44.979 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:44.979 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 1 2024-09-12T23:37:44.979 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.979 INFO:tasks.workunit.client.0.smithi136.stdout: "media_bytes_read": { 2024-09-12T23:37:44.979 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 0, 2024-09-12T23:37:44.979 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:44.979 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.980 INFO:tasks.workunit.client.0.smithi136.stdout: "nand_bytes_written": { 2024-09-12T23:37:44.980 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:44.980 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 47676652 2024-09-12T23:37:44.980 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.980 INFO:tasks.workunit.client.0.smithi136.stdout: "perf_stat_indicator": { 2024-09-12T23:37:44.980 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 0, 2024-09-12T23:37:44.980 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:44.980 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.980 INFO:tasks.workunit.client.0.smithi136.stdout: "pll_lock_loss_count": { 2024-09-12T23:37:44.980 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:44.980 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:44.981 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.981 INFO:tasks.workunit.client.0.smithi136.stdout: "program_fail_count": { 2024-09-12T23:37:44.981 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:44.981 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:44.981 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.981 INFO:tasks.workunit.client.0.smithi136.stdout: "re_alloc_sectr_cnt": { 2024-09-12T23:37:44.981 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 0, 2024-09-12T23:37:44.981 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:44.981 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.981 INFO:tasks.workunit.client.0.smithi136.stdout: "retry_buffer_overflow_count": { 2024-09-12T23:37:44.981 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:44.981 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:44.982 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.982 INFO:tasks.workunit.client.0.smithi136.stdout: "soft_ecc_err_rate": { 2024-09-12T23:37:44.982 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 0, 2024-09-12T23:37:44.982 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:44.982 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.982 INFO:tasks.workunit.client.0.smithi136.stdout: "thermal_throttle_status": { 2024-09-12T23:37:44.982 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:44.982 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": { 2024-09-12T23:37:44.982 INFO:tasks.workunit.client.0.smithi136.stdout: "cnt": 0, 2024-09-12T23:37:44.982 INFO:tasks.workunit.client.0.smithi136.stdout: "pct": 0 2024-09-12T23:37:44.982 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:44.982 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.983 INFO:tasks.workunit.client.0.smithi136.stdout: "timed_workload_host_reads": { 2024-09-12T23:37:44.983 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:44.983 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 65535 2024-09-12T23:37:44.983 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.983 INFO:tasks.workunit.client.0.smithi136.stdout: "timed_workload_media_wear": { 2024-09-12T23:37:44.983 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:44.983 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": "63.999023" 2024-09-12T23:37:44.983 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.983 INFO:tasks.workunit.client.0.smithi136.stdout: "timed_workload_timer": { 2024-09-12T23:37:44.983 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:44.983 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 65535 2024-09-12T23:37:44.983 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.983 INFO:tasks.workunit.client.0.smithi136.stdout: "unexp_power_loss": { 2024-09-12T23:37:44.983 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 0, 2024-09-12T23:37:44.983 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:44.984 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.984 INFO:tasks.workunit.client.0.smithi136.stdout: "wear_leveling": { 2024-09-12T23:37:44.984 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 90, 2024-09-12T23:37:44.984 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": { 2024-09-12T23:37:44.984 INFO:tasks.workunit.client.0.smithi136.stdout: "avg": 3275, 2024-09-12T23:37:44.984 INFO:tasks.workunit.client.0.smithi136.stdout: "max": 3290, 2024-09-12T23:37:44.984 INFO:tasks.workunit.client.0.smithi136.stdout: "min": 3260 2024-09-12T23:37:44.984 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:44.984 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:44.985 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.985 INFO:tasks.workunit.client.0.smithi136.stdout: "Intel Smart log": "nvme0n1" 2024-09-12T23:37:44.985 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.985 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_smart_health_information_log": { 2024-09-12T23:37:44.985 INFO:tasks.workunit.client.0.smithi136.stdout: "available_spare": 100, 2024-09-12T23:37:44.985 INFO:tasks.workunit.client.0.smithi136.stdout: "available_spare_threshold": 10, 2024-09-12T23:37:44.985 INFO:tasks.workunit.client.0.smithi136.stdout: "controller_busy_time": 50, 2024-09-12T23:37:44.985 INFO:tasks.workunit.client.0.smithi136.stdout: "critical_warning": 0, 2024-09-12T23:37:44.985 INFO:tasks.workunit.client.0.smithi136.stdout: "data_units_read": 360344011, 2024-09-12T23:37:44.985 INFO:tasks.workunit.client.0.smithi136.stdout: "data_units_written": 1743641130, 2024-09-12T23:37:44.985 INFO:tasks.workunit.client.0.smithi136.stdout: "host_reads": 5879482063, 2024-09-12T23:37:44.985 INFO:tasks.workunit.client.0.smithi136.stdout: "host_writes": 31824656568, 2024-09-12T23:37:44.985 INFO:tasks.workunit.client.0.smithi136.stdout: "media_errors": 0, 2024-09-12T23:37:44.986 INFO:tasks.workunit.client.0.smithi136.stdout: "num_err_log_entries": 0, 2024-09-12T23:37:44.986 INFO:tasks.workunit.client.0.smithi136.stdout: "percentage_used": 10, 2024-09-12T23:37:44.986 INFO:tasks.workunit.client.0.smithi136.stdout: "power_cycles": 56409, 2024-09-12T23:37:44.986 INFO:tasks.workunit.client.0.smithi136.stdout: "power_on_hours": 54400, 2024-09-12T23:37:44.986 INFO:tasks.workunit.client.0.smithi136.stdout: "temperature": 23, 2024-09-12T23:37:44.986 INFO:tasks.workunit.client.0.smithi136.stdout: "unsafe_shutdowns": 2575 2024-09-12T23:37:44.986 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.986 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_vendor": "intel", 2024-09-12T23:37:44.986 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_version": { 2024-09-12T23:37:44.986 INFO:tasks.workunit.client.0.smithi136.stdout: "string": "<1.2", 2024-09-12T23:37:44.986 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:44.986 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.986 INFO:tasks.workunit.client.0.smithi136.stdout: "power_cycle_count": 56409, 2024-09-12T23:37:44.986 INFO:tasks.workunit.client.0.smithi136.stdout: "power_on_time": { 2024-09-12T23:37:44.987 INFO:tasks.workunit.client.0.smithi136.stdout: "hours": 54400 2024-09-12T23:37:44.987 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.987 INFO:tasks.workunit.client.0.smithi136.stdout: "serial_number": "deleted", 2024-09-12T23:37:44.987 INFO:tasks.workunit.client.0.smithi136.stdout: "smart_status": { 2024-09-12T23:37:44.987 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme": { 2024-09-12T23:37:44.987 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:44.987 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.987 INFO:tasks.workunit.client.0.smithi136.stdout: "passed": true 2024-09-12T23:37:44.987 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.987 INFO:tasks.workunit.client.0.smithi136.stdout: "smartctl": { 2024-09-12T23:37:44.987 INFO:tasks.workunit.client.0.smithi136.stdout: "argv": [ 2024-09-12T23:37:44.987 INFO:tasks.workunit.client.0.smithi136.stdout: "smartctl", 2024-09-12T23:37:44.987 INFO:tasks.workunit.client.0.smithi136.stdout: "-x", 2024-09-12T23:37:44.987 INFO:tasks.workunit.client.0.smithi136.stdout: "--json=o", 2024-09-12T23:37:44.988 INFO:tasks.workunit.client.0.smithi136.stdout: "/dev/nvme0n1" 2024-09-12T23:37:44.988 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:44.988 INFO:tasks.workunit.client.0.smithi136.stdout: "build_info": "(local build)", 2024-09-12T23:37:44.988 INFO:tasks.workunit.client.0.smithi136.stdout: "exit_status": 0, 2024-09-12T23:37:44.988 INFO:tasks.workunit.client.0.smithi136.stdout: "output": [ 2024-09-12T23:37:44.988 INFO:tasks.workunit.client.0.smithi136.stdout: "smartctl 7.2 2020-12-30 r5155 [x86_64-linux-5.15.0-121-generic] (local build)", 2024-09-12T23:37:44.988 INFO:tasks.workunit.client.0.smithi136.stdout: "Copyright (C) 2002-20, Bruce Allen, Christian Franke, www.smartmontools.org", 2024-09-12T23:37:44.988 INFO:tasks.workunit.client.0.smithi136.stdout: "", 2024-09-12T23:37:44.988 INFO:tasks.workunit.client.0.smithi136.stdout: "=== START OF INFORMATION SECTION ===", 2024-09-12T23:37:44.988 INFO:tasks.workunit.client.0.smithi136.stdout: "Model Number: INTEL SSDPEDMD400G4", 2024-09-12T23:37:44.988 INFO:tasks.workunit.client.0.smithi136.stdout: "Serial Number: deleted", 2024-09-12T23:37:44.988 INFO:tasks.workunit.client.0.smithi136.stdout: "Firmware Version: 8DV101H0", 2024-09-12T23:37:44.988 INFO:tasks.workunit.client.0.smithi136.stdout: "PCI Vendor/Subsystem ID: 0x8086", 2024-09-12T23:37:44.988 INFO:tasks.workunit.client.0.smithi136.stdout: "IEEE OUI Identifier: 0x5cd2e4", 2024-09-12T23:37:44.989 INFO:tasks.workunit.client.0.smithi136.stdout: "Controller ID: 0", 2024-09-12T23:37:44.989 INFO:tasks.workunit.client.0.smithi136.stdout: "NVMe Version: <1.2", 2024-09-12T23:37:44.989 INFO:tasks.workunit.client.0.smithi136.stdout: "Number of Namespaces: 1", 2024-09-12T23:37:44.989 INFO:tasks.workunit.client.0.smithi136.stdout: "Namespace 1 Size/Capacity: 400,088,457,216 [400 GB]", 2024-09-12T23:37:44.989 INFO:tasks.workunit.client.0.smithi136.stdout: "Namespace 1 Formatted LBA Size: 512", 2024-09-12T23:37:44.989 INFO:tasks.workunit.client.0.smithi136.stdout: "Local Time is: Thu Sep 12 23:28:40 2024 UTC", 2024-09-12T23:37:44.989 INFO:tasks.workunit.client.0.smithi136.stdout: "Firmware Updates (0x02): 1 Slot", 2024-09-12T23:37:44.989 INFO:tasks.workunit.client.0.smithi136.stdout: "Optional Admin Commands (0x0006): Format Frmw_DL", 2024-09-12T23:37:44.989 INFO:tasks.workunit.client.0.smithi136.stdout: "Optional NVM Commands (0x0006): Wr_Unc DS_Mngmt", 2024-09-12T23:37:44.989 INFO:tasks.workunit.client.0.smithi136.stdout: "Log Page Attributes (0x02): Cmd_Eff_Lg", 2024-09-12T23:37:44.989 INFO:tasks.workunit.client.0.smithi136.stdout: "Maximum Data Transfer Size: 32 Pages", 2024-09-12T23:37:44.989 INFO:tasks.workunit.client.0.smithi136.stdout: "", 2024-09-12T23:37:44.990 INFO:tasks.workunit.client.0.smithi136.stdout: "Supported Power States", 2024-09-12T23:37:44.990 INFO:tasks.workunit.client.0.smithi136.stdout: "St Op Max Active Idle RL RT WL WT Ent_Lat Ex_Lat", 2024-09-12T23:37:44.990 INFO:tasks.workunit.client.0.smithi136.stdout: " 0 + 25.00W - - 0 0 0 0 0 0", 2024-09-12T23:37:44.990 INFO:tasks.workunit.client.0.smithi136.stdout: "", 2024-09-12T23:37:44.990 INFO:tasks.workunit.client.0.smithi136.stdout: "Supported LBA Sizes (NSID 0x1)", 2024-09-12T23:37:44.990 INFO:tasks.workunit.client.0.smithi136.stdout: "Id Fmt Data Metadt Rel_Perf", 2024-09-12T23:37:44.990 INFO:tasks.workunit.client.0.smithi136.stdout: " 0 + 512 0 2", 2024-09-12T23:37:44.990 INFO:tasks.workunit.client.0.smithi136.stdout: " 1 - 512 8 2", 2024-09-12T23:37:44.990 INFO:tasks.workunit.client.0.smithi136.stdout: " 2 - 512 16 2", 2024-09-12T23:37:44.990 INFO:tasks.workunit.client.0.smithi136.stdout: " 3 - 4096 0 0", 2024-09-12T23:37:44.990 INFO:tasks.workunit.client.0.smithi136.stdout: " 4 - 4096 8 0", 2024-09-12T23:37:44.990 INFO:tasks.workunit.client.0.smithi136.stdout: " 5 - 4096 64 0", 2024-09-12T23:37:44.990 INFO:tasks.workunit.client.0.smithi136.stdout: " 6 - 4096 128 0", 2024-09-12T23:37:44.990 INFO:tasks.workunit.client.0.smithi136.stdout: "", 2024-09-12T23:37:44.990 INFO:tasks.workunit.client.0.smithi136.stdout: "=== START OF SMART DATA SECTION ===", 2024-09-12T23:37:44.991 INFO:tasks.workunit.client.0.smithi136.stdout: "SMART overall-health self-assessment test result: PASSED", 2024-09-12T23:37:44.991 INFO:tasks.workunit.client.0.smithi136.stdout: "", 2024-09-12T23:37:44.991 INFO:tasks.workunit.client.0.smithi136.stdout: "SMART/Health Information (NVMe Log 0x02)", 2024-09-12T23:37:44.991 INFO:tasks.workunit.client.0.smithi136.stdout: "Critical Warning: 0x00", 2024-09-12T23:37:44.991 INFO:tasks.workunit.client.0.smithi136.stdout: "Temperature: 23 Celsius", 2024-09-12T23:37:44.991 INFO:tasks.workunit.client.0.smithi136.stdout: "Available Spare: 100%", 2024-09-12T23:37:44.991 INFO:tasks.workunit.client.0.smithi136.stdout: "Available Spare Threshold: 10%", 2024-09-12T23:37:44.991 INFO:tasks.workunit.client.0.smithi136.stdout: "Percentage Used: 10%", 2024-09-12T23:37:44.991 INFO:tasks.workunit.client.0.smithi136.stdout: "Data Units Read: 360,344,011 [184 TB]", 2024-09-12T23:37:44.991 INFO:tasks.workunit.client.0.smithi136.stdout: "Data Units Written: 1,743,641,130 [892 TB]", 2024-09-12T23:37:44.991 INFO:tasks.workunit.client.0.smithi136.stdout: "Host Read Commands: 5,879,482,063", 2024-09-12T23:37:44.991 INFO:tasks.workunit.client.0.smithi136.stdout: "Host Write Commands: 31,824,656,568", 2024-09-12T23:37:44.991 INFO:tasks.workunit.client.0.smithi136.stdout: "Controller Busy Time: 50", 2024-09-12T23:37:44.991 INFO:tasks.workunit.client.0.smithi136.stdout: "Power Cycles: 56,409", 2024-09-12T23:37:44.991 INFO:tasks.workunit.client.0.smithi136.stdout: "Power On Hours: 54,400", 2024-09-12T23:37:44.992 INFO:tasks.workunit.client.0.smithi136.stdout: "Unsafe Shutdowns: 2,575", 2024-09-12T23:37:44.992 INFO:tasks.workunit.client.0.smithi136.stdout: "Media and Data Integrity Errors: 0", 2024-09-12T23:37:44.992 INFO:tasks.workunit.client.0.smithi136.stdout: "Error Information Log Entries: 0", 2024-09-12T23:37:44.992 INFO:tasks.workunit.client.0.smithi136.stdout: "", 2024-09-12T23:37:44.992 INFO:tasks.workunit.client.0.smithi136.stdout: "Error Information (NVMe Log 0x01, 16 of 64 entries)", 2024-09-12T23:37:44.992 INFO:tasks.workunit.client.0.smithi136.stdout: "No Errors Logged", 2024-09-12T23:37:44.992 INFO:tasks.workunit.client.0.smithi136.stdout: "" 2024-09-12T23:37:44.992 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:44.992 INFO:tasks.workunit.client.0.smithi136.stdout: "platform_info": "x86_64-linux-5.15.0-121-generic", 2024-09-12T23:37:44.992 INFO:tasks.workunit.client.0.smithi136.stdout: "svn_revision": "5155", 2024-09-12T23:37:44.992 INFO:tasks.workunit.client.0.smithi136.stdout: "version": [ 2024-09-12T23:37:44.992 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:44.992 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:44.993 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:44.993 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.993 INFO:tasks.workunit.client.0.smithi136.stdout: "temperature": { 2024-09-12T23:37:44.993 INFO:tasks.workunit.client.0.smithi136.stdout: "current": 23 2024-09-12T23:37:44.993 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.993 INFO:tasks.workunit.client.0.smithi136.stdout: "user_capacity": { 2024-09-12T23:37:44.993 INFO:tasks.workunit.client.0.smithi136.stdout: "blocks": 781422768, 2024-09-12T23:37:44.993 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 400088457216 2024-09-12T23:37:44.993 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:44.993 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:44.993 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:44.993 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.994 INFO:tasks.workunit.client.0.smithi136.stdout: "02b4b8aa-7160-11ef-baee-0cc47a886e43": { 2024-09-12T23:37:44.994 INFO:tasks.workunit.client.0.smithi136.stdout: "INTEL_SSDPEDMX020T4_02b5a97c-7160-11ef-baee-0cc47a886e43": { 2024-09-12T23:37:44.994 INFO:tasks.workunit.client.0.smithi136.stdout: "20240912-232836": { 2024-09-12T23:37:44.994 INFO:tasks.workunit.client.0.smithi136.stdout: "device": { 2024-09-12T23:37:44.994 INFO:tasks.workunit.client.0.smithi136.stdout: "info_name": "/dev/nvme0n1", 2024-09-12T23:37:44.994 INFO:tasks.workunit.client.0.smithi136.stdout: "name": "/dev/nvme0n1", 2024-09-12T23:37:44.994 INFO:tasks.workunit.client.0.smithi136.stdout: "protocol": "NVMe", 2024-09-12T23:37:44.994 INFO:tasks.workunit.client.0.smithi136.stdout: "type": "nvme" 2024-09-12T23:37:44.994 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.994 INFO:tasks.workunit.client.0.smithi136.stdout: "firmware_version": "8DV10171", 2024-09-12T23:37:44.994 INFO:tasks.workunit.client.0.smithi136.stdout: "host_id": "02b4b8aa-7160-11ef-baee-0cc47a886e43", 2024-09-12T23:37:44.994 INFO:tasks.workunit.client.0.smithi136.stdout: "json_format_version": [ 2024-09-12T23:37:44.994 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:44.994 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:44.994 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:44.995 INFO:tasks.workunit.client.0.smithi136.stdout: "local_time": { 2024-09-12T23:37:44.995 INFO:tasks.workunit.client.0.smithi136.stdout: "asctime": "Thu Sep 12 23:28:36 2024 UTC", 2024-09-12T23:37:44.995 INFO:tasks.workunit.client.0.smithi136.stdout: "time_t": 1726183716 2024-09-12T23:37:44.995 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.995 INFO:tasks.workunit.client.0.smithi136.stdout: "logical_block_size": 512, 2024-09-12T23:37:44.995 INFO:tasks.workunit.client.0.smithi136.stdout: "model_name": "INTEL SSDPEDMX020T4", 2024-09-12T23:37:44.995 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_controller_id": 0, 2024-09-12T23:37:44.995 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_ieee_oui_identifier": 6083300, 2024-09-12T23:37:44.995 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_namespaces": [ 2024-09-12T23:37:44.995 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:44.995 INFO:tasks.workunit.client.0.smithi136.stdout: "capacity": { 2024-09-12T23:37:44.995 INFO:tasks.workunit.client.0.smithi136.stdout: "blocks": 3907029168, 2024-09-12T23:37:44.995 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 2000398934016 2024-09-12T23:37:44.995 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.995 INFO:tasks.workunit.client.0.smithi136.stdout: "formatted_lba_size": 512, 2024-09-12T23:37:44.995 INFO:tasks.workunit.client.0.smithi136.stdout: "id": 1, 2024-09-12T23:37:44.996 INFO:tasks.workunit.client.0.smithi136.stdout: "size": { 2024-09-12T23:37:44.996 INFO:tasks.workunit.client.0.smithi136.stdout: "blocks": 3907029168, 2024-09-12T23:37:44.996 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 2000398934016 2024-09-12T23:37:44.996 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.996 INFO:tasks.workunit.client.0.smithi136.stdout: "utilization": { 2024-09-12T23:37:44.996 INFO:tasks.workunit.client.0.smithi136.stdout: "blocks": 3907029168, 2024-09-12T23:37:44.996 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 2000398934016 2024-09-12T23:37:44.996 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:44.996 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:44.996 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:44.996 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_number_of_namespaces": 1, 2024-09-12T23:37:44.996 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_pci_vendor": { 2024-09-12T23:37:44.996 INFO:tasks.workunit.client.0.smithi136.stdout: "id": 32902, 2024-09-12T23:37:44.997 INFO:tasks.workunit.client.0.smithi136.stdout: "subsystem_id": 32902 2024-09-12T23:37:44.997 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.997 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_smart_health_information_add_log": { 2024-09-12T23:37:44.997 INFO:tasks.workunit.client.0.smithi136.stdout: "Device stats": { 2024-09-12T23:37:44.997 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_fw_downgrades": { 2024-09-12T23:37:44.997 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 0, 2024-09-12T23:37:44.997 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:44.997 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.997 INFO:tasks.workunit.client.0.smithi136.stdout: "crc_error_count": { 2024-09-12T23:37:44.997 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:44.998 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 2 2024-09-12T23:37:44.998 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.998 INFO:tasks.workunit.client.0.smithi136.stdout: "end_to_end_error_detection_count": { 2024-09-12T23:37:44.998 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:44.998 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:44.998 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.998 INFO:tasks.workunit.client.0.smithi136.stdout: "erase_fail_count": { 2024-09-12T23:37:44.998 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:44.998 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:44.998 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.999 INFO:tasks.workunit.client.0.smithi136.stdout: "host_bytes_written": { 2024-09-12T23:37:44.999 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:44.999 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 19052464 2024-09-12T23:37:44.999 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.999 INFO:tasks.workunit.client.0.smithi136.stdout: "host_ctx_wear_used": { 2024-09-12T23:37:44.999 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 0, 2024-09-12T23:37:44.999 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:44.999 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:44.999 INFO:tasks.workunit.client.0.smithi136.stdout: "media_bytes_read": { 2024-09-12T23:37:44.999 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 0, 2024-09-12T23:37:44.999 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:44.999 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.000 INFO:tasks.workunit.client.0.smithi136.stdout: "nand_bytes_written": { 2024-09-12T23:37:45.000 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:45.000 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 50938213 2024-09-12T23:37:45.000 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.000 INFO:tasks.workunit.client.0.smithi136.stdout: "perf_stat_indicator": { 2024-09-12T23:37:45.000 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 0, 2024-09-12T23:37:45.000 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:45.000 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.000 INFO:tasks.workunit.client.0.smithi136.stdout: "pll_lock_loss_count": { 2024-09-12T23:37:45.000 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:45.000 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:45.000 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.000 INFO:tasks.workunit.client.0.smithi136.stdout: "program_fail_count": { 2024-09-12T23:37:45.000 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:45.001 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:45.001 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.001 INFO:tasks.workunit.client.0.smithi136.stdout: "re_alloc_sectr_cnt": { 2024-09-12T23:37:45.001 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 0, 2024-09-12T23:37:45.001 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:45.001 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.001 INFO:tasks.workunit.client.0.smithi136.stdout: "retry_buffer_overflow_count": { 2024-09-12T23:37:45.001 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:45.001 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:45.001 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.001 INFO:tasks.workunit.client.0.smithi136.stdout: "soft_ecc_err_rate": { 2024-09-12T23:37:45.001 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 0, 2024-09-12T23:37:45.001 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:45.002 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.002 INFO:tasks.workunit.client.0.smithi136.stdout: "thermal_throttle_status": { 2024-09-12T23:37:45.002 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:45.002 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": { 2024-09-12T23:37:45.002 INFO:tasks.workunit.client.0.smithi136.stdout: "cnt": 0, 2024-09-12T23:37:45.002 INFO:tasks.workunit.client.0.smithi136.stdout: "pct": 0 2024-09-12T23:37:45.002 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:45.002 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.002 INFO:tasks.workunit.client.0.smithi136.stdout: "timed_workload_host_reads": { 2024-09-12T23:37:45.002 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:45.002 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 15 2024-09-12T23:37:45.002 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.002 INFO:tasks.workunit.client.0.smithi136.stdout: "timed_workload_media_wear": { 2024-09-12T23:37:45.002 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:45.003 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": "6.850586" 2024-09-12T23:37:45.003 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.003 INFO:tasks.workunit.client.0.smithi136.stdout: "timed_workload_timer": { 2024-09-12T23:37:45.003 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:45.003 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 1541953 2024-09-12T23:37:45.003 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.003 INFO:tasks.workunit.client.0.smithi136.stdout: "unexp_power_loss": { 2024-09-12T23:37:45.003 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 0, 2024-09-12T23:37:45.003 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:45.003 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.003 INFO:tasks.workunit.client.0.smithi136.stdout: "wear_leveling": { 2024-09-12T23:37:45.003 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 47, 2024-09-12T23:37:45.003 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": { 2024-09-12T23:37:45.003 INFO:tasks.workunit.client.0.smithi136.stdout: "avg": 1165, 2024-09-12T23:37:45.004 INFO:tasks.workunit.client.0.smithi136.stdout: "max": 1608, 2024-09-12T23:37:45.004 INFO:tasks.workunit.client.0.smithi136.stdout: "min": 765 2024-09-12T23:37:45.004 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:45.004 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:45.004 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.004 INFO:tasks.workunit.client.0.smithi136.stdout: "Intel Smart log": "nvme0n1" 2024-09-12T23:37:45.004 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.004 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_smart_health_information_log": { 2024-09-12T23:37:45.004 INFO:tasks.workunit.client.0.smithi136.stdout: "available_spare": 100, 2024-09-12T23:37:45.004 INFO:tasks.workunit.client.0.smithi136.stdout: "available_spare_threshold": 10, 2024-09-12T23:37:45.004 INFO:tasks.workunit.client.0.smithi136.stdout: "controller_busy_time": 358, 2024-09-12T23:37:45.004 INFO:tasks.workunit.client.0.smithi136.stdout: "critical_warning": 0, 2024-09-12T23:37:45.004 INFO:tasks.workunit.client.0.smithi136.stdout: "data_units_read": 215101386, 2024-09-12T23:37:45.004 INFO:tasks.workunit.client.0.smithi136.stdout: "data_units_written": 1248622303, 2024-09-12T23:37:45.005 INFO:tasks.workunit.client.0.smithi136.stdout: "host_reads": 4349473727, 2024-09-12T23:37:45.005 INFO:tasks.workunit.client.0.smithi136.stdout: "host_writes": 24841062282, 2024-09-12T23:37:45.005 INFO:tasks.workunit.client.0.smithi136.stdout: "media_errors": 0, 2024-09-12T23:37:45.005 INFO:tasks.workunit.client.0.smithi136.stdout: "num_err_log_entries": 0, 2024-09-12T23:37:45.005 INFO:tasks.workunit.client.0.smithi136.stdout: "percentage_used": 25, 2024-09-12T23:37:45.005 INFO:tasks.workunit.client.0.smithi136.stdout: "power_cycles": 45284, 2024-09-12T23:37:45.005 INFO:tasks.workunit.client.0.smithi136.stdout: "power_on_hours": 38404, 2024-09-12T23:37:45.005 INFO:tasks.workunit.client.0.smithi136.stdout: "temperature": 23, 2024-09-12T23:37:45.005 INFO:tasks.workunit.client.0.smithi136.stdout: "unsafe_shutdowns": 45156 2024-09-12T23:37:45.005 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.005 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_vendor": "intel", 2024-09-12T23:37:45.005 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_version": { 2024-09-12T23:37:45.005 INFO:tasks.workunit.client.0.smithi136.stdout: "string": "<1.2", 2024-09-12T23:37:45.006 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:45.006 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.006 INFO:tasks.workunit.client.0.smithi136.stdout: "power_cycle_count": 45284, 2024-09-12T23:37:45.006 INFO:tasks.workunit.client.0.smithi136.stdout: "power_on_time": { 2024-09-12T23:37:45.006 INFO:tasks.workunit.client.0.smithi136.stdout: "hours": 38404 2024-09-12T23:37:45.006 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.006 INFO:tasks.workunit.client.0.smithi136.stdout: "serial_number": "deleted", 2024-09-12T23:37:45.006 INFO:tasks.workunit.client.0.smithi136.stdout: "smart_status": { 2024-09-12T23:37:45.006 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme": { 2024-09-12T23:37:45.006 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:45.006 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.006 INFO:tasks.workunit.client.0.smithi136.stdout: "passed": true 2024-09-12T23:37:45.006 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.006 INFO:tasks.workunit.client.0.smithi136.stdout: "smartctl": { 2024-09-12T23:37:45.007 INFO:tasks.workunit.client.0.smithi136.stdout: "argv": [ 2024-09-12T23:37:45.007 INFO:tasks.workunit.client.0.smithi136.stdout: "smartctl", 2024-09-12T23:37:45.007 INFO:tasks.workunit.client.0.smithi136.stdout: "-x", 2024-09-12T23:37:45.007 INFO:tasks.workunit.client.0.smithi136.stdout: "--json=o", 2024-09-12T23:37:45.007 INFO:tasks.workunit.client.0.smithi136.stdout: "/dev/nvme0n1" 2024-09-12T23:37:45.007 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:45.007 INFO:tasks.workunit.client.0.smithi136.stdout: "build_info": "(local build)", 2024-09-12T23:37:45.007 INFO:tasks.workunit.client.0.smithi136.stdout: "exit_status": 0, 2024-09-12T23:37:45.007 INFO:tasks.workunit.client.0.smithi136.stdout: "output": [ 2024-09-12T23:37:45.007 INFO:tasks.workunit.client.0.smithi136.stdout: "smartctl 7.2 2020-12-30 r5155 [x86_64-linux-5.15.0-121-generic] (local build)", 2024-09-12T23:37:45.007 INFO:tasks.workunit.client.0.smithi136.stdout: "Copyright (C) 2002-20, Bruce Allen, Christian Franke, www.smartmontools.org", 2024-09-12T23:37:45.007 INFO:tasks.workunit.client.0.smithi136.stdout: "", 2024-09-12T23:37:45.008 INFO:tasks.workunit.client.0.smithi136.stdout: "=== START OF INFORMATION SECTION ===", 2024-09-12T23:37:45.008 INFO:tasks.workunit.client.0.smithi136.stdout: "Model Number: INTEL SSDPEDMX020T4", 2024-09-12T23:37:45.008 INFO:tasks.workunit.client.0.smithi136.stdout: "Serial Number: deleted", 2024-09-12T23:37:45.008 INFO:tasks.workunit.client.0.smithi136.stdout: "Firmware Version: 8DV10171", 2024-09-12T23:37:45.008 INFO:tasks.workunit.client.0.smithi136.stdout: "PCI Vendor/Subsystem ID: 0x8086", 2024-09-12T23:37:45.008 INFO:tasks.workunit.client.0.smithi136.stdout: "IEEE OUI Identifier: 0x5cd2e4", 2024-09-12T23:37:45.008 INFO:tasks.workunit.client.0.smithi136.stdout: "Controller ID: 0", 2024-09-12T23:37:45.008 INFO:tasks.workunit.client.0.smithi136.stdout: "NVMe Version: <1.2", 2024-09-12T23:37:45.008 INFO:tasks.workunit.client.0.smithi136.stdout: "Number of Namespaces: 1", 2024-09-12T23:37:45.008 INFO:tasks.workunit.client.0.smithi136.stdout: "Namespace 1 Size/Capacity: 2,000,398,934,016 [2.00 TB]", 2024-09-12T23:37:45.008 INFO:tasks.workunit.client.0.smithi136.stdout: "Namespace 1 Formatted LBA Size: 512", 2024-09-12T23:37:45.008 INFO:tasks.workunit.client.0.smithi136.stdout: "Local Time is: Thu Sep 12 23:28:36 2024 UTC", 2024-09-12T23:37:45.008 INFO:tasks.workunit.client.0.smithi136.stdout: "Firmware Updates (0x02): 1 Slot", 2024-09-12T23:37:45.008 INFO:tasks.workunit.client.0.smithi136.stdout: "Optional Admin Commands (0x0006): Format Frmw_DL", 2024-09-12T23:37:45.009 INFO:tasks.workunit.client.0.smithi136.stdout: "Optional NVM Commands (0x0006): Wr_Unc DS_Mngmt", 2024-09-12T23:37:45.009 INFO:tasks.workunit.client.0.smithi136.stdout: "Log Page Attributes (0x02): Cmd_Eff_Lg", 2024-09-12T23:37:45.009 INFO:tasks.workunit.client.0.smithi136.stdout: "Maximum Data Transfer Size: 32 Pages", 2024-09-12T23:37:45.009 INFO:tasks.workunit.client.0.smithi136.stdout: "", 2024-09-12T23:37:45.009 INFO:tasks.workunit.client.0.smithi136.stdout: "Supported Power States", 2024-09-12T23:37:45.009 INFO:tasks.workunit.client.0.smithi136.stdout: "St Op Max Active Idle RL RT WL WT Ent_Lat Ex_Lat", 2024-09-12T23:37:45.009 INFO:tasks.workunit.client.0.smithi136.stdout: " 0 + 25.00W - - 0 0 0 0 0 0", 2024-09-12T23:37:45.010 INFO:tasks.workunit.client.0.smithi136.stdout: "", 2024-09-12T23:37:45.010 INFO:tasks.workunit.client.0.smithi136.stdout: "Supported LBA Sizes (NSID 0x1)", 2024-09-12T23:37:45.010 INFO:tasks.workunit.client.0.smithi136.stdout: "Id Fmt Data Metadt Rel_Perf", 2024-09-12T23:37:45.010 INFO:tasks.workunit.client.0.smithi136.stdout: " 0 + 512 0 2", 2024-09-12T23:37:45.010 INFO:tasks.workunit.client.0.smithi136.stdout: " 1 - 512 8 2", 2024-09-12T23:37:45.010 INFO:tasks.workunit.client.0.smithi136.stdout: " 2 - 512 16 2", 2024-09-12T23:37:45.011 INFO:tasks.workunit.client.0.smithi136.stdout: " 3 - 4096 0 0", 2024-09-12T23:37:45.011 INFO:tasks.workunit.client.0.smithi136.stdout: " 4 - 4096 8 0", 2024-09-12T23:37:45.011 INFO:tasks.workunit.client.0.smithi136.stdout: " 5 - 4096 64 0", 2024-09-12T23:37:45.011 INFO:tasks.workunit.client.0.smithi136.stdout: " 6 - 4096 128 0", 2024-09-12T23:37:45.011 INFO:tasks.workunit.client.0.smithi136.stdout: "", 2024-09-12T23:37:45.011 INFO:tasks.workunit.client.0.smithi136.stdout: "=== START OF SMART DATA SECTION ===", 2024-09-12T23:37:45.011 INFO:tasks.workunit.client.0.smithi136.stdout: "SMART overall-health self-assessment test result: PASSED", 2024-09-12T23:37:45.012 INFO:tasks.workunit.client.0.smithi136.stdout: "", 2024-09-12T23:37:45.012 INFO:tasks.workunit.client.0.smithi136.stdout: "SMART/Health Information (NVMe Log 0x02)", 2024-09-12T23:37:45.012 INFO:tasks.workunit.client.0.smithi136.stdout: "Critical Warning: 0x00", 2024-09-12T23:37:45.012 INFO:tasks.workunit.client.0.smithi136.stdout: "Temperature: 23 Celsius", 2024-09-12T23:37:45.012 INFO:tasks.workunit.client.0.smithi136.stdout: "Available Spare: 100%", 2024-09-12T23:37:45.012 INFO:tasks.workunit.client.0.smithi136.stdout: "Available Spare Threshold: 10%", 2024-09-12T23:37:45.012 INFO:tasks.workunit.client.0.smithi136.stdout: "Percentage Used: 25%", 2024-09-12T23:37:45.012 INFO:tasks.workunit.client.0.smithi136.stdout: "Data Units Read: 215,101,386 [110 TB]", 2024-09-12T23:37:45.013 INFO:tasks.workunit.client.0.smithi136.stdout: "Data Units Written: 1,248,622,303 [639 TB]", 2024-09-12T23:37:45.013 INFO:tasks.workunit.client.0.smithi136.stdout: "Host Read Commands: 4,349,473,727", 2024-09-12T23:37:45.013 INFO:tasks.workunit.client.0.smithi136.stdout: "Host Write Commands: 24,841,062,282", 2024-09-12T23:37:45.013 INFO:tasks.workunit.client.0.smithi136.stdout: "Controller Busy Time: 358", 2024-09-12T23:37:45.013 INFO:tasks.workunit.client.0.smithi136.stdout: "Power Cycles: 45,284", 2024-09-12T23:37:45.013 INFO:tasks.workunit.client.0.smithi136.stdout: "Power On Hours: 38,404", 2024-09-12T23:37:45.013 INFO:tasks.workunit.client.0.smithi136.stdout: "Unsafe Shutdowns: 45,156", 2024-09-12T23:37:45.013 INFO:tasks.workunit.client.0.smithi136.stdout: "Media and Data Integrity Errors: 0", 2024-09-12T23:37:45.014 INFO:tasks.workunit.client.0.smithi136.stdout: "Error Information Log Entries: 0", 2024-09-12T23:37:45.014 INFO:tasks.workunit.client.0.smithi136.stdout: "", 2024-09-12T23:37:45.014 INFO:tasks.workunit.client.0.smithi136.stdout: "Error Information (NVMe Log 0x01, 16 of 64 entries)", 2024-09-12T23:37:45.014 INFO:tasks.workunit.client.0.smithi136.stdout: "No Errors Logged", 2024-09-12T23:37:45.014 INFO:tasks.workunit.client.0.smithi136.stdout: "" 2024-09-12T23:37:45.014 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:45.014 INFO:tasks.workunit.client.0.smithi136.stdout: "platform_info": "x86_64-linux-5.15.0-121-generic", 2024-09-12T23:37:45.014 INFO:tasks.workunit.client.0.smithi136.stdout: "svn_revision": "5155", 2024-09-12T23:37:45.015 INFO:tasks.workunit.client.0.smithi136.stdout: "version": [ 2024-09-12T23:37:45.015 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:45.015 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:45.015 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:45.015 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.015 INFO:tasks.workunit.client.0.smithi136.stdout: "temperature": { 2024-09-12T23:37:45.016 INFO:tasks.workunit.client.0.smithi136.stdout: "current": 23 2024-09-12T23:37:45.016 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.016 INFO:tasks.workunit.client.0.smithi136.stdout: "user_capacity": { 2024-09-12T23:37:45.016 INFO:tasks.workunit.client.0.smithi136.stdout: "blocks": 3907029168, 2024-09-12T23:37:45.016 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 2000398934016 2024-09-12T23:37:45.016 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:45.017 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:45.017 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:45.017 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:45.017 INFO:tasks.workunit.client.0.smithi136.stdout:} 2024-09-12T23:37:45.018 INFO:tasks.workunit.client.0.smithi136.stderr:+ ceph telemetry preview-all 2024-09-12T23:37:45.456 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:37:45 smithi181 bash[25717]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:37:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-12T23:37:45.558 INFO:tasks.workunit.client.0.smithi136.stdout:{ 2024-09-12T23:37:45.558 INFO:tasks.workunit.client.0.smithi136.stdout: "device_report": { 2024-09-12T23:37:45.558 INFO:tasks.workunit.client.0.smithi136.stdout: "02b0b25a-7160-11ef-baee-0cc47a886e43": { 2024-09-12T23:37:45.559 INFO:tasks.workunit.client.0.smithi136.stdout: "INTEL_SSDPEDMD400G4_02b1c294-7160-11ef-baee-0cc47a886e43": { 2024-09-12T23:37:45.559 INFO:tasks.workunit.client.0.smithi136.stdout: "20240912-232840": { 2024-09-12T23:37:45.559 INFO:tasks.workunit.client.0.smithi136.stdout: "device": { 2024-09-12T23:37:45.559 INFO:tasks.workunit.client.0.smithi136.stdout: "info_name": "/dev/nvme0n1", 2024-09-12T23:37:45.559 INFO:tasks.workunit.client.0.smithi136.stdout: "name": "/dev/nvme0n1", 2024-09-12T23:37:45.559 INFO:tasks.workunit.client.0.smithi136.stdout: "protocol": "NVMe", 2024-09-12T23:37:45.559 INFO:tasks.workunit.client.0.smithi136.stdout: "type": "nvme" 2024-09-12T23:37:45.559 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.559 INFO:tasks.workunit.client.0.smithi136.stdout: "firmware_version": "8DV101H0", 2024-09-12T23:37:45.559 INFO:tasks.workunit.client.0.smithi136.stdout: "host_id": "02b0b25a-7160-11ef-baee-0cc47a886e43", 2024-09-12T23:37:45.559 INFO:tasks.workunit.client.0.smithi136.stdout: "json_format_version": [ 2024-09-12T23:37:45.559 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:45.559 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:45.560 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:45.560 INFO:tasks.workunit.client.0.smithi136.stdout: "local_time": { 2024-09-12T23:37:45.560 INFO:tasks.workunit.client.0.smithi136.stdout: "asctime": "Thu Sep 12 23:28:40 2024 UTC", 2024-09-12T23:37:45.560 INFO:tasks.workunit.client.0.smithi136.stdout: "time_t": 1726183720 2024-09-12T23:37:45.560 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.560 INFO:tasks.workunit.client.0.smithi136.stdout: "logical_block_size": 512, 2024-09-12T23:37:45.560 INFO:tasks.workunit.client.0.smithi136.stdout: "model_name": "INTEL SSDPEDMD400G4", 2024-09-12T23:37:45.560 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_controller_id": 0, 2024-09-12T23:37:45.560 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_ieee_oui_identifier": 6083300, 2024-09-12T23:37:45.560 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_namespaces": [ 2024-09-12T23:37:45.560 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:45.560 INFO:tasks.workunit.client.0.smithi136.stdout: "capacity": { 2024-09-12T23:37:45.560 INFO:tasks.workunit.client.0.smithi136.stdout: "blocks": 781422768, 2024-09-12T23:37:45.560 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 400088457216 2024-09-12T23:37:45.561 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.561 INFO:tasks.workunit.client.0.smithi136.stdout: "formatted_lba_size": 512, 2024-09-12T23:37:45.561 INFO:tasks.workunit.client.0.smithi136.stdout: "id": 1, 2024-09-12T23:37:45.561 INFO:tasks.workunit.client.0.smithi136.stdout: "size": { 2024-09-12T23:37:45.561 INFO:tasks.workunit.client.0.smithi136.stdout: "blocks": 781422768, 2024-09-12T23:37:45.561 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 400088457216 2024-09-12T23:37:45.561 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.561 INFO:tasks.workunit.client.0.smithi136.stdout: "utilization": { 2024-09-12T23:37:45.561 INFO:tasks.workunit.client.0.smithi136.stdout: "blocks": 781422768, 2024-09-12T23:37:45.561 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 400088457216 2024-09-12T23:37:45.561 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:45.561 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:45.561 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:45.561 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_number_of_namespaces": 1, 2024-09-12T23:37:45.562 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_pci_vendor": { 2024-09-12T23:37:45.562 INFO:tasks.workunit.client.0.smithi136.stdout: "id": 32902, 2024-09-12T23:37:45.562 INFO:tasks.workunit.client.0.smithi136.stdout: "subsystem_id": 32902 2024-09-12T23:37:45.562 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.562 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_smart_health_information_add_log": { 2024-09-12T23:37:45.562 INFO:tasks.workunit.client.0.smithi136.stdout: "Device stats": { 2024-09-12T23:37:45.562 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_fw_downgrades": { 2024-09-12T23:37:45.562 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 0, 2024-09-12T23:37:45.562 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:45.562 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.562 INFO:tasks.workunit.client.0.smithi136.stdout: "crc_error_count": { 2024-09-12T23:37:45.562 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:45.562 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:45.562 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.563 INFO:tasks.workunit.client.0.smithi136.stdout: "end_to_end_error_detection_count": { 2024-09-12T23:37:45.563 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:45.563 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:45.563 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.563 INFO:tasks.workunit.client.0.smithi136.stdout: "erase_fail_count": { 2024-09-12T23:37:45.563 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:45.563 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:45.563 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.563 INFO:tasks.workunit.client.0.smithi136.stdout: "host_bytes_written": { 2024-09-12T23:37:45.563 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:45.563 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 26605852 2024-09-12T23:37:45.563 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.563 INFO:tasks.workunit.client.0.smithi136.stdout: "host_ctx_wear_used": { 2024-09-12T23:37:45.564 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:45.564 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 1 2024-09-12T23:37:45.564 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.564 INFO:tasks.workunit.client.0.smithi136.stdout: "media_bytes_read": { 2024-09-12T23:37:45.564 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 0, 2024-09-12T23:37:45.564 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:45.564 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.564 INFO:tasks.workunit.client.0.smithi136.stdout: "nand_bytes_written": { 2024-09-12T23:37:45.564 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:45.564 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 47676652 2024-09-12T23:37:45.564 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.564 INFO:tasks.workunit.client.0.smithi136.stdout: "perf_stat_indicator": { 2024-09-12T23:37:45.564 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 0, 2024-09-12T23:37:45.564 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:45.564 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.565 INFO:tasks.workunit.client.0.smithi136.stdout: "pll_lock_loss_count": { 2024-09-12T23:37:45.565 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:45.565 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:45.565 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.565 INFO:tasks.workunit.client.0.smithi136.stdout: "program_fail_count": { 2024-09-12T23:37:45.565 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:45.565 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:45.565 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.565 INFO:tasks.workunit.client.0.smithi136.stdout: "re_alloc_sectr_cnt": { 2024-09-12T23:37:45.565 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 0, 2024-09-12T23:37:45.565 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:45.565 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.565 INFO:tasks.workunit.client.0.smithi136.stdout: "retry_buffer_overflow_count": { 2024-09-12T23:37:45.566 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:45.566 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:45.566 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.566 INFO:tasks.workunit.client.0.smithi136.stdout: "soft_ecc_err_rate": { 2024-09-12T23:37:45.566 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 0, 2024-09-12T23:37:45.566 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:45.566 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.566 INFO:tasks.workunit.client.0.smithi136.stdout: "thermal_throttle_status": { 2024-09-12T23:37:45.566 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:45.566 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": { 2024-09-12T23:37:45.566 INFO:tasks.workunit.client.0.smithi136.stdout: "cnt": 0, 2024-09-12T23:37:45.566 INFO:tasks.workunit.client.0.smithi136.stdout: "pct": 0 2024-09-12T23:37:45.566 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:45.566 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.566 INFO:tasks.workunit.client.0.smithi136.stdout: "timed_workload_host_reads": { 2024-09-12T23:37:45.567 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:45.567 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 65535 2024-09-12T23:37:45.567 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.567 INFO:tasks.workunit.client.0.smithi136.stdout: "timed_workload_media_wear": { 2024-09-12T23:37:45.567 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:45.567 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": "63.999023" 2024-09-12T23:37:45.567 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.567 INFO:tasks.workunit.client.0.smithi136.stdout: "timed_workload_timer": { 2024-09-12T23:37:45.567 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:45.567 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 65535 2024-09-12T23:37:45.567 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.567 INFO:tasks.workunit.client.0.smithi136.stdout: "unexp_power_loss": { 2024-09-12T23:37:45.567 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 0, 2024-09-12T23:37:45.567 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:45.567 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.568 INFO:tasks.workunit.client.0.smithi136.stdout: "wear_leveling": { 2024-09-12T23:37:45.568 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 90, 2024-09-12T23:37:45.568 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": { 2024-09-12T23:37:45.568 INFO:tasks.workunit.client.0.smithi136.stdout: "avg": 3275, 2024-09-12T23:37:45.568 INFO:tasks.workunit.client.0.smithi136.stdout: "max": 3290, 2024-09-12T23:37:45.568 INFO:tasks.workunit.client.0.smithi136.stdout: "min": 3260 2024-09-12T23:37:45.568 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:45.568 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:45.568 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.568 INFO:tasks.workunit.client.0.smithi136.stdout: "Intel Smart log": "nvme0n1" 2024-09-12T23:37:45.568 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.568 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_smart_health_information_log": { 2024-09-12T23:37:45.568 INFO:tasks.workunit.client.0.smithi136.stdout: "available_spare": 100, 2024-09-12T23:37:45.568 INFO:tasks.workunit.client.0.smithi136.stdout: "available_spare_threshold": 10, 2024-09-12T23:37:45.568 INFO:tasks.workunit.client.0.smithi136.stdout: "controller_busy_time": 50, 2024-09-12T23:37:45.569 INFO:tasks.workunit.client.0.smithi136.stdout: "critical_warning": 0, 2024-09-12T23:37:45.569 INFO:tasks.workunit.client.0.smithi136.stdout: "data_units_read": 360344011, 2024-09-12T23:37:45.569 INFO:tasks.workunit.client.0.smithi136.stdout: "data_units_written": 1743641130, 2024-09-12T23:37:45.569 INFO:tasks.workunit.client.0.smithi136.stdout: "host_reads": 5879482063, 2024-09-12T23:37:45.569 INFO:tasks.workunit.client.0.smithi136.stdout: "host_writes": 31824656568, 2024-09-12T23:37:45.569 INFO:tasks.workunit.client.0.smithi136.stdout: "media_errors": 0, 2024-09-12T23:37:45.569 INFO:tasks.workunit.client.0.smithi136.stdout: "num_err_log_entries": 0, 2024-09-12T23:37:45.569 INFO:tasks.workunit.client.0.smithi136.stdout: "percentage_used": 10, 2024-09-12T23:37:45.569 INFO:tasks.workunit.client.0.smithi136.stdout: "power_cycles": 56409, 2024-09-12T23:37:45.569 INFO:tasks.workunit.client.0.smithi136.stdout: "power_on_hours": 54400, 2024-09-12T23:37:45.569 INFO:tasks.workunit.client.0.smithi136.stdout: "temperature": 23, 2024-09-12T23:37:45.569 INFO:tasks.workunit.client.0.smithi136.stdout: "unsafe_shutdowns": 2575 2024-09-12T23:37:45.569 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.569 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_vendor": "intel", 2024-09-12T23:37:45.570 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_version": { 2024-09-12T23:37:45.570 INFO:tasks.workunit.client.0.smithi136.stdout: "string": "<1.2", 2024-09-12T23:37:45.570 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:45.570 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.570 INFO:tasks.workunit.client.0.smithi136.stdout: "power_cycle_count": 56409, 2024-09-12T23:37:45.570 INFO:tasks.workunit.client.0.smithi136.stdout: "power_on_time": { 2024-09-12T23:37:45.570 INFO:tasks.workunit.client.0.smithi136.stdout: "hours": 54400 2024-09-12T23:37:45.570 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.570 INFO:tasks.workunit.client.0.smithi136.stdout: "serial_number": "deleted", 2024-09-12T23:37:45.570 INFO:tasks.workunit.client.0.smithi136.stdout: "smart_status": { 2024-09-12T23:37:45.570 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme": { 2024-09-12T23:37:45.570 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:45.570 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.570 INFO:tasks.workunit.client.0.smithi136.stdout: "passed": true 2024-09-12T23:37:45.571 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.571 INFO:tasks.workunit.client.0.smithi136.stdout: "smartctl": { 2024-09-12T23:37:45.571 INFO:tasks.workunit.client.0.smithi136.stdout: "argv": [ 2024-09-12T23:37:45.571 INFO:tasks.workunit.client.0.smithi136.stdout: "smartctl", 2024-09-12T23:37:45.571 INFO:tasks.workunit.client.0.smithi136.stdout: "-x", 2024-09-12T23:37:45.571 INFO:tasks.workunit.client.0.smithi136.stdout: "--json=o", 2024-09-12T23:37:45.571 INFO:tasks.workunit.client.0.smithi136.stdout: "/dev/nvme0n1" 2024-09-12T23:37:45.571 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:45.571 INFO:tasks.workunit.client.0.smithi136.stdout: "build_info": "(local build)", 2024-09-12T23:37:45.571 INFO:tasks.workunit.client.0.smithi136.stdout: "exit_status": 0, 2024-09-12T23:37:45.571 INFO:tasks.workunit.client.0.smithi136.stdout: "output": [ 2024-09-12T23:37:45.571 INFO:tasks.workunit.client.0.smithi136.stdout: "smartctl 7.2 2020-12-30 r5155 [x86_64-linux-5.15.0-121-generic] (local build)", 2024-09-12T23:37:45.571 INFO:tasks.workunit.client.0.smithi136.stdout: "Copyright (C) 2002-20, Bruce Allen, Christian Franke, www.smartmontools.org", 2024-09-12T23:37:45.571 INFO:tasks.workunit.client.0.smithi136.stdout: "", 2024-09-12T23:37:45.571 INFO:tasks.workunit.client.0.smithi136.stdout: "=== START OF INFORMATION SECTION ===", 2024-09-12T23:37:45.572 INFO:tasks.workunit.client.0.smithi136.stdout: "Model Number: INTEL SSDPEDMD400G4", 2024-09-12T23:37:45.572 INFO:tasks.workunit.client.0.smithi136.stdout: "Serial Number: deleted", 2024-09-12T23:37:45.572 INFO:tasks.workunit.client.0.smithi136.stdout: "Firmware Version: 8DV101H0", 2024-09-12T23:37:45.572 INFO:tasks.workunit.client.0.smithi136.stdout: "PCI Vendor/Subsystem ID: 0x8086", 2024-09-12T23:37:45.572 INFO:tasks.workunit.client.0.smithi136.stdout: "IEEE OUI Identifier: 0x5cd2e4", 2024-09-12T23:37:45.572 INFO:tasks.workunit.client.0.smithi136.stdout: "Controller ID: 0", 2024-09-12T23:37:45.572 INFO:tasks.workunit.client.0.smithi136.stdout: "NVMe Version: <1.2", 2024-09-12T23:37:45.572 INFO:tasks.workunit.client.0.smithi136.stdout: "Number of Namespaces: 1", 2024-09-12T23:37:45.572 INFO:tasks.workunit.client.0.smithi136.stdout: "Namespace 1 Size/Capacity: 400,088,457,216 [400 GB]", 2024-09-12T23:37:45.572 INFO:tasks.workunit.client.0.smithi136.stdout: "Namespace 1 Formatted LBA Size: 512", 2024-09-12T23:37:45.572 INFO:tasks.workunit.client.0.smithi136.stdout: "Local Time is: Thu Sep 12 23:28:40 2024 UTC", 2024-09-12T23:37:45.572 INFO:tasks.workunit.client.0.smithi136.stdout: "Firmware Updates (0x02): 1 Slot", 2024-09-12T23:37:45.572 INFO:tasks.workunit.client.0.smithi136.stdout: "Optional Admin Commands (0x0006): Format Frmw_DL", 2024-09-12T23:37:45.573 INFO:tasks.workunit.client.0.smithi136.stdout: "Optional NVM Commands (0x0006): Wr_Unc DS_Mngmt", 2024-09-12T23:37:45.573 INFO:tasks.workunit.client.0.smithi136.stdout: "Log Page Attributes (0x02): Cmd_Eff_Lg", 2024-09-12T23:37:45.573 INFO:tasks.workunit.client.0.smithi136.stdout: "Maximum Data Transfer Size: 32 Pages", 2024-09-12T23:37:45.573 INFO:tasks.workunit.client.0.smithi136.stdout: "", 2024-09-12T23:37:45.573 INFO:tasks.workunit.client.0.smithi136.stdout: "Supported Power States", 2024-09-12T23:37:45.573 INFO:tasks.workunit.client.0.smithi136.stdout: "St Op Max Active Idle RL RT WL WT Ent_Lat Ex_Lat", 2024-09-12T23:37:45.573 INFO:tasks.workunit.client.0.smithi136.stdout: " 0 + 25.00W - - 0 0 0 0 0 0", 2024-09-12T23:37:45.573 INFO:tasks.workunit.client.0.smithi136.stdout: "", 2024-09-12T23:37:45.573 INFO:tasks.workunit.client.0.smithi136.stdout: "Supported LBA Sizes (NSID 0x1)", 2024-09-12T23:37:45.573 INFO:tasks.workunit.client.0.smithi136.stdout: "Id Fmt Data Metadt Rel_Perf", 2024-09-12T23:37:45.573 INFO:tasks.workunit.client.0.smithi136.stdout: " 0 + 512 0 2", 2024-09-12T23:37:45.573 INFO:tasks.workunit.client.0.smithi136.stdout: " 1 - 512 8 2", 2024-09-12T23:37:45.573 INFO:tasks.workunit.client.0.smithi136.stdout: " 2 - 512 16 2", 2024-09-12T23:37:45.573 INFO:tasks.workunit.client.0.smithi136.stdout: " 3 - 4096 0 0", 2024-09-12T23:37:45.573 INFO:tasks.workunit.client.0.smithi136.stdout: " 4 - 4096 8 0", 2024-09-12T23:37:45.574 INFO:tasks.workunit.client.0.smithi136.stdout: " 5 - 4096 64 0", 2024-09-12T23:37:45.574 INFO:tasks.workunit.client.0.smithi136.stdout: " 6 - 4096 128 0", 2024-09-12T23:37:45.574 INFO:tasks.workunit.client.0.smithi136.stdout: "", 2024-09-12T23:37:45.574 INFO:tasks.workunit.client.0.smithi136.stdout: "=== START OF SMART DATA SECTION ===", 2024-09-12T23:37:45.574 INFO:tasks.workunit.client.0.smithi136.stdout: "SMART overall-health self-assessment test result: PASSED", 2024-09-12T23:37:45.574 INFO:tasks.workunit.client.0.smithi136.stdout: "", 2024-09-12T23:37:45.574 INFO:tasks.workunit.client.0.smithi136.stdout: "SMART/Health Information (NVMe Log 0x02)", 2024-09-12T23:37:45.574 INFO:tasks.workunit.client.0.smithi136.stdout: "Critical Warning: 0x00", 2024-09-12T23:37:45.574 INFO:tasks.workunit.client.0.smithi136.stdout: "Temperature: 23 Celsius", 2024-09-12T23:37:45.574 INFO:tasks.workunit.client.0.smithi136.stdout: "Available Spare: 100%", 2024-09-12T23:37:45.574 INFO:tasks.workunit.client.0.smithi136.stdout: "Available Spare Threshold: 10%", 2024-09-12T23:37:45.574 INFO:tasks.workunit.client.0.smithi136.stdout: "Percentage Used: 10%", 2024-09-12T23:37:45.574 INFO:tasks.workunit.client.0.smithi136.stdout: "Data Units Read: 360,344,011 [184 TB]", 2024-09-12T23:37:45.575 INFO:tasks.workunit.client.0.smithi136.stdout: "Data Units Written: 1,743,641,130 [892 TB]", 2024-09-12T23:37:45.575 INFO:tasks.workunit.client.0.smithi136.stdout: "Host Read Commands: 5,879,482,063", 2024-09-12T23:37:45.575 INFO:tasks.workunit.client.0.smithi136.stdout: "Host Write Commands: 31,824,656,568", 2024-09-12T23:37:45.575 INFO:tasks.workunit.client.0.smithi136.stdout: "Controller Busy Time: 50", 2024-09-12T23:37:45.575 INFO:tasks.workunit.client.0.smithi136.stdout: "Power Cycles: 56,409", 2024-09-12T23:37:45.575 INFO:tasks.workunit.client.0.smithi136.stdout: "Power On Hours: 54,400", 2024-09-12T23:37:45.575 INFO:tasks.workunit.client.0.smithi136.stdout: "Unsafe Shutdowns: 2,575", 2024-09-12T23:37:45.575 INFO:tasks.workunit.client.0.smithi136.stdout: "Media and Data Integrity Errors: 0", 2024-09-12T23:37:45.575 INFO:tasks.workunit.client.0.smithi136.stdout: "Error Information Log Entries: 0", 2024-09-12T23:37:45.575 INFO:tasks.workunit.client.0.smithi136.stdout: "", 2024-09-12T23:37:45.575 INFO:tasks.workunit.client.0.smithi136.stdout: "Error Information (NVMe Log 0x01, 16 of 64 entries)", 2024-09-12T23:37:45.575 INFO:tasks.workunit.client.0.smithi136.stdout: "No Errors Logged", 2024-09-12T23:37:45.575 INFO:tasks.workunit.client.0.smithi136.stdout: "" 2024-09-12T23:37:45.576 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:45.576 INFO:tasks.workunit.client.0.smithi136.stdout: "platform_info": "x86_64-linux-5.15.0-121-generic", 2024-09-12T23:37:45.576 INFO:tasks.workunit.client.0.smithi136.stdout: "svn_revision": "5155", 2024-09-12T23:37:45.576 INFO:tasks.workunit.client.0.smithi136.stdout: "version": [ 2024-09-12T23:37:45.576 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:45.576 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:45.576 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:45.576 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.576 INFO:tasks.workunit.client.0.smithi136.stdout: "temperature": { 2024-09-12T23:37:45.576 INFO:tasks.workunit.client.0.smithi136.stdout: "current": 23 2024-09-12T23:37:45.576 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.576 INFO:tasks.workunit.client.0.smithi136.stdout: "user_capacity": { 2024-09-12T23:37:45.577 INFO:tasks.workunit.client.0.smithi136.stdout: "blocks": 781422768, 2024-09-12T23:37:45.577 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 400088457216 2024-09-12T23:37:45.577 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:45.577 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:45.577 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:45.577 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.577 INFO:tasks.workunit.client.0.smithi136.stdout: "02b4b8aa-7160-11ef-baee-0cc47a886e43": { 2024-09-12T23:37:45.577 INFO:tasks.workunit.client.0.smithi136.stdout: "INTEL_SSDPEDMX020T4_02b5a97c-7160-11ef-baee-0cc47a886e43": { 2024-09-12T23:37:45.577 INFO:tasks.workunit.client.0.smithi136.stdout: "20240912-232836": { 2024-09-12T23:37:45.577 INFO:tasks.workunit.client.0.smithi136.stdout: "device": { 2024-09-12T23:37:45.577 INFO:tasks.workunit.client.0.smithi136.stdout: "info_name": "/dev/nvme0n1", 2024-09-12T23:37:45.577 INFO:tasks.workunit.client.0.smithi136.stdout: "name": "/dev/nvme0n1", 2024-09-12T23:37:45.577 INFO:tasks.workunit.client.0.smithi136.stdout: "protocol": "NVMe", 2024-09-12T23:37:45.578 INFO:tasks.workunit.client.0.smithi136.stdout: "type": "nvme" 2024-09-12T23:37:45.578 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.578 INFO:tasks.workunit.client.0.smithi136.stdout: "firmware_version": "8DV10171", 2024-09-12T23:37:45.578 INFO:tasks.workunit.client.0.smithi136.stdout: "host_id": "02b4b8aa-7160-11ef-baee-0cc47a886e43", 2024-09-12T23:37:45.578 INFO:tasks.workunit.client.0.smithi136.stdout: "json_format_version": [ 2024-09-12T23:37:45.578 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:45.578 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:45.578 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:45.578 INFO:tasks.workunit.client.0.smithi136.stdout: "local_time": { 2024-09-12T23:37:45.578 INFO:tasks.workunit.client.0.smithi136.stdout: "asctime": "Thu Sep 12 23:28:36 2024 UTC", 2024-09-12T23:37:45.578 INFO:tasks.workunit.client.0.smithi136.stdout: "time_t": 1726183716 2024-09-12T23:37:45.578 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.578 INFO:tasks.workunit.client.0.smithi136.stdout: "logical_block_size": 512, 2024-09-12T23:37:45.578 INFO:tasks.workunit.client.0.smithi136.stdout: "model_name": "INTEL SSDPEDMX020T4", 2024-09-12T23:37:45.579 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_controller_id": 0, 2024-09-12T23:37:45.579 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_ieee_oui_identifier": 6083300, 2024-09-12T23:37:45.579 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_namespaces": [ 2024-09-12T23:37:45.579 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:45.579 INFO:tasks.workunit.client.0.smithi136.stdout: "capacity": { 2024-09-12T23:37:45.579 INFO:tasks.workunit.client.0.smithi136.stdout: "blocks": 3907029168, 2024-09-12T23:37:45.579 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 2000398934016 2024-09-12T23:37:45.579 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.579 INFO:tasks.workunit.client.0.smithi136.stdout: "formatted_lba_size": 512, 2024-09-12T23:37:45.579 INFO:tasks.workunit.client.0.smithi136.stdout: "id": 1, 2024-09-12T23:37:45.579 INFO:tasks.workunit.client.0.smithi136.stdout: "size": { 2024-09-12T23:37:45.579 INFO:tasks.workunit.client.0.smithi136.stdout: "blocks": 3907029168, 2024-09-12T23:37:45.579 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 2000398934016 2024-09-12T23:37:45.579 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.580 INFO:tasks.workunit.client.0.smithi136.stdout: "utilization": { 2024-09-12T23:37:45.580 INFO:tasks.workunit.client.0.smithi136.stdout: "blocks": 3907029168, 2024-09-12T23:37:45.580 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 2000398934016 2024-09-12T23:37:45.580 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:45.580 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:45.580 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:45.580 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_number_of_namespaces": 1, 2024-09-12T23:37:45.580 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_pci_vendor": { 2024-09-12T23:37:45.580 INFO:tasks.workunit.client.0.smithi136.stdout: "id": 32902, 2024-09-12T23:37:45.580 INFO:tasks.workunit.client.0.smithi136.stdout: "subsystem_id": 32902 2024-09-12T23:37:45.580 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.580 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_smart_health_information_add_log": { 2024-09-12T23:37:45.580 INFO:tasks.workunit.client.0.smithi136.stdout: "Device stats": { 2024-09-12T23:37:45.581 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_fw_downgrades": { 2024-09-12T23:37:45.581 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 0, 2024-09-12T23:37:45.581 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:45.581 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.581 INFO:tasks.workunit.client.0.smithi136.stdout: "crc_error_count": { 2024-09-12T23:37:45.581 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:45.581 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 2 2024-09-12T23:37:45.581 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.581 INFO:tasks.workunit.client.0.smithi136.stdout: "end_to_end_error_detection_count": { 2024-09-12T23:37:45.581 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:45.581 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:45.581 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.581 INFO:tasks.workunit.client.0.smithi136.stdout: "erase_fail_count": { 2024-09-12T23:37:45.581 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:45.581 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:45.582 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.582 INFO:tasks.workunit.client.0.smithi136.stdout: "host_bytes_written": { 2024-09-12T23:37:45.582 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:45.582 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 19052464 2024-09-12T23:37:45.582 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.582 INFO:tasks.workunit.client.0.smithi136.stdout: "host_ctx_wear_used": { 2024-09-12T23:37:45.582 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 0, 2024-09-12T23:37:45.582 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:45.582 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.582 INFO:tasks.workunit.client.0.smithi136.stdout: "media_bytes_read": { 2024-09-12T23:37:45.582 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 0, 2024-09-12T23:37:45.582 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:45.582 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.582 INFO:tasks.workunit.client.0.smithi136.stdout: "nand_bytes_written": { 2024-09-12T23:37:45.583 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:45.583 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 50938213 2024-09-12T23:37:45.583 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.583 INFO:tasks.workunit.client.0.smithi136.stdout: "perf_stat_indicator": { 2024-09-12T23:37:45.583 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 0, 2024-09-12T23:37:45.583 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:45.583 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.583 INFO:tasks.workunit.client.0.smithi136.stdout: "pll_lock_loss_count": { 2024-09-12T23:37:45.583 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:45.583 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:45.583 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.583 INFO:tasks.workunit.client.0.smithi136.stdout: "program_fail_count": { 2024-09-12T23:37:45.583 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:45.583 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:45.584 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.584 INFO:tasks.workunit.client.0.smithi136.stdout: "re_alloc_sectr_cnt": { 2024-09-12T23:37:45.584 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 0, 2024-09-12T23:37:45.584 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:45.584 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.584 INFO:tasks.workunit.client.0.smithi136.stdout: "retry_buffer_overflow_count": { 2024-09-12T23:37:45.584 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:45.584 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:45.584 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.584 INFO:tasks.workunit.client.0.smithi136.stdout: "soft_ecc_err_rate": { 2024-09-12T23:37:45.584 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 0, 2024-09-12T23:37:45.584 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:45.584 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.584 INFO:tasks.workunit.client.0.smithi136.stdout: "thermal_throttle_status": { 2024-09-12T23:37:45.585 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:45.585 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": { 2024-09-12T23:37:45.585 INFO:tasks.workunit.client.0.smithi136.stdout: "cnt": 0, 2024-09-12T23:37:45.585 INFO:tasks.workunit.client.0.smithi136.stdout: "pct": 0 2024-09-12T23:37:45.585 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:45.585 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.585 INFO:tasks.workunit.client.0.smithi136.stdout: "timed_workload_host_reads": { 2024-09-12T23:37:45.585 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:45.585 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 15 2024-09-12T23:37:45.585 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.585 INFO:tasks.workunit.client.0.smithi136.stdout: "timed_workload_media_wear": { 2024-09-12T23:37:45.585 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:45.585 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": "6.850586" 2024-09-12T23:37:45.586 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.586 INFO:tasks.workunit.client.0.smithi136.stdout: "timed_workload_timer": { 2024-09-12T23:37:45.586 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:45.586 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 1541953 2024-09-12T23:37:45.586 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.586 INFO:tasks.workunit.client.0.smithi136.stdout: "unexp_power_loss": { 2024-09-12T23:37:45.586 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 0, 2024-09-12T23:37:45.586 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:45.586 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.586 INFO:tasks.workunit.client.0.smithi136.stdout: "wear_leveling": { 2024-09-12T23:37:45.586 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 47, 2024-09-12T23:37:45.586 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": { 2024-09-12T23:37:45.586 INFO:tasks.workunit.client.0.smithi136.stdout: "avg": 1165, 2024-09-12T23:37:45.587 INFO:tasks.workunit.client.0.smithi136.stdout: "max": 1608, 2024-09-12T23:37:45.587 INFO:tasks.workunit.client.0.smithi136.stdout: "min": 765 2024-09-12T23:37:45.587 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:45.587 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:45.587 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.587 INFO:tasks.workunit.client.0.smithi136.stdout: "Intel Smart log": "nvme0n1" 2024-09-12T23:37:45.587 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.587 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_smart_health_information_log": { 2024-09-12T23:37:45.587 INFO:tasks.workunit.client.0.smithi136.stdout: "available_spare": 100, 2024-09-12T23:37:45.587 INFO:tasks.workunit.client.0.smithi136.stdout: "available_spare_threshold": 10, 2024-09-12T23:37:45.587 INFO:tasks.workunit.client.0.smithi136.stdout: "controller_busy_time": 358, 2024-09-12T23:37:45.587 INFO:tasks.workunit.client.0.smithi136.stdout: "critical_warning": 0, 2024-09-12T23:37:45.587 INFO:tasks.workunit.client.0.smithi136.stdout: "data_units_read": 215101386, 2024-09-12T23:37:45.587 INFO:tasks.workunit.client.0.smithi136.stdout: "data_units_written": 1248622303, 2024-09-12T23:37:45.588 INFO:tasks.workunit.client.0.smithi136.stdout: "host_reads": 4349473727, 2024-09-12T23:37:45.588 INFO:tasks.workunit.client.0.smithi136.stdout: "host_writes": 24841062282, 2024-09-12T23:37:45.588 INFO:tasks.workunit.client.0.smithi136.stdout: "media_errors": 0, 2024-09-12T23:37:45.588 INFO:tasks.workunit.client.0.smithi136.stdout: "num_err_log_entries": 0, 2024-09-12T23:37:45.588 INFO:tasks.workunit.client.0.smithi136.stdout: "percentage_used": 25, 2024-09-12T23:37:45.588 INFO:tasks.workunit.client.0.smithi136.stdout: "power_cycles": 45284, 2024-09-12T23:37:45.588 INFO:tasks.workunit.client.0.smithi136.stdout: "power_on_hours": 38404, 2024-09-12T23:37:45.588 INFO:tasks.workunit.client.0.smithi136.stdout: "temperature": 23, 2024-09-12T23:37:45.588 INFO:tasks.workunit.client.0.smithi136.stdout: "unsafe_shutdowns": 45156 2024-09-12T23:37:45.588 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.588 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_vendor": "intel", 2024-09-12T23:37:45.588 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_version": { 2024-09-12T23:37:45.589 INFO:tasks.workunit.client.0.smithi136.stdout: "string": "<1.2", 2024-09-12T23:37:45.589 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:45.589 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.589 INFO:tasks.workunit.client.0.smithi136.stdout: "power_cycle_count": 45284, 2024-09-12T23:37:45.589 INFO:tasks.workunit.client.0.smithi136.stdout: "power_on_time": { 2024-09-12T23:37:45.589 INFO:tasks.workunit.client.0.smithi136.stdout: "hours": 38404 2024-09-12T23:37:45.589 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.589 INFO:tasks.workunit.client.0.smithi136.stdout: "serial_number": "deleted", 2024-09-12T23:37:45.589 INFO:tasks.workunit.client.0.smithi136.stdout: "smart_status": { 2024-09-12T23:37:45.589 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme": { 2024-09-12T23:37:45.589 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:45.589 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.589 INFO:tasks.workunit.client.0.smithi136.stdout: "passed": true 2024-09-12T23:37:45.589 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.590 INFO:tasks.workunit.client.0.smithi136.stdout: "smartctl": { 2024-09-12T23:37:45.590 INFO:tasks.workunit.client.0.smithi136.stdout: "argv": [ 2024-09-12T23:37:45.590 INFO:tasks.workunit.client.0.smithi136.stdout: "smartctl", 2024-09-12T23:37:45.590 INFO:tasks.workunit.client.0.smithi136.stdout: "-x", 2024-09-12T23:37:45.590 INFO:tasks.workunit.client.0.smithi136.stdout: "--json=o", 2024-09-12T23:37:45.590 INFO:tasks.workunit.client.0.smithi136.stdout: "/dev/nvme0n1" 2024-09-12T23:37:45.590 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:45.590 INFO:tasks.workunit.client.0.smithi136.stdout: "build_info": "(local build)", 2024-09-12T23:37:45.591 INFO:tasks.workunit.client.0.smithi136.stdout: "exit_status": 0, 2024-09-12T23:37:45.591 INFO:tasks.workunit.client.0.smithi136.stdout: "output": [ 2024-09-12T23:37:45.591 INFO:tasks.workunit.client.0.smithi136.stdout: "smartctl 7.2 2020-12-30 r5155 [x86_64-linux-5.15.0-121-generic] (local build)", 2024-09-12T23:37:45.591 INFO:tasks.workunit.client.0.smithi136.stdout: "Copyright (C) 2002-20, Bruce Allen, Christian Franke, www.smartmontools.org", 2024-09-12T23:37:45.591 INFO:tasks.workunit.client.0.smithi136.stdout: "", 2024-09-12T23:37:45.591 INFO:tasks.workunit.client.0.smithi136.stdout: "=== START OF INFORMATION SECTION ===", 2024-09-12T23:37:45.591 INFO:tasks.workunit.client.0.smithi136.stdout: "Model Number: INTEL SSDPEDMX020T4", 2024-09-12T23:37:45.591 INFO:tasks.workunit.client.0.smithi136.stdout: "Serial Number: deleted", 2024-09-12T23:37:45.591 INFO:tasks.workunit.client.0.smithi136.stdout: "Firmware Version: 8DV10171", 2024-09-12T23:37:45.591 INFO:tasks.workunit.client.0.smithi136.stdout: "PCI Vendor/Subsystem ID: 0x8086", 2024-09-12T23:37:45.591 INFO:tasks.workunit.client.0.smithi136.stdout: "IEEE OUI Identifier: 0x5cd2e4", 2024-09-12T23:37:45.591 INFO:tasks.workunit.client.0.smithi136.stdout: "Controller ID: 0", 2024-09-12T23:37:45.592 INFO:tasks.workunit.client.0.smithi136.stdout: "NVMe Version: <1.2", 2024-09-12T23:37:45.592 INFO:tasks.workunit.client.0.smithi136.stdout: "Number of Namespaces: 1", 2024-09-12T23:37:45.592 INFO:tasks.workunit.client.0.smithi136.stdout: "Namespace 1 Size/Capacity: 2,000,398,934,016 [2.00 TB]", 2024-09-12T23:37:45.592 INFO:tasks.workunit.client.0.smithi136.stdout: "Namespace 1 Formatted LBA Size: 512", 2024-09-12T23:37:45.592 INFO:tasks.workunit.client.0.smithi136.stdout: "Local Time is: Thu Sep 12 23:28:36 2024 UTC", 2024-09-12T23:37:45.592 INFO:tasks.workunit.client.0.smithi136.stdout: "Firmware Updates (0x02): 1 Slot", 2024-09-12T23:37:45.592 INFO:tasks.workunit.client.0.smithi136.stdout: "Optional Admin Commands (0x0006): Format Frmw_DL", 2024-09-12T23:37:45.592 INFO:tasks.workunit.client.0.smithi136.stdout: "Optional NVM Commands (0x0006): Wr_Unc DS_Mngmt", 2024-09-12T23:37:45.592 INFO:tasks.workunit.client.0.smithi136.stdout: "Log Page Attributes (0x02): Cmd_Eff_Lg", 2024-09-12T23:37:45.592 INFO:tasks.workunit.client.0.smithi136.stdout: "Maximum Data Transfer Size: 32 Pages", 2024-09-12T23:37:45.592 INFO:tasks.workunit.client.0.smithi136.stdout: "", 2024-09-12T23:37:45.592 INFO:tasks.workunit.client.0.smithi136.stdout: "Supported Power States", 2024-09-12T23:37:45.592 INFO:tasks.workunit.client.0.smithi136.stdout: "St Op Max Active Idle RL RT WL WT Ent_Lat Ex_Lat", 2024-09-12T23:37:45.592 INFO:tasks.workunit.client.0.smithi136.stdout: " 0 + 25.00W - - 0 0 0 0 0 0", 2024-09-12T23:37:45.593 INFO:tasks.workunit.client.0.smithi136.stdout: "", 2024-09-12T23:37:45.593 INFO:tasks.workunit.client.0.smithi136.stdout: "Supported LBA Sizes (NSID 0x1)", 2024-09-12T23:37:45.593 INFO:tasks.workunit.client.0.smithi136.stdout: "Id Fmt Data Metadt Rel_Perf", 2024-09-12T23:37:45.593 INFO:tasks.workunit.client.0.smithi136.stdout: " 0 + 512 0 2", 2024-09-12T23:37:45.593 INFO:tasks.workunit.client.0.smithi136.stdout: " 1 - 512 8 2", 2024-09-12T23:37:45.593 INFO:tasks.workunit.client.0.smithi136.stdout: " 2 - 512 16 2", 2024-09-12T23:37:45.593 INFO:tasks.workunit.client.0.smithi136.stdout: " 3 - 4096 0 0", 2024-09-12T23:37:45.593 INFO:tasks.workunit.client.0.smithi136.stdout: " 4 - 4096 8 0", 2024-09-12T23:37:45.593 INFO:tasks.workunit.client.0.smithi136.stdout: " 5 - 4096 64 0", 2024-09-12T23:37:45.593 INFO:tasks.workunit.client.0.smithi136.stdout: " 6 - 4096 128 0", 2024-09-12T23:37:45.593 INFO:tasks.workunit.client.0.smithi136.stdout: "", 2024-09-12T23:37:45.593 INFO:tasks.workunit.client.0.smithi136.stdout: "=== START OF SMART DATA SECTION ===", 2024-09-12T23:37:45.593 INFO:tasks.workunit.client.0.smithi136.stdout: "SMART overall-health self-assessment test result: PASSED", 2024-09-12T23:37:45.593 INFO:tasks.workunit.client.0.smithi136.stdout: "", 2024-09-12T23:37:45.594 INFO:tasks.workunit.client.0.smithi136.stdout: "SMART/Health Information (NVMe Log 0x02)", 2024-09-12T23:37:45.594 INFO:tasks.workunit.client.0.smithi136.stdout: "Critical Warning: 0x00", 2024-09-12T23:37:45.594 INFO:tasks.workunit.client.0.smithi136.stdout: "Temperature: 23 Celsius", 2024-09-12T23:37:45.594 INFO:tasks.workunit.client.0.smithi136.stdout: "Available Spare: 100%", 2024-09-12T23:37:45.594 INFO:tasks.workunit.client.0.smithi136.stdout: "Available Spare Threshold: 10%", 2024-09-12T23:37:45.594 INFO:tasks.workunit.client.0.smithi136.stdout: "Percentage Used: 25%", 2024-09-12T23:37:45.594 INFO:tasks.workunit.client.0.smithi136.stdout: "Data Units Read: 215,101,386 [110 TB]", 2024-09-12T23:37:45.594 INFO:tasks.workunit.client.0.smithi136.stdout: "Data Units Written: 1,248,622,303 [639 TB]", 2024-09-12T23:37:45.594 INFO:tasks.workunit.client.0.smithi136.stdout: "Host Read Commands: 4,349,473,727", 2024-09-12T23:37:45.594 INFO:tasks.workunit.client.0.smithi136.stdout: "Host Write Commands: 24,841,062,282", 2024-09-12T23:37:45.594 INFO:tasks.workunit.client.0.smithi136.stdout: "Controller Busy Time: 358", 2024-09-12T23:37:45.594 INFO:tasks.workunit.client.0.smithi136.stdout: "Power Cycles: 45,284", 2024-09-12T23:37:45.594 INFO:tasks.workunit.client.0.smithi136.stdout: "Power On Hours: 38,404", 2024-09-12T23:37:45.594 INFO:tasks.workunit.client.0.smithi136.stdout: "Unsafe Shutdowns: 45,156", 2024-09-12T23:37:45.595 INFO:tasks.workunit.client.0.smithi136.stdout: "Media and Data Integrity Errors: 0", 2024-09-12T23:37:45.595 INFO:tasks.workunit.client.0.smithi136.stdout: "Error Information Log Entries: 0", 2024-09-12T23:37:45.595 INFO:tasks.workunit.client.0.smithi136.stdout: "", 2024-09-12T23:37:45.595 INFO:tasks.workunit.client.0.smithi136.stdout: "Error Information (NVMe Log 0x01, 16 of 64 entries)", 2024-09-12T23:37:45.595 INFO:tasks.workunit.client.0.smithi136.stdout: "No Errors Logged", 2024-09-12T23:37:45.595 INFO:tasks.workunit.client.0.smithi136.stdout: "" 2024-09-12T23:37:45.595 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:45.595 INFO:tasks.workunit.client.0.smithi136.stdout: "platform_info": "x86_64-linux-5.15.0-121-generic", 2024-09-12T23:37:45.595 INFO:tasks.workunit.client.0.smithi136.stdout: "svn_revision": "5155", 2024-09-12T23:37:45.595 INFO:tasks.workunit.client.0.smithi136.stdout: "version": [ 2024-09-12T23:37:45.595 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:45.595 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:45.595 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:45.595 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.596 INFO:tasks.workunit.client.0.smithi136.stdout: "temperature": { 2024-09-12T23:37:45.596 INFO:tasks.workunit.client.0.smithi136.stdout: "current": 23 2024-09-12T23:37:45.596 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.596 INFO:tasks.workunit.client.0.smithi136.stdout: "user_capacity": { 2024-09-12T23:37:45.596 INFO:tasks.workunit.client.0.smithi136.stdout: "blocks": 3907029168, 2024-09-12T23:37:45.596 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 2000398934016 2024-09-12T23:37:45.596 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:45.596 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:45.596 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:45.596 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:45.596 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.596 INFO:tasks.workunit.client.0.smithi136.stdout: "report": { 2024-09-12T23:37:45.596 INFO:tasks.workunit.client.0.smithi136.stdout: "balancer": { 2024-09-12T23:37:45.597 INFO:tasks.workunit.client.0.smithi136.stdout: "active": true, 2024-09-12T23:37:45.597 INFO:tasks.workunit.client.0.smithi136.stdout: "mode": "" 2024-09-12T23:37:45.597 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.597 INFO:tasks.workunit.client.0.smithi136.stdout: "channels": [ 2024-09-12T23:37:45.597 INFO:tasks.workunit.client.0.smithi136.stdout: "basic", 2024-09-12T23:37:45.597 INFO:tasks.workunit.client.0.smithi136.stdout: "crash", 2024-09-12T23:37:45.597 INFO:tasks.workunit.client.0.smithi136.stdout: "device", 2024-09-12T23:37:45.597 INFO:tasks.workunit.client.0.smithi136.stdout: "ident" 2024-09-12T23:37:45.597 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:45.597 INFO:tasks.workunit.client.0.smithi136.stdout: "channels_available": [ 2024-09-12T23:37:45.597 INFO:tasks.workunit.client.0.smithi136.stdout: "basic", 2024-09-12T23:37:45.597 INFO:tasks.workunit.client.0.smithi136.stdout: "ident", 2024-09-12T23:37:45.597 INFO:tasks.workunit.client.0.smithi136.stdout: "crash", 2024-09-12T23:37:45.598 INFO:tasks.workunit.client.0.smithi136.stdout: "device", 2024-09-12T23:37:45.598 INFO:tasks.workunit.client.0.smithi136.stdout: "perf" 2024-09-12T23:37:45.598 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:45.598 INFO:tasks.workunit.client.0.smithi136.stdout: "collections_available": [ 2024-09-12T23:37:45.598 INFO:tasks.workunit.client.0.smithi136.stdout: "basic_base", 2024-09-12T23:37:45.598 INFO:tasks.workunit.client.0.smithi136.stdout: "device_base", 2024-09-12T23:37:45.598 INFO:tasks.workunit.client.0.smithi136.stdout: "crash_base", 2024-09-12T23:37:45.598 INFO:tasks.workunit.client.0.smithi136.stdout: "ident_base", 2024-09-12T23:37:45.598 INFO:tasks.workunit.client.0.smithi136.stdout: "perf_perf", 2024-09-12T23:37:45.598 INFO:tasks.workunit.client.0.smithi136.stdout: "basic_mds_metadata", 2024-09-12T23:37:45.598 INFO:tasks.workunit.client.0.smithi136.stdout: "basic_pool_usage", 2024-09-12T23:37:45.598 INFO:tasks.workunit.client.0.smithi136.stdout: "basic_usage_by_class", 2024-09-12T23:37:45.598 INFO:tasks.workunit.client.0.smithi136.stdout: "basic_rook_v01", 2024-09-12T23:37:45.598 INFO:tasks.workunit.client.0.smithi136.stdout: "perf_memory_metrics", 2024-09-12T23:37:45.599 INFO:tasks.workunit.client.0.smithi136.stdout: "basic_pool_options_bluestore" 2024-09-12T23:37:45.599 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:45.599 INFO:tasks.workunit.client.0.smithi136.stdout: "collections_opted_in": [ 2024-09-12T23:37:45.599 INFO:tasks.workunit.client.0.smithi136.stdout: "basic_base", 2024-09-12T23:37:45.599 INFO:tasks.workunit.client.0.smithi136.stdout: "device_base", 2024-09-12T23:37:45.599 INFO:tasks.workunit.client.0.smithi136.stdout: "crash_base", 2024-09-12T23:37:45.599 INFO:tasks.workunit.client.0.smithi136.stdout: "ident_base", 2024-09-12T23:37:45.599 INFO:tasks.workunit.client.0.smithi136.stdout: "perf_perf", 2024-09-12T23:37:45.599 INFO:tasks.workunit.client.0.smithi136.stdout: "basic_mds_metadata", 2024-09-12T23:37:45.599 INFO:tasks.workunit.client.0.smithi136.stdout: "basic_pool_usage", 2024-09-12T23:37:45.599 INFO:tasks.workunit.client.0.smithi136.stdout: "basic_usage_by_class", 2024-09-12T23:37:45.599 INFO:tasks.workunit.client.0.smithi136.stdout: "basic_rook_v01", 2024-09-12T23:37:45.599 INFO:tasks.workunit.client.0.smithi136.stdout: "perf_memory_metrics", 2024-09-12T23:37:45.599 INFO:tasks.workunit.client.0.smithi136.stdout: "basic_pool_options_bluestore" 2024-09-12T23:37:45.600 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:45.600 INFO:tasks.workunit.client.0.smithi136.stdout: "config": { 2024-09-12T23:37:45.600 INFO:tasks.workunit.client.0.smithi136.stdout: "active_changed": [ 2024-09-12T23:37:45.600 INFO:tasks.workunit.client.0.smithi136.stdout: "auth_allow_insecure_global_id_reclaim", 2024-09-12T23:37:45.600 INFO:tasks.workunit.client.0.smithi136.stdout: "auth_debug", 2024-09-12T23:37:45.600 INFO:tasks.workunit.client.0.smithi136.stdout: "auth_mon_ticket_ttl", 2024-09-12T23:37:45.600 INFO:tasks.workunit.client.0.smithi136.stdout: "auth_service_ticket_ttl", 2024-09-12T23:37:45.600 INFO:tasks.workunit.client.0.smithi136.stdout: "bdev_debug_aio", 2024-09-12T23:37:45.600 INFO:tasks.workunit.client.0.smithi136.stdout: "client_check_pool_perm", 2024-09-12T23:37:45.600 INFO:tasks.workunit.client.0.smithi136.stdout: "client_mount_gid", 2024-09-12T23:37:45.600 INFO:tasks.workunit.client.0.smithi136.stdout: "client_mount_uid", 2024-09-12T23:37:45.600 INFO:tasks.workunit.client.0.smithi136.stdout: "client_quota", 2024-09-12T23:37:45.600 INFO:tasks.workunit.client.0.smithi136.stdout: "container_image", 2024-09-12T23:37:45.600 INFO:tasks.workunit.client.0.smithi136.stdout: "daemonize", 2024-09-12T23:37:45.601 INFO:tasks.workunit.client.0.smithi136.stdout: "debug_asserts_on_shutdown", 2024-09-12T23:37:45.601 INFO:tasks.workunit.client.0.smithi136.stdout: "debug_mgr", 2024-09-12T23:37:45.601 INFO:tasks.workunit.client.0.smithi136.stdout: "debug_mon", 2024-09-12T23:37:45.601 INFO:tasks.workunit.client.0.smithi136.stdout: "debug_ms", 2024-09-12T23:37:45.601 INFO:tasks.workunit.client.0.smithi136.stdout: "debug_osd", 2024-09-12T23:37:45.601 INFO:tasks.workunit.client.0.smithi136.stdout: "debug_paxos", 2024-09-12T23:37:45.601 INFO:tasks.workunit.client.0.smithi136.stdout: "keyring", 2024-09-12T23:37:45.601 INFO:tasks.workunit.client.0.smithi136.stdout: "leveldb_block_size", 2024-09-12T23:37:45.601 INFO:tasks.workunit.client.0.smithi136.stdout: "leveldb_cache_size", 2024-09-12T23:37:45.601 INFO:tasks.workunit.client.0.smithi136.stdout: "leveldb_compression", 2024-09-12T23:37:45.601 INFO:tasks.workunit.client.0.smithi136.stdout: "leveldb_log", 2024-09-12T23:37:45.601 INFO:tasks.workunit.client.0.smithi136.stdout: "leveldb_write_buffer_size", 2024-09-12T23:37:45.601 INFO:tasks.workunit.client.0.smithi136.stdout: "log_stderr_prefix", 2024-09-12T23:37:45.601 INFO:tasks.workunit.client.0.smithi136.stdout: "log_to_file", 2024-09-12T23:37:45.602 INFO:tasks.workunit.client.0.smithi136.stdout: "log_to_journald", 2024-09-12T23:37:45.602 INFO:tasks.workunit.client.0.smithi136.stdout: "log_to_stderr", 2024-09-12T23:37:45.602 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_allow_pool_delete", 2024-09-12T23:37:45.602 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_clock_drift_allowed", 2024-09-12T23:37:45.602 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_cluster_log_to_file", 2024-09-12T23:37:45.602 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_cluster_log_to_stderr", 2024-09-12T23:37:45.602 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_data_avail_warn", 2024-09-12T23:37:45.602 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_election_default_strategy", 2024-09-12T23:37:45.602 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_host", 2024-09-12T23:37:45.602 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_max_pg_per_osd", 2024-09-12T23:37:45.602 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_mgr_mkfs_grace", 2024-09-12T23:37:45.602 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_osd_prime_pg_temp", 2024-09-12T23:37:45.602 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_osd_reporter_subtree_level", 2024-09-12T23:37:45.602 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_pg_warn_max_object_skew", 2024-09-12T23:37:45.603 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_reweight_min_bytes_per_osd", 2024-09-12T23:37:45.603 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_reweight_min_pgs_per_osd", 2024-09-12T23:37:45.603 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_crush_straw_calc_version_zero", 2024-09-12T23:37:45.603 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_insecure_global_id_reclaim", 2024-09-12T23:37:45.603 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed", 2024-09-12T23:37:45.603 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_legacy_crush_tunables", 2024-09-12T23:37:45.603 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_osd_down_out_interval_zero", 2024-09-12T23:37:45.603 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_pool_pg_num_not_power_of_two", 2024-09-12T23:37:45.603 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_too_few_osds", 2024-09-12T23:37:45.603 INFO:tasks.workunit.client.0.smithi136.stdout: "ms_die_on_bug", 2024-09-12T23:37:45.603 INFO:tasks.workunit.client.0.smithi136.stdout: "ms_die_on_old_message", 2024-09-12T23:37:45.603 INFO:tasks.workunit.client.0.smithi136.stdout: "no_config_file", 2024-09-12T23:37:45.603 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_class_default_list", 2024-09-12T23:37:45.603 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_class_load_list", 2024-09-12T23:37:45.604 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_misdirected_ops", 2024-09-12T23:37:45.604 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_op_order", 2024-09-12T23:37:45.604 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_pg_log_writeout", 2024-09-12T23:37:45.604 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_shutdown", 2024-09-12T23:37:45.604 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_verify_cached_snaps", 2024-09-12T23:37:45.604 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_verify_missing_on_start", 2024-09-12T23:37:45.604 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_verify_stray_on_activate", 2024-09-12T23:37:45.604 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_deep_scrub_update_digest_min_age", 2024-09-12T23:37:45.604 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mclock_profile", 2024-09-12T23:37:45.604 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mclock_scheduler_background_best_effort_lim", 2024-09-12T23:37:45.604 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mclock_scheduler_background_best_effort_res", 2024-09-12T23:37:45.604 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mclock_scheduler_background_best_effort_wgt", 2024-09-12T23:37:45.604 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mclock_scheduler_background_recovery_lim", 2024-09-12T23:37:45.604 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mclock_scheduler_background_recovery_res", 2024-09-12T23:37:45.605 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mclock_scheduler_background_recovery_wgt", 2024-09-12T23:37:45.605 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mclock_scheduler_client_lim", 2024-09-12T23:37:45.605 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mclock_scheduler_client_res", 2024-09-12T23:37:45.605 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mclock_scheduler_client_wgt", 2024-09-12T23:37:45.605 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_memory_target", 2024-09-12T23:37:45.605 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_memory_target_autotune", 2024-09-12T23:37:45.605 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_op_queue", 2024-09-12T23:37:45.605 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_op_queue_cut_off", 2024-09-12T23:37:45.605 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_pool_default_erasure_code_profile", 2024-09-12T23:37:45.605 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_pool_default_pg_autoscale_mode", 2024-09-12T23:37:45.605 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_recover_clone_overlap", 2024-09-12T23:37:45.605 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_recovery_max_chunk", 2024-09-12T23:37:45.605 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_scrub_load_threshold", 2024-09-12T23:37:45.605 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_scrub_max_interval", 2024-09-12T23:37:45.606 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_shutdown_pgref_assert", 2024-09-12T23:37:45.606 INFO:tasks.workunit.client.0.smithi136.stdout: "public_addr", 2024-09-12T23:37:45.606 INFO:tasks.workunit.client.0.smithi136.stdout: "public_addrv", 2024-09-12T23:37:45.606 INFO:tasks.workunit.client.0.smithi136.stdout: "public_network", 2024-09-12T23:37:45.606 INFO:tasks.workunit.client.0.smithi136.stdout: "rbd_default_features", 2024-09-12T23:37:45.606 INFO:tasks.workunit.client.0.smithi136.stdout: "rbd_qos_exclude_ops", 2024-09-12T23:37:45.606 INFO:tasks.workunit.client.0.smithi136.stdout: "setgroup", 2024-09-12T23:37:45.606 INFO:tasks.workunit.client.0.smithi136.stdout: "setuser" 2024-09-12T23:37:45.606 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:45.606 INFO:tasks.workunit.client.0.smithi136.stdout: "cluster_changed": [ 2024-09-12T23:37:45.606 INFO:tasks.workunit.client.0.smithi136.stdout: "auth_allow_insecure_global_id_reclaim", 2024-09-12T23:37:45.606 INFO:tasks.workunit.client.0.smithi136.stdout: "auth_debug", 2024-09-12T23:37:45.606 INFO:tasks.workunit.client.0.smithi136.stdout: "auth_mon_ticket_ttl", 2024-09-12T23:37:45.606 INFO:tasks.workunit.client.0.smithi136.stdout: "auth_service_ticket_ttl", 2024-09-12T23:37:45.606 INFO:tasks.workunit.client.0.smithi136.stdout: "bdev_debug_aio", 2024-09-12T23:37:45.607 INFO:tasks.workunit.client.0.smithi136.stdout: "container_image", 2024-09-12T23:37:45.607 INFO:tasks.workunit.client.0.smithi136.stdout: "debug_asserts_on_shutdown", 2024-09-12T23:37:45.607 INFO:tasks.workunit.client.0.smithi136.stdout: "debug_mgr", 2024-09-12T23:37:45.607 INFO:tasks.workunit.client.0.smithi136.stdout: "debug_mon", 2024-09-12T23:37:45.607 INFO:tasks.workunit.client.0.smithi136.stdout: "debug_ms", 2024-09-12T23:37:45.607 INFO:tasks.workunit.client.0.smithi136.stdout: "debug_osd", 2024-09-12T23:37:45.607 INFO:tasks.workunit.client.0.smithi136.stdout: "debug_paxos", 2024-09-12T23:37:45.607 INFO:tasks.workunit.client.0.smithi136.stdout: "log_to_file", 2024-09-12T23:37:45.607 INFO:tasks.workunit.client.0.smithi136.stdout: "log_to_journald", 2024-09-12T23:37:45.607 INFO:tasks.workunit.client.0.smithi136.stdout: "log_to_stderr", 2024-09-12T23:37:45.607 INFO:tasks.workunit.client.0.smithi136.stdout: "mds_join_fs", 2024-09-12T23:37:45.607 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/cephadm/allow_ptrace", 2024-09-12T23:37:45.607 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/cephadm/container_init", 2024-09-12T23:37:45.607 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/cephadm/migration_current", 2024-09-12T23:37:45.608 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/cephadm/use_repo_digest", 2024-09-12T23:37:45.608 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/dashboard/ALERTMANAGER_API_HOST", 2024-09-12T23:37:45.608 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/dashboard/GRAFANA_API_SSL_VERIFY", 2024-09-12T23:37:45.608 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/dashboard/GRAFANA_API_URL", 2024-09-12T23:37:45.608 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/dashboard/PROMETHEUS_API_HOST", 2024-09-12T23:37:45.608 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/dashboard/ssl_server_port", 2024-09-12T23:37:45.608 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/orchestrator/orchestrator", 2024-09-12T23:37:45.608 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/telemetry/channel_ident", 2024-09-12T23:37:45.608 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/telemetry/description", 2024-09-12T23:37:45.608 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/telemetry/organization", 2024-09-12T23:37:45.608 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_allow_pool_delete", 2024-09-12T23:37:45.608 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_clock_drift_allowed", 2024-09-12T23:37:45.608 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_data_avail_warn", 2024-09-12T23:37:45.609 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_election_default_strategy", 2024-09-12T23:37:45.609 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_max_pg_per_osd", 2024-09-12T23:37:45.609 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_mgr_mkfs_grace", 2024-09-12T23:37:45.609 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_osd_prime_pg_temp", 2024-09-12T23:37:45.609 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_osd_reporter_subtree_level", 2024-09-12T23:37:45.609 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_pg_warn_max_object_skew", 2024-09-12T23:37:45.609 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_reweight_min_bytes_per_osd", 2024-09-12T23:37:45.609 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_reweight_min_pgs_per_osd", 2024-09-12T23:37:45.609 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_crush_straw_calc_version_zero", 2024-09-12T23:37:45.609 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_insecure_global_id_reclaim", 2024-09-12T23:37:45.609 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed", 2024-09-12T23:37:45.609 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_legacy_crush_tunables", 2024-09-12T23:37:45.609 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_osd_down_out_interval_zero", 2024-09-12T23:37:45.609 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_pool_pg_num_not_power_of_two", 2024-09-12T23:37:45.610 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_too_few_osds", 2024-09-12T23:37:45.610 INFO:tasks.workunit.client.0.smithi136.stdout: "ms_die_on_bug", 2024-09-12T23:37:45.610 INFO:tasks.workunit.client.0.smithi136.stdout: "ms_die_on_old_message", 2024-09-12T23:37:45.610 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_class_default_list", 2024-09-12T23:37:45.610 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_class_load_list", 2024-09-12T23:37:45.610 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_misdirected_ops", 2024-09-12T23:37:45.610 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_op_order", 2024-09-12T23:37:45.610 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_pg_log_writeout", 2024-09-12T23:37:45.610 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_shutdown", 2024-09-12T23:37:45.610 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_verify_cached_snaps", 2024-09-12T23:37:45.610 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_verify_missing_on_start", 2024-09-12T23:37:45.610 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_verify_stray_on_activate", 2024-09-12T23:37:45.610 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_deep_scrub_update_digest_min_age", 2024-09-12T23:37:45.610 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mclock_profile", 2024-09-12T23:37:45.611 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_memory_target", 2024-09-12T23:37:45.611 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_memory_target_autotune", 2024-09-12T23:37:45.611 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_op_queue", 2024-09-12T23:37:45.611 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_op_queue_cut_off", 2024-09-12T23:37:45.611 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_pool_default_erasure_code_profile", 2024-09-12T23:37:45.611 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_pool_default_pg_autoscale_mode", 2024-09-12T23:37:45.611 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_recover_clone_overlap", 2024-09-12T23:37:45.611 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_recovery_max_chunk", 2024-09-12T23:37:45.611 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_scrub_load_threshold", 2024-09-12T23:37:45.611 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_scrub_max_interval", 2024-09-12T23:37:45.611 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_shutdown_pgref_assert", 2024-09-12T23:37:45.611 INFO:tasks.workunit.client.0.smithi136.stdout: "public_network", 2024-09-12T23:37:45.611 INFO:tasks.workunit.client.0.smithi136.stdout: "rgw_cache_enabled", 2024-09-12T23:37:45.611 INFO:tasks.workunit.client.0.smithi136.stdout: "rgw_enable_ops_log", 2024-09-12T23:37:45.612 INFO:tasks.workunit.client.0.smithi136.stdout: "rgw_enable_usage_log" 2024-09-12T23:37:45.612 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:45.612 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.612 INFO:tasks.workunit.client.0.smithi136.stdout: "contact": null, 2024-09-12T23:37:45.612 INFO:tasks.workunit.client.0.smithi136.stdout: "crashes": [], 2024-09-12T23:37:45.612 INFO:tasks.workunit.client.0.smithi136.stdout: "created": "2024-09-12T23:23:24.585690Z", 2024-09-12T23:37:45.612 INFO:tasks.workunit.client.0.smithi136.stdout: "crush": { 2024-09-12T23:37:45.612 INFO:tasks.workunit.client.0.smithi136.stdout: "bucket_algs": { 2024-09-12T23:37:45.612 INFO:tasks.workunit.client.0.smithi136.stdout: "straw2": 3 2024-09-12T23:37:45.612 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.612 INFO:tasks.workunit.client.0.smithi136.stdout: "bucket_sizes": { 2024-09-12T23:37:45.612 INFO:tasks.workunit.client.0.smithi136.stdout: "2": 1, 2024-09-12T23:37:45.612 INFO:tasks.workunit.client.0.smithi136.stdout: "4": 2 2024-09-12T23:37:45.613 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.613 INFO:tasks.workunit.client.0.smithi136.stdout: "bucket_types": { 2024-09-12T23:37:45.613 INFO:tasks.workunit.client.0.smithi136.stdout: "1": 2, 2024-09-12T23:37:45.613 INFO:tasks.workunit.client.0.smithi136.stdout: "11": 1 2024-09-12T23:37:45.613 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.613 INFO:tasks.workunit.client.0.smithi136.stdout: "compat_weight_set": false, 2024-09-12T23:37:45.613 INFO:tasks.workunit.client.0.smithi136.stdout: "device_classes": [ 2024-09-12T23:37:45.613 INFO:tasks.workunit.client.0.smithi136.stdout: 8 2024-09-12T23:37:45.613 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:45.613 INFO:tasks.workunit.client.0.smithi136.stdout: "num_buckets": 6, 2024-09-12T23:37:45.613 INFO:tasks.workunit.client.0.smithi136.stdout: "num_devices": 8, 2024-09-12T23:37:45.613 INFO:tasks.workunit.client.0.smithi136.stdout: "num_rules": 1, 2024-09-12T23:37:45.613 INFO:tasks.workunit.client.0.smithi136.stdout: "num_types": 12, 2024-09-12T23:37:45.613 INFO:tasks.workunit.client.0.smithi136.stdout: "num_weight_sets": 0, 2024-09-12T23:37:45.614 INFO:tasks.workunit.client.0.smithi136.stdout: "tunables": { 2024-09-12T23:37:45.614 INFO:tasks.workunit.client.0.smithi136.stdout: "allowed_bucket_algs": 54, 2024-09-12T23:37:45.614 INFO:tasks.workunit.client.0.smithi136.stdout: "choose_local_fallback_tries": 0, 2024-09-12T23:37:45.614 INFO:tasks.workunit.client.0.smithi136.stdout: "choose_local_tries": 0, 2024-09-12T23:37:45.614 INFO:tasks.workunit.client.0.smithi136.stdout: "choose_total_tries": 50, 2024-09-12T23:37:45.614 INFO:tasks.workunit.client.0.smithi136.stdout: "chooseleaf_descend_once": 1, 2024-09-12T23:37:45.614 INFO:tasks.workunit.client.0.smithi136.stdout: "chooseleaf_stable": 1, 2024-09-12T23:37:45.614 INFO:tasks.workunit.client.0.smithi136.stdout: "chooseleaf_vary_r": 1, 2024-09-12T23:37:45.614 INFO:tasks.workunit.client.0.smithi136.stdout: "has_v2_rules": 0, 2024-09-12T23:37:45.614 INFO:tasks.workunit.client.0.smithi136.stdout: "has_v3_rules": 0, 2024-09-12T23:37:45.614 INFO:tasks.workunit.client.0.smithi136.stdout: "has_v4_buckets": 1, 2024-09-12T23:37:45.614 INFO:tasks.workunit.client.0.smithi136.stdout: "has_v5_rules": 0, 2024-09-12T23:37:45.614 INFO:tasks.workunit.client.0.smithi136.stdout: "legacy_tunables": 0, 2024-09-12T23:37:45.614 INFO:tasks.workunit.client.0.smithi136.stdout: "minimum_required_version": "jewel", 2024-09-12T23:37:45.615 INFO:tasks.workunit.client.0.smithi136.stdout: "optimal_tunables": 1, 2024-09-12T23:37:45.615 INFO:tasks.workunit.client.0.smithi136.stdout: "profile": "jewel", 2024-09-12T23:37:45.615 INFO:tasks.workunit.client.0.smithi136.stdout: "require_feature_tunables": 1, 2024-09-12T23:37:45.615 INFO:tasks.workunit.client.0.smithi136.stdout: "require_feature_tunables2": 1, 2024-09-12T23:37:45.615 INFO:tasks.workunit.client.0.smithi136.stdout: "require_feature_tunables3": 1, 2024-09-12T23:37:45.615 INFO:tasks.workunit.client.0.smithi136.stdout: "require_feature_tunables5": 1, 2024-09-12T23:37:45.615 INFO:tasks.workunit.client.0.smithi136.stdout: "straw_calc_version": 1 2024-09-12T23:37:45.615 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:45.615 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.615 INFO:tasks.workunit.client.0.smithi136.stdout: "description": "upgrade test cluster", 2024-09-12T23:37:45.615 INFO:tasks.workunit.client.0.smithi136.stdout: "fs": { 2024-09-12T23:37:45.615 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1, 2024-09-12T23:37:45.615 INFO:tasks.workunit.client.0.smithi136.stdout: "feature_flags": { 2024-09-12T23:37:45.615 INFO:tasks.workunit.client.0.smithi136.stdout: "enable_multiple": true, 2024-09-12T23:37:45.616 INFO:tasks.workunit.client.0.smithi136.stdout: "ever_enabled_multiple": true 2024-09-12T23:37:45.616 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.616 INFO:tasks.workunit.client.0.smithi136.stdout: "filesystems": [ 2024-09-12T23:37:45.616 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:45.616 INFO:tasks.workunit.client.0.smithi136.stdout: "approx_ctime": "2024-09", 2024-09-12T23:37:45.616 INFO:tasks.workunit.client.0.smithi136.stdout: "balancer_enabled": false, 2024-09-12T23:37:45.616 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:45.616 INFO:tasks.workunit.client.0.smithi136.stdout: "cached_caps": 0, 2024-09-12T23:37:45.616 INFO:tasks.workunit.client.0.smithi136.stdout: "cached_dns": 10, 2024-09-12T23:37:45.616 INFO:tasks.workunit.client.0.smithi136.stdout: "cached_inos": 13, 2024-09-12T23:37:45.616 INFO:tasks.workunit.client.0.smithi136.stdout: "cached_subtrees": 2, 2024-09-12T23:37:45.616 INFO:tasks.workunit.client.0.smithi136.stdout: "ever_allowed_features": 0, 2024-09-12T23:37:45.616 INFO:tasks.workunit.client.0.smithi136.stdout: "explicitly_allowed_features": 0, 2024-09-12T23:37:45.616 INFO:tasks.workunit.client.0.smithi136.stdout: "files": 0, 2024-09-12T23:37:45.617 INFO:tasks.workunit.client.0.smithi136.stdout: "max_mds": 1, 2024-09-12T23:37:45.617 INFO:tasks.workunit.client.0.smithi136.stdout: "num_data_pools": 1, 2024-09-12T23:37:45.617 INFO:tasks.workunit.client.0.smithi136.stdout: "num_in": 1, 2024-09-12T23:37:45.617 INFO:tasks.workunit.client.0.smithi136.stdout: "num_mds": 1, 2024-09-12T23:37:45.617 INFO:tasks.workunit.client.0.smithi136.stdout: "num_sessions": 0, 2024-09-12T23:37:45.617 INFO:tasks.workunit.client.0.smithi136.stdout: "num_standby_replay": 0, 2024-09-12T23:37:45.617 INFO:tasks.workunit.client.0.smithi136.stdout: "num_up": 1, 2024-09-12T23:37:45.617 INFO:tasks.workunit.client.0.smithi136.stdout: "snaps": 0, 2024-09-12T23:37:45.617 INFO:tasks.workunit.client.0.smithi136.stdout: "standby_count_wanted": 0 2024-09-12T23:37:45.617 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:45.617 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:45.617 INFO:tasks.workunit.client.0.smithi136.stdout: "num_standby_mds": 1, 2024-09-12T23:37:45.617 INFO:tasks.workunit.client.0.smithi136.stdout: "total_num_mds": 2 2024-09-12T23:37:45.618 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.618 INFO:tasks.workunit.client.0.smithi136.stdout: "hosts": { 2024-09-12T23:37:45.618 INFO:tasks.workunit.client.0.smithi136.stdout: "num": 2, 2024-09-12T23:37:45.621 INFO:tasks.workunit.client.0.smithi136.stdout: "num_with_mds": 2, 2024-09-12T23:37:45.621 INFO:tasks.workunit.client.0.smithi136.stdout: "num_with_mgr": 2, 2024-09-12T23:37:45.621 INFO:tasks.workunit.client.0.smithi136.stdout: "num_with_mon": 2, 2024-09-12T23:37:45.621 INFO:tasks.workunit.client.0.smithi136.stdout: "num_with_osd": 2 2024-09-12T23:37:45.621 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.622 INFO:tasks.workunit.client.0.smithi136.stdout: "leaderboard": false, 2024-09-12T23:37:45.622 INFO:tasks.workunit.client.0.smithi136.stdout: "license": "sharing-1-0", 2024-09-12T23:37:45.622 INFO:tasks.workunit.client.0.smithi136.stdout: "metadata": { 2024-09-12T23:37:45.622 INFO:tasks.workunit.client.0.smithi136.stdout: "mds": { 2024-09-12T23:37:45.622 INFO:tasks.workunit.client.0.smithi136.stdout: "arch": { 2024-09-12T23:37:45.623 INFO:tasks.workunit.client.0.smithi136.stdout: "x86_64": 2 2024-09-12T23:37:45.623 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.623 INFO:tasks.workunit.client.0.smithi136.stdout: "ceph_version": { 2024-09-12T23:37:45.623 INFO:tasks.workunit.client.0.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-12T23:37:45.623 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.623 INFO:tasks.workunit.client.0.smithi136.stdout: "cpu": { 2024-09-12T23:37:45.624 INFO:tasks.workunit.client.0.smithi136.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 2 2024-09-12T23:37:45.624 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.624 INFO:tasks.workunit.client.0.smithi136.stdout: "distro": { 2024-09-12T23:37:45.624 INFO:tasks.workunit.client.0.smithi136.stdout: "centos": 2 2024-09-12T23:37:45.624 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.625 INFO:tasks.workunit.client.0.smithi136.stdout: "distro_description": { 2024-09-12T23:37:45.625 INFO:tasks.workunit.client.0.smithi136.stdout: "CentOS Stream 9": 2 2024-09-12T23:37:45.625 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.625 INFO:tasks.workunit.client.0.smithi136.stdout: "kernel_description": { 2024-09-12T23:37:45.625 INFO:tasks.workunit.client.0.smithi136.stdout: "#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024": 2 2024-09-12T23:37:45.625 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.626 INFO:tasks.workunit.client.0.smithi136.stdout: "kernel_version": { 2024-09-12T23:37:45.626 INFO:tasks.workunit.client.0.smithi136.stdout: "5.15.0-121-generic": 2 2024-09-12T23:37:45.626 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.626 INFO:tasks.workunit.client.0.smithi136.stdout: "os": { 2024-09-12T23:37:45.626 INFO:tasks.workunit.client.0.smithi136.stdout: "Linux": 2 2024-09-12T23:37:45.626 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:45.627 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.627 INFO:tasks.workunit.client.0.smithi136.stdout: "mon": { 2024-09-12T23:37:45.627 INFO:tasks.workunit.client.0.smithi136.stdout: "arch": { 2024-09-12T23:37:45.627 INFO:tasks.workunit.client.0.smithi136.stdout: "x86_64": 3 2024-09-12T23:37:45.627 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.628 INFO:tasks.workunit.client.0.smithi136.stdout: "ceph_version": { 2024-09-12T23:37:45.628 INFO:tasks.workunit.client.0.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 3 2024-09-12T23:37:45.628 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.628 INFO:tasks.workunit.client.0.smithi136.stdout: "cpu": { 2024-09-12T23:37:45.628 INFO:tasks.workunit.client.0.smithi136.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 3 2024-09-12T23:37:45.629 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.629 INFO:tasks.workunit.client.0.smithi136.stdout: "distro": { 2024-09-12T23:37:45.629 INFO:tasks.workunit.client.0.smithi136.stdout: "centos": 3 2024-09-12T23:37:45.629 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.629 INFO:tasks.workunit.client.0.smithi136.stdout: "distro_description": { 2024-09-12T23:37:45.629 INFO:tasks.workunit.client.0.smithi136.stdout: "CentOS Stream 9": 3 2024-09-12T23:37:45.630 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.630 INFO:tasks.workunit.client.0.smithi136.stdout: "kernel_description": { 2024-09-12T23:37:45.630 INFO:tasks.workunit.client.0.smithi136.stdout: "#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024": 3 2024-09-12T23:37:45.630 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.630 INFO:tasks.workunit.client.0.smithi136.stdout: "kernel_version": { 2024-09-12T23:37:45.631 INFO:tasks.workunit.client.0.smithi136.stdout: "5.15.0-121-generic": 3 2024-09-12T23:37:45.631 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.631 INFO:tasks.workunit.client.0.smithi136.stdout: "os": { 2024-09-12T23:37:45.631 INFO:tasks.workunit.client.0.smithi136.stdout: "Linux": 3 2024-09-12T23:37:45.631 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:45.632 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.632 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:45.632 INFO:tasks.workunit.client.0.smithi136.stdout: "arch": { 2024-09-12T23:37:45.632 INFO:tasks.workunit.client.0.smithi136.stdout: "x86_64": 8 2024-09-12T23:37:45.632 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.633 INFO:tasks.workunit.client.0.smithi136.stdout: "ceph_version": { 2024-09-12T23:37:45.633 INFO:tasks.workunit.client.0.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 8 2024-09-12T23:37:45.633 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.633 INFO:tasks.workunit.client.0.smithi136.stdout: "cpu": { 2024-09-12T23:37:45.633 INFO:tasks.workunit.client.0.smithi136.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 8 2024-09-12T23:37:45.633 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.634 INFO:tasks.workunit.client.0.smithi136.stdout: "distro": { 2024-09-12T23:37:45.634 INFO:tasks.workunit.client.0.smithi136.stdout: "centos": 8 2024-09-12T23:37:45.634 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.634 INFO:tasks.workunit.client.0.smithi136.stdout: "distro_description": { 2024-09-12T23:37:45.634 INFO:tasks.workunit.client.0.smithi136.stdout: "CentOS Stream 9": 8 2024-09-12T23:37:45.634 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.635 INFO:tasks.workunit.client.0.smithi136.stdout: "kernel_description": { 2024-09-12T23:37:45.635 INFO:tasks.workunit.client.0.smithi136.stdout: "#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024": 8 2024-09-12T23:37:45.635 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.635 INFO:tasks.workunit.client.0.smithi136.stdout: "kernel_version": { 2024-09-12T23:37:45.635 INFO:tasks.workunit.client.0.smithi136.stdout: "5.15.0-121-generic": 8 2024-09-12T23:37:45.635 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.636 INFO:tasks.workunit.client.0.smithi136.stdout: "os": { 2024-09-12T23:37:45.636 INFO:tasks.workunit.client.0.smithi136.stdout: "Linux": 8 2024-09-12T23:37:45.636 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.636 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_objectstore": { 2024-09-12T23:37:45.636 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore": 8 2024-09-12T23:37:45.637 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.637 INFO:tasks.workunit.client.0.smithi136.stdout: "rotational": { 2024-09-12T23:37:45.637 INFO:tasks.workunit.client.0.smithi136.stdout: "0": 8 2024-09-12T23:37:45.637 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:45.637 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:45.638 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.638 INFO:tasks.workunit.client.0.smithi136.stdout: "mon": { 2024-09-12T23:37:45.638 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 3, 2024-09-12T23:37:45.638 INFO:tasks.workunit.client.0.smithi136.stdout: "features": { 2024-09-12T23:37:45.638 INFO:tasks.workunit.client.0.smithi136.stdout: "optional": [], 2024-09-12T23:37:45.638 INFO:tasks.workunit.client.0.smithi136.stdout: "persistent": [ 2024-09-12T23:37:45.639 INFO:tasks.workunit.client.0.smithi136.stdout: "kraken", 2024-09-12T23:37:45.639 INFO:tasks.workunit.client.0.smithi136.stdout: "luminous", 2024-09-12T23:37:45.639 INFO:tasks.workunit.client.0.smithi136.stdout: "mimic", 2024-09-12T23:37:45.639 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap-prune", 2024-09-12T23:37:45.639 INFO:tasks.workunit.client.0.smithi136.stdout: "nautilus", 2024-09-12T23:37:45.639 INFO:tasks.workunit.client.0.smithi136.stdout: "octopus", 2024-09-12T23:37:45.640 INFO:tasks.workunit.client.0.smithi136.stdout: "pacific", 2024-09-12T23:37:45.640 INFO:tasks.workunit.client.0.smithi136.stdout: "elector-pinging", 2024-09-12T23:37:45.640 INFO:tasks.workunit.client.0.smithi136.stdout: "quincy" 2024-09-12T23:37:45.640 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:45.640 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.641 INFO:tasks.workunit.client.0.smithi136.stdout: "ipv4_addr_mons": 6, 2024-09-12T23:37:45.641 INFO:tasks.workunit.client.0.smithi136.stdout: "ipv6_addr_mons": 0, 2024-09-12T23:37:45.641 INFO:tasks.workunit.client.0.smithi136.stdout: "min_mon_release": 17, 2024-09-12T23:37:45.641 INFO:tasks.workunit.client.0.smithi136.stdout: "v1_addr_mons": 3, 2024-09-12T23:37:45.641 INFO:tasks.workunit.client.0.smithi136.stdout: "v2_addr_mons": 3 2024-09-12T23:37:45.641 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.642 INFO:tasks.workunit.client.0.smithi136.stdout: "organization": "ceph-qa", 2024-09-12T23:37:45.642 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:45.642 INFO:tasks.workunit.client.0.smithi136.stdout: "cluster_network": false, 2024-09-12T23:37:45.642 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 8, 2024-09-12T23:37:45.642 INFO:tasks.workunit.client.0.smithi136.stdout: "require_min_compat_client": "luminous", 2024-09-12T23:37:45.643 INFO:tasks.workunit.client.0.smithi136.stdout: "require_osd_release": "quincy" 2024-09-12T23:37:45.643 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.643 INFO:tasks.workunit.client.0.smithi136.stdout: "pools": [ 2024-09-12T23:37:45.643 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:45.643 INFO:tasks.workunit.client.0.smithi136.stdout: "application": [ 2024-09-12T23:37:45.643 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr" 2024-09-12T23:37:45.644 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:45.644 INFO:tasks.workunit.client.0.smithi136.stdout: "cache_mode": "none", 2024-09-12T23:37:45.644 INFO:tasks.workunit.client.0.smithi136.stdout: "erasure_code_profile": {}, 2024-09-12T23:37:45.644 INFO:tasks.workunit.client.0.smithi136.stdout: "min_size": 2, 2024-09-12T23:37:45.644 INFO:tasks.workunit.client.0.smithi136.stdout: "options": {}, 2024-09-12T23:37:45.645 INFO:tasks.workunit.client.0.smithi136.stdout: "pg_autoscale_mode": "off", 2024-09-12T23:37:45.645 INFO:tasks.workunit.client.0.smithi136.stdout: "pg_num": 1, 2024-09-12T23:37:45.645 INFO:tasks.workunit.client.0.smithi136.stdout: "pgp_num": 1, 2024-09-12T23:37:45.645 INFO:tasks.workunit.client.0.smithi136.stdout: "pool": 1, 2024-09-12T23:37:45.645 INFO:tasks.workunit.client.0.smithi136.stdout: "size": 3, 2024-09-12T23:37:45.645 INFO:tasks.workunit.client.0.smithi136.stdout: "stats": { 2024-09-12T23:37:45.646 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_raw": 727116228874, 2024-09-12T23:37:45.646 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_used": 1781760, 2024-09-12T23:37:45.646 INFO:tasks.workunit.client.0.smithi136.stdout: "compress_bytes_used": 0, 2024-09-12T23:37:45.646 INFO:tasks.workunit.client.0.smithi136.stdout: "compress_under_bytes": 0, 2024-09-12T23:37:45.646 INFO:tasks.workunit.client.0.smithi136.stdout: "data_bytes_used": 1781760, 2024-09-12T23:37:45.647 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty": 0, 2024-09-12T23:37:45.647 INFO:tasks.workunit.client.0.smithi136.stdout: "max_avail": 242372067328, 2024-09-12T23:37:45.647 INFO:tasks.workunit.client.0.smithi136.stdout: "objects": 2, 2024-09-12T23:37:45.647 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_bytes_used": 0, 2024-09-12T23:37:45.647 INFO:tasks.workunit.client.0.smithi136.stdout: "percent_used": 2.4504413431714056e-06, 2024-09-12T23:37:45.647 INFO:tasks.workunit.client.0.smithi136.stdout: "quota_bytes": 0, 2024-09-12T23:37:45.648 INFO:tasks.workunit.client.0.smithi136.stdout: "quota_objects": 0, 2024-09-12T23:37:45.648 INFO:tasks.workunit.client.0.smithi136.stdout: "rd": 126, 2024-09-12T23:37:45.648 INFO:tasks.workunit.client.0.smithi136.stdout: "rd_bytes": 236544, 2024-09-12T23:37:45.648 INFO:tasks.workunit.client.0.smithi136.stdout: "stored": 590368, 2024-09-12T23:37:45.648 INFO:tasks.workunit.client.0.smithi136.stdout: "stored_data": 590368, 2024-09-12T23:37:45.649 INFO:tasks.workunit.client.0.smithi136.stdout: "stored_omap": 0, 2024-09-12T23:37:45.649 INFO:tasks.workunit.client.0.smithi136.stdout: "stored_raw": 1771104, 2024-09-12T23:37:45.649 INFO:tasks.workunit.client.0.smithi136.stdout: "wr": 113, 2024-09-12T23:37:45.649 INFO:tasks.workunit.client.0.smithi136.stdout: "wr_bytes": 1667072 2024-09-12T23:37:45.649 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.649 INFO:tasks.workunit.client.0.smithi136.stdout: "target_max_bytes": 0, 2024-09-12T23:37:45.650 INFO:tasks.workunit.client.0.smithi136.stdout: "target_max_objects": 0, 2024-09-12T23:37:45.650 INFO:tasks.workunit.client.0.smithi136.stdout: "type": "replicated" 2024-09-12T23:37:45.650 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.650 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:45.650 INFO:tasks.workunit.client.0.smithi136.stdout: "application": [ 2024-09-12T23:37:45.651 INFO:tasks.workunit.client.0.smithi136.stdout: "cephfs" 2024-09-12T23:37:45.651 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:45.651 INFO:tasks.workunit.client.0.smithi136.stdout: "cache_mode": "none", 2024-09-12T23:37:45.651 INFO:tasks.workunit.client.0.smithi136.stdout: "erasure_code_profile": {}, 2024-09-12T23:37:45.651 INFO:tasks.workunit.client.0.smithi136.stdout: "min_size": 2, 2024-09-12T23:37:45.651 INFO:tasks.workunit.client.0.smithi136.stdout: "options": {}, 2024-09-12T23:37:45.651 INFO:tasks.workunit.client.0.smithi136.stdout: "pg_autoscale_mode": "off", 2024-09-12T23:37:45.652 INFO:tasks.workunit.client.0.smithi136.stdout: "pg_num": 32, 2024-09-12T23:37:45.652 INFO:tasks.workunit.client.0.smithi136.stdout: "pgp_num": 32, 2024-09-12T23:37:45.652 INFO:tasks.workunit.client.0.smithi136.stdout: "pool": 2, 2024-09-12T23:37:45.652 INFO:tasks.workunit.client.0.smithi136.stdout: "size": 3, 2024-09-12T23:37:45.652 INFO:tasks.workunit.client.0.smithi136.stdout: "stats": { 2024-09-12T23:37:45.653 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_raw": 727116228874, 2024-09-12T23:37:45.653 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_used": 98304, 2024-09-12T23:37:45.653 INFO:tasks.workunit.client.0.smithi136.stdout: "compress_bytes_used": 0, 2024-09-12T23:37:45.653 INFO:tasks.workunit.client.0.smithi136.stdout: "compress_under_bytes": 0, 2024-09-12T23:37:45.653 INFO:tasks.workunit.client.0.smithi136.stdout: "data_bytes_used": 98304, 2024-09-12T23:37:45.653 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty": 0, 2024-09-12T23:37:45.654 INFO:tasks.workunit.client.0.smithi136.stdout: "max_avail": 242372067328, 2024-09-12T23:37:45.654 INFO:tasks.workunit.client.0.smithi136.stdout: "objects": 22, 2024-09-12T23:37:45.654 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_bytes_used": 0, 2024-09-12T23:37:45.654 INFO:tasks.workunit.client.0.smithi136.stdout: "percent_used": 1.3519708375042683e-07, 2024-09-12T23:37:45.654 INFO:tasks.workunit.client.0.smithi136.stdout: "quota_bytes": 0, 2024-09-12T23:37:45.655 INFO:tasks.workunit.client.0.smithi136.stdout: "quota_objects": 0, 2024-09-12T23:37:45.655 INFO:tasks.workunit.client.0.smithi136.stdout: "rd": 0, 2024-09-12T23:37:45.655 INFO:tasks.workunit.client.0.smithi136.stdout: "rd_bytes": 0, 2024-09-12T23:37:45.655 INFO:tasks.workunit.client.0.smithi136.stdout: "stored": 3410, 2024-09-12T23:37:45.655 INFO:tasks.workunit.client.0.smithi136.stdout: "stored_data": 3410, 2024-09-12T23:37:45.655 INFO:tasks.workunit.client.0.smithi136.stdout: "stored_omap": 0, 2024-09-12T23:37:45.656 INFO:tasks.workunit.client.0.smithi136.stdout: "stored_raw": 10230, 2024-09-12T23:37:45.656 INFO:tasks.workunit.client.0.smithi136.stdout: "wr": 51, 2024-09-12T23:37:45.656 INFO:tasks.workunit.client.0.smithi136.stdout: "wr_bytes": 19456 2024-09-12T23:37:45.656 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.656 INFO:tasks.workunit.client.0.smithi136.stdout: "target_max_bytes": 0, 2024-09-12T23:37:45.656 INFO:tasks.workunit.client.0.smithi136.stdout: "target_max_objects": 0, 2024-09-12T23:37:45.657 INFO:tasks.workunit.client.0.smithi136.stdout: "type": "replicated" 2024-09-12T23:37:45.657 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.657 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:45.657 INFO:tasks.workunit.client.0.smithi136.stdout: "application": [ 2024-09-12T23:37:45.657 INFO:tasks.workunit.client.0.smithi136.stdout: "cephfs" 2024-09-12T23:37:45.658 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:45.658 INFO:tasks.workunit.client.0.smithi136.stdout: "cache_mode": "none", 2024-09-12T23:37:45.658 INFO:tasks.workunit.client.0.smithi136.stdout: "erasure_code_profile": {}, 2024-09-12T23:37:45.658 INFO:tasks.workunit.client.0.smithi136.stdout: "min_size": 2, 2024-09-12T23:37:45.658 INFO:tasks.workunit.client.0.smithi136.stdout: "options": {}, 2024-09-12T23:37:45.658 INFO:tasks.workunit.client.0.smithi136.stdout: "pg_autoscale_mode": "off", 2024-09-12T23:37:45.659 INFO:tasks.workunit.client.0.smithi136.stdout: "pg_num": 64, 2024-09-12T23:37:45.659 INFO:tasks.workunit.client.0.smithi136.stdout: "pgp_num": 64, 2024-09-12T23:37:45.659 INFO:tasks.workunit.client.0.smithi136.stdout: "pool": 3, 2024-09-12T23:37:45.659 INFO:tasks.workunit.client.0.smithi136.stdout: "size": 3, 2024-09-12T23:37:45.659 INFO:tasks.workunit.client.0.smithi136.stdout: "stats": { 2024-09-12T23:37:45.660 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_raw": 727116228874, 2024-09-12T23:37:45.660 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_used": 0, 2024-09-12T23:37:45.660 INFO:tasks.workunit.client.0.smithi136.stdout: "compress_bytes_used": 0, 2024-09-12T23:37:45.660 INFO:tasks.workunit.client.0.smithi136.stdout: "compress_under_bytes": 0, 2024-09-12T23:37:45.660 INFO:tasks.workunit.client.0.smithi136.stdout: "data_bytes_used": 0, 2024-09-12T23:37:45.660 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty": 0, 2024-09-12T23:37:45.661 INFO:tasks.workunit.client.0.smithi136.stdout: "max_avail": 242372067328, 2024-09-12T23:37:45.661 INFO:tasks.workunit.client.0.smithi136.stdout: "objects": 0, 2024-09-12T23:37:45.661 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_bytes_used": 0, 2024-09-12T23:37:45.661 INFO:tasks.workunit.client.0.smithi136.stdout: "percent_used": 0.0, 2024-09-12T23:37:45.661 INFO:tasks.workunit.client.0.smithi136.stdout: "quota_bytes": 0, 2024-09-12T23:37:45.662 INFO:tasks.workunit.client.0.smithi136.stdout: "quota_objects": 0, 2024-09-12T23:37:45.662 INFO:tasks.workunit.client.0.smithi136.stdout: "rd": 0, 2024-09-12T23:37:45.662 INFO:tasks.workunit.client.0.smithi136.stdout: "rd_bytes": 0, 2024-09-12T23:37:45.662 INFO:tasks.workunit.client.0.smithi136.stdout: "stored": 0, 2024-09-12T23:37:45.662 INFO:tasks.workunit.client.0.smithi136.stdout: "stored_data": 0, 2024-09-12T23:37:45.662 INFO:tasks.workunit.client.0.smithi136.stdout: "stored_omap": 0, 2024-09-12T23:37:45.663 INFO:tasks.workunit.client.0.smithi136.stdout: "stored_raw": 0, 2024-09-12T23:37:45.663 INFO:tasks.workunit.client.0.smithi136.stdout: "wr": 0, 2024-09-12T23:37:45.663 INFO:tasks.workunit.client.0.smithi136.stdout: "wr_bytes": 0 2024-09-12T23:37:45.663 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.663 INFO:tasks.workunit.client.0.smithi136.stdout: "target_max_bytes": 0, 2024-09-12T23:37:45.664 INFO:tasks.workunit.client.0.smithi136.stdout: "target_max_objects": 0, 2024-09-12T23:37:45.664 INFO:tasks.workunit.client.0.smithi136.stdout: "type": "replicated" 2024-09-12T23:37:45.664 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.664 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:45.664 INFO:tasks.workunit.client.0.smithi136.stdout: "application": [ 2024-09-12T23:37:45.665 INFO:tasks.workunit.client.0.smithi136.stdout: "rbd" 2024-09-12T23:37:45.665 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:45.665 INFO:tasks.workunit.client.0.smithi136.stdout: "cache_mode": "none", 2024-09-12T23:37:45.665 INFO:tasks.workunit.client.0.smithi136.stdout: "erasure_code_profile": {}, 2024-09-12T23:37:45.665 INFO:tasks.workunit.client.0.smithi136.stdout: "min_size": 2, 2024-09-12T23:37:45.666 INFO:tasks.workunit.client.0.smithi136.stdout: "options": {}, 2024-09-12T23:37:45.666 INFO:tasks.workunit.client.0.smithi136.stdout: "pg_autoscale_mode": "off", 2024-09-12T23:37:45.666 INFO:tasks.workunit.client.0.smithi136.stdout: "pg_num": 8, 2024-09-12T23:37:45.666 INFO:tasks.workunit.client.0.smithi136.stdout: "pgp_num": 8, 2024-09-12T23:37:45.666 INFO:tasks.workunit.client.0.smithi136.stdout: "pool": 4, 2024-09-12T23:37:45.666 INFO:tasks.workunit.client.0.smithi136.stdout: "size": 3, 2024-09-12T23:37:45.667 INFO:tasks.workunit.client.0.smithi136.stdout: "stats": { 2024-09-12T23:37:45.667 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_raw": 727116228874, 2024-09-12T23:37:45.667 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_used": 12288, 2024-09-12T23:37:45.667 INFO:tasks.workunit.client.0.smithi136.stdout: "compress_bytes_used": 0, 2024-09-12T23:37:45.667 INFO:tasks.workunit.client.0.smithi136.stdout: "compress_under_bytes": 0, 2024-09-12T23:37:45.668 INFO:tasks.workunit.client.0.smithi136.stdout: "data_bytes_used": 12288, 2024-09-12T23:37:45.668 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty": 0, 2024-09-12T23:37:45.668 INFO:tasks.workunit.client.0.smithi136.stdout: "max_avail": 242372067328, 2024-09-12T23:37:45.668 INFO:tasks.workunit.client.0.smithi136.stdout: "objects": 2, 2024-09-12T23:37:45.668 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_bytes_used": 0, 2024-09-12T23:37:45.669 INFO:tasks.workunit.client.0.smithi136.stdout: "percent_used": 1.6899637245160193e-08, 2024-09-12T23:37:45.669 INFO:tasks.workunit.client.0.smithi136.stdout: "quota_bytes": 0, 2024-09-12T23:37:45.669 INFO:tasks.workunit.client.0.smithi136.stdout: "quota_objects": 0, 2024-09-12T23:37:45.669 INFO:tasks.workunit.client.0.smithi136.stdout: "rd": 0, 2024-09-12T23:37:45.669 INFO:tasks.workunit.client.0.smithi136.stdout: "rd_bytes": 0, 2024-09-12T23:37:45.669 INFO:tasks.workunit.client.0.smithi136.stdout: "stored": 19, 2024-09-12T23:37:45.670 INFO:tasks.workunit.client.0.smithi136.stdout: "stored_data": 19, 2024-09-12T23:37:45.670 INFO:tasks.workunit.client.0.smithi136.stdout: "stored_omap": 0, 2024-09-12T23:37:45.670 INFO:tasks.workunit.client.0.smithi136.stdout: "stored_raw": 57, 2024-09-12T23:37:45.670 INFO:tasks.workunit.client.0.smithi136.stdout: "wr": 2, 2024-09-12T23:37:45.670 INFO:tasks.workunit.client.0.smithi136.stdout: "wr_bytes": 2048 2024-09-12T23:37:45.671 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.671 INFO:tasks.workunit.client.0.smithi136.stdout: "target_max_bytes": 0, 2024-09-12T23:37:45.671 INFO:tasks.workunit.client.0.smithi136.stdout: "target_max_objects": 0, 2024-09-12T23:37:45.671 INFO:tasks.workunit.client.0.smithi136.stdout: "type": "replicated" 2024-09-12T23:37:45.671 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:45.671 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:45.672 INFO:tasks.workunit.client.0.smithi136.stdout: "rbd": { 2024-09-12T23:37:45.672 INFO:tasks.workunit.client.0.smithi136.stdout: "mirroring_by_pool": [ 2024-09-12T23:37:45.672 INFO:tasks.workunit.client.0.smithi136.stdout: false 2024-09-12T23:37:45.672 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:45.672 INFO:tasks.workunit.client.0.smithi136.stdout: "num_images_by_pool": [ 2024-09-12T23:37:45.673 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:45.673 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:45.673 INFO:tasks.workunit.client.0.smithi136.stdout: "num_pools": 1 2024-09-12T23:37:45.673 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.673 INFO:tasks.workunit.client.0.smithi136.stdout: "report_id": "679f567b-70cd-4d20-9c08-b6621403ecae", 2024-09-12T23:37:45.674 INFO:tasks.workunit.client.0.smithi136.stdout: "report_timestamp": "2024-09-12T23:37:45.518085", 2024-09-12T23:37:45.674 INFO:tasks.workunit.client.0.smithi136.stdout: "report_version": 1, 2024-09-12T23:37:45.674 INFO:tasks.workunit.client.0.smithi136.stdout: "rook": { 2024-09-12T23:37:45.674 INFO:tasks.workunit.client.0.smithi136.stdout: "cluster": { 2024-09-12T23:37:45.674 INFO:tasks.workunit.client.0.smithi136.stdout: "external-mode": null, 2024-09-12T23:37:45.674 INFO:tasks.workunit.client.0.smithi136.stdout: "mon": { 2024-09-12T23:37:45.675 INFO:tasks.workunit.client.0.smithi136.stdout: "allow-multiple-per-node": null, 2024-09-12T23:37:45.675 INFO:tasks.workunit.client.0.smithi136.stdout: "count": null, 2024-09-12T23:37:45.675 INFO:tasks.workunit.client.0.smithi136.stdout: "max-id": null, 2024-09-12T23:37:45.675 INFO:tasks.workunit.client.0.smithi136.stdout: "pvc": { 2024-09-12T23:37:45.675 INFO:tasks.workunit.client.0.smithi136.stdout: "enabled": null 2024-09-12T23:37:45.676 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.676 INFO:tasks.workunit.client.0.smithi136.stdout: "stretch": { 2024-09-12T23:37:45.676 INFO:tasks.workunit.client.0.smithi136.stdout: "enabled": null 2024-09-12T23:37:45.676 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:45.676 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.676 INFO:tasks.workunit.client.0.smithi136.stdout: "network": { 2024-09-12T23:37:45.677 INFO:tasks.workunit.client.0.smithi136.stdout: "provider": null 2024-09-12T23:37:45.677 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.677 INFO:tasks.workunit.client.0.smithi136.stdout: "storage": { 2024-09-12T23:37:45.677 INFO:tasks.workunit.client.0.smithi136.stdout: "device-set": { 2024-09-12T23:37:45.677 INFO:tasks.workunit.client.0.smithi136.stdout: "count": { 2024-09-12T23:37:45.678 INFO:tasks.workunit.client.0.smithi136.stdout: "non-portable": null, 2024-09-12T23:37:45.678 INFO:tasks.workunit.client.0.smithi136.stdout: "portable": null, 2024-09-12T23:37:45.678 INFO:tasks.workunit.client.0.smithi136.stdout: "total": null 2024-09-12T23:37:45.678 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:45.678 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:45.679 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:45.679 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.679 INFO:tasks.workunit.client.0.smithi136.stdout: "csi": { 2024-09-12T23:37:45.679 INFO:tasks.workunit.client.0.smithi136.stdout: "version": null 2024-09-12T23:37:45.679 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.679 INFO:tasks.workunit.client.0.smithi136.stdout: "kubernetes": { 2024-09-12T23:37:45.680 INFO:tasks.workunit.client.0.smithi136.stdout: "version": null 2024-09-12T23:37:45.680 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.680 INFO:tasks.workunit.client.0.smithi136.stdout: "node": { 2024-09-12T23:37:45.680 INFO:tasks.workunit.client.0.smithi136.stdout: "count": { 2024-09-12T23:37:45.680 INFO:tasks.workunit.client.0.smithi136.stdout: "kubernetes-total": null, 2024-09-12T23:37:45.681 INFO:tasks.workunit.client.0.smithi136.stdout: "with-ceph-daemons": null, 2024-09-12T23:37:45.681 INFO:tasks.workunit.client.0.smithi136.stdout: "with-csi-cephfs-plugin": null, 2024-09-12T23:37:45.681 INFO:tasks.workunit.client.0.smithi136.stdout: "with-csi-nfs-plugin": null, 2024-09-12T23:37:45.681 INFO:tasks.workunit.client.0.smithi136.stdout: "with-csi-rbd-plugin": null 2024-09-12T23:37:45.681 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:45.681 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.682 INFO:tasks.workunit.client.0.smithi136.stdout: "usage": { 2024-09-12T23:37:45.682 INFO:tasks.workunit.client.0.smithi136.stdout: "storage-class": { 2024-09-12T23:37:45.682 INFO:tasks.workunit.client.0.smithi136.stdout: "count": { 2024-09-12T23:37:45.682 INFO:tasks.workunit.client.0.smithi136.stdout: "bucket": null, 2024-09-12T23:37:45.682 INFO:tasks.workunit.client.0.smithi136.stdout: "cephfs": null, 2024-09-12T23:37:45.682 INFO:tasks.workunit.client.0.smithi136.stdout: "nfs": null, 2024-09-12T23:37:45.682 INFO:tasks.workunit.client.0.smithi136.stdout: "rbd": null, 2024-09-12T23:37:45.682 INFO:tasks.workunit.client.0.smithi136.stdout: "total": null 2024-09-12T23:37:45.682 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:45.683 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:45.683 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.683 INFO:tasks.workunit.client.0.smithi136.stdout: "version": null 2024-09-12T23:37:45.683 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.683 INFO:tasks.workunit.client.0.smithi136.stdout: "services": {}, 2024-09-12T23:37:45.683 INFO:tasks.workunit.client.0.smithi136.stdout: "usage": { 2024-09-12T23:37:45.683 INFO:tasks.workunit.client.0.smithi136.stdout: "pg_num": 105, 2024-09-12T23:37:45.683 INFO:tasks.workunit.client.0.smithi136.stdout: "pools": 4, 2024-09-12T23:37:45.683 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_by_class": { 2024-09-12T23:37:45.683 INFO:tasks.workunit.client.0.smithi136.stdout: "ssd": { 2024-09-12T23:37:45.683 INFO:tasks.workunit.client.0.smithi136.stdout: "total_avail_bytes": 765517262848, 2024-09-12T23:37:45.683 INFO:tasks.workunit.client.0.smithi136.stdout: "total_bytes": 767960285184, 2024-09-12T23:37:45.683 INFO:tasks.workunit.client.0.smithi136.stdout: "total_used_bytes": 2443022336, 2024-09-12T23:37:45.683 INFO:tasks.workunit.client.0.smithi136.stdout: "total_used_raw_bytes": 2443022336, 2024-09-12T23:37:45.684 INFO:tasks.workunit.client.0.smithi136.stdout: "total_used_raw_ratio": 0.0031811832450330257 2024-09-12T23:37:45.684 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:45.684 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:45.684 INFO:tasks.workunit.client.0.smithi136.stdout: "total_avail_bytes": 765517262848, 2024-09-12T23:37:45.684 INFO:tasks.workunit.client.0.smithi136.stdout: "total_bytes": 767960285184, 2024-09-12T23:37:45.684 INFO:tasks.workunit.client.0.smithi136.stdout: "total_used_bytes": 2443022336 2024-09-12T23:37:45.684 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:45.684 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:45.684 INFO:tasks.workunit.client.0.smithi136.stdout:} 2024-09-12T23:37:45.685 INFO:tasks.workunit.client.0.smithi136.stderr:++ ceph telemetry collection ls 2024-09-12T23:37:45.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:45 smithi136 bash[22601]: cluster 2024-09-12T23:37:43.997324+0000 mgr.y (mgr.24446) 144 : cluster [DBG] pgmap v120: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:45.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:45 smithi136 bash[22601]: audit 2024-09-12T23:37:44.269398+0000 mgr.y (mgr.24446) 145 : audit [DBG] from='client.14850 -' entity='client.admin' cmd=[{"prefix": "telemetry preview", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:45.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:45 smithi136 bash[22601]: audit 2024-09-12T23:37:44.422618+0000 mgr.y (mgr.24446) 146 : audit [DBG] from='client.14850 -' entity='client.admin' cmd=[{"prefix": "telemetry preview", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:45.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:45 smithi136 bash[22601]: audit 2024-09-12T23:37:44.777408+0000 mon.a (mon.0) 924 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:37:45.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:45 smithi136 bash[22601]: audit 2024-09-12T23:37:44.783962+0000 mon.a (mon.0) 925 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:37:45.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:45 smithi136 bash[22601]: audit 2024-09-12T23:37:44.802953+0000 mon.a (mon.0) 926 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:37:45.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:45 smithi136 bash[22601]: audit 2024-09-12T23:37:44.809335+0000 mon.a (mon.0) 927 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:37:45.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:45 smithi136 bash[22601]: audit 2024-09-12T23:37:45.329042+0000 mon.a (mon.0) 928 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:37:45.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:45 smithi136 bash[22601]: audit 2024-09-12T23:37:45.335330+0000 mon.a (mon.0) 929 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-12T23:37:45.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:45 smithi136 bash[32189]: cluster 2024-09-12T23:37:43.997324+0000 mgr.y (mgr.24446) 144 : cluster [DBG] pgmap v120: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:45.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:45 smithi136 bash[32189]: audit 2024-09-12T23:37:44.269398+0000 mgr.y (mgr.24446) 145 : audit [DBG] from='client.14850 -' entity='client.admin' cmd=[{"prefix": "telemetry preview", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:45.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:45 smithi136 bash[32189]: audit 2024-09-12T23:37:44.422618+0000 mgr.y (mgr.24446) 146 : audit [DBG] from='client.14850 -' entity='client.admin' cmd=[{"prefix": "telemetry preview", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:45.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:45 smithi136 bash[32189]: audit 2024-09-12T23:37:44.777408+0000 mon.a (mon.0) 924 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:37:45.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:45 smithi136 bash[32189]: audit 2024-09-12T23:37:44.783962+0000 mon.a (mon.0) 925 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:37:45.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:45 smithi136 bash[32189]: audit 2024-09-12T23:37:44.802953+0000 mon.a (mon.0) 926 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:37:45.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:45 smithi136 bash[32189]: audit 2024-09-12T23:37:44.809335+0000 mon.a (mon.0) 927 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:37:45.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:45 smithi136 bash[32189]: audit 2024-09-12T23:37:45.329042+0000 mon.a (mon.0) 928 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:37:45.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:45 smithi136 bash[32189]: audit 2024-09-12T23:37:45.335330+0000 mon.a (mon.0) 929 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-12T23:37:45.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:45 smithi181 bash[24776]: cluster 2024-09-12T23:37:43.997324+0000 mgr.y (mgr.24446) 144 : cluster [DBG] pgmap v120: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:45.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:45 smithi181 bash[24776]: audit 2024-09-12T23:37:44.269398+0000 mgr.y (mgr.24446) 145 : audit [DBG] from='client.14850 -' entity='client.admin' cmd=[{"prefix": "telemetry preview", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:45.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:45 smithi181 bash[24776]: audit 2024-09-12T23:37:44.422618+0000 mgr.y (mgr.24446) 146 : audit [DBG] from='client.14850 -' entity='client.admin' cmd=[{"prefix": "telemetry preview", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:45.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:45 smithi181 bash[24776]: audit 2024-09-12T23:37:44.777408+0000 mon.a (mon.0) 924 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:37:45.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:45 smithi181 bash[24776]: audit 2024-09-12T23:37:44.783962+0000 mon.a (mon.0) 925 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:37:45.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:45 smithi181 bash[24776]: audit 2024-09-12T23:37:44.802953+0000 mon.a (mon.0) 926 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:37:45.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:45 smithi181 bash[24776]: audit 2024-09-12T23:37:44.809335+0000 mon.a (mon.0) 927 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:37:45.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:45 smithi181 bash[24776]: audit 2024-09-12T23:37:45.329042+0000 mon.a (mon.0) 928 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:37:45.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:45 smithi181 bash[24776]: audit 2024-09-12T23:37:45.335330+0000 mon.a (mon.0) 929 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-12T23:37:46.055 INFO:tasks.workunit.client.0.smithi136.stderr:+ COLLECTIONS='New collections are available: 2024-09-12T23:37:46.055 INFO:tasks.workunit.client.0.smithi136.stderr:['\''basic_base'\'', '\''basic_mds_metadata'\'', '\''basic_pool_options_bluestore'\'', '\''basic_pool_usage'\'', '\''basic_rook_v01'\'', '\''basic_usage_by_class'\'', '\''crash_base'\'', '\''device_base'\'', '\''ident_base'\'', '\''perf_memory_metrics'\'', '\''perf_perf'\''] 2024-09-12T23:37:46.055 INFO:tasks.workunit.client.0.smithi136.stderr:Run `ceph telemetry on` to opt-in to these collections. 2024-09-12T23:37:46.055 INFO:tasks.workunit.client.0.smithi136.stderr: 2024-09-12T23:37:46.056 INFO:tasks.workunit.client.0.smithi136.stderr:NAME STATUS DESC 2024-09-12T23:37:46.056 INFO:tasks.workunit.client.0.smithi136.stderr:basic_base NOT REPORTING: NOT OPTED-IN Basic information about the cluster (capacity, number and type of daemons, version, etc.) 2024-09-12T23:37:46.056 INFO:tasks.workunit.client.0.smithi136.stderr:basic_mds_metadata NOT REPORTING: NOT OPTED-IN MDS metadata 2024-09-12T23:37:46.056 INFO:tasks.workunit.client.0.smithi136.stderr:basic_pool_options_bluestore NOT REPORTING: NOT OPTED-IN Per-pool bluestore config options 2024-09-12T23:37:46.056 INFO:tasks.workunit.client.0.smithi136.stderr:basic_pool_usage NOT REPORTING: NOT OPTED-IN Default pool application and usage statistics 2024-09-12T23:37:46.056 INFO:tasks.workunit.client.0.smithi136.stderr:basic_rook_v01 NOT REPORTING: NOT OPTED-IN Basic Rook deployment data 2024-09-12T23:37:46.056 INFO:tasks.workunit.client.0.smithi136.stderr:basic_usage_by_class NOT REPORTING: NOT OPTED-IN Default device class usage statistics 2024-09-12T23:37:46.056 INFO:tasks.workunit.client.0.smithi136.stderr:crash_base NOT REPORTING: NOT OPTED-IN Information about daemon crashes (daemon type and version, backtrace, etc.) 2024-09-12T23:37:46.056 INFO:tasks.workunit.client.0.smithi136.stderr:device_base NOT REPORTING: NOT OPTED-IN Information about device health metrics 2024-09-12T23:37:46.056 INFO:tasks.workunit.client.0.smithi136.stderr:ident_base NOT REPORTING: NOT OPTED-IN User-provided identifying information about the cluster 2024-09-12T23:37:46.056 INFO:tasks.workunit.client.0.smithi136.stderr:perf_memory_metrics NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Heap stats and mempools for mon and mds 2024-09-12T23:37:46.056 INFO:tasks.workunit.client.0.smithi136.stderr:perf_perf NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Information about performance counters of the cluster ' 2024-09-12T23:37:46.057 INFO:tasks.workunit.client.0.smithi136.stderr:+ NEW_COLLECTIONS=("perf_perf" "basic_mds_metadata" "basic_pool_usage" "basic_rook_v01" "perf_memory_metrics") 2024-09-12T23:37:46.057 INFO:tasks.workunit.client.0.smithi136.stderr:+ for col in ${NEW_COLLECTIONS[@]} 2024-09-12T23:37:46.057 INFO:tasks.workunit.client.0.smithi136.stderr:+ [[ New collections are available: 2024-09-12T23:37:46.057 INFO:tasks.workunit.client.0.smithi136.stderr:['basic_base', 'basic_mds_metadata', 'basic_pool_options_bluestore', 'basic_pool_usage', 'basic_rook_v01', 'basic_usage_by_class', 'crash_base', 'device_base', 'ident_base', 'perf_memory_metrics', 'perf_perf'] 2024-09-12T23:37:46.057 INFO:tasks.workunit.client.0.smithi136.stderr:Run `ceph telemetry on` to opt-in to these collections. 2024-09-12T23:37:46.057 INFO:tasks.workunit.client.0.smithi136.stderr: 2024-09-12T23:37:46.057 INFO:tasks.workunit.client.0.smithi136.stderr:NAME STATUS DESC 2024-09-12T23:37:46.057 INFO:tasks.workunit.client.0.smithi136.stderr:basic_base NOT REPORTING: NOT OPTED-IN Basic information about the cluster (capacity, number and type of daemons, version, etc.) 2024-09-12T23:37:46.057 INFO:tasks.workunit.client.0.smithi136.stderr:basic_mds_metadata NOT REPORTING: NOT OPTED-IN MDS metadata 2024-09-12T23:37:46.057 INFO:tasks.workunit.client.0.smithi136.stderr:basic_pool_options_bluestore NOT REPORTING: NOT OPTED-IN Per-pool bluestore config options 2024-09-12T23:37:46.057 INFO:tasks.workunit.client.0.smithi136.stderr:basic_pool_usage NOT REPORTING: NOT OPTED-IN Default pool application and usage statistics 2024-09-12T23:37:46.057 INFO:tasks.workunit.client.0.smithi136.stderr:basic_rook_v01 NOT REPORTING: NOT OPTED-IN Basic Rook deployment data 2024-09-12T23:37:46.057 INFO:tasks.workunit.client.0.smithi136.stderr:basic_usage_by_class NOT REPORTING: NOT OPTED-IN Default device class usage statistics 2024-09-12T23:37:46.058 INFO:tasks.workunit.client.0.smithi136.stderr:crash_base NOT REPORTING: NOT OPTED-IN Information about daemon crashes (daemon type and version, backtrace, etc.) 2024-09-12T23:37:46.058 INFO:tasks.workunit.client.0.smithi136.stderr:device_base NOT REPORTING: NOT OPTED-IN Information about device health metrics 2024-09-12T23:37:46.058 INFO:tasks.workunit.client.0.smithi136.stderr:ident_base NOT REPORTING: NOT OPTED-IN User-provided identifying information about the cluster 2024-09-12T23:37:46.058 INFO:tasks.workunit.client.0.smithi136.stderr:perf_memory_metrics NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Heap stats and mempools for mon and mds 2024-09-12T23:37:46.058 INFO:tasks.workunit.client.0.smithi136.stderr:perf_perf NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Information about performance counters of the cluster == *perf_perf* ]] 2024-09-12T23:37:46.058 INFO:tasks.workunit.client.0.smithi136.stderr:+ for col in ${NEW_COLLECTIONS[@]} 2024-09-12T23:37:46.058 INFO:tasks.workunit.client.0.smithi136.stderr:+ [[ New collections are available: 2024-09-12T23:37:46.058 INFO:tasks.workunit.client.0.smithi136.stderr:['basic_base', 'basic_mds_metadata', 'basic_pool_options_bluestore', 'basic_pool_usage', 'basic_rook_v01', 'basic_usage_by_class', 'crash_base', 'device_base', 'ident_base', 'perf_memory_metrics', 'perf_perf'] 2024-09-12T23:37:46.058 INFO:tasks.workunit.client.0.smithi136.stderr:Run `ceph telemetry on` to opt-in to these collections. 2024-09-12T23:37:46.058 INFO:tasks.workunit.client.0.smithi136.stderr: 2024-09-12T23:37:46.058 INFO:tasks.workunit.client.0.smithi136.stderr:NAME STATUS DESC 2024-09-12T23:37:46.058 INFO:tasks.workunit.client.0.smithi136.stderr:basic_base NOT REPORTING: NOT OPTED-IN Basic information about the cluster (capacity, number and type of daemons, version, etc.) 2024-09-12T23:37:46.058 INFO:tasks.workunit.client.0.smithi136.stderr:basic_mds_metadata NOT REPORTING: NOT OPTED-IN MDS metadata 2024-09-12T23:37:46.058 INFO:tasks.workunit.client.0.smithi136.stderr:basic_pool_options_bluestore NOT REPORTING: NOT OPTED-IN Per-pool bluestore config options 2024-09-12T23:37:46.059 INFO:tasks.workunit.client.0.smithi136.stderr:basic_pool_usage NOT REPORTING: NOT OPTED-IN Default pool application and usage statistics 2024-09-12T23:37:46.059 INFO:tasks.workunit.client.0.smithi136.stderr:basic_rook_v01 NOT REPORTING: NOT OPTED-IN Basic Rook deployment data 2024-09-12T23:37:46.059 INFO:tasks.workunit.client.0.smithi136.stderr:basic_usage_by_class NOT REPORTING: NOT OPTED-IN Default device class usage statistics 2024-09-12T23:37:46.059 INFO:tasks.workunit.client.0.smithi136.stderr:crash_base NOT REPORTING: NOT OPTED-IN Information about daemon crashes (daemon type and version, backtrace, etc.) 2024-09-12T23:37:46.059 INFO:tasks.workunit.client.0.smithi136.stderr:device_base NOT REPORTING: NOT OPTED-IN Information about device health metrics 2024-09-12T23:37:46.059 INFO:tasks.workunit.client.0.smithi136.stderr:ident_base NOT REPORTING: NOT OPTED-IN User-provided identifying information about the cluster 2024-09-12T23:37:46.059 INFO:tasks.workunit.client.0.smithi136.stderr:perf_memory_metrics NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Heap stats and mempools for mon and mds 2024-09-12T23:37:46.059 INFO:tasks.workunit.client.0.smithi136.stderr:perf_perf NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Information about performance counters of the cluster == *basic_mds_metadata* ]] 2024-09-12T23:37:46.059 INFO:tasks.workunit.client.0.smithi136.stderr:+ for col in ${NEW_COLLECTIONS[@]} 2024-09-12T23:37:46.059 INFO:tasks.workunit.client.0.smithi136.stderr:+ [[ New collections are available: 2024-09-12T23:37:46.059 INFO:tasks.workunit.client.0.smithi136.stderr:['basic_base', 'basic_mds_metadata', 'basic_pool_options_bluestore', 'basic_pool_usage', 'basic_rook_v01', 'basic_usage_by_class', 'crash_base', 'device_base', 'ident_base', 'perf_memory_metrics', 'perf_perf'] 2024-09-12T23:37:46.059 INFO:tasks.workunit.client.0.smithi136.stderr:Run `ceph telemetry on` to opt-in to these collections. 2024-09-12T23:37:46.059 INFO:tasks.workunit.client.0.smithi136.stderr: 2024-09-12T23:37:46.059 INFO:tasks.workunit.client.0.smithi136.stderr:NAME STATUS DESC 2024-09-12T23:37:46.060 INFO:tasks.workunit.client.0.smithi136.stderr:basic_base NOT REPORTING: NOT OPTED-IN Basic information about the cluster (capacity, number and type of daemons, version, etc.) 2024-09-12T23:37:46.060 INFO:tasks.workunit.client.0.smithi136.stderr:basic_mds_metadata NOT REPORTING: NOT OPTED-IN MDS metadata 2024-09-12T23:37:46.060 INFO:tasks.workunit.client.0.smithi136.stderr:basic_pool_options_bluestore NOT REPORTING: NOT OPTED-IN Per-pool bluestore config options 2024-09-12T23:37:46.060 INFO:tasks.workunit.client.0.smithi136.stderr:basic_pool_usage NOT REPORTING: NOT OPTED-IN Default pool application and usage statistics 2024-09-12T23:37:46.060 INFO:tasks.workunit.client.0.smithi136.stderr:basic_rook_v01 NOT REPORTING: NOT OPTED-IN Basic Rook deployment data 2024-09-12T23:37:46.060 INFO:tasks.workunit.client.0.smithi136.stderr:basic_usage_by_class NOT REPORTING: NOT OPTED-IN Default device class usage statistics 2024-09-12T23:37:46.060 INFO:tasks.workunit.client.0.smithi136.stderr:crash_base NOT REPORTING: NOT OPTED-IN Information about daemon crashes (daemon type and version, backtrace, etc.) 2024-09-12T23:37:46.060 INFO:tasks.workunit.client.0.smithi136.stderr:device_base NOT REPORTING: NOT OPTED-IN Information about device health metrics 2024-09-12T23:37:46.060 INFO:tasks.workunit.client.0.smithi136.stderr:ident_base NOT REPORTING: NOT OPTED-IN User-provided identifying information about the cluster 2024-09-12T23:37:46.060 INFO:tasks.workunit.client.0.smithi136.stderr:perf_memory_metrics NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Heap stats and mempools for mon and mds 2024-09-12T23:37:46.060 INFO:tasks.workunit.client.0.smithi136.stderr:perf_perf NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Information about performance counters of the cluster == *basic_pool_usage* ]] 2024-09-12T23:37:46.060 INFO:tasks.workunit.client.0.smithi136.stderr:+ for col in ${NEW_COLLECTIONS[@]} 2024-09-12T23:37:46.060 INFO:tasks.workunit.client.0.smithi136.stderr:+ [[ New collections are available: 2024-09-12T23:37:46.061 INFO:tasks.workunit.client.0.smithi136.stderr:['basic_base', 'basic_mds_metadata', 'basic_pool_options_bluestore', 'basic_pool_usage', 'basic_rook_v01', 'basic_usage_by_class', 'crash_base', 'device_base', 'ident_base', 'perf_memory_metrics', 'perf_perf'] 2024-09-12T23:37:46.061 INFO:tasks.workunit.client.0.smithi136.stderr:Run `ceph telemetry on` to opt-in to these collections. 2024-09-12T23:37:46.061 INFO:tasks.workunit.client.0.smithi136.stderr: 2024-09-12T23:37:46.061 INFO:tasks.workunit.client.0.smithi136.stderr:NAME STATUS DESC 2024-09-12T23:37:46.061 INFO:tasks.workunit.client.0.smithi136.stderr:basic_base NOT REPORTING: NOT OPTED-IN Basic information about the cluster (capacity, number and type of daemons, version, etc.) 2024-09-12T23:37:46.061 INFO:tasks.workunit.client.0.smithi136.stderr:basic_mds_metadata NOT REPORTING: NOT OPTED-IN MDS metadata 2024-09-12T23:37:46.061 INFO:tasks.workunit.client.0.smithi136.stderr:basic_pool_options_bluestore NOT REPORTING: NOT OPTED-IN Per-pool bluestore config options 2024-09-12T23:37:46.061 INFO:tasks.workunit.client.0.smithi136.stderr:basic_pool_usage NOT REPORTING: NOT OPTED-IN Default pool application and usage statistics 2024-09-12T23:37:46.061 INFO:tasks.workunit.client.0.smithi136.stderr:basic_rook_v01 NOT REPORTING: NOT OPTED-IN Basic Rook deployment data 2024-09-12T23:37:46.061 INFO:tasks.workunit.client.0.smithi136.stderr:basic_usage_by_class NOT REPORTING: NOT OPTED-IN Default device class usage statistics 2024-09-12T23:37:46.061 INFO:tasks.workunit.client.0.smithi136.stderr:crash_base NOT REPORTING: NOT OPTED-IN Information about daemon crashes (daemon type and version, backtrace, etc.) 2024-09-12T23:37:46.061 INFO:tasks.workunit.client.0.smithi136.stderr:device_base NOT REPORTING: NOT OPTED-IN Information about device health metrics 2024-09-12T23:37:46.062 INFO:tasks.workunit.client.0.smithi136.stderr:ident_base NOT REPORTING: NOT OPTED-IN User-provided identifying information about the cluster 2024-09-12T23:37:46.062 INFO:tasks.workunit.client.0.smithi136.stderr:perf_memory_metrics NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Heap stats and mempools for mon and mds 2024-09-12T23:37:46.062 INFO:tasks.workunit.client.0.smithi136.stderr:perf_perf NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Information about performance counters of the cluster == *basic_rook_v01* ]] 2024-09-12T23:37:46.062 INFO:tasks.workunit.client.0.smithi136.stderr:+ for col in ${NEW_COLLECTIONS[@]} 2024-09-12T23:37:46.062 INFO:tasks.workunit.client.0.smithi136.stderr:+ [[ New collections are available: 2024-09-12T23:37:46.062 INFO:tasks.workunit.client.0.smithi136.stderr:['basic_base', 'basic_mds_metadata', 'basic_pool_options_bluestore', 'basic_pool_usage', 'basic_rook_v01', 'basic_usage_by_class', 'crash_base', 'device_base', 'ident_base', 'perf_memory_metrics', 'perf_perf'] 2024-09-12T23:37:46.062 INFO:tasks.workunit.client.0.smithi136.stderr:Run `ceph telemetry on` to opt-in to these collections. 2024-09-12T23:37:46.062 INFO:tasks.workunit.client.0.smithi136.stderr: 2024-09-12T23:37:46.062 INFO:tasks.workunit.client.0.smithi136.stderr:NAME STATUS DESC 2024-09-12T23:37:46.062 INFO:tasks.workunit.client.0.smithi136.stderr:basic_base NOT REPORTING: NOT OPTED-IN Basic information about the cluster (capacity, number and type of daemons, version, etc.) 2024-09-12T23:37:46.063 INFO:tasks.workunit.client.0.smithi136.stderr:basic_mds_metadata NOT REPORTING: NOT OPTED-IN MDS metadata 2024-09-12T23:37:46.063 INFO:tasks.workunit.client.0.smithi136.stderr:basic_pool_options_bluestore NOT REPORTING: NOT OPTED-IN Per-pool bluestore config options 2024-09-12T23:37:46.063 INFO:tasks.workunit.client.0.smithi136.stderr:basic_pool_usage NOT REPORTING: NOT OPTED-IN Default pool application and usage statistics 2024-09-12T23:37:46.063 INFO:tasks.workunit.client.0.smithi136.stderr:basic_rook_v01 NOT REPORTING: NOT OPTED-IN Basic Rook deployment data 2024-09-12T23:37:46.063 INFO:tasks.workunit.client.0.smithi136.stderr:basic_usage_by_class NOT REPORTING: NOT OPTED-IN Default device class usage statistics 2024-09-12T23:37:46.063 INFO:tasks.workunit.client.0.smithi136.stderr:crash_base NOT REPORTING: NOT OPTED-IN Information about daemon crashes (daemon type and version, backtrace, etc.) 2024-09-12T23:37:46.063 INFO:tasks.workunit.client.0.smithi136.stderr:device_base NOT REPORTING: NOT OPTED-IN Information about device health metrics 2024-09-12T23:37:46.063 INFO:tasks.workunit.client.0.smithi136.stderr:ident_base NOT REPORTING: NOT OPTED-IN User-provided identifying information about the cluster 2024-09-12T23:37:46.063 INFO:tasks.workunit.client.0.smithi136.stderr:perf_memory_metrics NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Heap stats and mempools for mon and mds 2024-09-12T23:37:46.063 INFO:tasks.workunit.client.0.smithi136.stderr:perf_perf NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Information about performance counters of the cluster == *perf_memory_metrics* ]] 2024-09-12T23:37:46.063 INFO:tasks.workunit.client.0.smithi136.stderr:+ ceph telemetry on --license sharing-1-0 2024-09-12T23:37:46.404 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:37:46 smithi136 bash[22852]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:37:46] "GET /metrics HTTP/1.1" 200 236586 "" "Prometheus/2.43.0" 2024-09-12T23:37:46.641 INFO:tasks.workunit.client.0.smithi136.stdout:Telemetry is on. 2024-09-12T23:37:46.641 INFO:tasks.workunit.client.0.smithi136.stdout:Some channels are disabled, please enable with: 2024-09-12T23:37:46.641 INFO:tasks.workunit.client.0.smithi136.stdout:`ceph telemetry enable channel perf` 2024-09-12T23:37:46.658 INFO:tasks.workunit.client.0.smithi136.stderr:+ ceph telemetry enable channel perf 2024-09-12T23:37:46.733 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:37:46 smithi136 bash[22852]: [12/Sep/2024:23:37:46] ENGINE Bus STOPPING 2024-09-12T23:37:46.733 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:37:46 smithi136 bash[22852]: [12/Sep/2024:23:37:46] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-12T23:37:46.733 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:37:46 smithi136 bash[22852]: [12/Sep/2024:23:37:46] ENGINE Bus STOPPED 2024-09-12T23:37:46.734 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:37:46 smithi136 bash[22852]: [12/Sep/2024:23:37:46] ENGINE Bus STARTING 2024-09-12T23:37:46.734 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:46 smithi136 bash[22601]: audit 2024-09-12T23:37:44.755089+0000 mgr.y (mgr.24446) 147 : audit [DBG] from='client.24776 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:46.734 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:46 smithi136 bash[22601]: audit 2024-09-12T23:37:44.952509+0000 mgr.y (mgr.24446) 148 : audit [DBG] from='client.24776 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:46.734 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:46 smithi136 bash[22601]: audit 2024-09-12T23:37:45.326526+0000 mgr.y (mgr.24446) 149 : audit [DBG] from='client.24782 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:46.734 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:46 smithi136 bash[22601]: audit 2024-09-12T23:37:45.517891+0000 mgr.y (mgr.24446) 150 : audit [DBG] from='client.24782 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:46.734 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:46 smithi136 bash[22601]: audit 2024-09-12T23:37:45.520213+0000 mon.a (mon.0) 930 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:37:46.734 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:46 smithi136 bash[22601]: audit 2024-09-12T23:37:45.526032+0000 mon.a (mon.0) 931 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-12T23:37:46.734 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:46 smithi136 bash[22601]: audit 2024-09-12T23:37:46.397852+0000 mon.a (mon.0) 932 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:37:46.734 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:46 smithi136 bash[22601]: audit 2024-09-12T23:37:46.408983+0000 mon.a (mon.0) 933 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:37:46.734 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:46 smithi136 bash[22601]: audit 2024-09-12T23:37:46.414224+0000 mon.a (mon.0) 934 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:37:46.735 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:46 smithi136 bash[22601]: audit 2024-09-12T23:37:46.419092+0000 mon.a (mon.0) 935 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:37:46.735 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:46 smithi136 bash[22601]: audit 2024-09-12T23:37:46.419375+0000 mon.a (mon.0) 936 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:37:46.735 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:46 smithi136 bash[22601]: audit 2024-09-12T23:37:46.421126+0000 mon.a (mon.0) 937 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:37:46.735 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:46 smithi136 bash[22601]: audit 2024-09-12T23:37:46.421581+0000 mon.a (mon.0) 938 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:37:46.735 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:46 smithi136 bash[22601]: audit 2024-09-12T23:37:46.425959+0000 mon.a (mon.0) 939 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-12T23:37:46.735 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:46 smithi136 bash[22601]: audit 2024-09-12T23:37:46.428844+0000 mon.a (mon.0) 940 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:37:46.735 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:46 smithi136 bash[32189]: audit 2024-09-12T23:37:44.755089+0000 mgr.y (mgr.24446) 147 : audit [DBG] from='client.24776 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:46.735 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:46 smithi136 bash[32189]: audit 2024-09-12T23:37:44.952509+0000 mgr.y (mgr.24446) 148 : audit [DBG] from='client.24776 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:46.735 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:46 smithi136 bash[32189]: audit 2024-09-12T23:37:45.326526+0000 mgr.y (mgr.24446) 149 : audit [DBG] from='client.24782 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:46.735 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:46 smithi136 bash[32189]: audit 2024-09-12T23:37:45.517891+0000 mgr.y (mgr.24446) 150 : audit [DBG] from='client.24782 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:46.735 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:46 smithi136 bash[32189]: audit 2024-09-12T23:37:45.520213+0000 mon.a (mon.0) 930 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:37:46.736 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:46 smithi136 bash[32189]: audit 2024-09-12T23:37:45.526032+0000 mon.a (mon.0) 931 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-12T23:37:46.736 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:46 smithi136 bash[32189]: audit 2024-09-12T23:37:46.397852+0000 mon.a (mon.0) 932 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:37:46.736 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:46 smithi136 bash[32189]: audit 2024-09-12T23:37:46.408983+0000 mon.a (mon.0) 933 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:37:46.736 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:46 smithi136 bash[32189]: audit 2024-09-12T23:37:46.414224+0000 mon.a (mon.0) 934 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:37:46.736 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:46 smithi136 bash[32189]: audit 2024-09-12T23:37:46.419092+0000 mon.a (mon.0) 935 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:37:46.736 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:46 smithi136 bash[32189]: audit 2024-09-12T23:37:46.419375+0000 mon.a (mon.0) 936 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:37:46.736 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:46 smithi136 bash[32189]: audit 2024-09-12T23:37:46.421126+0000 mon.a (mon.0) 937 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:37:46.736 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:46 smithi136 bash[32189]: audit 2024-09-12T23:37:46.421581+0000 mon.a (mon.0) 938 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:37:46.736 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:46 smithi136 bash[32189]: audit 2024-09-12T23:37:46.425959+0000 mon.a (mon.0) 939 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-12T23:37:46.736 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:46 smithi136 bash[32189]: audit 2024-09-12T23:37:46.428844+0000 mon.a (mon.0) 940 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:37:46.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:46 smithi181 bash[24776]: audit 2024-09-12T23:37:44.755089+0000 mgr.y (mgr.24446) 147 : audit [DBG] from='client.24776 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:46.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:46 smithi181 bash[24776]: audit 2024-09-12T23:37:44.952509+0000 mgr.y (mgr.24446) 148 : audit [DBG] from='client.24776 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:46.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:46 smithi181 bash[24776]: audit 2024-09-12T23:37:45.326526+0000 mgr.y (mgr.24446) 149 : audit [DBG] from='client.24782 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:46.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:46 smithi181 bash[24776]: audit 2024-09-12T23:37:45.517891+0000 mgr.y (mgr.24446) 150 : audit [DBG] from='client.24782 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:46.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:46 smithi181 bash[24776]: audit 2024-09-12T23:37:45.520213+0000 mon.a (mon.0) 930 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:37:46.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:46 smithi181 bash[24776]: audit 2024-09-12T23:37:45.526032+0000 mon.a (mon.0) 931 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-12T23:37:46.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:46 smithi181 bash[24776]: audit 2024-09-12T23:37:46.397852+0000 mon.a (mon.0) 932 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:37:46.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:46 smithi181 bash[24776]: audit 2024-09-12T23:37:46.408983+0000 mon.a (mon.0) 933 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:37:46.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:46 smithi181 bash[24776]: audit 2024-09-12T23:37:46.414224+0000 mon.a (mon.0) 934 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:37:46.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:46 smithi181 bash[24776]: audit 2024-09-12T23:37:46.419092+0000 mon.a (mon.0) 935 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:37:46.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:46 smithi181 bash[24776]: audit 2024-09-12T23:37:46.419375+0000 mon.a (mon.0) 936 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:37:46.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:46 smithi181 bash[24776]: audit 2024-09-12T23:37:46.421126+0000 mon.a (mon.0) 937 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:37:46.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:46 smithi181 bash[24776]: audit 2024-09-12T23:37:46.421581+0000 mon.a (mon.0) 938 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:37:46.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:46 smithi181 bash[24776]: audit 2024-09-12T23:37:46.425959+0000 mon.a (mon.0) 939 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-12T23:37:46.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:46 smithi181 bash[24776]: audit 2024-09-12T23:37:46.428844+0000 mon.a (mon.0) 940 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:37:47.005 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:37:46 smithi136 bash[22852]: [12/Sep/2024:23:37:46] ENGINE Serving on http://:::9283 2024-09-12T23:37:47.006 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:37:46 smithi136 bash[22852]: [12/Sep/2024:23:37:46] ENGINE Bus STARTED 2024-09-12T23:37:47.006 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:37:46 smithi136 bash[22852]: [12/Sep/2024:23:37:46] ENGINE Bus STOPPING 2024-09-12T23:37:47.148 INFO:tasks.workunit.client.0.smithi136.stdout:channel_perf is enabled 2024-09-12T23:37:47.166 INFO:tasks.workunit.client.0.smithi136.stderr:++ ceph config get mgr mgr/telemetry/last_opt_revision 2024-09-12T23:37:47.506 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[22852]: [12/Sep/2024:23:37:47] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-12T23:37:47.506 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[22852]: [12/Sep/2024:23:37:47] ENGINE Bus STOPPED 2024-09-12T23:37:47.506 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[22852]: [12/Sep/2024:23:37:47] ENGINE Bus STARTING 2024-09-12T23:37:47.506 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[22852]: [12/Sep/2024:23:37:47] ENGINE Serving on http://:::9283 2024-09-12T23:37:47.506 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[22852]: [12/Sep/2024:23:37:47] ENGINE Bus STARTED 2024-09-12T23:37:47.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[32189]: audit 2024-09-12T23:37:45.890964+0000 mgr.y (mgr.24446) 151 : audit [DBG] from='client.14874 -' entity='client.admin' cmd=[{"prefix": "telemetry collection ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:47.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[32189]: cluster 2024-09-12T23:37:45.997856+0000 mgr.y (mgr.24446) 152 : cluster [DBG] pgmap v121: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:47.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[32189]: audit 2024-09-12T23:37:46.038181+0000 mgr.y (mgr.24446) 153 : audit [DBG] from='client.14874 -' entity='client.admin' cmd=[{"prefix": "telemetry collection ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:47.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[32189]: audit 2024-09-12T23:37:46.392563+0000 mgr.y (mgr.24446) 154 : audit [DBG] from='client.24794 -' entity='client.admin' cmd=[{"prefix": "telemetry on", "license": "sharing-1-0", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:47.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[32189]: cluster 2024-09-12T23:37:46.477439+0000 mon.a (mon.0) 941 : cluster [DBG] mgrmap e22: y(active, since 3m), standbys: x 2024-09-12T23:37:47.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[32189]: audit 2024-09-12T23:37:46.524553+0000 mon.a (mon.0) 942 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:37:47.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[32189]: audit 2024-09-12T23:37:46.528332+0000 mon.a (mon.0) 943 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-12T23:37:47.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[32189]: audit 2024-09-12T23:37:46.633613+0000 mgr.y (mgr.24446) 155 : audit [DBG] from='client.24794 -' entity='client.admin' cmd=[{"prefix": "telemetry on", "license": "sharing-1-0", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:47.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[32189]: audit 2024-09-12T23:37:46.639362+0000 mon.a (mon.0) 944 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:37:47.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[32189]: audit 2024-09-12T23:37:46.645337+0000 mon.a (mon.0) 945 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:37:47.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[32189]: audit 2024-09-12T23:37:46.647200+0000 mon.a (mon.0) 946 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:37:47.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[32189]: audit 2024-09-12T23:37:46.650600+0000 mon.a (mon.0) 947 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-12T23:37:47.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[32189]: audit 2024-09-12T23:37:46.979546+0000 mon.a (mon.0) 948 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:37:47.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[32189]: audit 2024-09-12T23:37:46.990540+0000 mon.a (mon.0) 949 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:37:47.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[32189]: audit 2024-09-12T23:37:46.994081+0000 mon.a (mon.0) 950 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:37:47.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[32189]: audit 2024-09-12T23:37:46.996815+0000 mon.a (mon.0) 951 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:37:47.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[32189]: audit 2024-09-12T23:37:46.998456+0000 mon.a (mon.0) 952 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-12T23:37:47.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[32189]: audit 2024-09-12T23:37:47.001164+0000 mon.a (mon.0) 953 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:37:47.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[32189]: audit 2024-09-12T23:37:47.012564+0000 mon.a (mon.0) 954 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:37:47.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[32189]: audit 2024-09-12T23:37:47.127738+0000 mon.a (mon.0) 955 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:47.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[32189]: audit 2024-09-12T23:37:47.127938+0000 mon.a (mon.0) 956 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:47.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[32189]: audit 2024-09-12T23:37:47.128116+0000 mon.a (mon.0) 957 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:47.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[32189]: audit 2024-09-12T23:37:47.131557+0000 mon.a (mon.0) 958 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:47.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[32189]: audit 2024-09-12T23:37:47.131913+0000 mon.a (mon.0) 959 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:47.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[32189]: audit 2024-09-12T23:37:47.132193+0000 mon.a (mon.0) 960 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:47.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[32189]: audit 2024-09-12T23:37:47.134663+0000 mon.a (mon.0) 961 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:47.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[32189]: audit 2024-09-12T23:37:47.134975+0000 mon.a (mon.0) 962 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:47.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[32189]: audit 2024-09-12T23:37:47.135367+0000 mon.a (mon.0) 963 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:47.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[32189]: audit 2024-09-12T23:37:47.151012+0000 mon.a (mon.0) 964 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:37:47.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[32189]: audit 2024-09-12T23:37:47.151145+0000 mon.a (mon.0) 965 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:37:47.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[32189]: audit 2024-09-12T23:37:47.151716+0000 mon.a (mon.0) 966 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:47.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[32189]: audit 2024-09-12T23:37:47.152075+0000 mon.a (mon.0) 967 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:47.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[32189]: audit 2024-09-12T23:37:47.152414+0000 mon.a (mon.0) 968 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:47.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[32189]: audit 2024-09-12T23:37:47.154982+0000 mon.b (mon.1) 26 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:37:47.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[32189]: audit 2024-09-12T23:37:47.155100+0000 mon.b (mon.1) 27 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:37:47.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[32189]: audit 2024-09-12T23:37:47.155757+0000 mon.a (mon.0) 969 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:47.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[32189]: audit 2024-09-12T23:37:47.156566+0000 mon.a (mon.0) 970 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:47.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[32189]: audit 2024-09-12T23:37:47.156745+0000 mon.a (mon.0) 971 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:47.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[22601]: audit 2024-09-12T23:37:45.890964+0000 mgr.y (mgr.24446) 151 : audit [DBG] from='client.14874 -' entity='client.admin' cmd=[{"prefix": "telemetry collection ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:47.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[22601]: cluster 2024-09-12T23:37:45.997856+0000 mgr.y (mgr.24446) 152 : cluster [DBG] pgmap v121: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:47.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[22601]: audit 2024-09-12T23:37:46.038181+0000 mgr.y (mgr.24446) 153 : audit [DBG] from='client.14874 -' entity='client.admin' cmd=[{"prefix": "telemetry collection ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:47.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[22601]: audit 2024-09-12T23:37:46.392563+0000 mgr.y (mgr.24446) 154 : audit [DBG] from='client.24794 -' entity='client.admin' cmd=[{"prefix": "telemetry on", "license": "sharing-1-0", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:47.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[22601]: cluster 2024-09-12T23:37:46.477439+0000 mon.a (mon.0) 941 : cluster [DBG] mgrmap e22: y(active, since 3m), standbys: x 2024-09-12T23:37:47.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[22601]: audit 2024-09-12T23:37:46.524553+0000 mon.a (mon.0) 942 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:37:47.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[22601]: audit 2024-09-12T23:37:46.528332+0000 mon.a (mon.0) 943 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-12T23:37:47.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[22601]: audit 2024-09-12T23:37:46.633613+0000 mgr.y (mgr.24446) 155 : audit [DBG] from='client.24794 -' entity='client.admin' cmd=[{"prefix": "telemetry on", "license": "sharing-1-0", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:47.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[22601]: audit 2024-09-12T23:37:46.639362+0000 mon.a (mon.0) 944 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:37:47.511 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[22601]: audit 2024-09-12T23:37:46.645337+0000 mon.a (mon.0) 945 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:37:47.511 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[22601]: audit 2024-09-12T23:37:46.647200+0000 mon.a (mon.0) 946 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:37:47.511 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[22601]: audit 2024-09-12T23:37:46.650600+0000 mon.a (mon.0) 947 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-12T23:37:47.511 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[22601]: audit 2024-09-12T23:37:46.979546+0000 mon.a (mon.0) 948 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:37:47.511 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[22601]: audit 2024-09-12T23:37:46.990540+0000 mon.a (mon.0) 949 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:37:47.511 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[22601]: audit 2024-09-12T23:37:46.994081+0000 mon.a (mon.0) 950 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:37:47.511 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[22601]: audit 2024-09-12T23:37:46.996815+0000 mon.a (mon.0) 951 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:37:47.511 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[22601]: audit 2024-09-12T23:37:46.998456+0000 mon.a (mon.0) 952 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-12T23:37:47.511 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[22601]: audit 2024-09-12T23:37:47.001164+0000 mon.a (mon.0) 953 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:37:47.511 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[22601]: audit 2024-09-12T23:37:47.012564+0000 mon.a (mon.0) 954 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:37:47.512 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[32189]: audit 2024-09-12T23:37:47.158161+0000 mon.c (mon.2) 47 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:37:47.512 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[32189]: audit 2024-09-12T23:37:47.158241+0000 mon.c (mon.2) 48 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:37:47.512 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[32189]: audit 2024-09-12T23:37:47.158351+0000 mon.a (mon.0) 972 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:47.512 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[32189]: audit 2024-09-12T23:37:47.158695+0000 mon.a (mon.0) 973 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:47.512 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[32189]: audit 2024-09-12T23:37:47.158807+0000 mon.a (mon.0) 974 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:47.512 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[32189]: audit 2024-09-12T23:37:47.441698+0000 mon.c (mon.2) 49 : audit [DBG] from='client.? 172.21.15.136:0/109380988' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/telemetry/last_opt_revision"}]: dispatch 2024-09-12T23:37:47.512 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[22601]: audit 2024-09-12T23:37:47.127738+0000 mon.a (mon.0) 955 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:47.512 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[22601]: audit 2024-09-12T23:37:47.127938+0000 mon.a (mon.0) 956 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:47.512 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[22601]: audit 2024-09-12T23:37:47.128116+0000 mon.a (mon.0) 957 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:47.513 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[22601]: audit 2024-09-12T23:37:47.131557+0000 mon.a (mon.0) 958 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:47.513 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[22601]: audit 2024-09-12T23:37:47.131913+0000 mon.a (mon.0) 959 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:47.513 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[22601]: audit 2024-09-12T23:37:47.132193+0000 mon.a (mon.0) 960 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:47.513 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[22601]: audit 2024-09-12T23:37:47.134663+0000 mon.a (mon.0) 961 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:47.513 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[22601]: audit 2024-09-12T23:37:47.134975+0000 mon.a (mon.0) 962 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:47.513 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[22601]: audit 2024-09-12T23:37:47.135367+0000 mon.a (mon.0) 963 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:47.513 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[22601]: audit 2024-09-12T23:37:47.151012+0000 mon.a (mon.0) 964 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:37:47.513 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[22601]: audit 2024-09-12T23:37:47.151145+0000 mon.a (mon.0) 965 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:37:47.513 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[22601]: audit 2024-09-12T23:37:47.151716+0000 mon.a (mon.0) 966 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:47.513 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[22601]: audit 2024-09-12T23:37:47.152075+0000 mon.a (mon.0) 967 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:47.513 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[22601]: audit 2024-09-12T23:37:47.152414+0000 mon.a (mon.0) 968 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:47.513 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[22601]: audit 2024-09-12T23:37:47.154982+0000 mon.b (mon.1) 26 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:37:47.513 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[22601]: audit 2024-09-12T23:37:47.155100+0000 mon.b (mon.1) 27 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:37:47.514 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[22601]: audit 2024-09-12T23:37:47.155757+0000 mon.a (mon.0) 969 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:47.514 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[22601]: audit 2024-09-12T23:37:47.156566+0000 mon.a (mon.0) 970 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:47.514 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[22601]: audit 2024-09-12T23:37:47.156745+0000 mon.a (mon.0) 971 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:47.514 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[22601]: audit 2024-09-12T23:37:47.158161+0000 mon.c (mon.2) 47 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:37:47.514 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[22601]: audit 2024-09-12T23:37:47.158241+0000 mon.c (mon.2) 48 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:37:47.514 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[22601]: audit 2024-09-12T23:37:47.158351+0000 mon.a (mon.0) 972 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:47.514 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[22601]: audit 2024-09-12T23:37:47.158695+0000 mon.a (mon.0) 973 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:47.514 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[22601]: audit 2024-09-12T23:37:47.158807+0000 mon.a (mon.0) 974 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:47.514 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:47 smithi136 bash[22601]: audit 2024-09-12T23:37:47.441698+0000 mon.c (mon.2) 49 : audit [DBG] from='client.? 172.21.15.136:0/109380988' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/telemetry/last_opt_revision"}]: dispatch 2024-09-12T23:37:47.549 INFO:tasks.workunit.client.0.smithi136.stderr:+ LAST_OPT_REVISION=1 2024-09-12T23:37:47.549 INFO:tasks.workunit.client.0.smithi136.stderr:+ '[' 1 -ne 1 ']' 2024-09-12T23:37:47.549 INFO:tasks.workunit.client.0.smithi136.stderr:+ ceph telemetry show 2024-09-12T23:37:47.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:47 smithi181 bash[24776]: audit 2024-09-12T23:37:45.890964+0000 mgr.y (mgr.24446) 151 : audit [DBG] from='client.14874 -' entity='client.admin' cmd=[{"prefix": "telemetry collection ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:47.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:47 smithi181 bash[24776]: cluster 2024-09-12T23:37:45.997856+0000 mgr.y (mgr.24446) 152 : cluster [DBG] pgmap v121: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:47.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:47 smithi181 bash[24776]: audit 2024-09-12T23:37:46.038181+0000 mgr.y (mgr.24446) 153 : audit [DBG] from='client.14874 -' entity='client.admin' cmd=[{"prefix": "telemetry collection ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:47.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:47 smithi181 bash[24776]: audit 2024-09-12T23:37:46.392563+0000 mgr.y (mgr.24446) 154 : audit [DBG] from='client.24794 -' entity='client.admin' cmd=[{"prefix": "telemetry on", "license": "sharing-1-0", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:47.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:47 smithi181 bash[24776]: cluster 2024-09-12T23:37:46.477439+0000 mon.a (mon.0) 941 : cluster [DBG] mgrmap e22: y(active, since 3m), standbys: x 2024-09-12T23:37:47.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:47 smithi181 bash[24776]: audit 2024-09-12T23:37:46.524553+0000 mon.a (mon.0) 942 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:37:47.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:47 smithi181 bash[24776]: audit 2024-09-12T23:37:46.528332+0000 mon.a (mon.0) 943 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-12T23:37:47.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:47 smithi181 bash[24776]: audit 2024-09-12T23:37:46.633613+0000 mgr.y (mgr.24446) 155 : audit [DBG] from='client.24794 -' entity='client.admin' cmd=[{"prefix": "telemetry on", "license": "sharing-1-0", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:47.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:47 smithi181 bash[24776]: audit 2024-09-12T23:37:46.639362+0000 mon.a (mon.0) 944 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:37:47.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:47 smithi181 bash[24776]: audit 2024-09-12T23:37:46.645337+0000 mon.a (mon.0) 945 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:37:47.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:47 smithi181 bash[24776]: audit 2024-09-12T23:37:46.647200+0000 mon.a (mon.0) 946 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:37:47.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:47 smithi181 bash[24776]: audit 2024-09-12T23:37:46.650600+0000 mon.a (mon.0) 947 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-12T23:37:47.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:47 smithi181 bash[24776]: audit 2024-09-12T23:37:46.979546+0000 mon.a (mon.0) 948 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:37:47.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:47 smithi181 bash[24776]: audit 2024-09-12T23:37:46.990540+0000 mon.a (mon.0) 949 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:37:47.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:47 smithi181 bash[24776]: audit 2024-09-12T23:37:46.994081+0000 mon.a (mon.0) 950 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:37:47.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:47 smithi181 bash[24776]: audit 2024-09-12T23:37:46.996815+0000 mon.a (mon.0) 951 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:37:47.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:47 smithi181 bash[24776]: audit 2024-09-12T23:37:46.998456+0000 mon.a (mon.0) 952 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-12T23:37:47.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:47 smithi181 bash[24776]: audit 2024-09-12T23:37:47.001164+0000 mon.a (mon.0) 953 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:37:47.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:47 smithi181 bash[24776]: audit 2024-09-12T23:37:47.012564+0000 mon.a (mon.0) 954 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:37:47.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:47 smithi181 bash[24776]: audit 2024-09-12T23:37:47.127738+0000 mon.a (mon.0) 955 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:47.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:47 smithi181 bash[24776]: audit 2024-09-12T23:37:47.127938+0000 mon.a (mon.0) 956 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:47.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:47 smithi181 bash[24776]: audit 2024-09-12T23:37:47.128116+0000 mon.a (mon.0) 957 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:47.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:47 smithi181 bash[24776]: audit 2024-09-12T23:37:47.131557+0000 mon.a (mon.0) 958 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:47.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:47 smithi181 bash[24776]: audit 2024-09-12T23:37:47.131913+0000 mon.a (mon.0) 959 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:47.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:47 smithi181 bash[24776]: audit 2024-09-12T23:37:47.132193+0000 mon.a (mon.0) 960 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:47.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:47 smithi181 bash[24776]: audit 2024-09-12T23:37:47.134663+0000 mon.a (mon.0) 961 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:47.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:47 smithi181 bash[24776]: audit 2024-09-12T23:37:47.134975+0000 mon.a (mon.0) 962 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:47.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:47 smithi181 bash[24776]: audit 2024-09-12T23:37:47.135367+0000 mon.a (mon.0) 963 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:47.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:47 smithi181 bash[24776]: audit 2024-09-12T23:37:47.151012+0000 mon.a (mon.0) 964 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:37:47.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:47 smithi181 bash[24776]: audit 2024-09-12T23:37:47.151145+0000 mon.a (mon.0) 965 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:37:47.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:47 smithi181 bash[24776]: audit 2024-09-12T23:37:47.151716+0000 mon.a (mon.0) 966 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:47.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:47 smithi181 bash[24776]: audit 2024-09-12T23:37:47.152075+0000 mon.a (mon.0) 967 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:47.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:47 smithi181 bash[24776]: audit 2024-09-12T23:37:47.152414+0000 mon.a (mon.0) 968 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:47.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:47 smithi181 bash[24776]: audit 2024-09-12T23:37:47.154982+0000 mon.b (mon.1) 26 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:37:47.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:47 smithi181 bash[24776]: audit 2024-09-12T23:37:47.155100+0000 mon.b (mon.1) 27 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:37:47.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:47 smithi181 bash[24776]: audit 2024-09-12T23:37:47.155757+0000 mon.a (mon.0) 969 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:47.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:47 smithi181 bash[24776]: audit 2024-09-12T23:37:47.156566+0000 mon.a (mon.0) 970 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:47.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:47 smithi181 bash[24776]: audit 2024-09-12T23:37:47.156745+0000 mon.a (mon.0) 971 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:47.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:47 smithi181 bash[24776]: audit 2024-09-12T23:37:47.158161+0000 mon.c (mon.2) 47 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:37:47.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:47 smithi181 bash[24776]: audit 2024-09-12T23:37:47.158241+0000 mon.c (mon.2) 48 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:37:47.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:47 smithi181 bash[24776]: audit 2024-09-12T23:37:47.158351+0000 mon.a (mon.0) 972 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:47.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:47 smithi181 bash[24776]: audit 2024-09-12T23:37:47.158695+0000 mon.a (mon.0) 973 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:47.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:47 smithi181 bash[24776]: audit 2024-09-12T23:37:47.158807+0000 mon.a (mon.0) 974 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:47.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:47 smithi181 bash[24776]: audit 2024-09-12T23:37:47.441698+0000 mon.c (mon.2) 49 : audit [DBG] from='client.? 172.21.15.136:0/109380988' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/telemetry/last_opt_revision"}]: dispatch 2024-09-12T23:37:48.623 INFO:tasks.workunit.client.0.smithi136.stdout:{ 2024-09-12T23:37:48.623 INFO:tasks.workunit.client.0.smithi136.stdout: "balancer": { 2024-09-12T23:37:48.623 INFO:tasks.workunit.client.0.smithi136.stdout: "active": true, 2024-09-12T23:37:48.623 INFO:tasks.workunit.client.0.smithi136.stdout: "mode": "" 2024-09-12T23:37:48.623 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.623 INFO:tasks.workunit.client.0.smithi136.stdout: "channels": [ 2024-09-12T23:37:48.623 INFO:tasks.workunit.client.0.smithi136.stdout: "basic", 2024-09-12T23:37:48.624 INFO:tasks.workunit.client.0.smithi136.stdout: "crash", 2024-09-12T23:37:48.624 INFO:tasks.workunit.client.0.smithi136.stdout: "device", 2024-09-12T23:37:48.624 INFO:tasks.workunit.client.0.smithi136.stdout: "ident", 2024-09-12T23:37:48.624 INFO:tasks.workunit.client.0.smithi136.stdout: "perf" 2024-09-12T23:37:48.624 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:48.624 INFO:tasks.workunit.client.0.smithi136.stdout: "channels_available": [ 2024-09-12T23:37:48.624 INFO:tasks.workunit.client.0.smithi136.stdout: "basic", 2024-09-12T23:37:48.624 INFO:tasks.workunit.client.0.smithi136.stdout: "ident", 2024-09-12T23:37:48.624 INFO:tasks.workunit.client.0.smithi136.stdout: "crash", 2024-09-12T23:37:48.624 INFO:tasks.workunit.client.0.smithi136.stdout: "device", 2024-09-12T23:37:48.624 INFO:tasks.workunit.client.0.smithi136.stdout: "perf" 2024-09-12T23:37:48.624 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:48.624 INFO:tasks.workunit.client.0.smithi136.stdout: "collections_available": [ 2024-09-12T23:37:48.624 INFO:tasks.workunit.client.0.smithi136.stdout: "basic_base", 2024-09-12T23:37:48.625 INFO:tasks.workunit.client.0.smithi136.stdout: "device_base", 2024-09-12T23:37:48.625 INFO:tasks.workunit.client.0.smithi136.stdout: "crash_base", 2024-09-12T23:37:48.625 INFO:tasks.workunit.client.0.smithi136.stdout: "ident_base", 2024-09-12T23:37:48.625 INFO:tasks.workunit.client.0.smithi136.stdout: "perf_perf", 2024-09-12T23:37:48.625 INFO:tasks.workunit.client.0.smithi136.stdout: "basic_mds_metadata", 2024-09-12T23:37:48.625 INFO:tasks.workunit.client.0.smithi136.stdout: "basic_pool_usage", 2024-09-12T23:37:48.625 INFO:tasks.workunit.client.0.smithi136.stdout: "basic_usage_by_class", 2024-09-12T23:37:48.625 INFO:tasks.workunit.client.0.smithi136.stdout: "basic_rook_v01", 2024-09-12T23:37:48.625 INFO:tasks.workunit.client.0.smithi136.stdout: "perf_memory_metrics", 2024-09-12T23:37:48.625 INFO:tasks.workunit.client.0.smithi136.stdout: "basic_pool_options_bluestore" 2024-09-12T23:37:48.625 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:48.625 INFO:tasks.workunit.client.0.smithi136.stdout: "collections_opted_in": [ 2024-09-12T23:37:48.625 INFO:tasks.workunit.client.0.smithi136.stdout: "basic_base", 2024-09-12T23:37:48.626 INFO:tasks.workunit.client.0.smithi136.stdout: "device_base", 2024-09-12T23:37:48.626 INFO:tasks.workunit.client.0.smithi136.stdout: "crash_base", 2024-09-12T23:37:48.626 INFO:tasks.workunit.client.0.smithi136.stdout: "ident_base", 2024-09-12T23:37:48.626 INFO:tasks.workunit.client.0.smithi136.stdout: "perf_perf", 2024-09-12T23:37:48.626 INFO:tasks.workunit.client.0.smithi136.stdout: "basic_mds_metadata", 2024-09-12T23:37:48.626 INFO:tasks.workunit.client.0.smithi136.stdout: "basic_pool_usage", 2024-09-12T23:37:48.626 INFO:tasks.workunit.client.0.smithi136.stdout: "basic_usage_by_class", 2024-09-12T23:37:48.626 INFO:tasks.workunit.client.0.smithi136.stdout: "basic_rook_v01", 2024-09-12T23:37:48.626 INFO:tasks.workunit.client.0.smithi136.stdout: "perf_memory_metrics", 2024-09-12T23:37:48.626 INFO:tasks.workunit.client.0.smithi136.stdout: "basic_pool_options_bluestore" 2024-09-12T23:37:48.626 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:48.626 INFO:tasks.workunit.client.0.smithi136.stdout: "config": { 2024-09-12T23:37:48.626 INFO:tasks.workunit.client.0.smithi136.stdout: "active_changed": [ 2024-09-12T23:37:48.627 INFO:tasks.workunit.client.0.smithi136.stdout: "auth_allow_insecure_global_id_reclaim", 2024-09-12T23:37:48.627 INFO:tasks.workunit.client.0.smithi136.stdout: "auth_debug", 2024-09-12T23:37:48.627 INFO:tasks.workunit.client.0.smithi136.stdout: "auth_mon_ticket_ttl", 2024-09-12T23:37:48.627 INFO:tasks.workunit.client.0.smithi136.stdout: "auth_service_ticket_ttl", 2024-09-12T23:37:48.627 INFO:tasks.workunit.client.0.smithi136.stdout: "bdev_debug_aio", 2024-09-12T23:37:48.627 INFO:tasks.workunit.client.0.smithi136.stdout: "client_check_pool_perm", 2024-09-12T23:37:48.627 INFO:tasks.workunit.client.0.smithi136.stdout: "client_mount_gid", 2024-09-12T23:37:48.627 INFO:tasks.workunit.client.0.smithi136.stdout: "client_mount_uid", 2024-09-12T23:37:48.627 INFO:tasks.workunit.client.0.smithi136.stdout: "client_quota", 2024-09-12T23:37:48.627 INFO:tasks.workunit.client.0.smithi136.stdout: "container_image", 2024-09-12T23:37:48.627 INFO:tasks.workunit.client.0.smithi136.stdout: "daemonize", 2024-09-12T23:37:48.627 INFO:tasks.workunit.client.0.smithi136.stdout: "debug_asserts_on_shutdown", 2024-09-12T23:37:48.627 INFO:tasks.workunit.client.0.smithi136.stdout: "debug_mgr", 2024-09-12T23:37:48.627 INFO:tasks.workunit.client.0.smithi136.stdout: "debug_mon", 2024-09-12T23:37:48.627 INFO:tasks.workunit.client.0.smithi136.stdout: "debug_ms", 2024-09-12T23:37:48.628 INFO:tasks.workunit.client.0.smithi136.stdout: "debug_osd", 2024-09-12T23:37:48.628 INFO:tasks.workunit.client.0.smithi136.stdout: "debug_paxos", 2024-09-12T23:37:48.628 INFO:tasks.workunit.client.0.smithi136.stdout: "keyring", 2024-09-12T23:37:48.628 INFO:tasks.workunit.client.0.smithi136.stdout: "leveldb_block_size", 2024-09-12T23:37:48.628 INFO:tasks.workunit.client.0.smithi136.stdout: "leveldb_cache_size", 2024-09-12T23:37:48.628 INFO:tasks.workunit.client.0.smithi136.stdout: "leveldb_compression", 2024-09-12T23:37:48.628 INFO:tasks.workunit.client.0.smithi136.stdout: "leveldb_log", 2024-09-12T23:37:48.628 INFO:tasks.workunit.client.0.smithi136.stdout: "leveldb_write_buffer_size", 2024-09-12T23:37:48.628 INFO:tasks.workunit.client.0.smithi136.stdout: "log_stderr_prefix", 2024-09-12T23:37:48.628 INFO:tasks.workunit.client.0.smithi136.stdout: "log_to_file", 2024-09-12T23:37:48.628 INFO:tasks.workunit.client.0.smithi136.stdout: "log_to_journald", 2024-09-12T23:37:48.628 INFO:tasks.workunit.client.0.smithi136.stdout: "log_to_stderr", 2024-09-12T23:37:48.628 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_allow_pool_delete", 2024-09-12T23:37:48.628 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_clock_drift_allowed", 2024-09-12T23:37:48.629 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_cluster_log_to_file", 2024-09-12T23:37:48.629 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_cluster_log_to_stderr", 2024-09-12T23:37:48.629 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_data_avail_warn", 2024-09-12T23:37:48.629 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_election_default_strategy", 2024-09-12T23:37:48.629 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_host", 2024-09-12T23:37:48.629 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_max_pg_per_osd", 2024-09-12T23:37:48.629 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_mgr_mkfs_grace", 2024-09-12T23:37:48.629 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_osd_prime_pg_temp", 2024-09-12T23:37:48.629 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_osd_reporter_subtree_level", 2024-09-12T23:37:48.629 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_pg_warn_max_object_skew", 2024-09-12T23:37:48.629 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_reweight_min_bytes_per_osd", 2024-09-12T23:37:48.629 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_reweight_min_pgs_per_osd", 2024-09-12T23:37:48.629 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_crush_straw_calc_version_zero", 2024-09-12T23:37:48.629 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_insecure_global_id_reclaim", 2024-09-12T23:37:48.630 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed", 2024-09-12T23:37:48.630 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_legacy_crush_tunables", 2024-09-12T23:37:48.630 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_osd_down_out_interval_zero", 2024-09-12T23:37:48.630 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_pool_pg_num_not_power_of_two", 2024-09-12T23:37:48.630 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_too_few_osds", 2024-09-12T23:37:48.630 INFO:tasks.workunit.client.0.smithi136.stdout: "ms_die_on_bug", 2024-09-12T23:37:48.630 INFO:tasks.workunit.client.0.smithi136.stdout: "ms_die_on_old_message", 2024-09-12T23:37:48.630 INFO:tasks.workunit.client.0.smithi136.stdout: "no_config_file", 2024-09-12T23:37:48.630 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_class_default_list", 2024-09-12T23:37:48.630 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_class_load_list", 2024-09-12T23:37:48.630 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_misdirected_ops", 2024-09-12T23:37:48.630 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_op_order", 2024-09-12T23:37:48.630 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_pg_log_writeout", 2024-09-12T23:37:48.631 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_shutdown", 2024-09-12T23:37:48.631 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_verify_cached_snaps", 2024-09-12T23:37:48.631 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_verify_missing_on_start", 2024-09-12T23:37:48.631 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_verify_stray_on_activate", 2024-09-12T23:37:48.631 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_deep_scrub_update_digest_min_age", 2024-09-12T23:37:48.631 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mclock_profile", 2024-09-12T23:37:48.631 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mclock_scheduler_background_best_effort_lim", 2024-09-12T23:37:48.631 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mclock_scheduler_background_best_effort_res", 2024-09-12T23:37:48.631 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mclock_scheduler_background_best_effort_wgt", 2024-09-12T23:37:48.631 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mclock_scheduler_background_recovery_lim", 2024-09-12T23:37:48.631 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mclock_scheduler_background_recovery_res", 2024-09-12T23:37:48.631 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mclock_scheduler_background_recovery_wgt", 2024-09-12T23:37:48.631 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mclock_scheduler_client_lim", 2024-09-12T23:37:48.631 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mclock_scheduler_client_res", 2024-09-12T23:37:48.632 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mclock_scheduler_client_wgt", 2024-09-12T23:37:48.632 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_memory_target", 2024-09-12T23:37:48.632 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_memory_target_autotune", 2024-09-12T23:37:48.632 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_op_queue", 2024-09-12T23:37:48.632 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_op_queue_cut_off", 2024-09-12T23:37:48.632 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_pool_default_erasure_code_profile", 2024-09-12T23:37:48.632 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_pool_default_pg_autoscale_mode", 2024-09-12T23:37:48.632 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_recover_clone_overlap", 2024-09-12T23:37:48.632 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_recovery_max_chunk", 2024-09-12T23:37:48.632 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_scrub_load_threshold", 2024-09-12T23:37:48.632 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_scrub_max_interval", 2024-09-12T23:37:48.632 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_shutdown_pgref_assert", 2024-09-12T23:37:48.632 INFO:tasks.workunit.client.0.smithi136.stdout: "public_addr", 2024-09-12T23:37:48.633 INFO:tasks.workunit.client.0.smithi136.stdout: "public_addrv", 2024-09-12T23:37:48.633 INFO:tasks.workunit.client.0.smithi136.stdout: "public_network", 2024-09-12T23:37:48.633 INFO:tasks.workunit.client.0.smithi136.stdout: "rbd_default_features", 2024-09-12T23:37:48.633 INFO:tasks.workunit.client.0.smithi136.stdout: "rbd_qos_exclude_ops", 2024-09-12T23:37:48.633 INFO:tasks.workunit.client.0.smithi136.stdout: "setgroup", 2024-09-12T23:37:48.633 INFO:tasks.workunit.client.0.smithi136.stdout: "setuser" 2024-09-12T23:37:48.633 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:48.633 INFO:tasks.workunit.client.0.smithi136.stdout: "cluster_changed": [ 2024-09-12T23:37:48.633 INFO:tasks.workunit.client.0.smithi136.stdout: "auth_allow_insecure_global_id_reclaim", 2024-09-12T23:37:48.633 INFO:tasks.workunit.client.0.smithi136.stdout: "auth_debug", 2024-09-12T23:37:48.633 INFO:tasks.workunit.client.0.smithi136.stdout: "auth_mon_ticket_ttl", 2024-09-12T23:37:48.633 INFO:tasks.workunit.client.0.smithi136.stdout: "auth_service_ticket_ttl", 2024-09-12T23:37:48.633 INFO:tasks.workunit.client.0.smithi136.stdout: "bdev_debug_aio", 2024-09-12T23:37:48.634 INFO:tasks.workunit.client.0.smithi136.stdout: "container_image", 2024-09-12T23:37:48.634 INFO:tasks.workunit.client.0.smithi136.stdout: "debug_asserts_on_shutdown", 2024-09-12T23:37:48.634 INFO:tasks.workunit.client.0.smithi136.stdout: "debug_mgr", 2024-09-12T23:37:48.634 INFO:tasks.workunit.client.0.smithi136.stdout: "debug_mon", 2024-09-12T23:37:48.634 INFO:tasks.workunit.client.0.smithi136.stdout: "debug_ms", 2024-09-12T23:37:48.634 INFO:tasks.workunit.client.0.smithi136.stdout: "debug_osd", 2024-09-12T23:37:48.634 INFO:tasks.workunit.client.0.smithi136.stdout: "debug_paxos", 2024-09-12T23:37:48.634 INFO:tasks.workunit.client.0.smithi136.stdout: "log_to_file", 2024-09-12T23:37:48.634 INFO:tasks.workunit.client.0.smithi136.stdout: "log_to_journald", 2024-09-12T23:37:48.634 INFO:tasks.workunit.client.0.smithi136.stdout: "log_to_stderr", 2024-09-12T23:37:48.634 INFO:tasks.workunit.client.0.smithi136.stdout: "mds_join_fs", 2024-09-12T23:37:48.634 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/cephadm/allow_ptrace", 2024-09-12T23:37:48.634 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/cephadm/container_init", 2024-09-12T23:37:48.634 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/cephadm/migration_current", 2024-09-12T23:37:48.635 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/cephadm/use_repo_digest", 2024-09-12T23:37:48.635 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/dashboard/ALERTMANAGER_API_HOST", 2024-09-12T23:37:48.635 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/dashboard/GRAFANA_API_SSL_VERIFY", 2024-09-12T23:37:48.635 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/dashboard/GRAFANA_API_URL", 2024-09-12T23:37:48.635 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/dashboard/PROMETHEUS_API_HOST", 2024-09-12T23:37:48.635 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/dashboard/ssl_server_port", 2024-09-12T23:37:48.635 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/orchestrator/orchestrator", 2024-09-12T23:37:48.635 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/telemetry/channel_ident", 2024-09-12T23:37:48.635 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/telemetry/channel_perf", 2024-09-12T23:37:48.635 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/telemetry/description", 2024-09-12T23:37:48.635 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/telemetry/enabled", 2024-09-12T23:37:48.635 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/telemetry/organization", 2024-09-12T23:37:48.635 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_allow_pool_delete", 2024-09-12T23:37:48.636 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_clock_drift_allowed", 2024-09-12T23:37:48.636 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_data_avail_warn", 2024-09-12T23:37:48.636 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_election_default_strategy", 2024-09-12T23:37:48.636 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_max_pg_per_osd", 2024-09-12T23:37:48.636 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_mgr_mkfs_grace", 2024-09-12T23:37:48.636 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_osd_prime_pg_temp", 2024-09-12T23:37:48.636 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_osd_reporter_subtree_level", 2024-09-12T23:37:48.636 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_pg_warn_max_object_skew", 2024-09-12T23:37:48.636 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_reweight_min_bytes_per_osd", 2024-09-12T23:37:48.636 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_reweight_min_pgs_per_osd", 2024-09-12T23:37:48.636 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_crush_straw_calc_version_zero", 2024-09-12T23:37:48.636 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_insecure_global_id_reclaim", 2024-09-12T23:37:48.636 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed", 2024-09-12T23:37:48.636 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_legacy_crush_tunables", 2024-09-12T23:37:48.637 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_osd_down_out_interval_zero", 2024-09-12T23:37:48.637 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_pool_pg_num_not_power_of_two", 2024-09-12T23:37:48.637 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_too_few_osds", 2024-09-12T23:37:48.637 INFO:tasks.workunit.client.0.smithi136.stdout: "ms_die_on_bug", 2024-09-12T23:37:48.637 INFO:tasks.workunit.client.0.smithi136.stdout: "ms_die_on_old_message", 2024-09-12T23:37:48.637 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_class_default_list", 2024-09-12T23:37:48.637 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_class_load_list", 2024-09-12T23:37:48.637 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_misdirected_ops", 2024-09-12T23:37:48.637 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_op_order", 2024-09-12T23:37:48.637 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_pg_log_writeout", 2024-09-12T23:37:48.637 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_shutdown", 2024-09-12T23:37:48.637 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_verify_cached_snaps", 2024-09-12T23:37:48.637 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_verify_missing_on_start", 2024-09-12T23:37:48.637 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_verify_stray_on_activate", 2024-09-12T23:37:48.638 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_deep_scrub_update_digest_min_age", 2024-09-12T23:37:48.638 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mclock_profile", 2024-09-12T23:37:48.638 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_memory_target", 2024-09-12T23:37:48.638 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_memory_target_autotune", 2024-09-12T23:37:48.638 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_op_queue", 2024-09-12T23:37:48.638 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_op_queue_cut_off", 2024-09-12T23:37:48.638 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_pool_default_erasure_code_profile", 2024-09-12T23:37:48.638 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_pool_default_pg_autoscale_mode", 2024-09-12T23:37:48.638 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_recover_clone_overlap", 2024-09-12T23:37:48.638 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_recovery_max_chunk", 2024-09-12T23:37:48.638 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_scrub_load_threshold", 2024-09-12T23:37:48.638 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_scrub_max_interval", 2024-09-12T23:37:48.638 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_shutdown_pgref_assert", 2024-09-12T23:37:48.638 INFO:tasks.workunit.client.0.smithi136.stdout: "public_network", 2024-09-12T23:37:48.638 INFO:tasks.workunit.client.0.smithi136.stdout: "rgw_cache_enabled", 2024-09-12T23:37:48.639 INFO:tasks.workunit.client.0.smithi136.stdout: "rgw_enable_ops_log", 2024-09-12T23:37:48.639 INFO:tasks.workunit.client.0.smithi136.stdout: "rgw_enable_usage_log" 2024-09-12T23:37:48.639 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:48.639 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.639 INFO:tasks.workunit.client.0.smithi136.stdout: "contact": null, 2024-09-12T23:37:48.639 INFO:tasks.workunit.client.0.smithi136.stdout: "crashes": [], 2024-09-12T23:37:48.639 INFO:tasks.workunit.client.0.smithi136.stdout: "created": "2024-09-12T23:23:24.585690Z", 2024-09-12T23:37:48.639 INFO:tasks.workunit.client.0.smithi136.stdout: "crush": { 2024-09-12T23:37:48.639 INFO:tasks.workunit.client.0.smithi136.stdout: "bucket_algs": { 2024-09-12T23:37:48.639 INFO:tasks.workunit.client.0.smithi136.stdout: "straw2": 3 2024-09-12T23:37:48.639 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.639 INFO:tasks.workunit.client.0.smithi136.stdout: "bucket_sizes": { 2024-09-12T23:37:48.639 INFO:tasks.workunit.client.0.smithi136.stdout: "2": 1, 2024-09-12T23:37:48.639 INFO:tasks.workunit.client.0.smithi136.stdout: "4": 2 2024-09-12T23:37:48.640 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.640 INFO:tasks.workunit.client.0.smithi136.stdout: "bucket_types": { 2024-09-12T23:37:48.640 INFO:tasks.workunit.client.0.smithi136.stdout: "1": 2, 2024-09-12T23:37:48.640 INFO:tasks.workunit.client.0.smithi136.stdout: "11": 1 2024-09-12T23:37:48.640 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.640 INFO:tasks.workunit.client.0.smithi136.stdout: "compat_weight_set": false, 2024-09-12T23:37:48.640 INFO:tasks.workunit.client.0.smithi136.stdout: "device_classes": [ 2024-09-12T23:37:48.640 INFO:tasks.workunit.client.0.smithi136.stdout: 8 2024-09-12T23:37:48.640 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:48.640 INFO:tasks.workunit.client.0.smithi136.stdout: "num_buckets": 6, 2024-09-12T23:37:48.640 INFO:tasks.workunit.client.0.smithi136.stdout: "num_devices": 8, 2024-09-12T23:37:48.640 INFO:tasks.workunit.client.0.smithi136.stdout: "num_rules": 1, 2024-09-12T23:37:48.640 INFO:tasks.workunit.client.0.smithi136.stdout: "num_types": 12, 2024-09-12T23:37:48.640 INFO:tasks.workunit.client.0.smithi136.stdout: "num_weight_sets": 0, 2024-09-12T23:37:48.640 INFO:tasks.workunit.client.0.smithi136.stdout: "tunables": { 2024-09-12T23:37:48.641 INFO:tasks.workunit.client.0.smithi136.stdout: "allowed_bucket_algs": 54, 2024-09-12T23:37:48.641 INFO:tasks.workunit.client.0.smithi136.stdout: "choose_local_fallback_tries": 0, 2024-09-12T23:37:48.641 INFO:tasks.workunit.client.0.smithi136.stdout: "choose_local_tries": 0, 2024-09-12T23:37:48.641 INFO:tasks.workunit.client.0.smithi136.stdout: "choose_total_tries": 50, 2024-09-12T23:37:48.641 INFO:tasks.workunit.client.0.smithi136.stdout: "chooseleaf_descend_once": 1, 2024-09-12T23:37:48.641 INFO:tasks.workunit.client.0.smithi136.stdout: "chooseleaf_stable": 1, 2024-09-12T23:37:48.641 INFO:tasks.workunit.client.0.smithi136.stdout: "chooseleaf_vary_r": 1, 2024-09-12T23:37:48.641 INFO:tasks.workunit.client.0.smithi136.stdout: "has_v2_rules": 0, 2024-09-12T23:37:48.641 INFO:tasks.workunit.client.0.smithi136.stdout: "has_v3_rules": 0, 2024-09-12T23:37:48.641 INFO:tasks.workunit.client.0.smithi136.stdout: "has_v4_buckets": 1, 2024-09-12T23:37:48.641 INFO:tasks.workunit.client.0.smithi136.stdout: "has_v5_rules": 0, 2024-09-12T23:37:48.641 INFO:tasks.workunit.client.0.smithi136.stdout: "legacy_tunables": 0, 2024-09-12T23:37:48.641 INFO:tasks.workunit.client.0.smithi136.stdout: "minimum_required_version": "jewel", 2024-09-12T23:37:48.641 INFO:tasks.workunit.client.0.smithi136.stdout: "optimal_tunables": 1, 2024-09-12T23:37:48.642 INFO:tasks.workunit.client.0.smithi136.stdout: "profile": "jewel", 2024-09-12T23:37:48.642 INFO:tasks.workunit.client.0.smithi136.stdout: "require_feature_tunables": 1, 2024-09-12T23:37:48.642 INFO:tasks.workunit.client.0.smithi136.stdout: "require_feature_tunables2": 1, 2024-09-12T23:37:48.642 INFO:tasks.workunit.client.0.smithi136.stdout: "require_feature_tunables3": 1, 2024-09-12T23:37:48.642 INFO:tasks.workunit.client.0.smithi136.stdout: "require_feature_tunables5": 1, 2024-09-12T23:37:48.642 INFO:tasks.workunit.client.0.smithi136.stdout: "straw_calc_version": 1 2024-09-12T23:37:48.642 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.642 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.642 INFO:tasks.workunit.client.0.smithi136.stdout: "description": "upgrade test cluster", 2024-09-12T23:37:48.642 INFO:tasks.workunit.client.0.smithi136.stdout: "fs": { 2024-09-12T23:37:48.642 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1, 2024-09-12T23:37:48.642 INFO:tasks.workunit.client.0.smithi136.stdout: "feature_flags": { 2024-09-12T23:37:48.642 INFO:tasks.workunit.client.0.smithi136.stdout: "enable_multiple": true, 2024-09-12T23:37:48.642 INFO:tasks.workunit.client.0.smithi136.stdout: "ever_enabled_multiple": true 2024-09-12T23:37:48.643 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.643 INFO:tasks.workunit.client.0.smithi136.stdout: "filesystems": [ 2024-09-12T23:37:48.643 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:48.643 INFO:tasks.workunit.client.0.smithi136.stdout: "approx_ctime": "2024-09", 2024-09-12T23:37:48.643 INFO:tasks.workunit.client.0.smithi136.stdout: "balancer_enabled": false, 2024-09-12T23:37:48.643 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.643 INFO:tasks.workunit.client.0.smithi136.stdout: "cached_caps": 0, 2024-09-12T23:37:48.643 INFO:tasks.workunit.client.0.smithi136.stdout: "cached_dns": 10, 2024-09-12T23:37:48.643 INFO:tasks.workunit.client.0.smithi136.stdout: "cached_inos": 13, 2024-09-12T23:37:48.643 INFO:tasks.workunit.client.0.smithi136.stdout: "cached_subtrees": 2, 2024-09-12T23:37:48.643 INFO:tasks.workunit.client.0.smithi136.stdout: "ever_allowed_features": 0, 2024-09-12T23:37:48.643 INFO:tasks.workunit.client.0.smithi136.stdout: "explicitly_allowed_features": 0, 2024-09-12T23:37:48.643 INFO:tasks.workunit.client.0.smithi136.stdout: "files": 0, 2024-09-12T23:37:48.643 INFO:tasks.workunit.client.0.smithi136.stdout: "max_mds": 1, 2024-09-12T23:37:48.644 INFO:tasks.workunit.client.0.smithi136.stdout: "num_data_pools": 1, 2024-09-12T23:37:48.644 INFO:tasks.workunit.client.0.smithi136.stdout: "num_in": 1, 2024-09-12T23:37:48.644 INFO:tasks.workunit.client.0.smithi136.stdout: "num_mds": 1, 2024-09-12T23:37:48.644 INFO:tasks.workunit.client.0.smithi136.stdout: "num_sessions": 0, 2024-09-12T23:37:48.644 INFO:tasks.workunit.client.0.smithi136.stdout: "num_standby_replay": 0, 2024-09-12T23:37:48.644 INFO:tasks.workunit.client.0.smithi136.stdout: "num_up": 1, 2024-09-12T23:37:48.644 INFO:tasks.workunit.client.0.smithi136.stdout: "snaps": 0, 2024-09-12T23:37:48.644 INFO:tasks.workunit.client.0.smithi136.stdout: "standby_count_wanted": 0 2024-09-12T23:37:48.644 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.644 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:48.644 INFO:tasks.workunit.client.0.smithi136.stdout: "num_standby_mds": 1, 2024-09-12T23:37:48.644 INFO:tasks.workunit.client.0.smithi136.stdout: "total_num_mds": 2 2024-09-12T23:37:48.644 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.645 INFO:tasks.workunit.client.0.smithi136.stdout: "heap_stats": { 2024-09-12T23:37:48.645 INFO:tasks.workunit.client.0.smithi136.stdout: "mds": { 2024-09-12T23:37:48.645 INFO:tasks.workunit.client.0.smithi136.stdout: "mds.4585d541b331f263094839d9f58dd50a20d3f517": { 2024-09-12T23:37:48.645 INFO:tasks.workunit.client.0.smithi136.stdout: "actual_memory_used": 20627456, 2024-09-12T23:37:48.645 INFO:tasks.workunit.client.0.smithi136.stdout: "central_cache_freelist": 406296, 2024-09-12T23:37:48.645 INFO:tasks.workunit.client.0.smithi136.stdout: "malloc_metadata": 2621440, 2024-09-12T23:37:48.645 INFO:tasks.workunit.client.0.smithi136.stdout: "page_heap_freelist": 24576, 2024-09-12T23:37:48.645 INFO:tasks.workunit.client.0.smithi136.stdout: "released_to_os": 557056, 2024-09-12T23:37:48.645 INFO:tasks.workunit.client.0.smithi136.stdout: "spans_in_use": 429, 2024-09-12T23:37:48.645 INFO:tasks.workunit.client.0.smithi136.stdout: "tcmalloc_page_size": 8192, 2024-09-12T23:37:48.645 INFO:tasks.workunit.client.0.smithi136.stdout: "thread_cache_freelists": 1282040, 2024-09-12T23:37:48.645 INFO:tasks.workunit.client.0.smithi136.stdout: "thread_heaps_in_use": 20, 2024-09-12T23:37:48.645 INFO:tasks.workunit.client.0.smithi136.stdout: "transfer_cache_freelist": 1124864, 2024-09-12T23:37:48.645 INFO:tasks.workunit.client.0.smithi136.stdout: "use_by_application": 15168240, 2024-09-12T23:37:48.645 INFO:tasks.workunit.client.0.smithi136.stdout: "virtual_address_space_used": 21184512 2024-09-12T23:37:48.646 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.646 INFO:tasks.workunit.client.0.smithi136.stdout: "mds.48b2ee34c5040a1abbee529ca5897a6d3827a23a": { 2024-09-12T23:37:48.646 INFO:tasks.workunit.client.0.smithi136.stdout: "actual_memory_used": 20135936, 2024-09-12T23:37:48.646 INFO:tasks.workunit.client.0.smithi136.stdout: "central_cache_freelist": 365088, 2024-09-12T23:37:48.646 INFO:tasks.workunit.client.0.smithi136.stdout: "malloc_metadata": 2621440, 2024-09-12T23:37:48.646 INFO:tasks.workunit.client.0.smithi136.stdout: "page_heap_freelist": 811008, 2024-09-12T23:37:48.646 INFO:tasks.workunit.client.0.smithi136.stdout: "released_to_os": 0, 2024-09-12T23:37:48.646 INFO:tasks.workunit.client.0.smithi136.stdout: "spans_in_use": 351, 2024-09-12T23:37:48.646 INFO:tasks.workunit.client.0.smithi136.stdout: "tcmalloc_page_size": 8192, 2024-09-12T23:37:48.646 INFO:tasks.workunit.client.0.smithi136.stdout: "thread_cache_freelists": 872592, 2024-09-12T23:37:48.646 INFO:tasks.workunit.client.0.smithi136.stdout: "thread_heaps_in_use": 12, 2024-09-12T23:37:48.646 INFO:tasks.workunit.client.0.smithi136.stdout: "transfer_cache_freelist": 769024, 2024-09-12T23:37:48.646 INFO:tasks.workunit.client.0.smithi136.stdout: "use_by_application": 14696784, 2024-09-12T23:37:48.646 INFO:tasks.workunit.client.0.smithi136.stdout: "virtual_address_space_used": 20135936 2024-09-12T23:37:48.646 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.647 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.647 INFO:tasks.workunit.client.0.smithi136.stdout: "mon": { 2024-09-12T23:37:48.647 INFO:tasks.workunit.client.0.smithi136.stdout: "mon.0027af889fd200cedbdb75d916e7558e544c1ffa": { 2024-09-12T23:37:48.647 INFO:tasks.workunit.client.0.smithi136.stdout: "actual_memory_used": 39968768, 2024-09-12T23:37:48.647 INFO:tasks.workunit.client.0.smithi136.stdout: "central_cache_freelist": 1720776, 2024-09-12T23:37:48.647 INFO:tasks.workunit.client.0.smithi136.stdout: "malloc_metadata": 2621440, 2024-09-12T23:37:48.647 INFO:tasks.workunit.client.0.smithi136.stdout: "page_heap_freelist": 1892352, 2024-09-12T23:37:48.647 INFO:tasks.workunit.client.0.smithi136.stdout: "released_to_os": 3342336, 2024-09-12T23:37:48.647 INFO:tasks.workunit.client.0.smithi136.stdout: "spans_in_use": 1219, 2024-09-12T23:37:48.647 INFO:tasks.workunit.client.0.smithi136.stdout: "tcmalloc_page_size": 8192, 2024-09-12T23:37:48.647 INFO:tasks.workunit.client.0.smithi136.stdout: "thread_cache_freelists": 7749320, 2024-09-12T23:37:48.647 INFO:tasks.workunit.client.0.smithi136.stdout: "thread_heaps_in_use": 21, 2024-09-12T23:37:48.647 INFO:tasks.workunit.client.0.smithi136.stdout: "transfer_cache_freelist": 1193984, 2024-09-12T23:37:48.648 INFO:tasks.workunit.client.0.smithi136.stdout: "use_by_application": 24790896, 2024-09-12T23:37:48.648 INFO:tasks.workunit.client.0.smithi136.stdout: "virtual_address_space_used": 43311104 2024-09-12T23:37:48.648 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.648 INFO:tasks.workunit.client.0.smithi136.stdout: "mon.4585d541b331f263094839d9f58dd50a20d3f517": { 2024-09-12T23:37:48.648 INFO:tasks.workunit.client.0.smithi136.stdout: "actual_memory_used": 38518784, 2024-09-12T23:37:48.648 INFO:tasks.workunit.client.0.smithi136.stdout: "central_cache_freelist": 1225128, 2024-09-12T23:37:48.648 INFO:tasks.workunit.client.0.smithi136.stdout: "malloc_metadata": 2621440, 2024-09-12T23:37:48.648 INFO:tasks.workunit.client.0.smithi136.stdout: "page_heap_freelist": 0, 2024-09-12T23:37:48.648 INFO:tasks.workunit.client.0.smithi136.stdout: "released_to_os": 5849088, 2024-09-12T23:37:48.648 INFO:tasks.workunit.client.0.smithi136.stdout: "spans_in_use": 1201, 2024-09-12T23:37:48.648 INFO:tasks.workunit.client.0.smithi136.stdout: "tcmalloc_page_size": 8192, 2024-09-12T23:37:48.648 INFO:tasks.workunit.client.0.smithi136.stdout: "thread_cache_freelists": 8448216, 2024-09-12T23:37:48.648 INFO:tasks.workunit.client.0.smithi136.stdout: "thread_heaps_in_use": 21, 2024-09-12T23:37:48.648 INFO:tasks.workunit.client.0.smithi136.stdout: "transfer_cache_freelist": 1663488, 2024-09-12T23:37:48.649 INFO:tasks.workunit.client.0.smithi136.stdout: "use_by_application": 24560512, 2024-09-12T23:37:48.649 INFO:tasks.workunit.client.0.smithi136.stdout: "virtual_address_space_used": 44367872 2024-09-12T23:37:48.649 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.649 INFO:tasks.workunit.client.0.smithi136.stdout: "mon.48b2ee34c5040a1abbee529ca5897a6d3827a23a": { 2024-09-12T23:37:48.649 INFO:tasks.workunit.client.0.smithi136.stdout: "actual_memory_used": 47964160, 2024-09-12T23:37:48.649 INFO:tasks.workunit.client.0.smithi136.stdout: "central_cache_freelist": 1822376, 2024-09-12T23:37:48.649 INFO:tasks.workunit.client.0.smithi136.stdout: "malloc_metadata": 2621440, 2024-09-12T23:37:48.649 INFO:tasks.workunit.client.0.smithi136.stdout: "page_heap_freelist": 712704, 2024-09-12T23:37:48.649 INFO:tasks.workunit.client.0.smithi136.stdout: "released_to_os": 2605056, 2024-09-12T23:37:48.649 INFO:tasks.workunit.client.0.smithi136.stdout: "spans_in_use": 1753, 2024-09-12T23:37:48.649 INFO:tasks.workunit.client.0.smithi136.stdout: "tcmalloc_page_size": 8192, 2024-09-12T23:37:48.649 INFO:tasks.workunit.client.0.smithi136.stdout: "thread_cache_freelists": 10594080, 2024-09-12T23:37:48.649 INFO:tasks.workunit.client.0.smithi136.stdout: "thread_heaps_in_use": 23, 2024-09-12T23:37:48.650 INFO:tasks.workunit.client.0.smithi136.stdout: "transfer_cache_freelist": 2705408, 2024-09-12T23:37:48.650 INFO:tasks.workunit.client.0.smithi136.stdout: "use_by_application": 29508152, 2024-09-12T23:37:48.650 INFO:tasks.workunit.client.0.smithi136.stdout: "virtual_address_space_used": 50569216 2024-09-12T23:37:48.650 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.650 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.650 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:48.650 INFO:tasks.workunit.client.0.smithi136.stdout: "osd.0": { 2024-09-12T23:37:48.650 INFO:tasks.workunit.client.0.smithi136.stdout: "actual_memory_used": 277225472, 2024-09-12T23:37:48.650 INFO:tasks.workunit.client.0.smithi136.stdout: "central_cache_freelist": 716352, 2024-09-12T23:37:48.650 INFO:tasks.workunit.client.0.smithi136.stdout: "malloc_metadata": 4849664, 2024-09-12T23:37:48.650 INFO:tasks.workunit.client.0.smithi136.stdout: "page_heap_freelist": 0, 2024-09-12T23:37:48.650 INFO:tasks.workunit.client.0.smithi136.stdout: "released_to_os": 696320, 2024-09-12T23:37:48.650 INFO:tasks.workunit.client.0.smithi136.stdout: "spans_in_use": 1518, 2024-09-12T23:37:48.650 INFO:tasks.workunit.client.0.smithi136.stdout: "tcmalloc_page_size": 8192, 2024-09-12T23:37:48.651 INFO:tasks.workunit.client.0.smithi136.stdout: "thread_cache_freelists": 5286328, 2024-09-12T23:37:48.651 INFO:tasks.workunit.client.0.smithi136.stdout: "thread_heaps_in_use": 45, 2024-09-12T23:37:48.651 INFO:tasks.workunit.client.0.smithi136.stdout: "transfer_cache_freelist": 3107840, 2024-09-12T23:37:48.651 INFO:tasks.workunit.client.0.smithi136.stdout: "use_by_application": 263265288, 2024-09-12T23:37:48.651 INFO:tasks.workunit.client.0.smithi136.stdout: "virtual_address_space_used": 277921792 2024-09-12T23:37:48.651 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.651 INFO:tasks.workunit.client.0.smithi136.stdout: "osd.1": { 2024-09-12T23:37:48.651 INFO:tasks.workunit.client.0.smithi136.stdout: "actual_memory_used": 274178048, 2024-09-12T23:37:48.651 INFO:tasks.workunit.client.0.smithi136.stdout: "central_cache_freelist": 743152, 2024-09-12T23:37:48.651 INFO:tasks.workunit.client.0.smithi136.stdout: "malloc_metadata": 2752512, 2024-09-12T23:37:48.651 INFO:tasks.workunit.client.0.smithi136.stdout: "page_heap_freelist": 0, 2024-09-12T23:37:48.651 INFO:tasks.workunit.client.0.smithi136.stdout: "released_to_os": 598016, 2024-09-12T23:37:48.651 INFO:tasks.workunit.client.0.smithi136.stdout: "spans_in_use": 1455, 2024-09-12T23:37:48.651 INFO:tasks.workunit.client.0.smithi136.stdout: "tcmalloc_page_size": 8192, 2024-09-12T23:37:48.651 INFO:tasks.workunit.client.0.smithi136.stdout: "thread_cache_freelists": 5593832, 2024-09-12T23:37:48.651 INFO:tasks.workunit.client.0.smithi136.stdout: "thread_heaps_in_use": 47, 2024-09-12T23:37:48.652 INFO:tasks.workunit.client.0.smithi136.stdout: "transfer_cache_freelist": 2247424, 2024-09-12T23:37:48.652 INFO:tasks.workunit.client.0.smithi136.stdout: "use_by_application": 262841128, 2024-09-12T23:37:48.652 INFO:tasks.workunit.client.0.smithi136.stdout: "virtual_address_space_used": 274776064 2024-09-12T23:37:48.652 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.652 INFO:tasks.workunit.client.0.smithi136.stdout: "osd.2": { 2024-09-12T23:37:48.652 INFO:tasks.workunit.client.0.smithi136.stdout: "actual_memory_used": 274833408, 2024-09-12T23:37:48.652 INFO:tasks.workunit.client.0.smithi136.stdout: "central_cache_freelist": 761232, 2024-09-12T23:37:48.652 INFO:tasks.workunit.client.0.smithi136.stdout: "malloc_metadata": 4849664, 2024-09-12T23:37:48.652 INFO:tasks.workunit.client.0.smithi136.stdout: "page_heap_freelist": 0, 2024-09-12T23:37:48.652 INFO:tasks.workunit.client.0.smithi136.stdout: "released_to_os": 925696, 2024-09-12T23:37:48.652 INFO:tasks.workunit.client.0.smithi136.stdout: "spans_in_use": 1368, 2024-09-12T23:37:48.652 INFO:tasks.workunit.client.0.smithi136.stdout: "tcmalloc_page_size": 8192, 2024-09-12T23:37:48.652 INFO:tasks.workunit.client.0.smithi136.stdout: "thread_cache_freelists": 4992728, 2024-09-12T23:37:48.652 INFO:tasks.workunit.client.0.smithi136.stdout: "thread_heaps_in_use": 46, 2024-09-12T23:37:48.653 INFO:tasks.workunit.client.0.smithi136.stdout: "transfer_cache_freelist": 2522368, 2024-09-12T23:37:48.653 INFO:tasks.workunit.client.0.smithi136.stdout: "use_by_application": 261707416, 2024-09-12T23:37:48.653 INFO:tasks.workunit.client.0.smithi136.stdout: "virtual_address_space_used": 275759104 2024-09-12T23:37:48.653 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.653 INFO:tasks.workunit.client.0.smithi136.stdout: "osd.3": { 2024-09-12T23:37:48.653 INFO:tasks.workunit.client.0.smithi136.stdout: "actual_memory_used": 272195584, 2024-09-12T23:37:48.653 INFO:tasks.workunit.client.0.smithi136.stdout: "central_cache_freelist": 672080, 2024-09-12T23:37:48.653 INFO:tasks.workunit.client.0.smithi136.stdout: "malloc_metadata": 2752512, 2024-09-12T23:37:48.653 INFO:tasks.workunit.client.0.smithi136.stdout: "page_heap_freelist": 24576, 2024-09-12T23:37:48.653 INFO:tasks.workunit.client.0.smithi136.stdout: "released_to_os": 417792, 2024-09-12T23:37:48.653 INFO:tasks.workunit.client.0.smithi136.stdout: "spans_in_use": 1319, 2024-09-12T23:37:48.653 INFO:tasks.workunit.client.0.smithi136.stdout: "tcmalloc_page_size": 8192, 2024-09-12T23:37:48.653 INFO:tasks.workunit.client.0.smithi136.stdout: "thread_cache_freelists": 4434400, 2024-09-12T23:37:48.653 INFO:tasks.workunit.client.0.smithi136.stdout: "thread_heaps_in_use": 45, 2024-09-12T23:37:48.654 INFO:tasks.workunit.client.0.smithi136.stdout: "transfer_cache_freelist": 2056960, 2024-09-12T23:37:48.654 INFO:tasks.workunit.client.0.smithi136.stdout: "use_by_application": 262255056, 2024-09-12T23:37:48.654 INFO:tasks.workunit.client.0.smithi136.stdout: "virtual_address_space_used": 272613376 2024-09-12T23:37:48.654 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.654 INFO:tasks.workunit.client.0.smithi136.stdout: "osd.4": { 2024-09-12T23:37:48.654 INFO:tasks.workunit.client.0.smithi136.stdout: "actual_memory_used": 273113088, 2024-09-12T23:37:48.654 INFO:tasks.workunit.client.0.smithi136.stdout: "central_cache_freelist": 907432, 2024-09-12T23:37:48.654 INFO:tasks.workunit.client.0.smithi136.stdout: "malloc_metadata": 2752512, 2024-09-12T23:37:48.654 INFO:tasks.workunit.client.0.smithi136.stdout: "page_heap_freelist": 0, 2024-09-12T23:37:48.654 INFO:tasks.workunit.client.0.smithi136.stdout: "released_to_os": 827392, 2024-09-12T23:37:48.654 INFO:tasks.workunit.client.0.smithi136.stdout: "spans_in_use": 1307, 2024-09-12T23:37:48.654 INFO:tasks.workunit.client.0.smithi136.stdout: "tcmalloc_page_size": 8192, 2024-09-12T23:37:48.654 INFO:tasks.workunit.client.0.smithi136.stdout: "thread_cache_freelists": 4814664, 2024-09-12T23:37:48.654 INFO:tasks.workunit.client.0.smithi136.stdout: "thread_heaps_in_use": 46, 2024-09-12T23:37:48.654 INFO:tasks.workunit.client.0.smithi136.stdout: "transfer_cache_freelist": 1954048, 2024-09-12T23:37:48.654 INFO:tasks.workunit.client.0.smithi136.stdout: "use_by_application": 262684432, 2024-09-12T23:37:48.655 INFO:tasks.workunit.client.0.smithi136.stdout: "virtual_address_space_used": 273940480 2024-09-12T23:37:48.655 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.655 INFO:tasks.workunit.client.0.smithi136.stdout: "osd.5": { 2024-09-12T23:37:48.655 INFO:tasks.workunit.client.0.smithi136.stdout: "actual_memory_used": 272556032, 2024-09-12T23:37:48.655 INFO:tasks.workunit.client.0.smithi136.stdout: "central_cache_freelist": 630968, 2024-09-12T23:37:48.655 INFO:tasks.workunit.client.0.smithi136.stdout: "malloc_metadata": 2752512, 2024-09-12T23:37:48.655 INFO:tasks.workunit.client.0.smithi136.stdout: "page_heap_freelist": 0, 2024-09-12T23:37:48.655 INFO:tasks.workunit.client.0.smithi136.stdout: "released_to_os": 122880, 2024-09-12T23:37:48.655 INFO:tasks.workunit.client.0.smithi136.stdout: "spans_in_use": 1317, 2024-09-12T23:37:48.655 INFO:tasks.workunit.client.0.smithi136.stdout: "tcmalloc_page_size": 8192, 2024-09-12T23:37:48.655 INFO:tasks.workunit.client.0.smithi136.stdout: "thread_cache_freelists": 4658088, 2024-09-12T23:37:48.655 INFO:tasks.workunit.client.0.smithi136.stdout: "thread_heaps_in_use": 46, 2024-09-12T23:37:48.656 INFO:tasks.workunit.client.0.smithi136.stdout: "transfer_cache_freelist": 2168320, 2024-09-12T23:37:48.656 INFO:tasks.workunit.client.0.smithi136.stdout: "use_by_application": 262346144, 2024-09-12T23:37:48.656 INFO:tasks.workunit.client.0.smithi136.stdout: "virtual_address_space_used": 272678912 2024-09-12T23:37:48.656 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.656 INFO:tasks.workunit.client.0.smithi136.stdout: "osd.6": { 2024-09-12T23:37:48.656 INFO:tasks.workunit.client.0.smithi136.stdout: "actual_memory_used": 272678912, 2024-09-12T23:37:48.656 INFO:tasks.workunit.client.0.smithi136.stdout: "central_cache_freelist": 946456, 2024-09-12T23:37:48.656 INFO:tasks.workunit.client.0.smithi136.stdout: "malloc_metadata": 2752512, 2024-09-12T23:37:48.656 INFO:tasks.workunit.client.0.smithi136.stdout: "page_heap_freelist": 24576, 2024-09-12T23:37:48.656 INFO:tasks.workunit.client.0.smithi136.stdout: "released_to_os": 1261568, 2024-09-12T23:37:48.656 INFO:tasks.workunit.client.0.smithi136.stdout: "spans_in_use": 1294, 2024-09-12T23:37:48.656 INFO:tasks.workunit.client.0.smithi136.stdout: "tcmalloc_page_size": 8192, 2024-09-12T23:37:48.656 INFO:tasks.workunit.client.0.smithi136.stdout: "thread_cache_freelists": 4566936, 2024-09-12T23:37:48.656 INFO:tasks.workunit.client.0.smithi136.stdout: "thread_heaps_in_use": 46, 2024-09-12T23:37:48.656 INFO:tasks.workunit.client.0.smithi136.stdout: "transfer_cache_freelist": 1989120, 2024-09-12T23:37:48.657 INFO:tasks.workunit.client.0.smithi136.stdout: "use_by_application": 262399312, 2024-09-12T23:37:48.657 INFO:tasks.workunit.client.0.smithi136.stdout: "virtual_address_space_used": 273940480 2024-09-12T23:37:48.657 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.657 INFO:tasks.workunit.client.0.smithi136.stdout: "osd.7": { 2024-09-12T23:37:48.657 INFO:tasks.workunit.client.0.smithi136.stdout: "actual_memory_used": 273768448, 2024-09-12T23:37:48.657 INFO:tasks.workunit.client.0.smithi136.stdout: "central_cache_freelist": 940024, 2024-09-12T23:37:48.657 INFO:tasks.workunit.client.0.smithi136.stdout: "malloc_metadata": 2752512, 2024-09-12T23:37:48.657 INFO:tasks.workunit.client.0.smithi136.stdout: "page_heap_freelist": 24576, 2024-09-12T23:37:48.657 INFO:tasks.workunit.client.0.smithi136.stdout: "released_to_os": 106496, 2024-09-12T23:37:48.657 INFO:tasks.workunit.client.0.smithi136.stdout: "spans_in_use": 1347, 2024-09-12T23:37:48.657 INFO:tasks.workunit.client.0.smithi136.stdout: "tcmalloc_page_size": 8192, 2024-09-12T23:37:48.657 INFO:tasks.workunit.client.0.smithi136.stdout: "thread_cache_freelists": 5177896, 2024-09-12T23:37:48.658 INFO:tasks.workunit.client.0.smithi136.stdout: "thread_heaps_in_use": 46, 2024-09-12T23:37:48.658 INFO:tasks.workunit.client.0.smithi136.stdout: "transfer_cache_freelist": 2089728, 2024-09-12T23:37:48.658 INFO:tasks.workunit.client.0.smithi136.stdout: "use_by_application": 262783712, 2024-09-12T23:37:48.658 INFO:tasks.workunit.client.0.smithi136.stdout: "virtual_address_space_used": 273874944 2024-09-12T23:37:48.658 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.658 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.658 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.658 INFO:tasks.workunit.client.0.smithi136.stdout: "hosts": { 2024-09-12T23:37:48.658 INFO:tasks.workunit.client.0.smithi136.stdout: "num": 2, 2024-09-12T23:37:48.658 INFO:tasks.workunit.client.0.smithi136.stdout: "num_with_mds": 2, 2024-09-12T23:37:48.658 INFO:tasks.workunit.client.0.smithi136.stdout: "num_with_mgr": 2, 2024-09-12T23:37:48.658 INFO:tasks.workunit.client.0.smithi136.stdout: "num_with_mon": 2, 2024-09-12T23:37:48.658 INFO:tasks.workunit.client.0.smithi136.stdout: "num_with_osd": 2 2024-09-12T23:37:48.658 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.658 INFO:tasks.workunit.client.0.smithi136.stdout: "io_rate": { 2024-09-12T23:37:48.658 INFO:tasks.workunit.client.0.smithi136.stdout: "pg_stats_delta": { 2024-09-12T23:37:48.659 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": 0, 2024-09-12T23:37:48.659 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:48.659 INFO:tasks.workunit.client.0.smithi136.stdout: "num_store_stats": 0, 2024-09-12T23:37:48.659 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:48.659 INFO:tasks.workunit.client.0.smithi136.stdout: "stamp_delta": "12.003820", 2024-09-12T23:37:48.659 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:48.659 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:48.659 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:48.659 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:48.659 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:48.660 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:48.660 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:48.660 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:48.660 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:48.660 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:48.660 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:48.660 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:48.660 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:48.660 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:48.660 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:48.660 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:48.660 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:48.660 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:48.660 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:48.661 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:48.661 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:48.661 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:48.661 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:48.661 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:48.661 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:48.661 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:48.661 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:48.661 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:48.661 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:48.661 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:48.661 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:48.661 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:48.662 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:48.662 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:48.662 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:48.662 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:48.662 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:48.662 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:48.662 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:48.662 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:48.662 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:48.662 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.662 INFO:tasks.workunit.client.0.smithi136.stdout: "store_stats": { 2024-09-12T23:37:48.662 INFO:tasks.workunit.client.0.smithi136.stdout: "allocated": 0, 2024-09-12T23:37:48.662 INFO:tasks.workunit.client.0.smithi136.stdout: "available": 0, 2024-09-12T23:37:48.663 INFO:tasks.workunit.client.0.smithi136.stdout: "data_compressed": 0, 2024-09-12T23:37:48.663 INFO:tasks.workunit.client.0.smithi136.stdout: "data_compressed_allocated": 0, 2024-09-12T23:37:48.663 INFO:tasks.workunit.client.0.smithi136.stdout: "data_compressed_original": 0, 2024-09-12T23:37:48.663 INFO:tasks.workunit.client.0.smithi136.stdout: "data_stored": 0, 2024-09-12T23:37:48.663 INFO:tasks.workunit.client.0.smithi136.stdout: "internal_metadata": 0, 2024-09-12T23:37:48.663 INFO:tasks.workunit.client.0.smithi136.stdout: "internally_reserved": 0, 2024-09-12T23:37:48.663 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_allocated": 0, 2024-09-12T23:37:48.663 INFO:tasks.workunit.client.0.smithi136.stdout: "total": 0 2024-09-12T23:37:48.663 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.663 INFO:tasks.workunit.client.0.smithi136.stdout: "up": 0 2024-09-12T23:37:48.663 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.663 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.663 INFO:tasks.workunit.client.0.smithi136.stdout: "leaderboard": false, 2024-09-12T23:37:48.663 INFO:tasks.workunit.client.0.smithi136.stdout: "license": "sharing-1-0", 2024-09-12T23:37:48.663 INFO:tasks.workunit.client.0.smithi136.stdout: "mempool": { 2024-09-12T23:37:48.664 INFO:tasks.workunit.client.0.smithi136.stdout: "mds": { 2024-09-12T23:37:48.664 INFO:tasks.workunit.client.0.smithi136.stdout: "mds.4585d541b331f263094839d9f58dd50a20d3f517": { 2024-09-12T23:37:48.664 INFO:tasks.workunit.client.0.smithi136.stdout: "bloom_filter": { 2024-09-12T23:37:48.664 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.664 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.664 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.664 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs": { 2024-09-12T23:37:48.664 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.664 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.664 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.664 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs_file_reader": { 2024-09-12T23:37:48.664 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.664 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.664 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.665 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs_file_writer": { 2024-09-12T23:37:48.665 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.665 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.665 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.665 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Blob": { 2024-09-12T23:37:48.665 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.665 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.665 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.665 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Buffer": { 2024-09-12T23:37:48.665 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.665 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.665 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.665 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Extent": { 2024-09-12T23:37:48.665 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.665 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.666 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.666 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_SharedBlob": { 2024-09-12T23:37:48.666 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.666 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.666 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.666 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_alloc": { 2024-09-12T23:37:48.666 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.666 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.666 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.666 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_data": { 2024-09-12T23:37:48.666 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.666 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.667 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.667 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_meta": { 2024-09-12T23:37:48.667 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.667 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.667 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.667 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_onode": { 2024-09-12T23:37:48.667 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.667 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.667 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.667 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_other": { 2024-09-12T23:37:48.667 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.667 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.667 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.668 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_fsck": { 2024-09-12T23:37:48.668 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.668 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.668 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.668 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_inline_bl": { 2024-09-12T23:37:48.668 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.668 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.668 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.668 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_txc": { 2024-09-12T23:37:48.668 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.668 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.668 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.668 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_writing": { 2024-09-12T23:37:48.669 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.669 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.669 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.669 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_writing_deferred": { 2024-09-12T23:37:48.669 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.669 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.669 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.669 INFO:tasks.workunit.client.0.smithi136.stdout: "buffer_anon": { 2024-09-12T23:37:48.669 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 222471, 2024-09-12T23:37:48.669 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 72 2024-09-12T23:37:48.669 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.669 INFO:tasks.workunit.client.0.smithi136.stdout: "buffer_meta": { 2024-09-12T23:37:48.669 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.670 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.670 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.670 INFO:tasks.workunit.client.0.smithi136.stdout: "mds_co": { 2024-09-12T23:37:48.670 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 50488, 2024-09-12T23:37:48.670 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 208 2024-09-12T23:37:48.670 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.670 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:48.670 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.670 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.670 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.670 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mapbl": { 2024-09-12T23:37:48.670 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.670 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.671 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.671 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_pglog": { 2024-09-12T23:37:48.671 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.671 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.671 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.671 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap": { 2024-09-12T23:37:48.671 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 8128, 2024-09-12T23:37:48.671 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 139 2024-09-12T23:37:48.671 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.671 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap_mapping": { 2024-09-12T23:37:48.671 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.671 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.671 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.671 INFO:tasks.workunit.client.0.smithi136.stdout: "pgmap": { 2024-09-12T23:37:48.672 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.672 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.672 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.672 INFO:tasks.workunit.client.0.smithi136.stdout: "unittest_1": { 2024-09-12T23:37:48.672 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.672 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.672 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.672 INFO:tasks.workunit.client.0.smithi136.stdout: "unittest_2": { 2024-09-12T23:37:48.672 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.672 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.672 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.672 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.672 INFO:tasks.workunit.client.0.smithi136.stdout: "mds.48b2ee34c5040a1abbee529ca5897a6d3827a23a": { 2024-09-12T23:37:48.672 INFO:tasks.workunit.client.0.smithi136.stdout: "bloom_filter": { 2024-09-12T23:37:48.673 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.673 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.673 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.673 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs": { 2024-09-12T23:37:48.673 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.673 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.673 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.673 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs_file_reader": { 2024-09-12T23:37:48.673 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.673 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.673 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.673 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs_file_writer": { 2024-09-12T23:37:48.673 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.673 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.674 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.674 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Blob": { 2024-09-12T23:37:48.674 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.674 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.674 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.674 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Buffer": { 2024-09-12T23:37:48.674 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.674 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.674 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.674 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Extent": { 2024-09-12T23:37:48.674 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.674 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.674 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.675 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_SharedBlob": { 2024-09-12T23:37:48.675 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.675 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.675 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.675 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_alloc": { 2024-09-12T23:37:48.675 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.675 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.675 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.675 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_data": { 2024-09-12T23:37:48.675 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.675 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.675 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.675 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_meta": { 2024-09-12T23:37:48.676 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.676 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.676 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.676 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_onode": { 2024-09-12T23:37:48.676 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.676 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.676 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.676 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_other": { 2024-09-12T23:37:48.676 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.676 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.676 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.676 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_fsck": { 2024-09-12T23:37:48.676 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.676 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.677 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.677 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_inline_bl": { 2024-09-12T23:37:48.677 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.677 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.677 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.677 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_txc": { 2024-09-12T23:37:48.677 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.677 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.677 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.677 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_writing": { 2024-09-12T23:37:48.677 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.677 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.678 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.678 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_writing_deferred": { 2024-09-12T23:37:48.678 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.678 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.678 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.678 INFO:tasks.workunit.client.0.smithi136.stdout: "buffer_anon": { 2024-09-12T23:37:48.678 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 150088, 2024-09-12T23:37:48.678 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 33 2024-09-12T23:37:48.678 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.678 INFO:tasks.workunit.client.0.smithi136.stdout: "buffer_meta": { 2024-09-12T23:37:48.678 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.678 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.678 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.678 INFO:tasks.workunit.client.0.smithi136.stdout: "mds_co": { 2024-09-12T23:37:48.678 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 856, 2024-09-12T23:37:48.679 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 2 2024-09-12T23:37:48.679 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.679 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:48.679 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.679 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.679 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.679 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mapbl": { 2024-09-12T23:37:48.679 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.679 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.679 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.679 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_pglog": { 2024-09-12T23:37:48.679 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.679 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.679 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.680 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap": { 2024-09-12T23:37:48.680 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.680 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.680 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.680 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap_mapping": { 2024-09-12T23:37:48.680 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.680 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.680 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.680 INFO:tasks.workunit.client.0.smithi136.stdout: "pgmap": { 2024-09-12T23:37:48.680 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.680 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.680 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.680 INFO:tasks.workunit.client.0.smithi136.stdout: "unittest_1": { 2024-09-12T23:37:48.681 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.681 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.681 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.681 INFO:tasks.workunit.client.0.smithi136.stdout: "unittest_2": { 2024-09-12T23:37:48.681 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.681 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.681 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.681 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.681 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.681 INFO:tasks.workunit.client.0.smithi136.stdout: "mon": { 2024-09-12T23:37:48.681 INFO:tasks.workunit.client.0.smithi136.stdout: "mon.0027af889fd200cedbdb75d916e7558e544c1ffa": { 2024-09-12T23:37:48.681 INFO:tasks.workunit.client.0.smithi136.stdout: "bloom_filter": { 2024-09-12T23:37:48.681 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.681 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.682 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.682 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs": { 2024-09-12T23:37:48.682 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.682 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.682 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.682 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs_file_reader": { 2024-09-12T23:37:48.682 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.682 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.684 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.684 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs_file_writer": { 2024-09-12T23:37:48.684 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.684 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.684 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.684 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Blob": { 2024-09-12T23:37:48.684 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.684 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.684 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.684 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Buffer": { 2024-09-12T23:37:48.684 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.684 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.685 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.685 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Extent": { 2024-09-12T23:37:48.685 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.685 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.685 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.685 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_SharedBlob": { 2024-09-12T23:37:48.685 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.685 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.685 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.685 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_alloc": { 2024-09-12T23:37:48.685 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.685 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.685 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.686 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_data": { 2024-09-12T23:37:48.686 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.686 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.686 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.686 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_meta": { 2024-09-12T23:37:48.686 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.686 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.686 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.686 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_onode": { 2024-09-12T23:37:48.686 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.686 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.686 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.686 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_other": { 2024-09-12T23:37:48.687 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.687 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.687 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.687 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_fsck": { 2024-09-12T23:37:48.687 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.687 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.687 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.687 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_inline_bl": { 2024-09-12T23:37:48.687 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.687 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.687 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.687 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_txc": { 2024-09-12T23:37:48.687 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.688 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.688 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.688 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_writing": { 2024-09-12T23:37:48.688 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.688 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.688 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.688 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_writing_deferred": { 2024-09-12T23:37:48.688 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.688 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.688 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.688 INFO:tasks.workunit.client.0.smithi136.stdout: "buffer_anon": { 2024-09-12T23:37:48.688 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 1096719, 2024-09-12T23:37:48.688 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 361 2024-09-12T23:37:48.689 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.689 INFO:tasks.workunit.client.0.smithi136.stdout: "buffer_meta": { 2024-09-12T23:37:48.689 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 176, 2024-09-12T23:37:48.689 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 2 2024-09-12T23:37:48.689 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.689 INFO:tasks.workunit.client.0.smithi136.stdout: "mds_co": { 2024-09-12T23:37:48.689 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.689 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.689 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.689 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:48.689 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.689 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.689 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.689 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mapbl": { 2024-09-12T23:37:48.690 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.690 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.690 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.690 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_pglog": { 2024-09-12T23:37:48.690 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.690 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.690 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.690 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap": { 2024-09-12T23:37:48.690 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 6776, 2024-09-12T23:37:48.690 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 138 2024-09-12T23:37:48.690 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.690 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap_mapping": { 2024-09-12T23:37:48.690 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 12744, 2024-09-12T23:37:48.690 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 1546 2024-09-12T23:37:48.690 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.691 INFO:tasks.workunit.client.0.smithi136.stdout: "pgmap": { 2024-09-12T23:37:48.691 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 5312, 2024-09-12T23:37:48.691 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 103 2024-09-12T23:37:48.691 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.691 INFO:tasks.workunit.client.0.smithi136.stdout: "unittest_1": { 2024-09-12T23:37:48.691 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.691 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.691 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.691 INFO:tasks.workunit.client.0.smithi136.stdout: "unittest_2": { 2024-09-12T23:37:48.691 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.691 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.691 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.692 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.692 INFO:tasks.workunit.client.0.smithi136.stdout: "mon.4585d541b331f263094839d9f58dd50a20d3f517": { 2024-09-12T23:37:48.692 INFO:tasks.workunit.client.0.smithi136.stdout: "bloom_filter": { 2024-09-12T23:37:48.692 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.692 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.692 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.692 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs": { 2024-09-12T23:37:48.692 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.692 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.692 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.692 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs_file_reader": { 2024-09-12T23:37:48.692 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.692 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.693 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.693 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs_file_writer": { 2024-09-12T23:37:48.693 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.693 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.693 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.693 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Blob": { 2024-09-12T23:37:48.693 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.693 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.693 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.693 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Buffer": { 2024-09-12T23:37:48.693 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.693 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.693 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.693 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Extent": { 2024-09-12T23:37:48.694 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.694 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.694 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.694 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_SharedBlob": { 2024-09-12T23:37:48.694 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.694 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.694 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.694 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_alloc": { 2024-09-12T23:37:48.694 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.694 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.694 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.694 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_data": { 2024-09-12T23:37:48.694 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.694 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.695 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.695 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_meta": { 2024-09-12T23:37:48.695 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.695 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.695 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.695 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_onode": { 2024-09-12T23:37:48.695 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.695 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.695 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.695 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_other": { 2024-09-12T23:37:48.695 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.695 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.695 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.695 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_fsck": { 2024-09-12T23:37:48.696 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.696 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.696 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.696 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_inline_bl": { 2024-09-12T23:37:48.696 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.696 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.696 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.696 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_txc": { 2024-09-12T23:37:48.696 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.696 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.696 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.697 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_writing": { 2024-09-12T23:37:48.697 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.697 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.697 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.697 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_writing_deferred": { 2024-09-12T23:37:48.697 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.697 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.697 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.698 INFO:tasks.workunit.client.0.smithi136.stdout: "buffer_anon": { 2024-09-12T23:37:48.698 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 1087743, 2024-09-12T23:37:48.698 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 345 2024-09-12T23:37:48.698 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.698 INFO:tasks.workunit.client.0.smithi136.stdout: "buffer_meta": { 2024-09-12T23:37:48.698 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 176, 2024-09-12T23:37:48.698 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 2 2024-09-12T23:37:48.698 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.698 INFO:tasks.workunit.client.0.smithi136.stdout: "mds_co": { 2024-09-12T23:37:48.698 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.698 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.698 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.698 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:48.699 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.699 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.699 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.699 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mapbl": { 2024-09-12T23:37:48.699 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.699 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.699 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.699 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_pglog": { 2024-09-12T23:37:48.699 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.699 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.699 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.699 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap": { 2024-09-12T23:37:48.699 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 6776, 2024-09-12T23:37:48.699 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 138 2024-09-12T23:37:48.700 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.700 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap_mapping": { 2024-09-12T23:37:48.700 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 12744, 2024-09-12T23:37:48.700 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 1546 2024-09-12T23:37:48.700 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.700 INFO:tasks.workunit.client.0.smithi136.stdout: "pgmap": { 2024-09-12T23:37:48.700 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 5312, 2024-09-12T23:37:48.700 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 103 2024-09-12T23:37:48.700 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.700 INFO:tasks.workunit.client.0.smithi136.stdout: "unittest_1": { 2024-09-12T23:37:48.700 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.700 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.700 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.700 INFO:tasks.workunit.client.0.smithi136.stdout: "unittest_2": { 2024-09-12T23:37:48.701 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.701 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.701 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.701 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.701 INFO:tasks.workunit.client.0.smithi136.stdout: "mon.48b2ee34c5040a1abbee529ca5897a6d3827a23a": { 2024-09-12T23:37:48.701 INFO:tasks.workunit.client.0.smithi136.stdout: "bloom_filter": { 2024-09-12T23:37:48.701 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.701 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.701 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.701 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs": { 2024-09-12T23:37:48.701 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.701 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.701 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.701 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs_file_reader": { 2024-09-12T23:37:48.702 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.702 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.702 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.702 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs_file_writer": { 2024-09-12T23:37:48.702 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.702 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.702 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.702 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Blob": { 2024-09-12T23:37:48.702 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.702 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.702 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.702 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Buffer": { 2024-09-12T23:37:48.702 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.702 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.703 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.703 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Extent": { 2024-09-12T23:37:48.703 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.703 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.703 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.703 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_SharedBlob": { 2024-09-12T23:37:48.703 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.703 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.703 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.703 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_alloc": { 2024-09-12T23:37:48.703 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.703 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.703 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.704 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_data": { 2024-09-12T23:37:48.704 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.704 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.704 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.704 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_meta": { 2024-09-12T23:37:48.704 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.704 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.704 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.704 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_onode": { 2024-09-12T23:37:48.704 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.704 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.704 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.704 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_other": { 2024-09-12T23:37:48.704 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.705 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.705 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.705 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_fsck": { 2024-09-12T23:37:48.705 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.705 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.705 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.705 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_inline_bl": { 2024-09-12T23:37:48.705 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.705 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.705 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.705 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_txc": { 2024-09-12T23:37:48.705 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.705 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.705 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.706 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_writing": { 2024-09-12T23:37:48.706 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.706 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.706 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.706 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_writing_deferred": { 2024-09-12T23:37:48.706 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.706 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.706 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.706 INFO:tasks.workunit.client.0.smithi136.stdout: "buffer_anon": { 2024-09-12T23:37:48.706 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 1170703, 2024-09-12T23:37:48.706 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 354 2024-09-12T23:37:48.706 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.706 INFO:tasks.workunit.client.0.smithi136.stdout: "buffer_meta": { 2024-09-12T23:37:48.707 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 88, 2024-09-12T23:37:48.707 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 1 2024-09-12T23:37:48.707 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.707 INFO:tasks.workunit.client.0.smithi136.stdout: "mds_co": { 2024-09-12T23:37:48.707 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.707 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.707 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.707 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:48.707 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.707 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.707 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.707 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mapbl": { 2024-09-12T23:37:48.707 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.708 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.708 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.708 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_pglog": { 2024-09-12T23:37:48.708 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.708 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.708 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.708 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap": { 2024-09-12T23:37:48.708 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 6776, 2024-09-12T23:37:48.708 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 138 2024-09-12T23:37:48.708 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.708 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap_mapping": { 2024-09-12T23:37:48.708 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 12744, 2024-09-12T23:37:48.708 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 1546 2024-09-12T23:37:48.708 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.709 INFO:tasks.workunit.client.0.smithi136.stdout: "pgmap": { 2024-09-12T23:37:48.709 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 10624, 2024-09-12T23:37:48.709 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 206 2024-09-12T23:37:48.709 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.709 INFO:tasks.workunit.client.0.smithi136.stdout: "unittest_1": { 2024-09-12T23:37:48.709 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.709 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.709 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.709 INFO:tasks.workunit.client.0.smithi136.stdout: "unittest_2": { 2024-09-12T23:37:48.709 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.709 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.709 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.709 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.710 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.710 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:48.710 INFO:tasks.workunit.client.0.smithi136.stdout: "osd.0": { 2024-09-12T23:37:48.710 INFO:tasks.workunit.client.0.smithi136.stdout: "bloom_filter": { 2024-09-12T23:37:48.710 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.710 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.710 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.710 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs": { 2024-09-12T23:37:48.710 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 6304, 2024-09-12T23:37:48.710 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 115 2024-09-12T23:37:48.710 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.710 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs_file_reader": { 2024-09-12T23:37:48.710 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 16896, 2024-09-12T23:37:48.711 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 8 2024-09-12T23:37:48.711 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.711 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs_file_writer": { 2024-09-12T23:37:48.711 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 576, 2024-09-12T23:37:48.711 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 3 2024-09-12T23:37:48.711 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.711 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Blob": { 2024-09-12T23:37:48.711 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 19264, 2024-09-12T23:37:48.711 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 172 2024-09-12T23:37:48.711 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.711 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Buffer": { 2024-09-12T23:37:48.711 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 448, 2024-09-12T23:37:48.711 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 4 2024-09-12T23:37:48.711 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.712 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Extent": { 2024-09-12T23:37:48.712 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 8256, 2024-09-12T23:37:48.712 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 172 2024-09-12T23:37:48.712 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.712 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_SharedBlob": { 2024-09-12T23:37:48.712 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 19264, 2024-09-12T23:37:48.712 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 172 2024-09-12T23:37:48.712 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.712 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_alloc": { 2024-09-12T23:37:48.712 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 5280, 2024-09-12T23:37:48.712 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 66 2024-09-12T23:37:48.712 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.712 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_data": { 2024-09-12T23:37:48.713 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 22733, 2024-09-12T23:37:48.713 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 5 2024-09-12T23:37:48.713 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.713 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_meta": { 2024-09-12T23:37:48.713 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 52867, 2024-09-12T23:37:48.713 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 9516 2024-09-12T23:37:48.713 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.713 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_onode": { 2024-09-12T23:37:48.713 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 129880, 2024-09-12T23:37:48.713 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 191 2024-09-12T23:37:48.713 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.713 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_other": { 2024-09-12T23:37:48.713 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 10800, 2024-09-12T23:37:48.713 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 680 2024-09-12T23:37:48.713 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.714 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_fsck": { 2024-09-12T23:37:48.714 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.714 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.714 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.714 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_inline_bl": { 2024-09-12T23:37:48.714 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 30566, 2024-09-12T23:37:48.714 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 191 2024-09-12T23:37:48.714 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.714 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_txc": { 2024-09-12T23:37:48.714 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.714 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.714 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.714 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_writing": { 2024-09-12T23:37:48.714 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 366171, 2024-09-12T23:37:48.714 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 80 2024-09-12T23:37:48.714 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.715 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_writing_deferred": { 2024-09-12T23:37:48.715 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.715 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.715 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.715 INFO:tasks.workunit.client.0.smithi136.stdout: "buffer_anon": { 2024-09-12T23:37:48.715 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 2418575, 2024-09-12T23:37:48.715 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 192 2024-09-12T23:37:48.715 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.715 INFO:tasks.workunit.client.0.smithi136.stdout: "buffer_meta": { 2024-09-12T23:37:48.715 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 5808, 2024-09-12T23:37:48.715 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 66 2024-09-12T23:37:48.715 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.715 INFO:tasks.workunit.client.0.smithi136.stdout: "mds_co": { 2024-09-12T23:37:48.715 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.716 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.716 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.716 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:48.716 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 521280, 2024-09-12T23:37:48.716 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 45 2024-09-12T23:37:48.716 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.716 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mapbl": { 2024-09-12T23:37:48.716 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.716 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.716 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.716 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_pglog": { 2024-09-12T23:37:48.716 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 69784, 2024-09-12T23:37:48.717 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 139 2024-09-12T23:37:48.717 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.717 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap": { 2024-09-12T23:37:48.717 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 227704, 2024-09-12T23:37:48.717 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 3449 2024-09-12T23:37:48.717 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.717 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap_mapping": { 2024-09-12T23:37:48.717 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.717 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.717 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.717 INFO:tasks.workunit.client.0.smithi136.stdout: "pgmap": { 2024-09-12T23:37:48.717 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.717 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.717 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.718 INFO:tasks.workunit.client.0.smithi136.stdout: "unittest_1": { 2024-09-12T23:37:48.718 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.718 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.718 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.718 INFO:tasks.workunit.client.0.smithi136.stdout: "unittest_2": { 2024-09-12T23:37:48.718 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.718 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.718 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.718 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.718 INFO:tasks.workunit.client.0.smithi136.stdout: "osd.1": { 2024-09-12T23:37:48.718 INFO:tasks.workunit.client.0.smithi136.stdout: "bloom_filter": { 2024-09-12T23:37:48.718 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.718 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.719 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.719 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs": { 2024-09-12T23:37:48.719 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 6304, 2024-09-12T23:37:48.719 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 115 2024-09-12T23:37:48.719 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.719 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs_file_reader": { 2024-09-12T23:37:48.719 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 16896, 2024-09-12T23:37:48.719 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 8 2024-09-12T23:37:48.719 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.719 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs_file_writer": { 2024-09-12T23:37:48.719 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 576, 2024-09-12T23:37:48.719 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 3 2024-09-12T23:37:48.719 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.719 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Blob": { 2024-09-12T23:37:48.720 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 18144, 2024-09-12T23:37:48.720 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 162 2024-09-12T23:37:48.720 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.720 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Buffer": { 2024-09-12T23:37:48.720 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 336, 2024-09-12T23:37:48.720 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 3 2024-09-12T23:37:48.720 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.720 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Extent": { 2024-09-12T23:37:48.720 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 7776, 2024-09-12T23:37:48.720 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 162 2024-09-12T23:37:48.720 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.720 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_SharedBlob": { 2024-09-12T23:37:48.720 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 18144, 2024-09-12T23:37:48.720 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 162 2024-09-12T23:37:48.721 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.721 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_alloc": { 2024-09-12T23:37:48.721 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 4640, 2024-09-12T23:37:48.721 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 58 2024-09-12T23:37:48.721 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.721 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_data": { 2024-09-12T23:37:48.721 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 12288, 2024-09-12T23:37:48.721 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 3 2024-09-12T23:37:48.721 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.721 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_meta": { 2024-09-12T23:37:48.721 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 47344, 2024-09-12T23:37:48.721 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 8644 2024-09-12T23:37:48.721 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.721 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_onode": { 2024-09-12T23:37:48.722 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 122400, 2024-09-12T23:37:48.722 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 180 2024-09-12T23:37:48.722 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.722 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_other": { 2024-09-12T23:37:48.722 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 9004, 2024-09-12T23:37:48.722 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 510 2024-09-12T23:37:48.722 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.722 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_fsck": { 2024-09-12T23:37:48.722 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.722 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.722 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.722 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_inline_bl": { 2024-09-12T23:37:48.722 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 26810, 2024-09-12T23:37:48.722 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 180 2024-09-12T23:37:48.723 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.723 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_txc": { 2024-09-12T23:37:48.723 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.723 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.723 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.723 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_writing": { 2024-09-12T23:37:48.723 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 397408, 2024-09-12T23:37:48.723 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 87 2024-09-12T23:37:48.723 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.723 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_writing_deferred": { 2024-09-12T23:37:48.723 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.723 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.723 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.723 INFO:tasks.workunit.client.0.smithi136.stdout: "buffer_anon": { 2024-09-12T23:37:48.724 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 2405051, 2024-09-12T23:37:48.724 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 186 2024-09-12T23:37:48.724 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.724 INFO:tasks.workunit.client.0.smithi136.stdout: "buffer_meta": { 2024-09-12T23:37:48.724 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 4928, 2024-09-12T23:37:48.724 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 56 2024-09-12T23:37:48.724 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.724 INFO:tasks.workunit.client.0.smithi136.stdout: "mds_co": { 2024-09-12T23:37:48.724 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.724 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.724 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.724 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:48.724 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 451776, 2024-09-12T23:37:48.724 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 39 2024-09-12T23:37:48.725 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.725 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mapbl": { 2024-09-12T23:37:48.725 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.725 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.725 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.725 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_pglog": { 2024-09-12T23:37:48.725 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 8528, 2024-09-12T23:37:48.725 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 22 2024-09-12T23:37:48.725 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.725 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap": { 2024-09-12T23:37:48.725 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 227192, 2024-09-12T23:37:48.725 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 3425 2024-09-12T23:37:48.725 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.726 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap_mapping": { 2024-09-12T23:37:48.726 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.726 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.726 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.726 INFO:tasks.workunit.client.0.smithi136.stdout: "pgmap": { 2024-09-12T23:37:48.726 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.726 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.726 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.726 INFO:tasks.workunit.client.0.smithi136.stdout: "unittest_1": { 2024-09-12T23:37:48.726 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.726 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.726 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.726 INFO:tasks.workunit.client.0.smithi136.stdout: "unittest_2": { 2024-09-12T23:37:48.727 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.727 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.727 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.727 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.727 INFO:tasks.workunit.client.0.smithi136.stdout: "osd.2": { 2024-09-12T23:37:48.727 INFO:tasks.workunit.client.0.smithi136.stdout: "bloom_filter": { 2024-09-12T23:37:48.727 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.727 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.727 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.727 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs": { 2024-09-12T23:37:48.727 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 6304, 2024-09-12T23:37:48.727 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 115 2024-09-12T23:37:48.727 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.727 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs_file_reader": { 2024-09-12T23:37:48.727 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 16896, 2024-09-12T23:37:48.728 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 8 2024-09-12T23:37:48.728 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.728 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs_file_writer": { 2024-09-12T23:37:48.728 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 576, 2024-09-12T23:37:48.728 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 3 2024-09-12T23:37:48.728 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.728 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Blob": { 2024-09-12T23:37:48.728 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 17808, 2024-09-12T23:37:48.728 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 159 2024-09-12T23:37:48.728 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.728 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Buffer": { 2024-09-12T23:37:48.728 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 1120, 2024-09-12T23:37:48.728 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 10 2024-09-12T23:37:48.728 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.729 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Extent": { 2024-09-12T23:37:48.729 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 7632, 2024-09-12T23:37:48.729 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 159 2024-09-12T23:37:48.729 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.729 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_SharedBlob": { 2024-09-12T23:37:48.729 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 17920, 2024-09-12T23:37:48.729 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 160 2024-09-12T23:37:48.729 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.729 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_alloc": { 2024-09-12T23:37:48.729 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 3680, 2024-09-12T23:37:48.729 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 46 2024-09-12T23:37:48.729 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.729 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_data": { 2024-09-12T23:37:48.729 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 8192, 2024-09-12T23:37:48.730 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 2 2024-09-12T23:37:48.730 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.730 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_meta": { 2024-09-12T23:37:48.730 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 44904, 2024-09-12T23:37:48.730 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 8224 2024-09-12T23:37:48.730 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.730 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_onode": { 2024-09-12T23:37:48.730 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 116280, 2024-09-12T23:37:48.730 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 171 2024-09-12T23:37:48.730 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.730 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_other": { 2024-09-12T23:37:48.730 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 8080, 2024-09-12T23:37:48.730 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 498 2024-09-12T23:37:48.731 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.731 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_fsck": { 2024-09-12T23:37:48.731 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.731 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.731 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.731 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_inline_bl": { 2024-09-12T23:37:48.731 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 25010, 2024-09-12T23:37:48.731 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 171 2024-09-12T23:37:48.731 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.731 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_txc": { 2024-09-12T23:37:48.731 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 1472, 2024-09-12T23:37:48.731 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 2 2024-09-12T23:37:48.731 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.732 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_writing": { 2024-09-12T23:37:48.732 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 337827, 2024-09-12T23:37:48.732 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 73 2024-09-12T23:37:48.732 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.732 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_writing_deferred": { 2024-09-12T23:37:48.732 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 20096, 2024-09-12T23:37:48.732 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 5 2024-09-12T23:37:48.732 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.732 INFO:tasks.workunit.client.0.smithi136.stdout: "buffer_anon": { 2024-09-12T23:37:48.732 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 2419800, 2024-09-12T23:37:48.732 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 180 2024-09-12T23:37:48.732 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.732 INFO:tasks.workunit.client.0.smithi136.stdout: "buffer_meta": { 2024-09-12T23:37:48.732 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 4400, 2024-09-12T23:37:48.733 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 50 2024-09-12T23:37:48.733 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.733 INFO:tasks.workunit.client.0.smithi136.stdout: "mds_co": { 2024-09-12T23:37:48.733 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.733 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.733 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.733 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:48.733 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 359104, 2024-09-12T23:37:48.733 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 31 2024-09-12T23:37:48.733 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.733 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mapbl": { 2024-09-12T23:37:48.733 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.734 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.734 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.734 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_pglog": { 2024-09-12T23:37:48.734 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 9360, 2024-09-12T23:37:48.734 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 26 2024-09-12T23:37:48.734 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.734 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap": { 2024-09-12T23:37:48.734 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 227704, 2024-09-12T23:37:48.734 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 3449 2024-09-12T23:37:48.734 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.734 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap_mapping": { 2024-09-12T23:37:48.734 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.734 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.734 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.735 INFO:tasks.workunit.client.0.smithi136.stdout: "pgmap": { 2024-09-12T23:37:48.735 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.735 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.735 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.735 INFO:tasks.workunit.client.0.smithi136.stdout: "unittest_1": { 2024-09-12T23:37:48.735 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.735 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.735 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.735 INFO:tasks.workunit.client.0.smithi136.stdout: "unittest_2": { 2024-09-12T23:37:48.735 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.735 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.735 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.735 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.735 INFO:tasks.workunit.client.0.smithi136.stdout: "osd.3": { 2024-09-12T23:37:48.735 INFO:tasks.workunit.client.0.smithi136.stdout: "bloom_filter": { 2024-09-12T23:37:48.736 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.736 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.736 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.736 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs": { 2024-09-12T23:37:48.736 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 6304, 2024-09-12T23:37:48.736 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 115 2024-09-12T23:37:48.736 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.736 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs_file_reader": { 2024-09-12T23:37:48.736 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 16896, 2024-09-12T23:37:48.736 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 8 2024-09-12T23:37:48.736 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.737 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs_file_writer": { 2024-09-12T23:37:48.737 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 576, 2024-09-12T23:37:48.737 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 3 2024-09-12T23:37:48.737 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.737 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Blob": { 2024-09-12T23:37:48.737 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 18368, 2024-09-12T23:37:48.737 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 164 2024-09-12T23:37:48.737 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.737 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Buffer": { 2024-09-12T23:37:48.737 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 224, 2024-09-12T23:37:48.737 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 2 2024-09-12T23:37:48.737 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.737 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Extent": { 2024-09-12T23:37:48.738 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 7872, 2024-09-12T23:37:48.738 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 164 2024-09-12T23:37:48.738 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.738 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_SharedBlob": { 2024-09-12T23:37:48.738 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 18368, 2024-09-12T23:37:48.738 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 164 2024-09-12T23:37:48.738 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.738 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_alloc": { 2024-09-12T23:37:48.738 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 3520, 2024-09-12T23:37:48.738 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 44 2024-09-12T23:37:48.738 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.738 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_data": { 2024-09-12T23:37:48.738 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.738 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.739 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.739 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_meta": { 2024-09-12T23:37:48.739 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 45902, 2024-09-12T23:37:48.739 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 8497 2024-09-12T23:37:48.739 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.739 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_onode": { 2024-09-12T23:37:48.739 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 120360, 2024-09-12T23:37:48.739 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 177 2024-09-12T23:37:48.739 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.739 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_other": { 2024-09-12T23:37:48.739 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 8868, 2024-09-12T23:37:48.739 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 515 2024-09-12T23:37:48.739 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.740 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_fsck": { 2024-09-12T23:37:48.740 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.740 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.740 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.740 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_inline_bl": { 2024-09-12T23:37:48.740 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 25706, 2024-09-12T23:37:48.740 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 177 2024-09-12T23:37:48.740 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.740 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_txc": { 2024-09-12T23:37:48.740 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 2208, 2024-09-12T23:37:48.740 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 3 2024-09-12T23:37:48.740 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.740 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_writing": { 2024-09-12T23:37:48.740 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 249673, 2024-09-12T23:37:48.741 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 51 2024-09-12T23:37:48.741 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.741 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_writing_deferred": { 2024-09-12T23:37:48.741 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 30637, 2024-09-12T23:37:48.741 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 9 2024-09-12T23:37:48.741 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.741 INFO:tasks.workunit.client.0.smithi136.stdout: "buffer_anon": { 2024-09-12T23:37:48.741 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 2436764, 2024-09-12T23:37:48.741 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 187 2024-09-12T23:37:48.741 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.741 INFO:tasks.workunit.client.0.smithi136.stdout: "buffer_meta": { 2024-09-12T23:37:48.741 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 4752, 2024-09-12T23:37:48.741 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 54 2024-09-12T23:37:48.742 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.742 INFO:tasks.workunit.client.0.smithi136.stdout: "mds_co": { 2024-09-12T23:37:48.742 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.742 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.742 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.742 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:48.742 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 428608, 2024-09-12T23:37:48.742 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 37 2024-09-12T23:37:48.742 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.742 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mapbl": { 2024-09-12T23:37:48.742 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.742 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.742 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.743 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_pglog": { 2024-09-12T23:37:48.743 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 10400, 2024-09-12T23:37:48.743 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 20 2024-09-12T23:37:48.743 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.743 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap": { 2024-09-12T23:37:48.743 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 227192, 2024-09-12T23:37:48.743 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 3425 2024-09-12T23:37:48.743 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.743 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap_mapping": { 2024-09-12T23:37:48.743 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.743 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.743 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.743 INFO:tasks.workunit.client.0.smithi136.stdout: "pgmap": { 2024-09-12T23:37:48.744 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.744 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.744 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.744 INFO:tasks.workunit.client.0.smithi136.stdout: "unittest_1": { 2024-09-12T23:37:48.744 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.744 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.744 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.744 INFO:tasks.workunit.client.0.smithi136.stdout: "unittest_2": { 2024-09-12T23:37:48.744 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.744 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.744 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.744 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.744 INFO:tasks.workunit.client.0.smithi136.stdout: "osd.4": { 2024-09-12T23:37:48.744 INFO:tasks.workunit.client.0.smithi136.stdout: "bloom_filter": { 2024-09-12T23:37:48.745 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.745 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.745 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.745 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs": { 2024-09-12T23:37:48.745 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 6304, 2024-09-12T23:37:48.745 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 115 2024-09-12T23:37:48.745 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.745 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs_file_reader": { 2024-09-12T23:37:48.745 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 16896, 2024-09-12T23:37:48.745 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 8 2024-09-12T23:37:48.745 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.745 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs_file_writer": { 2024-09-12T23:37:48.745 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 576, 2024-09-12T23:37:48.746 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 3 2024-09-12T23:37:48.746 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.746 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Blob": { 2024-09-12T23:37:48.746 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 18368, 2024-09-12T23:37:48.746 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 164 2024-09-12T23:37:48.746 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.746 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Buffer": { 2024-09-12T23:37:48.746 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 1008, 2024-09-12T23:37:48.746 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 9 2024-09-12T23:37:48.746 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.746 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Extent": { 2024-09-12T23:37:48.746 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 7872, 2024-09-12T23:37:48.746 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 164 2024-09-12T23:37:48.746 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.747 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_SharedBlob": { 2024-09-12T23:37:48.747 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 18480, 2024-09-12T23:37:48.747 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 165 2024-09-12T23:37:48.747 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.747 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_alloc": { 2024-09-12T23:37:48.747 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 3040, 2024-09-12T23:37:48.747 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 38 2024-09-12T23:37:48.747 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.747 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_data": { 2024-09-12T23:37:48.747 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 4096, 2024-09-12T23:37:48.747 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 1 2024-09-12T23:37:48.747 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.747 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_meta": { 2024-09-12T23:37:48.747 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 46942, 2024-09-12T23:37:48.748 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 8602 2024-09-12T23:37:48.748 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.748 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_onode": { 2024-09-12T23:37:48.748 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 121720, 2024-09-12T23:37:48.748 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 179 2024-09-12T23:37:48.748 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.748 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_other": { 2024-09-12T23:37:48.748 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 8964, 2024-09-12T23:37:48.748 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 515 2024-09-12T23:37:48.748 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.748 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_fsck": { 2024-09-12T23:37:48.748 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.748 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.748 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.749 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_inline_bl": { 2024-09-12T23:37:48.749 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 26218, 2024-09-12T23:37:48.749 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 179 2024-09-12T23:37:48.749 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.749 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_txc": { 2024-09-12T23:37:48.749 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 1472, 2024-09-12T23:37:48.749 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 2 2024-09-12T23:37:48.749 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.749 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_writing": { 2024-09-12T23:37:48.749 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 873885, 2024-09-12T23:37:48.749 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 61 2024-09-12T23:37:48.749 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.749 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_writing_deferred": { 2024-09-12T23:37:48.750 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 20096, 2024-09-12T23:37:48.750 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 5 2024-09-12T23:37:48.750 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.750 INFO:tasks.workunit.client.0.smithi136.stdout: "buffer_anon": { 2024-09-12T23:37:48.750 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 2398270, 2024-09-12T23:37:48.750 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 173 2024-09-12T23:37:48.750 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.750 INFO:tasks.workunit.client.0.smithi136.stdout: "buffer_meta": { 2024-09-12T23:37:48.750 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 4752, 2024-09-12T23:37:48.750 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 54 2024-09-12T23:37:48.750 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.750 INFO:tasks.workunit.client.0.smithi136.stdout: "mds_co": { 2024-09-12T23:37:48.750 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.751 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.751 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.751 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:48.751 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 440192, 2024-09-12T23:37:48.751 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 38 2024-09-12T23:37:48.751 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.751 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mapbl": { 2024-09-12T23:37:48.751 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.751 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.751 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.751 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_pglog": { 2024-09-12T23:37:48.751 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 8944, 2024-09-12T23:37:48.751 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 18 2024-09-12T23:37:48.751 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.752 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap": { 2024-09-12T23:37:48.752 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 227448, 2024-09-12T23:37:48.752 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 3437 2024-09-12T23:37:48.752 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.752 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap_mapping": { 2024-09-12T23:37:48.752 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.752 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.752 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.752 INFO:tasks.workunit.client.0.smithi136.stdout: "pgmap": { 2024-09-12T23:37:48.752 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.752 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.752 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.752 INFO:tasks.workunit.client.0.smithi136.stdout: "unittest_1": { 2024-09-12T23:37:48.752 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.752 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.753 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.753 INFO:tasks.workunit.client.0.smithi136.stdout: "unittest_2": { 2024-09-12T23:37:48.753 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.753 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.753 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.753 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.753 INFO:tasks.workunit.client.0.smithi136.stdout: "osd.5": { 2024-09-12T23:37:48.753 INFO:tasks.workunit.client.0.smithi136.stdout: "bloom_filter": { 2024-09-12T23:37:48.753 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.753 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.753 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.753 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs": { 2024-09-12T23:37:48.753 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 6304, 2024-09-12T23:37:48.753 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 115 2024-09-12T23:37:48.753 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.754 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs_file_reader": { 2024-09-12T23:37:48.754 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 16896, 2024-09-12T23:37:48.754 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 8 2024-09-12T23:37:48.754 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.754 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs_file_writer": { 2024-09-12T23:37:48.754 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 576, 2024-09-12T23:37:48.754 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 3 2024-09-12T23:37:48.754 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.754 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Blob": { 2024-09-12T23:37:48.754 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 18144, 2024-09-12T23:37:48.754 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 162 2024-09-12T23:37:48.754 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.754 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Buffer": { 2024-09-12T23:37:48.754 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 336, 2024-09-12T23:37:48.754 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 3 2024-09-12T23:37:48.754 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.755 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Extent": { 2024-09-12T23:37:48.755 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 7776, 2024-09-12T23:37:48.755 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 162 2024-09-12T23:37:48.755 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.755 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_SharedBlob": { 2024-09-12T23:37:48.755 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 18144, 2024-09-12T23:37:48.755 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 162 2024-09-12T23:37:48.755 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.755 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_alloc": { 2024-09-12T23:37:48.755 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 2800, 2024-09-12T23:37:48.755 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 35 2024-09-12T23:37:48.755 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.755 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_data": { 2024-09-12T23:37:48.755 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 4096, 2024-09-12T23:37:48.756 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 1 2024-09-12T23:37:48.756 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.756 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_meta": { 2024-09-12T23:37:48.756 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 47550, 2024-09-12T23:37:48.756 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 8640 2024-09-12T23:37:48.756 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.756 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_onode": { 2024-09-12T23:37:48.756 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 122400, 2024-09-12T23:37:48.756 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 180 2024-09-12T23:37:48.756 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.756 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_other": { 2024-09-12T23:37:48.756 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 9164, 2024-09-12T23:37:48.756 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 514 2024-09-12T23:37:48.756 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.757 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_fsck": { 2024-09-12T23:37:48.757 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.757 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.757 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.757 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_inline_bl": { 2024-09-12T23:37:48.757 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 26810, 2024-09-12T23:37:48.757 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 180 2024-09-12T23:37:48.757 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.757 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_txc": { 2024-09-12T23:37:48.757 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 2208, 2024-09-12T23:37:48.757 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 3 2024-09-12T23:37:48.757 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.757 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_writing": { 2024-09-12T23:37:48.757 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 540192, 2024-09-12T23:37:48.758 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 43 2024-09-12T23:37:48.758 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.758 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_writing_deferred": { 2024-09-12T23:37:48.758 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 30637, 2024-09-12T23:37:48.758 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 9 2024-09-12T23:37:48.758 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.758 INFO:tasks.workunit.client.0.smithi136.stdout: "buffer_anon": { 2024-09-12T23:37:48.758 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 2437972, 2024-09-12T23:37:48.758 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 184 2024-09-12T23:37:48.758 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.758 INFO:tasks.workunit.client.0.smithi136.stdout: "buffer_meta": { 2024-09-12T23:37:48.758 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 5368, 2024-09-12T23:37:48.758 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 61 2024-09-12T23:37:48.758 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.759 INFO:tasks.workunit.client.0.smithi136.stdout: "mds_co": { 2024-09-12T23:37:48.759 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.759 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.759 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.759 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:48.759 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 463360, 2024-09-12T23:37:48.759 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 40 2024-09-12T23:37:48.759 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.759 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mapbl": { 2024-09-12T23:37:48.759 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.759 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.760 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.760 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_pglog": { 2024-09-12T23:37:48.760 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 13000, 2024-09-12T23:37:48.760 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 25 2024-09-12T23:37:48.760 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.760 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap": { 2024-09-12T23:37:48.760 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 227960, 2024-09-12T23:37:48.760 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 3461 2024-09-12T23:37:48.760 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.760 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap_mapping": { 2024-09-12T23:37:48.760 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.760 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.760 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.760 INFO:tasks.workunit.client.0.smithi136.stdout: "pgmap": { 2024-09-12T23:37:48.761 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.761 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.761 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.761 INFO:tasks.workunit.client.0.smithi136.stdout: "unittest_1": { 2024-09-12T23:37:48.761 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.761 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.761 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.761 INFO:tasks.workunit.client.0.smithi136.stdout: "unittest_2": { 2024-09-12T23:37:48.761 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.761 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.761 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.761 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.762 INFO:tasks.workunit.client.0.smithi136.stdout: "osd.6": { 2024-09-12T23:37:48.762 INFO:tasks.workunit.client.0.smithi136.stdout: "bloom_filter": { 2024-09-12T23:37:48.762 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.762 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.762 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.762 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs": { 2024-09-12T23:37:48.762 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 6304, 2024-09-12T23:37:48.762 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 115 2024-09-12T23:37:48.762 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.762 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs_file_reader": { 2024-09-12T23:37:48.762 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 16896, 2024-09-12T23:37:48.762 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 8 2024-09-12T23:37:48.762 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.762 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs_file_writer": { 2024-09-12T23:37:48.763 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 576, 2024-09-12T23:37:48.763 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 3 2024-09-12T23:37:48.763 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.763 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Blob": { 2024-09-12T23:37:48.763 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 19152, 2024-09-12T23:37:48.763 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 171 2024-09-12T23:37:48.763 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.763 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Buffer": { 2024-09-12T23:37:48.763 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 448, 2024-09-12T23:37:48.763 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 4 2024-09-12T23:37:48.763 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.763 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Extent": { 2024-09-12T23:37:48.763 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 8208, 2024-09-12T23:37:48.763 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 171 2024-09-12T23:37:48.764 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.764 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_SharedBlob": { 2024-09-12T23:37:48.764 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 19152, 2024-09-12T23:37:48.764 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 171 2024-09-12T23:37:48.764 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.764 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_alloc": { 2024-09-12T23:37:48.764 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 2480, 2024-09-12T23:37:48.764 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 31 2024-09-12T23:37:48.764 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.764 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_data": { 2024-09-12T23:37:48.764 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 22733, 2024-09-12T23:37:48.764 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 5 2024-09-12T23:37:48.764 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.764 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_meta": { 2024-09-12T23:37:48.764 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 47611, 2024-09-12T23:37:48.765 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 8932 2024-09-12T23:37:48.765 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.765 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_onode": { 2024-09-12T23:37:48.765 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 121720, 2024-09-12T23:37:48.765 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 179 2024-09-12T23:37:48.765 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.765 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_other": { 2024-09-12T23:37:48.765 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 10300, 2024-09-12T23:37:48.765 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 672 2024-09-12T23:37:48.765 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.765 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_fsck": { 2024-09-12T23:37:48.765 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.765 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.766 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.766 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_inline_bl": { 2024-09-12T23:37:48.766 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 27582, 2024-09-12T23:37:48.766 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 179 2024-09-12T23:37:48.766 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.766 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_txc": { 2024-09-12T23:37:48.766 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.766 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.766 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.766 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_writing": { 2024-09-12T23:37:48.766 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 780888, 2024-09-12T23:37:48.766 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 39 2024-09-12T23:37:48.766 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.766 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_writing_deferred": { 2024-09-12T23:37:48.767 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.767 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.767 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.767 INFO:tasks.workunit.client.0.smithi136.stdout: "buffer_anon": { 2024-09-12T23:37:48.767 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 2394070, 2024-09-12T23:37:48.767 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 164 2024-09-12T23:37:48.767 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.767 INFO:tasks.workunit.client.0.smithi136.stdout: "buffer_meta": { 2024-09-12T23:37:48.767 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 5016, 2024-09-12T23:37:48.767 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 57 2024-09-12T23:37:48.767 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.767 INFO:tasks.workunit.client.0.smithi136.stdout: "mds_co": { 2024-09-12T23:37:48.767 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.767 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.767 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.768 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:48.768 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 451776, 2024-09-12T23:37:48.768 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 39 2024-09-12T23:37:48.768 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.768 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mapbl": { 2024-09-12T23:37:48.768 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.768 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.768 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.768 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_pglog": { 2024-09-12T23:37:48.768 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 68952, 2024-09-12T23:37:48.768 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 143 2024-09-12T23:37:48.768 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.768 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap": { 2024-09-12T23:37:48.768 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 227192, 2024-09-12T23:37:48.769 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 3425 2024-09-12T23:37:48.769 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.769 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap_mapping": { 2024-09-12T23:37:48.769 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.769 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.769 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.769 INFO:tasks.workunit.client.0.smithi136.stdout: "pgmap": { 2024-09-12T23:37:48.769 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.769 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.769 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.769 INFO:tasks.workunit.client.0.smithi136.stdout: "unittest_1": { 2024-09-12T23:37:48.769 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.769 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.770 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.770 INFO:tasks.workunit.client.0.smithi136.stdout: "unittest_2": { 2024-09-12T23:37:48.770 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.770 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.770 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.770 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.770 INFO:tasks.workunit.client.0.smithi136.stdout: "osd.7": { 2024-09-12T23:37:48.770 INFO:tasks.workunit.client.0.smithi136.stdout: "bloom_filter": { 2024-09-12T23:37:48.770 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.770 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.770 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.770 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs": { 2024-09-12T23:37:48.770 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 6304, 2024-09-12T23:37:48.770 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 115 2024-09-12T23:37:48.771 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.771 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs_file_reader": { 2024-09-12T23:37:48.771 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 16896, 2024-09-12T23:37:48.771 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 8 2024-09-12T23:37:48.771 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.771 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs_file_writer": { 2024-09-12T23:37:48.771 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 576, 2024-09-12T23:37:48.771 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 3 2024-09-12T23:37:48.771 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.771 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Blob": { 2024-09-12T23:37:48.771 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 19152, 2024-09-12T23:37:48.771 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 171 2024-09-12T23:37:48.771 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.771 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Buffer": { 2024-09-12T23:37:48.772 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 1344, 2024-09-12T23:37:48.772 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 12 2024-09-12T23:37:48.772 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.772 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Extent": { 2024-09-12T23:37:48.772 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 8208, 2024-09-12T23:37:48.772 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 171 2024-09-12T23:37:48.772 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.772 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_SharedBlob": { 2024-09-12T23:37:48.772 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 19152, 2024-09-12T23:37:48.772 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 171 2024-09-12T23:37:48.772 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.772 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_alloc": { 2024-09-12T23:37:48.772 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 2000, 2024-09-12T23:37:48.773 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 25 2024-09-12T23:37:48.773 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.773 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_data": { 2024-09-12T23:37:48.773 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 297165, 2024-09-12T23:37:48.773 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 13 2024-09-12T23:37:48.773 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.773 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_meta": { 2024-09-12T23:37:48.773 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 51006, 2024-09-12T23:37:48.773 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 9370 2024-09-12T23:37:48.773 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.773 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_onode": { 2024-09-12T23:37:48.773 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 127840, 2024-09-12T23:37:48.773 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 188 2024-09-12T23:37:48.774 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.774 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_other": { 2024-09-12T23:37:48.774 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 11084, 2024-09-12T23:37:48.774 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 679 2024-09-12T23:37:48.774 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.774 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_fsck": { 2024-09-12T23:37:48.774 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.774 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.774 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.774 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_inline_bl": { 2024-09-12T23:37:48.774 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 29886, 2024-09-12T23:37:48.774 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 188 2024-09-12T23:37:48.774 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.774 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_txc": { 2024-09-12T23:37:48.774 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.775 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.775 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.775 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_writing": { 2024-09-12T23:37:48.775 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 736834, 2024-09-12T23:37:48.775 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 29 2024-09-12T23:37:48.775 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.775 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_writing_deferred": { 2024-09-12T23:37:48.775 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.775 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.775 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.775 INFO:tasks.workunit.client.0.smithi136.stdout: "buffer_anon": { 2024-09-12T23:37:48.775 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 2426906, 2024-09-12T23:37:48.775 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 219 2024-09-12T23:37:48.775 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.775 INFO:tasks.workunit.client.0.smithi136.stdout: "buffer_meta": { 2024-09-12T23:37:48.775 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 6512, 2024-09-12T23:37:48.776 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 74 2024-09-12T23:37:48.776 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.776 INFO:tasks.workunit.client.0.smithi136.stdout: "mds_co": { 2024-09-12T23:37:48.776 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.776 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.776 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.776 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:48.776 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 532864, 2024-09-12T23:37:48.776 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 46 2024-09-12T23:37:48.776 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.776 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mapbl": { 2024-09-12T23:37:48.777 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.777 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.777 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.777 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_pglog": { 2024-09-12T23:37:48.777 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 70408, 2024-09-12T23:37:48.777 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 169 2024-09-12T23:37:48.777 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.777 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap": { 2024-09-12T23:37:48.777 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 227192, 2024-09-12T23:37:48.777 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 3425 2024-09-12T23:37:48.777 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.777 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap_mapping": { 2024-09-12T23:37:48.777 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.777 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.777 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.778 INFO:tasks.workunit.client.0.smithi136.stdout: "pgmap": { 2024-09-12T23:37:48.778 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.778 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.778 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.778 INFO:tasks.workunit.client.0.smithi136.stdout: "unittest_1": { 2024-09-12T23:37:48.778 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.778 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.778 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.778 INFO:tasks.workunit.client.0.smithi136.stdout: "unittest_2": { 2024-09-12T23:37:48.778 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:48.778 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:48.778 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.778 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.778 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.778 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.779 INFO:tasks.workunit.client.0.smithi136.stdout: "metadata": { 2024-09-12T23:37:48.779 INFO:tasks.workunit.client.0.smithi136.stdout: "mds": { 2024-09-12T23:37:48.779 INFO:tasks.workunit.client.0.smithi136.stdout: "arch": { 2024-09-12T23:37:48.779 INFO:tasks.workunit.client.0.smithi136.stdout: "x86_64": 2 2024-09-12T23:37:48.779 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.779 INFO:tasks.workunit.client.0.smithi136.stdout: "ceph_version": { 2024-09-12T23:37:48.779 INFO:tasks.workunit.client.0.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-12T23:37:48.779 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.779 INFO:tasks.workunit.client.0.smithi136.stdout: "cpu": { 2024-09-12T23:37:48.779 INFO:tasks.workunit.client.0.smithi136.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 2 2024-09-12T23:37:48.779 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.779 INFO:tasks.workunit.client.0.smithi136.stdout: "distro": { 2024-09-12T23:37:48.779 INFO:tasks.workunit.client.0.smithi136.stdout: "centos": 2 2024-09-12T23:37:48.779 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.779 INFO:tasks.workunit.client.0.smithi136.stdout: "distro_description": { 2024-09-12T23:37:48.780 INFO:tasks.workunit.client.0.smithi136.stdout: "CentOS Stream 9": 2 2024-09-12T23:37:48.780 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.780 INFO:tasks.workunit.client.0.smithi136.stdout: "kernel_description": { 2024-09-12T23:37:48.780 INFO:tasks.workunit.client.0.smithi136.stdout: "#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024": 2 2024-09-12T23:37:48.780 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.780 INFO:tasks.workunit.client.0.smithi136.stdout: "kernel_version": { 2024-09-12T23:37:48.780 INFO:tasks.workunit.client.0.smithi136.stdout: "5.15.0-121-generic": 2 2024-09-12T23:37:48.780 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.780 INFO:tasks.workunit.client.0.smithi136.stdout: "os": { 2024-09-12T23:37:48.780 INFO:tasks.workunit.client.0.smithi136.stdout: "Linux": 2 2024-09-12T23:37:48.780 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.780 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.780 INFO:tasks.workunit.client.0.smithi136.stdout: "mon": { 2024-09-12T23:37:48.780 INFO:tasks.workunit.client.0.smithi136.stdout: "arch": { 2024-09-12T23:37:48.781 INFO:tasks.workunit.client.0.smithi136.stdout: "x86_64": 3 2024-09-12T23:37:48.781 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.781 INFO:tasks.workunit.client.0.smithi136.stdout: "ceph_version": { 2024-09-12T23:37:48.781 INFO:tasks.workunit.client.0.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 3 2024-09-12T23:37:48.781 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.781 INFO:tasks.workunit.client.0.smithi136.stdout: "cpu": { 2024-09-12T23:37:48.781 INFO:tasks.workunit.client.0.smithi136.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 3 2024-09-12T23:37:48.781 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.781 INFO:tasks.workunit.client.0.smithi136.stdout: "distro": { 2024-09-12T23:37:48.781 INFO:tasks.workunit.client.0.smithi136.stdout: "centos": 3 2024-09-12T23:37:48.781 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.781 INFO:tasks.workunit.client.0.smithi136.stdout: "distro_description": { 2024-09-12T23:37:48.781 INFO:tasks.workunit.client.0.smithi136.stdout: "CentOS Stream 9": 3 2024-09-12T23:37:48.782 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.782 INFO:tasks.workunit.client.0.smithi136.stdout: "kernel_description": { 2024-09-12T23:37:48.782 INFO:tasks.workunit.client.0.smithi136.stdout: "#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024": 3 2024-09-12T23:37:48.782 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.782 INFO:tasks.workunit.client.0.smithi136.stdout: "kernel_version": { 2024-09-12T23:37:48.782 INFO:tasks.workunit.client.0.smithi136.stdout: "5.15.0-121-generic": 3 2024-09-12T23:37:48.782 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.782 INFO:tasks.workunit.client.0.smithi136.stdout: "os": { 2024-09-12T23:37:48.782 INFO:tasks.workunit.client.0.smithi136.stdout: "Linux": 3 2024-09-12T23:37:48.782 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.782 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.782 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:48.782 INFO:tasks.workunit.client.0.smithi136.stdout: "arch": { 2024-09-12T23:37:48.783 INFO:tasks.workunit.client.0.smithi136.stdout: "x86_64": 8 2024-09-12T23:37:48.783 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.783 INFO:tasks.workunit.client.0.smithi136.stdout: "ceph_version": { 2024-09-12T23:37:48.783 INFO:tasks.workunit.client.0.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 8 2024-09-12T23:37:48.783 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.783 INFO:tasks.workunit.client.0.smithi136.stdout: "cpu": { 2024-09-12T23:37:48.783 INFO:tasks.workunit.client.0.smithi136.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 8 2024-09-12T23:37:48.783 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.783 INFO:tasks.workunit.client.0.smithi136.stdout: "distro": { 2024-09-12T23:37:48.783 INFO:tasks.workunit.client.0.smithi136.stdout: "centos": 8 2024-09-12T23:37:48.783 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.783 INFO:tasks.workunit.client.0.smithi136.stdout: "distro_description": { 2024-09-12T23:37:48.783 INFO:tasks.workunit.client.0.smithi136.stdout: "CentOS Stream 9": 8 2024-09-12T23:37:48.784 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.784 INFO:tasks.workunit.client.0.smithi136.stdout: "kernel_description": { 2024-09-12T23:37:48.784 INFO:tasks.workunit.client.0.smithi136.stdout: "#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024": 8 2024-09-12T23:37:48.784 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.784 INFO:tasks.workunit.client.0.smithi136.stdout: "kernel_version": { 2024-09-12T23:37:48.784 INFO:tasks.workunit.client.0.smithi136.stdout: "5.15.0-121-generic": 8 2024-09-12T23:37:48.784 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.784 INFO:tasks.workunit.client.0.smithi136.stdout: "os": { 2024-09-12T23:37:48.784 INFO:tasks.workunit.client.0.smithi136.stdout: "Linux": 8 2024-09-12T23:37:48.784 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.784 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_objectstore": { 2024-09-12T23:37:48.784 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore": 8 2024-09-12T23:37:48.784 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.784 INFO:tasks.workunit.client.0.smithi136.stdout: "rotational": { 2024-09-12T23:37:48.785 INFO:tasks.workunit.client.0.smithi136.stdout: "0": 8 2024-09-12T23:37:48.785 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.785 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.785 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.785 INFO:tasks.workunit.client.0.smithi136.stdout: "mon": { 2024-09-12T23:37:48.785 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 3, 2024-09-12T23:37:48.785 INFO:tasks.workunit.client.0.smithi136.stdout: "features": { 2024-09-12T23:37:48.785 INFO:tasks.workunit.client.0.smithi136.stdout: "optional": [], 2024-09-12T23:37:48.785 INFO:tasks.workunit.client.0.smithi136.stdout: "persistent": [ 2024-09-12T23:37:48.785 INFO:tasks.workunit.client.0.smithi136.stdout: "kraken", 2024-09-12T23:37:48.785 INFO:tasks.workunit.client.0.smithi136.stdout: "luminous", 2024-09-12T23:37:48.785 INFO:tasks.workunit.client.0.smithi136.stdout: "mimic", 2024-09-12T23:37:48.785 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap-prune", 2024-09-12T23:37:48.786 INFO:tasks.workunit.client.0.smithi136.stdout: "nautilus", 2024-09-12T23:37:48.786 INFO:tasks.workunit.client.0.smithi136.stdout: "octopus", 2024-09-12T23:37:48.786 INFO:tasks.workunit.client.0.smithi136.stdout: "pacific", 2024-09-12T23:37:48.786 INFO:tasks.workunit.client.0.smithi136.stdout: "elector-pinging", 2024-09-12T23:37:48.786 INFO:tasks.workunit.client.0.smithi136.stdout: "quincy" 2024-09-12T23:37:48.786 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:48.786 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.786 INFO:tasks.workunit.client.0.smithi136.stdout: "ipv4_addr_mons": 6, 2024-09-12T23:37:48.786 INFO:tasks.workunit.client.0.smithi136.stdout: "ipv6_addr_mons": 0, 2024-09-12T23:37:48.786 INFO:tasks.workunit.client.0.smithi136.stdout: "min_mon_release": 17, 2024-09-12T23:37:48.786 INFO:tasks.workunit.client.0.smithi136.stdout: "v1_addr_mons": 3, 2024-09-12T23:37:48.786 INFO:tasks.workunit.client.0.smithi136.stdout: "v2_addr_mons": 3 2024-09-12T23:37:48.786 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.786 INFO:tasks.workunit.client.0.smithi136.stdout: "organization": "ceph-qa", 2024-09-12T23:37:48.787 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:48.787 INFO:tasks.workunit.client.0.smithi136.stdout: "cluster_network": false, 2024-09-12T23:37:48.787 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 8, 2024-09-12T23:37:48.787 INFO:tasks.workunit.client.0.smithi136.stdout: "require_min_compat_client": "luminous", 2024-09-12T23:37:48.787 INFO:tasks.workunit.client.0.smithi136.stdout: "require_osd_release": "quincy" 2024-09-12T23:37:48.787 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.787 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_perf_histograms": [ 2024-09-12T23:37:48.787 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:48.787 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-12T23:37:48.787 INFO:tasks.workunit.client.0.smithi136.stdout: "axes": [ 2024-09-12T23:37:48.787 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:48.787 INFO:tasks.workunit.client.0.smithi136.stdout: "buckets": 32, 2024-09-12T23:37:48.787 INFO:tasks.workunit.client.0.smithi136.stdout: "min": 0, 2024-09-12T23:37:48.787 INFO:tasks.workunit.client.0.smithi136.stdout: "name": "Latency (usec)", 2024-09-12T23:37:48.788 INFO:tasks.workunit.client.0.smithi136.stdout: "quant_size": 100000, 2024-09-12T23:37:48.788 INFO:tasks.workunit.client.0.smithi136.stdout: "ranges": [ 2024-09-12T23:37:48.788 INFO:tasks.workunit.client.0.smithi136.stdout: "[None, -1]", 2024-09-12T23:37:48.788 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 99999]", 2024-09-12T23:37:48.788 INFO:tasks.workunit.client.0.smithi136.stdout: "[100000, 199999]", 2024-09-12T23:37:48.788 INFO:tasks.workunit.client.0.smithi136.stdout: "[200000, 399999]", 2024-09-12T23:37:48.788 INFO:tasks.workunit.client.0.smithi136.stdout: "[400000, 799999]", 2024-09-12T23:37:48.788 INFO:tasks.workunit.client.0.smithi136.stdout: "[800000, 1599999]", 2024-09-12T23:37:48.788 INFO:tasks.workunit.client.0.smithi136.stdout: "[1600000, 3199999]", 2024-09-12T23:37:48.788 INFO:tasks.workunit.client.0.smithi136.stdout: "[3200000, 6399999]", 2024-09-12T23:37:48.788 INFO:tasks.workunit.client.0.smithi136.stdout: "[6400000, 12799999]", 2024-09-12T23:37:48.788 INFO:tasks.workunit.client.0.smithi136.stdout: "[12800000, 25599999]", 2024-09-12T23:37:48.788 INFO:tasks.workunit.client.0.smithi136.stdout: "[25600000, 51199999]", 2024-09-12T23:37:48.789 INFO:tasks.workunit.client.0.smithi136.stdout: "[51200000, 102399999]", 2024-09-12T23:37:48.789 INFO:tasks.workunit.client.0.smithi136.stdout: "[102400000, 204799999]", 2024-09-12T23:37:48.789 INFO:tasks.workunit.client.0.smithi136.stdout: "[204800000, 409599999]", 2024-09-12T23:37:48.789 INFO:tasks.workunit.client.0.smithi136.stdout: "[409600000, 819199999]", 2024-09-12T23:37:48.789 INFO:tasks.workunit.client.0.smithi136.stdout: "[819200000, 1638399999]", 2024-09-12T23:37:48.789 INFO:tasks.workunit.client.0.smithi136.stdout: "[1638400000, 3276799999]", 2024-09-12T23:37:48.789 INFO:tasks.workunit.client.0.smithi136.stdout: "[3276800000, 6553599999]", 2024-09-12T23:37:48.789 INFO:tasks.workunit.client.0.smithi136.stdout: "[6553600000, 13107199999]", 2024-09-12T23:37:48.789 INFO:tasks.workunit.client.0.smithi136.stdout: "[13107200000, 26214399999]", 2024-09-12T23:37:48.789 INFO:tasks.workunit.client.0.smithi136.stdout: "[26214400000, 52428799999]", 2024-09-12T23:37:48.789 INFO:tasks.workunit.client.0.smithi136.stdout: "[52428800000, 104857599999]", 2024-09-12T23:37:48.789 INFO:tasks.workunit.client.0.smithi136.stdout: "[104857600000, 209715199999]", 2024-09-12T23:37:48.789 INFO:tasks.workunit.client.0.smithi136.stdout: "[209715200000, 419430399999]", 2024-09-12T23:37:48.789 INFO:tasks.workunit.client.0.smithi136.stdout: "[419430400000, 838860799999]", 2024-09-12T23:37:48.790 INFO:tasks.workunit.client.0.smithi136.stdout: "[838860800000, 1677721599999]", 2024-09-12T23:37:48.790 INFO:tasks.workunit.client.0.smithi136.stdout: "[1677721600000, 3355443199999]", 2024-09-12T23:37:48.790 INFO:tasks.workunit.client.0.smithi136.stdout: "[3355443200000, 6710886399999]", 2024-09-12T23:37:48.790 INFO:tasks.workunit.client.0.smithi136.stdout: "[6710886400000, 13421772799999]", 2024-09-12T23:37:48.790 INFO:tasks.workunit.client.0.smithi136.stdout: "[13421772800000, 26843545599999]", 2024-09-12T23:37:48.790 INFO:tasks.workunit.client.0.smithi136.stdout: "[26843545600000, 53687091199999]", 2024-09-12T23:37:48.790 INFO:tasks.workunit.client.0.smithi136.stdout: "[53687091200000, None]" 2024-09-12T23:37:48.790 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:48.790 INFO:tasks.workunit.client.0.smithi136.stdout: "scale_type": "log2" 2024-09-12T23:37:48.790 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.790 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:48.790 INFO:tasks.workunit.client.0.smithi136.stdout: "buckets": 32, 2024-09-12T23:37:48.790 INFO:tasks.workunit.client.0.smithi136.stdout: "min": 0, 2024-09-12T23:37:48.791 INFO:tasks.workunit.client.0.smithi136.stdout: "name": "Request size (bytes)", 2024-09-12T23:37:48.791 INFO:tasks.workunit.client.0.smithi136.stdout: "quant_size": 512, 2024-09-12T23:37:48.791 INFO:tasks.workunit.client.0.smithi136.stdout: "ranges": [ 2024-09-12T23:37:48.791 INFO:tasks.workunit.client.0.smithi136.stdout: "[None, -1]", 2024-09-12T23:37:48.791 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 511]", 2024-09-12T23:37:48.791 INFO:tasks.workunit.client.0.smithi136.stdout: "[512, 1023]", 2024-09-12T23:37:48.791 INFO:tasks.workunit.client.0.smithi136.stdout: "[1024, 2047]", 2024-09-12T23:37:48.791 INFO:tasks.workunit.client.0.smithi136.stdout: "[2048, 4095]", 2024-09-12T23:37:48.791 INFO:tasks.workunit.client.0.smithi136.stdout: "[4096, 8191]", 2024-09-12T23:37:48.791 INFO:tasks.workunit.client.0.smithi136.stdout: "[8192, 16383]", 2024-09-12T23:37:48.791 INFO:tasks.workunit.client.0.smithi136.stdout: "[16384, 32767]", 2024-09-12T23:37:48.791 INFO:tasks.workunit.client.0.smithi136.stdout: "[32768, 65535]", 2024-09-12T23:37:48.791 INFO:tasks.workunit.client.0.smithi136.stdout: "[65536, 131071]", 2024-09-12T23:37:48.792 INFO:tasks.workunit.client.0.smithi136.stdout: "[131072, 262143]", 2024-09-12T23:37:48.792 INFO:tasks.workunit.client.0.smithi136.stdout: "[262144, 524287]", 2024-09-12T23:37:48.792 INFO:tasks.workunit.client.0.smithi136.stdout: "[524288, 1048575]", 2024-09-12T23:37:48.792 INFO:tasks.workunit.client.0.smithi136.stdout: "[1048576, 2097151]", 2024-09-12T23:37:48.792 INFO:tasks.workunit.client.0.smithi136.stdout: "[2097152, 4194303]", 2024-09-12T23:37:48.792 INFO:tasks.workunit.client.0.smithi136.stdout: "[4194304, 8388607]", 2024-09-12T23:37:48.792 INFO:tasks.workunit.client.0.smithi136.stdout: "[8388608, 16777215]", 2024-09-12T23:37:48.792 INFO:tasks.workunit.client.0.smithi136.stdout: "[16777216, 33554431]", 2024-09-12T23:37:48.792 INFO:tasks.workunit.client.0.smithi136.stdout: "[33554432, 67108863]", 2024-09-12T23:37:48.792 INFO:tasks.workunit.client.0.smithi136.stdout: "[67108864, 134217727]", 2024-09-12T23:37:48.792 INFO:tasks.workunit.client.0.smithi136.stdout: "[134217728, 268435455]", 2024-09-12T23:37:48.792 INFO:tasks.workunit.client.0.smithi136.stdout: "[268435456, 536870911]", 2024-09-12T23:37:48.792 INFO:tasks.workunit.client.0.smithi136.stdout: "[536870912, 1073741823]", 2024-09-12T23:37:48.792 INFO:tasks.workunit.client.0.smithi136.stdout: "[1073741824, 2147483647]", 2024-09-12T23:37:48.793 INFO:tasks.workunit.client.0.smithi136.stdout: "[2147483648, 4294967295]", 2024-09-12T23:37:48.793 INFO:tasks.workunit.client.0.smithi136.stdout: "[4294967296, 8589934591]", 2024-09-12T23:37:48.793 INFO:tasks.workunit.client.0.smithi136.stdout: "[8589934592, 17179869183]", 2024-09-12T23:37:48.793 INFO:tasks.workunit.client.0.smithi136.stdout: "[17179869184, 34359738367]", 2024-09-12T23:37:48.793 INFO:tasks.workunit.client.0.smithi136.stdout: "[34359738368, 68719476735]", 2024-09-12T23:37:48.793 INFO:tasks.workunit.client.0.smithi136.stdout: "[68719476736, 137438953471]", 2024-09-12T23:37:48.793 INFO:tasks.workunit.client.0.smithi136.stdout: "[137438953472, 274877906943]", 2024-09-12T23:37:48.793 INFO:tasks.workunit.client.0.smithi136.stdout: "[274877906944, None]" 2024-09-12T23:37:48.793 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:48.793 INFO:tasks.workunit.client.0.smithi136.stdout: "scale_type": "log2" 2024-09-12T23:37:48.793 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.793 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:48.793 INFO:tasks.workunit.client.0.smithi136.stdout: "osds": [ 2024-09-12T23:37:48.794 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:48.794 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 0, 2024-09-12T23:37:48.794 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:48.794 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.794 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.794 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.794 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.794 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.794 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.794 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.794 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.794 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.794 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.794 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.795 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.795 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.795 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.795 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.795 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.795 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.795 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.795 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.795 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.795 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.795 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.795 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.795 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.796 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.796 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.796 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.796 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.796 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.796 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.796 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.796 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:48.796 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:48.796 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.796 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:48.796 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 1, 2024-09-12T23:37:48.796 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:48.797 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.797 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.797 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.797 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.797 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.797 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.797 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.797 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.797 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.797 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.797 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.797 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.798 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.798 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.798 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.798 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.798 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.798 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.798 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.798 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.798 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.798 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.798 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.798 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.798 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.798 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.798 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.799 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.799 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.799 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.799 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.799 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:48.799 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:48.799 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.799 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:48.799 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 2, 2024-09-12T23:37:48.799 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:48.799 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.799 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.799 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.800 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.800 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.800 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.800 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.800 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.800 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.800 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.800 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.800 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.800 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.800 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.800 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.800 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.800 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.801 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.801 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.801 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.801 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.801 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.801 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.801 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.801 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.801 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.801 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.801 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.801 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.801 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.801 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.802 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:48.802 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:48.802 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.802 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:48.802 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 3, 2024-09-12T23:37:48.802 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:48.802 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.802 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.802 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.802 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.802 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.802 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.802 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.802 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.803 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.803 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.803 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.803 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.803 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.803 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.803 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.803 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.803 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.803 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.803 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.803 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.803 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.803 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.804 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.804 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.804 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.804 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.804 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.804 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.804 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.804 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.804 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.804 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:48.804 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:48.804 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.805 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:48.805 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 4, 2024-09-12T23:37:48.805 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:48.805 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.805 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.805 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.805 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.805 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.805 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.805 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.805 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.805 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.806 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.806 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.806 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.806 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.806 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.806 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.806 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.806 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.806 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.806 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.806 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.806 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.806 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.806 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.807 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.807 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.807 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.807 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.807 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.807 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.807 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.807 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.807 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:48.807 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:48.807 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.807 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:48.808 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 5, 2024-09-12T23:37:48.808 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:48.808 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.808 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.808 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.808 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.808 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.808 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.808 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.808 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.808 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.808 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.808 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.808 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.808 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.809 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.809 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.809 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.809 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.809 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.809 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.809 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.809 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.809 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.809 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.809 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.810 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.810 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.810 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.810 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.810 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.810 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.810 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.810 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:48.810 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:48.810 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.810 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:48.810 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 6, 2024-09-12T23:37:48.811 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:48.811 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.811 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.811 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.811 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.811 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.811 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.811 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.811 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.811 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.811 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.811 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.811 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.811 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.811 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.812 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.812 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.812 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.812 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.812 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.812 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.812 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.812 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.812 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.812 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.812 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.812 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.812 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.813 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.813 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.813 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.813 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.813 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:48.813 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:48.813 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.813 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:48.813 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 7, 2024-09-12T23:37:48.813 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:48.813 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.813 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.813 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.814 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.814 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 33, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.814 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.814 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.814 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.814 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.814 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.814 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.814 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.814 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.814 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.814 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.814 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.814 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.815 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.815 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.815 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.815 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.815 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.815 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.815 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.815 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.815 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.815 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.815 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.815 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.815 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.816 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.816 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:48.816 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:48.816 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.816 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:48.816 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.816 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-12T23:37:48.816 INFO:tasks.workunit.client.0.smithi136.stdout: "axes": [ 2024-09-12T23:37:48.816 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:48.816 INFO:tasks.workunit.client.0.smithi136.stdout: "buckets": 32, 2024-09-12T23:37:48.816 INFO:tasks.workunit.client.0.smithi136.stdout: "min": 0, 2024-09-12T23:37:48.816 INFO:tasks.workunit.client.0.smithi136.stdout: "name": "Latency (usec)", 2024-09-12T23:37:48.816 INFO:tasks.workunit.client.0.smithi136.stdout: "quant_size": 100000, 2024-09-12T23:37:48.816 INFO:tasks.workunit.client.0.smithi136.stdout: "ranges": [ 2024-09-12T23:37:48.817 INFO:tasks.workunit.client.0.smithi136.stdout: "[None, -1]", 2024-09-12T23:37:48.817 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 99999]", 2024-09-12T23:37:48.817 INFO:tasks.workunit.client.0.smithi136.stdout: "[100000, 199999]", 2024-09-12T23:37:48.817 INFO:tasks.workunit.client.0.smithi136.stdout: "[200000, 399999]", 2024-09-12T23:37:48.817 INFO:tasks.workunit.client.0.smithi136.stdout: "[400000, 799999]", 2024-09-12T23:37:48.817 INFO:tasks.workunit.client.0.smithi136.stdout: "[800000, 1599999]", 2024-09-12T23:37:48.817 INFO:tasks.workunit.client.0.smithi136.stdout: "[1600000, 3199999]", 2024-09-12T23:37:48.817 INFO:tasks.workunit.client.0.smithi136.stdout: "[3200000, 6399999]", 2024-09-12T23:37:48.817 INFO:tasks.workunit.client.0.smithi136.stdout: "[6400000, 12799999]", 2024-09-12T23:37:48.817 INFO:tasks.workunit.client.0.smithi136.stdout: "[12800000, 25599999]", 2024-09-12T23:37:48.817 INFO:tasks.workunit.client.0.smithi136.stdout: "[25600000, 51199999]", 2024-09-12T23:37:48.817 INFO:tasks.workunit.client.0.smithi136.stdout: "[51200000, 102399999]", 2024-09-12T23:37:48.817 INFO:tasks.workunit.client.0.smithi136.stdout: "[102400000, 204799999]", 2024-09-12T23:37:48.818 INFO:tasks.workunit.client.0.smithi136.stdout: "[204800000, 409599999]", 2024-09-12T23:37:48.818 INFO:tasks.workunit.client.0.smithi136.stdout: "[409600000, 819199999]", 2024-09-12T23:37:48.818 INFO:tasks.workunit.client.0.smithi136.stdout: "[819200000, 1638399999]", 2024-09-12T23:37:48.818 INFO:tasks.workunit.client.0.smithi136.stdout: "[1638400000, 3276799999]", 2024-09-12T23:37:48.818 INFO:tasks.workunit.client.0.smithi136.stdout: "[3276800000, 6553599999]", 2024-09-12T23:37:48.818 INFO:tasks.workunit.client.0.smithi136.stdout: "[6553600000, 13107199999]", 2024-09-12T23:37:48.818 INFO:tasks.workunit.client.0.smithi136.stdout: "[13107200000, 26214399999]", 2024-09-12T23:37:48.818 INFO:tasks.workunit.client.0.smithi136.stdout: "[26214400000, 52428799999]", 2024-09-12T23:37:48.818 INFO:tasks.workunit.client.0.smithi136.stdout: "[52428800000, 104857599999]", 2024-09-12T23:37:48.818 INFO:tasks.workunit.client.0.smithi136.stdout: "[104857600000, 209715199999]", 2024-09-12T23:37:48.818 INFO:tasks.workunit.client.0.smithi136.stdout: "[209715200000, 419430399999]", 2024-09-12T23:37:48.818 INFO:tasks.workunit.client.0.smithi136.stdout: "[419430400000, 838860799999]", 2024-09-12T23:37:48.818 INFO:tasks.workunit.client.0.smithi136.stdout: "[838860800000, 1677721599999]", 2024-09-12T23:37:48.818 INFO:tasks.workunit.client.0.smithi136.stdout: "[1677721600000, 3355443199999]", 2024-09-12T23:37:48.819 INFO:tasks.workunit.client.0.smithi136.stdout: "[3355443200000, 6710886399999]", 2024-09-12T23:37:48.819 INFO:tasks.workunit.client.0.smithi136.stdout: "[6710886400000, 13421772799999]", 2024-09-12T23:37:48.819 INFO:tasks.workunit.client.0.smithi136.stdout: "[13421772800000, 26843545599999]", 2024-09-12T23:37:48.819 INFO:tasks.workunit.client.0.smithi136.stdout: "[26843545600000, 53687091199999]", 2024-09-12T23:37:48.819 INFO:tasks.workunit.client.0.smithi136.stdout: "[53687091200000, None]" 2024-09-12T23:37:48.819 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:48.819 INFO:tasks.workunit.client.0.smithi136.stdout: "scale_type": "log2" 2024-09-12T23:37:48.819 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.819 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:48.819 INFO:tasks.workunit.client.0.smithi136.stdout: "buckets": 32, 2024-09-12T23:37:48.819 INFO:tasks.workunit.client.0.smithi136.stdout: "min": 0, 2024-09-12T23:37:48.819 INFO:tasks.workunit.client.0.smithi136.stdout: "name": "Request size (bytes)", 2024-09-12T23:37:48.819 INFO:tasks.workunit.client.0.smithi136.stdout: "quant_size": 512, 2024-09-12T23:37:48.819 INFO:tasks.workunit.client.0.smithi136.stdout: "ranges": [ 2024-09-12T23:37:48.820 INFO:tasks.workunit.client.0.smithi136.stdout: "[None, -1]", 2024-09-12T23:37:48.820 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 511]", 2024-09-12T23:37:48.820 INFO:tasks.workunit.client.0.smithi136.stdout: "[512, 1023]", 2024-09-12T23:37:48.820 INFO:tasks.workunit.client.0.smithi136.stdout: "[1024, 2047]", 2024-09-12T23:37:48.820 INFO:tasks.workunit.client.0.smithi136.stdout: "[2048, 4095]", 2024-09-12T23:37:48.820 INFO:tasks.workunit.client.0.smithi136.stdout: "[4096, 8191]", 2024-09-12T23:37:48.820 INFO:tasks.workunit.client.0.smithi136.stdout: "[8192, 16383]", 2024-09-12T23:37:48.820 INFO:tasks.workunit.client.0.smithi136.stdout: "[16384, 32767]", 2024-09-12T23:37:48.820 INFO:tasks.workunit.client.0.smithi136.stdout: "[32768, 65535]", 2024-09-12T23:37:48.820 INFO:tasks.workunit.client.0.smithi136.stdout: "[65536, 131071]", 2024-09-12T23:37:48.820 INFO:tasks.workunit.client.0.smithi136.stdout: "[131072, 262143]", 2024-09-12T23:37:48.820 INFO:tasks.workunit.client.0.smithi136.stdout: "[262144, 524287]", 2024-09-12T23:37:48.820 INFO:tasks.workunit.client.0.smithi136.stdout: "[524288, 1048575]", 2024-09-12T23:37:48.821 INFO:tasks.workunit.client.0.smithi136.stdout: "[1048576, 2097151]", 2024-09-12T23:37:48.821 INFO:tasks.workunit.client.0.smithi136.stdout: "[2097152, 4194303]", 2024-09-12T23:37:48.821 INFO:tasks.workunit.client.0.smithi136.stdout: "[4194304, 8388607]", 2024-09-12T23:37:48.821 INFO:tasks.workunit.client.0.smithi136.stdout: "[8388608, 16777215]", 2024-09-12T23:37:48.821 INFO:tasks.workunit.client.0.smithi136.stdout: "[16777216, 33554431]", 2024-09-12T23:37:48.821 INFO:tasks.workunit.client.0.smithi136.stdout: "[33554432, 67108863]", 2024-09-12T23:37:48.821 INFO:tasks.workunit.client.0.smithi136.stdout: "[67108864, 134217727]", 2024-09-12T23:37:48.821 INFO:tasks.workunit.client.0.smithi136.stdout: "[134217728, 268435455]", 2024-09-12T23:37:48.821 INFO:tasks.workunit.client.0.smithi136.stdout: "[268435456, 536870911]", 2024-09-12T23:37:48.821 INFO:tasks.workunit.client.0.smithi136.stdout: "[536870912, 1073741823]", 2024-09-12T23:37:48.821 INFO:tasks.workunit.client.0.smithi136.stdout: "[1073741824, 2147483647]", 2024-09-12T23:37:48.821 INFO:tasks.workunit.client.0.smithi136.stdout: "[2147483648, 4294967295]", 2024-09-12T23:37:48.821 INFO:tasks.workunit.client.0.smithi136.stdout: "[4294967296, 8589934591]", 2024-09-12T23:37:48.822 INFO:tasks.workunit.client.0.smithi136.stdout: "[8589934592, 17179869183]", 2024-09-12T23:37:48.822 INFO:tasks.workunit.client.0.smithi136.stdout: "[17179869184, 34359738367]", 2024-09-12T23:37:48.822 INFO:tasks.workunit.client.0.smithi136.stdout: "[34359738368, 68719476735]", 2024-09-12T23:37:48.822 INFO:tasks.workunit.client.0.smithi136.stdout: "[68719476736, 137438953471]", 2024-09-12T23:37:48.822 INFO:tasks.workunit.client.0.smithi136.stdout: "[137438953472, 274877906943]", 2024-09-12T23:37:48.822 INFO:tasks.workunit.client.0.smithi136.stdout: "[274877906944, None]" 2024-09-12T23:37:48.822 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:48.822 INFO:tasks.workunit.client.0.smithi136.stdout: "scale_type": "log2" 2024-09-12T23:37:48.822 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.822 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:48.822 INFO:tasks.workunit.client.0.smithi136.stdout: "osds": [ 2024-09-12T23:37:48.822 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:48.822 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 0, 2024-09-12T23:37:48.823 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:48.823 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.823 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.823 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.823 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.823 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.823 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.823 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.823 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.823 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.823 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.823 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.823 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.823 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.824 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.824 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.824 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.824 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.824 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.824 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.824 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.824 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.824 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.824 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.824 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.824 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.824 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.825 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.825 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.825 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.825 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.825 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.825 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:48.825 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:48.825 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.825 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:48.825 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 1, 2024-09-12T23:37:48.825 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:48.825 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.825 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.825 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.826 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.826 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.826 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.826 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.826 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.826 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.826 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.826 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.826 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.826 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.826 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.826 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.826 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.827 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.827 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.827 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.827 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.827 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.827 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.827 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.827 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.827 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.827 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.827 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.827 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.827 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.827 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.828 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.828 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:48.828 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:48.828 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.828 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:48.828 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 2, 2024-09-12T23:37:48.828 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:48.828 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.828 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.828 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.828 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.828 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.828 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.828 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.828 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.829 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.829 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.829 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.829 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.829 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.829 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.829 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.829 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.829 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.829 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.829 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.829 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.830 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.830 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.830 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.830 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.830 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.830 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.830 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.830 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.830 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.830 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.830 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.830 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:48.830 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:48.831 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.831 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:48.831 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 3, 2024-09-12T23:37:48.831 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:48.831 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.831 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.831 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.831 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.831 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.831 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.831 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.831 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.832 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.832 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.832 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.832 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.832 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.832 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.832 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.832 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.832 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.832 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.832 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.832 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.832 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.832 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.833 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.833 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.833 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.833 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.833 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.833 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.833 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.833 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.833 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.833 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:48.833 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:48.833 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.833 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:48.834 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 4, 2024-09-12T23:37:48.834 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:48.834 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.834 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.834 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.834 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.834 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.834 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.834 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.834 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.834 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.834 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.834 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.834 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.835 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.835 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.835 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.835 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.835 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.835 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.835 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.835 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.835 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.835 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.835 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.835 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.835 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.836 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.836 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.836 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.836 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.836 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.836 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.836 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:48.836 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:48.836 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.836 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:48.836 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 5, 2024-09-12T23:37:48.836 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:48.836 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.837 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.837 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.837 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.837 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.837 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.837 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.837 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.837 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.837 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.837 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.837 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.837 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.837 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.837 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.838 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.838 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.838 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.838 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.838 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.838 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.838 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.838 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.838 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.838 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.838 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.838 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.838 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.838 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.839 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.839 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.839 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:48.839 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:48.839 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.839 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:48.839 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 6, 2024-09-12T23:37:48.839 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:48.839 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.839 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.839 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.839 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.839 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.839 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.840 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.840 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.840 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.840 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.840 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.840 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.840 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.840 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.840 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.840 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.840 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.840 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.841 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.841 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.841 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.841 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.841 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.841 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.841 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.841 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.841 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.841 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.841 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.841 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.841 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.842 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:48.842 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:48.842 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.842 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:48.842 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 7, 2024-09-12T23:37:48.842 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:48.842 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.842 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.842 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.842 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.842 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.842 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.842 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.843 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.843 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.843 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.843 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.843 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.843 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.843 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.843 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.843 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.843 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.843 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.843 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.843 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.843 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.844 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.844 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.844 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.844 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.844 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.844 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.844 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.844 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.844 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.844 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.844 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:48.844 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:48.844 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.845 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:48.845 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.845 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-12T23:37:48.845 INFO:tasks.workunit.client.0.smithi136.stdout: "axes": [ 2024-09-12T23:37:48.845 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:48.845 INFO:tasks.workunit.client.0.smithi136.stdout: "buckets": 32, 2024-09-12T23:37:48.845 INFO:tasks.workunit.client.0.smithi136.stdout: "min": 0, 2024-09-12T23:37:48.845 INFO:tasks.workunit.client.0.smithi136.stdout: "name": "Latency (usec)", 2024-09-12T23:37:48.845 INFO:tasks.workunit.client.0.smithi136.stdout: "quant_size": 100000, 2024-09-12T23:37:48.845 INFO:tasks.workunit.client.0.smithi136.stdout: "ranges": [ 2024-09-12T23:37:48.845 INFO:tasks.workunit.client.0.smithi136.stdout: "[None, -1]", 2024-09-12T23:37:48.845 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 99999]", 2024-09-12T23:37:48.845 INFO:tasks.workunit.client.0.smithi136.stdout: "[100000, 199999]", 2024-09-12T23:37:48.845 INFO:tasks.workunit.client.0.smithi136.stdout: "[200000, 399999]", 2024-09-12T23:37:48.846 INFO:tasks.workunit.client.0.smithi136.stdout: "[400000, 799999]", 2024-09-12T23:37:48.846 INFO:tasks.workunit.client.0.smithi136.stdout: "[800000, 1599999]", 2024-09-12T23:37:48.846 INFO:tasks.workunit.client.0.smithi136.stdout: "[1600000, 3199999]", 2024-09-12T23:37:48.846 INFO:tasks.workunit.client.0.smithi136.stdout: "[3200000, 6399999]", 2024-09-12T23:37:48.846 INFO:tasks.workunit.client.0.smithi136.stdout: "[6400000, 12799999]", 2024-09-12T23:37:48.846 INFO:tasks.workunit.client.0.smithi136.stdout: "[12800000, 25599999]", 2024-09-12T23:37:48.846 INFO:tasks.workunit.client.0.smithi136.stdout: "[25600000, 51199999]", 2024-09-12T23:37:48.846 INFO:tasks.workunit.client.0.smithi136.stdout: "[51200000, 102399999]", 2024-09-12T23:37:48.846 INFO:tasks.workunit.client.0.smithi136.stdout: "[102400000, 204799999]", 2024-09-12T23:37:48.846 INFO:tasks.workunit.client.0.smithi136.stdout: "[204800000, 409599999]", 2024-09-12T23:37:48.846 INFO:tasks.workunit.client.0.smithi136.stdout: "[409600000, 819199999]", 2024-09-12T23:37:48.846 INFO:tasks.workunit.client.0.smithi136.stdout: "[819200000, 1638399999]", 2024-09-12T23:37:48.846 INFO:tasks.workunit.client.0.smithi136.stdout: "[1638400000, 3276799999]", 2024-09-12T23:37:48.846 INFO:tasks.workunit.client.0.smithi136.stdout: "[3276800000, 6553599999]", 2024-09-12T23:37:48.846 INFO:tasks.workunit.client.0.smithi136.stdout: "[6553600000, 13107199999]", 2024-09-12T23:37:48.847 INFO:tasks.workunit.client.0.smithi136.stdout: "[13107200000, 26214399999]", 2024-09-12T23:37:48.847 INFO:tasks.workunit.client.0.smithi136.stdout: "[26214400000, 52428799999]", 2024-09-12T23:37:48.847 INFO:tasks.workunit.client.0.smithi136.stdout: "[52428800000, 104857599999]", 2024-09-12T23:37:48.847 INFO:tasks.workunit.client.0.smithi136.stdout: "[104857600000, 209715199999]", 2024-09-12T23:37:48.847 INFO:tasks.workunit.client.0.smithi136.stdout: "[209715200000, 419430399999]", 2024-09-12T23:37:48.847 INFO:tasks.workunit.client.0.smithi136.stdout: "[419430400000, 838860799999]", 2024-09-12T23:37:48.847 INFO:tasks.workunit.client.0.smithi136.stdout: "[838860800000, 1677721599999]", 2024-09-12T23:37:48.847 INFO:tasks.workunit.client.0.smithi136.stdout: "[1677721600000, 3355443199999]", 2024-09-12T23:37:48.847 INFO:tasks.workunit.client.0.smithi136.stdout: "[3355443200000, 6710886399999]", 2024-09-12T23:37:48.847 INFO:tasks.workunit.client.0.smithi136.stdout: "[6710886400000, 13421772799999]", 2024-09-12T23:37:48.847 INFO:tasks.workunit.client.0.smithi136.stdout: "[13421772800000, 26843545599999]", 2024-09-12T23:37:48.847 INFO:tasks.workunit.client.0.smithi136.stdout: "[26843545600000, 53687091199999]", 2024-09-12T23:37:48.847 INFO:tasks.workunit.client.0.smithi136.stdout: "[53687091200000, None]" 2024-09-12T23:37:48.848 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:48.848 INFO:tasks.workunit.client.0.smithi136.stdout: "scale_type": "log2" 2024-09-12T23:37:48.848 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.848 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:48.848 INFO:tasks.workunit.client.0.smithi136.stdout: "buckets": 32, 2024-09-12T23:37:48.848 INFO:tasks.workunit.client.0.smithi136.stdout: "min": 0, 2024-09-12T23:37:48.848 INFO:tasks.workunit.client.0.smithi136.stdout: "name": "Request size (bytes)", 2024-09-12T23:37:48.848 INFO:tasks.workunit.client.0.smithi136.stdout: "quant_size": 512, 2024-09-12T23:37:48.848 INFO:tasks.workunit.client.0.smithi136.stdout: "ranges": [ 2024-09-12T23:37:48.848 INFO:tasks.workunit.client.0.smithi136.stdout: "[None, -1]", 2024-09-12T23:37:48.848 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 511]", 2024-09-12T23:37:48.848 INFO:tasks.workunit.client.0.smithi136.stdout: "[512, 1023]", 2024-09-12T23:37:48.848 INFO:tasks.workunit.client.0.smithi136.stdout: "[1024, 2047]", 2024-09-12T23:37:48.849 INFO:tasks.workunit.client.0.smithi136.stdout: "[2048, 4095]", 2024-09-12T23:37:48.849 INFO:tasks.workunit.client.0.smithi136.stdout: "[4096, 8191]", 2024-09-12T23:37:48.849 INFO:tasks.workunit.client.0.smithi136.stdout: "[8192, 16383]", 2024-09-12T23:37:48.849 INFO:tasks.workunit.client.0.smithi136.stdout: "[16384, 32767]", 2024-09-12T23:37:48.849 INFO:tasks.workunit.client.0.smithi136.stdout: "[32768, 65535]", 2024-09-12T23:37:48.849 INFO:tasks.workunit.client.0.smithi136.stdout: "[65536, 131071]", 2024-09-12T23:37:48.849 INFO:tasks.workunit.client.0.smithi136.stdout: "[131072, 262143]", 2024-09-12T23:37:48.849 INFO:tasks.workunit.client.0.smithi136.stdout: "[262144, 524287]", 2024-09-12T23:37:48.849 INFO:tasks.workunit.client.0.smithi136.stdout: "[524288, 1048575]", 2024-09-12T23:37:48.849 INFO:tasks.workunit.client.0.smithi136.stdout: "[1048576, 2097151]", 2024-09-12T23:37:48.849 INFO:tasks.workunit.client.0.smithi136.stdout: "[2097152, 4194303]", 2024-09-12T23:37:48.849 INFO:tasks.workunit.client.0.smithi136.stdout: "[4194304, 8388607]", 2024-09-12T23:37:48.849 INFO:tasks.workunit.client.0.smithi136.stdout: "[8388608, 16777215]", 2024-09-12T23:37:48.850 INFO:tasks.workunit.client.0.smithi136.stdout: "[16777216, 33554431]", 2024-09-12T23:37:48.850 INFO:tasks.workunit.client.0.smithi136.stdout: "[33554432, 67108863]", 2024-09-12T23:37:48.850 INFO:tasks.workunit.client.0.smithi136.stdout: "[67108864, 134217727]", 2024-09-12T23:37:48.850 INFO:tasks.workunit.client.0.smithi136.stdout: "[134217728, 268435455]", 2024-09-12T23:37:48.850 INFO:tasks.workunit.client.0.smithi136.stdout: "[268435456, 536870911]", 2024-09-12T23:37:48.850 INFO:tasks.workunit.client.0.smithi136.stdout: "[536870912, 1073741823]", 2024-09-12T23:37:48.850 INFO:tasks.workunit.client.0.smithi136.stdout: "[1073741824, 2147483647]", 2024-09-12T23:37:48.850 INFO:tasks.workunit.client.0.smithi136.stdout: "[2147483648, 4294967295]", 2024-09-12T23:37:48.850 INFO:tasks.workunit.client.0.smithi136.stdout: "[4294967296, 8589934591]", 2024-09-12T23:37:48.850 INFO:tasks.workunit.client.0.smithi136.stdout: "[8589934592, 17179869183]", 2024-09-12T23:37:48.850 INFO:tasks.workunit.client.0.smithi136.stdout: "[17179869184, 34359738367]", 2024-09-12T23:37:48.850 INFO:tasks.workunit.client.0.smithi136.stdout: "[34359738368, 68719476735]", 2024-09-12T23:37:48.850 INFO:tasks.workunit.client.0.smithi136.stdout: "[68719476736, 137438953471]", 2024-09-12T23:37:48.850 INFO:tasks.workunit.client.0.smithi136.stdout: "[137438953472, 274877906943]", 2024-09-12T23:37:48.851 INFO:tasks.workunit.client.0.smithi136.stdout: "[274877906944, None]" 2024-09-12T23:37:48.851 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:48.851 INFO:tasks.workunit.client.0.smithi136.stdout: "scale_type": "log2" 2024-09-12T23:37:48.851 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.851 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:48.851 INFO:tasks.workunit.client.0.smithi136.stdout: "osds": [ 2024-09-12T23:37:48.851 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:48.851 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 0, 2024-09-12T23:37:48.851 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:48.851 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.851 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.851 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.851 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.852 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.852 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.852 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.852 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.852 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.852 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.852 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.852 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.852 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.852 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.852 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.852 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.852 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.853 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.853 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.853 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.853 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.853 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.853 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.853 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.853 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.853 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.853 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.853 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.853 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.853 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.854 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.854 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:48.854 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:48.854 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.854 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:48.854 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 1, 2024-09-12T23:37:48.854 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:48.854 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.854 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.854 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.854 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.854 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.854 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.854 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.855 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.855 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.855 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.855 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.855 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.855 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.855 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.855 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.855 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.855 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.855 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.855 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.856 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.856 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.856 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.856 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.856 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.856 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.856 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.856 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.856 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.856 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.856 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.856 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.856 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:48.856 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:48.857 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.857 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:48.857 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 2, 2024-09-12T23:37:48.857 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:48.857 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.857 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.857 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.857 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.857 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.857 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.857 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.857 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.857 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.857 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.858 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.858 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.858 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.858 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.858 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.858 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.858 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.858 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.858 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.858 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.858 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.858 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.859 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.859 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.859 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.859 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.859 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.859 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.859 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.859 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.859 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.859 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:48.860 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:48.860 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.860 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:48.860 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 3, 2024-09-12T23:37:48.860 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:48.860 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.860 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.860 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.860 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.861 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.861 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.861 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.861 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.861 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.861 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.861 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.861 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.862 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.862 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.862 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.862 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.862 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.862 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.862 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.862 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.862 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.862 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.862 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.862 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.862 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.863 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.863 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.863 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.863 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.863 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.863 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.863 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:48.863 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:48.863 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.863 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:48.863 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 4, 2024-09-12T23:37:48.864 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:48.864 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.864 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.864 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.864 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.864 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.864 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.864 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.864 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.864 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.864 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.864 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.864 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.864 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.865 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.865 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.865 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.865 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.865 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.865 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.865 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.865 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.865 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.865 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.865 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.865 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.865 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.865 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.866 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.866 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.866 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.866 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.866 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:48.866 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:48.866 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.866 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:48.866 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 5, 2024-09-12T23:37:48.866 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:48.866 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.866 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.866 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.867 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.867 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.867 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.867 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.867 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.867 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.867 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.867 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.868 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.868 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.868 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.868 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.868 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.868 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.868 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.868 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.868 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.869 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.869 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.869 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.869 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.869 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.869 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.869 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.869 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.869 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.869 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.869 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.869 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:48.869 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:48.869 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.870 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:48.870 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 6, 2024-09-12T23:37:48.870 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:48.870 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.870 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.870 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.870 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.870 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.870 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.870 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.870 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.871 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.871 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.871 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.871 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.871 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.871 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.871 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.871 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.871 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.871 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.871 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.872 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.872 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.872 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.872 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.872 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.872 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.872 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.872 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.872 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.872 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.873 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.873 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.873 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:48.873 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:48.873 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.873 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:48.873 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 7, 2024-09-12T23:37:48.873 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:48.873 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.873 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.873 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.874 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.874 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.874 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.874 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.874 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.874 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.874 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.874 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.874 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.874 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.874 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.875 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.875 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.875 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.875 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.875 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.875 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.875 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.875 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.875 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.875 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.875 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.875 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.875 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.875 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.876 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.876 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.876 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.876 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:48.876 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:48.876 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.876 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:48.876 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.876 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-12T23:37:48.876 INFO:tasks.workunit.client.0.smithi136.stdout: "axes": [ 2024-09-12T23:37:48.876 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:48.876 INFO:tasks.workunit.client.0.smithi136.stdout: "buckets": 32, 2024-09-12T23:37:48.876 INFO:tasks.workunit.client.0.smithi136.stdout: "min": 0, 2024-09-12T23:37:48.877 INFO:tasks.workunit.client.0.smithi136.stdout: "name": "Latency (usec)", 2024-09-12T23:37:48.877 INFO:tasks.workunit.client.0.smithi136.stdout: "quant_size": 100000, 2024-09-12T23:37:48.877 INFO:tasks.workunit.client.0.smithi136.stdout: "ranges": [ 2024-09-12T23:37:48.877 INFO:tasks.workunit.client.0.smithi136.stdout: "[None, -1]", 2024-09-12T23:37:48.877 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 99999]", 2024-09-12T23:37:48.877 INFO:tasks.workunit.client.0.smithi136.stdout: "[100000, 199999]", 2024-09-12T23:37:48.877 INFO:tasks.workunit.client.0.smithi136.stdout: "[200000, 399999]", 2024-09-12T23:37:48.877 INFO:tasks.workunit.client.0.smithi136.stdout: "[400000, 799999]", 2024-09-12T23:37:48.877 INFO:tasks.workunit.client.0.smithi136.stdout: "[800000, 1599999]", 2024-09-12T23:37:48.877 INFO:tasks.workunit.client.0.smithi136.stdout: "[1600000, 3199999]", 2024-09-12T23:37:48.878 INFO:tasks.workunit.client.0.smithi136.stdout: "[3200000, 6399999]", 2024-09-12T23:37:48.878 INFO:tasks.workunit.client.0.smithi136.stdout: "[6400000, 12799999]", 2024-09-12T23:37:48.878 INFO:tasks.workunit.client.0.smithi136.stdout: "[12800000, 25599999]", 2024-09-12T23:37:48.878 INFO:tasks.workunit.client.0.smithi136.stdout: "[25600000, 51199999]", 2024-09-12T23:37:48.878 INFO:tasks.workunit.client.0.smithi136.stdout: "[51200000, 102399999]", 2024-09-12T23:37:48.878 INFO:tasks.workunit.client.0.smithi136.stdout: "[102400000, 204799999]", 2024-09-12T23:37:48.878 INFO:tasks.workunit.client.0.smithi136.stdout: "[204800000, 409599999]", 2024-09-12T23:37:48.878 INFO:tasks.workunit.client.0.smithi136.stdout: "[409600000, 819199999]", 2024-09-12T23:37:48.878 INFO:tasks.workunit.client.0.smithi136.stdout: "[819200000, 1638399999]", 2024-09-12T23:37:48.878 INFO:tasks.workunit.client.0.smithi136.stdout: "[1638400000, 3276799999]", 2024-09-12T23:37:48.878 INFO:tasks.workunit.client.0.smithi136.stdout: "[3276800000, 6553599999]", 2024-09-12T23:37:48.878 INFO:tasks.workunit.client.0.smithi136.stdout: "[6553600000, 13107199999]", 2024-09-12T23:37:48.878 INFO:tasks.workunit.client.0.smithi136.stdout: "[13107200000, 26214399999]", 2024-09-12T23:37:48.878 INFO:tasks.workunit.client.0.smithi136.stdout: "[26214400000, 52428799999]", 2024-09-12T23:37:48.879 INFO:tasks.workunit.client.0.smithi136.stdout: "[52428800000, 104857599999]", 2024-09-12T23:37:48.879 INFO:tasks.workunit.client.0.smithi136.stdout: "[104857600000, 209715199999]", 2024-09-12T23:37:48.879 INFO:tasks.workunit.client.0.smithi136.stdout: "[209715200000, 419430399999]", 2024-09-12T23:37:48.879 INFO:tasks.workunit.client.0.smithi136.stdout: "[419430400000, 838860799999]", 2024-09-12T23:37:48.879 INFO:tasks.workunit.client.0.smithi136.stdout: "[838860800000, 1677721599999]", 2024-09-12T23:37:48.879 INFO:tasks.workunit.client.0.smithi136.stdout: "[1677721600000, 3355443199999]", 2024-09-12T23:37:48.879 INFO:tasks.workunit.client.0.smithi136.stdout: "[3355443200000, 6710886399999]", 2024-09-12T23:37:48.879 INFO:tasks.workunit.client.0.smithi136.stdout: "[6710886400000, 13421772799999]", 2024-09-12T23:37:48.879 INFO:tasks.workunit.client.0.smithi136.stdout: "[13421772800000, 26843545599999]", 2024-09-12T23:37:48.879 INFO:tasks.workunit.client.0.smithi136.stdout: "[26843545600000, 53687091199999]", 2024-09-12T23:37:48.879 INFO:tasks.workunit.client.0.smithi136.stdout: "[53687091200000, None]" 2024-09-12T23:37:48.879 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:48.879 INFO:tasks.workunit.client.0.smithi136.stdout: "scale_type": "log2" 2024-09-12T23:37:48.879 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.880 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:48.880 INFO:tasks.workunit.client.0.smithi136.stdout: "buckets": 32, 2024-09-12T23:37:48.880 INFO:tasks.workunit.client.0.smithi136.stdout: "min": 0, 2024-09-12T23:37:48.880 INFO:tasks.workunit.client.0.smithi136.stdout: "name": "Request size (bytes)", 2024-09-12T23:37:48.880 INFO:tasks.workunit.client.0.smithi136.stdout: "quant_size": 512, 2024-09-12T23:37:48.880 INFO:tasks.workunit.client.0.smithi136.stdout: "ranges": [ 2024-09-12T23:37:48.880 INFO:tasks.workunit.client.0.smithi136.stdout: "[None, -1]", 2024-09-12T23:37:48.880 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 511]", 2024-09-12T23:37:48.880 INFO:tasks.workunit.client.0.smithi136.stdout: "[512, 1023]", 2024-09-12T23:37:48.880 INFO:tasks.workunit.client.0.smithi136.stdout: "[1024, 2047]", 2024-09-12T23:37:48.880 INFO:tasks.workunit.client.0.smithi136.stdout: "[2048, 4095]", 2024-09-12T23:37:48.880 INFO:tasks.workunit.client.0.smithi136.stdout: "[4096, 8191]", 2024-09-12T23:37:48.880 INFO:tasks.workunit.client.0.smithi136.stdout: "[8192, 16383]", 2024-09-12T23:37:48.881 INFO:tasks.workunit.client.0.smithi136.stdout: "[16384, 32767]", 2024-09-12T23:37:48.881 INFO:tasks.workunit.client.0.smithi136.stdout: "[32768, 65535]", 2024-09-12T23:37:48.881 INFO:tasks.workunit.client.0.smithi136.stdout: "[65536, 131071]", 2024-09-12T23:37:48.881 INFO:tasks.workunit.client.0.smithi136.stdout: "[131072, 262143]", 2024-09-12T23:37:48.881 INFO:tasks.workunit.client.0.smithi136.stdout: "[262144, 524287]", 2024-09-12T23:37:48.881 INFO:tasks.workunit.client.0.smithi136.stdout: "[524288, 1048575]", 2024-09-12T23:37:48.881 INFO:tasks.workunit.client.0.smithi136.stdout: "[1048576, 2097151]", 2024-09-12T23:37:48.881 INFO:tasks.workunit.client.0.smithi136.stdout: "[2097152, 4194303]", 2024-09-12T23:37:48.881 INFO:tasks.workunit.client.0.smithi136.stdout: "[4194304, 8388607]", 2024-09-12T23:37:48.881 INFO:tasks.workunit.client.0.smithi136.stdout: "[8388608, 16777215]", 2024-09-12T23:37:48.882 INFO:tasks.workunit.client.0.smithi136.stdout: "[16777216, 33554431]", 2024-09-12T23:37:48.882 INFO:tasks.workunit.client.0.smithi136.stdout: "[33554432, 67108863]", 2024-09-12T23:37:48.882 INFO:tasks.workunit.client.0.smithi136.stdout: "[67108864, 134217727]", 2024-09-12T23:37:48.882 INFO:tasks.workunit.client.0.smithi136.stdout: "[134217728, 268435455]", 2024-09-12T23:37:48.882 INFO:tasks.workunit.client.0.smithi136.stdout: "[268435456, 536870911]", 2024-09-12T23:37:48.882 INFO:tasks.workunit.client.0.smithi136.stdout: "[536870912, 1073741823]", 2024-09-12T23:37:48.882 INFO:tasks.workunit.client.0.smithi136.stdout: "[1073741824, 2147483647]", 2024-09-12T23:37:48.882 INFO:tasks.workunit.client.0.smithi136.stdout: "[2147483648, 4294967295]", 2024-09-12T23:37:48.882 INFO:tasks.workunit.client.0.smithi136.stdout: "[4294967296, 8589934591]", 2024-09-12T23:37:48.882 INFO:tasks.workunit.client.0.smithi136.stdout: "[8589934592, 17179869183]", 2024-09-12T23:37:48.882 INFO:tasks.workunit.client.0.smithi136.stdout: "[17179869184, 34359738367]", 2024-09-12T23:37:48.882 INFO:tasks.workunit.client.0.smithi136.stdout: "[34359738368, 68719476735]", 2024-09-12T23:37:48.882 INFO:tasks.workunit.client.0.smithi136.stdout: "[68719476736, 137438953471]", 2024-09-12T23:37:48.882 INFO:tasks.workunit.client.0.smithi136.stdout: "[137438953472, 274877906943]", 2024-09-12T23:37:48.883 INFO:tasks.workunit.client.0.smithi136.stdout: "[274877906944, None]" 2024-09-12T23:37:48.883 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:48.883 INFO:tasks.workunit.client.0.smithi136.stdout: "scale_type": "log2" 2024-09-12T23:37:48.883 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.883 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:48.883 INFO:tasks.workunit.client.0.smithi136.stdout: "osds": [ 2024-09-12T23:37:48.883 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:48.883 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 0, 2024-09-12T23:37:48.883 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:48.883 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.883 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.883 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.883 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.884 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.884 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.884 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.884 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.884 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.884 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.884 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.884 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.884 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.884 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.884 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.885 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.885 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.885 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.885 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.885 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.885 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.885 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.885 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.885 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.885 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.885 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.886 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.886 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.886 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.886 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.886 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.886 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:48.886 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:48.886 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.888 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:48.888 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 1, 2024-09-12T23:37:48.888 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:48.888 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.888 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.888 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.888 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.888 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.888 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.888 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 7, 2, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.888 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 10, 2, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.889 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 10, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.889 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 8, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.889 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.889 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.889 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.889 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.889 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.889 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.889 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.889 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.889 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.889 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.889 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.890 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.890 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.890 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.890 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.890 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.890 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.890 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.890 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.890 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.890 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.891 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:48.891 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:48.891 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.891 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:48.891 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 2, 2024-09-12T23:37:48.891 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:48.891 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.891 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.891 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.891 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.891 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.891 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.891 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.891 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.892 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.892 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.892 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.892 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.892 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.892 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.892 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.892 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.892 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.892 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.892 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.892 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.892 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.893 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.893 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.893 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.893 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.893 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.893 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.893 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.893 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.893 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.893 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.893 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:48.893 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:48.893 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.894 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:48.894 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 3, 2024-09-12T23:37:48.894 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:48.894 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.894 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.894 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.894 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.894 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.894 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.894 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.894 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.894 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.894 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.894 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.895 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.895 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.895 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.895 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.895 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.895 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.895 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.895 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.895 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.895 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.895 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.895 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.895 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.896 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.896 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.896 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.896 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.896 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.896 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.896 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.896 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:48.896 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:48.896 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.896 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:48.897 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 4, 2024-09-12T23:37:48.897 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:48.897 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.897 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.897 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.897 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.897 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.897 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.897 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.897 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.897 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.898 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.898 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.898 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.898 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.898 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.898 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.898 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.898 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.898 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.898 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.898 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.898 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.898 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.899 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.899 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.899 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.899 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.899 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.899 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.899 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.899 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.899 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.899 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:48.899 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:48.899 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.899 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:48.899 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 5, 2024-09-12T23:37:48.900 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:48.900 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.900 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.900 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.900 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.900 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.900 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.900 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.900 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.900 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.900 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.900 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.900 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.901 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.901 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.901 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.901 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.901 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.901 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.901 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.901 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.901 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.901 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.901 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.901 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.901 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.901 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.902 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.902 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.902 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.902 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.902 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.902 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:48.902 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:48.902 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.902 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:48.902 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 6, 2024-09-12T23:37:48.902 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:48.902 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.902 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.903 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.903 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.903 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.903 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.903 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.903 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.903 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.903 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.903 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.903 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.903 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.903 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.903 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.903 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.904 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.904 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.904 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.904 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.904 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.904 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.904 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.904 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.904 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.904 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.904 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.904 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.904 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.904 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.905 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.905 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:48.905 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:48.905 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.905 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:48.905 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 7, 2024-09-12T23:37:48.905 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:48.905 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.905 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.905 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.906 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.906 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.906 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.906 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.906 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.906 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.906 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.906 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.906 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.906 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.906 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.907 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.907 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.907 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.907 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.907 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.907 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.907 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.907 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.907 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.907 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.907 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.907 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.907 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.908 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.908 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.908 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.908 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:48.908 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:48.908 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:48.908 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.908 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:48.908 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.908 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.908 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:48.908 INFO:tasks.workunit.client.0.smithi136.stdout: "perf_counters": { 2024-09-12T23:37:48.908 INFO:tasks.workunit.client.0.smithi136.stdout: "mds.4585d541b331f263094839d9f58dd50a20d3f517": { 2024-09-12T23:37:48.908 INFO:tasks.workunit.client.0.smithi136.stdout: "mds": { 2024-09-12T23:37:48.909 INFO:tasks.workunit.client.0.smithi136.stdout: "caps": { 2024-09-12T23:37:48.909 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.909 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.909 INFO:tasks.workunit.client.0.smithi136.stdout: "ceph_cap_op_flush_ack": { 2024-09-12T23:37:48.909 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.909 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.909 INFO:tasks.workunit.client.0.smithi136.stdout: "ceph_cap_op_flushsnap_ack": { 2024-09-12T23:37:48.909 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.909 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.909 INFO:tasks.workunit.client.0.smithi136.stdout: "ceph_cap_op_grant": { 2024-09-12T23:37:48.909 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.909 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.909 INFO:tasks.workunit.client.0.smithi136.stdout: "ceph_cap_op_revoke": { 2024-09-12T23:37:48.910 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.910 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.910 INFO:tasks.workunit.client.0.smithi136.stdout: "ceph_cap_op_trunc": { 2024-09-12T23:37:48.910 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.910 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.910 INFO:tasks.workunit.client.0.smithi136.stdout: "dir_commit": { 2024-09-12T23:37:48.910 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 12 2024-09-12T23:37:48.910 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.910 INFO:tasks.workunit.client.0.smithi136.stdout: "dir_fetch": { 2024-09-12T23:37:48.910 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.910 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.910 INFO:tasks.workunit.client.0.smithi136.stdout: "dir_merge": { 2024-09-12T23:37:48.911 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.911 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.911 INFO:tasks.workunit.client.0.smithi136.stdout: "dir_split": { 2024-09-12T23:37:48.911 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.911 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.911 INFO:tasks.workunit.client.0.smithi136.stdout: "exported_inodes": { 2024-09-12T23:37:48.911 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.911 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.911 INFO:tasks.workunit.client.0.smithi136.stdout: "forward": { 2024-09-12T23:37:48.911 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.911 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.911 INFO:tasks.workunit.client.0.smithi136.stdout: "handle_client_cap_release": { 2024-09-12T23:37:48.911 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.911 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.912 INFO:tasks.workunit.client.0.smithi136.stdout: "handle_client_caps": { 2024-09-12T23:37:48.912 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.912 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.912 INFO:tasks.workunit.client.0.smithi136.stdout: "handle_client_caps_dirty": { 2024-09-12T23:37:48.912 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.912 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.912 INFO:tasks.workunit.client.0.smithi136.stdout: "handle_inode_file_caps": { 2024-09-12T23:37:48.912 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.912 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.912 INFO:tasks.workunit.client.0.smithi136.stdout: "imported_inodes": { 2024-09-12T23:37:48.912 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.913 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.913 INFO:tasks.workunit.client.0.smithi136.stdout: "inodes": { 2024-09-12T23:37:48.913 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 10 2024-09-12T23:37:48.913 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.913 INFO:tasks.workunit.client.0.smithi136.stdout: "inodes_expired": { 2024-09-12T23:37:48.913 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.913 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.913 INFO:tasks.workunit.client.0.smithi136.stdout: "inodes_pinned": { 2024-09-12T23:37:48.913 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 10 2024-09-12T23:37:48.913 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.913 INFO:tasks.workunit.client.0.smithi136.stdout: "inodes_with_caps": { 2024-09-12T23:37:48.913 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.913 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.913 INFO:tasks.workunit.client.0.smithi136.stdout: "load_cent": { 2024-09-12T23:37:48.914 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.914 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.914 INFO:tasks.workunit.client.0.smithi136.stdout: "openino_dir_fetch": { 2024-09-12T23:37:48.914 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.914 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.914 INFO:tasks.workunit.client.0.smithi136.stdout: "process_request_cap_release": { 2024-09-12T23:37:48.914 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.914 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.914 INFO:tasks.workunit.client.0.smithi136.stdout: "reply_latency": { 2024-09-12T23:37:48.914 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 2, 2024-09-12T23:37:48.914 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 112659 2024-09-12T23:37:48.914 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.914 INFO:tasks.workunit.client.0.smithi136.stdout: "request": { 2024-09-12T23:37:48.914 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2 2024-09-12T23:37:48.915 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.915 INFO:tasks.workunit.client.0.smithi136.stdout: "root_rbytes": { 2024-09-12T23:37:48.915 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.915 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.915 INFO:tasks.workunit.client.0.smithi136.stdout: "root_rfiles": { 2024-09-12T23:37:48.915 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.915 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.915 INFO:tasks.workunit.client.0.smithi136.stdout: "root_rsnaps": { 2024-09-12T23:37:48.915 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.915 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.915 INFO:tasks.workunit.client.0.smithi136.stdout: "slow_reply": { 2024-09-12T23:37:48.916 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.916 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.916 INFO:tasks.workunit.client.0.smithi136.stdout: "subtrees": { 2024-09-12T23:37:48.916 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2 2024-09-12T23:37:48.916 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.916 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.916 INFO:tasks.workunit.client.0.smithi136.stdout: "mds_cache": { 2024-09-12T23:37:48.916 INFO:tasks.workunit.client.0.smithi136.stdout: "ireq_enqueue_scrub": { 2024-09-12T23:37:48.916 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.916 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.916 INFO:tasks.workunit.client.0.smithi136.stdout: "ireq_exportdir": { 2024-09-12T23:37:48.916 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.916 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.917 INFO:tasks.workunit.client.0.smithi136.stdout: "ireq_flush": { 2024-09-12T23:37:48.917 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.917 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.917 INFO:tasks.workunit.client.0.smithi136.stdout: "ireq_fragmentdir": { 2024-09-12T23:37:48.917 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.917 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.917 INFO:tasks.workunit.client.0.smithi136.stdout: "ireq_fragstats": { 2024-09-12T23:37:48.917 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.917 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.917 INFO:tasks.workunit.client.0.smithi136.stdout: "ireq_inodestats": { 2024-09-12T23:37:48.917 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.917 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.917 INFO:tasks.workunit.client.0.smithi136.stdout: "num_recovering_enqueued": { 2024-09-12T23:37:48.917 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.918 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.918 INFO:tasks.workunit.client.0.smithi136.stdout: "num_recovering_prioritized": { 2024-09-12T23:37:48.918 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.918 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.918 INFO:tasks.workunit.client.0.smithi136.stdout: "num_recovering_processing": { 2024-09-12T23:37:48.918 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.918 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.918 INFO:tasks.workunit.client.0.smithi136.stdout: "num_strays": { 2024-09-12T23:37:48.918 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.918 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.918 INFO:tasks.workunit.client.0.smithi136.stdout: "num_strays_delayed": { 2024-09-12T23:37:48.918 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.918 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.918 INFO:tasks.workunit.client.0.smithi136.stdout: "num_strays_enqueuing": { 2024-09-12T23:37:48.919 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.919 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.919 INFO:tasks.workunit.client.0.smithi136.stdout: "recovery_completed": { 2024-09-12T23:37:48.919 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.919 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.919 INFO:tasks.workunit.client.0.smithi136.stdout: "recovery_started": { 2024-09-12T23:37:48.919 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.919 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.919 INFO:tasks.workunit.client.0.smithi136.stdout: "strays_created": { 2024-09-12T23:37:48.919 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.919 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.919 INFO:tasks.workunit.client.0.smithi136.stdout: "strays_enqueued": { 2024-09-12T23:37:48.919 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.920 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.920 INFO:tasks.workunit.client.0.smithi136.stdout: "strays_migrated": { 2024-09-12T23:37:48.920 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.920 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.920 INFO:tasks.workunit.client.0.smithi136.stdout: "strays_reintegrated": { 2024-09-12T23:37:48.920 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.920 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.920 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.920 INFO:tasks.workunit.client.0.smithi136.stdout: "mds_log": { 2024-09-12T23:37:48.920 INFO:tasks.workunit.client.0.smithi136.stdout: "ev": { 2024-09-12T23:37:48.920 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5 2024-09-12T23:37:48.920 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.920 INFO:tasks.workunit.client.0.smithi136.stdout: "evadd": { 2024-09-12T23:37:48.920 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5 2024-09-12T23:37:48.921 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.921 INFO:tasks.workunit.client.0.smithi136.stdout: "evex": { 2024-09-12T23:37:48.921 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.921 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.921 INFO:tasks.workunit.client.0.smithi136.stdout: "evexd": { 2024-09-12T23:37:48.921 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.921 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.921 INFO:tasks.workunit.client.0.smithi136.stdout: "evexg": { 2024-09-12T23:37:48.921 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.921 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.921 INFO:tasks.workunit.client.0.smithi136.stdout: "evtrm": { 2024-09-12T23:37:48.921 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.921 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.921 INFO:tasks.workunit.client.0.smithi136.stdout: "jlat": { 2024-09-12T23:37:48.922 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 5, 2024-09-12T23:37:48.922 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 23682895 2024-09-12T23:37:48.922 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.922 INFO:tasks.workunit.client.0.smithi136.stdout: "replayed": { 2024-09-12T23:37:48.922 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.922 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.922 INFO:tasks.workunit.client.0.smithi136.stdout: "seg": { 2024-09-12T23:37:48.922 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1 2024-09-12T23:37:48.922 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.922 INFO:tasks.workunit.client.0.smithi136.stdout: "segadd": { 2024-09-12T23:37:48.922 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1 2024-09-12T23:37:48.922 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.922 INFO:tasks.workunit.client.0.smithi136.stdout: "segex": { 2024-09-12T23:37:48.922 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.923 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.923 INFO:tasks.workunit.client.0.smithi136.stdout: "segexd": { 2024-09-12T23:37:48.923 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.923 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.923 INFO:tasks.workunit.client.0.smithi136.stdout: "segexg": { 2024-09-12T23:37:48.923 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.923 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.923 INFO:tasks.workunit.client.0.smithi136.stdout: "segtrm": { 2024-09-12T23:37:48.923 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.923 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.923 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.923 INFO:tasks.workunit.client.0.smithi136.stdout: "mds_mem": { 2024-09-12T23:37:48.923 INFO:tasks.workunit.client.0.smithi136.stdout: "cap": { 2024-09-12T23:37:48.924 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.924 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.924 INFO:tasks.workunit.client.0.smithi136.stdout: "cap+": { 2024-09-12T23:37:48.924 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2 2024-09-12T23:37:48.924 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.924 INFO:tasks.workunit.client.0.smithi136.stdout: "cap-": { 2024-09-12T23:37:48.924 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2 2024-09-12T23:37:48.924 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.924 INFO:tasks.workunit.client.0.smithi136.stdout: "dir": { 2024-09-12T23:37:48.924 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 12 2024-09-12T23:37:48.924 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.924 INFO:tasks.workunit.client.0.smithi136.stdout: "dir+": { 2024-09-12T23:37:48.924 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 12 2024-09-12T23:37:48.924 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.925 INFO:tasks.workunit.client.0.smithi136.stdout: "dir-": { 2024-09-12T23:37:48.925 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.925 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.925 INFO:tasks.workunit.client.0.smithi136.stdout: "dn": { 2024-09-12T23:37:48.925 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 10 2024-09-12T23:37:48.925 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.925 INFO:tasks.workunit.client.0.smithi136.stdout: "dn+": { 2024-09-12T23:37:48.925 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 10 2024-09-12T23:37:48.925 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.925 INFO:tasks.workunit.client.0.smithi136.stdout: "dn-": { 2024-09-12T23:37:48.925 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.926 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.926 INFO:tasks.workunit.client.0.smithi136.stdout: "heap": { 2024-09-12T23:37:48.926 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 207068 2024-09-12T23:37:48.926 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.926 INFO:tasks.workunit.client.0.smithi136.stdout: "ino": { 2024-09-12T23:37:48.926 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 13 2024-09-12T23:37:48.926 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.926 INFO:tasks.workunit.client.0.smithi136.stdout: "ino+": { 2024-09-12T23:37:48.926 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 13 2024-09-12T23:37:48.926 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.926 INFO:tasks.workunit.client.0.smithi136.stdout: "ino-": { 2024-09-12T23:37:48.927 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.927 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.927 INFO:tasks.workunit.client.0.smithi136.stdout: "rss": { 2024-09-12T23:37:48.927 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 32760 2024-09-12T23:37:48.927 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.927 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.927 INFO:tasks.workunit.client.0.smithi136.stdout: "mds_server": { 2024-09-12T23:37:48.927 INFO:tasks.workunit.client.0.smithi136.stdout: "cap_acquisition_throttle": { 2024-09-12T23:37:48.927 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.927 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.927 INFO:tasks.workunit.client.0.smithi136.stdout: "cap_revoke_eviction": { 2024-09-12T23:37:48.927 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.927 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.927 INFO:tasks.workunit.client.0.smithi136.stdout: "handle_client_request": { 2024-09-12T23:37:48.928 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2 2024-09-12T23:37:48.928 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.928 INFO:tasks.workunit.client.0.smithi136.stdout: "handle_client_session": { 2024-09-12T23:37:48.928 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 16 2024-09-12T23:37:48.928 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.928 INFO:tasks.workunit.client.0.smithi136.stdout: "handle_peer_request": { 2024-09-12T23:37:48.928 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.928 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.928 INFO:tasks.workunit.client.0.smithi136.stdout: "req_create_latency": { 2024-09-12T23:37:48.928 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:48.928 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.929 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.929 INFO:tasks.workunit.client.0.smithi136.stdout: "req_getattr_latency": { 2024-09-12T23:37:48.929 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 2, 2024-09-12T23:37:48.929 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 112659 2024-09-12T23:37:48.929 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.929 INFO:tasks.workunit.client.0.smithi136.stdout: "req_getfilelock_latency": { 2024-09-12T23:37:48.929 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:48.929 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.929 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.929 INFO:tasks.workunit.client.0.smithi136.stdout: "req_getvxattr_latency": { 2024-09-12T23:37:48.929 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:48.929 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.929 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.930 INFO:tasks.workunit.client.0.smithi136.stdout: "req_link_latency": { 2024-09-12T23:37:48.930 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:48.930 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.930 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.930 INFO:tasks.workunit.client.0.smithi136.stdout: "req_lookup_latency": { 2024-09-12T23:37:48.930 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:48.930 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.930 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.930 INFO:tasks.workunit.client.0.smithi136.stdout: "req_lookuphash_latency": { 2024-09-12T23:37:48.930 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:48.931 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.931 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.931 INFO:tasks.workunit.client.0.smithi136.stdout: "req_lookupino_latency": { 2024-09-12T23:37:48.931 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:48.931 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.931 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.931 INFO:tasks.workunit.client.0.smithi136.stdout: "req_lookupname_latency": { 2024-09-12T23:37:48.931 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:48.931 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.931 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.931 INFO:tasks.workunit.client.0.smithi136.stdout: "req_lookupparent_latency": { 2024-09-12T23:37:48.931 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:48.932 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.932 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.932 INFO:tasks.workunit.client.0.smithi136.stdout: "req_lookupsnap_latency": { 2024-09-12T23:37:48.932 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:48.932 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.932 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.932 INFO:tasks.workunit.client.0.smithi136.stdout: "req_lssnap_latency": { 2024-09-12T23:37:48.932 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:48.932 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.932 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.932 INFO:tasks.workunit.client.0.smithi136.stdout: "req_mkdir_latency": { 2024-09-12T23:37:48.932 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:48.932 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.933 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.933 INFO:tasks.workunit.client.0.smithi136.stdout: "req_mknod_latency": { 2024-09-12T23:37:48.933 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:48.933 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.933 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.933 INFO:tasks.workunit.client.0.smithi136.stdout: "req_mksnap_latency": { 2024-09-12T23:37:48.933 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:48.933 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.933 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.933 INFO:tasks.workunit.client.0.smithi136.stdout: "req_open_latency": { 2024-09-12T23:37:48.933 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:48.933 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.933 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.933 INFO:tasks.workunit.client.0.smithi136.stdout: "req_readdir_latency": { 2024-09-12T23:37:48.933 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:48.934 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.934 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.934 INFO:tasks.workunit.client.0.smithi136.stdout: "req_rename_latency": { 2024-09-12T23:37:48.934 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:48.934 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.934 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.934 INFO:tasks.workunit.client.0.smithi136.stdout: "req_renamesnap_latency": { 2024-09-12T23:37:48.934 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:48.934 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.934 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.934 INFO:tasks.workunit.client.0.smithi136.stdout: "req_rmdir_latency": { 2024-09-12T23:37:48.934 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:48.934 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.935 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.935 INFO:tasks.workunit.client.0.smithi136.stdout: "req_rmsnap_latency": { 2024-09-12T23:37:48.935 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:48.935 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.935 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.935 INFO:tasks.workunit.client.0.smithi136.stdout: "req_rmxattr_latency": { 2024-09-12T23:37:48.935 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:48.935 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.935 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.935 INFO:tasks.workunit.client.0.smithi136.stdout: "req_setattr_latency": { 2024-09-12T23:37:48.935 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:48.935 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.935 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.936 INFO:tasks.workunit.client.0.smithi136.stdout: "req_setdirlayout_latency": { 2024-09-12T23:37:48.936 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:48.936 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.936 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.936 INFO:tasks.workunit.client.0.smithi136.stdout: "req_setfilelock_latency": { 2024-09-12T23:37:48.936 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:48.936 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.936 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.936 INFO:tasks.workunit.client.0.smithi136.stdout: "req_setlayout_latency": { 2024-09-12T23:37:48.936 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:48.937 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.937 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.937 INFO:tasks.workunit.client.0.smithi136.stdout: "req_setxattr_latency": { 2024-09-12T23:37:48.937 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:48.937 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.937 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.937 INFO:tasks.workunit.client.0.smithi136.stdout: "req_symlink_latency": { 2024-09-12T23:37:48.937 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:48.937 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.937 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.938 INFO:tasks.workunit.client.0.smithi136.stdout: "req_unlink_latency": { 2024-09-12T23:37:48.938 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:48.938 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.938 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.938 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.938 INFO:tasks.workunit.client.0.smithi136.stdout: "mds_sessions": { 2024-09-12T23:37:48.938 INFO:tasks.workunit.client.0.smithi136.stdout: "average_load": { 2024-09-12T23:37:48.938 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1 2024-09-12T23:37:48.938 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.938 INFO:tasks.workunit.client.0.smithi136.stdout: "avg_session_uptime": { 2024-09-12T23:37:48.938 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 85 2024-09-12T23:37:48.938 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.938 INFO:tasks.workunit.client.0.smithi136.stdout: "mdthresh_evicted": { 2024-09-12T23:37:48.939 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.939 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.939 INFO:tasks.workunit.client.0.smithi136.stdout: "session_add": { 2024-09-12T23:37:48.939 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2 2024-09-12T23:37:48.939 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.939 INFO:tasks.workunit.client.0.smithi136.stdout: "session_count": { 2024-09-12T23:37:48.939 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.939 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.939 INFO:tasks.workunit.client.0.smithi136.stdout: "session_remove": { 2024-09-12T23:37:48.939 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2 2024-09-12T23:37:48.939 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.939 INFO:tasks.workunit.client.0.smithi136.stdout: "sessions_open": { 2024-09-12T23:37:48.939 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.940 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.940 INFO:tasks.workunit.client.0.smithi136.stdout: "sessions_stale": { 2024-09-12T23:37:48.940 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.940 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.940 INFO:tasks.workunit.client.0.smithi136.stdout: "total_load": { 2024-09-12T23:37:48.940 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2 2024-09-12T23:37:48.940 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.940 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.940 INFO:tasks.workunit.client.0.smithi136.stdout: "objecter": { 2024-09-12T23:37:48.940 INFO:tasks.workunit.client.0.smithi136.stdout: "op_active": { 2024-09-12T23:37:48.940 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.940 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.940 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r": { 2024-09-12T23:37:48.941 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.941 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.941 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rmw": { 2024-09-12T23:37:48.941 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.941 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.941 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w": { 2024-09-12T23:37:48.941 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 45 2024-09-12T23:37:48.941 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.941 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.941 INFO:tasks.workunit.client.0.smithi136.stdout: "purge_queue": { 2024-09-12T23:37:48.942 INFO:tasks.workunit.client.0.smithi136.stdout: "pq_executed": { 2024-09-12T23:37:48.942 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.942 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.942 INFO:tasks.workunit.client.0.smithi136.stdout: "pq_executing": { 2024-09-12T23:37:48.942 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.942 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.942 INFO:tasks.workunit.client.0.smithi136.stdout: "pq_executing_high_water": { 2024-09-12T23:37:48.942 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.942 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.942 INFO:tasks.workunit.client.0.smithi136.stdout: "pq_executing_ops": { 2024-09-12T23:37:48.942 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.942 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.942 INFO:tasks.workunit.client.0.smithi136.stdout: "pq_executing_ops_high_water": { 2024-09-12T23:37:48.942 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.943 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.943 INFO:tasks.workunit.client.0.smithi136.stdout: "pq_item_in_journal": { 2024-09-12T23:37:48.943 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.943 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.943 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.943 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.943 INFO:tasks.workunit.client.0.smithi136.stdout: "mon.0027af889fd200cedbdb75d916e7558e544c1ffa": { 2024-09-12T23:37:48.943 INFO:tasks.workunit.client.0.smithi136.stdout: "mon": { 2024-09-12T23:37:48.943 INFO:tasks.workunit.client.0.smithi136.stdout: "election_call": { 2024-09-12T23:37:48.943 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1 2024-09-12T23:37:48.943 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.943 INFO:tasks.workunit.client.0.smithi136.stdout: "election_lose": { 2024-09-12T23:37:48.943 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1 2024-09-12T23:37:48.944 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.944 INFO:tasks.workunit.client.0.smithi136.stdout: "election_win": { 2024-09-12T23:37:48.944 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.944 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.944 INFO:tasks.workunit.client.0.smithi136.stdout: "num_elections": { 2024-09-12T23:37:48.944 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4 2024-09-12T23:37:48.944 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.944 INFO:tasks.workunit.client.0.smithi136.stdout: "num_sessions": { 2024-09-12T23:37:48.944 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 13 2024-09-12T23:37:48.944 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.944 INFO:tasks.workunit.client.0.smithi136.stdout: "session_add": { 2024-09-12T23:37:48.944 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5 2024-09-12T23:37:48.945 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.945 INFO:tasks.workunit.client.0.smithi136.stdout: "session_rm": { 2024-09-12T23:37:48.945 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 104 2024-09-12T23:37:48.945 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.945 INFO:tasks.workunit.client.0.smithi136.stdout: "session_trim": { 2024-09-12T23:37:48.945 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.945 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.945 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.945 INFO:tasks.workunit.client.0.smithi136.stdout: "paxos": { 2024-09-12T23:37:48.945 INFO:tasks.workunit.client.0.smithi136.stdout: "accept_timeout": { 2024-09-12T23:37:48.945 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.945 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.945 INFO:tasks.workunit.client.0.smithi136.stdout: "begin": { 2024-09-12T23:37:48.946 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1101 2024-09-12T23:37:48.946 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.946 INFO:tasks.workunit.client.0.smithi136.stdout: "begin_bytes": { 2024-09-12T23:37:48.946 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1101, 2024-09-12T23:37:48.946 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4471081 2024-09-12T23:37:48.946 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.946 INFO:tasks.workunit.client.0.smithi136.stdout: "begin_keys": { 2024-09-12T23:37:48.946 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:48.946 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.946 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.946 INFO:tasks.workunit.client.0.smithi136.stdout: "begin_latency": { 2024-09-12T23:37:48.946 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1101, 2024-09-12T23:37:48.947 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 391993482 2024-09-12T23:37:48.947 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.947 INFO:tasks.workunit.client.0.smithi136.stdout: "collect": { 2024-09-12T23:37:48.947 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1 2024-09-12T23:37:48.947 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.947 INFO:tasks.workunit.client.0.smithi136.stdout: "collect_bytes": { 2024-09-12T23:37:48.947 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1, 2024-09-12T23:37:48.947 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 73 2024-09-12T23:37:48.947 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.947 INFO:tasks.workunit.client.0.smithi136.stdout: "collect_keys": { 2024-09-12T23:37:48.947 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1, 2024-09-12T23:37:48.947 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1 2024-09-12T23:37:48.948 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.948 INFO:tasks.workunit.client.0.smithi136.stdout: "collect_latency": { 2024-09-12T23:37:48.948 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1, 2024-09-12T23:37:48.948 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.948 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.948 INFO:tasks.workunit.client.0.smithi136.stdout: "collect_timeout": { 2024-09-12T23:37:48.948 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.948 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.948 INFO:tasks.workunit.client.0.smithi136.stdout: "collect_uncommitted": { 2024-09-12T23:37:48.948 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.948 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.949 INFO:tasks.workunit.client.0.smithi136.stdout: "commit": { 2024-09-12T23:37:48.949 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1102 2024-09-12T23:37:48.949 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.949 INFO:tasks.workunit.client.0.smithi136.stdout: "commit_bytes": { 2024-09-12T23:37:48.949 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:48.949 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.949 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.949 INFO:tasks.workunit.client.0.smithi136.stdout: "commit_keys": { 2024-09-12T23:37:48.949 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:48.949 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.949 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.949 INFO:tasks.workunit.client.0.smithi136.stdout: "commit_latency": { 2024-09-12T23:37:48.949 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:48.950 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.950 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.950 INFO:tasks.workunit.client.0.smithi136.stdout: "lease_ack_timeout": { 2024-09-12T23:37:48.950 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.950 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.950 INFO:tasks.workunit.client.0.smithi136.stdout: "lease_timeout": { 2024-09-12T23:37:48.950 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.950 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.951 INFO:tasks.workunit.client.0.smithi136.stdout: "new_pn": { 2024-09-12T23:37:48.951 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.951 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.951 INFO:tasks.workunit.client.0.smithi136.stdout: "new_pn_latency": { 2024-09-12T23:37:48.951 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:48.951 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.951 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.951 INFO:tasks.workunit.client.0.smithi136.stdout: "refresh": { 2024-09-12T23:37:48.951 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1102 2024-09-12T23:37:48.951 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.951 INFO:tasks.workunit.client.0.smithi136.stdout: "refresh_latency": { 2024-09-12T23:37:48.951 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1102, 2024-09-12T23:37:48.952 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4775916648 2024-09-12T23:37:48.952 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.952 INFO:tasks.workunit.client.0.smithi136.stdout: "restart": { 2024-09-12T23:37:48.952 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 10 2024-09-12T23:37:48.952 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.952 INFO:tasks.workunit.client.0.smithi136.stdout: "share_state": { 2024-09-12T23:37:48.952 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.952 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.952 INFO:tasks.workunit.client.0.smithi136.stdout: "share_state_bytes": { 2024-09-12T23:37:48.952 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:48.952 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.952 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.952 INFO:tasks.workunit.client.0.smithi136.stdout: "share_state_keys": { 2024-09-12T23:37:48.953 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:48.953 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.953 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.953 INFO:tasks.workunit.client.0.smithi136.stdout: "start_leader": { 2024-09-12T23:37:48.953 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.953 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.953 INFO:tasks.workunit.client.0.smithi136.stdout: "start_peon": { 2024-09-12T23:37:48.953 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1 2024-09-12T23:37:48.953 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.953 INFO:tasks.workunit.client.0.smithi136.stdout: "store_state": { 2024-09-12T23:37:48.953 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1102 2024-09-12T23:37:48.953 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.953 INFO:tasks.workunit.client.0.smithi136.stdout: "store_state_bytes": { 2024-09-12T23:37:48.954 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1102, 2024-09-12T23:37:48.954 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 8783823 2024-09-12T23:37:48.954 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.954 INFO:tasks.workunit.client.0.smithi136.stdout: "store_state_keys": { 2024-09-12T23:37:48.954 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1102, 2024-09-12T23:37:48.954 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 7729 2024-09-12T23:37:48.954 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.954 INFO:tasks.workunit.client.0.smithi136.stdout: "store_state_latency": { 2024-09-12T23:37:48.954 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1102, 2024-09-12T23:37:48.954 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 535990454 2024-09-12T23:37:48.954 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.954 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.954 INFO:tasks.workunit.client.0.smithi136.stdout: "prioritycache": { 2024-09-12T23:37:48.954 INFO:tasks.workunit.client.0.smithi136.stdout: "cache_bytes": { 2024-09-12T23:37:48.955 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1020054731 2024-09-12T23:37:48.955 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.955 INFO:tasks.workunit.client.0.smithi136.stdout: "heap_bytes": { 2024-09-12T23:37:48.955 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 40689664 2024-09-12T23:37:48.955 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.955 INFO:tasks.workunit.client.0.smithi136.stdout: "mapped_bytes": { 2024-09-12T23:37:48.955 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 34250752 2024-09-12T23:37:48.955 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.955 INFO:tasks.workunit.client.0.smithi136.stdout: "target_bytes": { 2024-09-12T23:37:48.955 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2147483648 2024-09-12T23:37:48.955 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.956 INFO:tasks.workunit.client.0.smithi136.stdout: "unmapped_bytes": { 2024-09-12T23:37:48.956 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 6438912 2024-09-12T23:37:48.956 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.956 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.956 INFO:tasks.workunit.client.0.smithi136.stdout: "prioritycache:full": { 2024-09-12T23:37:48.956 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:48.956 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 348127232 2024-09-12T23:37:48.956 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.956 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:48.956 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.956 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.956 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:48.957 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.957 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.957 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:48.957 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 277028253 2024-09-12T23:37:48.957 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.957 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:48.957 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 254099 2024-09-12T23:37:48.957 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.957 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:48.957 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.958 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.958 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:48.958 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.958 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.958 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:48.958 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.958 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.958 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:48.958 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.958 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.958 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:48.958 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.958 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.959 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:48.959 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.959 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.959 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:48.959 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.959 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.959 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:48.959 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.959 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.959 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:48.959 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 70844880 2024-09-12T23:37:48.959 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.960 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.960 INFO:tasks.workunit.client.0.smithi136.stdout: "prioritycache:inc": { 2024-09-12T23:37:48.960 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:48.960 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 348127232 2024-09-12T23:37:48.960 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.960 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:48.960 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.960 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.960 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:48.960 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.960 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.960 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:48.960 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 277028253 2024-09-12T23:37:48.960 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.961 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:48.961 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 40255 2024-09-12T23:37:48.961 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.961 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:48.961 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.961 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.961 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:48.961 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.961 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.961 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:48.961 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.961 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.962 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:48.962 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.962 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.962 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:48.962 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.962 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.962 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:48.962 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.962 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.962 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:48.962 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.962 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.962 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:48.963 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.963 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.963 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:48.963 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 71058724 2024-09-12T23:37:48.963 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.963 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.963 INFO:tasks.workunit.client.0.smithi136.stdout: "prioritycache:kv": { 2024-09-12T23:37:48.963 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:48.963 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 322961408 2024-09-12T23:37:48.963 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.963 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:48.963 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67108864 2024-09-12T23:37:48.963 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.963 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:48.963 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.964 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.964 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:48.964 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 184685502 2024-09-12T23:37:48.964 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.964 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:48.964 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.964 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.964 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:48.964 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.964 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.964 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:48.964 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.964 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.965 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:48.965 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.965 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.965 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:48.965 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.965 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.965 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:48.965 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.965 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.965 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:48.965 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.965 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.965 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:48.965 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.966 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.966 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:48.966 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.966 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.966 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:48.966 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 71167042 2024-09-12T23:37:48.966 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.966 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.966 INFO:tasks.workunit.client.0.smithi136.stdout: "rocksdb": { 2024-09-12T23:37:48.966 INFO:tasks.workunit.client.0.smithi136.stdout: "compact": { 2024-09-12T23:37:48.966 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.966 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.966 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_completed": { 2024-09-12T23:37:48.967 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5 2024-09-12T23:37:48.967 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.967 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_lasted": { 2024-09-12T23:37:48.967 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 26588769 2024-09-12T23:37:48.967 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.967 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_queue_len": { 2024-09-12T23:37:48.967 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.967 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.967 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_queue_merge": { 2024-09-12T23:37:48.967 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.967 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.968 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_running": { 2024-09-12T23:37:48.968 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.968 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.968 INFO:tasks.workunit.client.0.smithi136.stdout: "get_latency": { 2024-09-12T23:37:48.968 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 46811, 2024-09-12T23:37:48.968 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 266662983 2024-09-12T23:37:48.968 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.968 INFO:tasks.workunit.client.0.smithi136.stdout: "rocksdb_write_delay_time": { 2024-09-12T23:37:48.968 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:48.968 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.968 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.968 INFO:tasks.workunit.client.0.smithi136.stdout: "rocksdb_write_memtable_time": { 2024-09-12T23:37:48.968 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:48.969 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.969 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.969 INFO:tasks.workunit.client.0.smithi136.stdout: "rocksdb_write_pre_and_post_time": { 2024-09-12T23:37:48.969 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:48.969 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.969 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.969 INFO:tasks.workunit.client.0.smithi136.stdout: "rocksdb_write_wal_time": { 2024-09-12T23:37:48.969 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:48.969 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.969 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.969 INFO:tasks.workunit.client.0.smithi136.stdout: "submit_latency": { 2024-09-12T23:37:48.969 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:48.969 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.970 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.970 INFO:tasks.workunit.client.0.smithi136.stdout: "submit_sync_latency": { 2024-09-12T23:37:48.970 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 2421, 2024-09-12T23:37:48.970 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1069742150 2024-09-12T23:37:48.970 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.970 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.970 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.970 INFO:tasks.workunit.client.0.smithi136.stdout: "mon.4585d541b331f263094839d9f58dd50a20d3f517": { 2024-09-12T23:37:48.970 INFO:tasks.workunit.client.0.smithi136.stdout: "mon": { 2024-09-12T23:37:48.970 INFO:tasks.workunit.client.0.smithi136.stdout: "election_call": { 2024-09-12T23:37:48.970 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2 2024-09-12T23:37:48.970 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.970 INFO:tasks.workunit.client.0.smithi136.stdout: "election_lose": { 2024-09-12T23:37:48.971 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2 2024-09-12T23:37:48.971 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.971 INFO:tasks.workunit.client.0.smithi136.stdout: "election_win": { 2024-09-12T23:37:48.971 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.971 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.971 INFO:tasks.workunit.client.0.smithi136.stdout: "num_elections": { 2024-09-12T23:37:48.971 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 8 2024-09-12T23:37:48.971 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.971 INFO:tasks.workunit.client.0.smithi136.stdout: "num_sessions": { 2024-09-12T23:37:48.971 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 9 2024-09-12T23:37:48.971 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.971 INFO:tasks.workunit.client.0.smithi136.stdout: "session_add": { 2024-09-12T23:37:48.971 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4 2024-09-12T23:37:48.971 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.971 INFO:tasks.workunit.client.0.smithi136.stdout: "session_rm": { 2024-09-12T23:37:48.972 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 51 2024-09-12T23:37:48.972 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.972 INFO:tasks.workunit.client.0.smithi136.stdout: "session_trim": { 2024-09-12T23:37:48.972 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.972 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.972 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.972 INFO:tasks.workunit.client.0.smithi136.stdout: "paxos": { 2024-09-12T23:37:48.972 INFO:tasks.workunit.client.0.smithi136.stdout: "accept_timeout": { 2024-09-12T23:37:48.972 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.972 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.972 INFO:tasks.workunit.client.0.smithi136.stdout: "begin": { 2024-09-12T23:37:48.972 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1108 2024-09-12T23:37:48.972 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.972 INFO:tasks.workunit.client.0.smithi136.stdout: "begin_bytes": { 2024-09-12T23:37:48.973 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1108, 2024-09-12T23:37:48.973 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4497075 2024-09-12T23:37:48.973 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.973 INFO:tasks.workunit.client.0.smithi136.stdout: "begin_keys": { 2024-09-12T23:37:48.973 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:48.973 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.973 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.973 INFO:tasks.workunit.client.0.smithi136.stdout: "begin_latency": { 2024-09-12T23:37:48.973 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1108, 2024-09-12T23:37:48.973 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 251996303 2024-09-12T23:37:48.973 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.973 INFO:tasks.workunit.client.0.smithi136.stdout: "collect": { 2024-09-12T23:37:48.973 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2 2024-09-12T23:37:48.974 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.974 INFO:tasks.workunit.client.0.smithi136.stdout: "collect_bytes": { 2024-09-12T23:37:48.974 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 2, 2024-09-12T23:37:48.974 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 146 2024-09-12T23:37:48.974 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.974 INFO:tasks.workunit.client.0.smithi136.stdout: "collect_keys": { 2024-09-12T23:37:48.974 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 2, 2024-09-12T23:37:48.974 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2 2024-09-12T23:37:48.974 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.974 INFO:tasks.workunit.client.0.smithi136.stdout: "collect_latency": { 2024-09-12T23:37:48.974 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 2, 2024-09-12T23:37:48.974 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 3999912 2024-09-12T23:37:48.974 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.974 INFO:tasks.workunit.client.0.smithi136.stdout: "collect_timeout": { 2024-09-12T23:37:48.975 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.975 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.975 INFO:tasks.workunit.client.0.smithi136.stdout: "collect_uncommitted": { 2024-09-12T23:37:48.975 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.975 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.975 INFO:tasks.workunit.client.0.smithi136.stdout: "commit": { 2024-09-12T23:37:48.975 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1109 2024-09-12T23:37:48.975 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.975 INFO:tasks.workunit.client.0.smithi136.stdout: "commit_bytes": { 2024-09-12T23:37:48.975 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:48.975 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.975 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.975 INFO:tasks.workunit.client.0.smithi136.stdout: "commit_keys": { 2024-09-12T23:37:48.975 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:48.976 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.976 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.976 INFO:tasks.workunit.client.0.smithi136.stdout: "commit_latency": { 2024-09-12T23:37:48.976 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:48.976 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.976 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.976 INFO:tasks.workunit.client.0.smithi136.stdout: "lease_ack_timeout": { 2024-09-12T23:37:48.976 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.976 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.976 INFO:tasks.workunit.client.0.smithi136.stdout: "lease_timeout": { 2024-09-12T23:37:48.976 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.976 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.976 INFO:tasks.workunit.client.0.smithi136.stdout: "new_pn": { 2024-09-12T23:37:48.976 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.977 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.977 INFO:tasks.workunit.client.0.smithi136.stdout: "new_pn_latency": { 2024-09-12T23:37:48.977 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:48.977 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.977 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.977 INFO:tasks.workunit.client.0.smithi136.stdout: "refresh": { 2024-09-12T23:37:48.977 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1109 2024-09-12T23:37:48.977 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.977 INFO:tasks.workunit.client.0.smithi136.stdout: "refresh_latency": { 2024-09-12T23:37:48.977 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1109, 2024-09-12T23:37:48.977 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4379937677 2024-09-12T23:37:48.977 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.978 INFO:tasks.workunit.client.0.smithi136.stdout: "restart": { 2024-09-12T23:37:48.978 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 14 2024-09-12T23:37:48.978 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.978 INFO:tasks.workunit.client.0.smithi136.stdout: "share_state": { 2024-09-12T23:37:48.978 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.978 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.978 INFO:tasks.workunit.client.0.smithi136.stdout: "share_state_bytes": { 2024-09-12T23:37:48.978 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:48.978 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.978 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.978 INFO:tasks.workunit.client.0.smithi136.stdout: "share_state_keys": { 2024-09-12T23:37:48.979 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:48.979 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.979 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.979 INFO:tasks.workunit.client.0.smithi136.stdout: "start_leader": { 2024-09-12T23:37:48.979 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.979 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.979 INFO:tasks.workunit.client.0.smithi136.stdout: "start_peon": { 2024-09-12T23:37:48.979 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2 2024-09-12T23:37:48.979 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.979 INFO:tasks.workunit.client.0.smithi136.stdout: "store_state": { 2024-09-12T23:37:48.979 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1109 2024-09-12T23:37:48.979 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.979 INFO:tasks.workunit.client.0.smithi136.stdout: "store_state_bytes": { 2024-09-12T23:37:48.979 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1109, 2024-09-12T23:37:48.980 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 8785300 2024-09-12T23:37:48.980 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.980 INFO:tasks.workunit.client.0.smithi136.stdout: "store_state_keys": { 2024-09-12T23:37:48.980 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1109, 2024-09-12T23:37:48.980 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 7740 2024-09-12T23:37:48.980 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.980 INFO:tasks.workunit.client.0.smithi136.stdout: "store_state_latency": { 2024-09-12T23:37:48.980 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1109, 2024-09-12T23:37:48.980 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 327995266 2024-09-12T23:37:48.980 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.980 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.980 INFO:tasks.workunit.client.0.smithi136.stdout: "prioritycache": { 2024-09-12T23:37:48.980 INFO:tasks.workunit.client.0.smithi136.stdout: "cache_bytes": { 2024-09-12T23:37:48.980 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1020054731 2024-09-12T23:37:48.981 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.981 INFO:tasks.workunit.client.0.smithi136.stdout: "heap_bytes": { 2024-09-12T23:37:48.981 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 41746432 2024-09-12T23:37:48.981 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.981 INFO:tasks.workunit.client.0.smithi136.stdout: "mapped_bytes": { 2024-09-12T23:37:48.981 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 34816000 2024-09-12T23:37:48.981 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.981 INFO:tasks.workunit.client.0.smithi136.stdout: "target_bytes": { 2024-09-12T23:37:48.981 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2147483648 2024-09-12T23:37:48.981 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.981 INFO:tasks.workunit.client.0.smithi136.stdout: "unmapped_bytes": { 2024-09-12T23:37:48.981 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 6930432 2024-09-12T23:37:48.981 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.982 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.982 INFO:tasks.workunit.client.0.smithi136.stdout: "prioritycache:full": { 2024-09-12T23:37:48.982 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:48.982 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 348127232 2024-09-12T23:37:48.982 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.982 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:48.982 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.982 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.982 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:48.982 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.982 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.982 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:48.982 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 277028253 2024-09-12T23:37:48.982 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.983 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:48.983 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 254099 2024-09-12T23:37:48.983 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.983 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:48.983 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.983 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.983 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:48.983 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.983 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.983 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:48.983 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.983 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.983 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:48.983 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.984 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.984 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:48.984 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.984 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.984 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:48.984 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.984 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.984 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:48.984 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.984 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.984 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:48.984 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.984 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.984 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:48.985 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 70844880 2024-09-12T23:37:48.985 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.985 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.985 INFO:tasks.workunit.client.0.smithi136.stdout: "prioritycache:inc": { 2024-09-12T23:37:48.985 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:48.985 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 348127232 2024-09-12T23:37:48.985 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.985 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:48.985 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.985 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.985 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:48.986 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.986 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.986 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:48.986 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 277028253 2024-09-12T23:37:48.986 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.986 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:48.986 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 40255 2024-09-12T23:37:48.986 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.986 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:48.986 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.986 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.986 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:48.986 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.986 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.987 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:48.987 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.987 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.987 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:48.987 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.987 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.987 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:48.987 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.987 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.987 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:48.987 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.987 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.987 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:48.988 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.988 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.988 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:48.988 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.988 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.988 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:48.988 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 71058724 2024-09-12T23:37:48.988 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.988 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.988 INFO:tasks.workunit.client.0.smithi136.stdout: "prioritycache:kv": { 2024-09-12T23:37:48.988 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:48.989 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 322961408 2024-09-12T23:37:48.989 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.989 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:48.989 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67108864 2024-09-12T23:37:48.989 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.989 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:48.989 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.989 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.989 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:48.989 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 184685502 2024-09-12T23:37:48.989 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.989 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:48.989 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.990 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.990 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:48.990 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.990 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.990 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:48.990 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.990 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.990 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:48.990 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.990 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.990 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:48.990 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.990 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.990 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:48.991 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.991 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.991 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:48.991 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.991 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.991 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:48.991 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.991 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.991 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:48.991 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.991 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.992 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:48.992 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 71167042 2024-09-12T23:37:48.992 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.992 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.992 INFO:tasks.workunit.client.0.smithi136.stdout: "rocksdb": { 2024-09-12T23:37:48.992 INFO:tasks.workunit.client.0.smithi136.stdout: "compact": { 2024-09-12T23:37:48.992 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.992 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.992 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_completed": { 2024-09-12T23:37:48.992 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5 2024-09-12T23:37:48.992 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.992 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_lasted": { 2024-09-12T23:37:48.992 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 39206331 2024-09-12T23:37:48.992 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.993 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_queue_len": { 2024-09-12T23:37:48.993 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.993 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.993 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_queue_merge": { 2024-09-12T23:37:48.993 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.993 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.993 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_running": { 2024-09-12T23:37:48.993 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.993 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.993 INFO:tasks.workunit.client.0.smithi136.stdout: "get_latency": { 2024-09-12T23:37:48.993 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 47099, 2024-09-12T23:37:48.993 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 242119711 2024-09-12T23:37:48.993 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.993 INFO:tasks.workunit.client.0.smithi136.stdout: "rocksdb_write_delay_time": { 2024-09-12T23:37:48.994 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:48.994 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.994 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.994 INFO:tasks.workunit.client.0.smithi136.stdout: "rocksdb_write_memtable_time": { 2024-09-12T23:37:48.994 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:48.994 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.994 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.994 INFO:tasks.workunit.client.0.smithi136.stdout: "rocksdb_write_pre_and_post_time": { 2024-09-12T23:37:48.994 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:48.994 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.994 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.994 INFO:tasks.workunit.client.0.smithi136.stdout: "rocksdb_write_wal_time": { 2024-09-12T23:37:48.994 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:48.994 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.995 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.995 INFO:tasks.workunit.client.0.smithi136.stdout: "submit_latency": { 2024-09-12T23:37:48.995 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:48.995 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.995 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.995 INFO:tasks.workunit.client.0.smithi136.stdout: "submit_sync_latency": { 2024-09-12T23:37:48.995 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 2440, 2024-09-12T23:37:48.995 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 789572610 2024-09-12T23:37:48.995 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.995 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.995 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.995 INFO:tasks.workunit.client.0.smithi136.stdout: "mon.48b2ee34c5040a1abbee529ca5897a6d3827a23a": { 2024-09-12T23:37:48.995 INFO:tasks.workunit.client.0.smithi136.stdout: "mon": { 2024-09-12T23:37:48.996 INFO:tasks.workunit.client.0.smithi136.stdout: "election_call": { 2024-09-12T23:37:48.996 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 3 2024-09-12T23:37:48.996 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.996 INFO:tasks.workunit.client.0.smithi136.stdout: "election_lose": { 2024-09-12T23:37:48.996 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.996 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.996 INFO:tasks.workunit.client.0.smithi136.stdout: "election_win": { 2024-09-12T23:37:48.996 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 3 2024-09-12T23:37:48.996 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.996 INFO:tasks.workunit.client.0.smithi136.stdout: "num_elections": { 2024-09-12T23:37:48.996 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 11 2024-09-12T23:37:48.996 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.996 INFO:tasks.workunit.client.0.smithi136.stdout: "num_sessions": { 2024-09-12T23:37:48.996 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 6 2024-09-12T23:37:48.997 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.997 INFO:tasks.workunit.client.0.smithi136.stdout: "session_add": { 2024-09-12T23:37:48.997 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 3 2024-09-12T23:37:48.997 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.997 INFO:tasks.workunit.client.0.smithi136.stdout: "session_rm": { 2024-09-12T23:37:48.997 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 183 2024-09-12T23:37:48.997 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.997 INFO:tasks.workunit.client.0.smithi136.stdout: "session_trim": { 2024-09-12T23:37:48.997 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.997 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:48.997 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.997 INFO:tasks.workunit.client.0.smithi136.stdout: "paxos": { 2024-09-12T23:37:48.997 INFO:tasks.workunit.client.0.smithi136.stdout: "accept_timeout": { 2024-09-12T23:37:48.998 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.998 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.998 INFO:tasks.workunit.client.0.smithi136.stdout: "begin": { 2024-09-12T23:37:48.998 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1271 2024-09-12T23:37:48.998 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.998 INFO:tasks.workunit.client.0.smithi136.stdout: "begin_bytes": { 2024-09-12T23:37:48.998 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1271, 2024-09-12T23:37:48.998 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5600837 2024-09-12T23:37:48.998 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.998 INFO:tasks.workunit.client.0.smithi136.stdout: "begin_keys": { 2024-09-12T23:37:48.998 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1271, 2024-09-12T23:37:48.999 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 3813 2024-09-12T23:37:48.999 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.999 INFO:tasks.workunit.client.0.smithi136.stdout: "begin_latency": { 2024-09-12T23:37:48.999 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1271, 2024-09-12T23:37:48.999 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 351992952 2024-09-12T23:37:48.999 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.999 INFO:tasks.workunit.client.0.smithi136.stdout: "collect": { 2024-09-12T23:37:48.999 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.999 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.999 INFO:tasks.workunit.client.0.smithi136.stdout: "collect_bytes": { 2024-09-12T23:37:48.999 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:48.999 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:48.999 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:48.999 INFO:tasks.workunit.client.0.smithi136.stdout: "collect_keys": { 2024-09-12T23:37:49.000 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.000 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.000 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.000 INFO:tasks.workunit.client.0.smithi136.stdout: "collect_latency": { 2024-09-12T23:37:49.000 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.000 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.000 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.000 INFO:tasks.workunit.client.0.smithi136.stdout: "collect_timeout": { 2024-09-12T23:37:49.000 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.000 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.000 INFO:tasks.workunit.client.0.smithi136.stdout: "collect_uncommitted": { 2024-09-12T23:37:49.000 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.000 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.001 INFO:tasks.workunit.client.0.smithi136.stdout: "commit": { 2024-09-12T23:37:49.001 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1271 2024-09-12T23:37:49.001 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.001 INFO:tasks.workunit.client.0.smithi136.stdout: "commit_bytes": { 2024-09-12T23:37:49.001 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1271, 2024-09-12T23:37:49.001 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5508956 2024-09-12T23:37:49.001 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.001 INFO:tasks.workunit.client.0.smithi136.stdout: "commit_keys": { 2024-09-12T23:37:49.001 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1271, 2024-09-12T23:37:49.001 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 7478 2024-09-12T23:37:49.001 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.001 INFO:tasks.workunit.client.0.smithi136.stdout: "commit_latency": { 2024-09-12T23:37:49.001 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1271, 2024-09-12T23:37:49.001 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 554603783 2024-09-12T23:37:49.002 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.002 INFO:tasks.workunit.client.0.smithi136.stdout: "lease_ack_timeout": { 2024-09-12T23:37:49.002 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.002 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.002 INFO:tasks.workunit.client.0.smithi136.stdout: "lease_timeout": { 2024-09-12T23:37:49.002 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.002 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.002 INFO:tasks.workunit.client.0.smithi136.stdout: "new_pn": { 2024-09-12T23:37:49.002 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2 2024-09-12T23:37:49.002 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.002 INFO:tasks.workunit.client.0.smithi136.stdout: "new_pn_latency": { 2024-09-12T23:37:49.002 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 2, 2024-09-12T23:37:49.002 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.002 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.003 INFO:tasks.workunit.client.0.smithi136.stdout: "refresh": { 2024-09-12T23:37:49.003 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1273 2024-09-12T23:37:49.003 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.003 INFO:tasks.workunit.client.0.smithi136.stdout: "refresh_latency": { 2024-09-12T23:37:49.003 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1273, 2024-09-12T23:37:49.003 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5651890777 2024-09-12T23:37:49.003 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.003 INFO:tasks.workunit.client.0.smithi136.stdout: "restart": { 2024-09-12T23:37:49.003 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 14 2024-09-12T23:37:49.003 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.003 INFO:tasks.workunit.client.0.smithi136.stdout: "share_state": { 2024-09-12T23:37:49.003 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2 2024-09-12T23:37:49.003 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.003 INFO:tasks.workunit.client.0.smithi136.stdout: "share_state_bytes": { 2024-09-12T23:37:49.004 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 2, 2024-09-12T23:37:49.004 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 25543 2024-09-12T23:37:49.004 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.004 INFO:tasks.workunit.client.0.smithi136.stdout: "share_state_keys": { 2024-09-12T23:37:49.004 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 2, 2024-09-12T23:37:49.004 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 8 2024-09-12T23:37:49.004 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.004 INFO:tasks.workunit.client.0.smithi136.stdout: "start_leader": { 2024-09-12T23:37:49.004 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 3 2024-09-12T23:37:49.004 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.004 INFO:tasks.workunit.client.0.smithi136.stdout: "start_peon": { 2024-09-12T23:37:49.004 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.004 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.005 INFO:tasks.workunit.client.0.smithi136.stdout: "store_state": { 2024-09-12T23:37:49.005 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.005 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.005 INFO:tasks.workunit.client.0.smithi136.stdout: "store_state_bytes": { 2024-09-12T23:37:49.005 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.005 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.005 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.005 INFO:tasks.workunit.client.0.smithi136.stdout: "store_state_keys": { 2024-09-12T23:37:49.005 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.005 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.005 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.005 INFO:tasks.workunit.client.0.smithi136.stdout: "store_state_latency": { 2024-09-12T23:37:49.006 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.006 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.006 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.006 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.006 INFO:tasks.workunit.client.0.smithi136.stdout: "prioritycache": { 2024-09-12T23:37:49.006 INFO:tasks.workunit.client.0.smithi136.stdout: "cache_bytes": { 2024-09-12T23:37:49.006 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1020054731 2024-09-12T23:37:49.006 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.006 INFO:tasks.workunit.client.0.smithi136.stdout: "heap_bytes": { 2024-09-12T23:37:49.006 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 47947776 2024-09-12T23:37:49.006 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.007 INFO:tasks.workunit.client.0.smithi136.stdout: "mapped_bytes": { 2024-09-12T23:37:49.007 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 43687936 2024-09-12T23:37:49.007 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.007 INFO:tasks.workunit.client.0.smithi136.stdout: "target_bytes": { 2024-09-12T23:37:49.007 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2147483648 2024-09-12T23:37:49.007 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.007 INFO:tasks.workunit.client.0.smithi136.stdout: "unmapped_bytes": { 2024-09-12T23:37:49.007 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4259840 2024-09-12T23:37:49.007 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.007 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.007 INFO:tasks.workunit.client.0.smithi136.stdout: "prioritycache:full": { 2024-09-12T23:37:49.007 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:49.007 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 348127232 2024-09-12T23:37:49.008 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.008 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:49.008 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.008 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.008 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:49.008 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.008 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.008 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:49.008 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 277029658 2024-09-12T23:37:49.008 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.008 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:49.008 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 253669 2024-09-12T23:37:49.008 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.008 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:49.009 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.009 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.009 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:49.009 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.009 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.009 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:49.009 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.009 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.009 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:49.009 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.009 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.009 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:49.009 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.009 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.010 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:49.010 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.010 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.010 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:49.010 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.010 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.010 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:49.010 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.010 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.010 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:49.010 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 70843905 2024-09-12T23:37:49.011 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.011 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.011 INFO:tasks.workunit.client.0.smithi136.stdout: "prioritycache:inc": { 2024-09-12T23:37:49.011 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:49.011 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 348127232 2024-09-12T23:37:49.011 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.011 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:49.011 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.011 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.011 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:49.011 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.011 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.011 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:49.011 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 277029658 2024-09-12T23:37:49.011 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.012 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:49.012 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 36939 2024-09-12T23:37:49.012 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.012 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:49.012 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.012 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.012 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:49.012 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.012 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.012 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:49.012 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.012 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.012 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:49.013 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.013 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.013 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:49.013 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.013 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.013 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:49.013 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.013 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.013 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:49.013 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.013 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.013 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:49.014 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.014 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.014 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:49.014 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 71060635 2024-09-12T23:37:49.014 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.014 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.014 INFO:tasks.workunit.client.0.smithi136.stdout: "prioritycache:kv": { 2024-09-12T23:37:49.014 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:49.014 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 322961408 2024-09-12T23:37:49.014 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.014 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:49.014 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67108864 2024-09-12T23:37:49.014 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.014 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:49.015 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.015 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.015 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:49.015 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 184686438 2024-09-12T23:37:49.015 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.015 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:49.015 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.015 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.015 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:49.015 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.015 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.015 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:49.015 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.015 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.016 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:49.016 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.016 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.016 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:49.016 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.016 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.016 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:49.016 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.016 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.016 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:49.016 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.016 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.017 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:49.017 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.017 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.017 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:49.017 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.017 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.017 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:49.017 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 71166106 2024-09-12T23:37:49.017 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.017 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.017 INFO:tasks.workunit.client.0.smithi136.stdout: "rocksdb": { 2024-09-12T23:37:49.017 INFO:tasks.workunit.client.0.smithi136.stdout: "compact": { 2024-09-12T23:37:49.017 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.017 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.018 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_completed": { 2024-09-12T23:37:49.018 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5 2024-09-12T23:37:49.018 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.018 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_lasted": { 2024-09-12T23:37:49.018 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 26925184 2024-09-12T23:37:49.018 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.018 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_queue_len": { 2024-09-12T23:37:49.018 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.018 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.018 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_queue_merge": { 2024-09-12T23:37:49.018 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.018 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.018 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_running": { 2024-09-12T23:37:49.018 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.019 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.019 INFO:tasks.workunit.client.0.smithi136.stdout: "get_latency": { 2024-09-12T23:37:49.019 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 56828, 2024-09-12T23:37:49.019 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 433189257 2024-09-12T23:37:49.019 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.019 INFO:tasks.workunit.client.0.smithi136.stdout: "rocksdb_write_delay_time": { 2024-09-12T23:37:49.019 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.019 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.019 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.019 INFO:tasks.workunit.client.0.smithi136.stdout: "rocksdb_write_memtable_time": { 2024-09-12T23:37:49.019 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.019 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.019 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.020 INFO:tasks.workunit.client.0.smithi136.stdout: "rocksdb_write_pre_and_post_time": { 2024-09-12T23:37:49.020 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.020 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.020 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.020 INFO:tasks.workunit.client.0.smithi136.stdout: "rocksdb_write_wal_time": { 2024-09-12T23:37:49.020 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.020 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.020 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.020 INFO:tasks.workunit.client.0.smithi136.stdout: "submit_latency": { 2024-09-12T23:37:49.020 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.020 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.020 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.020 INFO:tasks.workunit.client.0.smithi136.stdout: "submit_sync_latency": { 2024-09-12T23:37:49.021 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 2762, 2024-09-12T23:37:49.021 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 940334988 2024-09-12T23:37:49.021 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.021 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.021 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.021 INFO:tasks.workunit.client.0.smithi136.stdout: "osd.0": { 2024-09-12T23:37:49.021 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs": { 2024-09-12T23:37:49.021 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_slow_fallback": { 2024-09-12T23:37:49.021 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.021 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.021 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_slow_size_fallback": { 2024-09-12T23:37:49.021 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.022 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.022 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit_db": { 2024-09-12T23:37:49.022 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 65536 2024-09-12T23:37:49.022 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.022 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit_main": { 2024-09-12T23:37:49.022 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.022 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.022 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit_wal": { 2024-09-12T23:37:49.022 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.022 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.022 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_written_slow": { 2024-09-12T23:37:49.022 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.022 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.023 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_written_sst": { 2024-09-12T23:37:49.023 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 12288 2024-09-12T23:37:49.023 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.023 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_written_wal": { 2024-09-12T23:37:49.023 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2174976 2024-09-12T23:37:49.023 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.023 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_lat": { 2024-09-12T23:37:49.023 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.023 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.023 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.023 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_lock_lat": { 2024-09-12T23:37:49.023 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.023 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.023 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.024 INFO:tasks.workunit.client.0.smithi136.stdout: "db_total_bytes": { 2024-09-12T23:37:49.024 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 95995035648 2024-09-12T23:37:49.024 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.024 INFO:tasks.workunit.client.0.smithi136.stdout: "db_used_bytes": { 2024-09-12T23:37:49.024 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 304480256 2024-09-12T23:37:49.024 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.024 INFO:tasks.workunit.client.0.smithi136.stdout: "log_bytes": { 2024-09-12T23:37:49.024 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1441792 2024-09-12T23:37:49.024 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.024 INFO:tasks.workunit.client.0.smithi136.stdout: "logged_bytes": { 2024-09-12T23:37:49.024 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1302528 2024-09-12T23:37:49.024 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.024 INFO:tasks.workunit.client.0.smithi136.stdout: "max_bytes_db": { 2024-09-12T23:37:49.025 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 312868864 2024-09-12T23:37:49.025 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.025 INFO:tasks.workunit.client.0.smithi136.stdout: "max_bytes_slow": { 2024-09-12T23:37:49.025 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.025 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.025 INFO:tasks.workunit.client.0.smithi136.stdout: "max_bytes_wal": { 2024-09-12T23:37:49.025 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.025 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.025 INFO:tasks.workunit.client.0.smithi136.stdout: "num_files": { 2024-09-12T23:37:49.025 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 14 2024-09-12T23:37:49.025 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.025 INFO:tasks.workunit.client.0.smithi136.stdout: "read_bytes": { 2024-09-12T23:37:49.025 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 195697 2024-09-12T23:37:49.025 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.026 INFO:tasks.workunit.client.0.smithi136.stdout: "read_count": { 2024-09-12T23:37:49.026 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 23 2024-09-12T23:37:49.026 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.026 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes": { 2024-09-12T23:37:49.026 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1269760 2024-09-12T23:37:49.026 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.026 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes_db": { 2024-09-12T23:37:49.026 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1273856 2024-09-12T23:37:49.026 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.026 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes_slow": { 2024-09-12T23:37:49.026 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.027 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.027 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes_wal": { 2024-09-12T23:37:49.027 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.027 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.027 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_count": { 2024-09-12T23:37:49.027 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 15 2024-09-12T23:37:49.027 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.027 INFO:tasks.workunit.client.0.smithi136.stdout: "read_prefetch_bytes": { 2024-09-12T23:37:49.027 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4789 2024-09-12T23:37:49.027 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.027 INFO:tasks.workunit.client.0.smithi136.stdout: "read_prefetch_count": { 2024-09-12T23:37:49.027 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4 2024-09-12T23:37:49.027 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.028 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_buffer_bytes": { 2024-09-12T23:37:49.028 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5589 2024-09-12T23:37:49.028 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.028 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_buffer_count": { 2024-09-12T23:37:49.028 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 24 2024-09-12T23:37:49.028 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.028 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_bytes": { 2024-09-12T23:37:49.028 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5589 2024-09-12T23:37:49.028 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.028 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_count": { 2024-09-12T23:37:49.028 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 24 2024-09-12T23:37:49.028 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.028 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes": { 2024-09-12T23:37:49.029 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.029 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.029 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes_db": { 2024-09-12T23:37:49.029 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.029 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.029 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes_slow": { 2024-09-12T23:37:49.029 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.029 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.029 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes_wal": { 2024-09-12T23:37:49.029 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.029 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.029 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_count": { 2024-09-12T23:37:49.029 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.029 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.030 INFO:tasks.workunit.client.0.smithi136.stdout: "slow_total_bytes": { 2024-09-12T23:37:49.030 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.030 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.030 INFO:tasks.workunit.client.0.smithi136.stdout: "slow_used_bytes": { 2024-09-12T23:37:49.030 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.030 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.030 INFO:tasks.workunit.client.0.smithi136.stdout: "wal_total_bytes": { 2024-09-12T23:37:49.030 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.030 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.030 INFO:tasks.workunit.client.0.smithi136.stdout: "wal_used_bytes": { 2024-09-12T23:37:49.030 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.030 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.030 INFO:tasks.workunit.client.0.smithi136.stdout: "write_bytes": { 2024-09-12T23:37:49.030 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 3551232 2024-09-12T23:37:49.031 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.031 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.031 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore": { 2024-09-12T23:37:49.031 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit": { 2024-09-12T23:37:49.031 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4096 2024-09-12T23:37:49.031 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.031 INFO:tasks.workunit.client.0.smithi136.stdout: "allocated": { 2024-09-12T23:37:49.031 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1257472 2024-09-12T23:37:49.031 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.031 INFO:tasks.workunit.client.0.smithi136.stdout: "clist_lat": { 2024-09-12T23:37:49.031 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 4, 2024-09-12T23:37:49.031 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 332907 2024-09-12T23:37:49.032 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.032 INFO:tasks.workunit.client.0.smithi136.stdout: "compress_lat": { 2024-09-12T23:37:49.032 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.032 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.032 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.032 INFO:tasks.workunit.client.0.smithi136.stdout: "compressed": { 2024-09-12T23:37:49.032 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.032 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.032 INFO:tasks.workunit.client.0.smithi136.stdout: "compressed_allocated": { 2024-09-12T23:37:49.032 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.032 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.033 INFO:tasks.workunit.client.0.smithi136.stdout: "compressed_original": { 2024-09-12T23:37:49.033 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.033 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.033 INFO:tasks.workunit.client.0.smithi136.stdout: "csum_lat": { 2024-09-12T23:37:49.033 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 22, 2024-09-12T23:37:49.033 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 134139 2024-09-12T23:37:49.033 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.033 INFO:tasks.workunit.client.0.smithi136.stdout: "decompress_lat": { 2024-09-12T23:37:49.033 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.033 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.033 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.033 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_commit_lat": { 2024-09-12T23:37:49.033 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 301, 2024-09-12T23:37:49.033 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 112352271 2024-09-12T23:37:49.034 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.034 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_final_lat": { 2024-09-12T23:37:49.034 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 298, 2024-09-12T23:37:49.034 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 10508356 2024-09-12T23:37:49.034 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.034 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_flush_lat": { 2024-09-12T23:37:49.034 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 301, 2024-09-12T23:37:49.034 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 812782 2024-09-12T23:37:49.034 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.034 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_sync_lat": { 2024-09-12T23:37:49.034 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 301, 2024-09-12T23:37:49.034 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 113165053 2024-09-12T23:37:49.034 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.034 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_get_keys_lat": { 2024-09-12T23:37:49.035 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.035 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.035 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.035 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_get_values_lat": { 2024-09-12T23:37:49.035 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1, 2024-09-12T23:37:49.035 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 49690 2024-09-12T23:37:49.035 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.035 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_lower_bound_lat": { 2024-09-12T23:37:49.035 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 4, 2024-09-12T23:37:49.035 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 16074 2024-09-12T23:37:49.035 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.035 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_next_lat": { 2024-09-12T23:37:49.035 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.035 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.036 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.036 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_seek_to_first_lat": { 2024-09-12T23:37:49.036 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.036 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.036 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.036 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_upper_bound_lat": { 2024-09-12T23:37:49.036 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 4, 2024-09-12T23:37:49.036 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 23121 2024-09-12T23:37:49.036 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.036 INFO:tasks.workunit.client.0.smithi136.stdout: "onode_hits": { 2024-09-12T23:37:49.036 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 652 2024-09-12T23:37:49.036 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.036 INFO:tasks.workunit.client.0.smithi136.stdout: "onode_misses": { 2024-09-12T23:37:49.036 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 212 2024-09-12T23:37:49.037 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.037 INFO:tasks.workunit.client.0.smithi136.stdout: "read_lat": { 2024-09-12T23:37:49.037 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 100, 2024-09-12T23:37:49.037 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4193179 2024-09-12T23:37:49.037 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.037 INFO:tasks.workunit.client.0.smithi136.stdout: "read_onode_meta_lat": { 2024-09-12T23:37:49.037 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 244, 2024-09-12T23:37:49.037 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 371027 2024-09-12T23:37:49.037 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.037 INFO:tasks.workunit.client.0.smithi136.stdout: "read_wait_aio_lat": { 2024-09-12T23:37:49.037 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 144, 2024-09-12T23:37:49.037 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5076855 2024-09-12T23:37:49.037 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.038 INFO:tasks.workunit.client.0.smithi136.stdout: "reads_with_retries": { 2024-09-12T23:37:49.038 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.038 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.038 INFO:tasks.workunit.client.0.smithi136.stdout: "remove_lat": { 2024-09-12T23:37:49.038 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 2, 2024-09-12T23:37:49.038 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 81972 2024-09-12T23:37:49.038 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.038 INFO:tasks.workunit.client.0.smithi136.stdout: "state_aio_wait_lat": { 2024-09-12T23:37:49.038 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 411, 2024-09-12T23:37:49.038 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 8834203 2024-09-12T23:37:49.038 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.039 INFO:tasks.workunit.client.0.smithi136.stdout: "state_deferred_aio_wait_lat": { 2024-09-12T23:37:49.039 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 96, 2024-09-12T23:37:49.039 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 9066660 2024-09-12T23:37:49.039 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.039 INFO:tasks.workunit.client.0.smithi136.stdout: "state_deferred_cleanup_lat": { 2024-09-12T23:37:49.039 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 94, 2024-09-12T23:37:49.039 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 651742211831 2024-09-12T23:37:49.039 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.039 INFO:tasks.workunit.client.0.smithi136.stdout: "state_deferred_queued_lat": { 2024-09-12T23:37:49.039 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 96, 2024-09-12T23:37:49.039 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 126682261766 2024-09-12T23:37:49.039 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.039 INFO:tasks.workunit.client.0.smithi136.stdout: "state_done_lat": { 2024-09-12T23:37:49.039 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 409, 2024-09-12T23:37:49.040 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 66669782009 2024-09-12T23:37:49.040 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.040 INFO:tasks.workunit.client.0.smithi136.stdout: "state_finishing_lat": { 2024-09-12T23:37:49.040 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 409, 2024-09-12T23:37:49.040 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 74647 2024-09-12T23:37:49.040 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.040 INFO:tasks.workunit.client.0.smithi136.stdout: "state_io_done_lat": { 2024-09-12T23:37:49.040 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 411, 2024-09-12T23:37:49.040 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 718403 2024-09-12T23:37:49.040 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.040 INFO:tasks.workunit.client.0.smithi136.stdout: "state_kv_commiting_lat": { 2024-09-12T23:37:49.040 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 411, 2024-09-12T23:37:49.040 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 181282271 2024-09-12T23:37:49.041 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.041 INFO:tasks.workunit.client.0.smithi136.stdout: "state_kv_done_lat": { 2024-09-12T23:37:49.041 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 411, 2024-09-12T23:37:49.041 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 219925 2024-09-12T23:37:49.041 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.041 INFO:tasks.workunit.client.0.smithi136.stdout: "state_kv_queued_lat": { 2024-09-12T23:37:49.041 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 411, 2024-09-12T23:37:49.041 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 116565646 2024-09-12T23:37:49.041 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.041 INFO:tasks.workunit.client.0.smithi136.stdout: "state_prepare_lat": { 2024-09-12T23:37:49.041 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 411, 2024-09-12T23:37:49.042 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 34259378 2024-09-12T23:37:49.042 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.042 INFO:tasks.workunit.client.0.smithi136.stdout: "stored": { 2024-09-12T23:37:49.042 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 882305 2024-09-12T23:37:49.042 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.042 INFO:tasks.workunit.client.0.smithi136.stdout: "truncate_lat": { 2024-09-12T23:37:49.042 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 56, 2024-09-12T23:37:49.042 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 457853 2024-09-12T23:37:49.042 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.042 INFO:tasks.workunit.client.0.smithi136.stdout: "txc_commit_lat": { 2024-09-12T23:37:49.042 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 411, 2024-09-12T23:37:49.042 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 341760819 2024-09-12T23:37:49.042 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.043 INFO:tasks.workunit.client.0.smithi136.stdout: "txc_submit_lat": { 2024-09-12T23:37:49.043 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 411, 2024-09-12T23:37:49.043 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 40908287 2024-09-12T23:37:49.043 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.043 INFO:tasks.workunit.client.0.smithi136.stdout: "txc_throttle_lat": { 2024-09-12T23:37:49.043 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 411, 2024-09-12T23:37:49.043 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4190221 2024-09-12T23:37:49.043 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.043 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.043 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache": { 2024-09-12T23:37:49.043 INFO:tasks.workunit.client.0.smithi136.stdout: "cache_bytes": { 2024-09-12T23:37:49.043 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1900976301 2024-09-12T23:37:49.043 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.043 INFO:tasks.workunit.client.0.smithi136.stdout: "heap_bytes": { 2024-09-12T23:37:49.044 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 273072128 2024-09-12T23:37:49.044 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.044 INFO:tasks.workunit.client.0.smithi136.stdout: "mapped_bytes": { 2024-09-12T23:37:49.044 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 272039936 2024-09-12T23:37:49.044 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.044 INFO:tasks.workunit.client.0.smithi136.stdout: "target_bytes": { 2024-09-12T23:37:49.044 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 3183861964 2024-09-12T23:37:49.044 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.044 INFO:tasks.workunit.client.0.smithi136.stdout: "unmapped_bytes": { 2024-09-12T23:37:49.044 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1032192 2024-09-12T23:37:49.044 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.044 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.044 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:data": { 2024-09-12T23:37:49.045 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:49.045 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 159383552 2024-09-12T23:37:49.045 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.045 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:49.045 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.045 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.045 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:49.045 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.045 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.045 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:49.045 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 87868616 2024-09-12T23:37:49.045 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.045 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:49.045 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.046 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.046 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:49.046 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.046 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.046 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:49.046 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.046 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.046 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:49.046 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.046 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.046 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:49.046 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.046 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.046 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:49.047 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 16384 2024-09-12T23:37:49.047 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.047 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:49.047 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.047 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.047 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:49.047 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.047 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.047 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:49.047 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.047 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.047 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:49.047 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 71498552 2024-09-12T23:37:49.047 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.048 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.048 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:kv": { 2024-09-12T23:37:49.048 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:49.048 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 796917760 2024-09-12T23:37:49.048 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.048 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:49.048 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67108864 2024-09-12T23:37:49.048 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.048 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:49.048 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.048 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.049 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:49.049 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 659014623 2024-09-12T23:37:49.049 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.049 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:49.049 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.049 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.049 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:49.049 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.049 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.049 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:49.049 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.049 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.049 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:49.050 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.050 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.050 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:49.050 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.050 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.050 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:49.050 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2128 2024-09-12T23:37:49.050 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.050 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:49.050 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.050 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.050 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:49.050 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.050 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.051 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:49.051 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.051 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.051 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:49.051 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 70792145 2024-09-12T23:37:49.051 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.051 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.051 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:kv_onode": { 2024-09-12T23:37:49.051 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:49.052 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 192937984 2024-09-12T23:37:49.052 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.052 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:49.052 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67108864 2024-09-12T23:37:49.052 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.052 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:49.052 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 464 2024-09-12T23:37:49.052 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.052 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:49.052 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 58579077 2024-09-12T23:37:49.052 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.052 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:49.053 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.053 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.053 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:49.053 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.053 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.053 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:49.053 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.053 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.053 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:49.053 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.053 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.053 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:49.053 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.054 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.054 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:49.054 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.054 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.054 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:49.054 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.054 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.054 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:49.054 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.054 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.054 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:49.054 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.054 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.054 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:49.054 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67249579 2024-09-12T23:37:49.055 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.055 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.055 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:meta": { 2024-09-12T23:37:49.055 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:49.055 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 729808896 2024-09-12T23:37:49.055 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.055 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:49.055 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.055 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.055 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:49.055 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.055 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.055 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:49.055 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 659014623 2024-09-12T23:37:49.056 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.056 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:49.056 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.056 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.056 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:49.056 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.056 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.056 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:49.056 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.056 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.056 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:49.056 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.057 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.057 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:49.057 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.057 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.057 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:49.057 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 267056 2024-09-12T23:37:49.057 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.057 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:49.057 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.057 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.057 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:49.057 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.057 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.057 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:49.057 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.058 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.058 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:49.058 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 70527217 2024-09-12T23:37:49.058 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.058 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.058 INFO:tasks.workunit.client.0.smithi136.stdout: "objecter": { 2024-09-12T23:37:49.058 INFO:tasks.workunit.client.0.smithi136.stdout: "op_active": { 2024-09-12T23:37:49.058 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.058 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.058 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r": { 2024-09-12T23:37:49.058 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.058 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.058 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rmw": { 2024-09-12T23:37:49.058 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.059 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.059 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w": { 2024-09-12T23:37:49.059 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.059 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.059 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.059 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:49.059 INFO:tasks.workunit.client.0.smithi136.stdout: "numpg": { 2024-09-12T23:37:49.059 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 47 2024-09-12T23:37:49.059 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.059 INFO:tasks.workunit.client.0.smithi136.stdout: "numpg_removing": { 2024-09-12T23:37:49.059 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.059 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.059 INFO:tasks.workunit.client.0.smithi136.stdout: "op": { 2024-09-12T23:37:49.060 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 6 2024-09-12T23:37:49.060 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.060 INFO:tasks.workunit.client.0.smithi136.stdout: "op_delayed_degraded": { 2024-09-12T23:37:49.060 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.060 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.060 INFO:tasks.workunit.client.0.smithi136.stdout: "op_delayed_unreadable": { 2024-09-12T23:37:49.060 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.060 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.060 INFO:tasks.workunit.client.0.smithi136.stdout: "op_in_bytes": { 2024-09-12T23:37:49.060 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 604 2024-09-12T23:37:49.060 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.060 INFO:tasks.workunit.client.0.smithi136.stdout: "op_latency": { 2024-09-12T23:37:49.060 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 6, 2024-09-12T23:37:49.061 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 57198533 2024-09-12T23:37:49.061 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.061 INFO:tasks.workunit.client.0.smithi136.stdout: "op_out_bytes": { 2024-09-12T23:37:49.061 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.061 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.061 INFO:tasks.workunit.client.0.smithi136.stdout: "op_prepare_latency": { 2024-09-12T23:37:49.061 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 7, 2024-09-12T23:37:49.061 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5117491 2024-09-12T23:37:49.061 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.061 INFO:tasks.workunit.client.0.smithi136.stdout: "op_process_latency": { 2024-09-12T23:37:49.061 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 6, 2024-09-12T23:37:49.061 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 48672335 2024-09-12T23:37:49.061 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.061 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r": { 2024-09-12T23:37:49.061 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.062 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.062 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_latency": { 2024-09-12T23:37:49.062 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.062 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.062 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.062 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-12T23:37:49.062 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.062 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.062 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_out_bytes": { 2024-09-12T23:37:49.062 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.062 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.062 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_prepare_latency": { 2024-09-12T23:37:49.062 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.062 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.063 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.063 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_process_latency": { 2024-09-12T23:37:49.063 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.063 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.063 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.063 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw": { 2024-09-12T23:37:49.063 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.063 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.063 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_in_bytes": { 2024-09-12T23:37:49.063 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.063 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.063 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_latency": { 2024-09-12T23:37:49.063 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.063 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.064 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.064 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-12T23:37:49.064 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.064 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.064 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-12T23:37:49.064 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.064 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.064 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_out_bytes": { 2024-09-12T23:37:49.064 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.064 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.064 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_prepare_latency": { 2024-09-12T23:37:49.064 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.064 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.065 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.065 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_process_latency": { 2024-09-12T23:37:49.065 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.065 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.065 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.065 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w": { 2024-09-12T23:37:49.065 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 6 2024-09-12T23:37:49.065 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.065 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_in_bytes": { 2024-09-12T23:37:49.065 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 604 2024-09-12T23:37:49.065 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.065 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_latency": { 2024-09-12T23:37:49.065 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 6, 2024-09-12T23:37:49.065 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 57198533 2024-09-12T23:37:49.066 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.066 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-12T23:37:49.066 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.066 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.066 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_prepare_latency": { 2024-09-12T23:37:49.066 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 7, 2024-09-12T23:37:49.066 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5117491 2024-09-12T23:37:49.066 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.066 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_process_latency": { 2024-09-12T23:37:49.066 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 6, 2024-09-12T23:37:49.066 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 48672335 2024-09-12T23:37:49.066 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.066 INFO:tasks.workunit.client.0.smithi136.stdout: "op_wip": { 2024-09-12T23:37:49.066 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.067 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.067 INFO:tasks.workunit.client.0.smithi136.stdout: "recovery_bytes": { 2024-09-12T23:37:49.067 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.067 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.067 INFO:tasks.workunit.client.0.smithi136.stdout: "recovery_ops": { 2024-09-12T23:37:49.067 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.067 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.067 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_bytes": { 2024-09-12T23:37:49.067 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 95995035648 2024-09-12T23:37:49.067 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.067 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_bytes_used": { 2024-09-12T23:37:49.067 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 305754112 2024-09-12T23:37:49.067 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.067 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.068 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.068 INFO:tasks.workunit.client.0.smithi136.stdout: "osd.1": { 2024-09-12T23:37:49.068 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs": { 2024-09-12T23:37:49.068 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_slow_fallback": { 2024-09-12T23:37:49.068 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.068 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.068 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_slow_size_fallback": { 2024-09-12T23:37:49.068 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.068 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.068 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit_db": { 2024-09-12T23:37:49.068 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 65536 2024-09-12T23:37:49.068 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.068 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit_main": { 2024-09-12T23:37:49.068 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.069 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.069 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit_wal": { 2024-09-12T23:37:49.069 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.069 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.069 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_written_slow": { 2024-09-12T23:37:49.069 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.069 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.069 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_written_sst": { 2024-09-12T23:37:49.069 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 12288 2024-09-12T23:37:49.069 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.069 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_written_wal": { 2024-09-12T23:37:49.069 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2064384 2024-09-12T23:37:49.069 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.069 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_lat": { 2024-09-12T23:37:49.070 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.070 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.070 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.070 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_lock_lat": { 2024-09-12T23:37:49.070 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.070 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.070 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.070 INFO:tasks.workunit.client.0.smithi136.stdout: "db_total_bytes": { 2024-09-12T23:37:49.070 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 95995035648 2024-09-12T23:37:49.070 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.070 INFO:tasks.workunit.client.0.smithi136.stdout: "db_used_bytes": { 2024-09-12T23:37:49.070 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 304480256 2024-09-12T23:37:49.070 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.071 INFO:tasks.workunit.client.0.smithi136.stdout: "log_bytes": { 2024-09-12T23:37:49.071 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1417216 2024-09-12T23:37:49.071 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.071 INFO:tasks.workunit.client.0.smithi136.stdout: "logged_bytes": { 2024-09-12T23:37:49.071 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1277952 2024-09-12T23:37:49.071 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.071 INFO:tasks.workunit.client.0.smithi136.stdout: "max_bytes_db": { 2024-09-12T23:37:49.071 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 312868864 2024-09-12T23:37:49.071 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.071 INFO:tasks.workunit.client.0.smithi136.stdout: "max_bytes_slow": { 2024-09-12T23:37:49.071 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.071 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.071 INFO:tasks.workunit.client.0.smithi136.stdout: "max_bytes_wal": { 2024-09-12T23:37:49.071 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.072 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.072 INFO:tasks.workunit.client.0.smithi136.stdout: "num_files": { 2024-09-12T23:37:49.072 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 14 2024-09-12T23:37:49.072 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.072 INFO:tasks.workunit.client.0.smithi136.stdout: "read_bytes": { 2024-09-12T23:37:49.072 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 195697 2024-09-12T23:37:49.072 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.072 INFO:tasks.workunit.client.0.smithi136.stdout: "read_count": { 2024-09-12T23:37:49.072 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 23 2024-09-12T23:37:49.072 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.072 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes": { 2024-09-12T23:37:49.072 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1269760 2024-09-12T23:37:49.072 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.072 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes_db": { 2024-09-12T23:37:49.073 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1273856 2024-09-12T23:37:49.073 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.073 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes_slow": { 2024-09-12T23:37:49.073 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.073 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.073 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes_wal": { 2024-09-12T23:37:49.073 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.073 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.073 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_count": { 2024-09-12T23:37:49.073 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 15 2024-09-12T23:37:49.073 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.073 INFO:tasks.workunit.client.0.smithi136.stdout: "read_prefetch_bytes": { 2024-09-12T23:37:49.073 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4789 2024-09-12T23:37:49.073 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.074 INFO:tasks.workunit.client.0.smithi136.stdout: "read_prefetch_count": { 2024-09-12T23:37:49.074 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4 2024-09-12T23:37:49.074 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.074 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_buffer_bytes": { 2024-09-12T23:37:49.074 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5589 2024-09-12T23:37:49.074 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.074 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_buffer_count": { 2024-09-12T23:37:49.074 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 24 2024-09-12T23:37:49.074 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.074 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_bytes": { 2024-09-12T23:37:49.074 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5589 2024-09-12T23:37:49.074 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.074 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_count": { 2024-09-12T23:37:49.075 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 24 2024-09-12T23:37:49.075 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.075 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes": { 2024-09-12T23:37:49.075 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.075 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.075 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes_db": { 2024-09-12T23:37:49.075 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.075 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.075 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes_slow": { 2024-09-12T23:37:49.075 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.075 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.075 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes_wal": { 2024-09-12T23:37:49.075 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.075 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.076 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_count": { 2024-09-12T23:37:49.076 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.076 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.076 INFO:tasks.workunit.client.0.smithi136.stdout: "slow_total_bytes": { 2024-09-12T23:37:49.076 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.076 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.076 INFO:tasks.workunit.client.0.smithi136.stdout: "slow_used_bytes": { 2024-09-12T23:37:49.076 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.076 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.076 INFO:tasks.workunit.client.0.smithi136.stdout: "wal_total_bytes": { 2024-09-12T23:37:49.076 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.076 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.076 INFO:tasks.workunit.client.0.smithi136.stdout: "wal_used_bytes": { 2024-09-12T23:37:49.077 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.077 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.077 INFO:tasks.workunit.client.0.smithi136.stdout: "write_bytes": { 2024-09-12T23:37:49.077 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 3416064 2024-09-12T23:37:49.077 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.077 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.077 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore": { 2024-09-12T23:37:49.077 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit": { 2024-09-12T23:37:49.077 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4096 2024-09-12T23:37:49.077 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.077 INFO:tasks.workunit.client.0.smithi136.stdout: "allocated": { 2024-09-12T23:37:49.077 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 663552 2024-09-12T23:37:49.077 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.077 INFO:tasks.workunit.client.0.smithi136.stdout: "clist_lat": { 2024-09-12T23:37:49.077 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 5, 2024-09-12T23:37:49.078 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 379561 2024-09-12T23:37:49.078 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.078 INFO:tasks.workunit.client.0.smithi136.stdout: "compress_lat": { 2024-09-12T23:37:49.078 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.078 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.078 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.078 INFO:tasks.workunit.client.0.smithi136.stdout: "compressed": { 2024-09-12T23:37:49.078 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.078 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.078 INFO:tasks.workunit.client.0.smithi136.stdout: "compressed_allocated": { 2024-09-12T23:37:49.078 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.078 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.078 INFO:tasks.workunit.client.0.smithi136.stdout: "compressed_original": { 2024-09-12T23:37:49.078 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.079 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.079 INFO:tasks.workunit.client.0.smithi136.stdout: "csum_lat": { 2024-09-12T23:37:49.079 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 24, 2024-09-12T23:37:49.079 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 124553 2024-09-12T23:37:49.079 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.079 INFO:tasks.workunit.client.0.smithi136.stdout: "decompress_lat": { 2024-09-12T23:37:49.079 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.079 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.079 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.079 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_commit_lat": { 2024-09-12T23:37:49.079 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 295, 2024-09-12T23:37:49.079 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 102050526 2024-09-12T23:37:49.079 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.080 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_final_lat": { 2024-09-12T23:37:49.080 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 278, 2024-09-12T23:37:49.080 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 19355455 2024-09-12T23:37:49.080 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.080 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_flush_lat": { 2024-09-12T23:37:49.080 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 295, 2024-09-12T23:37:49.080 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 760153 2024-09-12T23:37:49.080 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.080 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_sync_lat": { 2024-09-12T23:37:49.080 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 295, 2024-09-12T23:37:49.080 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 102810679 2024-09-12T23:37:49.080 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.080 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_get_keys_lat": { 2024-09-12T23:37:49.081 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.081 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.081 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.081 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_get_values_lat": { 2024-09-12T23:37:49.081 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 3, 2024-09-12T23:37:49.081 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 40717 2024-09-12T23:37:49.081 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.081 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_lower_bound_lat": { 2024-09-12T23:37:49.081 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 7, 2024-09-12T23:37:49.081 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 38416 2024-09-12T23:37:49.081 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.081 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_next_lat": { 2024-09-12T23:37:49.081 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.081 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.081 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.082 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_seek_to_first_lat": { 2024-09-12T23:37:49.082 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.082 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.082 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.082 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_upper_bound_lat": { 2024-09-12T23:37:49.082 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 5, 2024-09-12T23:37:49.082 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 25928 2024-09-12T23:37:49.082 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.082 INFO:tasks.workunit.client.0.smithi136.stdout: "onode_hits": { 2024-09-12T23:37:49.082 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 679 2024-09-12T23:37:49.082 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.082 INFO:tasks.workunit.client.0.smithi136.stdout: "onode_misses": { 2024-09-12T23:37:49.082 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 220 2024-09-12T23:37:49.083 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.083 INFO:tasks.workunit.client.0.smithi136.stdout: "read_lat": { 2024-09-12T23:37:49.083 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 118, 2024-09-12T23:37:49.083 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4784720 2024-09-12T23:37:49.083 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.083 INFO:tasks.workunit.client.0.smithi136.stdout: "read_onode_meta_lat": { 2024-09-12T23:37:49.083 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 276, 2024-09-12T23:37:49.083 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 342650 2024-09-12T23:37:49.083 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.084 INFO:tasks.workunit.client.0.smithi136.stdout: "read_wait_aio_lat": { 2024-09-12T23:37:49.084 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 158, 2024-09-12T23:37:49.084 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5498560 2024-09-12T23:37:49.084 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.084 INFO:tasks.workunit.client.0.smithi136.stdout: "reads_with_retries": { 2024-09-12T23:37:49.084 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.084 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.084 INFO:tasks.workunit.client.0.smithi136.stdout: "remove_lat": { 2024-09-12T23:37:49.084 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 3, 2024-09-12T23:37:49.085 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 233801 2024-09-12T23:37:49.085 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.085 INFO:tasks.workunit.client.0.smithi136.stdout: "state_aio_wait_lat": { 2024-09-12T23:37:49.085 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 362, 2024-09-12T23:37:49.085 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 10768210 2024-09-12T23:37:49.085 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.085 INFO:tasks.workunit.client.0.smithi136.stdout: "state_deferred_aio_wait_lat": { 2024-09-12T23:37:49.085 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 92, 2024-09-12T23:37:49.085 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 9183607 2024-09-12T23:37:49.085 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.085 INFO:tasks.workunit.client.0.smithi136.stdout: "state_deferred_cleanup_lat": { 2024-09-12T23:37:49.086 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 92, 2024-09-12T23:37:49.086 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 579448531851 2024-09-12T23:37:49.086 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.086 INFO:tasks.workunit.client.0.smithi136.stdout: "state_deferred_queued_lat": { 2024-09-12T23:37:49.086 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 92, 2024-09-12T23:37:49.086 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 153525335504 2024-09-12T23:37:49.086 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.086 INFO:tasks.workunit.client.0.smithi136.stdout: "state_done_lat": { 2024-09-12T23:37:49.086 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 362, 2024-09-12T23:37:49.087 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 88666442401 2024-09-12T23:37:49.087 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.087 INFO:tasks.workunit.client.0.smithi136.stdout: "state_finishing_lat": { 2024-09-12T23:37:49.087 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 362, 2024-09-12T23:37:49.087 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 78889 2024-09-12T23:37:49.087 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.087 INFO:tasks.workunit.client.0.smithi136.stdout: "state_io_done_lat": { 2024-09-12T23:37:49.088 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 362, 2024-09-12T23:37:49.088 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 644822 2024-09-12T23:37:49.088 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.088 INFO:tasks.workunit.client.0.smithi136.stdout: "state_kv_commiting_lat": { 2024-09-12T23:37:49.088 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 362, 2024-09-12T23:37:49.088 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 446612270 2024-09-12T23:37:49.088 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.089 INFO:tasks.workunit.client.0.smithi136.stdout: "state_kv_done_lat": { 2024-09-12T23:37:49.089 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 362, 2024-09-12T23:37:49.089 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 217189 2024-09-12T23:37:49.089 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.089 INFO:tasks.workunit.client.0.smithi136.stdout: "state_kv_queued_lat": { 2024-09-12T23:37:49.089 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 362, 2024-09-12T23:37:49.089 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 79652699 2024-09-12T23:37:49.090 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.090 INFO:tasks.workunit.client.0.smithi136.stdout: "state_prepare_lat": { 2024-09-12T23:37:49.090 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 362, 2024-09-12T23:37:49.090 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 31013281 2024-09-12T23:37:49.090 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.090 INFO:tasks.workunit.client.0.smithi136.stdout: "stored": { 2024-09-12T23:37:49.091 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 291715 2024-09-12T23:37:49.091 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.091 INFO:tasks.workunit.client.0.smithi136.stdout: "truncate_lat": { 2024-09-12T23:37:49.091 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 51, 2024-09-12T23:37:49.091 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 434072 2024-09-12T23:37:49.091 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.091 INFO:tasks.workunit.client.0.smithi136.stdout: "txc_commit_lat": { 2024-09-12T23:37:49.092 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 362, 2024-09-12T23:37:49.092 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 568792135 2024-09-12T23:37:49.092 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.092 INFO:tasks.workunit.client.0.smithi136.stdout: "txc_submit_lat": { 2024-09-12T23:37:49.092 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 362, 2024-09-12T23:37:49.092 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 37787654 2024-09-12T23:37:49.092 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.093 INFO:tasks.workunit.client.0.smithi136.stdout: "txc_throttle_lat": { 2024-09-12T23:37:49.093 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 362, 2024-09-12T23:37:49.093 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4574541 2024-09-12T23:37:49.093 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.093 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.093 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache": { 2024-09-12T23:37:49.096 INFO:tasks.workunit.client.0.smithi136.stdout: "cache_bytes": { 2024-09-12T23:37:49.096 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1900976301 2024-09-12T23:37:49.096 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.096 INFO:tasks.workunit.client.0.smithi136.stdout: "heap_bytes": { 2024-09-12T23:37:49.096 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 272023552 2024-09-12T23:37:49.096 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.096 INFO:tasks.workunit.client.0.smithi136.stdout: "mapped_bytes": { 2024-09-12T23:37:49.097 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 271368192 2024-09-12T23:37:49.097 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.097 INFO:tasks.workunit.client.0.smithi136.stdout: "target_bytes": { 2024-09-12T23:37:49.097 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 3183861964 2024-09-12T23:37:49.097 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.097 INFO:tasks.workunit.client.0.smithi136.stdout: "unmapped_bytes": { 2024-09-12T23:37:49.097 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 655360 2024-09-12T23:37:49.098 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.098 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.098 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:data": { 2024-09-12T23:37:49.098 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:49.098 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 159383552 2024-09-12T23:37:49.098 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.098 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:49.099 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.099 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.099 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:49.099 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.099 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.099 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:49.099 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 87870230 2024-09-12T23:37:49.100 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.100 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:49.100 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.100 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.100 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:49.100 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.100 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.101 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:49.101 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.101 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.101 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:49.101 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.101 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.101 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:49.102 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.102 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.102 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:49.102 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 12288 2024-09-12T23:37:49.102 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.102 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:49.102 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.102 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.102 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:49.102 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.102 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.103 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:49.103 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.103 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.103 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:49.103 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 71501034 2024-09-12T23:37:49.103 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.103 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.103 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:kv": { 2024-09-12T23:37:49.103 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:49.103 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 796917760 2024-09-12T23:37:49.104 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.104 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:49.104 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67108864 2024-09-12T23:37:49.104 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.104 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:49.104 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.104 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.104 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:49.104 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 659026731 2024-09-12T23:37:49.104 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.105 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:49.105 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.105 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.105 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:49.105 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.105 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.105 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:49.105 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.105 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.105 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:49.105 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.106 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.106 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:49.106 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.106 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.106 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:49.106 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2128 2024-09-12T23:37:49.106 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.106 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:49.106 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.106 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.106 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:49.106 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.107 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.107 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:49.107 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.107 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.107 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:49.107 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 70780037 2024-09-12T23:37:49.107 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.107 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.107 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:kv_onode": { 2024-09-12T23:37:49.107 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:49.107 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 192937984 2024-09-12T23:37:49.108 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.108 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:49.108 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67108864 2024-09-12T23:37:49.108 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.108 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:49.108 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 464 2024-09-12T23:37:49.108 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.108 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:49.108 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 58580153 2024-09-12T23:37:49.108 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.108 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:49.108 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.109 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.109 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:49.109 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.109 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.109 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:49.109 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.109 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.109 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:49.109 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.109 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.109 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:49.110 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.110 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.110 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:49.110 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.110 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.110 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:49.110 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.110 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.110 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:49.111 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.111 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.111 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:49.111 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.111 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.111 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:49.111 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67248503 2024-09-12T23:37:49.111 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.111 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.111 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:meta": { 2024-09-12T23:37:49.111 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:49.112 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 729808896 2024-09-12T23:37:49.112 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.112 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:49.112 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.112 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.112 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:49.112 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.112 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.112 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:49.112 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 659026731 2024-09-12T23:37:49.113 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.113 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:49.113 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.113 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.113 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:49.113 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.113 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.113 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:49.113 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.113 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.113 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:49.113 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.114 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.114 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:49.114 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.114 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.114 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:49.114 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 249958 2024-09-12T23:37:49.114 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.114 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:49.114 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.114 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.114 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:49.115 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.115 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.115 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:49.115 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.115 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.115 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:49.115 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 70532207 2024-09-12T23:37:49.115 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.115 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.115 INFO:tasks.workunit.client.0.smithi136.stdout: "objecter": { 2024-09-12T23:37:49.115 INFO:tasks.workunit.client.0.smithi136.stdout: "op_active": { 2024-09-12T23:37:49.116 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.116 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.116 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r": { 2024-09-12T23:37:49.116 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.116 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.116 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rmw": { 2024-09-12T23:37:49.116 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.116 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.116 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w": { 2024-09-12T23:37:49.116 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.116 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.116 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.117 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:49.117 INFO:tasks.workunit.client.0.smithi136.stdout: "numpg": { 2024-09-12T23:37:49.117 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 37 2024-09-12T23:37:49.117 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.117 INFO:tasks.workunit.client.0.smithi136.stdout: "numpg_removing": { 2024-09-12T23:37:49.117 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.117 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.117 INFO:tasks.workunit.client.0.smithi136.stdout: "op": { 2024-09-12T23:37:49.117 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 97 2024-09-12T23:37:49.117 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.117 INFO:tasks.workunit.client.0.smithi136.stdout: "op_delayed_degraded": { 2024-09-12T23:37:49.118 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.118 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.118 INFO:tasks.workunit.client.0.smithi136.stdout: "op_delayed_unreadable": { 2024-09-12T23:37:49.118 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.118 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.118 INFO:tasks.workunit.client.0.smithi136.stdout: "op_in_bytes": { 2024-09-12T23:37:49.118 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1640827 2024-09-12T23:37:49.118 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.118 INFO:tasks.workunit.client.0.smithi136.stdout: "op_latency": { 2024-09-12T23:37:49.118 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 97, 2024-09-12T23:37:49.118 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1017073047 2024-09-12T23:37:49.119 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.119 INFO:tasks.workunit.client.0.smithi136.stdout: "op_out_bytes": { 2024-09-12T23:37:49.119 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 602 2024-09-12T23:37:49.119 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.119 INFO:tasks.workunit.client.0.smithi136.stdout: "op_prepare_latency": { 2024-09-12T23:37:49.119 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 97, 2024-09-12T23:37:49.119 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 63225715 2024-09-12T23:37:49.119 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.119 INFO:tasks.workunit.client.0.smithi136.stdout: "op_process_latency": { 2024-09-12T23:37:49.119 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 97, 2024-09-12T23:37:49.120 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 264349402 2024-09-12T23:37:49.120 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.120 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r": { 2024-09-12T23:37:49.120 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 18 2024-09-12T23:37:49.120 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.120 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_latency": { 2024-09-12T23:37:49.120 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 18, 2024-09-12T23:37:49.120 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5773383 2024-09-12T23:37:49.120 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.120 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-12T23:37:49.120 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.121 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.121 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_out_bytes": { 2024-09-12T23:37:49.121 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 602 2024-09-12T23:37:49.121 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.121 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_prepare_latency": { 2024-09-12T23:37:49.121 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 18, 2024-09-12T23:37:49.121 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4867433 2024-09-12T23:37:49.121 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.121 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_process_latency": { 2024-09-12T23:37:49.121 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 18, 2024-09-12T23:37:49.122 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4181989 2024-09-12T23:37:49.122 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.122 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw": { 2024-09-12T23:37:49.122 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 12 2024-09-12T23:37:49.122 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.122 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_in_bytes": { 2024-09-12T23:37:49.122 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.122 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.122 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_latency": { 2024-09-12T23:37:49.122 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 12, 2024-09-12T23:37:49.122 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 20095944 2024-09-12T23:37:49.122 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.123 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-12T23:37:49.123 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.123 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.123 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-12T23:37:49.123 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.123 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.123 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_out_bytes": { 2024-09-12T23:37:49.123 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.123 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.123 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_prepare_latency": { 2024-09-12T23:37:49.124 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 12, 2024-09-12T23:37:49.124 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 8126527 2024-09-12T23:37:49.124 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.124 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_process_latency": { 2024-09-12T23:37:49.124 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 12, 2024-09-12T23:37:49.124 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 18729918 2024-09-12T23:37:49.124 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.124 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w": { 2024-09-12T23:37:49.124 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67 2024-09-12T23:37:49.124 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.124 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_in_bytes": { 2024-09-12T23:37:49.125 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1640827 2024-09-12T23:37:49.125 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.125 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_latency": { 2024-09-12T23:37:49.125 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 67, 2024-09-12T23:37:49.125 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 991203720 2024-09-12T23:37:49.125 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.125 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-12T23:37:49.125 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.125 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.125 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_prepare_latency": { 2024-09-12T23:37:49.126 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 67, 2024-09-12T23:37:49.126 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 50231755 2024-09-12T23:37:49.126 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.126 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_process_latency": { 2024-09-12T23:37:49.126 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 67, 2024-09-12T23:37:49.126 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 241437495 2024-09-12T23:37:49.126 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.126 INFO:tasks.workunit.client.0.smithi136.stdout: "op_wip": { 2024-09-12T23:37:49.126 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.126 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.127 INFO:tasks.workunit.client.0.smithi136.stdout: "recovery_bytes": { 2024-09-12T23:37:49.127 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 590368 2024-09-12T23:37:49.127 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.127 INFO:tasks.workunit.client.0.smithi136.stdout: "recovery_ops": { 2024-09-12T23:37:49.127 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2 2024-09-12T23:37:49.127 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.127 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_bytes": { 2024-09-12T23:37:49.127 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 95995035648 2024-09-12T23:37:49.127 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.127 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_bytes_used": { 2024-09-12T23:37:49.128 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 305152000 2024-09-12T23:37:49.128 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.128 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.128 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.128 INFO:tasks.workunit.client.0.smithi136.stdout: "osd.2": { 2024-09-12T23:37:49.128 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs": { 2024-09-12T23:37:49.128 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_slow_fallback": { 2024-09-12T23:37:49.128 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.128 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.128 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_slow_size_fallback": { 2024-09-12T23:37:49.128 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.128 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.129 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit_db": { 2024-09-12T23:37:49.129 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 65536 2024-09-12T23:37:49.129 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.129 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit_main": { 2024-09-12T23:37:49.129 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.129 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.129 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit_wal": { 2024-09-12T23:37:49.129 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.129 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.129 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_written_slow": { 2024-09-12T23:37:49.130 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.130 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.130 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_written_sst": { 2024-09-12T23:37:49.130 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 12288 2024-09-12T23:37:49.130 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.130 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_written_wal": { 2024-09-12T23:37:49.131 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1630208 2024-09-12T23:37:49.131 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.131 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_lat": { 2024-09-12T23:37:49.131 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.131 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.131 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.131 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_lock_lat": { 2024-09-12T23:37:49.131 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.131 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.131 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.131 INFO:tasks.workunit.client.0.smithi136.stdout: "db_total_bytes": { 2024-09-12T23:37:49.131 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 95995035648 2024-09-12T23:37:49.132 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.132 INFO:tasks.workunit.client.0.smithi136.stdout: "db_used_bytes": { 2024-09-12T23:37:49.132 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 304480256 2024-09-12T23:37:49.132 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.132 INFO:tasks.workunit.client.0.smithi136.stdout: "log_bytes": { 2024-09-12T23:37:49.132 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1093632 2024-09-12T23:37:49.132 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.132 INFO:tasks.workunit.client.0.smithi136.stdout: "logged_bytes": { 2024-09-12T23:37:49.132 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 954368 2024-09-12T23:37:49.132 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.132 INFO:tasks.workunit.client.0.smithi136.stdout: "max_bytes_db": { 2024-09-12T23:37:49.133 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 312868864 2024-09-12T23:37:49.133 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.133 INFO:tasks.workunit.client.0.smithi136.stdout: "max_bytes_slow": { 2024-09-12T23:37:49.133 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.133 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.133 INFO:tasks.workunit.client.0.smithi136.stdout: "max_bytes_wal": { 2024-09-12T23:37:49.133 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.133 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.133 INFO:tasks.workunit.client.0.smithi136.stdout: "num_files": { 2024-09-12T23:37:49.133 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 14 2024-09-12T23:37:49.133 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.133 INFO:tasks.workunit.client.0.smithi136.stdout: "read_bytes": { 2024-09-12T23:37:49.134 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 195697 2024-09-12T23:37:49.134 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.134 INFO:tasks.workunit.client.0.smithi136.stdout: "read_count": { 2024-09-12T23:37:49.134 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 23 2024-09-12T23:37:49.134 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.135 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes": { 2024-09-12T23:37:49.135 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1269760 2024-09-12T23:37:49.135 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.135 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes_db": { 2024-09-12T23:37:49.135 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1273856 2024-09-12T23:37:49.135 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.135 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes_slow": { 2024-09-12T23:37:49.135 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.136 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.136 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes_wal": { 2024-09-12T23:37:49.136 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.136 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.136 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_count": { 2024-09-12T23:37:49.136 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 15 2024-09-12T23:37:49.136 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.136 INFO:tasks.workunit.client.0.smithi136.stdout: "read_prefetch_bytes": { 2024-09-12T23:37:49.136 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4789 2024-09-12T23:37:49.136 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.136 INFO:tasks.workunit.client.0.smithi136.stdout: "read_prefetch_count": { 2024-09-12T23:37:49.136 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4 2024-09-12T23:37:49.137 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.137 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_buffer_bytes": { 2024-09-12T23:37:49.137 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5589 2024-09-12T23:37:49.137 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.137 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_buffer_count": { 2024-09-12T23:37:49.137 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 24 2024-09-12T23:37:49.137 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.137 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_bytes": { 2024-09-12T23:37:49.137 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5589 2024-09-12T23:37:49.138 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.138 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_count": { 2024-09-12T23:37:49.138 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 24 2024-09-12T23:37:49.138 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.138 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes": { 2024-09-12T23:37:49.138 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.138 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.138 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes_db": { 2024-09-12T23:37:49.138 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.138 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.138 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes_slow": { 2024-09-12T23:37:49.139 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.139 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.139 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes_wal": { 2024-09-12T23:37:49.139 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.139 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.139 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_count": { 2024-09-12T23:37:49.139 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.139 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.139 INFO:tasks.workunit.client.0.smithi136.stdout: "slow_total_bytes": { 2024-09-12T23:37:49.140 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.140 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.140 INFO:tasks.workunit.client.0.smithi136.stdout: "slow_used_bytes": { 2024-09-12T23:37:49.140 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.140 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.140 INFO:tasks.workunit.client.0.smithi136.stdout: "wal_total_bytes": { 2024-09-12T23:37:49.140 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.140 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.140 INFO:tasks.workunit.client.0.smithi136.stdout: "wal_used_bytes": { 2024-09-12T23:37:49.140 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.141 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.141 INFO:tasks.workunit.client.0.smithi136.stdout: "write_bytes": { 2024-09-12T23:37:49.141 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2658304 2024-09-12T23:37:49.141 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.141 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.141 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore": { 2024-09-12T23:37:49.141 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit": { 2024-09-12T23:37:49.141 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4096 2024-09-12T23:37:49.141 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.141 INFO:tasks.workunit.client.0.smithi136.stdout: "allocated": { 2024-09-12T23:37:49.142 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 651264 2024-09-12T23:37:49.142 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.142 INFO:tasks.workunit.client.0.smithi136.stdout: "clist_lat": { 2024-09-12T23:37:49.142 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 4, 2024-09-12T23:37:49.142 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 363938 2024-09-12T23:37:49.142 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.142 INFO:tasks.workunit.client.0.smithi136.stdout: "compress_lat": { 2024-09-12T23:37:49.142 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.142 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.142 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.142 INFO:tasks.workunit.client.0.smithi136.stdout: "compressed": { 2024-09-12T23:37:49.142 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.143 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.143 INFO:tasks.workunit.client.0.smithi136.stdout: "compressed_allocated": { 2024-09-12T23:37:49.143 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.143 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.143 INFO:tasks.workunit.client.0.smithi136.stdout: "compressed_original": { 2024-09-12T23:37:49.143 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.143 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.143 INFO:tasks.workunit.client.0.smithi136.stdout: "csum_lat": { 2024-09-12T23:37:49.143 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 9, 2024-09-12T23:37:49.143 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 20087 2024-09-12T23:37:49.144 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.144 INFO:tasks.workunit.client.0.smithi136.stdout: "decompress_lat": { 2024-09-12T23:37:49.144 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.144 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.144 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.144 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_commit_lat": { 2024-09-12T23:37:49.144 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 216, 2024-09-12T23:37:49.144 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 98117890 2024-09-12T23:37:49.144 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.144 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_final_lat": { 2024-09-12T23:37:49.144 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 212, 2024-09-12T23:37:49.145 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5707053 2024-09-12T23:37:49.145 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.145 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_flush_lat": { 2024-09-12T23:37:49.145 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 216, 2024-09-12T23:37:49.145 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 629555 2024-09-12T23:37:49.145 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.145 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_sync_lat": { 2024-09-12T23:37:49.145 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 216, 2024-09-12T23:37:49.145 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 98747445 2024-09-12T23:37:49.145 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.146 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_get_keys_lat": { 2024-09-12T23:37:49.146 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.146 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.146 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.146 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_get_values_lat": { 2024-09-12T23:37:49.146 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 3, 2024-09-12T23:37:49.146 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 38992 2024-09-12T23:37:49.146 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.146 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_lower_bound_lat": { 2024-09-12T23:37:49.146 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 2, 2024-09-12T23:37:49.146 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4246 2024-09-12T23:37:49.146 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.147 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_next_lat": { 2024-09-12T23:37:49.147 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.147 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.147 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.147 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_seek_to_first_lat": { 2024-09-12T23:37:49.147 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.147 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.147 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.147 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_upper_bound_lat": { 2024-09-12T23:37:49.147 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 2, 2024-09-12T23:37:49.148 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 15665 2024-09-12T23:37:49.148 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.148 INFO:tasks.workunit.client.0.smithi136.stdout: "onode_hits": { 2024-09-12T23:37:49.148 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 486 2024-09-12T23:37:49.148 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.148 INFO:tasks.workunit.client.0.smithi136.stdout: "onode_misses": { 2024-09-12T23:37:49.148 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 207 2024-09-12T23:37:49.148 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.148 INFO:tasks.workunit.client.0.smithi136.stdout: "read_lat": { 2024-09-12T23:37:49.148 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 91, 2024-09-12T23:37:49.148 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1701769 2024-09-12T23:37:49.149 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.149 INFO:tasks.workunit.client.0.smithi136.stdout: "read_onode_meta_lat": { 2024-09-12T23:37:49.149 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 213, 2024-09-12T23:37:49.149 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 440111 2024-09-12T23:37:49.149 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.149 INFO:tasks.workunit.client.0.smithi136.stdout: "read_wait_aio_lat": { 2024-09-12T23:37:49.149 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 122, 2024-09-12T23:37:49.149 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1755222 2024-09-12T23:37:49.149 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.149 INFO:tasks.workunit.client.0.smithi136.stdout: "reads_with_retries": { 2024-09-12T23:37:49.149 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.149 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.150 INFO:tasks.workunit.client.0.smithi136.stdout: "remove_lat": { 2024-09-12T23:37:49.150 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 3, 2024-09-12T23:37:49.150 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 150110 2024-09-12T23:37:49.150 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.150 INFO:tasks.workunit.client.0.smithi136.stdout: "state_aio_wait_lat": { 2024-09-12T23:37:49.150 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 288, 2024-09-12T23:37:49.150 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 7414316 2024-09-12T23:37:49.150 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.150 INFO:tasks.workunit.client.0.smithi136.stdout: "state_deferred_aio_wait_lat": { 2024-09-12T23:37:49.150 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 82, 2024-09-12T23:37:49.150 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 8623654 2024-09-12T23:37:49.151 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.151 INFO:tasks.workunit.client.0.smithi136.stdout: "state_deferred_cleanup_lat": { 2024-09-12T23:37:49.151 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 80, 2024-09-12T23:37:49.151 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 493900778732 2024-09-12T23:37:49.151 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.151 INFO:tasks.workunit.client.0.smithi136.stdout: "state_deferred_queued_lat": { 2024-09-12T23:37:49.151 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 82, 2024-09-12T23:37:49.151 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 140893530151 2024-09-12T23:37:49.151 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.151 INFO:tasks.workunit.client.0.smithi136.stdout: "state_done_lat": { 2024-09-12T23:37:49.151 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 286, 2024-09-12T23:37:49.151 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 85018963499 2024-09-12T23:37:49.152 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.152 INFO:tasks.workunit.client.0.smithi136.stdout: "state_finishing_lat": { 2024-09-12T23:37:49.152 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 286, 2024-09-12T23:37:49.152 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 50988 2024-09-12T23:37:49.152 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.152 INFO:tasks.workunit.client.0.smithi136.stdout: "state_io_done_lat": { 2024-09-12T23:37:49.152 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 288, 2024-09-12T23:37:49.152 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 592890 2024-09-12T23:37:49.152 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.152 INFO:tasks.workunit.client.0.smithi136.stdout: "state_kv_commiting_lat": { 2024-09-12T23:37:49.152 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 288, 2024-09-12T23:37:49.153 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 199039664 2024-09-12T23:37:49.153 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.153 INFO:tasks.workunit.client.0.smithi136.stdout: "state_kv_done_lat": { 2024-09-12T23:37:49.153 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 288, 2024-09-12T23:37:49.153 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 160077 2024-09-12T23:37:49.153 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.153 INFO:tasks.workunit.client.0.smithi136.stdout: "state_kv_queued_lat": { 2024-09-12T23:37:49.153 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 288, 2024-09-12T23:37:49.153 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 224601766 2024-09-12T23:37:49.153 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.153 INFO:tasks.workunit.client.0.smithi136.stdout: "state_prepare_lat": { 2024-09-12T23:37:49.154 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 288, 2024-09-12T23:37:49.154 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 41974844 2024-09-12T23:37:49.154 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.154 INFO:tasks.workunit.client.0.smithi136.stdout: "stored": { 2024-09-12T23:37:49.154 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 291536 2024-09-12T23:37:49.154 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.154 INFO:tasks.workunit.client.0.smithi136.stdout: "truncate_lat": { 2024-09-12T23:37:49.154 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 43, 2024-09-12T23:37:49.154 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 363573 2024-09-12T23:37:49.154 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.154 INFO:tasks.workunit.client.0.smithi136.stdout: "txc_commit_lat": { 2024-09-12T23:37:49.154 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 288, 2024-09-12T23:37:49.155 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 473698711 2024-09-12T23:37:49.155 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.155 INFO:tasks.workunit.client.0.smithi136.stdout: "txc_submit_lat": { 2024-09-12T23:37:49.155 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 288, 2024-09-12T23:37:49.155 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 48019125 2024-09-12T23:37:49.155 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.155 INFO:tasks.workunit.client.0.smithi136.stdout: "txc_throttle_lat": { 2024-09-12T23:37:49.155 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 288, 2024-09-12T23:37:49.155 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 19681733 2024-09-12T23:37:49.155 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.155 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.155 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache": { 2024-09-12T23:37:49.156 INFO:tasks.workunit.client.0.smithi136.stdout: "cache_bytes": { 2024-09-12T23:37:49.156 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1900976301 2024-09-12T23:37:49.156 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.156 INFO:tasks.workunit.client.0.smithi136.stdout: "heap_bytes": { 2024-09-12T23:37:49.156 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 270909440 2024-09-12T23:37:49.156 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.156 INFO:tasks.workunit.client.0.smithi136.stdout: "mapped_bytes": { 2024-09-12T23:37:49.156 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 269885440 2024-09-12T23:37:49.156 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.156 INFO:tasks.workunit.client.0.smithi136.stdout: "target_bytes": { 2024-09-12T23:37:49.157 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 3183861964 2024-09-12T23:37:49.157 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.157 INFO:tasks.workunit.client.0.smithi136.stdout: "unmapped_bytes": { 2024-09-12T23:37:49.157 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1024000 2024-09-12T23:37:49.157 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.157 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.157 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:data": { 2024-09-12T23:37:49.157 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:49.157 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 159383552 2024-09-12T23:37:49.157 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.157 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:49.157 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.158 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.158 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:49.158 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.158 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.158 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:49.158 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 87871148 2024-09-12T23:37:49.158 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.158 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:49.158 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.158 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.158 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:49.158 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.159 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.159 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:49.159 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.159 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.159 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:49.159 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.159 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.159 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:49.159 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.159 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.159 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:49.159 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 8192 2024-09-12T23:37:49.159 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.160 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:49.160 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.160 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.160 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:49.160 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.160 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.160 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:49.160 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.160 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.160 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:49.160 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 71504212 2024-09-12T23:37:49.160 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.161 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.161 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:kv": { 2024-09-12T23:37:49.161 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:49.161 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 796917760 2024-09-12T23:37:49.161 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.161 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:49.161 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67108864 2024-09-12T23:37:49.161 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.161 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:49.162 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.162 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.162 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:49.162 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 659033616 2024-09-12T23:37:49.162 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.162 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:49.162 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.162 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.162 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:49.162 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.162 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.162 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:49.163 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.163 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.163 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:49.163 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.163 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.163 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:49.163 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.163 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.163 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:49.163 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2128 2024-09-12T23:37:49.163 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.164 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:49.164 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.164 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.164 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:49.164 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.164 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.164 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:49.164 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.164 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.164 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:49.165 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 70773152 2024-09-12T23:37:49.165 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.165 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.165 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:kv_onode": { 2024-09-12T23:37:49.165 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:49.165 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 192937984 2024-09-12T23:37:49.165 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.165 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:49.165 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67108864 2024-09-12T23:37:49.165 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.165 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:49.166 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 464 2024-09-12T23:37:49.166 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.166 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:49.166 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 58580765 2024-09-12T23:37:49.166 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.166 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:49.166 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.166 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.166 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:49.166 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.166 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.166 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:49.167 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.167 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.167 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:49.167 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.167 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.167 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:49.167 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.167 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.167 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:49.167 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.167 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.168 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:49.168 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.168 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.168 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:49.168 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.168 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.168 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:49.168 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.168 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.168 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:49.168 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67247891 2024-09-12T23:37:49.169 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.169 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.169 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:meta": { 2024-09-12T23:37:49.169 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:49.169 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 729808896 2024-09-12T23:37:49.169 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.169 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:49.169 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.169 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.169 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:49.169 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.170 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.170 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:49.170 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 659033616 2024-09-12T23:37:49.170 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.170 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:49.170 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.170 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.170 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:49.170 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.170 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.170 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:49.170 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.171 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.171 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:49.171 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.171 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.171 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:49.171 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.171 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.171 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:49.171 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 233169 2024-09-12T23:37:49.172 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.172 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:49.172 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.172 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.172 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:49.172 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.172 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.172 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:49.172 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.172 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.172 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:49.172 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 70542111 2024-09-12T23:37:49.173 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.173 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.173 INFO:tasks.workunit.client.0.smithi136.stdout: "objecter": { 2024-09-12T23:37:49.173 INFO:tasks.workunit.client.0.smithi136.stdout: "op_active": { 2024-09-12T23:37:49.173 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.173 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.173 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r": { 2024-09-12T23:37:49.173 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.173 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.173 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rmw": { 2024-09-12T23:37:49.174 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.174 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.174 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w": { 2024-09-12T23:37:49.174 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.174 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.174 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.174 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:49.174 INFO:tasks.workunit.client.0.smithi136.stdout: "numpg": { 2024-09-12T23:37:49.175 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 30 2024-09-12T23:37:49.175 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.175 INFO:tasks.workunit.client.0.smithi136.stdout: "numpg_removing": { 2024-09-12T23:37:49.175 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.175 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.175 INFO:tasks.workunit.client.0.smithi136.stdout: "op": { 2024-09-12T23:37:49.175 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 10 2024-09-12T23:37:49.175 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.175 INFO:tasks.workunit.client.0.smithi136.stdout: "op_delayed_degraded": { 2024-09-12T23:37:49.175 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.175 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.176 INFO:tasks.workunit.client.0.smithi136.stdout: "op_delayed_unreadable": { 2024-09-12T23:37:49.176 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.176 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.176 INFO:tasks.workunit.client.0.smithi136.stdout: "op_in_bytes": { 2024-09-12T23:37:49.176 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.176 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.176 INFO:tasks.workunit.client.0.smithi136.stdout: "op_latency": { 2024-09-12T23:37:49.176 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 10, 2024-09-12T23:37:49.177 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 228728255 2024-09-12T23:37:49.177 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.177 INFO:tasks.workunit.client.0.smithi136.stdout: "op_out_bytes": { 2024-09-12T23:37:49.177 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.177 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.177 INFO:tasks.workunit.client.0.smithi136.stdout: "op_prepare_latency": { 2024-09-12T23:37:49.177 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 10, 2024-09-12T23:37:49.177 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 6725617 2024-09-12T23:37:49.177 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.177 INFO:tasks.workunit.client.0.smithi136.stdout: "op_process_latency": { 2024-09-12T23:37:49.177 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 10, 2024-09-12T23:37:49.178 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 203254064 2024-09-12T23:37:49.178 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.178 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r": { 2024-09-12T23:37:49.178 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.178 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.178 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_latency": { 2024-09-12T23:37:49.178 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.178 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.178 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.178 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-12T23:37:49.178 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.179 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.179 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_out_bytes": { 2024-09-12T23:37:49.179 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.179 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.179 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_prepare_latency": { 2024-09-12T23:37:49.179 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.179 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.179 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.179 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_process_latency": { 2024-09-12T23:37:49.179 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.179 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.179 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.180 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw": { 2024-09-12T23:37:49.180 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.180 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.180 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_in_bytes": { 2024-09-12T23:37:49.180 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.180 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.180 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_latency": { 2024-09-12T23:37:49.180 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.180 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.180 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.180 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-12T23:37:49.181 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.181 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.181 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-12T23:37:49.181 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.181 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.181 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_out_bytes": { 2024-09-12T23:37:49.181 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.181 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.181 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_prepare_latency": { 2024-09-12T23:37:49.181 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.181 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.181 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.182 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_process_latency": { 2024-09-12T23:37:49.182 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.182 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.182 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.182 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w": { 2024-09-12T23:37:49.182 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 10 2024-09-12T23:37:49.182 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.182 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_in_bytes": { 2024-09-12T23:37:49.182 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.182 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.183 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_latency": { 2024-09-12T23:37:49.183 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 10, 2024-09-12T23:37:49.183 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 228728255 2024-09-12T23:37:49.183 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.183 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-12T23:37:49.183 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.183 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.183 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_prepare_latency": { 2024-09-12T23:37:49.183 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 10, 2024-09-12T23:37:49.183 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 6725617 2024-09-12T23:37:49.184 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.184 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_process_latency": { 2024-09-12T23:37:49.184 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 10, 2024-09-12T23:37:49.184 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 203254064 2024-09-12T23:37:49.184 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.184 INFO:tasks.workunit.client.0.smithi136.stdout: "op_wip": { 2024-09-12T23:37:49.184 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.184 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.184 INFO:tasks.workunit.client.0.smithi136.stdout: "recovery_bytes": { 2024-09-12T23:37:49.185 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.185 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.185 INFO:tasks.workunit.client.0.smithi136.stdout: "recovery_ops": { 2024-09-12T23:37:49.185 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1 2024-09-12T23:37:49.185 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.185 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_bytes": { 2024-09-12T23:37:49.185 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 95995035648 2024-09-12T23:37:49.185 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.185 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_bytes_used": { 2024-09-12T23:37:49.185 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 305147904 2024-09-12T23:37:49.185 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.185 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.186 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.186 INFO:tasks.workunit.client.0.smithi136.stdout: "osd.3": { 2024-09-12T23:37:49.186 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs": { 2024-09-12T23:37:49.186 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_slow_fallback": { 2024-09-12T23:37:49.186 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.186 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.186 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_slow_size_fallback": { 2024-09-12T23:37:49.186 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.186 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.186 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit_db": { 2024-09-12T23:37:49.186 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 65536 2024-09-12T23:37:49.186 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.187 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit_main": { 2024-09-12T23:37:49.187 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.187 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.187 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit_wal": { 2024-09-12T23:37:49.187 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.187 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.187 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_written_slow": { 2024-09-12T23:37:49.187 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.187 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.187 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_written_sst": { 2024-09-12T23:37:49.187 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 12288 2024-09-12T23:37:49.188 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.188 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_written_wal": { 2024-09-12T23:37:49.188 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1069056 2024-09-12T23:37:49.188 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.188 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_lat": { 2024-09-12T23:37:49.188 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.188 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.188 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.188 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_lock_lat": { 2024-09-12T23:37:49.188 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.189 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.189 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.189 INFO:tasks.workunit.client.0.smithi136.stdout: "db_total_bytes": { 2024-09-12T23:37:49.189 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 95995035648 2024-09-12T23:37:49.189 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.189 INFO:tasks.workunit.client.0.smithi136.stdout: "db_used_bytes": { 2024-09-12T23:37:49.189 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 304480256 2024-09-12T23:37:49.189 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.189 INFO:tasks.workunit.client.0.smithi136.stdout: "log_bytes": { 2024-09-12T23:37:49.189 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 823296 2024-09-12T23:37:49.190 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.190 INFO:tasks.workunit.client.0.smithi136.stdout: "logged_bytes": { 2024-09-12T23:37:49.190 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 684032 2024-09-12T23:37:49.190 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.190 INFO:tasks.workunit.client.0.smithi136.stdout: "max_bytes_db": { 2024-09-12T23:37:49.190 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 312868864 2024-09-12T23:37:49.190 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.190 INFO:tasks.workunit.client.0.smithi136.stdout: "max_bytes_slow": { 2024-09-12T23:37:49.190 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.190 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.190 INFO:tasks.workunit.client.0.smithi136.stdout: "max_bytes_wal": { 2024-09-12T23:37:49.190 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.191 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.191 INFO:tasks.workunit.client.0.smithi136.stdout: "num_files": { 2024-09-12T23:37:49.191 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 14 2024-09-12T23:37:49.191 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.191 INFO:tasks.workunit.client.0.smithi136.stdout: "read_bytes": { 2024-09-12T23:37:49.191 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 195697 2024-09-12T23:37:49.191 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.191 INFO:tasks.workunit.client.0.smithi136.stdout: "read_count": { 2024-09-12T23:37:49.191 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 23 2024-09-12T23:37:49.191 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.191 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes": { 2024-09-12T23:37:49.192 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1269760 2024-09-12T23:37:49.192 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.192 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes_db": { 2024-09-12T23:37:49.192 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1273856 2024-09-12T23:37:49.192 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.192 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes_slow": { 2024-09-12T23:37:49.192 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.192 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.192 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes_wal": { 2024-09-12T23:37:49.192 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.192 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.193 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_count": { 2024-09-12T23:37:49.193 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 15 2024-09-12T23:37:49.193 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.193 INFO:tasks.workunit.client.0.smithi136.stdout: "read_prefetch_bytes": { 2024-09-12T23:37:49.193 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4789 2024-09-12T23:37:49.193 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.193 INFO:tasks.workunit.client.0.smithi136.stdout: "read_prefetch_count": { 2024-09-12T23:37:49.193 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4 2024-09-12T23:37:49.193 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.193 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_buffer_bytes": { 2024-09-12T23:37:49.193 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5589 2024-09-12T23:37:49.194 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.194 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_buffer_count": { 2024-09-12T23:37:49.194 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 24 2024-09-12T23:37:49.194 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.194 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_bytes": { 2024-09-12T23:37:49.194 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5589 2024-09-12T23:37:49.194 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.194 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_count": { 2024-09-12T23:37:49.194 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 24 2024-09-12T23:37:49.194 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.194 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes": { 2024-09-12T23:37:49.194 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.195 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.195 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes_db": { 2024-09-12T23:37:49.195 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.195 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.195 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes_slow": { 2024-09-12T23:37:49.195 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.195 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.195 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes_wal": { 2024-09-12T23:37:49.195 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.195 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.195 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_count": { 2024-09-12T23:37:49.195 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.196 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.196 INFO:tasks.workunit.client.0.smithi136.stdout: "slow_total_bytes": { 2024-09-12T23:37:49.196 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.196 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.196 INFO:tasks.workunit.client.0.smithi136.stdout: "slow_used_bytes": { 2024-09-12T23:37:49.196 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.196 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.196 INFO:tasks.workunit.client.0.smithi136.stdout: "wal_total_bytes": { 2024-09-12T23:37:49.196 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.196 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.196 INFO:tasks.workunit.client.0.smithi136.stdout: "wal_used_bytes": { 2024-09-12T23:37:49.197 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.197 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.197 INFO:tasks.workunit.client.0.smithi136.stdout: "write_bytes": { 2024-09-12T23:37:49.197 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1826816 2024-09-12T23:37:49.197 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.197 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.197 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore": { 2024-09-12T23:37:49.197 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit": { 2024-09-12T23:37:49.197 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4096 2024-09-12T23:37:49.197 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.197 INFO:tasks.workunit.client.0.smithi136.stdout: "allocated": { 2024-09-12T23:37:49.198 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 671744 2024-09-12T23:37:49.198 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.198 INFO:tasks.workunit.client.0.smithi136.stdout: "clist_lat": { 2024-09-12T23:37:49.198 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.198 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.198 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.198 INFO:tasks.workunit.client.0.smithi136.stdout: "compress_lat": { 2024-09-12T23:37:49.198 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.198 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.198 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.199 INFO:tasks.workunit.client.0.smithi136.stdout: "compressed": { 2024-09-12T23:37:49.199 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.199 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.199 INFO:tasks.workunit.client.0.smithi136.stdout: "compressed_allocated": { 2024-09-12T23:37:49.199 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.199 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.199 INFO:tasks.workunit.client.0.smithi136.stdout: "compressed_original": { 2024-09-12T23:37:49.199 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.199 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.199 INFO:tasks.workunit.client.0.smithi136.stdout: "csum_lat": { 2024-09-12T23:37:49.199 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 3, 2024-09-12T23:37:49.199 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 7967 2024-09-12T23:37:49.200 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.200 INFO:tasks.workunit.client.0.smithi136.stdout: "decompress_lat": { 2024-09-12T23:37:49.200 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.200 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.200 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.200 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_commit_lat": { 2024-09-12T23:37:49.200 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 150, 2024-09-12T23:37:49.200 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 49119257 2024-09-12T23:37:49.200 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.200 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_final_lat": { 2024-09-12T23:37:49.200 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 144, 2024-09-12T23:37:49.201 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 11416815 2024-09-12T23:37:49.201 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.201 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_flush_lat": { 2024-09-12T23:37:49.201 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 150, 2024-09-12T23:37:49.201 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 428455 2024-09-12T23:37:49.201 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.201 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_sync_lat": { 2024-09-12T23:37:49.201 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 150, 2024-09-12T23:37:49.201 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 49547712 2024-09-12T23:37:49.201 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.201 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_get_keys_lat": { 2024-09-12T23:37:49.202 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.202 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.202 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.202 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_get_values_lat": { 2024-09-12T23:37:49.202 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1, 2024-09-12T23:37:49.202 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 42474 2024-09-12T23:37:49.202 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.202 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_lower_bound_lat": { 2024-09-12T23:37:49.202 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 2, 2024-09-12T23:37:49.202 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2754 2024-09-12T23:37:49.202 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.203 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_next_lat": { 2024-09-12T23:37:49.203 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.203 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.203 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.203 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_seek_to_first_lat": { 2024-09-12T23:37:49.203 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.203 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.203 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.203 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_upper_bound_lat": { 2024-09-12T23:37:49.203 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 3, 2024-09-12T23:37:49.203 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 17643 2024-09-12T23:37:49.203 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.203 INFO:tasks.workunit.client.0.smithi136.stdout: "onode_hits": { 2024-09-12T23:37:49.204 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 313 2024-09-12T23:37:49.204 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.204 INFO:tasks.workunit.client.0.smithi136.stdout: "onode_misses": { 2024-09-12T23:37:49.204 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 210 2024-09-12T23:37:49.204 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.204 INFO:tasks.workunit.client.0.smithi136.stdout: "read_lat": { 2024-09-12T23:37:49.204 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 79, 2024-09-12T23:37:49.204 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1230811 2024-09-12T23:37:49.204 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.205 INFO:tasks.workunit.client.0.smithi136.stdout: "read_onode_meta_lat": { 2024-09-12T23:37:49.205 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 135, 2024-09-12T23:37:49.205 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 370609 2024-09-12T23:37:49.205 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.205 INFO:tasks.workunit.client.0.smithi136.stdout: "read_wait_aio_lat": { 2024-09-12T23:37:49.205 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 56, 2024-09-12T23:37:49.205 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 542793 2024-09-12T23:37:49.205 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.205 INFO:tasks.workunit.client.0.smithi136.stdout: "reads_with_retries": { 2024-09-12T23:37:49.205 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.205 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.206 INFO:tasks.workunit.client.0.smithi136.stdout: "remove_lat": { 2024-09-12T23:37:49.206 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.206 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.206 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.206 INFO:tasks.workunit.client.0.smithi136.stdout: "state_aio_wait_lat": { 2024-09-12T23:37:49.206 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 228, 2024-09-12T23:37:49.206 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5822849 2024-09-12T23:37:49.206 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.206 INFO:tasks.workunit.client.0.smithi136.stdout: "state_deferred_aio_wait_lat": { 2024-09-12T23:37:49.206 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 46, 2024-09-12T23:37:49.206 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4403355 2024-09-12T23:37:49.207 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.207 INFO:tasks.workunit.client.0.smithi136.stdout: "state_deferred_cleanup_lat": { 2024-09-12T23:37:49.207 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 43, 2024-09-12T23:37:49.207 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 584895296342 2024-09-12T23:37:49.207 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.207 INFO:tasks.workunit.client.0.smithi136.stdout: "state_deferred_queued_lat": { 2024-09-12T23:37:49.207 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 46, 2024-09-12T23:37:49.207 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 69792450299 2024-09-12T23:37:49.207 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.207 INFO:tasks.workunit.client.0.smithi136.stdout: "state_done_lat": { 2024-09-12T23:37:49.207 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 225, 2024-09-12T23:37:49.208 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 875876 2024-09-12T23:37:49.208 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.208 INFO:tasks.workunit.client.0.smithi136.stdout: "state_finishing_lat": { 2024-09-12T23:37:49.208 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 225, 2024-09-12T23:37:49.208 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 36090 2024-09-12T23:37:49.208 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.208 INFO:tasks.workunit.client.0.smithi136.stdout: "state_io_done_lat": { 2024-09-12T23:37:49.208 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 228, 2024-09-12T23:37:49.208 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 6654008 2024-09-12T23:37:49.208 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.208 INFO:tasks.workunit.client.0.smithi136.stdout: "state_kv_commiting_lat": { 2024-09-12T23:37:49.209 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 228, 2024-09-12T23:37:49.209 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 115512990 2024-09-12T23:37:49.209 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.209 INFO:tasks.workunit.client.0.smithi136.stdout: "state_kv_done_lat": { 2024-09-12T23:37:49.209 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 228, 2024-09-12T23:37:49.209 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 102221 2024-09-12T23:37:49.209 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.209 INFO:tasks.workunit.client.0.smithi136.stdout: "state_kv_queued_lat": { 2024-09-12T23:37:49.209 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 228, 2024-09-12T23:37:49.209 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 86930755 2024-09-12T23:37:49.209 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.210 INFO:tasks.workunit.client.0.smithi136.stdout: "state_prepare_lat": { 2024-09-12T23:37:49.210 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 228, 2024-09-12T23:37:49.210 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 18258942 2024-09-12T23:37:49.210 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.210 INFO:tasks.workunit.client.0.smithi136.stdout: "stored": { 2024-09-12T23:37:49.210 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 294264 2024-09-12T23:37:49.210 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.210 INFO:tasks.workunit.client.0.smithi136.stdout: "truncate_lat": { 2024-09-12T23:37:49.210 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 37, 2024-09-12T23:37:49.210 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 329709 2024-09-12T23:37:49.210 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.210 INFO:tasks.workunit.client.0.smithi136.stdout: "txc_commit_lat": { 2024-09-12T23:37:49.211 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 228, 2024-09-12T23:37:49.211 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 233225664 2024-09-12T23:37:49.211 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.211 INFO:tasks.workunit.client.0.smithi136.stdout: "txc_submit_lat": { 2024-09-12T23:37:49.211 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 228, 2024-09-12T23:37:49.211 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 24580574 2024-09-12T23:37:49.211 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.211 INFO:tasks.workunit.client.0.smithi136.stdout: "txc_throttle_lat": { 2024-09-12T23:37:49.211 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 228, 2024-09-12T23:37:49.211 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 3909498 2024-09-12T23:37:49.211 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.212 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.212 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache": { 2024-09-12T23:37:49.212 INFO:tasks.workunit.client.0.smithi136.stdout: "cache_bytes": { 2024-09-12T23:37:49.212 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1900976301 2024-09-12T23:37:49.212 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.212 INFO:tasks.workunit.client.0.smithi136.stdout: "heap_bytes": { 2024-09-12T23:37:49.212 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 269860864 2024-09-12T23:37:49.212 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.212 INFO:tasks.workunit.client.0.smithi136.stdout: "mapped_bytes": { 2024-09-12T23:37:49.212 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 269434880 2024-09-12T23:37:49.212 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.213 INFO:tasks.workunit.client.0.smithi136.stdout: "target_bytes": { 2024-09-12T23:37:49.213 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 3183861964 2024-09-12T23:37:49.213 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.213 INFO:tasks.workunit.client.0.smithi136.stdout: "unmapped_bytes": { 2024-09-12T23:37:49.213 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 425984 2024-09-12T23:37:49.213 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.213 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.213 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:data": { 2024-09-12T23:37:49.213 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:49.213 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 159383552 2024-09-12T23:37:49.213 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.213 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:49.214 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.214 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.214 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:49.214 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.214 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.214 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:49.214 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 87871225 2024-09-12T23:37:49.214 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.214 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:49.214 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.214 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.214 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:49.215 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.215 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.215 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:49.215 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.215 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.215 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:49.215 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.215 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.215 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:49.215 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.215 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.215 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:49.215 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.215 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.216 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:49.216 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.216 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.216 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:49.216 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.216 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.216 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:49.216 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.216 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.216 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:49.216 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 71512327 2024-09-12T23:37:49.217 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.217 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.217 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:kv": { 2024-09-12T23:37:49.217 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:49.217 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 796917760 2024-09-12T23:37:49.217 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.217 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:49.217 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67108864 2024-09-12T23:37:49.217 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.217 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:49.217 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.217 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.217 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:49.218 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 659034191 2024-09-12T23:37:49.218 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.218 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:49.218 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.218 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.218 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:49.218 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.218 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.218 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:49.218 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.218 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.218 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:49.219 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.219 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.219 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:49.219 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.219 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.219 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:49.219 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2128 2024-09-12T23:37:49.219 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.219 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:49.219 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.219 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.219 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:49.220 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.220 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.220 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:49.220 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.220 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.220 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:49.220 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 70772577 2024-09-12T23:37:49.220 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.220 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.220 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:kv_onode": { 2024-09-12T23:37:49.220 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:49.220 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 192937984 2024-09-12T23:37:49.221 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.221 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:49.221 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67108864 2024-09-12T23:37:49.221 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.221 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:49.221 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 464 2024-09-12T23:37:49.221 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.221 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:49.221 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 58580817 2024-09-12T23:37:49.221 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.221 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:49.222 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.222 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.222 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:49.222 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.222 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.222 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:49.222 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.222 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.222 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:49.222 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.222 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.222 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:49.223 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.223 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.223 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:49.223 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.223 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.223 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:49.223 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.223 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.223 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:49.223 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.223 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.223 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:49.224 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.224 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.224 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:49.224 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67247839 2024-09-12T23:37:49.224 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.224 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.224 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:meta": { 2024-09-12T23:37:49.224 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:49.224 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 729808896 2024-09-12T23:37:49.224 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.225 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:49.225 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.225 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.225 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:49.225 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.225 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.225 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:49.225 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 659034191 2024-09-12T23:37:49.225 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.225 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:49.225 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.226 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.226 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:49.226 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.226 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.226 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:49.226 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.226 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.226 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:49.226 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.226 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.226 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:49.226 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.227 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.227 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:49.227 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 245668 2024-09-12T23:37:49.227 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.227 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:49.227 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.227 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.227 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:49.227 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.227 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.227 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:49.228 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.228 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.228 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:49.228 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 70529037 2024-09-12T23:37:49.228 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.228 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.228 INFO:tasks.workunit.client.0.smithi136.stdout: "objecter": { 2024-09-12T23:37:49.228 INFO:tasks.workunit.client.0.smithi136.stdout: "op_active": { 2024-09-12T23:37:49.228 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.228 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.228 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r": { 2024-09-12T23:37:49.229 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.229 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.229 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rmw": { 2024-09-12T23:37:49.229 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.229 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.229 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w": { 2024-09-12T23:37:49.229 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.229 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.229 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.229 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:49.229 INFO:tasks.workunit.client.0.smithi136.stdout: "numpg": { 2024-09-12T23:37:49.229 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 37 2024-09-12T23:37:49.230 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.230 INFO:tasks.workunit.client.0.smithi136.stdout: "numpg_removing": { 2024-09-12T23:37:49.230 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.230 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.230 INFO:tasks.workunit.client.0.smithi136.stdout: "op": { 2024-09-12T23:37:49.230 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.230 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.230 INFO:tasks.workunit.client.0.smithi136.stdout: "op_delayed_degraded": { 2024-09-12T23:37:49.230 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.230 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.230 INFO:tasks.workunit.client.0.smithi136.stdout: "op_delayed_unreadable": { 2024-09-12T23:37:49.231 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.231 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.231 INFO:tasks.workunit.client.0.smithi136.stdout: "op_in_bytes": { 2024-09-12T23:37:49.231 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.231 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.231 INFO:tasks.workunit.client.0.smithi136.stdout: "op_latency": { 2024-09-12T23:37:49.231 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.231 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.231 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.231 INFO:tasks.workunit.client.0.smithi136.stdout: "op_out_bytes": { 2024-09-12T23:37:49.232 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.232 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.232 INFO:tasks.workunit.client.0.smithi136.stdout: "op_prepare_latency": { 2024-09-12T23:37:49.232 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 2, 2024-09-12T23:37:49.232 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 956264 2024-09-12T23:37:49.232 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.232 INFO:tasks.workunit.client.0.smithi136.stdout: "op_process_latency": { 2024-09-12T23:37:49.232 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.232 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.232 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.233 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r": { 2024-09-12T23:37:49.233 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.233 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.233 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_latency": { 2024-09-12T23:37:49.233 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.233 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.233 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.233 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-12T23:37:49.233 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.233 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.234 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_out_bytes": { 2024-09-12T23:37:49.234 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.234 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.234 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_prepare_latency": { 2024-09-12T23:37:49.234 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.234 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.234 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.234 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_process_latency": { 2024-09-12T23:37:49.234 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.234 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.234 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.235 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw": { 2024-09-12T23:37:49.235 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.235 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.235 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_in_bytes": { 2024-09-12T23:37:49.235 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.235 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.235 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_latency": { 2024-09-12T23:37:49.235 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.235 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.235 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.235 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-12T23:37:49.235 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.236 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.236 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-12T23:37:49.236 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.236 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.236 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_out_bytes": { 2024-09-12T23:37:49.236 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.236 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.236 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_prepare_latency": { 2024-09-12T23:37:49.236 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.236 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.236 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.236 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_process_latency": { 2024-09-12T23:37:49.236 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.237 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.237 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.237 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w": { 2024-09-12T23:37:49.237 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.237 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.237 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_in_bytes": { 2024-09-12T23:37:49.237 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.237 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.237 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_latency": { 2024-09-12T23:37:49.237 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.238 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.238 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.238 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-12T23:37:49.238 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.238 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.238 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_prepare_latency": { 2024-09-12T23:37:49.238 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 2, 2024-09-12T23:37:49.238 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 956264 2024-09-12T23:37:49.238 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.238 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_process_latency": { 2024-09-12T23:37:49.238 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.239 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.239 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.239 INFO:tasks.workunit.client.0.smithi136.stdout: "op_wip": { 2024-09-12T23:37:49.239 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.239 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.239 INFO:tasks.workunit.client.0.smithi136.stdout: "recovery_bytes": { 2024-09-12T23:37:49.239 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.239 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.239 INFO:tasks.workunit.client.0.smithi136.stdout: "recovery_ops": { 2024-09-12T23:37:49.239 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.239 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.239 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_bytes": { 2024-09-12T23:37:49.240 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 95995035648 2024-09-12T23:37:49.240 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.240 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_bytes_used": { 2024-09-12T23:37:49.240 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 305160192 2024-09-12T23:37:49.240 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.240 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.240 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.240 INFO:tasks.workunit.client.0.smithi136.stdout: "osd.4": { 2024-09-12T23:37:49.240 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs": { 2024-09-12T23:37:49.240 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_slow_fallback": { 2024-09-12T23:37:49.241 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.241 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.241 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_slow_size_fallback": { 2024-09-12T23:37:49.241 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.241 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.241 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit_db": { 2024-09-12T23:37:49.241 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 65536 2024-09-12T23:37:49.241 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.241 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit_main": { 2024-09-12T23:37:49.241 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.241 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.242 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit_wal": { 2024-09-12T23:37:49.242 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.242 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.242 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_written_slow": { 2024-09-12T23:37:49.242 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.242 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.242 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_written_sst": { 2024-09-12T23:37:49.242 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 12288 2024-09-12T23:37:49.242 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.242 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_written_wal": { 2024-09-12T23:37:49.242 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1228800 2024-09-12T23:37:49.242 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.243 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_lat": { 2024-09-12T23:37:49.243 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.243 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.243 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.243 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_lock_lat": { 2024-09-12T23:37:49.243 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.243 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.243 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.243 INFO:tasks.workunit.client.0.smithi136.stdout: "db_total_bytes": { 2024-09-12T23:37:49.243 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 95995035648 2024-09-12T23:37:49.243 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.244 INFO:tasks.workunit.client.0.smithi136.stdout: "db_used_bytes": { 2024-09-12T23:37:49.244 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 304480256 2024-09-12T23:37:49.244 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.244 INFO:tasks.workunit.client.0.smithi136.stdout: "log_bytes": { 2024-09-12T23:37:49.244 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 954368 2024-09-12T23:37:49.244 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.244 INFO:tasks.workunit.client.0.smithi136.stdout: "logged_bytes": { 2024-09-12T23:37:49.244 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 815104 2024-09-12T23:37:49.244 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.244 INFO:tasks.workunit.client.0.smithi136.stdout: "max_bytes_db": { 2024-09-12T23:37:49.244 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 312868864 2024-09-12T23:37:49.245 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.245 INFO:tasks.workunit.client.0.smithi136.stdout: "max_bytes_slow": { 2024-09-12T23:37:49.245 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.245 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.245 INFO:tasks.workunit.client.0.smithi136.stdout: "max_bytes_wal": { 2024-09-12T23:37:49.245 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.245 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.245 INFO:tasks.workunit.client.0.smithi136.stdout: "num_files": { 2024-09-12T23:37:49.245 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 14 2024-09-12T23:37:49.246 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.246 INFO:tasks.workunit.client.0.smithi136.stdout: "read_bytes": { 2024-09-12T23:37:49.246 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 195697 2024-09-12T23:37:49.246 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.246 INFO:tasks.workunit.client.0.smithi136.stdout: "read_count": { 2024-09-12T23:37:49.246 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 23 2024-09-12T23:37:49.246 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.246 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes": { 2024-09-12T23:37:49.247 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1269760 2024-09-12T23:37:49.247 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.247 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes_db": { 2024-09-12T23:37:49.247 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1273856 2024-09-12T23:37:49.247 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.247 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes_slow": { 2024-09-12T23:37:49.248 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.248 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.248 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes_wal": { 2024-09-12T23:37:49.248 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.248 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.248 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_count": { 2024-09-12T23:37:49.249 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 15 2024-09-12T23:37:49.249 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.249 INFO:tasks.workunit.client.0.smithi136.stdout: "read_prefetch_bytes": { 2024-09-12T23:37:49.249 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4789 2024-09-12T23:37:49.249 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.250 INFO:tasks.workunit.client.0.smithi136.stdout: "read_prefetch_count": { 2024-09-12T23:37:49.250 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4 2024-09-12T23:37:49.250 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.250 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_buffer_bytes": { 2024-09-12T23:37:49.250 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5589 2024-09-12T23:37:49.250 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.251 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_buffer_count": { 2024-09-12T23:37:49.251 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 24 2024-09-12T23:37:49.251 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.251 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_bytes": { 2024-09-12T23:37:49.251 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5589 2024-09-12T23:37:49.252 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.252 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_count": { 2024-09-12T23:37:49.252 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 24 2024-09-12T23:37:49.252 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.252 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes": { 2024-09-12T23:37:49.252 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.253 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.253 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes_db": { 2024-09-12T23:37:49.253 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.253 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.253 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes_slow": { 2024-09-12T23:37:49.254 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.254 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.254 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes_wal": { 2024-09-12T23:37:49.254 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.254 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.255 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_count": { 2024-09-12T23:37:49.255 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.255 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.255 INFO:tasks.workunit.client.0.smithi136.stdout: "slow_total_bytes": { 2024-09-12T23:37:49.255 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.255 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.256 INFO:tasks.workunit.client.0.smithi136.stdout: "slow_used_bytes": { 2024-09-12T23:37:49.256 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.256 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.256 INFO:tasks.workunit.client.0.smithi136.stdout: "wal_total_bytes": { 2024-09-12T23:37:49.256 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.257 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.257 INFO:tasks.workunit.client.0.smithi136.stdout: "wal_used_bytes": { 2024-09-12T23:37:49.257 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.257 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.257 INFO:tasks.workunit.client.0.smithi136.stdout: "write_bytes": { 2024-09-12T23:37:49.258 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2117632 2024-09-12T23:37:49.258 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.258 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.258 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore": { 2024-09-12T23:37:49.258 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit": { 2024-09-12T23:37:49.259 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4096 2024-09-12T23:37:49.259 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.259 INFO:tasks.workunit.client.0.smithi136.stdout: "allocated": { 2024-09-12T23:37:49.259 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 671744 2024-09-12T23:37:49.259 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.260 INFO:tasks.workunit.client.0.smithi136.stdout: "clist_lat": { 2024-09-12T23:37:49.260 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 10, 2024-09-12T23:37:49.260 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 766836 2024-09-12T23:37:49.260 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.260 INFO:tasks.workunit.client.0.smithi136.stdout: "compress_lat": { 2024-09-12T23:37:49.261 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.261 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.261 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.261 INFO:tasks.workunit.client.0.smithi136.stdout: "compressed": { 2024-09-12T23:37:49.261 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.262 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.262 INFO:tasks.workunit.client.0.smithi136.stdout: "compressed_allocated": { 2024-09-12T23:37:49.262 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.262 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.262 INFO:tasks.workunit.client.0.smithi136.stdout: "compressed_original": { 2024-09-12T23:37:49.262 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.263 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.263 INFO:tasks.workunit.client.0.smithi136.stdout: "csum_lat": { 2024-09-12T23:37:49.263 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 7, 2024-09-12T23:37:49.263 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 28741 2024-09-12T23:37:49.263 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.263 INFO:tasks.workunit.client.0.smithi136.stdout: "decompress_lat": { 2024-09-12T23:37:49.264 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.264 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.264 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.264 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_commit_lat": { 2024-09-12T23:37:49.264 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 182, 2024-09-12T23:37:49.264 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 85389164 2024-09-12T23:37:49.265 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.265 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_final_lat": { 2024-09-12T23:37:49.265 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 177, 2024-09-12T23:37:49.265 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 15062744 2024-09-12T23:37:49.265 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.265 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_flush_lat": { 2024-09-12T23:37:49.265 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 182, 2024-09-12T23:37:49.265 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 380993 2024-09-12T23:37:49.265 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.265 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_sync_lat": { 2024-09-12T23:37:49.266 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 182, 2024-09-12T23:37:49.266 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 85770157 2024-09-12T23:37:49.266 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.266 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_get_keys_lat": { 2024-09-12T23:37:49.266 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.266 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.266 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.266 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_get_values_lat": { 2024-09-12T23:37:49.266 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 3, 2024-09-12T23:37:49.266 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 40241 2024-09-12T23:37:49.266 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.266 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_lower_bound_lat": { 2024-09-12T23:37:49.267 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 3, 2024-09-12T23:37:49.267 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 7368 2024-09-12T23:37:49.267 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.267 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_next_lat": { 2024-09-12T23:37:49.267 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.267 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.267 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.267 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_seek_to_first_lat": { 2024-09-12T23:37:49.267 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.267 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.267 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.267 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_upper_bound_lat": { 2024-09-12T23:37:49.268 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 3, 2024-09-12T23:37:49.268 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 14515 2024-09-12T23:37:49.268 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.268 INFO:tasks.workunit.client.0.smithi136.stdout: "onode_hits": { 2024-09-12T23:37:49.268 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 352 2024-09-12T23:37:49.268 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.268 INFO:tasks.workunit.client.0.smithi136.stdout: "onode_misses": { 2024-09-12T23:37:49.268 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 228 2024-09-12T23:37:49.268 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.268 INFO:tasks.workunit.client.0.smithi136.stdout: "read_lat": { 2024-09-12T23:37:49.268 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 86, 2024-09-12T23:37:49.269 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2185975 2024-09-12T23:37:49.269 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.269 INFO:tasks.workunit.client.0.smithi136.stdout: "read_onode_meta_lat": { 2024-09-12T23:37:49.269 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 141, 2024-09-12T23:37:49.269 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 577272 2024-09-12T23:37:49.269 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.269 INFO:tasks.workunit.client.0.smithi136.stdout: "read_wait_aio_lat": { 2024-09-12T23:37:49.269 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 55, 2024-09-12T23:37:49.269 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 986794 2024-09-12T23:37:49.269 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.269 INFO:tasks.workunit.client.0.smithi136.stdout: "reads_with_retries": { 2024-09-12T23:37:49.269 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.270 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.270 INFO:tasks.workunit.client.0.smithi136.stdout: "remove_lat": { 2024-09-12T23:37:49.270 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 6, 2024-09-12T23:37:49.270 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 288725 2024-09-12T23:37:49.270 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.270 INFO:tasks.workunit.client.0.smithi136.stdout: "state_aio_wait_lat": { 2024-09-12T23:37:49.270 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 264, 2024-09-12T23:37:49.270 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 7087454 2024-09-12T23:37:49.270 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.270 INFO:tasks.workunit.client.0.smithi136.stdout: "state_deferred_aio_wait_lat": { 2024-09-12T23:37:49.270 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 37, 2024-09-12T23:37:49.270 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4010336 2024-09-12T23:37:49.271 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.271 INFO:tasks.workunit.client.0.smithi136.stdout: "state_deferred_cleanup_lat": { 2024-09-12T23:37:49.271 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 35, 2024-09-12T23:37:49.271 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 407318272458 2024-09-12T23:37:49.271 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.271 INFO:tasks.workunit.client.0.smithi136.stdout: "state_deferred_queued_lat": { 2024-09-12T23:37:49.271 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 37, 2024-09-12T23:37:49.271 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 53628145715 2024-09-12T23:37:49.271 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.271 INFO:tasks.workunit.client.0.smithi136.stdout: "state_done_lat": { 2024-09-12T23:37:49.272 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 262, 2024-09-12T23:37:49.272 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 6392896272 2024-09-12T23:37:49.272 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.272 INFO:tasks.workunit.client.0.smithi136.stdout: "state_finishing_lat": { 2024-09-12T23:37:49.272 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 262, 2024-09-12T23:37:49.272 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 51575 2024-09-12T23:37:49.272 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.272 INFO:tasks.workunit.client.0.smithi136.stdout: "state_io_done_lat": { 2024-09-12T23:37:49.272 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 264, 2024-09-12T23:37:49.272 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 818704 2024-09-12T23:37:49.272 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.272 INFO:tasks.workunit.client.0.smithi136.stdout: "state_kv_commiting_lat": { 2024-09-12T23:37:49.273 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 264, 2024-09-12T23:37:49.273 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 165551935 2024-09-12T23:37:49.273 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.273 INFO:tasks.workunit.client.0.smithi136.stdout: "state_kv_done_lat": { 2024-09-12T23:37:49.273 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 264, 2024-09-12T23:37:49.273 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 136814 2024-09-12T23:37:49.273 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.273 INFO:tasks.workunit.client.0.smithi136.stdout: "state_kv_queued_lat": { 2024-09-12T23:37:49.273 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 264, 2024-09-12T23:37:49.273 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 237560396 2024-09-12T23:37:49.274 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.274 INFO:tasks.workunit.client.0.smithi136.stdout: "state_prepare_lat": { 2024-09-12T23:37:49.274 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 264, 2024-09-12T23:37:49.274 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 22324584 2024-09-12T23:37:49.274 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.274 INFO:tasks.workunit.client.0.smithi136.stdout: "stored": { 2024-09-12T23:37:49.274 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 292251 2024-09-12T23:37:49.274 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.274 INFO:tasks.workunit.client.0.smithi136.stdout: "truncate_lat": { 2024-09-12T23:37:49.274 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 35, 2024-09-12T23:37:49.275 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 257967 2024-09-12T23:37:49.275 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.275 INFO:tasks.workunit.client.0.smithi136.stdout: "txc_commit_lat": { 2024-09-12T23:37:49.275 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 264, 2024-09-12T23:37:49.275 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 433403579 2024-09-12T23:37:49.275 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.275 INFO:tasks.workunit.client.0.smithi136.stdout: "txc_submit_lat": { 2024-09-12T23:37:49.275 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 264, 2024-09-12T23:37:49.275 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 31406922 2024-09-12T23:37:49.275 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.275 INFO:tasks.workunit.client.0.smithi136.stdout: "txc_throttle_lat": { 2024-09-12T23:37:49.275 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 264, 2024-09-12T23:37:49.276 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 3032421 2024-09-12T23:37:49.276 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.276 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.276 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache": { 2024-09-12T23:37:49.276 INFO:tasks.workunit.client.0.smithi136.stdout: "cache_bytes": { 2024-09-12T23:37:49.276 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1900963507 2024-09-12T23:37:49.276 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.276 INFO:tasks.workunit.client.0.smithi136.stdout: "heap_bytes": { 2024-09-12T23:37:49.277 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 271187968 2024-09-12T23:37:49.277 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.277 INFO:tasks.workunit.client.0.smithi136.stdout: "mapped_bytes": { 2024-09-12T23:37:49.277 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 270163968 2024-09-12T23:37:49.277 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.277 INFO:tasks.workunit.client.0.smithi136.stdout: "target_bytes": { 2024-09-12T23:37:49.277 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 3183846912 2024-09-12T23:37:49.277 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.277 INFO:tasks.workunit.client.0.smithi136.stdout: "unmapped_bytes": { 2024-09-12T23:37:49.278 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1024000 2024-09-12T23:37:49.278 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.278 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.278 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:data": { 2024-09-12T23:37:49.278 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:49.278 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 159383552 2024-09-12T23:37:49.278 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.278 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:49.278 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.278 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.279 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:49.279 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.279 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.279 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:49.279 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 87869977 2024-09-12T23:37:49.279 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.279 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:49.279 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.279 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.279 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:49.279 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.279 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.280 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:49.280 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.280 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.280 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:49.280 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.280 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.280 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:49.280 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.281 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.281 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:49.281 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4096 2024-09-12T23:37:49.281 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.281 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:49.281 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.281 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.282 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:49.282 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.282 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.282 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:49.282 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.282 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.282 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:49.283 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 71509479 2024-09-12T23:37:49.283 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.283 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.283 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:kv": { 2024-09-12T23:37:49.283 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:49.283 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 796917760 2024-09-12T23:37:49.283 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.284 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:49.284 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67108864 2024-09-12T23:37:49.284 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.284 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:49.284 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.284 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.284 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:49.285 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 659024833 2024-09-12T23:37:49.285 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.285 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:49.285 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.285 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.285 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:49.285 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.286 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.286 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:49.286 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.286 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.286 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:49.286 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.286 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.287 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:49.287 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.287 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.287 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:49.287 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2128 2024-09-12T23:37:49.287 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.287 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:49.288 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.288 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.288 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:49.288 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.288 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.288 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:49.288 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.289 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.289 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:49.289 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 70781935 2024-09-12T23:37:49.289 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.289 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.289 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:kv_onode": { 2024-09-12T23:37:49.290 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:49.290 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 192937984 2024-09-12T23:37:49.290 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.290 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:49.290 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67108864 2024-09-12T23:37:49.290 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.290 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:49.291 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 464 2024-09-12T23:37:49.291 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.291 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:49.291 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 58579985 2024-09-12T23:37:49.291 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.291 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:49.291 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.292 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.292 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:49.292 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.292 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.292 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:49.292 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.292 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.293 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:49.293 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.293 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.293 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:49.293 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.293 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.293 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:49.294 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.294 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.294 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:49.294 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.294 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.294 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:49.294 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.294 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.294 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:49.294 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.295 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.295 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:49.295 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67248671 2024-09-12T23:37:49.295 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.295 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.295 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:meta": { 2024-09-12T23:37:49.295 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:49.295 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 729808896 2024-09-12T23:37:49.295 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.295 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:49.295 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.295 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.296 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:49.296 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.296 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.296 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:49.296 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 659024833 2024-09-12T23:37:49.296 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.296 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:49.296 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.296 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.296 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:49.296 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.297 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.297 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:49.297 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.297 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.297 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:49.297 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.297 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.297 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:49.297 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.297 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.298 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:49.298 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 241206 2024-09-12T23:37:49.298 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.298 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:49.298 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.298 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.298 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:49.298 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.298 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.298 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:49.298 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.299 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.299 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:49.299 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 70542857 2024-09-12T23:37:49.299 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.299 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.299 INFO:tasks.workunit.client.0.smithi136.stdout: "objecter": { 2024-09-12T23:37:49.299 INFO:tasks.workunit.client.0.smithi136.stdout: "op_active": { 2024-09-12T23:37:49.299 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.299 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.299 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r": { 2024-09-12T23:37:49.299 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.300 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.300 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rmw": { 2024-09-12T23:37:49.300 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.300 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.300 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w": { 2024-09-12T23:37:49.300 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.300 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.300 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.300 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:49.300 INFO:tasks.workunit.client.0.smithi136.stdout: "numpg": { 2024-09-12T23:37:49.300 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 41 2024-09-12T23:37:49.301 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.301 INFO:tasks.workunit.client.0.smithi136.stdout: "numpg_removing": { 2024-09-12T23:37:49.301 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.301 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.301 INFO:tasks.workunit.client.0.smithi136.stdout: "op": { 2024-09-12T23:37:49.301 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1 2024-09-12T23:37:49.301 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.301 INFO:tasks.workunit.client.0.smithi136.stdout: "op_delayed_degraded": { 2024-09-12T23:37:49.301 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.301 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.301 INFO:tasks.workunit.client.0.smithi136.stdout: "op_delayed_unreadable": { 2024-09-12T23:37:49.302 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.302 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.302 INFO:tasks.workunit.client.0.smithi136.stdout: "op_in_bytes": { 2024-09-12T23:37:49.302 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 22 2024-09-12T23:37:49.302 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.302 INFO:tasks.workunit.client.0.smithi136.stdout: "op_latency": { 2024-09-12T23:37:49.302 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1, 2024-09-12T23:37:49.302 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 10730533 2024-09-12T23:37:49.302 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.302 INFO:tasks.workunit.client.0.smithi136.stdout: "op_out_bytes": { 2024-09-12T23:37:49.302 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.302 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.303 INFO:tasks.workunit.client.0.smithi136.stdout: "op_prepare_latency": { 2024-09-12T23:37:49.304 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1, 2024-09-12T23:37:49.305 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 778785 2024-09-12T23:37:49.305 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.305 INFO:tasks.workunit.client.0.smithi136.stdout: "op_process_latency": { 2024-09-12T23:37:49.305 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1, 2024-09-12T23:37:49.305 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 10643160 2024-09-12T23:37:49.305 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.305 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r": { 2024-09-12T23:37:49.305 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.305 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.305 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_latency": { 2024-09-12T23:37:49.305 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.306 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.306 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.306 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-12T23:37:49.306 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.306 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.306 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_out_bytes": { 2024-09-12T23:37:49.306 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.306 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.306 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_prepare_latency": { 2024-09-12T23:37:49.306 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.306 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.306 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.307 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_process_latency": { 2024-09-12T23:37:49.307 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.307 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.307 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.307 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw": { 2024-09-12T23:37:49.307 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.307 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.307 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_in_bytes": { 2024-09-12T23:37:49.307 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.307 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.307 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_latency": { 2024-09-12T23:37:49.308 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.308 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.308 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.308 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-12T23:37:49.308 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.308 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.308 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-12T23:37:49.308 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.308 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.308 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_out_bytes": { 2024-09-12T23:37:49.308 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.309 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.309 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_prepare_latency": { 2024-09-12T23:37:49.309 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.309 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.309 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.309 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_process_latency": { 2024-09-12T23:37:49.309 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.309 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.309 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.309 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w": { 2024-09-12T23:37:49.309 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1 2024-09-12T23:37:49.310 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.310 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_in_bytes": { 2024-09-12T23:37:49.310 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 22 2024-09-12T23:37:49.310 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.310 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_latency": { 2024-09-12T23:37:49.310 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1, 2024-09-12T23:37:49.310 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 10730533 2024-09-12T23:37:49.310 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.310 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-12T23:37:49.310 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.310 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.311 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_prepare_latency": { 2024-09-12T23:37:49.311 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1, 2024-09-12T23:37:49.311 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 778785 2024-09-12T23:37:49.311 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.311 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_process_latency": { 2024-09-12T23:37:49.311 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1, 2024-09-12T23:37:49.311 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 10643160 2024-09-12T23:37:49.311 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.311 INFO:tasks.workunit.client.0.smithi136.stdout: "op_wip": { 2024-09-12T23:37:49.311 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.311 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.312 INFO:tasks.workunit.client.0.smithi136.stdout: "recovery_bytes": { 2024-09-12T23:37:49.312 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.312 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.312 INFO:tasks.workunit.client.0.smithi136.stdout: "recovery_ops": { 2024-09-12T23:37:49.312 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.312 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.312 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_bytes": { 2024-09-12T23:37:49.312 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 95995035648 2024-09-12T23:37:49.312 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.312 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_bytes_used": { 2024-09-12T23:37:49.312 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 305168384 2024-09-12T23:37:49.313 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.313 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.313 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.313 INFO:tasks.workunit.client.0.smithi136.stdout: "osd.5": { 2024-09-12T23:37:49.313 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs": { 2024-09-12T23:37:49.313 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_slow_fallback": { 2024-09-12T23:37:49.313 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.313 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.313 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_slow_size_fallback": { 2024-09-12T23:37:49.313 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.313 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.313 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit_db": { 2024-09-12T23:37:49.314 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 65536 2024-09-12T23:37:49.314 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.314 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit_main": { 2024-09-12T23:37:49.314 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.314 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.314 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit_wal": { 2024-09-12T23:37:49.314 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.314 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.314 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_written_slow": { 2024-09-12T23:37:49.314 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.314 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.315 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_written_sst": { 2024-09-12T23:37:49.315 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 12288 2024-09-12T23:37:49.315 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.315 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_written_wal": { 2024-09-12T23:37:49.315 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1200128 2024-09-12T23:37:49.315 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.315 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_lat": { 2024-09-12T23:37:49.315 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.315 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.315 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.315 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_lock_lat": { 2024-09-12T23:37:49.316 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.316 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.316 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.316 INFO:tasks.workunit.client.0.smithi136.stdout: "db_total_bytes": { 2024-09-12T23:37:49.316 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 95995035648 2024-09-12T23:37:49.316 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.316 INFO:tasks.workunit.client.0.smithi136.stdout: "db_used_bytes": { 2024-09-12T23:37:49.316 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 304480256 2024-09-12T23:37:49.316 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.316 INFO:tasks.workunit.client.0.smithi136.stdout: "log_bytes": { 2024-09-12T23:37:49.316 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 905216 2024-09-12T23:37:49.316 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.317 INFO:tasks.workunit.client.0.smithi136.stdout: "logged_bytes": { 2024-09-12T23:37:49.317 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 765952 2024-09-12T23:37:49.317 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.317 INFO:tasks.workunit.client.0.smithi136.stdout: "max_bytes_db": { 2024-09-12T23:37:49.317 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 312868864 2024-09-12T23:37:49.317 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.317 INFO:tasks.workunit.client.0.smithi136.stdout: "max_bytes_slow": { 2024-09-12T23:37:49.317 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.317 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.317 INFO:tasks.workunit.client.0.smithi136.stdout: "max_bytes_wal": { 2024-09-12T23:37:49.317 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.318 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.318 INFO:tasks.workunit.client.0.smithi136.stdout: "num_files": { 2024-09-12T23:37:49.318 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 14 2024-09-12T23:37:49.318 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.318 INFO:tasks.workunit.client.0.smithi136.stdout: "read_bytes": { 2024-09-12T23:37:49.318 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 195697 2024-09-12T23:37:49.318 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.318 INFO:tasks.workunit.client.0.smithi136.stdout: "read_count": { 2024-09-12T23:37:49.318 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 23 2024-09-12T23:37:49.318 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.318 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes": { 2024-09-12T23:37:49.318 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1269760 2024-09-12T23:37:49.319 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.319 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes_db": { 2024-09-12T23:37:49.319 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1273856 2024-09-12T23:37:49.319 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.319 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes_slow": { 2024-09-12T23:37:49.319 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.319 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.319 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes_wal": { 2024-09-12T23:37:49.319 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.319 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.319 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_count": { 2024-09-12T23:37:49.320 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 15 2024-09-12T23:37:49.320 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.320 INFO:tasks.workunit.client.0.smithi136.stdout: "read_prefetch_bytes": { 2024-09-12T23:37:49.320 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4789 2024-09-12T23:37:49.320 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.320 INFO:tasks.workunit.client.0.smithi136.stdout: "read_prefetch_count": { 2024-09-12T23:37:49.320 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4 2024-09-12T23:37:49.320 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.320 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_buffer_bytes": { 2024-09-12T23:37:49.320 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5589 2024-09-12T23:37:49.320 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.320 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_buffer_count": { 2024-09-12T23:37:49.321 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 24 2024-09-12T23:37:49.321 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.321 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_bytes": { 2024-09-12T23:37:49.321 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5589 2024-09-12T23:37:49.321 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.321 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_count": { 2024-09-12T23:37:49.321 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 24 2024-09-12T23:37:49.321 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.321 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes": { 2024-09-12T23:37:49.321 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.321 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.322 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes_db": { 2024-09-12T23:37:49.322 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.322 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.322 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes_slow": { 2024-09-12T23:37:49.322 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.322 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.322 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes_wal": { 2024-09-12T23:37:49.322 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.322 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.322 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_count": { 2024-09-12T23:37:49.322 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.322 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.323 INFO:tasks.workunit.client.0.smithi136.stdout: "slow_total_bytes": { 2024-09-12T23:37:49.323 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.323 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.323 INFO:tasks.workunit.client.0.smithi136.stdout: "slow_used_bytes": { 2024-09-12T23:37:49.323 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.323 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.323 INFO:tasks.workunit.client.0.smithi136.stdout: "wal_total_bytes": { 2024-09-12T23:37:49.323 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.323 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.323 INFO:tasks.workunit.client.0.smithi136.stdout: "wal_used_bytes": { 2024-09-12T23:37:49.323 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.324 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.324 INFO:tasks.workunit.client.0.smithi136.stdout: "write_bytes": { 2024-09-12T23:37:49.324 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2039808 2024-09-12T23:37:49.324 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.324 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.324 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore": { 2024-09-12T23:37:49.324 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit": { 2024-09-12T23:37:49.324 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4096 2024-09-12T23:37:49.324 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.324 INFO:tasks.workunit.client.0.smithi136.stdout: "allocated": { 2024-09-12T23:37:49.324 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 663552 2024-09-12T23:37:49.325 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.325 INFO:tasks.workunit.client.0.smithi136.stdout: "clist_lat": { 2024-09-12T23:37:49.325 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 8, 2024-09-12T23:37:49.325 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 618826 2024-09-12T23:37:49.325 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.325 INFO:tasks.workunit.client.0.smithi136.stdout: "compress_lat": { 2024-09-12T23:37:49.325 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.325 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.325 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.325 INFO:tasks.workunit.client.0.smithi136.stdout: "compressed": { 2024-09-12T23:37:49.326 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.326 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.326 INFO:tasks.workunit.client.0.smithi136.stdout: "compressed_allocated": { 2024-09-12T23:37:49.326 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.326 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.326 INFO:tasks.workunit.client.0.smithi136.stdout: "compressed_original": { 2024-09-12T23:37:49.326 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.326 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.326 INFO:tasks.workunit.client.0.smithi136.stdout: "csum_lat": { 2024-09-12T23:37:49.326 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 4, 2024-09-12T23:37:49.326 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 10745 2024-09-12T23:37:49.326 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.326 INFO:tasks.workunit.client.0.smithi136.stdout: "decompress_lat": { 2024-09-12T23:37:49.327 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.327 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.327 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.327 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_commit_lat": { 2024-09-12T23:37:49.327 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 170, 2024-09-12T23:37:49.327 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 62071808 2024-09-12T23:37:49.327 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.327 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_final_lat": { 2024-09-12T23:37:49.327 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 166, 2024-09-12T23:37:49.327 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 8289281 2024-09-12T23:37:49.327 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.327 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_flush_lat": { 2024-09-12T23:37:49.328 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 170, 2024-09-12T23:37:49.328 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 308319 2024-09-12T23:37:49.328 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.328 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_sync_lat": { 2024-09-12T23:37:49.328 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 170, 2024-09-12T23:37:49.328 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 62380127 2024-09-12T23:37:49.328 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.328 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_get_keys_lat": { 2024-09-12T23:37:49.328 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.328 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.329 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.329 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_get_values_lat": { 2024-09-12T23:37:49.329 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 3, 2024-09-12T23:37:49.329 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 45931 2024-09-12T23:37:49.329 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.329 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_lower_bound_lat": { 2024-09-12T23:37:49.329 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 2, 2024-09-12T23:37:49.329 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4017 2024-09-12T23:37:49.329 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.329 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_next_lat": { 2024-09-12T23:37:49.330 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.330 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.330 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.330 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_seek_to_first_lat": { 2024-09-12T23:37:49.330 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.330 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.330 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.330 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_upper_bound_lat": { 2024-09-12T23:37:49.330 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 2, 2024-09-12T23:37:49.330 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4566 2024-09-12T23:37:49.330 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.330 INFO:tasks.workunit.client.0.smithi136.stdout: "onode_hits": { 2024-09-12T23:37:49.331 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 339 2024-09-12T23:37:49.331 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.331 INFO:tasks.workunit.client.0.smithi136.stdout: "onode_misses": { 2024-09-12T23:37:49.331 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 230 2024-09-12T23:37:49.331 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.331 INFO:tasks.workunit.client.0.smithi136.stdout: "read_lat": { 2024-09-12T23:37:49.331 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 81, 2024-09-12T23:37:49.331 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1216840 2024-09-12T23:37:49.331 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.331 INFO:tasks.workunit.client.0.smithi136.stdout: "read_onode_meta_lat": { 2024-09-12T23:37:49.331 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 129, 2024-09-12T23:37:49.331 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 463065 2024-09-12T23:37:49.332 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.332 INFO:tasks.workunit.client.0.smithi136.stdout: "read_wait_aio_lat": { 2024-09-12T23:37:49.332 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 48, 2024-09-12T23:37:49.332 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 582298 2024-09-12T23:37:49.332 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.332 INFO:tasks.workunit.client.0.smithi136.stdout: "reads_with_retries": { 2024-09-12T23:37:49.332 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.332 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.332 INFO:tasks.workunit.client.0.smithi136.stdout: "remove_lat": { 2024-09-12T23:37:49.333 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 5, 2024-09-12T23:37:49.333 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 199719 2024-09-12T23:37:49.333 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.333 INFO:tasks.workunit.client.0.smithi136.stdout: "state_aio_wait_lat": { 2024-09-12T23:37:49.333 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 281, 2024-09-12T23:37:49.333 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4258657 2024-09-12T23:37:49.333 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.333 INFO:tasks.workunit.client.0.smithi136.stdout: "state_deferred_aio_wait_lat": { 2024-09-12T23:37:49.333 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 38, 2024-09-12T23:37:49.333 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 3695282 2024-09-12T23:37:49.333 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.333 INFO:tasks.workunit.client.0.smithi136.stdout: "state_deferred_cleanup_lat": { 2024-09-12T23:37:49.334 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 35, 2024-09-12T23:37:49.334 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 509531519261 2024-09-12T23:37:49.334 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.334 INFO:tasks.workunit.client.0.smithi136.stdout: "state_deferred_queued_lat": { 2024-09-12T23:37:49.334 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 38, 2024-09-12T23:37:49.334 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 64810170308 2024-09-12T23:37:49.334 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.334 INFO:tasks.workunit.client.0.smithi136.stdout: "state_done_lat": { 2024-09-12T23:37:49.335 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 278, 2024-09-12T23:37:49.335 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 6221345120 2024-09-12T23:37:49.335 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.335 INFO:tasks.workunit.client.0.smithi136.stdout: "state_finishing_lat": { 2024-09-12T23:37:49.335 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 278, 2024-09-12T23:37:49.335 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 47453 2024-09-12T23:37:49.335 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.335 INFO:tasks.workunit.client.0.smithi136.stdout: "state_io_done_lat": { 2024-09-12T23:37:49.335 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 281, 2024-09-12T23:37:49.335 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 355872 2024-09-12T23:37:49.335 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.335 INFO:tasks.workunit.client.0.smithi136.stdout: "state_kv_commiting_lat": { 2024-09-12T23:37:49.336 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 281, 2024-09-12T23:37:49.336 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 158103846 2024-09-12T23:37:49.336 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.336 INFO:tasks.workunit.client.0.smithi136.stdout: "state_kv_done_lat": { 2024-09-12T23:37:49.336 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 281, 2024-09-12T23:37:49.336 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 130632 2024-09-12T23:37:49.336 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.336 INFO:tasks.workunit.client.0.smithi136.stdout: "state_kv_queued_lat": { 2024-09-12T23:37:49.336 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 281, 2024-09-12T23:37:49.336 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 125272432 2024-09-12T23:37:49.337 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.337 INFO:tasks.workunit.client.0.smithi136.stdout: "state_prepare_lat": { 2024-09-12T23:37:49.337 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 281, 2024-09-12T23:37:49.337 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 21794104 2024-09-12T23:37:49.337 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.337 INFO:tasks.workunit.client.0.smithi136.stdout: "stored": { 2024-09-12T23:37:49.337 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 293639 2024-09-12T23:37:49.337 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.337 INFO:tasks.workunit.client.0.smithi136.stdout: "truncate_lat": { 2024-09-12T23:37:49.337 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 30, 2024-09-12T23:37:49.337 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 212811 2024-09-12T23:37:49.338 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.338 INFO:tasks.workunit.client.0.smithi136.stdout: "txc_commit_lat": { 2024-09-12T23:37:49.338 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 281, 2024-09-12T23:37:49.338 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 309841697 2024-09-12T23:37:49.338 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.338 INFO:tasks.workunit.client.0.smithi136.stdout: "txc_submit_lat": { 2024-09-12T23:37:49.338 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 281, 2024-09-12T23:37:49.338 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 25366187 2024-09-12T23:37:49.338 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.338 INFO:tasks.workunit.client.0.smithi136.stdout: "txc_throttle_lat": { 2024-09-12T23:37:49.338 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 281, 2024-09-12T23:37:49.339 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5054111 2024-09-12T23:37:49.339 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.339 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.339 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache": { 2024-09-12T23:37:49.339 INFO:tasks.workunit.client.0.smithi136.stdout: "cache_bytes": { 2024-09-12T23:37:49.339 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1900963507 2024-09-12T23:37:49.339 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.339 INFO:tasks.workunit.client.0.smithi136.stdout: "heap_bytes": { 2024-09-12T23:37:49.339 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 269926400 2024-09-12T23:37:49.339 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.339 INFO:tasks.workunit.client.0.smithi136.stdout: "mapped_bytes": { 2024-09-12T23:37:49.339 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 269713408 2024-09-12T23:37:49.340 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.340 INFO:tasks.workunit.client.0.smithi136.stdout: "target_bytes": { 2024-09-12T23:37:49.340 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 3183846912 2024-09-12T23:37:49.340 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.340 INFO:tasks.workunit.client.0.smithi136.stdout: "unmapped_bytes": { 2024-09-12T23:37:49.340 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 212992 2024-09-12T23:37:49.340 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.340 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.340 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:data": { 2024-09-12T23:37:49.340 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:49.340 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 159383552 2024-09-12T23:37:49.341 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.341 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:49.341 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.341 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.341 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:49.341 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.341 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.341 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:49.341 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 87869932 2024-09-12T23:37:49.342 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.342 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:49.342 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.342 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.342 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:49.342 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.342 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.342 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:49.342 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.342 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.342 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:49.342 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.343 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.343 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:49.343 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.343 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.343 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:49.343 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4096 2024-09-12T23:37:49.343 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.343 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:49.343 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.343 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.343 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:49.343 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.344 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.344 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:49.344 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.344 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.344 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:49.344 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 71509524 2024-09-12T23:37:49.344 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.344 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.344 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:kv": { 2024-09-12T23:37:49.344 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:49.344 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 796917760 2024-09-12T23:37:49.345 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.345 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:49.345 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67108864 2024-09-12T23:37:49.345 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.345 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:49.345 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.345 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.345 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:49.345 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 659024495 2024-09-12T23:37:49.345 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.346 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:49.346 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.346 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.346 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:49.346 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.346 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.346 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:49.346 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.346 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.346 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:49.347 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.347 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.347 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:49.347 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.347 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.347 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:49.347 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2128 2024-09-12T23:37:49.347 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.347 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:49.347 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.347 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.348 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:49.348 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.348 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.348 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:49.348 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.348 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.348 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:49.348 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 70782273 2024-09-12T23:37:49.348 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.348 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.348 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:kv_onode": { 2024-09-12T23:37:49.349 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:49.349 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 192937984 2024-09-12T23:37:49.349 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.349 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:49.349 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67108864 2024-09-12T23:37:49.349 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.349 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:49.349 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 464 2024-09-12T23:37:49.349 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.349 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:49.350 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 58579955 2024-09-12T23:37:49.350 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.350 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:49.350 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.350 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.350 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:49.350 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.350 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.350 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:49.350 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.350 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.351 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:49.351 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.351 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.351 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:49.351 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.351 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.351 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:49.351 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.351 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.351 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:49.351 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.351 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.352 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:49.352 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.352 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.352 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:49.352 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.352 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.352 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:49.352 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67248701 2024-09-12T23:37:49.352 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.352 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.352 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:meta": { 2024-09-12T23:37:49.352 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:49.353 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 729808896 2024-09-12T23:37:49.353 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.353 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:49.353 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.353 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.353 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:49.353 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.353 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.353 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:49.353 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 659024495 2024-09-12T23:37:49.353 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.354 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:49.354 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.354 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.354 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:49.354 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.354 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.354 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:49.354 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.354 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.354 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:49.355 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.355 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.355 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:49.355 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.355 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.355 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:49.355 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 250324 2024-09-12T23:37:49.355 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.355 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:49.355 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.355 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.356 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:49.356 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.356 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.356 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:49.356 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.356 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.356 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:49.356 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 70534077 2024-09-12T23:37:49.356 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.356 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.356 INFO:tasks.workunit.client.0.smithi136.stdout: "objecter": { 2024-09-12T23:37:49.357 INFO:tasks.workunit.client.0.smithi136.stdout: "op_active": { 2024-09-12T23:37:49.357 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.357 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.357 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r": { 2024-09-12T23:37:49.357 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.357 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.357 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rmw": { 2024-09-12T23:37:49.357 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.357 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.357 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w": { 2024-09-12T23:37:49.357 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.358 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.358 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.358 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:49.358 INFO:tasks.workunit.client.0.smithi136.stdout: "numpg": { 2024-09-12T23:37:49.358 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 42 2024-09-12T23:37:49.358 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.358 INFO:tasks.workunit.client.0.smithi136.stdout: "numpg_removing": { 2024-09-12T23:37:49.358 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.358 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.358 INFO:tasks.workunit.client.0.smithi136.stdout: "op": { 2024-09-12T23:37:49.358 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.358 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.359 INFO:tasks.workunit.client.0.smithi136.stdout: "op_delayed_degraded": { 2024-09-12T23:37:49.359 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.359 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.359 INFO:tasks.workunit.client.0.smithi136.stdout: "op_delayed_unreadable": { 2024-09-12T23:37:49.359 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.359 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.359 INFO:tasks.workunit.client.0.smithi136.stdout: "op_in_bytes": { 2024-09-12T23:37:49.359 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.359 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.359 INFO:tasks.workunit.client.0.smithi136.stdout: "op_latency": { 2024-09-12T23:37:49.359 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.360 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.360 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.360 INFO:tasks.workunit.client.0.smithi136.stdout: "op_out_bytes": { 2024-09-12T23:37:49.360 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.360 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.360 INFO:tasks.workunit.client.0.smithi136.stdout: "op_prepare_latency": { 2024-09-12T23:37:49.360 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.360 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.360 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.360 INFO:tasks.workunit.client.0.smithi136.stdout: "op_process_latency": { 2024-09-12T23:37:49.361 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.361 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.361 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.361 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r": { 2024-09-12T23:37:49.361 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.361 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.361 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_latency": { 2024-09-12T23:37:49.361 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.361 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.361 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.361 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-12T23:37:49.361 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.362 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.362 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_out_bytes": { 2024-09-12T23:37:49.362 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.362 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.362 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_prepare_latency": { 2024-09-12T23:37:49.362 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.362 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.362 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.362 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_process_latency": { 2024-09-12T23:37:49.362 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.363 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.363 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.363 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw": { 2024-09-12T23:37:49.363 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.363 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.363 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_in_bytes": { 2024-09-12T23:37:49.363 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.363 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.363 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_latency": { 2024-09-12T23:37:49.363 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.363 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.363 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.364 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-12T23:37:49.364 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.364 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.364 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-12T23:37:49.364 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.364 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.364 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_out_bytes": { 2024-09-12T23:37:49.364 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.364 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.364 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_prepare_latency": { 2024-09-12T23:37:49.364 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.365 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.365 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.365 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_process_latency": { 2024-09-12T23:37:49.365 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.365 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.365 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.365 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w": { 2024-09-12T23:37:49.365 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.365 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.366 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_in_bytes": { 2024-09-12T23:37:49.366 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.366 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.366 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_latency": { 2024-09-12T23:37:49.366 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.366 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.367 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.367 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-12T23:37:49.367 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.367 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.367 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_prepare_latency": { 2024-09-12T23:37:49.367 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.367 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.368 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.368 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_process_latency": { 2024-09-12T23:37:49.368 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.368 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.368 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.368 INFO:tasks.workunit.client.0.smithi136.stdout: "op_wip": { 2024-09-12T23:37:49.368 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.368 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.368 INFO:tasks.workunit.client.0.smithi136.stdout: "recovery_bytes": { 2024-09-12T23:37:49.368 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 590368 2024-09-12T23:37:49.368 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.369 INFO:tasks.workunit.client.0.smithi136.stdout: "recovery_ops": { 2024-09-12T23:37:49.369 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2 2024-09-12T23:37:49.369 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.369 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_bytes": { 2024-09-12T23:37:49.369 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 95995035648 2024-09-12T23:37:49.369 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.369 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_bytes_used": { 2024-09-12T23:37:49.369 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 305152000 2024-09-12T23:37:49.369 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.369 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.370 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.370 INFO:tasks.workunit.client.0.smithi136.stdout: "osd.6": { 2024-09-12T23:37:49.370 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs": { 2024-09-12T23:37:49.370 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_slow_fallback": { 2024-09-12T23:37:49.370 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.370 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.370 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_slow_size_fallback": { 2024-09-12T23:37:49.370 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.370 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.370 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit_db": { 2024-09-12T23:37:49.370 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 65536 2024-09-12T23:37:49.370 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.371 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit_main": { 2024-09-12T23:37:49.371 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.371 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.371 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit_wal": { 2024-09-12T23:37:49.371 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.371 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.371 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_written_slow": { 2024-09-12T23:37:49.371 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.371 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.371 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_written_sst": { 2024-09-12T23:37:49.371 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 12288 2024-09-12T23:37:49.372 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.372 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_written_wal": { 2024-09-12T23:37:49.372 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1208320 2024-09-12T23:37:49.372 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.372 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_lat": { 2024-09-12T23:37:49.372 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.372 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.372 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.372 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_lock_lat": { 2024-09-12T23:37:49.372 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.372 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.373 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.373 INFO:tasks.workunit.client.0.smithi136.stdout: "db_total_bytes": { 2024-09-12T23:37:49.373 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 95995035648 2024-09-12T23:37:49.373 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.373 INFO:tasks.workunit.client.0.smithi136.stdout: "db_used_bytes": { 2024-09-12T23:37:49.373 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 304480256 2024-09-12T23:37:49.373 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.373 INFO:tasks.workunit.client.0.smithi136.stdout: "log_bytes": { 2024-09-12T23:37:49.373 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 962560 2024-09-12T23:37:49.373 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.374 INFO:tasks.workunit.client.0.smithi136.stdout: "logged_bytes": { 2024-09-12T23:37:49.374 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 823296 2024-09-12T23:37:49.374 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.374 INFO:tasks.workunit.client.0.smithi136.stdout: "max_bytes_db": { 2024-09-12T23:37:49.374 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 312868864 2024-09-12T23:37:49.374 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.374 INFO:tasks.workunit.client.0.smithi136.stdout: "max_bytes_slow": { 2024-09-12T23:37:49.374 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.374 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.374 INFO:tasks.workunit.client.0.smithi136.stdout: "max_bytes_wal": { 2024-09-12T23:37:49.374 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.375 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.375 INFO:tasks.workunit.client.0.smithi136.stdout: "num_files": { 2024-09-12T23:37:49.375 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 14 2024-09-12T23:37:49.375 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.375 INFO:tasks.workunit.client.0.smithi136.stdout: "read_bytes": { 2024-09-12T23:37:49.375 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 195697 2024-09-12T23:37:49.375 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.375 INFO:tasks.workunit.client.0.smithi136.stdout: "read_count": { 2024-09-12T23:37:49.375 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 23 2024-09-12T23:37:49.375 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.376 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes": { 2024-09-12T23:37:49.376 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1269760 2024-09-12T23:37:49.376 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.376 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes_db": { 2024-09-12T23:37:49.376 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1273856 2024-09-12T23:37:49.376 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.376 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes_slow": { 2024-09-12T23:37:49.376 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.376 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.376 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes_wal": { 2024-09-12T23:37:49.376 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.377 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.377 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_count": { 2024-09-12T23:37:49.377 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 15 2024-09-12T23:37:49.377 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.377 INFO:tasks.workunit.client.0.smithi136.stdout: "read_prefetch_bytes": { 2024-09-12T23:37:49.377 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4789 2024-09-12T23:37:49.377 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.377 INFO:tasks.workunit.client.0.smithi136.stdout: "read_prefetch_count": { 2024-09-12T23:37:49.378 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4 2024-09-12T23:37:49.378 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.378 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_buffer_bytes": { 2024-09-12T23:37:49.378 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5589 2024-09-12T23:37:49.378 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.378 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_buffer_count": { 2024-09-12T23:37:49.378 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 24 2024-09-12T23:37:49.378 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.378 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_bytes": { 2024-09-12T23:37:49.378 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5589 2024-09-12T23:37:49.378 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.378 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_count": { 2024-09-12T23:37:49.379 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 24 2024-09-12T23:37:49.379 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.379 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes": { 2024-09-12T23:37:49.379 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.379 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.379 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes_db": { 2024-09-12T23:37:49.379 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.379 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.379 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes_slow": { 2024-09-12T23:37:49.379 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.380 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.380 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes_wal": { 2024-09-12T23:37:49.380 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.380 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.380 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_count": { 2024-09-12T23:37:49.380 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.380 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.380 INFO:tasks.workunit.client.0.smithi136.stdout: "slow_total_bytes": { 2024-09-12T23:37:49.380 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.380 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.380 INFO:tasks.workunit.client.0.smithi136.stdout: "slow_used_bytes": { 2024-09-12T23:37:49.381 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.381 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.381 INFO:tasks.workunit.client.0.smithi136.stdout: "wal_total_bytes": { 2024-09-12T23:37:49.381 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.381 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.381 INFO:tasks.workunit.client.0.smithi136.stdout: "wal_used_bytes": { 2024-09-12T23:37:49.381 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.381 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.381 INFO:tasks.workunit.client.0.smithi136.stdout: "write_bytes": { 2024-09-12T23:37:49.381 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2105344 2024-09-12T23:37:49.382 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.382 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.382 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore": { 2024-09-12T23:37:49.382 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit": { 2024-09-12T23:37:49.382 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4096 2024-09-12T23:37:49.382 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.382 INFO:tasks.workunit.client.0.smithi136.stdout: "allocated": { 2024-09-12T23:37:49.382 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1253376 2024-09-12T23:37:49.382 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.382 INFO:tasks.workunit.client.0.smithi136.stdout: "clist_lat": { 2024-09-12T23:37:49.382 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.383 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.383 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.383 INFO:tasks.workunit.client.0.smithi136.stdout: "compress_lat": { 2024-09-12T23:37:49.383 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.383 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.383 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.383 INFO:tasks.workunit.client.0.smithi136.stdout: "compressed": { 2024-09-12T23:37:49.383 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.383 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.383 INFO:tasks.workunit.client.0.smithi136.stdout: "compressed_allocated": { 2024-09-12T23:37:49.383 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.383 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.384 INFO:tasks.workunit.client.0.smithi136.stdout: "compressed_original": { 2024-09-12T23:37:49.384 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.384 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.384 INFO:tasks.workunit.client.0.smithi136.stdout: "csum_lat": { 2024-09-12T23:37:49.384 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 7, 2024-09-12T23:37:49.384 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 19807 2024-09-12T23:37:49.384 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.384 INFO:tasks.workunit.client.0.smithi136.stdout: "decompress_lat": { 2024-09-12T23:37:49.384 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.384 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.384 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.385 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_commit_lat": { 2024-09-12T23:37:49.385 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 184, 2024-09-12T23:37:49.385 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 93235014 2024-09-12T23:37:49.385 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.385 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_final_lat": { 2024-09-12T23:37:49.385 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 183, 2024-09-12T23:37:49.385 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 9170747 2024-09-12T23:37:49.385 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.385 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_flush_lat": { 2024-09-12T23:37:49.385 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 184, 2024-09-12T23:37:49.385 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 295510 2024-09-12T23:37:49.386 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.386 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_sync_lat": { 2024-09-12T23:37:49.386 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 184, 2024-09-12T23:37:49.386 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 93530524 2024-09-12T23:37:49.386 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.386 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_get_keys_lat": { 2024-09-12T23:37:49.386 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.386 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.386 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.386 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_get_values_lat": { 2024-09-12T23:37:49.386 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1, 2024-09-12T23:37:49.387 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 45213 2024-09-12T23:37:49.387 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.387 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_lower_bound_lat": { 2024-09-12T23:37:49.387 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 2, 2024-09-12T23:37:49.387 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2949 2024-09-12T23:37:49.387 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.387 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_next_lat": { 2024-09-12T23:37:49.387 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.387 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.387 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.388 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_seek_to_first_lat": { 2024-09-12T23:37:49.388 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.388 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.388 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.388 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_upper_bound_lat": { 2024-09-12T23:37:49.388 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 3, 2024-09-12T23:37:49.388 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 15110 2024-09-12T23:37:49.388 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.388 INFO:tasks.workunit.client.0.smithi136.stdout: "onode_hits": { 2024-09-12T23:37:49.388 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 305 2024-09-12T23:37:49.388 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.389 INFO:tasks.workunit.client.0.smithi136.stdout: "onode_misses": { 2024-09-12T23:37:49.389 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 234 2024-09-12T23:37:49.389 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.389 INFO:tasks.workunit.client.0.smithi136.stdout: "read_lat": { 2024-09-12T23:37:49.389 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 82, 2024-09-12T23:37:49.389 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1638877 2024-09-12T23:37:49.389 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.389 INFO:tasks.workunit.client.0.smithi136.stdout: "read_onode_meta_lat": { 2024-09-12T23:37:49.389 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 127, 2024-09-12T23:37:49.389 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 451371 2024-09-12T23:37:49.390 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.390 INFO:tasks.workunit.client.0.smithi136.stdout: "read_wait_aio_lat": { 2024-09-12T23:37:49.390 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 45, 2024-09-12T23:37:49.390 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1101545 2024-09-12T23:37:49.390 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.390 INFO:tasks.workunit.client.0.smithi136.stdout: "reads_with_retries": { 2024-09-12T23:37:49.390 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.390 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.390 INFO:tasks.workunit.client.0.smithi136.stdout: "remove_lat": { 2024-09-12T23:37:49.390 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.390 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.391 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.391 INFO:tasks.workunit.client.0.smithi136.stdout: "state_aio_wait_lat": { 2024-09-12T23:37:49.391 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 251, 2024-09-12T23:37:49.391 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 3751221 2024-09-12T23:37:49.391 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.391 INFO:tasks.workunit.client.0.smithi136.stdout: "state_deferred_aio_wait_lat": { 2024-09-12T23:37:49.391 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 33, 2024-09-12T23:37:49.391 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 3174770 2024-09-12T23:37:49.391 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.391 INFO:tasks.workunit.client.0.smithi136.stdout: "state_deferred_cleanup_lat": { 2024-09-12T23:37:49.392 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 30, 2024-09-12T23:37:49.392 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 205667623962 2024-09-12T23:37:49.392 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.392 INFO:tasks.workunit.client.0.smithi136.stdout: "state_deferred_queued_lat": { 2024-09-12T23:37:49.392 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 33, 2024-09-12T23:37:49.392 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 46789027563 2024-09-12T23:37:49.392 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.392 INFO:tasks.workunit.client.0.smithi136.stdout: "state_done_lat": { 2024-09-12T23:37:49.392 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 248, 2024-09-12T23:37:49.392 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 717633 2024-09-12T23:37:49.392 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.393 INFO:tasks.workunit.client.0.smithi136.stdout: "state_finishing_lat": { 2024-09-12T23:37:49.393 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 248, 2024-09-12T23:37:49.393 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 53906 2024-09-12T23:37:49.393 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.393 INFO:tasks.workunit.client.0.smithi136.stdout: "state_io_done_lat": { 2024-09-12T23:37:49.393 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 251, 2024-09-12T23:37:49.393 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 364264 2024-09-12T23:37:49.393 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.393 INFO:tasks.workunit.client.0.smithi136.stdout: "state_kv_commiting_lat": { 2024-09-12T23:37:49.393 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 251, 2024-09-12T23:37:49.393 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 166472352 2024-09-12T23:37:49.393 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.394 INFO:tasks.workunit.client.0.smithi136.stdout: "state_kv_done_lat": { 2024-09-12T23:37:49.394 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 251, 2024-09-12T23:37:49.394 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 150692 2024-09-12T23:37:49.394 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.394 INFO:tasks.workunit.client.0.smithi136.stdout: "state_kv_queued_lat": { 2024-09-12T23:37:49.394 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 251, 2024-09-12T23:37:49.394 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 173530334 2024-09-12T23:37:49.394 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.394 INFO:tasks.workunit.client.0.smithi136.stdout: "state_prepare_lat": { 2024-09-12T23:37:49.394 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 251, 2024-09-12T23:37:49.394 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 18550755 2024-09-12T23:37:49.395 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.395 INFO:tasks.workunit.client.0.smithi136.stdout: "stored": { 2024-09-12T23:37:49.395 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 884213 2024-09-12T23:37:49.395 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.395 INFO:tasks.workunit.client.0.smithi136.stdout: "truncate_lat": { 2024-09-12T23:37:49.395 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 26, 2024-09-12T23:37:49.395 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 207664 2024-09-12T23:37:49.395 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.395 INFO:tasks.workunit.client.0.smithi136.stdout: "txc_commit_lat": { 2024-09-12T23:37:49.395 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 251, 2024-09-12T23:37:49.395 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 362730762 2024-09-12T23:37:49.396 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.396 INFO:tasks.workunit.client.0.smithi136.stdout: "txc_submit_lat": { 2024-09-12T23:37:49.396 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 251, 2024-09-12T23:37:49.396 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 22391426 2024-09-12T23:37:49.396 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.396 INFO:tasks.workunit.client.0.smithi136.stdout: "txc_throttle_lat": { 2024-09-12T23:37:49.396 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 251, 2024-09-12T23:37:49.396 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2834441 2024-09-12T23:37:49.396 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.396 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.396 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache": { 2024-09-12T23:37:49.397 INFO:tasks.workunit.client.0.smithi136.stdout: "cache_bytes": { 2024-09-12T23:37:49.397 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1900963507 2024-09-12T23:37:49.397 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.397 INFO:tasks.workunit.client.0.smithi136.stdout: "heap_bytes": { 2024-09-12T23:37:49.397 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 271187968 2024-09-12T23:37:49.397 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.397 INFO:tasks.workunit.client.0.smithi136.stdout: "mapped_bytes": { 2024-09-12T23:37:49.397 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 269926400 2024-09-12T23:37:49.397 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.397 INFO:tasks.workunit.client.0.smithi136.stdout: "target_bytes": { 2024-09-12T23:37:49.397 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 3183846912 2024-09-12T23:37:49.398 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.398 INFO:tasks.workunit.client.0.smithi136.stdout: "unmapped_bytes": { 2024-09-12T23:37:49.398 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1261568 2024-09-12T23:37:49.398 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.398 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.398 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:data": { 2024-09-12T23:37:49.398 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:49.398 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 159383552 2024-09-12T23:37:49.398 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.398 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:49.399 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.399 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.399 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:49.399 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.399 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.399 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:49.399 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 87868942 2024-09-12T23:37:49.399 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.399 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:49.399 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.399 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.400 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:49.400 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.400 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.400 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:49.400 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.400 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.400 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:49.400 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.400 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.400 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:49.401 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.401 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.401 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:49.401 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 16384 2024-09-12T23:37:49.401 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.401 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:49.401 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.401 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.401 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:49.401 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.401 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.402 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:49.402 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.402 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.402 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:49.402 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 71498226 2024-09-12T23:37:49.402 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.402 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.402 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:kv": { 2024-09-12T23:37:49.402 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:49.402 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 796917760 2024-09-12T23:37:49.402 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.402 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:49.403 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67108864 2024-09-12T23:37:49.403 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.403 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:49.403 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.403 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.403 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:49.403 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 659017068 2024-09-12T23:37:49.403 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.403 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:49.403 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.403 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.403 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:49.404 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.404 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.404 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:49.404 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.404 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.404 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:49.404 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.404 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.404 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:49.404 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.404 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.405 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:49.405 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2128 2024-09-12T23:37:49.405 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.405 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:49.405 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.405 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.405 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:49.405 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.406 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.406 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:49.406 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.406 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.406 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:49.406 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 70789700 2024-09-12T23:37:49.406 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.406 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.407 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:kv_onode": { 2024-09-12T23:37:49.407 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:49.407 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 192937984 2024-09-12T23:37:49.407 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.407 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:49.407 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67108864 2024-09-12T23:37:49.407 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.407 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:49.407 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 464 2024-09-12T23:37:49.408 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.408 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:49.408 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 58579294 2024-09-12T23:37:49.408 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.408 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:49.408 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.408 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.408 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:49.408 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.408 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.409 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:49.409 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.409 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.409 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:49.409 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.409 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.409 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:49.409 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.409 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.409 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:49.409 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.410 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.410 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:49.410 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.410 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.410 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:49.410 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.410 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.410 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:49.410 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.411 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.411 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:49.411 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67249362 2024-09-12T23:37:49.411 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.411 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.411 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:meta": { 2024-09-12T23:37:49.411 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:49.411 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 729808896 2024-09-12T23:37:49.411 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.411 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:49.411 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.412 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.412 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:49.412 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.412 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.412 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:49.412 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 659017068 2024-09-12T23:37:49.412 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.412 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:49.412 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.412 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.412 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:49.413 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.413 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.413 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:49.413 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.413 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.413 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:49.413 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.413 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.413 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:49.413 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.413 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.413 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:49.414 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 254540 2024-09-12T23:37:49.414 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.414 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:49.414 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.414 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.414 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:49.414 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.414 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.414 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:49.414 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.414 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.415 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:49.415 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 70537288 2024-09-12T23:37:49.415 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.415 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.415 INFO:tasks.workunit.client.0.smithi136.stdout: "objecter": { 2024-09-12T23:37:49.415 INFO:tasks.workunit.client.0.smithi136.stdout: "op_active": { 2024-09-12T23:37:49.415 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.415 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.415 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r": { 2024-09-12T23:37:49.415 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.415 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.415 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rmw": { 2024-09-12T23:37:49.416 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.416 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.416 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w": { 2024-09-12T23:37:49.416 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.416 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.416 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.416 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:49.416 INFO:tasks.workunit.client.0.smithi136.stdout: "numpg": { 2024-09-12T23:37:49.416 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 39 2024-09-12T23:37:49.416 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.416 INFO:tasks.workunit.client.0.smithi136.stdout: "numpg_removing": { 2024-09-12T23:37:49.416 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.417 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.417 INFO:tasks.workunit.client.0.smithi136.stdout: "op": { 2024-09-12T23:37:49.417 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 13 2024-09-12T23:37:49.417 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.417 INFO:tasks.workunit.client.0.smithi136.stdout: "op_delayed_degraded": { 2024-09-12T23:37:49.417 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.417 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.417 INFO:tasks.workunit.client.0.smithi136.stdout: "op_delayed_unreadable": { 2024-09-12T23:37:49.417 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.417 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.418 INFO:tasks.workunit.client.0.smithi136.stdout: "op_in_bytes": { 2024-09-12T23:37:49.418 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2804 2024-09-12T23:37:49.418 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.418 INFO:tasks.workunit.client.0.smithi136.stdout: "op_latency": { 2024-09-12T23:37:49.418 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 13, 2024-09-12T23:37:49.418 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 143192594 2024-09-12T23:37:49.418 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.418 INFO:tasks.workunit.client.0.smithi136.stdout: "op_out_bytes": { 2024-09-12T23:37:49.418 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.418 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.418 INFO:tasks.workunit.client.0.smithi136.stdout: "op_prepare_latency": { 2024-09-12T23:37:49.418 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 15, 2024-09-12T23:37:49.419 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 10460108 2024-09-12T23:37:49.419 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.419 INFO:tasks.workunit.client.0.smithi136.stdout: "op_process_latency": { 2024-09-12T23:37:49.419 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 13, 2024-09-12T23:37:49.419 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 126492015 2024-09-12T23:37:49.419 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.419 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r": { 2024-09-12T23:37:49.419 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.419 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.419 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_latency": { 2024-09-12T23:37:49.419 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.420 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.420 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.420 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-12T23:37:49.420 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.420 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.420 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_out_bytes": { 2024-09-12T23:37:49.420 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.420 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.420 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_prepare_latency": { 2024-09-12T23:37:49.420 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.420 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.421 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.421 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_process_latency": { 2024-09-12T23:37:49.421 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.421 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.421 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.421 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw": { 2024-09-12T23:37:49.421 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.421 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.421 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_in_bytes": { 2024-09-12T23:37:49.421 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.422 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.422 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_latency": { 2024-09-12T23:37:49.422 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.422 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.422 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.422 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-12T23:37:49.422 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.422 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.422 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-12T23:37:49.422 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.423 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.423 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_out_bytes": { 2024-09-12T23:37:49.423 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.423 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.423 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_prepare_latency": { 2024-09-12T23:37:49.423 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.423 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.423 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.423 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_process_latency": { 2024-09-12T23:37:49.423 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.423 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.424 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.424 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w": { 2024-09-12T23:37:49.424 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 13 2024-09-12T23:37:49.424 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.424 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_in_bytes": { 2024-09-12T23:37:49.424 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2804 2024-09-12T23:37:49.424 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.424 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_latency": { 2024-09-12T23:37:49.424 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 13, 2024-09-12T23:37:49.424 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 143192594 2024-09-12T23:37:49.424 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.425 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-12T23:37:49.425 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.425 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.425 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_prepare_latency": { 2024-09-12T23:37:49.425 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 15, 2024-09-12T23:37:49.425 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 10460108 2024-09-12T23:37:49.425 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.425 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_process_latency": { 2024-09-12T23:37:49.425 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 13, 2024-09-12T23:37:49.425 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 126492015 2024-09-12T23:37:49.425 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.426 INFO:tasks.workunit.client.0.smithi136.stdout: "op_wip": { 2024-09-12T23:37:49.426 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.426 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.426 INFO:tasks.workunit.client.0.smithi136.stdout: "recovery_bytes": { 2024-09-12T23:37:49.426 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 590368 2024-09-12T23:37:49.426 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.426 INFO:tasks.workunit.client.0.smithi136.stdout: "recovery_ops": { 2024-09-12T23:37:49.426 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2 2024-09-12T23:37:49.426 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.426 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_bytes": { 2024-09-12T23:37:49.427 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 95995035648 2024-09-12T23:37:49.427 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.427 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_bytes_used": { 2024-09-12T23:37:49.427 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 305741824 2024-09-12T23:37:49.427 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.427 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.427 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.427 INFO:tasks.workunit.client.0.smithi136.stdout: "osd.7": { 2024-09-12T23:37:49.427 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs": { 2024-09-12T23:37:49.427 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_slow_fallback": { 2024-09-12T23:37:49.427 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.428 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.428 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_slow_size_fallback": { 2024-09-12T23:37:49.428 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.428 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.428 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit_db": { 2024-09-12T23:37:49.428 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 65536 2024-09-12T23:37:49.428 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.428 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit_main": { 2024-09-12T23:37:49.428 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.428 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.428 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit_wal": { 2024-09-12T23:37:49.429 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.429 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.429 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_written_slow": { 2024-09-12T23:37:49.429 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.429 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.429 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_written_sst": { 2024-09-12T23:37:49.429 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 12288 2024-09-12T23:37:49.429 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.429 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_written_wal": { 2024-09-12T23:37:49.429 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1163264 2024-09-12T23:37:49.429 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.429 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_lat": { 2024-09-12T23:37:49.430 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.430 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.430 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.430 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_lock_lat": { 2024-09-12T23:37:49.430 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.430 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.430 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.430 INFO:tasks.workunit.client.0.smithi136.stdout: "db_total_bytes": { 2024-09-12T23:37:49.430 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 95995035648 2024-09-12T23:37:49.430 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.430 INFO:tasks.workunit.client.0.smithi136.stdout: "db_used_bytes": { 2024-09-12T23:37:49.431 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 304480256 2024-09-12T23:37:49.431 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.431 INFO:tasks.workunit.client.0.smithi136.stdout: "log_bytes": { 2024-09-12T23:37:49.431 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 925696 2024-09-12T23:37:49.431 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.431 INFO:tasks.workunit.client.0.smithi136.stdout: "logged_bytes": { 2024-09-12T23:37:49.431 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 786432 2024-09-12T23:37:49.431 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.431 INFO:tasks.workunit.client.0.smithi136.stdout: "max_bytes_db": { 2024-09-12T23:37:49.431 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 312868864 2024-09-12T23:37:49.431 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.431 INFO:tasks.workunit.client.0.smithi136.stdout: "max_bytes_slow": { 2024-09-12T23:37:49.432 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.432 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.432 INFO:tasks.workunit.client.0.smithi136.stdout: "max_bytes_wal": { 2024-09-12T23:37:49.432 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.432 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.432 INFO:tasks.workunit.client.0.smithi136.stdout: "num_files": { 2024-09-12T23:37:49.432 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 14 2024-09-12T23:37:49.432 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.433 INFO:tasks.workunit.client.0.smithi136.stdout: "read_bytes": { 2024-09-12T23:37:49.433 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 195697 2024-09-12T23:37:49.433 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.433 INFO:tasks.workunit.client.0.smithi136.stdout: "read_count": { 2024-09-12T23:37:49.433 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 23 2024-09-12T23:37:49.433 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.433 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes": { 2024-09-12T23:37:49.433 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1269760 2024-09-12T23:37:49.433 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.433 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes_db": { 2024-09-12T23:37:49.433 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1273856 2024-09-12T23:37:49.434 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.434 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes_slow": { 2024-09-12T23:37:49.434 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.434 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.434 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes_wal": { 2024-09-12T23:37:49.434 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.434 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.434 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_count": { 2024-09-12T23:37:49.434 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 15 2024-09-12T23:37:49.435 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.435 INFO:tasks.workunit.client.0.smithi136.stdout: "read_prefetch_bytes": { 2024-09-12T23:37:49.435 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4789 2024-09-12T23:37:49.435 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.435 INFO:tasks.workunit.client.0.smithi136.stdout: "read_prefetch_count": { 2024-09-12T23:37:49.435 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4 2024-09-12T23:37:49.435 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.435 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_buffer_bytes": { 2024-09-12T23:37:49.435 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5589 2024-09-12T23:37:49.435 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.435 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_buffer_count": { 2024-09-12T23:37:49.436 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 24 2024-09-12T23:37:49.436 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.436 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_bytes": { 2024-09-12T23:37:49.436 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5589 2024-09-12T23:37:49.436 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.436 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_count": { 2024-09-12T23:37:49.436 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 24 2024-09-12T23:37:49.436 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.437 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes": { 2024-09-12T23:37:49.437 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.437 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.437 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes_db": { 2024-09-12T23:37:49.437 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.437 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.438 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes_slow": { 2024-09-12T23:37:49.438 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.438 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.438 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes_wal": { 2024-09-12T23:37:49.438 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.438 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.439 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_count": { 2024-09-12T23:37:49.439 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.439 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.439 INFO:tasks.workunit.client.0.smithi136.stdout: "slow_total_bytes": { 2024-09-12T23:37:49.439 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.439 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.439 INFO:tasks.workunit.client.0.smithi136.stdout: "slow_used_bytes": { 2024-09-12T23:37:49.439 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.439 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.439 INFO:tasks.workunit.client.0.smithi136.stdout: "wal_total_bytes": { 2024-09-12T23:37:49.439 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.440 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.440 INFO:tasks.workunit.client.0.smithi136.stdout: "wal_used_bytes": { 2024-09-12T23:37:49.440 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.440 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.440 INFO:tasks.workunit.client.0.smithi136.stdout: "write_bytes": { 2024-09-12T23:37:49.441 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2023424 2024-09-12T23:37:49.441 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.441 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.441 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore": { 2024-09-12T23:37:49.441 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit": { 2024-09-12T23:37:49.441 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4096 2024-09-12T23:37:49.442 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.442 INFO:tasks.workunit.client.0.smithi136.stdout: "allocated": { 2024-09-12T23:37:49.442 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1253376 2024-09-12T23:37:49.442 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.442 INFO:tasks.workunit.client.0.smithi136.stdout: "clist_lat": { 2024-09-12T23:37:49.442 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 5, 2024-09-12T23:37:49.442 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 381524 2024-09-12T23:37:49.442 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.442 INFO:tasks.workunit.client.0.smithi136.stdout: "compress_lat": { 2024-09-12T23:37:49.442 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.443 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.443 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.443 INFO:tasks.workunit.client.0.smithi136.stdout: "compressed": { 2024-09-12T23:37:49.443 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.443 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.444 INFO:tasks.workunit.client.0.smithi136.stdout: "compressed_allocated": { 2024-09-12T23:37:49.444 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.444 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.444 INFO:tasks.workunit.client.0.smithi136.stdout: "compressed_original": { 2024-09-12T23:37:49.444 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.444 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.444 INFO:tasks.workunit.client.0.smithi136.stdout: "csum_lat": { 2024-09-12T23:37:49.444 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 7, 2024-09-12T23:37:49.444 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 30960 2024-09-12T23:37:49.444 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.445 INFO:tasks.workunit.client.0.smithi136.stdout: "decompress_lat": { 2024-09-12T23:37:49.445 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.445 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.445 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.445 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_commit_lat": { 2024-09-12T23:37:49.445 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 175, 2024-09-12T23:37:49.445 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 96588735 2024-09-12T23:37:49.445 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.445 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_final_lat": { 2024-09-12T23:37:49.445 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 160, 2024-09-12T23:37:49.445 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 16173359 2024-09-12T23:37:49.446 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.446 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_flush_lat": { 2024-09-12T23:37:49.446 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 175, 2024-09-12T23:37:49.446 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 239595 2024-09-12T23:37:49.446 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.446 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_sync_lat": { 2024-09-12T23:37:49.446 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 175, 2024-09-12T23:37:49.446 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 96828330 2024-09-12T23:37:49.446 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.446 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_get_keys_lat": { 2024-09-12T23:37:49.447 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.447 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.447 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.447 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_get_values_lat": { 2024-09-12T23:37:49.447 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 2, 2024-09-12T23:37:49.447 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 52591 2024-09-12T23:37:49.447 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.447 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_lower_bound_lat": { 2024-09-12T23:37:49.447 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 3, 2024-09-12T23:37:49.448 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4340 2024-09-12T23:37:49.448 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.448 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_next_lat": { 2024-09-12T23:37:49.448 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.448 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.448 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.448 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_seek_to_first_lat": { 2024-09-12T23:37:49.448 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:49.448 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.448 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.448 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_upper_bound_lat": { 2024-09-12T23:37:49.449 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 2, 2024-09-12T23:37:49.449 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 8636 2024-09-12T23:37:49.449 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.449 INFO:tasks.workunit.client.0.smithi136.stdout: "onode_hits": { 2024-09-12T23:37:49.449 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 367 2024-09-12T23:37:49.449 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.449 INFO:tasks.workunit.client.0.smithi136.stdout: "onode_misses": { 2024-09-12T23:37:49.449 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 251 2024-09-12T23:37:49.449 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.449 INFO:tasks.workunit.client.0.smithi136.stdout: "read_lat": { 2024-09-12T23:37:49.450 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 84, 2024-09-12T23:37:49.450 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2431894 2024-09-12T23:37:49.450 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.450 INFO:tasks.workunit.client.0.smithi136.stdout: "read_onode_meta_lat": { 2024-09-12T23:37:49.450 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 121, 2024-09-12T23:37:49.450 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 442619 2024-09-12T23:37:49.450 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.450 INFO:tasks.workunit.client.0.smithi136.stdout: "read_wait_aio_lat": { 2024-09-12T23:37:49.450 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 37, 2024-09-12T23:37:49.450 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1549508 2024-09-12T23:37:49.451 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.451 INFO:tasks.workunit.client.0.smithi136.stdout: "reads_with_retries": { 2024-09-12T23:37:49.451 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.451 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.451 INFO:tasks.workunit.client.0.smithi136.stdout: "remove_lat": { 2024-09-12T23:37:49.451 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 2, 2024-09-12T23:37:49.451 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 85362 2024-09-12T23:37:49.451 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.451 INFO:tasks.workunit.client.0.smithi136.stdout: "state_aio_wait_lat": { 2024-09-12T23:37:49.451 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 287, 2024-09-12T23:37:49.451 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 3289529 2024-09-12T23:37:49.451 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.452 INFO:tasks.workunit.client.0.smithi136.stdout: "state_deferred_aio_wait_lat": { 2024-09-12T23:37:49.452 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 21, 2024-09-12T23:37:49.452 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2172027 2024-09-12T23:37:49.452 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.452 INFO:tasks.workunit.client.0.smithi136.stdout: "state_deferred_cleanup_lat": { 2024-09-12T23:37:49.452 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 20, 2024-09-12T23:37:49.452 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 192491977071 2024-09-12T23:37:49.452 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.452 INFO:tasks.workunit.client.0.smithi136.stdout: "state_deferred_queued_lat": { 2024-09-12T23:37:49.452 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 21, 2024-09-12T23:37:49.452 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 33206269672 2024-09-12T23:37:49.453 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.453 INFO:tasks.workunit.client.0.smithi136.stdout: "state_done_lat": { 2024-09-12T23:37:49.453 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 286, 2024-09-12T23:37:49.453 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1041937 2024-09-12T23:37:49.453 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.453 INFO:tasks.workunit.client.0.smithi136.stdout: "state_finishing_lat": { 2024-09-12T23:37:49.453 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 286, 2024-09-12T23:37:49.453 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 54002 2024-09-12T23:37:49.453 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.453 INFO:tasks.workunit.client.0.smithi136.stdout: "state_io_done_lat": { 2024-09-12T23:37:49.453 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 287, 2024-09-12T23:37:49.453 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 331018 2024-09-12T23:37:49.454 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.454 INFO:tasks.workunit.client.0.smithi136.stdout: "state_kv_commiting_lat": { 2024-09-12T23:37:49.454 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 287, 2024-09-12T23:37:49.454 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 337162547 2024-09-12T23:37:49.454 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.454 INFO:tasks.workunit.client.0.smithi136.stdout: "state_kv_done_lat": { 2024-09-12T23:37:49.454 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 287, 2024-09-12T23:37:49.454 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 145007 2024-09-12T23:37:49.454 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.454 INFO:tasks.workunit.client.0.smithi136.stdout: "state_kv_queued_lat": { 2024-09-12T23:37:49.454 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 287, 2024-09-12T23:37:49.455 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 157497798 2024-09-12T23:37:49.455 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.455 INFO:tasks.workunit.client.0.smithi136.stdout: "state_prepare_lat": { 2024-09-12T23:37:49.455 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 287, 2024-09-12T23:37:49.455 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 25402876 2024-09-12T23:37:49.455 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.455 INFO:tasks.workunit.client.0.smithi136.stdout: "stored": { 2024-09-12T23:37:49.455 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 882496 2024-09-12T23:37:49.455 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.455 INFO:tasks.workunit.client.0.smithi136.stdout: "truncate_lat": { 2024-09-12T23:37:49.455 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 20, 2024-09-12T23:37:49.455 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 146134 2024-09-12T23:37:49.456 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.456 INFO:tasks.workunit.client.0.smithi136.stdout: "txc_commit_lat": { 2024-09-12T23:37:49.456 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 287, 2024-09-12T23:37:49.456 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 523748960 2024-09-12T23:37:49.456 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.456 INFO:tasks.workunit.client.0.smithi136.stdout: "txc_submit_lat": { 2024-09-12T23:37:49.456 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 287, 2024-09-12T23:37:49.456 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 30886587 2024-09-12T23:37:49.456 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.456 INFO:tasks.workunit.client.0.smithi136.stdout: "txc_throttle_lat": { 2024-09-12T23:37:49.456 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 287, 2024-09-12T23:37:49.456 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 6887245 2024-09-12T23:37:49.457 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.457 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.457 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache": { 2024-09-12T23:37:49.457 INFO:tasks.workunit.client.0.smithi136.stdout: "cache_bytes": { 2024-09-12T23:37:49.457 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1900963507 2024-09-12T23:37:49.457 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.457 INFO:tasks.workunit.client.0.smithi136.stdout: "heap_bytes": { 2024-09-12T23:37:49.457 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 271122432 2024-09-12T23:37:49.457 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.458 INFO:tasks.workunit.client.0.smithi136.stdout: "mapped_bytes": { 2024-09-12T23:37:49.458 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 270557184 2024-09-12T23:37:49.458 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.458 INFO:tasks.workunit.client.0.smithi136.stdout: "target_bytes": { 2024-09-12T23:37:49.458 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 3183846912 2024-09-12T23:37:49.458 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.458 INFO:tasks.workunit.client.0.smithi136.stdout: "unmapped_bytes": { 2024-09-12T23:37:49.458 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 565248 2024-09-12T23:37:49.458 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.458 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.458 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:data": { 2024-09-12T23:37:49.459 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:49.459 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 159383552 2024-09-12T23:37:49.459 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.459 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:49.459 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.459 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.459 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:49.459 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.459 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.459 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:49.459 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 87860544 2024-09-12T23:37:49.459 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.460 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:49.460 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.460 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.460 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:49.460 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.460 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.460 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:49.460 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.460 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.461 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:49.461 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.461 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.461 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:49.461 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.461 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.462 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:49.462 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 143360 2024-09-12T23:37:49.462 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.462 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:49.462 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.462 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.462 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:49.462 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.462 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.462 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:49.463 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.463 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.463 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:49.463 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 71379648 2024-09-12T23:37:49.463 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.463 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.463 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:kv": { 2024-09-12T23:37:49.463 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:49.463 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 796917760 2024-09-12T23:37:49.464 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.464 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:49.464 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67108864 2024-09-12T23:37:49.464 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.464 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:49.464 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.464 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.465 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:49.465 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 658954084 2024-09-12T23:37:49.465 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.465 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:49.465 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.465 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.465 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:49.466 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.466 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.466 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:49.466 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.466 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.466 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:49.467 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.467 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.467 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:49.467 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.467 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.467 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:49.467 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2128 2024-09-12T23:37:49.468 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.468 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:49.468 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.468 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.468 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:49.468 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.468 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.469 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:49.469 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.469 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.469 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:49.469 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 70852684 2024-09-12T23:37:49.469 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.469 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.470 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:kv_onode": { 2024-09-12T23:37:49.470 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:49.470 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 192937984 2024-09-12T23:37:49.470 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.470 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:49.470 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67108864 2024-09-12T23:37:49.470 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.471 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:49.471 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 464 2024-09-12T23:37:49.471 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.471 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:49.471 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 58573696 2024-09-12T23:37:49.471 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.471 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:49.472 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.472 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.472 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:49.472 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.472 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.472 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:49.473 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.473 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.473 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:49.473 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.473 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.473 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:49.473 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.474 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.474 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:49.474 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.474 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.474 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:49.474 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.475 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.475 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:49.475 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.475 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.475 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:49.475 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.475 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.475 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:49.476 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67254960 2024-09-12T23:37:49.476 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.476 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.476 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:meta": { 2024-09-12T23:37:49.476 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:49.476 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 729808896 2024-09-12T23:37:49.477 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.477 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:49.477 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.477 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.477 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:49.477 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.477 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.477 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:49.478 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 658954084 2024-09-12T23:37:49.478 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.478 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:49.478 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.478 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.478 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:49.478 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.478 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.478 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:49.478 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.478 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.479 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:49.479 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.479 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.479 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:49.479 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.479 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.479 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:49.479 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 264681 2024-09-12T23:37:49.479 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.479 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:49.479 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.480 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.480 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:49.480 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.480 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.480 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:49.480 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.480 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.480 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:49.480 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 70590131 2024-09-12T23:37:49.480 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.480 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.481 INFO:tasks.workunit.client.0.smithi136.stdout: "objecter": { 2024-09-12T23:37:49.481 INFO:tasks.workunit.client.0.smithi136.stdout: "op_active": { 2024-09-12T23:37:49.481 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.481 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.481 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r": { 2024-09-12T23:37:49.481 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.481 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.481 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rmw": { 2024-09-12T23:37:49.481 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.482 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.482 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w": { 2024-09-12T23:37:49.482 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.482 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.482 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.482 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:49.482 INFO:tasks.workunit.client.0.smithi136.stdout: "numpg": { 2024-09-12T23:37:49.482 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 47 2024-09-12T23:37:49.482 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.482 INFO:tasks.workunit.client.0.smithi136.stdout: "numpg_removing": { 2024-09-12T23:37:49.482 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.483 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.483 INFO:tasks.workunit.client.0.smithi136.stdout: "op": { 2024-09-12T23:37:49.483 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 29 2024-09-12T23:37:49.483 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.483 INFO:tasks.workunit.client.0.smithi136.stdout: "op_delayed_degraded": { 2024-09-12T23:37:49.483 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.483 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.483 INFO:tasks.workunit.client.0.smithi136.stdout: "op_delayed_unreadable": { 2024-09-12T23:37:49.483 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.483 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.484 INFO:tasks.workunit.client.0.smithi136.stdout: "op_in_bytes": { 2024-09-12T23:37:49.484 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.484 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.484 INFO:tasks.workunit.client.0.smithi136.stdout: "op_latency": { 2024-09-12T23:37:49.484 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 29, 2024-09-12T23:37:49.484 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 113592836 2024-09-12T23:37:49.484 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.484 INFO:tasks.workunit.client.0.smithi136.stdout: "op_out_bytes": { 2024-09-12T23:37:49.484 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 131765 2024-09-12T23:37:49.484 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.485 INFO:tasks.workunit.client.0.smithi136.stdout: "op_prepare_latency": { 2024-09-12T23:37:49.485 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 31, 2024-09-12T23:37:49.485 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 16667817 2024-09-12T23:37:49.485 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.485 INFO:tasks.workunit.client.0.smithi136.stdout: "op_process_latency": { 2024-09-12T23:37:49.485 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 29, 2024-09-12T23:37:49.485 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 102850941 2024-09-12T23:37:49.485 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.485 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r": { 2024-09-12T23:37:49.485 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 18 2024-09-12T23:37:49.486 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.486 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_latency": { 2024-09-12T23:37:49.486 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 18, 2024-09-12T23:37:49.486 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 6949405 2024-09-12T23:37:49.486 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.486 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-12T23:37:49.486 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.486 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.486 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_out_bytes": { 2024-09-12T23:37:49.486 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 131765 2024-09-12T23:37:49.487 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.487 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_prepare_latency": { 2024-09-12T23:37:49.487 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 18, 2024-09-12T23:37:49.487 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 6070712 2024-09-12T23:37:49.487 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.487 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_process_latency": { 2024-09-12T23:37:49.487 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 18, 2024-09-12T23:37:49.487 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5060579 2024-09-12T23:37:49.487 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.487 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw": { 2024-09-12T23:37:49.487 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 6 2024-09-12T23:37:49.487 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.488 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_in_bytes": { 2024-09-12T23:37:49.488 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.488 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.488 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_latency": { 2024-09-12T23:37:49.488 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 6, 2024-09-12T23:37:49.488 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 11721532 2024-09-12T23:37:49.488 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.488 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-12T23:37:49.488 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.488 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.488 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-12T23:37:49.489 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.489 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.489 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_out_bytes": { 2024-09-12T23:37:49.489 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.489 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.489 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_prepare_latency": { 2024-09-12T23:37:49.489 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 6, 2024-09-12T23:37:49.489 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4767544 2024-09-12T23:37:49.489 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.489 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_process_latency": { 2024-09-12T23:37:49.489 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 6, 2024-09-12T23:37:49.489 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 10989424 2024-09-12T23:37:49.490 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.490 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w": { 2024-09-12T23:37:49.490 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5 2024-09-12T23:37:49.490 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.490 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_in_bytes": { 2024-09-12T23:37:49.490 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.490 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.490 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_latency": { 2024-09-12T23:37:49.490 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 5, 2024-09-12T23:37:49.490 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 94921899 2024-09-12T23:37:49.490 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.491 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-12T23:37:49.491 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.491 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.491 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_prepare_latency": { 2024-09-12T23:37:49.491 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 7, 2024-09-12T23:37:49.491 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5829561 2024-09-12T23:37:49.491 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.491 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_process_latency": { 2024-09-12T23:37:49.491 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 5, 2024-09-12T23:37:49.491 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 86800938 2024-09-12T23:37:49.491 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.492 INFO:tasks.workunit.client.0.smithi136.stdout: "op_wip": { 2024-09-12T23:37:49.492 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:49.492 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.492 INFO:tasks.workunit.client.0.smithi136.stdout: "recovery_bytes": { 2024-09-12T23:37:49.492 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 590368 2024-09-12T23:37:49.492 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.492 INFO:tasks.workunit.client.0.smithi136.stdout: "recovery_ops": { 2024-09-12T23:37:49.492 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2 2024-09-12T23:37:49.492 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.493 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_bytes": { 2024-09-12T23:37:49.493 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 95995035648 2024-09-12T23:37:49.493 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.493 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_bytes_used": { 2024-09-12T23:37:49.493 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 305745920 2024-09-12T23:37:49.493 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.493 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.493 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.493 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.493 INFO:tasks.workunit.client.0.smithi136.stdout: "pools": [ 2024-09-12T23:37:49.493 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.494 INFO:tasks.workunit.client.0.smithi136.stdout: "application": [ 2024-09-12T23:37:49.494 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr" 2024-09-12T23:37:49.494 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.494 INFO:tasks.workunit.client.0.smithi136.stdout: "cache_mode": "none", 2024-09-12T23:37:49.494 INFO:tasks.workunit.client.0.smithi136.stdout: "erasure_code_profile": {}, 2024-09-12T23:37:49.494 INFO:tasks.workunit.client.0.smithi136.stdout: "min_size": 2, 2024-09-12T23:37:49.494 INFO:tasks.workunit.client.0.smithi136.stdout: "options": {}, 2024-09-12T23:37:49.494 INFO:tasks.workunit.client.0.smithi136.stdout: "pg_autoscale_mode": "off", 2024-09-12T23:37:49.494 INFO:tasks.workunit.client.0.smithi136.stdout: "pg_num": 1, 2024-09-12T23:37:49.494 INFO:tasks.workunit.client.0.smithi136.stdout: "pgp_num": 1, 2024-09-12T23:37:49.494 INFO:tasks.workunit.client.0.smithi136.stdout: "pool": 1, 2024-09-12T23:37:49.494 INFO:tasks.workunit.client.0.smithi136.stdout: "size": 3, 2024-09-12T23:37:49.495 INFO:tasks.workunit.client.0.smithi136.stdout: "stats": { 2024-09-12T23:37:49.495 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_raw": 727116228874, 2024-09-12T23:37:49.495 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_used": 1781760, 2024-09-12T23:37:49.495 INFO:tasks.workunit.client.0.smithi136.stdout: "compress_bytes_used": 0, 2024-09-12T23:37:49.495 INFO:tasks.workunit.client.0.smithi136.stdout: "compress_under_bytes": 0, 2024-09-12T23:37:49.495 INFO:tasks.workunit.client.0.smithi136.stdout: "data_bytes_used": 1781760, 2024-09-12T23:37:49.495 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty": 0, 2024-09-12T23:37:49.495 INFO:tasks.workunit.client.0.smithi136.stdout: "max_avail": 242372067328, 2024-09-12T23:37:49.495 INFO:tasks.workunit.client.0.smithi136.stdout: "objects": 2, 2024-09-12T23:37:49.495 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_bytes_used": 0, 2024-09-12T23:37:49.495 INFO:tasks.workunit.client.0.smithi136.stdout: "percent_used": 2.4504413431714056e-06, 2024-09-12T23:37:49.496 INFO:tasks.workunit.client.0.smithi136.stdout: "quota_bytes": 0, 2024-09-12T23:37:49.496 INFO:tasks.workunit.client.0.smithi136.stdout: "quota_objects": 0, 2024-09-12T23:37:49.496 INFO:tasks.workunit.client.0.smithi136.stdout: "rd": 126, 2024-09-12T23:37:49.496 INFO:tasks.workunit.client.0.smithi136.stdout: "rd_bytes": 236544, 2024-09-12T23:37:49.496 INFO:tasks.workunit.client.0.smithi136.stdout: "stored": 590368, 2024-09-12T23:37:49.496 INFO:tasks.workunit.client.0.smithi136.stdout: "stored_data": 590368, 2024-09-12T23:37:49.496 INFO:tasks.workunit.client.0.smithi136.stdout: "stored_omap": 0, 2024-09-12T23:37:49.496 INFO:tasks.workunit.client.0.smithi136.stdout: "stored_raw": 1771104, 2024-09-12T23:37:49.496 INFO:tasks.workunit.client.0.smithi136.stdout: "wr": 113, 2024-09-12T23:37:49.496 INFO:tasks.workunit.client.0.smithi136.stdout: "wr_bytes": 1667072 2024-09-12T23:37:49.496 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.497 INFO:tasks.workunit.client.0.smithi136.stdout: "target_max_bytes": 0, 2024-09-12T23:37:49.497 INFO:tasks.workunit.client.0.smithi136.stdout: "target_max_objects": 0, 2024-09-12T23:37:49.497 INFO:tasks.workunit.client.0.smithi136.stdout: "type": "replicated" 2024-09-12T23:37:49.497 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.497 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.497 INFO:tasks.workunit.client.0.smithi136.stdout: "application": [ 2024-09-12T23:37:49.497 INFO:tasks.workunit.client.0.smithi136.stdout: "cephfs" 2024-09-12T23:37:49.497 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.497 INFO:tasks.workunit.client.0.smithi136.stdout: "cache_mode": "none", 2024-09-12T23:37:49.497 INFO:tasks.workunit.client.0.smithi136.stdout: "erasure_code_profile": {}, 2024-09-12T23:37:49.497 INFO:tasks.workunit.client.0.smithi136.stdout: "min_size": 2, 2024-09-12T23:37:49.498 INFO:tasks.workunit.client.0.smithi136.stdout: "options": {}, 2024-09-12T23:37:49.498 INFO:tasks.workunit.client.0.smithi136.stdout: "pg_autoscale_mode": "off", 2024-09-12T23:37:49.498 INFO:tasks.workunit.client.0.smithi136.stdout: "pg_num": 32, 2024-09-12T23:37:49.498 INFO:tasks.workunit.client.0.smithi136.stdout: "pgp_num": 32, 2024-09-12T23:37:49.498 INFO:tasks.workunit.client.0.smithi136.stdout: "pool": 2, 2024-09-12T23:37:49.498 INFO:tasks.workunit.client.0.smithi136.stdout: "size": 3, 2024-09-12T23:37:49.498 INFO:tasks.workunit.client.0.smithi136.stdout: "stats": { 2024-09-12T23:37:49.498 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_raw": 727116228874, 2024-09-12T23:37:49.498 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_used": 98304, 2024-09-12T23:37:49.498 INFO:tasks.workunit.client.0.smithi136.stdout: "compress_bytes_used": 0, 2024-09-12T23:37:49.498 INFO:tasks.workunit.client.0.smithi136.stdout: "compress_under_bytes": 0, 2024-09-12T23:37:49.498 INFO:tasks.workunit.client.0.smithi136.stdout: "data_bytes_used": 98304, 2024-09-12T23:37:49.499 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty": 0, 2024-09-12T23:37:49.499 INFO:tasks.workunit.client.0.smithi136.stdout: "max_avail": 242372067328, 2024-09-12T23:37:49.499 INFO:tasks.workunit.client.0.smithi136.stdout: "objects": 22, 2024-09-12T23:37:49.499 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_bytes_used": 0, 2024-09-12T23:37:49.499 INFO:tasks.workunit.client.0.smithi136.stdout: "percent_used": 1.3519708375042683e-07, 2024-09-12T23:37:49.499 INFO:tasks.workunit.client.0.smithi136.stdout: "quota_bytes": 0, 2024-09-12T23:37:49.499 INFO:tasks.workunit.client.0.smithi136.stdout: "quota_objects": 0, 2024-09-12T23:37:49.499 INFO:tasks.workunit.client.0.smithi136.stdout: "rd": 0, 2024-09-12T23:37:49.499 INFO:tasks.workunit.client.0.smithi136.stdout: "rd_bytes": 0, 2024-09-12T23:37:49.499 INFO:tasks.workunit.client.0.smithi136.stdout: "stored": 3410, 2024-09-12T23:37:49.499 INFO:tasks.workunit.client.0.smithi136.stdout: "stored_data": 3410, 2024-09-12T23:37:49.500 INFO:tasks.workunit.client.0.smithi136.stdout: "stored_omap": 0, 2024-09-12T23:37:49.500 INFO:tasks.workunit.client.0.smithi136.stdout: "stored_raw": 10230, 2024-09-12T23:37:49.500 INFO:tasks.workunit.client.0.smithi136.stdout: "wr": 51, 2024-09-12T23:37:49.500 INFO:tasks.workunit.client.0.smithi136.stdout: "wr_bytes": 19456 2024-09-12T23:37:49.500 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.500 INFO:tasks.workunit.client.0.smithi136.stdout: "target_max_bytes": 0, 2024-09-12T23:37:49.500 INFO:tasks.workunit.client.0.smithi136.stdout: "target_max_objects": 0, 2024-09-12T23:37:49.500 INFO:tasks.workunit.client.0.smithi136.stdout: "type": "replicated" 2024-09-12T23:37:49.500 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.500 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.500 INFO:tasks.workunit.client.0.smithi136.stdout: "application": [ 2024-09-12T23:37:49.500 INFO:tasks.workunit.client.0.smithi136.stdout: "cephfs" 2024-09-12T23:37:49.501 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.501 INFO:tasks.workunit.client.0.smithi136.stdout: "cache_mode": "none", 2024-09-12T23:37:49.501 INFO:tasks.workunit.client.0.smithi136.stdout: "erasure_code_profile": {}, 2024-09-12T23:37:49.501 INFO:tasks.workunit.client.0.smithi136.stdout: "min_size": 2, 2024-09-12T23:37:49.501 INFO:tasks.workunit.client.0.smithi136.stdout: "options": {}, 2024-09-12T23:37:49.501 INFO:tasks.workunit.client.0.smithi136.stdout: "pg_autoscale_mode": "off", 2024-09-12T23:37:49.501 INFO:tasks.workunit.client.0.smithi136.stdout: "pg_num": 64, 2024-09-12T23:37:49.501 INFO:tasks.workunit.client.0.smithi136.stdout: "pgp_num": 64, 2024-09-12T23:37:49.501 INFO:tasks.workunit.client.0.smithi136.stdout: "pool": 3, 2024-09-12T23:37:49.501 INFO:tasks.workunit.client.0.smithi136.stdout: "size": 3, 2024-09-12T23:37:49.502 INFO:tasks.workunit.client.0.smithi136.stdout: "stats": { 2024-09-12T23:37:49.502 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_raw": 727116228874, 2024-09-12T23:37:49.502 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_used": 0, 2024-09-12T23:37:49.502 INFO:tasks.workunit.client.0.smithi136.stdout: "compress_bytes_used": 0, 2024-09-12T23:37:49.502 INFO:tasks.workunit.client.0.smithi136.stdout: "compress_under_bytes": 0, 2024-09-12T23:37:49.502 INFO:tasks.workunit.client.0.smithi136.stdout: "data_bytes_used": 0, 2024-09-12T23:37:49.502 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty": 0, 2024-09-12T23:37:49.502 INFO:tasks.workunit.client.0.smithi136.stdout: "max_avail": 242372067328, 2024-09-12T23:37:49.502 INFO:tasks.workunit.client.0.smithi136.stdout: "objects": 0, 2024-09-12T23:37:49.502 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_bytes_used": 0, 2024-09-12T23:37:49.502 INFO:tasks.workunit.client.0.smithi136.stdout: "percent_used": 0.0, 2024-09-12T23:37:49.503 INFO:tasks.workunit.client.0.smithi136.stdout: "quota_bytes": 0, 2024-09-12T23:37:49.503 INFO:tasks.workunit.client.0.smithi136.stdout: "quota_objects": 0, 2024-09-12T23:37:49.503 INFO:tasks.workunit.client.0.smithi136.stdout: "rd": 0, 2024-09-12T23:37:49.503 INFO:tasks.workunit.client.0.smithi136.stdout: "rd_bytes": 0, 2024-09-12T23:37:49.503 INFO:tasks.workunit.client.0.smithi136.stdout: "stored": 0, 2024-09-12T23:37:49.503 INFO:tasks.workunit.client.0.smithi136.stdout: "stored_data": 0, 2024-09-12T23:37:49.503 INFO:tasks.workunit.client.0.smithi136.stdout: "stored_omap": 0, 2024-09-12T23:37:49.503 INFO:tasks.workunit.client.0.smithi136.stdout: "stored_raw": 0, 2024-09-12T23:37:49.503 INFO:tasks.workunit.client.0.smithi136.stdout: "wr": 0, 2024-09-12T23:37:49.503 INFO:tasks.workunit.client.0.smithi136.stdout: "wr_bytes": 0 2024-09-12T23:37:49.503 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.504 INFO:tasks.workunit.client.0.smithi136.stdout: "target_max_bytes": 0, 2024-09-12T23:37:49.504 INFO:tasks.workunit.client.0.smithi136.stdout: "target_max_objects": 0, 2024-09-12T23:37:49.504 INFO:tasks.workunit.client.0.smithi136.stdout: "type": "replicated" 2024-09-12T23:37:49.504 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.504 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.504 INFO:tasks.workunit.client.0.smithi136.stdout: "application": [ 2024-09-12T23:37:49.504 INFO:tasks.workunit.client.0.smithi136.stdout: "rbd" 2024-09-12T23:37:49.504 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.504 INFO:tasks.workunit.client.0.smithi136.stdout: "cache_mode": "none", 2024-09-12T23:37:49.504 INFO:tasks.workunit.client.0.smithi136.stdout: "erasure_code_profile": {}, 2024-09-12T23:37:49.504 INFO:tasks.workunit.client.0.smithi136.stdout: "min_size": 2, 2024-09-12T23:37:49.505 INFO:tasks.workunit.client.0.smithi136.stdout: "options": {}, 2024-09-12T23:37:49.505 INFO:tasks.workunit.client.0.smithi136.stdout: "pg_autoscale_mode": "off", 2024-09-12T23:37:49.505 INFO:tasks.workunit.client.0.smithi136.stdout: "pg_num": 8, 2024-09-12T23:37:49.505 INFO:tasks.workunit.client.0.smithi136.stdout: "pgp_num": 8, 2024-09-12T23:37:49.505 INFO:tasks.workunit.client.0.smithi136.stdout: "pool": 4, 2024-09-12T23:37:49.505 INFO:tasks.workunit.client.0.smithi136.stdout: "size": 3, 2024-09-12T23:37:49.505 INFO:tasks.workunit.client.0.smithi136.stdout: "stats": { 2024-09-12T23:37:49.506 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_raw": 727116228874, 2024-09-12T23:37:49.506 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_used": 12288, 2024-09-12T23:37:49.506 INFO:tasks.workunit.client.0.smithi136.stdout: "compress_bytes_used": 0, 2024-09-12T23:37:49.506 INFO:tasks.workunit.client.0.smithi136.stdout: "compress_under_bytes": 0, 2024-09-12T23:37:49.506 INFO:tasks.workunit.client.0.smithi136.stdout: "data_bytes_used": 12288, 2024-09-12T23:37:49.506 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty": 0, 2024-09-12T23:37:49.506 INFO:tasks.workunit.client.0.smithi136.stdout: "max_avail": 242372067328, 2024-09-12T23:37:49.506 INFO:tasks.workunit.client.0.smithi136.stdout: "objects": 2, 2024-09-12T23:37:49.507 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_bytes_used": 0, 2024-09-12T23:37:49.508 INFO:tasks.workunit.client.0.smithi136.stdout: "percent_used": 1.6899637245160193e-08, 2024-09-12T23:37:49.509 INFO:tasks.workunit.client.0.smithi136.stdout: "quota_bytes": 0, 2024-09-12T23:37:49.509 INFO:tasks.workunit.client.0.smithi136.stdout: "quota_objects": 0, 2024-09-12T23:37:49.509 INFO:tasks.workunit.client.0.smithi136.stdout: "rd": 0, 2024-09-12T23:37:49.509 INFO:tasks.workunit.client.0.smithi136.stdout: "rd_bytes": 0, 2024-09-12T23:37:49.509 INFO:tasks.workunit.client.0.smithi136.stdout: "stored": 19, 2024-09-12T23:37:49.509 INFO:tasks.workunit.client.0.smithi136.stdout: "stored_data": 19, 2024-09-12T23:37:49.509 INFO:tasks.workunit.client.0.smithi136.stdout: "stored_omap": 0, 2024-09-12T23:37:49.509 INFO:tasks.workunit.client.0.smithi136.stdout: "stored_raw": 57, 2024-09-12T23:37:49.509 INFO:tasks.workunit.client.0.smithi136.stdout: "wr": 2, 2024-09-12T23:37:49.509 INFO:tasks.workunit.client.0.smithi136.stdout: "wr_bytes": 2048 2024-09-12T23:37:49.509 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.510 INFO:tasks.workunit.client.0.smithi136.stdout: "target_max_bytes": 0, 2024-09-12T23:37:49.510 INFO:tasks.workunit.client.0.smithi136.stdout: "target_max_objects": 0, 2024-09-12T23:37:49.510 INFO:tasks.workunit.client.0.smithi136.stdout: "type": "replicated" 2024-09-12T23:37:49.510 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.510 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.510 INFO:tasks.workunit.client.0.smithi136.stdout: "rbd": { 2024-09-12T23:37:49.510 INFO:tasks.workunit.client.0.smithi136.stdout: "mirroring_by_pool": [ 2024-09-12T23:37:49.510 INFO:tasks.workunit.client.0.smithi136.stdout: false 2024-09-12T23:37:49.510 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.510 INFO:tasks.workunit.client.0.smithi136.stdout: "num_images_by_pool": [ 2024-09-12T23:37:49.510 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:49.511 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.511 INFO:tasks.workunit.client.0.smithi136.stdout: "num_pools": 1 2024-09-12T23:37:49.511 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.511 INFO:tasks.workunit.client.0.smithi136.stdout: "report_id": "679f567b-70cd-4d20-9c08-b6621403ecae", 2024-09-12T23:37:49.511 INFO:tasks.workunit.client.0.smithi136.stdout: "report_timestamp": "2024-09-12T23:37:48.299222", 2024-09-12T23:37:49.511 INFO:tasks.workunit.client.0.smithi136.stdout: "report_version": 1, 2024-09-12T23:37:49.511 INFO:tasks.workunit.client.0.smithi136.stdout: "rocksdb_stats": { 2024-09-12T23:37:49.511 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "6.15.5" 2024-09-12T23:37:49.511 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.511 INFO:tasks.workunit.client.0.smithi136.stdout: "rook": { 2024-09-12T23:37:49.512 INFO:tasks.workunit.client.0.smithi136.stdout: "cluster": { 2024-09-12T23:37:49.512 INFO:tasks.workunit.client.0.smithi136.stdout: "external-mode": null, 2024-09-12T23:37:49.512 INFO:tasks.workunit.client.0.smithi136.stdout: "mon": { 2024-09-12T23:37:49.512 INFO:tasks.workunit.client.0.smithi136.stdout: "allow-multiple-per-node": null, 2024-09-12T23:37:49.512 INFO:tasks.workunit.client.0.smithi136.stdout: "count": null, 2024-09-12T23:37:49.512 INFO:tasks.workunit.client.0.smithi136.stdout: "max-id": null, 2024-09-12T23:37:49.512 INFO:tasks.workunit.client.0.smithi136.stdout: "pvc": { 2024-09-12T23:37:49.512 INFO:tasks.workunit.client.0.smithi136.stdout: "enabled": null 2024-09-12T23:37:49.512 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.512 INFO:tasks.workunit.client.0.smithi136.stdout: "stretch": { 2024-09-12T23:37:49.512 INFO:tasks.workunit.client.0.smithi136.stdout: "enabled": null 2024-09-12T23:37:49.513 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.513 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.513 INFO:tasks.workunit.client.0.smithi136.stdout: "network": { 2024-09-12T23:37:49.513 INFO:tasks.workunit.client.0.smithi136.stdout: "provider": null 2024-09-12T23:37:49.513 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.513 INFO:tasks.workunit.client.0.smithi136.stdout: "storage": { 2024-09-12T23:37:49.513 INFO:tasks.workunit.client.0.smithi136.stdout: "device-set": { 2024-09-12T23:37:49.513 INFO:tasks.workunit.client.0.smithi136.stdout: "count": { 2024-09-12T23:37:49.513 INFO:tasks.workunit.client.0.smithi136.stdout: "non-portable": null, 2024-09-12T23:37:49.513 INFO:tasks.workunit.client.0.smithi136.stdout: "portable": null, 2024-09-12T23:37:49.513 INFO:tasks.workunit.client.0.smithi136.stdout: "total": null 2024-09-12T23:37:49.514 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.514 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.514 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.514 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.514 INFO:tasks.workunit.client.0.smithi136.stdout: "csi": { 2024-09-12T23:37:49.514 INFO:tasks.workunit.client.0.smithi136.stdout: "version": null 2024-09-12T23:37:49.514 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.514 INFO:tasks.workunit.client.0.smithi136.stdout: "kubernetes": { 2024-09-12T23:37:49.514 INFO:tasks.workunit.client.0.smithi136.stdout: "version": null 2024-09-12T23:37:49.514 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.514 INFO:tasks.workunit.client.0.smithi136.stdout: "node": { 2024-09-12T23:37:49.515 INFO:tasks.workunit.client.0.smithi136.stdout: "count": { 2024-09-12T23:37:49.515 INFO:tasks.workunit.client.0.smithi136.stdout: "kubernetes-total": null, 2024-09-12T23:37:49.515 INFO:tasks.workunit.client.0.smithi136.stdout: "with-ceph-daemons": null, 2024-09-12T23:37:49.515 INFO:tasks.workunit.client.0.smithi136.stdout: "with-csi-cephfs-plugin": null, 2024-09-12T23:37:49.515 INFO:tasks.workunit.client.0.smithi136.stdout: "with-csi-nfs-plugin": null, 2024-09-12T23:37:49.515 INFO:tasks.workunit.client.0.smithi136.stdout: "with-csi-rbd-plugin": null 2024-09-12T23:37:49.515 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.515 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.515 INFO:tasks.workunit.client.0.smithi136.stdout: "usage": { 2024-09-12T23:37:49.515 INFO:tasks.workunit.client.0.smithi136.stdout: "storage-class": { 2024-09-12T23:37:49.515 INFO:tasks.workunit.client.0.smithi136.stdout: "count": { 2024-09-12T23:37:49.515 INFO:tasks.workunit.client.0.smithi136.stdout: "bucket": null, 2024-09-12T23:37:49.516 INFO:tasks.workunit.client.0.smithi136.stdout: "cephfs": null, 2024-09-12T23:37:49.516 INFO:tasks.workunit.client.0.smithi136.stdout: "nfs": null, 2024-09-12T23:37:49.516 INFO:tasks.workunit.client.0.smithi136.stdout: "rbd": null, 2024-09-12T23:37:49.516 INFO:tasks.workunit.client.0.smithi136.stdout: "total": null 2024-09-12T23:37:49.516 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.516 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:49.516 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.516 INFO:tasks.workunit.client.0.smithi136.stdout: "version": null 2024-09-12T23:37:49.516 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.516 INFO:tasks.workunit.client.0.smithi136.stdout: "services": {}, 2024-09-12T23:37:49.516 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_per_pg": [ 2024-09-12T23:37:49.516 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.517 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:49.517 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:49.517 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:49.517 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:49.517 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.517 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 7, 2024-09-12T23:37:49.517 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:49.517 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:49.517 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:49.517 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:49.517 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:49.517 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:49.518 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.665027+0000", 2024-09-12T23:37:49.518 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.908161+0000", 2024-09-12T23:37:49.518 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.908161+0000", 2024-09-12T23:37:49.518 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.908602+0000", 2024-09-12T23:37:49.518 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.665027+0000", 2024-09-12T23:37:49.518 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.518 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:49.518 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.518 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:49.518 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.665027+0000", 2024-09-12T23:37:49.518 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.665027+0000", 2024-09-12T23:37:49.518 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.665027+0000", 2024-09-12T23:37:49.519 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:49.519 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:49.519 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.519 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.665027+0000", 2024-09-12T23:37:49.519 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.665027+0000", 2024-09-12T23:37:49.519 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:49.519 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:49.519 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:49.519 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:49.519 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:49.520 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:49.520 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:49.520 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:49.520 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:49.520 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:49.520 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:49.520 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:49.520 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:49.520 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.3f", 2024-09-12T23:37:49.520 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:49.521 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:49.521 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:49.521 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:49.521 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:49.521 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T02:18:25.662166+0000", 2024-09-12T23:37:49.521 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:49.521 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:49.521 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:49.521 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:49.521 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:49.521 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:49.522 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:49.522 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:49.522 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:49.522 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:49.522 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:49.522 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:49.522 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:49.522 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:49.522 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:49.522 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:49.522 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:49.523 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:49.523 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:49.523 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:49.523 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:49.523 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:49.523 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:49.523 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:49.523 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:49.523 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:49.523 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:49.523 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:49.523 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:49.524 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:49.524 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:49.524 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:49.524 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:49.524 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:49.524 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:49.524 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:49.524 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:49.524 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:49.524 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:49.524 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:49.524 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:49.524 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:49.525 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:49.525 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.525 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:49.525 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:49.525 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:49.525 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:49.525 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:49.525 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:49.525 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.525 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 7, 2024-09-12T23:37:49.526 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:49.526 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.526 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.526 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:49.526 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:49.526 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:49.526 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:49.526 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.526 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 1, 2024-09-12T23:37:49.526 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:49.526 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:49.527 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:49.527 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:49.527 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:49.527 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:49.527 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.664503+0000", 2024-09-12T23:37:49.527 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.906765+0000", 2024-09-12T23:37:49.527 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.906765+0000", 2024-09-12T23:37:49.527 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.907919+0000", 2024-09-12T23:37:49.527 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.664503+0000", 2024-09-12T23:37:49.527 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.528 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:49.528 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.528 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:49.528 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.664503+0000", 2024-09-12T23:37:49.528 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.664503+0000", 2024-09-12T23:37:49.528 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.664503+0000", 2024-09-12T23:37:49.528 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:49.528 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:49.528 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.528 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.664503+0000", 2024-09-12T23:37:49.528 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.664503+0000", 2024-09-12T23:37:49.529 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:49.529 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:49.529 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:49.529 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:49.529 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:49.529 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:49.529 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:49.529 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:49.529 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:49.529 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:49.529 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:49.530 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:49.530 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:49.530 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.3e", 2024-09-12T23:37:49.530 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:49.530 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:49.530 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:49.530 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:49.530 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:49.530 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T01:39:49.787701+0000", 2024-09-12T23:37:49.530 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:49.530 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:49.531 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:49.531 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:49.531 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:49.531 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:49.531 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:49.531 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:49.531 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:49.531 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:49.531 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:49.531 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:49.531 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:49.532 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:49.532 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:49.532 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:49.532 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:49.532 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:49.532 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:49.532 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:49.532 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:49.532 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:49.532 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:49.533 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:49.533 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:49.533 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:49.533 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:49.533 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:49.533 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:49.533 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:49.533 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:49.533 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:49.533 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:49.533 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:49.534 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:49.534 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:49.534 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:49.534 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:49.534 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:49.534 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:49.534 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:49.534 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:49.534 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:49.534 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.535 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:49.535 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:49.535 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:49.535 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:49.535 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:49.535 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:49.535 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.535 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 1, 2024-09-12T23:37:49.535 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:49.536 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.536 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.536 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:49.536 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:49.536 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:49.536 INFO:tasks.workunit.client.0.smithi136.stdout: 1 2024-09-12T23:37:49.536 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.536 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 0, 2024-09-12T23:37:49.536 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:49.536 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:49.536 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:49.537 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:49.537 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:49.537 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:49.537 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.671331+0000", 2024-09-12T23:37:49.537 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.924268+0000", 2024-09-12T23:37:49.537 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.924268+0000", 2024-09-12T23:37:49.537 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.924626+0000", 2024-09-12T23:37:49.537 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.671331+0000", 2024-09-12T23:37:49.537 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.538 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:49.538 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.538 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:49.538 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.671331+0000", 2024-09-12T23:37:49.538 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.671331+0000", 2024-09-12T23:37:49.538 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.671331+0000", 2024-09-12T23:37:49.538 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:49.538 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:49.538 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.538 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.671331+0000", 2024-09-12T23:37:49.539 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.671331+0000", 2024-09-12T23:37:49.539 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:49.539 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:49.539 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:49.539 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:49.539 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:49.539 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:49.539 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:49.539 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:49.539 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:49.539 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:49.540 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:49.540 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:49.540 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:49.540 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.3d", 2024-09-12T23:37:49.540 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:49.540 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:49.540 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:49.540 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:49.540 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:49.540 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T00:41:31.853826+0000", 2024-09-12T23:37:49.540 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:49.541 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:49.541 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:49.541 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:49.541 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:49.541 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:49.541 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:49.541 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:49.541 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:49.541 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:49.542 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:49.542 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:49.542 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:49.542 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:49.542 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:49.542 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:49.542 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:49.542 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:49.542 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:49.542 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:49.543 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:49.543 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:49.543 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:49.543 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:49.543 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:49.543 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:49.543 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:49.543 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:49.543 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:49.543 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:49.543 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:49.543 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:49.544 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:49.544 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:49.544 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:49.544 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:49.544 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:49.544 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:49.544 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:49.544 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:49.544 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:49.544 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:49.544 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:49.545 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.545 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:49.545 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:49.545 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:49.545 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:49.545 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:49.545 INFO:tasks.workunit.client.0.smithi136.stdout: 1 2024-09-12T23:37:49.545 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.545 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 0, 2024-09-12T23:37:49.545 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:49.545 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.546 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.546 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:49.546 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:49.546 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:49.546 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:49.546 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.546 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 5, 2024-09-12T23:37:49.546 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:49.546 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:49.546 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:49.546 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:49.547 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:49.547 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:49.547 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.670520+0000", 2024-09-12T23:37:49.547 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.913672+0000", 2024-09-12T23:37:49.547 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.913672+0000", 2024-09-12T23:37:49.547 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.913990+0000", 2024-09-12T23:37:49.547 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.670520+0000", 2024-09-12T23:37:49.547 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.548 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:49.548 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.548 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:49.548 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.670520+0000", 2024-09-12T23:37:49.548 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.670520+0000", 2024-09-12T23:37:49.548 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.670520+0000", 2024-09-12T23:37:49.548 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:49.548 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:49.548 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.548 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.670520+0000", 2024-09-12T23:37:49.548 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.670520+0000", 2024-09-12T23:37:49.549 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:49.549 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:49.549 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:49.549 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:49.549 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:49.549 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:49.549 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:49.549 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:49.549 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:49.549 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:49.549 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:49.549 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:49.550 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:49.550 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.3c", 2024-09-12T23:37:49.550 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:49.550 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:49.550 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:49.550 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:49.550 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:49.550 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T03:57:54.241724+0000", 2024-09-12T23:37:49.550 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:49.550 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:49.550 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:49.550 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:49.551 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:49.551 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:49.551 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:49.551 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:49.551 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:49.551 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:49.551 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:49.551 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:49.551 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:49.551 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:49.552 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:49.552 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:49.552 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:49.552 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:49.552 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:49.552 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:49.552 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:49.552 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:49.552 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:49.552 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:49.552 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:49.553 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:49.553 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:49.553 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:49.553 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:49.553 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:49.553 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:49.553 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:49.553 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:49.553 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:49.553 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:49.553 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:49.554 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:49.554 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:49.554 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:49.554 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:49.554 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:49.554 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:49.554 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:49.554 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.554 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:49.554 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:49.554 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:49.554 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:49.555 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:49.555 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:49.555 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.555 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 5, 2024-09-12T23:37:49.555 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:49.555 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.555 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.555 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:49.555 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:49.555 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:49.556 INFO:tasks.workunit.client.0.smithi136.stdout: 4 2024-09-12T23:37:49.556 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.556 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 7, 2024-09-12T23:37:49.556 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:49.556 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:49.556 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:49.556 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:49.556 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:49.556 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:49.556 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.663791+0000", 2024-09-12T23:37:49.556 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.904192+0000", 2024-09-12T23:37:49.556 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.904192+0000", 2024-09-12T23:37:49.557 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.904505+0000", 2024-09-12T23:37:49.557 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.663791+0000", 2024-09-12T23:37:49.557 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.557 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:49.557 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.557 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:49.557 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.663791+0000", 2024-09-12T23:37:49.557 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.663791+0000", 2024-09-12T23:37:49.557 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.663791+0000", 2024-09-12T23:37:49.557 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:49.557 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:49.557 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.558 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.663791+0000", 2024-09-12T23:37:49.558 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.663791+0000", 2024-09-12T23:37:49.558 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:49.558 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:49.558 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:49.558 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:49.558 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:49.558 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:49.558 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:49.558 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:49.559 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:49.559 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:49.559 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:49.559 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:49.559 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:49.559 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.3b", 2024-09-12T23:37:49.559 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:49.559 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:49.559 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:49.559 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:49.560 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:49.560 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T06:49:27.086584+0000", 2024-09-12T23:37:49.560 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:49.560 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:49.560 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:49.560 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:49.560 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:49.560 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:49.560 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:49.560 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:49.561 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:49.561 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:49.561 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:49.561 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:49.561 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:49.561 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:49.561 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:49.561 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:49.561 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:49.561 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:49.562 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:49.562 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:49.562 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:49.562 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:49.562 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:49.562 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:49.562 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:49.562 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:49.562 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:49.562 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:49.562 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:49.563 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:49.563 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:49.563 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:49.563 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:49.563 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:49.563 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:49.563 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:49.563 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:49.563 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:49.563 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:49.563 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:49.563 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:49.564 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:49.564 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:49.564 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.564 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:49.564 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:49.564 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:49.564 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:49.564 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:49.564 INFO:tasks.workunit.client.0.smithi136.stdout: 4 2024-09-12T23:37:49.564 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.564 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 7, 2024-09-12T23:37:49.565 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:49.565 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.565 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.565 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:49.565 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:49.565 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:49.565 INFO:tasks.workunit.client.0.smithi136.stdout: 1 2024-09-12T23:37:49.565 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.565 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 5, 2024-09-12T23:37:49.565 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:49.565 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:49.565 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:49.566 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:49.566 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:49.566 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:49.566 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.671038+0000", 2024-09-12T23:37:49.566 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.905154+0000", 2024-09-12T23:37:49.566 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.905154+0000", 2024-09-12T23:37:49.566 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.905467+0000", 2024-09-12T23:37:49.566 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.671038+0000", 2024-09-12T23:37:49.566 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.566 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:49.566 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.566 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:49.566 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.671038+0000", 2024-09-12T23:37:49.567 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.671038+0000", 2024-09-12T23:37:49.567 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.671038+0000", 2024-09-12T23:37:49.567 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:49.567 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:49.567 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.567 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.671038+0000", 2024-09-12T23:37:49.567 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.671038+0000", 2024-09-12T23:37:49.567 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:49.567 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:49.567 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:49.567 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:49.567 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:49.568 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:49.568 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:49.568 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:49.568 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:49.568 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:49.568 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:49.568 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:49.568 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:49.568 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.3a", 2024-09-12T23:37:49.568 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:49.569 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:49.569 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:49.569 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:49.569 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:49.569 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T01:49:00.596502+0000", 2024-09-12T23:37:49.569 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:49.569 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:49.569 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:49.569 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:49.569 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:49.569 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:49.569 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:49.570 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:49.570 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:49.570 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:49.570 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:49.570 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:49.570 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:49.570 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:49.570 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:49.570 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:49.570 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:49.570 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:49.570 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:49.571 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:49.571 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:49.571 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:49.571 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:49.571 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:49.571 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:49.571 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:49.571 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:49.571 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:49.571 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:49.571 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:49.571 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:49.571 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:49.572 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:49.572 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:49.572 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:49.572 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:49.572 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:49.572 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:49.572 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:49.572 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:49.572 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:49.573 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:49.573 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:49.573 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.573 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:49.573 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:49.573 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:49.573 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:49.573 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:49.573 INFO:tasks.workunit.client.0.smithi136.stdout: 1 2024-09-12T23:37:49.573 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.573 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 5, 2024-09-12T23:37:49.573 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:49.573 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.574 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.574 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:49.574 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:49.574 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:49.574 INFO:tasks.workunit.client.0.smithi136.stdout: 4 2024-09-12T23:37:49.574 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.574 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 7, 2024-09-12T23:37:49.574 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:49.574 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:49.574 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:49.574 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:49.574 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:49.575 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:49.575 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.664458+0000", 2024-09-12T23:37:49.575 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.908427+0000", 2024-09-12T23:37:49.575 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.908427+0000", 2024-09-12T23:37:49.575 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.908762+0000", 2024-09-12T23:37:49.575 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.664458+0000", 2024-09-12T23:37:49.575 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.575 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:49.575 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.575 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:49.575 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.664458+0000", 2024-09-12T23:37:49.575 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.664458+0000", 2024-09-12T23:37:49.576 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.664458+0000", 2024-09-12T23:37:49.576 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:49.576 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:49.576 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.576 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.664458+0000", 2024-09-12T23:37:49.576 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.664458+0000", 2024-09-12T23:37:49.576 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:49.576 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:49.576 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:49.576 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:49.576 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:49.576 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:49.577 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:49.577 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:49.577 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:49.577 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:49.577 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:49.577 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:49.577 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:49.577 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.39", 2024-09-12T23:37:49.577 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:49.577 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:49.577 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:49.577 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:49.578 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:49.578 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T10:44:26.357691+0000", 2024-09-12T23:37:49.578 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:49.578 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:49.578 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:49.578 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:49.578 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:49.578 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:49.578 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:49.578 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:49.578 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:49.578 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:49.579 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:49.579 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:49.579 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:49.579 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:49.579 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:49.579 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:49.579 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:49.579 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:49.579 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:49.579 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:49.579 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:49.580 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:49.580 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:49.580 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:49.580 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:49.580 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:49.580 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:49.580 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:49.580 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:49.580 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:49.580 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:49.580 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:49.581 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:49.581 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:49.581 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:49.581 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:49.581 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:49.581 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:49.581 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:49.581 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:49.581 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:49.581 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:49.581 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:49.581 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.582 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:49.582 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:49.582 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:49.582 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:49.582 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:49.582 INFO:tasks.workunit.client.0.smithi136.stdout: 4 2024-09-12T23:37:49.582 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.582 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 7, 2024-09-12T23:37:49.582 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:49.582 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.582 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.583 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:49.583 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:49.583 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:49.583 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:49.583 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.583 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 3, 2024-09-12T23:37:49.583 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:49.583 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:49.583 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:49.583 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:49.583 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:49.584 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:49.584 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.666998+0000", 2024-09-12T23:37:49.584 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.901611+0000", 2024-09-12T23:37:49.584 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.901611+0000", 2024-09-12T23:37:49.584 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.902033+0000", 2024-09-12T23:37:49.584 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.666998+0000", 2024-09-12T23:37:49.584 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.584 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:49.584 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.584 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:49.584 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.666998+0000", 2024-09-12T23:37:49.584 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.666998+0000", 2024-09-12T23:37:49.585 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.666998+0000", 2024-09-12T23:37:49.585 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:49.585 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:49.585 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.585 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.666998+0000", 2024-09-12T23:37:49.585 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.666998+0000", 2024-09-12T23:37:49.585 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:49.585 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:49.585 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:49.585 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:49.585 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:49.585 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:49.586 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:49.586 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:49.586 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:49.586 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:49.586 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:49.586 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:49.586 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:49.586 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.38", 2024-09-12T23:37:49.586 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:49.586 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:49.586 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:49.586 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:49.587 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:49.587 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T06:57:58.671558+0000", 2024-09-12T23:37:49.587 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:49.587 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:49.587 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:49.587 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:49.587 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:49.587 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:49.587 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:49.587 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:49.587 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:49.588 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:49.588 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:49.588 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:49.588 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:49.588 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:49.588 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:49.588 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:49.588 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:49.588 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:49.588 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:49.588 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:49.588 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:49.589 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:49.589 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:49.589 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:49.589 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:49.589 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:49.589 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:49.589 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:49.589 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:49.589 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:49.589 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:49.590 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:49.590 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:49.590 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:49.590 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:49.590 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:49.590 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:49.590 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:49.590 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:49.590 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:49.590 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:49.590 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:49.590 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:49.591 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.591 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:49.591 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:49.591 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:49.591 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:49.591 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:49.591 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:49.591 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.591 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 3, 2024-09-12T23:37:49.591 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:49.591 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.592 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.592 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:49.592 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:49.592 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:49.592 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:49.592 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.592 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 7, 2024-09-12T23:37:49.592 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:49.592 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:49.592 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:49.592 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:49.592 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:49.593 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:49.593 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:09.803676+0000", 2024-09-12T23:37:49.593 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:35:09.802434+0000", 2024-09-12T23:37:49.593 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:35:09.802434+0000", 2024-09-12T23:37:49.593 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:35:09.803676+0000", 2024-09-12T23:37:49.593 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:09.803676+0000", 2024-09-12T23:37:49.593 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.593 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:49.593 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.593 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 64, 2024-09-12T23:37:49.593 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:09.803676+0000", 2024-09-12T23:37:49.593 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:09.803676+0000", 2024-09-12T23:37:49.594 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:09.803676+0000", 2024-09-12T23:37:49.594 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:49.594 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:49.594 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.594 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:09.803676+0000", 2024-09-12T23:37:49.594 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:09.803676+0000", 2024-09-12T23:37:49.594 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:49.594 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:49.594 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:49.594 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:49.594 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 63, 2024-09-12T23:37:49.594 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:49.595 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:49.595 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:49.595 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:49.595 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:49.595 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:49.595 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:49.595 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:49.595 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.37", 2024-09-12T23:37:49.595 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:49.595 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:49.595 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 64, 2024-09-12T23:37:49.596 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 30, 2024-09-12T23:37:49.596 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:49.596 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T04:01:37.156347+0000", 2024-09-12T23:37:49.596 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:49.596 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:49.596 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:49.596 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:49.596 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:49.596 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:49.596 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:49.596 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:49.597 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:49.597 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:49.597 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:49.597 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:49.597 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:49.597 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:49.597 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:49.597 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:49.597 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:49.597 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:49.598 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:49.598 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:49.598 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:49.598 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:49.598 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:49.598 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:49.598 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:49.598 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:49.598 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:49.598 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:49.598 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:49.599 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:49.599 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:49.599 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:49.599 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:49.599 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:49.599 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:49.599 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:49.599 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:49.599 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:49.599 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:49.599 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:49.599 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:49.600 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:49.600 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:49.600 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.600 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:49.600 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:49.600 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:49.600 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:49.600 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:49.600 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:49.600 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.600 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 7, 2024-09-12T23:37:49.600 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:49.601 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.601 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.601 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:49.601 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:49.601 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:49.601 INFO:tasks.workunit.client.0.smithi136.stdout: 1 2024-09-12T23:37:49.601 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.601 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 6, 2024-09-12T23:37:49.601 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:49.601 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:49.601 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:49.601 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:49.602 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:49.602 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:49.602 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.666805+0000", 2024-09-12T23:37:49.602 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.892815+0000", 2024-09-12T23:37:49.602 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.892815+0000", 2024-09-12T23:37:49.602 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.893241+0000", 2024-09-12T23:37:49.602 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.666805+0000", 2024-09-12T23:37:49.602 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.602 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:49.602 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.602 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:49.602 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.666805+0000", 2024-09-12T23:37:49.603 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.666805+0000", 2024-09-12T23:37:49.603 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.666805+0000", 2024-09-12T23:37:49.603 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:49.603 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:49.603 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.603 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.666805+0000", 2024-09-12T23:37:49.603 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.666805+0000", 2024-09-12T23:37:49.603 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:49.603 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:49.603 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:49.603 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:49.603 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:49.604 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:49.604 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:49.604 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:49.604 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:49.604 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:49.604 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:49.604 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:49.604 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:49.604 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.36", 2024-09-12T23:37:49.604 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:49.604 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:49.604 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:49.605 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:49.605 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:49.605 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T01:46:21.539771+0000", 2024-09-12T23:37:49.605 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:49.605 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:49.605 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:49.605 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:49.605 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:49.605 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:49.605 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:49.605 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:49.605 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:49.606 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:49.606 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:49.606 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:49.606 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:49.606 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:49.606 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:49.606 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:49.606 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:49.606 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:49.606 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:49.607 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:49.607 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:49.607 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:49.607 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:49.607 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:49.607 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:49.607 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:49.607 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:49.607 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:49.608 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:49.608 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:49.608 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:49.608 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:49.608 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:49.608 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:49.608 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:49.608 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:49.608 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:49.608 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:49.608 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:49.609 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:49.609 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:49.609 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:49.609 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:49.609 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.609 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:49.609 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:49.609 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:49.609 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:49.609 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:49.610 INFO:tasks.workunit.client.0.smithi136.stdout: 1 2024-09-12T23:37:49.610 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.610 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 6, 2024-09-12T23:37:49.610 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:49.610 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.610 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.610 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:49.610 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:49.610 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:49.610 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:49.610 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.610 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 6, 2024-09-12T23:37:49.611 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:49.611 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:49.611 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:49.611 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:49.611 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:49.611 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:49.611 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.665734+0000", 2024-09-12T23:37:49.611 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.906493+0000", 2024-09-12T23:37:49.611 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.906493+0000", 2024-09-12T23:37:49.611 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.906810+0000", 2024-09-12T23:37:49.611 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.665734+0000", 2024-09-12T23:37:49.611 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.611 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:49.612 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.612 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:49.612 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.665734+0000", 2024-09-12T23:37:49.612 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.665734+0000", 2024-09-12T23:37:49.612 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.665734+0000", 2024-09-12T23:37:49.612 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:49.612 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:49.612 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.612 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.665734+0000", 2024-09-12T23:37:49.612 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.665734+0000", 2024-09-12T23:37:49.612 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:49.613 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:49.613 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:49.613 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:49.613 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:49.613 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:49.613 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:49.613 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:49.613 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:49.613 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:49.613 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:49.613 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:49.613 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:49.614 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.35", 2024-09-12T23:37:49.614 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:49.614 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:49.614 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:49.614 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:49.614 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:49.614 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T05:56:07.145258+0000", 2024-09-12T23:37:49.614 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:49.614 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:49.614 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:49.614 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:49.615 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:49.615 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:49.615 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:49.615 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:49.615 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:49.615 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:49.615 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:49.615 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:49.615 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:49.615 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:49.615 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:49.616 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:49.616 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:49.616 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:49.616 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:49.616 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:49.616 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:49.616 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:49.616 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:49.616 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:49.616 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:49.616 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:49.616 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:49.617 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:49.617 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:49.617 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:49.617 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:49.617 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:49.617 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:49.617 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:49.617 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:49.617 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:49.617 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:49.618 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:49.618 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:49.618 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:49.618 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:49.618 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:49.618 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:49.618 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.618 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:49.618 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:49.618 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:49.618 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:49.618 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:49.619 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:49.619 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.619 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 6, 2024-09-12T23:37:49.619 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:49.619 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.619 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.619 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:49.619 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:49.619 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:49.620 INFO:tasks.workunit.client.0.smithi136.stdout: 3 2024-09-12T23:37:49.620 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.620 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 4, 2024-09-12T23:37:49.620 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:49.620 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:49.620 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:49.620 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:49.620 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:49.620 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:49.621 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:09.802722+0000", 2024-09-12T23:37:49.621 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:35:09.801763+0000", 2024-09-12T23:37:49.621 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:35:09.801763+0000", 2024-09-12T23:37:49.621 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:35:09.802722+0000", 2024-09-12T23:37:49.621 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:09.802722+0000", 2024-09-12T23:37:49.621 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.621 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:49.621 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.621 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 64, 2024-09-12T23:37:49.621 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:09.802722+0000", 2024-09-12T23:37:49.622 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:09.802722+0000", 2024-09-12T23:37:49.622 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:09.802722+0000", 2024-09-12T23:37:49.622 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:49.622 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:49.622 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.622 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:09.802722+0000", 2024-09-12T23:37:49.622 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:09.802722+0000", 2024-09-12T23:37:49.622 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:49.622 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:49.622 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:49.622 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:49.623 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 63, 2024-09-12T23:37:49.623 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:49.623 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:49.623 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:49.623 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:49.623 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:49.623 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:49.624 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:49.624 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:49.624 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.34", 2024-09-12T23:37:49.624 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:49.624 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:49.624 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 64, 2024-09-12T23:37:49.625 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 30, 2024-09-12T23:37:49.625 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:49.625 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T10:33:24.574457+0000", 2024-09-12T23:37:49.625 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:49.625 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:49.625 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:49.625 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:49.625 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:49.625 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:49.625 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:49.626 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:49.626 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:49.626 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:49.626 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:49.626 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:49.626 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:49.626 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:49.626 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:49.626 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:49.626 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:49.626 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:49.627 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:49.627 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:49.627 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:49.627 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:49.627 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:49.627 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:49.627 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:49.627 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:49.627 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:49.627 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:49.627 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:49.628 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:49.628 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:49.628 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:49.628 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:49.628 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:49.628 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:49.628 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:49.628 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:49.628 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:49.628 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:49.628 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:49.629 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:49.629 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:49.629 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:49.629 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.629 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:49.629 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:49.629 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:49.629 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:49.629 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:49.629 INFO:tasks.workunit.client.0.smithi136.stdout: 3 2024-09-12T23:37:49.629 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.630 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 4, 2024-09-12T23:37:49.630 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:49.630 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.630 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.630 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:49.630 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:49.630 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:49.630 INFO:tasks.workunit.client.0.smithi136.stdout: 1 2024-09-12T23:37:49.631 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.631 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 3, 2024-09-12T23:37:49.631 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:49.631 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:49.631 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:49.631 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:49.631 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:49.631 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:49.631 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.668470+0000", 2024-09-12T23:37:49.631 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.906577+0000", 2024-09-12T23:37:49.631 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.906577+0000", 2024-09-12T23:37:49.632 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.907459+0000", 2024-09-12T23:37:49.632 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.668470+0000", 2024-09-12T23:37:49.632 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.632 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:49.632 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.632 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:49.632 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.668470+0000", 2024-09-12T23:37:49.632 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.668470+0000", 2024-09-12T23:37:49.632 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.668470+0000", 2024-09-12T23:37:49.632 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:49.632 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:49.633 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.633 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.668470+0000", 2024-09-12T23:37:49.633 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.668470+0000", 2024-09-12T23:37:49.633 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:49.633 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:49.633 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:49.633 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:49.633 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:49.633 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:49.633 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:49.633 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:49.633 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:49.634 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:49.634 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:49.634 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:49.634 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:49.634 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.33", 2024-09-12T23:37:49.634 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:49.634 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:49.634 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:49.634 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:49.634 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:49.634 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T07:21:41.131616+0000", 2024-09-12T23:37:49.635 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:49.635 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:49.635 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:49.635 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:49.635 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:49.635 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:49.635 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:49.635 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:49.635 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:49.636 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:49.636 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:49.636 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:49.636 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:49.636 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:49.636 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:49.636 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:49.636 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:49.636 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:49.636 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:49.637 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:49.637 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:49.637 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:49.637 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:49.637 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:49.637 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:49.637 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:49.637 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:49.637 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:49.638 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:49.638 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:49.638 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:49.638 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:49.638 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:49.638 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:49.638 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:49.638 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:49.638 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:49.638 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:49.639 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:49.639 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:49.639 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:49.639 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:49.639 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:49.639 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.639 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:49.639 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:49.639 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:49.639 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:49.639 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:49.640 INFO:tasks.workunit.client.0.smithi136.stdout: 1 2024-09-12T23:37:49.640 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.640 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 3, 2024-09-12T23:37:49.640 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:49.640 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.640 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.640 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:49.640 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:49.640 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:49.640 INFO:tasks.workunit.client.0.smithi136.stdout: 5 2024-09-12T23:37:49.641 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.641 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 0, 2024-09-12T23:37:49.641 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:49.641 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:49.641 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:49.641 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:49.641 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:49.641 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:49.641 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.670257+0000", 2024-09-12T23:37:49.641 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.918051+0000", 2024-09-12T23:37:49.642 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.918051+0000", 2024-09-12T23:37:49.642 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.920721+0000", 2024-09-12T23:37:49.642 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.670257+0000", 2024-09-12T23:37:49.642 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.642 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:49.642 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.642 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:49.642 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.670257+0000", 2024-09-12T23:37:49.642 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.670257+0000", 2024-09-12T23:37:49.642 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.670257+0000", 2024-09-12T23:37:49.643 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:49.643 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:49.643 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.643 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.670257+0000", 2024-09-12T23:37:49.643 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.670257+0000", 2024-09-12T23:37:49.643 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:49.643 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:49.643 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:49.643 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:49.643 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:49.643 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:49.644 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:49.644 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:49.644 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:49.644 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:49.644 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:49.644 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:49.645 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:49.645 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.32", 2024-09-12T23:37:49.645 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:49.645 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:49.645 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:49.645 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:49.645 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:49.646 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T10:32:27.077464+0000", 2024-09-12T23:37:49.646 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:49.646 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:49.646 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:49.646 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:49.646 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:49.646 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:49.646 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:49.647 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:49.647 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:49.647 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:49.647 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:49.647 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:49.647 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:49.647 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:49.648 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:49.648 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:49.648 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:49.648 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:49.648 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:49.648 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:49.648 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:49.649 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:49.649 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:49.649 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:49.649 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:49.649 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:49.649 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:49.650 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:49.650 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:49.650 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:49.650 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:49.650 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:49.650 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:49.650 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:49.651 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:49.651 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:49.651 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:49.651 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:49.651 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:49.651 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:49.651 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:49.652 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:49.652 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:49.652 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.652 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:49.652 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:49.652 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:49.653 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:49.653 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:49.653 INFO:tasks.workunit.client.0.smithi136.stdout: 5 2024-09-12T23:37:49.653 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.653 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 0, 2024-09-12T23:37:49.653 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:49.653 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.654 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.654 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:49.654 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:49.654 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:49.654 INFO:tasks.workunit.client.0.smithi136.stdout: 5 2024-09-12T23:37:49.654 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.654 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 1, 2024-09-12T23:37:49.655 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:49.655 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:49.655 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:49.655 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:49.655 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:49.655 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:49.655 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:09.683561+0000", 2024-09-12T23:37:49.656 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:35:09.682671+0000", 2024-09-12T23:37:49.656 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:35:09.682671+0000", 2024-09-12T23:37:49.656 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:35:09.683561+0000", 2024-09-12T23:37:49.656 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:09.683561+0000", 2024-09-12T23:37:49.656 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.656 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:49.656 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.657 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 64, 2024-09-12T23:37:49.657 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:09.683561+0000", 2024-09-12T23:37:49.657 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:09.683561+0000", 2024-09-12T23:37:49.657 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:09.683561+0000", 2024-09-12T23:37:49.657 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:49.657 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:49.657 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.658 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:09.683561+0000", 2024-09-12T23:37:49.658 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:09.683561+0000", 2024-09-12T23:37:49.658 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:49.658 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:49.658 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:49.658 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:49.658 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 63, 2024-09-12T23:37:49.659 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:49.659 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:49.659 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:49.659 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:49.659 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:49.659 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:49.659 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:49.660 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:49.660 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.31", 2024-09-12T23:37:49.660 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:49.660 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:49.660 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 64, 2024-09-12T23:37:49.660 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 30, 2024-09-12T23:37:49.661 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:49.661 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T05:45:39.393684+0000", 2024-09-12T23:37:49.661 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:49.661 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:49.661 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:49.661 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:49.661 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:49.661 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:49.661 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:49.662 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:49.662 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:49.662 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:49.662 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:49.662 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:49.662 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:49.662 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:49.662 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:49.662 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:49.662 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:49.663 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:49.663 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:49.663 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:49.663 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:49.663 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:49.663 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:49.663 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:49.663 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:49.663 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:49.663 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:49.664 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:49.664 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:49.664 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:49.664 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:49.664 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:49.664 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:49.664 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:49.664 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:49.664 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:49.664 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:49.664 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:49.664 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:49.665 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:49.665 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:49.665 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:49.665 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:49.665 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.665 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:49.665 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:49.665 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:49.665 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:49.665 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:49.665 INFO:tasks.workunit.client.0.smithi136.stdout: 5 2024-09-12T23:37:49.665 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.666 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 1, 2024-09-12T23:37:49.666 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:49.666 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.666 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.666 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:49.666 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:49.666 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:49.666 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:49.666 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.666 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 2, 2024-09-12T23:37:49.666 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:49.666 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:49.667 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:49.667 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:49.667 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:49.667 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:49.667 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:09.804288+0000", 2024-09-12T23:37:49.667 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:35:09.803450+0000", 2024-09-12T23:37:49.667 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:35:09.803450+0000", 2024-09-12T23:37:49.667 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:35:09.804288+0000", 2024-09-12T23:37:49.667 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:09.804288+0000", 2024-09-12T23:37:49.667 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.667 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:49.668 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.668 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 64, 2024-09-12T23:37:49.668 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:09.804288+0000", 2024-09-12T23:37:49.668 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:09.804288+0000", 2024-09-12T23:37:49.668 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:09.804288+0000", 2024-09-12T23:37:49.668 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:49.668 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:49.668 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.668 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:09.804288+0000", 2024-09-12T23:37:49.668 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:09.804288+0000", 2024-09-12T23:37:49.669 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:49.669 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:49.669 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:49.669 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:49.669 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 63, 2024-09-12T23:37:49.669 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:49.669 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:49.669 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:49.669 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:49.669 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:49.669 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:49.670 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:49.670 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:49.670 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.30", 2024-09-12T23:37:49.670 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:49.670 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:49.670 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 64, 2024-09-12T23:37:49.670 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 10, 2024-09-12T23:37:49.670 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:49.670 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T05:31:12.292070+0000", 2024-09-12T23:37:49.670 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:49.670 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:49.671 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:49.671 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:49.671 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:49.671 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:49.671 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:49.671 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:49.671 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:49.671 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:49.671 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:49.671 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:49.672 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:49.672 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:49.672 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:49.672 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:49.672 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:49.672 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:49.672 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:49.672 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:49.672 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:49.672 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:49.672 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:49.673 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:49.673 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:49.673 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:49.673 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:49.673 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:49.673 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:49.673 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:49.673 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:49.673 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:49.673 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:49.673 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:49.674 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:49.674 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:49.674 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:49.674 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:49.674 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:49.674 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:49.674 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:49.674 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:49.674 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:49.674 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.674 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:49.674 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:49.675 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:49.675 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:49.675 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:49.675 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:49.675 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.675 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 2, 2024-09-12T23:37:49.675 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:49.675 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.675 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.675 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:49.675 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:49.675 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:49.676 INFO:tasks.workunit.client.0.smithi136.stdout: 4 2024-09-12T23:37:49.676 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.676 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 5, 2024-09-12T23:37:49.676 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:49.676 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:49.676 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:49.676 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:49.676 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:49.676 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:49.676 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.670478+0000", 2024-09-12T23:37:49.676 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.911877+0000", 2024-09-12T23:37:49.677 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.911877+0000", 2024-09-12T23:37:49.677 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.913684+0000", 2024-09-12T23:37:49.677 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.670478+0000", 2024-09-12T23:37:49.677 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.677 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:49.677 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.677 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:49.677 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.670478+0000", 2024-09-12T23:37:49.677 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.670478+0000", 2024-09-12T23:37:49.677 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.670478+0000", 2024-09-12T23:37:49.678 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:49.678 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:49.678 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.678 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.670478+0000", 2024-09-12T23:37:49.678 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.670478+0000", 2024-09-12T23:37:49.678 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:49.678 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:49.678 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:49.678 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:49.678 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:49.678 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:49.678 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:49.679 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:49.679 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:49.679 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:49.679 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:49.679 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:49.679 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:49.679 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.2f", 2024-09-12T23:37:49.679 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:49.679 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:49.679 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:49.679 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:49.680 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:49.680 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T03:39:10.601087+0000", 2024-09-12T23:37:49.680 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:49.680 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:49.680 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:49.680 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:49.680 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:49.680 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:49.680 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:49.680 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:49.680 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:49.681 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:49.681 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:49.681 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:49.681 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:49.681 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:49.681 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:49.681 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:49.681 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:49.681 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:49.681 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:49.681 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:49.682 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:49.682 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:49.682 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:49.682 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:49.682 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:49.682 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:49.682 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:49.682 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:49.682 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:49.682 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:49.682 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:49.682 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:49.683 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:49.683 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:49.683 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:49.683 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:49.683 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:49.683 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:49.683 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:49.683 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:49.683 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:49.683 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:49.683 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:49.683 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.684 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:49.684 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:49.684 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:49.684 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:49.684 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:49.684 INFO:tasks.workunit.client.0.smithi136.stdout: 4 2024-09-12T23:37:49.684 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.684 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 5, 2024-09-12T23:37:49.684 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:49.684 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.684 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.684 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:49.684 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:49.685 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:49.685 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:49.685 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.685 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 2, 2024-09-12T23:37:49.685 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:49.685 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:49.685 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:49.685 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:49.685 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:49.685 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:49.685 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.661668+0000", 2024-09-12T23:37:49.686 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.882148+0000", 2024-09-12T23:37:49.686 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.882148+0000", 2024-09-12T23:37:49.686 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.882889+0000", 2024-09-12T23:37:49.686 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.661668+0000", 2024-09-12T23:37:49.686 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.686 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:49.686 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.686 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:49.686 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.661668+0000", 2024-09-12T23:37:49.686 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.661668+0000", 2024-09-12T23:37:49.686 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.661668+0000", 2024-09-12T23:37:49.686 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:49.687 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:49.687 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.687 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.661668+0000", 2024-09-12T23:37:49.687 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.661668+0000", 2024-09-12T23:37:49.687 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:49.687 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:49.687 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:49.687 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:49.687 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:49.687 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:49.687 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:49.687 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:49.688 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:49.688 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:49.688 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:49.688 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:49.688 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:49.688 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.2e", 2024-09-12T23:37:49.688 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:49.688 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:49.688 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:49.688 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:49.688 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:49.688 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T09:59:38.545795+0000", 2024-09-12T23:37:49.689 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:49.689 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:49.689 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:49.689 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:49.689 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:49.689 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:49.689 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:49.689 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:49.689 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:49.689 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:49.689 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:49.689 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:49.690 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:49.690 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:49.690 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:49.690 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:49.690 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:49.690 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:49.690 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:49.690 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:49.690 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:49.690 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:49.690 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:49.691 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:49.691 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:49.691 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:49.691 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:49.691 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:49.691 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:49.691 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:49.691 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:49.691 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:49.691 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:49.691 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:49.691 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:49.692 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:49.692 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:49.692 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:49.692 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:49.692 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:49.692 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:49.692 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:49.692 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:49.692 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.692 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:49.692 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:49.692 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:49.693 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:49.693 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:49.693 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:49.693 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.693 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 2, 2024-09-12T23:37:49.693 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:49.693 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.693 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.693 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:49.694 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:49.694 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:49.694 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:49.694 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.694 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 7, 2024-09-12T23:37:49.694 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:49.694 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:49.694 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:49.694 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:49.694 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:49.694 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:49.695 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.666136+0000", 2024-09-12T23:37:49.695 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.904247+0000", 2024-09-12T23:37:49.695 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.904247+0000", 2024-09-12T23:37:49.695 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.904552+0000", 2024-09-12T23:37:49.695 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.666136+0000", 2024-09-12T23:37:49.695 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.695 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:49.695 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.695 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:49.695 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.666136+0000", 2024-09-12T23:37:49.695 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.666136+0000", 2024-09-12T23:37:49.695 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.666136+0000", 2024-09-12T23:37:49.695 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:49.696 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:49.696 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.696 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.666136+0000", 2024-09-12T23:37:49.696 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.666136+0000", 2024-09-12T23:37:49.696 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:49.696 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:49.696 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:49.696 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:49.696 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:49.696 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:49.696 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:49.697 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:49.697 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:49.697 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:49.697 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:49.697 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:49.697 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:49.697 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.2d", 2024-09-12T23:37:49.697 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:49.697 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:49.697 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:49.697 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:49.697 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:49.698 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T00:30:06.600814+0000", 2024-09-12T23:37:49.698 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:49.698 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:49.698 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:49.698 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:49.698 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:49.698 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:49.698 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:49.698 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:49.698 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:49.698 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:49.698 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:49.699 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:49.699 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:49.699 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:49.699 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:49.699 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:49.699 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:49.699 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:49.699 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:49.699 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:49.699 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:49.699 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:49.699 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:49.700 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:49.700 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:49.700 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:49.700 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:49.700 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:49.700 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:49.700 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:49.700 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:49.700 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:49.700 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:49.700 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:49.700 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:49.701 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:49.701 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:49.701 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:49.701 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:49.701 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:49.701 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:49.701 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:49.701 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:49.701 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.701 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:49.701 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:49.702 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:49.702 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:49.702 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:49.702 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:49.702 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.702 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 7, 2024-09-12T23:37:49.702 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:49.702 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.702 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.702 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:49.702 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:49.702 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:49.703 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:49.703 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.703 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 6, 2024-09-12T23:37:49.703 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:49.703 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:49.703 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:49.703 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:49.703 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:49.703 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:49.703 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.665287+0000", 2024-09-12T23:37:49.703 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.905690+0000", 2024-09-12T23:37:49.703 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.905690+0000", 2024-09-12T23:37:49.704 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.905965+0000", 2024-09-12T23:37:49.704 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.665287+0000", 2024-09-12T23:37:49.704 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.704 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:49.704 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.704 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:49.704 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.665287+0000", 2024-09-12T23:37:49.704 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.665287+0000", 2024-09-12T23:37:49.704 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.665287+0000", 2024-09-12T23:37:49.704 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:49.705 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:49.705 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.705 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.665287+0000", 2024-09-12T23:37:49.705 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.665287+0000", 2024-09-12T23:37:49.705 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:49.705 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:49.705 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:49.705 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:49.705 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:49.705 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:49.705 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:49.706 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:49.706 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:49.706 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:49.706 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:49.706 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:49.706 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:49.706 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.2c", 2024-09-12T23:37:49.706 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:49.706 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:49.706 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:49.708 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:49.708 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:49.708 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T10:58:29.597052+0000", 2024-09-12T23:37:49.708 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:49.708 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:49.708 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:49.708 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:49.708 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:49.708 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:49.708 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:49.708 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:49.709 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:49.709 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:49.709 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:49.709 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:49.709 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:49.709 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:49.709 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:49.709 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:49.709 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:49.709 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:49.709 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:49.709 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:49.710 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:49.710 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:49.710 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:49.710 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:49.710 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:49.710 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:49.710 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:49.710 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:49.710 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:49.710 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:49.711 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:49.711 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:49.711 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:49.711 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:49.711 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:49.711 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:49.711 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:49.711 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:49.711 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:49.711 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:49.711 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:49.712 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:49.712 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:49.712 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.712 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:49.712 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:49.712 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:49.712 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:49.712 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:49.712 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:49.712 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.712 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 6, 2024-09-12T23:37:49.712 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:49.713 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.713 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.713 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:49.713 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:49.713 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:49.713 INFO:tasks.workunit.client.0.smithi136.stdout: 1 2024-09-12T23:37:49.713 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.713 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 0, 2024-09-12T23:37:49.713 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:49.713 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:49.713 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:49.714 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:49.714 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:49.714 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:49.714 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.669549+0000", 2024-09-12T23:37:49.714 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.917878+0000", 2024-09-12T23:37:49.714 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.917878+0000", 2024-09-12T23:37:49.714 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.918231+0000", 2024-09-12T23:37:49.714 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.669549+0000", 2024-09-12T23:37:49.714 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.714 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:49.714 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.715 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:49.715 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.669549+0000", 2024-09-12T23:37:49.715 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.669549+0000", 2024-09-12T23:37:49.715 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.669549+0000", 2024-09-12T23:37:49.715 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:49.715 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:49.715 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.715 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.669549+0000", 2024-09-12T23:37:49.715 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.669549+0000", 2024-09-12T23:37:49.715 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:49.715 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:49.716 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:49.716 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:49.716 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:49.716 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:49.716 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:49.716 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:49.716 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:49.716 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:49.716 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:49.716 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:49.716 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:49.716 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.2b", 2024-09-12T23:37:49.717 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:49.717 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:49.717 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:49.717 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:49.717 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:49.717 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T06:22:46.498222+0000", 2024-09-12T23:37:49.717 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:49.717 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:49.717 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:49.717 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:49.717 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:49.718 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:49.718 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:49.718 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:49.718 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:49.718 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:49.718 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:49.718 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:49.718 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:49.718 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:49.718 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:49.719 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:49.719 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:49.719 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:49.719 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:49.719 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:49.719 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:49.719 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:49.719 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:49.719 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:49.719 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:49.719 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:49.719 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:49.720 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:49.720 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:49.720 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:49.720 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:49.720 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:49.720 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:49.720 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:49.720 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:49.720 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:49.720 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:49.720 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:49.721 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:49.721 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:49.721 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:49.721 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:49.721 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:49.721 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.721 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:49.721 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:49.721 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:49.721 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:49.721 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:49.721 INFO:tasks.workunit.client.0.smithi136.stdout: 1 2024-09-12T23:37:49.722 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.722 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 0, 2024-09-12T23:37:49.722 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:49.722 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.722 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.722 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:49.722 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:49.722 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:49.722 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:49.722 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.723 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 5, 2024-09-12T23:37:49.723 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:49.723 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:49.723 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:49.723 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:49.723 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:49.723 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:49.723 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.671076+0000", 2024-09-12T23:37:49.723 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.909507+0000", 2024-09-12T23:37:49.723 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.909507+0000", 2024-09-12T23:37:49.723 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.909780+0000", 2024-09-12T23:37:49.723 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.671076+0000", 2024-09-12T23:37:49.724 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.724 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:49.724 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.724 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:49.724 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.671076+0000", 2024-09-12T23:37:49.724 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.671076+0000", 2024-09-12T23:37:49.724 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.671076+0000", 2024-09-12T23:37:49.724 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:49.724 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:49.724 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.724 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.671076+0000", 2024-09-12T23:37:49.725 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.671076+0000", 2024-09-12T23:37:49.725 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:49.725 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:49.725 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:49.725 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:49.725 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:49.725 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:49.725 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:49.725 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:49.725 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:49.725 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:49.726 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:49.726 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:49.726 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:49.726 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.2a", 2024-09-12T23:37:49.726 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:49.726 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:49.726 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:49.726 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:49.726 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:49.726 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T04:39:50.529545+0000", 2024-09-12T23:37:49.726 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:49.726 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:49.727 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:49.727 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:49.727 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:49.727 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:49.727 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:49.727 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:49.727 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:49.727 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:49.727 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:49.727 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:49.727 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:49.727 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:49.728 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:49.728 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:49.728 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:49.728 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:49.728 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:49.728 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:49.728 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:49.728 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:49.728 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:49.728 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:49.729 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:49.729 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:49.729 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:49.729 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:49.729 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:49.729 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:49.729 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:49.729 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:49.729 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:49.729 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:49.730 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:49.730 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:49.730 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:49.730 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:49.730 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:49.730 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:49.730 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:49.730 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:49.730 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:49.730 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.730 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:49.731 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:49.731 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:49.731 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:49.731 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:49.731 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:49.731 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.731 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 5, 2024-09-12T23:37:49.731 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:49.731 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.731 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.731 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:49.731 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:49.732 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:49.732 INFO:tasks.workunit.client.0.smithi136.stdout: 1 2024-09-12T23:37:49.732 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.732 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 3, 2024-09-12T23:37:49.732 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:49.732 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:49.732 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:49.732 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:49.732 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:49.732 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:49.732 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.667544+0000", 2024-09-12T23:37:49.733 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.905780+0000", 2024-09-12T23:37:49.733 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.905780+0000", 2024-09-12T23:37:49.733 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.906087+0000", 2024-09-12T23:37:49.733 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.667544+0000", 2024-09-12T23:37:49.733 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.733 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:49.733 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.733 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:49.733 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.667544+0000", 2024-09-12T23:37:49.733 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.667544+0000", 2024-09-12T23:37:49.733 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.667544+0000", 2024-09-12T23:37:49.734 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:49.734 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:49.734 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.734 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.667544+0000", 2024-09-12T23:37:49.734 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.667544+0000", 2024-09-12T23:37:49.734 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:49.734 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:49.734 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:49.734 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:49.734 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:49.734 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:49.735 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:49.735 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:49.735 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:49.735 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:49.735 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:49.735 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:49.735 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:49.735 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.29", 2024-09-12T23:37:49.735 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:49.735 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:49.735 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:49.735 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:49.736 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:49.736 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T07:00:40.777941+0000", 2024-09-12T23:37:49.736 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:49.736 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:49.736 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:49.736 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:49.736 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:49.736 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:49.736 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:49.736 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:49.737 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:49.737 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:49.737 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:49.737 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:49.737 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:49.737 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:49.737 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:49.737 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:49.737 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:49.737 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:49.737 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:49.737 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:49.738 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:49.738 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:49.738 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:49.738 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:49.738 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:49.738 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:49.738 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:49.738 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:49.738 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:49.738 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:49.738 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:49.738 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:49.739 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:49.739 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:49.739 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:49.739 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:49.739 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:49.739 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:49.739 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:49.739 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:49.739 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:49.739 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:49.740 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:49.740 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.740 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:49.740 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:49.740 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:49.740 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:49.740 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:49.740 INFO:tasks.workunit.client.0.smithi136.stdout: 1 2024-09-12T23:37:49.740 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.740 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 3, 2024-09-12T23:37:49.740 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:49.740 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.741 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.741 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:49.741 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:49.741 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:49.741 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:49.741 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.741 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 5, 2024-09-12T23:37:49.741 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:49.741 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:49.741 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:49.742 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:49.742 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:49.742 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:49.742 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.670507+0000", 2024-09-12T23:37:49.742 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.917757+0000", 2024-09-12T23:37:49.742 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.917757+0000", 2024-09-12T23:37:49.742 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.918286+0000", 2024-09-12T23:37:49.742 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.670507+0000", 2024-09-12T23:37:49.742 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.742 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:49.742 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.743 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:49.743 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.670507+0000", 2024-09-12T23:37:49.743 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.670507+0000", 2024-09-12T23:37:49.743 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.670507+0000", 2024-09-12T23:37:49.743 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:49.743 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:49.743 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.743 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.670507+0000", 2024-09-12T23:37:49.743 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.670507+0000", 2024-09-12T23:37:49.743 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:49.743 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:49.744 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:49.744 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:49.744 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:49.744 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:49.744 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:49.744 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:49.744 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:49.744 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:49.744 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:49.744 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:49.745 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:49.745 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.28", 2024-09-12T23:37:49.745 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:49.745 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:49.745 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:49.745 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:49.745 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:49.745 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T06:49:26.856852+0000", 2024-09-12T23:37:49.745 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:49.745 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:49.745 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:49.746 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:49.746 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:49.746 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:49.746 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:49.746 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:49.746 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:49.746 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:49.746 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:49.746 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:49.746 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:49.746 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:49.747 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:49.747 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:49.747 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:49.747 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:49.747 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:49.747 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:49.747 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:49.747 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:49.747 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:49.747 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:49.747 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:49.748 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:49.748 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:49.748 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:49.748 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:49.748 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:49.748 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:49.748 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:49.748 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:49.748 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:49.748 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:49.748 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:49.749 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:49.749 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:49.749 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:49.749 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:49.749 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:49.749 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:49.749 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:49.749 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.749 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:49.749 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:49.749 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:49.750 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:49.750 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:49.750 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:49.750 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.750 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 5, 2024-09-12T23:37:49.750 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:49.750 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.750 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.750 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:49.750 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:49.750 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:49.751 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:49.751 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.751 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 3, 2024-09-12T23:37:49.751 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:49.751 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:49.751 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:49.751 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:49.751 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:49.751 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:49.751 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.673387+0000", 2024-09-12T23:37:49.751 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.903081+0000", 2024-09-12T23:37:49.752 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.903081+0000", 2024-09-12T23:37:49.752 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.903381+0000", 2024-09-12T23:37:49.752 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.673387+0000", 2024-09-12T23:37:49.752 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.752 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:49.752 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.752 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:49.752 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.673387+0000", 2024-09-12T23:37:49.752 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.673387+0000", 2024-09-12T23:37:49.752 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.673387+0000", 2024-09-12T23:37:49.752 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:49.753 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:49.753 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.753 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.673387+0000", 2024-09-12T23:37:49.753 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.673387+0000", 2024-09-12T23:37:49.753 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:49.753 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:49.753 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:49.753 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:49.753 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:49.753 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:49.753 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:49.754 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:49.754 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:49.754 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:49.754 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:49.754 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:49.754 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:49.754 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.27", 2024-09-12T23:37:49.754 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:49.754 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:49.754 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:49.754 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:49.755 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:49.755 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T00:43:01.175691+0000", 2024-09-12T23:37:49.755 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:49.755 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:49.755 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:49.755 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:49.755 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:49.755 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:49.755 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:49.755 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:49.755 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:49.756 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:49.756 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:49.756 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:49.756 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:49.756 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:49.756 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:49.756 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:49.756 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:49.756 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:49.756 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:49.757 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:49.757 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:49.757 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:49.757 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:49.757 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:49.757 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:49.757 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:49.757 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:49.757 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:49.757 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:49.758 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:49.758 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:49.758 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:49.758 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:49.758 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:49.758 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:49.758 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:49.758 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:49.758 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:49.758 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:49.759 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:49.759 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:49.759 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:49.759 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:49.759 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.759 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:49.759 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:49.759 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:49.759 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:49.759 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:49.759 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:49.760 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.760 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 3, 2024-09-12T23:37:49.760 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:49.760 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.760 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.760 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:49.760 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:49.760 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:49.760 INFO:tasks.workunit.client.0.smithi136.stdout: 4 2024-09-12T23:37:49.760 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.760 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 6, 2024-09-12T23:37:49.761 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:49.761 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:49.761 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:49.761 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:49.761 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:49.761 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:49.761 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.666859+0000", 2024-09-12T23:37:49.761 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.896834+0000", 2024-09-12T23:37:49.761 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.896834+0000", 2024-09-12T23:37:49.761 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.897170+0000", 2024-09-12T23:37:49.762 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.666859+0000", 2024-09-12T23:37:49.762 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.762 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:49.762 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.762 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:49.762 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.666859+0000", 2024-09-12T23:37:49.762 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.666859+0000", 2024-09-12T23:37:49.762 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.666859+0000", 2024-09-12T23:37:49.762 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:49.762 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:49.762 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.763 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.666859+0000", 2024-09-12T23:37:49.763 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.666859+0000", 2024-09-12T23:37:49.763 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:49.763 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:49.763 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:49.763 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:49.763 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:49.763 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:49.763 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:49.763 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:49.764 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:49.764 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:49.764 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:49.764 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:49.764 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:49.764 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.26", 2024-09-12T23:37:49.764 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:49.764 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:49.764 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:49.764 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:49.764 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:49.765 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T00:47:43.534530+0000", 2024-09-12T23:37:49.765 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:49.765 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:49.765 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:49.765 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:49.765 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:49.765 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:49.765 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:49.765 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:49.765 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:49.766 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:49.766 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:49.766 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:49.766 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:49.766 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:49.766 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:49.766 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:49.766 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:49.766 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:49.766 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:49.767 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:49.767 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:49.767 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:49.767 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:49.767 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:49.767 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:49.767 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:49.767 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:49.767 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:49.767 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:49.767 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:49.767 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:49.768 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:49.768 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:49.768 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:49.768 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:49.768 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:49.768 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:49.768 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:49.768 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:49.768 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:49.768 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:49.769 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:49.769 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:49.769 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.769 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:49.769 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:49.769 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:49.769 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:49.769 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:49.769 INFO:tasks.workunit.client.0.smithi136.stdout: 4 2024-09-12T23:37:49.769 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.770 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 6, 2024-09-12T23:37:49.770 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:49.770 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.770 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.770 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:49.770 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:49.770 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:49.770 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:49.770 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.770 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 1, 2024-09-12T23:37:49.770 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:49.771 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:49.771 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:49.771 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:49.771 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:49.771 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:49.771 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.663571+0000", 2024-09-12T23:37:49.771 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.919185+0000", 2024-09-12T23:37:49.771 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.919185+0000", 2024-09-12T23:37:49.771 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.919500+0000", 2024-09-12T23:37:49.771 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.663571+0000", 2024-09-12T23:37:49.771 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.771 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:49.772 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.772 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:49.772 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.663571+0000", 2024-09-12T23:37:49.772 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.663571+0000", 2024-09-12T23:37:49.772 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.663571+0000", 2024-09-12T23:37:49.772 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:49.772 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:49.772 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.772 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.663571+0000", 2024-09-12T23:37:49.772 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.663571+0000", 2024-09-12T23:37:49.772 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:49.773 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:49.773 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:49.773 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:49.773 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:49.773 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:49.773 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:49.773 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:49.773 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:49.773 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:49.773 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:49.773 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:49.774 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:49.774 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.25", 2024-09-12T23:37:49.774 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:49.774 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:49.774 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:49.774 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:49.774 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:49.774 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T01:56:52.566524+0000", 2024-09-12T23:37:49.774 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:49.774 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:49.774 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:49.774 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:49.775 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:49.775 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:49.775 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:49.775 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:49.775 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:49.775 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:49.775 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:49.775 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:49.776 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:49.776 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:49.776 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:49.776 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:49.776 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:49.776 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:49.776 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:49.776 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:49.776 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:49.776 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:49.776 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:49.777 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:49.777 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:49.777 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:49.777 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:49.777 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:49.777 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:49.777 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:49.777 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:49.777 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:49.777 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:49.777 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:49.778 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:49.778 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:49.778 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:49.778 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:49.778 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:49.778 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:49.778 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:49.778 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:49.778 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:49.778 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.778 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:49.778 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:49.779 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:49.779 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:49.779 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:49.779 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:49.779 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.779 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 1, 2024-09-12T23:37:49.779 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:49.779 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.779 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.779 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:49.780 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:49.780 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:49.780 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:49.780 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.780 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 3, 2024-09-12T23:37:49.780 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:49.780 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:49.780 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:49.780 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:49.780 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:49.781 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:49.781 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.672875+0000", 2024-09-12T23:37:49.781 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.913193+0000", 2024-09-12T23:37:49.781 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.913193+0000", 2024-09-12T23:37:49.781 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.913907+0000", 2024-09-12T23:37:49.781 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.672875+0000", 2024-09-12T23:37:49.781 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.781 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:49.781 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.781 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:49.781 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.672875+0000", 2024-09-12T23:37:49.782 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.672875+0000", 2024-09-12T23:37:49.782 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.672875+0000", 2024-09-12T23:37:49.782 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:49.782 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:49.782 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.782 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.672875+0000", 2024-09-12T23:37:49.782 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.672875+0000", 2024-09-12T23:37:49.782 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:49.782 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:49.782 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:49.782 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:49.783 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:49.783 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:49.783 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:49.783 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:49.783 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:49.783 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:49.783 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:49.783 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:49.783 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:49.783 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.24", 2024-09-12T23:37:49.783 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:49.783 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:49.784 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:49.784 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:49.784 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:49.784 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T10:32:45.117056+0000", 2024-09-12T23:37:49.784 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:49.784 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:49.784 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:49.784 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:49.784 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:49.784 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:49.784 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:49.785 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:49.785 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:49.785 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:49.785 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:49.785 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:49.785 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:49.785 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:49.785 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:49.785 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:49.785 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:49.785 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:49.785 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:49.786 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:49.786 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:49.786 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:49.786 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:49.786 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:49.786 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:49.786 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:49.786 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:49.786 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:49.786 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:49.786 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:49.786 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:49.787 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:49.787 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:49.787 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:49.787 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:49.787 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:49.787 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:49.787 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:49.787 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:49.787 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:49.787 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:49.787 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:49.787 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:49.788 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.788 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:49.788 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:49.788 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:49.788 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:49.788 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:49.788 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:49.788 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.788 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 3, 2024-09-12T23:37:49.788 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:49.788 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.788 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.789 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:49.789 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:49.789 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:49.789 INFO:tasks.workunit.client.0.smithi136.stdout: 4 2024-09-12T23:37:49.789 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.789 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 0, 2024-09-12T23:37:49.789 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:49.789 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:49.789 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:49.790 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:49.790 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:49.790 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:49.790 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.669650+0000", 2024-09-12T23:37:49.790 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.911007+0000", 2024-09-12T23:37:49.790 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.911007+0000", 2024-09-12T23:37:49.790 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.911279+0000", 2024-09-12T23:37:49.790 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.669650+0000", 2024-09-12T23:37:49.790 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.790 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:49.790 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.790 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:49.791 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.669650+0000", 2024-09-12T23:37:49.791 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.669650+0000", 2024-09-12T23:37:49.791 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.669650+0000", 2024-09-12T23:37:49.791 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:49.791 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:49.791 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.791 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.669650+0000", 2024-09-12T23:37:49.791 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.669650+0000", 2024-09-12T23:37:49.791 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:49.791 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:49.791 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:49.792 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:49.792 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:49.792 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:49.792 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:49.792 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:49.792 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:49.792 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:49.792 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:49.792 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:49.792 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:49.792 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.23", 2024-09-12T23:37:49.793 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:49.793 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:49.793 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:49.793 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:49.793 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:49.793 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T06:26:51.842858+0000", 2024-09-12T23:37:49.793 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:49.793 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:49.793 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:49.793 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:49.793 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:49.794 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:49.794 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:49.794 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:49.794 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:49.794 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:49.794 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:49.794 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:49.794 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:49.794 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:49.794 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:49.794 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:49.795 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:49.795 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:49.795 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:49.795 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:49.795 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:49.795 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:49.795 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:49.795 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:49.795 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:49.795 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:49.795 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:49.796 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:49.796 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:49.796 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:49.796 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:49.796 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:49.796 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:49.796 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:49.796 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:49.796 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:49.796 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:49.796 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:49.797 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:49.797 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:49.797 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:49.797 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:49.797 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:49.797 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.797 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:49.797 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:49.797 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:49.797 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:49.797 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:49.798 INFO:tasks.workunit.client.0.smithi136.stdout: 4 2024-09-12T23:37:49.798 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.798 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 0, 2024-09-12T23:37:49.798 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:49.798 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.798 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.798 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:49.798 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:49.798 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:49.798 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:49.799 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.799 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 5, 2024-09-12T23:37:49.799 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:49.799 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:49.799 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:49.799 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:49.799 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:49.799 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:49.799 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.671116+0000", 2024-09-12T23:37:49.799 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.913659+0000", 2024-09-12T23:37:49.799 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.913659+0000", 2024-09-12T23:37:49.800 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.914015+0000", 2024-09-12T23:37:49.800 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.671116+0000", 2024-09-12T23:37:49.800 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.800 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:49.800 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.800 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:49.800 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.671116+0000", 2024-09-12T23:37:49.800 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.671116+0000", 2024-09-12T23:37:49.800 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.671116+0000", 2024-09-12T23:37:49.800 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:49.800 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:49.801 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.801 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.671116+0000", 2024-09-12T23:37:49.801 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.671116+0000", 2024-09-12T23:37:49.801 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:49.801 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:49.801 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:49.801 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:49.801 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:49.801 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:49.801 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:49.801 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:49.802 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:49.802 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:49.802 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:49.802 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:49.802 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:49.802 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.22", 2024-09-12T23:37:49.802 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:49.802 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:49.802 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:49.802 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:49.802 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:49.802 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T03:37:02.989629+0000", 2024-09-12T23:37:49.803 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:49.803 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:49.803 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:49.803 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:49.803 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:49.803 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:49.803 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:49.803 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:49.803 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:49.803 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:49.803 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:49.803 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:49.804 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:49.804 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:49.804 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:49.804 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:49.804 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:49.804 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:49.804 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:49.804 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:49.804 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:49.804 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:49.804 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:49.805 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:49.805 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:49.805 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:49.805 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:49.805 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:49.805 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:49.805 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:49.805 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:49.805 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:49.805 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:49.805 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:49.805 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:49.806 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:49.806 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:49.806 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:49.806 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:49.806 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:49.806 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:49.806 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:49.806 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:49.806 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.806 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:49.806 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:49.807 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:49.807 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:49.807 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:49.807 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:49.807 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.807 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 5, 2024-09-12T23:37:49.807 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:49.807 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.807 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.807 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:49.807 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:49.807 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:49.808 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:49.808 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.808 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 2, 2024-09-12T23:37:49.808 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:49.808 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:49.808 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:49.808 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:49.808 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:49.808 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:49.808 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.661319+0000", 2024-09-12T23:37:49.808 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.889548+0000", 2024-09-12T23:37:49.808 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.889548+0000", 2024-09-12T23:37:49.809 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.889782+0000", 2024-09-12T23:37:49.809 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.661319+0000", 2024-09-12T23:37:49.809 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.809 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:49.809 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.809 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:49.809 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.661319+0000", 2024-09-12T23:37:49.809 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.661319+0000", 2024-09-12T23:37:49.809 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.661319+0000", 2024-09-12T23:37:49.809 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:49.809 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:49.810 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.810 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.661319+0000", 2024-09-12T23:37:49.810 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.661319+0000", 2024-09-12T23:37:49.810 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:49.810 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:49.810 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:49.810 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:49.810 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:49.810 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:49.810 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:49.810 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:49.811 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:49.811 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:49.811 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:49.811 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:49.811 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:49.811 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.21", 2024-09-12T23:37:49.811 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:49.811 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:49.811 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:49.811 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:49.811 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:49.811 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-13T23:54:53.906619+0000", 2024-09-12T23:37:49.812 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:49.812 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:49.812 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:49.812 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:49.812 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:49.812 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:49.812 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:49.812 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:49.812 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:49.812 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:49.812 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:49.812 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:49.813 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:49.813 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:49.813 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:49.813 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:49.813 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:49.813 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:49.813 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:49.813 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:49.813 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:49.813 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:49.814 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:49.814 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:49.814 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:49.814 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:49.814 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:49.814 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:49.814 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:49.814 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:49.814 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:49.814 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:49.814 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:49.815 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:49.815 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:49.815 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:49.815 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:49.815 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:49.815 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:49.815 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:49.815 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:49.815 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:49.815 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:49.815 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.816 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:49.816 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:49.816 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:49.816 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:49.816 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:49.816 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:49.816 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.816 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 2, 2024-09-12T23:37:49.816 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:49.816 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.816 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.817 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:49.817 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:49.817 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:49.817 INFO:tasks.workunit.client.0.smithi136.stdout: 5 2024-09-12T23:37:49.817 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.817 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 6, 2024-09-12T23:37:49.817 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:49.817 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:49.817 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:49.817 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:49.817 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:49.818 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:49.818 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.665474+0000", 2024-09-12T23:37:49.818 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.895919+0000", 2024-09-12T23:37:49.818 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.895919+0000", 2024-09-12T23:37:49.818 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.896515+0000", 2024-09-12T23:37:49.818 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.665474+0000", 2024-09-12T23:37:49.818 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.818 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:49.818 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.818 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:49.818 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.665474+0000", 2024-09-12T23:37:49.819 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.665474+0000", 2024-09-12T23:37:49.819 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.665474+0000", 2024-09-12T23:37:49.819 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:49.819 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:49.819 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.819 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.665474+0000", 2024-09-12T23:37:49.819 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.665474+0000", 2024-09-12T23:37:49.819 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:49.819 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:49.819 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:49.819 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:49.819 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:49.820 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:49.820 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:49.820 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:49.820 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:49.820 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:49.820 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:49.820 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:49.820 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:49.820 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.20", 2024-09-12T23:37:49.820 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:49.820 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:49.820 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:49.821 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:49.821 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:49.821 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T05:17:53.675501+0000", 2024-09-12T23:37:49.821 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:49.821 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:49.821 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:49.821 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:49.821 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:49.821 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:49.821 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:49.821 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:49.821 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:49.822 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:49.822 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:49.822 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:49.822 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:49.822 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:49.822 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:49.822 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:49.822 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:49.822 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:49.822 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:49.822 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:49.822 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:49.823 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:49.823 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:49.823 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:49.823 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:49.823 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:49.823 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:49.823 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:49.823 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:49.823 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:49.823 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:49.823 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:49.823 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:49.824 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:49.824 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:49.824 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:49.824 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:49.824 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:49.824 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:49.824 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:49.824 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:49.824 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:49.824 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:49.824 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.825 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:49.825 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:49.825 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:49.825 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:49.825 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:49.825 INFO:tasks.workunit.client.0.smithi136.stdout: 5 2024-09-12T23:37:49.825 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.825 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 6, 2024-09-12T23:37:49.825 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:49.825 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.825 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.826 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:49.826 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:49.826 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:49.826 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:49.826 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.826 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 4, 2024-09-12T23:37:49.826 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:49.826 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:49.826 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:49.826 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:49.826 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:49.826 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:49.827 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.660905+0000", 2024-09-12T23:37:49.827 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.890839+0000", 2024-09-12T23:37:49.827 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.890839+0000", 2024-09-12T23:37:49.827 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.891596+0000", 2024-09-12T23:37:49.827 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.660905+0000", 2024-09-12T23:37:49.827 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.827 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:49.827 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.827 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:49.827 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.660905+0000", 2024-09-12T23:37:49.827 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.660905+0000", 2024-09-12T23:37:49.827 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.660905+0000", 2024-09-12T23:37:49.828 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:49.828 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:49.828 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.828 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.660905+0000", 2024-09-12T23:37:49.828 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.660905+0000", 2024-09-12T23:37:49.828 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:49.828 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:49.828 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:49.828 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:49.828 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:49.828 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:49.829 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:49.829 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:49.829 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:49.829 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:49.829 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:49.829 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:49.829 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:49.829 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.1a", 2024-09-12T23:37:49.829 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:49.829 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:49.829 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:49.829 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:49.830 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:49.830 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T09:29:49.995950+0000", 2024-09-12T23:37:49.830 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:49.830 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:49.830 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:49.830 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:49.830 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:49.830 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:49.830 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:49.830 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:49.830 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:49.830 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:49.830 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:49.831 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:49.831 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:49.831 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:49.831 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:49.831 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:49.831 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:49.831 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:49.831 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:49.831 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:49.831 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:49.831 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:49.832 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:49.832 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:49.832 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:49.832 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:49.832 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:49.832 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:49.832 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:49.832 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:49.832 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:49.832 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:49.832 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:49.832 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:49.833 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:49.833 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:49.833 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:49.833 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:49.833 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:49.833 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:49.833 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:49.833 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:49.833 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:49.833 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.834 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:49.834 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:49.834 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:49.834 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:49.834 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:49.834 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:49.834 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.834 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 4, 2024-09-12T23:37:49.834 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:49.834 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.834 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.834 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:49.835 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:49.835 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:49.835 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:49.835 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.835 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 3, 2024-09-12T23:37:49.835 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:49.835 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:49.835 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:49.835 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:49.835 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:49.835 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:49.835 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.668622+0000", 2024-09-12T23:37:49.836 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.882848+0000", 2024-09-12T23:37:49.836 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.882848+0000", 2024-09-12T23:37:49.836 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.883222+0000", 2024-09-12T23:37:49.836 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.668622+0000", 2024-09-12T23:37:49.836 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:49.836 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:49.836 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:49.836 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:49.836 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.668622+0000", 2024-09-12T23:37:49.836 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.668622+0000", 2024-09-12T23:37:49.836 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.668622+0000", 2024-09-12T23:37:49.837 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:49.837 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:49.837 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:49.837 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.668622+0000", 2024-09-12T23:37:49.837 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.668622+0000", 2024-09-12T23:37:49.837 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:49.837 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 1, 2024-09-12T23:37:49.837 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:49.837 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:49.837 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:49.837 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:49.838 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:49.838 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:49.838 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:49.838 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 1, 2024-09-12T23:37:49.838 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:49.838 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:49.838 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:49.838 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.1b", 2024-09-12T23:37:49.838 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:49.838 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:49.838 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:49.839 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 22, 2024-09-12T23:37:49.839 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:49.839 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T02:57:33.905996+0000", 2024-09-12T23:37:49.839 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:49.839 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:49.839 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:49.839 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:49.839 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:49.839 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:49.839 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:49.839 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:49.839 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:49.840 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:49.840 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:49.840 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:49.840 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:49.840 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:49.840 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:49.840 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:49.840 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:49.840 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:49.840 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:49.840 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:49.841 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:49.841 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:49.841 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:49.841 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:49.841 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:49.841 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:49.841 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:49.841 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:49.841 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:49.841 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:49.841 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:49.841 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:49.842 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:49.842 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:49.842 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:49.842 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:49.842 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:49.842 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:49.842 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:49.842 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:49.842 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:49.842 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:49.842 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:49.842 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.843 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:49.843 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:49.843 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:49.843 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:49.843 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:49.843 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:49.843 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.843 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 3, 2024-09-12T23:37:49.843 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "53'1" 2024-09-12T23:37:49.843 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.843 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.844 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:49.844 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:49.844 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:49.844 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:49.844 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.844 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 0, 2024-09-12T23:37:49.844 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:49.844 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:49.844 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:49.844 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:49.845 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:49.845 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:49.845 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.669749+0000", 2024-09-12T23:37:49.845 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.912406+0000", 2024-09-12T23:37:49.845 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.912406+0000", 2024-09-12T23:37:49.845 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.912763+0000", 2024-09-12T23:37:49.845 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.669749+0000", 2024-09-12T23:37:49.845 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.845 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:49.845 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.846 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:49.846 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.669749+0000", 2024-09-12T23:37:49.846 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.669749+0000", 2024-09-12T23:37:49.846 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.669749+0000", 2024-09-12T23:37:49.846 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:49.846 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:49.846 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.846 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.669749+0000", 2024-09-12T23:37:49.846 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.669749+0000", 2024-09-12T23:37:49.846 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:49.846 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:49.846 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:49.847 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:49.847 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:49.847 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:49.847 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:49.847 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:49.847 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:49.847 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:49.847 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:49.847 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:49.847 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:49.847 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.1b", 2024-09-12T23:37:49.847 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:49.848 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:49.848 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:49.848 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:49.848 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:49.848 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T01:49:06.828514+0000", 2024-09-12T23:37:49.848 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:49.848 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:49.848 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:49.848 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:49.848 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:49.848 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:49.849 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:49.849 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:49.849 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:49.849 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:49.849 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:49.849 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:49.849 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:49.849 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:49.849 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:49.849 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:49.849 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:49.850 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:49.850 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:49.850 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:49.850 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:49.850 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:49.850 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:49.850 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:49.850 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:49.850 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:49.850 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:49.850 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:49.851 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:49.851 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:49.851 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:49.851 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:49.851 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:49.851 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:49.851 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:49.851 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:49.851 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:49.851 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:49.851 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:49.851 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:49.852 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:49.852 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:49.852 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:49.852 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.852 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:49.852 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:49.852 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:49.852 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:49.852 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:49.852 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:49.852 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.853 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 0, 2024-09-12T23:37:49.853 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:49.853 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.853 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.853 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:49.853 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:49.853 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:49.853 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:49.853 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.853 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 6, 2024-09-12T23:37:49.854 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:49.854 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:49.854 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:49.854 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:49.854 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:49.854 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:49.854 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.667384+0000", 2024-09-12T23:37:49.854 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.884139+0000", 2024-09-12T23:37:49.854 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.884139+0000", 2024-09-12T23:37:49.854 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.884599+0000", 2024-09-12T23:37:49.854 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.667384+0000", 2024-09-12T23:37:49.855 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:49.855 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:49.855 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:49.855 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:49.855 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.667384+0000", 2024-09-12T23:37:49.855 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.667384+0000", 2024-09-12T23:37:49.855 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.667384+0000", 2024-09-12T23:37:49.855 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:49.855 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:49.855 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:49.856 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.667384+0000", 2024-09-12T23:37:49.856 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.667384+0000", 2024-09-12T23:37:49.856 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:49.856 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 1, 2024-09-12T23:37:49.856 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:49.856 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:49.856 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:49.856 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:49.856 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:49.857 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:49.857 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:49.857 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 1, 2024-09-12T23:37:49.857 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:49.857 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:49.857 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:49.857 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.1a", 2024-09-12T23:37:49.857 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:49.857 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:49.857 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:49.858 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 22, 2024-09-12T23:37:49.858 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:49.858 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T04:57:50.175287+0000", 2024-09-12T23:37:49.858 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:49.858 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:49.858 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:49.858 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:49.858 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:49.858 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:49.858 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:49.858 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:49.859 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:49.859 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:49.859 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:49.859 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:49.859 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:49.859 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:49.859 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:49.859 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:49.859 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:49.859 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:49.859 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:49.860 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:49.860 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:49.860 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:49.860 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:49.860 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:49.860 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:49.860 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:49.860 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:49.860 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:49.860 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:49.861 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:49.861 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:49.861 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:49.861 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:49.861 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:49.861 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:49.861 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:49.861 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:49.861 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:49.861 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:49.862 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:49.862 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:49.862 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:49.862 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:49.862 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.862 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:49.862 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:49.862 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:49.862 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:49.862 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:49.862 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:49.862 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.863 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 6, 2024-09-12T23:37:49.863 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "53'1" 2024-09-12T23:37:49.863 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.863 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.863 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:49.863 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:49.863 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:49.863 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:49.863 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.863 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 3, 2024-09-12T23:37:49.863 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:49.864 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:49.864 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:49.864 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:49.864 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:49.864 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:49.864 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.668064+0000", 2024-09-12T23:37:49.864 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.872404+0000", 2024-09-12T23:37:49.864 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.872404+0000", 2024-09-12T23:37:49.864 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.872822+0000", 2024-09-12T23:37:49.864 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.668064+0000", 2024-09-12T23:37:49.864 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:49.865 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:49.865 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:49.865 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:49.865 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.668064+0000", 2024-09-12T23:37:49.865 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.668064+0000", 2024-09-12T23:37:49.865 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.668064+0000", 2024-09-12T23:37:49.865 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:49.865 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:49.865 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:49.865 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.668064+0000", 2024-09-12T23:37:49.865 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.668064+0000", 2024-09-12T23:37:49.866 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:49.866 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:49.866 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:49.866 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:49.866 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:49.866 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:49.866 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:49.866 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:49.866 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:49.866 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:49.866 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:49.866 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:49.867 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:49.867 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.19", 2024-09-12T23:37:49.867 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:49.867 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:49.867 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:49.867 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 21, 2024-09-12T23:37:49.867 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:49.867 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-13T23:47:42.317408+0000", 2024-09-12T23:37:49.867 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:49.867 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:49.867 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:49.867 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:49.868 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:49.868 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:49.868 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:49.868 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:49.868 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:49.868 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:49.868 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:49.868 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:49.868 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:49.868 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:49.868 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:49.869 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:49.869 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:49.869 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:49.869 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:49.869 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:49.869 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:49.869 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:49.869 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:49.869 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:49.869 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:49.869 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:49.870 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:49.870 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:49.870 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:49.870 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:49.870 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:49.870 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:49.870 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:49.870 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:49.870 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:49.870 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:49.870 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:49.871 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:49.871 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:49.871 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:49.871 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:49.871 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:49.871 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:49.871 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.871 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:49.871 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:49.871 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:49.871 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:49.872 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:49.872 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:49.872 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.872 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 3, 2024-09-12T23:37:49.872 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:49.872 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.872 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.872 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:49.872 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:49.872 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:49.872 INFO:tasks.workunit.client.0.smithi136.stdout: 1 2024-09-12T23:37:49.873 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.873 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 3, 2024-09-12T23:37:49.873 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:49.873 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:49.873 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:49.873 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:49.873 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:49.873 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:49.873 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.667077+0000", 2024-09-12T23:37:49.873 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.907067+0000", 2024-09-12T23:37:49.873 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.907067+0000", 2024-09-12T23:37:49.874 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.907360+0000", 2024-09-12T23:37:49.874 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.667077+0000", 2024-09-12T23:37:49.874 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.874 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:49.874 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.874 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:49.874 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.667077+0000", 2024-09-12T23:37:49.874 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.667077+0000", 2024-09-12T23:37:49.874 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.667077+0000", 2024-09-12T23:37:49.874 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:49.875 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:49.875 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.875 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.667077+0000", 2024-09-12T23:37:49.875 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.667077+0000", 2024-09-12T23:37:49.875 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:49.875 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:49.875 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:49.875 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:49.875 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:49.875 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:49.875 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:49.876 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:49.876 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:49.876 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:49.876 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:49.876 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:49.876 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:49.876 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.18", 2024-09-12T23:37:49.876 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:49.876 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:49.876 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:49.876 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:49.877 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:49.877 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T11:26:38.084938+0000", 2024-09-12T23:37:49.877 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:49.877 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:49.877 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:49.877 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:49.877 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:49.877 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:49.877 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:49.877 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:49.878 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:49.878 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:49.878 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:49.878 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:49.878 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:49.878 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:49.878 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:49.878 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:49.878 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:49.878 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:49.878 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:49.878 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:49.879 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:49.879 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:49.879 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:49.879 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:49.879 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:49.879 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:49.879 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:49.879 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:49.879 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:49.879 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:49.879 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:49.879 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:49.880 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:49.880 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:49.880 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:49.880 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:49.880 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:49.880 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:49.880 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:49.880 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:49.880 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:49.880 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:49.880 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:49.881 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.881 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:49.881 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:49.881 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:49.881 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:49.881 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:49.881 INFO:tasks.workunit.client.0.smithi136.stdout: 1 2024-09-12T23:37:49.881 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.881 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 3, 2024-09-12T23:37:49.881 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:49.881 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.882 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.882 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:49.882 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:49.882 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:49.882 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:49.882 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.882 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 5, 2024-09-12T23:37:49.882 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:49.882 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:49.882 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:49.882 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:49.883 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:49.883 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:49.883 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.670406+0000", 2024-09-12T23:37:49.883 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.896167+0000", 2024-09-12T23:37:49.883 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.896167+0000", 2024-09-12T23:37:49.883 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.896704+0000", 2024-09-12T23:37:49.883 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.670406+0000", 2024-09-12T23:37:49.883 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:49.883 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:49.883 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:49.883 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:49.883 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.670406+0000", 2024-09-12T23:37:49.884 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.670406+0000", 2024-09-12T23:37:49.884 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.670406+0000", 2024-09-12T23:37:49.884 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:49.884 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:49.884 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:49.884 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.670406+0000", 2024-09-12T23:37:49.884 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.670406+0000", 2024-09-12T23:37:49.884 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:49.884 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:49.884 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:49.884 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:49.885 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:49.885 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:49.885 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:49.885 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:49.885 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:49.885 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:49.885 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:49.885 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:49.885 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:49.885 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.18", 2024-09-12T23:37:49.885 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:49.886 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:49.886 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:49.886 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 21, 2024-09-12T23:37:49.886 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:49.886 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T09:14:47.959501+0000", 2024-09-12T23:37:49.886 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:49.886 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:49.886 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:49.886 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:49.886 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:49.886 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:49.886 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:49.887 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:49.887 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:49.887 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:49.887 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:49.887 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:49.887 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:49.887 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:49.887 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:49.887 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:49.887 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:49.887 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:49.887 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:49.888 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:49.888 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:49.888 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:49.888 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:49.888 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:49.888 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:49.888 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:49.888 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:49.888 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:49.888 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:49.888 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:49.889 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:49.889 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:49.889 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:49.889 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:49.889 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:49.889 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:49.889 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:49.889 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:49.889 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:49.889 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:49.889 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:49.889 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:49.889 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:49.890 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.890 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:49.890 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:49.890 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:49.890 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:49.890 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:49.890 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:49.890 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.890 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 5, 2024-09-12T23:37:49.890 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:49.890 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.890 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.891 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:49.891 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:49.891 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:49.891 INFO:tasks.workunit.client.0.smithi136.stdout: 4 2024-09-12T23:37:49.891 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.891 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 1, 2024-09-12T23:37:49.891 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:49.891 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:49.891 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:49.891 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:49.891 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:49.891 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:49.892 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.663618+0000", 2024-09-12T23:37:49.892 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.892988+0000", 2024-09-12T23:37:49.892 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.892988+0000", 2024-09-12T23:37:49.892 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.893401+0000", 2024-09-12T23:37:49.892 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.663618+0000", 2024-09-12T23:37:49.892 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.892 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:49.892 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.892 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:49.892 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.663618+0000", 2024-09-12T23:37:49.892 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.663618+0000", 2024-09-12T23:37:49.892 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.663618+0000", 2024-09-12T23:37:49.893 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:49.893 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:49.893 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.893 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.663618+0000", 2024-09-12T23:37:49.893 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.663618+0000", 2024-09-12T23:37:49.893 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:49.893 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:49.893 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:49.893 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:49.893 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:49.893 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:49.893 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:49.894 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:49.894 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:49.894 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:49.894 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:49.894 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:49.894 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:49.894 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.19", 2024-09-12T23:37:49.894 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:49.894 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:49.894 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:49.894 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:49.894 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:49.895 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T07:30:04.271132+0000", 2024-09-12T23:37:49.895 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:49.895 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:49.895 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:49.895 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:49.895 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:49.895 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:49.895 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:49.895 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:49.895 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:49.895 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:49.895 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:49.896 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:49.896 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:49.896 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:49.896 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:49.896 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:49.896 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:49.896 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:49.896 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:49.896 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:49.896 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:49.896 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:49.896 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:49.897 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:49.897 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:49.897 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:49.897 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:49.897 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:49.897 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:49.897 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:49.897 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:49.897 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:49.897 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:49.897 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:49.897 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:49.898 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:49.898 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:49.898 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:49.898 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:49.898 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:49.898 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:49.898 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:49.898 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:49.898 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.898 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:49.898 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:49.898 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:49.899 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:49.899 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:49.899 INFO:tasks.workunit.client.0.smithi136.stdout: 4 2024-09-12T23:37:49.899 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.899 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 1, 2024-09-12T23:37:49.899 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:49.899 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.899 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.899 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:49.899 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:49.899 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:49.899 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:49.900 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.900 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 6, 2024-09-12T23:37:49.900 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:49.900 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:49.900 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:49.900 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:49.900 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:49.900 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:49.900 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.666010+0000", 2024-09-12T23:37:49.900 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.883028+0000", 2024-09-12T23:37:49.900 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.883028+0000", 2024-09-12T23:37:49.901 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.884106+0000", 2024-09-12T23:37:49.901 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.666010+0000", 2024-09-12T23:37:49.901 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:49.901 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:49.901 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:49.901 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:49.901 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.666010+0000", 2024-09-12T23:37:49.901 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.666010+0000", 2024-09-12T23:37:49.901 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.666010+0000", 2024-09-12T23:37:49.901 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:49.901 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:49.901 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:49.902 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.666010+0000", 2024-09-12T23:37:49.902 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.666010+0000", 2024-09-12T23:37:49.902 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:49.902 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:49.902 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:49.902 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:49.902 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:49.902 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:49.902 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:49.902 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:49.902 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:49.902 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:49.903 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:49.903 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:49.903 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:49.903 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.17", 2024-09-12T23:37:49.903 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:49.903 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:49.903 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:49.903 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 21, 2024-09-12T23:37:49.903 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:49.903 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T04:20:27.774655+0000", 2024-09-12T23:37:49.903 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:49.903 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:49.904 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:49.904 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:49.904 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:49.904 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:49.904 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:49.904 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:49.904 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:49.904 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:49.904 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:49.904 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:49.904 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:49.904 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:49.905 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:49.905 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:49.905 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:49.905 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:49.905 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:49.905 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:49.905 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:49.905 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:49.905 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:49.905 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:49.905 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:49.906 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:49.906 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:49.906 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:49.906 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:49.906 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:49.906 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:49.906 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:49.906 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:49.906 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:49.906 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:49.906 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:49.906 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:49.907 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:49.907 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:49.907 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:49.907 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:49.907 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:49.907 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:49.907 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.907 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:49.907 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:49.907 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:49.907 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:49.907 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:49.907 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:49.908 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.908 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 6, 2024-09-12T23:37:49.908 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:49.908 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.908 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.908 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:49.908 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:49.908 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:49.908 INFO:tasks.workunit.client.0.smithi136.stdout: 1 2024-09-12T23:37:49.908 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.908 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 5, 2024-09-12T23:37:49.909 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:49.909 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:49.909 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:49.909 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:49.909 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:49.909 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:49.909 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.670853+0000", 2024-09-12T23:37:49.909 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.909325+0000", 2024-09-12T23:37:49.909 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.909325+0000", 2024-09-12T23:37:49.909 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.909613+0000", 2024-09-12T23:37:49.909 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.670853+0000", 2024-09-12T23:37:49.912 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.912 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:49.913 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.913 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:49.913 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.670853+0000", 2024-09-12T23:37:49.913 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.670853+0000", 2024-09-12T23:37:49.913 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.670853+0000", 2024-09-12T23:37:49.913 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:49.913 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:49.913 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.913 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.670853+0000", 2024-09-12T23:37:49.913 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.670853+0000", 2024-09-12T23:37:49.913 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:49.913 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:49.914 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:49.914 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:49.914 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:49.914 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:49.914 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:49.914 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:49.914 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:49.914 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:49.914 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:49.914 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:49.914 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:49.914 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.16", 2024-09-12T23:37:49.915 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:49.915 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:49.915 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:49.915 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:49.915 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:49.915 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T06:50:21.069936+0000", 2024-09-12T23:37:49.915 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:49.915 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:49.915 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:49.915 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:49.915 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:49.915 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:49.916 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:49.916 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:49.916 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:49.916 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:49.916 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:49.916 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:49.916 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:49.916 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:49.916 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:49.916 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:49.916 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:49.917 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:49.917 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:49.917 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:49.917 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:49.917 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:49.917 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:49.917 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:49.917 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:49.917 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:49.917 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:49.917 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:49.917 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:49.918 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:49.918 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:49.918 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:49.918 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:49.918 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:49.918 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:49.918 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:49.918 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:49.918 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:49.918 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:49.918 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:49.918 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:49.919 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:49.919 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:49.919 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.919 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:49.919 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:49.919 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:49.919 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:49.919 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:49.919 INFO:tasks.workunit.client.0.smithi136.stdout: 1 2024-09-12T23:37:49.919 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.919 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 5, 2024-09-12T23:37:49.919 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:49.920 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.920 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.920 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:49.920 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:49.920 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:49.920 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:49.920 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.920 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 5, 2024-09-12T23:37:49.920 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:49.920 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:49.920 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:49.920 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:49.921 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:49.921 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:49.921 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.670886+0000", 2024-09-12T23:37:49.921 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.887477+0000", 2024-09-12T23:37:49.921 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.887477+0000", 2024-09-12T23:37:49.921 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.890012+0000", 2024-09-12T23:37:49.921 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.670886+0000", 2024-09-12T23:37:49.921 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:49.921 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:49.921 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:49.921 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:49.921 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.670886+0000", 2024-09-12T23:37:49.922 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.670886+0000", 2024-09-12T23:37:49.922 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.670886+0000", 2024-09-12T23:37:49.922 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:49.922 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:49.922 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:49.922 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.670886+0000", 2024-09-12T23:37:49.922 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.670886+0000", 2024-09-12T23:37:49.922 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:49.922 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:49.922 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:49.922 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:49.922 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:49.923 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:49.923 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:49.923 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:49.923 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:49.923 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:49.923 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:49.923 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:49.923 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:49.923 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.16", 2024-09-12T23:37:49.923 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:49.923 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:49.923 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:49.924 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 21, 2024-09-12T23:37:49.924 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:49.924 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T02:15:02.988389+0000", 2024-09-12T23:37:49.924 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:49.924 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:49.924 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:49.924 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:49.924 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:49.924 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:49.924 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:49.924 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:49.924 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:49.924 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:49.925 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:49.925 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:49.925 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:49.925 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:49.925 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:49.925 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:49.925 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:49.925 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:49.925 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:49.925 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:49.925 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:49.925 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:49.926 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:49.926 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:49.926 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:49.926 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:49.926 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:49.926 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:49.926 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:49.926 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:49.926 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:49.926 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:49.926 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:49.926 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:49.927 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:49.927 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:49.927 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:49.927 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:49.927 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:49.927 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:49.927 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:49.927 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:49.927 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:49.927 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.927 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:49.927 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:49.928 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:49.928 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:49.928 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:49.928 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:49.928 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.928 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 5, 2024-09-12T23:37:49.928 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:49.928 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.928 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.928 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:49.928 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:49.928 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:49.929 INFO:tasks.workunit.client.0.smithi136.stdout: 3 2024-09-12T23:37:49.929 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.929 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 0, 2024-09-12T23:37:49.929 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:49.929 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:49.929 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:49.929 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:49.929 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:49.929 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:49.929 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.670299+0000", 2024-09-12T23:37:49.929 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.916648+0000", 2024-09-12T23:37:49.929 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.916648+0000", 2024-09-12T23:37:49.930 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.917549+0000", 2024-09-12T23:37:49.930 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.670299+0000", 2024-09-12T23:37:49.930 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.930 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:49.930 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.930 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:49.930 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.670299+0000", 2024-09-12T23:37:49.930 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.670299+0000", 2024-09-12T23:37:49.930 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.670299+0000", 2024-09-12T23:37:49.930 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:49.930 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:49.930 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.931 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.670299+0000", 2024-09-12T23:37:49.931 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.670299+0000", 2024-09-12T23:37:49.931 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:49.931 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:49.931 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:49.931 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:49.931 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:49.931 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:49.931 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:49.931 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:49.931 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:49.931 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:49.932 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:49.932 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:49.932 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:49.932 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.17", 2024-09-12T23:37:49.932 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:49.932 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:49.932 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:49.932 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:49.932 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:49.932 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T09:37:55.708866+0000", 2024-09-12T23:37:49.932 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:49.932 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:49.933 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:49.933 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:49.933 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:49.933 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:49.933 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:49.933 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:49.933 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:49.933 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:49.933 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:49.933 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:49.933 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:49.933 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:49.934 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:49.934 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:49.934 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:49.934 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:49.934 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:49.934 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:49.934 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:49.934 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:49.934 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:49.934 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:49.934 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:49.934 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:49.934 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:49.935 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:49.935 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:49.935 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:49.935 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:49.935 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:49.935 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:49.935 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:49.935 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:49.935 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:49.935 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:49.935 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:49.935 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:49.936 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:49.936 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:49.936 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:49.936 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:49.936 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.936 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:49.936 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:49.936 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:49.936 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:49.936 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:49.936 INFO:tasks.workunit.client.0.smithi136.stdout: 3 2024-09-12T23:37:49.937 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.937 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 0, 2024-09-12T23:37:49.937 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:49.937 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.937 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.937 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:49.937 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:49.937 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:49.937 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:49.937 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.937 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 1, 2024-09-12T23:37:49.937 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:49.938 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:49.938 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:49.938 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:49.938 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:49.938 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:49.938 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.663816+0000", 2024-09-12T23:37:49.938 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.879399+0000", 2024-09-12T23:37:49.938 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.879399+0000", 2024-09-12T23:37:49.938 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.879732+0000", 2024-09-12T23:37:49.938 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.663816+0000", 2024-09-12T23:37:49.938 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:49.938 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:49.939 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:49.939 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:49.939 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.663816+0000", 2024-09-12T23:37:49.939 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.663816+0000", 2024-09-12T23:37:49.939 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.663816+0000", 2024-09-12T23:37:49.939 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:49.939 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:49.939 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:49.939 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.663816+0000", 2024-09-12T23:37:49.939 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.663816+0000", 2024-09-12T23:37:49.939 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:49.939 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 3, 2024-09-12T23:37:49.940 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:49.940 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:49.940 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:49.940 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:49.940 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:49.940 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:49.940 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:49.940 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 3, 2024-09-12T23:37:49.940 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:49.940 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:49.940 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:49.940 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.15", 2024-09-12T23:37:49.941 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:49.941 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:49.941 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:49.941 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 24, 2024-09-12T23:37:49.941 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:49.941 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T05:00:06.109352+0000", 2024-09-12T23:37:49.941 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:49.941 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:49.941 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:49.941 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 90, 2024-09-12T23:37:49.941 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:49.941 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:49.942 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:49.942 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:49.942 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:49.942 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:49.942 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:49.942 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:49.942 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:49.942 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:49.942 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:49.942 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:49.942 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:49.943 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:49.943 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:49.943 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 6, 2024-09-12T23:37:49.943 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 2, 2024-09-12T23:37:49.943 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:49.943 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 2, 2024-09-12T23:37:49.943 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:49.943 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:49.943 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:49.943 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:49.943 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:49.944 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 1, 2024-09-12T23:37:49.944 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:49.944 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:49.944 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:49.944 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:49.944 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:49.944 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:49.944 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:49.944 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:49.944 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:49.944 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:49.945 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:49.945 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:49.945 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 4, 2024-09-12T23:37:49.945 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 1 2024-09-12T23:37:49.945 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.945 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:49.945 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:49.945 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:49.945 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:49.945 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:49.945 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:49.946 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.946 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 1, 2024-09-12T23:37:49.946 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "53'3" 2024-09-12T23:37:49.946 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.946 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.946 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:49.946 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:49.946 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:49.946 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:49.946 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.946 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 2, 2024-09-12T23:37:49.947 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:49.947 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:49.947 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:49.947 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:49.947 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:49.947 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:49.947 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:09.684524+0000", 2024-09-12T23:37:49.947 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:35:09.683923+0000", 2024-09-12T23:37:49.947 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:35:09.683923+0000", 2024-09-12T23:37:49.947 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:35:09.684524+0000", 2024-09-12T23:37:49.947 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:09.684524+0000", 2024-09-12T23:37:49.948 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.948 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:49.948 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.948 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 64, 2024-09-12T23:37:49.948 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:09.684524+0000", 2024-09-12T23:37:49.948 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:09.684524+0000", 2024-09-12T23:37:49.948 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:09.684524+0000", 2024-09-12T23:37:49.948 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:49.948 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:49.948 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.948 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:09.684524+0000", 2024-09-12T23:37:49.949 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:09.684524+0000", 2024-09-12T23:37:49.949 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:49.949 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:49.949 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:49.949 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:49.949 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 63, 2024-09-12T23:37:49.949 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:49.949 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:49.949 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:49.949 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:49.950 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:49.950 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:49.950 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:49.950 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:49.950 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.14", 2024-09-12T23:37:49.950 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:49.950 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:49.950 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 64, 2024-09-12T23:37:49.950 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 10, 2024-09-12T23:37:49.950 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:49.950 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T06:30:40.963502+0000", 2024-09-12T23:37:49.951 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:49.951 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:49.951 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:49.951 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:49.951 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:49.951 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:49.951 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:49.951 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:49.951 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:49.951 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:49.951 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:49.951 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:49.952 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:49.952 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:49.952 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:49.952 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:49.952 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:49.952 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:49.952 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:49.952 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:49.952 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:49.952 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:49.952 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:49.952 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:49.953 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:49.953 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:49.953 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:49.953 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:49.953 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:49.953 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:49.953 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:49.953 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:49.953 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:49.953 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:49.953 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:49.954 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:49.954 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:49.954 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:49.954 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:49.954 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:49.954 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:49.954 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:49.954 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:49.954 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.954 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:49.954 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:49.955 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:49.955 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:49.955 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:49.955 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:49.955 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.955 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 2, 2024-09-12T23:37:49.955 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:49.955 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.955 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.955 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:49.955 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:49.955 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:49.956 INFO:tasks.workunit.client.0.smithi136.stdout: 5 2024-09-12T23:37:49.956 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.956 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 6, 2024-09-12T23:37:49.956 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:49.956 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:49.956 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:49.956 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:49.956 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:49.956 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:49.956 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:38.071488+0000", 2024-09-12T23:37:49.956 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.885991+0000", 2024-09-12T23:37:49.957 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.885991+0000", 2024-09-12T23:37:49.957 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.886930+0000", 2024-09-12T23:37:49.957 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:38.071488+0000", 2024-09-12T23:37:49.957 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:49.957 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:49.957 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:49.957 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:49.957 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:38.071488+0000", 2024-09-12T23:37:49.957 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:38.071488+0000", 2024-09-12T23:37:49.957 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:38.071488+0000", 2024-09-12T23:37:49.957 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:49.958 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:49.958 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:49.958 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:38.071488+0000", 2024-09-12T23:37:49.958 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:38.071488+0000", 2024-09-12T23:37:49.958 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:49.958 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 12, 2024-09-12T23:37:49.958 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:49.958 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:49.958 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:49.958 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:49.958 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:49.958 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:49.959 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:49.959 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 12, 2024-09-12T23:37:49.959 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:49.959 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:49.959 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:49.959 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.14", 2024-09-12T23:37:49.959 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:49.959 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:49.959 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 64, 2024-09-12T23:37:49.959 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 33, 2024-09-12T23:37:49.959 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:49.959 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T09:50:58.477301+0000", 2024-09-12T23:37:49.960 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:49.960 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:49.960 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:49.960 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 2624, 2024-09-12T23:37:49.960 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:49.960 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:49.960 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:49.960 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:49.960 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:49.960 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:49.960 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:49.960 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:49.961 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:49.961 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:49.961 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:49.961 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:49.961 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:49.961 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:49.961 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:49.961 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 12, 2024-09-12T23:37:49.961 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 4, 2024-09-12T23:37:49.961 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:49.961 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 4, 2024-09-12T23:37:49.961 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:49.962 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:49.962 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:49.962 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:49.962 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:49.962 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 1, 2024-09-12T23:37:49.962 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:49.962 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:49.962 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:49.962 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:49.962 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:49.962 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:49.962 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:49.963 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:49.963 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:49.963 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:49.963 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:49.963 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:49.963 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 12, 2024-09-12T23:37:49.963 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 9 2024-09-12T23:37:49.963 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.963 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:49.963 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:49.963 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:49.964 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:49.964 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:49.964 INFO:tasks.workunit.client.0.smithi136.stdout: 5 2024-09-12T23:37:49.964 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.964 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 6, 2024-09-12T23:37:49.964 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "64'12" 2024-09-12T23:37:49.964 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.964 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.964 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:49.964 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:49.964 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:49.964 INFO:tasks.workunit.client.0.smithi136.stdout: 4 2024-09-12T23:37:49.965 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.965 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 7, 2024-09-12T23:37:49.965 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:49.965 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:49.965 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:49.965 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:49.965 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:49.965 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:49.965 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.665823+0000", 2024-09-12T23:37:49.965 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.916622+0000", 2024-09-12T23:37:49.965 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.916622+0000", 2024-09-12T23:37:49.965 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.916929+0000", 2024-09-12T23:37:49.966 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.665823+0000", 2024-09-12T23:37:49.966 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.966 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:49.966 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.966 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:49.966 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.665823+0000", 2024-09-12T23:37:49.966 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.665823+0000", 2024-09-12T23:37:49.966 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.665823+0000", 2024-09-12T23:37:49.966 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:49.966 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:49.966 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.966 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.665823+0000", 2024-09-12T23:37:49.967 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.665823+0000", 2024-09-12T23:37:49.967 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:49.967 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:49.967 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:49.967 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:49.967 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:49.967 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:49.967 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:49.967 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:49.967 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:49.967 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:49.968 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:49.968 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:49.968 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:49.968 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.15", 2024-09-12T23:37:49.968 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:49.968 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:49.968 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:49.968 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:49.968 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:49.968 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T08:40:09.433278+0000", 2024-09-12T23:37:49.968 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:49.969 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:49.969 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:49.969 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:49.969 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:49.969 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:49.969 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:49.969 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:49.969 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:49.969 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:49.969 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:49.969 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:49.969 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:49.970 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:49.970 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:49.970 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:49.970 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:49.970 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:49.970 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:49.970 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:49.970 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:49.970 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:49.970 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:49.970 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:49.970 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:49.971 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:49.971 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:49.971 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:49.971 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:49.971 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:49.971 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:49.971 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:49.971 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:49.971 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:49.971 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:49.971 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:49.971 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:49.972 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:49.972 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:49.972 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:49.972 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:49.972 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:49.972 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:49.972 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.972 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:49.972 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:49.972 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:49.972 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:49.973 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:49.973 INFO:tasks.workunit.client.0.smithi136.stdout: 4 2024-09-12T23:37:49.973 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.973 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 7, 2024-09-12T23:37:49.973 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:49.973 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.973 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.973 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:49.973 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:49.973 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:49.973 INFO:tasks.workunit.client.0.smithi136.stdout: 3 2024-09-12T23:37:49.973 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.974 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 0, 2024-09-12T23:37:49.974 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:49.974 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:49.974 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:49.974 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:49.974 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:49.974 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:49.974 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.669915+0000", 2024-09-12T23:37:49.974 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.879825+0000", 2024-09-12T23:37:49.974 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.879825+0000", 2024-09-12T23:37:49.974 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.880385+0000", 2024-09-12T23:37:49.975 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.669915+0000", 2024-09-12T23:37:49.975 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:49.975 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:49.975 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:49.975 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:49.975 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.669915+0000", 2024-09-12T23:37:49.975 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.669915+0000", 2024-09-12T23:37:49.975 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.669915+0000", 2024-09-12T23:37:49.975 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:49.975 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:49.975 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:49.975 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.669915+0000", 2024-09-12T23:37:49.976 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.669915+0000", 2024-09-12T23:37:49.976 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:49.976 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 2, 2024-09-12T23:37:49.976 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:49.976 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:49.976 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:49.976 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:49.976 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:49.976 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:49.976 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:49.976 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 2, 2024-09-12T23:37:49.977 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:49.977 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:49.977 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:49.977 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.13", 2024-09-12T23:37:49.977 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:49.977 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:49.977 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:49.977 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 23, 2024-09-12T23:37:49.977 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:49.977 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T02:40:33.073593+0000", 2024-09-12T23:37:49.977 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:49.977 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:49.977 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:49.978 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 34, 2024-09-12T23:37:49.978 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:49.978 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:49.978 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:49.978 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:49.978 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:49.978 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:49.978 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:49.978 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:49.978 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:49.978 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:49.978 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:49.979 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:49.979 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:49.979 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:49.979 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:49.979 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 6, 2024-09-12T23:37:49.979 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 2, 2024-09-12T23:37:49.979 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:49.979 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 2, 2024-09-12T23:37:49.979 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:49.979 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:49.979 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:49.979 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:49.979 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:49.980 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 1, 2024-09-12T23:37:49.980 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:49.980 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:49.980 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:49.980 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:49.980 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:49.980 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:49.980 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:49.980 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:49.980 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:49.980 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:49.981 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:49.981 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:49.981 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 3, 2024-09-12T23:37:49.981 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 6 2024-09-12T23:37:49.981 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.981 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:49.981 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:49.981 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:49.981 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:49.981 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:49.981 INFO:tasks.workunit.client.0.smithi136.stdout: 3 2024-09-12T23:37:49.981 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.982 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 0, 2024-09-12T23:37:49.982 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "53'2" 2024-09-12T23:37:49.982 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.982 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.982 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:49.982 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:49.982 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:49.982 INFO:tasks.workunit.client.0.smithi136.stdout: 3 2024-09-12T23:37:49.982 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.982 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 0, 2024-09-12T23:37:49.982 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:49.982 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:49.983 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:49.983 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:49.983 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:49.983 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:49.983 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.670434+0000", 2024-09-12T23:37:49.983 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.918580+0000", 2024-09-12T23:37:49.983 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.918580+0000", 2024-09-12T23:37:49.983 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.919045+0000", 2024-09-12T23:37:49.983 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.670434+0000", 2024-09-12T23:37:49.983 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.983 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:49.984 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.984 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:49.984 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.670434+0000", 2024-09-12T23:37:49.984 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.670434+0000", 2024-09-12T23:37:49.984 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.670434+0000", 2024-09-12T23:37:49.984 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:49.984 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:49.984 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:49.984 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.670434+0000", 2024-09-12T23:37:49.984 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.670434+0000", 2024-09-12T23:37:49.984 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:49.984 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:49.985 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:49.985 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:49.985 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:49.985 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:49.985 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:49.985 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:49.985 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:49.985 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:49.985 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:49.985 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:49.985 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:49.986 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.12", 2024-09-12T23:37:49.986 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:49.986 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:49.986 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:49.986 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:49.986 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:49.986 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T01:39:32.320667+0000", 2024-09-12T23:37:49.986 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:49.986 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:49.986 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:49.986 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:49.986 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:49.987 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:49.987 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:49.987 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:49.987 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:49.987 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:49.987 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:49.987 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:49.987 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:49.987 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:49.987 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:49.987 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:49.988 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:49.988 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:49.988 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:49.988 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:49.988 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:49.988 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:49.988 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:49.988 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:49.988 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:49.988 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:49.988 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:49.988 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:49.989 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:49.989 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:49.989 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:49.989 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:49.989 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:49.989 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:49.989 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:49.989 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:49.989 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:49.989 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:49.989 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:49.990 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:49.990 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:49.990 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:49.990 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:49.990 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.990 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:49.990 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:49.990 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:49.990 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:49.990 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:49.990 INFO:tasks.workunit.client.0.smithi136.stdout: 3 2024-09-12T23:37:49.991 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.991 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 0, 2024-09-12T23:37:49.991 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:49.991 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.991 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:49.991 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:49.991 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:49.991 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:49.991 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:49.991 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:49.991 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 5, 2024-09-12T23:37:49.992 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:49.992 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:49.992 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:49.992 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:49.992 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:49.992 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:49.992 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.671158+0000", 2024-09-12T23:37:49.992 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.898942+0000", 2024-09-12T23:37:49.992 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.898942+0000", 2024-09-12T23:37:49.992 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.899424+0000", 2024-09-12T23:37:49.992 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.671158+0000", 2024-09-12T23:37:49.992 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:49.992 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:49.993 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:49.993 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:49.993 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.671158+0000", 2024-09-12T23:37:49.993 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.671158+0000", 2024-09-12T23:37:49.993 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.671158+0000", 2024-09-12T23:37:49.993 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:49.993 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:49.993 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:49.993 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.671158+0000", 2024-09-12T23:37:49.993 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.671158+0000", 2024-09-12T23:37:49.994 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:49.994 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:49.994 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:49.994 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:49.994 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:49.994 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:49.994 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:49.994 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:49.994 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:49.994 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:49.994 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:49.995 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:49.995 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:49.995 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.12", 2024-09-12T23:37:49.995 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:49.995 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:49.995 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:49.995 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 21, 2024-09-12T23:37:49.995 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:49.995 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T03:14:47.069896+0000", 2024-09-12T23:37:49.995 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:49.995 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:49.995 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:49.995 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:49.996 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:49.996 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:49.996 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:49.996 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:49.996 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:49.996 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:49.996 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:49.996 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:49.996 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:49.996 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:49.996 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:49.996 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:49.997 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:49.997 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:49.997 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:49.997 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:49.997 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:49.997 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:49.997 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:49.997 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:49.997 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:49.997 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:49.997 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:49.997 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:49.998 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:49.998 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:49.998 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:49.998 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:49.998 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:49.998 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:49.998 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:49.998 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:49.998 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:49.998 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:49.999 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:49.999 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:49.999 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:49.999 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:49.999 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:49.999 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:49.999 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:49.999 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:49.999 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:49.999 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:49.999 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:50.000 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:50.000 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.000 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 5, 2024-09-12T23:37:50.000 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:50.000 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.000 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.000 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.000 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:50.000 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:50.000 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:50.000 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.001 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 7, 2024-09-12T23:37:50.001 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.001 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.001 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:50.001 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.001 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.001 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.001 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.663898+0000", 2024-09-12T23:37:50.001 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.901060+0000", 2024-09-12T23:37:50.001 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.901060+0000", 2024-09-12T23:37:50.001 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.901397+0000", 2024-09-12T23:37:50.002 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.663898+0000", 2024-09-12T23:37:50.002 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.002 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.002 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.002 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:50.002 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.663898+0000", 2024-09-12T23:37:50.002 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.663898+0000", 2024-09-12T23:37:50.002 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.663898+0000", 2024-09-12T23:37:50.002 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.002 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.002 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.002 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.663898+0000", 2024-09-12T23:37:50.003 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.663898+0000", 2024-09-12T23:37:50.003 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.003 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:50.003 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.003 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.003 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:50.003 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.003 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.003 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.003 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.003 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:50.004 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.004 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.004 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.004 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.13", 2024-09-12T23:37:50.004 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.004 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.004 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:50.004 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:50.004 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.004 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T11:21:23.285498+0000", 2024-09-12T23:37:50.004 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:50.005 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.005 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.005 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:50.005 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.005 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.005 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.005 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.005 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.005 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.005 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.005 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.006 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.006 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.006 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.006 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.006 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.006 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.006 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.006 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:50.006 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:50.006 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.006 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:50.007 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.007 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.007 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.007 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.007 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.007 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:50.007 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.007 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.007 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.007 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.007 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.007 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.008 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.008 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.008 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.008 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.008 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.008 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.008 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:50.008 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:50.008 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.008 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.008 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.009 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.009 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:50.009 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:50.009 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:50.009 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.009 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 7, 2024-09-12T23:37:50.009 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:50.009 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.009 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.009 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.009 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:50.009 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:50.010 INFO:tasks.workunit.client.0.smithi136.stdout: 1 2024-09-12T23:37:50.010 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.010 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 6, 2024-09-12T23:37:50.010 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.010 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.010 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:50.010 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.010 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.010 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.010 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.665621+0000", 2024-09-12T23:37:50.010 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.869528+0000", 2024-09-12T23:37:50.010 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.869528+0000", 2024-09-12T23:37:50.011 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.871430+0000", 2024-09-12T23:37:50.011 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.665621+0000", 2024-09-12T23:37:50.011 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.011 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.011 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.011 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:50.011 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.665621+0000", 2024-09-12T23:37:50.011 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.665621+0000", 2024-09-12T23:37:50.011 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.665621+0000", 2024-09-12T23:37:50.011 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.011 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.011 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.012 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.665621+0000", 2024-09-12T23:37:50.012 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.665621+0000", 2024-09-12T23:37:50.012 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.012 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:50.012 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.012 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.012 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:50.012 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.012 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.012 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.012 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.013 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:50.013 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.013 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.013 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.013 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.11", 2024-09-12T23:37:50.013 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.013 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.013 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:50.013 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 21, 2024-09-12T23:37:50.013 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.013 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T11:27:43.449941+0000", 2024-09-12T23:37:50.013 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:50.014 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.014 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.014 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:50.014 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.014 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.014 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.014 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.014 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.014 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.014 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.014 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.014 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.014 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.015 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.015 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.015 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.015 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.015 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.015 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:50.015 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:50.015 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.015 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:50.015 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.015 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.015 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.016 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.016 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.016 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:50.016 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.016 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.016 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.016 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.016 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.016 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.016 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.016 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.016 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.017 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.017 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.017 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.017 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:50.017 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:50.017 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.017 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.017 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.017 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.017 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:50.017 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:50.018 INFO:tasks.workunit.client.0.smithi136.stdout: 1 2024-09-12T23:37:50.018 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.018 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 6, 2024-09-12T23:37:50.018 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:50.018 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.018 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.018 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.018 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:50.018 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:50.018 INFO:tasks.workunit.client.0.smithi136.stdout: 5 2024-09-12T23:37:50.018 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.018 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 6, 2024-09-12T23:37:50.018 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.019 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.019 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:50.019 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.019 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.019 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.019 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:09.683003+0000", 2024-09-12T23:37:50.019 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:35:09.681812+0000", 2024-09-12T23:37:50.019 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:35:09.681812+0000", 2024-09-12T23:37:50.019 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:35:09.683003+0000", 2024-09-12T23:37:50.019 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:09.683003+0000", 2024-09-12T23:37:50.019 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.019 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.020 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.020 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 64, 2024-09-12T23:37:50.020 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:09.683003+0000", 2024-09-12T23:37:50.020 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:09.683003+0000", 2024-09-12T23:37:50.020 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:09.683003+0000", 2024-09-12T23:37:50.020 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.020 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.020 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.020 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:09.683003+0000", 2024-09-12T23:37:50.020 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:09.683003+0000", 2024-09-12T23:37:50.020 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.021 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:50.021 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.021 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.021 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 63, 2024-09-12T23:37:50.021 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.021 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.021 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.021 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.021 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:50.021 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.021 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.021 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.022 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.10", 2024-09-12T23:37:50.022 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.022 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.022 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 64, 2024-09-12T23:37:50.022 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 30, 2024-09-12T23:37:50.022 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.022 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-13T23:47:15.002852+0000", 2024-09-12T23:37:50.022 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:50.022 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.022 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.022 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:50.022 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.023 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.023 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.023 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.023 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.023 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.023 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.023 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.023 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.023 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.023 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.023 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.023 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.023 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.024 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.024 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:50.024 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:50.024 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.024 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:50.024 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.024 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.024 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.024 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.024 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.024 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:50.025 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.025 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.025 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.025 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.025 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.025 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.025 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.025 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.025 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.025 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.025 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.025 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.026 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:50.026 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:50.026 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.026 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.026 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.026 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.026 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:50.026 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:50.026 INFO:tasks.workunit.client.0.smithi136.stdout: 5 2024-09-12T23:37:50.026 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.026 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 6, 2024-09-12T23:37:50.027 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:50.027 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.027 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.027 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.027 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:50.027 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:50.027 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:50.027 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.027 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 2, 2024-09-12T23:37:50.027 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.027 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.028 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:50.028 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.028 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.028 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.028 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.661383+0000", 2024-09-12T23:37:50.028 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.863479+0000", 2024-09-12T23:37:50.028 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.863479+0000", 2024-09-12T23:37:50.028 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.863788+0000", 2024-09-12T23:37:50.028 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.661383+0000", 2024-09-12T23:37:50.028 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.028 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.029 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.029 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:50.029 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.661383+0000", 2024-09-12T23:37:50.029 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.661383+0000", 2024-09-12T23:37:50.029 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.661383+0000", 2024-09-12T23:37:50.029 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.029 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.029 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.029 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.661383+0000", 2024-09-12T23:37:50.029 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.661383+0000", 2024-09-12T23:37:50.029 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.030 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 8, 2024-09-12T23:37:50.030 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.030 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.030 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:50.030 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.030 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.030 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.030 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.030 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 8, 2024-09-12T23:37:50.030 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.030 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.030 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.031 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.10", 2024-09-12T23:37:50.031 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.031 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.031 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:50.031 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 29, 2024-09-12T23:37:50.031 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.031 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T08:26:09.204038+0000", 2024-09-12T23:37:50.031 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:50.031 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.031 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.031 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:50.032 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.032 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.032 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.032 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.032 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.032 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.032 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.032 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.032 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.032 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.032 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.033 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.033 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.033 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.033 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.033 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 12, 2024-09-12T23:37:50.033 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 4, 2024-09-12T23:37:50.033 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.033 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 4, 2024-09-12T23:37:50.033 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.033 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.033 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.033 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.034 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.034 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 4, 2024-09-12T23:37:50.034 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.034 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.034 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.034 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.034 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.034 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.034 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.034 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.034 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.034 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.035 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.035 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.035 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 12, 2024-09-12T23:37:50.035 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:50.035 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.035 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.035 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.035 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.035 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:50.035 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:50.035 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:50.036 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.036 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 2, 2024-09-12T23:37:50.036 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "53'8" 2024-09-12T23:37:50.036 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.036 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.036 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.036 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:50.036 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:50.036 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:50.036 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.037 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 7, 2024-09-12T23:37:50.037 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.037 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.037 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:50.037 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.037 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.037 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.037 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.664571+0000", 2024-09-12T23:37:50.037 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.901803+0000", 2024-09-12T23:37:50.037 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.901803+0000", 2024-09-12T23:37:50.037 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.902101+0000", 2024-09-12T23:37:50.037 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.664571+0000", 2024-09-12T23:37:50.038 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.038 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.038 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.038 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:50.038 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.664571+0000", 2024-09-12T23:37:50.038 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.664571+0000", 2024-09-12T23:37:50.038 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.664571+0000", 2024-09-12T23:37:50.038 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.038 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.038 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.038 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.664571+0000", 2024-09-12T23:37:50.039 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.664571+0000", 2024-09-12T23:37:50.039 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.039 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:50.039 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.039 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.039 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:50.039 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.039 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.039 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.039 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.039 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:50.039 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.040 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.040 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.040 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.11", 2024-09-12T23:37:50.040 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.040 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.040 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:50.040 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:50.040 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.040 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T02:00:48.557318+0000", 2024-09-12T23:37:50.040 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:50.040 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.041 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.041 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:50.041 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.041 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.041 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.041 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.041 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.041 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.041 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.041 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.041 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.041 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.042 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.042 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.042 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.042 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.042 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.042 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:50.042 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:50.042 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.042 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:50.042 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.042 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.042 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.043 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.043 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.043 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:50.043 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.043 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.043 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.043 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.043 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.043 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.043 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.043 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.043 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.044 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.044 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.044 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.044 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:50.044 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:50.044 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.044 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.044 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.044 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.044 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:50.044 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:50.044 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:50.045 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.045 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 7, 2024-09-12T23:37:50.045 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:50.045 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.045 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.045 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.045 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:50.045 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:50.045 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:50.045 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.045 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 4, 2024-09-12T23:37:50.046 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.046 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.046 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:50.046 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.046 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.046 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.046 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.662210+0000", 2024-09-12T23:37:50.046 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.877913+0000", 2024-09-12T23:37:50.046 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.877913+0000", 2024-09-12T23:37:50.046 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.879318+0000", 2024-09-12T23:37:50.046 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.662210+0000", 2024-09-12T23:37:50.046 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.047 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.047 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.047 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:50.047 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.662210+0000", 2024-09-12T23:37:50.047 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.662210+0000", 2024-09-12T23:37:50.047 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.662210+0000", 2024-09-12T23:37:50.047 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.047 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.047 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.047 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.662210+0000", 2024-09-12T23:37:50.047 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.662210+0000", 2024-09-12T23:37:50.048 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.048 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 1, 2024-09-12T23:37:50.048 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.048 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.048 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:50.048 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.048 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.048 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.048 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.048 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 1, 2024-09-12T23:37:50.048 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.049 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.049 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.049 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.f", 2024-09-12T23:37:50.049 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.049 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.049 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:50.049 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 22, 2024-09-12T23:37:50.049 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.049 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T10:13:19.574352+0000", 2024-09-12T23:37:50.049 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:50.049 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.049 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.050 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 22, 2024-09-12T23:37:50.050 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.050 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.050 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.050 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.050 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.050 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.050 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.050 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.050 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.050 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.050 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.051 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.051 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.051 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.051 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.051 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 3, 2024-09-12T23:37:50.051 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 1, 2024-09-12T23:37:50.051 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.051 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 1, 2024-09-12T23:37:50.051 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.051 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.051 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.052 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.052 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.052 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:50.052 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.052 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.052 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.052 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.052 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.052 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.052 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.052 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.052 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.053 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.053 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.053 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.053 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 1, 2024-09-12T23:37:50.053 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 1 2024-09-12T23:37:50.053 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.053 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.053 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.053 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.053 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:50.053 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:50.054 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:50.054 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.054 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 4, 2024-09-12T23:37:50.054 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "53'1" 2024-09-12T23:37:50.054 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.054 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.054 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.054 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:50.054 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:50.054 INFO:tasks.workunit.client.0.smithi136.stdout: 1 2024-09-12T23:37:50.054 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.055 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 7, 2024-09-12T23:37:50.055 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.055 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.055 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:50.055 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.055 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.055 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.055 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.664646+0000", 2024-09-12T23:37:50.055 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.901052+0000", 2024-09-12T23:37:50.055 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.901052+0000", 2024-09-12T23:37:50.055 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.901391+0000", 2024-09-12T23:37:50.056 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.664646+0000", 2024-09-12T23:37:50.056 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.056 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.056 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.056 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:50.056 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.664646+0000", 2024-09-12T23:37:50.056 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.664646+0000", 2024-09-12T23:37:50.056 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.664646+0000", 2024-09-12T23:37:50.056 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.056 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.056 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.056 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.664646+0000", 2024-09-12T23:37:50.057 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.664646+0000", 2024-09-12T23:37:50.057 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.057 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:50.057 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.057 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.057 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:50.057 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.057 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.057 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.057 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.057 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:50.058 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.058 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.058 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.058 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.e", 2024-09-12T23:37:50.058 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.058 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.058 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:50.058 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:50.058 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.058 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T04:54:04.075006+0000", 2024-09-12T23:37:50.058 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:50.059 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.059 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.059 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:50.059 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.059 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.059 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.059 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.059 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.059 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.059 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.059 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.059 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.060 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.060 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.060 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.060 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.060 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.060 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.060 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:50.060 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:50.060 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.060 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:50.060 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.061 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.061 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.061 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.061 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.061 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:50.061 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.061 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.061 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.061 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.061 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.061 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.061 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.062 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.062 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.062 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.062 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.062 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.062 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:50.062 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:50.062 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.062 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.062 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.062 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.063 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:50.063 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:50.063 INFO:tasks.workunit.client.0.smithi136.stdout: 1 2024-09-12T23:37:50.063 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.063 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 7, 2024-09-12T23:37:50.063 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:50.063 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.063 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.063 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.063 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:50.063 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:50.063 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:50.064 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.064 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 2, 2024-09-12T23:37:50.064 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.064 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.064 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:50.064 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.064 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.064 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.064 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.661659+0000", 2024-09-12T23:37:50.064 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.887109+0000", 2024-09-12T23:37:50.064 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.887109+0000", 2024-09-12T23:37:50.065 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.887441+0000", 2024-09-12T23:37:50.065 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.661659+0000", 2024-09-12T23:37:50.065 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.065 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.065 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.065 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:50.065 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.661659+0000", 2024-09-12T23:37:50.065 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.661659+0000", 2024-09-12T23:37:50.065 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.661659+0000", 2024-09-12T23:37:50.065 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.065 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.066 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.066 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.661659+0000", 2024-09-12T23:37:50.066 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.661659+0000", 2024-09-12T23:37:50.066 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.066 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 2, 2024-09-12T23:37:50.066 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.066 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.066 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:50.066 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.066 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.066 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.067 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.067 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 2, 2024-09-12T23:37:50.067 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.067 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.067 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.067 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.e", 2024-09-12T23:37:50.067 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.067 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.067 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:50.067 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 23, 2024-09-12T23:37:50.067 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.067 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T07:35:14.071342+0000", 2024-09-12T23:37:50.068 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:50.068 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.068 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.068 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:50.068 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.068 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.068 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.068 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.068 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.068 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.068 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.069 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.069 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.069 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.069 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.069 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.069 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.069 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.069 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.069 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 3, 2024-09-12T23:37:50.069 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 1, 2024-09-12T23:37:50.069 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.070 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 1, 2024-09-12T23:37:50.070 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.070 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.070 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.070 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.070 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.070 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 1, 2024-09-12T23:37:50.070 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.070 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.070 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.070 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.071 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.071 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.071 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.071 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.071 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.071 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.071 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.071 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.071 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 3, 2024-09-12T23:37:50.071 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:50.071 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.072 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.072 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.072 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.072 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:50.072 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:50.072 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:50.072 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.072 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 2, 2024-09-12T23:37:50.072 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "53'2" 2024-09-12T23:37:50.072 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.072 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.073 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.073 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:50.073 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:50.073 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:50.073 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.073 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 7, 2024-09-12T23:37:50.073 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.073 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.073 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:50.073 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.073 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.074 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.074 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.665111+0000", 2024-09-12T23:37:50.074 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.907302+0000", 2024-09-12T23:37:50.074 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.907302+0000", 2024-09-12T23:37:50.074 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.907594+0000", 2024-09-12T23:37:50.074 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.665111+0000", 2024-09-12T23:37:50.074 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.074 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.074 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.074 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:50.074 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.665111+0000", 2024-09-12T23:37:50.075 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.665111+0000", 2024-09-12T23:37:50.075 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.665111+0000", 2024-09-12T23:37:50.075 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.075 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.075 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.075 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.665111+0000", 2024-09-12T23:37:50.075 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.665111+0000", 2024-09-12T23:37:50.075 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.075 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:50.075 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.075 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.076 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:50.076 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.076 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.076 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.076 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.076 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:50.076 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.076 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.076 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.076 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.f", 2024-09-12T23:37:50.076 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.077 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.077 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:50.077 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:50.077 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.077 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T05:45:37.988381+0000", 2024-09-12T23:37:50.077 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:50.077 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.077 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.077 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:50.077 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.077 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.078 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.078 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.078 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.078 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.078 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.078 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.078 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.078 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.078 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.078 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.078 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.078 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.079 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.079 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:50.079 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:50.079 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.079 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:50.079 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.079 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.079 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.079 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.079 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.079 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:50.080 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.080 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.080 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.080 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.080 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.080 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.080 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.080 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.080 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.080 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.080 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.080 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.081 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:50.081 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:50.081 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.081 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.081 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.081 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.081 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:50.081 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:50.081 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:50.081 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.081 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 7, 2024-09-12T23:37:50.081 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:50.082 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.082 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.082 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.082 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:50.082 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:50.082 INFO:tasks.workunit.client.0.smithi136.stdout: 3 2024-09-12T23:37:50.082 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.082 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 1, 2024-09-12T23:37:50.082 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.082 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.082 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:50.082 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.083 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.083 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.083 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.663915+0000", 2024-09-12T23:37:50.083 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.871628+0000", 2024-09-12T23:37:50.083 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.871628+0000", 2024-09-12T23:37:50.083 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.871944+0000", 2024-09-12T23:37:50.083 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.663915+0000", 2024-09-12T23:37:50.083 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.083 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.083 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.083 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:50.083 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.663915+0000", 2024-09-12T23:37:50.084 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.663915+0000", 2024-09-12T23:37:50.084 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.663915+0000", 2024-09-12T23:37:50.084 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.084 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.084 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.084 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.663915+0000", 2024-09-12T23:37:50.084 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.663915+0000", 2024-09-12T23:37:50.084 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.084 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 1, 2024-09-12T23:37:50.084 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.084 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.085 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:50.085 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.085 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.085 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.085 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.085 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 1, 2024-09-12T23:37:50.085 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.085 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.085 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.085 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.d", 2024-09-12T23:37:50.085 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.085 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.085 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:50.086 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 22, 2024-09-12T23:37:50.086 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.086 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T09:58:29.645624+0000", 2024-09-12T23:37:50.086 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:50.086 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.086 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.086 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 70, 2024-09-12T23:37:50.086 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.086 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.086 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.086 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.087 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.087 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.087 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.087 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.087 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.087 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.087 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.087 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.087 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.087 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.087 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.087 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 3, 2024-09-12T23:37:50.088 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 1, 2024-09-12T23:37:50.088 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.088 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 1, 2024-09-12T23:37:50.088 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.088 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.088 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.088 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.088 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.088 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:50.088 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.088 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.088 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.089 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.089 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.089 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.089 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.089 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.089 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.089 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.089 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.089 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.089 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 1, 2024-09-12T23:37:50.089 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 1 2024-09-12T23:37:50.090 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.090 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.090 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.090 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.090 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:50.090 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:50.090 INFO:tasks.workunit.client.0.smithi136.stdout: 3 2024-09-12T23:37:50.090 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.090 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 1, 2024-09-12T23:37:50.090 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "53'1" 2024-09-12T23:37:50.090 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.091 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.091 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.091 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:50.091 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:50.091 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:50.091 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.091 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 5, 2024-09-12T23:37:50.091 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.091 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.091 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:50.091 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.091 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.092 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.092 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.670659+0000", 2024-09-12T23:37:50.092 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.914416+0000", 2024-09-12T23:37:50.092 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.914416+0000", 2024-09-12T23:37:50.092 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.914833+0000", 2024-09-12T23:37:50.092 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.670659+0000", 2024-09-12T23:37:50.092 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.092 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.092 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.092 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:50.092 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.670659+0000", 2024-09-12T23:37:50.093 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.670659+0000", 2024-09-12T23:37:50.093 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.670659+0000", 2024-09-12T23:37:50.093 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.093 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.093 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.093 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.670659+0000", 2024-09-12T23:37:50.093 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.670659+0000", 2024-09-12T23:37:50.093 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.093 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:50.093 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.094 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.094 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:50.094 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.094 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.094 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.094 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.094 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:50.094 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.094 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.094 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.094 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.c", 2024-09-12T23:37:50.094 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.095 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.095 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:50.095 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:50.095 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.095 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T09:27:20.253954+0000", 2024-09-12T23:37:50.095 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:50.095 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.095 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.095 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:50.095 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.095 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.096 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.096 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.096 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.096 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.096 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.096 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.096 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.096 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.096 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.096 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.097 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.097 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.097 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.097 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:50.097 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:50.097 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.097 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:50.097 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.097 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.097 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.097 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.098 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.098 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:50.098 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.098 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.098 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.098 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.098 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.098 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.098 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.098 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.098 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.099 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.099 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.099 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.099 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:50.099 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:50.099 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.099 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.099 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.099 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.099 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:50.099 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:50.099 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:50.100 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.100 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 5, 2024-09-12T23:37:50.100 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:50.100 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.100 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.100 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.100 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:50.100 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:50.100 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:50.100 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.100 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 2, 2024-09-12T23:37:50.101 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.101 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.101 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:50.101 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.101 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.101 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.101 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.662019+0000", 2024-09-12T23:37:50.101 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.881774+0000", 2024-09-12T23:37:50.101 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.881774+0000", 2024-09-12T23:37:50.101 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.882138+0000", 2024-09-12T23:37:50.101 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.662019+0000", 2024-09-12T23:37:50.102 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.102 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.102 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.102 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:50.102 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.662019+0000", 2024-09-12T23:37:50.102 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.662019+0000", 2024-09-12T23:37:50.102 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.662019+0000", 2024-09-12T23:37:50.102 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.102 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.102 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.102 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.662019+0000", 2024-09-12T23:37:50.102 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.662019+0000", 2024-09-12T23:37:50.103 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.103 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:50.103 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.103 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.103 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:50.103 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.103 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.103 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.103 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.103 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:50.103 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.104 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.104 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.104 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.c", 2024-09-12T23:37:50.104 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.104 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.104 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:50.104 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 21, 2024-09-12T23:37:50.104 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.104 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T05:07:36.075590+0000", 2024-09-12T23:37:50.104 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:50.104 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.104 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.105 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:50.105 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.105 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.105 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.105 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.105 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.105 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.105 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.105 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.105 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.105 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.105 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.106 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.106 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.106 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.106 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.106 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:50.106 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:50.106 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.106 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:50.106 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.106 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.106 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.107 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.107 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.107 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:50.107 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.107 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.107 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.107 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.107 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.107 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.107 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.107 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.107 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.108 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.108 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.108 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.108 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:50.108 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:50.108 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.108 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.108 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.108 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.108 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:50.108 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:50.109 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:50.109 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.109 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 2, 2024-09-12T23:37:50.109 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:50.109 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.109 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.109 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.109 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:50.109 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:50.109 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:50.109 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.110 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 7, 2024-09-12T23:37:50.110 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.110 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.110 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:50.110 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.110 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.110 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.110 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.665947+0000", 2024-09-12T23:37:50.110 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.911244+0000", 2024-09-12T23:37:50.110 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.911244+0000", 2024-09-12T23:37:50.110 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.911549+0000", 2024-09-12T23:37:50.111 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.665947+0000", 2024-09-12T23:37:50.111 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.111 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.111 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.111 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:50.111 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.665947+0000", 2024-09-12T23:37:50.111 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.665947+0000", 2024-09-12T23:37:50.111 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.665947+0000", 2024-09-12T23:37:50.111 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.111 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.111 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.112 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.665947+0000", 2024-09-12T23:37:50.112 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.665947+0000", 2024-09-12T23:37:50.112 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.112 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:50.112 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.112 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.112 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:50.112 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.112 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.112 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.112 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.113 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:50.113 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.113 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.113 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.113 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.d", 2024-09-12T23:37:50.113 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.113 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.113 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:50.113 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:50.113 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.113 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T11:22:20.405567+0000", 2024-09-12T23:37:50.113 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:50.113 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.114 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.114 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:50.114 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.114 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.114 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.114 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.114 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.114 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.114 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.114 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.114 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.115 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.115 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.115 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.115 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.115 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.115 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.115 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:50.115 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:50.115 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.115 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:50.115 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.115 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.116 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.116 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.116 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.116 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:50.116 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.116 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.116 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.116 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.116 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.116 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.116 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.117 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.117 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.117 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.117 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.117 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.117 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:50.117 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:50.117 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.117 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.117 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.117 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.118 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:50.118 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:50.118 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:50.118 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.118 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 7, 2024-09-12T23:37:50.118 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:50.118 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.118 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.118 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.118 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:50.118 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:50.118 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:50.119 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.119 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 6, 2024-09-12T23:37:50.119 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.119 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.119 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 56, 2024-09-12T23:37:50.119 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.119 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.119 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.119 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.666935+0000", 2024-09-12T23:37:50.119 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:34:49.734724+0000", 2024-09-12T23:37:50.119 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:34:49.734724+0000", 2024-09-12T23:37:50.120 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:34:59.598602+0000", 2024-09-12T23:37:50.120 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.666935+0000", 2024-09-12T23:37:50.120 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:34:48.454503+0000", 2024-09-12T23:37:50.120 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.120 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:34:48.454503+0000", 2024-09-12T23:37:50.120 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 57, 2024-09-12T23:37:50.120 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.666935+0000", 2024-09-12T23:37:50.120 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.666935+0000", 2024-09-12T23:37:50.120 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.666935+0000", 2024-09-12T23:37:50.120 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.120 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.120 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:34:48.454503+0000", 2024-09-12T23:37:50.121 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.666935+0000", 2024-09-12T23:37:50.121 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.666935+0000", 2024-09-12T23:37:50.121 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.121 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:50.121 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.121 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.121 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 56, 2024-09-12T23:37:50.121 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.121 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.121 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.121 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.121 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:50.122 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.122 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.122 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.122 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "4.5", 2024-09-12T23:37:50.122 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.122 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.122 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:50.122 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 21, 2024-09-12T23:37:50.122 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.122 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T08:27:05.298933+0000", 2024-09-12T23:37:50.122 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.000791419, 2024-09-12T23:37:50.122 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.123 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.123 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:50.123 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.123 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.123 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.123 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.123 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.123 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.123 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.123 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.123 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.123 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.124 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.124 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.124 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.124 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.124 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.124 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:50.124 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:50.124 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.124 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:50.124 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.124 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.124 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.125 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.125 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.125 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:50.125 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.125 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.125 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.125 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.125 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.125 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.125 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.125 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.125 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.125 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.126 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.126 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.126 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:50.126 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:50.126 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.126 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.126 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.126 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.126 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:50.126 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:50.126 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:50.126 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.127 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 6, 2024-09-12T23:37:50.127 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:50.127 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.127 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.127 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.127 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:50.127 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:50.127 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:50.127 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.127 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 7, 2024-09-12T23:37:50.127 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.127 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.128 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 19, 2024-09-12T23:37:50.128 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.128 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.128 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.128 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.664101+0000", 2024-09-12T23:37:50.128 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:32:45.887205+0000", 2024-09-12T23:37:50.128 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:32:45.887205+0000", 2024-09-12T23:37:50.128 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:32:45.902460+0000", 2024-09-12T23:37:50.128 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.664101+0000", 2024-09-12T23:37:50.128 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:28:34.324093+0000", 2024-09-12T23:37:50.128 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.129 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:28:34.324093+0000", 2024-09-12T23:37:50.129 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 50, 2024-09-12T23:37:50.129 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.664101+0000", 2024-09-12T23:37:50.129 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.664101+0000", 2024-09-12T23:37:50.129 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.664101+0000", 2024-09-12T23:37:50.129 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.129 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.129 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:28:34.324093+0000", 2024-09-12T23:37:50.129 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.664101+0000", 2024-09-12T23:37:50.129 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.664101+0000", 2024-09-12T23:37:50.129 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.129 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 79, 2024-09-12T23:37:50.130 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.130 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.130 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 49, 2024-09-12T23:37:50.130 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.130 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.130 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.130 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.130 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 79, 2024-09-12T23:37:50.130 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.130 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.130 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.130 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "1.0", 2024-09-12T23:37:50.131 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.131 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.131 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:50.131 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 55, 2024-09-12T23:37:50.131 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.131 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T07:37:37.825919+0000", 2024-09-12T23:37:50.131 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:50.131 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.131 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.131 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 590368, 2024-09-12T23:37:50.131 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.132 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 590368, 2024-09-12T23:37:50.132 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.132 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.132 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.132 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.132 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.132 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.132 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.132 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.132 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.132 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.133 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.133 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.133 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.133 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 6, 2024-09-12T23:37:50.133 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 2, 2024-09-12T23:37:50.133 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.133 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 2, 2024-09-12T23:37:50.133 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.133 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.133 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.133 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.134 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.134 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:50.134 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.134 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 2, 2024-09-12T23:37:50.134 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.134 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.134 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.134 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.134 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.134 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 126, 2024-09-12T23:37:50.134 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 231, 2024-09-12T23:37:50.135 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.135 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.135 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.135 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 113, 2024-09-12T23:37:50.135 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 1628 2024-09-12T23:37:50.135 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.135 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.135 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.135 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.135 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:50.135 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:50.135 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:50.136 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.136 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 7, 2024-09-12T23:37:50.136 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "55'79" 2024-09-12T23:37:50.136 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.136 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.136 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.136 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:50.136 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:50.136 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:50.136 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.136 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 5, 2024-09-12T23:37:50.136 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.137 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.137 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:50.137 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.137 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.137 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.137 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.670187+0000", 2024-09-12T23:37:50.137 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.898329+0000", 2024-09-12T23:37:50.137 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.898329+0000", 2024-09-12T23:37:50.137 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.898796+0000", 2024-09-12T23:37:50.137 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.670187+0000", 2024-09-12T23:37:50.137 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.138 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.138 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.138 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:50.138 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.670187+0000", 2024-09-12T23:37:50.138 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.670187+0000", 2024-09-12T23:37:50.138 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.670187+0000", 2024-09-12T23:37:50.138 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.138 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.138 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.138 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.670187+0000", 2024-09-12T23:37:50.138 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.670187+0000", 2024-09-12T23:37:50.138 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.139 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:50.139 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.139 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.139 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:50.139 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.139 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.139 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.139 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.139 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:50.139 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.139 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.139 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.140 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.3", 2024-09-12T23:37:50.140 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.140 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.140 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:50.140 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 21, 2024-09-12T23:37:50.140 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.140 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T07:54:48.474286+0000", 2024-09-12T23:37:50.140 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:50.140 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.140 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.140 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:50.141 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.141 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.141 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.141 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.141 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.141 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.141 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.141 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.141 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.141 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.141 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.142 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.142 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.142 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.142 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.142 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:50.142 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:50.142 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.142 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:50.142 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.142 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.142 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.142 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.143 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.143 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:50.143 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.143 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.143 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.143 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.143 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.143 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.143 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.143 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.143 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.144 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.144 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.144 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.144 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:50.144 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:50.144 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.144 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.144 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.144 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.144 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:50.144 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:50.145 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:50.145 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.145 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 5, 2024-09-12T23:37:50.145 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:50.145 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.145 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.145 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.145 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:50.145 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:50.145 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:50.146 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.146 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 3, 2024-09-12T23:37:50.146 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.146 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.146 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:50.146 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.146 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.146 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.146 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.668367+0000", 2024-09-12T23:37:50.146 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.904695+0000", 2024-09-12T23:37:50.146 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.904695+0000", 2024-09-12T23:37:50.146 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.905028+0000", 2024-09-12T23:37:50.147 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.668367+0000", 2024-09-12T23:37:50.147 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.147 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.147 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.147 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:50.147 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.668367+0000", 2024-09-12T23:37:50.147 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.668367+0000", 2024-09-12T23:37:50.147 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.668367+0000", 2024-09-12T23:37:50.147 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.147 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.147 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.148 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.668367+0000", 2024-09-12T23:37:50.148 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.668367+0000", 2024-09-12T23:37:50.148 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.148 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:50.148 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.148 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.148 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:50.148 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.148 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.148 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.148 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.148 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:50.149 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.149 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.149 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.149 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.2", 2024-09-12T23:37:50.149 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.149 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.149 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:50.149 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:50.149 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.149 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T01:06:47.004852+0000", 2024-09-12T23:37:50.149 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:50.150 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.150 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.150 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:50.150 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.150 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.150 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.150 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.150 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.150 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.150 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.150 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.150 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.151 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.151 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.151 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.151 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.151 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.151 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.151 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:50.151 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:50.151 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.151 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:50.151 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.151 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.152 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.152 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.152 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.152 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:50.152 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.152 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.152 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.152 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.152 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.152 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.152 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.152 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.152 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.153 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.153 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.153 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.153 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:50.153 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:50.153 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.153 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.153 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.153 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.153 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:50.153 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:50.153 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:50.154 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.154 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 3, 2024-09-12T23:37:50.154 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:50.154 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.154 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.154 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.154 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:50.154 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:50.154 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:50.154 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.154 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 0, 2024-09-12T23:37:50.154 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.154 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.155 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 56, 2024-09-12T23:37:50.155 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.155 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.155 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.155 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.669471+0000", 2024-09-12T23:37:50.155 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:34:49.788254+0000", 2024-09-12T23:37:50.155 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:34:49.788254+0000", 2024-09-12T23:37:50.155 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:34:59.542653+0000", 2024-09-12T23:37:50.155 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.669471+0000", 2024-09-12T23:37:50.155 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:34:48.454503+0000", 2024-09-12T23:37:50.156 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.156 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:34:48.454503+0000", 2024-09-12T23:37:50.156 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 57, 2024-09-12T23:37:50.156 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.669471+0000", 2024-09-12T23:37:50.156 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.669471+0000", 2024-09-12T23:37:50.156 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.669471+0000", 2024-09-12T23:37:50.156 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.156 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.156 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:34:48.454503+0000", 2024-09-12T23:37:50.156 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.669471+0000", 2024-09-12T23:37:50.156 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.669471+0000", 2024-09-12T23:37:50.157 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.157 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:50.157 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.157 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.157 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 56, 2024-09-12T23:37:50.157 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.157 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.157 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.157 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.157 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:50.157 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.157 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.158 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.158 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "4.6", 2024-09-12T23:37:50.158 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.158 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.158 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:50.158 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 21, 2024-09-12T23:37:50.158 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.158 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T10:47:44.627533+0000", 2024-09-12T23:37:50.158 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.001202909, 2024-09-12T23:37:50.158 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.158 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.159 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:50.159 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.159 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.159 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.159 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.159 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.159 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.159 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.159 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.159 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.159 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.160 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.160 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.160 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.160 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.160 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.160 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:50.160 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:50.160 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.160 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:50.160 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.160 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.161 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.161 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.161 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.161 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:50.161 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.161 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.161 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.161 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.161 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.161 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.161 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.161 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.162 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.162 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.162 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.162 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.162 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:50.162 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:50.162 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.162 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.162 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.162 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.162 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:50.163 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:50.163 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:50.163 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.163 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 0, 2024-09-12T23:37:50.163 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:50.163 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.163 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.163 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.163 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:50.163 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:50.163 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:50.163 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.164 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 7, 2024-09-12T23:37:50.164 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.164 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.164 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:50.164 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.164 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.164 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.164 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.663998+0000", 2024-09-12T23:37:50.164 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.894687+0000", 2024-09-12T23:37:50.164 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.894687+0000", 2024-09-12T23:37:50.165 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.895135+0000", 2024-09-12T23:37:50.165 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.663998+0000", 2024-09-12T23:37:50.165 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.165 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.165 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.165 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:50.165 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.663998+0000", 2024-09-12T23:37:50.165 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.663998+0000", 2024-09-12T23:37:50.165 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.663998+0000", 2024-09-12T23:37:50.165 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.165 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.165 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.166 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.663998+0000", 2024-09-12T23:37:50.166 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.663998+0000", 2024-09-12T23:37:50.166 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.166 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:50.166 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.166 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.166 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:50.166 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.166 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.166 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.166 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.167 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:50.167 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.167 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.167 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.167 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.0", 2024-09-12T23:37:50.167 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.167 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.167 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:50.167 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 21, 2024-09-12T23:37:50.167 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.167 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T07:57:06.646238+0000", 2024-09-12T23:37:50.167 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:50.168 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.168 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.168 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:50.168 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.168 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.168 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.168 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.168 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.168 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.168 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.169 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.169 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.169 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.169 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.169 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.169 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.169 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.169 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.169 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:50.169 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:50.169 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.170 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:50.170 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.170 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.170 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.170 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.170 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.170 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:50.170 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.170 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.170 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.170 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.171 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.171 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.171 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.171 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.171 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.171 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.171 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.171 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.171 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:50.171 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:50.171 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.171 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.172 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.172 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.172 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:50.172 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:50.172 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:50.172 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.172 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 7, 2024-09-12T23:37:50.172 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:50.172 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.172 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.172 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.172 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:50.173 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:50.173 INFO:tasks.workunit.client.0.smithi136.stdout: 3 2024-09-12T23:37:50.173 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.173 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 0, 2024-09-12T23:37:50.173 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.173 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.173 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:50.173 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.173 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.173 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.173 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.669881+0000", 2024-09-12T23:37:50.174 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.909765+0000", 2024-09-12T23:37:50.174 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.909765+0000", 2024-09-12T23:37:50.174 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.910025+0000", 2024-09-12T23:37:50.174 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.669881+0000", 2024-09-12T23:37:50.174 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.174 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.174 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.174 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:50.174 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.669881+0000", 2024-09-12T23:37:50.174 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.669881+0000", 2024-09-12T23:37:50.174 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.669881+0000", 2024-09-12T23:37:50.175 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.175 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.175 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.175 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.669881+0000", 2024-09-12T23:37:50.175 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.669881+0000", 2024-09-12T23:37:50.175 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.175 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:50.175 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.175 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.175 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:50.175 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.175 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.176 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.176 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.176 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:50.176 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.176 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.176 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.176 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.1", 2024-09-12T23:37:50.176 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.176 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.176 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:50.176 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:50.177 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.177 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T00:23:35.389216+0000", 2024-09-12T23:37:50.177 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:50.177 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.177 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.177 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:50.177 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.177 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.177 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.177 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.177 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.178 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.178 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.178 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.178 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.178 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.178 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.178 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.178 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.178 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.178 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.178 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:50.178 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:50.178 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.179 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:50.179 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.179 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.179 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.179 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.179 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.179 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:50.179 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.179 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.179 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.179 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.180 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.180 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.180 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.180 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.180 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.180 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.180 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.180 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.180 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:50.180 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:50.180 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.180 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.181 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.181 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.181 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:50.181 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:50.181 INFO:tasks.workunit.client.0.smithi136.stdout: 3 2024-09-12T23:37:50.181 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.181 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 0, 2024-09-12T23:37:50.181 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:50.181 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.181 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.181 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.181 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:50.182 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:50.182 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:50.182 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.182 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 1, 2024-09-12T23:37:50.182 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.182 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.182 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 56, 2024-09-12T23:37:50.182 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.182 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.182 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.182 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.663799+0000", 2024-09-12T23:37:50.182 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:34:49.788699+0000", 2024-09-12T23:37:50.183 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:34:49.788699+0000", 2024-09-12T23:37:50.183 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:34:59.549321+0000", 2024-09-12T23:37:50.183 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.663799+0000", 2024-09-12T23:37:50.183 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:34:48.454503+0000", 2024-09-12T23:37:50.183 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.183 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:34:48.454503+0000", 2024-09-12T23:37:50.183 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 57, 2024-09-12T23:37:50.183 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.663799+0000", 2024-09-12T23:37:50.183 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.663799+0000", 2024-09-12T23:37:50.183 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.663799+0000", 2024-09-12T23:37:50.183 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.183 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.184 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:34:48.454503+0000", 2024-09-12T23:37:50.184 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.663799+0000", 2024-09-12T23:37:50.184 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.663799+0000", 2024-09-12T23:37:50.184 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.184 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:50.184 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.184 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.184 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 56, 2024-09-12T23:37:50.184 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.184 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.184 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.185 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.185 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:50.185 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.185 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.185 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.185 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "4.7", 2024-09-12T23:37:50.185 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.185 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.185 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:50.185 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 21, 2024-09-12T23:37:50.185 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.185 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T11:29:56.436091+0000", 2024-09-12T23:37:50.186 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.002029651, 2024-09-12T23:37:50.186 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.186 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.186 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:50.186 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.186 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.186 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.186 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.186 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.186 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.186 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.187 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.187 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.187 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.187 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.187 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.187 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.187 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.187 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.187 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:50.187 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:50.187 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.187 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:50.187 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.188 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.188 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.188 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.188 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.188 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:50.188 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.188 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.188 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.188 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.188 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.188 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.188 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.189 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.189 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.189 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.189 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.189 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.189 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:50.189 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:50.189 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.189 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.189 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.189 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.190 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:50.190 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:50.190 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:50.190 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.190 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 1, 2024-09-12T23:37:50.190 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:50.190 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.190 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.190 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.190 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:50.190 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:50.190 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:50.191 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.191 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 2, 2024-09-12T23:37:50.191 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.191 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.191 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:50.191 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.191 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.191 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.191 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.661420+0000", 2024-09-12T23:37:50.191 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.868587+0000", 2024-09-12T23:37:50.191 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.868587+0000", 2024-09-12T23:37:50.191 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.868957+0000", 2024-09-12T23:37:50.191 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.661420+0000", 2024-09-12T23:37:50.192 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.192 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.192 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.192 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:50.192 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.661420+0000", 2024-09-12T23:37:50.192 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.661420+0000", 2024-09-12T23:37:50.192 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.661420+0000", 2024-09-12T23:37:50.192 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.192 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.192 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.192 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.661420+0000", 2024-09-12T23:37:50.193 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.661420+0000", 2024-09-12T23:37:50.193 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.193 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:50.193 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.193 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.193 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:50.193 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.193 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.193 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.193 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.193 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:50.193 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.194 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.194 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.194 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.1", 2024-09-12T23:37:50.194 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.194 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.194 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:50.194 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 21, 2024-09-12T23:37:50.194 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.194 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T03:45:32.284209+0000", 2024-09-12T23:37:50.194 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:50.194 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.194 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.195 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:50.195 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.195 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.195 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.195 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.195 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.195 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.195 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.195 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.195 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.195 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.195 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.196 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.196 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.196 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.196 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.196 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:50.196 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:50.196 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.196 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:50.196 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.196 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.196 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.196 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.197 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.197 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:50.197 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.197 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.197 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.197 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.197 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.197 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.197 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.197 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.197 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.197 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.198 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.198 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.198 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:50.198 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:50.198 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.198 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.198 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.198 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.198 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:50.198 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:50.198 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:50.198 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.198 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 2, 2024-09-12T23:37:50.199 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:50.199 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.199 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.199 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.199 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:50.199 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:50.199 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:50.199 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.199 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 1, 2024-09-12T23:37:50.199 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.199 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.199 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:50.200 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.200 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.200 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.200 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.663738+0000", 2024-09-12T23:37:50.200 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.889575+0000", 2024-09-12T23:37:50.200 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.889575+0000", 2024-09-12T23:37:50.200 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.890360+0000", 2024-09-12T23:37:50.200 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.663738+0000", 2024-09-12T23:37:50.200 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.200 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.200 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.201 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:50.201 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.663738+0000", 2024-09-12T23:37:50.201 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.663738+0000", 2024-09-12T23:37:50.201 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.663738+0000", 2024-09-12T23:37:50.201 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.201 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.201 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.201 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.663738+0000", 2024-09-12T23:37:50.201 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.663738+0000", 2024-09-12T23:37:50.201 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.201 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:50.202 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.202 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.202 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:50.202 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.202 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.202 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.202 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.202 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:50.202 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.202 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.202 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.202 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.0", 2024-09-12T23:37:50.203 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.203 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.203 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:50.203 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:50.203 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.203 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T07:36:34.938054+0000", 2024-09-12T23:37:50.203 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:50.203 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.203 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.203 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:50.203 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.203 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.203 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.204 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.204 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.204 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.204 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.204 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.204 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.204 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.204 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.204 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.204 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.204 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.205 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.205 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:50.205 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:50.205 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.205 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:50.205 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.205 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.205 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.205 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.205 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.205 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:50.205 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.206 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.206 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.206 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.206 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.206 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.206 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.206 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.206 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.206 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.206 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.206 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.206 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:50.207 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:50.207 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.207 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.207 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.207 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.207 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:50.207 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:50.207 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:50.207 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.207 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 1, 2024-09-12T23:37:50.207 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:50.207 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.208 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.208 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.208 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:50.208 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:50.208 INFO:tasks.workunit.client.0.smithi136.stdout: 3 2024-09-12T23:37:50.208 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.208 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 1, 2024-09-12T23:37:50.208 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.208 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.208 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 56, 2024-09-12T23:37:50.208 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.208 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.209 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.209 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.663553+0000", 2024-09-12T23:37:50.209 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:34:49.787490+0000", 2024-09-12T23:37:50.209 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:34:49.787490+0000", 2024-09-12T23:37:50.209 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:34:59.548775+0000", 2024-09-12T23:37:50.209 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.663553+0000", 2024-09-12T23:37:50.209 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:34:48.454503+0000", 2024-09-12T23:37:50.209 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.209 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:34:48.454503+0000", 2024-09-12T23:37:50.209 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 57, 2024-09-12T23:37:50.209 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.663553+0000", 2024-09-12T23:37:50.209 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.663553+0000", 2024-09-12T23:37:50.210 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.663553+0000", 2024-09-12T23:37:50.210 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.210 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.210 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:34:48.454503+0000", 2024-09-12T23:37:50.210 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.663553+0000", 2024-09-12T23:37:50.210 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.663553+0000", 2024-09-12T23:37:50.210 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.210 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:50.210 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.210 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.210 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 56, 2024-09-12T23:37:50.210 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.211 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.211 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.211 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.211 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:50.211 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.211 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.211 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.211 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "4.4", 2024-09-12T23:37:50.211 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.211 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.211 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:50.212 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 21, 2024-09-12T23:37:50.212 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.212 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T06:19:10.283468+0000", 2024-09-12T23:37:50.212 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.002051994, 2024-09-12T23:37:50.212 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.212 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.212 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:50.212 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.212 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.212 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.212 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.212 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.213 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.213 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.213 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.213 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.213 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.213 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.213 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.213 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.213 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.213 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.213 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:50.213 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:50.214 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.214 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:50.214 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.214 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.214 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.214 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.214 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.214 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:50.214 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.214 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.214 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.214 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.215 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.215 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.215 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.215 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.215 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.215 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.215 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.215 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.215 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:50.215 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:50.215 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.215 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.216 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.216 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.216 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:50.216 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:50.216 INFO:tasks.workunit.client.0.smithi136.stdout: 3 2024-09-12T23:37:50.216 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.216 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 1, 2024-09-12T23:37:50.216 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:50.216 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.216 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.216 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.216 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:50.217 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:50.217 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:50.217 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.217 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 5, 2024-09-12T23:37:50.217 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.217 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.217 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:50.217 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.217 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.217 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.217 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.671197+0000", 2024-09-12T23:37:50.217 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.899410+0000", 2024-09-12T23:37:50.218 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.899410+0000", 2024-09-12T23:37:50.218 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.899931+0000", 2024-09-12T23:37:50.218 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.671197+0000", 2024-09-12T23:37:50.218 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.218 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.218 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.218 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:50.218 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.671197+0000", 2024-09-12T23:37:50.218 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.671197+0000", 2024-09-12T23:37:50.218 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.671197+0000", 2024-09-12T23:37:50.218 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.218 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.218 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.219 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.671197+0000", 2024-09-12T23:37:50.219 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.671197+0000", 2024-09-12T23:37:50.219 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.219 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:50.219 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.219 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.219 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:50.219 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.219 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.219 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.219 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.219 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:50.220 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.220 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.220 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.220 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.2", 2024-09-12T23:37:50.220 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.220 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.220 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:50.220 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 21, 2024-09-12T23:37:50.220 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.220 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T11:05:26.671172+0000", 2024-09-12T23:37:50.220 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:50.221 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.221 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.221 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:50.221 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.221 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.221 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.221 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.221 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.221 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.221 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.221 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.221 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.222 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.222 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.222 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.222 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.222 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.222 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.222 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:50.222 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:50.222 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.222 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:50.222 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.222 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.222 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.223 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.223 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.223 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:50.223 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.223 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.223 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.223 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.223 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.223 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.223 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.223 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.224 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.224 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.224 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.224 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.224 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:50.224 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:50.224 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.224 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.224 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.224 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.224 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:50.224 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:50.224 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:50.225 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.225 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 5, 2024-09-12T23:37:50.225 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:50.225 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.225 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.225 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.225 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:50.225 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:50.225 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:50.225 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.225 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 4, 2024-09-12T23:37:50.225 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.226 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.226 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:50.226 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.226 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.226 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.226 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.661569+0000", 2024-09-12T23:37:50.226 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.902561+0000", 2024-09-12T23:37:50.226 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.902561+0000", 2024-09-12T23:37:50.226 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.902878+0000", 2024-09-12T23:37:50.226 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.661569+0000", 2024-09-12T23:37:50.226 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.226 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.226 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.227 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:50.227 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.661569+0000", 2024-09-12T23:37:50.227 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.661569+0000", 2024-09-12T23:37:50.227 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.661569+0000", 2024-09-12T23:37:50.227 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.227 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.227 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.227 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.661569+0000", 2024-09-12T23:37:50.227 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.661569+0000", 2024-09-12T23:37:50.227 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.227 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:50.228 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.228 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.228 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:50.228 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.228 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.228 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.228 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.228 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:50.228 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.228 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.228 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.228 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.3", 2024-09-12T23:37:50.228 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.229 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.229 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:50.229 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:50.229 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.229 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T08:12:55.352139+0000", 2024-09-12T23:37:50.229 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:50.229 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.229 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.229 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:50.229 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.229 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.229 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.230 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.230 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.230 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.230 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.230 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.230 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.230 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.230 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.230 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.230 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.230 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.230 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.231 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:50.231 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:50.231 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.231 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:50.231 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.231 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.231 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.231 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.231 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.231 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:50.231 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.231 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.231 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.232 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.232 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.232 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.232 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.232 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.232 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.232 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.232 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.232 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.232 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:50.232 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:50.232 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.233 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.233 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.233 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.233 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:50.233 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:50.233 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:50.233 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.233 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 4, 2024-09-12T23:37:50.233 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:50.233 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.233 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.233 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.234 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:50.234 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:50.234 INFO:tasks.workunit.client.0.smithi136.stdout: 4 2024-09-12T23:37:50.234 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.234 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 1, 2024-09-12T23:37:50.234 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.234 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.234 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 56, 2024-09-12T23:37:50.234 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.234 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.234 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.234 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.664284+0000", 2024-09-12T23:37:50.234 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:34:49.792105+0000", 2024-09-12T23:37:50.235 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:34:49.792105+0000", 2024-09-12T23:37:50.235 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:34:59.548734+0000", 2024-09-12T23:37:50.235 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.664284+0000", 2024-09-12T23:37:50.235 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:34:48.454503+0000", 2024-09-12T23:37:50.235 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.235 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:34:48.454503+0000", 2024-09-12T23:37:50.235 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 57, 2024-09-12T23:37:50.235 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.664284+0000", 2024-09-12T23:37:50.235 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.664284+0000", 2024-09-12T23:37:50.235 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.664284+0000", 2024-09-12T23:37:50.235 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.235 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.236 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:34:48.454503+0000", 2024-09-12T23:37:50.236 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.664284+0000", 2024-09-12T23:37:50.236 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.664284+0000", 2024-09-12T23:37:50.236 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.236 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 2, 2024-09-12T23:37:50.236 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.236 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.236 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 56, 2024-09-12T23:37:50.236 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.236 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.236 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.237 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.237 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 2, 2024-09-12T23:37:50.237 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.237 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.237 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.237 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "4.2", 2024-09-12T23:37:50.237 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.237 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.237 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:50.237 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 23, 2024-09-12T23:37:50.237 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.237 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-13T23:37:40.496016+0000", 2024-09-12T23:37:50.238 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.002201358, 2024-09-12T23:37:50.238 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.238 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.238 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 19, 2024-09-12T23:37:50.238 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.238 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.238 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.238 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.238 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.238 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.238 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.238 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.239 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.239 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.239 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.239 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.239 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.239 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.239 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.239 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 3, 2024-09-12T23:37:50.239 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 1, 2024-09-12T23:37:50.239 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.239 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 1, 2024-09-12T23:37:50.240 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.240 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.240 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.240 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.240 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.240 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:50.240 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.240 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.240 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.240 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.240 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.241 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.241 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.241 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.241 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.241 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.241 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.241 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.241 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 2, 2024-09-12T23:37:50.241 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 2 2024-09-12T23:37:50.241 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.241 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.241 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.242 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.242 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:50.242 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:50.242 INFO:tasks.workunit.client.0.smithi136.stdout: 4 2024-09-12T23:37:50.242 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.242 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 1, 2024-09-12T23:37:50.242 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "61'2" 2024-09-12T23:37:50.242 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.242 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.242 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.242 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:50.242 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:50.243 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:50.243 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.243 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 1, 2024-09-12T23:37:50.243 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.243 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.243 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:50.243 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.243 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.243 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.243 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.664823+0000", 2024-09-12T23:37:50.243 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.867020+0000", 2024-09-12T23:37:50.243 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.867020+0000", 2024-09-12T23:37:50.244 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.867304+0000", 2024-09-12T23:37:50.244 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.664823+0000", 2024-09-12T23:37:50.244 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.244 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.244 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.244 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:50.244 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.664823+0000", 2024-09-12T23:37:50.244 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.664823+0000", 2024-09-12T23:37:50.244 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.664823+0000", 2024-09-12T23:37:50.244 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.244 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.245 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.245 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.664823+0000", 2024-09-12T23:37:50.245 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.664823+0000", 2024-09-12T23:37:50.245 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.245 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:50.245 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.245 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.245 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:50.245 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.245 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.245 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.245 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.246 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:50.246 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.246 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.246 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.246 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.4", 2024-09-12T23:37:50.246 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.246 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.246 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:50.246 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 21, 2024-09-12T23:37:50.246 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.246 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T09:51:22.908188+0000", 2024-09-12T23:37:50.247 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:50.247 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.247 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.247 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:50.247 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.247 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.247 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.247 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.247 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.247 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.247 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.248 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.248 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.248 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.248 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.248 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.248 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.248 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.248 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.248 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:50.248 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:50.248 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.248 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:50.249 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.249 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.249 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.249 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.249 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.249 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:50.249 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.249 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.249 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.249 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.249 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.250 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.250 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.250 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.250 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.250 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.250 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.250 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.250 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:50.250 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:50.250 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.250 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.250 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.251 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.251 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:50.251 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:50.251 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:50.251 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.251 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 1, 2024-09-12T23:37:50.251 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:50.251 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.251 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.251 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.251 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:50.252 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:50.252 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:50.252 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.252 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 5, 2024-09-12T23:37:50.252 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.252 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.252 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:50.252 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.252 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.252 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.252 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.670703+0000", 2024-09-12T23:37:50.252 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.914385+0000", 2024-09-12T23:37:50.253 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.914385+0000", 2024-09-12T23:37:50.253 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.914728+0000", 2024-09-12T23:37:50.253 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.670703+0000", 2024-09-12T23:37:50.253 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.253 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.253 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.253 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:50.253 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.670703+0000", 2024-09-12T23:37:50.254 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.670703+0000", 2024-09-12T23:37:50.254 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.670703+0000", 2024-09-12T23:37:50.254 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.254 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.254 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.254 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.670703+0000", 2024-09-12T23:37:50.254 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.670703+0000", 2024-09-12T23:37:50.254 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.254 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:50.254 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.254 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.254 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:50.255 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.255 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.255 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.255 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.255 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:50.255 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.255 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.255 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.255 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.5", 2024-09-12T23:37:50.255 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.255 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.256 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:50.256 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:50.256 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.256 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T03:45:02.815733+0000", 2024-09-12T23:37:50.256 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:50.256 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.256 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.256 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:50.256 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.256 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.256 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.257 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.257 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.257 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.257 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.257 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.257 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.257 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.257 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.257 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.257 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.257 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.257 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.258 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:50.258 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:50.258 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.258 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:50.258 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.258 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.258 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.258 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.258 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.258 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:50.258 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.259 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.259 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.259 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.259 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.259 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.259 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.259 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.259 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.259 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.259 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.260 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.260 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:50.260 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:50.260 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.260 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.260 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.260 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.260 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:50.260 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:50.260 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:50.260 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.260 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 5, 2024-09-12T23:37:50.261 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:50.261 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.261 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.261 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.261 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:50.261 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:50.261 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:50.261 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.261 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 0, 2024-09-12T23:37:50.261 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.261 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.262 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 56, 2024-09-12T23:37:50.262 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.262 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.262 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.262 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.669319+0000", 2024-09-12T23:37:50.262 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:34:49.787835+0000", 2024-09-12T23:37:50.262 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:34:49.787835+0000", 2024-09-12T23:37:50.262 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:34:59.541122+0000", 2024-09-12T23:37:50.262 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.669319+0000", 2024-09-12T23:37:50.262 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:34:48.454503+0000", 2024-09-12T23:37:50.262 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.262 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:34:48.454503+0000", 2024-09-12T23:37:50.263 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 57, 2024-09-12T23:37:50.263 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.669319+0000", 2024-09-12T23:37:50.263 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.669319+0000", 2024-09-12T23:37:50.263 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.669319+0000", 2024-09-12T23:37:50.263 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.263 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.263 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:34:48.454503+0000", 2024-09-12T23:37:50.263 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.669319+0000", 2024-09-12T23:37:50.263 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.669319+0000", 2024-09-12T23:37:50.263 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.263 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 1, 2024-09-12T23:37:50.264 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.264 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.264 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 56, 2024-09-12T23:37:50.264 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.264 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.264 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.264 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.264 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 1, 2024-09-12T23:37:50.264 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.264 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.264 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.265 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "4.3", 2024-09-12T23:37:50.265 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.265 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.265 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:50.265 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 22, 2024-09-12T23:37:50.265 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.265 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T07:07:48.849192+0000", 2024-09-12T23:37:50.265 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.000992265, 2024-09-12T23:37:50.265 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.265 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.265 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:50.265 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.266 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.266 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.266 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.266 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.266 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.266 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.266 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.266 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.266 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.266 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.266 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.267 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.267 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.267 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.267 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 3, 2024-09-12T23:37:50.267 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 1, 2024-09-12T23:37:50.267 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.267 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 1, 2024-09-12T23:37:50.267 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.267 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.267 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.267 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.267 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.268 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:50.268 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.268 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.268 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.268 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.268 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.268 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.268 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.268 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.268 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.268 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.269 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.269 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.269 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:50.269 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:50.269 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.269 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.269 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.269 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.269 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:50.269 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:50.269 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:50.270 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.270 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 0, 2024-09-12T23:37:50.270 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "59'1" 2024-09-12T23:37:50.270 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.270 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.270 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.270 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:50.270 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:50.270 INFO:tasks.workunit.client.0.smithi136.stdout: 4 2024-09-12T23:37:50.270 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.270 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 7, 2024-09-12T23:37:50.270 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.271 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.271 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:50.271 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.271 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.271 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.271 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.666213+0000", 2024-09-12T23:37:50.271 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.892268+0000", 2024-09-12T23:37:50.271 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.892268+0000", 2024-09-12T23:37:50.271 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.895077+0000", 2024-09-12T23:37:50.271 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.666213+0000", 2024-09-12T23:37:50.271 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.271 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.272 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.272 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:50.272 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.666213+0000", 2024-09-12T23:37:50.272 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.666213+0000", 2024-09-12T23:37:50.272 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.666213+0000", 2024-09-12T23:37:50.272 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.272 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.272 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.272 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.666213+0000", 2024-09-12T23:37:50.272 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.666213+0000", 2024-09-12T23:37:50.272 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.273 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:50.273 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.273 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.273 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:50.273 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.273 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.273 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.273 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.273 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:50.273 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.273 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.273 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.274 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.5", 2024-09-12T23:37:50.274 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.274 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.274 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:50.274 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 21, 2024-09-12T23:37:50.274 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.274 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T04:26:19.125877+0000", 2024-09-12T23:37:50.274 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:50.274 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.274 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.274 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:50.275 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.275 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.275 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.275 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.275 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.275 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.275 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.275 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.275 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.275 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.275 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.276 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.276 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.276 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.276 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.276 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:50.276 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:50.276 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.276 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:50.276 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.276 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.277 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.277 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.277 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.277 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:50.277 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.277 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.277 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.277 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.277 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.278 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.278 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.278 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.278 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.278 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.278 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.278 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.278 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:50.278 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:50.278 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.278 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.278 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.279 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.279 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:50.279 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:50.279 INFO:tasks.workunit.client.0.smithi136.stdout: 4 2024-09-12T23:37:50.279 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.279 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 7, 2024-09-12T23:37:50.279 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:50.279 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.279 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.279 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.279 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:50.280 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:50.280 INFO:tasks.workunit.client.0.smithi136.stdout: 5 2024-09-12T23:37:50.280 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.280 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 1, 2024-09-12T23:37:50.280 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.280 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.280 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:50.280 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.280 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.280 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.280 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.664444+0000", 2024-09-12T23:37:50.281 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.899381+0000", 2024-09-12T23:37:50.281 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.899381+0000", 2024-09-12T23:37:50.281 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.899700+0000", 2024-09-12T23:37:50.281 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.664444+0000", 2024-09-12T23:37:50.281 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.281 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.281 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.281 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:50.281 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.664444+0000", 2024-09-12T23:37:50.281 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.664444+0000", 2024-09-12T23:37:50.281 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.664444+0000", 2024-09-12T23:37:50.281 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.282 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.282 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.282 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.664444+0000", 2024-09-12T23:37:50.282 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.664444+0000", 2024-09-12T23:37:50.282 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.282 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:50.282 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.282 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.282 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:50.282 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.282 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.283 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.283 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.283 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:50.283 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.283 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.283 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.283 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.4", 2024-09-12T23:37:50.283 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.283 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.283 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:50.283 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:50.283 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.284 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T09:30:20.590373+0000", 2024-09-12T23:37:50.284 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:50.284 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.284 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.284 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:50.284 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.284 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.284 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.284 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.284 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.284 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.285 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.285 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.285 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.285 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.285 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.285 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.285 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.285 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.285 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.285 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:50.285 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:50.286 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.286 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:50.286 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.286 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.286 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.286 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.286 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.286 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:50.286 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.286 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.286 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.286 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.287 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.287 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.287 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.287 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.287 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.287 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.287 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.287 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.287 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:50.287 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:50.287 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.287 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.287 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.288 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.288 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:50.288 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:50.288 INFO:tasks.workunit.client.0.smithi136.stdout: 5 2024-09-12T23:37:50.288 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.288 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 1, 2024-09-12T23:37:50.288 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:50.288 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.288 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.288 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.288 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:50.288 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:50.289 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:50.289 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.289 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 3, 2024-09-12T23:37:50.289 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.289 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.289 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 56, 2024-09-12T23:37:50.289 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.289 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.289 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.289 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.666658+0000", 2024-09-12T23:37:50.290 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:34:49.736609+0000", 2024-09-12T23:37:50.290 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:34:49.736609+0000", 2024-09-12T23:37:50.290 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:34:59.899761+0000", 2024-09-12T23:37:50.290 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.666658+0000", 2024-09-12T23:37:50.290 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:34:48.454503+0000", 2024-09-12T23:37:50.290 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.290 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:34:48.454503+0000", 2024-09-12T23:37:50.290 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 57, 2024-09-12T23:37:50.290 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.666658+0000", 2024-09-12T23:37:50.290 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.666658+0000", 2024-09-12T23:37:50.290 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.666658+0000", 2024-09-12T23:37:50.290 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.291 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.291 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:34:48.454503+0000", 2024-09-12T23:37:50.291 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.666658+0000", 2024-09-12T23:37:50.291 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.666658+0000", 2024-09-12T23:37:50.291 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.291 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:50.291 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.291 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.291 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 56, 2024-09-12T23:37:50.291 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.291 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.291 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.292 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.292 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:50.292 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.292 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.292 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.292 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "4.0", 2024-09-12T23:37:50.292 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.292 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.292 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:50.292 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 21, 2024-09-12T23:37:50.292 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.292 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T06:01:06.682284+0000", 2024-09-12T23:37:50.293 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.001798914, 2024-09-12T23:37:50.293 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.293 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.293 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:50.293 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.293 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.293 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.293 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.293 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.293 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.293 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.293 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.294 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.294 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.294 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.294 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.294 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.294 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.294 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.294 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:50.294 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:50.294 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.294 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:50.295 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.295 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.295 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.295 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.295 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.295 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:50.295 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.295 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.295 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.295 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.295 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.296 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.296 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.296 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.296 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.296 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.296 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.296 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.296 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:50.296 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:50.296 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.297 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.297 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.297 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.297 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:50.297 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:50.297 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:50.297 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.297 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 3, 2024-09-12T23:37:50.297 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:50.297 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.297 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.298 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.298 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:50.298 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:50.298 INFO:tasks.workunit.client.0.smithi136.stdout: 4 2024-09-12T23:37:50.298 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.298 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 1, 2024-09-12T23:37:50.298 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.298 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.298 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:50.299 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.299 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.299 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.299 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.665023+0000", 2024-09-12T23:37:50.299 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.866005+0000", 2024-09-12T23:37:50.299 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.866005+0000", 2024-09-12T23:37:50.299 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.866367+0000", 2024-09-12T23:37:50.299 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.665023+0000", 2024-09-12T23:37:50.299 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.299 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.299 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.300 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:50.300 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.665023+0000", 2024-09-12T23:37:50.300 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.665023+0000", 2024-09-12T23:37:50.300 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.665023+0000", 2024-09-12T23:37:50.300 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.300 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.300 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.300 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.665023+0000", 2024-09-12T23:37:50.300 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.665023+0000", 2024-09-12T23:37:50.300 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.301 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 1, 2024-09-12T23:37:50.301 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.301 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.301 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:50.301 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.301 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.301 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.301 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.301 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 1, 2024-09-12T23:37:50.301 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.301 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.301 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.302 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.6", 2024-09-12T23:37:50.302 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.302 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.302 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:50.302 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 22, 2024-09-12T23:37:50.302 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.302 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T05:49:27.803045+0000", 2024-09-12T23:37:50.302 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:50.302 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.302 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.302 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:50.302 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.303 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.303 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.303 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.303 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.303 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.303 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.303 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.303 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.303 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.303 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.303 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.304 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.304 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.304 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.304 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 3, 2024-09-12T23:37:50.304 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 1, 2024-09-12T23:37:50.304 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.304 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 1, 2024-09-12T23:37:50.304 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.304 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.304 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.304 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.305 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.305 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 1, 2024-09-12T23:37:50.305 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.305 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.305 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.305 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.305 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.305 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.305 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.305 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.305 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.305 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.306 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.306 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.306 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 1, 2024-09-12T23:37:50.306 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:50.306 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.306 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.306 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.306 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.306 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:50.306 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:50.307 INFO:tasks.workunit.client.0.smithi136.stdout: 4 2024-09-12T23:37:50.307 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.307 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 1, 2024-09-12T23:37:50.307 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "54'1" 2024-09-12T23:37:50.307 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.307 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.307 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.307 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:50.307 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:50.307 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:50.307 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.308 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 3, 2024-09-12T23:37:50.308 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.308 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.308 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:50.308 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.308 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.308 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.308 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.673648+0000", 2024-09-12T23:37:50.308 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.914934+0000", 2024-09-12T23:37:50.308 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.914934+0000", 2024-09-12T23:37:50.308 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.915275+0000", 2024-09-12T23:37:50.308 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.673648+0000", 2024-09-12T23:37:50.309 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.309 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.309 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.309 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:50.309 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.673648+0000", 2024-09-12T23:37:50.309 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.673648+0000", 2024-09-12T23:37:50.309 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.673648+0000", 2024-09-12T23:37:50.309 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.309 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.309 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.310 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.673648+0000", 2024-09-12T23:37:50.310 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.673648+0000", 2024-09-12T23:37:50.310 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.310 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:50.310 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.310 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.310 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:50.310 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.310 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.310 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.310 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.311 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:50.311 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.311 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.311 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.311 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.7", 2024-09-12T23:37:50.311 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.311 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.311 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:50.311 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:50.311 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.311 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T06:43:27.562543+0000", 2024-09-12T23:37:50.312 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:50.312 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.312 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.312 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:50.312 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.312 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.312 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.312 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.312 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.312 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.312 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.313 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.313 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.313 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.313 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.313 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.313 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.313 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.313 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.313 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:50.313 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:50.313 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.313 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:50.314 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.314 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.314 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.314 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.314 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.314 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:50.314 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.314 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.314 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.314 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.314 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.314 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.314 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.315 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.315 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.315 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.315 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.315 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.315 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:50.315 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:50.315 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.315 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.315 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.315 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.315 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:50.316 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:50.316 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:50.316 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.316 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 3, 2024-09-12T23:37:50.316 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:50.316 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.316 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.316 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.316 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:50.316 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:50.316 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:50.317 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.317 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 4, 2024-09-12T23:37:50.317 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.317 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.317 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 56, 2024-09-12T23:37:50.317 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.317 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.317 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.317 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.661581+0000", 2024-09-12T23:37:50.317 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:34:49.785409+0000", 2024-09-12T23:37:50.317 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:34:49.785409+0000", 2024-09-12T23:37:50.318 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:34:59.550136+0000", 2024-09-12T23:37:50.318 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.661581+0000", 2024-09-12T23:37:50.318 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:34:48.454503+0000", 2024-09-12T23:37:50.318 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.318 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:34:48.454503+0000", 2024-09-12T23:37:50.318 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 57, 2024-09-12T23:37:50.318 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.661581+0000", 2024-09-12T23:37:50.320 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.661581+0000", 2024-09-12T23:37:50.320 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.661581+0000", 2024-09-12T23:37:50.320 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.320 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.320 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:34:48.454503+0000", 2024-09-12T23:37:50.320 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.661581+0000", 2024-09-12T23:37:50.320 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.661581+0000", 2024-09-12T23:37:50.320 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.321 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:50.321 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.321 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.321 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 56, 2024-09-12T23:37:50.321 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.321 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.321 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.321 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.321 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:50.321 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.322 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.322 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.322 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "4.1", 2024-09-12T23:37:50.322 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.322 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.322 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:50.322 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 21, 2024-09-12T23:37:50.322 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.322 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T07:30:29.736480+0000", 2024-09-12T23:37:50.322 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.00101623, 2024-09-12T23:37:50.322 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.322 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.322 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:50.323 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.323 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.323 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.323 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.323 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.323 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.323 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.323 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.323 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.323 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.323 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.323 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.324 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.324 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.324 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.324 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:50.324 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:50.324 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.324 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:50.324 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.324 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.324 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.324 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.325 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.325 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:50.325 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.325 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.325 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.325 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.325 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.325 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.325 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.325 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.325 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.326 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.326 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.326 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.326 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:50.326 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:50.326 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.326 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.326 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.326 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.326 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:50.326 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:50.327 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:50.327 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.327 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 4, 2024-09-12T23:37:50.327 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:50.327 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.327 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.327 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.327 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:50.328 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:50.328 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:50.328 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.328 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 6, 2024-09-12T23:37:50.328 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.328 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.328 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:50.328 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.328 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.328 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.329 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.665837+0000", 2024-09-12T23:37:50.329 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.886573+0000", 2024-09-12T23:37:50.329 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.886573+0000", 2024-09-12T23:37:50.329 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.887351+0000", 2024-09-12T23:37:50.329 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.665837+0000", 2024-09-12T23:37:50.329 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.329 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.329 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.329 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:50.329 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.665837+0000", 2024-09-12T23:37:50.330 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.665837+0000", 2024-09-12T23:37:50.330 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.665837+0000", 2024-09-12T23:37:50.330 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.330 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.330 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.330 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.665837+0000", 2024-09-12T23:37:50.330 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.665837+0000", 2024-09-12T23:37:50.330 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.330 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 2, 2024-09-12T23:37:50.330 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.330 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.331 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:50.331 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.331 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.331 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.331 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.331 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 2, 2024-09-12T23:37:50.331 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.331 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.331 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.331 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.7", 2024-09-12T23:37:50.331 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.332 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.332 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:50.332 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 23, 2024-09-12T23:37:50.332 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.332 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T06:58:40.820895+0000", 2024-09-12T23:37:50.332 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:50.332 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.332 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.332 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:50.332 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.332 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.332 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.333 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.333 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.333 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.333 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.333 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.333 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.333 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.333 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.333 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.333 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.334 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.334 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.334 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 3, 2024-09-12T23:37:50.334 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 1, 2024-09-12T23:37:50.334 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.334 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 1, 2024-09-12T23:37:50.334 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.334 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.334 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.334 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.334 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.335 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 1, 2024-09-12T23:37:50.335 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.335 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.335 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.335 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.335 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.335 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.335 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.335 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.335 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.335 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.335 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.336 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.336 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 3, 2024-09-12T23:37:50.336 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:50.336 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.336 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.336 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.336 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.336 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:50.336 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:50.336 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:50.336 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.336 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 6, 2024-09-12T23:37:50.337 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "53'2" 2024-09-12T23:37:50.337 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.337 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.337 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.337 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:50.337 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:50.337 INFO:tasks.workunit.client.0.smithi136.stdout: 4 2024-09-12T23:37:50.337 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.337 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 0, 2024-09-12T23:37:50.337 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.337 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.337 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:50.338 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.338 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.338 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.338 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.669756+0000", 2024-09-12T23:37:50.338 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.906278+0000", 2024-09-12T23:37:50.338 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.906278+0000", 2024-09-12T23:37:50.338 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.908474+0000", 2024-09-12T23:37:50.338 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.669756+0000", 2024-09-12T23:37:50.338 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.338 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.338 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.338 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:50.339 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.669756+0000", 2024-09-12T23:37:50.339 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.669756+0000", 2024-09-12T23:37:50.339 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.669756+0000", 2024-09-12T23:37:50.339 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.339 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.339 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.339 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.669756+0000", 2024-09-12T23:37:50.339 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.669756+0000", 2024-09-12T23:37:50.339 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.339 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:50.339 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.339 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.340 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:50.340 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.340 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.340 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.340 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.340 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:50.340 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.340 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.340 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.340 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.6", 2024-09-12T23:37:50.340 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.341 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.341 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:50.341 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:50.341 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.341 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T08:27:51.832430+0000", 2024-09-12T23:37:50.341 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:50.341 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.341 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.341 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:50.341 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.341 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.341 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.342 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.342 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.342 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.342 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.342 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.342 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.342 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.342 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.342 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.342 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.342 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.342 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.343 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:50.343 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:50.343 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.343 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:50.343 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.343 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.343 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.343 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.343 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.343 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:50.343 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.343 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.344 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.344 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.344 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.344 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.344 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.344 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.344 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.344 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.344 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.344 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.344 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:50.345 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:50.345 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.345 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.345 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.345 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.345 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:50.345 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:50.345 INFO:tasks.workunit.client.0.smithi136.stdout: 4 2024-09-12T23:37:50.345 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.345 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 0, 2024-09-12T23:37:50.345 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:50.345 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.346 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.346 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.346 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:50.346 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:50.346 INFO:tasks.workunit.client.0.smithi136.stdout: 1 2024-09-12T23:37:50.346 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.346 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 7, 2024-09-12T23:37:50.346 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.346 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.346 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:50.346 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.346 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.347 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.347 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.664547+0000", 2024-09-12T23:37:50.347 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.893861+0000", 2024-09-12T23:37:50.347 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.893861+0000", 2024-09-12T23:37:50.347 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.894286+0000", 2024-09-12T23:37:50.347 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.664547+0000", 2024-09-12T23:37:50.347 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.347 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.347 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.347 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:50.347 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.664547+0000", 2024-09-12T23:37:50.347 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.664547+0000", 2024-09-12T23:37:50.348 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.664547+0000", 2024-09-12T23:37:50.348 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.348 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.348 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.348 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.664547+0000", 2024-09-12T23:37:50.348 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.664547+0000", 2024-09-12T23:37:50.348 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.348 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:50.348 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.348 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.348 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:50.349 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.349 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.349 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.349 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.349 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:50.349 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.349 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.349 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.349 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.8", 2024-09-12T23:37:50.349 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.349 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.350 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:50.350 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 21, 2024-09-12T23:37:50.350 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.350 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T09:20:01.806723+0000", 2024-09-12T23:37:50.350 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:50.350 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.350 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.350 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:50.350 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.350 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.350 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.350 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.351 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.351 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.351 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.351 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.351 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.351 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.351 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.351 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.351 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.351 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.351 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.351 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:50.352 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:50.352 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.352 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:50.352 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.352 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.352 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.352 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.352 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.352 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:50.352 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.352 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.353 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.353 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.353 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.353 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.353 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.353 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.353 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.353 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.353 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.353 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.353 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:50.353 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:50.354 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.354 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.354 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.354 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.354 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:50.354 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:50.354 INFO:tasks.workunit.client.0.smithi136.stdout: 1 2024-09-12T23:37:50.354 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.354 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 7, 2024-09-12T23:37:50.354 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:50.354 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.355 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.355 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.355 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:50.355 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:50.355 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:50.355 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.355 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 1, 2024-09-12T23:37:50.355 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.355 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.355 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:50.355 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.355 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.356 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.356 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:09.676563+0000", 2024-09-12T23:37:50.356 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:35:09.675322+0000", 2024-09-12T23:37:50.356 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:35:09.675322+0000", 2024-09-12T23:37:50.356 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:35:09.676563+0000", 2024-09-12T23:37:50.356 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:09.676563+0000", 2024-09-12T23:37:50.356 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.356 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.356 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.356 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 64, 2024-09-12T23:37:50.356 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:09.676563+0000", 2024-09-12T23:37:50.357 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:09.676563+0000", 2024-09-12T23:37:50.357 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:09.676563+0000", 2024-09-12T23:37:50.357 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.357 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.357 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.357 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:09.676563+0000", 2024-09-12T23:37:50.357 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:09.676563+0000", 2024-09-12T23:37:50.357 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.357 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:50.357 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.357 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.357 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 63, 2024-09-12T23:37:50.358 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.358 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.358 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.358 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.358 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:50.358 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.358 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.358 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.358 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.9", 2024-09-12T23:37:50.358 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.358 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.359 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 64, 2024-09-12T23:37:50.359 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 10, 2024-09-12T23:37:50.359 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.359 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T03:11:37.262719+0000", 2024-09-12T23:37:50.359 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:50.359 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.359 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.359 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:50.359 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.359 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.359 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.359 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.360 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.360 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.360 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.360 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.360 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.360 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.360 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.360 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.360 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.360 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.360 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.361 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:50.361 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:50.361 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.361 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:50.361 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.361 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.361 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.361 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.361 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.361 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:50.361 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.361 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.362 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.362 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.362 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.362 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.362 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.362 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.362 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.362 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.362 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.362 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.362 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:50.363 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:50.363 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.363 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.363 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.363 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.363 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:50.363 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:50.363 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:50.363 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.363 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 1, 2024-09-12T23:37:50.363 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:50.363 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.364 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.364 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.364 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:50.364 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:50.364 INFO:tasks.workunit.client.0.smithi136.stdout: 3 2024-09-12T23:37:50.364 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.364 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 1, 2024-09-12T23:37:50.364 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.364 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.364 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:50.364 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.365 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.365 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.365 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.664006+0000", 2024-09-12T23:37:50.365 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.884027+0000", 2024-09-12T23:37:50.365 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.884027+0000", 2024-09-12T23:37:50.365 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.884360+0000", 2024-09-12T23:37:50.365 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.664006+0000", 2024-09-12T23:37:50.365 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.365 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.365 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.365 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:50.365 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.664006+0000", 2024-09-12T23:37:50.366 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.664006+0000", 2024-09-12T23:37:50.366 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.664006+0000", 2024-09-12T23:37:50.366 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.366 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.366 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.366 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.664006+0000", 2024-09-12T23:37:50.366 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.664006+0000", 2024-09-12T23:37:50.366 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.366 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:50.366 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.366 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.367 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:50.367 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.367 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.367 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.367 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.367 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:50.367 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.367 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.367 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.367 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.9", 2024-09-12T23:37:50.367 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.368 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.368 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:50.368 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 21, 2024-09-12T23:37:50.368 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.368 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T03:11:57.567074+0000", 2024-09-12T23:37:50.368 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:50.368 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.368 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.368 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:50.368 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.368 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.369 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.369 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.369 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.369 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.369 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.369 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.369 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.369 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.369 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.369 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.369 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.370 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.370 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.370 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:50.370 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:50.370 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.370 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:50.370 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.370 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.370 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.370 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.370 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.371 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:50.371 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.371 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.371 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.371 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.371 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.371 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.371 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.371 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.371 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.371 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.371 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.371 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.372 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:50.372 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:50.372 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.372 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.372 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.372 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.372 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:50.372 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:50.372 INFO:tasks.workunit.client.0.smithi136.stdout: 3 2024-09-12T23:37:50.372 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.372 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 1, 2024-09-12T23:37:50.373 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:50.373 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.373 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.373 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.373 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:50.373 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:50.373 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:50.373 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.373 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 3, 2024-09-12T23:37:50.373 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.373 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.374 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:50.374 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.374 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.374 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.374 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.667167+0000", 2024-09-12T23:37:50.374 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.914543+0000", 2024-09-12T23:37:50.374 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.914543+0000", 2024-09-12T23:37:50.374 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.915243+0000", 2024-09-12T23:37:50.374 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.667167+0000", 2024-09-12T23:37:50.374 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.374 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.374 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.375 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:50.375 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.667167+0000", 2024-09-12T23:37:50.375 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.667167+0000", 2024-09-12T23:37:50.375 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.667167+0000", 2024-09-12T23:37:50.375 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.375 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.375 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.375 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.667167+0000", 2024-09-12T23:37:50.375 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.667167+0000", 2024-09-12T23:37:50.375 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.375 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:50.376 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.376 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.376 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:50.376 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.376 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.376 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.376 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.376 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:50.376 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.376 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.376 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.377 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.8", 2024-09-12T23:37:50.377 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.377 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.377 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:50.377 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:50.377 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.377 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-13T23:48:16.023985+0000", 2024-09-12T23:37:50.377 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:50.377 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.377 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.377 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:50.378 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.378 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.378 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.378 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.378 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.378 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.378 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.378 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.378 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.378 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.378 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.379 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.379 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.379 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.379 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.379 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:50.379 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:50.379 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.379 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:50.379 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.379 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.379 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.379 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.380 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.380 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:50.380 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.380 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.380 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.380 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.380 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.380 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.380 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.380 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.380 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.381 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.381 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.381 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.381 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:50.381 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:50.381 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.381 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.381 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.381 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.381 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:50.381 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:50.382 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:50.382 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.382 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 3, 2024-09-12T23:37:50.382 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:50.382 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.382 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.382 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.382 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:50.382 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:50.382 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:50.382 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.382 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 1, 2024-09-12T23:37:50.383 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.383 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.383 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:50.383 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.383 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.383 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.383 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.664752+0000", 2024-09-12T23:37:50.383 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.883276+0000", 2024-09-12T23:37:50.383 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.883276+0000", 2024-09-12T23:37:50.383 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.883601+0000", 2024-09-12T23:37:50.383 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.664752+0000", 2024-09-12T23:37:50.384 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.384 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.384 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.384 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:50.384 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.664752+0000", 2024-09-12T23:37:50.384 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.664752+0000", 2024-09-12T23:37:50.384 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.664752+0000", 2024-09-12T23:37:50.384 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.384 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.384 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.384 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.664752+0000", 2024-09-12T23:37:50.385 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.664752+0000", 2024-09-12T23:37:50.385 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.385 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:50.385 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.385 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.385 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:50.385 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.385 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.385 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.385 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.385 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:50.386 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.386 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.386 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.386 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.a", 2024-09-12T23:37:50.386 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.386 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.386 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:50.386 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 21, 2024-09-12T23:37:50.386 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.386 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T00:26:21.733615+0000", 2024-09-12T23:37:50.386 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:50.387 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.387 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.387 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:50.387 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.387 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.387 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.387 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.387 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.387 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.387 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.387 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.388 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.388 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.388 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.388 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.388 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.388 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.388 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.388 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:50.388 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:50.388 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.388 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:50.389 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.389 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.389 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.389 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.389 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.389 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:50.389 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.389 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.389 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.389 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.389 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.390 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.390 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.390 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.390 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.390 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.390 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.390 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.390 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:50.390 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:50.390 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.390 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.391 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.391 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.391 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:50.391 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:50.391 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:50.391 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.391 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 1, 2024-09-12T23:37:50.391 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:50.391 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.391 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.391 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.391 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:50.392 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:50.392 INFO:tasks.workunit.client.0.smithi136.stdout: 4 2024-09-12T23:37:50.392 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.392 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 3, 2024-09-12T23:37:50.392 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.392 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.392 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:50.392 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.392 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.392 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.392 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.668547+0000", 2024-09-12T23:37:50.393 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.907814+0000", 2024-09-12T23:37:50.393 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.907814+0000", 2024-09-12T23:37:50.393 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.908185+0000", 2024-09-12T23:37:50.393 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.668547+0000", 2024-09-12T23:37:50.393 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.393 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.393 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.393 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:50.393 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.668547+0000", 2024-09-12T23:37:50.393 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.668547+0000", 2024-09-12T23:37:50.393 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.668547+0000", 2024-09-12T23:37:50.394 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.394 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.394 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.394 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.668547+0000", 2024-09-12T23:37:50.394 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.668547+0000", 2024-09-12T23:37:50.394 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.394 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:50.394 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.394 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.394 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:50.394 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.395 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.395 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.395 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.395 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:50.395 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.395 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.395 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.395 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.b", 2024-09-12T23:37:50.395 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.395 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.395 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:50.396 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:50.396 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.396 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T10:45:47.207242+0000", 2024-09-12T23:37:50.396 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:50.396 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.396 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.396 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:50.396 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.396 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.396 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.396 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.397 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.397 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.397 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.397 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.397 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.397 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.397 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.397 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.397 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.397 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.397 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.397 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:50.398 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:50.398 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.398 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:50.398 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.398 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.398 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.398 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.398 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.398 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:50.398 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.398 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.399 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.399 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.399 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.399 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.399 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.399 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.399 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.399 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.399 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.399 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.399 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:50.400 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:50.400 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.400 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.400 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.400 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.400 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:50.400 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:50.400 INFO:tasks.workunit.client.0.smithi136.stdout: 4 2024-09-12T23:37:50.400 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.400 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 3, 2024-09-12T23:37:50.400 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:50.400 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.401 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.401 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.401 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:50.401 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:50.401 INFO:tasks.workunit.client.0.smithi136.stdout: 5 2024-09-12T23:37:50.401 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.401 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 2, 2024-09-12T23:37:50.401 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.401 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.401 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:50.401 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.401 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.402 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.402 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:09.689819+0000", 2024-09-12T23:37:50.402 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:35:09.683737+0000", 2024-09-12T23:37:50.402 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:35:09.683737+0000", 2024-09-12T23:37:50.402 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:35:09.689819+0000", 2024-09-12T23:37:50.402 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:09.689819+0000", 2024-09-12T23:37:50.402 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.402 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.402 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.402 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 64, 2024-09-12T23:37:50.402 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:09.689819+0000", 2024-09-12T23:37:50.402 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:09.689819+0000", 2024-09-12T23:37:50.402 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:09.689819+0000", 2024-09-12T23:37:50.403 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.403 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.403 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.403 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:09.689819+0000", 2024-09-12T23:37:50.403 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:09.689819+0000", 2024-09-12T23:37:50.403 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.403 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 2, 2024-09-12T23:37:50.403 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.403 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.403 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 63, 2024-09-12T23:37:50.403 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.404 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.404 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.404 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.404 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 2, 2024-09-12T23:37:50.404 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.404 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.404 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.404 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.b", 2024-09-12T23:37:50.404 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.404 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.404 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 64, 2024-09-12T23:37:50.404 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 14, 2024-09-12T23:37:50.405 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.405 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T09:18:02.546637+0000", 2024-09-12T23:37:50.405 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:50.405 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.405 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.405 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:50.405 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.405 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.405 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.405 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.405 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.405 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.405 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.406 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.406 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.406 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.406 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.406 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.406 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.406 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.406 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.406 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 3, 2024-09-12T23:37:50.406 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 1, 2024-09-12T23:37:50.406 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.406 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 1, 2024-09-12T23:37:50.407 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.407 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.407 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.407 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.407 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.407 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 1, 2024-09-12T23:37:50.407 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.407 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 1, 2024-09-12T23:37:50.407 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.407 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.407 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.408 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.408 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.408 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.408 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.408 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.408 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.408 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.408 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 1, 2024-09-12T23:37:50.408 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:50.408 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.408 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.409 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.409 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.409 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:50.409 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:50.409 INFO:tasks.workunit.client.0.smithi136.stdout: 5 2024-09-12T23:37:50.409 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.409 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 2, 2024-09-12T23:37:50.409 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "53'2" 2024-09-12T23:37:50.409 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.409 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.409 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.409 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:50.410 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:50.410 INFO:tasks.workunit.client.0.smithi136.stdout: 1 2024-09-12T23:37:50.410 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.410 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 6, 2024-09-12T23:37:50.410 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.410 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.410 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:50.410 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.410 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.410 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.410 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.667383+0000", 2024-09-12T23:37:50.411 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.896761+0000", 2024-09-12T23:37:50.411 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.896761+0000", 2024-09-12T23:37:50.411 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.897132+0000", 2024-09-12T23:37:50.411 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.667383+0000", 2024-09-12T23:37:50.411 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.411 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.411 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.411 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:50.411 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.667383+0000", 2024-09-12T23:37:50.411 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.667383+0000", 2024-09-12T23:37:50.411 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.667383+0000", 2024-09-12T23:37:50.412 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.412 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.412 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.412 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.667383+0000", 2024-09-12T23:37:50.412 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.667383+0000", 2024-09-12T23:37:50.412 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.412 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:50.412 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.412 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.412 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:50.412 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.413 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.413 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.413 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.413 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:50.413 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.413 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.413 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.413 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.a", 2024-09-12T23:37:50.413 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.413 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.413 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:50.414 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:50.414 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.414 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T03:49:56.554393+0000", 2024-09-12T23:37:50.414 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:50.414 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.414 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.414 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:50.414 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.414 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.414 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.414 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.415 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.415 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.415 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.415 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.415 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.415 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.415 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.415 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.415 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.415 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.415 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.415 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:50.416 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:50.416 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.416 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:50.416 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.416 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.416 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.416 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.416 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.416 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:50.416 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.416 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.417 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.417 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.417 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.417 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.417 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.417 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.417 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.417 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.417 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.417 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.417 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:50.417 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:50.418 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.418 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.418 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.418 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.418 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:50.418 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:50.418 INFO:tasks.workunit.client.0.smithi136.stdout: 1 2024-09-12T23:37:50.418 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.418 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 6, 2024-09-12T23:37:50.418 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:50.418 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.419 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.419 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.419 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:50.419 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:50.419 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:50.419 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.419 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 5, 2024-09-12T23:37:50.419 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.419 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.419 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:50.419 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.420 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.420 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.420 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.670622+0000", 2024-09-12T23:37:50.420 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.911157+0000", 2024-09-12T23:37:50.420 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.911157+0000", 2024-09-12T23:37:50.420 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.911440+0000", 2024-09-12T23:37:50.420 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.670622+0000", 2024-09-12T23:37:50.420 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.420 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.420 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.420 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:50.420 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.670622+0000", 2024-09-12T23:37:50.421 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.670622+0000", 2024-09-12T23:37:50.421 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.670622+0000", 2024-09-12T23:37:50.421 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.421 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.421 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.421 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.670622+0000", 2024-09-12T23:37:50.421 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.670622+0000", 2024-09-12T23:37:50.421 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.421 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:50.421 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.421 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.421 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:50.422 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.422 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.422 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.422 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.422 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:50.422 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.422 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.422 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.422 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.1d", 2024-09-12T23:37:50.422 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.422 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.423 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:50.423 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:50.423 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.423 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T00:49:59.313436+0000", 2024-09-12T23:37:50.423 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:50.423 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.423 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.423 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:50.423 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.423 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.423 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.424 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.424 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.424 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.424 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.424 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.424 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.424 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.424 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.424 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.424 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.424 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.425 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.425 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:50.425 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:50.425 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.425 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:50.425 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.425 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.425 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.425 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.425 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.425 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:50.426 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.426 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.426 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.426 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.426 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.426 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.426 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.426 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.426 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.426 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.426 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.426 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.427 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:50.427 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:50.427 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.427 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.427 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.427 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.427 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:50.427 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:50.427 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:50.427 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.427 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 5, 2024-09-12T23:37:50.427 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:50.428 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.428 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.428 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.428 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:50.428 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:50.428 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:50.428 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.428 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 7, 2024-09-12T23:37:50.428 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.428 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.428 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:50.429 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.429 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.429 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.429 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.664158+0000", 2024-09-12T23:37:50.429 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.901755+0000", 2024-09-12T23:37:50.429 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.901755+0000", 2024-09-12T23:37:50.429 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.902224+0000", 2024-09-12T23:37:50.429 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.664158+0000", 2024-09-12T23:37:50.429 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.429 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.429 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.430 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:50.430 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.664158+0000", 2024-09-12T23:37:50.430 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.664158+0000", 2024-09-12T23:37:50.430 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.664158+0000", 2024-09-12T23:37:50.430 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.430 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.430 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.430 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.664158+0000", 2024-09-12T23:37:50.430 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.664158+0000", 2024-09-12T23:37:50.430 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.430 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 2, 2024-09-12T23:37:50.430 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.431 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.431 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:50.431 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.431 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.431 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.431 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.431 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 2, 2024-09-12T23:37:50.431 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.431 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.431 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.431 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.1c", 2024-09-12T23:37:50.432 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.432 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.432 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:50.432 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 23, 2024-09-12T23:37:50.432 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.432 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T08:06:15.504175+0000", 2024-09-12T23:37:50.432 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:50.432 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.432 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.432 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:50.432 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.432 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.433 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.433 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.433 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.433 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.433 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.433 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.433 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.433 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.433 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.433 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.433 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.434 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.434 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.434 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 3, 2024-09-12T23:37:50.434 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 1, 2024-09-12T23:37:50.434 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.434 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 1, 2024-09-12T23:37:50.434 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.434 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.434 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.434 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.434 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.434 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 1, 2024-09-12T23:37:50.435 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.435 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.435 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.435 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.435 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.435 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.435 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.435 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.435 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.435 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.435 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.436 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.436 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 3, 2024-09-12T23:37:50.436 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:50.436 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.436 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.436 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.436 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.436 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:50.436 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:50.436 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:50.436 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.437 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 7, 2024-09-12T23:37:50.437 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "53'2" 2024-09-12T23:37:50.437 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.437 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.437 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.437 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:50.437 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:50.437 INFO:tasks.workunit.client.0.smithi136.stdout: 1 2024-09-12T23:37:50.437 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.437 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 5, 2024-09-12T23:37:50.437 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.437 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.438 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:50.438 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.438 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.438 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.438 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.670603+0000", 2024-09-12T23:37:50.438 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.913521+0000", 2024-09-12T23:37:50.438 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.913521+0000", 2024-09-12T23:37:50.438 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.913814+0000", 2024-09-12T23:37:50.438 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.670603+0000", 2024-09-12T23:37:50.438 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.438 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.439 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.439 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:50.439 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.670603+0000", 2024-09-12T23:37:50.439 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.670603+0000", 2024-09-12T23:37:50.439 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.670603+0000", 2024-09-12T23:37:50.439 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.439 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.439 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.439 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.670603+0000", 2024-09-12T23:37:50.439 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.670603+0000", 2024-09-12T23:37:50.439 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.439 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:50.440 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.440 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.440 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:50.440 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.440 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.440 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.440 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.440 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:50.440 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.440 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.440 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.441 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.1c", 2024-09-12T23:37:50.441 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.441 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.441 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:50.441 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:50.441 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.441 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T04:30:35.754521+0000", 2024-09-12T23:37:50.441 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:50.441 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.441 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.441 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:50.442 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.442 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.442 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.442 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.442 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.442 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.442 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.442 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.442 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.442 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.442 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.442 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.443 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.443 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.443 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.443 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:50.443 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:50.443 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.443 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:50.443 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.443 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.443 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.443 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.443 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.444 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:50.444 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.444 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.444 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.444 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.444 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.444 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.444 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.444 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.444 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.445 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.445 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.445 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.445 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:50.445 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:50.445 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.445 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.445 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.445 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.445 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:50.445 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:50.446 INFO:tasks.workunit.client.0.smithi136.stdout: 1 2024-09-12T23:37:50.446 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.446 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 5, 2024-09-12T23:37:50.446 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:50.446 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.446 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.446 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.446 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:50.446 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:50.446 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:50.446 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.446 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 7, 2024-09-12T23:37:50.447 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.447 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.447 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:50.447 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.447 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.447 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.447 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.666037+0000", 2024-09-12T23:37:50.447 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.896936+0000", 2024-09-12T23:37:50.447 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.896936+0000", 2024-09-12T23:37:50.447 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.897466+0000", 2024-09-12T23:37:50.447 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.666037+0000", 2024-09-12T23:37:50.448 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.448 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.448 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.448 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:50.448 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.666037+0000", 2024-09-12T23:37:50.448 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.666037+0000", 2024-09-12T23:37:50.448 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.666037+0000", 2024-09-12T23:37:50.448 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.448 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.448 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.448 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.666037+0000", 2024-09-12T23:37:50.449 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.666037+0000", 2024-09-12T23:37:50.449 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.449 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 2, 2024-09-12T23:37:50.449 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.449 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.449 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:50.449 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.449 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.449 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.449 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.449 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 2, 2024-09-12T23:37:50.450 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.450 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.450 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.450 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.1d", 2024-09-12T23:37:50.450 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.450 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.450 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:50.450 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 23, 2024-09-12T23:37:50.450 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.450 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T10:29:38.732168+0000", 2024-09-12T23:37:50.450 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:50.451 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.451 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.451 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:50.451 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.451 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.451 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.451 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.451 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.451 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.451 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.451 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.452 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.452 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.452 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.452 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.452 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.452 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.452 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.452 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 3, 2024-09-12T23:37:50.452 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 1, 2024-09-12T23:37:50.452 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.452 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 1, 2024-09-12T23:37:50.453 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.453 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.453 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.453 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.453 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.453 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 1, 2024-09-12T23:37:50.453 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.453 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.453 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.453 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.453 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.454 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.454 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.454 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.454 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.454 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.454 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.454 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.454 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 3, 2024-09-12T23:37:50.454 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:50.454 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.454 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.454 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.455 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.455 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:50.455 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:50.455 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:50.455 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.455 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 7, 2024-09-12T23:37:50.455 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "53'2" 2024-09-12T23:37:50.455 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.455 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.455 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.455 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:50.455 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:50.456 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:50.456 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.456 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 0, 2024-09-12T23:37:50.456 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.456 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.456 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:50.456 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.456 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.456 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.456 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.670437+0000", 2024-09-12T23:37:50.456 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.924320+0000", 2024-09-12T23:37:50.457 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.924320+0000", 2024-09-12T23:37:50.457 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.924610+0000", 2024-09-12T23:37:50.457 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.670437+0000", 2024-09-12T23:37:50.457 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.457 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.457 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.457 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:50.457 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.670437+0000", 2024-09-12T23:37:50.457 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.670437+0000", 2024-09-12T23:37:50.457 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.670437+0000", 2024-09-12T23:37:50.457 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.457 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.458 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.458 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.670437+0000", 2024-09-12T23:37:50.458 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.670437+0000", 2024-09-12T23:37:50.458 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.458 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:50.458 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.458 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.458 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:50.458 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.458 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.459 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.459 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.459 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:50.459 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.459 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.459 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.459 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.1f", 2024-09-12T23:37:50.459 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.459 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.459 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:50.459 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:50.459 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.460 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T06:23:22.797301+0000", 2024-09-12T23:37:50.460 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:50.460 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.460 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.460 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:50.460 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.460 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.460 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.460 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.460 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.460 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.461 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.461 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.461 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.461 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.461 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.461 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.461 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.461 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.461 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.461 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:50.461 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:50.461 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.462 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:50.462 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.462 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.462 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.462 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.462 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.462 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:50.462 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.462 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.462 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.463 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.463 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.463 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.463 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.463 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.463 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.463 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.463 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.463 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.463 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:50.463 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:50.464 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.464 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.464 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.464 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.464 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:50.464 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:50.464 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:50.464 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.464 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 0, 2024-09-12T23:37:50.464 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:50.464 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.465 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.465 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.465 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:50.465 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:50.465 INFO:tasks.workunit.client.0.smithi136.stdout: 5 2024-09-12T23:37:50.465 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.465 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 3, 2024-09-12T23:37:50.465 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.465 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.465 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:50.465 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.466 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.466 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.466 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.673262+0000", 2024-09-12T23:37:50.466 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.893992+0000", 2024-09-12T23:37:50.466 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.893992+0000", 2024-09-12T23:37:50.466 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.894353+0000", 2024-09-12T23:37:50.466 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.673262+0000", 2024-09-12T23:37:50.466 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.466 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.466 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.466 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:50.467 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.673262+0000", 2024-09-12T23:37:50.467 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.673262+0000", 2024-09-12T23:37:50.467 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.673262+0000", 2024-09-12T23:37:50.467 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.467 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.467 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.467 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.673262+0000", 2024-09-12T23:37:50.467 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.673262+0000", 2024-09-12T23:37:50.467 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.467 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 1, 2024-09-12T23:37:50.467 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.468 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.468 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:50.468 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.468 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.468 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.468 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.468 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 1, 2024-09-12T23:37:50.468 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.468 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.468 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.468 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.1e", 2024-09-12T23:37:50.469 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.469 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.469 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:50.469 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 22, 2024-09-12T23:37:50.469 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.469 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T01:43:43.889025+0000", 2024-09-12T23:37:50.469 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:50.469 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.469 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.469 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:50.469 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.470 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.470 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.470 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.470 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.470 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.470 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.470 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.470 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.470 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.470 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.470 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.471 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.471 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.471 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.471 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:50.471 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:50.471 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.471 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:50.471 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.471 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.471 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.471 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.471 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.472 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:50.472 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.472 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.472 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.472 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.472 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.472 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.472 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.472 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.472 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.473 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.473 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.473 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.473 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:50.473 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:50.473 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.473 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.473 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.473 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.473 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:50.473 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:50.473 INFO:tasks.workunit.client.0.smithi136.stdout: 5 2024-09-12T23:37:50.474 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.474 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 3, 2024-09-12T23:37:50.474 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "53'1" 2024-09-12T23:37:50.474 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.474 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.474 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.474 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:50.474 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:50.474 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:50.474 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.475 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 3, 2024-09-12T23:37:50.475 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.475 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.475 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:50.475 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.475 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.475 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.475 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.673097+0000", 2024-09-12T23:37:50.475 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.890368+0000", 2024-09-12T23:37:50.475 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.890368+0000", 2024-09-12T23:37:50.475 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.891227+0000", 2024-09-12T23:37:50.476 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.673097+0000", 2024-09-12T23:37:50.476 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.476 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.476 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.476 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:50.476 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.673097+0000", 2024-09-12T23:37:50.476 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.673097+0000", 2024-09-12T23:37:50.476 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.673097+0000", 2024-09-12T23:37:50.476 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.476 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.476 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:50.476 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.673097+0000", 2024-09-12T23:37:50.477 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.673097+0000", 2024-09-12T23:37:50.477 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.477 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:50.477 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.477 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.477 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:50.477 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.477 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.477 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.477 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.477 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:50.478 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.478 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.478 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.478 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.1e", 2024-09-12T23:37:50.478 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.478 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.478 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:50.478 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:50.478 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.478 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T05:38:37.526610+0000", 2024-09-12T23:37:50.478 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:50.479 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.479 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.479 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:50.479 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.479 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.479 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.479 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.479 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.479 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.479 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.480 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.480 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.480 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.480 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.480 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.480 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.480 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.480 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.480 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:50.480 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:50.480 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.481 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:50.481 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.481 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.481 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.481 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.481 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.481 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:50.481 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.481 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.481 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.481 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.482 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.482 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.482 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.482 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.482 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.482 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.482 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.482 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.482 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:50.482 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:50.482 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.483 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.483 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.483 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.483 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:50.483 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:50.483 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:50.483 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.483 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 3, 2024-09-12T23:37:50.483 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:50.483 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.483 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.484 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:50.484 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:50.484 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:50.484 INFO:tasks.workunit.client.0.smithi136.stdout: 4 2024-09-12T23:37:50.484 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.484 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 0, 2024-09-12T23:37:50.484 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:50.484 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:50.484 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:50.484 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:50.485 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:50.485 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:50.485 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.670421+0000", 2024-09-12T23:37:50.485 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.885720+0000", 2024-09-12T23:37:50.485 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.885720+0000", 2024-09-12T23:37:50.485 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.886016+0000", 2024-09-12T23:37:50.485 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.670421+0000", 2024-09-12T23:37:50.485 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.485 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:50.485 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.485 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:50.486 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.670421+0000", 2024-09-12T23:37:50.486 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.670421+0000", 2024-09-12T23:37:50.486 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.670421+0000", 2024-09-12T23:37:50.486 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:50.486 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:50.486 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:50.486 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.670421+0000", 2024-09-12T23:37:50.486 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.670421+0000", 2024-09-12T23:37:50.486 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:50.486 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 4, 2024-09-12T23:37:50.487 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:50.487 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:50.487 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:50.487 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:50.487 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:50.487 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:50.487 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:50.487 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 4, 2024-09-12T23:37:50.487 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:50.487 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:50.488 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:50.488 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.1f", 2024-09-12T23:37:50.488 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:50.488 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:50.488 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:50.488 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 25, 2024-09-12T23:37:50.488 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:50.488 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T03:44:38.950039+0000", 2024-09-12T23:37:50.488 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:50.488 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:50.488 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.489 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 570, 2024-09-12T23:37:50.489 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.489 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.489 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.489 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.489 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.489 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.489 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.489 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.489 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.490 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.490 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.490 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.490 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.490 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.490 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.490 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 6, 2024-09-12T23:37:50.490 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 2, 2024-09-12T23:37:50.490 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.490 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 2, 2024-09-12T23:37:50.490 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.490 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.491 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.491 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.491 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.491 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 1, 2024-09-12T23:37:50.491 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.491 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.491 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.491 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.491 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.491 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.491 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.491 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.492 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.492 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.492 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.492 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.492 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 4, 2024-09-12T23:37:50.492 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 1 2024-09-12T23:37:50.492 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.492 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:50.492 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:50.492 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:50.492 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:50.493 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:50.493 INFO:tasks.workunit.client.0.smithi136.stdout: 4 2024-09-12T23:37:50.493 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.493 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 0, 2024-09-12T23:37:50.493 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "53'4" 2024-09-12T23:37:50.493 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.493 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.493 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_per_pool": [ 2024-09-12T23:37:50.493 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.493 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": 24, 2024-09-12T23:37:50.493 INFO:tasks.workunit.client.0.smithi136.stdout: "application": [ 2024-09-12T23:37:50.493 INFO:tasks.workunit.client.0.smithi136.stdout: "rbd" 2024-09-12T23:37:50.494 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.494 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 3, 2024-09-12T23:37:50.494 INFO:tasks.workunit.client.0.smithi136.stdout: "num_pg": 8, 2024-09-12T23:37:50.494 INFO:tasks.workunit.client.0.smithi136.stdout: "num_store_stats": 8, 2024-09-12T23:37:50.494 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 3, 2024-09-12T23:37:50.494 INFO:tasks.workunit.client.0.smithi136.stdout: "poolid": 4, 2024-09-12T23:37:50.494 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.494 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 19, 2024-09-12T23:37:50.494 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.494 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.494 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.495 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.495 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.495 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.495 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.495 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.495 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.495 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.495 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.495 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.495 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.496 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.496 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.496 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 6, 2024-09-12T23:37:50.496 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 2, 2024-09-12T23:37:50.496 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.496 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 2, 2024-09-12T23:37:50.496 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.496 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.496 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.496 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.496 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.497 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:50.497 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.497 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.497 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.497 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.497 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.497 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.497 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.497 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.497 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.497 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.497 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.498 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.498 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 2, 2024-09-12T23:37:50.498 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 2 2024-09-12T23:37:50.498 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.498 INFO:tasks.workunit.client.0.smithi136.stdout: "store_stats": { 2024-09-12T23:37:50.498 INFO:tasks.workunit.client.0.smithi136.stdout: "allocated": 12288, 2024-09-12T23:37:50.498 INFO:tasks.workunit.client.0.smithi136.stdout: "available": 0, 2024-09-12T23:37:50.498 INFO:tasks.workunit.client.0.smithi136.stdout: "data_compressed": 0, 2024-09-12T23:37:50.498 INFO:tasks.workunit.client.0.smithi136.stdout: "data_compressed_allocated": 0, 2024-09-12T23:37:50.498 INFO:tasks.workunit.client.0.smithi136.stdout: "data_compressed_original": 0, 2024-09-12T23:37:50.498 INFO:tasks.workunit.client.0.smithi136.stdout: "data_stored": 57, 2024-09-12T23:37:50.499 INFO:tasks.workunit.client.0.smithi136.stdout: "internal_metadata": 0, 2024-09-12T23:37:50.499 INFO:tasks.workunit.client.0.smithi136.stdout: "internally_reserved": 0, 2024-09-12T23:37:50.499 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_allocated": 0, 2024-09-12T23:37:50.499 INFO:tasks.workunit.client.0.smithi136.stdout: "total": 0 2024-09-12T23:37:50.499 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.499 INFO:tasks.workunit.client.0.smithi136.stdout: "up": 24 2024-09-12T23:37:50.499 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.499 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.499 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": 192, 2024-09-12T23:37:50.499 INFO:tasks.workunit.client.0.smithi136.stdout: "application": [ 2024-09-12T23:37:50.499 INFO:tasks.workunit.client.0.smithi136.stdout: "cephfs" 2024-09-12T23:37:50.499 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.500 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:50.500 INFO:tasks.workunit.client.0.smithi136.stdout: "num_pg": 64, 2024-09-12T23:37:50.500 INFO:tasks.workunit.client.0.smithi136.stdout: "num_store_stats": 8, 2024-09-12T23:37:50.500 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:50.500 INFO:tasks.workunit.client.0.smithi136.stdout: "poolid": 3, 2024-09-12T23:37:50.500 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.500 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:50.500 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.500 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.500 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.500 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.501 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.501 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.501 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.501 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.501 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.501 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.501 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.501 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.501 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.501 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.502 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.502 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:50.502 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:50.502 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.502 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:50.502 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.502 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.502 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.502 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.502 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.502 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:50.502 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.503 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.503 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.503 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.503 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.503 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.503 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.503 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.503 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.503 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.503 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.503 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.503 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:50.504 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:50.504 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.504 INFO:tasks.workunit.client.0.smithi136.stdout: "store_stats": { 2024-09-12T23:37:50.504 INFO:tasks.workunit.client.0.smithi136.stdout: "allocated": 0, 2024-09-12T23:37:50.504 INFO:tasks.workunit.client.0.smithi136.stdout: "available": 0, 2024-09-12T23:37:50.504 INFO:tasks.workunit.client.0.smithi136.stdout: "data_compressed": 0, 2024-09-12T23:37:50.504 INFO:tasks.workunit.client.0.smithi136.stdout: "data_compressed_allocated": 0, 2024-09-12T23:37:50.504 INFO:tasks.workunit.client.0.smithi136.stdout: "data_compressed_original": 0, 2024-09-12T23:37:50.504 INFO:tasks.workunit.client.0.smithi136.stdout: "data_stored": 0, 2024-09-12T23:37:50.504 INFO:tasks.workunit.client.0.smithi136.stdout: "internal_metadata": 0, 2024-09-12T23:37:50.504 INFO:tasks.workunit.client.0.smithi136.stdout: "internally_reserved": 0, 2024-09-12T23:37:50.505 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_allocated": 0, 2024-09-12T23:37:50.505 INFO:tasks.workunit.client.0.smithi136.stdout: "total": 0 2024-09-12T23:37:50.505 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.505 INFO:tasks.workunit.client.0.smithi136.stdout: "up": 192 2024-09-12T23:37:50.505 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.505 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.505 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": 96, 2024-09-12T23:37:50.505 INFO:tasks.workunit.client.0.smithi136.stdout: "application": [ 2024-09-12T23:37:50.505 INFO:tasks.workunit.client.0.smithi136.stdout: "cephfs" 2024-09-12T23:37:50.505 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.505 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 45, 2024-09-12T23:37:50.506 INFO:tasks.workunit.client.0.smithi136.stdout: "num_pg": 32, 2024-09-12T23:37:50.506 INFO:tasks.workunit.client.0.smithi136.stdout: "num_store_stats": 8, 2024-09-12T23:37:50.506 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 45, 2024-09-12T23:37:50.506 INFO:tasks.workunit.client.0.smithi136.stdout: "poolid": 2, 2024-09-12T23:37:50.506 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.506 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 3410, 2024-09-12T23:37:50.506 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.506 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.506 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.506 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.506 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.506 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.507 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.507 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.507 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.507 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.507 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.507 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.507 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.507 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.507 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.507 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 66, 2024-09-12T23:37:50.507 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 22, 2024-09-12T23:37:50.508 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.508 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 22, 2024-09-12T23:37:50.508 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.508 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.508 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.508 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.508 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.508 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 14, 2024-09-12T23:37:50.508 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.508 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 1, 2024-09-12T23:37:50.508 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.509 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.509 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.509 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.509 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.509 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:50.509 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:50.509 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.509 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.509 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.509 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 51, 2024-09-12T23:37:50.509 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 19 2024-09-12T23:37:50.510 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.510 INFO:tasks.workunit.client.0.smithi136.stdout: "store_stats": { 2024-09-12T23:37:50.510 INFO:tasks.workunit.client.0.smithi136.stdout: "allocated": 98304, 2024-09-12T23:37:50.510 INFO:tasks.workunit.client.0.smithi136.stdout: "available": 0, 2024-09-12T23:37:50.510 INFO:tasks.workunit.client.0.smithi136.stdout: "data_compressed": 0, 2024-09-12T23:37:50.510 INFO:tasks.workunit.client.0.smithi136.stdout: "data_compressed_allocated": 0, 2024-09-12T23:37:50.510 INFO:tasks.workunit.client.0.smithi136.stdout: "data_compressed_original": 0, 2024-09-12T23:37:50.510 INFO:tasks.workunit.client.0.smithi136.stdout: "data_stored": 10230, 2024-09-12T23:37:50.510 INFO:tasks.workunit.client.0.smithi136.stdout: "internal_metadata": 0, 2024-09-12T23:37:50.510 INFO:tasks.workunit.client.0.smithi136.stdout: "internally_reserved": 0, 2024-09-12T23:37:50.510 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_allocated": 0, 2024-09-12T23:37:50.511 INFO:tasks.workunit.client.0.smithi136.stdout: "total": 0 2024-09-12T23:37:50.511 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.511 INFO:tasks.workunit.client.0.smithi136.stdout: "up": 96 2024-09-12T23:37:50.511 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.511 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.511 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": 3, 2024-09-12T23:37:50.511 INFO:tasks.workunit.client.0.smithi136.stdout: "application": [ 2024-09-12T23:37:50.511 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr" 2024-09-12T23:37:50.511 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.511 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 79, 2024-09-12T23:37:50.511 INFO:tasks.workunit.client.0.smithi136.stdout: "num_pg": 1, 2024-09-12T23:37:50.511 INFO:tasks.workunit.client.0.smithi136.stdout: "num_store_stats": 3, 2024-09-12T23:37:50.512 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 79, 2024-09-12T23:37:50.512 INFO:tasks.workunit.client.0.smithi136.stdout: "poolid": 1, 2024-09-12T23:37:50.512 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.512 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 590368, 2024-09-12T23:37:50.512 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.512 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 590368, 2024-09-12T23:37:50.512 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.512 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.512 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.512 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.512 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.513 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.513 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.513 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.513 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.513 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.513 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.513 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.513 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.513 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 6, 2024-09-12T23:37:50.513 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 2, 2024-09-12T23:37:50.513 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.514 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 2, 2024-09-12T23:37:50.514 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.514 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.514 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.514 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.514 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.514 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:50.514 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.514 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 2, 2024-09-12T23:37:50.514 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.514 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.514 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.515 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.515 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.515 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 126, 2024-09-12T23:37:50.515 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 231, 2024-09-12T23:37:50.515 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.515 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.515 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.515 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 113, 2024-09-12T23:37:50.515 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 1628 2024-09-12T23:37:50.515 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.516 INFO:tasks.workunit.client.0.smithi136.stdout: "store_stats": { 2024-09-12T23:37:50.516 INFO:tasks.workunit.client.0.smithi136.stdout: "allocated": 1781760, 2024-09-12T23:37:50.516 INFO:tasks.workunit.client.0.smithi136.stdout: "available": 0, 2024-09-12T23:37:50.516 INFO:tasks.workunit.client.0.smithi136.stdout: "data_compressed": 0, 2024-09-12T23:37:50.516 INFO:tasks.workunit.client.0.smithi136.stdout: "data_compressed_allocated": 0, 2024-09-12T23:37:50.516 INFO:tasks.workunit.client.0.smithi136.stdout: "data_compressed_original": 0, 2024-09-12T23:37:50.516 INFO:tasks.workunit.client.0.smithi136.stdout: "data_stored": 1771104, 2024-09-12T23:37:50.516 INFO:tasks.workunit.client.0.smithi136.stdout: "internal_metadata": 0, 2024-09-12T23:37:50.516 INFO:tasks.workunit.client.0.smithi136.stdout: "internally_reserved": 0, 2024-09-12T23:37:50.516 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_allocated": 0, 2024-09-12T23:37:50.516 INFO:tasks.workunit.client.0.smithi136.stdout: "total": 0 2024-09-12T23:37:50.517 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.517 INFO:tasks.workunit.client.0.smithi136.stdout: "up": 3 2024-09-12T23:37:50.517 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.517 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.517 INFO:tasks.workunit.client.0.smithi136.stdout: "usage": { 2024-09-12T23:37:50.517 INFO:tasks.workunit.client.0.smithi136.stdout: "pg_num": 105, 2024-09-12T23:37:50.517 INFO:tasks.workunit.client.0.smithi136.stdout: "pools": 4, 2024-09-12T23:37:50.517 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_by_class": { 2024-09-12T23:37:50.517 INFO:tasks.workunit.client.0.smithi136.stdout: "ssd": { 2024-09-12T23:37:50.517 INFO:tasks.workunit.client.0.smithi136.stdout: "total_avail_bytes": 765517262848, 2024-09-12T23:37:50.517 INFO:tasks.workunit.client.0.smithi136.stdout: "total_bytes": 767960285184, 2024-09-12T23:37:50.517 INFO:tasks.workunit.client.0.smithi136.stdout: "total_used_bytes": 2443022336, 2024-09-12T23:37:50.518 INFO:tasks.workunit.client.0.smithi136.stdout: "total_used_raw_bytes": 2443022336, 2024-09-12T23:37:50.518 INFO:tasks.workunit.client.0.smithi136.stdout: "total_used_raw_ratio": 0.0031811832450330257 2024-09-12T23:37:50.518 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.518 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.518 INFO:tasks.workunit.client.0.smithi136.stdout: "total_avail_bytes": 765517262848, 2024-09-12T23:37:50.518 INFO:tasks.workunit.client.0.smithi136.stdout: "total_bytes": 767960285184, 2024-09-12T23:37:50.518 INFO:tasks.workunit.client.0.smithi136.stdout: "total_used_bytes": 2443022336 2024-09-12T23:37:50.518 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.518 INFO:tasks.workunit.client.0.smithi136.stdout:} 2024-09-12T23:37:50.518 INFO:tasks.workunit.client.0.smithi136.stdout:Device report is generated separately. To see it run 'ceph telemetry show-device'. 2024-09-12T23:37:50.519 INFO:tasks.workunit.client.0.smithi136.stderr:+ ceph telemetry show-device 2024-09-12T23:37:50.520 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[22601]: audit 2024-09-12T23:37:46.974834+0000 mgr.y (mgr.24446) 156 : audit [DBG] from='client.14886 -' entity='client.admin' cmd=[{"prefix": "telemetry enable channel", "channels": ["perf"], "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:50.520 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[22601]: audit 2024-09-12T23:37:47.152360+0000 mgr.y (mgr.24446) 157 : audit [DBG] from='client.14886 -' entity='client.admin' cmd=[{"prefix": "telemetry enable channel", "channels": ["perf"], "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:50.520 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[22601]: audit 2024-09-12T23:37:47.537860+0000 mon.c (mon.2) 50 : audit [DBG] from='client.? 172.21.15.136:0/109380988' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/telemetry/last_opt_revision"}]: dispatch 2024-09-12T23:37:50.520 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[22601]: audit 2024-09-12T23:37:47.892310+0000 mon.a (mon.0) 975 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:37:50.520 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[22601]: audit 2024-09-12T23:37:47.898470+0000 mon.a (mon.0) 976 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-12T23:37:50.520 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[22601]: audit 2024-09-12T23:37:48.028556+0000 mon.a (mon.0) 977 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:50.520 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[22601]: audit 2024-09-12T23:37:48.029039+0000 mon.a (mon.0) 978 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:50.521 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[22601]: audit 2024-09-12T23:37:48.029410+0000 mon.a (mon.0) 979 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:50.521 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[22601]: audit 2024-09-12T23:37:48.032945+0000 mon.a (mon.0) 980 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:50.521 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[22601]: audit 2024-09-12T23:37:48.033388+0000 mon.a (mon.0) 981 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:50.521 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[22601]: audit 2024-09-12T23:37:48.033766+0000 mon.a (mon.0) 982 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:50.521 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[22601]: audit 2024-09-12T23:37:48.036616+0000 mon.a (mon.0) 983 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:50.521 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[22601]: audit 2024-09-12T23:37:48.037072+0000 mon.a (mon.0) 984 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:50.521 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[22601]: audit 2024-09-12T23:37:48.037432+0000 mon.a (mon.0) 985 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:50.521 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[22601]: audit 2024-09-12T23:37:48.054304+0000 mon.a (mon.0) 986 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:37:50.521 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[22601]: audit 2024-09-12T23:37:48.054464+0000 mon.a (mon.0) 987 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:37:50.521 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[22601]: audit 2024-09-12T23:37:48.054938+0000 mon.a (mon.0) 988 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:50.521 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[22601]: audit 2024-09-12T23:37:48.055377+0000 mon.a (mon.0) 989 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:50.524 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[32189]: audit 2024-09-12T23:37:46.974834+0000 mgr.y (mgr.24446) 156 : audit [DBG] from='client.14886 -' entity='client.admin' cmd=[{"prefix": "telemetry enable channel", "channels": ["perf"], "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:50.524 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[32189]: audit 2024-09-12T23:37:47.152360+0000 mgr.y (mgr.24446) 157 : audit [DBG] from='client.14886 -' entity='client.admin' cmd=[{"prefix": "telemetry enable channel", "channels": ["perf"], "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:50.524 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[32189]: audit 2024-09-12T23:37:47.537860+0000 mon.c (mon.2) 50 : audit [DBG] from='client.? 172.21.15.136:0/109380988' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/telemetry/last_opt_revision"}]: dispatch 2024-09-12T23:37:50.524 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[32189]: audit 2024-09-12T23:37:47.892310+0000 mon.a (mon.0) 975 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:37:50.524 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[32189]: audit 2024-09-12T23:37:47.898470+0000 mon.a (mon.0) 976 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-12T23:37:50.524 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[32189]: audit 2024-09-12T23:37:48.028556+0000 mon.a (mon.0) 977 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:50.524 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[32189]: audit 2024-09-12T23:37:48.029039+0000 mon.a (mon.0) 978 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:50.524 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[32189]: audit 2024-09-12T23:37:48.029410+0000 mon.a (mon.0) 979 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:50.524 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[32189]: audit 2024-09-12T23:37:48.032945+0000 mon.a (mon.0) 980 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:50.524 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[32189]: audit 2024-09-12T23:37:48.033388+0000 mon.a (mon.0) 981 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:50.524 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[32189]: audit 2024-09-12T23:37:48.033766+0000 mon.a (mon.0) 982 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:50.525 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[32189]: audit 2024-09-12T23:37:48.036616+0000 mon.a (mon.0) 983 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:50.525 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[32189]: audit 2024-09-12T23:37:48.037072+0000 mon.a (mon.0) 984 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:50.525 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[32189]: audit 2024-09-12T23:37:48.037432+0000 mon.a (mon.0) 985 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:50.525 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[32189]: audit 2024-09-12T23:37:48.054304+0000 mon.a (mon.0) 986 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:37:50.525 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[32189]: audit 2024-09-12T23:37:48.054464+0000 mon.a (mon.0) 987 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:37:50.525 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[32189]: audit 2024-09-12T23:37:48.054938+0000 mon.a (mon.0) 988 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:50.525 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[32189]: audit 2024-09-12T23:37:48.055377+0000 mon.a (mon.0) 989 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:50.525 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:48 smithi181 bash[24776]: audit 2024-09-12T23:37:46.974834+0000 mgr.y (mgr.24446) 156 : audit [DBG] from='client.14886 -' entity='client.admin' cmd=[{"prefix": "telemetry enable channel", "channels": ["perf"], "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:50.526 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:48 smithi181 bash[24776]: audit 2024-09-12T23:37:47.152360+0000 mgr.y (mgr.24446) 157 : audit [DBG] from='client.14886 -' entity='client.admin' cmd=[{"prefix": "telemetry enable channel", "channels": ["perf"], "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:50.526 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:48 smithi181 bash[24776]: audit 2024-09-12T23:37:47.537860+0000 mon.c (mon.2) 50 : audit [DBG] from='client.? 172.21.15.136:0/109380988' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/telemetry/last_opt_revision"}]: dispatch 2024-09-12T23:37:50.526 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:48 smithi181 bash[24776]: audit 2024-09-12T23:37:47.892310+0000 mon.a (mon.0) 975 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:37:50.526 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:48 smithi181 bash[24776]: audit 2024-09-12T23:37:47.898470+0000 mon.a (mon.0) 976 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-12T23:37:50.526 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:48 smithi181 bash[24776]: audit 2024-09-12T23:37:48.028556+0000 mon.a (mon.0) 977 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:50.526 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:48 smithi181 bash[24776]: audit 2024-09-12T23:37:48.029039+0000 mon.a (mon.0) 978 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:50.526 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:48 smithi181 bash[24776]: audit 2024-09-12T23:37:48.029410+0000 mon.a (mon.0) 979 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:50.526 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:48 smithi181 bash[24776]: audit 2024-09-12T23:37:48.032945+0000 mon.a (mon.0) 980 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:50.526 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:48 smithi181 bash[24776]: audit 2024-09-12T23:37:48.033388+0000 mon.a (mon.0) 981 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:50.526 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:48 smithi181 bash[24776]: audit 2024-09-12T23:37:48.033766+0000 mon.a (mon.0) 982 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:50.526 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:48 smithi181 bash[24776]: audit 2024-09-12T23:37:48.036616+0000 mon.a (mon.0) 983 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:50.527 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:48 smithi181 bash[24776]: audit 2024-09-12T23:37:48.037072+0000 mon.a (mon.0) 984 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:50.527 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:48 smithi181 bash[24776]: audit 2024-09-12T23:37:48.037432+0000 mon.a (mon.0) 985 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:50.527 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:48 smithi181 bash[24776]: audit 2024-09-12T23:37:48.054304+0000 mon.a (mon.0) 986 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:37:50.527 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:48 smithi181 bash[24776]: audit 2024-09-12T23:37:48.054464+0000 mon.a (mon.0) 987 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:37:50.527 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:48 smithi181 bash[24776]: audit 2024-09-12T23:37:48.054938+0000 mon.a (mon.0) 988 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:50.527 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:48 smithi181 bash[24776]: audit 2024-09-12T23:37:48.055377+0000 mon.a (mon.0) 989 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:50.527 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:48 smithi181 bash[24776]: audit 2024-09-12T23:37:48.055725+0000 mon.a (mon.0) 990 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:50.527 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:48 smithi181 bash[24776]: audit 2024-09-12T23:37:48.058251+0000 mon.b (mon.1) 28 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:37:50.527 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:48 smithi181 bash[24776]: audit 2024-09-12T23:37:48.058392+0000 mon.b (mon.1) 29 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:37:50.527 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:48 smithi181 bash[24776]: audit 2024-09-12T23:37:48.058900+0000 mon.a (mon.0) 991 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:50.527 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:48 smithi181 bash[24776]: audit 2024-09-12T23:37:48.059683+0000 mon.a (mon.0) 992 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:50.527 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:48 smithi181 bash[24776]: audit 2024-09-12T23:37:48.059974+0000 mon.a (mon.0) 993 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:50.528 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:48 smithi181 bash[24776]: audit 2024-09-12T23:37:48.062630+0000 mon.c (mon.2) 51 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:37:50.528 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:48 smithi181 bash[24776]: audit 2024-09-12T23:37:48.062707+0000 mon.c (mon.2) 52 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:37:50.528 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:48 smithi181 bash[24776]: audit 2024-09-12T23:37:48.063351+0000 mon.a (mon.0) 994 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:50.528 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:48 smithi181 bash[24776]: audit 2024-09-12T23:37:48.063960+0000 mon.a (mon.0) 995 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:50.528 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:48 smithi181 bash[24776]: audit 2024-09-12T23:37:48.064296+0000 mon.a (mon.0) 996 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:50.528 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:48 smithi181 bash[24776]: audit 2024-09-12T23:37:48.300562+0000 mon.a (mon.0) 997 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:37:50.528 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:48 smithi181 bash[24776]: audit 2024-09-12T23:37:48.305053+0000 mon.a (mon.0) 998 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-12T23:37:50.528 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:49 smithi181 bash[24776]: audit 2024-09-12T23:37:47.889857+0000 mgr.y (mgr.24446) 158 : audit [DBG] from='client.24818 -' entity='client.admin' cmd=[{"prefix": "telemetry show", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:50.528 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:49 smithi181 bash[24776]: cluster 2024-09-12T23:37:47.998417+0000 mgr.y (mgr.24446) 159 : cluster [DBG] pgmap v122: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:50.528 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:49 smithi181 bash[24776]: audit 2024-09-12T23:37:48.299024+0000 mgr.y (mgr.24446) 160 : audit [DBG] from='client.24818 -' entity='client.admin' cmd=[{"prefix": "telemetry show", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:50.528 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:49 smithi181 bash[24776]: audit 2024-09-12T23:37:48.481240+0000 mon.a (mon.0) 999 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:50.529 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:49 smithi181 bash[24776]: audit 2024-09-12T23:37:48.494410+0000 mon.a (mon.0) 1000 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:50.529 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:49 smithi181 bash[24776]: audit 2024-09-12T23:37:48.494897+0000 mon.a (mon.0) 1001 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:50.529 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:49 smithi181 bash[24776]: audit 2024-09-12T23:37:48.498555+0000 mon.a (mon.0) 1002 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:50.529 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:49 smithi181 bash[24776]: audit 2024-09-12T23:37:48.499042+0000 mon.a (mon.0) 1003 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:50.529 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:49 smithi181 bash[24776]: audit 2024-09-12T23:37:48.499365+0000 mon.a (mon.0) 1004 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:50.529 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:49 smithi181 bash[24776]: audit 2024-09-12T23:37:48.502336+0000 mon.a (mon.0) 1005 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:50.529 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:49 smithi181 bash[24776]: audit 2024-09-12T23:37:48.502793+0000 mon.a (mon.0) 1006 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:50.529 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:49 smithi181 bash[24776]: audit 2024-09-12T23:37:48.503261+0000 mon.a (mon.0) 1007 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:50.529 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:49 smithi181 bash[24776]: audit 2024-09-12T23:37:48.521194+0000 mon.a (mon.0) 1008 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:37:50.529 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:49 smithi181 bash[24776]: audit 2024-09-12T23:37:48.521283+0000 mon.a (mon.0) 1009 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:37:50.530 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:49 smithi181 bash[24776]: audit 2024-09-12T23:37:48.521734+0000 mon.a (mon.0) 1010 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:50.530 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:49 smithi181 bash[24776]: audit 2024-09-12T23:37:48.522223+0000 mon.a (mon.0) 1011 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:50.530 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:49 smithi181 bash[24776]: audit 2024-09-12T23:37:48.522661+0000 mon.a (mon.0) 1012 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:50.530 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:49 smithi181 bash[24776]: audit 2024-09-12T23:37:48.525328+0000 mon.b (mon.1) 30 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:37:50.530 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:49 smithi181 bash[24776]: audit 2024-09-12T23:37:48.525419+0000 mon.b (mon.1) 31 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:37:50.530 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:49 smithi181 bash[24776]: audit 2024-09-12T23:37:48.525958+0000 mon.a (mon.0) 1013 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:50.530 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:49 smithi181 bash[24776]: audit 2024-09-12T23:37:48.526678+0000 mon.a (mon.0) 1014 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:50.530 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:49 smithi181 bash[24776]: audit 2024-09-12T23:37:48.527042+0000 mon.a (mon.0) 1015 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:50.530 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:49 smithi181 bash[24776]: audit 2024-09-12T23:37:48.530349+0000 mon.c (mon.2) 53 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:37:50.530 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:49 smithi181 bash[24776]: audit 2024-09-12T23:37:48.530482+0000 mon.c (mon.2) 54 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:37:50.530 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:49 smithi181 bash[24776]: audit 2024-09-12T23:37:48.530869+0000 mon.a (mon.0) 1016 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:50.531 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:49 smithi181 bash[24776]: audit 2024-09-12T23:37:48.531630+0000 mon.a (mon.0) 1017 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:50.531 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:49 smithi181 bash[24776]: audit 2024-09-12T23:37:48.531972+0000 mon.a (mon.0) 1018 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:50.531 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[22601]: audit 2024-09-12T23:37:48.055725+0000 mon.a (mon.0) 990 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:50.531 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[22601]: audit 2024-09-12T23:37:48.058251+0000 mon.b (mon.1) 28 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:37:50.531 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[22601]: audit 2024-09-12T23:37:48.058392+0000 mon.b (mon.1) 29 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:37:50.531 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[22601]: audit 2024-09-12T23:37:48.058900+0000 mon.a (mon.0) 991 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:50.531 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[22601]: audit 2024-09-12T23:37:48.059683+0000 mon.a (mon.0) 992 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:50.531 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[22601]: audit 2024-09-12T23:37:48.059974+0000 mon.a (mon.0) 993 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:50.531 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[22601]: audit 2024-09-12T23:37:48.062630+0000 mon.c (mon.2) 51 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:37:50.532 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[22601]: audit 2024-09-12T23:37:48.062707+0000 mon.c (mon.2) 52 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:37:50.532 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[22601]: audit 2024-09-12T23:37:48.063351+0000 mon.a (mon.0) 994 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:50.532 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[22601]: audit 2024-09-12T23:37:48.063960+0000 mon.a (mon.0) 995 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:50.532 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[22601]: audit 2024-09-12T23:37:48.064296+0000 mon.a (mon.0) 996 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:50.532 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[22601]: audit 2024-09-12T23:37:48.300562+0000 mon.a (mon.0) 997 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:37:50.532 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[22601]: audit 2024-09-12T23:37:48.305053+0000 mon.a (mon.0) 998 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-12T23:37:50.533 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[32189]: audit 2024-09-12T23:37:48.055725+0000 mon.a (mon.0) 990 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:50.533 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[32189]: audit 2024-09-12T23:37:48.058251+0000 mon.b (mon.1) 28 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:37:50.533 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[32189]: audit 2024-09-12T23:37:48.058392+0000 mon.b (mon.1) 29 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:37:50.533 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[32189]: audit 2024-09-12T23:37:48.058900+0000 mon.a (mon.0) 991 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:50.533 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[32189]: audit 2024-09-12T23:37:48.059683+0000 mon.a (mon.0) 992 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:50.533 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[32189]: audit 2024-09-12T23:37:48.059974+0000 mon.a (mon.0) 993 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:50.533 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[32189]: audit 2024-09-12T23:37:48.062630+0000 mon.c (mon.2) 51 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:37:50.533 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[32189]: audit 2024-09-12T23:37:48.062707+0000 mon.c (mon.2) 52 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:37:50.533 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[32189]: audit 2024-09-12T23:37:48.063351+0000 mon.a (mon.0) 994 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:50.533 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[32189]: audit 2024-09-12T23:37:48.063960+0000 mon.a (mon.0) 995 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:50.534 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[32189]: audit 2024-09-12T23:37:48.064296+0000 mon.a (mon.0) 996 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:50.534 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[32189]: audit 2024-09-12T23:37:48.300562+0000 mon.a (mon.0) 997 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:37:50.534 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:48 smithi136 bash[32189]: audit 2024-09-12T23:37:48.305053+0000 mon.a (mon.0) 998 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-12T23:37:50.534 INFO:tasks.workunit.client.0.smithi136.stdout:{ 2024-09-12T23:37:50.534 INFO:tasks.workunit.client.0.smithi136.stdout: "02b0b25a-7160-11ef-baee-0cc47a886e43": { 2024-09-12T23:37:50.534 INFO:tasks.workunit.client.0.smithi136.stdout: "INTEL_SSDPEDMD400G4_02b1c294-7160-11ef-baee-0cc47a886e43": { 2024-09-12T23:37:50.534 INFO:tasks.workunit.client.0.smithi136.stdout: "20240912-232840": { 2024-09-12T23:37:50.534 INFO:tasks.workunit.client.0.smithi136.stdout: "device": { 2024-09-12T23:37:50.535 INFO:tasks.workunit.client.0.smithi136.stdout: "info_name": "/dev/nvme0n1", 2024-09-12T23:37:50.535 INFO:tasks.workunit.client.0.smithi136.stdout: "name": "/dev/nvme0n1", 2024-09-12T23:37:50.535 INFO:tasks.workunit.client.0.smithi136.stdout: "protocol": "NVMe", 2024-09-12T23:37:50.535 INFO:tasks.workunit.client.0.smithi136.stdout: "type": "nvme" 2024-09-12T23:37:50.535 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.535 INFO:tasks.workunit.client.0.smithi136.stdout: "firmware_version": "8DV101H0", 2024-09-12T23:37:50.535 INFO:tasks.workunit.client.0.smithi136.stdout: "host_id": "02b0b25a-7160-11ef-baee-0cc47a886e43", 2024-09-12T23:37:50.535 INFO:tasks.workunit.client.0.smithi136.stdout: "json_format_version": [ 2024-09-12T23:37:50.535 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:50.535 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:50.535 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.536 INFO:tasks.workunit.client.0.smithi136.stdout: "local_time": { 2024-09-12T23:37:50.536 INFO:tasks.workunit.client.0.smithi136.stdout: "asctime": "Thu Sep 12 23:28:40 2024 UTC", 2024-09-12T23:37:50.536 INFO:tasks.workunit.client.0.smithi136.stdout: "time_t": 1726183720 2024-09-12T23:37:50.536 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.536 INFO:tasks.workunit.client.0.smithi136.stdout: "logical_block_size": 512, 2024-09-12T23:37:50.536 INFO:tasks.workunit.client.0.smithi136.stdout: "model_name": "INTEL SSDPEDMD400G4", 2024-09-12T23:37:50.536 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_controller_id": 0, 2024-09-12T23:37:50.536 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_ieee_oui_identifier": 6083300, 2024-09-12T23:37:50.536 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_namespaces": [ 2024-09-12T23:37:50.536 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.536 INFO:tasks.workunit.client.0.smithi136.stdout: "capacity": { 2024-09-12T23:37:50.537 INFO:tasks.workunit.client.0.smithi136.stdout: "blocks": 781422768, 2024-09-12T23:37:50.537 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 400088457216 2024-09-12T23:37:50.537 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.537 INFO:tasks.workunit.client.0.smithi136.stdout: "formatted_lba_size": 512, 2024-09-12T23:37:50.537 INFO:tasks.workunit.client.0.smithi136.stdout: "id": 1, 2024-09-12T23:37:50.537 INFO:tasks.workunit.client.0.smithi136.stdout: "size": { 2024-09-12T23:37:50.537 INFO:tasks.workunit.client.0.smithi136.stdout: "blocks": 781422768, 2024-09-12T23:37:50.537 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 400088457216 2024-09-12T23:37:50.537 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.537 INFO:tasks.workunit.client.0.smithi136.stdout: "utilization": { 2024-09-12T23:37:50.537 INFO:tasks.workunit.client.0.smithi136.stdout: "blocks": 781422768, 2024-09-12T23:37:50.538 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 400088457216 2024-09-12T23:37:50.538 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.538 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.538 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.538 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_number_of_namespaces": 1, 2024-09-12T23:37:50.538 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_pci_vendor": { 2024-09-12T23:37:50.538 INFO:tasks.workunit.client.0.smithi136.stdout: "id": 32902, 2024-09-12T23:37:50.538 INFO:tasks.workunit.client.0.smithi136.stdout: "subsystem_id": 32902 2024-09-12T23:37:50.538 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.538 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_smart_health_information_add_log": { 2024-09-12T23:37:50.538 INFO:tasks.workunit.client.0.smithi136.stdout: "Device stats": { 2024-09-12T23:37:50.539 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_fw_downgrades": { 2024-09-12T23:37:50.539 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 0, 2024-09-12T23:37:50.539 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:50.539 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.539 INFO:tasks.workunit.client.0.smithi136.stdout: "crc_error_count": { 2024-09-12T23:37:50.539 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:50.539 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:50.539 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.539 INFO:tasks.workunit.client.0.smithi136.stdout: "end_to_end_error_detection_count": { 2024-09-12T23:37:50.539 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:50.539 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:50.540 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.540 INFO:tasks.workunit.client.0.smithi136.stdout: "erase_fail_count": { 2024-09-12T23:37:50.540 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:50.540 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:50.540 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.540 INFO:tasks.workunit.client.0.smithi136.stdout: "host_bytes_written": { 2024-09-12T23:37:50.540 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:50.540 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 26605852 2024-09-12T23:37:50.540 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.540 INFO:tasks.workunit.client.0.smithi136.stdout: "host_ctx_wear_used": { 2024-09-12T23:37:50.540 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:50.541 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 1 2024-09-12T23:37:50.541 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.541 INFO:tasks.workunit.client.0.smithi136.stdout: "media_bytes_read": { 2024-09-12T23:37:50.541 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 0, 2024-09-12T23:37:50.541 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:50.541 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.541 INFO:tasks.workunit.client.0.smithi136.stdout: "nand_bytes_written": { 2024-09-12T23:37:50.541 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:50.541 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 47676652 2024-09-12T23:37:50.541 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.541 INFO:tasks.workunit.client.0.smithi136.stdout: "perf_stat_indicator": { 2024-09-12T23:37:50.542 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 0, 2024-09-12T23:37:50.542 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:50.542 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.542 INFO:tasks.workunit.client.0.smithi136.stdout: "pll_lock_loss_count": { 2024-09-12T23:37:50.542 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:50.542 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:50.542 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.542 INFO:tasks.workunit.client.0.smithi136.stdout: "program_fail_count": { 2024-09-12T23:37:50.542 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:50.542 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:50.542 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.543 INFO:tasks.workunit.client.0.smithi136.stdout: "re_alloc_sectr_cnt": { 2024-09-12T23:37:50.543 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 0, 2024-09-12T23:37:50.543 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:50.543 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.543 INFO:tasks.workunit.client.0.smithi136.stdout: "retry_buffer_overflow_count": { 2024-09-12T23:37:50.543 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:50.543 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:50.543 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.543 INFO:tasks.workunit.client.0.smithi136.stdout: "soft_ecc_err_rate": { 2024-09-12T23:37:50.543 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 0, 2024-09-12T23:37:50.543 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:50.544 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.544 INFO:tasks.workunit.client.0.smithi136.stdout: "thermal_throttle_status": { 2024-09-12T23:37:50.544 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:50.544 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": { 2024-09-12T23:37:50.544 INFO:tasks.workunit.client.0.smithi136.stdout: "cnt": 0, 2024-09-12T23:37:50.544 INFO:tasks.workunit.client.0.smithi136.stdout: "pct": 0 2024-09-12T23:37:50.544 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.544 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.544 INFO:tasks.workunit.client.0.smithi136.stdout: "timed_workload_host_reads": { 2024-09-12T23:37:50.544 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:50.544 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 65535 2024-09-12T23:37:50.544 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.545 INFO:tasks.workunit.client.0.smithi136.stdout: "timed_workload_media_wear": { 2024-09-12T23:37:50.545 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:50.545 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": "63.999023" 2024-09-12T23:37:50.545 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.545 INFO:tasks.workunit.client.0.smithi136.stdout: "timed_workload_timer": { 2024-09-12T23:37:50.545 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:50.545 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 65535 2024-09-12T23:37:50.545 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.545 INFO:tasks.workunit.client.0.smithi136.stdout: "unexp_power_loss": { 2024-09-12T23:37:50.545 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 0, 2024-09-12T23:37:50.545 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:50.545 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.546 INFO:tasks.workunit.client.0.smithi136.stdout: "wear_leveling": { 2024-09-12T23:37:50.546 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 90, 2024-09-12T23:37:50.546 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": { 2024-09-12T23:37:50.546 INFO:tasks.workunit.client.0.smithi136.stdout: "avg": 3275, 2024-09-12T23:37:50.546 INFO:tasks.workunit.client.0.smithi136.stdout: "max": 3290, 2024-09-12T23:37:50.546 INFO:tasks.workunit.client.0.smithi136.stdout: "min": 3260 2024-09-12T23:37:50.546 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.546 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.546 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.546 INFO:tasks.workunit.client.0.smithi136.stdout: "Intel Smart log": "nvme0n1" 2024-09-12T23:37:50.546 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.547 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_smart_health_information_log": { 2024-09-12T23:37:50.547 INFO:tasks.workunit.client.0.smithi136.stdout: "available_spare": 100, 2024-09-12T23:37:50.547 INFO:tasks.workunit.client.0.smithi136.stdout: "available_spare_threshold": 10, 2024-09-12T23:37:50.547 INFO:tasks.workunit.client.0.smithi136.stdout: "controller_busy_time": 50, 2024-09-12T23:37:50.547 INFO:tasks.workunit.client.0.smithi136.stdout: "critical_warning": 0, 2024-09-12T23:37:50.547 INFO:tasks.workunit.client.0.smithi136.stdout: "data_units_read": 360344011, 2024-09-12T23:37:50.547 INFO:tasks.workunit.client.0.smithi136.stdout: "data_units_written": 1743641130, 2024-09-12T23:37:50.547 INFO:tasks.workunit.client.0.smithi136.stdout: "host_reads": 5879482063, 2024-09-12T23:37:50.547 INFO:tasks.workunit.client.0.smithi136.stdout: "host_writes": 31824656568, 2024-09-12T23:37:50.547 INFO:tasks.workunit.client.0.smithi136.stdout: "media_errors": 0, 2024-09-12T23:37:50.547 INFO:tasks.workunit.client.0.smithi136.stdout: "num_err_log_entries": 0, 2024-09-12T23:37:50.548 INFO:tasks.workunit.client.0.smithi136.stdout: "percentage_used": 10, 2024-09-12T23:37:50.548 INFO:tasks.workunit.client.0.smithi136.stdout: "power_cycles": 56409, 2024-09-12T23:37:50.548 INFO:tasks.workunit.client.0.smithi136.stdout: "power_on_hours": 54400, 2024-09-12T23:37:50.548 INFO:tasks.workunit.client.0.smithi136.stdout: "temperature": 23, 2024-09-12T23:37:50.548 INFO:tasks.workunit.client.0.smithi136.stdout: "unsafe_shutdowns": 2575 2024-09-12T23:37:50.548 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.548 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_vendor": "intel", 2024-09-12T23:37:50.548 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_version": { 2024-09-12T23:37:50.548 INFO:tasks.workunit.client.0.smithi136.stdout: "string": "<1.2", 2024-09-12T23:37:50.548 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.548 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.549 INFO:tasks.workunit.client.0.smithi136.stdout: "power_cycle_count": 56409, 2024-09-12T23:37:50.549 INFO:tasks.workunit.client.0.smithi136.stdout: "power_on_time": { 2024-09-12T23:37:50.549 INFO:tasks.workunit.client.0.smithi136.stdout: "hours": 54400 2024-09-12T23:37:50.549 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.549 INFO:tasks.workunit.client.0.smithi136.stdout: "serial_number": "deleted", 2024-09-12T23:37:50.549 INFO:tasks.workunit.client.0.smithi136.stdout: "smart_status": { 2024-09-12T23:37:50.549 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme": { 2024-09-12T23:37:50.549 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.549 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.549 INFO:tasks.workunit.client.0.smithi136.stdout: "passed": true 2024-09-12T23:37:50.549 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.550 INFO:tasks.workunit.client.0.smithi136.stdout: "smartctl": { 2024-09-12T23:37:50.550 INFO:tasks.workunit.client.0.smithi136.stdout: "argv": [ 2024-09-12T23:37:50.550 INFO:tasks.workunit.client.0.smithi136.stdout: "smartctl", 2024-09-12T23:37:50.550 INFO:tasks.workunit.client.0.smithi136.stdout: "-x", 2024-09-12T23:37:50.550 INFO:tasks.workunit.client.0.smithi136.stdout: "--json=o", 2024-09-12T23:37:50.550 INFO:tasks.workunit.client.0.smithi136.stdout: "/dev/nvme0n1" 2024-09-12T23:37:50.550 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.550 INFO:tasks.workunit.client.0.smithi136.stdout: "build_info": "(local build)", 2024-09-12T23:37:50.550 INFO:tasks.workunit.client.0.smithi136.stdout: "exit_status": 0, 2024-09-12T23:37:50.550 INFO:tasks.workunit.client.0.smithi136.stdout: "output": [ 2024-09-12T23:37:50.550 INFO:tasks.workunit.client.0.smithi136.stdout: "smartctl 7.2 2020-12-30 r5155 [x86_64-linux-5.15.0-121-generic] (local build)", 2024-09-12T23:37:50.551 INFO:tasks.workunit.client.0.smithi136.stdout: "Copyright (C) 2002-20, Bruce Allen, Christian Franke, www.smartmontools.org", 2024-09-12T23:37:50.551 INFO:tasks.workunit.client.0.smithi136.stdout: "", 2024-09-12T23:37:50.551 INFO:tasks.workunit.client.0.smithi136.stdout: "=== START OF INFORMATION SECTION ===", 2024-09-12T23:37:50.551 INFO:tasks.workunit.client.0.smithi136.stdout: "Model Number: INTEL SSDPEDMD400G4", 2024-09-12T23:37:50.551 INFO:tasks.workunit.client.0.smithi136.stdout: "Serial Number: deleted", 2024-09-12T23:37:50.551 INFO:tasks.workunit.client.0.smithi136.stdout: "Firmware Version: 8DV101H0", 2024-09-12T23:37:50.551 INFO:tasks.workunit.client.0.smithi136.stdout: "PCI Vendor/Subsystem ID: 0x8086", 2024-09-12T23:37:50.551 INFO:tasks.workunit.client.0.smithi136.stdout: "IEEE OUI Identifier: 0x5cd2e4", 2024-09-12T23:37:50.551 INFO:tasks.workunit.client.0.smithi136.stdout: "Controller ID: 0", 2024-09-12T23:37:50.551 INFO:tasks.workunit.client.0.smithi136.stdout: "NVMe Version: <1.2", 2024-09-12T23:37:50.551 INFO:tasks.workunit.client.0.smithi136.stdout: "Number of Namespaces: 1", 2024-09-12T23:37:50.551 INFO:tasks.workunit.client.0.smithi136.stdout: "Namespace 1 Size/Capacity: 400,088,457,216 [400 GB]", 2024-09-12T23:37:50.552 INFO:tasks.workunit.client.0.smithi136.stdout: "Namespace 1 Formatted LBA Size: 512", 2024-09-12T23:37:50.552 INFO:tasks.workunit.client.0.smithi136.stdout: "Local Time is: Thu Sep 12 23:28:40 2024 UTC", 2024-09-12T23:37:50.552 INFO:tasks.workunit.client.0.smithi136.stdout: "Firmware Updates (0x02): 1 Slot", 2024-09-12T23:37:50.552 INFO:tasks.workunit.client.0.smithi136.stdout: "Optional Admin Commands (0x0006): Format Frmw_DL", 2024-09-12T23:37:50.552 INFO:tasks.workunit.client.0.smithi136.stdout: "Optional NVM Commands (0x0006): Wr_Unc DS_Mngmt", 2024-09-12T23:37:50.552 INFO:tasks.workunit.client.0.smithi136.stdout: "Log Page Attributes (0x02): Cmd_Eff_Lg", 2024-09-12T23:37:50.552 INFO:tasks.workunit.client.0.smithi136.stdout: "Maximum Data Transfer Size: 32 Pages", 2024-09-12T23:37:50.552 INFO:tasks.workunit.client.0.smithi136.stdout: "", 2024-09-12T23:37:50.552 INFO:tasks.workunit.client.0.smithi136.stdout: "Supported Power States", 2024-09-12T23:37:50.552 INFO:tasks.workunit.client.0.smithi136.stdout: "St Op Max Active Idle RL RT WL WT Ent_Lat Ex_Lat", 2024-09-12T23:37:50.552 INFO:tasks.workunit.client.0.smithi136.stdout: " 0 + 25.00W - - 0 0 0 0 0 0", 2024-09-12T23:37:50.553 INFO:tasks.workunit.client.0.smithi136.stdout: "", 2024-09-12T23:37:50.553 INFO:tasks.workunit.client.0.smithi136.stdout: "Supported LBA Sizes (NSID 0x1)", 2024-09-12T23:37:50.553 INFO:tasks.workunit.client.0.smithi136.stdout: "Id Fmt Data Metadt Rel_Perf", 2024-09-12T23:37:50.553 INFO:tasks.workunit.client.0.smithi136.stdout: " 0 + 512 0 2", 2024-09-12T23:37:50.553 INFO:tasks.workunit.client.0.smithi136.stdout: " 1 - 512 8 2", 2024-09-12T23:37:50.553 INFO:tasks.workunit.client.0.smithi136.stdout: " 2 - 512 16 2", 2024-09-12T23:37:50.553 INFO:tasks.workunit.client.0.smithi136.stdout: " 3 - 4096 0 0", 2024-09-12T23:37:50.553 INFO:tasks.workunit.client.0.smithi136.stdout: " 4 - 4096 8 0", 2024-09-12T23:37:50.553 INFO:tasks.workunit.client.0.smithi136.stdout: " 5 - 4096 64 0", 2024-09-12T23:37:50.553 INFO:tasks.workunit.client.0.smithi136.stdout: " 6 - 4096 128 0", 2024-09-12T23:37:50.554 INFO:tasks.workunit.client.0.smithi136.stdout: "", 2024-09-12T23:37:50.554 INFO:tasks.workunit.client.0.smithi136.stdout: "=== START OF SMART DATA SECTION ===", 2024-09-12T23:37:50.554 INFO:tasks.workunit.client.0.smithi136.stdout: "SMART overall-health self-assessment test result: PASSED", 2024-09-12T23:37:50.554 INFO:tasks.workunit.client.0.smithi136.stdout: "", 2024-09-12T23:37:50.554 INFO:tasks.workunit.client.0.smithi136.stdout: "SMART/Health Information (NVMe Log 0x02)", 2024-09-12T23:37:50.554 INFO:tasks.workunit.client.0.smithi136.stdout: "Critical Warning: 0x00", 2024-09-12T23:37:50.554 INFO:tasks.workunit.client.0.smithi136.stdout: "Temperature: 23 Celsius", 2024-09-12T23:37:50.554 INFO:tasks.workunit.client.0.smithi136.stdout: "Available Spare: 100%", 2024-09-12T23:37:50.554 INFO:tasks.workunit.client.0.smithi136.stdout: "Available Spare Threshold: 10%", 2024-09-12T23:37:50.554 INFO:tasks.workunit.client.0.smithi136.stdout: "Percentage Used: 10%", 2024-09-12T23:37:50.554 INFO:tasks.workunit.client.0.smithi136.stdout: "Data Units Read: 360,344,011 [184 TB]", 2024-09-12T23:37:50.555 INFO:tasks.workunit.client.0.smithi136.stdout: "Data Units Written: 1,743,641,130 [892 TB]", 2024-09-12T23:37:50.555 INFO:tasks.workunit.client.0.smithi136.stdout: "Host Read Commands: 5,879,482,063", 2024-09-12T23:37:50.555 INFO:tasks.workunit.client.0.smithi136.stdout: "Host Write Commands: 31,824,656,568", 2024-09-12T23:37:50.555 INFO:tasks.workunit.client.0.smithi136.stdout: "Controller Busy Time: 50", 2024-09-12T23:37:50.555 INFO:tasks.workunit.client.0.smithi136.stdout: "Power Cycles: 56,409", 2024-09-12T23:37:50.555 INFO:tasks.workunit.client.0.smithi136.stdout: "Power On Hours: 54,400", 2024-09-12T23:37:50.555 INFO:tasks.workunit.client.0.smithi136.stdout: "Unsafe Shutdowns: 2,575", 2024-09-12T23:37:50.555 INFO:tasks.workunit.client.0.smithi136.stdout: "Media and Data Integrity Errors: 0", 2024-09-12T23:37:50.555 INFO:tasks.workunit.client.0.smithi136.stdout: "Error Information Log Entries: 0", 2024-09-12T23:37:50.555 INFO:tasks.workunit.client.0.smithi136.stdout: "", 2024-09-12T23:37:50.555 INFO:tasks.workunit.client.0.smithi136.stdout: "Error Information (NVMe Log 0x01, 16 of 64 entries)", 2024-09-12T23:37:50.556 INFO:tasks.workunit.client.0.smithi136.stdout: "No Errors Logged", 2024-09-12T23:37:50.556 INFO:tasks.workunit.client.0.smithi136.stdout: "" 2024-09-12T23:37:50.556 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.556 INFO:tasks.workunit.client.0.smithi136.stdout: "platform_info": "x86_64-linux-5.15.0-121-generic", 2024-09-12T23:37:50.556 INFO:tasks.workunit.client.0.smithi136.stdout: "svn_revision": "5155", 2024-09-12T23:37:50.556 INFO:tasks.workunit.client.0.smithi136.stdout: "version": [ 2024-09-12T23:37:50.556 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:50.556 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:50.556 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:50.556 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.556 INFO:tasks.workunit.client.0.smithi136.stdout: "temperature": { 2024-09-12T23:37:50.557 INFO:tasks.workunit.client.0.smithi136.stdout: "current": 23 2024-09-12T23:37:50.557 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.557 INFO:tasks.workunit.client.0.smithi136.stdout: "user_capacity": { 2024-09-12T23:37:50.557 INFO:tasks.workunit.client.0.smithi136.stdout: "blocks": 781422768, 2024-09-12T23:37:50.557 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 400088457216 2024-09-12T23:37:50.557 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.557 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.557 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.557 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.557 INFO:tasks.workunit.client.0.smithi136.stdout: "02b4b8aa-7160-11ef-baee-0cc47a886e43": { 2024-09-12T23:37:50.557 INFO:tasks.workunit.client.0.smithi136.stdout: "INTEL_SSDPEDMX020T4_02b5a97c-7160-11ef-baee-0cc47a886e43": { 2024-09-12T23:37:50.558 INFO:tasks.workunit.client.0.smithi136.stdout: "20240912-232836": { 2024-09-12T23:37:50.558 INFO:tasks.workunit.client.0.smithi136.stdout: "device": { 2024-09-12T23:37:50.558 INFO:tasks.workunit.client.0.smithi136.stdout: "info_name": "/dev/nvme0n1", 2024-09-12T23:37:50.558 INFO:tasks.workunit.client.0.smithi136.stdout: "name": "/dev/nvme0n1", 2024-09-12T23:37:50.558 INFO:tasks.workunit.client.0.smithi136.stdout: "protocol": "NVMe", 2024-09-12T23:37:50.558 INFO:tasks.workunit.client.0.smithi136.stdout: "type": "nvme" 2024-09-12T23:37:50.558 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.558 INFO:tasks.workunit.client.0.smithi136.stdout: "firmware_version": "8DV10171", 2024-09-12T23:37:50.558 INFO:tasks.workunit.client.0.smithi136.stdout: "host_id": "02b4b8aa-7160-11ef-baee-0cc47a886e43", 2024-09-12T23:37:50.558 INFO:tasks.workunit.client.0.smithi136.stdout: "json_format_version": [ 2024-09-12T23:37:50.558 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:50.558 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:50.559 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.559 INFO:tasks.workunit.client.0.smithi136.stdout: "local_time": { 2024-09-12T23:37:50.559 INFO:tasks.workunit.client.0.smithi136.stdout: "asctime": "Thu Sep 12 23:28:36 2024 UTC", 2024-09-12T23:37:50.559 INFO:tasks.workunit.client.0.smithi136.stdout: "time_t": 1726183716 2024-09-12T23:37:50.559 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.559 INFO:tasks.workunit.client.0.smithi136.stdout: "logical_block_size": 512, 2024-09-12T23:37:50.559 INFO:tasks.workunit.client.0.smithi136.stdout: "model_name": "INTEL SSDPEDMX020T4", 2024-09-12T23:37:50.559 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_controller_id": 0, 2024-09-12T23:37:50.559 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_ieee_oui_identifier": 6083300, 2024-09-12T23:37:50.559 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_namespaces": [ 2024-09-12T23:37:50.560 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.560 INFO:tasks.workunit.client.0.smithi136.stdout: "capacity": { 2024-09-12T23:37:50.560 INFO:tasks.workunit.client.0.smithi136.stdout: "blocks": 3907029168, 2024-09-12T23:37:50.560 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 2000398934016 2024-09-12T23:37:50.560 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.560 INFO:tasks.workunit.client.0.smithi136.stdout: "formatted_lba_size": 512, 2024-09-12T23:37:50.560 INFO:tasks.workunit.client.0.smithi136.stdout: "id": 1, 2024-09-12T23:37:50.560 INFO:tasks.workunit.client.0.smithi136.stdout: "size": { 2024-09-12T23:37:50.560 INFO:tasks.workunit.client.0.smithi136.stdout: "blocks": 3907029168, 2024-09-12T23:37:50.560 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 2000398934016 2024-09-12T23:37:50.560 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.561 INFO:tasks.workunit.client.0.smithi136.stdout: "utilization": { 2024-09-12T23:37:50.561 INFO:tasks.workunit.client.0.smithi136.stdout: "blocks": 3907029168, 2024-09-12T23:37:50.561 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 2000398934016 2024-09-12T23:37:50.561 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.561 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.561 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.561 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_number_of_namespaces": 1, 2024-09-12T23:37:50.561 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_pci_vendor": { 2024-09-12T23:37:50.561 INFO:tasks.workunit.client.0.smithi136.stdout: "id": 32902, 2024-09-12T23:37:50.561 INFO:tasks.workunit.client.0.smithi136.stdout: "subsystem_id": 32902 2024-09-12T23:37:50.561 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.562 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_smart_health_information_add_log": { 2024-09-12T23:37:50.562 INFO:tasks.workunit.client.0.smithi136.stdout: "Device stats": { 2024-09-12T23:37:50.562 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_fw_downgrades": { 2024-09-12T23:37:50.562 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 0, 2024-09-12T23:37:50.562 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:50.562 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.562 INFO:tasks.workunit.client.0.smithi136.stdout: "crc_error_count": { 2024-09-12T23:37:50.562 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:50.562 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 2 2024-09-12T23:37:50.562 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.562 INFO:tasks.workunit.client.0.smithi136.stdout: "end_to_end_error_detection_count": { 2024-09-12T23:37:50.562 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:50.563 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:50.563 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.563 INFO:tasks.workunit.client.0.smithi136.stdout: "erase_fail_count": { 2024-09-12T23:37:50.563 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:50.563 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:50.563 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.563 INFO:tasks.workunit.client.0.smithi136.stdout: "host_bytes_written": { 2024-09-12T23:37:50.563 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:50.563 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 19052464 2024-09-12T23:37:50.563 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.563 INFO:tasks.workunit.client.0.smithi136.stdout: "host_ctx_wear_used": { 2024-09-12T23:37:50.564 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 0, 2024-09-12T23:37:50.564 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:50.564 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.564 INFO:tasks.workunit.client.0.smithi136.stdout: "media_bytes_read": { 2024-09-12T23:37:50.564 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 0, 2024-09-12T23:37:50.564 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:50.564 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.564 INFO:tasks.workunit.client.0.smithi136.stdout: "nand_bytes_written": { 2024-09-12T23:37:50.564 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:50.564 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 50938213 2024-09-12T23:37:50.564 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.565 INFO:tasks.workunit.client.0.smithi136.stdout: "perf_stat_indicator": { 2024-09-12T23:37:50.565 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 0, 2024-09-12T23:37:50.565 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:50.565 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.565 INFO:tasks.workunit.client.0.smithi136.stdout: "pll_lock_loss_count": { 2024-09-12T23:37:50.565 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:50.565 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:50.565 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.565 INFO:tasks.workunit.client.0.smithi136.stdout: "program_fail_count": { 2024-09-12T23:37:50.565 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:50.565 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:50.566 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.566 INFO:tasks.workunit.client.0.smithi136.stdout: "re_alloc_sectr_cnt": { 2024-09-12T23:37:50.566 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 0, 2024-09-12T23:37:50.566 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:50.566 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.566 INFO:tasks.workunit.client.0.smithi136.stdout: "retry_buffer_overflow_count": { 2024-09-12T23:37:50.566 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:50.566 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:50.566 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.566 INFO:tasks.workunit.client.0.smithi136.stdout: "soft_ecc_err_rate": { 2024-09-12T23:37:50.566 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 0, 2024-09-12T23:37:50.566 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:50.567 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.567 INFO:tasks.workunit.client.0.smithi136.stdout: "thermal_throttle_status": { 2024-09-12T23:37:50.567 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:50.567 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": { 2024-09-12T23:37:50.567 INFO:tasks.workunit.client.0.smithi136.stdout: "cnt": 0, 2024-09-12T23:37:50.567 INFO:tasks.workunit.client.0.smithi136.stdout: "pct": 0 2024-09-12T23:37:50.567 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.567 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.567 INFO:tasks.workunit.client.0.smithi136.stdout: "timed_workload_host_reads": { 2024-09-12T23:37:50.567 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:50.567 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 15 2024-09-12T23:37:50.567 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.568 INFO:tasks.workunit.client.0.smithi136.stdout: "timed_workload_media_wear": { 2024-09-12T23:37:50.568 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:50.568 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": "6.850586" 2024-09-12T23:37:50.568 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.568 INFO:tasks.workunit.client.0.smithi136.stdout: "timed_workload_timer": { 2024-09-12T23:37:50.568 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:50.568 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 1541953 2024-09-12T23:37:50.568 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.568 INFO:tasks.workunit.client.0.smithi136.stdout: "unexp_power_loss": { 2024-09-12T23:37:50.568 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 0, 2024-09-12T23:37:50.568 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:50.569 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.569 INFO:tasks.workunit.client.0.smithi136.stdout: "wear_leveling": { 2024-09-12T23:37:50.569 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 47, 2024-09-12T23:37:50.569 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": { 2024-09-12T23:37:50.569 INFO:tasks.workunit.client.0.smithi136.stdout: "avg": 1165, 2024-09-12T23:37:50.569 INFO:tasks.workunit.client.0.smithi136.stdout: "max": 1608, 2024-09-12T23:37:50.569 INFO:tasks.workunit.client.0.smithi136.stdout: "min": 765 2024-09-12T23:37:50.569 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.569 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.569 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.569 INFO:tasks.workunit.client.0.smithi136.stdout: "Intel Smart log": "nvme0n1" 2024-09-12T23:37:50.569 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.570 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_smart_health_information_log": { 2024-09-12T23:37:50.570 INFO:tasks.workunit.client.0.smithi136.stdout: "available_spare": 100, 2024-09-12T23:37:50.570 INFO:tasks.workunit.client.0.smithi136.stdout: "available_spare_threshold": 10, 2024-09-12T23:37:50.570 INFO:tasks.workunit.client.0.smithi136.stdout: "controller_busy_time": 358, 2024-09-12T23:37:50.570 INFO:tasks.workunit.client.0.smithi136.stdout: "critical_warning": 0, 2024-09-12T23:37:50.570 INFO:tasks.workunit.client.0.smithi136.stdout: "data_units_read": 215101386, 2024-09-12T23:37:50.570 INFO:tasks.workunit.client.0.smithi136.stdout: "data_units_written": 1248622303, 2024-09-12T23:37:50.570 INFO:tasks.workunit.client.0.smithi136.stdout: "host_reads": 4349473727, 2024-09-12T23:37:50.570 INFO:tasks.workunit.client.0.smithi136.stdout: "host_writes": 24841062282, 2024-09-12T23:37:50.570 INFO:tasks.workunit.client.0.smithi136.stdout: "media_errors": 0, 2024-09-12T23:37:50.570 INFO:tasks.workunit.client.0.smithi136.stdout: "num_err_log_entries": 0, 2024-09-12T23:37:50.571 INFO:tasks.workunit.client.0.smithi136.stdout: "percentage_used": 25, 2024-09-12T23:37:50.571 INFO:tasks.workunit.client.0.smithi136.stdout: "power_cycles": 45284, 2024-09-12T23:37:50.571 INFO:tasks.workunit.client.0.smithi136.stdout: "power_on_hours": 38404, 2024-09-12T23:37:50.571 INFO:tasks.workunit.client.0.smithi136.stdout: "temperature": 23, 2024-09-12T23:37:50.571 INFO:tasks.workunit.client.0.smithi136.stdout: "unsafe_shutdowns": 45156 2024-09-12T23:37:50.571 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.571 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_vendor": "intel", 2024-09-12T23:37:50.571 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_version": { 2024-09-12T23:37:50.571 INFO:tasks.workunit.client.0.smithi136.stdout: "string": "<1.2", 2024-09-12T23:37:50.571 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.571 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.571 INFO:tasks.workunit.client.0.smithi136.stdout: "power_cycle_count": 45284, 2024-09-12T23:37:50.572 INFO:tasks.workunit.client.0.smithi136.stdout: "power_on_time": { 2024-09-12T23:37:50.572 INFO:tasks.workunit.client.0.smithi136.stdout: "hours": 38404 2024-09-12T23:37:50.572 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.572 INFO:tasks.workunit.client.0.smithi136.stdout: "serial_number": "deleted", 2024-09-12T23:37:50.572 INFO:tasks.workunit.client.0.smithi136.stdout: "smart_status": { 2024-09-12T23:37:50.572 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme": { 2024-09-12T23:37:50.572 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.572 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.572 INFO:tasks.workunit.client.0.smithi136.stdout: "passed": true 2024-09-12T23:37:50.572 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.572 INFO:tasks.workunit.client.0.smithi136.stdout: "smartctl": { 2024-09-12T23:37:50.573 INFO:tasks.workunit.client.0.smithi136.stdout: "argv": [ 2024-09-12T23:37:50.573 INFO:tasks.workunit.client.0.smithi136.stdout: "smartctl", 2024-09-12T23:37:50.573 INFO:tasks.workunit.client.0.smithi136.stdout: "-x", 2024-09-12T23:37:50.573 INFO:tasks.workunit.client.0.smithi136.stdout: "--json=o", 2024-09-12T23:37:50.573 INFO:tasks.workunit.client.0.smithi136.stdout: "/dev/nvme0n1" 2024-09-12T23:37:50.573 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.573 INFO:tasks.workunit.client.0.smithi136.stdout: "build_info": "(local build)", 2024-09-12T23:37:50.573 INFO:tasks.workunit.client.0.smithi136.stdout: "exit_status": 0, 2024-09-12T23:37:50.573 INFO:tasks.workunit.client.0.smithi136.stdout: "output": [ 2024-09-12T23:37:50.573 INFO:tasks.workunit.client.0.smithi136.stdout: "smartctl 7.2 2020-12-30 r5155 [x86_64-linux-5.15.0-121-generic] (local build)", 2024-09-12T23:37:50.573 INFO:tasks.workunit.client.0.smithi136.stdout: "Copyright (C) 2002-20, Bruce Allen, Christian Franke, www.smartmontools.org", 2024-09-12T23:37:50.574 INFO:tasks.workunit.client.0.smithi136.stdout: "", 2024-09-12T23:37:50.574 INFO:tasks.workunit.client.0.smithi136.stdout: "=== START OF INFORMATION SECTION ===", 2024-09-12T23:37:50.574 INFO:tasks.workunit.client.0.smithi136.stdout: "Model Number: INTEL SSDPEDMX020T4", 2024-09-12T23:37:50.574 INFO:tasks.workunit.client.0.smithi136.stdout: "Serial Number: deleted", 2024-09-12T23:37:50.574 INFO:tasks.workunit.client.0.smithi136.stdout: "Firmware Version: 8DV10171", 2024-09-12T23:37:50.574 INFO:tasks.workunit.client.0.smithi136.stdout: "PCI Vendor/Subsystem ID: 0x8086", 2024-09-12T23:37:50.574 INFO:tasks.workunit.client.0.smithi136.stdout: "IEEE OUI Identifier: 0x5cd2e4", 2024-09-12T23:37:50.574 INFO:tasks.workunit.client.0.smithi136.stdout: "Controller ID: 0", 2024-09-12T23:37:50.574 INFO:tasks.workunit.client.0.smithi136.stdout: "NVMe Version: <1.2", 2024-09-12T23:37:50.574 INFO:tasks.workunit.client.0.smithi136.stdout: "Number of Namespaces: 1", 2024-09-12T23:37:50.574 INFO:tasks.workunit.client.0.smithi136.stdout: "Namespace 1 Size/Capacity: 2,000,398,934,016 [2.00 TB]", 2024-09-12T23:37:50.574 INFO:tasks.workunit.client.0.smithi136.stdout: "Namespace 1 Formatted LBA Size: 512", 2024-09-12T23:37:50.575 INFO:tasks.workunit.client.0.smithi136.stdout: "Local Time is: Thu Sep 12 23:28:36 2024 UTC", 2024-09-12T23:37:50.575 INFO:tasks.workunit.client.0.smithi136.stdout: "Firmware Updates (0x02): 1 Slot", 2024-09-12T23:37:50.575 INFO:tasks.workunit.client.0.smithi136.stdout: "Optional Admin Commands (0x0006): Format Frmw_DL", 2024-09-12T23:37:50.575 INFO:tasks.workunit.client.0.smithi136.stdout: "Optional NVM Commands (0x0006): Wr_Unc DS_Mngmt", 2024-09-12T23:37:50.575 INFO:tasks.workunit.client.0.smithi136.stdout: "Log Page Attributes (0x02): Cmd_Eff_Lg", 2024-09-12T23:37:50.575 INFO:tasks.workunit.client.0.smithi136.stdout: "Maximum Data Transfer Size: 32 Pages", 2024-09-12T23:37:50.575 INFO:tasks.workunit.client.0.smithi136.stdout: "", 2024-09-12T23:37:50.575 INFO:tasks.workunit.client.0.smithi136.stdout: "Supported Power States", 2024-09-12T23:37:50.575 INFO:tasks.workunit.client.0.smithi136.stdout: "St Op Max Active Idle RL RT WL WT Ent_Lat Ex_Lat", 2024-09-12T23:37:50.575 INFO:tasks.workunit.client.0.smithi136.stdout: " 0 + 25.00W - - 0 0 0 0 0 0", 2024-09-12T23:37:50.575 INFO:tasks.workunit.client.0.smithi136.stdout: "", 2024-09-12T23:37:50.576 INFO:tasks.workunit.client.0.smithi136.stdout: "Supported LBA Sizes (NSID 0x1)", 2024-09-12T23:37:50.576 INFO:tasks.workunit.client.0.smithi136.stdout: "Id Fmt Data Metadt Rel_Perf", 2024-09-12T23:37:50.576 INFO:tasks.workunit.client.0.smithi136.stdout: " 0 + 512 0 2", 2024-09-12T23:37:50.576 INFO:tasks.workunit.client.0.smithi136.stdout: " 1 - 512 8 2", 2024-09-12T23:37:50.576 INFO:tasks.workunit.client.0.smithi136.stdout: " 2 - 512 16 2", 2024-09-12T23:37:50.576 INFO:tasks.workunit.client.0.smithi136.stdout: " 3 - 4096 0 0", 2024-09-12T23:37:50.576 INFO:tasks.workunit.client.0.smithi136.stdout: " 4 - 4096 8 0", 2024-09-12T23:37:50.576 INFO:tasks.workunit.client.0.smithi136.stdout: " 5 - 4096 64 0", 2024-09-12T23:37:50.576 INFO:tasks.workunit.client.0.smithi136.stdout: " 6 - 4096 128 0", 2024-09-12T23:37:50.576 INFO:tasks.workunit.client.0.smithi136.stdout: "", 2024-09-12T23:37:50.577 INFO:tasks.workunit.client.0.smithi136.stdout: "=== START OF SMART DATA SECTION ===", 2024-09-12T23:37:50.577 INFO:tasks.workunit.client.0.smithi136.stdout: "SMART overall-health self-assessment test result: PASSED", 2024-09-12T23:37:50.577 INFO:tasks.workunit.client.0.smithi136.stdout: "", 2024-09-12T23:37:50.577 INFO:tasks.workunit.client.0.smithi136.stdout: "SMART/Health Information (NVMe Log 0x02)", 2024-09-12T23:37:50.577 INFO:tasks.workunit.client.0.smithi136.stdout: "Critical Warning: 0x00", 2024-09-12T23:37:50.577 INFO:tasks.workunit.client.0.smithi136.stdout: "Temperature: 23 Celsius", 2024-09-12T23:37:50.577 INFO:tasks.workunit.client.0.smithi136.stdout: "Available Spare: 100%", 2024-09-12T23:37:50.577 INFO:tasks.workunit.client.0.smithi136.stdout: "Available Spare Threshold: 10%", 2024-09-12T23:37:50.577 INFO:tasks.workunit.client.0.smithi136.stdout: "Percentage Used: 25%", 2024-09-12T23:37:50.577 INFO:tasks.workunit.client.0.smithi136.stdout: "Data Units Read: 215,101,386 [110 TB]", 2024-09-12T23:37:50.577 INFO:tasks.workunit.client.0.smithi136.stdout: "Data Units Written: 1,248,622,303 [639 TB]", 2024-09-12T23:37:50.578 INFO:tasks.workunit.client.0.smithi136.stdout: "Host Read Commands: 4,349,473,727", 2024-09-12T23:37:50.578 INFO:tasks.workunit.client.0.smithi136.stdout: "Host Write Commands: 24,841,062,282", 2024-09-12T23:37:50.578 INFO:tasks.workunit.client.0.smithi136.stdout: "Controller Busy Time: 358", 2024-09-12T23:37:50.578 INFO:tasks.workunit.client.0.smithi136.stdout: "Power Cycles: 45,284", 2024-09-12T23:37:50.578 INFO:tasks.workunit.client.0.smithi136.stdout: "Power On Hours: 38,404", 2024-09-12T23:37:50.578 INFO:tasks.workunit.client.0.smithi136.stdout: "Unsafe Shutdowns: 45,156", 2024-09-12T23:37:50.578 INFO:tasks.workunit.client.0.smithi136.stdout: "Media and Data Integrity Errors: 0", 2024-09-12T23:37:50.578 INFO:tasks.workunit.client.0.smithi136.stdout: "Error Information Log Entries: 0", 2024-09-12T23:37:50.578 INFO:tasks.workunit.client.0.smithi136.stdout: "", 2024-09-12T23:37:50.578 INFO:tasks.workunit.client.0.smithi136.stdout: "Error Information (NVMe Log 0x01, 16 of 64 entries)", 2024-09-12T23:37:50.578 INFO:tasks.workunit.client.0.smithi136.stdout: "No Errors Logged", 2024-09-12T23:37:50.579 INFO:tasks.workunit.client.0.smithi136.stdout: "" 2024-09-12T23:37:50.579 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.579 INFO:tasks.workunit.client.0.smithi136.stdout: "platform_info": "x86_64-linux-5.15.0-121-generic", 2024-09-12T23:37:50.579 INFO:tasks.workunit.client.0.smithi136.stdout: "svn_revision": "5155", 2024-09-12T23:37:50.579 INFO:tasks.workunit.client.0.smithi136.stdout: "version": [ 2024-09-12T23:37:50.579 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:50.579 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:50.579 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:50.579 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.579 INFO:tasks.workunit.client.0.smithi136.stdout: "temperature": { 2024-09-12T23:37:50.580 INFO:tasks.workunit.client.0.smithi136.stdout: "current": 23 2024-09-12T23:37:50.580 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.580 INFO:tasks.workunit.client.0.smithi136.stdout: "user_capacity": { 2024-09-12T23:37:50.580 INFO:tasks.workunit.client.0.smithi136.stdout: "blocks": 3907029168, 2024-09-12T23:37:50.580 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 2000398934016 2024-09-12T23:37:50.580 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.580 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.580 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.580 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.580 INFO:tasks.workunit.client.0.smithi136.stdout:} 2024-09-12T23:37:50.581 INFO:tasks.workunit.client.0.smithi136.stderr:+ ceph telemetry show-all 2024-09-12T23:37:50.581 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[22601]: audit 2024-09-12T23:37:47.889857+0000 mgr.y (mgr.24446) 158 : audit [DBG] from='client.24818 -' entity='client.admin' cmd=[{"prefix": "telemetry show", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:50.581 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[22601]: cluster 2024-09-12T23:37:47.998417+0000 mgr.y (mgr.24446) 159 : cluster [DBG] pgmap v122: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:50.581 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[22601]: audit 2024-09-12T23:37:48.299024+0000 mgr.y (mgr.24446) 160 : audit [DBG] from='client.24818 -' entity='client.admin' cmd=[{"prefix": "telemetry show", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:50.581 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[22601]: audit 2024-09-12T23:37:48.481240+0000 mon.a (mon.0) 999 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:50.581 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[22601]: audit 2024-09-12T23:37:48.494410+0000 mon.a (mon.0) 1000 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:50.581 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[22601]: audit 2024-09-12T23:37:48.494897+0000 mon.a (mon.0) 1001 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:50.582 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[22601]: audit 2024-09-12T23:37:48.498555+0000 mon.a (mon.0) 1002 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:50.582 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[22601]: audit 2024-09-12T23:37:48.499042+0000 mon.a (mon.0) 1003 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:50.582 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[22601]: audit 2024-09-12T23:37:48.499365+0000 mon.a (mon.0) 1004 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:50.582 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[22601]: audit 2024-09-12T23:37:48.502336+0000 mon.a (mon.0) 1005 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:50.582 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[22601]: audit 2024-09-12T23:37:48.502793+0000 mon.a (mon.0) 1006 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:50.582 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[22601]: audit 2024-09-12T23:37:48.503261+0000 mon.a (mon.0) 1007 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:50.582 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[22601]: audit 2024-09-12T23:37:48.521194+0000 mon.a (mon.0) 1008 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:37:50.582 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[22601]: audit 2024-09-12T23:37:48.521283+0000 mon.a (mon.0) 1009 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:37:50.582 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[22601]: audit 2024-09-12T23:37:48.521734+0000 mon.a (mon.0) 1010 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:50.582 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[22601]: audit 2024-09-12T23:37:48.522223+0000 mon.a (mon.0) 1011 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:50.583 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[22601]: audit 2024-09-12T23:37:48.522661+0000 mon.a (mon.0) 1012 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:50.583 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[22601]: audit 2024-09-12T23:37:48.525328+0000 mon.b (mon.1) 30 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:37:50.583 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[22601]: audit 2024-09-12T23:37:48.525419+0000 mon.b (mon.1) 31 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:37:50.583 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[32189]: audit 2024-09-12T23:37:47.889857+0000 mgr.y (mgr.24446) 158 : audit [DBG] from='client.24818 -' entity='client.admin' cmd=[{"prefix": "telemetry show", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:50.583 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[32189]: cluster 2024-09-12T23:37:47.998417+0000 mgr.y (mgr.24446) 159 : cluster [DBG] pgmap v122: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:37:50.583 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[32189]: audit 2024-09-12T23:37:48.299024+0000 mgr.y (mgr.24446) 160 : audit [DBG] from='client.24818 -' entity='client.admin' cmd=[{"prefix": "telemetry show", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:50.583 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[32189]: audit 2024-09-12T23:37:48.481240+0000 mon.a (mon.0) 999 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:50.583 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[32189]: audit 2024-09-12T23:37:48.494410+0000 mon.a (mon.0) 1000 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:50.583 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[32189]: audit 2024-09-12T23:37:48.494897+0000 mon.a (mon.0) 1001 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:50.584 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[32189]: audit 2024-09-12T23:37:48.498555+0000 mon.a (mon.0) 1002 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:50.584 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[32189]: audit 2024-09-12T23:37:48.499042+0000 mon.a (mon.0) 1003 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:50.584 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[32189]: audit 2024-09-12T23:37:48.499365+0000 mon.a (mon.0) 1004 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:50.584 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[32189]: audit 2024-09-12T23:37:48.502336+0000 mon.a (mon.0) 1005 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:50.584 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[32189]: audit 2024-09-12T23:37:48.502793+0000 mon.a (mon.0) 1006 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:50.584 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[32189]: audit 2024-09-12T23:37:48.503261+0000 mon.a (mon.0) 1007 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:50.584 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[32189]: audit 2024-09-12T23:37:48.521194+0000 mon.a (mon.0) 1008 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:37:50.584 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[32189]: audit 2024-09-12T23:37:48.521283+0000 mon.a (mon.0) 1009 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:37:50.584 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[32189]: audit 2024-09-12T23:37:48.521734+0000 mon.a (mon.0) 1010 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:50.584 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[32189]: audit 2024-09-12T23:37:48.522223+0000 mon.a (mon.0) 1011 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:50.585 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[32189]: audit 2024-09-12T23:37:48.522661+0000 mon.a (mon.0) 1012 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:50.585 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[32189]: audit 2024-09-12T23:37:48.525328+0000 mon.b (mon.1) 30 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:37:50.585 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[32189]: audit 2024-09-12T23:37:48.525419+0000 mon.b (mon.1) 31 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:37:50.585 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[22601]: audit 2024-09-12T23:37:48.525958+0000 mon.a (mon.0) 1013 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:50.585 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[22601]: audit 2024-09-12T23:37:48.526678+0000 mon.a (mon.0) 1014 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:50.585 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[22601]: audit 2024-09-12T23:37:48.527042+0000 mon.a (mon.0) 1015 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:50.585 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[22601]: audit 2024-09-12T23:37:48.530349+0000 mon.c (mon.2) 53 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:37:50.585 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[22601]: audit 2024-09-12T23:37:48.530482+0000 mon.c (mon.2) 54 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:37:50.586 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[22601]: audit 2024-09-12T23:37:48.530869+0000 mon.a (mon.0) 1016 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:50.586 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[22601]: audit 2024-09-12T23:37:48.531630+0000 mon.a (mon.0) 1017 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:50.586 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[22601]: audit 2024-09-12T23:37:48.531972+0000 mon.a (mon.0) 1018 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:50.586 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[32189]: audit 2024-09-12T23:37:48.525958+0000 mon.a (mon.0) 1013 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:50.586 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[32189]: audit 2024-09-12T23:37:48.526678+0000 mon.a (mon.0) 1014 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:50.586 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[32189]: audit 2024-09-12T23:37:48.527042+0000 mon.a (mon.0) 1015 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:50.586 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[32189]: audit 2024-09-12T23:37:48.530349+0000 mon.c (mon.2) 53 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:37:50.587 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[32189]: audit 2024-09-12T23:37:48.530482+0000 mon.c (mon.2) 54 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:37:50.587 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[32189]: audit 2024-09-12T23:37:48.530869+0000 mon.a (mon.0) 1016 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:50.587 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[32189]: audit 2024-09-12T23:37:48.531630+0000 mon.a (mon.0) 1017 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:50.587 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:49 smithi136 bash[32189]: audit 2024-09-12T23:37:48.531972+0000 mon.a (mon.0) 1018 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:50.593 INFO:tasks.workunit.client.0.smithi136.stdout:{ 2024-09-12T23:37:50.593 INFO:tasks.workunit.client.0.smithi136.stdout: "device_report": { 2024-09-12T23:37:50.593 INFO:tasks.workunit.client.0.smithi136.stdout: "02b0b25a-7160-11ef-baee-0cc47a886e43": { 2024-09-12T23:37:50.593 INFO:tasks.workunit.client.0.smithi136.stdout: "INTEL_SSDPEDMD400G4_02b1c294-7160-11ef-baee-0cc47a886e43": { 2024-09-12T23:37:50.593 INFO:tasks.workunit.client.0.smithi136.stdout: "20240912-232840": { 2024-09-12T23:37:50.593 INFO:tasks.workunit.client.0.smithi136.stdout: "device": { 2024-09-12T23:37:50.593 INFO:tasks.workunit.client.0.smithi136.stdout: "info_name": "/dev/nvme0n1", 2024-09-12T23:37:50.593 INFO:tasks.workunit.client.0.smithi136.stdout: "name": "/dev/nvme0n1", 2024-09-12T23:37:50.593 INFO:tasks.workunit.client.0.smithi136.stdout: "protocol": "NVMe", 2024-09-12T23:37:50.593 INFO:tasks.workunit.client.0.smithi136.stdout: "type": "nvme" 2024-09-12T23:37:50.594 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.594 INFO:tasks.workunit.client.0.smithi136.stdout: "firmware_version": "8DV101H0", 2024-09-12T23:37:50.594 INFO:tasks.workunit.client.0.smithi136.stdout: "host_id": "02b0b25a-7160-11ef-baee-0cc47a886e43", 2024-09-12T23:37:50.594 INFO:tasks.workunit.client.0.smithi136.stdout: "json_format_version": [ 2024-09-12T23:37:50.594 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:50.594 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:50.594 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.594 INFO:tasks.workunit.client.0.smithi136.stdout: "local_time": { 2024-09-12T23:37:50.594 INFO:tasks.workunit.client.0.smithi136.stdout: "asctime": "Thu Sep 12 23:28:40 2024 UTC", 2024-09-12T23:37:50.594 INFO:tasks.workunit.client.0.smithi136.stdout: "time_t": 1726183720 2024-09-12T23:37:50.594 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.595 INFO:tasks.workunit.client.0.smithi136.stdout: "logical_block_size": 512, 2024-09-12T23:37:50.595 INFO:tasks.workunit.client.0.smithi136.stdout: "model_name": "INTEL SSDPEDMD400G4", 2024-09-12T23:37:50.595 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_controller_id": 0, 2024-09-12T23:37:50.595 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_ieee_oui_identifier": 6083300, 2024-09-12T23:37:50.595 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_namespaces": [ 2024-09-12T23:37:50.595 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.595 INFO:tasks.workunit.client.0.smithi136.stdout: "capacity": { 2024-09-12T23:37:50.595 INFO:tasks.workunit.client.0.smithi136.stdout: "blocks": 781422768, 2024-09-12T23:37:50.595 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 400088457216 2024-09-12T23:37:50.595 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.595 INFO:tasks.workunit.client.0.smithi136.stdout: "formatted_lba_size": 512, 2024-09-12T23:37:50.595 INFO:tasks.workunit.client.0.smithi136.stdout: "id": 1, 2024-09-12T23:37:50.596 INFO:tasks.workunit.client.0.smithi136.stdout: "size": { 2024-09-12T23:37:50.596 INFO:tasks.workunit.client.0.smithi136.stdout: "blocks": 781422768, 2024-09-12T23:37:50.596 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 400088457216 2024-09-12T23:37:50.596 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.596 INFO:tasks.workunit.client.0.smithi136.stdout: "utilization": { 2024-09-12T23:37:50.596 INFO:tasks.workunit.client.0.smithi136.stdout: "blocks": 781422768, 2024-09-12T23:37:50.596 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 400088457216 2024-09-12T23:37:50.596 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.596 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.596 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.596 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_number_of_namespaces": 1, 2024-09-12T23:37:50.597 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_pci_vendor": { 2024-09-12T23:37:50.597 INFO:tasks.workunit.client.0.smithi136.stdout: "id": 32902, 2024-09-12T23:37:50.597 INFO:tasks.workunit.client.0.smithi136.stdout: "subsystem_id": 32902 2024-09-12T23:37:50.597 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.597 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_smart_health_information_add_log": { 2024-09-12T23:37:50.597 INFO:tasks.workunit.client.0.smithi136.stdout: "Device stats": { 2024-09-12T23:37:50.597 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_fw_downgrades": { 2024-09-12T23:37:50.597 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 0, 2024-09-12T23:37:50.597 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:50.597 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.598 INFO:tasks.workunit.client.0.smithi136.stdout: "crc_error_count": { 2024-09-12T23:37:50.598 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:50.598 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:50.598 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.598 INFO:tasks.workunit.client.0.smithi136.stdout: "end_to_end_error_detection_count": { 2024-09-12T23:37:50.598 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:50.598 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:50.598 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.598 INFO:tasks.workunit.client.0.smithi136.stdout: "erase_fail_count": { 2024-09-12T23:37:50.598 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:50.598 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:50.598 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.599 INFO:tasks.workunit.client.0.smithi136.stdout: "host_bytes_written": { 2024-09-12T23:37:50.599 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:50.599 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 26605852 2024-09-12T23:37:50.599 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.599 INFO:tasks.workunit.client.0.smithi136.stdout: "host_ctx_wear_used": { 2024-09-12T23:37:50.599 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:50.599 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 1 2024-09-12T23:37:50.599 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.599 INFO:tasks.workunit.client.0.smithi136.stdout: "media_bytes_read": { 2024-09-12T23:37:50.599 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 0, 2024-09-12T23:37:50.599 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:50.600 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.600 INFO:tasks.workunit.client.0.smithi136.stdout: "nand_bytes_written": { 2024-09-12T23:37:50.600 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:50.600 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 47676652 2024-09-12T23:37:50.600 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.600 INFO:tasks.workunit.client.0.smithi136.stdout: "perf_stat_indicator": { 2024-09-12T23:37:50.600 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 0, 2024-09-12T23:37:50.600 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:50.600 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.600 INFO:tasks.workunit.client.0.smithi136.stdout: "pll_lock_loss_count": { 2024-09-12T23:37:50.600 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:50.601 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:50.601 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.601 INFO:tasks.workunit.client.0.smithi136.stdout: "program_fail_count": { 2024-09-12T23:37:50.601 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:50.601 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:50.601 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.601 INFO:tasks.workunit.client.0.smithi136.stdout: "re_alloc_sectr_cnt": { 2024-09-12T23:37:50.601 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 0, 2024-09-12T23:37:50.601 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:50.601 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.601 INFO:tasks.workunit.client.0.smithi136.stdout: "retry_buffer_overflow_count": { 2024-09-12T23:37:50.602 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:50.602 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:50.602 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.602 INFO:tasks.workunit.client.0.smithi136.stdout: "soft_ecc_err_rate": { 2024-09-12T23:37:50.602 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 0, 2024-09-12T23:37:50.602 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:50.602 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.602 INFO:tasks.workunit.client.0.smithi136.stdout: "thermal_throttle_status": { 2024-09-12T23:37:50.602 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:50.602 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": { 2024-09-12T23:37:50.602 INFO:tasks.workunit.client.0.smithi136.stdout: "cnt": 0, 2024-09-12T23:37:50.603 INFO:tasks.workunit.client.0.smithi136.stdout: "pct": 0 2024-09-12T23:37:50.603 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.603 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.603 INFO:tasks.workunit.client.0.smithi136.stdout: "timed_workload_host_reads": { 2024-09-12T23:37:50.603 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:50.603 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 65535 2024-09-12T23:37:50.603 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.603 INFO:tasks.workunit.client.0.smithi136.stdout: "timed_workload_media_wear": { 2024-09-12T23:37:50.603 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:50.603 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": "63.999023" 2024-09-12T23:37:50.603 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.604 INFO:tasks.workunit.client.0.smithi136.stdout: "timed_workload_timer": { 2024-09-12T23:37:50.604 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:50.604 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 65535 2024-09-12T23:37:50.604 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.604 INFO:tasks.workunit.client.0.smithi136.stdout: "unexp_power_loss": { 2024-09-12T23:37:50.604 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 0, 2024-09-12T23:37:50.604 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:50.604 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.604 INFO:tasks.workunit.client.0.smithi136.stdout: "wear_leveling": { 2024-09-12T23:37:50.604 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 90, 2024-09-12T23:37:50.604 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": { 2024-09-12T23:37:50.604 INFO:tasks.workunit.client.0.smithi136.stdout: "avg": 3275, 2024-09-12T23:37:50.605 INFO:tasks.workunit.client.0.smithi136.stdout: "max": 3290, 2024-09-12T23:37:50.605 INFO:tasks.workunit.client.0.smithi136.stdout: "min": 3260 2024-09-12T23:37:50.605 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.605 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.605 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.605 INFO:tasks.workunit.client.0.smithi136.stdout: "Intel Smart log": "nvme0n1" 2024-09-12T23:37:50.605 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.605 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_smart_health_information_log": { 2024-09-12T23:37:50.605 INFO:tasks.workunit.client.0.smithi136.stdout: "available_spare": 100, 2024-09-12T23:37:50.605 INFO:tasks.workunit.client.0.smithi136.stdout: "available_spare_threshold": 10, 2024-09-12T23:37:50.605 INFO:tasks.workunit.client.0.smithi136.stdout: "controller_busy_time": 50, 2024-09-12T23:37:50.606 INFO:tasks.workunit.client.0.smithi136.stdout: "critical_warning": 0, 2024-09-12T23:37:50.606 INFO:tasks.workunit.client.0.smithi136.stdout: "data_units_read": 360344011, 2024-09-12T23:37:50.606 INFO:tasks.workunit.client.0.smithi136.stdout: "data_units_written": 1743641130, 2024-09-12T23:37:50.606 INFO:tasks.workunit.client.0.smithi136.stdout: "host_reads": 5879482063, 2024-09-12T23:37:50.606 INFO:tasks.workunit.client.0.smithi136.stdout: "host_writes": 31824656568, 2024-09-12T23:37:50.606 INFO:tasks.workunit.client.0.smithi136.stdout: "media_errors": 0, 2024-09-12T23:37:50.606 INFO:tasks.workunit.client.0.smithi136.stdout: "num_err_log_entries": 0, 2024-09-12T23:37:50.606 INFO:tasks.workunit.client.0.smithi136.stdout: "percentage_used": 10, 2024-09-12T23:37:50.606 INFO:tasks.workunit.client.0.smithi136.stdout: "power_cycles": 56409, 2024-09-12T23:37:50.606 INFO:tasks.workunit.client.0.smithi136.stdout: "power_on_hours": 54400, 2024-09-12T23:37:50.606 INFO:tasks.workunit.client.0.smithi136.stdout: "temperature": 23, 2024-09-12T23:37:50.606 INFO:tasks.workunit.client.0.smithi136.stdout: "unsafe_shutdowns": 2575 2024-09-12T23:37:50.607 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.607 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_vendor": "intel", 2024-09-12T23:37:50.607 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_version": { 2024-09-12T23:37:50.607 INFO:tasks.workunit.client.0.smithi136.stdout: "string": "<1.2", 2024-09-12T23:37:50.607 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.607 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.607 INFO:tasks.workunit.client.0.smithi136.stdout: "power_cycle_count": 56409, 2024-09-12T23:37:50.607 INFO:tasks.workunit.client.0.smithi136.stdout: "power_on_time": { 2024-09-12T23:37:50.607 INFO:tasks.workunit.client.0.smithi136.stdout: "hours": 54400 2024-09-12T23:37:50.607 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.607 INFO:tasks.workunit.client.0.smithi136.stdout: "serial_number": "deleted", 2024-09-12T23:37:50.607 INFO:tasks.workunit.client.0.smithi136.stdout: "smart_status": { 2024-09-12T23:37:50.608 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme": { 2024-09-12T23:37:50.608 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.608 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.608 INFO:tasks.workunit.client.0.smithi136.stdout: "passed": true 2024-09-12T23:37:50.608 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.608 INFO:tasks.workunit.client.0.smithi136.stdout: "smartctl": { 2024-09-12T23:37:50.608 INFO:tasks.workunit.client.0.smithi136.stdout: "argv": [ 2024-09-12T23:37:50.608 INFO:tasks.workunit.client.0.smithi136.stdout: "smartctl", 2024-09-12T23:37:50.608 INFO:tasks.workunit.client.0.smithi136.stdout: "-x", 2024-09-12T23:37:50.608 INFO:tasks.workunit.client.0.smithi136.stdout: "--json=o", 2024-09-12T23:37:50.608 INFO:tasks.workunit.client.0.smithi136.stdout: "/dev/nvme0n1" 2024-09-12T23:37:50.609 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.609 INFO:tasks.workunit.client.0.smithi136.stdout: "build_info": "(local build)", 2024-09-12T23:37:50.609 INFO:tasks.workunit.client.0.smithi136.stdout: "exit_status": 0, 2024-09-12T23:37:50.609 INFO:tasks.workunit.client.0.smithi136.stdout: "output": [ 2024-09-12T23:37:50.609 INFO:tasks.workunit.client.0.smithi136.stdout: "smartctl 7.2 2020-12-30 r5155 [x86_64-linux-5.15.0-121-generic] (local build)", 2024-09-12T23:37:50.609 INFO:tasks.workunit.client.0.smithi136.stdout: "Copyright (C) 2002-20, Bruce Allen, Christian Franke, www.smartmontools.org", 2024-09-12T23:37:50.609 INFO:tasks.workunit.client.0.smithi136.stdout: "", 2024-09-12T23:37:50.609 INFO:tasks.workunit.client.0.smithi136.stdout: "=== START OF INFORMATION SECTION ===", 2024-09-12T23:37:50.609 INFO:tasks.workunit.client.0.smithi136.stdout: "Model Number: INTEL SSDPEDMD400G4", 2024-09-12T23:37:50.609 INFO:tasks.workunit.client.0.smithi136.stdout: "Serial Number: deleted", 2024-09-12T23:37:50.609 INFO:tasks.workunit.client.0.smithi136.stdout: "Firmware Version: 8DV101H0", 2024-09-12T23:37:50.610 INFO:tasks.workunit.client.0.smithi136.stdout: "PCI Vendor/Subsystem ID: 0x8086", 2024-09-12T23:37:50.610 INFO:tasks.workunit.client.0.smithi136.stdout: "IEEE OUI Identifier: 0x5cd2e4", 2024-09-12T23:37:50.610 INFO:tasks.workunit.client.0.smithi136.stdout: "Controller ID: 0", 2024-09-12T23:37:50.610 INFO:tasks.workunit.client.0.smithi136.stdout: "NVMe Version: <1.2", 2024-09-12T23:37:50.610 INFO:tasks.workunit.client.0.smithi136.stdout: "Number of Namespaces: 1", 2024-09-12T23:37:50.610 INFO:tasks.workunit.client.0.smithi136.stdout: "Namespace 1 Size/Capacity: 400,088,457,216 [400 GB]", 2024-09-12T23:37:50.610 INFO:tasks.workunit.client.0.smithi136.stdout: "Namespace 1 Formatted LBA Size: 512", 2024-09-12T23:37:50.610 INFO:tasks.workunit.client.0.smithi136.stdout: "Local Time is: Thu Sep 12 23:28:40 2024 UTC", 2024-09-12T23:37:50.610 INFO:tasks.workunit.client.0.smithi136.stdout: "Firmware Updates (0x02): 1 Slot", 2024-09-12T23:37:50.610 INFO:tasks.workunit.client.0.smithi136.stdout: "Optional Admin Commands (0x0006): Format Frmw_DL", 2024-09-12T23:37:50.610 INFO:tasks.workunit.client.0.smithi136.stdout: "Optional NVM Commands (0x0006): Wr_Unc DS_Mngmt", 2024-09-12T23:37:50.610 INFO:tasks.workunit.client.0.smithi136.stdout: "Log Page Attributes (0x02): Cmd_Eff_Lg", 2024-09-12T23:37:50.611 INFO:tasks.workunit.client.0.smithi136.stdout: "Maximum Data Transfer Size: 32 Pages", 2024-09-12T23:37:50.611 INFO:tasks.workunit.client.0.smithi136.stdout: "", 2024-09-12T23:37:50.611 INFO:tasks.workunit.client.0.smithi136.stdout: "Supported Power States", 2024-09-12T23:37:50.611 INFO:tasks.workunit.client.0.smithi136.stdout: "St Op Max Active Idle RL RT WL WT Ent_Lat Ex_Lat", 2024-09-12T23:37:50.611 INFO:tasks.workunit.client.0.smithi136.stdout: " 0 + 25.00W - - 0 0 0 0 0 0", 2024-09-12T23:37:50.611 INFO:tasks.workunit.client.0.smithi136.stdout: "", 2024-09-12T23:37:50.611 INFO:tasks.workunit.client.0.smithi136.stdout: "Supported LBA Sizes (NSID 0x1)", 2024-09-12T23:37:50.611 INFO:tasks.workunit.client.0.smithi136.stdout: "Id Fmt Data Metadt Rel_Perf", 2024-09-12T23:37:50.611 INFO:tasks.workunit.client.0.smithi136.stdout: " 0 + 512 0 2", 2024-09-12T23:37:50.611 INFO:tasks.workunit.client.0.smithi136.stdout: " 1 - 512 8 2", 2024-09-12T23:37:50.611 INFO:tasks.workunit.client.0.smithi136.stdout: " 2 - 512 16 2", 2024-09-12T23:37:50.611 INFO:tasks.workunit.client.0.smithi136.stdout: " 3 - 4096 0 0", 2024-09-12T23:37:50.612 INFO:tasks.workunit.client.0.smithi136.stdout: " 4 - 4096 8 0", 2024-09-12T23:37:50.612 INFO:tasks.workunit.client.0.smithi136.stdout: " 5 - 4096 64 0", 2024-09-12T23:37:50.612 INFO:tasks.workunit.client.0.smithi136.stdout: " 6 - 4096 128 0", 2024-09-12T23:37:50.612 INFO:tasks.workunit.client.0.smithi136.stdout: "", 2024-09-12T23:37:50.612 INFO:tasks.workunit.client.0.smithi136.stdout: "=== START OF SMART DATA SECTION ===", 2024-09-12T23:37:50.612 INFO:tasks.workunit.client.0.smithi136.stdout: "SMART overall-health self-assessment test result: PASSED", 2024-09-12T23:37:50.612 INFO:tasks.workunit.client.0.smithi136.stdout: "", 2024-09-12T23:37:50.612 INFO:tasks.workunit.client.0.smithi136.stdout: "SMART/Health Information (NVMe Log 0x02)", 2024-09-12T23:37:50.612 INFO:tasks.workunit.client.0.smithi136.stdout: "Critical Warning: 0x00", 2024-09-12T23:37:50.612 INFO:tasks.workunit.client.0.smithi136.stdout: "Temperature: 23 Celsius", 2024-09-12T23:37:50.612 INFO:tasks.workunit.client.0.smithi136.stdout: "Available Spare: 100%", 2024-09-12T23:37:50.613 INFO:tasks.workunit.client.0.smithi136.stdout: "Available Spare Threshold: 10%", 2024-09-12T23:37:50.613 INFO:tasks.workunit.client.0.smithi136.stdout: "Percentage Used: 10%", 2024-09-12T23:37:50.613 INFO:tasks.workunit.client.0.smithi136.stdout: "Data Units Read: 360,344,011 [184 TB]", 2024-09-12T23:37:50.613 INFO:tasks.workunit.client.0.smithi136.stdout: "Data Units Written: 1,743,641,130 [892 TB]", 2024-09-12T23:37:50.613 INFO:tasks.workunit.client.0.smithi136.stdout: "Host Read Commands: 5,879,482,063", 2024-09-12T23:37:50.613 INFO:tasks.workunit.client.0.smithi136.stdout: "Host Write Commands: 31,824,656,568", 2024-09-12T23:37:50.613 INFO:tasks.workunit.client.0.smithi136.stdout: "Controller Busy Time: 50", 2024-09-12T23:37:50.613 INFO:tasks.workunit.client.0.smithi136.stdout: "Power Cycles: 56,409", 2024-09-12T23:37:50.613 INFO:tasks.workunit.client.0.smithi136.stdout: "Power On Hours: 54,400", 2024-09-12T23:37:50.613 INFO:tasks.workunit.client.0.smithi136.stdout: "Unsafe Shutdowns: 2,575", 2024-09-12T23:37:50.614 INFO:tasks.workunit.client.0.smithi136.stdout: "Media and Data Integrity Errors: 0", 2024-09-12T23:37:50.614 INFO:tasks.workunit.client.0.smithi136.stdout: "Error Information Log Entries: 0", 2024-09-12T23:37:50.614 INFO:tasks.workunit.client.0.smithi136.stdout: "", 2024-09-12T23:37:50.614 INFO:tasks.workunit.client.0.smithi136.stdout: "Error Information (NVMe Log 0x01, 16 of 64 entries)", 2024-09-12T23:37:50.614 INFO:tasks.workunit.client.0.smithi136.stdout: "No Errors Logged", 2024-09-12T23:37:50.614 INFO:tasks.workunit.client.0.smithi136.stdout: "" 2024-09-12T23:37:50.614 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.614 INFO:tasks.workunit.client.0.smithi136.stdout: "platform_info": "x86_64-linux-5.15.0-121-generic", 2024-09-12T23:37:50.614 INFO:tasks.workunit.client.0.smithi136.stdout: "svn_revision": "5155", 2024-09-12T23:37:50.614 INFO:tasks.workunit.client.0.smithi136.stdout: "version": [ 2024-09-12T23:37:50.614 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:50.615 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:50.615 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:50.615 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.615 INFO:tasks.workunit.client.0.smithi136.stdout: "temperature": { 2024-09-12T23:37:50.615 INFO:tasks.workunit.client.0.smithi136.stdout: "current": 23 2024-09-12T23:37:50.615 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.615 INFO:tasks.workunit.client.0.smithi136.stdout: "user_capacity": { 2024-09-12T23:37:50.615 INFO:tasks.workunit.client.0.smithi136.stdout: "blocks": 781422768, 2024-09-12T23:37:50.615 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 400088457216 2024-09-12T23:37:50.615 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.615 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.616 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.616 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.616 INFO:tasks.workunit.client.0.smithi136.stdout: "02b4b8aa-7160-11ef-baee-0cc47a886e43": { 2024-09-12T23:37:50.616 INFO:tasks.workunit.client.0.smithi136.stdout: "INTEL_SSDPEDMX020T4_02b5a97c-7160-11ef-baee-0cc47a886e43": { 2024-09-12T23:37:50.616 INFO:tasks.workunit.client.0.smithi136.stdout: "20240912-232836": { 2024-09-12T23:37:50.616 INFO:tasks.workunit.client.0.smithi136.stdout: "device": { 2024-09-12T23:37:50.616 INFO:tasks.workunit.client.0.smithi136.stdout: "info_name": "/dev/nvme0n1", 2024-09-12T23:37:50.616 INFO:tasks.workunit.client.0.smithi136.stdout: "name": "/dev/nvme0n1", 2024-09-12T23:37:50.616 INFO:tasks.workunit.client.0.smithi136.stdout: "protocol": "NVMe", 2024-09-12T23:37:50.616 INFO:tasks.workunit.client.0.smithi136.stdout: "type": "nvme" 2024-09-12T23:37:50.616 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.617 INFO:tasks.workunit.client.0.smithi136.stdout: "firmware_version": "8DV10171", 2024-09-12T23:37:50.617 INFO:tasks.workunit.client.0.smithi136.stdout: "host_id": "02b4b8aa-7160-11ef-baee-0cc47a886e43", 2024-09-12T23:37:50.617 INFO:tasks.workunit.client.0.smithi136.stdout: "json_format_version": [ 2024-09-12T23:37:50.617 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:50.617 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:50.617 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.617 INFO:tasks.workunit.client.0.smithi136.stdout: "local_time": { 2024-09-12T23:37:50.617 INFO:tasks.workunit.client.0.smithi136.stdout: "asctime": "Thu Sep 12 23:28:36 2024 UTC", 2024-09-12T23:37:50.617 INFO:tasks.workunit.client.0.smithi136.stdout: "time_t": 1726183716 2024-09-12T23:37:50.617 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.618 INFO:tasks.workunit.client.0.smithi136.stdout: "logical_block_size": 512, 2024-09-12T23:37:50.618 INFO:tasks.workunit.client.0.smithi136.stdout: "model_name": "INTEL SSDPEDMX020T4", 2024-09-12T23:37:50.618 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_controller_id": 0, 2024-09-12T23:37:50.618 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_ieee_oui_identifier": 6083300, 2024-09-12T23:37:50.618 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_namespaces": [ 2024-09-12T23:37:50.618 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.618 INFO:tasks.workunit.client.0.smithi136.stdout: "capacity": { 2024-09-12T23:37:50.618 INFO:tasks.workunit.client.0.smithi136.stdout: "blocks": 3907029168, 2024-09-12T23:37:50.618 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 2000398934016 2024-09-12T23:37:50.618 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.619 INFO:tasks.workunit.client.0.smithi136.stdout: "formatted_lba_size": 512, 2024-09-12T23:37:50.619 INFO:tasks.workunit.client.0.smithi136.stdout: "id": 1, 2024-09-12T23:37:50.619 INFO:tasks.workunit.client.0.smithi136.stdout: "size": { 2024-09-12T23:37:50.619 INFO:tasks.workunit.client.0.smithi136.stdout: "blocks": 3907029168, 2024-09-12T23:37:50.619 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 2000398934016 2024-09-12T23:37:50.619 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.619 INFO:tasks.workunit.client.0.smithi136.stdout: "utilization": { 2024-09-12T23:37:50.619 INFO:tasks.workunit.client.0.smithi136.stdout: "blocks": 3907029168, 2024-09-12T23:37:50.619 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 2000398934016 2024-09-12T23:37:50.619 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.620 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.620 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.620 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_number_of_namespaces": 1, 2024-09-12T23:37:50.620 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_pci_vendor": { 2024-09-12T23:37:50.620 INFO:tasks.workunit.client.0.smithi136.stdout: "id": 32902, 2024-09-12T23:37:50.620 INFO:tasks.workunit.client.0.smithi136.stdout: "subsystem_id": 32902 2024-09-12T23:37:50.620 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.620 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_smart_health_information_add_log": { 2024-09-12T23:37:50.620 INFO:tasks.workunit.client.0.smithi136.stdout: "Device stats": { 2024-09-12T23:37:50.620 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_fw_downgrades": { 2024-09-12T23:37:50.621 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 0, 2024-09-12T23:37:50.621 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:50.621 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.621 INFO:tasks.workunit.client.0.smithi136.stdout: "crc_error_count": { 2024-09-12T23:37:50.621 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:50.621 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 2 2024-09-12T23:37:50.621 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.621 INFO:tasks.workunit.client.0.smithi136.stdout: "end_to_end_error_detection_count": { 2024-09-12T23:37:50.621 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:50.621 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:50.621 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.621 INFO:tasks.workunit.client.0.smithi136.stdout: "erase_fail_count": { 2024-09-12T23:37:50.622 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:50.622 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:50.622 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.622 INFO:tasks.workunit.client.0.smithi136.stdout: "host_bytes_written": { 2024-09-12T23:37:50.622 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:50.622 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 19052464 2024-09-12T23:37:50.622 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.622 INFO:tasks.workunit.client.0.smithi136.stdout: "host_ctx_wear_used": { 2024-09-12T23:37:50.622 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 0, 2024-09-12T23:37:50.622 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:50.622 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.622 INFO:tasks.workunit.client.0.smithi136.stdout: "media_bytes_read": { 2024-09-12T23:37:50.623 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 0, 2024-09-12T23:37:50.623 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:50.623 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.623 INFO:tasks.workunit.client.0.smithi136.stdout: "nand_bytes_written": { 2024-09-12T23:37:50.623 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:50.623 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 50938213 2024-09-12T23:37:50.623 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.623 INFO:tasks.workunit.client.0.smithi136.stdout: "perf_stat_indicator": { 2024-09-12T23:37:50.623 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 0, 2024-09-12T23:37:50.623 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:50.623 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.624 INFO:tasks.workunit.client.0.smithi136.stdout: "pll_lock_loss_count": { 2024-09-12T23:37:50.624 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:50.624 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:50.624 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.624 INFO:tasks.workunit.client.0.smithi136.stdout: "program_fail_count": { 2024-09-12T23:37:50.624 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:50.624 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:50.624 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.624 INFO:tasks.workunit.client.0.smithi136.stdout: "re_alloc_sectr_cnt": { 2024-09-12T23:37:50.624 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 0, 2024-09-12T23:37:50.625 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:50.625 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.625 INFO:tasks.workunit.client.0.smithi136.stdout: "retry_buffer_overflow_count": { 2024-09-12T23:37:50.625 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:50.625 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:50.625 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.625 INFO:tasks.workunit.client.0.smithi136.stdout: "soft_ecc_err_rate": { 2024-09-12T23:37:50.625 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 0, 2024-09-12T23:37:50.625 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:50.625 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.625 INFO:tasks.workunit.client.0.smithi136.stdout: "thermal_throttle_status": { 2024-09-12T23:37:50.626 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:50.626 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": { 2024-09-12T23:37:50.626 INFO:tasks.workunit.client.0.smithi136.stdout: "cnt": 0, 2024-09-12T23:37:50.626 INFO:tasks.workunit.client.0.smithi136.stdout: "pct": 0 2024-09-12T23:37:50.626 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.626 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.626 INFO:tasks.workunit.client.0.smithi136.stdout: "timed_workload_host_reads": { 2024-09-12T23:37:50.626 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:50.626 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 15 2024-09-12T23:37:50.626 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.626 INFO:tasks.workunit.client.0.smithi136.stdout: "timed_workload_media_wear": { 2024-09-12T23:37:50.627 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:50.627 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": "6.850586" 2024-09-12T23:37:50.627 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.627 INFO:tasks.workunit.client.0.smithi136.stdout: "timed_workload_timer": { 2024-09-12T23:37:50.627 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 100, 2024-09-12T23:37:50.627 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 1541953 2024-09-12T23:37:50.627 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.627 INFO:tasks.workunit.client.0.smithi136.stdout: "unexp_power_loss": { 2024-09-12T23:37:50.627 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 0, 2024-09-12T23:37:50.627 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": 0 2024-09-12T23:37:50.627 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.627 INFO:tasks.workunit.client.0.smithi136.stdout: "wear_leveling": { 2024-09-12T23:37:50.628 INFO:tasks.workunit.client.0.smithi136.stdout: "normalized": 47, 2024-09-12T23:37:50.628 INFO:tasks.workunit.client.0.smithi136.stdout: "raw": { 2024-09-12T23:37:50.628 INFO:tasks.workunit.client.0.smithi136.stdout: "avg": 1165, 2024-09-12T23:37:50.628 INFO:tasks.workunit.client.0.smithi136.stdout: "max": 1608, 2024-09-12T23:37:50.628 INFO:tasks.workunit.client.0.smithi136.stdout: "min": 765 2024-09-12T23:37:50.628 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.628 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.628 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.628 INFO:tasks.workunit.client.0.smithi136.stdout: "Intel Smart log": "nvme0n1" 2024-09-12T23:37:50.628 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.628 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_smart_health_information_log": { 2024-09-12T23:37:50.628 INFO:tasks.workunit.client.0.smithi136.stdout: "available_spare": 100, 2024-09-12T23:37:50.629 INFO:tasks.workunit.client.0.smithi136.stdout: "available_spare_threshold": 10, 2024-09-12T23:37:50.629 INFO:tasks.workunit.client.0.smithi136.stdout: "controller_busy_time": 358, 2024-09-12T23:37:50.629 INFO:tasks.workunit.client.0.smithi136.stdout: "critical_warning": 0, 2024-09-12T23:37:50.629 INFO:tasks.workunit.client.0.smithi136.stdout: "data_units_read": 215101386, 2024-09-12T23:37:50.629 INFO:tasks.workunit.client.0.smithi136.stdout: "data_units_written": 1248622303, 2024-09-12T23:37:50.629 INFO:tasks.workunit.client.0.smithi136.stdout: "host_reads": 4349473727, 2024-09-12T23:37:50.629 INFO:tasks.workunit.client.0.smithi136.stdout: "host_writes": 24841062282, 2024-09-12T23:37:50.629 INFO:tasks.workunit.client.0.smithi136.stdout: "media_errors": 0, 2024-09-12T23:37:50.629 INFO:tasks.workunit.client.0.smithi136.stdout: "num_err_log_entries": 0, 2024-09-12T23:37:50.629 INFO:tasks.workunit.client.0.smithi136.stdout: "percentage_used": 25, 2024-09-12T23:37:50.629 INFO:tasks.workunit.client.0.smithi136.stdout: "power_cycles": 45284, 2024-09-12T23:37:50.630 INFO:tasks.workunit.client.0.smithi136.stdout: "power_on_hours": 38404, 2024-09-12T23:37:50.630 INFO:tasks.workunit.client.0.smithi136.stdout: "temperature": 23, 2024-09-12T23:37:50.630 INFO:tasks.workunit.client.0.smithi136.stdout: "unsafe_shutdowns": 45156 2024-09-12T23:37:50.630 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.630 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_vendor": "intel", 2024-09-12T23:37:50.630 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme_version": { 2024-09-12T23:37:50.630 INFO:tasks.workunit.client.0.smithi136.stdout: "string": "<1.2", 2024-09-12T23:37:50.630 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.630 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.630 INFO:tasks.workunit.client.0.smithi136.stdout: "power_cycle_count": 45284, 2024-09-12T23:37:50.630 INFO:tasks.workunit.client.0.smithi136.stdout: "power_on_time": { 2024-09-12T23:37:50.631 INFO:tasks.workunit.client.0.smithi136.stdout: "hours": 38404 2024-09-12T23:37:50.631 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.631 INFO:tasks.workunit.client.0.smithi136.stdout: "serial_number": "deleted", 2024-09-12T23:37:50.631 INFO:tasks.workunit.client.0.smithi136.stdout: "smart_status": { 2024-09-12T23:37:50.631 INFO:tasks.workunit.client.0.smithi136.stdout: "nvme": { 2024-09-12T23:37:50.631 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.631 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.631 INFO:tasks.workunit.client.0.smithi136.stdout: "passed": true 2024-09-12T23:37:50.631 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.631 INFO:tasks.workunit.client.0.smithi136.stdout: "smartctl": { 2024-09-12T23:37:50.631 INFO:tasks.workunit.client.0.smithi136.stdout: "argv": [ 2024-09-12T23:37:50.631 INFO:tasks.workunit.client.0.smithi136.stdout: "smartctl", 2024-09-12T23:37:50.632 INFO:tasks.workunit.client.0.smithi136.stdout: "-x", 2024-09-12T23:37:50.632 INFO:tasks.workunit.client.0.smithi136.stdout: "--json=o", 2024-09-12T23:37:50.632 INFO:tasks.workunit.client.0.smithi136.stdout: "/dev/nvme0n1" 2024-09-12T23:37:50.632 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.632 INFO:tasks.workunit.client.0.smithi136.stdout: "build_info": "(local build)", 2024-09-12T23:37:50.632 INFO:tasks.workunit.client.0.smithi136.stdout: "exit_status": 0, 2024-09-12T23:37:50.632 INFO:tasks.workunit.client.0.smithi136.stdout: "output": [ 2024-09-12T23:37:50.632 INFO:tasks.workunit.client.0.smithi136.stdout: "smartctl 7.2 2020-12-30 r5155 [x86_64-linux-5.15.0-121-generic] (local build)", 2024-09-12T23:37:50.632 INFO:tasks.workunit.client.0.smithi136.stdout: "Copyright (C) 2002-20, Bruce Allen, Christian Franke, www.smartmontools.org", 2024-09-12T23:37:50.632 INFO:tasks.workunit.client.0.smithi136.stdout: "", 2024-09-12T23:37:50.633 INFO:tasks.workunit.client.0.smithi136.stdout: "=== START OF INFORMATION SECTION ===", 2024-09-12T23:37:50.633 INFO:tasks.workunit.client.0.smithi136.stdout: "Model Number: INTEL SSDPEDMX020T4", 2024-09-12T23:37:50.633 INFO:tasks.workunit.client.0.smithi136.stdout: "Serial Number: deleted", 2024-09-12T23:37:50.633 INFO:tasks.workunit.client.0.smithi136.stdout: "Firmware Version: 8DV10171", 2024-09-12T23:37:50.633 INFO:tasks.workunit.client.0.smithi136.stdout: "PCI Vendor/Subsystem ID: 0x8086", 2024-09-12T23:37:50.633 INFO:tasks.workunit.client.0.smithi136.stdout: "IEEE OUI Identifier: 0x5cd2e4", 2024-09-12T23:37:50.633 INFO:tasks.workunit.client.0.smithi136.stdout: "Controller ID: 0", 2024-09-12T23:37:50.633 INFO:tasks.workunit.client.0.smithi136.stdout: "NVMe Version: <1.2", 2024-09-12T23:37:50.633 INFO:tasks.workunit.client.0.smithi136.stdout: "Number of Namespaces: 1", 2024-09-12T23:37:50.633 INFO:tasks.workunit.client.0.smithi136.stdout: "Namespace 1 Size/Capacity: 2,000,398,934,016 [2.00 TB]", 2024-09-12T23:37:50.633 INFO:tasks.workunit.client.0.smithi136.stdout: "Namespace 1 Formatted LBA Size: 512", 2024-09-12T23:37:50.633 INFO:tasks.workunit.client.0.smithi136.stdout: "Local Time is: Thu Sep 12 23:28:36 2024 UTC", 2024-09-12T23:37:50.634 INFO:tasks.workunit.client.0.smithi136.stdout: "Firmware Updates (0x02): 1 Slot", 2024-09-12T23:37:50.634 INFO:tasks.workunit.client.0.smithi136.stdout: "Optional Admin Commands (0x0006): Format Frmw_DL", 2024-09-12T23:37:50.634 INFO:tasks.workunit.client.0.smithi136.stdout: "Optional NVM Commands (0x0006): Wr_Unc DS_Mngmt", 2024-09-12T23:37:50.634 INFO:tasks.workunit.client.0.smithi136.stdout: "Log Page Attributes (0x02): Cmd_Eff_Lg", 2024-09-12T23:37:50.634 INFO:tasks.workunit.client.0.smithi136.stdout: "Maximum Data Transfer Size: 32 Pages", 2024-09-12T23:37:50.634 INFO:tasks.workunit.client.0.smithi136.stdout: "", 2024-09-12T23:37:50.634 INFO:tasks.workunit.client.0.smithi136.stdout: "Supported Power States", 2024-09-12T23:37:50.634 INFO:tasks.workunit.client.0.smithi136.stdout: "St Op Max Active Idle RL RT WL WT Ent_Lat Ex_Lat", 2024-09-12T23:37:50.634 INFO:tasks.workunit.client.0.smithi136.stdout: " 0 + 25.00W - - 0 0 0 0 0 0", 2024-09-12T23:37:50.634 INFO:tasks.workunit.client.0.smithi136.stdout: "", 2024-09-12T23:37:50.634 INFO:tasks.workunit.client.0.smithi136.stdout: "Supported LBA Sizes (NSID 0x1)", 2024-09-12T23:37:50.634 INFO:tasks.workunit.client.0.smithi136.stdout: "Id Fmt Data Metadt Rel_Perf", 2024-09-12T23:37:50.635 INFO:tasks.workunit.client.0.smithi136.stdout: " 0 + 512 0 2", 2024-09-12T23:37:50.635 INFO:tasks.workunit.client.0.smithi136.stdout: " 1 - 512 8 2", 2024-09-12T23:37:50.635 INFO:tasks.workunit.client.0.smithi136.stdout: " 2 - 512 16 2", 2024-09-12T23:37:50.635 INFO:tasks.workunit.client.0.smithi136.stdout: " 3 - 4096 0 0", 2024-09-12T23:37:50.635 INFO:tasks.workunit.client.0.smithi136.stdout: " 4 - 4096 8 0", 2024-09-12T23:37:50.635 INFO:tasks.workunit.client.0.smithi136.stdout: " 5 - 4096 64 0", 2024-09-12T23:37:50.635 INFO:tasks.workunit.client.0.smithi136.stdout: " 6 - 4096 128 0", 2024-09-12T23:37:50.635 INFO:tasks.workunit.client.0.smithi136.stdout: "", 2024-09-12T23:37:50.635 INFO:tasks.workunit.client.0.smithi136.stdout: "=== START OF SMART DATA SECTION ===", 2024-09-12T23:37:50.635 INFO:tasks.workunit.client.0.smithi136.stdout: "SMART overall-health self-assessment test result: PASSED", 2024-09-12T23:37:50.635 INFO:tasks.workunit.client.0.smithi136.stdout: "", 2024-09-12T23:37:50.635 INFO:tasks.workunit.client.0.smithi136.stdout: "SMART/Health Information (NVMe Log 0x02)", 2024-09-12T23:37:50.636 INFO:tasks.workunit.client.0.smithi136.stdout: "Critical Warning: 0x00", 2024-09-12T23:37:50.636 INFO:tasks.workunit.client.0.smithi136.stdout: "Temperature: 23 Celsius", 2024-09-12T23:37:50.636 INFO:tasks.workunit.client.0.smithi136.stdout: "Available Spare: 100%", 2024-09-12T23:37:50.636 INFO:tasks.workunit.client.0.smithi136.stdout: "Available Spare Threshold: 10%", 2024-09-12T23:37:50.636 INFO:tasks.workunit.client.0.smithi136.stdout: "Percentage Used: 25%", 2024-09-12T23:37:50.636 INFO:tasks.workunit.client.0.smithi136.stdout: "Data Units Read: 215,101,386 [110 TB]", 2024-09-12T23:37:50.636 INFO:tasks.workunit.client.0.smithi136.stdout: "Data Units Written: 1,248,622,303 [639 TB]", 2024-09-12T23:37:50.636 INFO:tasks.workunit.client.0.smithi136.stdout: "Host Read Commands: 4,349,473,727", 2024-09-12T23:37:50.636 INFO:tasks.workunit.client.0.smithi136.stdout: "Host Write Commands: 24,841,062,282", 2024-09-12T23:37:50.636 INFO:tasks.workunit.client.0.smithi136.stdout: "Controller Busy Time: 358", 2024-09-12T23:37:50.636 INFO:tasks.workunit.client.0.smithi136.stdout: "Power Cycles: 45,284", 2024-09-12T23:37:50.637 INFO:tasks.workunit.client.0.smithi136.stdout: "Power On Hours: 38,404", 2024-09-12T23:37:50.637 INFO:tasks.workunit.client.0.smithi136.stdout: "Unsafe Shutdowns: 45,156", 2024-09-12T23:37:50.637 INFO:tasks.workunit.client.0.smithi136.stdout: "Media and Data Integrity Errors: 0", 2024-09-12T23:37:50.637 INFO:tasks.workunit.client.0.smithi136.stdout: "Error Information Log Entries: 0", 2024-09-12T23:37:50.637 INFO:tasks.workunit.client.0.smithi136.stdout: "", 2024-09-12T23:37:50.637 INFO:tasks.workunit.client.0.smithi136.stdout: "Error Information (NVMe Log 0x01, 16 of 64 entries)", 2024-09-12T23:37:50.637 INFO:tasks.workunit.client.0.smithi136.stdout: "No Errors Logged", 2024-09-12T23:37:50.637 INFO:tasks.workunit.client.0.smithi136.stdout: "" 2024-09-12T23:37:50.637 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.637 INFO:tasks.workunit.client.0.smithi136.stdout: "platform_info": "x86_64-linux-5.15.0-121-generic", 2024-09-12T23:37:50.637 INFO:tasks.workunit.client.0.smithi136.stdout: "svn_revision": "5155", 2024-09-12T23:37:50.638 INFO:tasks.workunit.client.0.smithi136.stdout: "version": [ 2024-09-12T23:37:50.638 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:50.638 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:50.638 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:50.638 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.638 INFO:tasks.workunit.client.0.smithi136.stdout: "temperature": { 2024-09-12T23:37:50.638 INFO:tasks.workunit.client.0.smithi136.stdout: "current": 23 2024-09-12T23:37:50.638 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.638 INFO:tasks.workunit.client.0.smithi136.stdout: "user_capacity": { 2024-09-12T23:37:50.638 INFO:tasks.workunit.client.0.smithi136.stdout: "blocks": 3907029168, 2024-09-12T23:37:50.639 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 2000398934016 2024-09-12T23:37:50.639 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.639 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.639 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.639 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.639 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.639 INFO:tasks.workunit.client.0.smithi136.stdout: "report": { 2024-09-12T23:37:50.639 INFO:tasks.workunit.client.0.smithi136.stdout: "balancer": { 2024-09-12T23:37:50.639 INFO:tasks.workunit.client.0.smithi136.stdout: "active": true, 2024-09-12T23:37:50.639 INFO:tasks.workunit.client.0.smithi136.stdout: "mode": "" 2024-09-12T23:37:50.639 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.640 INFO:tasks.workunit.client.0.smithi136.stdout: "channels": [ 2024-09-12T23:37:50.640 INFO:tasks.workunit.client.0.smithi136.stdout: "basic", 2024-09-12T23:37:50.640 INFO:tasks.workunit.client.0.smithi136.stdout: "crash", 2024-09-12T23:37:50.640 INFO:tasks.workunit.client.0.smithi136.stdout: "device", 2024-09-12T23:37:50.640 INFO:tasks.workunit.client.0.smithi136.stdout: "ident", 2024-09-12T23:37:50.640 INFO:tasks.workunit.client.0.smithi136.stdout: "perf" 2024-09-12T23:37:50.640 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.640 INFO:tasks.workunit.client.0.smithi136.stdout: "channels_available": [ 2024-09-12T23:37:50.640 INFO:tasks.workunit.client.0.smithi136.stdout: "basic", 2024-09-12T23:37:50.640 INFO:tasks.workunit.client.0.smithi136.stdout: "ident", 2024-09-12T23:37:50.640 INFO:tasks.workunit.client.0.smithi136.stdout: "crash", 2024-09-12T23:37:50.640 INFO:tasks.workunit.client.0.smithi136.stdout: "device", 2024-09-12T23:37:50.641 INFO:tasks.workunit.client.0.smithi136.stdout: "perf" 2024-09-12T23:37:50.641 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.641 INFO:tasks.workunit.client.0.smithi136.stdout: "collections_available": [ 2024-09-12T23:37:50.641 INFO:tasks.workunit.client.0.smithi136.stdout: "basic_base", 2024-09-12T23:37:50.641 INFO:tasks.workunit.client.0.smithi136.stdout: "device_base", 2024-09-12T23:37:50.641 INFO:tasks.workunit.client.0.smithi136.stdout: "crash_base", 2024-09-12T23:37:50.641 INFO:tasks.workunit.client.0.smithi136.stdout: "ident_base", 2024-09-12T23:37:50.641 INFO:tasks.workunit.client.0.smithi136.stdout: "perf_perf", 2024-09-12T23:37:50.641 INFO:tasks.workunit.client.0.smithi136.stdout: "basic_mds_metadata", 2024-09-12T23:37:50.641 INFO:tasks.workunit.client.0.smithi136.stdout: "basic_pool_usage", 2024-09-12T23:37:50.641 INFO:tasks.workunit.client.0.smithi136.stdout: "basic_usage_by_class", 2024-09-12T23:37:50.642 INFO:tasks.workunit.client.0.smithi136.stdout: "basic_rook_v01", 2024-09-12T23:37:50.642 INFO:tasks.workunit.client.0.smithi136.stdout: "perf_memory_metrics", 2024-09-12T23:37:50.642 INFO:tasks.workunit.client.0.smithi136.stdout: "basic_pool_options_bluestore" 2024-09-12T23:37:50.642 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.642 INFO:tasks.workunit.client.0.smithi136.stdout: "collections_opted_in": [ 2024-09-12T23:37:50.642 INFO:tasks.workunit.client.0.smithi136.stdout: "basic_base", 2024-09-12T23:37:50.642 INFO:tasks.workunit.client.0.smithi136.stdout: "device_base", 2024-09-12T23:37:50.642 INFO:tasks.workunit.client.0.smithi136.stdout: "crash_base", 2024-09-12T23:37:50.642 INFO:tasks.workunit.client.0.smithi136.stdout: "ident_base", 2024-09-12T23:37:50.642 INFO:tasks.workunit.client.0.smithi136.stdout: "perf_perf", 2024-09-12T23:37:50.642 INFO:tasks.workunit.client.0.smithi136.stdout: "basic_mds_metadata", 2024-09-12T23:37:50.642 INFO:tasks.workunit.client.0.smithi136.stdout: "basic_pool_usage", 2024-09-12T23:37:50.643 INFO:tasks.workunit.client.0.smithi136.stdout: "basic_usage_by_class", 2024-09-12T23:37:50.643 INFO:tasks.workunit.client.0.smithi136.stdout: "basic_rook_v01", 2024-09-12T23:37:50.643 INFO:tasks.workunit.client.0.smithi136.stdout: "perf_memory_metrics", 2024-09-12T23:37:50.643 INFO:tasks.workunit.client.0.smithi136.stdout: "basic_pool_options_bluestore" 2024-09-12T23:37:50.643 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.643 INFO:tasks.workunit.client.0.smithi136.stdout: "config": { 2024-09-12T23:37:50.643 INFO:tasks.workunit.client.0.smithi136.stdout: "active_changed": [ 2024-09-12T23:37:50.643 INFO:tasks.workunit.client.0.smithi136.stdout: "auth_allow_insecure_global_id_reclaim", 2024-09-12T23:37:50.643 INFO:tasks.workunit.client.0.smithi136.stdout: "auth_debug", 2024-09-12T23:37:50.643 INFO:tasks.workunit.client.0.smithi136.stdout: "auth_mon_ticket_ttl", 2024-09-12T23:37:50.643 INFO:tasks.workunit.client.0.smithi136.stdout: "auth_service_ticket_ttl", 2024-09-12T23:37:50.643 INFO:tasks.workunit.client.0.smithi136.stdout: "bdev_debug_aio", 2024-09-12T23:37:50.644 INFO:tasks.workunit.client.0.smithi136.stdout: "client_check_pool_perm", 2024-09-12T23:37:50.644 INFO:tasks.workunit.client.0.smithi136.stdout: "client_mount_gid", 2024-09-12T23:37:50.644 INFO:tasks.workunit.client.0.smithi136.stdout: "client_mount_uid", 2024-09-12T23:37:50.644 INFO:tasks.workunit.client.0.smithi136.stdout: "client_quota", 2024-09-12T23:37:50.644 INFO:tasks.workunit.client.0.smithi136.stdout: "container_image", 2024-09-12T23:37:50.644 INFO:tasks.workunit.client.0.smithi136.stdout: "daemonize", 2024-09-12T23:37:50.644 INFO:tasks.workunit.client.0.smithi136.stdout: "debug_asserts_on_shutdown", 2024-09-12T23:37:50.644 INFO:tasks.workunit.client.0.smithi136.stdout: "debug_mgr", 2024-09-12T23:37:50.644 INFO:tasks.workunit.client.0.smithi136.stdout: "debug_mon", 2024-09-12T23:37:50.644 INFO:tasks.workunit.client.0.smithi136.stdout: "debug_ms", 2024-09-12T23:37:50.644 INFO:tasks.workunit.client.0.smithi136.stdout: "debug_osd", 2024-09-12T23:37:50.645 INFO:tasks.workunit.client.0.smithi136.stdout: "debug_paxos", 2024-09-12T23:37:50.645 INFO:tasks.workunit.client.0.smithi136.stdout: "keyring", 2024-09-12T23:37:50.645 INFO:tasks.workunit.client.0.smithi136.stdout: "leveldb_block_size", 2024-09-12T23:37:50.645 INFO:tasks.workunit.client.0.smithi136.stdout: "leveldb_cache_size", 2024-09-12T23:37:50.645 INFO:tasks.workunit.client.0.smithi136.stdout: "leveldb_compression", 2024-09-12T23:37:50.645 INFO:tasks.workunit.client.0.smithi136.stdout: "leveldb_log", 2024-09-12T23:37:50.645 INFO:tasks.workunit.client.0.smithi136.stdout: "leveldb_write_buffer_size", 2024-09-12T23:37:50.645 INFO:tasks.workunit.client.0.smithi136.stdout: "log_stderr_prefix", 2024-09-12T23:37:50.645 INFO:tasks.workunit.client.0.smithi136.stdout: "log_to_file", 2024-09-12T23:37:50.645 INFO:tasks.workunit.client.0.smithi136.stdout: "log_to_journald", 2024-09-12T23:37:50.645 INFO:tasks.workunit.client.0.smithi136.stdout: "log_to_stderr", 2024-09-12T23:37:50.646 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_allow_pool_delete", 2024-09-12T23:37:50.646 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_clock_drift_allowed", 2024-09-12T23:37:50.646 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_cluster_log_to_file", 2024-09-12T23:37:50.646 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_cluster_log_to_stderr", 2024-09-12T23:37:50.646 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_data_avail_warn", 2024-09-12T23:37:50.646 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_election_default_strategy", 2024-09-12T23:37:50.646 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_host", 2024-09-12T23:37:50.646 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_max_pg_per_osd", 2024-09-12T23:37:50.646 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_mgr_mkfs_grace", 2024-09-12T23:37:50.646 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_osd_prime_pg_temp", 2024-09-12T23:37:50.646 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_osd_reporter_subtree_level", 2024-09-12T23:37:50.646 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_pg_warn_max_object_skew", 2024-09-12T23:37:50.647 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_reweight_min_bytes_per_osd", 2024-09-12T23:37:50.647 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_reweight_min_pgs_per_osd", 2024-09-12T23:37:50.647 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_crush_straw_calc_version_zero", 2024-09-12T23:37:50.647 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_insecure_global_id_reclaim", 2024-09-12T23:37:50.647 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed", 2024-09-12T23:37:50.647 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_legacy_crush_tunables", 2024-09-12T23:37:50.647 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_osd_down_out_interval_zero", 2024-09-12T23:37:50.647 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_pool_pg_num_not_power_of_two", 2024-09-12T23:37:50.647 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_too_few_osds", 2024-09-12T23:37:50.647 INFO:tasks.workunit.client.0.smithi136.stdout: "ms_die_on_bug", 2024-09-12T23:37:50.647 INFO:tasks.workunit.client.0.smithi136.stdout: "ms_die_on_old_message", 2024-09-12T23:37:50.647 INFO:tasks.workunit.client.0.smithi136.stdout: "no_config_file", 2024-09-12T23:37:50.648 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_class_default_list", 2024-09-12T23:37:50.648 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_class_load_list", 2024-09-12T23:37:50.648 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_misdirected_ops", 2024-09-12T23:37:50.648 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_op_order", 2024-09-12T23:37:50.648 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_pg_log_writeout", 2024-09-12T23:37:50.648 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_shutdown", 2024-09-12T23:37:50.648 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_verify_cached_snaps", 2024-09-12T23:37:50.648 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_verify_missing_on_start", 2024-09-12T23:37:50.648 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_verify_stray_on_activate", 2024-09-12T23:37:50.648 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_deep_scrub_update_digest_min_age", 2024-09-12T23:37:50.648 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mclock_profile", 2024-09-12T23:37:50.649 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mclock_scheduler_background_best_effort_lim", 2024-09-12T23:37:50.649 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mclock_scheduler_background_best_effort_res", 2024-09-12T23:37:50.649 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mclock_scheduler_background_best_effort_wgt", 2024-09-12T23:37:50.649 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mclock_scheduler_background_recovery_lim", 2024-09-12T23:37:50.649 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mclock_scheduler_background_recovery_res", 2024-09-12T23:37:50.649 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mclock_scheduler_background_recovery_wgt", 2024-09-12T23:37:50.649 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mclock_scheduler_client_lim", 2024-09-12T23:37:50.649 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mclock_scheduler_client_res", 2024-09-12T23:37:50.649 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mclock_scheduler_client_wgt", 2024-09-12T23:37:50.649 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_memory_target", 2024-09-12T23:37:50.649 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_memory_target_autotune", 2024-09-12T23:37:50.650 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_op_queue", 2024-09-12T23:37:50.650 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_op_queue_cut_off", 2024-09-12T23:37:50.650 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_pool_default_erasure_code_profile", 2024-09-12T23:37:50.650 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_pool_default_pg_autoscale_mode", 2024-09-12T23:37:50.650 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_recover_clone_overlap", 2024-09-12T23:37:50.650 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_recovery_max_chunk", 2024-09-12T23:37:50.650 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_scrub_load_threshold", 2024-09-12T23:37:50.650 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_scrub_max_interval", 2024-09-12T23:37:50.650 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_shutdown_pgref_assert", 2024-09-12T23:37:50.650 INFO:tasks.workunit.client.0.smithi136.stdout: "public_addr", 2024-09-12T23:37:50.650 INFO:tasks.workunit.client.0.smithi136.stdout: "public_addrv", 2024-09-12T23:37:50.650 INFO:tasks.workunit.client.0.smithi136.stdout: "public_network", 2024-09-12T23:37:50.651 INFO:tasks.workunit.client.0.smithi136.stdout: "rbd_default_features", 2024-09-12T23:37:50.651 INFO:tasks.workunit.client.0.smithi136.stdout: "rbd_qos_exclude_ops", 2024-09-12T23:37:50.651 INFO:tasks.workunit.client.0.smithi136.stdout: "setgroup", 2024-09-12T23:37:50.651 INFO:tasks.workunit.client.0.smithi136.stdout: "setuser" 2024-09-12T23:37:50.651 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.651 INFO:tasks.workunit.client.0.smithi136.stdout: "cluster_changed": [ 2024-09-12T23:37:50.651 INFO:tasks.workunit.client.0.smithi136.stdout: "auth_allow_insecure_global_id_reclaim", 2024-09-12T23:37:50.651 INFO:tasks.workunit.client.0.smithi136.stdout: "auth_debug", 2024-09-12T23:37:50.651 INFO:tasks.workunit.client.0.smithi136.stdout: "auth_mon_ticket_ttl", 2024-09-12T23:37:50.651 INFO:tasks.workunit.client.0.smithi136.stdout: "auth_service_ticket_ttl", 2024-09-12T23:37:50.651 INFO:tasks.workunit.client.0.smithi136.stdout: "bdev_debug_aio", 2024-09-12T23:37:50.651 INFO:tasks.workunit.client.0.smithi136.stdout: "container_image", 2024-09-12T23:37:50.652 INFO:tasks.workunit.client.0.smithi136.stdout: "debug_asserts_on_shutdown", 2024-09-12T23:37:50.652 INFO:tasks.workunit.client.0.smithi136.stdout: "debug_mgr", 2024-09-12T23:37:50.652 INFO:tasks.workunit.client.0.smithi136.stdout: "debug_mon", 2024-09-12T23:37:50.652 INFO:tasks.workunit.client.0.smithi136.stdout: "debug_ms", 2024-09-12T23:37:50.652 INFO:tasks.workunit.client.0.smithi136.stdout: "debug_osd", 2024-09-12T23:37:50.652 INFO:tasks.workunit.client.0.smithi136.stdout: "debug_paxos", 2024-09-12T23:37:50.652 INFO:tasks.workunit.client.0.smithi136.stdout: "log_to_file", 2024-09-12T23:37:50.652 INFO:tasks.workunit.client.0.smithi136.stdout: "log_to_journald", 2024-09-12T23:37:50.652 INFO:tasks.workunit.client.0.smithi136.stdout: "log_to_stderr", 2024-09-12T23:37:50.652 INFO:tasks.workunit.client.0.smithi136.stdout: "mds_join_fs", 2024-09-12T23:37:50.652 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/cephadm/allow_ptrace", 2024-09-12T23:37:50.653 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/cephadm/container_init", 2024-09-12T23:37:50.653 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/cephadm/migration_current", 2024-09-12T23:37:50.653 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/cephadm/use_repo_digest", 2024-09-12T23:37:50.653 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/dashboard/ALERTMANAGER_API_HOST", 2024-09-12T23:37:50.653 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/dashboard/GRAFANA_API_SSL_VERIFY", 2024-09-12T23:37:50.653 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/dashboard/GRAFANA_API_URL", 2024-09-12T23:37:50.653 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/dashboard/PROMETHEUS_API_HOST", 2024-09-12T23:37:50.653 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/dashboard/ssl_server_port", 2024-09-12T23:37:50.653 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/orchestrator/orchestrator", 2024-09-12T23:37:50.653 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/telemetry/channel_ident", 2024-09-12T23:37:50.653 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/telemetry/channel_perf", 2024-09-12T23:37:50.653 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/telemetry/description", 2024-09-12T23:37:50.654 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/telemetry/enabled", 2024-09-12T23:37:50.654 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr/telemetry/organization", 2024-09-12T23:37:50.654 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_allow_pool_delete", 2024-09-12T23:37:50.654 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_clock_drift_allowed", 2024-09-12T23:37:50.654 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_data_avail_warn", 2024-09-12T23:37:50.654 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_election_default_strategy", 2024-09-12T23:37:50.654 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_max_pg_per_osd", 2024-09-12T23:37:50.654 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_mgr_mkfs_grace", 2024-09-12T23:37:50.654 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_osd_prime_pg_temp", 2024-09-12T23:37:50.654 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_osd_reporter_subtree_level", 2024-09-12T23:37:50.654 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_pg_warn_max_object_skew", 2024-09-12T23:37:50.655 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_reweight_min_bytes_per_osd", 2024-09-12T23:37:50.655 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_reweight_min_pgs_per_osd", 2024-09-12T23:37:50.655 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_crush_straw_calc_version_zero", 2024-09-12T23:37:50.655 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_insecure_global_id_reclaim", 2024-09-12T23:37:50.655 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed", 2024-09-12T23:37:50.655 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_legacy_crush_tunables", 2024-09-12T23:37:50.655 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_osd_down_out_interval_zero", 2024-09-12T23:37:50.655 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_pool_pg_num_not_power_of_two", 2024-09-12T23:37:50.655 INFO:tasks.workunit.client.0.smithi136.stdout: "mon_warn_on_too_few_osds", 2024-09-12T23:37:50.655 INFO:tasks.workunit.client.0.smithi136.stdout: "ms_die_on_bug", 2024-09-12T23:37:50.655 INFO:tasks.workunit.client.0.smithi136.stdout: "ms_die_on_old_message", 2024-09-12T23:37:50.655 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_class_default_list", 2024-09-12T23:37:50.656 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_class_load_list", 2024-09-12T23:37:50.656 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_misdirected_ops", 2024-09-12T23:37:50.656 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_op_order", 2024-09-12T23:37:50.656 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_pg_log_writeout", 2024-09-12T23:37:50.656 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_shutdown", 2024-09-12T23:37:50.656 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_verify_cached_snaps", 2024-09-12T23:37:50.656 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_verify_missing_on_start", 2024-09-12T23:37:50.656 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_debug_verify_stray_on_activate", 2024-09-12T23:37:50.656 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_deep_scrub_update_digest_min_age", 2024-09-12T23:37:50.656 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mclock_profile", 2024-09-12T23:37:50.656 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_memory_target", 2024-09-12T23:37:50.657 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_memory_target_autotune", 2024-09-12T23:37:50.657 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_op_queue", 2024-09-12T23:37:50.657 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_op_queue_cut_off", 2024-09-12T23:37:50.657 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_pool_default_erasure_code_profile", 2024-09-12T23:37:50.657 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_pool_default_pg_autoscale_mode", 2024-09-12T23:37:50.657 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_recover_clone_overlap", 2024-09-12T23:37:50.657 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_recovery_max_chunk", 2024-09-12T23:37:50.657 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_scrub_load_threshold", 2024-09-12T23:37:50.657 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_scrub_max_interval", 2024-09-12T23:37:50.657 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_shutdown_pgref_assert", 2024-09-12T23:37:50.657 INFO:tasks.workunit.client.0.smithi136.stdout: "public_network", 2024-09-12T23:37:50.658 INFO:tasks.workunit.client.0.smithi136.stdout: "rgw_cache_enabled", 2024-09-12T23:37:50.658 INFO:tasks.workunit.client.0.smithi136.stdout: "rgw_enable_ops_log", 2024-09-12T23:37:50.658 INFO:tasks.workunit.client.0.smithi136.stdout: "rgw_enable_usage_log" 2024-09-12T23:37:50.658 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:50.658 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.658 INFO:tasks.workunit.client.0.smithi136.stdout: "contact": null, 2024-09-12T23:37:50.658 INFO:tasks.workunit.client.0.smithi136.stdout: "crashes": [], 2024-09-12T23:37:50.658 INFO:tasks.workunit.client.0.smithi136.stdout: "created": "2024-09-12T23:23:24.585690Z", 2024-09-12T23:37:50.658 INFO:tasks.workunit.client.0.smithi136.stdout: "crush": { 2024-09-12T23:37:50.658 INFO:tasks.workunit.client.0.smithi136.stdout: "bucket_algs": { 2024-09-12T23:37:50.658 INFO:tasks.workunit.client.0.smithi136.stdout: "straw2": 3 2024-09-12T23:37:50.659 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.659 INFO:tasks.workunit.client.0.smithi136.stdout: "bucket_sizes": { 2024-09-12T23:37:50.659 INFO:tasks.workunit.client.0.smithi136.stdout: "2": 1, 2024-09-12T23:37:50.659 INFO:tasks.workunit.client.0.smithi136.stdout: "4": 2 2024-09-12T23:37:50.659 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.659 INFO:tasks.workunit.client.0.smithi136.stdout: "bucket_types": { 2024-09-12T23:37:50.659 INFO:tasks.workunit.client.0.smithi136.stdout: "1": 2, 2024-09-12T23:37:50.659 INFO:tasks.workunit.client.0.smithi136.stdout: "11": 1 2024-09-12T23:37:50.659 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.659 INFO:tasks.workunit.client.0.smithi136.stdout: "compat_weight_set": false, 2024-09-12T23:37:50.659 INFO:tasks.workunit.client.0.smithi136.stdout: "device_classes": [ 2024-09-12T23:37:50.660 INFO:tasks.workunit.client.0.smithi136.stdout: 8 2024-09-12T23:37:50.660 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.660 INFO:tasks.workunit.client.0.smithi136.stdout: "num_buckets": 6, 2024-09-12T23:37:50.660 INFO:tasks.workunit.client.0.smithi136.stdout: "num_devices": 8, 2024-09-12T23:37:50.660 INFO:tasks.workunit.client.0.smithi136.stdout: "num_rules": 1, 2024-09-12T23:37:50.660 INFO:tasks.workunit.client.0.smithi136.stdout: "num_types": 12, 2024-09-12T23:37:50.660 INFO:tasks.workunit.client.0.smithi136.stdout: "num_weight_sets": 0, 2024-09-12T23:37:50.660 INFO:tasks.workunit.client.0.smithi136.stdout: "tunables": { 2024-09-12T23:37:50.660 INFO:tasks.workunit.client.0.smithi136.stdout: "allowed_bucket_algs": 54, 2024-09-12T23:37:50.660 INFO:tasks.workunit.client.0.smithi136.stdout: "choose_local_fallback_tries": 0, 2024-09-12T23:37:50.660 INFO:tasks.workunit.client.0.smithi136.stdout: "choose_local_tries": 0, 2024-09-12T23:37:50.661 INFO:tasks.workunit.client.0.smithi136.stdout: "choose_total_tries": 50, 2024-09-12T23:37:50.661 INFO:tasks.workunit.client.0.smithi136.stdout: "chooseleaf_descend_once": 1, 2024-09-12T23:37:50.661 INFO:tasks.workunit.client.0.smithi136.stdout: "chooseleaf_stable": 1, 2024-09-12T23:37:50.661 INFO:tasks.workunit.client.0.smithi136.stdout: "chooseleaf_vary_r": 1, 2024-09-12T23:37:50.661 INFO:tasks.workunit.client.0.smithi136.stdout: "has_v2_rules": 0, 2024-09-12T23:37:50.661 INFO:tasks.workunit.client.0.smithi136.stdout: "has_v3_rules": 0, 2024-09-12T23:37:50.661 INFO:tasks.workunit.client.0.smithi136.stdout: "has_v4_buckets": 1, 2024-09-12T23:37:50.661 INFO:tasks.workunit.client.0.smithi136.stdout: "has_v5_rules": 0, 2024-09-12T23:37:50.661 INFO:tasks.workunit.client.0.smithi136.stdout: "legacy_tunables": 0, 2024-09-12T23:37:50.661 INFO:tasks.workunit.client.0.smithi136.stdout: "minimum_required_version": "jewel", 2024-09-12T23:37:50.661 INFO:tasks.workunit.client.0.smithi136.stdout: "optimal_tunables": 1, 2024-09-12T23:37:50.661 INFO:tasks.workunit.client.0.smithi136.stdout: "profile": "jewel", 2024-09-12T23:37:50.662 INFO:tasks.workunit.client.0.smithi136.stdout: "require_feature_tunables": 1, 2024-09-12T23:37:50.662 INFO:tasks.workunit.client.0.smithi136.stdout: "require_feature_tunables2": 1, 2024-09-12T23:37:50.662 INFO:tasks.workunit.client.0.smithi136.stdout: "require_feature_tunables3": 1, 2024-09-12T23:37:50.662 INFO:tasks.workunit.client.0.smithi136.stdout: "require_feature_tunables5": 1, 2024-09-12T23:37:50.662 INFO:tasks.workunit.client.0.smithi136.stdout: "straw_calc_version": 1 2024-09-12T23:37:50.662 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.662 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.662 INFO:tasks.workunit.client.0.smithi136.stdout: "description": "upgrade test cluster", 2024-09-12T23:37:50.662 INFO:tasks.workunit.client.0.smithi136.stdout: "fs": { 2024-09-12T23:37:50.662 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1, 2024-09-12T23:37:50.662 INFO:tasks.workunit.client.0.smithi136.stdout: "feature_flags": { 2024-09-12T23:37:50.663 INFO:tasks.workunit.client.0.smithi136.stdout: "enable_multiple": true, 2024-09-12T23:37:50.663 INFO:tasks.workunit.client.0.smithi136.stdout: "ever_enabled_multiple": true 2024-09-12T23:37:50.663 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.663 INFO:tasks.workunit.client.0.smithi136.stdout: "filesystems": [ 2024-09-12T23:37:50.663 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.663 INFO:tasks.workunit.client.0.smithi136.stdout: "approx_ctime": "2024-09", 2024-09-12T23:37:50.663 INFO:tasks.workunit.client.0.smithi136.stdout: "balancer_enabled": false, 2024-09-12T23:37:50.663 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.663 INFO:tasks.workunit.client.0.smithi136.stdout: "cached_caps": 0, 2024-09-12T23:37:50.663 INFO:tasks.workunit.client.0.smithi136.stdout: "cached_dns": 10, 2024-09-12T23:37:50.664 INFO:tasks.workunit.client.0.smithi136.stdout: "cached_inos": 13, 2024-09-12T23:37:50.664 INFO:tasks.workunit.client.0.smithi136.stdout: "cached_subtrees": 2, 2024-09-12T23:37:50.664 INFO:tasks.workunit.client.0.smithi136.stdout: "ever_allowed_features": 0, 2024-09-12T23:37:50.664 INFO:tasks.workunit.client.0.smithi136.stdout: "explicitly_allowed_features": 0, 2024-09-12T23:37:50.664 INFO:tasks.workunit.client.0.smithi136.stdout: "files": 0, 2024-09-12T23:37:50.664 INFO:tasks.workunit.client.0.smithi136.stdout: "max_mds": 1, 2024-09-12T23:37:50.664 INFO:tasks.workunit.client.0.smithi136.stdout: "num_data_pools": 1, 2024-09-12T23:37:50.664 INFO:tasks.workunit.client.0.smithi136.stdout: "num_in": 1, 2024-09-12T23:37:50.664 INFO:tasks.workunit.client.0.smithi136.stdout: "num_mds": 1, 2024-09-12T23:37:50.664 INFO:tasks.workunit.client.0.smithi136.stdout: "num_sessions": 0, 2024-09-12T23:37:50.664 INFO:tasks.workunit.client.0.smithi136.stdout: "num_standby_replay": 0, 2024-09-12T23:37:50.664 INFO:tasks.workunit.client.0.smithi136.stdout: "num_up": 1, 2024-09-12T23:37:50.665 INFO:tasks.workunit.client.0.smithi136.stdout: "snaps": 0, 2024-09-12T23:37:50.665 INFO:tasks.workunit.client.0.smithi136.stdout: "standby_count_wanted": 0 2024-09-12T23:37:50.665 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.665 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.665 INFO:tasks.workunit.client.0.smithi136.stdout: "num_standby_mds": 1, 2024-09-12T23:37:50.665 INFO:tasks.workunit.client.0.smithi136.stdout: "total_num_mds": 2 2024-09-12T23:37:50.665 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.665 INFO:tasks.workunit.client.0.smithi136.stdout: "heap_stats": { 2024-09-12T23:37:50.665 INFO:tasks.workunit.client.0.smithi136.stdout: "mds": { 2024-09-12T23:37:50.665 INFO:tasks.workunit.client.0.smithi136.stdout: "mds.4585d541b331f263094839d9f58dd50a20d3f517": { 2024-09-12T23:37:50.665 INFO:tasks.workunit.client.0.smithi136.stdout: "actual_memory_used": 20627456, 2024-09-12T23:37:50.665 INFO:tasks.workunit.client.0.smithi136.stdout: "central_cache_freelist": 405976, 2024-09-12T23:37:50.666 INFO:tasks.workunit.client.0.smithi136.stdout: "malloc_metadata": 2621440, 2024-09-12T23:37:50.666 INFO:tasks.workunit.client.0.smithi136.stdout: "page_heap_freelist": 8192, 2024-09-12T23:37:50.666 INFO:tasks.workunit.client.0.smithi136.stdout: "released_to_os": 557056, 2024-09-12T23:37:50.666 INFO:tasks.workunit.client.0.smithi136.stdout: "spans_in_use": 430, 2024-09-12T23:37:50.666 INFO:tasks.workunit.client.0.smithi136.stdout: "tcmalloc_page_size": 8192, 2024-09-12T23:37:50.666 INFO:tasks.workunit.client.0.smithi136.stdout: "thread_cache_freelists": 1302840, 2024-09-12T23:37:50.666 INFO:tasks.workunit.client.0.smithi136.stdout: "thread_heaps_in_use": 20, 2024-09-12T23:37:50.666 INFO:tasks.workunit.client.0.smithi136.stdout: "transfer_cache_freelist": 1120768, 2024-09-12T23:37:50.666 INFO:tasks.workunit.client.0.smithi136.stdout: "use_by_application": 15168240, 2024-09-12T23:37:50.666 INFO:tasks.workunit.client.0.smithi136.stdout: "virtual_address_space_used": 21184512 2024-09-12T23:37:50.666 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.666 INFO:tasks.workunit.client.0.smithi136.stdout: "mds.48b2ee34c5040a1abbee529ca5897a6d3827a23a": { 2024-09-12T23:37:50.667 INFO:tasks.workunit.client.0.smithi136.stdout: "actual_memory_used": 20135936, 2024-09-12T23:37:50.667 INFO:tasks.workunit.client.0.smithi136.stdout: "central_cache_freelist": 372784, 2024-09-12T23:37:50.667 INFO:tasks.workunit.client.0.smithi136.stdout: "malloc_metadata": 2621440, 2024-09-12T23:37:50.667 INFO:tasks.workunit.client.0.smithi136.stdout: "page_heap_freelist": 794624, 2024-09-12T23:37:50.667 INFO:tasks.workunit.client.0.smithi136.stdout: "released_to_os": 0, 2024-09-12T23:37:50.667 INFO:tasks.workunit.client.0.smithi136.stdout: "spans_in_use": 352, 2024-09-12T23:37:50.667 INFO:tasks.workunit.client.0.smithi136.stdout: "tcmalloc_page_size": 8192, 2024-09-12T23:37:50.667 INFO:tasks.workunit.client.0.smithi136.stdout: "thread_cache_freelists": 946816, 2024-09-12T23:37:50.667 INFO:tasks.workunit.client.0.smithi136.stdout: "thread_heaps_in_use": 12, 2024-09-12T23:37:50.667 INFO:tasks.workunit.client.0.smithi136.stdout: "transfer_cache_freelist": 703488, 2024-09-12T23:37:50.667 INFO:tasks.workunit.client.0.smithi136.stdout: "use_by_application": 14696784, 2024-09-12T23:37:50.667 INFO:tasks.workunit.client.0.smithi136.stdout: "virtual_address_space_used": 20135936 2024-09-12T23:37:50.668 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.668 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.668 INFO:tasks.workunit.client.0.smithi136.stdout: "mon": { 2024-09-12T23:37:50.668 INFO:tasks.workunit.client.0.smithi136.stdout: "mon.0027af889fd200cedbdb75d916e7558e544c1ffa": { 2024-09-12T23:37:50.668 INFO:tasks.workunit.client.0.smithi136.stdout: "actual_memory_used": 39985152, 2024-09-12T23:37:50.668 INFO:tasks.workunit.client.0.smithi136.stdout: "central_cache_freelist": 1619496, 2024-09-12T23:37:50.668 INFO:tasks.workunit.client.0.smithi136.stdout: "malloc_metadata": 2621440, 2024-09-12T23:37:50.668 INFO:tasks.workunit.client.0.smithi136.stdout: "page_heap_freelist": 1712128, 2024-09-12T23:37:50.668 INFO:tasks.workunit.client.0.smithi136.stdout: "released_to_os": 3325952, 2024-09-12T23:37:50.668 INFO:tasks.workunit.client.0.smithi136.stdout: "spans_in_use": 1220, 2024-09-12T23:37:50.668 INFO:tasks.workunit.client.0.smithi136.stdout: "tcmalloc_page_size": 8192, 2024-09-12T23:37:50.669 INFO:tasks.workunit.client.0.smithi136.stdout: "thread_cache_freelists": 7966104, 2024-09-12T23:37:50.669 INFO:tasks.workunit.client.0.smithi136.stdout: "thread_heaps_in_use": 21, 2024-09-12T23:37:50.669 INFO:tasks.workunit.client.0.smithi136.stdout: "transfer_cache_freelist": 1375232, 2024-09-12T23:37:50.669 INFO:tasks.workunit.client.0.smithi136.stdout: "use_by_application": 24690752, 2024-09-12T23:37:50.669 INFO:tasks.workunit.client.0.smithi136.stdout: "virtual_address_space_used": 43311104 2024-09-12T23:37:50.669 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.669 INFO:tasks.workunit.client.0.smithi136.stdout: "mon.4585d541b331f263094839d9f58dd50a20d3f517": { 2024-09-12T23:37:50.669 INFO:tasks.workunit.client.0.smithi136.stdout: "actual_memory_used": 38576128, 2024-09-12T23:37:50.669 INFO:tasks.workunit.client.0.smithi136.stdout: "central_cache_freelist": 1194632, 2024-09-12T23:37:50.669 INFO:tasks.workunit.client.0.smithi136.stdout: "malloc_metadata": 2621440, 2024-09-12T23:37:50.669 INFO:tasks.workunit.client.0.smithi136.stdout: "page_heap_freelist": 0, 2024-09-12T23:37:50.669 INFO:tasks.workunit.client.0.smithi136.stdout: "released_to_os": 5791744, 2024-09-12T23:37:50.670 INFO:tasks.workunit.client.0.smithi136.stdout: "spans_in_use": 1204, 2024-09-12T23:37:50.670 INFO:tasks.workunit.client.0.smithi136.stdout: "tcmalloc_page_size": 8192, 2024-09-12T23:37:50.670 INFO:tasks.workunit.client.0.smithi136.stdout: "thread_cache_freelists": 8453688, 2024-09-12T23:37:50.670 INFO:tasks.workunit.client.0.smithi136.stdout: "thread_heaps_in_use": 21, 2024-09-12T23:37:50.670 INFO:tasks.workunit.client.0.smithi136.stdout: "transfer_cache_freelist": 1777664, 2024-09-12T23:37:50.670 INFO:tasks.workunit.client.0.smithi136.stdout: "use_by_application": 24528704, 2024-09-12T23:37:50.670 INFO:tasks.workunit.client.0.smithi136.stdout: "virtual_address_space_used": 44367872 2024-09-12T23:37:50.670 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.670 INFO:tasks.workunit.client.0.smithi136.stdout: "mon.48b2ee34c5040a1abbee529ca5897a6d3827a23a": { 2024-09-12T23:37:50.670 INFO:tasks.workunit.client.0.smithi136.stdout: "actual_memory_used": 47964160, 2024-09-12T23:37:50.670 INFO:tasks.workunit.client.0.smithi136.stdout: "central_cache_freelist": 1841600, 2024-09-12T23:37:50.671 INFO:tasks.workunit.client.0.smithi136.stdout: "malloc_metadata": 2621440, 2024-09-12T23:37:50.671 INFO:tasks.workunit.client.0.smithi136.stdout: "page_heap_freelist": 778240, 2024-09-12T23:37:50.671 INFO:tasks.workunit.client.0.smithi136.stdout: "released_to_os": 2605056, 2024-09-12T23:37:50.671 INFO:tasks.workunit.client.0.smithi136.stdout: "spans_in_use": 1755, 2024-09-12T23:37:50.671 INFO:tasks.workunit.client.0.smithi136.stdout: "tcmalloc_page_size": 8192, 2024-09-12T23:37:50.671 INFO:tasks.workunit.client.0.smithi136.stdout: "thread_cache_freelists": 10211576, 2024-09-12T23:37:50.671 INFO:tasks.workunit.client.0.smithi136.stdout: "thread_heaps_in_use": 23, 2024-09-12T23:37:50.671 INFO:tasks.workunit.client.0.smithi136.stdout: "transfer_cache_freelist": 3070464, 2024-09-12T23:37:50.671 INFO:tasks.workunit.client.0.smithi136.stdout: "use_by_application": 29440840, 2024-09-12T23:37:50.671 INFO:tasks.workunit.client.0.smithi136.stdout: "virtual_address_space_used": 50569216 2024-09-12T23:37:50.671 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.671 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.672 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:50.672 INFO:tasks.workunit.client.0.smithi136.stdout: "osd.0": { 2024-09-12T23:37:50.672 INFO:tasks.workunit.client.0.smithi136.stdout: "actual_memory_used": 277266432, 2024-09-12T23:37:50.672 INFO:tasks.workunit.client.0.smithi136.stdout: "central_cache_freelist": 702624, 2024-09-12T23:37:50.672 INFO:tasks.workunit.client.0.smithi136.stdout: "malloc_metadata": 4849664, 2024-09-12T23:37:50.672 INFO:tasks.workunit.client.0.smithi136.stdout: "page_heap_freelist": 8192, 2024-09-12T23:37:50.672 INFO:tasks.workunit.client.0.smithi136.stdout: "released_to_os": 655360, 2024-09-12T23:37:50.672 INFO:tasks.workunit.client.0.smithi136.stdout: "spans_in_use": 1520, 2024-09-12T23:37:50.672 INFO:tasks.workunit.client.0.smithi136.stdout: "tcmalloc_page_size": 8192, 2024-09-12T23:37:50.672 INFO:tasks.workunit.client.0.smithi136.stdout: "thread_cache_freelists": 5198872, 2024-09-12T23:37:50.672 INFO:tasks.workunit.client.0.smithi136.stdout: "thread_heaps_in_use": 45, 2024-09-12T23:37:50.672 INFO:tasks.workunit.client.0.smithi136.stdout: "transfer_cache_freelist": 3229696, 2024-09-12T23:37:50.673 INFO:tasks.workunit.client.0.smithi136.stdout: "use_by_application": 263277384, 2024-09-12T23:37:50.673 INFO:tasks.workunit.client.0.smithi136.stdout: "virtual_address_space_used": 277921792 2024-09-12T23:37:50.673 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.673 INFO:tasks.workunit.client.0.smithi136.stdout: "osd.1": { 2024-09-12T23:37:50.673 INFO:tasks.workunit.client.0.smithi136.stdout: "actual_memory_used": 274178048, 2024-09-12T23:37:50.673 INFO:tasks.workunit.client.0.smithi136.stdout: "central_cache_freelist": 715664, 2024-09-12T23:37:50.673 INFO:tasks.workunit.client.0.smithi136.stdout: "malloc_metadata": 2752512, 2024-09-12T23:37:50.673 INFO:tasks.workunit.client.0.smithi136.stdout: "page_heap_freelist": 0, 2024-09-12T23:37:50.673 INFO:tasks.workunit.client.0.smithi136.stdout: "released_to_os": 598016, 2024-09-12T23:37:50.673 INFO:tasks.workunit.client.0.smithi136.stdout: "spans_in_use": 1455, 2024-09-12T23:37:50.673 INFO:tasks.workunit.client.0.smithi136.stdout: "tcmalloc_page_size": 8192, 2024-09-12T23:37:50.674 INFO:tasks.workunit.client.0.smithi136.stdout: "thread_cache_freelists": 5658184, 2024-09-12T23:37:50.674 INFO:tasks.workunit.client.0.smithi136.stdout: "thread_heaps_in_use": 47, 2024-09-12T23:37:50.674 INFO:tasks.workunit.client.0.smithi136.stdout: "transfer_cache_freelist": 2204416, 2024-09-12T23:37:50.674 INFO:tasks.workunit.client.0.smithi136.stdout: "use_by_application": 262847272, 2024-09-12T23:37:50.674 INFO:tasks.workunit.client.0.smithi136.stdout: "virtual_address_space_used": 274776064 2024-09-12T23:37:50.674 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.674 INFO:tasks.workunit.client.0.smithi136.stdout: "osd.2": { 2024-09-12T23:37:50.674 INFO:tasks.workunit.client.0.smithi136.stdout: "actual_memory_used": 274857984, 2024-09-12T23:37:50.674 INFO:tasks.workunit.client.0.smithi136.stdout: "central_cache_freelist": 779136, 2024-09-12T23:37:50.674 INFO:tasks.workunit.client.0.smithi136.stdout: "malloc_metadata": 4849664, 2024-09-12T23:37:50.674 INFO:tasks.workunit.client.0.smithi136.stdout: "page_heap_freelist": 0, 2024-09-12T23:37:50.675 INFO:tasks.workunit.client.0.smithi136.stdout: "released_to_os": 901120, 2024-09-12T23:37:50.675 INFO:tasks.workunit.client.0.smithi136.stdout: "spans_in_use": 1370, 2024-09-12T23:37:50.675 INFO:tasks.workunit.client.0.smithi136.stdout: "tcmalloc_page_size": 8192, 2024-09-12T23:37:50.675 INFO:tasks.workunit.client.0.smithi136.stdout: "thread_cache_freelists": 5042920, 2024-09-12T23:37:50.675 INFO:tasks.workunit.client.0.smithi136.stdout: "thread_heaps_in_use": 46, 2024-09-12T23:37:50.675 INFO:tasks.workunit.client.0.smithi136.stdout: "transfer_cache_freelist": 2475776, 2024-09-12T23:37:50.675 INFO:tasks.workunit.client.0.smithi136.stdout: "use_by_application": 261710488, 2024-09-12T23:37:50.675 INFO:tasks.workunit.client.0.smithi136.stdout: "virtual_address_space_used": 275759104 2024-09-12T23:37:50.675 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.675 INFO:tasks.workunit.client.0.smithi136.stdout: "osd.3": { 2024-09-12T23:37:50.675 INFO:tasks.workunit.client.0.smithi136.stdout: "actual_memory_used": 272171008, 2024-09-12T23:37:50.675 INFO:tasks.workunit.client.0.smithi136.stdout: "central_cache_freelist": 646656, 2024-09-12T23:37:50.676 INFO:tasks.workunit.client.0.smithi136.stdout: "malloc_metadata": 2752512, 2024-09-12T23:37:50.676 INFO:tasks.workunit.client.0.smithi136.stdout: "page_heap_freelist": 0, 2024-09-12T23:37:50.676 INFO:tasks.workunit.client.0.smithi136.stdout: "released_to_os": 442368, 2024-09-12T23:37:50.676 INFO:tasks.workunit.client.0.smithi136.stdout: "spans_in_use": 1319, 2024-09-12T23:37:50.676 INFO:tasks.workunit.client.0.smithi136.stdout: "tcmalloc_page_size": 8192, 2024-09-12T23:37:50.676 INFO:tasks.workunit.client.0.smithi136.stdout: "thread_cache_freelists": 4526896, 2024-09-12T23:37:50.676 INFO:tasks.workunit.client.0.smithi136.stdout: "thread_heaps_in_use": 45, 2024-09-12T23:37:50.676 INFO:tasks.workunit.client.0.smithi136.stdout: "transfer_cache_freelist": 1983744, 2024-09-12T23:37:50.676 INFO:tasks.workunit.client.0.smithi136.stdout: "use_by_application": 262261200, 2024-09-12T23:37:50.676 INFO:tasks.workunit.client.0.smithi136.stdout: "virtual_address_space_used": 272613376 2024-09-12T23:37:50.676 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.676 INFO:tasks.workunit.client.0.smithi136.stdout: "osd.4": { 2024-09-12T23:37:50.677 INFO:tasks.workunit.client.0.smithi136.stdout: "actual_memory_used": 273154048, 2024-09-12T23:37:50.677 INFO:tasks.workunit.client.0.smithi136.stdout: "central_cache_freelist": 899480, 2024-09-12T23:37:50.677 INFO:tasks.workunit.client.0.smithi136.stdout: "malloc_metadata": 2752512, 2024-09-12T23:37:50.677 INFO:tasks.workunit.client.0.smithi136.stdout: "page_heap_freelist": 0, 2024-09-12T23:37:50.677 INFO:tasks.workunit.client.0.smithi136.stdout: "released_to_os": 786432, 2024-09-12T23:37:50.677 INFO:tasks.workunit.client.0.smithi136.stdout: "spans_in_use": 1309, 2024-09-12T23:37:50.677 INFO:tasks.workunit.client.0.smithi136.stdout: "tcmalloc_page_size": 8192, 2024-09-12T23:37:50.677 INFO:tasks.workunit.client.0.smithi136.stdout: "thread_cache_freelists": 4802136, 2024-09-12T23:37:50.677 INFO:tasks.workunit.client.0.smithi136.stdout: "thread_heaps_in_use": 46, 2024-09-12T23:37:50.677 INFO:tasks.workunit.client.0.smithi136.stdout: "transfer_cache_freelist": 2009344, 2024-09-12T23:37:50.677 INFO:tasks.workunit.client.0.smithi136.stdout: "use_by_application": 262690576, 2024-09-12T23:37:50.678 INFO:tasks.workunit.client.0.smithi136.stdout: "virtual_address_space_used": 273940480 2024-09-12T23:37:50.678 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.678 INFO:tasks.workunit.client.0.smithi136.stdout: "osd.5": { 2024-09-12T23:37:50.678 INFO:tasks.workunit.client.0.smithi136.stdout: "actual_memory_used": 272596992, 2024-09-12T23:37:50.678 INFO:tasks.workunit.client.0.smithi136.stdout: "central_cache_freelist": 610776, 2024-09-12T23:37:50.678 INFO:tasks.workunit.client.0.smithi136.stdout: "malloc_metadata": 2752512, 2024-09-12T23:37:50.678 INFO:tasks.workunit.client.0.smithi136.stdout: "page_heap_freelist": 0, 2024-09-12T23:37:50.678 INFO:tasks.workunit.client.0.smithi136.stdout: "released_to_os": 81920, 2024-09-12T23:37:50.678 INFO:tasks.workunit.client.0.smithi136.stdout: "spans_in_use": 1318, 2024-09-12T23:37:50.678 INFO:tasks.workunit.client.0.smithi136.stdout: "tcmalloc_page_size": 8192, 2024-09-12T23:37:50.678 INFO:tasks.workunit.client.0.smithi136.stdout: "thread_cache_freelists": 4557448, 2024-09-12T23:37:50.678 INFO:tasks.workunit.client.0.smithi136.stdout: "thread_heaps_in_use": 46, 2024-09-12T23:37:50.679 INFO:tasks.workunit.client.0.smithi136.stdout: "transfer_cache_freelist": 2323968, 2024-09-12T23:37:50.679 INFO:tasks.workunit.client.0.smithi136.stdout: "use_by_application": 262352288, 2024-09-12T23:37:50.679 INFO:tasks.workunit.client.0.smithi136.stdout: "virtual_address_space_used": 272678912 2024-09-12T23:37:50.679 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.679 INFO:tasks.workunit.client.0.smithi136.stdout: "osd.6": { 2024-09-12T23:37:50.679 INFO:tasks.workunit.client.0.smithi136.stdout: "actual_memory_used": 272613376, 2024-09-12T23:37:50.679 INFO:tasks.workunit.client.0.smithi136.stdout: "central_cache_freelist": 898504, 2024-09-12T23:37:50.679 INFO:tasks.workunit.client.0.smithi136.stdout: "malloc_metadata": 2752512, 2024-09-12T23:37:50.679 INFO:tasks.workunit.client.0.smithi136.stdout: "page_heap_freelist": 0, 2024-09-12T23:37:50.679 INFO:tasks.workunit.client.0.smithi136.stdout: "released_to_os": 1327104, 2024-09-12T23:37:50.679 INFO:tasks.workunit.client.0.smithi136.stdout: "spans_in_use": 1294, 2024-09-12T23:37:50.680 INFO:tasks.workunit.client.0.smithi136.stdout: "tcmalloc_page_size": 8192, 2024-09-12T23:37:50.680 INFO:tasks.workunit.client.0.smithi136.stdout: "thread_cache_freelists": 4584616, 2024-09-12T23:37:50.680 INFO:tasks.workunit.client.0.smithi136.stdout: "thread_heaps_in_use": 46, 2024-09-12T23:37:50.680 INFO:tasks.workunit.client.0.smithi136.stdout: "transfer_cache_freelist": 1962496, 2024-09-12T23:37:50.680 INFO:tasks.workunit.client.0.smithi136.stdout: "use_by_application": 262415248, 2024-09-12T23:37:50.680 INFO:tasks.workunit.client.0.smithi136.stdout: "virtual_address_space_used": 273940480 2024-09-12T23:37:50.680 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.680 INFO:tasks.workunit.client.0.smithi136.stdout: "osd.7": { 2024-09-12T23:37:50.680 INFO:tasks.workunit.client.0.smithi136.stdout: "actual_memory_used": 273784832, 2024-09-12T23:37:50.680 INFO:tasks.workunit.client.0.smithi136.stdout: "central_cache_freelist": 1014952, 2024-09-12T23:37:50.680 INFO:tasks.workunit.client.0.smithi136.stdout: "malloc_metadata": 2752512, 2024-09-12T23:37:50.681 INFO:tasks.workunit.client.0.smithi136.stdout: "page_heap_freelist": 8192, 2024-09-12T23:37:50.681 INFO:tasks.workunit.client.0.smithi136.stdout: "released_to_os": 90112, 2024-09-12T23:37:50.681 INFO:tasks.workunit.client.0.smithi136.stdout: "spans_in_use": 1351, 2024-09-12T23:37:50.681 INFO:tasks.workunit.client.0.smithi136.stdout: "tcmalloc_page_size": 8192, 2024-09-12T23:37:50.681 INFO:tasks.workunit.client.0.smithi136.stdout: "thread_cache_freelists": 5147896, 2024-09-12T23:37:50.681 INFO:tasks.workunit.client.0.smithi136.stdout: "thread_heaps_in_use": 46, 2024-09-12T23:37:50.681 INFO:tasks.workunit.client.0.smithi136.stdout: "transfer_cache_freelist": 2059520, 2024-09-12T23:37:50.681 INFO:tasks.workunit.client.0.smithi136.stdout: "use_by_application": 262801760, 2024-09-12T23:37:50.681 INFO:tasks.workunit.client.0.smithi136.stdout: "virtual_address_space_used": 273874944 2024-09-12T23:37:50.681 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.681 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.681 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.682 INFO:tasks.workunit.client.0.smithi136.stdout: "hosts": { 2024-09-12T23:37:50.682 INFO:tasks.workunit.client.0.smithi136.stdout: "num": 2, 2024-09-12T23:37:50.682 INFO:tasks.workunit.client.0.smithi136.stdout: "num_with_mds": 2, 2024-09-12T23:37:50.682 INFO:tasks.workunit.client.0.smithi136.stdout: "num_with_mgr": 2, 2024-09-12T23:37:50.682 INFO:tasks.workunit.client.0.smithi136.stdout: "num_with_mon": 2, 2024-09-12T23:37:50.682 INFO:tasks.workunit.client.0.smithi136.stdout: "num_with_osd": 2 2024-09-12T23:37:50.682 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.682 INFO:tasks.workunit.client.0.smithi136.stdout: "io_rate": { 2024-09-12T23:37:50.682 INFO:tasks.workunit.client.0.smithi136.stdout: "pg_stats_delta": { 2024-09-12T23:37:50.682 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": 0, 2024-09-12T23:37:50.682 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:50.683 INFO:tasks.workunit.client.0.smithi136.stdout: "num_store_stats": 0, 2024-09-12T23:37:50.683 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:50.683 INFO:tasks.workunit.client.0.smithi136.stdout: "stamp_delta": "12.003996", 2024-09-12T23:37:50.683 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:50.683 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:50.683 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:50.683 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:50.683 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:50.683 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:50.683 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:50.683 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:50.683 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:50.684 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:50.684 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:50.684 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:50.684 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:50.684 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:50.684 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:50.684 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:50.684 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:50.684 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:50.684 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:50.684 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:50.685 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:50.685 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:50.685 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:50.685 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:50.685 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:50.685 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:50.685 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:50.685 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:50.685 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:50.685 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:50.685 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:50.685 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:50.686 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:50.686 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:50.686 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 290, 2024-09-12T23:37:50.686 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 368, 2024-09-12T23:37:50.686 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:50.686 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:50.686 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:50.686 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 64, 2024-09-12T23:37:50.686 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:50.686 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.686 INFO:tasks.workunit.client.0.smithi136.stdout: "store_stats": { 2024-09-12T23:37:50.687 INFO:tasks.workunit.client.0.smithi136.stdout: "allocated": 0, 2024-09-12T23:37:50.687 INFO:tasks.workunit.client.0.smithi136.stdout: "available": 0, 2024-09-12T23:37:50.687 INFO:tasks.workunit.client.0.smithi136.stdout: "data_compressed": 0, 2024-09-12T23:37:50.687 INFO:tasks.workunit.client.0.smithi136.stdout: "data_compressed_allocated": 0, 2024-09-12T23:37:50.687 INFO:tasks.workunit.client.0.smithi136.stdout: "data_compressed_original": 0, 2024-09-12T23:37:50.687 INFO:tasks.workunit.client.0.smithi136.stdout: "data_stored": 0, 2024-09-12T23:37:50.687 INFO:tasks.workunit.client.0.smithi136.stdout: "internal_metadata": 0, 2024-09-12T23:37:50.687 INFO:tasks.workunit.client.0.smithi136.stdout: "internally_reserved": 0, 2024-09-12T23:37:50.687 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_allocated": 0, 2024-09-12T23:37:50.687 INFO:tasks.workunit.client.0.smithi136.stdout: "total": 0 2024-09-12T23:37:50.687 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.688 INFO:tasks.workunit.client.0.smithi136.stdout: "up": 0 2024-09-12T23:37:50.688 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.688 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.688 INFO:tasks.workunit.client.0.smithi136.stdout: "leaderboard": false, 2024-09-12T23:37:50.688 INFO:tasks.workunit.client.0.smithi136.stdout: "license": "sharing-1-0", 2024-09-12T23:37:50.688 INFO:tasks.workunit.client.0.smithi136.stdout: "mempool": { 2024-09-12T23:37:50.688 INFO:tasks.workunit.client.0.smithi136.stdout: "mds": { 2024-09-12T23:37:50.688 INFO:tasks.workunit.client.0.smithi136.stdout: "mds.4585d541b331f263094839d9f58dd50a20d3f517": { 2024-09-12T23:37:50.688 INFO:tasks.workunit.client.0.smithi136.stdout: "bloom_filter": { 2024-09-12T23:37:50.688 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.688 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.688 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.689 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs": { 2024-09-12T23:37:50.689 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.689 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.689 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.689 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs_file_reader": { 2024-09-12T23:37:50.689 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.689 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.689 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.689 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs_file_writer": { 2024-09-12T23:37:50.689 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.689 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.689 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.690 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Blob": { 2024-09-12T23:37:50.690 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.690 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.690 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.690 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Buffer": { 2024-09-12T23:37:50.690 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.690 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.690 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.690 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Extent": { 2024-09-12T23:37:50.690 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.690 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.691 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.691 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_SharedBlob": { 2024-09-12T23:37:50.691 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.691 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.691 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.691 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_alloc": { 2024-09-12T23:37:50.691 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.691 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.691 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.691 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_data": { 2024-09-12T23:37:50.691 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.692 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.692 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.692 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_meta": { 2024-09-12T23:37:50.692 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.692 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.692 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.692 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_onode": { 2024-09-12T23:37:50.692 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.692 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.692 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.692 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_other": { 2024-09-12T23:37:50.692 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.693 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.693 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.693 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_fsck": { 2024-09-12T23:37:50.693 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.693 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.693 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.693 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_inline_bl": { 2024-09-12T23:37:50.693 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.693 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.693 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.694 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_txc": { 2024-09-12T23:37:50.694 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.694 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.694 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.694 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_writing": { 2024-09-12T23:37:50.694 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.694 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.694 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.694 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_writing_deferred": { 2024-09-12T23:37:50.694 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.694 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.695 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.695 INFO:tasks.workunit.client.0.smithi136.stdout: "buffer_anon": { 2024-09-12T23:37:50.695 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 222471, 2024-09-12T23:37:50.695 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 72 2024-09-12T23:37:50.695 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.695 INFO:tasks.workunit.client.0.smithi136.stdout: "buffer_meta": { 2024-09-12T23:37:50.695 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.695 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.695 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.695 INFO:tasks.workunit.client.0.smithi136.stdout: "mds_co": { 2024-09-12T23:37:50.695 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 50488, 2024-09-12T23:37:50.695 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 208 2024-09-12T23:37:50.696 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.696 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:50.696 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.696 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.696 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.696 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mapbl": { 2024-09-12T23:37:50.696 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.696 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.696 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.696 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_pglog": { 2024-09-12T23:37:50.696 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.697 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.697 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.697 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap": { 2024-09-12T23:37:50.697 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 8128, 2024-09-12T23:37:50.697 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 139 2024-09-12T23:37:50.697 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.697 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap_mapping": { 2024-09-12T23:37:50.697 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.697 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.697 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.697 INFO:tasks.workunit.client.0.smithi136.stdout: "pgmap": { 2024-09-12T23:37:50.697 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.698 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.698 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.698 INFO:tasks.workunit.client.0.smithi136.stdout: "unittest_1": { 2024-09-12T23:37:50.698 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.698 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.698 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.698 INFO:tasks.workunit.client.0.smithi136.stdout: "unittest_2": { 2024-09-12T23:37:50.698 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.698 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.698 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.698 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.698 INFO:tasks.workunit.client.0.smithi136.stdout: "mds.48b2ee34c5040a1abbee529ca5897a6d3827a23a": { 2024-09-12T23:37:50.699 INFO:tasks.workunit.client.0.smithi136.stdout: "bloom_filter": { 2024-09-12T23:37:50.699 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.699 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.699 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.699 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs": { 2024-09-12T23:37:50.699 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.699 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.699 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.699 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs_file_reader": { 2024-09-12T23:37:50.699 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.699 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.699 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.700 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs_file_writer": { 2024-09-12T23:37:50.700 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.700 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.700 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.700 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Blob": { 2024-09-12T23:37:50.700 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.700 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.700 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.700 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Buffer": { 2024-09-12T23:37:50.700 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.700 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.701 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.701 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Extent": { 2024-09-12T23:37:50.701 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.701 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.701 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.701 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_SharedBlob": { 2024-09-12T23:37:50.701 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.701 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.701 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.701 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_alloc": { 2024-09-12T23:37:50.701 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.701 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.702 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.702 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_data": { 2024-09-12T23:37:50.702 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.702 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.702 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.702 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_meta": { 2024-09-12T23:37:50.702 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.702 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.702 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.702 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_onode": { 2024-09-12T23:37:50.702 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.702 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.703 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.703 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_other": { 2024-09-12T23:37:50.703 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.703 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.703 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.703 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_fsck": { 2024-09-12T23:37:50.703 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.703 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.703 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.703 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_inline_bl": { 2024-09-12T23:37:50.703 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.704 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.704 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.704 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_txc": { 2024-09-12T23:37:50.704 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.704 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.704 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.704 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_writing": { 2024-09-12T23:37:50.704 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.704 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.704 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.704 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_writing_deferred": { 2024-09-12T23:37:50.704 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.705 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.705 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.705 INFO:tasks.workunit.client.0.smithi136.stdout: "buffer_anon": { 2024-09-12T23:37:50.705 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 150088, 2024-09-12T23:37:50.705 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 33 2024-09-12T23:37:50.705 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.705 INFO:tasks.workunit.client.0.smithi136.stdout: "buffer_meta": { 2024-09-12T23:37:50.705 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.705 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.705 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.705 INFO:tasks.workunit.client.0.smithi136.stdout: "mds_co": { 2024-09-12T23:37:50.706 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 856, 2024-09-12T23:37:50.706 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 2 2024-09-12T23:37:50.706 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.706 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:50.706 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.706 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.706 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.706 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mapbl": { 2024-09-12T23:37:50.706 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.706 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.706 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.707 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_pglog": { 2024-09-12T23:37:50.707 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.707 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.707 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.707 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap": { 2024-09-12T23:37:50.707 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.707 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.707 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.707 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap_mapping": { 2024-09-12T23:37:50.707 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.707 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.708 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.708 INFO:tasks.workunit.client.0.smithi136.stdout: "pgmap": { 2024-09-12T23:37:50.708 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.708 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.708 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.708 INFO:tasks.workunit.client.0.smithi136.stdout: "unittest_1": { 2024-09-12T23:37:50.708 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.708 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.708 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.708 INFO:tasks.workunit.client.0.smithi136.stdout: "unittest_2": { 2024-09-12T23:37:50.708 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.709 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.709 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.709 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.709 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.709 INFO:tasks.workunit.client.0.smithi136.stdout: "mon": { 2024-09-12T23:37:50.709 INFO:tasks.workunit.client.0.smithi136.stdout: "mon.0027af889fd200cedbdb75d916e7558e544c1ffa": { 2024-09-12T23:37:50.709 INFO:tasks.workunit.client.0.smithi136.stdout: "bloom_filter": { 2024-09-12T23:37:50.709 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.709 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.709 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.709 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs": { 2024-09-12T23:37:50.710 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.710 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.710 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.710 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs_file_reader": { 2024-09-12T23:37:50.710 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.710 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.710 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.710 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs_file_writer": { 2024-09-12T23:37:50.710 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.710 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.710 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.711 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Blob": { 2024-09-12T23:37:50.711 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.711 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.711 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.711 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Buffer": { 2024-09-12T23:37:50.711 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.711 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.711 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.711 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Extent": { 2024-09-12T23:37:50.711 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.711 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.711 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.712 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_SharedBlob": { 2024-09-12T23:37:50.712 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.712 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.712 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.712 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_alloc": { 2024-09-12T23:37:50.712 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.712 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.712 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.712 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_data": { 2024-09-12T23:37:50.712 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.712 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.713 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.713 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_meta": { 2024-09-12T23:37:50.713 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.713 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.713 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.713 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_onode": { 2024-09-12T23:37:50.713 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.713 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.713 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.713 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_other": { 2024-09-12T23:37:50.713 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.714 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.714 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.714 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_fsck": { 2024-09-12T23:37:50.714 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.714 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.714 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.714 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_inline_bl": { 2024-09-12T23:37:50.714 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.714 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.714 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.714 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_txc": { 2024-09-12T23:37:50.714 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.715 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.715 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.715 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_writing": { 2024-09-12T23:37:50.715 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.715 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.715 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.715 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_writing_deferred": { 2024-09-12T23:37:50.715 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.715 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.715 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.715 INFO:tasks.workunit.client.0.smithi136.stdout: "buffer_anon": { 2024-09-12T23:37:50.715 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 1055839, 2024-09-12T23:37:50.716 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 351 2024-09-12T23:37:50.716 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.716 INFO:tasks.workunit.client.0.smithi136.stdout: "buffer_meta": { 2024-09-12T23:37:50.716 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.716 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.716 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.716 INFO:tasks.workunit.client.0.smithi136.stdout: "mds_co": { 2024-09-12T23:37:50.716 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.716 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.716 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.716 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:50.716 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.717 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.717 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.717 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mapbl": { 2024-09-12T23:37:50.717 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.717 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.717 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.717 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_pglog": { 2024-09-12T23:37:50.717 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.717 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.717 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.717 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap": { 2024-09-12T23:37:50.718 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 6776, 2024-09-12T23:37:50.718 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 138 2024-09-12T23:37:50.718 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.718 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap_mapping": { 2024-09-12T23:37:50.718 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 12744, 2024-09-12T23:37:50.718 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 1546 2024-09-12T23:37:50.718 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.718 INFO:tasks.workunit.client.0.smithi136.stdout: "pgmap": { 2024-09-12T23:37:50.718 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 5312, 2024-09-12T23:37:50.718 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 103 2024-09-12T23:37:50.718 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.719 INFO:tasks.workunit.client.0.smithi136.stdout: "unittest_1": { 2024-09-12T23:37:50.719 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.719 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.719 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.719 INFO:tasks.workunit.client.0.smithi136.stdout: "unittest_2": { 2024-09-12T23:37:50.719 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.719 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.719 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.719 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.719 INFO:tasks.workunit.client.0.smithi136.stdout: "mon.4585d541b331f263094839d9f58dd50a20d3f517": { 2024-09-12T23:37:50.719 INFO:tasks.workunit.client.0.smithi136.stdout: "bloom_filter": { 2024-09-12T23:37:50.719 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.719 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.720 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.720 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs": { 2024-09-12T23:37:50.720 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.720 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.720 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.720 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs_file_reader": { 2024-09-12T23:37:50.720 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.720 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.720 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.720 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs_file_writer": { 2024-09-12T23:37:50.720 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.720 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.721 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.721 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Blob": { 2024-09-12T23:37:50.721 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.721 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.721 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.721 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Buffer": { 2024-09-12T23:37:50.721 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.721 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.721 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.721 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Extent": { 2024-09-12T23:37:50.721 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.721 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.722 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.722 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_SharedBlob": { 2024-09-12T23:37:50.722 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.722 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.722 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.722 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_alloc": { 2024-09-12T23:37:50.722 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.722 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.722 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.722 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_data": { 2024-09-12T23:37:50.722 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.723 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.723 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.723 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_meta": { 2024-09-12T23:37:50.723 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.723 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.723 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.723 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_onode": { 2024-09-12T23:37:50.723 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.723 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.723 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.723 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_other": { 2024-09-12T23:37:50.723 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.724 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.724 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.724 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_fsck": { 2024-09-12T23:37:50.724 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.724 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.724 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.724 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_inline_bl": { 2024-09-12T23:37:50.724 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.724 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.724 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.725 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_txc": { 2024-09-12T23:37:50.725 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.725 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.725 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.725 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_writing": { 2024-09-12T23:37:50.725 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.725 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.725 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.725 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_writing_deferred": { 2024-09-12T23:37:50.725 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.726 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.726 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.726 INFO:tasks.workunit.client.0.smithi136.stdout: "buffer_anon": { 2024-09-12T23:37:50.726 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 1051167, 2024-09-12T23:37:50.726 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 337 2024-09-12T23:37:50.726 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.726 INFO:tasks.workunit.client.0.smithi136.stdout: "buffer_meta": { 2024-09-12T23:37:50.726 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.726 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.726 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.727 INFO:tasks.workunit.client.0.smithi136.stdout: "mds_co": { 2024-09-12T23:37:50.727 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.727 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.727 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.727 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:50.727 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.727 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.727 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.727 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mapbl": { 2024-09-12T23:37:50.727 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.727 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.727 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.728 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_pglog": { 2024-09-12T23:37:50.728 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.728 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.728 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.728 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap": { 2024-09-12T23:37:50.728 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 6776, 2024-09-12T23:37:50.728 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 138 2024-09-12T23:37:50.728 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.728 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap_mapping": { 2024-09-12T23:37:50.728 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 12744, 2024-09-12T23:37:50.728 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 1546 2024-09-12T23:37:50.729 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.729 INFO:tasks.workunit.client.0.smithi136.stdout: "pgmap": { 2024-09-12T23:37:50.729 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 5312, 2024-09-12T23:37:50.729 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 103 2024-09-12T23:37:50.729 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.729 INFO:tasks.workunit.client.0.smithi136.stdout: "unittest_1": { 2024-09-12T23:37:50.729 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.729 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.729 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.729 INFO:tasks.workunit.client.0.smithi136.stdout: "unittest_2": { 2024-09-12T23:37:50.729 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.729 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.730 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.730 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.730 INFO:tasks.workunit.client.0.smithi136.stdout: "mon.48b2ee34c5040a1abbee529ca5897a6d3827a23a": { 2024-09-12T23:37:50.730 INFO:tasks.workunit.client.0.smithi136.stdout: "bloom_filter": { 2024-09-12T23:37:50.730 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.730 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.730 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.730 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs": { 2024-09-12T23:37:50.730 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.730 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.731 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.731 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs_file_reader": { 2024-09-12T23:37:50.731 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.731 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.731 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.731 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs_file_writer": { 2024-09-12T23:37:50.731 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.731 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.731 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.731 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Blob": { 2024-09-12T23:37:50.731 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.732 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.732 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.732 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Buffer": { 2024-09-12T23:37:50.732 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.732 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.732 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.732 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Extent": { 2024-09-12T23:37:50.732 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.732 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.732 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.732 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_SharedBlob": { 2024-09-12T23:37:50.733 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.733 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.733 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.733 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_alloc": { 2024-09-12T23:37:50.733 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.733 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.733 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.733 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_data": { 2024-09-12T23:37:50.733 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.733 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.733 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.734 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_meta": { 2024-09-12T23:37:50.734 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.734 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.734 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.734 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_onode": { 2024-09-12T23:37:50.734 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.734 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.734 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.734 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_other": { 2024-09-12T23:37:50.734 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.734 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.735 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.735 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_fsck": { 2024-09-12T23:37:50.735 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.735 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.735 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.735 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_inline_bl": { 2024-09-12T23:37:50.735 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.735 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.735 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.735 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_txc": { 2024-09-12T23:37:50.736 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.736 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.736 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.736 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_writing": { 2024-09-12T23:37:50.736 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.736 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.736 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.736 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_writing_deferred": { 2024-09-12T23:37:50.736 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.736 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.736 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.737 INFO:tasks.workunit.client.0.smithi136.stdout: "buffer_anon": { 2024-09-12T23:37:50.737 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 982047, 2024-09-12T23:37:50.737 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 309 2024-09-12T23:37:50.737 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.737 INFO:tasks.workunit.client.0.smithi136.stdout: "buffer_meta": { 2024-09-12T23:37:50.737 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.737 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.737 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.737 INFO:tasks.workunit.client.0.smithi136.stdout: "mds_co": { 2024-09-12T23:37:50.737 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.737 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.737 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.738 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:50.738 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.738 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.738 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.738 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mapbl": { 2024-09-12T23:37:50.738 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.738 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.738 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.738 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_pglog": { 2024-09-12T23:37:50.738 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.738 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.739 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.739 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap": { 2024-09-12T23:37:50.739 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 6776, 2024-09-12T23:37:50.739 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 138 2024-09-12T23:37:50.739 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.739 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap_mapping": { 2024-09-12T23:37:50.739 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 12744, 2024-09-12T23:37:50.739 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 1546 2024-09-12T23:37:50.739 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.739 INFO:tasks.workunit.client.0.smithi136.stdout: "pgmap": { 2024-09-12T23:37:50.739 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 10624, 2024-09-12T23:37:50.740 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 206 2024-09-12T23:37:50.740 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.740 INFO:tasks.workunit.client.0.smithi136.stdout: "unittest_1": { 2024-09-12T23:37:50.740 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.740 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.740 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.740 INFO:tasks.workunit.client.0.smithi136.stdout: "unittest_2": { 2024-09-12T23:37:50.740 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.740 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.740 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.740 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.740 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.741 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:50.741 INFO:tasks.workunit.client.0.smithi136.stdout: "osd.0": { 2024-09-12T23:37:50.741 INFO:tasks.workunit.client.0.smithi136.stdout: "bloom_filter": { 2024-09-12T23:37:50.741 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.741 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.741 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.741 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs": { 2024-09-12T23:37:50.741 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 6304, 2024-09-12T23:37:50.741 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 115 2024-09-12T23:37:50.741 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.741 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs_file_reader": { 2024-09-12T23:37:50.741 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 16896, 2024-09-12T23:37:50.742 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 8 2024-09-12T23:37:50.742 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.742 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs_file_writer": { 2024-09-12T23:37:50.742 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 576, 2024-09-12T23:37:50.742 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 3 2024-09-12T23:37:50.742 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.742 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Blob": { 2024-09-12T23:37:50.742 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 19264, 2024-09-12T23:37:50.742 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 172 2024-09-12T23:37:50.742 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.742 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Buffer": { 2024-09-12T23:37:50.742 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 448, 2024-09-12T23:37:50.743 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 4 2024-09-12T23:37:50.743 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.743 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Extent": { 2024-09-12T23:37:50.743 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 8256, 2024-09-12T23:37:50.743 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 172 2024-09-12T23:37:50.743 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.743 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_SharedBlob": { 2024-09-12T23:37:50.743 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 19264, 2024-09-12T23:37:50.743 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 172 2024-09-12T23:37:50.743 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.743 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_alloc": { 2024-09-12T23:37:50.743 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 5280, 2024-09-12T23:37:50.744 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 66 2024-09-12T23:37:50.744 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.744 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_data": { 2024-09-12T23:37:50.744 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 22733, 2024-09-12T23:37:50.744 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 5 2024-09-12T23:37:50.744 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.744 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_meta": { 2024-09-12T23:37:50.744 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 52867, 2024-09-12T23:37:50.744 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 9516 2024-09-12T23:37:50.744 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.744 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_onode": { 2024-09-12T23:37:50.744 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 129880, 2024-09-12T23:37:50.744 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 191 2024-09-12T23:37:50.745 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.745 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_other": { 2024-09-12T23:37:50.745 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 10800, 2024-09-12T23:37:50.745 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 680 2024-09-12T23:37:50.745 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.745 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_fsck": { 2024-09-12T23:37:50.745 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.745 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.745 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.745 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_inline_bl": { 2024-09-12T23:37:50.745 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 30566, 2024-09-12T23:37:50.745 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 191 2024-09-12T23:37:50.746 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.746 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_txc": { 2024-09-12T23:37:50.746 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.746 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.746 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.746 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_writing": { 2024-09-12T23:37:50.746 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 366171, 2024-09-12T23:37:50.746 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 80 2024-09-12T23:37:50.746 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.746 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_writing_deferred": { 2024-09-12T23:37:50.746 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.746 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.747 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.747 INFO:tasks.workunit.client.0.smithi136.stdout: "buffer_anon": { 2024-09-12T23:37:50.747 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 2418575, 2024-09-12T23:37:50.747 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 192 2024-09-12T23:37:50.747 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.747 INFO:tasks.workunit.client.0.smithi136.stdout: "buffer_meta": { 2024-09-12T23:37:50.747 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 5808, 2024-09-12T23:37:50.747 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 66 2024-09-12T23:37:50.747 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.747 INFO:tasks.workunit.client.0.smithi136.stdout: "mds_co": { 2024-09-12T23:37:50.747 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.747 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.748 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.748 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:50.748 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 521280, 2024-09-12T23:37:50.748 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 45 2024-09-12T23:37:50.748 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.748 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mapbl": { 2024-09-12T23:37:50.748 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.748 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.748 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.748 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_pglog": { 2024-09-12T23:37:50.748 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 76024, 2024-09-12T23:37:50.748 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 151 2024-09-12T23:37:50.749 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.749 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap": { 2024-09-12T23:37:50.749 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 227704, 2024-09-12T23:37:50.749 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 3449 2024-09-12T23:37:50.749 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.749 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap_mapping": { 2024-09-12T23:37:50.749 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.749 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.749 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.749 INFO:tasks.workunit.client.0.smithi136.stdout: "pgmap": { 2024-09-12T23:37:50.749 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.750 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.750 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.750 INFO:tasks.workunit.client.0.smithi136.stdout: "unittest_1": { 2024-09-12T23:37:50.750 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.750 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.750 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.750 INFO:tasks.workunit.client.0.smithi136.stdout: "unittest_2": { 2024-09-12T23:37:50.750 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.750 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.750 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.750 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.750 INFO:tasks.workunit.client.0.smithi136.stdout: "osd.1": { 2024-09-12T23:37:50.751 INFO:tasks.workunit.client.0.smithi136.stdout: "bloom_filter": { 2024-09-12T23:37:50.751 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.751 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.751 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.751 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs": { 2024-09-12T23:37:50.751 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 6304, 2024-09-12T23:37:50.751 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 115 2024-09-12T23:37:50.751 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.751 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs_file_reader": { 2024-09-12T23:37:50.751 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 16896, 2024-09-12T23:37:50.751 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 8 2024-09-12T23:37:50.751 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.752 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs_file_writer": { 2024-09-12T23:37:50.752 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 576, 2024-09-12T23:37:50.752 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 3 2024-09-12T23:37:50.752 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.752 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Blob": { 2024-09-12T23:37:50.752 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 18144, 2024-09-12T23:37:50.752 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 162 2024-09-12T23:37:50.752 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.752 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Buffer": { 2024-09-12T23:37:50.752 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 336, 2024-09-12T23:37:50.752 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 3 2024-09-12T23:37:50.752 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.753 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Extent": { 2024-09-12T23:37:50.753 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 7776, 2024-09-12T23:37:50.753 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 162 2024-09-12T23:37:50.753 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.753 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_SharedBlob": { 2024-09-12T23:37:50.753 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 18144, 2024-09-12T23:37:50.753 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 162 2024-09-12T23:37:50.753 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.753 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_alloc": { 2024-09-12T23:37:50.753 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 4640, 2024-09-12T23:37:50.753 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 58 2024-09-12T23:37:50.754 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.754 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_data": { 2024-09-12T23:37:50.754 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 12288, 2024-09-12T23:37:50.754 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 3 2024-09-12T23:37:50.754 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.754 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_meta": { 2024-09-12T23:37:50.754 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 47344, 2024-09-12T23:37:50.754 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 8644 2024-09-12T23:37:50.754 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.754 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_onode": { 2024-09-12T23:37:50.754 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 122400, 2024-09-12T23:37:50.755 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 180 2024-09-12T23:37:50.755 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.755 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_other": { 2024-09-12T23:37:50.755 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 9004, 2024-09-12T23:37:50.755 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 510 2024-09-12T23:37:50.755 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.755 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_fsck": { 2024-09-12T23:37:50.755 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.755 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.756 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.756 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_inline_bl": { 2024-09-12T23:37:50.756 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 26810, 2024-09-12T23:37:50.756 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 180 2024-09-12T23:37:50.756 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.756 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_txc": { 2024-09-12T23:37:50.756 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.756 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.756 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.756 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_writing": { 2024-09-12T23:37:50.756 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 397408, 2024-09-12T23:37:50.757 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 87 2024-09-12T23:37:50.757 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.757 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_writing_deferred": { 2024-09-12T23:37:50.757 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.757 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.757 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.757 INFO:tasks.workunit.client.0.smithi136.stdout: "buffer_anon": { 2024-09-12T23:37:50.757 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 2405051, 2024-09-12T23:37:50.757 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 186 2024-09-12T23:37:50.757 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.757 INFO:tasks.workunit.client.0.smithi136.stdout: "buffer_meta": { 2024-09-12T23:37:50.757 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 4928, 2024-09-12T23:37:50.758 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 56 2024-09-12T23:37:50.758 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.758 INFO:tasks.workunit.client.0.smithi136.stdout: "mds_co": { 2024-09-12T23:37:50.758 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.758 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.758 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.758 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:50.758 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 451776, 2024-09-12T23:37:50.758 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 39 2024-09-12T23:37:50.758 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.758 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mapbl": { 2024-09-12T23:37:50.758 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.759 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.759 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.759 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_pglog": { 2024-09-12T23:37:50.759 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 8528, 2024-09-12T23:37:50.759 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 22 2024-09-12T23:37:50.759 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.759 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap": { 2024-09-12T23:37:50.759 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 227192, 2024-09-12T23:37:50.759 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 3425 2024-09-12T23:37:50.759 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.759 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap_mapping": { 2024-09-12T23:37:50.759 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.760 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.760 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.760 INFO:tasks.workunit.client.0.smithi136.stdout: "pgmap": { 2024-09-12T23:37:50.760 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.760 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.760 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.760 INFO:tasks.workunit.client.0.smithi136.stdout: "unittest_1": { 2024-09-12T23:37:50.760 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.760 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.760 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.760 INFO:tasks.workunit.client.0.smithi136.stdout: "unittest_2": { 2024-09-12T23:37:50.761 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.761 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.761 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.761 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.761 INFO:tasks.workunit.client.0.smithi136.stdout: "osd.2": { 2024-09-12T23:37:50.761 INFO:tasks.workunit.client.0.smithi136.stdout: "bloom_filter": { 2024-09-12T23:37:50.761 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.761 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.761 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.761 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs": { 2024-09-12T23:37:50.761 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 6304, 2024-09-12T23:37:50.761 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 115 2024-09-12T23:37:50.762 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.762 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs_file_reader": { 2024-09-12T23:37:50.762 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 16896, 2024-09-12T23:37:50.762 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 8 2024-09-12T23:37:50.762 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.762 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs_file_writer": { 2024-09-12T23:37:50.762 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 576, 2024-09-12T23:37:50.762 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 3 2024-09-12T23:37:50.762 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.762 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Blob": { 2024-09-12T23:37:50.763 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 17808, 2024-09-12T23:37:50.763 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 159 2024-09-12T23:37:50.763 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.763 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Buffer": { 2024-09-12T23:37:50.763 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 1120, 2024-09-12T23:37:50.763 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 10 2024-09-12T23:37:50.763 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.763 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Extent": { 2024-09-12T23:37:50.763 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 7632, 2024-09-12T23:37:50.763 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 159 2024-09-12T23:37:50.763 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.763 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_SharedBlob": { 2024-09-12T23:37:50.764 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 17920, 2024-09-12T23:37:50.764 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 160 2024-09-12T23:37:50.764 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.764 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_alloc": { 2024-09-12T23:37:50.764 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 3680, 2024-09-12T23:37:50.764 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 46 2024-09-12T23:37:50.764 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.764 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_data": { 2024-09-12T23:37:50.764 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 8192, 2024-09-12T23:37:50.764 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 2 2024-09-12T23:37:50.764 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.764 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_meta": { 2024-09-12T23:37:50.765 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 44904, 2024-09-12T23:37:50.765 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 8224 2024-09-12T23:37:50.765 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.765 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_onode": { 2024-09-12T23:37:50.765 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 116280, 2024-09-12T23:37:50.765 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 171 2024-09-12T23:37:50.765 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.765 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_other": { 2024-09-12T23:37:50.765 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 8080, 2024-09-12T23:37:50.765 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 498 2024-09-12T23:37:50.765 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.766 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_fsck": { 2024-09-12T23:37:50.766 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.766 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.766 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.766 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_inline_bl": { 2024-09-12T23:37:50.766 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 25010, 2024-09-12T23:37:50.766 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 171 2024-09-12T23:37:50.766 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.766 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_txc": { 2024-09-12T23:37:50.766 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 1472, 2024-09-12T23:37:50.766 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 2 2024-09-12T23:37:50.766 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.766 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_writing": { 2024-09-12T23:37:50.767 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 337827, 2024-09-12T23:37:50.767 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 73 2024-09-12T23:37:50.767 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.767 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_writing_deferred": { 2024-09-12T23:37:50.767 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 20096, 2024-09-12T23:37:50.767 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 5 2024-09-12T23:37:50.767 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.767 INFO:tasks.workunit.client.0.smithi136.stdout: "buffer_anon": { 2024-09-12T23:37:50.767 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 2419800, 2024-09-12T23:37:50.767 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 180 2024-09-12T23:37:50.767 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.767 INFO:tasks.workunit.client.0.smithi136.stdout: "buffer_meta": { 2024-09-12T23:37:50.768 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 4400, 2024-09-12T23:37:50.768 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 50 2024-09-12T23:37:50.768 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.768 INFO:tasks.workunit.client.0.smithi136.stdout: "mds_co": { 2024-09-12T23:37:50.768 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.768 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.768 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.768 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:50.768 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 359104, 2024-09-12T23:37:50.768 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 31 2024-09-12T23:37:50.768 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.769 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mapbl": { 2024-09-12T23:37:50.769 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.769 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.769 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.769 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_pglog": { 2024-09-12T23:37:50.769 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 9360, 2024-09-12T23:37:50.769 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 26 2024-09-12T23:37:50.769 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.769 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap": { 2024-09-12T23:37:50.769 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 227704, 2024-09-12T23:37:50.769 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 3449 2024-09-12T23:37:50.769 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.770 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap_mapping": { 2024-09-12T23:37:50.770 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.770 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.770 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.770 INFO:tasks.workunit.client.0.smithi136.stdout: "pgmap": { 2024-09-12T23:37:50.770 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.770 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.770 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.770 INFO:tasks.workunit.client.0.smithi136.stdout: "unittest_1": { 2024-09-12T23:37:50.770 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.770 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.770 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.770 INFO:tasks.workunit.client.0.smithi136.stdout: "unittest_2": { 2024-09-12T23:37:50.771 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.771 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.771 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.771 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.771 INFO:tasks.workunit.client.0.smithi136.stdout: "osd.3": { 2024-09-12T23:37:50.771 INFO:tasks.workunit.client.0.smithi136.stdout: "bloom_filter": { 2024-09-12T23:37:50.771 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.771 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.771 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.771 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs": { 2024-09-12T23:37:50.771 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 6304, 2024-09-12T23:37:50.772 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 115 2024-09-12T23:37:50.772 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.772 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs_file_reader": { 2024-09-12T23:37:50.772 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 16896, 2024-09-12T23:37:50.772 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 8 2024-09-12T23:37:50.772 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.772 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs_file_writer": { 2024-09-12T23:37:50.772 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 576, 2024-09-12T23:37:50.772 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 3 2024-09-12T23:37:50.772 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.772 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Blob": { 2024-09-12T23:37:50.773 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 18368, 2024-09-12T23:37:50.773 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 164 2024-09-12T23:37:50.773 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.773 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Buffer": { 2024-09-12T23:37:50.773 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 224, 2024-09-12T23:37:50.773 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 2 2024-09-12T23:37:50.773 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.773 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Extent": { 2024-09-12T23:37:50.773 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 7872, 2024-09-12T23:37:50.773 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 164 2024-09-12T23:37:50.773 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.773 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_SharedBlob": { 2024-09-12T23:37:50.774 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 18368, 2024-09-12T23:37:50.774 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 164 2024-09-12T23:37:50.774 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.774 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_alloc": { 2024-09-12T23:37:50.774 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 3520, 2024-09-12T23:37:50.774 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 44 2024-09-12T23:37:50.774 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.774 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_data": { 2024-09-12T23:37:50.774 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.774 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.774 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.774 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_meta": { 2024-09-12T23:37:50.774 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 45902, 2024-09-12T23:37:50.775 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 8497 2024-09-12T23:37:50.775 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.775 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_onode": { 2024-09-12T23:37:50.775 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 120360, 2024-09-12T23:37:50.775 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 177 2024-09-12T23:37:50.775 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.775 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_other": { 2024-09-12T23:37:50.775 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 8868, 2024-09-12T23:37:50.775 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 515 2024-09-12T23:37:50.775 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.775 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_fsck": { 2024-09-12T23:37:50.775 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.776 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.776 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.776 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_inline_bl": { 2024-09-12T23:37:50.776 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 25706, 2024-09-12T23:37:50.776 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 177 2024-09-12T23:37:50.776 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.776 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_txc": { 2024-09-12T23:37:50.776 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 2208, 2024-09-12T23:37:50.776 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 3 2024-09-12T23:37:50.776 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.777 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_writing": { 2024-09-12T23:37:50.777 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 249673, 2024-09-12T23:37:50.777 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 51 2024-09-12T23:37:50.777 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.777 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_writing_deferred": { 2024-09-12T23:37:50.777 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 30637, 2024-09-12T23:37:50.777 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 9 2024-09-12T23:37:50.777 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.777 INFO:tasks.workunit.client.0.smithi136.stdout: "buffer_anon": { 2024-09-12T23:37:50.777 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 2436764, 2024-09-12T23:37:50.777 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 187 2024-09-12T23:37:50.777 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.778 INFO:tasks.workunit.client.0.smithi136.stdout: "buffer_meta": { 2024-09-12T23:37:50.778 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 4752, 2024-09-12T23:37:50.778 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 54 2024-09-12T23:37:50.778 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.778 INFO:tasks.workunit.client.0.smithi136.stdout: "mds_co": { 2024-09-12T23:37:50.778 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.778 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.778 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.778 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:50.778 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 428608, 2024-09-12T23:37:50.778 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 37 2024-09-12T23:37:50.778 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.779 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mapbl": { 2024-09-12T23:37:50.779 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.779 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.779 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.779 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_pglog": { 2024-09-12T23:37:50.779 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 10400, 2024-09-12T23:37:50.779 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 20 2024-09-12T23:37:50.779 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.779 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap": { 2024-09-12T23:37:50.779 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 227192, 2024-09-12T23:37:50.779 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 3425 2024-09-12T23:37:50.779 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.780 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap_mapping": { 2024-09-12T23:37:50.780 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.780 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.780 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.780 INFO:tasks.workunit.client.0.smithi136.stdout: "pgmap": { 2024-09-12T23:37:50.780 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.780 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.780 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.780 INFO:tasks.workunit.client.0.smithi136.stdout: "unittest_1": { 2024-09-12T23:37:50.780 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.780 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.780 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.781 INFO:tasks.workunit.client.0.smithi136.stdout: "unittest_2": { 2024-09-12T23:37:50.781 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.781 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.781 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.781 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.781 INFO:tasks.workunit.client.0.smithi136.stdout: "osd.4": { 2024-09-12T23:37:50.781 INFO:tasks.workunit.client.0.smithi136.stdout: "bloom_filter": { 2024-09-12T23:37:50.781 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.781 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.781 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.781 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs": { 2024-09-12T23:37:50.782 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 6304, 2024-09-12T23:37:50.782 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 115 2024-09-12T23:37:50.782 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.782 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs_file_reader": { 2024-09-12T23:37:50.782 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 16896, 2024-09-12T23:37:50.782 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 8 2024-09-12T23:37:50.782 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.782 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs_file_writer": { 2024-09-12T23:37:50.782 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 576, 2024-09-12T23:37:50.782 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 3 2024-09-12T23:37:50.782 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.782 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Blob": { 2024-09-12T23:37:50.783 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 18368, 2024-09-12T23:37:50.783 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 164 2024-09-12T23:37:50.783 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.783 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Buffer": { 2024-09-12T23:37:50.783 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 1008, 2024-09-12T23:37:50.783 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 9 2024-09-12T23:37:50.783 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.783 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Extent": { 2024-09-12T23:37:50.783 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 7872, 2024-09-12T23:37:50.783 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 164 2024-09-12T23:37:50.783 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.783 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_SharedBlob": { 2024-09-12T23:37:50.784 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 18480, 2024-09-12T23:37:50.784 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 165 2024-09-12T23:37:50.784 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.784 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_alloc": { 2024-09-12T23:37:50.784 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 3040, 2024-09-12T23:37:50.784 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 38 2024-09-12T23:37:50.784 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.784 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_data": { 2024-09-12T23:37:50.784 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 4096, 2024-09-12T23:37:50.784 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 1 2024-09-12T23:37:50.784 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.784 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_meta": { 2024-09-12T23:37:50.785 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 46942, 2024-09-12T23:37:50.785 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 8602 2024-09-12T23:37:50.785 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.785 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_onode": { 2024-09-12T23:37:50.785 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 121720, 2024-09-12T23:37:50.785 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 179 2024-09-12T23:37:50.785 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.785 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_other": { 2024-09-12T23:37:50.785 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 8964, 2024-09-12T23:37:50.785 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 515 2024-09-12T23:37:50.785 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.786 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_fsck": { 2024-09-12T23:37:50.786 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.786 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.786 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.786 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_inline_bl": { 2024-09-12T23:37:50.786 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 26218, 2024-09-12T23:37:50.786 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 179 2024-09-12T23:37:50.786 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.786 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_txc": { 2024-09-12T23:37:50.786 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 1472, 2024-09-12T23:37:50.786 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 2 2024-09-12T23:37:50.786 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.787 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_writing": { 2024-09-12T23:37:50.787 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 873885, 2024-09-12T23:37:50.787 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 61 2024-09-12T23:37:50.787 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.787 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_writing_deferred": { 2024-09-12T23:37:50.787 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 20096, 2024-09-12T23:37:50.787 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 5 2024-09-12T23:37:50.787 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.787 INFO:tasks.workunit.client.0.smithi136.stdout: "buffer_anon": { 2024-09-12T23:37:50.787 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 2398270, 2024-09-12T23:37:50.787 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 173 2024-09-12T23:37:50.787 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.788 INFO:tasks.workunit.client.0.smithi136.stdout: "buffer_meta": { 2024-09-12T23:37:50.788 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 4752, 2024-09-12T23:37:50.788 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 54 2024-09-12T23:37:50.788 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.788 INFO:tasks.workunit.client.0.smithi136.stdout: "mds_co": { 2024-09-12T23:37:50.788 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.788 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.788 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.788 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:50.788 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 440192, 2024-09-12T23:37:50.788 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 38 2024-09-12T23:37:50.788 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.789 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mapbl": { 2024-09-12T23:37:50.789 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.789 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.789 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.789 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_pglog": { 2024-09-12T23:37:50.789 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 8944, 2024-09-12T23:37:50.789 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 18 2024-09-12T23:37:50.789 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.789 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap": { 2024-09-12T23:37:50.789 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 227448, 2024-09-12T23:37:50.789 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 3437 2024-09-12T23:37:50.789 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.790 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap_mapping": { 2024-09-12T23:37:50.790 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.790 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.790 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.790 INFO:tasks.workunit.client.0.smithi136.stdout: "pgmap": { 2024-09-12T23:37:50.790 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.790 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.790 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.790 INFO:tasks.workunit.client.0.smithi136.stdout: "unittest_1": { 2024-09-12T23:37:50.790 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.790 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.790 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.791 INFO:tasks.workunit.client.0.smithi136.stdout: "unittest_2": { 2024-09-12T23:37:50.791 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.791 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.791 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.791 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.791 INFO:tasks.workunit.client.0.smithi136.stdout: "osd.5": { 2024-09-12T23:37:50.791 INFO:tasks.workunit.client.0.smithi136.stdout: "bloom_filter": { 2024-09-12T23:37:50.791 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.791 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.791 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.791 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs": { 2024-09-12T23:37:50.791 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 6304, 2024-09-12T23:37:50.792 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 115 2024-09-12T23:37:50.792 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.792 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs_file_reader": { 2024-09-12T23:37:50.792 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 16896, 2024-09-12T23:37:50.792 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 8 2024-09-12T23:37:50.792 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.792 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs_file_writer": { 2024-09-12T23:37:50.792 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 576, 2024-09-12T23:37:50.792 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 3 2024-09-12T23:37:50.792 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.792 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Blob": { 2024-09-12T23:37:50.792 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 18144, 2024-09-12T23:37:50.793 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 162 2024-09-12T23:37:50.793 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.793 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Buffer": { 2024-09-12T23:37:50.793 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 336, 2024-09-12T23:37:50.793 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 3 2024-09-12T23:37:50.793 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.793 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Extent": { 2024-09-12T23:37:50.793 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 7776, 2024-09-12T23:37:50.793 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 162 2024-09-12T23:37:50.793 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.793 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_SharedBlob": { 2024-09-12T23:37:50.793 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 18144, 2024-09-12T23:37:50.794 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 162 2024-09-12T23:37:50.794 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.794 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_alloc": { 2024-09-12T23:37:50.794 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 2800, 2024-09-12T23:37:50.794 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 35 2024-09-12T23:37:50.794 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.794 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_data": { 2024-09-12T23:37:50.794 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 4096, 2024-09-12T23:37:50.794 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 1 2024-09-12T23:37:50.794 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.794 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_meta": { 2024-09-12T23:37:50.794 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 47550, 2024-09-12T23:37:50.795 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 8640 2024-09-12T23:37:50.795 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.795 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_onode": { 2024-09-12T23:37:50.795 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 122400, 2024-09-12T23:37:50.795 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 180 2024-09-12T23:37:50.795 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.795 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_other": { 2024-09-12T23:37:50.795 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 9164, 2024-09-12T23:37:50.795 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 514 2024-09-12T23:37:50.795 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.795 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_fsck": { 2024-09-12T23:37:50.795 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.795 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.796 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.796 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_inline_bl": { 2024-09-12T23:37:50.796 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 26810, 2024-09-12T23:37:50.796 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 180 2024-09-12T23:37:50.796 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.796 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_txc": { 2024-09-12T23:37:50.796 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 2208, 2024-09-12T23:37:50.796 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 3 2024-09-12T23:37:50.796 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.796 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_writing": { 2024-09-12T23:37:50.797 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 540192, 2024-09-12T23:37:50.797 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 43 2024-09-12T23:37:50.797 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.797 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_writing_deferred": { 2024-09-12T23:37:50.797 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 30637, 2024-09-12T23:37:50.797 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 9 2024-09-12T23:37:50.797 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.797 INFO:tasks.workunit.client.0.smithi136.stdout: "buffer_anon": { 2024-09-12T23:37:50.797 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 2437972, 2024-09-12T23:37:50.797 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 184 2024-09-12T23:37:50.797 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.797 INFO:tasks.workunit.client.0.smithi136.stdout: "buffer_meta": { 2024-09-12T23:37:50.798 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 5368, 2024-09-12T23:37:50.798 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 61 2024-09-12T23:37:50.798 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.798 INFO:tasks.workunit.client.0.smithi136.stdout: "mds_co": { 2024-09-12T23:37:50.798 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.798 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.798 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.798 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:50.798 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 463360, 2024-09-12T23:37:50.798 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 40 2024-09-12T23:37:50.798 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.798 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mapbl": { 2024-09-12T23:37:50.799 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.799 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.799 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.799 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_pglog": { 2024-09-12T23:37:50.799 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 13000, 2024-09-12T23:37:50.799 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 25 2024-09-12T23:37:50.799 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.799 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap": { 2024-09-12T23:37:50.799 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 227960, 2024-09-12T23:37:50.799 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 3461 2024-09-12T23:37:50.799 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.800 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap_mapping": { 2024-09-12T23:37:50.800 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.800 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.800 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.800 INFO:tasks.workunit.client.0.smithi136.stdout: "pgmap": { 2024-09-12T23:37:50.800 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.800 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.800 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.800 INFO:tasks.workunit.client.0.smithi136.stdout: "unittest_1": { 2024-09-12T23:37:50.800 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.800 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.800 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.801 INFO:tasks.workunit.client.0.smithi136.stdout: "unittest_2": { 2024-09-12T23:37:50.801 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.801 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.801 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.801 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.801 INFO:tasks.workunit.client.0.smithi136.stdout: "osd.6": { 2024-09-12T23:37:50.801 INFO:tasks.workunit.client.0.smithi136.stdout: "bloom_filter": { 2024-09-12T23:37:50.801 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.801 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.801 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.801 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs": { 2024-09-12T23:37:50.801 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 6304, 2024-09-12T23:37:50.802 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 115 2024-09-12T23:37:50.802 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.802 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs_file_reader": { 2024-09-12T23:37:50.802 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 16896, 2024-09-12T23:37:50.802 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 8 2024-09-12T23:37:50.802 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.802 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs_file_writer": { 2024-09-12T23:37:50.802 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 576, 2024-09-12T23:37:50.802 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 3 2024-09-12T23:37:50.802 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.802 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Blob": { 2024-09-12T23:37:50.802 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 19152, 2024-09-12T23:37:50.803 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 171 2024-09-12T23:37:50.803 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.803 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Buffer": { 2024-09-12T23:37:50.803 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 448, 2024-09-12T23:37:50.803 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 4 2024-09-12T23:37:50.803 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.803 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Extent": { 2024-09-12T23:37:50.803 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 8208, 2024-09-12T23:37:50.803 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 171 2024-09-12T23:37:50.803 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.803 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_SharedBlob": { 2024-09-12T23:37:50.803 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 19152, 2024-09-12T23:37:50.804 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 171 2024-09-12T23:37:50.804 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.804 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_alloc": { 2024-09-12T23:37:50.804 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 2480, 2024-09-12T23:37:50.804 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 31 2024-09-12T23:37:50.804 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.804 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_data": { 2024-09-12T23:37:50.804 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 22733, 2024-09-12T23:37:50.804 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 5 2024-09-12T23:37:50.804 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.804 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_meta": { 2024-09-12T23:37:50.805 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 47611, 2024-09-12T23:37:50.805 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 8932 2024-09-12T23:37:50.805 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.805 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_onode": { 2024-09-12T23:37:50.805 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 121720, 2024-09-12T23:37:50.805 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 179 2024-09-12T23:37:50.805 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.805 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_other": { 2024-09-12T23:37:50.805 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 10300, 2024-09-12T23:37:50.805 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 672 2024-09-12T23:37:50.805 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.806 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_fsck": { 2024-09-12T23:37:50.806 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.806 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.806 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.806 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_inline_bl": { 2024-09-12T23:37:50.806 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 27582, 2024-09-12T23:37:50.806 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 179 2024-09-12T23:37:50.806 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.806 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_txc": { 2024-09-12T23:37:50.806 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.806 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.806 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.807 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_writing": { 2024-09-12T23:37:50.807 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 780888, 2024-09-12T23:37:50.807 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 39 2024-09-12T23:37:50.807 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.807 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_writing_deferred": { 2024-09-12T23:37:50.807 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.807 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.807 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.807 INFO:tasks.workunit.client.0.smithi136.stdout: "buffer_anon": { 2024-09-12T23:37:50.807 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 2394070, 2024-09-12T23:37:50.807 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 164 2024-09-12T23:37:50.808 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.808 INFO:tasks.workunit.client.0.smithi136.stdout: "buffer_meta": { 2024-09-12T23:37:50.808 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 5016, 2024-09-12T23:37:50.808 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 57 2024-09-12T23:37:50.808 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.808 INFO:tasks.workunit.client.0.smithi136.stdout: "mds_co": { 2024-09-12T23:37:50.808 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.808 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.808 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.808 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:50.808 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 451776, 2024-09-12T23:37:50.809 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 39 2024-09-12T23:37:50.809 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.809 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mapbl": { 2024-09-12T23:37:50.809 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.809 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.809 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.809 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_pglog": { 2024-09-12T23:37:50.809 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 75192, 2024-09-12T23:37:50.809 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 155 2024-09-12T23:37:50.809 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.810 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap": { 2024-09-12T23:37:50.810 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 227192, 2024-09-12T23:37:50.810 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 3425 2024-09-12T23:37:50.810 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.810 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap_mapping": { 2024-09-12T23:37:50.810 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.810 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.810 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.810 INFO:tasks.workunit.client.0.smithi136.stdout: "pgmap": { 2024-09-12T23:37:50.810 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.810 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.811 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.811 INFO:tasks.workunit.client.0.smithi136.stdout: "unittest_1": { 2024-09-12T23:37:50.811 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.811 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.811 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.811 INFO:tasks.workunit.client.0.smithi136.stdout: "unittest_2": { 2024-09-12T23:37:50.811 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.811 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.811 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.811 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.811 INFO:tasks.workunit.client.0.smithi136.stdout: "osd.7": { 2024-09-12T23:37:50.811 INFO:tasks.workunit.client.0.smithi136.stdout: "bloom_filter": { 2024-09-12T23:37:50.812 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.812 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.812 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.812 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs": { 2024-09-12T23:37:50.812 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 6304, 2024-09-12T23:37:50.812 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 115 2024-09-12T23:37:50.812 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.812 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs_file_reader": { 2024-09-12T23:37:50.812 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 16896, 2024-09-12T23:37:50.812 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 8 2024-09-12T23:37:50.812 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.813 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs_file_writer": { 2024-09-12T23:37:50.813 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 576, 2024-09-12T23:37:50.813 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 3 2024-09-12T23:37:50.813 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.813 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Blob": { 2024-09-12T23:37:50.813 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 19152, 2024-09-12T23:37:50.813 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 171 2024-09-12T23:37:50.813 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.813 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Buffer": { 2024-09-12T23:37:50.813 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 1344, 2024-09-12T23:37:50.813 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 12 2024-09-12T23:37:50.814 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.814 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_Extent": { 2024-09-12T23:37:50.814 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 8208, 2024-09-12T23:37:50.814 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 171 2024-09-12T23:37:50.814 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.814 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_SharedBlob": { 2024-09-12T23:37:50.814 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 19152, 2024-09-12T23:37:50.814 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 171 2024-09-12T23:37:50.814 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.814 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_alloc": { 2024-09-12T23:37:50.814 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 2000, 2024-09-12T23:37:50.814 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 25 2024-09-12T23:37:50.815 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.815 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_data": { 2024-09-12T23:37:50.815 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 297165, 2024-09-12T23:37:50.815 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 13 2024-09-12T23:37:50.815 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.815 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_meta": { 2024-09-12T23:37:50.815 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 51006, 2024-09-12T23:37:50.815 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 9370 2024-09-12T23:37:50.815 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.815 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_onode": { 2024-09-12T23:37:50.815 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 127840, 2024-09-12T23:37:50.816 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 188 2024-09-12T23:37:50.816 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.816 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_cache_other": { 2024-09-12T23:37:50.816 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 11084, 2024-09-12T23:37:50.816 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 679 2024-09-12T23:37:50.816 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.816 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_fsck": { 2024-09-12T23:37:50.816 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.816 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.816 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.816 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_inline_bl": { 2024-09-12T23:37:50.817 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 29886, 2024-09-12T23:37:50.817 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 188 2024-09-12T23:37:50.817 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.817 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_txc": { 2024-09-12T23:37:50.817 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.817 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.817 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.817 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_writing": { 2024-09-12T23:37:50.817 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 736834, 2024-09-12T23:37:50.817 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 29 2024-09-12T23:37:50.818 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.818 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore_writing_deferred": { 2024-09-12T23:37:50.818 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.818 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.818 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.818 INFO:tasks.workunit.client.0.smithi136.stdout: "buffer_anon": { 2024-09-12T23:37:50.818 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 2426906, 2024-09-12T23:37:50.818 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 219 2024-09-12T23:37:50.818 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.818 INFO:tasks.workunit.client.0.smithi136.stdout: "buffer_meta": { 2024-09-12T23:37:50.818 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 6512, 2024-09-12T23:37:50.819 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 74 2024-09-12T23:37:50.819 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.819 INFO:tasks.workunit.client.0.smithi136.stdout: "mds_co": { 2024-09-12T23:37:50.819 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.819 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.819 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.819 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:50.819 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 532864, 2024-09-12T23:37:50.819 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 46 2024-09-12T23:37:50.819 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.820 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_mapbl": { 2024-09-12T23:37:50.820 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.820 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.820 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.820 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_pglog": { 2024-09-12T23:37:50.820 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 77896, 2024-09-12T23:37:50.820 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 193 2024-09-12T23:37:50.820 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.820 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap": { 2024-09-12T23:37:50.820 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 227192, 2024-09-12T23:37:50.820 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 3425 2024-09-12T23:37:50.821 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.821 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap_mapping": { 2024-09-12T23:37:50.821 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.821 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.821 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.821 INFO:tasks.workunit.client.0.smithi136.stdout: "pgmap": { 2024-09-12T23:37:50.821 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.821 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.821 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.821 INFO:tasks.workunit.client.0.smithi136.stdout: "unittest_1": { 2024-09-12T23:37:50.821 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.822 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.822 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.822 INFO:tasks.workunit.client.0.smithi136.stdout: "unittest_2": { 2024-09-12T23:37:50.822 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes": 0, 2024-09-12T23:37:50.822 INFO:tasks.workunit.client.0.smithi136.stdout: "items": 0 2024-09-12T23:37:50.822 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.822 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.822 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.822 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.822 INFO:tasks.workunit.client.0.smithi136.stdout: "metadata": { 2024-09-12T23:37:50.822 INFO:tasks.workunit.client.0.smithi136.stdout: "mds": { 2024-09-12T23:37:50.822 INFO:tasks.workunit.client.0.smithi136.stdout: "arch": { 2024-09-12T23:37:50.823 INFO:tasks.workunit.client.0.smithi136.stdout: "x86_64": 2 2024-09-12T23:37:50.823 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.823 INFO:tasks.workunit.client.0.smithi136.stdout: "ceph_version": { 2024-09-12T23:37:50.823 INFO:tasks.workunit.client.0.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-12T23:37:50.823 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.823 INFO:tasks.workunit.client.0.smithi136.stdout: "cpu": { 2024-09-12T23:37:50.823 INFO:tasks.workunit.client.0.smithi136.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 2 2024-09-12T23:37:50.823 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.823 INFO:tasks.workunit.client.0.smithi136.stdout: "distro": { 2024-09-12T23:37:50.823 INFO:tasks.workunit.client.0.smithi136.stdout: "centos": 2 2024-09-12T23:37:50.823 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.824 INFO:tasks.workunit.client.0.smithi136.stdout: "distro_description": { 2024-09-12T23:37:50.824 INFO:tasks.workunit.client.0.smithi136.stdout: "CentOS Stream 9": 2 2024-09-12T23:37:50.824 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.824 INFO:tasks.workunit.client.0.smithi136.stdout: "kernel_description": { 2024-09-12T23:37:50.824 INFO:tasks.workunit.client.0.smithi136.stdout: "#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024": 2 2024-09-12T23:37:50.824 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.824 INFO:tasks.workunit.client.0.smithi136.stdout: "kernel_version": { 2024-09-12T23:37:50.824 INFO:tasks.workunit.client.0.smithi136.stdout: "5.15.0-121-generic": 2 2024-09-12T23:37:50.824 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.824 INFO:tasks.workunit.client.0.smithi136.stdout: "os": { 2024-09-12T23:37:50.824 INFO:tasks.workunit.client.0.smithi136.stdout: "Linux": 2 2024-09-12T23:37:50.824 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.825 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.825 INFO:tasks.workunit.client.0.smithi136.stdout: "mon": { 2024-09-12T23:37:50.825 INFO:tasks.workunit.client.0.smithi136.stdout: "arch": { 2024-09-12T23:37:50.825 INFO:tasks.workunit.client.0.smithi136.stdout: "x86_64": 3 2024-09-12T23:37:50.825 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.825 INFO:tasks.workunit.client.0.smithi136.stdout: "ceph_version": { 2024-09-12T23:37:50.825 INFO:tasks.workunit.client.0.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 3 2024-09-12T23:37:50.825 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.825 INFO:tasks.workunit.client.0.smithi136.stdout: "cpu": { 2024-09-12T23:37:50.825 INFO:tasks.workunit.client.0.smithi136.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 3 2024-09-12T23:37:50.825 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.825 INFO:tasks.workunit.client.0.smithi136.stdout: "distro": { 2024-09-12T23:37:50.826 INFO:tasks.workunit.client.0.smithi136.stdout: "centos": 3 2024-09-12T23:37:50.826 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.826 INFO:tasks.workunit.client.0.smithi136.stdout: "distro_description": { 2024-09-12T23:37:50.826 INFO:tasks.workunit.client.0.smithi136.stdout: "CentOS Stream 9": 3 2024-09-12T23:37:50.826 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.826 INFO:tasks.workunit.client.0.smithi136.stdout: "kernel_description": { 2024-09-12T23:37:50.826 INFO:tasks.workunit.client.0.smithi136.stdout: "#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024": 3 2024-09-12T23:37:50.826 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.826 INFO:tasks.workunit.client.0.smithi136.stdout: "kernel_version": { 2024-09-12T23:37:50.826 INFO:tasks.workunit.client.0.smithi136.stdout: "5.15.0-121-generic": 3 2024-09-12T23:37:50.826 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.827 INFO:tasks.workunit.client.0.smithi136.stdout: "os": { 2024-09-12T23:37:50.827 INFO:tasks.workunit.client.0.smithi136.stdout: "Linux": 3 2024-09-12T23:37:50.827 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.827 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.827 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:50.827 INFO:tasks.workunit.client.0.smithi136.stdout: "arch": { 2024-09-12T23:37:50.827 INFO:tasks.workunit.client.0.smithi136.stdout: "x86_64": 8 2024-09-12T23:37:50.827 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.827 INFO:tasks.workunit.client.0.smithi136.stdout: "ceph_version": { 2024-09-12T23:37:50.827 INFO:tasks.workunit.client.0.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 8 2024-09-12T23:37:50.827 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.827 INFO:tasks.workunit.client.0.smithi136.stdout: "cpu": { 2024-09-12T23:37:50.828 INFO:tasks.workunit.client.0.smithi136.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 8 2024-09-12T23:37:50.828 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.828 INFO:tasks.workunit.client.0.smithi136.stdout: "distro": { 2024-09-12T23:37:50.828 INFO:tasks.workunit.client.0.smithi136.stdout: "centos": 8 2024-09-12T23:37:50.828 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.828 INFO:tasks.workunit.client.0.smithi136.stdout: "distro_description": { 2024-09-12T23:37:50.828 INFO:tasks.workunit.client.0.smithi136.stdout: "CentOS Stream 9": 8 2024-09-12T23:37:50.828 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.828 INFO:tasks.workunit.client.0.smithi136.stdout: "kernel_description": { 2024-09-12T23:37:50.828 INFO:tasks.workunit.client.0.smithi136.stdout: "#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024": 8 2024-09-12T23:37:50.828 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.829 INFO:tasks.workunit.client.0.smithi136.stdout: "kernel_version": { 2024-09-12T23:37:50.829 INFO:tasks.workunit.client.0.smithi136.stdout: "5.15.0-121-generic": 8 2024-09-12T23:37:50.829 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.829 INFO:tasks.workunit.client.0.smithi136.stdout: "os": { 2024-09-12T23:37:50.829 INFO:tasks.workunit.client.0.smithi136.stdout: "Linux": 8 2024-09-12T23:37:50.829 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.829 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_objectstore": { 2024-09-12T23:37:50.829 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore": 8 2024-09-12T23:37:50.829 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.829 INFO:tasks.workunit.client.0.smithi136.stdout: "rotational": { 2024-09-12T23:37:50.829 INFO:tasks.workunit.client.0.smithi136.stdout: "0": 8 2024-09-12T23:37:50.829 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.830 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.830 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.830 INFO:tasks.workunit.client.0.smithi136.stdout: "mon": { 2024-09-12T23:37:50.830 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 3, 2024-09-12T23:37:50.830 INFO:tasks.workunit.client.0.smithi136.stdout: "features": { 2024-09-12T23:37:50.830 INFO:tasks.workunit.client.0.smithi136.stdout: "optional": [], 2024-09-12T23:37:50.830 INFO:tasks.workunit.client.0.smithi136.stdout: "persistent": [ 2024-09-12T23:37:50.830 INFO:tasks.workunit.client.0.smithi136.stdout: "kraken", 2024-09-12T23:37:50.830 INFO:tasks.workunit.client.0.smithi136.stdout: "luminous", 2024-09-12T23:37:50.830 INFO:tasks.workunit.client.0.smithi136.stdout: "mimic", 2024-09-12T23:37:50.830 INFO:tasks.workunit.client.0.smithi136.stdout: "osdmap-prune", 2024-09-12T23:37:50.830 INFO:tasks.workunit.client.0.smithi136.stdout: "nautilus", 2024-09-12T23:37:50.831 INFO:tasks.workunit.client.0.smithi136.stdout: "octopus", 2024-09-12T23:37:50.831 INFO:tasks.workunit.client.0.smithi136.stdout: "pacific", 2024-09-12T23:37:50.831 INFO:tasks.workunit.client.0.smithi136.stdout: "elector-pinging", 2024-09-12T23:37:50.831 INFO:tasks.workunit.client.0.smithi136.stdout: "quincy" 2024-09-12T23:37:50.831 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:50.831 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.831 INFO:tasks.workunit.client.0.smithi136.stdout: "ipv4_addr_mons": 6, 2024-09-12T23:37:50.831 INFO:tasks.workunit.client.0.smithi136.stdout: "ipv6_addr_mons": 0, 2024-09-12T23:37:50.831 INFO:tasks.workunit.client.0.smithi136.stdout: "min_mon_release": 17, 2024-09-12T23:37:50.831 INFO:tasks.workunit.client.0.smithi136.stdout: "v1_addr_mons": 3, 2024-09-12T23:37:50.831 INFO:tasks.workunit.client.0.smithi136.stdout: "v2_addr_mons": 3 2024-09-12T23:37:50.831 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.832 INFO:tasks.workunit.client.0.smithi136.stdout: "organization": "ceph-qa", 2024-09-12T23:37:50.832 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:50.832 INFO:tasks.workunit.client.0.smithi136.stdout: "cluster_network": false, 2024-09-12T23:37:50.832 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 8, 2024-09-12T23:37:50.832 INFO:tasks.workunit.client.0.smithi136.stdout: "require_min_compat_client": "luminous", 2024-09-12T23:37:50.832 INFO:tasks.workunit.client.0.smithi136.stdout: "require_osd_release": "quincy" 2024-09-12T23:37:50.832 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.832 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_perf_histograms": [ 2024-09-12T23:37:50.832 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.832 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-12T23:37:50.833 INFO:tasks.workunit.client.0.smithi136.stdout: "axes": [ 2024-09-12T23:37:50.833 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.833 INFO:tasks.workunit.client.0.smithi136.stdout: "buckets": 32, 2024-09-12T23:37:50.833 INFO:tasks.workunit.client.0.smithi136.stdout: "min": 0, 2024-09-12T23:37:50.833 INFO:tasks.workunit.client.0.smithi136.stdout: "name": "Latency (usec)", 2024-09-12T23:37:50.833 INFO:tasks.workunit.client.0.smithi136.stdout: "quant_size": 100000, 2024-09-12T23:37:50.833 INFO:tasks.workunit.client.0.smithi136.stdout: "ranges": [ 2024-09-12T23:37:50.833 INFO:tasks.workunit.client.0.smithi136.stdout: "[None, -1]", 2024-09-12T23:37:50.833 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 99999]", 2024-09-12T23:37:50.833 INFO:tasks.workunit.client.0.smithi136.stdout: "[100000, 199999]", 2024-09-12T23:37:50.833 INFO:tasks.workunit.client.0.smithi136.stdout: "[200000, 399999]", 2024-09-12T23:37:50.833 INFO:tasks.workunit.client.0.smithi136.stdout: "[400000, 799999]", 2024-09-12T23:37:50.834 INFO:tasks.workunit.client.0.smithi136.stdout: "[800000, 1599999]", 2024-09-12T23:37:50.834 INFO:tasks.workunit.client.0.smithi136.stdout: "[1600000, 3199999]", 2024-09-12T23:37:50.834 INFO:tasks.workunit.client.0.smithi136.stdout: "[3200000, 6399999]", 2024-09-12T23:37:50.834 INFO:tasks.workunit.client.0.smithi136.stdout: "[6400000, 12799999]", 2024-09-12T23:37:50.834 INFO:tasks.workunit.client.0.smithi136.stdout: "[12800000, 25599999]", 2024-09-12T23:37:50.834 INFO:tasks.workunit.client.0.smithi136.stdout: "[25600000, 51199999]", 2024-09-12T23:37:50.834 INFO:tasks.workunit.client.0.smithi136.stdout: "[51200000, 102399999]", 2024-09-12T23:37:50.834 INFO:tasks.workunit.client.0.smithi136.stdout: "[102400000, 204799999]", 2024-09-12T23:37:50.834 INFO:tasks.workunit.client.0.smithi136.stdout: "[204800000, 409599999]", 2024-09-12T23:37:50.834 INFO:tasks.workunit.client.0.smithi136.stdout: "[409600000, 819199999]", 2024-09-12T23:37:50.834 INFO:tasks.workunit.client.0.smithi136.stdout: "[819200000, 1638399999]", 2024-09-12T23:37:50.835 INFO:tasks.workunit.client.0.smithi136.stdout: "[1638400000, 3276799999]", 2024-09-12T23:37:50.835 INFO:tasks.workunit.client.0.smithi136.stdout: "[3276800000, 6553599999]", 2024-09-12T23:37:50.835 INFO:tasks.workunit.client.0.smithi136.stdout: "[6553600000, 13107199999]", 2024-09-12T23:37:50.835 INFO:tasks.workunit.client.0.smithi136.stdout: "[13107200000, 26214399999]", 2024-09-12T23:37:50.835 INFO:tasks.workunit.client.0.smithi136.stdout: "[26214400000, 52428799999]", 2024-09-12T23:37:50.835 INFO:tasks.workunit.client.0.smithi136.stdout: "[52428800000, 104857599999]", 2024-09-12T23:37:50.835 INFO:tasks.workunit.client.0.smithi136.stdout: "[104857600000, 209715199999]", 2024-09-12T23:37:50.835 INFO:tasks.workunit.client.0.smithi136.stdout: "[209715200000, 419430399999]", 2024-09-12T23:37:50.835 INFO:tasks.workunit.client.0.smithi136.stdout: "[419430400000, 838860799999]", 2024-09-12T23:37:50.835 INFO:tasks.workunit.client.0.smithi136.stdout: "[838860800000, 1677721599999]", 2024-09-12T23:37:50.835 INFO:tasks.workunit.client.0.smithi136.stdout: "[1677721600000, 3355443199999]", 2024-09-12T23:37:50.836 INFO:tasks.workunit.client.0.smithi136.stdout: "[3355443200000, 6710886399999]", 2024-09-12T23:37:50.836 INFO:tasks.workunit.client.0.smithi136.stdout: "[6710886400000, 13421772799999]", 2024-09-12T23:37:50.836 INFO:tasks.workunit.client.0.smithi136.stdout: "[13421772800000, 26843545599999]", 2024-09-12T23:37:50.836 INFO:tasks.workunit.client.0.smithi136.stdout: "[26843545600000, 53687091199999]", 2024-09-12T23:37:50.836 INFO:tasks.workunit.client.0.smithi136.stdout: "[53687091200000, None]" 2024-09-12T23:37:50.836 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.836 INFO:tasks.workunit.client.0.smithi136.stdout: "scale_type": "log2" 2024-09-12T23:37:50.836 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.836 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.836 INFO:tasks.workunit.client.0.smithi136.stdout: "buckets": 32, 2024-09-12T23:37:50.836 INFO:tasks.workunit.client.0.smithi136.stdout: "min": 0, 2024-09-12T23:37:50.836 INFO:tasks.workunit.client.0.smithi136.stdout: "name": "Request size (bytes)", 2024-09-12T23:37:50.837 INFO:tasks.workunit.client.0.smithi136.stdout: "quant_size": 512, 2024-09-12T23:37:50.837 INFO:tasks.workunit.client.0.smithi136.stdout: "ranges": [ 2024-09-12T23:37:50.837 INFO:tasks.workunit.client.0.smithi136.stdout: "[None, -1]", 2024-09-12T23:37:50.837 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 511]", 2024-09-12T23:37:50.837 INFO:tasks.workunit.client.0.smithi136.stdout: "[512, 1023]", 2024-09-12T23:37:50.837 INFO:tasks.workunit.client.0.smithi136.stdout: "[1024, 2047]", 2024-09-12T23:37:50.837 INFO:tasks.workunit.client.0.smithi136.stdout: "[2048, 4095]", 2024-09-12T23:37:50.837 INFO:tasks.workunit.client.0.smithi136.stdout: "[4096, 8191]", 2024-09-12T23:37:50.837 INFO:tasks.workunit.client.0.smithi136.stdout: "[8192, 16383]", 2024-09-12T23:37:50.837 INFO:tasks.workunit.client.0.smithi136.stdout: "[16384, 32767]", 2024-09-12T23:37:50.837 INFO:tasks.workunit.client.0.smithi136.stdout: "[32768, 65535]", 2024-09-12T23:37:50.837 INFO:tasks.workunit.client.0.smithi136.stdout: "[65536, 131071]", 2024-09-12T23:37:50.838 INFO:tasks.workunit.client.0.smithi136.stdout: "[131072, 262143]", 2024-09-12T23:37:50.838 INFO:tasks.workunit.client.0.smithi136.stdout: "[262144, 524287]", 2024-09-12T23:37:50.838 INFO:tasks.workunit.client.0.smithi136.stdout: "[524288, 1048575]", 2024-09-12T23:37:50.838 INFO:tasks.workunit.client.0.smithi136.stdout: "[1048576, 2097151]", 2024-09-12T23:37:50.838 INFO:tasks.workunit.client.0.smithi136.stdout: "[2097152, 4194303]", 2024-09-12T23:37:50.838 INFO:tasks.workunit.client.0.smithi136.stdout: "[4194304, 8388607]", 2024-09-12T23:37:50.838 INFO:tasks.workunit.client.0.smithi136.stdout: "[8388608, 16777215]", 2024-09-12T23:37:50.838 INFO:tasks.workunit.client.0.smithi136.stdout: "[16777216, 33554431]", 2024-09-12T23:37:50.838 INFO:tasks.workunit.client.0.smithi136.stdout: "[33554432, 67108863]", 2024-09-12T23:37:50.838 INFO:tasks.workunit.client.0.smithi136.stdout: "[67108864, 134217727]", 2024-09-12T23:37:50.838 INFO:tasks.workunit.client.0.smithi136.stdout: "[134217728, 268435455]", 2024-09-12T23:37:50.839 INFO:tasks.workunit.client.0.smithi136.stdout: "[268435456, 536870911]", 2024-09-12T23:37:50.839 INFO:tasks.workunit.client.0.smithi136.stdout: "[536870912, 1073741823]", 2024-09-12T23:37:50.839 INFO:tasks.workunit.client.0.smithi136.stdout: "[1073741824, 2147483647]", 2024-09-12T23:37:50.839 INFO:tasks.workunit.client.0.smithi136.stdout: "[2147483648, 4294967295]", 2024-09-12T23:37:50.839 INFO:tasks.workunit.client.0.smithi136.stdout: "[4294967296, 8589934591]", 2024-09-12T23:37:50.839 INFO:tasks.workunit.client.0.smithi136.stdout: "[8589934592, 17179869183]", 2024-09-12T23:37:50.839 INFO:tasks.workunit.client.0.smithi136.stdout: "[17179869184, 34359738367]", 2024-09-12T23:37:50.839 INFO:tasks.workunit.client.0.smithi136.stdout: "[34359738368, 68719476735]", 2024-09-12T23:37:50.839 INFO:tasks.workunit.client.0.smithi136.stdout: "[68719476736, 137438953471]", 2024-09-12T23:37:50.839 INFO:tasks.workunit.client.0.smithi136.stdout: "[137438953472, 274877906943]", 2024-09-12T23:37:50.839 INFO:tasks.workunit.client.0.smithi136.stdout: "[274877906944, None]" 2024-09-12T23:37:50.839 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.840 INFO:tasks.workunit.client.0.smithi136.stdout: "scale_type": "log2" 2024-09-12T23:37:50.840 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.840 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.840 INFO:tasks.workunit.client.0.smithi136.stdout: "osds": [ 2024-09-12T23:37:50.840 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.840 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 0, 2024-09-12T23:37:50.840 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:50.840 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.840 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.840 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.841 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.841 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.841 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.841 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.841 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.841 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.841 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.841 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.841 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.841 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.841 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.842 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.842 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.842 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.842 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.842 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.842 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.842 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.842 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.842 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.842 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.842 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.843 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.843 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.843 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.843 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.843 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.843 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.843 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:50.843 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:50.843 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.843 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.843 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 1, 2024-09-12T23:37:50.843 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:50.844 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.844 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.844 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.844 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.844 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.844 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.844 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.844 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.844 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.844 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.844 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.844 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.845 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.845 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.845 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.845 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.845 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.845 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.845 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.845 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.845 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.846 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.846 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.846 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.846 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.846 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.846 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.846 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.846 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.846 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.846 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.846 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:50.847 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:50.847 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.847 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.847 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 2, 2024-09-12T23:37:50.847 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:50.847 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.847 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.847 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.847 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.847 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.847 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.848 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.848 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.848 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.848 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.848 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.848 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.848 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.848 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.848 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.848 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.849 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.849 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.849 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.849 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.849 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.849 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.849 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.849 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.849 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.849 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.849 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.849 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.850 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.850 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.850 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.850 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:50.850 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:50.850 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.850 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.850 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 3, 2024-09-12T23:37:50.850 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:50.850 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.851 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.851 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.851 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.851 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.851 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.851 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.851 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.851 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.851 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.851 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.851 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.851 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.852 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.852 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.852 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.852 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.852 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.852 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.852 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.852 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.852 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.852 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.852 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.853 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.853 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.853 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.853 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.853 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.853 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.853 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.853 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:50.853 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:50.853 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.853 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.854 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 4, 2024-09-12T23:37:50.854 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:50.854 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.854 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.854 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.854 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.854 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.854 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.854 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.854 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.854 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.855 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.855 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.855 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.855 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.855 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.855 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.855 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.855 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.855 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.855 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.855 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.855 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.856 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.856 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.856 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.856 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.856 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.856 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.856 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.856 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.856 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.856 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.856 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:50.857 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:50.857 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.857 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.857 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 5, 2024-09-12T23:37:50.857 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:50.857 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.857 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.857 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.857 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.857 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.857 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.858 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.858 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.858 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.858 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.858 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.858 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.858 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.858 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.858 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.858 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.858 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.859 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.859 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.859 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.859 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.859 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.859 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.859 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.859 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.859 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.859 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.859 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.859 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.860 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.860 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.860 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:50.860 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:50.860 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.860 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.860 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 6, 2024-09-12T23:37:50.860 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:50.860 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.860 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.860 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.861 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.861 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.861 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.861 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.861 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.861 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.861 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.861 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.861 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.861 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.861 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.862 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.862 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.862 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.862 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.862 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.862 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.862 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.862 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.862 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.862 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.862 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.863 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.863 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.863 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.863 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.863 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.863 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.863 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:50.863 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:50.863 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.863 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.863 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 7, 2024-09-12T23:37:50.864 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:50.864 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.864 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.864 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.864 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.864 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 51, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.864 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.864 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.864 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.864 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.865 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.865 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.865 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.865 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.865 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.865 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.865 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.865 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.865 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.865 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.865 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.866 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.866 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.866 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.866 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.866 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.866 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.866 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.866 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.866 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.866 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.866 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.867 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:50.867 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:50.867 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.867 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:50.867 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.867 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-12T23:37:50.867 INFO:tasks.workunit.client.0.smithi136.stdout: "axes": [ 2024-09-12T23:37:50.867 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.867 INFO:tasks.workunit.client.0.smithi136.stdout: "buckets": 32, 2024-09-12T23:37:50.867 INFO:tasks.workunit.client.0.smithi136.stdout: "min": 0, 2024-09-12T23:37:50.867 INFO:tasks.workunit.client.0.smithi136.stdout: "name": "Latency (usec)", 2024-09-12T23:37:50.868 INFO:tasks.workunit.client.0.smithi136.stdout: "quant_size": 100000, 2024-09-12T23:37:50.868 INFO:tasks.workunit.client.0.smithi136.stdout: "ranges": [ 2024-09-12T23:37:50.868 INFO:tasks.workunit.client.0.smithi136.stdout: "[None, -1]", 2024-09-12T23:37:50.868 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 99999]", 2024-09-12T23:37:50.868 INFO:tasks.workunit.client.0.smithi136.stdout: "[100000, 199999]", 2024-09-12T23:37:50.868 INFO:tasks.workunit.client.0.smithi136.stdout: "[200000, 399999]", 2024-09-12T23:37:50.868 INFO:tasks.workunit.client.0.smithi136.stdout: "[400000, 799999]", 2024-09-12T23:37:50.868 INFO:tasks.workunit.client.0.smithi136.stdout: "[800000, 1599999]", 2024-09-12T23:37:50.868 INFO:tasks.workunit.client.0.smithi136.stdout: "[1600000, 3199999]", 2024-09-12T23:37:50.868 INFO:tasks.workunit.client.0.smithi136.stdout: "[3200000, 6399999]", 2024-09-12T23:37:50.868 INFO:tasks.workunit.client.0.smithi136.stdout: "[6400000, 12799999]", 2024-09-12T23:37:50.868 INFO:tasks.workunit.client.0.smithi136.stdout: "[12800000, 25599999]", 2024-09-12T23:37:50.869 INFO:tasks.workunit.client.0.smithi136.stdout: "[25600000, 51199999]", 2024-09-12T23:37:50.869 INFO:tasks.workunit.client.0.smithi136.stdout: "[51200000, 102399999]", 2024-09-12T23:37:50.869 INFO:tasks.workunit.client.0.smithi136.stdout: "[102400000, 204799999]", 2024-09-12T23:37:50.869 INFO:tasks.workunit.client.0.smithi136.stdout: "[204800000, 409599999]", 2024-09-12T23:37:50.869 INFO:tasks.workunit.client.0.smithi136.stdout: "[409600000, 819199999]", 2024-09-12T23:37:50.869 INFO:tasks.workunit.client.0.smithi136.stdout: "[819200000, 1638399999]", 2024-09-12T23:37:50.869 INFO:tasks.workunit.client.0.smithi136.stdout: "[1638400000, 3276799999]", 2024-09-12T23:37:50.869 INFO:tasks.workunit.client.0.smithi136.stdout: "[3276800000, 6553599999]", 2024-09-12T23:37:50.869 INFO:tasks.workunit.client.0.smithi136.stdout: "[6553600000, 13107199999]", 2024-09-12T23:37:50.869 INFO:tasks.workunit.client.0.smithi136.stdout: "[13107200000, 26214399999]", 2024-09-12T23:37:50.869 INFO:tasks.workunit.client.0.smithi136.stdout: "[26214400000, 52428799999]", 2024-09-12T23:37:50.870 INFO:tasks.workunit.client.0.smithi136.stdout: "[52428800000, 104857599999]", 2024-09-12T23:37:50.870 INFO:tasks.workunit.client.0.smithi136.stdout: "[104857600000, 209715199999]", 2024-09-12T23:37:50.870 INFO:tasks.workunit.client.0.smithi136.stdout: "[209715200000, 419430399999]", 2024-09-12T23:37:50.870 INFO:tasks.workunit.client.0.smithi136.stdout: "[419430400000, 838860799999]", 2024-09-12T23:37:50.870 INFO:tasks.workunit.client.0.smithi136.stdout: "[838860800000, 1677721599999]", 2024-09-12T23:37:50.870 INFO:tasks.workunit.client.0.smithi136.stdout: "[1677721600000, 3355443199999]", 2024-09-12T23:37:50.870 INFO:tasks.workunit.client.0.smithi136.stdout: "[3355443200000, 6710886399999]", 2024-09-12T23:37:50.870 INFO:tasks.workunit.client.0.smithi136.stdout: "[6710886400000, 13421772799999]", 2024-09-12T23:37:50.870 INFO:tasks.workunit.client.0.smithi136.stdout: "[13421772800000, 26843545599999]", 2024-09-12T23:37:50.870 INFO:tasks.workunit.client.0.smithi136.stdout: "[26843545600000, 53687091199999]", 2024-09-12T23:37:50.870 INFO:tasks.workunit.client.0.smithi136.stdout: "[53687091200000, None]" 2024-09-12T23:37:50.871 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.871 INFO:tasks.workunit.client.0.smithi136.stdout: "scale_type": "log2" 2024-09-12T23:37:50.871 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.871 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.871 INFO:tasks.workunit.client.0.smithi136.stdout: "buckets": 32, 2024-09-12T23:37:50.871 INFO:tasks.workunit.client.0.smithi136.stdout: "min": 0, 2024-09-12T23:37:50.871 INFO:tasks.workunit.client.0.smithi136.stdout: "name": "Request size (bytes)", 2024-09-12T23:37:50.871 INFO:tasks.workunit.client.0.smithi136.stdout: "quant_size": 512, 2024-09-12T23:37:50.871 INFO:tasks.workunit.client.0.smithi136.stdout: "ranges": [ 2024-09-12T23:37:50.871 INFO:tasks.workunit.client.0.smithi136.stdout: "[None, -1]", 2024-09-12T23:37:50.871 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 511]", 2024-09-12T23:37:50.872 INFO:tasks.workunit.client.0.smithi136.stdout: "[512, 1023]", 2024-09-12T23:37:50.872 INFO:tasks.workunit.client.0.smithi136.stdout: "[1024, 2047]", 2024-09-12T23:37:50.872 INFO:tasks.workunit.client.0.smithi136.stdout: "[2048, 4095]", 2024-09-12T23:37:50.872 INFO:tasks.workunit.client.0.smithi136.stdout: "[4096, 8191]", 2024-09-12T23:37:50.872 INFO:tasks.workunit.client.0.smithi136.stdout: "[8192, 16383]", 2024-09-12T23:37:50.872 INFO:tasks.workunit.client.0.smithi136.stdout: "[16384, 32767]", 2024-09-12T23:37:50.872 INFO:tasks.workunit.client.0.smithi136.stdout: "[32768, 65535]", 2024-09-12T23:37:50.872 INFO:tasks.workunit.client.0.smithi136.stdout: "[65536, 131071]", 2024-09-12T23:37:50.872 INFO:tasks.workunit.client.0.smithi136.stdout: "[131072, 262143]", 2024-09-12T23:37:50.872 INFO:tasks.workunit.client.0.smithi136.stdout: "[262144, 524287]", 2024-09-12T23:37:50.872 INFO:tasks.workunit.client.0.smithi136.stdout: "[524288, 1048575]", 2024-09-12T23:37:50.873 INFO:tasks.workunit.client.0.smithi136.stdout: "[1048576, 2097151]", 2024-09-12T23:37:50.873 INFO:tasks.workunit.client.0.smithi136.stdout: "[2097152, 4194303]", 2024-09-12T23:37:50.873 INFO:tasks.workunit.client.0.smithi136.stdout: "[4194304, 8388607]", 2024-09-12T23:37:50.873 INFO:tasks.workunit.client.0.smithi136.stdout: "[8388608, 16777215]", 2024-09-12T23:37:50.873 INFO:tasks.workunit.client.0.smithi136.stdout: "[16777216, 33554431]", 2024-09-12T23:37:50.873 INFO:tasks.workunit.client.0.smithi136.stdout: "[33554432, 67108863]", 2024-09-12T23:37:50.873 INFO:tasks.workunit.client.0.smithi136.stdout: "[67108864, 134217727]", 2024-09-12T23:37:50.873 INFO:tasks.workunit.client.0.smithi136.stdout: "[134217728, 268435455]", 2024-09-12T23:37:50.873 INFO:tasks.workunit.client.0.smithi136.stdout: "[268435456, 536870911]", 2024-09-12T23:37:50.873 INFO:tasks.workunit.client.0.smithi136.stdout: "[536870912, 1073741823]", 2024-09-12T23:37:50.873 INFO:tasks.workunit.client.0.smithi136.stdout: "[1073741824, 2147483647]", 2024-09-12T23:37:50.874 INFO:tasks.workunit.client.0.smithi136.stdout: "[2147483648, 4294967295]", 2024-09-12T23:37:50.874 INFO:tasks.workunit.client.0.smithi136.stdout: "[4294967296, 8589934591]", 2024-09-12T23:37:50.874 INFO:tasks.workunit.client.0.smithi136.stdout: "[8589934592, 17179869183]", 2024-09-12T23:37:50.874 INFO:tasks.workunit.client.0.smithi136.stdout: "[17179869184, 34359738367]", 2024-09-12T23:37:50.874 INFO:tasks.workunit.client.0.smithi136.stdout: "[34359738368, 68719476735]", 2024-09-12T23:37:50.874 INFO:tasks.workunit.client.0.smithi136.stdout: "[68719476736, 137438953471]", 2024-09-12T23:37:50.874 INFO:tasks.workunit.client.0.smithi136.stdout: "[137438953472, 274877906943]", 2024-09-12T23:37:50.874 INFO:tasks.workunit.client.0.smithi136.stdout: "[274877906944, None]" 2024-09-12T23:37:50.874 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.874 INFO:tasks.workunit.client.0.smithi136.stdout: "scale_type": "log2" 2024-09-12T23:37:50.874 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.875 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.875 INFO:tasks.workunit.client.0.smithi136.stdout: "osds": [ 2024-09-12T23:37:50.875 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.875 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 0, 2024-09-12T23:37:50.875 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:50.875 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.875 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.875 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.875 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.875 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.875 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.876 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.876 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.876 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.876 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.876 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.876 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.876 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.876 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.876 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.876 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.876 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.876 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.877 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.877 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.877 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.877 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.877 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.877 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.877 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.877 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.877 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.877 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.877 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.878 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.878 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.878 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:50.878 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:50.878 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.878 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.878 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 1, 2024-09-12T23:37:50.878 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:50.878 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.878 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.879 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.879 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.879 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.879 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.879 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.879 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.879 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.879 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.879 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.879 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.879 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.880 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.880 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.880 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.880 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.880 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.880 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.880 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.880 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.880 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.880 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.880 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.881 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.881 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.881 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.881 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.881 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.881 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.881 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.881 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:50.881 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:50.881 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.882 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.882 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 2, 2024-09-12T23:37:50.882 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:50.882 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.882 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.882 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.882 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.882 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.882 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.882 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.882 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.883 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.883 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.883 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.883 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.883 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.883 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.883 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.883 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.883 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.883 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.883 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.884 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.884 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.884 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.884 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.884 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.884 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.884 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.884 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.884 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.884 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.884 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.885 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.885 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:50.885 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:50.885 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.885 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.885 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 3, 2024-09-12T23:37:50.885 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:50.885 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.885 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.885 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.885 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.886 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.886 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.886 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.886 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.886 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.886 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.886 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.886 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.886 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.886 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.886 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.887 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.887 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.887 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.887 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.887 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.887 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.887 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.887 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.887 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.887 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.887 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.888 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.888 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.888 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.888 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.888 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.888 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:50.888 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:50.888 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.888 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.888 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 4, 2024-09-12T23:37:50.888 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:50.889 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.889 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.889 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.889 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.889 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.889 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.889 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.889 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.889 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.889 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.889 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.889 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.890 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.890 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.890 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.890 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.890 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.890 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.890 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.890 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.890 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.890 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.890 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.890 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.891 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.891 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.891 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.891 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.891 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.891 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.891 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.891 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:50.891 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:50.891 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.891 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.891 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 5, 2024-09-12T23:37:50.892 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:50.892 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.892 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.892 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.892 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.892 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.892 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.892 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.892 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.892 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.892 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.892 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.892 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.893 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.893 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.893 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.893 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.893 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.893 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.893 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.893 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.893 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.893 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.893 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.894 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.894 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.894 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.894 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.894 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.894 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.894 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.894 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.894 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:50.894 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:50.894 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.894 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.895 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 6, 2024-09-12T23:37:50.895 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:50.895 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.895 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.895 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.895 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.895 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.895 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.895 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.895 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.895 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.896 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.896 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.896 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.896 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.896 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.896 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.896 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.896 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.896 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.896 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.896 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.897 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.897 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.897 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.897 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.897 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.897 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.897 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.897 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.897 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.897 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.897 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.897 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:50.898 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:50.898 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.898 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.898 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 7, 2024-09-12T23:37:50.898 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:50.898 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.898 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.898 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.898 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.898 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.898 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.898 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.899 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.899 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.899 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.899 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.899 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.899 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.899 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.899 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.899 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.899 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.900 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.900 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.900 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.900 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.900 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.900 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.900 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.900 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.900 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.900 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.900 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.901 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.901 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.901 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.901 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:50.901 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:50.901 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.901 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:50.901 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.901 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-12T23:37:50.901 INFO:tasks.workunit.client.0.smithi136.stdout: "axes": [ 2024-09-12T23:37:50.902 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.902 INFO:tasks.workunit.client.0.smithi136.stdout: "buckets": 32, 2024-09-12T23:37:50.902 INFO:tasks.workunit.client.0.smithi136.stdout: "min": 0, 2024-09-12T23:37:50.902 INFO:tasks.workunit.client.0.smithi136.stdout: "name": "Latency (usec)", 2024-09-12T23:37:50.902 INFO:tasks.workunit.client.0.smithi136.stdout: "quant_size": 100000, 2024-09-12T23:37:50.902 INFO:tasks.workunit.client.0.smithi136.stdout: "ranges": [ 2024-09-12T23:37:50.902 INFO:tasks.workunit.client.0.smithi136.stdout: "[None, -1]", 2024-09-12T23:37:50.902 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 99999]", 2024-09-12T23:37:50.902 INFO:tasks.workunit.client.0.smithi136.stdout: "[100000, 199999]", 2024-09-12T23:37:50.902 INFO:tasks.workunit.client.0.smithi136.stdout: "[200000, 399999]", 2024-09-12T23:37:50.902 INFO:tasks.workunit.client.0.smithi136.stdout: "[400000, 799999]", 2024-09-12T23:37:50.902 INFO:tasks.workunit.client.0.smithi136.stdout: "[800000, 1599999]", 2024-09-12T23:37:50.903 INFO:tasks.workunit.client.0.smithi136.stdout: "[1600000, 3199999]", 2024-09-12T23:37:50.903 INFO:tasks.workunit.client.0.smithi136.stdout: "[3200000, 6399999]", 2024-09-12T23:37:50.903 INFO:tasks.workunit.client.0.smithi136.stdout: "[6400000, 12799999]", 2024-09-12T23:37:50.903 INFO:tasks.workunit.client.0.smithi136.stdout: "[12800000, 25599999]", 2024-09-12T23:37:50.903 INFO:tasks.workunit.client.0.smithi136.stdout: "[25600000, 51199999]", 2024-09-12T23:37:50.903 INFO:tasks.workunit.client.0.smithi136.stdout: "[51200000, 102399999]", 2024-09-12T23:37:50.903 INFO:tasks.workunit.client.0.smithi136.stdout: "[102400000, 204799999]", 2024-09-12T23:37:50.903 INFO:tasks.workunit.client.0.smithi136.stdout: "[204800000, 409599999]", 2024-09-12T23:37:50.903 INFO:tasks.workunit.client.0.smithi136.stdout: "[409600000, 819199999]", 2024-09-12T23:37:50.903 INFO:tasks.workunit.client.0.smithi136.stdout: "[819200000, 1638399999]", 2024-09-12T23:37:50.903 INFO:tasks.workunit.client.0.smithi136.stdout: "[1638400000, 3276799999]", 2024-09-12T23:37:50.903 INFO:tasks.workunit.client.0.smithi136.stdout: "[3276800000, 6553599999]", 2024-09-12T23:37:50.904 INFO:tasks.workunit.client.0.smithi136.stdout: "[6553600000, 13107199999]", 2024-09-12T23:37:50.904 INFO:tasks.workunit.client.0.smithi136.stdout: "[13107200000, 26214399999]", 2024-09-12T23:37:50.904 INFO:tasks.workunit.client.0.smithi136.stdout: "[26214400000, 52428799999]", 2024-09-12T23:37:50.904 INFO:tasks.workunit.client.0.smithi136.stdout: "[52428800000, 104857599999]", 2024-09-12T23:37:50.904 INFO:tasks.workunit.client.0.smithi136.stdout: "[104857600000, 209715199999]", 2024-09-12T23:37:50.904 INFO:tasks.workunit.client.0.smithi136.stdout: "[209715200000, 419430399999]", 2024-09-12T23:37:50.904 INFO:tasks.workunit.client.0.smithi136.stdout: "[419430400000, 838860799999]", 2024-09-12T23:37:50.904 INFO:tasks.workunit.client.0.smithi136.stdout: "[838860800000, 1677721599999]", 2024-09-12T23:37:50.904 INFO:tasks.workunit.client.0.smithi136.stdout: "[1677721600000, 3355443199999]", 2024-09-12T23:37:50.904 INFO:tasks.workunit.client.0.smithi136.stdout: "[3355443200000, 6710886399999]", 2024-09-12T23:37:50.904 INFO:tasks.workunit.client.0.smithi136.stdout: "[6710886400000, 13421772799999]", 2024-09-12T23:37:50.905 INFO:tasks.workunit.client.0.smithi136.stdout: "[13421772800000, 26843545599999]", 2024-09-12T23:37:50.905 INFO:tasks.workunit.client.0.smithi136.stdout: "[26843545600000, 53687091199999]", 2024-09-12T23:37:50.905 INFO:tasks.workunit.client.0.smithi136.stdout: "[53687091200000, None]" 2024-09-12T23:37:50.905 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.905 INFO:tasks.workunit.client.0.smithi136.stdout: "scale_type": "log2" 2024-09-12T23:37:50.905 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.905 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.905 INFO:tasks.workunit.client.0.smithi136.stdout: "buckets": 32, 2024-09-12T23:37:50.905 INFO:tasks.workunit.client.0.smithi136.stdout: "min": 0, 2024-09-12T23:37:50.905 INFO:tasks.workunit.client.0.smithi136.stdout: "name": "Request size (bytes)", 2024-09-12T23:37:50.905 INFO:tasks.workunit.client.0.smithi136.stdout: "quant_size": 512, 2024-09-12T23:37:50.906 INFO:tasks.workunit.client.0.smithi136.stdout: "ranges": [ 2024-09-12T23:37:50.906 INFO:tasks.workunit.client.0.smithi136.stdout: "[None, -1]", 2024-09-12T23:37:50.906 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 511]", 2024-09-12T23:37:50.906 INFO:tasks.workunit.client.0.smithi136.stdout: "[512, 1023]", 2024-09-12T23:37:50.906 INFO:tasks.workunit.client.0.smithi136.stdout: "[1024, 2047]", 2024-09-12T23:37:50.906 INFO:tasks.workunit.client.0.smithi136.stdout: "[2048, 4095]", 2024-09-12T23:37:50.906 INFO:tasks.workunit.client.0.smithi136.stdout: "[4096, 8191]", 2024-09-12T23:37:50.906 INFO:tasks.workunit.client.0.smithi136.stdout: "[8192, 16383]", 2024-09-12T23:37:50.906 INFO:tasks.workunit.client.0.smithi136.stdout: "[16384, 32767]", 2024-09-12T23:37:50.906 INFO:tasks.workunit.client.0.smithi136.stdout: "[32768, 65535]", 2024-09-12T23:37:50.906 INFO:tasks.workunit.client.0.smithi136.stdout: "[65536, 131071]", 2024-09-12T23:37:50.907 INFO:tasks.workunit.client.0.smithi136.stdout: "[131072, 262143]", 2024-09-12T23:37:50.907 INFO:tasks.workunit.client.0.smithi136.stdout: "[262144, 524287]", 2024-09-12T23:37:50.907 INFO:tasks.workunit.client.0.smithi136.stdout: "[524288, 1048575]", 2024-09-12T23:37:50.907 INFO:tasks.workunit.client.0.smithi136.stdout: "[1048576, 2097151]", 2024-09-12T23:37:50.907 INFO:tasks.workunit.client.0.smithi136.stdout: "[2097152, 4194303]", 2024-09-12T23:37:50.907 INFO:tasks.workunit.client.0.smithi136.stdout: "[4194304, 8388607]", 2024-09-12T23:37:50.907 INFO:tasks.workunit.client.0.smithi136.stdout: "[8388608, 16777215]", 2024-09-12T23:37:50.907 INFO:tasks.workunit.client.0.smithi136.stdout: "[16777216, 33554431]", 2024-09-12T23:37:50.907 INFO:tasks.workunit.client.0.smithi136.stdout: "[33554432, 67108863]", 2024-09-12T23:37:50.907 INFO:tasks.workunit.client.0.smithi136.stdout: "[67108864, 134217727]", 2024-09-12T23:37:50.907 INFO:tasks.workunit.client.0.smithi136.stdout: "[134217728, 268435455]", 2024-09-12T23:37:50.907 INFO:tasks.workunit.client.0.smithi136.stdout: "[268435456, 536870911]", 2024-09-12T23:37:50.908 INFO:tasks.workunit.client.0.smithi136.stdout: "[536870912, 1073741823]", 2024-09-12T23:37:50.908 INFO:tasks.workunit.client.0.smithi136.stdout: "[1073741824, 2147483647]", 2024-09-12T23:37:50.908 INFO:tasks.workunit.client.0.smithi136.stdout: "[2147483648, 4294967295]", 2024-09-12T23:37:50.908 INFO:tasks.workunit.client.0.smithi136.stdout: "[4294967296, 8589934591]", 2024-09-12T23:37:50.908 INFO:tasks.workunit.client.0.smithi136.stdout: "[8589934592, 17179869183]", 2024-09-12T23:37:50.908 INFO:tasks.workunit.client.0.smithi136.stdout: "[17179869184, 34359738367]", 2024-09-12T23:37:50.908 INFO:tasks.workunit.client.0.smithi136.stdout: "[34359738368, 68719476735]", 2024-09-12T23:37:50.908 INFO:tasks.workunit.client.0.smithi136.stdout: "[68719476736, 137438953471]", 2024-09-12T23:37:50.908 INFO:tasks.workunit.client.0.smithi136.stdout: "[137438953472, 274877906943]", 2024-09-12T23:37:50.908 INFO:tasks.workunit.client.0.smithi136.stdout: "[274877906944, None]" 2024-09-12T23:37:50.908 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.909 INFO:tasks.workunit.client.0.smithi136.stdout: "scale_type": "log2" 2024-09-12T23:37:50.909 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.909 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.909 INFO:tasks.workunit.client.0.smithi136.stdout: "osds": [ 2024-09-12T23:37:50.909 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.909 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 0, 2024-09-12T23:37:50.909 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:50.909 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.909 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.909 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.909 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.910 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.910 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.910 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.910 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.910 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.910 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.910 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.910 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.910 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.910 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.911 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.911 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.911 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.911 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.911 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.911 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.911 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.911 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.911 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.911 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.911 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.912 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.912 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.912 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.912 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.912 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.912 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.912 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:50.912 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:50.912 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.912 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.912 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 1, 2024-09-12T23:37:50.913 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:50.913 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.913 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.913 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.913 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.913 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.913 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.913 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.913 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.913 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.913 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.913 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.914 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.914 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.914 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.914 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.914 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.914 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.914 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.914 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.914 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.914 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.914 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.915 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.915 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.915 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.915 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.915 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.915 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.915 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.915 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.915 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.915 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:50.915 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:50.915 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.916 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.916 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 2, 2024-09-12T23:37:50.916 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:50.916 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.916 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.916 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.916 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.916 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.916 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.916 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.917 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.917 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.917 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.917 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.917 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.917 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.917 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.917 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.917 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.917 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.917 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.918 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.918 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.918 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.918 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.918 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.918 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.918 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.918 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.918 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.918 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.918 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.919 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.919 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.919 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:50.919 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:50.919 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.919 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.919 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 3, 2024-09-12T23:37:50.919 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:50.919 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.919 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.919 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.920 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.920 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.920 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.920 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.920 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.920 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.920 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.920 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.920 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.920 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.920 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.921 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.921 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.921 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.921 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.921 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.921 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.921 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.921 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.921 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.921 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.921 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.921 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.922 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.922 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.922 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.922 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.922 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.922 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:50.922 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:50.922 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.922 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.922 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 4, 2024-09-12T23:37:50.922 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:50.923 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.923 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.923 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.923 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.923 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.923 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.923 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.923 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.923 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.923 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.923 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.924 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.924 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.924 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.924 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.924 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.924 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.924 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.924 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.924 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.924 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.924 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.925 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.925 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.925 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.925 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.925 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.925 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.925 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.925 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.925 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.925 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:50.925 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:50.925 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.926 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.926 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 5, 2024-09-12T23:37:50.926 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:50.926 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.926 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.926 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.926 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.926 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.926 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.926 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.926 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.926 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.927 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.927 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.927 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.927 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.927 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.927 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.927 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.927 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.927 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.927 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.927 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.928 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.928 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.928 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.928 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.928 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.928 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.928 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.928 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.928 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.928 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.928 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.929 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:50.929 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:50.929 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.929 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.929 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 6, 2024-09-12T23:37:50.929 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:50.929 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.929 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.929 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.929 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.929 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.930 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.930 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.930 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.930 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.930 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.930 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.930 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.930 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.930 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.930 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.930 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.931 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.932 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.932 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.932 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.932 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.932 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.932 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.932 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.932 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.932 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.932 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.933 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.933 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.933 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.933 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.933 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:50.933 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:50.933 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.933 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.933 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 7, 2024-09-12T23:37:50.933 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:50.933 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.933 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.934 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.934 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.934 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.934 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.934 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.934 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.934 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.934 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.934 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.934 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.934 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.934 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.935 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.935 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.935 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.935 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.935 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.935 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.935 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.935 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.935 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.935 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.935 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.936 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.936 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.936 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.936 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.936 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.936 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.936 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:50.936 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:50.936 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.936 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:50.936 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.937 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-12T23:37:50.937 INFO:tasks.workunit.client.0.smithi136.stdout: "axes": [ 2024-09-12T23:37:50.937 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.937 INFO:tasks.workunit.client.0.smithi136.stdout: "buckets": 32, 2024-09-12T23:37:50.937 INFO:tasks.workunit.client.0.smithi136.stdout: "min": 0, 2024-09-12T23:37:50.937 INFO:tasks.workunit.client.0.smithi136.stdout: "name": "Latency (usec)", 2024-09-12T23:37:50.937 INFO:tasks.workunit.client.0.smithi136.stdout: "quant_size": 100000, 2024-09-12T23:37:50.937 INFO:tasks.workunit.client.0.smithi136.stdout: "ranges": [ 2024-09-12T23:37:50.937 INFO:tasks.workunit.client.0.smithi136.stdout: "[None, -1]", 2024-09-12T23:37:50.937 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 99999]", 2024-09-12T23:37:50.937 INFO:tasks.workunit.client.0.smithi136.stdout: "[100000, 199999]", 2024-09-12T23:37:50.937 INFO:tasks.workunit.client.0.smithi136.stdout: "[200000, 399999]", 2024-09-12T23:37:50.938 INFO:tasks.workunit.client.0.smithi136.stdout: "[400000, 799999]", 2024-09-12T23:37:50.938 INFO:tasks.workunit.client.0.smithi136.stdout: "[800000, 1599999]", 2024-09-12T23:37:50.938 INFO:tasks.workunit.client.0.smithi136.stdout: "[1600000, 3199999]", 2024-09-12T23:37:50.938 INFO:tasks.workunit.client.0.smithi136.stdout: "[3200000, 6399999]", 2024-09-12T23:37:50.938 INFO:tasks.workunit.client.0.smithi136.stdout: "[6400000, 12799999]", 2024-09-12T23:37:50.938 INFO:tasks.workunit.client.0.smithi136.stdout: "[12800000, 25599999]", 2024-09-12T23:37:50.938 INFO:tasks.workunit.client.0.smithi136.stdout: "[25600000, 51199999]", 2024-09-12T23:37:50.938 INFO:tasks.workunit.client.0.smithi136.stdout: "[51200000, 102399999]", 2024-09-12T23:37:50.938 INFO:tasks.workunit.client.0.smithi136.stdout: "[102400000, 204799999]", 2024-09-12T23:37:50.938 INFO:tasks.workunit.client.0.smithi136.stdout: "[204800000, 409599999]", 2024-09-12T23:37:50.938 INFO:tasks.workunit.client.0.smithi136.stdout: "[409600000, 819199999]", 2024-09-12T23:37:50.938 INFO:tasks.workunit.client.0.smithi136.stdout: "[819200000, 1638399999]", 2024-09-12T23:37:50.939 INFO:tasks.workunit.client.0.smithi136.stdout: "[1638400000, 3276799999]", 2024-09-12T23:37:50.939 INFO:tasks.workunit.client.0.smithi136.stdout: "[3276800000, 6553599999]", 2024-09-12T23:37:50.939 INFO:tasks.workunit.client.0.smithi136.stdout: "[6553600000, 13107199999]", 2024-09-12T23:37:50.939 INFO:tasks.workunit.client.0.smithi136.stdout: "[13107200000, 26214399999]", 2024-09-12T23:37:50.939 INFO:tasks.workunit.client.0.smithi136.stdout: "[26214400000, 52428799999]", 2024-09-12T23:37:50.939 INFO:tasks.workunit.client.0.smithi136.stdout: "[52428800000, 104857599999]", 2024-09-12T23:37:50.939 INFO:tasks.workunit.client.0.smithi136.stdout: "[104857600000, 209715199999]", 2024-09-12T23:37:50.939 INFO:tasks.workunit.client.0.smithi136.stdout: "[209715200000, 419430399999]", 2024-09-12T23:37:50.939 INFO:tasks.workunit.client.0.smithi136.stdout: "[419430400000, 838860799999]", 2024-09-12T23:37:50.939 INFO:tasks.workunit.client.0.smithi136.stdout: "[838860800000, 1677721599999]", 2024-09-12T23:37:50.939 INFO:tasks.workunit.client.0.smithi136.stdout: "[1677721600000, 3355443199999]", 2024-09-12T23:37:50.940 INFO:tasks.workunit.client.0.smithi136.stdout: "[3355443200000, 6710886399999]", 2024-09-12T23:37:50.940 INFO:tasks.workunit.client.0.smithi136.stdout: "[6710886400000, 13421772799999]", 2024-09-12T23:37:50.940 INFO:tasks.workunit.client.0.smithi136.stdout: "[13421772800000, 26843545599999]", 2024-09-12T23:37:50.940 INFO:tasks.workunit.client.0.smithi136.stdout: "[26843545600000, 53687091199999]", 2024-09-12T23:37:50.940 INFO:tasks.workunit.client.0.smithi136.stdout: "[53687091200000, None]" 2024-09-12T23:37:50.940 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.940 INFO:tasks.workunit.client.0.smithi136.stdout: "scale_type": "log2" 2024-09-12T23:37:50.940 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.940 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.940 INFO:tasks.workunit.client.0.smithi136.stdout: "buckets": 32, 2024-09-12T23:37:50.940 INFO:tasks.workunit.client.0.smithi136.stdout: "min": 0, 2024-09-12T23:37:50.941 INFO:tasks.workunit.client.0.smithi136.stdout: "name": "Request size (bytes)", 2024-09-12T23:37:50.941 INFO:tasks.workunit.client.0.smithi136.stdout: "quant_size": 512, 2024-09-12T23:37:50.941 INFO:tasks.workunit.client.0.smithi136.stdout: "ranges": [ 2024-09-12T23:37:50.941 INFO:tasks.workunit.client.0.smithi136.stdout: "[None, -1]", 2024-09-12T23:37:50.941 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 511]", 2024-09-12T23:37:50.941 INFO:tasks.workunit.client.0.smithi136.stdout: "[512, 1023]", 2024-09-12T23:37:50.941 INFO:tasks.workunit.client.0.smithi136.stdout: "[1024, 2047]", 2024-09-12T23:37:50.941 INFO:tasks.workunit.client.0.smithi136.stdout: "[2048, 4095]", 2024-09-12T23:37:50.941 INFO:tasks.workunit.client.0.smithi136.stdout: "[4096, 8191]", 2024-09-12T23:37:50.941 INFO:tasks.workunit.client.0.smithi136.stdout: "[8192, 16383]", 2024-09-12T23:37:50.941 INFO:tasks.workunit.client.0.smithi136.stdout: "[16384, 32767]", 2024-09-12T23:37:50.941 INFO:tasks.workunit.client.0.smithi136.stdout: "[32768, 65535]", 2024-09-12T23:37:50.942 INFO:tasks.workunit.client.0.smithi136.stdout: "[65536, 131071]", 2024-09-12T23:37:50.942 INFO:tasks.workunit.client.0.smithi136.stdout: "[131072, 262143]", 2024-09-12T23:37:50.942 INFO:tasks.workunit.client.0.smithi136.stdout: "[262144, 524287]", 2024-09-12T23:37:50.942 INFO:tasks.workunit.client.0.smithi136.stdout: "[524288, 1048575]", 2024-09-12T23:37:50.942 INFO:tasks.workunit.client.0.smithi136.stdout: "[1048576, 2097151]", 2024-09-12T23:37:50.942 INFO:tasks.workunit.client.0.smithi136.stdout: "[2097152, 4194303]", 2024-09-12T23:37:50.942 INFO:tasks.workunit.client.0.smithi136.stdout: "[4194304, 8388607]", 2024-09-12T23:37:50.942 INFO:tasks.workunit.client.0.smithi136.stdout: "[8388608, 16777215]", 2024-09-12T23:37:50.942 INFO:tasks.workunit.client.0.smithi136.stdout: "[16777216, 33554431]", 2024-09-12T23:37:50.942 INFO:tasks.workunit.client.0.smithi136.stdout: "[33554432, 67108863]", 2024-09-12T23:37:50.942 INFO:tasks.workunit.client.0.smithi136.stdout: "[67108864, 134217727]", 2024-09-12T23:37:50.943 INFO:tasks.workunit.client.0.smithi136.stdout: "[134217728, 268435455]", 2024-09-12T23:37:50.943 INFO:tasks.workunit.client.0.smithi136.stdout: "[268435456, 536870911]", 2024-09-12T23:37:50.943 INFO:tasks.workunit.client.0.smithi136.stdout: "[536870912, 1073741823]", 2024-09-12T23:37:50.943 INFO:tasks.workunit.client.0.smithi136.stdout: "[1073741824, 2147483647]", 2024-09-12T23:37:50.943 INFO:tasks.workunit.client.0.smithi136.stdout: "[2147483648, 4294967295]", 2024-09-12T23:37:50.943 INFO:tasks.workunit.client.0.smithi136.stdout: "[4294967296, 8589934591]", 2024-09-12T23:37:50.943 INFO:tasks.workunit.client.0.smithi136.stdout: "[8589934592, 17179869183]", 2024-09-12T23:37:50.943 INFO:tasks.workunit.client.0.smithi136.stdout: "[17179869184, 34359738367]", 2024-09-12T23:37:50.943 INFO:tasks.workunit.client.0.smithi136.stdout: "[34359738368, 68719476735]", 2024-09-12T23:37:50.943 INFO:tasks.workunit.client.0.smithi136.stdout: "[68719476736, 137438953471]", 2024-09-12T23:37:50.943 INFO:tasks.workunit.client.0.smithi136.stdout: "[137438953472, 274877906943]", 2024-09-12T23:37:50.943 INFO:tasks.workunit.client.0.smithi136.stdout: "[274877906944, None]" 2024-09-12T23:37:50.944 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.944 INFO:tasks.workunit.client.0.smithi136.stdout: "scale_type": "log2" 2024-09-12T23:37:50.944 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.944 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.944 INFO:tasks.workunit.client.0.smithi136.stdout: "osds": [ 2024-09-12T23:37:50.944 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.944 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 0, 2024-09-12T23:37:50.944 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:50.944 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.944 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.944 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.945 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.945 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.945 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.945 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.945 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.945 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.945 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.945 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.945 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.945 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.945 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.945 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.946 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.946 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.946 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.946 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.946 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.946 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.946 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.946 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.946 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.946 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.946 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.947 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.947 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.947 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.947 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.947 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.947 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:50.947 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:50.947 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.947 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.947 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 1, 2024-09-12T23:37:50.947 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:50.947 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.948 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.948 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.948 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.948 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.948 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.948 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 7, 2, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.948 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 10, 2, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.948 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 10, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.948 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 8, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.948 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.948 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.948 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.949 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.949 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.949 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.949 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.949 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.949 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.949 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.949 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.949 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.949 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.949 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.949 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.950 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.950 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.950 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.950 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.950 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.950 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.950 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:50.950 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:50.950 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.950 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.950 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 2, 2024-09-12T23:37:50.950 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:50.951 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.951 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.951 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.951 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.951 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.951 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.951 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.951 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.951 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.951 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.951 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.951 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.952 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.952 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.952 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.952 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.952 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.952 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.952 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.952 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.952 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.952 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.953 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.953 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.953 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.953 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.953 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.953 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.953 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.953 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.953 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.953 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:50.953 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:50.953 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.954 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.954 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 3, 2024-09-12T23:37:50.954 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:50.954 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.954 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.954 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.954 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.954 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.954 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.954 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.954 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.954 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.955 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.955 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.955 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.955 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.955 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.955 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.955 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.955 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.955 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.955 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.955 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.956 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.956 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.956 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.956 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.956 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.956 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.956 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.956 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.956 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.956 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.956 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.956 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:50.957 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:50.957 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.957 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.957 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 4, 2024-09-12T23:37:50.957 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:50.957 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.957 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.957 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.957 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.957 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.957 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.957 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.958 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.958 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.958 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.958 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.958 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.958 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.958 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.958 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.958 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.958 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.959 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.959 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.959 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.959 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.959 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.959 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.959 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.959 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.959 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.959 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.959 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.959 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.960 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.960 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.960 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:50.960 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:50.960 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.960 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.960 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 5, 2024-09-12T23:37:50.960 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:50.960 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.960 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.960 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.960 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.961 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.961 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.961 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.961 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.961 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.961 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.961 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.961 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.961 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.961 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.961 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.962 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.962 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.962 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.962 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.962 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.962 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.962 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.962 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.962 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.962 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.962 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.962 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.963 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.963 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.963 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.963 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.963 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:50.963 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:50.963 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.963 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.963 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 6, 2024-09-12T23:37:50.963 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:50.963 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.964 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.964 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.964 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.964 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.964 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.964 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.964 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.964 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.964 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.964 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.964 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.965 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.965 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.965 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.965 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.965 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.965 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.965 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.965 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.965 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.965 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.965 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.966 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.966 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.966 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.966 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.966 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.966 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.966 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.966 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.966 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:50.966 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:50.966 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.967 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:50.967 INFO:tasks.workunit.client.0.smithi136.stdout: "osd_id": 7, 2024-09-12T23:37:50.967 INFO:tasks.workunit.client.0.smithi136.stdout: "values": [ 2024-09-12T23:37:50.967 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.967 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.967 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.967 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.967 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.967 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.967 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.967 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.968 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.968 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.968 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.968 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.968 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.968 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.968 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.968 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.968 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.968 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.968 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.968 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.969 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.969 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.969 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.969 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.969 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.969 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.969 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.969 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.969 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.969 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.969 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-12T23:37:50.970 INFO:tasks.workunit.client.0.smithi136.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-12T23:37:50.970 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:50.970 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.970 INFO:tasks.workunit.client.0.smithi136.stdout: ] 2024-09-12T23:37:50.970 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.970 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.970 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:50.970 INFO:tasks.workunit.client.0.smithi136.stdout: "perf_counters": { 2024-09-12T23:37:50.970 INFO:tasks.workunit.client.0.smithi136.stdout: "mds.4585d541b331f263094839d9f58dd50a20d3f517": { 2024-09-12T23:37:50.970 INFO:tasks.workunit.client.0.smithi136.stdout: "mds": { 2024-09-12T23:37:50.970 INFO:tasks.workunit.client.0.smithi136.stdout: "caps": { 2024-09-12T23:37:50.971 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.971 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.971 INFO:tasks.workunit.client.0.smithi136.stdout: "ceph_cap_op_flush_ack": { 2024-09-12T23:37:50.971 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.971 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.971 INFO:tasks.workunit.client.0.smithi136.stdout: "ceph_cap_op_flushsnap_ack": { 2024-09-12T23:37:50.971 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.971 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.971 INFO:tasks.workunit.client.0.smithi136.stdout: "ceph_cap_op_grant": { 2024-09-12T23:37:50.971 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.971 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.971 INFO:tasks.workunit.client.0.smithi136.stdout: "ceph_cap_op_revoke": { 2024-09-12T23:37:50.972 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.972 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.972 INFO:tasks.workunit.client.0.smithi136.stdout: "ceph_cap_op_trunc": { 2024-09-12T23:37:50.972 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.972 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.972 INFO:tasks.workunit.client.0.smithi136.stdout: "dir_commit": { 2024-09-12T23:37:50.972 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 12 2024-09-12T23:37:50.972 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.972 INFO:tasks.workunit.client.0.smithi136.stdout: "dir_fetch": { 2024-09-12T23:37:50.972 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.972 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.973 INFO:tasks.workunit.client.0.smithi136.stdout: "dir_merge": { 2024-09-12T23:37:50.973 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.973 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.973 INFO:tasks.workunit.client.0.smithi136.stdout: "dir_split": { 2024-09-12T23:37:50.973 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.973 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.973 INFO:tasks.workunit.client.0.smithi136.stdout: "exported_inodes": { 2024-09-12T23:37:50.973 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.973 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.973 INFO:tasks.workunit.client.0.smithi136.stdout: "forward": { 2024-09-12T23:37:50.973 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.973 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.974 INFO:tasks.workunit.client.0.smithi136.stdout: "handle_client_cap_release": { 2024-09-12T23:37:50.974 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.974 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.974 INFO:tasks.workunit.client.0.smithi136.stdout: "handle_client_caps": { 2024-09-12T23:37:50.974 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.974 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.974 INFO:tasks.workunit.client.0.smithi136.stdout: "handle_client_caps_dirty": { 2024-09-12T23:37:50.974 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.974 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.974 INFO:tasks.workunit.client.0.smithi136.stdout: "handle_inode_file_caps": { 2024-09-12T23:37:50.974 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.975 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.975 INFO:tasks.workunit.client.0.smithi136.stdout: "imported_inodes": { 2024-09-12T23:37:50.975 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.975 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.975 INFO:tasks.workunit.client.0.smithi136.stdout: "inodes": { 2024-09-12T23:37:50.975 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 10 2024-09-12T23:37:50.975 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.975 INFO:tasks.workunit.client.0.smithi136.stdout: "inodes_expired": { 2024-09-12T23:37:50.975 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.975 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.975 INFO:tasks.workunit.client.0.smithi136.stdout: "inodes_pinned": { 2024-09-12T23:37:50.975 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 10 2024-09-12T23:37:50.976 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.976 INFO:tasks.workunit.client.0.smithi136.stdout: "inodes_with_caps": { 2024-09-12T23:37:50.976 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.976 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.976 INFO:tasks.workunit.client.0.smithi136.stdout: "load_cent": { 2024-09-12T23:37:50.976 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.976 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.976 INFO:tasks.workunit.client.0.smithi136.stdout: "openino_dir_fetch": { 2024-09-12T23:37:50.976 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.976 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.976 INFO:tasks.workunit.client.0.smithi136.stdout: "process_request_cap_release": { 2024-09-12T23:37:50.977 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.977 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.977 INFO:tasks.workunit.client.0.smithi136.stdout: "reply_latency": { 2024-09-12T23:37:50.977 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 2, 2024-09-12T23:37:50.977 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 112659 2024-09-12T23:37:50.977 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.977 INFO:tasks.workunit.client.0.smithi136.stdout: "request": { 2024-09-12T23:37:50.977 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2 2024-09-12T23:37:50.977 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.977 INFO:tasks.workunit.client.0.smithi136.stdout: "root_rbytes": { 2024-09-12T23:37:50.977 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.977 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.978 INFO:tasks.workunit.client.0.smithi136.stdout: "root_rfiles": { 2024-09-12T23:37:50.978 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.978 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.978 INFO:tasks.workunit.client.0.smithi136.stdout: "root_rsnaps": { 2024-09-12T23:37:50.978 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.978 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.978 INFO:tasks.workunit.client.0.smithi136.stdout: "slow_reply": { 2024-09-12T23:37:50.978 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.978 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.978 INFO:tasks.workunit.client.0.smithi136.stdout: "subtrees": { 2024-09-12T23:37:50.978 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2 2024-09-12T23:37:50.978 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.979 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.979 INFO:tasks.workunit.client.0.smithi136.stdout: "mds_cache": { 2024-09-12T23:37:50.979 INFO:tasks.workunit.client.0.smithi136.stdout: "ireq_enqueue_scrub": { 2024-09-12T23:37:50.979 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.979 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.979 INFO:tasks.workunit.client.0.smithi136.stdout: "ireq_exportdir": { 2024-09-12T23:37:50.979 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.979 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.979 INFO:tasks.workunit.client.0.smithi136.stdout: "ireq_flush": { 2024-09-12T23:37:50.979 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.979 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.980 INFO:tasks.workunit.client.0.smithi136.stdout: "ireq_fragmentdir": { 2024-09-12T23:37:50.980 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.980 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.980 INFO:tasks.workunit.client.0.smithi136.stdout: "ireq_fragstats": { 2024-09-12T23:37:50.980 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.980 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.980 INFO:tasks.workunit.client.0.smithi136.stdout: "ireq_inodestats": { 2024-09-12T23:37:50.980 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.980 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.980 INFO:tasks.workunit.client.0.smithi136.stdout: "num_recovering_enqueued": { 2024-09-12T23:37:50.980 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.980 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.981 INFO:tasks.workunit.client.0.smithi136.stdout: "num_recovering_prioritized": { 2024-09-12T23:37:50.981 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.981 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.981 INFO:tasks.workunit.client.0.smithi136.stdout: "num_recovering_processing": { 2024-09-12T23:37:50.981 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.981 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.981 INFO:tasks.workunit.client.0.smithi136.stdout: "num_strays": { 2024-09-12T23:37:50.981 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.981 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.981 INFO:tasks.workunit.client.0.smithi136.stdout: "num_strays_delayed": { 2024-09-12T23:37:50.981 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.982 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.982 INFO:tasks.workunit.client.0.smithi136.stdout: "num_strays_enqueuing": { 2024-09-12T23:37:50.982 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.982 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.982 INFO:tasks.workunit.client.0.smithi136.stdout: "recovery_completed": { 2024-09-12T23:37:50.982 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.982 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.982 INFO:tasks.workunit.client.0.smithi136.stdout: "recovery_started": { 2024-09-12T23:37:50.982 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.982 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.982 INFO:tasks.workunit.client.0.smithi136.stdout: "strays_created": { 2024-09-12T23:37:50.982 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.983 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.983 INFO:tasks.workunit.client.0.smithi136.stdout: "strays_enqueued": { 2024-09-12T23:37:50.983 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.983 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.983 INFO:tasks.workunit.client.0.smithi136.stdout: "strays_migrated": { 2024-09-12T23:37:50.983 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.983 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.983 INFO:tasks.workunit.client.0.smithi136.stdout: "strays_reintegrated": { 2024-09-12T23:37:50.983 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.983 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.983 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.983 INFO:tasks.workunit.client.0.smithi136.stdout: "mds_log": { 2024-09-12T23:37:50.984 INFO:tasks.workunit.client.0.smithi136.stdout: "ev": { 2024-09-12T23:37:50.984 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5 2024-09-12T23:37:50.984 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.984 INFO:tasks.workunit.client.0.smithi136.stdout: "evadd": { 2024-09-12T23:37:50.984 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5 2024-09-12T23:37:50.984 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.984 INFO:tasks.workunit.client.0.smithi136.stdout: "evex": { 2024-09-12T23:37:50.984 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.984 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.984 INFO:tasks.workunit.client.0.smithi136.stdout: "evexd": { 2024-09-12T23:37:50.984 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.985 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.985 INFO:tasks.workunit.client.0.smithi136.stdout: "evexg": { 2024-09-12T23:37:50.985 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.985 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.985 INFO:tasks.workunit.client.0.smithi136.stdout: "evtrm": { 2024-09-12T23:37:50.985 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.985 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.985 INFO:tasks.workunit.client.0.smithi136.stdout: "jlat": { 2024-09-12T23:37:50.985 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 5, 2024-09-12T23:37:50.985 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 23682895 2024-09-12T23:37:50.985 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.985 INFO:tasks.workunit.client.0.smithi136.stdout: "replayed": { 2024-09-12T23:37:50.985 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.986 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.986 INFO:tasks.workunit.client.0.smithi136.stdout: "seg": { 2024-09-12T23:37:50.986 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1 2024-09-12T23:37:50.986 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.986 INFO:tasks.workunit.client.0.smithi136.stdout: "segadd": { 2024-09-12T23:37:50.986 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1 2024-09-12T23:37:50.986 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.986 INFO:tasks.workunit.client.0.smithi136.stdout: "segex": { 2024-09-12T23:37:50.986 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.986 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.986 INFO:tasks.workunit.client.0.smithi136.stdout: "segexd": { 2024-09-12T23:37:50.987 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.987 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.987 INFO:tasks.workunit.client.0.smithi136.stdout: "segexg": { 2024-09-12T23:37:50.987 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.987 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.987 INFO:tasks.workunit.client.0.smithi136.stdout: "segtrm": { 2024-09-12T23:37:50.987 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.987 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.987 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.987 INFO:tasks.workunit.client.0.smithi136.stdout: "mds_mem": { 2024-09-12T23:37:50.987 INFO:tasks.workunit.client.0.smithi136.stdout: "cap": { 2024-09-12T23:37:50.987 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.988 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.988 INFO:tasks.workunit.client.0.smithi136.stdout: "cap+": { 2024-09-12T23:37:50.988 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2 2024-09-12T23:37:50.988 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.988 INFO:tasks.workunit.client.0.smithi136.stdout: "cap-": { 2024-09-12T23:37:50.988 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2 2024-09-12T23:37:50.988 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.988 INFO:tasks.workunit.client.0.smithi136.stdout: "dir": { 2024-09-12T23:37:50.988 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 12 2024-09-12T23:37:50.988 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.988 INFO:tasks.workunit.client.0.smithi136.stdout: "dir+": { 2024-09-12T23:37:50.988 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 12 2024-09-12T23:37:50.989 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.989 INFO:tasks.workunit.client.0.smithi136.stdout: "dir-": { 2024-09-12T23:37:50.989 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.989 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.989 INFO:tasks.workunit.client.0.smithi136.stdout: "dn": { 2024-09-12T23:37:50.989 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 10 2024-09-12T23:37:50.989 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.989 INFO:tasks.workunit.client.0.smithi136.stdout: "dn+": { 2024-09-12T23:37:50.989 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 10 2024-09-12T23:37:50.989 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.989 INFO:tasks.workunit.client.0.smithi136.stdout: "dn-": { 2024-09-12T23:37:50.989 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.990 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.990 INFO:tasks.workunit.client.0.smithi136.stdout: "heap": { 2024-09-12T23:37:50.990 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 207068 2024-09-12T23:37:50.990 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.990 INFO:tasks.workunit.client.0.smithi136.stdout: "ino": { 2024-09-12T23:37:50.990 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 13 2024-09-12T23:37:50.990 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.990 INFO:tasks.workunit.client.0.smithi136.stdout: "ino+": { 2024-09-12T23:37:50.990 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 13 2024-09-12T23:37:50.990 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.990 INFO:tasks.workunit.client.0.smithi136.stdout: "ino-": { 2024-09-12T23:37:50.990 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.991 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.991 INFO:tasks.workunit.client.0.smithi136.stdout: "rss": { 2024-09-12T23:37:50.991 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 32760 2024-09-12T23:37:50.991 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:50.991 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.991 INFO:tasks.workunit.client.0.smithi136.stdout: "mds_server": { 2024-09-12T23:37:50.991 INFO:tasks.workunit.client.0.smithi136.stdout: "cap_acquisition_throttle": { 2024-09-12T23:37:50.991 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.991 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.991 INFO:tasks.workunit.client.0.smithi136.stdout: "cap_revoke_eviction": { 2024-09-12T23:37:50.991 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.991 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.991 INFO:tasks.workunit.client.0.smithi136.stdout: "handle_client_request": { 2024-09-12T23:37:50.992 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2 2024-09-12T23:37:50.992 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.992 INFO:tasks.workunit.client.0.smithi136.stdout: "handle_client_session": { 2024-09-12T23:37:50.992 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 16 2024-09-12T23:37:50.992 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.992 INFO:tasks.workunit.client.0.smithi136.stdout: "handle_peer_request": { 2024-09-12T23:37:50.992 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.992 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.992 INFO:tasks.workunit.client.0.smithi136.stdout: "req_create_latency": { 2024-09-12T23:37:50.992 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:50.992 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.993 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.993 INFO:tasks.workunit.client.0.smithi136.stdout: "req_getattr_latency": { 2024-09-12T23:37:50.993 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 2, 2024-09-12T23:37:50.993 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 112659 2024-09-12T23:37:50.993 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.993 INFO:tasks.workunit.client.0.smithi136.stdout: "req_getfilelock_latency": { 2024-09-12T23:37:50.993 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:50.993 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.993 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.993 INFO:tasks.workunit.client.0.smithi136.stdout: "req_getvxattr_latency": { 2024-09-12T23:37:50.993 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:50.994 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.994 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.994 INFO:tasks.workunit.client.0.smithi136.stdout: "req_link_latency": { 2024-09-12T23:37:50.994 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:50.994 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.994 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.994 INFO:tasks.workunit.client.0.smithi136.stdout: "req_lookup_latency": { 2024-09-12T23:37:50.994 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:50.994 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.994 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.994 INFO:tasks.workunit.client.0.smithi136.stdout: "req_lookuphash_latency": { 2024-09-12T23:37:50.994 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:50.995 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.995 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.995 INFO:tasks.workunit.client.0.smithi136.stdout: "req_lookupino_latency": { 2024-09-12T23:37:50.995 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:50.995 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.995 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.995 INFO:tasks.workunit.client.0.smithi136.stdout: "req_lookupname_latency": { 2024-09-12T23:37:50.995 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:50.995 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.995 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.995 INFO:tasks.workunit.client.0.smithi136.stdout: "req_lookupparent_latency": { 2024-09-12T23:37:50.995 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:50.996 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.996 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.996 INFO:tasks.workunit.client.0.smithi136.stdout: "req_lookupsnap_latency": { 2024-09-12T23:37:50.996 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:50.996 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.996 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.996 INFO:tasks.workunit.client.0.smithi136.stdout: "req_lssnap_latency": { 2024-09-12T23:37:50.996 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:50.996 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.996 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.996 INFO:tasks.workunit.client.0.smithi136.stdout: "req_mkdir_latency": { 2024-09-12T23:37:50.997 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:50.997 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.997 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.997 INFO:tasks.workunit.client.0.smithi136.stdout: "req_mknod_latency": { 2024-09-12T23:37:50.997 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:50.997 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.997 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.997 INFO:tasks.workunit.client.0.smithi136.stdout: "req_mksnap_latency": { 2024-09-12T23:37:50.997 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:50.997 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.997 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.997 INFO:tasks.workunit.client.0.smithi136.stdout: "req_open_latency": { 2024-09-12T23:37:50.998 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:50.998 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.998 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.998 INFO:tasks.workunit.client.0.smithi136.stdout: "req_readdir_latency": { 2024-09-12T23:37:50.998 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:50.998 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.998 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.998 INFO:tasks.workunit.client.0.smithi136.stdout: "req_rename_latency": { 2024-09-12T23:37:50.998 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:50.998 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.998 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.999 INFO:tasks.workunit.client.0.smithi136.stdout: "req_renamesnap_latency": { 2024-09-12T23:37:50.999 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:50.999 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.999 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.999 INFO:tasks.workunit.client.0.smithi136.stdout: "req_rmdir_latency": { 2024-09-12T23:37:50.999 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:50.999 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.999 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:50.999 INFO:tasks.workunit.client.0.smithi136.stdout: "req_rmsnap_latency": { 2024-09-12T23:37:50.999 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:50.999 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:50.999 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.000 INFO:tasks.workunit.client.0.smithi136.stdout: "req_rmxattr_latency": { 2024-09-12T23:37:51.000 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.000 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.000 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.000 INFO:tasks.workunit.client.0.smithi136.stdout: "req_setattr_latency": { 2024-09-12T23:37:51.000 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.000 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.000 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.000 INFO:tasks.workunit.client.0.smithi136.stdout: "req_setdirlayout_latency": { 2024-09-12T23:37:51.000 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.000 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.000 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.001 INFO:tasks.workunit.client.0.smithi136.stdout: "req_setfilelock_latency": { 2024-09-12T23:37:51.001 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.001 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.001 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.001 INFO:tasks.workunit.client.0.smithi136.stdout: "req_setlayout_latency": { 2024-09-12T23:37:51.001 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.001 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.001 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.001 INFO:tasks.workunit.client.0.smithi136.stdout: "req_setxattr_latency": { 2024-09-12T23:37:51.001 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.001 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.001 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.002 INFO:tasks.workunit.client.0.smithi136.stdout: "req_symlink_latency": { 2024-09-12T23:37:51.002 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.002 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.002 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.002 INFO:tasks.workunit.client.0.smithi136.stdout: "req_unlink_latency": { 2024-09-12T23:37:51.002 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.002 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.002 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.002 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.002 INFO:tasks.workunit.client.0.smithi136.stdout: "mds_sessions": { 2024-09-12T23:37:51.002 INFO:tasks.workunit.client.0.smithi136.stdout: "average_load": { 2024-09-12T23:37:51.002 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1 2024-09-12T23:37:51.003 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.003 INFO:tasks.workunit.client.0.smithi136.stdout: "avg_session_uptime": { 2024-09-12T23:37:51.003 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 85 2024-09-12T23:37:51.003 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.003 INFO:tasks.workunit.client.0.smithi136.stdout: "mdthresh_evicted": { 2024-09-12T23:37:51.003 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.003 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.003 INFO:tasks.workunit.client.0.smithi136.stdout: "session_add": { 2024-09-12T23:37:51.003 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2 2024-09-12T23:37:51.003 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.003 INFO:tasks.workunit.client.0.smithi136.stdout: "session_count": { 2024-09-12T23:37:51.004 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.004 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.004 INFO:tasks.workunit.client.0.smithi136.stdout: "session_remove": { 2024-09-12T23:37:51.004 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2 2024-09-12T23:37:51.004 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.004 INFO:tasks.workunit.client.0.smithi136.stdout: "sessions_open": { 2024-09-12T23:37:51.004 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.004 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.004 INFO:tasks.workunit.client.0.smithi136.stdout: "sessions_stale": { 2024-09-12T23:37:51.004 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.004 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.005 INFO:tasks.workunit.client.0.smithi136.stdout: "total_load": { 2024-09-12T23:37:51.005 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2 2024-09-12T23:37:51.005 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.005 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.005 INFO:tasks.workunit.client.0.smithi136.stdout: "objecter": { 2024-09-12T23:37:51.005 INFO:tasks.workunit.client.0.smithi136.stdout: "op_active": { 2024-09-12T23:37:51.005 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.005 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.005 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r": { 2024-09-12T23:37:51.005 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.005 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.005 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rmw": { 2024-09-12T23:37:51.006 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.006 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.006 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w": { 2024-09-12T23:37:51.006 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 45 2024-09-12T23:37:51.006 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.006 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.006 INFO:tasks.workunit.client.0.smithi136.stdout: "purge_queue": { 2024-09-12T23:37:51.006 INFO:tasks.workunit.client.0.smithi136.stdout: "pq_executed": { 2024-09-12T23:37:51.006 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.006 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.006 INFO:tasks.workunit.client.0.smithi136.stdout: "pq_executing": { 2024-09-12T23:37:51.006 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.007 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.007 INFO:tasks.workunit.client.0.smithi136.stdout: "pq_executing_high_water": { 2024-09-12T23:37:51.007 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.007 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.007 INFO:tasks.workunit.client.0.smithi136.stdout: "pq_executing_ops": { 2024-09-12T23:37:51.007 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.007 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.007 INFO:tasks.workunit.client.0.smithi136.stdout: "pq_executing_ops_high_water": { 2024-09-12T23:37:51.007 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.007 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.007 INFO:tasks.workunit.client.0.smithi136.stdout: "pq_item_in_journal": { 2024-09-12T23:37:51.007 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.008 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.008 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.008 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.008 INFO:tasks.workunit.client.0.smithi136.stdout: "mon.0027af889fd200cedbdb75d916e7558e544c1ffa": { 2024-09-12T23:37:51.008 INFO:tasks.workunit.client.0.smithi136.stdout: "mon": { 2024-09-12T23:37:51.008 INFO:tasks.workunit.client.0.smithi136.stdout: "election_call": { 2024-09-12T23:37:51.008 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1 2024-09-12T23:37:51.008 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.008 INFO:tasks.workunit.client.0.smithi136.stdout: "election_lose": { 2024-09-12T23:37:51.008 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1 2024-09-12T23:37:51.008 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.009 INFO:tasks.workunit.client.0.smithi136.stdout: "election_win": { 2024-09-12T23:37:51.009 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.009 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.009 INFO:tasks.workunit.client.0.smithi136.stdout: "num_elections": { 2024-09-12T23:37:51.009 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4 2024-09-12T23:37:51.009 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.009 INFO:tasks.workunit.client.0.smithi136.stdout: "num_sessions": { 2024-09-12T23:37:51.009 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 14 2024-09-12T23:37:51.009 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.009 INFO:tasks.workunit.client.0.smithi136.stdout: "session_add": { 2024-09-12T23:37:51.009 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5 2024-09-12T23:37:51.010 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.010 INFO:tasks.workunit.client.0.smithi136.stdout: "session_rm": { 2024-09-12T23:37:51.010 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 119 2024-09-12T23:37:51.010 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.010 INFO:tasks.workunit.client.0.smithi136.stdout: "session_trim": { 2024-09-12T23:37:51.010 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.010 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.010 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.010 INFO:tasks.workunit.client.0.smithi136.stdout: "paxos": { 2024-09-12T23:37:51.010 INFO:tasks.workunit.client.0.smithi136.stdout: "accept_timeout": { 2024-09-12T23:37:51.010 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.010 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.011 INFO:tasks.workunit.client.0.smithi136.stdout: "begin": { 2024-09-12T23:37:51.011 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1121 2024-09-12T23:37:51.011 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.011 INFO:tasks.workunit.client.0.smithi136.stdout: "begin_bytes": { 2024-09-12T23:37:51.011 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1121, 2024-09-12T23:37:51.011 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4644951 2024-09-12T23:37:51.011 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.011 INFO:tasks.workunit.client.0.smithi136.stdout: "begin_keys": { 2024-09-12T23:37:51.011 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.011 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.011 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.012 INFO:tasks.workunit.client.0.smithi136.stdout: "begin_latency": { 2024-09-12T23:37:51.012 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1121, 2024-09-12T23:37:51.012 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 395993406 2024-09-12T23:37:51.012 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.012 INFO:tasks.workunit.client.0.smithi136.stdout: "collect": { 2024-09-12T23:37:51.012 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1 2024-09-12T23:37:51.012 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.012 INFO:tasks.workunit.client.0.smithi136.stdout: "collect_bytes": { 2024-09-12T23:37:51.012 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1, 2024-09-12T23:37:51.012 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 73 2024-09-12T23:37:51.012 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.012 INFO:tasks.workunit.client.0.smithi136.stdout: "collect_keys": { 2024-09-12T23:37:51.013 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1, 2024-09-12T23:37:51.013 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1 2024-09-12T23:37:51.013 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.013 INFO:tasks.workunit.client.0.smithi136.stdout: "collect_latency": { 2024-09-12T23:37:51.013 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1, 2024-09-12T23:37:51.013 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.013 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.013 INFO:tasks.workunit.client.0.smithi136.stdout: "collect_timeout": { 2024-09-12T23:37:51.013 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.013 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.013 INFO:tasks.workunit.client.0.smithi136.stdout: "collect_uncommitted": { 2024-09-12T23:37:51.014 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.014 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.014 INFO:tasks.workunit.client.0.smithi136.stdout: "commit": { 2024-09-12T23:37:51.014 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1122 2024-09-12T23:37:51.014 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.014 INFO:tasks.workunit.client.0.smithi136.stdout: "commit_bytes": { 2024-09-12T23:37:51.014 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.014 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.014 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.014 INFO:tasks.workunit.client.0.smithi136.stdout: "commit_keys": { 2024-09-12T23:37:51.014 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.014 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.015 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.015 INFO:tasks.workunit.client.0.smithi136.stdout: "commit_latency": { 2024-09-12T23:37:51.015 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.015 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.015 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.015 INFO:tasks.workunit.client.0.smithi136.stdout: "lease_ack_timeout": { 2024-09-12T23:37:51.015 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.015 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.015 INFO:tasks.workunit.client.0.smithi136.stdout: "lease_timeout": { 2024-09-12T23:37:51.015 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.015 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.016 INFO:tasks.workunit.client.0.smithi136.stdout: "new_pn": { 2024-09-12T23:37:51.016 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.016 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.016 INFO:tasks.workunit.client.0.smithi136.stdout: "new_pn_latency": { 2024-09-12T23:37:51.016 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.016 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.016 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.016 INFO:tasks.workunit.client.0.smithi136.stdout: "refresh": { 2024-09-12T23:37:51.016 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1122 2024-09-12T23:37:51.016 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.016 INFO:tasks.workunit.client.0.smithi136.stdout: "refresh_latency": { 2024-09-12T23:37:51.016 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1122, 2024-09-12T23:37:51.017 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4867914875 2024-09-12T23:37:51.017 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.017 INFO:tasks.workunit.client.0.smithi136.stdout: "restart": { 2024-09-12T23:37:51.017 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 10 2024-09-12T23:37:51.017 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.017 INFO:tasks.workunit.client.0.smithi136.stdout: "share_state": { 2024-09-12T23:37:51.017 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.017 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.017 INFO:tasks.workunit.client.0.smithi136.stdout: "share_state_bytes": { 2024-09-12T23:37:51.017 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.017 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.018 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.018 INFO:tasks.workunit.client.0.smithi136.stdout: "share_state_keys": { 2024-09-12T23:37:51.018 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.018 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.018 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.018 INFO:tasks.workunit.client.0.smithi136.stdout: "start_leader": { 2024-09-12T23:37:51.018 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.018 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.018 INFO:tasks.workunit.client.0.smithi136.stdout: "start_peon": { 2024-09-12T23:37:51.018 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1 2024-09-12T23:37:51.018 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.018 INFO:tasks.workunit.client.0.smithi136.stdout: "store_state": { 2024-09-12T23:37:51.019 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1122 2024-09-12T23:37:51.019 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.019 INFO:tasks.workunit.client.0.smithi136.stdout: "store_state_bytes": { 2024-09-12T23:37:51.019 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1122, 2024-09-12T23:37:51.019 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 9127763 2024-09-12T23:37:51.019 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.019 INFO:tasks.workunit.client.0.smithi136.stdout: "store_state_keys": { 2024-09-12T23:37:51.019 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1122, 2024-09-12T23:37:51.019 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 7949 2024-09-12T23:37:51.019 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.019 INFO:tasks.workunit.client.0.smithi136.stdout: "store_state_latency": { 2024-09-12T23:37:51.019 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1122, 2024-09-12T23:37:51.020 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 563989915 2024-09-12T23:37:51.020 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.020 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.020 INFO:tasks.workunit.client.0.smithi136.stdout: "prioritycache": { 2024-09-12T23:37:51.020 INFO:tasks.workunit.client.0.smithi136.stdout: "cache_bytes": { 2024-09-12T23:37:51.020 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1020054731 2024-09-12T23:37:51.020 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.020 INFO:tasks.workunit.client.0.smithi136.stdout: "heap_bytes": { 2024-09-12T23:37:51.020 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 40689664 2024-09-12T23:37:51.020 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.020 INFO:tasks.workunit.client.0.smithi136.stdout: "mapped_bytes": { 2024-09-12T23:37:51.021 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 36519936 2024-09-12T23:37:51.021 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.021 INFO:tasks.workunit.client.0.smithi136.stdout: "target_bytes": { 2024-09-12T23:37:51.021 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2147483648 2024-09-12T23:37:51.021 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.021 INFO:tasks.workunit.client.0.smithi136.stdout: "unmapped_bytes": { 2024-09-12T23:37:51.021 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4169728 2024-09-12T23:37:51.021 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.021 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.021 INFO:tasks.workunit.client.0.smithi136.stdout: "prioritycache:full": { 2024-09-12T23:37:51.021 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:51.022 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 348127232 2024-09-12T23:37:51.022 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.022 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:51.022 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.022 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.022 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:51.022 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.022 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.022 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:51.022 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 277028253 2024-09-12T23:37:51.022 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.023 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:51.023 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 254099 2024-09-12T23:37:51.023 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.023 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:51.023 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.023 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.023 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:51.023 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.023 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.023 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:51.023 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.023 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.024 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:51.024 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.024 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.024 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:51.024 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.024 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.024 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:51.024 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.024 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.024 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:51.024 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.025 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.025 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:51.025 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.025 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.025 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:51.025 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 70844880 2024-09-12T23:37:51.025 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.025 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.025 INFO:tasks.workunit.client.0.smithi136.stdout: "prioritycache:inc": { 2024-09-12T23:37:51.025 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:51.025 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 348127232 2024-09-12T23:37:51.026 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.026 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:51.026 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.026 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.026 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:51.026 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.026 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.026 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:51.026 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 277028253 2024-09-12T23:37:51.026 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.026 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:51.026 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 40255 2024-09-12T23:37:51.027 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.027 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:51.027 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.027 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.027 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:51.027 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.027 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.027 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:51.027 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.027 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.027 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:51.028 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.028 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.028 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:51.028 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.028 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.028 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:51.028 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.028 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.028 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:51.028 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.028 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.029 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:51.029 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.029 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.029 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:51.029 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 71058724 2024-09-12T23:37:51.029 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.029 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.029 INFO:tasks.workunit.client.0.smithi136.stdout: "prioritycache:kv": { 2024-09-12T23:37:51.029 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:51.029 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 322961408 2024-09-12T23:37:51.029 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.030 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:51.030 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67108864 2024-09-12T23:37:51.030 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.030 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:51.030 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.030 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.030 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:51.030 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 184685502 2024-09-12T23:37:51.030 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.030 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:51.030 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.030 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.031 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:51.031 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.031 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.031 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:51.031 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.031 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.031 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:51.031 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.031 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.031 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:51.031 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.031 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.032 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:51.032 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.032 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.032 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:51.032 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.032 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.032 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:51.032 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.032 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.032 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:51.032 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.033 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.033 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:51.033 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 71167042 2024-09-12T23:37:51.033 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.033 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.033 INFO:tasks.workunit.client.0.smithi136.stdout: "rocksdb": { 2024-09-12T23:37:51.033 INFO:tasks.workunit.client.0.smithi136.stdout: "compact": { 2024-09-12T23:37:51.033 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.033 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.033 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_completed": { 2024-09-12T23:37:51.033 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5 2024-09-12T23:37:51.033 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.034 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_lasted": { 2024-09-12T23:37:51.034 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 26588769 2024-09-12T23:37:51.034 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.034 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_queue_len": { 2024-09-12T23:37:51.034 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.034 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.034 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_queue_merge": { 2024-09-12T23:37:51.034 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.034 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.034 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_running": { 2024-09-12T23:37:51.034 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.034 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.035 INFO:tasks.workunit.client.0.smithi136.stdout: "get_latency": { 2024-09-12T23:37:51.035 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 47644, 2024-09-12T23:37:51.035 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 271657414 2024-09-12T23:37:51.035 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.035 INFO:tasks.workunit.client.0.smithi136.stdout: "rocksdb_write_delay_time": { 2024-09-12T23:37:51.035 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.035 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.035 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.036 INFO:tasks.workunit.client.0.smithi136.stdout: "rocksdb_write_memtable_time": { 2024-09-12T23:37:51.036 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.036 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.036 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.036 INFO:tasks.workunit.client.0.smithi136.stdout: "rocksdb_write_pre_and_post_time": { 2024-09-12T23:37:51.036 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.036 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.036 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.036 INFO:tasks.workunit.client.0.smithi136.stdout: "rocksdb_write_wal_time": { 2024-09-12T23:37:51.036 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.036 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.036 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.037 INFO:tasks.workunit.client.0.smithi136.stdout: "submit_latency": { 2024-09-12T23:37:51.037 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.037 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.037 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.037 INFO:tasks.workunit.client.0.smithi136.stdout: "submit_sync_latency": { 2024-09-12T23:37:51.037 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 2462, 2024-09-12T23:37:51.037 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1106897056 2024-09-12T23:37:51.037 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.037 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.037 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.037 INFO:tasks.workunit.client.0.smithi136.stdout: "mon.4585d541b331f263094839d9f58dd50a20d3f517": { 2024-09-12T23:37:51.038 INFO:tasks.workunit.client.0.smithi136.stdout: "mon": { 2024-09-12T23:37:51.038 INFO:tasks.workunit.client.0.smithi136.stdout: "election_call": { 2024-09-12T23:37:51.038 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2 2024-09-12T23:37:51.038 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.038 INFO:tasks.workunit.client.0.smithi136.stdout: "election_lose": { 2024-09-12T23:37:51.038 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2 2024-09-12T23:37:51.038 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.038 INFO:tasks.workunit.client.0.smithi136.stdout: "election_win": { 2024-09-12T23:37:51.038 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.038 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.038 INFO:tasks.workunit.client.0.smithi136.stdout: "num_elections": { 2024-09-12T23:37:51.038 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 8 2024-09-12T23:37:51.039 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.039 INFO:tasks.workunit.client.0.smithi136.stdout: "num_sessions": { 2024-09-12T23:37:51.039 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 9 2024-09-12T23:37:51.039 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.039 INFO:tasks.workunit.client.0.smithi136.stdout: "session_add": { 2024-09-12T23:37:51.039 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4 2024-09-12T23:37:51.039 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.039 INFO:tasks.workunit.client.0.smithi136.stdout: "session_rm": { 2024-09-12T23:37:51.039 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 57 2024-09-12T23:37:51.039 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.039 INFO:tasks.workunit.client.0.smithi136.stdout: "session_trim": { 2024-09-12T23:37:51.039 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.040 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.040 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.040 INFO:tasks.workunit.client.0.smithi136.stdout: "paxos": { 2024-09-12T23:37:51.040 INFO:tasks.workunit.client.0.smithi136.stdout: "accept_timeout": { 2024-09-12T23:37:51.040 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.040 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.040 INFO:tasks.workunit.client.0.smithi136.stdout: "begin": { 2024-09-12T23:37:51.040 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1128 2024-09-12T23:37:51.040 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.040 INFO:tasks.workunit.client.0.smithi136.stdout: "begin_bytes": { 2024-09-12T23:37:51.040 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1128, 2024-09-12T23:37:51.041 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4670945 2024-09-12T23:37:51.041 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.041 INFO:tasks.workunit.client.0.smithi136.stdout: "begin_keys": { 2024-09-12T23:37:51.041 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.041 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.041 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.041 INFO:tasks.workunit.client.0.smithi136.stdout: "begin_latency": { 2024-09-12T23:37:51.041 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1128, 2024-09-12T23:37:51.041 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 259996204 2024-09-12T23:37:51.041 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.041 INFO:tasks.workunit.client.0.smithi136.stdout: "collect": { 2024-09-12T23:37:51.042 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2 2024-09-12T23:37:51.042 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.042 INFO:tasks.workunit.client.0.smithi136.stdout: "collect_bytes": { 2024-09-12T23:37:51.042 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 2, 2024-09-12T23:37:51.042 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 146 2024-09-12T23:37:51.042 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.042 INFO:tasks.workunit.client.0.smithi136.stdout: "collect_keys": { 2024-09-12T23:37:51.042 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 2, 2024-09-12T23:37:51.042 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2 2024-09-12T23:37:51.042 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.042 INFO:tasks.workunit.client.0.smithi136.stdout: "collect_latency": { 2024-09-12T23:37:51.042 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 2, 2024-09-12T23:37:51.043 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 3999912 2024-09-12T23:37:51.043 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.043 INFO:tasks.workunit.client.0.smithi136.stdout: "collect_timeout": { 2024-09-12T23:37:51.043 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.043 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.043 INFO:tasks.workunit.client.0.smithi136.stdout: "collect_uncommitted": { 2024-09-12T23:37:51.043 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.043 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.043 INFO:tasks.workunit.client.0.smithi136.stdout: "commit": { 2024-09-12T23:37:51.043 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1129 2024-09-12T23:37:51.043 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.044 INFO:tasks.workunit.client.0.smithi136.stdout: "commit_bytes": { 2024-09-12T23:37:51.044 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.044 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.044 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.044 INFO:tasks.workunit.client.0.smithi136.stdout: "commit_keys": { 2024-09-12T23:37:51.044 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.044 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.044 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.044 INFO:tasks.workunit.client.0.smithi136.stdout: "commit_latency": { 2024-09-12T23:37:51.044 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.044 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.044 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.045 INFO:tasks.workunit.client.0.smithi136.stdout: "lease_ack_timeout": { 2024-09-12T23:37:51.045 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.045 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.045 INFO:tasks.workunit.client.0.smithi136.stdout: "lease_timeout": { 2024-09-12T23:37:51.045 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.045 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.045 INFO:tasks.workunit.client.0.smithi136.stdout: "new_pn": { 2024-09-12T23:37:51.045 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.045 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.045 INFO:tasks.workunit.client.0.smithi136.stdout: "new_pn_latency": { 2024-09-12T23:37:51.045 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.045 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.046 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.046 INFO:tasks.workunit.client.0.smithi136.stdout: "refresh": { 2024-09-12T23:37:51.046 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1129 2024-09-12T23:37:51.046 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.046 INFO:tasks.workunit.client.0.smithi136.stdout: "refresh_latency": { 2024-09-12T23:37:51.046 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1129, 2024-09-12T23:37:51.046 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4475936497 2024-09-12T23:37:51.046 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.046 INFO:tasks.workunit.client.0.smithi136.stdout: "restart": { 2024-09-12T23:37:51.046 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 14 2024-09-12T23:37:51.046 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.047 INFO:tasks.workunit.client.0.smithi136.stdout: "share_state": { 2024-09-12T23:37:51.047 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.047 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.047 INFO:tasks.workunit.client.0.smithi136.stdout: "share_state_bytes": { 2024-09-12T23:37:51.047 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.047 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.047 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.047 INFO:tasks.workunit.client.0.smithi136.stdout: "share_state_keys": { 2024-09-12T23:37:51.047 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.047 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.047 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.047 INFO:tasks.workunit.client.0.smithi136.stdout: "start_leader": { 2024-09-12T23:37:51.048 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.048 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.048 INFO:tasks.workunit.client.0.smithi136.stdout: "start_peon": { 2024-09-12T23:37:51.048 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2 2024-09-12T23:37:51.048 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.048 INFO:tasks.workunit.client.0.smithi136.stdout: "store_state": { 2024-09-12T23:37:51.048 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1129 2024-09-12T23:37:51.048 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.048 INFO:tasks.workunit.client.0.smithi136.stdout: "store_state_bytes": { 2024-09-12T23:37:51.048 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1129, 2024-09-12T23:37:51.048 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 9129240 2024-09-12T23:37:51.049 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.049 INFO:tasks.workunit.client.0.smithi136.stdout: "store_state_keys": { 2024-09-12T23:37:51.049 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1129, 2024-09-12T23:37:51.049 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 7960 2024-09-12T23:37:51.049 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.049 INFO:tasks.workunit.client.0.smithi136.stdout: "store_state_latency": { 2024-09-12T23:37:51.049 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1129, 2024-09-12T23:37:51.049 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 339995118 2024-09-12T23:37:51.049 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.049 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.049 INFO:tasks.workunit.client.0.smithi136.stdout: "prioritycache": { 2024-09-12T23:37:51.050 INFO:tasks.workunit.client.0.smithi136.stdout: "cache_bytes": { 2024-09-12T23:37:51.050 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1020054731 2024-09-12T23:37:51.050 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.050 INFO:tasks.workunit.client.0.smithi136.stdout: "heap_bytes": { 2024-09-12T23:37:51.050 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 41746432 2024-09-12T23:37:51.050 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.050 INFO:tasks.workunit.client.0.smithi136.stdout: "mapped_bytes": { 2024-09-12T23:37:51.050 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 35848192 2024-09-12T23:37:51.050 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.050 INFO:tasks.workunit.client.0.smithi136.stdout: "target_bytes": { 2024-09-12T23:37:51.050 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2147483648 2024-09-12T23:37:51.050 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.051 INFO:tasks.workunit.client.0.smithi136.stdout: "unmapped_bytes": { 2024-09-12T23:37:51.051 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5898240 2024-09-12T23:37:51.051 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.051 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.051 INFO:tasks.workunit.client.0.smithi136.stdout: "prioritycache:full": { 2024-09-12T23:37:51.051 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:51.051 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 348127232 2024-09-12T23:37:51.051 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.051 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:51.051 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.051 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.051 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:51.052 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.052 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.052 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:51.052 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 277028253 2024-09-12T23:37:51.052 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.052 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:51.052 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 254099 2024-09-12T23:37:51.052 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.052 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:51.052 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.052 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.053 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:51.053 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.053 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.053 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:51.053 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.053 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.053 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:51.053 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.053 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.053 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:51.053 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.054 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.054 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:51.054 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.054 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.054 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:51.054 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.054 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.054 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:51.054 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.054 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.054 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:51.054 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 70844880 2024-09-12T23:37:51.055 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.055 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.055 INFO:tasks.workunit.client.0.smithi136.stdout: "prioritycache:inc": { 2024-09-12T23:37:51.055 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:51.055 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 348127232 2024-09-12T23:37:51.055 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.055 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:51.055 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.055 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.055 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:51.055 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.056 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.056 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:51.056 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 277028253 2024-09-12T23:37:51.056 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.056 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:51.056 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 40255 2024-09-12T23:37:51.056 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.056 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:51.056 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.056 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.056 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:51.056 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.057 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.057 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:51.057 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.057 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.057 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:51.057 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.057 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.057 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:51.057 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.057 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.057 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:51.058 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.058 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.058 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:51.058 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.058 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.058 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:51.058 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.058 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.058 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:51.058 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 71058724 2024-09-12T23:37:51.058 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.059 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.059 INFO:tasks.workunit.client.0.smithi136.stdout: "prioritycache:kv": { 2024-09-12T23:37:51.059 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:51.059 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 322961408 2024-09-12T23:37:51.059 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.059 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:51.059 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67108864 2024-09-12T23:37:51.059 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.059 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:51.059 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.059 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.059 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:51.060 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 184685502 2024-09-12T23:37:51.060 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.060 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:51.060 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.060 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.060 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:51.060 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.060 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.060 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:51.060 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.060 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.060 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:51.061 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.061 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.061 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:51.061 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.061 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.061 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:51.061 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.061 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.061 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:51.061 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.061 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.061 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:51.062 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.062 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.062 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:51.062 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.062 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.062 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:51.062 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 71167042 2024-09-12T23:37:51.062 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.062 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.062 INFO:tasks.workunit.client.0.smithi136.stdout: "rocksdb": { 2024-09-12T23:37:51.062 INFO:tasks.workunit.client.0.smithi136.stdout: "compact": { 2024-09-12T23:37:51.062 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.063 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.063 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_completed": { 2024-09-12T23:37:51.063 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5 2024-09-12T23:37:51.063 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.063 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_lasted": { 2024-09-12T23:37:51.063 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 39206331 2024-09-12T23:37:51.063 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.063 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_queue_len": { 2024-09-12T23:37:51.063 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.063 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.063 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_queue_merge": { 2024-09-12T23:37:51.063 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.064 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.064 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_running": { 2024-09-12T23:37:51.064 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.064 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.064 INFO:tasks.workunit.client.0.smithi136.stdout: "get_latency": { 2024-09-12T23:37:51.064 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 47932, 2024-09-12T23:37:51.064 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 248442117 2024-09-12T23:37:51.064 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.064 INFO:tasks.workunit.client.0.smithi136.stdout: "rocksdb_write_delay_time": { 2024-09-12T23:37:51.064 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.064 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.064 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.065 INFO:tasks.workunit.client.0.smithi136.stdout: "rocksdb_write_memtable_time": { 2024-09-12T23:37:51.065 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.065 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.065 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.065 INFO:tasks.workunit.client.0.smithi136.stdout: "rocksdb_write_pre_and_post_time": { 2024-09-12T23:37:51.065 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.065 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.065 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.065 INFO:tasks.workunit.client.0.smithi136.stdout: "rocksdb_write_wal_time": { 2024-09-12T23:37:51.065 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.065 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.065 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.065 INFO:tasks.workunit.client.0.smithi136.stdout: "submit_latency": { 2024-09-12T23:37:51.066 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.066 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.066 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.066 INFO:tasks.workunit.client.0.smithi136.stdout: "submit_sync_latency": { 2024-09-12T23:37:51.066 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 2481, 2024-09-12T23:37:51.066 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 808510871 2024-09-12T23:37:51.066 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.066 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.066 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.066 INFO:tasks.workunit.client.0.smithi136.stdout: "mon.48b2ee34c5040a1abbee529ca5897a6d3827a23a": { 2024-09-12T23:37:51.066 INFO:tasks.workunit.client.0.smithi136.stdout: "mon": { 2024-09-12T23:37:51.067 INFO:tasks.workunit.client.0.smithi136.stdout: "election_call": { 2024-09-12T23:37:51.067 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 3 2024-09-12T23:37:51.067 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.067 INFO:tasks.workunit.client.0.smithi136.stdout: "election_lose": { 2024-09-12T23:37:51.067 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.067 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.067 INFO:tasks.workunit.client.0.smithi136.stdout: "election_win": { 2024-09-12T23:37:51.067 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 3 2024-09-12T23:37:51.067 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.067 INFO:tasks.workunit.client.0.smithi136.stdout: "num_elections": { 2024-09-12T23:37:51.067 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 11 2024-09-12T23:37:51.067 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.067 INFO:tasks.workunit.client.0.smithi136.stdout: "num_sessions": { 2024-09-12T23:37:51.068 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 6 2024-09-12T23:37:51.068 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.068 INFO:tasks.workunit.client.0.smithi136.stdout: "session_add": { 2024-09-12T23:37:51.068 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 3 2024-09-12T23:37:51.068 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.068 INFO:tasks.workunit.client.0.smithi136.stdout: "session_rm": { 2024-09-12T23:37:51.068 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 199 2024-09-12T23:37:51.068 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.068 INFO:tasks.workunit.client.0.smithi136.stdout: "session_trim": { 2024-09-12T23:37:51.068 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.068 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.068 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.069 INFO:tasks.workunit.client.0.smithi136.stdout: "paxos": { 2024-09-12T23:37:51.069 INFO:tasks.workunit.client.0.smithi136.stdout: "accept_timeout": { 2024-09-12T23:37:51.069 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.069 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.069 INFO:tasks.workunit.client.0.smithi136.stdout: "begin": { 2024-09-12T23:37:51.069 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1291 2024-09-12T23:37:51.069 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.069 INFO:tasks.workunit.client.0.smithi136.stdout: "begin_bytes": { 2024-09-12T23:37:51.069 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1291, 2024-09-12T23:37:51.069 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5774707 2024-09-12T23:37:51.069 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.069 INFO:tasks.workunit.client.0.smithi136.stdout: "begin_keys": { 2024-09-12T23:37:51.070 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1291, 2024-09-12T23:37:51.070 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 3873 2024-09-12T23:37:51.070 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.070 INFO:tasks.workunit.client.0.smithi136.stdout: "begin_latency": { 2024-09-12T23:37:51.070 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1291, 2024-09-12T23:37:51.070 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 351992952 2024-09-12T23:37:51.070 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.070 INFO:tasks.workunit.client.0.smithi136.stdout: "collect": { 2024-09-12T23:37:51.070 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.070 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.070 INFO:tasks.workunit.client.0.smithi136.stdout: "collect_bytes": { 2024-09-12T23:37:51.070 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.071 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.071 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.071 INFO:tasks.workunit.client.0.smithi136.stdout: "collect_keys": { 2024-09-12T23:37:51.071 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.071 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.071 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.071 INFO:tasks.workunit.client.0.smithi136.stdout: "collect_latency": { 2024-09-12T23:37:51.071 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.071 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.071 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.071 INFO:tasks.workunit.client.0.smithi136.stdout: "collect_timeout": { 2024-09-12T23:37:51.071 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.072 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.072 INFO:tasks.workunit.client.0.smithi136.stdout: "collect_uncommitted": { 2024-09-12T23:37:51.072 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.072 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.072 INFO:tasks.workunit.client.0.smithi136.stdout: "commit": { 2024-09-12T23:37:51.072 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1291 2024-09-12T23:37:51.072 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.072 INFO:tasks.workunit.client.0.smithi136.stdout: "commit_bytes": { 2024-09-12T23:37:51.072 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1291, 2024-09-12T23:37:51.072 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5681366 2024-09-12T23:37:51.072 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.072 INFO:tasks.workunit.client.0.smithi136.stdout: "commit_keys": { 2024-09-12T23:37:51.073 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1291, 2024-09-12T23:37:51.073 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 7678 2024-09-12T23:37:51.073 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.073 INFO:tasks.workunit.client.0.smithi136.stdout: "commit_latency": { 2024-09-12T23:37:51.073 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1291, 2024-09-12T23:37:51.073 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 562423819 2024-09-12T23:37:51.073 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.073 INFO:tasks.workunit.client.0.smithi136.stdout: "lease_ack_timeout": { 2024-09-12T23:37:51.073 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.073 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.073 INFO:tasks.workunit.client.0.smithi136.stdout: "lease_timeout": { 2024-09-12T23:37:51.073 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.074 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.074 INFO:tasks.workunit.client.0.smithi136.stdout: "new_pn": { 2024-09-12T23:37:51.074 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2 2024-09-12T23:37:51.074 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.074 INFO:tasks.workunit.client.0.smithi136.stdout: "new_pn_latency": { 2024-09-12T23:37:51.074 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 2, 2024-09-12T23:37:51.074 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.074 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.074 INFO:tasks.workunit.client.0.smithi136.stdout: "refresh": { 2024-09-12T23:37:51.074 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1293 2024-09-12T23:37:51.074 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.074 INFO:tasks.workunit.client.0.smithi136.stdout: "refresh_latency": { 2024-09-12T23:37:51.075 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1293, 2024-09-12T23:37:51.075 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5747888928 2024-09-12T23:37:51.075 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.075 INFO:tasks.workunit.client.0.smithi136.stdout: "restart": { 2024-09-12T23:37:51.075 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 14 2024-09-12T23:37:51.075 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.075 INFO:tasks.workunit.client.0.smithi136.stdout: "share_state": { 2024-09-12T23:37:51.075 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2 2024-09-12T23:37:51.075 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.075 INFO:tasks.workunit.client.0.smithi136.stdout: "share_state_bytes": { 2024-09-12T23:37:51.075 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 2, 2024-09-12T23:37:51.075 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 25543 2024-09-12T23:37:51.076 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.076 INFO:tasks.workunit.client.0.smithi136.stdout: "share_state_keys": { 2024-09-12T23:37:51.076 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 2, 2024-09-12T23:37:51.076 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 8 2024-09-12T23:37:51.076 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.076 INFO:tasks.workunit.client.0.smithi136.stdout: "start_leader": { 2024-09-12T23:37:51.076 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 3 2024-09-12T23:37:51.076 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.076 INFO:tasks.workunit.client.0.smithi136.stdout: "start_peon": { 2024-09-12T23:37:51.076 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.076 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.076 INFO:tasks.workunit.client.0.smithi136.stdout: "store_state": { 2024-09-12T23:37:51.077 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.077 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.077 INFO:tasks.workunit.client.0.smithi136.stdout: "store_state_bytes": { 2024-09-12T23:37:51.077 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.077 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.077 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.077 INFO:tasks.workunit.client.0.smithi136.stdout: "store_state_keys": { 2024-09-12T23:37:51.077 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.077 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.077 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.077 INFO:tasks.workunit.client.0.smithi136.stdout: "store_state_latency": { 2024-09-12T23:37:51.078 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.078 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.078 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.078 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.078 INFO:tasks.workunit.client.0.smithi136.stdout: "prioritycache": { 2024-09-12T23:37:51.078 INFO:tasks.workunit.client.0.smithi136.stdout: "cache_bytes": { 2024-09-12T23:37:51.078 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1020054731 2024-09-12T23:37:51.078 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.078 INFO:tasks.workunit.client.0.smithi136.stdout: "heap_bytes": { 2024-09-12T23:37:51.078 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 47947776 2024-09-12T23:37:51.078 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.078 INFO:tasks.workunit.client.0.smithi136.stdout: "mapped_bytes": { 2024-09-12T23:37:51.079 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 44032000 2024-09-12T23:37:51.079 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.079 INFO:tasks.workunit.client.0.smithi136.stdout: "target_bytes": { 2024-09-12T23:37:51.079 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2147483648 2024-09-12T23:37:51.079 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.079 INFO:tasks.workunit.client.0.smithi136.stdout: "unmapped_bytes": { 2024-09-12T23:37:51.079 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 3915776 2024-09-12T23:37:51.079 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.079 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.079 INFO:tasks.workunit.client.0.smithi136.stdout: "prioritycache:full": { 2024-09-12T23:37:51.079 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:51.079 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 348127232 2024-09-12T23:37:51.080 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.080 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:51.080 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.080 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.080 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:51.080 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.080 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.080 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:51.080 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 277029658 2024-09-12T23:37:51.080 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.080 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:51.080 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 253669 2024-09-12T23:37:51.081 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.081 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:51.081 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.081 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.081 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:51.081 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.081 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.081 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:51.081 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.081 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.081 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:51.081 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.082 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.082 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:51.082 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.082 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.082 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:51.082 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.082 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.082 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:51.082 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.082 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.082 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:51.082 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.083 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.083 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:51.083 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 70843905 2024-09-12T23:37:51.083 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.083 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.083 INFO:tasks.workunit.client.0.smithi136.stdout: "prioritycache:inc": { 2024-09-12T23:37:51.083 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:51.083 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 348127232 2024-09-12T23:37:51.083 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.083 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:51.083 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.083 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.084 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:51.084 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.084 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.084 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:51.084 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 277029658 2024-09-12T23:37:51.084 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.084 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:51.084 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 36939 2024-09-12T23:37:51.084 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.084 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:51.084 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.085 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.085 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:51.085 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.085 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.085 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:51.085 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.085 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.085 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:51.085 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.085 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.085 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:51.085 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.086 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.086 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:51.086 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.086 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.086 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:51.086 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.086 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.086 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:51.086 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.086 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.086 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:51.086 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 71060635 2024-09-12T23:37:51.086 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.087 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.087 INFO:tasks.workunit.client.0.smithi136.stdout: "prioritycache:kv": { 2024-09-12T23:37:51.087 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:51.087 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 322961408 2024-09-12T23:37:51.087 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.087 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:51.087 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67108864 2024-09-12T23:37:51.087 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.087 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:51.087 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.087 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.088 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:51.088 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 184686438 2024-09-12T23:37:51.088 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.088 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:51.088 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.088 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.088 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:51.088 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.088 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.088 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:51.088 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.088 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.089 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:51.089 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.089 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.089 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:51.089 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.089 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.089 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:51.089 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.089 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.089 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:51.089 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.090 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.090 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:51.090 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.090 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.090 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:51.090 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.090 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.090 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:51.090 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 71166106 2024-09-12T23:37:51.090 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.090 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.090 INFO:tasks.workunit.client.0.smithi136.stdout: "rocksdb": { 2024-09-12T23:37:51.091 INFO:tasks.workunit.client.0.smithi136.stdout: "compact": { 2024-09-12T23:37:51.091 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.091 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.091 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_completed": { 2024-09-12T23:37:51.091 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5 2024-09-12T23:37:51.091 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.091 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_lasted": { 2024-09-12T23:37:51.091 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 26925184 2024-09-12T23:37:51.091 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.091 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_queue_len": { 2024-09-12T23:37:51.091 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.091 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.092 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_queue_merge": { 2024-09-12T23:37:51.092 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.092 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.092 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_running": { 2024-09-12T23:37:51.092 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.092 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.092 INFO:tasks.workunit.client.0.smithi136.stdout: "get_latency": { 2024-09-12T23:37:51.092 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 57725, 2024-09-12T23:37:51.092 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 439413075 2024-09-12T23:37:51.092 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.092 INFO:tasks.workunit.client.0.smithi136.stdout: "rocksdb_write_delay_time": { 2024-09-12T23:37:51.093 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.093 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.093 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.093 INFO:tasks.workunit.client.0.smithi136.stdout: "rocksdb_write_memtable_time": { 2024-09-12T23:37:51.093 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.093 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.093 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.093 INFO:tasks.workunit.client.0.smithi136.stdout: "rocksdb_write_pre_and_post_time": { 2024-09-12T23:37:51.093 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.093 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.093 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.093 INFO:tasks.workunit.client.0.smithi136.stdout: "rocksdb_write_wal_time": { 2024-09-12T23:37:51.094 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.094 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.094 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.094 INFO:tasks.workunit.client.0.smithi136.stdout: "submit_latency": { 2024-09-12T23:37:51.094 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.094 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.094 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.094 INFO:tasks.workunit.client.0.smithi136.stdout: "submit_sync_latency": { 2024-09-12T23:37:51.094 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 2803, 2024-09-12T23:37:51.094 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 951053618 2024-09-12T23:37:51.094 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.094 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.095 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.095 INFO:tasks.workunit.client.0.smithi136.stdout: "osd.0": { 2024-09-12T23:37:51.095 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs": { 2024-09-12T23:37:51.095 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_slow_fallback": { 2024-09-12T23:37:51.095 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.095 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.095 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_slow_size_fallback": { 2024-09-12T23:37:51.095 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.095 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.095 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit_db": { 2024-09-12T23:37:51.095 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 65536 2024-09-12T23:37:51.095 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.096 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit_main": { 2024-09-12T23:37:51.096 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.096 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.096 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit_wal": { 2024-09-12T23:37:51.096 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.096 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.096 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_written_slow": { 2024-09-12T23:37:51.096 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.096 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.096 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_written_sst": { 2024-09-12T23:37:51.096 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 12288 2024-09-12T23:37:51.096 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.097 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_written_wal": { 2024-09-12T23:37:51.097 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2363392 2024-09-12T23:37:51.097 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.097 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_lat": { 2024-09-12T23:37:51.097 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.097 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.097 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.097 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_lock_lat": { 2024-09-12T23:37:51.097 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.097 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.097 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.097 INFO:tasks.workunit.client.0.smithi136.stdout: "db_total_bytes": { 2024-09-12T23:37:51.098 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 95995035648 2024-09-12T23:37:51.098 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.098 INFO:tasks.workunit.client.0.smithi136.stdout: "db_used_bytes": { 2024-09-12T23:37:51.098 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 304480256 2024-09-12T23:37:51.098 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.098 INFO:tasks.workunit.client.0.smithi136.stdout: "log_bytes": { 2024-09-12T23:37:51.098 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1572864 2024-09-12T23:37:51.098 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.098 INFO:tasks.workunit.client.0.smithi136.stdout: "logged_bytes": { 2024-09-12T23:37:51.098 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1433600 2024-09-12T23:37:51.098 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.098 INFO:tasks.workunit.client.0.smithi136.stdout: "max_bytes_db": { 2024-09-12T23:37:51.099 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 312868864 2024-09-12T23:37:51.099 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.099 INFO:tasks.workunit.client.0.smithi136.stdout: "max_bytes_slow": { 2024-09-12T23:37:51.099 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.099 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.099 INFO:tasks.workunit.client.0.smithi136.stdout: "max_bytes_wal": { 2024-09-12T23:37:51.099 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.099 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.099 INFO:tasks.workunit.client.0.smithi136.stdout: "num_files": { 2024-09-12T23:37:51.099 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 14 2024-09-12T23:37:51.099 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.100 INFO:tasks.workunit.client.0.smithi136.stdout: "read_bytes": { 2024-09-12T23:37:51.100 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 195697 2024-09-12T23:37:51.100 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.100 INFO:tasks.workunit.client.0.smithi136.stdout: "read_count": { 2024-09-12T23:37:51.100 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 23 2024-09-12T23:37:51.100 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.100 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes": { 2024-09-12T23:37:51.100 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1269760 2024-09-12T23:37:51.100 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.100 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes_db": { 2024-09-12T23:37:51.100 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1273856 2024-09-12T23:37:51.101 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.101 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes_slow": { 2024-09-12T23:37:51.101 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.101 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.101 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes_wal": { 2024-09-12T23:37:51.101 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.101 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.101 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_count": { 2024-09-12T23:37:51.101 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 15 2024-09-12T23:37:51.101 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.101 INFO:tasks.workunit.client.0.smithi136.stdout: "read_prefetch_bytes": { 2024-09-12T23:37:51.102 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4789 2024-09-12T23:37:51.102 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.102 INFO:tasks.workunit.client.0.smithi136.stdout: "read_prefetch_count": { 2024-09-12T23:37:51.102 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4 2024-09-12T23:37:51.102 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.102 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_buffer_bytes": { 2024-09-12T23:37:51.102 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5589 2024-09-12T23:37:51.102 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.102 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_buffer_count": { 2024-09-12T23:37:51.102 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 24 2024-09-12T23:37:51.102 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.102 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_bytes": { 2024-09-12T23:37:51.103 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5589 2024-09-12T23:37:51.103 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.103 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_count": { 2024-09-12T23:37:51.103 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 24 2024-09-12T23:37:51.103 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.103 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes": { 2024-09-12T23:37:51.103 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.103 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.103 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes_db": { 2024-09-12T23:37:51.103 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.103 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.103 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes_slow": { 2024-09-12T23:37:51.104 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.104 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.104 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes_wal": { 2024-09-12T23:37:51.104 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.104 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.104 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_count": { 2024-09-12T23:37:51.104 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.104 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.104 INFO:tasks.workunit.client.0.smithi136.stdout: "slow_total_bytes": { 2024-09-12T23:37:51.104 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.104 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.104 INFO:tasks.workunit.client.0.smithi136.stdout: "slow_used_bytes": { 2024-09-12T23:37:51.105 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.105 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.105 INFO:tasks.workunit.client.0.smithi136.stdout: "wal_total_bytes": { 2024-09-12T23:37:51.105 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.105 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.105 INFO:tasks.workunit.client.0.smithi136.stdout: "wal_used_bytes": { 2024-09-12T23:37:51.105 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.105 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.105 INFO:tasks.workunit.client.0.smithi136.stdout: "write_bytes": { 2024-09-12T23:37:51.105 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 3870720 2024-09-12T23:37:51.105 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.105 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.106 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore": { 2024-09-12T23:37:51.106 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit": { 2024-09-12T23:37:51.106 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4096 2024-09-12T23:37:51.106 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.106 INFO:tasks.workunit.client.0.smithi136.stdout: "allocated": { 2024-09-12T23:37:51.106 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1257472 2024-09-12T23:37:51.106 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.106 INFO:tasks.workunit.client.0.smithi136.stdout: "clist_lat": { 2024-09-12T23:37:51.106 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 4, 2024-09-12T23:37:51.106 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 332907 2024-09-12T23:37:51.106 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.107 INFO:tasks.workunit.client.0.smithi136.stdout: "compress_lat": { 2024-09-12T23:37:51.107 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.107 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.107 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.107 INFO:tasks.workunit.client.0.smithi136.stdout: "compressed": { 2024-09-12T23:37:51.107 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.107 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.107 INFO:tasks.workunit.client.0.smithi136.stdout: "compressed_allocated": { 2024-09-12T23:37:51.107 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.107 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.107 INFO:tasks.workunit.client.0.smithi136.stdout: "compressed_original": { 2024-09-12T23:37:51.107 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.108 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.108 INFO:tasks.workunit.client.0.smithi136.stdout: "csum_lat": { 2024-09-12T23:37:51.108 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 22, 2024-09-12T23:37:51.108 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 134139 2024-09-12T23:37:51.108 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.108 INFO:tasks.workunit.client.0.smithi136.stdout: "decompress_lat": { 2024-09-12T23:37:51.108 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.108 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.108 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.108 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_commit_lat": { 2024-09-12T23:37:51.108 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 333, 2024-09-12T23:37:51.108 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 119320326 2024-09-12T23:37:51.109 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.109 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_final_lat": { 2024-09-12T23:37:51.109 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 330, 2024-09-12T23:37:51.109 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 10918683 2024-09-12T23:37:51.109 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.109 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_flush_lat": { 2024-09-12T23:37:51.109 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 333, 2024-09-12T23:37:51.109 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 818662 2024-09-12T23:37:51.109 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.109 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_sync_lat": { 2024-09-12T23:37:51.109 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 333, 2024-09-12T23:37:51.110 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 120138988 2024-09-12T23:37:51.110 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.110 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_get_keys_lat": { 2024-09-12T23:37:51.110 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.110 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.110 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.110 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_get_values_lat": { 2024-09-12T23:37:51.110 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1, 2024-09-12T23:37:51.110 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 49690 2024-09-12T23:37:51.110 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.110 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_lower_bound_lat": { 2024-09-12T23:37:51.111 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 4, 2024-09-12T23:37:51.111 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 16074 2024-09-12T23:37:51.111 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.111 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_next_lat": { 2024-09-12T23:37:51.111 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.111 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.111 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.111 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_seek_to_first_lat": { 2024-09-12T23:37:51.111 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.111 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.111 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.111 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_upper_bound_lat": { 2024-09-12T23:37:51.112 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 4, 2024-09-12T23:37:51.112 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 23121 2024-09-12T23:37:51.112 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.112 INFO:tasks.workunit.client.0.smithi136.stdout: "onode_hits": { 2024-09-12T23:37:51.112 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 716 2024-09-12T23:37:51.112 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.112 INFO:tasks.workunit.client.0.smithi136.stdout: "onode_misses": { 2024-09-12T23:37:51.112 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 222 2024-09-12T23:37:51.112 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.112 INFO:tasks.workunit.client.0.smithi136.stdout: "read_lat": { 2024-09-12T23:37:51.112 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 100, 2024-09-12T23:37:51.112 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4193179 2024-09-12T23:37:51.113 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.113 INFO:tasks.workunit.client.0.smithi136.stdout: "read_onode_meta_lat": { 2024-09-12T23:37:51.113 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 244, 2024-09-12T23:37:51.113 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 371027 2024-09-12T23:37:51.113 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.113 INFO:tasks.workunit.client.0.smithi136.stdout: "read_wait_aio_lat": { 2024-09-12T23:37:51.113 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 144, 2024-09-12T23:37:51.113 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5076855 2024-09-12T23:37:51.113 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.113 INFO:tasks.workunit.client.0.smithi136.stdout: "reads_with_retries": { 2024-09-12T23:37:51.113 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.114 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.114 INFO:tasks.workunit.client.0.smithi136.stdout: "remove_lat": { 2024-09-12T23:37:51.114 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 2, 2024-09-12T23:37:51.114 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 81972 2024-09-12T23:37:51.114 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.114 INFO:tasks.workunit.client.0.smithi136.stdout: "state_aio_wait_lat": { 2024-09-12T23:37:51.114 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 443, 2024-09-12T23:37:51.114 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 8840179 2024-09-12T23:37:51.114 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.114 INFO:tasks.workunit.client.0.smithi136.stdout: "state_deferred_aio_wait_lat": { 2024-09-12T23:37:51.114 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 96, 2024-09-12T23:37:51.114 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 9066660 2024-09-12T23:37:51.115 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.115 INFO:tasks.workunit.client.0.smithi136.stdout: "state_deferred_cleanup_lat": { 2024-09-12T23:37:51.115 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 96, 2024-09-12T23:37:51.115 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 961368433264 2024-09-12T23:37:51.115 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.115 INFO:tasks.workunit.client.0.smithi136.stdout: "state_deferred_queued_lat": { 2024-09-12T23:37:51.115 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 96, 2024-09-12T23:37:51.115 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 126682261766 2024-09-12T23:37:51.115 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.115 INFO:tasks.workunit.client.0.smithi136.stdout: "state_done_lat": { 2024-09-12T23:37:51.115 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 443, 2024-09-12T23:37:51.115 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 66669853342 2024-09-12T23:37:51.116 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.116 INFO:tasks.workunit.client.0.smithi136.stdout: "state_finishing_lat": { 2024-09-12T23:37:51.116 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 443, 2024-09-12T23:37:51.116 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 82073 2024-09-12T23:37:51.116 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.116 INFO:tasks.workunit.client.0.smithi136.stdout: "state_io_done_lat": { 2024-09-12T23:37:51.116 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 443, 2024-09-12T23:37:51.116 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 733536 2024-09-12T23:37:51.116 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.116 INFO:tasks.workunit.client.0.smithi136.stdout: "state_kv_commiting_lat": { 2024-09-12T23:37:51.116 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 443, 2024-09-12T23:37:51.116 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 188705893 2024-09-12T23:37:51.117 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.117 INFO:tasks.workunit.client.0.smithi136.stdout: "state_kv_done_lat": { 2024-09-12T23:37:51.117 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 443, 2024-09-12T23:37:51.117 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 237987 2024-09-12T23:37:51.117 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.117 INFO:tasks.workunit.client.0.smithi136.stdout: "state_kv_queued_lat": { 2024-09-12T23:37:51.117 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 443, 2024-09-12T23:37:51.117 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 117485692 2024-09-12T23:37:51.117 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.117 INFO:tasks.workunit.client.0.smithi136.stdout: "state_prepare_lat": { 2024-09-12T23:37:51.117 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 443, 2024-09-12T23:37:51.117 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 35501236 2024-09-12T23:37:51.118 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.118 INFO:tasks.workunit.client.0.smithi136.stdout: "stored": { 2024-09-12T23:37:51.118 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 882305 2024-09-12T23:37:51.118 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.118 INFO:tasks.workunit.client.0.smithi136.stdout: "truncate_lat": { 2024-09-12T23:37:51.118 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 56, 2024-09-12T23:37:51.118 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 457853 2024-09-12T23:37:51.118 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.118 INFO:tasks.workunit.client.0.smithi136.stdout: "txc_commit_lat": { 2024-09-12T23:37:51.118 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 443, 2024-09-12T23:37:51.118 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 351375344 2024-09-12T23:37:51.119 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.119 INFO:tasks.workunit.client.0.smithi136.stdout: "txc_submit_lat": { 2024-09-12T23:37:51.119 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 443, 2024-09-12T23:37:51.119 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 42322019 2024-09-12T23:37:51.119 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.119 INFO:tasks.workunit.client.0.smithi136.stdout: "txc_throttle_lat": { 2024-09-12T23:37:51.119 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 443, 2024-09-12T23:37:51.119 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4507977 2024-09-12T23:37:51.119 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.119 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.119 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache": { 2024-09-12T23:37:51.120 INFO:tasks.workunit.client.0.smithi136.stdout: "cache_bytes": { 2024-09-12T23:37:51.120 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1900976301 2024-09-12T23:37:51.120 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.120 INFO:tasks.workunit.client.0.smithi136.stdout: "heap_bytes": { 2024-09-12T23:37:51.120 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 273072128 2024-09-12T23:37:51.120 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.120 INFO:tasks.workunit.client.0.smithi136.stdout: "mapped_bytes": { 2024-09-12T23:37:51.120 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 272375808 2024-09-12T23:37:51.120 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.120 INFO:tasks.workunit.client.0.smithi136.stdout: "target_bytes": { 2024-09-12T23:37:51.120 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 3183861964 2024-09-12T23:37:51.120 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.121 INFO:tasks.workunit.client.0.smithi136.stdout: "unmapped_bytes": { 2024-09-12T23:37:51.121 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 696320 2024-09-12T23:37:51.121 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.121 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.121 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:data": { 2024-09-12T23:37:51.121 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:51.121 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 159383552 2024-09-12T23:37:51.121 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.121 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:51.121 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.121 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.122 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:51.122 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.122 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.122 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:51.122 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 87868701 2024-09-12T23:37:51.122 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.122 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:51.122 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.122 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.122 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:51.122 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.122 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.123 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:51.123 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.123 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.123 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:51.123 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.123 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.123 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:51.123 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.123 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.123 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:51.123 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 16384 2024-09-12T23:37:51.123 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.124 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:51.124 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.124 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.124 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:51.124 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.124 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.124 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:51.124 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.124 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.124 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:51.124 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 71498467 2024-09-12T23:37:51.124 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.125 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.125 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:kv": { 2024-09-12T23:37:51.125 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:51.125 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 796917760 2024-09-12T23:37:51.125 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.125 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:51.125 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67108864 2024-09-12T23:37:51.125 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.125 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:51.125 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.126 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.126 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:51.126 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 659015264 2024-09-12T23:37:51.126 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.126 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:51.126 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.126 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.126 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:51.126 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.126 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.126 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:51.126 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.127 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.127 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:51.127 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.127 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.127 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:51.127 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.127 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.127 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:51.127 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2128 2024-09-12T23:37:51.127 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.127 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:51.127 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.128 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.128 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:51.128 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.128 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.128 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:51.128 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.128 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.128 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:51.128 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 70791504 2024-09-12T23:37:51.128 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.128 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.129 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:kv_onode": { 2024-09-12T23:37:51.129 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:51.129 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 192937984 2024-09-12T23:37:51.129 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.129 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:51.129 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67108864 2024-09-12T23:37:51.129 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.129 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:51.129 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 464 2024-09-12T23:37:51.129 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.129 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:51.129 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 58579134 2024-09-12T23:37:51.130 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.130 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:51.130 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.130 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.130 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:51.130 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.130 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.130 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:51.130 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.130 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.130 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:51.130 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.131 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.131 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:51.131 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.131 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.131 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:51.131 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.131 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.131 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:51.131 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.131 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.131 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:51.131 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.132 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.132 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:51.132 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.132 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.132 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:51.132 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67249522 2024-09-12T23:37:51.132 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.132 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.132 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:meta": { 2024-09-12T23:37:51.132 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:51.132 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 729808896 2024-09-12T23:37:51.133 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.133 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:51.133 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.133 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.133 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:51.133 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.133 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.133 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:51.133 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 659015264 2024-09-12T23:37:51.133 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.133 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:51.133 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.134 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.134 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:51.134 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2841 2024-09-12T23:37:51.134 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.134 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:51.134 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 9944 2024-09-12T23:37:51.134 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.134 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:51.136 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.136 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.136 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:51.136 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.136 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.136 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:51.136 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 258559 2024-09-12T23:37:51.136 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.136 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:51.136 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.137 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.137 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:51.137 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.137 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.137 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:51.137 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.137 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.137 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:51.137 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 70522288 2024-09-12T23:37:51.137 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.137 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.137 INFO:tasks.workunit.client.0.smithi136.stdout: "objecter": { 2024-09-12T23:37:51.137 INFO:tasks.workunit.client.0.smithi136.stdout: "op_active": { 2024-09-12T23:37:51.137 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.138 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.138 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r": { 2024-09-12T23:37:51.138 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.138 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.138 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rmw": { 2024-09-12T23:37:51.138 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.138 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.138 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w": { 2024-09-12T23:37:51.138 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.138 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.138 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.138 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:51.138 INFO:tasks.workunit.client.0.smithi136.stdout: "numpg": { 2024-09-12T23:37:51.138 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 47 2024-09-12T23:37:51.139 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.139 INFO:tasks.workunit.client.0.smithi136.stdout: "numpg_removing": { 2024-09-12T23:37:51.139 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.139 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.139 INFO:tasks.workunit.client.0.smithi136.stdout: "op": { 2024-09-12T23:37:51.139 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 6 2024-09-12T23:37:51.139 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.139 INFO:tasks.workunit.client.0.smithi136.stdout: "op_delayed_degraded": { 2024-09-12T23:37:51.139 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.139 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.139 INFO:tasks.workunit.client.0.smithi136.stdout: "op_delayed_unreadable": { 2024-09-12T23:37:51.139 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.139 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.139 INFO:tasks.workunit.client.0.smithi136.stdout: "op_in_bytes": { 2024-09-12T23:37:51.139 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 604 2024-09-12T23:37:51.140 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.140 INFO:tasks.workunit.client.0.smithi136.stdout: "op_latency": { 2024-09-12T23:37:51.140 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 6, 2024-09-12T23:37:51.140 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 57198533 2024-09-12T23:37:51.140 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.140 INFO:tasks.workunit.client.0.smithi136.stdout: "op_out_bytes": { 2024-09-12T23:37:51.140 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.140 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.140 INFO:tasks.workunit.client.0.smithi136.stdout: "op_prepare_latency": { 2024-09-12T23:37:51.140 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 7, 2024-09-12T23:37:51.140 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5117491 2024-09-12T23:37:51.140 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.140 INFO:tasks.workunit.client.0.smithi136.stdout: "op_process_latency": { 2024-09-12T23:37:51.140 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 6, 2024-09-12T23:37:51.141 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 48672335 2024-09-12T23:37:51.141 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.141 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r": { 2024-09-12T23:37:51.141 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.141 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.141 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_latency": { 2024-09-12T23:37:51.141 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.141 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.141 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.141 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-12T23:37:51.141 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.141 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.142 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_out_bytes": { 2024-09-12T23:37:51.142 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.142 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.142 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_prepare_latency": { 2024-09-12T23:37:51.142 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.142 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.142 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.142 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_process_latency": { 2024-09-12T23:37:51.142 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.142 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.142 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.142 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw": { 2024-09-12T23:37:51.142 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.142 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.143 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_in_bytes": { 2024-09-12T23:37:51.143 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.143 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.143 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_latency": { 2024-09-12T23:37:51.143 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.143 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.143 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.143 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-12T23:37:51.143 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.143 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.143 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-12T23:37:51.143 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.143 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.143 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_out_bytes": { 2024-09-12T23:37:51.144 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.144 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.144 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_prepare_latency": { 2024-09-12T23:37:51.144 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.144 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.144 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.144 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_process_latency": { 2024-09-12T23:37:51.144 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.144 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.144 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.144 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w": { 2024-09-12T23:37:51.144 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 6 2024-09-12T23:37:51.144 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.144 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_in_bytes": { 2024-09-12T23:37:51.145 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 604 2024-09-12T23:37:51.145 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.145 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_latency": { 2024-09-12T23:37:51.145 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 6, 2024-09-12T23:37:51.145 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 57198533 2024-09-12T23:37:51.145 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.145 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-12T23:37:51.145 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.145 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.145 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_prepare_latency": { 2024-09-12T23:37:51.145 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 7, 2024-09-12T23:37:51.145 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5117491 2024-09-12T23:37:51.145 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.146 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_process_latency": { 2024-09-12T23:37:51.146 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 6, 2024-09-12T23:37:51.146 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 48672335 2024-09-12T23:37:51.146 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.146 INFO:tasks.workunit.client.0.smithi136.stdout: "op_wip": { 2024-09-12T23:37:51.146 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.146 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.146 INFO:tasks.workunit.client.0.smithi136.stdout: "recovery_bytes": { 2024-09-12T23:37:51.146 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.146 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.146 INFO:tasks.workunit.client.0.smithi136.stdout: "recovery_ops": { 2024-09-12T23:37:51.146 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.146 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.146 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_bytes": { 2024-09-12T23:37:51.147 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 95995035648 2024-09-12T23:37:51.147 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.147 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_bytes_used": { 2024-09-12T23:37:51.147 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 305745920 2024-09-12T23:37:51.147 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.147 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.147 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.147 INFO:tasks.workunit.client.0.smithi136.stdout: "osd.1": { 2024-09-12T23:37:51.147 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs": { 2024-09-12T23:37:51.147 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_slow_fallback": { 2024-09-12T23:37:51.147 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.147 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.147 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_slow_size_fallback": { 2024-09-12T23:37:51.147 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.148 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.148 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit_db": { 2024-09-12T23:37:51.148 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 65536 2024-09-12T23:37:51.148 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.148 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit_main": { 2024-09-12T23:37:51.148 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.148 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.148 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit_wal": { 2024-09-12T23:37:51.148 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.148 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.148 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_written_slow": { 2024-09-12T23:37:51.148 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.148 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.148 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_written_sst": { 2024-09-12T23:37:51.149 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 12288 2024-09-12T23:37:51.149 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.149 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_written_wal": { 2024-09-12T23:37:51.149 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2064384 2024-09-12T23:37:51.149 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.149 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_lat": { 2024-09-12T23:37:51.149 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.149 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.149 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.149 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_lock_lat": { 2024-09-12T23:37:51.149 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.149 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.149 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.150 INFO:tasks.workunit.client.0.smithi136.stdout: "db_total_bytes": { 2024-09-12T23:37:51.150 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 95995035648 2024-09-12T23:37:51.150 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.150 INFO:tasks.workunit.client.0.smithi136.stdout: "db_used_bytes": { 2024-09-12T23:37:51.150 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 304480256 2024-09-12T23:37:51.150 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.150 INFO:tasks.workunit.client.0.smithi136.stdout: "log_bytes": { 2024-09-12T23:37:51.150 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1417216 2024-09-12T23:37:51.150 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.150 INFO:tasks.workunit.client.0.smithi136.stdout: "logged_bytes": { 2024-09-12T23:37:51.150 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1277952 2024-09-12T23:37:51.150 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.150 INFO:tasks.workunit.client.0.smithi136.stdout: "max_bytes_db": { 2024-09-12T23:37:51.150 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 312868864 2024-09-12T23:37:51.151 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.151 INFO:tasks.workunit.client.0.smithi136.stdout: "max_bytes_slow": { 2024-09-12T23:37:51.151 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.151 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.151 INFO:tasks.workunit.client.0.smithi136.stdout: "max_bytes_wal": { 2024-09-12T23:37:51.151 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.151 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.151 INFO:tasks.workunit.client.0.smithi136.stdout: "num_files": { 2024-09-12T23:37:51.151 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 14 2024-09-12T23:37:51.151 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.151 INFO:tasks.workunit.client.0.smithi136.stdout: "read_bytes": { 2024-09-12T23:37:51.151 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 195697 2024-09-12T23:37:51.151 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.152 INFO:tasks.workunit.client.0.smithi136.stdout: "read_count": { 2024-09-12T23:37:51.152 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 23 2024-09-12T23:37:51.152 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.152 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes": { 2024-09-12T23:37:51.152 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1269760 2024-09-12T23:37:51.152 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.152 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes_db": { 2024-09-12T23:37:51.152 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1273856 2024-09-12T23:37:51.152 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.152 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes_slow": { 2024-09-12T23:37:51.152 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.152 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.152 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes_wal": { 2024-09-12T23:37:51.152 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.152 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.153 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_count": { 2024-09-12T23:37:51.153 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 15 2024-09-12T23:37:51.153 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.153 INFO:tasks.workunit.client.0.smithi136.stdout: "read_prefetch_bytes": { 2024-09-12T23:37:51.153 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4789 2024-09-12T23:37:51.153 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.153 INFO:tasks.workunit.client.0.smithi136.stdout: "read_prefetch_count": { 2024-09-12T23:37:51.153 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4 2024-09-12T23:37:51.153 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.153 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_buffer_bytes": { 2024-09-12T23:37:51.153 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5589 2024-09-12T23:37:51.153 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.153 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_buffer_count": { 2024-09-12T23:37:51.153 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 24 2024-09-12T23:37:51.154 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.154 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_bytes": { 2024-09-12T23:37:51.154 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5589 2024-09-12T23:37:51.154 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.154 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_count": { 2024-09-12T23:37:51.154 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 24 2024-09-12T23:37:51.154 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.154 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes": { 2024-09-12T23:37:51.154 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.154 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.154 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes_db": { 2024-09-12T23:37:51.154 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.154 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.154 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes_slow": { 2024-09-12T23:37:51.155 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.155 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.155 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes_wal": { 2024-09-12T23:37:51.155 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.155 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.155 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_count": { 2024-09-12T23:37:51.155 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.155 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.155 INFO:tasks.workunit.client.0.smithi136.stdout: "slow_total_bytes": { 2024-09-12T23:37:51.155 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.155 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.155 INFO:tasks.workunit.client.0.smithi136.stdout: "slow_used_bytes": { 2024-09-12T23:37:51.155 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.155 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.155 INFO:tasks.workunit.client.0.smithi136.stdout: "wal_total_bytes": { 2024-09-12T23:37:51.156 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.156 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.156 INFO:tasks.workunit.client.0.smithi136.stdout: "wal_used_bytes": { 2024-09-12T23:37:51.156 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.156 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.156 INFO:tasks.workunit.client.0.smithi136.stdout: "write_bytes": { 2024-09-12T23:37:51.156 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 3416064 2024-09-12T23:37:51.156 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.156 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.156 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore": { 2024-09-12T23:37:51.156 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit": { 2024-09-12T23:37:51.156 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4096 2024-09-12T23:37:51.156 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.156 INFO:tasks.workunit.client.0.smithi136.stdout: "allocated": { 2024-09-12T23:37:51.157 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 663552 2024-09-12T23:37:51.157 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.157 INFO:tasks.workunit.client.0.smithi136.stdout: "clist_lat": { 2024-09-12T23:37:51.157 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 5, 2024-09-12T23:37:51.157 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 379561 2024-09-12T23:37:51.157 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.157 INFO:tasks.workunit.client.0.smithi136.stdout: "compress_lat": { 2024-09-12T23:37:51.157 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.157 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.157 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.157 INFO:tasks.workunit.client.0.smithi136.stdout: "compressed": { 2024-09-12T23:37:51.157 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.157 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.157 INFO:tasks.workunit.client.0.smithi136.stdout: "compressed_allocated": { 2024-09-12T23:37:51.158 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.158 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.158 INFO:tasks.workunit.client.0.smithi136.stdout: "compressed_original": { 2024-09-12T23:37:51.158 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.158 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.158 INFO:tasks.workunit.client.0.smithi136.stdout: "csum_lat": { 2024-09-12T23:37:51.158 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 24, 2024-09-12T23:37:51.158 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 124553 2024-09-12T23:37:51.158 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.158 INFO:tasks.workunit.client.0.smithi136.stdout: "decompress_lat": { 2024-09-12T23:37:51.158 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.158 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.158 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.158 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_commit_lat": { 2024-09-12T23:37:51.159 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 295, 2024-09-12T23:37:51.159 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 102050526 2024-09-12T23:37:51.159 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.159 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_final_lat": { 2024-09-12T23:37:51.159 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 278, 2024-09-12T23:37:51.159 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 19355455 2024-09-12T23:37:51.159 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.159 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_flush_lat": { 2024-09-12T23:37:51.159 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 295, 2024-09-12T23:37:51.159 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 760153 2024-09-12T23:37:51.159 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.159 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_sync_lat": { 2024-09-12T23:37:51.159 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 295, 2024-09-12T23:37:51.159 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 102810679 2024-09-12T23:37:51.160 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.160 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_get_keys_lat": { 2024-09-12T23:37:51.160 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.160 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.160 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.160 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_get_values_lat": { 2024-09-12T23:37:51.160 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 3, 2024-09-12T23:37:51.160 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 40717 2024-09-12T23:37:51.160 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.160 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_lower_bound_lat": { 2024-09-12T23:37:51.160 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 7, 2024-09-12T23:37:51.160 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 38416 2024-09-12T23:37:51.160 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.161 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_next_lat": { 2024-09-12T23:37:51.161 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.161 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.161 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.161 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_seek_to_first_lat": { 2024-09-12T23:37:51.161 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.161 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.161 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.161 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_upper_bound_lat": { 2024-09-12T23:37:51.161 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 5, 2024-09-12T23:37:51.161 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 25928 2024-09-12T23:37:51.161 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.161 INFO:tasks.workunit.client.0.smithi136.stdout: "onode_hits": { 2024-09-12T23:37:51.161 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 679 2024-09-12T23:37:51.162 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.162 INFO:tasks.workunit.client.0.smithi136.stdout: "onode_misses": { 2024-09-12T23:37:51.162 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 230 2024-09-12T23:37:51.162 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.162 INFO:tasks.workunit.client.0.smithi136.stdout: "read_lat": { 2024-09-12T23:37:51.162 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 118, 2024-09-12T23:37:51.162 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4784720 2024-09-12T23:37:51.162 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.162 INFO:tasks.workunit.client.0.smithi136.stdout: "read_onode_meta_lat": { 2024-09-12T23:37:51.162 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 276, 2024-09-12T23:37:51.162 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 342650 2024-09-12T23:37:51.162 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.162 INFO:tasks.workunit.client.0.smithi136.stdout: "read_wait_aio_lat": { 2024-09-12T23:37:51.162 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 158, 2024-09-12T23:37:51.163 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5498560 2024-09-12T23:37:51.163 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.163 INFO:tasks.workunit.client.0.smithi136.stdout: "reads_with_retries": { 2024-09-12T23:37:51.163 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.163 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.163 INFO:tasks.workunit.client.0.smithi136.stdout: "remove_lat": { 2024-09-12T23:37:51.163 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 3, 2024-09-12T23:37:51.163 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 233801 2024-09-12T23:37:51.163 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.163 INFO:tasks.workunit.client.0.smithi136.stdout: "state_aio_wait_lat": { 2024-09-12T23:37:51.163 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 362, 2024-09-12T23:37:51.163 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 10768210 2024-09-12T23:37:51.163 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.164 INFO:tasks.workunit.client.0.smithi136.stdout: "state_deferred_aio_wait_lat": { 2024-09-12T23:37:51.164 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 92, 2024-09-12T23:37:51.164 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 9183607 2024-09-12T23:37:51.164 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.164 INFO:tasks.workunit.client.0.smithi136.stdout: "state_deferred_cleanup_lat": { 2024-09-12T23:37:51.164 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 92, 2024-09-12T23:37:51.164 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 579448531851 2024-09-12T23:37:51.164 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.164 INFO:tasks.workunit.client.0.smithi136.stdout: "state_deferred_queued_lat": { 2024-09-12T23:37:51.164 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 92, 2024-09-12T23:37:51.164 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 153525335504 2024-09-12T23:37:51.164 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.164 INFO:tasks.workunit.client.0.smithi136.stdout: "state_done_lat": { 2024-09-12T23:37:51.164 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 362, 2024-09-12T23:37:51.165 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 88666442401 2024-09-12T23:37:51.165 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.165 INFO:tasks.workunit.client.0.smithi136.stdout: "state_finishing_lat": { 2024-09-12T23:37:51.165 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 362, 2024-09-12T23:37:51.165 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 78889 2024-09-12T23:37:51.165 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.165 INFO:tasks.workunit.client.0.smithi136.stdout: "state_io_done_lat": { 2024-09-12T23:37:51.165 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 362, 2024-09-12T23:37:51.165 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 644822 2024-09-12T23:37:51.165 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.165 INFO:tasks.workunit.client.0.smithi136.stdout: "state_kv_commiting_lat": { 2024-09-12T23:37:51.165 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 362, 2024-09-12T23:37:51.165 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 446612270 2024-09-12T23:37:51.166 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.166 INFO:tasks.workunit.client.0.smithi136.stdout: "state_kv_done_lat": { 2024-09-12T23:37:51.166 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 362, 2024-09-12T23:37:51.166 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 217189 2024-09-12T23:37:51.166 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.166 INFO:tasks.workunit.client.0.smithi136.stdout: "state_kv_queued_lat": { 2024-09-12T23:37:51.166 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 362, 2024-09-12T23:37:51.166 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 79652699 2024-09-12T23:37:51.166 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.166 INFO:tasks.workunit.client.0.smithi136.stdout: "state_prepare_lat": { 2024-09-12T23:37:51.166 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 362, 2024-09-12T23:37:51.166 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 31013281 2024-09-12T23:37:51.166 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.166 INFO:tasks.workunit.client.0.smithi136.stdout: "stored": { 2024-09-12T23:37:51.167 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 291715 2024-09-12T23:37:51.167 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.167 INFO:tasks.workunit.client.0.smithi136.stdout: "truncate_lat": { 2024-09-12T23:37:51.167 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 51, 2024-09-12T23:37:51.167 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 434072 2024-09-12T23:37:51.167 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.167 INFO:tasks.workunit.client.0.smithi136.stdout: "txc_commit_lat": { 2024-09-12T23:37:51.167 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 362, 2024-09-12T23:37:51.167 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 568792135 2024-09-12T23:37:51.167 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.167 INFO:tasks.workunit.client.0.smithi136.stdout: "txc_submit_lat": { 2024-09-12T23:37:51.167 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 362, 2024-09-12T23:37:51.167 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 37787654 2024-09-12T23:37:51.167 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.168 INFO:tasks.workunit.client.0.smithi136.stdout: "txc_throttle_lat": { 2024-09-12T23:37:51.168 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 362, 2024-09-12T23:37:51.168 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4574541 2024-09-12T23:37:51.168 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.168 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.168 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache": { 2024-09-12T23:37:51.168 INFO:tasks.workunit.client.0.smithi136.stdout: "cache_bytes": { 2024-09-12T23:37:51.168 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1900976301 2024-09-12T23:37:51.168 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.168 INFO:tasks.workunit.client.0.smithi136.stdout: "heap_bytes": { 2024-09-12T23:37:51.168 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 272023552 2024-09-12T23:37:51.168 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.168 INFO:tasks.workunit.client.0.smithi136.stdout: "mapped_bytes": { 2024-09-12T23:37:51.168 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 271384576 2024-09-12T23:37:51.169 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.169 INFO:tasks.workunit.client.0.smithi136.stdout: "target_bytes": { 2024-09-12T23:37:51.169 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 3183861964 2024-09-12T23:37:51.169 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.169 INFO:tasks.workunit.client.0.smithi136.stdout: "unmapped_bytes": { 2024-09-12T23:37:51.169 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 638976 2024-09-12T23:37:51.169 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.169 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.169 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:data": { 2024-09-12T23:37:51.169 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:51.169 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 159383552 2024-09-12T23:37:51.169 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.169 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:51.170 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.170 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.170 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:51.170 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.170 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.170 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:51.170 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 87870230 2024-09-12T23:37:51.170 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.170 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:51.170 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.170 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.170 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:51.170 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.170 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.170 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:51.171 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.171 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.171 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:51.171 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.171 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.171 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:51.171 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.171 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.171 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:51.171 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 12288 2024-09-12T23:37:51.171 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.171 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:51.171 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.171 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.172 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:51.172 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.172 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.172 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:51.172 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.172 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.172 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:51.172 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 71501034 2024-09-12T23:37:51.172 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.172 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.172 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:kv": { 2024-09-12T23:37:51.172 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:51.172 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 796917760 2024-09-12T23:37:51.172 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.173 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:51.173 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67108864 2024-09-12T23:37:51.173 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.173 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:51.173 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.173 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.173 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:51.173 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 659026731 2024-09-12T23:37:51.173 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.173 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:51.173 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.173 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.173 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:51.173 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.174 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.174 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:51.174 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.174 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.174 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:51.174 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.174 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.174 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:51.174 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.174 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.174 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:51.174 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2128 2024-09-12T23:37:51.174 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.175 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:51.175 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.175 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.175 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:51.175 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.175 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.175 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:51.175 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.175 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.175 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:51.175 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 70780037 2024-09-12T23:37:51.175 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.175 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.175 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:kv_onode": { 2024-09-12T23:37:51.175 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:51.176 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 192937984 2024-09-12T23:37:51.176 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.176 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:51.176 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67108864 2024-09-12T23:37:51.176 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.176 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:51.176 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 464 2024-09-12T23:37:51.176 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.176 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:51.176 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 58580153 2024-09-12T23:37:51.176 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.176 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:51.177 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.177 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.177 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:51.177 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.177 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.177 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:51.177 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.177 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.177 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:51.177 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.177 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.177 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:51.177 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.177 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.178 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:51.178 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.178 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.178 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:51.178 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.178 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.178 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:51.178 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.178 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.178 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:51.178 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.178 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.178 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:51.178 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67248503 2024-09-12T23:37:51.178 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.179 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.179 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:meta": { 2024-09-12T23:37:51.179 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:51.179 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 729808896 2024-09-12T23:37:51.179 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.179 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:51.179 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.179 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.179 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:51.179 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.179 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.179 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:51.179 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 659026731 2024-09-12T23:37:51.180 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.180 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:51.180 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.180 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.180 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:51.180 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.180 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.180 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:51.180 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.180 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.180 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:51.180 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.180 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.180 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:51.181 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.181 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.181 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:51.181 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 249958 2024-09-12T23:37:51.181 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.181 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:51.181 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.181 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.181 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:51.181 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.181 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.181 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:51.181 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.181 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.182 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:51.182 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 70532207 2024-09-12T23:37:51.182 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.182 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.182 INFO:tasks.workunit.client.0.smithi136.stdout: "objecter": { 2024-09-12T23:37:51.182 INFO:tasks.workunit.client.0.smithi136.stdout: "op_active": { 2024-09-12T23:37:51.182 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.182 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.182 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r": { 2024-09-12T23:37:51.182 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.182 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.182 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rmw": { 2024-09-12T23:37:51.182 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.182 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.183 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w": { 2024-09-12T23:37:51.183 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.183 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.183 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.183 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:51.183 INFO:tasks.workunit.client.0.smithi136.stdout: "numpg": { 2024-09-12T23:37:51.183 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 37 2024-09-12T23:37:51.183 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.183 INFO:tasks.workunit.client.0.smithi136.stdout: "numpg_removing": { 2024-09-12T23:37:51.183 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.183 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.183 INFO:tasks.workunit.client.0.smithi136.stdout: "op": { 2024-09-12T23:37:51.183 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 97 2024-09-12T23:37:51.184 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.184 INFO:tasks.workunit.client.0.smithi136.stdout: "op_delayed_degraded": { 2024-09-12T23:37:51.184 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.184 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.184 INFO:tasks.workunit.client.0.smithi136.stdout: "op_delayed_unreadable": { 2024-09-12T23:37:51.184 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.184 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.184 INFO:tasks.workunit.client.0.smithi136.stdout: "op_in_bytes": { 2024-09-12T23:37:51.184 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1640827 2024-09-12T23:37:51.184 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.184 INFO:tasks.workunit.client.0.smithi136.stdout: "op_latency": { 2024-09-12T23:37:51.184 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 97, 2024-09-12T23:37:51.184 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1017073047 2024-09-12T23:37:51.184 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.184 INFO:tasks.workunit.client.0.smithi136.stdout: "op_out_bytes": { 2024-09-12T23:37:51.185 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 602 2024-09-12T23:37:51.185 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.185 INFO:tasks.workunit.client.0.smithi136.stdout: "op_prepare_latency": { 2024-09-12T23:37:51.185 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 97, 2024-09-12T23:37:51.185 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 63225715 2024-09-12T23:37:51.185 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.185 INFO:tasks.workunit.client.0.smithi136.stdout: "op_process_latency": { 2024-09-12T23:37:51.185 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 97, 2024-09-12T23:37:51.185 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 264349402 2024-09-12T23:37:51.185 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.185 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r": { 2024-09-12T23:37:51.185 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 18 2024-09-12T23:37:51.185 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.186 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_latency": { 2024-09-12T23:37:51.186 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 18, 2024-09-12T23:37:51.186 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5773383 2024-09-12T23:37:51.186 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.186 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-12T23:37:51.186 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.186 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.186 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_out_bytes": { 2024-09-12T23:37:51.186 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 602 2024-09-12T23:37:51.186 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.186 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_prepare_latency": { 2024-09-12T23:37:51.186 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 18, 2024-09-12T23:37:51.186 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4867433 2024-09-12T23:37:51.187 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.187 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_process_latency": { 2024-09-12T23:37:51.187 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 18, 2024-09-12T23:37:51.187 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4181989 2024-09-12T23:37:51.187 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.187 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw": { 2024-09-12T23:37:51.187 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 12 2024-09-12T23:37:51.187 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.187 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_in_bytes": { 2024-09-12T23:37:51.187 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.187 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.187 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_latency": { 2024-09-12T23:37:51.187 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 12, 2024-09-12T23:37:51.188 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 20095944 2024-09-12T23:37:51.188 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.188 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-12T23:37:51.188 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.188 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.188 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-12T23:37:51.188 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.188 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.188 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_out_bytes": { 2024-09-12T23:37:51.188 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.188 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.188 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_prepare_latency": { 2024-09-12T23:37:51.188 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 12, 2024-09-12T23:37:51.188 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 8126527 2024-09-12T23:37:51.189 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.189 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_process_latency": { 2024-09-12T23:37:51.189 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 12, 2024-09-12T23:37:51.189 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 18729918 2024-09-12T23:37:51.189 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.189 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w": { 2024-09-12T23:37:51.189 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67 2024-09-12T23:37:51.189 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.189 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_in_bytes": { 2024-09-12T23:37:51.189 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1640827 2024-09-12T23:37:51.189 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.189 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_latency": { 2024-09-12T23:37:51.189 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 67, 2024-09-12T23:37:51.189 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 991203720 2024-09-12T23:37:51.190 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.190 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-12T23:37:51.190 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.190 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.190 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_prepare_latency": { 2024-09-12T23:37:51.190 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 67, 2024-09-12T23:37:51.190 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 50231755 2024-09-12T23:37:51.190 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.190 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_process_latency": { 2024-09-12T23:37:51.190 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 67, 2024-09-12T23:37:51.190 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 241437495 2024-09-12T23:37:51.190 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.190 INFO:tasks.workunit.client.0.smithi136.stdout: "op_wip": { 2024-09-12T23:37:51.190 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.191 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.191 INFO:tasks.workunit.client.0.smithi136.stdout: "recovery_bytes": { 2024-09-12T23:37:51.191 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 590368 2024-09-12T23:37:51.191 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.191 INFO:tasks.workunit.client.0.smithi136.stdout: "recovery_ops": { 2024-09-12T23:37:51.191 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2 2024-09-12T23:37:51.191 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.191 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_bytes": { 2024-09-12T23:37:51.191 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 95995035648 2024-09-12T23:37:51.191 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.191 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_bytes_used": { 2024-09-12T23:37:51.191 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 305152000 2024-09-12T23:37:51.191 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.191 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.191 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.191 INFO:tasks.workunit.client.0.smithi136.stdout: "osd.2": { 2024-09-12T23:37:51.192 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs": { 2024-09-12T23:37:51.192 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_slow_fallback": { 2024-09-12T23:37:51.192 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.192 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.192 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_slow_size_fallback": { 2024-09-12T23:37:51.192 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.192 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.192 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit_db": { 2024-09-12T23:37:51.192 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 65536 2024-09-12T23:37:51.192 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.192 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit_main": { 2024-09-12T23:37:51.192 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.192 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.192 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit_wal": { 2024-09-12T23:37:51.192 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.192 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.193 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_written_slow": { 2024-09-12T23:37:51.193 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.193 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.193 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_written_sst": { 2024-09-12T23:37:51.193 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 12288 2024-09-12T23:37:51.193 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.193 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_written_wal": { 2024-09-12T23:37:51.193 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1630208 2024-09-12T23:37:51.193 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.193 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_lat": { 2024-09-12T23:37:51.193 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.193 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.193 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.193 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_lock_lat": { 2024-09-12T23:37:51.194 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.194 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.194 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.194 INFO:tasks.workunit.client.0.smithi136.stdout: "db_total_bytes": { 2024-09-12T23:37:51.194 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 95995035648 2024-09-12T23:37:51.194 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.194 INFO:tasks.workunit.client.0.smithi136.stdout: "db_used_bytes": { 2024-09-12T23:37:51.194 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 304480256 2024-09-12T23:37:51.194 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.194 INFO:tasks.workunit.client.0.smithi136.stdout: "log_bytes": { 2024-09-12T23:37:51.194 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1093632 2024-09-12T23:37:51.194 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.194 INFO:tasks.workunit.client.0.smithi136.stdout: "logged_bytes": { 2024-09-12T23:37:51.194 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 954368 2024-09-12T23:37:51.194 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.195 INFO:tasks.workunit.client.0.smithi136.stdout: "max_bytes_db": { 2024-09-12T23:37:51.195 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 312868864 2024-09-12T23:37:51.195 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.195 INFO:tasks.workunit.client.0.smithi136.stdout: "max_bytes_slow": { 2024-09-12T23:37:51.195 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.195 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.195 INFO:tasks.workunit.client.0.smithi136.stdout: "max_bytes_wal": { 2024-09-12T23:37:51.195 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.195 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.195 INFO:tasks.workunit.client.0.smithi136.stdout: "num_files": { 2024-09-12T23:37:51.195 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 14 2024-09-12T23:37:51.195 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.195 INFO:tasks.workunit.client.0.smithi136.stdout: "read_bytes": { 2024-09-12T23:37:51.195 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 195697 2024-09-12T23:37:51.195 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.195 INFO:tasks.workunit.client.0.smithi136.stdout: "read_count": { 2024-09-12T23:37:51.196 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 23 2024-09-12T23:37:51.196 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.196 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes": { 2024-09-12T23:37:51.196 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1269760 2024-09-12T23:37:51.196 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.196 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes_db": { 2024-09-12T23:37:51.196 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1273856 2024-09-12T23:37:51.196 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.196 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes_slow": { 2024-09-12T23:37:51.196 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.196 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.196 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes_wal": { 2024-09-12T23:37:51.196 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.196 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.197 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_count": { 2024-09-12T23:37:51.197 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 15 2024-09-12T23:37:51.197 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.197 INFO:tasks.workunit.client.0.smithi136.stdout: "read_prefetch_bytes": { 2024-09-12T23:37:51.197 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4789 2024-09-12T23:37:51.197 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.197 INFO:tasks.workunit.client.0.smithi136.stdout: "read_prefetch_count": { 2024-09-12T23:37:51.197 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4 2024-09-12T23:37:51.197 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.197 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_buffer_bytes": { 2024-09-12T23:37:51.197 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5589 2024-09-12T23:37:51.197 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.197 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_buffer_count": { 2024-09-12T23:37:51.197 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 24 2024-09-12T23:37:51.198 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.198 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_bytes": { 2024-09-12T23:37:51.198 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5589 2024-09-12T23:37:51.198 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.198 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_count": { 2024-09-12T23:37:51.198 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 24 2024-09-12T23:37:51.198 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.198 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes": { 2024-09-12T23:37:51.198 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.198 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.198 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes_db": { 2024-09-12T23:37:51.198 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.198 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.198 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes_slow": { 2024-09-12T23:37:51.198 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.199 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.199 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes_wal": { 2024-09-12T23:37:51.199 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.199 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.199 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_count": { 2024-09-12T23:37:51.199 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.199 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.199 INFO:tasks.workunit.client.0.smithi136.stdout: "slow_total_bytes": { 2024-09-12T23:37:51.199 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.199 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.199 INFO:tasks.workunit.client.0.smithi136.stdout: "slow_used_bytes": { 2024-09-12T23:37:51.199 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.199 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.199 INFO:tasks.workunit.client.0.smithi136.stdout: "wal_total_bytes": { 2024-09-12T23:37:51.200 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.200 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.200 INFO:tasks.workunit.client.0.smithi136.stdout: "wal_used_bytes": { 2024-09-12T23:37:51.200 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.200 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.200 INFO:tasks.workunit.client.0.smithi136.stdout: "write_bytes": { 2024-09-12T23:37:51.200 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2658304 2024-09-12T23:37:51.200 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.200 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.200 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore": { 2024-09-12T23:37:51.200 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit": { 2024-09-12T23:37:51.200 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4096 2024-09-12T23:37:51.200 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.201 INFO:tasks.workunit.client.0.smithi136.stdout: "allocated": { 2024-09-12T23:37:51.201 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 651264 2024-09-12T23:37:51.201 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.201 INFO:tasks.workunit.client.0.smithi136.stdout: "clist_lat": { 2024-09-12T23:37:51.201 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 4, 2024-09-12T23:37:51.201 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 363938 2024-09-12T23:37:51.201 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.201 INFO:tasks.workunit.client.0.smithi136.stdout: "compress_lat": { 2024-09-12T23:37:51.201 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.201 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.201 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.201 INFO:tasks.workunit.client.0.smithi136.stdout: "compressed": { 2024-09-12T23:37:51.201 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.201 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.201 INFO:tasks.workunit.client.0.smithi136.stdout: "compressed_allocated": { 2024-09-12T23:37:51.202 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.202 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.202 INFO:tasks.workunit.client.0.smithi136.stdout: "compressed_original": { 2024-09-12T23:37:51.202 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.202 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.202 INFO:tasks.workunit.client.0.smithi136.stdout: "csum_lat": { 2024-09-12T23:37:51.202 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 9, 2024-09-12T23:37:51.202 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 20087 2024-09-12T23:37:51.202 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.202 INFO:tasks.workunit.client.0.smithi136.stdout: "decompress_lat": { 2024-09-12T23:37:51.202 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.202 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.202 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.202 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_commit_lat": { 2024-09-12T23:37:51.202 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 216, 2024-09-12T23:37:51.202 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 98117890 2024-09-12T23:37:51.203 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.203 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_final_lat": { 2024-09-12T23:37:51.203 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 212, 2024-09-12T23:37:51.203 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5707053 2024-09-12T23:37:51.203 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.203 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_flush_lat": { 2024-09-12T23:37:51.203 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 216, 2024-09-12T23:37:51.203 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 629555 2024-09-12T23:37:51.203 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.203 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_sync_lat": { 2024-09-12T23:37:51.203 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 216, 2024-09-12T23:37:51.203 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 98747445 2024-09-12T23:37:51.203 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.204 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_get_keys_lat": { 2024-09-12T23:37:51.204 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.204 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.204 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.204 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_get_values_lat": { 2024-09-12T23:37:51.204 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 3, 2024-09-12T23:37:51.204 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 38992 2024-09-12T23:37:51.204 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.204 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_lower_bound_lat": { 2024-09-12T23:37:51.204 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 2, 2024-09-12T23:37:51.204 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4246 2024-09-12T23:37:51.204 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.204 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_next_lat": { 2024-09-12T23:37:51.204 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.205 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.205 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.205 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_seek_to_first_lat": { 2024-09-12T23:37:51.205 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.205 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.205 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.205 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_upper_bound_lat": { 2024-09-12T23:37:51.205 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 2, 2024-09-12T23:37:51.205 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 15665 2024-09-12T23:37:51.205 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.205 INFO:tasks.workunit.client.0.smithi136.stdout: "onode_hits": { 2024-09-12T23:37:51.205 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 486 2024-09-12T23:37:51.205 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.205 INFO:tasks.workunit.client.0.smithi136.stdout: "onode_misses": { 2024-09-12T23:37:51.205 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 207 2024-09-12T23:37:51.206 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.206 INFO:tasks.workunit.client.0.smithi136.stdout: "read_lat": { 2024-09-12T23:37:51.206 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 91, 2024-09-12T23:37:51.206 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1701769 2024-09-12T23:37:51.206 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.206 INFO:tasks.workunit.client.0.smithi136.stdout: "read_onode_meta_lat": { 2024-09-12T23:37:51.206 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 213, 2024-09-12T23:37:51.206 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 440111 2024-09-12T23:37:51.206 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.206 INFO:tasks.workunit.client.0.smithi136.stdout: "read_wait_aio_lat": { 2024-09-12T23:37:51.206 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 122, 2024-09-12T23:37:51.206 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1755222 2024-09-12T23:37:51.206 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.206 INFO:tasks.workunit.client.0.smithi136.stdout: "reads_with_retries": { 2024-09-12T23:37:51.206 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.206 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.207 INFO:tasks.workunit.client.0.smithi136.stdout: "remove_lat": { 2024-09-12T23:37:51.207 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 3, 2024-09-12T23:37:51.207 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 150110 2024-09-12T23:37:51.207 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.207 INFO:tasks.workunit.client.0.smithi136.stdout: "state_aio_wait_lat": { 2024-09-12T23:37:51.207 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 288, 2024-09-12T23:37:51.207 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 7414316 2024-09-12T23:37:51.207 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.207 INFO:tasks.workunit.client.0.smithi136.stdout: "state_deferred_aio_wait_lat": { 2024-09-12T23:37:51.207 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 82, 2024-09-12T23:37:51.207 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 8623654 2024-09-12T23:37:51.207 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.207 INFO:tasks.workunit.client.0.smithi136.stdout: "state_deferred_cleanup_lat": { 2024-09-12T23:37:51.207 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 80, 2024-09-12T23:37:51.208 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 493900778732 2024-09-12T23:37:51.208 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.208 INFO:tasks.workunit.client.0.smithi136.stdout: "state_deferred_queued_lat": { 2024-09-12T23:37:51.208 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 82, 2024-09-12T23:37:51.208 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 140893530151 2024-09-12T23:37:51.208 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.208 INFO:tasks.workunit.client.0.smithi136.stdout: "state_done_lat": { 2024-09-12T23:37:51.208 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 286, 2024-09-12T23:37:51.208 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 85018963499 2024-09-12T23:37:51.208 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.208 INFO:tasks.workunit.client.0.smithi136.stdout: "state_finishing_lat": { 2024-09-12T23:37:51.208 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 286, 2024-09-12T23:37:51.208 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 50988 2024-09-12T23:37:51.208 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.208 INFO:tasks.workunit.client.0.smithi136.stdout: "state_io_done_lat": { 2024-09-12T23:37:51.208 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 288, 2024-09-12T23:37:51.209 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 592890 2024-09-12T23:37:51.209 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.209 INFO:tasks.workunit.client.0.smithi136.stdout: "state_kv_commiting_lat": { 2024-09-12T23:37:51.209 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 288, 2024-09-12T23:37:51.209 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 199039664 2024-09-12T23:37:51.209 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.209 INFO:tasks.workunit.client.0.smithi136.stdout: "state_kv_done_lat": { 2024-09-12T23:37:51.209 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 288, 2024-09-12T23:37:51.209 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 160077 2024-09-12T23:37:51.209 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.209 INFO:tasks.workunit.client.0.smithi136.stdout: "state_kv_queued_lat": { 2024-09-12T23:37:51.209 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 288, 2024-09-12T23:37:51.209 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 224601766 2024-09-12T23:37:51.209 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.209 INFO:tasks.workunit.client.0.smithi136.stdout: "state_prepare_lat": { 2024-09-12T23:37:51.210 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 288, 2024-09-12T23:37:51.210 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 41974844 2024-09-12T23:37:51.210 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.210 INFO:tasks.workunit.client.0.smithi136.stdout: "stored": { 2024-09-12T23:37:51.210 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 291536 2024-09-12T23:37:51.210 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.210 INFO:tasks.workunit.client.0.smithi136.stdout: "truncate_lat": { 2024-09-12T23:37:51.210 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 43, 2024-09-12T23:37:51.210 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 363573 2024-09-12T23:37:51.210 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.210 INFO:tasks.workunit.client.0.smithi136.stdout: "txc_commit_lat": { 2024-09-12T23:37:51.210 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 288, 2024-09-12T23:37:51.210 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 473698711 2024-09-12T23:37:51.210 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.211 INFO:tasks.workunit.client.0.smithi136.stdout: "txc_submit_lat": { 2024-09-12T23:37:51.211 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 288, 2024-09-12T23:37:51.211 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 48019125 2024-09-12T23:37:51.211 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.211 INFO:tasks.workunit.client.0.smithi136.stdout: "txc_throttle_lat": { 2024-09-12T23:37:51.211 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 288, 2024-09-12T23:37:51.211 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 19681733 2024-09-12T23:37:51.211 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.211 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.211 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache": { 2024-09-12T23:37:51.211 INFO:tasks.workunit.client.0.smithi136.stdout: "cache_bytes": { 2024-09-12T23:37:51.211 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1900976301 2024-09-12T23:37:51.211 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.211 INFO:tasks.workunit.client.0.smithi136.stdout: "heap_bytes": { 2024-09-12T23:37:51.212 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 270909440 2024-09-12T23:37:51.212 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.212 INFO:tasks.workunit.client.0.smithi136.stdout: "mapped_bytes": { 2024-09-12T23:37:51.212 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 269942784 2024-09-12T23:37:51.212 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.212 INFO:tasks.workunit.client.0.smithi136.stdout: "target_bytes": { 2024-09-12T23:37:51.212 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 3183861964 2024-09-12T23:37:51.212 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.212 INFO:tasks.workunit.client.0.smithi136.stdout: "unmapped_bytes": { 2024-09-12T23:37:51.212 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 966656 2024-09-12T23:37:51.212 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.212 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.212 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:data": { 2024-09-12T23:37:51.212 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:51.212 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 159383552 2024-09-12T23:37:51.212 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.213 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:51.213 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.213 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.213 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:51.213 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.213 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.213 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:51.213 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 87871148 2024-09-12T23:37:51.213 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.213 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:51.213 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.213 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.213 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:51.213 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.213 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.213 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:51.214 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.214 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.214 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:51.214 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.214 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.214 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:51.214 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.214 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.214 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:51.214 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 8192 2024-09-12T23:37:51.214 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.214 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:51.214 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.214 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.214 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:51.215 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.215 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.215 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:51.215 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.215 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.215 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:51.215 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 71504212 2024-09-12T23:37:51.215 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.215 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.215 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:kv": { 2024-09-12T23:37:51.215 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:51.215 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 796917760 2024-09-12T23:37:51.215 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.215 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:51.215 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67108864 2024-09-12T23:37:51.215 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.216 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:51.216 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.216 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.216 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:51.216 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 659033616 2024-09-12T23:37:51.216 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.216 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:51.216 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.216 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.216 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:51.216 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.216 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.216 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:51.216 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.216 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.217 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:51.217 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.217 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.217 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:51.217 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.217 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.217 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:51.217 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2128 2024-09-12T23:37:51.217 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.217 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:51.217 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.217 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.217 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:51.217 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.218 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.218 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:51.218 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.218 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.218 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:51.218 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 70773152 2024-09-12T23:37:51.218 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.218 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.218 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:kv_onode": { 2024-09-12T23:37:51.218 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:51.218 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 192937984 2024-09-12T23:37:51.218 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.218 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:51.218 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67108864 2024-09-12T23:37:51.218 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.218 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:51.219 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 464 2024-09-12T23:37:51.219 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.219 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:51.219 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 58580765 2024-09-12T23:37:51.219 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.219 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:51.219 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.219 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.219 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:51.219 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.219 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.219 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:51.219 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.219 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.220 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:51.220 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.220 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.220 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:51.220 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.220 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.220 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:51.220 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.220 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.220 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:51.220 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.220 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.220 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:51.220 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.220 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.221 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:51.221 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.221 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.221 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:51.221 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67247891 2024-09-12T23:37:51.221 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.221 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.221 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:meta": { 2024-09-12T23:37:51.221 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:51.221 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 729808896 2024-09-12T23:37:51.221 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.221 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:51.221 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.221 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.221 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:51.221 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.222 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.222 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:51.222 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 659033616 2024-09-12T23:37:51.222 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.222 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:51.222 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.222 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.222 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:51.222 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.222 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.222 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:51.222 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.222 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.222 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:51.222 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.223 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.223 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:51.223 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.223 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.223 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:51.223 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 233169 2024-09-12T23:37:51.223 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.223 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:51.223 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.223 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.223 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:51.223 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.223 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.223 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:51.224 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.224 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.224 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:51.224 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 70542111 2024-09-12T23:37:51.224 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.224 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.224 INFO:tasks.workunit.client.0.smithi136.stdout: "objecter": { 2024-09-12T23:37:51.224 INFO:tasks.workunit.client.0.smithi136.stdout: "op_active": { 2024-09-12T23:37:51.224 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.224 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.224 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r": { 2024-09-12T23:37:51.224 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.224 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.224 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rmw": { 2024-09-12T23:37:51.224 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.224 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.225 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w": { 2024-09-12T23:37:51.225 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.225 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.225 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.225 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:51.225 INFO:tasks.workunit.client.0.smithi136.stdout: "numpg": { 2024-09-12T23:37:51.225 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 30 2024-09-12T23:37:51.225 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.225 INFO:tasks.workunit.client.0.smithi136.stdout: "numpg_removing": { 2024-09-12T23:37:51.225 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.225 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.225 INFO:tasks.workunit.client.0.smithi136.stdout: "op": { 2024-09-12T23:37:51.225 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 10 2024-09-12T23:37:51.225 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.225 INFO:tasks.workunit.client.0.smithi136.stdout: "op_delayed_degraded": { 2024-09-12T23:37:51.226 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.226 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.226 INFO:tasks.workunit.client.0.smithi136.stdout: "op_delayed_unreadable": { 2024-09-12T23:37:51.226 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.226 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.226 INFO:tasks.workunit.client.0.smithi136.stdout: "op_in_bytes": { 2024-09-12T23:37:51.226 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.226 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.226 INFO:tasks.workunit.client.0.smithi136.stdout: "op_latency": { 2024-09-12T23:37:51.226 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 10, 2024-09-12T23:37:51.226 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 228728255 2024-09-12T23:37:51.226 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.226 INFO:tasks.workunit.client.0.smithi136.stdout: "op_out_bytes": { 2024-09-12T23:37:51.227 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.227 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.227 INFO:tasks.workunit.client.0.smithi136.stdout: "op_prepare_latency": { 2024-09-12T23:37:51.227 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 10, 2024-09-12T23:37:51.227 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 6725617 2024-09-12T23:37:51.227 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.227 INFO:tasks.workunit.client.0.smithi136.stdout: "op_process_latency": { 2024-09-12T23:37:51.227 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 10, 2024-09-12T23:37:51.227 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 203254064 2024-09-12T23:37:51.227 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.227 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r": { 2024-09-12T23:37:51.227 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.227 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.227 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_latency": { 2024-09-12T23:37:51.227 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.228 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.228 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.228 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-12T23:37:51.228 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.228 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.228 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_out_bytes": { 2024-09-12T23:37:51.228 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.228 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.228 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_prepare_latency": { 2024-09-12T23:37:51.228 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.228 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.228 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.228 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_process_latency": { 2024-09-12T23:37:51.229 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.229 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.229 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.229 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw": { 2024-09-12T23:37:51.229 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.229 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.229 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_in_bytes": { 2024-09-12T23:37:51.229 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.229 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.229 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_latency": { 2024-09-12T23:37:51.229 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.229 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.229 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.229 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-12T23:37:51.229 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.230 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.230 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-12T23:37:51.230 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.230 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.230 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_out_bytes": { 2024-09-12T23:37:51.230 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.230 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.230 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_prepare_latency": { 2024-09-12T23:37:51.230 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.230 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.230 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.230 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_process_latency": { 2024-09-12T23:37:51.230 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.230 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.230 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.231 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w": { 2024-09-12T23:37:51.231 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 10 2024-09-12T23:37:51.231 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.231 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_in_bytes": { 2024-09-12T23:37:51.231 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.231 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.231 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_latency": { 2024-09-12T23:37:51.231 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 10, 2024-09-12T23:37:51.231 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 228728255 2024-09-12T23:37:51.231 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.231 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-12T23:37:51.231 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.231 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.231 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_prepare_latency": { 2024-09-12T23:37:51.232 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 10, 2024-09-12T23:37:51.232 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 6725617 2024-09-12T23:37:51.232 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.232 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_process_latency": { 2024-09-12T23:37:51.232 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 10, 2024-09-12T23:37:51.232 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 203254064 2024-09-12T23:37:51.232 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.232 INFO:tasks.workunit.client.0.smithi136.stdout: "op_wip": { 2024-09-12T23:37:51.232 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.232 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.232 INFO:tasks.workunit.client.0.smithi136.stdout: "recovery_bytes": { 2024-09-12T23:37:51.232 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.232 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.232 INFO:tasks.workunit.client.0.smithi136.stdout: "recovery_ops": { 2024-09-12T23:37:51.233 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1 2024-09-12T23:37:51.233 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.233 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_bytes": { 2024-09-12T23:37:51.233 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 95995035648 2024-09-12T23:37:51.233 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.233 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_bytes_used": { 2024-09-12T23:37:51.233 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 305147904 2024-09-12T23:37:51.233 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.233 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.233 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.233 INFO:tasks.workunit.client.0.smithi136.stdout: "osd.3": { 2024-09-12T23:37:51.233 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs": { 2024-09-12T23:37:51.233 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_slow_fallback": { 2024-09-12T23:37:51.233 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.233 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.234 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_slow_size_fallback": { 2024-09-12T23:37:51.234 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.234 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.234 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit_db": { 2024-09-12T23:37:51.234 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 65536 2024-09-12T23:37:51.234 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.234 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit_main": { 2024-09-12T23:37:51.234 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.234 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.234 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit_wal": { 2024-09-12T23:37:51.234 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.234 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.234 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_written_slow": { 2024-09-12T23:37:51.234 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.234 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.235 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_written_sst": { 2024-09-12T23:37:51.235 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 12288 2024-09-12T23:37:51.235 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.235 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_written_wal": { 2024-09-12T23:37:51.235 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1069056 2024-09-12T23:37:51.235 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.235 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_lat": { 2024-09-12T23:37:51.235 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.235 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.235 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.235 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_lock_lat": { 2024-09-12T23:37:51.235 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.235 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.235 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.236 INFO:tasks.workunit.client.0.smithi136.stdout: "db_total_bytes": { 2024-09-12T23:37:51.236 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 95995035648 2024-09-12T23:37:51.236 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.236 INFO:tasks.workunit.client.0.smithi136.stdout: "db_used_bytes": { 2024-09-12T23:37:51.236 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 304480256 2024-09-12T23:37:51.236 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.236 INFO:tasks.workunit.client.0.smithi136.stdout: "log_bytes": { 2024-09-12T23:37:51.236 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 823296 2024-09-12T23:37:51.236 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.236 INFO:tasks.workunit.client.0.smithi136.stdout: "logged_bytes": { 2024-09-12T23:37:51.236 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 684032 2024-09-12T23:37:51.236 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.236 INFO:tasks.workunit.client.0.smithi136.stdout: "max_bytes_db": { 2024-09-12T23:37:51.237 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 312868864 2024-09-12T23:37:51.237 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.237 INFO:tasks.workunit.client.0.smithi136.stdout: "max_bytes_slow": { 2024-09-12T23:37:51.237 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.237 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.237 INFO:tasks.workunit.client.0.smithi136.stdout: "max_bytes_wal": { 2024-09-12T23:37:51.237 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.237 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.237 INFO:tasks.workunit.client.0.smithi136.stdout: "num_files": { 2024-09-12T23:37:51.237 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 14 2024-09-12T23:37:51.237 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.237 INFO:tasks.workunit.client.0.smithi136.stdout: "read_bytes": { 2024-09-12T23:37:51.237 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 195697 2024-09-12T23:37:51.237 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.238 INFO:tasks.workunit.client.0.smithi136.stdout: "read_count": { 2024-09-12T23:37:51.238 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 23 2024-09-12T23:37:51.238 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.238 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes": { 2024-09-12T23:37:51.238 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1269760 2024-09-12T23:37:51.238 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.238 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes_db": { 2024-09-12T23:37:51.238 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1273856 2024-09-12T23:37:51.238 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.238 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes_slow": { 2024-09-12T23:37:51.238 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.238 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.238 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes_wal": { 2024-09-12T23:37:51.238 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.239 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.239 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_count": { 2024-09-12T23:37:51.239 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 15 2024-09-12T23:37:51.239 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.239 INFO:tasks.workunit.client.0.smithi136.stdout: "read_prefetch_bytes": { 2024-09-12T23:37:51.239 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4789 2024-09-12T23:37:51.239 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.239 INFO:tasks.workunit.client.0.smithi136.stdout: "read_prefetch_count": { 2024-09-12T23:37:51.239 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4 2024-09-12T23:37:51.239 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.239 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_buffer_bytes": { 2024-09-12T23:37:51.239 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5589 2024-09-12T23:37:51.239 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.239 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_buffer_count": { 2024-09-12T23:37:51.239 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 24 2024-09-12T23:37:51.240 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.240 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_bytes": { 2024-09-12T23:37:51.240 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5589 2024-09-12T23:37:51.240 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.240 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_count": { 2024-09-12T23:37:51.240 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 24 2024-09-12T23:37:51.240 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.240 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes": { 2024-09-12T23:37:51.240 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.240 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.240 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes_db": { 2024-09-12T23:37:51.240 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.240 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.241 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes_slow": { 2024-09-12T23:37:51.241 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.241 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.241 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes_wal": { 2024-09-12T23:37:51.241 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.241 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.241 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_count": { 2024-09-12T23:37:51.241 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.241 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.241 INFO:tasks.workunit.client.0.smithi136.stdout: "slow_total_bytes": { 2024-09-12T23:37:51.241 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.241 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.241 INFO:tasks.workunit.client.0.smithi136.stdout: "slow_used_bytes": { 2024-09-12T23:37:51.241 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.242 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.242 INFO:tasks.workunit.client.0.smithi136.stdout: "wal_total_bytes": { 2024-09-12T23:37:51.242 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.242 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.242 INFO:tasks.workunit.client.0.smithi136.stdout: "wal_used_bytes": { 2024-09-12T23:37:51.242 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.242 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.242 INFO:tasks.workunit.client.0.smithi136.stdout: "write_bytes": { 2024-09-12T23:37:51.242 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1826816 2024-09-12T23:37:51.242 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.242 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.242 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore": { 2024-09-12T23:37:51.242 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit": { 2024-09-12T23:37:51.242 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4096 2024-09-12T23:37:51.242 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.242 INFO:tasks.workunit.client.0.smithi136.stdout: "allocated": { 2024-09-12T23:37:51.243 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 671744 2024-09-12T23:37:51.243 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.243 INFO:tasks.workunit.client.0.smithi136.stdout: "clist_lat": { 2024-09-12T23:37:51.243 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.243 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.243 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.243 INFO:tasks.workunit.client.0.smithi136.stdout: "compress_lat": { 2024-09-12T23:37:51.243 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.243 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.243 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.243 INFO:tasks.workunit.client.0.smithi136.stdout: "compressed": { 2024-09-12T23:37:51.243 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.243 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.243 INFO:tasks.workunit.client.0.smithi136.stdout: "compressed_allocated": { 2024-09-12T23:37:51.243 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.243 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.244 INFO:tasks.workunit.client.0.smithi136.stdout: "compressed_original": { 2024-09-12T23:37:51.244 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.244 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.244 INFO:tasks.workunit.client.0.smithi136.stdout: "csum_lat": { 2024-09-12T23:37:51.244 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 3, 2024-09-12T23:37:51.244 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 7967 2024-09-12T23:37:51.244 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.244 INFO:tasks.workunit.client.0.smithi136.stdout: "decompress_lat": { 2024-09-12T23:37:51.244 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.244 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.244 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.244 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_commit_lat": { 2024-09-12T23:37:51.244 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 150, 2024-09-12T23:37:51.244 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 49119257 2024-09-12T23:37:51.244 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.245 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_final_lat": { 2024-09-12T23:37:51.245 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 144, 2024-09-12T23:37:51.245 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 11416815 2024-09-12T23:37:51.245 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.245 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_flush_lat": { 2024-09-12T23:37:51.245 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 150, 2024-09-12T23:37:51.245 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 428455 2024-09-12T23:37:51.245 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.245 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_sync_lat": { 2024-09-12T23:37:51.245 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 150, 2024-09-12T23:37:51.245 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 49547712 2024-09-12T23:37:51.245 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.245 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_get_keys_lat": { 2024-09-12T23:37:51.245 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.245 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.246 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.246 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_get_values_lat": { 2024-09-12T23:37:51.246 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1, 2024-09-12T23:37:51.246 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 42474 2024-09-12T23:37:51.246 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.246 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_lower_bound_lat": { 2024-09-12T23:37:51.246 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 2, 2024-09-12T23:37:51.246 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2754 2024-09-12T23:37:51.246 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.246 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_next_lat": { 2024-09-12T23:37:51.246 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.246 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.246 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.246 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_seek_to_first_lat": { 2024-09-12T23:37:51.246 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.246 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.247 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.247 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_upper_bound_lat": { 2024-09-12T23:37:51.247 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 3, 2024-09-12T23:37:51.247 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 17643 2024-09-12T23:37:51.247 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.247 INFO:tasks.workunit.client.0.smithi136.stdout: "onode_hits": { 2024-09-12T23:37:51.247 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 313 2024-09-12T23:37:51.247 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.247 INFO:tasks.workunit.client.0.smithi136.stdout: "onode_misses": { 2024-09-12T23:37:51.247 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 210 2024-09-12T23:37:51.247 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.247 INFO:tasks.workunit.client.0.smithi136.stdout: "read_lat": { 2024-09-12T23:37:51.247 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 79, 2024-09-12T23:37:51.247 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1230811 2024-09-12T23:37:51.247 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.247 INFO:tasks.workunit.client.0.smithi136.stdout: "read_onode_meta_lat": { 2024-09-12T23:37:51.248 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 135, 2024-09-12T23:37:51.248 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 370609 2024-09-12T23:37:51.248 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.248 INFO:tasks.workunit.client.0.smithi136.stdout: "read_wait_aio_lat": { 2024-09-12T23:37:51.248 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 56, 2024-09-12T23:37:51.248 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 542793 2024-09-12T23:37:51.248 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.248 INFO:tasks.workunit.client.0.smithi136.stdout: "reads_with_retries": { 2024-09-12T23:37:51.248 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.248 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.248 INFO:tasks.workunit.client.0.smithi136.stdout: "remove_lat": { 2024-09-12T23:37:51.248 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.248 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.248 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.248 INFO:tasks.workunit.client.0.smithi136.stdout: "state_aio_wait_lat": { 2024-09-12T23:37:51.249 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 228, 2024-09-12T23:37:51.249 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5822849 2024-09-12T23:37:51.249 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.249 INFO:tasks.workunit.client.0.smithi136.stdout: "state_deferred_aio_wait_lat": { 2024-09-12T23:37:51.249 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 46, 2024-09-12T23:37:51.249 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4403355 2024-09-12T23:37:51.249 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.249 INFO:tasks.workunit.client.0.smithi136.stdout: "state_deferred_cleanup_lat": { 2024-09-12T23:37:51.249 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 43, 2024-09-12T23:37:51.249 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 584895296342 2024-09-12T23:37:51.249 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.249 INFO:tasks.workunit.client.0.smithi136.stdout: "state_deferred_queued_lat": { 2024-09-12T23:37:51.249 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 46, 2024-09-12T23:37:51.249 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 69792450299 2024-09-12T23:37:51.249 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.249 INFO:tasks.workunit.client.0.smithi136.stdout: "state_done_lat": { 2024-09-12T23:37:51.249 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 225, 2024-09-12T23:37:51.250 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 875876 2024-09-12T23:37:51.250 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.250 INFO:tasks.workunit.client.0.smithi136.stdout: "state_finishing_lat": { 2024-09-12T23:37:51.250 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 225, 2024-09-12T23:37:51.250 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 36090 2024-09-12T23:37:51.250 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.250 INFO:tasks.workunit.client.0.smithi136.stdout: "state_io_done_lat": { 2024-09-12T23:37:51.250 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 228, 2024-09-12T23:37:51.250 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 6654008 2024-09-12T23:37:51.250 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.250 INFO:tasks.workunit.client.0.smithi136.stdout: "state_kv_commiting_lat": { 2024-09-12T23:37:51.250 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 228, 2024-09-12T23:37:51.250 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 115512990 2024-09-12T23:37:51.250 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.251 INFO:tasks.workunit.client.0.smithi136.stdout: "state_kv_done_lat": { 2024-09-12T23:37:51.251 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 228, 2024-09-12T23:37:51.251 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 102221 2024-09-12T23:37:51.251 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.251 INFO:tasks.workunit.client.0.smithi136.stdout: "state_kv_queued_lat": { 2024-09-12T23:37:51.251 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 228, 2024-09-12T23:37:51.251 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 86930755 2024-09-12T23:37:51.251 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.251 INFO:tasks.workunit.client.0.smithi136.stdout: "state_prepare_lat": { 2024-09-12T23:37:51.251 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 228, 2024-09-12T23:37:51.251 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 18258942 2024-09-12T23:37:51.251 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.251 INFO:tasks.workunit.client.0.smithi136.stdout: "stored": { 2024-09-12T23:37:51.251 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 294264 2024-09-12T23:37:51.251 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.252 INFO:tasks.workunit.client.0.smithi136.stdout: "truncate_lat": { 2024-09-12T23:37:51.252 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 37, 2024-09-12T23:37:51.252 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 329709 2024-09-12T23:37:51.252 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.252 INFO:tasks.workunit.client.0.smithi136.stdout: "txc_commit_lat": { 2024-09-12T23:37:51.252 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 228, 2024-09-12T23:37:51.252 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 233225664 2024-09-12T23:37:51.252 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.252 INFO:tasks.workunit.client.0.smithi136.stdout: "txc_submit_lat": { 2024-09-12T23:37:51.252 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 228, 2024-09-12T23:37:51.252 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 24580574 2024-09-12T23:37:51.252 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.252 INFO:tasks.workunit.client.0.smithi136.stdout: "txc_throttle_lat": { 2024-09-12T23:37:51.252 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 228, 2024-09-12T23:37:51.252 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 3909498 2024-09-12T23:37:51.253 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.253 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.253 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache": { 2024-09-12T23:37:51.253 INFO:tasks.workunit.client.0.smithi136.stdout: "cache_bytes": { 2024-09-12T23:37:51.253 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1900976301 2024-09-12T23:37:51.253 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.253 INFO:tasks.workunit.client.0.smithi136.stdout: "heap_bytes": { 2024-09-12T23:37:51.253 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 269860864 2024-09-12T23:37:51.253 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.253 INFO:tasks.workunit.client.0.smithi136.stdout: "mapped_bytes": { 2024-09-12T23:37:51.253 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 269434880 2024-09-12T23:37:51.253 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.253 INFO:tasks.workunit.client.0.smithi136.stdout: "target_bytes": { 2024-09-12T23:37:51.253 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 3183861964 2024-09-12T23:37:51.253 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.253 INFO:tasks.workunit.client.0.smithi136.stdout: "unmapped_bytes": { 2024-09-12T23:37:51.254 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 425984 2024-09-12T23:37:51.254 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.254 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.254 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:data": { 2024-09-12T23:37:51.254 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:51.254 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 159383552 2024-09-12T23:37:51.254 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.254 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:51.254 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.254 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.254 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:51.254 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.254 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.254 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:51.254 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 87871225 2024-09-12T23:37:51.255 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.255 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:51.255 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.255 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.255 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:51.255 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.255 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.255 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:51.255 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.255 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.255 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:51.255 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.255 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.255 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:51.256 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.256 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.256 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:51.256 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.256 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.256 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:51.256 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.256 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.256 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:51.256 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.256 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.256 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:51.256 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.256 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.257 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:51.257 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 71512327 2024-09-12T23:37:51.257 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.257 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.257 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:kv": { 2024-09-12T23:37:51.257 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:51.257 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 796917760 2024-09-12T23:37:51.257 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.257 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:51.257 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67108864 2024-09-12T23:37:51.257 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.257 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:51.257 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.257 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.258 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:51.258 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 659034191 2024-09-12T23:37:51.258 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.258 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:51.258 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.258 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.258 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:51.258 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.258 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.258 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:51.258 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.258 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.258 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:51.258 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.259 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.259 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:51.259 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.259 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.259 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:51.259 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2128 2024-09-12T23:37:51.259 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.259 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:51.259 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.259 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.259 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:51.259 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.259 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.260 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:51.260 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.260 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.260 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:51.260 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 70772577 2024-09-12T23:37:51.260 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.260 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.260 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:kv_onode": { 2024-09-12T23:37:51.260 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:51.260 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 192937984 2024-09-12T23:37:51.260 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.260 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:51.260 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67108864 2024-09-12T23:37:51.261 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.261 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:51.261 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 464 2024-09-12T23:37:51.261 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.261 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:51.261 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 58580817 2024-09-12T23:37:51.261 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.261 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:51.261 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.261 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.261 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:51.261 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.261 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.262 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:51.262 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.262 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.262 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:51.262 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.262 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.262 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:51.262 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.262 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.262 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:51.262 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.262 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.262 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:51.262 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.262 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.262 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:51.263 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.263 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.263 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:51.263 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.263 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.263 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:51.263 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67247839 2024-09-12T23:37:51.263 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.263 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.263 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:meta": { 2024-09-12T23:37:51.263 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:51.263 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 729808896 2024-09-12T23:37:51.263 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.263 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:51.263 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.264 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.264 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:51.264 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.264 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.264 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:51.264 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 659034191 2024-09-12T23:37:51.264 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.264 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:51.264 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.264 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.264 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:51.264 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.264 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.265 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:51.265 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.265 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.265 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:51.265 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.265 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.265 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:51.265 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.265 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.265 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:51.265 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 245668 2024-09-12T23:37:51.265 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.265 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:51.265 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.265 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.266 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:51.266 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.266 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.266 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:51.266 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.266 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.266 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:51.266 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 70529037 2024-09-12T23:37:51.266 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.266 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.266 INFO:tasks.workunit.client.0.smithi136.stdout: "objecter": { 2024-09-12T23:37:51.266 INFO:tasks.workunit.client.0.smithi136.stdout: "op_active": { 2024-09-12T23:37:51.266 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.266 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.267 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r": { 2024-09-12T23:37:51.267 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.267 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.267 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rmw": { 2024-09-12T23:37:51.267 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.267 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.267 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w": { 2024-09-12T23:37:51.267 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.267 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.267 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.267 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:51.267 INFO:tasks.workunit.client.0.smithi136.stdout: "numpg": { 2024-09-12T23:37:51.267 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 37 2024-09-12T23:37:51.267 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.268 INFO:tasks.workunit.client.0.smithi136.stdout: "numpg_removing": { 2024-09-12T23:37:51.268 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.268 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.268 INFO:tasks.workunit.client.0.smithi136.stdout: "op": { 2024-09-12T23:37:51.268 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.268 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.268 INFO:tasks.workunit.client.0.smithi136.stdout: "op_delayed_degraded": { 2024-09-12T23:37:51.268 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.268 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.268 INFO:tasks.workunit.client.0.smithi136.stdout: "op_delayed_unreadable": { 2024-09-12T23:37:51.268 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.268 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.268 INFO:tasks.workunit.client.0.smithi136.stdout: "op_in_bytes": { 2024-09-12T23:37:51.268 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.269 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.269 INFO:tasks.workunit.client.0.smithi136.stdout: "op_latency": { 2024-09-12T23:37:51.269 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.269 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.269 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.269 INFO:tasks.workunit.client.0.smithi136.stdout: "op_out_bytes": { 2024-09-12T23:37:51.269 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.269 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.269 INFO:tasks.workunit.client.0.smithi136.stdout: "op_prepare_latency": { 2024-09-12T23:37:51.269 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 2, 2024-09-12T23:37:51.269 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 956264 2024-09-12T23:37:51.269 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.269 INFO:tasks.workunit.client.0.smithi136.stdout: "op_process_latency": { 2024-09-12T23:37:51.269 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.270 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.270 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.270 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r": { 2024-09-12T23:37:51.270 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.270 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.270 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_latency": { 2024-09-12T23:37:51.270 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.270 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.270 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.270 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-12T23:37:51.270 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.270 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.271 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_out_bytes": { 2024-09-12T23:37:51.271 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.271 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.271 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_prepare_latency": { 2024-09-12T23:37:51.271 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.271 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.271 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.271 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_process_latency": { 2024-09-12T23:37:51.271 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.271 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.271 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.271 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw": { 2024-09-12T23:37:51.271 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.271 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.271 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_in_bytes": { 2024-09-12T23:37:51.272 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.272 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.272 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_latency": { 2024-09-12T23:37:51.272 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.272 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.272 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.272 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-12T23:37:51.272 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.272 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.272 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-12T23:37:51.272 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.272 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.272 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_out_bytes": { 2024-09-12T23:37:51.272 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.273 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.273 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_prepare_latency": { 2024-09-12T23:37:51.273 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.273 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.273 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.273 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_process_latency": { 2024-09-12T23:37:51.273 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.273 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.273 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.273 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w": { 2024-09-12T23:37:51.273 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.273 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.273 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_in_bytes": { 2024-09-12T23:37:51.274 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.274 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.274 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_latency": { 2024-09-12T23:37:51.274 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.274 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.274 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.274 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-12T23:37:51.274 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.274 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.274 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_prepare_latency": { 2024-09-12T23:37:51.274 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 2, 2024-09-12T23:37:51.274 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 956264 2024-09-12T23:37:51.274 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.275 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_process_latency": { 2024-09-12T23:37:51.275 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.275 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.275 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.275 INFO:tasks.workunit.client.0.smithi136.stdout: "op_wip": { 2024-09-12T23:37:51.275 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.275 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.275 INFO:tasks.workunit.client.0.smithi136.stdout: "recovery_bytes": { 2024-09-12T23:37:51.275 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.275 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.275 INFO:tasks.workunit.client.0.smithi136.stdout: "recovery_ops": { 2024-09-12T23:37:51.275 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.275 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.275 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_bytes": { 2024-09-12T23:37:51.275 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 95995035648 2024-09-12T23:37:51.276 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.276 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_bytes_used": { 2024-09-12T23:37:51.276 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 305160192 2024-09-12T23:37:51.276 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.276 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.276 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.276 INFO:tasks.workunit.client.0.smithi136.stdout: "osd.4": { 2024-09-12T23:37:51.276 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs": { 2024-09-12T23:37:51.276 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_slow_fallback": { 2024-09-12T23:37:51.276 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.276 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.276 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_slow_size_fallback": { 2024-09-12T23:37:51.276 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.276 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.277 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit_db": { 2024-09-12T23:37:51.277 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 65536 2024-09-12T23:37:51.277 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.277 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit_main": { 2024-09-12T23:37:51.277 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.277 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.277 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit_wal": { 2024-09-12T23:37:51.277 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.277 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.277 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_written_slow": { 2024-09-12T23:37:51.277 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.277 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.277 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_written_sst": { 2024-09-12T23:37:51.277 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 12288 2024-09-12T23:37:51.277 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.278 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_written_wal": { 2024-09-12T23:37:51.278 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1228800 2024-09-12T23:37:51.278 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.278 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_lat": { 2024-09-12T23:37:51.278 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.278 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.278 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.278 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_lock_lat": { 2024-09-12T23:37:51.278 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.278 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.278 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.278 INFO:tasks.workunit.client.0.smithi136.stdout: "db_total_bytes": { 2024-09-12T23:37:51.278 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 95995035648 2024-09-12T23:37:51.278 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.279 INFO:tasks.workunit.client.0.smithi136.stdout: "db_used_bytes": { 2024-09-12T23:37:51.279 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 304480256 2024-09-12T23:37:51.279 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.279 INFO:tasks.workunit.client.0.smithi136.stdout: "log_bytes": { 2024-09-12T23:37:51.279 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 954368 2024-09-12T23:37:51.279 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.279 INFO:tasks.workunit.client.0.smithi136.stdout: "logged_bytes": { 2024-09-12T23:37:51.279 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 815104 2024-09-12T23:37:51.279 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.279 INFO:tasks.workunit.client.0.smithi136.stdout: "max_bytes_db": { 2024-09-12T23:37:51.279 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 312868864 2024-09-12T23:37:51.279 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.279 INFO:tasks.workunit.client.0.smithi136.stdout: "max_bytes_slow": { 2024-09-12T23:37:51.279 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.279 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.280 INFO:tasks.workunit.client.0.smithi136.stdout: "max_bytes_wal": { 2024-09-12T23:37:51.280 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.280 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.280 INFO:tasks.workunit.client.0.smithi136.stdout: "num_files": { 2024-09-12T23:37:51.280 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 14 2024-09-12T23:37:51.280 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.280 INFO:tasks.workunit.client.0.smithi136.stdout: "read_bytes": { 2024-09-12T23:37:51.280 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 195697 2024-09-12T23:37:51.280 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.280 INFO:tasks.workunit.client.0.smithi136.stdout: "read_count": { 2024-09-12T23:37:51.280 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 23 2024-09-12T23:37:51.280 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.280 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes": { 2024-09-12T23:37:51.280 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1269760 2024-09-12T23:37:51.281 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.281 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes_db": { 2024-09-12T23:37:51.281 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1273856 2024-09-12T23:37:51.281 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.281 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes_slow": { 2024-09-12T23:37:51.281 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.281 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.281 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes_wal": { 2024-09-12T23:37:51.281 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.281 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.281 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_count": { 2024-09-12T23:37:51.281 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 15 2024-09-12T23:37:51.281 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.281 INFO:tasks.workunit.client.0.smithi136.stdout: "read_prefetch_bytes": { 2024-09-12T23:37:51.281 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4789 2024-09-12T23:37:51.282 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.282 INFO:tasks.workunit.client.0.smithi136.stdout: "read_prefetch_count": { 2024-09-12T23:37:51.282 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4 2024-09-12T23:37:51.282 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.282 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_buffer_bytes": { 2024-09-12T23:37:51.282 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5589 2024-09-12T23:37:51.282 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.282 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_buffer_count": { 2024-09-12T23:37:51.282 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 24 2024-09-12T23:37:51.282 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.282 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_bytes": { 2024-09-12T23:37:51.282 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5589 2024-09-12T23:37:51.282 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.282 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_count": { 2024-09-12T23:37:51.283 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 24 2024-09-12T23:37:51.283 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.283 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes": { 2024-09-12T23:37:51.283 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.283 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.283 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes_db": { 2024-09-12T23:37:51.283 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.283 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.283 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes_slow": { 2024-09-12T23:37:51.283 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.283 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.283 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes_wal": { 2024-09-12T23:37:51.283 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.283 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.284 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_count": { 2024-09-12T23:37:51.284 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.284 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.284 INFO:tasks.workunit.client.0.smithi136.stdout: "slow_total_bytes": { 2024-09-12T23:37:51.284 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.284 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.284 INFO:tasks.workunit.client.0.smithi136.stdout: "slow_used_bytes": { 2024-09-12T23:37:51.284 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.284 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.284 INFO:tasks.workunit.client.0.smithi136.stdout: "wal_total_bytes": { 2024-09-12T23:37:51.284 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.284 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.284 INFO:tasks.workunit.client.0.smithi136.stdout: "wal_used_bytes": { 2024-09-12T23:37:51.285 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.285 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.285 INFO:tasks.workunit.client.0.smithi136.stdout: "write_bytes": { 2024-09-12T23:37:51.285 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2117632 2024-09-12T23:37:51.285 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.285 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.285 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore": { 2024-09-12T23:37:51.285 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit": { 2024-09-12T23:37:51.285 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4096 2024-09-12T23:37:51.285 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.285 INFO:tasks.workunit.client.0.smithi136.stdout: "allocated": { 2024-09-12T23:37:51.285 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 671744 2024-09-12T23:37:51.285 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.285 INFO:tasks.workunit.client.0.smithi136.stdout: "clist_lat": { 2024-09-12T23:37:51.286 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 10, 2024-09-12T23:37:51.286 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 766836 2024-09-12T23:37:51.286 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.286 INFO:tasks.workunit.client.0.smithi136.stdout: "compress_lat": { 2024-09-12T23:37:51.286 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.286 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.286 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.286 INFO:tasks.workunit.client.0.smithi136.stdout: "compressed": { 2024-09-12T23:37:51.286 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.286 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.286 INFO:tasks.workunit.client.0.smithi136.stdout: "compressed_allocated": { 2024-09-12T23:37:51.286 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.286 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.287 INFO:tasks.workunit.client.0.smithi136.stdout: "compressed_original": { 2024-09-12T23:37:51.287 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.287 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.287 INFO:tasks.workunit.client.0.smithi136.stdout: "csum_lat": { 2024-09-12T23:37:51.287 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 7, 2024-09-12T23:37:51.287 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 28741 2024-09-12T23:37:51.287 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.287 INFO:tasks.workunit.client.0.smithi136.stdout: "decompress_lat": { 2024-09-12T23:37:51.287 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.287 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.287 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.287 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_commit_lat": { 2024-09-12T23:37:51.287 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 182, 2024-09-12T23:37:51.287 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 85389164 2024-09-12T23:37:51.288 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.288 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_final_lat": { 2024-09-12T23:37:51.288 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 177, 2024-09-12T23:37:51.288 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 15062744 2024-09-12T23:37:51.288 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.288 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_flush_lat": { 2024-09-12T23:37:51.288 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 182, 2024-09-12T23:37:51.288 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 380993 2024-09-12T23:37:51.288 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.288 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_sync_lat": { 2024-09-12T23:37:51.288 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 182, 2024-09-12T23:37:51.288 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 85770157 2024-09-12T23:37:51.288 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.289 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_get_keys_lat": { 2024-09-12T23:37:51.289 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.289 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.289 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.289 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_get_values_lat": { 2024-09-12T23:37:51.289 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 3, 2024-09-12T23:37:51.289 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 40241 2024-09-12T23:37:51.289 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.289 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_lower_bound_lat": { 2024-09-12T23:37:51.289 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 3, 2024-09-12T23:37:51.289 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 7368 2024-09-12T23:37:51.289 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.289 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_next_lat": { 2024-09-12T23:37:51.289 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.290 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.290 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.290 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_seek_to_first_lat": { 2024-09-12T23:37:51.290 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.290 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.290 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.290 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_upper_bound_lat": { 2024-09-12T23:37:51.290 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 3, 2024-09-12T23:37:51.290 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 14515 2024-09-12T23:37:51.290 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.290 INFO:tasks.workunit.client.0.smithi136.stdout: "onode_hits": { 2024-09-12T23:37:51.290 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 352 2024-09-12T23:37:51.290 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.290 INFO:tasks.workunit.client.0.smithi136.stdout: "onode_misses": { 2024-09-12T23:37:51.291 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 228 2024-09-12T23:37:51.291 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.291 INFO:tasks.workunit.client.0.smithi136.stdout: "read_lat": { 2024-09-12T23:37:51.291 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 86, 2024-09-12T23:37:51.291 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2185975 2024-09-12T23:37:51.291 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.291 INFO:tasks.workunit.client.0.smithi136.stdout: "read_onode_meta_lat": { 2024-09-12T23:37:51.291 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 141, 2024-09-12T23:37:51.291 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 577272 2024-09-12T23:37:51.291 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.291 INFO:tasks.workunit.client.0.smithi136.stdout: "read_wait_aio_lat": { 2024-09-12T23:37:51.291 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 55, 2024-09-12T23:37:51.291 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 986794 2024-09-12T23:37:51.291 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.291 INFO:tasks.workunit.client.0.smithi136.stdout: "reads_with_retries": { 2024-09-12T23:37:51.292 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.292 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.292 INFO:tasks.workunit.client.0.smithi136.stdout: "remove_lat": { 2024-09-12T23:37:51.292 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 6, 2024-09-12T23:37:51.292 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 288725 2024-09-12T23:37:51.292 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.292 INFO:tasks.workunit.client.0.smithi136.stdout: "state_aio_wait_lat": { 2024-09-12T23:37:51.292 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 264, 2024-09-12T23:37:51.292 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 7087454 2024-09-12T23:37:51.292 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.292 INFO:tasks.workunit.client.0.smithi136.stdout: "state_deferred_aio_wait_lat": { 2024-09-12T23:37:51.292 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 37, 2024-09-12T23:37:51.292 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4010336 2024-09-12T23:37:51.292 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.293 INFO:tasks.workunit.client.0.smithi136.stdout: "state_deferred_cleanup_lat": { 2024-09-12T23:37:51.293 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 35, 2024-09-12T23:37:51.293 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 407318272458 2024-09-12T23:37:51.293 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.293 INFO:tasks.workunit.client.0.smithi136.stdout: "state_deferred_queued_lat": { 2024-09-12T23:37:51.293 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 37, 2024-09-12T23:37:51.293 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 53628145715 2024-09-12T23:37:51.293 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.293 INFO:tasks.workunit.client.0.smithi136.stdout: "state_done_lat": { 2024-09-12T23:37:51.293 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 262, 2024-09-12T23:37:51.293 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 6392896272 2024-09-12T23:37:51.293 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.293 INFO:tasks.workunit.client.0.smithi136.stdout: "state_finishing_lat": { 2024-09-12T23:37:51.293 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 262, 2024-09-12T23:37:51.294 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 51575 2024-09-12T23:37:51.294 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.294 INFO:tasks.workunit.client.0.smithi136.stdout: "state_io_done_lat": { 2024-09-12T23:37:51.294 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 264, 2024-09-12T23:37:51.294 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 818704 2024-09-12T23:37:51.294 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.294 INFO:tasks.workunit.client.0.smithi136.stdout: "state_kv_commiting_lat": { 2024-09-12T23:37:51.294 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 264, 2024-09-12T23:37:51.294 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 165551935 2024-09-12T23:37:51.294 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.294 INFO:tasks.workunit.client.0.smithi136.stdout: "state_kv_done_lat": { 2024-09-12T23:37:51.294 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 264, 2024-09-12T23:37:51.294 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 136814 2024-09-12T23:37:51.295 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.295 INFO:tasks.workunit.client.0.smithi136.stdout: "state_kv_queued_lat": { 2024-09-12T23:37:51.295 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 264, 2024-09-12T23:37:51.295 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 237560396 2024-09-12T23:37:51.295 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.295 INFO:tasks.workunit.client.0.smithi136.stdout: "state_prepare_lat": { 2024-09-12T23:37:51.295 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 264, 2024-09-12T23:37:51.295 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 22324584 2024-09-12T23:37:51.295 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.295 INFO:tasks.workunit.client.0.smithi136.stdout: "stored": { 2024-09-12T23:37:51.295 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 292251 2024-09-12T23:37:51.295 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.295 INFO:tasks.workunit.client.0.smithi136.stdout: "truncate_lat": { 2024-09-12T23:37:51.296 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 35, 2024-09-12T23:37:51.296 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 257967 2024-09-12T23:37:51.296 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.296 INFO:tasks.workunit.client.0.smithi136.stdout: "txc_commit_lat": { 2024-09-12T23:37:51.296 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 264, 2024-09-12T23:37:51.296 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 433403579 2024-09-12T23:37:51.296 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.296 INFO:tasks.workunit.client.0.smithi136.stdout: "txc_submit_lat": { 2024-09-12T23:37:51.296 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 264, 2024-09-12T23:37:51.296 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 31406922 2024-09-12T23:37:51.296 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.296 INFO:tasks.workunit.client.0.smithi136.stdout: "txc_throttle_lat": { 2024-09-12T23:37:51.296 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 264, 2024-09-12T23:37:51.296 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 3032421 2024-09-12T23:37:51.297 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.297 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.297 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache": { 2024-09-12T23:37:51.297 INFO:tasks.workunit.client.0.smithi136.stdout: "cache_bytes": { 2024-09-12T23:37:51.297 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1900963507 2024-09-12T23:37:51.297 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.297 INFO:tasks.workunit.client.0.smithi136.stdout: "heap_bytes": { 2024-09-12T23:37:51.297 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 271187968 2024-09-12T23:37:51.297 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.297 INFO:tasks.workunit.client.0.smithi136.stdout: "mapped_bytes": { 2024-09-12T23:37:51.297 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 270319616 2024-09-12T23:37:51.297 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.297 INFO:tasks.workunit.client.0.smithi136.stdout: "target_bytes": { 2024-09-12T23:37:51.297 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 3183846912 2024-09-12T23:37:51.298 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.298 INFO:tasks.workunit.client.0.smithi136.stdout: "unmapped_bytes": { 2024-09-12T23:37:51.298 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 868352 2024-09-12T23:37:51.298 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.298 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.298 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:data": { 2024-09-12T23:37:51.298 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:51.298 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 159383552 2024-09-12T23:37:51.298 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.298 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:51.298 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.298 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.298 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:51.298 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.299 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.299 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:51.299 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 87869977 2024-09-12T23:37:51.299 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.299 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:51.299 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.299 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.299 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:51.299 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.299 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.299 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:51.299 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.299 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.299 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:51.300 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.300 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.300 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:51.300 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.300 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.300 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:51.300 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4096 2024-09-12T23:37:51.300 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.300 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:51.300 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.300 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.300 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:51.300 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.300 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.301 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:51.301 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.301 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.301 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:51.301 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 71509479 2024-09-12T23:37:51.301 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.301 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.301 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:kv": { 2024-09-12T23:37:51.301 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:51.301 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 796917760 2024-09-12T23:37:51.301 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.301 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:51.301 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67108864 2024-09-12T23:37:51.302 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.302 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:51.302 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.302 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.302 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:51.302 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 659024833 2024-09-12T23:37:51.302 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.302 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:51.302 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.302 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.302 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:51.302 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.302 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.303 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:51.303 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.303 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.303 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:51.303 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.303 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.303 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:51.303 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.303 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.303 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:51.303 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2128 2024-09-12T23:37:51.303 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.304 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:51.304 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.304 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.304 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:51.304 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.304 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.304 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:51.304 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.304 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.304 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:51.304 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 70781935 2024-09-12T23:37:51.304 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.304 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.304 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:kv_onode": { 2024-09-12T23:37:51.305 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:51.305 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 192937984 2024-09-12T23:37:51.305 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.305 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:51.305 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67108864 2024-09-12T23:37:51.305 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.305 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:51.305 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 464 2024-09-12T23:37:51.305 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.305 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:51.305 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 58579985 2024-09-12T23:37:51.305 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.305 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:51.305 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.306 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.306 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:51.306 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.306 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.306 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:51.306 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.306 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.306 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:51.306 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.306 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.306 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:51.306 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.306 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.306 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:51.306 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.306 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.307 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:51.307 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.307 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.307 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:51.307 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.307 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.307 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:51.307 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.307 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.307 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:51.307 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67248671 2024-09-12T23:37:51.307 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.307 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.307 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:meta": { 2024-09-12T23:37:51.307 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:51.308 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 729808896 2024-09-12T23:37:51.308 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.308 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:51.308 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.308 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.308 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:51.308 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.308 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.308 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:51.308 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 659024833 2024-09-12T23:37:51.308 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.308 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:51.308 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.308 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.308 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:51.309 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.309 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.309 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:51.309 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.309 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.309 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:51.309 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.309 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.309 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:51.309 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.309 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.309 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:51.309 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 241206 2024-09-12T23:37:51.310 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.310 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:51.310 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.310 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.310 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:51.310 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.310 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.310 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:51.310 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.310 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.310 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:51.310 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 70542857 2024-09-12T23:37:51.310 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.310 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.311 INFO:tasks.workunit.client.0.smithi136.stdout: "objecter": { 2024-09-12T23:37:51.311 INFO:tasks.workunit.client.0.smithi136.stdout: "op_active": { 2024-09-12T23:37:51.311 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.311 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.311 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r": { 2024-09-12T23:37:51.311 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.311 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.311 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rmw": { 2024-09-12T23:37:51.311 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.311 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.311 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w": { 2024-09-12T23:37:51.311 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.311 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.311 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.312 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:51.312 INFO:tasks.workunit.client.0.smithi136.stdout: "numpg": { 2024-09-12T23:37:51.312 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 41 2024-09-12T23:37:51.312 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.312 INFO:tasks.workunit.client.0.smithi136.stdout: "numpg_removing": { 2024-09-12T23:37:51.312 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.312 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.312 INFO:tasks.workunit.client.0.smithi136.stdout: "op": { 2024-09-12T23:37:51.312 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1 2024-09-12T23:37:51.312 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.312 INFO:tasks.workunit.client.0.smithi136.stdout: "op_delayed_degraded": { 2024-09-12T23:37:51.312 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.312 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.313 INFO:tasks.workunit.client.0.smithi136.stdout: "op_delayed_unreadable": { 2024-09-12T23:37:51.313 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.313 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.313 INFO:tasks.workunit.client.0.smithi136.stdout: "op_in_bytes": { 2024-09-12T23:37:51.313 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 22 2024-09-12T23:37:51.313 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.313 INFO:tasks.workunit.client.0.smithi136.stdout: "op_latency": { 2024-09-12T23:37:51.313 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1, 2024-09-12T23:37:51.313 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 10730533 2024-09-12T23:37:51.313 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.313 INFO:tasks.workunit.client.0.smithi136.stdout: "op_out_bytes": { 2024-09-12T23:37:51.313 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.313 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.314 INFO:tasks.workunit.client.0.smithi136.stdout: "op_prepare_latency": { 2024-09-12T23:37:51.314 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1, 2024-09-12T23:37:51.314 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 778785 2024-09-12T23:37:51.314 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.314 INFO:tasks.workunit.client.0.smithi136.stdout: "op_process_latency": { 2024-09-12T23:37:51.314 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1, 2024-09-12T23:37:51.314 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 10643160 2024-09-12T23:37:51.314 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.314 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r": { 2024-09-12T23:37:51.314 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.314 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.314 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_latency": { 2024-09-12T23:37:51.314 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.315 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.315 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.315 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-12T23:37:51.315 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.315 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.315 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_out_bytes": { 2024-09-12T23:37:51.315 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.315 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.315 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_prepare_latency": { 2024-09-12T23:37:51.315 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.315 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.315 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.315 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_process_latency": { 2024-09-12T23:37:51.315 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.316 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.316 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.316 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw": { 2024-09-12T23:37:51.316 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.316 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.316 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_in_bytes": { 2024-09-12T23:37:51.316 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.316 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.316 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_latency": { 2024-09-12T23:37:51.316 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.316 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.316 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.316 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-12T23:37:51.316 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.317 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.317 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-12T23:37:51.317 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.317 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.317 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_out_bytes": { 2024-09-12T23:37:51.317 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.317 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.317 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_prepare_latency": { 2024-09-12T23:37:51.317 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.317 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.317 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.317 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_process_latency": { 2024-09-12T23:37:51.317 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.317 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.317 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.318 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w": { 2024-09-12T23:37:51.318 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1 2024-09-12T23:37:51.318 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.318 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_in_bytes": { 2024-09-12T23:37:51.318 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 22 2024-09-12T23:37:51.318 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.318 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_latency": { 2024-09-12T23:37:51.318 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1, 2024-09-12T23:37:51.318 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 10730533 2024-09-12T23:37:51.318 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.318 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-12T23:37:51.318 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.318 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.318 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_prepare_latency": { 2024-09-12T23:37:51.319 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1, 2024-09-12T23:37:51.319 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 778785 2024-09-12T23:37:51.319 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.319 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_process_latency": { 2024-09-12T23:37:51.319 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1, 2024-09-12T23:37:51.319 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 10643160 2024-09-12T23:37:51.319 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.319 INFO:tasks.workunit.client.0.smithi136.stdout: "op_wip": { 2024-09-12T23:37:51.319 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.319 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.319 INFO:tasks.workunit.client.0.smithi136.stdout: "recovery_bytes": { 2024-09-12T23:37:51.319 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.319 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.319 INFO:tasks.workunit.client.0.smithi136.stdout: "recovery_ops": { 2024-09-12T23:37:51.320 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.320 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.320 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_bytes": { 2024-09-12T23:37:51.320 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 95995035648 2024-09-12T23:37:51.320 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.320 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_bytes_used": { 2024-09-12T23:37:51.320 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 305168384 2024-09-12T23:37:51.320 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.320 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.320 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.320 INFO:tasks.workunit.client.0.smithi136.stdout: "osd.5": { 2024-09-12T23:37:51.320 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs": { 2024-09-12T23:37:51.320 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_slow_fallback": { 2024-09-12T23:37:51.320 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.321 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.321 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_slow_size_fallback": { 2024-09-12T23:37:51.321 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.321 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.321 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit_db": { 2024-09-12T23:37:51.321 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 65536 2024-09-12T23:37:51.321 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.321 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit_main": { 2024-09-12T23:37:51.321 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.321 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.321 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit_wal": { 2024-09-12T23:37:51.321 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.321 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.321 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_written_slow": { 2024-09-12T23:37:51.321 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.322 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.322 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_written_sst": { 2024-09-12T23:37:51.322 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 12288 2024-09-12T23:37:51.322 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.322 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_written_wal": { 2024-09-12T23:37:51.322 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1200128 2024-09-12T23:37:51.322 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.322 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_lat": { 2024-09-12T23:37:51.322 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.322 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.322 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.322 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_lock_lat": { 2024-09-12T23:37:51.322 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.322 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.322 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.322 INFO:tasks.workunit.client.0.smithi136.stdout: "db_total_bytes": { 2024-09-12T23:37:51.323 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 95995035648 2024-09-12T23:37:51.323 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.323 INFO:tasks.workunit.client.0.smithi136.stdout: "db_used_bytes": { 2024-09-12T23:37:51.323 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 304480256 2024-09-12T23:37:51.323 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.323 INFO:tasks.workunit.client.0.smithi136.stdout: "log_bytes": { 2024-09-12T23:37:51.323 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 905216 2024-09-12T23:37:51.323 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.323 INFO:tasks.workunit.client.0.smithi136.stdout: "logged_bytes": { 2024-09-12T23:37:51.323 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 765952 2024-09-12T23:37:51.323 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.323 INFO:tasks.workunit.client.0.smithi136.stdout: "max_bytes_db": { 2024-09-12T23:37:51.323 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 312868864 2024-09-12T23:37:51.324 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.324 INFO:tasks.workunit.client.0.smithi136.stdout: "max_bytes_slow": { 2024-09-12T23:37:51.324 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.324 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.324 INFO:tasks.workunit.client.0.smithi136.stdout: "max_bytes_wal": { 2024-09-12T23:37:51.324 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.324 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.324 INFO:tasks.workunit.client.0.smithi136.stdout: "num_files": { 2024-09-12T23:37:51.324 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 14 2024-09-12T23:37:51.324 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.324 INFO:tasks.workunit.client.0.smithi136.stdout: "read_bytes": { 2024-09-12T23:37:51.324 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 195697 2024-09-12T23:37:51.324 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.324 INFO:tasks.workunit.client.0.smithi136.stdout: "read_count": { 2024-09-12T23:37:51.324 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 23 2024-09-12T23:37:51.325 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.325 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes": { 2024-09-12T23:37:51.325 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1269760 2024-09-12T23:37:51.325 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.325 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes_db": { 2024-09-12T23:37:51.325 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1273856 2024-09-12T23:37:51.325 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.325 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes_slow": { 2024-09-12T23:37:51.325 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.325 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.325 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes_wal": { 2024-09-12T23:37:51.325 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.325 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.325 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_count": { 2024-09-12T23:37:51.325 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 15 2024-09-12T23:37:51.326 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.326 INFO:tasks.workunit.client.0.smithi136.stdout: "read_prefetch_bytes": { 2024-09-12T23:37:51.326 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4789 2024-09-12T23:37:51.326 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.326 INFO:tasks.workunit.client.0.smithi136.stdout: "read_prefetch_count": { 2024-09-12T23:37:51.326 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4 2024-09-12T23:37:51.326 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.326 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_buffer_bytes": { 2024-09-12T23:37:51.326 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5589 2024-09-12T23:37:51.326 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.326 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_buffer_count": { 2024-09-12T23:37:51.326 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 24 2024-09-12T23:37:51.326 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.326 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_bytes": { 2024-09-12T23:37:51.326 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5589 2024-09-12T23:37:51.326 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.327 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_count": { 2024-09-12T23:37:51.327 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 24 2024-09-12T23:37:51.327 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.327 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes": { 2024-09-12T23:37:51.327 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.327 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.327 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes_db": { 2024-09-12T23:37:51.327 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.327 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.327 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes_slow": { 2024-09-12T23:37:51.327 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.327 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.327 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes_wal": { 2024-09-12T23:37:51.327 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.328 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.328 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_count": { 2024-09-12T23:37:51.328 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.328 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.328 INFO:tasks.workunit.client.0.smithi136.stdout: "slow_total_bytes": { 2024-09-12T23:37:51.328 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.328 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.328 INFO:tasks.workunit.client.0.smithi136.stdout: "slow_used_bytes": { 2024-09-12T23:37:51.328 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.328 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.328 INFO:tasks.workunit.client.0.smithi136.stdout: "wal_total_bytes": { 2024-09-12T23:37:51.328 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.328 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.328 INFO:tasks.workunit.client.0.smithi136.stdout: "wal_used_bytes": { 2024-09-12T23:37:51.328 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.329 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.329 INFO:tasks.workunit.client.0.smithi136.stdout: "write_bytes": { 2024-09-12T23:37:51.329 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2039808 2024-09-12T23:37:51.329 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.329 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.329 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore": { 2024-09-12T23:37:51.329 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit": { 2024-09-12T23:37:51.329 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4096 2024-09-12T23:37:51.329 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.329 INFO:tasks.workunit.client.0.smithi136.stdout: "allocated": { 2024-09-12T23:37:51.329 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 663552 2024-09-12T23:37:51.329 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.329 INFO:tasks.workunit.client.0.smithi136.stdout: "clist_lat": { 2024-09-12T23:37:51.329 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 8, 2024-09-12T23:37:51.330 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 618826 2024-09-12T23:37:51.330 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.330 INFO:tasks.workunit.client.0.smithi136.stdout: "compress_lat": { 2024-09-12T23:37:51.330 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.330 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.330 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.330 INFO:tasks.workunit.client.0.smithi136.stdout: "compressed": { 2024-09-12T23:37:51.330 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.330 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.330 INFO:tasks.workunit.client.0.smithi136.stdout: "compressed_allocated": { 2024-09-12T23:37:51.330 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.330 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.330 INFO:tasks.workunit.client.0.smithi136.stdout: "compressed_original": { 2024-09-12T23:37:51.330 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.331 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.331 INFO:tasks.workunit.client.0.smithi136.stdout: "csum_lat": { 2024-09-12T23:37:51.331 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 4, 2024-09-12T23:37:51.331 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 10745 2024-09-12T23:37:51.331 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.331 INFO:tasks.workunit.client.0.smithi136.stdout: "decompress_lat": { 2024-09-12T23:37:51.331 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.331 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.331 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.331 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_commit_lat": { 2024-09-12T23:37:51.331 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 170, 2024-09-12T23:37:51.331 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 62071808 2024-09-12T23:37:51.331 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.331 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_final_lat": { 2024-09-12T23:37:51.332 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 166, 2024-09-12T23:37:51.332 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 8289281 2024-09-12T23:37:51.332 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.332 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_flush_lat": { 2024-09-12T23:37:51.332 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 170, 2024-09-12T23:37:51.332 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 308319 2024-09-12T23:37:51.332 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.332 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_sync_lat": { 2024-09-12T23:37:51.332 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 170, 2024-09-12T23:37:51.332 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 62380127 2024-09-12T23:37:51.332 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.332 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_get_keys_lat": { 2024-09-12T23:37:51.332 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.333 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.333 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.333 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_get_values_lat": { 2024-09-12T23:37:51.333 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 3, 2024-09-12T23:37:51.333 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 45931 2024-09-12T23:37:51.333 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.333 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_lower_bound_lat": { 2024-09-12T23:37:51.333 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 2, 2024-09-12T23:37:51.333 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4017 2024-09-12T23:37:51.333 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.333 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_next_lat": { 2024-09-12T23:37:51.333 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.333 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.334 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.334 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_seek_to_first_lat": { 2024-09-12T23:37:51.334 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.334 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.334 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.334 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_upper_bound_lat": { 2024-09-12T23:37:51.334 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 2, 2024-09-12T23:37:51.334 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4566 2024-09-12T23:37:51.334 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.334 INFO:tasks.workunit.client.0.smithi136.stdout: "onode_hits": { 2024-09-12T23:37:51.334 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 339 2024-09-12T23:37:51.334 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.334 INFO:tasks.workunit.client.0.smithi136.stdout: "onode_misses": { 2024-09-12T23:37:51.334 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 230 2024-09-12T23:37:51.334 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.335 INFO:tasks.workunit.client.0.smithi136.stdout: "read_lat": { 2024-09-12T23:37:51.335 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 81, 2024-09-12T23:37:51.335 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1216840 2024-09-12T23:37:51.335 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.335 INFO:tasks.workunit.client.0.smithi136.stdout: "read_onode_meta_lat": { 2024-09-12T23:37:51.335 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 129, 2024-09-12T23:37:51.335 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 463065 2024-09-12T23:37:51.335 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.335 INFO:tasks.workunit.client.0.smithi136.stdout: "read_wait_aio_lat": { 2024-09-12T23:37:51.335 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 48, 2024-09-12T23:37:51.335 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 582298 2024-09-12T23:37:51.335 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.335 INFO:tasks.workunit.client.0.smithi136.stdout: "reads_with_retries": { 2024-09-12T23:37:51.335 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.335 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.336 INFO:tasks.workunit.client.0.smithi136.stdout: "remove_lat": { 2024-09-12T23:37:51.336 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 5, 2024-09-12T23:37:51.336 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 199719 2024-09-12T23:37:51.336 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.336 INFO:tasks.workunit.client.0.smithi136.stdout: "state_aio_wait_lat": { 2024-09-12T23:37:51.336 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 281, 2024-09-12T23:37:51.336 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4258657 2024-09-12T23:37:51.336 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.336 INFO:tasks.workunit.client.0.smithi136.stdout: "state_deferred_aio_wait_lat": { 2024-09-12T23:37:51.336 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 38, 2024-09-12T23:37:51.336 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 3695282 2024-09-12T23:37:51.336 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.336 INFO:tasks.workunit.client.0.smithi136.stdout: "state_deferred_cleanup_lat": { 2024-09-12T23:37:51.336 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 35, 2024-09-12T23:37:51.336 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 509531519261 2024-09-12T23:37:51.337 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.337 INFO:tasks.workunit.client.0.smithi136.stdout: "state_deferred_queued_lat": { 2024-09-12T23:37:51.337 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 38, 2024-09-12T23:37:51.337 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 64810170308 2024-09-12T23:37:51.337 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.337 INFO:tasks.workunit.client.0.smithi136.stdout: "state_done_lat": { 2024-09-12T23:37:51.337 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 278, 2024-09-12T23:37:51.337 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 6221345120 2024-09-12T23:37:51.337 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.337 INFO:tasks.workunit.client.0.smithi136.stdout: "state_finishing_lat": { 2024-09-12T23:37:51.337 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 278, 2024-09-12T23:37:51.337 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 47453 2024-09-12T23:37:51.337 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.338 INFO:tasks.workunit.client.0.smithi136.stdout: "state_io_done_lat": { 2024-09-12T23:37:51.338 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 281, 2024-09-12T23:37:51.338 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 355872 2024-09-12T23:37:51.338 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.338 INFO:tasks.workunit.client.0.smithi136.stdout: "state_kv_commiting_lat": { 2024-09-12T23:37:51.338 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 281, 2024-09-12T23:37:51.338 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 158103846 2024-09-12T23:37:51.338 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.338 INFO:tasks.workunit.client.0.smithi136.stdout: "state_kv_done_lat": { 2024-09-12T23:37:51.338 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 281, 2024-09-12T23:37:51.338 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 130632 2024-09-12T23:37:51.338 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.338 INFO:tasks.workunit.client.0.smithi136.stdout: "state_kv_queued_lat": { 2024-09-12T23:37:51.338 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 281, 2024-09-12T23:37:51.339 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 125272432 2024-09-12T23:37:51.339 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.339 INFO:tasks.workunit.client.0.smithi136.stdout: "state_prepare_lat": { 2024-09-12T23:37:51.339 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 281, 2024-09-12T23:37:51.339 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 21794104 2024-09-12T23:37:51.339 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.339 INFO:tasks.workunit.client.0.smithi136.stdout: "stored": { 2024-09-12T23:37:51.339 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 293639 2024-09-12T23:37:51.339 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.339 INFO:tasks.workunit.client.0.smithi136.stdout: "truncate_lat": { 2024-09-12T23:37:51.339 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 30, 2024-09-12T23:37:51.339 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 212811 2024-09-12T23:37:51.339 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.339 INFO:tasks.workunit.client.0.smithi136.stdout: "txc_commit_lat": { 2024-09-12T23:37:51.340 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 281, 2024-09-12T23:37:51.340 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 309841697 2024-09-12T23:37:51.340 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.340 INFO:tasks.workunit.client.0.smithi136.stdout: "txc_submit_lat": { 2024-09-12T23:37:51.340 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 281, 2024-09-12T23:37:51.340 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 25366187 2024-09-12T23:37:51.340 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.340 INFO:tasks.workunit.client.0.smithi136.stdout: "txc_throttle_lat": { 2024-09-12T23:37:51.340 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 281, 2024-09-12T23:37:51.340 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5054111 2024-09-12T23:37:51.340 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.340 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.341 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache": { 2024-09-12T23:37:51.341 INFO:tasks.workunit.client.0.smithi136.stdout: "cache_bytes": { 2024-09-12T23:37:51.341 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1900963507 2024-09-12T23:37:51.341 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.341 INFO:tasks.workunit.client.0.smithi136.stdout: "heap_bytes": { 2024-09-12T23:37:51.341 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 269926400 2024-09-12T23:37:51.341 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.341 INFO:tasks.workunit.client.0.smithi136.stdout: "mapped_bytes": { 2024-09-12T23:37:51.341 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 269803520 2024-09-12T23:37:51.341 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.341 INFO:tasks.workunit.client.0.smithi136.stdout: "target_bytes": { 2024-09-12T23:37:51.341 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 3183846912 2024-09-12T23:37:51.341 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.341 INFO:tasks.workunit.client.0.smithi136.stdout: "unmapped_bytes": { 2024-09-12T23:37:51.342 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 122880 2024-09-12T23:37:51.342 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.342 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.342 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:data": { 2024-09-12T23:37:51.342 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:51.342 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 159383552 2024-09-12T23:37:51.342 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.344 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:51.344 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.344 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.344 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:51.344 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.344 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.344 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:51.344 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 87869932 2024-09-12T23:37:51.344 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.344 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:51.344 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.344 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.344 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:51.345 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.345 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.345 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:51.345 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.345 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.345 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:51.345 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.345 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.345 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:51.345 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.345 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.345 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:51.345 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4096 2024-09-12T23:37:51.346 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.346 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:51.346 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.346 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.346 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:51.346 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.346 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.346 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:51.346 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.346 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.346 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:51.346 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 71509524 2024-09-12T23:37:51.346 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.346 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.347 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:kv": { 2024-09-12T23:37:51.347 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:51.347 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 796917760 2024-09-12T23:37:51.347 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.347 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:51.347 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67108864 2024-09-12T23:37:51.347 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.347 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:51.347 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.347 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.347 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:51.347 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 659024495 2024-09-12T23:37:51.347 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.347 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:51.347 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.348 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.348 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:51.348 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.348 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.348 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:51.348 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.348 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.348 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:51.348 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.348 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.348 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:51.348 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.348 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.348 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:51.349 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2128 2024-09-12T23:37:51.349 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.349 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:51.349 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.349 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.349 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:51.349 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.349 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.349 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:51.349 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.349 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.349 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:51.349 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 70782273 2024-09-12T23:37:51.349 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.350 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.350 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:kv_onode": { 2024-09-12T23:37:51.350 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:51.350 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 192937984 2024-09-12T23:37:51.350 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.350 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:51.350 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67108864 2024-09-12T23:37:51.350 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.350 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:51.350 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 464 2024-09-12T23:37:51.350 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.350 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:51.350 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 58579955 2024-09-12T23:37:51.351 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.351 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:51.351 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.351 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.351 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:51.351 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.351 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.351 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:51.351 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.351 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.351 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:51.351 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.351 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.351 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:51.352 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.352 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.352 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:51.352 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.352 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.352 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:51.352 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.352 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.352 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:51.352 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.352 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.352 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:51.352 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.352 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.353 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:51.353 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67248701 2024-09-12T23:37:51.353 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.353 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.353 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:meta": { 2024-09-12T23:37:51.353 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:51.353 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 729808896 2024-09-12T23:37:51.353 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.353 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:51.353 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.353 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.353 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:51.353 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.353 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.354 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:51.354 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 659024495 2024-09-12T23:37:51.354 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.354 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:51.354 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.354 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.354 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:51.354 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.354 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.354 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:51.354 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.354 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.354 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:51.354 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.355 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.355 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:51.355 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.355 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.355 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:51.355 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 250324 2024-09-12T23:37:51.355 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.355 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:51.355 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.355 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.355 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:51.355 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.355 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.355 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:51.356 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.356 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.356 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:51.356 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 70534077 2024-09-12T23:37:51.356 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.356 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.356 INFO:tasks.workunit.client.0.smithi136.stdout: "objecter": { 2024-09-12T23:37:51.356 INFO:tasks.workunit.client.0.smithi136.stdout: "op_active": { 2024-09-12T23:37:51.356 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.356 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.356 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r": { 2024-09-12T23:37:51.356 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.356 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.356 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rmw": { 2024-09-12T23:37:51.357 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.357 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.357 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w": { 2024-09-12T23:37:51.357 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.357 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.357 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.357 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:51.357 INFO:tasks.workunit.client.0.smithi136.stdout: "numpg": { 2024-09-12T23:37:51.357 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 42 2024-09-12T23:37:51.357 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.357 INFO:tasks.workunit.client.0.smithi136.stdout: "numpg_removing": { 2024-09-12T23:37:51.357 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.357 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.357 INFO:tasks.workunit.client.0.smithi136.stdout: "op": { 2024-09-12T23:37:51.358 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.358 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.358 INFO:tasks.workunit.client.0.smithi136.stdout: "op_delayed_degraded": { 2024-09-12T23:37:51.358 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.358 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.358 INFO:tasks.workunit.client.0.smithi136.stdout: "op_delayed_unreadable": { 2024-09-12T23:37:51.358 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.358 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.358 INFO:tasks.workunit.client.0.smithi136.stdout: "op_in_bytes": { 2024-09-12T23:37:51.358 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.358 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.358 INFO:tasks.workunit.client.0.smithi136.stdout: "op_latency": { 2024-09-12T23:37:51.358 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.358 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.359 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.359 INFO:tasks.workunit.client.0.smithi136.stdout: "op_out_bytes": { 2024-09-12T23:37:51.359 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.359 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.359 INFO:tasks.workunit.client.0.smithi136.stdout: "op_prepare_latency": { 2024-09-12T23:37:51.359 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.359 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.359 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.359 INFO:tasks.workunit.client.0.smithi136.stdout: "op_process_latency": { 2024-09-12T23:37:51.359 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.359 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.359 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.359 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r": { 2024-09-12T23:37:51.359 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.360 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.360 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_latency": { 2024-09-12T23:37:51.360 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.360 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.360 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.360 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-12T23:37:51.360 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.360 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.360 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_out_bytes": { 2024-09-12T23:37:51.360 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.360 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.360 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_prepare_latency": { 2024-09-12T23:37:51.360 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.360 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.360 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.361 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_process_latency": { 2024-09-12T23:37:51.361 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.361 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.361 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.361 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw": { 2024-09-12T23:37:51.361 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.361 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.361 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_in_bytes": { 2024-09-12T23:37:51.361 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.361 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.361 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_latency": { 2024-09-12T23:37:51.361 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.361 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.362 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.362 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-12T23:37:51.362 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.362 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.362 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-12T23:37:51.362 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.362 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.362 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_out_bytes": { 2024-09-12T23:37:51.362 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.362 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.362 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_prepare_latency": { 2024-09-12T23:37:51.362 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.362 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.362 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.362 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_process_latency": { 2024-09-12T23:37:51.363 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.363 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.363 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.363 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w": { 2024-09-12T23:37:51.363 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.363 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.363 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_in_bytes": { 2024-09-12T23:37:51.363 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.363 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.363 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_latency": { 2024-09-12T23:37:51.363 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.363 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.363 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.363 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-12T23:37:51.364 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.364 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.364 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_prepare_latency": { 2024-09-12T23:37:51.364 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.364 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.364 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.364 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_process_latency": { 2024-09-12T23:37:51.364 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.364 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.364 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.364 INFO:tasks.workunit.client.0.smithi136.stdout: "op_wip": { 2024-09-12T23:37:51.364 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.364 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.365 INFO:tasks.workunit.client.0.smithi136.stdout: "recovery_bytes": { 2024-09-12T23:37:51.365 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 590368 2024-09-12T23:37:51.365 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.365 INFO:tasks.workunit.client.0.smithi136.stdout: "recovery_ops": { 2024-09-12T23:37:51.365 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2 2024-09-12T23:37:51.365 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.365 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_bytes": { 2024-09-12T23:37:51.365 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 95995035648 2024-09-12T23:37:51.365 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.365 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_bytes_used": { 2024-09-12T23:37:51.365 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 305152000 2024-09-12T23:37:51.365 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.365 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.365 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.365 INFO:tasks.workunit.client.0.smithi136.stdout: "osd.6": { 2024-09-12T23:37:51.366 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs": { 2024-09-12T23:37:51.366 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_slow_fallback": { 2024-09-12T23:37:51.366 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.366 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.366 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_slow_size_fallback": { 2024-09-12T23:37:51.366 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.366 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.366 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit_db": { 2024-09-12T23:37:51.366 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 65536 2024-09-12T23:37:51.366 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.366 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit_main": { 2024-09-12T23:37:51.366 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.366 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.366 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit_wal": { 2024-09-12T23:37:51.367 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.367 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.367 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_written_slow": { 2024-09-12T23:37:51.367 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.367 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.367 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_written_sst": { 2024-09-12T23:37:51.367 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 12288 2024-09-12T23:37:51.367 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.367 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_written_wal": { 2024-09-12T23:37:51.367 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1396736 2024-09-12T23:37:51.367 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.367 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_lat": { 2024-09-12T23:37:51.367 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.367 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.368 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.368 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_lock_lat": { 2024-09-12T23:37:51.368 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.368 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.368 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.368 INFO:tasks.workunit.client.0.smithi136.stdout: "db_total_bytes": { 2024-09-12T23:37:51.368 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 95995035648 2024-09-12T23:37:51.368 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.368 INFO:tasks.workunit.client.0.smithi136.stdout: "db_used_bytes": { 2024-09-12T23:37:51.368 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 304480256 2024-09-12T23:37:51.368 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.368 INFO:tasks.workunit.client.0.smithi136.stdout: "log_bytes": { 2024-09-12T23:37:51.368 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1093632 2024-09-12T23:37:51.369 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.369 INFO:tasks.workunit.client.0.smithi136.stdout: "logged_bytes": { 2024-09-12T23:37:51.369 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 954368 2024-09-12T23:37:51.369 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.369 INFO:tasks.workunit.client.0.smithi136.stdout: "max_bytes_db": { 2024-09-12T23:37:51.369 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 312868864 2024-09-12T23:37:51.369 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.369 INFO:tasks.workunit.client.0.smithi136.stdout: "max_bytes_slow": { 2024-09-12T23:37:51.369 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.369 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.369 INFO:tasks.workunit.client.0.smithi136.stdout: "max_bytes_wal": { 2024-09-12T23:37:51.369 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.369 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.369 INFO:tasks.workunit.client.0.smithi136.stdout: "num_files": { 2024-09-12T23:37:51.370 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 14 2024-09-12T23:37:51.370 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.370 INFO:tasks.workunit.client.0.smithi136.stdout: "read_bytes": { 2024-09-12T23:37:51.370 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 195697 2024-09-12T23:37:51.370 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.370 INFO:tasks.workunit.client.0.smithi136.stdout: "read_count": { 2024-09-12T23:37:51.370 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 23 2024-09-12T23:37:51.370 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.370 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes": { 2024-09-12T23:37:51.370 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1269760 2024-09-12T23:37:51.370 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.370 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes_db": { 2024-09-12T23:37:51.370 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1273856 2024-09-12T23:37:51.370 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.371 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes_slow": { 2024-09-12T23:37:51.371 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.371 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.371 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes_wal": { 2024-09-12T23:37:51.371 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.371 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.371 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_count": { 2024-09-12T23:37:51.371 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 15 2024-09-12T23:37:51.371 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.371 INFO:tasks.workunit.client.0.smithi136.stdout: "read_prefetch_bytes": { 2024-09-12T23:37:51.371 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4789 2024-09-12T23:37:51.371 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.371 INFO:tasks.workunit.client.0.smithi136.stdout: "read_prefetch_count": { 2024-09-12T23:37:51.371 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4 2024-09-12T23:37:51.372 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.372 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_buffer_bytes": { 2024-09-12T23:37:51.372 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5589 2024-09-12T23:37:51.372 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.372 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_buffer_count": { 2024-09-12T23:37:51.372 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 24 2024-09-12T23:37:51.372 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.372 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_bytes": { 2024-09-12T23:37:51.372 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5589 2024-09-12T23:37:51.372 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.372 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_count": { 2024-09-12T23:37:51.372 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 24 2024-09-12T23:37:51.372 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.372 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes": { 2024-09-12T23:37:51.373 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.373 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.373 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes_db": { 2024-09-12T23:37:51.373 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.373 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.373 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes_slow": { 2024-09-12T23:37:51.373 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.373 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.373 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes_wal": { 2024-09-12T23:37:51.373 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.373 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.373 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_count": { 2024-09-12T23:37:51.374 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.374 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.374 INFO:tasks.workunit.client.0.smithi136.stdout: "slow_total_bytes": { 2024-09-12T23:37:51.374 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.374 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.374 INFO:tasks.workunit.client.0.smithi136.stdout: "slow_used_bytes": { 2024-09-12T23:37:51.374 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.374 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.374 INFO:tasks.workunit.client.0.smithi136.stdout: "wal_total_bytes": { 2024-09-12T23:37:51.374 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.374 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.374 INFO:tasks.workunit.client.0.smithi136.stdout: "wal_used_bytes": { 2024-09-12T23:37:51.374 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.374 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.374 INFO:tasks.workunit.client.0.smithi136.stdout: "write_bytes": { 2024-09-12T23:37:51.375 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2424832 2024-09-12T23:37:51.375 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.375 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.375 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore": { 2024-09-12T23:37:51.375 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit": { 2024-09-12T23:37:51.375 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4096 2024-09-12T23:37:51.375 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.375 INFO:tasks.workunit.client.0.smithi136.stdout: "allocated": { 2024-09-12T23:37:51.375 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1253376 2024-09-12T23:37:51.375 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.375 INFO:tasks.workunit.client.0.smithi136.stdout: "clist_lat": { 2024-09-12T23:37:51.375 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.375 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.375 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.376 INFO:tasks.workunit.client.0.smithi136.stdout: "compress_lat": { 2024-09-12T23:37:51.376 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.376 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.376 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.376 INFO:tasks.workunit.client.0.smithi136.stdout: "compressed": { 2024-09-12T23:37:51.376 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.376 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.376 INFO:tasks.workunit.client.0.smithi136.stdout: "compressed_allocated": { 2024-09-12T23:37:51.376 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.376 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.376 INFO:tasks.workunit.client.0.smithi136.stdout: "compressed_original": { 2024-09-12T23:37:51.376 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.376 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.376 INFO:tasks.workunit.client.0.smithi136.stdout: "csum_lat": { 2024-09-12T23:37:51.377 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 7, 2024-09-12T23:37:51.377 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 19807 2024-09-12T23:37:51.377 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.377 INFO:tasks.workunit.client.0.smithi136.stdout: "decompress_lat": { 2024-09-12T23:37:51.377 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.377 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.377 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.377 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_commit_lat": { 2024-09-12T23:37:51.377 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 216, 2024-09-12T23:37:51.377 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 101200550 2024-09-12T23:37:51.377 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.377 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_final_lat": { 2024-09-12T23:37:51.377 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 215, 2024-09-12T23:37:51.378 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 9807463 2024-09-12T23:37:51.378 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.378 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_flush_lat": { 2024-09-12T23:37:51.378 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 216, 2024-09-12T23:37:51.378 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 325110 2024-09-12T23:37:51.378 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.378 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_sync_lat": { 2024-09-12T23:37:51.378 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 216, 2024-09-12T23:37:51.378 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 101525660 2024-09-12T23:37:51.378 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.378 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_get_keys_lat": { 2024-09-12T23:37:51.378 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.378 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.378 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.378 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_get_values_lat": { 2024-09-12T23:37:51.379 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 1, 2024-09-12T23:37:51.379 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 45213 2024-09-12T23:37:51.379 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.379 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_lower_bound_lat": { 2024-09-12T23:37:51.379 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 2, 2024-09-12T23:37:51.379 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2949 2024-09-12T23:37:51.379 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.379 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_next_lat": { 2024-09-12T23:37:51.379 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.379 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.379 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.379 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_seek_to_first_lat": { 2024-09-12T23:37:51.379 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.379 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.380 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.380 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_upper_bound_lat": { 2024-09-12T23:37:51.380 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 3, 2024-09-12T23:37:51.380 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 15110 2024-09-12T23:37:51.380 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.380 INFO:tasks.workunit.client.0.smithi136.stdout: "onode_hits": { 2024-09-12T23:37:51.380 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 369 2024-09-12T23:37:51.380 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.380 INFO:tasks.workunit.client.0.smithi136.stdout: "onode_misses": { 2024-09-12T23:37:51.380 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 234 2024-09-12T23:37:51.380 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.380 INFO:tasks.workunit.client.0.smithi136.stdout: "read_lat": { 2024-09-12T23:37:51.380 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 82, 2024-09-12T23:37:51.381 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1638877 2024-09-12T23:37:51.381 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.381 INFO:tasks.workunit.client.0.smithi136.stdout: "read_onode_meta_lat": { 2024-09-12T23:37:51.381 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 127, 2024-09-12T23:37:51.381 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 451371 2024-09-12T23:37:51.381 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.381 INFO:tasks.workunit.client.0.smithi136.stdout: "read_wait_aio_lat": { 2024-09-12T23:37:51.381 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 45, 2024-09-12T23:37:51.381 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1101545 2024-09-12T23:37:51.381 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.381 INFO:tasks.workunit.client.0.smithi136.stdout: "reads_with_retries": { 2024-09-12T23:37:51.381 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.381 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.381 INFO:tasks.workunit.client.0.smithi136.stdout: "remove_lat": { 2024-09-12T23:37:51.381 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.382 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.382 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.382 INFO:tasks.workunit.client.0.smithi136.stdout: "state_aio_wait_lat": { 2024-09-12T23:37:51.382 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 283, 2024-09-12T23:37:51.382 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 3761463 2024-09-12T23:37:51.382 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.382 INFO:tasks.workunit.client.0.smithi136.stdout: "state_deferred_aio_wait_lat": { 2024-09-12T23:37:51.382 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 33, 2024-09-12T23:37:51.382 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 3174770 2024-09-12T23:37:51.382 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.382 INFO:tasks.workunit.client.0.smithi136.stdout: "state_deferred_cleanup_lat": { 2024-09-12T23:37:51.382 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 33, 2024-09-12T23:37:51.382 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 580757760719 2024-09-12T23:37:51.382 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.382 INFO:tasks.workunit.client.0.smithi136.stdout: "state_deferred_queued_lat": { 2024-09-12T23:37:51.382 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 33, 2024-09-12T23:37:51.383 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 46789027563 2024-09-12T23:37:51.383 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.383 INFO:tasks.workunit.client.0.smithi136.stdout: "state_done_lat": { 2024-09-12T23:37:51.383 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 283, 2024-09-12T23:37:51.383 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 781028 2024-09-12T23:37:51.383 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.383 INFO:tasks.workunit.client.0.smithi136.stdout: "state_finishing_lat": { 2024-09-12T23:37:51.383 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 283, 2024-09-12T23:37:51.383 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 66907 2024-09-12T23:37:51.383 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.383 INFO:tasks.workunit.client.0.smithi136.stdout: "state_io_done_lat": { 2024-09-12T23:37:51.383 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 283, 2024-09-12T23:37:51.383 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 384998 2024-09-12T23:37:51.383 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.383 INFO:tasks.workunit.client.0.smithi136.stdout: "state_kv_commiting_lat": { 2024-09-12T23:37:51.384 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 283, 2024-09-12T23:37:51.384 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 175260161 2024-09-12T23:37:51.384 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.384 INFO:tasks.workunit.client.0.smithi136.stdout: "state_kv_done_lat": { 2024-09-12T23:37:51.384 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 283, 2024-09-12T23:37:51.384 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 180238 2024-09-12T23:37:51.384 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.384 INFO:tasks.workunit.client.0.smithi136.stdout: "state_kv_queued_lat": { 2024-09-12T23:37:51.384 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 283, 2024-09-12T23:37:51.384 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 174382174 2024-09-12T23:37:51.384 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.384 INFO:tasks.workunit.client.0.smithi136.stdout: "state_prepare_lat": { 2024-09-12T23:37:51.384 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 283, 2024-09-12T23:37:51.384 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 20382371 2024-09-12T23:37:51.385 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.385 INFO:tasks.workunit.client.0.smithi136.stdout: "stored": { 2024-09-12T23:37:51.385 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 884213 2024-09-12T23:37:51.385 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.385 INFO:tasks.workunit.client.0.smithi136.stdout: "truncate_lat": { 2024-09-12T23:37:51.385 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 26, 2024-09-12T23:37:51.385 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 207664 2024-09-12T23:37:51.385 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.385 INFO:tasks.workunit.client.0.smithi136.stdout: "txc_commit_lat": { 2024-09-12T23:37:51.385 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 283, 2024-09-12T23:37:51.385 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 374248117 2024-09-12T23:37:51.385 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.385 INFO:tasks.workunit.client.0.smithi136.stdout: "txc_submit_lat": { 2024-09-12T23:37:51.385 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 283, 2024-09-12T23:37:51.385 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 24468286 2024-09-12T23:37:51.386 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.386 INFO:tasks.workunit.client.0.smithi136.stdout: "txc_throttle_lat": { 2024-09-12T23:37:51.386 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 283, 2024-09-12T23:37:51.386 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 3266313 2024-09-12T23:37:51.386 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.386 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.386 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache": { 2024-09-12T23:37:51.386 INFO:tasks.workunit.client.0.smithi136.stdout: "cache_bytes": { 2024-09-12T23:37:51.386 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1900963507 2024-09-12T23:37:51.386 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.386 INFO:tasks.workunit.client.0.smithi136.stdout: "heap_bytes": { 2024-09-12T23:37:51.386 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 271187968 2024-09-12T23:37:51.386 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.386 INFO:tasks.workunit.client.0.smithi136.stdout: "mapped_bytes": { 2024-09-12T23:37:51.387 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 269885440 2024-09-12T23:37:51.387 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.387 INFO:tasks.workunit.client.0.smithi136.stdout: "target_bytes": { 2024-09-12T23:37:51.387 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 3183846912 2024-09-12T23:37:51.387 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.387 INFO:tasks.workunit.client.0.smithi136.stdout: "unmapped_bytes": { 2024-09-12T23:37:51.387 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1302528 2024-09-12T23:37:51.387 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.387 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.387 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:data": { 2024-09-12T23:37:51.387 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:51.387 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 159383552 2024-09-12T23:37:51.387 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.387 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:51.387 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.388 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.388 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:51.388 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.388 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.388 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:51.388 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 87868964 2024-09-12T23:37:51.388 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.388 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:51.388 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.388 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.388 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:51.388 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.388 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.388 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:51.388 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.389 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.389 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:51.389 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.389 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.389 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:51.389 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.389 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.389 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:51.389 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 16384 2024-09-12T23:37:51.389 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.389 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:51.389 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.389 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.389 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:51.390 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.390 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.390 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:51.390 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.390 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.390 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:51.390 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 71498204 2024-09-12T23:37:51.390 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.390 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.390 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:kv": { 2024-09-12T23:37:51.390 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:51.390 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 796917760 2024-09-12T23:37:51.390 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.390 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:51.390 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67108864 2024-09-12T23:37:51.391 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.391 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:51.391 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.391 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.391 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:51.391 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 659017234 2024-09-12T23:37:51.391 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.391 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:51.391 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.391 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.391 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:51.391 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.391 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.391 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:51.391 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.392 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.392 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:51.392 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.392 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.392 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:51.392 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.392 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.392 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:51.392 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2128 2024-09-12T23:37:51.392 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.392 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:51.392 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.392 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.392 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:51.392 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.393 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.393 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:51.393 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.393 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.393 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:51.393 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 70789534 2024-09-12T23:37:51.393 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.393 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.393 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:kv_onode": { 2024-09-12T23:37:51.393 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:51.393 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 192937984 2024-09-12T23:37:51.393 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.393 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:51.393 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67108864 2024-09-12T23:37:51.393 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.394 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:51.394 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 464 2024-09-12T23:37:51.394 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.394 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:51.394 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 58579309 2024-09-12T23:37:51.394 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.394 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:51.394 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.394 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.394 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:51.394 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.394 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.394 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:51.394 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.394 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.394 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:51.395 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.395 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.395 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:51.395 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.395 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.395 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:51.395 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.395 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.395 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:51.395 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.395 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.395 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:51.395 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.395 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.395 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:51.395 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.396 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.396 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:51.396 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67249347 2024-09-12T23:37:51.396 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.396 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.396 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:meta": { 2024-09-12T23:37:51.396 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:51.396 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 729808896 2024-09-12T23:37:51.396 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.396 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:51.396 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.396 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.396 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:51.396 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.397 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.397 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:51.397 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 659017234 2024-09-12T23:37:51.397 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.397 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:51.397 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.397 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.397 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:51.397 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2839 2024-09-12T23:37:51.397 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.397 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:51.397 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4259 2024-09-12T23:37:51.397 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.398 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:51.398 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.398 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.398 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:51.398 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.398 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.398 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:51.398 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 247073 2024-09-12T23:37:51.398 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.398 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:51.398 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.398 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.398 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:51.398 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.398 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.399 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:51.399 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.399 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.399 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:51.399 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 70537491 2024-09-12T23:37:51.399 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.399 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.399 INFO:tasks.workunit.client.0.smithi136.stdout: "objecter": { 2024-09-12T23:37:51.399 INFO:tasks.workunit.client.0.smithi136.stdout: "op_active": { 2024-09-12T23:37:51.399 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.399 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.399 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r": { 2024-09-12T23:37:51.399 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.399 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.399 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rmw": { 2024-09-12T23:37:51.400 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.400 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.400 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w": { 2024-09-12T23:37:51.400 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.400 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.400 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.400 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:51.400 INFO:tasks.workunit.client.0.smithi136.stdout: "numpg": { 2024-09-12T23:37:51.400 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 39 2024-09-12T23:37:51.400 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.400 INFO:tasks.workunit.client.0.smithi136.stdout: "numpg_removing": { 2024-09-12T23:37:51.400 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.400 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.400 INFO:tasks.workunit.client.0.smithi136.stdout: "op": { 2024-09-12T23:37:51.401 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 13 2024-09-12T23:37:51.401 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.401 INFO:tasks.workunit.client.0.smithi136.stdout: "op_delayed_degraded": { 2024-09-12T23:37:51.401 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.401 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.401 INFO:tasks.workunit.client.0.smithi136.stdout: "op_delayed_unreadable": { 2024-09-12T23:37:51.401 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.401 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.401 INFO:tasks.workunit.client.0.smithi136.stdout: "op_in_bytes": { 2024-09-12T23:37:51.401 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2804 2024-09-12T23:37:51.401 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.401 INFO:tasks.workunit.client.0.smithi136.stdout: "op_latency": { 2024-09-12T23:37:51.401 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 13, 2024-09-12T23:37:51.401 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 143192594 2024-09-12T23:37:51.402 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.402 INFO:tasks.workunit.client.0.smithi136.stdout: "op_out_bytes": { 2024-09-12T23:37:51.402 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.402 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.402 INFO:tasks.workunit.client.0.smithi136.stdout: "op_prepare_latency": { 2024-09-12T23:37:51.402 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 15, 2024-09-12T23:37:51.402 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 10460108 2024-09-12T23:37:51.402 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.402 INFO:tasks.workunit.client.0.smithi136.stdout: "op_process_latency": { 2024-09-12T23:37:51.402 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 13, 2024-09-12T23:37:51.402 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 126492015 2024-09-12T23:37:51.402 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.402 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r": { 2024-09-12T23:37:51.402 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.402 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.403 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_latency": { 2024-09-12T23:37:51.403 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.403 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.403 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.403 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-12T23:37:51.403 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.403 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.403 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_out_bytes": { 2024-09-12T23:37:51.403 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.403 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.403 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_prepare_latency": { 2024-09-12T23:37:51.403 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.403 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.403 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.404 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_process_latency": { 2024-09-12T23:37:51.404 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.404 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.404 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.404 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw": { 2024-09-12T23:37:51.404 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.404 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.404 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_in_bytes": { 2024-09-12T23:37:51.404 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.404 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.404 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_latency": { 2024-09-12T23:37:51.404 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.404 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.404 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.404 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-12T23:37:51.405 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.405 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.405 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-12T23:37:51.405 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.405 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.405 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_out_bytes": { 2024-09-12T23:37:51.405 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.405 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.405 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_prepare_latency": { 2024-09-12T23:37:51.405 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.405 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.405 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.405 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_process_latency": { 2024-09-12T23:37:51.406 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.406 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.406 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.406 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w": { 2024-09-12T23:37:51.406 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 13 2024-09-12T23:37:51.406 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.406 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_in_bytes": { 2024-09-12T23:37:51.406 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2804 2024-09-12T23:37:51.406 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.406 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_latency": { 2024-09-12T23:37:51.406 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 13, 2024-09-12T23:37:51.406 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 143192594 2024-09-12T23:37:51.406 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.406 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-12T23:37:51.407 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.407 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.407 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_prepare_latency": { 2024-09-12T23:37:51.407 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 15, 2024-09-12T23:37:51.407 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 10460108 2024-09-12T23:37:51.407 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.407 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_process_latency": { 2024-09-12T23:37:51.407 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 13, 2024-09-12T23:37:51.407 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 126492015 2024-09-12T23:37:51.407 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.407 INFO:tasks.workunit.client.0.smithi136.stdout: "op_wip": { 2024-09-12T23:37:51.407 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.407 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.407 INFO:tasks.workunit.client.0.smithi136.stdout: "recovery_bytes": { 2024-09-12T23:37:51.407 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 590368 2024-09-12T23:37:51.407 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.408 INFO:tasks.workunit.client.0.smithi136.stdout: "recovery_ops": { 2024-09-12T23:37:51.408 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2 2024-09-12T23:37:51.408 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.408 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_bytes": { 2024-09-12T23:37:51.408 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 95995035648 2024-09-12T23:37:51.408 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.408 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_bytes_used": { 2024-09-12T23:37:51.408 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 305741824 2024-09-12T23:37:51.408 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.408 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.408 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.408 INFO:tasks.workunit.client.0.smithi136.stdout: "osd.7": { 2024-09-12T23:37:51.408 INFO:tasks.workunit.client.0.smithi136.stdout: "bluefs": { 2024-09-12T23:37:51.408 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_slow_fallback": { 2024-09-12T23:37:51.409 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.409 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.409 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_slow_size_fallback": { 2024-09-12T23:37:51.409 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.409 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.409 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit_db": { 2024-09-12T23:37:51.409 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 65536 2024-09-12T23:37:51.409 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.409 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit_main": { 2024-09-12T23:37:51.409 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.409 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.409 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit_wal": { 2024-09-12T23:37:51.409 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.409 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.410 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_written_slow": { 2024-09-12T23:37:51.410 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.410 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.410 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_written_sst": { 2024-09-12T23:37:51.410 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 12288 2024-09-12T23:37:51.410 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.410 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_written_wal": { 2024-09-12T23:37:51.410 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1351680 2024-09-12T23:37:51.410 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.410 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_lat": { 2024-09-12T23:37:51.410 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.410 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.410 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.410 INFO:tasks.workunit.client.0.smithi136.stdout: "compact_lock_lat": { 2024-09-12T23:37:51.410 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.411 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.411 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.411 INFO:tasks.workunit.client.0.smithi136.stdout: "db_total_bytes": { 2024-09-12T23:37:51.411 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 95995035648 2024-09-12T23:37:51.411 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.411 INFO:tasks.workunit.client.0.smithi136.stdout: "db_used_bytes": { 2024-09-12T23:37:51.411 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 304480256 2024-09-12T23:37:51.411 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.411 INFO:tasks.workunit.client.0.smithi136.stdout: "log_bytes": { 2024-09-12T23:37:51.411 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1056768 2024-09-12T23:37:51.411 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.411 INFO:tasks.workunit.client.0.smithi136.stdout: "logged_bytes": { 2024-09-12T23:37:51.411 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 917504 2024-09-12T23:37:51.411 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.412 INFO:tasks.workunit.client.0.smithi136.stdout: "max_bytes_db": { 2024-09-12T23:37:51.412 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 312868864 2024-09-12T23:37:51.412 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.412 INFO:tasks.workunit.client.0.smithi136.stdout: "max_bytes_slow": { 2024-09-12T23:37:51.412 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.412 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.412 INFO:tasks.workunit.client.0.smithi136.stdout: "max_bytes_wal": { 2024-09-12T23:37:51.412 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.412 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.412 INFO:tasks.workunit.client.0.smithi136.stdout: "num_files": { 2024-09-12T23:37:51.412 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 14 2024-09-12T23:37:51.412 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.412 INFO:tasks.workunit.client.0.smithi136.stdout: "read_bytes": { 2024-09-12T23:37:51.412 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 195697 2024-09-12T23:37:51.412 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.412 INFO:tasks.workunit.client.0.smithi136.stdout: "read_count": { 2024-09-12T23:37:51.413 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 23 2024-09-12T23:37:51.413 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.413 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes": { 2024-09-12T23:37:51.413 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1269760 2024-09-12T23:37:51.413 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.413 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes_db": { 2024-09-12T23:37:51.413 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1273856 2024-09-12T23:37:51.413 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.413 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes_slow": { 2024-09-12T23:37:51.413 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.413 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.413 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_bytes_wal": { 2024-09-12T23:37:51.413 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.413 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.414 INFO:tasks.workunit.client.0.smithi136.stdout: "read_disk_count": { 2024-09-12T23:37:51.414 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 15 2024-09-12T23:37:51.414 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.414 INFO:tasks.workunit.client.0.smithi136.stdout: "read_prefetch_bytes": { 2024-09-12T23:37:51.414 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4789 2024-09-12T23:37:51.414 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.414 INFO:tasks.workunit.client.0.smithi136.stdout: "read_prefetch_count": { 2024-09-12T23:37:51.414 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4 2024-09-12T23:37:51.414 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.414 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_buffer_bytes": { 2024-09-12T23:37:51.414 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5589 2024-09-12T23:37:51.414 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.414 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_buffer_count": { 2024-09-12T23:37:51.414 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 24 2024-09-12T23:37:51.414 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.414 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_bytes": { 2024-09-12T23:37:51.415 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5589 2024-09-12T23:37:51.415 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.415 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_count": { 2024-09-12T23:37:51.415 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 24 2024-09-12T23:37:51.415 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.415 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes": { 2024-09-12T23:37:51.415 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.415 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.415 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes_db": { 2024-09-12T23:37:51.415 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.415 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.415 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes_slow": { 2024-09-12T23:37:51.415 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.415 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.415 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_bytes_wal": { 2024-09-12T23:37:51.416 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.416 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.416 INFO:tasks.workunit.client.0.smithi136.stdout: "read_random_disk_count": { 2024-09-12T23:37:51.416 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.416 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.416 INFO:tasks.workunit.client.0.smithi136.stdout: "slow_total_bytes": { 2024-09-12T23:37:51.416 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.416 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.416 INFO:tasks.workunit.client.0.smithi136.stdout: "slow_used_bytes": { 2024-09-12T23:37:51.416 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.416 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.416 INFO:tasks.workunit.client.0.smithi136.stdout: "wal_total_bytes": { 2024-09-12T23:37:51.416 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.416 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.417 INFO:tasks.workunit.client.0.smithi136.stdout: "wal_used_bytes": { 2024-09-12T23:37:51.417 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.417 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.417 INFO:tasks.workunit.client.0.smithi136.stdout: "write_bytes": { 2024-09-12T23:37:51.417 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2342912 2024-09-12T23:37:51.417 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.417 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.417 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore": { 2024-09-12T23:37:51.417 INFO:tasks.workunit.client.0.smithi136.stdout: "alloc_unit": { 2024-09-12T23:37:51.417 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4096 2024-09-12T23:37:51.417 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.417 INFO:tasks.workunit.client.0.smithi136.stdout: "allocated": { 2024-09-12T23:37:51.417 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1253376 2024-09-12T23:37:51.418 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.418 INFO:tasks.workunit.client.0.smithi136.stdout: "clist_lat": { 2024-09-12T23:37:51.418 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 5, 2024-09-12T23:37:51.418 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 381524 2024-09-12T23:37:51.418 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.418 INFO:tasks.workunit.client.0.smithi136.stdout: "compress_lat": { 2024-09-12T23:37:51.418 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.418 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.418 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.418 INFO:tasks.workunit.client.0.smithi136.stdout: "compressed": { 2024-09-12T23:37:51.418 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.418 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.418 INFO:tasks.workunit.client.0.smithi136.stdout: "compressed_allocated": { 2024-09-12T23:37:51.418 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.418 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.419 INFO:tasks.workunit.client.0.smithi136.stdout: "compressed_original": { 2024-09-12T23:37:51.419 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.419 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.419 INFO:tasks.workunit.client.0.smithi136.stdout: "csum_lat": { 2024-09-12T23:37:51.419 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 13, 2024-09-12T23:37:51.419 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 63196 2024-09-12T23:37:51.419 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.419 INFO:tasks.workunit.client.0.smithi136.stdout: "decompress_lat": { 2024-09-12T23:37:51.419 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.419 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.419 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.419 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_commit_lat": { 2024-09-12T23:37:51.419 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 207, 2024-09-12T23:37:51.419 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 106314966 2024-09-12T23:37:51.419 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.420 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_final_lat": { 2024-09-12T23:37:51.420 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 192, 2024-09-12T23:37:51.420 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 16760740 2024-09-12T23:37:51.420 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.420 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_flush_lat": { 2024-09-12T23:37:51.420 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 207, 2024-09-12T23:37:51.420 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 250075 2024-09-12T23:37:51.420 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.420 INFO:tasks.workunit.client.0.smithi136.stdout: "kv_sync_lat": { 2024-09-12T23:37:51.420 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 207, 2024-09-12T23:37:51.420 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 106565041 2024-09-12T23:37:51.420 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.420 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_get_keys_lat": { 2024-09-12T23:37:51.420 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.420 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.421 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.421 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_get_values_lat": { 2024-09-12T23:37:51.421 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 2, 2024-09-12T23:37:51.421 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 52591 2024-09-12T23:37:51.421 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.421 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_lower_bound_lat": { 2024-09-12T23:37:51.421 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 3, 2024-09-12T23:37:51.421 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4340 2024-09-12T23:37:51.421 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.421 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_next_lat": { 2024-09-12T23:37:51.421 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.421 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.421 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.421 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_seek_to_first_lat": { 2024-09-12T23:37:51.421 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 0, 2024-09-12T23:37:51.422 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.422 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.422 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_upper_bound_lat": { 2024-09-12T23:37:51.422 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 2, 2024-09-12T23:37:51.422 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 8636 2024-09-12T23:37:51.422 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.422 INFO:tasks.workunit.client.0.smithi136.stdout: "onode_hits": { 2024-09-12T23:37:51.422 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 725 2024-09-12T23:37:51.422 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.422 INFO:tasks.workunit.client.0.smithi136.stdout: "onode_misses": { 2024-09-12T23:37:51.422 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 267 2024-09-12T23:37:51.422 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.422 INFO:tasks.workunit.client.0.smithi136.stdout: "read_lat": { 2024-09-12T23:37:51.422 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 122, 2024-09-12T23:37:51.422 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4305057 2024-09-12T23:37:51.422 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.423 INFO:tasks.workunit.client.0.smithi136.stdout: "read_onode_meta_lat": { 2024-09-12T23:37:51.423 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 197, 2024-09-12T23:37:51.423 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 505144 2024-09-12T23:37:51.423 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.423 INFO:tasks.workunit.client.0.smithi136.stdout: "read_wait_aio_lat": { 2024-09-12T23:37:51.423 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 75, 2024-09-12T23:37:51.423 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2987455 2024-09-12T23:37:51.423 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.423 INFO:tasks.workunit.client.0.smithi136.stdout: "reads_with_retries": { 2024-09-12T23:37:51.423 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.423 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.423 INFO:tasks.workunit.client.0.smithi136.stdout: "remove_lat": { 2024-09-12T23:37:51.423 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 2, 2024-09-12T23:37:51.423 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 85362 2024-09-12T23:37:51.423 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.423 INFO:tasks.workunit.client.0.smithi136.stdout: "state_aio_wait_lat": { 2024-09-12T23:37:51.424 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 319, 2024-09-12T23:37:51.424 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 3298545 2024-09-12T23:37:51.424 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.424 INFO:tasks.workunit.client.0.smithi136.stdout: "state_deferred_aio_wait_lat": { 2024-09-12T23:37:51.424 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 21, 2024-09-12T23:37:51.424 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2172027 2024-09-12T23:37:51.424 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.424 INFO:tasks.workunit.client.0.smithi136.stdout: "state_deferred_cleanup_lat": { 2024-09-12T23:37:51.424 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 21, 2024-09-12T23:37:51.424 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 345186140205 2024-09-12T23:37:51.424 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.424 INFO:tasks.workunit.client.0.smithi136.stdout: "state_deferred_queued_lat": { 2024-09-12T23:37:51.424 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 21, 2024-09-12T23:37:51.424 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 33206269672 2024-09-12T23:37:51.425 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.425 INFO:tasks.workunit.client.0.smithi136.stdout: "state_done_lat": { 2024-09-12T23:37:51.425 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 319, 2024-09-12T23:37:51.425 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1115165 2024-09-12T23:37:51.425 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.425 INFO:tasks.workunit.client.0.smithi136.stdout: "state_finishing_lat": { 2024-09-12T23:37:51.425 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 319, 2024-09-12T23:37:51.425 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 64112 2024-09-12T23:37:51.425 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.425 INFO:tasks.workunit.client.0.smithi136.stdout: "state_io_done_lat": { 2024-09-12T23:37:51.425 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 319, 2024-09-12T23:37:51.425 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 351970 2024-09-12T23:37:51.425 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.425 INFO:tasks.workunit.client.0.smithi136.stdout: "state_kv_commiting_lat": { 2024-09-12T23:37:51.425 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 319, 2024-09-12T23:37:51.426 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 347629434 2024-09-12T23:37:51.426 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.426 INFO:tasks.workunit.client.0.smithi136.stdout: "state_kv_done_lat": { 2024-09-12T23:37:51.426 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 319, 2024-09-12T23:37:51.426 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 174259 2024-09-12T23:37:51.426 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.426 INFO:tasks.workunit.client.0.smithi136.stdout: "state_kv_queued_lat": { 2024-09-12T23:37:51.426 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 319, 2024-09-12T23:37:51.426 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 158575603 2024-09-12T23:37:51.426 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.426 INFO:tasks.workunit.client.0.smithi136.stdout: "state_prepare_lat": { 2024-09-12T23:37:51.426 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 319, 2024-09-12T23:37:51.426 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 27187291 2024-09-12T23:37:51.426 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.426 INFO:tasks.workunit.client.0.smithi136.stdout: "stored": { 2024-09-12T23:37:51.427 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 882496 2024-09-12T23:37:51.427 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.427 INFO:tasks.workunit.client.0.smithi136.stdout: "truncate_lat": { 2024-09-12T23:37:51.427 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 20, 2024-09-12T23:37:51.427 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 146134 2024-09-12T23:37:51.427 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.427 INFO:tasks.workunit.client.0.smithi136.stdout: "txc_commit_lat": { 2024-09-12T23:37:51.427 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 319, 2024-09-12T23:37:51.427 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 537123415 2024-09-12T23:37:51.427 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.427 INFO:tasks.workunit.client.0.smithi136.stdout: "txc_submit_lat": { 2024-09-12T23:37:51.427 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 319, 2024-09-12T23:37:51.427 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 32935662 2024-09-12T23:37:51.427 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.427 INFO:tasks.workunit.client.0.smithi136.stdout: "txc_throttle_lat": { 2024-09-12T23:37:51.427 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 319, 2024-09-12T23:37:51.428 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 7312883 2024-09-12T23:37:51.428 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.428 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.428 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache": { 2024-09-12T23:37:51.428 INFO:tasks.workunit.client.0.smithi136.stdout: "cache_bytes": { 2024-09-12T23:37:51.428 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 1900963507 2024-09-12T23:37:51.428 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.428 INFO:tasks.workunit.client.0.smithi136.stdout: "heap_bytes": { 2024-09-12T23:37:51.428 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 271122432 2024-09-12T23:37:51.428 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.428 INFO:tasks.workunit.client.0.smithi136.stdout: "mapped_bytes": { 2024-09-12T23:37:51.428 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 270991360 2024-09-12T23:37:51.428 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.428 INFO:tasks.workunit.client.0.smithi136.stdout: "target_bytes": { 2024-09-12T23:37:51.428 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 3183846912 2024-09-12T23:37:51.429 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.429 INFO:tasks.workunit.client.0.smithi136.stdout: "unmapped_bytes": { 2024-09-12T23:37:51.429 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 131072 2024-09-12T23:37:51.429 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.429 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.429 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:data": { 2024-09-12T23:37:51.429 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:51.429 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 159383552 2024-09-12T23:37:51.429 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.429 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:51.429 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.429 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.429 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:51.429 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.429 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.430 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:51.430 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 87851688 2024-09-12T23:37:51.430 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.430 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:51.430 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.430 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.430 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:51.430 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 8192 2024-09-12T23:37:51.430 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.430 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:51.430 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 147456 2024-09-12T23:37:51.430 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.430 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:51.430 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.430 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.431 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:51.431 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.431 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.431 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:51.431 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 135168 2024-09-12T23:37:51.431 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.431 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:51.431 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.431 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.431 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:51.431 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.431 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.431 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:51.431 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.431 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.431 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:51.432 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 71241048 2024-09-12T23:37:51.432 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.432 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.432 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:kv": { 2024-09-12T23:37:51.432 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:51.432 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 796917760 2024-09-12T23:37:51.432 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.432 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:51.432 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67108864 2024-09-12T23:37:51.432 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.432 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:51.432 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.432 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.432 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:51.433 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 658887665 2024-09-12T23:37:51.433 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.433 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:51.433 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.433 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.433 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:51.433 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.433 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.433 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:51.433 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.433 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.433 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:51.433 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.433 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.433 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:51.434 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.434 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.434 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:51.434 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2128 2024-09-12T23:37:51.434 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.434 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:51.434 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.434 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.434 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:51.434 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.434 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.434 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:51.434 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.434 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.434 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:51.435 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 70919103 2024-09-12T23:37:51.435 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.435 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.435 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:kv_onode": { 2024-09-12T23:37:51.435 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:51.435 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 192937984 2024-09-12T23:37:51.435 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.435 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:51.435 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67108864 2024-09-12T23:37:51.435 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.435 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:51.435 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 464 2024-09-12T23:37:51.435 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.435 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:51.435 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 58567792 2024-09-12T23:37:51.435 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.436 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:51.436 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.436 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.436 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:51.436 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.436 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.436 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:51.436 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.436 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.436 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:51.436 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.436 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.436 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:51.436 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.437 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.437 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:51.437 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.437 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.437 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:51.437 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.437 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.437 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:51.437 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.437 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.437 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:51.437 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.437 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.437 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:51.437 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 67260864 2024-09-12T23:37:51.437 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.438 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.438 INFO:tasks.workunit.client.0.smithi136.stdout: "bluestore-pricache:meta": { 2024-09-12T23:37:51.438 INFO:tasks.workunit.client.0.smithi136.stdout: "committed_bytes": { 2024-09-12T23:37:51.438 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 729808896 2024-09-12T23:37:51.438 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.438 INFO:tasks.workunit.client.0.smithi136.stdout: "pri0_bytes": { 2024-09-12T23:37:51.438 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.438 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.438 INFO:tasks.workunit.client.0.smithi136.stdout: "pri10_bytes": { 2024-09-12T23:37:51.438 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.438 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.438 INFO:tasks.workunit.client.0.smithi136.stdout: "pri11_bytes": { 2024-09-12T23:37:51.438 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 658887665 2024-09-12T23:37:51.438 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.439 INFO:tasks.workunit.client.0.smithi136.stdout: "pri1_bytes": { 2024-09-12T23:37:51.439 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.439 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.439 INFO:tasks.workunit.client.0.smithi136.stdout: "pri2_bytes": { 2024-09-12T23:37:51.439 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 4271 2024-09-12T23:37:51.439 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.439 INFO:tasks.workunit.client.0.smithi136.stdout: "pri3_bytes": { 2024-09-12T23:37:51.439 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 12814 2024-09-12T23:37:51.439 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.439 INFO:tasks.workunit.client.0.smithi136.stdout: "pri4_bytes": { 2024-09-12T23:37:51.439 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.439 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.439 INFO:tasks.workunit.client.0.smithi136.stdout: "pri5_bytes": { 2024-09-12T23:37:51.439 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.439 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.440 INFO:tasks.workunit.client.0.smithi136.stdout: "pri6_bytes": { 2024-09-12T23:37:51.440 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 250586 2024-09-12T23:37:51.440 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.440 INFO:tasks.workunit.client.0.smithi136.stdout: "pri7_bytes": { 2024-09-12T23:37:51.440 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.440 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.440 INFO:tasks.workunit.client.0.smithi136.stdout: "pri8_bytes": { 2024-09-12T23:37:51.440 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.440 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.440 INFO:tasks.workunit.client.0.smithi136.stdout: "pri9_bytes": { 2024-09-12T23:37:51.440 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.440 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.440 INFO:tasks.workunit.client.0.smithi136.stdout: "reserved_bytes": { 2024-09-12T23:37:51.440 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 70653560 2024-09-12T23:37:51.441 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.441 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.441 INFO:tasks.workunit.client.0.smithi136.stdout: "objecter": { 2024-09-12T23:37:51.441 INFO:tasks.workunit.client.0.smithi136.stdout: "op_active": { 2024-09-12T23:37:51.441 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.441 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.441 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r": { 2024-09-12T23:37:51.441 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.441 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.441 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rmw": { 2024-09-12T23:37:51.441 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.441 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.441 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w": { 2024-09-12T23:37:51.442 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.442 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.442 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.442 INFO:tasks.workunit.client.0.smithi136.stdout: "osd": { 2024-09-12T23:37:51.442 INFO:tasks.workunit.client.0.smithi136.stdout: "numpg": { 2024-09-12T23:37:51.442 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 47 2024-09-12T23:37:51.442 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.442 INFO:tasks.workunit.client.0.smithi136.stdout: "numpg_removing": { 2024-09-12T23:37:51.442 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.442 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.442 INFO:tasks.workunit.client.0.smithi136.stdout: "op": { 2024-09-12T23:37:51.442 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 143 2024-09-12T23:37:51.442 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.442 INFO:tasks.workunit.client.0.smithi136.stdout: "op_delayed_degraded": { 2024-09-12T23:37:51.443 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.443 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.443 INFO:tasks.workunit.client.0.smithi136.stdout: "op_delayed_unreadable": { 2024-09-12T23:37:51.443 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.443 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.443 INFO:tasks.workunit.client.0.smithi136.stdout: "op_in_bytes": { 2024-09-12T23:37:51.443 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.443 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.443 INFO:tasks.workunit.client.0.smithi136.stdout: "op_latency": { 2024-09-12T23:37:51.443 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 143, 2024-09-12T23:37:51.443 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 213105778 2024-09-12T23:37:51.443 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.443 INFO:tasks.workunit.client.0.smithi136.stdout: "op_out_bytes": { 2024-09-12T23:37:51.443 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 265829 2024-09-12T23:37:51.443 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.443 INFO:tasks.workunit.client.0.smithi136.stdout: "op_prepare_latency": { 2024-09-12T23:37:51.444 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 145, 2024-09-12T23:37:51.444 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 72616890 2024-09-12T23:37:51.444 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.444 INFO:tasks.workunit.client.0.smithi136.stdout: "op_process_latency": { 2024-09-12T23:37:51.444 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 143, 2024-09-12T23:37:51.444 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 186948728 2024-09-12T23:37:51.444 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.444 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r": { 2024-09-12T23:37:51.444 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 100 2024-09-12T23:37:51.444 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.444 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_latency": { 2024-09-12T23:37:51.444 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 100, 2024-09-12T23:37:51.444 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 39240678 2024-09-12T23:37:51.444 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.445 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-12T23:37:51.445 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.445 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.445 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_out_bytes": { 2024-09-12T23:37:51.445 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 265829 2024-09-12T23:37:51.445 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.445 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_prepare_latency": { 2024-09-12T23:37:51.445 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 100, 2024-09-12T23:37:51.445 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 32910534 2024-09-12T23:37:51.445 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.445 INFO:tasks.workunit.client.0.smithi136.stdout: "op_r_process_latency": { 2024-09-12T23:37:51.445 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 100, 2024-09-12T23:37:51.445 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 26512716 2024-09-12T23:37:51.445 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.445 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw": { 2024-09-12T23:37:51.445 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 38 2024-09-12T23:37:51.446 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.446 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_in_bytes": { 2024-09-12T23:37:51.446 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.446 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.446 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_latency": { 2024-09-12T23:37:51.446 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 38, 2024-09-12T23:37:51.446 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 78943201 2024-09-12T23:37:51.446 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.446 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-12T23:37:51.446 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.446 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.446 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-12T23:37:51.446 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.447 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.447 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_out_bytes": { 2024-09-12T23:37:51.447 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.447 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.447 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_prepare_latency": { 2024-09-12T23:37:51.447 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 38, 2024-09-12T23:37:51.447 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 33876795 2024-09-12T23:37:51.447 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.447 INFO:tasks.workunit.client.0.smithi136.stdout: "op_rw_process_latency": { 2024-09-12T23:37:51.447 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 38, 2024-09-12T23:37:51.447 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 73635074 2024-09-12T23:37:51.447 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.447 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w": { 2024-09-12T23:37:51.447 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5 2024-09-12T23:37:51.447 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.447 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_in_bytes": { 2024-09-12T23:37:51.448 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.448 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.448 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_latency": { 2024-09-12T23:37:51.448 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 5, 2024-09-12T23:37:51.448 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 94921899 2024-09-12T23:37:51.448 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.448 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-12T23:37:51.448 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.448 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.448 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_prepare_latency": { 2024-09-12T23:37:51.448 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 7, 2024-09-12T23:37:51.448 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 5829561 2024-09-12T23:37:51.448 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.448 INFO:tasks.workunit.client.0.smithi136.stdout: "op_w_process_latency": { 2024-09-12T23:37:51.448 INFO:tasks.workunit.client.0.smithi136.stdout: "count": 5, 2024-09-12T23:37:51.448 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 86800938 2024-09-12T23:37:51.449 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.449 INFO:tasks.workunit.client.0.smithi136.stdout: "op_wip": { 2024-09-12T23:37:51.449 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 0 2024-09-12T23:37:51.449 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.449 INFO:tasks.workunit.client.0.smithi136.stdout: "recovery_bytes": { 2024-09-12T23:37:51.449 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 590368 2024-09-12T23:37:51.449 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.449 INFO:tasks.workunit.client.0.smithi136.stdout: "recovery_ops": { 2024-09-12T23:37:51.449 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 2 2024-09-12T23:37:51.449 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.449 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_bytes": { 2024-09-12T23:37:51.449 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 95995035648 2024-09-12T23:37:51.449 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.449 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_bytes_used": { 2024-09-12T23:37:51.449 INFO:tasks.workunit.client.0.smithi136.stdout: "value": 305741824 2024-09-12T23:37:51.450 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.450 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.450 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.450 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.450 INFO:tasks.workunit.client.0.smithi136.stdout: "pools": [ 2024-09-12T23:37:51.450 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.450 INFO:tasks.workunit.client.0.smithi136.stdout: "application": [ 2024-09-12T23:37:51.450 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr" 2024-09-12T23:37:51.450 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.450 INFO:tasks.workunit.client.0.smithi136.stdout: "cache_mode": "none", 2024-09-12T23:37:51.450 INFO:tasks.workunit.client.0.smithi136.stdout: "erasure_code_profile": {}, 2024-09-12T23:37:51.450 INFO:tasks.workunit.client.0.smithi136.stdout: "min_size": 2, 2024-09-12T23:37:51.450 INFO:tasks.workunit.client.0.smithi136.stdout: "options": {}, 2024-09-12T23:37:51.450 INFO:tasks.workunit.client.0.smithi136.stdout: "pg_autoscale_mode": "off", 2024-09-12T23:37:51.450 INFO:tasks.workunit.client.0.smithi136.stdout: "pg_num": 1, 2024-09-12T23:37:51.451 INFO:tasks.workunit.client.0.smithi136.stdout: "pgp_num": 1, 2024-09-12T23:37:51.451 INFO:tasks.workunit.client.0.smithi136.stdout: "pool": 1, 2024-09-12T23:37:51.451 INFO:tasks.workunit.client.0.smithi136.stdout: "size": 3, 2024-09-12T23:37:51.451 INFO:tasks.workunit.client.0.smithi136.stdout: "stats": { 2024-09-12T23:37:51.451 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_raw": 727116294410, 2024-09-12T23:37:51.451 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_used": 1781760, 2024-09-12T23:37:51.451 INFO:tasks.workunit.client.0.smithi136.stdout: "compress_bytes_used": 0, 2024-09-12T23:37:51.451 INFO:tasks.workunit.client.0.smithi136.stdout: "compress_under_bytes": 0, 2024-09-12T23:37:51.451 INFO:tasks.workunit.client.0.smithi136.stdout: "data_bytes_used": 1781760, 2024-09-12T23:37:51.451 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty": 0, 2024-09-12T23:37:51.451 INFO:tasks.workunit.client.0.smithi136.stdout: "max_avail": 242372100096, 2024-09-12T23:37:51.451 INFO:tasks.workunit.client.0.smithi136.stdout: "objects": 2, 2024-09-12T23:37:51.451 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_bytes_used": 0, 2024-09-12T23:37:51.451 INFO:tasks.workunit.client.0.smithi136.stdout: "percent_used": 2.45044111579773e-06, 2024-09-12T23:37:51.451 INFO:tasks.workunit.client.0.smithi136.stdout: "quota_bytes": 0, 2024-09-12T23:37:51.452 INFO:tasks.workunit.client.0.smithi136.stdout: "quota_objects": 0, 2024-09-12T23:37:51.452 INFO:tasks.workunit.client.0.smithi136.stdout: "rd": 416, 2024-09-12T23:37:51.452 INFO:tasks.workunit.client.0.smithi136.stdout: "rd_bytes": 613376, 2024-09-12T23:37:51.452 INFO:tasks.workunit.client.0.smithi136.stdout: "stored": 590368, 2024-09-12T23:37:51.452 INFO:tasks.workunit.client.0.smithi136.stdout: "stored_data": 590368, 2024-09-12T23:37:51.452 INFO:tasks.workunit.client.0.smithi136.stdout: "stored_omap": 0, 2024-09-12T23:37:51.452 INFO:tasks.workunit.client.0.smithi136.stdout: "stored_raw": 1771104, 2024-09-12T23:37:51.452 INFO:tasks.workunit.client.0.smithi136.stdout: "wr": 177, 2024-09-12T23:37:51.452 INFO:tasks.workunit.client.0.smithi136.stdout: "wr_bytes": 1667072 2024-09-12T23:37:51.452 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.452 INFO:tasks.workunit.client.0.smithi136.stdout: "target_max_bytes": 0, 2024-09-12T23:37:51.452 INFO:tasks.workunit.client.0.smithi136.stdout: "target_max_objects": 0, 2024-09-12T23:37:51.452 INFO:tasks.workunit.client.0.smithi136.stdout: "type": "replicated" 2024-09-12T23:37:51.452 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.452 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.453 INFO:tasks.workunit.client.0.smithi136.stdout: "application": [ 2024-09-12T23:37:51.453 INFO:tasks.workunit.client.0.smithi136.stdout: "cephfs" 2024-09-12T23:37:51.453 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.453 INFO:tasks.workunit.client.0.smithi136.stdout: "cache_mode": "none", 2024-09-12T23:37:51.453 INFO:tasks.workunit.client.0.smithi136.stdout: "erasure_code_profile": {}, 2024-09-12T23:37:51.453 INFO:tasks.workunit.client.0.smithi136.stdout: "min_size": 2, 2024-09-12T23:37:51.453 INFO:tasks.workunit.client.0.smithi136.stdout: "options": {}, 2024-09-12T23:37:51.453 INFO:tasks.workunit.client.0.smithi136.stdout: "pg_autoscale_mode": "off", 2024-09-12T23:37:51.453 INFO:tasks.workunit.client.0.smithi136.stdout: "pg_num": 32, 2024-09-12T23:37:51.453 INFO:tasks.workunit.client.0.smithi136.stdout: "pgp_num": 32, 2024-09-12T23:37:51.453 INFO:tasks.workunit.client.0.smithi136.stdout: "pool": 2, 2024-09-12T23:37:51.453 INFO:tasks.workunit.client.0.smithi136.stdout: "size": 3, 2024-09-12T23:37:51.453 INFO:tasks.workunit.client.0.smithi136.stdout: "stats": { 2024-09-12T23:37:51.454 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_raw": 727116294410, 2024-09-12T23:37:51.454 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_used": 98304, 2024-09-12T23:37:51.454 INFO:tasks.workunit.client.0.smithi136.stdout: "compress_bytes_used": 0, 2024-09-12T23:37:51.454 INFO:tasks.workunit.client.0.smithi136.stdout: "compress_under_bytes": 0, 2024-09-12T23:37:51.454 INFO:tasks.workunit.client.0.smithi136.stdout: "data_bytes_used": 98304, 2024-09-12T23:37:51.454 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty": 0, 2024-09-12T23:37:51.454 INFO:tasks.workunit.client.0.smithi136.stdout: "max_avail": 242372100096, 2024-09-12T23:37:51.454 INFO:tasks.workunit.client.0.smithi136.stdout: "objects": 22, 2024-09-12T23:37:51.454 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_bytes_used": 0, 2024-09-12T23:37:51.454 INFO:tasks.workunit.client.0.smithi136.stdout: "percent_used": 1.351970553287174e-07, 2024-09-12T23:37:51.454 INFO:tasks.workunit.client.0.smithi136.stdout: "quota_bytes": 0, 2024-09-12T23:37:51.454 INFO:tasks.workunit.client.0.smithi136.stdout: "quota_objects": 0, 2024-09-12T23:37:51.454 INFO:tasks.workunit.client.0.smithi136.stdout: "rd": 0, 2024-09-12T23:37:51.454 INFO:tasks.workunit.client.0.smithi136.stdout: "rd_bytes": 0, 2024-09-12T23:37:51.455 INFO:tasks.workunit.client.0.smithi136.stdout: "stored": 3410, 2024-09-12T23:37:51.455 INFO:tasks.workunit.client.0.smithi136.stdout: "stored_data": 3410, 2024-09-12T23:37:51.455 INFO:tasks.workunit.client.0.smithi136.stdout: "stored_omap": 0, 2024-09-12T23:37:51.455 INFO:tasks.workunit.client.0.smithi136.stdout: "stored_raw": 10230, 2024-09-12T23:37:51.455 INFO:tasks.workunit.client.0.smithi136.stdout: "wr": 51, 2024-09-12T23:37:51.455 INFO:tasks.workunit.client.0.smithi136.stdout: "wr_bytes": 19456 2024-09-12T23:37:51.455 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.455 INFO:tasks.workunit.client.0.smithi136.stdout: "target_max_bytes": 0, 2024-09-12T23:37:51.455 INFO:tasks.workunit.client.0.smithi136.stdout: "target_max_objects": 0, 2024-09-12T23:37:51.455 INFO:tasks.workunit.client.0.smithi136.stdout: "type": "replicated" 2024-09-12T23:37:51.455 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.455 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.455 INFO:tasks.workunit.client.0.smithi136.stdout: "application": [ 2024-09-12T23:37:51.455 INFO:tasks.workunit.client.0.smithi136.stdout: "cephfs" 2024-09-12T23:37:51.455 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.455 INFO:tasks.workunit.client.0.smithi136.stdout: "cache_mode": "none", 2024-09-12T23:37:51.456 INFO:tasks.workunit.client.0.smithi136.stdout: "erasure_code_profile": {}, 2024-09-12T23:37:51.456 INFO:tasks.workunit.client.0.smithi136.stdout: "min_size": 2, 2024-09-12T23:37:51.456 INFO:tasks.workunit.client.0.smithi136.stdout: "options": {}, 2024-09-12T23:37:51.456 INFO:tasks.workunit.client.0.smithi136.stdout: "pg_autoscale_mode": "off", 2024-09-12T23:37:51.456 INFO:tasks.workunit.client.0.smithi136.stdout: "pg_num": 64, 2024-09-12T23:37:51.456 INFO:tasks.workunit.client.0.smithi136.stdout: "pgp_num": 64, 2024-09-12T23:37:51.456 INFO:tasks.workunit.client.0.smithi136.stdout: "pool": 3, 2024-09-12T23:37:51.456 INFO:tasks.workunit.client.0.smithi136.stdout: "size": 3, 2024-09-12T23:37:51.456 INFO:tasks.workunit.client.0.smithi136.stdout: "stats": { 2024-09-12T23:37:51.456 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_raw": 727116294410, 2024-09-12T23:37:51.456 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_used": 0, 2024-09-12T23:37:51.456 INFO:tasks.workunit.client.0.smithi136.stdout: "compress_bytes_used": 0, 2024-09-12T23:37:51.456 INFO:tasks.workunit.client.0.smithi136.stdout: "compress_under_bytes": 0, 2024-09-12T23:37:51.456 INFO:tasks.workunit.client.0.smithi136.stdout: "data_bytes_used": 0, 2024-09-12T23:37:51.457 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty": 0, 2024-09-12T23:37:51.457 INFO:tasks.workunit.client.0.smithi136.stdout: "max_avail": 242372100096, 2024-09-12T23:37:51.457 INFO:tasks.workunit.client.0.smithi136.stdout: "objects": 0, 2024-09-12T23:37:51.457 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_bytes_used": 0, 2024-09-12T23:37:51.457 INFO:tasks.workunit.client.0.smithi136.stdout: "percent_used": 0.0, 2024-09-12T23:37:51.457 INFO:tasks.workunit.client.0.smithi136.stdout: "quota_bytes": 0, 2024-09-12T23:37:51.457 INFO:tasks.workunit.client.0.smithi136.stdout: "quota_objects": 0, 2024-09-12T23:37:51.457 INFO:tasks.workunit.client.0.smithi136.stdout: "rd": 0, 2024-09-12T23:37:51.457 INFO:tasks.workunit.client.0.smithi136.stdout: "rd_bytes": 0, 2024-09-12T23:37:51.457 INFO:tasks.workunit.client.0.smithi136.stdout: "stored": 0, 2024-09-12T23:37:51.457 INFO:tasks.workunit.client.0.smithi136.stdout: "stored_data": 0, 2024-09-12T23:37:51.457 INFO:tasks.workunit.client.0.smithi136.stdout: "stored_omap": 0, 2024-09-12T23:37:51.457 INFO:tasks.workunit.client.0.smithi136.stdout: "stored_raw": 0, 2024-09-12T23:37:51.458 INFO:tasks.workunit.client.0.smithi136.stdout: "wr": 0, 2024-09-12T23:37:51.458 INFO:tasks.workunit.client.0.smithi136.stdout: "wr_bytes": 0 2024-09-12T23:37:51.458 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.458 INFO:tasks.workunit.client.0.smithi136.stdout: "target_max_bytes": 0, 2024-09-12T23:37:51.458 INFO:tasks.workunit.client.0.smithi136.stdout: "target_max_objects": 0, 2024-09-12T23:37:51.458 INFO:tasks.workunit.client.0.smithi136.stdout: "type": "replicated" 2024-09-12T23:37:51.458 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.458 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.458 INFO:tasks.workunit.client.0.smithi136.stdout: "application": [ 2024-09-12T23:37:51.458 INFO:tasks.workunit.client.0.smithi136.stdout: "rbd" 2024-09-12T23:37:51.458 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.458 INFO:tasks.workunit.client.0.smithi136.stdout: "cache_mode": "none", 2024-09-12T23:37:51.458 INFO:tasks.workunit.client.0.smithi136.stdout: "erasure_code_profile": {}, 2024-09-12T23:37:51.458 INFO:tasks.workunit.client.0.smithi136.stdout: "min_size": 2, 2024-09-12T23:37:51.459 INFO:tasks.workunit.client.0.smithi136.stdout: "options": {}, 2024-09-12T23:37:51.459 INFO:tasks.workunit.client.0.smithi136.stdout: "pg_autoscale_mode": "off", 2024-09-12T23:37:51.459 INFO:tasks.workunit.client.0.smithi136.stdout: "pg_num": 8, 2024-09-12T23:37:51.459 INFO:tasks.workunit.client.0.smithi136.stdout: "pgp_num": 8, 2024-09-12T23:37:51.459 INFO:tasks.workunit.client.0.smithi136.stdout: "pool": 4, 2024-09-12T23:37:51.459 INFO:tasks.workunit.client.0.smithi136.stdout: "size": 3, 2024-09-12T23:37:51.459 INFO:tasks.workunit.client.0.smithi136.stdout: "stats": { 2024-09-12T23:37:51.459 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_raw": 727116294410, 2024-09-12T23:37:51.459 INFO:tasks.workunit.client.0.smithi136.stdout: "bytes_used": 12288, 2024-09-12T23:37:51.459 INFO:tasks.workunit.client.0.smithi136.stdout: "compress_bytes_used": 0, 2024-09-12T23:37:51.459 INFO:tasks.workunit.client.0.smithi136.stdout: "compress_under_bytes": 0, 2024-09-12T23:37:51.459 INFO:tasks.workunit.client.0.smithi136.stdout: "data_bytes_used": 12288, 2024-09-12T23:37:51.459 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty": 0, 2024-09-12T23:37:51.459 INFO:tasks.workunit.client.0.smithi136.stdout: "max_avail": 242372100096, 2024-09-12T23:37:51.459 INFO:tasks.workunit.client.0.smithi136.stdout: "objects": 2, 2024-09-12T23:37:51.460 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_bytes_used": 0, 2024-09-12T23:37:51.460 INFO:tasks.workunit.client.0.smithi136.stdout: "percent_used": 1.6899635468803353e-08, 2024-09-12T23:37:51.460 INFO:tasks.workunit.client.0.smithi136.stdout: "quota_bytes": 0, 2024-09-12T23:37:51.460 INFO:tasks.workunit.client.0.smithi136.stdout: "quota_objects": 0, 2024-09-12T23:37:51.460 INFO:tasks.workunit.client.0.smithi136.stdout: "rd": 0, 2024-09-12T23:37:51.460 INFO:tasks.workunit.client.0.smithi136.stdout: "rd_bytes": 0, 2024-09-12T23:37:51.460 INFO:tasks.workunit.client.0.smithi136.stdout: "stored": 19, 2024-09-12T23:37:51.460 INFO:tasks.workunit.client.0.smithi136.stdout: "stored_data": 19, 2024-09-12T23:37:51.460 INFO:tasks.workunit.client.0.smithi136.stdout: "stored_omap": 0, 2024-09-12T23:37:51.460 INFO:tasks.workunit.client.0.smithi136.stdout: "stored_raw": 57, 2024-09-12T23:37:51.460 INFO:tasks.workunit.client.0.smithi136.stdout: "wr": 2, 2024-09-12T23:37:51.460 INFO:tasks.workunit.client.0.smithi136.stdout: "wr_bytes": 2048 2024-09-12T23:37:51.460 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.460 INFO:tasks.workunit.client.0.smithi136.stdout: "target_max_bytes": 0, 2024-09-12T23:37:51.460 INFO:tasks.workunit.client.0.smithi136.stdout: "target_max_objects": 0, 2024-09-12T23:37:51.461 INFO:tasks.workunit.client.0.smithi136.stdout: "type": "replicated" 2024-09-12T23:37:51.461 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.461 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.461 INFO:tasks.workunit.client.0.smithi136.stdout: "rbd": { 2024-09-12T23:37:51.461 INFO:tasks.workunit.client.0.smithi136.stdout: "mirroring_by_pool": [ 2024-09-12T23:37:51.461 INFO:tasks.workunit.client.0.smithi136.stdout: false 2024-09-12T23:37:51.461 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.461 INFO:tasks.workunit.client.0.smithi136.stdout: "num_images_by_pool": [ 2024-09-12T23:37:51.461 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:51.461 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.461 INFO:tasks.workunit.client.0.smithi136.stdout: "num_pools": 1 2024-09-12T23:37:51.461 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.461 INFO:tasks.workunit.client.0.smithi136.stdout: "report_id": "679f567b-70cd-4d20-9c08-b6621403ecae", 2024-09-12T23:37:51.461 INFO:tasks.workunit.client.0.smithi136.stdout: "report_timestamp": "2024-09-12T23:37:50.016135", 2024-09-12T23:37:51.461 INFO:tasks.workunit.client.0.smithi136.stdout: "report_version": 1, 2024-09-12T23:37:51.462 INFO:tasks.workunit.client.0.smithi136.stdout: "rocksdb_stats": { 2024-09-12T23:37:51.462 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "6.15.5" 2024-09-12T23:37:51.462 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.462 INFO:tasks.workunit.client.0.smithi136.stdout: "rook": { 2024-09-12T23:37:51.462 INFO:tasks.workunit.client.0.smithi136.stdout: "cluster": { 2024-09-12T23:37:51.462 INFO:tasks.workunit.client.0.smithi136.stdout: "external-mode": null, 2024-09-12T23:37:51.462 INFO:tasks.workunit.client.0.smithi136.stdout: "mon": { 2024-09-12T23:37:51.462 INFO:tasks.workunit.client.0.smithi136.stdout: "allow-multiple-per-node": null, 2024-09-12T23:37:51.462 INFO:tasks.workunit.client.0.smithi136.stdout: "count": null, 2024-09-12T23:37:51.462 INFO:tasks.workunit.client.0.smithi136.stdout: "max-id": null, 2024-09-12T23:37:51.462 INFO:tasks.workunit.client.0.smithi136.stdout: "pvc": { 2024-09-12T23:37:51.462 INFO:tasks.workunit.client.0.smithi136.stdout: "enabled": null 2024-09-12T23:37:51.462 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.462 INFO:tasks.workunit.client.0.smithi136.stdout: "stretch": { 2024-09-12T23:37:51.463 INFO:tasks.workunit.client.0.smithi136.stdout: "enabled": null 2024-09-12T23:37:51.463 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.463 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.463 INFO:tasks.workunit.client.0.smithi136.stdout: "network": { 2024-09-12T23:37:51.463 INFO:tasks.workunit.client.0.smithi136.stdout: "provider": null 2024-09-12T23:37:51.463 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.463 INFO:tasks.workunit.client.0.smithi136.stdout: "storage": { 2024-09-12T23:37:51.463 INFO:tasks.workunit.client.0.smithi136.stdout: "device-set": { 2024-09-12T23:37:51.463 INFO:tasks.workunit.client.0.smithi136.stdout: "count": { 2024-09-12T23:37:51.463 INFO:tasks.workunit.client.0.smithi136.stdout: "non-portable": null, 2024-09-12T23:37:51.463 INFO:tasks.workunit.client.0.smithi136.stdout: "portable": null, 2024-09-12T23:37:51.463 INFO:tasks.workunit.client.0.smithi136.stdout: "total": null 2024-09-12T23:37:51.463 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.463 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.463 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.463 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.464 INFO:tasks.workunit.client.0.smithi136.stdout: "csi": { 2024-09-12T23:37:51.464 INFO:tasks.workunit.client.0.smithi136.stdout: "version": null 2024-09-12T23:37:51.464 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.464 INFO:tasks.workunit.client.0.smithi136.stdout: "kubernetes": { 2024-09-12T23:37:51.464 INFO:tasks.workunit.client.0.smithi136.stdout: "version": null 2024-09-12T23:37:51.464 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.464 INFO:tasks.workunit.client.0.smithi136.stdout: "node": { 2024-09-12T23:37:51.464 INFO:tasks.workunit.client.0.smithi136.stdout: "count": { 2024-09-12T23:37:51.464 INFO:tasks.workunit.client.0.smithi136.stdout: "kubernetes-total": null, 2024-09-12T23:37:51.464 INFO:tasks.workunit.client.0.smithi136.stdout: "with-ceph-daemons": null, 2024-09-12T23:37:51.464 INFO:tasks.workunit.client.0.smithi136.stdout: "with-csi-cephfs-plugin": null, 2024-09-12T23:37:51.464 INFO:tasks.workunit.client.0.smithi136.stdout: "with-csi-nfs-plugin": null, 2024-09-12T23:37:51.464 INFO:tasks.workunit.client.0.smithi136.stdout: "with-csi-rbd-plugin": null 2024-09-12T23:37:51.464 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.464 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.465 INFO:tasks.workunit.client.0.smithi136.stdout: "usage": { 2024-09-12T23:37:51.465 INFO:tasks.workunit.client.0.smithi136.stdout: "storage-class": { 2024-09-12T23:37:51.465 INFO:tasks.workunit.client.0.smithi136.stdout: "count": { 2024-09-12T23:37:51.465 INFO:tasks.workunit.client.0.smithi136.stdout: "bucket": null, 2024-09-12T23:37:51.465 INFO:tasks.workunit.client.0.smithi136.stdout: "cephfs": null, 2024-09-12T23:37:51.465 INFO:tasks.workunit.client.0.smithi136.stdout: "nfs": null, 2024-09-12T23:37:51.465 INFO:tasks.workunit.client.0.smithi136.stdout: "rbd": null, 2024-09-12T23:37:51.465 INFO:tasks.workunit.client.0.smithi136.stdout: "total": null 2024-09-12T23:37:51.465 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.465 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:51.465 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.465 INFO:tasks.workunit.client.0.smithi136.stdout: "version": null 2024-09-12T23:37:51.465 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.465 INFO:tasks.workunit.client.0.smithi136.stdout: "services": {}, 2024-09-12T23:37:51.465 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_per_pg": [ 2024-09-12T23:37:51.466 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.466 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.466 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:51.466 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:51.466 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:51.466 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.466 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 7, 2024-09-12T23:37:51.466 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.466 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.466 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:51.466 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.466 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.466 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.466 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.665027+0000", 2024-09-12T23:37:51.467 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.908161+0000", 2024-09-12T23:37:51.467 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.908161+0000", 2024-09-12T23:37:51.467 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.908602+0000", 2024-09-12T23:37:51.467 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.665027+0000", 2024-09-12T23:37:51.467 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.467 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.467 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.467 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:51.467 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.665027+0000", 2024-09-12T23:37:51.467 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.665027+0000", 2024-09-12T23:37:51.467 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.665027+0000", 2024-09-12T23:37:51.467 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.467 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.467 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.467 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.665027+0000", 2024-09-12T23:37:51.467 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.665027+0000", 2024-09-12T23:37:51.468 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.468 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.468 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.468 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.468 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:51.468 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.468 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.468 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.468 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.468 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.468 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.468 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.468 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.468 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.3f", 2024-09-12T23:37:51.469 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.469 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.469 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.469 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:51.469 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.469 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T02:18:25.662166+0000", 2024-09-12T23:37:51.469 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.469 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.469 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.469 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.469 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.469 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.469 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.469 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.470 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.470 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.470 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.470 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.470 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.470 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.470 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.470 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.470 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.470 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.470 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.470 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.470 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.470 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.470 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.471 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.471 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.471 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.471 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.471 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.471 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.471 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.471 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.471 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.471 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.471 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.471 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.471 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.471 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.472 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.472 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.472 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.472 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.472 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.472 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.472 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.472 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.472 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.472 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.472 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:51.472 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:51.472 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:51.472 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.472 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 7, 2024-09-12T23:37:51.473 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.473 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.473 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.473 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.473 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:51.473 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:51.473 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:51.473 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.473 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 1, 2024-09-12T23:37:51.473 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.473 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.473 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:51.473 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.473 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.473 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.474 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.664503+0000", 2024-09-12T23:37:51.474 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.906765+0000", 2024-09-12T23:37:51.474 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.906765+0000", 2024-09-12T23:37:51.474 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.907919+0000", 2024-09-12T23:37:51.474 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.664503+0000", 2024-09-12T23:37:51.474 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.474 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.474 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.474 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:51.474 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.664503+0000", 2024-09-12T23:37:51.474 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.664503+0000", 2024-09-12T23:37:51.474 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.664503+0000", 2024-09-12T23:37:51.474 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.474 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.474 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.474 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.664503+0000", 2024-09-12T23:37:51.475 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.664503+0000", 2024-09-12T23:37:51.475 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.475 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.475 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.475 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.475 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:51.475 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.475 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.475 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.475 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.475 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.475 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.475 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.475 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.476 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.3e", 2024-09-12T23:37:51.476 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.476 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.476 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.476 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:51.476 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.476 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T01:39:49.787701+0000", 2024-09-12T23:37:51.476 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.476 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.476 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.476 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.477 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.477 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.477 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.477 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.477 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.477 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.477 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.477 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.477 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.477 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.477 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.477 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.477 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.477 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.478 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.478 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.478 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.478 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.478 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.478 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.478 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.478 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.478 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.478 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.478 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.478 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.478 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.478 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.479 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.479 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.479 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.479 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.479 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.479 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.479 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.479 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.479 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.479 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.479 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.479 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.479 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.479 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.479 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.480 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:51.480 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:51.480 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:51.480 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.480 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 1, 2024-09-12T23:37:51.480 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.480 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.480 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.480 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.480 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:51.480 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:51.480 INFO:tasks.workunit.client.0.smithi136.stdout: 1 2024-09-12T23:37:51.480 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.480 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 0, 2024-09-12T23:37:51.481 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.481 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.481 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:51.481 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.481 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.481 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.481 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.671331+0000", 2024-09-12T23:37:51.481 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.924268+0000", 2024-09-12T23:37:51.481 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.924268+0000", 2024-09-12T23:37:51.481 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.924626+0000", 2024-09-12T23:37:51.481 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.671331+0000", 2024-09-12T23:37:51.481 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.481 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.482 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.482 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:51.482 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.671331+0000", 2024-09-12T23:37:51.482 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.671331+0000", 2024-09-12T23:37:51.482 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.671331+0000", 2024-09-12T23:37:51.482 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.482 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.482 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.482 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.671331+0000", 2024-09-12T23:37:51.482 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.671331+0000", 2024-09-12T23:37:51.482 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.482 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.482 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.482 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.482 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:51.483 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.483 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.483 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.483 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.483 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.483 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.483 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.483 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.483 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.3d", 2024-09-12T23:37:51.483 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.483 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.483 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.483 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:51.483 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.484 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T00:41:31.853826+0000", 2024-09-12T23:37:51.484 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.484 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.484 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.484 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.484 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.484 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.484 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.484 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.484 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.484 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.484 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.484 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.485 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.485 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.485 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.485 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.485 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.485 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.485 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.485 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.485 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.485 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.485 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.485 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.485 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.485 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.486 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.486 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.486 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.486 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.486 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.486 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.486 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.486 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.486 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.486 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.486 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.486 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.486 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.487 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.487 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.487 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.487 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.487 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.487 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.487 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.487 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.487 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:51.487 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:51.487 INFO:tasks.workunit.client.0.smithi136.stdout: 1 2024-09-12T23:37:51.487 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.487 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 0, 2024-09-12T23:37:51.487 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.487 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.488 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.488 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.488 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:51.488 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:51.488 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:51.488 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.488 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 5, 2024-09-12T23:37:51.488 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.488 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.488 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:51.488 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.488 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.489 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.489 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.670520+0000", 2024-09-12T23:37:51.489 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.913672+0000", 2024-09-12T23:37:51.489 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.913672+0000", 2024-09-12T23:37:51.489 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.913990+0000", 2024-09-12T23:37:51.489 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.670520+0000", 2024-09-12T23:37:51.489 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.489 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.489 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.489 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:51.489 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.670520+0000", 2024-09-12T23:37:51.489 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.670520+0000", 2024-09-12T23:37:51.489 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.670520+0000", 2024-09-12T23:37:51.489 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.489 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.490 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.490 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.670520+0000", 2024-09-12T23:37:51.490 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.670520+0000", 2024-09-12T23:37:51.490 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.490 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.490 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.490 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.490 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:51.490 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.490 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.490 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.490 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.490 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.490 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.490 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.491 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.491 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.3c", 2024-09-12T23:37:51.491 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.491 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.491 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.491 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:51.491 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.491 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T03:57:54.241724+0000", 2024-09-12T23:37:51.491 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.491 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.491 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.491 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.491 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.491 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.491 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.492 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.492 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.492 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.492 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.492 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.492 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.492 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.492 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.492 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.492 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.492 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.492 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.492 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.492 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.492 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.493 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.493 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.493 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.493 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.493 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.493 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.493 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.493 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.493 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.493 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.493 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.494 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.494 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.494 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.494 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.494 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.494 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.494 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.494 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.494 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.494 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.494 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.494 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.494 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.494 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.495 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:51.495 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:51.495 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:51.495 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.495 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 5, 2024-09-12T23:37:51.495 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.495 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.495 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.495 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.495 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:51.495 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:51.495 INFO:tasks.workunit.client.0.smithi136.stdout: 4 2024-09-12T23:37:51.495 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.495 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 7, 2024-09-12T23:37:51.496 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.496 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.496 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:51.496 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.496 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.496 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.496 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.663791+0000", 2024-09-12T23:37:51.496 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.904192+0000", 2024-09-12T23:37:51.496 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.904192+0000", 2024-09-12T23:37:51.496 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.904505+0000", 2024-09-12T23:37:51.496 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.663791+0000", 2024-09-12T23:37:51.496 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.496 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.497 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.497 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:51.497 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.663791+0000", 2024-09-12T23:37:51.497 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.663791+0000", 2024-09-12T23:37:51.497 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.663791+0000", 2024-09-12T23:37:51.497 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.497 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.497 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.497 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.663791+0000", 2024-09-12T23:37:51.497 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.663791+0000", 2024-09-12T23:37:51.497 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.497 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.497 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.498 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.498 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:51.498 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.498 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.498 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.498 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.498 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.498 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.498 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.498 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.498 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.3b", 2024-09-12T23:37:51.498 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.498 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.498 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.498 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:51.499 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.499 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T06:49:27.086584+0000", 2024-09-12T23:37:51.499 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.499 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.499 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.499 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.499 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.499 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.499 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.499 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.499 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.499 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.499 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.499 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.499 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.500 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.500 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.500 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.500 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.500 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.500 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.500 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.500 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.500 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.500 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.500 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.500 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.500 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.500 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.501 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.501 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.501 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.501 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.501 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.501 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.501 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.501 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.501 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.501 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.501 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.501 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.501 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.501 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.502 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.502 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.502 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.502 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.502 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.502 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.502 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:51.502 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:51.502 INFO:tasks.workunit.client.0.smithi136.stdout: 4 2024-09-12T23:37:51.502 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.502 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 7, 2024-09-12T23:37:51.502 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.502 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.502 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.502 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.503 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:51.503 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:51.503 INFO:tasks.workunit.client.0.smithi136.stdout: 1 2024-09-12T23:37:51.503 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.503 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 5, 2024-09-12T23:37:51.503 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.503 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.503 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:51.503 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.503 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.503 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.503 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.671038+0000", 2024-09-12T23:37:51.503 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.905154+0000", 2024-09-12T23:37:51.504 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.905154+0000", 2024-09-12T23:37:51.504 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.905467+0000", 2024-09-12T23:37:51.504 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.671038+0000", 2024-09-12T23:37:51.504 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.504 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.504 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.504 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:51.504 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.671038+0000", 2024-09-12T23:37:51.504 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.671038+0000", 2024-09-12T23:37:51.504 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.671038+0000", 2024-09-12T23:37:51.504 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.504 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.504 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.504 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.671038+0000", 2024-09-12T23:37:51.505 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.671038+0000", 2024-09-12T23:37:51.505 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.505 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.505 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.505 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.505 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:51.505 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.505 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.505 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.505 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.505 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.505 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.505 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.505 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.506 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.3a", 2024-09-12T23:37:51.506 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.506 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.506 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.506 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:51.506 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.506 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T01:49:00.596502+0000", 2024-09-12T23:37:51.506 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.506 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.506 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.506 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.506 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.506 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.507 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.507 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.507 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.507 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.507 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.507 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.507 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.507 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.507 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.507 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.507 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.507 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.507 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.507 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.508 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.508 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.508 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.508 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.508 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.508 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.508 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.508 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.508 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.508 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.508 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.508 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.508 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.509 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.509 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.509 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.509 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.509 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.509 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.509 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.509 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.509 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.509 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.509 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.509 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.509 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.510 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.510 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:51.510 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:51.510 INFO:tasks.workunit.client.0.smithi136.stdout: 1 2024-09-12T23:37:51.510 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.510 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 5, 2024-09-12T23:37:51.510 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.510 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.510 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.510 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.510 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:51.510 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:51.510 INFO:tasks.workunit.client.0.smithi136.stdout: 4 2024-09-12T23:37:51.510 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.510 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 7, 2024-09-12T23:37:51.510 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.511 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.511 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:51.511 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.511 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.511 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.511 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.664458+0000", 2024-09-12T23:37:51.511 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.908427+0000", 2024-09-12T23:37:51.511 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.908427+0000", 2024-09-12T23:37:51.511 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.908762+0000", 2024-09-12T23:37:51.511 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.664458+0000", 2024-09-12T23:37:51.511 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.511 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.511 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.512 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:51.512 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.664458+0000", 2024-09-12T23:37:51.512 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.664458+0000", 2024-09-12T23:37:51.512 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.664458+0000", 2024-09-12T23:37:51.512 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.512 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.512 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.512 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.664458+0000", 2024-09-12T23:37:51.512 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.664458+0000", 2024-09-12T23:37:51.512 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.512 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.512 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.512 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.512 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:51.512 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.513 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.513 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.513 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.513 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.513 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.513 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.513 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.513 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.39", 2024-09-12T23:37:51.513 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.513 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.513 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.513 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:51.513 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.513 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T10:44:26.357691+0000", 2024-09-12T23:37:51.514 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.514 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.514 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.514 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.514 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.514 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.514 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.514 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.514 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.514 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.514 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.514 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.514 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.514 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.515 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.515 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.515 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.515 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.515 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.515 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.515 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.515 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.515 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.515 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.515 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.515 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.515 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.515 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.515 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.516 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.516 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.516 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.516 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.516 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.516 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.516 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.516 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.516 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.516 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.516 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.516 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.516 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.516 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.516 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.517 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.517 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.517 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.517 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:51.517 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:51.517 INFO:tasks.workunit.client.0.smithi136.stdout: 4 2024-09-12T23:37:51.517 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.517 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 7, 2024-09-12T23:37:51.517 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.517 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.517 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.517 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.517 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:51.517 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:51.518 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:51.518 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.518 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 3, 2024-09-12T23:37:51.518 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.518 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.518 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:51.518 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.518 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.518 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.518 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.666998+0000", 2024-09-12T23:37:51.518 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.901611+0000", 2024-09-12T23:37:51.518 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.901611+0000", 2024-09-12T23:37:51.518 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.902033+0000", 2024-09-12T23:37:51.518 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.666998+0000", 2024-09-12T23:37:51.519 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.519 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.519 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.519 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:51.519 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.666998+0000", 2024-09-12T23:37:51.519 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.666998+0000", 2024-09-12T23:37:51.519 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.666998+0000", 2024-09-12T23:37:51.519 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.519 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.519 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.519 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.666998+0000", 2024-09-12T23:37:51.519 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.666998+0000", 2024-09-12T23:37:51.519 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.519 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.519 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.520 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.520 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:51.520 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.520 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.520 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.520 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.520 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.520 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.520 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.520 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.520 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.38", 2024-09-12T23:37:51.520 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.520 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.521 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.521 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:51.521 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.521 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T06:57:58.671558+0000", 2024-09-12T23:37:51.521 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.521 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.521 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.521 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.521 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.521 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.521 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.521 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.521 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.521 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.522 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.522 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.522 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.522 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.522 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.522 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.522 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.522 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.522 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.522 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.522 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.522 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.522 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.522 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.523 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.523 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.523 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.523 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.523 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.523 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.523 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.523 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.523 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.523 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.523 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.523 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.523 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.523 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.524 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.524 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.524 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.524 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.524 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.524 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.524 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.524 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.524 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.524 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:51.524 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:51.524 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:51.524 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.524 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 3, 2024-09-12T23:37:51.525 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.525 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.525 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.525 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.525 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:51.525 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:51.525 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:51.525 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.525 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 7, 2024-09-12T23:37:51.525 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.525 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.525 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:51.525 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.525 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.526 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.526 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:09.803676+0000", 2024-09-12T23:37:51.526 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:35:09.802434+0000", 2024-09-12T23:37:51.526 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:35:09.802434+0000", 2024-09-12T23:37:51.526 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:35:09.803676+0000", 2024-09-12T23:37:51.526 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:09.803676+0000", 2024-09-12T23:37:51.526 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.526 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.526 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.526 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 64, 2024-09-12T23:37:51.526 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:09.803676+0000", 2024-09-12T23:37:51.526 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:09.803676+0000", 2024-09-12T23:37:51.526 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:09.803676+0000", 2024-09-12T23:37:51.527 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.527 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.527 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.527 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:09.803676+0000", 2024-09-12T23:37:51.527 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:09.803676+0000", 2024-09-12T23:37:51.527 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.527 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.527 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.527 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.527 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 63, 2024-09-12T23:37:51.527 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.527 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.527 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.527 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.528 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.528 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.528 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.528 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.528 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.37", 2024-09-12T23:37:51.528 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.528 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.528 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 64, 2024-09-12T23:37:51.528 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 30, 2024-09-12T23:37:51.528 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.528 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T04:01:37.156347+0000", 2024-09-12T23:37:51.528 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.528 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.528 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.528 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.529 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.529 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.529 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.529 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.529 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.529 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.529 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.529 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.529 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.529 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.529 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.529 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.529 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.529 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.530 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.530 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.530 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.530 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.530 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.530 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.530 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.530 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.530 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.530 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.530 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.530 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.530 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.530 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.531 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.531 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.531 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.531 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.531 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.531 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.531 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.531 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.531 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.531 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.531 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.531 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.531 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.531 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.532 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.532 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:51.532 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:51.532 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:51.532 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.532 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 7, 2024-09-12T23:37:51.532 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.532 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.532 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.532 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.532 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:51.532 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:51.532 INFO:tasks.workunit.client.0.smithi136.stdout: 1 2024-09-12T23:37:51.532 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.532 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 6, 2024-09-12T23:37:51.533 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.533 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.533 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:51.533 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.533 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.533 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.533 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.666805+0000", 2024-09-12T23:37:51.533 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.892815+0000", 2024-09-12T23:37:51.533 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.892815+0000", 2024-09-12T23:37:51.533 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.893241+0000", 2024-09-12T23:37:51.533 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.666805+0000", 2024-09-12T23:37:51.533 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.533 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.533 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.533 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:51.534 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.666805+0000", 2024-09-12T23:37:51.534 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.666805+0000", 2024-09-12T23:37:51.534 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.666805+0000", 2024-09-12T23:37:51.534 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.534 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.534 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.534 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.666805+0000", 2024-09-12T23:37:51.534 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.666805+0000", 2024-09-12T23:37:51.534 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.534 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.534 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.534 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.534 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:51.534 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.535 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.535 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.535 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.535 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.535 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.535 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.535 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.535 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.36", 2024-09-12T23:37:51.535 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.535 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.535 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.535 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:51.535 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.535 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T01:46:21.539771+0000", 2024-09-12T23:37:51.536 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.536 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.536 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.536 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.536 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.536 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.536 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.536 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.536 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.536 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.536 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.536 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.536 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.536 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.537 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.537 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.537 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.537 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.537 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.537 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.537 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.537 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.537 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.537 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.537 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.537 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.537 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.537 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.538 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.538 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.538 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.538 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.538 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.538 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.538 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.538 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.538 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.538 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.538 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.538 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.538 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.539 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.539 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.539 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.539 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.539 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.539 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.539 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:51.539 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:51.539 INFO:tasks.workunit.client.0.smithi136.stdout: 1 2024-09-12T23:37:51.539 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.539 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 6, 2024-09-12T23:37:51.539 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.540 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.540 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.540 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.540 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:51.540 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:51.540 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:51.540 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.540 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 6, 2024-09-12T23:37:51.540 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.540 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.540 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:51.540 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.540 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.540 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.541 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.665734+0000", 2024-09-12T23:37:51.541 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.906493+0000", 2024-09-12T23:37:51.541 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.906493+0000", 2024-09-12T23:37:51.541 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.906810+0000", 2024-09-12T23:37:51.541 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.665734+0000", 2024-09-12T23:37:51.541 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.541 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.541 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.541 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:51.541 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.665734+0000", 2024-09-12T23:37:51.541 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.665734+0000", 2024-09-12T23:37:51.541 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.665734+0000", 2024-09-12T23:37:51.541 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.542 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.542 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.542 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.665734+0000", 2024-09-12T23:37:51.542 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.665734+0000", 2024-09-12T23:37:51.542 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.542 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.542 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.542 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.542 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:51.542 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.542 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.542 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.542 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.542 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.542 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.543 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.543 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.543 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.35", 2024-09-12T23:37:51.543 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.543 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.543 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.543 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:51.543 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.543 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T05:56:07.145258+0000", 2024-09-12T23:37:51.543 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.543 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.543 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.543 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.543 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.543 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.544 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.544 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.544 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.544 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.544 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.544 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.544 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.544 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.544 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.544 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.544 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.544 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.544 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.544 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.545 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.545 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.545 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.545 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.545 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.545 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.545 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.545 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.545 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.545 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.545 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.545 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.545 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.545 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.546 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.546 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.546 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.546 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.546 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.546 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.546 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.546 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.546 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.546 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.546 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.546 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.546 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.546 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:51.547 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:51.547 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:51.547 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.547 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 6, 2024-09-12T23:37:51.547 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.547 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.547 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.547 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.547 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:51.547 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:51.547 INFO:tasks.workunit.client.0.smithi136.stdout: 3 2024-09-12T23:37:51.547 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.547 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 4, 2024-09-12T23:37:51.547 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.547 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.548 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:51.548 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.548 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.548 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.548 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:09.802722+0000", 2024-09-12T23:37:51.548 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:35:09.801763+0000", 2024-09-12T23:37:51.548 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:35:09.801763+0000", 2024-09-12T23:37:51.548 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:35:09.802722+0000", 2024-09-12T23:37:51.548 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:09.802722+0000", 2024-09-12T23:37:51.548 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.548 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.548 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.548 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 64, 2024-09-12T23:37:51.549 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:09.802722+0000", 2024-09-12T23:37:51.549 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:09.802722+0000", 2024-09-12T23:37:51.549 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:09.802722+0000", 2024-09-12T23:37:51.549 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.549 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.549 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.549 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:09.802722+0000", 2024-09-12T23:37:51.549 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:09.802722+0000", 2024-09-12T23:37:51.549 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.549 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.549 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.549 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.549 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 63, 2024-09-12T23:37:51.549 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.550 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.552 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.552 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.552 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.553 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.553 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.553 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.553 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.34", 2024-09-12T23:37:51.553 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.553 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.553 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 64, 2024-09-12T23:37:51.553 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 30, 2024-09-12T23:37:51.553 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.553 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T10:33:24.574457+0000", 2024-09-12T23:37:51.553 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.553 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.553 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.554 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.554 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.554 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.554 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.554 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.554 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.554 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.554 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.554 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.554 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.554 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.554 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.554 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.555 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.555 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.555 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.555 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.555 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.555 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.555 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.555 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.555 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.555 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.555 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.555 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.555 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.555 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.556 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.556 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.556 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.556 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.556 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.556 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.556 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.556 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.556 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.556 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.556 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.556 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.556 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.556 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.557 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.557 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.557 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.557 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:51.557 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:51.557 INFO:tasks.workunit.client.0.smithi136.stdout: 3 2024-09-12T23:37:51.557 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.557 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 4, 2024-09-12T23:37:51.557 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.557 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.557 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.557 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.557 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:51.557 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:51.558 INFO:tasks.workunit.client.0.smithi136.stdout: 1 2024-09-12T23:37:51.558 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.558 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 3, 2024-09-12T23:37:51.558 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.558 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.558 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:51.558 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.558 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.558 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.558 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.668470+0000", 2024-09-12T23:37:51.558 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.906577+0000", 2024-09-12T23:37:51.558 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.906577+0000", 2024-09-12T23:37:51.558 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.907459+0000", 2024-09-12T23:37:51.558 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.668470+0000", 2024-09-12T23:37:51.559 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.559 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.559 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.559 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:51.559 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.668470+0000", 2024-09-12T23:37:51.559 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.668470+0000", 2024-09-12T23:37:51.559 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.668470+0000", 2024-09-12T23:37:51.559 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.559 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.559 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.559 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.668470+0000", 2024-09-12T23:37:51.559 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.668470+0000", 2024-09-12T23:37:51.559 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.560 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.560 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.560 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.560 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:51.560 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.560 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.560 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.560 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.560 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.560 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.560 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.560 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.560 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.33", 2024-09-12T23:37:51.560 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.560 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.561 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.561 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:51.561 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.561 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T07:21:41.131616+0000", 2024-09-12T23:37:51.561 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.561 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.561 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.561 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.561 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.561 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.561 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.561 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.561 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.561 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.562 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.562 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.562 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.562 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.562 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.562 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.562 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.562 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.562 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.562 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.562 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.562 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.562 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.562 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.563 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.563 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.563 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.563 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.563 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.563 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.563 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.563 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.563 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.563 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.563 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.563 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.563 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.563 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.564 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.564 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.564 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.564 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.564 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.564 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.564 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.564 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.564 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.564 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:51.564 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:51.564 INFO:tasks.workunit.client.0.smithi136.stdout: 1 2024-09-12T23:37:51.564 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.565 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 3, 2024-09-12T23:37:51.565 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.565 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.565 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.565 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.565 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:51.565 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:51.565 INFO:tasks.workunit.client.0.smithi136.stdout: 5 2024-09-12T23:37:51.565 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.565 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 0, 2024-09-12T23:37:51.565 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.565 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.565 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:51.565 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.566 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.566 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.566 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.670257+0000", 2024-09-12T23:37:51.566 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.918051+0000", 2024-09-12T23:37:51.566 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.918051+0000", 2024-09-12T23:37:51.566 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.920721+0000", 2024-09-12T23:37:51.566 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.670257+0000", 2024-09-12T23:37:51.566 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.566 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.566 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.566 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:51.566 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.670257+0000", 2024-09-12T23:37:51.566 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.670257+0000", 2024-09-12T23:37:51.567 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.670257+0000", 2024-09-12T23:37:51.567 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.567 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.567 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.567 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.670257+0000", 2024-09-12T23:37:51.567 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.670257+0000", 2024-09-12T23:37:51.567 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.567 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.567 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.567 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.567 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:51.567 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.567 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.567 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.567 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.568 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.568 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.568 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.568 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.568 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.32", 2024-09-12T23:37:51.568 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.568 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.568 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.568 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:51.568 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.568 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T10:32:27.077464+0000", 2024-09-12T23:37:51.568 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.568 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.568 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.569 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.569 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.569 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.569 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.569 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.569 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.569 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.569 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.569 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.569 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.569 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.569 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.569 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.569 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.570 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.570 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.570 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.570 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.570 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.570 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.570 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.570 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.570 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.570 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.570 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.570 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.570 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.570 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.571 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.571 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.571 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.571 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.571 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.571 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.571 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.571 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.571 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.571 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.571 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.571 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.571 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.571 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.572 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.572 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.572 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:51.572 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:51.572 INFO:tasks.workunit.client.0.smithi136.stdout: 5 2024-09-12T23:37:51.572 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.572 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 0, 2024-09-12T23:37:51.572 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.572 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.572 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.572 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.572 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:51.572 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:51.573 INFO:tasks.workunit.client.0.smithi136.stdout: 5 2024-09-12T23:37:51.573 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.573 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 1, 2024-09-12T23:37:51.573 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.573 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.573 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:51.573 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.573 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.573 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.573 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:09.683561+0000", 2024-09-12T23:37:51.573 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:35:09.682671+0000", 2024-09-12T23:37:51.573 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:35:09.682671+0000", 2024-09-12T23:37:51.573 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:35:09.683561+0000", 2024-09-12T23:37:51.573 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:09.683561+0000", 2024-09-12T23:37:51.574 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.574 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.574 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.574 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 64, 2024-09-12T23:37:51.574 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:09.683561+0000", 2024-09-12T23:37:51.574 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:09.683561+0000", 2024-09-12T23:37:51.574 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:09.683561+0000", 2024-09-12T23:37:51.574 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.574 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.574 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.574 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:09.683561+0000", 2024-09-12T23:37:51.574 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:09.683561+0000", 2024-09-12T23:37:51.574 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.574 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.574 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.574 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.575 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 63, 2024-09-12T23:37:51.575 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.575 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.575 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.575 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.575 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.575 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.575 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.575 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.575 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.31", 2024-09-12T23:37:51.575 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.575 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.575 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 64, 2024-09-12T23:37:51.575 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 30, 2024-09-12T23:37:51.575 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.576 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T05:45:39.393684+0000", 2024-09-12T23:37:51.576 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.576 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.576 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.576 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.576 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.576 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.576 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.576 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.576 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.576 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.576 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.576 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.576 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.577 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.577 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.577 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.577 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.577 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.577 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.577 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.577 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.577 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.577 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.577 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.577 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.577 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.577 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.577 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.578 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.578 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.578 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.578 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.578 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.578 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.578 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.578 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.578 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.578 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.578 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.578 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.578 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.578 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.579 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.579 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.579 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.579 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.579 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.579 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:51.579 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:51.579 INFO:tasks.workunit.client.0.smithi136.stdout: 5 2024-09-12T23:37:51.579 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.579 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 1, 2024-09-12T23:37:51.579 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.579 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.579 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.579 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.579 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:51.579 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:51.580 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:51.580 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.580 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 2, 2024-09-12T23:37:51.580 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.580 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.580 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:51.580 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.580 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.580 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.580 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:09.804288+0000", 2024-09-12T23:37:51.580 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:35:09.803450+0000", 2024-09-12T23:37:51.580 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:35:09.803450+0000", 2024-09-12T23:37:51.580 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:35:09.804288+0000", 2024-09-12T23:37:51.580 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:09.804288+0000", 2024-09-12T23:37:51.580 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.581 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.581 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.581 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 64, 2024-09-12T23:37:51.581 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:09.804288+0000", 2024-09-12T23:37:51.581 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:09.804288+0000", 2024-09-12T23:37:51.581 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:09.804288+0000", 2024-09-12T23:37:51.581 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.581 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.581 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.581 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:09.804288+0000", 2024-09-12T23:37:51.581 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:09.804288+0000", 2024-09-12T23:37:51.581 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.581 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.581 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.582 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.582 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 63, 2024-09-12T23:37:51.582 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.582 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.582 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.582 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.582 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.582 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.582 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.582 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.582 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.30", 2024-09-12T23:37:51.582 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.582 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.582 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 64, 2024-09-12T23:37:51.582 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 10, 2024-09-12T23:37:51.582 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.583 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T05:31:12.292070+0000", 2024-09-12T23:37:51.583 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.583 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.583 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.583 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.583 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.583 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.583 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.583 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.583 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.583 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.583 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.583 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.583 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.583 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.584 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.584 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.584 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.584 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.584 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.584 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.584 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.584 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.584 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.584 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.584 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.584 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.584 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.584 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.584 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.585 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.585 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.585 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.585 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.585 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.585 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.585 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.585 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.585 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.585 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.585 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.585 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.585 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.586 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.586 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.586 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.586 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.586 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.586 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:51.586 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:51.586 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:51.586 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.586 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 2, 2024-09-12T23:37:51.586 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.586 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.586 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.586 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.587 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:51.587 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:51.587 INFO:tasks.workunit.client.0.smithi136.stdout: 4 2024-09-12T23:37:51.587 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.587 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 5, 2024-09-12T23:37:51.587 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.587 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.587 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:51.587 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.587 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.587 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.587 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.670478+0000", 2024-09-12T23:37:51.587 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.911877+0000", 2024-09-12T23:37:51.587 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.911877+0000", 2024-09-12T23:37:51.588 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.913684+0000", 2024-09-12T23:37:51.588 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.670478+0000", 2024-09-12T23:37:51.588 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.588 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.588 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.588 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:51.588 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.670478+0000", 2024-09-12T23:37:51.588 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.670478+0000", 2024-09-12T23:37:51.588 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.670478+0000", 2024-09-12T23:37:51.588 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.588 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.588 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.588 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.670478+0000", 2024-09-12T23:37:51.588 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.670478+0000", 2024-09-12T23:37:51.589 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.589 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.589 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.589 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.589 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:51.589 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.589 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.589 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.589 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.589 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.589 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.589 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.589 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.590 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.2f", 2024-09-12T23:37:51.590 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.590 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.590 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.590 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:51.590 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.590 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T03:39:10.601087+0000", 2024-09-12T23:37:51.590 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.590 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.590 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.590 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.590 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.590 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.590 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.591 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.591 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.591 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.591 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.591 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.591 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.591 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.591 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.591 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.591 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.591 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.591 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.591 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.591 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.591 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.592 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.592 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.592 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.592 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.592 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.592 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.592 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.592 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.592 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.592 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.592 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.592 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.592 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.592 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.593 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.593 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.593 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.593 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.593 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.593 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.593 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.593 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.593 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.593 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.593 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.593 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:51.593 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:51.593 INFO:tasks.workunit.client.0.smithi136.stdout: 4 2024-09-12T23:37:51.594 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.594 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 5, 2024-09-12T23:37:51.594 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.594 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.594 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.594 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.594 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:51.594 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:51.594 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:51.594 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.594 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 2, 2024-09-12T23:37:51.594 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.594 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.594 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:51.594 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.595 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.595 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.595 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.661668+0000", 2024-09-12T23:37:51.595 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.882148+0000", 2024-09-12T23:37:51.595 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.882148+0000", 2024-09-12T23:37:51.595 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.882889+0000", 2024-09-12T23:37:51.595 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.661668+0000", 2024-09-12T23:37:51.595 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.595 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.595 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.595 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:51.595 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.661668+0000", 2024-09-12T23:37:51.595 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.661668+0000", 2024-09-12T23:37:51.595 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.661668+0000", 2024-09-12T23:37:51.595 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.596 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.596 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.596 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.661668+0000", 2024-09-12T23:37:51.596 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.661668+0000", 2024-09-12T23:37:51.596 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.596 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.596 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.596 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.596 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:51.596 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.596 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.596 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.596 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.596 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.597 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.597 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.597 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.597 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.2e", 2024-09-12T23:37:51.597 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.597 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.597 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.597 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:51.597 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.597 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T09:59:38.545795+0000", 2024-09-12T23:37:51.597 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.597 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.597 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.598 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.598 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.598 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.598 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.598 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.598 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.598 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.598 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.598 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.598 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.598 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.598 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.598 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.598 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.599 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.599 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.599 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.599 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.599 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.599 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.599 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.599 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.599 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.599 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.599 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.599 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.599 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.599 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.600 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.600 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.600 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.600 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.600 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.600 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.600 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.600 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.600 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.600 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.600 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.600 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.600 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.600 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.601 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.601 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.601 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:51.601 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:51.601 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:51.601 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.601 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 2, 2024-09-12T23:37:51.601 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.601 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.601 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.601 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.601 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:51.601 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:51.601 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:51.602 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.602 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 7, 2024-09-12T23:37:51.602 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.602 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.602 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:51.602 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.602 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.602 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.602 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.666136+0000", 2024-09-12T23:37:51.602 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.904247+0000", 2024-09-12T23:37:51.602 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.904247+0000", 2024-09-12T23:37:51.602 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.904552+0000", 2024-09-12T23:37:51.602 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.666136+0000", 2024-09-12T23:37:51.602 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.602 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.603 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.603 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:51.603 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.666136+0000", 2024-09-12T23:37:51.603 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.666136+0000", 2024-09-12T23:37:51.603 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.666136+0000", 2024-09-12T23:37:51.603 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.603 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.603 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.603 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.666136+0000", 2024-09-12T23:37:51.603 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.666136+0000", 2024-09-12T23:37:51.603 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.603 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.603 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.603 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.603 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:51.604 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.604 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.604 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.604 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.604 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.604 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.604 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.604 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.604 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.2d", 2024-09-12T23:37:51.604 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.604 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.604 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.604 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:51.604 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.605 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T00:30:06.600814+0000", 2024-09-12T23:37:51.605 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.605 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.605 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.605 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.605 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.605 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.605 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.605 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.605 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.605 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.605 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.605 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.605 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.605 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.606 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.606 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.606 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.606 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.606 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.606 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.606 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.606 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.606 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.606 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.606 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.606 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.606 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.606 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.607 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.607 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.607 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.607 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.607 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.607 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.607 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.607 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.607 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.607 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.607 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.607 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.607 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.607 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.607 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.608 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.608 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.608 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.608 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.608 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:51.608 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:51.608 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:51.608 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.608 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 7, 2024-09-12T23:37:51.608 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.608 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.608 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.608 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.609 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:51.609 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:51.609 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:51.609 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.609 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 6, 2024-09-12T23:37:51.609 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.609 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.609 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:51.609 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.609 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.609 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.609 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.665287+0000", 2024-09-12T23:37:51.609 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.905690+0000", 2024-09-12T23:37:51.609 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.905690+0000", 2024-09-12T23:37:51.610 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.905965+0000", 2024-09-12T23:37:51.610 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.665287+0000", 2024-09-12T23:37:51.610 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.610 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.610 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.610 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:51.610 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.665287+0000", 2024-09-12T23:37:51.610 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.665287+0000", 2024-09-12T23:37:51.610 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.665287+0000", 2024-09-12T23:37:51.610 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.610 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.610 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.610 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.665287+0000", 2024-09-12T23:37:51.610 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.665287+0000", 2024-09-12T23:37:51.610 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.610 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.611 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.611 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.611 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:51.611 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.611 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.611 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.611 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.611 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.611 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.611 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.611 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.611 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.2c", 2024-09-12T23:37:51.611 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.611 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.612 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.612 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:51.612 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.612 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T10:58:29.597052+0000", 2024-09-12T23:37:51.612 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.612 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.612 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.612 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.612 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.612 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.612 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.612 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.612 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.613 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.613 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.613 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.613 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.613 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.613 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.613 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.613 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.613 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.613 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.613 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.613 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.613 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.613 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.614 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.614 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.614 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.614 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.614 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.614 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.614 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.614 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.614 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.614 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.614 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.614 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.614 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.614 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.614 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.615 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.615 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.615 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.615 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.615 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.615 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.615 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.615 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.615 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.615 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:51.615 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:51.615 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:51.615 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.615 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 6, 2024-09-12T23:37:51.615 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.616 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.616 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.616 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.616 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:51.616 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:51.616 INFO:tasks.workunit.client.0.smithi136.stdout: 1 2024-09-12T23:37:51.616 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.616 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 0, 2024-09-12T23:37:51.616 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.616 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.616 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:51.616 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.616 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.616 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.616 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.669549+0000", 2024-09-12T23:37:51.616 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.917878+0000", 2024-09-12T23:37:51.617 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.917878+0000", 2024-09-12T23:37:51.617 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.918231+0000", 2024-09-12T23:37:51.617 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.669549+0000", 2024-09-12T23:37:51.617 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.617 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.617 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.617 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:51.617 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.669549+0000", 2024-09-12T23:37:51.617 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.669549+0000", 2024-09-12T23:37:51.617 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.669549+0000", 2024-09-12T23:37:51.617 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.617 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.617 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.618 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.669549+0000", 2024-09-12T23:37:51.618 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.669549+0000", 2024-09-12T23:37:51.618 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.618 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.618 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.618 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.618 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:51.618 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.618 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.618 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.618 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.618 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.618 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.618 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.619 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.619 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.2b", 2024-09-12T23:37:51.619 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.619 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.619 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.619 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:51.619 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.619 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T06:22:46.498222+0000", 2024-09-12T23:37:51.619 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.619 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.619 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.619 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.619 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.619 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.620 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.620 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.620 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.620 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.620 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.620 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.620 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.620 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.620 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.620 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.620 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.620 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.620 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.620 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.620 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.620 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.621 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.621 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.621 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.621 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.621 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.621 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.621 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.621 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.621 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.621 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.621 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.621 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.621 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.621 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.622 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.622 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.622 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.622 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.622 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.622 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.622 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.622 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.622 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.622 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.622 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.622 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:51.622 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:51.622 INFO:tasks.workunit.client.0.smithi136.stdout: 1 2024-09-12T23:37:51.622 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.623 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 0, 2024-09-12T23:37:51.623 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.623 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.623 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.623 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.623 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:51.623 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:51.623 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:51.623 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.623 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 5, 2024-09-12T23:37:51.623 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.623 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.623 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:51.623 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.623 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.624 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.624 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.671076+0000", 2024-09-12T23:37:51.624 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.909507+0000", 2024-09-12T23:37:51.624 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.909507+0000", 2024-09-12T23:37:51.624 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.909780+0000", 2024-09-12T23:37:51.624 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.671076+0000", 2024-09-12T23:37:51.624 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.624 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.624 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.624 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:51.624 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.671076+0000", 2024-09-12T23:37:51.624 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.671076+0000", 2024-09-12T23:37:51.624 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.671076+0000", 2024-09-12T23:37:51.625 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.625 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.625 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.625 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.671076+0000", 2024-09-12T23:37:51.625 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.671076+0000", 2024-09-12T23:37:51.625 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.625 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.625 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.625 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.625 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:51.625 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.625 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.625 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.625 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.626 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.626 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.626 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.626 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.626 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.2a", 2024-09-12T23:37:51.626 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.626 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.626 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.626 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:51.626 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.626 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T04:39:50.529545+0000", 2024-09-12T23:37:51.626 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.626 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.627 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.627 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.627 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.627 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.627 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.627 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.627 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.627 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.627 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.627 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.627 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.627 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.627 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.627 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.627 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.628 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.628 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.628 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.628 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.628 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.628 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.628 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.628 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.628 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.628 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.628 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.628 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.628 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.628 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.629 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.629 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.629 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.629 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.629 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.629 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.629 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.629 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.629 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.629 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.629 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.629 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.629 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.629 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.630 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.630 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.630 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:51.630 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:51.630 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:51.630 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.630 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 5, 2024-09-12T23:37:51.630 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.630 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.630 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.630 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.630 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:51.630 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:51.630 INFO:tasks.workunit.client.0.smithi136.stdout: 1 2024-09-12T23:37:51.631 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.631 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 3, 2024-09-12T23:37:51.631 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.631 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.631 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:51.631 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.631 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.631 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.631 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.667544+0000", 2024-09-12T23:37:51.631 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.905780+0000", 2024-09-12T23:37:51.631 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.905780+0000", 2024-09-12T23:37:51.631 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.906087+0000", 2024-09-12T23:37:51.631 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.667544+0000", 2024-09-12T23:37:51.631 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.632 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.632 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.632 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:51.632 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.667544+0000", 2024-09-12T23:37:51.632 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.667544+0000", 2024-09-12T23:37:51.632 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.667544+0000", 2024-09-12T23:37:51.632 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.632 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.632 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.632 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.667544+0000", 2024-09-12T23:37:51.632 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.667544+0000", 2024-09-12T23:37:51.632 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.632 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.633 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.633 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.633 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:51.633 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.633 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.633 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.633 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.633 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.633 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.633 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.633 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.633 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.29", 2024-09-12T23:37:51.633 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.633 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.634 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.634 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:51.634 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.634 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T07:00:40.777941+0000", 2024-09-12T23:37:51.634 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.634 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.634 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.634 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.634 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.634 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.634 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.634 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.634 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.634 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.635 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.635 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.635 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.635 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.635 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.635 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.635 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.635 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.635 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.635 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.635 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.635 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.635 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.635 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.636 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.636 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.636 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.636 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.636 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.636 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.636 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.636 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.636 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.636 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.636 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.636 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.636 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.636 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.637 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.637 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.637 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.637 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.637 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.637 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.637 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.637 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.637 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.637 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:51.637 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:51.637 INFO:tasks.workunit.client.0.smithi136.stdout: 1 2024-09-12T23:37:51.637 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.637 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 3, 2024-09-12T23:37:51.637 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.638 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.638 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.638 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.638 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:51.638 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:51.638 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:51.638 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.638 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 5, 2024-09-12T23:37:51.638 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.638 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.638 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:51.638 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.638 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.638 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.639 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.670507+0000", 2024-09-12T23:37:51.639 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.917757+0000", 2024-09-12T23:37:51.639 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.917757+0000", 2024-09-12T23:37:51.639 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.918286+0000", 2024-09-12T23:37:51.639 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.670507+0000", 2024-09-12T23:37:51.639 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.639 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.639 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.639 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:51.639 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.670507+0000", 2024-09-12T23:37:51.639 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.670507+0000", 2024-09-12T23:37:51.639 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.670507+0000", 2024-09-12T23:37:51.639 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.639 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.640 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.640 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.670507+0000", 2024-09-12T23:37:51.640 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.670507+0000", 2024-09-12T23:37:51.640 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.640 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.640 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.640 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.640 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:51.640 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.640 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.640 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.640 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.640 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.640 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.641 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.641 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.641 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.28", 2024-09-12T23:37:51.641 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.641 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.641 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.641 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:51.641 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.641 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T06:49:26.856852+0000", 2024-09-12T23:37:51.641 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.641 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.641 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.641 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.641 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.642 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.642 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.642 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.642 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.642 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.642 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.642 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.642 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.642 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.642 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.642 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.642 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.642 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.642 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.643 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.643 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.643 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.643 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.643 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.643 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.643 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.643 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.643 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.643 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.643 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.643 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.643 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.644 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.644 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.644 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.644 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.644 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.644 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.644 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.644 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.644 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.644 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.644 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.644 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.644 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.644 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.645 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.645 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:51.645 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:51.645 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:51.645 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.645 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 5, 2024-09-12T23:37:51.645 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.645 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.645 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.645 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.645 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:51.645 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:51.645 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:51.645 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.645 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 3, 2024-09-12T23:37:51.646 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.646 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.646 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:51.646 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.646 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.646 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.646 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.673387+0000", 2024-09-12T23:37:51.646 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.903081+0000", 2024-09-12T23:37:51.646 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.903081+0000", 2024-09-12T23:37:51.646 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.903381+0000", 2024-09-12T23:37:51.646 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.673387+0000", 2024-09-12T23:37:51.646 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.646 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.646 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.647 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:51.647 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.673387+0000", 2024-09-12T23:37:51.647 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.673387+0000", 2024-09-12T23:37:51.647 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.673387+0000", 2024-09-12T23:37:51.647 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.647 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.647 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.647 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.673387+0000", 2024-09-12T23:37:51.647 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.673387+0000", 2024-09-12T23:37:51.647 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.647 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.647 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.647 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.648 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:51.648 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.648 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.648 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.648 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.648 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.648 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.648 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.648 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.648 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.27", 2024-09-12T23:37:51.648 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.648 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.648 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.648 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:51.649 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.649 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T00:43:01.175691+0000", 2024-09-12T23:37:51.649 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.649 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.649 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.649 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.649 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.649 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.649 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.649 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.649 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.649 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.649 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.649 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.649 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.650 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.650 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.650 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.650 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.650 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.650 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.650 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.650 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.650 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.650 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.650 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.650 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.650 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.650 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.651 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.651 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.651 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.651 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.651 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.651 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.651 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.651 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.651 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.651 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.651 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.651 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.651 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.651 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.652 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.652 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.652 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.652 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.652 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.652 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.652 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:51.652 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:51.652 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:51.652 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.652 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 3, 2024-09-12T23:37:51.652 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.652 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.653 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.653 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.653 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:51.653 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:51.653 INFO:tasks.workunit.client.0.smithi136.stdout: 4 2024-09-12T23:37:51.653 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.653 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 6, 2024-09-12T23:37:51.653 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.653 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.653 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:51.653 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.653 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.653 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.654 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.666859+0000", 2024-09-12T23:37:51.654 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.896834+0000", 2024-09-12T23:37:51.654 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.896834+0000", 2024-09-12T23:37:51.654 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.897170+0000", 2024-09-12T23:37:51.654 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.666859+0000", 2024-09-12T23:37:51.654 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.654 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.654 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.654 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:51.654 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.666859+0000", 2024-09-12T23:37:51.654 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.666859+0000", 2024-09-12T23:37:51.654 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.666859+0000", 2024-09-12T23:37:51.654 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.654 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.655 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.655 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.666859+0000", 2024-09-12T23:37:51.655 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.666859+0000", 2024-09-12T23:37:51.655 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.655 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.655 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.655 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.655 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:51.655 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.655 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.655 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.655 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.655 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.656 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.656 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.656 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.656 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.26", 2024-09-12T23:37:51.656 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.656 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.656 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.656 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:51.656 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.656 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T00:47:43.534530+0000", 2024-09-12T23:37:51.656 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.656 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.656 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.656 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.657 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.657 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.657 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.657 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.657 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.657 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.657 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.657 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.657 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.657 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.657 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.657 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.657 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.658 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.658 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.658 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.658 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.658 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.658 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.658 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.658 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.658 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.658 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.658 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.658 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.658 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.659 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.659 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.659 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.659 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.659 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.659 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.659 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.659 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.659 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.659 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.659 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.659 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.659 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.660 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.660 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.660 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.660 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.660 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:51.660 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:51.660 INFO:tasks.workunit.client.0.smithi136.stdout: 4 2024-09-12T23:37:51.660 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.660 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 6, 2024-09-12T23:37:51.660 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.660 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.660 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.660 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.660 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:51.661 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:51.661 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:51.661 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.661 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 1, 2024-09-12T23:37:51.661 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.661 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.661 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:51.661 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.661 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.661 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.661 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.663571+0000", 2024-09-12T23:37:51.661 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.919185+0000", 2024-09-12T23:37:51.661 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.919185+0000", 2024-09-12T23:37:51.662 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.919500+0000", 2024-09-12T23:37:51.662 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.663571+0000", 2024-09-12T23:37:51.662 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.662 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.662 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.662 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:51.662 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.663571+0000", 2024-09-12T23:37:51.662 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.663571+0000", 2024-09-12T23:37:51.662 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.663571+0000", 2024-09-12T23:37:51.662 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.662 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.662 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.662 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.663571+0000", 2024-09-12T23:37:51.663 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.663571+0000", 2024-09-12T23:37:51.663 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.663 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.663 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.663 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.663 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:51.663 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.663 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.663 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.663 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.663 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.663 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.663 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.663 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.664 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.25", 2024-09-12T23:37:51.664 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.664 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.664 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.664 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:51.664 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.664 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T01:56:52.566524+0000", 2024-09-12T23:37:51.664 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.664 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.664 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.664 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.664 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.664 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.665 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.665 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.665 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.665 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.665 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.665 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.665 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.665 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.665 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.665 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.665 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.665 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.665 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.665 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.666 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.666 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.666 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.666 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.666 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.666 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.666 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.666 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.666 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.666 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.666 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.666 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.666 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.667 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.667 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.667 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.667 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.667 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.667 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.667 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.667 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.667 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.667 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.667 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.667 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.667 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.667 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.668 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:51.668 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:51.668 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:51.668 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.668 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 1, 2024-09-12T23:37:51.668 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.668 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.668 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.668 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.668 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:51.668 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:51.668 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:51.668 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.668 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 3, 2024-09-12T23:37:51.668 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.669 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.669 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:51.669 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.669 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.669 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.669 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.672875+0000", 2024-09-12T23:37:51.669 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.913193+0000", 2024-09-12T23:37:51.669 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.913193+0000", 2024-09-12T23:37:51.669 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.913907+0000", 2024-09-12T23:37:51.669 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.672875+0000", 2024-09-12T23:37:51.669 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.669 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.669 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.669 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:51.670 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.672875+0000", 2024-09-12T23:37:51.670 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.672875+0000", 2024-09-12T23:37:51.670 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.672875+0000", 2024-09-12T23:37:51.670 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.670 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.670 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.670 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.672875+0000", 2024-09-12T23:37:51.670 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.672875+0000", 2024-09-12T23:37:51.670 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.670 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.670 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.670 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.670 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:51.670 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.671 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.671 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.671 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.671 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.671 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.671 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.671 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.671 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.24", 2024-09-12T23:37:51.671 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.671 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.671 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.671 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:51.671 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.671 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T10:32:45.117056+0000", 2024-09-12T23:37:51.672 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.672 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.672 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.672 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.672 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.672 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.672 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.672 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.672 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.672 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.672 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.672 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.672 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.673 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.673 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.673 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.673 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.673 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.673 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.673 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.673 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.673 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.673 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.673 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.673 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.673 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.674 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.674 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.674 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.674 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.674 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.674 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.674 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.674 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.674 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.674 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.674 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.674 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.674 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.674 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.675 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.675 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.675 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.675 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.675 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.675 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.675 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.675 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:51.675 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:51.675 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:51.675 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.675 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 3, 2024-09-12T23:37:51.675 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.676 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.676 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.676 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.676 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:51.676 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:51.676 INFO:tasks.workunit.client.0.smithi136.stdout: 4 2024-09-12T23:37:51.676 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.676 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 0, 2024-09-12T23:37:51.676 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.676 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.676 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:51.676 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.676 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.677 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.677 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.669650+0000", 2024-09-12T23:37:51.677 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.911007+0000", 2024-09-12T23:37:51.677 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.911007+0000", 2024-09-12T23:37:51.677 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.911279+0000", 2024-09-12T23:37:51.677 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.669650+0000", 2024-09-12T23:37:51.677 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.677 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.677 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.677 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:51.677 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.669650+0000", 2024-09-12T23:37:51.677 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.669650+0000", 2024-09-12T23:37:51.677 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.669650+0000", 2024-09-12T23:37:51.677 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.678 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.678 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.678 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.669650+0000", 2024-09-12T23:37:51.678 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.669650+0000", 2024-09-12T23:37:51.678 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.678 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.678 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.678 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.678 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:51.678 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.678 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.678 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.678 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.678 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.679 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.679 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.679 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.679 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.23", 2024-09-12T23:37:51.679 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.679 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.679 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.679 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:51.679 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.679 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T06:26:51.842858+0000", 2024-09-12T23:37:51.679 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.679 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.679 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.679 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.680 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.680 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.680 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.680 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.680 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.680 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.680 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.680 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.680 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.680 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.680 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.680 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.680 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.681 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.681 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.681 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.681 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.681 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.681 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.681 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.681 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.681 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.681 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.681 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.681 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.681 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.681 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.682 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.682 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.682 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.682 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.682 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.682 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.682 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.682 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.682 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.682 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.682 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.682 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.682 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.683 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.683 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.683 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.683 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:51.683 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:51.683 INFO:tasks.workunit.client.0.smithi136.stdout: 4 2024-09-12T23:37:51.683 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.683 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 0, 2024-09-12T23:37:51.683 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.683 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.683 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.683 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.683 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:51.683 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:51.683 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:51.684 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.684 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 5, 2024-09-12T23:37:51.684 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.684 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.684 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:51.684 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.684 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.684 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.684 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.671116+0000", 2024-09-12T23:37:51.684 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.913659+0000", 2024-09-12T23:37:51.684 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.913659+0000", 2024-09-12T23:37:51.684 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.914015+0000", 2024-09-12T23:37:51.684 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.671116+0000", 2024-09-12T23:37:51.685 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.685 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.685 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.685 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:51.685 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.671116+0000", 2024-09-12T23:37:51.685 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.671116+0000", 2024-09-12T23:37:51.685 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.671116+0000", 2024-09-12T23:37:51.685 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.685 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.685 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.685 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.671116+0000", 2024-09-12T23:37:51.685 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.671116+0000", 2024-09-12T23:37:51.685 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.685 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.686 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.686 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.686 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:51.686 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.686 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.686 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.686 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.686 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.686 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.686 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.686 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.686 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.22", 2024-09-12T23:37:51.686 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.687 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.687 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.687 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:51.687 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.687 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T03:37:02.989629+0000", 2024-09-12T23:37:51.687 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.687 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.687 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.687 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.687 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.687 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.687 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.687 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.688 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.688 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.688 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.688 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.688 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.688 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.688 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.688 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.688 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.688 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.688 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.688 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.688 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.688 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.689 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.689 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.689 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.689 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.689 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.689 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.689 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.689 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.689 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.689 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.689 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.689 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.689 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.689 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.690 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.690 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.690 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.690 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.690 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.690 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.690 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.690 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.690 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.690 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.690 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.690 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:51.690 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:51.690 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:51.691 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.691 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 5, 2024-09-12T23:37:51.691 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.691 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.691 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.691 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.691 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:51.691 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:51.691 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:51.691 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.691 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 2, 2024-09-12T23:37:51.691 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.691 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.691 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:51.692 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.692 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.692 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.692 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.661319+0000", 2024-09-12T23:37:51.692 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.889548+0000", 2024-09-12T23:37:51.692 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.889548+0000", 2024-09-12T23:37:51.692 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.889782+0000", 2024-09-12T23:37:51.692 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.661319+0000", 2024-09-12T23:37:51.692 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.692 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.692 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.692 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:51.692 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.661319+0000", 2024-09-12T23:37:51.693 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.661319+0000", 2024-09-12T23:37:51.693 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.661319+0000", 2024-09-12T23:37:51.693 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.693 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.693 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.693 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.661319+0000", 2024-09-12T23:37:51.693 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.661319+0000", 2024-09-12T23:37:51.693 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.693 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.693 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.693 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.693 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:51.693 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.693 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.693 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.694 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.694 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.694 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.694 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.694 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.694 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.21", 2024-09-12T23:37:51.694 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.694 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.694 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.694 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:51.694 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.694 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-13T23:54:53.906619+0000", 2024-09-12T23:37:51.694 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.695 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.695 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.695 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.695 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.695 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.695 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.695 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.695 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.695 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.695 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.695 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.695 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.695 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.695 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.696 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.696 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.696 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.696 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.696 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.696 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.696 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.696 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.696 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.696 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.696 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.696 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.696 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.696 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.697 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.697 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.697 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.697 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.697 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.697 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.697 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.697 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.697 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.697 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.697 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.697 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.697 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.698 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.698 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.698 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.698 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.698 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.698 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:51.698 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:51.698 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:51.698 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.698 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 2, 2024-09-12T23:37:51.698 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.698 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.698 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.698 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.698 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:51.699 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:51.699 INFO:tasks.workunit.client.0.smithi136.stdout: 5 2024-09-12T23:37:51.699 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.699 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 6, 2024-09-12T23:37:51.699 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.699 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.699 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:51.699 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.699 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.699 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.699 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.665474+0000", 2024-09-12T23:37:51.699 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.895919+0000", 2024-09-12T23:37:51.699 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.895919+0000", 2024-09-12T23:37:51.699 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.896515+0000", 2024-09-12T23:37:51.700 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.665474+0000", 2024-09-12T23:37:51.700 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.700 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.700 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.700 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:51.700 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.665474+0000", 2024-09-12T23:37:51.700 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.665474+0000", 2024-09-12T23:37:51.700 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.665474+0000", 2024-09-12T23:37:51.700 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.700 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.700 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.700 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.665474+0000", 2024-09-12T23:37:51.700 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.665474+0000", 2024-09-12T23:37:51.701 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.701 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.701 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.701 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.701 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:51.701 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.701 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.701 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.701 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.701 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.701 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.701 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.701 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.701 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.20", 2024-09-12T23:37:51.702 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.702 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.702 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.702 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:51.702 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.702 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T05:17:53.675501+0000", 2024-09-12T23:37:51.702 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.702 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.702 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.702 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.702 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.702 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.702 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.702 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.703 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.703 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.703 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.703 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.703 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.703 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.703 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.703 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.703 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.703 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.703 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.703 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.703 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.703 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.704 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.704 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.704 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.704 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.704 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.704 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.704 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.704 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.704 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.704 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.704 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.704 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.704 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.704 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.705 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.705 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.705 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.705 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.705 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.705 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.705 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.705 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.705 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.705 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.705 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.705 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:51.705 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:51.705 INFO:tasks.workunit.client.0.smithi136.stdout: 5 2024-09-12T23:37:51.706 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.706 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 6, 2024-09-12T23:37:51.706 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.706 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.706 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.706 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.706 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:51.706 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:51.706 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:51.706 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.706 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 4, 2024-09-12T23:37:51.706 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.706 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.706 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:51.707 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.707 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.707 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.707 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.660905+0000", 2024-09-12T23:37:51.707 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.890839+0000", 2024-09-12T23:37:51.707 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.890839+0000", 2024-09-12T23:37:51.707 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.891596+0000", 2024-09-12T23:37:51.707 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.660905+0000", 2024-09-12T23:37:51.707 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.707 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.707 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.707 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:51.707 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.660905+0000", 2024-09-12T23:37:51.707 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.660905+0000", 2024-09-12T23:37:51.707 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.660905+0000", 2024-09-12T23:37:51.708 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.708 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.708 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.708 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.660905+0000", 2024-09-12T23:37:51.708 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.660905+0000", 2024-09-12T23:37:51.708 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.708 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.708 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.708 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.708 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:51.708 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.708 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.708 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.709 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.709 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.709 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.709 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.709 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.709 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.1a", 2024-09-12T23:37:51.709 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.709 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.709 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.709 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:51.709 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.709 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T09:29:49.995950+0000", 2024-09-12T23:37:51.709 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.709 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.710 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.710 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.710 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.710 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.710 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.710 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.710 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.710 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.710 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.710 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.710 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.710 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.710 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.710 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.711 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.711 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.711 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.711 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.711 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.711 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.711 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.711 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.711 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.711 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.711 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.711 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.711 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.711 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.711 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.712 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.712 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.712 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.712 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.712 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.712 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.712 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.712 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.712 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.712 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.712 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.712 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.712 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.713 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.713 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.713 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.713 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:51.713 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:51.713 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:51.713 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.713 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 4, 2024-09-12T23:37:51.713 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.713 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.713 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.713 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.713 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:51.713 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:51.714 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:51.714 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.714 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 3, 2024-09-12T23:37:51.714 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.714 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.714 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:51.714 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.714 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.714 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.714 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.668622+0000", 2024-09-12T23:37:51.714 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.882848+0000", 2024-09-12T23:37:51.714 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.882848+0000", 2024-09-12T23:37:51.714 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.883222+0000", 2024-09-12T23:37:51.714 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.668622+0000", 2024-09-12T23:37:51.715 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:51.715 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.715 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:51.715 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:51.715 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.668622+0000", 2024-09-12T23:37:51.715 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.668622+0000", 2024-09-12T23:37:51.715 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.668622+0000", 2024-09-12T23:37:51.715 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.715 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.715 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:51.715 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.668622+0000", 2024-09-12T23:37:51.715 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.668622+0000", 2024-09-12T23:37:51.715 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.715 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 1, 2024-09-12T23:37:51.716 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.716 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.716 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:51.716 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.716 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.716 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.716 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.716 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 1, 2024-09-12T23:37:51.716 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.716 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.716 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.716 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.1b", 2024-09-12T23:37:51.716 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.716 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.717 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.717 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 22, 2024-09-12T23:37:51.717 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.717 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T02:57:33.905996+0000", 2024-09-12T23:37:51.717 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.717 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.717 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.717 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.717 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.717 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.717 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.717 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.717 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.718 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.718 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.718 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.718 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.718 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.718 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.718 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.718 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.718 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.718 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.718 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.718 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.718 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.718 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.719 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.719 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.719 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.719 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.719 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.719 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.719 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.719 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.719 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.719 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.719 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.719 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.719 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.719 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.720 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.720 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.720 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.720 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.720 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.720 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.720 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.720 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.720 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.720 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.720 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:51.720 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:51.720 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:51.720 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.721 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 3, 2024-09-12T23:37:51.721 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "53'1" 2024-09-12T23:37:51.721 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.721 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.721 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.721 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:51.721 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:51.721 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:51.721 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.721 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 0, 2024-09-12T23:37:51.721 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.721 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.721 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:51.721 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.722 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.722 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.722 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.669749+0000", 2024-09-12T23:37:51.722 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.912406+0000", 2024-09-12T23:37:51.722 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.912406+0000", 2024-09-12T23:37:51.722 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.912763+0000", 2024-09-12T23:37:51.722 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.669749+0000", 2024-09-12T23:37:51.722 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.722 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.722 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.722 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:51.722 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.669749+0000", 2024-09-12T23:37:51.722 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.669749+0000", 2024-09-12T23:37:51.722 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.669749+0000", 2024-09-12T23:37:51.723 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.723 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.723 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.723 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.669749+0000", 2024-09-12T23:37:51.723 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.669749+0000", 2024-09-12T23:37:51.723 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.723 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.723 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.723 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.723 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:51.723 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.723 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.723 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.723 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.724 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.724 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.724 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.724 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.724 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.1b", 2024-09-12T23:37:51.724 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.724 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.724 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.724 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:51.724 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.724 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T01:49:06.828514+0000", 2024-09-12T23:37:51.724 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.724 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.724 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.725 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.725 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.725 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.725 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.725 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.725 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.725 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.725 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.725 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.725 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.725 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.725 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.725 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.725 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.726 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.726 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.726 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.726 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.726 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.726 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.726 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.726 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.726 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.726 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.726 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.726 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.726 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.726 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.727 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.727 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.727 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.727 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.727 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.727 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.727 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.727 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.727 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.727 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.727 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.727 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.727 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.728 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.728 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.728 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.728 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:51.728 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:51.728 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:51.728 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.728 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 0, 2024-09-12T23:37:51.728 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.728 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.728 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.728 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.728 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:51.728 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:51.729 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:51.729 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.729 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 6, 2024-09-12T23:37:51.729 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.729 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.729 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:51.729 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.729 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.729 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.729 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.667384+0000", 2024-09-12T23:37:51.729 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.884139+0000", 2024-09-12T23:37:51.729 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.884139+0000", 2024-09-12T23:37:51.729 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.884599+0000", 2024-09-12T23:37:51.729 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.667384+0000", 2024-09-12T23:37:51.730 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:51.730 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.730 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:51.730 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:51.730 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.667384+0000", 2024-09-12T23:37:51.730 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.667384+0000", 2024-09-12T23:37:51.730 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.667384+0000", 2024-09-12T23:37:51.730 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.730 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.730 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:51.730 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.667384+0000", 2024-09-12T23:37:51.730 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.667384+0000", 2024-09-12T23:37:51.730 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.730 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 1, 2024-09-12T23:37:51.730 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.731 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.731 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:51.731 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.731 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.731 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.731 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.731 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 1, 2024-09-12T23:37:51.731 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.731 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.731 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.731 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.1a", 2024-09-12T23:37:51.731 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.731 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.731 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.732 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 22, 2024-09-12T23:37:51.732 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.732 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T04:57:50.175287+0000", 2024-09-12T23:37:51.732 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.732 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.732 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.732 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.732 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.732 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.732 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.732 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.732 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.732 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.732 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.733 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.733 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.733 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.733 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.733 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.733 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.733 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.733 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.733 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.733 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.733 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.733 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.733 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.733 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.734 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.734 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.734 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.734 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.734 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.734 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.734 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.734 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.734 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.734 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.734 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.734 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.734 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.734 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.735 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.735 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.735 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.735 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.735 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.735 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.735 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.735 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.735 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:51.735 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:51.735 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:51.735 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.735 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 6, 2024-09-12T23:37:51.735 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "53'1" 2024-09-12T23:37:51.736 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.736 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.736 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.736 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:51.736 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:51.736 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:51.736 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.736 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 3, 2024-09-12T23:37:51.736 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.736 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.736 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:51.736 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.736 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.736 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.737 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.668064+0000", 2024-09-12T23:37:51.737 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.872404+0000", 2024-09-12T23:37:51.737 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.872404+0000", 2024-09-12T23:37:51.737 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.872822+0000", 2024-09-12T23:37:51.737 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.668064+0000", 2024-09-12T23:37:51.737 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:51.737 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.737 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:51.737 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:51.737 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.668064+0000", 2024-09-12T23:37:51.737 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.668064+0000", 2024-09-12T23:37:51.737 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.668064+0000", 2024-09-12T23:37:51.737 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.738 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.738 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:51.738 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.668064+0000", 2024-09-12T23:37:51.738 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.668064+0000", 2024-09-12T23:37:51.738 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.738 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.738 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.738 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.738 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:51.738 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.738 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.738 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.738 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.738 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.739 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.739 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.739 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.739 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.19", 2024-09-12T23:37:51.739 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.739 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.739 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.739 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 21, 2024-09-12T23:37:51.739 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.739 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-13T23:47:42.317408+0000", 2024-09-12T23:37:51.739 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.739 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.739 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.739 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.740 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.740 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.740 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.740 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.740 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.740 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.740 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.740 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.740 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.740 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.740 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.740 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.740 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.740 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.741 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.741 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.741 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.741 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.741 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.741 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.741 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.741 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.741 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.741 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.741 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.741 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.741 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.741 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.742 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.742 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.742 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.742 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.742 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.742 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.742 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.742 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.742 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.742 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.742 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.742 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.742 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.743 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.743 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.743 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:51.743 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:51.743 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:51.743 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.743 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 3, 2024-09-12T23:37:51.743 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.743 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.743 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.743 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.743 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:51.743 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:51.743 INFO:tasks.workunit.client.0.smithi136.stdout: 1 2024-09-12T23:37:51.744 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.744 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 3, 2024-09-12T23:37:51.744 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.744 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.744 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:51.744 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.744 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.744 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.744 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.667077+0000", 2024-09-12T23:37:51.744 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.907067+0000", 2024-09-12T23:37:51.744 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.907067+0000", 2024-09-12T23:37:51.744 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.907360+0000", 2024-09-12T23:37:51.744 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.667077+0000", 2024-09-12T23:37:51.744 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.745 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.745 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.745 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:51.745 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.667077+0000", 2024-09-12T23:37:51.745 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.667077+0000", 2024-09-12T23:37:51.745 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.667077+0000", 2024-09-12T23:37:51.745 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.745 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.745 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.745 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.667077+0000", 2024-09-12T23:37:51.745 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.667077+0000", 2024-09-12T23:37:51.745 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.745 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.745 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.746 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.746 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:51.746 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.746 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.746 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.746 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.746 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.746 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.746 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.746 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.746 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.18", 2024-09-12T23:37:51.746 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.746 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.746 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.747 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:51.747 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.747 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T11:26:38.084938+0000", 2024-09-12T23:37:51.747 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.747 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.747 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.747 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.747 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.747 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.747 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.747 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.747 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.747 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.747 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.747 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.748 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.748 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.748 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.748 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.748 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.748 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.748 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.748 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.748 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.748 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.748 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.748 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.748 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.748 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.749 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.749 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.749 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.749 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.749 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.749 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.749 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.749 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.749 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.749 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.749 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.749 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.749 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.749 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.750 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.750 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.750 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.750 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.750 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.750 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.750 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.750 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:51.750 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:51.750 INFO:tasks.workunit.client.0.smithi136.stdout: 1 2024-09-12T23:37:51.750 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.750 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 3, 2024-09-12T23:37:51.750 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.750 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.751 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.751 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.751 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:51.751 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:51.751 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:51.751 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.751 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 5, 2024-09-12T23:37:51.751 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.751 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.751 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:51.751 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.751 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.751 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.751 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.670406+0000", 2024-09-12T23:37:51.751 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.896167+0000", 2024-09-12T23:37:51.752 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.896167+0000", 2024-09-12T23:37:51.752 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.896704+0000", 2024-09-12T23:37:51.752 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.670406+0000", 2024-09-12T23:37:51.752 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:51.752 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.752 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:51.752 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:51.752 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.670406+0000", 2024-09-12T23:37:51.752 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.670406+0000", 2024-09-12T23:37:51.752 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.670406+0000", 2024-09-12T23:37:51.752 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.752 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.752 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:51.753 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.670406+0000", 2024-09-12T23:37:51.753 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.670406+0000", 2024-09-12T23:37:51.753 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.753 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.753 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.753 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.753 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:51.753 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.753 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.753 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.753 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.753 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.753 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.753 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.754 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.754 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.18", 2024-09-12T23:37:51.754 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.754 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.754 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.754 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 21, 2024-09-12T23:37:51.754 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.754 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T09:14:47.959501+0000", 2024-09-12T23:37:51.754 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.754 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.754 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.754 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.754 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.754 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.755 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.755 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.755 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.755 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.755 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.755 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.755 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.755 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.755 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.755 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.755 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.755 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.755 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.755 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.756 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.756 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.756 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.756 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.756 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.756 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.756 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.756 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.756 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.756 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.756 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.756 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.756 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.756 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.757 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.757 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.757 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.757 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.757 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.757 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.757 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.757 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.757 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.759 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.759 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.759 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.760 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.760 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:51.760 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:51.760 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:51.760 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.760 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 5, 2024-09-12T23:37:51.760 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.760 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.760 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.760 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.760 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:51.760 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:51.760 INFO:tasks.workunit.client.0.smithi136.stdout: 4 2024-09-12T23:37:51.760 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.761 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 1, 2024-09-12T23:37:51.761 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.761 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.761 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:51.761 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.761 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.761 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.761 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.663618+0000", 2024-09-12T23:37:51.761 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.892988+0000", 2024-09-12T23:37:51.761 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.892988+0000", 2024-09-12T23:37:51.761 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.893401+0000", 2024-09-12T23:37:51.761 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.663618+0000", 2024-09-12T23:37:51.761 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.761 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.762 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.762 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:51.762 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.663618+0000", 2024-09-12T23:37:51.762 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.663618+0000", 2024-09-12T23:37:51.762 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.663618+0000", 2024-09-12T23:37:51.762 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.762 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.762 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.762 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.663618+0000", 2024-09-12T23:37:51.762 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.663618+0000", 2024-09-12T23:37:51.762 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.762 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.762 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.763 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.763 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:51.763 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.763 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.763 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.763 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.763 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.763 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.763 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.763 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.763 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.19", 2024-09-12T23:37:51.763 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.763 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.763 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.763 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:51.764 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.764 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T07:30:04.271132+0000", 2024-09-12T23:37:51.764 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.764 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.764 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.764 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.764 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.764 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.764 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.764 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.764 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.764 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.764 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.764 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.765 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.765 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.765 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.765 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.765 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.765 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.765 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.765 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.765 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.765 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.765 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.765 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.765 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.765 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.766 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.766 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.766 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.766 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.766 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.766 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.766 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.766 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.766 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.766 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.766 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.766 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.766 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.766 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.767 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.767 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.767 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.767 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.767 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.767 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.767 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.767 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:51.767 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:51.767 INFO:tasks.workunit.client.0.smithi136.stdout: 4 2024-09-12T23:37:51.767 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.767 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 1, 2024-09-12T23:37:51.767 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.767 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.768 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.768 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.768 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:51.768 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:51.768 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:51.768 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.768 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 6, 2024-09-12T23:37:51.768 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.768 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.768 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:51.768 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.768 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.768 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.768 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.666010+0000", 2024-09-12T23:37:51.769 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.883028+0000", 2024-09-12T23:37:51.769 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.883028+0000", 2024-09-12T23:37:51.769 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.884106+0000", 2024-09-12T23:37:51.769 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.666010+0000", 2024-09-12T23:37:51.769 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:51.769 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.769 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:51.769 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:51.769 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.666010+0000", 2024-09-12T23:37:51.769 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.666010+0000", 2024-09-12T23:37:51.769 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.666010+0000", 2024-09-12T23:37:51.769 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.769 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.769 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:51.769 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.666010+0000", 2024-09-12T23:37:51.770 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.666010+0000", 2024-09-12T23:37:51.770 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.770 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.770 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.770 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.770 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:51.770 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.770 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.770 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.770 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.770 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.770 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.770 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.770 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.770 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.17", 2024-09-12T23:37:51.771 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.771 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.771 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.771 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 21, 2024-09-12T23:37:51.771 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.771 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T04:20:27.774655+0000", 2024-09-12T23:37:51.771 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.771 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.771 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.771 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.771 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.771 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.771 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.771 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.772 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.772 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.772 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.772 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.772 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.772 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.772 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.772 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.772 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.772 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.772 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.772 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.772 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.773 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.773 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.773 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.773 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.773 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.773 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.773 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.773 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.773 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.773 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.773 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.773 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.773 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.773 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.774 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.774 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.774 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.774 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.774 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.774 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.774 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.774 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.774 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.774 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.774 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.774 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.774 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:51.774 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:51.775 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:51.775 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.775 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 6, 2024-09-12T23:37:51.775 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.775 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.775 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.775 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.775 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:51.775 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:51.775 INFO:tasks.workunit.client.0.smithi136.stdout: 1 2024-09-12T23:37:51.775 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.775 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 5, 2024-09-12T23:37:51.775 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.776 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.776 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:51.776 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.776 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.776 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.776 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.670853+0000", 2024-09-12T23:37:51.776 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.909325+0000", 2024-09-12T23:37:51.776 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.909325+0000", 2024-09-12T23:37:51.776 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.909613+0000", 2024-09-12T23:37:51.776 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.670853+0000", 2024-09-12T23:37:51.776 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.776 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.776 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.776 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:51.777 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.670853+0000", 2024-09-12T23:37:51.777 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.670853+0000", 2024-09-12T23:37:51.777 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.670853+0000", 2024-09-12T23:37:51.777 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.777 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.777 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.777 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.670853+0000", 2024-09-12T23:37:51.777 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.670853+0000", 2024-09-12T23:37:51.777 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.777 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.777 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.777 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.777 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:51.778 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.778 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.778 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.778 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.778 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.778 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.778 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.778 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.778 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.16", 2024-09-12T23:37:51.778 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.778 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.778 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.778 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:51.779 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.779 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T06:50:21.069936+0000", 2024-09-12T23:37:51.779 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.779 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.779 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.779 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.779 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.779 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.779 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.779 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.779 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.779 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.779 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.779 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.780 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.780 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.780 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.780 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.780 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.780 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.780 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.780 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.780 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.780 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.780 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.780 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.780 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.780 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.781 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.781 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.781 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.781 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.781 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.781 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.781 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.781 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.781 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.781 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.781 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.781 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.781 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.781 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.782 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.782 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.782 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.782 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.782 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.782 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.782 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.782 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:51.782 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:51.782 INFO:tasks.workunit.client.0.smithi136.stdout: 1 2024-09-12T23:37:51.782 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.782 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 5, 2024-09-12T23:37:51.782 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.782 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.783 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.783 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.783 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:51.783 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:51.783 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:51.783 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.783 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 5, 2024-09-12T23:37:51.783 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.783 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.783 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:51.783 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.783 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.783 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.783 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.670886+0000", 2024-09-12T23:37:51.784 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.887477+0000", 2024-09-12T23:37:51.784 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.887477+0000", 2024-09-12T23:37:51.784 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.890012+0000", 2024-09-12T23:37:51.784 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.670886+0000", 2024-09-12T23:37:51.784 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:51.784 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.784 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:51.784 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:51.784 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.670886+0000", 2024-09-12T23:37:51.784 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.670886+0000", 2024-09-12T23:37:51.784 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.670886+0000", 2024-09-12T23:37:51.784 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.784 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.784 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:51.785 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.670886+0000", 2024-09-12T23:37:51.785 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.670886+0000", 2024-09-12T23:37:51.785 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.785 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.785 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.785 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.785 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:51.785 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.785 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.785 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.785 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.785 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.785 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.785 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.786 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.786 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.16", 2024-09-12T23:37:51.786 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.786 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.786 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.786 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 21, 2024-09-12T23:37:51.786 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.786 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T02:15:02.988389+0000", 2024-09-12T23:37:51.786 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.786 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.786 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.786 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.786 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.786 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.787 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.787 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.787 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.787 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.787 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.787 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.787 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.787 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.787 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.787 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.787 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.787 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.787 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.788 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.788 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.788 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.788 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.788 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.788 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.788 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.788 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.788 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.788 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.788 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.788 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.788 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.788 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.789 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.789 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.789 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.789 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.789 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.789 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.789 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.789 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.789 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.789 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.789 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.789 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.789 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.789 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.790 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:51.790 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:51.790 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:51.790 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.790 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 5, 2024-09-12T23:37:51.790 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.790 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.790 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.790 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.790 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:51.790 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:51.790 INFO:tasks.workunit.client.0.smithi136.stdout: 3 2024-09-12T23:37:51.790 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.790 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 0, 2024-09-12T23:37:51.791 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.791 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.791 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:51.791 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.791 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.791 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.791 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.670299+0000", 2024-09-12T23:37:51.791 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.916648+0000", 2024-09-12T23:37:51.791 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.916648+0000", 2024-09-12T23:37:51.791 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.917549+0000", 2024-09-12T23:37:51.791 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.670299+0000", 2024-09-12T23:37:51.791 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.792 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.792 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.792 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:51.792 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.670299+0000", 2024-09-12T23:37:51.792 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.670299+0000", 2024-09-12T23:37:51.792 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.670299+0000", 2024-09-12T23:37:51.792 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.792 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.792 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.792 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.670299+0000", 2024-09-12T23:37:51.792 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.670299+0000", 2024-09-12T23:37:51.792 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.792 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.792 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.793 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.793 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:51.793 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.793 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.793 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.793 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.793 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.793 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.793 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.793 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.793 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.17", 2024-09-12T23:37:51.793 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.793 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.793 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.794 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:51.794 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.794 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T09:37:55.708866+0000", 2024-09-12T23:37:51.794 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.794 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.794 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.794 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.794 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.794 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.794 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.794 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.794 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.794 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.794 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.795 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.795 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.795 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.795 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.795 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.795 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.795 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.795 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.795 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.795 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.795 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.795 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.795 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.795 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.796 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.796 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.796 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.796 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.796 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.796 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.796 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.796 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.796 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.796 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.796 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.796 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.796 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.796 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.797 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.797 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.797 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.797 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.797 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.797 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.797 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.797 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.797 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:51.797 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:51.797 INFO:tasks.workunit.client.0.smithi136.stdout: 3 2024-09-12T23:37:51.797 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.797 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 0, 2024-09-12T23:37:51.798 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.798 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.798 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.798 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.798 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:51.798 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:51.798 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:51.798 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.798 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 1, 2024-09-12T23:37:51.798 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.798 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.798 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:51.798 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.798 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.798 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.799 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.663816+0000", 2024-09-12T23:37:51.799 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.879399+0000", 2024-09-12T23:37:51.799 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.879399+0000", 2024-09-12T23:37:51.799 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.879732+0000", 2024-09-12T23:37:51.799 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.663816+0000", 2024-09-12T23:37:51.799 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:51.799 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.799 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:51.799 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:51.799 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.663816+0000", 2024-09-12T23:37:51.799 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.663816+0000", 2024-09-12T23:37:51.799 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.663816+0000", 2024-09-12T23:37:51.799 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.800 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.800 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:51.800 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.663816+0000", 2024-09-12T23:37:51.800 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.663816+0000", 2024-09-12T23:37:51.800 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.800 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 3, 2024-09-12T23:37:51.800 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.800 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.800 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:51.800 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.800 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.800 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.800 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.801 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 3, 2024-09-12T23:37:51.801 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.801 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.801 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.801 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.15", 2024-09-12T23:37:51.801 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.801 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.801 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.801 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 24, 2024-09-12T23:37:51.801 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.801 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T05:00:06.109352+0000", 2024-09-12T23:37:51.801 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.801 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.802 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.802 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 90, 2024-09-12T23:37:51.802 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.802 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.802 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.802 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.802 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.802 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.802 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.802 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.802 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.802 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.802 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.802 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.803 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.803 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.803 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.803 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 6, 2024-09-12T23:37:51.803 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 2, 2024-09-12T23:37:51.803 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.803 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 2, 2024-09-12T23:37:51.803 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.803 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.803 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.803 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.803 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.803 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 1, 2024-09-12T23:37:51.803 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.803 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.804 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.804 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.804 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.804 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.804 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.804 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.804 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.804 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.804 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.804 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.804 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 4, 2024-09-12T23:37:51.804 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 1 2024-09-12T23:37:51.804 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.805 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.805 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.805 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.805 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:51.805 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:51.805 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:51.805 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.805 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 1, 2024-09-12T23:37:51.805 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "53'3" 2024-09-12T23:37:51.805 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.805 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.805 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.805 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:51.806 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:51.806 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:51.806 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.806 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 2, 2024-09-12T23:37:51.806 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.806 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.806 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:51.806 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.806 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.806 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.806 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:09.684524+0000", 2024-09-12T23:37:51.806 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:35:09.683923+0000", 2024-09-12T23:37:51.806 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:35:09.683923+0000", 2024-09-12T23:37:51.806 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:35:09.684524+0000", 2024-09-12T23:37:51.807 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:09.684524+0000", 2024-09-12T23:37:51.807 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.807 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.807 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.807 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 64, 2024-09-12T23:37:51.807 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:09.684524+0000", 2024-09-12T23:37:51.807 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:09.684524+0000", 2024-09-12T23:37:51.807 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:09.684524+0000", 2024-09-12T23:37:51.807 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.807 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.807 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.807 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:09.684524+0000", 2024-09-12T23:37:51.807 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:09.684524+0000", 2024-09-12T23:37:51.807 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.807 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.808 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.808 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.808 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 63, 2024-09-12T23:37:51.808 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.808 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.808 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.808 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.808 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.808 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.808 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.808 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.808 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.14", 2024-09-12T23:37:51.808 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.809 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.809 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 64, 2024-09-12T23:37:51.809 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 10, 2024-09-12T23:37:51.809 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.809 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T06:30:40.963502+0000", 2024-09-12T23:37:51.809 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.809 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.809 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.809 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.809 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.809 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.809 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.809 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.809 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.810 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.810 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.810 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.810 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.810 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.810 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.810 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.810 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.810 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.810 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.810 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.810 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.810 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.810 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.810 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.811 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.811 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.811 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.811 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.811 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.811 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.811 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.811 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.811 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.811 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.811 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.811 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.811 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.811 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.811 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.812 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.812 INFO:tasks.workunit.client.0.smithi136.stderr:+ echo OK 2024-09-12T23:37:51.815 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:48.982944+0000 mgr.y (mgr.24446) 161 : audit [DBG] from='client.14922 -' entity='client.admin' cmd=[{"prefix": "telemetry show-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:51.815 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:49.158481+0000 mgr.y (mgr.24446) 162 : audit [DBG] from='client.14922 -' entity='client.admin' cmd=[{"prefix": "telemetry show-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:51.815 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:49.531538+0000 mgr.y (mgr.24446) 163 : audit [DBG] from='client.24842 -' entity='client.admin' cmd=[{"prefix": "telemetry show-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:51.815 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:49.533958+0000 mon.a (mon.0) 1019 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:37:51.815 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:49.539773+0000 mon.a (mon.0) 1020 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-12T23:37:51.815 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:49.708317+0000 mon.a (mon.0) 1021 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:51.815 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:49.708730+0000 mon.a (mon.0) 1022 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:51.815 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:49.709061+0000 mon.a (mon.0) 1023 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:51.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:49.712354+0000 mon.a (mon.0) 1024 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:51.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:49.712801+0000 mon.a (mon.0) 1025 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:51.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:49.713255+0000 mon.a (mon.0) 1026 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:51.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:49.716555+0000 mon.a (mon.0) 1027 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:51.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:49.717028+0000 mon.a (mon.0) 1028 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:51.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:49.717383+0000 mon.a (mon.0) 1029 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:51.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:49.734268+0000 mon.a (mon.0) 1030 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:37:51.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:49.734363+0000 mon.a (mon.0) 1031 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:37:51.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:49.734812+0000 mon.a (mon.0) 1032 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:51.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:49.735249+0000 mon.a (mon.0) 1033 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:51.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:49.735647+0000 mon.a (mon.0) 1034 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:51.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:49.738169+0000 mon.b (mon.1) 32 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:37:51.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:49.738278+0000 mon.b (mon.1) 33 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:37:51.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:49.738882+0000 mon.a (mon.0) 1035 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:51.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:49.739463+0000 mon.a (mon.0) 1036 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:51.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:49.739752+0000 mon.a (mon.0) 1037 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:51.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:49.742337+0000 mon.c (mon.2) 55 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:37:51.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:49.742433+0000 mon.c (mon.2) 56 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:37:51.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:49.742861+0000 mon.a (mon.0) 1038 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:51.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:49.743604+0000 mon.a (mon.0) 1039 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:51.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:49.744064+0000 mon.a (mon.0) 1040 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:51.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:50.017366+0000 mon.a (mon.0) 1041 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:37:51.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:50.021067+0000 mon.a (mon.0) 1042 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-12T23:37:51.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:50.155926+0000 mon.a (mon.0) 1043 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:51.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:50.156366+0000 mon.a (mon.0) 1044 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:51.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:50.156726+0000 mon.a (mon.0) 1045 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:51.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:50.159660+0000 mon.a (mon.0) 1046 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:51.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:50.160183+0000 mon.a (mon.0) 1047 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:51.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:50.160535+0000 mon.a (mon.0) 1048 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:51.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:50.162948+0000 mon.a (mon.0) 1049 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:51.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:50.163436+0000 mon.a (mon.0) 1050 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:51.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:50.163724+0000 mon.a (mon.0) 1051 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:51.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:50.178285+0000 mon.a (mon.0) 1052 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:37:51.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:50.178336+0000 mon.a (mon.0) 1053 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:37:51.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:50.178736+0000 mon.a (mon.0) 1054 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:51.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:50.179150+0000 mon.a (mon.0) 1055 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:51.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:50.179483+0000 mon.a (mon.0) 1056 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:51.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:50.181858+0000 mon.b (mon.1) 34 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:37:51.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:50.181963+0000 mon.b (mon.1) 35 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:37:51.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:50.182585+0000 mon.a (mon.0) 1057 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:51.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:50.183201+0000 mon.a (mon.0) 1058 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:51.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:50.183535+0000 mon.a (mon.0) 1059 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:51.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:50.186058+0000 mon.c (mon.2) 57 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:37:51.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:50.186168+0000 mon.c (mon.2) 58 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:37:51.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:50.186423+0000 mon.a (mon.0) 1060 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:51.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:50.186984+0000 mon.a (mon.0) 1061 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:51.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:50 smithi181 bash[24776]: audit 2024-09-12T23:37:50.187252+0000 mon.a (mon.0) 1062 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:51.820 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.820 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.820 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.820 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.820 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.820 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.820 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.820 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:51.820 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:51.820 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:51.821 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.821 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 2, 2024-09-12T23:37:51.821 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.821 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.821 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.821 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.821 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:51.821 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:51.821 INFO:tasks.workunit.client.0.smithi136.stdout: 5 2024-09-12T23:37:51.821 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.821 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 6, 2024-09-12T23:37:51.821 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.821 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.821 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:51.822 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.822 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.822 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.822 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:38.071488+0000", 2024-09-12T23:37:51.822 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.885991+0000", 2024-09-12T23:37:51.822 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.885991+0000", 2024-09-12T23:37:51.822 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.886930+0000", 2024-09-12T23:37:51.822 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:38.071488+0000", 2024-09-12T23:37:51.822 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:51.822 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.822 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:51.822 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:51.822 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:38.071488+0000", 2024-09-12T23:37:51.823 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:38.071488+0000", 2024-09-12T23:37:51.823 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:38.071488+0000", 2024-09-12T23:37:51.823 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.823 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.823 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:51.823 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:38.071488+0000", 2024-09-12T23:37:51.823 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:38.071488+0000", 2024-09-12T23:37:51.823 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.823 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 12, 2024-09-12T23:37:51.823 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.823 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.823 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:51.823 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.824 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.824 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.824 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.824 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 12, 2024-09-12T23:37:51.824 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.824 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.824 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.824 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.14", 2024-09-12T23:37:51.824 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.824 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.824 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 64, 2024-09-12T23:37:51.824 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 33, 2024-09-12T23:37:51.824 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.825 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T09:50:58.477301+0000", 2024-09-12T23:37:51.825 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.825 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.825 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.825 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 2624, 2024-09-12T23:37:51.825 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.825 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.825 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.825 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.825 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.825 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.825 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.826 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.826 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.826 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.826 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.826 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.826 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.826 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.826 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.826 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 12, 2024-09-12T23:37:51.826 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 4, 2024-09-12T23:37:51.826 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.826 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 4, 2024-09-12T23:37:51.826 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.826 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.827 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.827 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.827 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.827 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 1, 2024-09-12T23:37:51.827 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.827 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.827 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.827 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.827 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.827 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.827 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.827 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.827 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.827 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.828 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.828 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.828 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 12, 2024-09-12T23:37:51.828 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 9 2024-09-12T23:37:51.828 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.828 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.828 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.828 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.828 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:51.828 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:51.828 INFO:tasks.workunit.client.0.smithi136.stdout: 5 2024-09-12T23:37:51.828 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.828 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 6, 2024-09-12T23:37:51.829 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "64'12" 2024-09-12T23:37:51.829 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.829 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.829 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.829 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:51.829 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:51.829 INFO:tasks.workunit.client.0.smithi136.stdout: 4 2024-09-12T23:37:51.829 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.829 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 7, 2024-09-12T23:37:51.829 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.829 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.829 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:51.829 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.830 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.830 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.830 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.665823+0000", 2024-09-12T23:37:51.830 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.916622+0000", 2024-09-12T23:37:51.830 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.916622+0000", 2024-09-12T23:37:51.830 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.916929+0000", 2024-09-12T23:37:51.830 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.665823+0000", 2024-09-12T23:37:51.830 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.830 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.830 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.830 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:51.830 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.665823+0000", 2024-09-12T23:37:51.830 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.665823+0000", 2024-09-12T23:37:51.830 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.665823+0000", 2024-09-12T23:37:51.831 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.831 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.831 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.831 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.665823+0000", 2024-09-12T23:37:51.831 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.665823+0000", 2024-09-12T23:37:51.831 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.831 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.831 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.831 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.831 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:51.831 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.831 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.831 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.831 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.832 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.832 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.832 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.832 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.832 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.15", 2024-09-12T23:37:51.832 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.832 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.832 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.832 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:51.832 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.832 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T08:40:09.433278+0000", 2024-09-12T23:37:51.832 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.832 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.833 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.833 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.833 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.833 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.833 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.833 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.833 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.833 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.833 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.833 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.833 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.833 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.833 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.833 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.834 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.834 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.834 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.834 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.834 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.834 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.834 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.834 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.834 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.834 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.834 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.834 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.834 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.834 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.835 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.835 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.835 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.835 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.835 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.835 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.835 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.835 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.835 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.835 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.835 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.835 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.835 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.836 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.836 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.836 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.836 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.836 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:51.836 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:51.836 INFO:tasks.workunit.client.0.smithi136.stdout: 4 2024-09-12T23:37:51.836 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.836 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 7, 2024-09-12T23:37:51.836 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.836 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.836 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.836 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.836 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:51.837 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:51.837 INFO:tasks.workunit.client.0.smithi136.stdout: 3 2024-09-12T23:37:51.837 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.837 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 0, 2024-09-12T23:37:51.837 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.837 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.837 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:51.837 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.837 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.837 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.837 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.669915+0000", 2024-09-12T23:37:51.837 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.879825+0000", 2024-09-12T23:37:51.838 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.879825+0000", 2024-09-12T23:37:51.838 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.880385+0000", 2024-09-12T23:37:51.838 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.669915+0000", 2024-09-12T23:37:51.838 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:51.838 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.838 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:51.838 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:51.838 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.669915+0000", 2024-09-12T23:37:51.838 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.669915+0000", 2024-09-12T23:37:51.838 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.669915+0000", 2024-09-12T23:37:51.838 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.838 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.838 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:51.838 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.669915+0000", 2024-09-12T23:37:51.838 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.669915+0000", 2024-09-12T23:37:51.839 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.839 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 2, 2024-09-12T23:37:51.839 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.839 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.839 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:51.839 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.839 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.839 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.839 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.839 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 2, 2024-09-12T23:37:51.839 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.839 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.839 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.839 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.13", 2024-09-12T23:37:51.840 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.840 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.840 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.840 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 23, 2024-09-12T23:37:51.840 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.840 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T02:40:33.073593+0000", 2024-09-12T23:37:51.840 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.840 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.840 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.840 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 34, 2024-09-12T23:37:51.840 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.840 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.840 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.840 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.841 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.841 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.841 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.841 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.841 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.841 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.841 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.841 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.841 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.841 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.841 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.841 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 6, 2024-09-12T23:37:51.841 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 2, 2024-09-12T23:37:51.842 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.842 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 2, 2024-09-12T23:37:51.842 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.842 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.842 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.842 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.842 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.842 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 1, 2024-09-12T23:37:51.842 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.842 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.842 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.842 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.842 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.843 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.843 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.843 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.843 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.843 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.843 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.843 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.843 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 3, 2024-09-12T23:37:51.843 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 6 2024-09-12T23:37:51.843 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.843 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.843 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.843 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.843 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:51.844 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:51.844 INFO:tasks.workunit.client.0.smithi136.stdout: 3 2024-09-12T23:37:51.844 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.844 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 0, 2024-09-12T23:37:51.844 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "53'2" 2024-09-12T23:37:51.844 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.844 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.844 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.844 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:51.844 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:51.844 INFO:tasks.workunit.client.0.smithi136.stdout: 3 2024-09-12T23:37:51.844 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.844 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 0, 2024-09-12T23:37:51.844 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.845 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.845 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:51.845 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.845 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.845 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.845 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.670434+0000", 2024-09-12T23:37:51.845 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.918580+0000", 2024-09-12T23:37:51.845 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.918580+0000", 2024-09-12T23:37:51.845 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.919045+0000", 2024-09-12T23:37:51.845 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.670434+0000", 2024-09-12T23:37:51.845 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.845 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.845 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.846 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:51.846 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.670434+0000", 2024-09-12T23:37:51.846 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.670434+0000", 2024-09-12T23:37:51.846 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.670434+0000", 2024-09-12T23:37:51.846 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.846 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.846 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.846 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.670434+0000", 2024-09-12T23:37:51.846 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.670434+0000", 2024-09-12T23:37:51.846 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.846 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.846 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.846 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.846 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:51.847 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.847 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.847 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.847 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.847 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.847 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.847 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.847 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.847 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.12", 2024-09-12T23:37:51.847 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.847 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.847 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.847 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:51.847 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.848 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T01:39:32.320667+0000", 2024-09-12T23:37:51.848 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.848 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.848 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.848 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.848 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.848 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.848 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.848 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.848 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.848 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.848 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.848 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.849 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.849 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.849 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.849 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.849 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.849 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.849 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.849 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.849 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.849 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.849 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.849 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.849 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.849 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.850 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.850 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.850 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.850 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.850 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.850 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.850 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.850 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.850 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.850 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.850 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.850 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.850 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.850 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.851 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.851 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.851 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.851 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.851 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.851 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.851 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.851 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:51.851 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:51.851 INFO:tasks.workunit.client.0.smithi136.stdout: 3 2024-09-12T23:37:51.851 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.851 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 0, 2024-09-12T23:37:51.851 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.851 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.852 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.852 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.852 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:51.852 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:51.852 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:51.852 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.852 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 5, 2024-09-12T23:37:51.852 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.852 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.852 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:51.852 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.852 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.852 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.853 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.671158+0000", 2024-09-12T23:37:51.853 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.898942+0000", 2024-09-12T23:37:51.853 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.898942+0000", 2024-09-12T23:37:51.853 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.899424+0000", 2024-09-12T23:37:51.853 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.671158+0000", 2024-09-12T23:37:51.853 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:51.853 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.853 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:51.853 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:51.853 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.671158+0000", 2024-09-12T23:37:51.853 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.671158+0000", 2024-09-12T23:37:51.853 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.671158+0000", 2024-09-12T23:37:51.853 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.854 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.854 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:51.854 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.671158+0000", 2024-09-12T23:37:51.854 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.671158+0000", 2024-09-12T23:37:51.854 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.854 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.854 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.854 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.854 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:51.854 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.854 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.854 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.854 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.855 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.855 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.855 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.855 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.855 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.12", 2024-09-12T23:37:51.855 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.855 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.855 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.855 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 21, 2024-09-12T23:37:51.855 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.855 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T03:14:47.069896+0000", 2024-09-12T23:37:51.855 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.855 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.855 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.856 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.856 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.856 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.856 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.856 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.856 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.856 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.856 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.856 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.856 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.856 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.856 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.856 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.857 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.857 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.857 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.857 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.857 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.857 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.857 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.857 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.857 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.857 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.857 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.857 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.857 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.857 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.858 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.858 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.858 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.858 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.858 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.858 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.858 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.858 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.858 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.858 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.858 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.858 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.858 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.859 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.859 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.859 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.859 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.859 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:51.859 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:51.859 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:51.859 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.859 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 5, 2024-09-12T23:37:51.859 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.859 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.859 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.859 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.859 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:51.859 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:51.860 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:51.860 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.860 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 7, 2024-09-12T23:37:51.860 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.860 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.860 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:51.860 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.860 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.860 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.860 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.663898+0000", 2024-09-12T23:37:51.860 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.901060+0000", 2024-09-12T23:37:51.860 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.901060+0000", 2024-09-12T23:37:51.860 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.901397+0000", 2024-09-12T23:37:51.861 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.663898+0000", 2024-09-12T23:37:51.861 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.861 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.861 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.861 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:51.861 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.663898+0000", 2024-09-12T23:37:51.861 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.663898+0000", 2024-09-12T23:37:51.861 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.663898+0000", 2024-09-12T23:37:51.861 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.861 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.861 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.861 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.663898+0000", 2024-09-12T23:37:51.861 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.663898+0000", 2024-09-12T23:37:51.861 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.862 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.862 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.862 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.862 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:51.862 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.862 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.862 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.862 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.862 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.862 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.862 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.862 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.862 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.13", 2024-09-12T23:37:51.863 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.863 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.863 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.863 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:51.863 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.863 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T11:21:23.285498+0000", 2024-09-12T23:37:51.863 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.863 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.863 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.863 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.863 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.863 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.863 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.863 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.864 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.864 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.864 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.864 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.864 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.864 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.864 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.864 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.864 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.864 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.864 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.864 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.864 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.864 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.865 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.865 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.865 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.865 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.865 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.865 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.865 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.865 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.865 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.865 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.865 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.865 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.865 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.866 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.866 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.866 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.866 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.866 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.866 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.866 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.866 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.866 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.866 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.866 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.866 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.866 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:51.866 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:51.867 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:51.867 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.867 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 7, 2024-09-12T23:37:51.867 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.867 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.867 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.867 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.867 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:51.867 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:51.867 INFO:tasks.workunit.client.0.smithi136.stdout: 1 2024-09-12T23:37:51.867 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.867 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 6, 2024-09-12T23:37:51.867 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.867 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.868 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:51.868 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.868 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.868 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.868 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.665621+0000", 2024-09-12T23:37:51.868 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.869528+0000", 2024-09-12T23:37:51.868 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.869528+0000", 2024-09-12T23:37:51.868 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.871430+0000", 2024-09-12T23:37:51.868 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.665621+0000", 2024-09-12T23:37:51.868 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:51.868 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.868 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:51.868 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:51.869 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.665621+0000", 2024-09-12T23:37:51.869 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.665621+0000", 2024-09-12T23:37:51.869 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.665621+0000", 2024-09-12T23:37:51.869 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.869 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.869 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:51.869 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.665621+0000", 2024-09-12T23:37:51.869 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.665621+0000", 2024-09-12T23:37:51.869 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.869 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.869 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.869 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.869 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:51.869 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.870 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.870 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.870 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.870 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.870 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.870 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.870 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.870 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.11", 2024-09-12T23:37:51.870 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.870 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.870 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.870 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 21, 2024-09-12T23:37:51.870 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.870 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T11:27:43.449941+0000", 2024-09-12T23:37:51.871 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.871 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.871 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.871 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.871 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.871 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.871 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.871 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.871 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.871 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.871 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.871 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.871 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.871 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.872 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.872 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.872 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.872 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.872 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.872 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.872 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.872 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.872 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.872 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.872 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.872 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.872 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.873 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.873 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.873 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.873 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.873 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.873 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.873 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.873 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.873 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.873 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.873 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.873 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.873 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.873 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.874 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.874 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.874 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.874 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.874 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.874 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.874 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:51.874 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:51.874 INFO:tasks.workunit.client.0.smithi136.stdout: 1 2024-09-12T23:37:51.874 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.874 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 6, 2024-09-12T23:37:51.874 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.874 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.874 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.875 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.875 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:51.875 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:51.875 INFO:tasks.workunit.client.0.smithi136.stdout: 5 2024-09-12T23:37:51.875 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.875 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 6, 2024-09-12T23:37:51.875 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.875 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.875 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:51.875 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.875 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.875 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.875 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:09.683003+0000", 2024-09-12T23:37:51.875 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:35:09.681812+0000", 2024-09-12T23:37:51.875 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:35:09.681812+0000", 2024-09-12T23:37:51.876 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:35:09.683003+0000", 2024-09-12T23:37:51.876 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:09.683003+0000", 2024-09-12T23:37:51.876 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.876 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.876 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.876 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 64, 2024-09-12T23:37:51.876 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:09.683003+0000", 2024-09-12T23:37:51.876 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:09.683003+0000", 2024-09-12T23:37:51.876 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:09.683003+0000", 2024-09-12T23:37:51.876 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.876 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.876 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.877 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:09.683003+0000", 2024-09-12T23:37:51.877 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:09.683003+0000", 2024-09-12T23:37:51.877 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.877 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.877 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.877 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.877 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 63, 2024-09-12T23:37:51.877 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.877 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.877 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.877 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.877 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.877 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.878 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.878 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.878 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.10", 2024-09-12T23:37:51.878 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.878 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.878 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 64, 2024-09-12T23:37:51.878 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 30, 2024-09-12T23:37:51.878 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.878 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-13T23:47:15.002852+0000", 2024-09-12T23:37:51.878 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.878 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.878 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.878 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.878 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.879 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.879 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.879 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.879 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.879 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.879 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.879 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.879 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.879 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.879 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.879 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.879 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.879 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.879 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.880 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.880 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.880 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.880 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.880 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.880 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.880 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.880 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.880 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.880 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.880 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.880 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.880 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.880 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.881 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.881 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.881 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.881 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.881 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.881 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.881 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.881 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.881 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.881 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.881 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.881 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.881 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.882 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.882 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:51.882 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:51.882 INFO:tasks.workunit.client.0.smithi136.stdout: 5 2024-09-12T23:37:51.882 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.882 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 6, 2024-09-12T23:37:51.882 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.882 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.882 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.882 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.882 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:51.882 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:51.882 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:51.882 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.883 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 2, 2024-09-12T23:37:51.883 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.883 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.883 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:51.883 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.883 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.883 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.883 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.661383+0000", 2024-09-12T23:37:51.883 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.863479+0000", 2024-09-12T23:37:51.884 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.863479+0000", 2024-09-12T23:37:51.884 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.863788+0000", 2024-09-12T23:37:51.884 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.661383+0000", 2024-09-12T23:37:51.884 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:51.884 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.884 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:51.884 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:51.884 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.661383+0000", 2024-09-12T23:37:51.884 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.661383+0000", 2024-09-12T23:37:51.884 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.661383+0000", 2024-09-12T23:37:51.884 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.885 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.885 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:51.885 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.661383+0000", 2024-09-12T23:37:51.885 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.661383+0000", 2024-09-12T23:37:51.885 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.885 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 8, 2024-09-12T23:37:51.885 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.885 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.885 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:51.885 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.886 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.886 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.886 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.886 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 8, 2024-09-12T23:37:51.886 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.886 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.886 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.886 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.10", 2024-09-12T23:37:51.886 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.886 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.886 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.887 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 29, 2024-09-12T23:37:51.887 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.887 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T08:26:09.204038+0000", 2024-09-12T23:37:51.887 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.887 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.887 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.887 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.887 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.887 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.887 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.887 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.888 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.888 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.888 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.888 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.888 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.888 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.888 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.888 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.888 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.888 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.888 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.888 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 12, 2024-09-12T23:37:51.889 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 4, 2024-09-12T23:37:51.889 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.889 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 4, 2024-09-12T23:37:51.889 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.889 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.889 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.889 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.889 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.889 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 4, 2024-09-12T23:37:51.889 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.889 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.890 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.890 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.890 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.890 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.890 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.890 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.890 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.890 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.890 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.890 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.890 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 12, 2024-09-12T23:37:51.890 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.891 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.891 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.891 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.891 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.891 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:51.891 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:51.891 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:51.891 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.891 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 2, 2024-09-12T23:37:51.891 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "53'8" 2024-09-12T23:37:51.891 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.892 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.892 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.892 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:51.892 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:51.892 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:51.892 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.892 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 7, 2024-09-12T23:37:51.892 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.892 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.892 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:51.892 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.893 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.893 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.893 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.664571+0000", 2024-09-12T23:37:51.893 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.901803+0000", 2024-09-12T23:37:51.893 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.901803+0000", 2024-09-12T23:37:51.893 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.902101+0000", 2024-09-12T23:37:51.893 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.664571+0000", 2024-09-12T23:37:51.893 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.893 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.893 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.893 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:51.894 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.664571+0000", 2024-09-12T23:37:51.894 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.664571+0000", 2024-09-12T23:37:51.894 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.664571+0000", 2024-09-12T23:37:51.894 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.894 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.894 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.894 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.664571+0000", 2024-09-12T23:37:51.894 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.664571+0000", 2024-09-12T23:37:51.894 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.894 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.894 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.895 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.895 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:51.895 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.895 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.895 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.895 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.895 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.895 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.895 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.895 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.896 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.11", 2024-09-12T23:37:51.896 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.896 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.896 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.896 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:51.896 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.896 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T02:00:48.557318+0000", 2024-09-12T23:37:51.896 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.896 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.896 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.896 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.896 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.897 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.897 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.897 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.897 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.897 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.897 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.897 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.897 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.897 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.897 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.897 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.898 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.898 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.898 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.898 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.898 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.898 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.898 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.898 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.898 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.898 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.898 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.898 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.899 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.899 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.899 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.899 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.899 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.899 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.899 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.899 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.899 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.899 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.899 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.900 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.900 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.900 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.900 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.900 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.900 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.900 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.900 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.900 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:51.900 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:51.900 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:51.901 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.901 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 7, 2024-09-12T23:37:51.901 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.901 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.901 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.901 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.901 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:51.901 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:51.901 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:51.901 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.902 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 4, 2024-09-12T23:37:51.902 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.902 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.902 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:51.902 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.902 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.902 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.902 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.662210+0000", 2024-09-12T23:37:51.902 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.877913+0000", 2024-09-12T23:37:51.902 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.877913+0000", 2024-09-12T23:37:51.902 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.879318+0000", 2024-09-12T23:37:51.903 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.662210+0000", 2024-09-12T23:37:51.903 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:51.903 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.903 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:51.903 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:51.903 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.662210+0000", 2024-09-12T23:37:51.903 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.662210+0000", 2024-09-12T23:37:51.903 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.662210+0000", 2024-09-12T23:37:51.903 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.903 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.903 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:51.903 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.662210+0000", 2024-09-12T23:37:51.904 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.662210+0000", 2024-09-12T23:37:51.904 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.904 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 1, 2024-09-12T23:37:51.904 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.904 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.904 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:51.904 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.904 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.904 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.904 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.904 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 1, 2024-09-12T23:37:51.905 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.905 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.905 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.905 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.f", 2024-09-12T23:37:51.905 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.905 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.905 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.905 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 22, 2024-09-12T23:37:51.905 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.905 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T10:13:19.574352+0000", 2024-09-12T23:37:51.905 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.906 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.906 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.906 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 22, 2024-09-12T23:37:51.906 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.906 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.906 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.906 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.906 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.906 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.906 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.906 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.906 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.907 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.907 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.907 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.907 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.907 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.907 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.907 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 3, 2024-09-12T23:37:51.907 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 1, 2024-09-12T23:37:51.907 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.907 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 1, 2024-09-12T23:37:51.907 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.907 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.908 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.908 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.908 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.908 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.908 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.908 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.908 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.908 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.908 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.908 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.908 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.909 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.909 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.909 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.909 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.909 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.909 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 1, 2024-09-12T23:37:51.909 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 1 2024-09-12T23:37:51.909 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.909 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.909 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.909 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.910 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:51.910 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:51.910 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:51.910 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.910 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 4, 2024-09-12T23:37:51.910 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "53'1" 2024-09-12T23:37:51.910 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.910 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.910 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.910 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:51.910 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:51.910 INFO:tasks.workunit.client.0.smithi136.stdout: 1 2024-09-12T23:37:51.911 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.911 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 7, 2024-09-12T23:37:51.911 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.911 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.911 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:51.911 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.911 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.911 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.911 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.664646+0000", 2024-09-12T23:37:51.911 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.901052+0000", 2024-09-12T23:37:51.911 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.901052+0000", 2024-09-12T23:37:51.911 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.901391+0000", 2024-09-12T23:37:51.912 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.664646+0000", 2024-09-12T23:37:51.912 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.912 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.912 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.912 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:51.912 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.664646+0000", 2024-09-12T23:37:51.912 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.664646+0000", 2024-09-12T23:37:51.912 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.664646+0000", 2024-09-12T23:37:51.912 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.912 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.912 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.913 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.664646+0000", 2024-09-12T23:37:51.913 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.664646+0000", 2024-09-12T23:37:51.913 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.913 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.913 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.913 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.913 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:51.913 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.913 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.913 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.913 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.914 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.914 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.914 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.914 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.914 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.e", 2024-09-12T23:37:51.914 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.914 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.914 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.914 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:51.914 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.914 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T04:54:04.075006+0000", 2024-09-12T23:37:51.915 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.915 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.915 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.915 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.915 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.915 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.915 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.915 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.915 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.915 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.915 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.915 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.916 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.916 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.916 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.916 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.916 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.916 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.916 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.916 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.916 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.916 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.916 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.917 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.917 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.917 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.917 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.917 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.917 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.917 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.917 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.917 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.917 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.917 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.918 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.918 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.918 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.918 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.918 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.918 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.918 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.918 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.918 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.918 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.918 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.918 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.919 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.919 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:51.919 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:51.919 INFO:tasks.workunit.client.0.smithi136.stdout: 1 2024-09-12T23:37:51.919 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.919 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 7, 2024-09-12T23:37:51.919 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.919 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.919 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.919 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.919 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:51.920 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:51.920 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:51.920 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.920 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 2, 2024-09-12T23:37:51.920 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.920 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.920 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:51.920 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.920 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.920 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.920 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.661659+0000", 2024-09-12T23:37:51.921 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.887109+0000", 2024-09-12T23:37:51.921 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.887109+0000", 2024-09-12T23:37:51.921 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.887441+0000", 2024-09-12T23:37:51.921 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.661659+0000", 2024-09-12T23:37:51.921 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:51.921 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.921 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:51.921 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:51.921 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.661659+0000", 2024-09-12T23:37:51.921 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.661659+0000", 2024-09-12T23:37:51.921 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.661659+0000", 2024-09-12T23:37:51.922 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.922 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.922 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:51.922 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.661659+0000", 2024-09-12T23:37:51.922 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.661659+0000", 2024-09-12T23:37:51.922 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.922 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 2, 2024-09-12T23:37:51.922 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.922 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.922 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:51.922 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.923 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.923 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.923 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.923 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 2, 2024-09-12T23:37:51.923 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.923 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.923 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.923 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.e", 2024-09-12T23:37:51.923 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.923 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.923 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.924 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 23, 2024-09-12T23:37:51.924 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.924 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T07:35:14.071342+0000", 2024-09-12T23:37:51.924 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.924 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.924 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.924 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.924 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.924 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.924 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.924 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.924 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.925 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.925 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.925 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.925 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.925 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.925 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.925 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.925 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.925 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.925 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.925 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 3, 2024-09-12T23:37:51.925 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 1, 2024-09-12T23:37:51.926 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.926 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 1, 2024-09-12T23:37:51.926 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.926 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.926 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.926 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.926 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.926 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 1, 2024-09-12T23:37:51.926 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.926 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.926 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.927 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.927 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.927 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.927 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.927 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.927 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.927 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.927 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.927 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.927 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 3, 2024-09-12T23:37:51.927 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.927 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.928 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.928 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.928 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.928 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:51.928 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:51.928 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:51.928 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.928 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 2, 2024-09-12T23:37:51.928 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "53'2" 2024-09-12T23:37:51.928 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.928 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.929 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.929 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:51.929 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:51.929 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:51.929 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.929 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 7, 2024-09-12T23:37:51.929 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.929 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.929 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:51.929 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.929 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.929 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.930 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.665111+0000", 2024-09-12T23:37:51.930 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.907302+0000", 2024-09-12T23:37:51.930 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.907302+0000", 2024-09-12T23:37:51.930 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.907594+0000", 2024-09-12T23:37:51.930 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.665111+0000", 2024-09-12T23:37:51.930 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.930 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.930 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.930 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:51.930 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.665111+0000", 2024-09-12T23:37:51.930 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.665111+0000", 2024-09-12T23:37:51.931 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.665111+0000", 2024-09-12T23:37:51.931 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.931 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.931 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.931 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.665111+0000", 2024-09-12T23:37:51.931 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.665111+0000", 2024-09-12T23:37:51.931 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.931 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.931 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.931 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.931 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:51.931 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.932 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.932 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.932 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.932 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.932 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.932 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.932 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.932 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.f", 2024-09-12T23:37:51.932 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.932 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.932 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.932 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:51.933 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.933 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T05:45:37.988381+0000", 2024-09-12T23:37:51.933 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.933 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.933 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.933 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.933 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.933 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.933 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.933 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.933 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.933 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.934 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.934 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.934 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.934 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.934 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.934 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.934 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.934 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.934 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.934 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.934 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.934 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.935 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.935 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.935 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.935 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.935 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.935 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.935 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.935 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.935 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.935 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.935 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.936 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.936 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.936 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.936 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.936 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.936 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.936 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.936 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.936 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.936 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.936 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.937 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.937 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.937 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.937 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:51.937 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:51.937 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:51.937 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.937 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 7, 2024-09-12T23:37:51.937 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.937 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.937 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.938 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.938 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:51.938 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:51.938 INFO:tasks.workunit.client.0.smithi136.stdout: 3 2024-09-12T23:37:51.938 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.938 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 1, 2024-09-12T23:37:51.938 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.938 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.938 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:51.938 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.938 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.939 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.939 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.663915+0000", 2024-09-12T23:37:51.939 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.871628+0000", 2024-09-12T23:37:51.939 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.871628+0000", 2024-09-12T23:37:51.939 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.871944+0000", 2024-09-12T23:37:51.939 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.663915+0000", 2024-09-12T23:37:51.939 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:51.939 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.939 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:51.939 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:51.939 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.663915+0000", 2024-09-12T23:37:51.940 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.663915+0000", 2024-09-12T23:37:51.940 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.663915+0000", 2024-09-12T23:37:51.940 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.940 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.940 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:51.940 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.663915+0000", 2024-09-12T23:37:51.940 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.663915+0000", 2024-09-12T23:37:51.940 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.940 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 1, 2024-09-12T23:37:51.940 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.940 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.941 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:51.941 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.941 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.941 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.941 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.941 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 1, 2024-09-12T23:37:51.941 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.941 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.941 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.941 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.d", 2024-09-12T23:37:51.941 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.942 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.942 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.942 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 22, 2024-09-12T23:37:51.942 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.942 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T09:58:29.645624+0000", 2024-09-12T23:37:51.942 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.942 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.942 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.942 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 70, 2024-09-12T23:37:51.942 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.942 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.942 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.943 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.943 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.943 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.943 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.943 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.943 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.943 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.943 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.943 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.943 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.943 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.944 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.944 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 3, 2024-09-12T23:37:51.944 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 1, 2024-09-12T23:37:51.944 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.944 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 1, 2024-09-12T23:37:51.944 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.944 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.944 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.944 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.944 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.944 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.945 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.945 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.945 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.945 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.945 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.945 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.945 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.945 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.945 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.945 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.946 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.946 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.946 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 1, 2024-09-12T23:37:51.946 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 1 2024-09-12T23:37:51.946 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.946 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.946 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.946 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.946 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:51.946 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:51.946 INFO:tasks.workunit.client.0.smithi136.stdout: 3 2024-09-12T23:37:51.946 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.947 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 1, 2024-09-12T23:37:51.947 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "53'1" 2024-09-12T23:37:51.947 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.947 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.947 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.947 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:51.947 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:51.947 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:51.947 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.947 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 5, 2024-09-12T23:37:51.947 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.948 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.948 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:51.948 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.948 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.948 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.948 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.670659+0000", 2024-09-12T23:37:51.948 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.914416+0000", 2024-09-12T23:37:51.948 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.914416+0000", 2024-09-12T23:37:51.948 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.914833+0000", 2024-09-12T23:37:51.948 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.670659+0000", 2024-09-12T23:37:51.948 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.949 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.949 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.949 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:51.949 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.670659+0000", 2024-09-12T23:37:51.949 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.670659+0000", 2024-09-12T23:37:51.949 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.670659+0000", 2024-09-12T23:37:51.949 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.949 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.949 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.949 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.670659+0000", 2024-09-12T23:37:51.949 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.670659+0000", 2024-09-12T23:37:51.950 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.950 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.950 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.950 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.950 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:51.950 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.950 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.950 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.950 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.950 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.950 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.950 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.951 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.951 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.c", 2024-09-12T23:37:51.951 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.951 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.951 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.951 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:51.951 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.951 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T09:27:20.253954+0000", 2024-09-12T23:37:51.951 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.951 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.951 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.952 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.952 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.952 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.952 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.952 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.952 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.952 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.952 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.952 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.952 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.952 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.953 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.953 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.953 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.953 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.953 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.953 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.953 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.953 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.953 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.953 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.953 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.954 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.954 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.954 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.954 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.954 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.954 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.954 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.954 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.954 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.954 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.954 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.954 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.955 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.955 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.955 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.955 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.955 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.955 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.955 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.955 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.955 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.955 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.955 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:51.955 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:51.956 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:51.956 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.956 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 5, 2024-09-12T23:37:51.956 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.956 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.956 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.956 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.956 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:51.956 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:51.956 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:51.956 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.957 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 2, 2024-09-12T23:37:51.957 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.957 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.957 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:51.957 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.957 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.957 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.957 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.662019+0000", 2024-09-12T23:37:51.957 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.881774+0000", 2024-09-12T23:37:51.957 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.881774+0000", 2024-09-12T23:37:51.957 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.882138+0000", 2024-09-12T23:37:51.957 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.662019+0000", 2024-09-12T23:37:51.958 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:51.958 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.958 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:51.958 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:51.958 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.662019+0000", 2024-09-12T23:37:51.958 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.662019+0000", 2024-09-12T23:37:51.958 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.662019+0000", 2024-09-12T23:37:51.958 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.958 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.958 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:51.958 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.662019+0000", 2024-09-12T23:37:51.958 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.662019+0000", 2024-09-12T23:37:51.959 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.959 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.959 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.959 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.959 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:51.959 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.959 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.959 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.959 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.959 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.960 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.960 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.960 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.960 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.c", 2024-09-12T23:37:51.960 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.960 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.960 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.960 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 21, 2024-09-12T23:37:51.960 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.960 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T05:07:36.075590+0000", 2024-09-12T23:37:51.960 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.961 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.961 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.961 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.961 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.961 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.961 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.961 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.961 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.961 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.961 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.961 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.962 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.962 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.962 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.962 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.962 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.962 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.962 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.962 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.962 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.962 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.963 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.963 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.963 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.963 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.963 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.963 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.963 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.963 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.963 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.963 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.963 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.964 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.964 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.964 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.964 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.964 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.964 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.964 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.964 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.964 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.964 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.964 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.965 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.965 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.965 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.965 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:51.965 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:51.965 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:51.965 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.965 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 2, 2024-09-12T23:37:51.965 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.965 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.965 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.966 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.966 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:51.966 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:51.966 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:51.966 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.966 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 7, 2024-09-12T23:37:51.968 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.968 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.968 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:51.968 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.968 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.968 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.968 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.665947+0000", 2024-09-12T23:37:51.968 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.911244+0000", 2024-09-12T23:37:51.968 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.911244+0000", 2024-09-12T23:37:51.968 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.911549+0000", 2024-09-12T23:37:51.968 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.665947+0000", 2024-09-12T23:37:51.969 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.969 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.969 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.969 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:51.969 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.665947+0000", 2024-09-12T23:37:51.969 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.665947+0000", 2024-09-12T23:37:51.969 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.665947+0000", 2024-09-12T23:37:51.969 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.969 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.969 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:51.969 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.665947+0000", 2024-09-12T23:37:51.970 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.665947+0000", 2024-09-12T23:37:51.970 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.970 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.970 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.970 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.970 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:51.970 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.970 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.970 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.970 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.970 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.971 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.971 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.971 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.971 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.d", 2024-09-12T23:37:51.971 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.971 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.971 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.971 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:51.971 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.971 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T11:22:20.405567+0000", 2024-09-12T23:37:51.971 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.971 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.972 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.972 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.972 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.972 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.972 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.972 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.972 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.972 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.972 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.972 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.972 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.973 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.973 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.973 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.973 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.973 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.973 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.973 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.973 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.973 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.973 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.973 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.973 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.974 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.974 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.974 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.974 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.974 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.974 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.974 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.974 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.974 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.974 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.974 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.974 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.975 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.975 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.975 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.975 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.975 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.975 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.975 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.975 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.975 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.975 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.975 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:51.976 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:51.976 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:51.976 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.976 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 7, 2024-09-12T23:37:51.976 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.976 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.976 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.976 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.976 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:51.976 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:51.976 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:51.976 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.977 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 6, 2024-09-12T23:37:51.977 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.977 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.977 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 56, 2024-09-12T23:37:51.977 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.977 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.977 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.977 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.666935+0000", 2024-09-12T23:37:51.977 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:34:49.734724+0000", 2024-09-12T23:37:51.977 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:34:49.734724+0000", 2024-09-12T23:37:51.977 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:34:59.598602+0000", 2024-09-12T23:37:51.978 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.666935+0000", 2024-09-12T23:37:51.978 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:34:48.454503+0000", 2024-09-12T23:37:51.978 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.978 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:34:48.454503+0000", 2024-09-12T23:37:51.978 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 57, 2024-09-12T23:37:51.978 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.666935+0000", 2024-09-12T23:37:51.978 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.666935+0000", 2024-09-12T23:37:51.978 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.666935+0000", 2024-09-12T23:37:51.978 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.978 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.978 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:34:48.454503+0000", 2024-09-12T23:37:51.979 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.666935+0000", 2024-09-12T23:37:51.979 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.666935+0000", 2024-09-12T23:37:51.979 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.979 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.979 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.979 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.979 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 56, 2024-09-12T23:37:51.979 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.979 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.979 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.980 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.980 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.980 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.980 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.980 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.980 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "4.5", 2024-09-12T23:37:51.980 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.980 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.980 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.980 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 21, 2024-09-12T23:37:51.980 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.981 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T08:27:05.298933+0000", 2024-09-12T23:37:51.981 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.000791419, 2024-09-12T23:37:51.981 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.981 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.981 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.981 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.981 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:51.981 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.981 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.981 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.981 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.982 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.982 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.982 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.982 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.982 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.982 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.982 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.982 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.982 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.982 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:51.982 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:51.982 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.983 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:51.983 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.983 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.983 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.983 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.983 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.983 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.983 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.983 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:51.983 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.983 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.983 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.984 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.984 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.984 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:51.984 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:51.984 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.984 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.984 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.984 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:51.984 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:51.984 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.985 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.985 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.985 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.985 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:51.985 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:51.985 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:51.985 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.985 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 6, 2024-09-12T23:37:51.985 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:51.985 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.985 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.986 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.986 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:51.986 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:51.986 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:51.986 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.986 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 7, 2024-09-12T23:37:51.986 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.986 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.986 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 19, 2024-09-12T23:37:51.986 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.986 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.987 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.987 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:37:47.220105+0000", 2024-09-12T23:37:51.987 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:32:45.887205+0000", 2024-09-12T23:37:51.987 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:32:45.887205+0000", 2024-09-12T23:37:51.987 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:32:45.902460+0000", 2024-09-12T23:37:51.987 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:37:47.220105+0000", 2024-09-12T23:37:51.987 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:28:34.324093+0000", 2024-09-12T23:37:51.987 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.987 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:28:34.324093+0000", 2024-09-12T23:37:51.987 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 50, 2024-09-12T23:37:51.987 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:37:47.220105+0000", 2024-09-12T23:37:51.988 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:37:47.220105+0000", 2024-09-12T23:37:51.988 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:37:47.220105+0000", 2024-09-12T23:37:51.988 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.988 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.988 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:28:34.324093+0000", 2024-09-12T23:37:51.988 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:37:47.220105+0000", 2024-09-12T23:37:51.988 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:37:47.220105+0000", 2024-09-12T23:37:51.988 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.988 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 111, 2024-09-12T23:37:51.988 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.988 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.989 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 49, 2024-09-12T23:37:51.989 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.989 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.989 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.989 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.989 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 111, 2024-09-12T23:37:51.989 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.989 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.989 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.989 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "1.0", 2024-09-12T23:37:51.989 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.989 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.990 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 64, 2024-09-12T23:37:51.990 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 169, 2024-09-12T23:37:51.990 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.990 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T07:37:37.825919+0000", 2024-09-12T23:37:51.990 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.990 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.990 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.990 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 590368, 2024-09-12T23:37:51.990 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.990 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 590368, 2024-09-12T23:37:51.990 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:51.991 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:51.991 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:51.991 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:51.991 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:51.991 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:51.991 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:51.991 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:51.991 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:51.991 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:51.991 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:51.991 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:51.992 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:51.992 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 6, 2024-09-12T23:37:51.992 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 2, 2024-09-12T23:37:51.992 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:51.992 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 2, 2024-09-12T23:37:51.992 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:51.992 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:51.992 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:51.992 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:51.992 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:51.992 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:51.993 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:51.993 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 2, 2024-09-12T23:37:51.993 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:51.993 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:51.993 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:51.993 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:51.993 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:51.993 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 416, 2024-09-12T23:37:51.993 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 599, 2024-09-12T23:37:51.993 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:51.993 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:51.994 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:51.994 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 177, 2024-09-12T23:37:51.994 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 1628 2024-09-12T23:37:51.994 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.994 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:51.994 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:51.994 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:51.994 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:51.994 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:51.994 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:51.994 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.994 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 7, 2024-09-12T23:37:51.994 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "64'111" 2024-09-12T23:37:51.995 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:51.995 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:51.995 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:51.995 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:51.995 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:51.995 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:51.995 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:51.995 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 5, 2024-09-12T23:37:51.995 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:51.995 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:51.995 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:51.995 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:51.996 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:51.996 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:51.996 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.670187+0000", 2024-09-12T23:37:51.996 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.898329+0000", 2024-09-12T23:37:51.996 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.898329+0000", 2024-09-12T23:37:51.996 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.898796+0000", 2024-09-12T23:37:51.996 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.670187+0000", 2024-09-12T23:37:51.996 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:51.996 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:51.996 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:51.997 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:51.997 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.670187+0000", 2024-09-12T23:37:51.997 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.670187+0000", 2024-09-12T23:37:51.997 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.670187+0000", 2024-09-12T23:37:51.997 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:51.997 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:51.997 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:51.997 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.670187+0000", 2024-09-12T23:37:51.997 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.670187+0000", 2024-09-12T23:37:51.997 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:51.997 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:51.997 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:51.998 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:51.998 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:51.998 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:51.998 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:51.998 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:51.998 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:51.998 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:51.998 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:51.998 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:51.998 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:51.998 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.3", 2024-09-12T23:37:51.998 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:51.999 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:51.999 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:51.999 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 21, 2024-09-12T23:37:51.999 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:51.999 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T07:54:48.474286+0000", 2024-09-12T23:37:51.999 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:51.999 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:51.999 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:51.999 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:51.999 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:51.999 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:52.000 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:52.000 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:52.000 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:52.000 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:52.000 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:52.000 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:52.000 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:52.000 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:52.000 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:52.000 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:52.000 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:52.000 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:52.001 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:52.001 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:52.001 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:52.001 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:52.001 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:52.001 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:52.001 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:52.001 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:52.001 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:52.001 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:52.001 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:52.001 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:52.002 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:52.002 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:52.002 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:52.002 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:52.002 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:52.002 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:52.002 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:52.002 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:52.002 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:52.002 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:52.002 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:52.002 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:52.003 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:52.003 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.003 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:52.003 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:52.003 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:52.003 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:52.003 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:52.003 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:52.003 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.003 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 5, 2024-09-12T23:37:52.003 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:52.004 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.004 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:52.004 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:52.004 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:52.004 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:52.004 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:52.004 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.004 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 3, 2024-09-12T23:37:52.004 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:52.004 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:52.004 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:52.005 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:52.005 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:52.005 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:52.005 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.668367+0000", 2024-09-12T23:37:52.005 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.904695+0000", 2024-09-12T23:37:52.005 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.904695+0000", 2024-09-12T23:37:52.005 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.905028+0000", 2024-09-12T23:37:52.005 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.668367+0000", 2024-09-12T23:37:52.005 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:52.005 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:52.005 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:52.005 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:52.006 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.668367+0000", 2024-09-12T23:37:52.006 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.668367+0000", 2024-09-12T23:37:52.006 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.668367+0000", 2024-09-12T23:37:52.006 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:52.006 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:52.006 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:52.006 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.668367+0000", 2024-09-12T23:37:52.006 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.668367+0000", 2024-09-12T23:37:52.006 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:52.006 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:52.006 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:52.007 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:52.007 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:52.007 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:52.007 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:52.007 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:52.007 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:52.007 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:52.007 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:52.007 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:52.007 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:52.007 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.2", 2024-09-12T23:37:52.007 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:52.008 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:52.008 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:52.008 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:52.008 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:52.008 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T01:06:47.004852+0000", 2024-09-12T23:37:52.008 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:52.008 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:52.008 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:52.008 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:52.008 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:52.008 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:52.008 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:52.009 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:52.009 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:52.009 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:52.009 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:52.009 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:52.009 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:52.009 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:52.009 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:52.009 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:52.009 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:52.010 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:52.010 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:52.010 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:52.010 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:52.010 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:52.010 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:52.010 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:52.010 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:52.010 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:52.010 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:52.010 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:52.010 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:52.010 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:52.011 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:52.011 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:52.011 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:52.011 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:52.011 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:52.011 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:52.011 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:52.011 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:52.011 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:52.011 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:52.011 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:52.012 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:52.012 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:52.012 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.012 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:52.012 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:52.012 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:52.012 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:52.012 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:52.012 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:52.012 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.012 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 3, 2024-09-12T23:37:52.012 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:52.013 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.013 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:52.013 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:52.013 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:52.013 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:52.013 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:52.013 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.013 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 0, 2024-09-12T23:37:52.013 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:52.013 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:52.013 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 56, 2024-09-12T23:37:52.014 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:52.014 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:52.014 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:52.014 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.669471+0000", 2024-09-12T23:37:52.014 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:34:49.788254+0000", 2024-09-12T23:37:52.014 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:34:49.788254+0000", 2024-09-12T23:37:52.014 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:34:59.542653+0000", 2024-09-12T23:37:52.014 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.669471+0000", 2024-09-12T23:37:52.014 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:34:48.454503+0000", 2024-09-12T23:37:52.014 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:52.014 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:34:48.454503+0000", 2024-09-12T23:37:52.015 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 57, 2024-09-12T23:37:52.015 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.669471+0000", 2024-09-12T23:37:52.015 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.669471+0000", 2024-09-12T23:37:52.015 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.669471+0000", 2024-09-12T23:37:52.015 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:52.015 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:52.015 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:34:48.454503+0000", 2024-09-12T23:37:52.015 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.669471+0000", 2024-09-12T23:37:52.015 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.669471+0000", 2024-09-12T23:37:52.015 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:52.015 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:52.015 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:52.016 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:52.016 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 56, 2024-09-12T23:37:52.016 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:52.016 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:52.016 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:52.016 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:52.016 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:52.016 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:52.016 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:52.016 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:52.016 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "4.6", 2024-09-12T23:37:52.017 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:52.017 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:52.017 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:52.017 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 21, 2024-09-12T23:37:52.017 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:52.017 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T10:47:44.627533+0000", 2024-09-12T23:37:52.017 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.001202909, 2024-09-12T23:37:52.017 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:52.017 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:52.017 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:52.017 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:52.018 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:52.018 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:52.018 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:52.018 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:52.018 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:52.018 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:52.018 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:52.018 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:52.018 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:52.018 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:52.018 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:52.019 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:52.019 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:52.019 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:52.019 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:52.019 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:52.019 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:52.019 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:52.019 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:52.019 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:52.019 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:52.019 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:52.019 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:52.020 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:52.020 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:52.020 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:52.020 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:52.020 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:52.020 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:52.020 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:52.020 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:52.020 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:52.020 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:52.020 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:52.020 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:52.021 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:52.021 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:52.021 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:52.021 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.021 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:52.021 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:52.021 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:52.021 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:52.021 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:52.021 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:52.021 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.022 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 0, 2024-09-12T23:37:52.022 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:52.022 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.022 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:52.022 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:52.022 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:52.022 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:52.022 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:52.022 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.022 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 7, 2024-09-12T23:37:52.022 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:52.022 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:52.023 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:52.023 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:52.023 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:52.023 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:52.023 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.663998+0000", 2024-09-12T23:37:52.023 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.894687+0000", 2024-09-12T23:37:52.023 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.894687+0000", 2024-09-12T23:37:52.023 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.895135+0000", 2024-09-12T23:37:52.023 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.663998+0000", 2024-09-12T23:37:52.023 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:52.023 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:52.023 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:52.024 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:52.024 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.663998+0000", 2024-09-12T23:37:52.024 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.663998+0000", 2024-09-12T23:37:52.024 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.663998+0000", 2024-09-12T23:37:52.024 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:52.024 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:52.024 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:52.024 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.663998+0000", 2024-09-12T23:37:52.024 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.663998+0000", 2024-09-12T23:37:52.024 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:52.024 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:52.024 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:52.025 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:52.025 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:52.025 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:52.025 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:52.025 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:52.025 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:52.025 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:52.025 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:52.025 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:52.025 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:52.025 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.0", 2024-09-12T23:37:52.026 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:52.026 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:52.026 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:52.026 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 21, 2024-09-12T23:37:52.026 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:52.026 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T07:57:06.646238+0000", 2024-09-12T23:37:52.026 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:52.026 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:52.026 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:52.026 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:52.026 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:52.026 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:52.027 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:52.027 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:52.027 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:52.027 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:52.027 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:52.027 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:52.027 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:52.027 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:52.027 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:52.027 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:52.027 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:52.027 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:52.028 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:52.028 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:52.028 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:52.028 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:52.028 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:52.028 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:52.028 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:52.028 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:52.028 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:52.028 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:52.028 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:52.029 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:52.029 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:52.029 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:52.029 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:52.029 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:52.029 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:52.029 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:52.029 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:52.029 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:52.029 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:52.030 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:52.030 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:52.030 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:52.030 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:52.030 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.030 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:52.030 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:52.030 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:52.030 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:52.030 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:52.030 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:52.030 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.031 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 7, 2024-09-12T23:37:52.031 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:52.031 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.031 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:52.031 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:52.031 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:52.031 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:52.031 INFO:tasks.workunit.client.0.smithi136.stdout: 3 2024-09-12T23:37:52.031 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.031 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 0, 2024-09-12T23:37:52.031 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:52.031 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:52.032 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:52.032 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:52.032 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:52.032 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:52.032 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.669881+0000", 2024-09-12T23:37:52.032 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.909765+0000", 2024-09-12T23:37:52.032 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.909765+0000", 2024-09-12T23:37:52.032 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.910025+0000", 2024-09-12T23:37:52.032 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.669881+0000", 2024-09-12T23:37:52.032 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:52.032 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:52.033 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:52.033 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:52.033 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.669881+0000", 2024-09-12T23:37:52.033 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.669881+0000", 2024-09-12T23:37:52.033 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.669881+0000", 2024-09-12T23:37:52.033 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:52.033 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:52.033 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:52.033 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.669881+0000", 2024-09-12T23:37:52.033 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.669881+0000", 2024-09-12T23:37:52.033 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:52.033 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:52.034 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:52.034 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:52.034 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:52.034 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:52.034 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:52.034 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:52.034 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:52.034 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:52.034 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:52.034 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:52.034 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:52.035 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.1", 2024-09-12T23:37:52.035 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:52.035 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:52.035 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:52.035 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:52.035 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:52.035 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T00:23:35.389216+0000", 2024-09-12T23:37:52.035 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:52.035 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:52.035 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:52.035 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:52.035 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:52.036 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:52.036 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:52.036 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:52.036 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:52.036 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:52.036 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:52.036 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:52.036 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:52.036 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:52.036 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:52.036 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:52.037 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:52.037 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:52.037 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:52.037 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:52.037 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:52.037 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:52.037 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:52.037 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:52.037 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:52.037 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:52.037 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:52.038 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:52.038 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:52.038 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:52.038 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:52.038 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:52.038 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:52.038 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:52.038 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:52.038 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:52.038 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:52.038 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:52.038 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:52.039 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:52.039 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:52.039 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:52.039 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:52.039 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.039 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:52.039 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:52.039 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:52.039 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:52.039 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:52.039 INFO:tasks.workunit.client.0.smithi136.stdout: 3 2024-09-12T23:37:52.040 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.040 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 0, 2024-09-12T23:37:52.040 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:52.040 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.040 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:52.040 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:52.040 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:52.040 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:52.040 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:52.040 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.040 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 1, 2024-09-12T23:37:52.041 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:52.041 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:52.041 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 56, 2024-09-12T23:37:52.041 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:52.041 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:52.041 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:52.041 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.663799+0000", 2024-09-12T23:37:52.041 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:34:49.788699+0000", 2024-09-12T23:37:52.041 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:34:49.788699+0000", 2024-09-12T23:37:52.041 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:34:59.549321+0000", 2024-09-12T23:37:52.041 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.663799+0000", 2024-09-12T23:37:52.042 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:34:48.454503+0000", 2024-09-12T23:37:52.042 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:52.042 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:34:48.454503+0000", 2024-09-12T23:37:52.042 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 57, 2024-09-12T23:37:52.042 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.663799+0000", 2024-09-12T23:37:52.042 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.663799+0000", 2024-09-12T23:37:52.042 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.663799+0000", 2024-09-12T23:37:52.042 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:52.042 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:52.042 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:34:48.454503+0000", 2024-09-12T23:37:52.042 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.663799+0000", 2024-09-12T23:37:52.042 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.663799+0000", 2024-09-12T23:37:52.043 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:52.043 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:52.043 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:52.043 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:52.043 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 56, 2024-09-12T23:37:52.043 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:52.043 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:52.043 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:52.043 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:52.043 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:52.043 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:52.043 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:52.044 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:52.044 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "4.7", 2024-09-12T23:37:52.044 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:52.044 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:52.044 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:52.044 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 21, 2024-09-12T23:37:52.044 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:52.044 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T11:29:56.436091+0000", 2024-09-12T23:37:52.044 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.002029651, 2024-09-12T23:37:52.044 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:52.044 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:52.044 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:52.045 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:52.045 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:52.045 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:52.045 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:52.045 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:52.045 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:52.045 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:52.045 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:52.045 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:52.045 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:52.046 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:52.046 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:52.046 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:52.046 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:52.046 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:52.046 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:52.046 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:52.046 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:52.046 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:52.046 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:52.046 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:52.046 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:52.047 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:52.047 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:52.047 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:52.047 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:52.047 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:52.047 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:52.047 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:52.047 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:52.047 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:52.047 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:52.048 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:52.048 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:52.048 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:52.048 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:52.048 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:52.048 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:52.048 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:52.048 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.048 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:52.048 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:52.048 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:52.049 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:52.049 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:52.049 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:52.049 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.049 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 1, 2024-09-12T23:37:52.049 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:52.049 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.049 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:52.049 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:52.049 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:52.049 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:52.050 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:52.050 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.050 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 2, 2024-09-12T23:37:52.050 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:52.050 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:52.050 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:52.050 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:52.050 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:52.050 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:52.050 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.661420+0000", 2024-09-12T23:37:52.050 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.868587+0000", 2024-09-12T23:37:52.051 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.868587+0000", 2024-09-12T23:37:52.051 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.868957+0000", 2024-09-12T23:37:52.051 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.661420+0000", 2024-09-12T23:37:52.051 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:52.051 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:52.051 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:52.051 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:52.051 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.661420+0000", 2024-09-12T23:37:52.051 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.661420+0000", 2024-09-12T23:37:52.051 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.661420+0000", 2024-09-12T23:37:52.051 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:52.052 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:52.052 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:52.052 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.661420+0000", 2024-09-12T23:37:52.052 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.661420+0000", 2024-09-12T23:37:52.052 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:52.052 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:52.052 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:52.052 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:52.052 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:52.052 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:52.052 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:52.053 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:52.053 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:52.053 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:52.053 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:52.053 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:52.053 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:52.053 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.1", 2024-09-12T23:37:52.053 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:52.053 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:52.053 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:52.054 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 21, 2024-09-12T23:37:52.054 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:52.054 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T03:45:32.284209+0000", 2024-09-12T23:37:52.054 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:52.054 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:52.054 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:52.054 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:52.054 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:52.054 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:52.054 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:52.054 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:52.054 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:52.055 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:52.055 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:52.055 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:52.055 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:52.055 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:52.055 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:52.055 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:52.055 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:52.055 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:52.055 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:52.055 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:52.055 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:52.056 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:52.056 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:52.056 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:52.056 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:52.056 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:52.056 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:52.056 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:52.056 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:52.056 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:52.056 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:52.057 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:52.057 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:52.057 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:52.057 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:52.057 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:52.057 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:52.057 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:52.057 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:52.057 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:52.057 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:52.057 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:52.057 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:52.058 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.058 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:52.058 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:52.058 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:52.058 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:52.058 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:52.058 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:52.058 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.058 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 2, 2024-09-12T23:37:52.058 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:52.058 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.058 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:52.059 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:52.059 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:52.059 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:52.059 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:52.059 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.059 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 1, 2024-09-12T23:37:52.059 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:52.059 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:52.059 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:52.059 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:52.059 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:52.060 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:52.060 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.663738+0000", 2024-09-12T23:37:52.060 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.889575+0000", 2024-09-12T23:37:52.060 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.889575+0000", 2024-09-12T23:37:52.060 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.890360+0000", 2024-09-12T23:37:52.060 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.663738+0000", 2024-09-12T23:37:52.060 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:52.060 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:52.060 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:52.060 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:52.060 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.663738+0000", 2024-09-12T23:37:52.060 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.663738+0000", 2024-09-12T23:37:52.061 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.663738+0000", 2024-09-12T23:37:52.061 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:52.061 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:52.061 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:52.061 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.663738+0000", 2024-09-12T23:37:52.061 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.663738+0000", 2024-09-12T23:37:52.061 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:52.061 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:52.061 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:52.061 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:52.061 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:52.061 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:52.062 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:52.062 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:52.062 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:52.062 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:52.062 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:52.062 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:52.062 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:52.062 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.0", 2024-09-12T23:37:52.062 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:52.062 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:52.062 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:52.063 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:52.063 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:52.063 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T07:36:34.938054+0000", 2024-09-12T23:37:52.063 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:52.063 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:52.063 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:52.063 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:52.063 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:52.063 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:52.063 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:52.063 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:52.064 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:52.064 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:52.064 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:52.064 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:52.064 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:52.064 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:52.064 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:52.064 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:52.064 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:52.064 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:52.064 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:52.065 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:52.065 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:52.065 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:52.065 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:52.065 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:52.065 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:52.065 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:52.065 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:52.065 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:52.065 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:52.065 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:52.066 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:52.066 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:52.066 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:52.066 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:52.066 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:52.066 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:52.066 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:52.066 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:52.066 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:52.066 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:52.066 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:52.066 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:52.067 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:52.067 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.067 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:52.067 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:52.067 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:52.067 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:52.067 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:52.067 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:52.067 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.067 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 1, 2024-09-12T23:37:52.067 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:52.067 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.067 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:52.068 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:52.068 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:52.068 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:52.068 INFO:tasks.workunit.client.0.smithi136.stdout: 3 2024-09-12T23:37:52.068 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.068 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 1, 2024-09-12T23:37:52.068 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:52.068 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:52.068 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 56, 2024-09-12T23:37:52.068 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:52.068 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:52.069 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:52.069 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.663553+0000", 2024-09-12T23:37:52.069 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:34:49.787490+0000", 2024-09-12T23:37:52.069 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:34:49.787490+0000", 2024-09-12T23:37:52.069 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:34:59.548775+0000", 2024-09-12T23:37:52.069 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.663553+0000", 2024-09-12T23:37:52.069 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:34:48.454503+0000", 2024-09-12T23:37:52.069 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:52.069 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:34:48.454503+0000", 2024-09-12T23:37:52.069 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 57, 2024-09-12T23:37:52.069 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.663553+0000", 2024-09-12T23:37:52.069 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.663553+0000", 2024-09-12T23:37:52.070 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.663553+0000", 2024-09-12T23:37:52.070 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:52.070 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:52.070 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:34:48.454503+0000", 2024-09-12T23:37:52.070 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.663553+0000", 2024-09-12T23:37:52.070 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.663553+0000", 2024-09-12T23:37:52.070 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:52.070 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:52.070 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:52.070 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:52.070 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 56, 2024-09-12T23:37:52.071 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:52.071 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:52.071 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:52.071 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:52.071 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:52.071 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:52.071 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:52.071 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:52.071 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "4.4", 2024-09-12T23:37:52.071 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:52.071 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:52.072 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:52.072 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 21, 2024-09-12T23:37:52.072 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:52.072 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T06:19:10.283468+0000", 2024-09-12T23:37:52.072 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.002051994, 2024-09-12T23:37:52.072 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:52.072 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:52.072 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:52.072 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:52.072 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:52.072 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:52.073 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:52.073 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:52.073 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:52.073 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:52.073 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:52.073 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:52.073 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:52.073 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:52.073 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:52.073 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:52.073 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:52.074 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:52.074 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:52.074 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:52.074 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:52.074 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:52.074 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:52.074 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:52.074 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:52.074 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:52.074 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:52.074 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:52.074 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:52.075 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:52.075 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:52.075 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:52.075 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:52.075 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:52.075 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:52.075 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:52.075 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:52.075 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:52.075 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:52.075 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:52.075 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:52.076 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:52.076 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.076 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:52.076 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:52.076 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:52.076 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:52.076 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:52.076 INFO:tasks.workunit.client.0.smithi136.stdout: 3 2024-09-12T23:37:52.076 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.076 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 1, 2024-09-12T23:37:52.076 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:52.076 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.077 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:52.077 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:52.077 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:52.077 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:52.077 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:52.077 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.077 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 5, 2024-09-12T23:37:52.077 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:52.077 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:52.077 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:52.077 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:52.077 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:52.078 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:52.078 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.671197+0000", 2024-09-12T23:37:52.078 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.899410+0000", 2024-09-12T23:37:52.078 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.899410+0000", 2024-09-12T23:37:52.078 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.899931+0000", 2024-09-12T23:37:52.078 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.671197+0000", 2024-09-12T23:37:52.078 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:52.078 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:52.078 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:52.078 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:52.078 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.671197+0000", 2024-09-12T23:37:52.078 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.671197+0000", 2024-09-12T23:37:52.079 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.671197+0000", 2024-09-12T23:37:52.079 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:52.079 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:52.079 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:52.079 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.671197+0000", 2024-09-12T23:37:52.079 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.671197+0000", 2024-09-12T23:37:52.079 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:52.079 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:52.079 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:52.079 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:52.079 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:52.080 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:52.080 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:52.080 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:52.080 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:52.080 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:52.080 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:52.080 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:52.080 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:52.080 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.2", 2024-09-12T23:37:52.080 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:52.081 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:52.081 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:52.081 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 21, 2024-09-12T23:37:52.081 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:52.081 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T11:05:26.671172+0000", 2024-09-12T23:37:52.081 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:52.081 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:52.081 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:52.081 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:52.081 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:52.081 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:52.082 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:52.082 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:52.082 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:52.082 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:52.082 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:52.082 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:52.082 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:52.082 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:52.082 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:52.082 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:52.082 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:52.083 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:52.083 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:52.083 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:52.083 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:52.083 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:52.083 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:52.083 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:52.083 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:52.083 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:52.083 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:52.083 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:52.084 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:52.084 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:52.084 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:52.084 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:52.084 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:52.084 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:52.084 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:52.084 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:52.084 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:52.084 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:52.085 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:52.085 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:52.085 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:52.085 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:52.085 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:52.085 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.085 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:52.085 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:52.085 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:52.085 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:52.085 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:52.086 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:52.086 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.086 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 5, 2024-09-12T23:37:52.086 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:52.086 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.086 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:52.086 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:52.086 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:52.086 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:52.086 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:52.086 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.087 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 4, 2024-09-12T23:37:52.087 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:52.087 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:52.087 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:52.087 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:52.087 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:52.087 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:52.087 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.661569+0000", 2024-09-12T23:37:52.087 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.902561+0000", 2024-09-12T23:37:52.087 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.902561+0000", 2024-09-12T23:37:52.087 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.902878+0000", 2024-09-12T23:37:52.088 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.661569+0000", 2024-09-12T23:37:52.088 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:52.088 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:52.088 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:52.088 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:52.088 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.661569+0000", 2024-09-12T23:37:52.088 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.661569+0000", 2024-09-12T23:37:52.088 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.661569+0000", 2024-09-12T23:37:52.088 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:52.088 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:52.089 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:52.089 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.661569+0000", 2024-09-12T23:37:52.089 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.661569+0000", 2024-09-12T23:37:52.089 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:52.089 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:52.089 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:52.089 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:52.089 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:52.089 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:52.089 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:52.089 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:52.090 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:52.090 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:52.090 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:52.090 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:52.090 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:52.090 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.3", 2024-09-12T23:37:52.090 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:52.090 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:52.090 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:52.090 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:52.090 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:52.091 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T08:12:55.352139+0000", 2024-09-12T23:37:52.091 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:52.091 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:52.091 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:52.091 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:52.091 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:52.091 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:52.091 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:52.091 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:52.091 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:52.091 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:52.092 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:52.092 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:52.092 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:52.092 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:52.092 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:52.092 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:52.092 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:52.092 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:52.092 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:52.092 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:52.092 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:52.092 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:52.093 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:52.093 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:52.093 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:52.093 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:52.093 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:52.093 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:52.093 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:52.093 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:52.093 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:52.093 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:52.093 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:52.093 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:52.094 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:52.094 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:52.094 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:52.094 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:52.094 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:52.094 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:52.094 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:52.094 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:52.094 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:52.094 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.094 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:52.095 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:52.095 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:52.095 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:52.095 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:52.095 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:52.095 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.095 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 4, 2024-09-12T23:37:52.095 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:52.095 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.095 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:52.095 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:52.095 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:52.096 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:52.096 INFO:tasks.workunit.client.0.smithi136.stdout: 4 2024-09-12T23:37:52.096 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.096 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 1, 2024-09-12T23:37:52.096 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:52.096 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:52.096 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 56, 2024-09-12T23:37:52.096 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:52.096 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:52.096 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:52.096 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.664284+0000", 2024-09-12T23:37:52.097 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:34:49.792105+0000", 2024-09-12T23:37:52.097 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:34:49.792105+0000", 2024-09-12T23:37:52.097 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:34:59.548734+0000", 2024-09-12T23:37:52.097 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.664284+0000", 2024-09-12T23:37:52.097 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:34:48.454503+0000", 2024-09-12T23:37:52.097 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:52.097 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:34:48.454503+0000", 2024-09-12T23:37:52.097 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 57, 2024-09-12T23:37:52.097 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.664284+0000", 2024-09-12T23:37:52.097 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.664284+0000", 2024-09-12T23:37:52.097 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.664284+0000", 2024-09-12T23:37:52.098 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:52.098 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:52.098 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:34:48.454503+0000", 2024-09-12T23:37:52.098 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.664284+0000", 2024-09-12T23:37:52.098 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.664284+0000", 2024-09-12T23:37:52.098 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:52.098 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 2, 2024-09-12T23:37:52.098 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:52.098 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:52.098 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 56, 2024-09-12T23:37:52.098 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:52.099 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:52.099 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:52.099 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:52.099 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 2, 2024-09-12T23:37:52.099 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:52.099 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:52.099 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:52.099 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "4.2", 2024-09-12T23:37:52.099 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:52.099 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:52.099 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:52.099 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 23, 2024-09-12T23:37:52.100 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:52.100 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-13T23:37:40.496016+0000", 2024-09-12T23:37:52.100 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.002201358, 2024-09-12T23:37:52.100 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:52.100 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:52.100 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 19, 2024-09-12T23:37:52.100 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:52.100 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:52.100 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:52.100 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:52.100 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:52.100 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:52.101 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:52.101 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:52.101 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:52.101 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:52.101 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:52.101 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:52.101 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:52.101 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:52.101 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:52.101 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 3, 2024-09-12T23:37:52.101 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 1, 2024-09-12T23:37:52.101 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:52.102 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 1, 2024-09-12T23:37:52.102 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:52.102 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:52.102 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:52.102 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:52.102 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:52.102 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:52.102 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:52.102 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:52.102 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:52.102 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:52.103 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:52.103 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:52.103 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:52.103 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:52.103 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:52.103 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:52.103 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:52.103 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:52.103 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 2, 2024-09-12T23:37:52.103 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 2 2024-09-12T23:37:52.103 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.103 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:52.104 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:52.104 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:52.104 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:52.104 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:52.104 INFO:tasks.workunit.client.0.smithi136.stdout: 4 2024-09-12T23:37:52.104 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.104 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 1, 2024-09-12T23:37:52.104 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "61'2" 2024-09-12T23:37:52.104 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.104 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:52.104 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:52.105 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:52.105 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:52.105 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:52.105 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.105 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 1, 2024-09-12T23:37:52.105 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:52.105 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:52.105 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:52.105 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:52.105 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:52.105 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:52.105 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.664823+0000", 2024-09-12T23:37:52.106 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.867020+0000", 2024-09-12T23:37:52.106 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.867020+0000", 2024-09-12T23:37:52.106 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.867304+0000", 2024-09-12T23:37:52.106 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.664823+0000", 2024-09-12T23:37:52.106 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:52.106 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:52.106 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:52.106 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:52.106 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.664823+0000", 2024-09-12T23:37:52.106 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.664823+0000", 2024-09-12T23:37:52.106 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.664823+0000", 2024-09-12T23:37:52.106 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:52.106 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:52.107 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:52.107 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.664823+0000", 2024-09-12T23:37:52.107 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.664823+0000", 2024-09-12T23:37:52.107 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:52.107 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:52.107 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:52.107 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:52.107 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:52.107 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:52.107 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:52.107 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:52.108 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:52.108 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:52.108 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:52.108 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:52.108 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:52.108 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.4", 2024-09-12T23:37:52.108 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:52.108 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:52.108 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:52.108 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 21, 2024-09-12T23:37:52.108 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:52.108 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T09:51:22.908188+0000", 2024-09-12T23:37:52.109 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:52.109 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:52.109 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:52.109 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:52.109 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:52.109 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:52.109 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:52.109 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:52.109 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:52.109 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:52.109 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:52.109 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:52.110 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:52.110 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:52.110 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:52.110 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:52.110 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:52.110 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:52.110 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:52.110 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:52.110 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:52.110 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:52.110 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:52.110 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:52.111 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:52.111 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:52.111 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:52.111 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:52.111 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:52.111 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:52.111 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:52.111 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:52.111 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:52.111 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:52.111 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:52.111 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:52.111 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:52.112 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:52.112 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:52.112 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:52.112 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:52.112 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:52.112 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:52.112 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.112 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:52.112 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:52.112 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:52.112 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:52.113 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:52.113 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:52.113 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.113 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 1, 2024-09-12T23:37:52.113 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:52.113 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.113 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:52.113 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:52.113 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:52.113 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:52.113 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:52.114 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.114 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 5, 2024-09-12T23:37:52.114 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:52.114 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:52.114 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:52.114 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:52.114 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:52.114 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:52.114 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.670703+0000", 2024-09-12T23:37:52.114 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.914385+0000", 2024-09-12T23:37:52.114 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.914385+0000", 2024-09-12T23:37:52.114 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.914728+0000", 2024-09-12T23:37:52.115 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.670703+0000", 2024-09-12T23:37:52.115 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:52.115 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:52.115 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:52.115 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:52.115 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.670703+0000", 2024-09-12T23:37:52.115 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.670703+0000", 2024-09-12T23:37:52.115 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.670703+0000", 2024-09-12T23:37:52.115 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:52.115 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:52.115 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:52.115 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.670703+0000", 2024-09-12T23:37:52.116 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.670703+0000", 2024-09-12T23:37:52.116 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:52.116 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:52.116 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:52.116 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:52.116 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:52.116 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:52.116 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:52.116 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:52.116 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:52.116 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:52.116 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:52.117 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:52.117 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:52.117 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.5", 2024-09-12T23:37:52.117 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:52.117 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:52.117 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:52.117 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:52.117 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:52.117 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T03:45:02.815733+0000", 2024-09-12T23:37:52.117 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:52.117 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:52.118 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:52.118 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:52.118 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:52.118 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:52.118 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:52.118 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:52.118 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:52.118 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:52.118 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:52.118 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:52.118 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:52.118 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:52.118 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:52.119 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:52.119 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:52.119 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:52.119 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:52.119 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:52.119 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:52.119 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:52.119 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:52.119 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:52.119 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:52.119 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:52.119 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:52.120 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:52.120 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:52.120 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:52.120 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:52.120 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:52.120 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:52.120 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:52.120 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:52.120 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:52.120 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:52.120 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:52.120 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:52.121 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:52.121 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:52.121 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:52.121 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:52.121 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.121 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:52.121 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:52.121 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:52.121 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:52.121 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:52.121 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:52.121 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.122 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 5, 2024-09-12T23:37:52.122 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:52.122 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.122 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:52.122 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:52.122 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:52.122 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:52.122 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:52.122 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.122 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 0, 2024-09-12T23:37:52.122 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:52.122 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:52.122 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 56, 2024-09-12T23:37:52.123 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:52.123 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:52.123 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:52.123 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.669319+0000", 2024-09-12T23:37:52.123 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:34:49.787835+0000", 2024-09-12T23:37:52.123 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:34:49.787835+0000", 2024-09-12T23:37:52.123 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:34:59.541122+0000", 2024-09-12T23:37:52.123 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.669319+0000", 2024-09-12T23:37:52.123 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:34:48.454503+0000", 2024-09-12T23:37:52.123 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:52.123 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:34:48.454503+0000", 2024-09-12T23:37:52.123 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 57, 2024-09-12T23:37:52.124 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.669319+0000", 2024-09-12T23:37:52.124 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.669319+0000", 2024-09-12T23:37:52.124 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.669319+0000", 2024-09-12T23:37:52.124 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:52.124 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:52.124 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:34:48.454503+0000", 2024-09-12T23:37:52.124 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.669319+0000", 2024-09-12T23:37:52.124 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.669319+0000", 2024-09-12T23:37:52.124 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:52.124 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 1, 2024-09-12T23:37:52.124 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:52.124 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:52.125 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 56, 2024-09-12T23:37:52.125 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:52.125 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:52.125 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:52.125 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:52.125 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 1, 2024-09-12T23:37:52.125 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:52.125 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:52.125 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:52.125 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "4.3", 2024-09-12T23:37:52.125 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:52.125 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:52.126 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:52.126 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 22, 2024-09-12T23:37:52.126 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:52.126 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T07:07:48.849192+0000", 2024-09-12T23:37:52.126 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.000992265, 2024-09-12T23:37:52.126 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:52.126 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:52.126 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:52.126 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:52.126 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:52.126 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:52.126 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:52.127 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:52.127 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:52.127 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:52.127 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:52.127 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:52.127 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:52.127 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:52.127 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:52.127 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:52.127 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:52.127 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:52.127 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 3, 2024-09-12T23:37:52.127 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 1, 2024-09-12T23:37:52.128 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:52.128 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 1, 2024-09-12T23:37:52.128 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:52.128 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:52.128 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:52.128 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:52.128 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:52.128 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:52.128 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:52.128 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:52.128 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:52.129 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:52.129 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:52.129 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:52.129 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:52.129 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:52.129 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:52.129 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:52.129 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:52.129 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:52.129 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:52.129 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:52.130 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.130 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:52.130 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:52.130 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:52.130 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:52.130 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:52.130 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:52.130 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.130 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 0, 2024-09-12T23:37:52.130 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "59'1" 2024-09-12T23:37:52.130 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.130 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:52.131 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:52.131 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:52.131 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:52.131 INFO:tasks.workunit.client.0.smithi136.stdout: 4 2024-09-12T23:37:52.131 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.131 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 7, 2024-09-12T23:37:52.131 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:52.131 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:52.131 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:52.131 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:52.132 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:52.132 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:52.132 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.666213+0000", 2024-09-12T23:37:52.132 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.892268+0000", 2024-09-12T23:37:52.132 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.892268+0000", 2024-09-12T23:37:52.132 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.895077+0000", 2024-09-12T23:37:52.132 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.666213+0000", 2024-09-12T23:37:52.132 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:52.132 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:52.132 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:52.132 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:52.133 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.666213+0000", 2024-09-12T23:37:52.133 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.666213+0000", 2024-09-12T23:37:52.133 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.666213+0000", 2024-09-12T23:37:52.133 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:52.133 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:52.133 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:52.133 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.666213+0000", 2024-09-12T23:37:52.133 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.666213+0000", 2024-09-12T23:37:52.133 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:52.133 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:52.133 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:52.134 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:52.134 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:52.134 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:52.134 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:52.134 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:52.134 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:52.134 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:52.134 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:52.134 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:52.134 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:52.134 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.5", 2024-09-12T23:37:52.134 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:52.135 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:52.135 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:52.135 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 21, 2024-09-12T23:37:52.135 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:52.135 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T04:26:19.125877+0000", 2024-09-12T23:37:52.135 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:52.135 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:52.135 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:52.135 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:52.135 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:52.135 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:52.135 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:52.136 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:52.136 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:52.136 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:52.136 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:52.136 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:52.136 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:52.136 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:52.136 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:52.136 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:52.136 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:52.136 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:52.136 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:52.137 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:52.137 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:52.137 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:52.137 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:52.137 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:52.137 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:52.137 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:52.137 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:52.137 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:52.137 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:52.137 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:52.138 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:52.138 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:52.138 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:52.138 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:52.138 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:52.138 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:52.138 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:52.138 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:52.138 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:52.138 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:52.138 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:52.139 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:52.139 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:52.139 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.139 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:52.139 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:52.139 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:52.139 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:52.139 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:52.139 INFO:tasks.workunit.client.0.smithi136.stdout: 4 2024-09-12T23:37:52.139 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.139 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 7, 2024-09-12T23:37:52.140 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:52.140 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.140 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:52.140 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:52.140 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:52.140 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:52.140 INFO:tasks.workunit.client.0.smithi136.stdout: 5 2024-09-12T23:37:52.140 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.140 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 1, 2024-09-12T23:37:52.140 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:52.140 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:52.140 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:52.141 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:52.141 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:52.141 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:52.141 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.664444+0000", 2024-09-12T23:37:52.141 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.899381+0000", 2024-09-12T23:37:52.141 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.899381+0000", 2024-09-12T23:37:52.141 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.899700+0000", 2024-09-12T23:37:52.141 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.664444+0000", 2024-09-12T23:37:52.141 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:52.141 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:52.141 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:52.142 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:52.142 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.664444+0000", 2024-09-12T23:37:52.142 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.664444+0000", 2024-09-12T23:37:52.142 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.664444+0000", 2024-09-12T23:37:52.142 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:52.142 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:52.142 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:52.142 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.664444+0000", 2024-09-12T23:37:52.142 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.664444+0000", 2024-09-12T23:37:52.142 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:52.142 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:52.143 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:52.143 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:52.143 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:52.143 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:52.143 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:52.143 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:52.143 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:52.143 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:52.143 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:52.143 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:52.143 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:52.143 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.4", 2024-09-12T23:37:52.144 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:52.144 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:52.144 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:52.144 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:52.144 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:52.144 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T09:30:20.590373+0000", 2024-09-12T23:37:52.144 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:52.144 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:52.144 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:52.144 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:52.144 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:52.144 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:52.145 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:52.145 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:52.145 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:52.145 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:52.145 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:52.145 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:52.145 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:52.145 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:52.145 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:52.145 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:52.145 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:52.146 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:52.146 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:52.146 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:52.146 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:52.146 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:52.146 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:52.146 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:52.146 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:52.146 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:52.146 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:52.146 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:52.146 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:52.147 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:52.147 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:52.147 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:52.147 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:52.147 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:52.147 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:52.147 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:52.147 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:52.147 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:52.147 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:52.147 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:52.147 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:52.148 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:52.148 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:52.148 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.148 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:52.148 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:52.148 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:52.148 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:52.148 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:52.148 INFO:tasks.workunit.client.0.smithi136.stdout: 5 2024-09-12T23:37:52.148 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.148 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 1, 2024-09-12T23:37:52.149 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:52.149 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.149 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:52.149 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:52.149 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:52.149 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:52.149 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:52.149 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.149 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 3, 2024-09-12T23:37:52.149 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:52.149 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:52.149 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 56, 2024-09-12T23:37:52.150 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:52.150 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:52.150 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:52.150 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.666658+0000", 2024-09-12T23:37:52.150 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:34:49.736609+0000", 2024-09-12T23:37:52.150 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:34:49.736609+0000", 2024-09-12T23:37:52.150 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:34:59.899761+0000", 2024-09-12T23:37:52.150 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.666658+0000", 2024-09-12T23:37:52.150 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:34:48.454503+0000", 2024-09-12T23:37:52.150 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:52.150 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:34:48.454503+0000", 2024-09-12T23:37:52.151 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 57, 2024-09-12T23:37:52.151 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.666658+0000", 2024-09-12T23:37:52.151 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.666658+0000", 2024-09-12T23:37:52.151 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.666658+0000", 2024-09-12T23:37:52.151 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:52.151 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:52.151 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:34:48.454503+0000", 2024-09-12T23:37:52.151 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.666658+0000", 2024-09-12T23:37:52.151 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.666658+0000", 2024-09-12T23:37:52.151 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:52.152 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:52.152 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:52.152 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:52.152 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 56, 2024-09-12T23:37:52.152 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:52.152 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:52.152 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:52.152 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:52.152 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:52.152 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:52.152 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:52.152 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:52.153 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "4.0", 2024-09-12T23:37:52.153 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:52.153 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:52.153 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:52.153 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 21, 2024-09-12T23:37:52.153 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:52.153 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T06:01:06.682284+0000", 2024-09-12T23:37:52.153 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.001798914, 2024-09-12T23:37:52.153 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:52.153 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:52.153 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:52.153 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:52.154 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:52.154 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:52.154 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:52.154 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:52.154 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:52.154 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:52.154 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:52.154 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:52.154 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:52.154 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:52.155 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:52.155 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:52.155 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:52.155 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:52.155 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:52.155 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:52.155 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:52.155 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:52.155 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:52.155 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:52.155 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:52.155 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:52.155 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:52.156 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:52.156 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:52.156 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:52.156 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:52.156 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:52.156 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:52.156 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:52.156 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:52.156 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:52.156 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:52.156 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:52.156 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:52.156 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:52.156 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:52.156 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:52.157 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.157 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:52.157 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:52.157 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:52.157 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:52.157 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:52.157 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:52.157 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.157 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 3, 2024-09-12T23:37:52.157 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:52.157 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.157 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:52.157 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:52.157 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:52.158 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:52.158 INFO:tasks.workunit.client.0.smithi136.stdout: 4 2024-09-12T23:37:52.158 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.158 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 1, 2024-09-12T23:37:52.158 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:52.158 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:52.158 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:52.158 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:52.158 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:52.158 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:52.158 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.665023+0000", 2024-09-12T23:37:52.158 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.866005+0000", 2024-09-12T23:37:52.158 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.866005+0000", 2024-09-12T23:37:52.158 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.866367+0000", 2024-09-12T23:37:52.159 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.665023+0000", 2024-09-12T23:37:52.159 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:52.159 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:52.159 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:52.159 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:52.159 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.665023+0000", 2024-09-12T23:37:52.159 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.665023+0000", 2024-09-12T23:37:52.159 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.665023+0000", 2024-09-12T23:37:52.159 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:52.159 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:52.159 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:52.159 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.665023+0000", 2024-09-12T23:37:52.159 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.665023+0000", 2024-09-12T23:37:52.159 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:52.159 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 1, 2024-09-12T23:37:52.160 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:52.160 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:52.160 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:52.160 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:52.160 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:52.160 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:52.160 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:52.160 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 1, 2024-09-12T23:37:52.160 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:52.160 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:52.160 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:52.160 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.6", 2024-09-12T23:37:52.160 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:52.161 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:52.161 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:52.161 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 22, 2024-09-12T23:37:52.161 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:52.161 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T05:49:27.803045+0000", 2024-09-12T23:37:52.161 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:52.161 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:52.161 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:52.161 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:52.161 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:52.161 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:52.161 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:52.161 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:52.161 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:52.162 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:52.162 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:52.162 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:52.162 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:52.162 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:52.162 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:52.162 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:52.162 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:52.162 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:52.162 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:52.162 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 3, 2024-09-12T23:37:52.162 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 1, 2024-09-12T23:37:52.162 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:52.162 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 1, 2024-09-12T23:37:52.163 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:52.163 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:52.163 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:52.163 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:52.163 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:52.163 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 1, 2024-09-12T23:37:52.163 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:52.163 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:52.163 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:52.163 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:52.163 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:52.163 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:52.163 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:52.163 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:52.164 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:52.164 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:52.164 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:52.164 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:52.164 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 1, 2024-09-12T23:37:52.164 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:52.164 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.164 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:52.164 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:52.164 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:52.164 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:52.164 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:52.164 INFO:tasks.workunit.client.0.smithi136.stdout: 4 2024-09-12T23:37:52.164 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.165 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 1, 2024-09-12T23:37:52.165 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "54'1" 2024-09-12T23:37:52.165 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.165 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:52.165 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:52.165 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:52.165 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:52.165 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:52.165 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.165 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 3, 2024-09-12T23:37:52.165 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:52.165 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:52.165 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:52.165 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:52.166 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:52.166 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:52.166 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.673648+0000", 2024-09-12T23:37:52.166 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.914934+0000", 2024-09-12T23:37:52.166 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.914934+0000", 2024-09-12T23:37:52.166 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.915275+0000", 2024-09-12T23:37:52.166 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.673648+0000", 2024-09-12T23:37:52.166 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:52.166 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:52.166 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:52.166 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:52.166 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.673648+0000", 2024-09-12T23:37:52.166 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.673648+0000", 2024-09-12T23:37:52.166 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.673648+0000", 2024-09-12T23:37:52.167 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:52.167 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:52.167 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:52.167 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.673648+0000", 2024-09-12T23:37:52.167 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.673648+0000", 2024-09-12T23:37:52.167 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:52.167 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:52.167 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:52.167 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:52.167 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:52.167 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:52.167 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:52.167 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:52.168 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:52.168 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:52.168 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:52.168 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:52.168 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:52.168 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.7", 2024-09-12T23:37:52.168 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:52.168 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:52.168 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:52.168 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:52.168 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:52.168 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T06:43:27.562543+0000", 2024-09-12T23:37:52.168 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:52.168 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:52.169 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:52.169 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:52.169 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:52.169 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:52.169 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:52.169 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:52.169 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:52.169 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:52.169 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:52.169 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:52.169 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:52.169 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:52.169 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:52.169 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:52.170 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:52.170 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:52.170 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:52.170 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:52.170 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:52.170 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:52.170 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:52.170 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:52.170 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:52.170 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:52.170 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:52.170 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:52.170 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:52.171 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:52.171 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:52.171 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:52.171 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:52.171 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:52.171 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:52.171 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:52.171 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:52.171 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:52.171 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:52.171 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:52.171 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:52.171 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:52.171 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:52.171 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.172 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:52.172 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:52.172 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:52.172 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:52.172 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:52.172 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:52.172 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.172 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 3, 2024-09-12T23:37:52.172 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:52.172 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.172 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:52.172 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:52.172 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:52.172 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:52.173 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:52.173 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.173 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 4, 2024-09-12T23:37:52.173 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:52.173 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:52.173 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 56, 2024-09-12T23:37:52.173 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:52.173 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:52.173 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:52.173 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.661581+0000", 2024-09-12T23:37:52.173 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:34:49.785409+0000", 2024-09-12T23:37:52.173 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:34:49.785409+0000", 2024-09-12T23:37:52.173 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:34:59.550136+0000", 2024-09-12T23:37:52.173 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.661581+0000", 2024-09-12T23:37:52.173 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:34:48.454503+0000", 2024-09-12T23:37:52.174 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:52.174 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:34:48.454503+0000", 2024-09-12T23:37:52.174 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 57, 2024-09-12T23:37:52.174 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.661581+0000", 2024-09-12T23:37:52.174 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.661581+0000", 2024-09-12T23:37:52.175 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.661581+0000", 2024-09-12T23:37:52.175 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:52.175 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:52.175 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:34:48.454503+0000", 2024-09-12T23:37:52.176 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.661581+0000", 2024-09-12T23:37:52.176 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.661581+0000", 2024-09-12T23:37:52.176 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:52.176 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:52.176 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:52.176 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:52.176 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 56, 2024-09-12T23:37:52.176 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:52.176 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:52.176 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:52.176 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:52.176 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:52.176 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:52.176 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:52.177 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:52.177 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "4.1", 2024-09-12T23:37:52.177 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:52.177 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:52.177 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:52.177 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 21, 2024-09-12T23:37:52.177 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:52.177 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T07:30:29.736480+0000", 2024-09-12T23:37:52.177 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.00101623, 2024-09-12T23:37:52.177 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:52.177 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:52.177 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:52.177 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:52.177 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:52.178 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:52.178 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:52.178 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:52.178 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:52.178 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:52.178 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:52.178 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:52.178 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:52.178 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:52.178 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:52.178 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:52.178 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:52.178 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:52.178 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:52.179 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:52.179 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:52.179 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:52.179 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:52.179 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:52.179 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:52.179 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:52.179 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:52.179 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:52.179 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:52.179 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:52.179 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:52.179 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:52.179 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:52.180 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:52.180 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:52.180 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:52.180 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:52.180 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:52.180 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:52.180 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:52.180 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:52.180 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:52.180 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.180 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:52.180 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:52.180 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:52.180 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:52.181 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:52.181 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:52.181 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.181 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 4, 2024-09-12T23:37:52.181 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:52.181 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.181 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:52.181 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:52.181 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:52.181 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:52.181 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:52.181 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.181 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 6, 2024-09-12T23:37:52.181 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:52.182 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:52.182 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:52.182 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:52.182 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:52.182 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:52.182 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.665837+0000", 2024-09-12T23:37:52.182 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.886573+0000", 2024-09-12T23:37:52.182 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.886573+0000", 2024-09-12T23:37:52.182 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.887351+0000", 2024-09-12T23:37:52.182 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.665837+0000", 2024-09-12T23:37:52.182 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:52.182 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:52.182 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:52.182 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:52.182 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.665837+0000", 2024-09-12T23:37:52.183 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.665837+0000", 2024-09-12T23:37:52.183 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.665837+0000", 2024-09-12T23:37:52.183 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:52.183 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:52.183 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:52.183 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.665837+0000", 2024-09-12T23:37:52.183 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.665837+0000", 2024-09-12T23:37:52.183 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:52.183 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 2, 2024-09-12T23:37:52.183 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:52.183 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:52.183 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:52.183 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:52.184 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:52.184 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:52.184 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:52.184 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 2, 2024-09-12T23:37:52.184 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:52.184 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:52.184 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:52.184 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.7", 2024-09-12T23:37:52.184 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:52.184 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:52.184 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:52.184 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 23, 2024-09-12T23:37:52.184 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:52.184 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T06:58:40.820895+0000", 2024-09-12T23:37:52.185 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:52.185 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:52.185 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:52.185 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:52.185 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:52.185 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:52.185 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:52.185 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:52.185 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:52.185 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:52.185 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:52.185 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:52.185 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:52.186 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:52.186 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:52.186 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:52.186 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:52.186 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:52.186 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:52.186 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 3, 2024-09-12T23:37:52.186 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 1, 2024-09-12T23:37:52.186 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:52.186 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 1, 2024-09-12T23:37:52.186 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:52.186 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:52.186 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:52.186 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:52.187 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:52.187 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 1, 2024-09-12T23:37:52.187 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:52.187 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:52.187 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:52.187 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:52.187 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:52.187 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:52.187 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:52.187 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:52.187 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:52.187 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:52.187 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:52.187 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:52.188 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 3, 2024-09-12T23:37:52.188 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:52.188 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.188 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:52.188 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:52.188 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:52.188 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:52.188 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:52.188 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:52.188 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.188 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 6, 2024-09-12T23:37:52.188 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "53'2" 2024-09-12T23:37:52.188 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.188 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:52.189 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:52.189 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:52.189 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:52.189 INFO:tasks.workunit.client.0.smithi136.stdout: 4 2024-09-12T23:37:52.189 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.189 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 0, 2024-09-12T23:37:52.189 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:52.189 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:52.189 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:52.189 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:52.189 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:52.189 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:52.189 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.669756+0000", 2024-09-12T23:37:52.189 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.906278+0000", 2024-09-12T23:37:52.190 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.906278+0000", 2024-09-12T23:37:52.190 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.908474+0000", 2024-09-12T23:37:52.190 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.669756+0000", 2024-09-12T23:37:52.190 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:52.190 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:52.190 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:52.190 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:52.190 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.669756+0000", 2024-09-12T23:37:52.190 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.669756+0000", 2024-09-12T23:37:52.190 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.669756+0000", 2024-09-12T23:37:52.190 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:52.190 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:52.190 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:52.191 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.669756+0000", 2024-09-12T23:37:52.191 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.669756+0000", 2024-09-12T23:37:52.191 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:52.191 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:52.191 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:52.191 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:52.191 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:52.191 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:52.191 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:52.191 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:52.191 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:52.191 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:52.191 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:52.191 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:52.191 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:52.192 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.6", 2024-09-12T23:37:52.192 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:52.192 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:52.192 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:52.192 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:52.192 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:52.192 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T08:27:51.832430+0000", 2024-09-12T23:37:52.192 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:52.192 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:52.192 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:52.192 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:52.192 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:52.192 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:52.193 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:52.193 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:52.193 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:52.193 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:52.193 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:52.193 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:52.193 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:52.193 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:52.193 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:52.193 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:52.193 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:52.193 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:52.193 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:52.193 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:52.194 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:52.194 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:52.194 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:52.194 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:52.194 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:52.194 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:52.194 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:52.194 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:52.194 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:52.194 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:52.194 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:52.194 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:52.194 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:52.194 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:52.195 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:52.195 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:52.195 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:52.195 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:52.195 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:52.195 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:52.195 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:52.195 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:52.195 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:52.195 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.195 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:52.195 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:52.195 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:52.195 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:52.196 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:52.196 INFO:tasks.workunit.client.0.smithi136.stdout: 4 2024-09-12T23:37:52.196 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.196 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 0, 2024-09-12T23:37:52.196 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:52.196 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.196 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:52.196 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:52.196 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:52.196 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:52.196 INFO:tasks.workunit.client.0.smithi136.stdout: 1 2024-09-12T23:37:52.196 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.196 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 7, 2024-09-12T23:37:52.196 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:52.197 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:52.197 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:52.197 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:52.197 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:52.197 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:52.197 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.664547+0000", 2024-09-12T23:37:52.197 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.893861+0000", 2024-09-12T23:37:52.197 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.893861+0000", 2024-09-12T23:37:52.197 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.894286+0000", 2024-09-12T23:37:52.197 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.664547+0000", 2024-09-12T23:37:52.197 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:52.197 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:52.197 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:52.197 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:52.198 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.664547+0000", 2024-09-12T23:37:52.198 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.664547+0000", 2024-09-12T23:37:52.198 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.664547+0000", 2024-09-12T23:37:52.198 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:52.198 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:52.198 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:52.198 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.664547+0000", 2024-09-12T23:37:52.198 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.664547+0000", 2024-09-12T23:37:52.198 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:52.198 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:52.198 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:52.198 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:52.198 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:52.199 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:52.199 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:52.199 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:52.199 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:52.199 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:52.199 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:52.199 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:52.199 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:52.199 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.8", 2024-09-12T23:37:52.199 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:52.199 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:52.199 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:52.199 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 21, 2024-09-12T23:37:52.199 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:52.199 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T09:20:01.806723+0000", 2024-09-12T23:37:52.200 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:52.200 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:52.200 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:52.200 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:52.200 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:52.200 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:52.200 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:52.200 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:52.200 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:52.200 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:52.200 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:52.200 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:52.200 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:52.200 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:52.201 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:52.201 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:52.201 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:52.201 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:52.201 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:52.201 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:52.201 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:52.201 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:52.201 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:52.201 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:52.201 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:52.201 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:52.201 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:52.202 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:52.202 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:52.202 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:52.202 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:52.202 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:52.202 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:52.202 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:52.202 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:52.202 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:52.202 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:52.202 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:52.202 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:52.202 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:52.203 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:52.203 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:52.203 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:52.203 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.203 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:52.203 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:52.203 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:52.203 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:52.203 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:52.203 INFO:tasks.workunit.client.0.smithi136.stdout: 1 2024-09-12T23:37:52.203 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.203 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 7, 2024-09-12T23:37:52.203 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:52.203 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.203 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:52.204 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:52.204 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:52.204 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:52.204 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:52.204 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.204 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 1, 2024-09-12T23:37:52.204 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:52.204 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:52.204 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:52.204 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:52.204 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:52.204 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:52.204 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:09.676563+0000", 2024-09-12T23:37:52.204 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:35:09.675322+0000", 2024-09-12T23:37:52.205 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:35:09.675322+0000", 2024-09-12T23:37:52.205 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:35:09.676563+0000", 2024-09-12T23:37:52.205 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:09.676563+0000", 2024-09-12T23:37:52.205 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:52.205 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:52.205 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:52.205 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 64, 2024-09-12T23:37:52.205 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:09.676563+0000", 2024-09-12T23:37:52.205 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:09.676563+0000", 2024-09-12T23:37:52.205 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:09.676563+0000", 2024-09-12T23:37:52.205 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:52.205 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:52.205 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:52.206 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:09.676563+0000", 2024-09-12T23:37:52.206 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:09.676563+0000", 2024-09-12T23:37:52.206 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:52.206 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:52.206 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:52.206 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:52.206 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 63, 2024-09-12T23:37:52.206 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:52.206 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:52.206 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:52.206 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:52.206 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:52.206 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:52.206 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:52.207 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:52.207 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.9", 2024-09-12T23:37:52.207 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:52.207 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:52.207 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 64, 2024-09-12T23:37:52.207 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 10, 2024-09-12T23:37:52.207 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:52.207 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T03:11:37.262719+0000", 2024-09-12T23:37:52.207 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:52.207 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:52.207 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:52.207 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:52.207 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:52.207 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:52.208 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:52.208 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:52.208 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:52.208 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:52.208 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:52.208 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:52.208 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:52.208 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:52.208 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:52.208 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:52.208 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:52.208 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:52.208 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:52.208 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:52.209 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:52.209 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:52.209 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:52.209 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:52.209 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:52.209 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:52.209 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:52.209 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:52.209 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:52.209 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:52.209 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:52.209 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:52.209 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:52.210 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:52.210 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:52.210 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:52.210 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:52.210 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:52.210 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:52.210 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:52.210 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:52.210 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:52.210 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:52.210 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.210 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:52.210 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:52.210 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:52.211 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:52.211 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:52.211 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:52.211 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.211 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 1, 2024-09-12T23:37:52.211 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:52.211 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.211 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:52.211 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:52.211 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:52.211 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:52.211 INFO:tasks.workunit.client.0.smithi136.stdout: 3 2024-09-12T23:37:52.211 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.211 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 1, 2024-09-12T23:37:52.211 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:52.212 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:52.212 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:52.212 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:52.212 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:52.212 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:52.212 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.664006+0000", 2024-09-12T23:37:52.212 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.884027+0000", 2024-09-12T23:37:52.212 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.884027+0000", 2024-09-12T23:37:52.212 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.884360+0000", 2024-09-12T23:37:52.212 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.664006+0000", 2024-09-12T23:37:52.212 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:52.212 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:52.212 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:52.213 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:52.213 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.664006+0000", 2024-09-12T23:37:52.213 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.664006+0000", 2024-09-12T23:37:52.213 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.664006+0000", 2024-09-12T23:37:52.213 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:52.213 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:52.213 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:52.213 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.664006+0000", 2024-09-12T23:37:52.213 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.664006+0000", 2024-09-12T23:37:52.213 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:52.213 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:52.213 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:52.213 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:52.213 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:52.213 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:52.214 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:52.214 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:52.214 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:52.214 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:52.214 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:52.214 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:52.214 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:52.214 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.9", 2024-09-12T23:37:52.214 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:52.214 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:52.214 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:52.214 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 21, 2024-09-12T23:37:52.214 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:52.214 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T03:11:57.567074+0000", 2024-09-12T23:37:52.215 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:52.215 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:52.215 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:52.215 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:52.215 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:52.215 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:52.215 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:52.215 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:52.215 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:52.215 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:52.215 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:52.215 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:52.215 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:52.215 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:52.216 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:52.216 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:52.216 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:52.216 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:52.216 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:52.216 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:52.216 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:52.216 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:52.216 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:52.216 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:52.216 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:52.216 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:52.216 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:52.216 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:52.217 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:52.217 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:52.217 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:52.217 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:52.217 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:52.217 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:52.217 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:52.217 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:52.217 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:52.217 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:52.217 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:52.217 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:52.217 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:52.217 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:52.218 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:52.218 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.218 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:52.218 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:52.218 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:52.218 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:52.218 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:52.218 INFO:tasks.workunit.client.0.smithi136.stdout: 3 2024-09-12T23:37:52.218 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.218 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 1, 2024-09-12T23:37:52.218 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:52.218 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.218 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:52.218 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:52.219 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:52.219 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:52.219 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:52.219 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.219 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 3, 2024-09-12T23:37:52.219 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:52.219 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:52.219 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:52.219 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:52.219 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:52.219 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:52.219 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.667167+0000", 2024-09-12T23:37:52.219 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.914543+0000", 2024-09-12T23:37:52.219 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.914543+0000", 2024-09-12T23:37:52.220 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.915243+0000", 2024-09-12T23:37:52.220 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.667167+0000", 2024-09-12T23:37:52.220 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:52.220 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:52.220 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:52.220 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:52.220 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.667167+0000", 2024-09-12T23:37:52.220 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.667167+0000", 2024-09-12T23:37:52.220 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.667167+0000", 2024-09-12T23:37:52.220 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:52.220 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:52.220 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:52.220 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.667167+0000", 2024-09-12T23:37:52.220 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.667167+0000", 2024-09-12T23:37:52.221 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:52.221 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:52.221 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:52.221 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:52.221 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:52.221 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:52.221 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:52.221 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:52.221 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:52.221 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:52.221 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:52.221 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:52.221 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:52.221 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.8", 2024-09-12T23:37:52.222 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:52.222 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:52.222 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:52.222 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:52.222 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:52.222 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-13T23:48:16.023985+0000", 2024-09-12T23:37:52.222 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:52.222 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:52.222 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:52.222 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:52.222 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:52.222 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:52.222 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:52.222 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:52.223 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:52.223 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:52.223 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:52.223 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:52.223 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:52.223 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:52.223 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:52.223 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:52.223 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:52.223 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:52.223 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:52.223 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:52.223 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:52.224 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:52.224 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:52.224 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:52.224 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:52.224 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:52.224 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:52.224 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:52.224 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:52.224 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:52.224 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:52.224 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:52.224 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:52.224 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:52.224 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:52.225 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:52.225 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:52.225 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:52.225 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:52.225 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:52.225 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:52.225 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:52.225 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:52.225 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.225 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:52.225 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:52.225 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:52.225 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:52.225 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:52.226 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:52.226 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.226 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 3, 2024-09-12T23:37:52.226 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:52.226 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.226 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:52.226 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:52.226 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:52.226 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:52.226 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:52.226 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.226 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 1, 2024-09-12T23:37:52.226 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:52.226 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:52.227 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:52.227 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:52.227 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:52.227 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:52.227 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.664752+0000", 2024-09-12T23:37:52.227 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.883276+0000", 2024-09-12T23:37:52.227 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.883276+0000", 2024-09-12T23:37:52.227 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.883601+0000", 2024-09-12T23:37:52.227 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.664752+0000", 2024-09-12T23:37:52.227 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:52.227 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:52.227 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:52.227 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:52.227 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.664752+0000", 2024-09-12T23:37:52.228 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.664752+0000", 2024-09-12T23:37:52.228 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.664752+0000", 2024-09-12T23:37:52.228 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:52.228 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:52.228 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:52.228 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.664752+0000", 2024-09-12T23:37:52.228 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.664752+0000", 2024-09-12T23:37:52.228 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:52.228 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:52.228 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:52.228 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:52.228 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:52.228 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:52.228 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:52.229 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:52.229 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:52.229 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:52.229 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:52.229 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:52.229 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:52.229 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.a", 2024-09-12T23:37:52.229 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:52.229 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:52.229 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:52.229 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 21, 2024-09-12T23:37:52.229 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:52.229 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T00:26:21.733615+0000", 2024-09-12T23:37:52.229 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:52.230 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:52.230 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:52.230 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:52.230 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:52.230 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:52.230 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:52.230 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:52.230 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:52.230 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:52.230 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:52.230 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:52.230 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:52.230 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:52.230 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:52.231 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:52.231 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:52.231 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:52.231 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:52.231 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:52.231 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:52.231 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:52.231 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:52.231 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:52.231 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:52.231 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:52.231 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:52.231 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:52.232 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:52.232 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:52.232 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:52.232 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:52.232 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:52.232 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:52.232 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:52.232 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:52.232 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:52.232 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:52.232 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:52.232 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:52.232 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:52.232 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:52.233 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:52.233 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.233 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:52.233 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:52.233 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:52.233 INFO:tasks.workunit.client.0.smithi136.stdout: 1, 2024-09-12T23:37:52.233 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:52.233 INFO:tasks.workunit.client.0.smithi136.stdout: 7 2024-09-12T23:37:52.233 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.233 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 1, 2024-09-12T23:37:52.233 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:52.233 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.233 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:52.233 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:52.234 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:52.234 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:52.234 INFO:tasks.workunit.client.0.smithi136.stdout: 4 2024-09-12T23:37:52.234 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.234 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 3, 2024-09-12T23:37:52.234 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:52.234 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:52.234 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:52.234 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:52.234 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:52.234 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:52.234 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.668547+0000", 2024-09-12T23:37:52.234 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.907814+0000", 2024-09-12T23:37:52.234 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.907814+0000", 2024-09-12T23:37:52.234 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.908185+0000", 2024-09-12T23:37:52.235 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.668547+0000", 2024-09-12T23:37:52.235 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:52.235 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:52.235 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:52.235 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:52.235 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.668547+0000", 2024-09-12T23:37:52.235 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.668547+0000", 2024-09-12T23:37:52.235 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.668547+0000", 2024-09-12T23:37:52.235 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:52.235 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:52.235 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:52.235 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.668547+0000", 2024-09-12T23:37:52.235 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.668547+0000", 2024-09-12T23:37:52.235 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:52.236 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:52.236 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:52.236 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:52.236 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:52.236 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:52.236 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:52.236 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:52.236 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:52.236 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:52.236 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:52.236 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:52.236 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:52.236 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.b", 2024-09-12T23:37:52.236 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:52.237 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:52.237 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:52.237 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:52.237 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:52.237 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T10:45:47.207242+0000", 2024-09-12T23:37:52.237 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:52.237 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:52.237 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:52.237 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:52.237 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:52.237 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:52.237 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:52.237 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:52.238 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:52.238 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:52.238 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:52.238 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:52.238 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:52.238 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:52.238 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:52.238 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:52.238 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:52.238 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:52.238 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:52.238 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:52.238 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:52.238 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:52.239 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:52.239 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:52.239 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:52.239 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:52.239 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:52.239 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:52.239 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:52.239 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:52.239 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:52.239 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:52.239 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:52.239 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:52.239 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:52.239 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:52.240 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:52.240 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:52.240 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:52.240 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:52.240 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:52.240 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:52.240 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:52.240 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.240 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:52.240 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:52.240 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:52.240 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:52.240 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:52.240 INFO:tasks.workunit.client.0.smithi136.stdout: 4 2024-09-12T23:37:52.241 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.241 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 3, 2024-09-12T23:37:52.241 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:52.241 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.241 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:52.241 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:52.241 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:52.241 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:52.241 INFO:tasks.workunit.client.0.smithi136.stdout: 5 2024-09-12T23:37:52.241 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.241 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 2, 2024-09-12T23:37:52.241 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:52.241 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:52.241 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:52.242 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:52.242 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:52.242 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:52.242 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:09.689819+0000", 2024-09-12T23:37:52.242 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:35:09.683737+0000", 2024-09-12T23:37:52.242 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:35:09.683737+0000", 2024-09-12T23:37:52.242 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:35:09.689819+0000", 2024-09-12T23:37:52.242 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:09.689819+0000", 2024-09-12T23:37:52.242 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:52.242 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:52.242 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:52.242 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 64, 2024-09-12T23:37:52.242 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:09.689819+0000", 2024-09-12T23:37:52.242 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:09.689819+0000", 2024-09-12T23:37:52.243 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:09.689819+0000", 2024-09-12T23:37:52.243 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:52.243 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:52.243 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:52.243 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:09.689819+0000", 2024-09-12T23:37:52.243 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:09.689819+0000", 2024-09-12T23:37:52.243 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:52.243 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 2, 2024-09-12T23:37:52.243 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:52.243 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:52.243 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 63, 2024-09-12T23:37:52.243 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:52.243 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:52.243 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:52.243 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:52.244 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 2, 2024-09-12T23:37:52.244 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:52.244 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:52.244 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:52.244 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.b", 2024-09-12T23:37:52.244 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:52.244 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:52.244 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 64, 2024-09-12T23:37:52.244 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 14, 2024-09-12T23:37:52.244 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:52.244 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T09:18:02.546637+0000", 2024-09-12T23:37:52.244 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:52.244 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:52.245 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:52.245 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:52.245 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:52.245 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:52.245 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:52.245 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:52.245 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:52.245 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:52.245 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:52.245 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:52.245 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:52.245 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:52.245 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:52.246 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:52.246 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:52.246 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:52.246 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:52.246 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 3, 2024-09-12T23:37:52.246 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 1, 2024-09-12T23:37:52.246 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:52.246 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 1, 2024-09-12T23:37:52.246 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:52.246 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:52.246 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:52.246 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:52.246 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:52.246 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 1, 2024-09-12T23:37:52.247 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:52.247 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 1, 2024-09-12T23:37:52.247 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:52.247 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:52.247 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:52.247 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:52.247 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:52.247 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:52.247 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:52.247 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:52.247 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:52.247 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:52.247 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 1, 2024-09-12T23:37:52.248 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:52.248 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.248 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:52.248 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:52.248 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:52.248 INFO:tasks.workunit.client.0.smithi136.stdout: 2, 2024-09-12T23:37:52.248 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:52.248 INFO:tasks.workunit.client.0.smithi136.stdout: 5 2024-09-12T23:37:52.248 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.248 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 2, 2024-09-12T23:37:52.248 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "53'2" 2024-09-12T23:37:52.248 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.248 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:52.248 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:52.249 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:52.249 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:52.249 INFO:tasks.workunit.client.0.smithi136.stdout: 1 2024-09-12T23:37:52.249 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.249 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 6, 2024-09-12T23:37:52.249 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:52.249 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:52.249 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:52.249 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:52.249 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:52.249 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:52.249 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.667383+0000", 2024-09-12T23:37:52.249 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.896761+0000", 2024-09-12T23:37:52.250 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.896761+0000", 2024-09-12T23:37:52.250 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.897132+0000", 2024-09-12T23:37:52.250 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.667383+0000", 2024-09-12T23:37:52.250 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:52.250 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:52.250 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:52.250 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:52.250 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.667383+0000", 2024-09-12T23:37:52.250 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.667383+0000", 2024-09-12T23:37:52.250 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.667383+0000", 2024-09-12T23:37:52.250 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:52.250 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:52.250 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:52.250 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.667383+0000", 2024-09-12T23:37:52.251 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.667383+0000", 2024-09-12T23:37:52.251 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:52.251 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:52.251 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:52.251 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:52.251 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:52.251 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:52.251 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:52.251 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:52.251 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:52.251 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:52.251 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:52.251 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:52.251 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:52.252 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.a", 2024-09-12T23:37:52.252 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:52.252 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:52.252 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:52.252 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:52.252 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:52.252 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T03:49:56.554393+0000", 2024-09-12T23:37:52.252 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:52.252 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:52.252 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:52.252 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:52.252 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:52.252 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:52.253 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:52.253 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:52.253 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:52.253 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:52.253 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:52.253 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:52.253 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:52.253 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:52.253 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:52.253 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:52.253 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:52.253 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:52.253 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:52.253 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:52.254 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:52.254 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:52.254 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:52.254 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:52.254 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:52.254 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:52.254 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:52.254 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:52.254 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:52.254 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:52.254 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:52.254 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:52.254 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:52.254 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:52.255 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:52.255 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:52.255 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:52.255 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:52.255 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:52.255 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:52.255 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:52.255 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:52.255 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:52.255 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.255 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:52.255 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:52.255 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:52.255 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:52.256 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:52.256 INFO:tasks.workunit.client.0.smithi136.stdout: 1 2024-09-12T23:37:52.256 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.256 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 6, 2024-09-12T23:37:52.256 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:52.256 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.256 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:52.256 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:52.256 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:52.256 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:52.256 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:52.256 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.256 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 5, 2024-09-12T23:37:52.256 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:52.257 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:52.257 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:52.257 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:52.257 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:52.257 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:52.257 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.670622+0000", 2024-09-12T23:37:52.257 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.911157+0000", 2024-09-12T23:37:52.257 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.911157+0000", 2024-09-12T23:37:52.257 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.911440+0000", 2024-09-12T23:37:52.257 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.670622+0000", 2024-09-12T23:37:52.257 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:52.257 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:52.257 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:52.257 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:52.258 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.670622+0000", 2024-09-12T23:37:52.258 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.670622+0000", 2024-09-12T23:37:52.258 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.670622+0000", 2024-09-12T23:37:52.258 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:52.258 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:52.258 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:52.258 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.670622+0000", 2024-09-12T23:37:52.258 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.670622+0000", 2024-09-12T23:37:52.258 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:52.258 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:52.258 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:52.258 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:52.258 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:52.258 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:52.259 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:52.259 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:52.259 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:52.259 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:52.259 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:52.259 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:52.259 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:52.259 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.1d", 2024-09-12T23:37:52.259 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:52.259 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:52.259 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:52.259 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:52.259 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:52.259 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T00:49:59.313436+0000", 2024-09-12T23:37:52.260 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:52.260 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:52.260 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:52.260 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:52.260 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:52.260 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:52.260 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:52.260 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:52.260 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:52.260 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:52.260 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:52.260 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:52.260 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:52.261 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:52.261 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:52.261 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:52.261 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:52.261 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:52.261 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:52.261 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:52.261 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:52.261 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:52.261 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:52.261 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:52.261 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:52.261 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:52.261 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:52.262 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:52.262 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:52.262 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:52.262 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:52.262 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:52.262 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:52.262 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:52.262 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:52.262 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:52.262 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:52.262 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:52.262 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:52.262 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:52.262 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:52.263 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:52.263 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:52.263 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.263 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:52.263 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:52.263 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:52.263 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:52.263 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:52.263 INFO:tasks.workunit.client.0.smithi136.stdout: 6 2024-09-12T23:37:52.263 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.263 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 5, 2024-09-12T23:37:52.263 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:52.263 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.263 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:52.264 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:52.264 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:52.264 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:52.264 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:52.264 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.264 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 7, 2024-09-12T23:37:52.264 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:52.264 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:52.264 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:52.264 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:52.264 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:52.264 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:52.264 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.664158+0000", 2024-09-12T23:37:52.265 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.901755+0000", 2024-09-12T23:37:52.265 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.901755+0000", 2024-09-12T23:37:52.265 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.902224+0000", 2024-09-12T23:37:52.265 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.664158+0000", 2024-09-12T23:37:52.265 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:52.265 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:52.265 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:52.265 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:52.265 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.664158+0000", 2024-09-12T23:37:52.265 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.664158+0000", 2024-09-12T23:37:52.265 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.664158+0000", 2024-09-12T23:37:52.265 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:52.265 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:52.265 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:52.266 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.664158+0000", 2024-09-12T23:37:52.266 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.664158+0000", 2024-09-12T23:37:52.266 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:52.266 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 2, 2024-09-12T23:37:52.266 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:52.266 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:52.266 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:52.266 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:52.266 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:52.266 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:52.266 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:52.266 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 2, 2024-09-12T23:37:52.266 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:52.266 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:52.267 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:52.267 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.1c", 2024-09-12T23:37:52.267 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:52.267 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:52.267 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:52.267 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 23, 2024-09-12T23:37:52.267 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:52.267 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T08:06:15.504175+0000", 2024-09-12T23:37:52.267 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:52.267 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:52.267 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:52.267 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:52.267 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:52.267 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:52.267 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:52.268 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:52.268 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:52.268 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:52.268 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:52.268 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:52.268 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:52.268 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:52.268 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:52.268 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:52.268 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:52.268 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:52.268 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:52.268 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 3, 2024-09-12T23:37:52.268 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 1, 2024-09-12T23:37:52.269 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:52.269 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 1, 2024-09-12T23:37:52.269 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:52.269 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:52.269 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:52.269 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:52.269 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:52.269 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 1, 2024-09-12T23:37:52.269 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:52.269 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:52.269 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:52.269 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:52.269 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:52.269 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:52.270 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:52.270 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:52.270 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:52.270 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:52.270 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:52.270 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:52.270 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 3, 2024-09-12T23:37:52.270 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:52.270 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.270 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:52.270 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:52.270 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:52.270 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:52.270 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:52.271 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:52.271 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.271 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 7, 2024-09-12T23:37:52.271 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "53'2" 2024-09-12T23:37:52.271 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.271 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:52.271 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:52.271 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:52.271 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:52.271 INFO:tasks.workunit.client.0.smithi136.stdout: 1 2024-09-12T23:37:52.271 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.271 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 5, 2024-09-12T23:37:52.271 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:52.271 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:52.271 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:52.272 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:52.272 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:52.272 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:52.272 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.670603+0000", 2024-09-12T23:37:52.272 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.913521+0000", 2024-09-12T23:37:52.272 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.913521+0000", 2024-09-12T23:37:52.272 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.913814+0000", 2024-09-12T23:37:52.272 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.670603+0000", 2024-09-12T23:37:52.272 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:52.272 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:52.272 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:52.272 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:52.272 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.670603+0000", 2024-09-12T23:37:52.272 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.670603+0000", 2024-09-12T23:37:52.273 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.670603+0000", 2024-09-12T23:37:52.273 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:52.273 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:52.273 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:52.273 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.670603+0000", 2024-09-12T23:37:52.273 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.670603+0000", 2024-09-12T23:37:52.273 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:52.273 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:52.273 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:52.273 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:52.273 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:52.273 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:52.273 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:52.273 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:52.274 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:52.274 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:52.274 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:52.274 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:52.274 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:52.274 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.1c", 2024-09-12T23:37:52.274 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:52.274 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:52.274 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:52.274 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:52.274 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:52.274 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T04:30:35.754521+0000", 2024-09-12T23:37:52.274 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:52.274 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:52.274 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:52.275 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:52.275 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:52.275 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:52.275 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:52.275 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:52.275 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:52.275 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:52.275 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:52.275 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:52.275 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:52.275 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:52.275 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:52.275 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:52.275 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:52.276 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:52.276 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:52.276 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:52.276 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:52.276 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:52.276 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:52.276 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:52.276 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:52.276 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:52.276 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:52.276 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:52.276 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:52.276 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:52.276 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:52.277 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:52.277 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:52.277 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:52.277 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:52.277 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:52.277 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:52.277 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:52.277 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:52.277 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:52.277 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:52.277 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:52.277 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:52.277 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.277 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:52.278 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:52.278 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:52.278 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:52.278 INFO:tasks.workunit.client.0.smithi136.stdout: 4, 2024-09-12T23:37:52.278 INFO:tasks.workunit.client.0.smithi136.stdout: 1 2024-09-12T23:37:52.278 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.278 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 5, 2024-09-12T23:37:52.278 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:52.278 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.278 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:52.278 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:52.278 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:52.278 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:52.279 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:52.279 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.279 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 7, 2024-09-12T23:37:52.279 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:52.279 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:52.279 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:52.279 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:52.279 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:52.279 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:52.279 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.666037+0000", 2024-09-12T23:37:52.279 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.896936+0000", 2024-09-12T23:37:52.279 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.896936+0000", 2024-09-12T23:37:52.279 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.897466+0000", 2024-09-12T23:37:52.279 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.666037+0000", 2024-09-12T23:37:52.279 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:52.280 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:52.280 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:52.280 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:52.280 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.666037+0000", 2024-09-12T23:37:52.280 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.666037+0000", 2024-09-12T23:37:52.280 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.666037+0000", 2024-09-12T23:37:52.280 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:52.280 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:52.280 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:52.280 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.666037+0000", 2024-09-12T23:37:52.280 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.666037+0000", 2024-09-12T23:37:52.280 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:52.280 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 2, 2024-09-12T23:37:52.280 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:52.281 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:52.281 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:52.281 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:52.281 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:52.281 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:52.281 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:52.281 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 2, 2024-09-12T23:37:52.281 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:52.281 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:52.281 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:52.281 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.1d", 2024-09-12T23:37:52.281 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:52.281 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:52.282 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:52.282 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 23, 2024-09-12T23:37:52.282 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:52.282 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T10:29:38.732168+0000", 2024-09-12T23:37:52.282 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:52.282 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:52.282 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:52.282 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:52.282 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:52.282 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:52.282 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:52.282 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:52.282 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:52.282 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:52.283 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:52.283 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:52.283 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:52.283 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:52.283 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:52.283 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:52.283 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:52.283 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:52.283 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:52.283 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 3, 2024-09-12T23:37:52.283 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 1, 2024-09-12T23:37:52.283 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:52.283 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 1, 2024-09-12T23:37:52.283 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:52.284 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:52.284 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:52.284 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:52.284 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:52.284 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 1, 2024-09-12T23:37:52.284 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:52.284 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:52.284 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:52.284 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:52.284 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:52.284 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:52.284 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:52.284 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:52.284 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:52.284 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:52.285 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:52.285 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:52.285 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 3, 2024-09-12T23:37:52.285 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:52.285 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.285 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:52.285 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:52.285 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:52.285 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:52.285 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:52.285 INFO:tasks.workunit.client.0.smithi136.stdout: 0 2024-09-12T23:37:52.285 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.285 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 7, 2024-09-12T23:37:52.285 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "53'2" 2024-09-12T23:37:52.286 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.286 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:52.286 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:52.286 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:52.286 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:52.286 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:52.286 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.286 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 0, 2024-09-12T23:37:52.286 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:52.286 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:52.286 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:52.286 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:52.286 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:52.287 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:52.287 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.670437+0000", 2024-09-12T23:37:52.287 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.924320+0000", 2024-09-12T23:37:52.287 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.924320+0000", 2024-09-12T23:37:52.287 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.924610+0000", 2024-09-12T23:37:52.287 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.670437+0000", 2024-09-12T23:37:52.287 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:52.287 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:52.287 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:52.287 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:52.287 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.670437+0000", 2024-09-12T23:37:52.287 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.670437+0000", 2024-09-12T23:37:52.287 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.670437+0000", 2024-09-12T23:37:52.287 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:52.288 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:52.288 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:52.288 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.670437+0000", 2024-09-12T23:37:52.288 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.670437+0000", 2024-09-12T23:37:52.288 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:52.288 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:52.288 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:52.288 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:52.288 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:52.288 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:52.288 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:52.288 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:52.288 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:52.289 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:52.289 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:52.289 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:52.289 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:52.289 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.1f", 2024-09-12T23:37:52.289 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:52.289 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:52.289 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:52.289 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:52.289 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:52.289 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T06:23:22.797301+0000", 2024-09-12T23:37:52.289 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:52.289 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:52.290 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:52.290 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:52.290 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:52.290 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:52.290 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:52.290 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:52.290 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:52.290 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:52.290 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:52.290 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:52.290 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:52.290 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:52.290 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:52.290 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:52.291 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:52.291 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:52.291 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:52.291 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:52.291 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:52.291 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:52.291 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:52.291 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:52.291 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:52.291 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:52.291 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:52.291 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:52.291 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:52.291 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:52.292 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:52.292 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:52.292 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:52.292 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:52.292 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:52.292 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:52.292 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:52.292 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:52.292 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:52.292 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:52.292 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:52.292 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:52.292 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:52.292 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.293 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:52.293 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:52.293 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:52.293 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:52.293 INFO:tasks.workunit.client.0.smithi136.stdout: 5, 2024-09-12T23:37:52.293 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:52.293 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.293 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 0, 2024-09-12T23:37:52.293 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:52.293 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.293 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:52.293 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:52.293 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:52.294 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:52.294 INFO:tasks.workunit.client.0.smithi136.stdout: 5 2024-09-12T23:37:52.294 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.294 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 3, 2024-09-12T23:37:52.294 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:52.294 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:52.294 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:52.294 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:52.294 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:52.294 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:52.294 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.673262+0000", 2024-09-12T23:37:52.294 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.893992+0000", 2024-09-12T23:37:52.294 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.893992+0000", 2024-09-12T23:37:52.294 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.894353+0000", 2024-09-12T23:37:52.295 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.673262+0000", 2024-09-12T23:37:52.295 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:52.295 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:52.295 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:52.295 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:52.295 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.673262+0000", 2024-09-12T23:37:52.295 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.673262+0000", 2024-09-12T23:37:52.295 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.673262+0000", 2024-09-12T23:37:52.295 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:52.295 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:52.295 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:52.295 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.673262+0000", 2024-09-12T23:37:52.295 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.673262+0000", 2024-09-12T23:37:52.295 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:52.296 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 1, 2024-09-12T23:37:52.296 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:52.296 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:52.296 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:52.296 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:52.296 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:52.296 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:52.296 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:52.296 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 1, 2024-09-12T23:37:52.296 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:52.296 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:52.296 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:52.296 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.1e", 2024-09-12T23:37:52.297 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:52.297 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:52.297 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:52.297 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 22, 2024-09-12T23:37:52.297 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:52.297 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T01:43:43.889025+0000", 2024-09-12T23:37:52.297 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:52.297 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:52.297 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:52.297 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:52.297 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:52.297 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:52.297 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:52.297 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:52.298 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:52.298 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:52.298 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:52.298 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:52.298 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:52.298 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:52.298 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:52.298 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:52.298 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:52.298 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:52.298 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:52.298 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:52.298 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:52.298 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:52.298 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:52.299 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:52.299 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:52.299 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:52.299 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:52.299 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:52.299 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:52.299 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:52.299 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:52.299 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:52.299 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:52.299 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:52.299 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:52.299 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:52.300 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:52.300 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:52.300 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:52.300 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:52.300 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:52.300 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:52.300 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:52.300 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.300 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:52.300 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:52.300 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:52.300 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:52.300 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:52.301 INFO:tasks.workunit.client.0.smithi136.stdout: 5 2024-09-12T23:37:52.301 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.301 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 3, 2024-09-12T23:37:52.301 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "53'1" 2024-09-12T23:37:52.301 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.301 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:52.301 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:52.301 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:52.301 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:52.301 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:52.301 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.301 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 3, 2024-09-12T23:37:52.301 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:52.301 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:52.302 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 52, 2024-09-12T23:37:52.302 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:52.302 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:52.302 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:52.302 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.673097+0000", 2024-09-12T23:37:52.302 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:29.890368+0000", 2024-09-12T23:37:52.302 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:29.890368+0000", 2024-09-12T23:37:52.302 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:29.891227+0000", 2024-09-12T23:37:52.302 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.673097+0000", 2024-09-12T23:37:52.302 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:52.302 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:52.302 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:52.302 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 53, 2024-09-12T23:37:52.302 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.673097+0000", 2024-09-12T23:37:52.303 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.673097+0000", 2024-09-12T23:37:52.303 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.673097+0000", 2024-09-12T23:37:52.303 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:52.303 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:52.303 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:28.834450+0000", 2024-09-12T23:37:52.303 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.673097+0000", 2024-09-12T23:37:52.303 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.673097+0000", 2024-09-12T23:37:52.303 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:52.303 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:52.303 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:52.303 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:52.303 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 52, 2024-09-12T23:37:52.303 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:52.303 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:52.304 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:52.304 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:52.304 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:52.304 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:52.304 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:52.304 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:52.304 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "3.1e", 2024-09-12T23:37:52.304 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:52.304 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:52.304 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:52.304 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 20, 2024-09-12T23:37:52.304 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:52.304 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T05:38:37.526610+0000", 2024-09-12T23:37:52.305 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:52.305 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:52.305 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:52.305 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:52.305 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:52.305 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:52.305 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:52.305 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:52.305 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:52.305 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:52.305 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:52.305 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:52.305 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:52.306 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:52.306 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:52.306 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:52.306 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:52.306 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:52.306 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:52.306 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:52.306 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:52.306 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:52.306 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:52.306 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:52.306 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:52.306 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:52.306 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:52.307 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:52.307 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:52.307 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:52.307 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:52.307 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:52.307 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:52.307 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:52.307 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:52.307 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:52.307 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:52.307 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:52.307 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:52.307 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:52.307 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:52.308 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:52.308 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:52.308 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.308 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:52.308 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:52.308 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:52.308 INFO:tasks.workunit.client.0.smithi136.stdout: 3, 2024-09-12T23:37:52.308 INFO:tasks.workunit.client.0.smithi136.stdout: 6, 2024-09-12T23:37:52.308 INFO:tasks.workunit.client.0.smithi136.stdout: 2 2024-09-12T23:37:52.308 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.308 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 3, 2024-09-12T23:37:52.308 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "0'0" 2024-09-12T23:37:52.308 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.308 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:52.309 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": [ 2024-09-12T23:37:52.309 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:52.309 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:52.309 INFO:tasks.workunit.client.0.smithi136.stdout: 4 2024-09-12T23:37:52.309 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.309 INFO:tasks.workunit.client.0.smithi136.stdout: "acting_primary": 0, 2024-09-12T23:37:52.309 INFO:tasks.workunit.client.0.smithi136.stdout: "avail_no_missing": [], 2024-09-12T23:37:52.309 INFO:tasks.workunit.client.0.smithi136.stdout: "blocked_by": [], 2024-09-12T23:37:52.309 INFO:tasks.workunit.client.0.smithi136.stdout: "created": 51, 2024-09-12T23:37:52.309 INFO:tasks.workunit.client.0.smithi136.stdout: "dirty_stats_invalid": false, 2024-09-12T23:37:52.309 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_bytes_stats_invalid": false, 2024-09-12T23:37:52.309 INFO:tasks.workunit.client.0.smithi136.stdout: "hitset_stats_invalid": false, 2024-09-12T23:37:52.309 INFO:tasks.workunit.client.0.smithi136.stdout: "last_active": "2024-09-12T23:35:08.670421+0000", 2024-09-12T23:37:52.309 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_active": "2024-09-12T23:33:28.885720+0000", 2024-09-12T23:37:52.310 INFO:tasks.workunit.client.0.smithi136.stdout: "last_became_peered": "2024-09-12T23:33:28.885720+0000", 2024-09-12T23:37:52.310 INFO:tasks.workunit.client.0.smithi136.stdout: "last_change": "2024-09-12T23:33:28.886016+0000", 2024-09-12T23:37:52.310 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean": "2024-09-12T23:35:08.670421+0000", 2024-09-12T23:37:52.310 INFO:tasks.workunit.client.0.smithi136.stdout: "last_clean_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:52.310 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub": "0'0", 2024-09-12T23:37:52.310 INFO:tasks.workunit.client.0.smithi136.stdout: "last_deep_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:52.310 INFO:tasks.workunit.client.0.smithi136.stdout: "last_epoch_clean": 52, 2024-09-12T23:37:52.310 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fresh": "2024-09-12T23:35:08.670421+0000", 2024-09-12T23:37:52.310 INFO:tasks.workunit.client.0.smithi136.stdout: "last_fullsized": "2024-09-12T23:35:08.670421+0000", 2024-09-12T23:37:52.310 INFO:tasks.workunit.client.0.smithi136.stdout: "last_peered": "2024-09-12T23:35:08.670421+0000", 2024-09-12T23:37:52.310 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub": "0'0", 2024-09-12T23:37:52.310 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_duration": 0, 2024-09-12T23:37:52.310 INFO:tasks.workunit.client.0.smithi136.stdout: "last_scrub_stamp": "2024-09-12T23:33:27.830370+0000", 2024-09-12T23:37:52.310 INFO:tasks.workunit.client.0.smithi136.stdout: "last_undegraded": "2024-09-12T23:35:08.670421+0000", 2024-09-12T23:37:52.311 INFO:tasks.workunit.client.0.smithi136.stdout: "last_unstale": "2024-09-12T23:35:08.670421+0000", 2024-09-12T23:37:52.311 INFO:tasks.workunit.client.0.smithi136.stdout: "log_dups_size": 0, 2024-09-12T23:37:52.311 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 4, 2024-09-12T23:37:52.311 INFO:tasks.workunit.client.0.smithi136.stdout: "log_start": "0'0", 2024-09-12T23:37:52.311 INFO:tasks.workunit.client.0.smithi136.stdout: "manifest_stats_invalid": false, 2024-09-12T23:37:52.311 INFO:tasks.workunit.client.0.smithi136.stdout: "mapping_epoch": 51, 2024-09-12T23:37:52.311 INFO:tasks.workunit.client.0.smithi136.stdout: "object_location_counts": [], 2024-09-12T23:37:52.311 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_scrubbed": 0, 2024-09-12T23:37:52.311 INFO:tasks.workunit.client.0.smithi136.stdout: "objects_trimmed": 0, 2024-09-12T23:37:52.311 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_stats_invalid": false, 2024-09-12T23:37:52.311 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 4, 2024-09-12T23:37:52.311 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_start": "0'0", 2024-09-12T23:37:52.311 INFO:tasks.workunit.client.0.smithi136.stdout: "parent": "0.0", 2024-09-12T23:37:52.311 INFO:tasks.workunit.client.0.smithi136.stdout: "parent_split_bits": 0, 2024-09-12T23:37:52.312 INFO:tasks.workunit.client.0.smithi136.stdout: "pgid": "2.1f", 2024-09-12T23:37:52.312 INFO:tasks.workunit.client.0.smithi136.stdout: "pin_stats_invalid": false, 2024-09-12T23:37:52.312 INFO:tasks.workunit.client.0.smithi136.stdout: "purged_snaps": [], 2024-09-12T23:37:52.312 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_epoch": 63, 2024-09-12T23:37:52.312 INFO:tasks.workunit.client.0.smithi136.stdout: "reported_seq": 25, 2024-09-12T23:37:52.312 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_duration": 0.0, 2024-09-12T23:37:52.312 INFO:tasks.workunit.client.0.smithi136.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T03:44:38.950039+0000", 2024-09-12T23:37:52.312 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrim_duration": 0.0, 2024-09-12T23:37:52.312 INFO:tasks.workunit.client.0.smithi136.stdout: "snaptrimq_len": 0, 2024-09-12T23:37:52.312 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:52.312 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 570, 2024-09-12T23:37:52.312 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:52.312 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:52.313 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:52.313 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:52.313 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:52.313 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:52.313 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:52.313 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:52.313 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:52.313 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:52.313 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:52.313 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:52.313 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:52.313 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:52.313 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:52.313 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 6, 2024-09-12T23:37:52.314 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 2, 2024-09-12T23:37:52.314 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:52.314 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 2, 2024-09-12T23:37:52.314 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:52.314 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:52.314 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:52.314 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:52.314 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:52.314 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 1, 2024-09-12T23:37:52.314 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:52.314 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:52.314 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:52.314 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:52.315 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:52.315 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:52.315 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:52.315 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:52.315 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:52.315 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:52.315 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:52.315 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:52.315 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 4, 2024-09-12T23:37:52.315 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 1 2024-09-12T23:37:52.315 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.315 INFO:tasks.workunit.client.0.smithi136.stdout: "state": "active+clean", 2024-09-12T23:37:52.315 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_invalid": false, 2024-09-12T23:37:52.315 INFO:tasks.workunit.client.0.smithi136.stdout: "up": [ 2024-09-12T23:37:52.315 INFO:tasks.workunit.client.0.smithi136.stdout: 0, 2024-09-12T23:37:52.316 INFO:tasks.workunit.client.0.smithi136.stdout: 7, 2024-09-12T23:37:52.316 INFO:tasks.workunit.client.0.smithi136.stdout: 4 2024-09-12T23:37:52.316 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.316 INFO:tasks.workunit.client.0.smithi136.stdout: "up_primary": 0, 2024-09-12T23:37:52.316 INFO:tasks.workunit.client.0.smithi136.stdout: "version": "53'4" 2024-09-12T23:37:52.316 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:52.316 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.316 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_per_pool": [ 2024-09-12T23:37:52.316 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:52.316 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": 24, 2024-09-12T23:37:52.316 INFO:tasks.workunit.client.0.smithi136.stdout: "application": [ 2024-09-12T23:37:52.316 INFO:tasks.workunit.client.0.smithi136.stdout: "rbd" 2024-09-12T23:37:52.316 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.317 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 3, 2024-09-12T23:37:52.317 INFO:tasks.workunit.client.0.smithi136.stdout: "num_pg": 8, 2024-09-12T23:37:52.317 INFO:tasks.workunit.client.0.smithi136.stdout: "num_store_stats": 8, 2024-09-12T23:37:52.317 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 3, 2024-09-12T23:37:52.317 INFO:tasks.workunit.client.0.smithi136.stdout: "poolid": 4, 2024-09-12T23:37:52.317 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:52.317 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 19, 2024-09-12T23:37:52.317 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:52.317 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:52.317 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:52.317 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:52.317 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:52.317 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:52.317 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:52.318 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:52.318 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:52.318 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:52.318 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:52.318 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:52.318 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:52.318 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:52.318 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:52.318 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 6, 2024-09-12T23:37:52.318 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 2, 2024-09-12T23:37:52.318 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:52.318 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 2, 2024-09-12T23:37:52.318 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:52.318 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:52.319 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:52.319 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:52.319 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:52.319 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:52.319 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:52.319 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:52.319 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:52.319 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:52.319 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:52.319 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:52.319 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:52.319 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:52.319 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:52.319 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:52.319 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:52.320 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:52.320 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 2, 2024-09-12T23:37:52.320 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 2 2024-09-12T23:37:52.320 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.320 INFO:tasks.workunit.client.0.smithi136.stdout: "store_stats": { 2024-09-12T23:37:52.320 INFO:tasks.workunit.client.0.smithi136.stdout: "allocated": 12288, 2024-09-12T23:37:52.320 INFO:tasks.workunit.client.0.smithi136.stdout: "available": 0, 2024-09-12T23:37:52.320 INFO:tasks.workunit.client.0.smithi136.stdout: "data_compressed": 0, 2024-09-12T23:37:52.320 INFO:tasks.workunit.client.0.smithi136.stdout: "data_compressed_allocated": 0, 2024-09-12T23:37:52.320 INFO:tasks.workunit.client.0.smithi136.stdout: "data_compressed_original": 0, 2024-09-12T23:37:52.320 INFO:tasks.workunit.client.0.smithi136.stdout: "data_stored": 57, 2024-09-12T23:37:52.320 INFO:tasks.workunit.client.0.smithi136.stdout: "internal_metadata": 0, 2024-09-12T23:37:52.321 INFO:tasks.workunit.client.0.smithi136.stdout: "internally_reserved": 0, 2024-09-12T23:37:52.321 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_allocated": 0, 2024-09-12T23:37:52.321 INFO:tasks.workunit.client.0.smithi136.stdout: "total": 0 2024-09-12T23:37:52.321 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.321 INFO:tasks.workunit.client.0.smithi136.stdout: "up": 24 2024-09-12T23:37:52.321 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.321 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:52.321 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": 192, 2024-09-12T23:37:52.321 INFO:tasks.workunit.client.0.smithi136.stdout: "application": [ 2024-09-12T23:37:52.321 INFO:tasks.workunit.client.0.smithi136.stdout: "cephfs" 2024-09-12T23:37:52.321 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.321 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 0, 2024-09-12T23:37:52.321 INFO:tasks.workunit.client.0.smithi136.stdout: "num_pg": 64, 2024-09-12T23:37:52.321 INFO:tasks.workunit.client.0.smithi136.stdout: "num_store_stats": 8, 2024-09-12T23:37:52.322 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 0, 2024-09-12T23:37:52.322 INFO:tasks.workunit.client.0.smithi136.stdout: "poolid": 3, 2024-09-12T23:37:52.322 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:52.322 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 0, 2024-09-12T23:37:52.322 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:52.322 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:52.322 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:52.322 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:52.322 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:52.322 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:52.322 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:52.322 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:52.322 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:52.322 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:52.323 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:52.323 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:52.323 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:52.323 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:52.323 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:52.323 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 0, 2024-09-12T23:37:52.323 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 0, 2024-09-12T23:37:52.323 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:52.323 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 0, 2024-09-12T23:37:52.323 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:52.323 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:52.323 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:52.323 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:52.324 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:52.324 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:52.324 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:52.324 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 0, 2024-09-12T23:37:52.324 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:52.324 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:52.324 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:52.324 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:52.324 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:52.324 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:52.324 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:52.324 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:52.324 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:52.325 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:52.325 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 0, 2024-09-12T23:37:52.325 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 0 2024-09-12T23:37:52.325 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.325 INFO:tasks.workunit.client.0.smithi136.stdout: "store_stats": { 2024-09-12T23:37:52.325 INFO:tasks.workunit.client.0.smithi136.stdout: "allocated": 0, 2024-09-12T23:37:52.325 INFO:tasks.workunit.client.0.smithi136.stdout: "available": 0, 2024-09-12T23:37:52.325 INFO:tasks.workunit.client.0.smithi136.stdout: "data_compressed": 0, 2024-09-12T23:37:52.325 INFO:tasks.workunit.client.0.smithi136.stdout: "data_compressed_allocated": 0, 2024-09-12T23:37:52.325 INFO:tasks.workunit.client.0.smithi136.stdout: "data_compressed_original": 0, 2024-09-12T23:37:52.325 INFO:tasks.workunit.client.0.smithi136.stdout: "data_stored": 0, 2024-09-12T23:37:52.325 INFO:tasks.workunit.client.0.smithi136.stdout: "internal_metadata": 0, 2024-09-12T23:37:52.325 INFO:tasks.workunit.client.0.smithi136.stdout: "internally_reserved": 0, 2024-09-12T23:37:52.325 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_allocated": 0, 2024-09-12T23:37:52.326 INFO:tasks.workunit.client.0.smithi136.stdout: "total": 0 2024-09-12T23:37:52.326 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.326 INFO:tasks.workunit.client.0.smithi136.stdout: "up": 192 2024-09-12T23:37:52.326 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.326 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:52.326 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": 96, 2024-09-12T23:37:52.326 INFO:tasks.workunit.client.0.smithi136.stdout: "application": [ 2024-09-12T23:37:52.326 INFO:tasks.workunit.client.0.smithi136.stdout: "cephfs" 2024-09-12T23:37:52.326 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.326 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 45, 2024-09-12T23:37:52.326 INFO:tasks.workunit.client.0.smithi136.stdout: "num_pg": 32, 2024-09-12T23:37:52.326 INFO:tasks.workunit.client.0.smithi136.stdout: "num_store_stats": 8, 2024-09-12T23:37:52.326 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 45, 2024-09-12T23:37:52.326 INFO:tasks.workunit.client.0.smithi136.stdout: "poolid": 2, 2024-09-12T23:37:52.326 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:52.327 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 3410, 2024-09-12T23:37:52.327 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:52.327 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 0, 2024-09-12T23:37:52.327 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:52.327 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:52.327 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:52.327 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:52.327 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:52.327 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:52.327 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:52.327 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:52.327 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:52.327 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:52.327 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:52.328 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:52.328 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:52.328 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 66, 2024-09-12T23:37:52.328 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 22, 2024-09-12T23:37:52.328 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:52.328 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 22, 2024-09-12T23:37:52.328 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:52.328 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:52.328 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:52.328 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:52.328 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:52.328 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 14, 2024-09-12T23:37:52.328 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:52.329 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 1, 2024-09-12T23:37:52.329 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:52.329 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:52.329 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:52.329 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:52.329 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:52.329 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 0, 2024-09-12T23:37:52.329 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 0, 2024-09-12T23:37:52.329 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:52.329 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:52.329 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:52.329 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 51, 2024-09-12T23:37:52.329 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 19 2024-09-12T23:37:52.330 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.330 INFO:tasks.workunit.client.0.smithi136.stdout: "store_stats": { 2024-09-12T23:37:52.330 INFO:tasks.workunit.client.0.smithi136.stdout: "allocated": 98304, 2024-09-12T23:37:52.330 INFO:tasks.workunit.client.0.smithi136.stdout: "available": 0, 2024-09-12T23:37:52.330 INFO:tasks.workunit.client.0.smithi136.stdout: "data_compressed": 0, 2024-09-12T23:37:52.330 INFO:tasks.workunit.client.0.smithi136.stdout: "data_compressed_allocated": 0, 2024-09-12T23:37:52.330 INFO:tasks.workunit.client.0.smithi136.stdout: "data_compressed_original": 0, 2024-09-12T23:37:52.330 INFO:tasks.workunit.client.0.smithi136.stdout: "data_stored": 10230, 2024-09-12T23:37:52.330 INFO:tasks.workunit.client.0.smithi136.stdout: "internal_metadata": 0, 2024-09-12T23:37:52.330 INFO:tasks.workunit.client.0.smithi136.stdout: "internally_reserved": 0, 2024-09-12T23:37:52.330 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_allocated": 0, 2024-09-12T23:37:52.330 INFO:tasks.workunit.client.0.smithi136.stdout: "total": 0 2024-09-12T23:37:52.330 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.330 INFO:tasks.workunit.client.0.smithi136.stdout: "up": 96 2024-09-12T23:37:52.331 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.331 INFO:tasks.workunit.client.0.smithi136.stdout: { 2024-09-12T23:37:52.331 INFO:tasks.workunit.client.0.smithi136.stdout: "acting": 3, 2024-09-12T23:37:52.331 INFO:tasks.workunit.client.0.smithi136.stdout: "application": [ 2024-09-12T23:37:52.331 INFO:tasks.workunit.client.0.smithi136.stdout: "mgr" 2024-09-12T23:37:52.331 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.331 INFO:tasks.workunit.client.0.smithi136.stdout: "log_size": 111, 2024-09-12T23:37:52.331 INFO:tasks.workunit.client.0.smithi136.stdout: "num_pg": 1, 2024-09-12T23:37:52.331 INFO:tasks.workunit.client.0.smithi136.stdout: "num_store_stats": 3, 2024-09-12T23:37:52.331 INFO:tasks.workunit.client.0.smithi136.stdout: "ondisk_log_size": 111, 2024-09-12T23:37:52.331 INFO:tasks.workunit.client.0.smithi136.stdout: "poolid": 1, 2024-09-12T23:37:52.331 INFO:tasks.workunit.client.0.smithi136.stdout: "stat_sum": { 2024-09-12T23:37:52.331 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes": 590368, 2024-09-12T23:37:52.331 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_hit_set_archive": 0, 2024-09-12T23:37:52.332 INFO:tasks.workunit.client.0.smithi136.stdout: "num_bytes_recovered": 590368, 2024-09-12T23:37:52.332 INFO:tasks.workunit.client.0.smithi136.stdout: "num_deep_scrub_errors": 0, 2024-09-12T23:37:52.332 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict": 0, 2024-09-12T23:37:52.332 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_kb": 0, 2024-09-12T23:37:52.332 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_full": 0, 2024-09-12T23:37:52.332 INFO:tasks.workunit.client.0.smithi136.stdout: "num_evict_mode_some": 0, 2024-09-12T23:37:52.332 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush": 0, 2024-09-12T23:37:52.332 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_kb": 0, 2024-09-12T23:37:52.332 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_high": 0, 2024-09-12T23:37:52.332 INFO:tasks.workunit.client.0.smithi136.stdout: "num_flush_mode_low": 0, 2024-09-12T23:37:52.332 INFO:tasks.workunit.client.0.smithi136.stdout: "num_keys_recovered": 0, 2024-09-12T23:37:52.332 INFO:tasks.workunit.client.0.smithi136.stdout: "num_large_omap_objects": 0, 2024-09-12T23:37:52.332 INFO:tasks.workunit.client.0.smithi136.stdout: "num_legacy_snapsets": 0, 2024-09-12T23:37:52.332 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_clones": 0, 2024-09-12T23:37:52.333 INFO:tasks.workunit.client.0.smithi136.stdout: "num_object_copies": 6, 2024-09-12T23:37:52.333 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects": 2, 2024-09-12T23:37:52.333 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_degraded": 0, 2024-09-12T23:37:52.333 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_dirty": 2, 2024-09-12T23:37:52.333 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_hit_set_archive": 0, 2024-09-12T23:37:52.333 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_manifest": 0, 2024-09-12T23:37:52.333 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_misplaced": 0, 2024-09-12T23:37:52.333 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing": 0, 2024-09-12T23:37:52.333 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_missing_on_primary": 0, 2024-09-12T23:37:52.333 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_omap": 0, 2024-09-12T23:37:52.333 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_pinned": 0, 2024-09-12T23:37:52.333 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_recovered": 2, 2024-09-12T23:37:52.333 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_repaired": 0, 2024-09-12T23:37:52.333 INFO:tasks.workunit.client.0.smithi136.stdout: "num_objects_unfound": 0, 2024-09-12T23:37:52.334 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_bytes": 0, 2024-09-12T23:37:52.334 INFO:tasks.workunit.client.0.smithi136.stdout: "num_omap_keys": 0, 2024-09-12T23:37:52.334 INFO:tasks.workunit.client.0.smithi136.stdout: "num_promote": 0, 2024-09-12T23:37:52.334 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read": 416, 2024-09-12T23:37:52.334 INFO:tasks.workunit.client.0.smithi136.stdout: "num_read_kb": 599, 2024-09-12T23:37:52.334 INFO:tasks.workunit.client.0.smithi136.stdout: "num_scrub_errors": 0, 2024-09-12T23:37:52.334 INFO:tasks.workunit.client.0.smithi136.stdout: "num_shallow_scrub_errors": 0, 2024-09-12T23:37:52.334 INFO:tasks.workunit.client.0.smithi136.stdout: "num_whiteouts": 0, 2024-09-12T23:37:52.334 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write": 177, 2024-09-12T23:37:52.334 INFO:tasks.workunit.client.0.smithi136.stdout: "num_write_kb": 1628 2024-09-12T23:37:52.334 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.334 INFO:tasks.workunit.client.0.smithi136.stdout: "store_stats": { 2024-09-12T23:37:52.334 INFO:tasks.workunit.client.0.smithi136.stdout: "allocated": 1781760, 2024-09-12T23:37:52.334 INFO:tasks.workunit.client.0.smithi136.stdout: "available": 0, 2024-09-12T23:37:52.335 INFO:tasks.workunit.client.0.smithi136.stdout: "data_compressed": 0, 2024-09-12T23:37:52.335 INFO:tasks.workunit.client.0.smithi136.stdout: "data_compressed_allocated": 0, 2024-09-12T23:37:52.335 INFO:tasks.workunit.client.0.smithi136.stdout: "data_compressed_original": 0, 2024-09-12T23:37:52.335 INFO:tasks.workunit.client.0.smithi136.stdout: "data_stored": 1771104, 2024-09-12T23:37:52.335 INFO:tasks.workunit.client.0.smithi136.stdout: "internal_metadata": 0, 2024-09-12T23:37:52.335 INFO:tasks.workunit.client.0.smithi136.stdout: "internally_reserved": 0, 2024-09-12T23:37:52.335 INFO:tasks.workunit.client.0.smithi136.stdout: "omap_allocated": 0, 2024-09-12T23:37:52.335 INFO:tasks.workunit.client.0.smithi136.stdout: "total": 0 2024-09-12T23:37:52.335 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.335 INFO:tasks.workunit.client.0.smithi136.stdout: "up": 3 2024-09-12T23:37:52.335 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:52.335 INFO:tasks.workunit.client.0.smithi136.stdout: ], 2024-09-12T23:37:52.335 INFO:tasks.workunit.client.0.smithi136.stdout: "usage": { 2024-09-12T23:37:52.335 INFO:tasks.workunit.client.0.smithi136.stdout: "pg_num": 105, 2024-09-12T23:37:52.335 INFO:tasks.workunit.client.0.smithi136.stdout: "pools": 4, 2024-09-12T23:37:52.336 INFO:tasks.workunit.client.0.smithi136.stdout: "stats_by_class": { 2024-09-12T23:37:52.336 INFO:tasks.workunit.client.0.smithi136.stdout: "ssd": { 2024-09-12T23:37:52.336 INFO:tasks.workunit.client.0.smithi136.stdout: "total_avail_bytes": 765517275136, 2024-09-12T23:37:52.336 INFO:tasks.workunit.client.0.smithi136.stdout: "total_bytes": 767960285184, 2024-09-12T23:37:52.336 INFO:tasks.workunit.client.0.smithi136.stdout: "total_used_bytes": 2443010048, 2024-09-12T23:37:52.336 INFO:tasks.workunit.client.0.smithi136.stdout: "total_used_raw_bytes": 2443010048, 2024-09-12T23:37:52.336 INFO:tasks.workunit.client.0.smithi136.stdout: "total_used_raw_ratio": 0.0031811671797186136 2024-09-12T23:37:52.336 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:52.336 INFO:tasks.workunit.client.0.smithi136.stdout: }, 2024-09-12T23:37:52.336 INFO:tasks.workunit.client.0.smithi136.stdout: "total_avail_bytes": 765517275136, 2024-09-12T23:37:52.336 INFO:tasks.workunit.client.0.smithi136.stdout: "total_bytes": 767960285184, 2024-09-12T23:37:52.336 INFO:tasks.workunit.client.0.smithi136.stdout: "total_used_bytes": 2443010048 2024-09-12T23:37:52.336 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:52.336 INFO:tasks.workunit.client.0.smithi136.stdout: } 2024-09-12T23:37:52.336 INFO:tasks.workunit.client.0.smithi136.stdout:} 2024-09-12T23:37:52.337 INFO:tasks.workunit.client.0.smithi136.stdout:OK 2024-09-12T23:37:52.337 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-12T23:37:52.337 DEBUG:teuthology.orchestra.run.smithi136:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-12T23:37:52.338 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:48.982944+0000 mgr.y (mgr.24446) 161 : audit [DBG] from='client.14922 -' entity='client.admin' cmd=[{"prefix": "telemetry show-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:52.338 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:49.158481+0000 mgr.y (mgr.24446) 162 : audit [DBG] from='client.14922 -' entity='client.admin' cmd=[{"prefix": "telemetry show-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:52.338 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:49.531538+0000 mgr.y (mgr.24446) 163 : audit [DBG] from='client.24842 -' entity='client.admin' cmd=[{"prefix": "telemetry show-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:52.338 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:49.533958+0000 mon.a (mon.0) 1019 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:37:52.338 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:49.539773+0000 mon.a (mon.0) 1020 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-12T23:37:52.338 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:49.708317+0000 mon.a (mon.0) 1021 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:52.338 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:49.708730+0000 mon.a (mon.0) 1022 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:52.338 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:49.709061+0000 mon.a (mon.0) 1023 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:52.338 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:49.712354+0000 mon.a (mon.0) 1024 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:52.338 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:49.712801+0000 mon.a (mon.0) 1025 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:52.338 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:49.713255+0000 mon.a (mon.0) 1026 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:52.339 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:49.716555+0000 mon.a (mon.0) 1027 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:52.339 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:49.717028+0000 mon.a (mon.0) 1028 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:52.339 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:49.717383+0000 mon.a (mon.0) 1029 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:52.339 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:49.734268+0000 mon.a (mon.0) 1030 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:37:52.339 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:49.734363+0000 mon.a (mon.0) 1031 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:37:52.339 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:49.734812+0000 mon.a (mon.0) 1032 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:52.339 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:49.735249+0000 mon.a (mon.0) 1033 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:52.339 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:48.982944+0000 mgr.y (mgr.24446) 161 : audit [DBG] from='client.14922 -' entity='client.admin' cmd=[{"prefix": "telemetry show-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:52.339 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:49.158481+0000 mgr.y (mgr.24446) 162 : audit [DBG] from='client.14922 -' entity='client.admin' cmd=[{"prefix": "telemetry show-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:52.340 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:49.531538+0000 mgr.y (mgr.24446) 163 : audit [DBG] from='client.24842 -' entity='client.admin' cmd=[{"prefix": "telemetry show-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:52.340 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:49.533958+0000 mon.a (mon.0) 1019 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:37:52.340 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:49.539773+0000 mon.a (mon.0) 1020 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-12T23:37:52.340 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:49.708317+0000 mon.a (mon.0) 1021 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:52.340 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:49.708730+0000 mon.a (mon.0) 1022 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:52.340 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:49.709061+0000 mon.a (mon.0) 1023 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:52.340 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:49.712354+0000 mon.a (mon.0) 1024 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:52.340 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:49.712801+0000 mon.a (mon.0) 1025 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:52.340 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:49.713255+0000 mon.a (mon.0) 1026 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:52.340 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:49.716555+0000 mon.a (mon.0) 1027 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:52.340 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:49.717028+0000 mon.a (mon.0) 1028 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:52.340 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:49.717383+0000 mon.a (mon.0) 1029 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:52.340 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:49.734268+0000 mon.a (mon.0) 1030 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:37:52.341 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:49.734363+0000 mon.a (mon.0) 1031 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:37:52.341 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:49.734812+0000 mon.a (mon.0) 1032 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:52.341 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:49.735249+0000 mon.a (mon.0) 1033 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:52.341 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:49.735647+0000 mon.a (mon.0) 1034 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:52.341 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:49.738169+0000 mon.b (mon.1) 32 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:37:52.341 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:49.738278+0000 mon.b (mon.1) 33 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:37:52.341 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:49.738882+0000 mon.a (mon.0) 1035 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:52.341 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:49.739463+0000 mon.a (mon.0) 1036 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:52.341 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:49.739752+0000 mon.a (mon.0) 1037 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:52.341 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:49.742337+0000 mon.c (mon.2) 55 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:37:52.341 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:49.742433+0000 mon.c (mon.2) 56 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:37:52.341 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:49.742861+0000 mon.a (mon.0) 1038 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:52.341 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:49.743604+0000 mon.a (mon.0) 1039 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:52.342 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:49.744064+0000 mon.a (mon.0) 1040 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:52.342 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:50.017366+0000 mon.a (mon.0) 1041 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:37:52.342 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:50.021067+0000 mon.a (mon.0) 1042 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-12T23:37:52.342 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:50.155926+0000 mon.a (mon.0) 1043 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:52.342 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:50.156366+0000 mon.a (mon.0) 1044 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:52.342 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:50.156726+0000 mon.a (mon.0) 1045 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:52.342 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:50.159660+0000 mon.a (mon.0) 1046 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:52.342 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:50.160183+0000 mon.a (mon.0) 1047 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:52.342 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:50.160535+0000 mon.a (mon.0) 1048 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:52.343 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:49.735647+0000 mon.a (mon.0) 1034 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:52.343 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:49.738169+0000 mon.b (mon.1) 32 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:37:52.343 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:49.738278+0000 mon.b (mon.1) 33 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:37:52.343 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:49.738882+0000 mon.a (mon.0) 1035 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:52.343 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:49.739463+0000 mon.a (mon.0) 1036 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:52.343 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:49.739752+0000 mon.a (mon.0) 1037 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:52.343 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:49.742337+0000 mon.c (mon.2) 55 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:37:52.343 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:49.742433+0000 mon.c (mon.2) 56 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:37:52.343 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:49.742861+0000 mon.a (mon.0) 1038 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:52.343 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:49.743604+0000 mon.a (mon.0) 1039 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:52.343 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:49.744064+0000 mon.a (mon.0) 1040 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:52.343 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:50.017366+0000 mon.a (mon.0) 1041 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:37:52.344 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:50.021067+0000 mon.a (mon.0) 1042 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-12T23:37:52.344 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:50.155926+0000 mon.a (mon.0) 1043 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:52.344 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:50.156366+0000 mon.a (mon.0) 1044 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:52.344 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:50.156726+0000 mon.a (mon.0) 1045 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:52.344 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:50.159660+0000 mon.a (mon.0) 1046 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:52.344 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:50.160183+0000 mon.a (mon.0) 1047 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:52.344 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:50.160535+0000 mon.a (mon.0) 1048 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:52.344 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:50.162948+0000 mon.a (mon.0) 1049 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:52.344 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:50.163436+0000 mon.a (mon.0) 1050 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:52.344 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:50.163724+0000 mon.a (mon.0) 1051 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:52.344 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:50.178285+0000 mon.a (mon.0) 1052 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:37:52.344 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:50.178336+0000 mon.a (mon.0) 1053 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:37:52.344 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:50.178736+0000 mon.a (mon.0) 1054 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:52.345 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:50.179150+0000 mon.a (mon.0) 1055 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:52.345 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:50.179483+0000 mon.a (mon.0) 1056 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:52.345 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:50.181858+0000 mon.b (mon.1) 34 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:37:52.345 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:50.181963+0000 mon.b (mon.1) 35 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:37:52.345 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:50.182585+0000 mon.a (mon.0) 1057 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:52.345 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:50.183201+0000 mon.a (mon.0) 1058 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:52.345 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:50.183535+0000 mon.a (mon.0) 1059 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:52.345 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:50.186058+0000 mon.c (mon.2) 57 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:37:52.345 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:50.186168+0000 mon.c (mon.2) 58 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:37:52.345 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:50.186423+0000 mon.a (mon.0) 1060 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:52.345 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:50.186984+0000 mon.a (mon.0) 1061 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:52.345 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[22601]: audit 2024-09-12T23:37:50.187252+0000 mon.a (mon.0) 1062 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:52.346 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:50.162948+0000 mon.a (mon.0) 1049 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:52.346 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:50.163436+0000 mon.a (mon.0) 1050 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:52.346 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:50.163724+0000 mon.a (mon.0) 1051 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:52.346 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:50.178285+0000 mon.a (mon.0) 1052 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:37:52.346 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:50.178336+0000 mon.a (mon.0) 1053 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:37:52.346 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:50.178736+0000 mon.a (mon.0) 1054 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:52.346 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:50.179150+0000 mon.a (mon.0) 1055 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:52.346 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:50.179483+0000 mon.a (mon.0) 1056 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:52.346 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:50.181858+0000 mon.b (mon.1) 34 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:37:52.346 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:50.181963+0000 mon.b (mon.1) 35 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:37:52.346 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:50.182585+0000 mon.a (mon.0) 1057 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:52.347 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:50.183201+0000 mon.a (mon.0) 1058 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:52.347 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:50.183535+0000 mon.a (mon.0) 1059 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:52.347 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:50.186058+0000 mon.c (mon.2) 57 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:37:52.347 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:50.186168+0000 mon.c (mon.2) 58 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:37:52.347 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:50.186423+0000 mon.a (mon.0) 1060 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:37:52.347 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:50.186984+0000 mon.a (mon.0) 1061 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:37:52.347 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:50 smithi136 bash[32189]: audit 2024-09-12T23:37:50.187252+0000 mon.a (mon.0) 1062 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:37:52.347 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:51 smithi136 bash[22601]: cluster 2024-09-12T23:37:49.999581+0000 mgr.y (mgr.24446) 164 : cluster [DBG] pgmap v123: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 31 KiB/s rd, 0 B/s wr, 29 op/s 2024-09-12T23:37:52.347 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:51 smithi136 bash[22601]: audit 2024-09-12T23:37:50.015955+0000 mgr.y (mgr.24446) 165 : audit [DBG] from='client.24842 -' entity='client.admin' cmd=[{"prefix": "telemetry show-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:52.348 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:51 smithi136 bash[32189]: cluster 2024-09-12T23:37:49.999581+0000 mgr.y (mgr.24446) 164 : cluster [DBG] pgmap v123: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 31 KiB/s rd, 0 B/s wr, 29 op/s 2024-09-12T23:37:52.348 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:51 smithi136 bash[32189]: audit 2024-09-12T23:37:50.015955+0000 mgr.y (mgr.24446) 165 : audit [DBG] from='client.24842 -' entity='client.admin' cmd=[{"prefix": "telemetry show-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:52.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:51 smithi181 bash[24776]: cluster 2024-09-12T23:37:49.999581+0000 mgr.y (mgr.24446) 164 : cluster [DBG] pgmap v123: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 31 KiB/s rd, 0 B/s wr, 29 op/s 2024-09-12T23:37:52.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:51 smithi181 bash[24776]: audit 2024-09-12T23:37:50.015955+0000 mgr.y (mgr.24446) 165 : audit [DBG] from='client.24842 -' entity='client.admin' cmd=[{"prefix": "telemetry show-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:37:52.362 INFO:tasks.workunit:Stopping ['test_telemetry_quincy.sh'] on client.0... 2024-09-12T23:37:52.362 DEBUG:teuthology.orchestra.run.smithi136:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2024-09-12T23:37:52.691 DEBUG:teuthology.parallel:result is None 2024-09-12T23:37:52.691 DEBUG:teuthology.orchestra.run.smithi136:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2024-09-12T23:37:52.704 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2024-09-12T23:37:52.704 DEBUG:teuthology.orchestra.run.smithi136:> rmdir -- /home/ubuntu/cephtest/mnt.0 2024-09-12T23:37:52.766 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2024-09-12T23:37:52.767 INFO:teuthology.run_tasks:Running task print... 2024-09-12T23:37:52.777 INFO:teuthology.task.print:**** done end telemetry quincy... 2024-09-12T23:37:52.777 INFO:teuthology.run_tasks:Running task print... 2024-09-12T23:37:52.823 INFO:teuthology.task.print:**** done start parallel 2024-09-12T23:37:52.823 INFO:teuthology.run_tasks:Running task parallel... 2024-09-12T23:37:52.863 INFO:teuthology.task.parallel:starting parallel... 2024-09-12T23:37:52.863 INFO:teuthology.task.parallel:In parallel, running task full_sequential... 2024-09-12T23:37:52.864 INFO:teuthology.task.full_sequential:In full_sequential, running task print... 2024-09-12T23:37:52.864 INFO:teuthology.task.print:**** done start ec-rados-default.yaml 2024-09-12T23:37:52.864 INFO:teuthology.task.full_sequential:In full_sequential, running task rados... 2024-09-12T23:37:52.866 INFO:tasks.rados:Beginning rados... 2024-09-12T23:37:52.867 INFO:tasks.rados:config is {'clients': ['client.0'], 'ec_pool': True, 'objects': 50, 'op_weights': {'append': 100, 'copy_from': 50, 'delete': 50, 'read': 100, 'rmattr': 25, 'rollback': 50, 'setattr': 25, 'snap_create': 50, 'snap_remove': 50, 'write': 0}, 'ops': 4000, 'write_append_excl': False} 2024-09-12T23:37:52.867 INFO:tasks.rados:overrides is {'admin_socket': {'branch': 'reef'}, 'ceph': {'conf': {'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': '7a853b1706d076d58e0da7a5d3b9b9eb36276880'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'extra_system_packages': ['python3-pytest'], 'flavor': 'default', 'sha1': '7a853b1706d076d58e0da7a5d3b9b9eb36276880'}}, 'workunit': {'branch': 'reef', 'sha1': '7a853b1706d076d58e0da7a5d3b9b9eb36276880'}} 2024-09-12T23:37:52.867 INFO:tasks.rados:config is {'clients': ['client.0'], 'ec_pool': True, 'objects': 50, 'op_weights': {'append': 100, 'copy_from': 50, 'delete': 50, 'read': 100, 'rmattr': 25, 'rollback': 50, 'setattr': 25, 'snap_create': 50, 'snap_remove': 50, 'write': 0}, 'ops': 4000, 'write_append_excl': False} 2024-09-12T23:37:52.867 INFO:tasks.rados:joining rados 2024-09-12T23:37:52.867 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-12T23:37:52.868 INFO:teuthology.task.sequential:In sequential, running task print... 2024-09-12T23:37:52.868 INFO:teuthology.task.print:**** done start upgrade, wait 2024-09-12T23:37:52.868 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-12T23:37:52.868 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi136.front.sepia.ceph.com 2024-09-12T23:37:52.869 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid da58a47c-715d-11ef-bceb-c7b262605968 -e sha1=7a853b1706d076d58e0da7a5d3b9b9eb36276880 -- bash -c 'ceph config set global log_to_journald false --force' 2024-09-12T23:37:52.869 INFO:tasks.rados:clients are ['client.0', 'client.1'] 2024-09-12T23:37:52.869 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph osd erasure-code-profile set teuthologyprofile k=2 m=1 crush-failure-domain=osd 2024-09-12T23:37:53.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:53 smithi136 bash[22601]: cluster 2024-09-12T23:37:52.000303+0000 mgr.y (mgr.24446) 166 : cluster [DBG] pgmap v124: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 31 KiB/s rd, 0 B/s wr, 29 op/s 2024-09-12T23:37:53.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:53 smithi136 bash[22601]: audit 2024-09-12T23:37:53.013143+0000 mon.a (mon.0) 1063 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:37:53.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:53 smithi136 bash[32189]: cluster 2024-09-12T23:37:52.000303+0000 mgr.y (mgr.24446) 166 : cluster [DBG] pgmap v124: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 31 KiB/s rd, 0 B/s wr, 29 op/s 2024-09-12T23:37:53.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:53 smithi136 bash[32189]: audit 2024-09-12T23:37:53.013143+0000 mon.a (mon.0) 1063 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:37:53.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:53 smithi181 bash[24776]: cluster 2024-09-12T23:37:52.000303+0000 mgr.y (mgr.24446) 166 : cluster [DBG] pgmap v124: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 31 KiB/s rd, 0 B/s wr, 29 op/s 2024-09-12T23:37:53.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:53 smithi181 bash[24776]: audit 2024-09-12T23:37:53.013143+0000 mon.a (mon.0) 1063 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:37:55.508 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:37:55 smithi181 bash[25717]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:37:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-12T23:37:55.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:55 smithi181 bash[24776]: cluster 2024-09-12T23:37:54.001544+0000 mgr.y (mgr.24446) 167 : cluster [DBG] pgmap v125: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 44 op/s 2024-09-12T23:37:56.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:55 smithi136 bash[22601]: cluster 2024-09-12T23:37:54.001544+0000 mgr.y (mgr.24446) 167 : cluster [DBG] pgmap v125: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 44 op/s 2024-09-12T23:37:56.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:55 smithi136 bash[32189]: cluster 2024-09-12T23:37:54.001544+0000 mgr.y (mgr.24446) 167 : cluster [DBG] pgmap v125: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 44 op/s 2024-09-12T23:37:56.506 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:37:56 smithi136 bash[22852]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:37:56] "GET /metrics HTTP/1.1" 200 236622 "" "Prometheus/2.43.0" 2024-09-12T23:37:57.664 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:37:57.665 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:37:57.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:57 smithi181 bash[24776]: cluster 2024-09-12T23:37:56.002192+0000 mgr.y (mgr.24446) 168 : cluster [DBG] pgmap v126: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 44 op/s 2024-09-12T23:37:57.893 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:57 smithi136 bash[22601]: cluster 2024-09-12T23:37:56.002192+0000 mgr.y (mgr.24446) 168 : cluster [DBG] pgmap v126: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 44 op/s 2024-09-12T23:37:57.893 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:57 smithi136 bash[32189]: cluster 2024-09-12T23:37:56.002192+0000 mgr.y (mgr.24446) 168 : cluster [DBG] pgmap v126: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 44 op/s 2024-09-12T23:37:59.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:37:59 smithi136 bash[22601]: cluster 2024-09-12T23:37:58.002886+0000 mgr.y (mgr.24446) 169 : cluster [DBG] pgmap v127: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 44 op/s 2024-09-12T23:37:59.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:37:59 smithi136 bash[32189]: cluster 2024-09-12T23:37:58.002886+0000 mgr.y (mgr.24446) 169 : cluster [DBG] pgmap v127: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 44 op/s 2024-09-12T23:37:59.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:37:59 smithi181 bash[24776]: cluster 2024-09-12T23:37:58.002886+0000 mgr.y (mgr.24446) 169 : cluster [DBG] pgmap v127: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 44 op/s 2024-09-12T23:38:01.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:01 smithi181 bash[24776]: cluster 2024-09-12T23:38:00.004164+0000 mgr.y (mgr.24446) 170 : cluster [DBG] pgmap v128: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 44 op/s 2024-09-12T23:38:01.889 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:01 smithi136 bash[32189]: cluster 2024-09-12T23:38:00.004164+0000 mgr.y (mgr.24446) 170 : cluster [DBG] pgmap v128: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 44 op/s 2024-09-12T23:38:01.889 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:01 smithi136 bash[22601]: cluster 2024-09-12T23:38:00.004164+0000 mgr.y (mgr.24446) 170 : cluster [DBG] pgmap v128: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 44 op/s 2024-09-12T23:38:02.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:02 smithi181 bash[24776]: audit 2024-09-12T23:38:02.232498+0000 mon.c (mon.2) 59 : audit [INF] from='client.? 172.21.15.136:0/2495713333' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-12T23:38:02.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:02 smithi181 bash[24776]: audit 2024-09-12T23:38:02.232909+0000 mon.a (mon.0) 1064 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-12T23:38:03.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:02 smithi136 bash[22601]: audit 2024-09-12T23:38:02.232498+0000 mon.c (mon.2) 59 : audit [INF] from='client.? 172.21.15.136:0/2495713333' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-12T23:38:03.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:02 smithi136 bash[22601]: audit 2024-09-12T23:38:02.232909+0000 mon.a (mon.0) 1064 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-12T23:38:03.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:02 smithi136 bash[32189]: audit 2024-09-12T23:38:02.232498+0000 mon.c (mon.2) 59 : audit [INF] from='client.? 172.21.15.136:0/2495713333' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-12T23:38:03.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:02 smithi136 bash[32189]: audit 2024-09-12T23:38:02.232909+0000 mon.a (mon.0) 1064 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-12T23:38:03.068 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid da58a47c-715d-11ef-bceb-c7b262605968 -e sha1=7a853b1706d076d58e0da7a5d3b9b9eb36276880 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-09-12T23:38:03.411 INFO:tasks.rados:starting run 0 out of 1 2024-09-12T23:38:03.412 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2024-09-12T23:38:03.412 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph osd pool create unique_pool_0 16 16 erasure teuthologyprofile 2024-09-12T23:38:03.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:03 smithi136 bash[22601]: cluster 2024-09-12T23:38:02.004559+0000 mgr.y (mgr.24446) 171 : cluster [DBG] pgmap v129: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 0 B/s wr, 14 op/s 2024-09-12T23:38:03.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:03 smithi136 bash[22601]: audit 2024-09-12T23:38:02.539591+0000 mon.a (mon.0) 1065 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2024-09-12T23:38:03.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:03 smithi136 bash[22601]: cluster 2024-09-12T23:38:02.539726+0000 mon.a (mon.0) 1066 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2024-09-12T23:38:03.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:03 smithi136 bash[32189]: cluster 2024-09-12T23:38:02.004559+0000 mgr.y (mgr.24446) 171 : cluster [DBG] pgmap v129: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 0 B/s wr, 14 op/s 2024-09-12T23:38:03.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:03 smithi136 bash[32189]: audit 2024-09-12T23:38:02.539591+0000 mon.a (mon.0) 1065 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2024-09-12T23:38:03.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:03 smithi136 bash[32189]: cluster 2024-09-12T23:38:02.539726+0000 mon.a (mon.0) 1066 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2024-09-12T23:38:03.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:03 smithi181 bash[24776]: cluster 2024-09-12T23:38:02.004559+0000 mgr.y (mgr.24446) 171 : cluster [DBG] pgmap v129: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 0 B/s wr, 14 op/s 2024-09-12T23:38:03.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:03 smithi181 bash[24776]: audit 2024-09-12T23:38:02.539591+0000 mon.a (mon.0) 1065 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2024-09-12T23:38:03.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:03 smithi181 bash[24776]: cluster 2024-09-12T23:38:02.539726+0000 mon.a (mon.0) 1066 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2024-09-12T23:38:05.535 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:38:05 smithi181 bash[25717]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:38:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-12T23:38:05.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:05 smithi181 bash[24776]: cluster 2024-09-12T23:38:04.005694+0000 mgr.y (mgr.24446) 172 : cluster [DBG] pgmap v131: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:38:06.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:05 smithi136 bash[22601]: cluster 2024-09-12T23:38:04.005694+0000 mgr.y (mgr.24446) 172 : cluster [DBG] pgmap v131: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:38:06.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:05 smithi136 bash[32189]: cluster 2024-09-12T23:38:04.005694+0000 mgr.y (mgr.24446) 172 : cluster [DBG] pgmap v131: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:38:06.505 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:38:06 smithi136 bash[22852]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:38:06] "GET /metrics HTTP/1.1" 200 236622 "" "Prometheus/2.43.0" 2024-09-12T23:38:07.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:07 smithi181 bash[24776]: cluster 2024-09-12T23:38:06.006349+0000 mgr.y (mgr.24446) 173 : cluster [DBG] pgmap v132: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:38:07.858 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:38:07.858 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:38:07.989 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:07 smithi136 bash[22601]: cluster 2024-09-12T23:38:06.006349+0000 mgr.y (mgr.24446) 173 : cluster [DBG] pgmap v132: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:38:07.989 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:07 smithi136 bash[32189]: cluster 2024-09-12T23:38:06.006349+0000 mgr.y (mgr.24446) 173 : cluster [DBG] pgmap v132: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:38:08.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:08 smithi181 bash[24776]: audit 2024-09-12T23:38:08.013029+0000 mon.a (mon.0) 1067 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:38:09.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:08 smithi136 bash[22601]: audit 2024-09-12T23:38:08.013029+0000 mon.a (mon.0) 1067 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:38:09.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:08 smithi136 bash[32189]: audit 2024-09-12T23:38:08.013029+0000 mon.a (mon.0) 1067 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:38:09.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:09 smithi136 bash[22601]: cluster 2024-09-12T23:38:08.007011+0000 mgr.y (mgr.24446) 174 : cluster [DBG] pgmap v133: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:38:09.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:09 smithi136 bash[32189]: cluster 2024-09-12T23:38:08.007011+0000 mgr.y (mgr.24446) 174 : cluster [DBG] pgmap v133: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:38:09.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:09 smithi181 bash[24776]: cluster 2024-09-12T23:38:08.007011+0000 mgr.y (mgr.24446) 174 : cluster [DBG] pgmap v133: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:38:11.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:11 smithi181 bash[24776]: cluster 2024-09-12T23:38:10.008274+0000 mgr.y (mgr.24446) 175 : cluster [DBG] pgmap v134: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:38:11.961 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:11 smithi136 bash[22601]: cluster 2024-09-12T23:38:10.008274+0000 mgr.y (mgr.24446) 175 : cluster [DBG] pgmap v134: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:38:11.961 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:11 smithi136 bash[32189]: cluster 2024-09-12T23:38:10.008274+0000 mgr.y (mgr.24446) 175 : cluster [DBG] pgmap v134: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:38:12.339 INFO:teuthology.orchestra.run.smithi136.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:7a853b1706d076d58e0da7a5d3b9b9eb36276880 2024-09-12T23:38:13.049 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid da58a47c-715d-11ef-bceb-c7b262605968 -e sha1=7a853b1706d076d58e0da7a5d3b9b9eb36276880 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done' 2024-09-12T23:38:13.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:13 smithi181 bash[24776]: cluster 2024-09-12T23:38:12.008775+0000 mgr.y (mgr.24446) 176 : cluster [DBG] pgmap v135: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:38:13.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:13 smithi181 bash[24776]: audit 2024-09-12T23:38:12.337239+0000 mgr.y (mgr.24446) 177 : audit [DBG] from='client.24872 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:7a853b1706d076d58e0da7a5d3b9b9eb36276880", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:38:13.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:13 smithi181 bash[24776]: cephadm 2024-09-12T23:38:12.338007+0000 mgr.y (mgr.24446) 178 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:7a853b1706d076d58e0da7a5d3b9b9eb36276880 2024-09-12T23:38:13.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:13 smithi181 bash[24776]: audit 2024-09-12T23:38:12.343504+0000 mon.a (mon.0) 1068 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:38:13.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:13 smithi181 bash[24776]: audit 2024-09-12T23:38:12.344273+0000 mon.a (mon.0) 1069 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:38:13.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:13 smithi181 bash[24776]: audit 2024-09-12T23:38:12.346309+0000 mon.a (mon.0) 1070 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:38:13.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:13 smithi181 bash[24776]: audit 2024-09-12T23:38:12.347370+0000 mon.a (mon.0) 1071 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:38:13.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:13 smithi181 bash[24776]: audit 2024-09-12T23:38:12.353749+0000 mon.a (mon.0) 1072 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:38:13.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:13 smithi181 bash[24776]: cephadm 2024-09-12T23:38:12.394099+0000 mgr.y (mgr.24446) 179 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:7a853b1706d076d58e0da7a5d3b9b9eb36276880 2024-09-12T23:38:13.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:13 smithi181 bash[24776]: audit 2024-09-12T23:38:12.444019+0000 mon.c (mon.2) 60 : audit [INF] from='client.? 172.21.15.136:0/1020754763' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]: dispatch 2024-09-12T23:38:13.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:13 smithi181 bash[24776]: audit 2024-09-12T23:38:12.444417+0000 mon.a (mon.0) 1073 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]: dispatch 2024-09-12T23:38:13.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:13 smithi136 bash[32189]: cluster 2024-09-12T23:38:12.008775+0000 mgr.y (mgr.24446) 176 : cluster [DBG] pgmap v135: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:38:13.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:13 smithi136 bash[32189]: audit 2024-09-12T23:38:12.337239+0000 mgr.y (mgr.24446) 177 : audit [DBG] from='client.24872 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:7a853b1706d076d58e0da7a5d3b9b9eb36276880", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:38:13.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:13 smithi136 bash[32189]: cephadm 2024-09-12T23:38:12.338007+0000 mgr.y (mgr.24446) 178 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:7a853b1706d076d58e0da7a5d3b9b9eb36276880 2024-09-12T23:38:13.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:13 smithi136 bash[32189]: audit 2024-09-12T23:38:12.343504+0000 mon.a (mon.0) 1068 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:38:13.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:13 smithi136 bash[32189]: audit 2024-09-12T23:38:12.344273+0000 mon.a (mon.0) 1069 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:38:13.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:13 smithi136 bash[32189]: audit 2024-09-12T23:38:12.346309+0000 mon.a (mon.0) 1070 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:38:13.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:13 smithi136 bash[32189]: audit 2024-09-12T23:38:12.347370+0000 mon.a (mon.0) 1071 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:38:13.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:13 smithi136 bash[32189]: audit 2024-09-12T23:38:12.353749+0000 mon.a (mon.0) 1072 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:38:13.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:13 smithi136 bash[32189]: cephadm 2024-09-12T23:38:12.394099+0000 mgr.y (mgr.24446) 179 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:7a853b1706d076d58e0da7a5d3b9b9eb36276880 2024-09-12T23:38:13.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:13 smithi136 bash[32189]: audit 2024-09-12T23:38:12.444019+0000 mon.c (mon.2) 60 : audit [INF] from='client.? 172.21.15.136:0/1020754763' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]: dispatch 2024-09-12T23:38:13.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:13 smithi136 bash[32189]: audit 2024-09-12T23:38:12.444417+0000 mon.a (mon.0) 1073 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]: dispatch 2024-09-12T23:38:13.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:13 smithi136 bash[22601]: cluster 2024-09-12T23:38:12.008775+0000 mgr.y (mgr.24446) 176 : cluster [DBG] pgmap v135: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:38:13.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:13 smithi136 bash[22601]: audit 2024-09-12T23:38:12.337239+0000 mgr.y (mgr.24446) 177 : audit [DBG] from='client.24872 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:7a853b1706d076d58e0da7a5d3b9b9eb36276880", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:38:13.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:13 smithi136 bash[22601]: cephadm 2024-09-12T23:38:12.338007+0000 mgr.y (mgr.24446) 178 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:7a853b1706d076d58e0da7a5d3b9b9eb36276880 2024-09-12T23:38:13.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:13 smithi136 bash[22601]: audit 2024-09-12T23:38:12.343504+0000 mon.a (mon.0) 1068 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:38:13.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:13 smithi136 bash[22601]: audit 2024-09-12T23:38:12.344273+0000 mon.a (mon.0) 1069 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:38:13.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:13 smithi136 bash[22601]: audit 2024-09-12T23:38:12.346309+0000 mon.a (mon.0) 1070 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:38:13.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:13 smithi136 bash[22601]: audit 2024-09-12T23:38:12.347370+0000 mon.a (mon.0) 1071 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:38:13.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:13 smithi136 bash[22601]: audit 2024-09-12T23:38:12.353749+0000 mon.a (mon.0) 1072 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:38:13.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:13 smithi136 bash[22601]: cephadm 2024-09-12T23:38:12.394099+0000 mgr.y (mgr.24446) 179 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:7a853b1706d076d58e0da7a5d3b9b9eb36276880 2024-09-12T23:38:13.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:13 smithi136 bash[22601]: audit 2024-09-12T23:38:12.444019+0000 mon.c (mon.2) 60 : audit [INF] from='client.? 172.21.15.136:0/1020754763' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]: dispatch 2024-09-12T23:38:13.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:13 smithi136 bash[22601]: audit 2024-09-12T23:38:12.444417+0000 mon.a (mon.0) 1073 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]: dispatch 2024-09-12T23:38:14.384 INFO:teuthology.orchestra.run.smithi136.stderr:pool 'unique_pool_0' created 2024-09-12T23:38:14.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:14 smithi136 bash[22601]: cluster 2024-09-12T23:38:13.804591+0000 mon.a (mon.0) 1074 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2024-09-12T23:38:14.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:14 smithi136 bash[32189]: cluster 2024-09-12T23:38:13.804591+0000 mon.a (mon.0) 1074 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2024-09-12T23:38:14.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:14 smithi181 bash[24776]: cluster 2024-09-12T23:38:13.804591+0000 mon.a (mon.0) 1074 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2024-09-12T23:38:15.447 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2024-09-12T23:38:15.595 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:38:15 smithi181 bash[25717]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:38:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-12T23:38:15.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:15 smithi181 bash[24776]: cluster 2024-09-12T23:38:14.009967+0000 mgr.y (mgr.24446) 180 : cluster [DBG] pgmap v137: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:38:15.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:15 smithi181 bash[24776]: audit 2024-09-12T23:38:14.381674+0000 mon.a (mon.0) 1075 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]': finished 2024-09-12T23:38:15.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:15 smithi181 bash[24776]: cluster 2024-09-12T23:38:14.381882+0000 mon.a (mon.0) 1076 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2024-09-12T23:38:15.755 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:15 smithi136 bash[32189]: cluster 2024-09-12T23:38:14.009967+0000 mgr.y (mgr.24446) 180 : cluster [DBG] pgmap v137: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:38:15.755 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:15 smithi136 bash[32189]: audit 2024-09-12T23:38:14.381674+0000 mon.a (mon.0) 1075 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]': finished 2024-09-12T23:38:15.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:15 smithi136 bash[32189]: cluster 2024-09-12T23:38:14.381882+0000 mon.a (mon.0) 1076 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2024-09-12T23:38:15.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:15 smithi136 bash[22601]: cluster 2024-09-12T23:38:14.009967+0000 mgr.y (mgr.24446) 180 : cluster [DBG] pgmap v137: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:38:15.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:15 smithi136 bash[22601]: audit 2024-09-12T23:38:14.381674+0000 mon.a (mon.0) 1075 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]': finished 2024-09-12T23:38:15.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:15 smithi136 bash[22601]: cluster 2024-09-12T23:38:14.381882+0000 mon.a (mon.0) 1076 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2024-09-12T23:38:16.388 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:38:16 smithi136 bash[22852]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:38:16] "GET /metrics HTTP/1.1" 200 236607 "" "Prometheus/2.43.0" 2024-09-12T23:38:16.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:16 smithi136 bash[22601]: cluster 2024-09-12T23:38:15.377312+0000 mon.a (mon.0) 1077 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2024-09-12T23:38:16.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:16 smithi136 bash[32189]: cluster 2024-09-12T23:38:15.377312+0000 mon.a (mon.0) 1077 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2024-09-12T23:38:16.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:16 smithi181 bash[24776]: cluster 2024-09-12T23:38:15.377312+0000 mon.a (mon.0) 1077 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2024-09-12T23:38:17.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:17 smithi136 bash[22601]: cluster 2024-09-12T23:38:16.010393+0000 mgr.y (mgr.24446) 181 : cluster [DBG] pgmap v140: 121 pgs: 16 unknown, 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:38:17.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:17 smithi136 bash[22601]: cluster 2024-09-12T23:38:16.381875+0000 mon.a (mon.0) 1078 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:38:17.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:17 smithi136 bash[22601]: cluster 2024-09-12T23:38:16.399619+0000 mon.a (mon.0) 1079 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2024-09-12T23:38:17.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:17 smithi136 bash[32189]: cluster 2024-09-12T23:38:16.010393+0000 mgr.y (mgr.24446) 181 : cluster [DBG] pgmap v140: 121 pgs: 16 unknown, 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:38:17.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:17 smithi136 bash[32189]: cluster 2024-09-12T23:38:16.381875+0000 mon.a (mon.0) 1078 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:38:17.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:17 smithi136 bash[32189]: cluster 2024-09-12T23:38:16.399619+0000 mon.a (mon.0) 1079 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2024-09-12T23:38:17.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:17 smithi181 bash[24776]: cluster 2024-09-12T23:38:16.010393+0000 mgr.y (mgr.24446) 181 : cluster [DBG] pgmap v140: 121 pgs: 16 unknown, 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:38:17.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:17 smithi181 bash[24776]: cluster 2024-09-12T23:38:16.381875+0000 mon.a (mon.0) 1078 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:38:17.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:17 smithi181 bash[24776]: cluster 2024-09-12T23:38:16.399619+0000 mon.a (mon.0) 1079 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2024-09-12T23:38:17.852 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:38:19.669 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:19 smithi136 bash[32189]: cluster 2024-09-12T23:38:18.010830+0000 mgr.y (mgr.24446) 182 : cluster [DBG] pgmap v142: 121 pgs: 16 unknown, 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:38:19.669 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:19 smithi136 bash[22601]: cluster 2024-09-12T23:38:18.010830+0000 mgr.y (mgr.24446) 182 : cluster [DBG] pgmap v142: 121 pgs: 16 unknown, 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:38:19.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:19 smithi181 bash[24776]: cluster 2024-09-12T23:38:18.010830+0000 mgr.y (mgr.24446) 182 : cluster [DBG] pgmap v142: 121 pgs: 16 unknown, 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:38:19.905 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:38:21.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:21 smithi136 bash[32189]: cluster 2024-09-12T23:38:20.011945+0000 mgr.y (mgr.24446) 183 : cluster [DBG] pgmap v143: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:38:21.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:21 smithi136 bash[22601]: cluster 2024-09-12T23:38:20.011945+0000 mgr.y (mgr.24446) 183 : cluster [DBG] pgmap v143: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:38:21.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:21 smithi181 bash[24776]: cluster 2024-09-12T23:38:20.011945+0000 mgr.y (mgr.24446) 183 : cluster [DBG] pgmap v143: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:38:23.048 INFO:teuthology.orchestra.run.smithi136.stdout:true 2024-09-12T23:38:23.362 INFO:teuthology.orchestra.run.smithi136.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-12T23:38:23.363 INFO:teuthology.orchestra.run.smithi136.stdout:alertmanager.a smithi136 *:9093,9094 running (3m) 2m ago 3m 14.3M - 0.25.0 c8568f914cd2 d6c106834390 2024-09-12T23:38:23.363 INFO:teuthology.orchestra.run.smithi136.stdout:grafana.a smithi181 *:3000 running (3m) 2m ago 3m 29.0M - 9.4.7 954c08fa6188 85ee0f95160c 2024-09-12T23:38:23.363 INFO:teuthology.orchestra.run.smithi136.stdout:mds.a smithi136 running (5m) 2m ago 5m 12.1M - 17.2.7-1627-gbbdadfa3 214985049bdb 713b9795ad19 2024-09-12T23:38:23.363 INFO:teuthology.orchestra.run.smithi136.stdout:mds.b smithi181 running (5m) 2m ago 5m 13.9M - 17.2.7-1627-gbbdadfa3 214985049bdb ad13179f0868 2024-09-12T23:38:23.363 INFO:teuthology.orchestra.run.smithi136.stdout:mgr.x smithi181 *:8443 running (12m) 2m ago 12m 387M - 17.2.7-1627-gbbdadfa3 214985049bdb d07cbbb66db7 2024-09-12T23:38:23.363 INFO:teuthology.orchestra.run.smithi136.stdout:mgr.y smithi136 *:9283,8765 running (14m) 2m ago 14m 436M - 17.2.7-1627-gbbdadfa3 214985049bdb 56608cdf6b27 2024-09-12T23:38:23.363 INFO:teuthology.orchestra.run.smithi136.stdout:mon.a smithi136 running (14m) 2m ago 14m 44.1M 2048M 17.2.7-1627-gbbdadfa3 214985049bdb 0462f4f61475 2024-09-12T23:38:23.363 INFO:teuthology.orchestra.run.smithi136.stdout:mon.b smithi181 running (12m) 2m ago 12m 37.6M 2048M 17.2.7-1627-gbbdadfa3 214985049bdb 8b3d784af845 2024-09-12T23:38:23.363 INFO:teuthology.orchestra.run.smithi136.stdout:mon.c smithi136 running (12m) 2m ago 12m 36.8M 2048M 17.2.7-1627-gbbdadfa3 214985049bdb 647b028ea963 2024-09-12T23:38:23.363 INFO:teuthology.orchestra.run.smithi136.stdout:node-exporter.a smithi136 *:9100 running (4m) 2m ago 4m 3143k - 1.5.0 0da6a335fe13 8f54fc5b0da3 2024-09-12T23:38:23.363 INFO:teuthology.orchestra.run.smithi136.stdout:node-exporter.b smithi181 *:9100 running (4m) 2m ago 4m 2895k - 1.5.0 0da6a335fe13 575eff556149 2024-09-12T23:38:23.364 INFO:teuthology.orchestra.run.smithi136.stdout:osd.0 smithi136 running (11m) 2m ago 11m 47.0M 3036M 17.2.7-1627-gbbdadfa3 214985049bdb cbc0b7df0115 2024-09-12T23:38:23.364 INFO:teuthology.orchestra.run.smithi136.stdout:osd.1 smithi136 running (10m) 2m ago 10m 44.4M 3036M 17.2.7-1627-gbbdadfa3 214985049bdb c4daee7da836 2024-09-12T23:38:23.364 INFO:teuthology.orchestra.run.smithi136.stdout:osd.2 smithi136 running (9m) 2m ago 9m 44.8M 3036M 17.2.7-1627-gbbdadfa3 214985049bdb 4ab8fd79cdb4 2024-09-12T23:38:23.364 INFO:teuthology.orchestra.run.smithi136.stdout:osd.3 smithi136 running (9m) 2m ago 9m 42.5M 3036M 17.2.7-1627-gbbdadfa3 214985049bdb 9ceb14a27521 2024-09-12T23:38:23.364 INFO:teuthology.orchestra.run.smithi136.stdout:osd.4 smithi181 running (8m) 2m ago 8m 42.7M 3036M 17.2.7-1627-gbbdadfa3 214985049bdb a35cdca79055 2024-09-12T23:38:23.364 INFO:teuthology.orchestra.run.smithi136.stdout:osd.5 smithi181 running (7m) 2m ago 7m 42.2M 3036M 17.2.7-1627-gbbdadfa3 214985049bdb 8f2e8d494fef 2024-09-12T23:38:23.364 INFO:teuthology.orchestra.run.smithi136.stdout:osd.6 smithi181 running (6m) 2m ago 6m 41.2M 3036M 17.2.7-1627-gbbdadfa3 214985049bdb 9c6bec406408 2024-09-12T23:38:23.364 INFO:teuthology.orchestra.run.smithi136.stdout:osd.7 smithi181 running (5m) 2m ago 5m 42.3M 3036M 17.2.7-1627-gbbdadfa3 214985049bdb bc0632ae2145 2024-09-12T23:38:23.364 INFO:teuthology.orchestra.run.smithi136.stdout:prometheus.a smithi181 *:9095 running (3m) 2m ago 4m 22.6M - 2.43.0 a07b618ecd1d 24e591310569 2024-09-12T23:38:23.733 INFO:teuthology.orchestra.run.smithi136.stdout:{ 2024-09-12T23:38:23.733 INFO:teuthology.orchestra.run.smithi136.stdout: "mon": { 2024-09-12T23:38:23.733 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 3 2024-09-12T23:38:23.733 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:38:23.733 INFO:teuthology.orchestra.run.smithi136.stdout: "mgr": { 2024-09-12T23:38:23.733 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-12T23:38:23.733 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:38:23.733 INFO:teuthology.orchestra.run.smithi136.stdout: "osd": { 2024-09-12T23:38:23.734 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 8 2024-09-12T23:38:23.734 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:38:23.734 INFO:teuthology.orchestra.run.smithi136.stdout: "mds": { 2024-09-12T23:38:23.734 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-12T23:38:23.734 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:38:23.734 INFO:teuthology.orchestra.run.smithi136.stdout: "overall": { 2024-09-12T23:38:23.734 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 15 2024-09-12T23:38:23.734 INFO:teuthology.orchestra.run.smithi136.stdout: } 2024-09-12T23:38:23.734 INFO:teuthology.orchestra.run.smithi136.stdout:} 2024-09-12T23:38:23.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:23 smithi136 bash[22601]: cluster 2024-09-12T23:38:22.012597+0000 mgr.y (mgr.24446) 184 : cluster [DBG] pgmap v144: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:38:23.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:23 smithi136 bash[22601]: audit 2024-09-12T23:38:23.017688+0000 mon.a (mon.0) 1080 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:38:23.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:23 smithi136 bash[22601]: audit 2024-09-12T23:38:23.018966+0000 mon.a (mon.0) 1081 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:38:23.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:23 smithi136 bash[32189]: cluster 2024-09-12T23:38:22.012597+0000 mgr.y (mgr.24446) 184 : cluster [DBG] pgmap v144: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:38:23.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:23 smithi136 bash[32189]: audit 2024-09-12T23:38:23.017688+0000 mon.a (mon.0) 1080 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:38:23.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:23 smithi136 bash[32189]: audit 2024-09-12T23:38:23.018966+0000 mon.a (mon.0) 1081 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:38:23.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:23 smithi181 bash[24776]: cluster 2024-09-12T23:38:22.012597+0000 mgr.y (mgr.24446) 184 : cluster [DBG] pgmap v144: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:38:23.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:23 smithi181 bash[24776]: audit 2024-09-12T23:38:23.017688+0000 mon.a (mon.0) 1080 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:38:23.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:23 smithi181 bash[24776]: audit 2024-09-12T23:38:23.018966+0000 mon.a (mon.0) 1081 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:38:24.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:24 smithi136 bash[22601]: audit 2024-09-12T23:38:23.039891+0000 mgr.y (mgr.24446) 185 : audit [DBG] from='client.14970 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:38:24.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:24 smithi136 bash[22601]: audit 2024-09-12T23:38:23.353133+0000 mgr.y (mgr.24446) 186 : audit [DBG] from='client.24890 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:38:24.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:24 smithi136 bash[22601]: audit 2024-09-12T23:38:23.737625+0000 mon.c (mon.2) 61 : audit [DBG] from='client.? 172.21.15.136:0/2211571592' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:38:24.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:24 smithi136 bash[32189]: audit 2024-09-12T23:38:23.039891+0000 mgr.y (mgr.24446) 185 : audit [DBG] from='client.14970 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:38:24.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:24 smithi136 bash[32189]: audit 2024-09-12T23:38:23.353133+0000 mgr.y (mgr.24446) 186 : audit [DBG] from='client.24890 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:38:24.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:24 smithi136 bash[32189]: audit 2024-09-12T23:38:23.737625+0000 mon.c (mon.2) 61 : audit [DBG] from='client.? 172.21.15.136:0/2211571592' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:38:24.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:24 smithi181 bash[24776]: audit 2024-09-12T23:38:23.039891+0000 mgr.y (mgr.24446) 185 : audit [DBG] from='client.14970 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:38:24.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:24 smithi181 bash[24776]: audit 2024-09-12T23:38:23.353133+0000 mgr.y (mgr.24446) 186 : audit [DBG] from='client.24890 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:38:24.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:24 smithi181 bash[24776]: audit 2024-09-12T23:38:23.737625+0000 mon.c (mon.2) 61 : audit [DBG] from='client.? 172.21.15.136:0/2211571592' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:38:25.418 INFO:teuthology.orchestra.run.smithi136.stderr:enabled application 'rados' on pool 'unique_pool_0' 2024-09-12T23:38:25.422 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:38:25 smithi181 bash[25717]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:38:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-12T23:38:25.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:25 smithi136 bash[32189]: cluster 2024-09-12T23:38:24.014196+0000 mgr.y (mgr.24446) 187 : cluster [DBG] pgmap v145: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:38:25.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:25 smithi136 bash[32189]: audit 2024-09-12T23:38:24.835072+0000 mon.c (mon.2) 62 : audit [INF] from='client.? 172.21.15.136:0/2332622084' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:38:25.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:25 smithi136 bash[32189]: audit 2024-09-12T23:38:24.835537+0000 mon.a (mon.0) 1082 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:38:25.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:25 smithi136 bash[22601]: cluster 2024-09-12T23:38:24.014196+0000 mgr.y (mgr.24446) 187 : cluster [DBG] pgmap v145: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:38:25.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:25 smithi136 bash[22601]: audit 2024-09-12T23:38:24.835072+0000 mon.c (mon.2) 62 : audit [INF] from='client.? 172.21.15.136:0/2332622084' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:38:25.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:25 smithi136 bash[22601]: audit 2024-09-12T23:38:24.835537+0000 mon.a (mon.0) 1082 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:38:25.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:25 smithi181 bash[24776]: cluster 2024-09-12T23:38:24.014196+0000 mgr.y (mgr.24446) 187 : cluster [DBG] pgmap v145: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:38:25.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:25 smithi181 bash[24776]: audit 2024-09-12T23:38:24.835072+0000 mon.c (mon.2) 62 : audit [INF] from='client.? 172.21.15.136:0/2332622084' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:38:25.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:25 smithi181 bash[24776]: audit 2024-09-12T23:38:24.835537+0000 mon.a (mon.0) 1082 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:38:26.423 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:38:26 smithi136 bash[22852]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:38:26] "GET /metrics HTTP/1.1" 200 238982 "" "Prometheus/2.43.0" 2024-09-12T23:38:26.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:26 smithi136 bash[22601]: audit 2024-09-12T23:38:25.422485+0000 mon.a (mon.0) 1083 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-12T23:38:26.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:26 smithi136 bash[22601]: cluster 2024-09-12T23:38:25.422645+0000 mon.a (mon.0) 1084 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2024-09-12T23:38:26.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:26 smithi136 bash[32189]: audit 2024-09-12T23:38:25.422485+0000 mon.a (mon.0) 1083 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-12T23:38:26.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:26 smithi136 bash[32189]: cluster 2024-09-12T23:38:25.422645+0000 mon.a (mon.0) 1084 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2024-09-12T23:38:26.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:26 smithi181 bash[24776]: audit 2024-09-12T23:38:25.422485+0000 mon.a (mon.0) 1083 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-12T23:38:26.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:26 smithi181 bash[24776]: cluster 2024-09-12T23:38:25.422645+0000 mon.a (mon.0) 1084 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2024-09-12T23:38:27.364 DEBUG:teuthology.orchestra.run.smithi136:> CEPH_CLIENT_ID=0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph_test_rados --no-omap --ec-pool --max-ops 4000 --objects 50 --max-in-flight 16 --size 4000000 --min-stride-size 400000 --max-stride-size 800000 --max-seconds 0 --op read 100 --op write 0 --op delete 50 --op snap_create 50 --op snap_remove 50 --op rollback 50 --op setattr 25 --op rmattr 25 --op copy_from 50 --op append 100 --pool unique_pool_0 2024-09-12T23:38:27.382 INFO:tasks.rados.rados.0.smithi136.stdout:adding op weight read -> 100 2024-09-12T23:38:27.382 INFO:tasks.rados.rados.0.smithi136.stdout:adding op weight delete -> 50 2024-09-12T23:38:27.382 INFO:tasks.rados.rados.0.smithi136.stdout:adding op weight snap_create -> 50 2024-09-12T23:38:27.382 INFO:tasks.rados.rados.0.smithi136.stdout:adding op weight snap_remove -> 50 2024-09-12T23:38:27.382 INFO:tasks.rados.rados.0.smithi136.stdout:adding op weight rollback -> 50 2024-09-12T23:38:27.382 INFO:tasks.rados.rados.0.smithi136.stdout:adding op weight setattr -> 25 2024-09-12T23:38:27.382 INFO:tasks.rados.rados.0.smithi136.stdout:adding op weight rmattr -> 25 2024-09-12T23:38:27.382 INFO:tasks.rados.rados.0.smithi136.stdout:adding op weight copy_from -> 50 2024-09-12T23:38:27.382 INFO:tasks.rados.rados.0.smithi136.stdout:adding op weight append -> 100 2024-09-12T23:38:27.382 INFO:tasks.rados.rados.0.smithi136.stdout:ceph version 17.2.7 (b12291d110049b2f35e32e0de30d70e9a4c060d2) quincy (stable) 2024-09-12T23:38:27.383 INFO:tasks.rados.rados.0.smithi136.stdout:Configuration: 2024-09-12T23:38:27.383 INFO:tasks.rados.rados.0.smithi136.stdout: Number of operations: 4000 2024-09-12T23:38:27.383 INFO:tasks.rados.rados.0.smithi136.stdout: Number of objects: 50 2024-09-12T23:38:27.383 INFO:tasks.rados.rados.0.smithi136.stdout: Max in flight operations: 16 2024-09-12T23:38:27.383 INFO:tasks.rados.rados.0.smithi136.stdout: Object size (in bytes): 4000000 2024-09-12T23:38:27.383 INFO:tasks.rados.rados.0.smithi136.stdout: Write stride min: 400000 2024-09-12T23:38:27.383 INFO:tasks.rados.rados.0.smithi136.stdout: Write stride max: 800000 2024-09-12T23:38:27.435 INFO:tasks.rados.rados.0.smithi136.stdout:1: write initial oid 1 2024-09-12T23:38:27.435 INFO:tasks.rados.rados.0.smithi136.stdout:1: seq_num 0 ranges {0=24576} 2024-09-12T23:38:27.436 INFO:tasks.rados.rados.0.smithi136.stdout:1: writing smithi13667081-1 from 0 to 24576 tid 1 2024-09-12T23:38:27.436 INFO:tasks.rados.rados.0.smithi136.stdout:2: write initial oid 2 2024-09-12T23:38:27.437 INFO:tasks.rados.rados.0.smithi136.stdout:2: seq_num 1 ranges {0=24576} 2024-09-12T23:38:27.437 INFO:tasks.rados.rados.0.smithi136.stdout:2: writing smithi13667081-2 from 0 to 24576 tid 1 2024-09-12T23:38:27.437 INFO:tasks.rados.rados.0.smithi136.stdout:3: write initial oid 3 2024-09-12T23:38:27.437 INFO:tasks.rados.rados.0.smithi136.stdout:3: seq_num 2 ranges {0=40960} 2024-09-12T23:38:27.437 INFO:tasks.rados.rados.0.smithi136.stdout:3: writing smithi13667081-3 from 0 to 40960 tid 1 2024-09-12T23:38:27.437 INFO:tasks.rados.rados.0.smithi136.stdout:4: write initial oid 4 2024-09-12T23:38:27.437 INFO:tasks.rados.rados.0.smithi136.stdout:4: seq_num 3 ranges {0=57344} 2024-09-12T23:38:27.437 INFO:tasks.rados.rados.0.smithi136.stdout:4: writing smithi13667081-4 from 0 to 57344 tid 1 2024-09-12T23:38:27.438 INFO:tasks.rados.rados.0.smithi136.stdout:5: write initial oid 5 2024-09-12T23:38:27.438 INFO:tasks.rados.rados.0.smithi136.stdout:5: seq_num 4 ranges {0=73728} 2024-09-12T23:38:27.440 INFO:tasks.rados.rados.0.smithi136.stdout:5: writing smithi13667081-5 from 0 to 73728 tid 1 2024-09-12T23:38:27.440 INFO:tasks.rados.rados.0.smithi136.stdout:6: write initial oid 6 2024-09-12T23:38:27.440 INFO:tasks.rados.rados.0.smithi136.stdout:6: seq_num 5 ranges {0=90112} 2024-09-12T23:38:27.440 INFO:tasks.rados.rados.0.smithi136.stdout:6: writing smithi13667081-6 from 0 to 90112 tid 1 2024-09-12T23:38:27.441 INFO:tasks.rados.rados.0.smithi136.stdout:7: write initial oid 7 2024-09-12T23:38:27.441 INFO:tasks.rados.rados.0.smithi136.stdout:7: seq_num 6 ranges {0=106496} 2024-09-12T23:38:27.441 INFO:tasks.rados.rados.0.smithi136.stdout:7: writing smithi13667081-7 from 0 to 106496 tid 1 2024-09-12T23:38:27.441 INFO:tasks.rados.rados.0.smithi136.stdout:8: write initial oid 8 2024-09-12T23:38:27.441 INFO:tasks.rados.rados.0.smithi136.stdout:8: seq_num 7 ranges {0=122880} 2024-09-12T23:38:27.441 INFO:tasks.rados.rados.0.smithi136.stdout:8: writing smithi13667081-8 from 0 to 122880 tid 1 2024-09-12T23:38:27.441 INFO:tasks.rados.rados.0.smithi136.stdout:9: write initial oid 9 2024-09-12T23:38:27.441 INFO:tasks.rados.rados.0.smithi136.stdout:9: seq_num 8 ranges {0=139264} 2024-09-12T23:38:27.442 INFO:tasks.rados.rados.0.smithi136.stdout:9: writing smithi13667081-9 from 0 to 139264 tid 1 2024-09-12T23:38:27.442 INFO:tasks.rados.rados.0.smithi136.stdout:10: write initial oid 10 2024-09-12T23:38:27.442 INFO:tasks.rados.rados.0.smithi136.stdout:10: seq_num 9 ranges {0=155648} 2024-09-12T23:38:27.443 INFO:tasks.rados.rados.0.smithi136.stdout:10: writing smithi13667081-10 from 0 to 155648 tid 1 2024-09-12T23:38:27.443 INFO:tasks.rados.rados.0.smithi136.stdout:11: write initial oid 11 2024-09-12T23:38:27.443 INFO:tasks.rados.rados.0.smithi136.stdout:11: seq_num 10 ranges {0=172032} 2024-09-12T23:38:27.447 INFO:tasks.rados.rados.0.smithi136.stdout:11: writing smithi13667081-11 from 0 to 172032 tid 1 2024-09-12T23:38:27.447 INFO:tasks.rados.rados.0.smithi136.stdout:1: finishing write tid 1 to smithi13667081-1 2024-09-12T23:38:27.447 INFO:tasks.rados.rados.0.smithi136.stdout:1: finishing write tid 2 to smithi13667081-1 2024-09-12T23:38:27.447 INFO:tasks.rados.rados.0.smithi136.stdout:3: finishing write tid 1 to smithi13667081-3 2024-09-12T23:38:27.447 INFO:tasks.rados.rados.0.smithi136.stdout:4: finishing write tid 1 to smithi13667081-4 2024-09-12T23:38:27.447 INFO:tasks.rados.rados.0.smithi136.stdout:1: finishing write tid 3 to smithi13667081-1 2024-09-12T23:38:27.447 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 1 v 2 (ObjNum 0 snap 0 seq_num 0) dirty exists 2024-09-12T23:38:27.447 INFO:tasks.rados.rados.0.smithi136.stdout:1: left oid 1 (ObjNum 0 snap 0 seq_num 0) 2024-09-12T23:38:27.447 INFO:tasks.rados.rados.0.smithi136.stdout:1: done (10 left) 2024-09-12T23:38:27.447 INFO:tasks.rados.rados.0.smithi136.stdout:12: write initial oid 12 2024-09-12T23:38:27.447 INFO:tasks.rados.rados.0.smithi136.stdout:12: seq_num 11 ranges {0=188416} 2024-09-12T23:38:27.448 INFO:tasks.rados.rados.0.smithi136.stdout:12: writing smithi13667081-12 from 0 to 188416 tid 1 2024-09-12T23:38:27.448 INFO:tasks.rados.rados.0.smithi136.stdout:13: write initial oid 13 2024-09-12T23:38:27.448 INFO:tasks.rados.rados.0.smithi136.stdout:13: seq_num 12 ranges {0=204800} 2024-09-12T23:38:27.450 INFO:tasks.rados.rados.0.smithi136.stdout:13: writing smithi13667081-13 from 0 to 204800 tid 1 2024-09-12T23:38:27.450 INFO:tasks.rados.rados.0.smithi136.stdout:14: write initial oid 14 2024-09-12T23:38:27.450 INFO:tasks.rados.rados.0.smithi136.stdout:14: seq_num 13 ranges {0=221184} 2024-09-12T23:38:27.451 INFO:tasks.rados.rados.0.smithi136.stdout:14: writing smithi13667081-14 from 0 to 221184 tid 1 2024-09-12T23:38:27.451 INFO:tasks.rados.rados.0.smithi136.stdout:15: write initial oid 15 2024-09-12T23:38:27.452 INFO:tasks.rados.rados.0.smithi136.stdout:15: seq_num 14 ranges {0=237568} 2024-09-12T23:38:27.453 INFO:tasks.rados.rados.0.smithi136.stdout:15: writing smithi13667081-15 from 0 to 237568 tid 1 2024-09-12T23:38:27.454 INFO:tasks.rados.rados.0.smithi136.stdout:16: write initial oid 16 2024-09-12T23:38:27.454 INFO:tasks.rados.rados.0.smithi136.stdout:16: seq_num 15 ranges {0=253952} 2024-09-12T23:38:27.455 INFO:tasks.rados.rados.0.smithi136.stdout:16: writing smithi13667081-16 from 0 to 253952 tid 1 2024-09-12T23:38:27.455 INFO:tasks.rados.rados.0.smithi136.stdout:17: write initial oid 17 2024-09-12T23:38:27.456 INFO:tasks.rados.rados.0.smithi136.stdout:17: seq_num 16 ranges {0=270336} 2024-09-12T23:38:27.459 INFO:tasks.rados.rados.0.smithi136.stdout:17: writing smithi13667081-17 from 0 to 270336 tid 1 2024-09-12T23:38:27.459 INFO:tasks.rados.rados.0.smithi136.stdout: waiting on 16 2024-09-12T23:38:27.460 INFO:tasks.rados.rados.0.smithi136.stdout:4: finishing write tid 2 to smithi13667081-4 2024-09-12T23:38:27.460 INFO:tasks.rados.rados.0.smithi136.stdout:2: finishing write tid 1 to smithi13667081-2 2024-09-12T23:38:27.460 INFO:tasks.rados.rados.0.smithi136.stdout:2: finishing write tid 2 to smithi13667081-2 2024-09-12T23:38:27.460 INFO:tasks.rados.rados.0.smithi136.stdout:3: finishing write tid 2 to smithi13667081-3 2024-09-12T23:38:27.460 INFO:tasks.rados.rados.0.smithi136.stdout:9: finishing write tid 1 to smithi13667081-9 2024-09-12T23:38:27.460 INFO:tasks.rados.rados.0.smithi136.stdout:9: finishing write tid 2 to smithi13667081-9 2024-09-12T23:38:27.460 INFO:tasks.rados.rados.0.smithi136.stdout:9: finishing write tid 3 to smithi13667081-9 2024-09-12T23:38:27.460 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 9 v 2 (ObjNum 8 snap 0 seq_num 8) dirty exists 2024-09-12T23:38:27.460 INFO:tasks.rados.rados.0.smithi136.stdout:9: left oid 9 (ObjNum 8 snap 0 seq_num 8) 2024-09-12T23:38:27.460 INFO:tasks.rados.rados.0.smithi136.stdout:5: finishing write tid 1 to smithi13667081-5 2024-09-12T23:38:27.460 INFO:tasks.rados.rados.0.smithi136.stdout:5: finishing write tid 2 to smithi13667081-5 2024-09-12T23:38:27.460 INFO:tasks.rados.rados.0.smithi136.stdout:5: finishing write tid 3 to smithi13667081-5 2024-09-12T23:38:27.460 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 5 v 2 (ObjNum 4 snap 0 seq_num 4) dirty exists 2024-09-12T23:38:27.461 INFO:tasks.rados.rados.0.smithi136.stdout:5: left oid 5 (ObjNum 4 snap 0 seq_num 4) 2024-09-12T23:38:27.461 INFO:tasks.rados.rados.0.smithi136.stdout:11: finishing write tid 1 to smithi13667081-11 2024-09-12T23:38:27.461 INFO:tasks.rados.rados.0.smithi136.stdout:11: finishing write tid 2 to smithi13667081-11 2024-09-12T23:38:27.461 INFO:tasks.rados.rados.0.smithi136.stdout:11: finishing write tid 3 to smithi13667081-11 2024-09-12T23:38:27.461 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 11 v 4 (ObjNum 10 snap 0 seq_num 10) dirty exists 2024-09-12T23:38:27.461 INFO:tasks.rados.rados.0.smithi136.stdout:11: left oid 11 (ObjNum 10 snap 0 seq_num 10) 2024-09-12T23:38:27.461 INFO:tasks.rados.rados.0.smithi136.stdout:6: finishing write tid 1 to smithi13667081-6 2024-09-12T23:38:27.461 INFO:tasks.rados.rados.0.smithi136.stdout:6: finishing write tid 2 to smithi13667081-6 2024-09-12T23:38:27.461 INFO:tasks.rados.rados.0.smithi136.stdout:6: finishing write tid 3 to smithi13667081-6 2024-09-12T23:38:27.461 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 6 v 2 (ObjNum 5 snap 0 seq_num 5) dirty exists 2024-09-12T23:38:27.461 INFO:tasks.rados.rados.0.smithi136.stdout:6: left oid 6 (ObjNum 5 snap 0 seq_num 5) 2024-09-12T23:38:27.461 INFO:tasks.rados.rados.0.smithi136.stdout:2: finishing write tid 3 to smithi13667081-2 2024-09-12T23:38:27.462 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 2 v 2 (ObjNum 1 snap 0 seq_num 1) dirty exists 2024-09-12T23:38:27.462 INFO:tasks.rados.rados.0.smithi136.stdout:2: left oid 2 (ObjNum 1 snap 0 seq_num 1) 2024-09-12T23:38:27.462 INFO:tasks.rados.rados.0.smithi136.stdout:4: finishing write tid 3 to smithi13667081-4 2024-09-12T23:38:27.462 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 4 v 2 (ObjNum 3 snap 0 seq_num 3) dirty exists 2024-09-12T23:38:27.462 INFO:tasks.rados.rados.0.smithi136.stdout:4: left oid 4 (ObjNum 3 snap 0 seq_num 3) 2024-09-12T23:38:27.462 INFO:tasks.rados.rados.0.smithi136.stdout:3: finishing write tid 3 to smithi13667081-3 2024-09-12T23:38:27.462 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 3 v 2 (ObjNum 2 snap 0 seq_num 2) dirty exists 2024-09-12T23:38:27.462 INFO:tasks.rados.rados.0.smithi136.stdout:3: left oid 3 (ObjNum 2 snap 0 seq_num 2) 2024-09-12T23:38:27.462 INFO:tasks.rados.rados.0.smithi136.stdout:12: finishing write tid 1 to smithi13667081-12 2024-09-12T23:38:27.462 INFO:tasks.rados.rados.0.smithi136.stdout:2: done (15 left) 2024-09-12T23:38:27.462 INFO:tasks.rados.rados.0.smithi136.stdout:3: done (14 left) 2024-09-12T23:38:27.462 INFO:tasks.rados.rados.0.smithi136.stdout:4: done (13 left) 2024-09-12T23:38:27.462 INFO:tasks.rados.rados.0.smithi136.stdout:5: done (12 left) 2024-09-12T23:38:27.463 INFO:tasks.rados.rados.0.smithi136.stdout:6: done (11 left) 2024-09-12T23:38:27.463 INFO:tasks.rados.rados.0.smithi136.stdout:9: done (10 left) 2024-09-12T23:38:27.463 INFO:tasks.rados.rados.0.smithi136.stdout:11: done (9 left) 2024-09-12T23:38:27.463 INFO:tasks.rados.rados.0.smithi136.stdout:18: write initial oid 18 2024-09-12T23:38:27.463 INFO:tasks.rados.rados.0.smithi136.stdout:18: seq_num 17 ranges {0=286720} 2024-09-12T23:38:27.463 INFO:tasks.rados.rados.0.smithi136.stdout:18: writing smithi13667081-18 from 0 to 286720 tid 1 2024-09-12T23:38:27.464 INFO:tasks.rados.rados.0.smithi136.stdout:19: write initial oid 19 2024-09-12T23:38:27.464 INFO:tasks.rados.rados.0.smithi136.stdout:19: seq_num 18 ranges {0=303104} 2024-09-12T23:38:27.464 INFO:tasks.rados.rados.0.smithi136.stdout:19: writing smithi13667081-19 from 0 to 303104 tid 1 2024-09-12T23:38:27.464 INFO:tasks.rados.rados.0.smithi136.stdout:20: write initial oid 20 2024-09-12T23:38:27.464 INFO:tasks.rados.rados.0.smithi136.stdout:20: seq_num 19 ranges {0=319488} 2024-09-12T23:38:27.465 INFO:tasks.rados.rados.0.smithi136.stdout:20: writing smithi13667081-20 from 0 to 319488 tid 1 2024-09-12T23:38:27.466 INFO:tasks.rados.rados.0.smithi136.stdout:21: write initial oid 21 2024-09-12T23:38:27.466 INFO:tasks.rados.rados.0.smithi136.stdout:21: seq_num 20 ranges {0=344064} 2024-09-12T23:38:27.468 INFO:tasks.rados.rados.0.smithi136.stdout:21: writing smithi13667081-21 from 0 to 344064 tid 1 2024-09-12T23:38:27.468 INFO:tasks.rados.rados.0.smithi136.stdout:12: finishing write tid 2 to smithi13667081-12 2024-09-12T23:38:27.468 INFO:tasks.rados.rados.0.smithi136.stdout:8: finishing write tid 1 to smithi13667081-8 2024-09-12T23:38:27.469 INFO:tasks.rados.rados.0.smithi136.stdout:15: finishing write tid 1 to smithi13667081-15 2024-09-12T23:38:27.469 INFO:tasks.rados.rados.0.smithi136.stdout:7: finishing write tid 1 to smithi13667081-7 2024-09-12T23:38:27.469 INFO:tasks.rados.rados.0.smithi136.stdout:7: finishing write tid 2 to smithi13667081-7 2024-09-12T23:38:27.469 INFO:tasks.rados.rados.0.smithi136.stdout:15: finishing write tid 2 to smithi13667081-15 2024-09-12T23:38:27.469 INFO:tasks.rados.rados.0.smithi136.stdout:12: finishing write tid 3 to smithi13667081-12 2024-09-12T23:38:27.469 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 12 v 4 (ObjNum 11 snap 0 seq_num 11) dirty exists 2024-09-12T23:38:27.469 INFO:tasks.rados.rados.0.smithi136.stdout:12: left oid 12 (ObjNum 11 snap 0 seq_num 11) 2024-09-12T23:38:27.469 INFO:tasks.rados.rados.0.smithi136.stdout:17: finishing write tid 1 to smithi13667081-17 2024-09-12T23:38:27.469 INFO:tasks.rados.rados.0.smithi136.stdout:17: finishing write tid 2 to smithi13667081-17 2024-09-12T23:38:27.469 INFO:tasks.rados.rados.0.smithi136.stdout:15: finishing write tid 3 to smithi13667081-15 2024-09-12T23:38:27.469 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 15 v 6 (ObjNum 14 snap 0 seq_num 14) dirty exists 2024-09-12T23:38:27.469 INFO:tasks.rados.rados.0.smithi136.stdout:15: left oid 15 (ObjNum 14 snap 0 seq_num 14) 2024-09-12T23:38:27.470 INFO:tasks.rados.rados.0.smithi136.stdout:17: finishing write tid 3 to smithi13667081-17 2024-09-12T23:38:27.470 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 17 v 6 (ObjNum 16 snap 0 seq_num 16) dirty exists 2024-09-12T23:38:27.470 INFO:tasks.rados.rados.0.smithi136.stdout:17: left oid 17 (ObjNum 16 snap 0 seq_num 16) 2024-09-12T23:38:27.470 INFO:tasks.rados.rados.0.smithi136.stdout:14: finishing write tid 1 to smithi13667081-14 2024-09-12T23:38:27.470 INFO:tasks.rados.rados.0.smithi136.stdout:14: finishing write tid 2 to smithi13667081-14 2024-09-12T23:38:27.470 INFO:tasks.rados.rados.0.smithi136.stdout:16: finishing write tid 1 to smithi13667081-16 2024-09-12T23:38:27.470 INFO:tasks.rados.rados.0.smithi136.stdout:10: finishing write tid 1 to smithi13667081-10 2024-09-12T23:38:27.470 INFO:tasks.rados.rados.0.smithi136.stdout:16: finishing write tid 2 to smithi13667081-16 2024-09-12T23:38:27.470 INFO:tasks.rados.rados.0.smithi136.stdout:8: finishing write tid 2 to smithi13667081-8 2024-09-12T23:38:27.470 INFO:tasks.rados.rados.0.smithi136.stdout:12: done (12 left) 2024-09-12T23:38:27.470 INFO:tasks.rados.rados.0.smithi136.stdout:15: done (11 left) 2024-09-12T23:38:27.470 INFO:tasks.rados.rados.0.smithi136.stdout:17: done (10 left) 2024-09-12T23:38:27.471 INFO:tasks.rados.rados.0.smithi136.stdout:22: write initial oid 22 2024-09-12T23:38:27.471 INFO:tasks.rados.rados.0.smithi136.stdout:22: seq_num 21 ranges {0=360448} 2024-09-12T23:38:27.471 INFO:tasks.rados.rados.0.smithi136.stdout:22: writing smithi13667081-22 from 0 to 360448 tid 1 2024-09-12T23:38:27.471 INFO:tasks.rados.rados.0.smithi136.stdout:23: write initial oid 23 2024-09-12T23:38:27.471 INFO:tasks.rados.rados.0.smithi136.stdout:23: seq_num 22 ranges {0=376832} 2024-09-12T23:38:27.473 INFO:tasks.rados.rados.0.smithi136.stdout:23: writing smithi13667081-23 from 0 to 376832 tid 1 2024-09-12T23:38:27.473 INFO:tasks.rados.rados.0.smithi136.stdout:8: finishing write tid 3 to smithi13667081-8 2024-09-12T23:38:27.473 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 8 v 2 (ObjNum 7 snap 0 seq_num 7) dirty exists 2024-09-12T23:38:27.474 INFO:tasks.rados.rados.0.smithi136.stdout:8: left oid 8 (ObjNum 7 snap 0 seq_num 7) 2024-09-12T23:38:27.474 INFO:tasks.rados.rados.0.smithi136.stdout:20: finishing write tid 1 to smithi13667081-20 2024-09-12T23:38:27.474 INFO:tasks.rados.rados.0.smithi136.stdout:20: finishing write tid 2 to smithi13667081-20 2024-09-12T23:38:27.474 INFO:tasks.rados.rados.0.smithi136.stdout:10: finishing write tid 2 to smithi13667081-10 2024-09-12T23:38:27.474 INFO:tasks.rados.rados.0.smithi136.stdout:13: finishing write tid 1 to smithi13667081-13 2024-09-12T23:38:27.474 INFO:tasks.rados.rados.0.smithi136.stdout:20: finishing write tid 3 to smithi13667081-20 2024-09-12T23:38:27.474 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 20 v 4 (ObjNum 19 snap 0 seq_num 19) dirty exists 2024-09-12T23:38:27.474 INFO:tasks.rados.rados.0.smithi136.stdout:20: left oid 20 (ObjNum 19 snap 0 seq_num 19) 2024-09-12T23:38:27.474 INFO:tasks.rados.rados.0.smithi136.stdout:14: finishing write tid 3 to smithi13667081-14 2024-09-12T23:38:27.474 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 14 v 2 (ObjNum 13 snap 0 seq_num 13) dirty exists 2024-09-12T23:38:27.474 INFO:tasks.rados.rados.0.smithi136.stdout:14: left oid 14 (ObjNum 13 snap 0 seq_num 13) 2024-09-12T23:38:27.474 INFO:tasks.rados.rados.0.smithi136.stdout:16: finishing write tid 3 to smithi13667081-16 2024-09-12T23:38:27.474 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 16 v 4 (ObjNum 15 snap 0 seq_num 15) dirty exists 2024-09-12T23:38:27.475 INFO:tasks.rados.rados.0.smithi136.stdout:16: left oid 16 (ObjNum 15 snap 0 seq_num 15) 2024-09-12T23:38:27.475 INFO:tasks.rados.rados.0.smithi136.stdout:7: finishing write tid 3 to smithi13667081-7 2024-09-12T23:38:27.475 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 7 v 2 (ObjNum 6 snap 0 seq_num 6) dirty exists 2024-09-12T23:38:27.475 INFO:tasks.rados.rados.0.smithi136.stdout:7: left oid 7 (ObjNum 6 snap 0 seq_num 6) 2024-09-12T23:38:27.475 INFO:tasks.rados.rados.0.smithi136.stdout:13: finishing write tid 2 to smithi13667081-13 2024-09-12T23:38:27.475 INFO:tasks.rados.rados.0.smithi136.stdout:18: finishing write tid 1 to smithi13667081-18 2024-09-12T23:38:27.475 INFO:tasks.rados.rados.0.smithi136.stdout:7: done (11 left) 2024-09-12T23:38:27.475 INFO:tasks.rados.rados.0.smithi136.stdout:8: done (10 left) 2024-09-12T23:38:27.475 INFO:tasks.rados.rados.0.smithi136.stdout:14: done (9 left) 2024-09-12T23:38:27.475 INFO:tasks.rados.rados.0.smithi136.stdout:16: done (8 left) 2024-09-12T23:38:27.475 INFO:tasks.rados.rados.0.smithi136.stdout:20: done (7 left) 2024-09-12T23:38:27.475 INFO:tasks.rados.rados.0.smithi136.stdout:24: write initial oid 24 2024-09-12T23:38:27.475 INFO:tasks.rados.rados.0.smithi136.stdout:24: seq_num 23 ranges {0=393216} 2024-09-12T23:38:27.476 INFO:tasks.rados.rados.0.smithi136.stdout:24: writing smithi13667081-24 from 0 to 393216 tid 1 2024-09-12T23:38:27.476 INFO:tasks.rados.rados.0.smithi136.stdout:25: write initial oid 25 2024-09-12T23:38:27.476 INFO:tasks.rados.rados.0.smithi136.stdout:25: seq_num 24 ranges {0=409600} 2024-09-12T23:38:27.479 INFO:tasks.rados.rados.0.smithi136.stdout:25: writing smithi13667081-25 from 0 to 409600 tid 1 2024-09-12T23:38:27.479 INFO:tasks.rados.rados.0.smithi136.stdout:26: write initial oid 26 2024-09-12T23:38:27.479 INFO:tasks.rados.rados.0.smithi136.stdout:26: seq_num 25 ranges {0=425984} 2024-09-12T23:38:27.482 INFO:tasks.rados.rados.0.smithi136.stdout:26: writing smithi13667081-26 from 0 to 425984 tid 1 2024-09-12T23:38:27.482 INFO:tasks.rados.rados.0.smithi136.stdout:27: write initial oid 27 2024-09-12T23:38:27.482 INFO:tasks.rados.rados.0.smithi136.stdout:27: seq_num 26 ranges {0=442368} 2024-09-12T23:38:27.485 INFO:tasks.rados.rados.0.smithi136.stdout:27: writing smithi13667081-27 from 0 to 442368 tid 1 2024-09-12T23:38:27.485 INFO:tasks.rados.rados.0.smithi136.stdout:28: write initial oid 28 2024-09-12T23:38:27.485 INFO:tasks.rados.rados.0.smithi136.stdout:28: seq_num 27 ranges {0=458752} 2024-09-12T23:38:27.488 INFO:tasks.rados.rados.0.smithi136.stdout:28: writing smithi13667081-28 from 0 to 458752 tid 1 2024-09-12T23:38:27.488 INFO:tasks.rados.rados.0.smithi136.stdout:29: write initial oid 29 2024-09-12T23:38:27.488 INFO:tasks.rados.rados.0.smithi136.stdout:29: seq_num 28 ranges {0=475136} 2024-09-12T23:38:27.491 INFO:tasks.rados.rados.0.smithi136.stdout:29: writing smithi13667081-29 from 0 to 475136 tid 1 2024-09-12T23:38:27.491 INFO:tasks.rados.rados.0.smithi136.stdout:30: write initial oid 30 2024-09-12T23:38:27.491 INFO:tasks.rados.rados.0.smithi136.stdout:30: seq_num 29 ranges {0=491520} 2024-09-12T23:38:27.494 INFO:tasks.rados.rados.0.smithi136.stdout:30: writing smithi13667081-30 from 0 to 491520 tid 1 2024-09-12T23:38:27.494 INFO:tasks.rados.rados.0.smithi136.stdout:31: write initial oid 31 2024-09-12T23:38:27.494 INFO:tasks.rados.rados.0.smithi136.stdout:31: seq_num 30 ranges {0=507904} 2024-09-12T23:38:27.497 INFO:tasks.rados.rados.0.smithi136.stdout:31: writing smithi13667081-31 from 0 to 507904 tid 1 2024-09-12T23:38:27.497 INFO:tasks.rados.rados.0.smithi136.stdout:32: write initial oid 32 2024-09-12T23:38:27.497 INFO:tasks.rados.rados.0.smithi136.stdout:32: seq_num 31 ranges {0=524288} 2024-09-12T23:38:27.501 INFO:tasks.rados.rados.0.smithi136.stdout:32: writing smithi13667081-32 from 0 to 524288 tid 1 2024-09-12T23:38:27.501 INFO:tasks.rados.rados.0.smithi136.stdout: waiting on 16 2024-09-12T23:38:27.501 INFO:tasks.rados.rados.0.smithi136.stdout:19: finishing write tid 1 to smithi13667081-19 2024-09-12T23:38:27.501 INFO:tasks.rados.rados.0.smithi136.stdout:18: finishing write tid 2 to smithi13667081-18 2024-09-12T23:38:27.501 INFO:tasks.rados.rados.0.smithi136.stdout:19: finishing write tid 2 to smithi13667081-19 2024-09-12T23:38:27.502 INFO:tasks.rados.rados.0.smithi136.stdout:21: finishing write tid 1 to smithi13667081-21 2024-09-12T23:38:27.502 INFO:tasks.rados.rados.0.smithi136.stdout:21: finishing write tid 2 to smithi13667081-21 2024-09-12T23:38:27.502 INFO:tasks.rados.rados.0.smithi136.stdout:10: finishing write tid 3 to smithi13667081-10 2024-09-12T23:38:27.502 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 10 v 2 (ObjNum 9 snap 0 seq_num 9) dirty exists 2024-09-12T23:38:27.502 INFO:tasks.rados.rados.0.smithi136.stdout:10: left oid 10 (ObjNum 9 snap 0 seq_num 9) 2024-09-12T23:38:27.502 INFO:tasks.rados.rados.0.smithi136.stdout:13: finishing write tid 3 to smithi13667081-13 2024-09-12T23:38:27.502 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 13 v 2 (ObjNum 12 snap 0 seq_num 12) dirty exists 2024-09-12T23:38:27.502 INFO:tasks.rados.rados.0.smithi136.stdout:13: left oid 13 (ObjNum 12 snap 0 seq_num 12) 2024-09-12T23:38:27.502 INFO:tasks.rados.rados.0.smithi136.stdout:19: finishing write tid 3 to smithi13667081-19 2024-09-12T23:38:27.502 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 19 v 4 (ObjNum 18 snap 0 seq_num 18) dirty exists 2024-09-12T23:38:27.502 INFO:tasks.rados.rados.0.smithi136.stdout:19: left oid 19 (ObjNum 18 snap 0 seq_num 18) 2024-09-12T23:38:27.502 INFO:tasks.rados.rados.0.smithi136.stdout:22: finishing write tid 1 to smithi13667081-22 2024-09-12T23:38:27.503 INFO:tasks.rados.rados.0.smithi136.stdout:22: finishing write tid 2 to smithi13667081-22 2024-09-12T23:38:27.503 INFO:tasks.rados.rados.0.smithi136.stdout:21: finishing write tid 3 to smithi13667081-21 2024-09-12T23:38:27.503 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 21 v 4 (ObjNum 20 snap 0 seq_num 20) dirty exists 2024-09-12T23:38:27.503 INFO:tasks.rados.rados.0.smithi136.stdout:21: left oid 21 (ObjNum 20 snap 0 seq_num 20) 2024-09-12T23:38:27.503 INFO:tasks.rados.rados.0.smithi136.stdout:22: finishing write tid 3 to smithi13667081-22 2024-09-12T23:38:27.503 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 22 v 6 (ObjNum 21 snap 0 seq_num 21) dirty exists 2024-09-12T23:38:27.503 INFO:tasks.rados.rados.0.smithi136.stdout:22: left oid 22 (ObjNum 21 snap 0 seq_num 21) 2024-09-12T23:38:27.503 INFO:tasks.rados.rados.0.smithi136.stdout:23: finishing write tid 1 to smithi13667081-23 2024-09-12T23:38:27.503 INFO:tasks.rados.rados.0.smithi136.stdout:23: finishing write tid 2 to smithi13667081-23 2024-09-12T23:38:27.503 INFO:tasks.rados.rados.0.smithi136.stdout:10: done (15 left) 2024-09-12T23:38:27.503 INFO:tasks.rados.rados.0.smithi136.stdout:13: done (14 left) 2024-09-12T23:38:27.503 INFO:tasks.rados.rados.0.smithi136.stdout:19: done (13 left) 2024-09-12T23:38:27.503 INFO:tasks.rados.rados.0.smithi136.stdout:21: done (12 left) 2024-09-12T23:38:27.503 INFO:tasks.rados.rados.0.smithi136.stdout:22: done (11 left) 2024-09-12T23:38:27.504 INFO:tasks.rados.rados.0.smithi136.stdout:33: write initial oid 33 2024-09-12T23:38:27.504 INFO:tasks.rados.rados.0.smithi136.stdout:33: seq_num 32 ranges {0=540672} 2024-09-12T23:38:27.505 INFO:tasks.rados.rados.0.smithi136.stdout:33: writing smithi13667081-33 from 0 to 540672 tid 1 2024-09-12T23:38:27.505 INFO:tasks.rados.rados.0.smithi136.stdout:18: finishing write tid 3 to smithi13667081-18 2024-09-12T23:38:27.505 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 18 v 4 (ObjNum 17 snap 0 seq_num 17) dirty exists 2024-09-12T23:38:27.505 INFO:tasks.rados.rados.0.smithi136.stdout:18: left oid 18 (ObjNum 17 snap 0 seq_num 17) 2024-09-12T23:38:27.505 INFO:tasks.rados.rados.0.smithi136.stdout:23: finishing write tid 3 to smithi13667081-23 2024-09-12T23:38:27.505 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 23 v 8 (ObjNum 22 snap 0 seq_num 22) dirty exists 2024-09-12T23:38:27.505 INFO:tasks.rados.rados.0.smithi136.stdout:23: left oid 23 (ObjNum 22 snap 0 seq_num 22) 2024-09-12T23:38:27.506 INFO:tasks.rados.rados.0.smithi136.stdout:24: finishing write tid 1 to smithi13667081-24 2024-09-12T23:38:27.506 INFO:tasks.rados.rados.0.smithi136.stdout:24: finishing write tid 2 to smithi13667081-24 2024-09-12T23:38:27.506 INFO:tasks.rados.rados.0.smithi136.stdout:24: finishing write tid 3 to smithi13667081-24 2024-09-12T23:38:27.506 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 24 v 2 (ObjNum 23 snap 0 seq_num 23) dirty exists 2024-09-12T23:38:27.506 INFO:tasks.rados.rados.0.smithi136.stdout:24: left oid 24 (ObjNum 23 snap 0 seq_num 23) 2024-09-12T23:38:27.506 INFO:tasks.rados.rados.0.smithi136.stdout:25: finishing write tid 1 to smithi13667081-25 2024-09-12T23:38:27.506 INFO:tasks.rados.rados.0.smithi136.stdout:25: finishing write tid 2 to smithi13667081-25 2024-09-12T23:38:27.506 INFO:tasks.rados.rados.0.smithi136.stdout:26: finishing write tid 1 to smithi13667081-26 2024-09-12T23:38:27.506 INFO:tasks.rados.rados.0.smithi136.stdout:26: finishing write tid 2 to smithi13667081-26 2024-09-12T23:38:27.506 INFO:tasks.rados.rados.0.smithi136.stdout:25: finishing write tid 3 to smithi13667081-25 2024-09-12T23:38:27.506 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 25 v 2 (ObjNum 24 snap 0 seq_num 24) dirty exists 2024-09-12T23:38:27.506 INFO:tasks.rados.rados.0.smithi136.stdout:25: left oid 25 (ObjNum 24 snap 0 seq_num 24) 2024-09-12T23:38:27.507 INFO:tasks.rados.rados.0.smithi136.stdout:26: finishing write tid 3 to smithi13667081-26 2024-09-12T23:38:27.507 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 26 v 4 (ObjNum 25 snap 0 seq_num 25) dirty exists 2024-09-12T23:38:27.507 INFO:tasks.rados.rados.0.smithi136.stdout:26: left oid 26 (ObjNum 25 snap 0 seq_num 25) 2024-09-12T23:38:27.507 INFO:tasks.rados.rados.0.smithi136.stdout:27: finishing write tid 1 to smithi13667081-27 2024-09-12T23:38:27.507 INFO:tasks.rados.rados.0.smithi136.stdout:27: finishing write tid 2 to smithi13667081-27 2024-09-12T23:38:27.507 INFO:tasks.rados.rados.0.smithi136.stdout:27: finishing write tid 3 to smithi13667081-27 2024-09-12T23:38:27.507 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 27 v 10 (ObjNum 26 snap 0 seq_num 26) dirty exists 2024-09-12T23:38:27.507 INFO:tasks.rados.rados.0.smithi136.stdout:27: left oid 27 (ObjNum 26 snap 0 seq_num 26) 2024-09-12T23:38:27.507 INFO:tasks.rados.rados.0.smithi136.stdout:28: finishing write tid 1 to smithi13667081-28 2024-09-12T23:38:27.507 INFO:tasks.rados.rados.0.smithi136.stdout:28: finishing write tid 2 to smithi13667081-28 2024-09-12T23:38:27.507 INFO:tasks.rados.rados.0.smithi136.stdout:28: finishing write tid 3 to smithi13667081-28 2024-09-12T23:38:27.507 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 28 v 6 (ObjNum 27 snap 0 seq_num 27) dirty exists 2024-09-12T23:38:27.507 INFO:tasks.rados.rados.0.smithi136.stdout:28: left oid 28 (ObjNum 27 snap 0 seq_num 27) 2024-09-12T23:38:27.508 INFO:tasks.rados.rados.0.smithi136.stdout:29: finishing write tid 1 to smithi13667081-29 2024-09-12T23:38:27.508 INFO:tasks.rados.rados.0.smithi136.stdout:29: finishing write tid 2 to smithi13667081-29 2024-09-12T23:38:27.508 INFO:tasks.rados.rados.0.smithi136.stdout:29: finishing write tid 3 to smithi13667081-29 2024-09-12T23:38:27.508 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 29 v 4 (ObjNum 28 snap 0 seq_num 28) dirty exists 2024-09-12T23:38:27.508 INFO:tasks.rados.rados.0.smithi136.stdout:29: left oid 29 (ObjNum 28 snap 0 seq_num 28) 2024-09-12T23:38:27.508 INFO:tasks.rados.rados.0.smithi136.stdout:30: finishing write tid 1 to smithi13667081-30 2024-09-12T23:38:27.508 INFO:tasks.rados.rados.0.smithi136.stdout:31: finishing write tid 1 to smithi13667081-31 2024-09-12T23:38:27.508 INFO:tasks.rados.rados.0.smithi136.stdout:30: finishing write tid 2 to smithi13667081-30 2024-09-12T23:38:27.508 INFO:tasks.rados.rados.0.smithi136.stdout:31: finishing write tid 2 to smithi13667081-31 2024-09-12T23:38:27.508 INFO:tasks.rados.rados.0.smithi136.stdout:30: finishing write tid 3 to smithi13667081-30 2024-09-12T23:38:27.508 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 30 v 4 (ObjNum 29 snap 0 seq_num 29) dirty exists 2024-09-12T23:38:27.508 INFO:tasks.rados.rados.0.smithi136.stdout:30: left oid 30 (ObjNum 29 snap 0 seq_num 29) 2024-09-12T23:38:27.508 INFO:tasks.rados.rados.0.smithi136.stdout:31: finishing write tid 3 to smithi13667081-31 2024-09-12T23:38:27.509 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 31 v 8 (ObjNum 30 snap 0 seq_num 30) dirty exists 2024-09-12T23:38:27.509 INFO:tasks.rados.rados.0.smithi136.stdout:31: left oid 31 (ObjNum 30 snap 0 seq_num 30) 2024-09-12T23:38:27.509 INFO:tasks.rados.rados.0.smithi136.stdout:18: done (11 left) 2024-09-12T23:38:27.509 INFO:tasks.rados.rados.0.smithi136.stdout:23: done (10 left) 2024-09-12T23:38:27.509 INFO:tasks.rados.rados.0.smithi136.stdout:24: done (9 left) 2024-09-12T23:38:27.509 INFO:tasks.rados.rados.0.smithi136.stdout:25: done (8 left) 2024-09-12T23:38:27.509 INFO:tasks.rados.rados.0.smithi136.stdout:26: done (7 left) 2024-09-12T23:38:27.509 INFO:tasks.rados.rados.0.smithi136.stdout:27: done (6 left) 2024-09-12T23:38:27.509 INFO:tasks.rados.rados.0.smithi136.stdout:28: done (5 left) 2024-09-12T23:38:27.509 INFO:tasks.rados.rados.0.smithi136.stdout:29: done (4 left) 2024-09-12T23:38:27.509 INFO:tasks.rados.rados.0.smithi136.stdout:30: done (3 left) 2024-09-12T23:38:27.509 INFO:tasks.rados.rados.0.smithi136.stdout:31: done (2 left) 2024-09-12T23:38:27.509 INFO:tasks.rados.rados.0.smithi136.stdout:34: write initial oid 34 2024-09-12T23:38:27.509 INFO:tasks.rados.rados.0.smithi136.stdout:34: seq_num 33 ranges {0=557056} 2024-09-12T23:38:27.510 INFO:tasks.rados.rados.0.smithi136.stdout:34: writing smithi13667081-34 from 0 to 557056 tid 1 2024-09-12T23:38:27.510 INFO:tasks.rados.rados.0.smithi136.stdout:35: write initial oid 35 2024-09-12T23:38:27.510 INFO:tasks.rados.rados.0.smithi136.stdout:35: seq_num 34 ranges {0=573440} 2024-09-12T23:38:27.512 INFO:tasks.rados.rados.0.smithi136.stdout:35: writing smithi13667081-35 from 0 to 573440 tid 1 2024-09-12T23:38:27.512 INFO:tasks.rados.rados.0.smithi136.stdout:36: write initial oid 36 2024-09-12T23:38:27.512 INFO:tasks.rados.rados.0.smithi136.stdout:36: seq_num 35 ranges {0=589824} 2024-09-12T23:38:27.516 INFO:tasks.rados.rados.0.smithi136.stdout:36: writing smithi13667081-36 from 0 to 589824 tid 1 2024-09-12T23:38:27.516 INFO:tasks.rados.rados.0.smithi136.stdout:37: write initial oid 37 2024-09-12T23:38:27.516 INFO:tasks.rados.rados.0.smithi136.stdout:37: seq_num 36 ranges {0=606208} 2024-09-12T23:38:27.519 INFO:tasks.rados.rados.0.smithi136.stdout:37: writing smithi13667081-37 from 0 to 606208 tid 1 2024-09-12T23:38:27.519 INFO:tasks.rados.rados.0.smithi136.stdout:38: write initial oid 38 2024-09-12T23:38:27.519 INFO:tasks.rados.rados.0.smithi136.stdout:38: seq_num 37 ranges {0=622592} 2024-09-12T23:38:27.523 INFO:tasks.rados.rados.0.smithi136.stdout:38: writing smithi13667081-38 from 0 to 622592 tid 1 2024-09-12T23:38:27.523 INFO:tasks.rados.rados.0.smithi136.stdout:39: write initial oid 39 2024-09-12T23:38:27.524 INFO:tasks.rados.rados.0.smithi136.stdout:39: seq_num 38 ranges {0=638976} 2024-09-12T23:38:27.527 INFO:tasks.rados.rados.0.smithi136.stdout:39: writing smithi13667081-39 from 0 to 638976 tid 1 2024-09-12T23:38:27.527 INFO:tasks.rados.rados.0.smithi136.stdout:40: write initial oid 40 2024-09-12T23:38:27.527 INFO:tasks.rados.rados.0.smithi136.stdout:40: seq_num 39 ranges {0=663552} 2024-09-12T23:38:27.531 INFO:tasks.rados.rados.0.smithi136.stdout:40: writing smithi13667081-40 from 0 to 663552 tid 1 2024-09-12T23:38:27.531 INFO:tasks.rados.rados.0.smithi136.stdout:41: write initial oid 41 2024-09-12T23:38:27.532 INFO:tasks.rados.rados.0.smithi136.stdout:32: finishing write tid 1 to smithi13667081-32 2024-09-12T23:38:27.532 INFO:tasks.rados.rados.0.smithi136.stdout:41: seq_num 40 ranges {0=679936} 2024-09-12T23:38:27.535 INFO:tasks.rados.rados.0.smithi136.stdout:41: writing smithi13667081-41 from 0 to 679936 tid 1 2024-09-12T23:38:27.536 INFO:tasks.rados.rados.0.smithi136.stdout:42: write initial oid 42 2024-09-12T23:38:27.536 INFO:tasks.rados.rados.0.smithi136.stdout:42: seq_num 41 ranges {0=696320} 2024-09-12T23:38:27.540 INFO:tasks.rados.rados.0.smithi136.stdout:42: writing smithi13667081-42 from 0 to 696320 tid 1 2024-09-12T23:38:27.540 INFO:tasks.rados.rados.0.smithi136.stdout:43: write initial oid 43 2024-09-12T23:38:27.540 INFO:tasks.rados.rados.0.smithi136.stdout:43: seq_num 42 ranges {0=712704} 2024-09-12T23:38:27.545 INFO:tasks.rados.rados.0.smithi136.stdout:43: writing smithi13667081-43 from 0 to 712704 tid 1 2024-09-12T23:38:27.545 INFO:tasks.rados.rados.0.smithi136.stdout:44: write initial oid 44 2024-09-12T23:38:27.545 INFO:tasks.rados.rados.0.smithi136.stdout:44: seq_num 43 ranges {0=729088} 2024-09-12T23:38:27.549 INFO:tasks.rados.rados.0.smithi136.stdout:44: writing smithi13667081-44 from 0 to 729088 tid 1 2024-09-12T23:38:27.549 INFO:tasks.rados.rados.0.smithi136.stdout:45: write initial oid 45 2024-09-12T23:38:27.549 INFO:tasks.rados.rados.0.smithi136.stdout:45: seq_num 44 ranges {0=745472} 2024-09-12T23:38:27.554 INFO:tasks.rados.rados.0.smithi136.stdout:45: writing smithi13667081-45 from 0 to 745472 tid 1 2024-09-12T23:38:27.554 INFO:tasks.rados.rados.0.smithi136.stdout:46: write initial oid 46 2024-09-12T23:38:27.554 INFO:tasks.rados.rados.0.smithi136.stdout:46: seq_num 45 ranges {0=761856} 2024-09-12T23:38:27.558 INFO:tasks.rados.rados.0.smithi136.stdout:46: writing smithi13667081-46 from 0 to 761856 tid 1 2024-09-12T23:38:27.558 INFO:tasks.rados.rados.0.smithi136.stdout:47: write initial oid 47 2024-09-12T23:38:27.558 INFO:tasks.rados.rados.0.smithi136.stdout:47: seq_num 46 ranges {0=778240} 2024-09-12T23:38:27.564 INFO:tasks.rados.rados.0.smithi136.stdout:47: writing smithi13667081-47 from 0 to 778240 tid 1 2024-09-12T23:38:27.564 INFO:tasks.rados.rados.0.smithi136.stdout: waiting on 16 2024-09-12T23:38:27.564 INFO:tasks.rados.rados.0.smithi136.stdout:32: finishing write tid 2 to smithi13667081-32 2024-09-12T23:38:27.564 INFO:tasks.rados.rados.0.smithi136.stdout:33: finishing write tid 1 to smithi13667081-33 2024-09-12T23:38:27.564 INFO:tasks.rados.rados.0.smithi136.stdout:32: finishing write tid 3 to smithi13667081-32 2024-09-12T23:38:27.564 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 32 v 4 (ObjNum 31 snap 0 seq_num 31) dirty exists 2024-09-12T23:38:27.564 INFO:tasks.rados.rados.0.smithi136.stdout:32: left oid 32 (ObjNum 31 snap 0 seq_num 31) 2024-09-12T23:38:27.564 INFO:tasks.rados.rados.0.smithi136.stdout:33: finishing write tid 2 to smithi13667081-33 2024-09-12T23:38:27.564 INFO:tasks.rados.rados.0.smithi136.stdout:33: finishing write tid 3 to smithi13667081-33 2024-09-12T23:38:27.565 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 33 v 4 (ObjNum 32 snap 0 seq_num 32) dirty exists 2024-09-12T23:38:27.565 INFO:tasks.rados.rados.0.smithi136.stdout:33: left oid 33 (ObjNum 32 snap 0 seq_num 32) 2024-09-12T23:38:27.565 INFO:tasks.rados.rados.0.smithi136.stdout:34: finishing write tid 1 to smithi13667081-34 2024-09-12T23:38:27.565 INFO:tasks.rados.rados.0.smithi136.stdout:34: finishing write tid 2 to smithi13667081-34 2024-09-12T23:38:27.565 INFO:tasks.rados.rados.0.smithi136.stdout:34: finishing write tid 3 to smithi13667081-34 2024-09-12T23:38:27.565 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 34 v 6 (ObjNum 33 snap 0 seq_num 33) dirty exists 2024-09-12T23:38:27.565 INFO:tasks.rados.rados.0.smithi136.stdout:34: left oid 34 (ObjNum 33 snap 0 seq_num 33) 2024-09-12T23:38:27.565 INFO:tasks.rados.rados.0.smithi136.stdout:35: finishing write tid 1 to smithi13667081-35 2024-09-12T23:38:27.565 INFO:tasks.rados.rados.0.smithi136.stdout:35: finishing write tid 2 to smithi13667081-35 2024-09-12T23:38:27.565 INFO:tasks.rados.rados.0.smithi136.stdout:32: done (15 left) 2024-09-12T23:38:27.565 INFO:tasks.rados.rados.0.smithi136.stdout:33: done (14 left) 2024-09-12T23:38:27.565 INFO:tasks.rados.rados.0.smithi136.stdout:34: done (13 left) 2024-09-12T23:38:27.566 INFO:tasks.rados.rados.0.smithi136.stdout:48: write initial oid 48 2024-09-12T23:38:27.566 INFO:tasks.rados.rados.0.smithi136.stdout:48: seq_num 47 ranges {0=794624} 2024-09-12T23:38:27.568 INFO:tasks.rados.rados.0.smithi136.stdout:48: writing smithi13667081-48 from 0 to 794624 tid 1 2024-09-12T23:38:27.569 INFO:tasks.rados.rados.0.smithi136.stdout:49: write initial oid 49 2024-09-12T23:38:27.569 INFO:tasks.rados.rados.0.smithi136.stdout:49: seq_num 48 ranges {0=409600,409600=401408} 2024-09-12T23:38:27.571 INFO:tasks.rados.rados.0.smithi136.stdout:49: writing smithi13667081-49 from 0 to 409600 tid 1 2024-09-12T23:38:27.574 INFO:tasks.rados.rados.0.smithi136.stdout:49: writing smithi13667081-49 from 409600 to 811008 tid 2 2024-09-12T23:38:27.574 INFO:tasks.rados.rados.0.smithi136.stdout:50: write initial oid 50 2024-09-12T23:38:27.575 INFO:tasks.rados.rados.0.smithi136.stdout:50: seq_num 49 ranges {0=425984,425984=401408} 2024-09-12T23:38:27.576 INFO:tasks.rados.rados.0.smithi136.stdout:50: writing smithi13667081-50 from 0 to 425984 tid 1 2024-09-12T23:38:27.580 INFO:tasks.rados.rados.0.smithi136.stdout:50: writing smithi13667081-50 from 425984 to 827392 tid 2 2024-09-12T23:38:27.580 INFO:tasks.rados.rados.0.smithi136.stdout: waiting on 16 2024-09-12T23:38:27.580 INFO:tasks.rados.rados.0.smithi136.stdout:35: finishing write tid 3 to smithi13667081-35 2024-09-12T23:38:27.580 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 35 v 12 (ObjNum 34 snap 0 seq_num 34) dirty exists 2024-09-12T23:38:27.580 INFO:tasks.rados.rados.0.smithi136.stdout:35: left oid 35 (ObjNum 34 snap 0 seq_num 34) 2024-09-12T23:38:27.580 INFO:tasks.rados.rados.0.smithi136.stdout:36: finishing write tid 1 to smithi13667081-36 2024-09-12T23:38:27.580 INFO:tasks.rados.rados.0.smithi136.stdout:36: finishing write tid 2 to smithi13667081-36 2024-09-12T23:38:27.580 INFO:tasks.rados.rados.0.smithi136.stdout:37: finishing write tid 1 to smithi13667081-37 2024-09-12T23:38:27.580 INFO:tasks.rados.rados.0.smithi136.stdout:35: done (15 left) 2024-09-12T23:38:27.580 INFO:tasks.rados.rados.0.smithi136.stdout:51: copy_from oid 2 from oid 11 current snap is 0 2024-09-12T23:38:27.580 INFO:tasks.rados.rados.0.smithi136.stdout:37: finishing write tid 2 to smithi13667081-37 2024-09-12T23:38:27.581 INFO:tasks.rados.rados.0.smithi136.stdout:36: finishing write tid 3 to smithi13667081-36 2024-09-12T23:38:27.581 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 36 v 4 (ObjNum 35 snap 0 seq_num 35) dirty exists 2024-09-12T23:38:27.581 INFO:tasks.rados.rados.0.smithi136.stdout:36: left oid 36 (ObjNum 35 snap 0 seq_num 35) 2024-09-12T23:38:27.581 INFO:tasks.rados.rados.0.smithi136.stdout:37: finishing write tid 3 to smithi13667081-37 2024-09-12T23:38:27.581 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 37 v 8 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-09-12T23:38:27.581 INFO:tasks.rados.rados.0.smithi136.stdout:37: left oid 37 (ObjNum 36 snap 0 seq_num 36) 2024-09-12T23:38:27.581 INFO:tasks.rados.rados.0.smithi136.stdout:38: finishing write tid 1 to smithi13667081-38 2024-09-12T23:38:27.581 INFO:tasks.rados.rados.0.smithi136.stdout:38: finishing write tid 2 to smithi13667081-38 2024-09-12T23:38:27.581 INFO:tasks.rados.rados.0.smithi136.stdout:38: finishing write tid 3 to smithi13667081-38 2024-09-12T23:38:27.581 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 38 v 6 (ObjNum 37 snap 0 seq_num 37) dirty exists 2024-09-12T23:38:27.581 INFO:tasks.rados.rados.0.smithi136.stdout:38: left oid 38 (ObjNum 37 snap 0 seq_num 37) 2024-09-12T23:38:27.581 INFO:tasks.rados.rados.0.smithi136.stdout:39: finishing write tid 1 to smithi13667081-39 2024-09-12T23:38:27.581 INFO:tasks.rados.rados.0.smithi136.stdout:39: finishing write tid 2 to smithi13667081-39 2024-09-12T23:38:27.581 INFO:tasks.rados.rados.0.smithi136.stdout:40: finishing write tid 1 to smithi13667081-40 2024-09-12T23:38:27.582 INFO:tasks.rados.rados.0.smithi136.stdout:40: finishing write tid 2 to smithi13667081-40 2024-09-12T23:38:27.582 INFO:tasks.rados.rados.0.smithi136.stdout:36: done (15 left) 2024-09-12T23:38:27.582 INFO:tasks.rados.rados.0.smithi136.stdout:37: done (14 left) 2024-09-12T23:38:27.582 INFO:tasks.rados.rados.0.smithi136.stdout:38: done (13 left) 2024-09-12T23:38:27.582 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 15 current snap is 0 2024-09-12T23:38:27.582 INFO:tasks.rados.rados.0.smithi136.stdout:52: seq_num 50 ranges {237568=442368,679936=401408} 2024-09-12T23:38:27.583 INFO:tasks.rados.rados.0.smithi136.stdout:52: writing smithi13667081-15 from 237568 to 679936 tid 1 2024-09-12T23:38:27.587 INFO:tasks.rados.rados.0.smithi136.stdout:52: writing smithi13667081-15 from 679936 to 1081344 tid 2 2024-09-12T23:38:27.587 INFO:tasks.rados.rados.0.smithi136.stdout:53: rmattr oid 6 current snap is 0 2024-09-12T23:38:27.587 INFO:tasks.rados.rados.0.smithi136.stdout:53: done (14 left) 2024-09-12T23:38:27.587 INFO:tasks.rados.rados.0.smithi136.stdout:54: delete oid 6 current snap is 0 2024-09-12T23:38:27.587 INFO:tasks.rados.rados.0.smithi136.stdout:39: finishing write tid 3 to smithi13667081-39 2024-09-12T23:38:27.587 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 39 v 4 (ObjNum 38 snap 0 seq_num 38) dirty exists 2024-09-12T23:38:27.587 INFO:tasks.rados.rados.0.smithi136.stdout:39: left oid 39 (ObjNum 38 snap 0 seq_num 38) 2024-09-12T23:38:27.587 INFO:tasks.rados.rados.0.smithi136.stdout:40: finishing write tid 3 to smithi13667081-40 2024-09-12T23:38:27.587 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 40 v 14 (ObjNum 39 snap 0 seq_num 39) dirty exists 2024-09-12T23:38:27.587 INFO:tasks.rados.rados.0.smithi136.stdout:40: left oid 40 (ObjNum 39 snap 0 seq_num 39) 2024-09-12T23:38:27.588 INFO:tasks.rados.rados.0.smithi136.stdout:41: finishing write tid 1 to smithi13667081-41 2024-09-12T23:38:27.588 INFO:tasks.rados.rados.0.smithi136.stdout:41: finishing write tid 2 to smithi13667081-41 2024-09-12T23:38:27.588 INFO:tasks.rados.rados.0.smithi136.stdout:41: finishing write tid 3 to smithi13667081-41 2024-09-12T23:38:27.588 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 41 v 10 (ObjNum 40 snap 0 seq_num 40) dirty exists 2024-09-12T23:38:27.588 INFO:tasks.rados.rados.0.smithi136.stdout:41: left oid 41 (ObjNum 40 snap 0 seq_num 40) 2024-09-12T23:38:27.588 INFO:tasks.rados.rados.0.smithi136.stdout:42: finishing write tid 1 to smithi13667081-42 2024-09-12T23:38:27.588 INFO:tasks.rados.rados.0.smithi136.stdout:42: finishing write tid 2 to smithi13667081-42 2024-09-12T23:38:27.588 INFO:tasks.rados.rados.0.smithi136.stdout:42: finishing write tid 3 to smithi13667081-42 2024-09-12T23:38:27.588 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 42 v 8 (ObjNum 41 snap 0 seq_num 41) dirty exists 2024-09-12T23:38:27.588 INFO:tasks.rados.rados.0.smithi136.stdout:42: left oid 42 (ObjNum 41 snap 0 seq_num 41) 2024-09-12T23:38:27.588 INFO:tasks.rados.rados.0.smithi136.stdout:43: finishing write tid 1 to smithi13667081-43 2024-09-12T23:38:27.588 INFO:tasks.rados.rados.0.smithi136.stdout:43: finishing write tid 2 to smithi13667081-43 2024-09-12T23:38:27.588 INFO:tasks.rados.rados.0.smithi136.stdout:43: finishing write tid 3 to smithi13667081-43 2024-09-12T23:38:27.589 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 43 v 8 (ObjNum 42 snap 0 seq_num 42) dirty exists 2024-09-12T23:38:27.589 INFO:tasks.rados.rados.0.smithi136.stdout:43: left oid 43 (ObjNum 42 snap 0 seq_num 42) 2024-09-12T23:38:27.589 INFO:tasks.rados.rados.0.smithi136.stdout:44: finishing write tid 1 to smithi13667081-44 2024-09-12T23:38:27.589 INFO:tasks.rados.rados.0.smithi136.stdout:44: finishing write tid 2 to smithi13667081-44 2024-09-12T23:38:27.589 INFO:tasks.rados.rados.0.smithi136.stdout:44: finishing write tid 3 to smithi13667081-44 2024-09-12T23:38:27.589 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 44 v 10 (ObjNum 43 snap 0 seq_num 43) dirty exists 2024-09-12T23:38:27.589 INFO:tasks.rados.rados.0.smithi136.stdout:44: left oid 44 (ObjNum 43 snap 0 seq_num 43) 2024-09-12T23:38:27.589 INFO:tasks.rados.rados.0.smithi136.stdout:45: finishing write tid 1 to smithi13667081-45 2024-09-12T23:38:27.589 INFO:tasks.rados.rados.0.smithi136.stdout:45: finishing write tid 2 to smithi13667081-45 2024-09-12T23:38:27.589 INFO:tasks.rados.rados.0.smithi136.stdout:46: finishing write tid 1 to smithi13667081-46 2024-09-12T23:38:27.589 INFO:tasks.rados.rados.0.smithi136.stdout:46: finishing write tid 2 to smithi13667081-46 2024-09-12T23:38:27.589 INFO:tasks.rados.rados.0.smithi136.stdout:45: finishing write tid 3 to smithi13667081-45 2024-09-12T23:38:27.590 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 45 v 6 (ObjNum 44 snap 0 seq_num 44) dirty exists 2024-09-12T23:38:27.590 INFO:tasks.rados.rados.0.smithi136.stdout:45: left oid 45 (ObjNum 44 snap 0 seq_num 44) 2024-09-12T23:38:27.590 INFO:tasks.rados.rados.0.smithi136.stdout:46: finishing write tid 3 to smithi13667081-46 2024-09-12T23:38:27.590 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 46 v 6 (ObjNum 45 snap 0 seq_num 45) dirty exists 2024-09-12T23:38:27.590 INFO:tasks.rados.rados.0.smithi136.stdout:46: left oid 46 (ObjNum 45 snap 0 seq_num 45) 2024-09-12T23:38:27.590 INFO:tasks.rados.rados.0.smithi136.stdout:47: finishing write tid 1 to smithi13667081-47 2024-09-12T23:38:27.590 INFO:tasks.rados.rados.0.smithi136.stdout:47: finishing write tid 2 to smithi13667081-47 2024-09-12T23:38:27.590 INFO:tasks.rados.rados.0.smithi136.stdout:47: finishing write tid 3 to smithi13667081-47 2024-09-12T23:38:27.590 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 47 v 6 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-09-12T23:38:27.590 INFO:tasks.rados.rados.0.smithi136.stdout:47: left oid 47 (ObjNum 46 snap 0 seq_num 46) 2024-09-12T23:38:27.590 INFO:tasks.rados.rados.0.smithi136.stdout:48: finishing write tid 1 to smithi13667081-48 2024-09-12T23:38:27.590 INFO:tasks.rados.rados.0.smithi136.stdout:48: finishing write tid 2 to smithi13667081-48 2024-09-12T23:38:27.590 INFO:tasks.rados.rados.0.smithi136.stdout:49: finishing write tid 1 to smithi13667081-49 2024-09-12T23:38:27.590 INFO:tasks.rados.rados.0.smithi136.stdout:48: finishing write tid 3 to smithi13667081-48 2024-09-12T23:38:27.591 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 48 v 12 (ObjNum 47 snap 0 seq_num 47) dirty exists 2024-09-12T23:38:27.591 INFO:tasks.rados.rados.0.smithi136.stdout:48: left oid 48 (ObjNum 47 snap 0 seq_num 47) 2024-09-12T23:38:27.591 INFO:tasks.rados.rados.0.smithi136.stdout:49: finishing write tid 2 to smithi13667081-49 2024-09-12T23:38:27.591 INFO:tasks.rados.rados.0.smithi136.stdout:49: finishing write tid 3 to smithi13667081-49 2024-09-12T23:38:27.591 INFO:tasks.rados.rados.0.smithi136.stdout:49: finishing write tid 4 to smithi13667081-49 2024-09-12T23:38:27.591 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 49 v 11 (ObjNum 48 snap 0 seq_num 48) dirty exists 2024-09-12T23:38:27.591 INFO:tasks.rados.rados.0.smithi136.stdout:49: left oid 49 (ObjNum 48 snap 0 seq_num 48) 2024-09-12T23:38:27.591 INFO:tasks.rados.rados.0.smithi136.stdout:50: finishing write tid 1 to smithi13667081-50 2024-09-12T23:38:27.591 INFO:tasks.rados.rados.0.smithi136.stdout:50: finishing write tid 2 to smithi13667081-50 2024-09-12T23:38:27.591 INFO:tasks.rados.rados.0.smithi136.stdout:50: finishing write tid 3 to smithi13667081-50 2024-09-12T23:38:27.591 INFO:tasks.rados.rados.0.smithi136.stdout:51: finishing copy_from racing read to smithi13667081-2 2024-09-12T23:38:27.591 INFO:tasks.rados.rados.0.smithi136.stdout:51: finishing copy_from to smithi13667081-2 2024-09-12T23:38:27.591 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 2 v 4 (ObjNum 10 snap 0 seq_num 10) dirty exists 2024-09-12T23:38:27.591 INFO:tasks.rados.rados.0.smithi136.stdout:50: finishing write tid 4 to smithi13667081-50 2024-09-12T23:38:27.592 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 50 v 9 (ObjNum 49 snap 0 seq_num 49) dirty exists 2024-09-12T23:38:27.592 INFO:tasks.rados.rados.0.smithi136.stdout:50: left oid 50 (ObjNum 49 snap 0 seq_num 49) 2024-09-12T23:38:27.592 INFO:tasks.rados.rados.0.smithi136.stdout:52: finishing write tid 1 to smithi13667081-15 2024-09-12T23:38:27.593 INFO:tasks.rados.rados.0.smithi136.stdout:39: done (14 left) 2024-09-12T23:38:27.593 INFO:tasks.rados.rados.0.smithi136.stdout:40: done (13 left) 2024-09-12T23:38:27.593 INFO:tasks.rados.rados.0.smithi136.stdout:41: done (12 left) 2024-09-12T23:38:27.593 INFO:tasks.rados.rados.0.smithi136.stdout:42: done (11 left) 2024-09-12T23:38:27.593 INFO:tasks.rados.rados.0.smithi136.stdout:43: done (10 left) 2024-09-12T23:38:27.593 INFO:tasks.rados.rados.0.smithi136.stdout:44: done (9 left) 2024-09-12T23:38:27.594 INFO:tasks.rados.rados.0.smithi136.stdout:45: done (8 left) 2024-09-12T23:38:27.594 INFO:tasks.rados.rados.0.smithi136.stdout:46: done (7 left) 2024-09-12T23:38:27.594 INFO:tasks.rados.rados.0.smithi136.stdout:47: done (6 left) 2024-09-12T23:38:27.594 INFO:tasks.rados.rados.0.smithi136.stdout:48: done (5 left) 2024-09-12T23:38:27.594 INFO:tasks.rados.rados.0.smithi136.stdout:49: done (4 left) 2024-09-12T23:38:27.594 INFO:tasks.rados.rados.0.smithi136.stdout:50: done (3 left) 2024-09-12T23:38:27.594 INFO:tasks.rados.rados.0.smithi136.stdout:51: done (2 left) 2024-09-12T23:38:27.594 INFO:tasks.rados.rados.0.smithi136.stdout:54: done (1 left) 2024-09-12T23:38:27.594 INFO:tasks.rados.rados.0.smithi136.stdout:55: read oid 16 snap -1 2024-09-12T23:38:27.594 INFO:tasks.rados.rados.0.smithi136.stdout:55: expect (ObjNum 15 snap 0 seq_num 15) 2024-09-12T23:38:27.594 INFO:tasks.rados.rados.0.smithi136.stdout:56: rollback oid 31 current snap is 0 2024-09-12T23:38:27.594 INFO:tasks.rados.rados.0.smithi136.stdout:56: done (2 left) 2024-09-12T23:38:27.594 INFO:tasks.rados.rados.0.smithi136.stdout:57: read oid 28 snap -1 2024-09-12T23:38:27.595 INFO:tasks.rados.rados.0.smithi136.stdout:57: expect (ObjNum 27 snap 0 seq_num 27) 2024-09-12T23:38:27.595 INFO:tasks.rados.rados.0.smithi136.stdout:58: copy_from oid 29 from oid 47 current snap is 0 2024-09-12T23:38:27.595 INFO:tasks.rados.rados.0.smithi136.stdout:59: rollback oid 45 current snap is 0 2024-09-12T23:38:27.595 INFO:tasks.rados.rados.0.smithi136.stdout:59: done (4 left) 2024-09-12T23:38:27.595 INFO:tasks.rados.rados.0.smithi136.stdout:60: copy_from oid 23 from oid 48 current snap is 0 2024-09-12T23:38:27.595 INFO:tasks.rados.rados.0.smithi136.stdout:61: delete oid 1 current snap is 0 2024-09-12T23:38:27.595 INFO:tasks.rados.rados.0.smithi136.stdout:52: finishing write tid 2 to smithi13667081-15 2024-09-12T23:38:27.595 INFO:tasks.rados.rados.0.smithi136.stdout:61: done (5 left) 2024-09-12T23:38:27.595 INFO:tasks.rados.rados.0.smithi136.stdout:62: read oid 18 snap -1 2024-09-12T23:38:27.595 INFO:tasks.rados.rados.0.smithi136.stdout:62: expect (ObjNum 17 snap 0 seq_num 17) 2024-09-12T23:38:27.595 INFO:tasks.rados.rados.0.smithi136.stdout:63: snap_create 2024-09-12T23:38:27.595 INFO:tasks.rados.rados.0.smithi136.stdout:52: finishing write tid 3 to smithi13667081-15 2024-09-12T23:38:27.596 INFO:tasks.rados.rados.0.smithi136.stdout:52: finishing write tid 4 to smithi13667081-15 2024-09-12T23:38:27.596 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 15 v 14 (ObjNum 50 snap 0 seq_num 50) dirty exists 2024-09-12T23:38:27.596 INFO:tasks.rados.rados.0.smithi136.stdout:52: left oid 15 (ObjNum 50 snap 0 seq_num 50) 2024-09-12T23:38:27.596 INFO:tasks.rados.rados.0.smithi136.stdout:55: expect (ObjNum 15 snap 0 seq_num 15) 2024-09-12T23:38:27.600 INFO:tasks.rados.rados.0.smithi136.stdout:57: expect (ObjNum 27 snap 0 seq_num 27) 2024-09-12T23:38:27.612 INFO:tasks.rados.rados.0.smithi136.stdout:62: expect (ObjNum 17 snap 0 seq_num 17) 2024-09-12T23:38:27.620 INFO:tasks.rados.rados.0.smithi136.stdout:60: finishing copy_from racing read to smithi13667081-23 2024-09-12T23:38:27.620 INFO:tasks.rados.rados.0.smithi136.stdout:60: finishing copy_from to smithi13667081-23 2024-09-12T23:38:27.620 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 23 v 12 (ObjNum 47 snap 0 seq_num 47) dirty exists 2024-09-12T23:38:27.620 INFO:tasks.rados.rados.0.smithi136.stdout:58: finishing copy_from racing read to smithi13667081-29 2024-09-12T23:38:27.620 INFO:tasks.rados.rados.0.smithi136.stdout:58: finishing copy_from to smithi13667081-29 2024-09-12T23:38:27.621 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 29 v 6 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-09-12T23:38:27.755 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:27 smithi136 bash[32189]: cluster 2024-09-12T23:38:26.014888+0000 mgr.y (mgr.24446) 188 : cluster [DBG] pgmap v147: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:38:27.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:27 smithi136 bash[32189]: cluster 2024-09-12T23:38:26.426222+0000 mon.a (mon.0) 1085 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:38:27.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:27 smithi136 bash[32189]: cluster 2024-09-12T23:38:26.426276+0000 mon.a (mon.0) 1086 : cluster [INF] Cluster is now healthy 2024-09-12T23:38:27.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:27 smithi136 bash[32189]: audit 2024-09-12T23:38:27.402116+0000 mon.a (mon.0) 1087 : audit [INF] from='client.? 172.21.15.136:0/2828853910' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2024-09-12T23:38:27.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:27 smithi136 bash[22601]: cluster 2024-09-12T23:38:26.014888+0000 mgr.y (mgr.24446) 188 : cluster [DBG] pgmap v147: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:38:27.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:27 smithi136 bash[22601]: cluster 2024-09-12T23:38:26.426222+0000 mon.a (mon.0) 1085 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:38:27.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:27 smithi136 bash[22601]: cluster 2024-09-12T23:38:26.426276+0000 mon.a (mon.0) 1086 : cluster [INF] Cluster is now healthy 2024-09-12T23:38:27.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:27 smithi136 bash[22601]: audit 2024-09-12T23:38:27.402116+0000 mon.a (mon.0) 1087 : audit [INF] from='client.? 172.21.15.136:0/2828853910' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2024-09-12T23:38:27.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:27 smithi181 bash[24776]: cluster 2024-09-12T23:38:26.014888+0000 mgr.y (mgr.24446) 188 : cluster [DBG] pgmap v147: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:38:27.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:27 smithi181 bash[24776]: cluster 2024-09-12T23:38:26.426222+0000 mon.a (mon.0) 1085 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:38:27.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:27 smithi181 bash[24776]: cluster 2024-09-12T23:38:26.426276+0000 mon.a (mon.0) 1086 : cluster [INF] Cluster is now healthy 2024-09-12T23:38:27.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:27 smithi181 bash[24776]: audit 2024-09-12T23:38:27.402116+0000 mon.a (mon.0) 1087 : audit [INF] from='client.? 172.21.15.136:0/2828853910' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2024-09-12T23:38:28.455 INFO:tasks.rados.rados.0.smithi136.stdout:52: done (6 left) 2024-09-12T23:38:28.455 INFO:tasks.rados.rados.0.smithi136.stdout:55: done (5 left) 2024-09-12T23:38:28.455 INFO:tasks.rados.rados.0.smithi136.stdout:57: done (4 left) 2024-09-12T23:38:28.455 INFO:tasks.rados.rados.0.smithi136.stdout:58: done (3 left) 2024-09-12T23:38:28.455 INFO:tasks.rados.rados.0.smithi136.stdout:60: done (2 left) 2024-09-12T23:38:28.456 INFO:tasks.rados.rados.0.smithi136.stdout:62: done (1 left) 2024-09-12T23:38:28.456 INFO:tasks.rados.rados.0.smithi136.stdout:63: done (0 left) 2024-09-12T23:38:28.456 INFO:tasks.rados.rados.0.smithi136.stdout:64: copy_from oid 15 from oid 2 current snap is 1 2024-09-12T23:38:28.456 INFO:tasks.rados.rados.0.smithi136.stdout:65: read oid 30 snap -1 2024-09-12T23:38:28.456 INFO:tasks.rados.rados.0.smithi136.stdout:65: expect (ObjNum 29 snap 0 seq_num 29) 2024-09-12T23:38:28.456 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 23 current snap is 1 2024-09-12T23:38:28.456 INFO:tasks.rados.rados.0.smithi136.stdout:66: seq_num 53 ranges {794624=491520,1286144=401408} 2024-09-12T23:38:28.463 INFO:tasks.rados.rados.0.smithi136.stdout:66: writing smithi13667081-23 from 794624 to 1286144 tid 1 2024-09-12T23:38:28.467 INFO:tasks.rados.rados.0.smithi136.stdout:66: writing smithi13667081-23 from 1286144 to 1687552 tid 2 2024-09-12T23:38:28.467 INFO:tasks.rados.rados.0.smithi136.stdout:67: rollback oid 10 current snap is 1 2024-09-12T23:38:28.467 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 10 to 0 2024-09-12T23:38:28.467 INFO:tasks.rados.rados.0.smithi136.stdout:65: expect (ObjNum 29 snap 0 seq_num 29) 2024-09-12T23:38:28.484 INFO:tasks.rados.rados.0.smithi136.stdout:64: finishing copy_from racing read to smithi13667081-15 2024-09-12T23:38:28.484 INFO:tasks.rados.rados.0.smithi136.stdout:67: finishing rollback tid 0 to smithi13667081-10 2024-09-12T23:38:28.485 INFO:tasks.rados.rados.0.smithi136.stdout:67: finishing rollback tid 1 to smithi13667081-10 2024-09-12T23:38:28.485 INFO:tasks.rados.rados.0.smithi136.stdout:64: finishing copy_from to smithi13667081-15 2024-09-12T23:38:28.485 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 15 v 4 (ObjNum 10 snap 0 seq_num 10) dirty exists 2024-09-12T23:38:28.485 INFO:tasks.rados.rados.0.smithi136.stdout:67: finishing rollback tid 2 to smithi13667081-10 2024-09-12T23:38:28.485 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 10 v 8 (ObjNum 9 snap 0 seq_num 9) dirty exists 2024-09-12T23:38:28.485 INFO:tasks.rados.rados.0.smithi136.stdout:66: finishing write tid 1 to smithi13667081-23 2024-09-12T23:38:28.485 INFO:tasks.rados.rados.0.smithi136.stdout:66: finishing write tid 2 to smithi13667081-23 2024-09-12T23:38:28.485 INFO:tasks.rados.rados.0.smithi136.stdout:64: done (3 left) 2024-09-12T23:38:28.485 INFO:tasks.rados.rados.0.smithi136.stdout:65: done (2 left) 2024-09-12T23:38:28.485 INFO:tasks.rados.rados.0.smithi136.stdout:67: done (1 left) 2024-09-12T23:38:28.485 INFO:tasks.rados.rados.0.smithi136.stdout:68: setattr oid 31 current snap is 1 2024-09-12T23:38:28.485 INFO:tasks.rados.rados.0.smithi136.stdout:66: finishing write tid 3 to smithi13667081-23 2024-09-12T23:38:28.486 INFO:tasks.rados.rados.0.smithi136.stdout:66: finishing write tid 4 to smithi13667081-23 2024-09-12T23:38:28.486 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 23 v 19 (ObjNum 53 snap 1 seq_num 53) dirty exists 2024-09-12T23:38:28.486 INFO:tasks.rados.rados.0.smithi136.stdout:66: left oid 23 (ObjNum 53 snap 1 seq_num 53) 2024-09-12T23:38:28.487 INFO:tasks.rados.rados.0.smithi136.stdout:66: done (1 left) 2024-09-12T23:38:28.487 INFO:tasks.rados.rados.0.smithi136.stdout:69: copy_from oid 45 from oid 34 current snap is 1 2024-09-12T23:38:28.487 INFO:tasks.rados.rados.0.smithi136.stdout:70: read oid 23 snap 0 2024-09-12T23:38:28.488 INFO:tasks.rados.rados.0.smithi136.stdout:70: expect (ObjNum 47 snap 0 seq_num 47) 2024-09-12T23:38:28.488 INFO:tasks.rados.rados.0.smithi136.stdout:71: copy_from oid 36 from oid 27 current snap is 1 2024-09-12T23:38:28.488 INFO:tasks.rados.rados.0.smithi136.stdout:72: rollback oid 49 current snap is 1 2024-09-12T23:38:28.488 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 49 to 0 2024-09-12T23:38:28.488 INFO:tasks.rados.rados.0.smithi136.stdout:73: read oid 4 snap 0 2024-09-12T23:38:28.488 INFO:tasks.rados.rados.0.smithi136.stdout:73: expect (ObjNum 3 snap 0 seq_num 3) 2024-09-12T23:38:28.488 INFO:tasks.rados.rados.0.smithi136.stdout:74: rmattr oid 7 current snap is 1 2024-09-12T23:38:28.488 INFO:tasks.rados.rados.0.smithi136.stdout:74: done (6 left) 2024-09-12T23:38:28.488 INFO:tasks.rados.rados.0.smithi136.stdout:75: copy_from oid 20 from oid 12 current snap is 1 2024-09-12T23:38:28.488 INFO:tasks.rados.rados.0.smithi136.stdout:76: read oid 39 snap -1 2024-09-12T23:38:28.488 INFO:tasks.rados.rados.0.smithi136.stdout:76: expect (ObjNum 38 snap 0 seq_num 38) 2024-09-12T23:38:28.488 INFO:tasks.rados.rados.0.smithi136.stdout:77: copy_from oid 6 from oid 37 current snap is 1 2024-09-12T23:38:28.489 INFO:tasks.rados.rados.0.smithi136.stdout:78: read oid 2 snap -1 2024-09-12T23:38:28.489 INFO:tasks.rados.rados.0.smithi136.stdout:78: expect (ObjNum 10 snap 0 seq_num 10) 2024-09-12T23:38:28.489 INFO:tasks.rados.rados.0.smithi136.stdout:79: setattr oid 41 current snap is 1 2024-09-12T23:38:28.489 INFO:tasks.rados.rados.0.smithi136.stdout:80: snap_create 2024-09-12T23:38:28.493 INFO:tasks.rados.rados.0.smithi136.stdout:73: expect (ObjNum 3 snap 0 seq_num 3) 2024-09-12T23:38:28.499 INFO:tasks.rados.rados.0.smithi136.stdout:70: expect (ObjNum 47 snap 0 seq_num 47) 2024-09-12T23:38:28.521 INFO:tasks.rados.rados.0.smithi136.stdout:76: expect (ObjNum 38 snap 0 seq_num 38) 2024-09-12T23:38:28.537 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 31 v 18 (ObjNum 30 snap 0 seq_num 30) dirty exists 2024-09-12T23:38:28.537 INFO:tasks.rados.rados.0.smithi136.stdout:72: finishing rollback tid 0 to smithi13667081-49 2024-09-12T23:38:28.537 INFO:tasks.rados.rados.0.smithi136.stdout:72: finishing rollback tid 1 to smithi13667081-49 2024-09-12T23:38:28.537 INFO:tasks.rados.rados.0.smithi136.stdout:78: expect (ObjNum 10 snap 0 seq_num 10) 2024-09-12T23:38:28.542 INFO:tasks.rados.rados.0.smithi136.stdout:75: finishing copy_from racing read to smithi13667081-20 2024-09-12T23:38:28.542 INFO:tasks.rados.rados.0.smithi136.stdout:72: finishing rollback tid 2 to smithi13667081-49 2024-09-12T23:38:28.542 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 49 v 22 (ObjNum 48 snap 0 seq_num 48) dirty exists 2024-09-12T23:38:28.542 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 41 v 13 (ObjNum 40 snap 0 seq_num 40) dirty exists 2024-09-12T23:38:28.542 INFO:tasks.rados.rados.0.smithi136.stdout:77: finishing copy_from racing read to smithi13667081-6 2024-09-12T23:38:28.542 INFO:tasks.rados.rados.0.smithi136.stdout:75: finishing copy_from to smithi13667081-20 2024-09-12T23:38:28.542 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 20 v 4 (ObjNum 11 snap 0 seq_num 11) dirty exists 2024-09-12T23:38:28.542 INFO:tasks.rados.rados.0.smithi136.stdout:77: finishing copy_from to smithi13667081-6 2024-09-12T23:38:28.543 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 6 v 8 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-09-12T23:38:28.543 INFO:tasks.rados.rados.0.smithi136.stdout:69: finishing copy_from racing read to smithi13667081-45 2024-09-12T23:38:28.543 INFO:tasks.rados.rados.0.smithi136.stdout:71: finishing copy_from racing read to smithi13667081-36 2024-09-12T23:38:28.543 INFO:tasks.rados.rados.0.smithi136.stdout:71: finishing copy_from to smithi13667081-36 2024-09-12T23:38:28.543 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 36 v 10 (ObjNum 26 snap 0 seq_num 26) dirty exists 2024-09-12T23:38:28.543 INFO:tasks.rados.rados.0.smithi136.stdout:69: finishing copy_from to smithi13667081-45 2024-09-12T23:38:28.543 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 45 v 6 (ObjNum 33 snap 0 seq_num 33) dirty exists 2024-09-12T23:38:28.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:28 smithi136 bash[22601]: audit 2024-09-12T23:38:27.440351+0000 mon.a (mon.0) 1088 : audit [INF] from='client.? 172.21.15.136:0/2828853910' entity='client.0' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]': finished 2024-09-12T23:38:28.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:28 smithi136 bash[22601]: cluster 2024-09-12T23:38:27.440439+0000 mon.a (mon.0) 1089 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2024-09-12T23:38:28.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:28 smithi136 bash[32189]: audit 2024-09-12T23:38:27.440351+0000 mon.a (mon.0) 1088 : audit [INF] from='client.? 172.21.15.136:0/2828853910' entity='client.0' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]': finished 2024-09-12T23:38:28.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:28 smithi136 bash[32189]: cluster 2024-09-12T23:38:27.440439+0000 mon.a (mon.0) 1089 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2024-09-12T23:38:28.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:28 smithi181 bash[24776]: audit 2024-09-12T23:38:27.440351+0000 mon.a (mon.0) 1088 : audit [INF] from='client.? 172.21.15.136:0/2828853910' entity='client.0' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]': finished 2024-09-12T23:38:28.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:28 smithi181 bash[24776]: cluster 2024-09-12T23:38:27.440439+0000 mon.a (mon.0) 1089 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2024-09-12T23:38:29.453 INFO:tasks.rados.rados.0.smithi136.stdout:68: done (11 left) 2024-09-12T23:38:29.453 INFO:tasks.rados.rados.0.smithi136.stdout:69: done (10 left) 2024-09-12T23:38:29.453 INFO:tasks.rados.rados.0.smithi136.stdout:70: done (9 left) 2024-09-12T23:38:29.453 INFO:tasks.rados.rados.0.smithi136.stdout:71: done (8 left) 2024-09-12T23:38:29.453 INFO:tasks.rados.rados.0.smithi136.stdout:72: done (7 left) 2024-09-12T23:38:29.453 INFO:tasks.rados.rados.0.smithi136.stdout:73: done (6 left) 2024-09-12T23:38:29.454 INFO:tasks.rados.rados.0.smithi136.stdout:75: done (5 left) 2024-09-12T23:38:29.454 INFO:tasks.rados.rados.0.smithi136.stdout:76: done (4 left) 2024-09-12T23:38:29.454 INFO:tasks.rados.rados.0.smithi136.stdout:77: done (3 left) 2024-09-12T23:38:29.454 INFO:tasks.rados.rados.0.smithi136.stdout:78: done (2 left) 2024-09-12T23:38:29.454 INFO:tasks.rados.rados.0.smithi136.stdout:79: done (1 left) 2024-09-12T23:38:29.454 INFO:tasks.rados.rados.0.smithi136.stdout:80: done (0 left) 2024-09-12T23:38:29.454 INFO:tasks.rados.rados.0.smithi136.stdout:81: rollback oid 40 current snap is 2 2024-09-12T23:38:29.454 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 40 to 1 2024-09-12T23:38:29.454 INFO:tasks.rados.rados.0.smithi136.stdout:82: snap_create 2024-09-12T23:38:29.469 INFO:tasks.rados.rados.0.smithi136.stdout:81: finishing rollback tid 0 to smithi13667081-40 2024-09-12T23:38:29.470 INFO:tasks.rados.rados.0.smithi136.stdout:81: finishing rollback tid 1 to smithi13667081-40 2024-09-12T23:38:29.470 INFO:tasks.rados.rados.0.smithi136.stdout:81: finishing rollback tid 2 to smithi13667081-40 2024-09-12T23:38:29.470 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 40 v 23 (ObjNum 39 snap 0 seq_num 39) dirty exists 2024-09-12T23:38:29.755 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:29 smithi136 bash[32189]: cluster 2024-09-12T23:38:28.015694+0000 mgr.y (mgr.24446) 189 : cluster [DBG] pgmap v149: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:38:29.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:29 smithi136 bash[32189]: cluster 2024-09-12T23:38:28.458936+0000 mon.a (mon.0) 1090 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2024-09-12T23:38:29.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:29 smithi136 bash[22601]: cluster 2024-09-12T23:38:28.015694+0000 mgr.y (mgr.24446) 189 : cluster [DBG] pgmap v149: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:38:29.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:29 smithi136 bash[22601]: cluster 2024-09-12T23:38:28.458936+0000 mon.a (mon.0) 1090 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2024-09-12T23:38:29.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:29 smithi181 bash[24776]: cluster 2024-09-12T23:38:28.015694+0000 mgr.y (mgr.24446) 189 : cluster [DBG] pgmap v149: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:38:29.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:29 smithi181 bash[24776]: cluster 2024-09-12T23:38:28.458936+0000 mon.a (mon.0) 1090 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2024-09-12T23:38:30.457 INFO:tasks.rados.rados.0.smithi136.stdout:81: done (1 left) 2024-09-12T23:38:30.457 INFO:tasks.rados.rados.0.smithi136.stdout:82: done (0 left) 2024-09-12T23:38:30.457 INFO:tasks.rados.rados.0.smithi136.stdout:83: rollback oid 9 current snap is 3 2024-09-12T23:38:30.457 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 9 to 0 2024-09-12T23:38:30.457 INFO:tasks.rados.rados.0.smithi136.stdout:84: read oid 20 snap 1 2024-09-12T23:38:30.457 INFO:tasks.rados.rados.0.smithi136.stdout:84: expect (ObjNum 11 snap 0 seq_num 11) 2024-09-12T23:38:30.457 INFO:tasks.rados.rados.0.smithi136.stdout:85: read oid 41 snap -1 2024-09-12T23:38:30.458 INFO:tasks.rados.rados.0.smithi136.stdout:85: expect (ObjNum 40 snap 0 seq_num 40) 2024-09-12T23:38:30.458 INFO:tasks.rados.rados.0.smithi136.stdout:86: snap_create 2024-09-12T23:38:30.468 INFO:tasks.rados.rados.0.smithi136.stdout:83: finishing rollback tid 0 to smithi13667081-9 2024-09-12T23:38:30.475 INFO:tasks.rados.rados.0.smithi136.stdout:85: expect (ObjNum 40 snap 0 seq_num 40) 2024-09-12T23:38:30.508 INFO:tasks.rados.rados.0.smithi136.stdout:83: finishing rollback tid 1 to smithi13667081-9 2024-09-12T23:38:30.509 INFO:tasks.rados.rados.0.smithi136.stdout:83: finishing rollback tid 2 to smithi13667081-9 2024-09-12T23:38:30.509 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 9 v 10 (ObjNum 8 snap 0 seq_num 8) dirty exists 2024-09-12T23:38:30.509 INFO:tasks.rados.rados.0.smithi136.stdout:84: expect (ObjNum 11 snap 0 seq_num 11) 2024-09-12T23:38:30.755 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:30 smithi136 bash[32189]: cluster 2024-09-12T23:38:29.455699+0000 mon.a (mon.0) 1091 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2024-09-12T23:38:30.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:30 smithi136 bash[32189]: cluster 2024-09-12T23:38:30.459528+0000 mon.a (mon.0) 1092 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2024-09-12T23:38:30.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:30 smithi136 bash[22601]: cluster 2024-09-12T23:38:29.455699+0000 mon.a (mon.0) 1091 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2024-09-12T23:38:30.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:30 smithi136 bash[22601]: cluster 2024-09-12T23:38:30.459528+0000 mon.a (mon.0) 1092 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2024-09-12T23:38:30.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:30 smithi181 bash[24776]: cluster 2024-09-12T23:38:29.455699+0000 mon.a (mon.0) 1091 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2024-09-12T23:38:30.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:30 smithi181 bash[24776]: cluster 2024-09-12T23:38:30.459528+0000 mon.a (mon.0) 1092 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2024-09-12T23:38:31.458 INFO:tasks.rados.rados.0.smithi136.stdout:83: done (3 left) 2024-09-12T23:38:31.458 INFO:tasks.rados.rados.0.smithi136.stdout:84: done (2 left) 2024-09-12T23:38:31.458 INFO:tasks.rados.rados.0.smithi136.stdout:85: done (1 left) 2024-09-12T23:38:31.458 INFO:tasks.rados.rados.0.smithi136.stdout:86: done (0 left) 2024-09-12T23:38:31.459 INFO:tasks.rados.rados.0.smithi136.stdout:87: snap_remove snap 0 2024-09-12T23:38:31.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:31 smithi136 bash[22601]: cluster 2024-09-12T23:38:30.016362+0000 mgr.y (mgr.24446) 190 : cluster [DBG] pgmap v152: 121 pgs: 121 active+clean; 25 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 5.4 MiB/s wr, 66 op/s 2024-09-12T23:38:31.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:31 smithi136 bash[22601]: cluster 2024-09-12T23:38:31.461975+0000 mon.a (mon.0) 1093 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2024-09-12T23:38:31.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:31 smithi136 bash[32189]: cluster 2024-09-12T23:38:30.016362+0000 mgr.y (mgr.24446) 190 : cluster [DBG] pgmap v152: 121 pgs: 121 active+clean; 25 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 5.4 MiB/s wr, 66 op/s 2024-09-12T23:38:31.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:31 smithi136 bash[32189]: cluster 2024-09-12T23:38:31.461975+0000 mon.a (mon.0) 1093 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2024-09-12T23:38:31.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:31 smithi181 bash[24776]: cluster 2024-09-12T23:38:30.016362+0000 mgr.y (mgr.24446) 190 : cluster [DBG] pgmap v152: 121 pgs: 121 active+clean; 25 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 5.4 MiB/s wr, 66 op/s 2024-09-12T23:38:31.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:31 smithi181 bash[24776]: cluster 2024-09-12T23:38:31.461975+0000 mon.a (mon.0) 1093 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2024-09-12T23:38:32.462 INFO:tasks.rados.rados.0.smithi136.stdout:87: done (0 left) 2024-09-12T23:38:32.462 INFO:tasks.rados.rados.0.smithi136.stdout:88: read oid 29 snap 3 2024-09-12T23:38:32.462 INFO:tasks.rados.rados.0.smithi136.stdout:88: expect (ObjNum 46 snap 0 seq_num 46) 2024-09-12T23:38:32.463 INFO:tasks.rados.rados.0.smithi136.stdout:89: read oid 46 snap -1 2024-09-12T23:38:32.463 INFO:tasks.rados.rados.0.smithi136.stdout:89: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-12T23:38:32.463 INFO:tasks.rados.rados.0.smithi136.stdout:90: copy_from oid 36 from oid 2 current snap is 4 2024-09-12T23:38:32.463 INFO:tasks.rados.rados.0.smithi136.stdout:91: rmattr oid 49 current snap is 4 2024-09-12T23:38:32.463 INFO:tasks.rados.rados.0.smithi136.stdout:91: done (3 left) 2024-09-12T23:38:32.463 INFO:tasks.rados.rados.0.smithi136.stdout:92: snap_remove snap 2 2024-09-12T23:38:33.475 INFO:tasks.rados.rados.0.smithi136.stdout:92: done (3 left) 2024-09-12T23:38:33.475 INFO:tasks.rados.rados.0.smithi136.stdout:93: rmattr oid 30 current snap is 4 2024-09-12T23:38:33.475 INFO:tasks.rados.rados.0.smithi136.stdout:93: done (3 left) 2024-09-12T23:38:33.475 INFO:tasks.rados.rados.0.smithi136.stdout:94: read oid 42 snap 1 2024-09-12T23:38:33.475 INFO:tasks.rados.rados.0.smithi136.stdout:94: expect (ObjNum 41 snap 0 seq_num 41) 2024-09-12T23:38:33.475 INFO:tasks.rados.rados.0.smithi136.stdout:95: rmattr oid 14 current snap is 4 2024-09-12T23:38:33.475 INFO:tasks.rados.rados.0.smithi136.stdout:95: done (4 left) 2024-09-12T23:38:33.475 INFO:tasks.rados.rados.0.smithi136.stdout:96: read oid 9 snap 1 2024-09-12T23:38:33.475 INFO:tasks.rados.rados.0.smithi136.stdout:96: expect (ObjNum 8 snap 0 seq_num 8) 2024-09-12T23:38:33.475 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 24 current snap is 4 2024-09-12T23:38:33.475 INFO:tasks.rados.rados.0.smithi136.stdout:97: seq_num 54 ranges {393216=507904,901120=401408} 2024-09-12T23:38:33.483 INFO:tasks.rados.rados.0.smithi136.stdout:97: writing smithi13667081-24 from 393216 to 901120 tid 1 2024-09-12T23:38:33.487 INFO:tasks.rados.rados.0.smithi136.stdout:97: writing smithi13667081-24 from 901120 to 1302528 tid 2 2024-09-12T23:38:33.487 INFO:tasks.rados.rados.0.smithi136.stdout:98: copy_from oid 31 from oid 6 current snap is 4 2024-09-12T23:38:33.487 INFO:tasks.rados.rados.0.smithi136.stdout:90: finishing copy_from racing read to smithi13667081-36 2024-09-12T23:38:33.487 INFO:tasks.rados.rados.0.smithi136.stdout:89: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-12T23:38:33.515 INFO:tasks.rados.rados.0.smithi136.stdout:90: finishing copy_from to smithi13667081-36 2024-09-12T23:38:33.516 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 36 v 4 (ObjNum 10 snap 0 seq_num 10) dirty exists 2024-09-12T23:38:33.516 INFO:tasks.rados.rados.0.smithi136.stdout:88: expect (ObjNum 46 snap 0 seq_num 46) 2024-09-12T23:38:33.539 INFO:tasks.rados.rados.0.smithi136.stdout:94: expect (ObjNum 41 snap 0 seq_num 41) 2024-09-12T23:38:33.562 INFO:tasks.rados.rados.0.smithi136.stdout:97: finishing write tid 1 to smithi13667081-24 2024-09-12T23:38:33.562 INFO:tasks.rados.rados.0.smithi136.stdout:97: finishing write tid 2 to smithi13667081-24 2024-09-12T23:38:33.562 INFO:tasks.rados.rados.0.smithi136.stdout:88: done (6 left) 2024-09-12T23:38:33.562 INFO:tasks.rados.rados.0.smithi136.stdout:89: done (5 left) 2024-09-12T23:38:33.562 INFO:tasks.rados.rados.0.smithi136.stdout:90: done (4 left) 2024-09-12T23:38:33.562 INFO:tasks.rados.rados.0.smithi136.stdout:94: done (3 left) 2024-09-12T23:38:33.562 INFO:tasks.rados.rados.0.smithi136.stdout:99: snap_remove snap 3 2024-09-12T23:38:33.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:33 smithi136 bash[22601]: cluster 2024-09-12T23:38:32.017053+0000 mgr.y (mgr.24446) 191 : cluster [DBG] pgmap v155: 121 pgs: 121 active+clean; 25 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 6.3 MiB/s wr, 75 op/s 2024-09-12T23:38:33.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:33 smithi136 bash[22601]: cluster 2024-09-12T23:38:32.465712+0000 mon.a (mon.0) 1094 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2024-09-12T23:38:33.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:33 smithi136 bash[32189]: cluster 2024-09-12T23:38:32.017053+0000 mgr.y (mgr.24446) 191 : cluster [DBG] pgmap v155: 121 pgs: 121 active+clean; 25 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 6.3 MiB/s wr, 75 op/s 2024-09-12T23:38:33.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:33 smithi136 bash[32189]: cluster 2024-09-12T23:38:32.465712+0000 mon.a (mon.0) 1094 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2024-09-12T23:38:33.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:33 smithi181 bash[24776]: cluster 2024-09-12T23:38:32.017053+0000 mgr.y (mgr.24446) 191 : cluster [DBG] pgmap v155: 121 pgs: 121 active+clean; 25 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 6.3 MiB/s wr, 75 op/s 2024-09-12T23:38:33.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:33 smithi181 bash[24776]: cluster 2024-09-12T23:38:32.465712+0000 mon.a (mon.0) 1094 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2024-09-12T23:38:34.479 INFO:tasks.rados.rados.0.smithi136.stdout:99: done (3 left) 2024-09-12T23:38:34.479 INFO:tasks.rados.rados.0.smithi136.stdout:100: copy_from oid 37 from oid 35 current snap is 4 2024-09-12T23:38:34.479 INFO:tasks.rados.rados.0.smithi136.stdout:97: finishing write tid 3 to smithi13667081-24 2024-09-12T23:38:34.479 INFO:tasks.rados.rados.0.smithi136.stdout:96: expect (ObjNum 8 snap 0 seq_num 8) 2024-09-12T23:38:34.484 INFO:tasks.rados.rados.0.smithi136.stdout:97: finishing write tid 4 to smithi13667081-24 2024-09-12T23:38:34.484 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 24 v 10 (ObjNum 54 snap 4 seq_num 54) dirty exists 2024-09-12T23:38:34.484 INFO:tasks.rados.rados.0.smithi136.stdout:97: left oid 24 (ObjNum 54 snap 4 seq_num 54) 2024-09-12T23:38:34.484 INFO:tasks.rados.rados.0.smithi136.stdout:98: finishing copy_from racing read to smithi13667081-31 2024-09-12T23:38:34.484 INFO:tasks.rados.rados.0.smithi136.stdout:98: finishing copy_from to smithi13667081-31 2024-09-12T23:38:34.484 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 31 v 8 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-09-12T23:38:34.484 INFO:tasks.rados.rados.0.smithi136.stdout:96: done (3 left) 2024-09-12T23:38:34.484 INFO:tasks.rados.rados.0.smithi136.stdout:97: done (2 left) 2024-09-12T23:38:34.485 INFO:tasks.rados.rados.0.smithi136.stdout:98: done (1 left) 2024-09-12T23:38:34.485 INFO:tasks.rados.rados.0.smithi136.stdout:101: read oid 14 snap 1 2024-09-12T23:38:34.485 INFO:tasks.rados.rados.0.smithi136.stdout:101: expect (ObjNum 13 snap 0 seq_num 13) 2024-09-12T23:38:34.487 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 49 current snap is 4 2024-09-12T23:38:34.487 INFO:tasks.rados.rados.0.smithi136.stdout:102: seq_num 55 ranges {811008=524288,1335296=401408} 2024-09-12T23:38:34.492 INFO:tasks.rados.rados.0.smithi136.stdout:102: writing smithi13667081-49 from 811008 to 1335296 tid 1 2024-09-12T23:38:34.495 INFO:tasks.rados.rados.0.smithi136.stdout:102: writing smithi13667081-49 from 1335296 to 1736704 tid 2 2024-09-12T23:38:34.495 INFO:tasks.rados.rados.0.smithi136.stdout:103: delete oid 43 current snap is 4 2024-09-12T23:38:34.495 INFO:tasks.rados.rados.0.smithi136.stdout:101: expect (ObjNum 13 snap 0 seq_num 13) 2024-09-12T23:38:34.504 INFO:tasks.rados.rados.0.smithi136.stdout:102: finishing write tid 1 to smithi13667081-49 2024-09-12T23:38:34.504 INFO:tasks.rados.rados.0.smithi136.stdout:102: finishing write tid 2 to smithi13667081-49 2024-09-12T23:38:34.504 INFO:tasks.rados.rados.0.smithi136.stdout:102: finishing write tid 3 to smithi13667081-49 2024-09-12T23:38:34.504 INFO:tasks.rados.rados.0.smithi136.stdout:100: finishing copy_from racing read to smithi13667081-37 2024-09-12T23:38:34.504 INFO:tasks.rados.rados.0.smithi136.stdout:102: finishing write tid 4 to smithi13667081-49 2024-09-12T23:38:34.505 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 49 v 36 (ObjNum 55 snap 4 seq_num 55) dirty exists 2024-09-12T23:38:34.505 INFO:tasks.rados.rados.0.smithi136.stdout:102: left oid 49 (ObjNum 55 snap 4 seq_num 55) 2024-09-12T23:38:34.505 INFO:tasks.rados.rados.0.smithi136.stdout:100: finishing copy_from to smithi13667081-37 2024-09-12T23:38:34.505 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 37 v 12 (ObjNum 34 snap 0 seq_num 34) dirty exists 2024-09-12T23:38:34.505 INFO:tasks.rados.rados.0.smithi136.stdout:100: done (3 left) 2024-09-12T23:38:34.505 INFO:tasks.rados.rados.0.smithi136.stdout:101: done (2 left) 2024-09-12T23:38:34.505 INFO:tasks.rados.rados.0.smithi136.stdout:102: done (1 left) 2024-09-12T23:38:34.505 INFO:tasks.rados.rados.0.smithi136.stdout:103: done (0 left) 2024-09-12T23:38:34.505 INFO:tasks.rados.rados.0.smithi136.stdout:104: read oid 43 snap -1 2024-09-12T23:38:34.505 INFO:tasks.rados.rados.0.smithi136.stdout:104: expect deleted 2024-09-12T23:38:34.505 INFO:tasks.rados.rados.0.smithi136.stdout:105: rollback oid 38 current snap is 4 2024-09-12T23:38:34.505 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 38 to 1 2024-09-12T23:38:34.505 INFO:tasks.rados.rados.0.smithi136.stdout:106: delete oid 12 current snap is 4 2024-09-12T23:38:34.507 INFO:tasks.rados.rados.0.smithi136.stdout:104: done (2 left) 2024-09-12T23:38:34.507 INFO:tasks.rados.rados.0.smithi136.stdout:106: done (1 left) 2024-09-12T23:38:34.507 INFO:tasks.rados.rados.0.smithi136.stdout:107: read oid 9 snap 1 2024-09-12T23:38:34.507 INFO:tasks.rados.rados.0.smithi136.stdout:107: expect (ObjNum 8 snap 0 seq_num 8) 2024-09-12T23:38:34.507 INFO:tasks.rados.rados.0.smithi136.stdout:108: rollback oid 14 current snap is 4 2024-09-12T23:38:34.507 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 14 to 1 2024-09-12T23:38:34.507 INFO:tasks.rados.rados.0.smithi136.stdout:109: delete oid 13 current snap is 4 2024-09-12T23:38:34.509 INFO:tasks.rados.rados.0.smithi136.stdout:107: expect (ObjNum 8 snap 0 seq_num 8) 2024-09-12T23:38:34.513 INFO:tasks.rados.rados.0.smithi136.stdout:108: finishing rollback tid 0 to smithi13667081-14 2024-09-12T23:38:34.514 INFO:tasks.rados.rados.0.smithi136.stdout:108: finishing rollback tid 1 to smithi13667081-14 2024-09-12T23:38:34.514 INFO:tasks.rados.rados.0.smithi136.stdout:108: finishing rollback tid 2 to smithi13667081-14 2024-09-12T23:38:34.514 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 14 v 18 (ObjNum 13 snap 0 seq_num 13) dirty exists 2024-09-12T23:38:34.514 INFO:tasks.rados.rados.0.smithi136.stdout:105: finishing rollback tid 0 to smithi13667081-38 2024-09-12T23:38:34.514 INFO:tasks.rados.rados.0.smithi136.stdout:105: finishing rollback tid 1 to smithi13667081-38 2024-09-12T23:38:34.514 INFO:tasks.rados.rados.0.smithi136.stdout:105: finishing rollback tid 2 to smithi13667081-38 2024-09-12T23:38:34.514 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 38 v 14 (ObjNum 37 snap 0 seq_num 37) dirty exists 2024-09-12T23:38:34.515 INFO:tasks.rados.rados.0.smithi136.stdout:105: done (3 left) 2024-09-12T23:38:34.515 INFO:tasks.rados.rados.0.smithi136.stdout:107: done (2 left) 2024-09-12T23:38:34.515 INFO:tasks.rados.rados.0.smithi136.stdout:108: done (1 left) 2024-09-12T23:38:34.515 INFO:tasks.rados.rados.0.smithi136.stdout:109: done (0 left) 2024-09-12T23:38:34.515 INFO:tasks.rados.rados.0.smithi136.stdout:110: setattr oid 35 current snap is 4 2024-09-12T23:38:34.515 INFO:tasks.rados.rados.0.smithi136.stdout:111: snap_create 2024-09-12T23:38:34.518 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 35 v 29 (ObjNum 34 snap 0 seq_num 34) dirty exists 2024-09-12T23:38:34.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:34 smithi136 bash[22601]: cluster 2024-09-12T23:38:33.478154+0000 mon.a (mon.0) 1095 : cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2024-09-12T23:38:34.755 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:34 smithi136 bash[32189]: cluster 2024-09-12T23:38:33.478154+0000 mon.a (mon.0) 1095 : cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2024-09-12T23:38:34.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:34 smithi181 bash[24776]: cluster 2024-09-12T23:38:33.478154+0000 mon.a (mon.0) 1095 : cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2024-09-12T23:38:35.439 INFO:tasks.rados.rados.0.smithi136.stdout:110: done (1 left) 2024-09-12T23:38:35.439 INFO:tasks.rados.rados.0.smithi136.stdout:111: done (0 left) 2024-09-12T23:38:35.439 INFO:tasks.rados.rados.0.smithi136.stdout:112: copy_from oid 25 from oid 6 current snap is 5 2024-09-12T23:38:35.439 INFO:tasks.rados.rados.0.smithi136.stdout:113: setattr oid 23 current snap is 5 2024-09-12T23:38:35.440 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 15 current snap is 5 2024-09-12T23:38:35.440 INFO:tasks.rados.rados.0.smithi136.stdout:114: seq_num 59 ranges {172032=598016,770048=401408} 2024-09-12T23:38:35.447 INFO:tasks.rados.rados.0.smithi136.stdout:114: writing smithi13667081-15 from 172032 to 770048 tid 1 2024-09-12T23:38:35.451 INFO:tasks.rados.rados.0.smithi136.stdout:114: writing smithi13667081-15 from 770048 to 1171456 tid 2 2024-09-12T23:38:35.451 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 20 current snap is 5 2024-09-12T23:38:35.451 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 23 v 31 (ObjNum 53 snap 1 seq_num 53) dirty exists 2024-09-12T23:38:35.451 INFO:tasks.rados.rados.0.smithi136.stdout:115: seq_num 60 ranges {188416=614400,802816=401408} 2024-09-12T23:38:35.459 INFO:tasks.rados.rados.0.smithi136.stdout:115: writing smithi13667081-20 from 188416 to 802816 tid 1 2024-09-12T23:38:35.463 INFO:tasks.rados.rados.0.smithi136.stdout:115: writing smithi13667081-20 from 802816 to 1204224 tid 2 2024-09-12T23:38:35.463 INFO:tasks.rados.rados.0.smithi136.stdout:113: done (3 left) 2024-09-12T23:38:35.463 INFO:tasks.rados.rados.0.smithi136.stdout:116: rollback oid 14 current snap is 5 2024-09-12T23:38:35.463 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 14 to 1 2024-09-12T23:38:35.464 INFO:tasks.rados.rados.0.smithi136.stdout:114: finishing write tid 1 to smithi13667081-15 2024-09-12T23:38:35.464 INFO:tasks.rados.rados.0.smithi136.stdout:114: finishing write tid 2 to smithi13667081-15 2024-09-12T23:38:35.464 INFO:tasks.rados.rados.0.smithi136.stdout:117: read oid 31 snap -1 2024-09-12T23:38:35.464 INFO:tasks.rados.rados.0.smithi136.stdout:117: expect (ObjNum 36 snap 0 seq_num 36) 2024-09-12T23:38:35.464 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 49 current snap is 5 2024-09-12T23:38:35.464 INFO:tasks.rados.rados.0.smithi136.stdout:118: seq_num 61 ranges {1736704=630784,2367488=401408} 2024-09-12T23:38:35.475 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:38:35 smithi181 bash[25717]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:38:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-12T23:38:35.483 INFO:tasks.rados.rados.0.smithi136.stdout:118: writing smithi13667081-49 from 1736704 to 2367488 tid 1 2024-09-12T23:38:35.488 INFO:tasks.rados.rados.0.smithi136.stdout:118: writing smithi13667081-49 from 2367488 to 2768896 tid 2 2024-09-12T23:38:35.488 INFO:tasks.rados.rados.0.smithi136.stdout:119: snap_create 2024-09-12T23:38:35.488 INFO:tasks.rados.rados.0.smithi136.stdout:112: finishing copy_from racing read to smithi13667081-25 2024-09-12T23:38:35.488 INFO:tasks.rados.rados.0.smithi136.stdout:112: finishing copy_from to smithi13667081-25 2024-09-12T23:38:35.488 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 25 v 8 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-09-12T23:38:35.488 INFO:tasks.rados.rados.0.smithi136.stdout:114: finishing write tid 3 to smithi13667081-15 2024-09-12T23:38:35.488 INFO:tasks.rados.rados.0.smithi136.stdout:115: finishing write tid 1 to smithi13667081-20 2024-09-12T23:38:35.489 INFO:tasks.rados.rados.0.smithi136.stdout:115: finishing write tid 2 to smithi13667081-20 2024-09-12T23:38:35.489 INFO:tasks.rados.rados.0.smithi136.stdout:116: finishing rollback tid 0 to smithi13667081-14 2024-09-12T23:38:35.489 INFO:tasks.rados.rados.0.smithi136.stdout:116: finishing rollback tid 1 to smithi13667081-14 2024-09-12T23:38:35.489 INFO:tasks.rados.rados.0.smithi136.stdout:115: finishing write tid 3 to smithi13667081-20 2024-09-12T23:38:35.489 INFO:tasks.rados.rados.0.smithi136.stdout:116: finishing rollback tid 2 to smithi13667081-14 2024-09-12T23:38:35.489 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 14 v 22 (ObjNum 13 snap 0 seq_num 13) dirty exists 2024-09-12T23:38:35.489 INFO:tasks.rados.rados.0.smithi136.stdout:115: finishing write tid 4 to smithi13667081-20 2024-09-12T23:38:35.489 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 20 v 14 (ObjNum 60 snap 5 seq_num 60) dirty exists 2024-09-12T23:38:35.489 INFO:tasks.rados.rados.0.smithi136.stdout:115: left oid 20 (ObjNum 60 snap 5 seq_num 60) 2024-09-12T23:38:35.489 INFO:tasks.rados.rados.0.smithi136.stdout:114: finishing write tid 4 to smithi13667081-15 2024-09-12T23:38:35.489 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 15 v 42 (ObjNum 59 snap 5 seq_num 59) dirty exists 2024-09-12T23:38:35.489 INFO:tasks.rados.rados.0.smithi136.stdout:114: left oid 15 (ObjNum 59 snap 5 seq_num 59) 2024-09-12T23:38:35.490 INFO:tasks.rados.rados.0.smithi136.stdout:117: expect (ObjNum 36 snap 0 seq_num 36) 2024-09-12T23:38:35.506 INFO:tasks.rados.rados.0.smithi136.stdout:118: finishing write tid 1 to smithi13667081-49 2024-09-12T23:38:35.506 INFO:tasks.rados.rados.0.smithi136.stdout:118: finishing write tid 2 to smithi13667081-49 2024-09-12T23:38:35.506 INFO:tasks.rados.rados.0.smithi136.stdout:118: finishing write tid 3 to smithi13667081-49 2024-09-12T23:38:35.506 INFO:tasks.rados.rados.0.smithi136.stdout:118: finishing write tid 4 to smithi13667081-49 2024-09-12T23:38:35.506 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 49 v 46 (ObjNum 61 snap 5 seq_num 61) dirty exists 2024-09-12T23:38:35.506 INFO:tasks.rados.rados.0.smithi136.stdout:118: left oid 49 (ObjNum 61 snap 5 seq_num 61) 2024-09-12T23:38:35.755 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:35 smithi136 bash[32189]: cluster 2024-09-12T23:38:34.017772+0000 mgr.y (mgr.24446) 192 : cluster [DBG] pgmap v158: 121 pgs: 121 active+clean; 24 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 412 KiB/s wr, 8 op/s 2024-09-12T23:38:35.755 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:35 smithi136 bash[32189]: cluster 2024-09-12T23:38:34.483969+0000 mon.a (mon.0) 1096 : cluster [DBG] osdmap e78: 8 total, 8 up, 8 in 2024-09-12T23:38:35.755 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:35 smithi136 bash[32189]: cluster 2024-09-12T23:38:35.442982+0000 mon.a (mon.0) 1097 : cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2024-09-12T23:38:35.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:35 smithi136 bash[22601]: cluster 2024-09-12T23:38:34.017772+0000 mgr.y (mgr.24446) 192 : cluster [DBG] pgmap v158: 121 pgs: 121 active+clean; 24 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 412 KiB/s wr, 8 op/s 2024-09-12T23:38:35.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:35 smithi136 bash[22601]: cluster 2024-09-12T23:38:34.483969+0000 mon.a (mon.0) 1096 : cluster [DBG] osdmap e78: 8 total, 8 up, 8 in 2024-09-12T23:38:35.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:35 smithi136 bash[22601]: cluster 2024-09-12T23:38:35.442982+0000 mon.a (mon.0) 1097 : cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2024-09-12T23:38:35.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:35 smithi181 bash[24776]: cluster 2024-09-12T23:38:34.017772+0000 mgr.y (mgr.24446) 192 : cluster [DBG] pgmap v158: 121 pgs: 121 active+clean; 24 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 412 KiB/s wr, 8 op/s 2024-09-12T23:38:35.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:35 smithi181 bash[24776]: cluster 2024-09-12T23:38:34.483969+0000 mon.a (mon.0) 1096 : cluster [DBG] osdmap e78: 8 total, 8 up, 8 in 2024-09-12T23:38:35.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:35 smithi181 bash[24776]: cluster 2024-09-12T23:38:35.442982+0000 mon.a (mon.0) 1097 : cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2024-09-12T23:38:36.488 INFO:tasks.rados.rados.0.smithi136.stdout:112: done (6 left) 2024-09-12T23:38:36.488 INFO:tasks.rados.rados.0.smithi136.stdout:114: done (5 left) 2024-09-12T23:38:36.488 INFO:tasks.rados.rados.0.smithi136.stdout:115: done (4 left) 2024-09-12T23:38:36.488 INFO:tasks.rados.rados.0.smithi136.stdout:116: done (3 left) 2024-09-12T23:38:36.488 INFO:tasks.rados.rados.0.smithi136.stdout:117: done (2 left) 2024-09-12T23:38:36.488 INFO:tasks.rados.rados.0.smithi136.stdout:118: done (1 left) 2024-09-12T23:38:36.488 INFO:tasks.rados.rados.0.smithi136.stdout:119: done (0 left) 2024-09-12T23:38:36.489 INFO:tasks.rados.rados.0.smithi136.stdout:120: read oid 48 snap 4 2024-09-12T23:38:36.489 INFO:tasks.rados.rados.0.smithi136.stdout:120: expect (ObjNum 47 snap 0 seq_num 47) 2024-09-12T23:38:36.489 INFO:tasks.rados.rados.0.smithi136.stdout:121: read oid 49 snap 5 2024-09-12T23:38:36.489 INFO:tasks.rados.rados.0.smithi136.stdout:121: expect (ObjNum 61 snap 5 seq_num 61) 2024-09-12T23:38:36.489 INFO:tasks.rados.rados.0.smithi136.stdout:122: delete oid 44 current snap is 6 2024-09-12T23:38:36.497 INFO:tasks.rados.rados.0.smithi136.stdout:120: expect (ObjNum 47 snap 0 seq_num 47) 2024-09-12T23:38:36.505 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:38:36 smithi136 bash[22852]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:38:36] "GET /metrics HTTP/1.1" 200 238982 "" "Prometheus/2.43.0" 2024-09-12T23:38:36.537 INFO:tasks.rados.rados.0.smithi136.stdout:121: expect (ObjNum 61 snap 5 seq_num 61) 2024-09-12T23:38:36.678 INFO:tasks.rados.rados.0.smithi136.stdout:120: done (2 left) 2024-09-12T23:38:36.678 INFO:tasks.rados.rados.0.smithi136.stdout:121: done (1 left) 2024-09-12T23:38:36.678 INFO:tasks.rados.rados.0.smithi136.stdout:122: done (0 left) 2024-09-12T23:38:36.679 INFO:tasks.rados.rados.0.smithi136.stdout:123: rollback oid 49 current snap is 6 2024-09-12T23:38:36.679 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 49 to 4 2024-09-12T23:38:36.679 INFO:tasks.rados.rados.0.smithi136.stdout:124: read oid 27 snap -1 2024-09-12T23:38:36.679 INFO:tasks.rados.rados.0.smithi136.stdout:124: expect (ObjNum 26 snap 0 seq_num 26) 2024-09-12T23:38:36.679 INFO:tasks.rados.rados.0.smithi136.stdout:125: setattr oid 41 current snap is 6 2024-09-12T23:38:36.680 INFO:tasks.rados.rados.0.smithi136.stdout:126: delete oid 28 current snap is 6 2024-09-12T23:38:36.682 INFO:tasks.rados.rados.0.smithi136.stdout:123: finishing rollback tid 0 to smithi13667081-49 2024-09-12T23:38:36.683 INFO:tasks.rados.rados.0.smithi136.stdout:126: done (3 left) 2024-09-12T23:38:36.683 INFO:tasks.rados.rados.0.smithi136.stdout:127: snap_remove snap 5 2024-09-12T23:38:37.507 INFO:tasks.rados.rados.0.smithi136.stdout:127: done (3 left) 2024-09-12T23:38:37.507 INFO:tasks.rados.rados.0.smithi136.stdout:128: setattr oid 50 current snap is 6 2024-09-12T23:38:37.507 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 32 current snap is 6 2024-09-12T23:38:37.507 INFO:tasks.rados.rados.0.smithi136.stdout:129: seq_num 64 ranges {524288=679936,1204224=401408} 2024-09-12T23:38:37.513 INFO:tasks.rados.rados.0.smithi136.stdout:129: writing smithi13667081-32 from 524288 to 1204224 tid 1 2024-09-12T23:38:37.516 INFO:tasks.rados.rados.0.smithi136.stdout:129: writing smithi13667081-32 from 1204224 to 1605632 tid 2 2024-09-12T23:38:37.516 INFO:tasks.rados.rados.0.smithi136.stdout:130: read oid 44 snap 1 2024-09-12T23:38:37.516 INFO:tasks.rados.rados.0.smithi136.stdout:130: expect (ObjNum 43 snap 0 seq_num 43) 2024-09-12T23:38:37.516 INFO:tasks.rados.rados.0.smithi136.stdout:131: rollback oid 11 current snap is 6 2024-09-12T23:38:37.516 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 11 to 1 2024-09-12T23:38:37.516 INFO:tasks.rados.rados.0.smithi136.stdout:123: finishing rollback tid 1 to smithi13667081-49 2024-09-12T23:38:37.516 INFO:tasks.rados.rados.0.smithi136.stdout:123: finishing rollback tid 2 to smithi13667081-49 2024-09-12T23:38:37.516 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 49 v 50 (ObjNum 55 snap 4 seq_num 55) dirty exists 2024-09-12T23:38:37.516 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 41 v 19 (ObjNum 40 snap 0 seq_num 40) dirty exists 2024-09-12T23:38:37.517 INFO:tasks.rados.rados.0.smithi136.stdout:124: expect (ObjNum 26 snap 0 seq_num 26) 2024-09-12T23:38:37.532 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 50 v 18 (ObjNum 49 snap 0 seq_num 49) dirty exists 2024-09-12T23:38:37.532 INFO:tasks.rados.rados.0.smithi136.stdout:131: finishing rollback tid 0 to smithi13667081-11 2024-09-12T23:38:37.532 INFO:tasks.rados.rados.0.smithi136.stdout:131: finishing rollback tid 1 to smithi13667081-11 2024-09-12T23:38:37.532 INFO:tasks.rados.rados.0.smithi136.stdout:131: finishing rollback tid 2 to smithi13667081-11 2024-09-12T23:38:37.532 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 11 v 35 (ObjNum 10 snap 0 seq_num 10) dirty exists 2024-09-12T23:38:37.532 INFO:tasks.rados.rados.0.smithi136.stdout:130: expect (ObjNum 43 snap 0 seq_num 43) 2024-09-12T23:38:37.556 INFO:tasks.rados.rados.0.smithi136.stdout:129: finishing write tid 1 to smithi13667081-32 2024-09-12T23:38:37.556 INFO:tasks.rados.rados.0.smithi136.stdout:129: finishing write tid 2 to smithi13667081-32 2024-09-12T23:38:37.556 INFO:tasks.rados.rados.0.smithi136.stdout:129: finishing write tid 3 to smithi13667081-32 2024-09-12T23:38:37.556 INFO:tasks.rados.rados.0.smithi136.stdout:129: finishing write tid 4 to smithi13667081-32 2024-09-12T23:38:37.556 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 32 v 8 (ObjNum 64 snap 6 seq_num 64) dirty exists 2024-09-12T23:38:37.556 INFO:tasks.rados.rados.0.smithi136.stdout:129: left oid 32 (ObjNum 64 snap 6 seq_num 64) 2024-09-12T23:38:37.556 INFO:tasks.rados.rados.0.smithi136.stdout:123: done (6 left) 2024-09-12T23:38:37.556 INFO:tasks.rados.rados.0.smithi136.stdout:124: done (5 left) 2024-09-12T23:38:37.556 INFO:tasks.rados.rados.0.smithi136.stdout:125: done (4 left) 2024-09-12T23:38:37.556 INFO:tasks.rados.rados.0.smithi136.stdout:128: done (3 left) 2024-09-12T23:38:37.556 INFO:tasks.rados.rados.0.smithi136.stdout:129: done (2 left) 2024-09-12T23:38:37.557 INFO:tasks.rados.rados.0.smithi136.stdout:130: done (1 left) 2024-09-12T23:38:37.557 INFO:tasks.rados.rados.0.smithi136.stdout:131: done (0 left) 2024-09-12T23:38:37.557 INFO:tasks.rados.rados.0.smithi136.stdout:132: delete oid 20 current snap is 6 2024-09-12T23:38:37.558 INFO:tasks.rados.rados.0.smithi136.stdout:132: done (0 left) 2024-09-12T23:38:37.558 INFO:tasks.rados.rados.0.smithi136.stdout:133: delete oid 42 current snap is 6 2024-09-12T23:38:37.560 INFO:tasks.rados.rados.0.smithi136.stdout:133: done (0 left) 2024-09-12T23:38:37.561 INFO:tasks.rados.rados.0.smithi136.stdout:134: copy_from oid 36 from oid 1 current snap is 6 2024-09-12T23:38:37.561 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 31 current snap is 6 2024-09-12T23:38:37.561 INFO:tasks.rados.rados.0.smithi136.stdout:135: seq_num 67 ranges {606208=729088,1335296=401408} 2024-09-12T23:38:37.569 INFO:tasks.rados.rados.0.smithi136.stdout:135: writing smithi13667081-31 from 606208 to 1335296 tid 1 2024-09-12T23:38:37.572 INFO:tasks.rados.rados.0.smithi136.stdout:135: writing smithi13667081-31 from 1335296 to 1736704 tid 2 2024-09-12T23:38:37.572 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 8 current snap is 6 2024-09-12T23:38:37.572 INFO:tasks.rados.rados.0.smithi136.stdout:134: finishing copy_from to smithi13667081-36 2024-09-12T23:38:37.572 INFO:tasks.rados.rados.0.smithi136.stdout:134: got expected ENOENT (src dne) 2024-09-12T23:38:37.572 INFO:tasks.rados.rados.0.smithi136.stdout:136: seq_num 68 ranges {122880=745472,868352=401408} 2024-09-12T23:38:37.578 INFO:tasks.rados.rados.0.smithi136.stdout:136: writing smithi13667081-8 from 122880 to 868352 tid 1 2024-09-12T23:38:37.581 INFO:tasks.rados.rados.0.smithi136.stdout:136: writing smithi13667081-8 from 868352 to 1269760 tid 2 2024-09-12T23:38:37.581 INFO:tasks.rados.rados.0.smithi136.stdout:137: copy_from oid 41 from oid 17 current snap is 6 2024-09-12T23:38:37.581 INFO:tasks.rados.rados.0.smithi136.stdout:134: finishing copy_from racing read to smithi13667081-36 2024-09-12T23:38:37.581 INFO:tasks.rados.rados.0.smithi136.stdout:135: finishing write tid 1 to smithi13667081-31 2024-09-12T23:38:37.581 INFO:tasks.rados.rados.0.smithi136.stdout:135: finishing write tid 2 to smithi13667081-31 2024-09-12T23:38:37.581 INFO:tasks.rados.rados.0.smithi136.stdout:135: finishing write tid 3 to smithi13667081-31 2024-09-12T23:38:37.581 INFO:tasks.rados.rados.0.smithi136.stdout:134: done (3 left) 2024-09-12T23:38:37.581 INFO:tasks.rados.rados.0.smithi136.stdout:138: copy_from oid 47 from oid 20 current snap is 6 2024-09-12T23:38:37.581 INFO:tasks.rados.rados.0.smithi136.stdout:139: snap_create 2024-09-12T23:38:37.583 INFO:tasks.rados.rados.0.smithi136.stdout:135: finishing write tid 4 to smithi13667081-31 2024-09-12T23:38:37.583 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 31 v 56 (ObjNum 67 snap 6 seq_num 67) dirty exists 2024-09-12T23:38:37.584 INFO:tasks.rados.rados.0.smithi136.stdout:135: left oid 31 (ObjNum 67 snap 6 seq_num 67) 2024-09-12T23:38:37.584 INFO:tasks.rados.rados.0.smithi136.stdout:138: finishing copy_from to smithi13667081-47 2024-09-12T23:38:37.584 INFO:tasks.rados.rados.0.smithi136.stdout:138: got expected ENOENT (src dne) 2024-09-12T23:38:37.584 INFO:tasks.rados.rados.0.smithi136.stdout:138: finishing copy_from racing read to smithi13667081-47 2024-09-12T23:38:37.584 INFO:tasks.rados.rados.0.smithi136.stdout:137: finishing copy_from racing read to smithi13667081-41 2024-09-12T23:38:37.585 INFO:tasks.rados.rados.0.smithi136.stdout:137: finishing copy_from to smithi13667081-41 2024-09-12T23:38:37.585 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 41 v 6 (ObjNum 16 snap 0 seq_num 16) dirty exists 2024-09-12T23:38:37.587 INFO:tasks.rados.rados.0.smithi136.stdout:136: finishing write tid 1 to smithi13667081-8 2024-09-12T23:38:37.588 INFO:tasks.rados.rados.0.smithi136.stdout:136: finishing write tid 2 to smithi13667081-8 2024-09-12T23:38:37.588 INFO:tasks.rados.rados.0.smithi136.stdout:136: finishing write tid 3 to smithi13667081-8 2024-09-12T23:38:37.591 INFO:tasks.rados.rados.0.smithi136.stdout:136: finishing write tid 4 to smithi13667081-8 2024-09-12T23:38:37.591 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 8 v 24 (ObjNum 68 snap 6 seq_num 68) dirty exists 2024-09-12T23:38:37.591 INFO:tasks.rados.rados.0.smithi136.stdout:136: left oid 8 (ObjNum 68 snap 6 seq_num 68) 2024-09-12T23:38:37.755 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:37 smithi136 bash[32189]: cluster 2024-09-12T23:38:36.018373+0000 mgr.y (mgr.24446) 193 : cluster [DBG] pgmap v161: 121 pgs: 121 active+clean; 24 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 412 KiB/s wr, 8 op/s 2024-09-12T23:38:37.755 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:37 smithi136 bash[32189]: cluster 2024-09-12T23:38:36.492102+0000 mon.a (mon.0) 1098 : cluster [DBG] osdmap e80: 8 total, 8 up, 8 in 2024-09-12T23:38:37.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:37 smithi136 bash[22601]: cluster 2024-09-12T23:38:36.018373+0000 mgr.y (mgr.24446) 193 : cluster [DBG] pgmap v161: 121 pgs: 121 active+clean; 24 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 412 KiB/s wr, 8 op/s 2024-09-12T23:38:37.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:37 smithi136 bash[22601]: cluster 2024-09-12T23:38:36.492102+0000 mon.a (mon.0) 1098 : cluster [DBG] osdmap e80: 8 total, 8 up, 8 in 2024-09-12T23:38:37.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:37 smithi181 bash[24776]: cluster 2024-09-12T23:38:36.018373+0000 mgr.y (mgr.24446) 193 : cluster [DBG] pgmap v161: 121 pgs: 121 active+clean; 24 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 412 KiB/s wr, 8 op/s 2024-09-12T23:38:37.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:37 smithi181 bash[24776]: cluster 2024-09-12T23:38:36.492102+0000 mon.a (mon.0) 1098 : cluster [DBG] osdmap e80: 8 total, 8 up, 8 in 2024-09-12T23:38:38.512 INFO:tasks.rados.rados.0.smithi136.stdout:135: done (4 left) 2024-09-12T23:38:38.513 INFO:tasks.rados.rados.0.smithi136.stdout:136: done (3 left) 2024-09-12T23:38:38.513 INFO:tasks.rados.rados.0.smithi136.stdout:137: done (2 left) 2024-09-12T23:38:38.513 INFO:tasks.rados.rados.0.smithi136.stdout:138: done (1 left) 2024-09-12T23:38:38.513 INFO:tasks.rados.rados.0.smithi136.stdout:139: done (0 left) 2024-09-12T23:38:38.513 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 9 current snap is 7 2024-09-12T23:38:38.513 INFO:tasks.rados.rados.0.smithi136.stdout:140: seq_num 69 ranges {139264=761856,901120=401408} 2024-09-12T23:38:38.518 INFO:tasks.rados.rados.0.smithi136.stdout:140: writing smithi13667081-9 from 139264 to 901120 tid 1 2024-09-12T23:38:38.522 INFO:tasks.rados.rados.0.smithi136.stdout:140: writing smithi13667081-9 from 901120 to 1302528 tid 2 2024-09-12T23:38:38.522 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 12 current snap is 7 2024-09-12T23:38:38.522 INFO:tasks.rados.rados.0.smithi136.stdout:141: seq_num 70 ranges {0=778240,778240=401408} 2024-09-12T23:38:38.528 INFO:tasks.rados.rados.0.smithi136.stdout:141: writing smithi13667081-12 from 0 to 778240 tid 1 2024-09-12T23:38:38.531 INFO:tasks.rados.rados.0.smithi136.stdout:141: writing smithi13667081-12 from 778240 to 1179648 tid 2 2024-09-12T23:38:38.532 INFO:tasks.rados.rados.0.smithi136.stdout:142: rollback oid 22 current snap is 7 2024-09-12T23:38:38.532 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 22 to 1 2024-09-12T23:38:38.532 INFO:tasks.rados.rados.0.smithi136.stdout:140: finishing write tid 1 to smithi13667081-9 2024-09-12T23:38:38.532 INFO:tasks.rados.rados.0.smithi136.stdout:140: finishing write tid 2 to smithi13667081-9 2024-09-12T23:38:38.532 INFO:tasks.rados.rados.0.smithi136.stdout:140: finishing write tid 3 to smithi13667081-9 2024-09-12T23:38:38.532 INFO:tasks.rados.rados.0.smithi136.stdout:140: finishing write tid 4 to smithi13667081-9 2024-09-12T23:38:38.532 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 9 v 18 (ObjNum 69 snap 7 seq_num 69) dirty exists 2024-09-12T23:38:38.532 INFO:tasks.rados.rados.0.smithi136.stdout:140: left oid 9 (ObjNum 69 snap 7 seq_num 69) 2024-09-12T23:38:38.532 INFO:tasks.rados.rados.0.smithi136.stdout:140: done (2 left) 2024-09-12T23:38:38.532 INFO:tasks.rados.rados.0.smithi136.stdout:143: snap_create 2024-09-12T23:38:38.540 INFO:tasks.rados.rados.0.smithi136.stdout:142: finishing rollback tid 0 to smithi13667081-22 2024-09-12T23:38:38.541 INFO:tasks.rados.rados.0.smithi136.stdout:142: finishing rollback tid 1 to smithi13667081-22 2024-09-12T23:38:38.541 INFO:tasks.rados.rados.0.smithi136.stdout:142: finishing rollback tid 2 to smithi13667081-22 2024-09-12T23:38:38.541 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 22 v 15 (ObjNum 21 snap 0 seq_num 21) dirty exists 2024-09-12T23:38:38.542 INFO:tasks.rados.rados.0.smithi136.stdout:141: finishing write tid 1 to smithi13667081-12 2024-09-12T23:38:38.542 INFO:tasks.rados.rados.0.smithi136.stdout:141: finishing write tid 2 to smithi13667081-12 2024-09-12T23:38:38.542 INFO:tasks.rados.rados.0.smithi136.stdout:141: finishing write tid 3 to smithi13667081-12 2024-09-12T23:38:38.544 INFO:tasks.rados.rados.0.smithi136.stdout:141: finishing write tid 4 to smithi13667081-12 2024-09-12T23:38:38.544 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 12 v 59 (ObjNum 70 snap 7 seq_num 70) dirty exists 2024-09-12T23:38:38.544 INFO:tasks.rados.rados.0.smithi136.stdout:141: left oid 12 (ObjNum 70 snap 7 seq_num 70) 2024-09-12T23:38:38.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:38 smithi136 bash[22601]: cluster 2024-09-12T23:38:37.505096+0000 mon.a (mon.0) 1099 : cluster [DBG] osdmap e81: 8 total, 8 up, 8 in 2024-09-12T23:38:38.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:38 smithi136 bash[22601]: audit 2024-09-12T23:38:38.017614+0000 mon.a (mon.0) 1100 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:38:38.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:38 smithi136 bash[22601]: audit 2024-09-12T23:38:38.018542+0000 mon.a (mon.0) 1101 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:38:38.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:38 smithi136 bash[22601]: audit 2024-09-12T23:38:38.179766+0000 mon.a (mon.0) 1102 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:38:38.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:38 smithi136 bash[32189]: cluster 2024-09-12T23:38:37.505096+0000 mon.a (mon.0) 1099 : cluster [DBG] osdmap e81: 8 total, 8 up, 8 in 2024-09-12T23:38:38.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:38 smithi136 bash[32189]: audit 2024-09-12T23:38:38.017614+0000 mon.a (mon.0) 1100 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:38:38.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:38 smithi136 bash[32189]: audit 2024-09-12T23:38:38.018542+0000 mon.a (mon.0) 1101 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:38:38.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:38 smithi136 bash[32189]: audit 2024-09-12T23:38:38.179766+0000 mon.a (mon.0) 1102 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:38:38.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:38 smithi181 bash[24776]: cluster 2024-09-12T23:38:37.505096+0000 mon.a (mon.0) 1099 : cluster [DBG] osdmap e81: 8 total, 8 up, 8 in 2024-09-12T23:38:38.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:38 smithi181 bash[24776]: audit 2024-09-12T23:38:38.017614+0000 mon.a (mon.0) 1100 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:38:38.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:38 smithi181 bash[24776]: audit 2024-09-12T23:38:38.018542+0000 mon.a (mon.0) 1101 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:38:38.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:38 smithi181 bash[24776]: audit 2024-09-12T23:38:38.179766+0000 mon.a (mon.0) 1102 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:38:39.511 INFO:tasks.rados.rados.0.smithi136.stdout:141: done (2 left) 2024-09-12T23:38:39.511 INFO:tasks.rados.rados.0.smithi136.stdout:142: done (1 left) 2024-09-12T23:38:39.511 INFO:tasks.rados.rados.0.smithi136.stdout:143: done (0 left) 2024-09-12T23:38:39.512 INFO:tasks.rados.rados.0.smithi136.stdout:144: read oid 33 snap -1 2024-09-12T23:38:39.512 INFO:tasks.rados.rados.0.smithi136.stdout:144: expect (ObjNum 32 snap 0 seq_num 32) 2024-09-12T23:38:39.512 INFO:tasks.rados.rados.0.smithi136.stdout:145: read oid 45 snap -1 2024-09-12T23:38:39.512 INFO:tasks.rados.rados.0.smithi136.stdout:145: expect (ObjNum 33 snap 0 seq_num 33) 2024-09-12T23:38:39.512 INFO:tasks.rados.rados.0.smithi136.stdout:146: snap_create 2024-09-12T23:38:39.518 INFO:tasks.rados.rados.0.smithi136.stdout:144: expect (ObjNum 32 snap 0 seq_num 32) 2024-09-12T23:38:39.535 INFO:tasks.rados.rados.0.smithi136.stdout:145: expect (ObjNum 33 snap 0 seq_num 33) 2024-09-12T23:38:39.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:39 smithi181 bash[24776]: cluster 2024-09-12T23:38:38.018778+0000 mgr.y (mgr.24446) 194 : cluster [DBG] pgmap v164: 121 pgs: 121 active+clean; 24 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:38:39.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:39 smithi181 bash[24776]: cluster 2024-09-12T23:38:38.517065+0000 mon.a (mon.0) 1103 : cluster [DBG] osdmap e82: 8 total, 8 up, 8 in 2024-09-12T23:38:40.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:39 smithi136 bash[32189]: cluster 2024-09-12T23:38:38.018778+0000 mgr.y (mgr.24446) 194 : cluster [DBG] pgmap v164: 121 pgs: 121 active+clean; 24 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:38:40.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:39 smithi136 bash[32189]: cluster 2024-09-12T23:38:38.517065+0000 mon.a (mon.0) 1103 : cluster [DBG] osdmap e82: 8 total, 8 up, 8 in 2024-09-12T23:38:40.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:39 smithi136 bash[22601]: cluster 2024-09-12T23:38:38.018778+0000 mgr.y (mgr.24446) 194 : cluster [DBG] pgmap v164: 121 pgs: 121 active+clean; 24 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:38:40.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:39 smithi136 bash[22601]: cluster 2024-09-12T23:38:38.517065+0000 mon.a (mon.0) 1103 : cluster [DBG] osdmap e82: 8 total, 8 up, 8 in 2024-09-12T23:38:40.518 INFO:tasks.rados.rados.0.smithi136.stdout:144: done (2 left) 2024-09-12T23:38:40.518 INFO:tasks.rados.rados.0.smithi136.stdout:145: done (1 left) 2024-09-12T23:38:40.519 INFO:tasks.rados.rados.0.smithi136.stdout:146: done (0 left) 2024-09-12T23:38:40.519 INFO:tasks.rados.rados.0.smithi136.stdout:147: copy_from oid 48 from oid 42 current snap is 9 2024-09-12T23:38:40.519 INFO:tasks.rados.rados.0.smithi136.stdout:148: delete oid 29 current snap is 9 2024-09-12T23:38:40.526 INFO:tasks.rados.rados.0.smithi136.stdout:148: done (1 left) 2024-09-12T23:38:40.526 INFO:tasks.rados.rados.0.smithi136.stdout:149: snap_create 2024-09-12T23:38:40.527 INFO:tasks.rados.rados.0.smithi136.stdout:147: finishing copy_from to smithi13667081-48 2024-09-12T23:38:40.527 INFO:tasks.rados.rados.0.smithi136.stdout:147: got expected ENOENT (src dne) 2024-09-12T23:38:40.527 INFO:tasks.rados.rados.0.smithi136.stdout:147: finishing copy_from racing read to smithi13667081-48 2024-09-12T23:38:40.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:40 smithi181 bash[24776]: cluster 2024-09-12T23:38:39.515626+0000 mon.a (mon.0) 1104 : cluster [DBG] osdmap e83: 8 total, 8 up, 8 in 2024-09-12T23:38:41.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:40 smithi136 bash[32189]: cluster 2024-09-12T23:38:39.515626+0000 mon.a (mon.0) 1104 : cluster [DBG] osdmap e83: 8 total, 8 up, 8 in 2024-09-12T23:38:41.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:40 smithi136 bash[22601]: cluster 2024-09-12T23:38:39.515626+0000 mon.a (mon.0) 1104 : cluster [DBG] osdmap e83: 8 total, 8 up, 8 in 2024-09-12T23:38:41.529 INFO:tasks.rados.rados.0.smithi136.stdout:147: done (1 left) 2024-09-12T23:38:41.529 INFO:tasks.rados.rados.0.smithi136.stdout:149: done (0 left) 2024-09-12T23:38:41.529 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 43 current snap is 10 2024-09-12T23:38:41.530 INFO:tasks.rados.rados.0.smithi136.stdout:150: seq_num 72 ranges {0=409600,409600=524288,933888=278528} 2024-09-12T23:38:41.534 INFO:tasks.rados.rados.0.smithi136.stdout:150: writing smithi13667081-43 from 0 to 409600 tid 1 2024-09-12T23:38:41.541 INFO:tasks.rados.rados.0.smithi136.stdout:150: writing smithi13667081-43 from 409600 to 933888 tid 2 2024-09-12T23:38:41.545 INFO:tasks.rados.rados.0.smithi136.stdout:150: writing smithi13667081-43 from 933888 to 1212416 tid 3 2024-09-12T23:38:41.545 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 10 current snap is 10 2024-09-12T23:38:41.545 INFO:tasks.rados.rados.0.smithi136.stdout:151: seq_num 73 ranges {155648=425984,581632=409600,991232=393216} 2024-09-12T23:38:41.553 INFO:tasks.rados.rados.0.smithi136.stdout:151: writing smithi13667081-10 from 155648 to 581632 tid 1 2024-09-12T23:38:41.558 INFO:tasks.rados.rados.0.smithi136.stdout:151: writing smithi13667081-10 from 581632 to 991232 tid 2 2024-09-12T23:38:41.564 INFO:tasks.rados.rados.0.smithi136.stdout:151: writing smithi13667081-10 from 991232 to 1384448 tid 3 2024-09-12T23:38:41.564 INFO:tasks.rados.rados.0.smithi136.stdout:152: snap_create 2024-09-12T23:38:41.564 INFO:tasks.rados.rados.0.smithi136.stdout:150: finishing write tid 1 to smithi13667081-43 2024-09-12T23:38:41.564 INFO:tasks.rados.rados.0.smithi136.stdout:150: finishing write tid 2 to smithi13667081-43 2024-09-12T23:38:41.564 INFO:tasks.rados.rados.0.smithi136.stdout:150: finishing write tid 3 to smithi13667081-43 2024-09-12T23:38:41.564 INFO:tasks.rados.rados.0.smithi136.stdout:150: finishing write tid 4 to smithi13667081-43 2024-09-12T23:38:41.564 INFO:tasks.rados.rados.0.smithi136.stdout:150: finishing write tid 5 to smithi13667081-43 2024-09-12T23:38:41.564 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 43 v 34 (ObjNum 72 snap 10 seq_num 72) dirty exists 2024-09-12T23:38:41.565 INFO:tasks.rados.rados.0.smithi136.stdout:150: left oid 43 (ObjNum 72 snap 10 seq_num 72) 2024-09-12T23:38:41.565 INFO:tasks.rados.rados.0.smithi136.stdout:151: finishing write tid 1 to smithi13667081-10 2024-09-12T23:38:41.565 INFO:tasks.rados.rados.0.smithi136.stdout:151: finishing write tid 2 to smithi13667081-10 2024-09-12T23:38:41.568 INFO:tasks.rados.rados.0.smithi136.stdout:151: finishing write tid 3 to smithi13667081-10 2024-09-12T23:38:41.569 INFO:tasks.rados.rados.0.smithi136.stdout:151: finishing write tid 4 to smithi13667081-10 2024-09-12T23:38:41.571 INFO:tasks.rados.rados.0.smithi136.stdout:151: finishing write tid 5 to smithi13667081-10 2024-09-12T23:38:41.571 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 10 v 15 (ObjNum 73 snap 10 seq_num 73) dirty exists 2024-09-12T23:38:41.571 INFO:tasks.rados.rados.0.smithi136.stdout:151: left oid 10 (ObjNum 73 snap 10 seq_num 73) 2024-09-12T23:38:41.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:41 smithi181 bash[24776]: cluster 2024-09-12T23:38:40.019258+0000 mgr.y (mgr.24446) 195 : cluster [DBG] pgmap v167: 121 pgs: 121 active+clean; 32 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 2.7 MiB/s wr, 43 op/s 2024-09-12T23:38:41.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:41 smithi181 bash[24776]: cluster 2024-09-12T23:38:40.522020+0000 mon.a (mon.0) 1105 : cluster [DBG] osdmap e84: 8 total, 8 up, 8 in 2024-09-12T23:38:41.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:41 smithi181 bash[24776]: cluster 2024-09-12T23:38:41.533434+0000 mon.a (mon.0) 1106 : cluster [DBG] osdmap e85: 8 total, 8 up, 8 in 2024-09-12T23:38:42.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:41 smithi136 bash[22601]: cluster 2024-09-12T23:38:40.019258+0000 mgr.y (mgr.24446) 195 : cluster [DBG] pgmap v167: 121 pgs: 121 active+clean; 32 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 2.7 MiB/s wr, 43 op/s 2024-09-12T23:38:42.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:41 smithi136 bash[22601]: cluster 2024-09-12T23:38:40.522020+0000 mon.a (mon.0) 1105 : cluster [DBG] osdmap e84: 8 total, 8 up, 8 in 2024-09-12T23:38:42.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:41 smithi136 bash[22601]: cluster 2024-09-12T23:38:41.533434+0000 mon.a (mon.0) 1106 : cluster [DBG] osdmap e85: 8 total, 8 up, 8 in 2024-09-12T23:38:42.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:41 smithi136 bash[32189]: cluster 2024-09-12T23:38:40.019258+0000 mgr.y (mgr.24446) 195 : cluster [DBG] pgmap v167: 121 pgs: 121 active+clean; 32 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 2.7 MiB/s wr, 43 op/s 2024-09-12T23:38:42.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:41 smithi136 bash[32189]: cluster 2024-09-12T23:38:40.522020+0000 mon.a (mon.0) 1105 : cluster [DBG] osdmap e84: 8 total, 8 up, 8 in 2024-09-12T23:38:42.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:41 smithi136 bash[32189]: cluster 2024-09-12T23:38:41.533434+0000 mon.a (mon.0) 1106 : cluster [DBG] osdmap e85: 8 total, 8 up, 8 in 2024-09-12T23:38:42.548 INFO:tasks.rados.rados.0.smithi136.stdout:150: done (2 left) 2024-09-12T23:38:42.548 INFO:tasks.rados.rados.0.smithi136.stdout:151: done (1 left) 2024-09-12T23:38:42.548 INFO:tasks.rados.rados.0.smithi136.stdout:152: done (0 left) 2024-09-12T23:38:42.548 INFO:tasks.rados.rados.0.smithi136.stdout:153: snap_create 2024-09-12T23:38:43.553 INFO:tasks.rados.rados.0.smithi136.stdout:153: done (0 left) 2024-09-12T23:38:43.554 INFO:tasks.rados.rados.0.smithi136.stdout:154: read oid 42 snap -1 2024-09-12T23:38:43.554 INFO:tasks.rados.rados.0.smithi136.stdout:154: expect deleted 2024-09-12T23:38:43.554 INFO:tasks.rados.rados.0.smithi136.stdout:155: rmattr oid 17 current snap is 12 2024-09-12T23:38:43.554 INFO:tasks.rados.rados.0.smithi136.stdout:155: done (1 left) 2024-09-12T23:38:43.554 INFO:tasks.rados.rados.0.smithi136.stdout:156: setattr oid 28 current snap is 12 2024-09-12T23:38:43.558 INFO:tasks.rados.rados.0.smithi136.stdout:157: copy_from oid 33 from oid 16 current snap is 12 2024-09-12T23:38:43.558 INFO:tasks.rados.rados.0.smithi136.stdout:158: setattr oid 32 current snap is 12 2024-09-12T23:38:43.563 INFO:tasks.rados.rados.0.smithi136.stdout:154: done (3 left) 2024-09-12T23:38:43.564 INFO:tasks.rados.rados.0.smithi136.stdout:159: snap_remove snap 1 2024-09-12T23:38:43.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:43 smithi181 bash[24776]: cluster 2024-09-12T23:38:42.019664+0000 mgr.y (mgr.24446) 196 : cluster [DBG] pgmap v170: 121 pgs: 121 active+clean; 32 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 2.7 MiB/s wr, 43 op/s 2024-09-12T23:38:43.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:43 smithi181 bash[24776]: cluster 2024-09-12T23:38:42.551914+0000 mon.a (mon.0) 1107 : cluster [DBG] osdmap e86: 8 total, 8 up, 8 in 2024-09-12T23:38:44.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:43 smithi136 bash[22601]: cluster 2024-09-12T23:38:42.019664+0000 mgr.y (mgr.24446) 196 : cluster [DBG] pgmap v170: 121 pgs: 121 active+clean; 32 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 2.7 MiB/s wr, 43 op/s 2024-09-12T23:38:44.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:43 smithi136 bash[22601]: cluster 2024-09-12T23:38:42.551914+0000 mon.a (mon.0) 1107 : cluster [DBG] osdmap e86: 8 total, 8 up, 8 in 2024-09-12T23:38:44.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:43 smithi136 bash[32189]: cluster 2024-09-12T23:38:42.019664+0000 mgr.y (mgr.24446) 196 : cluster [DBG] pgmap v170: 121 pgs: 121 active+clean; 32 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 2.7 MiB/s wr, 43 op/s 2024-09-12T23:38:44.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:43 smithi136 bash[32189]: cluster 2024-09-12T23:38:42.551914+0000 mon.a (mon.0) 1107 : cluster [DBG] osdmap e86: 8 total, 8 up, 8 in 2024-09-12T23:38:44.582 INFO:tasks.rados.rados.0.smithi136.stdout:159: done (3 left) 2024-09-12T23:38:44.582 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 47 current snap is 12 2024-09-12T23:38:44.582 INFO:tasks.rados.rados.0.smithi136.stdout:160: seq_num 74 ranges {778240=442368,1220608=696320,1916928=106496} 2024-09-12T23:38:44.599 INFO:tasks.rados.rados.0.smithi136.stdout:160: writing smithi13667081-47 from 778240 to 1220608 tid 1 2024-09-12T23:38:44.606 INFO:tasks.rados.rados.0.smithi136.stdout:160: writing smithi13667081-47 from 1220608 to 1916928 tid 2 2024-09-12T23:38:44.608 INFO:tasks.rados.rados.0.smithi136.stdout:160: writing smithi13667081-47 from 1916928 to 2023424 tid 3 2024-09-12T23:38:44.608 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 26 current snap is 12 2024-09-12T23:38:44.608 INFO:tasks.rados.rados.0.smithi136.stdout:161: seq_num 75 ranges {425984=458752,884736=581632,1466368=221184} 2024-09-12T23:38:44.616 INFO:tasks.rados.rados.0.smithi136.stdout:161: writing smithi13667081-26 from 425984 to 884736 tid 1 2024-09-12T23:38:44.621 INFO:tasks.rados.rados.0.smithi136.stdout:161: writing smithi13667081-26 from 884736 to 1466368 tid 2 2024-09-12T23:38:44.623 INFO:tasks.rados.rados.0.smithi136.stdout:161: writing smithi13667081-26 from 1466368 to 1687552 tid 3 2024-09-12T23:38:44.623 INFO:tasks.rados.rados.0.smithi136.stdout:162: snap_remove snap 11 2024-09-12T23:38:44.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:44 smithi181 bash[24776]: cluster 2024-09-12T23:38:43.556224+0000 mon.a (mon.0) 1108 : cluster [DBG] osdmap e87: 8 total, 8 up, 8 in 2024-09-12T23:38:45.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:44 smithi136 bash[22601]: cluster 2024-09-12T23:38:43.556224+0000 mon.a (mon.0) 1108 : cluster [DBG] osdmap e87: 8 total, 8 up, 8 in 2024-09-12T23:38:45.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:44 smithi136 bash[32189]: cluster 2024-09-12T23:38:43.556224+0000 mon.a (mon.0) 1108 : cluster [DBG] osdmap e87: 8 total, 8 up, 8 in 2024-09-12T23:38:45.442 INFO:tasks.rados.rados.0.smithi136.stdout:162: done (5 left) 2024-09-12T23:38:45.443 INFO:tasks.rados.rados.0.smithi136.stdout:163: delete oid 8 current snap is 12 2024-09-12T23:38:45.443 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 32 v 10 (ObjNum 64 snap 6 seq_num 64) dirty exists 2024-09-12T23:38:45.443 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 28 v 35 (ObjNum 65 snap 0 seq_num 1751995491) dirty exists 2024-09-12T23:38:45.443 INFO:tasks.rados.rados.0.smithi136.stdout:157: finishing copy_from racing read to smithi13667081-33 2024-09-12T23:38:45.443 INFO:tasks.rados.rados.0.smithi136.stdout:157: finishing copy_from to smithi13667081-33 2024-09-12T23:38:45.443 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 33 v 4 (ObjNum 15 snap 0 seq_num 15) dirty exists 2024-09-12T23:38:45.443 INFO:tasks.rados.rados.0.smithi136.stdout:160: finishing write tid 1 to smithi13667081-47 2024-09-12T23:38:45.443 INFO:tasks.rados.rados.0.smithi136.stdout:160: finishing write tid 2 to smithi13667081-47 2024-09-12T23:38:45.443 INFO:tasks.rados.rados.0.smithi136.stdout:160: finishing write tid 3 to smithi13667081-47 2024-09-12T23:38:45.443 INFO:tasks.rados.rados.0.smithi136.stdout:160: finishing write tid 4 to smithi13667081-47 2024-09-12T23:38:45.443 INFO:tasks.rados.rados.0.smithi136.stdout:160: finishing write tid 5 to smithi13667081-47 2024-09-12T23:38:45.443 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 47 v 25 (ObjNum 74 snap 12 seq_num 74) dirty exists 2024-09-12T23:38:45.443 INFO:tasks.rados.rados.0.smithi136.stdout:160: left oid 47 (ObjNum 74 snap 12 seq_num 74) 2024-09-12T23:38:45.443 INFO:tasks.rados.rados.0.smithi136.stdout:161: finishing write tid 1 to smithi13667081-26 2024-09-12T23:38:45.444 INFO:tasks.rados.rados.0.smithi136.stdout:161: finishing write tid 2 to smithi13667081-26 2024-09-12T23:38:45.444 INFO:tasks.rados.rados.0.smithi136.stdout:161: finishing write tid 3 to smithi13667081-26 2024-09-12T23:38:45.444 INFO:tasks.rados.rados.0.smithi136.stdout:161: finishing write tid 4 to smithi13667081-26 2024-09-12T23:38:45.444 INFO:tasks.rados.rados.0.smithi136.stdout:161: finishing write tid 5 to smithi13667081-26 2024-09-12T23:38:45.444 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 26 v 19 (ObjNum 75 snap 12 seq_num 75) dirty exists 2024-09-12T23:38:45.444 INFO:tasks.rados.rados.0.smithi136.stdout:161: left oid 26 (ObjNum 75 snap 12 seq_num 75) 2024-09-12T23:38:45.453 INFO:tasks.rados.rados.0.smithi136.stdout:156: done (5 left) 2024-09-12T23:38:45.454 INFO:tasks.rados.rados.0.smithi136.stdout:157: done (4 left) 2024-09-12T23:38:45.454 INFO:tasks.rados.rados.0.smithi136.stdout:158: done (3 left) 2024-09-12T23:38:45.454 INFO:tasks.rados.rados.0.smithi136.stdout:160: done (2 left) 2024-09-12T23:38:45.454 INFO:tasks.rados.rados.0.smithi136.stdout:161: done (1 left) 2024-09-12T23:38:45.454 INFO:tasks.rados.rados.0.smithi136.stdout:163: done (0 left) 2024-09-12T23:38:45.454 INFO:tasks.rados.rados.0.smithi136.stdout:164: snap_create 2024-09-12T23:38:45.572 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:38:45 smithi181 bash[25717]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:38:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-12T23:38:45.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:45 smithi181 bash[24776]: cluster 2024-09-12T23:38:44.021189+0000 mgr.y (mgr.24446) 197 : cluster [DBG] pgmap v173: 121 pgs: 121 active+clean; 35 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 805 KiB/s rd, 658 KiB/s wr, 15 op/s 2024-09-12T23:38:45.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:45 smithi181 bash[24776]: cluster 2024-09-12T23:38:44.585520+0000 mon.a (mon.0) 1109 : cluster [DBG] osdmap e88: 8 total, 8 up, 8 in 2024-09-12T23:38:45.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:45 smithi181 bash[24776]: cluster 2024-09-12T23:38:45.445311+0000 mon.a (mon.0) 1110 : cluster [DBG] osdmap e89: 8 total, 8 up, 8 in 2024-09-12T23:38:46.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:45 smithi136 bash[32189]: cluster 2024-09-12T23:38:44.021189+0000 mgr.y (mgr.24446) 197 : cluster [DBG] pgmap v173: 121 pgs: 121 active+clean; 35 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 805 KiB/s rd, 658 KiB/s wr, 15 op/s 2024-09-12T23:38:46.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:45 smithi136 bash[32189]: cluster 2024-09-12T23:38:44.585520+0000 mon.a (mon.0) 1109 : cluster [DBG] osdmap e88: 8 total, 8 up, 8 in 2024-09-12T23:38:46.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:45 smithi136 bash[32189]: cluster 2024-09-12T23:38:45.445311+0000 mon.a (mon.0) 1110 : cluster [DBG] osdmap e89: 8 total, 8 up, 8 in 2024-09-12T23:38:46.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:45 smithi136 bash[22601]: cluster 2024-09-12T23:38:44.021189+0000 mgr.y (mgr.24446) 197 : cluster [DBG] pgmap v173: 121 pgs: 121 active+clean; 35 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 805 KiB/s rd, 658 KiB/s wr, 15 op/s 2024-09-12T23:38:46.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:45 smithi136 bash[22601]: cluster 2024-09-12T23:38:44.585520+0000 mon.a (mon.0) 1109 : cluster [DBG] osdmap e88: 8 total, 8 up, 8 in 2024-09-12T23:38:46.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:45 smithi136 bash[22601]: cluster 2024-09-12T23:38:45.445311+0000 mon.a (mon.0) 1110 : cluster [DBG] osdmap e89: 8 total, 8 up, 8 in 2024-09-12T23:38:46.444 INFO:tasks.rados.rados.0.smithi136.stdout:164: done (0 left) 2024-09-12T23:38:46.444 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 31 current snap is 13 2024-09-12T23:38:46.444 INFO:tasks.rados.rados.0.smithi136.stdout:165: seq_num 77 ranges {1736704=491520,2228224=794624,3022848=8192} 2024-09-12T23:38:46.475 INFO:tasks.rados.rados.0.smithi136.stdout:165: writing smithi13667081-31 from 1736704 to 2228224 tid 1 2024-09-12T23:38:46.488 INFO:tasks.rados.rados.0.smithi136.stdout:165: writing smithi13667081-31 from 2228224 to 3022848 tid 2 2024-09-12T23:38:46.488 INFO:tasks.rados.rados.0.smithi136.stdout:165: writing smithi13667081-31 from 3022848 to 3031040 tid 3 2024-09-12T23:38:46.488 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 16 current snap is 13 2024-09-12T23:38:46.488 INFO:tasks.rados.rados.0.smithi136.stdout:165: finishing write tid 1 to smithi13667081-31 2024-09-12T23:38:46.488 INFO:tasks.rados.rados.0.smithi136.stdout:166: seq_num 78 ranges {253952=516096,770048=679936,1449984=122880} 2024-09-12T23:38:46.499 INFO:tasks.rados.rados.0.smithi136.stdout:166: writing smithi13667081-16 from 253952 to 770048 tid 1 2024-09-12T23:38:46.505 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:38:46 smithi136 bash[22852]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:38:46] "GET /metrics HTTP/1.1" 200 239768 "" "Prometheus/2.43.0" 2024-09-12T23:38:46.508 INFO:tasks.rados.rados.0.smithi136.stdout:166: writing smithi13667081-16 from 770048 to 1449984 tid 2 2024-09-12T23:38:46.511 INFO:tasks.rados.rados.0.smithi136.stdout:166: writing smithi13667081-16 from 1449984 to 1572864 tid 3 2024-09-12T23:38:46.511 INFO:tasks.rados.rados.0.smithi136.stdout:167: copy_from oid 10 from oid 39 current snap is 13 2024-09-12T23:38:46.511 INFO:tasks.rados.rados.0.smithi136.stdout:165: finishing write tid 2 to smithi13667081-31 2024-09-12T23:38:46.511 INFO:tasks.rados.rados.0.smithi136.stdout:165: finishing write tid 3 to smithi13667081-31 2024-09-12T23:38:46.511 INFO:tasks.rados.rados.0.smithi136.stdout:165: finishing write tid 4 to smithi13667081-31 2024-09-12T23:38:46.511 INFO:tasks.rados.rados.0.smithi136.stdout:165: finishing write tid 5 to smithi13667081-31 2024-09-12T23:38:46.511 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 31 v 72 (ObjNum 77 snap 13 seq_num 77) dirty exists 2024-09-12T23:38:46.511 INFO:tasks.rados.rados.0.smithi136.stdout:165: left oid 31 (ObjNum 77 snap 13 seq_num 77) 2024-09-12T23:38:46.511 INFO:tasks.rados.rados.0.smithi136.stdout:166: finishing write tid 1 to smithi13667081-16 2024-09-12T23:38:46.511 INFO:tasks.rados.rados.0.smithi136.stdout:165: done (2 left) 2024-09-12T23:38:46.512 INFO:tasks.rados.rados.0.smithi136.stdout:168: read oid 25 snap 8 2024-09-12T23:38:46.512 INFO:tasks.rados.rados.0.smithi136.stdout:168: expect (ObjNum 36 snap 0 seq_num 36) 2024-09-12T23:38:46.512 INFO:tasks.rados.rados.0.smithi136.stdout:169: copy_from oid 21 from oid 35 current snap is 13 2024-09-12T23:38:46.512 INFO:tasks.rados.rados.0.smithi136.stdout:170: snap_remove snap 7 2024-09-12T23:38:47.447 INFO:tasks.rados.rados.0.smithi136.stdout:166: finishing write tid 2 to smithi13667081-16 2024-09-12T23:38:47.448 INFO:tasks.rados.rados.0.smithi136.stdout:166: finishing write tid 3 to smithi13667081-16 2024-09-12T23:38:47.448 INFO:tasks.rados.rados.0.smithi136.stdout:166: finishing write tid 4 to smithi13667081-16 2024-09-12T23:38:47.448 INFO:tasks.rados.rados.0.smithi136.stdout:166: finishing write tid 5 to smithi13667081-16 2024-09-12T23:38:47.448 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 16 v 48 (ObjNum 78 snap 13 seq_num 78) dirty exists 2024-09-12T23:38:47.448 INFO:tasks.rados.rados.0.smithi136.stdout:166: left oid 16 (ObjNum 78 snap 13 seq_num 78) 2024-09-12T23:38:47.448 INFO:tasks.rados.rados.0.smithi136.stdout:168: expect (ObjNum 36 snap 0 seq_num 36) 2024-09-12T23:38:47.486 INFO:tasks.rados.rados.0.smithi136.stdout:169: finishing copy_from racing read to smithi13667081-21 2024-09-12T23:38:47.487 INFO:tasks.rados.rados.0.smithi136.stdout:169: finishing copy_from to smithi13667081-21 2024-09-12T23:38:47.487 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 21 v 29 (ObjNum 34 snap 0 seq_num 34) dirty exists 2024-09-12T23:38:47.487 INFO:tasks.rados.rados.0.smithi136.stdout:167: finishing copy_from racing read to smithi13667081-10 2024-09-12T23:38:47.487 INFO:tasks.rados.rados.0.smithi136.stdout:167: finishing copy_from to smithi13667081-10 2024-09-12T23:38:47.487 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 10 v 4 (ObjNum 38 snap 0 seq_num 38) dirty exists 2024-09-12T23:38:47.487 INFO:tasks.rados.rados.0.smithi136.stdout:166: done (4 left) 2024-09-12T23:38:47.487 INFO:tasks.rados.rados.0.smithi136.stdout:167: done (3 left) 2024-09-12T23:38:47.487 INFO:tasks.rados.rados.0.smithi136.stdout:168: done (2 left) 2024-09-12T23:38:47.487 INFO:tasks.rados.rados.0.smithi136.stdout:169: done (1 left) 2024-09-12T23:38:47.487 INFO:tasks.rados.rados.0.smithi136.stdout:170: done (0 left) 2024-09-12T23:38:47.488 INFO:tasks.rados.rados.0.smithi136.stdout:171: delete oid 48 current snap is 13 2024-09-12T23:38:47.494 INFO:tasks.rados.rados.0.smithi136.stdout:171: done (0 left) 2024-09-12T23:38:47.494 INFO:tasks.rados.rados.0.smithi136.stdout:172: snap_remove snap 8 2024-09-12T23:38:47.755 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:47 smithi136 bash[32189]: cluster 2024-09-12T23:38:46.021789+0000 mgr.y (mgr.24446) 198 : cluster [DBG] pgmap v176: 121 pgs: 121 active+clean; 35 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 805 KiB/s rd, 658 KiB/s wr, 15 op/s 2024-09-12T23:38:47.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:47 smithi136 bash[32189]: cluster 2024-09-12T23:38:46.447991+0000 mon.a (mon.0) 1111 : cluster [DBG] osdmap e90: 8 total, 8 up, 8 in 2024-09-12T23:38:47.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:47 smithi136 bash[22601]: cluster 2024-09-12T23:38:46.021789+0000 mgr.y (mgr.24446) 198 : cluster [DBG] pgmap v176: 121 pgs: 121 active+clean; 35 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 805 KiB/s rd, 658 KiB/s wr, 15 op/s 2024-09-12T23:38:47.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:47 smithi136 bash[22601]: cluster 2024-09-12T23:38:46.447991+0000 mon.a (mon.0) 1111 : cluster [DBG] osdmap e90: 8 total, 8 up, 8 in 2024-09-12T23:38:47.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:47 smithi181 bash[24776]: cluster 2024-09-12T23:38:46.021789+0000 mgr.y (mgr.24446) 198 : cluster [DBG] pgmap v176: 121 pgs: 121 active+clean; 35 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 805 KiB/s rd, 658 KiB/s wr, 15 op/s 2024-09-12T23:38:47.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:47 smithi181 bash[24776]: cluster 2024-09-12T23:38:46.447991+0000 mon.a (mon.0) 1111 : cluster [DBG] osdmap e90: 8 total, 8 up, 8 in 2024-09-12T23:38:48.470 INFO:tasks.rados.rados.0.smithi136.stdout:172: done (0 left) 2024-09-12T23:38:48.470 INFO:tasks.rados.rados.0.smithi136.stdout:173: snap_create 2024-09-12T23:38:48.755 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:48 smithi136 bash[32189]: cluster 2024-09-12T23:38:47.450826+0000 mon.a (mon.0) 1112 : cluster [DBG] osdmap e91: 8 total, 8 up, 8 in 2024-09-12T23:38:48.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:48 smithi136 bash[22601]: cluster 2024-09-12T23:38:47.450826+0000 mon.a (mon.0) 1112 : cluster [DBG] osdmap e91: 8 total, 8 up, 8 in 2024-09-12T23:38:48.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:48 smithi181 bash[24776]: cluster 2024-09-12T23:38:47.450826+0000 mon.a (mon.0) 1112 : cluster [DBG] osdmap e91: 8 total, 8 up, 8 in 2024-09-12T23:38:49.472 INFO:tasks.rados.rados.0.smithi136.stdout:173: done (0 left) 2024-09-12T23:38:49.472 INFO:tasks.rados.rados.0.smithi136.stdout:174: read oid 46 snap -1 2024-09-12T23:38:49.472 INFO:tasks.rados.rados.0.smithi136.stdout:174: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-12T23:38:49.472 INFO:tasks.rados.rados.0.smithi136.stdout:175: read oid 7 snap 4 2024-09-12T23:38:49.472 INFO:tasks.rados.rados.0.smithi136.stdout:175: expect (ObjNum 6 snap 0 seq_num 6) 2024-09-12T23:38:49.472 INFO:tasks.rados.rados.0.smithi136.stdout:176: snap_create 2024-09-12T23:38:49.483 INFO:tasks.rados.rados.0.smithi136.stdout:175: expect (ObjNum 6 snap 0 seq_num 6) 2024-09-12T23:38:49.491 INFO:tasks.rados.rados.0.smithi136.stdout:174: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-12T23:38:49.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:49 smithi136 bash[22601]: cluster 2024-09-12T23:38:48.022422+0000 mgr.y (mgr.24446) 199 : cluster [DBG] pgmap v179: 121 pgs: 121 active+clean; 35 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:38:49.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:49 smithi136 bash[22601]: cluster 2024-09-12T23:38:48.473459+0000 mon.a (mon.0) 1113 : cluster [DBG] osdmap e92: 8 total, 8 up, 8 in 2024-09-12T23:38:49.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:49 smithi136 bash[32189]: cluster 2024-09-12T23:38:48.022422+0000 mgr.y (mgr.24446) 199 : cluster [DBG] pgmap v179: 121 pgs: 121 active+clean; 35 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:38:49.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:49 smithi136 bash[32189]: cluster 2024-09-12T23:38:48.473459+0000 mon.a (mon.0) 1113 : cluster [DBG] osdmap e92: 8 total, 8 up, 8 in 2024-09-12T23:38:49.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:49 smithi181 bash[24776]: cluster 2024-09-12T23:38:48.022422+0000 mgr.y (mgr.24446) 199 : cluster [DBG] pgmap v179: 121 pgs: 121 active+clean; 35 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:38:49.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:49 smithi181 bash[24776]: cluster 2024-09-12T23:38:48.473459+0000 mon.a (mon.0) 1113 : cluster [DBG] osdmap e92: 8 total, 8 up, 8 in 2024-09-12T23:38:50.477 INFO:tasks.rados.rados.0.smithi136.stdout:174: done (2 left) 2024-09-12T23:38:50.477 INFO:tasks.rados.rados.0.smithi136.stdout:175: done (1 left) 2024-09-12T23:38:50.477 INFO:tasks.rados.rados.0.smithi136.stdout:176: done (0 left) 2024-09-12T23:38:50.477 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 13 current snap is 15 2024-09-12T23:38:50.477 INFO:tasks.rados.rados.0.smithi136.stdout:177: seq_num 80 ranges {0=548864,548864=450560,999424=352256} 2024-09-12T23:38:50.486 INFO:tasks.rados.rados.0.smithi136.stdout:177: writing smithi13667081-13 from 0 to 548864 tid 1 2024-09-12T23:38:50.494 INFO:tasks.rados.rados.0.smithi136.stdout:177: writing smithi13667081-13 from 548864 to 999424 tid 2 2024-09-12T23:38:50.501 INFO:tasks.rados.rados.0.smithi136.stdout:177: writing smithi13667081-13 from 999424 to 1351680 tid 3 2024-09-12T23:38:50.501 INFO:tasks.rados.rados.0.smithi136.stdout:178: copy_from oid 15 from oid 5 current snap is 15 2024-09-12T23:38:50.501 INFO:tasks.rados.rados.0.smithi136.stdout:177: finishing write tid 1 to smithi13667081-13 2024-09-12T23:38:50.501 INFO:tasks.rados.rados.0.smithi136.stdout:177: finishing write tid 2 to smithi13667081-13 2024-09-12T23:38:50.501 INFO:tasks.rados.rados.0.smithi136.stdout:179: rollback oid 19 current snap is 15 2024-09-12T23:38:50.501 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 19 to 9 2024-09-12T23:38:50.501 INFO:tasks.rados.rados.0.smithi136.stdout:180: copy_from oid 21 from oid 24 current snap is 15 2024-09-12T23:38:50.501 INFO:tasks.rados.rados.0.smithi136.stdout:181: copy_from oid 3 from oid 7 current snap is 15 2024-09-12T23:38:50.501 INFO:tasks.rados.rados.0.smithi136.stdout:182: read oid 33 snap -1 2024-09-12T23:38:50.501 INFO:tasks.rados.rados.0.smithi136.stdout:182: expect (ObjNum 15 snap 0 seq_num 15) 2024-09-12T23:38:50.502 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 16 current snap is 15 2024-09-12T23:38:50.502 INFO:tasks.rados.rados.0.smithi136.stdout:183: seq_num 81 ranges {1572864=565248,2138112=737280,2875392=65536} 2024-09-12T23:38:50.535 INFO:tasks.rados.rados.0.smithi136.stdout:183: writing smithi13667081-16 from 1572864 to 2138112 tid 1 2024-09-12T23:38:50.544 INFO:tasks.rados.rados.0.smithi136.stdout:183: writing smithi13667081-16 from 2138112 to 2875392 tid 2 2024-09-12T23:38:50.545 INFO:tasks.rados.rados.0.smithi136.stdout:183: writing smithi13667081-16 from 2875392 to 2940928 tid 3 2024-09-12T23:38:50.545 INFO:tasks.rados.rados.0.smithi136.stdout:184: rmattr oid 34 current snap is 15 2024-09-12T23:38:50.545 INFO:tasks.rados.rados.0.smithi136.stdout:184: done (7 left) 2024-09-12T23:38:50.545 INFO:tasks.rados.rados.0.smithi136.stdout:185: snap_remove snap 4 2024-09-12T23:38:50.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:50 smithi136 bash[22601]: cluster 2024-09-12T23:38:49.475152+0000 mon.a (mon.0) 1114 : cluster [DBG] osdmap e93: 8 total, 8 up, 8 in 2024-09-12T23:38:50.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:50 smithi136 bash[32189]: cluster 2024-09-12T23:38:49.475152+0000 mon.a (mon.0) 1114 : cluster [DBG] osdmap e93: 8 total, 8 up, 8 in 2024-09-12T23:38:50.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:50 smithi181 bash[24776]: cluster 2024-09-12T23:38:49.475152+0000 mon.a (mon.0) 1114 : cluster [DBG] osdmap e93: 8 total, 8 up, 8 in 2024-09-12T23:38:51.506 INFO:tasks.rados.rados.0.smithi136.stdout:185: done (7 left) 2024-09-12T23:38:51.506 INFO:tasks.rados.rados.0.smithi136.stdout:186: read oid 39 snap -1 2024-09-12T23:38:51.506 INFO:tasks.rados.rados.0.smithi136.stdout:186: expect (ObjNum 38 snap 0 seq_num 38) 2024-09-12T23:38:51.515 INFO:tasks.rados.rados.0.smithi136.stdout:177: finishing write tid 3 to smithi13667081-13 2024-09-12T23:38:51.515 INFO:tasks.rados.rados.0.smithi136.stdout:177: finishing write tid 4 to smithi13667081-13 2024-09-12T23:38:51.515 INFO:tasks.rados.rados.0.smithi136.stdout:177: finishing write tid 5 to smithi13667081-13 2024-09-12T23:38:51.515 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 13 v 36 (ObjNum 80 snap 15 seq_num 80) dirty exists 2024-09-12T23:38:51.515 INFO:tasks.rados.rados.0.smithi136.stdout:177: left oid 13 (ObjNum 80 snap 15 seq_num 80) 2024-09-12T23:38:51.515 INFO:tasks.rados.rados.0.smithi136.stdout:182: expect (ObjNum 15 snap 0 seq_num 15) 2024-09-12T23:38:51.518 INFO:tasks.rados.rados.0.smithi136.stdout:181: finishing copy_from racing read to smithi13667081-3 2024-09-12T23:38:51.518 INFO:tasks.rados.rados.0.smithi136.stdout:179: finishing rollback tid 0 to smithi13667081-19 2024-09-12T23:38:51.518 INFO:tasks.rados.rados.0.smithi136.stdout:179: finishing rollback tid 1 to smithi13667081-19 2024-09-12T23:38:51.518 INFO:tasks.rados.rados.0.smithi136.stdout:181: finishing copy_from to smithi13667081-3 2024-09-12T23:38:51.518 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 3 v 2 (ObjNum 6 snap 0 seq_num 6) dirty exists 2024-09-12T23:38:51.518 INFO:tasks.rados.rados.0.smithi136.stdout:178: finishing copy_from racing read to smithi13667081-15 2024-09-12T23:38:51.518 INFO:tasks.rados.rados.0.smithi136.stdout:179: finishing rollback tid 2 to smithi13667081-19 2024-09-12T23:38:51.518 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 19 v 35 (ObjNum 18 snap 0 seq_num 18) dirty exists 2024-09-12T23:38:51.519 INFO:tasks.rados.rados.0.smithi136.stdout:178: finishing copy_from to smithi13667081-15 2024-09-12T23:38:51.519 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 15 v 2 (ObjNum 4 snap 0 seq_num 4) dirty exists 2024-09-12T23:38:51.519 INFO:tasks.rados.rados.0.smithi136.stdout:180: finishing copy_from racing read to smithi13667081-21 2024-09-12T23:38:51.519 INFO:tasks.rados.rados.0.smithi136.stdout:180: finishing copy_from to smithi13667081-21 2024-09-12T23:38:51.519 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 21 v 10 (ObjNum 54 snap 4 seq_num 54) dirty exists 2024-09-12T23:38:51.519 INFO:tasks.rados.rados.0.smithi136.stdout:183: finishing write tid 1 to smithi13667081-16 2024-09-12T23:38:51.519 INFO:tasks.rados.rados.0.smithi136.stdout:183: finishing write tid 2 to smithi13667081-16 2024-09-12T23:38:51.519 INFO:tasks.rados.rados.0.smithi136.stdout:183: finishing write tid 3 to smithi13667081-16 2024-09-12T23:38:51.519 INFO:tasks.rados.rados.0.smithi136.stdout:183: finishing write tid 4 to smithi13667081-16 2024-09-12T23:38:51.519 INFO:tasks.rados.rados.0.smithi136.stdout:183: finishing write tid 5 to smithi13667081-16 2024-09-12T23:38:51.519 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 16 v 61 (ObjNum 81 snap 15 seq_num 81) dirty exists 2024-09-12T23:38:51.519 INFO:tasks.rados.rados.0.smithi136.stdout:183: left oid 16 (ObjNum 81 snap 15 seq_num 81) 2024-09-12T23:38:51.519 INFO:tasks.rados.rados.0.smithi136.stdout:177: done (7 left) 2024-09-12T23:38:51.519 INFO:tasks.rados.rados.0.smithi136.stdout:178: done (6 left) 2024-09-12T23:38:51.519 INFO:tasks.rados.rados.0.smithi136.stdout:179: done (5 left) 2024-09-12T23:38:51.520 INFO:tasks.rados.rados.0.smithi136.stdout:180: done (4 left) 2024-09-12T23:38:51.520 INFO:tasks.rados.rados.0.smithi136.stdout:181: done (3 left) 2024-09-12T23:38:51.520 INFO:tasks.rados.rados.0.smithi136.stdout:182: done (2 left) 2024-09-12T23:38:51.520 INFO:tasks.rados.rados.0.smithi136.stdout:183: done (1 left) 2024-09-12T23:38:51.520 INFO:tasks.rados.rados.0.smithi136.stdout:187: rmattr oid 1 current snap is 15 2024-09-12T23:38:51.520 INFO:tasks.rados.rados.0.smithi136.stdout:187: done (1 left) 2024-09-12T23:38:51.520 INFO:tasks.rados.rados.0.smithi136.stdout:188: snap_create 2024-09-12T23:38:51.531 INFO:tasks.rados.rados.0.smithi136.stdout:186: expect (ObjNum 38 snap 0 seq_num 38) 2024-09-12T23:38:51.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:51 smithi136 bash[22601]: cluster 2024-09-12T23:38:50.023270+0000 mgr.y (mgr.24446) 200 : cluster [DBG] pgmap v182: 121 pgs: 121 active+clean; 38 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 445 KiB/s rd, 1.5 MiB/s wr, 7 op/s 2024-09-12T23:38:51.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:51 smithi136 bash[22601]: cluster 2024-09-12T23:38:50.480444+0000 mon.a (mon.0) 1115 : cluster [DBG] osdmap e94: 8 total, 8 up, 8 in 2024-09-12T23:38:51.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:51 smithi136 bash[32189]: cluster 2024-09-12T23:38:50.023270+0000 mgr.y (mgr.24446) 200 : cluster [DBG] pgmap v182: 121 pgs: 121 active+clean; 38 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 445 KiB/s rd, 1.5 MiB/s wr, 7 op/s 2024-09-12T23:38:51.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:51 smithi136 bash[32189]: cluster 2024-09-12T23:38:50.480444+0000 mon.a (mon.0) 1115 : cluster [DBG] osdmap e94: 8 total, 8 up, 8 in 2024-09-12T23:38:51.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:51 smithi181 bash[24776]: cluster 2024-09-12T23:38:50.023270+0000 mgr.y (mgr.24446) 200 : cluster [DBG] pgmap v182: 121 pgs: 121 active+clean; 38 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 445 KiB/s rd, 1.5 MiB/s wr, 7 op/s 2024-09-12T23:38:51.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:51 smithi181 bash[24776]: cluster 2024-09-12T23:38:50.480444+0000 mon.a (mon.0) 1115 : cluster [DBG] osdmap e94: 8 total, 8 up, 8 in 2024-09-12T23:38:52.517 INFO:tasks.rados.rados.0.smithi136.stdout:186: done (1 left) 2024-09-12T23:38:52.518 INFO:tasks.rados.rados.0.smithi136.stdout:188: done (0 left) 2024-09-12T23:38:52.518 INFO:tasks.rados.rados.0.smithi136.stdout:189: copy_from oid 17 from oid 8 current snap is 16 2024-09-12T23:38:52.518 INFO:tasks.rados.rados.0.smithi136.stdout:190: read oid 32 snap -1 2024-09-12T23:38:52.518 INFO:tasks.rados.rados.0.smithi136.stdout:190: expect (ObjNum 64 snap 6 seq_num 64) 2024-09-12T23:38:52.518 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 43 current snap is 16 2024-09-12T23:38:52.518 INFO:tasks.rados.rados.0.smithi136.stdout:191: seq_num 82 ranges {1212416=581632,1794048=622592,2416640=180224} 2024-09-12T23:38:52.540 INFO:tasks.rados.rados.0.smithi136.stdout:191: writing smithi13667081-43 from 1212416 to 1794048 tid 1 2024-09-12T23:38:52.549 INFO:tasks.rados.rados.0.smithi136.stdout:191: writing smithi13667081-43 from 1794048 to 2416640 tid 2 2024-09-12T23:38:52.554 INFO:tasks.rados.rados.0.smithi136.stdout:191: writing smithi13667081-43 from 2416640 to 2596864 tid 3 2024-09-12T23:38:52.554 INFO:tasks.rados.rados.0.smithi136.stdout:192: setattr oid 39 current snap is 16 2024-09-12T23:38:52.555 INFO:tasks.rados.rados.0.smithi136.stdout:189: finishing copy_from to smithi13667081-17 2024-09-12T23:38:52.555 INFO:tasks.rados.rados.0.smithi136.stdout:189: got expected ENOENT (src dne) 2024-09-12T23:38:52.555 INFO:tasks.rados.rados.0.smithi136.stdout:189: finishing copy_from racing read to smithi13667081-17 2024-09-12T23:38:52.555 INFO:tasks.rados.rados.0.smithi136.stdout:190: expect (ObjNum 64 snap 6 seq_num 64) 2024-09-12T23:38:52.649 INFO:tasks.rados.rados.0.smithi136.stdout:191: finishing write tid 1 to smithi13667081-43 2024-09-12T23:38:52.650 INFO:tasks.rados.rados.0.smithi136.stdout:191: finishing write tid 2 to smithi13667081-43 2024-09-12T23:38:52.650 INFO:tasks.rados.rados.0.smithi136.stdout:191: finishing write tid 3 to smithi13667081-43 2024-09-12T23:38:52.650 INFO:tasks.rados.rados.0.smithi136.stdout:191: finishing write tid 4 to smithi13667081-43 2024-09-12T23:38:52.650 INFO:tasks.rados.rados.0.smithi136.stdout:191: finishing write tid 5 to smithi13667081-43 2024-09-12T23:38:52.650 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 43 v 76 (ObjNum 82 snap 16 seq_num 82) dirty exists 2024-09-12T23:38:52.650 INFO:tasks.rados.rados.0.smithi136.stdout:191: left oid 43 (ObjNum 82 snap 16 seq_num 82) 2024-09-12T23:38:52.650 INFO:tasks.rados.rados.0.smithi136.stdout:189: done (3 left) 2024-09-12T23:38:52.650 INFO:tasks.rados.rados.0.smithi136.stdout:190: done (2 left) 2024-09-12T23:38:52.650 INFO:tasks.rados.rados.0.smithi136.stdout:191: done (1 left) 2024-09-12T23:38:52.650 INFO:tasks.rados.rados.0.smithi136.stdout:193: read oid 21 snap 14 2024-09-12T23:38:52.650 INFO:tasks.rados.rados.0.smithi136.stdout:193: expect (ObjNum 34 snap 0 seq_num 34) 2024-09-12T23:38:52.650 INFO:tasks.rados.rados.0.smithi136.stdout:194: snap_create 2024-09-12T23:38:52.655 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 39 v 12 (ObjNum 38 snap 0 seq_num 38) dirty exists 2024-09-12T23:38:52.657 INFO:tasks.rados.rados.0.smithi136.stdout:193: expect (ObjNum 34 snap 0 seq_num 34) 2024-09-12T23:38:52.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:52 smithi181 bash[24776]: cluster 2024-09-12T23:38:51.509841+0000 mon.a (mon.0) 1116 : cluster [DBG] osdmap e95: 8 total, 8 up, 8 in 2024-09-12T23:38:53.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:52 smithi136 bash[22601]: cluster 2024-09-12T23:38:51.509841+0000 mon.a (mon.0) 1116 : cluster [DBG] osdmap e95: 8 total, 8 up, 8 in 2024-09-12T23:38:53.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:52 smithi136 bash[32189]: cluster 2024-09-12T23:38:51.509841+0000 mon.a (mon.0) 1116 : cluster [DBG] osdmap e95: 8 total, 8 up, 8 in 2024-09-12T23:38:53.530 INFO:tasks.rados.rados.0.smithi136.stdout:192: done (2 left) 2024-09-12T23:38:53.530 INFO:tasks.rados.rados.0.smithi136.stdout:193: done (1 left) 2024-09-12T23:38:53.530 INFO:tasks.rados.rados.0.smithi136.stdout:194: done (0 left) 2024-09-12T23:38:53.530 INFO:tasks.rados.rados.0.smithi136.stdout:195: rmattr oid 47 current snap is 17 2024-09-12T23:38:53.530 INFO:tasks.rados.rados.0.smithi136.stdout:195: done (0 left) 2024-09-12T23:38:53.530 INFO:tasks.rados.rados.0.smithi136.stdout:196: rmattr oid 39 current snap is 17 2024-09-12T23:38:53.530 INFO:tasks.rados.rados.0.smithi136.stdout:197: read oid 5 snap -1 2024-09-12T23:38:53.530 INFO:tasks.rados.rados.0.smithi136.stdout:197: expect (ObjNum 4 snap 0 seq_num 4) 2024-09-12T23:38:53.530 INFO:tasks.rados.rados.0.smithi136.stdout:198: read oid 48 snap -1 2024-09-12T23:38:53.530 INFO:tasks.rados.rados.0.smithi136.stdout:198: expect deleted 2024-09-12T23:38:53.530 INFO:tasks.rados.rados.0.smithi136.stdout:199: setattr oid 38 current snap is 17 2024-09-12T23:38:53.534 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 37 current snap is 17 2024-09-12T23:38:53.534 INFO:tasks.rados.rados.0.smithi136.stdout:200: seq_num 83 ranges {573440=598016,1171456=499712,1671168=303104} 2024-09-12T23:38:53.554 INFO:tasks.rados.rados.0.smithi136.stdout:200: writing smithi13667081-37 from 573440 to 1171456 tid 1 2024-09-12T23:38:53.563 INFO:tasks.rados.rados.0.smithi136.stdout:200: writing smithi13667081-37 from 1171456 to 1671168 tid 2 2024-09-12T23:38:53.572 INFO:tasks.rados.rados.0.smithi136.stdout:200: writing smithi13667081-37 from 1671168 to 1974272 tid 3 2024-09-12T23:38:53.572 INFO:tasks.rados.rados.0.smithi136.stdout:201: copy_from oid 41 from oid 31 current snap is 17 2024-09-12T23:38:53.572 INFO:tasks.rados.rados.0.smithi136.stdout:202: snap_create 2024-09-12T23:38:53.572 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 39 v 14 (ObjNum 38 snap 0 seq_num 38) dirty exists 2024-09-12T23:38:53.573 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 38 v 42 (ObjNum 37 snap 0 seq_num 37) dirty exists 2024-09-12T23:38:53.573 INFO:tasks.rados.rados.0.smithi136.stdout:197: expect (ObjNum 4 snap 0 seq_num 4) 2024-09-12T23:38:53.573 INFO:tasks.rados.rados.0.smithi136.stdout:200: finishing write tid 1 to smithi13667081-37 2024-09-12T23:38:53.573 INFO:tasks.rados.rados.0.smithi136.stdout:200: finishing write tid 2 to smithi13667081-37 2024-09-12T23:38:53.575 INFO:tasks.rados.rados.0.smithi136.stdout:200: finishing write tid 3 to smithi13667081-37 2024-09-12T23:38:53.575 INFO:tasks.rados.rados.0.smithi136.stdout:200: finishing write tid 4 to smithi13667081-37 2024-09-12T23:38:53.578 INFO:tasks.rados.rados.0.smithi136.stdout:200: finishing write tid 5 to smithi13667081-37 2024-09-12T23:38:53.579 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 37 v 45 (ObjNum 83 snap 17 seq_num 83) dirty exists 2024-09-12T23:38:53.579 INFO:tasks.rados.rados.0.smithi136.stdout:200: left oid 37 (ObjNum 83 snap 17 seq_num 83) 2024-09-12T23:38:53.604 INFO:tasks.rados.rados.0.smithi136.stdout:201: finishing copy_from racing read to smithi13667081-41 2024-09-12T23:38:53.604 INFO:tasks.rados.rados.0.smithi136.stdout:201: finishing copy_from to smithi13667081-41 2024-09-12T23:38:53.604 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 41 v 72 (ObjNum 77 snap 13 seq_num 77) dirty exists 2024-09-12T23:38:53.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:53 smithi181 bash[24776]: cluster 2024-09-12T23:38:52.024011+0000 mgr.y (mgr.24446) 201 : cluster [DBG] pgmap v185: 121 pgs: 121 active+clean; 38 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 445 KiB/s rd, 1.5 MiB/s wr, 7 op/s 2024-09-12T23:38:53.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:53 smithi181 bash[24776]: cluster 2024-09-12T23:38:52.521502+0000 mon.a (mon.0) 1117 : cluster [DBG] osdmap e96: 8 total, 8 up, 8 in 2024-09-12T23:38:53.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:53 smithi181 bash[24776]: audit 2024-09-12T23:38:53.013943+0000 mon.a (mon.0) 1118 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:38:54.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:53 smithi136 bash[22601]: cluster 2024-09-12T23:38:52.024011+0000 mgr.y (mgr.24446) 201 : cluster [DBG] pgmap v185: 121 pgs: 121 active+clean; 38 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 445 KiB/s rd, 1.5 MiB/s wr, 7 op/s 2024-09-12T23:38:54.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:53 smithi136 bash[22601]: cluster 2024-09-12T23:38:52.521502+0000 mon.a (mon.0) 1117 : cluster [DBG] osdmap e96: 8 total, 8 up, 8 in 2024-09-12T23:38:54.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:53 smithi136 bash[22601]: audit 2024-09-12T23:38:53.013943+0000 mon.a (mon.0) 1118 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:38:54.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:53 smithi136 bash[32189]: cluster 2024-09-12T23:38:52.024011+0000 mgr.y (mgr.24446) 201 : cluster [DBG] pgmap v185: 121 pgs: 121 active+clean; 38 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 445 KiB/s rd, 1.5 MiB/s wr, 7 op/s 2024-09-12T23:38:54.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:53 smithi136 bash[32189]: cluster 2024-09-12T23:38:52.521502+0000 mon.a (mon.0) 1117 : cluster [DBG] osdmap e96: 8 total, 8 up, 8 in 2024-09-12T23:38:54.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:53 smithi136 bash[32189]: audit 2024-09-12T23:38:53.013943+0000 mon.a (mon.0) 1118 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:38:54.046 INFO:teuthology.orchestra.run.smithi136.stdout:true 2024-09-12T23:38:54.343 INFO:teuthology.orchestra.run.smithi136.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-12T23:38:54.343 INFO:teuthology.orchestra.run.smithi136.stdout:alertmanager.a smithi136 *:9093,9094 running (3m) 3m ago 4m 14.3M - 0.25.0 c8568f914cd2 d6c106834390 2024-09-12T23:38:54.343 INFO:teuthology.orchestra.run.smithi136.stdout:grafana.a smithi181 *:3000 running (3m) 3m ago 3m 29.0M - 9.4.7 954c08fa6188 85ee0f95160c 2024-09-12T23:38:54.343 INFO:teuthology.orchestra.run.smithi136.stdout:mds.a smithi136 running (5m) 3m ago 5m 12.1M - 17.2.7-1627-gbbdadfa3 214985049bdb 713b9795ad19 2024-09-12T23:38:54.343 INFO:teuthology.orchestra.run.smithi136.stdout:mds.b smithi181 running (5m) 3m ago 5m 13.9M - 17.2.7-1627-gbbdadfa3 214985049bdb ad13179f0868 2024-09-12T23:38:54.343 INFO:teuthology.orchestra.run.smithi136.stdout:mgr.x smithi181 *:8443 running (12m) 3m ago 12m 387M - 17.2.7-1627-gbbdadfa3 214985049bdb d07cbbb66db7 2024-09-12T23:38:54.343 INFO:teuthology.orchestra.run.smithi136.stdout:mgr.y smithi136 *:9283,8765 running (15m) 3m ago 15m 436M - 17.2.7-1627-gbbdadfa3 214985049bdb 56608cdf6b27 2024-09-12T23:38:54.343 INFO:teuthology.orchestra.run.smithi136.stdout:mon.a smithi136 running (15m) 3m ago 15m 44.1M 2048M 17.2.7-1627-gbbdadfa3 214985049bdb 0462f4f61475 2024-09-12T23:38:54.343 INFO:teuthology.orchestra.run.smithi136.stdout:mon.b smithi181 running (13m) 3m ago 13m 37.6M 2048M 17.2.7-1627-gbbdadfa3 214985049bdb 8b3d784af845 2024-09-12T23:38:54.343 INFO:teuthology.orchestra.run.smithi136.stdout:mon.c smithi136 running (13m) 3m ago 13m 36.8M 2048M 17.2.7-1627-gbbdadfa3 214985049bdb 647b028ea963 2024-09-12T23:38:54.344 INFO:teuthology.orchestra.run.smithi136.stdout:node-exporter.a smithi136 *:9100 running (4m) 3m ago 4m 3143k - 1.5.0 0da6a335fe13 8f54fc5b0da3 2024-09-12T23:38:54.344 INFO:teuthology.orchestra.run.smithi136.stdout:node-exporter.b smithi181 *:9100 running (4m) 3m ago 4m 2895k - 1.5.0 0da6a335fe13 575eff556149 2024-09-12T23:38:54.344 INFO:teuthology.orchestra.run.smithi136.stdout:osd.0 smithi136 running (12m) 3m ago 12m 47.0M 3036M 17.2.7-1627-gbbdadfa3 214985049bdb cbc0b7df0115 2024-09-12T23:38:54.344 INFO:teuthology.orchestra.run.smithi136.stdout:osd.1 smithi136 running (11m) 3m ago 11m 44.4M 3036M 17.2.7-1627-gbbdadfa3 214985049bdb c4daee7da836 2024-09-12T23:38:54.344 INFO:teuthology.orchestra.run.smithi136.stdout:osd.2 smithi136 running (10m) 3m ago 10m 44.8M 3036M 17.2.7-1627-gbbdadfa3 214985049bdb 4ab8fd79cdb4 2024-09-12T23:38:54.344 INFO:teuthology.orchestra.run.smithi136.stdout:osd.3 smithi136 running (9m) 3m ago 9m 42.5M 3036M 17.2.7-1627-gbbdadfa3 214985049bdb 9ceb14a27521 2024-09-12T23:38:54.344 INFO:teuthology.orchestra.run.smithi136.stdout:osd.4 smithi181 running (8m) 3m ago 8m 42.7M 3036M 17.2.7-1627-gbbdadfa3 214985049bdb a35cdca79055 2024-09-12T23:38:54.344 INFO:teuthology.orchestra.run.smithi136.stdout:osd.5 smithi181 running (7m) 3m ago 7m 42.2M 3036M 17.2.7-1627-gbbdadfa3 214985049bdb 8f2e8d494fef 2024-09-12T23:38:54.344 INFO:teuthology.orchestra.run.smithi136.stdout:osd.6 smithi181 running (7m) 3m ago 7m 41.2M 3036M 17.2.7-1627-gbbdadfa3 214985049bdb 9c6bec406408 2024-09-12T23:38:54.344 INFO:teuthology.orchestra.run.smithi136.stdout:osd.7 smithi181 running (6m) 3m ago 6m 42.3M 3036M 17.2.7-1627-gbbdadfa3 214985049bdb bc0632ae2145 2024-09-12T23:38:54.344 INFO:teuthology.orchestra.run.smithi136.stdout:prometheus.a smithi181 *:9095 running (3m) 3m ago 4m 22.6M - 2.43.0 a07b618ecd1d 24e591310569 2024-09-12T23:38:54.534 INFO:tasks.rados.rados.0.smithi136.stdout:196: done (6 left) 2024-09-12T23:38:54.534 INFO:tasks.rados.rados.0.smithi136.stdout:197: done (5 left) 2024-09-12T23:38:54.534 INFO:tasks.rados.rados.0.smithi136.stdout:198: done (4 left) 2024-09-12T23:38:54.534 INFO:tasks.rados.rados.0.smithi136.stdout:199: done (3 left) 2024-09-12T23:38:54.534 INFO:tasks.rados.rados.0.smithi136.stdout:200: done (2 left) 2024-09-12T23:38:54.534 INFO:tasks.rados.rados.0.smithi136.stdout:201: done (1 left) 2024-09-12T23:38:54.534 INFO:tasks.rados.rados.0.smithi136.stdout:202: done (0 left) 2024-09-12T23:38:54.534 INFO:tasks.rados.rados.0.smithi136.stdout:203: delete oid 31 current snap is 18 2024-09-12T23:38:54.541 INFO:tasks.rados.rados.0.smithi136.stdout:203: done (0 left) 2024-09-12T23:38:54.541 INFO:tasks.rados.rados.0.smithi136.stdout:204: delete oid 26 current snap is 18 2024-09-12T23:38:54.544 INFO:tasks.rados.rados.0.smithi136.stdout:204: done (0 left) 2024-09-12T23:38:54.544 INFO:tasks.rados.rados.0.smithi136.stdout:205: copy_from oid 20 from oid 42 current snap is 18 2024-09-12T23:38:54.544 INFO:tasks.rados.rados.0.smithi136.stdout:206: delete oid 5 current snap is 18 2024-09-12T23:38:54.548 INFO:tasks.rados.rados.0.smithi136.stdout:206: done (1 left) 2024-09-12T23:38:54.548 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 13 current snap is 18 2024-09-12T23:38:54.549 INFO:tasks.rados.rados.0.smithi136.stdout:207: seq_num 87 ranges {1351680=663552,2015232=491520,2506752=311296} 2024-09-12T23:38:54.581 INFO:tasks.rados.rados.0.smithi136.stdout:207: writing smithi13667081-13 from 1351680 to 2015232 tid 1 2024-09-12T23:38:54.588 INFO:tasks.rados.rados.0.smithi136.stdout:207: writing smithi13667081-13 from 2015232 to 2506752 tid 2 2024-09-12T23:38:54.593 INFO:tasks.rados.rados.0.smithi136.stdout:207: writing smithi13667081-13 from 2506752 to 2818048 tid 3 2024-09-12T23:38:54.593 INFO:tasks.rados.rados.0.smithi136.stdout:208: read oid 19 snap -1 2024-09-12T23:38:54.593 INFO:tasks.rados.rados.0.smithi136.stdout:208: expect (ObjNum 18 snap 0 seq_num 18) 2024-09-12T23:38:54.593 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 21 current snap is 18 2024-09-12T23:38:54.593 INFO:tasks.rados.rados.0.smithi136.stdout:205: finishing copy_from to smithi13667081-20 2024-09-12T23:38:54.593 INFO:tasks.rados.rados.0.smithi136.stdout:205: got expected ENOENT (src dne) 2024-09-12T23:38:54.593 INFO:tasks.rados.rados.0.smithi136.stdout:205: finishing copy_from racing read to smithi13667081-20 2024-09-12T23:38:54.593 INFO:tasks.rados.rados.0.smithi136.stdout:207: finishing write tid 1 to smithi13667081-13 2024-09-12T23:38:54.593 INFO:tasks.rados.rados.0.smithi136.stdout:209: seq_num 88 ranges {1302528=679936,1982464=778240,2760704=24576} 2024-09-12T23:38:54.616 INFO:tasks.rados.rados.0.smithi136.stdout:209: writing smithi13667081-21 from 1302528 to 1982464 tid 1 2024-09-12T23:38:54.625 INFO:tasks.rados.rados.0.smithi136.stdout:209: writing smithi13667081-21 from 1982464 to 2760704 tid 2 2024-09-12T23:38:54.625 INFO:tasks.rados.rados.0.smithi136.stdout:209: writing smithi13667081-21 from 2760704 to 2785280 tid 3 2024-09-12T23:38:54.625 INFO:tasks.rados.rados.0.smithi136.stdout:205: done (3 left) 2024-09-12T23:38:54.625 INFO:tasks.rados.rados.0.smithi136.stdout:210: delete oid 11 current snap is 18 2024-09-12T23:38:54.625 INFO:tasks.rados.rados.0.smithi136.stdout:207: finishing write tid 2 to smithi13667081-13 2024-09-12T23:38:54.626 INFO:tasks.rados.rados.0.smithi136.stdout:207: finishing write tid 3 to smithi13667081-13 2024-09-12T23:38:54.626 INFO:tasks.rados.rados.0.smithi136.stdout:208: expect (ObjNum 18 snap 0 seq_num 18) 2024-09-12T23:38:54.652 INFO:tasks.rados.rados.0.smithi136.stdout:207: finishing write tid 4 to smithi13667081-13 2024-09-12T23:38:54.652 INFO:tasks.rados.rados.0.smithi136.stdout:207: finishing write tid 5 to smithi13667081-13 2024-09-12T23:38:54.653 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 13 v 47 (ObjNum 87 snap 18 seq_num 87) dirty exists 2024-09-12T23:38:54.653 INFO:tasks.rados.rados.0.smithi136.stdout:207: left oid 13 (ObjNum 87 snap 18 seq_num 87) 2024-09-12T23:38:54.653 INFO:tasks.rados.rados.0.smithi136.stdout:209: finishing write tid 1 to smithi13667081-21 2024-09-12T23:38:54.653 INFO:tasks.rados.rados.0.smithi136.stdout:209: finishing write tid 2 to smithi13667081-21 2024-09-12T23:38:54.653 INFO:tasks.rados.rados.0.smithi136.stdout:207: done (3 left) 2024-09-12T23:38:54.653 INFO:tasks.rados.rados.0.smithi136.stdout:208: done (2 left) 2024-09-12T23:38:54.653 INFO:tasks.rados.rados.0.smithi136.stdout:210: done (1 left) 2024-09-12T23:38:54.653 INFO:tasks.rados.rados.0.smithi136.stdout:211: read oid 32 snap -1 2024-09-12T23:38:54.653 INFO:tasks.rados.rados.0.smithi136.stdout:211: expect (ObjNum 64 snap 6 seq_num 64) 2024-09-12T23:38:54.653 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 49 current snap is 18 2024-09-12T23:38:54.653 INFO:tasks.rados.rados.0.smithi136.stdout:209: finishing write tid 3 to smithi13667081-21 2024-09-12T23:38:54.653 INFO:tasks.rados.rados.0.smithi136.stdout:209: finishing write tid 4 to smithi13667081-21 2024-09-12T23:38:54.654 INFO:tasks.rados.rados.0.smithi136.stdout:212: seq_num 90 ranges {1736704=712704,2449408=540672,2990080=262144} 2024-09-12T23:38:54.681 INFO:tasks.rados.rados.0.smithi136.stdout:212: writing smithi13667081-49 from 1736704 to 2449408 tid 1 2024-09-12T23:38:54.686 INFO:tasks.rados.rados.0.smithi136.stdout:212: writing smithi13667081-49 from 2449408 to 2990080 tid 2 2024-09-12T23:38:54.689 INFO:tasks.rados.rados.0.smithi136.stdout:212: writing smithi13667081-49 from 2990080 to 3252224 tid 3 2024-09-12T23:38:54.689 INFO:tasks.rados.rados.0.smithi136.stdout:213: delete oid 15 current snap is 18 2024-09-12T23:38:54.689 INFO:tasks.rados.rados.0.smithi136.stdout:209: finishing write tid 5 to smithi13667081-21 2024-09-12T23:38:54.689 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 21 v 38 (ObjNum 88 snap 18 seq_num 88) dirty exists 2024-09-12T23:38:54.689 INFO:tasks.rados.rados.0.smithi136.stdout:209: left oid 21 (ObjNum 88 snap 18 seq_num 88) 2024-09-12T23:38:54.689 INFO:tasks.rados.rados.0.smithi136.stdout:211: expect (ObjNum 64 snap 6 seq_num 64) 2024-09-12T23:38:54.697 INFO:teuthology.orchestra.run.smithi136.stdout:{ 2024-09-12T23:38:54.698 INFO:teuthology.orchestra.run.smithi136.stdout: "mon": { 2024-09-12T23:38:54.698 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 3 2024-09-12T23:38:54.698 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:38:54.698 INFO:teuthology.orchestra.run.smithi136.stdout: "mgr": { 2024-09-12T23:38:54.698 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-12T23:38:54.698 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:38:54.698 INFO:teuthology.orchestra.run.smithi136.stdout: "osd": { 2024-09-12T23:38:54.698 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 8 2024-09-12T23:38:54.698 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:38:54.698 INFO:teuthology.orchestra.run.smithi136.stdout: "mds": { 2024-09-12T23:38:54.698 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-12T23:38:54.698 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:38:54.699 INFO:teuthology.orchestra.run.smithi136.stdout: "overall": { 2024-09-12T23:38:54.699 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 15 2024-09-12T23:38:54.699 INFO:teuthology.orchestra.run.smithi136.stdout: } 2024-09-12T23:38:54.699 INFO:teuthology.orchestra.run.smithi136.stdout:} 2024-09-12T23:38:54.763 INFO:tasks.rados.rados.0.smithi136.stdout:212: finishing write tid 1 to smithi13667081-49 2024-09-12T23:38:54.763 INFO:tasks.rados.rados.0.smithi136.stdout:212: finishing write tid 2 to smithi13667081-49 2024-09-12T23:38:54.763 INFO:tasks.rados.rados.0.smithi136.stdout:212: finishing write tid 3 to smithi13667081-49 2024-09-12T23:38:54.763 INFO:tasks.rados.rados.0.smithi136.stdout:212: finishing write tid 4 to smithi13667081-49 2024-09-12T23:38:54.763 INFO:tasks.rados.rados.0.smithi136.stdout:212: finishing write tid 5 to smithi13667081-49 2024-09-12T23:38:54.763 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 49 v 95 (ObjNum 90 snap 18 seq_num 90) dirty exists 2024-09-12T23:38:54.763 INFO:tasks.rados.rados.0.smithi136.stdout:212: left oid 49 (ObjNum 90 snap 18 seq_num 90) 2024-09-12T23:38:54.763 INFO:tasks.rados.rados.0.smithi136.stdout:209: done (3 left) 2024-09-12T23:38:54.764 INFO:tasks.rados.rados.0.smithi136.stdout:211: done (2 left) 2024-09-12T23:38:54.764 INFO:tasks.rados.rados.0.smithi136.stdout:212: done (1 left) 2024-09-12T23:38:54.764 INFO:tasks.rados.rados.0.smithi136.stdout:213: done (0 left) 2024-09-12T23:38:54.764 INFO:tasks.rados.rados.0.smithi136.stdout:214: delete oid 35 current snap is 18 2024-09-12T23:38:54.766 INFO:tasks.rados.rados.0.smithi136.stdout:214: done (0 left) 2024-09-12T23:38:54.766 INFO:tasks.rados.rados.0.smithi136.stdout:215: setattr oid 29 current snap is 18 2024-09-12T23:38:54.768 INFO:tasks.rados.rados.0.smithi136.stdout:216: snap_remove snap 6 2024-09-12T23:38:54.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:54 smithi181 bash[24776]: cluster 2024-09-12T23:38:53.533768+0000 mon.a (mon.0) 1119 : cluster [DBG] osdmap e97: 8 total, 8 up, 8 in 2024-09-12T23:38:55.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:54 smithi136 bash[22601]: cluster 2024-09-12T23:38:53.533768+0000 mon.a (mon.0) 1119 : cluster [DBG] osdmap e97: 8 total, 8 up, 8 in 2024-09-12T23:38:55.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:54 smithi136 bash[32189]: cluster 2024-09-12T23:38:53.533768+0000 mon.a (mon.0) 1119 : cluster [DBG] osdmap e97: 8 total, 8 up, 8 in 2024-09-12T23:38:55.447 INFO:tasks.rados.rados.0.smithi136.stdout:216: done (1 left) 2024-09-12T23:38:55.447 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 38 current snap is 18 2024-09-12T23:38:55.447 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 29 v 48 (ObjNum 0 snap 0 seq_num 304) dirty exists 2024-09-12T23:38:55.447 INFO:tasks.rados.rados.0.smithi136.stdout:217: seq_num 93 ranges {622592=761856,1384448=647168,2031616=155648} 2024-09-12T23:38:55.472 INFO:tasks.rados.rados.0.smithi136.stdout:217: writing smithi13667081-38 from 622592 to 1384448 tid 1 2024-09-12T23:38:55.486 INFO:tasks.rados.rados.0.smithi136.stdout:217: writing smithi13667081-38 from 1384448 to 2031616 tid 2 2024-09-12T23:38:55.489 INFO:tasks.rados.rados.0.smithi136.stdout:217: writing smithi13667081-38 from 2031616 to 2187264 tid 3 2024-09-12T23:38:55.489 INFO:tasks.rados.rados.0.smithi136.stdout:215: done (1 left) 2024-09-12T23:38:55.489 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 2 current snap is 18 2024-09-12T23:38:55.489 INFO:tasks.rados.rados.0.smithi136.stdout:217: finishing write tid 1 to smithi13667081-38 2024-09-12T23:38:55.490 INFO:tasks.rados.rados.0.smithi136.stdout:218: seq_num 94 ranges {172032=778240,950272=532480,1482752=270336} 2024-09-12T23:38:55.503 INFO:tasks.rados.rados.0.smithi136.stdout:218: writing smithi13667081-2 from 172032 to 950272 tid 1 2024-09-12T23:38:55.511 INFO:tasks.rados.rados.0.smithi136.stdout:218: writing smithi13667081-2 from 950272 to 1482752 tid 2 2024-09-12T23:38:55.516 INFO:tasks.rados.rados.0.smithi136.stdout:218: writing smithi13667081-2 from 1482752 to 1753088 tid 3 2024-09-12T23:38:55.516 INFO:tasks.rados.rados.0.smithi136.stdout:219: copy_from oid 27 from oid 26 current snap is 18 2024-09-12T23:38:55.516 INFO:tasks.rados.rados.0.smithi136.stdout:217: finishing write tid 2 to smithi13667081-38 2024-09-12T23:38:55.516 INFO:tasks.rados.rados.0.smithi136.stdout:217: finishing write tid 3 to smithi13667081-38 2024-09-12T23:38:55.516 INFO:tasks.rados.rados.0.smithi136.stdout:217: finishing write tid 4 to smithi13667081-38 2024-09-12T23:38:55.516 INFO:tasks.rados.rados.0.smithi136.stdout:217: finishing write tid 5 to smithi13667081-38 2024-09-12T23:38:55.516 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 38 v 57 (ObjNum 93 snap 18 seq_num 93) dirty exists 2024-09-12T23:38:55.516 INFO:tasks.rados.rados.0.smithi136.stdout:217: left oid 38 (ObjNum 93 snap 18 seq_num 93) 2024-09-12T23:38:55.516 INFO:tasks.rados.rados.0.smithi136.stdout:218: finishing write tid 1 to smithi13667081-2 2024-09-12T23:38:55.516 INFO:tasks.rados.rados.0.smithi136.stdout:218: finishing write tid 2 to smithi13667081-2 2024-09-12T23:38:55.517 INFO:tasks.rados.rados.0.smithi136.stdout:217: done (2 left) 2024-09-12T23:38:55.517 INFO:tasks.rados.rados.0.smithi136.stdout:220: snap_create 2024-09-12T23:38:55.518 INFO:tasks.rados.rados.0.smithi136.stdout:219: finishing copy_from to smithi13667081-27 2024-09-12T23:38:55.518 INFO:tasks.rados.rados.0.smithi136.stdout:219: got expected ENOENT (src dne) 2024-09-12T23:38:55.518 INFO:tasks.rados.rados.0.smithi136.stdout:219: finishing copy_from racing read to smithi13667081-27 2024-09-12T23:38:55.519 INFO:tasks.rados.rados.0.smithi136.stdout:218: finishing write tid 3 to smithi13667081-2 2024-09-12T23:38:55.519 INFO:tasks.rados.rados.0.smithi136.stdout:218: finishing write tid 4 to smithi13667081-2 2024-09-12T23:38:55.520 INFO:tasks.rados.rados.0.smithi136.stdout:218: finishing write tid 5 to smithi13667081-2 2024-09-12T23:38:55.520 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 2 v 76 (ObjNum 94 snap 18 seq_num 94) dirty exists 2024-09-12T23:38:55.521 INFO:tasks.rados.rados.0.smithi136.stdout:218: left oid 2 (ObjNum 94 snap 18 seq_num 94) 2024-09-12T23:38:55.530 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:38:55 smithi181 bash[25717]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:38:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-12T23:38:55.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:55 smithi181 bash[24776]: cluster 2024-09-12T23:38:54.025318+0000 mgr.y (mgr.24446) 202 : cluster [DBG] pgmap v188: 121 pgs: 121 active+clean; 45 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.4 MiB/s wr, 26 op/s 2024-09-12T23:38:55.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:55 smithi181 bash[24776]: audit 2024-09-12T23:38:54.037098+0000 mgr.y (mgr.24446) 203 : audit [DBG] from='client.24914 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:38:55.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:55 smithi181 bash[24776]: audit 2024-09-12T23:38:54.312942+0000 mgr.y (mgr.24446) 204 : audit [DBG] from='client.24920 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:38:55.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:55 smithi181 bash[24776]: cluster 2024-09-12T23:38:54.537671+0000 mon.a (mon.0) 1120 : cluster [DBG] osdmap e98: 8 total, 8 up, 8 in 2024-09-12T23:38:55.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:55 smithi181 bash[24776]: audit 2024-09-12T23:38:54.702712+0000 mon.a (mon.0) 1121 : audit [DBG] from='client.? 172.21.15.136:0/185062950' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:38:55.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:55 smithi181 bash[24776]: cluster 2024-09-12T23:38:55.450710+0000 mon.a (mon.0) 1122 : cluster [DBG] osdmap e99: 8 total, 8 up, 8 in 2024-09-12T23:38:56.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:55 smithi136 bash[32189]: cluster 2024-09-12T23:38:54.025318+0000 mgr.y (mgr.24446) 202 : cluster [DBG] pgmap v188: 121 pgs: 121 active+clean; 45 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.4 MiB/s wr, 26 op/s 2024-09-12T23:38:56.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:55 smithi136 bash[32189]: audit 2024-09-12T23:38:54.037098+0000 mgr.y (mgr.24446) 203 : audit [DBG] from='client.24914 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:38:56.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:55 smithi136 bash[32189]: audit 2024-09-12T23:38:54.312942+0000 mgr.y (mgr.24446) 204 : audit [DBG] from='client.24920 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:38:56.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:55 smithi136 bash[32189]: cluster 2024-09-12T23:38:54.537671+0000 mon.a (mon.0) 1120 : cluster [DBG] osdmap e98: 8 total, 8 up, 8 in 2024-09-12T23:38:56.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:55 smithi136 bash[32189]: audit 2024-09-12T23:38:54.702712+0000 mon.a (mon.0) 1121 : audit [DBG] from='client.? 172.21.15.136:0/185062950' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:38:56.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:55 smithi136 bash[32189]: cluster 2024-09-12T23:38:55.450710+0000 mon.a (mon.0) 1122 : cluster [DBG] osdmap e99: 8 total, 8 up, 8 in 2024-09-12T23:38:56.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:55 smithi136 bash[22601]: cluster 2024-09-12T23:38:54.025318+0000 mgr.y (mgr.24446) 202 : cluster [DBG] pgmap v188: 121 pgs: 121 active+clean; 45 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.4 MiB/s wr, 26 op/s 2024-09-12T23:38:56.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:55 smithi136 bash[22601]: audit 2024-09-12T23:38:54.037098+0000 mgr.y (mgr.24446) 203 : audit [DBG] from='client.24914 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:38:56.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:55 smithi136 bash[22601]: audit 2024-09-12T23:38:54.312942+0000 mgr.y (mgr.24446) 204 : audit [DBG] from='client.24920 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:38:56.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:55 smithi136 bash[22601]: cluster 2024-09-12T23:38:54.537671+0000 mon.a (mon.0) 1120 : cluster [DBG] osdmap e98: 8 total, 8 up, 8 in 2024-09-12T23:38:56.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:55 smithi136 bash[22601]: audit 2024-09-12T23:38:54.702712+0000 mon.a (mon.0) 1121 : audit [DBG] from='client.? 172.21.15.136:0/185062950' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:38:56.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:55 smithi136 bash[22601]: cluster 2024-09-12T23:38:55.450710+0000 mon.a (mon.0) 1122 : cluster [DBG] osdmap e99: 8 total, 8 up, 8 in 2024-09-12T23:38:56.450 INFO:tasks.rados.rados.0.smithi136.stdout:218: done (2 left) 2024-09-12T23:38:56.450 INFO:tasks.rados.rados.0.smithi136.stdout:219: done (1 left) 2024-09-12T23:38:56.450 INFO:tasks.rados.rados.0.smithi136.stdout:220: done (0 left) 2024-09-12T23:38:56.450 INFO:tasks.rados.rados.0.smithi136.stdout:221: snap_create 2024-09-12T23:38:56.505 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:38:56 smithi136 bash[22852]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:38:56] "GET /metrics HTTP/1.1" 200 239875 "" "Prometheus/2.43.0" 2024-09-12T23:38:57.453 INFO:tasks.rados.rados.0.smithi136.stdout:221: done (0 left) 2024-09-12T23:38:57.453 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 17 current snap is 20 2024-09-12T23:38:57.453 INFO:tasks.rados.rados.0.smithi136.stdout:222: seq_num 95 ranges {270336=794624,1064960=409600,1474560=393216} 2024-09-12T23:38:57.467 INFO:tasks.rados.rados.0.smithi136.stdout:222: writing smithi13667081-17 from 270336 to 1064960 tid 1 2024-09-12T23:38:57.474 INFO:tasks.rados.rados.0.smithi136.stdout:222: writing smithi13667081-17 from 1064960 to 1474560 tid 2 2024-09-12T23:38:57.481 INFO:tasks.rados.rados.0.smithi136.stdout:222: writing smithi13667081-17 from 1474560 to 1867776 tid 3 2024-09-12T23:38:57.481 INFO:tasks.rados.rados.0.smithi136.stdout:223: rollback oid 8 current snap is 20 2024-09-12T23:38:57.481 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 8 to 17 2024-09-12T23:38:57.482 INFO:tasks.rados.rados.0.smithi136.stdout:222: finishing write tid 1 to smithi13667081-17 2024-09-12T23:38:57.482 INFO:tasks.rados.rados.0.smithi136.stdout:224: rmattr oid 26 current snap is 20 2024-09-12T23:38:57.482 INFO:tasks.rados.rados.0.smithi136.stdout:224: done (2 left) 2024-09-12T23:38:57.482 INFO:tasks.rados.rados.0.smithi136.stdout:225: delete oid 2 current snap is 20 2024-09-12T23:38:57.487 INFO:tasks.rados.rados.0.smithi136.stdout:222: finishing write tid 2 to smithi13667081-17 2024-09-12T23:38:57.489 INFO:tasks.rados.rados.0.smithi136.stdout:223: finishing rollback tid 1 to smithi13667081-8 2024-09-12T23:38:57.489 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 8 v 30 (ObjNum 129 snap 0 seq_num 4242603448) dirty dne 2024-09-12T23:38:57.489 INFO:tasks.rados.rados.0.smithi136.stdout:223: done (2 left) 2024-09-12T23:38:57.489 INFO:tasks.rados.rados.0.smithi136.stdout:225: done (1 left) 2024-09-12T23:38:57.489 INFO:tasks.rados.rados.0.smithi136.stdout:226: rmattr oid 10 current snap is 20 2024-09-12T23:38:57.489 INFO:tasks.rados.rados.0.smithi136.stdout:226: done (1 left) 2024-09-12T23:38:57.489 INFO:tasks.rados.rados.0.smithi136.stdout:227: snap_create 2024-09-12T23:38:57.489 INFO:tasks.rados.rados.0.smithi136.stdout:222: finishing write tid 3 to smithi13667081-17 2024-09-12T23:38:57.490 INFO:tasks.rados.rados.0.smithi136.stdout:222: finishing write tid 4 to smithi13667081-17 2024-09-12T23:38:57.492 INFO:tasks.rados.rados.0.smithi136.stdout:222: finishing write tid 5 to smithi13667081-17 2024-09-12T23:38:57.492 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 17 v 64 (ObjNum 95 snap 20 seq_num 95) dirty exists 2024-09-12T23:38:57.492 INFO:tasks.rados.rados.0.smithi136.stdout:222: left oid 17 (ObjNum 95 snap 20 seq_num 95) 2024-09-12T23:38:57.755 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:57 smithi136 bash[32189]: cluster 2024-09-12T23:38:56.025973+0000 mgr.y (mgr.24446) 205 : cluster [DBG] pgmap v191: 121 pgs: 121 active+clean; 45 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.4 MiB/s wr, 26 op/s 2024-09-12T23:38:57.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:57 smithi136 bash[32189]: cluster 2024-09-12T23:38:56.453223+0000 mon.a (mon.0) 1123 : cluster [DBG] osdmap e100: 8 total, 8 up, 8 in 2024-09-12T23:38:57.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:57 smithi136 bash[22601]: cluster 2024-09-12T23:38:56.025973+0000 mgr.y (mgr.24446) 205 : cluster [DBG] pgmap v191: 121 pgs: 121 active+clean; 45 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.4 MiB/s wr, 26 op/s 2024-09-12T23:38:57.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:57 smithi136 bash[22601]: cluster 2024-09-12T23:38:56.453223+0000 mon.a (mon.0) 1123 : cluster [DBG] osdmap e100: 8 total, 8 up, 8 in 2024-09-12T23:38:57.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:57 smithi181 bash[24776]: cluster 2024-09-12T23:38:56.025973+0000 mgr.y (mgr.24446) 205 : cluster [DBG] pgmap v191: 121 pgs: 121 active+clean; 45 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.4 MiB/s wr, 26 op/s 2024-09-12T23:38:57.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:57 smithi181 bash[24776]: cluster 2024-09-12T23:38:56.453223+0000 mon.a (mon.0) 1123 : cluster [DBG] osdmap e100: 8 total, 8 up, 8 in 2024-09-12T23:38:58.469 INFO:tasks.rados.rados.0.smithi136.stdout:222: done (1 left) 2024-09-12T23:38:58.469 INFO:tasks.rados.rados.0.smithi136.stdout:227: done (0 left) 2024-09-12T23:38:58.469 INFO:tasks.rados.rados.0.smithi136.stdout:228: copy_from oid 48 from oid 47 current snap is 21 2024-09-12T23:38:58.469 INFO:tasks.rados.rados.0.smithi136.stdout:229: read oid 34 snap -1 2024-09-12T23:38:58.469 INFO:tasks.rados.rados.0.smithi136.stdout:229: expect (ObjNum 33 snap 0 seq_num 33) 2024-09-12T23:38:58.469 INFO:tasks.rados.rados.0.smithi136.stdout:230: snap_create 2024-09-12T23:38:58.477 INFO:tasks.rados.rados.0.smithi136.stdout:229: expect (ObjNum 33 snap 0 seq_num 33) 2024-09-12T23:38:58.507 INFO:tasks.rados.rados.0.smithi136.stdout:228: finishing copy_from racing read to smithi13667081-48 2024-09-12T23:38:58.507 INFO:tasks.rados.rados.0.smithi136.stdout:228: finishing copy_from to smithi13667081-48 2024-09-12T23:38:58.507 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 48 v 25 (ObjNum 74 snap 12 seq_num 74) dirty exists 2024-09-12T23:38:58.701 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:58 smithi136 bash[22601]: cluster 2024-09-12T23:38:57.456346+0000 mon.a (mon.0) 1124 : cluster [DBG] osdmap e101: 8 total, 8 up, 8 in 2024-09-12T23:38:58.701 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:58 smithi136 bash[32189]: cluster 2024-09-12T23:38:57.456346+0000 mon.a (mon.0) 1124 : cluster [DBG] osdmap e101: 8 total, 8 up, 8 in 2024-09-12T23:38:58.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:58 smithi181 bash[24776]: cluster 2024-09-12T23:38:57.456346+0000 mon.a (mon.0) 1124 : cluster [DBG] osdmap e101: 8 total, 8 up, 8 in 2024-09-12T23:38:59.479 INFO:tasks.rados.rados.0.smithi136.stdout:228: done (2 left) 2024-09-12T23:38:59.479 INFO:tasks.rados.rados.0.smithi136.stdout:229: done (1 left) 2024-09-12T23:38:59.479 INFO:tasks.rados.rados.0.smithi136.stdout:230: done (0 left) 2024-09-12T23:38:59.479 INFO:tasks.rados.rados.0.smithi136.stdout:231: read oid 47 snap -1 2024-09-12T23:38:59.479 INFO:tasks.rados.rados.0.smithi136.stdout:231: expect (ObjNum 74 snap 12 seq_num 74) 2024-09-12T23:38:59.479 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 45 current snap is 22 2024-09-12T23:38:59.479 INFO:tasks.rados.rados.0.smithi136.stdout:232: seq_num 97 ranges {557056=434176,991232=581632,1572864=540672,2113536=81920} 2024-09-12T23:38:59.495 INFO:tasks.rados.rados.0.smithi136.stdout:232: writing smithi13667081-45 from 557056 to 991232 tid 1 2024-09-12T23:38:59.505 INFO:tasks.rados.rados.0.smithi136.stdout:232: writing smithi13667081-45 from 991232 to 1572864 tid 2 2024-09-12T23:38:59.513 INFO:tasks.rados.rados.0.smithi136.stdout:232: writing smithi13667081-45 from 1572864 to 2113536 tid 3 2024-09-12T23:38:59.515 INFO:tasks.rados.rados.0.smithi136.stdout:232: writing smithi13667081-45 from 2113536 to 2195456 tid 4 2024-09-12T23:38:59.515 INFO:tasks.rados.rados.0.smithi136.stdout:233: read oid 49 snap 16 2024-09-12T23:38:59.515 INFO:tasks.rados.rados.0.smithi136.stdout:233: expect (ObjNum 55 snap 4 seq_num 55) 2024-09-12T23:38:59.515 INFO:tasks.rados.rados.0.smithi136.stdout:234: read oid 28 snap -1 2024-09-12T23:38:59.516 INFO:tasks.rados.rados.0.smithi136.stdout:234: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-12T23:38:59.516 INFO:tasks.rados.rados.0.smithi136.stdout:235: read oid 11 snap -1 2024-09-12T23:38:59.516 INFO:tasks.rados.rados.0.smithi136.stdout:235: expect deleted 2024-09-12T23:38:59.516 INFO:tasks.rados.rados.0.smithi136.stdout:236: read oid 5 snap 13 2024-09-12T23:38:59.516 INFO:tasks.rados.rados.0.smithi136.stdout:236: expect (ObjNum 4 snap 0 seq_num 4) 2024-09-12T23:38:59.516 INFO:tasks.rados.rados.0.smithi136.stdout:237: setattr oid 48 current snap is 22 2024-09-12T23:38:59.516 INFO:tasks.rados.rados.0.smithi136.stdout:231: expect (ObjNum 74 snap 12 seq_num 74) 2024-09-12T23:38:59.635 INFO:tasks.rados.rados.0.smithi136.stdout:232: finishing write tid 1 to smithi13667081-45 2024-09-12T23:38:59.635 INFO:tasks.rados.rados.0.smithi136.stdout:232: finishing write tid 2 to smithi13667081-45 2024-09-12T23:38:59.635 INFO:tasks.rados.rados.0.smithi136.stdout:234: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-12T23:38:59.637 INFO:tasks.rados.rados.0.smithi136.stdout:232: finishing write tid 3 to smithi13667081-45 2024-09-12T23:38:59.638 INFO:tasks.rados.rados.0.smithi136.stdout:232: finishing write tid 4 to smithi13667081-45 2024-09-12T23:38:59.638 INFO:tasks.rados.rados.0.smithi136.stdout:232: finishing write tid 5 to smithi13667081-45 2024-09-12T23:38:59.638 INFO:tasks.rados.rados.0.smithi136.stdout:232: finishing write tid 6 to smithi13667081-45 2024-09-12T23:38:59.638 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 45 v 45 (ObjNum 97 snap 22 seq_num 97) dirty exists 2024-09-12T23:38:59.638 INFO:tasks.rados.rados.0.smithi136.stdout:232: left oid 45 (ObjNum 97 snap 22 seq_num 97) 2024-09-12T23:38:59.638 INFO:tasks.rados.rados.0.smithi136.stdout:236: expect (ObjNum 4 snap 0 seq_num 4) 2024-09-12T23:38:59.639 INFO:tasks.rados.rados.0.smithi136.stdout:233: expect (ObjNum 55 snap 4 seq_num 55) 2024-09-12T23:38:59.670 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:59 smithi181 bash[24776]: cluster 2024-09-12T23:38:58.026669+0000 mgr.y (mgr.24446) 206 : cluster [DBG] pgmap v194: 121 pgs: 121 active+clean; 45 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:38:59.670 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:59 smithi181 bash[24776]: cluster 2024-09-12T23:38:58.473521+0000 mon.a (mon.0) 1125 : cluster [DBG] osdmap e102: 8 total, 8 up, 8 in 2024-09-12T23:38:59.670 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:59 smithi181 bash[24776]: audit 2024-09-12T23:38:59.142055+0000 mon.a (mon.0) 1126 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:38:59.671 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:59 smithi181 bash[24776]: audit 2024-09-12T23:38:59.144520+0000 mon.a (mon.0) 1127 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:38:59.671 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:38:59 smithi181 bash[24776]: audit 2024-09-12T23:38:59.153952+0000 mon.a (mon.0) 1128 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:38:59.701 INFO:tasks.rados.rados.0.smithi136.stdout:231: done (6 left) 2024-09-12T23:38:59.701 INFO:tasks.rados.rados.0.smithi136.stdout:232: done (5 left) 2024-09-12T23:38:59.701 INFO:tasks.rados.rados.0.smithi136.stdout:233: done (4 left) 2024-09-12T23:38:59.701 INFO:tasks.rados.rados.0.smithi136.stdout:234: done (3 left) 2024-09-12T23:38:59.701 INFO:tasks.rados.rados.0.smithi136.stdout:235: done (2 left) 2024-09-12T23:38:59.701 INFO:tasks.rados.rados.0.smithi136.stdout:236: done (1 left) 2024-09-12T23:38:59.702 INFO:tasks.rados.rados.0.smithi136.stdout:238: rollback oid 15 current snap is 22 2024-09-12T23:38:59.702 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 15 to 21 2024-09-12T23:38:59.702 INFO:tasks.rados.rados.0.smithi136.stdout:239: delete oid 44 current snap is 22 2024-09-12T23:38:59.702 INFO:tasks.rados.rados.0.smithi136.stdout:238: finishing rollback tid 1 to smithi13667081-15 2024-09-12T23:38:59.702 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 15 v 97 (ObjNum 97 snap 0 seq_num 4243134256) dirty dne 2024-09-12T23:38:59.703 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 48 v 83 (ObjNum 74 snap 12 seq_num 74) dirty exists 2024-09-12T23:38:59.703 INFO:tasks.rados.rados.0.smithi136.stdout:237: done (2 left) 2024-09-12T23:38:59.703 INFO:tasks.rados.rados.0.smithi136.stdout:238: done (1 left) 2024-09-12T23:38:59.703 INFO:tasks.rados.rados.0.smithi136.stdout:239: done (0 left) 2024-09-12T23:38:59.703 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 26 current snap is 22 2024-09-12T23:38:59.703 INFO:tasks.rados.rados.0.smithi136.stdout:240: seq_num 99 ranges {0=466944,466944=802816,1269760=401408} 2024-09-12T23:38:59.705 INFO:tasks.rados.rados.0.smithi136.stdout:240: writing smithi13667081-26 from 0 to 466944 tid 1 2024-09-12T23:38:59.710 INFO:tasks.rados.rados.0.smithi136.stdout:240: writing smithi13667081-26 from 466944 to 1269760 tid 2 2024-09-12T23:38:59.712 INFO:tasks.rados.rados.0.smithi136.stdout:240: writing smithi13667081-26 from 1269760 to 1671168 tid 3 2024-09-12T23:38:59.713 INFO:tasks.rados.rados.0.smithi136.stdout:241: setattr oid 47 current snap is 22 2024-09-12T23:38:59.713 INFO:tasks.rados.rados.0.smithi136.stdout:240: finishing write tid 1 to smithi13667081-26 2024-09-12T23:38:59.713 INFO:tasks.rados.rados.0.smithi136.stdout:242: setattr oid 30 current snap is 22 2024-09-12T23:38:59.715 INFO:tasks.rados.rados.0.smithi136.stdout:243: snap_remove snap 12 2024-09-12T23:38:59.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:59 smithi136 bash[22601]: cluster 2024-09-12T23:38:58.026669+0000 mgr.y (mgr.24446) 206 : cluster [DBG] pgmap v194: 121 pgs: 121 active+clean; 45 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:38:59.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:59 smithi136 bash[22601]: cluster 2024-09-12T23:38:58.473521+0000 mon.a (mon.0) 1125 : cluster [DBG] osdmap e102: 8 total, 8 up, 8 in 2024-09-12T23:38:59.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:59 smithi136 bash[22601]: audit 2024-09-12T23:38:59.142055+0000 mon.a (mon.0) 1126 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:38:59.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:59 smithi136 bash[22601]: audit 2024-09-12T23:38:59.144520+0000 mon.a (mon.0) 1127 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:38:59.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:38:59 smithi136 bash[22601]: audit 2024-09-12T23:38:59.153952+0000 mon.a (mon.0) 1128 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:38:59.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:59 smithi136 bash[32189]: cluster 2024-09-12T23:38:58.026669+0000 mgr.y (mgr.24446) 206 : cluster [DBG] pgmap v194: 121 pgs: 121 active+clean; 45 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:38:59.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:59 smithi136 bash[32189]: cluster 2024-09-12T23:38:58.473521+0000 mon.a (mon.0) 1125 : cluster [DBG] osdmap e102: 8 total, 8 up, 8 in 2024-09-12T23:38:59.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:59 smithi136 bash[32189]: audit 2024-09-12T23:38:59.142055+0000 mon.a (mon.0) 1126 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:38:59.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:59 smithi136 bash[32189]: audit 2024-09-12T23:38:59.144520+0000 mon.a (mon.0) 1127 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:38:59.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:38:59 smithi136 bash[32189]: audit 2024-09-12T23:38:59.153952+0000 mon.a (mon.0) 1128 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:39:00.491 INFO:tasks.rados.rados.0.smithi136.stdout:243: done (3 left) 2024-09-12T23:39:00.491 INFO:tasks.rados.rados.0.smithi136.stdout:244: read oid 49 snap -1 2024-09-12T23:39:00.491 INFO:tasks.rados.rados.0.smithi136.stdout:244: expect (ObjNum 90 snap 18 seq_num 90) 2024-09-12T23:39:00.491 INFO:tasks.rados.rados.0.smithi136.stdout:245: read oid 42 snap -1 2024-09-12T23:39:00.492 INFO:tasks.rados.rados.0.smithi136.stdout:245: expect deleted 2024-09-12T23:39:00.492 INFO:tasks.rados.rados.0.smithi136.stdout:246: read oid 33 snap -1 2024-09-12T23:39:00.492 INFO:tasks.rados.rados.0.smithi136.stdout:246: expect (ObjNum 15 snap 0 seq_num 15) 2024-09-12T23:39:00.492 INFO:tasks.rados.rados.0.smithi136.stdout:247: snap_remove snap 16 2024-09-12T23:39:00.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:00 smithi136 bash[22601]: cephadm 2024-09-12T23:38:59.142769+0000 mgr.y (mgr.24446) 207 : cephadm [INF] Upgrade: Target is version 18.2.4-837-g7a853b17 (unknown) 2024-09-12T23:39:00.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:00 smithi136 bash[22601]: cephadm 2024-09-12T23:38:59.142887+0000 mgr.y (mgr.24446) 208 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:ef14fa6264e59c23a50911ff257310d925626ef2ed8aade654f62752810262e6, digests ['quay.ceph.io/ceph-ci/ceph@sha256:ef14fa6264e59c23a50911ff257310d925626ef2ed8aade654f62752810262e6'] 2024-09-12T23:39:00.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:00 smithi136 bash[22601]: cephadm 2024-09-12T23:38:59.155273+0000 mgr.y (mgr.24446) 209 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2024-09-12T23:39:00.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:00 smithi136 bash[22601]: cephadm 2024-09-12T23:38:59.407434+0000 mgr.y (mgr.24446) 210 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:ef14fa6264e59c23a50911ff257310d925626ef2ed8aade654f62752810262e6 on smithi181 2024-09-12T23:39:00.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:00 smithi136 bash[22601]: cluster 2024-09-12T23:38:59.482818+0000 mon.a (mon.0) 1129 : cluster [DBG] osdmap e103: 8 total, 8 up, 8 in 2024-09-12T23:39:00.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:00 smithi136 bash[32189]: cephadm 2024-09-12T23:38:59.142769+0000 mgr.y (mgr.24446) 207 : cephadm [INF] Upgrade: Target is version 18.2.4-837-g7a853b17 (unknown) 2024-09-12T23:39:00.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:00 smithi136 bash[32189]: cephadm 2024-09-12T23:38:59.142887+0000 mgr.y (mgr.24446) 208 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:ef14fa6264e59c23a50911ff257310d925626ef2ed8aade654f62752810262e6, digests ['quay.ceph.io/ceph-ci/ceph@sha256:ef14fa6264e59c23a50911ff257310d925626ef2ed8aade654f62752810262e6'] 2024-09-12T23:39:00.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:00 smithi136 bash[32189]: cephadm 2024-09-12T23:38:59.155273+0000 mgr.y (mgr.24446) 209 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2024-09-12T23:39:00.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:00 smithi136 bash[32189]: cephadm 2024-09-12T23:38:59.407434+0000 mgr.y (mgr.24446) 210 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:ef14fa6264e59c23a50911ff257310d925626ef2ed8aade654f62752810262e6 on smithi181 2024-09-12T23:39:00.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:00 smithi136 bash[32189]: cluster 2024-09-12T23:38:59.482818+0000 mon.a (mon.0) 1129 : cluster [DBG] osdmap e103: 8 total, 8 up, 8 in 2024-09-12T23:39:00.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:00 smithi181 bash[24776]: cephadm 2024-09-12T23:38:59.142769+0000 mgr.y (mgr.24446) 207 : cephadm [INF] Upgrade: Target is version 18.2.4-837-g7a853b17 (unknown) 2024-09-12T23:39:00.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:00 smithi181 bash[24776]: cephadm 2024-09-12T23:38:59.142887+0000 mgr.y (mgr.24446) 208 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:ef14fa6264e59c23a50911ff257310d925626ef2ed8aade654f62752810262e6, digests ['quay.ceph.io/ceph-ci/ceph@sha256:ef14fa6264e59c23a50911ff257310d925626ef2ed8aade654f62752810262e6'] 2024-09-12T23:39:00.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:00 smithi181 bash[24776]: cephadm 2024-09-12T23:38:59.155273+0000 mgr.y (mgr.24446) 209 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2024-09-12T23:39:00.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:00 smithi181 bash[24776]: cephadm 2024-09-12T23:38:59.407434+0000 mgr.y (mgr.24446) 210 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:ef14fa6264e59c23a50911ff257310d925626ef2ed8aade654f62752810262e6 on smithi181 2024-09-12T23:39:00.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:00 smithi181 bash[24776]: cluster 2024-09-12T23:38:59.482818+0000 mon.a (mon.0) 1129 : cluster [DBG] osdmap e103: 8 total, 8 up, 8 in 2024-09-12T23:39:01.492 INFO:tasks.rados.rados.0.smithi136.stdout:247: done (6 left) 2024-09-12T23:39:01.492 INFO:tasks.rados.rados.0.smithi136.stdout:248: copy_from oid 15 from oid 40 current snap is 22 2024-09-12T23:39:01.493 INFO:tasks.rados.rados.0.smithi136.stdout:249: read oid 45 snap -1 2024-09-12T23:39:01.493 INFO:tasks.rados.rados.0.smithi136.stdout:249: expect (ObjNum 97 snap 22 seq_num 97) 2024-09-12T23:39:01.493 INFO:tasks.rados.rados.0.smithi136.stdout:250: snap_remove snap 15 2024-09-12T23:39:01.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:01 smithi136 bash[22601]: cluster 2024-09-12T23:39:00.027338+0000 mgr.y (mgr.24446) 211 : cluster [DBG] pgmap v197: 121 pgs: 121 active+clean; 55 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.8 MiB/s rd, 2.7 MiB/s wr, 19 op/s 2024-09-12T23:39:01.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:01 smithi136 bash[22601]: cluster 2024-09-12T23:39:00.494875+0000 mon.a (mon.0) 1130 : cluster [DBG] osdmap e104: 8 total, 8 up, 8 in 2024-09-12T23:39:01.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:01 smithi136 bash[32189]: cluster 2024-09-12T23:39:00.027338+0000 mgr.y (mgr.24446) 211 : cluster [DBG] pgmap v197: 121 pgs: 121 active+clean; 55 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.8 MiB/s rd, 2.7 MiB/s wr, 19 op/s 2024-09-12T23:39:01.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:01 smithi136 bash[32189]: cluster 2024-09-12T23:39:00.494875+0000 mon.a (mon.0) 1130 : cluster [DBG] osdmap e104: 8 total, 8 up, 8 in 2024-09-12T23:39:01.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:01 smithi181 bash[24776]: cluster 2024-09-12T23:39:00.027338+0000 mgr.y (mgr.24446) 211 : cluster [DBG] pgmap v197: 121 pgs: 121 active+clean; 55 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.8 MiB/s rd, 2.7 MiB/s wr, 19 op/s 2024-09-12T23:39:01.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:01 smithi181 bash[24776]: cluster 2024-09-12T23:39:00.494875+0000 mon.a (mon.0) 1130 : cluster [DBG] osdmap e104: 8 total, 8 up, 8 in 2024-09-12T23:39:02.495 INFO:tasks.rados.rados.0.smithi136.stdout:250: done (8 left) 2024-09-12T23:39:02.496 INFO:tasks.rados.rados.0.smithi136.stdout:251: read oid 11 snap -1 2024-09-12T23:39:02.496 INFO:tasks.rados.rados.0.smithi136.stdout:251: expect deleted 2024-09-12T23:39:02.496 INFO:tasks.rados.rados.0.smithi136.stdout:252: delete oid 21 current snap is 22 2024-09-12T23:39:02.496 INFO:tasks.rados.rados.0.smithi136.stdout:240: finishing write tid 2 to smithi13667081-26 2024-09-12T23:39:02.496 INFO:tasks.rados.rados.0.smithi136.stdout:240: finishing write tid 3 to smithi13667081-26 2024-09-12T23:39:02.496 INFO:tasks.rados.rados.0.smithi136.stdout:240: finishing write tid 4 to smithi13667081-26 2024-09-12T23:39:02.496 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 47 v 49 (ObjNum 74 snap 12 seq_num 74) dirty exists 2024-09-12T23:39:02.496 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 30 v 29 (ObjNum 29 snap 0 seq_num 29) dirty exists 2024-09-12T23:39:02.497 INFO:tasks.rados.rados.0.smithi136.stdout:240: finishing write tid 5 to smithi13667081-26 2024-09-12T23:39:02.497 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 26 v 35 (ObjNum 99 snap 22 seq_num 99) dirty exists 2024-09-12T23:39:02.497 INFO:tasks.rados.rados.0.smithi136.stdout:240: left oid 26 (ObjNum 99 snap 22 seq_num 99) 2024-09-12T23:39:02.497 INFO:tasks.rados.rados.0.smithi136.stdout:246: expect (ObjNum 15 snap 0 seq_num 15) 2024-09-12T23:39:02.507 INFO:tasks.rados.rados.0.smithi136.stdout:244: expect (ObjNum 90 snap 18 seq_num 90) 2024-09-12T23:39:02.653 INFO:tasks.rados.rados.0.smithi136.stdout:248: finishing copy_from racing read to smithi13667081-15 2024-09-12T23:39:02.654 INFO:tasks.rados.rados.0.smithi136.stdout:248: finishing copy_from to smithi13667081-15 2024-09-12T23:39:02.654 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 15 v 23 (ObjNum 39 snap 0 seq_num 39) dirty exists 2024-09-12T23:39:02.654 INFO:tasks.rados.rados.0.smithi136.stdout:249: expect (ObjNum 97 snap 22 seq_num 97) 2024-09-12T23:39:02.718 INFO:tasks.rados.rados.0.smithi136.stdout:240: done (9 left) 2024-09-12T23:39:02.718 INFO:tasks.rados.rados.0.smithi136.stdout:241: done (8 left) 2024-09-12T23:39:02.719 INFO:tasks.rados.rados.0.smithi136.stdout:242: done (7 left) 2024-09-12T23:39:02.719 INFO:tasks.rados.rados.0.smithi136.stdout:244: done (6 left) 2024-09-12T23:39:02.719 INFO:tasks.rados.rados.0.smithi136.stdout:245: done (5 left) 2024-09-12T23:39:02.719 INFO:tasks.rados.rados.0.smithi136.stdout:246: done (4 left) 2024-09-12T23:39:02.720 INFO:tasks.rados.rados.0.smithi136.stdout:248: done (3 left) 2024-09-12T23:39:02.720 INFO:tasks.rados.rados.0.smithi136.stdout:249: done (2 left) 2024-09-12T23:39:02.720 INFO:tasks.rados.rados.0.smithi136.stdout:251: done (1 left) 2024-09-12T23:39:02.720 INFO:tasks.rados.rados.0.smithi136.stdout:252: done (0 left) 2024-09-12T23:39:02.720 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 30 current snap is 22 2024-09-12T23:39:02.720 INFO:tasks.rados.rados.0.smithi136.stdout:253: seq_num 101 ranges {491520=499712,991232=573440,1564672=524288,2088960=106496} 2024-09-12T23:39:02.735 INFO:tasks.rados.rados.0.smithi136.stdout:253: writing smithi13667081-30 from 491520 to 991232 tid 1 2024-09-12T23:39:02.745 INFO:tasks.rados.rados.0.smithi136.stdout:253: writing smithi13667081-30 from 991232 to 1564672 tid 2 2024-09-12T23:39:02.754 INFO:tasks.rados.rados.0.smithi136.stdout:253: writing smithi13667081-30 from 1564672 to 2088960 tid 3 2024-09-12T23:39:02.756 INFO:tasks.rados.rados.0.smithi136.stdout:253: writing smithi13667081-30 from 2088960 to 2195456 tid 4 2024-09-12T23:39:02.756 INFO:tasks.rados.rados.0.smithi136.stdout:254: rmattr oid 17 current snap is 22 2024-09-12T23:39:02.756 INFO:tasks.rados.rados.0.smithi136.stdout:254: done (1 left) 2024-09-12T23:39:02.756 INFO:tasks.rados.rados.0.smithi136.stdout:255: snap_remove snap 9 2024-09-12T23:39:02.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:02 smithi181 bash[24776]: cluster 2024-09-12T23:39:01.495368+0000 mon.a (mon.0) 1131 : cluster [DBG] osdmap e105: 8 total, 8 up, 8 in 2024-09-12T23:39:02.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:02 smithi181 bash[24776]: cluster 2024-09-12T23:39:02.498841+0000 mon.a (mon.0) 1132 : cluster [DBG] osdmap e106: 8 total, 8 up, 8 in 2024-09-12T23:39:03.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:02 smithi136 bash[22601]: cluster 2024-09-12T23:39:01.495368+0000 mon.a (mon.0) 1131 : cluster [DBG] osdmap e105: 8 total, 8 up, 8 in 2024-09-12T23:39:03.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:02 smithi136 bash[22601]: cluster 2024-09-12T23:39:02.498841+0000 mon.a (mon.0) 1132 : cluster [DBG] osdmap e106: 8 total, 8 up, 8 in 2024-09-12T23:39:03.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:02 smithi136 bash[32189]: cluster 2024-09-12T23:39:01.495368+0000 mon.a (mon.0) 1131 : cluster [DBG] osdmap e105: 8 total, 8 up, 8 in 2024-09-12T23:39:03.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:02 smithi136 bash[32189]: cluster 2024-09-12T23:39:02.498841+0000 mon.a (mon.0) 1132 : cluster [DBG] osdmap e106: 8 total, 8 up, 8 in 2024-09-12T23:39:03.533 INFO:tasks.rados.rados.0.smithi136.stdout:255: done (1 left) 2024-09-12T23:39:03.533 INFO:tasks.rados.rados.0.smithi136.stdout:256: snap_remove snap 18 2024-09-12T23:39:03.533 INFO:tasks.rados.rados.0.smithi136.stdout:253: finishing write tid 1 to smithi13667081-30 2024-09-12T23:39:03.533 INFO:tasks.rados.rados.0.smithi136.stdout:253: finishing write tid 2 to smithi13667081-30 2024-09-12T23:39:03.533 INFO:tasks.rados.rados.0.smithi136.stdout:253: finishing write tid 3 to smithi13667081-30 2024-09-12T23:39:03.533 INFO:tasks.rados.rados.0.smithi136.stdout:253: finishing write tid 4 to smithi13667081-30 2024-09-12T23:39:03.533 INFO:tasks.rados.rados.0.smithi136.stdout:253: finishing write tid 5 to smithi13667081-30 2024-09-12T23:39:03.533 INFO:tasks.rados.rados.0.smithi136.stdout:253: finishing write tid 6 to smithi13667081-30 2024-09-12T23:39:03.533 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 30 v 42 (ObjNum 101 snap 22 seq_num 101) dirty exists 2024-09-12T23:39:03.533 INFO:tasks.rados.rados.0.smithi136.stdout:253: left oid 30 (ObjNum 101 snap 22 seq_num 101) 2024-09-12T23:39:03.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:03 smithi181 bash[24776]: cluster 2024-09-12T23:39:02.028084+0000 mgr.y (mgr.24446) 212 : cluster [DBG] pgmap v200: 121 pgs: 121 active+clean; 55 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.8 MiB/s rd, 2.7 MiB/s wr, 19 op/s 2024-09-12T23:39:04.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:03 smithi136 bash[22601]: cluster 2024-09-12T23:39:02.028084+0000 mgr.y (mgr.24446) 212 : cluster [DBG] pgmap v200: 121 pgs: 121 active+clean; 55 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.8 MiB/s rd, 2.7 MiB/s wr, 19 op/s 2024-09-12T23:39:04.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:03 smithi136 bash[32189]: cluster 2024-09-12T23:39:02.028084+0000 mgr.y (mgr.24446) 212 : cluster [DBG] pgmap v200: 121 pgs: 121 active+clean; 55 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.8 MiB/s rd, 2.7 MiB/s wr, 19 op/s 2024-09-12T23:39:04.542 INFO:tasks.rados.rados.0.smithi136.stdout:253: done (1 left) 2024-09-12T23:39:04.542 INFO:tasks.rados.rados.0.smithi136.stdout:256: done (0 left) 2024-09-12T23:39:04.542 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 14 current snap is 22 2024-09-12T23:39:04.543 INFO:tasks.rados.rados.0.smithi136.stdout:257: seq_num 102 ranges {221184=516096,737280=450560,1187840=729088,1916928=24576} 2024-09-12T23:39:04.546 INFO:tasks.rados.rados.0.smithi136.stdout:257: writing smithi13667081-14 from 221184 to 737280 tid 1 2024-09-12T23:39:04.549 INFO:tasks.rados.rados.0.smithi136.stdout:257: writing smithi13667081-14 from 737280 to 1187840 tid 2 2024-09-12T23:39:04.554 INFO:tasks.rados.rados.0.smithi136.stdout:257: writing smithi13667081-14 from 1187840 to 1916928 tid 3 2024-09-12T23:39:04.555 INFO:tasks.rados.rados.0.smithi136.stdout:257: writing smithi13667081-14 from 1916928 to 1941504 tid 4 2024-09-12T23:39:04.555 INFO:tasks.rados.rados.0.smithi136.stdout:258: read oid 3 snap -1 2024-09-12T23:39:04.555 INFO:tasks.rados.rados.0.smithi136.stdout:258: expect (ObjNum 6 snap 0 seq_num 6) 2024-09-12T23:39:04.555 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 20 current snap is 22 2024-09-12T23:39:04.555 INFO:tasks.rados.rados.0.smithi136.stdout:259: seq_num 103 ranges {0=532480,532480=737280,1269760=466944} 2024-09-12T23:39:04.558 INFO:tasks.rados.rados.0.smithi136.stdout:259: writing smithi13667081-20 from 0 to 532480 tid 1 2024-09-12T23:39:04.563 INFO:tasks.rados.rados.0.smithi136.stdout:259: writing smithi13667081-20 from 532480 to 1269760 tid 2 2024-09-12T23:39:04.567 INFO:tasks.rados.rados.0.smithi136.stdout:259: writing smithi13667081-20 from 1269760 to 1736704 tid 3 2024-09-12T23:39:04.567 INFO:tasks.rados.rados.0.smithi136.stdout:260: snap_create 2024-09-12T23:39:04.567 INFO:tasks.rados.rados.0.smithi136.stdout:258: expect (ObjNum 6 snap 0 seq_num 6) 2024-09-12T23:39:04.581 INFO:tasks.rados.rados.0.smithi136.stdout:259: finishing write tid 1 to smithi13667081-20 2024-09-12T23:39:04.581 INFO:tasks.rados.rados.0.smithi136.stdout:257: finishing write tid 1 to smithi13667081-14 2024-09-12T23:39:04.581 INFO:tasks.rados.rados.0.smithi136.stdout:257: finishing write tid 2 to smithi13667081-14 2024-09-12T23:39:04.581 INFO:tasks.rados.rados.0.smithi136.stdout:257: finishing write tid 3 to smithi13667081-14 2024-09-12T23:39:04.581 INFO:tasks.rados.rados.0.smithi136.stdout:257: finishing write tid 4 to smithi13667081-14 2024-09-12T23:39:04.582 INFO:tasks.rados.rados.0.smithi136.stdout:259: finishing write tid 2 to smithi13667081-20 2024-09-12T23:39:04.582 INFO:tasks.rados.rados.0.smithi136.stdout:259: finishing write tid 3 to smithi13667081-20 2024-09-12T23:39:04.582 INFO:tasks.rados.rados.0.smithi136.stdout:257: finishing write tid 5 to smithi13667081-14 2024-09-12T23:39:04.583 INFO:tasks.rados.rados.0.smithi136.stdout:259: finishing write tid 4 to smithi13667081-20 2024-09-12T23:39:04.583 INFO:tasks.rados.rados.0.smithi136.stdout:257: finishing write tid 6 to smithi13667081-14 2024-09-12T23:39:04.584 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 14 v 102 (ObjNum 102 snap 22 seq_num 102) dirty exists 2024-09-12T23:39:04.584 INFO:tasks.rados.rados.0.smithi136.stdout:257: left oid 14 (ObjNum 102 snap 22 seq_num 102) 2024-09-12T23:39:04.587 INFO:tasks.rados.rados.0.smithi136.stdout:259: finishing write tid 5 to smithi13667081-20 2024-09-12T23:39:04.587 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 20 v 23 (ObjNum 103 snap 22 seq_num 103) dirty exists 2024-09-12T23:39:04.587 INFO:tasks.rados.rados.0.smithi136.stdout:259: left oid 20 (ObjNum 103 snap 22 seq_num 103) 2024-09-12T23:39:04.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:04 smithi181 bash[24776]: cluster 2024-09-12T23:39:03.535915+0000 mon.a (mon.0) 1133 : cluster [DBG] osdmap e107: 8 total, 8 up, 8 in 2024-09-12T23:39:05.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:04 smithi136 bash[22601]: cluster 2024-09-12T23:39:03.535915+0000 mon.a (mon.0) 1133 : cluster [DBG] osdmap e107: 8 total, 8 up, 8 in 2024-09-12T23:39:05.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:04 smithi136 bash[32189]: cluster 2024-09-12T23:39:03.535915+0000 mon.a (mon.0) 1133 : cluster [DBG] osdmap e107: 8 total, 8 up, 8 in 2024-09-12T23:39:05.450 INFO:tasks.rados.rados.0.smithi136.stdout:257: done (3 left) 2024-09-12T23:39:05.450 INFO:tasks.rados.rados.0.smithi136.stdout:258: done (2 left) 2024-09-12T23:39:05.450 INFO:tasks.rados.rados.0.smithi136.stdout:259: done (1 left) 2024-09-12T23:39:05.451 INFO:tasks.rados.rados.0.smithi136.stdout:260: done (0 left) 2024-09-12T23:39:05.451 INFO:tasks.rados.rados.0.smithi136.stdout:261: delete oid 41 current snap is 23 2024-09-12T23:39:05.459 INFO:tasks.rados.rados.0.smithi136.stdout:261: done (0 left) 2024-09-12T23:39:05.459 INFO:tasks.rados.rados.0.smithi136.stdout:262: read oid 13 snap -1 2024-09-12T23:39:05.459 INFO:tasks.rados.rados.0.smithi136.stdout:262: expect (ObjNum 87 snap 18 seq_num 87) 2024-09-12T23:39:05.460 INFO:tasks.rados.rados.0.smithi136.stdout:263: setattr oid 26 current snap is 23 2024-09-12T23:39:05.465 INFO:tasks.rados.rados.0.smithi136.stdout:264: copy_from oid 32 from oid 7 current snap is 23 2024-09-12T23:39:05.465 INFO:tasks.rados.rados.0.smithi136.stdout:265: read oid 22 snap -1 2024-09-12T23:39:05.465 INFO:tasks.rados.rados.0.smithi136.stdout:265: expect (ObjNum 21 snap 0 seq_num 21) 2024-09-12T23:39:05.465 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 15 current snap is 23 2024-09-12T23:39:05.465 INFO:tasks.rados.rados.0.smithi136.stdout:266: seq_num 105 ranges {663552=565248,1228800=507904,1736704=507904,2244608=188416} 2024-09-12T23:39:05.478 INFO:tasks.rados.rados.0.smithi136.stdout:266: writing smithi13667081-15 from 663552 to 1228800 tid 1 2024-09-12T23:39:05.484 INFO:tasks.rados.rados.0.smithi136.stdout:266: writing smithi13667081-15 from 1228800 to 1736704 tid 2 2024-09-12T23:39:05.490 INFO:tasks.rados.rados.0.smithi136.stdout:266: writing smithi13667081-15 from 1736704 to 2244608 tid 3 2024-09-12T23:39:05.493 INFO:tasks.rados.rados.0.smithi136.stdout:266: writing smithi13667081-15 from 2244608 to 2433024 tid 4 2024-09-12T23:39:05.494 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 49 current snap is 23 2024-09-12T23:39:05.494 INFO:tasks.rados.rados.0.smithi136.stdout:267: seq_num 106 ranges {3252224=581632,3833856=794624,4628480=409600} 2024-09-12T23:39:05.525 INFO:tasks.rados.rados.0.smithi136.stdout:267: writing smithi13667081-49 from 3252224 to 3833856 tid 1 2024-09-12T23:39:05.531 INFO:tasks.rados.rados.0.smithi136.stdout:267: writing smithi13667081-49 from 3833856 to 4628480 tid 2 2024-09-12T23:39:05.534 INFO:tasks.rados.rados.0.smithi136.stdout:267: writing smithi13667081-49 from 4628480 to 5038080 tid 3 2024-09-12T23:39:05.534 INFO:tasks.rados.rados.0.smithi136.stdout:268: rollback oid 33 current snap is 23 2024-09-12T23:39:05.534 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 33 to 19 2024-09-12T23:39:05.534 INFO:tasks.rados.rados.0.smithi136.stdout:265: expect (ObjNum 21 snap 0 seq_num 21) 2024-09-12T23:39:05.538 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:39:05 smithi181 bash[25717]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:39:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-12T23:39:05.566 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 26 v 45 (ObjNum 99 snap 22 seq_num 99) dirty exists 2024-09-12T23:39:05.566 INFO:tasks.rados.rados.0.smithi136.stdout:266: finishing write tid 1 to smithi13667081-15 2024-09-12T23:39:05.566 INFO:tasks.rados.rados.0.smithi136.stdout:264: finishing copy_from racing read to smithi13667081-32 2024-09-12T23:39:05.566 INFO:tasks.rados.rados.0.smithi136.stdout:266: finishing write tid 2 to smithi13667081-15 2024-09-12T23:39:05.566 INFO:tasks.rados.rados.0.smithi136.stdout:264: finishing copy_from to smithi13667081-32 2024-09-12T23:39:05.566 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 32 v 2 (ObjNum 6 snap 0 seq_num 6) dirty exists 2024-09-12T23:39:05.566 INFO:tasks.rados.rados.0.smithi136.stdout:263: done (6 left) 2024-09-12T23:39:05.566 INFO:tasks.rados.rados.0.smithi136.stdout:264: done (5 left) 2024-09-12T23:39:05.566 INFO:tasks.rados.rados.0.smithi136.stdout:265: done (4 left) 2024-09-12T23:39:05.566 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 2 current snap is 23 2024-09-12T23:39:05.566 INFO:tasks.rados.rados.0.smithi136.stdout:269: seq_num 107 ranges {0=598016,598016=729088,1327104=466944,1794048=8192} 2024-09-12T23:39:05.569 INFO:tasks.rados.rados.0.smithi136.stdout:269: writing smithi13667081-2 from 0 to 598016 tid 1 2024-09-12T23:39:05.574 INFO:tasks.rados.rados.0.smithi136.stdout:269: writing smithi13667081-2 from 598016 to 1327104 tid 2 2024-09-12T23:39:05.578 INFO:tasks.rados.rados.0.smithi136.stdout:269: writing smithi13667081-2 from 1327104 to 1794048 tid 3 2024-09-12T23:39:05.578 INFO:tasks.rados.rados.0.smithi136.stdout:269: writing smithi13667081-2 from 1794048 to 1802240 tid 4 2024-09-12T23:39:05.578 INFO:tasks.rados.rados.0.smithi136.stdout:270: read oid 22 snap -1 2024-09-12T23:39:05.578 INFO:tasks.rados.rados.0.smithi136.stdout:270: expect (ObjNum 21 snap 0 seq_num 21) 2024-09-12T23:39:05.579 INFO:tasks.rados.rados.0.smithi136.stdout:271: rollback oid 45 current snap is 23 2024-09-12T23:39:05.579 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 45 to 17 2024-09-12T23:39:05.579 INFO:tasks.rados.rados.0.smithi136.stdout:262: expect (ObjNum 87 snap 18 seq_num 87) 2024-09-12T23:39:05.716 INFO:tasks.rados.rados.0.smithi136.stdout:266: finishing write tid 3 to smithi13667081-15 2024-09-12T23:39:05.716 INFO:tasks.rados.rados.0.smithi136.stdout:266: finishing write tid 4 to smithi13667081-15 2024-09-12T23:39:05.716 INFO:tasks.rados.rados.0.smithi136.stdout:266: finishing write tid 5 to smithi13667081-15 2024-09-12T23:39:05.716 INFO:tasks.rados.rados.0.smithi136.stdout:266: finishing write tid 6 to smithi13667081-15 2024-09-12T23:39:05.716 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 15 v 141 (ObjNum 105 snap 23 seq_num 105) dirty exists 2024-09-12T23:39:05.716 INFO:tasks.rados.rados.0.smithi136.stdout:266: left oid 15 (ObjNum 105 snap 23 seq_num 105) 2024-09-12T23:39:05.716 INFO:tasks.rados.rados.0.smithi136.stdout:267: finishing write tid 1 to smithi13667081-49 2024-09-12T23:39:05.716 INFO:tasks.rados.rados.0.smithi136.stdout:267: finishing write tid 2 to smithi13667081-49 2024-09-12T23:39:05.716 INFO:tasks.rados.rados.0.smithi136.stdout:267: finishing write tid 3 to smithi13667081-49 2024-09-12T23:39:05.717 INFO:tasks.rados.rados.0.smithi136.stdout:267: finishing write tid 4 to smithi13667081-49 2024-09-12T23:39:05.717 INFO:tasks.rados.rados.0.smithi136.stdout:267: finishing write tid 5 to smithi13667081-49 2024-09-12T23:39:05.717 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 49 v 146 (ObjNum 106 snap 23 seq_num 106) dirty exists 2024-09-12T23:39:05.717 INFO:tasks.rados.rados.0.smithi136.stdout:267: left oid 49 (ObjNum 106 snap 23 seq_num 106) 2024-09-12T23:39:05.717 INFO:tasks.rados.rados.0.smithi136.stdout:268: finishing rollback tid 0 to smithi13667081-33 2024-09-12T23:39:05.717 INFO:tasks.rados.rados.0.smithi136.stdout:262: done (6 left) 2024-09-12T23:39:05.717 INFO:tasks.rados.rados.0.smithi136.stdout:266: done (5 left) 2024-09-12T23:39:05.717 INFO:tasks.rados.rados.0.smithi136.stdout:267: done (4 left) 2024-09-12T23:39:05.717 INFO:tasks.rados.rados.0.smithi136.stdout:272: rmattr oid 38 current snap is 23 2024-09-12T23:39:05.717 INFO:tasks.rados.rados.0.smithi136.stdout:268: finishing rollback tid 1 to smithi13667081-33 2024-09-12T23:39:05.717 INFO:tasks.rados.rados.0.smithi136.stdout:268: finishing rollback tid 2 to smithi13667081-33 2024-09-12T23:39:05.717 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 33 v 94 (ObjNum 15 snap 0 seq_num 15) dirty exists 2024-09-12T23:39:05.717 INFO:tasks.rados.rados.0.smithi136.stdout:269: finishing write tid 1 to smithi13667081-2 2024-09-12T23:39:05.717 INFO:tasks.rados.rados.0.smithi136.stdout:269: finishing write tid 2 to smithi13667081-2 2024-09-12T23:39:05.718 INFO:tasks.rados.rados.0.smithi136.stdout:270: expect (ObjNum 21 snap 0 seq_num 21) 2024-09-12T23:39:05.725 INFO:tasks.rados.rados.0.smithi136.stdout:269: finishing write tid 3 to smithi13667081-2 2024-09-12T23:39:05.725 INFO:tasks.rados.rados.0.smithi136.stdout:271: finishing rollback tid 0 to smithi13667081-45 2024-09-12T23:39:05.725 INFO:tasks.rados.rados.0.smithi136.stdout:269: finishing write tid 4 to smithi13667081-2 2024-09-12T23:39:05.725 INFO:tasks.rados.rados.0.smithi136.stdout:269: finishing write tid 5 to smithi13667081-2 2024-09-12T23:39:05.725 INFO:tasks.rados.rados.0.smithi136.stdout:271: finishing rollback tid 1 to smithi13667081-45 2024-09-12T23:39:05.725 INFO:tasks.rados.rados.0.smithi136.stdout:271: finishing rollback tid 2 to smithi13667081-45 2024-09-12T23:39:05.725 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 45 v 61 (ObjNum 33 snap 0 seq_num 33) dirty exists 2024-09-12T23:39:05.725 INFO:tasks.rados.rados.0.smithi136.stdout:269: finishing write tid 6 to smithi13667081-2 2024-09-12T23:39:05.725 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 2 v 99 (ObjNum 107 snap 23 seq_num 107) dirty exists 2024-09-12T23:39:05.725 INFO:tasks.rados.rados.0.smithi136.stdout:269: left oid 2 (ObjNum 107 snap 23 seq_num 107) 2024-09-12T23:39:05.725 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 38 v 71 (ObjNum 93 snap 18 seq_num 93) dirty exists 2024-09-12T23:39:05.725 INFO:tasks.rados.rados.0.smithi136.stdout:268: done (4 left) 2024-09-12T23:39:05.726 INFO:tasks.rados.rados.0.smithi136.stdout:269: done (3 left) 2024-09-12T23:39:05.726 INFO:tasks.rados.rados.0.smithi136.stdout:270: done (2 left) 2024-09-12T23:39:05.726 INFO:tasks.rados.rados.0.smithi136.stdout:271: done (1 left) 2024-09-12T23:39:05.726 INFO:tasks.rados.rados.0.smithi136.stdout:272: done (0 left) 2024-09-12T23:39:05.726 INFO:tasks.rados.rados.0.smithi136.stdout:273: copy_from oid 29 from oid 28 current snap is 23 2024-09-12T23:39:05.726 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 37 current snap is 23 2024-09-12T23:39:05.726 INFO:tasks.rados.rados.0.smithi136.stdout:274: seq_num 108 ranges {1974272=614400,2588672=606208,3194880=598016} 2024-09-12T23:39:05.753 INFO:tasks.rados.rados.0.smithi136.stdout:274: writing smithi13667081-37 from 1974272 to 2588672 tid 1 2024-09-12T23:39:05.760 INFO:tasks.rados.rados.0.smithi136.stdout:274: writing smithi13667081-37 from 2588672 to 3194880 tid 2 2024-09-12T23:39:05.767 INFO:tasks.rados.rados.0.smithi136.stdout:274: writing smithi13667081-37 from 3194880 to 3792896 tid 3 2024-09-12T23:39:05.767 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 45 current snap is 23 2024-09-12T23:39:05.767 INFO:tasks.rados.rados.0.smithi136.stdout:273: finishing copy_from racing read to smithi13667081-29 2024-09-12T23:39:05.767 INFO:tasks.rados.rados.0.smithi136.stdout:273: finishing copy_from to smithi13667081-29 2024-09-12T23:39:05.768 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 29 v 35 (ObjNum 4 snap 0 seq_num 304) dirty exists 2024-09-12T23:39:05.768 INFO:tasks.rados.rados.0.smithi136.stdout:274: finishing write tid 1 to smithi13667081-37 2024-09-12T23:39:05.768 INFO:tasks.rados.rados.0.smithi136.stdout:274: finishing write tid 2 to smithi13667081-37 2024-09-12T23:39:05.768 INFO:tasks.rados.rados.0.smithi136.stdout:275: seq_num 109 ranges {557056=630784,1187840=491520,1679360=483328,2162688=229376} 2024-09-12T23:39:05.781 INFO:tasks.rados.rados.0.smithi136.stdout:275: writing smithi13667081-45 from 557056 to 1187840 tid 1 2024-09-12T23:39:05.786 INFO:tasks.rados.rados.0.smithi136.stdout:275: writing smithi13667081-45 from 1187840 to 1679360 tid 2 2024-09-12T23:39:05.791 INFO:tasks.rados.rados.0.smithi136.stdout:275: writing smithi13667081-45 from 1679360 to 2162688 tid 3 2024-09-12T23:39:05.794 INFO:tasks.rados.rados.0.smithi136.stdout:275: writing smithi13667081-45 from 2162688 to 2392064 tid 4 2024-09-12T23:39:05.794 INFO:tasks.rados.rados.0.smithi136.stdout:273: done (2 left) 2024-09-12T23:39:05.794 INFO:tasks.rados.rados.0.smithi136.stdout:276: read oid 12 snap 21 2024-09-12T23:39:05.794 INFO:tasks.rados.rados.0.smithi136.stdout:276: expect (ObjNum 70 snap 7 seq_num 70) 2024-09-12T23:39:05.794 INFO:tasks.rados.rados.0.smithi136.stdout:277: rmattr oid 32 current snap is 23 2024-09-12T23:39:05.794 INFO:tasks.rados.rados.0.smithi136.stdout:277: done (3 left) 2024-09-12T23:39:05.794 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 46 current snap is 23 2024-09-12T23:39:05.794 INFO:tasks.rados.rados.0.smithi136.stdout:278: seq_num 110 ranges {761856=647168,1409024=778240,2187264=425984} 2024-09-12T23:39:05.806 INFO:tasks.rados.rados.0.smithi136.stdout:278: writing smithi13667081-46 from 761856 to 1409024 tid 1 2024-09-12T23:39:05.813 INFO:tasks.rados.rados.0.smithi136.stdout:278: writing smithi13667081-46 from 1409024 to 2187264 tid 2 2024-09-12T23:39:05.817 INFO:tasks.rados.rados.0.smithi136.stdout:278: writing smithi13667081-46 from 2187264 to 2613248 tid 3 2024-09-12T23:39:05.817 INFO:tasks.rados.rados.0.smithi136.stdout:279: read oid 2 snap -1 2024-09-12T23:39:05.817 INFO:tasks.rados.rados.0.smithi136.stdout:279: expect (ObjNum 107 snap 23 seq_num 107) 2024-09-12T23:39:05.817 INFO:tasks.rados.rados.0.smithi136.stdout:280: read oid 7 snap 14 2024-09-12T23:39:05.817 INFO:tasks.rados.rados.0.smithi136.stdout:280: expect (ObjNum 6 snap 0 seq_num 6) 2024-09-12T23:39:05.817 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 41 current snap is 23 2024-09-12T23:39:05.817 INFO:tasks.rados.rados.0.smithi136.stdout:274: finishing write tid 3 to smithi13667081-37 2024-09-12T23:39:05.817 INFO:tasks.rados.rados.0.smithi136.stdout:274: finishing write tid 4 to smithi13667081-37 2024-09-12T23:39:05.817 INFO:tasks.rados.rados.0.smithi136.stdout:274: finishing write tid 5 to smithi13667081-37 2024-09-12T23:39:05.817 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 37 v 104 (ObjNum 108 snap 23 seq_num 108) dirty exists 2024-09-12T23:39:05.817 INFO:tasks.rados.rados.0.smithi136.stdout:274: left oid 37 (ObjNum 108 snap 23 seq_num 108) 2024-09-12T23:39:05.818 INFO:tasks.rados.rados.0.smithi136.stdout:275: finishing write tid 1 to smithi13667081-45 2024-09-12T23:39:05.818 INFO:tasks.rados.rados.0.smithi136.stdout:275: finishing write tid 2 to smithi13667081-45 2024-09-12T23:39:05.818 INFO:tasks.rados.rados.0.smithi136.stdout:275: finishing write tid 3 to smithi13667081-45 2024-09-12T23:39:05.818 INFO:tasks.rados.rados.0.smithi136.stdout:275: finishing write tid 4 to smithi13667081-45 2024-09-12T23:39:05.818 INFO:tasks.rados.rados.0.smithi136.stdout:275: finishing write tid 5 to smithi13667081-45 2024-09-12T23:39:05.818 INFO:tasks.rados.rados.0.smithi136.stdout:281: seq_num 111 ranges {0=663552,663552=663552,1327104=450560,1777664=90112} 2024-09-12T23:39:05.821 INFO:tasks.rados.rados.0.smithi136.stdout:281: writing smithi13667081-41 from 0 to 663552 tid 1 2024-09-12T23:39:05.827 INFO:tasks.rados.rados.0.smithi136.stdout:281: writing smithi13667081-41 from 663552 to 1327104 tid 2 2024-09-12T23:39:05.830 INFO:tasks.rados.rados.0.smithi136.stdout:281: writing smithi13667081-41 from 1327104 to 1777664 tid 3 2024-09-12T23:39:05.831 INFO:tasks.rados.rados.0.smithi136.stdout:281: writing smithi13667081-41 from 1777664 to 1867776 tid 4 2024-09-12T23:39:05.831 INFO:tasks.rados.rados.0.smithi136.stdout:274: done (6 left) 2024-09-12T23:39:05.831 INFO:tasks.rados.rados.0.smithi136.stdout:282: snap_remove snap 17 2024-09-12T23:39:05.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:05 smithi181 bash[24776]: cluster 2024-09-12T23:39:04.029481+0000 mgr.y (mgr.24446) 213 : cluster [DBG] pgmap v203: 121 pgs: 121 active+clean; 60 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.8 MiB/s rd, 1.4 MiB/s wr, 27 op/s 2024-09-12T23:39:05.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:05 smithi181 bash[24776]: cluster 2024-09-12T23:39:04.546339+0000 mon.a (mon.0) 1134 : cluster [DBG] osdmap e108: 8 total, 8 up, 8 in 2024-09-12T23:39:05.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:05 smithi181 bash[24776]: cluster 2024-09-12T23:39:05.454093+0000 mon.a (mon.0) 1135 : cluster [DBG] osdmap e109: 8 total, 8 up, 8 in 2024-09-12T23:39:06.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:05 smithi136 bash[22601]: cluster 2024-09-12T23:39:04.029481+0000 mgr.y (mgr.24446) 213 : cluster [DBG] pgmap v203: 121 pgs: 121 active+clean; 60 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.8 MiB/s rd, 1.4 MiB/s wr, 27 op/s 2024-09-12T23:39:06.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:05 smithi136 bash[22601]: cluster 2024-09-12T23:39:04.546339+0000 mon.a (mon.0) 1134 : cluster [DBG] osdmap e108: 8 total, 8 up, 8 in 2024-09-12T23:39:06.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:05 smithi136 bash[22601]: cluster 2024-09-12T23:39:05.454093+0000 mon.a (mon.0) 1135 : cluster [DBG] osdmap e109: 8 total, 8 up, 8 in 2024-09-12T23:39:06.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:05 smithi136 bash[32189]: cluster 2024-09-12T23:39:04.029481+0000 mgr.y (mgr.24446) 213 : cluster [DBG] pgmap v203: 121 pgs: 121 active+clean; 60 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.8 MiB/s rd, 1.4 MiB/s wr, 27 op/s 2024-09-12T23:39:06.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:05 smithi136 bash[32189]: cluster 2024-09-12T23:39:04.546339+0000 mon.a (mon.0) 1134 : cluster [DBG] osdmap e108: 8 total, 8 up, 8 in 2024-09-12T23:39:06.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:05 smithi136 bash[32189]: cluster 2024-09-12T23:39:05.454093+0000 mon.a (mon.0) 1135 : cluster [DBG] osdmap e109: 8 total, 8 up, 8 in 2024-09-12T23:39:06.505 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:39:06 smithi136 bash[22852]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:39:06] "GET /metrics HTTP/1.1" 200 239875 "" "Prometheus/2.43.0" 2024-09-12T23:39:06.553 INFO:tasks.rados.rados.0.smithi136.stdout:282: done (6 left) 2024-09-12T23:39:06.553 INFO:tasks.rados.rados.0.smithi136.stdout:283: copy_from oid 20 from oid 40 current snap is 23 2024-09-12T23:39:06.553 INFO:tasks.rados.rados.0.smithi136.stdout:276: expect (ObjNum 70 snap 7 seq_num 70) 2024-09-12T23:39:06.635 INFO:tasks.rados.rados.0.smithi136.stdout:275: finishing write tid 6 to smithi13667081-45 2024-09-12T23:39:06.635 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 45 v 66 (ObjNum 109 snap 23 seq_num 109) dirty exists 2024-09-12T23:39:06.635 INFO:tasks.rados.rados.0.smithi136.stdout:275: left oid 45 (ObjNum 109 snap 23 seq_num 109) 2024-09-12T23:39:06.635 INFO:tasks.rados.rados.0.smithi136.stdout:278: finishing write tid 1 to smithi13667081-46 2024-09-12T23:39:06.636 INFO:tasks.rados.rados.0.smithi136.stdout:278: finishing write tid 2 to smithi13667081-46 2024-09-12T23:39:06.636 INFO:tasks.rados.rados.0.smithi136.stdout:280: expect (ObjNum 6 snap 0 seq_num 6) 2024-09-12T23:39:06.639 INFO:tasks.rados.rados.0.smithi136.stdout:278: finishing write tid 3 to smithi13667081-46 2024-09-12T23:39:06.639 INFO:tasks.rados.rados.0.smithi136.stdout:278: finishing write tid 4 to smithi13667081-46 2024-09-12T23:39:06.639 INFO:tasks.rados.rados.0.smithi136.stdout:279: expect (ObjNum 107 snap 23 seq_num 107) 2024-09-12T23:39:06.689 INFO:tasks.rados.rados.0.smithi136.stdout:278: finishing write tid 5 to smithi13667081-46 2024-09-12T23:39:06.689 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 46 v 50 (ObjNum 110 snap 23 seq_num 110) dirty exists 2024-09-12T23:39:06.689 INFO:tasks.rados.rados.0.smithi136.stdout:278: left oid 46 (ObjNum 110 snap 23 seq_num 110) 2024-09-12T23:39:06.689 INFO:tasks.rados.rados.0.smithi136.stdout:281: finishing write tid 1 to smithi13667081-41 2024-09-12T23:39:06.689 INFO:tasks.rados.rados.0.smithi136.stdout:281: finishing write tid 2 to smithi13667081-41 2024-09-12T23:39:06.689 INFO:tasks.rados.rados.0.smithi136.stdout:281: finishing write tid 3 to smithi13667081-41 2024-09-12T23:39:06.689 INFO:tasks.rados.rados.0.smithi136.stdout:281: finishing write tid 4 to smithi13667081-41 2024-09-12T23:39:06.689 INFO:tasks.rados.rados.0.smithi136.stdout:281: finishing write tid 5 to smithi13667081-41 2024-09-12T23:39:06.689 INFO:tasks.rados.rados.0.smithi136.stdout:281: finishing write tid 6 to smithi13667081-41 2024-09-12T23:39:06.689 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 41 v 109 (ObjNum 111 snap 23 seq_num 111) dirty exists 2024-09-12T23:39:06.689 INFO:tasks.rados.rados.0.smithi136.stdout:281: left oid 41 (ObjNum 111 snap 23 seq_num 111) 2024-09-12T23:39:06.689 INFO:tasks.rados.rados.0.smithi136.stdout:283: finishing copy_from racing read to smithi13667081-20 2024-09-12T23:39:06.689 INFO:tasks.rados.rados.0.smithi136.stdout:283: finishing copy_from to smithi13667081-20 2024-09-12T23:39:06.690 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 20 v 23 (ObjNum 39 snap 0 seq_num 39) dirty exists 2024-09-12T23:39:06.690 INFO:tasks.rados.rados.0.smithi136.stdout:275: done (6 left) 2024-09-12T23:39:06.690 INFO:tasks.rados.rados.0.smithi136.stdout:276: done (5 left) 2024-09-12T23:39:06.690 INFO:tasks.rados.rados.0.smithi136.stdout:278: done (4 left) 2024-09-12T23:39:06.690 INFO:tasks.rados.rados.0.smithi136.stdout:279: done (3 left) 2024-09-12T23:39:06.690 INFO:tasks.rados.rados.0.smithi136.stdout:280: done (2 left) 2024-09-12T23:39:06.690 INFO:tasks.rados.rados.0.smithi136.stdout:281: done (1 left) 2024-09-12T23:39:06.690 INFO:tasks.rados.rados.0.smithi136.stdout:283: done (0 left) 2024-09-12T23:39:06.690 INFO:tasks.rados.rados.0.smithi136.stdout:284: rollback oid 18 current snap is 23 2024-09-12T23:39:06.690 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 18 to 10 2024-09-12T23:39:06.690 INFO:tasks.rados.rados.0.smithi136.stdout:285: rollback oid 50 current snap is 23 2024-09-12T23:39:06.690 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 50 to 13 2024-09-12T23:39:06.691 INFO:tasks.rados.rados.0.smithi136.stdout:286: rmattr oid 5 current snap is 23 2024-09-12T23:39:06.691 INFO:tasks.rados.rados.0.smithi136.stdout:286: done (2 left) 2024-09-12T23:39:06.691 INFO:tasks.rados.rados.0.smithi136.stdout:287: rollback oid 27 current snap is 23 2024-09-12T23:39:06.691 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 27 to 19 2024-09-12T23:39:06.691 INFO:tasks.rados.rados.0.smithi136.stdout:288: rmattr oid 38 current snap is 23 2024-09-12T23:39:06.691 INFO:tasks.rados.rados.0.smithi136.stdout:289: copy_from oid 20 from oid 39 current snap is 23 2024-09-12T23:39:06.691 INFO:tasks.rados.rados.0.smithi136.stdout:290: rollback oid 7 current snap is 23 2024-09-12T23:39:06.691 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 7 to 13 2024-09-12T23:39:06.691 INFO:tasks.rados.rados.0.smithi136.stdout:291: snap_remove snap 14 2024-09-12T23:39:07.564 INFO:tasks.rados.rados.0.smithi136.stdout:291: done (6 left) 2024-09-12T23:39:07.564 INFO:tasks.rados.rados.0.smithi136.stdout:292: snap_create 2024-09-12T23:39:07.565 INFO:tasks.rados.rados.0.smithi136.stdout:287: finishing rollback tid 0 to smithi13667081-27 2024-09-12T23:39:07.565 INFO:tasks.rados.rados.0.smithi136.stdout:287: finishing rollback tid 1 to smithi13667081-27 2024-09-12T23:39:07.565 INFO:tasks.rados.rados.0.smithi136.stdout:287: finishing rollback tid 2 to smithi13667081-27 2024-09-12T23:39:07.565 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 27 v 110 (ObjNum 26 snap 0 seq_num 26) dirty exists 2024-09-12T23:39:07.565 INFO:tasks.rados.rados.0.smithi136.stdout:284: finishing rollback tid 0 to smithi13667081-18 2024-09-12T23:39:07.565 INFO:tasks.rados.rados.0.smithi136.stdout:284: finishing rollback tid 1 to smithi13667081-18 2024-09-12T23:39:07.565 INFO:tasks.rados.rados.0.smithi136.stdout:289: finishing copy_from racing read to smithi13667081-20 2024-09-12T23:39:07.565 INFO:tasks.rados.rados.0.smithi136.stdout:284: finishing rollback tid 2 to smithi13667081-18 2024-09-12T23:39:07.565 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 18 v 8 (ObjNum 17 snap 0 seq_num 17) dirty exists 2024-09-12T23:39:07.565 INFO:tasks.rados.rados.0.smithi136.stdout:285: finishing rollback tid 0 to smithi13667081-50 2024-09-12T23:39:07.565 INFO:tasks.rados.rados.0.smithi136.stdout:289: finishing copy_from to smithi13667081-20 2024-09-12T23:39:07.565 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 20 v 14 (ObjNum 38 snap 0 seq_num 38) dirty exists 2024-09-12T23:39:07.565 INFO:tasks.rados.rados.0.smithi136.stdout:290: finishing rollback tid 0 to smithi13667081-7 2024-09-12T23:39:07.566 INFO:tasks.rados.rados.0.smithi136.stdout:285: finishing rollback tid 1 to smithi13667081-50 2024-09-12T23:39:07.566 INFO:tasks.rados.rados.0.smithi136.stdout:290: finishing rollback tid 1 to smithi13667081-7 2024-09-12T23:39:07.566 INFO:tasks.rados.rados.0.smithi136.stdout:290: finishing rollback tid 2 to smithi13667081-7 2024-09-12T23:39:07.566 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 7 v 12 (ObjNum 6 snap 0 seq_num 6) dirty exists 2024-09-12T23:39:07.566 INFO:tasks.rados.rados.0.smithi136.stdout:285: finishing rollback tid 2 to smithi13667081-50 2024-09-12T23:39:07.566 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 50 v 81 (ObjNum 49 snap 0 seq_num 49) dirty exists 2024-09-12T23:39:07.566 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 38 v 82 (ObjNum 93 snap 18 seq_num 93) dirty exists 2024-09-12T23:39:07.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:07 smithi181 bash[24776]: cluster 2024-09-12T23:39:06.030178+0000 mgr.y (mgr.24446) 214 : cluster [DBG] pgmap v206: 121 pgs: 121 active+clean; 60 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.8 MiB/s rd, 1.4 MiB/s wr, 27 op/s 2024-09-12T23:39:07.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:07 smithi181 bash[24776]: cluster 2024-09-12T23:39:06.557388+0000 mon.a (mon.0) 1136 : cluster [DBG] osdmap e110: 8 total, 8 up, 8 in 2024-09-12T23:39:08.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:07 smithi136 bash[22601]: cluster 2024-09-12T23:39:06.030178+0000 mgr.y (mgr.24446) 214 : cluster [DBG] pgmap v206: 121 pgs: 121 active+clean; 60 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.8 MiB/s rd, 1.4 MiB/s wr, 27 op/s 2024-09-12T23:39:08.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:07 smithi136 bash[22601]: cluster 2024-09-12T23:39:06.557388+0000 mon.a (mon.0) 1136 : cluster [DBG] osdmap e110: 8 total, 8 up, 8 in 2024-09-12T23:39:08.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:07 smithi136 bash[32189]: cluster 2024-09-12T23:39:06.030178+0000 mgr.y (mgr.24446) 214 : cluster [DBG] pgmap v206: 121 pgs: 121 active+clean; 60 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.8 MiB/s rd, 1.4 MiB/s wr, 27 op/s 2024-09-12T23:39:08.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:07 smithi136 bash[32189]: cluster 2024-09-12T23:39:06.557388+0000 mon.a (mon.0) 1136 : cluster [DBG] osdmap e110: 8 total, 8 up, 8 in 2024-09-12T23:39:08.573 INFO:tasks.rados.rados.0.smithi136.stdout:284: done (6 left) 2024-09-12T23:39:08.573 INFO:tasks.rados.rados.0.smithi136.stdout:285: done (5 left) 2024-09-12T23:39:08.573 INFO:tasks.rados.rados.0.smithi136.stdout:287: done (4 left) 2024-09-12T23:39:08.573 INFO:tasks.rados.rados.0.smithi136.stdout:288: done (3 left) 2024-09-12T23:39:08.573 INFO:tasks.rados.rados.0.smithi136.stdout:289: done (2 left) 2024-09-12T23:39:08.573 INFO:tasks.rados.rados.0.smithi136.stdout:290: done (1 left) 2024-09-12T23:39:08.574 INFO:tasks.rados.rados.0.smithi136.stdout:292: done (0 left) 2024-09-12T23:39:08.574 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 16 current snap is 24 2024-09-12T23:39:08.574 INFO:tasks.rados.rados.0.smithi136.stdout:293: seq_num 112 ranges {2940928=679936,3620864=548864,4169728=655360} 2024-09-12T23:39:08.629 INFO:tasks.rados.rados.0.smithi136.stdout:293: writing smithi13667081-16 from 2940928 to 3620864 tid 1 2024-09-12T23:39:08.636 INFO:tasks.rados.rados.0.smithi136.stdout:293: writing smithi13667081-16 from 3620864 to 4169728 tid 2 2024-09-12T23:39:08.644 INFO:tasks.rados.rados.0.smithi136.stdout:293: writing smithi13667081-16 from 4169728 to 4825088 tid 3 2024-09-12T23:39:08.644 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 18 current snap is 24 2024-09-12T23:39:08.644 INFO:tasks.rados.rados.0.smithi136.stdout:293: finishing write tid 1 to smithi13667081-16 2024-09-12T23:39:08.644 INFO:tasks.rados.rados.0.smithi136.stdout:294: seq_num 113 ranges {286720=696320,983040=434176,1417216=466944,1884160=303104} 2024-09-12T23:39:08.653 INFO:tasks.rados.rados.0.smithi136.stdout:294: writing smithi13667081-18 from 286720 to 983040 tid 1 2024-09-12T23:39:08.657 INFO:tasks.rados.rados.0.smithi136.stdout:294: writing smithi13667081-18 from 983040 to 1417216 tid 2 2024-09-12T23:39:08.662 INFO:tasks.rados.rados.0.smithi136.stdout:294: writing smithi13667081-18 from 1417216 to 1884160 tid 3 2024-09-12T23:39:08.665 INFO:tasks.rados.rados.0.smithi136.stdout:294: writing smithi13667081-18 from 1884160 to 2187264 tid 4 2024-09-12T23:39:08.665 INFO:tasks.rados.rados.0.smithi136.stdout:295: setattr oid 45 current snap is 24 2024-09-12T23:39:08.665 INFO:tasks.rados.rados.0.smithi136.stdout:293: finishing write tid 2 to smithi13667081-16 2024-09-12T23:39:08.665 INFO:tasks.rados.rados.0.smithi136.stdout:293: finishing write tid 3 to smithi13667081-16 2024-09-12T23:39:08.665 INFO:tasks.rados.rados.0.smithi136.stdout:293: finishing write tid 4 to smithi13667081-16 2024-09-12T23:39:08.665 INFO:tasks.rados.rados.0.smithi136.stdout:293: finishing write tid 5 to smithi13667081-16 2024-09-12T23:39:08.665 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 16 v 115 (ObjNum 112 snap 24 seq_num 112) dirty exists 2024-09-12T23:39:08.665 INFO:tasks.rados.rados.0.smithi136.stdout:293: left oid 16 (ObjNum 112 snap 24 seq_num 112) 2024-09-12T23:39:08.666 INFO:tasks.rados.rados.0.smithi136.stdout:294: finishing write tid 1 to smithi13667081-18 2024-09-12T23:39:08.667 INFO:tasks.rados.rados.0.smithi136.stdout:293: done (2 left) 2024-09-12T23:39:08.667 INFO:tasks.rados.rados.0.smithi136.stdout:296: snap_remove snap 10 2024-09-12T23:39:08.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:08 smithi181 bash[24776]: cluster 2024-09-12T23:39:07.568310+0000 mon.a (mon.0) 1137 : cluster [DBG] osdmap e111: 8 total, 8 up, 8 in 2024-09-12T23:39:08.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:08 smithi181 bash[24776]: audit 2024-09-12T23:39:08.014345+0000 mon.a (mon.0) 1138 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:39:09.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:08 smithi136 bash[32189]: cluster 2024-09-12T23:39:07.568310+0000 mon.a (mon.0) 1137 : cluster [DBG] osdmap e111: 8 total, 8 up, 8 in 2024-09-12T23:39:09.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:08 smithi136 bash[32189]: audit 2024-09-12T23:39:08.014345+0000 mon.a (mon.0) 1138 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:39:09.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:08 smithi136 bash[22601]: cluster 2024-09-12T23:39:07.568310+0000 mon.a (mon.0) 1137 : cluster [DBG] osdmap e111: 8 total, 8 up, 8 in 2024-09-12T23:39:09.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:08 smithi136 bash[22601]: audit 2024-09-12T23:39:08.014345+0000 mon.a (mon.0) 1138 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:39:09.582 INFO:tasks.rados.rados.0.smithi136.stdout:296: done (2 left) 2024-09-12T23:39:09.582 INFO:tasks.rados.rados.0.smithi136.stdout:297: setattr oid 15 current snap is 24 2024-09-12T23:39:09.582 INFO:tasks.rados.rados.0.smithi136.stdout:294: finishing write tid 2 to smithi13667081-18 2024-09-12T23:39:09.582 INFO:tasks.rados.rados.0.smithi136.stdout:294: finishing write tid 3 to smithi13667081-18 2024-09-12T23:39:09.582 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 45 v 72 (ObjNum 109 snap 23 seq_num 109) dirty exists 2024-09-12T23:39:09.582 INFO:tasks.rados.rados.0.smithi136.stdout:294: finishing write tid 4 to smithi13667081-18 2024-09-12T23:39:09.582 INFO:tasks.rados.rados.0.smithi136.stdout:294: finishing write tid 5 to smithi13667081-18 2024-09-12T23:39:09.582 INFO:tasks.rados.rados.0.smithi136.stdout:294: finishing write tid 6 to smithi13667081-18 2024-09-12T23:39:09.582 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 18 v 22 (ObjNum 113 snap 24 seq_num 113) dirty exists 2024-09-12T23:39:09.583 INFO:tasks.rados.rados.0.smithi136.stdout:294: left oid 18 (ObjNum 113 snap 24 seq_num 113) 2024-09-12T23:39:09.585 INFO:tasks.rados.rados.0.smithi136.stdout:294: done (2 left) 2024-09-12T23:39:09.585 INFO:tasks.rados.rados.0.smithi136.stdout:295: done (1 left) 2024-09-12T23:39:09.586 INFO:tasks.rados.rados.0.smithi136.stdout:298: snap_create 2024-09-12T23:39:09.604 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 15 v 162 (ObjNum 105 snap 23 seq_num 105) dirty exists 2024-09-12T23:39:09.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:09 smithi181 bash[24776]: cluster 2024-09-12T23:39:08.030808+0000 mgr.y (mgr.24446) 215 : cluster [DBG] pgmap v209: 121 pgs: 121 active+clean; 60 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:39:09.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:09 smithi181 bash[24776]: cluster 2024-09-12T23:39:08.577373+0000 mon.a (mon.0) 1139 : cluster [DBG] osdmap e112: 8 total, 8 up, 8 in 2024-09-12T23:39:10.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:09 smithi136 bash[32189]: cluster 2024-09-12T23:39:08.030808+0000 mgr.y (mgr.24446) 215 : cluster [DBG] pgmap v209: 121 pgs: 121 active+clean; 60 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:39:10.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:09 smithi136 bash[32189]: cluster 2024-09-12T23:39:08.577373+0000 mon.a (mon.0) 1139 : cluster [DBG] osdmap e112: 8 total, 8 up, 8 in 2024-09-12T23:39:10.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:09 smithi136 bash[22601]: cluster 2024-09-12T23:39:08.030808+0000 mgr.y (mgr.24446) 215 : cluster [DBG] pgmap v209: 121 pgs: 121 active+clean; 60 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:39:10.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:09 smithi136 bash[22601]: cluster 2024-09-12T23:39:08.577373+0000 mon.a (mon.0) 1139 : cluster [DBG] osdmap e112: 8 total, 8 up, 8 in 2024-09-12T23:39:10.592 INFO:tasks.rados.rados.0.smithi136.stdout:297: done (1 left) 2024-09-12T23:39:10.593 INFO:tasks.rados.rados.0.smithi136.stdout:298: done (0 left) 2024-09-12T23:39:10.593 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 33 current snap is 25 2024-09-12T23:39:10.593 INFO:tasks.rados.rados.0.smithi136.stdout:299: seq_num 114 ranges {253952=712704,966656=712704,1679360=491520} 2024-09-12T23:39:10.605 INFO:tasks.rados.rados.0.smithi136.stdout:299: writing smithi13667081-33 from 253952 to 966656 tid 1 2024-09-12T23:39:10.618 INFO:tasks.rados.rados.0.smithi136.stdout:299: writing smithi13667081-33 from 966656 to 1679360 tid 2 2024-09-12T23:39:10.624 INFO:tasks.rados.rados.0.smithi136.stdout:299: writing smithi13667081-33 from 1679360 to 2170880 tid 3 2024-09-12T23:39:10.624 INFO:tasks.rados.rados.0.smithi136.stdout:300: copy_from oid 18 from oid 13 current snap is 25 2024-09-12T23:39:10.625 INFO:tasks.rados.rados.0.smithi136.stdout:299: finishing write tid 1 to smithi13667081-33 2024-09-12T23:39:10.625 INFO:tasks.rados.rados.0.smithi136.stdout:299: finishing write tid 2 to smithi13667081-33 2024-09-12T23:39:10.625 INFO:tasks.rados.rados.0.smithi136.stdout:301: snap_create 2024-09-12T23:39:10.630 INFO:tasks.rados.rados.0.smithi136.stdout:299: finishing write tid 3 to smithi13667081-33 2024-09-12T23:39:10.630 INFO:tasks.rados.rados.0.smithi136.stdout:299: finishing write tid 4 to smithi13667081-33 2024-09-12T23:39:10.633 INFO:tasks.rados.rados.0.smithi136.stdout:299: finishing write tid 5 to smithi13667081-33 2024-09-12T23:39:10.633 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 33 v 138 (ObjNum 114 snap 25 seq_num 114) dirty exists 2024-09-12T23:39:10.633 INFO:tasks.rados.rados.0.smithi136.stdout:299: left oid 33 (ObjNum 114 snap 25 seq_num 114) 2024-09-12T23:39:10.658 INFO:tasks.rados.rados.0.smithi136.stdout:300: finishing copy_from racing read to smithi13667081-18 2024-09-12T23:39:10.659 INFO:tasks.rados.rados.0.smithi136.stdout:300: finishing copy_from to smithi13667081-18 2024-09-12T23:39:10.659 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 18 v 47 (ObjNum 87 snap 18 seq_num 87) dirty exists 2024-09-12T23:39:10.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:10 smithi181 bash[24776]: cluster 2024-09-12T23:39:09.586270+0000 mon.a (mon.0) 1140 : cluster [DBG] osdmap e113: 8 total, 8 up, 8 in 2024-09-12T23:39:11.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:10 smithi136 bash[22601]: cluster 2024-09-12T23:39:09.586270+0000 mon.a (mon.0) 1140 : cluster [DBG] osdmap e113: 8 total, 8 up, 8 in 2024-09-12T23:39:11.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:10 smithi136 bash[32189]: cluster 2024-09-12T23:39:09.586270+0000 mon.a (mon.0) 1140 : cluster [DBG] osdmap e113: 8 total, 8 up, 8 in 2024-09-12T23:39:11.592 INFO:tasks.rados.rados.0.smithi136.stdout:299: done (2 left) 2024-09-12T23:39:11.593 INFO:tasks.rados.rados.0.smithi136.stdout:300: done (1 left) 2024-09-12T23:39:11.593 INFO:tasks.rados.rados.0.smithi136.stdout:301: done (0 left) 2024-09-12T23:39:11.593 INFO:tasks.rados.rados.0.smithi136.stdout:302: snap_create 2024-09-12T23:39:12.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:11 smithi136 bash[22601]: cluster 2024-09-12T23:39:10.031509+0000 mgr.y (mgr.24446) 216 : cluster [DBG] pgmap v212: 121 pgs: 121 active+clean; 80 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.8 MiB/s rd, 5.1 MiB/s wr, 39 op/s 2024-09-12T23:39:12.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:11 smithi136 bash[22601]: cluster 2024-09-12T23:39:10.597036+0000 mon.a (mon.0) 1141 : cluster [DBG] osdmap e114: 8 total, 8 up, 8 in 2024-09-12T23:39:12.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:11 smithi136 bash[32189]: cluster 2024-09-12T23:39:10.031509+0000 mgr.y (mgr.24446) 216 : cluster [DBG] pgmap v212: 121 pgs: 121 active+clean; 80 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.8 MiB/s rd, 5.1 MiB/s wr, 39 op/s 2024-09-12T23:39:12.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:11 smithi136 bash[32189]: cluster 2024-09-12T23:39:10.597036+0000 mon.a (mon.0) 1141 : cluster [DBG] osdmap e114: 8 total, 8 up, 8 in 2024-09-12T23:39:12.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:11 smithi181 bash[24776]: cluster 2024-09-12T23:39:10.031509+0000 mgr.y (mgr.24446) 216 : cluster [DBG] pgmap v212: 121 pgs: 121 active+clean; 80 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.8 MiB/s rd, 5.1 MiB/s wr, 39 op/s 2024-09-12T23:39:12.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:11 smithi181 bash[24776]: cluster 2024-09-12T23:39:10.597036+0000 mon.a (mon.0) 1141 : cluster [DBG] osdmap e114: 8 total, 8 up, 8 in 2024-09-12T23:39:12.597 INFO:tasks.rados.rados.0.smithi136.stdout:302: done (0 left) 2024-09-12T23:39:12.597 INFO:tasks.rados.rados.0.smithi136.stdout:303: read oid 15 snap -1 2024-09-12T23:39:12.597 INFO:tasks.rados.rados.0.smithi136.stdout:303: expect (ObjNum 105 snap 23 seq_num 105) 2024-09-12T23:39:12.597 INFO:tasks.rados.rados.0.smithi136.stdout:304: snap_create 2024-09-12T23:39:12.621 INFO:tasks.rados.rados.0.smithi136.stdout:303: expect (ObjNum 105 snap 23 seq_num 105) 2024-09-12T23:39:13.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:12 smithi136 bash[32189]: cluster 2024-09-12T23:39:11.596950+0000 mon.a (mon.0) 1142 : cluster [DBG] osdmap e115: 8 total, 8 up, 8 in 2024-09-12T23:39:13.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:12 smithi136 bash[22601]: cluster 2024-09-12T23:39:11.596950+0000 mon.a (mon.0) 1142 : cluster [DBG] osdmap e115: 8 total, 8 up, 8 in 2024-09-12T23:39:13.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:12 smithi181 bash[24776]: cluster 2024-09-12T23:39:11.596950+0000 mon.a (mon.0) 1142 : cluster [DBG] osdmap e115: 8 total, 8 up, 8 in 2024-09-12T23:39:13.599 INFO:tasks.rados.rados.0.smithi136.stdout:303: done (1 left) 2024-09-12T23:39:13.599 INFO:tasks.rados.rados.0.smithi136.stdout:304: done (0 left) 2024-09-12T23:39:13.600 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 23 current snap is 28 2024-09-12T23:39:13.600 INFO:tasks.rados.rados.0.smithi136.stdout:305: seq_num 115 ranges {1687552=729088,2416640=647168,3063808=434176,3497984=122880} 2024-09-12T23:39:13.639 INFO:tasks.rados.rados.0.smithi136.stdout:305: writing smithi13667081-23 from 1687552 to 2416640 tid 1 2024-09-12T23:39:13.650 INFO:tasks.rados.rados.0.smithi136.stdout:305: writing smithi13667081-23 from 2416640 to 3063808 tid 2 2024-09-12T23:39:13.655 INFO:tasks.rados.rados.0.smithi136.stdout:305: writing smithi13667081-23 from 3063808 to 3497984 tid 3 2024-09-12T23:39:13.657 INFO:tasks.rados.rados.0.smithi136.stdout:305: writing smithi13667081-23 from 3497984 to 3620864 tid 4 2024-09-12T23:39:13.657 INFO:tasks.rados.rados.0.smithi136.stdout:306: rollback oid 9 current snap is 28 2024-09-12T23:39:13.657 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 9 to 22 2024-09-12T23:39:13.657 INFO:tasks.rados.rados.0.smithi136.stdout:305: finishing write tid 1 to smithi13667081-23 2024-09-12T23:39:13.658 INFO:tasks.rados.rados.0.smithi136.stdout:305: finishing write tid 2 to smithi13667081-23 2024-09-12T23:39:13.658 INFO:tasks.rados.rados.0.smithi136.stdout:307: snap_remove snap 20 2024-09-12T23:39:14.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:13 smithi136 bash[32189]: cluster 2024-09-12T23:39:12.032214+0000 mgr.y (mgr.24446) 217 : cluster [DBG] pgmap v215: 121 pgs: 121 active+clean; 80 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.8 MiB/s rd, 5.1 MiB/s wr, 39 op/s 2024-09-12T23:39:14.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:13 smithi136 bash[32189]: cluster 2024-09-12T23:39:12.599402+0000 mon.a (mon.0) 1143 : cluster [DBG] osdmap e116: 8 total, 8 up, 8 in 2024-09-12T23:39:14.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:13 smithi136 bash[32189]: cluster 2024-09-12T23:39:13.601801+0000 mon.a (mon.0) 1144 : cluster [DBG] osdmap e117: 8 total, 8 up, 8 in 2024-09-12T23:39:14.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:13 smithi136 bash[22601]: cluster 2024-09-12T23:39:12.032214+0000 mgr.y (mgr.24446) 217 : cluster [DBG] pgmap v215: 121 pgs: 121 active+clean; 80 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.8 MiB/s rd, 5.1 MiB/s wr, 39 op/s 2024-09-12T23:39:14.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:13 smithi136 bash[22601]: cluster 2024-09-12T23:39:12.599402+0000 mon.a (mon.0) 1143 : cluster [DBG] osdmap e116: 8 total, 8 up, 8 in 2024-09-12T23:39:14.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:13 smithi136 bash[22601]: cluster 2024-09-12T23:39:13.601801+0000 mon.a (mon.0) 1144 : cluster [DBG] osdmap e117: 8 total, 8 up, 8 in 2024-09-12T23:39:14.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:13 smithi181 bash[24776]: cluster 2024-09-12T23:39:12.032214+0000 mgr.y (mgr.24446) 217 : cluster [DBG] pgmap v215: 121 pgs: 121 active+clean; 80 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.8 MiB/s rd, 5.1 MiB/s wr, 39 op/s 2024-09-12T23:39:14.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:13 smithi181 bash[24776]: cluster 2024-09-12T23:39:12.599402+0000 mon.a (mon.0) 1143 : cluster [DBG] osdmap e116: 8 total, 8 up, 8 in 2024-09-12T23:39:14.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:13 smithi181 bash[24776]: cluster 2024-09-12T23:39:13.601801+0000 mon.a (mon.0) 1144 : cluster [DBG] osdmap e117: 8 total, 8 up, 8 in 2024-09-12T23:39:14.624 INFO:tasks.rados.rados.0.smithi136.stdout:307: done (2 left) 2024-09-12T23:39:14.624 INFO:tasks.rados.rados.0.smithi136.stdout:308: read oid 18 snap 22 2024-09-12T23:39:14.624 INFO:tasks.rados.rados.0.smithi136.stdout:308: expect (ObjNum 17 snap 0 seq_num 17) 2024-09-12T23:39:14.624 INFO:tasks.rados.rados.0.smithi136.stdout:309: snap_create 2024-09-12T23:39:14.625 INFO:tasks.rados.rados.0.smithi136.stdout:305: finishing write tid 3 to smithi13667081-23 2024-09-12T23:39:14.625 INFO:tasks.rados.rados.0.smithi136.stdout:306: finishing rollback tid 0 to smithi13667081-9 2024-09-12T23:39:14.625 INFO:tasks.rados.rados.0.smithi136.stdout:306: finishing rollback tid 1 to smithi13667081-9 2024-09-12T23:39:14.625 INFO:tasks.rados.rados.0.smithi136.stdout:306: finishing rollback tid 2 to smithi13667081-9 2024-09-12T23:39:14.625 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 9 v 77 (ObjNum 69 snap 7 seq_num 69) dirty exists 2024-09-12T23:39:14.625 INFO:tasks.rados.rados.0.smithi136.stdout:305: finishing write tid 4 to smithi13667081-23 2024-09-12T23:39:14.625 INFO:tasks.rados.rados.0.smithi136.stdout:305: finishing write tid 5 to smithi13667081-23 2024-09-12T23:39:14.625 INFO:tasks.rados.rados.0.smithi136.stdout:305: finishing write tid 6 to smithi13667081-23 2024-09-12T23:39:14.625 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 23 v 136 (ObjNum 115 snap 28 seq_num 115) dirty exists 2024-09-12T23:39:14.625 INFO:tasks.rados.rados.0.smithi136.stdout:305: left oid 23 (ObjNum 115 snap 28 seq_num 115) 2024-09-12T23:39:14.652 INFO:tasks.rados.rados.0.smithi136.stdout:308: expect (ObjNum 17 snap 0 seq_num 17) 2024-09-12T23:39:15.452 INFO:tasks.rados.rados.0.smithi136.stdout:305: done (3 left) 2024-09-12T23:39:15.452 INFO:tasks.rados.rados.0.smithi136.stdout:306: done (2 left) 2024-09-12T23:39:15.452 INFO:tasks.rados.rados.0.smithi136.stdout:308: done (1 left) 2024-09-12T23:39:15.452 INFO:tasks.rados.rados.0.smithi136.stdout:309: done (0 left) 2024-09-12T23:39:15.452 INFO:tasks.rados.rados.0.smithi136.stdout:310: read oid 23 snap -1 2024-09-12T23:39:15.452 INFO:tasks.rados.rados.0.smithi136.stdout:310: expect (ObjNum 115 snap 28 seq_num 115) 2024-09-12T23:39:15.452 INFO:tasks.rados.rados.0.smithi136.stdout:311: delete oid 34 current snap is 29 2024-09-12T23:39:15.462 INFO:tasks.rados.rados.0.smithi136.stdout:311: done (1 left) 2024-09-12T23:39:15.462 INFO:tasks.rados.rados.0.smithi136.stdout:312: delete oid 10 current snap is 29 2024-09-12T23:39:15.466 INFO:tasks.rados.rados.0.smithi136.stdout:312: done (1 left) 2024-09-12T23:39:15.466 INFO:tasks.rados.rados.0.smithi136.stdout:313: read oid 37 snap -1 2024-09-12T23:39:15.466 INFO:tasks.rados.rados.0.smithi136.stdout:313: expect (ObjNum 108 snap 23 seq_num 108) 2024-09-12T23:39:15.466 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 48 current snap is 29 2024-09-12T23:39:15.467 INFO:tasks.rados.rados.0.smithi136.stdout:314: seq_num 118 ranges {2023424=786432,2809856=704512,3514368=499712} 2024-09-12T23:39:15.482 INFO:tasks.rados.rados.0.smithi136.stdout:314: writing smithi13667081-48 from 2023424 to 2809856 tid 1 2024-09-12T23:39:15.486 INFO:tasks.rados.rados.0.smithi136.stdout:314: writing smithi13667081-48 from 2809856 to 3514368 tid 2 2024-09-12T23:39:15.490 INFO:tasks.rados.rados.0.smithi136.stdout:314: writing smithi13667081-48 from 3514368 to 4014080 tid 3 2024-09-12T23:39:15.490 INFO:tasks.rados.rados.0.smithi136.stdout:315: snap_remove snap 24 2024-09-12T23:39:15.594 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:39:15 smithi181 bash[25717]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:39:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-12T23:39:16.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:15 smithi136 bash[22601]: cluster 2024-09-12T23:39:14.033584+0000 mgr.y (mgr.24446) 218 : cluster [DBG] pgmap v218: 121 pgs: 121 active+clean; 82 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.6 MiB/s wr, 9 op/s 2024-09-12T23:39:16.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:15 smithi136 bash[22601]: cluster 2024-09-12T23:39:14.627733+0000 mon.a (mon.0) 1145 : cluster [DBG] osdmap e118: 8 total, 8 up, 8 in 2024-09-12T23:39:16.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:15 smithi136 bash[22601]: cluster 2024-09-12T23:39:15.455714+0000 mon.a (mon.0) 1146 : cluster [DBG] osdmap e119: 8 total, 8 up, 8 in 2024-09-12T23:39:16.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:15 smithi136 bash[32189]: cluster 2024-09-12T23:39:14.033584+0000 mgr.y (mgr.24446) 218 : cluster [DBG] pgmap v218: 121 pgs: 121 active+clean; 82 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.6 MiB/s wr, 9 op/s 2024-09-12T23:39:16.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:15 smithi136 bash[32189]: cluster 2024-09-12T23:39:14.627733+0000 mon.a (mon.0) 1145 : cluster [DBG] osdmap e118: 8 total, 8 up, 8 in 2024-09-12T23:39:16.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:15 smithi136 bash[32189]: cluster 2024-09-12T23:39:15.455714+0000 mon.a (mon.0) 1146 : cluster [DBG] osdmap e119: 8 total, 8 up, 8 in 2024-09-12T23:39:16.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:15 smithi181 bash[24776]: cluster 2024-09-12T23:39:14.033584+0000 mgr.y (mgr.24446) 218 : cluster [DBG] pgmap v218: 121 pgs: 121 active+clean; 82 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.6 MiB/s wr, 9 op/s 2024-09-12T23:39:16.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:15 smithi181 bash[24776]: cluster 2024-09-12T23:39:14.627733+0000 mon.a (mon.0) 1145 : cluster [DBG] osdmap e118: 8 total, 8 up, 8 in 2024-09-12T23:39:16.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:15 smithi181 bash[24776]: cluster 2024-09-12T23:39:15.455714+0000 mon.a (mon.0) 1146 : cluster [DBG] osdmap e119: 8 total, 8 up, 8 in 2024-09-12T23:39:16.455 INFO:tasks.rados.rados.0.smithi136.stdout:315: done (3 left) 2024-09-12T23:39:16.455 INFO:tasks.rados.rados.0.smithi136.stdout:316: read oid 35 snap -1 2024-09-12T23:39:16.455 INFO:tasks.rados.rados.0.smithi136.stdout:316: expect deleted 2024-09-12T23:39:16.455 INFO:tasks.rados.rados.0.smithi136.stdout:317: read oid 22 snap -1 2024-09-12T23:39:16.456 INFO:tasks.rados.rados.0.smithi136.stdout:317: expect (ObjNum 21 snap 0 seq_num 21) 2024-09-12T23:39:16.456 INFO:tasks.rados.rados.0.smithi136.stdout:318: copy_from oid 7 from oid 20 current snap is 29 2024-09-12T23:39:16.456 INFO:tasks.rados.rados.0.smithi136.stdout:310: expect (ObjNum 115 snap 28 seq_num 115) 2024-09-12T23:39:16.505 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:39:16 smithi136 bash[22852]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:39:16] "GET /metrics HTTP/1.1" 200 240003 "" "Prometheus/2.43.0" 2024-09-12T23:39:16.641 INFO:tasks.rados.rados.0.smithi136.stdout:313: expect (ObjNum 108 snap 23 seq_num 108) 2024-09-12T23:39:16.784 INFO:tasks.rados.rados.0.smithi136.stdout:314: finishing write tid 1 to smithi13667081-48 2024-09-12T23:39:16.785 INFO:tasks.rados.rados.0.smithi136.stdout:314: finishing write tid 2 to smithi13667081-48 2024-09-12T23:39:16.785 INFO:tasks.rados.rados.0.smithi136.stdout:314: finishing write tid 3 to smithi13667081-48 2024-09-12T23:39:16.785 INFO:tasks.rados.rados.0.smithi136.stdout:310: done (5 left) 2024-09-12T23:39:16.785 INFO:tasks.rados.rados.0.smithi136.stdout:313: done (4 left) 2024-09-12T23:39:16.785 INFO:tasks.rados.rados.0.smithi136.stdout:319: delete oid 9 current snap is 29 2024-09-12T23:39:16.786 INFO:tasks.rados.rados.0.smithi136.stdout:314: finishing write tid 4 to smithi13667081-48 2024-09-12T23:39:16.786 INFO:tasks.rados.rados.0.smithi136.stdout:314: finishing write tid 5 to smithi13667081-48 2024-09-12T23:39:16.786 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 48 v 132 (ObjNum 118 snap 29 seq_num 118) dirty exists 2024-09-12T23:39:16.786 INFO:tasks.rados.rados.0.smithi136.stdout:314: left oid 48 (ObjNum 118 snap 29 seq_num 118) 2024-09-12T23:39:16.786 INFO:tasks.rados.rados.0.smithi136.stdout:317: expect (ObjNum 21 snap 0 seq_num 21) 2024-09-12T23:39:16.795 INFO:tasks.rados.rados.0.smithi136.stdout:318: finishing copy_from racing read to smithi13667081-7 2024-09-12T23:39:16.796 INFO:tasks.rados.rados.0.smithi136.stdout:318: finishing copy_from to smithi13667081-7 2024-09-12T23:39:16.796 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 7 v 14 (ObjNum 38 snap 0 seq_num 38) dirty exists 2024-09-12T23:39:16.796 INFO:tasks.rados.rados.0.smithi136.stdout:314: done (4 left) 2024-09-12T23:39:16.796 INFO:tasks.rados.rados.0.smithi136.stdout:316: done (3 left) 2024-09-12T23:39:16.796 INFO:tasks.rados.rados.0.smithi136.stdout:317: done (2 left) 2024-09-12T23:39:16.796 INFO:tasks.rados.rados.0.smithi136.stdout:318: done (1 left) 2024-09-12T23:39:16.796 INFO:tasks.rados.rados.0.smithi136.stdout:319: done (0 left) 2024-09-12T23:39:16.796 INFO:tasks.rados.rados.0.smithi136.stdout:320: snap_create 2024-09-12T23:39:17.631 INFO:tasks.rados.rados.0.smithi136.stdout:320: done (0 left) 2024-09-12T23:39:17.631 INFO:tasks.rados.rados.0.smithi136.stdout:321: delete oid 19 current snap is 30 2024-09-12T23:39:17.638 INFO:tasks.rados.rados.0.smithi136.stdout:321: done (0 left) 2024-09-12T23:39:17.638 INFO:tasks.rados.rados.0.smithi136.stdout:322: delete oid 8 current snap is 30 2024-09-12T23:39:17.647 INFO:tasks.rados.rados.0.smithi136.stdout:322: done (0 left) 2024-09-12T23:39:17.647 INFO:tasks.rados.rados.0.smithi136.stdout:323: delete oid 49 current snap is 30 2024-09-12T23:39:17.652 INFO:tasks.rados.rados.0.smithi136.stdout:323: done (0 left) 2024-09-12T23:39:17.653 INFO:tasks.rados.rados.0.smithi136.stdout:324: copy_from oid 33 from oid 9 current snap is 30 2024-09-12T23:39:17.653 INFO:tasks.rados.rados.0.smithi136.stdout:325: read oid 27 snap -1 2024-09-12T23:39:17.653 INFO:tasks.rados.rados.0.smithi136.stdout:325: expect (ObjNum 26 snap 0 seq_num 26) 2024-09-12T23:39:17.653 INFO:tasks.rados.rados.0.smithi136.stdout:326: rollback oid 42 current snap is 30 2024-09-12T23:39:17.653 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 42 to 27 2024-09-12T23:39:17.653 INFO:tasks.rados.rados.0.smithi136.stdout:327: read oid 29 snap -1 2024-09-12T23:39:17.653 INFO:tasks.rados.rados.0.smithi136.stdout:327: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-12T23:39:17.653 INFO:tasks.rados.rados.0.smithi136.stdout:328: rollback oid 11 current snap is 30 2024-09-12T23:39:17.653 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 11 to 21 2024-09-12T23:39:17.654 INFO:tasks.rados.rados.0.smithi136.stdout:329: read oid 21 snap 13 2024-09-12T23:39:17.654 INFO:tasks.rados.rados.0.smithi136.stdout:329: expect (ObjNum 34 snap 0 seq_num 34) 2024-09-12T23:39:17.654 INFO:tasks.rados.rados.0.smithi136.stdout:330: snap_create 2024-09-12T23:39:17.660 INFO:tasks.rados.rados.0.smithi136.stdout:327: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-12T23:39:17.660 INFO:tasks.rados.rados.0.smithi136.stdout:326: finishing rollback tid 1 to smithi13667081-42 2024-09-12T23:39:17.660 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 42 v 0 (ObjNum 1427291360 snap 32541 seq_num 304) dirty dne 2024-09-12T23:39:17.660 INFO:tasks.rados.rados.0.smithi136.stdout:324: finishing copy_from to smithi13667081-33 2024-09-12T23:39:17.661 INFO:tasks.rados.rados.0.smithi136.stdout:324: got expected ENOENT (src dne) 2024-09-12T23:39:17.661 INFO:tasks.rados.rados.0.smithi136.stdout:324: finishing copy_from racing read to smithi13667081-33 2024-09-12T23:39:17.662 INFO:tasks.rados.rados.0.smithi136.stdout:329: expect (ObjNum 34 snap 0 seq_num 34) 2024-09-12T23:39:17.689 INFO:tasks.rados.rados.0.smithi136.stdout:325: expect (ObjNum 26 snap 0 seq_num 26) 2024-09-12T23:39:17.708 INFO:tasks.rados.rados.0.smithi136.stdout:328: finishing rollback tid 1 to smithi13667081-11 2024-09-12T23:39:17.708 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 11 v 51 (ObjNum 65 snap 0 seq_num 1616079456) dirty dne 2024-09-12T23:39:17.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:17 smithi136 bash[22601]: cluster 2024-09-12T23:39:16.034254+0000 mgr.y (mgr.24446) 219 : cluster [DBG] pgmap v221: 121 pgs: 121 active+clean; 82 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.6 MiB/s wr, 9 op/s 2024-09-12T23:39:17.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:17 smithi136 bash[22601]: cluster 2024-09-12T23:39:16.459108+0000 mon.a (mon.0) 1147 : cluster [DBG] osdmap e120: 8 total, 8 up, 8 in 2024-09-12T23:39:17.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:17 smithi136 bash[32189]: cluster 2024-09-12T23:39:16.034254+0000 mgr.y (mgr.24446) 219 : cluster [DBG] pgmap v221: 121 pgs: 121 active+clean; 82 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.6 MiB/s wr, 9 op/s 2024-09-12T23:39:17.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:17 smithi136 bash[32189]: cluster 2024-09-12T23:39:16.459108+0000 mon.a (mon.0) 1147 : cluster [DBG] osdmap e120: 8 total, 8 up, 8 in 2024-09-12T23:39:17.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:17 smithi181 bash[24776]: cluster 2024-09-12T23:39:16.034254+0000 mgr.y (mgr.24446) 219 : cluster [DBG] pgmap v221: 121 pgs: 121 active+clean; 82 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.6 MiB/s wr, 9 op/s 2024-09-12T23:39:17.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:17 smithi181 bash[24776]: cluster 2024-09-12T23:39:16.459108+0000 mon.a (mon.0) 1147 : cluster [DBG] osdmap e120: 8 total, 8 up, 8 in 2024-09-12T23:39:18.642 INFO:tasks.rados.rados.0.smithi136.stdout:324: done (6 left) 2024-09-12T23:39:18.642 INFO:tasks.rados.rados.0.smithi136.stdout:325: done (5 left) 2024-09-12T23:39:18.642 INFO:tasks.rados.rados.0.smithi136.stdout:326: done (4 left) 2024-09-12T23:39:18.642 INFO:tasks.rados.rados.0.smithi136.stdout:327: done (3 left) 2024-09-12T23:39:18.642 INFO:tasks.rados.rados.0.smithi136.stdout:328: done (2 left) 2024-09-12T23:39:18.642 INFO:tasks.rados.rados.0.smithi136.stdout:329: done (1 left) 2024-09-12T23:39:18.643 INFO:tasks.rados.rados.0.smithi136.stdout:330: done (0 left) 2024-09-12T23:39:18.643 INFO:tasks.rados.rados.0.smithi136.stdout:331: copy_from oid 45 from oid 39 current snap is 31 2024-09-12T23:39:18.643 INFO:tasks.rados.rados.0.smithi136.stdout:332: snap_create 2024-09-12T23:39:18.669 INFO:tasks.rados.rados.0.smithi136.stdout:331: finishing copy_from racing read to smithi13667081-45 2024-09-12T23:39:18.672 INFO:tasks.rados.rados.0.smithi136.stdout:331: finishing copy_from to smithi13667081-45 2024-09-12T23:39:18.672 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 45 v 14 (ObjNum 38 snap 0 seq_num 38) dirty exists 2024-09-12T23:39:19.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:18 smithi136 bash[22601]: cluster 2024-09-12T23:39:17.634706+0000 mon.a (mon.0) 1148 : cluster [DBG] osdmap e121: 8 total, 8 up, 8 in 2024-09-12T23:39:19.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:18 smithi136 bash[32189]: cluster 2024-09-12T23:39:17.634706+0000 mon.a (mon.0) 1148 : cluster [DBG] osdmap e121: 8 total, 8 up, 8 in 2024-09-12T23:39:19.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:18 smithi181 bash[24776]: cluster 2024-09-12T23:39:17.634706+0000 mon.a (mon.0) 1148 : cluster [DBG] osdmap e121: 8 total, 8 up, 8 in 2024-09-12T23:39:19.643 INFO:tasks.rados.rados.0.smithi136.stdout:331: done (1 left) 2024-09-12T23:39:19.643 INFO:tasks.rados.rados.0.smithi136.stdout:332: done (0 left) 2024-09-12T23:39:19.643 INFO:tasks.rados.rados.0.smithi136.stdout:333: delete oid 32 current snap is 32 2024-09-12T23:39:19.657 INFO:tasks.rados.rados.0.smithi136.stdout:333: done (0 left) 2024-09-12T23:39:19.657 INFO:tasks.rados.rados.0.smithi136.stdout:334: delete oid 20 current snap is 32 2024-09-12T23:39:19.666 INFO:tasks.rados.rados.0.smithi136.stdout:334: done (0 left) 2024-09-12T23:39:19.666 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 42 current snap is 32 2024-09-12T23:39:19.666 INFO:tasks.rados.rados.0.smithi136.stdout:335: seq_num 125 ranges {0=499712,499712=745472,1245184=409600,1654784=450560} 2024-09-12T23:39:19.674 INFO:tasks.rados.rados.0.smithi136.stdout:335: writing smithi13667081-42 from 0 to 499712 tid 1 2024-09-12T23:39:19.687 INFO:tasks.rados.rados.0.smithi136.stdout:335: writing smithi13667081-42 from 499712 to 1245184 tid 2 2024-09-12T23:39:19.694 INFO:tasks.rados.rados.0.smithi136.stdout:335: writing smithi13667081-42 from 1245184 to 1654784 tid 3 2024-09-12T23:39:19.702 INFO:tasks.rados.rados.0.smithi136.stdout:335: writing smithi13667081-42 from 1654784 to 2105344 tid 4 2024-09-12T23:39:19.703 INFO:tasks.rados.rados.0.smithi136.stdout:336: snap_create 2024-09-12T23:39:19.703 INFO:tasks.rados.rados.0.smithi136.stdout:335: finishing write tid 1 to smithi13667081-42 2024-09-12T23:39:19.703 INFO:tasks.rados.rados.0.smithi136.stdout:335: finishing write tid 2 to smithi13667081-42 2024-09-12T23:39:19.703 INFO:tasks.rados.rados.0.smithi136.stdout:335: finishing write tid 3 to smithi13667081-42 2024-09-12T23:39:19.707 INFO:tasks.rados.rados.0.smithi136.stdout:335: finishing write tid 4 to smithi13667081-42 2024-09-12T23:39:19.707 INFO:tasks.rados.rados.0.smithi136.stdout:335: finishing write tid 5 to smithi13667081-42 2024-09-12T23:39:19.710 INFO:tasks.rados.rados.0.smithi136.stdout:335: finishing write tid 6 to smithi13667081-42 2024-09-12T23:39:19.710 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 42 v 67 (ObjNum 125 snap 32 seq_num 125) dirty exists 2024-09-12T23:39:19.710 INFO:tasks.rados.rados.0.smithi136.stdout:335: left oid 42 (ObjNum 125 snap 32 seq_num 125) 2024-09-12T23:39:20.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:19 smithi136 bash[22601]: cluster 2024-09-12T23:39:18.034920+0000 mgr.y (mgr.24446) 220 : cluster [DBG] pgmap v224: 121 pgs: 121 active+clean; 82 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:39:20.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:19 smithi136 bash[22601]: cluster 2024-09-12T23:39:18.646337+0000 mon.a (mon.0) 1149 : cluster [DBG] osdmap e122: 8 total, 8 up, 8 in 2024-09-12T23:39:20.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:19 smithi136 bash[32189]: cluster 2024-09-12T23:39:18.034920+0000 mgr.y (mgr.24446) 220 : cluster [DBG] pgmap v224: 121 pgs: 121 active+clean; 82 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:39:20.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:19 smithi136 bash[32189]: cluster 2024-09-12T23:39:18.646337+0000 mon.a (mon.0) 1149 : cluster [DBG] osdmap e122: 8 total, 8 up, 8 in 2024-09-12T23:39:20.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:19 smithi181 bash[24776]: cluster 2024-09-12T23:39:18.034920+0000 mgr.y (mgr.24446) 220 : cluster [DBG] pgmap v224: 121 pgs: 121 active+clean; 82 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:39:20.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:19 smithi181 bash[24776]: cluster 2024-09-12T23:39:18.646337+0000 mon.a (mon.0) 1149 : cluster [DBG] osdmap e122: 8 total, 8 up, 8 in 2024-09-12T23:39:20.667 INFO:tasks.rados.rados.0.smithi136.stdout:335: done (1 left) 2024-09-12T23:39:20.667 INFO:tasks.rados.rados.0.smithi136.stdout:336: done (0 left) 2024-09-12T23:39:20.667 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 5 current snap is 33 2024-09-12T23:39:20.667 INFO:tasks.rados.rados.0.smithi136.stdout:337: seq_num 126 ranges {0=516096,516096=630784,1146880=614400,1761280=360448} 2024-09-12T23:39:20.675 INFO:tasks.rados.rados.0.smithi136.stdout:337: writing smithi13667081-5 from 0 to 516096 tid 1 2024-09-12T23:39:20.687 INFO:tasks.rados.rados.0.smithi136.stdout:337: writing smithi13667081-5 from 516096 to 1146880 tid 2 2024-09-12T23:39:20.697 INFO:tasks.rados.rados.0.smithi136.stdout:337: writing smithi13667081-5 from 1146880 to 1761280 tid 3 2024-09-12T23:39:20.704 INFO:tasks.rados.rados.0.smithi136.stdout:337: writing smithi13667081-5 from 1761280 to 2121728 tid 4 2024-09-12T23:39:20.704 INFO:tasks.rados.rados.0.smithi136.stdout:338: rollback oid 31 current snap is 33 2024-09-12T23:39:20.704 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 31 to 22 2024-09-12T23:39:20.704 INFO:tasks.rados.rados.0.smithi136.stdout:337: finishing write tid 1 to smithi13667081-5 2024-09-12T23:39:20.704 INFO:tasks.rados.rados.0.smithi136.stdout:337: finishing write tid 2 to smithi13667081-5 2024-09-12T23:39:20.704 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 49 current snap is 33 2024-09-12T23:39:20.704 INFO:tasks.rados.rados.0.smithi136.stdout:337: finishing write tid 3 to smithi13667081-5 2024-09-12T23:39:20.704 INFO:tasks.rados.rados.0.smithi136.stdout:339: seq_num 127 ranges {0=532480,532480=507904,1040384=778240,1818624=319488} 2024-09-12T23:39:20.714 INFO:tasks.rados.rados.0.smithi136.stdout:339: writing smithi13667081-49 from 0 to 532480 tid 1 2024-09-12T23:39:20.723 INFO:tasks.rados.rados.0.smithi136.stdout:339: writing smithi13667081-49 from 532480 to 1040384 tid 2 2024-09-12T23:39:20.734 INFO:tasks.rados.rados.0.smithi136.stdout:339: writing smithi13667081-49 from 1040384 to 1818624 tid 3 2024-09-12T23:39:20.739 INFO:tasks.rados.rados.0.smithi136.stdout:339: writing smithi13667081-49 from 1818624 to 2138112 tid 4 2024-09-12T23:39:20.739 INFO:tasks.rados.rados.0.smithi136.stdout:340: rmattr oid 41 current snap is 33 2024-09-12T23:39:20.739 INFO:tasks.rados.rados.0.smithi136.stdout:340: done (3 left) 2024-09-12T23:39:20.739 INFO:tasks.rados.rados.0.smithi136.stdout:341: read oid 7 snap -1 2024-09-12T23:39:20.739 INFO:tasks.rados.rados.0.smithi136.stdout:341: expect (ObjNum 38 snap 0 seq_num 38) 2024-09-12T23:39:20.740 INFO:tasks.rados.rados.0.smithi136.stdout:342: snap_remove snap 30 2024-09-12T23:39:21.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:20 smithi136 bash[22601]: cluster 2024-09-12T23:39:19.647025+0000 mon.a (mon.0) 1150 : cluster [DBG] osdmap e123: 8 total, 8 up, 8 in 2024-09-12T23:39:21.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:20 smithi136 bash[32189]: cluster 2024-09-12T23:39:19.647025+0000 mon.a (mon.0) 1150 : cluster [DBG] osdmap e123: 8 total, 8 up, 8 in 2024-09-12T23:39:21.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:20 smithi181 bash[24776]: cluster 2024-09-12T23:39:19.647025+0000 mon.a (mon.0) 1150 : cluster [DBG] osdmap e123: 8 total, 8 up, 8 in 2024-09-12T23:39:21.678 INFO:tasks.rados.rados.0.smithi136.stdout:342: done (4 left) 2024-09-12T23:39:21.679 INFO:tasks.rados.rados.0.smithi136.stdout:343: rollback oid 8 current snap is 33 2024-09-12T23:39:21.679 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 8 to 27 2024-09-12T23:39:21.679 INFO:tasks.rados.rados.0.smithi136.stdout:337: finishing write tid 4 to smithi13667081-5 2024-09-12T23:39:21.679 INFO:tasks.rados.rados.0.smithi136.stdout:337: finishing write tid 5 to smithi13667081-5 2024-09-12T23:39:21.679 INFO:tasks.rados.rados.0.smithi136.stdout:338: finishing rollback tid 1 to smithi13667081-31 2024-09-12T23:39:21.679 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 31 v 90 (ObjNum 145 snap 0 seq_num 4243166848) dirty dne 2024-09-12T23:39:21.679 INFO:tasks.rados.rados.0.smithi136.stdout:337: finishing write tid 6 to smithi13667081-5 2024-09-12T23:39:21.679 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 5 v 148 (ObjNum 126 snap 33 seq_num 126) dirty exists 2024-09-12T23:39:21.679 INFO:tasks.rados.rados.0.smithi136.stdout:337: left oid 5 (ObjNum 126 snap 33 seq_num 126) 2024-09-12T23:39:21.679 INFO:tasks.rados.rados.0.smithi136.stdout:339: finishing write tid 1 to smithi13667081-49 2024-09-12T23:39:21.679 INFO:tasks.rados.rados.0.smithi136.stdout:339: finishing write tid 2 to smithi13667081-49 2024-09-12T23:39:21.679 INFO:tasks.rados.rados.0.smithi136.stdout:339: finishing write tid 3 to smithi13667081-49 2024-09-12T23:39:21.679 INFO:tasks.rados.rados.0.smithi136.stdout:339: finishing write tid 4 to smithi13667081-49 2024-09-12T23:39:21.680 INFO:tasks.rados.rados.0.smithi136.stdout:339: finishing write tid 5 to smithi13667081-49 2024-09-12T23:39:21.680 INFO:tasks.rados.rados.0.smithi136.stdout:339: finishing write tid 6 to smithi13667081-49 2024-09-12T23:39:21.680 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 49 v 180 (ObjNum 127 snap 33 seq_num 127) dirty exists 2024-09-12T23:39:21.680 INFO:tasks.rados.rados.0.smithi136.stdout:339: left oid 49 (ObjNum 127 snap 33 seq_num 127) 2024-09-12T23:39:21.680 INFO:tasks.rados.rados.0.smithi136.stdout:341: expect (ObjNum 38 snap 0 seq_num 38) 2024-09-12T23:39:21.733 INFO:tasks.rados.rados.0.smithi136.stdout:343: finishing rollback tid 1 to smithi13667081-8 2024-09-12T23:39:21.733 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 8 v 0 (ObjNum 65 snap 0 seq_num 1649300571) dirty dne 2024-09-12T23:39:21.733 INFO:tasks.rados.rados.0.smithi136.stdout:337: done (4 left) 2024-09-12T23:39:21.733 INFO:tasks.rados.rados.0.smithi136.stdout:338: done (3 left) 2024-09-12T23:39:21.733 INFO:tasks.rados.rados.0.smithi136.stdout:339: done (2 left) 2024-09-12T23:39:21.733 INFO:tasks.rados.rados.0.smithi136.stdout:341: done (1 left) 2024-09-12T23:39:21.733 INFO:tasks.rados.rados.0.smithi136.stdout:343: done (0 left) 2024-09-12T23:39:21.733 INFO:tasks.rados.rados.0.smithi136.stdout:344: read oid 1 snap -1 2024-09-12T23:39:21.733 INFO:tasks.rados.rados.0.smithi136.stdout:344: expect deleted 2024-09-12T23:39:21.733 INFO:tasks.rados.rados.0.smithi136.stdout:345: read oid 14 snap -1 2024-09-12T23:39:21.733 INFO:tasks.rados.rados.0.smithi136.stdout:345: expect (ObjNum 102 snap 22 seq_num 102) 2024-09-12T23:39:21.733 INFO:tasks.rados.rados.0.smithi136.stdout:346: delete oid 29 current snap is 33 2024-09-12T23:39:21.735 INFO:tasks.rados.rados.0.smithi136.stdout:344: done (2 left) 2024-09-12T23:39:21.736 INFO:tasks.rados.rados.0.smithi136.stdout:346: done (1 left) 2024-09-12T23:39:21.736 INFO:tasks.rados.rados.0.smithi136.stdout:347: rollback oid 12 current snap is 33 2024-09-12T23:39:21.736 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 12 to 29 2024-09-12T23:39:21.736 INFO:tasks.rados.rados.0.smithi136.stdout:348: snap_remove snap 27 2024-09-12T23:39:22.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:21 smithi136 bash[22601]: cluster 2024-09-12T23:39:20.035604+0000 mgr.y (mgr.24446) 221 : cluster [DBG] pgmap v227: 121 pgs: 121 active+clean; 84 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.5 MiB/s rd, 798 KiB/s wr, 11 op/s 2024-09-12T23:39:22.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:21 smithi136 bash[22601]: cluster 2024-09-12T23:39:20.671317+0000 mon.a (mon.0) 1151 : cluster [DBG] osdmap e124: 8 total, 8 up, 8 in 2024-09-12T23:39:22.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:21 smithi136 bash[32189]: cluster 2024-09-12T23:39:20.035604+0000 mgr.y (mgr.24446) 221 : cluster [DBG] pgmap v227: 121 pgs: 121 active+clean; 84 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.5 MiB/s rd, 798 KiB/s wr, 11 op/s 2024-09-12T23:39:22.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:21 smithi136 bash[32189]: cluster 2024-09-12T23:39:20.671317+0000 mon.a (mon.0) 1151 : cluster [DBG] osdmap e124: 8 total, 8 up, 8 in 2024-09-12T23:39:22.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:21 smithi181 bash[24776]: cluster 2024-09-12T23:39:20.035604+0000 mgr.y (mgr.24446) 221 : cluster [DBG] pgmap v227: 121 pgs: 121 active+clean; 84 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.5 MiB/s rd, 798 KiB/s wr, 11 op/s 2024-09-12T23:39:22.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:21 smithi181 bash[24776]: cluster 2024-09-12T23:39:20.671317+0000 mon.a (mon.0) 1151 : cluster [DBG] osdmap e124: 8 total, 8 up, 8 in 2024-09-12T23:39:22.688 INFO:tasks.rados.rados.0.smithi136.stdout:348: done (2 left) 2024-09-12T23:39:22.688 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 18 current snap is 33 2024-09-12T23:39:22.688 INFO:tasks.rados.rados.0.smithi136.stdout:349: seq_num 129 ranges {2818048=565248,3383296=679936,4063232=786432,4849664=139264} 2024-09-12T23:39:22.743 INFO:tasks.rados.rados.0.smithi136.stdout:349: writing smithi13667081-18 from 2818048 to 3383296 tid 1 2024-09-12T23:39:22.752 INFO:tasks.rados.rados.0.smithi136.stdout:349: writing smithi13667081-18 from 3383296 to 4063232 tid 2 2024-09-12T23:39:22.763 INFO:tasks.rados.rados.0.smithi136.stdout:349: writing smithi13667081-18 from 4063232 to 4849664 tid 3 2024-09-12T23:39:22.763 INFO:tasks.rados.rados.0.smithi136.stdout:349: writing smithi13667081-18 from 4849664 to 4988928 tid 4 2024-09-12T23:39:22.766 INFO:tasks.rados.rados.0.smithi136.stdout:350: read oid 32 snap -1 2024-09-12T23:39:22.766 INFO:tasks.rados.rados.0.smithi136.stdout:350: expect deleted 2024-09-12T23:39:22.766 INFO:tasks.rados.rados.0.smithi136.stdout:351: setattr oid 35 current snap is 33 2024-09-12T23:39:22.766 INFO:tasks.rados.rados.0.smithi136.stdout:347: finishing rollback tid 0 to smithi13667081-12 2024-09-12T23:39:22.766 INFO:tasks.rados.rados.0.smithi136.stdout:347: finishing rollback tid 1 to smithi13667081-12 2024-09-12T23:39:22.766 INFO:tasks.rados.rados.0.smithi136.stdout:347: finishing rollback tid 2 to smithi13667081-12 2024-09-12T23:39:22.766 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 12 v 184 (ObjNum 70 snap 7 seq_num 70) dirty exists 2024-09-12T23:39:22.766 INFO:tasks.rados.rados.0.smithi136.stdout:345: expect (ObjNum 102 snap 22 seq_num 102) 2024-09-12T23:39:22.863 INFO:tasks.rados.rados.0.smithi136.stdout:349: finishing write tid 1 to smithi13667081-18 2024-09-12T23:39:22.863 INFO:tasks.rados.rados.0.smithi136.stdout:349: finishing write tid 2 to smithi13667081-18 2024-09-12T23:39:22.863 INFO:tasks.rados.rados.0.smithi136.stdout:349: finishing write tid 3 to smithi13667081-18 2024-09-12T23:39:22.863 INFO:tasks.rados.rados.0.smithi136.stdout:349: finishing write tid 4 to smithi13667081-18 2024-09-12T23:39:22.863 INFO:tasks.rados.rados.0.smithi136.stdout:349: finishing write tid 5 to smithi13667081-18 2024-09-12T23:39:22.863 INFO:tasks.rados.rados.0.smithi136.stdout:349: finishing write tid 6 to smithi13667081-18 2024-09-12T23:39:22.863 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 18 v 52 (ObjNum 129 snap 33 seq_num 129) dirty exists 2024-09-12T23:39:22.863 INFO:tasks.rados.rados.0.smithi136.stdout:349: left oid 18 (ObjNum 129 snap 33 seq_num 129) 2024-09-12T23:39:22.863 INFO:tasks.rados.rados.0.smithi136.stdout:345: done (4 left) 2024-09-12T23:39:22.864 INFO:tasks.rados.rados.0.smithi136.stdout:347: done (3 left) 2024-09-12T23:39:22.864 INFO:tasks.rados.rados.0.smithi136.stdout:349: done (2 left) 2024-09-12T23:39:22.864 INFO:tasks.rados.rados.0.smithi136.stdout:350: done (1 left) 2024-09-12T23:39:22.864 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 36 current snap is 33 2024-09-12T23:39:22.864 INFO:tasks.rados.rados.0.smithi136.stdout:352: seq_num 130 ranges {172032=581632,753664=614400,1368064=598016,1966080=393216} 2024-09-12T23:39:22.869 INFO:tasks.rados.rados.0.smithi136.stdout:352: writing smithi13667081-36 from 172032 to 753664 tid 1 2024-09-12T23:39:22.875 INFO:tasks.rados.rados.0.smithi136.stdout:352: writing smithi13667081-36 from 753664 to 1368064 tid 2 2024-09-12T23:39:22.880 INFO:tasks.rados.rados.0.smithi136.stdout:352: writing smithi13667081-36 from 1368064 to 1966080 tid 3 2024-09-12T23:39:22.884 INFO:tasks.rados.rados.0.smithi136.stdout:352: writing smithi13667081-36 from 1966080 to 2359296 tid 4 2024-09-12T23:39:22.884 INFO:tasks.rados.rados.0.smithi136.stdout:353: snap_create 2024-09-12T23:39:22.884 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 35 v 151 (ObjNum 289 snap 0 seq_num 0) dirty exists 2024-09-12T23:39:22.884 INFO:tasks.rados.rados.0.smithi136.stdout:352: finishing write tid 1 to smithi13667081-36 2024-09-12T23:39:22.884 INFO:tasks.rados.rados.0.smithi136.stdout:352: finishing write tid 2 to smithi13667081-36 2024-09-12T23:39:22.884 INFO:tasks.rados.rados.0.smithi136.stdout:352: finishing write tid 3 to smithi13667081-36 2024-09-12T23:39:22.886 INFO:tasks.rados.rados.0.smithi136.stdout:352: finishing write tid 4 to smithi13667081-36 2024-09-12T23:39:22.886 INFO:tasks.rados.rados.0.smithi136.stdout:352: finishing write tid 5 to smithi13667081-36 2024-09-12T23:39:22.889 INFO:tasks.rados.rados.0.smithi136.stdout:352: finishing write tid 6 to smithi13667081-36 2024-09-12T23:39:22.889 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 36 v 91 (ObjNum 130 snap 33 seq_num 130) dirty exists 2024-09-12T23:39:22.890 INFO:tasks.rados.rados.0.smithi136.stdout:352: left oid 36 (ObjNum 130 snap 33 seq_num 130) 2024-09-12T23:39:23.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:22 smithi136 bash[22601]: cluster 2024-09-12T23:39:21.678732+0000 mon.a (mon.0) 1152 : cluster [DBG] osdmap e125: 8 total, 8 up, 8 in 2024-09-12T23:39:23.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:22 smithi136 bash[32189]: cluster 2024-09-12T23:39:21.678732+0000 mon.a (mon.0) 1152 : cluster [DBG] osdmap e125: 8 total, 8 up, 8 in 2024-09-12T23:39:23.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:22 smithi181 bash[24776]: cluster 2024-09-12T23:39:21.678732+0000 mon.a (mon.0) 1152 : cluster [DBG] osdmap e125: 8 total, 8 up, 8 in 2024-09-12T23:39:23.687 INFO:tasks.rados.rados.0.smithi136.stdout:351: done (2 left) 2024-09-12T23:39:23.687 INFO:tasks.rados.rados.0.smithi136.stdout:352: done (1 left) 2024-09-12T23:39:23.687 INFO:tasks.rados.rados.0.smithi136.stdout:353: done (0 left) 2024-09-12T23:39:23.687 INFO:tasks.rados.rados.0.smithi136.stdout:354: read oid 25 snap 29 2024-09-12T23:39:23.687 INFO:tasks.rados.rados.0.smithi136.stdout:354: expect (ObjNum 36 snap 0 seq_num 36) 2024-09-12T23:39:23.687 INFO:tasks.rados.rados.0.smithi136.stdout:355: snap_remove snap 26 2024-09-12T23:39:24.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:23 smithi136 bash[22601]: cluster 2024-09-12T23:39:22.036310+0000 mgr.y (mgr.24446) 222 : cluster [DBG] pgmap v230: 121 pgs: 121 active+clean; 84 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.5 MiB/s rd, 798 KiB/s wr, 11 op/s 2024-09-12T23:39:24.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:23 smithi136 bash[22601]: cluster 2024-09-12T23:39:22.691593+0000 mon.a (mon.0) 1153 : cluster [DBG] osdmap e126: 8 total, 8 up, 8 in 2024-09-12T23:39:24.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:23 smithi136 bash[22601]: audit 2024-09-12T23:39:23.014754+0000 mon.a (mon.0) 1154 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:39:24.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:23 smithi136 bash[32189]: cluster 2024-09-12T23:39:22.036310+0000 mgr.y (mgr.24446) 222 : cluster [DBG] pgmap v230: 121 pgs: 121 active+clean; 84 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.5 MiB/s rd, 798 KiB/s wr, 11 op/s 2024-09-12T23:39:24.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:23 smithi136 bash[32189]: cluster 2024-09-12T23:39:22.691593+0000 mon.a (mon.0) 1153 : cluster [DBG] osdmap e126: 8 total, 8 up, 8 in 2024-09-12T23:39:24.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:23 smithi136 bash[32189]: audit 2024-09-12T23:39:23.014754+0000 mon.a (mon.0) 1154 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:39:24.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:23 smithi181 bash[24776]: cluster 2024-09-12T23:39:22.036310+0000 mgr.y (mgr.24446) 222 : cluster [DBG] pgmap v230: 121 pgs: 121 active+clean; 84 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.5 MiB/s rd, 798 KiB/s wr, 11 op/s 2024-09-12T23:39:24.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:23 smithi181 bash[24776]: cluster 2024-09-12T23:39:22.691593+0000 mon.a (mon.0) 1153 : cluster [DBG] osdmap e126: 8 total, 8 up, 8 in 2024-09-12T23:39:24.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:23 smithi181 bash[24776]: audit 2024-09-12T23:39:23.014754+0000 mon.a (mon.0) 1154 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:39:24.690 INFO:tasks.rados.rados.0.smithi136.stdout:355: done (1 left) 2024-09-12T23:39:24.690 INFO:tasks.rados.rados.0.smithi136.stdout:356: copy_from oid 9 from oid 36 current snap is 34 2024-09-12T23:39:24.690 INFO:tasks.rados.rados.0.smithi136.stdout:354: expect (ObjNum 36 snap 0 seq_num 36) 2024-09-12T23:39:24.722 INFO:tasks.rados.rados.0.smithi136.stdout:354: done (1 left) 2024-09-12T23:39:24.722 INFO:tasks.rados.rados.0.smithi136.stdout:357: snap_create 2024-09-12T23:39:24.723 INFO:tasks.rados.rados.0.smithi136.stdout:356: finishing copy_from racing read to smithi13667081-9 2024-09-12T23:39:24.726 INFO:tasks.rados.rados.0.smithi136.stdout:356: finishing copy_from to smithi13667081-9 2024-09-12T23:39:24.727 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 9 v 91 (ObjNum 130 snap 33 seq_num 130) dirty exists 2024-09-12T23:39:25.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:24 smithi136 bash[22601]: cluster 2024-09-12T23:39:23.690715+0000 mon.a (mon.0) 1155 : cluster [DBG] osdmap e127: 8 total, 8 up, 8 in 2024-09-12T23:39:25.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:24 smithi136 bash[32189]: cluster 2024-09-12T23:39:23.690715+0000 mon.a (mon.0) 1155 : cluster [DBG] osdmap e127: 8 total, 8 up, 8 in 2024-09-12T23:39:25.007 INFO:teuthology.orchestra.run.smithi136.stdout:true 2024-09-12T23:39:25.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:24 smithi181 bash[24776]: cluster 2024-09-12T23:39:23.690715+0000 mon.a (mon.0) 1155 : cluster [DBG] osdmap e127: 8 total, 8 up, 8 in 2024-09-12T23:39:25.282 INFO:teuthology.orchestra.run.smithi136.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-12T23:39:25.283 INFO:teuthology.orchestra.run.smithi136.stdout:alertmanager.a smithi136 *:9093,9094 running (4m) 3m ago 4m 14.3M - 0.25.0 c8568f914cd2 d6c106834390 2024-09-12T23:39:25.283 INFO:teuthology.orchestra.run.smithi136.stdout:grafana.a smithi181 *:3000 running (4m) 3m ago 4m 29.0M - 9.4.7 954c08fa6188 85ee0f95160c 2024-09-12T23:39:25.283 INFO:teuthology.orchestra.run.smithi136.stdout:mds.a smithi136 running (6m) 3m ago 6m 12.1M - 17.2.7-1627-gbbdadfa3 214985049bdb 713b9795ad19 2024-09-12T23:39:25.283 INFO:teuthology.orchestra.run.smithi136.stdout:mds.b smithi181 running (6m) 3m ago 6m 13.9M - 17.2.7-1627-gbbdadfa3 214985049bdb ad13179f0868 2024-09-12T23:39:25.283 INFO:teuthology.orchestra.run.smithi136.stdout:mgr.x smithi181 *:8443 running (13m) 3m ago 13m 387M - 17.2.7-1627-gbbdadfa3 214985049bdb d07cbbb66db7 2024-09-12T23:39:25.283 INFO:teuthology.orchestra.run.smithi136.stdout:mgr.y smithi136 *:9283,8765 running (15m) 3m ago 15m 436M - 17.2.7-1627-gbbdadfa3 214985049bdb 56608cdf6b27 2024-09-12T23:39:25.283 INFO:teuthology.orchestra.run.smithi136.stdout:mon.a smithi136 running (15m) 3m ago 15m 44.1M 2048M 17.2.7-1627-gbbdadfa3 214985049bdb 0462f4f61475 2024-09-12T23:39:25.283 INFO:teuthology.orchestra.run.smithi136.stdout:mon.b smithi181 running (13m) 3m ago 13m 37.6M 2048M 17.2.7-1627-gbbdadfa3 214985049bdb 8b3d784af845 2024-09-12T23:39:25.283 INFO:teuthology.orchestra.run.smithi136.stdout:mon.c smithi136 running (13m) 3m ago 13m 36.8M 2048M 17.2.7-1627-gbbdadfa3 214985049bdb 647b028ea963 2024-09-12T23:39:25.283 INFO:teuthology.orchestra.run.smithi136.stdout:node-exporter.a smithi136 *:9100 running (5m) 3m ago 5m 3143k - 1.5.0 0da6a335fe13 8f54fc5b0da3 2024-09-12T23:39:25.283 INFO:teuthology.orchestra.run.smithi136.stdout:node-exporter.b smithi181 *:9100 running (5m) 3m ago 5m 2895k - 1.5.0 0da6a335fe13 575eff556149 2024-09-12T23:39:25.284 INFO:teuthology.orchestra.run.smithi136.stdout:osd.0 smithi136 running (12m) 3m ago 12m 47.0M 3036M 17.2.7-1627-gbbdadfa3 214985049bdb cbc0b7df0115 2024-09-12T23:39:25.284 INFO:teuthology.orchestra.run.smithi136.stdout:osd.1 smithi136 running (11m) 3m ago 11m 44.4M 3036M 17.2.7-1627-gbbdadfa3 214985049bdb c4daee7da836 2024-09-12T23:39:25.284 INFO:teuthology.orchestra.run.smithi136.stdout:osd.2 smithi136 running (10m) 3m ago 11m 44.8M 3036M 17.2.7-1627-gbbdadfa3 214985049bdb 4ab8fd79cdb4 2024-09-12T23:39:25.284 INFO:teuthology.orchestra.run.smithi136.stdout:osd.3 smithi136 running (10m) 3m ago 10m 42.5M 3036M 17.2.7-1627-gbbdadfa3 214985049bdb 9ceb14a27521 2024-09-12T23:39:25.284 INFO:teuthology.orchestra.run.smithi136.stdout:osd.4 smithi181 running (9m) 3m ago 9m 42.7M 3036M 17.2.7-1627-gbbdadfa3 214985049bdb a35cdca79055 2024-09-12T23:39:25.284 INFO:teuthology.orchestra.run.smithi136.stdout:osd.5 smithi181 running (8m) 3m ago 8m 42.2M 3036M 17.2.7-1627-gbbdadfa3 214985049bdb 8f2e8d494fef 2024-09-12T23:39:25.284 INFO:teuthology.orchestra.run.smithi136.stdout:osd.6 smithi181 running (7m) 3m ago 7m 41.2M 3036M 17.2.7-1627-gbbdadfa3 214985049bdb 9c6bec406408 2024-09-12T23:39:25.284 INFO:teuthology.orchestra.run.smithi136.stdout:osd.7 smithi181 running (6m) 3m ago 6m 42.3M 3036M 17.2.7-1627-gbbdadfa3 214985049bdb bc0632ae2145 2024-09-12T23:39:25.284 INFO:teuthology.orchestra.run.smithi136.stdout:prometheus.a smithi181 *:9095 running (4m) 3m ago 5m 22.6M - 2.43.0 a07b618ecd1d 24e591310569 2024-09-12T23:39:25.453 INFO:tasks.rados.rados.0.smithi136.stdout:356: done (1 left) 2024-09-12T23:39:25.453 INFO:tasks.rados.rados.0.smithi136.stdout:357: done (0 left) 2024-09-12T23:39:25.453 INFO:tasks.rados.rados.0.smithi136.stdout:358: snap_remove snap 33 2024-09-12T23:39:25.594 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:39:25 smithi181 bash[25717]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:39:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-12T23:39:25.616 INFO:teuthology.orchestra.run.smithi136.stdout:{ 2024-09-12T23:39:25.616 INFO:teuthology.orchestra.run.smithi136.stdout: "mon": { 2024-09-12T23:39:25.616 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 3 2024-09-12T23:39:25.616 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:39:25.616 INFO:teuthology.orchestra.run.smithi136.stdout: "mgr": { 2024-09-12T23:39:25.616 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-12T23:39:25.616 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:39:25.616 INFO:teuthology.orchestra.run.smithi136.stdout: "osd": { 2024-09-12T23:39:25.616 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 8 2024-09-12T23:39:25.616 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:39:25.616 INFO:teuthology.orchestra.run.smithi136.stdout: "mds": { 2024-09-12T23:39:25.617 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-12T23:39:25.617 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:39:25.617 INFO:teuthology.orchestra.run.smithi136.stdout: "overall": { 2024-09-12T23:39:25.617 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 15 2024-09-12T23:39:25.617 INFO:teuthology.orchestra.run.smithi136.stdout: } 2024-09-12T23:39:25.617 INFO:teuthology.orchestra.run.smithi136.stdout:} 2024-09-12T23:39:26.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:25 smithi136 bash[22601]: cluster 2024-09-12T23:39:24.037683+0000 mgr.y (mgr.24446) 223 : cluster [DBG] pgmap v233: 121 pgs: 121 active+clean; 94 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.6 MiB/s wr, 17 op/s 2024-09-12T23:39:26.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:25 smithi136 bash[22601]: cluster 2024-09-12T23:39:24.693303+0000 mon.a (mon.0) 1156 : cluster [DBG] osdmap e128: 8 total, 8 up, 8 in 2024-09-12T23:39:26.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:25 smithi136 bash[22601]: audit 2024-09-12T23:39:24.999233+0000 mgr.y (mgr.24446) 224 : audit [DBG] from='client.15018 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:39:26.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:25 smithi136 bash[22601]: audit 2024-09-12T23:39:25.275006+0000 mgr.y (mgr.24446) 225 : audit [DBG] from='client.24938 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:39:26.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:25 smithi136 bash[22601]: cluster 2024-09-12T23:39:25.457262+0000 mon.a (mon.0) 1157 : cluster [DBG] osdmap e129: 8 total, 8 up, 8 in 2024-09-12T23:39:26.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:25 smithi136 bash[22601]: audit 2024-09-12T23:39:25.620398+0000 mon.a (mon.0) 1158 : audit [DBG] from='client.? 172.21.15.136:0/1994524657' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:39:26.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:25 smithi136 bash[32189]: cluster 2024-09-12T23:39:24.037683+0000 mgr.y (mgr.24446) 223 : cluster [DBG] pgmap v233: 121 pgs: 121 active+clean; 94 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.6 MiB/s wr, 17 op/s 2024-09-12T23:39:26.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:25 smithi136 bash[32189]: cluster 2024-09-12T23:39:24.693303+0000 mon.a (mon.0) 1156 : cluster [DBG] osdmap e128: 8 total, 8 up, 8 in 2024-09-12T23:39:26.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:25 smithi136 bash[32189]: audit 2024-09-12T23:39:24.999233+0000 mgr.y (mgr.24446) 224 : audit [DBG] from='client.15018 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:39:26.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:25 smithi136 bash[32189]: audit 2024-09-12T23:39:25.275006+0000 mgr.y (mgr.24446) 225 : audit [DBG] from='client.24938 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:39:26.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:25 smithi136 bash[32189]: cluster 2024-09-12T23:39:25.457262+0000 mon.a (mon.0) 1157 : cluster [DBG] osdmap e129: 8 total, 8 up, 8 in 2024-09-12T23:39:26.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:25 smithi136 bash[32189]: audit 2024-09-12T23:39:25.620398+0000 mon.a (mon.0) 1158 : audit [DBG] from='client.? 172.21.15.136:0/1994524657' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:39:26.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:25 smithi181 bash[24776]: cluster 2024-09-12T23:39:24.037683+0000 mgr.y (mgr.24446) 223 : cluster [DBG] pgmap v233: 121 pgs: 121 active+clean; 94 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.6 MiB/s wr, 17 op/s 2024-09-12T23:39:26.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:25 smithi181 bash[24776]: cluster 2024-09-12T23:39:24.693303+0000 mon.a (mon.0) 1156 : cluster [DBG] osdmap e128: 8 total, 8 up, 8 in 2024-09-12T23:39:26.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:25 smithi181 bash[24776]: audit 2024-09-12T23:39:24.999233+0000 mgr.y (mgr.24446) 224 : audit [DBG] from='client.15018 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:39:26.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:25 smithi181 bash[24776]: audit 2024-09-12T23:39:25.275006+0000 mgr.y (mgr.24446) 225 : audit [DBG] from='client.24938 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:39:26.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:25 smithi181 bash[24776]: cluster 2024-09-12T23:39:25.457262+0000 mon.a (mon.0) 1157 : cluster [DBG] osdmap e129: 8 total, 8 up, 8 in 2024-09-12T23:39:26.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:25 smithi181 bash[24776]: audit 2024-09-12T23:39:25.620398+0000 mon.a (mon.0) 1158 : audit [DBG] from='client.? 172.21.15.136:0/1994524657' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:39:26.458 INFO:tasks.rados.rados.0.smithi136.stdout:358: done (0 left) 2024-09-12T23:39:26.458 INFO:tasks.rados.rados.0.smithi136.stdout:359: snap_remove snap 21 2024-09-12T23:39:26.505 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:39:26 smithi136 bash[22852]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:39:26] "GET /metrics HTTP/1.1" 200 240083 "" "Prometheus/2.43.0" 2024-09-12T23:39:27.465 INFO:tasks.rados.rados.0.smithi136.stdout:359: done (0 left) 2024-09-12T23:39:27.465 INFO:tasks.rados.rados.0.smithi136.stdout:360: copy_from oid 43 from oid 3 current snap is 35 2024-09-12T23:39:27.465 INFO:tasks.rados.rados.0.smithi136.stdout:361: read oid 37 snap -1 2024-09-12T23:39:27.465 INFO:tasks.rados.rados.0.smithi136.stdout:361: expect (ObjNum 108 snap 23 seq_num 108) 2024-09-12T23:39:27.465 INFO:tasks.rados.rados.0.smithi136.stdout:362: delete oid 12 current snap is 35 2024-09-12T23:39:27.503 INFO:tasks.rados.rados.0.smithi136.stdout:362: done (2 left) 2024-09-12T23:39:27.503 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 13 current snap is 35 2024-09-12T23:39:27.503 INFO:tasks.rados.rados.0.smithi136.stdout:363: seq_num 132 ranges {2818048=614400,3432448=786432,4218880=565248,4784128=253952} 2024-09-12T23:39:27.548 INFO:tasks.rados.rados.0.smithi136.stdout:363: writing smithi13667081-13 from 2818048 to 3432448 tid 1 2024-09-12T23:39:27.557 INFO:tasks.rados.rados.0.smithi136.stdout:363: writing smithi13667081-13 from 3432448 to 4218880 tid 2 2024-09-12T23:39:27.563 INFO:tasks.rados.rados.0.smithi136.stdout:363: writing smithi13667081-13 from 4218880 to 4784128 tid 3 2024-09-12T23:39:27.565 INFO:tasks.rados.rados.0.smithi136.stdout:363: writing smithi13667081-13 from 4784128 to 5038080 tid 4 2024-09-12T23:39:27.566 INFO:tasks.rados.rados.0.smithi136.stdout:364: delete oid 9 current snap is 35 2024-09-12T23:39:27.566 INFO:tasks.rados.rados.0.smithi136.stdout:361: expect (ObjNum 108 snap 23 seq_num 108) 2024-09-12T23:39:27.754 INFO:tasks.rados.rados.0.smithi136.stdout:360: finishing copy_from racing read to smithi13667081-43 2024-09-12T23:39:27.754 INFO:tasks.rados.rados.0.smithi136.stdout:360: finishing copy_from to smithi13667081-43 2024-09-12T23:39:27.754 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 43 v 2 (ObjNum 6 snap 0 seq_num 6) dirty exists 2024-09-12T23:39:27.754 INFO:tasks.rados.rados.0.smithi136.stdout:363: finishing write tid 1 to smithi13667081-13 2024-09-12T23:39:27.754 INFO:tasks.rados.rados.0.smithi136.stdout:363: finishing write tid 2 to smithi13667081-13 2024-09-12T23:39:27.754 INFO:tasks.rados.rados.0.smithi136.stdout:363: finishing write tid 3 to smithi13667081-13 2024-09-12T23:39:27.754 INFO:tasks.rados.rados.0.smithi136.stdout:363: finishing write tid 4 to smithi13667081-13 2024-09-12T23:39:27.755 INFO:tasks.rados.rados.0.smithi136.stdout:363: finishing write tid 5 to smithi13667081-13 2024-09-12T23:39:27.755 INFO:tasks.rados.rados.0.smithi136.stdout:363: finishing write tid 6 to smithi13667081-13 2024-09-12T23:39:27.755 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 13 v 114 (ObjNum 132 snap 35 seq_num 132) dirty exists 2024-09-12T23:39:27.755 INFO:tasks.rados.rados.0.smithi136.stdout:363: left oid 13 (ObjNum 132 snap 35 seq_num 132) 2024-09-12T23:39:27.755 INFO:tasks.rados.rados.0.smithi136.stdout:360: done (3 left) 2024-09-12T23:39:27.755 INFO:tasks.rados.rados.0.smithi136.stdout:361: done (2 left) 2024-09-12T23:39:27.755 INFO:tasks.rados.rados.0.smithi136.stdout:363: done (1 left) 2024-09-12T23:39:27.755 INFO:tasks.rados.rados.0.smithi136.stdout:364: done (0 left) 2024-09-12T23:39:27.756 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 37 current snap is 35 2024-09-12T23:39:27.756 INFO:tasks.rados.rados.0.smithi136.stdout:365: seq_num 134 ranges {3792896=647168,4440064=548864,4988928=573440,5562368=483328} 2024-09-12T23:39:27.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:27 smithi136 bash[22601]: cluster 2024-09-12T23:39:26.038365+0000 mgr.y (mgr.24446) 226 : cluster [DBG] pgmap v236: 121 pgs: 121 active+clean; 94 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.6 MiB/s wr, 17 op/s 2024-09-12T23:39:27.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:27 smithi136 bash[22601]: cluster 2024-09-12T23:39:26.462836+0000 mon.a (mon.0) 1159 : cluster [DBG] osdmap e130: 8 total, 8 up, 8 in 2024-09-12T23:39:27.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:27 smithi136 bash[32189]: cluster 2024-09-12T23:39:26.038365+0000 mgr.y (mgr.24446) 226 : cluster [DBG] pgmap v236: 121 pgs: 121 active+clean; 94 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.6 MiB/s wr, 17 op/s 2024-09-12T23:39:27.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:27 smithi136 bash[32189]: cluster 2024-09-12T23:39:26.462836+0000 mon.a (mon.0) 1159 : cluster [DBG] osdmap e130: 8 total, 8 up, 8 in 2024-09-12T23:39:27.795 INFO:tasks.rados.rados.0.smithi136.stdout:365: writing smithi13667081-37 from 3792896 to 4440064 tid 1 2024-09-12T23:39:27.801 INFO:tasks.rados.rados.0.smithi136.stdout:365: writing smithi13667081-37 from 4440064 to 4988928 tid 2 2024-09-12T23:39:27.807 INFO:tasks.rados.rados.0.smithi136.stdout:365: writing smithi13667081-37 from 4988928 to 5562368 tid 3 2024-09-12T23:39:27.811 INFO:tasks.rados.rados.0.smithi136.stdout:365: writing smithi13667081-37 from 5562368 to 6045696 tid 4 2024-09-12T23:39:27.811 INFO:tasks.rados.rados.0.smithi136.stdout:366: copy_from oid 21 from oid 32 current snap is 35 2024-09-12T23:39:27.811 INFO:tasks.rados.rados.0.smithi136.stdout:365: finishing write tid 1 to smithi13667081-37 2024-09-12T23:39:27.811 INFO:tasks.rados.rados.0.smithi136.stdout:367: setattr oid 43 current snap is 35 2024-09-12T23:39:27.812 INFO:tasks.rados.rados.0.smithi136.stdout:365: finishing write tid 2 to smithi13667081-37 2024-09-12T23:39:27.812 INFO:tasks.rados.rados.0.smithi136.stdout:365: finishing write tid 3 to smithi13667081-37 2024-09-12T23:39:27.813 INFO:tasks.rados.rados.0.smithi136.stdout:368: read oid 34 snap -1 2024-09-12T23:39:27.813 INFO:tasks.rados.rados.0.smithi136.stdout:368: expect deleted 2024-09-12T23:39:27.813 INFO:tasks.rados.rados.0.smithi136.stdout:369: snap_create 2024-09-12T23:39:27.814 INFO:tasks.rados.rados.0.smithi136.stdout:366: finishing copy_from to smithi13667081-21 2024-09-12T23:39:27.814 INFO:tasks.rados.rados.0.smithi136.stdout:366: got expected ENOENT (src dne) 2024-09-12T23:39:27.814 INFO:tasks.rados.rados.0.smithi136.stdout:366: finishing copy_from racing read to smithi13667081-21 2024-09-12T23:39:27.818 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 43 v 147 (ObjNum 6 snap 0 seq_num 6) dirty exists 2024-09-12T23:39:27.818 INFO:tasks.rados.rados.0.smithi136.stdout:365: finishing write tid 4 to smithi13667081-37 2024-09-12T23:39:27.819 INFO:tasks.rados.rados.0.smithi136.stdout:365: finishing write tid 5 to smithi13667081-37 2024-09-12T23:39:27.821 INFO:tasks.rados.rados.0.smithi136.stdout:365: finishing write tid 6 to smithi13667081-37 2024-09-12T23:39:27.821 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 37 v 168 (ObjNum 134 snap 35 seq_num 134) dirty exists 2024-09-12T23:39:27.821 INFO:tasks.rados.rados.0.smithi136.stdout:365: left oid 37 (ObjNum 134 snap 35 seq_num 134) 2024-09-12T23:39:27.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:27 smithi181 bash[24776]: cluster 2024-09-12T23:39:26.038365+0000 mgr.y (mgr.24446) 226 : cluster [DBG] pgmap v236: 121 pgs: 121 active+clean; 94 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.6 MiB/s wr, 17 op/s 2024-09-12T23:39:27.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:27 smithi181 bash[24776]: cluster 2024-09-12T23:39:26.462836+0000 mon.a (mon.0) 1159 : cluster [DBG] osdmap e130: 8 total, 8 up, 8 in 2024-09-12T23:39:28.490 INFO:tasks.rados.rados.0.smithi136.stdout:365: done (4 left) 2024-09-12T23:39:28.490 INFO:tasks.rados.rados.0.smithi136.stdout:366: done (3 left) 2024-09-12T23:39:28.490 INFO:tasks.rados.rados.0.smithi136.stdout:367: done (2 left) 2024-09-12T23:39:28.490 INFO:tasks.rados.rados.0.smithi136.stdout:368: done (1 left) 2024-09-12T23:39:28.490 INFO:tasks.rados.rados.0.smithi136.stdout:369: done (0 left) 2024-09-12T23:39:28.491 INFO:tasks.rados.rados.0.smithi136.stdout:370: read oid 31 snap -1 2024-09-12T23:39:28.491 INFO:tasks.rados.rados.0.smithi136.stdout:370: expect deleted 2024-09-12T23:39:28.491 INFO:tasks.rados.rados.0.smithi136.stdout:371: rollback oid 22 current snap is 36 2024-09-12T23:39:28.491 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 22 to 28 2024-09-12T23:39:28.491 INFO:tasks.rados.rados.0.smithi136.stdout:372: read oid 48 snap -1 2024-09-12T23:39:28.491 INFO:tasks.rados.rados.0.smithi136.stdout:372: expect (ObjNum 118 snap 29 seq_num 118) 2024-09-12T23:39:28.491 INFO:tasks.rados.rados.0.smithi136.stdout:373: rollback oid 3 current snap is 36 2024-09-12T23:39:28.491 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 3 to 25 2024-09-12T23:39:28.491 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 18 current snap is 36 2024-09-12T23:39:28.491 INFO:tasks.rados.rados.0.smithi136.stdout:374: seq_num 135 ranges {4988928=663552,5652480=434176,6086656=786432,6873088=385024} 2024-09-12T23:39:28.534 INFO:tasks.rados.rados.0.smithi136.stdout:374: writing smithi13667081-18 from 4988928 to 5652480 tid 1 2024-09-12T23:39:28.537 INFO:tasks.rados.rados.0.smithi136.stdout:374: writing smithi13667081-18 from 5652480 to 6086656 tid 2 2024-09-12T23:39:28.543 INFO:tasks.rados.rados.0.smithi136.stdout:374: writing smithi13667081-18 from 6086656 to 6873088 tid 3 2024-09-12T23:39:28.546 INFO:tasks.rados.rados.0.smithi136.stdout:374: writing smithi13667081-18 from 6873088 to 7258112 tid 4 2024-09-12T23:39:28.546 INFO:tasks.rados.rados.0.smithi136.stdout:375: rmattr oid 24 current snap is 36 2024-09-12T23:39:28.546 INFO:tasks.rados.rados.0.smithi136.stdout:375: done (5 left) 2024-09-12T23:39:28.546 INFO:tasks.rados.rados.0.smithi136.stdout:376: rmattr oid 41 current snap is 36 2024-09-12T23:39:28.546 INFO:tasks.rados.rados.0.smithi136.stdout:376: done (5 left) 2024-09-12T23:39:28.546 INFO:tasks.rados.rados.0.smithi136.stdout:377: rmattr oid 2 current snap is 36 2024-09-12T23:39:28.546 INFO:tasks.rados.rados.0.smithi136.stdout:377: done (5 left) 2024-09-12T23:39:28.547 INFO:tasks.rados.rados.0.smithi136.stdout:378: copy_from oid 32 from oid 24 current snap is 36 2024-09-12T23:39:28.547 INFO:tasks.rados.rados.0.smithi136.stdout:371: finishing rollback tid 0 to smithi13667081-22 2024-09-12T23:39:28.547 INFO:tasks.rados.rados.0.smithi136.stdout:373: finishing rollback tid 0 to smithi13667081-3 2024-09-12T23:39:28.547 INFO:tasks.rados.rados.0.smithi136.stdout:371: finishing rollback tid 1 to smithi13667081-22 2024-09-12T23:39:28.547 INFO:tasks.rados.rados.0.smithi136.stdout:371: finishing rollback tid 2 to smithi13667081-22 2024-09-12T23:39:28.547 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 22 v 73 (ObjNum 21 snap 0 seq_num 21) dirty exists 2024-09-12T23:39:28.547 INFO:tasks.rados.rados.0.smithi136.stdout:373: finishing rollback tid 1 to smithi13667081-3 2024-09-12T23:39:28.547 INFO:tasks.rados.rados.0.smithi136.stdout:373: finishing rollback tid 2 to smithi13667081-3 2024-09-12T23:39:28.547 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 3 v 202 (ObjNum 6 snap 0 seq_num 6) dirty exists 2024-09-12T23:39:28.547 INFO:tasks.rados.rados.0.smithi136.stdout:370: done (5 left) 2024-09-12T23:39:28.547 INFO:tasks.rados.rados.0.smithi136.stdout:371: done (4 left) 2024-09-12T23:39:28.547 INFO:tasks.rados.rados.0.smithi136.stdout:373: done (3 left) 2024-09-12T23:39:28.547 INFO:tasks.rados.rados.0.smithi136.stdout:379: delete oid 7 current snap is 36 2024-09-12T23:39:28.547 INFO:tasks.rados.rados.0.smithi136.stdout:372: expect (ObjNum 118 snap 29 seq_num 118) 2024-09-12T23:39:28.723 INFO:tasks.rados.rados.0.smithi136.stdout:374: finishing write tid 1 to smithi13667081-18 2024-09-12T23:39:28.723 INFO:tasks.rados.rados.0.smithi136.stdout:374: finishing write tid 2 to smithi13667081-18 2024-09-12T23:39:28.723 INFO:tasks.rados.rados.0.smithi136.stdout:374: finishing write tid 3 to smithi13667081-18 2024-09-12T23:39:28.724 INFO:tasks.rados.rados.0.smithi136.stdout:374: finishing write tid 4 to smithi13667081-18 2024-09-12T23:39:28.724 INFO:tasks.rados.rados.0.smithi136.stdout:374: finishing write tid 5 to smithi13667081-18 2024-09-12T23:39:28.724 INFO:tasks.rados.rados.0.smithi136.stdout:374: finishing write tid 6 to smithi13667081-18 2024-09-12T23:39:28.724 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 18 v 58 (ObjNum 135 snap 36 seq_num 135) dirty exists 2024-09-12T23:39:28.724 INFO:tasks.rados.rados.0.smithi136.stdout:374: left oid 18 (ObjNum 135 snap 36 seq_num 135) 2024-09-12T23:39:28.724 INFO:tasks.rados.rados.0.smithi136.stdout:378: finishing copy_from racing read to smithi13667081-32 2024-09-12T23:39:28.724 INFO:tasks.rados.rados.0.smithi136.stdout:378: finishing copy_from to smithi13667081-32 2024-09-12T23:39:28.724 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 32 v 10 (ObjNum 54 snap 4 seq_num 54) dirty exists 2024-09-12T23:39:28.724 INFO:tasks.rados.rados.0.smithi136.stdout:372: done (3 left) 2024-09-12T23:39:28.725 INFO:tasks.rados.rados.0.smithi136.stdout:374: done (2 left) 2024-09-12T23:39:28.725 INFO:tasks.rados.rados.0.smithi136.stdout:378: done (1 left) 2024-09-12T23:39:28.725 INFO:tasks.rados.rados.0.smithi136.stdout:379: done (0 left) 2024-09-12T23:39:28.725 INFO:tasks.rados.rados.0.smithi136.stdout:380: rollback oid 30 current snap is 36 2024-09-12T23:39:28.725 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 30 to 13 2024-09-12T23:39:28.725 INFO:tasks.rados.rados.0.smithi136.stdout:381: read oid 19 snap -1 2024-09-12T23:39:28.725 INFO:tasks.rados.rados.0.smithi136.stdout:381: expect deleted 2024-09-12T23:39:28.725 INFO:tasks.rados.rados.0.smithi136.stdout:382: snap_remove snap 31 2024-09-12T23:39:28.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:28 smithi136 bash[22601]: cluster 2024-09-12T23:39:27.466440+0000 mon.a (mon.0) 1160 : cluster [DBG] osdmap e131: 8 total, 8 up, 8 in 2024-09-12T23:39:28.755 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:28 smithi136 bash[32189]: cluster 2024-09-12T23:39:27.466440+0000 mon.a (mon.0) 1160 : cluster [DBG] osdmap e131: 8 total, 8 up, 8 in 2024-09-12T23:39:28.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:28 smithi181 bash[24776]: cluster 2024-09-12T23:39:27.466440+0000 mon.a (mon.0) 1160 : cluster [DBG] osdmap e131: 8 total, 8 up, 8 in 2024-09-12T23:39:29.498 INFO:tasks.rados.rados.0.smithi136.stdout:382: done (2 left) 2024-09-12T23:39:29.498 INFO:tasks.rados.rados.0.smithi136.stdout:383: delete oid 12 current snap is 36 2024-09-12T23:39:29.498 INFO:tasks.rados.rados.0.smithi136.stdout:380: finishing rollback tid 0 to smithi13667081-30 2024-09-12T23:39:29.499 INFO:tasks.rados.rados.0.smithi136.stdout:380: finishing rollback tid 1 to smithi13667081-30 2024-09-12T23:39:29.499 INFO:tasks.rados.rados.0.smithi136.stdout:380: finishing rollback tid 2 to smithi13667081-30 2024-09-12T23:39:29.499 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 30 v 74 (ObjNum 29 snap 0 seq_num 29) dirty exists 2024-09-12T23:39:29.519 INFO:tasks.rados.rados.0.smithi136.stdout:380: done (2 left) 2024-09-12T23:39:29.519 INFO:tasks.rados.rados.0.smithi136.stdout:381: done (1 left) 2024-09-12T23:39:29.519 INFO:tasks.rados.rados.0.smithi136.stdout:383: done (0 left) 2024-09-12T23:39:29.519 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 12 current snap is 36 2024-09-12T23:39:29.519 INFO:tasks.rados.rados.0.smithi136.stdout:384: seq_num 138 ranges {0=720896,720896=540672,1261568=557056,1818624=507904} 2024-09-12T23:39:29.526 INFO:tasks.rados.rados.0.smithi136.stdout:384: writing smithi13667081-12 from 0 to 720896 tid 1 2024-09-12T23:39:29.532 INFO:tasks.rados.rados.0.smithi136.stdout:384: writing smithi13667081-12 from 720896 to 1261568 tid 2 2024-09-12T23:39:29.537 INFO:tasks.rados.rados.0.smithi136.stdout:384: writing smithi13667081-12 from 1261568 to 1818624 tid 3 2024-09-12T23:39:29.542 INFO:tasks.rados.rados.0.smithi136.stdout:384: writing smithi13667081-12 from 1818624 to 2326528 tid 4 2024-09-12T23:39:29.542 INFO:tasks.rados.rados.0.smithi136.stdout:385: snap_remove snap 35 2024-09-12T23:39:29.755 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:29 smithi136 bash[32189]: cluster 2024-09-12T23:39:28.039021+0000 mgr.y (mgr.24446) 227 : cluster [DBG] pgmap v239: 121 pgs: 121 active+clean; 94 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:39:29.755 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:29 smithi136 bash[32189]: cluster 2024-09-12T23:39:28.494225+0000 mon.a (mon.0) 1161 : cluster [DBG] osdmap e132: 8 total, 8 up, 8 in 2024-09-12T23:39:29.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:29 smithi136 bash[22601]: cluster 2024-09-12T23:39:28.039021+0000 mgr.y (mgr.24446) 227 : cluster [DBG] pgmap v239: 121 pgs: 121 active+clean; 94 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:39:29.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:29 smithi136 bash[22601]: cluster 2024-09-12T23:39:28.494225+0000 mon.a (mon.0) 1161 : cluster [DBG] osdmap e132: 8 total, 8 up, 8 in 2024-09-12T23:39:29.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:29 smithi181 bash[24776]: cluster 2024-09-12T23:39:28.039021+0000 mgr.y (mgr.24446) 227 : cluster [DBG] pgmap v239: 121 pgs: 121 active+clean; 94 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:39:29.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:29 smithi181 bash[24776]: cluster 2024-09-12T23:39:28.494225+0000 mon.a (mon.0) 1161 : cluster [DBG] osdmap e132: 8 total, 8 up, 8 in 2024-09-12T23:39:30.512 INFO:tasks.rados.rados.0.smithi136.stdout:385: done (1 left) 2024-09-12T23:39:30.512 INFO:tasks.rados.rados.0.smithi136.stdout:386: snap_remove snap 32 2024-09-12T23:39:30.755 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:30 smithi136 bash[32189]: cluster 2024-09-12T23:39:29.501986+0000 mon.a (mon.0) 1162 : cluster [DBG] osdmap e133: 8 total, 8 up, 8 in 2024-09-12T23:39:30.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:30 smithi136 bash[22601]: cluster 2024-09-12T23:39:29.501986+0000 mon.a (mon.0) 1162 : cluster [DBG] osdmap e133: 8 total, 8 up, 8 in 2024-09-12T23:39:30.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:30 smithi181 bash[24776]: cluster 2024-09-12T23:39:29.501986+0000 mon.a (mon.0) 1162 : cluster [DBG] osdmap e133: 8 total, 8 up, 8 in 2024-09-12T23:39:31.511 INFO:tasks.rados.rados.0.smithi136.stdout:386: done (1 left) 2024-09-12T23:39:31.511 INFO:tasks.rados.rados.0.smithi136.stdout:387: snap_remove snap 34 2024-09-12T23:39:31.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:31 smithi181 bash[24776]: cluster 2024-09-12T23:39:30.039693+0000 mgr.y (mgr.24446) 228 : cluster [DBG] pgmap v242: 121 pgs: 121 active+clean; 104 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.6 MiB/s rd, 2.5 MiB/s wr, 15 op/s 2024-09-12T23:39:31.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:31 smithi181 bash[24776]: cluster 2024-09-12T23:39:30.516016+0000 mon.a (mon.0) 1163 : cluster [DBG] osdmap e134: 8 total, 8 up, 8 in 2024-09-12T23:39:32.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:31 smithi136 bash[32189]: cluster 2024-09-12T23:39:30.039693+0000 mgr.y (mgr.24446) 228 : cluster [DBG] pgmap v242: 121 pgs: 121 active+clean; 104 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.6 MiB/s rd, 2.5 MiB/s wr, 15 op/s 2024-09-12T23:39:32.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:31 smithi136 bash[32189]: cluster 2024-09-12T23:39:30.516016+0000 mon.a (mon.0) 1163 : cluster [DBG] osdmap e134: 8 total, 8 up, 8 in 2024-09-12T23:39:32.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:31 smithi136 bash[22601]: cluster 2024-09-12T23:39:30.039693+0000 mgr.y (mgr.24446) 228 : cluster [DBG] pgmap v242: 121 pgs: 121 active+clean; 104 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.6 MiB/s rd, 2.5 MiB/s wr, 15 op/s 2024-09-12T23:39:32.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:31 smithi136 bash[22601]: cluster 2024-09-12T23:39:30.516016+0000 mon.a (mon.0) 1163 : cluster [DBG] osdmap e134: 8 total, 8 up, 8 in 2024-09-12T23:39:32.515 INFO:tasks.rados.rados.0.smithi136.stdout:387: done (1 left) 2024-09-12T23:39:32.515 INFO:tasks.rados.rados.0.smithi136.stdout:388: snap_remove snap 25 2024-09-12T23:39:32.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:32 smithi181 bash[24776]: cluster 2024-09-12T23:39:31.515330+0000 mon.a (mon.0) 1164 : cluster [DBG] osdmap e135: 8 total, 8 up, 8 in 2024-09-12T23:39:32.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:32 smithi181 bash[24776]: cluster 2024-09-12T23:39:32.519343+0000 mon.a (mon.0) 1165 : cluster [DBG] osdmap e136: 8 total, 8 up, 8 in 2024-09-12T23:39:33.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:32 smithi136 bash[22601]: cluster 2024-09-12T23:39:31.515330+0000 mon.a (mon.0) 1164 : cluster [DBG] osdmap e135: 8 total, 8 up, 8 in 2024-09-12T23:39:33.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:32 smithi136 bash[22601]: cluster 2024-09-12T23:39:32.519343+0000 mon.a (mon.0) 1165 : cluster [DBG] osdmap e136: 8 total, 8 up, 8 in 2024-09-12T23:39:33.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:32 smithi136 bash[32189]: cluster 2024-09-12T23:39:31.515330+0000 mon.a (mon.0) 1164 : cluster [DBG] osdmap e135: 8 total, 8 up, 8 in 2024-09-12T23:39:33.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:32 smithi136 bash[32189]: cluster 2024-09-12T23:39:32.519343+0000 mon.a (mon.0) 1165 : cluster [DBG] osdmap e136: 8 total, 8 up, 8 in 2024-09-12T23:39:33.519 INFO:tasks.rados.rados.0.smithi136.stdout:388: done (1 left) 2024-09-12T23:39:33.519 INFO:tasks.rados.rados.0.smithi136.stdout:389: setattr oid 21 current snap is 36 2024-09-12T23:39:33.519 INFO:tasks.rados.rados.0.smithi136.stdout:384: finishing write tid 1 to smithi13667081-12 2024-09-12T23:39:33.519 INFO:tasks.rados.rados.0.smithi136.stdout:384: finishing write tid 2 to smithi13667081-12 2024-09-12T23:39:33.520 INFO:tasks.rados.rados.0.smithi136.stdout:384: finishing write tid 3 to smithi13667081-12 2024-09-12T23:39:33.520 INFO:tasks.rados.rados.0.smithi136.stdout:384: finishing write tid 4 to smithi13667081-12 2024-09-12T23:39:33.520 INFO:tasks.rados.rados.0.smithi136.stdout:384: finishing write tid 5 to smithi13667081-12 2024-09-12T23:39:33.520 INFO:tasks.rados.rados.0.smithi136.stdout:384: finishing write tid 6 to smithi13667081-12 2024-09-12T23:39:33.520 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 12 v 212 (ObjNum 138 snap 36 seq_num 138) dirty exists 2024-09-12T23:39:33.520 INFO:tasks.rados.rados.0.smithi136.stdout:384: left oid 12 (ObjNum 138 snap 36 seq_num 138) 2024-09-12T23:39:33.524 INFO:tasks.rados.rados.0.smithi136.stdout:384: done (1 left) 2024-09-12T23:39:33.524 INFO:tasks.rados.rados.0.smithi136.stdout:390: read oid 27 snap -1 2024-09-12T23:39:33.524 INFO:tasks.rados.rados.0.smithi136.stdout:390: expect (ObjNum 26 snap 0 seq_num 26) 2024-09-12T23:39:33.524 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 32 current snap is 36 2024-09-12T23:39:33.524 INFO:tasks.rados.rados.0.smithi136.stdout:391: seq_num 139 ranges {1302528=737280,2039808=425984,2465792=761856,3227648=417792} 2024-09-12T23:39:33.548 INFO:tasks.rados.rados.0.smithi136.stdout:391: writing smithi13667081-32 from 1302528 to 2039808 tid 1 2024-09-12T23:39:33.554 INFO:tasks.rados.rados.0.smithi136.stdout:391: writing smithi13667081-32 from 2039808 to 2465792 tid 2 2024-09-12T23:39:33.563 INFO:tasks.rados.rados.0.smithi136.stdout:391: writing smithi13667081-32 from 2465792 to 3227648 tid 3 2024-09-12T23:39:33.568 INFO:tasks.rados.rados.0.smithi136.stdout:391: writing smithi13667081-32 from 3227648 to 3645440 tid 4 2024-09-12T23:39:33.568 INFO:tasks.rados.rados.0.smithi136.stdout:392: setattr oid 8 current snap is 36 2024-09-12T23:39:33.569 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 21 v 82 (ObjNum 273 snap 0 seq_num 0) dirty exists 2024-09-12T23:39:33.569 INFO:tasks.rados.rados.0.smithi136.stdout:390: expect (ObjNum 26 snap 0 seq_num 26) 2024-09-12T23:39:33.605 INFO:tasks.rados.rados.0.smithi136.stdout:391: finishing write tid 1 to smithi13667081-32 2024-09-12T23:39:33.605 INFO:tasks.rados.rados.0.smithi136.stdout:391: finishing write tid 2 to smithi13667081-32 2024-09-12T23:39:33.605 INFO:tasks.rados.rados.0.smithi136.stdout:391: finishing write tid 3 to smithi13667081-32 2024-09-12T23:39:33.605 INFO:tasks.rados.rados.0.smithi136.stdout:391: finishing write tid 4 to smithi13667081-32 2024-09-12T23:39:33.605 INFO:tasks.rados.rados.0.smithi136.stdout:391: finishing write tid 5 to smithi13667081-32 2024-09-12T23:39:33.606 INFO:tasks.rados.rados.0.smithi136.stdout:391: finishing write tid 6 to smithi13667081-32 2024-09-12T23:39:33.606 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 32 v 54 (ObjNum 139 snap 36 seq_num 139) dirty exists 2024-09-12T23:39:33.606 INFO:tasks.rados.rados.0.smithi136.stdout:391: left oid 32 (ObjNum 139 snap 36 seq_num 139) 2024-09-12T23:39:33.606 INFO:tasks.rados.rados.0.smithi136.stdout:389: done (3 left) 2024-09-12T23:39:33.606 INFO:tasks.rados.rados.0.smithi136.stdout:390: done (2 left) 2024-09-12T23:39:33.606 INFO:tasks.rados.rados.0.smithi136.stdout:391: done (1 left) 2024-09-12T23:39:33.606 INFO:tasks.rados.rados.0.smithi136.stdout:393: read oid 47 snap -1 2024-09-12T23:39:33.606 INFO:tasks.rados.rados.0.smithi136.stdout:393: expect (ObjNum 74 snap 12 seq_num 74) 2024-09-12T23:39:33.606 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 28 current snap is 36 2024-09-12T23:39:33.606 INFO:tasks.rados.rados.0.smithi136.stdout:394: seq_num 140 ranges {0=753664,753664=704512,1458176=524288,1982464=376832} 2024-09-12T23:39:33.610 INFO:tasks.rados.rados.0.smithi136.stdout:394: writing smithi13667081-28 from 0 to 753664 tid 1 2024-09-12T23:39:33.618 INFO:tasks.rados.rados.0.smithi136.stdout:394: writing smithi13667081-28 from 753664 to 1458176 tid 2 2024-09-12T23:39:33.624 INFO:tasks.rados.rados.0.smithi136.stdout:394: writing smithi13667081-28 from 1458176 to 1982464 tid 3 2024-09-12T23:39:33.628 INFO:tasks.rados.rados.0.smithi136.stdout:394: writing smithi13667081-28 from 1982464 to 2359296 tid 4 2024-09-12T23:39:33.628 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 45 current snap is 36 2024-09-12T23:39:33.628 INFO:tasks.rados.rados.0.smithi136.stdout:395: seq_num 141 ranges {638976=770048,1409024=589824,1998848=729088,2727936=286720} 2024-09-12T23:39:33.643 INFO:tasks.rados.rados.0.smithi136.stdout:395: writing smithi13667081-45 from 638976 to 1409024 tid 1 2024-09-12T23:39:33.648 INFO:tasks.rados.rados.0.smithi136.stdout:395: writing smithi13667081-45 from 1409024 to 1998848 tid 2 2024-09-12T23:39:33.654 INFO:tasks.rados.rados.0.smithi136.stdout:395: writing smithi13667081-45 from 1998848 to 2727936 tid 3 2024-09-12T23:39:33.657 INFO:tasks.rados.rados.0.smithi136.stdout:395: writing smithi13667081-45 from 2727936 to 3014656 tid 4 2024-09-12T23:39:33.657 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 49 current snap is 36 2024-09-12T23:39:33.658 INFO:tasks.rados.rados.0.smithi136.stdout:396: seq_num 142 ranges {2138112=786432,2924544=475136,3399680=540672,3940352=589824} 2024-09-12T23:39:33.676 INFO:tasks.rados.rados.0.smithi136.stdout:396: writing smithi13667081-49 from 2138112 to 2924544 tid 1 2024-09-12T23:39:33.679 INFO:tasks.rados.rados.0.smithi136.stdout:396: writing smithi13667081-49 from 2924544 to 3399680 tid 2 2024-09-12T23:39:33.683 INFO:tasks.rados.rados.0.smithi136.stdout:396: writing smithi13667081-49 from 3399680 to 3940352 tid 3 2024-09-12T23:39:33.687 INFO:tasks.rados.rados.0.smithi136.stdout:396: writing smithi13667081-49 from 3940352 to 4530176 tid 4 2024-09-12T23:39:33.687 INFO:tasks.rados.rados.0.smithi136.stdout:397: delete oid 25 current snap is 36 2024-09-12T23:39:33.687 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 8 v 108 (ObjNum 161 snap 0 seq_num 1427291504) dirty exists 2024-09-12T23:39:33.687 INFO:tasks.rados.rados.0.smithi136.stdout:393: expect (ObjNum 74 snap 12 seq_num 74) 2024-09-12T23:39:33.802 INFO:tasks.rados.rados.0.smithi136.stdout:394: finishing write tid 1 to smithi13667081-28 2024-09-12T23:39:33.802 INFO:tasks.rados.rados.0.smithi136.stdout:394: finishing write tid 2 to smithi13667081-28 2024-09-12T23:39:33.802 INFO:tasks.rados.rados.0.smithi136.stdout:394: finishing write tid 3 to smithi13667081-28 2024-09-12T23:39:33.802 INFO:tasks.rados.rados.0.smithi136.stdout:394: finishing write tid 4 to smithi13667081-28 2024-09-12T23:39:33.803 INFO:tasks.rados.rados.0.smithi136.stdout:394: finishing write tid 5 to smithi13667081-28 2024-09-12T23:39:33.803 INFO:tasks.rados.rados.0.smithi136.stdout:394: finishing write tid 6 to smithi13667081-28 2024-09-12T23:39:33.803 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 28 v 163 (ObjNum 140 snap 36 seq_num 140) dirty exists 2024-09-12T23:39:33.803 INFO:tasks.rados.rados.0.smithi136.stdout:394: left oid 28 (ObjNum 140 snap 36 seq_num 140) 2024-09-12T23:39:33.803 INFO:tasks.rados.rados.0.smithi136.stdout:395: finishing write tid 1 to smithi13667081-45 2024-09-12T23:39:33.803 INFO:tasks.rados.rados.0.smithi136.stdout:395: finishing write tid 2 to smithi13667081-45 2024-09-12T23:39:33.803 INFO:tasks.rados.rados.0.smithi136.stdout:395: finishing write tid 3 to smithi13667081-45 2024-09-12T23:39:33.803 INFO:tasks.rados.rados.0.smithi136.stdout:395: finishing write tid 4 to smithi13667081-45 2024-09-12T23:39:33.803 INFO:tasks.rados.rados.0.smithi136.stdout:395: finishing write tid 5 to smithi13667081-45 2024-09-12T23:39:33.803 INFO:tasks.rados.rados.0.smithi136.stdout:395: finishing write tid 6 to smithi13667081-45 2024-09-12T23:39:33.803 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 45 v 113 (ObjNum 141 snap 36 seq_num 141) dirty exists 2024-09-12T23:39:33.803 INFO:tasks.rados.rados.0.smithi136.stdout:395: left oid 45 (ObjNum 141 snap 36 seq_num 141) 2024-09-12T23:39:33.803 INFO:tasks.rados.rados.0.smithi136.stdout:396: finishing write tid 1 to smithi13667081-49 2024-09-12T23:39:33.804 INFO:tasks.rados.rados.0.smithi136.stdout:396: finishing write tid 2 to smithi13667081-49 2024-09-12T23:39:33.804 INFO:tasks.rados.rados.0.smithi136.stdout:396: finishing write tid 3 to smithi13667081-49 2024-09-12T23:39:33.804 INFO:tasks.rados.rados.0.smithi136.stdout:396: finishing write tid 4 to smithi13667081-49 2024-09-12T23:39:33.804 INFO:tasks.rados.rados.0.smithi136.stdout:396: finishing write tid 5 to smithi13667081-49 2024-09-12T23:39:33.804 INFO:tasks.rados.rados.0.smithi136.stdout:396: finishing write tid 6 to smithi13667081-49 2024-09-12T23:39:33.804 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 49 v 233 (ObjNum 142 snap 36 seq_num 142) dirty exists 2024-09-12T23:39:33.804 INFO:tasks.rados.rados.0.smithi136.stdout:396: left oid 49 (ObjNum 142 snap 36 seq_num 142) 2024-09-12T23:39:33.804 INFO:tasks.rados.rados.0.smithi136.stdout:392: done (5 left) 2024-09-12T23:39:33.804 INFO:tasks.rados.rados.0.smithi136.stdout:393: done (4 left) 2024-09-12T23:39:33.804 INFO:tasks.rados.rados.0.smithi136.stdout:394: done (3 left) 2024-09-12T23:39:33.804 INFO:tasks.rados.rados.0.smithi136.stdout:395: done (2 left) 2024-09-12T23:39:33.804 INFO:tasks.rados.rados.0.smithi136.stdout:396: done (1 left) 2024-09-12T23:39:33.804 INFO:tasks.rados.rados.0.smithi136.stdout:397: done (0 left) 2024-09-12T23:39:33.804 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 25 current snap is 36 2024-09-12T23:39:33.805 INFO:tasks.rados.rados.0.smithi136.stdout:398: seq_num 144 ranges {0=16384} 2024-09-12T23:39:33.805 INFO:tasks.rados.rados.0.smithi136.stdout:398: writing smithi13667081-25 from 0 to 16384 tid 1 2024-09-12T23:39:33.805 INFO:tasks.rados.rados.0.smithi136.stdout:399: setattr oid 48 current snap is 36 2024-09-12T23:39:33.805 INFO:tasks.rados.rados.0.smithi136.stdout:400: read oid 14 snap -1 2024-09-12T23:39:33.805 INFO:tasks.rados.rados.0.smithi136.stdout:400: expect (ObjNum 102 snap 22 seq_num 102) 2024-09-12T23:39:33.805 INFO:tasks.rados.rados.0.smithi136.stdout:401: snap_create 2024-09-12T23:39:33.806 INFO:tasks.rados.rados.0.smithi136.stdout:398: finishing write tid 1 to smithi13667081-25 2024-09-12T23:39:33.806 INFO:tasks.rados.rados.0.smithi136.stdout:398: finishing write tid 2 to smithi13667081-25 2024-09-12T23:39:33.807 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 48 v 165 (ObjNum 118 snap 29 seq_num 118) dirty exists 2024-09-12T23:39:33.807 INFO:tasks.rados.rados.0.smithi136.stdout:398: finishing write tid 3 to smithi13667081-25 2024-09-12T23:39:33.807 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 25 v 32 (ObjNum 144 snap 36 seq_num 144) dirty exists 2024-09-12T23:39:33.807 INFO:tasks.rados.rados.0.smithi136.stdout:398: left oid 25 (ObjNum 144 snap 36 seq_num 144) 2024-09-12T23:39:33.813 INFO:tasks.rados.rados.0.smithi136.stdout:400: expect (ObjNum 102 snap 22 seq_num 102) 2024-09-12T23:39:33.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:33 smithi181 bash[24776]: cluster 2024-09-12T23:39:32.040446+0000 mgr.y (mgr.24446) 229 : cluster [DBG] pgmap v245: 121 pgs: 121 active+clean; 104 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.6 MiB/s rd, 2.5 MiB/s wr, 15 op/s 2024-09-12T23:39:33.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:33 smithi181 bash[24776]: cluster 2024-09-12T23:39:33.523159+0000 mon.a (mon.0) 1166 : cluster [DBG] osdmap e137: 8 total, 8 up, 8 in 2024-09-12T23:39:34.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:33 smithi136 bash[22601]: cluster 2024-09-12T23:39:32.040446+0000 mgr.y (mgr.24446) 229 : cluster [DBG] pgmap v245: 121 pgs: 121 active+clean; 104 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.6 MiB/s rd, 2.5 MiB/s wr, 15 op/s 2024-09-12T23:39:34.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:33 smithi136 bash[22601]: cluster 2024-09-12T23:39:33.523159+0000 mon.a (mon.0) 1166 : cluster [DBG] osdmap e137: 8 total, 8 up, 8 in 2024-09-12T23:39:34.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:33 smithi136 bash[32189]: cluster 2024-09-12T23:39:32.040446+0000 mgr.y (mgr.24446) 229 : cluster [DBG] pgmap v245: 121 pgs: 121 active+clean; 104 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.6 MiB/s rd, 2.5 MiB/s wr, 15 op/s 2024-09-12T23:39:34.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:33 smithi136 bash[32189]: cluster 2024-09-12T23:39:33.523159+0000 mon.a (mon.0) 1166 : cluster [DBG] osdmap e137: 8 total, 8 up, 8 in 2024-09-12T23:39:34.557 INFO:tasks.rados.rados.0.smithi136.stdout:398: done (3 left) 2024-09-12T23:39:34.557 INFO:tasks.rados.rados.0.smithi136.stdout:399: done (2 left) 2024-09-12T23:39:34.557 INFO:tasks.rados.rados.0.smithi136.stdout:400: done (1 left) 2024-09-12T23:39:34.558 INFO:tasks.rados.rados.0.smithi136.stdout:401: done (0 left) 2024-09-12T23:39:34.558 INFO:tasks.rados.rados.0.smithi136.stdout:402: delete oid 27 current snap is 37 2024-09-12T23:39:34.560 INFO:tasks.rados.rados.0.smithi136.stdout:402: done (0 left) 2024-09-12T23:39:34.560 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 3 current snap is 37 2024-09-12T23:39:34.560 INFO:tasks.rados.rados.0.smithi136.stdout:403: seq_num 146 ranges {106496=49152} 2024-09-12T23:39:34.562 INFO:tasks.rados.rados.0.smithi136.stdout:403: writing smithi13667081-3 from 106496 to 155648 tid 1 2024-09-12T23:39:34.562 INFO:tasks.rados.rados.0.smithi136.stdout:404: snap_create 2024-09-12T23:39:34.570 INFO:tasks.rados.rados.0.smithi136.stdout:403: finishing write tid 1 to smithi13667081-3 2024-09-12T23:39:34.570 INFO:tasks.rados.rados.0.smithi136.stdout:403: finishing write tid 2 to smithi13667081-3 2024-09-12T23:39:34.575 INFO:tasks.rados.rados.0.smithi136.stdout:403: finishing write tid 3 to smithi13667081-3 2024-09-12T23:39:34.575 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 3 v 236 (ObjNum 146 snap 37 seq_num 146) dirty exists 2024-09-12T23:39:34.575 INFO:tasks.rados.rados.0.smithi136.stdout:403: left oid 3 (ObjNum 146 snap 37 seq_num 146) 2024-09-12T23:39:35.462 INFO:tasks.rados.rados.0.smithi136.stdout:403: done (1 left) 2024-09-12T23:39:35.462 INFO:tasks.rados.rados.0.smithi136.stdout:404: done (0 left) 2024-09-12T23:39:35.462 INFO:tasks.rados.rados.0.smithi136.stdout:405: rollback oid 27 current snap is 38 2024-09-12T23:39:35.462 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 27 to 37 2024-09-12T23:39:35.462 INFO:tasks.rados.rados.0.smithi136.stdout:406: setattr oid 20 current snap is 38 2024-09-12T23:39:35.470 INFO:tasks.rados.rados.0.smithi136.stdout:405: finishing rollback tid 1 to smithi13667081-27 2024-09-12T23:39:35.470 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 27 v 161 (ObjNum 33 snap 0 seq_num 4242855296) dirty dne 2024-09-12T23:39:35.470 INFO:tasks.rados.rados.0.smithi136.stdout:405: done (1 left) 2024-09-12T23:39:35.471 INFO:tasks.rados.rados.0.smithi136.stdout:407: snap_create 2024-09-12T23:39:35.480 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 20 v 39 (ObjNum 0 snap 0 seq_num 304) dirty exists 2024-09-12T23:39:35.554 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:39:35 smithi181 bash[25717]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:39:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-12T23:39:35.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:35 smithi181 bash[24776]: cluster 2024-09-12T23:39:34.041856+0000 mgr.y (mgr.24446) 230 : cluster [DBG] pgmap v248: 121 pgs: 121 active+clean; 113 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.8 MiB/s wr, 26 op/s 2024-09-12T23:39:35.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:35 smithi181 bash[24776]: cluster 2024-09-12T23:39:34.561155+0000 mon.a (mon.0) 1167 : cluster [DBG] osdmap e138: 8 total, 8 up, 8 in 2024-09-12T23:39:35.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:35 smithi181 bash[24776]: cluster 2024-09-12T23:39:35.466021+0000 mon.a (mon.0) 1168 : cluster [DBG] osdmap e139: 8 total, 8 up, 8 in 2024-09-12T23:39:36.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:35 smithi136 bash[22601]: cluster 2024-09-12T23:39:34.041856+0000 mgr.y (mgr.24446) 230 : cluster [DBG] pgmap v248: 121 pgs: 121 active+clean; 113 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.8 MiB/s wr, 26 op/s 2024-09-12T23:39:36.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:35 smithi136 bash[22601]: cluster 2024-09-12T23:39:34.561155+0000 mon.a (mon.0) 1167 : cluster [DBG] osdmap e138: 8 total, 8 up, 8 in 2024-09-12T23:39:36.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:35 smithi136 bash[22601]: cluster 2024-09-12T23:39:35.466021+0000 mon.a (mon.0) 1168 : cluster [DBG] osdmap e139: 8 total, 8 up, 8 in 2024-09-12T23:39:36.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:35 smithi136 bash[32189]: cluster 2024-09-12T23:39:34.041856+0000 mgr.y (mgr.24446) 230 : cluster [DBG] pgmap v248: 121 pgs: 121 active+clean; 113 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.8 MiB/s wr, 26 op/s 2024-09-12T23:39:36.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:35 smithi136 bash[32189]: cluster 2024-09-12T23:39:34.561155+0000 mon.a (mon.0) 1167 : cluster [DBG] osdmap e138: 8 total, 8 up, 8 in 2024-09-12T23:39:36.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:35 smithi136 bash[32189]: cluster 2024-09-12T23:39:35.466021+0000 mon.a (mon.0) 1168 : cluster [DBG] osdmap e139: 8 total, 8 up, 8 in 2024-09-12T23:39:36.464 INFO:tasks.rados.rados.0.smithi136.stdout:406: done (1 left) 2024-09-12T23:39:36.464 INFO:tasks.rados.rados.0.smithi136.stdout:407: done (0 left) 2024-09-12T23:39:36.464 INFO:tasks.rados.rados.0.smithi136.stdout:408: read oid 46 snap -1 2024-09-12T23:39:36.464 INFO:tasks.rados.rados.0.smithi136.stdout:408: expect (ObjNum 110 snap 23 seq_num 110) 2024-09-12T23:39:36.464 INFO:tasks.rados.rados.0.smithi136.stdout:409: snap_remove snap 22 2024-09-12T23:39:36.505 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:39:36 smithi136 bash[22852]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:39:36] "GET /metrics HTTP/1.1" 200 240083 "" "Prometheus/2.43.0" 2024-09-12T23:39:37.472 INFO:tasks.rados.rados.0.smithi136.stdout:409: done (1 left) 2024-09-12T23:39:37.472 INFO:tasks.rados.rados.0.smithi136.stdout:410: read oid 12 snap -1 2024-09-12T23:39:37.473 INFO:tasks.rados.rados.0.smithi136.stdout:410: expect (ObjNum 138 snap 36 seq_num 138) 2024-09-12T23:39:37.473 INFO:tasks.rados.rados.0.smithi136.stdout:411: read oid 38 snap 23 2024-09-12T23:39:37.473 INFO:tasks.rados.rados.0.smithi136.stdout:411: expect (ObjNum 93 snap 18 seq_num 93) 2024-09-12T23:39:37.473 INFO:tasks.rados.rados.0.smithi136.stdout:412: read oid 36 snap 19 2024-09-12T23:39:37.473 INFO:tasks.rados.rados.0.smithi136.stdout:412: expect (ObjNum 10 snap 0 seq_num 10) 2024-09-12T23:39:37.473 INFO:tasks.rados.rados.0.smithi136.stdout:413: setattr oid 24 current snap is 39 2024-09-12T23:39:37.479 INFO:tasks.rados.rados.0.smithi136.stdout:408: expect (ObjNum 110 snap 23 seq_num 110) 2024-09-12T23:39:37.598 INFO:tasks.rados.rados.0.smithi136.stdout:412: expect (ObjNum 10 snap 0 seq_num 10) 2024-09-12T23:39:37.602 INFO:tasks.rados.rados.0.smithi136.stdout:410: expect (ObjNum 138 snap 36 seq_num 138) 2024-09-12T23:39:37.660 INFO:tasks.rados.rados.0.smithi136.stdout:411: expect (ObjNum 93 snap 18 seq_num 93) 2024-09-12T23:39:37.726 INFO:tasks.rados.rados.0.smithi136.stdout:408: done (4 left) 2024-09-12T23:39:37.726 INFO:tasks.rados.rados.0.smithi136.stdout:410: done (3 left) 2024-09-12T23:39:37.727 INFO:tasks.rados.rados.0.smithi136.stdout:411: done (2 left) 2024-09-12T23:39:37.727 INFO:tasks.rados.rados.0.smithi136.stdout:412: done (1 left) 2024-09-12T23:39:37.727 INFO:tasks.rados.rados.0.smithi136.stdout:414: rmattr oid 38 current snap is 39 2024-09-12T23:39:37.727 INFO:tasks.rados.rados.0.smithi136.stdout:415: snap_remove snap 38 2024-09-12T23:39:37.755 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:37 smithi136 bash[32189]: cluster 2024-09-12T23:39:36.042548+0000 mgr.y (mgr.24446) 231 : cluster [DBG] pgmap v251: 121 pgs: 121 active+clean; 113 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.8 MiB/s wr, 26 op/s 2024-09-12T23:39:37.755 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:37 smithi136 bash[32189]: cluster 2024-09-12T23:39:36.468195+0000 mon.a (mon.0) 1169 : cluster [DBG] osdmap e140: 8 total, 8 up, 8 in 2024-09-12T23:39:37.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:37 smithi136 bash[22601]: cluster 2024-09-12T23:39:36.042548+0000 mgr.y (mgr.24446) 231 : cluster [DBG] pgmap v251: 121 pgs: 121 active+clean; 113 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.8 MiB/s wr, 26 op/s 2024-09-12T23:39:37.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:37 smithi136 bash[22601]: cluster 2024-09-12T23:39:36.468195+0000 mon.a (mon.0) 1169 : cluster [DBG] osdmap e140: 8 total, 8 up, 8 in 2024-09-12T23:39:37.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:37 smithi181 bash[24776]: cluster 2024-09-12T23:39:36.042548+0000 mgr.y (mgr.24446) 231 : cluster [DBG] pgmap v251: 121 pgs: 121 active+clean; 113 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.8 MiB/s wr, 26 op/s 2024-09-12T23:39:37.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:37 smithi181 bash[24776]: cluster 2024-09-12T23:39:36.468195+0000 mon.a (mon.0) 1169 : cluster [DBG] osdmap e140: 8 total, 8 up, 8 in 2024-09-12T23:39:38.500 INFO:tasks.rados.rados.0.smithi136.stdout:415: done (2 left) 2024-09-12T23:39:38.500 INFO:tasks.rados.rados.0.smithi136.stdout:416: read oid 45 snap 23 2024-09-12T23:39:38.500 INFO:tasks.rados.rados.0.smithi136.stdout:416: expect (ObjNum 109 snap 23 seq_num 109) 2024-09-12T23:39:38.501 INFO:tasks.rados.rados.0.smithi136.stdout:417: delete oid 8 current snap is 39 2024-09-12T23:39:38.501 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 38 v 138 (ObjNum 93 snap 18 seq_num 93) dirty exists 2024-09-12T23:39:38.501 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 24 v 72 (ObjNum 54 snap 4 seq_num 54) dirty exists 2024-09-12T23:39:38.528 INFO:tasks.rados.rados.0.smithi136.stdout:416: expect (ObjNum 109 snap 23 seq_num 109) 2024-09-12T23:39:38.598 INFO:tasks.rados.rados.0.smithi136.stdout:413: done (3 left) 2024-09-12T23:39:38.598 INFO:tasks.rados.rados.0.smithi136.stdout:414: done (2 left) 2024-09-12T23:39:38.598 INFO:tasks.rados.rados.0.smithi136.stdout:416: done (1 left) 2024-09-12T23:39:38.598 INFO:tasks.rados.rados.0.smithi136.stdout:417: done (0 left) 2024-09-12T23:39:38.599 INFO:tasks.rados.rados.0.smithi136.stdout:418: copy_from oid 11 from oid 50 current snap is 39 2024-09-12T23:39:38.599 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 16 current snap is 39 2024-09-12T23:39:38.599 INFO:tasks.rados.rados.0.smithi136.stdout:419: seq_num 148 ranges {4825088=81920} 2024-09-12T23:39:38.669 INFO:tasks.rados.rados.0.smithi136.stdout:419: writing smithi13667081-16 from 4825088 to 4907008 tid 1 2024-09-12T23:39:38.670 INFO:tasks.rados.rados.0.smithi136.stdout:420: rmattr oid 2 current snap is 39 2024-09-12T23:39:38.670 INFO:tasks.rados.rados.0.smithi136.stdout:420: done (2 left) 2024-09-12T23:39:38.670 INFO:tasks.rados.rados.0.smithi136.stdout:421: delete oid 35 current snap is 39 2024-09-12T23:39:38.670 INFO:tasks.rados.rados.0.smithi136.stdout:418: finishing copy_from racing read to smithi13667081-11 2024-09-12T23:39:38.670 INFO:tasks.rados.rados.0.smithi136.stdout:418: finishing copy_from to smithi13667081-11 2024-09-12T23:39:38.670 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 11 v 81 (ObjNum 49 snap 0 seq_num 49) dirty exists 2024-09-12T23:39:38.679 INFO:tasks.rados.rados.0.smithi136.stdout:418: done (2 left) 2024-09-12T23:39:38.679 INFO:tasks.rados.rados.0.smithi136.stdout:421: done (1 left) 2024-09-12T23:39:38.679 INFO:tasks.rados.rados.0.smithi136.stdout:422: read oid 3 snap -1 2024-09-12T23:39:38.680 INFO:tasks.rados.rados.0.smithi136.stdout:422: expect (ObjNum 146 snap 37 seq_num 146) 2024-09-12T23:39:38.680 INFO:tasks.rados.rados.0.smithi136.stdout:423: snap_create 2024-09-12T23:39:38.680 INFO:tasks.rados.rados.0.smithi136.stdout:419: finishing write tid 1 to smithi13667081-16 2024-09-12T23:39:38.681 INFO:tasks.rados.rados.0.smithi136.stdout:419: finishing write tid 2 to smithi13667081-16 2024-09-12T23:39:38.683 INFO:tasks.rados.rados.0.smithi136.stdout:422: expect (ObjNum 146 snap 37 seq_num 146) 2024-09-12T23:39:38.690 INFO:tasks.rados.rados.0.smithi136.stdout:419: finishing write tid 3 to smithi13667081-16 2024-09-12T23:39:38.690 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 16 v 176 (ObjNum 148 snap 39 seq_num 148) dirty exists 2024-09-12T23:39:38.690 INFO:tasks.rados.rados.0.smithi136.stdout:419: left oid 16 (ObjNum 148 snap 39 seq_num 148) 2024-09-12T23:39:38.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:38 smithi136 bash[22601]: cluster 2024-09-12T23:39:37.470694+0000 mon.a (mon.0) 1170 : cluster [DBG] osdmap e141: 8 total, 8 up, 8 in 2024-09-12T23:39:38.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:38 smithi136 bash[22601]: audit 2024-09-12T23:39:38.015142+0000 mon.a (mon.0) 1171 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:39:38.755 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:38 smithi136 bash[32189]: cluster 2024-09-12T23:39:37.470694+0000 mon.a (mon.0) 1170 : cluster [DBG] osdmap e141: 8 total, 8 up, 8 in 2024-09-12T23:39:38.755 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:38 smithi136 bash[32189]: audit 2024-09-12T23:39:38.015142+0000 mon.a (mon.0) 1171 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:39:38.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:38 smithi181 bash[24776]: cluster 2024-09-12T23:39:37.470694+0000 mon.a (mon.0) 1170 : cluster [DBG] osdmap e141: 8 total, 8 up, 8 in 2024-09-12T23:39:38.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:38 smithi181 bash[24776]: audit 2024-09-12T23:39:38.015142+0000 mon.a (mon.0) 1171 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:39:39.498 INFO:tasks.rados.rados.0.smithi136.stdout:419: done (2 left) 2024-09-12T23:39:39.499 INFO:tasks.rados.rados.0.smithi136.stdout:422: done (1 left) 2024-09-12T23:39:39.499 INFO:tasks.rados.rados.0.smithi136.stdout:423: done (0 left) 2024-09-12T23:39:39.499 INFO:tasks.rados.rados.0.smithi136.stdout:424: rmattr oid 6 current snap is 40 2024-09-12T23:39:39.499 INFO:tasks.rados.rados.0.smithi136.stdout:425: rollback oid 3 current snap is 40 2024-09-12T23:39:39.499 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 3 to 36 2024-09-12T23:39:39.499 INFO:tasks.rados.rados.0.smithi136.stdout:426: rollback oid 16 current snap is 40 2024-09-12T23:39:39.499 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 16 to 29 2024-09-12T23:39:39.499 INFO:tasks.rados.rados.0.smithi136.stdout:427: snap_create 2024-09-12T23:39:39.507 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 6 v 0 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-09-12T23:39:39.513 INFO:tasks.rados.rados.0.smithi136.stdout:425: finishing rollback tid 0 to smithi13667081-3 2024-09-12T23:39:39.513 INFO:tasks.rados.rados.0.smithi136.stdout:426: finishing rollback tid 0 to smithi13667081-16 2024-09-12T23:39:39.513 INFO:tasks.rados.rados.0.smithi136.stdout:425: finishing rollback tid 1 to smithi13667081-3 2024-09-12T23:39:39.514 INFO:tasks.rados.rados.0.smithi136.stdout:425: finishing rollback tid 2 to smithi13667081-3 2024-09-12T23:39:39.514 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 3 v 248 (ObjNum 6 snap 0 seq_num 6) dirty exists 2024-09-12T23:39:39.514 INFO:tasks.rados.rados.0.smithi136.stdout:426: finishing rollback tid 1 to smithi13667081-16 2024-09-12T23:39:39.515 INFO:tasks.rados.rados.0.smithi136.stdout:426: finishing rollback tid 2 to smithi13667081-16 2024-09-12T23:39:39.515 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 16 v 180 (ObjNum 112 snap 24 seq_num 112) dirty exists 2024-09-12T23:39:39.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:39 smithi136 bash[22601]: cluster 2024-09-12T23:39:38.043578+0000 mgr.y (mgr.24446) 232 : cluster [DBG] pgmap v254: 121 pgs: 121 active+clean; 113 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:39:39.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:39 smithi136 bash[22601]: cluster 2024-09-12T23:39:38.502274+0000 mon.a (mon.0) 1172 : cluster [DBG] osdmap e142: 8 total, 8 up, 8 in 2024-09-12T23:39:39.755 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:39 smithi136 bash[32189]: cluster 2024-09-12T23:39:38.043578+0000 mgr.y (mgr.24446) 232 : cluster [DBG] pgmap v254: 121 pgs: 121 active+clean; 113 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:39:39.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:39 smithi136 bash[32189]: cluster 2024-09-12T23:39:38.502274+0000 mon.a (mon.0) 1172 : cluster [DBG] osdmap e142: 8 total, 8 up, 8 in 2024-09-12T23:39:39.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:39 smithi181 bash[24776]: cluster 2024-09-12T23:39:38.043578+0000 mgr.y (mgr.24446) 232 : cluster [DBG] pgmap v254: 121 pgs: 121 active+clean; 113 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:39:39.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:39 smithi181 bash[24776]: cluster 2024-09-12T23:39:38.502274+0000 mon.a (mon.0) 1172 : cluster [DBG] osdmap e142: 8 total, 8 up, 8 in 2024-09-12T23:39:40.498 INFO:tasks.rados.rados.0.smithi136.stdout:424: done (3 left) 2024-09-12T23:39:40.498 INFO:tasks.rados.rados.0.smithi136.stdout:425: done (2 left) 2024-09-12T23:39:40.499 INFO:tasks.rados.rados.0.smithi136.stdout:426: done (1 left) 2024-09-12T23:39:40.499 INFO:tasks.rados.rados.0.smithi136.stdout:427: done (0 left) 2024-09-12T23:39:40.499 INFO:tasks.rados.rados.0.smithi136.stdout:428: rollback oid 39 current snap is 41 2024-09-12T23:39:40.499 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 39 to 36 2024-09-12T23:39:40.499 INFO:tasks.rados.rados.0.smithi136.stdout:429: rollback oid 21 current snap is 41 2024-09-12T23:39:40.499 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 21 to 29 2024-09-12T23:39:40.499 INFO:tasks.rados.rados.0.smithi136.stdout:430: delete oid 24 current snap is 41 2024-09-12T23:39:40.509 INFO:tasks.rados.rados.0.smithi136.stdout:429: finishing rollback tid 0 to smithi13667081-21 2024-09-12T23:39:40.510 INFO:tasks.rados.rados.0.smithi136.stdout:429: finishing rollback tid 1 to smithi13667081-21 2024-09-12T23:39:40.510 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 21 v 89 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-12T23:39:40.514 INFO:tasks.rados.rados.0.smithi136.stdout:428: finishing rollback tid 0 to smithi13667081-39 2024-09-12T23:39:40.518 INFO:tasks.rados.rados.0.smithi136.stdout:429: done (2 left) 2024-09-12T23:39:40.518 INFO:tasks.rados.rados.0.smithi136.stdout:430: done (1 left) 2024-09-12T23:39:40.518 INFO:tasks.rados.rados.0.smithi136.stdout:431: delete oid 4 current snap is 41 2024-09-12T23:39:40.519 INFO:tasks.rados.rados.0.smithi136.stdout:428: finishing rollback tid 1 to smithi13667081-39 2024-09-12T23:39:40.519 INFO:tasks.rados.rados.0.smithi136.stdout:428: finishing rollback tid 2 to smithi13667081-39 2024-09-12T23:39:40.519 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 39 v 38 (ObjNum 38 snap 0 seq_num 38) dirty exists 2024-09-12T23:39:40.531 INFO:tasks.rados.rados.0.smithi136.stdout:428: done (1 left) 2024-09-12T23:39:40.531 INFO:tasks.rados.rados.0.smithi136.stdout:431: done (0 left) 2024-09-12T23:39:40.531 INFO:tasks.rados.rados.0.smithi136.stdout:432: read oid 33 snap 40 2024-09-12T23:39:40.531 INFO:tasks.rados.rados.0.smithi136.stdout:432: expect (ObjNum 114 snap 25 seq_num 114) 2024-09-12T23:39:40.531 INFO:tasks.rados.rados.0.smithi136.stdout:433: setattr oid 15 current snap is 41 2024-09-12T23:39:40.531 INFO:tasks.rados.rados.0.smithi136.stdout:434: setattr oid 29 current snap is 41 2024-09-12T23:39:40.533 INFO:tasks.rados.rados.0.smithi136.stdout:435: delete oid 34 current snap is 41 2024-09-12T23:39:40.539 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 15 v 250 (ObjNum 105 snap 23 seq_num 105) dirty exists 2024-09-12T23:39:40.540 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 29 v 141 (ObjNum 33 snap 0 seq_num 4243324688) dirty exists 2024-09-12T23:39:40.547 INFO:tasks.rados.rados.0.smithi136.stdout:432: expect (ObjNum 114 snap 25 seq_num 114) 2024-09-12T23:39:40.649 INFO:tasks.rados.rados.0.smithi136.stdout:432: done (3 left) 2024-09-12T23:39:40.649 INFO:tasks.rados.rados.0.smithi136.stdout:433: done (2 left) 2024-09-12T23:39:40.649 INFO:tasks.rados.rados.0.smithi136.stdout:434: done (1 left) 2024-09-12T23:39:40.649 INFO:tasks.rados.rados.0.smithi136.stdout:435: done (0 left) 2024-09-12T23:39:40.649 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 21 current snap is 41 2024-09-12T23:39:40.649 INFO:tasks.rados.rados.0.smithi136.stdout:436: seq_num 153 ranges {0=163840} 2024-09-12T23:39:40.651 INFO:tasks.rados.rados.0.smithi136.stdout:436: writing smithi13667081-21 from 0 to 163840 tid 1 2024-09-12T23:39:40.652 INFO:tasks.rados.rados.0.smithi136.stdout:437: snap_remove snap 23 2024-09-12T23:39:40.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:40 smithi181 bash[24776]: cluster 2024-09-12T23:39:39.502709+0000 mon.a (mon.0) 1173 : cluster [DBG] osdmap e143: 8 total, 8 up, 8 in 2024-09-12T23:39:40.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:40 smithi181 bash[24776]: cluster 2024-09-12T23:39:40.501786+0000 mon.a (mon.0) 1174 : cluster [DBG] osdmap e144: 8 total, 8 up, 8 in 2024-09-12T23:39:41.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:40 smithi136 bash[32189]: cluster 2024-09-12T23:39:39.502709+0000 mon.a (mon.0) 1173 : cluster [DBG] osdmap e143: 8 total, 8 up, 8 in 2024-09-12T23:39:41.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:40 smithi136 bash[32189]: cluster 2024-09-12T23:39:40.501786+0000 mon.a (mon.0) 1174 : cluster [DBG] osdmap e144: 8 total, 8 up, 8 in 2024-09-12T23:39:41.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:40 smithi136 bash[22601]: cluster 2024-09-12T23:39:39.502709+0000 mon.a (mon.0) 1173 : cluster [DBG] osdmap e143: 8 total, 8 up, 8 in 2024-09-12T23:39:41.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:40 smithi136 bash[22601]: cluster 2024-09-12T23:39:40.501786+0000 mon.a (mon.0) 1174 : cluster [DBG] osdmap e144: 8 total, 8 up, 8 in 2024-09-12T23:39:41.529 INFO:tasks.rados.rados.0.smithi136.stdout:437: done (1 left) 2024-09-12T23:39:41.529 INFO:tasks.rados.rados.0.smithi136.stdout:438: copy_from oid 41 from oid 28 current snap is 41 2024-09-12T23:39:41.529 INFO:tasks.rados.rados.0.smithi136.stdout:439: read oid 27 snap -1 2024-09-12T23:39:41.529 INFO:tasks.rados.rados.0.smithi136.stdout:439: expect deleted 2024-09-12T23:39:41.529 INFO:tasks.rados.rados.0.smithi136.stdout:440: rollback oid 13 current snap is 41 2024-09-12T23:39:41.529 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 13 to 29 2024-09-12T23:39:41.529 INFO:tasks.rados.rados.0.smithi136.stdout:441: snap_remove snap 37 2024-09-12T23:39:41.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:41 smithi181 bash[24776]: cluster 2024-09-12T23:39:40.044358+0000 mgr.y (mgr.24446) 233 : cluster [DBG] pgmap v257: 121 pgs: 121 active+clean; 111 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.0 MiB/s rd, 234 KiB/s wr, 20 op/s 2024-09-12T23:39:42.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:41 smithi136 bash[22601]: cluster 2024-09-12T23:39:40.044358+0000 mgr.y (mgr.24446) 233 : cluster [DBG] pgmap v257: 121 pgs: 121 active+clean; 111 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.0 MiB/s rd, 234 KiB/s wr, 20 op/s 2024-09-12T23:39:42.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:41 smithi136 bash[32189]: cluster 2024-09-12T23:39:40.044358+0000 mgr.y (mgr.24446) 233 : cluster [DBG] pgmap v257: 121 pgs: 121 active+clean; 111 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.0 MiB/s rd, 234 KiB/s wr, 20 op/s 2024-09-12T23:39:42.544 INFO:tasks.rados.rados.0.smithi136.stdout:441: done (4 left) 2024-09-12T23:39:42.544 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 29 current snap is 41 2024-09-12T23:39:42.544 INFO:tasks.rados.rados.0.smithi136.stdout:442: seq_num 154 ranges {0=180224} 2024-09-12T23:39:42.544 INFO:tasks.rados.rados.0.smithi136.stdout:442: writing smithi13667081-29 from 0 to 180224 tid 1 2024-09-12T23:39:42.545 INFO:tasks.rados.rados.0.smithi136.stdout:436: finishing write tid 1 to smithi13667081-21 2024-09-12T23:39:42.545 INFO:tasks.rados.rados.0.smithi136.stdout:436: finishing write tid 2 to smithi13667081-21 2024-09-12T23:39:42.545 INFO:tasks.rados.rados.0.smithi136.stdout:436: finishing write tid 3 to smithi13667081-21 2024-09-12T23:39:42.545 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 21 v 91 (ObjNum 153 snap 41 seq_num 153) dirty exists 2024-09-12T23:39:42.545 INFO:tasks.rados.rados.0.smithi136.stdout:436: left oid 21 (ObjNum 153 snap 41 seq_num 153) 2024-09-12T23:39:42.545 INFO:tasks.rados.rados.0.smithi136.stdout:440: finishing rollback tid 0 to smithi13667081-13 2024-09-12T23:39:42.545 INFO:tasks.rados.rados.0.smithi136.stdout:440: finishing rollback tid 1 to smithi13667081-13 2024-09-12T23:39:42.545 INFO:tasks.rados.rados.0.smithi136.stdout:440: finishing rollback tid 2 to smithi13667081-13 2024-09-12T23:39:42.545 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 13 v 149 (ObjNum 87 snap 18 seq_num 87) dirty exists 2024-09-12T23:39:42.545 INFO:tasks.rados.rados.0.smithi136.stdout:438: finishing copy_from racing read to smithi13667081-41 2024-09-12T23:39:42.545 INFO:tasks.rados.rados.0.smithi136.stdout:438: finishing copy_from to smithi13667081-41 2024-09-12T23:39:42.545 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 41 v 163 (ObjNum 140 snap 36 seq_num 140) dirty exists 2024-09-12T23:39:42.545 INFO:tasks.rados.rados.0.smithi136.stdout:436: done (4 left) 2024-09-12T23:39:42.546 INFO:tasks.rados.rados.0.smithi136.stdout:438: done (3 left) 2024-09-12T23:39:42.546 INFO:tasks.rados.rados.0.smithi136.stdout:439: done (2 left) 2024-09-12T23:39:42.546 INFO:tasks.rados.rados.0.smithi136.stdout:440: done (1 left) 2024-09-12T23:39:42.546 INFO:tasks.rados.rados.0.smithi136.stdout:443: delete oid 5 current snap is 41 2024-09-12T23:39:42.563 INFO:tasks.rados.rados.0.smithi136.stdout:443: done (1 left) 2024-09-12T23:39:42.563 INFO:tasks.rados.rados.0.smithi136.stdout:444: read oid 25 snap 39 2024-09-12T23:39:42.563 INFO:tasks.rados.rados.0.smithi136.stdout:444: expect (ObjNum 144 snap 36 seq_num 144) 2024-09-12T23:39:42.563 INFO:tasks.rados.rados.0.smithi136.stdout:445: read oid 47 snap -1 2024-09-12T23:39:42.563 INFO:tasks.rados.rados.0.smithi136.stdout:445: expect (ObjNum 74 snap 12 seq_num 74) 2024-09-12T23:39:42.563 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 36 current snap is 41 2024-09-12T23:39:42.563 INFO:tasks.rados.rados.0.smithi136.stdout:446: seq_num 156 ranges {2359296=221184} 2024-09-12T23:39:42.582 INFO:tasks.rados.rados.0.smithi136.stdout:446: writing smithi13667081-36 from 2359296 to 2580480 tid 1 2024-09-12T23:39:42.582 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 44 current snap is 41 2024-09-12T23:39:42.583 INFO:tasks.rados.rados.0.smithi136.stdout:447: seq_num 157 ranges {0=237568} 2024-09-12T23:39:42.584 INFO:tasks.rados.rados.0.smithi136.stdout:447: writing smithi13667081-44 from 0 to 237568 tid 1 2024-09-12T23:39:42.584 INFO:tasks.rados.rados.0.smithi136.stdout:448: snap_remove snap 36 2024-09-12T23:39:43.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:42 smithi136 bash[22601]: cluster 2024-09-12T23:39:41.532459+0000 mon.a (mon.0) 1175 : cluster [DBG] osdmap e145: 8 total, 8 up, 8 in 2024-09-12T23:39:43.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:42 smithi136 bash[32189]: cluster 2024-09-12T23:39:41.532459+0000 mon.a (mon.0) 1175 : cluster [DBG] osdmap e145: 8 total, 8 up, 8 in 2024-09-12T23:39:43.194 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:42 smithi181 bash[24776]: cluster 2024-09-12T23:39:41.532459+0000 mon.a (mon.0) 1175 : cluster [DBG] osdmap e145: 8 total, 8 up, 8 in 2024-09-12T23:39:43.574 INFO:tasks.rados.rados.0.smithi136.stdout:442: finishing write tid 1 to smithi13667081-29 2024-09-12T23:39:43.574 INFO:tasks.rados.rados.0.smithi136.stdout:442: finishing write tid 2 to smithi13667081-29 2024-09-12T23:39:43.574 INFO:tasks.rados.rados.0.smithi136.stdout:442: finishing write tid 3 to smithi13667081-29 2024-09-12T23:39:43.574 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 29 v 157 (ObjNum 154 snap 41 seq_num 154) dirty exists 2024-09-12T23:39:43.575 INFO:tasks.rados.rados.0.smithi136.stdout:442: left oid 29 (ObjNum 154 snap 41 seq_num 154) 2024-09-12T23:39:43.575 INFO:tasks.rados.rados.0.smithi136.stdout:444: expect (ObjNum 144 snap 36 seq_num 144) 2024-09-12T23:39:43.575 INFO:tasks.rados.rados.0.smithi136.stdout:445: expect (ObjNum 74 snap 12 seq_num 74) 2024-09-12T23:39:43.667 INFO:tasks.rados.rados.0.smithi136.stdout:447: finishing write tid 1 to smithi13667081-44 2024-09-12T23:39:43.667 INFO:tasks.rados.rados.0.smithi136.stdout:447: finishing write tid 2 to smithi13667081-44 2024-09-12T23:39:43.667 INFO:tasks.rados.rados.0.smithi136.stdout:446: finishing write tid 1 to smithi13667081-36 2024-09-12T23:39:43.667 INFO:tasks.rados.rados.0.smithi136.stdout:446: finishing write tid 2 to smithi13667081-36 2024-09-12T23:39:43.667 INFO:tasks.rados.rados.0.smithi136.stdout:447: finishing write tid 3 to smithi13667081-44 2024-09-12T23:39:43.667 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 44 v 192 (ObjNum 157 snap 41 seq_num 157) dirty exists 2024-09-12T23:39:43.667 INFO:tasks.rados.rados.0.smithi136.stdout:447: left oid 44 (ObjNum 157 snap 41 seq_num 157) 2024-09-12T23:39:43.667 INFO:tasks.rados.rados.0.smithi136.stdout:446: finishing write tid 3 to smithi13667081-36 2024-09-12T23:39:43.667 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 36 v 128 (ObjNum 156 snap 41 seq_num 156) dirty exists 2024-09-12T23:39:43.667 INFO:tasks.rados.rados.0.smithi136.stdout:446: left oid 36 (ObjNum 156 snap 41 seq_num 156) 2024-09-12T23:39:43.668 INFO:tasks.rados.rados.0.smithi136.stdout:442: done (5 left) 2024-09-12T23:39:43.668 INFO:tasks.rados.rados.0.smithi136.stdout:444: done (4 left) 2024-09-12T23:39:43.668 INFO:tasks.rados.rados.0.smithi136.stdout:445: done (3 left) 2024-09-12T23:39:43.668 INFO:tasks.rados.rados.0.smithi136.stdout:446: done (2 left) 2024-09-12T23:39:43.668 INFO:tasks.rados.rados.0.smithi136.stdout:447: done (1 left) 2024-09-12T23:39:43.668 INFO:tasks.rados.rados.0.smithi136.stdout:448: done (0 left) 2024-09-12T23:39:43.668 INFO:tasks.rados.rados.0.smithi136.stdout:449: rollback oid 24 current snap is 41 2024-09-12T23:39:43.668 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 24 to 28 2024-09-12T23:39:43.668 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 14 current snap is 41 2024-09-12T23:39:43.668 INFO:tasks.rados.rados.0.smithi136.stdout:450: seq_num 158 ranges {1941504=253952} 2024-09-12T23:39:43.697 INFO:tasks.rados.rados.0.smithi136.stdout:450: writing smithi13667081-14 from 1941504 to 2195456 tid 1 2024-09-12T23:39:43.698 INFO:tasks.rados.rados.0.smithi136.stdout:451: copy_from oid 6 from oid 19 current snap is 41 2024-09-12T23:39:43.698 INFO:tasks.rados.rados.0.smithi136.stdout:449: finishing rollback tid 1 to smithi13667081-24 2024-09-12T23:39:43.698 INFO:tasks.rados.rados.0.smithi136.stdout:449: finishing rollback tid 2 to smithi13667081-24 2024-09-12T23:39:43.698 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 24 v 84 (ObjNum 54 snap 4 seq_num 54) dirty exists 2024-09-12T23:39:43.698 INFO:tasks.rados.rados.0.smithi136.stdout:449: done (2 left) 2024-09-12T23:39:43.698 INFO:tasks.rados.rados.0.smithi136.stdout:452: snap_create 2024-09-12T23:39:43.700 INFO:tasks.rados.rados.0.smithi136.stdout:451: finishing copy_from to smithi13667081-6 2024-09-12T23:39:43.700 INFO:tasks.rados.rados.0.smithi136.stdout:451: got expected ENOENT (src dne) 2024-09-12T23:39:43.701 INFO:tasks.rados.rados.0.smithi136.stdout:451: finishing copy_from racing read to smithi13667081-6 2024-09-12T23:39:43.702 INFO:tasks.rados.rados.0.smithi136.stdout:450: finishing write tid 1 to smithi13667081-14 2024-09-12T23:39:43.703 INFO:tasks.rados.rados.0.smithi136.stdout:450: finishing write tid 2 to smithi13667081-14 2024-09-12T23:39:43.704 INFO:tasks.rados.rados.0.smithi136.stdout:450: finishing write tid 3 to smithi13667081-14 2024-09-12T23:39:43.704 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 14 v 197 (ObjNum 158 snap 41 seq_num 158) dirty exists 2024-09-12T23:39:43.704 INFO:tasks.rados.rados.0.smithi136.stdout:450: left oid 14 (ObjNum 158 snap 41 seq_num 158) 2024-09-12T23:39:43.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:43 smithi181 bash[24776]: cluster 2024-09-12T23:39:42.045120+0000 mgr.y (mgr.24446) 234 : cluster [DBG] pgmap v260: 121 pgs: 121 active+clean; 111 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.0 MiB/s rd, 234 KiB/s wr, 20 op/s 2024-09-12T23:39:43.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:43 smithi181 bash[24776]: cluster 2024-09-12T23:39:42.539313+0000 mon.a (mon.0) 1176 : cluster [DBG] osdmap e146: 8 total, 8 up, 8 in 2024-09-12T23:39:44.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:43 smithi136 bash[22601]: cluster 2024-09-12T23:39:42.045120+0000 mgr.y (mgr.24446) 234 : cluster [DBG] pgmap v260: 121 pgs: 121 active+clean; 111 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.0 MiB/s rd, 234 KiB/s wr, 20 op/s 2024-09-12T23:39:44.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:43 smithi136 bash[22601]: cluster 2024-09-12T23:39:42.539313+0000 mon.a (mon.0) 1176 : cluster [DBG] osdmap e146: 8 total, 8 up, 8 in 2024-09-12T23:39:44.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:43 smithi136 bash[32189]: cluster 2024-09-12T23:39:42.045120+0000 mgr.y (mgr.24446) 234 : cluster [DBG] pgmap v260: 121 pgs: 121 active+clean; 111 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.0 MiB/s rd, 234 KiB/s wr, 20 op/s 2024-09-12T23:39:44.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:43 smithi136 bash[32189]: cluster 2024-09-12T23:39:42.539313+0000 mon.a (mon.0) 1176 : cluster [DBG] osdmap e146: 8 total, 8 up, 8 in 2024-09-12T23:39:44.586 INFO:tasks.rados.rados.0.smithi136.stdout:450: done (2 left) 2024-09-12T23:39:44.586 INFO:tasks.rados.rados.0.smithi136.stdout:451: done (1 left) 2024-09-12T23:39:44.586 INFO:tasks.rados.rados.0.smithi136.stdout:452: done (0 left) 2024-09-12T23:39:44.587 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 12 current snap is 42 2024-09-12T23:39:44.587 INFO:tasks.rados.rados.0.smithi136.stdout:453: seq_num 159 ranges {2326528=270336} 2024-09-12T23:39:44.628 INFO:tasks.rados.rados.0.smithi136.stdout:453: writing smithi13667081-12 from 2326528 to 2596864 tid 1 2024-09-12T23:39:44.629 INFO:tasks.rados.rados.0.smithi136.stdout:454: read oid 9 snap -1 2024-09-12T23:39:44.629 INFO:tasks.rados.rados.0.smithi136.stdout:454: expect deleted 2024-09-12T23:39:44.629 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 26 current snap is 42 2024-09-12T23:39:44.629 INFO:tasks.rados.rados.0.smithi136.stdout:455: seq_num 160 ranges {1671168=286720} 2024-09-12T23:39:44.653 INFO:tasks.rados.rados.0.smithi136.stdout:455: writing smithi13667081-26 from 1671168 to 1957888 tid 1 2024-09-12T23:39:44.653 INFO:tasks.rados.rados.0.smithi136.stdout:456: snap_create 2024-09-12T23:39:44.653 INFO:tasks.rados.rados.0.smithi136.stdout:453: finishing write tid 1 to smithi13667081-12 2024-09-12T23:39:44.653 INFO:tasks.rados.rados.0.smithi136.stdout:453: finishing write tid 2 to smithi13667081-12 2024-09-12T23:39:44.654 INFO:tasks.rados.rados.0.smithi136.stdout:453: finishing write tid 3 to smithi13667081-12 2024-09-12T23:39:44.654 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 12 v 269 (ObjNum 159 snap 42 seq_num 159) dirty exists 2024-09-12T23:39:44.654 INFO:tasks.rados.rados.0.smithi136.stdout:453: left oid 12 (ObjNum 159 snap 42 seq_num 159) 2024-09-12T23:39:44.660 INFO:tasks.rados.rados.0.smithi136.stdout:455: finishing write tid 1 to smithi13667081-26 2024-09-12T23:39:44.661 INFO:tasks.rados.rados.0.smithi136.stdout:455: finishing write tid 2 to smithi13667081-26 2024-09-12T23:39:44.664 INFO:tasks.rados.rados.0.smithi136.stdout:455: finishing write tid 3 to smithi13667081-26 2024-09-12T23:39:44.664 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 26 v 87 (ObjNum 160 snap 42 seq_num 160) dirty exists 2024-09-12T23:39:44.664 INFO:tasks.rados.rados.0.smithi136.stdout:455: left oid 26 (ObjNum 160 snap 42 seq_num 160) 2024-09-12T23:39:44.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:44 smithi181 bash[24776]: cluster 2024-09-12T23:39:43.578043+0000 mon.a (mon.0) 1177 : cluster [DBG] osdmap e147: 8 total, 8 up, 8 in 2024-09-12T23:39:45.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:44 smithi136 bash[22601]: cluster 2024-09-12T23:39:43.578043+0000 mon.a (mon.0) 1177 : cluster [DBG] osdmap e147: 8 total, 8 up, 8 in 2024-09-12T23:39:45.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:44 smithi136 bash[32189]: cluster 2024-09-12T23:39:43.578043+0000 mon.a (mon.0) 1177 : cluster [DBG] osdmap e147: 8 total, 8 up, 8 in 2024-09-12T23:39:45.465 INFO:tasks.rados.rados.0.smithi136.stdout:453: done (3 left) 2024-09-12T23:39:45.466 INFO:tasks.rados.rados.0.smithi136.stdout:454: done (2 left) 2024-09-12T23:39:45.466 INFO:tasks.rados.rados.0.smithi136.stdout:455: done (1 left) 2024-09-12T23:39:45.466 INFO:tasks.rados.rados.0.smithi136.stdout:456: done (0 left) 2024-09-12T23:39:45.466 INFO:tasks.rados.rados.0.smithi136.stdout:457: copy_from oid 15 from oid 40 current snap is 43 2024-09-12T23:39:45.466 INFO:tasks.rados.rados.0.smithi136.stdout:458: read oid 7 snap 39 2024-09-12T23:39:45.466 INFO:tasks.rados.rados.0.smithi136.stdout:458: expect deleted 2024-09-12T23:39:45.466 INFO:tasks.rados.rados.0.smithi136.stdout:459: rollback oid 31 current snap is 43 2024-09-12T23:39:45.466 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 31 to 19 2024-09-12T23:39:45.466 INFO:tasks.rados.rados.0.smithi136.stdout:460: delete oid 35 current snap is 43 2024-09-12T23:39:45.468 INFO:tasks.rados.rados.0.smithi136.stdout:458: done (3 left) 2024-09-12T23:39:45.468 INFO:tasks.rados.rados.0.smithi136.stdout:460: done (2 left) 2024-09-12T23:39:45.468 INFO:tasks.rados.rados.0.smithi136.stdout:461: read oid 8 snap -1 2024-09-12T23:39:45.468 INFO:tasks.rados.rados.0.smithi136.stdout:461: expect deleted 2024-09-12T23:39:45.469 INFO:tasks.rados.rados.0.smithi136.stdout:462: delete oid 17 current snap is 43 2024-09-12T23:39:45.479 INFO:tasks.rados.rados.0.smithi136.stdout:461: done (3 left) 2024-09-12T23:39:45.479 INFO:tasks.rados.rados.0.smithi136.stdout:462: done (2 left) 2024-09-12T23:39:45.479 INFO:tasks.rados.rados.0.smithi136.stdout:463: rmattr oid 7 current snap is 43 2024-09-12T23:39:45.479 INFO:tasks.rados.rados.0.smithi136.stdout:463: done (2 left) 2024-09-12T23:39:45.479 INFO:tasks.rados.rados.0.smithi136.stdout:464: copy_from oid 14 from oid 45 current snap is 43 2024-09-12T23:39:45.479 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 16 current snap is 43 2024-09-12T23:39:45.479 INFO:tasks.rados.rados.0.smithi136.stdout:465: seq_num 163 ranges {4825088=335872} 2024-09-12T23:39:45.530 INFO:tasks.rados.rados.0.smithi136.stdout:465: writing smithi13667081-16 from 4825088 to 5160960 tid 1 2024-09-12T23:39:45.530 INFO:tasks.rados.rados.0.smithi136.stdout:466: rollback oid 28 current snap is 43 2024-09-12T23:39:45.530 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 28 to 28 2024-09-12T23:39:45.531 INFO:tasks.rados.rados.0.smithi136.stdout:457: finishing copy_from racing read to smithi13667081-15 2024-09-12T23:39:45.531 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 8 current snap is 43 2024-09-12T23:39:45.531 INFO:tasks.rados.rados.0.smithi136.stdout:459: finishing rollback tid 1 to smithi13667081-31 2024-09-12T23:39:45.531 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 31 v 90 (ObjNum 4113 snap 0 seq_num 1427292928) dirty dne 2024-09-12T23:39:45.531 INFO:tasks.rados.rados.0.smithi136.stdout:467: seq_num 164 ranges {0=352256} 2024-09-12T23:39:45.534 INFO:tasks.rados.rados.0.smithi136.stdout:467: writing smithi13667081-8 from 0 to 352256 tid 1 2024-09-12T23:39:45.534 INFO:tasks.rados.rados.0.smithi136.stdout:459: done (5 left) 2024-09-12T23:39:45.534 INFO:tasks.rados.rados.0.smithi136.stdout:468: copy_from oid 30 from oid 44 current snap is 43 2024-09-12T23:39:45.534 INFO:tasks.rados.rados.0.smithi136.stdout:457: finishing copy_from to smithi13667081-15 2024-09-12T23:39:45.534 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 15 v 23 (ObjNum 39 snap 0 seq_num 39) dirty exists 2024-09-12T23:39:45.535 INFO:tasks.rados.rados.0.smithi136.stdout:464: finishing copy_from racing read to smithi13667081-14 2024-09-12T23:39:45.535 INFO:tasks.rados.rados.0.smithi136.stdout:464: finishing copy_from to smithi13667081-14 2024-09-12T23:39:45.535 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 14 v 113 (ObjNum 141 snap 36 seq_num 141) dirty exists 2024-09-12T23:39:45.535 INFO:tasks.rados.rados.0.smithi136.stdout:457: done (5 left) 2024-09-12T23:39:45.535 INFO:tasks.rados.rados.0.smithi136.stdout:464: done (4 left) 2024-09-12T23:39:45.535 INFO:tasks.rados.rados.0.smithi136.stdout:469: read oid 21 snap -1 2024-09-12T23:39:45.535 INFO:tasks.rados.rados.0.smithi136.stdout:469: expect (ObjNum 153 snap 41 seq_num 153) 2024-09-12T23:39:45.535 INFO:tasks.rados.rados.0.smithi136.stdout:470: copy_from oid 9 from oid 3 current snap is 43 2024-09-12T23:39:45.535 INFO:tasks.rados.rados.0.smithi136.stdout:471: rmattr oid 4 current snap is 43 2024-09-12T23:39:45.535 INFO:tasks.rados.rados.0.smithi136.stdout:471: done (6 left) 2024-09-12T23:39:45.535 INFO:tasks.rados.rados.0.smithi136.stdout:472: rmattr oid 25 current snap is 43 2024-09-12T23:39:45.535 INFO:tasks.rados.rados.0.smithi136.stdout:472: done (6 left) 2024-09-12T23:39:45.535 INFO:tasks.rados.rados.0.smithi136.stdout:473: setattr oid 1 current snap is 43 2024-09-12T23:39:45.538 INFO:tasks.rados.rados.0.smithi136.stdout:465: finishing write tid 1 to smithi13667081-16 2024-09-12T23:39:45.538 INFO:tasks.rados.rados.0.smithi136.stdout:474: rollback oid 27 current snap is 43 2024-09-12T23:39:45.538 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 27 to 29 2024-09-12T23:39:45.538 INFO:tasks.rados.rados.0.smithi136.stdout:475: delete oid 41 current snap is 43 2024-09-12T23:39:45.540 INFO:tasks.rados.rados.0.smithi136.stdout:465: finishing write tid 2 to smithi13667081-16 2024-09-12T23:39:45.542 INFO:tasks.rados.rados.0.smithi136.stdout:469: expect (ObjNum 153 snap 41 seq_num 153) 2024-09-12T23:39:45.555 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 1 v 98 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-12T23:39:45.555 INFO:tasks.rados.rados.0.smithi136.stdout:465: finishing write tid 3 to smithi13667081-16 2024-09-12T23:39:45.555 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 16 v 202 (ObjNum 163 snap 43 seq_num 163) dirty exists 2024-09-12T23:39:45.556 INFO:tasks.rados.rados.0.smithi136.stdout:465: left oid 16 (ObjNum 163 snap 43 seq_num 163) 2024-09-12T23:39:45.556 INFO:tasks.rados.rados.0.smithi136.stdout:470: finishing copy_from racing read to smithi13667081-9 2024-09-12T23:39:45.556 INFO:tasks.rados.rados.0.smithi136.stdout:466: finishing rollback tid 0 to smithi13667081-28 2024-09-12T23:39:45.556 INFO:tasks.rados.rados.0.smithi136.stdout:474: finishing rollback tid 1 to smithi13667081-27 2024-09-12T23:39:45.556 INFO:tasks.rados.rados.0.smithi136.stdout:467: finishing write tid 1 to smithi13667081-8 2024-09-12T23:39:45.556 INFO:tasks.rados.rados.0.smithi136.stdout:466: finishing rollback tid 1 to smithi13667081-28 2024-09-12T23:39:45.556 INFO:tasks.rados.rados.0.smithi136.stdout:467: finishing write tid 2 to smithi13667081-8 2024-09-12T23:39:45.556 INFO:tasks.rados.rados.0.smithi136.stdout:474: finishing rollback tid 2 to smithi13667081-27 2024-09-12T23:39:45.556 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 27 v 188 (ObjNum 26 snap 0 seq_num 26) dirty exists 2024-09-12T23:39:45.556 INFO:tasks.rados.rados.0.smithi136.stdout:466: finishing rollback tid 2 to smithi13667081-28 2024-09-12T23:39:45.556 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 28 v 206 (ObjNum 161 snap 0 seq_num 0) dirty exists 2024-09-12T23:39:45.557 INFO:tasks.rados.rados.0.smithi136.stdout:470: finishing copy_from to smithi13667081-9 2024-09-12T23:39:45.557 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 9 v 248 (ObjNum 6 snap 0 seq_num 6) dirty exists 2024-09-12T23:39:45.557 INFO:tasks.rados.rados.0.smithi136.stdout:468: finishing copy_from racing read to smithi13667081-30 2024-09-12T23:39:45.557 INFO:tasks.rados.rados.0.smithi136.stdout:465: done (8 left) 2024-09-12T23:39:45.557 INFO:tasks.rados.rados.0.smithi136.stdout:466: done (7 left) 2024-09-12T23:39:45.557 INFO:tasks.rados.rados.0.smithi136.stdout:469: done (6 left) 2024-09-12T23:39:45.557 INFO:tasks.rados.rados.0.smithi136.stdout:470: done (5 left) 2024-09-12T23:39:45.557 INFO:tasks.rados.rados.0.smithi136.stdout:473: done (4 left) 2024-09-12T23:39:45.557 INFO:tasks.rados.rados.0.smithi136.stdout:474: done (3 left) 2024-09-12T23:39:45.557 INFO:tasks.rados.rados.0.smithi136.stdout:475: done (2 left) 2024-09-12T23:39:45.557 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 22 current snap is 43 2024-09-12T23:39:45.557 INFO:tasks.rados.rados.0.smithi136.stdout:467: finishing write tid 3 to smithi13667081-8 2024-09-12T23:39:45.557 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 8 v 134 (ObjNum 164 snap 43 seq_num 164) dirty exists 2024-09-12T23:39:45.557 INFO:tasks.rados.rados.0.smithi136.stdout:467: left oid 8 (ObjNum 164 snap 43 seq_num 164) 2024-09-12T23:39:45.558 INFO:tasks.rados.rados.0.smithi136.stdout:476: seq_num 166 ranges {360448=385024} 2024-09-12T23:39:45.559 INFO:tasks.rados.rados.0.smithi136.stdout:476: writing smithi13667081-22 from 360448 to 745472 tid 1 2024-09-12T23:39:45.559 INFO:tasks.rados.rados.0.smithi136.stdout:467: done (2 left) 2024-09-12T23:39:45.559 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 4 current snap is 43 2024-09-12T23:39:45.559 INFO:tasks.rados.rados.0.smithi136.stdout:477: seq_num 167 ranges {0=401408} 2024-09-12T23:39:45.562 INFO:tasks.rados.rados.0.smithi136.stdout:477: writing smithi13667081-4 from 0 to 401408 tid 1 2024-09-12T23:39:45.562 INFO:tasks.rados.rados.0.smithi136.stdout:478: read oid 11 snap -1 2024-09-12T23:39:45.562 INFO:tasks.rados.rados.0.smithi136.stdout:478: expect (ObjNum 49 snap 0 seq_num 49) 2024-09-12T23:39:45.562 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 3 current snap is 43 2024-09-12T23:39:45.562 INFO:tasks.rados.rados.0.smithi136.stdout:468: finishing copy_from to smithi13667081-30 2024-09-12T23:39:45.562 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 30 v 192 (ObjNum 157 snap 41 seq_num 157) dirty exists 2024-09-12T23:39:45.562 INFO:tasks.rados.rados.0.smithi136.stdout:479: seq_num 168 ranges {106496=417792} 2024-09-12T23:39:45.566 INFO:tasks.rados.rados.0.smithi136.stdout:479: writing smithi13667081-3 from 106496 to 524288 tid 1 2024-09-12T23:39:45.566 INFO:tasks.rados.rados.0.smithi136.stdout:468: done (4 left) 2024-09-12T23:39:45.566 INFO:tasks.rados.rados.0.smithi136.stdout:480: read oid 10 snap 40 2024-09-12T23:39:45.566 INFO:tasks.rados.rados.0.smithi136.stdout:480: expect deleted 2024-09-12T23:39:45.566 INFO:tasks.rados.rados.0.smithi136.stdout:481: rollback oid 35 current snap is 43 2024-09-12T23:39:45.566 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 35 to 39 2024-09-12T23:39:45.566 INFO:tasks.rados.rados.0.smithi136.stdout:482: delete oid 40 current snap is 43 2024-09-12T23:39:45.567 INFO:tasks.rados.rados.0.smithi136.stdout:476: finishing write tid 1 to smithi13667081-22 2024-09-12T23:39:45.567 INFO:tasks.rados.rados.0.smithi136.stdout:476: finishing write tid 2 to smithi13667081-22 2024-09-12T23:39:45.568 INFO:tasks.rados.rados.0.smithi136.stdout:477: finishing write tid 1 to smithi13667081-4 2024-09-12T23:39:45.568 INFO:tasks.rados.rados.0.smithi136.stdout:476: finishing write tid 3 to smithi13667081-22 2024-09-12T23:39:45.568 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 22 v 101 (ObjNum 166 snap 43 seq_num 166) dirty exists 2024-09-12T23:39:45.568 INFO:tasks.rados.rados.0.smithi136.stdout:476: left oid 22 (ObjNum 166 snap 43 seq_num 166) 2024-09-12T23:39:45.568 INFO:tasks.rados.rados.0.smithi136.stdout:477: finishing write tid 2 to smithi13667081-4 2024-09-12T23:39:45.575 INFO:tasks.rados.rados.0.smithi136.stdout:478: expect (ObjNum 49 snap 0 seq_num 49) 2024-09-12T23:39:45.579 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:39:45 smithi181 bash[25717]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:39:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-12T23:39:45.624 INFO:tasks.rados.rados.0.smithi136.stdout:479: finishing write tid 1 to smithi13667081-3 2024-09-12T23:39:45.624 INFO:tasks.rados.rados.0.smithi136.stdout:477: finishing write tid 3 to smithi13667081-4 2024-09-12T23:39:45.624 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 4 v 68 (ObjNum 167 snap 43 seq_num 167) dirty exists 2024-09-12T23:39:45.624 INFO:tasks.rados.rados.0.smithi136.stdout:477: left oid 4 (ObjNum 167 snap 43 seq_num 167) 2024-09-12T23:39:45.624 INFO:tasks.rados.rados.0.smithi136.stdout:479: finishing write tid 2 to smithi13667081-3 2024-09-12T23:39:45.624 INFO:tasks.rados.rados.0.smithi136.stdout:481: finishing rollback tid 1 to smithi13667081-35 2024-09-12T23:39:45.624 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 35 v 169 (ObjNum 161 snap 0 seq_num 0) dirty dne 2024-09-12T23:39:45.625 INFO:tasks.rados.rados.0.smithi136.stdout:479: finishing write tid 3 to smithi13667081-3 2024-09-12T23:39:45.625 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 3 v 275 (ObjNum 168 snap 43 seq_num 168) dirty exists 2024-09-12T23:39:45.625 INFO:tasks.rados.rados.0.smithi136.stdout:479: left oid 3 (ObjNum 168 snap 43 seq_num 168) 2024-09-12T23:39:45.625 INFO:tasks.rados.rados.0.smithi136.stdout:476: done (6 left) 2024-09-12T23:39:45.625 INFO:tasks.rados.rados.0.smithi136.stdout:477: done (5 left) 2024-09-12T23:39:45.625 INFO:tasks.rados.rados.0.smithi136.stdout:478: done (4 left) 2024-09-12T23:39:45.625 INFO:tasks.rados.rados.0.smithi136.stdout:479: done (3 left) 2024-09-12T23:39:45.625 INFO:tasks.rados.rados.0.smithi136.stdout:480: done (2 left) 2024-09-12T23:39:45.625 INFO:tasks.rados.rados.0.smithi136.stdout:481: done (1 left) 2024-09-12T23:39:45.625 INFO:tasks.rados.rados.0.smithi136.stdout:482: done (0 left) 2024-09-12T23:39:45.625 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 28 current snap is 43 2024-09-12T23:39:45.625 INFO:tasks.rados.rados.0.smithi136.stdout:483: seq_num 170 ranges {0=450560} 2024-09-12T23:39:45.629 INFO:tasks.rados.rados.0.smithi136.stdout:483: writing smithi13667081-28 from 0 to 450560 tid 1 2024-09-12T23:39:45.629 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 17 current snap is 43 2024-09-12T23:39:45.629 INFO:tasks.rados.rados.0.smithi136.stdout:484: seq_num 171 ranges {0=466944} 2024-09-12T23:39:45.635 INFO:tasks.rados.rados.0.smithi136.stdout:484: writing smithi13667081-17 from 0 to 466944 tid 1 2024-09-12T23:39:45.635 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 12 current snap is 43 2024-09-12T23:39:45.635 INFO:tasks.rados.rados.0.smithi136.stdout:483: finishing write tid 1 to smithi13667081-28 2024-09-12T23:39:45.635 INFO:tasks.rados.rados.0.smithi136.stdout:483: finishing write tid 2 to smithi13667081-28 2024-09-12T23:39:45.635 INFO:tasks.rados.rados.0.smithi136.stdout:485: seq_num 172 ranges {2596864=483328} 2024-09-12T23:39:45.661 INFO:tasks.rados.rados.0.smithi136.stdout:485: writing smithi13667081-12 from 2596864 to 3080192 tid 1 2024-09-12T23:39:45.661 INFO:tasks.rados.rados.0.smithi136.stdout:486: read oid 34 snap -1 2024-09-12T23:39:45.661 INFO:tasks.rados.rados.0.smithi136.stdout:486: expect deleted 2024-09-12T23:39:45.661 INFO:tasks.rados.rados.0.smithi136.stdout:487: snap_remove snap 39 2024-09-12T23:39:45.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:45 smithi181 bash[24776]: cluster 2024-09-12T23:39:44.046407+0000 mgr.y (mgr.24446) 235 : cluster [DBG] pgmap v263: 121 pgs: 121 active+clean; 115 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s rd, 834 KiB/s wr, 21 op/s 2024-09-12T23:39:45.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:45 smithi181 bash[24776]: cluster 2024-09-12T23:39:44.590900+0000 mon.a (mon.0) 1178 : cluster [DBG] osdmap e148: 8 total, 8 up, 8 in 2024-09-12T23:39:45.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:45 smithi181 bash[24776]: cluster 2024-09-12T23:39:45.469447+0000 mon.a (mon.0) 1179 : cluster [DBG] osdmap e149: 8 total, 8 up, 8 in 2024-09-12T23:39:46.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:45 smithi136 bash[22601]: cluster 2024-09-12T23:39:44.046407+0000 mgr.y (mgr.24446) 235 : cluster [DBG] pgmap v263: 121 pgs: 121 active+clean; 115 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s rd, 834 KiB/s wr, 21 op/s 2024-09-12T23:39:46.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:45 smithi136 bash[22601]: cluster 2024-09-12T23:39:44.590900+0000 mon.a (mon.0) 1178 : cluster [DBG] osdmap e148: 8 total, 8 up, 8 in 2024-09-12T23:39:46.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:45 smithi136 bash[22601]: cluster 2024-09-12T23:39:45.469447+0000 mon.a (mon.0) 1179 : cluster [DBG] osdmap e149: 8 total, 8 up, 8 in 2024-09-12T23:39:46.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:45 smithi136 bash[32189]: cluster 2024-09-12T23:39:44.046407+0000 mgr.y (mgr.24446) 235 : cluster [DBG] pgmap v263: 121 pgs: 121 active+clean; 115 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s rd, 834 KiB/s wr, 21 op/s 2024-09-12T23:39:46.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:45 smithi136 bash[32189]: cluster 2024-09-12T23:39:44.590900+0000 mon.a (mon.0) 1178 : cluster [DBG] osdmap e148: 8 total, 8 up, 8 in 2024-09-12T23:39:46.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:45 smithi136 bash[32189]: cluster 2024-09-12T23:39:45.469447+0000 mon.a (mon.0) 1179 : cluster [DBG] osdmap e149: 8 total, 8 up, 8 in 2024-09-12T23:39:46.505 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:39:46 smithi136 bash[22852]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:39:46] "GET /metrics HTTP/1.1" 200 240116 "" "Prometheus/2.43.0" 2024-09-12T23:39:46.597 INFO:tasks.rados.rados.0.smithi136.stdout:487: done (4 left) 2024-09-12T23:39:46.597 INFO:tasks.rados.rados.0.smithi136.stdout:488: rollback oid 1 current snap is 43 2024-09-12T23:39:46.597 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 1 to 40 2024-09-12T23:39:46.597 INFO:tasks.rados.rados.0.smithi136.stdout:483: finishing write tid 3 to smithi13667081-28 2024-09-12T23:39:46.597 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 28 v 208 (ObjNum 170 snap 43 seq_num 170) dirty exists 2024-09-12T23:39:46.597 INFO:tasks.rados.rados.0.smithi136.stdout:483: left oid 28 (ObjNum 170 snap 43 seq_num 170) 2024-09-12T23:39:46.597 INFO:tasks.rados.rados.0.smithi136.stdout:483: done (4 left) 2024-09-12T23:39:46.597 INFO:tasks.rados.rados.0.smithi136.stdout:489: delete oid 38 current snap is 43 2024-09-12T23:39:46.597 INFO:tasks.rados.rados.0.smithi136.stdout:484: finishing write tid 1 to smithi13667081-17 2024-09-12T23:39:46.597 INFO:tasks.rados.rados.0.smithi136.stdout:484: finishing write tid 2 to smithi13667081-17 2024-09-12T23:39:46.597 INFO:tasks.rados.rados.0.smithi136.stdout:484: finishing write tid 3 to smithi13667081-17 2024-09-12T23:39:46.598 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 17 v 193 (ObjNum 171 snap 43 seq_num 171) dirty exists 2024-09-12T23:39:46.598 INFO:tasks.rados.rados.0.smithi136.stdout:484: left oid 17 (ObjNum 171 snap 43 seq_num 171) 2024-09-12T23:39:46.598 INFO:tasks.rados.rados.0.smithi136.stdout:485: finishing write tid 1 to smithi13667081-12 2024-09-12T23:39:46.598 INFO:tasks.rados.rados.0.smithi136.stdout:485: finishing write tid 2 to smithi13667081-12 2024-09-12T23:39:46.598 INFO:tasks.rados.rados.0.smithi136.stdout:485: finishing write tid 3 to smithi13667081-12 2024-09-12T23:39:46.598 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 12 v 278 (ObjNum 172 snap 43 seq_num 172) dirty exists 2024-09-12T23:39:46.598 INFO:tasks.rados.rados.0.smithi136.stdout:485: left oid 12 (ObjNum 172 snap 43 seq_num 172) 2024-09-12T23:39:46.610 INFO:tasks.rados.rados.0.smithi136.stdout:484: done (4 left) 2024-09-12T23:39:46.610 INFO:tasks.rados.rados.0.smithi136.stdout:485: done (3 left) 2024-09-12T23:39:46.610 INFO:tasks.rados.rados.0.smithi136.stdout:486: done (2 left) 2024-09-12T23:39:46.610 INFO:tasks.rados.rados.0.smithi136.stdout:489: done (1 left) 2024-09-12T23:39:46.611 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 4 current snap is 43 2024-09-12T23:39:46.611 INFO:tasks.rados.rados.0.smithi136.stdout:490: seq_num 174 ranges {401408=516096} 2024-09-12T23:39:46.622 INFO:tasks.rados.rados.0.smithi136.stdout:490: writing smithi13667081-4 from 401408 to 917504 tid 1 2024-09-12T23:39:46.623 INFO:tasks.rados.rados.0.smithi136.stdout:491: snap_remove snap 41 2024-09-12T23:39:47.598 INFO:tasks.rados.rados.0.smithi136.stdout:491: done (2 left) 2024-09-12T23:39:47.598 INFO:tasks.rados.rados.0.smithi136.stdout:492: copy_from oid 14 from oid 43 current snap is 43 2024-09-12T23:39:47.599 INFO:tasks.rados.rados.0.smithi136.stdout:493: setattr oid 48 current snap is 43 2024-09-12T23:39:47.605 INFO:tasks.rados.rados.0.smithi136.stdout:494: rollback oid 21 current snap is 43 2024-09-12T23:39:47.605 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 21 to 19 2024-09-12T23:39:47.605 INFO:tasks.rados.rados.0.smithi136.stdout:495: setattr oid 24 current snap is 43 2024-09-12T23:39:47.607 INFO:tasks.rados.rados.0.smithi136.stdout:488: finishing rollback tid 0 to smithi13667081-1 2024-09-12T23:39:47.607 INFO:tasks.rados.rados.0.smithi136.stdout:488: finishing rollback tid 1 to smithi13667081-1 2024-09-12T23:39:47.607 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 1 v 103 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-12T23:39:47.607 INFO:tasks.rados.rados.0.smithi136.stdout:490: finishing write tid 1 to smithi13667081-4 2024-09-12T23:39:47.607 INFO:tasks.rados.rados.0.smithi136.stdout:490: finishing write tid 2 to smithi13667081-4 2024-09-12T23:39:47.607 INFO:tasks.rados.rados.0.smithi136.stdout:490: finishing write tid 3 to smithi13667081-4 2024-09-12T23:39:47.607 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 4 v 72 (ObjNum 174 snap 43 seq_num 174) dirty exists 2024-09-12T23:39:47.607 INFO:tasks.rados.rados.0.smithi136.stdout:490: left oid 4 (ObjNum 174 snap 43 seq_num 174) 2024-09-12T23:39:47.612 INFO:tasks.rados.rados.0.smithi136.stdout:488: done (5 left) 2024-09-12T23:39:47.612 INFO:tasks.rados.rados.0.smithi136.stdout:490: done (4 left) 2024-09-12T23:39:47.612 INFO:tasks.rados.rados.0.smithi136.stdout:496: snap_create 2024-09-12T23:39:47.622 INFO:tasks.rados.rados.0.smithi136.stdout:494: finishing rollback tid 0 to smithi13667081-21 2024-09-12T23:39:47.622 INFO:tasks.rados.rados.0.smithi136.stdout:494: finishing rollback tid 1 to smithi13667081-21 2024-09-12T23:39:47.624 INFO:tasks.rados.rados.0.smithi136.stdout:494: finishing rollback tid 2 to smithi13667081-21 2024-09-12T23:39:47.624 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 21 v 113 (ObjNum 88 snap 18 seq_num 88) dirty exists 2024-09-12T23:39:47.631 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 24 v 93 (ObjNum 54 snap 4 seq_num 54) dirty exists 2024-09-12T23:39:47.632 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 48 v 216 (ObjNum 118 snap 29 seq_num 118) dirty exists 2024-09-12T23:39:47.637 INFO:tasks.rados.rados.0.smithi136.stdout:492: finishing copy_from racing read to smithi13667081-14 2024-09-12T23:39:47.638 INFO:tasks.rados.rados.0.smithi136.stdout:492: finishing copy_from to smithi13667081-14 2024-09-12T23:39:47.638 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 14 v 147 (ObjNum 6 snap 0 seq_num 6) dirty exists 2024-09-12T23:39:47.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:47 smithi181 bash[24776]: cluster 2024-09-12T23:39:46.047084+0000 mgr.y (mgr.24446) 236 : cluster [DBG] pgmap v266: 121 pgs: 121 active+clean; 115 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s rd, 834 KiB/s wr, 21 op/s 2024-09-12T23:39:47.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:47 smithi181 bash[24776]: cluster 2024-09-12T23:39:46.599497+0000 mon.a (mon.0) 1180 : cluster [DBG] osdmap e150: 8 total, 8 up, 8 in 2024-09-12T23:39:47.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:47 smithi181 bash[24776]: audit 2024-09-12T23:39:47.150720+0000 mon.a (mon.0) 1181 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:39:47.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:47 smithi181 bash[24776]: audit 2024-09-12T23:39:47.153077+0000 mon.a (mon.0) 1182 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-12T23:39:47.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:47 smithi181 bash[24776]: audit 2024-09-12T23:39:47.154836+0000 mon.a (mon.0) 1183 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-12T23:39:47.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:47 smithi181 bash[24776]: audit 2024-09-12T23:39:47.156591+0000 mon.a (mon.0) 1184 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:39:48.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:47 smithi136 bash[22601]: cluster 2024-09-12T23:39:46.047084+0000 mgr.y (mgr.24446) 236 : cluster [DBG] pgmap v266: 121 pgs: 121 active+clean; 115 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s rd, 834 KiB/s wr, 21 op/s 2024-09-12T23:39:48.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:47 smithi136 bash[22601]: cluster 2024-09-12T23:39:46.599497+0000 mon.a (mon.0) 1180 : cluster [DBG] osdmap e150: 8 total, 8 up, 8 in 2024-09-12T23:39:48.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:47 smithi136 bash[22601]: audit 2024-09-12T23:39:47.150720+0000 mon.a (mon.0) 1181 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:39:48.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:47 smithi136 bash[22601]: audit 2024-09-12T23:39:47.153077+0000 mon.a (mon.0) 1182 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-12T23:39:48.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:47 smithi136 bash[22601]: audit 2024-09-12T23:39:47.154836+0000 mon.a (mon.0) 1183 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-12T23:39:48.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:47 smithi136 bash[22601]: audit 2024-09-12T23:39:47.156591+0000 mon.a (mon.0) 1184 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:39:48.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:47 smithi136 bash[32189]: cluster 2024-09-12T23:39:46.047084+0000 mgr.y (mgr.24446) 236 : cluster [DBG] pgmap v266: 121 pgs: 121 active+clean; 115 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s rd, 834 KiB/s wr, 21 op/s 2024-09-12T23:39:48.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:47 smithi136 bash[32189]: cluster 2024-09-12T23:39:46.599497+0000 mon.a (mon.0) 1180 : cluster [DBG] osdmap e150: 8 total, 8 up, 8 in 2024-09-12T23:39:48.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:47 smithi136 bash[32189]: audit 2024-09-12T23:39:47.150720+0000 mon.a (mon.0) 1181 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:39:48.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:47 smithi136 bash[32189]: audit 2024-09-12T23:39:47.153077+0000 mon.a (mon.0) 1182 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-12T23:39:48.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:47 smithi136 bash[32189]: audit 2024-09-12T23:39:47.154836+0000 mon.a (mon.0) 1183 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-12T23:39:48.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:47 smithi136 bash[32189]: audit 2024-09-12T23:39:47.156591+0000 mon.a (mon.0) 1184 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:39:48.632 INFO:tasks.rados.rados.0.smithi136.stdout:492: done (4 left) 2024-09-12T23:39:48.633 INFO:tasks.rados.rados.0.smithi136.stdout:493: done (3 left) 2024-09-12T23:39:48.633 INFO:tasks.rados.rados.0.smithi136.stdout:494: done (2 left) 2024-09-12T23:39:48.633 INFO:tasks.rados.rados.0.smithi136.stdout:495: done (1 left) 2024-09-12T23:39:48.633 INFO:tasks.rados.rados.0.smithi136.stdout:496: done (0 left) 2024-09-12T23:39:48.633 INFO:tasks.rados.rados.0.smithi136.stdout:497: snap_create 2024-09-12T23:39:48.958 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:48 smithi181 bash[24776]: cephadm 2024-09-12T23:39:47.140753+0000 mgr.y (mgr.24446) 237 : cephadm [INF] Upgrade: Updating mgr.x 2024-09-12T23:39:48.958 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:48 smithi181 bash[24776]: cephadm 2024-09-12T23:39:47.158332+0000 mgr.y (mgr.24446) 238 : cephadm [INF] Deploying daemon mgr.x on smithi181 2024-09-12T23:39:48.958 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:48 smithi181 bash[24776]: cluster 2024-09-12T23:39:47.602360+0000 mon.a (mon.0) 1185 : cluster [DBG] osdmap e151: 8 total, 8 up, 8 in 2024-09-12T23:39:49.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:48 smithi136 bash[22601]: cephadm 2024-09-12T23:39:47.140753+0000 mgr.y (mgr.24446) 237 : cephadm [INF] Upgrade: Updating mgr.x 2024-09-12T23:39:49.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:48 smithi136 bash[22601]: cephadm 2024-09-12T23:39:47.158332+0000 mgr.y (mgr.24446) 238 : cephadm [INF] Deploying daemon mgr.x on smithi181 2024-09-12T23:39:49.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:48 smithi136 bash[22601]: cluster 2024-09-12T23:39:47.602360+0000 mon.a (mon.0) 1185 : cluster [DBG] osdmap e151: 8 total, 8 up, 8 in 2024-09-12T23:39:49.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:48 smithi136 bash[32189]: cephadm 2024-09-12T23:39:47.140753+0000 mgr.y (mgr.24446) 237 : cephadm [INF] Upgrade: Updating mgr.x 2024-09-12T23:39:49.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:48 smithi136 bash[32189]: cephadm 2024-09-12T23:39:47.158332+0000 mgr.y (mgr.24446) 238 : cephadm [INF] Deploying daemon mgr.x on smithi181 2024-09-12T23:39:49.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:48 smithi136 bash[32189]: cluster 2024-09-12T23:39:47.602360+0000 mon.a (mon.0) 1185 : cluster [DBG] osdmap e151: 8 total, 8 up, 8 in 2024-09-12T23:39:49.217 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:39:49 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:39:49.218 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 12 23:39:49 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:39:49.218 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:39:49 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:39:49.219 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:39:49 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:39:49.219 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:49 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:39:49.219 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:39:49 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:39:49.220 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 12 23:39:49 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:39:49.220 INFO:journalctl@ceph.mds.b.smithi181.stdout:Sep 12 23:39:49 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:39:49.221 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:39:49 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:39:49.221 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:39:49 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:39:49.503 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:39:49 smithi181 systemd[1]: Stopping Ceph mgr.x for da58a47c-715d-11ef-bceb-c7b262605968... 2024-09-12T23:39:49.633 INFO:tasks.rados.rados.0.smithi136.stdout:497: done (0 left) 2024-09-12T23:39:49.633 INFO:tasks.rados.rados.0.smithi136.stdout:498: setattr oid 4 current snap is 45 2024-09-12T23:39:49.638 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 14 current snap is 45 2024-09-12T23:39:49.638 INFO:tasks.rados.rados.0.smithi136.stdout:499: seq_num 175 ranges {106496=540672} 2024-09-12T23:39:49.646 INFO:tasks.rados.rados.0.smithi136.stdout:499: writing smithi13667081-14 from 106496 to 647168 tid 1 2024-09-12T23:39:49.646 INFO:tasks.rados.rados.0.smithi136.stdout:500: read oid 16 snap 19 2024-09-12T23:39:49.646 INFO:tasks.rados.rados.0.smithi136.stdout:500: expect (ObjNum 81 snap 15 seq_num 81) 2024-09-12T23:39:49.646 INFO:tasks.rados.rados.0.smithi136.stdout:501: rmattr oid 10 current snap is 45 2024-09-12T23:39:49.646 INFO:tasks.rados.rados.0.smithi136.stdout:501: done (3 left) 2024-09-12T23:39:49.646 INFO:tasks.rados.rados.0.smithi136.stdout:502: copy_from oid 41 from oid 17 current snap is 45 2024-09-12T23:39:49.646 INFO:tasks.rados.rados.0.smithi136.stdout:503: delete oid 32 current snap is 45 2024-09-12T23:39:49.656 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 4 v 74 (ObjNum 174 snap 43 seq_num 174) dirty exists 2024-09-12T23:39:49.657 INFO:tasks.rados.rados.0.smithi136.stdout:498: done (4 left) 2024-09-12T23:39:49.657 INFO:tasks.rados.rados.0.smithi136.stdout:503: done (3 left) 2024-09-12T23:39:49.658 INFO:tasks.rados.rados.0.smithi136.stdout:504: rmattr oid 28 current snap is 45 2024-09-12T23:39:49.662 INFO:tasks.rados.rados.0.smithi136.stdout:505: rollback oid 31 current snap is 45 2024-09-12T23:39:49.663 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 31 to 28 2024-09-12T23:39:49.663 INFO:tasks.rados.rados.0.smithi136.stdout:506: copy_from oid 19 from oid 46 current snap is 45 2024-09-12T23:39:49.663 INFO:tasks.rados.rados.0.smithi136.stdout:507: read oid 45 snap -1 2024-09-12T23:39:49.663 INFO:tasks.rados.rados.0.smithi136.stdout:507: expect (ObjNum 141 snap 36 seq_num 141) 2024-09-12T23:39:49.663 INFO:tasks.rados.rados.0.smithi136.stdout:508: read oid 26 snap -1 2024-09-12T23:39:49.663 INFO:tasks.rados.rados.0.smithi136.stdout:508: expect (ObjNum 160 snap 42 seq_num 160) 2024-09-12T23:39:49.663 INFO:tasks.rados.rados.0.smithi136.stdout:509: copy_from oid 8 from oid 34 current snap is 45 2024-09-12T23:39:49.663 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 20 current snap is 45 2024-09-12T23:39:49.663 INFO:tasks.rados.rados.0.smithi136.stdout:510: seq_num 177 ranges {0=573440} 2024-09-12T23:39:49.664 INFO:tasks.rados.rados.0.smithi136.stdout:510: writing smithi13667081-20 from 0 to 573440 tid 1 2024-09-12T23:39:49.664 INFO:tasks.rados.rados.0.smithi136.stdout:499: finishing write tid 1 to smithi13667081-14 2024-09-12T23:39:49.665 INFO:tasks.rados.rados.0.smithi136.stdout:511: read oid 6 snap -1 2024-09-12T23:39:49.665 INFO:tasks.rados.rados.0.smithi136.stdout:511: expect (ObjNum 36 snap 0 seq_num 36) 2024-09-12T23:39:49.665 INFO:tasks.rados.rados.0.smithi136.stdout:512: copy_from oid 23 from oid 47 current snap is 45 2024-09-12T23:39:49.665 INFO:tasks.rados.rados.0.smithi136.stdout:513: read oid 25 snap -1 2024-09-12T23:39:49.665 INFO:tasks.rados.rados.0.smithi136.stdout:513: expect (ObjNum 144 snap 36 seq_num 144) 2024-09-12T23:39:49.665 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 35 current snap is 45 2024-09-12T23:39:49.665 INFO:tasks.rados.rados.0.smithi136.stdout:514: seq_num 178 ranges {0=589824} 2024-09-12T23:39:49.671 INFO:tasks.rados.rados.0.smithi136.stdout:514: writing smithi13667081-35 from 0 to 589824 tid 1 2024-09-12T23:39:49.671 INFO:tasks.rados.rados.0.smithi136.stdout:515: read oid 18 snap 40 2024-09-12T23:39:49.671 INFO:tasks.rados.rados.0.smithi136.stdout:515: expect (ObjNum 135 snap 36 seq_num 135) 2024-09-12T23:39:49.671 INFO:tasks.rados.rados.0.smithi136.stdout:516: snap_remove snap 44 2024-09-12T23:39:49.808 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:49 smithi181 bash[24776]: cluster 2024-09-12T23:39:48.047857+0000 mgr.y (mgr.24446) 239 : cluster [DBG] pgmap v269: 121 pgs: 121 active+clean; 115 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:39:49.808 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:49 smithi181 bash[24776]: cluster 2024-09-12T23:39:48.636951+0000 mon.a (mon.0) 1186 : cluster [DBG] osdmap e152: 8 total, 8 up, 8 in 2024-09-12T23:39:50.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:49 smithi136 bash[32189]: cluster 2024-09-12T23:39:48.047857+0000 mgr.y (mgr.24446) 239 : cluster [DBG] pgmap v269: 121 pgs: 121 active+clean; 115 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:39:50.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:49 smithi136 bash[32189]: cluster 2024-09-12T23:39:48.636951+0000 mon.a (mon.0) 1186 : cluster [DBG] osdmap e152: 8 total, 8 up, 8 in 2024-09-12T23:39:50.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:49 smithi136 bash[22601]: cluster 2024-09-12T23:39:48.047857+0000 mgr.y (mgr.24446) 239 : cluster [DBG] pgmap v269: 121 pgs: 121 active+clean; 115 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:39:50.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:49 smithi136 bash[22601]: cluster 2024-09-12T23:39:48.636951+0000 mon.a (mon.0) 1186 : cluster [DBG] osdmap e152: 8 total, 8 up, 8 in 2024-09-12T23:39:50.094 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:39:49 smithi181 bash[52247]: ceph-da58a47c-715d-11ef-bceb-c7b262605968-mgr-x 2024-09-12T23:39:50.094 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:39:49 smithi181 systemd[1]: ceph-da58a47c-715d-11ef-bceb-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-09-12T23:39:50.094 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:39:50 smithi181 bash[52309]: Error response from daemon: No such container: ceph-da58a47c-715d-11ef-bceb-c7b262605968-mgr-x 2024-09-12T23:39:50.094 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:39:50 smithi181 systemd[1]: ceph-da58a47c-715d-11ef-bceb-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-09-12T23:39:50.094 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:39:50 smithi181 systemd[1]: Stopped Ceph mgr.x for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:39:50.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:50 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:39:50.594 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:39:50 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:39:50.594 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 12 23:39:50 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:39:50.595 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 12 23:39:50 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:39:50.595 INFO:journalctl@ceph.mds.b.smithi181.stdout:Sep 12 23:39:50 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:39:50.595 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:39:50 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:39:50.596 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:39:50 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:39:50.596 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:39:50 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:39:50.596 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:39:50 smithi181 systemd[1]: Started Ceph mgr.x for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:39:50.596 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:39:50 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:39:50.596 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:39:50 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:39:50.654 INFO:tasks.rados.rados.0.smithi136.stdout:516: done (15 left) 2024-09-12T23:39:50.654 INFO:tasks.rados.rados.0.smithi136.stdout:517: snap_remove snap 43 2024-09-12T23:39:51.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:50 smithi136 bash[22601]: cluster 2024-09-12T23:39:49.637475+0000 mon.a (mon.0) 1187 : cluster [DBG] osdmap e153: 8 total, 8 up, 8 in 2024-09-12T23:39:51.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:50 smithi136 bash[22601]: audit 2024-09-12T23:39:50.301545+0000 mon.a (mon.0) 1188 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:39:51.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:50 smithi136 bash[22601]: audit 2024-09-12T23:39:50.311486+0000 mon.a (mon.0) 1189 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:39:51.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:50 smithi136 bash[22601]: audit 2024-09-12T23:39:50.313625+0000 mon.a (mon.0) 1190 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:39:51.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:50 smithi136 bash[32189]: cluster 2024-09-12T23:39:49.637475+0000 mon.a (mon.0) 1187 : cluster [DBG] osdmap e153: 8 total, 8 up, 8 in 2024-09-12T23:39:51.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:50 smithi136 bash[32189]: audit 2024-09-12T23:39:50.301545+0000 mon.a (mon.0) 1188 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:39:51.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:50 smithi136 bash[32189]: audit 2024-09-12T23:39:50.311486+0000 mon.a (mon.0) 1189 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:39:51.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:50 smithi136 bash[32189]: audit 2024-09-12T23:39:50.313625+0000 mon.a (mon.0) 1190 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:39:51.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:50 smithi181 bash[24776]: cluster 2024-09-12T23:39:49.637475+0000 mon.a (mon.0) 1187 : cluster [DBG] osdmap e153: 8 total, 8 up, 8 in 2024-09-12T23:39:51.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:50 smithi181 bash[24776]: audit 2024-09-12T23:39:50.301545+0000 mon.a (mon.0) 1188 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:39:51.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:50 smithi181 bash[24776]: audit 2024-09-12T23:39:50.311486+0000 mon.a (mon.0) 1189 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:39:51.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:50 smithi181 bash[24776]: audit 2024-09-12T23:39:50.313625+0000 mon.a (mon.0) 1190 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:39:51.648 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:39:51 smithi181 bash[52367]: debug 2024-09-12T23:39:51.511+0000 7fc99f05f640 1 -- 172.21.15.181:0/1590365827 <== mon.1 v2:172.21.15.181:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x558c97cded00 con 0x558c97cd1400 2024-09-12T23:39:51.648 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:39:51 smithi181 bash[52367]: debug 2024-09-12T23:39:51.627+0000 7fc9a18c8040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-12T23:39:51.661 INFO:tasks.rados.rados.0.smithi136.stdout:517: done (15 left) 2024-09-12T23:39:51.661 INFO:tasks.rados.rados.0.smithi136.stdout:518: delete oid 29 current snap is 45 2024-09-12T23:39:51.661 INFO:tasks.rados.rados.0.smithi136.stdout:499: finishing write tid 2 to smithi13667081-14 2024-09-12T23:39:51.661 INFO:tasks.rados.rados.0.smithi136.stdout:509: finishing copy_from to smithi13667081-8 2024-09-12T23:39:51.661 INFO:tasks.rados.rados.0.smithi136.stdout:509: got expected ENOENT (src dne) 2024-09-12T23:39:51.661 INFO:tasks.rados.rados.0.smithi136.stdout:509: finishing copy_from racing read to smithi13667081-8 2024-09-12T23:39:51.661 INFO:tasks.rados.rados.0.smithi136.stdout:510: finishing write tid 1 to smithi13667081-20 2024-09-12T23:39:51.662 INFO:tasks.rados.rados.0.smithi136.stdout:510: finishing write tid 2 to smithi13667081-20 2024-09-12T23:39:51.662 INFO:tasks.rados.rados.0.smithi136.stdout:502: finishing copy_from racing read to smithi13667081-41 2024-09-12T23:39:51.662 INFO:tasks.rados.rados.0.smithi136.stdout:505: finishing rollback tid 1 to smithi13667081-31 2024-09-12T23:39:51.662 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 31 v 90 (ObjNum 177 snap 0 seq_num 1) dirty dne 2024-09-12T23:39:51.662 INFO:tasks.rados.rados.0.smithi136.stdout:507: expect (ObjNum 141 snap 36 seq_num 141) 2024-09-12T23:39:51.805 INFO:tasks.rados.rados.0.smithi136.stdout:511: expect (ObjNum 36 snap 0 seq_num 36) 2024-09-12T23:39:51.820 INFO:tasks.rados.rados.0.smithi136.stdout:510: finishing write tid 3 to smithi13667081-20 2024-09-12T23:39:51.820 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 20 v 46 (ObjNum 177 snap 45 seq_num 177) dirty exists 2024-09-12T23:39:51.820 INFO:tasks.rados.rados.0.smithi136.stdout:510: left oid 20 (ObjNum 177 snap 45 seq_num 177) 2024-09-12T23:39:51.820 INFO:tasks.rados.rados.0.smithi136.stdout:502: finishing copy_from to smithi13667081-41 2024-09-12T23:39:51.820 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 41 v 193 (ObjNum 171 snap 43 seq_num 171) dirty exists 2024-09-12T23:39:51.820 INFO:tasks.rados.rados.0.smithi136.stdout:506: finishing copy_from racing read to smithi13667081-19 2024-09-12T23:39:51.820 INFO:tasks.rados.rados.0.smithi136.stdout:500: expect (ObjNum 81 snap 15 seq_num 81) 2024-09-12T23:39:51.924 INFO:tasks.rados.rados.0.smithi136.stdout:499: finishing write tid 3 to smithi13667081-14 2024-09-12T23:39:51.924 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 14 v 226 (ObjNum 175 snap 45 seq_num 175) dirty exists 2024-09-12T23:39:51.925 INFO:tasks.rados.rados.0.smithi136.stdout:499: left oid 14 (ObjNum 175 snap 45 seq_num 175) 2024-09-12T23:39:51.925 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 28 v 228 (ObjNum 170 snap 43 seq_num 170) dirty exists 2024-09-12T23:39:51.925 INFO:tasks.rados.rados.0.smithi136.stdout:512: finishing copy_from racing read to smithi13667081-23 2024-09-12T23:39:51.925 INFO:tasks.rados.rados.0.smithi136.stdout:514: finishing write tid 1 to smithi13667081-35 2024-09-12T23:39:51.925 INFO:tasks.rados.rados.0.smithi136.stdout:513: expect (ObjNum 144 snap 36 seq_num 144) 2024-09-12T23:39:51.925 INFO:tasks.rados.rados.0.smithi136.stdout:514: finishing write tid 2 to smithi13667081-35 2024-09-12T23:39:51.925 INFO:tasks.rados.rados.0.smithi136.stdout:512: finishing copy_from to smithi13667081-23 2024-09-12T23:39:51.926 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 23 v 49 (ObjNum 74 snap 12 seq_num 74) dirty exists 2024-09-12T23:39:51.926 INFO:tasks.rados.rados.0.smithi136.stdout:506: finishing copy_from to smithi13667081-19 2024-09-12T23:39:51.926 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 19 v 50 (ObjNum 110 snap 23 seq_num 110) dirty exists 2024-09-12T23:39:51.926 INFO:tasks.rados.rados.0.smithi136.stdout:514: finishing write tid 3 to smithi13667081-35 2024-09-12T23:39:51.926 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 35 v 205 (ObjNum 178 snap 45 seq_num 178) dirty exists 2024-09-12T23:39:51.926 INFO:tasks.rados.rados.0.smithi136.stdout:514: left oid 35 (ObjNum 178 snap 45 seq_num 178) 2024-09-12T23:39:51.926 INFO:tasks.rados.rados.0.smithi136.stdout:508: expect (ObjNum 160 snap 42 seq_num 160) 2024-09-12T23:39:52.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:51 smithi136 bash[22601]: cluster 2024-09-12T23:39:50.048639+0000 mgr.y (mgr.24446) 240 : cluster [DBG] pgmap v272: 121 pgs: 121 active+clean; 123 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 751 KiB/s rd, 2.0 MiB/s wr, 30 op/s 2024-09-12T23:39:52.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:51 smithi136 bash[22601]: cluster 2024-09-12T23:39:50.656532+0000 mon.a (mon.0) 1191 : cluster [DBG] osdmap e154: 8 total, 8 up, 8 in 2024-09-12T23:39:52.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:51 smithi136 bash[32189]: cluster 2024-09-12T23:39:50.048639+0000 mgr.y (mgr.24446) 240 : cluster [DBG] pgmap v272: 121 pgs: 121 active+clean; 123 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 751 KiB/s rd, 2.0 MiB/s wr, 30 op/s 2024-09-12T23:39:52.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:51 smithi136 bash[32189]: cluster 2024-09-12T23:39:50.656532+0000 mon.a (mon.0) 1191 : cluster [DBG] osdmap e154: 8 total, 8 up, 8 in 2024-09-12T23:39:52.009 INFO:tasks.rados.rados.0.smithi136.stdout:515: expect (ObjNum 135 snap 36 seq_num 135) 2024-09-12T23:39:52.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:51 smithi181 bash[24776]: cluster 2024-09-12T23:39:50.048639+0000 mgr.y (mgr.24446) 240 : cluster [DBG] pgmap v272: 121 pgs: 121 active+clean; 123 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 751 KiB/s rd, 2.0 MiB/s wr, 30 op/s 2024-09-12T23:39:52.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:51 smithi181 bash[24776]: cluster 2024-09-12T23:39:50.656532+0000 mon.a (mon.0) 1191 : cluster [DBG] osdmap e154: 8 total, 8 up, 8 in 2024-09-12T23:39:52.341 INFO:tasks.rados.rados.0.smithi136.stdout:499: done (15 left) 2024-09-12T23:39:52.341 INFO:tasks.rados.rados.0.smithi136.stdout:500: done (14 left) 2024-09-12T23:39:52.341 INFO:tasks.rados.rados.0.smithi136.stdout:502: done (13 left) 2024-09-12T23:39:52.342 INFO:tasks.rados.rados.0.smithi136.stdout:504: done (12 left) 2024-09-12T23:39:52.342 INFO:tasks.rados.rados.0.smithi136.stdout:505: done (11 left) 2024-09-12T23:39:52.342 INFO:tasks.rados.rados.0.smithi136.stdout:506: done (10 left) 2024-09-12T23:39:52.342 INFO:tasks.rados.rados.0.smithi136.stdout:507: done (9 left) 2024-09-12T23:39:52.342 INFO:tasks.rados.rados.0.smithi136.stdout:508: done (8 left) 2024-09-12T23:39:52.342 INFO:tasks.rados.rados.0.smithi136.stdout:509: done (7 left) 2024-09-12T23:39:52.342 INFO:tasks.rados.rados.0.smithi136.stdout:510: done (6 left) 2024-09-12T23:39:52.342 INFO:tasks.rados.rados.0.smithi136.stdout:511: done (5 left) 2024-09-12T23:39:52.343 INFO:tasks.rados.rados.0.smithi136.stdout:512: done (4 left) 2024-09-12T23:39:52.343 INFO:tasks.rados.rados.0.smithi136.stdout:513: done (3 left) 2024-09-12T23:39:52.343 INFO:tasks.rados.rados.0.smithi136.stdout:514: done (2 left) 2024-09-12T23:39:52.343 INFO:tasks.rados.rados.0.smithi136.stdout:515: done (1 left) 2024-09-12T23:39:52.343 INFO:tasks.rados.rados.0.smithi136.stdout:518: done (0 left) 2024-09-12T23:39:52.344 INFO:tasks.rados.rados.0.smithi136.stdout:519: snap_remove snap 42 2024-09-12T23:39:52.672 INFO:tasks.rados.rados.0.smithi136.stdout:519: done (0 left) 2024-09-12T23:39:52.672 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 44 current snap is 45 2024-09-12T23:39:52.672 INFO:tasks.rados.rados.0.smithi136.stdout:520: seq_num 180 ranges {237568=622592} 2024-09-12T23:39:52.689 INFO:tasks.rados.rados.0.smithi136.stdout:520: writing smithi13667081-44 from 237568 to 860160 tid 1 2024-09-12T23:39:52.689 INFO:tasks.rados.rados.0.smithi136.stdout:521: snap_create 2024-09-12T23:39:52.699 INFO:tasks.rados.rados.0.smithi136.stdout:520: finishing write tid 1 to smithi13667081-44 2024-09-12T23:39:52.700 INFO:tasks.rados.rados.0.smithi136.stdout:520: finishing write tid 2 to smithi13667081-44 2024-09-12T23:39:52.702 INFO:tasks.rados.rados.0.smithi136.stdout:520: finishing write tid 3 to smithi13667081-44 2024-09-12T23:39:52.702 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 44 v 238 (ObjNum 180 snap 45 seq_num 180) dirty exists 2024-09-12T23:39:52.702 INFO:tasks.rados.rados.0.smithi136.stdout:520: left oid 44 (ObjNum 180 snap 45 seq_num 180) 2024-09-12T23:39:53.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:52 smithi136 bash[22601]: cluster 2024-09-12T23:39:51.665445+0000 mon.a (mon.0) 1192 : cluster [DBG] osdmap e155: 8 total, 8 up, 8 in 2024-09-12T23:39:53.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:52 smithi136 bash[32189]: cluster 2024-09-12T23:39:51.665445+0000 mon.a (mon.0) 1192 : cluster [DBG] osdmap e155: 8 total, 8 up, 8 in 2024-09-12T23:39:53.050 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:52 smithi181 bash[24776]: cluster 2024-09-12T23:39:51.665445+0000 mon.a (mon.0) 1192 : cluster [DBG] osdmap e155: 8 total, 8 up, 8 in 2024-09-12T23:39:53.050 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:39:52 smithi181 bash[52367]: debug 2024-09-12T23:39:52.835+0000 7fc9a18c8040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-12T23:39:53.051 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:39:52 smithi181 bash[52367]: debug 2024-09-12T23:39:52.903+0000 7fc9a18c8040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-12T23:39:53.344 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:39:53 smithi181 bash[52367]: debug 2024-09-12T23:39:53.051+0000 7fc9a18c8040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-12T23:39:53.685 INFO:tasks.rados.rados.0.smithi136.stdout:520: done (1 left) 2024-09-12T23:39:53.685 INFO:tasks.rados.rados.0.smithi136.stdout:521: done (0 left) 2024-09-12T23:39:53.685 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 17 current snap is 46 2024-09-12T23:39:53.685 INFO:tasks.rados.rados.0.smithi136.stdout:522: seq_num 181 ranges {466944=638976} 2024-09-12T23:39:53.703 INFO:tasks.rados.rados.0.smithi136.stdout:522: writing smithi13667081-17 from 466944 to 1105920 tid 1 2024-09-12T23:39:53.703 INFO:tasks.rados.rados.0.smithi136.stdout:523: copy_from oid 28 from oid 42 current snap is 46 2024-09-12T23:39:53.703 INFO:tasks.rados.rados.0.smithi136.stdout:524: rollback oid 15 current snap is 46 2024-09-12T23:39:53.703 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 15 to 40 2024-09-12T23:39:53.703 INFO:tasks.rados.rados.0.smithi136.stdout:525: rollback oid 33 current snap is 46 2024-09-12T23:39:53.704 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 33 to 45 2024-09-12T23:39:53.704 INFO:tasks.rados.rados.0.smithi136.stdout:526: delete oid 46 current snap is 46 2024-09-12T23:39:53.716 INFO:tasks.rados.rados.0.smithi136.stdout:526: done (4 left) 2024-09-12T23:39:53.717 INFO:tasks.rados.rados.0.smithi136.stdout:527: read oid 11 snap 13 2024-09-12T23:39:53.717 INFO:tasks.rados.rados.0.smithi136.stdout:527: expect (ObjNum 10 snap 0 seq_num 10) 2024-09-12T23:39:53.717 INFO:tasks.rados.rados.0.smithi136.stdout:528: read oid 19 snap -1 2024-09-12T23:39:53.717 INFO:tasks.rados.rados.0.smithi136.stdout:528: expect (ObjNum 110 snap 23 seq_num 110) 2024-09-12T23:39:53.717 INFO:tasks.rados.rados.0.smithi136.stdout:529: copy_from oid 14 from oid 7 current snap is 46 2024-09-12T23:39:53.717 INFO:tasks.rados.rados.0.smithi136.stdout:530: delete oid 47 current snap is 46 2024-09-12T23:39:53.723 INFO:tasks.rados.rados.0.smithi136.stdout:522: finishing write tid 1 to smithi13667081-17 2024-09-12T23:39:53.723 INFO:tasks.rados.rados.0.smithi136.stdout:525: finishing rollback tid 0 to smithi13667081-33 2024-09-12T23:39:53.723 INFO:tasks.rados.rados.0.smithi136.stdout:522: finishing write tid 2 to smithi13667081-17 2024-09-12T23:39:53.725 INFO:tasks.rados.rados.0.smithi136.stdout:525: finishing rollback tid 1 to smithi13667081-33 2024-09-12T23:39:53.725 INFO:tasks.rados.rados.0.smithi136.stdout:524: finishing rollback tid 0 to smithi13667081-15 2024-09-12T23:39:53.726 INFO:tasks.rados.rados.0.smithi136.stdout:524: finishing rollback tid 1 to smithi13667081-15 2024-09-12T23:39:53.726 INFO:tasks.rados.rados.0.smithi136.stdout:525: finishing rollback tid 2 to smithi13667081-33 2024-09-12T23:39:53.726 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 33 v 212 (ObjNum 114 snap 25 seq_num 114) dirty exists 2024-09-12T23:39:53.727 INFO:tasks.rados.rados.0.smithi136.stdout:529: finishing copy_from to smithi13667081-14 2024-09-12T23:39:53.728 INFO:tasks.rados.rados.0.smithi136.stdout:529: got expected ENOENT (src dne) 2024-09-12T23:39:53.728 INFO:tasks.rados.rados.0.smithi136.stdout:529: finishing copy_from racing read to smithi13667081-14 2024-09-12T23:39:53.729 INFO:tasks.rados.rados.0.smithi136.stdout:524: finishing rollback tid 2 to smithi13667081-15 2024-09-12T23:39:53.729 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 15 v 301 (ObjNum 105 snap 23 seq_num 105) dirty exists 2024-09-12T23:39:53.729 INFO:tasks.rados.rados.0.smithi136.stdout:527: expect (ObjNum 10 snap 0 seq_num 10) 2024-09-12T23:39:53.736 INFO:tasks.rados.rados.0.smithi136.stdout:522: finishing write tid 3 to smithi13667081-17 2024-09-12T23:39:53.736 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 17 v 220 (ObjNum 181 snap 46 seq_num 181) dirty exists 2024-09-12T23:39:53.736 INFO:tasks.rados.rados.0.smithi136.stdout:522: left oid 17 (ObjNum 181 snap 46 seq_num 181) 2024-09-12T23:39:53.736 INFO:tasks.rados.rados.0.smithi136.stdout:523: finishing copy_from racing read to smithi13667081-28 2024-09-12T23:39:53.738 INFO:tasks.rados.rados.0.smithi136.stdout:523: finishing copy_from to smithi13667081-28 2024-09-12T23:39:53.738 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 28 v 67 (ObjNum 125 snap 32 seq_num 125) dirty exists 2024-09-12T23:39:53.741 INFO:tasks.rados.rados.0.smithi136.stdout:528: expect (ObjNum 110 snap 23 seq_num 110) 2024-09-12T23:39:53.824 INFO:tasks.rados.rados.0.smithi136.stdout:522: done (7 left) 2024-09-12T23:39:53.824 INFO:tasks.rados.rados.0.smithi136.stdout:523: done (6 left) 2024-09-12T23:39:53.824 INFO:tasks.rados.rados.0.smithi136.stdout:524: done (5 left) 2024-09-12T23:39:53.825 INFO:tasks.rados.rados.0.smithi136.stdout:525: done (4 left) 2024-09-12T23:39:53.825 INFO:tasks.rados.rados.0.smithi136.stdout:527: done (3 left) 2024-09-12T23:39:53.825 INFO:tasks.rados.rados.0.smithi136.stdout:528: done (2 left) 2024-09-12T23:39:53.825 INFO:tasks.rados.rados.0.smithi136.stdout:529: done (1 left) 2024-09-12T23:39:53.825 INFO:tasks.rados.rados.0.smithi136.stdout:530: done (0 left) 2024-09-12T23:39:53.825 INFO:tasks.rados.rados.0.smithi136.stdout:531: read oid 19 snap -1 2024-09-12T23:39:53.825 INFO:tasks.rados.rados.0.smithi136.stdout:531: expect (ObjNum 110 snap 23 seq_num 110) 2024-09-12T23:39:53.825 INFO:tasks.rados.rados.0.smithi136.stdout:532: delete oid 32 current snap is 46 2024-09-12T23:39:53.827 INFO:tasks.rados.rados.0.smithi136.stdout:532: done (1 left) 2024-09-12T23:39:53.827 INFO:tasks.rados.rados.0.smithi136.stdout:533: copy_from oid 1 from oid 31 current snap is 46 2024-09-12T23:39:53.827 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 4 current snap is 46 2024-09-12T23:39:53.827 INFO:tasks.rados.rados.0.smithi136.stdout:534: seq_num 185 ranges {917504=704512} 2024-09-12T23:39:53.857 INFO:tasks.rados.rados.0.smithi136.stdout:534: writing smithi13667081-4 from 917504 to 1622016 tid 1 2024-09-12T23:39:53.857 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 40 current snap is 46 2024-09-12T23:39:53.858 INFO:tasks.rados.rados.0.smithi136.stdout:533: finishing copy_from to smithi13667081-1 2024-09-12T23:39:53.858 INFO:tasks.rados.rados.0.smithi136.stdout:533: got expected ENOENT (src dne) 2024-09-12T23:39:53.858 INFO:tasks.rados.rados.0.smithi136.stdout:533: finishing copy_from racing read to smithi13667081-1 2024-09-12T23:39:53.858 INFO:tasks.rados.rados.0.smithi136.stdout:535: seq_num 186 ranges {0=720896} 2024-09-12T23:39:53.867 INFO:tasks.rados.rados.0.smithi136.stdout:535: writing smithi13667081-40 from 0 to 720896 tid 1 2024-09-12T23:39:53.867 INFO:tasks.rados.rados.0.smithi136.stdout:533: done (3 left) 2024-09-12T23:39:53.867 INFO:tasks.rados.rados.0.smithi136.stdout:536: rollback oid 22 current snap is 46 2024-09-12T23:39:53.867 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 22 to 19 2024-09-12T23:39:53.867 INFO:tasks.rados.rados.0.smithi136.stdout:531: expect (ObjNum 110 snap 23 seq_num 110) 2024-09-12T23:39:53.977 INFO:tasks.rados.rados.0.smithi136.stdout:534: finishing write tid 1 to smithi13667081-4 2024-09-12T23:39:53.977 INFO:tasks.rados.rados.0.smithi136.stdout:534: finishing write tid 2 to smithi13667081-4 2024-09-12T23:39:53.977 INFO:tasks.rados.rados.0.smithi136.stdout:534: finishing write tid 3 to smithi13667081-4 2024-09-12T23:39:53.977 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 4 v 90 (ObjNum 185 snap 46 seq_num 185) dirty exists 2024-09-12T23:39:53.977 INFO:tasks.rados.rados.0.smithi136.stdout:534: left oid 4 (ObjNum 185 snap 46 seq_num 185) 2024-09-12T23:39:53.977 INFO:tasks.rados.rados.0.smithi136.stdout:536: finishing rollback tid 0 to smithi13667081-22 2024-09-12T23:39:53.977 INFO:tasks.rados.rados.0.smithi136.stdout:535: finishing write tid 1 to smithi13667081-40 2024-09-12T23:39:53.977 INFO:tasks.rados.rados.0.smithi136.stdout:535: finishing write tid 2 to smithi13667081-40 2024-09-12T23:39:53.977 INFO:tasks.rados.rados.0.smithi136.stdout:536: finishing rollback tid 1 to smithi13667081-22 2024-09-12T23:39:53.977 INFO:tasks.rados.rados.0.smithi136.stdout:536: finishing rollback tid 2 to smithi13667081-22 2024-09-12T23:39:53.977 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 22 v 121 (ObjNum 21 snap 0 seq_num 21) dirty exists 2024-09-12T23:39:53.978 INFO:tasks.rados.rados.0.smithi136.stdout:535: finishing write tid 3 to smithi13667081-40 2024-09-12T23:39:53.978 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 40 v 222 (ObjNum 186 snap 46 seq_num 186) dirty exists 2024-09-12T23:39:53.978 INFO:tasks.rados.rados.0.smithi136.stdout:535: left oid 40 (ObjNum 186 snap 46 seq_num 186) 2024-09-12T23:39:53.978 INFO:tasks.rados.rados.0.smithi136.stdout:531: done (3 left) 2024-09-12T23:39:53.978 INFO:tasks.rados.rados.0.smithi136.stdout:534: done (2 left) 2024-09-12T23:39:53.978 INFO:tasks.rados.rados.0.smithi136.stdout:535: done (1 left) 2024-09-12T23:39:53.978 INFO:tasks.rados.rados.0.smithi136.stdout:536: done (0 left) 2024-09-12T23:39:53.978 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 15 current snap is 46 2024-09-12T23:39:53.978 INFO:tasks.rados.rados.0.smithi136.stdout:537: seq_num 187 ranges {2433024=737280} 2024-09-12T23:39:54.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:53 smithi136 bash[22601]: cluster 2024-09-12T23:39:52.049358+0000 mgr.y (mgr.24446) 241 : cluster [DBG] pgmap v275: 121 pgs: 121 active+clean; 123 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 751 KiB/s rd, 2.0 MiB/s wr, 30 op/s 2024-09-12T23:39:54.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:53 smithi136 bash[22601]: cluster 2024-09-12T23:39:52.676432+0000 mon.a (mon.0) 1193 : cluster [DBG] osdmap e156: 8 total, 8 up, 8 in 2024-09-12T23:39:54.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:53 smithi136 bash[22601]: audit 2024-09-12T23:39:53.015581+0000 mon.a (mon.0) 1194 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:39:54.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:53 smithi136 bash[32189]: cluster 2024-09-12T23:39:52.049358+0000 mgr.y (mgr.24446) 241 : cluster [DBG] pgmap v275: 121 pgs: 121 active+clean; 123 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 751 KiB/s rd, 2.0 MiB/s wr, 30 op/s 2024-09-12T23:39:54.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:53 smithi136 bash[32189]: cluster 2024-09-12T23:39:52.676432+0000 mon.a (mon.0) 1193 : cluster [DBG] osdmap e156: 8 total, 8 up, 8 in 2024-09-12T23:39:54.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:53 smithi136 bash[32189]: audit 2024-09-12T23:39:53.015581+0000 mon.a (mon.0) 1194 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:39:54.025 INFO:tasks.rados.rados.0.smithi136.stdout:537: writing smithi13667081-15 from 2433024 to 3170304 tid 1 2024-09-12T23:39:54.025 INFO:tasks.rados.rados.0.smithi136.stdout:538: delete oid 35 current snap is 46 2024-09-12T23:39:54.028 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:53 smithi181 bash[24776]: cluster 2024-09-12T23:39:52.049358+0000 mgr.y (mgr.24446) 241 : cluster [DBG] pgmap v275: 121 pgs: 121 active+clean; 123 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 751 KiB/s rd, 2.0 MiB/s wr, 30 op/s 2024-09-12T23:39:54.029 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:53 smithi181 bash[24776]: cluster 2024-09-12T23:39:52.676432+0000 mon.a (mon.0) 1193 : cluster [DBG] osdmap e156: 8 total, 8 up, 8 in 2024-09-12T23:39:54.029 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:53 smithi181 bash[24776]: audit 2024-09-12T23:39:53.015581+0000 mon.a (mon.0) 1194 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:39:54.029 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:39:53 smithi181 bash[52367]: debug 2024-09-12T23:39:53.695+0000 7fc9a18c8040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-12T23:39:54.029 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:39:53 smithi181 bash[52367]: debug 2024-09-12T23:39:53.771+0000 7fc9a18c8040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-12T23:39:54.029 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:39:53 smithi181 bash[52367]: debug 2024-09-12T23:39:53.839+0000 7fc9a18c8040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-12T23:39:54.029 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:39:54 smithi181 bash[52367]: debug 2024-09-12T23:39:54.031+0000 7fc9a18c8040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-12T23:39:54.029 INFO:tasks.rados.rados.0.smithi136.stdout:538: done (1 left) 2024-09-12T23:39:54.030 INFO:tasks.rados.rados.0.smithi136.stdout:539: delete oid 10 current snap is 46 2024-09-12T23:39:54.032 INFO:tasks.rados.rados.0.smithi136.stdout:539: done (1 left) 2024-09-12T23:39:54.032 INFO:tasks.rados.rados.0.smithi136.stdout:540: rmattr oid 3 current snap is 46 2024-09-12T23:39:54.032 INFO:tasks.rados.rados.0.smithi136.stdout:540: done (1 left) 2024-09-12T23:39:54.032 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 38 current snap is 46 2024-09-12T23:39:54.032 INFO:tasks.rados.rados.0.smithi136.stdout:537: finishing write tid 1 to smithi13667081-15 2024-09-12T23:39:54.032 INFO:tasks.rados.rados.0.smithi136.stdout:541: seq_num 190 ranges {0=786432} 2024-09-12T23:39:54.038 INFO:tasks.rados.rados.0.smithi136.stdout:541: writing smithi13667081-38 from 0 to 786432 tid 1 2024-09-12T23:39:54.039 INFO:tasks.rados.rados.0.smithi136.stdout:542: read oid 3 snap -1 2024-09-12T23:39:54.039 INFO:tasks.rados.rados.0.smithi136.stdout:542: expect (ObjNum 168 snap 43 seq_num 168) 2024-09-12T23:39:54.039 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 12 current snap is 46 2024-09-12T23:39:54.039 INFO:tasks.rados.rados.0.smithi136.stdout:537: finishing write tid 2 to smithi13667081-15 2024-09-12T23:39:54.039 INFO:tasks.rados.rados.0.smithi136.stdout:537: finishing write tid 3 to smithi13667081-15 2024-09-12T23:39:54.039 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 15 v 303 (ObjNum 187 snap 46 seq_num 187) dirty exists 2024-09-12T23:39:54.039 INFO:tasks.rados.rados.0.smithi136.stdout:537: left oid 15 (ObjNum 187 snap 46 seq_num 187) 2024-09-12T23:39:54.039 INFO:tasks.rados.rados.0.smithi136.stdout:543: seq_num 191 ranges {3080192=802816} 2024-09-12T23:39:54.076 INFO:tasks.rados.rados.0.smithi136.stdout:543: writing smithi13667081-12 from 3080192 to 3883008 tid 1 2024-09-12T23:39:54.076 INFO:tasks.rados.rados.0.smithi136.stdout:537: done (3 left) 2024-09-12T23:39:54.076 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 36 current snap is 46 2024-09-12T23:39:54.076 INFO:tasks.rados.rados.0.smithi136.stdout:544: seq_num 192 ranges {2580480=417792,2998272=401408} 2024-09-12T23:39:54.097 INFO:tasks.rados.rados.0.smithi136.stdout:544: writing smithi13667081-36 from 2580480 to 2998272 tid 1 2024-09-12T23:39:54.100 INFO:tasks.rados.rados.0.smithi136.stdout:544: writing smithi13667081-36 from 2998272 to 3399680 tid 2 2024-09-12T23:39:54.100 INFO:tasks.rados.rados.0.smithi136.stdout:545: snap_remove snap 13 2024-09-12T23:39:54.344 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:39:54 smithi181 bash[52367]: debug 2024-09-12T23:39:54.119+0000 7fc9a18c8040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-12T23:39:54.344 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:39:54 smithi181 bash[52367]: debug 2024-09-12T23:39:54.187+0000 7fc9a18c8040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-12T23:39:54.690 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:39:54 smithi181 bash[52367]: debug 2024-09-12T23:39:54.435+0000 7fc9a18c8040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-12T23:39:54.690 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:39:54 smithi181 bash[52367]: debug 2024-09-12T23:39:54.503+0000 7fc9a18c8040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-12T23:39:54.697 INFO:tasks.rados.rados.0.smithi136.stdout:545: done (4 left) 2024-09-12T23:39:54.697 INFO:tasks.rados.rados.0.smithi136.stdout:546: snap_create 2024-09-12T23:39:54.697 INFO:tasks.rados.rados.0.smithi136.stdout:542: expect (ObjNum 168 snap 43 seq_num 168) 2024-09-12T23:39:54.722 INFO:tasks.rados.rados.0.smithi136.stdout:541: finishing write tid 1 to smithi13667081-38 2024-09-12T23:39:54.722 INFO:tasks.rados.rados.0.smithi136.stdout:541: finishing write tid 2 to smithi13667081-38 2024-09-12T23:39:54.723 INFO:tasks.rados.rados.0.smithi136.stdout:541: finishing write tid 3 to smithi13667081-38 2024-09-12T23:39:54.723 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 38 v 175 (ObjNum 190 snap 46 seq_num 190) dirty exists 2024-09-12T23:39:54.723 INFO:tasks.rados.rados.0.smithi136.stdout:541: left oid 38 (ObjNum 190 snap 46 seq_num 190) 2024-09-12T23:39:54.723 INFO:tasks.rados.rados.0.smithi136.stdout:543: finishing write tid 1 to smithi13667081-12 2024-09-12T23:39:54.723 INFO:tasks.rados.rados.0.smithi136.stdout:543: finishing write tid 2 to smithi13667081-12 2024-09-12T23:39:54.723 INFO:tasks.rados.rados.0.smithi136.stdout:543: finishing write tid 3 to smithi13667081-12 2024-09-12T23:39:54.723 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 12 v 306 (ObjNum 191 snap 46 seq_num 191) dirty exists 2024-09-12T23:39:54.723 INFO:tasks.rados.rados.0.smithi136.stdout:543: left oid 12 (ObjNum 191 snap 46 seq_num 191) 2024-09-12T23:39:54.723 INFO:tasks.rados.rados.0.smithi136.stdout:544: finishing write tid 1 to smithi13667081-36 2024-09-12T23:39:54.723 INFO:tasks.rados.rados.0.smithi136.stdout:544: finishing write tid 2 to smithi13667081-36 2024-09-12T23:39:54.723 INFO:tasks.rados.rados.0.smithi136.stdout:544: finishing write tid 3 to smithi13667081-36 2024-09-12T23:39:54.723 INFO:tasks.rados.rados.0.smithi136.stdout:544: finishing write tid 4 to smithi13667081-36 2024-09-12T23:39:54.723 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 36 v 140 (ObjNum 192 snap 46 seq_num 192) dirty exists 2024-09-12T23:39:54.724 INFO:tasks.rados.rados.0.smithi136.stdout:544: left oid 36 (ObjNum 192 snap 46 seq_num 192) 2024-09-12T23:39:55.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:54 smithi136 bash[22601]: cluster 2024-09-12T23:39:53.689374+0000 mon.a (mon.0) 1195 : cluster [DBG] osdmap e157: 8 total, 8 up, 8 in 2024-09-12T23:39:55.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:54 smithi136 bash[32189]: cluster 2024-09-12T23:39:53.689374+0000 mon.a (mon.0) 1195 : cluster [DBG] osdmap e157: 8 total, 8 up, 8 in 2024-09-12T23:39:55.037 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:54 smithi181 bash[24776]: cluster 2024-09-12T23:39:53.689374+0000 mon.a (mon.0) 1195 : cluster [DBG] osdmap e157: 8 total, 8 up, 8 in 2024-09-12T23:39:55.037 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:39:54 smithi181 bash[52367]: debug 2024-09-12T23:39:54.703+0000 7fc9a18c8040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-12T23:39:55.037 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:39:54 smithi181 bash[52367]: debug 2024-09-12T23:39:54.791+0000 7fc9a18c8040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-12T23:39:55.344 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:39:55 smithi181 bash[52367]: debug 2024-09-12T23:39:55.039+0000 7fc9a18c8040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-12T23:39:55.344 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:39:55 smithi181 bash[52367]: debug 2024-09-12T23:39:55.211+0000 7fc9a18c8040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-12T23:39:55.473 INFO:tasks.rados.rados.0.smithi136.stdout:541: done (4 left) 2024-09-12T23:39:55.473 INFO:tasks.rados.rados.0.smithi136.stdout:542: done (3 left) 2024-09-12T23:39:55.473 INFO:tasks.rados.rados.0.smithi136.stdout:543: done (2 left) 2024-09-12T23:39:55.473 INFO:tasks.rados.rados.0.smithi136.stdout:544: done (1 left) 2024-09-12T23:39:55.473 INFO:tasks.rados.rados.0.smithi136.stdout:546: done (0 left) 2024-09-12T23:39:55.474 INFO:tasks.rados.rados.0.smithi136.stdout:547: snap_remove snap 19 2024-09-12T23:39:55.626 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:39:55 smithi181 bash[52367]: debug 2024-09-12T23:39:55.371+0000 7fc9a18c8040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-12T23:39:55.626 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:39:55 smithi181 bash[52367]: debug 2024-09-12T23:39:55.447+0000 7fc9a18c8040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-12T23:39:55.931 INFO:teuthology.orchestra.run.smithi136.stdout:true 2024-09-12T23:39:56.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:55 smithi136 bash[32189]: cluster 2024-09-12T23:39:54.050106+0000 mgr.y (mgr.24446) 242 : cluster [DBG] pgmap v278: 121 pgs: 121 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 2.8 MiB/s wr, 26 op/s 2024-09-12T23:39:56.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:55 smithi136 bash[32189]: cluster 2024-09-12T23:39:54.698928+0000 mon.a (mon.0) 1196 : cluster [DBG] osdmap e158: 8 total, 8 up, 8 in 2024-09-12T23:39:56.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:55 smithi136 bash[32189]: cluster 2024-09-12T23:39:55.477628+0000 mon.a (mon.0) 1197 : cluster [DBG] osdmap e159: 8 total, 8 up, 8 in 2024-09-12T23:39:56.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:55 smithi136 bash[22601]: cluster 2024-09-12T23:39:54.050106+0000 mgr.y (mgr.24446) 242 : cluster [DBG] pgmap v278: 121 pgs: 121 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 2.8 MiB/s wr, 26 op/s 2024-09-12T23:39:56.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:55 smithi136 bash[22601]: cluster 2024-09-12T23:39:54.698928+0000 mon.a (mon.0) 1196 : cluster [DBG] osdmap e158: 8 total, 8 up, 8 in 2024-09-12T23:39:56.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:55 smithi136 bash[22601]: cluster 2024-09-12T23:39:55.477628+0000 mon.a (mon.0) 1197 : cluster [DBG] osdmap e159: 8 total, 8 up, 8 in 2024-09-12T23:39:56.094 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:39:55 smithi181 bash[52367]: debug 2024-09-12T23:39:55.627+0000 7fc9a18c8040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-12T23:39:56.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:55 smithi181 bash[24776]: cluster 2024-09-12T23:39:54.050106+0000 mgr.y (mgr.24446) 242 : cluster [DBG] pgmap v278: 121 pgs: 121 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 2.8 MiB/s wr, 26 op/s 2024-09-12T23:39:56.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:55 smithi181 bash[24776]: cluster 2024-09-12T23:39:54.698928+0000 mon.a (mon.0) 1196 : cluster [DBG] osdmap e158: 8 total, 8 up, 8 in 2024-09-12T23:39:56.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:55 smithi181 bash[24776]: cluster 2024-09-12T23:39:55.477628+0000 mon.a (mon.0) 1197 : cluster [DBG] osdmap e159: 8 total, 8 up, 8 in 2024-09-12T23:39:56.219 INFO:teuthology.orchestra.run.smithi136.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-12T23:39:56.219 INFO:teuthology.orchestra.run.smithi136.stdout:alertmanager.a smithi136 *:9093,9094 running (4m) 4m ago 5m 14.3M - 0.25.0 c8568f914cd2 d6c106834390 2024-09-12T23:39:56.219 INFO:teuthology.orchestra.run.smithi136.stdout:grafana.a smithi181 *:3000 running (4m) 4m ago 4m 29.0M - 9.4.7 954c08fa6188 85ee0f95160c 2024-09-12T23:39:56.219 INFO:teuthology.orchestra.run.smithi136.stdout:mds.a smithi136 running (6m) 4m ago 6m 12.1M - 17.2.7-1627-gbbdadfa3 214985049bdb 713b9795ad19 2024-09-12T23:39:56.219 INFO:teuthology.orchestra.run.smithi136.stdout:mds.b smithi181 running (6m) 4m ago 6m 13.9M - 17.2.7-1627-gbbdadfa3 214985049bdb ad13179f0868 2024-09-12T23:39:56.220 INFO:teuthology.orchestra.run.smithi136.stdout:mgr.x smithi181 *:8443,9283 starting - - - - 2024-09-12T23:39:56.220 INFO:teuthology.orchestra.run.smithi136.stdout:mgr.y smithi136 *:9283,8765 running (16m) 4m ago 16m 436M - 17.2.7-1627-gbbdadfa3 214985049bdb 56608cdf6b27 2024-09-12T23:39:56.220 INFO:teuthology.orchestra.run.smithi136.stdout:mon.a smithi136 running (16m) 4m ago 16m 44.1M 2048M 17.2.7-1627-gbbdadfa3 214985049bdb 0462f4f61475 2024-09-12T23:39:56.220 INFO:teuthology.orchestra.run.smithi136.stdout:mon.b smithi181 running (14m) 4m ago 14m 37.6M 2048M 17.2.7-1627-gbbdadfa3 214985049bdb 8b3d784af845 2024-09-12T23:39:56.220 INFO:teuthology.orchestra.run.smithi136.stdout:mon.c smithi136 running (14m) 4m ago 14m 36.8M 2048M 17.2.7-1627-gbbdadfa3 214985049bdb 647b028ea963 2024-09-12T23:39:56.220 INFO:teuthology.orchestra.run.smithi136.stdout:node-exporter.a smithi136 *:9100 running (5m) 4m ago 5m 3143k - 1.5.0 0da6a335fe13 8f54fc5b0da3 2024-09-12T23:39:56.220 INFO:teuthology.orchestra.run.smithi136.stdout:node-exporter.b smithi181 *:9100 running (5m) 4m ago 5m 2895k - 1.5.0 0da6a335fe13 575eff556149 2024-09-12T23:39:56.220 INFO:teuthology.orchestra.run.smithi136.stdout:osd.0 smithi136 running (13m) 4m ago 13m 47.0M 3036M 17.2.7-1627-gbbdadfa3 214985049bdb cbc0b7df0115 2024-09-12T23:39:56.220 INFO:teuthology.orchestra.run.smithi136.stdout:osd.1 smithi136 running (12m) 4m ago 12m 44.4M 3036M 17.2.7-1627-gbbdadfa3 214985049bdb c4daee7da836 2024-09-12T23:39:56.220 INFO:teuthology.orchestra.run.smithi136.stdout:osd.2 smithi136 running (11m) 4m ago 11m 44.8M 3036M 17.2.7-1627-gbbdadfa3 214985049bdb 4ab8fd79cdb4 2024-09-12T23:39:56.220 INFO:teuthology.orchestra.run.smithi136.stdout:osd.3 smithi136 running (10m) 4m ago 10m 42.5M 3036M 17.2.7-1627-gbbdadfa3 214985049bdb 9ceb14a27521 2024-09-12T23:39:56.220 INFO:teuthology.orchestra.run.smithi136.stdout:osd.4 smithi181 running (9m) 4m ago 9m 42.7M 3036M 17.2.7-1627-gbbdadfa3 214985049bdb a35cdca79055 2024-09-12T23:39:56.221 INFO:teuthology.orchestra.run.smithi136.stdout:osd.5 smithi181 running (8m) 4m ago 9m 42.2M 3036M 17.2.7-1627-gbbdadfa3 214985049bdb 8f2e8d494fef 2024-09-12T23:39:56.221 INFO:teuthology.orchestra.run.smithi136.stdout:osd.6 smithi181 running (8m) 4m ago 8m 41.2M 3036M 17.2.7-1627-gbbdadfa3 214985049bdb 9c6bec406408 2024-09-12T23:39:56.221 INFO:teuthology.orchestra.run.smithi136.stdout:osd.7 smithi181 running (7m) 4m ago 7m 42.3M 3036M 17.2.7-1627-gbbdadfa3 214985049bdb bc0632ae2145 2024-09-12T23:39:56.221 INFO:teuthology.orchestra.run.smithi136.stdout:prometheus.a smithi181 *:9095 running (4m) 4m ago 5m 22.6M - 2.43.0 a07b618ecd1d 24e591310569 2024-09-12T23:39:56.475 INFO:tasks.rados.rados.0.smithi136.stdout:547: done (0 left) 2024-09-12T23:39:56.476 INFO:tasks.rados.rados.0.smithi136.stdout:548: read oid 16 snap 45 2024-09-12T23:39:56.476 INFO:tasks.rados.rados.0.smithi136.stdout:548: expect (ObjNum 163 snap 43 seq_num 163) 2024-09-12T23:39:56.476 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 2 current snap is 47 2024-09-12T23:39:56.476 INFO:tasks.rados.rados.0.smithi136.stdout:549: seq_num 193 ranges {1802240=434176,2236416=401408} 2024-09-12T23:39:56.494 INFO:tasks.rados.rados.0.smithi136.stdout:549: writing smithi13667081-2 from 1802240 to 2236416 tid 1 2024-09-12T23:39:56.497 INFO:tasks.rados.rados.0.smithi136.stdout:549: writing smithi13667081-2 from 2236416 to 2637824 tid 2 2024-09-12T23:39:56.498 INFO:tasks.rados.rados.0.smithi136.stdout:550: setattr oid 49 current snap is 47 2024-09-12T23:39:56.500 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 45 current snap is 47 2024-09-12T23:39:56.500 INFO:tasks.rados.rados.0.smithi136.stdout:551: seq_num 194 ranges {3014656=458752,3473408=401408} 2024-09-12T23:39:56.505 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:39:56 smithi136 bash[22852]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:39:56] "GET /metrics HTTP/1.1" 200 240174 "" "Prometheus/2.43.0" 2024-09-12T23:39:56.528 INFO:tasks.rados.rados.0.smithi136.stdout:551: writing smithi13667081-45 from 3014656 to 3473408 tid 1 2024-09-12T23:39:56.530 INFO:tasks.rados.rados.0.smithi136.stdout:551: writing smithi13667081-45 from 3473408 to 3874816 tid 2 2024-09-12T23:39:56.531 INFO:tasks.rados.rados.0.smithi136.stdout:552: read oid 35 snap -1 2024-09-12T23:39:56.531 INFO:tasks.rados.rados.0.smithi136.stdout:552: expect deleted 2024-09-12T23:39:56.531 INFO:tasks.rados.rados.0.smithi136.stdout:553: read oid 14 snap -1 2024-09-12T23:39:56.531 INFO:tasks.rados.rados.0.smithi136.stdout:553: expect (ObjNum 175 snap 45 seq_num 175) 2024-09-12T23:39:56.531 INFO:tasks.rados.rados.0.smithi136.stdout:554: delete oid 39 current snap is 47 2024-09-12T23:39:56.531 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 49 v 318 (ObjNum 142 snap 36 seq_num 142) dirty exists 2024-09-12T23:39:56.531 INFO:tasks.rados.rados.0.smithi136.stdout:549: finishing write tid 1 to smithi13667081-2 2024-09-12T23:39:56.531 INFO:tasks.rados.rados.0.smithi136.stdout:549: finishing write tid 2 to smithi13667081-2 2024-09-12T23:39:56.531 INFO:tasks.rados.rados.0.smithi136.stdout:549: finishing write tid 3 to smithi13667081-2 2024-09-12T23:39:56.531 INFO:tasks.rados.rados.0.smithi136.stdout:548: expect (ObjNum 163 snap 43 seq_num 163) 2024-09-12T23:39:56.600 INFO:teuthology.orchestra.run.smithi136.stdout:{ 2024-09-12T23:39:56.600 INFO:teuthology.orchestra.run.smithi136.stdout: "mon": { 2024-09-12T23:39:56.601 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 3 2024-09-12T23:39:56.601 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:39:56.601 INFO:teuthology.orchestra.run.smithi136.stdout: "mgr": { 2024-09-12T23:39:56.601 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-12T23:39:56.601 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:39:56.601 INFO:teuthology.orchestra.run.smithi136.stdout: "osd": { 2024-09-12T23:39:56.601 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 8 2024-09-12T23:39:56.601 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:39:56.601 INFO:teuthology.orchestra.run.smithi136.stdout: "mds": { 2024-09-12T23:39:56.601 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-12T23:39:56.601 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:39:56.601 INFO:teuthology.orchestra.run.smithi136.stdout: "overall": { 2024-09-12T23:39:56.601 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 15 2024-09-12T23:39:56.601 INFO:teuthology.orchestra.run.smithi136.stdout: } 2024-09-12T23:39:56.602 INFO:teuthology.orchestra.run.smithi136.stdout:} 2024-09-12T23:39:56.780 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:39:56 smithi181 bash[52367]: debug 2024-09-12T23:39:56.443+0000 7fc9a18c8040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-12T23:39:56.781 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:39:56 smithi181 bash[52367]: debug 2024-09-12T23:39:56.539+0000 7fc9a18c8040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-12T23:39:56.820 INFO:tasks.rados.rados.0.smithi136.stdout:549: finishing write tid 4 to smithi13667081-2 2024-09-12T23:39:56.820 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 2 v 230 (ObjNum 193 snap 47 seq_num 193) dirty exists 2024-09-12T23:39:56.820 INFO:tasks.rados.rados.0.smithi136.stdout:549: left oid 2 (ObjNum 193 snap 47 seq_num 193) 2024-09-12T23:39:56.820 INFO:tasks.rados.rados.0.smithi136.stdout:551: finishing write tid 1 to smithi13667081-45 2024-09-12T23:39:56.820 INFO:tasks.rados.rados.0.smithi136.stdout:551: finishing write tid 2 to smithi13667081-45 2024-09-12T23:39:56.820 INFO:tasks.rados.rados.0.smithi136.stdout:551: finishing write tid 3 to smithi13667081-45 2024-09-12T23:39:56.820 INFO:tasks.rados.rados.0.smithi136.stdout:553: expect (ObjNum 175 snap 45 seq_num 175) 2024-09-12T23:39:56.839 INFO:tasks.rados.rados.0.smithi136.stdout:551: finishing write tid 4 to smithi13667081-45 2024-09-12T23:39:56.839 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 45 v 152 (ObjNum 194 snap 47 seq_num 194) dirty exists 2024-09-12T23:39:56.839 INFO:tasks.rados.rados.0.smithi136.stdout:551: left oid 45 (ObjNum 194 snap 47 seq_num 194) 2024-09-12T23:39:56.839 INFO:tasks.rados.rados.0.smithi136.stdout:548: done (6 left) 2024-09-12T23:39:56.839 INFO:tasks.rados.rados.0.smithi136.stdout:549: done (5 left) 2024-09-12T23:39:56.839 INFO:tasks.rados.rados.0.smithi136.stdout:550: done (4 left) 2024-09-12T23:39:56.839 INFO:tasks.rados.rados.0.smithi136.stdout:551: done (3 left) 2024-09-12T23:39:56.839 INFO:tasks.rados.rados.0.smithi136.stdout:552: done (2 left) 2024-09-12T23:39:56.839 INFO:tasks.rados.rados.0.smithi136.stdout:553: done (1 left) 2024-09-12T23:39:56.840 INFO:tasks.rados.rados.0.smithi136.stdout:554: done (0 left) 2024-09-12T23:39:56.840 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 40 current snap is 47 2024-09-12T23:39:56.840 INFO:tasks.rados.rados.0.smithi136.stdout:555: seq_num 196 ranges {720896=491520,1212416=401408} 2024-09-12T23:39:56.851 INFO:tasks.rados.rados.0.smithi136.stdout:555: writing smithi13667081-40 from 720896 to 1212416 tid 1 2024-09-12T23:39:56.855 INFO:tasks.rados.rados.0.smithi136.stdout:555: writing smithi13667081-40 from 1212416 to 1613824 tid 2 2024-09-12T23:39:56.855 INFO:tasks.rados.rados.0.smithi136.stdout:556: snap_create 2024-09-12T23:39:56.859 INFO:tasks.rados.rados.0.smithi136.stdout:555: finishing write tid 1 to smithi13667081-40 2024-09-12T23:39:56.859 INFO:tasks.rados.rados.0.smithi136.stdout:555: finishing write tid 2 to smithi13667081-40 2024-09-12T23:39:56.860 INFO:tasks.rados.rados.0.smithi136.stdout:555: finishing write tid 3 to smithi13667081-40 2024-09-12T23:39:56.861 INFO:tasks.rados.rados.0.smithi136.stdout:555: finishing write tid 4 to smithi13667081-40 2024-09-12T23:39:56.861 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 40 v 250 (ObjNum 196 snap 47 seq_num 196) dirty exists 2024-09-12T23:39:56.861 INFO:tasks.rados.rados.0.smithi136.stdout:555: left oid 40 (ObjNum 196 snap 47 seq_num 196) 2024-09-12T23:39:57.094 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:39:56 smithi181 bash[52367]: debug 2024-09-12T23:39:56.783+0000 7fc9a18c8040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-12T23:39:57.094 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:39:56 smithi181 bash[52367]: debug 2024-09-12T23:39:56.863+0000 7fc9a18c8040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-12T23:39:57.594 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:39:57 smithi181 bash[52367]: debug 2024-09-12T23:39:57.287+0000 7fc9a18c8040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-12T23:39:57.594 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:39:57 smithi181 bash[52367]: debug 2024-09-12T23:39:57.367+0000 7fc9a18c8040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-12T23:39:57.594 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:39:57 smithi181 bash[52367]: [12/Sep/2024:23:39:57] ENGINE Bus STARTING 2024-09-12T23:39:57.594 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:39:57 smithi181 bash[52367]: CherryPy Checker: 2024-09-12T23:39:57.594 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:39:57 smithi181 bash[52367]: The Application mounted at '' has an empty config. 2024-09-12T23:39:57.595 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:39:57 smithi181 bash[52367]: [12/Sep/2024:23:39:57] ENGINE Serving on http://:::9283 2024-09-12T23:39:57.595 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:39:57 smithi181 bash[52367]: [12/Sep/2024:23:39:57] ENGINE Bus STARTED 2024-09-12T23:39:57.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:57 smithi181 bash[24776]: audit 2024-09-12T23:39:55.923998+0000 mgr.y (mgr.24446) 243 : audit [DBG] from='client.24956 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:39:57.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:57 smithi181 bash[24776]: cluster 2024-09-12T23:39:56.050480+0000 mgr.y (mgr.24446) 244 : cluster [DBG] pgmap v281: 121 pgs: 121 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 2.8 MiB/s wr, 26 op/s 2024-09-12T23:39:57.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:57 smithi181 bash[24776]: audit 2024-09-12T23:39:56.207845+0000 mgr.y (mgr.24446) 245 : audit [DBG] from='client.24962 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:39:57.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:57 smithi181 bash[24776]: cluster 2024-09-12T23:39:56.480111+0000 mon.a (mon.0) 1198 : cluster [DBG] osdmap e160: 8 total, 8 up, 8 in 2024-09-12T23:39:57.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:57 smithi181 bash[24776]: audit 2024-09-12T23:39:56.605317+0000 mon.a (mon.0) 1199 : audit [DBG] from='client.? 172.21.15.136:0/3958653305' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:39:57.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:57 smithi181 bash[24776]: audit 2024-09-12T23:39:57.086182+0000 mon.a (mon.0) 1200 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:39:57.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:57 smithi181 bash[24776]: audit 2024-09-12T23:39:57.097657+0000 mon.a (mon.0) 1201 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:39:57.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:57 smithi181 bash[24776]: cluster 2024-09-12T23:39:57.373591+0000 mon.a (mon.0) 1202 : cluster [DBG] Standby manager daemon x restarted 2024-09-12T23:39:57.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:57 smithi181 bash[24776]: cluster 2024-09-12T23:39:57.373840+0000 mon.a (mon.0) 1203 : cluster [DBG] Standby manager daemon x started 2024-09-12T23:39:57.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:57 smithi181 bash[24776]: audit 2024-09-12T23:39:57.374411+0000 mon.b (mon.1) 36 : audit [DBG] from='mgr.? 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-12T23:39:57.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:57 smithi181 bash[24776]: audit 2024-09-12T23:39:57.375068+0000 mon.b (mon.1) 37 : audit [DBG] from='mgr.? 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-12T23:39:57.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:57 smithi181 bash[24776]: audit 2024-09-12T23:39:57.376936+0000 mon.b (mon.1) 38 : audit [DBG] from='mgr.? 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-12T23:39:57.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:57 smithi181 bash[24776]: audit 2024-09-12T23:39:57.378100+0000 mon.b (mon.1) 39 : audit [DBG] from='mgr.? 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-12T23:39:57.702 INFO:tasks.rados.rados.0.smithi136.stdout:555: done (1 left) 2024-09-12T23:39:57.703 INFO:tasks.rados.rados.0.smithi136.stdout:556: done (0 left) 2024-09-12T23:39:57.703 INFO:tasks.rados.rados.0.smithi136.stdout:557: setattr oid 5 current snap is 48 2024-09-12T23:39:57.705 INFO:tasks.rados.rados.0.smithi136.stdout:558: rollback oid 50 current snap is 48 2024-09-12T23:39:57.705 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 50 to 46 2024-09-12T23:39:57.705 INFO:tasks.rados.rados.0.smithi136.stdout:559: setattr oid 47 current snap is 48 2024-09-12T23:39:57.709 INFO:tasks.rados.rados.0.smithi136.stdout:560: rollback oid 10 current snap is 48 2024-09-12T23:39:57.709 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 10 to 29 2024-09-12T23:39:57.709 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 11 current snap is 48 2024-09-12T23:39:57.709 INFO:tasks.rados.rados.0.smithi136.stdout:561: seq_num 197 ranges {827392=507904,1335296=401408} 2024-09-12T23:39:57.726 INFO:tasks.rados.rados.0.smithi136.stdout:561: writing smithi13667081-11 from 827392 to 1335296 tid 1 2024-09-12T23:39:57.731 INFO:tasks.rados.rados.0.smithi136.stdout:561: writing smithi13667081-11 from 1335296 to 1736704 tid 2 2024-09-12T23:39:57.732 INFO:tasks.rados.rados.0.smithi136.stdout:562: snap_remove snap 28 2024-09-12T23:39:57.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:57 smithi136 bash[22601]: audit 2024-09-12T23:39:55.923998+0000 mgr.y (mgr.24446) 243 : audit [DBG] from='client.24956 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:39:57.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:57 smithi136 bash[22601]: cluster 2024-09-12T23:39:56.050480+0000 mgr.y (mgr.24446) 244 : cluster [DBG] pgmap v281: 121 pgs: 121 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 2.8 MiB/s wr, 26 op/s 2024-09-12T23:39:57.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:57 smithi136 bash[22601]: audit 2024-09-12T23:39:56.207845+0000 mgr.y (mgr.24446) 245 : audit [DBG] from='client.24962 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:39:57.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:57 smithi136 bash[22601]: cluster 2024-09-12T23:39:56.480111+0000 mon.a (mon.0) 1198 : cluster [DBG] osdmap e160: 8 total, 8 up, 8 in 2024-09-12T23:39:57.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:57 smithi136 bash[22601]: audit 2024-09-12T23:39:56.605317+0000 mon.a (mon.0) 1199 : audit [DBG] from='client.? 172.21.15.136:0/3958653305' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:39:57.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:57 smithi136 bash[22601]: audit 2024-09-12T23:39:57.086182+0000 mon.a (mon.0) 1200 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:39:57.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:57 smithi136 bash[22601]: audit 2024-09-12T23:39:57.097657+0000 mon.a (mon.0) 1201 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:39:57.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:57 smithi136 bash[22601]: cluster 2024-09-12T23:39:57.373591+0000 mon.a (mon.0) 1202 : cluster [DBG] Standby manager daemon x restarted 2024-09-12T23:39:57.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:57 smithi136 bash[22601]: cluster 2024-09-12T23:39:57.373840+0000 mon.a (mon.0) 1203 : cluster [DBG] Standby manager daemon x started 2024-09-12T23:39:57.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:57 smithi136 bash[22601]: audit 2024-09-12T23:39:57.374411+0000 mon.b (mon.1) 36 : audit [DBG] from='mgr.? 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-12T23:39:57.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:57 smithi136 bash[22601]: audit 2024-09-12T23:39:57.375068+0000 mon.b (mon.1) 37 : audit [DBG] from='mgr.? 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-12T23:39:57.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:57 smithi136 bash[22601]: audit 2024-09-12T23:39:57.376936+0000 mon.b (mon.1) 38 : audit [DBG] from='mgr.? 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-12T23:39:57.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:57 smithi136 bash[22601]: audit 2024-09-12T23:39:57.378100+0000 mon.b (mon.1) 39 : audit [DBG] from='mgr.? 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-12T23:39:57.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:57 smithi136 bash[32189]: audit 2024-09-12T23:39:55.923998+0000 mgr.y (mgr.24446) 243 : audit [DBG] from='client.24956 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:39:57.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:57 smithi136 bash[32189]: cluster 2024-09-12T23:39:56.050480+0000 mgr.y (mgr.24446) 244 : cluster [DBG] pgmap v281: 121 pgs: 121 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 2.8 MiB/s wr, 26 op/s 2024-09-12T23:39:57.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:57 smithi136 bash[32189]: audit 2024-09-12T23:39:56.207845+0000 mgr.y (mgr.24446) 245 : audit [DBG] from='client.24962 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:39:57.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:57 smithi136 bash[32189]: cluster 2024-09-12T23:39:56.480111+0000 mon.a (mon.0) 1198 : cluster [DBG] osdmap e160: 8 total, 8 up, 8 in 2024-09-12T23:39:57.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:57 smithi136 bash[32189]: audit 2024-09-12T23:39:56.605317+0000 mon.a (mon.0) 1199 : audit [DBG] from='client.? 172.21.15.136:0/3958653305' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:39:57.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:57 smithi136 bash[32189]: audit 2024-09-12T23:39:57.086182+0000 mon.a (mon.0) 1200 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:39:57.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:57 smithi136 bash[32189]: audit 2024-09-12T23:39:57.097657+0000 mon.a (mon.0) 1201 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:39:57.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:57 smithi136 bash[32189]: cluster 2024-09-12T23:39:57.373591+0000 mon.a (mon.0) 1202 : cluster [DBG] Standby manager daemon x restarted 2024-09-12T23:39:57.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:57 smithi136 bash[32189]: cluster 2024-09-12T23:39:57.373840+0000 mon.a (mon.0) 1203 : cluster [DBG] Standby manager daemon x started 2024-09-12T23:39:57.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:57 smithi136 bash[32189]: audit 2024-09-12T23:39:57.374411+0000 mon.b (mon.1) 36 : audit [DBG] from='mgr.? 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-12T23:39:57.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:57 smithi136 bash[32189]: audit 2024-09-12T23:39:57.375068+0000 mon.b (mon.1) 37 : audit [DBG] from='mgr.? 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-12T23:39:57.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:57 smithi136 bash[32189]: audit 2024-09-12T23:39:57.376936+0000 mon.b (mon.1) 38 : audit [DBG] from='mgr.? 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-12T23:39:57.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:57 smithi136 bash[32189]: audit 2024-09-12T23:39:57.378100+0000 mon.b (mon.1) 39 : audit [DBG] from='mgr.? 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-12T23:39:58.713 INFO:tasks.rados.rados.0.smithi136.stdout:562: done (5 left) 2024-09-12T23:39:58.713 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 23 current snap is 48 2024-09-12T23:39:58.713 INFO:tasks.rados.rados.0.smithi136.stdout:560: finishing rollback tid 1 to smithi13667081-10 2024-09-12T23:39:58.713 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 10 v 60 (ObjNum 161 snap 0 seq_num 4242934064) dirty dne 2024-09-12T23:39:58.713 INFO:tasks.rados.rados.0.smithi136.stdout:558: finishing rollback tid 0 to smithi13667081-50 2024-09-12T23:39:58.713 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 5 v 251 (ObjNum 33 snap 0 seq_num 4242592000) dirty exists 2024-09-12T23:39:58.714 INFO:tasks.rados.rados.0.smithi136.stdout:558: finishing rollback tid 1 to smithi13667081-50 2024-09-12T23:39:58.714 INFO:tasks.rados.rados.0.smithi136.stdout:558: finishing rollback tid 2 to smithi13667081-50 2024-09-12T23:39:58.714 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 50 v 191 (ObjNum 49 snap 0 seq_num 49) dirty exists 2024-09-12T23:39:58.714 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 47 v 250 (ObjNum 81 snap 0 seq_num 0) dirty exists 2024-09-12T23:39:58.714 INFO:tasks.rados.rados.0.smithi136.stdout:561: finishing write tid 1 to smithi13667081-11 2024-09-12T23:39:58.714 INFO:tasks.rados.rados.0.smithi136.stdout:561: finishing write tid 2 to smithi13667081-11 2024-09-12T23:39:58.714 INFO:tasks.rados.rados.0.smithi136.stdout:561: finishing write tid 3 to smithi13667081-11 2024-09-12T23:39:58.714 INFO:tasks.rados.rados.0.smithi136.stdout:561: finishing write tid 4 to smithi13667081-11 2024-09-12T23:39:58.714 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 11 v 255 (ObjNum 197 snap 48 seq_num 197) dirty exists 2024-09-12T23:39:58.714 INFO:tasks.rados.rados.0.smithi136.stdout:561: left oid 11 (ObjNum 197 snap 48 seq_num 197) 2024-09-12T23:39:58.714 INFO:tasks.rados.rados.0.smithi136.stdout:563: seq_num 198 ranges {2023424=524288,2547712=401408} 2024-09-12T23:39:58.728 INFO:tasks.rados.rados.0.smithi136.stdout:563: writing smithi13667081-23 from 2023424 to 2547712 tid 1 2024-09-12T23:39:58.731 INFO:tasks.rados.rados.0.smithi136.stdout:563: writing smithi13667081-23 from 2547712 to 2949120 tid 2 2024-09-12T23:39:58.732 INFO:tasks.rados.rados.0.smithi136.stdout:557: done (5 left) 2024-09-12T23:39:58.732 INFO:tasks.rados.rados.0.smithi136.stdout:558: done (4 left) 2024-09-12T23:39:58.732 INFO:tasks.rados.rados.0.smithi136.stdout:559: done (3 left) 2024-09-12T23:39:58.732 INFO:tasks.rados.rados.0.smithi136.stdout:560: done (2 left) 2024-09-12T23:39:58.732 INFO:tasks.rados.rados.0.smithi136.stdout:561: done (1 left) 2024-09-12T23:39:58.732 INFO:tasks.rados.rados.0.smithi136.stdout:564: rollback oid 44 current snap is 48 2024-09-12T23:39:58.732 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 44 to 29 2024-09-12T23:39:58.732 INFO:tasks.rados.rados.0.smithi136.stdout:565: rmattr oid 40 current snap is 48 2024-09-12T23:39:58.732 INFO:tasks.rados.rados.0.smithi136.stdout:565: done (2 left) 2024-09-12T23:39:58.732 INFO:tasks.rados.rados.0.smithi136.stdout:566: read oid 36 snap -1 2024-09-12T23:39:58.732 INFO:tasks.rados.rados.0.smithi136.stdout:566: expect (ObjNum 192 snap 46 seq_num 192) 2024-09-12T23:39:58.733 INFO:tasks.rados.rados.0.smithi136.stdout:567: snap_create 2024-09-12T23:39:58.743 INFO:tasks.rados.rados.0.smithi136.stdout:564: finishing rollback tid 0 to smithi13667081-44 2024-09-12T23:39:58.745 INFO:tasks.rados.rados.0.smithi136.stdout:563: finishing write tid 1 to smithi13667081-23 2024-09-12T23:39:58.746 INFO:tasks.rados.rados.0.smithi136.stdout:564: finishing rollback tid 1 to smithi13667081-44 2024-09-12T23:39:58.746 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 44 v 267 (ObjNum 65 snap 0 seq_num 1902792295) dirty dne 2024-09-12T23:39:58.748 INFO:tasks.rados.rados.0.smithi136.stdout:563: finishing write tid 2 to smithi13667081-23 2024-09-12T23:39:58.748 INFO:tasks.rados.rados.0.smithi136.stdout:563: finishing write tid 3 to smithi13667081-23 2024-09-12T23:39:58.751 INFO:tasks.rados.rados.0.smithi136.stdout:563: finishing write tid 4 to smithi13667081-23 2024-09-12T23:39:58.751 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 23 v 263 (ObjNum 198 snap 48 seq_num 198) dirty exists 2024-09-12T23:39:58.751 INFO:tasks.rados.rados.0.smithi136.stdout:563: left oid 23 (ObjNum 198 snap 48 seq_num 198) 2024-09-12T23:39:58.762 INFO:tasks.rados.rados.0.smithi136.stdout:566: expect (ObjNum 192 snap 46 seq_num 192) 2024-09-12T23:39:59.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:58 smithi136 bash[22601]: audit 2024-09-12T23:39:57.685053+0000 mon.a (mon.0) 1204 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:39:59.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:58 smithi136 bash[22601]: audit 2024-09-12T23:39:57.697709+0000 mon.a (mon.0) 1205 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:39:59.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:58 smithi136 bash[22601]: cluster 2024-09-12T23:39:57.706163+0000 mon.a (mon.0) 1206 : cluster [DBG] osdmap e161: 8 total, 8 up, 8 in 2024-09-12T23:39:59.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:58 smithi136 bash[22601]: cluster 2024-09-12T23:39:58.108608+0000 mon.a (mon.0) 1207 : cluster [DBG] mgrmap e23: y(active, since 5m), standbys: x 2024-09-12T23:39:59.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:58 smithi136 bash[32189]: audit 2024-09-12T23:39:57.685053+0000 mon.a (mon.0) 1204 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:39:59.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:58 smithi136 bash[32189]: audit 2024-09-12T23:39:57.697709+0000 mon.a (mon.0) 1205 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:39:59.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:58 smithi136 bash[32189]: cluster 2024-09-12T23:39:57.706163+0000 mon.a (mon.0) 1206 : cluster [DBG] osdmap e161: 8 total, 8 up, 8 in 2024-09-12T23:39:59.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:58 smithi136 bash[32189]: cluster 2024-09-12T23:39:58.108608+0000 mon.a (mon.0) 1207 : cluster [DBG] mgrmap e23: y(active, since 5m), standbys: x 2024-09-12T23:39:59.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:58 smithi181 bash[24776]: audit 2024-09-12T23:39:57.685053+0000 mon.a (mon.0) 1204 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:39:59.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:58 smithi181 bash[24776]: audit 2024-09-12T23:39:57.697709+0000 mon.a (mon.0) 1205 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:39:59.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:58 smithi181 bash[24776]: cluster 2024-09-12T23:39:57.706163+0000 mon.a (mon.0) 1206 : cluster [DBG] osdmap e161: 8 total, 8 up, 8 in 2024-09-12T23:39:59.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:58 smithi181 bash[24776]: cluster 2024-09-12T23:39:58.108608+0000 mon.a (mon.0) 1207 : cluster [DBG] mgrmap e23: y(active, since 5m), standbys: x 2024-09-12T23:39:59.722 INFO:tasks.rados.rados.0.smithi136.stdout:563: done (3 left) 2024-09-12T23:39:59.722 INFO:tasks.rados.rados.0.smithi136.stdout:564: done (2 left) 2024-09-12T23:39:59.722 INFO:tasks.rados.rados.0.smithi136.stdout:566: done (1 left) 2024-09-12T23:39:59.722 INFO:tasks.rados.rados.0.smithi136.stdout:567: done (0 left) 2024-09-12T23:39:59.722 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 3 current snap is 49 2024-09-12T23:39:59.722 INFO:tasks.rados.rados.0.smithi136.stdout:568: seq_num 199 ranges {524288=540672,1064960=401408} 2024-09-12T23:39:59.736 INFO:tasks.rados.rados.0.smithi136.stdout:568: writing smithi13667081-3 from 524288 to 1064960 tid 1 2024-09-12T23:39:59.742 INFO:tasks.rados.rados.0.smithi136.stdout:568: writing smithi13667081-3 from 1064960 to 1466368 tid 2 2024-09-12T23:39:59.742 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 49 current snap is 49 2024-09-12T23:39:59.742 INFO:tasks.rados.rados.0.smithi136.stdout:569: seq_num 200 ranges {4530176=557056,5087232=401408} 2024-09-12T23:39:59.796 INFO:tasks.rados.rados.0.smithi136.stdout:569: writing smithi13667081-49 from 4530176 to 5087232 tid 1 2024-09-12T23:39:59.800 INFO:tasks.rados.rados.0.smithi136.stdout:569: writing smithi13667081-49 from 5087232 to 5488640 tid 2 2024-09-12T23:39:59.800 INFO:tasks.rados.rados.0.smithi136.stdout:570: read oid 32 snap 48 2024-09-12T23:39:59.800 INFO:tasks.rados.rados.0.smithi136.stdout:570: expect deleted 2024-09-12T23:39:59.800 INFO:tasks.rados.rados.0.smithi136.stdout:571: rollback oid 50 current snap is 49 2024-09-12T23:39:59.801 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 50 to 46 2024-09-12T23:39:59.801 INFO:tasks.rados.rados.0.smithi136.stdout:568: finishing write tid 1 to smithi13667081-3 2024-09-12T23:39:59.801 INFO:tasks.rados.rados.0.smithi136.stdout:568: finishing write tid 2 to smithi13667081-3 2024-09-12T23:39:59.801 INFO:tasks.rados.rados.0.smithi136.stdout:568: finishing write tid 3 to smithi13667081-3 2024-09-12T23:39:59.801 INFO:tasks.rados.rados.0.smithi136.stdout:568: finishing write tid 4 to smithi13667081-3 2024-09-12T23:39:59.801 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 3 v 336 (ObjNum 199 snap 49 seq_num 199) dirty exists 2024-09-12T23:39:59.801 INFO:tasks.rados.rados.0.smithi136.stdout:568: left oid 3 (ObjNum 199 snap 49 seq_num 199) 2024-09-12T23:39:59.801 INFO:tasks.rados.rados.0.smithi136.stdout:568: done (3 left) 2024-09-12T23:39:59.801 INFO:tasks.rados.rados.0.smithi136.stdout:572: snap_create 2024-09-12T23:39:59.806 INFO:tasks.rados.rados.0.smithi136.stdout:569: finishing write tid 1 to smithi13667081-49 2024-09-12T23:39:59.807 INFO:tasks.rados.rados.0.smithi136.stdout:569: finishing write tid 2 to smithi13667081-49 2024-09-12T23:39:59.807 INFO:tasks.rados.rados.0.smithi136.stdout:569: finishing write tid 3 to smithi13667081-49 2024-09-12T23:39:59.808 INFO:tasks.rados.rados.0.smithi136.stdout:571: finishing rollback tid 0 to smithi13667081-50 2024-09-12T23:39:59.809 INFO:tasks.rados.rados.0.smithi136.stdout:571: finishing rollback tid 1 to smithi13667081-50 2024-09-12T23:39:59.809 INFO:tasks.rados.rados.0.smithi136.stdout:569: finishing write tid 4 to smithi13667081-49 2024-09-12T23:39:59.809 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 49 v 340 (ObjNum 200 snap 49 seq_num 200) dirty exists 2024-09-12T23:39:59.810 INFO:tasks.rados.rados.0.smithi136.stdout:569: left oid 49 (ObjNum 200 snap 49 seq_num 200) 2024-09-12T23:39:59.810 INFO:tasks.rados.rados.0.smithi136.stdout:571: finishing rollback tid 2 to smithi13667081-50 2024-09-12T23:39:59.810 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 50 v 203 (ObjNum 49 snap 0 seq_num 49) dirty exists 2024-09-12T23:40:00.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:59 smithi136 bash[32189]: cluster 2024-09-12T23:39:58.051134+0000 mgr.y (mgr.24446) 246 : cluster [DBG] pgmap v284: 121 pgs: 121 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:40:00.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:39:59 smithi136 bash[32189]: cluster 2024-09-12T23:39:58.716091+0000 mon.a (mon.0) 1208 : cluster [DBG] osdmap e162: 8 total, 8 up, 8 in 2024-09-12T23:40:00.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:59 smithi136 bash[22601]: cluster 2024-09-12T23:39:58.051134+0000 mgr.y (mgr.24446) 246 : cluster [DBG] pgmap v284: 121 pgs: 121 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:40:00.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:39:59 smithi136 bash[22601]: cluster 2024-09-12T23:39:58.716091+0000 mon.a (mon.0) 1208 : cluster [DBG] osdmap e162: 8 total, 8 up, 8 in 2024-09-12T23:40:00.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:59 smithi181 bash[24776]: cluster 2024-09-12T23:39:58.051134+0000 mgr.y (mgr.24446) 246 : cluster [DBG] pgmap v284: 121 pgs: 121 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:40:00.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:39:59 smithi181 bash[24776]: cluster 2024-09-12T23:39:58.716091+0000 mon.a (mon.0) 1208 : cluster [DBG] osdmap e162: 8 total, 8 up, 8 in 2024-09-12T23:40:00.732 INFO:tasks.rados.rados.0.smithi136.stdout:569: done (3 left) 2024-09-12T23:40:00.732 INFO:tasks.rados.rados.0.smithi136.stdout:570: done (2 left) 2024-09-12T23:40:00.732 INFO:tasks.rados.rados.0.smithi136.stdout:571: done (1 left) 2024-09-12T23:40:00.732 INFO:tasks.rados.rados.0.smithi136.stdout:572: done (0 left) 2024-09-12T23:40:00.732 INFO:tasks.rados.rados.0.smithi136.stdout:573: read oid 22 snap -1 2024-09-12T23:40:00.732 INFO:tasks.rados.rados.0.smithi136.stdout:573: expect (ObjNum 21 snap 0 seq_num 21) 2024-09-12T23:40:00.732 INFO:tasks.rados.rados.0.smithi136.stdout:574: rollback oid 47 current snap is 50 2024-09-12T23:40:00.732 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 47 to 48 2024-09-12T23:40:00.732 INFO:tasks.rados.rados.0.smithi136.stdout:575: snap_remove snap 29 2024-09-12T23:40:01.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:00 smithi136 bash[32189]: cluster 2024-09-12T23:39:59.726696+0000 mon.a (mon.0) 1209 : cluster [DBG] osdmap e163: 8 total, 8 up, 8 in 2024-09-12T23:40:01.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:00 smithi136 bash[32189]: cluster 2024-09-12T23:40:00.000157+0000 mon.a (mon.0) 1210 : cluster [INF] overall HEALTH_OK 2024-09-12T23:40:01.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:00 smithi136 bash[32189]: cluster 2024-09-12T23:40:00.051811+0000 mgr.y (mgr.24446) 247 : cluster [DBG] pgmap v287: 121 pgs: 121 active+clean; 122 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.0 MiB/s rd, 1.8 MiB/s wr, 28 op/s 2024-09-12T23:40:01.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:00 smithi136 bash[22601]: cluster 2024-09-12T23:39:59.726696+0000 mon.a (mon.0) 1209 : cluster [DBG] osdmap e163: 8 total, 8 up, 8 in 2024-09-12T23:40:01.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:00 smithi136 bash[22601]: cluster 2024-09-12T23:40:00.000157+0000 mon.a (mon.0) 1210 : cluster [INF] overall HEALTH_OK 2024-09-12T23:40:01.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:00 smithi136 bash[22601]: cluster 2024-09-12T23:40:00.051811+0000 mgr.y (mgr.24446) 247 : cluster [DBG] pgmap v287: 121 pgs: 121 active+clean; 122 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.0 MiB/s rd, 1.8 MiB/s wr, 28 op/s 2024-09-12T23:40:01.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:00 smithi181 bash[24776]: cluster 2024-09-12T23:39:59.726696+0000 mon.a (mon.0) 1209 : cluster [DBG] osdmap e163: 8 total, 8 up, 8 in 2024-09-12T23:40:01.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:00 smithi181 bash[24776]: cluster 2024-09-12T23:40:00.000157+0000 mon.a (mon.0) 1210 : cluster [INF] overall HEALTH_OK 2024-09-12T23:40:01.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:00 smithi181 bash[24776]: cluster 2024-09-12T23:40:00.051811+0000 mgr.y (mgr.24446) 247 : cluster [DBG] pgmap v287: 121 pgs: 121 active+clean; 122 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.0 MiB/s rd, 1.8 MiB/s wr, 28 op/s 2024-09-12T23:40:01.739 INFO:tasks.rados.rados.0.smithi136.stdout:575: done (2 left) 2024-09-12T23:40:01.740 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 6 current snap is 50 2024-09-12T23:40:01.740 INFO:tasks.rados.rados.0.smithi136.stdout:576: seq_num 201 ranges {606208=573440,1179648=401408} 2024-09-12T23:40:01.746 INFO:tasks.rados.rados.0.smithi136.stdout:576: writing smithi13667081-6 from 606208 to 1179648 tid 1 2024-09-12T23:40:01.749 INFO:tasks.rados.rados.0.smithi136.stdout:576: writing smithi13667081-6 from 1179648 to 1581056 tid 2 2024-09-12T23:40:01.750 INFO:tasks.rados.rados.0.smithi136.stdout:577: copy_from oid 41 from oid 46 current snap is 50 2024-09-12T23:40:01.750 INFO:tasks.rados.rados.0.smithi136.stdout:573: expect (ObjNum 21 snap 0 seq_num 21) 2024-09-12T23:40:01.790 INFO:tasks.rados.rados.0.smithi136.stdout:574: finishing rollback tid 0 to smithi13667081-47 2024-09-12T23:40:01.790 INFO:tasks.rados.rados.0.smithi136.stdout:574: finishing rollback tid 1 to smithi13667081-47 2024-09-12T23:40:01.790 INFO:tasks.rados.rados.0.smithi136.stdout:574: finishing rollback tid 2 to smithi13667081-47 2024-09-12T23:40:01.790 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 47 v 253 (ObjNum 161 snap 0 seq_num 0) dirty exists 2024-09-12T23:40:01.790 INFO:tasks.rados.rados.0.smithi136.stdout:573: done (3 left) 2024-09-12T23:40:01.790 INFO:tasks.rados.rados.0.smithi136.stdout:574: done (2 left) 2024-09-12T23:40:01.790 INFO:tasks.rados.rados.0.smithi136.stdout:578: rollback oid 27 current snap is 50 2024-09-12T23:40:01.790 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 27 to 46 2024-09-12T23:40:01.790 INFO:tasks.rados.rados.0.smithi136.stdout:577: finishing copy_from to smithi13667081-41 2024-09-12T23:40:01.791 INFO:tasks.rados.rados.0.smithi136.stdout:577: got expected ENOENT (src dne) 2024-09-12T23:40:01.791 INFO:tasks.rados.rados.0.smithi136.stdout:577: finishing copy_from racing read to smithi13667081-41 2024-09-12T23:40:01.791 INFO:tasks.rados.rados.0.smithi136.stdout:576: finishing write tid 1 to smithi13667081-6 2024-09-12T23:40:01.791 INFO:tasks.rados.rados.0.smithi136.stdout:576: finishing write tid 2 to smithi13667081-6 2024-09-12T23:40:01.791 INFO:tasks.rados.rados.0.smithi136.stdout:576: finishing write tid 3 to smithi13667081-6 2024-09-12T23:40:01.791 INFO:tasks.rados.rados.0.smithi136.stdout:576: finishing write tid 4 to smithi13667081-6 2024-09-12T23:40:01.791 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 6 v 60 (ObjNum 201 snap 50 seq_num 201) dirty exists 2024-09-12T23:40:01.791 INFO:tasks.rados.rados.0.smithi136.stdout:576: left oid 6 (ObjNum 201 snap 50 seq_num 201) 2024-09-12T23:40:01.791 INFO:tasks.rados.rados.0.smithi136.stdout:576: done (2 left) 2024-09-12T23:40:01.791 INFO:tasks.rados.rados.0.smithi136.stdout:577: done (1 left) 2024-09-12T23:40:01.791 INFO:tasks.rados.rados.0.smithi136.stdout:579: snap_remove snap 40 2024-09-12T23:40:02.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:01 smithi136 bash[22601]: cluster 2024-09-12T23:40:00.736016+0000 mon.a (mon.0) 1211 : cluster [DBG] osdmap e164: 8 total, 8 up, 8 in 2024-09-12T23:40:02.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:01 smithi136 bash[32189]: cluster 2024-09-12T23:40:00.736016+0000 mon.a (mon.0) 1211 : cluster [DBG] osdmap e164: 8 total, 8 up, 8 in 2024-09-12T23:40:02.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:01 smithi181 bash[24776]: cluster 2024-09-12T23:40:00.736016+0000 mon.a (mon.0) 1211 : cluster [DBG] osdmap e164: 8 total, 8 up, 8 in 2024-09-12T23:40:02.745 INFO:tasks.rados.rados.0.smithi136.stdout:578: finishing rollback tid 0 to smithi13667081-27 2024-09-12T23:40:02.745 INFO:tasks.rados.rados.0.smithi136.stdout:578: finishing rollback tid 1 to smithi13667081-27 2024-09-12T23:40:02.745 INFO:tasks.rados.rados.0.smithi136.stdout:578: finishing rollback tid 2 to smithi13667081-27 2024-09-12T23:40:02.745 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 27 v 279 (ObjNum 26 snap 0 seq_num 26) dirty exists 2024-09-12T23:40:02.745 INFO:tasks.rados.rados.0.smithi136.stdout:578: done (1 left) 2024-09-12T23:40:02.745 INFO:tasks.rados.rados.0.smithi136.stdout:579: done (0 left) 2024-09-12T23:40:02.745 INFO:tasks.rados.rados.0.smithi136.stdout:580: snap_remove snap 48 2024-09-12T23:40:03.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:02 smithi136 bash[22601]: cluster 2024-09-12T23:40:01.743905+0000 mon.a (mon.0) 1212 : cluster [DBG] osdmap e165: 8 total, 8 up, 8 in 2024-09-12T23:40:03.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:02 smithi136 bash[22601]: cluster 2024-09-12T23:40:02.052489+0000 mgr.y (mgr.24446) 248 : cluster [DBG] pgmap v290: 121 pgs: 121 active+clean; 122 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.0 MiB/s rd, 1.8 MiB/s wr, 28 op/s 2024-09-12T23:40:03.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:02 smithi136 bash[32189]: cluster 2024-09-12T23:40:01.743905+0000 mon.a (mon.0) 1212 : cluster [DBG] osdmap e165: 8 total, 8 up, 8 in 2024-09-12T23:40:03.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:02 smithi136 bash[32189]: cluster 2024-09-12T23:40:02.052489+0000 mgr.y (mgr.24446) 248 : cluster [DBG] pgmap v290: 121 pgs: 121 active+clean; 122 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.0 MiB/s rd, 1.8 MiB/s wr, 28 op/s 2024-09-12T23:40:03.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:02 smithi181 bash[24776]: cluster 2024-09-12T23:40:01.743905+0000 mon.a (mon.0) 1212 : cluster [DBG] osdmap e165: 8 total, 8 up, 8 in 2024-09-12T23:40:03.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:02 smithi181 bash[24776]: cluster 2024-09-12T23:40:02.052489+0000 mgr.y (mgr.24446) 248 : cluster [DBG] pgmap v290: 121 pgs: 121 active+clean; 122 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.0 MiB/s rd, 1.8 MiB/s wr, 28 op/s 2024-09-12T23:40:03.745 INFO:tasks.rados.rados.0.smithi136.stdout:580: done (0 left) 2024-09-12T23:40:03.745 INFO:tasks.rados.rados.0.smithi136.stdout:581: snap_create 2024-09-12T23:40:04.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:03 smithi136 bash[22601]: cluster 2024-09-12T23:40:02.748001+0000 mon.a (mon.0) 1213 : cluster [DBG] osdmap e166: 8 total, 8 up, 8 in 2024-09-12T23:40:04.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:03 smithi136 bash[22601]: cluster 2024-09-12T23:40:03.749744+0000 mon.a (mon.0) 1214 : cluster [DBG] osdmap e167: 8 total, 8 up, 8 in 2024-09-12T23:40:04.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:03 smithi136 bash[32189]: cluster 2024-09-12T23:40:02.748001+0000 mon.a (mon.0) 1213 : cluster [DBG] osdmap e166: 8 total, 8 up, 8 in 2024-09-12T23:40:04.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:03 smithi136 bash[32189]: cluster 2024-09-12T23:40:03.749744+0000 mon.a (mon.0) 1214 : cluster [DBG] osdmap e167: 8 total, 8 up, 8 in 2024-09-12T23:40:04.093 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:03 smithi181 bash[24776]: cluster 2024-09-12T23:40:02.748001+0000 mon.a (mon.0) 1213 : cluster [DBG] osdmap e166: 8 total, 8 up, 8 in 2024-09-12T23:40:04.093 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:03 smithi181 bash[24776]: cluster 2024-09-12T23:40:03.749744+0000 mon.a (mon.0) 1214 : cluster [DBG] osdmap e167: 8 total, 8 up, 8 in 2024-09-12T23:40:04.748 INFO:tasks.rados.rados.0.smithi136.stdout:581: done (0 left) 2024-09-12T23:40:04.748 INFO:tasks.rados.rados.0.smithi136.stdout:582: read oid 47 snap -1 2024-09-12T23:40:04.748 INFO:tasks.rados.rados.0.smithi136.stdout:582: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-12T23:40:04.748 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 43 current snap is 51 2024-09-12T23:40:04.748 INFO:tasks.rados.rados.0.smithi136.stdout:583: seq_num 202 ranges {106496=589824,696320=401408} 2024-09-12T23:40:04.759 INFO:tasks.rados.rados.0.smithi136.stdout:583: writing smithi13667081-43 from 106496 to 696320 tid 1 2024-09-12T23:40:04.766 INFO:tasks.rados.rados.0.smithi136.stdout:583: writing smithi13667081-43 from 696320 to 1097728 tid 2 2024-09-12T23:40:04.766 INFO:tasks.rados.rados.0.smithi136.stdout:584: rollback oid 29 current snap is 51 2024-09-12T23:40:04.767 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 29 to 46 2024-09-12T23:40:04.767 INFO:tasks.rados.rados.0.smithi136.stdout:585: delete oid 16 current snap is 51 2024-09-12T23:40:04.767 INFO:tasks.rados.rados.0.smithi136.stdout:582: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-12T23:40:04.774 INFO:tasks.rados.rados.0.smithi136.stdout:584: finishing rollback tid 1 to smithi13667081-29 2024-09-12T23:40:04.774 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 29 v 0 (ObjNum 1649 snap 0 seq_num 4243020752) dirty dne 2024-09-12T23:40:04.780 INFO:tasks.rados.rados.0.smithi136.stdout:583: finishing write tid 1 to smithi13667081-43 2024-09-12T23:40:04.781 INFO:tasks.rados.rados.0.smithi136.stdout:583: finishing write tid 2 to smithi13667081-43 2024-09-12T23:40:04.783 INFO:tasks.rados.rados.0.smithi136.stdout:583: finishing write tid 3 to smithi13667081-43 2024-09-12T23:40:04.785 INFO:tasks.rados.rados.0.smithi136.stdout:582: done (3 left) 2024-09-12T23:40:04.785 INFO:tasks.rados.rados.0.smithi136.stdout:584: done (2 left) 2024-09-12T23:40:04.785 INFO:tasks.rados.rados.0.smithi136.stdout:585: done (1 left) 2024-09-12T23:40:04.786 INFO:tasks.rados.rados.0.smithi136.stdout:586: read oid 38 snap -1 2024-09-12T23:40:04.786 INFO:tasks.rados.rados.0.smithi136.stdout:586: expect (ObjNum 190 snap 46 seq_num 190) 2024-09-12T23:40:04.786 INFO:tasks.rados.rados.0.smithi136.stdout:587: copy_from oid 28 from oid 17 current snap is 51 2024-09-12T23:40:04.786 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 15 current snap is 51 2024-09-12T23:40:04.786 INFO:tasks.rados.rados.0.smithi136.stdout:588: seq_num 204 ranges {3170304=622592,3792896=401408} 2024-09-12T23:40:04.831 INFO:tasks.rados.rados.0.smithi136.stdout:588: writing smithi13667081-15 from 3170304 to 3792896 tid 1 2024-09-12T23:40:04.834 INFO:tasks.rados.rados.0.smithi136.stdout:588: writing smithi13667081-15 from 3792896 to 4194304 tid 2 2024-09-12T23:40:04.834 INFO:tasks.rados.rados.0.smithi136.stdout:589: delete oid 35 current snap is 51 2024-09-12T23:40:04.835 INFO:tasks.rados.rados.0.smithi136.stdout:583: finishing write tid 4 to smithi13667081-43 2024-09-12T23:40:04.835 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 43 v 299 (ObjNum 202 snap 51 seq_num 202) dirty exists 2024-09-12T23:40:04.835 INFO:tasks.rados.rados.0.smithi136.stdout:583: left oid 43 (ObjNum 202 snap 51 seq_num 202) 2024-09-12T23:40:04.835 INFO:tasks.rados.rados.0.smithi136.stdout:586: expect (ObjNum 190 snap 46 seq_num 190) 2024-09-12T23:40:04.893 INFO:tasks.rados.rados.0.smithi136.stdout:587: finishing copy_from racing read to smithi13667081-28 2024-09-12T23:40:04.893 INFO:tasks.rados.rados.0.smithi136.stdout:587: finishing copy_from to smithi13667081-28 2024-09-12T23:40:04.893 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 28 v 220 (ObjNum 181 snap 46 seq_num 181) dirty exists 2024-09-12T23:40:04.893 INFO:tasks.rados.rados.0.smithi136.stdout:588: finishing write tid 1 to smithi13667081-15 2024-09-12T23:40:04.893 INFO:tasks.rados.rados.0.smithi136.stdout:588: finishing write tid 2 to smithi13667081-15 2024-09-12T23:40:04.893 INFO:tasks.rados.rados.0.smithi136.stdout:588: finishing write tid 3 to smithi13667081-15 2024-09-12T23:40:04.893 INFO:tasks.rados.rados.0.smithi136.stdout:588: finishing write tid 4 to smithi13667081-15 2024-09-12T23:40:04.894 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 15 v 364 (ObjNum 204 snap 51 seq_num 204) dirty exists 2024-09-12T23:40:04.894 INFO:tasks.rados.rados.0.smithi136.stdout:588: left oid 15 (ObjNum 204 snap 51 seq_num 204) 2024-09-12T23:40:04.894 INFO:tasks.rados.rados.0.smithi136.stdout:583: done (4 left) 2024-09-12T23:40:04.894 INFO:tasks.rados.rados.0.smithi136.stdout:586: done (3 left) 2024-09-12T23:40:04.894 INFO:tasks.rados.rados.0.smithi136.stdout:587: done (2 left) 2024-09-12T23:40:04.894 INFO:tasks.rados.rados.0.smithi136.stdout:588: done (1 left) 2024-09-12T23:40:04.894 INFO:tasks.rados.rados.0.smithi136.stdout:589: done (0 left) 2024-09-12T23:40:04.894 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 38 current snap is 51 2024-09-12T23:40:04.894 INFO:tasks.rados.rados.0.smithi136.stdout:590: seq_num 206 ranges {786432=655360,1441792=401408} 2024-09-12T23:40:04.910 INFO:tasks.rados.rados.0.smithi136.stdout:590: writing smithi13667081-38 from 786432 to 1441792 tid 1 2024-09-12T23:40:04.915 INFO:tasks.rados.rados.0.smithi136.stdout:590: writing smithi13667081-38 from 1441792 to 1843200 tid 2 2024-09-12T23:40:04.915 INFO:tasks.rados.rados.0.smithi136.stdout:591: setattr oid 26 current snap is 51 2024-09-12T23:40:04.918 INFO:tasks.rados.rados.0.smithi136.stdout:592: delete oid 43 current snap is 51 2024-09-12T23:40:04.920 INFO:tasks.rados.rados.0.smithi136.stdout:590: finishing write tid 1 to smithi13667081-38 2024-09-12T23:40:04.921 INFO:tasks.rados.rados.0.smithi136.stdout:590: finishing write tid 2 to smithi13667081-38 2024-09-12T23:40:04.922 INFO:tasks.rados.rados.0.smithi136.stdout:590: finishing write tid 3 to smithi13667081-38 2024-09-12T23:40:04.923 INFO:tasks.rados.rados.0.smithi136.stdout:592: done (2 left) 2024-09-12T23:40:04.923 INFO:tasks.rados.rados.0.smithi136.stdout:593: snap_remove snap 45 2024-09-12T23:40:05.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:04 smithi181 bash[24776]: cluster 2024-09-12T23:40:04.053869+0000 mgr.y (mgr.24446) 249 : cluster [DBG] pgmap v293: 121 pgs: 121 active+clean; 85 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 265 KiB/s rd, 702 KiB/s wr, 5 op/s 2024-09-12T23:40:05.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:04 smithi181 bash[24776]: cluster 2024-09-12T23:40:04.752635+0000 mon.a (mon.0) 1215 : cluster [DBG] osdmap e168: 8 total, 8 up, 8 in 2024-09-12T23:40:05.255 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:04 smithi136 bash[22601]: cluster 2024-09-12T23:40:04.053869+0000 mgr.y (mgr.24446) 249 : cluster [DBG] pgmap v293: 121 pgs: 121 active+clean; 85 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 265 KiB/s rd, 702 KiB/s wr, 5 op/s 2024-09-12T23:40:05.255 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:04 smithi136 bash[22601]: cluster 2024-09-12T23:40:04.752635+0000 mon.a (mon.0) 1215 : cluster [DBG] osdmap e168: 8 total, 8 up, 8 in 2024-09-12T23:40:05.255 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:04 smithi136 bash[32189]: cluster 2024-09-12T23:40:04.053869+0000 mgr.y (mgr.24446) 249 : cluster [DBG] pgmap v293: 121 pgs: 121 active+clean; 85 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 265 KiB/s rd, 702 KiB/s wr, 5 op/s 2024-09-12T23:40:05.255 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:04 smithi136 bash[32189]: cluster 2024-09-12T23:40:04.752635+0000 mon.a (mon.0) 1215 : cluster [DBG] osdmap e168: 8 total, 8 up, 8 in 2024-09-12T23:40:05.477 INFO:tasks.rados.rados.0.smithi136.stdout:593: done (2 left) 2024-09-12T23:40:05.478 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 33 current snap is 51 2024-09-12T23:40:05.478 INFO:tasks.rados.rados.0.smithi136.stdout:594: seq_num 208 ranges {2170880=688128,2859008=401408} 2024-09-12T23:40:05.519 INFO:tasks.rados.rados.0.smithi136.stdout:594: writing smithi13667081-33 from 2170880 to 2859008 tid 1 2024-09-12T23:40:05.525 INFO:tasks.rados.rados.0.smithi136.stdout:594: writing smithi13667081-33 from 2859008 to 3260416 tid 2 2024-09-12T23:40:05.525 INFO:tasks.rados.rados.0.smithi136.stdout:595: snap_create 2024-09-12T23:40:05.525 INFO:tasks.rados.rados.0.smithi136.stdout:590: finishing write tid 4 to smithi13667081-38 2024-09-12T23:40:05.525 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 38 v 223 (ObjNum 206 snap 51 seq_num 206) dirty exists 2024-09-12T23:40:05.525 INFO:tasks.rados.rados.0.smithi136.stdout:590: left oid 38 (ObjNum 206 snap 51 seq_num 206) 2024-09-12T23:40:05.525 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 26 v 129 (ObjNum 160 snap 42 seq_num 160) dirty exists 2024-09-12T23:40:05.530 INFO:tasks.rados.rados.0.smithi136.stdout:594: finishing write tid 1 to smithi13667081-33 2024-09-12T23:40:05.530 INFO:tasks.rados.rados.0.smithi136.stdout:594: finishing write tid 2 to smithi13667081-33 2024-09-12T23:40:05.531 INFO:tasks.rados.rados.0.smithi136.stdout:594: finishing write tid 3 to smithi13667081-33 2024-09-12T23:40:05.533 INFO:tasks.rados.rados.0.smithi136.stdout:594: finishing write tid 4 to smithi13667081-33 2024-09-12T23:40:05.533 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 33 v 268 (ObjNum 208 snap 51 seq_num 208) dirty exists 2024-09-12T23:40:05.533 INFO:tasks.rados.rados.0.smithi136.stdout:594: left oid 33 (ObjNum 208 snap 51 seq_num 208) 2024-09-12T23:40:05.594 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:40:05 smithi181 bash[52367]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:40:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-12T23:40:06.479 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:06 smithi136 bash[22852]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:40:06] "GET /metrics HTTP/1.1" 200 240174 "" "Prometheus/2.43.0" 2024-09-12T23:40:06.488 INFO:tasks.rados.rados.0.smithi136.stdout:590: done (3 left) 2024-09-12T23:40:06.488 INFO:tasks.rados.rados.0.smithi136.stdout:591: done (2 left) 2024-09-12T23:40:06.488 INFO:tasks.rados.rados.0.smithi136.stdout:594: done (1 left) 2024-09-12T23:40:06.488 INFO:tasks.rados.rados.0.smithi136.stdout:595: done (0 left) 2024-09-12T23:40:06.488 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 27 current snap is 52 2024-09-12T23:40:06.488 INFO:tasks.rados.rados.0.smithi136.stdout:596: seq_num 209 ranges {442368=704512,1146880=401408} 2024-09-12T23:40:06.502 INFO:tasks.rados.rados.0.smithi136.stdout:596: writing smithi13667081-27 from 442368 to 1146880 tid 1 2024-09-12T23:40:06.507 INFO:tasks.rados.rados.0.smithi136.stdout:596: writing smithi13667081-27 from 1146880 to 1548288 tid 2 2024-09-12T23:40:06.507 INFO:tasks.rados.rados.0.smithi136.stdout:597: rollback oid 16 current snap is 52 2024-09-12T23:40:06.507 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 16 to 46 2024-09-12T23:40:06.508 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 32 current snap is 52 2024-09-12T23:40:06.508 INFO:tasks.rados.rados.0.smithi136.stdout:598: seq_num 210 ranges {0=720896,720896=401408} 2024-09-12T23:40:06.517 INFO:tasks.rados.rados.0.smithi136.stdout:598: writing smithi13667081-32 from 0 to 720896 tid 1 2024-09-12T23:40:06.524 INFO:tasks.rados.rados.0.smithi136.stdout:598: writing smithi13667081-32 from 720896 to 1122304 tid 2 2024-09-12T23:40:06.524 INFO:tasks.rados.rados.0.smithi136.stdout:599: snap_create 2024-09-12T23:40:06.524 INFO:tasks.rados.rados.0.smithi136.stdout:596: finishing write tid 1 to smithi13667081-27 2024-09-12T23:40:06.524 INFO:tasks.rados.rados.0.smithi136.stdout:596: finishing write tid 2 to smithi13667081-27 2024-09-12T23:40:06.524 INFO:tasks.rados.rados.0.smithi136.stdout:596: finishing write tid 3 to smithi13667081-27 2024-09-12T23:40:06.524 INFO:tasks.rados.rados.0.smithi136.stdout:596: finishing write tid 4 to smithi13667081-27 2024-09-12T23:40:06.524 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 27 v 306 (ObjNum 209 snap 52 seq_num 209) dirty exists 2024-09-12T23:40:06.524 INFO:tasks.rados.rados.0.smithi136.stdout:596: left oid 27 (ObjNum 209 snap 52 seq_num 209) 2024-09-12T23:40:06.524 INFO:tasks.rados.rados.0.smithi136.stdout:597: finishing rollback tid 1 to smithi13667081-16 2024-09-12T23:40:06.524 INFO:tasks.rados.rados.0.smithi136.stdout:597: finishing rollback tid 2 to smithi13667081-16 2024-09-12T23:40:06.524 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 16 v 314 (ObjNum 163 snap 43 seq_num 163) dirty exists 2024-09-12T23:40:06.527 INFO:tasks.rados.rados.0.smithi136.stdout:598: finishing write tid 1 to smithi13667081-32 2024-09-12T23:40:06.528 INFO:tasks.rados.rados.0.smithi136.stdout:598: finishing write tid 2 to smithi13667081-32 2024-09-12T23:40:06.528 INFO:tasks.rados.rados.0.smithi136.stdout:598: finishing write tid 3 to smithi13667081-32 2024-09-12T23:40:06.529 INFO:tasks.rados.rados.0.smithi136.stdout:598: finishing write tid 4 to smithi13667081-32 2024-09-12T23:40:06.530 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 32 v 115 (ObjNum 210 snap 52 seq_num 210) dirty exists 2024-09-12T23:40:06.530 INFO:tasks.rados.rados.0.smithi136.stdout:598: left oid 32 (ObjNum 210 snap 52 seq_num 210) 2024-09-12T23:40:06.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:06 smithi136 bash[22601]: cluster 2024-09-12T23:40:05.481350+0000 mon.a (mon.0) 1216 : cluster [DBG] osdmap e169: 8 total, 8 up, 8 in 2024-09-12T23:40:06.755 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:06 smithi136 bash[32189]: cluster 2024-09-12T23:40:05.481350+0000 mon.a (mon.0) 1216 : cluster [DBG] osdmap e169: 8 total, 8 up, 8 in 2024-09-12T23:40:06.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:06 smithi181 bash[24776]: cluster 2024-09-12T23:40:05.481350+0000 mon.a (mon.0) 1216 : cluster [DBG] osdmap e169: 8 total, 8 up, 8 in 2024-09-12T23:40:07.488 INFO:tasks.rados.rados.0.smithi136.stdout:596: done (3 left) 2024-09-12T23:40:07.488 INFO:tasks.rados.rados.0.smithi136.stdout:597: done (2 left) 2024-09-12T23:40:07.488 INFO:tasks.rados.rados.0.smithi136.stdout:598: done (1 left) 2024-09-12T23:40:07.488 INFO:tasks.rados.rados.0.smithi136.stdout:599: done (0 left) 2024-09-12T23:40:07.488 INFO:tasks.rados.rados.0.smithi136.stdout:600: setattr oid 7 current snap is 53 2024-09-12T23:40:07.490 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 19 current snap is 53 2024-09-12T23:40:07.490 INFO:tasks.rados.rados.0.smithi136.stdout:601: seq_num 211 ranges {2613248=737280,3350528=401408} 2024-09-12T23:40:07.515 INFO:tasks.rados.rados.0.smithi136.stdout:601: writing smithi13667081-19 from 2613248 to 3350528 tid 1 2024-09-12T23:40:07.518 INFO:tasks.rados.rados.0.smithi136.stdout:601: writing smithi13667081-19 from 3350528 to 3751936 tid 2 2024-09-12T23:40:07.519 INFO:tasks.rados.rados.0.smithi136.stdout:602: copy_from oid 13 from oid 8 current snap is 53 2024-09-12T23:40:07.519 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 7 v 105 (ObjNum 49 snap 0 seq_num 1953066355) dirty exists 2024-09-12T23:40:07.519 INFO:tasks.rados.rados.0.smithi136.stdout:600: done (2 left) 2024-09-12T23:40:07.519 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 41 current snap is 53 2024-09-12T23:40:07.519 INFO:tasks.rados.rados.0.smithi136.stdout:603: seq_num 212 ranges {466944=753664,1220608=401408} 2024-09-12T23:40:07.528 INFO:tasks.rados.rados.0.smithi136.stdout:603: writing smithi13667081-41 from 466944 to 1220608 tid 1 2024-09-12T23:40:07.532 INFO:tasks.rados.rados.0.smithi136.stdout:603: writing smithi13667081-41 from 1220608 to 1622016 tid 2 2024-09-12T23:40:07.532 INFO:tasks.rados.rados.0.smithi136.stdout:604: read oid 20 snap 46 2024-09-12T23:40:07.532 INFO:tasks.rados.rados.0.smithi136.stdout:604: expect (ObjNum 177 snap 45 seq_num 177) 2024-09-12T23:40:07.532 INFO:tasks.rados.rados.0.smithi136.stdout:605: snap_remove snap 49 2024-09-12T23:40:07.755 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:07 smithi136 bash[22852]: ignoring --setuser ceph since I am not root 2024-09-12T23:40:07.755 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:07 smithi136 bash[22852]: ignoring --setgroup ceph since I am not root 2024-09-12T23:40:07.755 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:07 smithi136 bash[22852]: debug 2024-09-12T23:40:07.605+0000 7fc5030f2640 1 -- 172.21.15.136:0/1360186464 <== mon.0 v2:172.21.15.136:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55951c036340 con 0x55951c043000 2024-09-12T23:40:07.755 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:07 smithi136 bash[22852]: debug 2024-09-12T23:40:07.713+0000 7fc50594cec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-12T23:40:07.755 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:07 smithi136 bash[32189]: cluster 2024-09-12T23:40:06.054556+0000 mgr.y (mgr.24446) 250 : cluster [DBG] pgmap v296: 121 pgs: 121 active+clean; 85 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 265 KiB/s rd, 702 KiB/s wr, 5 op/s 2024-09-12T23:40:07.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:07 smithi136 bash[32189]: cluster 2024-09-12T23:40:06.491397+0000 mon.a (mon.0) 1217 : cluster [DBG] osdmap e170: 8 total, 8 up, 8 in 2024-09-12T23:40:07.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:07 smithi136 bash[32189]: audit 2024-09-12T23:40:07.418902+0000 mon.a (mon.0) 1218 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:40:07.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:07 smithi136 bash[32189]: audit 2024-09-12T23:40:07.428893+0000 mon.a (mon.0) 1219 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:40:07.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:07 smithi136 bash[32189]: audit 2024-09-12T23:40:07.430926+0000 mon.a (mon.0) 1220 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:40:07.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:07 smithi136 bash[32189]: audit 2024-09-12T23:40:07.432733+0000 mon.a (mon.0) 1221 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:40:07.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:07 smithi136 bash[32189]: audit 2024-09-12T23:40:07.442779+0000 mon.a (mon.0) 1222 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:40:07.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:07 smithi136 bash[22601]: cluster 2024-09-12T23:40:06.054556+0000 mgr.y (mgr.24446) 250 : cluster [DBG] pgmap v296: 121 pgs: 121 active+clean; 85 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 265 KiB/s rd, 702 KiB/s wr, 5 op/s 2024-09-12T23:40:07.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:07 smithi136 bash[22601]: cluster 2024-09-12T23:40:06.491397+0000 mon.a (mon.0) 1217 : cluster [DBG] osdmap e170: 8 total, 8 up, 8 in 2024-09-12T23:40:07.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:07 smithi136 bash[22601]: audit 2024-09-12T23:40:07.418902+0000 mon.a (mon.0) 1218 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:40:07.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:07 smithi136 bash[22601]: audit 2024-09-12T23:40:07.428893+0000 mon.a (mon.0) 1219 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:40:07.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:07 smithi136 bash[22601]: audit 2024-09-12T23:40:07.430926+0000 mon.a (mon.0) 1220 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:40:07.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:07 smithi136 bash[22601]: audit 2024-09-12T23:40:07.432733+0000 mon.a (mon.0) 1221 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:40:07.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:07 smithi136 bash[22601]: audit 2024-09-12T23:40:07.442779+0000 mon.a (mon.0) 1222 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:40:07.787 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:40:07 smithi181 bash[52367]: [12/Sep/2024:23:40:07] ENGINE Bus STOPPING 2024-09-12T23:40:07.788 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:40:07 smithi181 bash[52367]: [12/Sep/2024:23:40:07] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-12T23:40:07.788 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:40:07 smithi181 bash[52367]: [12/Sep/2024:23:40:07] ENGINE Bus STOPPED 2024-09-12T23:40:07.788 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:40:07 smithi181 bash[52367]: [12/Sep/2024:23:40:07] ENGINE Bus STARTING 2024-09-12T23:40:07.788 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:07 smithi181 bash[24776]: cluster 2024-09-12T23:40:06.054556+0000 mgr.y (mgr.24446) 250 : cluster [DBG] pgmap v296: 121 pgs: 121 active+clean; 85 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 265 KiB/s rd, 702 KiB/s wr, 5 op/s 2024-09-12T23:40:07.788 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:07 smithi181 bash[24776]: cluster 2024-09-12T23:40:06.491397+0000 mon.a (mon.0) 1217 : cluster [DBG] osdmap e170: 8 total, 8 up, 8 in 2024-09-12T23:40:07.788 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:07 smithi181 bash[24776]: audit 2024-09-12T23:40:07.418902+0000 mon.a (mon.0) 1218 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:40:07.788 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:07 smithi181 bash[24776]: audit 2024-09-12T23:40:07.428893+0000 mon.a (mon.0) 1219 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:40:07.788 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:07 smithi181 bash[24776]: audit 2024-09-12T23:40:07.430926+0000 mon.a (mon.0) 1220 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:40:07.789 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:07 smithi181 bash[24776]: audit 2024-09-12T23:40:07.432733+0000 mon.a (mon.0) 1221 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:40:07.789 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:07 smithi181 bash[24776]: audit 2024-09-12T23:40:07.442779+0000 mon.a (mon.0) 1222 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' 2024-09-12T23:40:08.094 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:40:07 smithi181 bash[52367]: debug 2024-09-12T23:40:07.791+0000 7fc90b43c640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-12T23:40:08.094 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:40:07 smithi181 bash[52367]: debug 2024-09-12T23:40:07.791+0000 7fc90b43c640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-12T23:40:08.094 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:40:07 smithi181 bash[52367]: debug 2024-09-12T23:40:07.791+0000 7fc90b43c640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-12T23:40:08.094 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:40:07 smithi181 bash[52367]: debug 2024-09-12T23:40:07.791+0000 7fc90b43c640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-12T23:40:08.094 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:40:07 smithi181 bash[52367]: debug 2024-09-12T23:40:07.791+0000 7fc90b43c640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-12T23:40:08.094 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:40:07 smithi181 bash[52367]: debug 2024-09-12T23:40:07.791+0000 7fc908436640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-12T23:40:08.094 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:40:07 smithi181 bash[52367]: debug 2024-09-12T23:40:07.791+0000 7fc908436640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-12T23:40:08.094 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:40:07 smithi181 bash[52367]: debug 2024-09-12T23:40:07.791+0000 7fc908436640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-12T23:40:08.094 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:40:07 smithi181 bash[52367]: debug 2024-09-12T23:40:07.791+0000 7fc908436640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-12T23:40:08.095 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:40:07 smithi181 bash[52367]: debug 2024-09-12T23:40:07.791+0000 7fc908436640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-12T23:40:08.095 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:40:07 smithi181 bash[52367]: [12/Sep/2024:23:40:07] ENGINE Serving on http://:::9283 2024-09-12T23:40:08.095 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:40:07 smithi181 bash[52367]: [12/Sep/2024:23:40:07] ENGINE Bus STARTED 2024-09-12T23:40:08.543 INFO:tasks.rados.rados.0.smithi136.stdout:601: finishing write tid 1 to smithi13667081-19 2024-09-12T23:40:08.543 INFO:tasks.rados.rados.0.smithi136.stdout:601: finishing write tid 2 to smithi13667081-19 2024-09-12T23:40:08.543 INFO:tasks.rados.rados.0.smithi136.stdout:601: finishing write tid 3 to smithi13667081-19 2024-09-12T23:40:08.543 INFO:tasks.rados.rados.0.smithi136.stdout:601: finishing write tid 4 to smithi13667081-19 2024-09-12T23:40:08.543 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 19 v 256 (ObjNum 211 snap 53 seq_num 211) dirty exists 2024-09-12T23:40:08.543 INFO:tasks.rados.rados.0.smithi136.stdout:601: left oid 19 (ObjNum 211 snap 53 seq_num 211) 2024-09-12T23:40:08.543 INFO:tasks.rados.rados.0.smithi136.stdout:602: finishing copy_from racing read to smithi13667081-13 2024-09-12T23:40:08.544 INFO:tasks.rados.rados.0.smithi136.stdout:602: finishing copy_from to smithi13667081-13 2024-09-12T23:40:08.544 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 13 v 134 (ObjNum 164 snap 43 seq_num 164) dirty exists 2024-09-12T23:40:08.544 INFO:tasks.rados.rados.0.smithi136.stdout:604: expect (ObjNum 177 snap 45 seq_num 177) 2024-09-12T23:40:08.606 INFO:tasks.rados.rados.0.smithi136.stdout:603: finishing write tid 1 to smithi13667081-41 2024-09-12T23:40:08.607 INFO:tasks.rados.rados.0.smithi136.stdout:603: finishing write tid 2 to smithi13667081-41 2024-09-12T23:40:08.607 INFO:tasks.rados.rados.0.smithi136.stdout:603: finishing write tid 3 to smithi13667081-41 2024-09-12T23:40:08.607 INFO:tasks.rados.rados.0.smithi136.stdout:603: finishing write tid 4 to smithi13667081-41 2024-09-12T23:40:08.607 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 41 v 272 (ObjNum 212 snap 53 seq_num 212) dirty exists 2024-09-12T23:40:08.607 INFO:tasks.rados.rados.0.smithi136.stdout:603: left oid 41 (ObjNum 212 snap 53 seq_num 212) 2024-09-12T23:40:08.607 INFO:tasks.rados.rados.0.smithi136.stdout:601: done (4 left) 2024-09-12T23:40:08.607 INFO:tasks.rados.rados.0.smithi136.stdout:602: done (3 left) 2024-09-12T23:40:08.607 INFO:tasks.rados.rados.0.smithi136.stdout:603: done (2 left) 2024-09-12T23:40:08.607 INFO:tasks.rados.rados.0.smithi136.stdout:604: done (1 left) 2024-09-12T23:40:08.607 INFO:tasks.rados.rados.0.smithi136.stdout:605: done (0 left) 2024-09-12T23:40:08.607 INFO:tasks.rados.rados.0.smithi136.stdout:606: read oid 46 snap -1 2024-09-12T23:40:08.607 INFO:tasks.rados.rados.0.smithi136.stdout:606: expect deleted 2024-09-12T23:40:08.608 INFO:tasks.rados.rados.0.smithi136.stdout:607: rollback oid 42 current snap is 53 2024-09-12T23:40:08.608 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 42 to 47 2024-09-12T23:40:08.608 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 26 current snap is 53 2024-09-12T23:40:08.608 INFO:tasks.rados.rados.0.smithi136.stdout:608: seq_num 213 ranges {1957888=770048,2727936=401408} 2024-09-12T23:40:08.627 INFO:tasks.rados.rados.0.smithi136.stdout:608: writing smithi13667081-26 from 1957888 to 2727936 tid 1 2024-09-12T23:40:08.634 INFO:tasks.rados.rados.0.smithi136.stdout:608: writing smithi13667081-26 from 2727936 to 3129344 tid 2 2024-09-12T23:40:08.634 INFO:tasks.rados.rados.0.smithi136.stdout:609: copy_from oid 41 from oid 40 current snap is 53 2024-09-12T23:40:08.634 INFO:tasks.rados.rados.0.smithi136.stdout:607: finishing rollback tid 0 to smithi13667081-42 2024-09-12T23:40:08.634 INFO:tasks.rados.rados.0.smithi136.stdout:607: finishing rollback tid 1 to smithi13667081-42 2024-09-12T23:40:08.634 INFO:tasks.rados.rados.0.smithi136.stdout:607: finishing rollback tid 2 to smithi13667081-42 2024-09-12T23:40:08.634 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 42 v 143 (ObjNum 125 snap 32 seq_num 125) dirty exists 2024-09-12T23:40:08.634 INFO:tasks.rados.rados.0.smithi136.stdout:608: finishing write tid 1 to smithi13667081-26 2024-09-12T23:40:08.634 INFO:tasks.rados.rados.0.smithi136.stdout:606: done (3 left) 2024-09-12T23:40:08.634 INFO:tasks.rados.rados.0.smithi136.stdout:607: done (2 left) 2024-09-12T23:40:08.635 INFO:tasks.rados.rados.0.smithi136.stdout:610: rollback oid 16 current snap is 53 2024-09-12T23:40:08.635 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 16 to 51 2024-09-12T23:40:08.635 INFO:tasks.rados.rados.0.smithi136.stdout:611: copy_from oid 18 from oid 23 current snap is 53 2024-09-12T23:40:08.635 INFO:tasks.rados.rados.0.smithi136.stdout:612: rollback oid 17 current snap is 53 2024-09-12T23:40:08.635 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 17 to 50 2024-09-12T23:40:08.635 INFO:tasks.rados.rados.0.smithi136.stdout:613: delete oid 6 current snap is 53 2024-09-12T23:40:08.635 INFO:tasks.rados.rados.0.smithi136.stdout:613: done (5 left) 2024-09-12T23:40:08.636 INFO:tasks.rados.rados.0.smithi136.stdout:614: delete oid 35 current snap is 53 2024-09-12T23:40:08.638 INFO:tasks.rados.rados.0.smithi136.stdout:610: finishing rollback tid 0 to smithi13667081-16 2024-09-12T23:40:08.639 INFO:tasks.rados.rados.0.smithi136.stdout:610: finishing rollback tid 1 to smithi13667081-16 2024-09-12T23:40:08.639 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 16 v 323 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-12T23:40:08.639 INFO:tasks.rados.rados.0.smithi136.stdout:608: finishing write tid 2 to smithi13667081-26 2024-09-12T23:40:08.639 INFO:tasks.rados.rados.0.smithi136.stdout:608: finishing write tid 3 to smithi13667081-26 2024-09-12T23:40:08.641 INFO:tasks.rados.rados.0.smithi136.stdout:608: finishing write tid 4 to smithi13667081-26 2024-09-12T23:40:08.641 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 26 v 139 (ObjNum 213 snap 53 seq_num 213) dirty exists 2024-09-12T23:40:08.641 INFO:tasks.rados.rados.0.smithi136.stdout:608: left oid 26 (ObjNum 213 snap 53 seq_num 213) 2024-09-12T23:40:08.643 INFO:tasks.rados.rados.0.smithi136.stdout:612: finishing rollback tid 0 to smithi13667081-17 2024-09-12T23:40:08.643 INFO:tasks.rados.rados.0.smithi136.stdout:612: finishing rollback tid 1 to smithi13667081-17 2024-09-12T23:40:08.643 INFO:tasks.rados.rados.0.smithi136.stdout:612: finishing rollback tid 2 to smithi13667081-17 2024-09-12T23:40:08.644 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 17 v 312 (ObjNum 181 snap 46 seq_num 181) dirty exists 2024-09-12T23:40:08.647 INFO:tasks.rados.rados.0.smithi136.stdout:608: done (5 left) 2024-09-12T23:40:08.647 INFO:tasks.rados.rados.0.smithi136.stdout:610: done (4 left) 2024-09-12T23:40:08.647 INFO:tasks.rados.rados.0.smithi136.stdout:612: done (3 left) 2024-09-12T23:40:08.647 INFO:tasks.rados.rados.0.smithi136.stdout:614: done (2 left) 2024-09-12T23:40:08.647 INFO:tasks.rados.rados.0.smithi136.stdout:615: read oid 4 snap -1 2024-09-12T23:40:08.647 INFO:tasks.rados.rados.0.smithi136.stdout:615: expect (ObjNum 185 snap 46 seq_num 185) 2024-09-12T23:40:08.647 INFO:tasks.rados.rados.0.smithi136.stdout:616: snap_create 2024-09-12T23:40:08.649 INFO:tasks.rados.rados.0.smithi136.stdout:609: finishing copy_from racing read to smithi13667081-41 2024-09-12T23:40:08.650 INFO:tasks.rados.rados.0.smithi136.stdout:609: finishing copy_from to smithi13667081-41 2024-09-12T23:40:08.650 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 41 v 250 (ObjNum 196 snap 47 seq_num 196) dirty exists 2024-09-12T23:40:08.657 INFO:tasks.rados.rados.0.smithi136.stdout:615: expect (ObjNum 185 snap 46 seq_num 185) 2024-09-12T23:40:08.725 INFO:tasks.rados.rados.0.smithi136.stdout:611: finishing copy_from racing read to smithi13667081-18 2024-09-12T23:40:08.725 INFO:tasks.rados.rados.0.smithi136.stdout:611: finishing copy_from to smithi13667081-18 2024-09-12T23:40:08.725 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 18 v 263 (ObjNum 198 snap 48 seq_num 198) dirty exists 2024-09-12T23:40:08.755 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[32189]: cluster 2024-09-12T23:40:07.491724+0000 mon.a (mon.0) 1223 : cluster [DBG] osdmap e171: 8 total, 8 up, 8 in 2024-09-12T23:40:08.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[32189]: audit 2024-09-12T23:40:07.516069+0000 mon.a (mon.0) 1224 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:40:08.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[32189]: audit 2024-09-12T23:40:07.531189+0000 mon.a (mon.0) 1225 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-12T23:40:08.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[32189]: cluster 2024-09-12T23:40:07.532392+0000 mon.a (mon.0) 1226 : cluster [INF] Activating manager daemon x 2024-09-12T23:40:08.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[32189]: cluster 2024-09-12T23:40:07.550085+0000 mon.a (mon.0) 1227 : cluster [DBG] osdmap e172: 8 total, 8 up, 8 in 2024-09-12T23:40:08.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[32189]: audit 2024-09-12T23:40:07.550430+0000 mon.b (mon.1) 40 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:40:08.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[32189]: audit 2024-09-12T23:40:07.550650+0000 mon.b (mon.1) 41 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:40:08.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[32189]: audit 2024-09-12T23:40:07.551038+0000 mon.b (mon.1) 42 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:40:08.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[32189]: audit 2024-09-12T23:40:07.551794+0000 mon.b (mon.1) 43 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-12T23:40:08.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[32189]: audit 2024-09-12T23:40:07.552225+0000 mon.b (mon.1) 44 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-12T23:40:08.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[32189]: audit 2024-09-12T23:40:07.552605+0000 mon.b (mon.1) 45 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-12T23:40:08.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[32189]: audit 2024-09-12T23:40:07.552922+0000 mon.b (mon.1) 46 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-12T23:40:08.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[32189]: audit 2024-09-12T23:40:07.553356+0000 mon.b (mon.1) 47 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-12T23:40:08.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[32189]: audit 2024-09-12T23:40:07.553542+0000 mon.a (mon.0) 1228 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-09-12T23:40:08.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[32189]: cluster 2024-09-12T23:40:07.553652+0000 mon.a (mon.0) 1229 : cluster [DBG] mgrmap e24: x(active, starting, since 0.0212551s) 2024-09-12T23:40:08.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[32189]: audit 2024-09-12T23:40:07.553681+0000 mon.b (mon.1) 48 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-12T23:40:08.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[32189]: audit 2024-09-12T23:40:07.553997+0000 mon.b (mon.1) 49 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-12T23:40:08.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[32189]: audit 2024-09-12T23:40:07.554290+0000 mon.b (mon.1) 50 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-12T23:40:08.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[32189]: audit 2024-09-12T23:40:07.554605+0000 mon.b (mon.1) 51 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-12T23:40:08.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[32189]: audit 2024-09-12T23:40:07.554895+0000 mon.b (mon.1) 52 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-12T23:40:08.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[32189]: audit 2024-09-12T23:40:07.555264+0000 mon.b (mon.1) 53 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-12T23:40:08.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[32189]: audit 2024-09-12T23:40:07.555655+0000 mon.b (mon.1) 54 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-12T23:40:08.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[32189]: audit 2024-09-12T23:40:07.556022+0000 mon.b (mon.1) 55 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-12T23:40:08.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[32189]: audit 2024-09-12T23:40:07.556686+0000 mon.b (mon.1) 56 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-12T23:40:08.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[32189]: cluster 2024-09-12T23:40:07.679429+0000 mon.a (mon.0) 1230 : cluster [INF] Manager daemon x is now available 2024-09-12T23:40:08.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[32189]: audit 2024-09-12T23:40:07.704891+0000 mon.b (mon.1) 57 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:40:08.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[32189]: audit 2024-09-12T23:40:07.734618+0000 mon.b (mon.1) 58 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:40:08.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[32189]: audit 2024-09-12T23:40:07.745534+0000 mon.b (mon.1) 59 : audit [INF] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-12T23:40:08.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[32189]: audit 2024-09-12T23:40:07.746146+0000 mon.a (mon.0) 1231 : audit [INF] from='mgr.24517 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-12T23:40:08.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[32189]: audit 2024-09-12T23:40:07.761812+0000 mon.b (mon.1) 60 : audit [INF] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-12T23:40:08.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[32189]: audit 2024-09-12T23:40:07.762537+0000 mon.a (mon.0) 1232 : audit [INF] from='mgr.24517 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-12T23:40:08.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[32189]: audit 2024-09-12T23:40:08.417560+0000 mon.a (mon.0) 1233 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:08.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[32189]: audit 2024-09-12T23:40:08.441319+0000 mon.a (mon.0) 1234 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:08.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[22601]: cluster 2024-09-12T23:40:07.491724+0000 mon.a (mon.0) 1223 : cluster [DBG] osdmap e171: 8 total, 8 up, 8 in 2024-09-12T23:40:08.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[22601]: audit 2024-09-12T23:40:07.516069+0000 mon.a (mon.0) 1224 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:40:08.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[22601]: audit 2024-09-12T23:40:07.531189+0000 mon.a (mon.0) 1225 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-12T23:40:08.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[22601]: cluster 2024-09-12T23:40:07.532392+0000 mon.a (mon.0) 1226 : cluster [INF] Activating manager daemon x 2024-09-12T23:40:08.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[22601]: cluster 2024-09-12T23:40:07.550085+0000 mon.a (mon.0) 1227 : cluster [DBG] osdmap e172: 8 total, 8 up, 8 in 2024-09-12T23:40:08.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[22601]: audit 2024-09-12T23:40:07.550430+0000 mon.b (mon.1) 40 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:40:08.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[22601]: audit 2024-09-12T23:40:07.550650+0000 mon.b (mon.1) 41 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:40:08.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[22601]: audit 2024-09-12T23:40:07.551038+0000 mon.b (mon.1) 42 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:40:08.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[22601]: audit 2024-09-12T23:40:07.551794+0000 mon.b (mon.1) 43 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-12T23:40:08.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[22601]: audit 2024-09-12T23:40:07.552225+0000 mon.b (mon.1) 44 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-12T23:40:08.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[22601]: audit 2024-09-12T23:40:07.552605+0000 mon.b (mon.1) 45 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-12T23:40:08.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[22601]: audit 2024-09-12T23:40:07.552922+0000 mon.b (mon.1) 46 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-12T23:40:08.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[22601]: audit 2024-09-12T23:40:07.553356+0000 mon.b (mon.1) 47 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-12T23:40:08.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[22601]: audit 2024-09-12T23:40:07.553542+0000 mon.a (mon.0) 1228 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-09-12T23:40:08.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[22601]: cluster 2024-09-12T23:40:07.553652+0000 mon.a (mon.0) 1229 : cluster [DBG] mgrmap e24: x(active, starting, since 0.0212551s) 2024-09-12T23:40:08.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[22601]: audit 2024-09-12T23:40:07.553681+0000 mon.b (mon.1) 48 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-12T23:40:08.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[22601]: audit 2024-09-12T23:40:07.553997+0000 mon.b (mon.1) 49 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-12T23:40:08.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[22601]: audit 2024-09-12T23:40:07.554290+0000 mon.b (mon.1) 50 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-12T23:40:08.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[22601]: audit 2024-09-12T23:40:07.554605+0000 mon.b (mon.1) 51 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-12T23:40:08.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[22601]: audit 2024-09-12T23:40:07.554895+0000 mon.b (mon.1) 52 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-12T23:40:08.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[22601]: audit 2024-09-12T23:40:07.555264+0000 mon.b (mon.1) 53 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-12T23:40:08.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[22601]: audit 2024-09-12T23:40:07.555655+0000 mon.b (mon.1) 54 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-12T23:40:08.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[22601]: audit 2024-09-12T23:40:07.556022+0000 mon.b (mon.1) 55 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-12T23:40:08.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[22601]: audit 2024-09-12T23:40:07.556686+0000 mon.b (mon.1) 56 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-12T23:40:08.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[22601]: cluster 2024-09-12T23:40:07.679429+0000 mon.a (mon.0) 1230 : cluster [INF] Manager daemon x is now available 2024-09-12T23:40:08.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[22601]: audit 2024-09-12T23:40:07.704891+0000 mon.b (mon.1) 57 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:40:08.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[22601]: audit 2024-09-12T23:40:07.734618+0000 mon.b (mon.1) 58 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:40:08.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[22601]: audit 2024-09-12T23:40:07.745534+0000 mon.b (mon.1) 59 : audit [INF] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-12T23:40:08.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[22601]: audit 2024-09-12T23:40:07.746146+0000 mon.a (mon.0) 1231 : audit [INF] from='mgr.24517 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-12T23:40:08.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[22601]: audit 2024-09-12T23:40:07.761812+0000 mon.b (mon.1) 60 : audit [INF] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-12T23:40:08.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[22601]: audit 2024-09-12T23:40:07.762537+0000 mon.a (mon.0) 1232 : audit [INF] from='mgr.24517 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-12T23:40:08.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[22601]: audit 2024-09-12T23:40:08.417560+0000 mon.a (mon.0) 1233 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:08.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[22601]: audit 2024-09-12T23:40:08.441319+0000 mon.a (mon.0) 1234 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:08.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:08 smithi181 bash[24776]: cluster 2024-09-12T23:40:07.491724+0000 mon.a (mon.0) 1223 : cluster [DBG] osdmap e171: 8 total, 8 up, 8 in 2024-09-12T23:40:08.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:08 smithi181 bash[24776]: audit 2024-09-12T23:40:07.516069+0000 mon.a (mon.0) 1224 : audit [DBG] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:40:08.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:08 smithi181 bash[24776]: audit 2024-09-12T23:40:07.531189+0000 mon.a (mon.0) 1225 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-12T23:40:08.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:08 smithi181 bash[24776]: cluster 2024-09-12T23:40:07.532392+0000 mon.a (mon.0) 1226 : cluster [INF] Activating manager daemon x 2024-09-12T23:40:08.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:08 smithi181 bash[24776]: cluster 2024-09-12T23:40:07.550085+0000 mon.a (mon.0) 1227 : cluster [DBG] osdmap e172: 8 total, 8 up, 8 in 2024-09-12T23:40:08.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:08 smithi181 bash[24776]: audit 2024-09-12T23:40:07.550430+0000 mon.b (mon.1) 40 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:40:08.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:08 smithi181 bash[24776]: audit 2024-09-12T23:40:07.550650+0000 mon.b (mon.1) 41 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:40:08.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:08 smithi181 bash[24776]: audit 2024-09-12T23:40:07.551038+0000 mon.b (mon.1) 42 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:40:08.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:08 smithi181 bash[24776]: audit 2024-09-12T23:40:07.551794+0000 mon.b (mon.1) 43 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-12T23:40:08.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:08 smithi181 bash[24776]: audit 2024-09-12T23:40:07.552225+0000 mon.b (mon.1) 44 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-12T23:40:08.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:08 smithi181 bash[24776]: audit 2024-09-12T23:40:07.552605+0000 mon.b (mon.1) 45 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-12T23:40:08.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:08 smithi181 bash[24776]: audit 2024-09-12T23:40:07.552922+0000 mon.b (mon.1) 46 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-12T23:40:08.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:08 smithi181 bash[24776]: audit 2024-09-12T23:40:07.553356+0000 mon.b (mon.1) 47 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-12T23:40:08.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:08 smithi181 bash[24776]: audit 2024-09-12T23:40:07.553542+0000 mon.a (mon.0) 1228 : audit [INF] from='mgr.24446 172.21.15.136:0/159905798' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-09-12T23:40:08.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:08 smithi181 bash[24776]: cluster 2024-09-12T23:40:07.553652+0000 mon.a (mon.0) 1229 : cluster [DBG] mgrmap e24: x(active, starting, since 0.0212551s) 2024-09-12T23:40:08.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:08 smithi181 bash[24776]: audit 2024-09-12T23:40:07.553681+0000 mon.b (mon.1) 48 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-12T23:40:08.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:08 smithi181 bash[24776]: audit 2024-09-12T23:40:07.553997+0000 mon.b (mon.1) 49 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-12T23:40:08.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:08 smithi181 bash[24776]: audit 2024-09-12T23:40:07.554290+0000 mon.b (mon.1) 50 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-12T23:40:08.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:08 smithi181 bash[24776]: audit 2024-09-12T23:40:07.554605+0000 mon.b (mon.1) 51 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-12T23:40:08.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:08 smithi181 bash[24776]: audit 2024-09-12T23:40:07.554895+0000 mon.b (mon.1) 52 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-12T23:40:08.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:08 smithi181 bash[24776]: audit 2024-09-12T23:40:07.555264+0000 mon.b (mon.1) 53 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-12T23:40:08.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:08 smithi181 bash[24776]: audit 2024-09-12T23:40:07.555655+0000 mon.b (mon.1) 54 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-12T23:40:08.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:08 smithi181 bash[24776]: audit 2024-09-12T23:40:07.556022+0000 mon.b (mon.1) 55 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-12T23:40:08.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:08 smithi181 bash[24776]: audit 2024-09-12T23:40:07.556686+0000 mon.b (mon.1) 56 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-12T23:40:08.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:08 smithi181 bash[24776]: cluster 2024-09-12T23:40:07.679429+0000 mon.a (mon.0) 1230 : cluster [INF] Manager daemon x is now available 2024-09-12T23:40:08.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:08 smithi181 bash[24776]: audit 2024-09-12T23:40:07.704891+0000 mon.b (mon.1) 57 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:40:08.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:08 smithi181 bash[24776]: audit 2024-09-12T23:40:07.734618+0000 mon.b (mon.1) 58 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:40:08.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:08 smithi181 bash[24776]: audit 2024-09-12T23:40:07.745534+0000 mon.b (mon.1) 59 : audit [INF] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-12T23:40:08.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:08 smithi181 bash[24776]: audit 2024-09-12T23:40:07.746146+0000 mon.a (mon.0) 1231 : audit [INF] from='mgr.24517 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-12T23:40:08.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:08 smithi181 bash[24776]: audit 2024-09-12T23:40:07.761812+0000 mon.b (mon.1) 60 : audit [INF] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-12T23:40:08.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:08 smithi181 bash[24776]: audit 2024-09-12T23:40:07.762537+0000 mon.a (mon.0) 1232 : audit [INF] from='mgr.24517 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-12T23:40:08.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:08 smithi181 bash[24776]: audit 2024-09-12T23:40:08.417560+0000 mon.a (mon.0) 1233 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:08.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:08 smithi181 bash[24776]: audit 2024-09-12T23:40:08.441319+0000 mon.a (mon.0) 1234 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:09.254 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:08 smithi136 bash[22852]: debug 2024-09-12T23:40:08.981+0000 7fc50594cec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-12T23:40:09.255 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:09 smithi136 bash[22852]: debug 2024-09-12T23:40:09.045+0000 7fc50594cec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-12T23:40:09.255 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:09 smithi136 bash[22852]: debug 2024-09-12T23:40:09.201+0000 7fc50594cec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-12T23:40:09.562 INFO:tasks.rados.rados.0.smithi136.stdout:609: done (3 left) 2024-09-12T23:40:09.562 INFO:tasks.rados.rados.0.smithi136.stdout:611: done (2 left) 2024-09-12T23:40:09.562 INFO:tasks.rados.rados.0.smithi136.stdout:615: done (1 left) 2024-09-12T23:40:09.562 INFO:tasks.rados.rados.0.smithi136.stdout:616: done (0 left) 2024-09-12T23:40:09.562 INFO:tasks.rados.rados.0.smithi136.stdout:617: read oid 35 snap 51 2024-09-12T23:40:09.562 INFO:tasks.rados.rados.0.smithi136.stdout:617: expect deleted 2024-09-12T23:40:09.563 INFO:tasks.rados.rados.0.smithi136.stdout:618: copy_from oid 45 from oid 23 current snap is 54 2024-09-12T23:40:09.563 INFO:tasks.rados.rados.0.smithi136.stdout:619: delete oid 1 current snap is 54 2024-09-12T23:40:09.565 INFO:tasks.rados.rados.0.smithi136.stdout:617: done (2 left) 2024-09-12T23:40:09.565 INFO:tasks.rados.rados.0.smithi136.stdout:619: done (1 left) 2024-09-12T23:40:09.565 INFO:tasks.rados.rados.0.smithi136.stdout:620: snap_create 2024-09-12T23:40:09.595 INFO:tasks.rados.rados.0.smithi136.stdout:618: finishing copy_from racing read to smithi13667081-45 2024-09-12T23:40:09.596 INFO:tasks.rados.rados.0.smithi136.stdout:618: finishing copy_from to smithi13667081-45 2024-09-12T23:40:09.597 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 45 v 263 (ObjNum 198 snap 48 seq_num 198) dirty exists 2024-09-12T23:40:09.755 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:09 smithi136 bash[32189]: cephadm 2024-09-12T23:40:08.230665+0000 mgr.x (mgr.24517) 1 : cephadm [INF] Deploying cephadm binary to smithi181 2024-09-12T23:40:09.755 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:09 smithi136 bash[32189]: cephadm 2024-09-12T23:40:08.285007+0000 mgr.x (mgr.24517) 2 : cephadm [INF] Deploying cephadm binary to smithi136 2024-09-12T23:40:09.755 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:09 smithi136 bash[32189]: cluster 2024-09-12T23:40:08.546402+0000 mon.a (mon.0) 1235 : cluster [DBG] osdmap e173: 8 total, 8 up, 8 in 2024-09-12T23:40:09.755 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:09 smithi136 bash[32189]: cluster 2024-09-12T23:40:08.563655+0000 mon.a (mon.0) 1236 : cluster [DBG] mgrmap e25: x(active, since 1.03123s) 2024-09-12T23:40:09.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:09 smithi136 bash[22601]: cephadm 2024-09-12T23:40:08.230665+0000 mgr.x (mgr.24517) 1 : cephadm [INF] Deploying cephadm binary to smithi181 2024-09-12T23:40:09.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:09 smithi136 bash[22601]: cephadm 2024-09-12T23:40:08.285007+0000 mgr.x (mgr.24517) 2 : cephadm [INF] Deploying cephadm binary to smithi136 2024-09-12T23:40:09.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:09 smithi136 bash[22601]: cluster 2024-09-12T23:40:08.546402+0000 mon.a (mon.0) 1235 : cluster [DBG] osdmap e173: 8 total, 8 up, 8 in 2024-09-12T23:40:09.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:09 smithi136 bash[22601]: cluster 2024-09-12T23:40:08.563655+0000 mon.a (mon.0) 1236 : cluster [DBG] mgrmap e25: x(active, since 1.03123s) 2024-09-12T23:40:09.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:09 smithi181 bash[24776]: cephadm 2024-09-12T23:40:08.230665+0000 mgr.x (mgr.24517) 1 : cephadm [INF] Deploying cephadm binary to smithi181 2024-09-12T23:40:09.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:09 smithi181 bash[24776]: cephadm 2024-09-12T23:40:08.285007+0000 mgr.x (mgr.24517) 2 : cephadm [INF] Deploying cephadm binary to smithi136 2024-09-12T23:40:09.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:09 smithi181 bash[24776]: cluster 2024-09-12T23:40:08.546402+0000 mon.a (mon.0) 1235 : cluster [DBG] osdmap e173: 8 total, 8 up, 8 in 2024-09-12T23:40:09.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:09 smithi181 bash[24776]: cluster 2024-09-12T23:40:08.563655+0000 mon.a (mon.0) 1236 : cluster [DBG] mgrmap e25: x(active, since 1.03123s) 2024-09-12T23:40:10.141 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:09 smithi136 bash[22852]: debug 2024-09-12T23:40:09.845+0000 7fc50594cec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-12T23:40:10.141 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:09 smithi136 bash[22852]: debug 2024-09-12T23:40:09.913+0000 7fc50594cec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-12T23:40:10.141 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:09 smithi136 bash[22852]: debug 2024-09-12T23:40:09.985+0000 7fc50594cec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-12T23:40:10.141 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:10 smithi136 bash[22852]: debug 2024-09-12T23:40:10.141+0000 7fc50594cec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-12T23:40:10.478 INFO:tasks.rados.rados.0.smithi136.stdout:618: done (1 left) 2024-09-12T23:40:10.478 INFO:tasks.rados.rados.0.smithi136.stdout:620: done (0 left) 2024-09-12T23:40:10.478 INFO:tasks.rados.rados.0.smithi136.stdout:621: read oid 45 snap -1 2024-09-12T23:40:10.478 INFO:tasks.rados.rados.0.smithi136.stdout:621: expect (ObjNum 198 snap 48 seq_num 198) 2024-09-12T23:40:10.478 INFO:tasks.rados.rados.0.smithi136.stdout:622: setattr oid 1 current snap is 55 2024-09-12T23:40:10.481 INFO:tasks.rados.rados.0.smithi136.stdout:623: rmattr oid 34 current snap is 55 2024-09-12T23:40:10.481 INFO:tasks.rados.rados.0.smithi136.stdout:623: done (2 left) 2024-09-12T23:40:10.481 INFO:tasks.rados.rados.0.smithi136.stdout:624: read oid 10 snap -1 2024-09-12T23:40:10.481 INFO:tasks.rados.rados.0.smithi136.stdout:624: expect deleted 2024-09-12T23:40:10.481 INFO:tasks.rados.rados.0.smithi136.stdout:625: rollback oid 46 current snap is 55 2024-09-12T23:40:10.481 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 46 to 46 2024-09-12T23:40:10.481 INFO:tasks.rados.rados.0.smithi136.stdout:626: copy_from oid 25 from oid 44 current snap is 55 2024-09-12T23:40:10.481 INFO:tasks.rados.rados.0.smithi136.stdout:627: rollback oid 41 current snap is 55 2024-09-12T23:40:10.481 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 41 to 51 2024-09-12T23:40:10.481 INFO:tasks.rados.rados.0.smithi136.stdout:628: rollback oid 50 current snap is 55 2024-09-12T23:40:10.481 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 50 to 47 2024-09-12T23:40:10.482 INFO:tasks.rados.rados.0.smithi136.stdout:629: read oid 5 snap -1 2024-09-12T23:40:10.482 INFO:tasks.rados.rados.0.smithi136.stdout:629: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-12T23:40:10.482 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 36 current snap is 55 2024-09-12T23:40:10.482 INFO:tasks.rados.rados.0.smithi136.stdout:630: seq_num 217 ranges {3399680=442368,3842048=696320,4538368=106496} 2024-09-12T23:40:10.501 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:10 smithi136 bash[22852]: debug 2024-09-12T23:40:10.217+0000 7fc50594cec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-12T23:40:10.501 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:10 smithi136 bash[22852]: debug 2024-09-12T23:40:10.285+0000 7fc50594cec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-12T23:40:10.516 INFO:tasks.rados.rados.0.smithi136.stdout:630: writing smithi13667081-36 from 3399680 to 3842048 tid 1 2024-09-12T23:40:10.522 INFO:tasks.rados.rados.0.smithi136.stdout:630: writing smithi13667081-36 from 3842048 to 4538368 tid 2 2024-09-12T23:40:10.524 INFO:tasks.rados.rados.0.smithi136.stdout:630: writing smithi13667081-36 from 4538368 to 4644864 tid 3 2024-09-12T23:40:10.524 INFO:tasks.rados.rados.0.smithi136.stdout:631: rollback oid 49 current snap is 55 2024-09-12T23:40:10.524 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 49 to 47 2024-09-12T23:40:10.524 INFO:tasks.rados.rados.0.smithi136.stdout:625: finishing rollback tid 1 to smithi13667081-46 2024-09-12T23:40:10.524 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 46 v 0 (ObjNum 49 snap 0 seq_num 2) dirty dne 2024-09-12T23:40:10.524 INFO:tasks.rados.rados.0.smithi136.stdout:628: finishing rollback tid 0 to smithi13667081-50 2024-09-12T23:40:10.524 INFO:tasks.rados.rados.0.smithi136.stdout:628: finishing rollback tid 1 to smithi13667081-50 2024-09-12T23:40:10.524 INFO:tasks.rados.rados.0.smithi136.stdout:626: finishing copy_from to smithi13667081-25 2024-09-12T23:40:10.524 INFO:tasks.rados.rados.0.smithi136.stdout:626: got expected ENOENT (src dne) 2024-09-12T23:40:10.524 INFO:tasks.rados.rados.0.smithi136.stdout:626: finishing copy_from racing read to smithi13667081-25 2024-09-12T23:40:10.525 INFO:tasks.rados.rados.0.smithi136.stdout:624: done (9 left) 2024-09-12T23:40:10.525 INFO:tasks.rados.rados.0.smithi136.stdout:625: done (8 left) 2024-09-12T23:40:10.525 INFO:tasks.rados.rados.0.smithi136.stdout:626: done (7 left) 2024-09-12T23:40:10.525 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 28 current snap is 55 2024-09-12T23:40:10.525 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 1 v 145 (ObjNum 33 snap 0 seq_num 4243276896) dirty exists 2024-09-12T23:40:10.525 INFO:tasks.rados.rados.0.smithi136.stdout:627: finishing rollback tid 0 to smithi13667081-41 2024-09-12T23:40:10.525 INFO:tasks.rados.rados.0.smithi136.stdout:628: finishing rollback tid 2 to smithi13667081-50 2024-09-12T23:40:10.525 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 50 v 235 (ObjNum 49 snap 0 seq_num 49) dirty exists 2024-09-12T23:40:10.525 INFO:tasks.rados.rados.0.smithi136.stdout:627: finishing rollback tid 1 to smithi13667081-41 2024-09-12T23:40:10.525 INFO:tasks.rados.rados.0.smithi136.stdout:627: finishing rollback tid 2 to smithi13667081-41 2024-09-12T23:40:10.525 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 41 v 281 (ObjNum 171 snap 43 seq_num 171) dirty exists 2024-09-12T23:40:10.525 INFO:tasks.rados.rados.0.smithi136.stdout:632: seq_num 218 ranges {1105920=458752,1564672=581632,2146304=221184} 2024-09-12T23:40:10.540 INFO:tasks.rados.rados.0.smithi136.stdout:632: writing smithi13667081-28 from 1105920 to 1564672 tid 1 2024-09-12T23:40:10.546 INFO:tasks.rados.rados.0.smithi136.stdout:632: writing smithi13667081-28 from 1564672 to 2146304 tid 2 2024-09-12T23:40:10.549 INFO:tasks.rados.rados.0.smithi136.stdout:632: writing smithi13667081-28 from 2146304 to 2367488 tid 3 2024-09-12T23:40:10.549 INFO:tasks.rados.rados.0.smithi136.stdout:622: done (7 left) 2024-09-12T23:40:10.549 INFO:tasks.rados.rados.0.smithi136.stdout:627: done (6 left) 2024-09-12T23:40:10.549 INFO:tasks.rados.rados.0.smithi136.stdout:628: done (5 left) 2024-09-12T23:40:10.549 INFO:tasks.rados.rados.0.smithi136.stdout:633: rollback oid 23 current snap is 55 2024-09-12T23:40:10.549 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 23 to 47 2024-09-12T23:40:10.549 INFO:tasks.rados.rados.0.smithi136.stdout:629: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-12T23:40:10.554 INFO:tasks.rados.rados.0.smithi136.stdout:621: expect (ObjNum 198 snap 48 seq_num 198) 2024-09-12T23:40:10.710 INFO:tasks.rados.rados.0.smithi136.stdout:630: finishing write tid 1 to smithi13667081-36 2024-09-12T23:40:10.710 INFO:tasks.rados.rados.0.smithi136.stdout:630: finishing write tid 2 to smithi13667081-36 2024-09-12T23:40:10.710 INFO:tasks.rados.rados.0.smithi136.stdout:630: finishing write tid 3 to smithi13667081-36 2024-09-12T23:40:10.711 INFO:tasks.rados.rados.0.smithi136.stdout:630: finishing write tid 4 to smithi13667081-36 2024-09-12T23:40:10.711 INFO:tasks.rados.rados.0.smithi136.stdout:630: finishing write tid 5 to smithi13667081-36 2024-09-12T23:40:10.711 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 36 v 267 (ObjNum 217 snap 55 seq_num 217) dirty exists 2024-09-12T23:40:10.711 INFO:tasks.rados.rados.0.smithi136.stdout:630: left oid 36 (ObjNum 217 snap 55 seq_num 217) 2024-09-12T23:40:10.711 INFO:tasks.rados.rados.0.smithi136.stdout:631: finishing rollback tid 0 to smithi13667081-49 2024-09-12T23:40:10.711 INFO:tasks.rados.rados.0.smithi136.stdout:631: finishing rollback tid 1 to smithi13667081-49 2024-09-12T23:40:10.711 INFO:tasks.rados.rados.0.smithi136.stdout:631: finishing rollback tid 2 to smithi13667081-49 2024-09-12T23:40:10.711 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 49 v 378 (ObjNum 142 snap 36 seq_num 142) dirty exists 2024-09-12T23:40:10.711 INFO:tasks.rados.rados.0.smithi136.stdout:632: finishing write tid 1 to smithi13667081-28 2024-09-12T23:40:10.711 INFO:tasks.rados.rados.0.smithi136.stdout:633: finishing rollback tid 0 to smithi13667081-23 2024-09-12T23:40:10.711 INFO:tasks.rados.rados.0.smithi136.stdout:632: finishing write tid 2 to smithi13667081-28 2024-09-12T23:40:10.711 INFO:tasks.rados.rados.0.smithi136.stdout:633: finishing rollback tid 1 to smithi13667081-23 2024-09-12T23:40:10.711 INFO:tasks.rados.rados.0.smithi136.stdout:632: finishing write tid 3 to smithi13667081-28 2024-09-12T23:40:10.711 INFO:tasks.rados.rados.0.smithi136.stdout:633: finishing rollback tid 2 to smithi13667081-23 2024-09-12T23:40:10.712 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 23 v 317 (ObjNum 74 snap 12 seq_num 74) dirty exists 2024-09-12T23:40:10.712 INFO:tasks.rados.rados.0.smithi136.stdout:621: done (5 left) 2024-09-12T23:40:10.712 INFO:tasks.rados.rados.0.smithi136.stdout:629: done (4 left) 2024-09-12T23:40:10.712 INFO:tasks.rados.rados.0.smithi136.stdout:630: done (3 left) 2024-09-12T23:40:10.712 INFO:tasks.rados.rados.0.smithi136.stdout:631: done (2 left) 2024-09-12T23:40:10.712 INFO:tasks.rados.rados.0.smithi136.stdout:633: done (1 left) 2024-09-12T23:40:10.712 INFO:tasks.rados.rados.0.smithi136.stdout:634: rollback oid 25 current snap is 55 2024-09-12T23:40:10.712 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 25 to 51 2024-09-12T23:40:10.712 INFO:tasks.rados.rados.0.smithi136.stdout:632: finishing write tid 4 to smithi13667081-28 2024-09-12T23:40:10.712 INFO:tasks.rados.rados.0.smithi136.stdout:632: finishing write tid 5 to smithi13667081-28 2024-09-12T23:40:10.712 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 28 v 328 (ObjNum 218 snap 55 seq_num 218) dirty exists 2024-09-12T23:40:10.712 INFO:tasks.rados.rados.0.smithi136.stdout:632: left oid 28 (ObjNum 218 snap 55 seq_num 218) 2024-09-12T23:40:10.713 INFO:tasks.rados.rados.0.smithi136.stdout:632: done (1 left) 2024-09-12T23:40:10.713 INFO:tasks.rados.rados.0.smithi136.stdout:635: read oid 37 snap 51 2024-09-12T23:40:10.713 INFO:tasks.rados.rados.0.smithi136.stdout:635: expect (ObjNum 134 snap 35 seq_num 134) 2024-09-12T23:40:10.713 INFO:tasks.rados.rados.0.smithi136.stdout:636: copy_from oid 4 from oid 44 current snap is 55 2024-09-12T23:40:10.713 INFO:tasks.rados.rados.0.smithi136.stdout:637: snap_remove snap 47 2024-09-12T23:40:10.755 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:10 smithi136 bash[22852]: debug 2024-09-12T23:40:10.521+0000 7fc50594cec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-12T23:40:10.755 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:10 smithi136 bash[22852]: debug 2024-09-12T23:40:10.597+0000 7fc50594cec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-12T23:40:10.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:10 smithi136 bash[22601]: cluster 2024-09-12T23:40:08.592775+0000 mgr.x (mgr.24517) 3 : cluster [DBG] pgmap v4: 121 pgs: 121 active+clean; 92 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:40:10.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:10 smithi136 bash[22601]: cluster 2024-09-12T23:40:09.566025+0000 mon.a (mon.0) 1237 : cluster [DBG] osdmap e174: 8 total, 8 up, 8 in 2024-09-12T23:40:10.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:10 smithi136 bash[22601]: cluster 2024-09-12T23:40:09.566506+0000 mon.a (mon.0) 1238 : cluster [DBG] mgrmap e26: x(active, since 2s) 2024-09-12T23:40:10.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:10 smithi136 bash[22601]: cluster 2024-09-12T23:40:10.482633+0000 mon.a (mon.0) 1239 : cluster [DBG] osdmap e175: 8 total, 8 up, 8 in 2024-09-12T23:40:10.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:10 smithi136 bash[32189]: cluster 2024-09-12T23:40:08.592775+0000 mgr.x (mgr.24517) 3 : cluster [DBG] pgmap v4: 121 pgs: 121 active+clean; 92 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:40:10.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:10 smithi136 bash[32189]: cluster 2024-09-12T23:40:09.566025+0000 mon.a (mon.0) 1237 : cluster [DBG] osdmap e174: 8 total, 8 up, 8 in 2024-09-12T23:40:10.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:10 smithi136 bash[32189]: cluster 2024-09-12T23:40:09.566506+0000 mon.a (mon.0) 1238 : cluster [DBG] mgrmap e26: x(active, since 2s) 2024-09-12T23:40:10.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:10 smithi136 bash[32189]: cluster 2024-09-12T23:40:10.482633+0000 mon.a (mon.0) 1239 : cluster [DBG] osdmap e175: 8 total, 8 up, 8 in 2024-09-12T23:40:10.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:10 smithi181 bash[24776]: cluster 2024-09-12T23:40:08.592775+0000 mgr.x (mgr.24517) 3 : cluster [DBG] pgmap v4: 121 pgs: 121 active+clean; 92 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:40:10.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:10 smithi181 bash[24776]: cluster 2024-09-12T23:40:09.566025+0000 mon.a (mon.0) 1237 : cluster [DBG] osdmap e174: 8 total, 8 up, 8 in 2024-09-12T23:40:10.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:10 smithi181 bash[24776]: cluster 2024-09-12T23:40:09.566506+0000 mon.a (mon.0) 1238 : cluster [DBG] mgrmap e26: x(active, since 2s) 2024-09-12T23:40:10.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:10 smithi181 bash[24776]: cluster 2024-09-12T23:40:10.482633+0000 mon.a (mon.0) 1239 : cluster [DBG] osdmap e175: 8 total, 8 up, 8 in 2024-09-12T23:40:10.845 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:40:10 smithi181 bash[52367]: debug 2024-09-12T23:40:10.387+0000 7fc8f1408640 -1 log_channel(cephadm) log [ERR] : [12/Sep/2024:23:40:10] ENGINE Error in HTTPServer.serve 2024-09-12T23:40:10.845 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:40:10 smithi181 bash[52367]: Traceback (most recent call last): 2024-09-12T23:40:10.845 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:40:10 smithi181 bash[52367]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-12T23:40:10.845 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:40:10 smithi181 bash[52367]: self._connections.run(self.expiration_interval) 2024-09-12T23:40:10.845 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:40:10 smithi181 bash[52367]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-12T23:40:10.845 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:40:10 smithi181 bash[52367]: self._run(expiration_interval) 2024-09-12T23:40:10.845 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:40:10 smithi181 bash[52367]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-12T23:40:10.845 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:40:10 smithi181 bash[52367]: new_conn = self._from_server_socket(self.server.socket) 2024-09-12T23:40:10.845 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:40:10 smithi181 bash[52367]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-12T23:40:10.845 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:40:10 smithi181 bash[52367]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-12T23:40:10.845 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:40:10 smithi181 bash[52367]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-12T23:40:10.845 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:40:10 smithi181 bash[52367]: s = self.context.wrap_socket( 2024-09-12T23:40:10.845 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:40:10 smithi181 bash[52367]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-12T23:40:10.846 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:40:10 smithi181 bash[52367]: return self.sslsocket_class._create( 2024-09-12T23:40:10.846 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:40:10 smithi181 bash[52367]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-12T23:40:10.846 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:40:10 smithi181 bash[52367]: self.do_handshake() 2024-09-12T23:40:10.846 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:40:10 smithi181 bash[52367]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-12T23:40:10.846 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:40:10 smithi181 bash[52367]: self._sslobj.do_handshake() 2024-09-12T23:40:10.846 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:40:10 smithi181 bash[52367]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-12T23:40:11.075 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:10 smithi136 bash[22852]: debug 2024-09-12T23:40:10.769+0000 7fc50594cec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-12T23:40:11.075 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:10 smithi136 bash[22852]: debug 2024-09-12T23:40:10.845+0000 7fc50594cec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-12T23:40:11.464 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:11 smithi136 bash[22852]: debug 2024-09-12T23:40:11.073+0000 7fc50594cec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-12T23:40:11.464 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:11 smithi136 bash[22852]: debug 2024-09-12T23:40:11.233+0000 7fc50594cec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-12T23:40:11.464 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:11 smithi136 bash[22852]: debug 2024-09-12T23:40:11.313+0000 7fc50594cec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-12T23:40:11.464 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:11 smithi136 bash[22852]: debug 2024-09-12T23:40:11.465+0000 7fc50594cec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-12T23:40:11.566 INFO:tasks.rados.rados.0.smithi136.stdout:636: finishing copy_from to smithi13667081-4 2024-09-12T23:40:11.566 INFO:tasks.rados.rados.0.smithi136.stdout:636: got expected ENOENT (src dne) 2024-09-12T23:40:11.566 INFO:tasks.rados.rados.0.smithi136.stdout:636: finishing copy_from racing read to smithi13667081-4 2024-09-12T23:40:11.566 INFO:tasks.rados.rados.0.smithi136.stdout:634: finishing rollback tid 0 to smithi13667081-25 2024-09-12T23:40:11.567 INFO:tasks.rados.rados.0.smithi136.stdout:634: finishing rollback tid 1 to smithi13667081-25 2024-09-12T23:40:11.567 INFO:tasks.rados.rados.0.smithi136.stdout:634: finishing rollback tid 2 to smithi13667081-25 2024-09-12T23:40:11.567 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 25 v 74 (ObjNum 144 snap 36 seq_num 144) dirty exists 2024-09-12T23:40:11.567 INFO:tasks.rados.rados.0.smithi136.stdout:635: expect (ObjNum 134 snap 35 seq_num 134) 2024-09-12T23:40:11.755 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:11 smithi136 bash[32189]: cluster 2024-09-12T23:40:09.556039+0000 mgr.x (mgr.24517) 4 : cluster [DBG] pgmap v5: 121 pgs: 121 active+clean; 92 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:40:11.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:11 smithi136 bash[32189]: cephadm 2024-09-12T23:40:10.267213+0000 mgr.x (mgr.24517) 5 : cephadm [INF] [12/Sep/2024:23:40:10] ENGINE Bus STARTING 2024-09-12T23:40:11.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:11 smithi136 bash[32189]: cephadm 2024-09-12T23:40:10.388594+0000 mgr.x (mgr.24517) 6 : cephadm [INF] [12/Sep/2024:23:40:10] ENGINE Serving on https://172.21.15.181:7150 2024-09-12T23:40:11.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:11 smithi136 bash[32189]: cephadm 2024-09-12T23:40:10.391763+0000 mgr.x (mgr.24517) 7 : cephadm [ERR] [12/Sep/2024:23:40:10] ENGINE Error in HTTPServer.serve 2024-09-12T23:40:11.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:11 smithi136 bash[32189]: Traceback (most recent call last): 2024-09-12T23:40:11.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:11 smithi136 bash[32189]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-12T23:40:11.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:11 smithi136 bash[32189]: self._connections.run(self.expiration_interval) 2024-09-12T23:40:11.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:11 smithi136 bash[32189]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-12T23:40:11.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:11 smithi136 bash[32189]: self._run(expiration_interval) 2024-09-12T23:40:11.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:11 smithi136 bash[32189]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-12T23:40:11.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:11 smithi136 bash[32189]: new_conn = self._from_server_socket(self.server.socket) 2024-09-12T23:40:11.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:11 smithi136 bash[32189]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-12T23:40:11.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:11 smithi136 bash[32189]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-12T23:40:11.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:11 smithi136 bash[32189]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-12T23:40:11.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:11 smithi136 bash[32189]: s = self.context.wrap_socket( 2024-09-12T23:40:11.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:11 smithi136 bash[32189]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-12T23:40:11.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:11 smithi136 bash[32189]: return self.sslsocket_class._create( 2024-09-12T23:40:11.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:11 smithi136 bash[32189]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-12T23:40:11.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:11 smithi136 bash[32189]: self.do_handshake() 2024-09-12T23:40:11.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:11 smithi136 bash[32189]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-12T23:40:11.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:11 smithi136 bash[32189]: self._sslobj.do_handshake() 2024-09-12T23:40:11.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:11 smithi136 bash[32189]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-12T23:40:11.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:11 smithi136 bash[32189]: cephadm 2024-09-12T23:40:10.493402+0000 mgr.x (mgr.24517) 8 : cephadm [INF] [12/Sep/2024:23:40:10] ENGINE Serving on http://172.21.15.181:8765 2024-09-12T23:40:11.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:11 smithi136 bash[32189]: cephadm 2024-09-12T23:40:10.493571+0000 mgr.x (mgr.24517) 9 : cephadm [INF] [12/Sep/2024:23:40:10] ENGINE Bus STARTED 2024-09-12T23:40:11.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:11 smithi136 bash[22601]: cluster 2024-09-12T23:40:09.556039+0000 mgr.x (mgr.24517) 4 : cluster [DBG] pgmap v5: 121 pgs: 121 active+clean; 92 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:40:11.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:11 smithi136 bash[22601]: cephadm 2024-09-12T23:40:10.267213+0000 mgr.x (mgr.24517) 5 : cephadm [INF] [12/Sep/2024:23:40:10] ENGINE Bus STARTING 2024-09-12T23:40:11.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:11 smithi136 bash[22601]: cephadm 2024-09-12T23:40:10.388594+0000 mgr.x (mgr.24517) 6 : cephadm [INF] [12/Sep/2024:23:40:10] ENGINE Serving on https://172.21.15.181:7150 2024-09-12T23:40:11.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:11 smithi136 bash[22601]: cephadm 2024-09-12T23:40:10.391763+0000 mgr.x (mgr.24517) 7 : cephadm [ERR] [12/Sep/2024:23:40:10] ENGINE Error in HTTPServer.serve 2024-09-12T23:40:11.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:11 smithi136 bash[22601]: Traceback (most recent call last): 2024-09-12T23:40:11.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:11 smithi136 bash[22601]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-12T23:40:11.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:11 smithi136 bash[22601]: self._connections.run(self.expiration_interval) 2024-09-12T23:40:11.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:11 smithi136 bash[22601]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-12T23:40:11.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:11 smithi136 bash[22601]: self._run(expiration_interval) 2024-09-12T23:40:11.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:11 smithi136 bash[22601]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-12T23:40:11.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:11 smithi136 bash[22601]: new_conn = self._from_server_socket(self.server.socket) 2024-09-12T23:40:11.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:11 smithi136 bash[22601]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-12T23:40:11.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:11 smithi136 bash[22601]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-12T23:40:11.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:11 smithi136 bash[22601]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-12T23:40:11.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:11 smithi136 bash[22601]: s = self.context.wrap_socket( 2024-09-12T23:40:11.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:11 smithi136 bash[22601]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-12T23:40:11.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:11 smithi136 bash[22601]: return self.sslsocket_class._create( 2024-09-12T23:40:11.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:11 smithi136 bash[22601]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-12T23:40:11.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:11 smithi136 bash[22601]: self.do_handshake() 2024-09-12T23:40:11.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:11 smithi136 bash[22601]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-12T23:40:11.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:11 smithi136 bash[22601]: self._sslobj.do_handshake() 2024-09-12T23:40:11.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:11 smithi136 bash[22601]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-12T23:40:11.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:11 smithi136 bash[22601]: cephadm 2024-09-12T23:40:10.493402+0000 mgr.x (mgr.24517) 8 : cephadm [INF] [12/Sep/2024:23:40:10] ENGINE Serving on http://172.21.15.181:8765 2024-09-12T23:40:11.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:11 smithi136 bash[22601]: cephadm 2024-09-12T23:40:10.493571+0000 mgr.x (mgr.24517) 9 : cephadm [INF] [12/Sep/2024:23:40:10] ENGINE Bus STARTED 2024-09-12T23:40:11.832 INFO:tasks.rados.rados.0.smithi136.stdout:634: done (3 left) 2024-09-12T23:40:11.832 INFO:tasks.rados.rados.0.smithi136.stdout:635: done (2 left) 2024-09-12T23:40:11.832 INFO:tasks.rados.rados.0.smithi136.stdout:636: done (1 left) 2024-09-12T23:40:11.832 INFO:tasks.rados.rados.0.smithi136.stdout:637: done (0 left) 2024-09-12T23:40:11.832 INFO:tasks.rados.rados.0.smithi136.stdout:638: snap_remove snap 46 2024-09-12T23:40:11.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:11 smithi181 bash[24776]: cluster 2024-09-12T23:40:09.556039+0000 mgr.x (mgr.24517) 4 : cluster [DBG] pgmap v5: 121 pgs: 121 active+clean; 92 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:40:11.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:11 smithi181 bash[24776]: cephadm 2024-09-12T23:40:10.267213+0000 mgr.x (mgr.24517) 5 : cephadm [INF] [12/Sep/2024:23:40:10] ENGINE Bus STARTING 2024-09-12T23:40:11.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:11 smithi181 bash[24776]: cephadm 2024-09-12T23:40:10.388594+0000 mgr.x (mgr.24517) 6 : cephadm [INF] [12/Sep/2024:23:40:10] ENGINE Serving on https://172.21.15.181:7150 2024-09-12T23:40:11.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:11 smithi181 bash[24776]: cephadm 2024-09-12T23:40:10.391763+0000 mgr.x (mgr.24517) 7 : cephadm [ERR] [12/Sep/2024:23:40:10] ENGINE Error in HTTPServer.serve 2024-09-12T23:40:11.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:11 smithi181 bash[24776]: Traceback (most recent call last): 2024-09-12T23:40:11.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:11 smithi181 bash[24776]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-12T23:40:11.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:11 smithi181 bash[24776]: self._connections.run(self.expiration_interval) 2024-09-12T23:40:11.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:11 smithi181 bash[24776]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-12T23:40:11.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:11 smithi181 bash[24776]: self._run(expiration_interval) 2024-09-12T23:40:11.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:11 smithi181 bash[24776]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-12T23:40:11.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:11 smithi181 bash[24776]: new_conn = self._from_server_socket(self.server.socket) 2024-09-12T23:40:11.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:11 smithi181 bash[24776]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-12T23:40:11.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:11 smithi181 bash[24776]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-12T23:40:11.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:11 smithi181 bash[24776]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-12T23:40:11.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:11 smithi181 bash[24776]: s = self.context.wrap_socket( 2024-09-12T23:40:11.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:11 smithi181 bash[24776]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-12T23:40:11.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:11 smithi181 bash[24776]: return self.sslsocket_class._create( 2024-09-12T23:40:11.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:11 smithi181 bash[24776]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-12T23:40:11.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:11 smithi181 bash[24776]: self.do_handshake() 2024-09-12T23:40:11.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:11 smithi181 bash[24776]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-12T23:40:11.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:11 smithi181 bash[24776]: self._sslobj.do_handshake() 2024-09-12T23:40:11.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:11 smithi181 bash[24776]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-12T23:40:11.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:11 smithi181 bash[24776]: cephadm 2024-09-12T23:40:10.493402+0000 mgr.x (mgr.24517) 8 : cephadm [INF] [12/Sep/2024:23:40:10] ENGINE Serving on http://172.21.15.181:8765 2024-09-12T23:40:11.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:11 smithi181 bash[24776]: cephadm 2024-09-12T23:40:10.493571+0000 mgr.x (mgr.24517) 9 : cephadm [INF] [12/Sep/2024:23:40:10] ENGINE Bus STARTED 2024-09-12T23:40:12.536 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:12 smithi136 bash[22852]: debug 2024-09-12T23:40:12.265+0000 7fc50594cec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-12T23:40:12.536 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:12 smithi136 bash[22852]: debug 2024-09-12T23:40:12.341+0000 7fc50594cec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-12T23:40:12.582 INFO:tasks.rados.rados.0.smithi136.stdout:638: done (0 left) 2024-09-12T23:40:12.582 INFO:tasks.rados.rados.0.smithi136.stdout:639: rollback oid 28 current snap is 55 2024-09-12T23:40:12.582 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 28 to 54 2024-09-12T23:40:12.582 INFO:tasks.rados.rados.0.smithi136.stdout:640: delete oid 15 current snap is 55 2024-09-12T23:40:12.610 INFO:tasks.rados.rados.0.smithi136.stdout:640: done (1 left) 2024-09-12T23:40:12.611 INFO:tasks.rados.rados.0.smithi136.stdout:641: read oid 10 snap -1 2024-09-12T23:40:12.611 INFO:tasks.rados.rados.0.smithi136.stdout:641: expect deleted 2024-09-12T23:40:12.611 INFO:tasks.rados.rados.0.smithi136.stdout:642: setattr oid 34 current snap is 55 2024-09-12T23:40:12.611 INFO:tasks.rados.rados.0.smithi136.stdout:639: finishing rollback tid 0 to smithi13667081-28 2024-09-12T23:40:12.611 INFO:tasks.rados.rados.0.smithi136.stdout:639: finishing rollback tid 1 to smithi13667081-28 2024-09-12T23:40:12.611 INFO:tasks.rados.rados.0.smithi136.stdout:639: finishing rollback tid 2 to smithi13667081-28 2024-09-12T23:40:12.611 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 28 v 339 (ObjNum 181 snap 46 seq_num 181) dirty exists 2024-09-12T23:40:12.613 INFO:tasks.rados.rados.0.smithi136.stdout:639: done (2 left) 2024-09-12T23:40:12.614 INFO:tasks.rados.rados.0.smithi136.stdout:641: done (1 left) 2024-09-12T23:40:12.614 INFO:tasks.rados.rados.0.smithi136.stdout:643: rmattr oid 24 current snap is 55 2024-09-12T23:40:12.614 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 4 current snap is 55 2024-09-12T23:40:12.614 INFO:tasks.rados.rados.0.smithi136.stdout:644: seq_num 220 ranges {1622016=491520,2113536=753664,2867200=49152} 2024-09-12T23:40:12.640 INFO:tasks.rados.rados.0.smithi136.stdout:644: writing smithi13667081-4 from 1622016 to 2113536 tid 1 2024-09-12T23:40:12.648 INFO:tasks.rados.rados.0.smithi136.stdout:644: writing smithi13667081-4 from 2113536 to 2867200 tid 2 2024-09-12T23:40:12.649 INFO:tasks.rados.rados.0.smithi136.stdout:644: writing smithi13667081-4 from 2867200 to 2916352 tid 3 2024-09-12T23:40:12.649 INFO:tasks.rados.rados.0.smithi136.stdout:645: snap_create 2024-09-12T23:40:12.649 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 34 v 286 (ObjNum 449 snap 0 seq_num 4243285488) dirty exists 2024-09-12T23:40:12.649 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 24 v 145 (ObjNum 54 snap 4 seq_num 54) dirty exists 2024-09-12T23:40:12.650 INFO:tasks.rados.rados.0.smithi136.stdout:644: finishing write tid 1 to smithi13667081-4 2024-09-12T23:40:12.655 INFO:tasks.rados.rados.0.smithi136.stdout:644: finishing write tid 2 to smithi13667081-4 2024-09-12T23:40:12.656 INFO:tasks.rados.rados.0.smithi136.stdout:644: finishing write tid 3 to smithi13667081-4 2024-09-12T23:40:12.656 INFO:tasks.rados.rados.0.smithi136.stdout:644: finishing write tid 4 to smithi13667081-4 2024-09-12T23:40:12.658 INFO:tasks.rados.rados.0.smithi136.stdout:644: finishing write tid 5 to smithi13667081-4 2024-09-12T23:40:12.658 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 4 v 120 (ObjNum 220 snap 55 seq_num 220) dirty exists 2024-09-12T23:40:12.658 INFO:tasks.rados.rados.0.smithi136.stdout:644: left oid 4 (ObjNum 220 snap 55 seq_num 220) 2024-09-12T23:40:12.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:12 smithi181 bash[24776]: cluster 2024-09-12T23:40:11.556771+0000 mgr.x (mgr.24517) 10 : cluster [DBG] pgmap v8: 121 pgs: 121 active+clean; 92 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:40:12.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:12 smithi181 bash[24776]: cluster 2024-09-12T23:40:11.564581+0000 mon.a (mon.0) 1240 : cluster [DBG] osdmap e176: 8 total, 8 up, 8 in 2024-09-12T23:40:12.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:12 smithi181 bash[24776]: cluster 2024-09-12T23:40:11.577787+0000 mon.a (mon.0) 1241 : cluster [DBG] mgrmap e27: x(active, since 4s) 2024-09-12T23:40:12.948 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:12 smithi136 bash[22601]: cluster 2024-09-12T23:40:11.556771+0000 mgr.x (mgr.24517) 10 : cluster [DBG] pgmap v8: 121 pgs: 121 active+clean; 92 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:40:12.948 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:12 smithi136 bash[22601]: cluster 2024-09-12T23:40:11.564581+0000 mon.a (mon.0) 1240 : cluster [DBG] osdmap e176: 8 total, 8 up, 8 in 2024-09-12T23:40:12.948 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:12 smithi136 bash[22601]: cluster 2024-09-12T23:40:11.577787+0000 mon.a (mon.0) 1241 : cluster [DBG] mgrmap e27: x(active, since 4s) 2024-09-12T23:40:12.948 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:12 smithi136 bash[22852]: debug 2024-09-12T23:40:12.537+0000 7fc50594cec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-12T23:40:12.949 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:12 smithi136 bash[22852]: debug 2024-09-12T23:40:12.629+0000 7fc50594cec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-12T23:40:12.949 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:12 smithi136 bash[32189]: cluster 2024-09-12T23:40:11.556771+0000 mgr.x (mgr.24517) 10 : cluster [DBG] pgmap v8: 121 pgs: 121 active+clean; 92 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:40:12.949 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:12 smithi136 bash[32189]: cluster 2024-09-12T23:40:11.564581+0000 mon.a (mon.0) 1240 : cluster [DBG] osdmap e176: 8 total, 8 up, 8 in 2024-09-12T23:40:12.949 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:12 smithi136 bash[32189]: cluster 2024-09-12T23:40:11.577787+0000 mon.a (mon.0) 1241 : cluster [DBG] mgrmap e27: x(active, since 4s) 2024-09-12T23:40:13.255 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:12 smithi136 bash[22852]: debug 2024-09-12T23:40:12.949+0000 7fc50594cec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-12T23:40:13.255 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:13 smithi136 bash[22852]: debug 2024-09-12T23:40:13.025+0000 7fc50594cec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-12T23:40:13.255 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:13 smithi136 bash[22852]: [12/Sep/2024:23:40:13] ENGINE Bus STARTING 2024-09-12T23:40:13.255 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:13 smithi136 bash[22852]: CherryPy Checker: 2024-09-12T23:40:13.255 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:13 smithi136 bash[22852]: The Application mounted at '' has an empty config. 2024-09-12T23:40:13.255 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:13 smithi136 bash[22852]: [12/Sep/2024:23:40:13] ENGINE Serving on http://:::9283 2024-09-12T23:40:13.255 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:13 smithi136 bash[22852]: [12/Sep/2024:23:40:13] ENGINE Bus STARTED 2024-09-12T23:40:13.599 INFO:tasks.rados.rados.0.smithi136.stdout:642: done (3 left) 2024-09-12T23:40:13.599 INFO:tasks.rados.rados.0.smithi136.stdout:643: done (2 left) 2024-09-12T23:40:13.599 INFO:tasks.rados.rados.0.smithi136.stdout:644: done (1 left) 2024-09-12T23:40:13.599 INFO:tasks.rados.rados.0.smithi136.stdout:645: done (0 left) 2024-09-12T23:40:13.599 INFO:tasks.rados.rados.0.smithi136.stdout:646: snap_remove snap 50 2024-09-12T23:40:13.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:13 smithi181 bash[24776]: cluster 2024-09-12T23:40:12.581035+0000 mon.a (mon.0) 1242 : cluster [DBG] osdmap e177: 8 total, 8 up, 8 in 2024-09-12T23:40:13.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:13 smithi181 bash[24776]: cluster 2024-09-12T23:40:13.052286+0000 mon.a (mon.0) 1243 : cluster [DBG] Standby manager daemon y started 2024-09-12T23:40:13.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:13 smithi181 bash[24776]: audit 2024-09-12T23:40:13.055263+0000 mon.c (mon.2) 63 : audit [DBG] from='mgr.? 172.21.15.136:0/2285748149' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-12T23:40:13.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:13 smithi181 bash[24776]: audit 2024-09-12T23:40:13.057758+0000 mon.c (mon.2) 64 : audit [DBG] from='mgr.? 172.21.15.136:0/2285748149' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-12T23:40:13.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:13 smithi181 bash[24776]: audit 2024-09-12T23:40:13.059044+0000 mon.c (mon.2) 65 : audit [DBG] from='mgr.? 172.21.15.136:0/2285748149' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-12T23:40:13.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:13 smithi181 bash[24776]: audit 2024-09-12T23:40:13.059361+0000 mon.c (mon.2) 66 : audit [DBG] from='mgr.? 172.21.15.136:0/2285748149' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-12T23:40:14.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:13 smithi136 bash[32189]: cluster 2024-09-12T23:40:12.581035+0000 mon.a (mon.0) 1242 : cluster [DBG] osdmap e177: 8 total, 8 up, 8 in 2024-09-12T23:40:14.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:13 smithi136 bash[32189]: cluster 2024-09-12T23:40:13.052286+0000 mon.a (mon.0) 1243 : cluster [DBG] Standby manager daemon y started 2024-09-12T23:40:14.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:13 smithi136 bash[32189]: audit 2024-09-12T23:40:13.055263+0000 mon.c (mon.2) 63 : audit [DBG] from='mgr.? 172.21.15.136:0/2285748149' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-12T23:40:14.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:13 smithi136 bash[32189]: audit 2024-09-12T23:40:13.057758+0000 mon.c (mon.2) 64 : audit [DBG] from='mgr.? 172.21.15.136:0/2285748149' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-12T23:40:14.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:13 smithi136 bash[32189]: audit 2024-09-12T23:40:13.059044+0000 mon.c (mon.2) 65 : audit [DBG] from='mgr.? 172.21.15.136:0/2285748149' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-12T23:40:14.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:13 smithi136 bash[32189]: audit 2024-09-12T23:40:13.059361+0000 mon.c (mon.2) 66 : audit [DBG] from='mgr.? 172.21.15.136:0/2285748149' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-12T23:40:14.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:13 smithi136 bash[22601]: cluster 2024-09-12T23:40:12.581035+0000 mon.a (mon.0) 1242 : cluster [DBG] osdmap e177: 8 total, 8 up, 8 in 2024-09-12T23:40:14.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:13 smithi136 bash[22601]: cluster 2024-09-12T23:40:13.052286+0000 mon.a (mon.0) 1243 : cluster [DBG] Standby manager daemon y started 2024-09-12T23:40:14.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:13 smithi136 bash[22601]: audit 2024-09-12T23:40:13.055263+0000 mon.c (mon.2) 63 : audit [DBG] from='mgr.? 172.21.15.136:0/2285748149' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-12T23:40:14.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:13 smithi136 bash[22601]: audit 2024-09-12T23:40:13.057758+0000 mon.c (mon.2) 64 : audit [DBG] from='mgr.? 172.21.15.136:0/2285748149' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-12T23:40:14.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:13 smithi136 bash[22601]: audit 2024-09-12T23:40:13.059044+0000 mon.c (mon.2) 65 : audit [DBG] from='mgr.? 172.21.15.136:0/2285748149' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-12T23:40:14.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:13 smithi136 bash[22601]: audit 2024-09-12T23:40:13.059361+0000 mon.c (mon.2) 66 : audit [DBG] from='mgr.? 172.21.15.136:0/2285748149' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-12T23:40:14.611 INFO:tasks.rados.rados.0.smithi136.stdout:646: done (0 left) 2024-09-12T23:40:14.612 INFO:tasks.rados.rados.0.smithi136.stdout:647: snap_remove snap 53 2024-09-12T23:40:14.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:14 smithi181 bash[24776]: cluster 2024-09-12T23:40:13.557312+0000 mgr.x (mgr.24517) 11 : cluster [DBG] pgmap v11: 121 pgs: 121 active+clean; 92 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:40:14.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:14 smithi181 bash[24776]: audit 2024-09-12T23:40:13.588412+0000 mon.b (mon.1) 61 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-12T23:40:14.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:14 smithi181 bash[24776]: cluster 2024-09-12T23:40:13.590914+0000 mon.a (mon.0) 1244 : cluster [DBG] mgrmap e28: x(active, since 6s), standbys: y 2024-09-12T23:40:14.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:14 smithi181 bash[24776]: cluster 2024-09-12T23:40:13.603088+0000 mon.a (mon.0) 1245 : cluster [DBG] osdmap e178: 8 total, 8 up, 8 in 2024-09-12T23:40:14.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:14 smithi181 bash[24776]: audit 2024-09-12T23:40:14.343006+0000 mon.a (mon.0) 1246 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:14.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:14 smithi181 bash[24776]: audit 2024-09-12T23:40:14.355532+0000 mon.a (mon.0) 1247 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:14.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:14 smithi181 bash[24776]: audit 2024-09-12T23:40:14.437402+0000 mon.a (mon.0) 1248 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:14.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:14 smithi181 bash[24776]: audit 2024-09-12T23:40:14.450495+0000 mon.a (mon.0) 1249 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:15.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:14 smithi136 bash[22601]: cluster 2024-09-12T23:40:13.557312+0000 mgr.x (mgr.24517) 11 : cluster [DBG] pgmap v11: 121 pgs: 121 active+clean; 92 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:40:15.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:14 smithi136 bash[22601]: audit 2024-09-12T23:40:13.588412+0000 mon.b (mon.1) 61 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-12T23:40:15.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:14 smithi136 bash[22601]: cluster 2024-09-12T23:40:13.590914+0000 mon.a (mon.0) 1244 : cluster [DBG] mgrmap e28: x(active, since 6s), standbys: y 2024-09-12T23:40:15.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:14 smithi136 bash[22601]: cluster 2024-09-12T23:40:13.603088+0000 mon.a (mon.0) 1245 : cluster [DBG] osdmap e178: 8 total, 8 up, 8 in 2024-09-12T23:40:15.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:14 smithi136 bash[22601]: audit 2024-09-12T23:40:14.343006+0000 mon.a (mon.0) 1246 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:15.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:14 smithi136 bash[22601]: audit 2024-09-12T23:40:14.355532+0000 mon.a (mon.0) 1247 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:15.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:14 smithi136 bash[22601]: audit 2024-09-12T23:40:14.437402+0000 mon.a (mon.0) 1248 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:15.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:14 smithi136 bash[22601]: audit 2024-09-12T23:40:14.450495+0000 mon.a (mon.0) 1249 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:15.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:14 smithi136 bash[32189]: cluster 2024-09-12T23:40:13.557312+0000 mgr.x (mgr.24517) 11 : cluster [DBG] pgmap v11: 121 pgs: 121 active+clean; 92 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:40:15.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:14 smithi136 bash[32189]: audit 2024-09-12T23:40:13.588412+0000 mon.b (mon.1) 61 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-12T23:40:15.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:14 smithi136 bash[32189]: cluster 2024-09-12T23:40:13.590914+0000 mon.a (mon.0) 1244 : cluster [DBG] mgrmap e28: x(active, since 6s), standbys: y 2024-09-12T23:40:15.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:14 smithi136 bash[32189]: cluster 2024-09-12T23:40:13.603088+0000 mon.a (mon.0) 1245 : cluster [DBG] osdmap e178: 8 total, 8 up, 8 in 2024-09-12T23:40:15.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:14 smithi136 bash[32189]: audit 2024-09-12T23:40:14.343006+0000 mon.a (mon.0) 1246 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:15.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:14 smithi136 bash[32189]: audit 2024-09-12T23:40:14.355532+0000 mon.a (mon.0) 1247 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:15.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:14 smithi136 bash[32189]: audit 2024-09-12T23:40:14.437402+0000 mon.a (mon.0) 1248 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:15.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:14 smithi136 bash[32189]: audit 2024-09-12T23:40:14.450495+0000 mon.a (mon.0) 1249 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:15.594 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:40:15 smithi181 bash[52367]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:40:15] "GET /metrics HTTP/1.1" 200 33189 "" "Prometheus/2.43.0" 2024-09-12T23:40:15.615 INFO:tasks.rados.rados.0.smithi136.stdout:647: done (0 left) 2024-09-12T23:40:15.615 INFO:tasks.rados.rados.0.smithi136.stdout:648: rollback oid 12 current snap is 56 2024-09-12T23:40:15.615 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 12 to 55 2024-09-12T23:40:15.615 INFO:tasks.rados.rados.0.smithi136.stdout:649: read oid 21 snap -1 2024-09-12T23:40:15.616 INFO:tasks.rados.rados.0.smithi136.stdout:649: expect (ObjNum 88 snap 18 seq_num 88) 2024-09-12T23:40:15.616 INFO:tasks.rados.rados.0.smithi136.stdout:650: snap_remove snap 54 2024-09-12T23:40:16.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:15 smithi136 bash[22601]: cluster 2024-09-12T23:40:14.616516+0000 mon.a (mon.0) 1250 : cluster [DBG] osdmap e179: 8 total, 8 up, 8 in 2024-09-12T23:40:16.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:15 smithi136 bash[22601]: audit 2024-09-12T23:40:15.171114+0000 mon.a (mon.0) 1251 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:16.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:15 smithi136 bash[22601]: audit 2024-09-12T23:40:15.178395+0000 mon.a (mon.0) 1252 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:16.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:15 smithi136 bash[22601]: audit 2024-09-12T23:40:15.185641+0000 mon.a (mon.0) 1253 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:16.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:15 smithi136 bash[22601]: audit 2024-09-12T23:40:15.194971+0000 mon.a (mon.0) 1254 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:16.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:15 smithi136 bash[22601]: audit 2024-09-12T23:40:15.199197+0000 mon.b (mon.1) 62 : audit [INF] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:40:16.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:15 smithi136 bash[22601]: audit 2024-09-12T23:40:15.199774+0000 mon.a (mon.0) 1255 : audit [INF] from='mgr.24517 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:40:16.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:15 smithi136 bash[22601]: audit 2024-09-12T23:40:15.201352+0000 mon.b (mon.1) 63 : audit [INF] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:40:16.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:15 smithi136 bash[22601]: audit 2024-09-12T23:40:15.201865+0000 mon.a (mon.0) 1256 : audit [INF] from='mgr.24517 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:40:16.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:15 smithi136 bash[22601]: audit 2024-09-12T23:40:15.203213+0000 mon.b (mon.1) 64 : audit [INF] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:40:16.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:15 smithi136 bash[22601]: audit 2024-09-12T23:40:15.203952+0000 mon.a (mon.0) 1257 : audit [INF] from='mgr.24517 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:40:16.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:15 smithi136 bash[22601]: audit 2024-09-12T23:40:15.205175+0000 mon.b (mon.1) 65 : audit [INF] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:40:16.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:15 smithi136 bash[22601]: audit 2024-09-12T23:40:15.205549+0000 mon.a (mon.0) 1258 : audit [INF] from='mgr.24517 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:40:16.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:15 smithi136 bash[22601]: cephadm 2024-09-12T23:40:15.206749+0000 mgr.x (mgr.24517) 12 : cephadm [INF] Adjusting osd_memory_target on smithi136 to 2524M 2024-09-12T23:40:16.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:15 smithi136 bash[22601]: audit 2024-09-12T23:40:15.214398+0000 mon.a (mon.0) 1259 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:16.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:15 smithi136 bash[32189]: cluster 2024-09-12T23:40:14.616516+0000 mon.a (mon.0) 1250 : cluster [DBG] osdmap e179: 8 total, 8 up, 8 in 2024-09-12T23:40:16.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:15 smithi136 bash[32189]: audit 2024-09-12T23:40:15.171114+0000 mon.a (mon.0) 1251 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:16.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:15 smithi136 bash[32189]: audit 2024-09-12T23:40:15.178395+0000 mon.a (mon.0) 1252 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:16.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:15 smithi136 bash[32189]: audit 2024-09-12T23:40:15.185641+0000 mon.a (mon.0) 1253 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:16.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:15 smithi136 bash[32189]: audit 2024-09-12T23:40:15.194971+0000 mon.a (mon.0) 1254 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:16.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:15 smithi136 bash[32189]: audit 2024-09-12T23:40:15.199197+0000 mon.b (mon.1) 62 : audit [INF] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:40:16.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:15 smithi136 bash[32189]: audit 2024-09-12T23:40:15.199774+0000 mon.a (mon.0) 1255 : audit [INF] from='mgr.24517 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:40:16.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:15 smithi136 bash[32189]: audit 2024-09-12T23:40:15.201352+0000 mon.b (mon.1) 63 : audit [INF] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:40:16.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:15 smithi136 bash[32189]: audit 2024-09-12T23:40:15.201865+0000 mon.a (mon.0) 1256 : audit [INF] from='mgr.24517 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:40:16.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:15 smithi136 bash[32189]: audit 2024-09-12T23:40:15.203213+0000 mon.b (mon.1) 64 : audit [INF] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:40:16.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:15 smithi136 bash[32189]: audit 2024-09-12T23:40:15.203952+0000 mon.a (mon.0) 1257 : audit [INF] from='mgr.24517 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:40:16.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:15 smithi136 bash[32189]: audit 2024-09-12T23:40:15.205175+0000 mon.b (mon.1) 65 : audit [INF] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:40:16.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:15 smithi136 bash[32189]: audit 2024-09-12T23:40:15.205549+0000 mon.a (mon.0) 1258 : audit [INF] from='mgr.24517 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:40:16.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:15 smithi136 bash[32189]: cephadm 2024-09-12T23:40:15.206749+0000 mgr.x (mgr.24517) 12 : cephadm [INF] Adjusting osd_memory_target on smithi136 to 2524M 2024-09-12T23:40:16.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:15 smithi136 bash[32189]: audit 2024-09-12T23:40:15.214398+0000 mon.a (mon.0) 1259 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:16.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:15 smithi181 bash[24776]: cluster 2024-09-12T23:40:14.616516+0000 mon.a (mon.0) 1250 : cluster [DBG] osdmap e179: 8 total, 8 up, 8 in 2024-09-12T23:40:16.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:15 smithi181 bash[24776]: audit 2024-09-12T23:40:15.171114+0000 mon.a (mon.0) 1251 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:16.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:15 smithi181 bash[24776]: audit 2024-09-12T23:40:15.178395+0000 mon.a (mon.0) 1252 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:16.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:15 smithi181 bash[24776]: audit 2024-09-12T23:40:15.185641+0000 mon.a (mon.0) 1253 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:16.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:15 smithi181 bash[24776]: audit 2024-09-12T23:40:15.194971+0000 mon.a (mon.0) 1254 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:16.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:15 smithi181 bash[24776]: audit 2024-09-12T23:40:15.199197+0000 mon.b (mon.1) 62 : audit [INF] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:40:16.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:15 smithi181 bash[24776]: audit 2024-09-12T23:40:15.199774+0000 mon.a (mon.0) 1255 : audit [INF] from='mgr.24517 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:40:16.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:15 smithi181 bash[24776]: audit 2024-09-12T23:40:15.201352+0000 mon.b (mon.1) 63 : audit [INF] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:40:16.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:15 smithi181 bash[24776]: audit 2024-09-12T23:40:15.201865+0000 mon.a (mon.0) 1256 : audit [INF] from='mgr.24517 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:40:16.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:15 smithi181 bash[24776]: audit 2024-09-12T23:40:15.203213+0000 mon.b (mon.1) 64 : audit [INF] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:40:16.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:15 smithi181 bash[24776]: audit 2024-09-12T23:40:15.203952+0000 mon.a (mon.0) 1257 : audit [INF] from='mgr.24517 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:40:16.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:15 smithi181 bash[24776]: audit 2024-09-12T23:40:15.205175+0000 mon.b (mon.1) 65 : audit [INF] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:40:16.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:15 smithi181 bash[24776]: audit 2024-09-12T23:40:15.205549+0000 mon.a (mon.0) 1258 : audit [INF] from='mgr.24517 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:40:16.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:15 smithi181 bash[24776]: cephadm 2024-09-12T23:40:15.206749+0000 mgr.x (mgr.24517) 12 : cephadm [INF] Adjusting osd_memory_target on smithi136 to 2524M 2024-09-12T23:40:16.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:15 smithi181 bash[24776]: audit 2024-09-12T23:40:15.214398+0000 mon.a (mon.0) 1259 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:16.505 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:16 smithi136 bash[22852]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:40:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-12T23:40:16.616 INFO:tasks.rados.rados.0.smithi136.stdout:650: done (2 left) 2024-09-12T23:40:16.616 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 44 current snap is 56 2024-09-12T23:40:16.616 INFO:tasks.rados.rados.0.smithi136.stdout:651: seq_num 221 ranges {0=507904,507904=688128,1196032=114688} 2024-09-12T23:40:16.626 INFO:tasks.rados.rados.0.smithi136.stdout:651: writing smithi13667081-44 from 0 to 507904 tid 1 2024-09-12T23:40:16.641 INFO:tasks.rados.rados.0.smithi136.stdout:651: writing smithi13667081-44 from 507904 to 1196032 tid 2 2024-09-12T23:40:16.643 INFO:tasks.rados.rados.0.smithi136.stdout:651: writing smithi13667081-44 from 1196032 to 1310720 tid 3 2024-09-12T23:40:16.644 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 28 current snap is 56 2024-09-12T23:40:16.644 INFO:tasks.rados.rados.0.smithi136.stdout:652: seq_num 222 ranges {1105920=524288,1630208=565248,2195456=237568} 2024-09-12T23:40:16.670 INFO:tasks.rados.rados.0.smithi136.stdout:652: writing smithi13667081-28 from 1105920 to 1630208 tid 1 2024-09-12T23:40:16.678 INFO:tasks.rados.rados.0.smithi136.stdout:652: writing smithi13667081-28 from 1630208 to 2195456 tid 2 2024-09-12T23:40:16.681 INFO:tasks.rados.rados.0.smithi136.stdout:652: writing smithi13667081-28 from 2195456 to 2433024 tid 3 2024-09-12T23:40:16.681 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 11 current snap is 56 2024-09-12T23:40:16.681 INFO:tasks.rados.rados.0.smithi136.stdout:648: finishing rollback tid 0 to smithi13667081-12 2024-09-12T23:40:16.681 INFO:tasks.rados.rados.0.smithi136.stdout:648: finishing rollback tid 1 to smithi13667081-12 2024-09-12T23:40:16.681 INFO:tasks.rados.rados.0.smithi136.stdout:653: seq_num 223 ranges {1736704=540672,2277376=450560,2727936=352256} 2024-09-12T23:40:16.706 INFO:tasks.rados.rados.0.smithi136.stdout:653: writing smithi13667081-11 from 1736704 to 2277376 tid 1 2024-09-12T23:40:16.711 INFO:tasks.rados.rados.0.smithi136.stdout:653: writing smithi13667081-11 from 2277376 to 2727936 tid 2 2024-09-12T23:40:16.715 INFO:tasks.rados.rados.0.smithi136.stdout:653: writing smithi13667081-11 from 2727936 to 3080192 tid 3 2024-09-12T23:40:16.715 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 10 current snap is 56 2024-09-12T23:40:16.715 INFO:tasks.rados.rados.0.smithi136.stdout:654: seq_num 224 ranges {0=557056,557056=737280,1294336=65536} 2024-09-12T23:40:16.721 INFO:tasks.rados.rados.0.smithi136.stdout:654: writing smithi13667081-10 from 0 to 557056 tid 1 2024-09-12T23:40:16.729 INFO:tasks.rados.rados.0.smithi136.stdout:654: writing smithi13667081-10 from 557056 to 1294336 tid 2 2024-09-12T23:40:16.730 INFO:tasks.rados.rados.0.smithi136.stdout:654: writing smithi13667081-10 from 1294336 to 1359872 tid 3 2024-09-12T23:40:16.730 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 39 current snap is 56 2024-09-12T23:40:16.730 INFO:tasks.rados.rados.0.smithi136.stdout:648: finishing rollback tid 2 to smithi13667081-12 2024-09-12T23:40:16.730 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 12 v 400 (ObjNum 191 snap 46 seq_num 191) dirty exists 2024-09-12T23:40:16.730 INFO:tasks.rados.rados.0.smithi136.stdout:649: expect (ObjNum 88 snap 18 seq_num 88) 2024-09-12T23:40:16.874 INFO:tasks.rados.rados.0.smithi136.stdout:651: finishing write tid 1 to smithi13667081-44 2024-09-12T23:40:16.874 INFO:tasks.rados.rados.0.smithi136.stdout:651: finishing write tid 2 to smithi13667081-44 2024-09-12T23:40:16.874 INFO:tasks.rados.rados.0.smithi136.stdout:651: finishing write tid 3 to smithi13667081-44 2024-09-12T23:40:16.875 INFO:tasks.rados.rados.0.smithi136.stdout:651: finishing write tid 4 to smithi13667081-44 2024-09-12T23:40:16.875 INFO:tasks.rados.rados.0.smithi136.stdout:651: finishing write tid 5 to smithi13667081-44 2024-09-12T23:40:16.875 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 44 v 361 (ObjNum 221 snap 56 seq_num 221) dirty exists 2024-09-12T23:40:16.875 INFO:tasks.rados.rados.0.smithi136.stdout:651: left oid 44 (ObjNum 221 snap 56 seq_num 221) 2024-09-12T23:40:16.875 INFO:tasks.rados.rados.0.smithi136.stdout:652: finishing write tid 1 to smithi13667081-28 2024-09-12T23:40:16.875 INFO:tasks.rados.rados.0.smithi136.stdout:652: finishing write tid 2 to smithi13667081-28 2024-09-12T23:40:16.875 INFO:tasks.rados.rados.0.smithi136.stdout:652: finishing write tid 3 to smithi13667081-28 2024-09-12T23:40:16.875 INFO:tasks.rados.rados.0.smithi136.stdout:652: finishing write tid 4 to smithi13667081-28 2024-09-12T23:40:16.875 INFO:tasks.rados.rados.0.smithi136.stdout:652: finishing write tid 5 to smithi13667081-28 2024-09-12T23:40:16.875 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 28 v 366 (ObjNum 222 snap 56 seq_num 222) dirty exists 2024-09-12T23:40:16.875 INFO:tasks.rados.rados.0.smithi136.stdout:652: left oid 28 (ObjNum 222 snap 56 seq_num 222) 2024-09-12T23:40:16.876 INFO:tasks.rados.rados.0.smithi136.stdout:653: finishing write tid 1 to smithi13667081-11 2024-09-12T23:40:16.876 INFO:tasks.rados.rados.0.smithi136.stdout:653: finishing write tid 2 to smithi13667081-11 2024-09-12T23:40:16.876 INFO:tasks.rados.rados.0.smithi136.stdout:653: finishing write tid 3 to smithi13667081-11 2024-09-12T23:40:16.876 INFO:tasks.rados.rados.0.smithi136.stdout:653: finishing write tid 4 to smithi13667081-11 2024-09-12T23:40:16.876 INFO:tasks.rados.rados.0.smithi136.stdout:653: finishing write tid 5 to smithi13667081-11 2024-09-12T23:40:16.876 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 11 v 350 (ObjNum 223 snap 56 seq_num 223) dirty exists 2024-09-12T23:40:16.876 INFO:tasks.rados.rados.0.smithi136.stdout:653: left oid 11 (ObjNum 223 snap 56 seq_num 223) 2024-09-12T23:40:16.876 INFO:tasks.rados.rados.0.smithi136.stdout:654: finishing write tid 1 to smithi13667081-10 2024-09-12T23:40:16.876 INFO:tasks.rados.rados.0.smithi136.stdout:654: finishing write tid 2 to smithi13667081-10 2024-09-12T23:40:16.876 INFO:tasks.rados.rados.0.smithi136.stdout:654: finishing write tid 3 to smithi13667081-10 2024-09-12T23:40:16.876 INFO:tasks.rados.rados.0.smithi136.stdout:654: finishing write tid 4 to smithi13667081-10 2024-09-12T23:40:16.876 INFO:tasks.rados.rados.0.smithi136.stdout:654: finishing write tid 5 to smithi13667081-10 2024-09-12T23:40:16.877 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 10 v 196 (ObjNum 224 snap 56 seq_num 224) dirty exists 2024-09-12T23:40:16.877 INFO:tasks.rados.rados.0.smithi136.stdout:654: left oid 10 (ObjNum 224 snap 56 seq_num 224) 2024-09-12T23:40:16.877 INFO:tasks.rados.rados.0.smithi136.stdout:655: seq_num 225 ranges {0=573440,573440=622592,1196032=180224} 2024-09-12T23:40:16.879 INFO:tasks.rados.rados.0.smithi136.stdout:655: writing smithi13667081-39 from 0 to 573440 tid 1 2024-09-12T23:40:16.885 INFO:tasks.rados.rados.0.smithi136.stdout:655: writing smithi13667081-39 from 573440 to 1196032 tid 2 2024-09-12T23:40:16.887 INFO:tasks.rados.rados.0.smithi136.stdout:655: writing smithi13667081-39 from 1196032 to 1376256 tid 3 2024-09-12T23:40:16.887 INFO:tasks.rados.rados.0.smithi136.stdout:655: finishing write tid 1 to smithi13667081-39 2024-09-12T23:40:16.887 INFO:tasks.rados.rados.0.smithi136.stdout:648: done (6 left) 2024-09-12T23:40:16.887 INFO:tasks.rados.rados.0.smithi136.stdout:649: done (5 left) 2024-09-12T23:40:16.887 INFO:tasks.rados.rados.0.smithi136.stdout:651: done (4 left) 2024-09-12T23:40:16.887 INFO:tasks.rados.rados.0.smithi136.stdout:652: done (3 left) 2024-09-12T23:40:16.887 INFO:tasks.rados.rados.0.smithi136.stdout:653: done (2 left) 2024-09-12T23:40:16.888 INFO:tasks.rados.rados.0.smithi136.stdout:654: done (1 left) 2024-09-12T23:40:16.888 INFO:tasks.rados.rados.0.smithi136.stdout:656: setattr oid 7 current snap is 56 2024-09-12T23:40:16.888 INFO:tasks.rados.rados.0.smithi136.stdout:657: delete oid 6 current snap is 56 2024-09-12T23:40:16.891 INFO:tasks.rados.rados.0.smithi136.stdout:657: done (2 left) 2024-09-12T23:40:16.891 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 24 current snap is 56 2024-09-12T23:40:16.891 INFO:tasks.rados.rados.0.smithi136.stdout:658: seq_num 227 ranges {1302528=606208,1908736=794624,2703360=8192} 2024-09-12T23:40:16.907 INFO:tasks.rados.rados.0.smithi136.stdout:658: writing smithi13667081-24 from 1302528 to 1908736 tid 1 2024-09-12T23:40:16.914 INFO:tasks.rados.rados.0.smithi136.stdout:658: writing smithi13667081-24 from 1908736 to 2703360 tid 2 2024-09-12T23:40:16.914 INFO:tasks.rados.rados.0.smithi136.stdout:658: writing smithi13667081-24 from 2703360 to 2711552 tid 3 2024-09-12T23:40:16.914 INFO:tasks.rados.rados.0.smithi136.stdout:659: copy_from oid 44 from oid 50 current snap is 56 2024-09-12T23:40:16.914 INFO:tasks.rados.rados.0.smithi136.stdout:655: finishing write tid 2 to smithi13667081-39 2024-09-12T23:40:16.914 INFO:tasks.rados.rados.0.smithi136.stdout:660: rollback oid 23 current snap is 56 2024-09-12T23:40:16.914 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 23 to 52 2024-09-12T23:40:16.914 INFO:tasks.rados.rados.0.smithi136.stdout:655: finishing write tid 3 to smithi13667081-39 2024-09-12T23:40:16.915 INFO:tasks.rados.rados.0.smithi136.stdout:655: finishing write tid 4 to smithi13667081-39 2024-09-12T23:40:16.915 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 7 v 264 (ObjNum 161 snap 0 seq_num 0) dirty exists 2024-09-12T23:40:16.915 INFO:tasks.rados.rados.0.smithi136.stdout:655: finishing write tid 5 to smithi13667081-39 2024-09-12T23:40:16.915 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 39 v 90 (ObjNum 225 snap 56 seq_num 225) dirty exists 2024-09-12T23:40:16.915 INFO:tasks.rados.rados.0.smithi136.stdout:655: left oid 39 (ObjNum 225 snap 56 seq_num 225) 2024-09-12T23:40:16.915 INFO:tasks.rados.rados.0.smithi136.stdout:655: done (4 left) 2024-09-12T23:40:16.915 INFO:tasks.rados.rados.0.smithi136.stdout:656: done (3 left) 2024-09-12T23:40:16.915 INFO:tasks.rados.rados.0.smithi136.stdout:661: copy_from oid 33 from oid 18 current snap is 56 2024-09-12T23:40:16.915 INFO:tasks.rados.rados.0.smithi136.stdout:658: finishing write tid 1 to smithi13667081-24 2024-09-12T23:40:16.915 INFO:tasks.rados.rados.0.smithi136.stdout:662: copy_from oid 17 from oid 8 current snap is 56 2024-09-12T23:40:16.915 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 16 current snap is 56 2024-09-12T23:40:16.915 INFO:tasks.rados.rados.0.smithi136.stdout:663: seq_num 228 ranges {0=622592,622592=671744,1294336=131072} 2024-09-12T23:40:16.919 INFO:tasks.rados.rados.0.smithi136.stdout:663: writing smithi13667081-16 from 0 to 622592 tid 1 2024-09-12T23:40:16.926 INFO:tasks.rados.rados.0.smithi136.stdout:663: writing smithi13667081-16 from 622592 to 1294336 tid 2 2024-09-12T23:40:16.928 INFO:tasks.rados.rados.0.smithi136.stdout:663: writing smithi13667081-16 from 1294336 to 1425408 tid 3 2024-09-12T23:40:16.928 INFO:tasks.rados.rados.0.smithi136.stdout:664: snap_remove snap 55 2024-09-12T23:40:17.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:16 smithi136 bash[22601]: cluster 2024-09-12T23:40:15.557944+0000 mgr.x (mgr.24517) 13 : cluster [DBG] pgmap v14: 121 pgs: 121 active+clean; 100 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 3.0 MiB/s wr, 57 op/s 2024-09-12T23:40:17.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:16 smithi136 bash[22601]: cluster 2024-09-12T23:40:15.618257+0000 mon.a (mon.0) 1260 : cluster [DBG] osdmap e180: 8 total, 8 up, 8 in 2024-09-12T23:40:17.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:16 smithi136 bash[22601]: cluster 2024-09-12T23:40:16.619970+0000 mon.a (mon.0) 1261 : cluster [DBG] osdmap e181: 8 total, 8 up, 8 in 2024-09-12T23:40:17.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:16 smithi136 bash[32189]: cluster 2024-09-12T23:40:15.557944+0000 mgr.x (mgr.24517) 13 : cluster [DBG] pgmap v14: 121 pgs: 121 active+clean; 100 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 3.0 MiB/s wr, 57 op/s 2024-09-12T23:40:17.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:16 smithi136 bash[32189]: cluster 2024-09-12T23:40:15.618257+0000 mon.a (mon.0) 1260 : cluster [DBG] osdmap e180: 8 total, 8 up, 8 in 2024-09-12T23:40:17.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:16 smithi136 bash[32189]: cluster 2024-09-12T23:40:16.619970+0000 mon.a (mon.0) 1261 : cluster [DBG] osdmap e181: 8 total, 8 up, 8 in 2024-09-12T23:40:17.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:16 smithi181 bash[24776]: cluster 2024-09-12T23:40:15.557944+0000 mgr.x (mgr.24517) 13 : cluster [DBG] pgmap v14: 121 pgs: 121 active+clean; 100 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 3.0 MiB/s wr, 57 op/s 2024-09-12T23:40:17.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:16 smithi181 bash[24776]: cluster 2024-09-12T23:40:15.618257+0000 mon.a (mon.0) 1260 : cluster [DBG] osdmap e180: 8 total, 8 up, 8 in 2024-09-12T23:40:17.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:16 smithi181 bash[24776]: cluster 2024-09-12T23:40:16.619970+0000 mon.a (mon.0) 1261 : cluster [DBG] osdmap e181: 8 total, 8 up, 8 in 2024-09-12T23:40:17.639 INFO:tasks.rados.rados.0.smithi136.stdout:664: done (6 left) 2024-09-12T23:40:17.640 INFO:tasks.rados.rados.0.smithi136.stdout:665: read oid 7 snap -1 2024-09-12T23:40:17.640 INFO:tasks.rados.rados.0.smithi136.stdout:665: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-12T23:40:17.640 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 12 current snap is 56 2024-09-12T23:40:17.640 INFO:tasks.rados.rados.0.smithi136.stdout:660: finishing rollback tid 0 to smithi13667081-23 2024-09-12T23:40:17.640 INFO:tasks.rados.rados.0.smithi136.stdout:660: finishing rollback tid 1 to smithi13667081-23 2024-09-12T23:40:17.640 INFO:tasks.rados.rados.0.smithi136.stdout:660: finishing rollback tid 2 to smithi13667081-23 2024-09-12T23:40:17.640 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 23 v 354 (ObjNum 198 snap 48 seq_num 198) dirty exists 2024-09-12T23:40:17.640 INFO:tasks.rados.rados.0.smithi136.stdout:658: finishing write tid 2 to smithi13667081-24 2024-09-12T23:40:17.640 INFO:tasks.rados.rados.0.smithi136.stdout:658: finishing write tid 3 to smithi13667081-24 2024-09-12T23:40:17.640 INFO:tasks.rados.rados.0.smithi136.stdout:658: finishing write tid 4 to smithi13667081-24 2024-09-12T23:40:17.640 INFO:tasks.rados.rados.0.smithi136.stdout:662: finishing copy_from racing read to smithi13667081-17 2024-09-12T23:40:17.641 INFO:tasks.rados.rados.0.smithi136.stdout:658: finishing write tid 5 to smithi13667081-24 2024-09-12T23:40:17.641 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 24 v 158 (ObjNum 227 snap 56 seq_num 227) dirty exists 2024-09-12T23:40:17.641 INFO:tasks.rados.rados.0.smithi136.stdout:658: left oid 24 (ObjNum 227 snap 56 seq_num 227) 2024-09-12T23:40:17.641 INFO:tasks.rados.rados.0.smithi136.stdout:662: finishing copy_from to smithi13667081-17 2024-09-12T23:40:17.641 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 17 v 134 (ObjNum 164 snap 43 seq_num 164) dirty exists 2024-09-12T23:40:17.641 INFO:tasks.rados.rados.0.smithi136.stdout:659: finishing copy_from racing read to smithi13667081-44 2024-09-12T23:40:17.641 INFO:tasks.rados.rados.0.smithi136.stdout:666: seq_num 229 ranges {3883008=638976,4521984=606208,5128192=196608} 2024-09-12T23:40:17.691 INFO:tasks.rados.rados.0.smithi136.stdout:666: writing smithi13667081-12 from 3883008 to 4521984 tid 1 2024-09-12T23:40:17.697 INFO:tasks.rados.rados.0.smithi136.stdout:666: writing smithi13667081-12 from 4521984 to 5128192 tid 2 2024-09-12T23:40:17.700 INFO:tasks.rados.rados.0.smithi136.stdout:666: writing smithi13667081-12 from 5128192 to 5324800 tid 3 2024-09-12T23:40:17.700 INFO:tasks.rados.rados.0.smithi136.stdout:658: done (7 left) 2024-09-12T23:40:17.700 INFO:tasks.rados.rados.0.smithi136.stdout:660: done (6 left) 2024-09-12T23:40:17.700 INFO:tasks.rados.rados.0.smithi136.stdout:662: done (5 left) 2024-09-12T23:40:17.700 INFO:tasks.rados.rados.0.smithi136.stdout:667: read oid 1 snap -1 2024-09-12T23:40:17.701 INFO:tasks.rados.rados.0.smithi136.stdout:667: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-12T23:40:17.701 INFO:tasks.rados.rados.0.smithi136.stdout:668: rollback oid 43 current snap is 56 2024-09-12T23:40:17.701 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 43 to 51 2024-09-12T23:40:17.701 INFO:tasks.rados.rados.0.smithi136.stdout:669: delete oid 11 current snap is 56 2024-09-12T23:40:17.701 INFO:tasks.rados.rados.0.smithi136.stdout:663: finishing write tid 1 to smithi13667081-16 2024-09-12T23:40:17.701 INFO:tasks.rados.rados.0.smithi136.stdout:659: finishing copy_from to smithi13667081-44 2024-09-12T23:40:17.701 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 44 v 235 (ObjNum 49 snap 0 seq_num 49) dirty exists 2024-09-12T23:40:17.701 INFO:tasks.rados.rados.0.smithi136.stdout:663: finishing write tid 2 to smithi13667081-16 2024-09-12T23:40:17.701 INFO:tasks.rados.rados.0.smithi136.stdout:663: finishing write tid 3 to smithi13667081-16 2024-09-12T23:40:17.701 INFO:tasks.rados.rados.0.smithi136.stdout:663: finishing write tid 4 to smithi13667081-16 2024-09-12T23:40:17.701 INFO:tasks.rados.rados.0.smithi136.stdout:661: finishing copy_from racing read to smithi13667081-33 2024-09-12T23:40:17.701 INFO:tasks.rados.rados.0.smithi136.stdout:661: finishing copy_from to smithi13667081-33 2024-09-12T23:40:17.702 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 33 v 263 (ObjNum 198 snap 48 seq_num 198) dirty exists 2024-09-12T23:40:17.702 INFO:tasks.rados.rados.0.smithi136.stdout:663: finishing write tid 5 to smithi13667081-16 2024-09-12T23:40:17.702 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 16 v 371 (ObjNum 228 snap 56 seq_num 228) dirty exists 2024-09-12T23:40:17.702 INFO:tasks.rados.rados.0.smithi136.stdout:663: left oid 16 (ObjNum 228 snap 56 seq_num 228) 2024-09-12T23:40:17.702 INFO:tasks.rados.rados.0.smithi136.stdout:665: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-12T23:40:17.710 INFO:tasks.rados.rados.0.smithi136.stdout:668: finishing rollback tid 1 to smithi13667081-43 2024-09-12T23:40:17.710 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 43 v 0 (ObjNum 241 snap 0 seq_num 1427291584) dirty dne 2024-09-12T23:40:17.713 INFO:tasks.rados.rados.0.smithi136.stdout:666: finishing write tid 1 to smithi13667081-12 2024-09-12T23:40:17.714 INFO:tasks.rados.rados.0.smithi136.stdout:667: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-12T23:40:17.717 INFO:tasks.rados.rados.0.smithi136.stdout:666: finishing write tid 2 to smithi13667081-12 2024-09-12T23:40:17.717 INFO:tasks.rados.rados.0.smithi136.stdout:666: finishing write tid 3 to smithi13667081-12 2024-09-12T23:40:17.717 INFO:tasks.rados.rados.0.smithi136.stdout:666: finishing write tid 4 to smithi13667081-12 2024-09-12T23:40:17.718 INFO:tasks.rados.rados.0.smithi136.stdout:659: done (7 left) 2024-09-12T23:40:17.718 INFO:tasks.rados.rados.0.smithi136.stdout:661: done (6 left) 2024-09-12T23:40:17.719 INFO:tasks.rados.rados.0.smithi136.stdout:663: done (5 left) 2024-09-12T23:40:17.719 INFO:tasks.rados.rados.0.smithi136.stdout:665: done (4 left) 2024-09-12T23:40:17.719 INFO:tasks.rados.rados.0.smithi136.stdout:667: done (3 left) 2024-09-12T23:40:17.719 INFO:tasks.rados.rados.0.smithi136.stdout:668: done (2 left) 2024-09-12T23:40:17.719 INFO:tasks.rados.rados.0.smithi136.stdout:669: done (1 left) 2024-09-12T23:40:17.719 INFO:tasks.rados.rados.0.smithi136.stdout:670: read oid 7 snap -1 2024-09-12T23:40:17.719 INFO:tasks.rados.rados.0.smithi136.stdout:670: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-12T23:40:17.719 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 47 current snap is 56 2024-09-12T23:40:17.719 INFO:tasks.rados.rados.0.smithi136.stdout:671: seq_num 231 ranges {0=671744,671744=778240,1449984=24576} 2024-09-12T23:40:17.725 INFO:tasks.rados.rados.0.smithi136.stdout:671: writing smithi13667081-47 from 0 to 671744 tid 1 2024-09-12T23:40:17.732 INFO:tasks.rados.rados.0.smithi136.stdout:671: writing smithi13667081-47 from 671744 to 1449984 tid 2 2024-09-12T23:40:17.732 INFO:tasks.rados.rados.0.smithi136.stdout:671: writing smithi13667081-47 from 1449984 to 1474560 tid 3 2024-09-12T23:40:17.732 INFO:tasks.rados.rados.0.smithi136.stdout:672: delete oid 10 current snap is 56 2024-09-12T23:40:17.733 INFO:tasks.rados.rados.0.smithi136.stdout:666: finishing write tid 5 to smithi13667081-12 2024-09-12T23:40:17.733 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 12 v 414 (ObjNum 229 snap 56 seq_num 229) dirty exists 2024-09-12T23:40:17.733 INFO:tasks.rados.rados.0.smithi136.stdout:666: left oid 12 (ObjNum 229 snap 56 seq_num 229) 2024-09-12T23:40:17.733 INFO:tasks.rados.rados.0.smithi136.stdout:670: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-12T23:40:17.745 INFO:tasks.rados.rados.0.smithi136.stdout:671: finishing write tid 1 to smithi13667081-47 2024-09-12T23:40:17.745 INFO:tasks.rados.rados.0.smithi136.stdout:671: finishing write tid 2 to smithi13667081-47 2024-09-12T23:40:17.745 INFO:tasks.rados.rados.0.smithi136.stdout:671: finishing write tid 3 to smithi13667081-47 2024-09-12T23:40:17.746 INFO:tasks.rados.rados.0.smithi136.stdout:671: finishing write tid 4 to smithi13667081-47 2024-09-12T23:40:17.746 INFO:tasks.rados.rados.0.smithi136.stdout:671: finishing write tid 5 to smithi13667081-47 2024-09-12T23:40:17.746 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 47 v 260 (ObjNum 231 snap 56 seq_num 231) dirty exists 2024-09-12T23:40:17.746 INFO:tasks.rados.rados.0.smithi136.stdout:671: left oid 47 (ObjNum 231 snap 56 seq_num 231) 2024-09-12T23:40:17.746 INFO:tasks.rados.rados.0.smithi136.stdout:666: done (3 left) 2024-09-12T23:40:17.746 INFO:tasks.rados.rados.0.smithi136.stdout:670: done (2 left) 2024-09-12T23:40:17.746 INFO:tasks.rados.rados.0.smithi136.stdout:671: done (1 left) 2024-09-12T23:40:17.746 INFO:tasks.rados.rados.0.smithi136.stdout:672: done (0 left) 2024-09-12T23:40:17.746 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 46 current snap is 56 2024-09-12T23:40:17.746 INFO:tasks.rados.rados.0.smithi136.stdout:673: seq_num 233 ranges {0=712704,712704=548864,1261568=253952} 2024-09-12T23:40:17.751 INFO:tasks.rados.rados.0.smithi136.stdout:673: writing smithi13667081-46 from 0 to 712704 tid 1 2024-09-12T23:40:17.756 INFO:tasks.rados.rados.0.smithi136.stdout:673: writing smithi13667081-46 from 712704 to 1261568 tid 2 2024-09-12T23:40:17.759 INFO:tasks.rados.rados.0.smithi136.stdout:673: writing smithi13667081-46 from 1261568 to 1515520 tid 3 2024-09-12T23:40:17.759 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 23 current snap is 56 2024-09-12T23:40:17.759 INFO:tasks.rados.rados.0.smithi136.stdout:674: seq_num 234 ranges {2949120=729088,3678208=434176,4112384=368640} 2024-09-12T23:40:17.784 INFO:tasks.rados.rados.0.smithi136.stdout:674: writing smithi13667081-23 from 2949120 to 3678208 tid 1 2024-09-12T23:40:17.787 INFO:tasks.rados.rados.0.smithi136.stdout:674: writing smithi13667081-23 from 3678208 to 4112384 tid 2 2024-09-12T23:40:17.790 INFO:tasks.rados.rados.0.smithi136.stdout:674: writing smithi13667081-23 from 4112384 to 4481024 tid 3 2024-09-12T23:40:17.790 INFO:tasks.rados.rados.0.smithi136.stdout:675: snap_create 2024-09-12T23:40:17.790 INFO:tasks.rados.rados.0.smithi136.stdout:673: finishing write tid 1 to smithi13667081-46 2024-09-12T23:40:17.790 INFO:tasks.rados.rados.0.smithi136.stdout:673: finishing write tid 2 to smithi13667081-46 2024-09-12T23:40:17.790 INFO:tasks.rados.rados.0.smithi136.stdout:673: finishing write tid 3 to smithi13667081-46 2024-09-12T23:40:17.790 INFO:tasks.rados.rados.0.smithi136.stdout:673: finishing write tid 4 to smithi13667081-46 2024-09-12T23:40:17.790 INFO:tasks.rados.rados.0.smithi136.stdout:673: finishing write tid 5 to smithi13667081-46 2024-09-12T23:40:17.790 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 46 v 164 (ObjNum 233 snap 56 seq_num 233) dirty exists 2024-09-12T23:40:17.790 INFO:tasks.rados.rados.0.smithi136.stdout:673: left oid 46 (ObjNum 233 snap 56 seq_num 233) 2024-09-12T23:40:17.791 INFO:tasks.rados.rados.0.smithi136.stdout:674: finishing write tid 1 to smithi13667081-23 2024-09-12T23:40:17.794 INFO:tasks.rados.rados.0.smithi136.stdout:674: finishing write tid 2 to smithi13667081-23 2024-09-12T23:40:17.794 INFO:tasks.rados.rados.0.smithi136.stdout:674: finishing write tid 3 to smithi13667081-23 2024-09-12T23:40:17.795 INFO:tasks.rados.rados.0.smithi136.stdout:674: finishing write tid 4 to smithi13667081-23 2024-09-12T23:40:17.798 INFO:tasks.rados.rados.0.smithi136.stdout:674: finishing write tid 5 to smithi13667081-23 2024-09-12T23:40:17.798 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 23 v 367 (ObjNum 234 snap 56 seq_num 234) dirty exists 2024-09-12T23:40:17.798 INFO:tasks.rados.rados.0.smithi136.stdout:674: left oid 23 (ObjNum 234 snap 56 seq_num 234) 2024-09-12T23:40:18.654 INFO:tasks.rados.rados.0.smithi136.stdout:673: done (2 left) 2024-09-12T23:40:18.654 INFO:tasks.rados.rados.0.smithi136.stdout:674: done (1 left) 2024-09-12T23:40:18.654 INFO:tasks.rados.rados.0.smithi136.stdout:675: done (0 left) 2024-09-12T23:40:18.654 INFO:tasks.rados.rados.0.smithi136.stdout:676: rollback oid 13 current snap is 57 2024-09-12T23:40:18.654 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 13 to 56 2024-09-12T23:40:18.654 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 35 current snap is 57 2024-09-12T23:40:18.654 INFO:tasks.rados.rados.0.smithi136.stdout:677: seq_num 235 ranges {0=745472,745472=712704,1458176=90112} 2024-09-12T23:40:18.659 INFO:tasks.rados.rados.0.smithi136.stdout:677: writing smithi13667081-35 from 0 to 745472 tid 1 2024-09-12T23:40:18.664 INFO:tasks.rados.rados.0.smithi136.stdout:677: writing smithi13667081-35 from 745472 to 1458176 tid 2 2024-09-12T23:40:18.666 INFO:tasks.rados.rados.0.smithi136.stdout:677: writing smithi13667081-35 from 1458176 to 1548288 tid 3 2024-09-12T23:40:18.666 INFO:tasks.rados.rados.0.smithi136.stdout:678: read oid 1 snap -1 2024-09-12T23:40:18.666 INFO:tasks.rados.rados.0.smithi136.stdout:678: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-12T23:40:18.666 INFO:tasks.rados.rados.0.smithi136.stdout:679: rollback oid 26 current snap is 57 2024-09-12T23:40:18.666 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 26 to 52 2024-09-12T23:40:18.666 INFO:tasks.rados.rados.0.smithi136.stdout:676: finishing rollback tid 0 to smithi13667081-13 2024-09-12T23:40:18.666 INFO:tasks.rados.rados.0.smithi136.stdout:680: rollback oid 5 current snap is 57 2024-09-12T23:40:18.666 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 5 to 51 2024-09-12T23:40:18.666 INFO:tasks.rados.rados.0.smithi136.stdout:681: setattr oid 22 current snap is 57 2024-09-12T23:40:18.667 INFO:tasks.rados.rados.0.smithi136.stdout:676: finishing rollback tid 1 to smithi13667081-13 2024-09-12T23:40:18.667 INFO:tasks.rados.rados.0.smithi136.stdout:676: finishing rollback tid 2 to smithi13667081-13 2024-09-12T23:40:18.667 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 13 v 261 (ObjNum 164 snap 43 seq_num 164) dirty exists 2024-09-12T23:40:18.669 INFO:tasks.rados.rados.0.smithi136.stdout:678: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-12T23:40:18.677 INFO:tasks.rados.rados.0.smithi136.stdout:677: finishing write tid 1 to smithi13667081-35 2024-09-12T23:40:18.677 INFO:tasks.rados.rados.0.smithi136.stdout:677: finishing write tid 2 to smithi13667081-35 2024-09-12T23:40:18.677 INFO:tasks.rados.rados.0.smithi136.stdout:676: done (5 left) 2024-09-12T23:40:18.677 INFO:tasks.rados.rados.0.smithi136.stdout:678: done (4 left) 2024-09-12T23:40:18.677 INFO:tasks.rados.rados.0.smithi136.stdout:682: delete oid 30 current snap is 57 2024-09-12T23:40:18.684 INFO:tasks.rados.rados.0.smithi136.stdout:677: finishing write tid 3 to smithi13667081-35 2024-09-12T23:40:18.684 INFO:tasks.rados.rados.0.smithi136.stdout:677: finishing write tid 4 to smithi13667081-35 2024-09-12T23:40:18.685 INFO:tasks.rados.rados.0.smithi136.stdout:677: finishing write tid 5 to smithi13667081-35 2024-09-12T23:40:18.685 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 35 v 371 (ObjNum 235 snap 57 seq_num 235) dirty exists 2024-09-12T23:40:18.686 INFO:tasks.rados.rados.0.smithi136.stdout:677: left oid 35 (ObjNum 235 snap 57 seq_num 235) 2024-09-12T23:40:18.687 INFO:tasks.rados.rados.0.smithi136.stdout:677: done (4 left) 2024-09-12T23:40:18.687 INFO:tasks.rados.rados.0.smithi136.stdout:682: done (3 left) 2024-09-12T23:40:18.687 INFO:tasks.rados.rados.0.smithi136.stdout:683: delete oid 12 current snap is 57 2024-09-12T23:40:18.687 INFO:tasks.rados.rados.0.smithi136.stdout:679: finishing rollback tid 0 to smithi13667081-26 2024-09-12T23:40:18.688 INFO:tasks.rados.rados.0.smithi136.stdout:680: finishing rollback tid 0 to smithi13667081-5 2024-09-12T23:40:18.688 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 22 v 153 (ObjNum 21 snap 0 seq_num 21) dirty exists 2024-09-12T23:40:18.688 INFO:tasks.rados.rados.0.smithi136.stdout:680: finishing rollback tid 1 to smithi13667081-5 2024-09-12T23:40:18.690 INFO:tasks.rados.rados.0.smithi136.stdout:680: finishing rollback tid 2 to smithi13667081-5 2024-09-12T23:40:18.690 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 5 v 375 (ObjNum 33 snap 0 seq_num 4243323840) dirty exists 2024-09-12T23:40:18.691 INFO:tasks.rados.rados.0.smithi136.stdout:679: finishing rollback tid 1 to smithi13667081-26 2024-09-12T23:40:18.691 INFO:tasks.rados.rados.0.smithi136.stdout:679: finishing rollback tid 2 to smithi13667081-26 2024-09-12T23:40:18.692 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 26 v 168 (ObjNum 160 snap 42 seq_num 160) dirty exists 2024-09-12T23:40:18.692 INFO:tasks.rados.rados.0.smithi136.stdout:679: done (3 left) 2024-09-12T23:40:18.692 INFO:tasks.rados.rados.0.smithi136.stdout:680: done (2 left) 2024-09-12T23:40:18.692 INFO:tasks.rados.rados.0.smithi136.stdout:681: done (1 left) 2024-09-12T23:40:18.692 INFO:tasks.rados.rados.0.smithi136.stdout:683: done (0 left) 2024-09-12T23:40:18.692 INFO:tasks.rados.rados.0.smithi136.stdout:684: rmattr oid 13 current snap is 57 2024-09-12T23:40:18.692 INFO:tasks.rados.rados.0.smithi136.stdout:684: done (0 left) 2024-09-12T23:40:18.692 INFO:tasks.rados.rados.0.smithi136.stdout:685: read oid 1 snap -1 2024-09-12T23:40:18.692 INFO:tasks.rados.rados.0.smithi136.stdout:685: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-12T23:40:18.692 INFO:tasks.rados.rados.0.smithi136.stdout:686: delete oid 8 current snap is 57 2024-09-12T23:40:18.693 INFO:tasks.rados.rados.0.smithi136.stdout:685: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-12T23:40:18.697 INFO:tasks.rados.rados.0.smithi136.stdout:685: done (1 left) 2024-09-12T23:40:18.697 INFO:tasks.rados.rados.0.smithi136.stdout:686: done (0 left) 2024-09-12T23:40:18.697 INFO:tasks.rados.rados.0.smithi136.stdout:687: rollback oid 9 current snap is 57 2024-09-12T23:40:18.698 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 9 to 56 2024-09-12T23:40:18.698 INFO:tasks.rados.rados.0.smithi136.stdout:688: read oid 6 snap -1 2024-09-12T23:40:18.698 INFO:tasks.rados.rados.0.smithi136.stdout:688: expect deleted 2024-09-12T23:40:18.698 INFO:tasks.rados.rados.0.smithi136.stdout:689: read oid 24 snap 51 2024-09-12T23:40:18.698 INFO:tasks.rados.rados.0.smithi136.stdout:689: expect (ObjNum 54 snap 4 seq_num 54) 2024-09-12T23:40:18.698 INFO:tasks.rados.rados.0.smithi136.stdout:690: snap_create 2024-09-12T23:40:18.702 INFO:tasks.rados.rados.0.smithi136.stdout:687: finishing rollback tid 0 to smithi13667081-9 2024-09-12T23:40:18.702 INFO:tasks.rados.rados.0.smithi136.stdout:687: finishing rollback tid 1 to smithi13667081-9 2024-09-12T23:40:18.703 INFO:tasks.rados.rados.0.smithi136.stdout:687: finishing rollback tid 2 to smithi13667081-9 2024-09-12T23:40:18.703 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 9 v 263 (ObjNum 6 snap 0 seq_num 6) dirty exists 2024-09-12T23:40:18.704 INFO:tasks.rados.rados.0.smithi136.stdout:689: expect (ObjNum 54 snap 4 seq_num 54) 2024-09-12T23:40:19.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[22601]: cluster 2024-09-12T23:40:17.558610+0000 mgr.x (mgr.24517) 14 : cluster [DBG] pgmap v17: 121 pgs: 121 active+clean; 100 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 3.0 MiB/s wr, 57 op/s 2024-09-12T23:40:19.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[22601]: cluster 2024-09-12T23:40:17.644169+0000 mon.a (mon.0) 1262 : cluster [DBG] osdmap e182: 8 total, 8 up, 8 in 2024-09-12T23:40:19.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[22601]: audit 2024-09-12T23:40:17.777265+0000 mon.b (mon.1) 66 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:40:19.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[22601]: audit 2024-09-12T23:40:17.796619+0000 mon.b (mon.1) 67 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-12T23:40:19.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[22601]: audit 2024-09-12T23:40:17.904483+0000 mon.b (mon.1) 68 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:40:19.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[22601]: audit 2024-09-12T23:40:17.905006+0000 mon.b (mon.1) 69 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:40:19.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[22601]: audit 2024-09-12T23:40:17.905429+0000 mon.b (mon.1) 70 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:40:19.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[22601]: audit 2024-09-12T23:40:17.908849+0000 mon.b (mon.1) 71 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:40:19.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[22601]: audit 2024-09-12T23:40:17.909282+0000 mon.b (mon.1) 72 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:40:19.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[22601]: audit 2024-09-12T23:40:17.909651+0000 mon.b (mon.1) 73 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:40:19.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[22601]: audit 2024-09-12T23:40:17.912990+0000 mon.b (mon.1) 74 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:40:19.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[22601]: audit 2024-09-12T23:40:17.913452+0000 mon.b (mon.1) 75 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:40:19.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[22601]: audit 2024-09-12T23:40:17.913830+0000 mon.b (mon.1) 76 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:40:19.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[22601]: audit 2024-09-12T23:40:17.934286+0000 mon.a (mon.0) 1263 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:40:19.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[22601]: audit 2024-09-12T23:40:17.934395+0000 mon.a (mon.0) 1264 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:40:19.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[22601]: audit 2024-09-12T23:40:17.934922+0000 mon.b (mon.1) 77 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:40:19.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[22601]: audit 2024-09-12T23:40:17.935301+0000 mon.b (mon.1) 78 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:40:19.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[22601]: audit 2024-09-12T23:40:17.935632+0000 mon.b (mon.1) 79 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:40:19.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[22601]: audit 2024-09-12T23:40:17.937605+0000 mon.b (mon.1) 80 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:40:19.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[32189]: cluster 2024-09-12T23:40:17.558610+0000 mgr.x (mgr.24517) 14 : cluster [DBG] pgmap v17: 121 pgs: 121 active+clean; 100 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 3.0 MiB/s wr, 57 op/s 2024-09-12T23:40:19.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[32189]: cluster 2024-09-12T23:40:17.644169+0000 mon.a (mon.0) 1262 : cluster [DBG] osdmap e182: 8 total, 8 up, 8 in 2024-09-12T23:40:19.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[32189]: audit 2024-09-12T23:40:17.777265+0000 mon.b (mon.1) 66 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:40:19.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[32189]: audit 2024-09-12T23:40:17.796619+0000 mon.b (mon.1) 67 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-12T23:40:19.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[32189]: audit 2024-09-12T23:40:17.904483+0000 mon.b (mon.1) 68 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:40:19.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[32189]: audit 2024-09-12T23:40:17.905006+0000 mon.b (mon.1) 69 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:40:19.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[32189]: audit 2024-09-12T23:40:17.905429+0000 mon.b (mon.1) 70 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:40:19.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[32189]: audit 2024-09-12T23:40:17.908849+0000 mon.b (mon.1) 71 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:40:19.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[32189]: audit 2024-09-12T23:40:17.909282+0000 mon.b (mon.1) 72 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:40:19.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[32189]: audit 2024-09-12T23:40:17.909651+0000 mon.b (mon.1) 73 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:40:19.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[32189]: audit 2024-09-12T23:40:17.912990+0000 mon.b (mon.1) 74 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:40:19.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[32189]: audit 2024-09-12T23:40:17.913452+0000 mon.b (mon.1) 75 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:40:19.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[32189]: audit 2024-09-12T23:40:17.913830+0000 mon.b (mon.1) 76 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:40:19.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[32189]: audit 2024-09-12T23:40:17.934286+0000 mon.a (mon.0) 1263 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:40:19.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[32189]: audit 2024-09-12T23:40:17.934395+0000 mon.a (mon.0) 1264 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:40:19.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[32189]: audit 2024-09-12T23:40:17.934922+0000 mon.b (mon.1) 77 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:40:19.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[32189]: audit 2024-09-12T23:40:17.935301+0000 mon.b (mon.1) 78 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:40:19.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[32189]: audit 2024-09-12T23:40:17.935632+0000 mon.b (mon.1) 79 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:40:19.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[32189]: audit 2024-09-12T23:40:17.937605+0000 mon.b (mon.1) 80 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:40:19.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[22601]: audit 2024-09-12T23:40:17.937688+0000 mon.b (mon.1) 81 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:40:19.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[22601]: audit 2024-09-12T23:40:17.938110+0000 mon.b (mon.1) 82 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:40:19.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[22601]: audit 2024-09-12T23:40:17.938370+0000 mon.b (mon.1) 83 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:40:19.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[22601]: audit 2024-09-12T23:40:17.938658+0000 mon.b (mon.1) 84 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:40:19.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[22601]: audit 2024-09-12T23:40:17.941207+0000 mon.c (mon.2) 67 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:40:19.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[22601]: audit 2024-09-12T23:40:17.941305+0000 mon.c (mon.2) 68 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:40:19.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[22601]: audit 2024-09-12T23:40:17.941564+0000 mon.b (mon.1) 85 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:40:19.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[22601]: audit 2024-09-12T23:40:17.941891+0000 mon.b (mon.1) 86 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:40:19.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[22601]: audit 2024-09-12T23:40:17.942242+0000 mon.b (mon.1) 87 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:40:19.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[32189]: audit 2024-09-12T23:40:17.937688+0000 mon.b (mon.1) 81 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:40:19.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[32189]: audit 2024-09-12T23:40:17.938110+0000 mon.b (mon.1) 82 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:40:19.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[32189]: audit 2024-09-12T23:40:17.938370+0000 mon.b (mon.1) 83 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:40:19.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[32189]: audit 2024-09-12T23:40:17.938658+0000 mon.b (mon.1) 84 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:40:19.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[32189]: audit 2024-09-12T23:40:17.941207+0000 mon.c (mon.2) 67 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:40:19.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[32189]: audit 2024-09-12T23:40:17.941305+0000 mon.c (mon.2) 68 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:40:19.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[32189]: audit 2024-09-12T23:40:17.941564+0000 mon.b (mon.1) 85 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:40:19.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[32189]: audit 2024-09-12T23:40:17.941891+0000 mon.b (mon.1) 86 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:40:19.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:18 smithi136 bash[32189]: audit 2024-09-12T23:40:17.942242+0000 mon.b (mon.1) 87 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:40:19.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:18 smithi181 bash[24776]: cluster 2024-09-12T23:40:17.558610+0000 mgr.x (mgr.24517) 14 : cluster [DBG] pgmap v17: 121 pgs: 121 active+clean; 100 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 3.0 MiB/s wr, 57 op/s 2024-09-12T23:40:19.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:18 smithi181 bash[24776]: cluster 2024-09-12T23:40:17.644169+0000 mon.a (mon.0) 1262 : cluster [DBG] osdmap e182: 8 total, 8 up, 8 in 2024-09-12T23:40:19.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:18 smithi181 bash[24776]: audit 2024-09-12T23:40:17.777265+0000 mon.b (mon.1) 66 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:40:19.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:18 smithi181 bash[24776]: audit 2024-09-12T23:40:17.796619+0000 mon.b (mon.1) 67 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-12T23:40:19.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:18 smithi181 bash[24776]: audit 2024-09-12T23:40:17.904483+0000 mon.b (mon.1) 68 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:40:19.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:18 smithi181 bash[24776]: audit 2024-09-12T23:40:17.905006+0000 mon.b (mon.1) 69 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:40:19.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:18 smithi181 bash[24776]: audit 2024-09-12T23:40:17.905429+0000 mon.b (mon.1) 70 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:40:19.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:18 smithi181 bash[24776]: audit 2024-09-12T23:40:17.908849+0000 mon.b (mon.1) 71 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:40:19.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:18 smithi181 bash[24776]: audit 2024-09-12T23:40:17.909282+0000 mon.b (mon.1) 72 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:40:19.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:18 smithi181 bash[24776]: audit 2024-09-12T23:40:17.909651+0000 mon.b (mon.1) 73 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:40:19.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:18 smithi181 bash[24776]: audit 2024-09-12T23:40:17.912990+0000 mon.b (mon.1) 74 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:40:19.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:18 smithi181 bash[24776]: audit 2024-09-12T23:40:17.913452+0000 mon.b (mon.1) 75 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:40:19.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:18 smithi181 bash[24776]: audit 2024-09-12T23:40:17.913830+0000 mon.b (mon.1) 76 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:40:19.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:18 smithi181 bash[24776]: audit 2024-09-12T23:40:17.934286+0000 mon.a (mon.0) 1263 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:40:19.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:18 smithi181 bash[24776]: audit 2024-09-12T23:40:17.934395+0000 mon.a (mon.0) 1264 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:40:19.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:18 smithi181 bash[24776]: audit 2024-09-12T23:40:17.934922+0000 mon.b (mon.1) 77 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:40:19.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:18 smithi181 bash[24776]: audit 2024-09-12T23:40:17.935301+0000 mon.b (mon.1) 78 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:40:19.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:18 smithi181 bash[24776]: audit 2024-09-12T23:40:17.935632+0000 mon.b (mon.1) 79 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:40:19.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:18 smithi181 bash[24776]: audit 2024-09-12T23:40:17.937605+0000 mon.b (mon.1) 80 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:40:19.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:18 smithi181 bash[24776]: audit 2024-09-12T23:40:17.937688+0000 mon.b (mon.1) 81 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:40:19.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:18 smithi181 bash[24776]: audit 2024-09-12T23:40:17.938110+0000 mon.b (mon.1) 82 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:40:19.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:18 smithi181 bash[24776]: audit 2024-09-12T23:40:17.938370+0000 mon.b (mon.1) 83 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:40:19.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:18 smithi181 bash[24776]: audit 2024-09-12T23:40:17.938658+0000 mon.b (mon.1) 84 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:40:19.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:18 smithi181 bash[24776]: audit 2024-09-12T23:40:17.941207+0000 mon.c (mon.2) 67 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:40:19.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:18 smithi181 bash[24776]: audit 2024-09-12T23:40:17.941305+0000 mon.c (mon.2) 68 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:40:19.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:18 smithi181 bash[24776]: audit 2024-09-12T23:40:17.941564+0000 mon.b (mon.1) 85 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:40:19.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:18 smithi181 bash[24776]: audit 2024-09-12T23:40:17.941891+0000 mon.b (mon.1) 86 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:40:19.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:18 smithi181 bash[24776]: audit 2024-09-12T23:40:17.942242+0000 mon.b (mon.1) 87 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:40:19.671 INFO:tasks.rados.rados.0.smithi136.stdout:687: done (3 left) 2024-09-12T23:40:19.672 INFO:tasks.rados.rados.0.smithi136.stdout:688: done (2 left) 2024-09-12T23:40:19.672 INFO:tasks.rados.rados.0.smithi136.stdout:689: done (1 left) 2024-09-12T23:40:19.672 INFO:tasks.rados.rados.0.smithi136.stdout:690: done (0 left) 2024-09-12T23:40:19.672 INFO:tasks.rados.rados.0.smithi136.stdout:691: delete oid 35 current snap is 58 2024-09-12T23:40:19.684 INFO:tasks.rados.rados.0.smithi136.stdout:691: done (0 left) 2024-09-12T23:40:19.684 INFO:tasks.rados.rados.0.smithi136.stdout:692: snap_create 2024-09-12T23:40:20.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:19 smithi136 bash[22601]: cluster 2024-09-12T23:40:18.658474+0000 mon.a (mon.0) 1265 : cluster [DBG] osdmap e183: 8 total, 8 up, 8 in 2024-09-12T23:40:20.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:19 smithi136 bash[32189]: cluster 2024-09-12T23:40:18.658474+0000 mon.a (mon.0) 1265 : cluster [DBG] osdmap e183: 8 total, 8 up, 8 in 2024-09-12T23:40:20.093 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:19 smithi181 bash[24776]: cluster 2024-09-12T23:40:18.658474+0000 mon.a (mon.0) 1265 : cluster [DBG] osdmap e183: 8 total, 8 up, 8 in 2024-09-12T23:40:20.480 INFO:tasks.rados.rados.0.smithi136.stdout:692: done (0 left) 2024-09-12T23:40:20.481 INFO:tasks.rados.rados.0.smithi136.stdout:693: snap_create 2024-09-12T23:40:21.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:20 smithi136 bash[22601]: cluster 2024-09-12T23:40:19.559289+0000 mgr.x (mgr.24517) 15 : cluster [DBG] pgmap v20: 121 pgs: 121 active+clean; 108 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.7 MiB/s wr, 37 op/s 2024-09-12T23:40:21.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:20 smithi136 bash[22601]: cluster 2024-09-12T23:40:19.675672+0000 mon.a (mon.0) 1266 : cluster [DBG] osdmap e184: 8 total, 8 up, 8 in 2024-09-12T23:40:21.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:20 smithi136 bash[22601]: cluster 2024-09-12T23:40:19.676642+0000 mon.a (mon.0) 1267 : cluster [DBG] mgrmap e29: x(active, since 12s), standbys: y 2024-09-12T23:40:21.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:20 smithi136 bash[22601]: cluster 2024-09-12T23:40:20.485251+0000 mon.a (mon.0) 1268 : cluster [DBG] osdmap e185: 8 total, 8 up, 8 in 2024-09-12T23:40:21.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:20 smithi136 bash[32189]: cluster 2024-09-12T23:40:19.559289+0000 mgr.x (mgr.24517) 15 : cluster [DBG] pgmap v20: 121 pgs: 121 active+clean; 108 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.7 MiB/s wr, 37 op/s 2024-09-12T23:40:21.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:20 smithi136 bash[32189]: cluster 2024-09-12T23:40:19.675672+0000 mon.a (mon.0) 1266 : cluster [DBG] osdmap e184: 8 total, 8 up, 8 in 2024-09-12T23:40:21.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:20 smithi136 bash[32189]: cluster 2024-09-12T23:40:19.676642+0000 mon.a (mon.0) 1267 : cluster [DBG] mgrmap e29: x(active, since 12s), standbys: y 2024-09-12T23:40:21.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:20 smithi136 bash[32189]: cluster 2024-09-12T23:40:20.485251+0000 mon.a (mon.0) 1268 : cluster [DBG] osdmap e185: 8 total, 8 up, 8 in 2024-09-12T23:40:21.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:20 smithi181 bash[24776]: cluster 2024-09-12T23:40:19.559289+0000 mgr.x (mgr.24517) 15 : cluster [DBG] pgmap v20: 121 pgs: 121 active+clean; 108 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.7 MiB/s wr, 37 op/s 2024-09-12T23:40:21.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:20 smithi181 bash[24776]: cluster 2024-09-12T23:40:19.675672+0000 mon.a (mon.0) 1266 : cluster [DBG] osdmap e184: 8 total, 8 up, 8 in 2024-09-12T23:40:21.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:20 smithi181 bash[24776]: cluster 2024-09-12T23:40:19.676642+0000 mon.a (mon.0) 1267 : cluster [DBG] mgrmap e29: x(active, since 12s), standbys: y 2024-09-12T23:40:21.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:20 smithi181 bash[24776]: cluster 2024-09-12T23:40:20.485251+0000 mon.a (mon.0) 1268 : cluster [DBG] osdmap e185: 8 total, 8 up, 8 in 2024-09-12T23:40:21.483 INFO:tasks.rados.rados.0.smithi136.stdout:693: done (0 left) 2024-09-12T23:40:21.484 INFO:tasks.rados.rados.0.smithi136.stdout:694: read oid 22 snap -1 2024-09-12T23:40:21.484 INFO:tasks.rados.rados.0.smithi136.stdout:694: expect (ObjNum 21 snap 0 seq_num 21) 2024-09-12T23:40:21.484 INFO:tasks.rados.rados.0.smithi136.stdout:695: copy_from oid 40 from oid 5 current snap is 60 2024-09-12T23:40:21.484 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 20 current snap is 60 2024-09-12T23:40:21.484 INFO:tasks.rados.rados.0.smithi136.stdout:696: seq_num 240 ranges {573440=425984,999424=589824,1589248=483328,2072576=131072} 2024-09-12T23:40:21.501 INFO:tasks.rados.rados.0.smithi136.stdout:696: writing smithi13667081-20 from 573440 to 999424 tid 1 2024-09-12T23:40:21.511 INFO:tasks.rados.rados.0.smithi136.stdout:696: writing smithi13667081-20 from 999424 to 1589248 tid 2 2024-09-12T23:40:21.519 INFO:tasks.rados.rados.0.smithi136.stdout:696: writing smithi13667081-20 from 1589248 to 2072576 tid 3 2024-09-12T23:40:21.521 INFO:tasks.rados.rados.0.smithi136.stdout:696: writing smithi13667081-20 from 2072576 to 2203648 tid 4 2024-09-12T23:40:21.521 INFO:tasks.rados.rados.0.smithi136.stdout:697: snap_remove snap 51 2024-09-12T23:40:22.494 INFO:tasks.rados.rados.0.smithi136.stdout:697: done (3 left) 2024-09-12T23:40:22.494 INFO:tasks.rados.rados.0.smithi136.stdout:698: delete oid 24 current snap is 60 2024-09-12T23:40:22.494 INFO:tasks.rados.rados.0.smithi136.stdout:695: finishing copy_from racing read to smithi13667081-40 2024-09-12T23:40:22.494 INFO:tasks.rados.rados.0.smithi136.stdout:695: finishing copy_from to smithi13667081-40 2024-09-12T23:40:22.495 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 40 v 375 (ObjNum 33 snap 0 seq_num 4242616896) dirty exists 2024-09-12T23:40:22.495 INFO:tasks.rados.rados.0.smithi136.stdout:694: expect (ObjNum 21 snap 0 seq_num 21) 2024-09-12T23:40:22.544 INFO:tasks.rados.rados.0.smithi136.stdout:696: finishing write tid 1 to smithi13667081-20 2024-09-12T23:40:22.544 INFO:tasks.rados.rados.0.smithi136.stdout:696: finishing write tid 2 to smithi13667081-20 2024-09-12T23:40:22.545 INFO:tasks.rados.rados.0.smithi136.stdout:696: finishing write tid 3 to smithi13667081-20 2024-09-12T23:40:22.545 INFO:tasks.rados.rados.0.smithi136.stdout:696: finishing write tid 4 to smithi13667081-20 2024-09-12T23:40:22.545 INFO:tasks.rados.rados.0.smithi136.stdout:696: finishing write tid 5 to smithi13667081-20 2024-09-12T23:40:22.545 INFO:tasks.rados.rados.0.smithi136.stdout:696: finishing write tid 6 to smithi13667081-20 2024-09-12T23:40:22.545 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 20 v 85 (ObjNum 240 snap 60 seq_num 240) dirty exists 2024-09-12T23:40:22.545 INFO:tasks.rados.rados.0.smithi136.stdout:696: left oid 20 (ObjNum 240 snap 60 seq_num 240) 2024-09-12T23:40:22.545 INFO:tasks.rados.rados.0.smithi136.stdout:694: done (3 left) 2024-09-12T23:40:22.545 INFO:tasks.rados.rados.0.smithi136.stdout:695: done (2 left) 2024-09-12T23:40:22.545 INFO:tasks.rados.rados.0.smithi136.stdout:696: done (1 left) 2024-09-12T23:40:22.545 INFO:tasks.rados.rados.0.smithi136.stdout:698: done (0 left) 2024-09-12T23:40:22.545 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 24 current snap is 60 2024-09-12T23:40:22.546 INFO:tasks.rados.rados.0.smithi136.stdout:699: seq_num 242 ranges {0=458752,458752=753664,1212416=450560} 2024-09-12T23:40:22.550 INFO:tasks.rados.rados.0.smithi136.stdout:699: writing smithi13667081-24 from 0 to 458752 tid 1 2024-09-12T23:40:22.560 INFO:tasks.rados.rados.0.smithi136.stdout:699: writing smithi13667081-24 from 458752 to 1212416 tid 2 2024-09-12T23:40:22.566 INFO:tasks.rados.rados.0.smithi136.stdout:699: writing smithi13667081-24 from 1212416 to 1662976 tid 3 2024-09-12T23:40:22.566 INFO:tasks.rados.rados.0.smithi136.stdout:700: copy_from oid 49 from oid 22 current snap is 60 2024-09-12T23:40:22.566 INFO:tasks.rados.rados.0.smithi136.stdout:699: finishing write tid 1 to smithi13667081-24 2024-09-12T23:40:22.567 INFO:tasks.rados.rados.0.smithi136.stdout:699: finishing write tid 2 to smithi13667081-24 2024-09-12T23:40:22.567 INFO:tasks.rados.rados.0.smithi136.stdout:701: delete oid 35 current snap is 60 2024-09-12T23:40:22.569 INFO:tasks.rados.rados.0.smithi136.stdout:701: done (2 left) 2024-09-12T23:40:22.569 INFO:tasks.rados.rados.0.smithi136.stdout:702: read oid 11 snap -1 2024-09-12T23:40:22.569 INFO:tasks.rados.rados.0.smithi136.stdout:702: expect deleted 2024-09-12T23:40:22.569 INFO:tasks.rados.rados.0.smithi136.stdout:703: snap_remove snap 57 2024-09-12T23:40:22.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:22 smithi136 bash[22601]: cluster 2024-09-12T23:40:21.487823+0000 mon.a (mon.0) 1269 : cluster [DBG] osdmap e186: 8 total, 8 up, 8 in 2024-09-12T23:40:22.755 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:22 smithi136 bash[32189]: cluster 2024-09-12T23:40:21.487823+0000 mon.a (mon.0) 1269 : cluster [DBG] osdmap e186: 8 total, 8 up, 8 in 2024-09-12T23:40:22.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:22 smithi181 bash[24776]: cluster 2024-09-12T23:40:21.487823+0000 mon.a (mon.0) 1269 : cluster [DBG] osdmap e186: 8 total, 8 up, 8 in 2024-09-12T23:40:23.505 INFO:tasks.rados.rados.0.smithi136.stdout:703: done (3 left) 2024-09-12T23:40:23.506 INFO:tasks.rados.rados.0.smithi136.stdout:704: snap_remove snap 52 2024-09-12T23:40:23.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:23 smithi136 bash[22601]: cluster 2024-09-12T23:40:21.559924+0000 mgr.x (mgr.24517) 16 : cluster [DBG] pgmap v24: 121 pgs: 121 active+clean; 108 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 4.8 MiB/s wr, 38 op/s 2024-09-12T23:40:23.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:23 smithi136 bash[22601]: cluster 2024-09-12T23:40:22.498570+0000 mon.a (mon.0) 1270 : cluster [DBG] osdmap e187: 8 total, 8 up, 8 in 2024-09-12T23:40:23.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:23 smithi136 bash[22601]: audit 2024-09-12T23:40:22.735679+0000 mon.b (mon.1) 88 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:40:23.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:23 smithi136 bash[32189]: cluster 2024-09-12T23:40:21.559924+0000 mgr.x (mgr.24517) 16 : cluster [DBG] pgmap v24: 121 pgs: 121 active+clean; 108 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 4.8 MiB/s wr, 38 op/s 2024-09-12T23:40:23.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:23 smithi136 bash[32189]: cluster 2024-09-12T23:40:22.498570+0000 mon.a (mon.0) 1270 : cluster [DBG] osdmap e187: 8 total, 8 up, 8 in 2024-09-12T23:40:23.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:23 smithi136 bash[32189]: audit 2024-09-12T23:40:22.735679+0000 mon.b (mon.1) 88 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:40:23.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:23 smithi181 bash[24776]: cluster 2024-09-12T23:40:21.559924+0000 mgr.x (mgr.24517) 16 : cluster [DBG] pgmap v24: 121 pgs: 121 active+clean; 108 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 4.8 MiB/s wr, 38 op/s 2024-09-12T23:40:23.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:23 smithi181 bash[24776]: cluster 2024-09-12T23:40:22.498570+0000 mon.a (mon.0) 1270 : cluster [DBG] osdmap e187: 8 total, 8 up, 8 in 2024-09-12T23:40:23.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:23 smithi181 bash[24776]: audit 2024-09-12T23:40:22.735679+0000 mon.b (mon.1) 88 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:40:24.514 INFO:tasks.rados.rados.0.smithi136.stdout:704: done (3 left) 2024-09-12T23:40:24.515 INFO:tasks.rados.rados.0.smithi136.stdout:705: setattr oid 12 current snap is 60 2024-09-12T23:40:24.515 INFO:tasks.rados.rados.0.smithi136.stdout:699: finishing write tid 3 to smithi13667081-24 2024-09-12T23:40:24.515 INFO:tasks.rados.rados.0.smithi136.stdout:699: finishing write tid 4 to smithi13667081-24 2024-09-12T23:40:24.515 INFO:tasks.rados.rados.0.smithi136.stdout:699: finishing write tid 5 to smithi13667081-24 2024-09-12T23:40:24.515 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 24 v 178 (ObjNum 242 snap 60 seq_num 242) dirty exists 2024-09-12T23:40:24.515 INFO:tasks.rados.rados.0.smithi136.stdout:699: left oid 24 (ObjNum 242 snap 60 seq_num 242) 2024-09-12T23:40:24.515 INFO:tasks.rados.rados.0.smithi136.stdout:700: finishing copy_from racing read to smithi13667081-49 2024-09-12T23:40:24.515 INFO:tasks.rados.rados.0.smithi136.stdout:700: finishing copy_from to smithi13667081-49 2024-09-12T23:40:24.515 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 49 v 153 (ObjNum 21 snap 0 seq_num 21) dirty exists 2024-09-12T23:40:24.517 INFO:tasks.rados.rados.0.smithi136.stdout:699: done (3 left) 2024-09-12T23:40:24.517 INFO:tasks.rados.rados.0.smithi136.stdout:700: done (2 left) 2024-09-12T23:40:24.517 INFO:tasks.rados.rados.0.smithi136.stdout:702: done (1 left) 2024-09-12T23:40:24.517 INFO:tasks.rados.rados.0.smithi136.stdout:706: copy_from oid 15 from oid 10 current snap is 60 2024-09-12T23:40:24.518 INFO:tasks.rados.rados.0.smithi136.stdout:707: rollback oid 19 current snap is 60 2024-09-12T23:40:24.518 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 19 to 59 2024-09-12T23:40:24.518 INFO:tasks.rados.rados.0.smithi136.stdout:708: read oid 36 snap -1 2024-09-12T23:40:24.518 INFO:tasks.rados.rados.0.smithi136.stdout:708: expect (ObjNum 217 snap 55 seq_num 217) 2024-09-12T23:40:24.518 INFO:tasks.rados.rados.0.smithi136.stdout:709: snap_remove snap 58 2024-09-12T23:40:24.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:24 smithi181 bash[24776]: cluster 2024-09-12T23:40:23.508276+0000 mon.a (mon.0) 1271 : cluster [DBG] osdmap e188: 8 total, 8 up, 8 in 2024-09-12T23:40:25.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:24 smithi136 bash[22601]: cluster 2024-09-12T23:40:23.508276+0000 mon.a (mon.0) 1271 : cluster [DBG] osdmap e188: 8 total, 8 up, 8 in 2024-09-12T23:40:25.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:24 smithi136 bash[32189]: cluster 2024-09-12T23:40:23.508276+0000 mon.a (mon.0) 1271 : cluster [DBG] osdmap e188: 8 total, 8 up, 8 in 2024-09-12T23:40:25.513 INFO:tasks.rados.rados.0.smithi136.stdout:709: done (4 left) 2024-09-12T23:40:25.513 INFO:tasks.rados.rados.0.smithi136.stdout:710: read oid 25 snap -1 2024-09-12T23:40:25.513 INFO:tasks.rados.rados.0.smithi136.stdout:710: expect (ObjNum 144 snap 36 seq_num 144) 2024-09-12T23:40:25.514 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 32 current snap is 60 2024-09-12T23:40:25.514 INFO:tasks.rados.rados.0.smithi136.stdout:711: seq_num 244 ranges {1122304=491520,1613824=573440,2187264=458752,2646016=172032} 2024-09-12T23:40:25.520 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:40:25 smithi181 bash[52367]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:40:25] "GET /metrics HTTP/1.1" 200 36148 "" "Prometheus/2.43.0" 2024-09-12T23:40:25.529 INFO:tasks.rados.rados.0.smithi136.stdout:711: writing smithi13667081-32 from 1122304 to 1613824 tid 1 2024-09-12T23:40:25.534 INFO:tasks.rados.rados.0.smithi136.stdout:711: writing smithi13667081-32 from 1613824 to 2187264 tid 2 2024-09-12T23:40:25.538 INFO:tasks.rados.rados.0.smithi136.stdout:711: writing smithi13667081-32 from 2187264 to 2646016 tid 3 2024-09-12T23:40:25.540 INFO:tasks.rados.rados.0.smithi136.stdout:711: writing smithi13667081-32 from 2646016 to 2818048 tid 4 2024-09-12T23:40:25.541 INFO:tasks.rados.rados.0.smithi136.stdout:712: read oid 16 snap -1 2024-09-12T23:40:25.541 INFO:tasks.rados.rados.0.smithi136.stdout:712: expect (ObjNum 228 snap 56 seq_num 228) 2024-09-12T23:40:25.541 INFO:tasks.rados.rados.0.smithi136.stdout:713: snap_create 2024-09-12T23:40:25.541 INFO:tasks.rados.rados.0.smithi136.stdout:706: finishing copy_from to smithi13667081-15 2024-09-12T23:40:25.541 INFO:tasks.rados.rados.0.smithi136.stdout:706: got expected ENOENT (src dne) 2024-09-12T23:40:25.541 INFO:tasks.rados.rados.0.smithi136.stdout:706: finishing copy_from racing read to smithi13667081-15 2024-09-12T23:40:25.541 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 12 v 429 (ObjNum 3345 snap 0 seq_num 4243120368) dirty exists 2024-09-12T23:40:25.541 INFO:tasks.rados.rados.0.smithi136.stdout:707: finishing rollback tid 0 to smithi13667081-19 2024-09-12T23:40:25.541 INFO:tasks.rados.rados.0.smithi136.stdout:707: finishing rollback tid 1 to smithi13667081-19 2024-09-12T23:40:25.541 INFO:tasks.rados.rados.0.smithi136.stdout:707: finishing rollback tid 2 to smithi13667081-19 2024-09-12T23:40:25.541 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 19 v 266 (ObjNum 211 snap 53 seq_num 211) dirty exists 2024-09-12T23:40:25.541 INFO:tasks.rados.rados.0.smithi136.stdout:708: expect (ObjNum 217 snap 55 seq_num 217) 2024-09-12T23:40:25.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:25 smithi181 bash[24776]: cluster 2024-09-12T23:40:23.560504+0000 mgr.x (mgr.24517) 17 : cluster [DBG] pgmap v27: 121 pgs: 121 active+clean; 108 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:40:25.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:25 smithi181 bash[24776]: cluster 2024-09-12T23:40:24.518696+0000 mon.a (mon.0) 1272 : cluster [DBG] osdmap e189: 8 total, 8 up, 8 in 2024-09-12T23:40:25.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:25 smithi181 bash[24776]: audit 2024-09-12T23:40:24.730872+0000 mon.a (mon.0) 1273 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:25.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:25 smithi181 bash[24776]: audit 2024-09-12T23:40:24.745535+0000 mon.a (mon.0) 1274 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:25.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:25 smithi181 bash[24776]: audit 2024-09-12T23:40:24.749519+0000 mon.b (mon.1) 89 : audit [INF] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:40:25.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:25 smithi181 bash[24776]: audit 2024-09-12T23:40:24.750054+0000 mon.a (mon.0) 1275 : audit [INF] from='mgr.24517 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:40:25.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:25 smithi181 bash[24776]: audit 2024-09-12T23:40:24.752565+0000 mon.b (mon.1) 90 : audit [INF] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:40:25.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:25 smithi181 bash[24776]: audit 2024-09-12T23:40:24.753060+0000 mon.a (mon.0) 1276 : audit [INF] from='mgr.24517 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:40:25.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:25 smithi181 bash[24776]: audit 2024-09-12T23:40:24.755418+0000 mon.b (mon.1) 91 : audit [INF] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:40:25.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:25 smithi181 bash[24776]: audit 2024-09-12T23:40:24.756073+0000 mon.a (mon.0) 1277 : audit [INF] from='mgr.24517 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:40:25.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:25 smithi181 bash[24776]: audit 2024-09-12T23:40:24.758566+0000 mon.b (mon.1) 92 : audit [INF] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:40:25.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:25 smithi181 bash[24776]: audit 2024-09-12T23:40:24.759266+0000 mon.a (mon.0) 1278 : audit [INF] from='mgr.24517 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:40:25.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:25 smithi181 bash[24776]: audit 2024-09-12T23:40:24.773236+0000 mon.a (mon.0) 1279 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:25.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:25 smithi181 bash[24776]: audit 2024-09-12T23:40:24.775870+0000 mon.b (mon.1) 93 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:40:25.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:25 smithi181 bash[24776]: audit 2024-09-12T23:40:24.777386+0000 mon.b (mon.1) 94 : audit [INF] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:40:25.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:25 smithi181 bash[24776]: audit 2024-09-12T23:40:25.344859+0000 mon.a (mon.0) 1280 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:25.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:25 smithi181 bash[24776]: audit 2024-09-12T23:40:25.351811+0000 mon.a (mon.0) 1281 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:25.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:25 smithi181 bash[24776]: audit 2024-09-12T23:40:25.374321+0000 mon.a (mon.0) 1282 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:25.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:25 smithi181 bash[24776]: audit 2024-09-12T23:40:25.383022+0000 mon.a (mon.0) 1283 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:25.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:25 smithi181 bash[24776]: audit 2024-09-12T23:40:25.389257+0000 mon.a (mon.0) 1284 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:25.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:25 smithi181 bash[24776]: audit 2024-09-12T23:40:25.432234+0000 mon.a (mon.0) 1285 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:25.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:25 smithi181 bash[24776]: audit 2024-09-12T23:40:25.441153+0000 mon.a (mon.0) 1286 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:25.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:25 smithi181 bash[24776]: audit 2024-09-12T23:40:25.449290+0000 mon.a (mon.0) 1287 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:25.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:25 smithi181 bash[24776]: audit 2024-09-12T23:40:25.457752+0000 mon.a (mon.0) 1288 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:25.847 INFO:tasks.rados.rados.0.smithi136.stdout:710: expect (ObjNum 144 snap 36 seq_num 144) 2024-09-12T23:40:25.847 INFO:tasks.rados.rados.0.smithi136.stdout:711: finishing write tid 1 to smithi13667081-32 2024-09-12T23:40:25.847 INFO:tasks.rados.rados.0.smithi136.stdout:711: finishing write tid 2 to smithi13667081-32 2024-09-12T23:40:25.847 INFO:tasks.rados.rados.0.smithi136.stdout:711: finishing write tid 3 to smithi13667081-32 2024-09-12T23:40:25.847 INFO:tasks.rados.rados.0.smithi136.stdout:711: finishing write tid 4 to smithi13667081-32 2024-09-12T23:40:25.847 INFO:tasks.rados.rados.0.smithi136.stdout:711: finishing write tid 5 to smithi13667081-32 2024-09-12T23:40:25.847 INFO:tasks.rados.rados.0.smithi136.stdout:712: expect (ObjNum 228 snap 56 seq_num 228) 2024-09-12T23:40:25.866 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:25 smithi136 systemd[1]: Stopping Ceph node-exporter.a for da58a47c-715d-11ef-bceb-c7b262605968... 2024-09-12T23:40:25.866 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:25 smithi136 bash[22601]: cluster 2024-09-12T23:40:23.560504+0000 mgr.x (mgr.24517) 17 : cluster [DBG] pgmap v27: 121 pgs: 121 active+clean; 108 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:40:25.866 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:25 smithi136 bash[22601]: cluster 2024-09-12T23:40:24.518696+0000 mon.a (mon.0) 1272 : cluster [DBG] osdmap e189: 8 total, 8 up, 8 in 2024-09-12T23:40:25.866 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:25 smithi136 bash[22601]: audit 2024-09-12T23:40:24.730872+0000 mon.a (mon.0) 1273 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:25.866 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:25 smithi136 bash[22601]: audit 2024-09-12T23:40:24.745535+0000 mon.a (mon.0) 1274 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:25.866 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:25 smithi136 bash[22601]: audit 2024-09-12T23:40:24.749519+0000 mon.b (mon.1) 89 : audit [INF] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:40:25.866 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:25 smithi136 bash[22601]: audit 2024-09-12T23:40:24.750054+0000 mon.a (mon.0) 1275 : audit [INF] from='mgr.24517 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:40:25.867 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:25 smithi136 bash[22601]: audit 2024-09-12T23:40:24.752565+0000 mon.b (mon.1) 90 : audit [INF] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:40:25.867 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:25 smithi136 bash[22601]: audit 2024-09-12T23:40:24.753060+0000 mon.a (mon.0) 1276 : audit [INF] from='mgr.24517 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:40:25.867 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:25 smithi136 bash[22601]: audit 2024-09-12T23:40:24.755418+0000 mon.b (mon.1) 91 : audit [INF] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:40:25.867 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:25 smithi136 bash[22601]: audit 2024-09-12T23:40:24.756073+0000 mon.a (mon.0) 1277 : audit [INF] from='mgr.24517 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:40:25.867 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:25 smithi136 bash[22601]: audit 2024-09-12T23:40:24.758566+0000 mon.b (mon.1) 92 : audit [INF] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:40:25.867 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:25 smithi136 bash[22601]: audit 2024-09-12T23:40:24.759266+0000 mon.a (mon.0) 1278 : audit [INF] from='mgr.24517 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:40:25.867 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:25 smithi136 bash[22601]: audit 2024-09-12T23:40:24.773236+0000 mon.a (mon.0) 1279 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:25.867 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:25 smithi136 bash[22601]: audit 2024-09-12T23:40:24.775870+0000 mon.b (mon.1) 93 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:40:25.867 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:25 smithi136 bash[22601]: audit 2024-09-12T23:40:24.777386+0000 mon.b (mon.1) 94 : audit [INF] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:40:25.867 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:25 smithi136 bash[22601]: audit 2024-09-12T23:40:25.344859+0000 mon.a (mon.0) 1280 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:25.867 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:25 smithi136 bash[22601]: audit 2024-09-12T23:40:25.351811+0000 mon.a (mon.0) 1281 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:25.867 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:25 smithi136 bash[22601]: audit 2024-09-12T23:40:25.374321+0000 mon.a (mon.0) 1282 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:25.867 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:25 smithi136 bash[22601]: audit 2024-09-12T23:40:25.383022+0000 mon.a (mon.0) 1283 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:25.867 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:25 smithi136 bash[22601]: audit 2024-09-12T23:40:25.389257+0000 mon.a (mon.0) 1284 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:25.868 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:25 smithi136 bash[22601]: audit 2024-09-12T23:40:25.432234+0000 mon.a (mon.0) 1285 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:25.868 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:25 smithi136 bash[32189]: cluster 2024-09-12T23:40:23.560504+0000 mgr.x (mgr.24517) 17 : cluster [DBG] pgmap v27: 121 pgs: 121 active+clean; 108 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:40:25.868 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:25 smithi136 bash[32189]: cluster 2024-09-12T23:40:24.518696+0000 mon.a (mon.0) 1272 : cluster [DBG] osdmap e189: 8 total, 8 up, 8 in 2024-09-12T23:40:25.868 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:25 smithi136 bash[32189]: audit 2024-09-12T23:40:24.730872+0000 mon.a (mon.0) 1273 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:25.868 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:25 smithi136 bash[32189]: audit 2024-09-12T23:40:24.745535+0000 mon.a (mon.0) 1274 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:25.868 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:25 smithi136 bash[32189]: audit 2024-09-12T23:40:24.749519+0000 mon.b (mon.1) 89 : audit [INF] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:40:25.868 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:25 smithi136 bash[32189]: audit 2024-09-12T23:40:24.750054+0000 mon.a (mon.0) 1275 : audit [INF] from='mgr.24517 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:40:25.868 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:25 smithi136 bash[32189]: audit 2024-09-12T23:40:24.752565+0000 mon.b (mon.1) 90 : audit [INF] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:40:25.868 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:25 smithi136 bash[32189]: audit 2024-09-12T23:40:24.753060+0000 mon.a (mon.0) 1276 : audit [INF] from='mgr.24517 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:40:25.868 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:25 smithi136 bash[32189]: audit 2024-09-12T23:40:24.755418+0000 mon.b (mon.1) 91 : audit [INF] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:40:25.869 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:25 smithi136 bash[32189]: audit 2024-09-12T23:40:24.756073+0000 mon.a (mon.0) 1277 : audit [INF] from='mgr.24517 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:40:25.869 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:25 smithi136 bash[32189]: audit 2024-09-12T23:40:24.758566+0000 mon.b (mon.1) 92 : audit [INF] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:40:25.869 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:25 smithi136 bash[32189]: audit 2024-09-12T23:40:24.759266+0000 mon.a (mon.0) 1278 : audit [INF] from='mgr.24517 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-12T23:40:25.869 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:25 smithi136 bash[32189]: audit 2024-09-12T23:40:24.773236+0000 mon.a (mon.0) 1279 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:25.869 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:25 smithi136 bash[32189]: audit 2024-09-12T23:40:24.775870+0000 mon.b (mon.1) 93 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:40:25.869 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:25 smithi136 bash[32189]: audit 2024-09-12T23:40:24.777386+0000 mon.b (mon.1) 94 : audit [INF] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:40:25.869 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:25 smithi136 bash[32189]: audit 2024-09-12T23:40:25.344859+0000 mon.a (mon.0) 1280 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:25.869 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:25 smithi136 bash[32189]: audit 2024-09-12T23:40:25.351811+0000 mon.a (mon.0) 1281 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:25.869 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:25 smithi136 bash[32189]: audit 2024-09-12T23:40:25.374321+0000 mon.a (mon.0) 1282 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:25.869 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:25 smithi136 bash[32189]: audit 2024-09-12T23:40:25.383022+0000 mon.a (mon.0) 1283 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:25.869 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:25 smithi136 bash[32189]: audit 2024-09-12T23:40:25.389257+0000 mon.a (mon.0) 1284 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:25.869 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:25 smithi136 bash[32189]: audit 2024-09-12T23:40:25.432234+0000 mon.a (mon.0) 1285 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:25.870 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:25 smithi136 bash[22601]: audit 2024-09-12T23:40:25.441153+0000 mon.a (mon.0) 1286 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:25.870 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:25 smithi136 bash[22601]: audit 2024-09-12T23:40:25.449290+0000 mon.a (mon.0) 1287 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:25.870 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:25 smithi136 bash[22601]: audit 2024-09-12T23:40:25.457752+0000 mon.a (mon.0) 1288 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:25.870 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:25 smithi136 bash[32189]: audit 2024-09-12T23:40:25.441153+0000 mon.a (mon.0) 1286 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:25.870 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:25 smithi136 bash[32189]: audit 2024-09-12T23:40:25.449290+0000 mon.a (mon.0) 1287 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:25.870 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:25 smithi136 bash[32189]: audit 2024-09-12T23:40:25.457752+0000 mon.a (mon.0) 1288 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:25.883 INFO:tasks.rados.rados.0.smithi136.stdout:711: finishing write tid 6 to smithi13667081-32 2024-09-12T23:40:25.884 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 32 v 136 (ObjNum 244 snap 60 seq_num 244) dirty exists 2024-09-12T23:40:25.884 INFO:tasks.rados.rados.0.smithi136.stdout:711: left oid 32 (ObjNum 244 snap 60 seq_num 244) 2024-09-12T23:40:26.505 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:26 smithi136 bash[22852]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:40:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-12T23:40:26.540 INFO:tasks.rados.rados.0.smithi136.stdout:705: done (7 left) 2024-09-12T23:40:26.540 INFO:tasks.rados.rados.0.smithi136.stdout:706: done (6 left) 2024-09-12T23:40:26.540 INFO:tasks.rados.rados.0.smithi136.stdout:707: done (5 left) 2024-09-12T23:40:26.540 INFO:tasks.rados.rados.0.smithi136.stdout:708: done (4 left) 2024-09-12T23:40:26.540 INFO:tasks.rados.rados.0.smithi136.stdout:710: done (3 left) 2024-09-12T23:40:26.540 INFO:tasks.rados.rados.0.smithi136.stdout:711: done (2 left) 2024-09-12T23:40:26.540 INFO:tasks.rados.rados.0.smithi136.stdout:712: done (1 left) 2024-09-12T23:40:26.540 INFO:tasks.rados.rados.0.smithi136.stdout:713: done (0 left) 2024-09-12T23:40:26.540 INFO:tasks.rados.rados.0.smithi136.stdout:714: setattr oid 5 current snap is 61 2024-09-12T23:40:26.542 INFO:tasks.rados.rados.0.smithi136.stdout:715: snap_create 2024-09-12T23:40:26.554 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 5 v 410 (ObjNum 3857 snap 0 seq_num 1427292912) dirty exists 2024-09-12T23:40:26.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:26 smithi181 bash[24776]: cephadm 2024-09-12T23:40:24.761257+0000 mgr.x (mgr.24517) 18 : cephadm [INF] Adjusting osd_memory_target on smithi181 to 2524M 2024-09-12T23:40:26.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:26 smithi181 bash[24776]: cephadm 2024-09-12T23:40:24.779589+0000 mgr.x (mgr.24517) 19 : cephadm [INF] Updating smithi136:/etc/ceph/ceph.conf 2024-09-12T23:40:26.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:26 smithi181 bash[24776]: cephadm 2024-09-12T23:40:24.780041+0000 mgr.x (mgr.24517) 20 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.conf 2024-09-12T23:40:26.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:26 smithi181 bash[24776]: cephadm 2024-09-12T23:40:24.960423+0000 mgr.x (mgr.24517) 21 : cephadm [INF] Updating smithi181:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-12T23:40:26.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:26 smithi181 bash[24776]: cephadm 2024-09-12T23:40:24.960590+0000 mgr.x (mgr.24517) 22 : cephadm [INF] Updating smithi136:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-12T23:40:26.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:26 smithi181 bash[24776]: cephadm 2024-09-12T23:40:25.085356+0000 mgr.x (mgr.24517) 23 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2024-09-12T23:40:26.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:26 smithi181 bash[24776]: cephadm 2024-09-12T23:40:25.103892+0000 mgr.x (mgr.24517) 24 : cephadm [INF] Updating smithi136:/etc/ceph/ceph.client.admin.keyring 2024-09-12T23:40:26.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:26 smithi181 bash[24776]: cephadm 2024-09-12T23:40:25.210776+0000 mgr.x (mgr.24517) 25 : cephadm [INF] Updating smithi181:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-12T23:40:26.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:26 smithi181 bash[24776]: cephadm 2024-09-12T23:40:25.240088+0000 mgr.x (mgr.24517) 26 : cephadm [INF] Updating smithi136:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-12T23:40:26.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:26 smithi181 bash[24776]: cephadm 2024-09-12T23:40:25.460997+0000 mgr.x (mgr.24517) 27 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2024-09-12T23:40:26.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:26 smithi181 bash[24776]: cephadm 2024-09-12T23:40:25.461604+0000 mgr.x (mgr.24517) 28 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi136 2024-09-12T23:40:26.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:26 smithi181 bash[24776]: cluster 2024-09-12T23:40:25.518133+0000 mon.a (mon.0) 1289 : cluster [DBG] osdmap e190: 8 total, 8 up, 8 in 2024-09-12T23:40:26.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:26 smithi181 bash[24776]: cluster 2024-09-12T23:40:25.561153+0000 mgr.x (mgr.24517) 29 : cluster [DBG] pgmap v30: 121 pgs: 121 active+clean; 111 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.2 MiB/s wr, 17 op/s 2024-09-12T23:40:26.885 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:26 smithi136 bash[22601]: cephadm 2024-09-12T23:40:24.761257+0000 mgr.x (mgr.24517) 18 : cephadm [INF] Adjusting osd_memory_target on smithi181 to 2524M 2024-09-12T23:40:26.885 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:26 smithi136 bash[22601]: cephadm 2024-09-12T23:40:24.779589+0000 mgr.x (mgr.24517) 19 : cephadm [INF] Updating smithi136:/etc/ceph/ceph.conf 2024-09-12T23:40:26.886 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:26 smithi136 bash[22601]: cephadm 2024-09-12T23:40:24.780041+0000 mgr.x (mgr.24517) 20 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.conf 2024-09-12T23:40:26.886 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:26 smithi136 bash[22601]: cephadm 2024-09-12T23:40:24.960423+0000 mgr.x (mgr.24517) 21 : cephadm [INF] Updating smithi181:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-12T23:40:26.886 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:26 smithi136 bash[22601]: cephadm 2024-09-12T23:40:24.960590+0000 mgr.x (mgr.24517) 22 : cephadm [INF] Updating smithi136:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-12T23:40:26.886 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:26 smithi136 bash[22601]: cephadm 2024-09-12T23:40:25.085356+0000 mgr.x (mgr.24517) 23 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2024-09-12T23:40:26.886 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:26 smithi136 bash[22601]: cephadm 2024-09-12T23:40:25.103892+0000 mgr.x (mgr.24517) 24 : cephadm [INF] Updating smithi136:/etc/ceph/ceph.client.admin.keyring 2024-09-12T23:40:26.886 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:26 smithi136 bash[22601]: cephadm 2024-09-12T23:40:25.210776+0000 mgr.x (mgr.24517) 25 : cephadm [INF] Updating smithi181:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-12T23:40:26.886 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:26 smithi136 bash[22601]: cephadm 2024-09-12T23:40:25.240088+0000 mgr.x (mgr.24517) 26 : cephadm [INF] Updating smithi136:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-12T23:40:26.886 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:26 smithi136 bash[22601]: cephadm 2024-09-12T23:40:25.460997+0000 mgr.x (mgr.24517) 27 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2024-09-12T23:40:26.886 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:26 smithi136 bash[22601]: cephadm 2024-09-12T23:40:25.461604+0000 mgr.x (mgr.24517) 28 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi136 2024-09-12T23:40:26.886 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:26 smithi136 bash[22601]: cluster 2024-09-12T23:40:25.518133+0000 mon.a (mon.0) 1289 : cluster [DBG] osdmap e190: 8 total, 8 up, 8 in 2024-09-12T23:40:26.886 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:26 smithi136 bash[22601]: cluster 2024-09-12T23:40:25.561153+0000 mgr.x (mgr.24517) 29 : cluster [DBG] pgmap v30: 121 pgs: 121 active+clean; 111 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.2 MiB/s wr, 17 op/s 2024-09-12T23:40:26.887 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:26 smithi136 bash[32189]: cephadm 2024-09-12T23:40:24.761257+0000 mgr.x (mgr.24517) 18 : cephadm [INF] Adjusting osd_memory_target on smithi181 to 2524M 2024-09-12T23:40:26.887 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:26 smithi136 bash[32189]: cephadm 2024-09-12T23:40:24.779589+0000 mgr.x (mgr.24517) 19 : cephadm [INF] Updating smithi136:/etc/ceph/ceph.conf 2024-09-12T23:40:26.887 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:26 smithi136 bash[32189]: cephadm 2024-09-12T23:40:24.780041+0000 mgr.x (mgr.24517) 20 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.conf 2024-09-12T23:40:26.887 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:26 smithi136 bash[32189]: cephadm 2024-09-12T23:40:24.960423+0000 mgr.x (mgr.24517) 21 : cephadm [INF] Updating smithi181:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-12T23:40:26.887 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:26 smithi136 bash[32189]: cephadm 2024-09-12T23:40:24.960590+0000 mgr.x (mgr.24517) 22 : cephadm [INF] Updating smithi136:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-12T23:40:26.887 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:26 smithi136 bash[32189]: cephadm 2024-09-12T23:40:25.085356+0000 mgr.x (mgr.24517) 23 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2024-09-12T23:40:26.887 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:26 smithi136 bash[32189]: cephadm 2024-09-12T23:40:25.103892+0000 mgr.x (mgr.24517) 24 : cephadm [INF] Updating smithi136:/etc/ceph/ceph.client.admin.keyring 2024-09-12T23:40:26.887 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:26 smithi136 bash[32189]: cephadm 2024-09-12T23:40:25.210776+0000 mgr.x (mgr.24517) 25 : cephadm [INF] Updating smithi181:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-12T23:40:26.887 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:26 smithi136 bash[32189]: cephadm 2024-09-12T23:40:25.240088+0000 mgr.x (mgr.24517) 26 : cephadm [INF] Updating smithi136:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-12T23:40:26.887 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:26 smithi136 bash[32189]: cephadm 2024-09-12T23:40:25.460997+0000 mgr.x (mgr.24517) 27 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2024-09-12T23:40:26.887 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:26 smithi136 bash[32189]: cephadm 2024-09-12T23:40:25.461604+0000 mgr.x (mgr.24517) 28 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi136 2024-09-12T23:40:26.887 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:26 smithi136 bash[32189]: cluster 2024-09-12T23:40:25.518133+0000 mon.a (mon.0) 1289 : cluster [DBG] osdmap e190: 8 total, 8 up, 8 in 2024-09-12T23:40:26.888 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:26 smithi136 bash[32189]: cluster 2024-09-12T23:40:25.561153+0000 mgr.x (mgr.24517) 29 : cluster [DBG] pgmap v30: 121 pgs: 121 active+clean; 111 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.2 MiB/s wr, 17 op/s 2024-09-12T23:40:26.923 INFO:teuthology.orchestra.run.smithi136.stdout:true 2024-09-12T23:40:27.200 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:26 smithi136 bash[67907]: ceph-da58a47c-715d-11ef-bceb-c7b262605968-node-exporter-a 2024-09-12T23:40:27.200 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:27 smithi136 systemd[1]: ceph-da58a47c-715d-11ef-bceb-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-09-12T23:40:27.200 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:27 smithi136 bash[68013]: Error response from daemon: No such container: ceph-da58a47c-715d-11ef-bceb-c7b262605968-node-exporter-a 2024-09-12T23:40:27.200 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:27 smithi136 systemd[1]: ceph-da58a47c-715d-11ef-bceb-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-09-12T23:40:27.200 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:27 smithi136 systemd[1]: Stopped Ceph node-exporter.a for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:40:27.201 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:27 smithi136 systemd[1]: Started Ceph node-exporter.a for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:40:27.231 INFO:teuthology.orchestra.run.smithi136.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-12T23:40:27.231 INFO:teuthology.orchestra.run.smithi136.stdout:alertmanager.a smithi136 *:9093,9094 running (5m) 12s ago 6m 14.5M - 0.25.0 c8568f914cd2 d6c106834390 2024-09-12T23:40:27.231 INFO:teuthology.orchestra.run.smithi136.stdout:grafana.a smithi181 *:3000 running (5m) 12s ago 5m 79.7M - 9.4.7 954c08fa6188 85ee0f95160c 2024-09-12T23:40:27.231 INFO:teuthology.orchestra.run.smithi136.stdout:mds.a smithi136 running (7m) 12s ago 7m 13.4M - 17.2.7-1627-gbbdadfa3 214985049bdb 713b9795ad19 2024-09-12T23:40:27.231 INFO:teuthology.orchestra.run.smithi136.stdout:mds.b smithi181 running (7m) 12s ago 7m 16.0M - 17.2.7-1627-gbbdadfa3 214985049bdb ad13179f0868 2024-09-12T23:40:27.232 INFO:teuthology.orchestra.run.smithi136.stdout:mgr.x smithi181 *:8443,9283 running (36s) 12s ago 14m 459M - 18.2.4-837-g7a853b17 f21ba1eea74e 761bc101e333 2024-09-12T23:40:27.232 INFO:teuthology.orchestra.run.smithi136.stdout:mgr.y smithi136 *:9283,8765 running (16m) 12s ago 16m 285M - 17.2.7-1627-gbbdadfa3 214985049bdb 56608cdf6b27 2024-09-12T23:40:27.232 INFO:teuthology.orchestra.run.smithi136.stdout:mon.a smithi136 running (16m) 12s ago 16m 54.0M 2048M 17.2.7-1627-gbbdadfa3 214985049bdb 0462f4f61475 2024-09-12T23:40:27.232 INFO:teuthology.orchestra.run.smithi136.stdout:mon.b smithi181 running (14m) 12s ago 14m 43.9M 2048M 17.2.7-1627-gbbdadfa3 214985049bdb 8b3d784af845 2024-09-12T23:40:27.232 INFO:teuthology.orchestra.run.smithi136.stdout:mon.c smithi136 running (14m) 12s ago 14m 44.5M 2048M 17.2.7-1627-gbbdadfa3 214985049bdb 647b028ea963 2024-09-12T23:40:27.232 INFO:teuthology.orchestra.run.smithi136.stdout:node-exporter.a smithi136 *:9100 starting - - - - 2024-09-12T23:40:27.232 INFO:teuthology.orchestra.run.smithi136.stdout:node-exporter.b smithi181 *:9100 running (6m) 12s ago 6m 8603k - 1.5.0 0da6a335fe13 575eff556149 2024-09-12T23:40:27.232 INFO:teuthology.orchestra.run.smithi136.stdout:osd.0 smithi136 running (13m) 12s ago 13m 99.4M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb cbc0b7df0115 2024-09-12T23:40:27.232 INFO:teuthology.orchestra.run.smithi136.stdout:osd.1 smithi136 running (12m) 12s ago 12m 102M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb c4daee7da836 2024-09-12T23:40:27.232 INFO:teuthology.orchestra.run.smithi136.stdout:osd.2 smithi136 running (12m) 12s ago 12m 121M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 4ab8fd79cdb4 2024-09-12T23:40:27.232 INFO:teuthology.orchestra.run.smithi136.stdout:osd.3 smithi136 running (11m) 12s ago 11m 88.5M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 9ceb14a27521 2024-09-12T23:40:27.232 INFO:teuthology.orchestra.run.smithi136.stdout:osd.4 smithi181 running (10m) 12s ago 10m 114M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb a35cdca79055 2024-09-12T23:40:27.232 INFO:teuthology.orchestra.run.smithi136.stdout:osd.5 smithi181 running (9m) 12s ago 9m 101M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 8f2e8d494fef 2024-09-12T23:40:27.233 INFO:teuthology.orchestra.run.smithi136.stdout:osd.6 smithi181 running (8m) 12s ago 8m 66.5M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 9c6bec406408 2024-09-12T23:40:27.233 INFO:teuthology.orchestra.run.smithi136.stdout:osd.7 smithi181 running (7m) 12s ago 7m 86.7M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb bc0632ae2145 2024-09-12T23:40:27.233 INFO:teuthology.orchestra.run.smithi136.stdout:prometheus.a smithi181 *:9095 running (5m) 12s ago 6m 49.7M - 2.43.0 a07b618ecd1d 24e591310569 2024-09-12T23:40:27.533 INFO:tasks.rados.rados.0.smithi136.stdout:714: done (1 left) 2024-09-12T23:40:27.533 INFO:tasks.rados.rados.0.smithi136.stdout:715: done (0 left) 2024-09-12T23:40:27.533 INFO:tasks.rados.rados.0.smithi136.stdout:716: read oid 2 snap 59 2024-09-12T23:40:27.533 INFO:tasks.rados.rados.0.smithi136.stdout:716: expect (ObjNum 193 snap 47 seq_num 193) 2024-09-12T23:40:27.533 INFO:tasks.rados.rados.0.smithi136.stdout:717: read oid 33 snap -1 2024-09-12T23:40:27.533 INFO:tasks.rados.rados.0.smithi136.stdout:717: expect (ObjNum 198 snap 48 seq_num 198) 2024-09-12T23:40:27.533 INFO:tasks.rados.rados.0.smithi136.stdout:718: rollback oid 14 current snap is 62 2024-09-12T23:40:27.533 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 14 to 61 2024-09-12T23:40:27.533 INFO:tasks.rados.rados.0.smithi136.stdout:719: read oid 45 snap -1 2024-09-12T23:40:27.533 INFO:tasks.rados.rados.0.smithi136.stdout:719: expect (ObjNum 198 snap 48 seq_num 198) 2024-09-12T23:40:27.533 INFO:tasks.rados.rados.0.smithi136.stdout:720: snap_create 2024-09-12T23:40:27.549 INFO:tasks.rados.rados.0.smithi136.stdout:718: finishing rollback tid 0 to smithi13667081-14 2024-09-12T23:40:27.552 INFO:tasks.rados.rados.0.smithi136.stdout:716: expect (ObjNum 193 snap 47 seq_num 193) 2024-09-12T23:40:27.564 INFO:teuthology.orchestra.run.smithi136.stdout:{ 2024-09-12T23:40:27.564 INFO:teuthology.orchestra.run.smithi136.stdout: "mon": { 2024-09-12T23:40:27.564 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 3 2024-09-12T23:40:27.564 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:40:27.564 INFO:teuthology.orchestra.run.smithi136.stdout: "mgr": { 2024-09-12T23:40:27.564 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 1, 2024-09-12T23:40:27.564 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 1 2024-09-12T23:40:27.565 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:40:27.565 INFO:teuthology.orchestra.run.smithi136.stdout: "osd": { 2024-09-12T23:40:27.565 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 8 2024-09-12T23:40:27.565 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:40:27.565 INFO:teuthology.orchestra.run.smithi136.stdout: "mds": { 2024-09-12T23:40:27.565 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-12T23:40:27.565 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:40:27.565 INFO:teuthology.orchestra.run.smithi136.stdout: "overall": { 2024-09-12T23:40:27.565 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 14, 2024-09-12T23:40:27.565 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 1 2024-09-12T23:40:27.565 INFO:teuthology.orchestra.run.smithi136.stdout: } 2024-09-12T23:40:27.565 INFO:teuthology.orchestra.run.smithi136.stdout:} 2024-09-12T23:40:27.654 INFO:tasks.rados.rados.0.smithi136.stdout:718: finishing rollback tid 1 to smithi13667081-14 2024-09-12T23:40:27.655 INFO:tasks.rados.rados.0.smithi136.stdout:718: finishing rollback tid 2 to smithi13667081-14 2024-09-12T23:40:27.655 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 14 v 383 (ObjNum 175 snap 45 seq_num 175) dirty exists 2024-09-12T23:40:27.655 INFO:tasks.rados.rados.0.smithi136.stdout:717: expect (ObjNum 198 snap 48 seq_num 198) 2024-09-12T23:40:27.770 INFO:tasks.rados.rados.0.smithi136.stdout:719: expect (ObjNum 198 snap 48 seq_num 198) 2024-09-12T23:40:27.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:27 smithi181 bash[24776]: cluster 2024-09-12T23:40:26.544835+0000 mon.a (mon.0) 1290 : cluster [DBG] osdmap e191: 8 total, 8 up, 8 in 2024-09-12T23:40:27.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:27 smithi181 bash[24776]: audit 2024-09-12T23:40:26.914721+0000 mgr.x (mgr.24517) 30 : audit [DBG] from='client.15105 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:40:27.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:27 smithi181 bash[24776]: audit 2024-09-12T23:40:27.142503+0000 mon.a (mon.0) 1291 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:27.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:27 smithi181 bash[24776]: audit 2024-09-12T23:40:27.154083+0000 mon.a (mon.0) 1292 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:27.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:27 smithi181 bash[24776]: cephadm 2024-09-12T23:40:27.159608+0000 mgr.x (mgr.24517) 31 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2024-09-12T23:40:27.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:27 smithi181 bash[24776]: cephadm 2024-09-12T23:40:27.186621+0000 mgr.x (mgr.24517) 32 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi136 2024-09-12T23:40:27.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:27 smithi181 bash[24776]: audit 2024-09-12T23:40:27.199328+0000 mgr.x (mgr.24517) 33 : audit [DBG] from='client.15111 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:40:28.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:27 smithi136 bash[22601]: cluster 2024-09-12T23:40:26.544835+0000 mon.a (mon.0) 1290 : cluster [DBG] osdmap e191: 8 total, 8 up, 8 in 2024-09-12T23:40:28.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:27 smithi136 bash[22601]: audit 2024-09-12T23:40:26.914721+0000 mgr.x (mgr.24517) 30 : audit [DBG] from='client.15105 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:40:28.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:27 smithi136 bash[22601]: audit 2024-09-12T23:40:27.142503+0000 mon.a (mon.0) 1291 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:28.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:27 smithi136 bash[22601]: audit 2024-09-12T23:40:27.154083+0000 mon.a (mon.0) 1292 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:28.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:27 smithi136 bash[22601]: cephadm 2024-09-12T23:40:27.159608+0000 mgr.x (mgr.24517) 31 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2024-09-12T23:40:28.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:27 smithi136 bash[22601]: cephadm 2024-09-12T23:40:27.186621+0000 mgr.x (mgr.24517) 32 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi136 2024-09-12T23:40:28.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:27 smithi136 bash[22601]: audit 2024-09-12T23:40:27.199328+0000 mgr.x (mgr.24517) 33 : audit [DBG] from='client.15111 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:40:28.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:27 smithi136 bash[32189]: cluster 2024-09-12T23:40:26.544835+0000 mon.a (mon.0) 1290 : cluster [DBG] osdmap e191: 8 total, 8 up, 8 in 2024-09-12T23:40:28.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:27 smithi136 bash[32189]: audit 2024-09-12T23:40:26.914721+0000 mgr.x (mgr.24517) 30 : audit [DBG] from='client.15105 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:40:28.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:27 smithi136 bash[32189]: audit 2024-09-12T23:40:27.142503+0000 mon.a (mon.0) 1291 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:28.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:27 smithi136 bash[32189]: audit 2024-09-12T23:40:27.154083+0000 mon.a (mon.0) 1292 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:28.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:27 smithi136 bash[32189]: cephadm 2024-09-12T23:40:27.159608+0000 mgr.x (mgr.24517) 31 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2024-09-12T23:40:28.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:27 smithi136 bash[32189]: cephadm 2024-09-12T23:40:27.186621+0000 mgr.x (mgr.24517) 32 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi136 2024-09-12T23:40:28.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:27 smithi136 bash[32189]: audit 2024-09-12T23:40:27.199328+0000 mgr.x (mgr.24517) 33 : audit [DBG] from='client.15111 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:40:28.543 INFO:tasks.rados.rados.0.smithi136.stdout:716: done (4 left) 2024-09-12T23:40:28.543 INFO:tasks.rados.rados.0.smithi136.stdout:717: done (3 left) 2024-09-12T23:40:28.543 INFO:tasks.rados.rados.0.smithi136.stdout:718: done (2 left) 2024-09-12T23:40:28.543 INFO:tasks.rados.rados.0.smithi136.stdout:719: done (1 left) 2024-09-12T23:40:28.543 INFO:tasks.rados.rados.0.smithi136.stdout:720: done (0 left) 2024-09-12T23:40:28.543 INFO:tasks.rados.rados.0.smithi136.stdout:721: rmattr oid 13 current snap is 63 2024-09-12T23:40:28.543 INFO:tasks.rados.rados.0.smithi136.stdout:721: done (0 left) 2024-09-12T23:40:28.543 INFO:tasks.rados.rados.0.smithi136.stdout:722: snap_create 2024-09-12T23:40:28.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:28 smithi181 bash[24776]: cluster 2024-09-12T23:40:27.537446+0000 mon.a (mon.0) 1293 : cluster [DBG] osdmap e192: 8 total, 8 up, 8 in 2024-09-12T23:40:28.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:28 smithi181 bash[24776]: cluster 2024-09-12T23:40:27.561836+0000 mgr.x (mgr.24517) 34 : cluster [DBG] pgmap v33: 121 pgs: 121 active+clean; 111 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.2 MiB/s wr, 17 op/s 2024-09-12T23:40:28.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:28 smithi181 bash[24776]: audit 2024-09-12T23:40:27.569537+0000 mon.c (mon.2) 69 : audit [DBG] from='client.? 172.21.15.136:0/2322553228' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:40:28.855 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[22601]: cluster 2024-09-12T23:40:27.537446+0000 mon.a (mon.0) 1293 : cluster [DBG] osdmap e192: 8 total, 8 up, 8 in 2024-09-12T23:40:28.855 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[22601]: cluster 2024-09-12T23:40:27.561836+0000 mgr.x (mgr.24517) 34 : cluster [DBG] pgmap v33: 121 pgs: 121 active+clean; 111 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.2 MiB/s wr, 17 op/s 2024-09-12T23:40:28.855 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[22601]: audit 2024-09-12T23:40:27.569537+0000 mon.c (mon.2) 69 : audit [DBG] from='client.? 172.21.15.136:0/2322553228' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:40:28.856 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[32189]: cluster 2024-09-12T23:40:27.537446+0000 mon.a (mon.0) 1293 : cluster [DBG] osdmap e192: 8 total, 8 up, 8 in 2024-09-12T23:40:28.856 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[32189]: cluster 2024-09-12T23:40:27.561836+0000 mgr.x (mgr.24517) 34 : cluster [DBG] pgmap v33: 121 pgs: 121 active+clean; 111 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.2 MiB/s wr, 17 op/s 2024-09-12T23:40:28.856 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[32189]: audit 2024-09-12T23:40:27.569537+0000 mon.c (mon.2) 69 : audit [DBG] from='client.? 172.21.15.136:0/2322553228' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:40:29.123 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[68041]: ts=2024-09-12T23:40:28.942Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-09-12T23:40:29.123 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[68041]: ts=2024-09-12T23:40:28.942Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-09-12T23:40:29.123 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[68041]: ts=2024-09-12T23:40:28.943Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-12T23:40:29.124 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[68041]: ts=2024-09-12T23:40:28.943Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-12T23:40:29.124 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[68041]: ts=2024-09-12T23:40:28.943Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-12T23:40:29.124 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[68041]: ts=2024-09-12T23:40:28.944Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-12T23:40:29.124 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[68041]: ts=2024-09-12T23:40:28.944Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-12T23:40:29.124 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[68041]: ts=2024-09-12T23:40:28.944Z caller=node_exporter.go:117 level=info collector=arp 2024-09-12T23:40:29.124 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[68041]: ts=2024-09-12T23:40:28.944Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-12T23:40:29.124 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[68041]: ts=2024-09-12T23:40:28.944Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-12T23:40:29.124 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[68041]: ts=2024-09-12T23:40:28.944Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-12T23:40:29.124 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[68041]: ts=2024-09-12T23:40:28.944Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-12T23:40:29.124 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[68041]: ts=2024-09-12T23:40:28.944Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-12T23:40:29.124 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[68041]: ts=2024-09-12T23:40:28.944Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-12T23:40:29.124 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[68041]: ts=2024-09-12T23:40:28.944Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-12T23:40:29.125 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[68041]: ts=2024-09-12T23:40:28.944Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-12T23:40:29.125 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[68041]: ts=2024-09-12T23:40:28.944Z caller=node_exporter.go:117 level=info collector=edac 2024-09-12T23:40:29.125 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[68041]: ts=2024-09-12T23:40:28.944Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-12T23:40:29.125 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[68041]: ts=2024-09-12T23:40:28.944Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-12T23:40:29.125 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[68041]: ts=2024-09-12T23:40:28.944Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-12T23:40:29.125 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[68041]: ts=2024-09-12T23:40:28.944Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-12T23:40:29.125 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[68041]: ts=2024-09-12T23:40:28.944Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-12T23:40:29.125 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[68041]: ts=2024-09-12T23:40:28.944Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-12T23:40:29.125 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[68041]: ts=2024-09-12T23:40:28.944Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-12T23:40:29.125 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[68041]: ts=2024-09-12T23:40:28.944Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-12T23:40:29.125 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[68041]: ts=2024-09-12T23:40:28.944Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-12T23:40:29.125 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[68041]: ts=2024-09-12T23:40:28.944Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-12T23:40:29.125 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[68041]: ts=2024-09-12T23:40:28.944Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-12T23:40:29.125 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[68041]: ts=2024-09-12T23:40:28.944Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-12T23:40:29.126 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[68041]: ts=2024-09-12T23:40:28.944Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-12T23:40:29.126 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[68041]: ts=2024-09-12T23:40:28.944Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-12T23:40:29.126 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[68041]: ts=2024-09-12T23:40:28.944Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-12T23:40:29.126 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[68041]: ts=2024-09-12T23:40:28.944Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-12T23:40:29.126 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[68041]: ts=2024-09-12T23:40:28.944Z caller=node_exporter.go:117 level=info collector=os 2024-09-12T23:40:29.126 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[68041]: ts=2024-09-12T23:40:28.944Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-12T23:40:29.126 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[68041]: ts=2024-09-12T23:40:28.944Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-12T23:40:29.126 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[68041]: ts=2024-09-12T23:40:28.944Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-12T23:40:29.126 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[68041]: ts=2024-09-12T23:40:28.944Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-12T23:40:29.126 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[68041]: ts=2024-09-12T23:40:28.944Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-12T23:40:29.126 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[68041]: ts=2024-09-12T23:40:28.944Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-12T23:40:29.126 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[68041]: ts=2024-09-12T23:40:28.944Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-12T23:40:29.126 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[68041]: ts=2024-09-12T23:40:28.944Z caller=node_exporter.go:117 level=info collector=stat 2024-09-12T23:40:29.127 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[68041]: ts=2024-09-12T23:40:28.944Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-12T23:40:29.127 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[68041]: ts=2024-09-12T23:40:28.944Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-12T23:40:29.127 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[68041]: ts=2024-09-12T23:40:28.944Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-12T23:40:29.127 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[68041]: ts=2024-09-12T23:40:28.944Z caller=node_exporter.go:117 level=info collector=time 2024-09-12T23:40:29.127 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[68041]: ts=2024-09-12T23:40:28.944Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-12T23:40:29.127 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[68041]: ts=2024-09-12T23:40:28.944Z caller=node_exporter.go:117 level=info collector=uname 2024-09-12T23:40:29.127 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[68041]: ts=2024-09-12T23:40:28.944Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-12T23:40:29.127 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[68041]: ts=2024-09-12T23:40:28.944Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-12T23:40:29.127 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[68041]: ts=2024-09-12T23:40:28.944Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-12T23:40:29.127 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[68041]: ts=2024-09-12T23:40:28.944Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-09-12T23:40:29.127 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:28 smithi136 bash[68041]: ts=2024-09-12T23:40:28.944Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-12T23:40:29.540 INFO:tasks.rados.rados.0.smithi136.stdout:722: done (0 left) 2024-09-12T23:40:29.540 INFO:tasks.rados.rados.0.smithi136.stdout:723: copy_from oid 43 from oid 39 current snap is 64 2024-09-12T23:40:29.540 INFO:tasks.rados.rados.0.smithi136.stdout:724: delete oid 49 current snap is 64 2024-09-12T23:40:29.555 INFO:tasks.rados.rados.0.smithi136.stdout:724: done (1 left) 2024-09-12T23:40:29.555 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 23 current snap is 64 2024-09-12T23:40:29.555 INFO:tasks.rados.rados.0.smithi136.stdout:725: seq_num 246 ranges {4481024=524288,5005312=745472,5750784=425984,6176768=32768} 2024-09-12T23:40:29.620 INFO:tasks.rados.rados.0.smithi136.stdout:725: writing smithi13667081-23 from 4481024 to 5005312 tid 1 2024-09-12T23:40:29.627 INFO:tasks.rados.rados.0.smithi136.stdout:725: writing smithi13667081-23 from 5005312 to 5750784 tid 2 2024-09-12T23:40:29.631 INFO:tasks.rados.rados.0.smithi136.stdout:725: writing smithi13667081-23 from 5750784 to 6176768 tid 3 2024-09-12T23:40:29.632 INFO:tasks.rados.rados.0.smithi136.stdout:725: writing smithi13667081-23 from 6176768 to 6209536 tid 4 2024-09-12T23:40:29.632 INFO:tasks.rados.rados.0.smithi136.stdout:726: rollback oid 3 current snap is 64 2024-09-12T23:40:29.632 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 3 to 59 2024-09-12T23:40:29.632 INFO:tasks.rados.rados.0.smithi136.stdout:723: finishing copy_from racing read to smithi13667081-43 2024-09-12T23:40:29.632 INFO:tasks.rados.rados.0.smithi136.stdout:723: finishing copy_from to smithi13667081-43 2024-09-12T23:40:29.632 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 43 v 90 (ObjNum 225 snap 56 seq_num 225) dirty exists 2024-09-12T23:40:29.632 INFO:tasks.rados.rados.0.smithi136.stdout:723: done (2 left) 2024-09-12T23:40:29.632 INFO:tasks.rados.rados.0.smithi136.stdout:727: snap_create 2024-09-12T23:40:29.637 INFO:tasks.rados.rados.0.smithi136.stdout:725: finishing write tid 1 to smithi13667081-23 2024-09-12T23:40:29.637 INFO:tasks.rados.rados.0.smithi136.stdout:726: finishing rollback tid 0 to smithi13667081-3 2024-09-12T23:40:29.637 INFO:tasks.rados.rados.0.smithi136.stdout:726: finishing rollback tid 1 to smithi13667081-3 2024-09-12T23:40:29.637 INFO:tasks.rados.rados.0.smithi136.stdout:726: finishing rollback tid 2 to smithi13667081-3 2024-09-12T23:40:29.637 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 3 v 443 (ObjNum 199 snap 49 seq_num 199) dirty exists 2024-09-12T23:40:29.639 INFO:tasks.rados.rados.0.smithi136.stdout:725: finishing write tid 2 to smithi13667081-23 2024-09-12T23:40:29.640 INFO:tasks.rados.rados.0.smithi136.stdout:725: finishing write tid 3 to smithi13667081-23 2024-09-12T23:40:29.640 INFO:tasks.rados.rados.0.smithi136.stdout:725: finishing write tid 4 to smithi13667081-23 2024-09-12T23:40:29.641 INFO:tasks.rados.rados.0.smithi136.stdout:725: finishing write tid 5 to smithi13667081-23 2024-09-12T23:40:29.642 INFO:tasks.rados.rados.0.smithi136.stdout:725: finishing write tid 6 to smithi13667081-23 2024-09-12T23:40:29.642 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 23 v 416 (ObjNum 246 snap 64 seq_num 246) dirty exists 2024-09-12T23:40:29.642 INFO:tasks.rados.rados.0.smithi136.stdout:725: left oid 23 (ObjNum 246 snap 64 seq_num 246) 2024-09-12T23:40:29.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:29 smithi181 bash[24776]: cluster 2024-09-12T23:40:28.547412+0000 mon.a (mon.0) 1294 : cluster [DBG] osdmap e193: 8 total, 8 up, 8 in 2024-09-12T23:40:29.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:29 smithi181 bash[24776]: cluster 2024-09-12T23:40:29.545126+0000 mon.a (mon.0) 1295 : cluster [DBG] osdmap e194: 8 total, 8 up, 8 in 2024-09-12T23:40:29.936 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:29 smithi136 bash[32189]: cluster 2024-09-12T23:40:28.547412+0000 mon.a (mon.0) 1294 : cluster [DBG] osdmap e193: 8 total, 8 up, 8 in 2024-09-12T23:40:29.937 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:29 smithi136 bash[32189]: cluster 2024-09-12T23:40:29.545126+0000 mon.a (mon.0) 1295 : cluster [DBG] osdmap e194: 8 total, 8 up, 8 in 2024-09-12T23:40:29.937 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:29 smithi136 bash[22601]: cluster 2024-09-12T23:40:28.547412+0000 mon.a (mon.0) 1294 : cluster [DBG] osdmap e193: 8 total, 8 up, 8 in 2024-09-12T23:40:29.937 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:29 smithi136 bash[22601]: cluster 2024-09-12T23:40:29.545126+0000 mon.a (mon.0) 1295 : cluster [DBG] osdmap e194: 8 total, 8 up, 8 in 2024-09-12T23:40:30.486 INFO:tasks.rados.rados.0.smithi136.stdout:725: done (2 left) 2024-09-12T23:40:30.486 INFO:tasks.rados.rados.0.smithi136.stdout:726: done (1 left) 2024-09-12T23:40:30.486 INFO:tasks.rados.rados.0.smithi136.stdout:727: done (0 left) 2024-09-12T23:40:30.486 INFO:tasks.rados.rados.0.smithi136.stdout:728: read oid 42 snap -1 2024-09-12T23:40:30.487 INFO:tasks.rados.rados.0.smithi136.stdout:728: expect (ObjNum 125 snap 32 seq_num 125) 2024-09-12T23:40:30.487 INFO:tasks.rados.rados.0.smithi136.stdout:729: read oid 24 snap -1 2024-09-12T23:40:30.487 INFO:tasks.rados.rados.0.smithi136.stdout:729: expect (ObjNum 242 snap 60 seq_num 242) 2024-09-12T23:40:30.487 INFO:tasks.rados.rados.0.smithi136.stdout:730: snap_create 2024-09-12T23:40:30.508 INFO:tasks.rados.rados.0.smithi136.stdout:728: expect (ObjNum 125 snap 32 seq_num 125) 2024-09-12T23:40:30.579 INFO:tasks.rados.rados.0.smithi136.stdout:729: expect (ObjNum 242 snap 60 seq_num 242) 2024-09-12T23:40:31.503 INFO:tasks.rados.rados.0.smithi136.stdout:728: done (2 left) 2024-09-12T23:40:31.503 INFO:tasks.rados.rados.0.smithi136.stdout:729: done (1 left) 2024-09-12T23:40:31.503 INFO:tasks.rados.rados.0.smithi136.stdout:730: done (0 left) 2024-09-12T23:40:31.504 INFO:tasks.rados.rados.0.smithi136.stdout:731: delete oid 10 current snap is 66 2024-09-12T23:40:31.506 INFO:tasks.rados.rados.0.smithi136.stdout:731: done (0 left) 2024-09-12T23:40:31.506 INFO:tasks.rados.rados.0.smithi136.stdout:732: snap_create 2024-09-12T23:40:31.596 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:31 smithi136 bash[22601]: cluster 2024-09-12T23:40:29.563245+0000 mgr.x (mgr.24517) 35 : cluster [DBG] pgmap v36: 121 pgs: 121 active+clean; 81 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 414 KiB/s wr, 18 op/s 2024-09-12T23:40:31.596 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:31 smithi136 bash[22601]: cluster 2024-09-12T23:40:30.490043+0000 mon.a (mon.0) 1296 : cluster [DBG] osdmap e195: 8 total, 8 up, 8 in 2024-09-12T23:40:31.596 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:31 smithi136 bash[32189]: cluster 2024-09-12T23:40:29.563245+0000 mgr.x (mgr.24517) 35 : cluster [DBG] pgmap v36: 121 pgs: 121 active+clean; 81 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 414 KiB/s wr, 18 op/s 2024-09-12T23:40:31.596 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:31 smithi136 bash[32189]: cluster 2024-09-12T23:40:30.490043+0000 mon.a (mon.0) 1296 : cluster [DBG] osdmap e195: 8 total, 8 up, 8 in 2024-09-12T23:40:31.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:31 smithi181 bash[24776]: cluster 2024-09-12T23:40:29.563245+0000 mgr.x (mgr.24517) 35 : cluster [DBG] pgmap v36: 121 pgs: 121 active+clean; 81 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 414 KiB/s wr, 18 op/s 2024-09-12T23:40:31.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:31 smithi181 bash[24776]: cluster 2024-09-12T23:40:30.490043+0000 mon.a (mon.0) 1296 : cluster [DBG] osdmap e195: 8 total, 8 up, 8 in 2024-09-12T23:40:31.938 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:40:31 smithi136 systemd[1]: Stopping Ceph alertmanager.a for da58a47c-715d-11ef-bceb-c7b262605968... 2024-09-12T23:40:31.938 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:40:31 smithi136 bash[60087]: ts=2024-09-12T23:40:31.708Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-12T23:40:32.499 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:40:32 smithi136 bash[68302]: ceph-da58a47c-715d-11ef-bceb-c7b262605968-alertmanager-a 2024-09-12T23:40:32.499 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:40:32 smithi136 bash[68363]: Error response from daemon: No such container: ceph-da58a47c-715d-11ef-bceb-c7b262605968-alertmanager-a 2024-09-12T23:40:32.500 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:40:32 smithi136 systemd[1]: ceph-da58a47c-715d-11ef-bceb-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-09-12T23:40:32.500 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:40:32 smithi136 systemd[1]: Stopped Ceph alertmanager.a for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:40:32.500 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:40:32 smithi136 systemd[1]: Started Ceph alertmanager.a for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:40:32.503 INFO:tasks.rados.rados.0.smithi136.stdout:732: done (0 left) 2024-09-12T23:40:32.503 INFO:tasks.rados.rados.0.smithi136.stdout:733: read oid 17 snap -1 2024-09-12T23:40:32.503 INFO:tasks.rados.rados.0.smithi136.stdout:733: expect (ObjNum 164 snap 43 seq_num 164) 2024-09-12T23:40:32.503 INFO:tasks.rados.rados.0.smithi136.stdout:734: rollback oid 6 current snap is 67 2024-09-12T23:40:32.503 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 6 to 56 2024-09-12T23:40:32.503 INFO:tasks.rados.rados.0.smithi136.stdout:735: read oid 27 snap 56 2024-09-12T23:40:32.503 INFO:tasks.rados.rados.0.smithi136.stdout:735: expect (ObjNum 209 snap 52 seq_num 209) 2024-09-12T23:40:32.503 INFO:tasks.rados.rados.0.smithi136.stdout:736: copy_from oid 18 from oid 22 current snap is 67 2024-09-12T23:40:32.511 INFO:tasks.rados.rados.0.smithi136.stdout:737: rmattr oid 14 current snap is 67 2024-09-12T23:40:32.511 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 10 current snap is 67 2024-09-12T23:40:32.511 INFO:tasks.rados.rados.0.smithi136.stdout:738: seq_num 248 ranges {0=557056,557056=507904,1064960=442368,1507328=253952} 2024-09-12T23:40:32.511 INFO:tasks.rados.rados.0.smithi136.stdout:738: writing smithi13667081-10 from 0 to 557056 tid 1 2024-09-12T23:40:32.511 INFO:tasks.rados.rados.0.smithi136.stdout:738: writing smithi13667081-10 from 557056 to 1064960 tid 2 2024-09-12T23:40:32.513 INFO:tasks.rados.rados.0.smithi136.stdout:738: writing smithi13667081-10 from 1064960 to 1507328 tid 3 2024-09-12T23:40:32.515 INFO:tasks.rados.rados.0.smithi136.stdout:738: writing smithi13667081-10 from 1507328 to 1761280 tid 4 2024-09-12T23:40:32.515 INFO:tasks.rados.rados.0.smithi136.stdout:739: setattr oid 4 current snap is 67 2024-09-12T23:40:32.515 INFO:tasks.rados.rados.0.smithi136.stdout:734: finishing rollback tid 1 to smithi13667081-6 2024-09-12T23:40:32.516 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 6 v 0 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-12T23:40:32.516 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 14 v 386 (ObjNum 175 snap 45 seq_num 175) dirty exists 2024-09-12T23:40:32.516 INFO:tasks.rados.rados.0.smithi136.stdout:733: expect (ObjNum 164 snap 43 seq_num 164) 2024-09-12T23:40:32.524 INFO:tasks.rados.rados.0.smithi136.stdout:735: expect (ObjNum 209 snap 52 seq_num 209) 2024-09-12T23:40:32.573 INFO:tasks.rados.rados.0.smithi136.stdout:736: finishing copy_from racing read to smithi13667081-18 2024-09-12T23:40:32.573 INFO:tasks.rados.rados.0.smithi136.stdout:736: finishing copy_from to smithi13667081-18 2024-09-12T23:40:32.573 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 18 v 153 (ObjNum 21 snap 0 seq_num 21) dirty exists 2024-09-12T23:40:32.573 INFO:tasks.rados.rados.0.smithi136.stdout:738: finishing write tid 1 to smithi13667081-10 2024-09-12T23:40:32.573 INFO:tasks.rados.rados.0.smithi136.stdout:738: finishing write tid 2 to smithi13667081-10 2024-09-12T23:40:32.573 INFO:tasks.rados.rados.0.smithi136.stdout:738: finishing write tid 3 to smithi13667081-10 2024-09-12T23:40:32.573 INFO:tasks.rados.rados.0.smithi136.stdout:738: finishing write tid 4 to smithi13667081-10 2024-09-12T23:40:32.574 INFO:tasks.rados.rados.0.smithi136.stdout:738: finishing write tid 5 to smithi13667081-10 2024-09-12T23:40:32.574 INFO:tasks.rados.rados.0.smithi136.stdout:738: finishing write tid 6 to smithi13667081-10 2024-09-12T23:40:32.574 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 10 v 203 (ObjNum 248 snap 67 seq_num 248) dirty exists 2024-09-12T23:40:32.574 INFO:tasks.rados.rados.0.smithi136.stdout:738: left oid 10 (ObjNum 248 snap 67 seq_num 248) 2024-09-12T23:40:32.574 INFO:tasks.rados.rados.0.smithi136.stdout:733: done (6 left) 2024-09-12T23:40:32.574 INFO:tasks.rados.rados.0.smithi136.stdout:734: done (5 left) 2024-09-12T23:40:32.574 INFO:tasks.rados.rados.0.smithi136.stdout:735: done (4 left) 2024-09-12T23:40:32.574 INFO:tasks.rados.rados.0.smithi136.stdout:736: done (3 left) 2024-09-12T23:40:32.574 INFO:tasks.rados.rados.0.smithi136.stdout:737: done (2 left) 2024-09-12T23:40:32.574 INFO:tasks.rados.rados.0.smithi136.stdout:738: done (1 left) 2024-09-12T23:40:32.574 INFO:tasks.rados.rados.0.smithi136.stdout:740: read oid 37 snap -1 2024-09-12T23:40:32.574 INFO:tasks.rados.rados.0.smithi136.stdout:740: expect (ObjNum 134 snap 35 seq_num 134) 2024-09-12T23:40:32.574 INFO:tasks.rados.rados.0.smithi136.stdout:741: delete oid 27 current snap is 67 2024-09-12T23:40:32.576 INFO:tasks.rados.rados.0.smithi136.stdout:741: done (2 left) 2024-09-12T23:40:32.576 INFO:tasks.rados.rados.0.smithi136.stdout:742: delete oid 31 current snap is 67 2024-09-12T23:40:32.579 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 4 v 138 (ObjNum 220 snap 55 seq_num 220) dirty exists 2024-09-12T23:40:32.579 INFO:tasks.rados.rados.0.smithi136.stdout:739: done (2 left) 2024-09-12T23:40:32.580 INFO:tasks.rados.rados.0.smithi136.stdout:742: done (1 left) 2024-09-12T23:40:32.580 INFO:tasks.rados.rados.0.smithi136.stdout:743: delete oid 20 current snap is 67 2024-09-12T23:40:32.585 INFO:tasks.rados.rados.0.smithi136.stdout:743: done (1 left) 2024-09-12T23:40:32.585 INFO:tasks.rados.rados.0.smithi136.stdout:744: snap_create 2024-09-12T23:40:32.601 INFO:tasks.rados.rados.0.smithi136.stdout:740: expect (ObjNum 134 snap 35 seq_num 134) 2024-09-12T23:40:32.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:32 smithi136 bash[22601]: cluster 2024-09-12T23:40:31.508239+0000 mon.a (mon.0) 1297 : cluster [DBG] osdmap e196: 8 total, 8 up, 8 in 2024-09-12T23:40:32.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:32 smithi136 bash[22601]: audit 2024-09-12T23:40:32.479981+0000 mon.a (mon.0) 1298 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:32.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:32 smithi136 bash[22601]: audit 2024-09-12T23:40:32.493265+0000 mon.a (mon.0) 1299 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:32.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:32 smithi136 bash[32189]: cluster 2024-09-12T23:40:31.508239+0000 mon.a (mon.0) 1297 : cluster [DBG] osdmap e196: 8 total, 8 up, 8 in 2024-09-12T23:40:32.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:32 smithi136 bash[32189]: audit 2024-09-12T23:40:32.479981+0000 mon.a (mon.0) 1298 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:32.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:32 smithi136 bash[32189]: audit 2024-09-12T23:40:32.493265+0000 mon.a (mon.0) 1299 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:32.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:32 smithi181 bash[24776]: cluster 2024-09-12T23:40:31.508239+0000 mon.a (mon.0) 1297 : cluster [DBG] osdmap e196: 8 total, 8 up, 8 in 2024-09-12T23:40:32.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:32 smithi181 bash[24776]: audit 2024-09-12T23:40:32.479981+0000 mon.a (mon.0) 1298 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:32.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:32 smithi181 bash[24776]: audit 2024-09-12T23:40:32.493265+0000 mon.a (mon.0) 1299 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:33.514 INFO:tasks.rados.rados.0.smithi136.stdout:740: done (1 left) 2024-09-12T23:40:33.514 INFO:tasks.rados.rados.0.smithi136.stdout:744: done (0 left) 2024-09-12T23:40:33.514 INFO:tasks.rados.rados.0.smithi136.stdout:745: snap_create 2024-09-12T23:40:33.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:33 smithi136 bash[22601]: cluster 2024-09-12T23:40:31.564050+0000 mgr.x (mgr.24517) 36 : cluster [DBG] pgmap v39: 121 pgs: 121 active+clean; 81 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 414 KiB/s wr, 18 op/s 2024-09-12T23:40:33.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:33 smithi136 bash[22601]: cephadm 2024-09-12T23:40:32.500102+0000 mgr.x (mgr.24517) 37 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-09-12T23:40:33.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:33 smithi136 bash[22601]: cluster 2024-09-12T23:40:32.508137+0000 mon.a (mon.0) 1300 : cluster [DBG] osdmap e197: 8 total, 8 up, 8 in 2024-09-12T23:40:33.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:33 smithi136 bash[32189]: cluster 2024-09-12T23:40:31.564050+0000 mgr.x (mgr.24517) 36 : cluster [DBG] pgmap v39: 121 pgs: 121 active+clean; 81 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 414 KiB/s wr, 18 op/s 2024-09-12T23:40:33.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:33 smithi136 bash[32189]: cephadm 2024-09-12T23:40:32.500102+0000 mgr.x (mgr.24517) 37 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-09-12T23:40:33.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:33 smithi136 bash[32189]: cluster 2024-09-12T23:40:32.508137+0000 mon.a (mon.0) 1300 : cluster [DBG] osdmap e197: 8 total, 8 up, 8 in 2024-09-12T23:40:33.756 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:40:33 smithi136 bash[68390]: ts=2024-09-12T23:40:33.580Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-09-12T23:40:33.756 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:40:33 smithi136 bash[68390]: ts=2024-09-12T23:40:33.580Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-09-12T23:40:33.756 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:40:33 smithi136 bash[68390]: ts=2024-09-12T23:40:33.582Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.136 port=9094 2024-09-12T23:40:33.757 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:40:33 smithi136 bash[68390]: ts=2024-09-12T23:40:33.583Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-12T23:40:33.757 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:40:33 smithi136 bash[68390]: ts=2024-09-12T23:40:33.613Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-12T23:40:33.757 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:40:33 smithi136 bash[68390]: ts=2024-09-12T23:40:33.614Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-12T23:40:33.757 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:40:33 smithi136 bash[68390]: ts=2024-09-12T23:40:33.617Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-09-12T23:40:33.757 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:40:33 smithi136 bash[68390]: ts=2024-09-12T23:40:33.617Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-09-12T23:40:33.761 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:40:33 smithi181 bash[51487]: ts=2024-09-12T23:40:33.509Z caller=notifier.go:532 level=error component=notifier alertmanager=http://smithi136.front.sepia.ceph.com:9093/api/v2/alerts count=1 msg="Error sending alert" err="Post \"http://smithi136.front.sepia.ceph.com:9093/api/v2/alerts\": dial tcp 172.21.15.136:9093: connect: connection refused" 2024-09-12T23:40:33.761 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:33 smithi181 bash[24776]: cluster 2024-09-12T23:40:31.564050+0000 mgr.x (mgr.24517) 36 : cluster [DBG] pgmap v39: 121 pgs: 121 active+clean; 81 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 414 KiB/s wr, 18 op/s 2024-09-12T23:40:33.761 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:33 smithi181 bash[24776]: cephadm 2024-09-12T23:40:32.500102+0000 mgr.x (mgr.24517) 37 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-09-12T23:40:33.761 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:33 smithi181 bash[24776]: cluster 2024-09-12T23:40:32.508137+0000 mon.a (mon.0) 1300 : cluster [DBG] osdmap e197: 8 total, 8 up, 8 in 2024-09-12T23:40:34.527 INFO:tasks.rados.rados.0.smithi136.stdout:745: done (0 left) 2024-09-12T23:40:34.527 INFO:tasks.rados.rados.0.smithi136.stdout:746: copy_from oid 48 from oid 1 current snap is 69 2024-09-12T23:40:34.527 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 41 current snap is 69 2024-09-12T23:40:34.527 INFO:tasks.rados.rados.0.smithi136.stdout:747: seq_num 252 ranges {466944=630784,1097728=499712,1597440=425984,2023424=278528} 2024-09-12T23:40:34.545 INFO:tasks.rados.rados.0.smithi136.stdout:747: writing smithi13667081-41 from 466944 to 1097728 tid 1 2024-09-12T23:40:34.555 INFO:tasks.rados.rados.0.smithi136.stdout:747: writing smithi13667081-41 from 1097728 to 1597440 tid 2 2024-09-12T23:40:34.562 INFO:tasks.rados.rados.0.smithi136.stdout:747: writing smithi13667081-41 from 1597440 to 2023424 tid 3 2024-09-12T23:40:34.567 INFO:tasks.rados.rados.0.smithi136.stdout:747: writing smithi13667081-41 from 2023424 to 2301952 tid 4 2024-09-12T23:40:34.567 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 25 current snap is 69 2024-09-12T23:40:34.567 INFO:tasks.rados.rados.0.smithi136.stdout:746: finishing copy_from racing read to smithi13667081-48 2024-09-12T23:40:34.568 INFO:tasks.rados.rados.0.smithi136.stdout:746: finishing copy_from to smithi13667081-48 2024-09-12T23:40:34.568 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 48 v 145 (ObjNum 113 snap 0 seq_num 2878166161) dirty exists 2024-09-12T23:40:34.568 INFO:tasks.rados.rados.0.smithi136.stdout:747: finishing write tid 1 to smithi13667081-41 2024-09-12T23:40:34.568 INFO:tasks.rados.rados.0.smithi136.stdout:747: finishing write tid 2 to smithi13667081-41 2024-09-12T23:40:34.568 INFO:tasks.rados.rados.0.smithi136.stdout:747: finishing write tid 3 to smithi13667081-41 2024-09-12T23:40:34.568 INFO:tasks.rados.rados.0.smithi136.stdout:748: seq_num 253 ranges {16384=647168,663552=786432,1449984=417792} 2024-09-12T23:40:34.579 INFO:tasks.rados.rados.0.smithi136.stdout:748: writing smithi13667081-25 from 16384 to 663552 tid 1 2024-09-12T23:40:34.591 INFO:tasks.rados.rados.0.smithi136.stdout:748: writing smithi13667081-25 from 663552 to 1449984 tid 2 2024-09-12T23:40:34.598 INFO:tasks.rados.rados.0.smithi136.stdout:748: writing smithi13667081-25 from 1449984 to 1867776 tid 3 2024-09-12T23:40:34.598 INFO:tasks.rados.rados.0.smithi136.stdout:746: done (2 left) 2024-09-12T23:40:34.598 INFO:tasks.rados.rados.0.smithi136.stdout:749: rmattr oid 9 current snap is 69 2024-09-12T23:40:34.598 INFO:tasks.rados.rados.0.smithi136.stdout:749: done (2 left) 2024-09-12T23:40:34.598 INFO:tasks.rados.rados.0.smithi136.stdout:750: read oid 21 snap -1 2024-09-12T23:40:34.598 INFO:tasks.rados.rados.0.smithi136.stdout:750: expect (ObjNum 88 snap 18 seq_num 88) 2024-09-12T23:40:34.598 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 38 current snap is 69 2024-09-12T23:40:34.598 INFO:tasks.rados.rados.0.smithi136.stdout:747: finishing write tid 4 to smithi13667081-41 2024-09-12T23:40:34.598 INFO:tasks.rados.rados.0.smithi136.stdout:747: finishing write tid 5 to smithi13667081-41 2024-09-12T23:40:34.598 INFO:tasks.rados.rados.0.smithi136.stdout:747: finishing write tid 6 to smithi13667081-41 2024-09-12T23:40:34.598 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 41 v 318 (ObjNum 252 snap 69 seq_num 252) dirty exists 2024-09-12T23:40:34.599 INFO:tasks.rados.rados.0.smithi136.stdout:747: left oid 41 (ObjNum 252 snap 69 seq_num 252) 2024-09-12T23:40:34.599 INFO:tasks.rados.rados.0.smithi136.stdout:748: finishing write tid 1 to smithi13667081-25 2024-09-12T23:40:34.599 INFO:tasks.rados.rados.0.smithi136.stdout:748: finishing write tid 2 to smithi13667081-25 2024-09-12T23:40:34.599 INFO:tasks.rados.rados.0.smithi136.stdout:751: seq_num 254 ranges {1843200=663552,2506752=663552,3170304=540672} 2024-09-12T23:40:34.628 INFO:tasks.rados.rados.0.smithi136.stdout:751: writing smithi13667081-38 from 1843200 to 2506752 tid 1 2024-09-12T23:40:34.635 INFO:tasks.rados.rados.0.smithi136.stdout:751: writing smithi13667081-38 from 2506752 to 3170304 tid 2 2024-09-12T23:40:34.640 INFO:tasks.rados.rados.0.smithi136.stdout:751: writing smithi13667081-38 from 3170304 to 3710976 tid 3 2024-09-12T23:40:34.640 INFO:tasks.rados.rados.0.smithi136.stdout:747: done (3 left) 2024-09-12T23:40:34.640 INFO:tasks.rados.rados.0.smithi136.stdout:752: copy_from oid 19 from oid 2 current snap is 69 2024-09-12T23:40:34.640 INFO:tasks.rados.rados.0.smithi136.stdout:748: finishing write tid 3 to smithi13667081-25 2024-09-12T23:40:34.641 INFO:tasks.rados.rados.0.smithi136.stdout:748: finishing write tid 4 to smithi13667081-25 2024-09-12T23:40:34.641 INFO:tasks.rados.rados.0.smithi136.stdout:748: finishing write tid 5 to smithi13667081-25 2024-09-12T23:40:34.641 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 25 v 99 (ObjNum 253 snap 69 seq_num 253) dirty exists 2024-09-12T23:40:34.641 INFO:tasks.rados.rados.0.smithi136.stdout:748: left oid 25 (ObjNum 253 snap 69 seq_num 253) 2024-09-12T23:40:34.641 INFO:tasks.rados.rados.0.smithi136.stdout:750: expect (ObjNum 88 snap 18 seq_num 88) 2024-09-12T23:40:34.689 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:34 smithi181 bash[24776]: cephadm 2024-09-12T23:40:32.796541+0000 mgr.x (mgr.24517) 38 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi181 2024-09-12T23:40:34.689 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:34 smithi181 bash[24776]: cluster 2024-09-12T23:40:33.518830+0000 mon.a (mon.0) 1301 : cluster [DBG] osdmap e198: 8 total, 8 up, 8 in 2024-09-12T23:40:34.785 INFO:tasks.rados.rados.0.smithi136.stdout:751: finishing write tid 1 to smithi13667081-38 2024-09-12T23:40:34.785 INFO:tasks.rados.rados.0.smithi136.stdout:751: finishing write tid 2 to smithi13667081-38 2024-09-12T23:40:34.785 INFO:tasks.rados.rados.0.smithi136.stdout:751: finishing write tid 3 to smithi13667081-38 2024-09-12T23:40:34.786 INFO:tasks.rados.rados.0.smithi136.stdout:751: finishing write tid 4 to smithi13667081-38 2024-09-12T23:40:34.786 INFO:tasks.rados.rados.0.smithi136.stdout:752: finishing copy_from racing read to smithi13667081-19 2024-09-12T23:40:34.786 INFO:tasks.rados.rados.0.smithi136.stdout:751: finishing write tid 5 to smithi13667081-38 2024-09-12T23:40:34.786 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 38 v 274 (ObjNum 254 snap 69 seq_num 254) dirty exists 2024-09-12T23:40:34.786 INFO:tasks.rados.rados.0.smithi136.stdout:751: left oid 38 (ObjNum 254 snap 69 seq_num 254) 2024-09-12T23:40:34.786 INFO:tasks.rados.rados.0.smithi136.stdout:752: finishing copy_from to smithi13667081-19 2024-09-12T23:40:34.786 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 19 v 230 (ObjNum 193 snap 47 seq_num 193) dirty exists 2024-09-12T23:40:34.786 INFO:tasks.rados.rados.0.smithi136.stdout:748: done (3 left) 2024-09-12T23:40:34.786 INFO:tasks.rados.rados.0.smithi136.stdout:750: done (2 left) 2024-09-12T23:40:34.786 INFO:tasks.rados.rados.0.smithi136.stdout:751: done (1 left) 2024-09-12T23:40:34.786 INFO:tasks.rados.rados.0.smithi136.stdout:752: done (0 left) 2024-09-12T23:40:34.786 INFO:tasks.rados.rados.0.smithi136.stdout:753: snap_create 2024-09-12T23:40:35.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:34 smithi136 bash[32189]: cephadm 2024-09-12T23:40:32.796541+0000 mgr.x (mgr.24517) 38 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi181 2024-09-12T23:40:35.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:34 smithi136 bash[32189]: cluster 2024-09-12T23:40:33.518830+0000 mon.a (mon.0) 1301 : cluster [DBG] osdmap e198: 8 total, 8 up, 8 in 2024-09-12T23:40:35.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:34 smithi136 bash[22601]: cephadm 2024-09-12T23:40:32.796541+0000 mgr.x (mgr.24517) 38 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi181 2024-09-12T23:40:35.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:34 smithi136 bash[22601]: cluster 2024-09-12T23:40:33.518830+0000 mon.a (mon.0) 1301 : cluster [DBG] osdmap e198: 8 total, 8 up, 8 in 2024-09-12T23:40:35.528 INFO:tasks.rados.rados.0.smithi136.stdout:753: done (0 left) 2024-09-12T23:40:35.528 INFO:tasks.rados.rados.0.smithi136.stdout:754: rollback oid 46 current snap is 70 2024-09-12T23:40:35.528 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 46 to 65 2024-09-12T23:40:35.529 INFO:tasks.rados.rados.0.smithi136.stdout:755: copy_from oid 35 from oid 11 current snap is 70 2024-09-12T23:40:35.529 INFO:tasks.rados.rados.0.smithi136.stdout:756: read oid 30 snap -1 2024-09-12T23:40:35.529 INFO:tasks.rados.rados.0.smithi136.stdout:756: expect deleted 2024-09-12T23:40:35.529 INFO:tasks.rados.rados.0.smithi136.stdout:757: copy_from oid 6 from oid 38 current snap is 70 2024-09-12T23:40:35.529 INFO:tasks.rados.rados.0.smithi136.stdout:758: delete oid 33 current snap is 70 2024-09-12T23:40:35.531 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:40:35 smithi181 bash[52367]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:40:35] "GET /metrics HTTP/1.1" 200 36148 "" "Prometheus/2.43.0" 2024-09-12T23:40:35.534 INFO:tasks.rados.rados.0.smithi136.stdout:755: finishing copy_from to smithi13667081-35 2024-09-12T23:40:35.534 INFO:tasks.rados.rados.0.smithi136.stdout:755: got expected ENOENT (src dne) 2024-09-12T23:40:35.534 INFO:tasks.rados.rados.0.smithi136.stdout:755: finishing copy_from racing read to smithi13667081-35 2024-09-12T23:40:35.539 INFO:tasks.rados.rados.0.smithi136.stdout:754: finishing rollback tid 0 to smithi13667081-46 2024-09-12T23:40:35.539 INFO:tasks.rados.rados.0.smithi136.stdout:754: finishing rollback tid 1 to smithi13667081-46 2024-09-12T23:40:35.540 INFO:tasks.rados.rados.0.smithi136.stdout:754: finishing rollback tid 2 to smithi13667081-46 2024-09-12T23:40:35.540 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 46 v 190 (ObjNum 233 snap 56 seq_num 233) dirty exists 2024-09-12T23:40:35.543 INFO:tasks.rados.rados.0.smithi136.stdout:754: done (4 left) 2024-09-12T23:40:35.544 INFO:tasks.rados.rados.0.smithi136.stdout:755: done (3 left) 2024-09-12T23:40:35.544 INFO:tasks.rados.rados.0.smithi136.stdout:756: done (2 left) 2024-09-12T23:40:35.544 INFO:tasks.rados.rados.0.smithi136.stdout:758: done (1 left) 2024-09-12T23:40:35.544 INFO:tasks.rados.rados.0.smithi136.stdout:759: read oid 9 snap -1 2024-09-12T23:40:35.544 INFO:tasks.rados.rados.0.smithi136.stdout:759: expect (ObjNum 6 snap 0 seq_num 6) 2024-09-12T23:40:35.544 INFO:tasks.rados.rados.0.smithi136.stdout:760: read oid 24 snap 69 2024-09-12T23:40:35.544 INFO:tasks.rados.rados.0.smithi136.stdout:760: expect (ObjNum 242 snap 60 seq_num 242) 2024-09-12T23:40:35.544 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 21 current snap is 70 2024-09-12T23:40:35.544 INFO:tasks.rados.rados.0.smithi136.stdout:761: seq_num 256 ranges {2785280=696320,3481600=434176,3915776=770048} 2024-09-12T23:40:35.585 INFO:tasks.rados.rados.0.smithi136.stdout:761: writing smithi13667081-21 from 2785280 to 3481600 tid 1 2024-09-12T23:40:35.591 INFO:tasks.rados.rados.0.smithi136.stdout:761: writing smithi13667081-21 from 3481600 to 3915776 tid 2 2024-09-12T23:40:35.599 INFO:tasks.rados.rados.0.smithi136.stdout:761: writing smithi13667081-21 from 3915776 to 4685824 tid 3 2024-09-12T23:40:35.599 INFO:tasks.rados.rados.0.smithi136.stdout:762: rmattr oid 23 current snap is 70 2024-09-12T23:40:35.599 INFO:tasks.rados.rados.0.smithi136.stdout:759: expect (ObjNum 6 snap 0 seq_num 6) 2024-09-12T23:40:35.603 INFO:tasks.rados.rados.0.smithi136.stdout:760: expect (ObjNum 242 snap 60 seq_num 242) 2024-09-12T23:40:35.652 INFO:tasks.rados.rados.0.smithi136.stdout:757: finishing copy_from racing read to smithi13667081-6 2024-09-12T23:40:35.652 INFO:tasks.rados.rados.0.smithi136.stdout:757: finishing copy_from to smithi13667081-6 2024-09-12T23:40:35.652 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 6 v 274 (ObjNum 254 snap 69 seq_num 254) dirty exists 2024-09-12T23:40:35.652 INFO:tasks.rados.rados.0.smithi136.stdout:761: finishing write tid 1 to smithi13667081-21 2024-09-12T23:40:35.652 INFO:tasks.rados.rados.0.smithi136.stdout:761: finishing write tid 2 to smithi13667081-21 2024-09-12T23:40:35.652 INFO:tasks.rados.rados.0.smithi136.stdout:761: finishing write tid 3 to smithi13667081-21 2024-09-12T23:40:35.652 INFO:tasks.rados.rados.0.smithi136.stdout:761: finishing write tid 4 to smithi13667081-21 2024-09-12T23:40:35.652 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 23 v 420 (ObjNum 246 snap 64 seq_num 246) dirty exists 2024-09-12T23:40:35.653 INFO:tasks.rados.rados.0.smithi136.stdout:761: finishing write tid 5 to smithi13667081-21 2024-09-12T23:40:35.653 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 21 v 166 (ObjNum 256 snap 70 seq_num 256) dirty exists 2024-09-12T23:40:35.653 INFO:tasks.rados.rados.0.smithi136.stdout:761: left oid 21 (ObjNum 256 snap 70 seq_num 256) 2024-09-12T23:40:35.653 INFO:tasks.rados.rados.0.smithi136.stdout:757: done (4 left) 2024-09-12T23:40:35.653 INFO:tasks.rados.rados.0.smithi136.stdout:759: done (3 left) 2024-09-12T23:40:35.653 INFO:tasks.rados.rados.0.smithi136.stdout:760: done (2 left) 2024-09-12T23:40:35.653 INFO:tasks.rados.rados.0.smithi136.stdout:761: done (1 left) 2024-09-12T23:40:35.653 INFO:tasks.rados.rados.0.smithi136.stdout:762: done (0 left) 2024-09-12T23:40:35.653 INFO:tasks.rados.rados.0.smithi136.stdout:763: snap_remove snap 60 2024-09-12T23:40:35.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:35 smithi181 bash[24776]: cluster 2024-09-12T23:40:33.564737+0000 mgr.x (mgr.24517) 39 : cluster [DBG] pgmap v42: 121 pgs: 121 active+clean; 81 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:40:35.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:35 smithi181 bash[24776]: cluster 2024-09-12T23:40:34.531369+0000 mon.a (mon.0) 1302 : cluster [DBG] osdmap e199: 8 total, 8 up, 8 in 2024-09-12T23:40:36.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:35 smithi136 bash[22601]: cluster 2024-09-12T23:40:33.564737+0000 mgr.x (mgr.24517) 39 : cluster [DBG] pgmap v42: 121 pgs: 121 active+clean; 81 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:40:36.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:35 smithi136 bash[22601]: cluster 2024-09-12T23:40:34.531369+0000 mon.a (mon.0) 1302 : cluster [DBG] osdmap e199: 8 total, 8 up, 8 in 2024-09-12T23:40:36.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:35 smithi136 bash[32189]: cluster 2024-09-12T23:40:33.564737+0000 mgr.x (mgr.24517) 39 : cluster [DBG] pgmap v42: 121 pgs: 121 active+clean; 81 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:40:36.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:35 smithi136 bash[32189]: cluster 2024-09-12T23:40:34.531369+0000 mon.a (mon.0) 1302 : cluster [DBG] osdmap e199: 8 total, 8 up, 8 in 2024-09-12T23:40:36.006 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:40:35 smithi136 bash[68390]: ts=2024-09-12T23:40:35.584Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.00057317s 2024-09-12T23:40:36.505 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:36 smithi136 bash[22852]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:40:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-12T23:40:36.553 INFO:tasks.rados.rados.0.smithi136.stdout:763: done (0 left) 2024-09-12T23:40:36.553 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 23 current snap is 70 2024-09-12T23:40:36.553 INFO:tasks.rados.rados.0.smithi136.stdout:764: seq_num 257 ranges {6209536=712704,6922240=720896,7643136=483328} 2024-09-12T23:40:36.610 INFO:tasks.rados.rados.0.smithi136.stdout:764: writing smithi13667081-23 from 6209536 to 6922240 tid 1 2024-09-12T23:40:36.615 INFO:tasks.rados.rados.0.smithi136.stdout:764: writing smithi13667081-23 from 6922240 to 7643136 tid 2 2024-09-12T23:40:36.619 INFO:tasks.rados.rados.0.smithi136.stdout:764: writing smithi13667081-23 from 7643136 to 8126464 tid 3 2024-09-12T23:40:36.619 INFO:tasks.rados.rados.0.smithi136.stdout:765: rollback oid 44 current snap is 70 2024-09-12T23:40:36.619 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 44 to 66 2024-09-12T23:40:36.619 INFO:tasks.rados.rados.0.smithi136.stdout:764: finishing write tid 1 to smithi13667081-23 2024-09-12T23:40:36.619 INFO:tasks.rados.rados.0.smithi136.stdout:766: snap_create 2024-09-12T23:40:36.623 INFO:tasks.rados.rados.0.smithi136.stdout:764: finishing write tid 2 to smithi13667081-23 2024-09-12T23:40:36.624 INFO:tasks.rados.rados.0.smithi136.stdout:764: finishing write tid 3 to smithi13667081-23 2024-09-12T23:40:36.625 INFO:tasks.rados.rados.0.smithi136.stdout:764: finishing write tid 4 to smithi13667081-23 2024-09-12T23:40:36.625 INFO:tasks.rados.rados.0.smithi136.stdout:765: finishing rollback tid 0 to smithi13667081-44 2024-09-12T23:40:36.626 INFO:tasks.rados.rados.0.smithi136.stdout:765: finishing rollback tid 1 to smithi13667081-44 2024-09-12T23:40:36.626 INFO:tasks.rados.rados.0.smithi136.stdout:764: finishing write tid 5 to smithi13667081-23 2024-09-12T23:40:36.626 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 23 v 430 (ObjNum 257 snap 70 seq_num 257) dirty exists 2024-09-12T23:40:36.627 INFO:tasks.rados.rados.0.smithi136.stdout:764: left oid 23 (ObjNum 257 snap 70 seq_num 257) 2024-09-12T23:40:36.627 INFO:tasks.rados.rados.0.smithi136.stdout:765: finishing rollback tid 2 to smithi13667081-44 2024-09-12T23:40:36.627 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 44 v 396 (ObjNum 49 snap 0 seq_num 49) dirty exists 2024-09-12T23:40:36.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:36 smithi181 bash[24776]: cluster 2024-09-12T23:40:35.532239+0000 mon.a (mon.0) 1303 : cluster [DBG] osdmap e200: 8 total, 8 up, 8 in 2024-09-12T23:40:36.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:36 smithi181 bash[24776]: cluster 2024-09-12T23:40:35.565416+0000 mgr.x (mgr.24517) 40 : cluster [DBG] pgmap v45: 121 pgs: 121 active+clean; 86 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.4 MiB/s rd, 1.2 MiB/s wr, 17 op/s 2024-09-12T23:40:37.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:36 smithi136 bash[22601]: cluster 2024-09-12T23:40:35.532239+0000 mon.a (mon.0) 1303 : cluster [DBG] osdmap e200: 8 total, 8 up, 8 in 2024-09-12T23:40:37.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:36 smithi136 bash[22601]: cluster 2024-09-12T23:40:35.565416+0000 mgr.x (mgr.24517) 40 : cluster [DBG] pgmap v45: 121 pgs: 121 active+clean; 86 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.4 MiB/s rd, 1.2 MiB/s wr, 17 op/s 2024-09-12T23:40:37.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:36 smithi136 bash[32189]: cluster 2024-09-12T23:40:35.532239+0000 mon.a (mon.0) 1303 : cluster [DBG] osdmap e200: 8 total, 8 up, 8 in 2024-09-12T23:40:37.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:36 smithi136 bash[32189]: cluster 2024-09-12T23:40:35.565416+0000 mgr.x (mgr.24517) 40 : cluster [DBG] pgmap v45: 121 pgs: 121 active+clean; 86 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.4 MiB/s rd, 1.2 MiB/s wr, 17 op/s 2024-09-12T23:40:37.344 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:40:36 smithi181 systemd[1]: Stopping Ceph prometheus.a for da58a47c-715d-11ef-bceb-c7b262605968... 2024-09-12T23:40:37.344 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:40:37 smithi181 bash[51487]: ts=2024-09-12T23:40:37.137Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-12T23:40:37.344 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:40:37 smithi181 bash[51487]: ts=2024-09-12T23:40:37.138Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-12T23:40:37.344 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:40:37 smithi181 bash[51487]: ts=2024-09-12T23:40:37.138Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-12T23:40:37.344 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:40:37 smithi181 bash[51487]: ts=2024-09-12T23:40:37.138Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-12T23:40:37.345 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:40:37 smithi181 bash[51487]: ts=2024-09-12T23:40:37.138Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-12T23:40:37.345 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:40:37 smithi181 bash[51487]: ts=2024-09-12T23:40:37.138Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-12T23:40:37.345 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:40:37 smithi181 bash[51487]: ts=2024-09-12T23:40:37.138Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-12T23:40:37.345 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:40:37 smithi181 bash[51487]: ts=2024-09-12T23:40:37.138Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-12T23:40:37.345 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:40:37 smithi181 bash[51487]: ts=2024-09-12T23:40:37.138Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-12T23:40:37.345 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:40:37 smithi181 bash[51487]: ts=2024-09-12T23:40:37.139Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-12T23:40:37.345 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:40:37 smithi181 bash[51487]: ts=2024-09-12T23:40:37.139Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-12T23:40:37.345 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:40:37 smithi181 bash[51487]: ts=2024-09-12T23:40:37.139Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-12T23:40:37.556 INFO:tasks.rados.rados.0.smithi136.stdout:764: done (2 left) 2024-09-12T23:40:37.556 INFO:tasks.rados.rados.0.smithi136.stdout:765: done (1 left) 2024-09-12T23:40:37.557 INFO:tasks.rados.rados.0.smithi136.stdout:766: done (0 left) 2024-09-12T23:40:37.557 INFO:tasks.rados.rados.0.smithi136.stdout:767: copy_from oid 48 from oid 7 current snap is 71 2024-09-12T23:40:37.557 INFO:tasks.rados.rados.0.smithi136.stdout:768: read oid 2 snap -1 2024-09-12T23:40:37.557 INFO:tasks.rados.rados.0.smithi136.stdout:768: expect (ObjNum 193 snap 47 seq_num 193) 2024-09-12T23:40:37.557 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 13 current snap is 71 2024-09-12T23:40:37.557 INFO:tasks.rados.rados.0.smithi136.stdout:769: seq_num 258 ranges {352256=729088,1081344=606208,1687552=598016} 2024-09-12T23:40:37.564 INFO:tasks.rados.rados.0.smithi136.stdout:769: writing smithi13667081-13 from 352256 to 1081344 tid 1 2024-09-12T23:40:37.569 INFO:tasks.rados.rados.0.smithi136.stdout:769: writing smithi13667081-13 from 1081344 to 1687552 tid 2 2024-09-12T23:40:37.574 INFO:tasks.rados.rados.0.smithi136.stdout:769: writing smithi13667081-13 from 1687552 to 2285568 tid 3 2024-09-12T23:40:37.574 INFO:tasks.rados.rados.0.smithi136.stdout:770: read oid 47 snap -1 2024-09-12T23:40:37.574 INFO:tasks.rados.rados.0.smithi136.stdout:770: expect (ObjNum 231 snap 56 seq_num 231) 2024-09-12T23:40:37.574 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 18 current snap is 71 2024-09-12T23:40:37.574 INFO:tasks.rados.rados.0.smithi136.stdout:771: seq_num 259 ranges {360448=745472,1105920=540672,1646592=581632,2228224=81920} 2024-09-12T23:40:37.582 INFO:tasks.rados.rados.0.smithi136.stdout:771: writing smithi13667081-18 from 360448 to 1105920 tid 1 2024-09-12T23:40:37.585 INFO:tasks.rados.rados.0.smithi136.stdout:771: writing smithi13667081-18 from 1105920 to 1646592 tid 2 2024-09-12T23:40:37.590 INFO:tasks.rados.rados.0.smithi136.stdout:771: writing smithi13667081-18 from 1646592 to 2228224 tid 3 2024-09-12T23:40:37.590 INFO:tasks.rados.rados.0.smithi136.stdout:771: writing smithi13667081-18 from 2228224 to 2310144 tid 4 2024-09-12T23:40:37.591 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 33 current snap is 71 2024-09-12T23:40:37.591 INFO:tasks.rados.rados.0.smithi136.stdout:772: seq_num 260 ranges {0=761856,761856=417792,1179648=786432} 2024-09-12T23:40:37.596 INFO:tasks.rados.rados.0.smithi136.stdout:772: writing smithi13667081-33 from 0 to 761856 tid 1 2024-09-12T23:40:37.599 INFO:tasks.rados.rados.0.smithi136.stdout:772: writing smithi13667081-33 from 761856 to 1179648 tid 2 2024-09-12T23:40:37.605 INFO:tasks.rados.rados.0.smithi136.stdout:772: writing smithi13667081-33 from 1179648 to 1966080 tid 3 2024-09-12T23:40:37.605 INFO:tasks.rados.rados.0.smithi136.stdout:773: read oid 37 snap -1 2024-09-12T23:40:37.605 INFO:tasks.rados.rados.0.smithi136.stdout:773: expect (ObjNum 134 snap 35 seq_num 134) 2024-09-12T23:40:37.605 INFO:tasks.rados.rados.0.smithi136.stdout:774: read oid 38 snap -1 2024-09-12T23:40:37.605 INFO:tasks.rados.rados.0.smithi136.stdout:774: expect (ObjNum 254 snap 69 seq_num 254) 2024-09-12T23:40:37.605 INFO:tasks.rados.rados.0.smithi136.stdout:775: copy_from oid 50 from oid 12 current snap is 71 2024-09-12T23:40:37.606 INFO:tasks.rados.rados.0.smithi136.stdout:769: finishing write tid 1 to smithi13667081-13 2024-09-12T23:40:37.606 INFO:tasks.rados.rados.0.smithi136.stdout:768: expect (ObjNum 193 snap 47 seq_num 193) 2024-09-12T23:40:37.665 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:37 smithi181 bash[24776]: cluster 2024-09-12T23:40:36.556217+0000 mon.a (mon.0) 1304 : cluster [DBG] osdmap e201: 8 total, 8 up, 8 in 2024-09-12T23:40:37.724 INFO:tasks.rados.rados.0.smithi136.stdout:767: finishing copy_from racing read to smithi13667081-48 2024-09-12T23:40:37.724 INFO:tasks.rados.rados.0.smithi136.stdout:769: finishing write tid 2 to smithi13667081-13 2024-09-12T23:40:37.724 INFO:tasks.rados.rados.0.smithi136.stdout:769: finishing write tid 3 to smithi13667081-13 2024-09-12T23:40:37.724 INFO:tasks.rados.rados.0.smithi136.stdout:768: done (8 left) 2024-09-12T23:40:37.724 INFO:tasks.rados.rados.0.smithi136.stdout:776: snap_remove snap 69 2024-09-12T23:40:37.957 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:40:37 smithi181 bash[58609]: ceph-da58a47c-715d-11ef-bceb-c7b262605968-prometheus-a 2024-09-12T23:40:37.957 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:40:37 smithi181 bash[58669]: ceph-da58a47c-715d-11ef-bceb-c7b262605968-prometheus-a 2024-09-12T23:40:37.957 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:40:37 smithi181 systemd[1]: ceph-da58a47c-715d-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-12T23:40:37.957 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:40:37 smithi181 systemd[1]: Stopped Ceph prometheus.a for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:40:37.958 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:40:37 smithi181 systemd[1]: Started Ceph prometheus.a for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:40:38.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:37 smithi136 bash[22601]: cluster 2024-09-12T23:40:36.556217+0000 mon.a (mon.0) 1304 : cluster [DBG] osdmap e201: 8 total, 8 up, 8 in 2024-09-12T23:40:38.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:37 smithi136 bash[32189]: cluster 2024-09-12T23:40:36.556217+0000 mon.a (mon.0) 1304 : cluster [DBG] osdmap e201: 8 total, 8 up, 8 in 2024-09-12T23:40:38.344 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:38 smithi181 systemd[1]: Stopping Ceph node-exporter.b for da58a47c-715d-11ef-bceb-c7b262605968... 2024-09-12T23:40:38.568 INFO:tasks.rados.rados.0.smithi136.stdout:776: done (8 left) 2024-09-12T23:40:38.568 INFO:tasks.rados.rados.0.smithi136.stdout:777: delete oid 28 current snap is 71 2024-09-12T23:40:38.568 INFO:tasks.rados.rados.0.smithi136.stdout:767: finishing copy_from to smithi13667081-48 2024-09-12T23:40:38.568 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 48 v 264 (ObjNum 33 snap 0 seq_num 4243655008) dirty exists 2024-09-12T23:40:38.568 INFO:tasks.rados.rados.0.smithi136.stdout:769: finishing write tid 4 to smithi13667081-13 2024-09-12T23:40:38.568 INFO:tasks.rados.rados.0.smithi136.stdout:769: finishing write tid 5 to smithi13667081-13 2024-09-12T23:40:38.568 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 13 v 281 (ObjNum 258 snap 71 seq_num 258) dirty exists 2024-09-12T23:40:38.568 INFO:tasks.rados.rados.0.smithi136.stdout:769: left oid 13 (ObjNum 258 snap 71 seq_num 258) 2024-09-12T23:40:38.568 INFO:tasks.rados.rados.0.smithi136.stdout:770: expect (ObjNum 231 snap 56 seq_num 231) 2024-09-12T23:40:38.617 INFO:tasks.rados.rados.0.smithi136.stdout:771: finishing write tid 1 to smithi13667081-18 2024-09-12T23:40:38.617 INFO:tasks.rados.rados.0.smithi136.stdout:771: finishing write tid 2 to smithi13667081-18 2024-09-12T23:40:38.617 INFO:tasks.rados.rados.0.smithi136.stdout:771: finishing write tid 3 to smithi13667081-18 2024-09-12T23:40:38.617 INFO:tasks.rados.rados.0.smithi136.stdout:771: finishing write tid 4 to smithi13667081-18 2024-09-12T23:40:38.617 INFO:tasks.rados.rados.0.smithi136.stdout:767: done (8 left) 2024-09-12T23:40:38.617 INFO:tasks.rados.rados.0.smithi136.stdout:769: done (7 left) 2024-09-12T23:40:38.618 INFO:tasks.rados.rados.0.smithi136.stdout:770: done (6 left) 2024-09-12T23:40:38.618 INFO:tasks.rados.rados.0.smithi136.stdout:777: done (5 left) 2024-09-12T23:40:38.618 INFO:tasks.rados.rados.0.smithi136.stdout:778: read oid 24 snap -1 2024-09-12T23:40:38.618 INFO:tasks.rados.rados.0.smithi136.stdout:778: expect (ObjNum 242 snap 60 seq_num 242) 2024-09-12T23:40:38.618 INFO:tasks.rados.rados.0.smithi136.stdout:779: snap_remove snap 56 2024-09-12T23:40:38.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:38 smithi181 bash[24776]: cluster 2024-09-12T23:40:37.560826+0000 mon.a (mon.0) 1305 : cluster [DBG] osdmap e202: 8 total, 8 up, 8 in 2024-09-12T23:40:38.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:38 smithi181 bash[24776]: cluster 2024-09-12T23:40:37.566141+0000 mgr.x (mgr.24517) 41 : cluster [DBG] pgmap v48: 121 pgs: 121 active+clean; 86 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.4 MiB/s rd, 1.2 MiB/s wr, 17 op/s 2024-09-12T23:40:38.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:38 smithi181 bash[24776]: audit 2024-09-12T23:40:37.735337+0000 mon.b (mon.1) 95 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:40:38.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:38 smithi181 bash[24776]: audit 2024-09-12T23:40:37.894020+0000 mon.a (mon.0) 1306 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:38.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:38 smithi181 bash[24776]: audit 2024-09-12T23:40:37.904687+0000 mon.a (mon.0) 1307 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:38.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:38 smithi181 bash[24776]: cephadm 2024-09-12T23:40:37.907099+0000 mgr.x (mgr.24517) 42 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2024-09-12T23:40:38.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:38 smithi181 bash[24776]: cephadm 2024-09-12T23:40:37.907471+0000 mgr.x (mgr.24517) 43 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi181 2024-09-12T23:40:39.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:38 smithi136 bash[22601]: cluster 2024-09-12T23:40:37.560826+0000 mon.a (mon.0) 1305 : cluster [DBG] osdmap e202: 8 total, 8 up, 8 in 2024-09-12T23:40:39.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:38 smithi136 bash[22601]: cluster 2024-09-12T23:40:37.566141+0000 mgr.x (mgr.24517) 41 : cluster [DBG] pgmap v48: 121 pgs: 121 active+clean; 86 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.4 MiB/s rd, 1.2 MiB/s wr, 17 op/s 2024-09-12T23:40:39.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:38 smithi136 bash[22601]: audit 2024-09-12T23:40:37.735337+0000 mon.b (mon.1) 95 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:40:39.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:38 smithi136 bash[22601]: audit 2024-09-12T23:40:37.894020+0000 mon.a (mon.0) 1306 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:39.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:38 smithi136 bash[22601]: audit 2024-09-12T23:40:37.904687+0000 mon.a (mon.0) 1307 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:39.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:38 smithi136 bash[22601]: cephadm 2024-09-12T23:40:37.907099+0000 mgr.x (mgr.24517) 42 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2024-09-12T23:40:39.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:38 smithi136 bash[22601]: cephadm 2024-09-12T23:40:37.907471+0000 mgr.x (mgr.24517) 43 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi181 2024-09-12T23:40:39.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:38 smithi136 bash[32189]: cluster 2024-09-12T23:40:37.560826+0000 mon.a (mon.0) 1305 : cluster [DBG] osdmap e202: 8 total, 8 up, 8 in 2024-09-12T23:40:39.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:38 smithi136 bash[32189]: cluster 2024-09-12T23:40:37.566141+0000 mgr.x (mgr.24517) 41 : cluster [DBG] pgmap v48: 121 pgs: 121 active+clean; 86 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.4 MiB/s rd, 1.2 MiB/s wr, 17 op/s 2024-09-12T23:40:39.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:38 smithi136 bash[32189]: audit 2024-09-12T23:40:37.735337+0000 mon.b (mon.1) 95 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:40:39.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:38 smithi136 bash[32189]: audit 2024-09-12T23:40:37.894020+0000 mon.a (mon.0) 1306 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:39.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:38 smithi136 bash[32189]: audit 2024-09-12T23:40:37.904687+0000 mon.a (mon.0) 1307 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:39.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:38 smithi136 bash[32189]: cephadm 2024-09-12T23:40:37.907099+0000 mgr.x (mgr.24517) 42 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2024-09-12T23:40:39.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:38 smithi136 bash[32189]: cephadm 2024-09-12T23:40:37.907471+0000 mgr.x (mgr.24517) 43 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi181 2024-09-12T23:40:39.569 INFO:tasks.rados.rados.0.smithi136.stdout:779: done (6 left) 2024-09-12T23:40:39.569 INFO:tasks.rados.rados.0.smithi136.stdout:780: read oid 11 snap -1 2024-09-12T23:40:39.569 INFO:tasks.rados.rados.0.smithi136.stdout:780: expect deleted 2024-09-12T23:40:39.574 INFO:tasks.rados.rados.0.smithi136.stdout:781: read oid 39 snap -1 2024-09-12T23:40:39.575 INFO:tasks.rados.rados.0.smithi136.stdout:781: expect (ObjNum 225 snap 56 seq_num 225) 2024-09-12T23:40:39.575 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 41 current snap is 71 2024-09-12T23:40:39.575 INFO:tasks.rados.rados.0.smithi136.stdout:782: seq_num 262 ranges {2301952=794624,3096576=589824,3686400=614400} 2024-09-12T23:40:39.589 INFO:tasks.rados.rados.0.smithi136.stdout:782: writing smithi13667081-41 from 2301952 to 3096576 tid 1 2024-09-12T23:40:39.593 INFO:tasks.rados.rados.0.smithi136.stdout:782: writing smithi13667081-41 from 3096576 to 3686400 tid 2 2024-09-12T23:40:39.597 INFO:tasks.rados.rados.0.smithi136.stdout:782: writing smithi13667081-41 from 3686400 to 4300800 tid 3 2024-09-12T23:40:39.597 INFO:tasks.rados.rados.0.smithi136.stdout:771: finishing write tid 5 to smithi13667081-18 2024-09-12T23:40:39.598 INFO:tasks.rados.rados.0.smithi136.stdout:771: finishing write tid 6 to smithi13667081-18 2024-09-12T23:40:39.598 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 18 v 269 (ObjNum 259 snap 71 seq_num 259) dirty exists 2024-09-12T23:40:39.598 INFO:tasks.rados.rados.0.smithi136.stdout:771: left oid 18 (ObjNum 259 snap 71 seq_num 259) 2024-09-12T23:40:39.598 INFO:tasks.rados.rados.0.smithi136.stdout:772: finishing write tid 1 to smithi13667081-33 2024-09-12T23:40:39.598 INFO:tasks.rados.rados.0.smithi136.stdout:772: finishing write tid 2 to smithi13667081-33 2024-09-12T23:40:39.598 INFO:tasks.rados.rados.0.smithi136.stdout:772: finishing write tid 3 to smithi13667081-33 2024-09-12T23:40:39.598 INFO:tasks.rados.rados.0.smithi136.stdout:772: finishing write tid 4 to smithi13667081-33 2024-09-12T23:40:39.598 INFO:tasks.rados.rados.0.smithi136.stdout:774: expect (ObjNum 254 snap 69 seq_num 254) 2024-09-12T23:40:39.684 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:39 smithi181 bash[24776]: cluster 2024-09-12T23:40:38.569842+0000 mon.a (mon.0) 1308 : cluster [DBG] osdmap e203: 8 total, 8 up, 8 in 2024-09-12T23:40:39.684 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:39 smithi181 bash[58742]: ceph-da58a47c-715d-11ef-bceb-c7b262605968-node-exporter-b 2024-09-12T23:40:39.684 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:39 smithi181 systemd[1]: ceph-da58a47c-715d-11ef-bceb-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-09-12T23:40:39.684 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:39 smithi181 systemd[1]: ceph-da58a47c-715d-11ef-bceb-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-09-12T23:40:39.685 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:39 smithi181 systemd[1]: Stopped Ceph node-exporter.b for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:40:39.685 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:39 smithi181 systemd[1]: Started Ceph node-exporter.b for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:40:39.743 INFO:tasks.rados.rados.0.smithi136.stdout:775: finishing copy_from racing read to smithi13667081-50 2024-09-12T23:40:39.743 INFO:tasks.rados.rados.0.smithi136.stdout:775: finishing copy_from to smithi13667081-50 2024-09-12T23:40:39.743 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 50 v 429 (ObjNum 33 snap 0 seq_num 4242676064) dirty exists 2024-09-12T23:40:39.743 INFO:tasks.rados.rados.0.smithi136.stdout:772: finishing write tid 5 to smithi13667081-33 2024-09-12T23:40:39.743 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 33 v 328 (ObjNum 260 snap 71 seq_num 260) dirty exists 2024-09-12T23:40:39.743 INFO:tasks.rados.rados.0.smithi136.stdout:772: left oid 33 (ObjNum 260 snap 71 seq_num 260) 2024-09-12T23:40:39.743 INFO:tasks.rados.rados.0.smithi136.stdout:773: expect (ObjNum 134 snap 35 seq_num 134) 2024-09-12T23:40:39.994 INFO:tasks.rados.rados.0.smithi136.stdout:778: expect (ObjNum 242 snap 60 seq_num 242) 2024-09-12T23:40:40.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:39 smithi136 bash[32189]: cluster 2024-09-12T23:40:38.569842+0000 mon.a (mon.0) 1308 : cluster [DBG] osdmap e203: 8 total, 8 up, 8 in 2024-09-12T23:40:40.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:39 smithi136 bash[22601]: cluster 2024-09-12T23:40:38.569842+0000 mon.a (mon.0) 1308 : cluster [DBG] osdmap e203: 8 total, 8 up, 8 in 2024-09-12T23:40:40.034 INFO:tasks.rados.rados.0.smithi136.stdout:781: expect (ObjNum 225 snap 56 seq_num 225) 2024-09-12T23:40:40.067 INFO:tasks.rados.rados.0.smithi136.stdout:782: finishing write tid 1 to smithi13667081-41 2024-09-12T23:40:40.067 INFO:tasks.rados.rados.0.smithi136.stdout:782: finishing write tid 2 to smithi13667081-41 2024-09-12T23:40:40.067 INFO:tasks.rados.rados.0.smithi136.stdout:782: finishing write tid 3 to smithi13667081-41 2024-09-12T23:40:40.067 INFO:tasks.rados.rados.0.smithi136.stdout:782: finishing write tid 4 to smithi13667081-41 2024-09-12T23:40:40.067 INFO:tasks.rados.rados.0.smithi136.stdout:782: finishing write tid 5 to smithi13667081-41 2024-09-12T23:40:40.067 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 41 v 339 (ObjNum 262 snap 71 seq_num 262) dirty exists 2024-09-12T23:40:40.067 INFO:tasks.rados.rados.0.smithi136.stdout:782: left oid 41 (ObjNum 262 snap 71 seq_num 262) 2024-09-12T23:40:40.067 INFO:tasks.rados.rados.0.smithi136.stdout:771: done (8 left) 2024-09-12T23:40:40.067 INFO:tasks.rados.rados.0.smithi136.stdout:772: done (7 left) 2024-09-12T23:40:40.067 INFO:tasks.rados.rados.0.smithi136.stdout:773: done (6 left) 2024-09-12T23:40:40.068 INFO:tasks.rados.rados.0.smithi136.stdout:774: done (5 left) 2024-09-12T23:40:40.068 INFO:tasks.rados.rados.0.smithi136.stdout:775: done (4 left) 2024-09-12T23:40:40.068 INFO:tasks.rados.rados.0.smithi136.stdout:778: done (3 left) 2024-09-12T23:40:40.068 INFO:tasks.rados.rados.0.smithi136.stdout:780: done (2 left) 2024-09-12T23:40:40.068 INFO:tasks.rados.rados.0.smithi136.stdout:781: done (1 left) 2024-09-12T23:40:40.068 INFO:tasks.rados.rados.0.smithi136.stdout:782: done (0 left) 2024-09-12T23:40:40.068 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 45 current snap is 71 2024-09-12T23:40:40.068 INFO:tasks.rados.rados.0.smithi136.stdout:783: seq_num 263 ranges {2949120=409600,3358720=475136,3833856=557056,4390912=573440} 2024-09-12T23:40:40.094 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:40:39 smithi181 bash[58696]: ts=2024-09-12T23:40:39.794Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-12T23:40:40.094 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:40:39 smithi181 bash[58696]: ts=2024-09-12T23:40:39.794Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-12T23:40:40.094 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:40:39 smithi181 bash[58696]: ts=2024-09-12T23:40:39.794Z caller=main.go:570 level=info host_details="(Linux 5.15.0-121-generic #131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024 x86_64 smithi181 (none))" 2024-09-12T23:40:40.094 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:40:39 smithi181 bash[58696]: ts=2024-09-12T23:40:39.794Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-12T23:40:40.095 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:40:39 smithi181 bash[58696]: ts=2024-09-12T23:40:39.795Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-12T23:40:40.095 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:40:39 smithi181 bash[58696]: ts=2024-09-12T23:40:39.796Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-12T23:40:40.095 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:40:39 smithi181 bash[58696]: ts=2024-09-12T23:40:39.797Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-12T23:40:40.095 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:40:39 smithi181 bash[58696]: ts=2024-09-12T23:40:39.798Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-12T23:40:40.095 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:40:39 smithi181 bash[58696]: ts=2024-09-12T23:40:39.798Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-12T23:40:40.095 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:40:39 smithi181 bash[58696]: ts=2024-09-12T23:40:39.801Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-12T23:40:40.095 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:40:39 smithi181 bash[58696]: ts=2024-09-12T23:40:39.801Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.777µs 2024-09-12T23:40:40.095 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:40:39 smithi181 bash[58696]: ts=2024-09-12T23:40:39.801Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-12T23:40:40.095 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:40:39 smithi181 bash[58696]: ts=2024-09-12T23:40:39.802Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-09-12T23:40:40.095 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:40:39 smithi181 bash[58696]: ts=2024-09-12T23:40:39.824Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-09-12T23:40:40.095 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:40:39 smithi181 bash[58696]: ts=2024-09-12T23:40:39.824Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-09-12T23:40:40.095 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:40:39 smithi181 bash[58696]: ts=2024-09-12T23:40:39.824Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=22.781µs wal_replay_duration=22.855824ms wbl_replay_duration=148ns total_replay_duration=22.897855ms 2024-09-12T23:40:40.095 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:40:39 smithi181 bash[58696]: ts=2024-09-12T23:40:39.828Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-12T23:40:40.096 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:40:39 smithi181 bash[58696]: ts=2024-09-12T23:40:39.828Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-12T23:40:40.096 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:40:39 smithi181 bash[58696]: ts=2024-09-12T23:40:39.828Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-12T23:40:40.096 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:40:39 smithi181 bash[58696]: ts=2024-09-12T23:40:39.854Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=25.744788ms db_storage=1.202µs remote_storage=1.256µs web_handler=447ns query_engine=1.005µs scrape=299.288µs scrape_sd=98.126µs notify=10.911µs notify_sd=10.654µs rules=24.813432ms tracing=6.672µs 2024-09-12T23:40:40.096 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:40:39 smithi181 bash[58696]: ts=2024-09-12T23:40:39.854Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-12T23:40:40.096 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:40:39 smithi181 bash[58696]: ts=2024-09-12T23:40:39.854Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-12T23:40:40.119 INFO:tasks.rados.rados.0.smithi136.stdout:783: writing smithi13667081-45 from 2949120 to 3358720 tid 1 2024-09-12T23:40:40.124 INFO:tasks.rados.rados.0.smithi136.stdout:783: writing smithi13667081-45 from 3358720 to 3833856 tid 2 2024-09-12T23:40:40.130 INFO:tasks.rados.rados.0.smithi136.stdout:783: writing smithi13667081-45 from 3833856 to 4390912 tid 3 2024-09-12T23:40:40.136 INFO:tasks.rados.rados.0.smithi136.stdout:783: writing smithi13667081-45 from 4390912 to 4964352 tid 4 2024-09-12T23:40:40.136 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 19 current snap is 71 2024-09-12T23:40:40.136 INFO:tasks.rados.rados.0.smithi136.stdout:784: seq_num 264 ranges {2637824=425984,3063808=761856,3825664=770048,4595712=73728} 2024-09-12T23:40:40.161 INFO:tasks.rados.rados.0.smithi136.stdout:784: writing smithi13667081-19 from 2637824 to 3063808 tid 1 2024-09-12T23:40:40.166 INFO:tasks.rados.rados.0.smithi136.stdout:784: writing smithi13667081-19 from 3063808 to 3825664 tid 2 2024-09-12T23:40:40.171 INFO:tasks.rados.rados.0.smithi136.stdout:784: writing smithi13667081-19 from 3825664 to 4595712 tid 3 2024-09-12T23:40:40.172 INFO:tasks.rados.rados.0.smithi136.stdout:784: writing smithi13667081-19 from 4595712 to 4669440 tid 4 2024-09-12T23:40:40.172 INFO:tasks.rados.rados.0.smithi136.stdout:785: rmattr oid 37 current snap is 71 2024-09-12T23:40:40.172 INFO:tasks.rados.rados.0.smithi136.stdout:785: done (2 left) 2024-09-12T23:40:40.172 INFO:tasks.rados.rados.0.smithi136.stdout:786: snap_remove snap 62 2024-09-12T23:40:40.491 INFO:tasks.rados.rados.0.smithi136.stdout:786: done (2 left) 2024-09-12T23:40:40.492 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 50 current snap is 71 2024-09-12T23:40:40.492 INFO:tasks.rados.rados.0.smithi136.stdout:787: seq_num 265 ranges {0=442368,442368=647168,1089536=573440,1662976=385024} 2024-09-12T23:40:40.497 INFO:tasks.rados.rados.0.smithi136.stdout:787: writing smithi13667081-50 from 0 to 442368 tid 1 2024-09-12T23:40:40.502 INFO:tasks.rados.rados.0.smithi136.stdout:787: writing smithi13667081-50 from 442368 to 1089536 tid 2 2024-09-12T23:40:40.506 INFO:tasks.rados.rados.0.smithi136.stdout:787: writing smithi13667081-50 from 1089536 to 1662976 tid 3 2024-09-12T23:40:40.509 INFO:tasks.rados.rados.0.smithi136.stdout:787: writing smithi13667081-50 from 1662976 to 2048000 tid 4 2024-09-12T23:40:40.509 INFO:tasks.rados.rados.0.smithi136.stdout:788: snap_create 2024-09-12T23:40:40.510 INFO:tasks.rados.rados.0.smithi136.stdout:783: finishing write tid 1 to smithi13667081-45 2024-09-12T23:40:40.510 INFO:tasks.rados.rados.0.smithi136.stdout:783: finishing write tid 2 to smithi13667081-45 2024-09-12T23:40:40.510 INFO:tasks.rados.rados.0.smithi136.stdout:783: finishing write tid 3 to smithi13667081-45 2024-09-12T23:40:40.510 INFO:tasks.rados.rados.0.smithi136.stdout:783: finishing write tid 4 to smithi13667081-45 2024-09-12T23:40:40.510 INFO:tasks.rados.rados.0.smithi136.stdout:783: finishing write tid 5 to smithi13667081-45 2024-09-12T23:40:40.510 INFO:tasks.rados.rados.0.smithi136.stdout:783: finishing write tid 6 to smithi13667081-45 2024-09-12T23:40:40.510 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 45 v 273 (ObjNum 263 snap 71 seq_num 263) dirty exists 2024-09-12T23:40:40.510 INFO:tasks.rados.rados.0.smithi136.stdout:783: left oid 45 (ObjNum 263 snap 71 seq_num 263) 2024-09-12T23:40:40.510 INFO:tasks.rados.rados.0.smithi136.stdout:784: finishing write tid 1 to smithi13667081-19 2024-09-12T23:40:40.510 INFO:tasks.rados.rados.0.smithi136.stdout:784: finishing write tid 2 to smithi13667081-19 2024-09-12T23:40:40.510 INFO:tasks.rados.rados.0.smithi136.stdout:784: finishing write tid 3 to smithi13667081-19 2024-09-12T23:40:40.511 INFO:tasks.rados.rados.0.smithi136.stdout:784: finishing write tid 4 to smithi13667081-19 2024-09-12T23:40:40.511 INFO:tasks.rados.rados.0.smithi136.stdout:784: finishing write tid 5 to smithi13667081-19 2024-09-12T23:40:40.511 INFO:tasks.rados.rados.0.smithi136.stdout:784: finishing write tid 6 to smithi13667081-19 2024-09-12T23:40:40.511 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 19 v 271 (ObjNum 264 snap 71 seq_num 264) dirty exists 2024-09-12T23:40:40.511 INFO:tasks.rados.rados.0.smithi136.stdout:784: left oid 19 (ObjNum 264 snap 71 seq_num 264) 2024-09-12T23:40:40.526 INFO:tasks.rados.rados.0.smithi136.stdout:787: finishing write tid 1 to smithi13667081-50 2024-09-12T23:40:40.526 INFO:tasks.rados.rados.0.smithi136.stdout:787: finishing write tid 2 to smithi13667081-50 2024-09-12T23:40:40.527 INFO:tasks.rados.rados.0.smithi136.stdout:787: finishing write tid 3 to smithi13667081-50 2024-09-12T23:40:40.529 INFO:tasks.rados.rados.0.smithi136.stdout:787: finishing write tid 4 to smithi13667081-50 2024-09-12T23:40:40.530 INFO:tasks.rados.rados.0.smithi136.stdout:787: finishing write tid 5 to smithi13667081-50 2024-09-12T23:40:40.537 INFO:tasks.rados.rados.0.smithi136.stdout:787: finishing write tid 6 to smithi13667081-50 2024-09-12T23:40:40.537 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 50 v 434 (ObjNum 265 snap 71 seq_num 265) dirty exists 2024-09-12T23:40:40.537 INFO:tasks.rados.rados.0.smithi136.stdout:787: left oid 50 (ObjNum 265 snap 71 seq_num 265) 2024-09-12T23:40:40.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:40 smithi181 bash[24776]: cluster 2024-09-12T23:40:39.567968+0000 mgr.x (mgr.24517) 44 : cluster [DBG] pgmap v50: 121 pgs: 3 active+clean+snaptrim, 118 active+clean; 106 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 5.1 MiB/s wr, 23 op/s 2024-09-12T23:40:40.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:40 smithi181 bash[24776]: cluster 2024-09-12T23:40:39.571028+0000 mon.a (mon.0) 1309 : cluster [DBG] osdmap e204: 8 total, 8 up, 8 in 2024-09-12T23:40:40.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:40 smithi181 bash[24776]: audit 2024-09-12T23:40:39.645592+0000 mon.a (mon.0) 1310 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:40.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:40 smithi181 bash[24776]: audit 2024-09-12T23:40:39.667569+0000 mon.a (mon.0) 1311 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:40.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:40 smithi181 bash[24776]: cephadm 2024-09-12T23:40:39.669723+0000 mgr.x (mgr.24517) 45 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2024-09-12T23:40:40.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:40 smithi181 bash[24776]: cephadm 2024-09-12T23:40:39.719862+0000 mgr.x (mgr.24517) 46 : cephadm [INF] Reconfiguring daemon grafana.a on smithi181 2024-09-12T23:40:40.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:40 smithi181 bash[24776]: cluster 2024-09-12T23:40:40.495451+0000 mon.a (mon.0) 1312 : cluster [DBG] osdmap e205: 8 total, 8 up, 8 in 2024-09-12T23:40:41.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:40 smithi136 bash[22601]: cluster 2024-09-12T23:40:39.567968+0000 mgr.x (mgr.24517) 44 : cluster [DBG] pgmap v50: 121 pgs: 3 active+clean+snaptrim, 118 active+clean; 106 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 5.1 MiB/s wr, 23 op/s 2024-09-12T23:40:41.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:40 smithi136 bash[22601]: cluster 2024-09-12T23:40:39.571028+0000 mon.a (mon.0) 1309 : cluster [DBG] osdmap e204: 8 total, 8 up, 8 in 2024-09-12T23:40:41.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:40 smithi136 bash[22601]: audit 2024-09-12T23:40:39.645592+0000 mon.a (mon.0) 1310 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:41.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:40 smithi136 bash[22601]: audit 2024-09-12T23:40:39.667569+0000 mon.a (mon.0) 1311 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:41.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:40 smithi136 bash[22601]: cephadm 2024-09-12T23:40:39.669723+0000 mgr.x (mgr.24517) 45 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2024-09-12T23:40:41.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:40 smithi136 bash[22601]: cephadm 2024-09-12T23:40:39.719862+0000 mgr.x (mgr.24517) 46 : cephadm [INF] Reconfiguring daemon grafana.a on smithi181 2024-09-12T23:40:41.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:40 smithi136 bash[22601]: cluster 2024-09-12T23:40:40.495451+0000 mon.a (mon.0) 1312 : cluster [DBG] osdmap e205: 8 total, 8 up, 8 in 2024-09-12T23:40:41.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:40 smithi136 bash[32189]: cluster 2024-09-12T23:40:39.567968+0000 mgr.x (mgr.24517) 44 : cluster [DBG] pgmap v50: 121 pgs: 3 active+clean+snaptrim, 118 active+clean; 106 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 5.1 MiB/s wr, 23 op/s 2024-09-12T23:40:41.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:40 smithi136 bash[32189]: cluster 2024-09-12T23:40:39.571028+0000 mon.a (mon.0) 1309 : cluster [DBG] osdmap e204: 8 total, 8 up, 8 in 2024-09-12T23:40:41.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:40 smithi136 bash[32189]: audit 2024-09-12T23:40:39.645592+0000 mon.a (mon.0) 1310 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:41.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:40 smithi136 bash[32189]: audit 2024-09-12T23:40:39.667569+0000 mon.a (mon.0) 1311 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:41.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:40 smithi136 bash[32189]: cephadm 2024-09-12T23:40:39.669723+0000 mgr.x (mgr.24517) 45 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2024-09-12T23:40:41.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:40 smithi136 bash[32189]: cephadm 2024-09-12T23:40:39.719862+0000 mgr.x (mgr.24517) 46 : cephadm [INF] Reconfiguring daemon grafana.a on smithi181 2024-09-12T23:40:41.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:40 smithi136 bash[32189]: cluster 2024-09-12T23:40:40.495451+0000 mon.a (mon.0) 1312 : cluster [DBG] osdmap e205: 8 total, 8 up, 8 in 2024-09-12T23:40:41.491 INFO:tasks.rados.rados.0.smithi136.stdout:783: done (3 left) 2024-09-12T23:40:41.491 INFO:tasks.rados.rados.0.smithi136.stdout:784: done (2 left) 2024-09-12T23:40:41.491 INFO:tasks.rados.rados.0.smithi136.stdout:787: done (1 left) 2024-09-12T23:40:41.491 INFO:tasks.rados.rados.0.smithi136.stdout:788: done (0 left) 2024-09-12T23:40:41.491 INFO:tasks.rados.rados.0.smithi136.stdout:789: delete oid 46 current snap is 72 2024-09-12T23:40:41.497 INFO:tasks.rados.rados.0.smithi136.stdout:789: done (0 left) 2024-09-12T23:40:41.497 INFO:tasks.rados.rados.0.smithi136.stdout:790: delete oid 44 current snap is 72 2024-09-12T23:40:41.504 INFO:tasks.rados.rados.0.smithi136.stdout:790: done (0 left) 2024-09-12T23:40:41.504 INFO:tasks.rados.rados.0.smithi136.stdout:791: read oid 46 snap -1 2024-09-12T23:40:41.504 INFO:tasks.rados.rados.0.smithi136.stdout:791: expect deleted 2024-09-12T23:40:41.504 INFO:tasks.rados.rados.0.smithi136.stdout:792: rollback oid 47 current snap is 72 2024-09-12T23:40:41.504 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 47 to 70 2024-09-12T23:40:41.505 INFO:tasks.rados.rados.0.smithi136.stdout:793: read oid 43 snap -1 2024-09-12T23:40:41.505 INFO:tasks.rados.rados.0.smithi136.stdout:793: expect (ObjNum 225 snap 56 seq_num 225) 2024-09-12T23:40:41.505 INFO:tasks.rados.rados.0.smithi136.stdout:794: copy_from oid 41 from oid 49 current snap is 72 2024-09-12T23:40:41.505 INFO:tasks.rados.rados.0.smithi136.stdout:795: snap_remove snap 71 2024-09-12T23:40:41.844 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:41 smithi181 bash[58825]: ts=2024-09-12T23:40:41.663Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-09-12T23:40:41.844 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:41 smithi181 bash[58825]: ts=2024-09-12T23:40:41.663Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-09-12T23:40:41.844 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:41 smithi181 bash[58825]: ts=2024-09-12T23:40:41.663Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-12T23:40:41.844 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:41 smithi181 bash[58825]: ts=2024-09-12T23:40:41.663Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-12T23:40:41.844 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:41 smithi181 bash[58825]: ts=2024-09-12T23:40:41.663Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-12T23:40:41.845 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:41 smithi181 bash[58825]: ts=2024-09-12T23:40:41.663Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-12T23:40:41.845 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:41 smithi181 bash[58825]: ts=2024-09-12T23:40:41.664Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-12T23:40:41.845 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:41 smithi181 bash[58825]: ts=2024-09-12T23:40:41.664Z caller=node_exporter.go:117 level=info collector=arp 2024-09-12T23:40:41.845 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:41 smithi181 bash[58825]: ts=2024-09-12T23:40:41.664Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-12T23:40:41.845 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:41 smithi181 bash[58825]: ts=2024-09-12T23:40:41.664Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-12T23:40:41.845 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:41 smithi181 bash[58825]: ts=2024-09-12T23:40:41.664Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-12T23:40:41.845 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:41 smithi181 bash[58825]: ts=2024-09-12T23:40:41.664Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-12T23:40:41.845 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:41 smithi181 bash[58825]: ts=2024-09-12T23:40:41.664Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-12T23:40:41.845 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:41 smithi181 bash[58825]: ts=2024-09-12T23:40:41.664Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-12T23:40:41.845 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:41 smithi181 bash[58825]: ts=2024-09-12T23:40:41.664Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-12T23:40:41.845 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:41 smithi181 bash[58825]: ts=2024-09-12T23:40:41.664Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-12T23:40:41.846 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:41 smithi181 bash[58825]: ts=2024-09-12T23:40:41.664Z caller=node_exporter.go:117 level=info collector=edac 2024-09-12T23:40:41.846 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:41 smithi181 bash[58825]: ts=2024-09-12T23:40:41.664Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-12T23:40:41.846 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:41 smithi181 bash[58825]: ts=2024-09-12T23:40:41.664Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-12T23:40:41.846 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:41 smithi181 bash[58825]: ts=2024-09-12T23:40:41.664Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-12T23:40:41.846 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:41 smithi181 bash[58825]: ts=2024-09-12T23:40:41.664Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-12T23:40:41.846 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:41 smithi181 bash[58825]: ts=2024-09-12T23:40:41.664Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-12T23:40:41.846 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:41 smithi181 bash[58825]: ts=2024-09-12T23:40:41.664Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-12T23:40:41.846 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:41 smithi181 bash[58825]: ts=2024-09-12T23:40:41.664Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-12T23:40:41.846 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:41 smithi181 bash[58825]: ts=2024-09-12T23:40:41.664Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-12T23:40:41.846 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:41 smithi181 bash[58825]: ts=2024-09-12T23:40:41.664Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-12T23:40:41.846 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:41 smithi181 bash[58825]: ts=2024-09-12T23:40:41.664Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-12T23:40:41.846 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:41 smithi181 bash[58825]: ts=2024-09-12T23:40:41.664Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-12T23:40:41.847 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:41 smithi181 bash[58825]: ts=2024-09-12T23:40:41.664Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-12T23:40:41.847 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:41 smithi181 bash[58825]: ts=2024-09-12T23:40:41.664Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-12T23:40:41.847 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:41 smithi181 bash[58825]: ts=2024-09-12T23:40:41.664Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-12T23:40:41.847 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:41 smithi181 bash[58825]: ts=2024-09-12T23:40:41.664Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-12T23:40:41.847 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:41 smithi181 bash[58825]: ts=2024-09-12T23:40:41.664Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-12T23:40:41.847 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:41 smithi181 bash[58825]: ts=2024-09-12T23:40:41.664Z caller=node_exporter.go:117 level=info collector=os 2024-09-12T23:40:41.847 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:41 smithi181 bash[58825]: ts=2024-09-12T23:40:41.665Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-12T23:40:41.847 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:41 smithi181 bash[58825]: ts=2024-09-12T23:40:41.665Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-12T23:40:41.847 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:41 smithi181 bash[58825]: ts=2024-09-12T23:40:41.665Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-12T23:40:41.847 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:41 smithi181 bash[58825]: ts=2024-09-12T23:40:41.665Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-12T23:40:41.847 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:41 smithi181 bash[58825]: ts=2024-09-12T23:40:41.665Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-12T23:40:41.847 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:41 smithi181 bash[58825]: ts=2024-09-12T23:40:41.665Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-12T23:40:41.847 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:41 smithi181 bash[58825]: ts=2024-09-12T23:40:41.665Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-12T23:40:41.848 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:41 smithi181 bash[58825]: ts=2024-09-12T23:40:41.665Z caller=node_exporter.go:117 level=info collector=stat 2024-09-12T23:40:41.848 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:41 smithi181 bash[58825]: ts=2024-09-12T23:40:41.665Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-12T23:40:41.848 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:41 smithi181 bash[58825]: ts=2024-09-12T23:40:41.665Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-12T23:40:41.848 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:41 smithi181 bash[58825]: ts=2024-09-12T23:40:41.665Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-12T23:40:41.848 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:41 smithi181 bash[58825]: ts=2024-09-12T23:40:41.665Z caller=node_exporter.go:117 level=info collector=time 2024-09-12T23:40:41.848 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:41 smithi181 bash[58825]: ts=2024-09-12T23:40:41.665Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-12T23:40:41.848 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:41 smithi181 bash[58825]: ts=2024-09-12T23:40:41.665Z caller=node_exporter.go:117 level=info collector=uname 2024-09-12T23:40:41.848 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:41 smithi181 bash[58825]: ts=2024-09-12T23:40:41.665Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-12T23:40:41.848 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:41 smithi181 bash[58825]: ts=2024-09-12T23:40:41.665Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-12T23:40:41.848 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:41 smithi181 bash[58825]: ts=2024-09-12T23:40:41.665Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-12T23:40:41.848 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:41 smithi181 bash[58825]: ts=2024-09-12T23:40:41.665Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-09-12T23:40:41.848 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:40:41 smithi181 bash[58825]: ts=2024-09-12T23:40:41.665Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-12T23:40:42.507 INFO:tasks.rados.rados.0.smithi136.stdout:794: finishing copy_from to smithi13667081-41 2024-09-12T23:40:42.507 INFO:tasks.rados.rados.0.smithi136.stdout:794: got expected ENOENT (src dne) 2024-09-12T23:40:42.507 INFO:tasks.rados.rados.0.smithi136.stdout:794: finishing copy_from racing read to smithi13667081-41 2024-09-12T23:40:42.507 INFO:tasks.rados.rados.0.smithi136.stdout:792: finishing rollback tid 0 to smithi13667081-47 2024-09-12T23:40:42.507 INFO:tasks.rados.rados.0.smithi136.stdout:792: finishing rollback tid 1 to smithi13667081-47 2024-09-12T23:40:42.507 INFO:tasks.rados.rados.0.smithi136.stdout:792: finishing rollback tid 2 to smithi13667081-47 2024-09-12T23:40:42.507 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 47 v 274 (ObjNum 231 snap 56 seq_num 231) dirty exists 2024-09-12T23:40:42.508 INFO:tasks.rados.rados.0.smithi136.stdout:793: expect (ObjNum 225 snap 56 seq_num 225) 2024-09-12T23:40:42.570 INFO:tasks.rados.rados.0.smithi136.stdout:791: done (4 left) 2024-09-12T23:40:42.570 INFO:tasks.rados.rados.0.smithi136.stdout:792: done (3 left) 2024-09-12T23:40:42.570 INFO:tasks.rados.rados.0.smithi136.stdout:793: done (2 left) 2024-09-12T23:40:42.570 INFO:tasks.rados.rados.0.smithi136.stdout:794: done (1 left) 2024-09-12T23:40:42.570 INFO:tasks.rados.rados.0.smithi136.stdout:795: done (0 left) 2024-09-12T23:40:42.570 INFO:tasks.rados.rados.0.smithi136.stdout:796: rmattr oid 17 current snap is 72 2024-09-12T23:40:42.570 INFO:tasks.rados.rados.0.smithi136.stdout:796: done (0 left) 2024-09-12T23:40:42.570 INFO:tasks.rados.rados.0.smithi136.stdout:797: read oid 24 snap -1 2024-09-12T23:40:42.570 INFO:tasks.rados.rados.0.smithi136.stdout:797: expect (ObjNum 242 snap 60 seq_num 242) 2024-09-12T23:40:42.570 INFO:tasks.rados.rados.0.smithi136.stdout:798: rollback oid 34 current snap is 72 2024-09-12T23:40:42.570 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 34 to 65 2024-09-12T23:40:42.570 INFO:tasks.rados.rados.0.smithi136.stdout:799: copy_from oid 29 from oid 25 current snap is 72 2024-09-12T23:40:42.571 INFO:tasks.rados.rados.0.smithi136.stdout:800: read oid 17 snap -1 2024-09-12T23:40:42.571 INFO:tasks.rados.rados.0.smithi136.stdout:800: expect (ObjNum 164 snap 43 seq_num 164) 2024-09-12T23:40:42.571 INFO:tasks.rados.rados.0.smithi136.stdout:801: rollback oid 4 current snap is 72 2024-09-12T23:40:42.571 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 4 to 70 2024-09-12T23:40:42.571 INFO:tasks.rados.rados.0.smithi136.stdout:802: snap_create 2024-09-12T23:40:42.591 INFO:tasks.rados.rados.0.smithi136.stdout:800: expect (ObjNum 164 snap 43 seq_num 164) 2024-09-12T23:40:42.605 INFO:tasks.rados.rados.0.smithi136.stdout:801: finishing rollback tid 0 to smithi13667081-4 2024-09-12T23:40:42.605 INFO:tasks.rados.rados.0.smithi136.stdout:797: expect (ObjNum 242 snap 60 seq_num 242) 2024-09-12T23:40:42.661 INFO:tasks.rados.rados.0.smithi136.stdout:798: finishing rollback tid 0 to smithi13667081-34 2024-09-12T23:40:42.661 INFO:tasks.rados.rados.0.smithi136.stdout:801: finishing rollback tid 1 to smithi13667081-4 2024-09-12T23:40:42.661 INFO:tasks.rados.rados.0.smithi136.stdout:798: finishing rollback tid 1 to smithi13667081-34 2024-09-12T23:40:42.661 INFO:tasks.rados.rados.0.smithi136.stdout:801: finishing rollback tid 2 to smithi13667081-4 2024-09-12T23:40:42.661 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 4 v 150 (ObjNum 220 snap 55 seq_num 220) dirty exists 2024-09-12T23:40:42.661 INFO:tasks.rados.rados.0.smithi136.stdout:798: finishing rollback tid 2 to smithi13667081-34 2024-09-12T23:40:42.661 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 34 v 347 (ObjNum 1265 snap 0 seq_num 4242749600) dirty exists 2024-09-12T23:40:42.661 INFO:tasks.rados.rados.0.smithi136.stdout:799: finishing copy_from racing read to smithi13667081-29 2024-09-12T23:40:42.661 INFO:tasks.rados.rados.0.smithi136.stdout:799: finishing copy_from to smithi13667081-29 2024-09-12T23:40:42.661 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 29 v 99 (ObjNum 253 snap 69 seq_num 253) dirty exists 2024-09-12T23:40:42.755 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:42 smithi136 bash[32189]: cluster 2024-09-12T23:40:41.495386+0000 mon.a (mon.0) 1313 : cluster [DBG] osdmap e206: 8 total, 8 up, 8 in 2024-09-12T23:40:42.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:42 smithi136 bash[22601]: cluster 2024-09-12T23:40:41.495386+0000 mon.a (mon.0) 1313 : cluster [DBG] osdmap e206: 8 total, 8 up, 8 in 2024-09-12T23:40:42.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:42 smithi181 bash[24776]: cluster 2024-09-12T23:40:41.495386+0000 mon.a (mon.0) 1313 : cluster [DBG] osdmap e206: 8 total, 8 up, 8 in 2024-09-12T23:40:43.515 INFO:tasks.rados.rados.0.smithi136.stdout:797: done (5 left) 2024-09-12T23:40:43.515 INFO:tasks.rados.rados.0.smithi136.stdout:798: done (4 left) 2024-09-12T23:40:43.515 INFO:tasks.rados.rados.0.smithi136.stdout:799: done (3 left) 2024-09-12T23:40:43.515 INFO:tasks.rados.rados.0.smithi136.stdout:800: done (2 left) 2024-09-12T23:40:43.515 INFO:tasks.rados.rados.0.smithi136.stdout:801: done (1 left) 2024-09-12T23:40:43.515 INFO:tasks.rados.rados.0.smithi136.stdout:802: done (0 left) 2024-09-12T23:40:43.515 INFO:tasks.rados.rados.0.smithi136.stdout:803: copy_from oid 36 from oid 21 current snap is 73 2024-09-12T23:40:43.515 INFO:tasks.rados.rados.0.smithi136.stdout:804: read oid 12 snap -1 2024-09-12T23:40:43.516 INFO:tasks.rados.rados.0.smithi136.stdout:804: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-12T23:40:43.516 INFO:tasks.rados.rados.0.smithi136.stdout:805: rmattr oid 13 current snap is 73 2024-09-12T23:40:43.516 INFO:tasks.rados.rados.0.smithi136.stdout:805: done (2 left) 2024-09-12T23:40:43.516 INFO:tasks.rados.rados.0.smithi136.stdout:806: setattr oid 42 current snap is 73 2024-09-12T23:40:43.520 INFO:tasks.rados.rados.0.smithi136.stdout:804: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-12T23:40:43.523 INFO:tasks.rados.rados.0.smithi136.stdout:804: done (2 left) 2024-09-12T23:40:43.523 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 20 current snap is 73 2024-09-12T23:40:43.523 INFO:tasks.rados.rados.0.smithi136.stdout:807: seq_num 268 ranges {0=491520,491520=745472,1236992=745472,1982464=114688} 2024-09-12T23:40:43.531 INFO:tasks.rados.rados.0.smithi136.stdout:807: writing smithi13667081-20 from 0 to 491520 tid 1 2024-09-12T23:40:43.544 INFO:tasks.rados.rados.0.smithi136.stdout:807: writing smithi13667081-20 from 491520 to 1236992 tid 2 2024-09-12T23:40:43.557 INFO:tasks.rados.rados.0.smithi136.stdout:807: writing smithi13667081-20 from 1236992 to 1982464 tid 3 2024-09-12T23:40:43.560 INFO:tasks.rados.rados.0.smithi136.stdout:807: writing smithi13667081-20 from 1982464 to 2097152 tid 4 2024-09-12T23:40:43.560 INFO:tasks.rados.rados.0.smithi136.stdout:808: rollback oid 43 current snap is 73 2024-09-12T23:40:43.560 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 43 to 61 2024-09-12T23:40:43.560 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 42 v 178 (ObjNum 125 snap 32 seq_num 125) dirty exists 2024-09-12T23:40:43.560 INFO:tasks.rados.rados.0.smithi136.stdout:807: finishing write tid 1 to smithi13667081-20 2024-09-12T23:40:43.560 INFO:tasks.rados.rados.0.smithi136.stdout:803: finishing copy_from racing read to smithi13667081-36 2024-09-12T23:40:43.560 INFO:tasks.rados.rados.0.smithi136.stdout:803: finishing copy_from to smithi13667081-36 2024-09-12T23:40:43.560 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 36 v 166 (ObjNum 256 snap 70 seq_num 256) dirty exists 2024-09-12T23:40:43.561 INFO:tasks.rados.rados.0.smithi136.stdout:807: finishing write tid 2 to smithi13667081-20 2024-09-12T23:40:43.561 INFO:tasks.rados.rados.0.smithi136.stdout:803: done (3 left) 2024-09-12T23:40:43.561 INFO:tasks.rados.rados.0.smithi136.stdout:806: done (2 left) 2024-09-12T23:40:43.561 INFO:tasks.rados.rados.0.smithi136.stdout:809: snap_remove snap 64 2024-09-12T23:40:43.755 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:40:43 smithi136 bash[68390]: ts=2024-09-12T23:40:43.586Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003161862s 2024-09-12T23:40:43.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:43 smithi136 bash[22601]: cluster 2024-09-12T23:40:41.568905+0000 mgr.x (mgr.24517) 47 : cluster [DBG] pgmap v54: 121 pgs: 3 active+clean+snaptrim, 118 active+clean; 106 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 5.1 MiB/s wr, 23 op/s 2024-09-12T23:40:43.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:43 smithi136 bash[22601]: cluster 2024-09-12T23:40:42.508699+0000 mon.a (mon.0) 1314 : cluster [DBG] osdmap e207: 8 total, 8 up, 8 in 2024-09-12T23:40:43.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:43 smithi136 bash[32189]: cluster 2024-09-12T23:40:41.568905+0000 mgr.x (mgr.24517) 47 : cluster [DBG] pgmap v54: 121 pgs: 3 active+clean+snaptrim, 118 active+clean; 106 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 5.1 MiB/s wr, 23 op/s 2024-09-12T23:40:43.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:43 smithi136 bash[32189]: cluster 2024-09-12T23:40:42.508699+0000 mon.a (mon.0) 1314 : cluster [DBG] osdmap e207: 8 total, 8 up, 8 in 2024-09-12T23:40:43.762 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:43 smithi181 bash[24776]: cluster 2024-09-12T23:40:41.568905+0000 mgr.x (mgr.24517) 47 : cluster [DBG] pgmap v54: 121 pgs: 3 active+clean+snaptrim, 118 active+clean; 106 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 5.1 MiB/s wr, 23 op/s 2024-09-12T23:40:43.762 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:43 smithi181 bash[24776]: cluster 2024-09-12T23:40:42.508699+0000 mon.a (mon.0) 1314 : cluster [DBG] osdmap e207: 8 total, 8 up, 8 in 2024-09-12T23:40:44.504 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:40:44 smithi181 systemd[1]: Stopping Ceph grafana.a for da58a47c-715d-11ef-bceb-c7b262605968... 2024-09-12T23:40:44.525 INFO:tasks.rados.rados.0.smithi136.stdout:809: done (2 left) 2024-09-12T23:40:44.525 INFO:tasks.rados.rados.0.smithi136.stdout:810: copy_from oid 44 from oid 39 current snap is 73 2024-09-12T23:40:44.525 INFO:tasks.rados.rados.0.smithi136.stdout:807: finishing write tid 3 to smithi13667081-20 2024-09-12T23:40:44.525 INFO:tasks.rados.rados.0.smithi136.stdout:808: finishing rollback tid 0 to smithi13667081-43 2024-09-12T23:40:44.525 INFO:tasks.rados.rados.0.smithi136.stdout:808: finishing rollback tid 1 to smithi13667081-43 2024-09-12T23:40:44.526 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 43 v 427 (ObjNum 33 snap 0 seq_num 4243420592) dirty dne 2024-09-12T23:40:44.526 INFO:tasks.rados.rados.0.smithi136.stdout:807: finishing write tid 4 to smithi13667081-20 2024-09-12T23:40:44.526 INFO:tasks.rados.rados.0.smithi136.stdout:807: finishing write tid 5 to smithi13667081-20 2024-09-12T23:40:44.526 INFO:tasks.rados.rados.0.smithi136.stdout:807: finishing write tid 6 to smithi13667081-20 2024-09-12T23:40:44.526 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 20 v 279 (ObjNum 268 snap 73 seq_num 268) dirty exists 2024-09-12T23:40:44.526 INFO:tasks.rados.rados.0.smithi136.stdout:807: left oid 20 (ObjNum 268 snap 73 seq_num 268) 2024-09-12T23:40:44.526 INFO:tasks.rados.rados.0.smithi136.stdout:807: done (2 left) 2024-09-12T23:40:44.526 INFO:tasks.rados.rados.0.smithi136.stdout:808: done (1 left) 2024-09-12T23:40:44.526 INFO:tasks.rados.rados.0.smithi136.stdout:811: copy_from oid 46 from oid 16 current snap is 73 2024-09-12T23:40:44.526 INFO:tasks.rados.rados.0.smithi136.stdout:812: delete oid 28 current snap is 73 2024-09-12T23:40:44.549 INFO:tasks.rados.rados.0.smithi136.stdout:812: done (2 left) 2024-09-12T23:40:44.550 INFO:tasks.rados.rados.0.smithi136.stdout:813: snap_remove snap 72 2024-09-12T23:40:44.763 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:44 smithi181 bash[24776]: cluster 2024-09-12T23:40:43.519001+0000 mon.a (mon.0) 1315 : cluster [DBG] osdmap e208: 8 total, 8 up, 8 in 2024-09-12T23:40:45.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:44 smithi136 bash[22601]: cluster 2024-09-12T23:40:43.519001+0000 mon.a (mon.0) 1315 : cluster [DBG] osdmap e208: 8 total, 8 up, 8 in 2024-09-12T23:40:45.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:44 smithi136 bash[32189]: cluster 2024-09-12T23:40:43.519001+0000 mon.a (mon.0) 1315 : cluster [DBG] osdmap e208: 8 total, 8 up, 8 in 2024-09-12T23:40:45.058 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:40:44 smithi181 bash[59113]: ceph-da58a47c-715d-11ef-bceb-c7b262605968-grafana-a 2024-09-12T23:40:45.059 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:40:44 smithi181 systemd[1]: ceph-da58a47c-715d-11ef-bceb-c7b262605968@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-09-12T23:40:45.059 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:40:44 smithi181 bash[59173]: Error response from daemon: No such container: ceph-da58a47c-715d-11ef-bceb-c7b262605968-grafana-a 2024-09-12T23:40:45.059 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:40:44 smithi181 systemd[1]: ceph-da58a47c-715d-11ef-bceb-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-09-12T23:40:45.059 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:40:44 smithi181 systemd[1]: Stopped Ceph grafana.a for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:40:45.059 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:40:44 smithi181 systemd[1]: Started Ceph grafana.a for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:40:45.541 INFO:tasks.rados.rados.0.smithi136.stdout:813: done (2 left) 2024-09-12T23:40:45.541 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 11 current snap is 73 2024-09-12T23:40:45.541 INFO:tasks.rados.rados.0.smithi136.stdout:811: finishing copy_from racing read to smithi13667081-46 2024-09-12T23:40:45.541 INFO:tasks.rados.rados.0.smithi136.stdout:811: finishing copy_from to smithi13667081-46 2024-09-12T23:40:45.542 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 46 v 371 (ObjNum 228 snap 56 seq_num 228) dirty exists 2024-09-12T23:40:45.542 INFO:tasks.rados.rados.0.smithi136.stdout:810: finishing copy_from racing read to smithi13667081-44 2024-09-12T23:40:45.542 INFO:tasks.rados.rados.0.smithi136.stdout:810: finishing copy_from to smithi13667081-44 2024-09-12T23:40:45.542 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 44 v 90 (ObjNum 225 snap 56 seq_num 225) dirty exists 2024-09-12T23:40:45.542 INFO:tasks.rados.rados.0.smithi136.stdout:814: seq_num 270 ranges {0=524288,524288=516096,1040384=761856,1802240=327680} 2024-09-12T23:40:45.549 INFO:tasks.rados.rados.0.smithi136.stdout:814: writing smithi13667081-11 from 0 to 524288 tid 1 2024-09-12T23:40:45.559 INFO:tasks.rados.rados.0.smithi136.stdout:814: writing smithi13667081-11 from 524288 to 1040384 tid 2 2024-09-12T23:40:45.574 INFO:tasks.rados.rados.0.smithi136.stdout:814: writing smithi13667081-11 from 1040384 to 1802240 tid 3 2024-09-12T23:40:45.579 INFO:tasks.rados.rados.0.smithi136.stdout:814: writing smithi13667081-11 from 1802240 to 2129920 tid 4 2024-09-12T23:40:45.579 INFO:tasks.rados.rados.0.smithi136.stdout:810: done (2 left) 2024-09-12T23:40:45.580 INFO:tasks.rados.rados.0.smithi136.stdout:811: done (1 left) 2024-09-12T23:40:45.580 INFO:tasks.rados.rados.0.smithi136.stdout:815: delete oid 3 current snap is 73 2024-09-12T23:40:45.580 INFO:tasks.rados.rados.0.smithi136.stdout:814: finishing write tid 1 to smithi13667081-11 2024-09-12T23:40:45.580 INFO:tasks.rados.rados.0.smithi136.stdout:814: finishing write tid 2 to smithi13667081-11 2024-09-12T23:40:45.585 INFO:tasks.rados.rados.0.smithi136.stdout:814: finishing write tid 3 to smithi13667081-11 2024-09-12T23:40:45.588 INFO:tasks.rados.rados.0.smithi136.stdout:814: finishing write tid 4 to smithi13667081-11 2024-09-12T23:40:45.588 INFO:tasks.rados.rados.0.smithi136.stdout:814: finishing write tid 5 to smithi13667081-11 2024-09-12T23:40:45.590 INFO:tasks.rados.rados.0.smithi136.stdout:815: done (1 left) 2024-09-12T23:40:45.591 INFO:tasks.rados.rados.0.smithi136.stdout:816: snap_remove snap 66 2024-09-12T23:40:45.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:45 smithi136 bash[22601]: cluster 2024-09-12T23:40:43.569351+0000 mgr.x (mgr.24517) 48 : cluster [DBG] pgmap v57: 121 pgs: 3 active+clean+snaptrim, 118 active+clean; 106 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:40:45.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:45 smithi136 bash[22601]: cluster 2024-09-12T23:40:44.529727+0000 mon.a (mon.0) 1316 : cluster [DBG] osdmap e209: 8 total, 8 up, 8 in 2024-09-12T23:40:45.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:45 smithi136 bash[22601]: audit 2024-09-12T23:40:44.995794+0000 mon.a (mon.0) 1317 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:45.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:45 smithi136 bash[22601]: audit 2024-09-12T23:40:45.007485+0000 mon.a (mon.0) 1318 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:45.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:45 smithi136 bash[22601]: audit 2024-09-12T23:40:45.012921+0000 mon.b (mon.1) 96 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-12T23:40:45.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:45 smithi136 bash[22601]: audit 2024-09-12T23:40:45.016797+0000 mon.b (mon.1) 97 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-12T23:40:45.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:45 smithi136 bash[22601]: audit 2024-09-12T23:40:45.020188+0000 mon.b (mon.1) 98 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-12T23:40:45.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:45 smithi136 bash[22601]: audit 2024-09-12T23:40:45.061052+0000 mon.b (mon.1) 99 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:40:45.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:45 smithi136 bash[32189]: cluster 2024-09-12T23:40:43.569351+0000 mgr.x (mgr.24517) 48 : cluster [DBG] pgmap v57: 121 pgs: 3 active+clean+snaptrim, 118 active+clean; 106 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:40:45.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:45 smithi136 bash[32189]: cluster 2024-09-12T23:40:44.529727+0000 mon.a (mon.0) 1316 : cluster [DBG] osdmap e209: 8 total, 8 up, 8 in 2024-09-12T23:40:45.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:45 smithi136 bash[32189]: audit 2024-09-12T23:40:44.995794+0000 mon.a (mon.0) 1317 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:45.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:45 smithi136 bash[32189]: audit 2024-09-12T23:40:45.007485+0000 mon.a (mon.0) 1318 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:45.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:45 smithi136 bash[32189]: audit 2024-09-12T23:40:45.012921+0000 mon.b (mon.1) 96 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-12T23:40:45.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:45 smithi136 bash[32189]: audit 2024-09-12T23:40:45.016797+0000 mon.b (mon.1) 97 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-12T23:40:45.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:45 smithi136 bash[32189]: audit 2024-09-12T23:40:45.020188+0000 mon.b (mon.1) 98 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-12T23:40:45.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:45 smithi136 bash[32189]: audit 2024-09-12T23:40:45.061052+0000 mon.b (mon.1) 99 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:40:45.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:45 smithi181 bash[24776]: cluster 2024-09-12T23:40:43.569351+0000 mgr.x (mgr.24517) 48 : cluster [DBG] pgmap v57: 121 pgs: 3 active+clean+snaptrim, 118 active+clean; 106 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:40:45.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:45 smithi181 bash[24776]: cluster 2024-09-12T23:40:44.529727+0000 mon.a (mon.0) 1316 : cluster [DBG] osdmap e209: 8 total, 8 up, 8 in 2024-09-12T23:40:45.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:45 smithi181 bash[24776]: audit 2024-09-12T23:40:44.995794+0000 mon.a (mon.0) 1317 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:45.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:45 smithi181 bash[24776]: audit 2024-09-12T23:40:45.007485+0000 mon.a (mon.0) 1318 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:45.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:45 smithi181 bash[24776]: audit 2024-09-12T23:40:45.012921+0000 mon.b (mon.1) 96 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-12T23:40:45.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:45 smithi181 bash[24776]: audit 2024-09-12T23:40:45.016797+0000 mon.b (mon.1) 97 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-12T23:40:45.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:45 smithi181 bash[24776]: audit 2024-09-12T23:40:45.020188+0000 mon.b (mon.1) 98 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-12T23:40:45.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:45 smithi181 bash[24776]: audit 2024-09-12T23:40:45.061052+0000 mon.b (mon.1) 99 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:40:46.544 INFO:tasks.rados.rados.0.smithi136.stdout:816: done (1 left) 2024-09-12T23:40:46.544 INFO:tasks.rados.rados.0.smithi136.stdout:817: read oid 9 snap -1 2024-09-12T23:40:46.544 INFO:tasks.rados.rados.0.smithi136.stdout:817: expect (ObjNum 6 snap 0 seq_num 6) 2024-09-12T23:40:46.544 INFO:tasks.rados.rados.0.smithi136.stdout:818: snap_remove snap 65 2024-09-12T23:40:46.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:46 smithi181 bash[24776]: audit 2024-09-12T23:40:45.013377+0000 mgr.x (mgr.24517) 49 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-12T23:40:46.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:46 smithi181 bash[24776]: audit 2024-09-12T23:40:45.017191+0000 mgr.x (mgr.24517) 50 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-12T23:40:46.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:46 smithi181 bash[24776]: audit 2024-09-12T23:40:45.020528+0000 mgr.x (mgr.24517) 51 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-12T23:40:46.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:46 smithi181 bash[24776]: cephadm 2024-09-12T23:40:45.065644+0000 mgr.x (mgr.24517) 52 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2024-09-12T23:40:46.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:46 smithi181 bash[24776]: cephadm 2024-09-12T23:40:45.066159+0000 mgr.x (mgr.24517) 53 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2024-09-12T23:40:46.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:46 smithi181 bash[24776]: cluster 2024-09-12T23:40:45.544411+0000 mon.a (mon.0) 1319 : cluster [DBG] osdmap e210: 8 total, 8 up, 8 in 2024-09-12T23:40:46.844 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:40:46 smithi181 bash[59202]: logger=settings t=2024-09-12T23:40:46.375891336Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-09-12T23:40:46.845 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:40:46 smithi181 bash[59202]: logger=settings t=2024-09-12T23:40:46.376108776Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-09-12T23:40:46.845 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:40:46 smithi181 bash[59202]: logger=settings t=2024-09-12T23:40:46.376132525Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-09-12T23:40:46.845 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:40:46 smithi181 bash[59202]: logger=settings t=2024-09-12T23:40:46.376151574Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-09-12T23:40:46.845 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:40:46 smithi181 bash[59202]: logger=settings t=2024-09-12T23:40:46.376162367Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-09-12T23:40:46.845 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:40:46 smithi181 bash[59202]: logger=settings t=2024-09-12T23:40:46.376170913Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-09-12T23:40:46.845 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:40:46 smithi181 bash[59202]: logger=settings t=2024-09-12T23:40:46.376182525Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-09-12T23:40:46.845 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:40:46 smithi181 bash[59202]: logger=settings t=2024-09-12T23:40:46.376191753Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-09-12T23:40:46.845 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:40:46 smithi181 bash[59202]: logger=settings t=2024-09-12T23:40:46.376198796Z level=info msg="Path Home" path=/usr/share/grafana 2024-09-12T23:40:46.845 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:40:46 smithi181 bash[59202]: logger=settings t=2024-09-12T23:40:46.376205965Z level=info msg="Path Data" path=/var/lib/grafana 2024-09-12T23:40:46.845 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:40:46 smithi181 bash[59202]: logger=settings t=2024-09-12T23:40:46.37621261Z level=info msg="Path Logs" path=/var/log/grafana 2024-09-12T23:40:46.845 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:40:46 smithi181 bash[59202]: logger=settings t=2024-09-12T23:40:46.376219713Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-09-12T23:40:46.845 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:40:46 smithi181 bash[59202]: logger=settings t=2024-09-12T23:40:46.376227315Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-09-12T23:40:46.845 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:40:46 smithi181 bash[59202]: logger=settings t=2024-09-12T23:40:46.376237223Z level=info msg="App mode production" 2024-09-12T23:40:46.846 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:40:46 smithi181 bash[59202]: logger=sqlstore t=2024-09-12T23:40:46.376320416Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-09-12T23:40:46.846 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:40:46 smithi181 bash[59202]: logger=sqlstore t=2024-09-12T23:40:46.376350604Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-09-12T23:40:46.846 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:40:46 smithi181 bash[59202]: logger=migrator t=2024-09-12T23:40:46.393119515Z level=info msg="Starting DB migrations" 2024-09-12T23:40:46.846 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:40:46 smithi181 bash[59202]: logger=migrator t=2024-09-12T23:40:46.398750462Z level=info msg="migrations completed" performed=0 skipped=481 duration=530.145µs 2024-09-12T23:40:46.846 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:40:46 smithi181 bash[59202]: logger=sqlstore t=2024-09-12T23:40:46.399590507Z level=info msg="Created default organization" 2024-09-12T23:40:46.846 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:40:46 smithi181 bash[59202]: logger=plugin.loader t=2024-09-12T23:40:46.516608653Z level=info msg="Plugin registered" pluginID=input 2024-09-12T23:40:46.846 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:40:46 smithi181 bash[59202]: logger=plugin.loader t=2024-09-12T23:40:46.553494875Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-09-12T23:40:46.846 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:40:46 smithi181 bash[59202]: logger=plugin.loader t=2024-09-12T23:40:46.553550107Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-09-12T23:40:46.846 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:40:46 smithi181 bash[59202]: logger=secrets t=2024-09-12T23:40:46.55364325Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-09-12T23:40:46.846 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:40:46 smithi181 bash[59202]: logger=query_data t=2024-09-12T23:40:46.558579931Z level=info msg="Query Service initialization" 2024-09-12T23:40:46.846 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:40:46 smithi181 bash[59202]: logger=live.push_http t=2024-09-12T23:40:46.563459009Z level=info msg="Live Push Gateway initialization" 2024-09-12T23:40:46.866 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:46 smithi136 bash[32189]: audit 2024-09-12T23:40:45.013377+0000 mgr.x (mgr.24517) 49 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-12T23:40:46.866 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:46 smithi136 bash[32189]: audit 2024-09-12T23:40:45.017191+0000 mgr.x (mgr.24517) 50 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-12T23:40:46.866 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:46 smithi136 bash[32189]: audit 2024-09-12T23:40:45.020528+0000 mgr.x (mgr.24517) 51 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-12T23:40:46.866 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:46 smithi136 bash[32189]: cephadm 2024-09-12T23:40:45.065644+0000 mgr.x (mgr.24517) 52 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2024-09-12T23:40:46.867 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:46 smithi136 bash[32189]: cephadm 2024-09-12T23:40:45.066159+0000 mgr.x (mgr.24517) 53 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2024-09-12T23:40:46.867 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:46 smithi136 bash[32189]: cluster 2024-09-12T23:40:45.544411+0000 mon.a (mon.0) 1319 : cluster [DBG] osdmap e210: 8 total, 8 up, 8 in 2024-09-12T23:40:46.867 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:46 smithi136 bash[22601]: audit 2024-09-12T23:40:45.013377+0000 mgr.x (mgr.24517) 49 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-12T23:40:46.867 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:46 smithi136 bash[22601]: audit 2024-09-12T23:40:45.017191+0000 mgr.x (mgr.24517) 50 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-12T23:40:46.867 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:46 smithi136 bash[22601]: audit 2024-09-12T23:40:45.020528+0000 mgr.x (mgr.24517) 51 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-12T23:40:46.867 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:46 smithi136 bash[22601]: cephadm 2024-09-12T23:40:45.065644+0000 mgr.x (mgr.24517) 52 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2024-09-12T23:40:46.867 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:46 smithi136 bash[22601]: cephadm 2024-09-12T23:40:45.066159+0000 mgr.x (mgr.24517) 53 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2024-09-12T23:40:46.867 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:46 smithi136 bash[22601]: cluster 2024-09-12T23:40:45.544411+0000 mon.a (mon.0) 1319 : cluster [DBG] osdmap e210: 8 total, 8 up, 8 in 2024-09-12T23:40:47.555 INFO:tasks.rados.rados.0.smithi136.stdout:818: done (2 left) 2024-09-12T23:40:47.555 INFO:tasks.rados.rados.0.smithi136.stdout:819: read oid 47 snap -1 2024-09-12T23:40:47.555 INFO:tasks.rados.rados.0.smithi136.stdout:819: expect (ObjNum 231 snap 56 seq_num 231) 2024-09-12T23:40:47.555 INFO:tasks.rados.rados.0.smithi136.stdout:820: snap_remove snap 70 2024-09-12T23:40:47.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:47 smithi181 bash[24776]: cluster 2024-09-12T23:40:45.570002+0000 mgr.x (mgr.24517) 54 : cluster [DBG] pgmap v60: 121 pgs: 121 active+clean; 116 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 4.0 MiB/s wr, 21 op/s 2024-09-12T23:40:47.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:47 smithi181 bash[24776]: cluster 2024-09-12T23:40:46.548330+0000 mon.a (mon.0) 1320 : cluster [DBG] osdmap e211: 8 total, 8 up, 8 in 2024-09-12T23:40:47.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:47 smithi181 bash[24776]: audit 2024-09-12T23:40:47.509652+0000 mon.a (mon.0) 1321 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:47.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:47 smithi181 bash[24776]: audit 2024-09-12T23:40:47.512428+0000 mon.b (mon.1) 100 : audit [INF] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-12T23:40:47.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:47 smithi181 bash[24776]: audit 2024-09-12T23:40:47.512497+0000 mon.a (mon.0) 1322 : audit [INF] from='mgr.24517 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-12T23:40:47.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:47 smithi181 bash[24776]: audit 2024-09-12T23:40:47.514970+0000 mon.b (mon.1) 101 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-12T23:40:47.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:47 smithi181 bash[24776]: audit 2024-09-12T23:40:47.516414+0000 mon.b (mon.1) 102 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:40:47.879 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:47 smithi136 bash[32189]: cluster 2024-09-12T23:40:45.570002+0000 mgr.x (mgr.24517) 54 : cluster [DBG] pgmap v60: 121 pgs: 121 active+clean; 116 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 4.0 MiB/s wr, 21 op/s 2024-09-12T23:40:47.879 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:47 smithi136 bash[32189]: cluster 2024-09-12T23:40:46.548330+0000 mon.a (mon.0) 1320 : cluster [DBG] osdmap e211: 8 total, 8 up, 8 in 2024-09-12T23:40:47.879 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:47 smithi136 bash[32189]: audit 2024-09-12T23:40:47.509652+0000 mon.a (mon.0) 1321 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:47.879 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:47 smithi136 bash[32189]: audit 2024-09-12T23:40:47.512428+0000 mon.b (mon.1) 100 : audit [INF] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-12T23:40:47.879 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:47 smithi136 bash[32189]: audit 2024-09-12T23:40:47.512497+0000 mon.a (mon.0) 1322 : audit [INF] from='mgr.24517 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-12T23:40:47.879 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:47 smithi136 bash[32189]: audit 2024-09-12T23:40:47.514970+0000 mon.b (mon.1) 101 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-12T23:40:47.879 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:47 smithi136 bash[32189]: audit 2024-09-12T23:40:47.516414+0000 mon.b (mon.1) 102 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:40:47.880 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:47 smithi136 bash[22601]: cluster 2024-09-12T23:40:45.570002+0000 mgr.x (mgr.24517) 54 : cluster [DBG] pgmap v60: 121 pgs: 121 active+clean; 116 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 4.0 MiB/s wr, 21 op/s 2024-09-12T23:40:47.880 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:47 smithi136 bash[22601]: cluster 2024-09-12T23:40:46.548330+0000 mon.a (mon.0) 1320 : cluster [DBG] osdmap e211: 8 total, 8 up, 8 in 2024-09-12T23:40:47.880 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:47 smithi136 bash[22601]: audit 2024-09-12T23:40:47.509652+0000 mon.a (mon.0) 1321 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:47.880 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:47 smithi136 bash[22601]: audit 2024-09-12T23:40:47.512428+0000 mon.b (mon.1) 100 : audit [INF] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-12T23:40:47.880 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:47 smithi136 bash[22601]: audit 2024-09-12T23:40:47.512497+0000 mon.a (mon.0) 1322 : audit [INF] from='mgr.24517 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-12T23:40:47.880 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:47 smithi136 bash[22601]: audit 2024-09-12T23:40:47.514970+0000 mon.b (mon.1) 101 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-12T23:40:47.880 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:47 smithi136 bash[22601]: audit 2024-09-12T23:40:47.516414+0000 mon.b (mon.1) 102 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:40:48.093 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:40:47 smithi181 bash[59202]: logger=infra.usagestats.collector t=2024-09-12T23:40:47.823566878Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-09-12T23:40:48.094 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:40:47 smithi181 bash[59202]: logger=provisioning.datasources t=2024-09-12T23:40:47.943987794Z level=info msg="deleted correlations based on configuration" ds_name=Dashboard1 2024-09-12T23:40:48.094 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:40:47 smithi181 bash[59202]: logger=provisioning.datasources t=2024-09-12T23:40:47.944074554Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2024-09-12T23:40:48.094 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:40:47 smithi181 bash[59202]: logger=provisioning.datasources t=2024-09-12T23:40:47.955532999Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-09-12T23:40:48.526 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:40:48 smithi181 bash[59202]: logger=provisioning.alerting t=2024-09-12T23:40:48.135099755Z level=info msg="starting to provision alerting" 2024-09-12T23:40:48.526 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:40:48 smithi181 bash[59202]: logger=provisioning.alerting t=2024-09-12T23:40:48.135166501Z level=info msg="finished to provision alerting" 2024-09-12T23:40:48.526 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:40:48 smithi181 bash[59202]: logger=grafanaStorageLogger t=2024-09-12T23:40:48.135729506Z level=info msg="storage starting" 2024-09-12T23:40:48.527 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:40:48 smithi181 bash[59202]: logger=ngalert.state.manager t=2024-09-12T23:40:48.135945002Z level=info msg="Warming state cache for startup" 2024-09-12T23:40:48.527 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:40:48 smithi181 bash[59202]: logger=http.server t=2024-09-12T23:40:48.138762406Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-09-12T23:40:48.527 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:40:48 smithi181 bash[59202]: logger=ngalert.state.manager t=2024-09-12T23:40:48.184544328Z level=info msg="State cache has been initialized" states=0 duration=48.598396ms 2024-09-12T23:40:48.527 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:40:48 smithi181 bash[59202]: logger=ngalert.multiorg.alertmanager t=2024-09-12T23:40:48.184604197Z level=info msg="starting MultiOrg Alertmanager" 2024-09-12T23:40:48.527 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:40:48 smithi181 bash[59202]: logger=ticker t=2024-09-12T23:40:48.184657151Z level=info msg=starting first_tick=2024-09-12T23:40:50Z 2024-09-12T23:40:48.556 INFO:tasks.rados.rados.0.smithi136.stdout:820: done (3 left) 2024-09-12T23:40:48.556 INFO:tasks.rados.rados.0.smithi136.stdout:821: rollback oid 20 current snap is 73 2024-09-12T23:40:48.556 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 20 to 59 2024-09-12T23:40:48.556 INFO:tasks.rados.rados.0.smithi136.stdout:814: finishing write tid 6 to smithi13667081-11 2024-09-12T23:40:48.556 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 11 v 453 (ObjNum 270 snap 73 seq_num 270) dirty exists 2024-09-12T23:40:48.556 INFO:tasks.rados.rados.0.smithi136.stdout:814: left oid 11 (ObjNum 270 snap 73 seq_num 270) 2024-09-12T23:40:48.557 INFO:tasks.rados.rados.0.smithi136.stdout:817: expect (ObjNum 6 snap 0 seq_num 6) 2024-09-12T23:40:48.579 INFO:tasks.rados.rados.0.smithi136.stdout:814: done (3 left) 2024-09-12T23:40:48.579 INFO:tasks.rados.rados.0.smithi136.stdout:817: done (2 left) 2024-09-12T23:40:48.579 INFO:tasks.rados.rados.0.smithi136.stdout:822: read oid 19 snap -1 2024-09-12T23:40:48.579 INFO:tasks.rados.rados.0.smithi136.stdout:822: expect (ObjNum 264 snap 71 seq_num 264) 2024-09-12T23:40:48.579 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 28 current snap is 73 2024-09-12T23:40:48.579 INFO:tasks.rados.rados.0.smithi136.stdout:823: seq_num 272 ranges {0=565248,565248=688128,1253376=729088,1982464=188416} 2024-09-12T23:40:48.580 INFO:tasks.rados.rados.0.smithi136.stdout:823: writing smithi13667081-28 from 0 to 565248 tid 1 2024-09-12T23:40:48.590 INFO:tasks.rados.rados.0.smithi136.stdout:823: writing smithi13667081-28 from 565248 to 1253376 tid 2 2024-09-12T23:40:48.599 INFO:tasks.rados.rados.0.smithi136.stdout:823: writing smithi13667081-28 from 1253376 to 1982464 tid 3 2024-09-12T23:40:48.601 INFO:tasks.rados.rados.0.smithi136.stdout:823: writing smithi13667081-28 from 1982464 to 2170880 tid 4 2024-09-12T23:40:48.601 INFO:tasks.rados.rados.0.smithi136.stdout:824: setattr oid 14 current snap is 73 2024-09-12T23:40:48.601 INFO:tasks.rados.rados.0.smithi136.stdout:819: expect (ObjNum 231 snap 56 seq_num 231) 2024-09-12T23:40:48.661 INFO:tasks.rados.rados.0.smithi136.stdout:821: finishing rollback tid 0 to smithi13667081-20 2024-09-12T23:40:48.661 INFO:tasks.rados.rados.0.smithi136.stdout:821: finishing rollback tid 1 to smithi13667081-20 2024-09-12T23:40:48.661 INFO:tasks.rados.rados.0.smithi136.stdout:821: finishing rollback tid 2 to smithi13667081-20 2024-09-12T23:40:48.661 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 20 v 282 (ObjNum 177 snap 45 seq_num 177) dirty exists 2024-09-12T23:40:48.661 INFO:tasks.rados.rados.0.smithi136.stdout:823: finishing write tid 1 to smithi13667081-28 2024-09-12T23:40:48.661 INFO:tasks.rados.rados.0.smithi136.stdout:823: finishing write tid 2 to smithi13667081-28 2024-09-12T23:40:48.661 INFO:tasks.rados.rados.0.smithi136.stdout:822: expect (ObjNum 264 snap 71 seq_num 264) 2024-09-12T23:40:48.793 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:48 smithi136 bash[22601]: cephadm 2024-09-12T23:40:47.500346+0000 mgr.x (mgr.24517) 55 : cephadm [INF] Upgrade: Updating mgr.y 2024-09-12T23:40:48.793 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:48 smithi136 bash[22601]: cephadm 2024-09-12T23:40:47.517722+0000 mgr.x (mgr.24517) 56 : cephadm [INF] Deploying daemon mgr.y on smithi136 2024-09-12T23:40:48.793 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:48 smithi136 bash[22601]: cluster 2024-09-12T23:40:47.557576+0000 mon.a (mon.0) 1323 : cluster [DBG] osdmap e212: 8 total, 8 up, 8 in 2024-09-12T23:40:48.793 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:48 smithi136 bash[22601]: cluster 2024-09-12T23:40:47.570517+0000 mgr.x (mgr.24517) 57 : cluster [DBG] pgmap v63: 121 pgs: 121 active+clean; 116 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 4.0 MiB/s wr, 21 op/s 2024-09-12T23:40:48.793 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:48 smithi136 bash[32189]: cephadm 2024-09-12T23:40:47.500346+0000 mgr.x (mgr.24517) 55 : cephadm [INF] Upgrade: Updating mgr.y 2024-09-12T23:40:48.793 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:48 smithi136 bash[32189]: cephadm 2024-09-12T23:40:47.517722+0000 mgr.x (mgr.24517) 56 : cephadm [INF] Deploying daemon mgr.y on smithi136 2024-09-12T23:40:48.794 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:48 smithi136 bash[32189]: cluster 2024-09-12T23:40:47.557576+0000 mon.a (mon.0) 1323 : cluster [DBG] osdmap e212: 8 total, 8 up, 8 in 2024-09-12T23:40:48.794 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:48 smithi136 bash[32189]: cluster 2024-09-12T23:40:47.570517+0000 mgr.x (mgr.24517) 57 : cluster [DBG] pgmap v63: 121 pgs: 121 active+clean; 116 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 4.0 MiB/s wr, 21 op/s 2024-09-12T23:40:48.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:48 smithi181 bash[24776]: cephadm 2024-09-12T23:40:47.500346+0000 mgr.x (mgr.24517) 55 : cephadm [INF] Upgrade: Updating mgr.y 2024-09-12T23:40:48.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:48 smithi181 bash[24776]: cephadm 2024-09-12T23:40:47.517722+0000 mgr.x (mgr.24517) 56 : cephadm [INF] Deploying daemon mgr.y on smithi136 2024-09-12T23:40:48.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:48 smithi181 bash[24776]: cluster 2024-09-12T23:40:47.557576+0000 mon.a (mon.0) 1323 : cluster [DBG] osdmap e212: 8 total, 8 up, 8 in 2024-09-12T23:40:48.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:48 smithi181 bash[24776]: cluster 2024-09-12T23:40:47.570517+0000 mgr.x (mgr.24517) 57 : cluster [DBG] pgmap v63: 121 pgs: 121 active+clean; 116 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 4.0 MiB/s wr, 21 op/s 2024-09-12T23:40:48.856 INFO:tasks.rados.rados.0.smithi136.stdout:823: finishing write tid 3 to smithi13667081-28 2024-09-12T23:40:48.856 INFO:tasks.rados.rados.0.smithi136.stdout:823: finishing write tid 4 to smithi13667081-28 2024-09-12T23:40:48.856 INFO:tasks.rados.rados.0.smithi136.stdout:823: finishing write tid 5 to smithi13667081-28 2024-09-12T23:40:48.856 INFO:tasks.rados.rados.0.smithi136.stdout:823: finishing write tid 6 to smithi13667081-28 2024-09-12T23:40:48.856 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 28 v 474 (ObjNum 272 snap 73 seq_num 272) dirty exists 2024-09-12T23:40:48.856 INFO:tasks.rados.rados.0.smithi136.stdout:823: left oid 28 (ObjNum 272 snap 73 seq_num 272) 2024-09-12T23:40:48.856 INFO:tasks.rados.rados.0.smithi136.stdout:819: done (4 left) 2024-09-12T23:40:48.857 INFO:tasks.rados.rados.0.smithi136.stdout:821: done (3 left) 2024-09-12T23:40:48.857 INFO:tasks.rados.rados.0.smithi136.stdout:822: done (2 left) 2024-09-12T23:40:48.857 INFO:tasks.rados.rados.0.smithi136.stdout:823: done (1 left) 2024-09-12T23:40:48.857 INFO:tasks.rados.rados.0.smithi136.stdout:825: rmattr oid 30 current snap is 73 2024-09-12T23:40:48.857 INFO:tasks.rados.rados.0.smithi136.stdout:825: done (1 left) 2024-09-12T23:40:48.857 INFO:tasks.rados.rados.0.smithi136.stdout:826: setattr oid 32 current snap is 73 2024-09-12T23:40:48.858 INFO:tasks.rados.rados.0.smithi136.stdout:827: delete oid 44 current snap is 73 2024-09-12T23:40:48.860 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 14 v 476 (ObjNum 175 snap 45 seq_num 175) dirty exists 2024-09-12T23:40:48.861 INFO:tasks.rados.rados.0.smithi136.stdout:824: done (2 left) 2024-09-12T23:40:48.861 INFO:tasks.rados.rados.0.smithi136.stdout:827: done (1 left) 2024-09-12T23:40:48.861 INFO:tasks.rados.rados.0.smithi136.stdout:828: copy_from oid 27 from oid 10 current snap is 73 2024-09-12T23:40:48.861 INFO:tasks.rados.rados.0.smithi136.stdout:829: read oid 42 snap -1 2024-09-12T23:40:48.861 INFO:tasks.rados.rados.0.smithi136.stdout:829: expect (ObjNum 125 snap 32 seq_num 125) 2024-09-12T23:40:48.861 INFO:tasks.rados.rados.0.smithi136.stdout:830: snap_create 2024-09-12T23:40:48.863 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 32 v 160 (ObjNum 244 snap 60 seq_num 244) dirty exists 2024-09-12T23:40:48.867 INFO:tasks.rados.rados.0.smithi136.stdout:829: expect (ObjNum 125 snap 32 seq_num 125) 2024-09-12T23:40:48.926 INFO:tasks.rados.rados.0.smithi136.stdout:828: finishing copy_from racing read to smithi13667081-27 2024-09-12T23:40:48.926 INFO:tasks.rados.rados.0.smithi136.stdout:828: finishing copy_from to smithi13667081-27 2024-09-12T23:40:48.927 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 27 v 203 (ObjNum 248 snap 67 seq_num 248) dirty exists 2024-09-12T23:40:49.343 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:40:48 smithi181 bash[52367]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:40:48] "GET /metrics HTTP/1.1" 200 36140 "" "Prometheus/2.43.0" 2024-09-12T23:40:49.569 INFO:tasks.rados.rados.0.smithi136.stdout:826: done (3 left) 2024-09-12T23:40:49.569 INFO:tasks.rados.rados.0.smithi136.stdout:828: done (2 left) 2024-09-12T23:40:49.569 INFO:tasks.rados.rados.0.smithi136.stdout:829: done (1 left) 2024-09-12T23:40:49.569 INFO:tasks.rados.rados.0.smithi136.stdout:830: done (0 left) 2024-09-12T23:40:49.569 INFO:tasks.rados.rados.0.smithi136.stdout:831: rmattr oid 35 current snap is 74 2024-09-12T23:40:49.569 INFO:tasks.rados.rados.0.smithi136.stdout:831: done (0 left) 2024-09-12T23:40:49.570 INFO:tasks.rados.rados.0.smithi136.stdout:832: read oid 27 snap -1 2024-09-12T23:40:49.570 INFO:tasks.rados.rados.0.smithi136.stdout:832: expect (ObjNum 248 snap 67 seq_num 248) 2024-09-12T23:40:49.570 INFO:tasks.rados.rados.0.smithi136.stdout:833: setattr oid 18 current snap is 74 2024-09-12T23:40:49.574 INFO:tasks.rados.rados.0.smithi136.stdout:834: rollback oid 25 current snap is 74 2024-09-12T23:40:49.574 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 25 to 59 2024-09-12T23:40:49.574 INFO:tasks.rados.rados.0.smithi136.stdout:835: snap_remove snap 68 2024-09-12T23:40:49.835 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:49 smithi136 bash[22601]: cluster 2024-09-12T23:40:48.560219+0000 mon.a (mon.0) 1324 : cluster [DBG] osdmap e213: 8 total, 8 up, 8 in 2024-09-12T23:40:49.835 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:49 smithi136 bash[32189]: cluster 2024-09-12T23:40:48.560219+0000 mon.a (mon.0) 1324 : cluster [DBG] osdmap e213: 8 total, 8 up, 8 in 2024-09-12T23:40:49.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:49 smithi181 bash[24776]: cluster 2024-09-12T23:40:48.560219+0000 mon.a (mon.0) 1324 : cluster [DBG] osdmap e213: 8 total, 8 up, 8 in 2024-09-12T23:40:50.092 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:49 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:40:50.092 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:49 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:40:50.092 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 12 23:40:49 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:40:50.093 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 12 23:40:49 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:40:50.093 INFO:journalctl@ceph.mds.a.smithi136.stdout:Sep 12 23:40:49 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:40:50.093 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:40:49 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:40:50.093 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 12 23:40:49 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:40:50.094 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:49 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:40:50.094 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:49 smithi136 systemd[1]: Stopping Ceph mgr.y for da58a47c-715d-11ef-bceb-c7b262605968... 2024-09-12T23:40:50.094 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 12 23:40:49 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:40:50.094 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:49 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:40:50.492 INFO:tasks.rados.rados.0.smithi136.stdout:835: done (3 left) 2024-09-12T23:40:50.492 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 40 current snap is 74 2024-09-12T23:40:50.492 INFO:tasks.rados.rados.0.smithi136.stdout:836: seq_num 274 ranges {0=598016,598016=499712,1097728=745472,1843200=360448} 2024-09-12T23:40:50.498 INFO:tasks.rados.rados.0.smithi136.stdout:836: writing smithi13667081-40 from 0 to 598016 tid 1 2024-09-12T23:40:50.509 INFO:tasks.rados.rados.0.smithi136.stdout:836: writing smithi13667081-40 from 598016 to 1097728 tid 2 2024-09-12T23:40:50.523 INFO:tasks.rados.rados.0.smithi136.stdout:836: writing smithi13667081-40 from 1097728 to 1843200 tid 3 2024-09-12T23:40:50.529 INFO:tasks.rados.rados.0.smithi136.stdout:836: writing smithi13667081-40 from 1843200 to 2203648 tid 4 2024-09-12T23:40:50.529 INFO:tasks.rados.rados.0.smithi136.stdout:837: copy_from oid 34 from oid 9 current snap is 74 2024-09-12T23:40:50.530 INFO:tasks.rados.rados.0.smithi136.stdout:832: expect (ObjNum 248 snap 67 seq_num 248) 2024-09-12T23:40:50.606 INFO:tasks.rados.rados.0.smithi136.stdout:834: finishing rollback tid 0 to smithi13667081-25 2024-09-12T23:40:50.606 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 18 v 270 (ObjNum 259 snap 71 seq_num 259) dirty exists 2024-09-12T23:40:50.606 INFO:tasks.rados.rados.0.smithi136.stdout:834: finishing rollback tid 1 to smithi13667081-25 2024-09-12T23:40:50.606 INFO:tasks.rados.rados.0.smithi136.stdout:834: finishing rollback tid 2 to smithi13667081-25 2024-09-12T23:40:50.606 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 25 v 115 (ObjNum 144 snap 36 seq_num 144) dirty exists 2024-09-12T23:40:50.606 INFO:tasks.rados.rados.0.smithi136.stdout:836: finishing write tid 1 to smithi13667081-40 2024-09-12T23:40:50.606 INFO:tasks.rados.rados.0.smithi136.stdout:836: finishing write tid 2 to smithi13667081-40 2024-09-12T23:40:50.606 INFO:tasks.rados.rados.0.smithi136.stdout:836: finishing write tid 3 to smithi13667081-40 2024-09-12T23:40:50.607 INFO:tasks.rados.rados.0.smithi136.stdout:836: finishing write tid 4 to smithi13667081-40 2024-09-12T23:40:50.607 INFO:tasks.rados.rados.0.smithi136.stdout:837: finishing copy_from racing read to smithi13667081-34 2024-09-12T23:40:50.607 INFO:tasks.rados.rados.0.smithi136.stdout:836: finishing write tid 5 to smithi13667081-40 2024-09-12T23:40:50.607 INFO:tasks.rados.rados.0.smithi136.stdout:836: finishing write tid 6 to smithi13667081-40 2024-09-12T23:40:50.607 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 40 v 470 (ObjNum 274 snap 74 seq_num 274) dirty exists 2024-09-12T23:40:50.607 INFO:tasks.rados.rados.0.smithi136.stdout:836: left oid 40 (ObjNum 274 snap 74 seq_num 274) 2024-09-12T23:40:50.607 INFO:tasks.rados.rados.0.smithi136.stdout:837: finishing copy_from to smithi13667081-34 2024-09-12T23:40:50.607 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 34 v 263 (ObjNum 6 snap 0 seq_num 6) dirty exists 2024-09-12T23:40:50.607 INFO:tasks.rados.rados.0.smithi136.stdout:832: done (4 left) 2024-09-12T23:40:50.607 INFO:tasks.rados.rados.0.smithi136.stdout:833: done (3 left) 2024-09-12T23:40:50.607 INFO:tasks.rados.rados.0.smithi136.stdout:834: done (2 left) 2024-09-12T23:40:50.607 INFO:tasks.rados.rados.0.smithi136.stdout:836: done (1 left) 2024-09-12T23:40:50.608 INFO:tasks.rados.rados.0.smithi136.stdout:837: done (0 left) 2024-09-12T23:40:50.608 INFO:tasks.rados.rados.0.smithi136.stdout:838: read oid 36 snap -1 2024-09-12T23:40:50.608 INFO:tasks.rados.rados.0.smithi136.stdout:838: expect (ObjNum 256 snap 70 seq_num 256) 2024-09-12T23:40:50.608 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 1 current snap is 74 2024-09-12T23:40:50.608 INFO:tasks.rados.rados.0.smithi136.stdout:839: seq_num 275 ranges {0=614400,614400=786432,1400832=499712,1900544=319488} 2024-09-12T23:40:50.615 INFO:tasks.rados.rados.0.smithi136.stdout:839: writing smithi13667081-1 from 0 to 614400 tid 1 2024-09-12T23:40:50.623 INFO:tasks.rados.rados.0.smithi136.stdout:839: writing smithi13667081-1 from 614400 to 1400832 tid 2 2024-09-12T23:40:50.627 INFO:tasks.rados.rados.0.smithi136.stdout:839: writing smithi13667081-1 from 1400832 to 1900544 tid 3 2024-09-12T23:40:50.630 INFO:tasks.rados.rados.0.smithi136.stdout:839: writing smithi13667081-1 from 1900544 to 2220032 tid 4 2024-09-12T23:40:50.631 INFO:tasks.rados.rados.0.smithi136.stdout:839: finishing write tid 1 to smithi13667081-1 2024-09-12T23:40:50.631 INFO:tasks.rados.rados.0.smithi136.stdout:839: finishing write tid 2 to smithi13667081-1 2024-09-12T23:40:50.632 INFO:tasks.rados.rados.0.smithi136.stdout:840: copy_from oid 12 from oid 19 current snap is 74 2024-09-12T23:40:50.632 INFO:tasks.rados.rados.0.smithi136.stdout:841: read oid 17 snap -1 2024-09-12T23:40:50.632 INFO:tasks.rados.rados.0.smithi136.stdout:841: expect (ObjNum 164 snap 43 seq_num 164) 2024-09-12T23:40:50.632 INFO:tasks.rados.rados.0.smithi136.stdout:842: delete oid 25 current snap is 74 2024-09-12T23:40:50.634 INFO:tasks.rados.rados.0.smithi136.stdout:839: finishing write tid 3 to smithi13667081-1 2024-09-12T23:40:50.635 INFO:tasks.rados.rados.0.smithi136.stdout:841: expect (ObjNum 164 snap 43 seq_num 164) 2024-09-12T23:40:50.647 INFO:tasks.rados.rados.0.smithi136.stdout:839: finishing write tid 4 to smithi13667081-1 2024-09-12T23:40:50.647 INFO:tasks.rados.rados.0.smithi136.stdout:839: finishing write tid 5 to smithi13667081-1 2024-09-12T23:40:50.648 INFO:tasks.rados.rados.0.smithi136.stdout:839: finishing write tid 6 to smithi13667081-1 2024-09-12T23:40:50.648 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 1 v 204 (ObjNum 275 snap 74 seq_num 275) dirty exists 2024-09-12T23:40:50.648 INFO:tasks.rados.rados.0.smithi136.stdout:839: left oid 1 (ObjNum 275 snap 74 seq_num 275) 2024-09-12T23:40:50.648 INFO:tasks.rados.rados.0.smithi136.stdout:838: expect (ObjNum 256 snap 70 seq_num 256) 2024-09-12T23:40:50.828 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:50 smithi136 bash[68674]: ceph-da58a47c-715d-11ef-bceb-c7b262605968-mgr-y 2024-09-12T23:40:50.828 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:50 smithi136 bash[32189]: cluster 2024-09-12T23:40:49.571134+0000 mgr.x (mgr.24517) 58 : cluster [DBG] pgmap v66: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 119 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 1.3 MiB/s wr, 5 op/s 2024-09-12T23:40:50.828 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:50 smithi136 bash[32189]: cluster 2024-09-12T23:40:49.571473+0000 mon.a (mon.0) 1325 : cluster [DBG] osdmap e214: 8 total, 8 up, 8 in 2024-09-12T23:40:50.828 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:50 smithi136 bash[32189]: cluster 2024-09-12T23:40:50.496335+0000 mon.a (mon.0) 1326 : cluster [DBG] osdmap e215: 8 total, 8 up, 8 in 2024-09-12T23:40:50.829 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:50 smithi136 bash[22601]: cluster 2024-09-12T23:40:49.571134+0000 mgr.x (mgr.24517) 58 : cluster [DBG] pgmap v66: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 119 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 1.3 MiB/s wr, 5 op/s 2024-09-12T23:40:50.829 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:50 smithi136 bash[22601]: cluster 2024-09-12T23:40:49.571473+0000 mon.a (mon.0) 1325 : cluster [DBG] osdmap e214: 8 total, 8 up, 8 in 2024-09-12T23:40:50.829 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:50 smithi136 bash[22601]: cluster 2024-09-12T23:40:50.496335+0000 mon.a (mon.0) 1326 : cluster [DBG] osdmap e215: 8 total, 8 up, 8 in 2024-09-12T23:40:50.842 INFO:tasks.rados.rados.0.smithi136.stdout:840: finishing copy_from racing read to smithi13667081-12 2024-09-12T23:40:50.842 INFO:tasks.rados.rados.0.smithi136.stdout:840: finishing copy_from to smithi13667081-12 2024-09-12T23:40:50.842 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 12 v 271 (ObjNum 264 snap 71 seq_num 264) dirty exists 2024-09-12T23:40:50.842 INFO:tasks.rados.rados.0.smithi136.stdout:838: done (4 left) 2024-09-12T23:40:50.842 INFO:tasks.rados.rados.0.smithi136.stdout:839: done (3 left) 2024-09-12T23:40:50.843 INFO:tasks.rados.rados.0.smithi136.stdout:840: done (2 left) 2024-09-12T23:40:50.843 INFO:tasks.rados.rados.0.smithi136.stdout:841: done (1 left) 2024-09-12T23:40:50.843 INFO:tasks.rados.rados.0.smithi136.stdout:842: done (0 left) 2024-09-12T23:40:50.843 INFO:tasks.rados.rados.0.smithi136.stdout:843: snap_remove snap 59 2024-09-12T23:40:50.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:50 smithi181 bash[24776]: cluster 2024-09-12T23:40:49.571134+0000 mgr.x (mgr.24517) 58 : cluster [DBG] pgmap v66: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 119 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 1.3 MiB/s wr, 5 op/s 2024-09-12T23:40:50.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:50 smithi181 bash[24776]: cluster 2024-09-12T23:40:49.571473+0000 mon.a (mon.0) 1325 : cluster [DBG] osdmap e214: 8 total, 8 up, 8 in 2024-09-12T23:40:50.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:50 smithi181 bash[24776]: cluster 2024-09-12T23:40:50.496335+0000 mon.a (mon.0) 1326 : cluster [DBG] osdmap e215: 8 total, 8 up, 8 in 2024-09-12T23:40:51.094 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:50 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:40:51.094 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 12 23:40:50 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:40:51.095 INFO:journalctl@ceph.mds.a.smithi136.stdout:Sep 12 23:40:50 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:40:51.095 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:50 smithi136 systemd[1]: ceph-da58a47c-715d-11ef-bceb-c7b262605968@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-09-12T23:40:51.095 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:50 smithi136 bash[68731]: Error response from daemon: No such container: ceph-da58a47c-715d-11ef-bceb-c7b262605968-mgr-y 2024-09-12T23:40:51.095 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:50 smithi136 systemd[1]: ceph-da58a47c-715d-11ef-bceb-c7b262605968@mgr.y.service: Failed with result 'exit-code'. 2024-09-12T23:40:51.095 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:50 smithi136 systemd[1]: Stopped Ceph mgr.y for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:40:51.095 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:50 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:40:51.096 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 12 23:40:50 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:40:51.096 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 12 23:40:50 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:40:51.096 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:40:50 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:40:51.097 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:50 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:40:51.097 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:40:50 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:40:51.097 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 12 23:40:50 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:40:51.505 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:51 smithi136 systemd[1]: Started Ceph mgr.y for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:40:51.584 INFO:tasks.rados.rados.0.smithi136.stdout:843: done (0 left) 2024-09-12T23:40:51.584 INFO:tasks.rados.rados.0.smithi136.stdout:844: snap_remove snap 63 2024-09-12T23:40:52.489 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:52 smithi136 bash[68787]: debug 2024-09-12T23:40:52.376+0000 7f982654b640 1 -- 172.21.15.136:0/831398684 <== mon.2 v2:172.21.15.136:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55c314a0cd00 con 0x55c3149ff400 2024-09-12T23:40:52.489 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:52 smithi136 bash[22601]: audit 2024-09-12T23:40:51.138066+0000 mon.a (mon.0) 1327 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:52.489 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:52 smithi136 bash[22601]: audit 2024-09-12T23:40:51.150390+0000 mon.a (mon.0) 1328 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:52.489 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:52 smithi136 bash[22601]: audit 2024-09-12T23:40:51.155389+0000 mon.b (mon.1) 103 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:40:52.489 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:52 smithi136 bash[22601]: cluster 2024-09-12T23:40:51.584725+0000 mon.a (mon.0) 1329 : cluster [DBG] osdmap e216: 8 total, 8 up, 8 in 2024-09-12T23:40:52.490 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:52 smithi136 bash[32189]: audit 2024-09-12T23:40:51.138066+0000 mon.a (mon.0) 1327 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:52.490 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:52 smithi136 bash[32189]: audit 2024-09-12T23:40:51.150390+0000 mon.a (mon.0) 1328 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:52.490 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:52 smithi136 bash[32189]: audit 2024-09-12T23:40:51.155389+0000 mon.b (mon.1) 103 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:40:52.490 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:52 smithi136 bash[32189]: cluster 2024-09-12T23:40:51.584725+0000 mon.a (mon.0) 1329 : cluster [DBG] osdmap e216: 8 total, 8 up, 8 in 2024-09-12T23:40:52.582 INFO:tasks.rados.rados.0.smithi136.stdout:844: done (0 left) 2024-09-12T23:40:52.582 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 14 current snap is 74 2024-09-12T23:40:52.582 INFO:tasks.rados.rados.0.smithi136.stdout:845: seq_num 277 ranges {647168=647168,1294336=557056,1851392=516096,2367488=532480} 2024-09-12T23:40:52.593 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:52 smithi181 bash[24776]: audit 2024-09-12T23:40:51.138066+0000 mon.a (mon.0) 1327 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:52.593 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:52 smithi181 bash[24776]: audit 2024-09-12T23:40:51.150390+0000 mon.a (mon.0) 1328 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:52.593 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:52 smithi181 bash[24776]: audit 2024-09-12T23:40:51.155389+0000 mon.b (mon.1) 103 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:40:52.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:52 smithi181 bash[24776]: cluster 2024-09-12T23:40:51.584725+0000 mon.a (mon.0) 1329 : cluster [DBG] osdmap e216: 8 total, 8 up, 8 in 2024-09-12T23:40:52.606 INFO:tasks.rados.rados.0.smithi136.stdout:845: writing smithi13667081-14 from 647168 to 1294336 tid 1 2024-09-12T23:40:52.618 INFO:tasks.rados.rados.0.smithi136.stdout:845: writing smithi13667081-14 from 1294336 to 1851392 tid 2 2024-09-12T23:40:52.626 INFO:tasks.rados.rados.0.smithi136.stdout:845: writing smithi13667081-14 from 1851392 to 2367488 tid 3 2024-09-12T23:40:52.635 INFO:tasks.rados.rados.0.smithi136.stdout:845: writing smithi13667081-14 from 2367488 to 2899968 tid 4 2024-09-12T23:40:52.635 INFO:tasks.rados.rados.0.smithi136.stdout:846: read oid 31 snap -1 2024-09-12T23:40:52.635 INFO:tasks.rados.rados.0.smithi136.stdout:846: expect deleted 2024-09-12T23:40:52.635 INFO:tasks.rados.rados.0.smithi136.stdout:847: rollback oid 17 current snap is 74 2024-09-12T23:40:52.635 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 17 to 67 2024-09-12T23:40:52.635 INFO:tasks.rados.rados.0.smithi136.stdout:845: finishing write tid 1 to smithi13667081-14 2024-09-12T23:40:52.635 INFO:tasks.rados.rados.0.smithi136.stdout:845: finishing write tid 2 to smithi13667081-14 2024-09-12T23:40:52.635 INFO:tasks.rados.rados.0.smithi136.stdout:845: finishing write tid 3 to smithi13667081-14 2024-09-12T23:40:52.635 INFO:tasks.rados.rados.0.smithi136.stdout:848: rollback oid 2 current snap is 74 2024-09-12T23:40:52.635 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 2 to 61 2024-09-12T23:40:52.635 INFO:tasks.rados.rados.0.smithi136.stdout:849: read oid 39 snap -1 2024-09-12T23:40:52.635 INFO:tasks.rados.rados.0.smithi136.stdout:849: expect (ObjNum 225 snap 56 seq_num 225) 2024-09-12T23:40:52.635 INFO:tasks.rados.rados.0.smithi136.stdout:850: read oid 46 snap 73 2024-09-12T23:40:52.636 INFO:tasks.rados.rados.0.smithi136.stdout:850: expect (ObjNum 228 snap 56 seq_num 228) 2024-09-12T23:40:52.636 INFO:tasks.rados.rados.0.smithi136.stdout:851: snap_create 2024-09-12T23:40:52.639 INFO:tasks.rados.rados.0.smithi136.stdout:847: finishing rollback tid 0 to smithi13667081-17 2024-09-12T23:40:52.640 INFO:tasks.rados.rados.0.smithi136.stdout:847: finishing rollback tid 1 to smithi13667081-17 2024-09-12T23:40:52.640 INFO:tasks.rados.rados.0.smithi136.stdout:847: finishing rollback tid 2 to smithi13667081-17 2024-09-12T23:40:52.640 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 17 v 488 (ObjNum 164 snap 43 seq_num 164) dirty exists 2024-09-12T23:40:52.641 INFO:tasks.rados.rados.0.smithi136.stdout:845: finishing write tid 4 to smithi13667081-14 2024-09-12T23:40:52.641 INFO:tasks.rados.rados.0.smithi136.stdout:845: finishing write tid 5 to smithi13667081-14 2024-09-12T23:40:52.642 INFO:tasks.rados.rados.0.smithi136.stdout:848: finishing rollback tid 0 to smithi13667081-2 2024-09-12T23:40:52.642 INFO:tasks.rados.rados.0.smithi136.stdout:848: finishing rollback tid 1 to smithi13667081-2 2024-09-12T23:40:52.643 INFO:tasks.rados.rados.0.smithi136.stdout:845: finishing write tid 6 to smithi13667081-14 2024-09-12T23:40:52.643 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 14 v 509 (ObjNum 277 snap 74 seq_num 277) dirty exists 2024-09-12T23:40:52.643 INFO:tasks.rados.rados.0.smithi136.stdout:845: left oid 14 (ObjNum 277 snap 74 seq_num 277) 2024-09-12T23:40:52.643 INFO:tasks.rados.rados.0.smithi136.stdout:848: finishing rollback tid 2 to smithi13667081-2 2024-09-12T23:40:52.643 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 2 v 393 (ObjNum 193 snap 47 seq_num 193) dirty exists 2024-09-12T23:40:52.650 INFO:tasks.rados.rados.0.smithi136.stdout:849: expect (ObjNum 225 snap 56 seq_num 225) 2024-09-12T23:40:52.682 INFO:tasks.rados.rados.0.smithi136.stdout:850: expect (ObjNum 228 snap 56 seq_num 228) 2024-09-12T23:40:52.755 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:52 smithi136 bash[68787]: debug 2024-09-12T23:40:52.492+0000 7f9828db4040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-12T23:40:53.505 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:53 smithi136 bash[32189]: cluster 2024-09-12T23:40:51.571785+0000 mgr.x (mgr.24517) 59 : cluster [DBG] pgmap v68: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 119 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.3 MiB/s wr, 5 op/s 2024-09-12T23:40:53.505 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:53 smithi136 bash[32189]: cluster 2024-09-12T23:40:52.584371+0000 mon.a (mon.0) 1330 : cluster [DBG] osdmap e217: 8 total, 8 up, 8 in 2024-09-12T23:40:53.505 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:53 smithi136 bash[32189]: audit 2024-09-12T23:40:52.736287+0000 mon.b (mon.1) 104 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:40:53.505 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:53 smithi136 bash[22601]: cluster 2024-09-12T23:40:51.571785+0000 mgr.x (mgr.24517) 59 : cluster [DBG] pgmap v68: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 119 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.3 MiB/s wr, 5 op/s 2024-09-12T23:40:53.505 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:53 smithi136 bash[22601]: cluster 2024-09-12T23:40:52.584371+0000 mon.a (mon.0) 1330 : cluster [DBG] osdmap e217: 8 total, 8 up, 8 in 2024-09-12T23:40:53.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:53 smithi136 bash[22601]: audit 2024-09-12T23:40:52.736287+0000 mon.b (mon.1) 104 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:40:53.506 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:53 smithi181 bash[24776]: cluster 2024-09-12T23:40:51.571785+0000 mgr.x (mgr.24517) 59 : cluster [DBG] pgmap v68: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 119 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.3 MiB/s wr, 5 op/s 2024-09-12T23:40:53.506 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:53 smithi181 bash[24776]: cluster 2024-09-12T23:40:52.584371+0000 mon.a (mon.0) 1330 : cluster [DBG] osdmap e217: 8 total, 8 up, 8 in 2024-09-12T23:40:53.506 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:53 smithi181 bash[24776]: audit 2024-09-12T23:40:52.736287+0000 mon.b (mon.1) 104 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:40:53.585 INFO:tasks.rados.rados.0.smithi136.stdout:845: done (6 left) 2024-09-12T23:40:53.585 INFO:tasks.rados.rados.0.smithi136.stdout:846: done (5 left) 2024-09-12T23:40:53.585 INFO:tasks.rados.rados.0.smithi136.stdout:847: done (4 left) 2024-09-12T23:40:53.585 INFO:tasks.rados.rados.0.smithi136.stdout:848: done (3 left) 2024-09-12T23:40:53.586 INFO:tasks.rados.rados.0.smithi136.stdout:849: done (2 left) 2024-09-12T23:40:53.586 INFO:tasks.rados.rados.0.smithi136.stdout:850: done (1 left) 2024-09-12T23:40:53.586 INFO:tasks.rados.rados.0.smithi136.stdout:851: done (0 left) 2024-09-12T23:40:53.586 INFO:tasks.rados.rados.0.smithi136.stdout:852: read oid 27 snap -1 2024-09-12T23:40:53.586 INFO:tasks.rados.rados.0.smithi136.stdout:852: expect (ObjNum 248 snap 67 seq_num 248) 2024-09-12T23:40:53.586 INFO:tasks.rados.rados.0.smithi136.stdout:853: snap_remove snap 74 2024-09-12T23:40:53.843 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:40:53 smithi181 bash[58696]: ts=2024-09-12T23:40:53.510Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi136\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:40:54.005 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:53 smithi136 bash[68787]: debug 2024-09-12T23:40:53.732+0000 7f9828db4040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-12T23:40:54.005 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:53 smithi136 bash[68787]: debug 2024-09-12T23:40:53.796+0000 7f9828db4040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-12T23:40:54.005 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:53 smithi136 bash[68787]: debug 2024-09-12T23:40:53.948+0000 7f9828db4040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-12T23:40:54.590 INFO:tasks.rados.rados.0.smithi136.stdout:853: done (1 left) 2024-09-12T23:40:54.590 INFO:tasks.rados.rados.0.smithi136.stdout:854: snap_create 2024-09-12T23:40:54.590 INFO:tasks.rados.rados.0.smithi136.stdout:852: expect (ObjNum 248 snap 67 seq_num 248) 2024-09-12T23:40:54.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:54 smithi181 bash[24776]: cluster 2024-09-12T23:40:53.572611+0000 mgr.x (mgr.24517) 60 : cluster [DBG] pgmap v71: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 119 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:40:54.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:54 smithi181 bash[24776]: cluster 2024-09-12T23:40:53.587228+0000 mon.a (mon.0) 1331 : cluster [DBG] osdmap e218: 8 total, 8 up, 8 in 2024-09-12T23:40:54.844 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:40:54 smithi181 bash[58696]: ts=2024-09-12T23:40:54.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:40:54.844 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:40:54 smithi181 bash[58696]: ts=2024-09-12T23:40:54.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:40:54.916 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:54 smithi136 bash[32189]: cluster 2024-09-12T23:40:53.572611+0000 mgr.x (mgr.24517) 60 : cluster [DBG] pgmap v71: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 119 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:40:54.917 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:54 smithi136 bash[32189]: cluster 2024-09-12T23:40:53.587228+0000 mon.a (mon.0) 1331 : cluster [DBG] osdmap e218: 8 total, 8 up, 8 in 2024-09-12T23:40:54.917 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:54 smithi136 bash[22601]: cluster 2024-09-12T23:40:53.572611+0000 mgr.x (mgr.24517) 60 : cluster [DBG] pgmap v71: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 119 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:40:54.917 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:54 smithi136 bash[22601]: cluster 2024-09-12T23:40:53.587228+0000 mon.a (mon.0) 1331 : cluster [DBG] osdmap e218: 8 total, 8 up, 8 in 2024-09-12T23:40:54.917 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:54 smithi136 bash[68787]: debug 2024-09-12T23:40:54.592+0000 7f9828db4040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-12T23:40:54.917 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:54 smithi136 bash[68787]: debug 2024-09-12T23:40:54.664+0000 7f9828db4040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-12T23:40:54.917 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:54 smithi136 bash[68787]: debug 2024-09-12T23:40:54.732+0000 7f9828db4040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-12T23:40:54.918 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:54 smithi136 bash[68787]: debug 2024-09-12T23:40:54.920+0000 7f9828db4040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-12T23:40:55.255 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:54 smithi136 bash[68787]: debug 2024-09-12T23:40:54.996+0000 7f9828db4040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-12T23:40:55.255 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:55 smithi136 bash[68787]: debug 2024-09-12T23:40:55.064+0000 7f9828db4040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-12T23:40:55.576 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:55 smithi136 bash[68787]: debug 2024-09-12T23:40:55.316+0000 7f9828db4040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-12T23:40:55.577 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:55 smithi136 bash[68787]: debug 2024-09-12T23:40:55.384+0000 7f9828db4040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-12T23:40:55.585 INFO:tasks.rados.rados.0.smithi136.stdout:852: done (1 left) 2024-09-12T23:40:55.585 INFO:tasks.rados.rados.0.smithi136.stdout:854: done (0 left) 2024-09-12T23:40:55.585 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 48 current snap is 76 2024-09-12T23:40:55.585 INFO:tasks.rados.rados.0.smithi136.stdout:855: seq_num 278 ranges {0=663552,663552=442368,1105920=720896,1826816=442368} 2024-09-12T23:40:55.591 INFO:tasks.rados.rados.0.smithi136.stdout:855: writing smithi13667081-48 from 0 to 663552 tid 1 2024-09-12T23:40:55.597 INFO:tasks.rados.rados.0.smithi136.stdout:855: writing smithi13667081-48 from 663552 to 1105920 tid 2 2024-09-12T23:40:55.604 INFO:tasks.rados.rados.0.smithi136.stdout:855: writing smithi13667081-48 from 1105920 to 1826816 tid 3 2024-09-12T23:40:55.609 INFO:tasks.rados.rados.0.smithi136.stdout:855: writing smithi13667081-48 from 1826816 to 2269184 tid 4 2024-09-12T23:40:55.609 INFO:tasks.rados.rados.0.smithi136.stdout:856: copy_from oid 23 from oid 35 current snap is 76 2024-09-12T23:40:55.609 INFO:tasks.rados.rados.0.smithi136.stdout:857: delete oid 34 current snap is 76 2024-09-12T23:40:55.611 INFO:tasks.rados.rados.0.smithi136.stdout:855: finishing write tid 1 to smithi13667081-48 2024-09-12T23:40:55.611 INFO:tasks.rados.rados.0.smithi136.stdout:855: finishing write tid 2 to smithi13667081-48 2024-09-12T23:40:55.614 INFO:tasks.rados.rados.0.smithi136.stdout:855: finishing write tid 3 to smithi13667081-48 2024-09-12T23:40:55.614 INFO:tasks.rados.rados.0.smithi136.stdout:857: done (2 left) 2024-09-12T23:40:55.614 INFO:tasks.rados.rados.0.smithi136.stdout:858: snap_create 2024-09-12T23:40:55.615 INFO:tasks.rados.rados.0.smithi136.stdout:856: finishing copy_from to smithi13667081-23 2024-09-12T23:40:55.615 INFO:tasks.rados.rados.0.smithi136.stdout:856: got expected ENOENT (src dne) 2024-09-12T23:40:55.615 INFO:tasks.rados.rados.0.smithi136.stdout:856: finishing copy_from racing read to smithi13667081-23 2024-09-12T23:40:55.616 INFO:tasks.rados.rados.0.smithi136.stdout:855: finishing write tid 4 to smithi13667081-48 2024-09-12T23:40:55.616 INFO:tasks.rados.rados.0.smithi136.stdout:855: finishing write tid 5 to smithi13667081-48 2024-09-12T23:40:55.618 INFO:tasks.rados.rados.0.smithi136.stdout:855: finishing write tid 6 to smithi13667081-48 2024-09-12T23:40:55.618 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 48 v 515 (ObjNum 278 snap 76 seq_num 278) dirty exists 2024-09-12T23:40:55.618 INFO:tasks.rados.rados.0.smithi136.stdout:855: left oid 48 (ObjNum 278 snap 76 seq_num 278) 2024-09-12T23:40:55.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:55 smithi181 bash[24776]: cluster 2024-09-12T23:40:54.594594+0000 mon.a (mon.0) 1332 : cluster [DBG] osdmap e219: 8 total, 8 up, 8 in 2024-09-12T23:40:55.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:55 smithi181 bash[24776]: cluster 2024-09-12T23:40:55.590379+0000 mon.a (mon.0) 1333 : cluster [DBG] osdmap e220: 8 total, 8 up, 8 in 2024-09-12T23:40:55.914 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:55 smithi136 bash[32189]: cluster 2024-09-12T23:40:54.594594+0000 mon.a (mon.0) 1332 : cluster [DBG] osdmap e219: 8 total, 8 up, 8 in 2024-09-12T23:40:55.914 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:55 smithi136 bash[32189]: cluster 2024-09-12T23:40:55.590379+0000 mon.a (mon.0) 1333 : cluster [DBG] osdmap e220: 8 total, 8 up, 8 in 2024-09-12T23:40:55.914 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:55 smithi136 bash[68787]: debug 2024-09-12T23:40:55.580+0000 7f9828db4040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-12T23:40:55.914 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:55 smithi136 bash[68787]: debug 2024-09-12T23:40:55.664+0000 7f9828db4040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-12T23:40:55.914 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:55 smithi136 bash[22601]: cluster 2024-09-12T23:40:54.594594+0000 mon.a (mon.0) 1332 : cluster [DBG] osdmap e219: 8 total, 8 up, 8 in 2024-09-12T23:40:55.915 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:55 smithi136 bash[22601]: cluster 2024-09-12T23:40:55.590379+0000 mon.a (mon.0) 1333 : cluster [DBG] osdmap e220: 8 total, 8 up, 8 in 2024-09-12T23:40:56.255 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:55 smithi136 bash[68787]: debug 2024-09-12T23:40:55.916+0000 7f9828db4040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-12T23:40:56.255 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:56 smithi136 bash[68787]: debug 2024-09-12T23:40:56.092+0000 7f9828db4040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-12T23:40:56.544 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:56 smithi136 bash[68787]: debug 2024-09-12T23:40:56.268+0000 7f9828db4040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-12T23:40:56.545 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:56 smithi136 bash[68787]: debug 2024-09-12T23:40:56.348+0000 7f9828db4040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-12T23:40:56.545 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:56 smithi136 bash[68787]: debug 2024-09-12T23:40:56.548+0000 7f9828db4040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-12T23:40:56.608 INFO:tasks.rados.rados.0.smithi136.stdout:855: done (2 left) 2024-09-12T23:40:56.608 INFO:tasks.rados.rados.0.smithi136.stdout:856: done (1 left) 2024-09-12T23:40:56.608 INFO:tasks.rados.rados.0.smithi136.stdout:858: done (0 left) 2024-09-12T23:40:56.608 INFO:tasks.rados.rados.0.smithi136.stdout:859: read oid 28 snap -1 2024-09-12T23:40:56.608 INFO:tasks.rados.rados.0.smithi136.stdout:859: expect (ObjNum 272 snap 73 seq_num 272) 2024-09-12T23:40:56.608 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 2 current snap is 77 2024-09-12T23:40:56.608 INFO:tasks.rados.rados.0.smithi136.stdout:860: seq_num 280 ranges {2637824=696320,3334144=606208,3940352=688128,4628480=311296} 2024-09-12T23:40:56.627 INFO:tasks.rados.rados.0.smithi136.stdout:860: writing smithi13667081-2 from 2637824 to 3334144 tid 1 2024-09-12T23:40:56.632 INFO:tasks.rados.rados.0.smithi136.stdout:860: writing smithi13667081-2 from 3334144 to 3940352 tid 2 2024-09-12T23:40:56.637 INFO:tasks.rados.rados.0.smithi136.stdout:860: writing smithi13667081-2 from 3940352 to 4628480 tid 3 2024-09-12T23:40:56.640 INFO:tasks.rados.rados.0.smithi136.stdout:860: writing smithi13667081-2 from 4628480 to 4939776 tid 4 2024-09-12T23:40:56.640 INFO:tasks.rados.rados.0.smithi136.stdout:859: expect (ObjNum 272 snap 73 seq_num 272) 2024-09-12T23:40:56.718 INFO:tasks.rados.rados.0.smithi136.stdout:860: finishing write tid 1 to smithi13667081-2 2024-09-12T23:40:56.719 INFO:tasks.rados.rados.0.smithi136.stdout:860: finishing write tid 2 to smithi13667081-2 2024-09-12T23:40:56.719 INFO:tasks.rados.rados.0.smithi136.stdout:859: done (1 left) 2024-09-12T23:40:56.719 INFO:tasks.rados.rados.0.smithi136.stdout:861: delete oid 48 current snap is 77 2024-09-12T23:40:56.719 INFO:tasks.rados.rados.0.smithi136.stdout:860: finishing write tid 3 to smithi13667081-2 2024-09-12T23:40:56.719 INFO:tasks.rados.rados.0.smithi136.stdout:860: finishing write tid 4 to smithi13667081-2 2024-09-12T23:40:56.719 INFO:tasks.rados.rados.0.smithi136.stdout:860: finishing write tid 5 to smithi13667081-2 2024-09-12T23:40:56.719 INFO:tasks.rados.rados.0.smithi136.stdout:860: finishing write tid 6 to smithi13667081-2 2024-09-12T23:40:56.719 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 2 v 401 (ObjNum 280 snap 77 seq_num 280) dirty exists 2024-09-12T23:40:56.719 INFO:tasks.rados.rados.0.smithi136.stdout:860: left oid 2 (ObjNum 280 snap 77 seq_num 280) 2024-09-12T23:40:56.721 INFO:tasks.rados.rados.0.smithi136.stdout:860: done (1 left) 2024-09-12T23:40:56.721 INFO:tasks.rados.rados.0.smithi136.stdout:861: done (0 left) 2024-09-12T23:40:56.721 INFO:tasks.rados.rados.0.smithi136.stdout:862: snap_create 2024-09-12T23:40:57.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:56 smithi136 bash[22601]: cluster 2024-09-12T23:40:55.573294+0000 mgr.x (mgr.24517) 61 : cluster [DBG] pgmap v74: 121 pgs: 121 active+clean; 124 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 3.5 MiB/s wr, 31 op/s 2024-09-12T23:40:57.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:56 smithi136 bash[32189]: cluster 2024-09-12T23:40:55.573294+0000 mgr.x (mgr.24517) 61 : cluster [DBG] pgmap v74: 121 pgs: 121 active+clean; 124 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 3.5 MiB/s wr, 31 op/s 2024-09-12T23:40:57.093 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:56 smithi181 bash[24776]: cluster 2024-09-12T23:40:55.573294+0000 mgr.x (mgr.24517) 61 : cluster [DBG] pgmap v74: 121 pgs: 121 active+clean; 124 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 3.5 MiB/s wr, 31 op/s 2024-09-12T23:40:57.720 INFO:tasks.rados.rados.0.smithi136.stdout:862: done (0 left) 2024-09-12T23:40:57.720 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 27 current snap is 78 2024-09-12T23:40:57.720 INFO:tasks.rados.rados.0.smithi136.stdout:863: seq_num 282 ranges {1761280=729088,2490368=425984,2916352=704512,3620864=475136} 2024-09-12T23:40:57.733 INFO:tasks.rados.rados.0.smithi136.stdout:863: writing smithi13667081-27 from 1761280 to 2490368 tid 1 2024-09-12T23:40:57.736 INFO:tasks.rados.rados.0.smithi136.stdout:863: writing smithi13667081-27 from 2490368 to 2916352 tid 2 2024-09-12T23:40:57.741 INFO:tasks.rados.rados.0.smithi136.stdout:863: writing smithi13667081-27 from 2916352 to 3620864 tid 3 2024-09-12T23:40:57.744 INFO:tasks.rados.rados.0.smithi136.stdout:863: writing smithi13667081-27 from 3620864 to 4096000 tid 4 2024-09-12T23:40:57.745 INFO:tasks.rados.rados.0.smithi136.stdout:864: read oid 26 snap -1 2024-09-12T23:40:57.745 INFO:tasks.rados.rados.0.smithi136.stdout:864: expect (ObjNum 160 snap 42 seq_num 160) 2024-09-12T23:40:57.745 INFO:tasks.rados.rados.0.smithi136.stdout:865: rmattr oid 4 current snap is 78 2024-09-12T23:40:57.745 INFO:tasks.rados.rados.0.smithi136.stdout:863: finishing write tid 1 to smithi13667081-27 2024-09-12T23:40:57.745 INFO:tasks.rados.rados.0.smithi136.stdout:863: finishing write tid 2 to smithi13667081-27 2024-09-12T23:40:57.745 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 24 current snap is 78 2024-09-12T23:40:57.745 INFO:tasks.rados.rados.0.smithi136.stdout:866: seq_num 283 ranges {1662976=745472,2408448=712704,3121152=458752,3579904=434176} 2024-09-12T23:40:57.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:57 smithi136 bash[22601]: cluster 2024-09-12T23:40:56.612926+0000 mon.a (mon.0) 1334 : cluster [DBG] osdmap e221: 8 total, 8 up, 8 in 2024-09-12T23:40:57.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:57 smithi136 bash[22601]: audit 2024-09-12T23:40:56.709754+0000 mon.a (mon.0) 1335 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:57.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:57 smithi136 bash[22601]: audit 2024-09-12T23:40:56.720684+0000 mon.a (mon.0) 1336 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:57.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:57 smithi136 bash[22601]: audit 2024-09-12T23:40:57.005083+0000 mon.a (mon.0) 1337 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:57.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:57 smithi136 bash[22601]: audit 2024-09-12T23:40:57.015600+0000 mon.a (mon.0) 1338 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:57.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:57 smithi136 bash[22601]: audit 2024-09-12T23:40:57.429540+0000 mon.a (mon.0) 1339 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:57.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:57 smithi136 bash[22601]: audit 2024-09-12T23:40:57.438998+0000 mon.a (mon.0) 1340 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:57.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:57 smithi136 bash[68787]: debug 2024-09-12T23:40:57.420+0000 7f9828db4040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-12T23:40:57.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:57 smithi136 bash[68787]: debug 2024-09-12T23:40:57.500+0000 7f9828db4040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-12T23:40:57.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:57 smithi136 bash[32189]: cluster 2024-09-12T23:40:56.612926+0000 mon.a (mon.0) 1334 : cluster [DBG] osdmap e221: 8 total, 8 up, 8 in 2024-09-12T23:40:57.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:57 smithi136 bash[32189]: audit 2024-09-12T23:40:56.709754+0000 mon.a (mon.0) 1335 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:57.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:57 smithi136 bash[32189]: audit 2024-09-12T23:40:56.720684+0000 mon.a (mon.0) 1336 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:57.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:57 smithi136 bash[32189]: audit 2024-09-12T23:40:57.005083+0000 mon.a (mon.0) 1337 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:57.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:57 smithi136 bash[32189]: audit 2024-09-12T23:40:57.015600+0000 mon.a (mon.0) 1338 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:57.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:57 smithi136 bash[32189]: audit 2024-09-12T23:40:57.429540+0000 mon.a (mon.0) 1339 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:57.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:57 smithi136 bash[32189]: audit 2024-09-12T23:40:57.438998+0000 mon.a (mon.0) 1340 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:57.758 INFO:tasks.rados.rados.0.smithi136.stdout:866: writing smithi13667081-24 from 1662976 to 2408448 tid 1 2024-09-12T23:40:57.763 INFO:tasks.rados.rados.0.smithi136.stdout:866: writing smithi13667081-24 from 2408448 to 3121152 tid 2 2024-09-12T23:40:57.765 INFO:tasks.rados.rados.0.smithi136.stdout:866: writing smithi13667081-24 from 3121152 to 3579904 tid 3 2024-09-12T23:40:57.769 INFO:tasks.rados.rados.0.smithi136.stdout:866: writing smithi13667081-24 from 3579904 to 4014080 tid 4 2024-09-12T23:40:57.769 INFO:tasks.rados.rados.0.smithi136.stdout:867: rmattr oid 1 current snap is 78 2024-09-12T23:40:57.769 INFO:tasks.rados.rados.0.smithi136.stdout:863: finishing write tid 3 to smithi13667081-27 2024-09-12T23:40:57.769 INFO:tasks.rados.rados.0.smithi136.stdout:863: finishing write tid 4 to smithi13667081-27 2024-09-12T23:40:57.769 INFO:tasks.rados.rados.0.smithi136.stdout:863: finishing write tid 5 to smithi13667081-27 2024-09-12T23:40:57.769 INFO:tasks.rados.rados.0.smithi136.stdout:863: finishing write tid 6 to smithi13667081-27 2024-09-12T23:40:57.769 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 27 v 494 (ObjNum 282 snap 78 seq_num 282) dirty exists 2024-09-12T23:40:57.769 INFO:tasks.rados.rados.0.smithi136.stdout:863: left oid 27 (ObjNum 282 snap 78 seq_num 282) 2024-09-12T23:40:57.769 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 4 v 174 (ObjNum 220 snap 55 seq_num 220) dirty exists 2024-09-12T23:40:57.769 INFO:tasks.rados.rados.0.smithi136.stdout:864: expect (ObjNum 160 snap 42 seq_num 160) 2024-09-12T23:40:57.854 INFO:tasks.rados.rados.0.smithi136.stdout:866: finishing write tid 1 to smithi13667081-24 2024-09-12T23:40:57.855 INFO:tasks.rados.rados.0.smithi136.stdout:866: finishing write tid 2 to smithi13667081-24 2024-09-12T23:40:57.855 INFO:tasks.rados.rados.0.smithi136.stdout:866: finishing write tid 3 to smithi13667081-24 2024-09-12T23:40:57.855 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 1 v 218 (ObjNum 275 snap 74 seq_num 275) dirty exists 2024-09-12T23:40:57.855 INFO:tasks.rados.rados.0.smithi136.stdout:866: finishing write tid 4 to smithi13667081-24 2024-09-12T23:40:57.855 INFO:tasks.rados.rados.0.smithi136.stdout:866: finishing write tid 5 to smithi13667081-24 2024-09-12T23:40:57.855 INFO:tasks.rados.rados.0.smithi136.stdout:866: finishing write tid 6 to smithi13667081-24 2024-09-12T23:40:57.855 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 24 v 376 (ObjNum 283 snap 78 seq_num 283) dirty exists 2024-09-12T23:40:57.855 INFO:tasks.rados.rados.0.smithi136.stdout:866: left oid 24 (ObjNum 283 snap 78 seq_num 283) 2024-09-12T23:40:57.855 INFO:tasks.rados.rados.0.smithi136.stdout:863: done (4 left) 2024-09-12T23:40:57.855 INFO:tasks.rados.rados.0.smithi136.stdout:864: done (3 left) 2024-09-12T23:40:57.855 INFO:tasks.rados.rados.0.smithi136.stdout:865: done (2 left) 2024-09-12T23:40:57.855 INFO:tasks.rados.rados.0.smithi136.stdout:866: done (1 left) 2024-09-12T23:40:57.855 INFO:tasks.rados.rados.0.smithi136.stdout:867: done (0 left) 2024-09-12T23:40:57.856 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 3 current snap is 78 2024-09-12T23:40:57.856 INFO:tasks.rados.rados.0.smithi136.stdout:868: seq_num 284 ranges {0=761856,761856=598016,1359872=671744,2031616=335872} 2024-09-12T23:40:57.859 INFO:tasks.rados.rados.0.smithi136.stdout:868: writing smithi13667081-3 from 0 to 761856 tid 1 2024-09-12T23:40:57.863 INFO:tasks.rados.rados.0.smithi136.stdout:868: writing smithi13667081-3 from 761856 to 1359872 tid 2 2024-09-12T23:40:57.868 INFO:tasks.rados.rados.0.smithi136.stdout:868: writing smithi13667081-3 from 1359872 to 2031616 tid 3 2024-09-12T23:40:57.871 INFO:tasks.rados.rados.0.smithi136.stdout:868: writing smithi13667081-3 from 2031616 to 2367488 tid 4 2024-09-12T23:40:57.871 INFO:tasks.rados.rados.0.smithi136.stdout:869: read oid 48 snap 77 2024-09-12T23:40:57.871 INFO:tasks.rados.rados.0.smithi136.stdout:869: expect deleted 2024-09-12T23:40:57.871 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 9 current snap is 78 2024-09-12T23:40:57.871 INFO:tasks.rados.rados.0.smithi136.stdout:868: finishing write tid 1 to smithi13667081-3 2024-09-12T23:40:57.871 INFO:tasks.rados.rados.0.smithi136.stdout:868: finishing write tid 2 to smithi13667081-3 2024-09-12T23:40:57.871 INFO:tasks.rados.rados.0.smithi136.stdout:870: seq_num 285 ranges {106496=778240,884736=475136,1359872=475136,1835008=655360} 2024-09-12T23:40:57.877 INFO:tasks.rados.rados.0.smithi136.stdout:870: writing smithi13667081-9 from 106496 to 884736 tid 1 2024-09-12T23:40:57.881 INFO:tasks.rados.rados.0.smithi136.stdout:870: writing smithi13667081-9 from 884736 to 1359872 tid 2 2024-09-12T23:40:57.884 INFO:tasks.rados.rados.0.smithi136.stdout:870: writing smithi13667081-9 from 1359872 to 1835008 tid 3 2024-09-12T23:40:57.890 INFO:tasks.rados.rados.0.smithi136.stdout:870: writing smithi13667081-9 from 1835008 to 2490368 tid 4 2024-09-12T23:40:57.890 INFO:tasks.rados.rados.0.smithi136.stdout:871: read oid 32 snap 77 2024-09-12T23:40:57.890 INFO:tasks.rados.rados.0.smithi136.stdout:871: expect (ObjNum 244 snap 60 seq_num 244) 2024-09-12T23:40:57.890 INFO:tasks.rados.rados.0.smithi136.stdout:872: rollback oid 23 current snap is 78 2024-09-12T23:40:57.890 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 23 to 75 2024-09-12T23:40:57.890 INFO:tasks.rados.rados.0.smithi136.stdout:868: finishing write tid 3 to smithi13667081-3 2024-09-12T23:40:57.890 INFO:tasks.rados.rados.0.smithi136.stdout:868: finishing write tid 4 to smithi13667081-3 2024-09-12T23:40:57.890 INFO:tasks.rados.rados.0.smithi136.stdout:868: finishing write tid 5 to smithi13667081-3 2024-09-12T23:40:57.890 INFO:tasks.rados.rados.0.smithi136.stdout:868: finishing write tid 6 to smithi13667081-3 2024-09-12T23:40:57.890 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 3 v 485 (ObjNum 284 snap 78 seq_num 284) dirty exists 2024-09-12T23:40:57.890 INFO:tasks.rados.rados.0.smithi136.stdout:868: left oid 3 (ObjNum 284 snap 78 seq_num 284) 2024-09-12T23:40:57.890 INFO:tasks.rados.rados.0.smithi136.stdout:870: finishing write tid 1 to smithi13667081-9 2024-09-12T23:40:57.891 INFO:tasks.rados.rados.0.smithi136.stdout:870: finishing write tid 2 to smithi13667081-9 2024-09-12T23:40:57.891 INFO:tasks.rados.rados.0.smithi136.stdout:868: done (4 left) 2024-09-12T23:40:57.891 INFO:tasks.rados.rados.0.smithi136.stdout:869: done (3 left) 2024-09-12T23:40:57.891 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 8 current snap is 78 2024-09-12T23:40:57.891 INFO:tasks.rados.rados.0.smithi136.stdout:870: finishing write tid 3 to smithi13667081-9 2024-09-12T23:40:57.891 INFO:tasks.rados.rados.0.smithi136.stdout:873: seq_num 286 ranges {0=794624,794624=761856,1556480=679936,2236416=163840} 2024-09-12T23:40:57.896 INFO:tasks.rados.rados.0.smithi136.stdout:873: writing smithi13667081-8 from 0 to 794624 tid 1 2024-09-12T23:40:57.901 INFO:tasks.rados.rados.0.smithi136.stdout:873: writing smithi13667081-8 from 794624 to 1556480 tid 2 2024-09-12T23:40:57.907 INFO:tasks.rados.rados.0.smithi136.stdout:873: writing smithi13667081-8 from 1556480 to 2236416 tid 3 2024-09-12T23:40:57.909 INFO:tasks.rados.rados.0.smithi136.stdout:873: writing smithi13667081-8 from 2236416 to 2400256 tid 4 2024-09-12T23:40:57.909 INFO:tasks.rados.rados.0.smithi136.stdout:874: copy_from oid 20 from oid 25 current snap is 78 2024-09-12T23:40:57.909 INFO:tasks.rados.rados.0.smithi136.stdout:872: finishing rollback tid 0 to smithi13667081-23 2024-09-12T23:40:57.909 INFO:tasks.rados.rados.0.smithi136.stdout:872: finishing rollback tid 1 to smithi13667081-23 2024-09-12T23:40:57.909 INFO:tasks.rados.rados.0.smithi136.stdout:870: finishing write tid 4 to smithi13667081-9 2024-09-12T23:40:57.909 INFO:tasks.rados.rados.0.smithi136.stdout:872: finishing rollback tid 2 to smithi13667081-23 2024-09-12T23:40:57.909 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 23 v 498 (ObjNum 257 snap 70 seq_num 257) dirty exists 2024-09-12T23:40:57.909 INFO:tasks.rados.rados.0.smithi136.stdout:870: finishing write tid 5 to smithi13667081-9 2024-09-12T23:40:57.909 INFO:tasks.rados.rados.0.smithi136.stdout:870: finishing write tid 6 to smithi13667081-9 2024-09-12T23:40:57.909 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 9 v 279 (ObjNum 285 snap 78 seq_num 285) dirty exists 2024-09-12T23:40:57.909 INFO:tasks.rados.rados.0.smithi136.stdout:870: left oid 9 (ObjNum 285 snap 78 seq_num 285) 2024-09-12T23:40:57.909 INFO:tasks.rados.rados.0.smithi136.stdout:870: done (4 left) 2024-09-12T23:40:57.910 INFO:tasks.rados.rados.0.smithi136.stdout:872: done (3 left) 2024-09-12T23:40:57.910 INFO:tasks.rados.rados.0.smithi136.stdout:875: snap_remove snap 76 2024-09-12T23:40:57.918 INFO:teuthology.orchestra.run.smithi136.stdout:true 2024-09-12T23:40:58.093 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:57 smithi181 bash[24776]: cluster 2024-09-12T23:40:56.612926+0000 mon.a (mon.0) 1334 : cluster [DBG] osdmap e221: 8 total, 8 up, 8 in 2024-09-12T23:40:58.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:57 smithi181 bash[24776]: audit 2024-09-12T23:40:56.709754+0000 mon.a (mon.0) 1335 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:58.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:57 smithi181 bash[24776]: audit 2024-09-12T23:40:56.720684+0000 mon.a (mon.0) 1336 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:58.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:57 smithi181 bash[24776]: audit 2024-09-12T23:40:57.005083+0000 mon.a (mon.0) 1337 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:58.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:57 smithi181 bash[24776]: audit 2024-09-12T23:40:57.015600+0000 mon.a (mon.0) 1338 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:58.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:57 smithi181 bash[24776]: audit 2024-09-12T23:40:57.429540+0000 mon.a (mon.0) 1339 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:58.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:57 smithi181 bash[24776]: audit 2024-09-12T23:40:57.438998+0000 mon.a (mon.0) 1340 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:40:58.209 INFO:teuthology.orchestra.run.smithi136.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-12T23:40:58.209 INFO:teuthology.orchestra.run.smithi136.stdout:alertmanager.a smithi136 *:9093,9094 running (25s) 1s ago 6m 13.5M - 0.25.0 c8568f914cd2 b5197f020eeb 2024-09-12T23:40:58.209 INFO:teuthology.orchestra.run.smithi136.stdout:grafana.a smithi181 *:3000 running (13s) 1s ago 5m 63.6M - 9.4.7 954c08fa6188 256c181be984 2024-09-12T23:40:58.209 INFO:teuthology.orchestra.run.smithi136.stdout:mds.a smithi136 running (7m) 1s ago 7m 13.6M - 17.2.7-1627-gbbdadfa3 214985049bdb 713b9795ad19 2024-09-12T23:40:58.209 INFO:teuthology.orchestra.run.smithi136.stdout:mds.b smithi181 running (7m) 1s ago 7m 16.4M - 17.2.7-1627-gbbdadfa3 214985049bdb ad13179f0868 2024-09-12T23:40:58.209 INFO:teuthology.orchestra.run.smithi136.stdout:mgr.x smithi181 *:8443,9283 running (67s) 1s ago 14m 480M - 18.2.4-837-g7a853b17 f21ba1eea74e 761bc101e333 2024-09-12T23:40:58.209 INFO:teuthology.orchestra.run.smithi136.stdout:mgr.y smithi136 *:8443,9283,8765 running (7s) 1s ago 17m 126M - 18.2.4-837-g7a853b17 f21ba1eea74e 272f10b42fc3 2024-09-12T23:40:58.210 INFO:teuthology.orchestra.run.smithi136.stdout:mon.a smithi136 running (17m) 1s ago 17m 55.4M 2048M 17.2.7-1627-gbbdadfa3 214985049bdb 0462f4f61475 2024-09-12T23:40:58.210 INFO:teuthology.orchestra.run.smithi136.stdout:mon.b smithi181 running (15m) 1s ago 15m 48.4M 2048M 17.2.7-1627-gbbdadfa3 214985049bdb 8b3d784af845 2024-09-12T23:40:58.210 INFO:teuthology.orchestra.run.smithi136.stdout:mon.c smithi136 running (15m) 1s ago 15m 46.5M 2048M 17.2.7-1627-gbbdadfa3 214985049bdb 647b028ea963 2024-09-12T23:40:58.210 INFO:teuthology.orchestra.run.smithi136.stdout:node-exporter.a smithi136 *:9100 running (31s) 1s ago 6m 7680k - 1.5.0 0da6a335fe13 ccfc37c96620 2024-09-12T23:40:58.210 INFO:teuthology.orchestra.run.smithi136.stdout:node-exporter.b smithi181 *:9100 running (18s) 1s ago 6m 3344k - 1.5.0 0da6a335fe13 80bf0e383230 2024-09-12T23:40:58.210 INFO:teuthology.orchestra.run.smithi136.stdout:osd.0 smithi136 running (14m) 1s ago 14m 126M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb cbc0b7df0115 2024-09-12T23:40:58.210 INFO:teuthology.orchestra.run.smithi136.stdout:osd.1 smithi136 running (13m) 1s ago 13m 138M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb c4daee7da836 2024-09-12T23:40:58.210 INFO:teuthology.orchestra.run.smithi136.stdout:osd.2 smithi136 running (12m) 1s ago 12m 145M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 4ab8fd79cdb4 2024-09-12T23:40:58.210 INFO:teuthology.orchestra.run.smithi136.stdout:osd.3 smithi136 running (11m) 1s ago 11m 117M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 9ceb14a27521 2024-09-12T23:40:58.210 INFO:teuthology.orchestra.run.smithi136.stdout:osd.4 smithi181 running (10m) 1s ago 10m 152M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb a35cdca79055 2024-09-12T23:40:58.210 INFO:teuthology.orchestra.run.smithi136.stdout:osd.5 smithi181 running (10m) 1s ago 10m 118M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 8f2e8d494fef 2024-09-12T23:40:58.210 INFO:teuthology.orchestra.run.smithi136.stdout:osd.6 smithi181 running (9m) 1s ago 9m 76.3M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 9c6bec406408 2024-09-12T23:40:58.210 INFO:teuthology.orchestra.run.smithi136.stdout:osd.7 smithi181 running (8m) 1s ago 8m 101M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb bc0632ae2145 2024-09-12T23:40:58.210 INFO:teuthology.orchestra.run.smithi136.stdout:prometheus.a smithi181 *:9095 running (20s) 1s ago 6m 38.1M - 2.43.0 a07b618ecd1d c44e669360fc 2024-09-12T23:40:58.251 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:57 smithi136 bash[68787]: debug 2024-09-12T23:40:57.756+0000 7f9828db4040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-12T23:40:58.251 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:57 smithi136 bash[68787]: debug 2024-09-12T23:40:57.840+0000 7f9828db4040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-12T23:40:58.505 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:58 smithi136 bash[68787]: debug 2024-09-12T23:40:58.252+0000 7f9828db4040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-12T23:40:58.505 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:58 smithi136 bash[68787]: debug 2024-09-12T23:40:58.336+0000 7f9828db4040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-12T23:40:58.505 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:58 smithi136 bash[68787]: [12/Sep/2024:23:40:58] ENGINE Bus STARTING 2024-09-12T23:40:58.505 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:58 smithi136 bash[68787]: CherryPy Checker: 2024-09-12T23:40:58.505 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:58 smithi136 bash[68787]: The Application mounted at '' has an empty config. 2024-09-12T23:40:58.505 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:58 smithi136 bash[68787]: [12/Sep/2024:23:40:58] ENGINE Serving on http://:::9283 2024-09-12T23:40:58.505 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:40:58 smithi136 bash[68787]: [12/Sep/2024:23:40:58] ENGINE Bus STARTED 2024-09-12T23:40:58.535 INFO:teuthology.orchestra.run.smithi136.stdout:{ 2024-09-12T23:40:58.535 INFO:teuthology.orchestra.run.smithi136.stdout: "mon": { 2024-09-12T23:40:58.535 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 3 2024-09-12T23:40:58.535 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:40:58.535 INFO:teuthology.orchestra.run.smithi136.stdout: "mgr": { 2024-09-12T23:40:58.535 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 1, 2024-09-12T23:40:58.535 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 1 2024-09-12T23:40:58.536 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:40:58.536 INFO:teuthology.orchestra.run.smithi136.stdout: "osd": { 2024-09-12T23:40:58.536 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 8 2024-09-12T23:40:58.536 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:40:58.536 INFO:teuthology.orchestra.run.smithi136.stdout: "mds": { 2024-09-12T23:40:58.536 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-12T23:40:58.536 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:40:58.536 INFO:teuthology.orchestra.run.smithi136.stdout: "overall": { 2024-09-12T23:40:58.536 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 14, 2024-09-12T23:40:58.536 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 1 2024-09-12T23:40:58.536 INFO:teuthology.orchestra.run.smithi136.stdout: } 2024-09-12T23:40:58.536 INFO:teuthology.orchestra.run.smithi136.stdout:} 2024-09-12T23:40:58.745 INFO:tasks.rados.rados.0.smithi136.stdout:875: done (3 left) 2024-09-12T23:40:58.746 INFO:tasks.rados.rados.0.smithi136.stdout:876: rollback oid 31 current snap is 78 2024-09-12T23:40:58.746 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 31 to 75 2024-09-12T23:40:58.746 INFO:tasks.rados.rados.0.smithi136.stdout:874: finishing copy_from to smithi13667081-20 2024-09-12T23:40:58.746 INFO:tasks.rados.rados.0.smithi136.stdout:874: got expected ENOENT (src dne) 2024-09-12T23:40:58.746 INFO:tasks.rados.rados.0.smithi136.stdout:874: finishing copy_from racing read to smithi13667081-20 2024-09-12T23:40:58.746 INFO:tasks.rados.rados.0.smithi136.stdout:871: expect (ObjNum 244 snap 60 seq_num 244) 2024-09-12T23:40:58.860 INFO:tasks.rados.rados.0.smithi136.stdout:873: finishing write tid 1 to smithi13667081-8 2024-09-12T23:40:58.861 INFO:tasks.rados.rados.0.smithi136.stdout:873: finishing write tid 2 to smithi13667081-8 2024-09-12T23:40:58.861 INFO:tasks.rados.rados.0.smithi136.stdout:873: finishing write tid 3 to smithi13667081-8 2024-09-12T23:40:58.861 INFO:tasks.rados.rados.0.smithi136.stdout:873: finishing write tid 4 to smithi13667081-8 2024-09-12T23:40:58.861 INFO:tasks.rados.rados.0.smithi136.stdout:873: finishing write tid 5 to smithi13667081-8 2024-09-12T23:40:58.861 INFO:tasks.rados.rados.0.smithi136.stdout:873: finishing write tid 6 to smithi13667081-8 2024-09-12T23:40:58.861 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 8 v 278 (ObjNum 286 snap 78 seq_num 286) dirty exists 2024-09-12T23:40:58.861 INFO:tasks.rados.rados.0.smithi136.stdout:873: left oid 8 (ObjNum 286 snap 78 seq_num 286) 2024-09-12T23:40:58.861 INFO:tasks.rados.rados.0.smithi136.stdout:876: finishing rollback tid 1 to smithi13667081-31 2024-09-12T23:40:58.861 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 31 v 0 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-09-12T23:40:58.862 INFO:tasks.rados.rados.0.smithi136.stdout:871: done (3 left) 2024-09-12T23:40:58.862 INFO:tasks.rados.rados.0.smithi136.stdout:873: done (2 left) 2024-09-12T23:40:58.862 INFO:tasks.rados.rados.0.smithi136.stdout:874: done (1 left) 2024-09-12T23:40:58.862 INFO:tasks.rados.rados.0.smithi136.stdout:876: done (0 left) 2024-09-12T23:40:58.862 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 16 current snap is 78 2024-09-12T23:40:58.862 INFO:tasks.rados.rados.0.smithi136.stdout:877: seq_num 287 ranges {1425408=8192} 2024-09-12T23:40:58.873 INFO:tasks.rados.rados.0.smithi136.stdout:877: writing smithi13667081-16 from 1425408 to 1433600 tid 1 2024-09-12T23:40:58.874 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 30 current snap is 78 2024-09-12T23:40:58.874 INFO:tasks.rados.rados.0.smithi136.stdout:878: seq_num 288 ranges {0=24576} 2024-09-12T23:40:58.874 INFO:tasks.rados.rados.0.smithi136.stdout:878: writing smithi13667081-30 from 0 to 24576 tid 1 2024-09-12T23:40:58.874 INFO:tasks.rados.rados.0.smithi136.stdout:879: snap_create 2024-09-12T23:40:58.879 INFO:tasks.rados.rados.0.smithi136.stdout:878: finishing write tid 1 to smithi13667081-30 2024-09-12T23:40:58.879 INFO:tasks.rados.rados.0.smithi136.stdout:877: finishing write tid 1 to smithi13667081-16 2024-09-12T23:40:58.879 INFO:tasks.rados.rados.0.smithi136.stdout:878: finishing write tid 2 to smithi13667081-30 2024-09-12T23:40:58.879 INFO:tasks.rados.rados.0.smithi136.stdout:877: finishing write tid 2 to smithi13667081-16 2024-09-12T23:40:58.881 INFO:tasks.rados.rados.0.smithi136.stdout:877: finishing write tid 3 to smithi13667081-16 2024-09-12T23:40:58.881 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 16 v 522 (ObjNum 287 snap 78 seq_num 287) dirty exists 2024-09-12T23:40:58.881 INFO:tasks.rados.rados.0.smithi136.stdout:877: left oid 16 (ObjNum 287 snap 78 seq_num 287) 2024-09-12T23:40:58.884 INFO:tasks.rados.rados.0.smithi136.stdout:878: finishing write tid 3 to smithi13667081-30 2024-09-12T23:40:58.884 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 30 v 205 (ObjNum 288 snap 78 seq_num 288) dirty exists 2024-09-12T23:40:58.884 INFO:tasks.rados.rados.0.smithi136.stdout:878: left oid 30 (ObjNum 288 snap 78 seq_num 288) 2024-09-12T23:40:58.986 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:40:58 smithi181 bash[58696]: ts=2024-09-12T23:40:58.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:40:58.987 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:58 smithi181 bash[24776]: cluster 2024-09-12T23:40:57.574008+0000 mgr.x (mgr.24517) 62 : cluster [DBG] pgmap v77: 121 pgs: 121 active+clean; 124 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 3.5 MiB/s wr, 31 op/s 2024-09-12T23:40:58.987 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:58 smithi181 bash[24776]: cluster 2024-09-12T23:40:57.725026+0000 mon.a (mon.0) 1341 : cluster [DBG] osdmap e222: 8 total, 8 up, 8 in 2024-09-12T23:40:58.987 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:58 smithi181 bash[24776]: audit 2024-09-12T23:40:57.908787+0000 mgr.x (mgr.24517) 63 : audit [DBG] from='client.15129 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:40:58.987 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:58 smithi181 bash[24776]: audit 2024-09-12T23:40:58.179150+0000 mgr.x (mgr.24517) 64 : audit [DBG] from='client.25055 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:40:58.987 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:58 smithi181 bash[24776]: cluster 2024-09-12T23:40:58.357872+0000 mon.a (mon.0) 1342 : cluster [DBG] Standby manager daemon y restarted 2024-09-12T23:40:58.987 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:58 smithi181 bash[24776]: cluster 2024-09-12T23:40:58.358003+0000 mon.a (mon.0) 1343 : cluster [DBG] Standby manager daemon y started 2024-09-12T23:40:58.987 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:58 smithi181 bash[24776]: audit 2024-09-12T23:40:58.361341+0000 mon.a (mon.0) 1344 : audit [DBG] from='mgr.? 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-12T23:40:58.987 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:58 smithi181 bash[24776]: audit 2024-09-12T23:40:58.361798+0000 mon.a (mon.0) 1345 : audit [DBG] from='mgr.? 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-12T23:40:58.988 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:58 smithi181 bash[24776]: audit 2024-09-12T23:40:58.363251+0000 mon.a (mon.0) 1346 : audit [DBG] from='mgr.? 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-12T23:40:58.988 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:58 smithi181 bash[24776]: audit 2024-09-12T23:40:58.363531+0000 mon.a (mon.0) 1347 : audit [DBG] from='mgr.? 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-12T23:40:58.988 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:58 smithi181 bash[24776]: audit 2024-09-12T23:40:58.540123+0000 mon.a (mon.0) 1348 : audit [DBG] from='client.? 172.21.15.136:0/4201067683' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:40:59.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:58 smithi136 bash[32189]: cluster 2024-09-12T23:40:57.574008+0000 mgr.x (mgr.24517) 62 : cluster [DBG] pgmap v77: 121 pgs: 121 active+clean; 124 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 3.5 MiB/s wr, 31 op/s 2024-09-12T23:40:59.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:58 smithi136 bash[32189]: cluster 2024-09-12T23:40:57.725026+0000 mon.a (mon.0) 1341 : cluster [DBG] osdmap e222: 8 total, 8 up, 8 in 2024-09-12T23:40:59.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:58 smithi136 bash[32189]: audit 2024-09-12T23:40:57.908787+0000 mgr.x (mgr.24517) 63 : audit [DBG] from='client.15129 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:40:59.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:58 smithi136 bash[32189]: audit 2024-09-12T23:40:58.179150+0000 mgr.x (mgr.24517) 64 : audit [DBG] from='client.25055 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:40:59.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:58 smithi136 bash[32189]: cluster 2024-09-12T23:40:58.357872+0000 mon.a (mon.0) 1342 : cluster [DBG] Standby manager daemon y restarted 2024-09-12T23:40:59.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:58 smithi136 bash[32189]: cluster 2024-09-12T23:40:58.358003+0000 mon.a (mon.0) 1343 : cluster [DBG] Standby manager daemon y started 2024-09-12T23:40:59.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:58 smithi136 bash[32189]: audit 2024-09-12T23:40:58.361341+0000 mon.a (mon.0) 1344 : audit [DBG] from='mgr.? 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-12T23:40:59.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:58 smithi136 bash[32189]: audit 2024-09-12T23:40:58.361798+0000 mon.a (mon.0) 1345 : audit [DBG] from='mgr.? 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-12T23:40:59.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:58 smithi136 bash[32189]: audit 2024-09-12T23:40:58.363251+0000 mon.a (mon.0) 1346 : audit [DBG] from='mgr.? 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-12T23:40:59.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:58 smithi136 bash[32189]: audit 2024-09-12T23:40:58.363531+0000 mon.a (mon.0) 1347 : audit [DBG] from='mgr.? 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-12T23:40:59.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:58 smithi136 bash[32189]: audit 2024-09-12T23:40:58.540123+0000 mon.a (mon.0) 1348 : audit [DBG] from='client.? 172.21.15.136:0/4201067683' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:40:59.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:58 smithi136 bash[22601]: cluster 2024-09-12T23:40:57.574008+0000 mgr.x (mgr.24517) 62 : cluster [DBG] pgmap v77: 121 pgs: 121 active+clean; 124 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 3.5 MiB/s wr, 31 op/s 2024-09-12T23:40:59.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:58 smithi136 bash[22601]: cluster 2024-09-12T23:40:57.725026+0000 mon.a (mon.0) 1341 : cluster [DBG] osdmap e222: 8 total, 8 up, 8 in 2024-09-12T23:40:59.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:58 smithi136 bash[22601]: audit 2024-09-12T23:40:57.908787+0000 mgr.x (mgr.24517) 63 : audit [DBG] from='client.15129 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:40:59.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:58 smithi136 bash[22601]: audit 2024-09-12T23:40:58.179150+0000 mgr.x (mgr.24517) 64 : audit [DBG] from='client.25055 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:40:59.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:58 smithi136 bash[22601]: cluster 2024-09-12T23:40:58.357872+0000 mon.a (mon.0) 1342 : cluster [DBG] Standby manager daemon y restarted 2024-09-12T23:40:59.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:58 smithi136 bash[22601]: cluster 2024-09-12T23:40:58.358003+0000 mon.a (mon.0) 1343 : cluster [DBG] Standby manager daemon y started 2024-09-12T23:40:59.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:58 smithi136 bash[22601]: audit 2024-09-12T23:40:58.361341+0000 mon.a (mon.0) 1344 : audit [DBG] from='mgr.? 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-12T23:40:59.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:58 smithi136 bash[22601]: audit 2024-09-12T23:40:58.361798+0000 mon.a (mon.0) 1345 : audit [DBG] from='mgr.? 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-12T23:40:59.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:58 smithi136 bash[22601]: audit 2024-09-12T23:40:58.363251+0000 mon.a (mon.0) 1346 : audit [DBG] from='mgr.? 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-12T23:40:59.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:58 smithi136 bash[22601]: audit 2024-09-12T23:40:58.363531+0000 mon.a (mon.0) 1347 : audit [DBG] from='mgr.? 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-12T23:40:59.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:58 smithi136 bash[22601]: audit 2024-09-12T23:40:58.540123+0000 mon.a (mon.0) 1348 : audit [DBG] from='client.? 172.21.15.136:0/4201067683' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:40:59.343 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:40:58 smithi181 bash[52367]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:40:58] "GET /metrics HTTP/1.1" 200 36138 "" "Prometheus/2.43.0" 2024-09-12T23:40:59.746 INFO:tasks.rados.rados.0.smithi136.stdout:877: done (2 left) 2024-09-12T23:40:59.747 INFO:tasks.rados.rados.0.smithi136.stdout:878: done (1 left) 2024-09-12T23:40:59.747 INFO:tasks.rados.rados.0.smithi136.stdout:879: done (0 left) 2024-09-12T23:40:59.747 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 45 current snap is 79 2024-09-12T23:40:59.747 INFO:tasks.rados.rados.0.smithi136.stdout:880: seq_num 289 ranges {4964352=40960} 2024-09-12T23:40:59.798 INFO:tasks.rados.rados.0.smithi136.stdout:880: writing smithi13667081-45 from 4964352 to 5005312 tid 1 2024-09-12T23:40:59.798 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 10 current snap is 79 2024-09-12T23:40:59.798 INFO:tasks.rados.rados.0.smithi136.stdout:881: seq_num 290 ranges {1761280=57344} 2024-09-12T23:40:59.814 INFO:tasks.rados.rados.0.smithi136.stdout:881: writing smithi13667081-10 from 1761280 to 1818624 tid 1 2024-09-12T23:40:59.814 INFO:tasks.rados.rados.0.smithi136.stdout:882: delete oid 7 current snap is 79 2024-09-12T23:40:59.814 INFO:tasks.rados.rados.0.smithi136.stdout:880: finishing write tid 1 to smithi13667081-45 2024-09-12T23:40:59.814 INFO:tasks.rados.rados.0.smithi136.stdout:880: finishing write tid 2 to smithi13667081-45 2024-09-12T23:40:59.815 INFO:tasks.rados.rados.0.smithi136.stdout:880: finishing write tid 3 to smithi13667081-45 2024-09-12T23:40:59.815 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 45 v 280 (ObjNum 289 snap 79 seq_num 289) dirty exists 2024-09-12T23:40:59.815 INFO:tasks.rados.rados.0.smithi136.stdout:880: left oid 45 (ObjNum 289 snap 79 seq_num 289) 2024-09-12T23:40:59.820 INFO:tasks.rados.rados.0.smithi136.stdout:881: finishing write tid 1 to smithi13667081-10 2024-09-12T23:40:59.821 INFO:tasks.rados.rados.0.smithi136.stdout:881: finishing write tid 2 to smithi13667081-10 2024-09-12T23:40:59.823 INFO:tasks.rados.rados.0.smithi136.stdout:881: finishing write tid 3 to smithi13667081-10 2024-09-12T23:40:59.823 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 10 v 207 (ObjNum 290 snap 79 seq_num 290) dirty exists 2024-09-12T23:40:59.823 INFO:tasks.rados.rados.0.smithi136.stdout:881: left oid 10 (ObjNum 290 snap 79 seq_num 290) 2024-09-12T23:40:59.824 INFO:tasks.rados.rados.0.smithi136.stdout:880: done (2 left) 2024-09-12T23:40:59.825 INFO:tasks.rados.rados.0.smithi136.stdout:881: done (1 left) 2024-09-12T23:40:59.825 INFO:tasks.rados.rados.0.smithi136.stdout:882: done (0 left) 2024-09-12T23:40:59.825 INFO:tasks.rados.rados.0.smithi136.stdout:883: rmattr oid 17 current snap is 79 2024-09-12T23:40:59.825 INFO:tasks.rados.rados.0.smithi136.stdout:883: done (0 left) 2024-09-12T23:40:59.825 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 30 current snap is 79 2024-09-12T23:40:59.826 INFO:tasks.rados.rados.0.smithi136.stdout:884: seq_num 292 ranges {24576=98304} 2024-09-12T23:40:59.826 INFO:tasks.rados.rados.0.smithi136.stdout:884: writing smithi13667081-30 from 24576 to 122880 tid 1 2024-09-12T23:40:59.827 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 27 current snap is 79 2024-09-12T23:40:59.827 INFO:tasks.rados.rados.0.smithi136.stdout:885: seq_num 293 ranges {4096000=114688} 2024-09-12T23:40:59.857 INFO:tasks.rados.rados.0.smithi136.stdout:885: writing smithi13667081-27 from 4096000 to 4210688 tid 1 2024-09-12T23:40:59.857 INFO:tasks.rados.rados.0.smithi136.stdout:886: delete oid 20 current snap is 79 2024-09-12T23:40:59.857 INFO:tasks.rados.rados.0.smithi136.stdout:884: finishing write tid 1 to smithi13667081-30 2024-09-12T23:40:59.857 INFO:tasks.rados.rados.0.smithi136.stdout:884: finishing write tid 2 to smithi13667081-30 2024-09-12T23:40:59.857 INFO:tasks.rados.rados.0.smithi136.stdout:884: finishing write tid 3 to smithi13667081-30 2024-09-12T23:40:59.857 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 30 v 209 (ObjNum 292 snap 79 seq_num 292) dirty exists 2024-09-12T23:40:59.857 INFO:tasks.rados.rados.0.smithi136.stdout:884: left oid 30 (ObjNum 292 snap 79 seq_num 292) 2024-09-12T23:40:59.861 INFO:tasks.rados.rados.0.smithi136.stdout:885: finishing write tid 1 to smithi13667081-27 2024-09-12T23:40:59.861 INFO:tasks.rados.rados.0.smithi136.stdout:885: finishing write tid 2 to smithi13667081-27 2024-09-12T23:40:59.863 INFO:tasks.rados.rados.0.smithi136.stdout:884: done (2 left) 2024-09-12T23:40:59.863 INFO:tasks.rados.rados.0.smithi136.stdout:886: done (1 left) 2024-09-12T23:40:59.863 INFO:tasks.rados.rados.0.smithi136.stdout:887: snap_create 2024-09-12T23:40:59.865 INFO:tasks.rados.rados.0.smithi136.stdout:885: finishing write tid 3 to smithi13667081-27 2024-09-12T23:40:59.865 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 27 v 505 (ObjNum 293 snap 79 seq_num 293) dirty exists 2024-09-12T23:40:59.865 INFO:tasks.rados.rados.0.smithi136.stdout:885: left oid 27 (ObjNum 293 snap 79 seq_num 293) 2024-09-12T23:41:00.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:59 smithi136 bash[22601]: cluster 2024-09-12T23:40:58.746657+0000 mon.a (mon.0) 1349 : cluster [DBG] mgrmap e30: x(active, since 51s), standbys: y 2024-09-12T23:41:00.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:40:59 smithi136 bash[22601]: cluster 2024-09-12T23:40:58.746852+0000 mon.a (mon.0) 1350 : cluster [DBG] osdmap e223: 8 total, 8 up, 8 in 2024-09-12T23:41:00.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:59 smithi136 bash[32189]: cluster 2024-09-12T23:40:58.746657+0000 mon.a (mon.0) 1349 : cluster [DBG] mgrmap e30: x(active, since 51s), standbys: y 2024-09-12T23:41:00.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:40:59 smithi136 bash[32189]: cluster 2024-09-12T23:40:58.746852+0000 mon.a (mon.0) 1350 : cluster [DBG] osdmap e223: 8 total, 8 up, 8 in 2024-09-12T23:41:00.093 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:59 smithi181 bash[24776]: cluster 2024-09-12T23:40:58.746657+0000 mon.a (mon.0) 1349 : cluster [DBG] mgrmap e30: x(active, since 51s), standbys: y 2024-09-12T23:41:00.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:40:59 smithi181 bash[24776]: cluster 2024-09-12T23:40:58.746852+0000 mon.a (mon.0) 1350 : cluster [DBG] osdmap e223: 8 total, 8 up, 8 in 2024-09-12T23:41:00.495 INFO:tasks.rados.rados.0.smithi136.stdout:885: done (1 left) 2024-09-12T23:41:00.495 INFO:tasks.rados.rados.0.smithi136.stdout:887: done (0 left) 2024-09-12T23:41:00.495 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 13 current snap is 80 2024-09-12T23:41:00.495 INFO:tasks.rados.rados.0.smithi136.stdout:888: seq_num 295 ranges {2285568=147456} 2024-09-12T23:41:00.514 INFO:tasks.rados.rados.0.smithi136.stdout:888: writing smithi13667081-13 from 2285568 to 2433024 tid 1 2024-09-12T23:41:00.514 INFO:tasks.rados.rados.0.smithi136.stdout:889: delete oid 50 current snap is 80 2024-09-12T23:41:00.524 INFO:tasks.rados.rados.0.smithi136.stdout:888: finishing write tid 1 to smithi13667081-13 2024-09-12T23:41:00.524 INFO:tasks.rados.rados.0.smithi136.stdout:888: finishing write tid 2 to smithi13667081-13 2024-09-12T23:41:00.526 INFO:tasks.rados.rados.0.smithi136.stdout:888: finishing write tid 3 to smithi13667081-13 2024-09-12T23:41:00.526 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 13 v 436 (ObjNum 295 snap 80 seq_num 295) dirty exists 2024-09-12T23:41:00.526 INFO:tasks.rados.rados.0.smithi136.stdout:888: left oid 13 (ObjNum 295 snap 80 seq_num 295) 2024-09-12T23:41:00.526 INFO:tasks.rados.rados.0.smithi136.stdout:888: done (1 left) 2024-09-12T23:41:00.526 INFO:tasks.rados.rados.0.smithi136.stdout:889: done (0 left) 2024-09-12T23:41:00.527 INFO:tasks.rados.rados.0.smithi136.stdout:890: rollback oid 45 current snap is 80 2024-09-12T23:41:00.527 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 45 to 73 2024-09-12T23:41:00.527 INFO:tasks.rados.rados.0.smithi136.stdout:891: snap_remove snap 77 2024-09-12T23:41:01.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:00 smithi136 bash[22601]: cluster 2024-09-12T23:40:59.574741+0000 mgr.x (mgr.24517) 65 : cluster [DBG] pgmap v80: 121 pgs: 121 active+clean; 139 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.4 MiB/s rd, 3.9 MiB/s wr, 22 op/s 2024-09-12T23:41:01.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:00 smithi136 bash[22601]: cluster 2024-09-12T23:40:59.751400+0000 mon.a (mon.0) 1351 : cluster [DBG] osdmap e224: 8 total, 8 up, 8 in 2024-09-12T23:41:01.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:00 smithi136 bash[22601]: cluster 2024-09-12T23:41:00.499704+0000 mon.a (mon.0) 1352 : cluster [DBG] osdmap e225: 8 total, 8 up, 8 in 2024-09-12T23:41:01.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:00 smithi136 bash[32189]: cluster 2024-09-12T23:40:59.574741+0000 mgr.x (mgr.24517) 65 : cluster [DBG] pgmap v80: 121 pgs: 121 active+clean; 139 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.4 MiB/s rd, 3.9 MiB/s wr, 22 op/s 2024-09-12T23:41:01.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:00 smithi136 bash[32189]: cluster 2024-09-12T23:40:59.751400+0000 mon.a (mon.0) 1351 : cluster [DBG] osdmap e224: 8 total, 8 up, 8 in 2024-09-12T23:41:01.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:00 smithi136 bash[32189]: cluster 2024-09-12T23:41:00.499704+0000 mon.a (mon.0) 1352 : cluster [DBG] osdmap e225: 8 total, 8 up, 8 in 2024-09-12T23:41:01.093 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:00 smithi181 bash[24776]: cluster 2024-09-12T23:40:59.574741+0000 mgr.x (mgr.24517) 65 : cluster [DBG] pgmap v80: 121 pgs: 121 active+clean; 139 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.4 MiB/s rd, 3.9 MiB/s wr, 22 op/s 2024-09-12T23:41:01.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:00 smithi181 bash[24776]: cluster 2024-09-12T23:40:59.751400+0000 mon.a (mon.0) 1351 : cluster [DBG] osdmap e224: 8 total, 8 up, 8 in 2024-09-12T23:41:01.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:00 smithi181 bash[24776]: cluster 2024-09-12T23:41:00.499704+0000 mon.a (mon.0) 1352 : cluster [DBG] osdmap e225: 8 total, 8 up, 8 in 2024-09-12T23:41:01.499 INFO:tasks.rados.rados.0.smithi136.stdout:891: done (1 left) 2024-09-12T23:41:01.499 INFO:tasks.rados.rados.0.smithi136.stdout:892: snap_create 2024-09-12T23:41:01.499 INFO:tasks.rados.rados.0.smithi136.stdout:890: finishing rollback tid 0 to smithi13667081-45 2024-09-12T23:41:01.499 INFO:tasks.rados.rados.0.smithi136.stdout:890: finishing rollback tid 1 to smithi13667081-45 2024-09-12T23:41:01.500 INFO:tasks.rados.rados.0.smithi136.stdout:890: finishing rollback tid 2 to smithi13667081-45 2024-09-12T23:41:01.500 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 45 v 283 (ObjNum 263 snap 71 seq_num 263) dirty exists 2024-09-12T23:41:02.506 INFO:tasks.rados.rados.0.smithi136.stdout:890: done (1 left) 2024-09-12T23:41:02.506 INFO:tasks.rados.rados.0.smithi136.stdout:892: done (0 left) 2024-09-12T23:41:02.507 INFO:tasks.rados.rados.0.smithi136.stdout:893: read oid 46 snap -1 2024-09-12T23:41:02.507 INFO:tasks.rados.rados.0.smithi136.stdout:893: expect (ObjNum 228 snap 56 seq_num 228) 2024-09-12T23:41:02.507 INFO:tasks.rados.rados.0.smithi136.stdout:894: read oid 12 snap -1 2024-09-12T23:41:02.507 INFO:tasks.rados.rados.0.smithi136.stdout:894: expect (ObjNum 264 snap 71 seq_num 264) 2024-09-12T23:41:02.507 INFO:tasks.rados.rados.0.smithi136.stdout:895: read oid 29 snap -1 2024-09-12T23:41:02.507 INFO:tasks.rados.rados.0.smithi136.stdout:895: expect (ObjNum 253 snap 69 seq_num 253) 2024-09-12T23:41:02.507 INFO:tasks.rados.rados.0.smithi136.stdout:896: snap_remove snap 79 2024-09-12T23:41:02.593 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:02 smithi181 bash[24776]: cluster 2024-09-12T23:41:01.503288+0000 mon.a (mon.0) 1353 : cluster [DBG] osdmap e226: 8 total, 8 up, 8 in 2024-09-12T23:41:02.594 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:02 smithi181 bash[58696]: ts=2024-09-12T23:41:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:41:02.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:02 smithi136 bash[22601]: cluster 2024-09-12T23:41:01.503288+0000 mon.a (mon.0) 1353 : cluster [DBG] osdmap e226: 8 total, 8 up, 8 in 2024-09-12T23:41:02.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:02 smithi136 bash[32189]: cluster 2024-09-12T23:41:01.503288+0000 mon.a (mon.0) 1353 : cluster [DBG] osdmap e226: 8 total, 8 up, 8 in 2024-09-12T23:41:03.519 INFO:tasks.rados.rados.0.smithi136.stdout:893: expect (ObjNum 228 snap 56 seq_num 228) 2024-09-12T23:41:03.596 INFO:tasks.rados.rados.0.smithi136.stdout:895: expect (ObjNum 253 snap 69 seq_num 253) 2024-09-12T23:41:03.650 INFO:tasks.rados.rados.0.smithi136.stdout:894: expect (ObjNum 264 snap 71 seq_num 264) 2024-09-12T23:41:03.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:03 smithi136 bash[22601]: cluster 2024-09-12T23:41:01.575464+0000 mgr.x (mgr.24517) 66 : cluster [DBG] pgmap v84: 121 pgs: 121 active+clean; 139 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 4.1 MiB/s wr, 23 op/s 2024-09-12T23:41:03.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:03 smithi136 bash[22601]: cluster 2024-09-12T23:41:02.510356+0000 mon.a (mon.0) 1354 : cluster [DBG] osdmap e227: 8 total, 8 up, 8 in 2024-09-12T23:41:03.755 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:03 smithi136 bash[32189]: cluster 2024-09-12T23:41:01.575464+0000 mgr.x (mgr.24517) 66 : cluster [DBG] pgmap v84: 121 pgs: 121 active+clean; 139 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 4.1 MiB/s wr, 23 op/s 2024-09-12T23:41:03.755 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:03 smithi136 bash[32189]: cluster 2024-09-12T23:41:02.510356+0000 mon.a (mon.0) 1354 : cluster [DBG] osdmap e227: 8 total, 8 up, 8 in 2024-09-12T23:41:03.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:03 smithi181 bash[24776]: cluster 2024-09-12T23:41:01.575464+0000 mgr.x (mgr.24517) 66 : cluster [DBG] pgmap v84: 121 pgs: 121 active+clean; 139 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 4.1 MiB/s wr, 23 op/s 2024-09-12T23:41:03.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:03 smithi181 bash[24776]: cluster 2024-09-12T23:41:02.510356+0000 mon.a (mon.0) 1354 : cluster [DBG] osdmap e227: 8 total, 8 up, 8 in 2024-09-12T23:41:03.844 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:03 smithi181 bash[58696]: ts=2024-09-12T23:41:03.512Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi136\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:41:03.846 INFO:tasks.rados.rados.0.smithi136.stdout:893: done (3 left) 2024-09-12T23:41:03.846 INFO:tasks.rados.rados.0.smithi136.stdout:894: done (2 left) 2024-09-12T23:41:03.846 INFO:tasks.rados.rados.0.smithi136.stdout:895: done (1 left) 2024-09-12T23:41:03.846 INFO:tasks.rados.rados.0.smithi136.stdout:896: done (0 left) 2024-09-12T23:41:03.846 INFO:tasks.rados.rados.0.smithi136.stdout:897: setattr oid 27 current snap is 81 2024-09-12T23:41:03.848 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 12 current snap is 81 2024-09-12T23:41:03.848 INFO:tasks.rados.rados.0.smithi136.stdout:898: seq_num 297 ranges {4669440=180224} 2024-09-12T23:41:03.876 INFO:tasks.rados.rados.0.smithi136.stdout:898: writing smithi13667081-12 from 4669440 to 4849664 tid 1 2024-09-12T23:41:03.876 INFO:tasks.rados.rados.0.smithi136.stdout:899: read oid 21 snap 78 2024-09-12T23:41:03.876 INFO:tasks.rados.rados.0.smithi136.stdout:899: expect (ObjNum 256 snap 70 seq_num 256) 2024-09-12T23:41:03.876 INFO:tasks.rados.rados.0.smithi136.stdout:900: read oid 35 snap 67 2024-09-12T23:41:03.876 INFO:tasks.rados.rados.0.smithi136.stdout:900: expect deleted 2024-09-12T23:41:03.876 INFO:tasks.rados.rados.0.smithi136.stdout:901: snap_create 2024-09-12T23:41:03.876 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 27 v 511 (ObjNum 293 snap 79 seq_num 293) dirty exists 2024-09-12T23:41:03.881 INFO:tasks.rados.rados.0.smithi136.stdout:898: finishing write tid 1 to smithi13667081-12 2024-09-12T23:41:03.881 INFO:tasks.rados.rados.0.smithi136.stdout:898: finishing write tid 2 to smithi13667081-12 2024-09-12T23:41:03.884 INFO:tasks.rados.rados.0.smithi136.stdout:898: finishing write tid 3 to smithi13667081-12 2024-09-12T23:41:03.884 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 12 v 488 (ObjNum 297 snap 81 seq_num 297) dirty exists 2024-09-12T23:41:03.884 INFO:tasks.rados.rados.0.smithi136.stdout:898: left oid 12 (ObjNum 297 snap 81 seq_num 297) 2024-09-12T23:41:03.895 INFO:tasks.rados.rados.0.smithi136.stdout:899: expect (ObjNum 256 snap 70 seq_num 256) 2024-09-12T23:41:04.526 INFO:tasks.rados.rados.0.smithi136.stdout:897: done (4 left) 2024-09-12T23:41:04.526 INFO:tasks.rados.rados.0.smithi136.stdout:898: done (3 left) 2024-09-12T23:41:04.526 INFO:tasks.rados.rados.0.smithi136.stdout:899: done (2 left) 2024-09-12T23:41:04.526 INFO:tasks.rados.rados.0.smithi136.stdout:900: done (1 left) 2024-09-12T23:41:04.526 INFO:tasks.rados.rados.0.smithi136.stdout:901: done (0 left) 2024-09-12T23:41:04.526 INFO:tasks.rados.rados.0.smithi136.stdout:902: rollback oid 40 current snap is 82 2024-09-12T23:41:04.526 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 40 to 81 2024-09-12T23:41:04.526 INFO:tasks.rados.rados.0.smithi136.stdout:903: rollback oid 16 current snap is 82 2024-09-12T23:41:04.527 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 16 to 67 2024-09-12T23:41:04.527 INFO:tasks.rados.rados.0.smithi136.stdout:904: rollback oid 26 current snap is 82 2024-09-12T23:41:04.527 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 26 to 67 2024-09-12T23:41:04.527 INFO:tasks.rados.rados.0.smithi136.stdout:905: read oid 18 snap -1 2024-09-12T23:41:04.527 INFO:tasks.rados.rados.0.smithi136.stdout:905: expect (ObjNum 259 snap 71 seq_num 259) 2024-09-12T23:41:04.527 INFO:tasks.rados.rados.0.smithi136.stdout:906: rollback oid 15 current snap is 82 2024-09-12T23:41:04.527 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 15 to 75 2024-09-12T23:41:04.527 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 9 current snap is 82 2024-09-12T23:41:04.527 INFO:tasks.rados.rados.0.smithi136.stdout:907: seq_num 298 ranges {2490368=196608} 2024-09-12T23:41:04.545 INFO:tasks.rados.rados.0.smithi136.stdout:907: writing smithi13667081-9 from 2490368 to 2686976 tid 1 2024-09-12T23:41:04.545 INFO:tasks.rados.rados.0.smithi136.stdout:908: read oid 36 snap -1 2024-09-12T23:41:04.545 INFO:tasks.rados.rados.0.smithi136.stdout:908: expect (ObjNum 256 snap 70 seq_num 256) 2024-09-12T23:41:04.545 INFO:tasks.rados.rados.0.smithi136.stdout:909: read oid 23 snap -1 2024-09-12T23:41:04.545 INFO:tasks.rados.rados.0.smithi136.stdout:909: expect (ObjNum 257 snap 70 seq_num 257) 2024-09-12T23:41:04.545 INFO:tasks.rados.rados.0.smithi136.stdout:910: delete oid 10 current snap is 82 2024-09-12T23:41:04.545 INFO:tasks.rados.rados.0.smithi136.stdout:906: finishing rollback tid 1 to smithi13667081-15 2024-09-12T23:41:04.545 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 15 v 0 (ObjNum 33 snap 0 seq_num 1) dirty dne 2024-09-12T23:41:04.545 INFO:tasks.rados.rados.0.smithi136.stdout:902: finishing rollback tid 0 to smithi13667081-40 2024-09-12T23:41:04.545 INFO:tasks.rados.rados.0.smithi136.stdout:902: finishing rollback tid 1 to smithi13667081-40 2024-09-12T23:41:04.546 INFO:tasks.rados.rados.0.smithi136.stdout:903: finishing rollback tid 0 to smithi13667081-16 2024-09-12T23:41:04.546 INFO:tasks.rados.rados.0.smithi136.stdout:902: finishing rollback tid 2 to smithi13667081-40 2024-09-12T23:41:04.546 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 40 v 515 (ObjNum 274 snap 74 seq_num 274) dirty exists 2024-09-12T23:41:04.546 INFO:tasks.rados.rados.0.smithi136.stdout:903: finishing rollback tid 1 to smithi13667081-16 2024-09-12T23:41:04.546 INFO:tasks.rados.rados.0.smithi136.stdout:903: finishing rollback tid 2 to smithi13667081-16 2024-09-12T23:41:04.546 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 16 v 528 (ObjNum 228 snap 56 seq_num 228) dirty exists 2024-09-12T23:41:04.549 INFO:tasks.rados.rados.0.smithi136.stdout:904: finishing rollback tid 0 to smithi13667081-26 2024-09-12T23:41:04.551 INFO:tasks.rados.rados.0.smithi136.stdout:907: finishing write tid 1 to smithi13667081-9 2024-09-12T23:41:04.552 INFO:tasks.rados.rados.0.smithi136.stdout:904: finishing rollback tid 1 to smithi13667081-26 2024-09-12T23:41:04.552 INFO:tasks.rados.rados.0.smithi136.stdout:904: finishing rollback tid 2 to smithi13667081-26 2024-09-12T23:41:04.552 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 26 v 379 (ObjNum 160 snap 42 seq_num 160) dirty exists 2024-09-12T23:41:04.555 INFO:tasks.rados.rados.0.smithi136.stdout:902: done (8 left) 2024-09-12T23:41:04.555 INFO:tasks.rados.rados.0.smithi136.stdout:903: done (7 left) 2024-09-12T23:41:04.556 INFO:tasks.rados.rados.0.smithi136.stdout:904: done (6 left) 2024-09-12T23:41:04.556 INFO:tasks.rados.rados.0.smithi136.stdout:906: done (5 left) 2024-09-12T23:41:04.556 INFO:tasks.rados.rados.0.smithi136.stdout:910: done (4 left) 2024-09-12T23:41:04.556 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 38 current snap is 82 2024-09-12T23:41:04.556 INFO:tasks.rados.rados.0.smithi136.stdout:911: seq_num 300 ranges {3710976=229376} 2024-09-12T23:41:04.579 INFO:tasks.rados.rados.0.smithi136.stdout:911: writing smithi13667081-38 from 3710976 to 3940352 tid 1 2024-09-12T23:41:04.579 INFO:tasks.rados.rados.0.smithi136.stdout:912: read oid 35 snap -1 2024-09-12T23:41:04.579 INFO:tasks.rados.rados.0.smithi136.stdout:912: expect deleted 2024-09-12T23:41:04.579 INFO:tasks.rados.rados.0.smithi136.stdout:913: delete oid 19 current snap is 82 2024-09-12T23:41:04.579 INFO:tasks.rados.rados.0.smithi136.stdout:907: finishing write tid 2 to smithi13667081-9 2024-09-12T23:41:04.579 INFO:tasks.rados.rados.0.smithi136.stdout:905: expect (ObjNum 259 snap 71 seq_num 259) 2024-09-12T23:41:04.651 INFO:tasks.rados.rados.0.smithi136.stdout:907: finishing write tid 3 to smithi13667081-9 2024-09-12T23:41:04.651 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 9 v 281 (ObjNum 298 snap 82 seq_num 298) dirty exists 2024-09-12T23:41:04.651 INFO:tasks.rados.rados.0.smithi136.stdout:907: left oid 9 (ObjNum 298 snap 82 seq_num 298) 2024-09-12T23:41:04.651 INFO:tasks.rados.rados.0.smithi136.stdout:908: expect (ObjNum 256 snap 70 seq_num 256) 2024-09-12T23:41:04.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:04 smithi181 bash[24776]: cluster 2024-09-12T23:41:03.523640+0000 mon.a (mon.0) 1355 : cluster [DBG] osdmap e228: 8 total, 8 up, 8 in 2024-09-12T23:41:04.844 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:04 smithi181 bash[58696]: ts=2024-09-12T23:41:04.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:41:04.844 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:04 smithi181 bash[58696]: ts=2024-09-12T23:41:04.456Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:41:04.846 INFO:tasks.rados.rados.0.smithi136.stdout:909: expect (ObjNum 257 snap 70 seq_num 257) 2024-09-12T23:41:05.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:04 smithi136 bash[22601]: cluster 2024-09-12T23:41:03.523640+0000 mon.a (mon.0) 1355 : cluster [DBG] osdmap e228: 8 total, 8 up, 8 in 2024-09-12T23:41:05.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:04 smithi136 bash[32189]: cluster 2024-09-12T23:41:03.523640+0000 mon.a (mon.0) 1355 : cluster [DBG] osdmap e228: 8 total, 8 up, 8 in 2024-09-12T23:41:05.297 INFO:tasks.rados.rados.0.smithi136.stdout:911: finishing write tid 1 to smithi13667081-38 2024-09-12T23:41:05.297 INFO:tasks.rados.rados.0.smithi136.stdout:911: finishing write tid 2 to smithi13667081-38 2024-09-12T23:41:05.297 INFO:tasks.rados.rados.0.smithi136.stdout:911: finishing write tid 3 to smithi13667081-38 2024-09-12T23:41:05.297 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 38 v 439 (ObjNum 300 snap 82 seq_num 300) dirty exists 2024-09-12T23:41:05.297 INFO:tasks.rados.rados.0.smithi136.stdout:911: left oid 38 (ObjNum 300 snap 82 seq_num 300) 2024-09-12T23:41:05.297 INFO:tasks.rados.rados.0.smithi136.stdout:905: done (6 left) 2024-09-12T23:41:05.297 INFO:tasks.rados.rados.0.smithi136.stdout:907: done (5 left) 2024-09-12T23:41:05.297 INFO:tasks.rados.rados.0.smithi136.stdout:908: done (4 left) 2024-09-12T23:41:05.297 INFO:tasks.rados.rados.0.smithi136.stdout:909: done (3 left) 2024-09-12T23:41:05.298 INFO:tasks.rados.rados.0.smithi136.stdout:911: done (2 left) 2024-09-12T23:41:05.298 INFO:tasks.rados.rados.0.smithi136.stdout:912: done (1 left) 2024-09-12T23:41:05.298 INFO:tasks.rados.rados.0.smithi136.stdout:913: done (0 left) 2024-09-12T23:41:05.298 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 33 current snap is 82 2024-09-12T23:41:05.298 INFO:tasks.rados.rados.0.smithi136.stdout:914: seq_num 302 ranges {1966080=262144} 2024-09-12T23:41:05.327 INFO:tasks.rados.rados.0.smithi136.stdout:914: writing smithi13667081-33 from 1966080 to 2228224 tid 1 2024-09-12T23:41:05.327 INFO:tasks.rados.rados.0.smithi136.stdout:915: rollback oid 47 current snap is 82 2024-09-12T23:41:05.327 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 47 to 75 2024-09-12T23:41:05.328 INFO:tasks.rados.rados.0.smithi136.stdout:916: setattr oid 44 current snap is 82 2024-09-12T23:41:05.330 INFO:tasks.rados.rados.0.smithi136.stdout:917: snap_remove snap 61 2024-09-12T23:41:05.537 INFO:tasks.rados.rados.0.smithi136.stdout:917: done (3 left) 2024-09-12T23:41:05.537 INFO:tasks.rados.rados.0.smithi136.stdout:918: snap_create 2024-09-12T23:41:05.537 INFO:tasks.rados.rados.0.smithi136.stdout:914: finishing write tid 1 to smithi13667081-33 2024-09-12T23:41:05.537 INFO:tasks.rados.rados.0.smithi136.stdout:914: finishing write tid 2 to smithi13667081-33 2024-09-12T23:41:05.537 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 44 v 529 (ObjNum 81 snap 0 seq_num 2877858670) dirty exists 2024-09-12T23:41:05.537 INFO:tasks.rados.rados.0.smithi136.stdout:914: finishing write tid 3 to smithi13667081-33 2024-09-12T23:41:05.537 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 33 v 406 (ObjNum 302 snap 82 seq_num 302) dirty exists 2024-09-12T23:41:05.538 INFO:tasks.rados.rados.0.smithi136.stdout:914: left oid 33 (ObjNum 302 snap 82 seq_num 302) 2024-09-12T23:41:05.538 INFO:tasks.rados.rados.0.smithi136.stdout:915: finishing rollback tid 0 to smithi13667081-47 2024-09-12T23:41:05.538 INFO:tasks.rados.rados.0.smithi136.stdout:915: finishing rollback tid 1 to smithi13667081-47 2024-09-12T23:41:05.538 INFO:tasks.rados.rados.0.smithi136.stdout:915: finishing rollback tid 2 to smithi13667081-47 2024-09-12T23:41:05.538 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 47 v 285 (ObjNum 231 snap 56 seq_num 231) dirty exists 2024-09-12T23:41:05.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:05 smithi181 bash[24776]: cluster 2024-09-12T23:41:03.576203+0000 mgr.x (mgr.24517) 67 : cluster [DBG] pgmap v87: 121 pgs: 121 active+clean; 139 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:41:05.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:05 smithi181 bash[24776]: cluster 2024-09-12T23:41:04.529731+0000 mon.a (mon.0) 1356 : cluster [DBG] osdmap e229: 8 total, 8 up, 8 in 2024-09-12T23:41:06.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:05 smithi136 bash[22601]: cluster 2024-09-12T23:41:03.576203+0000 mgr.x (mgr.24517) 67 : cluster [DBG] pgmap v87: 121 pgs: 121 active+clean; 139 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:41:06.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:05 smithi136 bash[22601]: cluster 2024-09-12T23:41:04.529731+0000 mon.a (mon.0) 1356 : cluster [DBG] osdmap e229: 8 total, 8 up, 8 in 2024-09-12T23:41:06.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:05 smithi136 bash[32189]: cluster 2024-09-12T23:41:03.576203+0000 mgr.x (mgr.24517) 67 : cluster [DBG] pgmap v87: 121 pgs: 121 active+clean; 139 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:41:06.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:05 smithi136 bash[32189]: cluster 2024-09-12T23:41:04.529731+0000 mon.a (mon.0) 1356 : cluster [DBG] osdmap e229: 8 total, 8 up, 8 in 2024-09-12T23:41:06.546 INFO:tasks.rados.rados.0.smithi136.stdout:914: done (3 left) 2024-09-12T23:41:06.546 INFO:tasks.rados.rados.0.smithi136.stdout:915: done (2 left) 2024-09-12T23:41:06.546 INFO:tasks.rados.rados.0.smithi136.stdout:916: done (1 left) 2024-09-12T23:41:06.546 INFO:tasks.rados.rados.0.smithi136.stdout:918: done (0 left) 2024-09-12T23:41:06.547 INFO:tasks.rados.rados.0.smithi136.stdout:919: rmattr oid 17 current snap is 83 2024-09-12T23:41:06.547 INFO:tasks.rados.rados.0.smithi136.stdout:919: done (0 left) 2024-09-12T23:41:06.547 INFO:tasks.rados.rados.0.smithi136.stdout:920: read oid 48 snap -1 2024-09-12T23:41:06.547 INFO:tasks.rados.rados.0.smithi136.stdout:920: expect deleted 2024-09-12T23:41:06.547 INFO:tasks.rados.rados.0.smithi136.stdout:921: rollback oid 9 current snap is 83 2024-09-12T23:41:06.547 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 9 to 81 2024-09-12T23:41:06.547 INFO:tasks.rados.rados.0.smithi136.stdout:922: delete oid 45 current snap is 83 2024-09-12T23:41:06.552 INFO:tasks.rados.rados.0.smithi136.stdout:920: done (2 left) 2024-09-12T23:41:06.553 INFO:tasks.rados.rados.0.smithi136.stdout:922: done (1 left) 2024-09-12T23:41:06.553 INFO:tasks.rados.rados.0.smithi136.stdout:923: snap_create 2024-09-12T23:41:06.553 INFO:tasks.rados.rados.0.smithi136.stdout:921: finishing rollback tid 0 to smithi13667081-9 2024-09-12T23:41:06.555 INFO:tasks.rados.rados.0.smithi136.stdout:921: finishing rollback tid 1 to smithi13667081-9 2024-09-12T23:41:06.555 INFO:tasks.rados.rados.0.smithi136.stdout:921: finishing rollback tid 2 to smithi13667081-9 2024-09-12T23:41:06.555 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 9 v 288 (ObjNum 285 snap 78 seq_num 285) dirty exists 2024-09-12T23:41:06.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:06 smithi136 bash[22601]: cluster 2024-09-12T23:41:05.541079+0000 mon.a (mon.0) 1357 : cluster [DBG] osdmap e230: 8 total, 8 up, 8 in 2024-09-12T23:41:06.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:06 smithi136 bash[22601]: cluster 2024-09-12T23:41:05.576902+0000 mgr.x (mgr.24517) 68 : cluster [DBG] pgmap v90: 121 pgs: 121 active+clean; 138 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 120 KiB/s wr, 9 op/s 2024-09-12T23:41:06.755 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:06 smithi136 bash[32189]: cluster 2024-09-12T23:41:05.541079+0000 mon.a (mon.0) 1357 : cluster [DBG] osdmap e230: 8 total, 8 up, 8 in 2024-09-12T23:41:06.755 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:06 smithi136 bash[32189]: cluster 2024-09-12T23:41:05.576902+0000 mgr.x (mgr.24517) 68 : cluster [DBG] pgmap v90: 121 pgs: 121 active+clean; 138 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 120 KiB/s wr, 9 op/s 2024-09-12T23:41:06.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:06 smithi181 bash[24776]: cluster 2024-09-12T23:41:05.541079+0000 mon.a (mon.0) 1357 : cluster [DBG] osdmap e230: 8 total, 8 up, 8 in 2024-09-12T23:41:06.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:06 smithi181 bash[24776]: cluster 2024-09-12T23:41:05.576902+0000 mgr.x (mgr.24517) 68 : cluster [DBG] pgmap v90: 121 pgs: 121 active+clean; 138 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 120 KiB/s wr, 9 op/s 2024-09-12T23:41:07.419 INFO:tasks.rados.rados.0.smithi136.stdout:921: done (1 left) 2024-09-12T23:41:07.419 INFO:tasks.rados.rados.0.smithi136.stdout:923: done (0 left) 2024-09-12T23:41:07.419 INFO:tasks.rados.rados.0.smithi136.stdout:924: setattr oid 39 current snap is 84 2024-09-12T23:41:07.431 INFO:tasks.rados.rados.0.smithi136.stdout:925: read oid 37 snap -1 2024-09-12T23:41:07.431 INFO:tasks.rados.rados.0.smithi136.stdout:925: expect (ObjNum 134 snap 35 seq_num 134) 2024-09-12T23:41:07.431 INFO:tasks.rados.rados.0.smithi136.stdout:926: read oid 25 snap -1 2024-09-12T23:41:07.431 INFO:tasks.rados.rados.0.smithi136.stdout:926: expect deleted 2024-09-12T23:41:07.431 INFO:tasks.rados.rados.0.smithi136.stdout:927: read oid 43 snap -1 2024-09-12T23:41:07.431 INFO:tasks.rados.rados.0.smithi136.stdout:927: expect deleted 2024-09-12T23:41:07.431 INFO:tasks.rados.rados.0.smithi136.stdout:928: rmattr oid 12 current snap is 84 2024-09-12T23:41:07.431 INFO:tasks.rados.rados.0.smithi136.stdout:928: done (4 left) 2024-09-12T23:41:07.431 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 9 current snap is 84 2024-09-12T23:41:07.432 INFO:tasks.rados.rados.0.smithi136.stdout:929: seq_num 304 ranges {2490368=294912} 2024-09-12T23:41:07.450 INFO:tasks.rados.rados.0.smithi136.stdout:929: writing smithi13667081-9 from 2490368 to 2785280 tid 1 2024-09-12T23:41:07.450 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 26 current snap is 84 2024-09-12T23:41:07.450 INFO:tasks.rados.rados.0.smithi136.stdout:930: seq_num 305 ranges {1957888=311296} 2024-09-12T23:41:07.463 INFO:tasks.rados.rados.0.smithi136.stdout:930: writing smithi13667081-26 from 1957888 to 2269184 tid 1 2024-09-12T23:41:07.464 INFO:tasks.rados.rados.0.smithi136.stdout:931: rmattr oid 31 current snap is 84 2024-09-12T23:41:07.464 INFO:tasks.rados.rados.0.smithi136.stdout:931: done (6 left) 2024-09-12T23:41:07.464 INFO:tasks.rados.rados.0.smithi136.stdout:932: snap_create 2024-09-12T23:41:07.464 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 39 v 126 (ObjNum 225 snap 56 seq_num 225) dirty exists 2024-09-12T23:41:07.464 INFO:tasks.rados.rados.0.smithi136.stdout:925: expect (ObjNum 134 snap 35 seq_num 134) 2024-09-12T23:41:07.703 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[32189]: cluster 2024-09-12T23:41:06.550964+0000 mon.a (mon.0) 1358 : cluster [DBG] osdmap e231: 8 total, 8 up, 8 in 2024-09-12T23:41:07.703 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[32189]: audit 2024-09-12T23:41:07.207455+0000 mon.a (mon.0) 1359 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:41:07.704 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[32189]: audit 2024-09-12T23:41:07.220370+0000 mon.a (mon.0) 1360 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:41:07.704 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[32189]: audit 2024-09-12T23:41:07.225783+0000 mon.b (mon.1) 105 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:41:07.704 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[32189]: audit 2024-09-12T23:41:07.227592+0000 mon.b (mon.1) 106 : audit [INF] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:41:07.704 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[32189]: audit 2024-09-12T23:41:07.237685+0000 mon.a (mon.0) 1361 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:41:07.704 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[32189]: audit 2024-09-12T23:41:07.399797+0000 mon.b (mon.1) 107 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:41:07.704 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[32189]: audit 2024-09-12T23:41:07.408756+0000 mon.b (mon.1) 108 : audit [INF] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-12T23:41:07.704 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[32189]: audit 2024-09-12T23:41:07.408846+0000 mon.a (mon.0) 1362 : audit [INF] from='mgr.24517 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-12T23:41:07.704 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[32189]: cluster 2024-09-12T23:41:07.409781+0000 mon.a (mon.0) 1363 : cluster [INF] Activating manager daemon y 2024-09-12T23:41:07.704 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[32189]: cluster 2024-09-12T23:41:07.422440+0000 mon.a (mon.0) 1364 : cluster [DBG] osdmap e232: 8 total, 8 up, 8 in 2024-09-12T23:41:07.704 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[32189]: audit 2024-09-12T23:41:07.422993+0000 mon.a (mon.0) 1365 : audit [INF] from='mgr.24517 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-12T23:41:07.704 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[32189]: cluster 2024-09-12T23:41:07.423140+0000 mon.a (mon.0) 1366 : cluster [DBG] mgrmap e31: y(active, starting, since 0.0133541s) 2024-09-12T23:41:07.704 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[32189]: audit 2024-09-12T23:41:07.427739+0000 mon.b (mon.1) 109 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:41:07.705 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[32189]: audit 2024-09-12T23:41:07.427986+0000 mon.b (mon.1) 110 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:41:07.705 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[32189]: audit 2024-09-12T23:41:07.428205+0000 mon.b (mon.1) 111 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:41:07.705 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[32189]: audit 2024-09-12T23:41:07.429215+0000 mon.b (mon.1) 112 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-12T23:41:07.705 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[32189]: audit 2024-09-12T23:41:07.429834+0000 mon.b (mon.1) 113 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-12T23:41:07.705 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[32189]: audit 2024-09-12T23:41:07.434518+0000 mon.b (mon.1) 114 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-12T23:41:07.705 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[32189]: audit 2024-09-12T23:41:07.434815+0000 mon.b (mon.1) 115 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-12T23:41:07.705 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[32189]: audit 2024-09-12T23:41:07.435119+0000 mon.b (mon.1) 116 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-12T23:41:07.705 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[32189]: audit 2024-09-12T23:41:07.435400+0000 mon.b (mon.1) 117 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-12T23:41:07.705 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[32189]: audit 2024-09-12T23:41:07.435698+0000 mon.b (mon.1) 118 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-12T23:41:07.705 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[32189]: audit 2024-09-12T23:41:07.435978+0000 mon.b (mon.1) 119 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-12T23:41:07.705 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[32189]: audit 2024-09-12T23:41:07.436275+0000 mon.b (mon.1) 120 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-12T23:41:07.705 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[32189]: audit 2024-09-12T23:41:07.436537+0000 mon.b (mon.1) 121 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-12T23:41:07.705 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[32189]: audit 2024-09-12T23:41:07.436813+0000 mon.b (mon.1) 122 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-12T23:41:07.705 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[32189]: audit 2024-09-12T23:41:07.437114+0000 mon.b (mon.1) 123 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-12T23:41:07.706 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[32189]: audit 2024-09-12T23:41:07.437444+0000 mon.b (mon.1) 124 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-12T23:41:07.706 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[32189]: audit 2024-09-12T23:41:07.438046+0000 mon.b (mon.1) 125 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-12T23:41:07.706 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[22601]: cluster 2024-09-12T23:41:06.550964+0000 mon.a (mon.0) 1358 : cluster [DBG] osdmap e231: 8 total, 8 up, 8 in 2024-09-12T23:41:07.706 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[22601]: audit 2024-09-12T23:41:07.207455+0000 mon.a (mon.0) 1359 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:41:07.706 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[22601]: audit 2024-09-12T23:41:07.220370+0000 mon.a (mon.0) 1360 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:41:07.706 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[22601]: audit 2024-09-12T23:41:07.225783+0000 mon.b (mon.1) 105 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:41:07.706 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[22601]: audit 2024-09-12T23:41:07.227592+0000 mon.b (mon.1) 106 : audit [INF] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:41:07.707 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[22601]: audit 2024-09-12T23:41:07.237685+0000 mon.a (mon.0) 1361 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:41:07.707 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[22601]: audit 2024-09-12T23:41:07.399797+0000 mon.b (mon.1) 107 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:41:07.707 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[22601]: audit 2024-09-12T23:41:07.408756+0000 mon.b (mon.1) 108 : audit [INF] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-12T23:41:07.707 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[22601]: audit 2024-09-12T23:41:07.408846+0000 mon.a (mon.0) 1362 : audit [INF] from='mgr.24517 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-12T23:41:07.707 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[22601]: cluster 2024-09-12T23:41:07.409781+0000 mon.a (mon.0) 1363 : cluster [INF] Activating manager daemon y 2024-09-12T23:41:07.707 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[22601]: cluster 2024-09-12T23:41:07.422440+0000 mon.a (mon.0) 1364 : cluster [DBG] osdmap e232: 8 total, 8 up, 8 in 2024-09-12T23:41:07.707 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[22601]: audit 2024-09-12T23:41:07.422993+0000 mon.a (mon.0) 1365 : audit [INF] from='mgr.24517 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-12T23:41:07.707 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[22601]: cluster 2024-09-12T23:41:07.423140+0000 mon.a (mon.0) 1366 : cluster [DBG] mgrmap e31: y(active, starting, since 0.0133541s) 2024-09-12T23:41:07.707 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[22601]: audit 2024-09-12T23:41:07.427739+0000 mon.b (mon.1) 109 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:41:07.707 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[22601]: audit 2024-09-12T23:41:07.427986+0000 mon.b (mon.1) 110 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:41:07.707 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[22601]: audit 2024-09-12T23:41:07.428205+0000 mon.b (mon.1) 111 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:41:07.707 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[22601]: audit 2024-09-12T23:41:07.429215+0000 mon.b (mon.1) 112 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-12T23:41:07.707 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[22601]: audit 2024-09-12T23:41:07.429834+0000 mon.b (mon.1) 113 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-12T23:41:07.707 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[22601]: audit 2024-09-12T23:41:07.434518+0000 mon.b (mon.1) 114 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-12T23:41:07.708 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[22601]: audit 2024-09-12T23:41:07.434815+0000 mon.b (mon.1) 115 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-12T23:41:07.708 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[22601]: audit 2024-09-12T23:41:07.435119+0000 mon.b (mon.1) 116 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-12T23:41:07.708 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[22601]: audit 2024-09-12T23:41:07.435400+0000 mon.b (mon.1) 117 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-12T23:41:07.708 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[22601]: audit 2024-09-12T23:41:07.435698+0000 mon.b (mon.1) 118 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-12T23:41:07.708 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[22601]: audit 2024-09-12T23:41:07.435978+0000 mon.b (mon.1) 119 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-12T23:41:07.708 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[22601]: audit 2024-09-12T23:41:07.436275+0000 mon.b (mon.1) 120 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-12T23:41:07.708 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[22601]: audit 2024-09-12T23:41:07.436537+0000 mon.b (mon.1) 121 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-12T23:41:07.708 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[22601]: audit 2024-09-12T23:41:07.436813+0000 mon.b (mon.1) 122 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-12T23:41:07.708 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[22601]: audit 2024-09-12T23:41:07.437114+0000 mon.b (mon.1) 123 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-12T23:41:07.708 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[22601]: audit 2024-09-12T23:41:07.437444+0000 mon.b (mon.1) 124 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-12T23:41:07.708 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[22601]: audit 2024-09-12T23:41:07.438046+0000 mon.b (mon.1) 125 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-12T23:41:07.708 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[68787]: [12/Sep/2024:23:41:07] ENGINE Bus STOPPING 2024-09-12T23:41:07.709 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[68787]: [12/Sep/2024:23:41:07] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-12T23:41:07.709 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[68787]: [12/Sep/2024:23:41:07] ENGINE Bus STOPPED 2024-09-12T23:41:07.709 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[68787]: [12/Sep/2024:23:41:07] ENGINE Bus STARTING 2024-09-12T23:41:07.762 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:07 smithi181 bash[52367]: debug 2024-09-12T23:41:07.418+0000 7fc99c85a640 -1 mgr handle_mgr_map I was active but no longer am 2024-09-12T23:41:07.762 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:07 smithi181 bash[52367]: ignoring --setuser ceph since I am not root 2024-09-12T23:41:07.762 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:07 smithi181 bash[52367]: ignoring --setgroup ceph since I am not root 2024-09-12T23:41:07.762 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:07 smithi181 bash[52367]: debug 2024-09-12T23:41:07.482+0000 7f7d06579640 1 -- 172.21.15.181:0/1718831323 <== mon.1 v2:172.21.15.181:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55d0fae70d00 con 0x55d0fae63400 2024-09-12T23:41:07.762 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:07 smithi181 bash[52367]: debug 2024-09-12T23:41:07.594+0000 7f7d08dd4040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-12T23:41:07.762 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:07 smithi181 bash[24776]: cluster 2024-09-12T23:41:06.550964+0000 mon.a (mon.0) 1358 : cluster [DBG] osdmap e231: 8 total, 8 up, 8 in 2024-09-12T23:41:07.763 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:07 smithi181 bash[24776]: audit 2024-09-12T23:41:07.207455+0000 mon.a (mon.0) 1359 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:41:07.763 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:07 smithi181 bash[24776]: audit 2024-09-12T23:41:07.220370+0000 mon.a (mon.0) 1360 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:41:07.763 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:07 smithi181 bash[24776]: audit 2024-09-12T23:41:07.225783+0000 mon.b (mon.1) 105 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:41:07.763 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:07 smithi181 bash[24776]: audit 2024-09-12T23:41:07.227592+0000 mon.b (mon.1) 106 : audit [INF] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:41:07.763 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:07 smithi181 bash[24776]: audit 2024-09-12T23:41:07.237685+0000 mon.a (mon.0) 1361 : audit [INF] from='mgr.24517 ' entity='mgr.x' 2024-09-12T23:41:07.763 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:07 smithi181 bash[24776]: audit 2024-09-12T23:41:07.399797+0000 mon.b (mon.1) 107 : audit [DBG] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:41:07.763 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:07 smithi181 bash[24776]: audit 2024-09-12T23:41:07.408756+0000 mon.b (mon.1) 108 : audit [INF] from='mgr.24517 172.21.15.181:0/1780410443' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-12T23:41:07.763 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:07 smithi181 bash[24776]: audit 2024-09-12T23:41:07.408846+0000 mon.a (mon.0) 1362 : audit [INF] from='mgr.24517 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-12T23:41:07.763 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:07 smithi181 bash[24776]: cluster 2024-09-12T23:41:07.409781+0000 mon.a (mon.0) 1363 : cluster [INF] Activating manager daemon y 2024-09-12T23:41:07.763 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:07 smithi181 bash[24776]: cluster 2024-09-12T23:41:07.422440+0000 mon.a (mon.0) 1364 : cluster [DBG] osdmap e232: 8 total, 8 up, 8 in 2024-09-12T23:41:07.763 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:07 smithi181 bash[24776]: audit 2024-09-12T23:41:07.422993+0000 mon.a (mon.0) 1365 : audit [INF] from='mgr.24517 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-12T23:41:07.763 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:07 smithi181 bash[24776]: cluster 2024-09-12T23:41:07.423140+0000 mon.a (mon.0) 1366 : cluster [DBG] mgrmap e31: y(active, starting, since 0.0133541s) 2024-09-12T23:41:07.763 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:07 smithi181 bash[24776]: audit 2024-09-12T23:41:07.427739+0000 mon.b (mon.1) 109 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:41:07.764 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:07 smithi181 bash[24776]: audit 2024-09-12T23:41:07.427986+0000 mon.b (mon.1) 110 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:41:07.764 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:07 smithi181 bash[24776]: audit 2024-09-12T23:41:07.428205+0000 mon.b (mon.1) 111 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:41:07.764 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:07 smithi181 bash[24776]: audit 2024-09-12T23:41:07.429215+0000 mon.b (mon.1) 112 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-12T23:41:07.764 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:07 smithi181 bash[24776]: audit 2024-09-12T23:41:07.429834+0000 mon.b (mon.1) 113 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-12T23:41:07.764 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:07 smithi181 bash[24776]: audit 2024-09-12T23:41:07.434518+0000 mon.b (mon.1) 114 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-12T23:41:07.764 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:07 smithi181 bash[24776]: audit 2024-09-12T23:41:07.434815+0000 mon.b (mon.1) 115 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-12T23:41:07.764 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:07 smithi181 bash[24776]: audit 2024-09-12T23:41:07.435119+0000 mon.b (mon.1) 116 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-12T23:41:07.764 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:07 smithi181 bash[24776]: audit 2024-09-12T23:41:07.435400+0000 mon.b (mon.1) 117 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-12T23:41:07.764 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:07 smithi181 bash[24776]: audit 2024-09-12T23:41:07.435698+0000 mon.b (mon.1) 118 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-12T23:41:07.764 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:07 smithi181 bash[24776]: audit 2024-09-12T23:41:07.435978+0000 mon.b (mon.1) 119 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-12T23:41:07.764 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:07 smithi181 bash[24776]: audit 2024-09-12T23:41:07.436275+0000 mon.b (mon.1) 120 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-12T23:41:07.764 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:07 smithi181 bash[24776]: audit 2024-09-12T23:41:07.436537+0000 mon.b (mon.1) 121 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-12T23:41:07.764 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:07 smithi181 bash[24776]: audit 2024-09-12T23:41:07.436813+0000 mon.b (mon.1) 122 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-12T23:41:07.765 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:07 smithi181 bash[24776]: audit 2024-09-12T23:41:07.437114+0000 mon.b (mon.1) 123 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-12T23:41:07.765 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:07 smithi181 bash[24776]: audit 2024-09-12T23:41:07.437444+0000 mon.b (mon.1) 124 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-12T23:41:07.765 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:07 smithi181 bash[24776]: audit 2024-09-12T23:41:07.438046+0000 mon.b (mon.1) 125 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-12T23:41:07.961 INFO:tasks.rados.rados.0.smithi136.stdout:929: finishing write tid 1 to smithi13667081-9 2024-09-12T23:41:07.961 INFO:tasks.rados.rados.0.smithi136.stdout:929: finishing write tid 2 to smithi13667081-9 2024-09-12T23:41:07.961 INFO:tasks.rados.rados.0.smithi136.stdout:929: finishing write tid 3 to smithi13667081-9 2024-09-12T23:41:07.961 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 9 v 290 (ObjNum 304 snap 84 seq_num 304) dirty exists 2024-09-12T23:41:07.961 INFO:tasks.rados.rados.0.smithi136.stdout:929: left oid 9 (ObjNum 304 snap 84 seq_num 304) 2024-09-12T23:41:07.961 INFO:tasks.rados.rados.0.smithi136.stdout:930: finishing write tid 1 to smithi13667081-26 2024-09-12T23:41:07.961 INFO:tasks.rados.rados.0.smithi136.stdout:930: finishing write tid 2 to smithi13667081-26 2024-09-12T23:41:07.961 INFO:tasks.rados.rados.0.smithi136.stdout:930: finishing write tid 3 to smithi13667081-26 2024-09-12T23:41:07.962 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 26 v 381 (ObjNum 305 snap 84 seq_num 305) dirty exists 2024-09-12T23:41:07.962 INFO:tasks.rados.rados.0.smithi136.stdout:930: left oid 26 (ObjNum 305 snap 84 seq_num 305) 2024-09-12T23:41:08.005 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[68787]: debug 2024-09-12T23:41:07.703+0000 7f9792968640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-12T23:41:08.005 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[68787]: debug 2024-09-12T23:41:07.703+0000 7f9792968640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-12T23:41:08.005 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[68787]: debug 2024-09-12T23:41:07.703+0000 7f9792968640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-12T23:41:08.005 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[68787]: debug 2024-09-12T23:41:07.703+0000 7f9792968640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-12T23:41:08.005 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[68787]: debug 2024-09-12T23:41:07.703+0000 7f9792968640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-12T23:41:08.005 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[68787]: debug 2024-09-12T23:41:07.711+0000 7f978f962640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-12T23:41:08.005 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[68787]: debug 2024-09-12T23:41:07.711+0000 7f978f962640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-12T23:41:08.006 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[68787]: debug 2024-09-12T23:41:07.711+0000 7f978f962640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-12T23:41:08.006 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[68787]: debug 2024-09-12T23:41:07.711+0000 7f978f962640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-12T23:41:08.006 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[68787]: debug 2024-09-12T23:41:07.711+0000 7f978f962640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-12T23:41:08.006 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[68787]: [12/Sep/2024:23:41:07] ENGINE Serving on http://:::9283 2024-09-12T23:41:08.006 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:41:07 smithi136 bash[68787]: [12/Sep/2024:23:41:07] ENGINE Bus STARTED 2024-09-12T23:41:08.414 INFO:tasks.rados.rados.0.smithi136.stdout:924: done (6 left) 2024-09-12T23:41:08.414 INFO:tasks.rados.rados.0.smithi136.stdout:925: done (5 left) 2024-09-12T23:41:08.414 INFO:tasks.rados.rados.0.smithi136.stdout:926: done (4 left) 2024-09-12T23:41:08.414 INFO:tasks.rados.rados.0.smithi136.stdout:927: done (3 left) 2024-09-12T23:41:08.414 INFO:tasks.rados.rados.0.smithi136.stdout:929: done (2 left) 2024-09-12T23:41:08.414 INFO:tasks.rados.rados.0.smithi136.stdout:930: done (1 left) 2024-09-12T23:41:08.414 INFO:tasks.rados.rados.0.smithi136.stdout:932: done (0 left) 2024-09-12T23:41:08.414 INFO:tasks.rados.rados.0.smithi136.stdout:933: snap_create 2024-09-12T23:41:08.813 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:08 smithi181 bash[58696]: ts=2024-09-12T23:41:08.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:41:08.814 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:08 smithi181 bash[24776]: cluster 2024-09-12T23:41:07.584469+0000 mon.a (mon.0) 1367 : cluster [INF] Manager daemon y is now available 2024-09-12T23:41:08.814 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:08 smithi181 bash[24776]: audit 2024-09-12T23:41:07.605224+0000 mon.b (mon.1) 126 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:41:08.814 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:08 smithi181 bash[24776]: audit 2024-09-12T23:41:07.642306+0000 mon.b (mon.1) 127 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:41:08.814 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:08 smithi181 bash[24776]: audit 2024-09-12T23:41:07.657720+0000 mon.a (mon.0) 1368 : audit [INF] from='mgr.15123 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-12T23:41:08.814 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:08 smithi181 bash[24776]: audit 2024-09-12T23:41:07.658010+0000 mon.b (mon.1) 128 : audit [INF] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-12T23:41:08.814 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:08 smithi181 bash[24776]: audit 2024-09-12T23:41:07.687913+0000 mon.a (mon.0) 1369 : audit [INF] from='mgr.15123 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-12T23:41:08.814 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:08 smithi181 bash[24776]: audit 2024-09-12T23:41:07.688183+0000 mon.b (mon.1) 129 : audit [INF] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-12T23:41:08.814 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:08 smithi181 bash[24776]: cluster 2024-09-12T23:41:08.418910+0000 mon.a (mon.0) 1370 : cluster [DBG] osdmap e233: 8 total, 8 up, 8 in 2024-09-12T23:41:09.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:08 smithi136 bash[32189]: cluster 2024-09-12T23:41:07.584469+0000 mon.a (mon.0) 1367 : cluster [INF] Manager daemon y is now available 2024-09-12T23:41:09.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:08 smithi136 bash[32189]: audit 2024-09-12T23:41:07.605224+0000 mon.b (mon.1) 126 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:41:09.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:08 smithi136 bash[32189]: audit 2024-09-12T23:41:07.642306+0000 mon.b (mon.1) 127 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:41:09.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:08 smithi136 bash[32189]: audit 2024-09-12T23:41:07.657720+0000 mon.a (mon.0) 1368 : audit [INF] from='mgr.15123 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-12T23:41:09.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:08 smithi136 bash[32189]: audit 2024-09-12T23:41:07.658010+0000 mon.b (mon.1) 128 : audit [INF] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-12T23:41:09.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:08 smithi136 bash[32189]: audit 2024-09-12T23:41:07.687913+0000 mon.a (mon.0) 1369 : audit [INF] from='mgr.15123 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-12T23:41:09.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:08 smithi136 bash[32189]: audit 2024-09-12T23:41:07.688183+0000 mon.b (mon.1) 129 : audit [INF] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-12T23:41:09.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:08 smithi136 bash[32189]: cluster 2024-09-12T23:41:08.418910+0000 mon.a (mon.0) 1370 : cluster [DBG] osdmap e233: 8 total, 8 up, 8 in 2024-09-12T23:41:09.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:08 smithi136 bash[22601]: cluster 2024-09-12T23:41:07.584469+0000 mon.a (mon.0) 1367 : cluster [INF] Manager daemon y is now available 2024-09-12T23:41:09.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:08 smithi136 bash[22601]: audit 2024-09-12T23:41:07.605224+0000 mon.b (mon.1) 126 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:41:09.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:08 smithi136 bash[22601]: audit 2024-09-12T23:41:07.642306+0000 mon.b (mon.1) 127 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:41:09.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:08 smithi136 bash[22601]: audit 2024-09-12T23:41:07.657720+0000 mon.a (mon.0) 1368 : audit [INF] from='mgr.15123 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-12T23:41:09.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:08 smithi136 bash[22601]: audit 2024-09-12T23:41:07.658010+0000 mon.b (mon.1) 128 : audit [INF] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-12T23:41:09.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:08 smithi136 bash[22601]: audit 2024-09-12T23:41:07.687913+0000 mon.a (mon.0) 1369 : audit [INF] from='mgr.15123 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-12T23:41:09.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:08 smithi136 bash[22601]: audit 2024-09-12T23:41:07.688183+0000 mon.b (mon.1) 129 : audit [INF] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-12T23:41:09.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:08 smithi136 bash[22601]: cluster 2024-09-12T23:41:08.418910+0000 mon.a (mon.0) 1370 : cluster [DBG] osdmap e233: 8 total, 8 up, 8 in 2024-09-12T23:41:09.093 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:08 smithi181 bash[52367]: debug 2024-09-12T23:41:08.814+0000 7f7d08dd4040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-12T23:41:09.093 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:08 smithi181 bash[52367]: debug 2024-09-12T23:41:08.878+0000 7f7d08dd4040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-12T23:41:09.093 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:09 smithi181 bash[52367]: debug 2024-09-12T23:41:09.030+0000 7f7d08dd4040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-12T23:41:09.416 INFO:tasks.rados.rados.0.smithi136.stdout:933: done (0 left) 2024-09-12T23:41:09.416 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 17 current snap is 86 2024-09-12T23:41:09.416 INFO:tasks.rados.rados.0.smithi136.stdout:934: seq_num 306 ranges {352256=327680} 2024-09-12T23:41:09.421 INFO:tasks.rados.rados.0.smithi136.stdout:934: writing smithi13667081-17 from 352256 to 679936 tid 1 2024-09-12T23:41:09.421 INFO:tasks.rados.rados.0.smithi136.stdout:935: delete oid 15 current snap is 86 2024-09-12T23:41:09.426 INFO:tasks.rados.rados.0.smithi136.stdout:935: done (1 left) 2024-09-12T23:41:09.426 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 27 current snap is 86 2024-09-12T23:41:09.426 INFO:tasks.rados.rados.0.smithi136.stdout:936: seq_num 308 ranges {4210688=360448} 2024-09-12T23:41:09.455 INFO:tasks.rados.rados.0.smithi136.stdout:936: writing smithi13667081-27 from 4210688 to 4571136 tid 1 2024-09-12T23:41:09.455 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 10 current snap is 86 2024-09-12T23:41:09.455 INFO:tasks.rados.rados.0.smithi136.stdout:937: seq_num 309 ranges {0=376832} 2024-09-12T23:41:09.460 INFO:tasks.rados.rados.0.smithi136.stdout:937: writing smithi13667081-10 from 0 to 376832 tid 1 2024-09-12T23:41:09.461 INFO:tasks.rados.rados.0.smithi136.stdout:938: delete oid 32 current snap is 86 2024-09-12T23:41:09.461 INFO:tasks.rados.rados.0.smithi136.stdout:934: finishing write tid 1 to smithi13667081-17 2024-09-12T23:41:09.461 INFO:tasks.rados.rados.0.smithi136.stdout:934: finishing write tid 2 to smithi13667081-17 2024-09-12T23:41:09.461 INFO:tasks.rados.rados.0.smithi136.stdout:934: finishing write tid 3 to smithi13667081-17 2024-09-12T23:41:09.461 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 17 v 526 (ObjNum 306 snap 86 seq_num 306) dirty exists 2024-09-12T23:41:09.461 INFO:tasks.rados.rados.0.smithi136.stdout:934: left oid 17 (ObjNum 306 snap 86 seq_num 306) 2024-09-12T23:41:09.461 INFO:tasks.rados.rados.0.smithi136.stdout:936: finishing write tid 1 to smithi13667081-27 2024-09-12T23:41:09.461 INFO:tasks.rados.rados.0.smithi136.stdout:936: finishing write tid 2 to smithi13667081-27 2024-09-12T23:41:09.464 INFO:tasks.rados.rados.0.smithi136.stdout:936: finishing write tid 3 to smithi13667081-27 2024-09-12T23:41:09.464 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 27 v 529 (ObjNum 308 snap 86 seq_num 308) dirty exists 2024-09-12T23:41:09.465 INFO:tasks.rados.rados.0.smithi136.stdout:936: left oid 27 (ObjNum 308 snap 86 seq_num 308) 2024-09-12T23:41:09.465 INFO:tasks.rados.rados.0.smithi136.stdout:937: finishing write tid 1 to smithi13667081-10 2024-09-12T23:41:09.466 INFO:tasks.rados.rados.0.smithi136.stdout:934: done (3 left) 2024-09-12T23:41:09.466 INFO:tasks.rados.rados.0.smithi136.stdout:936: done (2 left) 2024-09-12T23:41:09.466 INFO:tasks.rados.rados.0.smithi136.stdout:938: done (1 left) 2024-09-12T23:41:09.466 INFO:tasks.rados.rados.0.smithi136.stdout:939: read oid 41 snap -1 2024-09-12T23:41:09.466 INFO:tasks.rados.rados.0.smithi136.stdout:939: expect (ObjNum 262 snap 71 seq_num 262) 2024-09-12T23:41:09.466 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 29 current snap is 86 2024-09-12T23:41:09.466 INFO:tasks.rados.rados.0.smithi136.stdout:940: seq_num 311 ranges {1867776=417792} 2024-09-12T23:41:09.482 INFO:tasks.rados.rados.0.smithi136.stdout:940: writing smithi13667081-29 from 1867776 to 2285568 tid 1 2024-09-12T23:41:09.483 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 39 current snap is 86 2024-09-12T23:41:09.483 INFO:tasks.rados.rados.0.smithi136.stdout:941: seq_num 312 ranges {1376256=434176} 2024-09-12T23:41:09.497 INFO:tasks.rados.rados.0.smithi136.stdout:941: writing smithi13667081-39 from 1376256 to 1810432 tid 1 2024-09-12T23:41:09.497 INFO:tasks.rados.rados.0.smithi136.stdout:942: delete oid 46 current snap is 86 2024-09-12T23:41:09.497 INFO:tasks.rados.rados.0.smithi136.stdout:937: finishing write tid 2 to smithi13667081-10 2024-09-12T23:41:09.497 INFO:tasks.rados.rados.0.smithi136.stdout:937: finishing write tid 3 to smithi13667081-10 2024-09-12T23:41:09.497 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 10 v 212 (ObjNum 309 snap 86 seq_num 309) dirty exists 2024-09-12T23:41:09.497 INFO:tasks.rados.rados.0.smithi136.stdout:937: left oid 10 (ObjNum 309 snap 86 seq_num 309) 2024-09-12T23:41:09.497 INFO:tasks.rados.rados.0.smithi136.stdout:939: expect (ObjNum 262 snap 71 seq_num 262) 2024-09-12T23:41:09.701 INFO:tasks.rados.rados.0.smithi136.stdout:940: finishing write tid 1 to smithi13667081-29 2024-09-12T23:41:09.701 INFO:tasks.rados.rados.0.smithi136.stdout:940: finishing write tid 2 to smithi13667081-29 2024-09-12T23:41:09.701 INFO:tasks.rados.rados.0.smithi136.stdout:937: done (4 left) 2024-09-12T23:41:09.701 INFO:tasks.rados.rados.0.smithi136.stdout:939: done (3 left) 2024-09-12T23:41:09.701 INFO:tasks.rados.rados.0.smithi136.stdout:942: done (2 left) 2024-09-12T23:41:09.701 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 10 current snap is 86 2024-09-12T23:41:09.701 INFO:tasks.rados.rados.0.smithi136.stdout:940: finishing write tid 3 to smithi13667081-29 2024-09-12T23:41:09.701 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 29 v 441 (ObjNum 311 snap 86 seq_num 311) dirty exists 2024-09-12T23:41:09.701 INFO:tasks.rados.rados.0.smithi136.stdout:940: left oid 29 (ObjNum 311 snap 86 seq_num 311) 2024-09-12T23:41:09.702 INFO:tasks.rados.rados.0.smithi136.stdout:943: seq_num 314 ranges {376832=466944} 2024-09-12T23:41:09.705 INFO:tasks.rados.rados.0.smithi136.stdout:943: writing smithi13667081-10 from 376832 to 843776 tid 1 2024-09-12T23:41:09.706 INFO:tasks.rados.rados.0.smithi136.stdout:940: done (2 left) 2024-09-12T23:41:09.706 INFO:tasks.rados.rados.0.smithi136.stdout:944: copy_from oid 35 from oid 41 current snap is 86 2024-09-12T23:41:09.706 INFO:tasks.rados.rados.0.smithi136.stdout:941: finishing write tid 1 to smithi13667081-39 2024-09-12T23:41:09.706 INFO:tasks.rados.rados.0.smithi136.stdout:941: finishing write tid 2 to smithi13667081-39 2024-09-12T23:41:09.706 INFO:tasks.rados.rados.0.smithi136.stdout:941: finishing write tid 3 to smithi13667081-39 2024-09-12T23:41:09.706 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 39 v 129 (ObjNum 312 snap 86 seq_num 312) dirty exists 2024-09-12T23:41:09.706 INFO:tasks.rados.rados.0.smithi136.stdout:941: left oid 39 (ObjNum 312 snap 86 seq_num 312) 2024-09-12T23:41:09.706 INFO:tasks.rados.rados.0.smithi136.stdout:941: done (2 left) 2024-09-12T23:41:09.706 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 50 current snap is 86 2024-09-12T23:41:09.706 INFO:tasks.rados.rados.0.smithi136.stdout:945: seq_num 315 ranges {0=483328} 2024-09-12T23:41:09.708 INFO:tasks.rados.rados.0.smithi136.stdout:945: writing smithi13667081-50 from 0 to 483328 tid 1 2024-09-12T23:41:09.708 INFO:tasks.rados.rados.0.smithi136.stdout:946: delete oid 19 current snap is 86 2024-09-12T23:41:09.711 INFO:tasks.rados.rados.0.smithi136.stdout:943: finishing write tid 1 to smithi13667081-10 2024-09-12T23:41:09.711 INFO:tasks.rados.rados.0.smithi136.stdout:946: done (3 left) 2024-09-12T23:41:09.711 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 23 current snap is 86 2024-09-12T23:41:09.711 INFO:tasks.rados.rados.0.smithi136.stdout:947: seq_num 317 ranges {8126464=516096} 2024-09-12T23:41:09.768 INFO:tasks.rados.rados.0.smithi136.stdout:947: writing smithi13667081-23 from 8126464 to 8642560 tid 1 2024-09-12T23:41:09.768 INFO:tasks.rados.rados.0.smithi136.stdout:948: snap_create 2024-09-12T23:41:09.768 INFO:tasks.rados.rados.0.smithi136.stdout:943: finishing write tid 2 to smithi13667081-10 2024-09-12T23:41:09.768 INFO:tasks.rados.rados.0.smithi136.stdout:943: finishing write tid 3 to smithi13667081-10 2024-09-12T23:41:09.768 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 10 v 214 (ObjNum 314 snap 86 seq_num 314) dirty exists 2024-09-12T23:41:09.768 INFO:tasks.rados.rados.0.smithi136.stdout:943: left oid 10 (ObjNum 314 snap 86 seq_num 314) 2024-09-12T23:41:09.768 INFO:tasks.rados.rados.0.smithi136.stdout:945: finishing write tid 1 to smithi13667081-50 2024-09-12T23:41:09.768 INFO:tasks.rados.rados.0.smithi136.stdout:945: finishing write tid 2 to smithi13667081-50 2024-09-12T23:41:09.769 INFO:tasks.rados.rados.0.smithi136.stdout:944: finishing copy_from racing read to smithi13667081-35 2024-09-12T23:41:09.769 INFO:tasks.rados.rados.0.smithi136.stdout:945: finishing write tid 3 to smithi13667081-50 2024-09-12T23:41:09.769 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 50 v 443 (ObjNum 315 snap 86 seq_num 315) dirty exists 2024-09-12T23:41:09.769 INFO:tasks.rados.rados.0.smithi136.stdout:945: left oid 50 (ObjNum 315 snap 86 seq_num 315) 2024-09-12T23:41:09.769 INFO:tasks.rados.rados.0.smithi136.stdout:944: finishing copy_from to smithi13667081-35 2024-09-12T23:41:09.769 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 35 v 339 (ObjNum 262 snap 71 seq_num 262) dirty exists 2024-09-12T23:41:09.772 INFO:tasks.rados.rados.0.smithi136.stdout:947: finishing write tid 1 to smithi13667081-23 2024-09-12T23:41:09.772 INFO:tasks.rados.rados.0.smithi136.stdout:947: finishing write tid 2 to smithi13667081-23 2024-09-12T23:41:09.773 INFO:tasks.rados.rados.0.smithi136.stdout:947: finishing write tid 3 to smithi13667081-23 2024-09-12T23:41:09.773 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 23 v 533 (ObjNum 317 snap 86 seq_num 317) dirty exists 2024-09-12T23:41:09.773 INFO:tasks.rados.rados.0.smithi136.stdout:947: left oid 23 (ObjNum 317 snap 86 seq_num 317) 2024-09-12T23:41:09.830 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:09 smithi181 bash[52367]: debug 2024-09-12T23:41:09.690+0000 7f7d08dd4040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-12T23:41:09.830 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:09 smithi181 bash[52367]: debug 2024-09-12T23:41:09.762+0000 7f7d08dd4040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-12T23:41:09.830 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:09 smithi181 bash[24776]: cluster 2024-09-12T23:41:08.562621+0000 mon.a (mon.0) 1371 : cluster [DBG] mgrmap e32: y(active, since 1.15284s) 2024-09-12T23:41:09.830 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:09 smithi181 bash[24776]: cluster 2024-09-12T23:41:08.598197+0000 mgr.y (mgr.15123) 1 : cluster [DBG] pgmap v4: 121 pgs: 121 active+clean; 132 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:41:09.830 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:09 smithi181 bash[24776]: cluster 2024-09-12T23:41:09.420497+0000 mon.a (mon.0) 1372 : cluster [DBG] osdmap e234: 8 total, 8 up, 8 in 2024-09-12T23:41:10.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:09 smithi136 bash[22601]: cluster 2024-09-12T23:41:08.562621+0000 mon.a (mon.0) 1371 : cluster [DBG] mgrmap e32: y(active, since 1.15284s) 2024-09-12T23:41:10.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:09 smithi136 bash[22601]: cluster 2024-09-12T23:41:08.598197+0000 mgr.y (mgr.15123) 1 : cluster [DBG] pgmap v4: 121 pgs: 121 active+clean; 132 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:41:10.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:09 smithi136 bash[22601]: cluster 2024-09-12T23:41:09.420497+0000 mon.a (mon.0) 1372 : cluster [DBG] osdmap e234: 8 total, 8 up, 8 in 2024-09-12T23:41:10.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:09 smithi136 bash[32189]: cluster 2024-09-12T23:41:08.562621+0000 mon.a (mon.0) 1371 : cluster [DBG] mgrmap e32: y(active, since 1.15284s) 2024-09-12T23:41:10.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:09 smithi136 bash[32189]: cluster 2024-09-12T23:41:08.598197+0000 mgr.y (mgr.15123) 1 : cluster [DBG] pgmap v4: 121 pgs: 121 active+clean; 132 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:41:10.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:09 smithi136 bash[32189]: cluster 2024-09-12T23:41:09.420497+0000 mon.a (mon.0) 1372 : cluster [DBG] osdmap e234: 8 total, 8 up, 8 in 2024-09-12T23:41:10.006 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:41:09 smithi136 bash[68787]: debug 2024-09-12T23:41:09.747+0000 7f9776930640 -1 log_channel(cephadm) log [ERR] : [12/Sep/2024:23:41:09] ENGINE Error in HTTPServer.serve 2024-09-12T23:41:10.006 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:41:09 smithi136 bash[68787]: Traceback (most recent call last): 2024-09-12T23:41:10.006 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:41:09 smithi136 bash[68787]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-12T23:41:10.006 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:41:09 smithi136 bash[68787]: self._connections.run(self.expiration_interval) 2024-09-12T23:41:10.007 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:41:09 smithi136 bash[68787]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-12T23:41:10.007 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:41:09 smithi136 bash[68787]: self._run(expiration_interval) 2024-09-12T23:41:10.007 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:41:09 smithi136 bash[68787]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-12T23:41:10.007 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:41:09 smithi136 bash[68787]: new_conn = self._from_server_socket(self.server.socket) 2024-09-12T23:41:10.007 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:41:09 smithi136 bash[68787]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-12T23:41:10.007 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:41:09 smithi136 bash[68787]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-12T23:41:10.007 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:41:09 smithi136 bash[68787]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-12T23:41:10.007 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:41:09 smithi136 bash[68787]: s = self.context.wrap_socket( 2024-09-12T23:41:10.007 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:41:09 smithi136 bash[68787]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-12T23:41:10.007 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:41:09 smithi136 bash[68787]: return self.sslsocket_class._create( 2024-09-12T23:41:10.007 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:41:09 smithi136 bash[68787]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-12T23:41:10.007 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:41:09 smithi136 bash[68787]: self.do_handshake() 2024-09-12T23:41:10.007 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:41:09 smithi136 bash[68787]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-12T23:41:10.008 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:41:09 smithi136 bash[68787]: self._sslobj.do_handshake() 2024-09-12T23:41:10.008 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:41:09 smithi136 bash[68787]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-12T23:41:10.091 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:09 smithi181 bash[52367]: debug 2024-09-12T23:41:09.830+0000 7f7d08dd4040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-12T23:41:10.091 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:10 smithi181 bash[52367]: debug 2024-09-12T23:41:10.014+0000 7f7d08dd4040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-12T23:41:10.091 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:10 smithi181 bash[52367]: debug 2024-09-12T23:41:10.094+0000 7f7d08dd4040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-12T23:41:10.343 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:10 smithi181 bash[52367]: debug 2024-09-12T23:41:10.162+0000 7f7d08dd4040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-12T23:41:10.500 INFO:tasks.rados.rados.0.smithi136.stdout:943: done (4 left) 2024-09-12T23:41:10.500 INFO:tasks.rados.rados.0.smithi136.stdout:944: done (3 left) 2024-09-12T23:41:10.500 INFO:tasks.rados.rados.0.smithi136.stdout:945: done (2 left) 2024-09-12T23:41:10.501 INFO:tasks.rados.rados.0.smithi136.stdout:947: done (1 left) 2024-09-12T23:41:10.501 INFO:tasks.rados.rados.0.smithi136.stdout:948: done (0 left) 2024-09-12T23:41:10.501 INFO:tasks.rados.rados.0.smithi136.stdout:949: snap_remove snap 67 2024-09-12T23:41:10.675 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:10 smithi181 bash[52367]: debug 2024-09-12T23:41:10.410+0000 7f7d08dd4040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-12T23:41:10.676 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:10 smithi181 bash[52367]: debug 2024-09-12T23:41:10.482+0000 7f7d08dd4040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-12T23:41:10.676 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:10 smithi181 bash[52367]: debug 2024-09-12T23:41:10.678+0000 7f7d08dd4040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-12T23:41:10.676 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:10 smithi181 bash[24776]: cephadm 2024-09-12T23:41:09.526095+0000 mgr.y (mgr.15123) 3 : cephadm [INF] [12/Sep/2024:23:41:09] ENGINE Bus STARTING 2024-09-12T23:41:10.676 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:10 smithi181 bash[24776]: cephadm 2024-09-12T23:41:09.627921+0000 mgr.y (mgr.15123) 4 : cephadm [INF] [12/Sep/2024:23:41:09] ENGINE Serving on http://172.21.15.136:8765 2024-09-12T23:41:10.676 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:10 smithi181 bash[24776]: cephadm 2024-09-12T23:41:09.749365+0000 mgr.y (mgr.15123) 5 : cephadm [INF] [12/Sep/2024:23:41:09] ENGINE Serving on https://172.21.15.136:7150 2024-09-12T23:41:10.676 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:10 smithi181 bash[24776]: cephadm 2024-09-12T23:41:09.749586+0000 mgr.y (mgr.15123) 6 : cephadm [INF] [12/Sep/2024:23:41:09] ENGINE Bus STARTED 2024-09-12T23:41:10.676 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:10 smithi181 bash[24776]: cephadm 2024-09-12T23:41:09.751149+0000 mgr.y (mgr.15123) 7 : cephadm [ERR] [12/Sep/2024:23:41:09] ENGINE Error in HTTPServer.serve 2024-09-12T23:41:10.676 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:10 smithi181 bash[24776]: Traceback (most recent call last): 2024-09-12T23:41:10.677 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:10 smithi181 bash[24776]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-12T23:41:10.677 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:10 smithi181 bash[24776]: self._connections.run(self.expiration_interval) 2024-09-12T23:41:10.677 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:10 smithi181 bash[24776]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-12T23:41:10.677 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:10 smithi181 bash[24776]: self._run(expiration_interval) 2024-09-12T23:41:10.677 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:10 smithi181 bash[24776]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-12T23:41:10.677 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:10 smithi181 bash[24776]: new_conn = self._from_server_socket(self.server.socket) 2024-09-12T23:41:10.677 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:10 smithi181 bash[24776]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-12T23:41:10.677 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:10 smithi181 bash[24776]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-12T23:41:10.677 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:10 smithi181 bash[24776]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-12T23:41:10.677 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:10 smithi181 bash[24776]: s = self.context.wrap_socket( 2024-09-12T23:41:10.677 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:10 smithi181 bash[24776]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-12T23:41:10.677 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:10 smithi181 bash[24776]: return self.sslsocket_class._create( 2024-09-12T23:41:10.678 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:10 smithi181 bash[24776]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-12T23:41:10.678 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:10 smithi181 bash[24776]: self.do_handshake() 2024-09-12T23:41:10.678 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:10 smithi181 bash[24776]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-12T23:41:10.678 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:10 smithi181 bash[24776]: self._sslobj.do_handshake() 2024-09-12T23:41:10.678 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:10 smithi181 bash[24776]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-12T23:41:10.678 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:10 smithi181 bash[24776]: cluster 2024-09-12T23:41:10.503486+0000 mon.a (mon.0) 1373 : cluster [DBG] osdmap e235: 8 total, 8 up, 8 in 2024-09-12T23:41:11.004 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:10 smithi181 bash[52367]: debug 2024-09-12T23:41:10.754+0000 7f7d08dd4040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-12T23:41:11.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:10 smithi136 bash[32189]: cephadm 2024-09-12T23:41:09.526095+0000 mgr.y (mgr.15123) 3 : cephadm [INF] [12/Sep/2024:23:41:09] ENGINE Bus STARTING 2024-09-12T23:41:11.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:10 smithi136 bash[32189]: cephadm 2024-09-12T23:41:09.627921+0000 mgr.y (mgr.15123) 4 : cephadm [INF] [12/Sep/2024:23:41:09] ENGINE Serving on http://172.21.15.136:8765 2024-09-12T23:41:11.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:10 smithi136 bash[32189]: cephadm 2024-09-12T23:41:09.749365+0000 mgr.y (mgr.15123) 5 : cephadm [INF] [12/Sep/2024:23:41:09] ENGINE Serving on https://172.21.15.136:7150 2024-09-12T23:41:11.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:10 smithi136 bash[32189]: cephadm 2024-09-12T23:41:09.749586+0000 mgr.y (mgr.15123) 6 : cephadm [INF] [12/Sep/2024:23:41:09] ENGINE Bus STARTED 2024-09-12T23:41:11.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:10 smithi136 bash[32189]: cephadm 2024-09-12T23:41:09.751149+0000 mgr.y (mgr.15123) 7 : cephadm [ERR] [12/Sep/2024:23:41:09] ENGINE Error in HTTPServer.serve 2024-09-12T23:41:11.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:10 smithi136 bash[32189]: Traceback (most recent call last): 2024-09-12T23:41:11.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:10 smithi136 bash[32189]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-12T23:41:11.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:10 smithi136 bash[32189]: self._connections.run(self.expiration_interval) 2024-09-12T23:41:11.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:10 smithi136 bash[32189]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-12T23:41:11.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:10 smithi136 bash[32189]: self._run(expiration_interval) 2024-09-12T23:41:11.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:10 smithi136 bash[32189]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-12T23:41:11.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:10 smithi136 bash[32189]: new_conn = self._from_server_socket(self.server.socket) 2024-09-12T23:41:11.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:10 smithi136 bash[32189]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-12T23:41:11.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:10 smithi136 bash[32189]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-12T23:41:11.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:10 smithi136 bash[32189]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-12T23:41:11.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:10 smithi136 bash[32189]: s = self.context.wrap_socket( 2024-09-12T23:41:11.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:10 smithi136 bash[32189]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-12T23:41:11.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:10 smithi136 bash[32189]: return self.sslsocket_class._create( 2024-09-12T23:41:11.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:10 smithi136 bash[32189]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-12T23:41:11.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:10 smithi136 bash[32189]: self.do_handshake() 2024-09-12T23:41:11.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:10 smithi136 bash[32189]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-12T23:41:11.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:10 smithi136 bash[32189]: self._sslobj.do_handshake() 2024-09-12T23:41:11.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:10 smithi136 bash[32189]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-12T23:41:11.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:10 smithi136 bash[32189]: cluster 2024-09-12T23:41:10.503486+0000 mon.a (mon.0) 1373 : cluster [DBG] osdmap e235: 8 total, 8 up, 8 in 2024-09-12T23:41:11.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:10 smithi136 bash[22601]: cephadm 2024-09-12T23:41:09.526095+0000 mgr.y (mgr.15123) 3 : cephadm [INF] [12/Sep/2024:23:41:09] ENGINE Bus STARTING 2024-09-12T23:41:11.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:10 smithi136 bash[22601]: cephadm 2024-09-12T23:41:09.627921+0000 mgr.y (mgr.15123) 4 : cephadm [INF] [12/Sep/2024:23:41:09] ENGINE Serving on http://172.21.15.136:8765 2024-09-12T23:41:11.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:10 smithi136 bash[22601]: cephadm 2024-09-12T23:41:09.749365+0000 mgr.y (mgr.15123) 5 : cephadm [INF] [12/Sep/2024:23:41:09] ENGINE Serving on https://172.21.15.136:7150 2024-09-12T23:41:11.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:10 smithi136 bash[22601]: cephadm 2024-09-12T23:41:09.749586+0000 mgr.y (mgr.15123) 6 : cephadm [INF] [12/Sep/2024:23:41:09] ENGINE Bus STARTED 2024-09-12T23:41:11.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:10 smithi136 bash[22601]: cephadm 2024-09-12T23:41:09.751149+0000 mgr.y (mgr.15123) 7 : cephadm [ERR] [12/Sep/2024:23:41:09] ENGINE Error in HTTPServer.serve 2024-09-12T23:41:11.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:10 smithi136 bash[22601]: Traceback (most recent call last): 2024-09-12T23:41:11.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:10 smithi136 bash[22601]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-12T23:41:11.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:10 smithi136 bash[22601]: self._connections.run(self.expiration_interval) 2024-09-12T23:41:11.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:10 smithi136 bash[22601]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-12T23:41:11.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:10 smithi136 bash[22601]: self._run(expiration_interval) 2024-09-12T23:41:11.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:10 smithi136 bash[22601]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-12T23:41:11.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:10 smithi136 bash[22601]: new_conn = self._from_server_socket(self.server.socket) 2024-09-12T23:41:11.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:10 smithi136 bash[22601]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-12T23:41:11.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:10 smithi136 bash[22601]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-12T23:41:11.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:10 smithi136 bash[22601]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-12T23:41:11.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:10 smithi136 bash[22601]: s = self.context.wrap_socket( 2024-09-12T23:41:11.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:10 smithi136 bash[22601]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-12T23:41:11.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:10 smithi136 bash[22601]: return self.sslsocket_class._create( 2024-09-12T23:41:11.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:10 smithi136 bash[22601]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-12T23:41:11.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:10 smithi136 bash[22601]: self.do_handshake() 2024-09-12T23:41:11.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:10 smithi136 bash[22601]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-12T23:41:11.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:10 smithi136 bash[22601]: self._sslobj.do_handshake() 2024-09-12T23:41:11.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:10 smithi136 bash[22601]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-12T23:41:11.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:10 smithi136 bash[22601]: cluster 2024-09-12T23:41:10.503486+0000 mon.a (mon.0) 1373 : cluster [DBG] osdmap e235: 8 total, 8 up, 8 in 2024-09-12T23:41:11.341 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:11 smithi181 bash[52367]: debug 2024-09-12T23:41:11.006+0000 7f7d08dd4040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-12T23:41:11.341 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:11 smithi181 bash[52367]: debug 2024-09-12T23:41:11.178+0000 7f7d08dd4040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-12T23:41:11.341 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:11 smithi181 bash[52367]: debug 2024-09-12T23:41:11.342+0000 7f7d08dd4040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-12T23:41:11.505 INFO:tasks.rados.rados.0.smithi136.stdout:949: done (0 left) 2024-09-12T23:41:11.505 INFO:tasks.rados.rados.0.smithi136.stdout:950: rmattr oid 46 current snap is 87 2024-09-12T23:41:11.505 INFO:tasks.rados.rados.0.smithi136.stdout:950: done (0 left) 2024-09-12T23:41:11.505 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 21 current snap is 87 2024-09-12T23:41:11.505 INFO:tasks.rados.rados.0.smithi136.stdout:951: seq_num 318 ranges {4685824=532480} 2024-09-12T23:41:11.549 INFO:tasks.rados.rados.0.smithi136.stdout:951: writing smithi13667081-21 from 4685824 to 5218304 tid 1 2024-09-12T23:41:11.550 INFO:tasks.rados.rados.0.smithi136.stdout:952: delete oid 39 current snap is 87 2024-09-12T23:41:11.554 INFO:tasks.rados.rados.0.smithi136.stdout:952: done (1 left) 2024-09-12T23:41:11.554 INFO:tasks.rados.rados.0.smithi136.stdout:953: read oid 49 snap -1 2024-09-12T23:41:11.554 INFO:tasks.rados.rados.0.smithi136.stdout:953: expect deleted 2024-09-12T23:41:11.554 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 28 current snap is 87 2024-09-12T23:41:11.555 INFO:tasks.rados.rados.0.smithi136.stdout:954: seq_num 320 ranges {2170880=565248} 2024-09-12T23:41:11.570 INFO:tasks.rados.rados.0.smithi136.stdout:954: writing smithi13667081-28 from 2170880 to 2736128 tid 1 2024-09-12T23:41:11.570 INFO:tasks.rados.rados.0.smithi136.stdout:955: read oid 39 snap -1 2024-09-12T23:41:11.570 INFO:tasks.rados.rados.0.smithi136.stdout:955: expect deleted 2024-09-12T23:41:11.570 INFO:tasks.rados.rados.0.smithi136.stdout:956: rmattr oid 12 current snap is 87 2024-09-12T23:41:11.571 INFO:tasks.rados.rados.0.smithi136.stdout:956: done (4 left) 2024-09-12T23:41:11.571 INFO:tasks.rados.rados.0.smithi136.stdout:957: delete oid 3 current snap is 87 2024-09-12T23:41:11.571 INFO:tasks.rados.rados.0.smithi136.stdout:951: finishing write tid 1 to smithi13667081-21 2024-09-12T23:41:11.571 INFO:tasks.rados.rados.0.smithi136.stdout:951: finishing write tid 2 to smithi13667081-21 2024-09-12T23:41:11.571 INFO:tasks.rados.rados.0.smithi136.stdout:951: finishing write tid 3 to smithi13667081-21 2024-09-12T23:41:11.571 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 21 v 237 (ObjNum 318 snap 87 seq_num 318) dirty exists 2024-09-12T23:41:11.571 INFO:tasks.rados.rados.0.smithi136.stdout:951: left oid 21 (ObjNum 318 snap 87 seq_num 318) 2024-09-12T23:41:11.574 INFO:tasks.rados.rados.0.smithi136.stdout:951: done (4 left) 2024-09-12T23:41:11.574 INFO:tasks.rados.rados.0.smithi136.stdout:953: done (3 left) 2024-09-12T23:41:11.574 INFO:tasks.rados.rados.0.smithi136.stdout:955: done (2 left) 2024-09-12T23:41:11.574 INFO:tasks.rados.rados.0.smithi136.stdout:957: done (1 left) 2024-09-12T23:41:11.574 INFO:tasks.rados.rados.0.smithi136.stdout:958: copy_from oid 32 from oid 19 current snap is 87 2024-09-12T23:41:11.574 INFO:tasks.rados.rados.0.smithi136.stdout:959: snap_remove snap 73 2024-09-12T23:41:11.593 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:11 smithi181 bash[24776]: cluster 2024-09-12T23:41:10.583197+0000 mon.a (mon.0) 1374 : cluster [DBG] mgrmap e33: y(active, since 3s) 2024-09-12T23:41:11.593 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:11 smithi181 bash[24776]: cluster 2024-09-12T23:41:11.506186+0000 mon.a (mon.0) 1375 : cluster [DBG] osdmap e236: 8 total, 8 up, 8 in 2024-09-12T23:41:11.594 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:11 smithi181 bash[52367]: debug 2024-09-12T23:41:11.422+0000 7f7d08dd4040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-12T23:41:12.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:11 smithi136 bash[22601]: cluster 2024-09-12T23:41:10.583197+0000 mon.a (mon.0) 1374 : cluster [DBG] mgrmap e33: y(active, since 3s) 2024-09-12T23:41:12.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:11 smithi136 bash[22601]: cluster 2024-09-12T23:41:11.506186+0000 mon.a (mon.0) 1375 : cluster [DBG] osdmap e236: 8 total, 8 up, 8 in 2024-09-12T23:41:12.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:11 smithi136 bash[32189]: cluster 2024-09-12T23:41:10.583197+0000 mon.a (mon.0) 1374 : cluster [DBG] mgrmap e33: y(active, since 3s) 2024-09-12T23:41:12.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:11 smithi136 bash[32189]: cluster 2024-09-12T23:41:11.506186+0000 mon.a (mon.0) 1375 : cluster [DBG] osdmap e236: 8 total, 8 up, 8 in 2024-09-12T23:41:12.093 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:11 smithi181 bash[52367]: debug 2024-09-12T23:41:11.622+0000 7f7d08dd4040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-12T23:41:12.581 INFO:tasks.rados.rados.0.smithi136.stdout:959: done (2 left) 2024-09-12T23:41:12.581 INFO:tasks.rados.rados.0.smithi136.stdout:960: snap_create 2024-09-12T23:41:12.582 INFO:tasks.rados.rados.0.smithi136.stdout:958: finishing copy_from to smithi13667081-32 2024-09-12T23:41:12.582 INFO:tasks.rados.rados.0.smithi136.stdout:958: got expected ENOENT (src dne) 2024-09-12T23:41:12.582 INFO:tasks.rados.rados.0.smithi136.stdout:958: finishing copy_from racing read to smithi13667081-32 2024-09-12T23:41:12.582 INFO:tasks.rados.rados.0.smithi136.stdout:954: finishing write tid 1 to smithi13667081-28 2024-09-12T23:41:12.582 INFO:tasks.rados.rados.0.smithi136.stdout:954: finishing write tid 2 to smithi13667081-28 2024-09-12T23:41:12.582 INFO:tasks.rados.rados.0.smithi136.stdout:954: finishing write tid 3 to smithi13667081-28 2024-09-12T23:41:12.582 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 28 v 554 (ObjNum 320 snap 87 seq_num 320) dirty exists 2024-09-12T23:41:12.582 INFO:tasks.rados.rados.0.smithi136.stdout:954: left oid 28 (ObjNum 320 snap 87 seq_num 320) 2024-09-12T23:41:12.588 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:12 smithi181 bash[52367]: debug 2024-09-12T23:41:12.426+0000 7f7d08dd4040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-12T23:41:12.588 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:12 smithi181 bash[52367]: debug 2024-09-12T23:41:12.506+0000 7f7d08dd4040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-12T23:41:12.588 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:12 smithi181 bash[58696]: ts=2024-09-12T23:41:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:41:12.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:12 smithi181 bash[24776]: cluster 2024-09-12T23:41:11.435170+0000 mgr.y (mgr.15123) 8 : cluster [DBG] pgmap v8: 121 pgs: 121 active+clean; 132 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:41:12.843 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:12 smithi181 bash[52367]: debug 2024-09-12T23:41:12.742+0000 7f7d08dd4040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-12T23:41:12.844 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:12 smithi181 bash[52367]: debug 2024-09-12T23:41:12.814+0000 7f7d08dd4040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-12T23:41:13.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:12 smithi136 bash[22601]: cluster 2024-09-12T23:41:11.435170+0000 mgr.y (mgr.15123) 8 : cluster [DBG] pgmap v8: 121 pgs: 121 active+clean; 132 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:41:13.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:12 smithi136 bash[32189]: cluster 2024-09-12T23:41:11.435170+0000 mgr.y (mgr.15123) 8 : cluster [DBG] pgmap v8: 121 pgs: 121 active+clean; 132 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:41:13.500 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:13 smithi181 bash[52367]: debug 2024-09-12T23:41:13.222+0000 7f7d08dd4040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-12T23:41:13.501 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:13 smithi181 bash[52367]: debug 2024-09-12T23:41:13.306+0000 7f7d08dd4040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-12T23:41:13.501 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:13 smithi181 bash[52367]: [12/Sep/2024:23:41:13] ENGINE Bus STARTING 2024-09-12T23:41:13.501 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:13 smithi181 bash[52367]: CherryPy Checker: 2024-09-12T23:41:13.501 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:13 smithi181 bash[52367]: The Application mounted at '' has an empty config. 2024-09-12T23:41:13.501 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:13 smithi181 bash[52367]: [12/Sep/2024:23:41:13] ENGINE Serving on http://:::9283 2024-09-12T23:41:13.501 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:13 smithi181 bash[52367]: [12/Sep/2024:23:41:13] ENGINE Bus STARTED 2024-09-12T23:41:13.578 INFO:tasks.rados.rados.0.smithi136.stdout:954: done (2 left) 2024-09-12T23:41:13.578 INFO:tasks.rados.rados.0.smithi136.stdout:958: done (1 left) 2024-09-12T23:41:13.579 INFO:tasks.rados.rados.0.smithi136.stdout:960: done (0 left) 2024-09-12T23:41:13.579 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 21 current snap is 88 2024-09-12T23:41:13.579 INFO:tasks.rados.rados.0.smithi136.stdout:961: seq_num 322 ranges {5218304=598016} 2024-09-12T23:41:13.617 INFO:tasks.rados.rados.0.smithi136.stdout:961: writing smithi13667081-21 from 5218304 to 5816320 tid 1 2024-09-12T23:41:13.617 INFO:tasks.rados.rados.0.smithi136.stdout:962: rollback oid 42 current snap is 88 2024-09-12T23:41:13.617 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 42 to 83 2024-09-12T23:41:13.617 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 47 current snap is 88 2024-09-12T23:41:13.617 INFO:tasks.rados.rados.0.smithi136.stdout:963: seq_num 323 ranges {1474560=614400} 2024-09-12T23:41:13.630 INFO:tasks.rados.rados.0.smithi136.stdout:963: writing smithi13667081-47 from 1474560 to 2088960 tid 1 2024-09-12T23:41:13.630 INFO:tasks.rados.rados.0.smithi136.stdout:964: copy_from oid 13 from oid 7 current snap is 88 2024-09-12T23:41:13.630 INFO:tasks.rados.rados.0.smithi136.stdout:961: finishing write tid 1 to smithi13667081-21 2024-09-12T23:41:13.630 INFO:tasks.rados.rados.0.smithi136.stdout:961: finishing write tid 2 to smithi13667081-21 2024-09-12T23:41:13.630 INFO:tasks.rados.rados.0.smithi136.stdout:962: finishing rollback tid 0 to smithi13667081-42 2024-09-12T23:41:13.630 INFO:tasks.rados.rados.0.smithi136.stdout:961: finishing write tid 3 to smithi13667081-21 2024-09-12T23:41:13.630 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 21 v 244 (ObjNum 322 snap 88 seq_num 322) dirty exists 2024-09-12T23:41:13.630 INFO:tasks.rados.rados.0.smithi136.stdout:961: left oid 21 (ObjNum 322 snap 88 seq_num 322) 2024-09-12T23:41:13.630 INFO:tasks.rados.rados.0.smithi136.stdout:962: finishing rollback tid 1 to smithi13667081-42 2024-09-12T23:41:13.630 INFO:tasks.rados.rados.0.smithi136.stdout:962: finishing rollback tid 2 to smithi13667081-42 2024-09-12T23:41:13.630 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 42 v 248 (ObjNum 125 snap 32 seq_num 125) dirty exists 2024-09-12T23:41:13.631 INFO:tasks.rados.rados.0.smithi136.stdout:961: done (3 left) 2024-09-12T23:41:13.631 INFO:tasks.rados.rados.0.smithi136.stdout:962: done (2 left) 2024-09-12T23:41:13.631 INFO:tasks.rados.rados.0.smithi136.stdout:965: snap_create 2024-09-12T23:41:13.631 INFO:tasks.rados.rados.0.smithi136.stdout:964: finishing copy_from to smithi13667081-13 2024-09-12T23:41:13.631 INFO:tasks.rados.rados.0.smithi136.stdout:964: got expected ENOENT (src dne) 2024-09-12T23:41:13.631 INFO:tasks.rados.rados.0.smithi136.stdout:964: finishing copy_from racing read to smithi13667081-13 2024-09-12T23:41:13.633 INFO:tasks.rados.rados.0.smithi136.stdout:963: finishing write tid 1 to smithi13667081-47 2024-09-12T23:41:13.633 INFO:tasks.rados.rados.0.smithi136.stdout:963: finishing write tid 2 to smithi13667081-47 2024-09-12T23:41:13.634 INFO:tasks.rados.rados.0.smithi136.stdout:963: finishing write tid 3 to smithi13667081-47 2024-09-12T23:41:13.634 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 47 v 294 (ObjNum 323 snap 88 seq_num 323) dirty exists 2024-09-12T23:41:13.634 INFO:tasks.rados.rados.0.smithi136.stdout:963: left oid 47 (ObjNum 323 snap 88 seq_num 323) 2024-09-12T23:41:13.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:13 smithi181 bash[24776]: cluster 2024-09-12T23:41:12.585058+0000 mon.a (mon.0) 1376 : cluster [DBG] osdmap e237: 8 total, 8 up, 8 in 2024-09-12T23:41:13.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:13 smithi181 bash[24776]: cluster 2024-09-12T23:41:13.329149+0000 mon.a (mon.0) 1377 : cluster [DBG] Standby manager daemon x started 2024-09-12T23:41:13.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:13 smithi181 bash[24776]: audit 2024-09-12T23:41:13.331939+0000 mon.b (mon.1) 130 : audit [DBG] from='mgr.? 172.21.15.181:0/2224483215' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-12T23:41:13.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:13 smithi181 bash[24776]: audit 2024-09-12T23:41:13.332416+0000 mon.b (mon.1) 131 : audit [DBG] from='mgr.? 172.21.15.181:0/2224483215' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-12T23:41:13.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:13 smithi181 bash[24776]: audit 2024-09-12T23:41:13.333999+0000 mon.b (mon.1) 132 : audit [DBG] from='mgr.? 172.21.15.181:0/2224483215' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-12T23:41:13.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:13 smithi181 bash[24776]: audit 2024-09-12T23:41:13.335260+0000 mon.b (mon.1) 133 : audit [DBG] from='mgr.? 172.21.15.181:0/2224483215' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-12T23:41:13.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:13 smithi181 bash[24776]: cluster 2024-09-12T23:41:13.583292+0000 mon.a (mon.0) 1378 : cluster [DBG] osdmap e238: 8 total, 8 up, 8 in 2024-09-12T23:41:13.844 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:13 smithi181 bash[58696]: ts=2024-09-12T23:41:13.506Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi136\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:41:14.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:13 smithi136 bash[22601]: cluster 2024-09-12T23:41:12.585058+0000 mon.a (mon.0) 1376 : cluster [DBG] osdmap e237: 8 total, 8 up, 8 in 2024-09-12T23:41:14.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:13 smithi136 bash[22601]: cluster 2024-09-12T23:41:13.329149+0000 mon.a (mon.0) 1377 : cluster [DBG] Standby manager daemon x started 2024-09-12T23:41:14.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:13 smithi136 bash[22601]: audit 2024-09-12T23:41:13.331939+0000 mon.b (mon.1) 130 : audit [DBG] from='mgr.? 172.21.15.181:0/2224483215' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-12T23:41:14.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:13 smithi136 bash[22601]: audit 2024-09-12T23:41:13.332416+0000 mon.b (mon.1) 131 : audit [DBG] from='mgr.? 172.21.15.181:0/2224483215' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-12T23:41:14.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:13 smithi136 bash[22601]: audit 2024-09-12T23:41:13.333999+0000 mon.b (mon.1) 132 : audit [DBG] from='mgr.? 172.21.15.181:0/2224483215' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-12T23:41:14.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:13 smithi136 bash[22601]: audit 2024-09-12T23:41:13.335260+0000 mon.b (mon.1) 133 : audit [DBG] from='mgr.? 172.21.15.181:0/2224483215' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-12T23:41:14.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:13 smithi136 bash[22601]: cluster 2024-09-12T23:41:13.583292+0000 mon.a (mon.0) 1378 : cluster [DBG] osdmap e238: 8 total, 8 up, 8 in 2024-09-12T23:41:14.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:13 smithi136 bash[32189]: cluster 2024-09-12T23:41:12.585058+0000 mon.a (mon.0) 1376 : cluster [DBG] osdmap e237: 8 total, 8 up, 8 in 2024-09-12T23:41:14.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:13 smithi136 bash[32189]: cluster 2024-09-12T23:41:13.329149+0000 mon.a (mon.0) 1377 : cluster [DBG] Standby manager daemon x started 2024-09-12T23:41:14.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:13 smithi136 bash[32189]: audit 2024-09-12T23:41:13.331939+0000 mon.b (mon.1) 130 : audit [DBG] from='mgr.? 172.21.15.181:0/2224483215' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-12T23:41:14.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:13 smithi136 bash[32189]: audit 2024-09-12T23:41:13.332416+0000 mon.b (mon.1) 131 : audit [DBG] from='mgr.? 172.21.15.181:0/2224483215' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-12T23:41:14.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:13 smithi136 bash[32189]: audit 2024-09-12T23:41:13.333999+0000 mon.b (mon.1) 132 : audit [DBG] from='mgr.? 172.21.15.181:0/2224483215' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-12T23:41:14.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:13 smithi136 bash[32189]: audit 2024-09-12T23:41:13.335260+0000 mon.b (mon.1) 133 : audit [DBG] from='mgr.? 172.21.15.181:0/2224483215' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-12T23:41:14.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:13 smithi136 bash[32189]: cluster 2024-09-12T23:41:13.583292+0000 mon.a (mon.0) 1378 : cluster [DBG] osdmap e238: 8 total, 8 up, 8 in 2024-09-12T23:41:14.594 INFO:tasks.rados.rados.0.smithi136.stdout:963: done (2 left) 2024-09-12T23:41:14.594 INFO:tasks.rados.rados.0.smithi136.stdout:964: done (1 left) 2024-09-12T23:41:14.594 INFO:tasks.rados.rados.0.smithi136.stdout:965: done (0 left) 2024-09-12T23:41:14.594 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 15 current snap is 89 2024-09-12T23:41:14.594 INFO:tasks.rados.rados.0.smithi136.stdout:966: seq_num 324 ranges {0=630784} 2024-09-12T23:41:14.598 INFO:tasks.rados.rados.0.smithi136.stdout:966: writing smithi13667081-15 from 0 to 630784 tid 1 2024-09-12T23:41:14.598 INFO:tasks.rados.rados.0.smithi136.stdout:967: delete oid 31 current snap is 89 2024-09-12T23:41:14.603 INFO:tasks.rados.rados.0.smithi136.stdout:966: finishing write tid 1 to smithi13667081-15 2024-09-12T23:41:14.603 INFO:tasks.rados.rados.0.smithi136.stdout:966: finishing write tid 2 to smithi13667081-15 2024-09-12T23:41:14.604 INFO:tasks.rados.rados.0.smithi136.stdout:967: done (1 left) 2024-09-12T23:41:14.604 INFO:tasks.rados.rados.0.smithi136.stdout:968: snap_create 2024-09-12T23:41:14.605 INFO:tasks.rados.rados.0.smithi136.stdout:966: finishing write tid 3 to smithi13667081-15 2024-09-12T23:41:14.605 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 15 v 505 (ObjNum 324 snap 89 seq_num 324) dirty exists 2024-09-12T23:41:14.605 INFO:tasks.rados.rados.0.smithi136.stdout:966: left oid 15 (ObjNum 324 snap 89 seq_num 324) 2024-09-12T23:41:15.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:14 smithi136 bash[22601]: cluster 2024-09-12T23:41:13.435608+0000 mgr.y (mgr.15123) 9 : cluster [DBG] pgmap v11: 121 pgs: 121 active+clean; 132 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:41:15.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:14 smithi136 bash[22601]: cluster 2024-09-12T23:41:13.601680+0000 mon.a (mon.0) 1379 : cluster [DBG] mgrmap e34: y(active, since 6s), standbys: x 2024-09-12T23:41:15.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:14 smithi136 bash[22601]: audit 2024-09-12T23:41:13.603596+0000 mon.b (mon.1) 134 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-12T23:41:15.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:14 smithi136 bash[22601]: audit 2024-09-12T23:41:13.679545+0000 mon.a (mon.0) 1380 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:15.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:14 smithi136 bash[22601]: audit 2024-09-12T23:41:13.686838+0000 mon.a (mon.0) 1381 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:15.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:14 smithi136 bash[22601]: audit 2024-09-12T23:41:13.905843+0000 mon.a (mon.0) 1382 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:15.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:14 smithi136 bash[22601]: audit 2024-09-12T23:41:13.917636+0000 mon.a (mon.0) 1383 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:15.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:14 smithi136 bash[22601]: audit 2024-09-12T23:41:14.350862+0000 mon.a (mon.0) 1384 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:15.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:14 smithi136 bash[22601]: audit 2024-09-12T23:41:14.361251+0000 mon.a (mon.0) 1385 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:15.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:14 smithi136 bash[32189]: cluster 2024-09-12T23:41:13.435608+0000 mgr.y (mgr.15123) 9 : cluster [DBG] pgmap v11: 121 pgs: 121 active+clean; 132 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:41:15.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:14 smithi136 bash[32189]: cluster 2024-09-12T23:41:13.601680+0000 mon.a (mon.0) 1379 : cluster [DBG] mgrmap e34: y(active, since 6s), standbys: x 2024-09-12T23:41:15.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:14 smithi136 bash[32189]: audit 2024-09-12T23:41:13.603596+0000 mon.b (mon.1) 134 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-12T23:41:15.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:14 smithi136 bash[32189]: audit 2024-09-12T23:41:13.679545+0000 mon.a (mon.0) 1380 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:15.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:14 smithi136 bash[32189]: audit 2024-09-12T23:41:13.686838+0000 mon.a (mon.0) 1381 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:15.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:14 smithi136 bash[32189]: audit 2024-09-12T23:41:13.905843+0000 mon.a (mon.0) 1382 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:15.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:14 smithi136 bash[32189]: audit 2024-09-12T23:41:13.917636+0000 mon.a (mon.0) 1383 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:15.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:14 smithi136 bash[32189]: audit 2024-09-12T23:41:14.350862+0000 mon.a (mon.0) 1384 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:15.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:14 smithi136 bash[32189]: audit 2024-09-12T23:41:14.361251+0000 mon.a (mon.0) 1385 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:15.093 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:14 smithi181 bash[24776]: cluster 2024-09-12T23:41:13.435608+0000 mgr.y (mgr.15123) 9 : cluster [DBG] pgmap v11: 121 pgs: 121 active+clean; 132 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:41:15.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:14 smithi181 bash[24776]: cluster 2024-09-12T23:41:13.601680+0000 mon.a (mon.0) 1379 : cluster [DBG] mgrmap e34: y(active, since 6s), standbys: x 2024-09-12T23:41:15.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:14 smithi181 bash[24776]: audit 2024-09-12T23:41:13.603596+0000 mon.b (mon.1) 134 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-12T23:41:15.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:14 smithi181 bash[24776]: audit 2024-09-12T23:41:13.679545+0000 mon.a (mon.0) 1380 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:15.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:14 smithi181 bash[24776]: audit 2024-09-12T23:41:13.686838+0000 mon.a (mon.0) 1381 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:15.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:14 smithi181 bash[24776]: audit 2024-09-12T23:41:13.905843+0000 mon.a (mon.0) 1382 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:15.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:14 smithi181 bash[24776]: audit 2024-09-12T23:41:13.917636+0000 mon.a (mon.0) 1383 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:15.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:14 smithi181 bash[24776]: audit 2024-09-12T23:41:14.350862+0000 mon.a (mon.0) 1384 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:15.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:14 smithi181 bash[24776]: audit 2024-09-12T23:41:14.361251+0000 mon.a (mon.0) 1385 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:15.617 INFO:tasks.rados.rados.0.smithi136.stdout:966: done (1 left) 2024-09-12T23:41:15.617 INFO:tasks.rados.rados.0.smithi136.stdout:968: done (0 left) 2024-09-12T23:41:15.617 INFO:tasks.rados.rados.0.smithi136.stdout:969: copy_from oid 33 from oid 44 current snap is 90 2024-09-12T23:41:15.617 INFO:tasks.rados.rados.0.smithi136.stdout:970: delete oid 3 current snap is 90 2024-09-12T23:41:15.625 INFO:tasks.rados.rados.0.smithi136.stdout:969: finishing copy_from racing read to smithi13667081-33 2024-09-12T23:41:15.628 INFO:tasks.rados.rados.0.smithi136.stdout:969: finishing copy_from to smithi13667081-33 2024-09-12T23:41:15.628 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 33 v 529 (ObjNum 129 snap 0 seq_num 4243303312) dirty exists 2024-09-12T23:41:15.628 INFO:tasks.rados.rados.0.smithi136.stdout:969: done (1 left) 2024-09-12T23:41:15.628 INFO:tasks.rados.rados.0.smithi136.stdout:970: done (0 left) 2024-09-12T23:41:15.628 INFO:tasks.rados.rados.0.smithi136.stdout:971: snap_remove snap 85 2024-09-12T23:41:16.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:15 smithi136 bash[22601]: cluster 2024-09-12T23:41:14.599104+0000 mon.a (mon.0) 1386 : cluster [DBG] osdmap e239: 8 total, 8 up, 8 in 2024-09-12T23:41:16.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:15 smithi136 bash[22601]: audit 2024-09-12T23:41:14.627879+0000 mon.a (mon.0) 1387 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:16.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:15 smithi136 bash[22601]: audit 2024-09-12T23:41:14.640060+0000 mon.a (mon.0) 1388 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:16.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:15 smithi136 bash[32189]: cluster 2024-09-12T23:41:14.599104+0000 mon.a (mon.0) 1386 : cluster [DBG] osdmap e239: 8 total, 8 up, 8 in 2024-09-12T23:41:16.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:15 smithi136 bash[32189]: audit 2024-09-12T23:41:14.627879+0000 mon.a (mon.0) 1387 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:16.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:15 smithi136 bash[32189]: audit 2024-09-12T23:41:14.640060+0000 mon.a (mon.0) 1388 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:16.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:15 smithi181 bash[24776]: cluster 2024-09-12T23:41:14.599104+0000 mon.a (mon.0) 1386 : cluster [DBG] osdmap e239: 8 total, 8 up, 8 in 2024-09-12T23:41:16.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:15 smithi181 bash[24776]: audit 2024-09-12T23:41:14.627879+0000 mon.a (mon.0) 1387 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:16.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:15 smithi181 bash[24776]: audit 2024-09-12T23:41:14.640060+0000 mon.a (mon.0) 1388 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:16.627 INFO:tasks.rados.rados.0.smithi136.stdout:971: done (0 left) 2024-09-12T23:41:16.627 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 34 current snap is 90 2024-09-12T23:41:16.627 INFO:tasks.rados.rados.0.smithi136.stdout:972: seq_num 327 ranges {0=679936} 2024-09-12T23:41:16.632 INFO:tasks.rados.rados.0.smithi136.stdout:972: writing smithi13667081-34 from 0 to 679936 tid 1 2024-09-12T23:41:16.632 INFO:tasks.rados.rados.0.smithi136.stdout:973: read oid 25 snap -1 2024-09-12T23:41:16.632 INFO:tasks.rados.rados.0.smithi136.stdout:973: expect deleted 2024-09-12T23:41:16.632 INFO:tasks.rados.rados.0.smithi136.stdout:974: read oid 39 snap -1 2024-09-12T23:41:16.632 INFO:tasks.rados.rados.0.smithi136.stdout:974: expect deleted 2024-09-12T23:41:16.632 INFO:tasks.rados.rados.0.smithi136.stdout:975: delete oid 37 current snap is 90 2024-09-12T23:41:16.651 INFO:tasks.rados.rados.0.smithi136.stdout:972: finishing write tid 1 to smithi13667081-34 2024-09-12T23:41:16.652 INFO:tasks.rados.rados.0.smithi136.stdout:972: finishing write tid 2 to smithi13667081-34 2024-09-12T23:41:16.652 INFO:tasks.rados.rados.0.smithi136.stdout:973: done (3 left) 2024-09-12T23:41:16.653 INFO:tasks.rados.rados.0.smithi136.stdout:974: done (2 left) 2024-09-12T23:41:16.653 INFO:tasks.rados.rados.0.smithi136.stdout:975: done (1 left) 2024-09-12T23:41:16.653 INFO:tasks.rados.rados.0.smithi136.stdout:976: read oid 14 snap -1 2024-09-12T23:41:16.653 INFO:tasks.rados.rados.0.smithi136.stdout:976: expect (ObjNum 277 snap 74 seq_num 277) 2024-09-12T23:41:16.653 INFO:tasks.rados.rados.0.smithi136.stdout:977: copy_from oid 15 from oid 26 current snap is 90 2024-09-12T23:41:16.653 INFO:tasks.rados.rados.0.smithi136.stdout:978: rollback oid 39 current snap is 90 2024-09-12T23:41:16.653 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 39 to 86 2024-09-12T23:41:16.653 INFO:tasks.rados.rados.0.smithi136.stdout:979: delete oid 7 current snap is 90 2024-09-12T23:41:16.655 INFO:tasks.rados.rados.0.smithi136.stdout:972: finishing write tid 3 to smithi13667081-34 2024-09-12T23:41:16.655 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 34 v 531 (ObjNum 327 snap 90 seq_num 327) dirty exists 2024-09-12T23:41:16.655 INFO:tasks.rados.rados.0.smithi136.stdout:972: left oid 34 (ObjNum 327 snap 90 seq_num 327) 2024-09-12T23:41:16.657 INFO:tasks.rados.rados.0.smithi136.stdout:972: done (4 left) 2024-09-12T23:41:16.657 INFO:tasks.rados.rados.0.smithi136.stdout:979: done (3 left) 2024-09-12T23:41:16.657 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 45 current snap is 90 2024-09-12T23:41:16.657 INFO:tasks.rados.rados.0.smithi136.stdout:980: seq_num 330 ranges {0=737280} 2024-09-12T23:41:16.664 INFO:tasks.rados.rados.0.smithi136.stdout:980: writing smithi13667081-45 from 0 to 737280 tid 1 2024-09-12T23:41:16.664 INFO:tasks.rados.rados.0.smithi136.stdout:981: read oid 30 snap -1 2024-09-12T23:41:16.664 INFO:tasks.rados.rados.0.smithi136.stdout:981: expect (ObjNum 292 snap 79 seq_num 292) 2024-09-12T23:41:16.664 INFO:tasks.rados.rados.0.smithi136.stdout:982: snap_remove snap 82 2024-09-12T23:41:17.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:16 smithi136 bash[22601]: cluster 2024-09-12T23:41:15.436306+0000 mgr.y (mgr.15123) 10 : cluster [DBG] pgmap v14: 121 pgs: 121 active+clean; 116 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.1 MiB/s wr, 39 op/s 2024-09-12T23:41:17.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:16 smithi136 bash[22601]: cluster 2024-09-12T23:41:15.618255+0000 mon.a (mon.0) 1389 : cluster [DBG] osdmap e240: 8 total, 8 up, 8 in 2024-09-12T23:41:17.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:16 smithi136 bash[32189]: cluster 2024-09-12T23:41:15.436306+0000 mgr.y (mgr.15123) 10 : cluster [DBG] pgmap v14: 121 pgs: 121 active+clean; 116 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.1 MiB/s wr, 39 op/s 2024-09-12T23:41:17.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:16 smithi136 bash[32189]: cluster 2024-09-12T23:41:15.618255+0000 mon.a (mon.0) 1389 : cluster [DBG] osdmap e240: 8 total, 8 up, 8 in 2024-09-12T23:41:17.093 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:16 smithi181 bash[24776]: cluster 2024-09-12T23:41:15.436306+0000 mgr.y (mgr.15123) 10 : cluster [DBG] pgmap v14: 121 pgs: 121 active+clean; 116 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.1 MiB/s wr, 39 op/s 2024-09-12T23:41:17.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:16 smithi181 bash[24776]: cluster 2024-09-12T23:41:15.618255+0000 mon.a (mon.0) 1389 : cluster [DBG] osdmap e240: 8 total, 8 up, 8 in 2024-09-12T23:41:17.637 INFO:tasks.rados.rados.0.smithi136.stdout:982: done (5 left) 2024-09-12T23:41:17.638 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 40 current snap is 90 2024-09-12T23:41:17.638 INFO:tasks.rados.rados.0.smithi136.stdout:983: seq_num 331 ranges {2203648=753664} 2024-09-12T23:41:17.688 INFO:tasks.rados.rados.0.smithi136.stdout:983: writing smithi13667081-40 from 2203648 to 2957312 tid 1 2024-09-12T23:41:17.688 INFO:tasks.rados.rados.0.smithi136.stdout:984: snap_create 2024-09-12T23:41:17.688 INFO:tasks.rados.rados.0.smithi136.stdout:978: finishing rollback tid 1 to smithi13667081-39 2024-09-12T23:41:17.688 INFO:tasks.rados.rados.0.smithi136.stdout:978: finishing rollback tid 2 to smithi13667081-39 2024-09-12T23:41:17.688 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 39 v 143 (ObjNum 312 snap 86 seq_num 312) dirty exists 2024-09-12T23:41:17.688 INFO:tasks.rados.rados.0.smithi136.stdout:981: expect (ObjNum 292 snap 79 seq_num 292) 2024-09-12T23:41:17.700 INFO:tasks.rados.rados.0.smithi136.stdout:980: finishing write tid 1 to smithi13667081-45 2024-09-12T23:41:17.700 INFO:tasks.rados.rados.0.smithi136.stdout:980: finishing write tid 2 to smithi13667081-45 2024-09-12T23:41:17.700 INFO:tasks.rados.rados.0.smithi136.stdout:976: expect (ObjNum 277 snap 74 seq_num 277) 2024-09-12T23:41:17.800 INFO:tasks.rados.rados.0.smithi136.stdout:980: finishing write tid 3 to smithi13667081-45 2024-09-12T23:41:17.800 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 45 v 286 (ObjNum 330 snap 90 seq_num 330) dirty exists 2024-09-12T23:41:17.800 INFO:tasks.rados.rados.0.smithi136.stdout:980: left oid 45 (ObjNum 330 snap 90 seq_num 330) 2024-09-12T23:41:17.800 INFO:tasks.rados.rados.0.smithi136.stdout:977: finishing copy_from racing read to smithi13667081-15 2024-09-12T23:41:17.800 INFO:tasks.rados.rados.0.smithi136.stdout:977: finishing copy_from to smithi13667081-15 2024-09-12T23:41:17.800 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 15 v 381 (ObjNum 305 snap 84 seq_num 305) dirty exists 2024-09-12T23:41:17.800 INFO:tasks.rados.rados.0.smithi136.stdout:983: finishing write tid 1 to smithi13667081-40 2024-09-12T23:41:17.801 INFO:tasks.rados.rados.0.smithi136.stdout:983: finishing write tid 2 to smithi13667081-40 2024-09-12T23:41:17.801 INFO:tasks.rados.rados.0.smithi136.stdout:983: finishing write tid 3 to smithi13667081-40 2024-09-12T23:41:17.801 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 40 v 562 (ObjNum 331 snap 90 seq_num 331) dirty exists 2024-09-12T23:41:17.801 INFO:tasks.rados.rados.0.smithi136.stdout:983: left oid 40 (ObjNum 331 snap 90 seq_num 331) 2024-09-12T23:41:18.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:17 smithi136 bash[22601]: cluster 2024-09-12T23:41:16.627348+0000 mon.a (mon.0) 1390 : cluster [DBG] osdmap e241: 8 total, 8 up, 8 in 2024-09-12T23:41:18.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:17 smithi136 bash[32189]: cluster 2024-09-12T23:41:16.627348+0000 mon.a (mon.0) 1390 : cluster [DBG] osdmap e241: 8 total, 8 up, 8 in 2024-09-12T23:41:18.093 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:17 smithi181 bash[24776]: cluster 2024-09-12T23:41:16.627348+0000 mon.a (mon.0) 1390 : cluster [DBG] osdmap e241: 8 total, 8 up, 8 in 2024-09-12T23:41:18.662 INFO:tasks.rados.rados.0.smithi136.stdout:976: done (6 left) 2024-09-12T23:41:18.663 INFO:tasks.rados.rados.0.smithi136.stdout:977: done (5 left) 2024-09-12T23:41:18.663 INFO:tasks.rados.rados.0.smithi136.stdout:978: done (4 left) 2024-09-12T23:41:18.663 INFO:tasks.rados.rados.0.smithi136.stdout:980: done (3 left) 2024-09-12T23:41:18.663 INFO:tasks.rados.rados.0.smithi136.stdout:981: done (2 left) 2024-09-12T23:41:18.663 INFO:tasks.rados.rados.0.smithi136.stdout:983: done (1 left) 2024-09-12T23:41:18.663 INFO:tasks.rados.rados.0.smithi136.stdout:984: done (0 left) 2024-09-12T23:41:18.663 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 19 current snap is 91 2024-09-12T23:41:18.663 INFO:tasks.rados.rados.0.smithi136.stdout:985: seq_num 332 ranges {0=770048} 2024-09-12T23:41:18.673 INFO:tasks.rados.rados.0.smithi136.stdout:985: writing smithi13667081-19 from 0 to 770048 tid 1 2024-09-12T23:41:18.673 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 25 current snap is 91 2024-09-12T23:41:18.673 INFO:tasks.rados.rados.0.smithi136.stdout:986: seq_num 333 ranges {0=786432} 2024-09-12T23:41:18.684 INFO:tasks.rados.rados.0.smithi136.stdout:986: writing smithi13667081-25 from 0 to 786432 tid 1 2024-09-12T23:41:18.685 INFO:tasks.rados.rados.0.smithi136.stdout:987: rmattr oid 42 current snap is 91 2024-09-12T23:41:18.685 INFO:tasks.rados.rados.0.smithi136.stdout:985: finishing write tid 1 to smithi13667081-19 2024-09-12T23:41:18.685 INFO:tasks.rados.rados.0.smithi136.stdout:988: delete oid 10 current snap is 91 2024-09-12T23:41:18.685 INFO:tasks.rados.rados.0.smithi136.stdout:985: finishing write tid 2 to smithi13667081-19 2024-09-12T23:41:18.688 INFO:tasks.rados.rados.0.smithi136.stdout:985: finishing write tid 3 to smithi13667081-19 2024-09-12T23:41:18.688 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 19 v 302 (ObjNum 332 snap 91 seq_num 332) dirty exists 2024-09-12T23:41:18.688 INFO:tasks.rados.rados.0.smithi136.stdout:985: left oid 19 (ObjNum 332 snap 91 seq_num 332) 2024-09-12T23:41:18.691 INFO:tasks.rados.rados.0.smithi136.stdout:985: done (3 left) 2024-09-12T23:41:18.691 INFO:tasks.rados.rados.0.smithi136.stdout:988: done (2 left) 2024-09-12T23:41:18.692 INFO:tasks.rados.rados.0.smithi136.stdout:989: copy_from oid 21 from oid 40 current snap is 91 2024-09-12T23:41:18.692 INFO:tasks.rados.rados.0.smithi136.stdout:990: snap_create 2024-09-12T23:41:18.697 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 42 v 258 (ObjNum 125 snap 32 seq_num 125) dirty exists 2024-09-12T23:41:18.704 INFO:tasks.rados.rados.0.smithi136.stdout:986: finishing write tid 1 to smithi13667081-25 2024-09-12T23:41:18.704 INFO:tasks.rados.rados.0.smithi136.stdout:986: finishing write tid 2 to smithi13667081-25 2024-09-12T23:41:18.706 INFO:tasks.rados.rados.0.smithi136.stdout:986: finishing write tid 3 to smithi13667081-25 2024-09-12T23:41:18.706 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 25 v 147 (ObjNum 333 snap 91 seq_num 333) dirty exists 2024-09-12T23:41:18.706 INFO:tasks.rados.rados.0.smithi136.stdout:986: left oid 25 (ObjNum 333 snap 91 seq_num 333) 2024-09-12T23:41:18.713 INFO:tasks.rados.rados.0.smithi136.stdout:989: finishing copy_from racing read to smithi13667081-21 2024-09-12T23:41:18.714 INFO:tasks.rados.rados.0.smithi136.stdout:989: finishing copy_from to smithi13667081-21 2024-09-12T23:41:18.714 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 21 v 562 (ObjNum 331 snap 90 seq_num 331) dirty exists 2024-09-12T23:41:18.985 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:18 smithi181 bash[24776]: cluster 2024-09-12T23:41:17.437079+0000 mgr.y (mgr.15123) 11 : cluster [DBG] pgmap v17: 121 pgs: 121 active+clean; 116 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.1 MiB/s wr, 39 op/s 2024-09-12T23:41:18.986 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:18 smithi181 bash[24776]: cluster 2024-09-12T23:41:17.641382+0000 mon.a (mon.0) 1391 : cluster [DBG] osdmap e242: 8 total, 8 up, 8 in 2024-09-12T23:41:18.986 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:18 smithi181 bash[24776]: audit 2024-09-12T23:41:17.700041+0000 mon.b (mon.1) 135 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:41:18.986 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:18 smithi181 bash[24776]: audit 2024-09-12T23:41:17.719521+0000 mon.b (mon.1) 136 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-12T23:41:18.986 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:18 smithi181 bash[24776]: audit 2024-09-12T23:41:17.828890+0000 mon.b (mon.1) 137 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:41:18.986 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:18 smithi181 bash[24776]: audit 2024-09-12T23:41:17.829320+0000 mon.b (mon.1) 138 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:41:18.986 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:18 smithi181 bash[24776]: audit 2024-09-12T23:41:17.829627+0000 mon.b (mon.1) 139 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:41:18.986 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:18 smithi181 bash[24776]: audit 2024-09-12T23:41:17.832422+0000 mon.b (mon.1) 140 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:41:18.986 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:18 smithi181 bash[24776]: audit 2024-09-12T23:41:17.832818+0000 mon.b (mon.1) 141 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:41:18.986 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:18 smithi181 bash[24776]: audit 2024-09-12T23:41:17.833119+0000 mon.b (mon.1) 142 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:41:18.986 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:18 smithi181 bash[24776]: audit 2024-09-12T23:41:17.835367+0000 mon.b (mon.1) 143 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:41:18.986 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:18 smithi181 bash[24776]: audit 2024-09-12T23:41:17.835751+0000 mon.b (mon.1) 144 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:41:18.987 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:18 smithi181 bash[24776]: audit 2024-09-12T23:41:17.836087+0000 mon.b (mon.1) 145 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:41:18.987 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:18 smithi181 bash[24776]: audit 2024-09-12T23:41:17.850346+0000 mon.a (mon.0) 1392 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:41:18.987 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:18 smithi181 bash[24776]: audit 2024-09-12T23:41:17.850477+0000 mon.a (mon.0) 1393 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:41:18.987 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:18 smithi181 bash[24776]: audit 2024-09-12T23:41:17.851928+0000 mon.b (mon.1) 146 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:41:18.987 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:18 smithi181 bash[24776]: audit 2024-09-12T23:41:17.852282+0000 mon.b (mon.1) 147 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:41:18.987 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:18 smithi181 bash[24776]: audit 2024-09-12T23:41:17.852582+0000 mon.b (mon.1) 148 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:41:18.987 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:18 smithi181 bash[24776]: audit 2024-09-12T23:41:17.854820+0000 mon.b (mon.1) 149 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:41:18.987 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:18 smithi181 bash[24776]: audit 2024-09-12T23:41:17.854943+0000 mon.b (mon.1) 150 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:41:18.987 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:18 smithi181 bash[24776]: audit 2024-09-12T23:41:17.855306+0000 mon.b (mon.1) 151 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:41:18.987 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:18 smithi181 bash[24776]: audit 2024-09-12T23:41:17.855638+0000 mon.b (mon.1) 152 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:41:18.987 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:18 smithi181 bash[24776]: audit 2024-09-12T23:41:17.855988+0000 mon.b (mon.1) 153 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:41:18.987 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:18 smithi181 bash[24776]: audit 2024-09-12T23:41:17.856620+0000 mon.c (mon.2) 70 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:41:18.987 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:18 smithi181 bash[24776]: audit 2024-09-12T23:41:17.856786+0000 mon.c (mon.2) 71 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:41:18.987 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:18 smithi181 bash[24776]: audit 2024-09-12T23:41:17.857953+0000 mon.b (mon.1) 154 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:41:18.988 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:18 smithi181 bash[24776]: audit 2024-09-12T23:41:17.858300+0000 mon.b (mon.1) 155 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:41:18.988 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:18 smithi181 bash[24776]: audit 2024-09-12T23:41:17.858605+0000 mon.b (mon.1) 156 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:41:19.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[22601]: cluster 2024-09-12T23:41:17.437079+0000 mgr.y (mgr.15123) 11 : cluster [DBG] pgmap v17: 121 pgs: 121 active+clean; 116 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.1 MiB/s wr, 39 op/s 2024-09-12T23:41:19.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[22601]: cluster 2024-09-12T23:41:17.641382+0000 mon.a (mon.0) 1391 : cluster [DBG] osdmap e242: 8 total, 8 up, 8 in 2024-09-12T23:41:19.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[22601]: audit 2024-09-12T23:41:17.700041+0000 mon.b (mon.1) 135 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:41:19.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[22601]: audit 2024-09-12T23:41:17.719521+0000 mon.b (mon.1) 136 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-12T23:41:19.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[22601]: audit 2024-09-12T23:41:17.828890+0000 mon.b (mon.1) 137 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:41:19.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[22601]: audit 2024-09-12T23:41:17.829320+0000 mon.b (mon.1) 138 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:41:19.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[22601]: audit 2024-09-12T23:41:17.829627+0000 mon.b (mon.1) 139 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:41:19.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[22601]: audit 2024-09-12T23:41:17.832422+0000 mon.b (mon.1) 140 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:41:19.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[22601]: audit 2024-09-12T23:41:17.832818+0000 mon.b (mon.1) 141 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:41:19.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[22601]: audit 2024-09-12T23:41:17.833119+0000 mon.b (mon.1) 142 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:41:19.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[22601]: audit 2024-09-12T23:41:17.835367+0000 mon.b (mon.1) 143 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:41:19.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[22601]: audit 2024-09-12T23:41:17.835751+0000 mon.b (mon.1) 144 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:41:19.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[22601]: audit 2024-09-12T23:41:17.836087+0000 mon.b (mon.1) 145 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:41:19.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[22601]: audit 2024-09-12T23:41:17.850346+0000 mon.a (mon.0) 1392 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:41:19.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[22601]: audit 2024-09-12T23:41:17.850477+0000 mon.a (mon.0) 1393 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:41:19.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[22601]: audit 2024-09-12T23:41:17.851928+0000 mon.b (mon.1) 146 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:41:19.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[22601]: audit 2024-09-12T23:41:17.852282+0000 mon.b (mon.1) 147 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:41:19.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[22601]: audit 2024-09-12T23:41:17.852582+0000 mon.b (mon.1) 148 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:41:19.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[22601]: audit 2024-09-12T23:41:17.854820+0000 mon.b (mon.1) 149 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:41:19.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[32189]: cluster 2024-09-12T23:41:17.437079+0000 mgr.y (mgr.15123) 11 : cluster [DBG] pgmap v17: 121 pgs: 121 active+clean; 116 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.1 MiB/s wr, 39 op/s 2024-09-12T23:41:19.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[32189]: cluster 2024-09-12T23:41:17.641382+0000 mon.a (mon.0) 1391 : cluster [DBG] osdmap e242: 8 total, 8 up, 8 in 2024-09-12T23:41:19.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[32189]: audit 2024-09-12T23:41:17.700041+0000 mon.b (mon.1) 135 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:41:19.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[32189]: audit 2024-09-12T23:41:17.719521+0000 mon.b (mon.1) 136 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-12T23:41:19.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[32189]: audit 2024-09-12T23:41:17.828890+0000 mon.b (mon.1) 137 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:41:19.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[32189]: audit 2024-09-12T23:41:17.829320+0000 mon.b (mon.1) 138 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:41:19.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[32189]: audit 2024-09-12T23:41:17.829627+0000 mon.b (mon.1) 139 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:41:19.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[32189]: audit 2024-09-12T23:41:17.832422+0000 mon.b (mon.1) 140 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:41:19.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[32189]: audit 2024-09-12T23:41:17.832818+0000 mon.b (mon.1) 141 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:41:19.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[32189]: audit 2024-09-12T23:41:17.833119+0000 mon.b (mon.1) 142 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:41:19.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[32189]: audit 2024-09-12T23:41:17.835367+0000 mon.b (mon.1) 143 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:41:19.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[32189]: audit 2024-09-12T23:41:17.835751+0000 mon.b (mon.1) 144 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:41:19.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[32189]: audit 2024-09-12T23:41:17.836087+0000 mon.b (mon.1) 145 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:41:19.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[32189]: audit 2024-09-12T23:41:17.850346+0000 mon.a (mon.0) 1392 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:41:19.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[32189]: audit 2024-09-12T23:41:17.850477+0000 mon.a (mon.0) 1393 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:41:19.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[32189]: audit 2024-09-12T23:41:17.851928+0000 mon.b (mon.1) 146 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:41:19.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[32189]: audit 2024-09-12T23:41:17.852282+0000 mon.b (mon.1) 147 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:41:19.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[32189]: audit 2024-09-12T23:41:17.852582+0000 mon.b (mon.1) 148 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:41:19.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[32189]: audit 2024-09-12T23:41:17.854820+0000 mon.b (mon.1) 149 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:41:19.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[32189]: audit 2024-09-12T23:41:17.854943+0000 mon.b (mon.1) 150 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:41:19.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[32189]: audit 2024-09-12T23:41:17.855306+0000 mon.b (mon.1) 151 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:41:19.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[32189]: audit 2024-09-12T23:41:17.855638+0000 mon.b (mon.1) 152 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:41:19.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[32189]: audit 2024-09-12T23:41:17.855988+0000 mon.b (mon.1) 153 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:41:19.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[32189]: audit 2024-09-12T23:41:17.856620+0000 mon.c (mon.2) 70 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:41:19.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[32189]: audit 2024-09-12T23:41:17.856786+0000 mon.c (mon.2) 71 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:41:19.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[32189]: audit 2024-09-12T23:41:17.857953+0000 mon.b (mon.1) 154 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:41:19.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[32189]: audit 2024-09-12T23:41:17.858300+0000 mon.b (mon.1) 155 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:41:19.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[32189]: audit 2024-09-12T23:41:17.858605+0000 mon.b (mon.1) 156 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:41:19.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[22601]: audit 2024-09-12T23:41:17.854943+0000 mon.b (mon.1) 150 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:41:19.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[22601]: audit 2024-09-12T23:41:17.855306+0000 mon.b (mon.1) 151 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:41:19.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[22601]: audit 2024-09-12T23:41:17.855638+0000 mon.b (mon.1) 152 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:41:19.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[22601]: audit 2024-09-12T23:41:17.855988+0000 mon.b (mon.1) 153 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:41:19.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[22601]: audit 2024-09-12T23:41:17.856620+0000 mon.c (mon.2) 70 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:41:19.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[22601]: audit 2024-09-12T23:41:17.856786+0000 mon.c (mon.2) 71 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:41:19.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[22601]: audit 2024-09-12T23:41:17.857953+0000 mon.b (mon.1) 154 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:41:19.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[22601]: audit 2024-09-12T23:41:17.858300+0000 mon.b (mon.1) 155 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:41:19.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:18 smithi136 bash[22601]: audit 2024-09-12T23:41:17.858605+0000 mon.b (mon.1) 156 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:41:19.343 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:18 smithi181 bash[52367]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:41:18] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-12T23:41:19.670 INFO:tasks.rados.rados.0.smithi136.stdout:986: done (3 left) 2024-09-12T23:41:19.671 INFO:tasks.rados.rados.0.smithi136.stdout:987: done (2 left) 2024-09-12T23:41:19.671 INFO:tasks.rados.rados.0.smithi136.stdout:989: done (1 left) 2024-09-12T23:41:19.671 INFO:tasks.rados.rados.0.smithi136.stdout:990: done (0 left) 2024-09-12T23:41:19.671 INFO:tasks.rados.rados.0.smithi136.stdout:991: rollback oid 44 current snap is 92 2024-09-12T23:41:19.671 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 44 to 83 2024-09-12T23:41:19.671 INFO:tasks.rados.rados.0.smithi136.stdout:992: copy_from oid 18 from oid 38 current snap is 92 2024-09-12T23:41:19.671 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 3 current snap is 92 2024-09-12T23:41:19.671 INFO:tasks.rados.rados.0.smithi136.stdout:993: seq_num 335 ranges {0=417792,417792=401408} 2024-09-12T23:41:19.678 INFO:tasks.rados.rados.0.smithi136.stdout:993: writing smithi13667081-3 from 0 to 417792 tid 1 2024-09-12T23:41:19.687 INFO:tasks.rados.rados.0.smithi136.stdout:993: writing smithi13667081-3 from 417792 to 819200 tid 2 2024-09-12T23:41:19.687 INFO:tasks.rados.rados.0.smithi136.stdout:994: snap_remove snap 84 2024-09-12T23:41:20.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:19 smithi136 bash[22601]: cluster 2024-09-12T23:41:18.649239+0000 mon.a (mon.0) 1394 : cluster [DBG] mgrmap e35: y(active, since 11s), standbys: x 2024-09-12T23:41:20.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:19 smithi136 bash[22601]: cluster 2024-09-12T23:41:18.665745+0000 mon.a (mon.0) 1395 : cluster [DBG] osdmap e243: 8 total, 8 up, 8 in 2024-09-12T23:41:20.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:19 smithi136 bash[32189]: cluster 2024-09-12T23:41:18.649239+0000 mon.a (mon.0) 1394 : cluster [DBG] mgrmap e35: y(active, since 11s), standbys: x 2024-09-12T23:41:20.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:19 smithi136 bash[32189]: cluster 2024-09-12T23:41:18.665745+0000 mon.a (mon.0) 1395 : cluster [DBG] osdmap e243: 8 total, 8 up, 8 in 2024-09-12T23:41:20.093 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:19 smithi181 bash[24776]: cluster 2024-09-12T23:41:18.649239+0000 mon.a (mon.0) 1394 : cluster [DBG] mgrmap e35: y(active, since 11s), standbys: x 2024-09-12T23:41:20.093 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:19 smithi181 bash[24776]: cluster 2024-09-12T23:41:18.665745+0000 mon.a (mon.0) 1395 : cluster [DBG] osdmap e243: 8 total, 8 up, 8 in 2024-09-12T23:41:20.505 INFO:tasks.rados.rados.0.smithi136.stdout:991: finishing rollback tid 0 to smithi13667081-44 2024-09-12T23:41:20.505 INFO:tasks.rados.rados.0.smithi136.stdout:991: finishing rollback tid 1 to smithi13667081-44 2024-09-12T23:41:20.505 INFO:tasks.rados.rados.0.smithi136.stdout:993: finishing write tid 1 to smithi13667081-3 2024-09-12T23:41:20.505 INFO:tasks.rados.rados.0.smithi136.stdout:991: finishing rollback tid 2 to smithi13667081-44 2024-09-12T23:41:20.505 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 44 v 570 (ObjNum 129 snap 0 seq_num 4243317768) dirty exists 2024-09-12T23:41:20.506 INFO:tasks.rados.rados.0.smithi136.stdout:993: finishing write tid 2 to smithi13667081-3 2024-09-12T23:41:20.506 INFO:tasks.rados.rados.0.smithi136.stdout:993: finishing write tid 3 to smithi13667081-3 2024-09-12T23:41:20.506 INFO:tasks.rados.rados.0.smithi136.stdout:993: finishing write tid 4 to smithi13667081-3 2024-09-12T23:41:20.506 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 3 v 516 (ObjNum 335 snap 92 seq_num 335) dirty exists 2024-09-12T23:41:20.506 INFO:tasks.rados.rados.0.smithi136.stdout:993: left oid 3 (ObjNum 335 snap 92 seq_num 335) 2024-09-12T23:41:20.506 INFO:tasks.rados.rados.0.smithi136.stdout:992: finishing copy_from racing read to smithi13667081-18 2024-09-12T23:41:20.506 INFO:tasks.rados.rados.0.smithi136.stdout:992: finishing copy_from to smithi13667081-18 2024-09-12T23:41:20.506 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 18 v 439 (ObjNum 300 snap 82 seq_num 300) dirty exists 2024-09-12T23:41:20.506 INFO:tasks.rados.rados.0.smithi136.stdout:991: done (3 left) 2024-09-12T23:41:20.506 INFO:tasks.rados.rados.0.smithi136.stdout:992: done (2 left) 2024-09-12T23:41:20.506 INFO:tasks.rados.rados.0.smithi136.stdout:993: done (1 left) 2024-09-12T23:41:20.506 INFO:tasks.rados.rados.0.smithi136.stdout:994: done (0 left) 2024-09-12T23:41:20.506 INFO:tasks.rados.rados.0.smithi136.stdout:995: snap_create 2024-09-12T23:41:20.937 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:20 smithi136 bash[32189]: cluster 2024-09-12T23:41:19.437797+0000 mgr.y (mgr.15123) 12 : cluster [DBG] pgmap v20: 121 pgs: 121 active+clean; 124 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.5 MiB/s wr, 20 op/s 2024-09-12T23:41:20.937 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:20 smithi136 bash[32189]: cluster 2024-09-12T23:41:19.674060+0000 mon.a (mon.0) 1396 : cluster [DBG] osdmap e244: 8 total, 8 up, 8 in 2024-09-12T23:41:20.937 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:20 smithi136 bash[32189]: cluster 2024-09-12T23:41:20.508138+0000 mon.a (mon.0) 1397 : cluster [DBG] osdmap e245: 8 total, 8 up, 8 in 2024-09-12T23:41:20.937 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:20 smithi136 bash[22601]: cluster 2024-09-12T23:41:19.437797+0000 mgr.y (mgr.15123) 12 : cluster [DBG] pgmap v20: 121 pgs: 121 active+clean; 124 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.5 MiB/s wr, 20 op/s 2024-09-12T23:41:20.937 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:20 smithi136 bash[22601]: cluster 2024-09-12T23:41:19.674060+0000 mon.a (mon.0) 1396 : cluster [DBG] osdmap e244: 8 total, 8 up, 8 in 2024-09-12T23:41:20.937 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:20 smithi136 bash[22601]: cluster 2024-09-12T23:41:20.508138+0000 mon.a (mon.0) 1397 : cluster [DBG] osdmap e245: 8 total, 8 up, 8 in 2024-09-12T23:41:21.093 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:20 smithi181 bash[24776]: cluster 2024-09-12T23:41:19.437797+0000 mgr.y (mgr.15123) 12 : cluster [DBG] pgmap v20: 121 pgs: 121 active+clean; 124 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.5 MiB/s wr, 20 op/s 2024-09-12T23:41:21.093 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:20 smithi181 bash[24776]: cluster 2024-09-12T23:41:19.674060+0000 mon.a (mon.0) 1396 : cluster [DBG] osdmap e244: 8 total, 8 up, 8 in 2024-09-12T23:41:21.093 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:20 smithi181 bash[24776]: cluster 2024-09-12T23:41:20.508138+0000 mon.a (mon.0) 1397 : cluster [DBG] osdmap e245: 8 total, 8 up, 8 in 2024-09-12T23:41:21.506 INFO:tasks.rados.rados.0.smithi136.stdout:995: done (0 left) 2024-09-12T23:41:21.506 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 1 current snap is 93 2024-09-12T23:41:21.506 INFO:tasks.rados.rados.0.smithi136.stdout:996: seq_num 336 ranges {2220032=434176,2654208=401408} 2024-09-12T23:41:21.531 INFO:tasks.rados.rados.0.smithi136.stdout:996: writing smithi13667081-1 from 2220032 to 2654208 tid 1 2024-09-12T23:41:21.534 INFO:tasks.rados.rados.0.smithi136.stdout:996: writing smithi13667081-1 from 2654208 to 3055616 tid 2 2024-09-12T23:41:21.534 INFO:tasks.rados.rados.0.smithi136.stdout:997: read oid 35 snap -1 2024-09-12T23:41:21.534 INFO:tasks.rados.rados.0.smithi136.stdout:997: expect (ObjNum 262 snap 71 seq_num 262) 2024-09-12T23:41:21.534 INFO:tasks.rados.rados.0.smithi136.stdout:998: snap_create 2024-09-12T23:41:21.545 INFO:tasks.rados.rados.0.smithi136.stdout:996: finishing write tid 1 to smithi13667081-1 2024-09-12T23:41:21.546 INFO:tasks.rados.rados.0.smithi136.stdout:996: finishing write tid 2 to smithi13667081-1 2024-09-12T23:41:21.550 INFO:tasks.rados.rados.0.smithi136.stdout:996: finishing write tid 3 to smithi13667081-1 2024-09-12T23:41:21.556 INFO:tasks.rados.rados.0.smithi136.stdout:996: finishing write tid 4 to smithi13667081-1 2024-09-12T23:41:21.556 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 1 v 565 (ObjNum 336 snap 93 seq_num 336) dirty exists 2024-09-12T23:41:21.556 INFO:tasks.rados.rados.0.smithi136.stdout:996: left oid 1 (ObjNum 336 snap 93 seq_num 336) 2024-09-12T23:41:21.563 INFO:tasks.rados.rados.0.smithi136.stdout:997: expect (ObjNum 262 snap 71 seq_num 262) 2024-09-12T23:41:22.520 INFO:tasks.rados.rados.0.smithi136.stdout:996: done (2 left) 2024-09-12T23:41:22.521 INFO:tasks.rados.rados.0.smithi136.stdout:997: done (1 left) 2024-09-12T23:41:22.521 INFO:tasks.rados.rados.0.smithi136.stdout:998: done (0 left) 2024-09-12T23:41:22.521 INFO:tasks.rados.rados.0.smithi136.stdout:999: rollback oid 37 current snap is 94 2024-09-12T23:41:22.521 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 37 to 86 2024-09-12T23:41:22.521 INFO:tasks.rados.rados.0.smithi136.stdout:1000: delete oid 44 current snap is 94 2024-09-12T23:41:22.522 INFO:tasks.rados.rados.0.smithi136.stdout:999: finishing rollback tid 1 to smithi13667081-37 2024-09-12T23:41:22.526 INFO:tasks.rados.rados.0.smithi136.stdout:999: finishing rollback tid 2 to smithi13667081-37 2024-09-12T23:41:22.526 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 37 v 534 (ObjNum 134 snap 35 seq_num 134) dirty exists 2024-09-12T23:41:22.526 INFO:tasks.rados.rados.0.smithi136.stdout:999: done (1 left) 2024-09-12T23:41:22.526 INFO:tasks.rados.rados.0.smithi136.stdout:1000: done (0 left) 2024-09-12T23:41:22.527 INFO:tasks.rados.rados.0.smithi136.stdout:1001: copy_from oid 5 from oid 45 current snap is 94 2024-09-12T23:41:22.527 INFO:tasks.rados.rados.0.smithi136.stdout:1002: read oid 12 snap 78 2024-09-12T23:41:22.527 INFO:tasks.rados.rados.0.smithi136.stdout:1002: expect (ObjNum 264 snap 71 seq_num 264) 2024-09-12T23:41:22.527 INFO:tasks.rados.rados.0.smithi136.stdout:1003: rollback oid 47 current snap is 94 2024-09-12T23:41:22.527 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 47 to 89 2024-09-12T23:41:22.527 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 28 current snap is 94 2024-09-12T23:41:22.527 INFO:tasks.rados.rados.0.smithi136.stdout:1004: seq_num 338 ranges {2736128=466944,3203072=401408} 2024-09-12T23:41:22.547 INFO:tasks.rados.rados.0.smithi136.stdout:1004: writing smithi13667081-28 from 2736128 to 3203072 tid 1 2024-09-12T23:41:22.550 INFO:tasks.rados.rados.0.smithi136.stdout:1004: writing smithi13667081-28 from 3203072 to 3604480 tid 2 2024-09-12T23:41:22.550 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 36 current snap is 94 2024-09-12T23:41:22.550 INFO:tasks.rados.rados.0.smithi136.stdout:1003: finishing rollback tid 0 to smithi13667081-47 2024-09-12T23:41:22.550 INFO:tasks.rados.rados.0.smithi136.stdout:1003: finishing rollback tid 1 to smithi13667081-47 2024-09-12T23:41:22.550 INFO:tasks.rados.rados.0.smithi136.stdout:1003: finishing rollback tid 2 to smithi13667081-47 2024-09-12T23:41:22.551 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 47 v 308 (ObjNum 323 snap 88 seq_num 323) dirty exists 2024-09-12T23:41:22.551 INFO:tasks.rados.rados.0.smithi136.stdout:1001: finishing copy_from racing read to smithi13667081-5 2024-09-12T23:41:22.551 INFO:tasks.rados.rados.0.smithi136.stdout:1002: expect (ObjNum 264 snap 71 seq_num 264) 2024-09-12T23:41:22.593 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:22 smithi181 bash[24776]: cluster 2024-09-12T23:41:21.438350+0000 mgr.y (mgr.15123) 13 : cluster [DBG] pgmap v23: 121 pgs: 121 active+clean; 124 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.5 MiB/s wr, 20 op/s 2024-09-12T23:41:22.593 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:22 smithi181 bash[24776]: cluster 2024-09-12T23:41:21.509479+0000 mon.a (mon.0) 1398 : cluster [DBG] osdmap e246: 8 total, 8 up, 8 in 2024-09-12T23:41:22.594 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:22 smithi181 bash[58696]: ts=2024-09-12T23:41:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:41:22.749 INFO:tasks.rados.rados.0.smithi136.stdout:1001: finishing copy_from to smithi13667081-5 2024-09-12T23:41:22.749 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 5 v 286 (ObjNum 330 snap 90 seq_num 330) dirty exists 2024-09-12T23:41:22.749 INFO:tasks.rados.rados.0.smithi136.stdout:1004: finishing write tid 1 to smithi13667081-28 2024-09-12T23:41:22.749 INFO:tasks.rados.rados.0.smithi136.stdout:1004: finishing write tid 2 to smithi13667081-28 2024-09-12T23:41:22.749 INFO:tasks.rados.rados.0.smithi136.stdout:1004: finishing write tid 3 to smithi13667081-28 2024-09-12T23:41:22.749 INFO:tasks.rados.rados.0.smithi136.stdout:1004: finishing write tid 4 to smithi13667081-28 2024-09-12T23:41:22.749 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 28 v 580 (ObjNum 338 snap 94 seq_num 338) dirty exists 2024-09-12T23:41:22.749 INFO:tasks.rados.rados.0.smithi136.stdout:1004: left oid 28 (ObjNum 338 snap 94 seq_num 338) 2024-09-12T23:41:22.749 INFO:tasks.rados.rados.0.smithi136.stdout:1005: seq_num 339 ranges {4685824=483328,5169152=401408} 2024-09-12T23:41:22.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:22 smithi136 bash[22601]: cluster 2024-09-12T23:41:21.438350+0000 mgr.y (mgr.15123) 13 : cluster [DBG] pgmap v23: 121 pgs: 121 active+clean; 124 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.5 MiB/s wr, 20 op/s 2024-09-12T23:41:22.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:22 smithi136 bash[22601]: cluster 2024-09-12T23:41:21.509479+0000 mon.a (mon.0) 1398 : cluster [DBG] osdmap e246: 8 total, 8 up, 8 in 2024-09-12T23:41:22.755 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:22 smithi136 bash[32189]: cluster 2024-09-12T23:41:21.438350+0000 mgr.y (mgr.15123) 13 : cluster [DBG] pgmap v23: 121 pgs: 121 active+clean; 124 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.5 MiB/s wr, 20 op/s 2024-09-12T23:41:22.755 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:22 smithi136 bash[32189]: cluster 2024-09-12T23:41:21.509479+0000 mon.a (mon.0) 1398 : cluster [DBG] osdmap e246: 8 total, 8 up, 8 in 2024-09-12T23:41:22.789 INFO:tasks.rados.rados.0.smithi136.stdout:1005: writing smithi13667081-36 from 4685824 to 5169152 tid 1 2024-09-12T23:41:22.793 INFO:tasks.rados.rados.0.smithi136.stdout:1005: writing smithi13667081-36 from 5169152 to 5570560 tid 2 2024-09-12T23:41:22.793 INFO:tasks.rados.rados.0.smithi136.stdout:1001: done (4 left) 2024-09-12T23:41:22.793 INFO:tasks.rados.rados.0.smithi136.stdout:1002: done (3 left) 2024-09-12T23:41:22.793 INFO:tasks.rados.rados.0.smithi136.stdout:1003: done (2 left) 2024-09-12T23:41:22.793 INFO:tasks.rados.rados.0.smithi136.stdout:1004: done (1 left) 2024-09-12T23:41:22.794 INFO:tasks.rados.rados.0.smithi136.stdout:1006: snap_remove snap 80 2024-09-12T23:41:23.532 INFO:tasks.rados.rados.0.smithi136.stdout:1005: finishing write tid 1 to smithi13667081-36 2024-09-12T23:41:23.532 INFO:tasks.rados.rados.0.smithi136.stdout:1005: finishing write tid 2 to smithi13667081-36 2024-09-12T23:41:23.532 INFO:tasks.rados.rados.0.smithi136.stdout:1005: finishing write tid 3 to smithi13667081-36 2024-09-12T23:41:23.532 INFO:tasks.rados.rados.0.smithi136.stdout:1005: finishing write tid 4 to smithi13667081-36 2024-09-12T23:41:23.532 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 36 v 289 (ObjNum 339 snap 94 seq_num 339) dirty exists 2024-09-12T23:41:23.532 INFO:tasks.rados.rados.0.smithi136.stdout:1005: left oid 36 (ObjNum 339 snap 94 seq_num 339) 2024-09-12T23:41:23.532 INFO:tasks.rados.rados.0.smithi136.stdout:1005: done (1 left) 2024-09-12T23:41:23.532 INFO:tasks.rados.rados.0.smithi136.stdout:1006: done (0 left) 2024-09-12T23:41:23.532 INFO:tasks.rados.rados.0.smithi136.stdout:1007: rmattr oid 25 current snap is 94 2024-09-12T23:41:23.532 INFO:tasks.rados.rados.0.smithi136.stdout:1007: done (0 left) 2024-09-12T23:41:23.532 INFO:tasks.rados.rados.0.smithi136.stdout:1008: setattr oid 27 current snap is 94 2024-09-12T23:41:23.535 INFO:tasks.rados.rados.0.smithi136.stdout:1009: snap_remove snap 86 2024-09-12T23:41:23.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:23 smithi181 bash[24776]: cluster 2024-09-12T23:41:22.521313+0000 mon.a (mon.0) 1399 : cluster [DBG] osdmap e247: 8 total, 8 up, 8 in 2024-09-12T23:41:23.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:23 smithi181 bash[24776]: audit 2024-09-12T23:41:22.642487+0000 mon.b (mon.1) 157 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:41:23.844 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:23 smithi181 bash[58696]: ts=2024-09-12T23:41:23.509Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi136\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:41:24.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:23 smithi136 bash[22601]: cluster 2024-09-12T23:41:22.521313+0000 mon.a (mon.0) 1399 : cluster [DBG] osdmap e247: 8 total, 8 up, 8 in 2024-09-12T23:41:24.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:23 smithi136 bash[22601]: audit 2024-09-12T23:41:22.642487+0000 mon.b (mon.1) 157 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:41:24.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:23 smithi136 bash[32189]: cluster 2024-09-12T23:41:22.521313+0000 mon.a (mon.0) 1399 : cluster [DBG] osdmap e247: 8 total, 8 up, 8 in 2024-09-12T23:41:24.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:23 smithi136 bash[32189]: audit 2024-09-12T23:41:22.642487+0000 mon.b (mon.1) 157 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:41:24.541 INFO:tasks.rados.rados.0.smithi136.stdout:1009: done (1 left) 2024-09-12T23:41:24.541 INFO:tasks.rados.rados.0.smithi136.stdout:1010: rollback oid 16 current snap is 94 2024-09-12T23:41:24.542 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 16 to 92 2024-09-12T23:41:24.542 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 27 v 578 (ObjNum 308 snap 86 seq_num 308) dirty exists 2024-09-12T23:41:24.542 INFO:tasks.rados.rados.0.smithi136.stdout:1008: done (1 left) 2024-09-12T23:41:24.542 INFO:tasks.rados.rados.0.smithi136.stdout:1011: snap_remove snap 89 2024-09-12T23:41:24.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:24 smithi181 bash[24776]: cluster 2024-09-12T23:41:23.438842+0000 mgr.y (mgr.15123) 14 : cluster [DBG] pgmap v26: 121 pgs: 121 active+clean; 124 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:41:24.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:24 smithi181 bash[24776]: cluster 2024-09-12T23:41:23.532619+0000 mon.a (mon.0) 1400 : cluster [DBG] osdmap e248: 8 total, 8 up, 8 in 2024-09-12T23:41:24.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:24 smithi181 bash[24776]: audit 2024-09-12T23:41:24.349972+0000 mon.a (mon.0) 1401 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:24.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:24 smithi181 bash[24776]: audit 2024-09-12T23:41:24.363219+0000 mon.a (mon.0) 1402 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:24.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:24 smithi181 bash[24776]: audit 2024-09-12T23:41:24.369166+0000 mon.b (mon.1) 158 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:41:24.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:24 smithi181 bash[24776]: audit 2024-09-12T23:41:24.371181+0000 mon.b (mon.1) 159 : audit [INF] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:41:24.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:24 smithi181 bash[24776]: cephadm 2024-09-12T23:41:24.372534+0000 mgr.y (mgr.15123) 15 : cephadm [INF] Updating smithi136:/etc/ceph/ceph.conf 2024-09-12T23:41:24.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:24 smithi181 bash[24776]: cephadm 2024-09-12T23:41:24.372996+0000 mgr.y (mgr.15123) 16 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.conf 2024-09-12T23:41:25.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:24 smithi136 bash[22601]: cluster 2024-09-12T23:41:23.438842+0000 mgr.y (mgr.15123) 14 : cluster [DBG] pgmap v26: 121 pgs: 121 active+clean; 124 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:41:25.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:24 smithi136 bash[22601]: cluster 2024-09-12T23:41:23.532619+0000 mon.a (mon.0) 1400 : cluster [DBG] osdmap e248: 8 total, 8 up, 8 in 2024-09-12T23:41:25.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:24 smithi136 bash[22601]: audit 2024-09-12T23:41:24.349972+0000 mon.a (mon.0) 1401 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:25.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:24 smithi136 bash[22601]: audit 2024-09-12T23:41:24.363219+0000 mon.a (mon.0) 1402 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:25.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:24 smithi136 bash[22601]: audit 2024-09-12T23:41:24.369166+0000 mon.b (mon.1) 158 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:41:25.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:24 smithi136 bash[22601]: audit 2024-09-12T23:41:24.371181+0000 mon.b (mon.1) 159 : audit [INF] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:41:25.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:24 smithi136 bash[22601]: cephadm 2024-09-12T23:41:24.372534+0000 mgr.y (mgr.15123) 15 : cephadm [INF] Updating smithi136:/etc/ceph/ceph.conf 2024-09-12T23:41:25.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:24 smithi136 bash[22601]: cephadm 2024-09-12T23:41:24.372996+0000 mgr.y (mgr.15123) 16 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.conf 2024-09-12T23:41:25.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:24 smithi136 bash[32189]: cluster 2024-09-12T23:41:23.438842+0000 mgr.y (mgr.15123) 14 : cluster [DBG] pgmap v26: 121 pgs: 121 active+clean; 124 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:41:25.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:24 smithi136 bash[32189]: cluster 2024-09-12T23:41:23.532619+0000 mon.a (mon.0) 1400 : cluster [DBG] osdmap e248: 8 total, 8 up, 8 in 2024-09-12T23:41:25.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:24 smithi136 bash[32189]: audit 2024-09-12T23:41:24.349972+0000 mon.a (mon.0) 1401 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:25.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:24 smithi136 bash[32189]: audit 2024-09-12T23:41:24.363219+0000 mon.a (mon.0) 1402 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:25.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:24 smithi136 bash[32189]: audit 2024-09-12T23:41:24.369166+0000 mon.b (mon.1) 158 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:41:25.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:24 smithi136 bash[32189]: audit 2024-09-12T23:41:24.371181+0000 mon.b (mon.1) 159 : audit [INF] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:41:25.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:24 smithi136 bash[32189]: cephadm 2024-09-12T23:41:24.372534+0000 mgr.y (mgr.15123) 15 : cephadm [INF] Updating smithi136:/etc/ceph/ceph.conf 2024-09-12T23:41:25.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:24 smithi136 bash[32189]: cephadm 2024-09-12T23:41:24.372996+0000 mgr.y (mgr.15123) 16 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.conf 2024-09-12T23:41:25.544 INFO:tasks.rados.rados.0.smithi136.stdout:1011: done (1 left) 2024-09-12T23:41:25.544 INFO:tasks.rados.rados.0.smithi136.stdout:1012: read oid 46 snap -1 2024-09-12T23:41:25.544 INFO:tasks.rados.rados.0.smithi136.stdout:1012: expect deleted 2024-09-12T23:41:25.544 INFO:tasks.rados.rados.0.smithi136.stdout:1013: copy_from oid 21 from oid 11 current snap is 94 2024-09-12T23:41:25.544 INFO:tasks.rados.rados.0.smithi136.stdout:1010: finishing rollback tid 0 to smithi13667081-16 2024-09-12T23:41:25.544 INFO:tasks.rados.rados.0.smithi136.stdout:1010: finishing rollback tid 1 to smithi13667081-16 2024-09-12T23:41:25.544 INFO:tasks.rados.rados.0.smithi136.stdout:1010: finishing rollback tid 2 to smithi13667081-16 2024-09-12T23:41:25.544 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 16 v 588 (ObjNum 228 snap 56 seq_num 228) dirty exists 2024-09-12T23:41:25.544 INFO:tasks.rados.rados.0.smithi136.stdout:1010: done (2 left) 2024-09-12T23:41:25.544 INFO:tasks.rados.rados.0.smithi136.stdout:1014: snap_remove snap 75 2024-09-12T23:41:25.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:25 smithi181 bash[24776]: cluster 2024-09-12T23:41:24.542528+0000 mon.a (mon.0) 1403 : cluster [DBG] osdmap e249: 8 total, 8 up, 8 in 2024-09-12T23:41:25.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:25 smithi181 bash[24776]: cephadm 2024-09-12T23:41:24.577188+0000 mgr.y (mgr.15123) 17 : cephadm [INF] Updating smithi181:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-12T23:41:25.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:25 smithi181 bash[24776]: cephadm 2024-09-12T23:41:24.589310+0000 mgr.y (mgr.15123) 18 : cephadm [INF] Updating smithi136:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-12T23:41:25.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:25 smithi181 bash[24776]: cephadm 2024-09-12T23:41:24.714444+0000 mgr.y (mgr.15123) 19 : cephadm [INF] Updating smithi136:/etc/ceph/ceph.client.admin.keyring 2024-09-12T23:41:25.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:25 smithi181 bash[24776]: cephadm 2024-09-12T23:41:24.714604+0000 mgr.y (mgr.15123) 20 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2024-09-12T23:41:25.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:25 smithi181 bash[24776]: cephadm 2024-09-12T23:41:24.834862+0000 mgr.y (mgr.15123) 21 : cephadm [INF] Updating smithi136:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-12T23:41:25.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:25 smithi181 bash[24776]: cephadm 2024-09-12T23:41:24.853395+0000 mgr.y (mgr.15123) 22 : cephadm [INF] Updating smithi181:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-12T23:41:25.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:25 smithi181 bash[24776]: audit 2024-09-12T23:41:24.958047+0000 mon.a (mon.0) 1404 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:25.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:25 smithi181 bash[24776]: audit 2024-09-12T23:41:24.965992+0000 mon.a (mon.0) 1405 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:25.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:25 smithi181 bash[24776]: audit 2024-09-12T23:41:24.987891+0000 mon.a (mon.0) 1406 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:25.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:25 smithi181 bash[24776]: audit 2024-09-12T23:41:24.998699+0000 mon.a (mon.0) 1407 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:25.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:25 smithi181 bash[24776]: audit 2024-09-12T23:41:25.008794+0000 mon.a (mon.0) 1408 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:25.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:25 smithi181 bash[24776]: cephadm 2024-09-12T23:41:25.047681+0000 mgr.y (mgr.15123) 23 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-09-12T23:41:25.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:25 smithi181 bash[24776]: cephadm 2024-09-12T23:41:25.304635+0000 mgr.y (mgr.15123) 24 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi181 2024-09-12T23:41:26.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:25 smithi136 bash[22601]: cluster 2024-09-12T23:41:24.542528+0000 mon.a (mon.0) 1403 : cluster [DBG] osdmap e249: 8 total, 8 up, 8 in 2024-09-12T23:41:26.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:25 smithi136 bash[22601]: cephadm 2024-09-12T23:41:24.577188+0000 mgr.y (mgr.15123) 17 : cephadm [INF] Updating smithi181:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-12T23:41:26.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:25 smithi136 bash[22601]: cephadm 2024-09-12T23:41:24.589310+0000 mgr.y (mgr.15123) 18 : cephadm [INF] Updating smithi136:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-12T23:41:26.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:25 smithi136 bash[22601]: cephadm 2024-09-12T23:41:24.714444+0000 mgr.y (mgr.15123) 19 : cephadm [INF] Updating smithi136:/etc/ceph/ceph.client.admin.keyring 2024-09-12T23:41:26.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:25 smithi136 bash[22601]: cephadm 2024-09-12T23:41:24.714604+0000 mgr.y (mgr.15123) 20 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2024-09-12T23:41:26.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:25 smithi136 bash[22601]: cephadm 2024-09-12T23:41:24.834862+0000 mgr.y (mgr.15123) 21 : cephadm [INF] Updating smithi136:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-12T23:41:26.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:25 smithi136 bash[22601]: cephadm 2024-09-12T23:41:24.853395+0000 mgr.y (mgr.15123) 22 : cephadm [INF] Updating smithi181:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-12T23:41:26.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:25 smithi136 bash[22601]: audit 2024-09-12T23:41:24.958047+0000 mon.a (mon.0) 1404 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:26.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:25 smithi136 bash[22601]: audit 2024-09-12T23:41:24.965992+0000 mon.a (mon.0) 1405 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:26.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:25 smithi136 bash[22601]: audit 2024-09-12T23:41:24.987891+0000 mon.a (mon.0) 1406 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:26.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:25 smithi136 bash[22601]: audit 2024-09-12T23:41:24.998699+0000 mon.a (mon.0) 1407 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:26.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:25 smithi136 bash[22601]: audit 2024-09-12T23:41:25.008794+0000 mon.a (mon.0) 1408 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:26.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:25 smithi136 bash[22601]: cephadm 2024-09-12T23:41:25.047681+0000 mgr.y (mgr.15123) 23 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-09-12T23:41:26.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:25 smithi136 bash[22601]: cephadm 2024-09-12T23:41:25.304635+0000 mgr.y (mgr.15123) 24 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi181 2024-09-12T23:41:26.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:25 smithi136 bash[32189]: cluster 2024-09-12T23:41:24.542528+0000 mon.a (mon.0) 1403 : cluster [DBG] osdmap e249: 8 total, 8 up, 8 in 2024-09-12T23:41:26.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:25 smithi136 bash[32189]: cephadm 2024-09-12T23:41:24.577188+0000 mgr.y (mgr.15123) 17 : cephadm [INF] Updating smithi181:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-12T23:41:26.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:25 smithi136 bash[32189]: cephadm 2024-09-12T23:41:24.589310+0000 mgr.y (mgr.15123) 18 : cephadm [INF] Updating smithi136:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-12T23:41:26.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:25 smithi136 bash[32189]: cephadm 2024-09-12T23:41:24.714444+0000 mgr.y (mgr.15123) 19 : cephadm [INF] Updating smithi136:/etc/ceph/ceph.client.admin.keyring 2024-09-12T23:41:26.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:25 smithi136 bash[32189]: cephadm 2024-09-12T23:41:24.714604+0000 mgr.y (mgr.15123) 20 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2024-09-12T23:41:26.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:25 smithi136 bash[32189]: cephadm 2024-09-12T23:41:24.834862+0000 mgr.y (mgr.15123) 21 : cephadm [INF] Updating smithi136:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-12T23:41:26.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:25 smithi136 bash[32189]: cephadm 2024-09-12T23:41:24.853395+0000 mgr.y (mgr.15123) 22 : cephadm [INF] Updating smithi181:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-12T23:41:26.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:25 smithi136 bash[32189]: audit 2024-09-12T23:41:24.958047+0000 mon.a (mon.0) 1404 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:26.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:25 smithi136 bash[32189]: audit 2024-09-12T23:41:24.965992+0000 mon.a (mon.0) 1405 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:26.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:25 smithi136 bash[32189]: audit 2024-09-12T23:41:24.987891+0000 mon.a (mon.0) 1406 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:26.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:25 smithi136 bash[32189]: audit 2024-09-12T23:41:24.998699+0000 mon.a (mon.0) 1407 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:26.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:25 smithi136 bash[32189]: audit 2024-09-12T23:41:25.008794+0000 mon.a (mon.0) 1408 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:26.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:25 smithi136 bash[32189]: cephadm 2024-09-12T23:41:25.047681+0000 mgr.y (mgr.15123) 23 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-09-12T23:41:26.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:25 smithi136 bash[32189]: cephadm 2024-09-12T23:41:25.304635+0000 mgr.y (mgr.15123) 24 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi181 2024-09-12T23:41:26.555 INFO:tasks.rados.rados.0.smithi136.stdout:1014: done (2 left) 2024-09-12T23:41:26.555 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 17 current snap is 94 2024-09-12T23:41:26.555 INFO:tasks.rados.rados.0.smithi136.stdout:1015: seq_num 340 ranges {679936=499712,1179648=401408} 2024-09-12T23:41:26.561 INFO:tasks.rados.rados.0.smithi136.stdout:1015: writing smithi13667081-17 from 679936 to 1179648 tid 1 2024-09-12T23:41:26.564 INFO:tasks.rados.rados.0.smithi136.stdout:1015: writing smithi13667081-17 from 1179648 to 1581056 tid 2 2024-09-12T23:41:26.564 INFO:tasks.rados.rados.0.smithi136.stdout:1016: delete oid 29 current snap is 94 2024-09-12T23:41:26.565 INFO:tasks.rados.rados.0.smithi136.stdout:1013: finishing copy_from racing read to smithi13667081-21 2024-09-12T23:41:26.565 INFO:tasks.rados.rados.0.smithi136.stdout:1013: finishing copy_from to smithi13667081-21 2024-09-12T23:41:26.565 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 21 v 453 (ObjNum 270 snap 73 seq_num 270) dirty exists 2024-09-12T23:41:26.575 INFO:tasks.rados.rados.0.smithi136.stdout:1012: done (3 left) 2024-09-12T23:41:26.575 INFO:tasks.rados.rados.0.smithi136.stdout:1013: done (2 left) 2024-09-12T23:41:26.575 INFO:tasks.rados.rados.0.smithi136.stdout:1016: done (1 left) 2024-09-12T23:41:26.575 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 2 current snap is 94 2024-09-12T23:41:26.576 INFO:tasks.rados.rados.0.smithi136.stdout:1017: seq_num 342 ranges {4939776=532480,5472256=401408} 2024-09-12T23:41:26.616 INFO:tasks.rados.rados.0.smithi136.stdout:1017: writing smithi13667081-2 from 4939776 to 5472256 tid 1 2024-09-12T23:41:26.619 INFO:tasks.rados.rados.0.smithi136.stdout:1017: writing smithi13667081-2 from 5472256 to 5873664 tid 2 2024-09-12T23:41:26.619 INFO:tasks.rados.rados.0.smithi136.stdout:1018: rollback oid 36 current snap is 94 2024-09-12T23:41:26.619 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 36 to 83 2024-09-12T23:41:26.619 INFO:tasks.rados.rados.0.smithi136.stdout:1015: finishing write tid 1 to smithi13667081-17 2024-09-12T23:41:26.619 INFO:tasks.rados.rados.0.smithi136.stdout:1015: finishing write tid 2 to smithi13667081-17 2024-09-12T23:41:26.619 INFO:tasks.rados.rados.0.smithi136.stdout:1015: finishing write tid 3 to smithi13667081-17 2024-09-12T23:41:26.619 INFO:tasks.rados.rados.0.smithi136.stdout:1015: finishing write tid 4 to smithi13667081-17 2024-09-12T23:41:26.619 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 17 v 604 (ObjNum 340 snap 94 seq_num 340) dirty exists 2024-09-12T23:41:26.620 INFO:tasks.rados.rados.0.smithi136.stdout:1015: left oid 17 (ObjNum 340 snap 94 seq_num 340) 2024-09-12T23:41:26.620 INFO:tasks.rados.rados.0.smithi136.stdout:1015: done (2 left) 2024-09-12T23:41:26.620 INFO:tasks.rados.rados.0.smithi136.stdout:1019: snap_create 2024-09-12T23:41:26.623 INFO:tasks.rados.rados.0.smithi136.stdout:1018: finishing rollback tid 0 to smithi13667081-36 2024-09-12T23:41:26.623 INFO:tasks.rados.rados.0.smithi136.stdout:1017: finishing write tid 1 to smithi13667081-2 2024-09-12T23:41:26.624 INFO:tasks.rados.rados.0.smithi136.stdout:1018: finishing rollback tid 1 to smithi13667081-36 2024-09-12T23:41:26.624 INFO:tasks.rados.rados.0.smithi136.stdout:1017: finishing write tid 2 to smithi13667081-2 2024-09-12T23:41:26.624 INFO:tasks.rados.rados.0.smithi136.stdout:1018: finishing rollback tid 2 to smithi13667081-36 2024-09-12T23:41:26.624 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 36 v 300 (ObjNum 256 snap 70 seq_num 256) dirty exists 2024-09-12T23:41:26.624 INFO:tasks.rados.rados.0.smithi136.stdout:1017: finishing write tid 3 to smithi13667081-2 2024-09-12T23:41:26.628 INFO:tasks.rados.rados.0.smithi136.stdout:1017: finishing write tid 4 to smithi13667081-2 2024-09-12T23:41:26.628 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 2 v 537 (ObjNum 342 snap 94 seq_num 342) dirty exists 2024-09-12T23:41:26.628 INFO:tasks.rados.rados.0.smithi136.stdout:1017: left oid 2 (ObjNum 342 snap 94 seq_num 342) 2024-09-12T23:41:26.838 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:26 smithi181 bash[24776]: cluster 2024-09-12T23:41:25.439547+0000 mgr.y (mgr.15123) 25 : cluster [DBG] pgmap v29: 121 pgs: 121 active+clean; 142 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 3.0 MiB/s wr, 21 op/s 2024-09-12T23:41:26.838 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:26 smithi181 bash[24776]: cluster 2024-09-12T23:41:25.544622+0000 mon.a (mon.0) 1409 : cluster [DBG] osdmap e250: 8 total, 8 up, 8 in 2024-09-12T23:41:26.838 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:26 smithi181 bash[24776]: cluster 2024-09-12T23:41:26.549449+0000 mon.a (mon.0) 1410 : cluster [DBG] osdmap e251: 8 total, 8 up, 8 in 2024-09-12T23:41:27.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:26 smithi136 bash[22601]: cluster 2024-09-12T23:41:25.439547+0000 mgr.y (mgr.15123) 25 : cluster [DBG] pgmap v29: 121 pgs: 121 active+clean; 142 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 3.0 MiB/s wr, 21 op/s 2024-09-12T23:41:27.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:26 smithi136 bash[22601]: cluster 2024-09-12T23:41:25.544622+0000 mon.a (mon.0) 1409 : cluster [DBG] osdmap e250: 8 total, 8 up, 8 in 2024-09-12T23:41:27.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:26 smithi136 bash[22601]: cluster 2024-09-12T23:41:26.549449+0000 mon.a (mon.0) 1410 : cluster [DBG] osdmap e251: 8 total, 8 up, 8 in 2024-09-12T23:41:27.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:26 smithi136 bash[32189]: cluster 2024-09-12T23:41:25.439547+0000 mgr.y (mgr.15123) 25 : cluster [DBG] pgmap v29: 121 pgs: 121 active+clean; 142 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 3.0 MiB/s wr, 21 op/s 2024-09-12T23:41:27.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:26 smithi136 bash[32189]: cluster 2024-09-12T23:41:25.544622+0000 mon.a (mon.0) 1409 : cluster [DBG] osdmap e250: 8 total, 8 up, 8 in 2024-09-12T23:41:27.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:26 smithi136 bash[32189]: cluster 2024-09-12T23:41:26.549449+0000 mon.a (mon.0) 1410 : cluster [DBG] osdmap e251: 8 total, 8 up, 8 in 2024-09-12T23:41:27.573 INFO:tasks.rados.rados.0.smithi136.stdout:1017: done (2 left) 2024-09-12T23:41:27.574 INFO:tasks.rados.rados.0.smithi136.stdout:1018: done (1 left) 2024-09-12T23:41:27.574 INFO:tasks.rados.rados.0.smithi136.stdout:1019: done (0 left) 2024-09-12T23:41:27.574 INFO:tasks.rados.rados.0.smithi136.stdout:1020: rmattr oid 32 current snap is 95 2024-09-12T23:41:27.574 INFO:tasks.rados.rados.0.smithi136.stdout:1020: done (0 left) 2024-09-12T23:41:27.574 INFO:tasks.rados.rados.0.smithi136.stdout:1021: read oid 50 snap -1 2024-09-12T23:41:27.574 INFO:tasks.rados.rados.0.smithi136.stdout:1021: expect (ObjNum 315 snap 86 seq_num 315) 2024-09-12T23:41:27.574 INFO:tasks.rados.rados.0.smithi136.stdout:1022: rollback oid 48 current snap is 95 2024-09-12T23:41:27.574 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 48 to 87 2024-09-12T23:41:27.574 INFO:tasks.rados.rados.0.smithi136.stdout:1023: delete oid 5 current snap is 95 2024-09-12T23:41:27.580 INFO:tasks.rados.rados.0.smithi136.stdout:1022: finishing rollback tid 1 to smithi13667081-48 2024-09-12T23:41:27.580 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 48 v 0 (ObjNum 673 snap 0 seq_num 4242747744) dirty dne 2024-09-12T23:41:27.583 INFO:tasks.rados.rados.0.smithi136.stdout:1022: done (2 left) 2024-09-12T23:41:27.583 INFO:tasks.rados.rados.0.smithi136.stdout:1023: done (1 left) 2024-09-12T23:41:27.584 INFO:tasks.rados.rados.0.smithi136.stdout:1024: delete oid 45 current snap is 95 2024-09-12T23:41:27.586 INFO:tasks.rados.rados.0.smithi136.stdout:1021: expect (ObjNum 315 snap 86 seq_num 315) 2024-09-12T23:41:27.622 INFO:tasks.rados.rados.0.smithi136.stdout:1021: done (1 left) 2024-09-12T23:41:27.622 INFO:tasks.rados.rados.0.smithi136.stdout:1024: done (0 left) 2024-09-12T23:41:27.623 INFO:tasks.rados.rados.0.smithi136.stdout:1025: copy_from oid 25 from oid 31 current snap is 95 2024-09-12T23:41:27.623 INFO:tasks.rados.rados.0.smithi136.stdout:1026: read oid 49 snap 90 2024-09-12T23:41:27.623 INFO:tasks.rados.rados.0.smithi136.stdout:1026: expect deleted 2024-09-12T23:41:27.623 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 23 current snap is 95 2024-09-12T23:41:27.623 INFO:tasks.rados.rados.0.smithi136.stdout:1027: seq_num 345 ranges {8642560=581632,9224192=401408} 2024-09-12T23:41:27.690 INFO:tasks.rados.rados.0.smithi136.stdout:1027: writing smithi13667081-23 from 8642560 to 9224192 tid 1 2024-09-12T23:41:27.692 INFO:tasks.rados.rados.0.smithi136.stdout:1027: writing smithi13667081-23 from 9224192 to 9625600 tid 2 2024-09-12T23:41:27.692 INFO:tasks.rados.rados.0.smithi136.stdout:1028: copy_from oid 32 from oid 42 current snap is 95 2024-09-12T23:41:27.693 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 4 current snap is 95 2024-09-12T23:41:27.693 INFO:tasks.rados.rados.0.smithi136.stdout:1029: seq_num 346 ranges {2916352=598016,3514368=401408} 2024-09-12T23:41:27.714 INFO:tasks.rados.rados.0.smithi136.stdout:1029: writing smithi13667081-4 from 2916352 to 3514368 tid 1 2024-09-12T23:41:27.716 INFO:tasks.rados.rados.0.smithi136.stdout:1029: writing smithi13667081-4 from 3514368 to 3915776 tid 2 2024-09-12T23:41:27.716 INFO:tasks.rados.rados.0.smithi136.stdout:1030: read oid 50 snap 81 2024-09-12T23:41:27.716 INFO:tasks.rados.rados.0.smithi136.stdout:1030: expect deleted 2024-09-12T23:41:27.716 INFO:tasks.rados.rados.0.smithi136.stdout:1031: read oid 39 snap -1 2024-09-12T23:41:27.717 INFO:tasks.rados.rados.0.smithi136.stdout:1031: expect (ObjNum 312 snap 86 seq_num 312) 2024-09-12T23:41:27.717 INFO:tasks.rados.rados.0.smithi136.stdout:1032: read oid 6 snap -1 2024-09-12T23:41:27.717 INFO:tasks.rados.rados.0.smithi136.stdout:1032: expect (ObjNum 254 snap 69 seq_num 254) 2024-09-12T23:41:27.717 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 16 current snap is 95 2024-09-12T23:41:27.717 INFO:tasks.rados.rados.0.smithi136.stdout:1025: finishing copy_from to smithi13667081-25 2024-09-12T23:41:27.717 INFO:tasks.rados.rados.0.smithi136.stdout:1025: got expected ENOENT (src dne) 2024-09-12T23:41:27.717 INFO:tasks.rados.rados.0.smithi136.stdout:1025: finishing copy_from racing read to smithi13667081-25 2024-09-12T23:41:27.717 INFO:tasks.rados.rados.0.smithi136.stdout:1033: seq_num 347 ranges {1425408=614400,2039808=401408} 2024-09-12T23:41:27.728 INFO:tasks.rados.rados.0.smithi136.stdout:1033: writing smithi13667081-16 from 1425408 to 2039808 tid 1 2024-09-12T23:41:27.731 INFO:tasks.rados.rados.0.smithi136.stdout:1033: writing smithi13667081-16 from 2039808 to 2441216 tid 2 2024-09-12T23:41:27.731 INFO:tasks.rados.rados.0.smithi136.stdout:1025: done (8 left) 2024-09-12T23:41:27.731 INFO:tasks.rados.rados.0.smithi136.stdout:1026: done (7 left) 2024-09-12T23:41:27.731 INFO:tasks.rados.rados.0.smithi136.stdout:1034: read oid 20 snap -1 2024-09-12T23:41:27.731 INFO:tasks.rados.rados.0.smithi136.stdout:1034: expect deleted 2024-09-12T23:41:27.732 INFO:tasks.rados.rados.0.smithi136.stdout:1035: setattr oid 38 current snap is 95 2024-09-12T23:41:27.732 INFO:tasks.rados.rados.0.smithi136.stdout:1027: finishing write tid 1 to smithi13667081-23 2024-09-12T23:41:27.732 INFO:tasks.rados.rados.0.smithi136.stdout:1027: finishing write tid 2 to smithi13667081-23 2024-09-12T23:41:27.732 INFO:tasks.rados.rados.0.smithi136.stdout:1027: finishing write tid 3 to smithi13667081-23 2024-09-12T23:41:27.732 INFO:tasks.rados.rados.0.smithi136.stdout:1027: finishing write tid 4 to smithi13667081-23 2024-09-12T23:41:27.732 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 23 v 616 (ObjNum 345 snap 95 seq_num 345) dirty exists 2024-09-12T23:41:27.732 INFO:tasks.rados.rados.0.smithi136.stdout:1027: left oid 23 (ObjNum 345 snap 95 seq_num 345) 2024-09-12T23:41:27.732 INFO:tasks.rados.rados.0.smithi136.stdout:1028: finishing copy_from racing read to smithi13667081-32 2024-09-12T23:41:27.732 INFO:tasks.rados.rados.0.smithi136.stdout:1028: finishing copy_from to smithi13667081-32 2024-09-12T23:41:27.732 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 32 v 258 (ObjNum 125 snap 32 seq_num 125) dirty exists 2024-09-12T23:41:27.732 INFO:tasks.rados.rados.0.smithi136.stdout:1029: finishing write tid 1 to smithi13667081-4 2024-09-12T23:41:27.732 INFO:tasks.rados.rados.0.smithi136.stdout:1029: finishing write tid 2 to smithi13667081-4 2024-09-12T23:41:27.732 INFO:tasks.rados.rados.0.smithi136.stdout:1029: finishing write tid 3 to smithi13667081-4 2024-09-12T23:41:27.732 INFO:tasks.rados.rados.0.smithi136.stdout:1029: finishing write tid 4 to smithi13667081-4 2024-09-12T23:41:27.732 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 4 v 261 (ObjNum 346 snap 95 seq_num 346) dirty exists 2024-09-12T23:41:27.733 INFO:tasks.rados.rados.0.smithi136.stdout:1029: left oid 4 (ObjNum 346 snap 95 seq_num 346) 2024-09-12T23:41:27.733 INFO:tasks.rados.rados.0.smithi136.stdout:1027: done (8 left) 2024-09-12T23:41:27.733 INFO:tasks.rados.rados.0.smithi136.stdout:1028: done (7 left) 2024-09-12T23:41:27.733 INFO:tasks.rados.rados.0.smithi136.stdout:1029: done (6 left) 2024-09-12T23:41:27.733 INFO:tasks.rados.rados.0.smithi136.stdout:1030: done (5 left) 2024-09-12T23:41:27.733 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 5 current snap is 95 2024-09-12T23:41:27.733 INFO:tasks.rados.rados.0.smithi136.stdout:1036: seq_num 348 ranges {0=630784,630784=401408} 2024-09-12T23:41:27.737 INFO:tasks.rados.rados.0.smithi136.stdout:1036: writing smithi13667081-5 from 0 to 630784 tid 1 2024-09-12T23:41:27.740 INFO:tasks.rados.rados.0.smithi136.stdout:1036: writing smithi13667081-5 from 630784 to 1032192 tid 2 2024-09-12T23:41:27.740 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 15 current snap is 95 2024-09-12T23:41:27.740 INFO:tasks.rados.rados.0.smithi136.stdout:1031: expect (ObjNum 312 snap 86 seq_num 312) 2024-09-12T23:41:27.858 INFO:tasks.rados.rados.0.smithi136.stdout:1033: finishing write tid 1 to smithi13667081-16 2024-09-12T23:41:27.858 INFO:tasks.rados.rados.0.smithi136.stdout:1033: finishing write tid 2 to smithi13667081-16 2024-09-12T23:41:27.858 INFO:tasks.rados.rados.0.smithi136.stdout:1033: finishing write tid 3 to smithi13667081-16 2024-09-12T23:41:27.858 INFO:tasks.rados.rados.0.smithi136.stdout:1032: expect (ObjNum 254 snap 69 seq_num 254) 2024-09-12T23:41:27.996 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 38 v 445 (ObjNum 300 snap 82 seq_num 300) dirty exists 2024-09-12T23:41:27.996 INFO:tasks.rados.rados.0.smithi136.stdout:1036: finishing write tid 1 to smithi13667081-5 2024-09-12T23:41:27.996 INFO:tasks.rados.rados.0.smithi136.stdout:1033: finishing write tid 4 to smithi13667081-16 2024-09-12T23:41:27.996 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 16 v 608 (ObjNum 347 snap 95 seq_num 347) dirty exists 2024-09-12T23:41:27.996 INFO:tasks.rados.rados.0.smithi136.stdout:1033: left oid 16 (ObjNum 347 snap 95 seq_num 347) 2024-09-12T23:41:27.996 INFO:tasks.rados.rados.0.smithi136.stdout:1036: finishing write tid 2 to smithi13667081-5 2024-09-12T23:41:27.996 INFO:tasks.rados.rados.0.smithi136.stdout:1036: finishing write tid 3 to smithi13667081-5 2024-09-12T23:41:27.996 INFO:tasks.rados.rados.0.smithi136.stdout:1036: finishing write tid 4 to smithi13667081-5 2024-09-12T23:41:27.996 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 5 v 619 (ObjNum 348 snap 95 seq_num 348) dirty exists 2024-09-12T23:41:27.996 INFO:tasks.rados.rados.0.smithi136.stdout:1036: left oid 5 (ObjNum 348 snap 95 seq_num 348) 2024-09-12T23:41:27.996 INFO:tasks.rados.rados.0.smithi136.stdout:1037: seq_num 349 ranges {2269184=655360,2924544=401408} 2024-09-12T23:41:28.017 INFO:tasks.rados.rados.0.smithi136.stdout:1037: writing smithi13667081-15 from 2269184 to 2924544 tid 1 2024-09-12T23:41:28.021 INFO:tasks.rados.rados.0.smithi136.stdout:1037: writing smithi13667081-15 from 2924544 to 3325952 tid 2 2024-09-12T23:41:28.021 INFO:tasks.rados.rados.0.smithi136.stdout:1031: done (6 left) 2024-09-12T23:41:28.021 INFO:tasks.rados.rados.0.smithi136.stdout:1032: done (5 left) 2024-09-12T23:41:28.021 INFO:tasks.rados.rados.0.smithi136.stdout:1033: done (4 left) 2024-09-12T23:41:28.021 INFO:tasks.rados.rados.0.smithi136.stdout:1034: done (3 left) 2024-09-12T23:41:28.021 INFO:tasks.rados.rados.0.smithi136.stdout:1035: done (2 left) 2024-09-12T23:41:28.021 INFO:tasks.rados.rados.0.smithi136.stdout:1036: done (1 left) 2024-09-12T23:41:28.021 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 16 current snap is 95 2024-09-12T23:41:28.021 INFO:tasks.rados.rados.0.smithi136.stdout:1038: seq_num 350 ranges {2441216=671744,3112960=401408} 2024-09-12T23:41:28.044 INFO:tasks.rados.rados.0.smithi136.stdout:1038: writing smithi13667081-16 from 2441216 to 3112960 tid 1 2024-09-12T23:41:28.047 INFO:tasks.rados.rados.0.smithi136.stdout:1038: writing smithi13667081-16 from 3112960 to 3514368 tid 2 2024-09-12T23:41:28.047 INFO:tasks.rados.rados.0.smithi136.stdout:1039: snap_remove snap 92 2024-09-12T23:41:28.578 INFO:tasks.rados.rados.0.smithi136.stdout:1039: done (2 left) 2024-09-12T23:41:28.578 INFO:tasks.rados.rados.0.smithi136.stdout:1040: setattr oid 39 current snap is 95 2024-09-12T23:41:28.578 INFO:tasks.rados.rados.0.smithi136.stdout:1037: finishing write tid 1 to smithi13667081-15 2024-09-12T23:41:28.578 INFO:tasks.rados.rados.0.smithi136.stdout:1037: finishing write tid 2 to smithi13667081-15 2024-09-12T23:41:28.578 INFO:tasks.rados.rados.0.smithi136.stdout:1037: finishing write tid 3 to smithi13667081-15 2024-09-12T23:41:28.578 INFO:tasks.rados.rados.0.smithi136.stdout:1037: finishing write tid 4 to smithi13667081-15 2024-09-12T23:41:28.578 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 15 v 532 (ObjNum 349 snap 95 seq_num 349) dirty exists 2024-09-12T23:41:28.578 INFO:tasks.rados.rados.0.smithi136.stdout:1037: left oid 15 (ObjNum 349 snap 95 seq_num 349) 2024-09-12T23:41:28.578 INFO:tasks.rados.rados.0.smithi136.stdout:1038: finishing write tid 1 to smithi13667081-16 2024-09-12T23:41:28.578 INFO:tasks.rados.rados.0.smithi136.stdout:1038: finishing write tid 2 to smithi13667081-16 2024-09-12T23:41:28.578 INFO:tasks.rados.rados.0.smithi136.stdout:1038: finishing write tid 3 to smithi13667081-16 2024-09-12T23:41:28.579 INFO:tasks.rados.rados.0.smithi136.stdout:1038: finishing write tid 4 to smithi13667081-16 2024-09-12T23:41:28.579 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 16 v 611 (ObjNum 350 snap 95 seq_num 350) dirty exists 2024-09-12T23:41:28.579 INFO:tasks.rados.rados.0.smithi136.stdout:1038: left oid 16 (ObjNum 350 snap 95 seq_num 350) 2024-09-12T23:41:28.579 INFO:tasks.rados.rados.0.smithi136.stdout:1037: done (2 left) 2024-09-12T23:41:28.579 INFO:tasks.rados.rados.0.smithi136.stdout:1038: done (1 left) 2024-09-12T23:41:28.579 INFO:tasks.rados.rados.0.smithi136.stdout:1041: rollback oid 24 current snap is 95 2024-09-12T23:41:28.580 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 24 to 81 2024-09-12T23:41:28.580 INFO:tasks.rados.rados.0.smithi136.stdout:1042: read oid 32 snap -1 2024-09-12T23:41:28.580 INFO:tasks.rados.rados.0.smithi136.stdout:1042: expect (ObjNum 125 snap 32 seq_num 125) 2024-09-12T23:41:28.580 INFO:tasks.rados.rados.0.smithi136.stdout:1043: snap_create 2024-09-12T23:41:28.590 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 39 v 157 (ObjNum 312 snap 86 seq_num 312) dirty exists 2024-09-12T23:41:28.594 INFO:tasks.rados.rados.0.smithi136.stdout:1041: finishing rollback tid 0 to smithi13667081-24 2024-09-12T23:41:28.597 INFO:tasks.rados.rados.0.smithi136.stdout:1041: finishing rollback tid 1 to smithi13667081-24 2024-09-12T23:41:28.597 INFO:tasks.rados.rados.0.smithi136.stdout:1041: finishing rollback tid 2 to smithi13667081-24 2024-09-12T23:41:28.597 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 24 v 385 (ObjNum 283 snap 78 seq_num 283) dirty exists 2024-09-12T23:41:28.598 INFO:tasks.rados.rados.0.smithi136.stdout:1042: expect (ObjNum 125 snap 32 seq_num 125) 2024-09-12T23:41:28.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:28 smithi181 bash[24776]: cluster 2024-09-12T23:41:27.440241+0000 mgr.y (mgr.15123) 26 : cluster [DBG] pgmap v32: 121 pgs: 121 active+clean; 142 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 3.0 MiB/s wr, 21 op/s 2024-09-12T23:41:28.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:28 smithi181 bash[24776]: cluster 2024-09-12T23:41:27.575930+0000 mon.a (mon.0) 1411 : cluster [DBG] osdmap e252: 8 total, 8 up, 8 in 2024-09-12T23:41:28.873 INFO:teuthology.orchestra.run.smithi136.stdout:true 2024-09-12T23:41:29.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:28 smithi136 bash[22601]: cluster 2024-09-12T23:41:27.440241+0000 mgr.y (mgr.15123) 26 : cluster [DBG] pgmap v32: 121 pgs: 121 active+clean; 142 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 3.0 MiB/s wr, 21 op/s 2024-09-12T23:41:29.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:28 smithi136 bash[22601]: cluster 2024-09-12T23:41:27.575930+0000 mon.a (mon.0) 1411 : cluster [DBG] osdmap e252: 8 total, 8 up, 8 in 2024-09-12T23:41:29.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:28 smithi136 bash[32189]: cluster 2024-09-12T23:41:27.440241+0000 mgr.y (mgr.15123) 26 : cluster [DBG] pgmap v32: 121 pgs: 121 active+clean; 142 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 3.0 MiB/s wr, 21 op/s 2024-09-12T23:41:29.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:28 smithi136 bash[32189]: cluster 2024-09-12T23:41:27.575930+0000 mon.a (mon.0) 1411 : cluster [DBG] osdmap e252: 8 total, 8 up, 8 in 2024-09-12T23:41:29.188 INFO:teuthology.orchestra.run.smithi136.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-12T23:41:29.188 INFO:teuthology.orchestra.run.smithi136.stdout:alertmanager.a smithi136 *:9093,9094 running (56s) 15s ago 7m 13.5M - 0.25.0 c8568f914cd2 b5197f020eeb 2024-09-12T23:41:29.188 INFO:teuthology.orchestra.run.smithi136.stdout:grafana.a smithi181 *:3000 running (44s) 15s ago 6m 73.7M - 9.4.7 954c08fa6188 256c181be984 2024-09-12T23:41:29.188 INFO:teuthology.orchestra.run.smithi136.stdout:mds.a smithi136 running (8m) 15s ago 8m 13.7M - 17.2.7-1627-gbbdadfa3 214985049bdb 713b9795ad19 2024-09-12T23:41:29.188 INFO:teuthology.orchestra.run.smithi136.stdout:mds.b smithi181 running (8m) 15s ago 8m 16.7M - 17.2.7-1627-gbbdadfa3 214985049bdb ad13179f0868 2024-09-12T23:41:29.188 INFO:teuthology.orchestra.run.smithi136.stdout:mgr.x smithi181 *:8443,9283 running (98s) 15s ago 15m 243M - 18.2.4-837-g7a853b17 f21ba1eea74e 761bc101e333 2024-09-12T23:41:29.188 INFO:teuthology.orchestra.run.smithi136.stdout:mgr.y smithi136 *:8443,9283,8765 running (38s) 15s ago 17m 457M - 18.2.4-837-g7a853b17 f21ba1eea74e 272f10b42fc3 2024-09-12T23:41:29.188 INFO:teuthology.orchestra.run.smithi136.stdout:mon.a smithi136 running (17m) 15s ago 18m 59.4M 2048M 17.2.7-1627-gbbdadfa3 214985049bdb 0462f4f61475 2024-09-12T23:41:29.188 INFO:teuthology.orchestra.run.smithi136.stdout:mon.b smithi181 running (15m) 15s ago 15m 49.8M 2048M 17.2.7-1627-gbbdadfa3 214985049bdb 8b3d784af845 2024-09-12T23:41:29.188 INFO:teuthology.orchestra.run.smithi136.stdout:mon.c smithi136 running (15m) 15s ago 15m 51.5M 2048M 17.2.7-1627-gbbdadfa3 214985049bdb 647b028ea963 2024-09-12T23:41:29.189 INFO:teuthology.orchestra.run.smithi136.stdout:node-exporter.a smithi136 *:9100 running (62s) 15s ago 7m 7707k - 1.5.0 0da6a335fe13 ccfc37c96620 2024-09-12T23:41:29.189 INFO:teuthology.orchestra.run.smithi136.stdout:node-exporter.b smithi181 *:9100 running (49s) 15s ago 7m 7799k - 1.5.0 0da6a335fe13 80bf0e383230 2024-09-12T23:41:29.189 INFO:teuthology.orchestra.run.smithi136.stdout:osd.0 smithi136 running (14m) 15s ago 14m 140M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb cbc0b7df0115 2024-09-12T23:41:29.189 INFO:teuthology.orchestra.run.smithi136.stdout:osd.1 smithi136 running (13m) 15s ago 13m 148M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb c4daee7da836 2024-09-12T23:41:29.189 INFO:teuthology.orchestra.run.smithi136.stdout:osd.2 smithi136 running (13m) 15s ago 13m 153M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 4ab8fd79cdb4 2024-09-12T23:41:29.189 INFO:teuthology.orchestra.run.smithi136.stdout:osd.3 smithi136 running (12m) 15s ago 12m 122M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 9ceb14a27521 2024-09-12T23:41:29.189 INFO:teuthology.orchestra.run.smithi136.stdout:osd.4 smithi181 running (11m) 15s ago 11m 165M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb a35cdca79055 2024-09-12T23:41:29.189 INFO:teuthology.orchestra.run.smithi136.stdout:osd.5 smithi181 running (10m) 15s ago 10m 127M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 8f2e8d494fef 2024-09-12T23:41:29.189 INFO:teuthology.orchestra.run.smithi136.stdout:osd.6 smithi181 running (9m) 15s ago 9m 77.7M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 9c6bec406408 2024-09-12T23:41:29.189 INFO:teuthology.orchestra.run.smithi136.stdout:osd.7 smithi181 running (8m) 15s ago 8m 105M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb bc0632ae2145 2024-09-12T23:41:29.189 INFO:teuthology.orchestra.run.smithi136.stdout:prometheus.a smithi181 *:9095 running (51s) 15s ago 7m 41.4M - 2.43.0 a07b618ecd1d c44e669360fc 2024-09-12T23:41:29.253 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:28 smithi181 bash[52367]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:41:28] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-12T23:41:29.532 INFO:teuthology.orchestra.run.smithi136.stdout:{ 2024-09-12T23:41:29.533 INFO:teuthology.orchestra.run.smithi136.stdout: "mon": { 2024-09-12T23:41:29.533 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 3 2024-09-12T23:41:29.533 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:41:29.533 INFO:teuthology.orchestra.run.smithi136.stdout: "mgr": { 2024-09-12T23:41:29.533 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 2 2024-09-12T23:41:29.533 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:41:29.533 INFO:teuthology.orchestra.run.smithi136.stdout: "osd": { 2024-09-12T23:41:29.533 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 8 2024-09-12T23:41:29.533 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:41:29.533 INFO:teuthology.orchestra.run.smithi136.stdout: "mds": { 2024-09-12T23:41:29.534 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-12T23:41:29.534 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:41:29.534 INFO:teuthology.orchestra.run.smithi136.stdout: "overall": { 2024-09-12T23:41:29.534 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 13, 2024-09-12T23:41:29.534 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 2 2024-09-12T23:41:29.534 INFO:teuthology.orchestra.run.smithi136.stdout: } 2024-09-12T23:41:29.534 INFO:teuthology.orchestra.run.smithi136.stdout:} 2024-09-12T23:41:29.580 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:29 smithi181 systemd[1]: Stopping Ceph prometheus.a for da58a47c-715d-11ef-bceb-c7b262605968... 2024-09-12T23:41:29.581 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:29 smithi181 bash[58696]: ts=2024-09-12T23:41:29.381Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-12T23:41:29.581 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:29 smithi181 bash[58696]: ts=2024-09-12T23:41:29.381Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-12T23:41:29.581 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:29 smithi181 bash[58696]: ts=2024-09-12T23:41:29.381Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-12T23:41:29.581 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:29 smithi181 bash[58696]: ts=2024-09-12T23:41:29.381Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-12T23:41:29.581 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:29 smithi181 bash[58696]: ts=2024-09-12T23:41:29.381Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-12T23:41:29.581 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:29 smithi181 bash[58696]: ts=2024-09-12T23:41:29.381Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-12T23:41:29.581 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:29 smithi181 bash[58696]: ts=2024-09-12T23:41:29.382Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-12T23:41:29.581 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:29 smithi181 bash[58696]: ts=2024-09-12T23:41:29.382Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-12T23:41:29.581 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:29 smithi181 bash[58696]: ts=2024-09-12T23:41:29.382Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-12T23:41:29.581 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:29 smithi181 bash[58696]: ts=2024-09-12T23:41:29.382Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-12T23:41:29.581 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:29 smithi181 bash[58696]: ts=2024-09-12T23:41:29.382Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-12T23:41:29.581 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:29 smithi181 bash[58696]: ts=2024-09-12T23:41:29.382Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-12T23:41:29.592 INFO:tasks.rados.rados.0.smithi136.stdout:1040: done (3 left) 2024-09-12T23:41:29.592 INFO:tasks.rados.rados.0.smithi136.stdout:1041: done (2 left) 2024-09-12T23:41:29.592 INFO:tasks.rados.rados.0.smithi136.stdout:1042: done (1 left) 2024-09-12T23:41:29.593 INFO:tasks.rados.rados.0.smithi136.stdout:1043: done (0 left) 2024-09-12T23:41:29.593 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 37 current snap is 96 2024-09-12T23:41:29.593 INFO:tasks.rados.rados.0.smithi136.stdout:1044: seq_num 351 ranges {6045696=688128,6733824=401408} 2024-09-12T23:41:29.655 INFO:tasks.rados.rados.0.smithi136.stdout:1044: writing smithi13667081-37 from 6045696 to 6733824 tid 1 2024-09-12T23:41:29.658 INFO:tasks.rados.rados.0.smithi136.stdout:1044: writing smithi13667081-37 from 6733824 to 7135232 tid 2 2024-09-12T23:41:29.658 INFO:tasks.rados.rados.0.smithi136.stdout:1045: rmattr oid 2 current snap is 96 2024-09-12T23:41:29.658 INFO:tasks.rados.rados.0.smithi136.stdout:1045: done (1 left) 2024-09-12T23:41:29.658 INFO:tasks.rados.rados.0.smithi136.stdout:1046: read oid 2 snap 90 2024-09-12T23:41:29.659 INFO:tasks.rados.rados.0.smithi136.stdout:1046: expect (ObjNum 280 snap 77 seq_num 280) 2024-09-12T23:41:29.659 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 1 current snap is 96 2024-09-12T23:41:29.659 INFO:tasks.rados.rados.0.smithi136.stdout:1047: seq_num 352 ranges {3055616=704512,3760128=401408} 2024-09-12T23:41:29.683 INFO:tasks.rados.rados.0.smithi136.stdout:1047: writing smithi13667081-1 from 3055616 to 3760128 tid 1 2024-09-12T23:41:29.686 INFO:tasks.rados.rados.0.smithi136.stdout:1047: writing smithi13667081-1 from 3760128 to 4161536 tid 2 2024-09-12T23:41:29.686 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 32 current snap is 96 2024-09-12T23:41:29.686 INFO:tasks.rados.rados.0.smithi136.stdout:1044: finishing write tid 1 to smithi13667081-37 2024-09-12T23:41:29.686 INFO:tasks.rados.rados.0.smithi136.stdout:1044: finishing write tid 2 to smithi13667081-37 2024-09-12T23:41:29.686 INFO:tasks.rados.rados.0.smithi136.stdout:1044: finishing write tid 3 to smithi13667081-37 2024-09-12T23:41:29.686 INFO:tasks.rados.rados.0.smithi136.stdout:1044: finishing write tid 4 to smithi13667081-37 2024-09-12T23:41:29.686 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 37 v 540 (ObjNum 351 snap 96 seq_num 351) dirty exists 2024-09-12T23:41:29.686 INFO:tasks.rados.rados.0.smithi136.stdout:1044: left oid 37 (ObjNum 351 snap 96 seq_num 351) 2024-09-12T23:41:29.686 INFO:tasks.rados.rados.0.smithi136.stdout:1048: seq_num 353 ranges {2105344=720896,2826240=401408} 2024-09-12T23:41:29.702 INFO:tasks.rados.rados.0.smithi136.stdout:1048: writing smithi13667081-32 from 2105344 to 2826240 tid 1 2024-09-12T23:41:29.704 INFO:tasks.rados.rados.0.smithi136.stdout:1048: writing smithi13667081-32 from 2826240 to 3227648 tid 2 2024-09-12T23:41:29.704 INFO:tasks.rados.rados.0.smithi136.stdout:1044: done (3 left) 2024-09-12T23:41:29.705 INFO:tasks.rados.rados.0.smithi136.stdout:1049: rmattr oid 49 current snap is 96 2024-09-12T23:41:29.705 INFO:tasks.rados.rados.0.smithi136.stdout:1049: done (3 left) 2024-09-12T23:41:29.705 INFO:tasks.rados.rados.0.smithi136.stdout:1050: setattr oid 50 current snap is 96 2024-09-12T23:41:29.705 INFO:tasks.rados.rados.0.smithi136.stdout:1047: finishing write tid 1 to smithi13667081-1 2024-09-12T23:41:29.705 INFO:tasks.rados.rados.0.smithi136.stdout:1047: finishing write tid 2 to smithi13667081-1 2024-09-12T23:41:29.705 INFO:tasks.rados.rados.0.smithi136.stdout:1047: finishing write tid 3 to smithi13667081-1 2024-09-12T23:41:29.705 INFO:tasks.rados.rados.0.smithi136.stdout:1047: finishing write tid 4 to smithi13667081-1 2024-09-12T23:41:29.705 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 1 v 568 (ObjNum 352 snap 96 seq_num 352) dirty exists 2024-09-12T23:41:29.705 INFO:tasks.rados.rados.0.smithi136.stdout:1047: left oid 1 (ObjNum 352 snap 96 seq_num 352) 2024-09-12T23:41:29.705 INFO:tasks.rados.rados.0.smithi136.stdout:1046: expect (ObjNum 280 snap 77 seq_num 280) 2024-09-12T23:41:29.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:29 smithi181 bash[24776]: cluster 2024-09-12T23:41:28.582228+0000 mon.a (mon.0) 1412 : cluster [DBG] osdmap e253: 8 total, 8 up, 8 in 2024-09-12T23:41:29.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:29 smithi181 bash[24776]: audit 2024-09-12T23:41:28.862161+0000 mgr.y (mgr.15123) 27 : audit [DBG] from='client.15192 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:41:29.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:29 smithi181 bash[24776]: audit 2024-09-12T23:41:29.156041+0000 mgr.y (mgr.15123) 28 : audit [DBG] from='client.25121 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:41:29.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:29 smithi181 bash[24776]: audit 2024-09-12T23:41:29.536886+0000 mon.a (mon.0) 1413 : audit [DBG] from='client.? 172.21.15.136:0/1699145481' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:41:29.931 INFO:tasks.rados.rados.0.smithi136.stdout:1048: finishing write tid 1 to smithi13667081-32 2024-09-12T23:41:29.931 INFO:tasks.rados.rados.0.smithi136.stdout:1048: finishing write tid 2 to smithi13667081-32 2024-09-12T23:41:29.931 INFO:tasks.rados.rados.0.smithi136.stdout:1048: finishing write tid 3 to smithi13667081-32 2024-09-12T23:41:29.931 INFO:tasks.rados.rados.0.smithi136.stdout:1048: finishing write tid 4 to smithi13667081-32 2024-09-12T23:41:29.931 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 32 v 264 (ObjNum 353 snap 96 seq_num 353) dirty exists 2024-09-12T23:41:29.931 INFO:tasks.rados.rados.0.smithi136.stdout:1048: left oid 32 (ObjNum 353 snap 96 seq_num 353) 2024-09-12T23:41:29.932 INFO:tasks.rados.rados.0.smithi136.stdout:1046: done (3 left) 2024-09-12T23:41:29.932 INFO:tasks.rados.rados.0.smithi136.stdout:1047: done (2 left) 2024-09-12T23:41:29.932 INFO:tasks.rados.rados.0.smithi136.stdout:1048: done (1 left) 2024-09-12T23:41:29.932 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 15 current snap is 96 2024-09-12T23:41:29.932 INFO:tasks.rados.rados.0.smithi136.stdout:1051: seq_num 354 ranges {3325952=737280,4063232=401408} 2024-09-12T23:41:29.957 INFO:tasks.rados.rados.0.smithi136.stdout:1051: writing smithi13667081-15 from 3325952 to 4063232 tid 1 2024-09-12T23:41:29.960 INFO:tasks.rados.rados.0.smithi136.stdout:1051: writing smithi13667081-15 from 4063232 to 4464640 tid 2 2024-09-12T23:41:29.960 INFO:tasks.rados.rados.0.smithi136.stdout:1052: delete oid 39 current snap is 96 2024-09-12T23:41:29.961 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 50 v 446 (ObjNum 315 snap 86 seq_num 315) dirty exists 2024-09-12T23:41:29.964 INFO:tasks.rados.rados.0.smithi136.stdout:1050: done (2 left) 2024-09-12T23:41:29.964 INFO:tasks.rados.rados.0.smithi136.stdout:1052: done (1 left) 2024-09-12T23:41:29.964 INFO:tasks.rados.rados.0.smithi136.stdout:1053: read oid 5 snap -1 2024-09-12T23:41:29.964 INFO:tasks.rados.rados.0.smithi136.stdout:1053: expect (ObjNum 348 snap 95 seq_num 348) 2024-09-12T23:41:29.964 INFO:tasks.rados.rados.0.smithi136.stdout:1054: rmattr oid 32 current snap is 96 2024-09-12T23:41:29.964 INFO:tasks.rados.rados.0.smithi136.stdout:1054: done (2 left) 2024-09-12T23:41:29.964 INFO:tasks.rados.rados.0.smithi136.stdout:1055: rmattr oid 47 current snap is 96 2024-09-12T23:41:29.964 INFO:tasks.rados.rados.0.smithi136.stdout:1056: setattr oid 21 current snap is 96 2024-09-12T23:41:29.965 INFO:tasks.rados.rados.0.smithi136.stdout:1051: finishing write tid 1 to smithi13667081-15 2024-09-12T23:41:29.966 INFO:tasks.rados.rados.0.smithi136.stdout:1051: finishing write tid 2 to smithi13667081-15 2024-09-12T23:41:29.970 INFO:tasks.rados.rados.0.smithi136.stdout:1057: read oid 25 snap 90 2024-09-12T23:41:29.970 INFO:tasks.rados.rados.0.smithi136.stdout:1057: expect deleted 2024-09-12T23:41:29.970 INFO:tasks.rados.rados.0.smithi136.stdout:1058: rollback oid 29 current snap is 96 2024-09-12T23:41:29.970 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 29 to 93 2024-09-12T23:41:29.970 INFO:tasks.rados.rados.0.smithi136.stdout:1059: setattr oid 17 current snap is 96 2024-09-12T23:41:29.970 INFO:tasks.rados.rados.0.smithi136.stdout:1051: finishing write tid 3 to smithi13667081-15 2024-09-12T23:41:29.970 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 47 v 328 (ObjNum 323 snap 88 seq_num 323) dirty exists 2024-09-12T23:41:29.971 INFO:tasks.rados.rados.0.smithi136.stdout:1051: finishing write tid 4 to smithi13667081-15 2024-09-12T23:41:29.971 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 15 v 538 (ObjNum 354 snap 96 seq_num 354) dirty exists 2024-09-12T23:41:29.971 INFO:tasks.rados.rados.0.smithi136.stdout:1051: left oid 15 (ObjNum 354 snap 96 seq_num 354) 2024-09-12T23:41:29.971 INFO:tasks.rados.rados.0.smithi136.stdout:1051: done (6 left) 2024-09-12T23:41:29.971 INFO:tasks.rados.rados.0.smithi136.stdout:1055: done (5 left) 2024-09-12T23:41:29.971 INFO:tasks.rados.rados.0.smithi136.stdout:1057: done (4 left) 2024-09-12T23:41:29.971 INFO:tasks.rados.rados.0.smithi136.stdout:1060: rollback oid 10 current snap is 96 2024-09-12T23:41:29.971 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 10 to 83 2024-09-12T23:41:29.971 INFO:tasks.rados.rados.0.smithi136.stdout:1061: rollback oid 25 current snap is 96 2024-09-12T23:41:29.971 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 25 to 83 2024-09-12T23:41:29.971 INFO:tasks.rados.rados.0.smithi136.stdout:1062: read oid 44 snap -1 2024-09-12T23:41:29.971 INFO:tasks.rados.rados.0.smithi136.stdout:1062: expect deleted 2024-09-12T23:41:29.971 INFO:tasks.rados.rados.0.smithi136.stdout:1063: snap_create 2024-09-12T23:41:29.972 INFO:tasks.rados.rados.0.smithi136.stdout:1058: finishing rollback tid 1 to smithi13667081-29 2024-09-12T23:41:29.972 INFO:tasks.rados.rados.0.smithi136.stdout:1058: finishing rollback tid 2 to smithi13667081-29 2024-09-12T23:41:29.972 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 29 v 448 (ObjNum 311 snap 86 seq_num 311) dirty exists 2024-09-12T23:41:29.972 INFO:tasks.rados.rados.0.smithi136.stdout:1060: finishing rollback tid 1 to smithi13667081-10 2024-09-12T23:41:29.973 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 10 v 215 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-12T23:41:29.973 INFO:tasks.rados.rados.0.smithi136.stdout:1061: finishing rollback tid 0 to smithi13667081-25 2024-09-12T23:41:29.974 INFO:tasks.rados.rados.0.smithi136.stdout:1061: finishing rollback tid 1 to smithi13667081-25 2024-09-12T23:41:29.974 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 25 v 162 (ObjNum 97 snap 0 seq_num 2878127530) dirty dne 2024-09-12T23:41:29.974 INFO:tasks.rados.rados.0.smithi136.stdout:1053: expect (ObjNum 348 snap 95 seq_num 348) 2024-09-12T23:41:30.002 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 21 v 569 (ObjNum 270 snap 73 seq_num 270) dirty exists 2024-09-12T23:41:30.002 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 17 v 629 (ObjNum 340 snap 94 seq_num 340) dirty exists 2024-09-12T23:41:30.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:29 smithi136 bash[32189]: cluster 2024-09-12T23:41:28.582228+0000 mon.a (mon.0) 1412 : cluster [DBG] osdmap e253: 8 total, 8 up, 8 in 2024-09-12T23:41:30.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:29 smithi136 bash[32189]: audit 2024-09-12T23:41:28.862161+0000 mgr.y (mgr.15123) 27 : audit [DBG] from='client.15192 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:41:30.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:29 smithi136 bash[32189]: audit 2024-09-12T23:41:29.156041+0000 mgr.y (mgr.15123) 28 : audit [DBG] from='client.25121 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:41:30.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:29 smithi136 bash[32189]: audit 2024-09-12T23:41:29.536886+0000 mon.a (mon.0) 1413 : audit [DBG] from='client.? 172.21.15.136:0/1699145481' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:41:30.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:29 smithi136 bash[22601]: cluster 2024-09-12T23:41:28.582228+0000 mon.a (mon.0) 1412 : cluster [DBG] osdmap e253: 8 total, 8 up, 8 in 2024-09-12T23:41:30.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:29 smithi136 bash[22601]: audit 2024-09-12T23:41:28.862161+0000 mgr.y (mgr.15123) 27 : audit [DBG] from='client.15192 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:41:30.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:29 smithi136 bash[22601]: audit 2024-09-12T23:41:29.156041+0000 mgr.y (mgr.15123) 28 : audit [DBG] from='client.25121 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:41:30.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:29 smithi136 bash[22601]: audit 2024-09-12T23:41:29.536886+0000 mon.a (mon.0) 1413 : audit [DBG] from='client.? 172.21.15.136:0/1699145481' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:41:30.226 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:29 smithi181 bash[60382]: ceph-da58a47c-715d-11ef-bceb-c7b262605968-prometheus-a 2024-09-12T23:41:30.226 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:30 smithi181 bash[60443]: Error response from daemon: No such container: ceph-da58a47c-715d-11ef-bceb-c7b262605968-prometheus-a 2024-09-12T23:41:30.226 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:30 smithi181 systemd[1]: ceph-da58a47c-715d-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-12T23:41:30.226 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:30 smithi181 systemd[1]: Stopped Ceph prometheus.a for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:41:30.227 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:30 smithi181 systemd[1]: Started Ceph prometheus.a for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:41:30.508 INFO:tasks.rados.rados.0.smithi136.stdout:1053: done (7 left) 2024-09-12T23:41:30.508 INFO:tasks.rados.rados.0.smithi136.stdout:1056: done (6 left) 2024-09-12T23:41:30.509 INFO:tasks.rados.rados.0.smithi136.stdout:1058: done (5 left) 2024-09-12T23:41:30.509 INFO:tasks.rados.rados.0.smithi136.stdout:1059: done (4 left) 2024-09-12T23:41:30.509 INFO:tasks.rados.rados.0.smithi136.stdout:1060: done (3 left) 2024-09-12T23:41:30.509 INFO:tasks.rados.rados.0.smithi136.stdout:1061: done (2 left) 2024-09-12T23:41:30.509 INFO:tasks.rados.rados.0.smithi136.stdout:1062: done (1 left) 2024-09-12T23:41:30.509 INFO:tasks.rados.rados.0.smithi136.stdout:1063: done (0 left) 2024-09-12T23:41:30.509 INFO:tasks.rados.rados.0.smithi136.stdout:1064: copy_from oid 2 from oid 18 current snap is 97 2024-09-12T23:41:30.509 INFO:tasks.rados.rados.0.smithi136.stdout:1065: snap_create 2024-09-12T23:41:30.543 INFO:tasks.rados.rados.0.smithi136.stdout:1064: finishing copy_from racing read to smithi13667081-2 2024-09-12T23:41:30.544 INFO:tasks.rados.rados.0.smithi136.stdout:1064: finishing copy_from to smithi13667081-2 2024-09-12T23:41:30.544 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 2 v 439 (ObjNum 300 snap 82 seq_num 300) dirty exists 2024-09-12T23:41:30.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:30 smithi181 bash[24776]: cluster 2024-09-12T23:41:29.441167+0000 mgr.y (mgr.15123) 29 : cluster [DBG] pgmap v35: 121 pgs: 2 active+clean+snaptrim_wait, 6 active+clean+snaptrim, 113 active+clean; 151 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 2.9 MiB/s wr, 25 op/s 2024-09-12T23:41:30.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:30 smithi181 bash[24776]: cluster 2024-09-12T23:41:29.596785+0000 mon.a (mon.0) 1414 : cluster [DBG] osdmap e254: 8 total, 8 up, 8 in 2024-09-12T23:41:30.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:30 smithi181 bash[24776]: audit 2024-09-12T23:41:30.161803+0000 mon.a (mon.0) 1415 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:30.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:30 smithi181 bash[24776]: audit 2024-09-12T23:41:30.171570+0000 mon.a (mon.0) 1416 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:30.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:30 smithi181 bash[24776]: audit 2024-09-12T23:41:30.179519+0000 mgr.y (mgr.15123) 30 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-12T23:41:30.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:30 smithi181 bash[24776]: audit 2024-09-12T23:41:30.180090+0000 mon.b (mon.1) 160 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-12T23:41:30.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:30 smithi181 bash[24776]: audit 2024-09-12T23:41:30.219709+0000 mon.b (mon.1) 161 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:41:30.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:30 smithi181 bash[24776]: cluster 2024-09-12T23:41:30.510920+0000 mon.a (mon.0) 1417 : cluster [DBG] osdmap e255: 8 total, 8 up, 8 in 2024-09-12T23:41:31.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:30 smithi136 bash[32189]: cluster 2024-09-12T23:41:29.441167+0000 mgr.y (mgr.15123) 29 : cluster [DBG] pgmap v35: 121 pgs: 2 active+clean+snaptrim_wait, 6 active+clean+snaptrim, 113 active+clean; 151 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 2.9 MiB/s wr, 25 op/s 2024-09-12T23:41:31.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:30 smithi136 bash[32189]: cluster 2024-09-12T23:41:29.596785+0000 mon.a (mon.0) 1414 : cluster [DBG] osdmap e254: 8 total, 8 up, 8 in 2024-09-12T23:41:31.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:30 smithi136 bash[32189]: audit 2024-09-12T23:41:30.161803+0000 mon.a (mon.0) 1415 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:31.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:30 smithi136 bash[32189]: audit 2024-09-12T23:41:30.171570+0000 mon.a (mon.0) 1416 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:31.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:30 smithi136 bash[32189]: audit 2024-09-12T23:41:30.179519+0000 mgr.y (mgr.15123) 30 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-12T23:41:31.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:30 smithi136 bash[32189]: audit 2024-09-12T23:41:30.180090+0000 mon.b (mon.1) 160 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-12T23:41:31.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:30 smithi136 bash[32189]: audit 2024-09-12T23:41:30.219709+0000 mon.b (mon.1) 161 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:41:31.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:30 smithi136 bash[32189]: cluster 2024-09-12T23:41:30.510920+0000 mon.a (mon.0) 1417 : cluster [DBG] osdmap e255: 8 total, 8 up, 8 in 2024-09-12T23:41:31.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:30 smithi136 bash[22601]: cluster 2024-09-12T23:41:29.441167+0000 mgr.y (mgr.15123) 29 : cluster [DBG] pgmap v35: 121 pgs: 2 active+clean+snaptrim_wait, 6 active+clean+snaptrim, 113 active+clean; 151 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 2.9 MiB/s wr, 25 op/s 2024-09-12T23:41:31.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:30 smithi136 bash[22601]: cluster 2024-09-12T23:41:29.596785+0000 mon.a (mon.0) 1414 : cluster [DBG] osdmap e254: 8 total, 8 up, 8 in 2024-09-12T23:41:31.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:30 smithi136 bash[22601]: audit 2024-09-12T23:41:30.161803+0000 mon.a (mon.0) 1415 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:31.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:30 smithi136 bash[22601]: audit 2024-09-12T23:41:30.171570+0000 mon.a (mon.0) 1416 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:31.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:30 smithi136 bash[22601]: audit 2024-09-12T23:41:30.179519+0000 mgr.y (mgr.15123) 30 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-12T23:41:31.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:30 smithi136 bash[22601]: audit 2024-09-12T23:41:30.180090+0000 mon.b (mon.1) 160 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-12T23:41:31.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:30 smithi136 bash[22601]: audit 2024-09-12T23:41:30.219709+0000 mon.b (mon.1) 161 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:41:31.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:30 smithi136 bash[22601]: cluster 2024-09-12T23:41:30.510920+0000 mon.a (mon.0) 1417 : cluster [DBG] osdmap e255: 8 total, 8 up, 8 in 2024-09-12T23:41:31.508 INFO:tasks.rados.rados.0.smithi136.stdout:1064: done (1 left) 2024-09-12T23:41:31.508 INFO:tasks.rados.rados.0.smithi136.stdout:1065: done (0 left) 2024-09-12T23:41:31.508 INFO:tasks.rados.rados.0.smithi136.stdout:1066: read oid 19 snap -1 2024-09-12T23:41:31.508 INFO:tasks.rados.rados.0.smithi136.stdout:1066: expect (ObjNum 332 snap 91 seq_num 332) 2024-09-12T23:41:31.509 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 44 current snap is 98 2024-09-12T23:41:31.509 INFO:tasks.rados.rados.0.smithi136.stdout:1067: seq_num 356 ranges {0=770048,770048=401408} 2024-09-12T23:41:31.516 INFO:tasks.rados.rados.0.smithi136.stdout:1067: writing smithi13667081-44 from 0 to 770048 tid 1 2024-09-12T23:41:31.520 INFO:tasks.rados.rados.0.smithi136.stdout:1067: writing smithi13667081-44 from 770048 to 1171456 tid 2 2024-09-12T23:41:31.520 INFO:tasks.rados.rados.0.smithi136.stdout:1068: snap_create 2024-09-12T23:41:31.525 INFO:tasks.rados.rados.0.smithi136.stdout:1066: expect (ObjNum 332 snap 91 seq_num 332) 2024-09-12T23:41:31.553 INFO:tasks.rados.rados.0.smithi136.stdout:1067: finishing write tid 1 to smithi13667081-44 2024-09-12T23:41:31.553 INFO:tasks.rados.rados.0.smithi136.stdout:1067: finishing write tid 2 to smithi13667081-44 2024-09-12T23:41:31.553 INFO:tasks.rados.rados.0.smithi136.stdout:1067: finishing write tid 3 to smithi13667081-44 2024-09-12T23:41:31.554 INFO:tasks.rados.rados.0.smithi136.stdout:1067: finishing write tid 4 to smithi13667081-44 2024-09-12T23:41:31.554 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 44 v 620 (ObjNum 356 snap 98 seq_num 356) dirty exists 2024-09-12T23:41:31.554 INFO:tasks.rados.rados.0.smithi136.stdout:1067: left oid 44 (ObjNum 356 snap 98 seq_num 356) 2024-09-12T23:41:32.020 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:31 smithi181 bash[60471]: ts=2024-09-12T23:41:31.878Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-12T23:41:32.020 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:31 smithi181 bash[60471]: ts=2024-09-12T23:41:31.879Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-12T23:41:32.020 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:31 smithi181 bash[60471]: ts=2024-09-12T23:41:31.879Z caller=main.go:570 level=info host_details="(Linux 5.15.0-121-generic #131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024 x86_64 smithi181 (none))" 2024-09-12T23:41:32.020 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:31 smithi181 bash[60471]: ts=2024-09-12T23:41:31.879Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-12T23:41:32.020 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:31 smithi181 bash[60471]: ts=2024-09-12T23:41:31.879Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-12T23:41:32.020 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:31 smithi181 bash[60471]: ts=2024-09-12T23:41:31.880Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-12T23:41:32.020 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:31 smithi181 bash[60471]: ts=2024-09-12T23:41:31.881Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-12T23:41:32.020 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:31 smithi181 bash[60471]: ts=2024-09-12T23:41:31.882Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-12T23:41:32.021 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:31 smithi181 bash[60471]: ts=2024-09-12T23:41:31.882Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-12T23:41:32.021 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:31 smithi181 bash[60471]: ts=2024-09-12T23:41:31.886Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-12T23:41:32.021 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:31 smithi181 bash[60471]: ts=2024-09-12T23:41:31.886Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.882µs 2024-09-12T23:41:32.021 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:31 smithi181 bash[60471]: ts=2024-09-12T23:41:31.886Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-12T23:41:32.021 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:31 smithi181 bash[60471]: ts=2024-09-12T23:41:31.887Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-09-12T23:41:32.021 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:31 smithi181 bash[60471]: ts=2024-09-12T23:41:31.917Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-09-12T23:41:32.021 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:31 smithi181 bash[60471]: ts=2024-09-12T23:41:31.924Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-09-12T23:41:32.021 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:31 smithi181 bash[60471]: ts=2024-09-12T23:41:31.924Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-09-12T23:41:32.021 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:31 smithi181 bash[60471]: ts=2024-09-12T23:41:31.924Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=24.665µs wal_replay_duration=37.611753ms wbl_replay_duration=177ns total_replay_duration=37.656809ms 2024-09-12T23:41:32.021 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:31 smithi181 bash[60471]: ts=2024-09-12T23:41:31.927Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-12T23:41:32.021 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:31 smithi181 bash[60471]: ts=2024-09-12T23:41:31.927Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-12T23:41:32.021 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:31 smithi181 bash[60471]: ts=2024-09-12T23:41:31.927Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-12T23:41:32.021 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:31 smithi181 bash[60471]: ts=2024-09-12T23:41:31.952Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=25.127457ms db_storage=1.011µs remote_storage=1.213µs web_handler=391ns query_engine=693ns scrape=238.9µs scrape_sd=94.488µs notify=22.672µs notify_sd=47.202µs rules=24.272924ms tracing=5.82µs 2024-09-12T23:41:32.022 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:31 smithi181 bash[60471]: ts=2024-09-12T23:41:31.952Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-12T23:41:32.022 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:31 smithi181 bash[60471]: ts=2024-09-12T23:41:31.952Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-12T23:41:32.520 INFO:tasks.rados.rados.0.smithi136.stdout:1066: done (2 left) 2024-09-12T23:41:32.520 INFO:tasks.rados.rados.0.smithi136.stdout:1067: done (1 left) 2024-09-12T23:41:32.521 INFO:tasks.rados.rados.0.smithi136.stdout:1068: done (0 left) 2024-09-12T23:41:32.521 INFO:tasks.rados.rados.0.smithi136.stdout:1069: copy_from oid 44 from oid 47 current snap is 99 2024-09-12T23:41:32.521 INFO:tasks.rados.rados.0.smithi136.stdout:1070: copy_from oid 18 from oid 21 current snap is 99 2024-09-12T23:41:32.521 INFO:tasks.rados.rados.0.smithi136.stdout:1071: snap_create 2024-09-12T23:41:32.550 INFO:tasks.rados.rados.0.smithi136.stdout:1069: finishing copy_from racing read to smithi13667081-44 2024-09-12T23:41:32.552 INFO:tasks.rados.rados.0.smithi136.stdout:1069: finishing copy_from to smithi13667081-44 2024-09-12T23:41:32.552 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 44 v 328 (ObjNum 323 snap 88 seq_num 323) dirty exists 2024-09-12T23:41:32.556 INFO:tasks.rados.rados.0.smithi136.stdout:1070: finishing copy_from racing read to smithi13667081-18 2024-09-12T23:41:32.557 INFO:tasks.rados.rados.0.smithi136.stdout:1070: finishing copy_from to smithi13667081-18 2024-09-12T23:41:32.557 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 18 v 569 (ObjNum 270 snap 73 seq_num 270) dirty exists 2024-09-12T23:41:32.593 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:32 smithi181 bash[24776]: cluster 2024-09-12T23:41:31.441869+0000 mgr.y (mgr.15123) 31 : cluster [DBG] pgmap v38: 121 pgs: 2 active+clean+snaptrim_wait, 6 active+clean+snaptrim, 113 active+clean; 151 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 2.9 MiB/s wr, 25 op/s 2024-09-12T23:41:32.593 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:32 smithi181 bash[24776]: cluster 2024-09-12T23:41:31.512459+0000 mon.a (mon.0) 1418 : cluster [DBG] osdmap e256: 8 total, 8 up, 8 in 2024-09-12T23:41:32.593 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:32 smithi181 bash[60471]: ts=2024-09-12T23:41:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:41:32.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:32 smithi136 bash[22601]: cluster 2024-09-12T23:41:31.441869+0000 mgr.y (mgr.15123) 31 : cluster [DBG] pgmap v38: 121 pgs: 2 active+clean+snaptrim_wait, 6 active+clean+snaptrim, 113 active+clean; 151 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 2.9 MiB/s wr, 25 op/s 2024-09-12T23:41:32.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:32 smithi136 bash[22601]: cluster 2024-09-12T23:41:31.512459+0000 mon.a (mon.0) 1418 : cluster [DBG] osdmap e256: 8 total, 8 up, 8 in 2024-09-12T23:41:32.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:32 smithi136 bash[32189]: cluster 2024-09-12T23:41:31.441869+0000 mgr.y (mgr.15123) 31 : cluster [DBG] pgmap v38: 121 pgs: 2 active+clean+snaptrim_wait, 6 active+clean+snaptrim, 113 active+clean; 151 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 2.9 MiB/s wr, 25 op/s 2024-09-12T23:41:32.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:32 smithi136 bash[32189]: cluster 2024-09-12T23:41:31.512459+0000 mon.a (mon.0) 1418 : cluster [DBG] osdmap e256: 8 total, 8 up, 8 in 2024-09-12T23:41:33.534 INFO:tasks.rados.rados.0.smithi136.stdout:1069: done (2 left) 2024-09-12T23:41:33.534 INFO:tasks.rados.rados.0.smithi136.stdout:1070: done (1 left) 2024-09-12T23:41:33.534 INFO:tasks.rados.rados.0.smithi136.stdout:1071: done (0 left) 2024-09-12T23:41:33.534 INFO:tasks.rados.rados.0.smithi136.stdout:1072: read oid 16 snap 87 2024-09-12T23:41:33.535 INFO:tasks.rados.rados.0.smithi136.stdout:1072: expect (ObjNum 228 snap 56 seq_num 228) 2024-09-12T23:41:33.535 INFO:tasks.rados.rados.0.smithi136.stdout:1073: setattr oid 20 current snap is 100 2024-09-12T23:41:33.537 INFO:tasks.rados.rados.0.smithi136.stdout:1074: copy_from oid 23 from oid 36 current snap is 100 2024-09-12T23:41:33.537 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 17 current snap is 100 2024-09-12T23:41:33.537 INFO:tasks.rados.rados.0.smithi136.stdout:1075: seq_num 357 ranges {1581056=786432,2367488=401408} 2024-09-12T23:41:33.569 INFO:tasks.rados.rados.0.smithi136.stdout:1075: writing smithi13667081-17 from 1581056 to 2367488 tid 1 2024-09-12T23:41:33.573 INFO:tasks.rados.rados.0.smithi136.stdout:1075: writing smithi13667081-17 from 2367488 to 2768896 tid 2 2024-09-12T23:41:33.573 INFO:tasks.rados.rados.0.smithi136.stdout:1076: setattr oid 13 current snap is 100 2024-09-12T23:41:33.573 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 20 v 284 (ObjNum 33 snap 0 seq_num 4243647216) dirty exists 2024-09-12T23:41:33.573 INFO:tasks.rados.rados.0.smithi136.stdout:1072: expect (ObjNum 228 snap 56 seq_num 228) 2024-09-12T23:41:33.648 INFO:tasks.rados.rados.0.smithi136.stdout:1074: finishing copy_from racing read to smithi13667081-23 2024-09-12T23:41:33.648 INFO:tasks.rados.rados.0.smithi136.stdout:1075: finishing write tid 1 to smithi13667081-17 2024-09-12T23:41:33.648 INFO:tasks.rados.rados.0.smithi136.stdout:1074: finishing copy_from to smithi13667081-23 2024-09-12T23:41:33.649 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 23 v 300 (ObjNum 256 snap 70 seq_num 256) dirty exists 2024-09-12T23:41:33.649 INFO:tasks.rados.rados.0.smithi136.stdout:1075: finishing write tid 2 to smithi13667081-17 2024-09-12T23:41:33.649 INFO:tasks.rados.rados.0.smithi136.stdout:1075: finishing write tid 3 to smithi13667081-17 2024-09-12T23:41:33.649 INFO:tasks.rados.rados.0.smithi136.stdout:1075: finishing write tid 4 to smithi13667081-17 2024-09-12T23:41:33.649 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 17 v 635 (ObjNum 357 snap 100 seq_num 357) dirty exists 2024-09-12T23:41:33.649 INFO:tasks.rados.rados.0.smithi136.stdout:1075: left oid 17 (ObjNum 357 snap 100 seq_num 357) 2024-09-12T23:41:33.649 INFO:tasks.rados.rados.0.smithi136.stdout:1072: done (4 left) 2024-09-12T23:41:33.649 INFO:tasks.rados.rados.0.smithi136.stdout:1073: done (3 left) 2024-09-12T23:41:33.649 INFO:tasks.rados.rados.0.smithi136.stdout:1074: done (2 left) 2024-09-12T23:41:33.649 INFO:tasks.rados.rados.0.smithi136.stdout:1075: done (1 left) 2024-09-12T23:41:33.649 INFO:tasks.rados.rados.0.smithi136.stdout:1077: snap_create 2024-09-12T23:41:33.657 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 13 v 449 (ObjNum 295 snap 80 seq_num 295) dirty exists 2024-09-12T23:41:33.843 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:33 smithi181 bash[60471]: ts=2024-09-12T23:41:33.509Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi136\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:41:33.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:33 smithi181 bash[24776]: cluster 2024-09-12T23:41:32.524522+0000 mon.a (mon.0) 1419 : cluster [DBG] osdmap e257: 8 total, 8 up, 8 in 2024-09-12T23:41:33.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:33 smithi181 bash[24776]: cephadm 2024-09-12T23:41:32.693068+0000 mgr.y (mgr.15123) 32 : cephadm [INF] Upgrade: Updating mgr.x 2024-09-12T23:41:33.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:33 smithi181 bash[24776]: audit 2024-09-12T23:41:32.695155+0000 mon.a (mon.0) 1420 : audit [INF] from='mgr.15123 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-12T23:41:33.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:33 smithi181 bash[24776]: audit 2024-09-12T23:41:32.695189+0000 mon.b (mon.1) 162 : audit [INF] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-12T23:41:33.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:33 smithi181 bash[24776]: audit 2024-09-12T23:41:32.699255+0000 mon.b (mon.1) 163 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-12T23:41:33.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:33 smithi181 bash[24776]: audit 2024-09-12T23:41:32.701271+0000 mon.b (mon.1) 164 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:41:33.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:33 smithi181 bash[24776]: cephadm 2024-09-12T23:41:32.702130+0000 mgr.y (mgr.15123) 33 : cephadm [INF] Deploying daemon mgr.x on smithi181 2024-09-12T23:41:34.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:33 smithi136 bash[32189]: cluster 2024-09-12T23:41:32.524522+0000 mon.a (mon.0) 1419 : cluster [DBG] osdmap e257: 8 total, 8 up, 8 in 2024-09-12T23:41:34.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:33 smithi136 bash[32189]: cephadm 2024-09-12T23:41:32.693068+0000 mgr.y (mgr.15123) 32 : cephadm [INF] Upgrade: Updating mgr.x 2024-09-12T23:41:34.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:33 smithi136 bash[32189]: audit 2024-09-12T23:41:32.695155+0000 mon.a (mon.0) 1420 : audit [INF] from='mgr.15123 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-12T23:41:34.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:33 smithi136 bash[32189]: audit 2024-09-12T23:41:32.695189+0000 mon.b (mon.1) 162 : audit [INF] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-12T23:41:34.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:33 smithi136 bash[32189]: audit 2024-09-12T23:41:32.699255+0000 mon.b (mon.1) 163 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-12T23:41:34.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:33 smithi136 bash[32189]: audit 2024-09-12T23:41:32.701271+0000 mon.b (mon.1) 164 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:41:34.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:33 smithi136 bash[32189]: cephadm 2024-09-12T23:41:32.702130+0000 mgr.y (mgr.15123) 33 : cephadm [INF] Deploying daemon mgr.x on smithi181 2024-09-12T23:41:34.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:33 smithi136 bash[22601]: cluster 2024-09-12T23:41:32.524522+0000 mon.a (mon.0) 1419 : cluster [DBG] osdmap e257: 8 total, 8 up, 8 in 2024-09-12T23:41:34.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:33 smithi136 bash[22601]: cephadm 2024-09-12T23:41:32.693068+0000 mgr.y (mgr.15123) 32 : cephadm [INF] Upgrade: Updating mgr.x 2024-09-12T23:41:34.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:33 smithi136 bash[22601]: audit 2024-09-12T23:41:32.695155+0000 mon.a (mon.0) 1420 : audit [INF] from='mgr.15123 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-12T23:41:34.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:33 smithi136 bash[22601]: audit 2024-09-12T23:41:32.695189+0000 mon.b (mon.1) 162 : audit [INF] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-12T23:41:34.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:33 smithi136 bash[22601]: audit 2024-09-12T23:41:32.699255+0000 mon.b (mon.1) 163 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-12T23:41:34.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:33 smithi136 bash[22601]: audit 2024-09-12T23:41:32.701271+0000 mon.b (mon.1) 164 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:41:34.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:33 smithi136 bash[22601]: cephadm 2024-09-12T23:41:32.702130+0000 mgr.y (mgr.15123) 33 : cephadm [INF] Deploying daemon mgr.x on smithi181 2024-09-12T23:41:34.541 INFO:tasks.rados.rados.0.smithi136.stdout:1076: done (1 left) 2024-09-12T23:41:34.541 INFO:tasks.rados.rados.0.smithi136.stdout:1077: done (0 left) 2024-09-12T23:41:34.541 INFO:tasks.rados.rados.0.smithi136.stdout:1078: snap_create 2024-09-12T23:41:34.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:34 smithi181 bash[24776]: cluster 2024-09-12T23:41:33.442583+0000 mgr.y (mgr.15123) 34 : cluster [DBG] pgmap v41: 121 pgs: 2 active+clean+snaptrim_wait, 6 active+clean+snaptrim, 113 active+clean; 151 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:41:34.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:34 smithi181 bash[24776]: cluster 2024-09-12T23:41:33.538361+0000 mon.a (mon.0) 1421 : cluster [DBG] osdmap e258: 8 total, 8 up, 8 in 2024-09-12T23:41:35.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:34 smithi136 bash[32189]: cluster 2024-09-12T23:41:33.442583+0000 mgr.y (mgr.15123) 34 : cluster [DBG] pgmap v41: 121 pgs: 2 active+clean+snaptrim_wait, 6 active+clean+snaptrim, 113 active+clean; 151 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:41:35.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:34 smithi136 bash[32189]: cluster 2024-09-12T23:41:33.538361+0000 mon.a (mon.0) 1421 : cluster [DBG] osdmap e258: 8 total, 8 up, 8 in 2024-09-12T23:41:35.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:34 smithi136 bash[22601]: cluster 2024-09-12T23:41:33.442583+0000 mgr.y (mgr.15123) 34 : cluster [DBG] pgmap v41: 121 pgs: 2 active+clean+snaptrim_wait, 6 active+clean+snaptrim, 113 active+clean; 151 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:41:35.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:34 smithi136 bash[22601]: cluster 2024-09-12T23:41:33.538361+0000 mon.a (mon.0) 1421 : cluster [DBG] osdmap e258: 8 total, 8 up, 8 in 2024-09-12T23:41:35.093 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:41:34 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:41:35.093 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:41:34 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:41:35.094 INFO:journalctl@ceph.mds.b.smithi181.stdout:Sep 12 23:41:34 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:41:35.094 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:34 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:41:35.094 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:34 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:41:35.095 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:41:34 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:41:35.095 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 12 23:41:34 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:41:35.095 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:41:34 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:41:35.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:34 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:41:35.096 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 12 23:41:34 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:41:35.471 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:35 smithi181 systemd[1]: Stopping Ceph mgr.x for da58a47c-715d-11ef-bceb-c7b262605968... 2024-09-12T23:41:35.551 INFO:tasks.rados.rados.0.smithi136.stdout:1078: done (0 left) 2024-09-12T23:41:35.551 INFO:tasks.rados.rados.0.smithi136.stdout:1079: copy_from oid 10 from oid 16 current snap is 102 2024-09-12T23:41:35.551 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 5 current snap is 102 2024-09-12T23:41:35.551 INFO:tasks.rados.rados.0.smithi136.stdout:1080: seq_num 358 ranges {1032192=802816,1835008=401408} 2024-09-12T23:41:35.581 INFO:tasks.rados.rados.0.smithi136.stdout:1080: writing smithi13667081-5 from 1032192 to 1835008 tid 1 2024-09-12T23:41:35.589 INFO:tasks.rados.rados.0.smithi136.stdout:1080: writing smithi13667081-5 from 1835008 to 2236416 tid 2 2024-09-12T23:41:35.589 INFO:tasks.rados.rados.0.smithi136.stdout:1079: finishing copy_from racing read to smithi13667081-10 2024-09-12T23:41:35.589 INFO:tasks.rados.rados.0.smithi136.stdout:1081: read oid 23 snap 78 2024-09-12T23:41:35.590 INFO:tasks.rados.rados.0.smithi136.stdout:1081: expect (ObjNum 257 snap 70 seq_num 257) 2024-09-12T23:41:35.590 INFO:tasks.rados.rados.0.smithi136.stdout:1082: snap_remove snap 88 2024-09-12T23:41:35.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:35 smithi181 bash[24776]: cluster 2024-09-12T23:41:34.545278+0000 mon.a (mon.0) 1422 : cluster [DBG] osdmap e259: 8 total, 8 up, 8 in 2024-09-12T23:41:36.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:35 smithi136 bash[22601]: cluster 2024-09-12T23:41:34.545278+0000 mon.a (mon.0) 1422 : cluster [DBG] osdmap e259: 8 total, 8 up, 8 in 2024-09-12T23:41:36.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:35 smithi136 bash[32189]: cluster 2024-09-12T23:41:34.545278+0000 mon.a (mon.0) 1422 : cluster [DBG] osdmap e259: 8 total, 8 up, 8 in 2024-09-12T23:41:36.343 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:35 smithi181 bash[60751]: ceph-da58a47c-715d-11ef-bceb-c7b262605968-mgr-x 2024-09-12T23:41:36.343 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:36 smithi181 systemd[1]: ceph-da58a47c-715d-11ef-bceb-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-09-12T23:41:36.343 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:36 smithi181 bash[60812]: Error response from daemon: No such container: ceph-da58a47c-715d-11ef-bceb-c7b262605968-mgr-x 2024-09-12T23:41:36.343 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:36 smithi181 systemd[1]: ceph-da58a47c-715d-11ef-bceb-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-09-12T23:41:36.343 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:36 smithi181 systemd[1]: Stopped Ceph mgr.x for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:41:36.556 INFO:tasks.rados.rados.0.smithi136.stdout:1082: done (3 left) 2024-09-12T23:41:36.556 INFO:tasks.rados.rados.0.smithi136.stdout:1083: snap_remove snap 90 2024-09-12T23:41:36.627 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 12 23:41:36 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:41:36.627 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:41:36 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:41:36.628 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:41:36 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:41:36.628 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 12 23:41:36 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:41:36.628 INFO:journalctl@ceph.mds.b.smithi181.stdout:Sep 12 23:41:36 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:41:36.629 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:41:36 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:41:36.629 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:36 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:41:36.629 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:41:36 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:41:36.629 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:36 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:41:36.630 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:36 smithi181 systemd[1]: Started Ceph mgr.x for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:41:36.630 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:36 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:41:36.630 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:36 smithi181 bash[24776]: cluster 2024-09-12T23:41:35.443289+0000 mgr.y (mgr.15123) 35 : cluster [DBG] pgmap v44: 121 pgs: 121 active+clean; 172 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 4.6 MiB/s wr, 34 op/s 2024-09-12T23:41:36.630 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:36 smithi181 bash[24776]: cluster 2024-09-12T23:41:35.554928+0000 mon.a (mon.0) 1423 : cluster [DBG] osdmap e260: 8 total, 8 up, 8 in 2024-09-12T23:41:36.630 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:36 smithi181 bash[24776]: audit 2024-09-12T23:41:36.520057+0000 mon.a (mon.0) 1424 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:36.630 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:36 smithi181 bash[24776]: audit 2024-09-12T23:41:36.531396+0000 mon.a (mon.0) 1425 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:36.630 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:36 smithi181 bash[24776]: audit 2024-09-12T23:41:36.535475+0000 mon.b (mon.1) 165 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:41:37.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:36 smithi136 bash[22601]: cluster 2024-09-12T23:41:35.443289+0000 mgr.y (mgr.15123) 35 : cluster [DBG] pgmap v44: 121 pgs: 121 active+clean; 172 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 4.6 MiB/s wr, 34 op/s 2024-09-12T23:41:37.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:36 smithi136 bash[22601]: cluster 2024-09-12T23:41:35.554928+0000 mon.a (mon.0) 1423 : cluster [DBG] osdmap e260: 8 total, 8 up, 8 in 2024-09-12T23:41:37.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:36 smithi136 bash[22601]: audit 2024-09-12T23:41:36.520057+0000 mon.a (mon.0) 1424 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:37.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:36 smithi136 bash[22601]: audit 2024-09-12T23:41:36.531396+0000 mon.a (mon.0) 1425 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:37.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:36 smithi136 bash[22601]: audit 2024-09-12T23:41:36.535475+0000 mon.b (mon.1) 165 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:41:37.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:36 smithi136 bash[32189]: cluster 2024-09-12T23:41:35.443289+0000 mgr.y (mgr.15123) 35 : cluster [DBG] pgmap v44: 121 pgs: 121 active+clean; 172 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 4.6 MiB/s wr, 34 op/s 2024-09-12T23:41:37.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:36 smithi136 bash[32189]: cluster 2024-09-12T23:41:35.554928+0000 mon.a (mon.0) 1423 : cluster [DBG] osdmap e260: 8 total, 8 up, 8 in 2024-09-12T23:41:37.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:36 smithi136 bash[32189]: audit 2024-09-12T23:41:36.520057+0000 mon.a (mon.0) 1424 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:37.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:36 smithi136 bash[32189]: audit 2024-09-12T23:41:36.531396+0000 mon.a (mon.0) 1425 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:37.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:36 smithi136 bash[32189]: audit 2024-09-12T23:41:36.535475+0000 mon.b (mon.1) 165 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:41:37.577 INFO:tasks.rados.rados.0.smithi136.stdout:1083: done (3 left) 2024-09-12T23:41:37.577 INFO:tasks.rados.rados.0.smithi136.stdout:1084: snap_create 2024-09-12T23:41:37.577 INFO:tasks.rados.rados.0.smithi136.stdout:1079: finishing copy_from to smithi13667081-10 2024-09-12T23:41:37.577 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 10 v 611 (ObjNum 350 snap 95 seq_num 350) dirty exists 2024-09-12T23:41:37.577 INFO:tasks.rados.rados.0.smithi136.stdout:1080: finishing write tid 1 to smithi13667081-5 2024-09-12T23:41:37.577 INFO:tasks.rados.rados.0.smithi136.stdout:1080: finishing write tid 2 to smithi13667081-5 2024-09-12T23:41:37.577 INFO:tasks.rados.rados.0.smithi136.stdout:1080: finishing write tid 3 to smithi13667081-5 2024-09-12T23:41:37.577 INFO:tasks.rados.rados.0.smithi136.stdout:1081: expect (ObjNum 257 snap 70 seq_num 257) 2024-09-12T23:41:37.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:37 smithi181 bash[24776]: cluster 2024-09-12T23:41:36.556588+0000 mon.a (mon.0) 1426 : cluster [DBG] osdmap e261: 8 total, 8 up, 8 in 2024-09-12T23:41:38.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:37 smithi136 bash[22601]: cluster 2024-09-12T23:41:36.556588+0000 mon.a (mon.0) 1426 : cluster [DBG] osdmap e261: 8 total, 8 up, 8 in 2024-09-12T23:41:38.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:37 smithi136 bash[32189]: cluster 2024-09-12T23:41:36.556588+0000 mon.a (mon.0) 1426 : cluster [DBG] osdmap e261: 8 total, 8 up, 8 in 2024-09-12T23:41:38.126 INFO:tasks.rados.rados.0.smithi136.stdout:1080: finishing write tid 4 to smithi13667081-5 2024-09-12T23:41:38.126 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 5 v 639 (ObjNum 358 snap 102 seq_num 358) dirty exists 2024-09-12T23:41:38.126 INFO:tasks.rados.rados.0.smithi136.stdout:1080: left oid 5 (ObjNum 358 snap 102 seq_num 358) 2024-09-12T23:41:38.343 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:37 smithi181 bash[60867]: debug 2024-09-12T23:41:37.994+0000 7f0d1c974640 1 -- 172.21.15.181:0/2286834989 <== mon.1 v2:172.21.15.181:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x56287f0f4d00 con 0x56287f0e7400 2024-09-12T23:41:38.343 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:38 smithi181 bash[60867]: debug 2024-09-12T23:41:38.106+0000 7f0d1f1dd040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-12T23:41:38.580 INFO:tasks.rados.rados.0.smithi136.stdout:1079: done (3 left) 2024-09-12T23:41:38.580 INFO:tasks.rados.rados.0.smithi136.stdout:1080: done (2 left) 2024-09-12T23:41:38.580 INFO:tasks.rados.rados.0.smithi136.stdout:1081: done (1 left) 2024-09-12T23:41:38.580 INFO:tasks.rados.rados.0.smithi136.stdout:1084: done (0 left) 2024-09-12T23:41:38.580 INFO:tasks.rados.rados.0.smithi136.stdout:1085: rollback oid 48 current snap is 103 2024-09-12T23:41:38.580 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 48 to 100 2024-09-12T23:41:38.580 INFO:tasks.rados.rados.0.smithi136.stdout:1086: copy_from oid 4 from oid 24 current snap is 103 2024-09-12T23:41:38.580 INFO:tasks.rados.rados.0.smithi136.stdout:1087: snap_create 2024-09-12T23:41:38.595 INFO:tasks.rados.rados.0.smithi136.stdout:1085: finishing rollback tid 1 to smithi13667081-48 2024-09-12T23:41:38.595 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 48 v 0 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-12T23:41:38.622 INFO:tasks.rados.rados.0.smithi136.stdout:1086: finishing copy_from racing read to smithi13667081-4 2024-09-12T23:41:38.628 INFO:tasks.rados.rados.0.smithi136.stdout:1086: finishing copy_from to smithi13667081-4 2024-09-12T23:41:38.628 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 4 v 385 (ObjNum 283 snap 78 seq_num 283) dirty exists 2024-09-12T23:41:38.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:38 smithi181 bash[24776]: cluster 2024-09-12T23:41:37.444020+0000 mgr.y (mgr.15123) 36 : cluster [DBG] pgmap v47: 121 pgs: 121 active+clean; 172 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 4.6 MiB/s wr, 34 op/s 2024-09-12T23:41:38.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:38 smithi181 bash[24776]: cluster 2024-09-12T23:41:37.580282+0000 mon.a (mon.0) 1427 : cluster [DBG] osdmap e262: 8 total, 8 up, 8 in 2024-09-12T23:41:38.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:38 smithi181 bash[24776]: audit 2024-09-12T23:41:37.643072+0000 mon.b (mon.1) 166 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:41:39.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:38 smithi136 bash[22601]: cluster 2024-09-12T23:41:37.444020+0000 mgr.y (mgr.15123) 36 : cluster [DBG] pgmap v47: 121 pgs: 121 active+clean; 172 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 4.6 MiB/s wr, 34 op/s 2024-09-12T23:41:39.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:38 smithi136 bash[22601]: cluster 2024-09-12T23:41:37.580282+0000 mon.a (mon.0) 1427 : cluster [DBG] osdmap e262: 8 total, 8 up, 8 in 2024-09-12T23:41:39.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:38 smithi136 bash[22601]: audit 2024-09-12T23:41:37.643072+0000 mon.b (mon.1) 166 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:41:39.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:38 smithi136 bash[32189]: cluster 2024-09-12T23:41:37.444020+0000 mgr.y (mgr.15123) 36 : cluster [DBG] pgmap v47: 121 pgs: 121 active+clean; 172 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 4.6 MiB/s wr, 34 op/s 2024-09-12T23:41:39.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:38 smithi136 bash[32189]: cluster 2024-09-12T23:41:37.580282+0000 mon.a (mon.0) 1427 : cluster [DBG] osdmap e262: 8 total, 8 up, 8 in 2024-09-12T23:41:39.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:38 smithi136 bash[32189]: audit 2024-09-12T23:41:37.643072+0000 mon.b (mon.1) 166 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:41:39.578 INFO:tasks.rados.rados.0.smithi136.stdout:1085: done (2 left) 2024-09-12T23:41:39.578 INFO:tasks.rados.rados.0.smithi136.stdout:1086: done (1 left) 2024-09-12T23:41:39.578 INFO:tasks.rados.rados.0.smithi136.stdout:1087: done (0 left) 2024-09-12T23:41:39.578 INFO:tasks.rados.rados.0.smithi136.stdout:1088: read oid 34 snap 100 2024-09-12T23:41:39.578 INFO:tasks.rados.rados.0.smithi136.stdout:1088: expect (ObjNum 327 snap 90 seq_num 327) 2024-09-12T23:41:39.581 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 45 current snap is 104 2024-09-12T23:41:39.581 INFO:tasks.rados.rados.0.smithi136.stdout:1089: seq_num 359 ranges {0=417792,417792=417792,835584=385024} 2024-09-12T23:41:39.587 INFO:tasks.rados.rados.0.smithi136.stdout:1089: writing smithi13667081-45 from 0 to 417792 tid 1 2024-09-12T23:41:39.593 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:39 smithi181 bash[60867]: debug 2024-09-12T23:41:39.306+0000 7f0d1f1dd040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-12T23:41:39.593 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:39 smithi181 bash[60867]: debug 2024-09-12T23:41:39.370+0000 7f0d1f1dd040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-12T23:41:39.593 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:39 smithi181 bash[60867]: debug 2024-09-12T23:41:39.518+0000 7f0d1f1dd040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-12T23:41:39.595 INFO:tasks.rados.rados.0.smithi136.stdout:1089: writing smithi13667081-45 from 417792 to 835584 tid 2 2024-09-12T23:41:39.604 INFO:tasks.rados.rados.0.smithi136.stdout:1089: writing smithi13667081-45 from 835584 to 1220608 tid 3 2024-09-12T23:41:39.604 INFO:tasks.rados.rados.0.smithi136.stdout:1090: delete oid 48 current snap is 104 2024-09-12T23:41:39.604 INFO:tasks.rados.rados.0.smithi136.stdout:1088: expect (ObjNum 327 snap 90 seq_num 327) 2024-09-12T23:41:39.623 INFO:tasks.rados.rados.0.smithi136.stdout:1089: finishing write tid 1 to smithi13667081-45 2024-09-12T23:41:39.623 INFO:tasks.rados.rados.0.smithi136.stdout:1089: finishing write tid 2 to smithi13667081-45 2024-09-12T23:41:39.623 INFO:tasks.rados.rados.0.smithi136.stdout:1089: finishing write tid 3 to smithi13667081-45 2024-09-12T23:41:39.623 INFO:tasks.rados.rados.0.smithi136.stdout:1089: finishing write tid 4 to smithi13667081-45 2024-09-12T23:41:39.623 INFO:tasks.rados.rados.0.smithi136.stdout:1089: finishing write tid 5 to smithi13667081-45 2024-09-12T23:41:39.624 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 45 v 316 (ObjNum 359 snap 104 seq_num 359) dirty exists 2024-09-12T23:41:39.624 INFO:tasks.rados.rados.0.smithi136.stdout:1089: left oid 45 (ObjNum 359 snap 104 seq_num 359) 2024-09-12T23:41:39.624 INFO:tasks.rados.rados.0.smithi136.stdout:1088: done (2 left) 2024-09-12T23:41:39.624 INFO:tasks.rados.rados.0.smithi136.stdout:1089: done (1 left) 2024-09-12T23:41:39.624 INFO:tasks.rados.rados.0.smithi136.stdout:1090: done (0 left) 2024-09-12T23:41:39.624 INFO:tasks.rados.rados.0.smithi136.stdout:1091: setattr oid 29 current snap is 104 2024-09-12T23:41:39.627 INFO:tasks.rados.rados.0.smithi136.stdout:1092: rollback oid 3 current snap is 104 2024-09-12T23:41:39.627 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 3 to 98 2024-09-12T23:41:39.627 INFO:tasks.rados.rados.0.smithi136.stdout:1093: snap_create 2024-09-12T23:41:39.633 INFO:tasks.rados.rados.0.smithi136.stdout:1092: finishing rollback tid 0 to smithi13667081-3 2024-09-12T23:41:39.634 INFO:tasks.rados.rados.0.smithi136.stdout:1092: finishing rollback tid 1 to smithi13667081-3 2024-09-12T23:41:39.634 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 29 v 450 (ObjNum 311 snap 86 seq_num 311) dirty exists 2024-09-12T23:41:39.634 INFO:tasks.rados.rados.0.smithi136.stdout:1092: finishing rollback tid 2 to smithi13667081-3 2024-09-12T23:41:39.635 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 3 v 544 (ObjNum 335 snap 92 seq_num 335) dirty exists 2024-09-12T23:41:39.755 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:41:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:41:39] "GET /metrics HTTP/1.1" 200 36139 "" "Prometheus/2.43.0" 2024-09-12T23:41:39.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:39 smithi136 bash[22601]: cluster 2024-09-12T23:41:38.581601+0000 mon.a (mon.0) 1428 : cluster [DBG] osdmap e263: 8 total, 8 up, 8 in 2024-09-12T23:41:39.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:39 smithi136 bash[22601]: cluster 2024-09-12T23:41:39.582410+0000 mon.a (mon.0) 1429 : cluster [DBG] osdmap e264: 8 total, 8 up, 8 in 2024-09-12T23:41:39.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:39 smithi136 bash[32189]: cluster 2024-09-12T23:41:38.581601+0000 mon.a (mon.0) 1428 : cluster [DBG] osdmap e263: 8 total, 8 up, 8 in 2024-09-12T23:41:39.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:39 smithi136 bash[32189]: cluster 2024-09-12T23:41:39.582410+0000 mon.a (mon.0) 1429 : cluster [DBG] osdmap e264: 8 total, 8 up, 8 in 2024-09-12T23:41:40.093 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:39 smithi181 bash[24776]: cluster 2024-09-12T23:41:38.581601+0000 mon.a (mon.0) 1428 : cluster [DBG] osdmap e263: 8 total, 8 up, 8 in 2024-09-12T23:41:40.093 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:39 smithi181 bash[24776]: cluster 2024-09-12T23:41:39.582410+0000 mon.a (mon.0) 1429 : cluster [DBG] osdmap e264: 8 total, 8 up, 8 in 2024-09-12T23:41:40.483 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:40 smithi181 bash[60867]: debug 2024-09-12T23:41:40.166+0000 7f0d1f1dd040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-12T23:41:40.484 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:40 smithi181 bash[60867]: debug 2024-09-12T23:41:40.234+0000 7f0d1f1dd040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-12T23:41:40.484 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:40 smithi181 bash[60867]: debug 2024-09-12T23:41:40.302+0000 7f0d1f1dd040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-12T23:41:40.512 INFO:tasks.rados.rados.0.smithi136.stdout:1091: done (2 left) 2024-09-12T23:41:40.512 INFO:tasks.rados.rados.0.smithi136.stdout:1092: done (1 left) 2024-09-12T23:41:40.512 INFO:tasks.rados.rados.0.smithi136.stdout:1093: done (0 left) 2024-09-12T23:41:40.512 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 36 current snap is 105 2024-09-12T23:41:40.512 INFO:tasks.rados.rados.0.smithi136.stdout:1094: seq_num 361 ranges {4685824=450560,5136384=581632,5718016=221184} 2024-09-12T23:41:40.584 INFO:tasks.rados.rados.0.smithi136.stdout:1094: writing smithi13667081-36 from 4685824 to 5136384 tid 1 2024-09-12T23:41:40.589 INFO:tasks.rados.rados.0.smithi136.stdout:1094: writing smithi13667081-36 from 5136384 to 5718016 tid 2 2024-09-12T23:41:40.591 INFO:tasks.rados.rados.0.smithi136.stdout:1094: writing smithi13667081-36 from 5718016 to 5939200 tid 3 2024-09-12T23:41:40.591 INFO:tasks.rados.rados.0.smithi136.stdout:1095: read oid 24 snap -1 2024-09-12T23:41:40.591 INFO:tasks.rados.rados.0.smithi136.stdout:1095: expect (ObjNum 283 snap 78 seq_num 283) 2024-09-12T23:41:40.592 INFO:tasks.rados.rados.0.smithi136.stdout:1096: snap_remove snap 81 2024-09-12T23:41:40.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:40 smithi181 bash[24776]: cluster 2024-09-12T23:41:39.445482+0000 mgr.y (mgr.15123) 37 : cluster [DBG] pgmap v50: 121 pgs: 121 active+clean; 176 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-09-12T23:41:40.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:40 smithi181 bash[24776]: cluster 2024-09-12T23:41:40.515958+0000 mon.a (mon.0) 1430 : cluster [DBG] osdmap e265: 8 total, 8 up, 8 in 2024-09-12T23:41:40.844 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:40 smithi181 bash[60867]: debug 2024-09-12T23:41:40.486+0000 7f0d1f1dd040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-12T23:41:40.844 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:40 smithi181 bash[60867]: debug 2024-09-12T23:41:40.562+0000 7f0d1f1dd040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-12T23:41:40.844 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:40 smithi181 bash[60867]: debug 2024-09-12T23:41:40.642+0000 7f0d1f1dd040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-12T23:41:41.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:40 smithi136 bash[22601]: cluster 2024-09-12T23:41:39.445482+0000 mgr.y (mgr.15123) 37 : cluster [DBG] pgmap v50: 121 pgs: 121 active+clean; 176 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-09-12T23:41:41.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:40 smithi136 bash[22601]: cluster 2024-09-12T23:41:40.515958+0000 mon.a (mon.0) 1430 : cluster [DBG] osdmap e265: 8 total, 8 up, 8 in 2024-09-12T23:41:41.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:40 smithi136 bash[32189]: cluster 2024-09-12T23:41:39.445482+0000 mgr.y (mgr.15123) 37 : cluster [DBG] pgmap v50: 121 pgs: 121 active+clean; 176 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-09-12T23:41:41.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:40 smithi136 bash[32189]: cluster 2024-09-12T23:41:40.515958+0000 mon.a (mon.0) 1430 : cluster [DBG] osdmap e265: 8 total, 8 up, 8 in 2024-09-12T23:41:41.149 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:40 smithi181 bash[60867]: debug 2024-09-12T23:41:40.894+0000 7f0d1f1dd040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-12T23:41:41.149 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:40 smithi181 bash[60867]: debug 2024-09-12T23:41:40.962+0000 7f0d1f1dd040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-12T23:41:41.149 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:41 smithi181 bash[60867]: debug 2024-09-12T23:41:41.150+0000 7f0d1f1dd040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-12T23:41:41.470 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:41 smithi181 bash[60867]: debug 2024-09-12T23:41:41.226+0000 7f0d1f1dd040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-12T23:41:41.531 INFO:tasks.rados.rados.0.smithi136.stdout:1096: done (2 left) 2024-09-12T23:41:41.531 INFO:tasks.rados.rados.0.smithi136.stdout:1097: read oid 22 snap -1 2024-09-12T23:41:41.531 INFO:tasks.rados.rados.0.smithi136.stdout:1097: expect (ObjNum 21 snap 0 seq_num 21) 2024-09-12T23:41:41.531 INFO:tasks.rados.rados.0.smithi136.stdout:1098: rollback oid 39 current snap is 105 2024-09-12T23:41:41.532 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 39 to 95 2024-09-12T23:41:41.532 INFO:tasks.rados.rados.0.smithi136.stdout:1099: delete oid 17 current snap is 105 2024-09-12T23:41:41.532 INFO:tasks.rados.rados.0.smithi136.stdout:1094: finishing write tid 1 to smithi13667081-36 2024-09-12T23:41:41.532 INFO:tasks.rados.rados.0.smithi136.stdout:1094: finishing write tid 2 to smithi13667081-36 2024-09-12T23:41:41.532 INFO:tasks.rados.rados.0.smithi136.stdout:1094: finishing write tid 3 to smithi13667081-36 2024-09-12T23:41:41.532 INFO:tasks.rados.rados.0.smithi136.stdout:1094: finishing write tid 4 to smithi13667081-36 2024-09-12T23:41:41.532 INFO:tasks.rados.rados.0.smithi136.stdout:1094: finishing write tid 5 to smithi13667081-36 2024-09-12T23:41:41.532 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 36 v 321 (ObjNum 361 snap 105 seq_num 361) dirty exists 2024-09-12T23:41:41.532 INFO:tasks.rados.rados.0.smithi136.stdout:1094: left oid 36 (ObjNum 361 snap 105 seq_num 361) 2024-09-12T23:41:41.532 INFO:tasks.rados.rados.0.smithi136.stdout:1095: expect (ObjNum 283 snap 78 seq_num 283) 2024-09-12T23:41:41.663 INFO:tasks.rados.rados.0.smithi136.stdout:1098: finishing rollback tid 1 to smithi13667081-39 2024-09-12T23:41:41.663 INFO:tasks.rados.rados.0.smithi136.stdout:1098: finishing rollback tid 2 to smithi13667081-39 2024-09-12T23:41:41.663 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 39 v 168 (ObjNum 312 snap 86 seq_num 312) dirty exists 2024-09-12T23:41:41.663 INFO:tasks.rados.rados.0.smithi136.stdout:1097: expect (ObjNum 21 snap 0 seq_num 21) 2024-09-12T23:41:41.675 INFO:tasks.rados.rados.0.smithi136.stdout:1094: done (4 left) 2024-09-12T23:41:41.676 INFO:tasks.rados.rados.0.smithi136.stdout:1095: done (3 left) 2024-09-12T23:41:41.676 INFO:tasks.rados.rados.0.smithi136.stdout:1097: done (2 left) 2024-09-12T23:41:41.676 INFO:tasks.rados.rados.0.smithi136.stdout:1098: done (1 left) 2024-09-12T23:41:41.676 INFO:tasks.rados.rados.0.smithi136.stdout:1099: done (0 left) 2024-09-12T23:41:41.676 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 47 current snap is 105 2024-09-12T23:41:41.676 INFO:tasks.rados.rados.0.smithi136.stdout:1100: seq_num 363 ranges {2088960=483328,2572288=753664,3325952=49152} 2024-09-12T23:41:41.714 INFO:tasks.rados.rados.0.smithi136.stdout:1100: writing smithi13667081-47 from 2088960 to 2572288 tid 1 2024-09-12T23:41:41.725 INFO:tasks.rados.rados.0.smithi136.stdout:1100: writing smithi13667081-47 from 2572288 to 3325952 tid 2 2024-09-12T23:41:41.726 INFO:tasks.rados.rados.0.smithi136.stdout:1100: writing smithi13667081-47 from 3325952 to 3375104 tid 3 2024-09-12T23:41:41.726 INFO:tasks.rados.rados.0.smithi136.stdout:1101: read oid 18 snap -1 2024-09-12T23:41:41.726 INFO:tasks.rados.rados.0.smithi136.stdout:1101: expect (ObjNum 270 snap 73 seq_num 270) 2024-09-12T23:41:41.726 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 2 current snap is 105 2024-09-12T23:41:41.726 INFO:tasks.rados.rados.0.smithi136.stdout:1100: finishing write tid 1 to smithi13667081-47 2024-09-12T23:41:41.726 INFO:tasks.rados.rados.0.smithi136.stdout:1102: seq_num 364 ranges {3940352=499712,4440064=638976,5079040=163840} 2024-09-12T23:41:41.774 INFO:tasks.rados.rados.0.smithi136.stdout:1102: writing smithi13667081-2 from 3940352 to 4440064 tid 1 2024-09-12T23:41:41.780 INFO:tasks.rados.rados.0.smithi136.stdout:1102: writing smithi13667081-2 from 4440064 to 5079040 tid 2 2024-09-12T23:41:41.782 INFO:tasks.rados.rados.0.smithi136.stdout:1102: writing smithi13667081-2 from 5079040 to 5242880 tid 3 2024-09-12T23:41:41.782 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 33 current snap is 105 2024-09-12T23:41:41.782 INFO:tasks.rados.rados.0.smithi136.stdout:1103: seq_num 365 ranges {0=516096,516096=573440,1089536=229376} 2024-09-12T23:41:41.785 INFO:tasks.rados.rados.0.smithi136.stdout:1103: writing smithi13667081-33 from 0 to 516096 tid 1 2024-09-12T23:41:41.789 INFO:tasks.rados.rados.0.smithi136.stdout:1103: writing smithi13667081-33 from 516096 to 1089536 tid 2 2024-09-12T23:41:41.791 INFO:tasks.rados.rados.0.smithi136.stdout:1103: writing smithi13667081-33 from 1089536 to 1318912 tid 3 2024-09-12T23:41:41.791 INFO:tasks.rados.rados.0.smithi136.stdout:1104: copy_from oid 19 from oid 21 current snap is 105 2024-09-12T23:41:41.791 INFO:tasks.rados.rados.0.smithi136.stdout:1100: finishing write tid 2 to smithi13667081-47 2024-09-12T23:41:41.791 INFO:tasks.rados.rados.0.smithi136.stdout:1100: finishing write tid 3 to smithi13667081-47 2024-09-12T23:41:41.791 INFO:tasks.rados.rados.0.smithi136.stdout:1100: finishing write tid 4 to smithi13667081-47 2024-09-12T23:41:41.791 INFO:tasks.rados.rados.0.smithi136.stdout:1100: finishing write tid 5 to smithi13667081-47 2024-09-12T23:41:41.791 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 47 v 343 (ObjNum 363 snap 105 seq_num 363) dirty exists 2024-09-12T23:41:41.791 INFO:tasks.rados.rados.0.smithi136.stdout:1100: left oid 47 (ObjNum 363 snap 105 seq_num 363) 2024-09-12T23:41:41.791 INFO:tasks.rados.rados.0.smithi136.stdout:1101: expect (ObjNum 270 snap 73 seq_num 270) 2024-09-12T23:41:41.826 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:41 smithi181 bash[60867]: debug 2024-09-12T23:41:41.474+0000 7f0d1f1dd040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-12T23:41:41.826 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:41 smithi181 bash[60867]: debug 2024-09-12T23:41:41.662+0000 7f0d1f1dd040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-12T23:41:41.852 INFO:tasks.rados.rados.0.smithi136.stdout:1102: finishing write tid 1 to smithi13667081-2 2024-09-12T23:41:41.852 INFO:tasks.rados.rados.0.smithi136.stdout:1102: finishing write tid 2 to smithi13667081-2 2024-09-12T23:41:41.852 INFO:tasks.rados.rados.0.smithi136.stdout:1102: finishing write tid 3 to smithi13667081-2 2024-09-12T23:41:41.852 INFO:tasks.rados.rados.0.smithi136.stdout:1102: finishing write tid 4 to smithi13667081-2 2024-09-12T23:41:41.852 INFO:tasks.rados.rados.0.smithi136.stdout:1102: finishing write tid 5 to smithi13667081-2 2024-09-12T23:41:41.852 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 2 v 544 (ObjNum 364 snap 105 seq_num 364) dirty exists 2024-09-12T23:41:41.852 INFO:tasks.rados.rados.0.smithi136.stdout:1102: left oid 2 (ObjNum 364 snap 105 seq_num 364) 2024-09-12T23:41:41.853 INFO:tasks.rados.rados.0.smithi136.stdout:1103: finishing write tid 1 to smithi13667081-33 2024-09-12T23:41:41.853 INFO:tasks.rados.rados.0.smithi136.stdout:1103: finishing write tid 2 to smithi13667081-33 2024-09-12T23:41:41.853 INFO:tasks.rados.rados.0.smithi136.stdout:1103: finishing write tid 3 to smithi13667081-33 2024-09-12T23:41:41.853 INFO:tasks.rados.rados.0.smithi136.stdout:1103: finishing write tid 4 to smithi13667081-33 2024-09-12T23:41:41.853 INFO:tasks.rados.rados.0.smithi136.stdout:1103: finishing write tid 5 to smithi13667081-33 2024-09-12T23:41:41.853 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 33 v 548 (ObjNum 365 snap 105 seq_num 365) dirty exists 2024-09-12T23:41:41.853 INFO:tasks.rados.rados.0.smithi136.stdout:1103: left oid 33 (ObjNum 365 snap 105 seq_num 365) 2024-09-12T23:41:41.853 INFO:tasks.rados.rados.0.smithi136.stdout:1100: done (4 left) 2024-09-12T23:41:41.853 INFO:tasks.rados.rados.0.smithi136.stdout:1101: done (3 left) 2024-09-12T23:41:41.853 INFO:tasks.rados.rados.0.smithi136.stdout:1102: done (2 left) 2024-09-12T23:41:41.853 INFO:tasks.rados.rados.0.smithi136.stdout:1103: done (1 left) 2024-09-12T23:41:41.853 INFO:tasks.rados.rados.0.smithi136.stdout:1105: rollback oid 49 current snap is 105 2024-09-12T23:41:41.853 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 49 to 100 2024-09-12T23:41:41.853 INFO:tasks.rados.rados.0.smithi136.stdout:1106: copy_from oid 33 from oid 50 current snap is 105 2024-09-12T23:41:41.854 INFO:tasks.rados.rados.0.smithi136.stdout:1107: read oid 12 snap 93 2024-09-12T23:41:41.854 INFO:tasks.rados.rados.0.smithi136.stdout:1107: expect (ObjNum 297 snap 81 seq_num 297) 2024-09-12T23:41:41.854 INFO:tasks.rados.rados.0.smithi136.stdout:1108: copy_from oid 48 from oid 5 current snap is 105 2024-09-12T23:41:41.854 INFO:tasks.rados.rados.0.smithi136.stdout:1109: snap_create 2024-09-12T23:41:41.854 INFO:tasks.rados.rados.0.smithi136.stdout:1105: finishing rollback tid 1 to smithi13667081-49 2024-09-12T23:41:41.854 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 49 v 0 (ObjNum 161 snap 0 seq_num 0) dirty dne 2024-09-12T23:41:41.873 INFO:tasks.rados.rados.0.smithi136.stdout:1104: finishing copy_from racing read to smithi13667081-19 2024-09-12T23:41:41.873 INFO:tasks.rados.rados.0.smithi136.stdout:1104: finishing copy_from to smithi13667081-19 2024-09-12T23:41:41.873 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 19 v 569 (ObjNum 270 snap 73 seq_num 270) dirty exists 2024-09-12T23:41:41.879 INFO:tasks.rados.rados.0.smithi136.stdout:1107: expect (ObjNum 297 snap 81 seq_num 297) 2024-09-12T23:41:42.086 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:41 smithi181 bash[60867]: debug 2024-09-12T23:41:41.830+0000 7f0d1f1dd040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-12T23:41:42.086 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:41 smithi181 bash[60867]: debug 2024-09-12T23:41:41.910+0000 7f0d1f1dd040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-12T23:41:42.157 INFO:tasks.rados.rados.0.smithi136.stdout:1108: finishing copy_from racing read to smithi13667081-48 2024-09-12T23:41:42.157 INFO:tasks.rados.rados.0.smithi136.stdout:1106: finishing copy_from racing read to smithi13667081-33 2024-09-12T23:41:42.157 INFO:tasks.rados.rados.0.smithi136.stdout:1106: finishing copy_from to smithi13667081-33 2024-09-12T23:41:42.157 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 33 v 446 (ObjNum 315 snap 86 seq_num 315) dirty exists 2024-09-12T23:41:42.157 INFO:tasks.rados.rados.0.smithi136.stdout:1108: finishing copy_from to smithi13667081-48 2024-09-12T23:41:42.157 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 48 v 639 (ObjNum 358 snap 102 seq_num 358) dirty exists 2024-09-12T23:41:42.343 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:42 smithi181 bash[60867]: debug 2024-09-12T23:41:42.086+0000 7f0d1f1dd040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-12T23:41:42.343 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:42 smithi181 bash[60471]: ts=2024-09-12T23:41:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:41:42.614 INFO:tasks.rados.rados.0.smithi136.stdout:1104: done (5 left) 2024-09-12T23:41:42.614 INFO:tasks.rados.rados.0.smithi136.stdout:1105: done (4 left) 2024-09-12T23:41:42.614 INFO:tasks.rados.rados.0.smithi136.stdout:1106: done (3 left) 2024-09-12T23:41:42.614 INFO:tasks.rados.rados.0.smithi136.stdout:1107: done (2 left) 2024-09-12T23:41:42.614 INFO:tasks.rados.rados.0.smithi136.stdout:1108: done (1 left) 2024-09-12T23:41:42.614 INFO:tasks.rados.rados.0.smithi136.stdout:1109: done (0 left) 2024-09-12T23:41:42.614 INFO:tasks.rados.rados.0.smithi136.stdout:1110: read oid 48 snap -1 2024-09-12T23:41:42.614 INFO:tasks.rados.rados.0.smithi136.stdout:1110: expect (ObjNum 358 snap 102 seq_num 358) 2024-09-12T23:41:42.614 INFO:tasks.rados.rados.0.smithi136.stdout:1111: read oid 35 snap 87 2024-09-12T23:41:42.614 INFO:tasks.rados.rados.0.smithi136.stdout:1111: expect (ObjNum 262 snap 71 seq_num 262) 2024-09-12T23:41:42.614 INFO:tasks.rados.rados.0.smithi136.stdout:1112: rollback oid 21 current snap is 106 2024-09-12T23:41:42.615 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 21 to 99 2024-09-12T23:41:42.615 INFO:tasks.rados.rados.0.smithi136.stdout:1113: snap_remove snap 83 2024-09-12T23:41:42.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:42 smithi181 bash[24776]: cluster 2024-09-12T23:41:41.446222+0000 mgr.y (mgr.15123) 38 : cluster [DBG] pgmap v53: 121 pgs: 121 active+clean; 176 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-09-12T23:41:42.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:42 smithi181 bash[24776]: cluster 2024-09-12T23:41:41.517274+0000 mon.a (mon.0) 1431 : cluster [DBG] osdmap e266: 8 total, 8 up, 8 in 2024-09-12T23:41:43.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:42 smithi136 bash[22601]: cluster 2024-09-12T23:41:41.446222+0000 mgr.y (mgr.15123) 38 : cluster [DBG] pgmap v53: 121 pgs: 121 active+clean; 176 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-09-12T23:41:43.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:42 smithi136 bash[22601]: cluster 2024-09-12T23:41:41.517274+0000 mon.a (mon.0) 1431 : cluster [DBG] osdmap e266: 8 total, 8 up, 8 in 2024-09-12T23:41:43.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:42 smithi136 bash[32189]: cluster 2024-09-12T23:41:41.446222+0000 mgr.y (mgr.15123) 38 : cluster [DBG] pgmap v53: 121 pgs: 121 active+clean; 176 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-09-12T23:41:43.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:42 smithi136 bash[32189]: cluster 2024-09-12T23:41:41.517274+0000 mon.a (mon.0) 1431 : cluster [DBG] osdmap e266: 8 total, 8 up, 8 in 2024-09-12T23:41:43.242 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:42 smithi181 bash[60867]: debug 2024-09-12T23:41:42.922+0000 7f0d1f1dd040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-12T23:41:43.242 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:43 smithi181 bash[60867]: debug 2024-09-12T23:41:43.006+0000 7f0d1f1dd040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-12T23:41:43.502 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:43 smithi181 bash[60867]: debug 2024-09-12T23:41:43.246+0000 7f0d1f1dd040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-12T23:41:43.502 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:43 smithi181 bash[60867]: debug 2024-09-12T23:41:43.326+0000 7f0d1f1dd040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-12T23:41:43.623 INFO:tasks.rados.rados.0.smithi136.stdout:1113: done (3 left) 2024-09-12T23:41:43.623 INFO:tasks.rados.rados.0.smithi136.stdout:1114: read oid 32 snap -1 2024-09-12T23:41:43.623 INFO:tasks.rados.rados.0.smithi136.stdout:1114: expect (ObjNum 353 snap 96 seq_num 353) 2024-09-12T23:41:43.624 INFO:tasks.rados.rados.0.smithi136.stdout:1115: read oid 25 snap -1 2024-09-12T23:41:43.624 INFO:tasks.rados.rados.0.smithi136.stdout:1115: expect deleted 2024-09-12T23:41:43.624 INFO:tasks.rados.rados.0.smithi136.stdout:1116: read oid 23 snap -1 2024-09-12T23:41:43.624 INFO:tasks.rados.rados.0.smithi136.stdout:1116: expect (ObjNum 256 snap 70 seq_num 256) 2024-09-12T23:41:43.625 INFO:tasks.rados.rados.0.smithi136.stdout:1117: rollback oid 33 current snap is 106 2024-09-12T23:41:43.625 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 33 to 93 2024-09-12T23:41:43.625 INFO:tasks.rados.rados.0.smithi136.stdout:1112: finishing rollback tid 0 to smithi13667081-21 2024-09-12T23:41:43.625 INFO:tasks.rados.rados.0.smithi136.stdout:1110: expect (ObjNum 358 snap 102 seq_num 358) 2024-09-12T23:41:43.717 INFO:tasks.rados.rados.0.smithi136.stdout:1112: finishing rollback tid 1 to smithi13667081-21 2024-09-12T23:41:43.717 INFO:tasks.rados.rados.0.smithi136.stdout:1111: expect (ObjNum 262 snap 71 seq_num 262) 2024-09-12T23:41:43.823 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:43 smithi181 bash[60867]: debug 2024-09-12T23:41:43.742+0000 7f0d1f1dd040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-12T23:41:43.824 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:43 smithi181 bash[60471]: ts=2024-09-12T23:41:43.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi136\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:41:43.824 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:43 smithi181 bash[24776]: cluster 2024-09-12T23:41:42.617804+0000 mon.a (mon.0) 1432 : cluster [DBG] osdmap e267: 8 total, 8 up, 8 in 2024-09-12T23:41:43.824 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:43 smithi181 bash[24776]: audit 2024-09-12T23:41:43.367024+0000 mon.a (mon.0) 1433 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:43.824 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:43 smithi181 bash[24776]: audit 2024-09-12T23:41:43.379513+0000 mon.a (mon.0) 1434 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:43.872 INFO:tasks.rados.rados.0.smithi136.stdout:1112: finishing rollback tid 2 to smithi13667081-21 2024-09-12T23:41:43.872 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 21 v 572 (ObjNum 270 snap 73 seq_num 270) dirty exists 2024-09-12T23:41:43.872 INFO:tasks.rados.rados.0.smithi136.stdout:1117: finishing rollback tid 0 to smithi13667081-33 2024-09-12T23:41:43.872 INFO:tasks.rados.rados.0.smithi136.stdout:1117: finishing rollback tid 1 to smithi13667081-33 2024-09-12T23:41:43.872 INFO:tasks.rados.rados.0.smithi136.stdout:1117: finishing rollback tid 2 to smithi13667081-33 2024-09-12T23:41:43.872 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 33 v 551 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-12T23:41:43.872 INFO:tasks.rados.rados.0.smithi136.stdout:1110: done (6 left) 2024-09-12T23:41:43.872 INFO:tasks.rados.rados.0.smithi136.stdout:1111: done (5 left) 2024-09-12T23:41:43.872 INFO:tasks.rados.rados.0.smithi136.stdout:1112: done (4 left) 2024-09-12T23:41:43.872 INFO:tasks.rados.rados.0.smithi136.stdout:1115: done (3 left) 2024-09-12T23:41:43.872 INFO:tasks.rados.rados.0.smithi136.stdout:1117: done (2 left) 2024-09-12T23:41:43.873 INFO:tasks.rados.rados.0.smithi136.stdout:1118: snap_remove snap 97 2024-09-12T23:41:44.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:43 smithi136 bash[22601]: cluster 2024-09-12T23:41:42.617804+0000 mon.a (mon.0) 1432 : cluster [DBG] osdmap e267: 8 total, 8 up, 8 in 2024-09-12T23:41:44.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:43 smithi136 bash[22601]: audit 2024-09-12T23:41:43.367024+0000 mon.a (mon.0) 1433 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:44.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:43 smithi136 bash[22601]: audit 2024-09-12T23:41:43.379513+0000 mon.a (mon.0) 1434 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:44.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:43 smithi136 bash[32189]: cluster 2024-09-12T23:41:42.617804+0000 mon.a (mon.0) 1432 : cluster [DBG] osdmap e267: 8 total, 8 up, 8 in 2024-09-12T23:41:44.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:43 smithi136 bash[32189]: audit 2024-09-12T23:41:43.367024+0000 mon.a (mon.0) 1433 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:44.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:43 smithi136 bash[32189]: audit 2024-09-12T23:41:43.379513+0000 mon.a (mon.0) 1434 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:44.093 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:43 smithi181 bash[60867]: debug 2024-09-12T23:41:43.826+0000 7f0d1f1dd040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-12T23:41:44.093 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:43 smithi181 bash[60867]: [12/Sep/2024:23:41:43] ENGINE Bus STARTING 2024-09-12T23:41:44.094 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:43 smithi181 bash[60867]: CherryPy Checker: 2024-09-12T23:41:44.094 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:43 smithi181 bash[60867]: The Application mounted at '' has an empty config. 2024-09-12T23:41:44.094 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:43 smithi181 bash[60867]: [12/Sep/2024:23:41:43] ENGINE Serving on http://:::9283 2024-09-12T23:41:44.094 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:41:43 smithi181 bash[60867]: [12/Sep/2024:23:41:43] ENGINE Bus STARTED 2024-09-12T23:41:44.667 INFO:tasks.rados.rados.0.smithi136.stdout:1118: done (2 left) 2024-09-12T23:41:44.668 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 1 current snap is 106 2024-09-12T23:41:44.668 INFO:tasks.rados.rados.0.smithi136.stdout:1119: seq_num 366 ranges {4161536=532480,4694016=458752,5152768=344064} 2024-09-12T23:41:44.728 INFO:tasks.rados.rados.0.smithi136.stdout:1119: writing smithi13667081-1 from 4161536 to 4694016 tid 1 2024-09-12T23:41:44.732 INFO:tasks.rados.rados.0.smithi136.stdout:1119: writing smithi13667081-1 from 4694016 to 5152768 tid 2 2024-09-12T23:41:44.736 INFO:tasks.rados.rados.0.smithi136.stdout:1119: writing smithi13667081-1 from 5152768 to 5496832 tid 3 2024-09-12T23:41:44.736 INFO:tasks.rados.rados.0.smithi136.stdout:1120: snap_remove snap 104 2024-09-12T23:41:44.843 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:44 smithi181 bash[60471]: ts=2024-09-12T23:41:44.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:41:44.844 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:44 smithi181 bash[60471]: ts=2024-09-12T23:41:44.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:41:44.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:44 smithi181 bash[24776]: cluster 2024-09-12T23:41:43.446937+0000 mgr.y (mgr.15123) 39 : cluster [DBG] pgmap v56: 121 pgs: 121 active+clean; 176 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:41:44.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:44 smithi181 bash[24776]: cluster 2024-09-12T23:41:43.621813+0000 mon.a (mon.0) 1435 : cluster [DBG] osdmap e268: 8 total, 8 up, 8 in 2024-09-12T23:41:44.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:44 smithi181 bash[24776]: cluster 2024-09-12T23:41:43.849314+0000 mon.a (mon.0) 1436 : cluster [DBG] Standby manager daemon x restarted 2024-09-12T23:41:44.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:44 smithi181 bash[24776]: cluster 2024-09-12T23:41:43.849552+0000 mon.a (mon.0) 1437 : cluster [DBG] Standby manager daemon x started 2024-09-12T23:41:44.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:44 smithi181 bash[24776]: audit 2024-09-12T23:41:43.852663+0000 mon.b (mon.1) 167 : audit [DBG] from='mgr.? 172.21.15.181:0/2917489088' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-12T23:41:44.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:44 smithi181 bash[24776]: audit 2024-09-12T23:41:43.853424+0000 mon.b (mon.1) 168 : audit [DBG] from='mgr.? 172.21.15.181:0/2917489088' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-12T23:41:44.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:44 smithi181 bash[24776]: audit 2024-09-12T23:41:43.854737+0000 mon.b (mon.1) 169 : audit [DBG] from='mgr.? 172.21.15.181:0/2917489088' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-12T23:41:44.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:44 smithi181 bash[24776]: audit 2024-09-12T23:41:43.855589+0000 mon.b (mon.1) 170 : audit [DBG] from='mgr.? 172.21.15.181:0/2917489088' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-12T23:41:44.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:44 smithi181 bash[24776]: audit 2024-09-12T23:41:44.165121+0000 mon.a (mon.0) 1438 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:44.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:44 smithi181 bash[24776]: audit 2024-09-12T23:41:44.178053+0000 mon.a (mon.0) 1439 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:45.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:44 smithi136 bash[22601]: cluster 2024-09-12T23:41:43.446937+0000 mgr.y (mgr.15123) 39 : cluster [DBG] pgmap v56: 121 pgs: 121 active+clean; 176 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:41:45.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:44 smithi136 bash[22601]: cluster 2024-09-12T23:41:43.621813+0000 mon.a (mon.0) 1435 : cluster [DBG] osdmap e268: 8 total, 8 up, 8 in 2024-09-12T23:41:45.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:44 smithi136 bash[22601]: cluster 2024-09-12T23:41:43.849314+0000 mon.a (mon.0) 1436 : cluster [DBG] Standby manager daemon x restarted 2024-09-12T23:41:45.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:44 smithi136 bash[22601]: cluster 2024-09-12T23:41:43.849552+0000 mon.a (mon.0) 1437 : cluster [DBG] Standby manager daemon x started 2024-09-12T23:41:45.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:44 smithi136 bash[22601]: audit 2024-09-12T23:41:43.852663+0000 mon.b (mon.1) 167 : audit [DBG] from='mgr.? 172.21.15.181:0/2917489088' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-12T23:41:45.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:44 smithi136 bash[22601]: audit 2024-09-12T23:41:43.853424+0000 mon.b (mon.1) 168 : audit [DBG] from='mgr.? 172.21.15.181:0/2917489088' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-12T23:41:45.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:44 smithi136 bash[22601]: audit 2024-09-12T23:41:43.854737+0000 mon.b (mon.1) 169 : audit [DBG] from='mgr.? 172.21.15.181:0/2917489088' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-12T23:41:45.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:44 smithi136 bash[22601]: audit 2024-09-12T23:41:43.855589+0000 mon.b (mon.1) 170 : audit [DBG] from='mgr.? 172.21.15.181:0/2917489088' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-12T23:41:45.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:44 smithi136 bash[22601]: audit 2024-09-12T23:41:44.165121+0000 mon.a (mon.0) 1438 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:45.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:44 smithi136 bash[22601]: audit 2024-09-12T23:41:44.178053+0000 mon.a (mon.0) 1439 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:45.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:44 smithi136 bash[32189]: cluster 2024-09-12T23:41:43.446937+0000 mgr.y (mgr.15123) 39 : cluster [DBG] pgmap v56: 121 pgs: 121 active+clean; 176 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:41:45.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:44 smithi136 bash[32189]: cluster 2024-09-12T23:41:43.621813+0000 mon.a (mon.0) 1435 : cluster [DBG] osdmap e268: 8 total, 8 up, 8 in 2024-09-12T23:41:45.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:44 smithi136 bash[32189]: cluster 2024-09-12T23:41:43.849314+0000 mon.a (mon.0) 1436 : cluster [DBG] Standby manager daemon x restarted 2024-09-12T23:41:45.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:44 smithi136 bash[32189]: cluster 2024-09-12T23:41:43.849552+0000 mon.a (mon.0) 1437 : cluster [DBG] Standby manager daemon x started 2024-09-12T23:41:45.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:44 smithi136 bash[32189]: audit 2024-09-12T23:41:43.852663+0000 mon.b (mon.1) 167 : audit [DBG] from='mgr.? 172.21.15.181:0/2917489088' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-12T23:41:45.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:44 smithi136 bash[32189]: audit 2024-09-12T23:41:43.853424+0000 mon.b (mon.1) 168 : audit [DBG] from='mgr.? 172.21.15.181:0/2917489088' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-12T23:41:45.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:44 smithi136 bash[32189]: audit 2024-09-12T23:41:43.854737+0000 mon.b (mon.1) 169 : audit [DBG] from='mgr.? 172.21.15.181:0/2917489088' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-12T23:41:45.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:44 smithi136 bash[32189]: audit 2024-09-12T23:41:43.855589+0000 mon.b (mon.1) 170 : audit [DBG] from='mgr.? 172.21.15.181:0/2917489088' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-12T23:41:45.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:44 smithi136 bash[32189]: audit 2024-09-12T23:41:44.165121+0000 mon.a (mon.0) 1438 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:45.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:44 smithi136 bash[32189]: audit 2024-09-12T23:41:44.178053+0000 mon.a (mon.0) 1439 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:45.671 INFO:tasks.rados.rados.0.smithi136.stdout:1120: done (3 left) 2024-09-12T23:41:45.671 INFO:tasks.rados.rados.0.smithi136.stdout:1121: delete oid 8 current snap is 106 2024-09-12T23:41:45.671 INFO:tasks.rados.rados.0.smithi136.stdout:1114: expect (ObjNum 353 snap 96 seq_num 353) 2024-09-12T23:41:45.792 INFO:tasks.rados.rados.0.smithi136.stdout:1116: expect (ObjNum 256 snap 70 seq_num 256) 2024-09-12T23:41:45.991 INFO:tasks.rados.rados.0.smithi136.stdout:1119: finishing write tid 1 to smithi13667081-1 2024-09-12T23:41:45.992 INFO:tasks.rados.rados.0.smithi136.stdout:1119: finishing write tid 2 to smithi13667081-1 2024-09-12T23:41:45.992 INFO:tasks.rados.rados.0.smithi136.stdout:1119: finishing write tid 3 to smithi13667081-1 2024-09-12T23:41:45.992 INFO:tasks.rados.rados.0.smithi136.stdout:1119: finishing write tid 4 to smithi13667081-1 2024-09-12T23:41:45.992 INFO:tasks.rados.rados.0.smithi136.stdout:1119: finishing write tid 5 to smithi13667081-1 2024-09-12T23:41:45.992 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 1 v 576 (ObjNum 366 snap 106 seq_num 366) dirty exists 2024-09-12T23:41:45.992 INFO:tasks.rados.rados.0.smithi136.stdout:1119: left oid 1 (ObjNum 366 snap 106 seq_num 366) 2024-09-12T23:41:45.992 INFO:tasks.rados.rados.0.smithi136.stdout:1114: done (3 left) 2024-09-12T23:41:45.992 INFO:tasks.rados.rados.0.smithi136.stdout:1116: done (2 left) 2024-09-12T23:41:45.992 INFO:tasks.rados.rados.0.smithi136.stdout:1119: done (1 left) 2024-09-12T23:41:45.992 INFO:tasks.rados.rados.0.smithi136.stdout:1121: done (0 left) 2024-09-12T23:41:45.992 INFO:tasks.rados.rados.0.smithi136.stdout:1122: snap_remove snap 78 2024-09-12T23:41:46.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:45 smithi136 bash[22601]: cluster 2024-09-12T23:41:44.658562+0000 mon.a (mon.0) 1440 : cluster [DBG] osdmap e269: 8 total, 8 up, 8 in 2024-09-12T23:41:46.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:45 smithi136 bash[22601]: cluster 2024-09-12T23:41:44.659957+0000 mon.a (mon.0) 1441 : cluster [DBG] mgrmap e36: y(active, since 37s), standbys: x 2024-09-12T23:41:46.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:45 smithi136 bash[32189]: cluster 2024-09-12T23:41:44.658562+0000 mon.a (mon.0) 1440 : cluster [DBG] osdmap e269: 8 total, 8 up, 8 in 2024-09-12T23:41:46.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:45 smithi136 bash[32189]: cluster 2024-09-12T23:41:44.659957+0000 mon.a (mon.0) 1441 : cluster [DBG] mgrmap e36: y(active, since 37s), standbys: x 2024-09-12T23:41:46.093 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:45 smithi181 bash[24776]: cluster 2024-09-12T23:41:44.658562+0000 mon.a (mon.0) 1440 : cluster [DBG] osdmap e269: 8 total, 8 up, 8 in 2024-09-12T23:41:46.093 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:45 smithi181 bash[24776]: cluster 2024-09-12T23:41:44.659957+0000 mon.a (mon.0) 1441 : cluster [DBG] mgrmap e36: y(active, since 37s), standbys: x 2024-09-12T23:41:46.677 INFO:tasks.rados.rados.0.smithi136.stdout:1122: done (0 left) 2024-09-12T23:41:46.677 INFO:tasks.rados.rados.0.smithi136.stdout:1123: copy_from oid 31 from oid 35 current snap is 106 2024-09-12T23:41:46.677 INFO:tasks.rados.rados.0.smithi136.stdout:1124: delete oid 47 current snap is 106 2024-09-12T23:41:46.707 INFO:tasks.rados.rados.0.smithi136.stdout:1124: done (1 left) 2024-09-12T23:41:46.708 INFO:tasks.rados.rados.0.smithi136.stdout:1125: snap_remove snap 105 2024-09-12T23:41:47.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:46 smithi136 bash[22601]: cluster 2024-09-12T23:41:45.447651+0000 mgr.y (mgr.15123) 40 : cluster [DBG] pgmap v59: 121 pgs: 121 active+clean; 191 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 3.6 MiB/s wr, 21 op/s 2024-09-12T23:41:47.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:46 smithi136 bash[22601]: cluster 2024-09-12T23:41:45.673746+0000 mon.a (mon.0) 1442 : cluster [DBG] osdmap e270: 8 total, 8 up, 8 in 2024-09-12T23:41:47.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:46 smithi136 bash[32189]: cluster 2024-09-12T23:41:45.447651+0000 mgr.y (mgr.15123) 40 : cluster [DBG] pgmap v59: 121 pgs: 121 active+clean; 191 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 3.6 MiB/s wr, 21 op/s 2024-09-12T23:41:47.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:46 smithi136 bash[32189]: cluster 2024-09-12T23:41:45.673746+0000 mon.a (mon.0) 1442 : cluster [DBG] osdmap e270: 8 total, 8 up, 8 in 2024-09-12T23:41:47.093 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:46 smithi181 bash[24776]: cluster 2024-09-12T23:41:45.447651+0000 mgr.y (mgr.15123) 40 : cluster [DBG] pgmap v59: 121 pgs: 121 active+clean; 191 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 3.6 MiB/s wr, 21 op/s 2024-09-12T23:41:47.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:46 smithi181 bash[24776]: cluster 2024-09-12T23:41:45.673746+0000 mon.a (mon.0) 1442 : cluster [DBG] osdmap e270: 8 total, 8 up, 8 in 2024-09-12T23:41:47.686 INFO:tasks.rados.rados.0.smithi136.stdout:1125: done (1 left) 2024-09-12T23:41:47.686 INFO:tasks.rados.rados.0.smithi136.stdout:1126: snap_create 2024-09-12T23:41:47.686 INFO:tasks.rados.rados.0.smithi136.stdout:1123: finishing copy_from racing read to smithi13667081-31 2024-09-12T23:41:47.686 INFO:tasks.rados.rados.0.smithi136.stdout:1123: finishing copy_from to smithi13667081-31 2024-09-12T23:41:47.687 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 31 v 339 (ObjNum 262 snap 71 seq_num 262) dirty exists 2024-09-12T23:41:48.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:47 smithi136 bash[22601]: cluster 2024-09-12T23:41:46.679779+0000 mon.a (mon.0) 1443 : cluster [DBG] osdmap e271: 8 total, 8 up, 8 in 2024-09-12T23:41:48.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:47 smithi136 bash[32189]: cluster 2024-09-12T23:41:46.679779+0000 mon.a (mon.0) 1443 : cluster [DBG] osdmap e271: 8 total, 8 up, 8 in 2024-09-12T23:41:48.093 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:47 smithi181 bash[24776]: cluster 2024-09-12T23:41:46.679779+0000 mon.a (mon.0) 1443 : cluster [DBG] osdmap e271: 8 total, 8 up, 8 in 2024-09-12T23:41:48.688 INFO:tasks.rados.rados.0.smithi136.stdout:1123: done (1 left) 2024-09-12T23:41:48.688 INFO:tasks.rados.rados.0.smithi136.stdout:1126: done (0 left) 2024-09-12T23:41:48.688 INFO:tasks.rados.rados.0.smithi136.stdout:1127: copy_from oid 20 from oid 32 current snap is 107 2024-09-12T23:41:48.688 INFO:tasks.rados.rados.0.smithi136.stdout:1128: delete oid 3 current snap is 107 2024-09-12T23:41:48.704 INFO:tasks.rados.rados.0.smithi136.stdout:1128: done (1 left) 2024-09-12T23:41:48.705 INFO:tasks.rados.rados.0.smithi136.stdout:1129: copy_from oid 45 from oid 37 current snap is 107 2024-09-12T23:41:48.705 INFO:tasks.rados.rados.0.smithi136.stdout:1130: snap_remove snap 98 2024-09-12T23:41:49.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:48 smithi136 bash[22601]: cluster 2024-09-12T23:41:47.448362+0000 mgr.y (mgr.15123) 41 : cluster [DBG] pgmap v62: 121 pgs: 121 active+clean; 191 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 3.6 MiB/s wr, 21 op/s 2024-09-12T23:41:49.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:48 smithi136 bash[22601]: cluster 2024-09-12T23:41:47.689873+0000 mon.a (mon.0) 1444 : cluster [DBG] osdmap e272: 8 total, 8 up, 8 in 2024-09-12T23:41:49.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:48 smithi136 bash[32189]: cluster 2024-09-12T23:41:47.448362+0000 mgr.y (mgr.15123) 41 : cluster [DBG] pgmap v62: 121 pgs: 121 active+clean; 191 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 3.6 MiB/s wr, 21 op/s 2024-09-12T23:41:49.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:48 smithi136 bash[32189]: cluster 2024-09-12T23:41:47.689873+0000 mon.a (mon.0) 1444 : cluster [DBG] osdmap e272: 8 total, 8 up, 8 in 2024-09-12T23:41:49.093 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:48 smithi181 bash[24776]: cluster 2024-09-12T23:41:47.448362+0000 mgr.y (mgr.15123) 41 : cluster [DBG] pgmap v62: 121 pgs: 121 active+clean; 191 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 3.6 MiB/s wr, 21 op/s 2024-09-12T23:41:49.093 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:48 smithi181 bash[24776]: cluster 2024-09-12T23:41:47.689873+0000 mon.a (mon.0) 1444 : cluster [DBG] osdmap e272: 8 total, 8 up, 8 in 2024-09-12T23:41:49.094 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:48 smithi181 bash[60471]: ts=2024-09-12T23:41:48.662Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:41:49.698 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:41:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:41:49] "GET /metrics HTTP/1.1" 200 36139 "" "Prometheus/2.43.0" 2024-09-12T23:41:49.707 INFO:tasks.rados.rados.0.smithi136.stdout:1130: done (2 left) 2024-09-12T23:41:49.707 INFO:tasks.rados.rados.0.smithi136.stdout:1131: snap_create 2024-09-12T23:41:49.707 INFO:tasks.rados.rados.0.smithi136.stdout:1127: finishing copy_from racing read to smithi13667081-20 2024-09-12T23:41:49.707 INFO:tasks.rados.rados.0.smithi136.stdout:1127: finishing copy_from to smithi13667081-20 2024-09-12T23:41:49.707 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 20 v 264 (ObjNum 353 snap 96 seq_num 353) dirty exists 2024-09-12T23:41:49.708 INFO:tasks.rados.rados.0.smithi136.stdout:1129: finishing copy_from racing read to smithi13667081-45 2024-09-12T23:41:49.708 INFO:tasks.rados.rados.0.smithi136.stdout:1129: finishing copy_from to smithi13667081-45 2024-09-12T23:41:49.708 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 45 v 540 (ObjNum 351 snap 96 seq_num 351) dirty exists 2024-09-12T23:41:50.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:49 smithi136 bash[22601]: cluster 2024-09-12T23:41:48.690813+0000 mon.a (mon.0) 1445 : cluster [DBG] osdmap e273: 8 total, 8 up, 8 in 2024-09-12T23:41:50.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:49 smithi136 bash[32189]: cluster 2024-09-12T23:41:48.690813+0000 mon.a (mon.0) 1445 : cluster [DBG] osdmap e273: 8 total, 8 up, 8 in 2024-09-12T23:41:50.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:49 smithi181 bash[24776]: cluster 2024-09-12T23:41:48.690813+0000 mon.a (mon.0) 1445 : cluster [DBG] osdmap e273: 8 total, 8 up, 8 in 2024-09-12T23:41:50.523 INFO:tasks.rados.rados.0.smithi136.stdout:1127: done (2 left) 2024-09-12T23:41:50.523 INFO:tasks.rados.rados.0.smithi136.stdout:1129: done (1 left) 2024-09-12T23:41:50.523 INFO:tasks.rados.rados.0.smithi136.stdout:1131: done (0 left) 2024-09-12T23:41:50.523 INFO:tasks.rados.rados.0.smithi136.stdout:1132: snap_remove snap 95 2024-09-12T23:41:51.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:50 smithi136 bash[22601]: cluster 2024-09-12T23:41:49.449162+0000 mgr.y (mgr.15123) 42 : cluster [DBG] pgmap v65: 121 pgs: 121 active+clean; 174 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 1.3 MiB/s wr, 5 op/s 2024-09-12T23:41:51.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:50 smithi136 bash[22601]: cluster 2024-09-12T23:41:49.711348+0000 mon.a (mon.0) 1446 : cluster [DBG] osdmap e274: 8 total, 8 up, 8 in 2024-09-12T23:41:51.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:50 smithi136 bash[22601]: cluster 2024-09-12T23:41:50.526623+0000 mon.a (mon.0) 1447 : cluster [DBG] osdmap e275: 8 total, 8 up, 8 in 2024-09-12T23:41:51.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:50 smithi136 bash[32189]: cluster 2024-09-12T23:41:49.449162+0000 mgr.y (mgr.15123) 42 : cluster [DBG] pgmap v65: 121 pgs: 121 active+clean; 174 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 1.3 MiB/s wr, 5 op/s 2024-09-12T23:41:51.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:50 smithi136 bash[32189]: cluster 2024-09-12T23:41:49.711348+0000 mon.a (mon.0) 1446 : cluster [DBG] osdmap e274: 8 total, 8 up, 8 in 2024-09-12T23:41:51.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:50 smithi136 bash[32189]: cluster 2024-09-12T23:41:50.526623+0000 mon.a (mon.0) 1447 : cluster [DBG] osdmap e275: 8 total, 8 up, 8 in 2024-09-12T23:41:51.093 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:50 smithi181 bash[24776]: cluster 2024-09-12T23:41:49.449162+0000 mgr.y (mgr.15123) 42 : cluster [DBG] pgmap v65: 121 pgs: 121 active+clean; 174 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 1.3 MiB/s wr, 5 op/s 2024-09-12T23:41:51.093 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:50 smithi181 bash[24776]: cluster 2024-09-12T23:41:49.711348+0000 mon.a (mon.0) 1446 : cluster [DBG] osdmap e274: 8 total, 8 up, 8 in 2024-09-12T23:41:51.093 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:50 smithi181 bash[24776]: cluster 2024-09-12T23:41:50.526623+0000 mon.a (mon.0) 1447 : cluster [DBG] osdmap e275: 8 total, 8 up, 8 in 2024-09-12T23:41:51.523 INFO:tasks.rados.rados.0.smithi136.stdout:1132: done (0 left) 2024-09-12T23:41:51.523 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 25 current snap is 108 2024-09-12T23:41:51.523 INFO:tasks.rados.rados.0.smithi136.stdout:1133: seq_num 370 ranges {0=606208,606208=794624,1400832=8192} 2024-09-12T23:41:51.526 INFO:tasks.rados.rados.0.smithi136.stdout:1133: writing smithi13667081-25 from 0 to 606208 tid 1 2024-09-12T23:41:51.533 INFO:tasks.rados.rados.0.smithi136.stdout:1133: writing smithi13667081-25 from 606208 to 1400832 tid 2 2024-09-12T23:41:51.533 INFO:tasks.rados.rados.0.smithi136.stdout:1133: writing smithi13667081-25 from 1400832 to 1409024 tid 3 2024-09-12T23:41:51.533 INFO:tasks.rados.rados.0.smithi136.stdout:1134: copy_from oid 1 from oid 10 current snap is 108 2024-09-12T23:41:51.533 INFO:tasks.rados.rados.0.smithi136.stdout:1135: read oid 46 snap -1 2024-09-12T23:41:51.533 INFO:tasks.rados.rados.0.smithi136.stdout:1135: expect deleted 2024-09-12T23:41:51.533 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 24 current snap is 108 2024-09-12T23:41:51.533 INFO:tasks.rados.rados.0.smithi136.stdout:1136: seq_num 371 ranges {4014080=622592,4636672=679936,5316608=122880} 2024-09-12T23:41:51.561 INFO:tasks.rados.rados.0.smithi136.stdout:1136: writing smithi13667081-24 from 4014080 to 4636672 tid 1 2024-09-12T23:41:51.565 INFO:tasks.rados.rados.0.smithi136.stdout:1136: writing smithi13667081-24 from 4636672 to 5316608 tid 2 2024-09-12T23:41:51.566 INFO:tasks.rados.rados.0.smithi136.stdout:1136: writing smithi13667081-24 from 5316608 to 5439488 tid 3 2024-09-12T23:41:51.566 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 5 current snap is 108 2024-09-12T23:41:51.567 INFO:tasks.rados.rados.0.smithi136.stdout:1137: seq_num 372 ranges {2236416=638976,2875392=565248,3440640=237568} 2024-09-12T23:41:51.583 INFO:tasks.rados.rados.0.smithi136.stdout:1137: writing smithi13667081-5 from 2236416 to 2875392 tid 1 2024-09-12T23:41:51.587 INFO:tasks.rados.rados.0.smithi136.stdout:1137: writing smithi13667081-5 from 2875392 to 3440640 tid 2 2024-09-12T23:41:51.589 INFO:tasks.rados.rados.0.smithi136.stdout:1137: writing smithi13667081-5 from 3440640 to 3678208 tid 3 2024-09-12T23:41:51.589 INFO:tasks.rados.rados.0.smithi136.stdout:1138: rollback oid 26 current snap is 108 2024-09-12T23:41:51.589 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 26 to 106 2024-09-12T23:41:51.590 INFO:tasks.rados.rados.0.smithi136.stdout:1133: finishing write tid 1 to smithi13667081-25 2024-09-12T23:41:51.590 INFO:tasks.rados.rados.0.smithi136.stdout:1133: finishing write tid 2 to smithi13667081-25 2024-09-12T23:41:51.590 INFO:tasks.rados.rados.0.smithi136.stdout:1133: finishing write tid 3 to smithi13667081-25 2024-09-12T23:41:51.590 INFO:tasks.rados.rados.0.smithi136.stdout:1133: finishing write tid 4 to smithi13667081-25 2024-09-12T23:41:51.590 INFO:tasks.rados.rados.0.smithi136.stdout:1133: finishing write tid 5 to smithi13667081-25 2024-09-12T23:41:51.590 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 25 v 180 (ObjNum 370 snap 108 seq_num 370) dirty exists 2024-09-12T23:41:51.590 INFO:tasks.rados.rados.0.smithi136.stdout:1133: left oid 25 (ObjNum 370 snap 108 seq_num 370) 2024-09-12T23:41:51.590 INFO:tasks.rados.rados.0.smithi136.stdout:1136: finishing write tid 1 to smithi13667081-24 2024-09-12T23:41:51.591 INFO:tasks.rados.rados.0.smithi136.stdout:1136: finishing write tid 2 to smithi13667081-24 2024-09-12T23:41:51.591 INFO:tasks.rados.rados.0.smithi136.stdout:1136: finishing write tid 3 to smithi13667081-24 2024-09-12T23:41:51.591 INFO:tasks.rados.rados.0.smithi136.stdout:1136: finishing write tid 4 to smithi13667081-24 2024-09-12T23:41:51.591 INFO:tasks.rados.rados.0.smithi136.stdout:1134: finishing copy_from racing read to smithi13667081-1 2024-09-12T23:41:51.591 INFO:tasks.rados.rados.0.smithi136.stdout:1134: finishing copy_from to smithi13667081-1 2024-09-12T23:41:51.591 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 1 v 611 (ObjNum 350 snap 95 seq_num 350) dirty exists 2024-09-12T23:41:51.591 INFO:tasks.rados.rados.0.smithi136.stdout:1136: finishing write tid 5 to smithi13667081-24 2024-09-12T23:41:51.591 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 24 v 389 (ObjNum 371 snap 108 seq_num 371) dirty exists 2024-09-12T23:41:51.591 INFO:tasks.rados.rados.0.smithi136.stdout:1136: left oid 24 (ObjNum 371 snap 108 seq_num 371) 2024-09-12T23:41:51.591 INFO:tasks.rados.rados.0.smithi136.stdout:1133: done (5 left) 2024-09-12T23:41:51.591 INFO:tasks.rados.rados.0.smithi136.stdout:1134: done (4 left) 2024-09-12T23:41:51.592 INFO:tasks.rados.rados.0.smithi136.stdout:1135: done (3 left) 2024-09-12T23:41:51.592 INFO:tasks.rados.rados.0.smithi136.stdout:1136: done (2 left) 2024-09-12T23:41:51.592 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 35 current snap is 108 2024-09-12T23:41:51.592 INFO:tasks.rados.rados.0.smithi136.stdout:1139: seq_num 373 ranges {4300800=655360,4956160=499712,5455872=303104} 2024-09-12T23:41:51.616 INFO:tasks.rados.rados.0.smithi136.stdout:1139: writing smithi13667081-35 from 4300800 to 4956160 tid 1 2024-09-12T23:41:51.619 INFO:tasks.rados.rados.0.smithi136.stdout:1139: writing smithi13667081-35 from 4956160 to 5455872 tid 2 2024-09-12T23:41:51.622 INFO:tasks.rados.rados.0.smithi136.stdout:1139: writing smithi13667081-35 from 5455872 to 5758976 tid 3 2024-09-12T23:41:51.622 INFO:tasks.rados.rados.0.smithi136.stdout:1140: copy_from oid 50 from oid 30 current snap is 108 2024-09-12T23:41:51.622 INFO:tasks.rados.rados.0.smithi136.stdout:1137: finishing write tid 1 to smithi13667081-5 2024-09-12T23:41:51.622 INFO:tasks.rados.rados.0.smithi136.stdout:1137: finishing write tid 2 to smithi13667081-5 2024-09-12T23:41:51.622 INFO:tasks.rados.rados.0.smithi136.stdout:1137: finishing write tid 3 to smithi13667081-5 2024-09-12T23:41:51.622 INFO:tasks.rados.rados.0.smithi136.stdout:1138: finishing rollback tid 0 to smithi13667081-26 2024-09-12T23:41:51.622 INFO:tasks.rados.rados.0.smithi136.stdout:1137: finishing write tid 4 to smithi13667081-5 2024-09-12T23:41:51.622 INFO:tasks.rados.rados.0.smithi136.stdout:1138: finishing rollback tid 1 to smithi13667081-26 2024-09-12T23:41:51.623 INFO:tasks.rados.rados.0.smithi136.stdout:1137: finishing write tid 5 to smithi13667081-5 2024-09-12T23:41:51.623 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 5 v 714 (ObjNum 372 snap 108 seq_num 372) dirty exists 2024-09-12T23:41:51.623 INFO:tasks.rados.rados.0.smithi136.stdout:1137: left oid 5 (ObjNum 372 snap 108 seq_num 372) 2024-09-12T23:41:51.623 INFO:tasks.rados.rados.0.smithi136.stdout:1138: finishing rollback tid 2 to smithi13667081-26 2024-09-12T23:41:51.623 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 26 v 392 (ObjNum 305 snap 84 seq_num 305) dirty exists 2024-09-12T23:41:51.623 INFO:tasks.rados.rados.0.smithi136.stdout:1137: done (3 left) 2024-09-12T23:41:51.623 INFO:tasks.rados.rados.0.smithi136.stdout:1138: done (2 left) 2024-09-12T23:41:51.623 INFO:tasks.rados.rados.0.smithi136.stdout:1141: copy_from oid 41 from oid 3 current snap is 108 2024-09-12T23:41:51.623 INFO:tasks.rados.rados.0.smithi136.stdout:1142: read oid 42 snap -1 2024-09-12T23:41:51.623 INFO:tasks.rados.rados.0.smithi136.stdout:1142: expect (ObjNum 125 snap 32 seq_num 125) 2024-09-12T23:41:51.623 INFO:tasks.rados.rados.0.smithi136.stdout:1143: setattr oid 15 current snap is 108 2024-09-12T23:41:51.623 INFO:tasks.rados.rados.0.smithi136.stdout:1139: finishing write tid 1 to smithi13667081-35 2024-09-12T23:41:51.625 INFO:tasks.rados.rados.0.smithi136.stdout:1144: rollback oid 23 current snap is 108 2024-09-12T23:41:51.625 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 23 to 93 2024-09-12T23:41:51.625 INFO:tasks.rados.rados.0.smithi136.stdout:1145: rollback oid 46 current snap is 108 2024-09-12T23:41:51.625 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 46 to 99 2024-09-12T23:41:51.625 INFO:tasks.rados.rados.0.smithi136.stdout:1146: read oid 40 snap -1 2024-09-12T23:41:51.625 INFO:tasks.rados.rados.0.smithi136.stdout:1146: expect (ObjNum 331 snap 90 seq_num 331) 2024-09-12T23:41:51.625 INFO:tasks.rados.rados.0.smithi136.stdout:1147: delete oid 28 current snap is 108 2024-09-12T23:41:51.625 INFO:tasks.rados.rados.0.smithi136.stdout:1141: finishing copy_from to smithi13667081-41 2024-09-12T23:41:51.625 INFO:tasks.rados.rados.0.smithi136.stdout:1141: got expected ENOENT (src dne) 2024-09-12T23:41:51.625 INFO:tasks.rados.rados.0.smithi136.stdout:1141: finishing copy_from racing read to smithi13667081-41 2024-09-12T23:41:51.625 INFO:tasks.rados.rados.0.smithi136.stdout:1145: finishing rollback tid 1 to smithi13667081-46 2024-09-12T23:41:51.625 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 46 v 0 (ObjNum 33 snap 0 seq_num 1784769907) dirty dne 2024-09-12T23:41:51.629 INFO:tasks.rados.rados.0.smithi136.stdout:1140: finishing copy_from racing read to smithi13667081-50 2024-09-12T23:41:51.631 INFO:tasks.rados.rados.0.smithi136.stdout:1141: done (8 left) 2024-09-12T23:41:51.632 INFO:tasks.rados.rados.0.smithi136.stdout:1145: done (7 left) 2024-09-12T23:41:51.632 INFO:tasks.rados.rados.0.smithi136.stdout:1147: done (6 left) 2024-09-12T23:41:51.632 INFO:tasks.rados.rados.0.smithi136.stdout:1148: snap_remove snap 106 2024-09-12T23:41:52.525 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 15 v 565 (ObjNum 354 snap 96 seq_num 354) dirty exists 2024-09-12T23:41:52.525 INFO:tasks.rados.rados.0.smithi136.stdout:1140: finishing copy_from to smithi13667081-50 2024-09-12T23:41:52.525 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 50 v 209 (ObjNum 292 snap 79 seq_num 292) dirty exists 2024-09-12T23:41:52.525 INFO:tasks.rados.rados.0.smithi136.stdout:1139: finishing write tid 2 to smithi13667081-35 2024-09-12T23:41:52.525 INFO:tasks.rados.rados.0.smithi136.stdout:1142: expect (ObjNum 125 snap 32 seq_num 125) 2024-09-12T23:41:52.584 INFO:tasks.rados.rados.0.smithi136.stdout:1139: finishing write tid 3 to smithi13667081-35 2024-09-12T23:41:52.584 INFO:tasks.rados.rados.0.smithi136.stdout:1139: finishing write tid 4 to smithi13667081-35 2024-09-12T23:41:52.585 INFO:tasks.rados.rados.0.smithi136.stdout:1144: finishing rollback tid 0 to smithi13667081-23 2024-09-12T23:41:52.585 INFO:tasks.rados.rados.0.smithi136.stdout:1144: finishing rollback tid 1 to smithi13667081-23 2024-09-12T23:41:52.585 INFO:tasks.rados.rados.0.smithi136.stdout:1144: finishing rollback tid 2 to smithi13667081-23 2024-09-12T23:41:52.585 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 23 v 723 (ObjNum 317 snap 86 seq_num 317) dirty exists 2024-09-12T23:41:52.585 INFO:tasks.rados.rados.0.smithi136.stdout:1139: finishing write tid 5 to smithi13667081-35 2024-09-12T23:41:52.585 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 35 v 719 (ObjNum 373 snap 108 seq_num 373) dirty exists 2024-09-12T23:41:52.585 INFO:tasks.rados.rados.0.smithi136.stdout:1139: left oid 35 (ObjNum 373 snap 108 seq_num 373) 2024-09-12T23:41:52.585 INFO:tasks.rados.rados.0.smithi136.stdout:1146: expect (ObjNum 331 snap 90 seq_num 331) 2024-09-12T23:41:52.593 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:52 smithi181 bash[24776]: cluster 2024-09-12T23:41:51.449919+0000 mgr.y (mgr.15123) 43 : cluster [DBG] pgmap v68: 121 pgs: 121 active+clean; 174 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 1.3 MiB/s wr, 5 op/s 2024-09-12T23:41:52.593 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:52 smithi181 bash[24776]: cluster 2024-09-12T23:41:51.526959+0000 mon.a (mon.0) 1448 : cluster [DBG] osdmap e276: 8 total, 8 up, 8 in 2024-09-12T23:41:52.593 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:52 smithi181 bash[60471]: ts=2024-09-12T23:41:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:41:52.700 INFO:tasks.rados.rados.0.smithi136.stdout:1139: done (6 left) 2024-09-12T23:41:52.700 INFO:tasks.rados.rados.0.smithi136.stdout:1140: done (5 left) 2024-09-12T23:41:52.701 INFO:tasks.rados.rados.0.smithi136.stdout:1142: done (4 left) 2024-09-12T23:41:52.701 INFO:tasks.rados.rados.0.smithi136.stdout:1143: done (3 left) 2024-09-12T23:41:52.701 INFO:tasks.rados.rados.0.smithi136.stdout:1144: done (2 left) 2024-09-12T23:41:52.701 INFO:tasks.rados.rados.0.smithi136.stdout:1146: done (1 left) 2024-09-12T23:41:52.701 INFO:tasks.rados.rados.0.smithi136.stdout:1148: done (0 left) 2024-09-12T23:41:52.701 INFO:tasks.rados.rados.0.smithi136.stdout:1149: read oid 26 snap -1 2024-09-12T23:41:52.701 INFO:tasks.rados.rados.0.smithi136.stdout:1149: expect (ObjNum 305 snap 84 seq_num 305) 2024-09-12T23:41:52.701 INFO:tasks.rados.rados.0.smithi136.stdout:1150: rollback oid 47 current snap is 108 2024-09-12T23:41:52.701 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 47 to 101 2024-09-12T23:41:52.701 INFO:tasks.rados.rados.0.smithi136.stdout:1151: copy_from oid 21 from oid 35 current snap is 108 2024-09-12T23:41:52.701 INFO:tasks.rados.rados.0.smithi136.stdout:1152: delete oid 3 current snap is 108 2024-09-12T23:41:52.705 INFO:tasks.rados.rados.0.smithi136.stdout:1150: finishing rollback tid 1 to smithi13667081-47 2024-09-12T23:41:52.705 INFO:tasks.rados.rados.0.smithi136.stdout:1152: done (3 left) 2024-09-12T23:41:52.705 INFO:tasks.rados.rados.0.smithi136.stdout:1153: rmattr oid 23 current snap is 108 2024-09-12T23:41:52.705 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 9 current snap is 108 2024-09-12T23:41:52.705 INFO:tasks.rados.rados.0.smithi136.stdout:1154: seq_num 376 ranges {2785280=704512,3489792=548864,4038656=253952} 2024-09-12T23:41:52.725 INFO:tasks.rados.rados.0.smithi136.stdout:1154: writing smithi13667081-9 from 2785280 to 3489792 tid 1 2024-09-12T23:41:52.728 INFO:tasks.rados.rados.0.smithi136.stdout:1154: writing smithi13667081-9 from 3489792 to 4038656 tid 2 2024-09-12T23:41:52.730 INFO:tasks.rados.rados.0.smithi136.stdout:1154: writing smithi13667081-9 from 4038656 to 4292608 tid 3 2024-09-12T23:41:52.730 INFO:tasks.rados.rados.0.smithi136.stdout:1155: snap_create 2024-09-12T23:41:52.730 INFO:tasks.rados.rados.0.smithi136.stdout:1150: finishing rollback tid 2 to smithi13667081-47 2024-09-12T23:41:52.730 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 47 v 572 (ObjNum 323 snap 88 seq_num 323) dirty exists 2024-09-12T23:41:52.731 INFO:tasks.rados.rados.0.smithi136.stdout:1149: expect (ObjNum 305 snap 84 seq_num 305) 2024-09-12T23:41:52.857 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 23 v 730 (ObjNum 317 snap 86 seq_num 317) dirty exists 2024-09-12T23:41:52.857 INFO:tasks.rados.rados.0.smithi136.stdout:1154: finishing write tid 1 to smithi13667081-9 2024-09-12T23:41:52.857 INFO:tasks.rados.rados.0.smithi136.stdout:1154: finishing write tid 2 to smithi13667081-9 2024-09-12T23:41:52.857 INFO:tasks.rados.rados.0.smithi136.stdout:1154: finishing write tid 3 to smithi13667081-9 2024-09-12T23:41:52.857 INFO:tasks.rados.rados.0.smithi136.stdout:1154: finishing write tid 4 to smithi13667081-9 2024-09-12T23:41:52.857 INFO:tasks.rados.rados.0.smithi136.stdout:1151: finishing copy_from racing read to smithi13667081-21 2024-09-12T23:41:52.857 INFO:tasks.rados.rados.0.smithi136.stdout:1154: finishing write tid 5 to smithi13667081-9 2024-09-12T23:41:52.857 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 9 v 576 (ObjNum 376 snap 108 seq_num 376) dirty exists 2024-09-12T23:41:52.857 INFO:tasks.rados.rados.0.smithi136.stdout:1154: left oid 9 (ObjNum 376 snap 108 seq_num 376) 2024-09-12T23:41:52.858 INFO:tasks.rados.rados.0.smithi136.stdout:1151: finishing copy_from to smithi13667081-21 2024-09-12T23:41:52.858 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 21 v 719 (ObjNum 373 snap 108 seq_num 373) dirty exists 2024-09-12T23:41:53.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:52 smithi136 bash[22601]: cluster 2024-09-12T23:41:51.449919+0000 mgr.y (mgr.15123) 43 : cluster [DBG] pgmap v68: 121 pgs: 121 active+clean; 174 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 1.3 MiB/s wr, 5 op/s 2024-09-12T23:41:53.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:52 smithi136 bash[22601]: cluster 2024-09-12T23:41:51.526959+0000 mon.a (mon.0) 1448 : cluster [DBG] osdmap e276: 8 total, 8 up, 8 in 2024-09-12T23:41:53.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:52 smithi136 bash[32189]: cluster 2024-09-12T23:41:51.449919+0000 mgr.y (mgr.15123) 43 : cluster [DBG] pgmap v68: 121 pgs: 121 active+clean; 174 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 1.3 MiB/s wr, 5 op/s 2024-09-12T23:41:53.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:52 smithi136 bash[32189]: cluster 2024-09-12T23:41:51.526959+0000 mon.a (mon.0) 1448 : cluster [DBG] osdmap e276: 8 total, 8 up, 8 in 2024-09-12T23:41:53.549 INFO:tasks.rados.rados.0.smithi136.stdout:1149: done (5 left) 2024-09-12T23:41:53.550 INFO:tasks.rados.rados.0.smithi136.stdout:1150: done (4 left) 2024-09-12T23:41:53.550 INFO:tasks.rados.rados.0.smithi136.stdout:1151: done (3 left) 2024-09-12T23:41:53.550 INFO:tasks.rados.rados.0.smithi136.stdout:1153: done (2 left) 2024-09-12T23:41:53.550 INFO:tasks.rados.rados.0.smithi136.stdout:1154: done (1 left) 2024-09-12T23:41:53.550 INFO:tasks.rados.rados.0.smithi136.stdout:1155: done (0 left) 2024-09-12T23:41:53.550 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 39 current snap is 109 2024-09-12T23:41:53.550 INFO:tasks.rados.rados.0.smithi136.stdout:1156: seq_num 377 ranges {1810432=720896,2531328=434176,2965504=368640} 2024-09-12T23:41:53.579 INFO:tasks.rados.rados.0.smithi136.stdout:1156: writing smithi13667081-39 from 1810432 to 2531328 tid 1 2024-09-12T23:41:53.583 INFO:tasks.rados.rados.0.smithi136.stdout:1156: writing smithi13667081-39 from 2531328 to 2965504 tid 2 2024-09-12T23:41:53.587 INFO:tasks.rados.rados.0.smithi136.stdout:1156: writing smithi13667081-39 from 2965504 to 3334144 tid 3 2024-09-12T23:41:53.587 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 43 current snap is 109 2024-09-12T23:41:53.587 INFO:tasks.rados.rados.0.smithi136.stdout:1157: seq_num 378 ranges {0=737280,737280=720896,1458176=81920} 2024-09-12T23:41:53.595 INFO:tasks.rados.rados.0.smithi136.stdout:1157: writing smithi13667081-43 from 0 to 737280 tid 1 2024-09-12T23:41:53.603 INFO:tasks.rados.rados.0.smithi136.stdout:1157: writing smithi13667081-43 from 737280 to 1458176 tid 2 2024-09-12T23:41:53.604 INFO:tasks.rados.rados.0.smithi136.stdout:1157: writing smithi13667081-43 from 1458176 to 1540096 tid 3 2024-09-12T23:41:53.604 INFO:tasks.rados.rados.0.smithi136.stdout:1158: read oid 2 snap -1 2024-09-12T23:41:53.604 INFO:tasks.rados.rados.0.smithi136.stdout:1158: expect (ObjNum 364 snap 105 seq_num 364) 2024-09-12T23:41:53.604 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 11 current snap is 109 2024-09-12T23:41:53.604 INFO:tasks.rados.rados.0.smithi136.stdout:1156: finishing write tid 1 to smithi13667081-39 2024-09-12T23:41:53.604 INFO:tasks.rados.rados.0.smithi136.stdout:1156: finishing write tid 2 to smithi13667081-39 2024-09-12T23:41:53.605 INFO:tasks.rados.rados.0.smithi136.stdout:1156: finishing write tid 3 to smithi13667081-39 2024-09-12T23:41:53.605 INFO:tasks.rados.rados.0.smithi136.stdout:1156: finishing write tid 4 to smithi13667081-39 2024-09-12T23:41:53.605 INFO:tasks.rados.rados.0.smithi136.stdout:1156: finishing write tid 5 to smithi13667081-39 2024-09-12T23:41:53.605 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 39 v 185 (ObjNum 377 snap 109 seq_num 377) dirty exists 2024-09-12T23:41:53.605 INFO:tasks.rados.rados.0.smithi136.stdout:1156: left oid 39 (ObjNum 377 snap 109 seq_num 377) 2024-09-12T23:41:53.605 INFO:tasks.rados.rados.0.smithi136.stdout:1157: finishing write tid 1 to smithi13667081-43 2024-09-12T23:41:53.605 INFO:tasks.rados.rados.0.smithi136.stdout:1159: seq_num 379 ranges {2129920=753664,2883584=606208,3489792=196608} 2024-09-12T23:41:53.629 INFO:tasks.rados.rados.0.smithi136.stdout:1159: writing smithi13667081-11 from 2129920 to 2883584 tid 1 2024-09-12T23:41:53.634 INFO:tasks.rados.rados.0.smithi136.stdout:1159: writing smithi13667081-11 from 2883584 to 3489792 tid 2 2024-09-12T23:41:53.636 INFO:tasks.rados.rados.0.smithi136.stdout:1159: writing smithi13667081-11 from 3489792 to 3686400 tid 3 2024-09-12T23:41:53.636 INFO:tasks.rados.rados.0.smithi136.stdout:1156: done (3 left) 2024-09-12T23:41:53.636 INFO:tasks.rados.rados.0.smithi136.stdout:1160: rmattr oid 48 current snap is 109 2024-09-12T23:41:53.636 INFO:tasks.rados.rados.0.smithi136.stdout:1160: done (3 left) 2024-09-12T23:41:53.636 INFO:tasks.rados.rados.0.smithi136.stdout:1161: rmattr oid 19 current snap is 109 2024-09-12T23:41:53.636 INFO:tasks.rados.rados.0.smithi136.stdout:1157: finishing write tid 2 to smithi13667081-43 2024-09-12T23:41:53.636 INFO:tasks.rados.rados.0.smithi136.stdout:1157: finishing write tid 3 to smithi13667081-43 2024-09-12T23:41:53.636 INFO:tasks.rados.rados.0.smithi136.stdout:1157: finishing write tid 4 to smithi13667081-43 2024-09-12T23:41:53.643 INFO:tasks.rados.rados.0.smithi136.stdout:1157: finishing write tid 5 to smithi13667081-43 2024-09-12T23:41:53.643 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 43 v 661 (ObjNum 378 snap 109 seq_num 378) dirty exists 2024-09-12T23:41:53.643 INFO:tasks.rados.rados.0.smithi136.stdout:1157: left oid 43 (ObjNum 378 snap 109 seq_num 378) 2024-09-12T23:41:53.644 INFO:tasks.rados.rados.0.smithi136.stdout:1158: expect (ObjNum 364 snap 105 seq_num 364) 2024-09-12T23:41:53.843 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:53 smithi181 bash[60471]: ts=2024-09-12T23:41:53.515Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi136\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:41:53.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:53 smithi181 bash[24776]: cluster 2024-09-12T23:41:52.528084+0000 mon.a (mon.0) 1449 : cluster [DBG] osdmap e277: 8 total, 8 up, 8 in 2024-09-12T23:41:53.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:53 smithi181 bash[24776]: audit 2024-09-12T23:41:52.642858+0000 mon.b (mon.1) 171 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:41:53.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:53 smithi181 bash[24776]: audit 2024-09-12T23:41:53.419800+0000 mon.a (mon.0) 1450 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:53.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:53 smithi181 bash[24776]: audit 2024-09-12T23:41:53.433289+0000 mon.a (mon.0) 1451 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:53.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:53 smithi181 bash[24776]: audit 2024-09-12T23:41:53.438093+0000 mon.b (mon.1) 172 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:41:53.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:53 smithi181 bash[24776]: audit 2024-09-12T23:41:53.440156+0000 mon.b (mon.1) 173 : audit [INF] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:41:53.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:53 smithi181 bash[24776]: audit 2024-09-12T23:41:53.450251+0000 mon.a (mon.0) 1452 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:53.945 INFO:tasks.rados.rados.0.smithi136.stdout:1159: finishing write tid 1 to smithi13667081-11 2024-09-12T23:41:53.945 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 19 v 577 (ObjNum 270 snap 73 seq_num 270) dirty exists 2024-09-12T23:41:53.945 INFO:tasks.rados.rados.0.smithi136.stdout:1159: finishing write tid 2 to smithi13667081-11 2024-09-12T23:41:53.946 INFO:tasks.rados.rados.0.smithi136.stdout:1159: finishing write tid 3 to smithi13667081-11 2024-09-12T23:41:53.946 INFO:tasks.rados.rados.0.smithi136.stdout:1157: done (3 left) 2024-09-12T23:41:53.946 INFO:tasks.rados.rados.0.smithi136.stdout:1158: done (2 left) 2024-09-12T23:41:53.946 INFO:tasks.rados.rados.0.smithi136.stdout:1161: done (1 left) 2024-09-12T23:41:53.946 INFO:tasks.rados.rados.0.smithi136.stdout:1162: snap_remove snap 107 2024-09-12T23:41:54.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:53 smithi136 bash[22601]: cluster 2024-09-12T23:41:52.528084+0000 mon.a (mon.0) 1449 : cluster [DBG] osdmap e277: 8 total, 8 up, 8 in 2024-09-12T23:41:54.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:53 smithi136 bash[22601]: audit 2024-09-12T23:41:52.642858+0000 mon.b (mon.1) 171 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:41:54.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:53 smithi136 bash[22601]: audit 2024-09-12T23:41:53.419800+0000 mon.a (mon.0) 1450 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:54.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:53 smithi136 bash[22601]: audit 2024-09-12T23:41:53.433289+0000 mon.a (mon.0) 1451 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:54.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:53 smithi136 bash[22601]: audit 2024-09-12T23:41:53.438093+0000 mon.b (mon.1) 172 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:41:54.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:53 smithi136 bash[22601]: audit 2024-09-12T23:41:53.440156+0000 mon.b (mon.1) 173 : audit [INF] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:41:54.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:53 smithi136 bash[22601]: audit 2024-09-12T23:41:53.450251+0000 mon.a (mon.0) 1452 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:54.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:53 smithi136 bash[32189]: cluster 2024-09-12T23:41:52.528084+0000 mon.a (mon.0) 1449 : cluster [DBG] osdmap e277: 8 total, 8 up, 8 in 2024-09-12T23:41:54.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:53 smithi136 bash[32189]: audit 2024-09-12T23:41:52.642858+0000 mon.b (mon.1) 171 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:41:54.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:53 smithi136 bash[32189]: audit 2024-09-12T23:41:53.419800+0000 mon.a (mon.0) 1450 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:54.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:53 smithi136 bash[32189]: audit 2024-09-12T23:41:53.433289+0000 mon.a (mon.0) 1451 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:54.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:53 smithi136 bash[32189]: audit 2024-09-12T23:41:53.438093+0000 mon.b (mon.1) 172 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:41:54.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:53 smithi136 bash[32189]: audit 2024-09-12T23:41:53.440156+0000 mon.b (mon.1) 173 : audit [INF] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:41:54.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:53 smithi136 bash[32189]: audit 2024-09-12T23:41:53.450251+0000 mon.a (mon.0) 1452 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:54.657 INFO:tasks.rados.rados.0.smithi136.stdout:1162: done (1 left) 2024-09-12T23:41:54.657 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 30 current snap is 109 2024-09-12T23:41:54.657 INFO:tasks.rados.rados.0.smithi136.stdout:1159: finishing write tid 4 to smithi13667081-11 2024-09-12T23:41:54.657 INFO:tasks.rados.rados.0.smithi136.stdout:1163: seq_num 380 ranges {122880=770048,892928=483328,1376256=319488} 2024-09-12T23:41:54.663 INFO:tasks.rados.rados.0.smithi136.stdout:1163: writing smithi13667081-30 from 122880 to 892928 tid 1 2024-09-12T23:41:54.666 INFO:tasks.rados.rados.0.smithi136.stdout:1163: writing smithi13667081-30 from 892928 to 1376256 tid 2 2024-09-12T23:41:54.668 INFO:tasks.rados.rados.0.smithi136.stdout:1163: writing smithi13667081-30 from 1376256 to 1695744 tid 3 2024-09-12T23:41:54.668 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 29 current snap is 109 2024-09-12T23:41:54.669 INFO:tasks.rados.rados.0.smithi136.stdout:1159: finishing write tid 5 to smithi13667081-11 2024-09-12T23:41:54.669 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 11 v 735 (ObjNum 379 snap 109 seq_num 379) dirty exists 2024-09-12T23:41:54.669 INFO:tasks.rados.rados.0.smithi136.stdout:1159: left oid 11 (ObjNum 379 snap 109 seq_num 379) 2024-09-12T23:41:54.669 INFO:tasks.rados.rados.0.smithi136.stdout:1164: seq_num 381 ranges {2285568=786432,3072000=417792,3489792=385024} 2024-09-12T23:41:54.687 INFO:tasks.rados.rados.0.smithi136.stdout:1164: writing smithi13667081-29 from 2285568 to 3072000 tid 1 2024-09-12T23:41:54.689 INFO:tasks.rados.rados.0.smithi136.stdout:1164: writing smithi13667081-29 from 3072000 to 3489792 tid 2 2024-09-12T23:41:54.692 INFO:tasks.rados.rados.0.smithi136.stdout:1164: writing smithi13667081-29 from 3489792 to 3874816 tid 3 2024-09-12T23:41:54.692 INFO:tasks.rados.rados.0.smithi136.stdout:1159: done (2 left) 2024-09-12T23:41:54.692 INFO:tasks.rados.rados.0.smithi136.stdout:1165: snap_create 2024-09-12T23:41:54.693 INFO:tasks.rados.rados.0.smithi136.stdout:1163: finishing write tid 1 to smithi13667081-30 2024-09-12T23:41:54.693 INFO:tasks.rados.rados.0.smithi136.stdout:1163: finishing write tid 2 to smithi13667081-30 2024-09-12T23:41:54.693 INFO:tasks.rados.rados.0.smithi136.stdout:1163: finishing write tid 3 to smithi13667081-30 2024-09-12T23:41:54.693 INFO:tasks.rados.rados.0.smithi136.stdout:1163: finishing write tid 4 to smithi13667081-30 2024-09-12T23:41:54.693 INFO:tasks.rados.rados.0.smithi136.stdout:1163: finishing write tid 5 to smithi13667081-30 2024-09-12T23:41:54.693 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 30 v 615 (ObjNum 380 snap 109 seq_num 380) dirty exists 2024-09-12T23:41:54.693 INFO:tasks.rados.rados.0.smithi136.stdout:1163: left oid 30 (ObjNum 380 snap 109 seq_num 380) 2024-09-12T23:41:54.699 INFO:tasks.rados.rados.0.smithi136.stdout:1164: finishing write tid 1 to smithi13667081-29 2024-09-12T23:41:54.700 INFO:tasks.rados.rados.0.smithi136.stdout:1164: finishing write tid 2 to smithi13667081-29 2024-09-12T23:41:54.700 INFO:tasks.rados.rados.0.smithi136.stdout:1164: finishing write tid 3 to smithi13667081-29 2024-09-12T23:41:54.703 INFO:tasks.rados.rados.0.smithi136.stdout:1164: finishing write tid 4 to smithi13667081-29 2024-09-12T23:41:54.705 INFO:tasks.rados.rados.0.smithi136.stdout:1164: finishing write tid 5 to smithi13667081-29 2024-09-12T23:41:54.705 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 29 v 479 (ObjNum 381 snap 109 seq_num 381) dirty exists 2024-09-12T23:41:54.706 INFO:tasks.rados.rados.0.smithi136.stdout:1164: left oid 29 (ObjNum 381 snap 109 seq_num 381) 2024-09-12T23:41:54.843 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:54 smithi181 bash[60471]: ts=2024-09-12T23:41:54.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:41:54.844 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:54 smithi181 bash[60471]: ts=2024-09-12T23:41:54.456Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:41:54.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:54 smithi181 bash[24776]: cluster 2024-09-12T23:41:53.450823+0000 mgr.y (mgr.15123) 44 : cluster [DBG] pgmap v71: 121 pgs: 121 active+clean; 174 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:41:54.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:54 smithi181 bash[24776]: cluster 2024-09-12T23:41:53.553590+0000 mon.a (mon.0) 1453 : cluster [DBG] osdmap e278: 8 total, 8 up, 8 in 2024-09-12T23:41:54.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:54 smithi181 bash[24776]: audit 2024-09-12T23:41:53.621889+0000 mon.b (mon.1) 174 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:41:54.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:54 smithi181 bash[24776]: audit 2024-09-12T23:41:53.627923+0000 mon.b (mon.1) 175 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:41:54.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:54 smithi181 bash[24776]: cephadm 2024-09-12T23:41:53.628513+0000 mgr.y (mgr.15123) 45 : cephadm [INF] Upgrade: Setting container_image for all mgr 2024-09-12T23:41:54.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:54 smithi181 bash[24776]: audit 2024-09-12T23:41:53.634894+0000 mon.a (mon.0) 1454 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:54.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:54 smithi181 bash[24776]: audit 2024-09-12T23:41:53.640874+0000 mon.a (mon.0) 1455 : audit [INF] from='mgr.15123 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-12T23:41:54.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:54 smithi181 bash[24776]: audit 2024-09-12T23:41:53.641488+0000 mon.b (mon.1) 176 : audit [INF] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-12T23:41:54.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:54 smithi181 bash[24776]: audit 2024-09-12T23:41:53.646270+0000 mon.a (mon.0) 1456 : audit [INF] from='mgr.15123 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-12T23:41:54.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:54 smithi181 bash[24776]: audit 2024-09-12T23:41:53.651681+0000 mon.a (mon.0) 1457 : audit [INF] from='mgr.15123 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-12T23:41:54.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:54 smithi181 bash[24776]: audit 2024-09-12T23:41:53.652386+0000 mon.b (mon.1) 177 : audit [INF] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-12T23:41:54.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:54 smithi181 bash[24776]: audit 2024-09-12T23:41:53.655407+0000 mon.a (mon.0) 1458 : audit [INF] from='mgr.15123 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-12T23:41:54.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:54 smithi181 bash[24776]: audit 2024-09-12T23:41:53.661916+0000 mon.b (mon.1) 178 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-12T23:41:54.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:54 smithi181 bash[24776]: cephadm 2024-09-12T23:41:53.663270+0000 mgr.y (mgr.15123) 46 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2024-09-12T23:41:54.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:54 smithi181 bash[24776]: audit 2024-09-12T23:41:53.663487+0000 mon.b (mon.1) 179 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-12T23:41:55.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:54 smithi136 bash[22601]: cluster 2024-09-12T23:41:53.450823+0000 mgr.y (mgr.15123) 44 : cluster [DBG] pgmap v71: 121 pgs: 121 active+clean; 174 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:41:55.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:54 smithi136 bash[22601]: cluster 2024-09-12T23:41:53.553590+0000 mon.a (mon.0) 1453 : cluster [DBG] osdmap e278: 8 total, 8 up, 8 in 2024-09-12T23:41:55.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:54 smithi136 bash[22601]: audit 2024-09-12T23:41:53.621889+0000 mon.b (mon.1) 174 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:41:55.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:54 smithi136 bash[22601]: audit 2024-09-12T23:41:53.627923+0000 mon.b (mon.1) 175 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:41:55.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:54 smithi136 bash[22601]: cephadm 2024-09-12T23:41:53.628513+0000 mgr.y (mgr.15123) 45 : cephadm [INF] Upgrade: Setting container_image for all mgr 2024-09-12T23:41:55.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:54 smithi136 bash[22601]: audit 2024-09-12T23:41:53.634894+0000 mon.a (mon.0) 1454 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:55.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:54 smithi136 bash[22601]: audit 2024-09-12T23:41:53.640874+0000 mon.a (mon.0) 1455 : audit [INF] from='mgr.15123 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-12T23:41:55.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:54 smithi136 bash[22601]: audit 2024-09-12T23:41:53.641488+0000 mon.b (mon.1) 176 : audit [INF] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-12T23:41:55.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:54 smithi136 bash[22601]: audit 2024-09-12T23:41:53.646270+0000 mon.a (mon.0) 1456 : audit [INF] from='mgr.15123 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-12T23:41:55.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:54 smithi136 bash[22601]: audit 2024-09-12T23:41:53.651681+0000 mon.a (mon.0) 1457 : audit [INF] from='mgr.15123 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-12T23:41:55.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:54 smithi136 bash[22601]: audit 2024-09-12T23:41:53.652386+0000 mon.b (mon.1) 177 : audit [INF] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-12T23:41:55.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:54 smithi136 bash[22601]: audit 2024-09-12T23:41:53.655407+0000 mon.a (mon.0) 1458 : audit [INF] from='mgr.15123 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-12T23:41:55.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:54 smithi136 bash[22601]: audit 2024-09-12T23:41:53.661916+0000 mon.b (mon.1) 178 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-12T23:41:55.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:54 smithi136 bash[22601]: cephadm 2024-09-12T23:41:53.663270+0000 mgr.y (mgr.15123) 46 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2024-09-12T23:41:55.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:54 smithi136 bash[22601]: audit 2024-09-12T23:41:53.663487+0000 mon.b (mon.1) 179 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-12T23:41:55.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:54 smithi136 bash[32189]: cluster 2024-09-12T23:41:53.450823+0000 mgr.y (mgr.15123) 44 : cluster [DBG] pgmap v71: 121 pgs: 121 active+clean; 174 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:41:55.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:54 smithi136 bash[32189]: cluster 2024-09-12T23:41:53.553590+0000 mon.a (mon.0) 1453 : cluster [DBG] osdmap e278: 8 total, 8 up, 8 in 2024-09-12T23:41:55.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:54 smithi136 bash[32189]: audit 2024-09-12T23:41:53.621889+0000 mon.b (mon.1) 174 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:41:55.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:54 smithi136 bash[32189]: audit 2024-09-12T23:41:53.627923+0000 mon.b (mon.1) 175 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:41:55.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:54 smithi136 bash[32189]: cephadm 2024-09-12T23:41:53.628513+0000 mgr.y (mgr.15123) 45 : cephadm [INF] Upgrade: Setting container_image for all mgr 2024-09-12T23:41:55.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:54 smithi136 bash[32189]: audit 2024-09-12T23:41:53.634894+0000 mon.a (mon.0) 1454 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:55.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:54 smithi136 bash[32189]: audit 2024-09-12T23:41:53.640874+0000 mon.a (mon.0) 1455 : audit [INF] from='mgr.15123 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-12T23:41:55.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:54 smithi136 bash[32189]: audit 2024-09-12T23:41:53.641488+0000 mon.b (mon.1) 176 : audit [INF] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-12T23:41:55.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:54 smithi136 bash[32189]: audit 2024-09-12T23:41:53.646270+0000 mon.a (mon.0) 1456 : audit [INF] from='mgr.15123 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-12T23:41:55.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:54 smithi136 bash[32189]: audit 2024-09-12T23:41:53.651681+0000 mon.a (mon.0) 1457 : audit [INF] from='mgr.15123 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-12T23:41:55.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:54 smithi136 bash[32189]: audit 2024-09-12T23:41:53.652386+0000 mon.b (mon.1) 177 : audit [INF] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-12T23:41:55.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:54 smithi136 bash[32189]: audit 2024-09-12T23:41:53.655407+0000 mon.a (mon.0) 1458 : audit [INF] from='mgr.15123 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-12T23:41:55.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:54 smithi136 bash[32189]: audit 2024-09-12T23:41:53.661916+0000 mon.b (mon.1) 178 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-12T23:41:55.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:54 smithi136 bash[32189]: cephadm 2024-09-12T23:41:53.663270+0000 mgr.y (mgr.15123) 46 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2024-09-12T23:41:55.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:54 smithi136 bash[32189]: audit 2024-09-12T23:41:53.663487+0000 mon.b (mon.1) 179 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-12T23:41:55.666 INFO:tasks.rados.rados.0.smithi136.stdout:1163: done (2 left) 2024-09-12T23:41:55.666 INFO:tasks.rados.rados.0.smithi136.stdout:1164: done (1 left) 2024-09-12T23:41:55.667 INFO:tasks.rados.rados.0.smithi136.stdout:1165: done (0 left) 2024-09-12T23:41:55.667 INFO:tasks.rados.rados.0.smithi136.stdout:1166: read oid 5 snap 102 2024-09-12T23:41:55.667 INFO:tasks.rados.rados.0.smithi136.stdout:1166: expect (ObjNum 358 snap 102 seq_num 358) 2024-09-12T23:41:55.667 INFO:tasks.rados.rados.0.smithi136.stdout:1167: read oid 47 snap 109 2024-09-12T23:41:55.667 INFO:tasks.rados.rados.0.smithi136.stdout:1167: expect (ObjNum 323 snap 88 seq_num 323) 2024-09-12T23:41:55.667 INFO:tasks.rados.rados.0.smithi136.stdout:1168: delete oid 20 current snap is 110 2024-09-12T23:41:55.672 INFO:tasks.rados.rados.0.smithi136.stdout:1168: done (2 left) 2024-09-12T23:41:55.672 INFO:tasks.rados.rados.0.smithi136.stdout:1169: snap_create 2024-09-12T23:41:55.685 INFO:tasks.rados.rados.0.smithi136.stdout:1167: expect (ObjNum 323 snap 88 seq_num 323) 2024-09-12T23:41:55.811 INFO:tasks.rados.rados.0.smithi136.stdout:1166: expect (ObjNum 358 snap 102 seq_num 358) 2024-09-12T23:41:56.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:55 smithi136 bash[32189]: cluster 2024-09-12T23:41:54.660929+0000 mon.a (mon.0) 1459 : cluster [DBG] osdmap e279: 8 total, 8 up, 8 in 2024-09-12T23:41:56.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:55 smithi136 bash[22601]: cluster 2024-09-12T23:41:54.660929+0000 mon.a (mon.0) 1459 : cluster [DBG] osdmap e279: 8 total, 8 up, 8 in 2024-09-12T23:41:56.093 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:55 smithi181 bash[24776]: cluster 2024-09-12T23:41:54.660929+0000 mon.a (mon.0) 1459 : cluster [DBG] osdmap e279: 8 total, 8 up, 8 in 2024-09-12T23:41:56.674 INFO:tasks.rados.rados.0.smithi136.stdout:1166: done (2 left) 2024-09-12T23:41:56.674 INFO:tasks.rados.rados.0.smithi136.stdout:1167: done (1 left) 2024-09-12T23:41:56.674 INFO:tasks.rados.rados.0.smithi136.stdout:1169: done (0 left) 2024-09-12T23:41:56.674 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 47 current snap is 111 2024-09-12T23:41:56.674 INFO:tasks.rados.rados.0.smithi136.stdout:1170: seq_num 383 ranges {2088960=417792,2506752=589824,3096576=417792,3514368=196608} 2024-09-12T23:41:56.710 INFO:tasks.rados.rados.0.smithi136.stdout:1170: writing smithi13667081-47 from 2088960 to 2506752 tid 1 2024-09-12T23:41:56.718 INFO:tasks.rados.rados.0.smithi136.stdout:1170: writing smithi13667081-47 from 2506752 to 3096576 tid 2 2024-09-12T23:41:56.724 INFO:tasks.rados.rados.0.smithi136.stdout:1170: writing smithi13667081-47 from 3096576 to 3514368 tid 3 2024-09-12T23:41:56.727 INFO:tasks.rados.rados.0.smithi136.stdout:1170: writing smithi13667081-47 from 3514368 to 3710976 tid 4 2024-09-12T23:41:56.727 INFO:tasks.rados.rados.0.smithi136.stdout:1171: snap_remove snap 102 2024-09-12T23:41:57.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:56 smithi136 bash[22601]: cluster 2024-09-12T23:41:55.451292+0000 mgr.y (mgr.15123) 47 : cluster [DBG] pgmap v74: 121 pgs: 121 active+clean; 210 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.3 MiB/s rd, 7.0 MiB/s wr, 21 op/s 2024-09-12T23:41:57.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:56 smithi136 bash[22601]: cluster 2024-09-12T23:41:55.670443+0000 mon.a (mon.0) 1460 : cluster [DBG] osdmap e280: 8 total, 8 up, 8 in 2024-09-12T23:41:57.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:56 smithi136 bash[22601]: cephadm 2024-09-12T23:41:56.192751+0000 mgr.y (mgr.15123) 48 : cephadm [INF] Upgrade: Updating mon.a 2024-09-12T23:41:57.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:56 smithi136 bash[22601]: audit 2024-09-12T23:41:56.203508+0000 mon.a (mon.0) 1461 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:57.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:56 smithi136 bash[22601]: audit 2024-09-12T23:41:56.207046+0000 mon.b (mon.1) 180 : audit [INF] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-12T23:41:57.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:56 smithi136 bash[22601]: audit 2024-09-12T23:41:56.208628+0000 mon.b (mon.1) 181 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-12T23:41:57.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:56 smithi136 bash[22601]: audit 2024-09-12T23:41:56.210371+0000 mon.b (mon.1) 182 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:41:57.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:56 smithi136 bash[22601]: cephadm 2024-09-12T23:41:56.211377+0000 mgr.y (mgr.15123) 49 : cephadm [INF] Deploying daemon mon.a on smithi136 2024-09-12T23:41:57.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:56 smithi136 bash[32189]: cluster 2024-09-12T23:41:55.451292+0000 mgr.y (mgr.15123) 47 : cluster [DBG] pgmap v74: 121 pgs: 121 active+clean; 210 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.3 MiB/s rd, 7.0 MiB/s wr, 21 op/s 2024-09-12T23:41:57.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:56 smithi136 bash[32189]: cluster 2024-09-12T23:41:55.670443+0000 mon.a (mon.0) 1460 : cluster [DBG] osdmap e280: 8 total, 8 up, 8 in 2024-09-12T23:41:57.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:56 smithi136 bash[32189]: cephadm 2024-09-12T23:41:56.192751+0000 mgr.y (mgr.15123) 48 : cephadm [INF] Upgrade: Updating mon.a 2024-09-12T23:41:57.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:56 smithi136 bash[32189]: audit 2024-09-12T23:41:56.203508+0000 mon.a (mon.0) 1461 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:57.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:56 smithi136 bash[32189]: audit 2024-09-12T23:41:56.207046+0000 mon.b (mon.1) 180 : audit [INF] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-12T23:41:57.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:56 smithi136 bash[32189]: audit 2024-09-12T23:41:56.208628+0000 mon.b (mon.1) 181 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-12T23:41:57.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:56 smithi136 bash[32189]: audit 2024-09-12T23:41:56.210371+0000 mon.b (mon.1) 182 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:41:57.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:56 smithi136 bash[32189]: cephadm 2024-09-12T23:41:56.211377+0000 mgr.y (mgr.15123) 49 : cephadm [INF] Deploying daemon mon.a on smithi136 2024-09-12T23:41:57.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:56 smithi181 bash[24776]: cluster 2024-09-12T23:41:55.451292+0000 mgr.y (mgr.15123) 47 : cluster [DBG] pgmap v74: 121 pgs: 121 active+clean; 210 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.3 MiB/s rd, 7.0 MiB/s wr, 21 op/s 2024-09-12T23:41:57.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:56 smithi181 bash[24776]: cluster 2024-09-12T23:41:55.670443+0000 mon.a (mon.0) 1460 : cluster [DBG] osdmap e280: 8 total, 8 up, 8 in 2024-09-12T23:41:57.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:56 smithi181 bash[24776]: cephadm 2024-09-12T23:41:56.192751+0000 mgr.y (mgr.15123) 48 : cephadm [INF] Upgrade: Updating mon.a 2024-09-12T23:41:57.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:56 smithi181 bash[24776]: audit 2024-09-12T23:41:56.203508+0000 mon.a (mon.0) 1461 : audit [INF] from='mgr.15123 ' entity='mgr.y' 2024-09-12T23:41:57.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:56 smithi181 bash[24776]: audit 2024-09-12T23:41:56.207046+0000 mon.b (mon.1) 180 : audit [INF] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-12T23:41:57.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:56 smithi181 bash[24776]: audit 2024-09-12T23:41:56.208628+0000 mon.b (mon.1) 181 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-12T23:41:57.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:56 smithi181 bash[24776]: audit 2024-09-12T23:41:56.210371+0000 mon.b (mon.1) 182 : audit [DBG] from='mgr.15123 172.21.15.136:0/2019264702' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:41:57.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:56 smithi181 bash[24776]: cephadm 2024-09-12T23:41:56.211377+0000 mgr.y (mgr.15123) 49 : cephadm [INF] Deploying daemon mon.a on smithi136 2024-09-12T23:41:57.687 INFO:tasks.rados.rados.0.smithi136.stdout:1171: done (1 left) 2024-09-12T23:41:57.687 INFO:tasks.rados.rados.0.smithi136.stdout:1172: delete oid 9 current snap is 111 2024-09-12T23:41:57.687 INFO:tasks.rados.rados.0.smithi136.stdout:1170: finishing write tid 1 to smithi13667081-47 2024-09-12T23:41:57.687 INFO:tasks.rados.rados.0.smithi136.stdout:1170: finishing write tid 2 to smithi13667081-47 2024-09-12T23:41:57.688 INFO:tasks.rados.rados.0.smithi136.stdout:1170: finishing write tid 3 to smithi13667081-47 2024-09-12T23:41:57.688 INFO:tasks.rados.rados.0.smithi136.stdout:1170: finishing write tid 4 to smithi13667081-47 2024-09-12T23:41:57.688 INFO:tasks.rados.rados.0.smithi136.stdout:1170: finishing write tid 5 to smithi13667081-47 2024-09-12T23:41:57.688 INFO:tasks.rados.rados.0.smithi136.stdout:1170: finishing write tid 6 to smithi13667081-47 2024-09-12T23:41:57.688 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 47 v 582 (ObjNum 383 snap 111 seq_num 383) dirty exists 2024-09-12T23:41:57.688 INFO:tasks.rados.rados.0.smithi136.stdout:1170: left oid 47 (ObjNum 383 snap 111 seq_num 383) 2024-09-12T23:41:57.703 INFO:tasks.rados.rados.0.smithi136.stdout:1170: done (1 left) 2024-09-12T23:41:57.703 INFO:tasks.rados.rados.0.smithi136.stdout:1172: done (0 left) 2024-09-12T23:41:57.704 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 46 current snap is 111 2024-09-12T23:41:57.704 INFO:tasks.rados.rados.0.smithi136.stdout:1173: seq_num 385 ranges {0=450560,450560=761856,1212416=442368} 2024-09-12T23:41:57.715 INFO:tasks.rados.rados.0.smithi136.stdout:1173: writing smithi13667081-46 from 0 to 450560 tid 1 2024-09-12T23:41:57.716 INFO:tasks.rados.rados.0.smithi136.stdout:1173: writing smithi13667081-46 from 450560 to 1212416 tid 2 2024-09-12T23:41:57.716 INFO:tasks.rados.rados.0.smithi136.stdout:1173: writing smithi13667081-46 from 1212416 to 1654784 tid 3 2024-09-12T23:41:57.716 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 15 current snap is 111 2024-09-12T23:41:57.716 INFO:tasks.rados.rados.0.smithi136.stdout:1174: seq_num 386 ranges {4464640=466944,4931584=638976,5570560=565248} 2024-09-12T23:41:57.742 INFO:tasks.rados.rados.0.smithi136.stdout:1174: writing smithi13667081-15 from 4464640 to 4931584 tid 1 2024-09-12T23:41:57.746 INFO:tasks.rados.rados.0.smithi136.stdout:1174: writing smithi13667081-15 from 4931584 to 5570560 tid 2 2024-09-12T23:41:57.750 INFO:tasks.rados.rados.0.smithi136.stdout:1174: writing smithi13667081-15 from 5570560 to 6135808 tid 3 2024-09-12T23:41:57.750 INFO:tasks.rados.rados.0.smithi136.stdout:1175: read oid 31 snap -1 2024-09-12T23:41:57.750 INFO:tasks.rados.rados.0.smithi136.stdout:1175: expect (ObjNum 262 snap 71 seq_num 262) 2024-09-12T23:41:57.750 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 23 current snap is 111 2024-09-12T23:41:57.750 INFO:tasks.rados.rados.0.smithi136.stdout:1173: finishing write tid 1 to smithi13667081-46 2024-09-12T23:41:57.750 INFO:tasks.rados.rados.0.smithi136.stdout:1176: seq_num 387 ranges {8642560=483328,9125888=524288,9650176=679936} 2024-09-12T23:41:57.801 INFO:tasks.rados.rados.0.smithi136.stdout:1176: writing smithi13667081-23 from 8642560 to 9125888 tid 1 2024-09-12T23:41:57.804 INFO:tasks.rados.rados.0.smithi136.stdout:1176: writing smithi13667081-23 from 9125888 to 9650176 tid 2 2024-09-12T23:41:57.808 INFO:tasks.rados.rados.0.smithi136.stdout:1176: writing smithi13667081-23 from 9650176 to 10330112 tid 3 2024-09-12T23:41:57.808 INFO:tasks.rados.rados.0.smithi136.stdout:1177: snap_remove snap 103 2024-09-12T23:41:57.944 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:57 smithi136 bash[32189]: cluster 2024-09-12T23:41:56.677711+0000 mon.a (mon.0) 1462 : cluster [DBG] osdmap e281: 8 total, 8 up, 8 in 2024-09-12T23:41:57.945 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:57 smithi136 bash[22601]: cluster 2024-09-12T23:41:56.677711+0000 mon.a (mon.0) 1462 : cluster [DBG] osdmap e281: 8 total, 8 up, 8 in 2024-09-12T23:41:58.093 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:57 smithi181 bash[24776]: cluster 2024-09-12T23:41:56.677711+0000 mon.a (mon.0) 1462 : cluster [DBG] osdmap e281: 8 total, 8 up, 8 in 2024-09-12T23:41:58.696 INFO:tasks.rados.rados.0.smithi136.stdout:1177: done (4 left) 2024-09-12T23:41:58.696 INFO:tasks.rados.rados.0.smithi136.stdout:1178: rmattr oid 47 current snap is 111 2024-09-12T23:41:58.696 INFO:tasks.rados.rados.0.smithi136.stdout:1173: finishing write tid 2 to smithi13667081-46 2024-09-12T23:41:58.696 INFO:tasks.rados.rados.0.smithi136.stdout:1173: finishing write tid 3 to smithi13667081-46 2024-09-12T23:41:58.697 INFO:tasks.rados.rados.0.smithi136.stdout:1173: finishing write tid 4 to smithi13667081-46 2024-09-12T23:41:58.697 INFO:tasks.rados.rados.0.smithi136.stdout:1173: finishing write tid 5 to smithi13667081-46 2024-09-12T23:41:58.697 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 46 v 396 (ObjNum 385 snap 111 seq_num 385) dirty exists 2024-09-12T23:41:58.697 INFO:tasks.rados.rados.0.smithi136.stdout:1173: left oid 46 (ObjNum 385 snap 111 seq_num 385) 2024-09-12T23:41:58.697 INFO:tasks.rados.rados.0.smithi136.stdout:1174: finishing write tid 1 to smithi13667081-15 2024-09-12T23:41:58.697 INFO:tasks.rados.rados.0.smithi136.stdout:1174: finishing write tid 2 to smithi13667081-15 2024-09-12T23:41:58.697 INFO:tasks.rados.rados.0.smithi136.stdout:1174: finishing write tid 3 to smithi13667081-15 2024-09-12T23:41:58.697 INFO:tasks.rados.rados.0.smithi136.stdout:1174: finishing write tid 4 to smithi13667081-15 2024-09-12T23:41:58.697 INFO:tasks.rados.rados.0.smithi136.stdout:1175: expect (ObjNum 262 snap 71 seq_num 262) 2024-09-12T23:41:58.709 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 12 23:41:58 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:41:58.710 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:58 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:41:58.710 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:58 smithi136 bash[32189]: cluster 2024-09-12T23:41:57.451959+0000 mgr.y (mgr.15123) 50 : cluster [DBG] pgmap v77: 121 pgs: 121 active+clean; 210 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.3 MiB/s rd, 7.0 MiB/s wr, 21 op/s 2024-09-12T23:41:58.710 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:58 smithi136 bash[32189]: cluster 2024-09-12T23:41:57.691691+0000 mon.a (mon.0) 1463 : cluster [DBG] osdmap e282: 8 total, 8 up, 8 in 2024-09-12T23:41:58.710 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 12 23:41:58 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:41:58.710 INFO:journalctl@ceph.mds.a.smithi136.stdout:Sep 12 23:41:58 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:41:58.711 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:58 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:41:58.711 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:58 smithi136 bash[22601]: cluster 2024-09-12T23:41:57.451959+0000 mgr.y (mgr.15123) 50 : cluster [DBG] pgmap v77: 121 pgs: 121 active+clean; 210 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.3 MiB/s rd, 7.0 MiB/s wr, 21 op/s 2024-09-12T23:41:58.711 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:58 smithi136 bash[22601]: cluster 2024-09-12T23:41:57.691691+0000 mon.a (mon.0) 1463 : cluster [DBG] osdmap e282: 8 total, 8 up, 8 in 2024-09-12T23:41:58.711 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:41:58 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:41:58.712 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 12 23:41:58 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:41:58.712 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:41:58 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:41:58.712 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 12 23:41:58 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:41:58.712 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:41:58 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:41:58.869 INFO:tasks.rados.rados.0.smithi136.stdout:1174: finishing write tid 5 to smithi13667081-15 2024-09-12T23:41:58.869 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 15 v 581 (ObjNum 386 snap 111 seq_num 386) dirty exists 2024-09-12T23:41:58.869 INFO:tasks.rados.rados.0.smithi136.stdout:1174: left oid 15 (ObjNum 386 snap 111 seq_num 386) 2024-09-12T23:41:58.869 INFO:tasks.rados.rados.0.smithi136.stdout:1176: finishing write tid 1 to smithi13667081-23 2024-09-12T23:41:58.869 INFO:tasks.rados.rados.0.smithi136.stdout:1173: done (4 left) 2024-09-12T23:41:58.870 INFO:tasks.rados.rados.0.smithi136.stdout:1174: done (3 left) 2024-09-12T23:41:58.870 INFO:tasks.rados.rados.0.smithi136.stdout:1175: done (2 left) 2024-09-12T23:41:58.870 INFO:tasks.rados.rados.0.smithi136.stdout:1179: snap_remove snap 108 2024-09-12T23:41:59.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:58 smithi136 systemd[1]: Stopping Ceph mon.a for da58a47c-715d-11ef-bceb-c7b262605968... 2024-09-12T23:41:59.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:58 smithi136 bash[22601]: debug 2024-09-12T23:41:58.826+0000 7ff5e171a640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-09-12T23:41:59.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:58 smithi136 bash[22601]: debug 2024-09-12T23:41:58.826+0000 7ff5e171a640 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-09-12T23:41:59.093 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:58 smithi181 bash[24776]: cluster 2024-09-12T23:41:57.451959+0000 mgr.y (mgr.15123) 50 : cluster [DBG] pgmap v77: 121 pgs: 121 active+clean; 210 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.3 MiB/s rd, 7.0 MiB/s wr, 21 op/s 2024-09-12T23:41:59.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:41:58 smithi181 bash[24776]: cluster 2024-09-12T23:41:57.691691+0000 mon.a (mon.0) 1463 : cluster [DBG] osdmap e282: 8 total, 8 up, 8 in 2024-09-12T23:41:59.094 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:41:58 smithi181 bash[60471]: ts=2024-09-12T23:41:58.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:41:59.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:59 smithi136 bash[75002]: ceph-da58a47c-715d-11ef-bceb-c7b262605968-mon-a 2024-09-12T23:41:59.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:41:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:41:59] "GET /metrics HTTP/1.1" 200 36142 "" "Prometheus/2.43.0" 2024-09-12T23:41:59.860 INFO:teuthology.orchestra.run.smithi136.stdout:true 2024-09-12T23:42:00.089 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:41:59 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:42:00.090 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 12 23:41:59 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:42:00.090 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 12 23:41:59 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:42:00.090 INFO:journalctl@ceph.mds.a.smithi136.stdout:Sep 12 23:41:59 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:42:00.090 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:59 smithi136 bash[75085]: Error response from daemon: No such container: ceph-da58a47c-715d-11ef-bceb-c7b262605968-mon-a 2024-09-12T23:42:00.091 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:59 smithi136 systemd[1]: ceph-da58a47c-715d-11ef-bceb-c7b262605968@mon.a.service: Deactivated successfully. 2024-09-12T23:42:00.091 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:59 smithi136 systemd[1]: Stopped Ceph mon.a for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:42:00.091 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:41:59 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:42:00.091 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:41:59 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:42:00.091 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:41:59 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:42:00.092 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:41:59 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:42:00.092 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 12 23:41:59 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:42:00.092 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 12 23:41:59 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:42:00.139 INFO:teuthology.orchestra.run.smithi136.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-12T23:42:00.139 INFO:teuthology.orchestra.run.smithi136.stdout:alertmanager.a smithi136 *:9093,9094 running (87s) 46s ago 7m 13.5M - 0.25.0 c8568f914cd2 b5197f020eeb 2024-09-12T23:42:00.139 INFO:teuthology.orchestra.run.smithi136.stdout:grafana.a smithi181 *:3000 running (75s) 16s ago 6m 74.5M - 9.4.7 954c08fa6188 256c181be984 2024-09-12T23:42:00.139 INFO:teuthology.orchestra.run.smithi136.stdout:mds.a smithi136 running (8m) 46s ago 8m 13.7M - 17.2.7-1627-gbbdadfa3 214985049bdb 713b9795ad19 2024-09-12T23:42:00.139 INFO:teuthology.orchestra.run.smithi136.stdout:mds.b smithi181 running (8m) 16s ago 8m 17.0M - 17.2.7-1627-gbbdadfa3 214985049bdb ad13179f0868 2024-09-12T23:42:00.139 INFO:teuthology.orchestra.run.smithi136.stdout:mgr.x smithi181 *:8443,9283,8765 running (23s) 16s ago 15m 174M - 18.2.4-837-g7a853b17 f21ba1eea74e cc664dc337f7 2024-09-12T23:42:00.139 INFO:teuthology.orchestra.run.smithi136.stdout:mgr.y smithi136 *:8443,9283,8765 running (68s) 46s ago 18m 457M - 18.2.4-837-g7a853b17 f21ba1eea74e 272f10b42fc3 2024-09-12T23:42:00.139 INFO:teuthology.orchestra.run.smithi136.stdout:mon.a smithi136 starting - - - 2048M 2024-09-12T23:42:00.139 INFO:teuthology.orchestra.run.smithi136.stdout:mon.b smithi181 running (16m) 16s ago 16m 49.4M 2048M 17.2.7-1627-gbbdadfa3 214985049bdb 8b3d784af845 2024-09-12T23:42:00.139 INFO:teuthology.orchestra.run.smithi136.stdout:mon.c smithi136 running (16m) 46s ago 16m 51.5M 2048M 17.2.7-1627-gbbdadfa3 214985049bdb 647b028ea963 2024-09-12T23:42:00.140 INFO:teuthology.orchestra.run.smithi136.stdout:node-exporter.a smithi136 *:9100 running (92s) 46s ago 7m 7707k - 1.5.0 0da6a335fe13 ccfc37c96620 2024-09-12T23:42:00.140 INFO:teuthology.orchestra.run.smithi136.stdout:node-exporter.b smithi181 *:9100 running (80s) 16s ago 7m 8024k - 1.5.0 0da6a335fe13 80bf0e383230 2024-09-12T23:42:00.140 INFO:teuthology.orchestra.run.smithi136.stdout:osd.0 smithi136 running (15m) 46s ago 15m 140M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb cbc0b7df0115 2024-09-12T23:42:00.140 INFO:teuthology.orchestra.run.smithi136.stdout:osd.1 smithi136 running (14m) 46s ago 14m 148M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb c4daee7da836 2024-09-12T23:42:00.140 INFO:teuthology.orchestra.run.smithi136.stdout:osd.2 smithi136 running (13m) 46s ago 13m 153M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 4ab8fd79cdb4 2024-09-12T23:42:00.140 INFO:teuthology.orchestra.run.smithi136.stdout:osd.3 smithi136 running (12m) 46s ago 12m 122M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 9ceb14a27521 2024-09-12T23:42:00.140 INFO:teuthology.orchestra.run.smithi136.stdout:osd.4 smithi181 running (11m) 16s ago 11m 180M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb a35cdca79055 2024-09-12T23:42:00.140 INFO:teuthology.orchestra.run.smithi136.stdout:osd.5 smithi181 running (11m) 16s ago 11m 147M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 8f2e8d494fef 2024-09-12T23:42:00.140 INFO:teuthology.orchestra.run.smithi136.stdout:osd.6 smithi181 running (10m) 16s ago 10m 88.9M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 9c6bec406408 2024-09-12T23:42:00.140 INFO:teuthology.orchestra.run.smithi136.stdout:osd.7 smithi181 running (9m) 16s ago 9m 123M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb bc0632ae2145 2024-09-12T23:42:00.140 INFO:teuthology.orchestra.run.smithi136.stdout:prometheus.a smithi181 *:9095 running (29s) 16s ago 7m 39.0M - 2.43.0 a07b618ecd1d 3853d003a5cc 2024-09-12T23:42:00.470 INFO:teuthology.orchestra.run.smithi136.stdout:{ 2024-09-12T23:42:00.470 INFO:teuthology.orchestra.run.smithi136.stdout: "mon": { 2024-09-12T23:42:00.470 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 3 2024-09-12T23:42:00.470 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:42:00.471 INFO:teuthology.orchestra.run.smithi136.stdout: "mgr": { 2024-09-12T23:42:00.471 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 2 2024-09-12T23:42:00.471 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:42:00.471 INFO:teuthology.orchestra.run.smithi136.stdout: "osd": { 2024-09-12T23:42:00.471 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 8 2024-09-12T23:42:00.471 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:42:00.471 INFO:teuthology.orchestra.run.smithi136.stdout: "mds": { 2024-09-12T23:42:00.471 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-12T23:42:00.471 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:42:00.471 INFO:teuthology.orchestra.run.smithi136.stdout: "overall": { 2024-09-12T23:42:00.471 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 13, 2024-09-12T23:42:00.471 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 2 2024-09-12T23:42:00.471 INFO:teuthology.orchestra.run.smithi136.stdout: } 2024-09-12T23:42:00.472 INFO:teuthology.orchestra.run.smithi136.stdout:} 2024-09-12T23:42:00.505 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:00 smithi136 systemd[1]: Started Ceph mon.a for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:42:01.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.242+0000 7f6a8b721b00 0 set uid:gid to 167:167 (ceph:ceph) 2024-09-12T23:42:01.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.242+0000 7f6a8b721b00 0 ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable), process ceph-mon, pid 7 2024-09-12T23:42:01.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.242+0000 7f6a8b721b00 0 pidfile_write: ignore empty --pid-file 2024-09-12T23:42:01.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.242+0000 7f6a8b721b00 0 load: jerasure load: lrc 2024-09-12T23:42:01.262 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: RocksDB version: 7.9.2 2024-09-12T23:42:01.262 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Git sha 0 2024-09-12T23:42:01.262 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Compile date 2024-09-10 15:58:06 2024-09-12T23:42:01.262 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: DB SUMMARY 2024-09-12T23:42:01.262 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: DB Session ID: 49RYPC0YO3ZWJ8CTEIKV 2024-09-12T23:42:01.262 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: CURRENT file: CURRENT 2024-09-12T23:42:01.262 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: IDENTITY file: IDENTITY 2024-09-12T23:42:01.262 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: MANIFEST file: MANIFEST-000015 size: 1536 Bytes 2024-09-12T23:42:01.262 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000039.sst 2024-09-12T23:42:01.262 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000037.log size: 298233 ; 2024-09-12T23:42:01.262 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.error_if_exists: 0 2024-09-12T23:42:01.263 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.create_if_missing: 0 2024-09-12T23:42:01.263 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.paranoid_checks: 1 2024-09-12T23:42:01.263 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.flush_verify_memtable_count: 1 2024-09-12T23:42:01.263 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-12T23:42:01.263 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-12T23:42:01.263 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.env: 0x55b354f07bc0 2024-09-12T23:42:01.263 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.fs: PosixFileSystem 2024-09-12T23:42:01.263 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.info_log: 0x55b356f80be0 2024-09-12T23:42:01.263 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.max_file_opening_threads: 16 2024-09-12T23:42:01.263 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.statistics: (nil) 2024-09-12T23:42:01.263 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.use_fsync: 0 2024-09-12T23:42:01.264 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.max_log_file_size: 0 2024-09-12T23:42:01.264 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-12T23:42:01.264 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.log_file_time_to_roll: 0 2024-09-12T23:42:01.264 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.keep_log_file_num: 1000 2024-09-12T23:42:01.264 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.recycle_log_file_num: 0 2024-09-12T23:42:01.264 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.allow_fallocate: 1 2024-09-12T23:42:01.264 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.allow_mmap_reads: 0 2024-09-12T23:42:01.264 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.allow_mmap_writes: 0 2024-09-12T23:42:01.264 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.use_direct_reads: 0 2024-09-12T23:42:01.264 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-12T23:42:01.264 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.create_missing_column_families: 0 2024-09-12T23:42:01.264 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.db_log_dir: 2024-09-12T23:42:01.264 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.wal_dir: 2024-09-12T23:42:01.265 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.table_cache_numshardbits: 6 2024-09-12T23:42:01.265 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-09-12T23:42:01.265 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-09-12T23:42:01.265 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-12T23:42:01.265 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-12T23:42:01.265 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-09-12T23:42:01.265 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.advise_random_on_open: 1 2024-09-12T23:42:01.265 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.db_write_buffer_size: 0 2024-09-12T23:42:01.265 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.write_buffer_manager: 0x55b356f8eb40 2024-09-12T23:42:01.265 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-12T23:42:01.265 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-12T23:42:01.265 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.use_adaptive_mutex: 0 2024-09-12T23:42:01.266 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.rate_limiter: (nil) 2024-09-12T23:42:01.266 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-12T23:42:01.266 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.wal_recovery_mode: 2 2024-09-12T23:42:01.268 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.enable_thread_tracking: 0 2024-09-12T23:42:01.268 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.enable_pipelined_write: 0 2024-09-12T23:42:01.268 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.unordered_write: 0 2024-09-12T23:42:01.268 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-12T23:42:01.268 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-12T23:42:01.268 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-12T23:42:01.268 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-12T23:42:01.268 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.row_cache: None 2024-09-12T23:42:01.268 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.wal_filter: None 2024-09-12T23:42:01.269 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-12T23:42:01.269 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.allow_ingest_behind: 0 2024-09-12T23:42:01.269 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.two_write_queues: 0 2024-09-12T23:42:01.269 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.manual_wal_flush: 0 2024-09-12T23:42:01.269 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.wal_compression: 0 2024-09-12T23:42:01.269 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.atomic_flush: 0 2024-09-12T23:42:01.269 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-12T23:42:01.269 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.persist_stats_to_disk: 0 2024-09-12T23:42:01.269 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-09-12T23:42:01.269 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.log_readahead_size: 0 2024-09-12T23:42:01.269 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-12T23:42:01.269 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.best_efforts_recovery: 0 2024-09-12T23:42:01.269 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-12T23:42:01.270 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-12T23:42:01.270 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.allow_data_in_errors: 0 2024-09-12T23:42:01.270 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.db_host_id: __hostname__ 2024-09-12T23:42:01.270 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.enforce_single_del_contracts: true 2024-09-12T23:42:01.270 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.max_background_jobs: 2 2024-09-12T23:42:01.270 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.max_background_compactions: -1 2024-09-12T23:42:01.270 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.max_subcompactions: 1 2024-09-12T23:42:01.270 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-12T23:42:01.270 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-12T23:42:01.270 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.delayed_write_rate : 16777216 2024-09-12T23:42:01.270 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.max_total_wal_size: 0 2024-09-12T23:42:01.270 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-12T23:42:01.271 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.stats_dump_period_sec: 600 2024-09-12T23:42:01.271 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.stats_persist_period_sec: 600 2024-09-12T23:42:01.271 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-12T23:42:01.271 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.max_open_files: -1 2024-09-12T23:42:01.271 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.bytes_per_sync: 0 2024-09-12T23:42:01.271 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-09-12T23:42:01.271 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-09-12T23:42:01.271 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.compaction_readahead_size: 0 2024-09-12T23:42:01.271 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.max_background_flushes: -1 2024-09-12T23:42:01.271 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Compression algorithms supported: 2024-09-12T23:42:01.271 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: kZSTD supported: 0 2024-09-12T23:42:01.271 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: kXpressCompression supported: 0 2024-09-12T23:42:01.271 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: kBZip2Compression supported: 0 2024-09-12T23:42:01.271 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-12T23:42:01.272 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: kLZ4Compression supported: 1 2024-09-12T23:42:01.272 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: kZlibCompression supported: 1 2024-09-12T23:42:01.272 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: kLZ4HCCompression supported: 1 2024-09-12T23:42:01.272 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: kSnappyCompression supported: 1 2024-09-12T23:42:01.272 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-09-12T23:42:01.272 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: DMutex implementation: pthread_mutex_t 2024-09-12T23:42:01.272 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-09-12T23:42:01.272 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-12T23:42:01.272 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-12T23:42:01.272 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.merge_operator: 2024-09-12T23:42:01.272 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.compaction_filter: None 2024-09-12T23:42:01.272 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.compaction_filter_factory: None 2024-09-12T23:42:01.273 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.sst_partitioner_factory: None 2024-09-12T23:42:01.273 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-09-12T23:42:01.273 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.table_factory: BlockBasedTable 2024-09-12T23:42:01.273 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55b356f807e0) 2024-09-12T23:42:01.273 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: cache_index_and_filter_blocks: 1 2024-09-12T23:42:01.273 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-12T23:42:01.273 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-12T23:42:01.273 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: pin_top_level_index_and_filter: 1 2024-09-12T23:42:01.273 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: index_type: 0 2024-09-12T23:42:01.273 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: data_block_index_type: 0 2024-09-12T23:42:01.273 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: index_shortening: 1 2024-09-12T23:42:01.273 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: data_block_hash_table_util_ratio: 0.750000 2024-09-12T23:42:01.273 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: checksum: 4 2024-09-12T23:42:01.274 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: no_block_cache: 0 2024-09-12T23:42:01.274 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: block_cache: 0x55b356f771f0 2024-09-12T23:42:01.274 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: block_cache_name: BinnedLRUCache 2024-09-12T23:42:01.274 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: block_cache_options: 2024-09-12T23:42:01.274 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: capacity : 536870912 2024-09-12T23:42:01.274 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: num_shard_bits : 4 2024-09-12T23:42:01.274 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: strict_capacity_limit : 0 2024-09-12T23:42:01.274 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: high_pri_pool_ratio: 0.000 2024-09-12T23:42:01.274 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: block_cache_compressed: (nil) 2024-09-12T23:42:01.274 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: persistent_cache: (nil) 2024-09-12T23:42:01.274 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: block_size: 4096 2024-09-12T23:42:01.274 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: block_size_deviation: 10 2024-09-12T23:42:01.274 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: block_restart_interval: 16 2024-09-12T23:42:01.275 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: index_block_restart_interval: 1 2024-09-12T23:42:01.275 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: metadata_block_size: 4096 2024-09-12T23:42:01.275 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: partition_filters: 0 2024-09-12T23:42:01.275 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: use_delta_encoding: 1 2024-09-12T23:42:01.275 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: filter_policy: bloomfilter 2024-09-12T23:42:01.275 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: whole_key_filtering: 1 2024-09-12T23:42:01.275 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: verify_compression: 0 2024-09-12T23:42:01.275 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: read_amp_bytes_per_bit: 0 2024-09-12T23:42:01.275 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: format_version: 5 2024-09-12T23:42:01.275 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: enable_index_compression: 1 2024-09-12T23:42:01.275 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: block_align: 0 2024-09-12T23:42:01.275 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: max_auto_readahead_size: 262144 2024-09-12T23:42:01.276 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: prepopulate_block_cache: 0 2024-09-12T23:42:01.276 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: initial_auto_readahead_size: 8192 2024-09-12T23:42:01.276 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: num_file_reads_for_auto_readahead: 2 2024-09-12T23:42:01.276 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.write_buffer_size: 33554432 2024-09-12T23:42:01.276 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.max_write_buffer_number: 2 2024-09-12T23:42:01.276 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.compression: NoCompression 2024-09-12T23:42:01.276 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.bottommost_compression: Disabled 2024-09-12T23:42:01.276 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.prefix_extractor: nullptr 2024-09-12T23:42:01.276 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-12T23:42:01.276 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.num_levels: 7 2024-09-12T23:42:01.276 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-12T23:42:01.276 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-12T23:42:01.276 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-12T23:42:01.277 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-12T23:42:01.277 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-12T23:42:01.277 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-12T23:42:01.277 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-12T23:42:01.277 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-12T23:42:01.277 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-12T23:42:01.277 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-12T23:42:01.277 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-12T23:42:01.277 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-12T23:42:01.277 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.compression_opts.window_bits: -14 2024-09-12T23:42:01.277 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.compression_opts.level: 32767 2024-09-12T23:42:01.277 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.compression_opts.strategy: 0 2024-09-12T23:42:01.277 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-12T23:42:01.277 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-12T23:42:01.278 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-12T23:42:01.278 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-12T23:42:01.278 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.compression_opts.enabled: false 2024-09-12T23:42:01.278 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-12T23:42:01.278 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-12T23:42:01.278 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-12T23:42:01.278 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-12T23:42:01.278 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.target_file_size_base: 67108864 2024-09-12T23:42:01.278 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.target_file_size_multiplier: 1 2024-09-12T23:42:01.278 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-12T23:42:01.278 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-12T23:42:01.278 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-12T23:42:01.278 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-12T23:42:01.279 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-12T23:42:01.279 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-12T23:42:01.279 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-12T23:42:01.279 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-12T23:42:01.279 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-12T23:42:01.279 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-12T23:42:01.279 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-12T23:42:01.279 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-12T23:42:01.279 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-12T23:42:01.279 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.arena_block_size: 1048576 2024-09-12T23:42:01.279 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-12T23:42:01.279 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-12T23:42:01.279 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.disable_auto_compactions: 0 2024-09-12T23:42:01.280 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-12T23:42:01.280 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-12T23:42:01.280 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-12T23:42:01.280 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-12T23:42:01.280 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-12T23:42:01.280 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-12T23:42:01.280 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-12T23:42:01.280 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-12T23:42:01.280 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-12T23:42:01.280 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-12T23:42:01.280 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-12T23:42:01.280 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.inplace_update_support: 0 2024-09-12T23:42:01.281 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-09-12T23:42:01.281 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-12T23:42:01.281 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-12T23:42:01.281 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.memtable_huge_page_size: 0 2024-09-12T23:42:01.281 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.bloom_locality: 0 2024-09-12T23:42:01.281 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.max_successive_merges: 0 2024-09-12T23:42:01.281 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-09-12T23:42:01.281 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.paranoid_file_checks: 0 2024-09-12T23:42:01.281 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.force_consistency_checks: 1 2024-09-12T23:42:01.281 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.report_bg_io_stats: 0 2024-09-12T23:42:01.281 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.ttl: 2592000 2024-09-12T23:42:01.281 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-09-12T23:42:01.282 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-12T23:42:01.282 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-12T23:42:01.282 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.enable_blob_files: false 2024-09-12T23:42:01.282 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.min_blob_size: 0 2024-09-12T23:42:01.282 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.blob_file_size: 268435456 2024-09-12T23:42:01.282 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.blob_compression_type: NoCompression 2024-09-12T23:42:01.282 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.enable_blob_garbage_collection: false 2024-09-12T23:42:01.282 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-12T23:42:01.282 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-12T23:42:01.282 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-12T23:42:01.282 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.blob_file_starting_level: 0 2024-09-12T23:42:01.282 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-12T23:42:01.282 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a884fa640 3 rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 39.sst 2024-09-12T23:42:01.283 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-12T23:42:01.283 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 41, last_sequence is 18898, log_number is 37,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-12T23:42:01.283 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 37 2024-09-12T23:42:01.283 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: a6cd1e2e-cca4-4d2f-8e18-1a5c2d73d75a 2024-09-12T23:42:01.283 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726184521251136, "job": 1, "event": "recovery_started", "wal_files": [37]} 2024-09-12T23:42:01.283 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.246+0000 7f6a8b721b00 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #37 mode 2 2024-09-12T23:42:01.283 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.250+0000 7f6a8b721b00 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726184521253211, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 42, "file_size": 295381, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 18899, "largest_seqno": 19154, "table_properties": {"data_size": 293136, "index_size": 821, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 389, "raw_key_size": 3530, "raw_average_key_size": 26, "raw_value_size": 290056, "raw_average_value_size": 2197, "num_data_blocks": 35, "num_entries": 132, "num_filter_entries": 132, "num_deletions": 4, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726184521, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "a6cd1e2e-cca4-4d2f-8e18-1a5c2d73d75a", "db_session_id": "49RYPC0YO3ZWJ8CTEIKV", "orig_file_number": 42, "seqno_to_time_mapping": "N/A"}} 2024-09-12T23:42:01.283 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.250+0000 7f6a8b721b00 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726184521253338, "job": 1, "event": "recovery_finished"} 2024-09-12T23:42:01.283 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.250+0000 7f6a8b721b00 4 rocksdb: [db/version_set.cc:5047] Creating manifest 44 2024-09-12T23:42:01.283 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.250+0000 7f6a8b721b00 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-12T23:42:01.283 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.250+0000 7f6a8b721b00 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000037.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-12T23:42:01.283 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.250+0000 7f6a8b721b00 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55b356fa0e00 2024-09-12T23:42:01.283 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.250+0000 7f6a8b721b00 4 rocksdb: DB pointer 0x55b357098000 2024-09-12T23:42:01.284 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.250+0000 7f6a814ec640 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-12T23:42:01.284 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.250+0000 7f6a814ec640 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-12T23:42:01.284 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: ** DB Stats ** 2024-09-12T23:42:01.284 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: Uptime(secs): 0.0 total, 0.0 interval 2024-09-12T23:42:01.284 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-12T23:42:01.284 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-12T23:42:01.284 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-12T23:42:01.284 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-12T23:42:01.284 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-12T23:42:01.284 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-12T23:42:01.284 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: ** Compaction Stats [default] ** 2024-09-12T23:42:01.284 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-12T23:42:01.285 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-12T23:42:01.285 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: L0 1/0 288.46 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 198.5 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-12T23:42:01.285 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: L6 1/0 11.08 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-12T23:42:01.285 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: Sum 2/0 11.36 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 198.5 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-12T23:42:01.285 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 198.5 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-12T23:42:01.285 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: ** Compaction Stats [default] ** 2024-09-12T23:42:01.285 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-12T23:42:01.285 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-12T23:42:01.285 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 198.5 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-12T23:42:01.285 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-12T23:42:01.285 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: Uptime(secs): 0.0 total, 0.0 interval 2024-09-12T23:42:01.285 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: Flush(GB): cumulative 0.000, interval 0.000 2024-09-12T23:42:01.285 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-12T23:42:01.286 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: AddFile(Total Files): cumulative 0, interval 0 2024-09-12T23:42:01.286 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: AddFile(L0 Files): cumulative 0, interval 0 2024-09-12T23:42:01.286 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: AddFile(Keys): cumulative 0, interval 0 2024-09-12T23:42:01.286 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: Cumulative compaction: 0.00 GB write, 55.61 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-12T23:42:01.286 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: Interval compaction: 0.00 GB write, 55.61 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-12T23:42:01.286 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-12T23:42:01.286 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: Block cache BinnedLRUCache@0x55b356f771f0#7 capacity: 512.00 MB usage: 1.39 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.5e-05 secs_since: 0 2024-09-12T23:42:01.286 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: Block cache entry stats(count,size,portion): FilterBlock(1,0.44 KB,8.34465e-05%) IndexBlock(1,0.95 KB,0.000181794%) Misc(1,0.00 KB,0%) 2024-09-12T23:42:01.286 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: ** File Read Latency Histogram By Level [default] ** 2024-09-12T23:42:01.286 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.250+0000 7f6a8b721b00 0 starting mon.a rank 0 at public addrs [v2:172.21.15.136:3300/0,v1:172.21.15.136:6789/0] at bind addrs [v2:172.21.15.136:3300/0,v1:172.21.15.136:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid da58a47c-715d-11ef-bceb-c7b262605968 2024-09-12T23:42:01.286 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.250+0000 7f6a8b721b00 1 mon.a@-1(???) e3 preinit fsid da58a47c-715d-11ef-bceb-c7b262605968 2024-09-12T23:42:01.286 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.254+0000 7f6a8b721b00 0 mon.a@-1(???).mds e7 new map 2024-09-12T23:42:01.287 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.254+0000 7f6a8b721b00 0 mon.a@-1(???).mds e7 print_map 2024-09-12T23:42:01.287 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: e7 2024-09-12T23:42:01.287 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: enable_multiple, ever_enabled_multiple: 1,1 2024-09-12T23:42:01.287 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-12T23:42:01.287 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: legacy client fscid: 1 2024-09-12T23:42:01.287 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: 2024-09-12T23:42:01.287 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: Filesystem 'cephfs' (1) 2024-09-12T23:42:01.287 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: fs_name cephfs 2024-09-12T23:42:01.287 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: epoch 7 2024-09-12T23:42:01.287 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-12T23:42:01.287 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: created 2024-09-12T23:33:29.192396+0000 2024-09-12T23:42:01.287 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: modified 2024-09-12T23:33:32.368306+0000 2024-09-12T23:42:01.287 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: tableserver 0 2024-09-12T23:42:01.287 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: root 0 2024-09-12T23:42:01.288 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: session_timeout 60 2024-09-12T23:42:01.288 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: session_autoclose 300 2024-09-12T23:42:01.288 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: max_file_size 1099511627776 2024-09-12T23:42:01.288 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: max_xattr_size 65536 2024-09-12T23:42:01.288 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: required_client_features {} 2024-09-12T23:42:01.288 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: last_failure 0 2024-09-12T23:42:01.288 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: last_failure_osd_epoch 0 2024-09-12T23:42:01.288 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-12T23:42:01.288 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: max_mds 1 2024-09-12T23:42:01.288 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: in 0 2024-09-12T23:42:01.288 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: up {0=24316} 2024-09-12T23:42:01.288 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: failed 2024-09-12T23:42:01.289 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: damaged 2024-09-12T23:42:01.289 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: stopped 2024-09-12T23:42:01.289 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: data_pools [3] 2024-09-12T23:42:01.289 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: metadata_pool 2 2024-09-12T23:42:01.289 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: inline_data disabled 2024-09-12T23:42:01.289 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: balancer 2024-09-12T23:42:01.289 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: bal_rank_mask -1 2024-09-12T23:42:01.289 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: standby_count_wanted 0 2024-09-12T23:42:01.289 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: [mds.b{0:24316} state up:active seq 3 addr [v2:172.21.15.181:6832/2077070284,v1:172.21.15.181:6833/2077070284] compat {c=[1],r=[1],i=[7ff]}] 2024-09-12T23:42:01.289 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: 2024-09-12T23:42:01.289 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: 2024-09-12T23:42:01.289 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: Standby daemons: 2024-09-12T23:42:01.289 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: 2024-09-12T23:42:01.289 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: [mds.a{-1:14391} state up:standby seq 1 addr [v2:172.21.15.136:6834/3940972569,v1:172.21.15.136:6835/3940972569] compat {c=[1],r=[1],i=[7ff]}] 2024-09-12T23:42:01.290 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.254+0000 7f6a8b721b00 0 mon.a@-1(???).osd e283 crush map has features 3314933069573799936, adjusting msgr requires 2024-09-12T23:42:01.290 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.254+0000 7f6a8b721b00 0 mon.a@-1(???).osd e283 crush map has features 432629308056666112, adjusting msgr requires 2024-09-12T23:42:01.290 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.254+0000 7f6a8b721b00 0 mon.a@-1(???).osd e283 crush map has features 432629308056666112, adjusting msgr requires 2024-09-12T23:42:01.290 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.254+0000 7f6a8b721b00 0 mon.a@-1(???).osd e283 crush map has features 432629308056666112, adjusting msgr requires 2024-09-12T23:42:01.290 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:01 smithi136 bash[75164]: debug 2024-09-12T23:42:01.254+0000 7f6a8b721b00 1 mon.a@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2024-09-12T23:42:02.593 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:42:02 smithi181 bash[60471]: ts=2024-09-12T23:42:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:42:02.781 INFO:tasks.rados.rados.0.smithi136.stdout:1176: finishing write tid 2 to smithi13667081-23 2024-09-12T23:42:02.782 INFO:tasks.rados.rados.0.smithi136.stdout:1176: finishing write tid 3 to smithi13667081-23 2024-09-12T23:42:02.782 INFO:tasks.rados.rados.0.smithi136.stdout:1176: finishing write tid 4 to smithi13667081-23 2024-09-12T23:42:02.782 INFO:tasks.rados.rados.0.smithi136.stdout:1176: finishing write tid 5 to smithi13667081-23 2024-09-12T23:42:02.782 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 23 v 758 (ObjNum 387 snap 111 seq_num 387) dirty exists 2024-09-12T23:42:02.782 INFO:tasks.rados.rados.0.smithi136.stdout:1176: left oid 23 (ObjNum 387 snap 111 seq_num 387) 2024-09-12T23:42:02.782 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 47 v 584 (ObjNum 383 snap 111 seq_num 383) dirty exists 2024-09-12T23:42:02.782 INFO:tasks.rados.rados.0.smithi136.stdout:1176: done (2 left) 2024-09-12T23:42:02.782 INFO:tasks.rados.rados.0.smithi136.stdout:1178: done (1 left) 2024-09-12T23:42:02.782 INFO:tasks.rados.rados.0.smithi136.stdout:1179: done (0 left) 2024-09-12T23:42:02.782 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 19 current snap is 111 2024-09-12T23:42:02.782 INFO:tasks.rados.rados.0.smithi136.stdout:1180: seq_num 388 ranges {2129920=507904,2637824=458752,3096576=606208,3702784=139264} 2024-09-12T23:42:02.797 INFO:tasks.rados.rados.0.smithi136.stdout:1180: writing smithi13667081-19 from 2129920 to 2637824 tid 1 2024-09-12T23:42:02.801 INFO:tasks.rados.rados.0.smithi136.stdout:1180: writing smithi13667081-19 from 2637824 to 3096576 tid 2 2024-09-12T23:42:02.805 INFO:tasks.rados.rados.0.smithi136.stdout:1180: writing smithi13667081-19 from 3096576 to 3702784 tid 3 2024-09-12T23:42:02.806 INFO:tasks.rados.rados.0.smithi136.stdout:1180: writing smithi13667081-19 from 3702784 to 3842048 tid 4 2024-09-12T23:42:02.806 INFO:tasks.rados.rados.0.smithi136.stdout:1181: read oid 41 snap -1 2024-09-12T23:42:02.806 INFO:tasks.rados.rados.0.smithi136.stdout:1181: expect (ObjNum 262 snap 71 seq_num 262) 2024-09-12T23:42:02.806 INFO:tasks.rados.rados.0.smithi136.stdout:1182: snap_remove snap 101 2024-09-12T23:42:03.094 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:42:02 smithi181 bash[60867]: ignoring --setuser ceph since I am not root 2024-09-12T23:42:03.094 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:42:02 smithi181 bash[60867]: ignoring --setgroup ceph since I am not root 2024-09-12T23:42:03.094 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:42:02 smithi181 bash[60867]: debug 2024-09-12T23:42:02.817+0000 7ff860ea5640 1 -- 172.21.15.181:0/27297741 <== mon.1 v2:172.21.15.181:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x564cf2270d00 con 0x564cf2262800 2024-09-12T23:42:03.094 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:42:02 smithi181 bash[60867]: debug 2024-09-12T23:42:02.929+0000 7ff863700040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-12T23:42:03.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:02 smithi181 bash[24776]: cluster 2024-09-12T23:41:59.452746+0000 mgr.y (mgr.15123) 51 : cluster [DBG] pgmap v80: 121 pgs: 121 active+clean; 220 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.9 MiB/s rd, 2.9 MiB/s wr, 20 op/s 2024-09-12T23:42:03.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:02 smithi181 bash[24776]: audit 2024-09-12T23:41:59.847135+0000 mgr.y (mgr.15123) 52 : audit [DBG] from='client.25139 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:42:03.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:02 smithi181 bash[24776]: audit 2024-09-12T23:42:00.131857+0000 mgr.y (mgr.15123) 53 : audit [DBG] from='client.25145 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:42:03.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:02 smithi181 bash[24776]: audit 2024-09-12T23:42:00.474502+0000 mon.c (mon.2) 72 : audit [DBG] from='client.? 172.21.15.136:0/3360582411' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:42:03.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:02 smithi181 bash[24776]: cluster 2024-09-12T23:42:01.453472+0000 mgr.y (mgr.15123) 54 : cluster [DBG] pgmap v81: 121 pgs: 121 active+clean; 220 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.9 MiB/s rd, 2.0 MiB/s wr, 14 op/s 2024-09-12T23:42:03.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:02 smithi181 bash[24776]: cluster 2024-09-12T23:42:01.875117+0000 mon.a (mon.0) 1 : cluster [INF] mon.a calling monitor election 2024-09-12T23:42:03.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:02 smithi181 bash[24776]: cluster 2024-09-12T23:42:02.698459+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-12T23:42:03.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:02 smithi181 bash[24776]: cluster 2024-09-12T23:42:02.709885+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.136:3300/0,v1:172.21.15.136:6789/0],b=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0],c=[v2:172.21.15.136:3301/0,v1:172.21.15.136:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-12T23:42:03.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:02 smithi181 bash[24776]: cluster 2024-09-12T23:42:02.711169+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-12T23:42:03.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:02 smithi181 bash[24776]: cluster 2024-09-12T23:42:02.711254+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e283: 8 total, 8 up, 8 in 2024-09-12T23:42:03.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:02 smithi181 bash[24776]: cluster 2024-09-12T23:42:02.713560+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e36: y(active, since 55s), standbys: x 2024-09-12T23:42:03.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:02 smithi181 bash[24776]: cluster 2024-09-12T23:42:02.714578+0000 mon.a (mon.0) 7 : cluster [INF] overall HEALTH_OK 2024-09-12T23:42:03.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:02 smithi181 bash[24776]: cluster 2024-09-12T23:42:02.740321+0000 mon.a (mon.0) 8 : cluster [DBG] mgrmap e37: y(active, since 55s), standbys: x 2024-09-12T23:42:03.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:02 smithi181 bash[24776]: audit 2024-09-12T23:42:02.740494+0000 mon.a (mon.0) 9 : audit [INF] from='mgr.15123 ' entity='' 2024-09-12T23:42:03.256 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:02 smithi136 bash[68787]: ignoring --setuser ceph since I am not root 2024-09-12T23:42:03.256 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:02 smithi136 bash[68787]: ignoring --setgroup ceph since I am not root 2024-09-12T23:42:03.256 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:02 smithi136 bash[68787]: debug 2024-09-12T23:42:02.818+0000 7f5ea91e5640 1 -- 172.21.15.136:0/929128135 <== mon.0 v2:172.21.15.136:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5615c0044d00 con 0x5615c0037800 2024-09-12T23:42:03.256 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:02 smithi136 bash[68787]: debug 2024-09-12T23:42:02.930+0000 7f5eaba40040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-12T23:42:03.256 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:02 smithi136 bash[32189]: cluster 2024-09-12T23:41:59.452746+0000 mgr.y (mgr.15123) 51 : cluster [DBG] pgmap v80: 121 pgs: 121 active+clean; 220 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.9 MiB/s rd, 2.9 MiB/s wr, 20 op/s 2024-09-12T23:42:03.256 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:02 smithi136 bash[32189]: audit 2024-09-12T23:41:59.847135+0000 mgr.y (mgr.15123) 52 : audit [DBG] from='client.25139 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:42:03.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:02 smithi136 bash[32189]: audit 2024-09-12T23:42:00.131857+0000 mgr.y (mgr.15123) 53 : audit [DBG] from='client.25145 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:42:03.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:02 smithi136 bash[32189]: audit 2024-09-12T23:42:00.474502+0000 mon.c (mon.2) 72 : audit [DBG] from='client.? 172.21.15.136:0/3360582411' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:42:03.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:02 smithi136 bash[32189]: cluster 2024-09-12T23:42:01.453472+0000 mgr.y (mgr.15123) 54 : cluster [DBG] pgmap v81: 121 pgs: 121 active+clean; 220 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.9 MiB/s rd, 2.0 MiB/s wr, 14 op/s 2024-09-12T23:42:03.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:02 smithi136 bash[32189]: cluster 2024-09-12T23:42:01.875117+0000 mon.a (mon.0) 1 : cluster [INF] mon.a calling monitor election 2024-09-12T23:42:03.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:02 smithi136 bash[32189]: cluster 2024-09-12T23:42:02.698459+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-12T23:42:03.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:02 smithi136 bash[32189]: cluster 2024-09-12T23:42:02.709885+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.136:3300/0,v1:172.21.15.136:6789/0],b=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0],c=[v2:172.21.15.136:3301/0,v1:172.21.15.136:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-12T23:42:03.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:02 smithi136 bash[32189]: cluster 2024-09-12T23:42:02.711169+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-12T23:42:03.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:02 smithi136 bash[32189]: cluster 2024-09-12T23:42:02.711254+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e283: 8 total, 8 up, 8 in 2024-09-12T23:42:03.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:02 smithi136 bash[32189]: cluster 2024-09-12T23:42:02.713560+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e36: y(active, since 55s), standbys: x 2024-09-12T23:42:03.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:02 smithi136 bash[32189]: cluster 2024-09-12T23:42:02.714578+0000 mon.a (mon.0) 7 : cluster [INF] overall HEALTH_OK 2024-09-12T23:42:03.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:02 smithi136 bash[32189]: cluster 2024-09-12T23:42:02.740321+0000 mon.a (mon.0) 8 : cluster [DBG] mgrmap e37: y(active, since 55s), standbys: x 2024-09-12T23:42:03.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:02 smithi136 bash[32189]: audit 2024-09-12T23:42:02.740494+0000 mon.a (mon.0) 9 : audit [INF] from='mgr.15123 ' entity='' 2024-09-12T23:42:03.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:02 smithi136 bash[75164]: cluster 2024-09-12T23:41:59.452746+0000 mgr.y (mgr.15123) 51 : cluster [DBG] pgmap v80: 121 pgs: 121 active+clean; 220 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.9 MiB/s rd, 2.9 MiB/s wr, 20 op/s 2024-09-12T23:42:03.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:02 smithi136 bash[75164]: audit 2024-09-12T23:41:59.847135+0000 mgr.y (mgr.15123) 52 : audit [DBG] from='client.25139 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:42:03.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:02 smithi136 bash[75164]: audit 2024-09-12T23:42:00.131857+0000 mgr.y (mgr.15123) 53 : audit [DBG] from='client.25145 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:42:03.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:02 smithi136 bash[75164]: audit 2024-09-12T23:42:00.474502+0000 mon.c (mon.2) 72 : audit [DBG] from='client.? 172.21.15.136:0/3360582411' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:42:03.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:02 smithi136 bash[75164]: cluster 2024-09-12T23:42:01.453472+0000 mgr.y (mgr.15123) 54 : cluster [DBG] pgmap v81: 121 pgs: 121 active+clean; 220 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.9 MiB/s rd, 2.0 MiB/s wr, 14 op/s 2024-09-12T23:42:03.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:02 smithi136 bash[75164]: cluster 2024-09-12T23:42:01.875117+0000 mon.a (mon.0) 1 : cluster [INF] mon.a calling monitor election 2024-09-12T23:42:03.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:02 smithi136 bash[75164]: cluster 2024-09-12T23:42:02.698459+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-12T23:42:03.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:02 smithi136 bash[75164]: cluster 2024-09-12T23:42:02.709885+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.136:3300/0,v1:172.21.15.136:6789/0],b=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0],c=[v2:172.21.15.136:3301/0,v1:172.21.15.136:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-12T23:42:03.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:02 smithi136 bash[75164]: cluster 2024-09-12T23:42:02.711169+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-12T23:42:03.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:02 smithi136 bash[75164]: cluster 2024-09-12T23:42:02.711254+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e283: 8 total, 8 up, 8 in 2024-09-12T23:42:03.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:02 smithi136 bash[75164]: cluster 2024-09-12T23:42:02.713560+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e36: y(active, since 55s), standbys: x 2024-09-12T23:42:03.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:02 smithi136 bash[75164]: cluster 2024-09-12T23:42:02.714578+0000 mon.a (mon.0) 7 : cluster [INF] overall HEALTH_OK 2024-09-12T23:42:03.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:02 smithi136 bash[75164]: cluster 2024-09-12T23:42:02.740321+0000 mon.a (mon.0) 8 : cluster [DBG] mgrmap e37: y(active, since 55s), standbys: x 2024-09-12T23:42:03.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:02 smithi136 bash[75164]: audit 2024-09-12T23:42:02.740494+0000 mon.a (mon.0) 9 : audit [INF] from='mgr.15123 ' entity='' 2024-09-12T23:42:03.777 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:42:03 smithi181 bash[60471]: ts=2024-09-12T23:42:03.507Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi136\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:42:03.798 INFO:tasks.rados.rados.0.smithi136.stdout:1182: done (2 left) 2024-09-12T23:42:03.798 INFO:tasks.rados.rados.0.smithi136.stdout:1180: finishing write tid 1 to smithi13667081-19 2024-09-12T23:42:03.798 INFO:tasks.rados.rados.0.smithi136.stdout:1180: finishing write tid 2 to smithi13667081-19 2024-09-12T23:42:03.798 INFO:tasks.rados.rados.0.smithi136.stdout:1180: finishing write tid 3 to smithi13667081-19 2024-09-12T23:42:03.798 INFO:tasks.rados.rados.0.smithi136.stdout:1180: finishing write tid 4 to smithi13667081-19 2024-09-12T23:42:03.798 INFO:tasks.rados.rados.0.smithi136.stdout:1180: finishing write tid 5 to smithi13667081-19 2024-09-12T23:42:03.798 INFO:tasks.rados.rados.0.smithi136.stdout:1180: finishing write tid 6 to smithi13667081-19 2024-09-12T23:42:03.798 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 19 v 589 (ObjNum 388 snap 111 seq_num 388) dirty exists 2024-09-12T23:42:03.799 INFO:tasks.rados.rados.0.smithi136.stdout:1180: left oid 19 (ObjNum 388 snap 111 seq_num 388) 2024-09-12T23:42:03.799 INFO:tasks.rados.rados.0.smithi136.stdout:1181: expect (ObjNum 262 snap 71 seq_num 262) 2024-09-12T23:42:03.958 INFO:tasks.rados.rados.0.smithi136.stdout:1183: read oid 7 snap -1 2024-09-12T23:42:03.958 INFO:tasks.rados.rados.0.smithi136.stdout:1183: expect deleted 2024-09-12T23:42:03.958 INFO:tasks.rados.rados.0.smithi136.stdout:1180: done (2 left) 2024-09-12T23:42:03.959 INFO:tasks.rados.rados.0.smithi136.stdout:1181: done (1 left) 2024-09-12T23:42:03.959 INFO:tasks.rados.rados.0.smithi136.stdout:1184: snap_create 2024-09-12T23:42:04.093 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:03 smithi181 bash[24776]: cluster 2024-09-12T23:42:02.785315+0000 mon.a (mon.0) 10 : cluster [DBG] osdmap e284: 8 total, 8 up, 8 in 2024-09-12T23:42:04.133 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:03 smithi136 bash[32189]: cluster 2024-09-12T23:42:02.785315+0000 mon.a (mon.0) 10 : cluster [DBG] osdmap e284: 8 total, 8 up, 8 in 2024-09-12T23:42:04.133 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:03 smithi136 bash[75164]: cluster 2024-09-12T23:42:02.785315+0000 mon.a (mon.0) 10 : cluster [DBG] osdmap e284: 8 total, 8 up, 8 in 2024-09-12T23:42:04.450 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:42:04 smithi181 bash[60867]: debug 2024-09-12T23:42:04.157+0000 7ff863700040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-12T23:42:04.450 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:42:04 smithi181 bash[60867]: debug 2024-09-12T23:42:04.221+0000 7ff863700040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-12T23:42:04.450 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:42:04 smithi181 bash[60867]: debug 2024-09-12T23:42:04.369+0000 7ff863700040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-12T23:42:04.506 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:04 smithi136 bash[68787]: debug 2024-09-12T23:42:04.134+0000 7f5eaba40040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-12T23:42:04.506 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:04 smithi136 bash[68787]: debug 2024-09-12T23:42:04.198+0000 7f5eaba40040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-12T23:42:04.506 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:04 smithi136 bash[68787]: debug 2024-09-12T23:42:04.346+0000 7f5eaba40040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-12T23:42:04.786 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:42:04 smithi181 bash[60471]: ts=2024-09-12T23:42:04.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:42:04.786 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:42:04 smithi181 bash[60471]: ts=2024-09-12T23:42:04.456Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:42:04.800 INFO:tasks.rados.rados.0.smithi136.stdout:1183: done (1 left) 2024-09-12T23:42:04.800 INFO:tasks.rados.rados.0.smithi136.stdout:1184: done (0 left) 2024-09-12T23:42:04.800 INFO:tasks.rados.rados.0.smithi136.stdout:1185: snap_remove snap 99 2024-09-12T23:42:05.045 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:04 smithi136 bash[32189]: cluster 2024-09-12T23:42:03.793655+0000 mon.a (mon.0) 11 : cluster [DBG] osdmap e285: 8 total, 8 up, 8 in 2024-09-12T23:42:05.046 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:04 smithi136 bash[68787]: debug 2024-09-12T23:42:04.978+0000 7f5eaba40040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-12T23:42:05.046 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:04 smithi136 bash[75164]: cluster 2024-09-12T23:42:03.793655+0000 mon.a (mon.0) 11 : cluster [DBG] osdmap e285: 8 total, 8 up, 8 in 2024-09-12T23:42:05.070 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:04 smithi181 bash[24776]: cluster 2024-09-12T23:42:03.793655+0000 mon.a (mon.0) 11 : cluster [DBG] osdmap e285: 8 total, 8 up, 8 in 2024-09-12T23:42:05.070 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:42:05 smithi181 bash[60867]: debug 2024-09-12T23:42:05.001+0000 7ff863700040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-12T23:42:05.297 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:05 smithi136 bash[68787]: debug 2024-09-12T23:42:05.046+0000 7f5eaba40040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-12T23:42:05.297 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:05 smithi136 bash[68787]: debug 2024-09-12T23:42:05.114+0000 7f5eaba40040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-12T23:42:05.321 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:42:05 smithi181 bash[60867]: debug 2024-09-12T23:42:05.073+0000 7ff863700040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-12T23:42:05.322 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:42:05 smithi181 bash[60867]: debug 2024-09-12T23:42:05.141+0000 7ff863700040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-12T23:42:05.593 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:42:05 smithi181 bash[60867]: debug 2024-09-12T23:42:05.321+0000 7ff863700040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-12T23:42:05.594 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:42:05 smithi181 bash[60867]: debug 2024-09-12T23:42:05.401+0000 7ff863700040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-12T23:42:05.594 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:42:05 smithi181 bash[60867]: debug 2024-09-12T23:42:05.465+0000 7ff863700040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-12T23:42:05.691 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:05 smithi136 bash[68787]: debug 2024-09-12T23:42:05.294+0000 7f5eaba40040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-12T23:42:05.692 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:05 smithi136 bash[68787]: debug 2024-09-12T23:42:05.374+0000 7f5eaba40040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-12T23:42:05.692 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:05 smithi136 bash[68787]: debug 2024-09-12T23:42:05.438+0000 7f5eaba40040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-12T23:42:05.692 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:05 smithi136 bash[68787]: debug 2024-09-12T23:42:05.690+0000 7f5eaba40040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-12T23:42:05.804 INFO:tasks.rados.rados.0.smithi136.stdout:1185: done (0 left) 2024-09-12T23:42:05.805 INFO:tasks.rados.rados.0.smithi136.stdout:1186: copy_from oid 25 from oid 20 current snap is 112 2024-09-12T23:42:05.805 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 47 current snap is 112 2024-09-12T23:42:05.805 INFO:tasks.rados.rados.0.smithi136.stdout:1187: seq_num 389 ranges {3710976=524288,4235264=745472,4980736=458752} 2024-09-12T23:42:05.837 INFO:tasks.rados.rados.0.smithi136.stdout:1187: writing smithi13667081-47 from 3710976 to 4235264 tid 1 2024-09-12T23:42:05.842 INFO:tasks.rados.rados.0.smithi136.stdout:1187: writing smithi13667081-47 from 4235264 to 4980736 tid 2 2024-09-12T23:42:05.847 INFO:tasks.rados.rados.0.smithi136.stdout:1187: writing smithi13667081-47 from 4980736 to 5439488 tid 3 2024-09-12T23:42:05.847 INFO:tasks.rados.rados.0.smithi136.stdout:1188: read oid 50 snap -1 2024-09-12T23:42:05.847 INFO:tasks.rados.rados.0.smithi136.stdout:1188: expect (ObjNum 292 snap 79 seq_num 292) 2024-09-12T23:42:05.847 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 21 current snap is 112 2024-09-12T23:42:05.847 INFO:tasks.rados.rados.0.smithi136.stdout:1189: seq_num 390 ranges {5758976=540672,6299648=630784,6930432=573440} 2024-09-12T23:42:05.888 INFO:tasks.rados.rados.0.smithi136.stdout:1189: writing smithi13667081-21 from 5758976 to 6299648 tid 1 2024-09-12T23:42:05.891 INFO:tasks.rados.rados.0.smithi136.stdout:1189: writing smithi13667081-21 from 6299648 to 6930432 tid 2 2024-09-12T23:42:05.895 INFO:tasks.rados.rados.0.smithi136.stdout:1189: writing smithi13667081-21 from 6930432 to 7503872 tid 3 2024-09-12T23:42:05.895 INFO:tasks.rados.rados.0.smithi136.stdout:1190: snap_create 2024-09-12T23:42:05.895 INFO:tasks.rados.rados.0.smithi136.stdout:1186: finishing copy_from to smithi13667081-25 2024-09-12T23:42:05.895 INFO:tasks.rados.rados.0.smithi136.stdout:1186: got expected ENOENT (src dne) 2024-09-12T23:42:05.895 INFO:tasks.rados.rados.0.smithi136.stdout:1186: finishing copy_from racing read to smithi13667081-25 2024-09-12T23:42:05.895 INFO:tasks.rados.rados.0.smithi136.stdout:1187: finishing write tid 1 to smithi13667081-47 2024-09-12T23:42:05.895 INFO:tasks.rados.rados.0.smithi136.stdout:1187: finishing write tid 2 to smithi13667081-47 2024-09-12T23:42:05.896 INFO:tasks.rados.rados.0.smithi136.stdout:1187: finishing write tid 3 to smithi13667081-47 2024-09-12T23:42:05.896 INFO:tasks.rados.rados.0.smithi136.stdout:1187: finishing write tid 4 to smithi13667081-47 2024-09-12T23:42:05.896 INFO:tasks.rados.rados.0.smithi136.stdout:1188: expect (ObjNum 292 snap 79 seq_num 292) 2024-09-12T23:42:05.901 INFO:tasks.rados.rados.0.smithi136.stdout:1187: finishing write tid 5 to smithi13667081-47 2024-09-12T23:42:05.901 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 47 v 593 (ObjNum 389 snap 112 seq_num 389) dirty exists 2024-09-12T23:42:05.902 INFO:tasks.rados.rados.0.smithi136.stdout:1187: left oid 47 (ObjNum 389 snap 112 seq_num 389) 2024-09-12T23:42:05.902 INFO:tasks.rados.rados.0.smithi136.stdout:1189: finishing write tid 1 to smithi13667081-21 2024-09-12T23:42:05.902 INFO:tasks.rados.rados.0.smithi136.stdout:1189: finishing write tid 2 to smithi13667081-21 2024-09-12T23:42:05.902 INFO:tasks.rados.rados.0.smithi136.stdout:1189: finishing write tid 3 to smithi13667081-21 2024-09-12T23:42:05.902 INFO:tasks.rados.rados.0.smithi136.stdout:1189: finishing write tid 4 to smithi13667081-21 2024-09-12T23:42:05.903 INFO:tasks.rados.rados.0.smithi136.stdout:1189: finishing write tid 5 to smithi13667081-21 2024-09-12T23:42:05.903 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 21 v 723 (ObjNum 390 snap 112 seq_num 390) dirty exists 2024-09-12T23:42:05.903 INFO:tasks.rados.rados.0.smithi136.stdout:1189: left oid 21 (ObjNum 390 snap 112 seq_num 390) 2024-09-12T23:42:05.975 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:05 smithi136 bash[32189]: cluster 2024-09-12T23:42:04.802186+0000 mon.a (mon.0) 12 : cluster [DBG] osdmap e286: 8 total, 8 up, 8 in 2024-09-12T23:42:05.975 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:05 smithi136 bash[75164]: cluster 2024-09-12T23:42:04.802186+0000 mon.a (mon.0) 12 : cluster [DBG] osdmap e286: 8 total, 8 up, 8 in 2024-09-12T23:42:05.976 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:05 smithi136 bash[68787]: debug 2024-09-12T23:42:05.758+0000 7f5eaba40040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-12T23:42:05.996 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:05 smithi181 bash[24776]: cluster 2024-09-12T23:42:04.802186+0000 mon.a (mon.0) 12 : cluster [DBG] osdmap e286: 8 total, 8 up, 8 in 2024-09-12T23:42:05.996 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:42:05 smithi181 bash[60867]: debug 2024-09-12T23:42:05.721+0000 7ff863700040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-12T23:42:05.996 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:42:05 smithi181 bash[60867]: debug 2024-09-12T23:42:05.785+0000 7ff863700040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-12T23:42:06.255 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:05 smithi136 bash[68787]: debug 2024-09-12T23:42:05.974+0000 7f5eaba40040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-12T23:42:06.256 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:06 smithi136 bash[68787]: debug 2024-09-12T23:42:06.058+0000 7f5eaba40040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-12T23:42:06.273 INFO:tasks.rados.rados.0.smithi136.stdout:1186: done (4 left) 2024-09-12T23:42:06.273 INFO:tasks.rados.rados.0.smithi136.stdout:1187: done (3 left) 2024-09-12T23:42:06.273 INFO:tasks.rados.rados.0.smithi136.stdout:1188: done (2 left) 2024-09-12T23:42:06.273 INFO:tasks.rados.rados.0.smithi136.stdout:1189: done (1 left) 2024-09-12T23:42:06.273 INFO:tasks.rados.rados.0.smithi136.stdout:1190: done (0 left) 2024-09-12T23:42:06.274 INFO:tasks.rados.rados.0.smithi136.stdout:1191: read oid 7 snap 110 2024-09-12T23:42:06.274 INFO:tasks.rados.rados.0.smithi136.stdout:1191: expect deleted 2024-09-12T23:42:06.274 INFO:tasks.rados.rados.0.smithi136.stdout:1192: copy_from oid 13 from oid 41 current snap is 113 2024-09-12T23:42:06.274 INFO:tasks.rados.rados.0.smithi136.stdout:1193: snap_remove snap 112 2024-09-12T23:42:06.342 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:42:06 smithi181 bash[60867]: debug 2024-09-12T23:42:05.997+0000 7ff863700040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-12T23:42:06.342 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:42:06 smithi181 bash[60867]: debug 2024-09-12T23:42:06.077+0000 7ff863700040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-12T23:42:06.593 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:42:06 smithi181 bash[60867]: debug 2024-09-12T23:42:06.345+0000 7ff863700040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-12T23:42:06.593 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:42:06 smithi181 bash[60867]: debug 2024-09-12T23:42:06.517+0000 7ff863700040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-12T23:42:06.665 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:06 smithi136 bash[68787]: debug 2024-09-12T23:42:06.330+0000 7f5eaba40040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-12T23:42:06.665 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:06 smithi136 bash[68787]: debug 2024-09-12T23:42:06.506+0000 7f5eaba40040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-12T23:42:06.919 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:06 smithi136 bash[68787]: debug 2024-09-12T23:42:06.666+0000 7f5eaba40040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-12T23:42:06.919 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:06 smithi136 bash[68787]: debug 2024-09-12T23:42:06.742+0000 7f5eaba40040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-12T23:42:06.920 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:06 smithi136 bash[75164]: cluster 2024-09-12T23:42:05.805538+0000 mon.a (mon.0) 13 : cluster [DBG] osdmap e287: 8 total, 8 up, 8 in 2024-09-12T23:42:06.920 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:06 smithi136 bash[75164]: cluster 2024-09-12T23:42:06.277377+0000 mon.a (mon.0) 14 : cluster [DBG] osdmap e288: 8 total, 8 up, 8 in 2024-09-12T23:42:06.920 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:06 smithi136 bash[32189]: cluster 2024-09-12T23:42:05.805538+0000 mon.a (mon.0) 13 : cluster [DBG] osdmap e287: 8 total, 8 up, 8 in 2024-09-12T23:42:06.920 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:06 smithi136 bash[32189]: cluster 2024-09-12T23:42:06.277377+0000 mon.a (mon.0) 14 : cluster [DBG] osdmap e288: 8 total, 8 up, 8 in 2024-09-12T23:42:06.933 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:06 smithi181 bash[24776]: cluster 2024-09-12T23:42:05.805538+0000 mon.a (mon.0) 13 : cluster [DBG] osdmap e287: 8 total, 8 up, 8 in 2024-09-12T23:42:06.934 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:06 smithi181 bash[24776]: cluster 2024-09-12T23:42:06.277377+0000 mon.a (mon.0) 14 : cluster [DBG] osdmap e288: 8 total, 8 up, 8 in 2024-09-12T23:42:06.934 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:42:06 smithi181 bash[60867]: debug 2024-09-12T23:42:06.677+0000 7ff863700040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-12T23:42:06.934 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:42:06 smithi181 bash[60867]: debug 2024-09-12T23:42:06.757+0000 7ff863700040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-12T23:42:07.255 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:06 smithi136 bash[68787]: debug 2024-09-12T23:42:06.918+0000 7f5eaba40040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-12T23:42:07.280 INFO:tasks.rados.rados.0.smithi136.stdout:1192: finishing copy_from racing read to smithi13667081-13 2024-09-12T23:42:07.280 INFO:tasks.rados.rados.0.smithi136.stdout:1192: finishing copy_from to smithi13667081-13 2024-09-12T23:42:07.280 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 13 v 339 (ObjNum 262 snap 71 seq_num 262) dirty exists 2024-09-12T23:42:07.280 INFO:tasks.rados.rados.0.smithi136.stdout:1191: done (2 left) 2024-09-12T23:42:07.281 INFO:tasks.rados.rados.0.smithi136.stdout:1192: done (1 left) 2024-09-12T23:42:07.281 INFO:tasks.rados.rados.0.smithi136.stdout:1193: done (0 left) 2024-09-12T23:42:07.281 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 19 current snap is 113 2024-09-12T23:42:07.281 INFO:tasks.rados.rados.0.smithi136.stdout:1194: seq_num 391 ranges {3842048=557056,4399104=516096,4915200=688128} 2024-09-12T23:42:07.309 INFO:tasks.rados.rados.0.smithi136.stdout:1194: writing smithi13667081-19 from 3842048 to 4399104 tid 1 2024-09-12T23:42:07.315 INFO:tasks.rados.rados.0.smithi136.stdout:1194: writing smithi13667081-19 from 4399104 to 4915200 tid 2 2024-09-12T23:42:07.325 INFO:tasks.rados.rados.0.smithi136.stdout:1194: writing smithi13667081-19 from 4915200 to 5603328 tid 3 2024-09-12T23:42:07.325 INFO:tasks.rados.rados.0.smithi136.stdout:1195: snap_create 2024-09-12T23:42:07.326 INFO:tasks.rados.rados.0.smithi136.stdout:1194: finishing write tid 1 to smithi13667081-19 2024-09-12T23:42:07.326 INFO:tasks.rados.rados.0.smithi136.stdout:1194: finishing write tid 2 to smithi13667081-19 2024-09-12T23:42:07.332 INFO:tasks.rados.rados.0.smithi136.stdout:1194: finishing write tid 3 to smithi13667081-19 2024-09-12T23:42:07.332 INFO:tasks.rados.rados.0.smithi136.stdout:1194: finishing write tid 4 to smithi13667081-19 2024-09-12T23:42:07.335 INFO:tasks.rados.rados.0.smithi136.stdout:1194: finishing write tid 5 to smithi13667081-19 2024-09-12T23:42:07.335 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 19 v 597 (ObjNum 391 snap 113 seq_num 391) dirty exists 2024-09-12T23:42:07.335 INFO:tasks.rados.rados.0.smithi136.stdout:1194: left oid 19 (ObjNum 391 snap 113 seq_num 391) 2024-09-12T23:42:07.344 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:42:06 smithi181 bash[60867]: debug 2024-09-12T23:42:06.933+0000 7ff863700040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-12T23:42:08.048 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:07 smithi136 bash[68787]: debug 2024-09-12T23:42:07.757+0000 7f5eaba40040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-12T23:42:08.048 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:07 smithi136 bash[68787]: debug 2024-09-12T23:42:07.829+0000 7f5eaba40040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-12T23:42:08.053 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:42:07 smithi181 bash[60867]: debug 2024-09-12T23:42:07.757+0000 7ff863700040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-12T23:42:08.053 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:42:07 smithi181 bash[60867]: debug 2024-09-12T23:42:07.833+0000 7ff863700040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-12T23:42:08.278 INFO:tasks.rados.rados.0.smithi136.stdout:1194: done (1 left) 2024-09-12T23:42:08.278 INFO:tasks.rados.rados.0.smithi136.stdout:1195: done (0 left) 2024-09-12T23:42:08.278 INFO:tasks.rados.rados.0.smithi136.stdout:1196: snap_create 2024-09-12T23:42:08.343 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:08 smithi181 bash[24776]: cluster 2024-09-12T23:42:07.281683+0000 mon.a (mon.0) 15 : cluster [DBG] osdmap e289: 8 total, 8 up, 8 in 2024-09-12T23:42:08.344 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:42:08 smithi181 bash[60867]: debug 2024-09-12T23:42:08.053+0000 7ff863700040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-12T23:42:08.344 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:42:08 smithi181 bash[60867]: debug 2024-09-12T23:42:08.129+0000 7ff863700040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-12T23:42:08.505 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:08 smithi136 bash[75164]: cluster 2024-09-12T23:42:07.281683+0000 mon.a (mon.0) 15 : cluster [DBG] osdmap e289: 8 total, 8 up, 8 in 2024-09-12T23:42:08.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:08 smithi136 bash[32189]: cluster 2024-09-12T23:42:07.281683+0000 mon.a (mon.0) 15 : cluster [DBG] osdmap e289: 8 total, 8 up, 8 in 2024-09-12T23:42:08.506 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:08 smithi136 bash[68787]: debug 2024-09-12T23:42:08.049+0000 7f5eaba40040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-12T23:42:08.506 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:08 smithi136 bash[68787]: debug 2024-09-12T23:42:08.121+0000 7f5eaba40040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-12T23:42:08.620 INFO:tasks.rados.rados.0.smithi136.stdout:1196: done (0 left) 2024-09-12T23:42:08.620 INFO:tasks.rados.rados.0.smithi136.stdout:1197: snap_remove snap 96 2024-09-12T23:42:08.796 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:08 smithi136 bash[68787]: debug 2024-09-12T23:42:08.505+0000 7f5eaba40040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-12T23:42:08.796 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:08 smithi136 bash[68787]: debug 2024-09-12T23:42:08.581+0000 7f5eaba40040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-12T23:42:08.796 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:08 smithi136 bash[68787]: [12/Sep/2024:23:42:08] ENGINE Bus STARTING 2024-09-12T23:42:08.796 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:08 smithi136 bash[68787]: CherryPy Checker: 2024-09-12T23:42:08.796 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:08 smithi136 bash[68787]: The Application mounted at '' has an empty config. 2024-09-12T23:42:08.843 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:42:08 smithi181 bash[60471]: ts=2024-09-12T23:42:08.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:42:08.844 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:42:08 smithi181 bash[60867]: debug 2024-09-12T23:42:08.513+0000 7ff863700040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-12T23:42:08.844 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:42:08 smithi181 bash[60867]: debug 2024-09-12T23:42:08.589+0000 7ff863700040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-12T23:42:08.844 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:42:08 smithi181 bash[60867]: [12/Sep/2024:23:42:08] ENGINE Bus STARTING 2024-09-12T23:42:08.844 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:42:08 smithi181 bash[60867]: CherryPy Checker: 2024-09-12T23:42:08.844 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:42:08 smithi181 bash[60867]: The Application mounted at '' has an empty config. 2024-09-12T23:42:08.844 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:42:08 smithi181 bash[60867]: [12/Sep/2024:23:42:08] ENGINE Serving on http://:::9283 2024-09-12T23:42:08.844 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:42:08 smithi181 bash[60867]: [12/Sep/2024:23:42:08] ENGINE Bus STARTED 2024-09-12T23:42:09.255 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:08 smithi136 bash[68787]: debug 2024-09-12T23:42:08.797+0000 7f5e15482640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-12T23:42:09.255 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:08 smithi136 bash[68787]: debug 2024-09-12T23:42:08.797+0000 7f5e15482640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-12T23:42:09.256 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:08 smithi136 bash[68787]: debug 2024-09-12T23:42:08.797+0000 7f5e15482640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-12T23:42:09.256 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:08 smithi136 bash[68787]: debug 2024-09-12T23:42:08.797+0000 7f5e15482640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-12T23:42:09.256 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:08 smithi136 bash[68787]: debug 2024-09-12T23:42:08.797+0000 7f5e15482640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-12T23:42:09.256 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:08 smithi136 bash[68787]: debug 2024-09-12T23:42:08.797+0000 7f5e12c7d640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-12T23:42:09.256 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:08 smithi136 bash[68787]: debug 2024-09-12T23:42:08.797+0000 7f5e12c7d640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-12T23:42:09.256 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:08 smithi136 bash[68787]: debug 2024-09-12T23:42:08.797+0000 7f5e12c7d640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-12T23:42:09.256 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:08 smithi136 bash[68787]: debug 2024-09-12T23:42:08.797+0000 7f5e12c7d640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-12T23:42:09.256 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:08 smithi136 bash[68787]: debug 2024-09-12T23:42:08.797+0000 7f5e12c7d640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-12T23:42:09.256 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:08 smithi136 bash[68787]: [12/Sep/2024:23:42:08] ENGINE Serving on http://:::9283 2024-09-12T23:42:09.256 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:08 smithi136 bash[68787]: [12/Sep/2024:23:42:08] ENGINE Bus STARTED 2024-09-12T23:42:09.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:09 smithi181 bash[24776]: cluster 2024-09-12T23:42:08.281604+0000 mon.a (mon.0) 16 : cluster [DBG] osdmap e290: 8 total, 8 up, 8 in 2024-09-12T23:42:09.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:09 smithi181 bash[24776]: cluster 2024-09-12T23:42:08.612472+0000 mon.a (mon.0) 17 : cluster [INF] Active manager daemon y restarted 2024-09-12T23:42:09.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:09 smithi181 bash[24776]: cluster 2024-09-12T23:42:08.613251+0000 mon.a (mon.0) 18 : cluster [INF] Activating manager daemon y 2024-09-12T23:42:09.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:09 smithi181 bash[24776]: cluster 2024-09-12T23:42:08.624970+0000 mon.a (mon.0) 19 : cluster [DBG] osdmap e291: 8 total, 8 up, 8 in 2024-09-12T23:42:09.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:09 smithi181 bash[24776]: cluster 2024-09-12T23:42:08.625824+0000 mon.a (mon.0) 20 : cluster [DBG] mgrmap e38: y(active, starting, since 0.0127233s), standbys: x 2024-09-12T23:42:09.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:09 smithi181 bash[24776]: cluster 2024-09-12T23:42:08.632529+0000 mon.a (mon.0) 21 : cluster [DBG] Standby manager daemon x restarted 2024-09-12T23:42:09.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:09 smithi181 bash[24776]: cluster 2024-09-12T23:42:08.632635+0000 mon.a (mon.0) 22 : cluster [DBG] Standby manager daemon x started 2024-09-12T23:42:09.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:09 smithi181 bash[24776]: audit 2024-09-12T23:42:08.633367+0000 mon.b (mon.1) 183 : audit [DBG] from='mgr.? 172.21.15.181:0/3495880576' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-12T23:42:09.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:09 smithi181 bash[24776]: audit 2024-09-12T23:42:08.633933+0000 mon.b (mon.1) 184 : audit [DBG] from='mgr.? 172.21.15.181:0/3495880576' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-12T23:42:09.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:09 smithi181 bash[24776]: audit 2024-09-12T23:42:08.634066+0000 mon.c (mon.2) 73 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:42:09.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:09 smithi181 bash[24776]: audit 2024-09-12T23:42:08.634570+0000 mon.c (mon.2) 74 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:42:09.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:09 smithi181 bash[24776]: audit 2024-09-12T23:42:08.634943+0000 mon.c (mon.2) 75 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:42:09.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:09 smithi181 bash[24776]: audit 2024-09-12T23:42:08.634958+0000 mon.b (mon.1) 185 : audit [DBG] from='mgr.? 172.21.15.181:0/3495880576' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-12T23:42:09.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:09 smithi181 bash[24776]: audit 2024-09-12T23:42:08.635303+0000 mon.b (mon.1) 186 : audit [DBG] from='mgr.? 172.21.15.181:0/3495880576' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-12T23:42:09.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:09 smithi181 bash[24776]: audit 2024-09-12T23:42:08.637434+0000 mon.c (mon.2) 76 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-12T23:42:09.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:09 smithi181 bash[24776]: audit 2024-09-12T23:42:08.637968+0000 mon.c (mon.2) 77 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-12T23:42:09.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:09 smithi181 bash[24776]: audit 2024-09-12T23:42:08.639451+0000 mon.c (mon.2) 78 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-12T23:42:09.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:09 smithi181 bash[24776]: audit 2024-09-12T23:42:08.640130+0000 mon.c (mon.2) 79 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-12T23:42:09.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:09 smithi181 bash[24776]: audit 2024-09-12T23:42:08.640570+0000 mon.c (mon.2) 80 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-12T23:42:09.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:09 smithi181 bash[24776]: audit 2024-09-12T23:42:08.641180+0000 mon.c (mon.2) 81 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-12T23:42:09.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:09 smithi181 bash[24776]: audit 2024-09-12T23:42:08.641650+0000 mon.c (mon.2) 82 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-12T23:42:09.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:09 smithi181 bash[24776]: audit 2024-09-12T23:42:08.642099+0000 mon.c (mon.2) 83 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-12T23:42:09.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:09 smithi181 bash[24776]: audit 2024-09-12T23:42:08.642544+0000 mon.c (mon.2) 84 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-12T23:42:09.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:09 smithi181 bash[24776]: audit 2024-09-12T23:42:08.642976+0000 mon.c (mon.2) 85 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-12T23:42:09.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:09 smithi181 bash[24776]: audit 2024-09-12T23:42:08.643375+0000 mon.c (mon.2) 86 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-12T23:42:09.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:09 smithi181 bash[24776]: audit 2024-09-12T23:42:08.643799+0000 mon.c (mon.2) 87 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-12T23:42:09.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:09 smithi181 bash[24776]: audit 2024-09-12T23:42:08.644235+0000 mon.c (mon.2) 88 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-12T23:42:09.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:09 smithi181 bash[24776]: audit 2024-09-12T23:42:08.644781+0000 mon.c (mon.2) 89 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-12T23:42:09.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:09 smithi181 bash[24776]: audit 2024-09-12T23:42:08.645828+0000 mon.c (mon.2) 90 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-12T23:42:09.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:09 smithi181 bash[24776]: cluster 2024-09-12T23:42:08.675575+0000 mon.a (mon.0) 23 : cluster [INF] Manager daemon y is now available 2024-09-12T23:42:09.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:09 smithi181 bash[24776]: audit 2024-09-12T23:42:08.696369+0000 mon.c (mon.2) 91 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:42:09.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:09 smithi181 bash[24776]: audit 2024-09-12T23:42:08.721111+0000 mon.c (mon.2) 92 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:42:09.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:09 smithi181 bash[24776]: audit 2024-09-12T23:42:08.727809+0000 mon.c (mon.2) 93 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-12T23:42:09.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:09 smithi181 bash[24776]: audit 2024-09-12T23:42:08.728141+0000 mon.a (mon.0) 24 : audit [INF] from='mgr.24661 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-12T23:42:09.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:09 smithi181 bash[24776]: audit 2024-09-12T23:42:08.750455+0000 mon.c (mon.2) 94 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-12T23:42:09.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:09 smithi181 bash[24776]: audit 2024-09-12T23:42:08.750945+0000 mon.a (mon.0) 25 : audit [INF] from='mgr.24661 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-12T23:42:09.638 INFO:tasks.rados.rados.0.smithi136.stdout:1197: done (0 left) 2024-09-12T23:42:09.638 INFO:tasks.rados.rados.0.smithi136.stdout:1198: snap_remove snap 114 2024-09-12T23:42:09.645 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[75164]: cluster 2024-09-12T23:42:08.281604+0000 mon.a (mon.0) 16 : cluster [DBG] osdmap e290: 8 total, 8 up, 8 in 2024-09-12T23:42:09.645 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[75164]: cluster 2024-09-12T23:42:08.612472+0000 mon.a (mon.0) 17 : cluster [INF] Active manager daemon y restarted 2024-09-12T23:42:09.645 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[75164]: cluster 2024-09-12T23:42:08.613251+0000 mon.a (mon.0) 18 : cluster [INF] Activating manager daemon y 2024-09-12T23:42:09.645 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[75164]: cluster 2024-09-12T23:42:08.624970+0000 mon.a (mon.0) 19 : cluster [DBG] osdmap e291: 8 total, 8 up, 8 in 2024-09-12T23:42:09.645 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[75164]: cluster 2024-09-12T23:42:08.625824+0000 mon.a (mon.0) 20 : cluster [DBG] mgrmap e38: y(active, starting, since 0.0127233s), standbys: x 2024-09-12T23:42:09.645 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[75164]: cluster 2024-09-12T23:42:08.632529+0000 mon.a (mon.0) 21 : cluster [DBG] Standby manager daemon x restarted 2024-09-12T23:42:09.645 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[75164]: cluster 2024-09-12T23:42:08.632635+0000 mon.a (mon.0) 22 : cluster [DBG] Standby manager daemon x started 2024-09-12T23:42:09.646 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[75164]: audit 2024-09-12T23:42:08.633367+0000 mon.b (mon.1) 183 : audit [DBG] from='mgr.? 172.21.15.181:0/3495880576' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-12T23:42:09.646 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[75164]: audit 2024-09-12T23:42:08.633933+0000 mon.b (mon.1) 184 : audit [DBG] from='mgr.? 172.21.15.181:0/3495880576' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-12T23:42:09.646 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[75164]: audit 2024-09-12T23:42:08.634066+0000 mon.c (mon.2) 73 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:42:09.646 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[75164]: audit 2024-09-12T23:42:08.634570+0000 mon.c (mon.2) 74 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:42:09.646 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[75164]: audit 2024-09-12T23:42:08.634943+0000 mon.c (mon.2) 75 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:42:09.646 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[75164]: audit 2024-09-12T23:42:08.634958+0000 mon.b (mon.1) 185 : audit [DBG] from='mgr.? 172.21.15.181:0/3495880576' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-12T23:42:09.646 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[75164]: audit 2024-09-12T23:42:08.635303+0000 mon.b (mon.1) 186 : audit [DBG] from='mgr.? 172.21.15.181:0/3495880576' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-12T23:42:09.646 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[75164]: audit 2024-09-12T23:42:08.637434+0000 mon.c (mon.2) 76 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-12T23:42:09.646 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[75164]: audit 2024-09-12T23:42:08.637968+0000 mon.c (mon.2) 77 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-12T23:42:09.646 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[75164]: audit 2024-09-12T23:42:08.639451+0000 mon.c (mon.2) 78 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-12T23:42:09.646 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[75164]: audit 2024-09-12T23:42:08.640130+0000 mon.c (mon.2) 79 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-12T23:42:09.646 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[75164]: audit 2024-09-12T23:42:08.640570+0000 mon.c (mon.2) 80 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-12T23:42:09.647 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[75164]: audit 2024-09-12T23:42:08.641180+0000 mon.c (mon.2) 81 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-12T23:42:09.647 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:42:09] "GET /metrics HTTP/1.1" 200 33394 "" "Prometheus/2.43.0" 2024-09-12T23:42:09.647 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[75164]: audit 2024-09-12T23:42:08.641650+0000 mon.c (mon.2) 82 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-12T23:42:09.647 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[75164]: audit 2024-09-12T23:42:08.642099+0000 mon.c (mon.2) 83 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-12T23:42:09.647 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[75164]: audit 2024-09-12T23:42:08.642544+0000 mon.c (mon.2) 84 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-12T23:42:09.647 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[75164]: audit 2024-09-12T23:42:08.642976+0000 mon.c (mon.2) 85 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-12T23:42:09.647 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[75164]: audit 2024-09-12T23:42:08.643375+0000 mon.c (mon.2) 86 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-12T23:42:09.648 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[75164]: audit 2024-09-12T23:42:08.643799+0000 mon.c (mon.2) 87 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-12T23:42:09.648 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[75164]: audit 2024-09-12T23:42:08.644235+0000 mon.c (mon.2) 88 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-12T23:42:09.648 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[75164]: audit 2024-09-12T23:42:08.644781+0000 mon.c (mon.2) 89 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-12T23:42:09.648 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[75164]: audit 2024-09-12T23:42:08.645828+0000 mon.c (mon.2) 90 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-12T23:42:09.648 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[75164]: cluster 2024-09-12T23:42:08.675575+0000 mon.a (mon.0) 23 : cluster [INF] Manager daemon y is now available 2024-09-12T23:42:09.648 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[75164]: audit 2024-09-12T23:42:08.696369+0000 mon.c (mon.2) 91 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:42:09.648 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[75164]: audit 2024-09-12T23:42:08.721111+0000 mon.c (mon.2) 92 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:42:09.648 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[75164]: audit 2024-09-12T23:42:08.727809+0000 mon.c (mon.2) 93 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-12T23:42:09.648 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[75164]: audit 2024-09-12T23:42:08.728141+0000 mon.a (mon.0) 24 : audit [INF] from='mgr.24661 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-12T23:42:09.648 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[75164]: audit 2024-09-12T23:42:08.750455+0000 mon.c (mon.2) 94 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-12T23:42:09.648 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[75164]: audit 2024-09-12T23:42:08.750945+0000 mon.a (mon.0) 25 : audit [INF] from='mgr.24661 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-12T23:42:09.649 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[32189]: cluster 2024-09-12T23:42:08.281604+0000 mon.a (mon.0) 16 : cluster [DBG] osdmap e290: 8 total, 8 up, 8 in 2024-09-12T23:42:09.649 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[32189]: cluster 2024-09-12T23:42:08.612472+0000 mon.a (mon.0) 17 : cluster [INF] Active manager daemon y restarted 2024-09-12T23:42:09.649 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[32189]: cluster 2024-09-12T23:42:08.613251+0000 mon.a (mon.0) 18 : cluster [INF] Activating manager daemon y 2024-09-12T23:42:09.649 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[32189]: cluster 2024-09-12T23:42:08.624970+0000 mon.a (mon.0) 19 : cluster [DBG] osdmap e291: 8 total, 8 up, 8 in 2024-09-12T23:42:09.649 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[32189]: cluster 2024-09-12T23:42:08.625824+0000 mon.a (mon.0) 20 : cluster [DBG] mgrmap e38: y(active, starting, since 0.0127233s), standbys: x 2024-09-12T23:42:09.649 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[32189]: cluster 2024-09-12T23:42:08.632529+0000 mon.a (mon.0) 21 : cluster [DBG] Standby manager daemon x restarted 2024-09-12T23:42:09.649 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[32189]: cluster 2024-09-12T23:42:08.632635+0000 mon.a (mon.0) 22 : cluster [DBG] Standby manager daemon x started 2024-09-12T23:42:09.649 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[32189]: audit 2024-09-12T23:42:08.633367+0000 mon.b (mon.1) 183 : audit [DBG] from='mgr.? 172.21.15.181:0/3495880576' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-12T23:42:09.649 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[32189]: audit 2024-09-12T23:42:08.633933+0000 mon.b (mon.1) 184 : audit [DBG] from='mgr.? 172.21.15.181:0/3495880576' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-12T23:42:09.650 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[32189]: audit 2024-09-12T23:42:08.634066+0000 mon.c (mon.2) 73 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:42:09.650 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[32189]: audit 2024-09-12T23:42:08.634570+0000 mon.c (mon.2) 74 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:42:09.650 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[32189]: audit 2024-09-12T23:42:08.634943+0000 mon.c (mon.2) 75 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:42:09.650 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[32189]: audit 2024-09-12T23:42:08.634958+0000 mon.b (mon.1) 185 : audit [DBG] from='mgr.? 172.21.15.181:0/3495880576' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-12T23:42:09.650 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[32189]: audit 2024-09-12T23:42:08.635303+0000 mon.b (mon.1) 186 : audit [DBG] from='mgr.? 172.21.15.181:0/3495880576' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-12T23:42:09.650 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[32189]: audit 2024-09-12T23:42:08.637434+0000 mon.c (mon.2) 76 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-12T23:42:09.650 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[32189]: audit 2024-09-12T23:42:08.637968+0000 mon.c (mon.2) 77 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-12T23:42:09.650 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[32189]: audit 2024-09-12T23:42:08.639451+0000 mon.c (mon.2) 78 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-12T23:42:09.650 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[32189]: audit 2024-09-12T23:42:08.640130+0000 mon.c (mon.2) 79 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-12T23:42:09.650 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[32189]: audit 2024-09-12T23:42:08.640570+0000 mon.c (mon.2) 80 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-12T23:42:09.651 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[32189]: audit 2024-09-12T23:42:08.641180+0000 mon.c (mon.2) 81 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-12T23:42:09.651 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[32189]: audit 2024-09-12T23:42:08.641650+0000 mon.c (mon.2) 82 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-12T23:42:09.651 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[32189]: audit 2024-09-12T23:42:08.642099+0000 mon.c (mon.2) 83 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-12T23:42:09.651 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[32189]: audit 2024-09-12T23:42:08.642544+0000 mon.c (mon.2) 84 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-12T23:42:09.651 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[32189]: audit 2024-09-12T23:42:08.642976+0000 mon.c (mon.2) 85 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-12T23:42:09.651 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[32189]: audit 2024-09-12T23:42:08.643375+0000 mon.c (mon.2) 86 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-12T23:42:09.651 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[32189]: audit 2024-09-12T23:42:08.643799+0000 mon.c (mon.2) 87 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-12T23:42:09.651 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[32189]: audit 2024-09-12T23:42:08.644235+0000 mon.c (mon.2) 88 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-12T23:42:09.651 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[32189]: audit 2024-09-12T23:42:08.644781+0000 mon.c (mon.2) 89 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-12T23:42:09.651 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[32189]: audit 2024-09-12T23:42:08.645828+0000 mon.c (mon.2) 90 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-12T23:42:09.651 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[32189]: cluster 2024-09-12T23:42:08.675575+0000 mon.a (mon.0) 23 : cluster [INF] Manager daemon y is now available 2024-09-12T23:42:09.651 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[32189]: audit 2024-09-12T23:42:08.696369+0000 mon.c (mon.2) 91 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:42:09.651 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[32189]: audit 2024-09-12T23:42:08.721111+0000 mon.c (mon.2) 92 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:42:09.652 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[32189]: audit 2024-09-12T23:42:08.727809+0000 mon.c (mon.2) 93 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-12T23:42:09.652 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[32189]: audit 2024-09-12T23:42:08.728141+0000 mon.a (mon.0) 24 : audit [INF] from='mgr.24661 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-12T23:42:09.652 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[32189]: audit 2024-09-12T23:42:08.750455+0000 mon.c (mon.2) 94 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-12T23:42:09.652 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[32189]: audit 2024-09-12T23:42:08.750945+0000 mon.a (mon.0) 25 : audit [INF] from='mgr.24661 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-12T23:42:10.005 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:09 smithi136 bash[68787]: debug 2024-09-12T23:42:09.641+0000 7f5e4b0ad640 -1 mgr.server handle_report got status from non-daemon mon.a 2024-09-12T23:42:10.643 INFO:tasks.rados.rados.0.smithi136.stdout:1198: done (0 left) 2024-09-12T23:42:10.643 INFO:tasks.rados.rados.0.smithi136.stdout:1199: delete oid 6 current snap is 115 2024-09-12T23:42:10.648 INFO:tasks.rados.rados.0.smithi136.stdout:1199: done (0 left) 2024-09-12T23:42:10.649 INFO:tasks.rados.rados.0.smithi136.stdout:1200: read oid 23 snap 91 2024-09-12T23:42:10.649 INFO:tasks.rados.rados.0.smithi136.stdout:1200: expect (ObjNum 317 snap 86 seq_num 317) 2024-09-12T23:42:10.649 INFO:tasks.rados.rados.0.smithi136.stdout:1201: delete oid 47 current snap is 115 2024-09-12T23:42:10.660 INFO:tasks.rados.rados.0.smithi136.stdout:1201: done (1 left) 2024-09-12T23:42:10.660 INFO:tasks.rados.rados.0.smithi136.stdout:1202: rollback oid 41 current snap is 115 2024-09-12T23:42:10.660 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 41 to 91 2024-09-12T23:42:10.661 INFO:tasks.rados.rados.0.smithi136.stdout:1203: snap_remove snap 113 2024-09-12T23:42:10.755 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:10 smithi136 bash[32189]: cluster 2024-09-12T23:42:09.632056+0000 mon.a (mon.0) 26 : cluster [DBG] mgrmap e39: y(active, since 1.01894s), standbys: x 2024-09-12T23:42:10.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:10 smithi136 bash[32189]: cluster 2024-09-12T23:42:09.641542+0000 mon.a (mon.0) 27 : cluster [DBG] osdmap e292: 8 total, 8 up, 8 in 2024-09-12T23:42:10.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:10 smithi136 bash[32189]: cluster 2024-09-12T23:42:09.658258+0000 mgr.y (mgr.24661) 1 : cluster [DBG] pgmap v4: 121 pgs: 4 active+clean+snaptrim, 117 active+clean; 231 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 56 GiB/s rd, 25 GiB/s wr, 296.29k op/s; 35 MiB/s, 185 objects/s recovering 2024-09-12T23:42:10.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:10 smithi136 bash[75164]: cluster 2024-09-12T23:42:09.632056+0000 mon.a (mon.0) 26 : cluster [DBG] mgrmap e39: y(active, since 1.01894s), standbys: x 2024-09-12T23:42:10.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:10 smithi136 bash[75164]: cluster 2024-09-12T23:42:09.641542+0000 mon.a (mon.0) 27 : cluster [DBG] osdmap e292: 8 total, 8 up, 8 in 2024-09-12T23:42:10.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:10 smithi136 bash[75164]: cluster 2024-09-12T23:42:09.658258+0000 mgr.y (mgr.24661) 1 : cluster [DBG] pgmap v4: 121 pgs: 4 active+clean+snaptrim, 117 active+clean; 231 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 56 GiB/s rd, 25 GiB/s wr, 296.29k op/s; 35 MiB/s, 185 objects/s recovering 2024-09-12T23:42:10.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:10 smithi136 bash[68787]: debug 2024-09-12T23:42:10.401+0000 7f5df9f4c640 -1 log_channel(cephadm) log [ERR] : [12/Sep/2024:23:42:10] ENGINE Error in HTTPServer.serve 2024-09-12T23:42:10.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:10 smithi136 bash[68787]: Traceback (most recent call last): 2024-09-12T23:42:10.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:10 smithi136 bash[68787]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-12T23:42:10.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:10 smithi136 bash[68787]: self._connections.run(self.expiration_interval) 2024-09-12T23:42:10.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:10 smithi136 bash[68787]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-12T23:42:10.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:10 smithi136 bash[68787]: self._run(expiration_interval) 2024-09-12T23:42:10.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:10 smithi136 bash[68787]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-12T23:42:10.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:10 smithi136 bash[68787]: new_conn = self._from_server_socket(self.server.socket) 2024-09-12T23:42:10.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:10 smithi136 bash[68787]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-12T23:42:10.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:10 smithi136 bash[68787]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-12T23:42:10.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:10 smithi136 bash[68787]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-12T23:42:10.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:10 smithi136 bash[68787]: s = self.context.wrap_socket( 2024-09-12T23:42:10.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:10 smithi136 bash[68787]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-12T23:42:10.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:10 smithi136 bash[68787]: return self.sslsocket_class._create( 2024-09-12T23:42:10.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:10 smithi136 bash[68787]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-12T23:42:10.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:10 smithi136 bash[68787]: self.do_handshake() 2024-09-12T23:42:10.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:10 smithi136 bash[68787]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-12T23:42:10.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:10 smithi136 bash[68787]: self._sslobj.do_handshake() 2024-09-12T23:42:10.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:10 smithi136 bash[68787]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-12T23:42:11.093 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:10 smithi181 bash[24776]: cluster 2024-09-12T23:42:09.632056+0000 mon.a (mon.0) 26 : cluster [DBG] mgrmap e39: y(active, since 1.01894s), standbys: x 2024-09-12T23:42:11.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:10 smithi181 bash[24776]: cluster 2024-09-12T23:42:09.641542+0000 mon.a (mon.0) 27 : cluster [DBG] osdmap e292: 8 total, 8 up, 8 in 2024-09-12T23:42:11.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:10 smithi181 bash[24776]: cluster 2024-09-12T23:42:09.658258+0000 mgr.y (mgr.24661) 1 : cluster [DBG] pgmap v4: 121 pgs: 4 active+clean+snaptrim, 117 active+clean; 231 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 56 GiB/s rd, 25 GiB/s wr, 296.29k op/s; 35 MiB/s, 185 objects/s recovering 2024-09-12T23:42:11.282 INFO:tasks.rados.rados.0.smithi136.stdout:1203: done (2 left) 2024-09-12T23:42:11.282 INFO:tasks.rados.rados.0.smithi136.stdout:1204: read oid 29 snap -1 2024-09-12T23:42:11.282 INFO:tasks.rados.rados.0.smithi136.stdout:1204: expect (ObjNum 381 snap 109 seq_num 381) 2024-09-12T23:42:11.282 INFO:tasks.rados.rados.0.smithi136.stdout:1205: copy_from oid 4 from oid 5 current snap is 115 2024-09-12T23:42:11.282 INFO:tasks.rados.rados.0.smithi136.stdout:1202: finishing rollback tid 0 to smithi13667081-41 2024-09-12T23:42:11.282 INFO:tasks.rados.rados.0.smithi136.stdout:1202: finishing rollback tid 1 to smithi13667081-41 2024-09-12T23:42:11.282 INFO:tasks.rados.rados.0.smithi136.stdout:1202: finishing rollback tid 2 to smithi13667081-41 2024-09-12T23:42:11.282 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 41 v 563 (ObjNum 262 snap 71 seq_num 262) dirty exists 2024-09-12T23:42:11.282 INFO:tasks.rados.rados.0.smithi136.stdout:1200: expect (ObjNum 317 snap 86 seq_num 317) 2024-09-12T23:42:11.976 INFO:tasks.rados.rados.0.smithi136.stdout:1205: finishing copy_from racing read to smithi13667081-4 2024-09-12T23:42:11.976 INFO:tasks.rados.rados.0.smithi136.stdout:1205: finishing copy_from to smithi13667081-4 2024-09-12T23:42:11.977 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 4 v 714 (ObjNum 372 snap 108 seq_num 372) dirty exists 2024-09-12T23:42:11.977 INFO:tasks.rados.rados.0.smithi136.stdout:1204: expect (ObjNum 381 snap 109 seq_num 381) 2024-09-12T23:42:12.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[75164]: cephadm 2024-09-12T23:42:10.181239+0000 mgr.y (mgr.24661) 2 : cephadm [INF] [12/Sep/2024:23:42:10] ENGINE Bus STARTING 2024-09-12T23:42:12.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[75164]: cephadm 2024-09-12T23:42:10.282868+0000 mgr.y (mgr.24661) 3 : cephadm [INF] [12/Sep/2024:23:42:10] ENGINE Serving on http://172.21.15.136:8765 2024-09-12T23:42:12.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[75164]: cephadm 2024-09-12T23:42:10.404062+0000 mgr.y (mgr.24661) 4 : cephadm [INF] [12/Sep/2024:23:42:10] ENGINE Serving on https://172.21.15.136:7150 2024-09-12T23:42:12.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[75164]: cephadm 2024-09-12T23:42:10.404238+0000 mgr.y (mgr.24661) 5 : cephadm [INF] [12/Sep/2024:23:42:10] ENGINE Bus STARTED 2024-09-12T23:42:12.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[75164]: cephadm 2024-09-12T23:42:10.406230+0000 mgr.y (mgr.24661) 6 : cephadm [ERR] [12/Sep/2024:23:42:10] ENGINE Error in HTTPServer.serve 2024-09-12T23:42:12.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[75164]: Traceback (most recent call last): 2024-09-12T23:42:12.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[75164]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-12T23:42:12.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[75164]: self._connections.run(self.expiration_interval) 2024-09-12T23:42:12.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[75164]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-12T23:42:12.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[75164]: self._run(expiration_interval) 2024-09-12T23:42:12.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[75164]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-12T23:42:12.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[75164]: new_conn = self._from_server_socket(self.server.socket) 2024-09-12T23:42:12.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[75164]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-12T23:42:12.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[75164]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-12T23:42:12.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[75164]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-12T23:42:12.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[75164]: s = self.context.wrap_socket( 2024-09-12T23:42:12.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[75164]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-12T23:42:12.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[75164]: return self.sslsocket_class._create( 2024-09-12T23:42:12.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[75164]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-12T23:42:12.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[75164]: self.do_handshake() 2024-09-12T23:42:12.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[75164]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-12T23:42:12.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[75164]: self._sslobj.do_handshake() 2024-09-12T23:42:12.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[75164]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-12T23:42:12.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[75164]: cluster 2024-09-12T23:42:10.640185+0000 mgr.y (mgr.24661) 7 : cluster [DBG] pgmap v5: 121 pgs: 4 active+clean+snaptrim, 117 active+clean; 231 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 939 MiB/s rd, 409 MiB/s wr, 4.81k op/s; 578 KiB/s, 3 objects/s recovering 2024-09-12T23:42:12.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[75164]: cluster 2024-09-12T23:42:10.646578+0000 mon.a (mon.0) 28 : cluster [DBG] osdmap e293: 8 total, 8 up, 8 in 2024-09-12T23:42:12.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[75164]: cluster 2024-09-12T23:42:11.281132+0000 mon.a (mon.0) 29 : cluster [DBG] osdmap e294: 8 total, 8 up, 8 in 2024-09-12T23:42:12.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[75164]: cluster 2024-09-12T23:42:11.638006+0000 mon.a (mon.0) 30 : cluster [DBG] mgrmap e40: y(active, since 3s), standbys: x 2024-09-12T23:42:12.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[32189]: cephadm 2024-09-12T23:42:10.181239+0000 mgr.y (mgr.24661) 2 : cephadm [INF] [12/Sep/2024:23:42:10] ENGINE Bus STARTING 2024-09-12T23:42:12.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[32189]: cephadm 2024-09-12T23:42:10.282868+0000 mgr.y (mgr.24661) 3 : cephadm [INF] [12/Sep/2024:23:42:10] ENGINE Serving on http://172.21.15.136:8765 2024-09-12T23:42:12.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[32189]: cephadm 2024-09-12T23:42:10.404062+0000 mgr.y (mgr.24661) 4 : cephadm [INF] [12/Sep/2024:23:42:10] ENGINE Serving on https://172.21.15.136:7150 2024-09-12T23:42:12.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[32189]: cephadm 2024-09-12T23:42:10.404238+0000 mgr.y (mgr.24661) 5 : cephadm [INF] [12/Sep/2024:23:42:10] ENGINE Bus STARTED 2024-09-12T23:42:12.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[32189]: cephadm 2024-09-12T23:42:10.406230+0000 mgr.y (mgr.24661) 6 : cephadm [ERR] [12/Sep/2024:23:42:10] ENGINE Error in HTTPServer.serve 2024-09-12T23:42:12.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[32189]: Traceback (most recent call last): 2024-09-12T23:42:12.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[32189]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-12T23:42:12.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[32189]: self._connections.run(self.expiration_interval) 2024-09-12T23:42:12.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[32189]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-12T23:42:12.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[32189]: self._run(expiration_interval) 2024-09-12T23:42:12.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[32189]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-12T23:42:12.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[32189]: new_conn = self._from_server_socket(self.server.socket) 2024-09-12T23:42:12.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[32189]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-12T23:42:12.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[32189]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-12T23:42:12.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[32189]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-12T23:42:12.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[32189]: s = self.context.wrap_socket( 2024-09-12T23:42:12.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[32189]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-12T23:42:12.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[32189]: return self.sslsocket_class._create( 2024-09-12T23:42:12.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[32189]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-12T23:42:12.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[32189]: self.do_handshake() 2024-09-12T23:42:12.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[32189]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-12T23:42:12.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[32189]: self._sslobj.do_handshake() 2024-09-12T23:42:12.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[32189]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-12T23:42:12.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[32189]: cluster 2024-09-12T23:42:10.640185+0000 mgr.y (mgr.24661) 7 : cluster [DBG] pgmap v5: 121 pgs: 4 active+clean+snaptrim, 117 active+clean; 231 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 939 MiB/s rd, 409 MiB/s wr, 4.81k op/s; 578 KiB/s, 3 objects/s recovering 2024-09-12T23:42:12.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[32189]: cluster 2024-09-12T23:42:10.646578+0000 mon.a (mon.0) 28 : cluster [DBG] osdmap e293: 8 total, 8 up, 8 in 2024-09-12T23:42:12.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[32189]: cluster 2024-09-12T23:42:11.281132+0000 mon.a (mon.0) 29 : cluster [DBG] osdmap e294: 8 total, 8 up, 8 in 2024-09-12T23:42:12.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:11 smithi136 bash[32189]: cluster 2024-09-12T23:42:11.638006+0000 mon.a (mon.0) 30 : cluster [DBG] mgrmap e40: y(active, since 3s), standbys: x 2024-09-12T23:42:12.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:11 smithi181 bash[24776]: cephadm 2024-09-12T23:42:10.181239+0000 mgr.y (mgr.24661) 2 : cephadm [INF] [12/Sep/2024:23:42:10] ENGINE Bus STARTING 2024-09-12T23:42:12.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:11 smithi181 bash[24776]: cephadm 2024-09-12T23:42:10.282868+0000 mgr.y (mgr.24661) 3 : cephadm [INF] [12/Sep/2024:23:42:10] ENGINE Serving on http://172.21.15.136:8765 2024-09-12T23:42:12.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:11 smithi181 bash[24776]: cephadm 2024-09-12T23:42:10.404062+0000 mgr.y (mgr.24661) 4 : cephadm [INF] [12/Sep/2024:23:42:10] ENGINE Serving on https://172.21.15.136:7150 2024-09-12T23:42:12.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:11 smithi181 bash[24776]: cephadm 2024-09-12T23:42:10.404238+0000 mgr.y (mgr.24661) 5 : cephadm [INF] [12/Sep/2024:23:42:10] ENGINE Bus STARTED 2024-09-12T23:42:12.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:11 smithi181 bash[24776]: cephadm 2024-09-12T23:42:10.406230+0000 mgr.y (mgr.24661) 6 : cephadm [ERR] [12/Sep/2024:23:42:10] ENGINE Error in HTTPServer.serve 2024-09-12T23:42:12.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:11 smithi181 bash[24776]: Traceback (most recent call last): 2024-09-12T23:42:12.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:11 smithi181 bash[24776]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-12T23:42:12.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:11 smithi181 bash[24776]: self._connections.run(self.expiration_interval) 2024-09-12T23:42:12.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:11 smithi181 bash[24776]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-12T23:42:12.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:11 smithi181 bash[24776]: self._run(expiration_interval) 2024-09-12T23:42:12.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:11 smithi181 bash[24776]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-12T23:42:12.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:11 smithi181 bash[24776]: new_conn = self._from_server_socket(self.server.socket) 2024-09-12T23:42:12.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:11 smithi181 bash[24776]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-12T23:42:12.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:11 smithi181 bash[24776]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-12T23:42:12.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:11 smithi181 bash[24776]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-12T23:42:12.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:11 smithi181 bash[24776]: s = self.context.wrap_socket( 2024-09-12T23:42:12.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:11 smithi181 bash[24776]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-12T23:42:12.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:11 smithi181 bash[24776]: return self.sslsocket_class._create( 2024-09-12T23:42:12.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:11 smithi181 bash[24776]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-12T23:42:12.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:11 smithi181 bash[24776]: self.do_handshake() 2024-09-12T23:42:12.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:11 smithi181 bash[24776]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-12T23:42:12.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:11 smithi181 bash[24776]: self._sslobj.do_handshake() 2024-09-12T23:42:12.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:11 smithi181 bash[24776]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-12T23:42:12.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:11 smithi181 bash[24776]: cluster 2024-09-12T23:42:10.640185+0000 mgr.y (mgr.24661) 7 : cluster [DBG] pgmap v5: 121 pgs: 4 active+clean+snaptrim, 117 active+clean; 231 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 939 MiB/s rd, 409 MiB/s wr, 4.81k op/s; 578 KiB/s, 3 objects/s recovering 2024-09-12T23:42:12.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:11 smithi181 bash[24776]: cluster 2024-09-12T23:42:10.646578+0000 mon.a (mon.0) 28 : cluster [DBG] osdmap e293: 8 total, 8 up, 8 in 2024-09-12T23:42:12.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:11 smithi181 bash[24776]: cluster 2024-09-12T23:42:11.281132+0000 mon.a (mon.0) 29 : cluster [DBG] osdmap e294: 8 total, 8 up, 8 in 2024-09-12T23:42:12.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:11 smithi181 bash[24776]: cluster 2024-09-12T23:42:11.638006+0000 mon.a (mon.0) 30 : cluster [DBG] mgrmap e40: y(active, since 3s), standbys: x 2024-09-12T23:42:12.136 INFO:tasks.rados.rados.0.smithi136.stdout:1200: done (3 left) 2024-09-12T23:42:12.139 INFO:tasks.rados.rados.0.smithi136.stdout:1202: done (2 left) 2024-09-12T23:42:12.139 INFO:tasks.rados.rados.0.smithi136.stdout:1204: done (1 left) 2024-09-12T23:42:12.139 INFO:tasks.rados.rados.0.smithi136.stdout:1205: done (0 left) 2024-09-12T23:42:12.139 INFO:tasks.rados.rados.0.smithi136.stdout:1206: rollback oid 33 current snap is 115 2024-09-12T23:42:12.139 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 33 to 87 2024-09-12T23:42:12.139 INFO:tasks.rados.rados.0.smithi136.stdout:1207: copy_from oid 22 from oid 6 current snap is 115 2024-09-12T23:42:12.139 INFO:tasks.rados.rados.0.smithi136.stdout:1208: setattr oid 36 current snap is 115 2024-09-12T23:42:12.142 INFO:tasks.rados.rados.0.smithi136.stdout:1207: finishing copy_from to smithi13667081-22 2024-09-12T23:42:12.142 INFO:tasks.rados.rados.0.smithi136.stdout:1207: got expected ENOENT (src dne) 2024-09-12T23:42:12.142 INFO:tasks.rados.rados.0.smithi136.stdout:1209: delete oid 2 current snap is 115 2024-09-12T23:42:12.142 INFO:tasks.rados.rados.0.smithi136.stdout:1207: finishing copy_from racing read to smithi13667081-22 2024-09-12T23:42:12.143 INFO:tasks.rados.rados.0.smithi136.stdout:1206: finishing rollback tid 0 to smithi13667081-33 2024-09-12T23:42:12.146 INFO:tasks.rados.rados.0.smithi136.stdout:1206: finishing rollback tid 1 to smithi13667081-33 2024-09-12T23:42:12.146 INFO:tasks.rados.rados.0.smithi136.stdout:1206: finishing rollback tid 2 to smithi13667081-33 2024-09-12T23:42:12.146 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 33 v 569 (ObjNum 302 snap 82 seq_num 302) dirty exists 2024-09-12T23:42:12.146 INFO:tasks.rados.rados.0.smithi136.stdout:1206: done (3 left) 2024-09-12T23:42:12.146 INFO:tasks.rados.rados.0.smithi136.stdout:1207: done (2 left) 2024-09-12T23:42:12.146 INFO:tasks.rados.rados.0.smithi136.stdout:1209: done (1 left) 2024-09-12T23:42:12.146 INFO:tasks.rados.rados.0.smithi136.stdout:1210: read oid 12 snap -1 2024-09-12T23:42:12.146 INFO:tasks.rados.rados.0.smithi136.stdout:1210: expect (ObjNum 297 snap 81 seq_num 297) 2024-09-12T23:42:12.146 INFO:tasks.rados.rados.0.smithi136.stdout:1211: delete oid 38 current snap is 115 2024-09-12T23:42:12.147 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 36 v 541 (ObjNum 361 snap 105 seq_num 361) dirty exists 2024-09-12T23:42:12.152 INFO:tasks.rados.rados.0.smithi136.stdout:1208: done (2 left) 2024-09-12T23:42:12.152 INFO:tasks.rados.rados.0.smithi136.stdout:1211: done (1 left) 2024-09-12T23:42:12.152 INFO:tasks.rados.rados.0.smithi136.stdout:1212: snap_remove snap 100 2024-09-12T23:42:12.593 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:42:12 smithi181 bash[60471]: ts=2024-09-12T23:42:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:42:12.666 INFO:tasks.rados.rados.0.smithi136.stdout:1212: done (1 left) 2024-09-12T23:42:12.666 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 34 current snap is 115 2024-09-12T23:42:12.666 INFO:tasks.rados.rados.0.smithi136.stdout:1213: seq_num 396 ranges {679936=638976,1318912=786432,2105344=417792} 2024-09-12T23:42:12.691 INFO:tasks.rados.rados.0.smithi136.stdout:1213: writing smithi13667081-34 from 679936 to 1318912 tid 1 2024-09-12T23:42:12.706 INFO:tasks.rados.rados.0.smithi136.stdout:1213: writing smithi13667081-34 from 1318912 to 2105344 tid 2 2024-09-12T23:42:12.714 INFO:tasks.rados.rados.0.smithi136.stdout:1213: writing smithi13667081-34 from 2105344 to 2523136 tid 3 2024-09-12T23:42:12.714 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 4 current snap is 115 2024-09-12T23:42:12.714 INFO:tasks.rados.rados.0.smithi136.stdout:1214: seq_num 397 ranges {3678208=655360,4333568=671744,5005312=532480} 2024-09-12T23:42:12.756 INFO:tasks.rados.rados.0.smithi136.stdout:1214: writing smithi13667081-4 from 3678208 to 4333568 tid 1 2024-09-12T23:42:12.761 INFO:tasks.rados.rados.0.smithi136.stdout:1214: writing smithi13667081-4 from 4333568 to 5005312 tid 2 2024-09-12T23:42:12.765 INFO:tasks.rados.rados.0.smithi136.stdout:1214: writing smithi13667081-4 from 5005312 to 5537792 tid 3 2024-09-12T23:42:12.765 INFO:tasks.rados.rados.0.smithi136.stdout:1215: copy_from oid 7 from oid 1 current snap is 115 2024-09-12T23:42:12.765 INFO:tasks.rados.rados.0.smithi136.stdout:1210: expect (ObjNum 297 snap 81 seq_num 297) 2024-09-12T23:42:13.084 INFO:tasks.rados.rados.0.smithi136.stdout:1213: finishing write tid 1 to smithi13667081-34 2024-09-12T23:42:13.084 INFO:tasks.rados.rados.0.smithi136.stdout:1213: finishing write tid 2 to smithi13667081-34 2024-09-12T23:42:13.085 INFO:tasks.rados.rados.0.smithi136.stdout:1213: finishing write tid 3 to smithi13667081-34 2024-09-12T23:42:13.085 INFO:tasks.rados.rados.0.smithi136.stdout:1213: finishing write tid 4 to smithi13667081-34 2024-09-12T23:42:13.085 INFO:tasks.rados.rados.0.smithi136.stdout:1213: finishing write tid 5 to smithi13667081-34 2024-09-12T23:42:13.085 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 34 v 582 (ObjNum 396 snap 115 seq_num 396) dirty exists 2024-09-12T23:42:13.085 INFO:tasks.rados.rados.0.smithi136.stdout:1213: left oid 34 (ObjNum 396 snap 115 seq_num 396) 2024-09-12T23:42:13.085 INFO:tasks.rados.rados.0.smithi136.stdout:1214: finishing write tid 1 to smithi13667081-4 2024-09-12T23:42:13.085 INFO:tasks.rados.rados.0.smithi136.stdout:1214: finishing write tid 2 to smithi13667081-4 2024-09-12T23:42:13.085 INFO:tasks.rados.rados.0.smithi136.stdout:1214: finishing write tid 3 to smithi13667081-4 2024-09-12T23:42:13.085 INFO:tasks.rados.rados.0.smithi136.stdout:1214: finishing write tid 4 to smithi13667081-4 2024-09-12T23:42:13.085 INFO:tasks.rados.rados.0.smithi136.stdout:1214: finishing write tid 5 to smithi13667081-4 2024-09-12T23:42:13.085 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 4 v 718 (ObjNum 397 snap 115 seq_num 397) dirty exists 2024-09-12T23:42:13.085 INFO:tasks.rados.rados.0.smithi136.stdout:1214: left oid 4 (ObjNum 397 snap 115 seq_num 397) 2024-09-12T23:42:13.086 INFO:tasks.rados.rados.0.smithi136.stdout:1215: finishing copy_from racing read to smithi13667081-7 2024-09-12T23:42:13.086 INFO:tasks.rados.rados.0.smithi136.stdout:1215: finishing copy_from to smithi13667081-7 2024-09-12T23:42:13.086 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 7 v 611 (ObjNum 350 snap 95 seq_num 350) dirty exists 2024-09-12T23:42:13.086 INFO:tasks.rados.rados.0.smithi136.stdout:1210: done (3 left) 2024-09-12T23:42:13.086 INFO:tasks.rados.rados.0.smithi136.stdout:1213: done (2 left) 2024-09-12T23:42:13.086 INFO:tasks.rados.rados.0.smithi136.stdout:1214: done (1 left) 2024-09-12T23:42:13.086 INFO:tasks.rados.rados.0.smithi136.stdout:1215: done (0 left) 2024-09-12T23:42:13.086 INFO:tasks.rados.rados.0.smithi136.stdout:1216: snap_remove snap 94 2024-09-12T23:42:13.685 INFO:tasks.rados.rados.0.smithi136.stdout:1216: done (0 left) 2024-09-12T23:42:13.685 INFO:tasks.rados.rados.0.smithi136.stdout:1217: read oid 5 snap 91 2024-09-12T23:42:13.685 INFO:tasks.rados.rados.0.smithi136.stdout:1217: expect (ObjNum 1140868000 snap 32541 seq_num 4242700592) 2024-09-12T23:42:13.685 INFO:tasks.rados.rados.0.smithi136.stdout:1218: read oid 50 snap 111 2024-09-12T23:42:13.685 INFO:tasks.rados.rados.0.smithi136.stdout:1218: expect (ObjNum 292 snap 79 seq_num 292) 2024-09-12T23:42:13.685 INFO:tasks.rados.rados.0.smithi136.stdout:1219: snap_create 2024-09-12T23:42:13.698 INFO:tasks.rados.rados.0.smithi136.stdout:1217: expect (ObjNum 1140868000 snap 32541 seq_num 4242700592) 2024-09-12T23:42:13.704 INFO:tasks.rados.rados.0.smithi136.stdout:1218: expect (ObjNum 292 snap 79 seq_num 292) 2024-09-12T23:42:13.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:13 smithi181 bash[24776]: cluster 2024-09-12T23:42:12.640941+0000 mgr.y (mgr.24661) 8 : cluster [DBG] pgmap v8: 121 pgs: 4 active+clean+snaptrim, 117 active+clean; 231 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 312 MiB/s rd, 136 MiB/s wr, 1.60k op/s; 192 KiB/s, 1 objects/s recovering 2024-09-12T23:42:13.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:13 smithi181 bash[24776]: cluster 2024-09-12T23:42:12.667225+0000 mon.a (mon.0) 31 : cluster [DBG] osdmap e295: 8 total, 8 up, 8 in 2024-09-12T23:42:13.844 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:42:13 smithi181 bash[60471]: ts=2024-09-12T23:42:13.507Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi136\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:42:14.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:13 smithi136 bash[75164]: cluster 2024-09-12T23:42:12.640941+0000 mgr.y (mgr.24661) 8 : cluster [DBG] pgmap v8: 121 pgs: 4 active+clean+snaptrim, 117 active+clean; 231 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 312 MiB/s rd, 136 MiB/s wr, 1.60k op/s; 192 KiB/s, 1 objects/s recovering 2024-09-12T23:42:14.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:13 smithi136 bash[75164]: cluster 2024-09-12T23:42:12.667225+0000 mon.a (mon.0) 31 : cluster [DBG] osdmap e295: 8 total, 8 up, 8 in 2024-09-12T23:42:14.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:13 smithi136 bash[32189]: cluster 2024-09-12T23:42:12.640941+0000 mgr.y (mgr.24661) 8 : cluster [DBG] pgmap v8: 121 pgs: 4 active+clean+snaptrim, 117 active+clean; 231 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 312 MiB/s rd, 136 MiB/s wr, 1.60k op/s; 192 KiB/s, 1 objects/s recovering 2024-09-12T23:42:14.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:13 smithi136 bash[32189]: cluster 2024-09-12T23:42:12.667225+0000 mon.a (mon.0) 31 : cluster [DBG] osdmap e295: 8 total, 8 up, 8 in 2024-09-12T23:42:14.700 INFO:tasks.rados.rados.0.smithi136.stdout:1217: done (2 left) 2024-09-12T23:42:14.700 INFO:tasks.rados.rados.0.smithi136.stdout:1218: done (1 left) 2024-09-12T23:42:14.700 INFO:tasks.rados.rados.0.smithi136.stdout:1219: done (0 left) 2024-09-12T23:42:14.700 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 14 current snap is 116 2024-09-12T23:42:14.700 INFO:tasks.rados.rados.0.smithi136.stdout:1220: seq_num 398 ranges {2899968=671744,3571712=557056,4128768=532480,4661248=114688} 2024-09-12T23:42:14.720 INFO:tasks.rados.rados.0.smithi136.stdout:1220: writing smithi13667081-14 from 2899968 to 3571712 tid 1 2024-09-12T23:42:14.723 INFO:tasks.rados.rados.0.smithi136.stdout:1220: writing smithi13667081-14 from 3571712 to 4128768 tid 2 2024-09-12T23:42:14.727 INFO:tasks.rados.rados.0.smithi136.stdout:1220: writing smithi13667081-14 from 4128768 to 4661248 tid 3 2024-09-12T23:42:14.728 INFO:tasks.rados.rados.0.smithi136.stdout:1220: writing smithi13667081-14 from 4661248 to 4775936 tid 4 2024-09-12T23:42:14.728 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 39 current snap is 116 2024-09-12T23:42:14.728 INFO:tasks.rados.rados.0.smithi136.stdout:1220: finishing write tid 1 to smithi13667081-14 2024-09-12T23:42:14.728 INFO:tasks.rados.rados.0.smithi136.stdout:1220: finishing write tid 2 to smithi13667081-14 2024-09-12T23:42:14.728 INFO:tasks.rados.rados.0.smithi136.stdout:1221: seq_num 399 ranges {3334144=688128,4022272=434176,4456448=745472,5201920=24576} 2024-09-12T23:42:14.751 INFO:tasks.rados.rados.0.smithi136.stdout:1221: writing smithi13667081-39 from 3334144 to 4022272 tid 1 2024-09-12T23:42:14.753 INFO:tasks.rados.rados.0.smithi136.stdout:1221: writing smithi13667081-39 from 4022272 to 4456448 tid 2 2024-09-12T23:42:14.757 INFO:tasks.rados.rados.0.smithi136.stdout:1221: writing smithi13667081-39 from 4456448 to 5201920 tid 3 2024-09-12T23:42:14.758 INFO:tasks.rados.rados.0.smithi136.stdout:1221: writing smithi13667081-39 from 5201920 to 5226496 tid 4 2024-09-12T23:42:14.758 INFO:tasks.rados.rados.0.smithi136.stdout:1222: read oid 9 snap -1 2024-09-12T23:42:14.758 INFO:tasks.rados.rados.0.smithi136.stdout:1222: expect deleted 2024-09-12T23:42:14.758 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 28 current snap is 116 2024-09-12T23:42:14.758 INFO:tasks.rados.rados.0.smithi136.stdout:1220: finishing write tid 3 to smithi13667081-14 2024-09-12T23:42:14.758 INFO:tasks.rados.rados.0.smithi136.stdout:1223: seq_num 400 ranges {0=704512,704512=720896,1425408=483328} 2024-09-12T23:42:14.762 INFO:tasks.rados.rados.0.smithi136.stdout:1223: writing smithi13667081-28 from 0 to 704512 tid 1 2024-09-12T23:42:14.766 INFO:tasks.rados.rados.0.smithi136.stdout:1223: writing smithi13667081-28 from 704512 to 1425408 tid 2 2024-09-12T23:42:14.770 INFO:tasks.rados.rados.0.smithi136.stdout:1223: writing smithi13667081-28 from 1425408 to 1908736 tid 3 2024-09-12T23:42:14.770 INFO:tasks.rados.rados.0.smithi136.stdout:1224: snap_create 2024-09-12T23:42:14.770 INFO:tasks.rados.rados.0.smithi136.stdout:1220: finishing write tid 4 to smithi13667081-14 2024-09-12T23:42:14.770 INFO:tasks.rados.rados.0.smithi136.stdout:1220: finishing write tid 5 to smithi13667081-14 2024-09-12T23:42:14.770 INFO:tasks.rados.rados.0.smithi136.stdout:1220: finishing write tid 6 to smithi13667081-14 2024-09-12T23:42:14.770 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 14 v 685 (ObjNum 398 snap 116 seq_num 398) dirty exists 2024-09-12T23:42:14.770 INFO:tasks.rados.rados.0.smithi136.stdout:1220: left oid 14 (ObjNum 398 snap 116 seq_num 398) 2024-09-12T23:42:14.770 INFO:tasks.rados.rados.0.smithi136.stdout:1221: finishing write tid 1 to smithi13667081-39 2024-09-12T23:42:14.770 INFO:tasks.rados.rados.0.smithi136.stdout:1221: finishing write tid 2 to smithi13667081-39 2024-09-12T23:42:14.770 INFO:tasks.rados.rados.0.smithi136.stdout:1221: finishing write tid 3 to smithi13667081-39 2024-09-12T23:42:14.770 INFO:tasks.rados.rados.0.smithi136.stdout:1221: finishing write tid 4 to smithi13667081-39 2024-09-12T23:42:14.771 INFO:tasks.rados.rados.0.smithi136.stdout:1221: finishing write tid 5 to smithi13667081-39 2024-09-12T23:42:14.771 INFO:tasks.rados.rados.0.smithi136.stdout:1221: finishing write tid 6 to smithi13667081-39 2024-09-12T23:42:14.771 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 39 v 199 (ObjNum 399 snap 116 seq_num 399) dirty exists 2024-09-12T23:42:14.771 INFO:tasks.rados.rados.0.smithi136.stdout:1221: left oid 39 (ObjNum 399 snap 116 seq_num 399) 2024-09-12T23:42:14.771 INFO:tasks.rados.rados.0.smithi136.stdout:1223: finishing write tid 1 to smithi13667081-28 2024-09-12T23:42:14.771 INFO:tasks.rados.rados.0.smithi136.stdout:1223: finishing write tid 2 to smithi13667081-28 2024-09-12T23:42:14.772 INFO:tasks.rados.rados.0.smithi136.stdout:1223: finishing write tid 3 to smithi13667081-28 2024-09-12T23:42:14.772 INFO:tasks.rados.rados.0.smithi136.stdout:1223: finishing write tid 4 to smithi13667081-28 2024-09-12T23:42:14.774 INFO:tasks.rados.rados.0.smithi136.stdout:1223: finishing write tid 5 to smithi13667081-28 2024-09-12T23:42:14.774 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 28 v 689 (ObjNum 400 snap 116 seq_num 400) dirty exists 2024-09-12T23:42:14.774 INFO:tasks.rados.rados.0.smithi136.stdout:1223: left oid 28 (ObjNum 400 snap 116 seq_num 400) 2024-09-12T23:42:14.843 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:42:14 smithi181 bash[60471]: ts=2024-09-12T23:42:14.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:42:14.844 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:42:14 smithi181 bash[60471]: ts=2024-09-12T23:42:14.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:42:14.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:14 smithi181 bash[24776]: cluster 2024-09-12T23:42:13.669861+0000 mon.a (mon.0) 32 : cluster [DBG] mgrmap e41: y(active, since 5s), standbys: x 2024-09-12T23:42:14.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:14 smithi181 bash[24776]: cluster 2024-09-12T23:42:13.686005+0000 mon.a (mon.0) 33 : cluster [DBG] osdmap e296: 8 total, 8 up, 8 in 2024-09-12T23:42:14.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:14 smithi181 bash[24776]: audit 2024-09-12T23:42:14.651212+0000 mon.a (mon.0) 34 : audit [INF] from='mgr.24661 ' entity='mgr.y' 2024-09-12T23:42:14.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:14 smithi181 bash[24776]: audit 2024-09-12T23:42:14.657333+0000 mon.a (mon.0) 35 : audit [INF] from='mgr.24661 ' entity='mgr.y' 2024-09-12T23:42:15.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:14 smithi136 bash[75164]: cluster 2024-09-12T23:42:13.669861+0000 mon.a (mon.0) 32 : cluster [DBG] mgrmap e41: y(active, since 5s), standbys: x 2024-09-12T23:42:15.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:14 smithi136 bash[75164]: cluster 2024-09-12T23:42:13.686005+0000 mon.a (mon.0) 33 : cluster [DBG] osdmap e296: 8 total, 8 up, 8 in 2024-09-12T23:42:15.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:14 smithi136 bash[75164]: audit 2024-09-12T23:42:14.651212+0000 mon.a (mon.0) 34 : audit [INF] from='mgr.24661 ' entity='mgr.y' 2024-09-12T23:42:15.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:14 smithi136 bash[75164]: audit 2024-09-12T23:42:14.657333+0000 mon.a (mon.0) 35 : audit [INF] from='mgr.24661 ' entity='mgr.y' 2024-09-12T23:42:15.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:14 smithi136 bash[32189]: cluster 2024-09-12T23:42:13.669861+0000 mon.a (mon.0) 32 : cluster [DBG] mgrmap e41: y(active, since 5s), standbys: x 2024-09-12T23:42:15.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:14 smithi136 bash[32189]: cluster 2024-09-12T23:42:13.686005+0000 mon.a (mon.0) 33 : cluster [DBG] osdmap e296: 8 total, 8 up, 8 in 2024-09-12T23:42:15.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:14 smithi136 bash[32189]: audit 2024-09-12T23:42:14.651212+0000 mon.a (mon.0) 34 : audit [INF] from='mgr.24661 ' entity='mgr.y' 2024-09-12T23:42:15.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:14 smithi136 bash[32189]: audit 2024-09-12T23:42:14.657333+0000 mon.a (mon.0) 35 : audit [INF] from='mgr.24661 ' entity='mgr.y' 2024-09-12T23:42:15.709 INFO:tasks.rados.rados.0.smithi136.stdout:1220: done (4 left) 2024-09-12T23:42:15.709 INFO:tasks.rados.rados.0.smithi136.stdout:1221: done (3 left) 2024-09-12T23:42:15.709 INFO:tasks.rados.rados.0.smithi136.stdout:1222: done (2 left) 2024-09-12T23:42:15.709 INFO:tasks.rados.rados.0.smithi136.stdout:1223: done (1 left) 2024-09-12T23:42:15.709 INFO:tasks.rados.rados.0.smithi136.stdout:1224: done (0 left) 2024-09-12T23:42:15.709 INFO:tasks.rados.rados.0.smithi136.stdout:1225: snap_create 2024-09-12T23:42:16.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:15 smithi136 bash[75164]: cluster 2024-09-12T23:42:14.641469+0000 mgr.y (mgr.24661) 9 : cluster [DBG] pgmap v11: 121 pgs: 4 active+clean+snaptrim, 117 active+clean; 229 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-12T23:42:16.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:15 smithi136 bash[75164]: cluster 2024-09-12T23:42:14.704155+0000 mon.a (mon.0) 36 : cluster [DBG] osdmap e297: 8 total, 8 up, 8 in 2024-09-12T23:42:16.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:15 smithi136 bash[75164]: audit 2024-09-12T23:42:14.900275+0000 mon.a (mon.0) 37 : audit [INF] from='mgr.24661 ' entity='mgr.y' 2024-09-12T23:42:16.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:15 smithi136 bash[75164]: audit 2024-09-12T23:42:14.915782+0000 mon.a (mon.0) 38 : audit [INF] from='mgr.24661 ' entity='mgr.y' 2024-09-12T23:42:16.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:15 smithi136 bash[75164]: audit 2024-09-12T23:42:15.336145+0000 mon.a (mon.0) 39 : audit [INF] from='mgr.24661 ' entity='mgr.y' 2024-09-12T23:42:16.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:15 smithi136 bash[75164]: audit 2024-09-12T23:42:15.344998+0000 mon.a (mon.0) 40 : audit [INF] from='mgr.24661 ' entity='mgr.y' 2024-09-12T23:42:16.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:15 smithi136 bash[75164]: audit 2024-09-12T23:42:15.616701+0000 mon.a (mon.0) 41 : audit [INF] from='mgr.24661 ' entity='mgr.y' 2024-09-12T23:42:16.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:15 smithi136 bash[75164]: audit 2024-09-12T23:42:15.632335+0000 mon.a (mon.0) 42 : audit [INF] from='mgr.24661 ' entity='mgr.y' 2024-09-12T23:42:16.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:15 smithi136 bash[32189]: cluster 2024-09-12T23:42:14.641469+0000 mgr.y (mgr.24661) 9 : cluster [DBG] pgmap v11: 121 pgs: 4 active+clean+snaptrim, 117 active+clean; 229 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-12T23:42:16.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:15 smithi136 bash[32189]: cluster 2024-09-12T23:42:14.704155+0000 mon.a (mon.0) 36 : cluster [DBG] osdmap e297: 8 total, 8 up, 8 in 2024-09-12T23:42:16.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:15 smithi136 bash[32189]: audit 2024-09-12T23:42:14.900275+0000 mon.a (mon.0) 37 : audit [INF] from='mgr.24661 ' entity='mgr.y' 2024-09-12T23:42:16.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:15 smithi136 bash[32189]: audit 2024-09-12T23:42:14.915782+0000 mon.a (mon.0) 38 : audit [INF] from='mgr.24661 ' entity='mgr.y' 2024-09-12T23:42:16.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:15 smithi136 bash[32189]: audit 2024-09-12T23:42:15.336145+0000 mon.a (mon.0) 39 : audit [INF] from='mgr.24661 ' entity='mgr.y' 2024-09-12T23:42:16.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:15 smithi136 bash[32189]: audit 2024-09-12T23:42:15.344998+0000 mon.a (mon.0) 40 : audit [INF] from='mgr.24661 ' entity='mgr.y' 2024-09-12T23:42:16.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:15 smithi136 bash[32189]: audit 2024-09-12T23:42:15.616701+0000 mon.a (mon.0) 41 : audit [INF] from='mgr.24661 ' entity='mgr.y' 2024-09-12T23:42:16.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:15 smithi136 bash[32189]: audit 2024-09-12T23:42:15.632335+0000 mon.a (mon.0) 42 : audit [INF] from='mgr.24661 ' entity='mgr.y' 2024-09-12T23:42:16.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:15 smithi181 bash[24776]: cluster 2024-09-12T23:42:14.641469+0000 mgr.y (mgr.24661) 9 : cluster [DBG] pgmap v11: 121 pgs: 4 active+clean+snaptrim, 117 active+clean; 229 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-12T23:42:16.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:15 smithi181 bash[24776]: cluster 2024-09-12T23:42:14.704155+0000 mon.a (mon.0) 36 : cluster [DBG] osdmap e297: 8 total, 8 up, 8 in 2024-09-12T23:42:16.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:15 smithi181 bash[24776]: audit 2024-09-12T23:42:14.900275+0000 mon.a (mon.0) 37 : audit [INF] from='mgr.24661 ' entity='mgr.y' 2024-09-12T23:42:16.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:15 smithi181 bash[24776]: audit 2024-09-12T23:42:14.915782+0000 mon.a (mon.0) 38 : audit [INF] from='mgr.24661 ' entity='mgr.y' 2024-09-12T23:42:16.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:15 smithi181 bash[24776]: audit 2024-09-12T23:42:15.336145+0000 mon.a (mon.0) 39 : audit [INF] from='mgr.24661 ' entity='mgr.y' 2024-09-12T23:42:16.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:15 smithi181 bash[24776]: audit 2024-09-12T23:42:15.344998+0000 mon.a (mon.0) 40 : audit [INF] from='mgr.24661 ' entity='mgr.y' 2024-09-12T23:42:16.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:15 smithi181 bash[24776]: audit 2024-09-12T23:42:15.616701+0000 mon.a (mon.0) 41 : audit [INF] from='mgr.24661 ' entity='mgr.y' 2024-09-12T23:42:16.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:15 smithi181 bash[24776]: audit 2024-09-12T23:42:15.632335+0000 mon.a (mon.0) 42 : audit [INF] from='mgr.24661 ' entity='mgr.y' 2024-09-12T23:42:16.728 INFO:tasks.rados.rados.0.smithi136.stdout:1225: done (0 left) 2024-09-12T23:42:16.729 INFO:tasks.rados.rados.0.smithi136.stdout:1226: delete oid 18 current snap is 118 2024-09-12T23:42:16.741 INFO:tasks.rados.rados.0.smithi136.stdout:1226: done (0 left) 2024-09-12T23:42:16.741 INFO:tasks.rados.rados.0.smithi136.stdout:1227: read oid 32 snap -1 2024-09-12T23:42:16.741 INFO:tasks.rados.rados.0.smithi136.stdout:1227: expect (ObjNum 353 snap 96 seq_num 353) 2024-09-12T23:42:16.741 INFO:tasks.rados.rados.0.smithi136.stdout:1228: read oid 48 snap -1 2024-09-12T23:42:16.741 INFO:tasks.rados.rados.0.smithi136.stdout:1228: expect (ObjNum 358 snap 102 seq_num 358) 2024-09-12T23:42:16.741 INFO:tasks.rados.rados.0.smithi136.stdout:1229: read oid 14 snap 87 2024-09-12T23:42:16.741 INFO:tasks.rados.rados.0.smithi136.stdout:1229: expect (ObjNum 277 snap 74 seq_num 277) 2024-09-12T23:42:16.741 INFO:tasks.rados.rados.0.smithi136.stdout:1230: copy_from oid 28 from oid 34 current snap is 118 2024-09-12T23:42:16.741 INFO:tasks.rados.rados.0.smithi136.stdout:1231: snap_create 2024-09-12T23:42:16.767 INFO:tasks.rados.rados.0.smithi136.stdout:1228: expect (ObjNum 358 snap 102 seq_num 358) 2024-09-12T23:42:16.884 INFO:tasks.rados.rados.0.smithi136.stdout:1227: expect (ObjNum 353 snap 96 seq_num 353) 2024-09-12T23:42:17.002 INFO:tasks.rados.rados.0.smithi136.stdout:1230: finishing copy_from racing read to smithi13667081-28 2024-09-12T23:42:17.002 INFO:tasks.rados.rados.0.smithi136.stdout:1229: expect (ObjNum 277 snap 74 seq_num 277) 2024-09-12T23:42:17.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:16 smithi136 bash[32189]: cluster 2024-09-12T23:42:15.708491+0000 mon.a (mon.0) 43 : cluster [DBG] osdmap e298: 8 total, 8 up, 8 in 2024-09-12T23:42:17.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:16 smithi136 bash[75164]: cluster 2024-09-12T23:42:15.708491+0000 mon.a (mon.0) 43 : cluster [DBG] osdmap e298: 8 total, 8 up, 8 in 2024-09-12T23:42:17.091 INFO:tasks.rados.rados.0.smithi136.stdout:1230: finishing copy_from to smithi13667081-28 2024-09-12T23:42:17.092 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 28 v 582 (ObjNum 396 snap 115 seq_num 396) dirty exists 2024-09-12T23:42:17.093 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:16 smithi181 bash[24776]: cluster 2024-09-12T23:42:15.708491+0000 mon.a (mon.0) 43 : cluster [DBG] osdmap e298: 8 total, 8 up, 8 in 2024-09-12T23:42:17.737 INFO:tasks.rados.rados.0.smithi136.stdout:1227: done (4 left) 2024-09-12T23:42:17.738 INFO:tasks.rados.rados.0.smithi136.stdout:1228: done (3 left) 2024-09-12T23:42:17.738 INFO:tasks.rados.rados.0.smithi136.stdout:1229: done (2 left) 2024-09-12T23:42:17.738 INFO:tasks.rados.rados.0.smithi136.stdout:1230: done (1 left) 2024-09-12T23:42:17.738 INFO:tasks.rados.rados.0.smithi136.stdout:1231: done (0 left) 2024-09-12T23:42:17.738 INFO:tasks.rados.rados.0.smithi136.stdout:1232: copy_from oid 34 from oid 25 current snap is 119 2024-09-12T23:42:17.738 INFO:tasks.rados.rados.0.smithi136.stdout:1233: rollback oid 49 current snap is 119 2024-09-12T23:42:17.738 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 49 to 87 2024-09-12T23:42:17.739 INFO:tasks.rados.rados.0.smithi136.stdout:1234: delete oid 4 current snap is 119 2024-09-12T23:42:17.743 INFO:tasks.rados.rados.0.smithi136.stdout:1233: finishing rollback tid 1 to smithi13667081-49 2024-09-12T23:42:17.744 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 49 v 0 (ObjNum 113 snap 0 seq_num 2878056952) dirty dne 2024-09-12T23:42:17.746 INFO:tasks.rados.rados.0.smithi136.stdout:1233: done (2 left) 2024-09-12T23:42:17.746 INFO:tasks.rados.rados.0.smithi136.stdout:1234: done (1 left) 2024-09-12T23:42:17.746 INFO:tasks.rados.rados.0.smithi136.stdout:1235: delete oid 42 current snap is 119 2024-09-12T23:42:17.753 INFO:tasks.rados.rados.0.smithi136.stdout:1235: done (1 left) 2024-09-12T23:42:17.754 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 3 current snap is 119 2024-09-12T23:42:17.754 INFO:tasks.rados.rados.0.smithi136.stdout:1236: seq_num 404 ranges {0=770048,770048=712704,1482752=491520} 2024-09-12T23:42:17.768 INFO:tasks.rados.rados.0.smithi136.stdout:1236: writing smithi13667081-3 from 0 to 770048 tid 1 2024-09-12T23:42:17.780 INFO:tasks.rados.rados.0.smithi136.stdout:1236: writing smithi13667081-3 from 770048 to 1482752 tid 2 2024-09-12T23:42:17.789 INFO:tasks.rados.rados.0.smithi136.stdout:1236: writing smithi13667081-3 from 1482752 to 1974272 tid 3 2024-09-12T23:42:17.789 INFO:tasks.rados.rados.0.smithi136.stdout:1237: rmattr oid 41 current snap is 119 2024-09-12T23:42:17.789 INFO:tasks.rados.rados.0.smithi136.stdout:1237: done (2 left) 2024-09-12T23:42:17.789 INFO:tasks.rados.rados.0.smithi136.stdout:1238: read oid 38 snap 87 2024-09-12T23:42:17.789 INFO:tasks.rados.rados.0.smithi136.stdout:1238: expect (ObjNum 300 snap 82 seq_num 300) 2024-09-12T23:42:17.790 INFO:tasks.rados.rados.0.smithi136.stdout:1239: snap_remove snap 115 2024-09-12T23:42:18.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:17 smithi136 bash[75164]: cluster 2024-09-12T23:42:16.642172+0000 mgr.y (mgr.24661) 10 : cluster [DBG] pgmap v14: 121 pgs: 121 active+clean; 228 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 2.6 MiB/s wr, 44 op/s 2024-09-12T23:42:18.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:17 smithi136 bash[75164]: cluster 2024-09-12T23:42:16.729109+0000 mon.a (mon.0) 44 : cluster [DBG] osdmap e299: 8 total, 8 up, 8 in 2024-09-12T23:42:18.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:17 smithi136 bash[32189]: cluster 2024-09-12T23:42:16.642172+0000 mgr.y (mgr.24661) 10 : cluster [DBG] pgmap v14: 121 pgs: 121 active+clean; 228 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 2.6 MiB/s wr, 44 op/s 2024-09-12T23:42:18.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:17 smithi136 bash[32189]: cluster 2024-09-12T23:42:16.729109+0000 mon.a (mon.0) 44 : cluster [DBG] osdmap e299: 8 total, 8 up, 8 in 2024-09-12T23:42:18.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:17 smithi181 bash[24776]: cluster 2024-09-12T23:42:16.642172+0000 mgr.y (mgr.24661) 10 : cluster [DBG] pgmap v14: 121 pgs: 121 active+clean; 228 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 2.6 MiB/s wr, 44 op/s 2024-09-12T23:42:18.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:17 smithi181 bash[24776]: cluster 2024-09-12T23:42:16.729109+0000 mon.a (mon.0) 44 : cluster [DBG] osdmap e299: 8 total, 8 up, 8 in 2024-09-12T23:42:18.748 INFO:tasks.rados.rados.0.smithi136.stdout:1239: done (3 left) 2024-09-12T23:42:18.748 INFO:tasks.rados.rados.0.smithi136.stdout:1240: read oid 23 snap -1 2024-09-12T23:42:18.748 INFO:tasks.rados.rados.0.smithi136.stdout:1240: expect (ObjNum 387 snap 111 seq_num 387) 2024-09-12T23:42:18.748 INFO:tasks.rados.rados.0.smithi136.stdout:1241: delete oid 22 current snap is 119 2024-09-12T23:42:18.748 INFO:tasks.rados.rados.0.smithi136.stdout:1232: finishing copy_from racing read to smithi13667081-34 2024-09-12T23:42:18.748 INFO:tasks.rados.rados.0.smithi136.stdout:1232: finishing copy_from to smithi13667081-34 2024-09-12T23:42:18.748 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 34 v 180 (ObjNum 370 snap 108 seq_num 370) dirty exists 2024-09-12T23:42:18.748 INFO:tasks.rados.rados.0.smithi136.stdout:1236: finishing write tid 1 to smithi13667081-3 2024-09-12T23:42:18.748 INFO:tasks.rados.rados.0.smithi136.stdout:1236: finishing write tid 2 to smithi13667081-3 2024-09-12T23:42:18.749 INFO:tasks.rados.rados.0.smithi136.stdout:1236: finishing write tid 3 to smithi13667081-3 2024-09-12T23:42:18.749 INFO:tasks.rados.rados.0.smithi136.stdout:1236: finishing write tid 4 to smithi13667081-3 2024-09-12T23:42:18.749 INFO:tasks.rados.rados.0.smithi136.stdout:1236: finishing write tid 5 to smithi13667081-3 2024-09-12T23:42:18.749 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 3 v 612 (ObjNum 404 snap 119 seq_num 404) dirty exists 2024-09-12T23:42:18.749 INFO:tasks.rados.rados.0.smithi136.stdout:1236: left oid 3 (ObjNum 404 snap 119 seq_num 404) 2024-09-12T23:42:18.749 INFO:tasks.rados.rados.0.smithi136.stdout:1238: expect (ObjNum 300 snap 82 seq_num 300) 2024-09-12T23:42:19.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:18 smithi136 bash[75164]: cluster 2024-09-12T23:42:17.739058+0000 mon.a (mon.0) 45 : cluster [DBG] osdmap e300: 8 total, 8 up, 8 in 2024-09-12T23:42:19.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:18 smithi136 bash[32189]: cluster 2024-09-12T23:42:17.739058+0000 mon.a (mon.0) 45 : cluster [DBG] osdmap e300: 8 total, 8 up, 8 in 2024-09-12T23:42:19.011 INFO:tasks.rados.rados.0.smithi136.stdout:1240: expect (ObjNum 387 snap 111 seq_num 387) 2024-09-12T23:42:19.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:18 smithi181 bash[24776]: cluster 2024-09-12T23:42:17.739058+0000 mon.a (mon.0) 45 : cluster [DBG] osdmap e300: 8 total, 8 up, 8 in 2024-09-12T23:42:19.094 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:42:18 smithi181 bash[60471]: ts=2024-09-12T23:42:18.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:42:19.740 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:42:19] "GET /metrics HTTP/1.1" 200 33394 "" "Prometheus/2.43.0" 2024-09-12T23:42:19.795 INFO:tasks.rados.rados.0.smithi136.stdout:1232: done (4 left) 2024-09-12T23:42:19.795 INFO:tasks.rados.rados.0.smithi136.stdout:1236: done (3 left) 2024-09-12T23:42:19.795 INFO:tasks.rados.rados.0.smithi136.stdout:1238: done (2 left) 2024-09-12T23:42:19.796 INFO:tasks.rados.rados.0.smithi136.stdout:1240: done (1 left) 2024-09-12T23:42:19.799 INFO:tasks.rados.rados.0.smithi136.stdout:1241: done (0 left) 2024-09-12T23:42:19.799 INFO:tasks.rados.rados.0.smithi136.stdout:1242: rollback oid 33 current snap is 119 2024-09-12T23:42:19.799 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 33 to 87 2024-09-12T23:42:19.799 INFO:tasks.rados.rados.0.smithi136.stdout:1243: snap_remove snap 91 2024-09-12T23:42:20.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[32189]: cluster 2024-09-12T23:42:18.642920+0000 mgr.y (mgr.24661) 11 : cluster [DBG] pgmap v17: 121 pgs: 121 active+clean; 228 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 2.6 MiB/s wr, 44 op/s 2024-09-12T23:42:20.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[32189]: cluster 2024-09-12T23:42:18.748975+0000 mon.a (mon.0) 46 : cluster [DBG] osdmap e301: 8 total, 8 up, 8 in 2024-09-12T23:42:20.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[32189]: audit 2024-09-12T23:42:18.769349+0000 mon.c (mon.2) 95 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:42:20.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[32189]: audit 2024-09-12T23:42:18.791100+0000 mon.c (mon.2) 96 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-12T23:42:20.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[32189]: audit 2024-09-12T23:42:18.891341+0000 mon.c (mon.2) 97 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:42:20.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[32189]: audit 2024-09-12T23:42:18.891563+0000 mon.c (mon.2) 98 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:42:20.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[32189]: audit 2024-09-12T23:42:18.891781+0000 mon.c (mon.2) 99 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:42:20.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[32189]: audit 2024-09-12T23:42:18.894316+0000 mon.c (mon.2) 100 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:42:20.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[32189]: audit 2024-09-12T23:42:18.894541+0000 mon.c (mon.2) 101 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:42:20.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[32189]: audit 2024-09-12T23:42:18.894789+0000 mon.c (mon.2) 102 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:42:20.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[32189]: audit 2024-09-12T23:42:18.896677+0000 mon.c (mon.2) 103 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:42:20.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[32189]: audit 2024-09-12T23:42:18.896931+0000 mon.c (mon.2) 104 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:42:20.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[32189]: audit 2024-09-12T23:42:18.897324+0000 mon.c (mon.2) 105 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:42:20.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[32189]: audit 2024-09-12T23:42:18.911115+0000 mon.a (mon.0) 47 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:42:20.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[32189]: audit 2024-09-12T23:42:18.911250+0000 mon.a (mon.0) 48 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:42:20.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[32189]: audit 2024-09-12T23:42:18.911302+0000 mon.c (mon.2) 106 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:42:20.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[32189]: audit 2024-09-12T23:42:18.911893+0000 mon.c (mon.2) 107 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:42:20.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[32189]: audit 2024-09-12T23:42:18.912048+0000 mon.c (mon.2) 108 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:42:20.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[32189]: audit 2024-09-12T23:42:18.914584+0000 mon.c (mon.2) 109 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:42:20.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[75164]: cluster 2024-09-12T23:42:18.642920+0000 mgr.y (mgr.24661) 11 : cluster [DBG] pgmap v17: 121 pgs: 121 active+clean; 228 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 2.6 MiB/s wr, 44 op/s 2024-09-12T23:42:20.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[75164]: cluster 2024-09-12T23:42:18.748975+0000 mon.a (mon.0) 46 : cluster [DBG] osdmap e301: 8 total, 8 up, 8 in 2024-09-12T23:42:20.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[75164]: audit 2024-09-12T23:42:18.769349+0000 mon.c (mon.2) 95 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:42:20.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[75164]: audit 2024-09-12T23:42:18.791100+0000 mon.c (mon.2) 96 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-12T23:42:20.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[75164]: audit 2024-09-12T23:42:18.891341+0000 mon.c (mon.2) 97 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:42:20.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[75164]: audit 2024-09-12T23:42:18.891563+0000 mon.c (mon.2) 98 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:42:20.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[75164]: audit 2024-09-12T23:42:18.891781+0000 mon.c (mon.2) 99 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:42:20.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[75164]: audit 2024-09-12T23:42:18.894316+0000 mon.c (mon.2) 100 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:42:20.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[75164]: audit 2024-09-12T23:42:18.894541+0000 mon.c (mon.2) 101 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:42:20.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[75164]: audit 2024-09-12T23:42:18.894789+0000 mon.c (mon.2) 102 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:42:20.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[75164]: audit 2024-09-12T23:42:18.896677+0000 mon.c (mon.2) 103 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:42:20.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[75164]: audit 2024-09-12T23:42:18.896931+0000 mon.c (mon.2) 104 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:42:20.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[75164]: audit 2024-09-12T23:42:18.897324+0000 mon.c (mon.2) 105 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:42:20.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[75164]: audit 2024-09-12T23:42:18.911115+0000 mon.a (mon.0) 47 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:42:20.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[75164]: audit 2024-09-12T23:42:18.911250+0000 mon.a (mon.0) 48 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:42:20.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[75164]: audit 2024-09-12T23:42:18.911302+0000 mon.c (mon.2) 106 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:42:20.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[75164]: audit 2024-09-12T23:42:18.911893+0000 mon.c (mon.2) 107 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:42:20.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[75164]: audit 2024-09-12T23:42:18.912048+0000 mon.c (mon.2) 108 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:42:20.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[75164]: audit 2024-09-12T23:42:18.914584+0000 mon.c (mon.2) 109 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:42:20.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[32189]: audit 2024-09-12T23:42:18.914795+0000 mon.c (mon.2) 110 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:42:20.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[32189]: audit 2024-09-12T23:42:18.915024+0000 mon.c (mon.2) 111 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:42:20.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[32189]: audit 2024-09-12T23:42:18.915533+0000 mon.b (mon.1) 187 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:42:20.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[32189]: audit 2024-09-12T23:42:18.915655+0000 mon.b (mon.1) 188 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:42:20.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[32189]: audit 2024-09-12T23:42:18.916629+0000 mon.c (mon.2) 112 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:42:20.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[32189]: audit 2024-09-12T23:42:18.916739+0000 mon.c (mon.2) 113 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:42:20.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[32189]: audit 2024-09-12T23:42:18.916931+0000 mon.c (mon.2) 114 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:42:20.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[32189]: audit 2024-09-12T23:42:18.917168+0000 mon.c (mon.2) 115 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:42:20.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[32189]: audit 2024-09-12T23:42:18.917436+0000 mon.c (mon.2) 116 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:42:20.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[75164]: audit 2024-09-12T23:42:18.914795+0000 mon.c (mon.2) 110 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:42:20.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[75164]: audit 2024-09-12T23:42:18.915024+0000 mon.c (mon.2) 111 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:42:20.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[75164]: audit 2024-09-12T23:42:18.915533+0000 mon.b (mon.1) 187 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:42:20.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[75164]: audit 2024-09-12T23:42:18.915655+0000 mon.b (mon.1) 188 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:42:20.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[75164]: audit 2024-09-12T23:42:18.916629+0000 mon.c (mon.2) 112 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:42:20.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[75164]: audit 2024-09-12T23:42:18.916739+0000 mon.c (mon.2) 113 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:42:20.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[75164]: audit 2024-09-12T23:42:18.916931+0000 mon.c (mon.2) 114 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:42:20.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[75164]: audit 2024-09-12T23:42:18.917168+0000 mon.c (mon.2) 115 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:42:20.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:19 smithi136 bash[75164]: audit 2024-09-12T23:42:18.917436+0000 mon.c (mon.2) 116 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:42:20.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:19 smithi181 bash[24776]: cluster 2024-09-12T23:42:18.642920+0000 mgr.y (mgr.24661) 11 : cluster [DBG] pgmap v17: 121 pgs: 121 active+clean; 228 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 2.6 MiB/s wr, 44 op/s 2024-09-12T23:42:20.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:19 smithi181 bash[24776]: cluster 2024-09-12T23:42:18.748975+0000 mon.a (mon.0) 46 : cluster [DBG] osdmap e301: 8 total, 8 up, 8 in 2024-09-12T23:42:20.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:19 smithi181 bash[24776]: audit 2024-09-12T23:42:18.769349+0000 mon.c (mon.2) 95 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:42:20.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:19 smithi181 bash[24776]: audit 2024-09-12T23:42:18.791100+0000 mon.c (mon.2) 96 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-12T23:42:20.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:19 smithi181 bash[24776]: audit 2024-09-12T23:42:18.891341+0000 mon.c (mon.2) 97 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:42:20.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:19 smithi181 bash[24776]: audit 2024-09-12T23:42:18.891563+0000 mon.c (mon.2) 98 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:42:20.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:19 smithi181 bash[24776]: audit 2024-09-12T23:42:18.891781+0000 mon.c (mon.2) 99 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:42:20.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:19 smithi181 bash[24776]: audit 2024-09-12T23:42:18.894316+0000 mon.c (mon.2) 100 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:42:20.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:19 smithi181 bash[24776]: audit 2024-09-12T23:42:18.894541+0000 mon.c (mon.2) 101 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:42:20.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:19 smithi181 bash[24776]: audit 2024-09-12T23:42:18.894789+0000 mon.c (mon.2) 102 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:42:20.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:19 smithi181 bash[24776]: audit 2024-09-12T23:42:18.896677+0000 mon.c (mon.2) 103 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:42:20.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:19 smithi181 bash[24776]: audit 2024-09-12T23:42:18.896931+0000 mon.c (mon.2) 104 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:42:20.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:19 smithi181 bash[24776]: audit 2024-09-12T23:42:18.897324+0000 mon.c (mon.2) 105 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:42:20.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:19 smithi181 bash[24776]: audit 2024-09-12T23:42:18.911115+0000 mon.a (mon.0) 47 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:42:20.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:19 smithi181 bash[24776]: audit 2024-09-12T23:42:18.911250+0000 mon.a (mon.0) 48 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:42:20.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:19 smithi181 bash[24776]: audit 2024-09-12T23:42:18.911302+0000 mon.c (mon.2) 106 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:42:20.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:19 smithi181 bash[24776]: audit 2024-09-12T23:42:18.911893+0000 mon.c (mon.2) 107 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:42:20.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:19 smithi181 bash[24776]: audit 2024-09-12T23:42:18.912048+0000 mon.c (mon.2) 108 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:42:20.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:19 smithi181 bash[24776]: audit 2024-09-12T23:42:18.914584+0000 mon.c (mon.2) 109 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:42:20.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:19 smithi181 bash[24776]: audit 2024-09-12T23:42:18.914795+0000 mon.c (mon.2) 110 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:42:20.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:19 smithi181 bash[24776]: audit 2024-09-12T23:42:18.915024+0000 mon.c (mon.2) 111 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:42:20.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:19 smithi181 bash[24776]: audit 2024-09-12T23:42:18.915533+0000 mon.b (mon.1) 187 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:42:20.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:19 smithi181 bash[24776]: audit 2024-09-12T23:42:18.915655+0000 mon.b (mon.1) 188 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:42:20.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:19 smithi181 bash[24776]: audit 2024-09-12T23:42:18.916629+0000 mon.c (mon.2) 112 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:42:20.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:19 smithi181 bash[24776]: audit 2024-09-12T23:42:18.916739+0000 mon.c (mon.2) 113 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:42:20.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:19 smithi181 bash[24776]: audit 2024-09-12T23:42:18.916931+0000 mon.c (mon.2) 114 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:42:20.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:19 smithi181 bash[24776]: audit 2024-09-12T23:42:18.917168+0000 mon.c (mon.2) 115 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:42:20.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:19 smithi181 bash[24776]: audit 2024-09-12T23:42:18.917436+0000 mon.c (mon.2) 116 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:42:20.776 INFO:tasks.rados.rados.0.smithi136.stdout:1243: done (1 left) 2024-09-12T23:42:20.776 INFO:tasks.rados.rados.0.smithi136.stdout:1244: read oid 18 snap -1 2024-09-12T23:42:20.776 INFO:tasks.rados.rados.0.smithi136.stdout:1244: expect deleted 2024-09-12T23:42:20.776 INFO:tasks.rados.rados.0.smithi136.stdout:1245: delete oid 48 current snap is 119 2024-09-12T23:42:20.776 INFO:tasks.rados.rados.0.smithi136.stdout:1242: finishing rollback tid 0 to smithi13667081-33 2024-09-12T23:42:20.776 INFO:tasks.rados.rados.0.smithi136.stdout:1242: finishing rollback tid 1 to smithi13667081-33 2024-09-12T23:42:20.776 INFO:tasks.rados.rados.0.smithi136.stdout:1242: finishing rollback tid 2 to smithi13667081-33 2024-09-12T23:42:20.776 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 33 v 600 (ObjNum 302 snap 82 seq_num 302) dirty exists 2024-09-12T23:42:20.795 INFO:tasks.rados.rados.0.smithi136.stdout:1242: done (2 left) 2024-09-12T23:42:20.795 INFO:tasks.rados.rados.0.smithi136.stdout:1244: done (1 left) 2024-09-12T23:42:20.795 INFO:tasks.rados.rados.0.smithi136.stdout:1245: done (0 left) 2024-09-12T23:42:20.795 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 43 current snap is 119 2024-09-12T23:42:20.795 INFO:tasks.rados.rados.0.smithi136.stdout:1246: seq_num 407 ranges {1540096=425984,1966080=761856,2727936=704512,3432448=139264} 2024-09-12T23:42:20.807 INFO:tasks.rados.rados.0.smithi136.stdout:1246: writing smithi13667081-43 from 1540096 to 1966080 tid 1 2024-09-12T23:42:20.812 INFO:tasks.rados.rados.0.smithi136.stdout:1246: writing smithi13667081-43 from 1966080 to 2727936 tid 2 2024-09-12T23:42:20.816 INFO:tasks.rados.rados.0.smithi136.stdout:1246: writing smithi13667081-43 from 2727936 to 3432448 tid 3 2024-09-12T23:42:20.818 INFO:tasks.rados.rados.0.smithi136.stdout:1246: writing smithi13667081-43 from 3432448 to 3571712 tid 4 2024-09-12T23:42:20.818 INFO:tasks.rados.rados.0.smithi136.stdout:1247: rollback oid 23 current snap is 119 2024-09-12T23:42:20.818 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 23 to 111 2024-09-12T23:42:20.818 INFO:tasks.rados.rados.0.smithi136.stdout:1246: finishing write tid 1 to smithi13667081-43 2024-09-12T23:42:20.818 INFO:tasks.rados.rados.0.smithi136.stdout:1246: finishing write tid 2 to smithi13667081-43 2024-09-12T23:42:20.818 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 35 current snap is 119 2024-09-12T23:42:20.818 INFO:tasks.rados.rados.0.smithi136.stdout:1248: seq_num 408 ranges {5758976=442368,6201344=647168,6848512=507904,7356416=450560} 2024-09-12T23:42:20.852 INFO:tasks.rados.rados.0.smithi136.stdout:1248: writing smithi13667081-35 from 5758976 to 6201344 tid 1 2024-09-12T23:42:20.856 INFO:tasks.rados.rados.0.smithi136.stdout:1248: writing smithi13667081-35 from 6201344 to 6848512 tid 2 2024-09-12T23:42:20.861 INFO:tasks.rados.rados.0.smithi136.stdout:1248: writing smithi13667081-35 from 6848512 to 7356416 tid 3 2024-09-12T23:42:20.863 INFO:tasks.rados.rados.0.smithi136.stdout:1248: writing smithi13667081-35 from 7356416 to 7806976 tid 4 2024-09-12T23:42:20.863 INFO:tasks.rados.rados.0.smithi136.stdout:1249: delete oid 36 current snap is 119 2024-09-12T23:42:20.863 INFO:tasks.rados.rados.0.smithi136.stdout:1247: finishing rollback tid 0 to smithi13667081-23 2024-09-12T23:42:20.863 INFO:tasks.rados.rados.0.smithi136.stdout:1247: finishing rollback tid 1 to smithi13667081-23 2024-09-12T23:42:20.863 INFO:tasks.rados.rados.0.smithi136.stdout:1247: finishing rollback tid 2 to smithi13667081-23 2024-09-12T23:42:20.863 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 23 v 838 (ObjNum 387 snap 111 seq_num 387) dirty exists 2024-09-12T23:42:20.863 INFO:tasks.rados.rados.0.smithi136.stdout:1246: finishing write tid 3 to smithi13667081-43 2024-09-12T23:42:20.864 INFO:tasks.rados.rados.0.smithi136.stdout:1246: finishing write tid 4 to smithi13667081-43 2024-09-12T23:42:20.864 INFO:tasks.rados.rados.0.smithi136.stdout:1246: finishing write tid 5 to smithi13667081-43 2024-09-12T23:42:20.864 INFO:tasks.rados.rados.0.smithi136.stdout:1246: finishing write tid 6 to smithi13667081-43 2024-09-12T23:42:20.864 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 43 v 709 (ObjNum 407 snap 119 seq_num 407) dirty exists 2024-09-12T23:42:20.864 INFO:tasks.rados.rados.0.smithi136.stdout:1246: left oid 43 (ObjNum 407 snap 119 seq_num 407) 2024-09-12T23:42:20.864 INFO:tasks.rados.rados.0.smithi136.stdout:1248: finishing write tid 1 to smithi13667081-35 2024-09-12T23:42:20.864 INFO:tasks.rados.rados.0.smithi136.stdout:1248: finishing write tid 2 to smithi13667081-35 2024-09-12T23:42:20.865 INFO:tasks.rados.rados.0.smithi136.stdout:1248: finishing write tid 3 to smithi13667081-35 2024-09-12T23:42:20.866 INFO:tasks.rados.rados.0.smithi136.stdout:1248: finishing write tid 4 to smithi13667081-35 2024-09-12T23:42:20.867 INFO:tasks.rados.rados.0.smithi136.stdout:1248: finishing write tid 5 to smithi13667081-35 2024-09-12T23:42:20.867 INFO:tasks.rados.rados.0.smithi136.stdout:1246: done (3 left) 2024-09-12T23:42:20.867 INFO:tasks.rados.rados.0.smithi136.stdout:1247: done (2 left) 2024-09-12T23:42:20.867 INFO:tasks.rados.rados.0.smithi136.stdout:1249: done (1 left) 2024-09-12T23:42:20.867 INFO:tasks.rados.rados.0.smithi136.stdout:1250: rollback oid 6 current snap is 119 2024-09-12T23:42:20.867 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 6 to 109 2024-09-12T23:42:20.867 INFO:tasks.rados.rados.0.smithi136.stdout:1251: snap_remove snap 117 2024-09-12T23:42:21.282 INFO:tasks.rados.rados.0.smithi136.stdout:1251: done (2 left) 2024-09-12T23:42:21.282 INFO:tasks.rados.rados.0.smithi136.stdout:1252: read oid 7 snap -1 2024-09-12T23:42:21.282 INFO:tasks.rados.rados.0.smithi136.stdout:1252: expect (ObjNum 350 snap 95 seq_num 350) 2024-09-12T23:42:21.282 INFO:tasks.rados.rados.0.smithi136.stdout:1253: delete oid 36 current snap is 119 2024-09-12T23:42:21.282 INFO:tasks.rados.rados.0.smithi136.stdout:1248: finishing write tid 6 to smithi13667081-35 2024-09-12T23:42:21.282 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 35 v 844 (ObjNum 408 snap 119 seq_num 408) dirty exists 2024-09-12T23:42:21.282 INFO:tasks.rados.rados.0.smithi136.stdout:1248: left oid 35 (ObjNum 408 snap 119 seq_num 408) 2024-09-12T23:42:21.282 INFO:tasks.rados.rados.0.smithi136.stdout:1250: finishing rollback tid 1 to smithi13667081-6 2024-09-12T23:42:21.283 INFO:tasks.rados.rados.0.smithi136.stdout:1250: finishing rollback tid 2 to smithi13667081-6 2024-09-12T23:42:21.283 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 6 v 288 (ObjNum 254 snap 69 seq_num 254) dirty exists 2024-09-12T23:42:21.318 INFO:tasks.rados.rados.0.smithi136.stdout:1248: done (3 left) 2024-09-12T23:42:21.319 INFO:tasks.rados.rados.0.smithi136.stdout:1250: done (2 left) 2024-09-12T23:42:21.319 INFO:tasks.rados.rados.0.smithi136.stdout:1253: done (1 left) 2024-09-12T23:42:21.319 INFO:tasks.rados.rados.0.smithi136.stdout:1254: rollback oid 16 current snap is 119 2024-09-12T23:42:21.319 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 16 to 87 2024-09-12T23:42:21.319 INFO:tasks.rados.rados.0.smithi136.stdout:1255: rmattr oid 43 current snap is 119 2024-09-12T23:42:21.319 INFO:tasks.rados.rados.0.smithi136.stdout:1255: done (2 left) 2024-09-12T23:42:21.319 INFO:tasks.rados.rados.0.smithi136.stdout:1256: delete oid 8 current snap is 119 2024-09-12T23:42:21.323 INFO:tasks.rados.rados.0.smithi136.stdout:1256: done (2 left) 2024-09-12T23:42:21.324 INFO:tasks.rados.rados.0.smithi136.stdout:1257: read oid 41 snap -1 2024-09-12T23:42:21.324 INFO:tasks.rados.rados.0.smithi136.stdout:1257: expect (ObjNum 262 snap 71 seq_num 262) 2024-09-12T23:42:21.324 INFO:tasks.rados.rados.0.smithi136.stdout:1258: snap_create 2024-09-12T23:42:21.328 INFO:tasks.rados.rados.0.smithi136.stdout:1254: finishing rollback tid 0 to smithi13667081-16 2024-09-12T23:42:21.328 INFO:tasks.rados.rados.0.smithi136.stdout:1254: finishing rollback tid 1 to smithi13667081-16 2024-09-12T23:42:21.329 INFO:tasks.rados.rados.0.smithi136.stdout:1254: finishing rollback tid 2 to smithi13667081-16 2024-09-12T23:42:21.329 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 16 v 719 (ObjNum 228 snap 56 seq_num 228) dirty exists 2024-09-12T23:42:21.335 INFO:tasks.rados.rados.0.smithi136.stdout:1252: expect (ObjNum 350 snap 95 seq_num 350) 2024-09-12T23:42:21.529 INFO:tasks.rados.rados.0.smithi136.stdout:1257: expect (ObjNum 262 snap 71 seq_num 262) 2024-09-12T23:42:22.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:21 smithi136 bash[75164]: cluster 2024-09-12T23:42:20.644382+0000 mgr.y (mgr.24661) 12 : cluster [DBG] pgmap v19: 121 pgs: 121 active+clean; 239 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 2.2 MiB/s wr, 19 op/s 2024-09-12T23:42:22.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:21 smithi136 bash[75164]: cluster 2024-09-12T23:42:20.769272+0000 mon.a (mon.0) 49 : cluster [DBG] mgrmap e42: y(active, since 12s), standbys: x 2024-09-12T23:42:22.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:21 smithi136 bash[75164]: cluster 2024-09-12T23:42:20.769486+0000 mon.a (mon.0) 50 : cluster [DBG] osdmap e302: 8 total, 8 up, 8 in 2024-09-12T23:42:22.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:21 smithi136 bash[75164]: cluster 2024-09-12T23:42:21.283843+0000 mon.a (mon.0) 51 : cluster [DBG] osdmap e303: 8 total, 8 up, 8 in 2024-09-12T23:42:22.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:21 smithi136 bash[32189]: cluster 2024-09-12T23:42:20.644382+0000 mgr.y (mgr.24661) 12 : cluster [DBG] pgmap v19: 121 pgs: 121 active+clean; 239 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 2.2 MiB/s wr, 19 op/s 2024-09-12T23:42:22.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:21 smithi136 bash[32189]: cluster 2024-09-12T23:42:20.769272+0000 mon.a (mon.0) 49 : cluster [DBG] mgrmap e42: y(active, since 12s), standbys: x 2024-09-12T23:42:22.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:21 smithi136 bash[32189]: cluster 2024-09-12T23:42:20.769486+0000 mon.a (mon.0) 50 : cluster [DBG] osdmap e302: 8 total, 8 up, 8 in 2024-09-12T23:42:22.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:21 smithi136 bash[32189]: cluster 2024-09-12T23:42:21.283843+0000 mon.a (mon.0) 51 : cluster [DBG] osdmap e303: 8 total, 8 up, 8 in 2024-09-12T23:42:22.088 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:21 smithi181 bash[24776]: cluster 2024-09-12T23:42:20.644382+0000 mgr.y (mgr.24661) 12 : cluster [DBG] pgmap v19: 121 pgs: 121 active+clean; 239 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 2.2 MiB/s wr, 19 op/s 2024-09-12T23:42:22.088 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:21 smithi181 bash[24776]: cluster 2024-09-12T23:42:20.769272+0000 mon.a (mon.0) 49 : cluster [DBG] mgrmap e42: y(active, since 12s), standbys: x 2024-09-12T23:42:22.089 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:21 smithi181 bash[24776]: cluster 2024-09-12T23:42:20.769486+0000 mon.a (mon.0) 50 : cluster [DBG] osdmap e302: 8 total, 8 up, 8 in 2024-09-12T23:42:22.089 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:21 smithi181 bash[24776]: cluster 2024-09-12T23:42:21.283843+0000 mon.a (mon.0) 51 : cluster [DBG] osdmap e303: 8 total, 8 up, 8 in 2024-09-12T23:42:22.287 INFO:tasks.rados.rados.0.smithi136.stdout:1252: done (3 left) 2024-09-12T23:42:22.287 INFO:tasks.rados.rados.0.smithi136.stdout:1254: done (2 left) 2024-09-12T23:42:22.287 INFO:tasks.rados.rados.0.smithi136.stdout:1257: done (1 left) 2024-09-12T23:42:22.287 INFO:tasks.rados.rados.0.smithi136.stdout:1258: done (0 left) 2024-09-12T23:42:22.287 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 40 current snap is 120 2024-09-12T23:42:22.287 INFO:tasks.rados.rados.0.smithi136.stdout:1259: seq_num 412 ranges {2957312=507904,3465216=630784,4096000=491520,4587520=483328} 2024-09-12T23:42:22.312 INFO:tasks.rados.rados.0.smithi136.stdout:1259: writing smithi13667081-40 from 2957312 to 3465216 tid 1 2024-09-12T23:42:22.317 INFO:tasks.rados.rados.0.smithi136.stdout:1259: writing smithi13667081-40 from 3465216 to 4096000 tid 2 2024-09-12T23:42:22.322 INFO:tasks.rados.rados.0.smithi136.stdout:1259: writing smithi13667081-40 from 4096000 to 4587520 tid 3 2024-09-12T23:42:22.327 INFO:tasks.rados.rados.0.smithi136.stdout:1259: writing smithi13667081-40 from 4587520 to 5070848 tid 4 2024-09-12T23:42:22.327 INFO:tasks.rados.rados.0.smithi136.stdout:1260: delete oid 2 current snap is 120 2024-09-12T23:42:22.327 INFO:tasks.rados.rados.0.smithi136.stdout:1259: finishing write tid 1 to smithi13667081-40 2024-09-12T23:42:22.330 INFO:tasks.rados.rados.0.smithi136.stdout:1259: finishing write tid 2 to smithi13667081-40 2024-09-12T23:42:22.330 INFO:tasks.rados.rados.0.smithi136.stdout:1259: finishing write tid 3 to smithi13667081-40 2024-09-12T23:42:22.330 INFO:tasks.rados.rados.0.smithi136.stdout:1260: done (1 left) 2024-09-12T23:42:22.331 INFO:tasks.rados.rados.0.smithi136.stdout:1261: snap_remove snap 119 2024-09-12T23:42:22.344 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:42:22 smithi181 bash[60471]: ts=2024-09-12T23:42:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:42:23.296 INFO:tasks.rados.rados.0.smithi136.stdout:1261: done (1 left) 2024-09-12T23:42:23.334 INFO:tasks.rados.rados.0.smithi136.stdout:1262: copy_from oid 43 from oid 39 current snap is 120 2024-09-12T23:42:23.334 INFO:tasks.rados.rados.0.smithi136.stdout:1259: finishing write tid 4 to smithi13667081-40 2024-09-12T23:42:23.334 INFO:tasks.rados.rados.0.smithi136.stdout:1259: finishing write tid 5 to smithi13667081-40 2024-09-12T23:42:23.335 INFO:tasks.rados.rados.0.smithi136.stdout:1259: finishing write tid 6 to smithi13667081-40 2024-09-12T23:42:23.335 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 40 v 854 (ObjNum 412 snap 120 seq_num 412) dirty exists 2024-09-12T23:42:23.335 INFO:tasks.rados.rados.0.smithi136.stdout:1259: left oid 40 (ObjNum 412 snap 120 seq_num 412) 2024-09-12T23:42:23.335 INFO:tasks.rados.rados.0.smithi136.stdout:1259: done (1 left) 2024-09-12T23:42:23.335 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 49 current snap is 120 2024-09-12T23:42:23.335 INFO:tasks.rados.rados.0.smithi136.stdout:1263: seq_num 414 ranges {0=540672,540672=802816,1343488=458752,1802240=344064} 2024-09-12T23:42:23.336 INFO:tasks.rados.rados.0.smithi136.stdout:1263: writing smithi13667081-49 from 0 to 540672 tid 1 2024-09-12T23:42:23.336 INFO:tasks.rados.rados.0.smithi136.stdout:1263: writing smithi13667081-49 from 540672 to 1343488 tid 2 2024-09-12T23:42:23.336 INFO:tasks.rados.rados.0.smithi136.stdout:1263: writing smithi13667081-49 from 1343488 to 1802240 tid 3 2024-09-12T23:42:23.336 INFO:tasks.rados.rados.0.smithi136.stdout:1263: writing smithi13667081-49 from 1802240 to 2146304 tid 4 2024-09-12T23:42:23.337 INFO:tasks.rados.rados.0.smithi136.stdout:1264: delete oid 26 current snap is 120 2024-09-12T23:42:23.337 INFO:tasks.rados.rados.0.smithi136.stdout:1264: done (2 left) 2024-09-12T23:42:23.337 INFO:tasks.rados.rados.0.smithi136.stdout:1265: read oid 32 snap 116 2024-09-12T23:42:23.337 INFO:tasks.rados.rados.0.smithi136.stdout:1265: expect (ObjNum 353 snap 96 seq_num 353) 2024-09-12T23:42:23.337 INFO:tasks.rados.rados.0.smithi136.stdout:1266: rollback oid 35 current snap is 120 2024-09-12T23:42:23.337 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 35 to 87 2024-09-12T23:42:23.337 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 24 current snap is 120 2024-09-12T23:42:23.337 INFO:tasks.rados.rados.0.smithi136.stdout:1267: seq_num 416 ranges {5439488=573440,6012928=573440,6586368=475136,7061504=557056} 2024-09-12T23:42:23.354 INFO:tasks.rados.rados.0.smithi136.stdout:1267: writing smithi13667081-24 from 5439488 to 6012928 tid 1 2024-09-12T23:42:23.359 INFO:tasks.rados.rados.0.smithi136.stdout:1267: writing smithi13667081-24 from 6012928 to 6586368 tid 2 2024-09-12T23:42:23.362 INFO:tasks.rados.rados.0.smithi136.stdout:1267: writing smithi13667081-24 from 6586368 to 7061504 tid 3 2024-09-12T23:42:23.366 INFO:tasks.rados.rados.0.smithi136.stdout:1267: writing smithi13667081-24 from 7061504 to 7618560 tid 4 2024-09-12T23:42:23.367 INFO:tasks.rados.rados.0.smithi136.stdout:1268: read oid 2 snap -1 2024-09-12T23:42:23.367 INFO:tasks.rados.rados.0.smithi136.stdout:1268: expect deleted 2024-09-12T23:42:23.367 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 25 current snap is 120 2024-09-12T23:42:23.367 INFO:tasks.rados.rados.0.smithi136.stdout:1263: finishing write tid 1 to smithi13667081-49 2024-09-12T23:42:23.367 INFO:tasks.rados.rados.0.smithi136.stdout:1263: finishing write tid 2 to smithi13667081-49 2024-09-12T23:42:23.367 INFO:tasks.rados.rados.0.smithi136.stdout:1263: finishing write tid 3 to smithi13667081-49 2024-09-12T23:42:23.367 INFO:tasks.rados.rados.0.smithi136.stdout:1263: finishing write tid 4 to smithi13667081-49 2024-09-12T23:42:23.367 INFO:tasks.rados.rados.0.smithi136.stdout:1263: finishing write tid 5 to smithi13667081-49 2024-09-12T23:42:23.367 INFO:tasks.rados.rados.0.smithi136.stdout:1269: seq_num 417 ranges {1409024=589824,1998848=458752,2457600=679936,3137536=466944} 2024-09-12T23:42:23.377 INFO:tasks.rados.rados.0.smithi136.stdout:1269: writing smithi13667081-25 from 1409024 to 1998848 tid 1 2024-09-12T23:42:23.379 INFO:tasks.rados.rados.0.smithi136.stdout:1269: writing smithi13667081-25 from 1998848 to 2457600 tid 2 2024-09-12T23:42:23.383 INFO:tasks.rados.rados.0.smithi136.stdout:1269: writing smithi13667081-25 from 2457600 to 3137536 tid 3 2024-09-12T23:42:23.387 INFO:tasks.rados.rados.0.smithi136.stdout:1269: writing smithi13667081-25 from 3137536 to 3604480 tid 4 2024-09-12T23:42:23.387 INFO:tasks.rados.rados.0.smithi136.stdout:1270: read oid 22 snap -1 2024-09-12T23:42:23.387 INFO:tasks.rados.rados.0.smithi136.stdout:1270: expect deleted 2024-09-12T23:42:23.387 INFO:tasks.rados.rados.0.smithi136.stdout:1271: snap_create 2024-09-12T23:42:23.387 INFO:tasks.rados.rados.0.smithi136.stdout:1266: finishing rollback tid 0 to smithi13667081-35 2024-09-12T23:42:23.387 INFO:tasks.rados.rados.0.smithi136.stdout:1266: finishing rollback tid 1 to smithi13667081-35 2024-09-12T23:42:23.387 INFO:tasks.rados.rados.0.smithi136.stdout:1263: finishing write tid 6 to smithi13667081-49 2024-09-12T23:42:23.387 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 49 v 619 (ObjNum 414 snap 120 seq_num 414) dirty exists 2024-09-12T23:42:23.387 INFO:tasks.rados.rados.0.smithi136.stdout:1263: left oid 49 (ObjNum 414 snap 120 seq_num 414) 2024-09-12T23:42:23.388 INFO:tasks.rados.rados.0.smithi136.stdout:1266: finishing rollback tid 2 to smithi13667081-35 2024-09-12T23:42:23.388 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 35 v 859 (ObjNum 262 snap 71 seq_num 262) dirty exists 2024-09-12T23:42:23.388 INFO:tasks.rados.rados.0.smithi136.stdout:1265: expect (ObjNum 353 snap 96 seq_num 353) 2024-09-12T23:42:23.549 INFO:tasks.rados.rados.0.smithi136.stdout:1262: finishing copy_from racing read to smithi13667081-43 2024-09-12T23:42:23.549 INFO:tasks.rados.rados.0.smithi136.stdout:1262: finishing copy_from to smithi13667081-43 2024-09-12T23:42:23.549 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 43 v 199 (ObjNum 399 snap 116 seq_num 399) dirty exists 2024-09-12T23:42:23.549 INFO:tasks.rados.rados.0.smithi136.stdout:1267: finishing write tid 1 to smithi13667081-24 2024-09-12T23:42:23.549 INFO:tasks.rados.rados.0.smithi136.stdout:1267: finishing write tid 2 to smithi13667081-24 2024-09-12T23:42:23.549 INFO:tasks.rados.rados.0.smithi136.stdout:1267: finishing write tid 3 to smithi13667081-24 2024-09-12T23:42:23.549 INFO:tasks.rados.rados.0.smithi136.stdout:1267: finishing write tid 4 to smithi13667081-24 2024-09-12T23:42:23.549 INFO:tasks.rados.rados.0.smithi136.stdout:1267: finishing write tid 5 to smithi13667081-24 2024-09-12T23:42:23.549 INFO:tasks.rados.rados.0.smithi136.stdout:1267: finishing write tid 6 to smithi13667081-24 2024-09-12T23:42:23.550 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 24 v 402 (ObjNum 416 snap 120 seq_num 416) dirty exists 2024-09-12T23:42:23.550 INFO:tasks.rados.rados.0.smithi136.stdout:1267: left oid 24 (ObjNum 416 snap 120 seq_num 416) 2024-09-12T23:42:23.550 INFO:tasks.rados.rados.0.smithi136.stdout:1269: finishing write tid 1 to smithi13667081-25 2024-09-12T23:42:23.550 INFO:tasks.rados.rados.0.smithi136.stdout:1269: finishing write tid 2 to smithi13667081-25 2024-09-12T23:42:23.550 INFO:tasks.rados.rados.0.smithi136.stdout:1269: finishing write tid 3 to smithi13667081-25 2024-09-12T23:42:23.550 INFO:tasks.rados.rados.0.smithi136.stdout:1269: finishing write tid 4 to smithi13667081-25 2024-09-12T23:42:23.550 INFO:tasks.rados.rados.0.smithi136.stdout:1269: finishing write tid 5 to smithi13667081-25 2024-09-12T23:42:23.550 INFO:tasks.rados.rados.0.smithi136.stdout:1269: finishing write tid 6 to smithi13667081-25 2024-09-12T23:42:23.550 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 25 v 211 (ObjNum 417 snap 120 seq_num 417) dirty exists 2024-09-12T23:42:23.550 INFO:tasks.rados.rados.0.smithi136.stdout:1269: left oid 25 (ObjNum 417 snap 120 seq_num 417) 2024-09-12T23:42:23.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:23 smithi181 bash[24776]: cluster 2024-09-12T23:42:22.287331+0000 mon.a (mon.0) 52 : cluster [DBG] osdmap e304: 8 total, 8 up, 8 in 2024-09-12T23:42:23.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:23 smithi181 bash[24776]: cluster 2024-09-12T23:42:22.645160+0000 mgr.y (mgr.24661) 13 : cluster [DBG] pgmap v23: 121 pgs: 121 active+clean; 239 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 2.8 MiB/s wr, 24 op/s 2024-09-12T23:42:23.594 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:42:23 smithi181 bash[60471]: ts=2024-09-12T23:42:23.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi136\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:42:23.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:23 smithi136 bash[75164]: cluster 2024-09-12T23:42:22.287331+0000 mon.a (mon.0) 52 : cluster [DBG] osdmap e304: 8 total, 8 up, 8 in 2024-09-12T23:42:23.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:23 smithi136 bash[75164]: cluster 2024-09-12T23:42:22.645160+0000 mgr.y (mgr.24661) 13 : cluster [DBG] pgmap v23: 121 pgs: 121 active+clean; 239 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 2.8 MiB/s wr, 24 op/s 2024-09-12T23:42:23.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:23 smithi136 bash[32189]: cluster 2024-09-12T23:42:22.287331+0000 mon.a (mon.0) 52 : cluster [DBG] osdmap e304: 8 total, 8 up, 8 in 2024-09-12T23:42:23.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:23 smithi136 bash[32189]: cluster 2024-09-12T23:42:22.645160+0000 mgr.y (mgr.24661) 13 : cluster [DBG] pgmap v23: 121 pgs: 121 active+clean; 239 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 2.8 MiB/s wr, 24 op/s 2024-09-12T23:42:24.303 INFO:tasks.rados.rados.0.smithi136.stdout:1262: done (8 left) 2024-09-12T23:42:24.303 INFO:tasks.rados.rados.0.smithi136.stdout:1263: done (7 left) 2024-09-12T23:42:24.303 INFO:tasks.rados.rados.0.smithi136.stdout:1265: done (6 left) 2024-09-12T23:42:24.304 INFO:tasks.rados.rados.0.smithi136.stdout:1266: done (5 left) 2024-09-12T23:42:24.304 INFO:tasks.rados.rados.0.smithi136.stdout:1267: done (4 left) 2024-09-12T23:42:24.304 INFO:tasks.rados.rados.0.smithi136.stdout:1268: done (3 left) 2024-09-12T23:42:24.304 INFO:tasks.rados.rados.0.smithi136.stdout:1269: done (2 left) 2024-09-12T23:42:24.304 INFO:tasks.rados.rados.0.smithi136.stdout:1270: done (1 left) 2024-09-12T23:42:24.304 INFO:tasks.rados.rados.0.smithi136.stdout:1271: done (0 left) 2024-09-12T23:42:24.304 INFO:tasks.rados.rados.0.smithi136.stdout:1272: read oid 18 snap -1 2024-09-12T23:42:24.304 INFO:tasks.rados.rados.0.smithi136.stdout:1272: expect deleted 2024-09-12T23:42:24.304 INFO:tasks.rados.rados.0.smithi136.stdout:1273: setattr oid 1 current snap is 121 2024-09-12T23:42:24.304 INFO:tasks.rados.rados.0.smithi136.stdout:1274: snap_create 2024-09-12T23:42:24.309 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 1 v 726 (ObjNum 350 snap 95 seq_num 350) dirty exists 2024-09-12T23:42:24.552 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:24 smithi181 bash[24776]: cluster 2024-09-12T23:42:23.296627+0000 mon.a (mon.0) 53 : cluster [DBG] osdmap e305: 8 total, 8 up, 8 in 2024-09-12T23:42:24.552 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:24 smithi181 bash[24776]: audit 2024-09-12T23:42:23.722004+0000 mon.c (mon.2) 117 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:42:24.553 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:42:24 smithi181 bash[60471]: ts=2024-09-12T23:42:24.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:42:24.553 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:42:24 smithi181 bash[60471]: ts=2024-09-12T23:42:24.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:42:24.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:24 smithi136 bash[32189]: cluster 2024-09-12T23:42:23.296627+0000 mon.a (mon.0) 53 : cluster [DBG] osdmap e305: 8 total, 8 up, 8 in 2024-09-12T23:42:24.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:24 smithi136 bash[32189]: audit 2024-09-12T23:42:23.722004+0000 mon.c (mon.2) 117 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:42:24.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:24 smithi136 bash[75164]: cluster 2024-09-12T23:42:23.296627+0000 mon.a (mon.0) 53 : cluster [DBG] osdmap e305: 8 total, 8 up, 8 in 2024-09-12T23:42:24.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:24 smithi136 bash[75164]: audit 2024-09-12T23:42:23.722004+0000 mon.c (mon.2) 117 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:42:25.317 INFO:tasks.rados.rados.0.smithi136.stdout:1272: done (2 left) 2024-09-12T23:42:25.317 INFO:tasks.rados.rados.0.smithi136.stdout:1273: done (1 left) 2024-09-12T23:42:25.317 INFO:tasks.rados.rados.0.smithi136.stdout:1274: done (0 left) 2024-09-12T23:42:25.317 INFO:tasks.rados.rados.0.smithi136.stdout:1275: setattr oid 40 current snap is 122 2024-09-12T23:42:25.319 INFO:tasks.rados.rados.0.smithi136.stdout:1276: read oid 8 snap -1 2024-09-12T23:42:25.319 INFO:tasks.rados.rados.0.smithi136.stdout:1276: expect deleted 2024-09-12T23:42:25.319 INFO:tasks.rados.rados.0.smithi136.stdout:1277: snap_remove snap 111 2024-09-12T23:42:25.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:25 smithi181 bash[24776]: cluster 2024-09-12T23:42:24.306126+0000 mon.a (mon.0) 54 : cluster [DBG] osdmap e306: 8 total, 8 up, 8 in 2024-09-12T23:42:25.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:25 smithi181 bash[24776]: cluster 2024-09-12T23:42:24.646268+0000 mgr.y (mgr.24661) 14 : cluster [DBG] pgmap v26: 121 pgs: 121 active+clean; 256 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 3.9 MiB/s wr, 17 op/s 2024-09-12T23:42:25.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:25 smithi181 bash[24776]: audit 2024-09-12T23:42:25.243739+0000 mon.a (mon.0) 55 : audit [INF] from='mgr.24661 ' entity='mgr.y' 2024-09-12T23:42:25.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:25 smithi181 bash[24776]: audit 2024-09-12T23:42:25.257286+0000 mon.a (mon.0) 56 : audit [INF] from='mgr.24661 ' entity='mgr.y' 2024-09-12T23:42:25.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:25 smithi181 bash[24776]: audit 2024-09-12T23:42:25.261710+0000 mon.c (mon.2) 118 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:42:25.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:25 smithi181 bash[24776]: audit 2024-09-12T23:42:25.263724+0000 mon.c (mon.2) 119 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:42:25.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:25 smithi136 bash[75164]: cluster 2024-09-12T23:42:24.306126+0000 mon.a (mon.0) 54 : cluster [DBG] osdmap e306: 8 total, 8 up, 8 in 2024-09-12T23:42:25.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:25 smithi136 bash[75164]: cluster 2024-09-12T23:42:24.646268+0000 mgr.y (mgr.24661) 14 : cluster [DBG] pgmap v26: 121 pgs: 121 active+clean; 256 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 3.9 MiB/s wr, 17 op/s 2024-09-12T23:42:25.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:25 smithi136 bash[75164]: audit 2024-09-12T23:42:25.243739+0000 mon.a (mon.0) 55 : audit [INF] from='mgr.24661 ' entity='mgr.y' 2024-09-12T23:42:25.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:25 smithi136 bash[75164]: audit 2024-09-12T23:42:25.257286+0000 mon.a (mon.0) 56 : audit [INF] from='mgr.24661 ' entity='mgr.y' 2024-09-12T23:42:25.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:25 smithi136 bash[75164]: audit 2024-09-12T23:42:25.261710+0000 mon.c (mon.2) 118 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:42:25.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:25 smithi136 bash[75164]: audit 2024-09-12T23:42:25.263724+0000 mon.c (mon.2) 119 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:42:25.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:25 smithi136 bash[32189]: cluster 2024-09-12T23:42:24.306126+0000 mon.a (mon.0) 54 : cluster [DBG] osdmap e306: 8 total, 8 up, 8 in 2024-09-12T23:42:25.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:25 smithi136 bash[32189]: cluster 2024-09-12T23:42:24.646268+0000 mgr.y (mgr.24661) 14 : cluster [DBG] pgmap v26: 121 pgs: 121 active+clean; 256 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 3.9 MiB/s wr, 17 op/s 2024-09-12T23:42:25.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:25 smithi136 bash[32189]: audit 2024-09-12T23:42:25.243739+0000 mon.a (mon.0) 55 : audit [INF] from='mgr.24661 ' entity='mgr.y' 2024-09-12T23:42:25.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:25 smithi136 bash[32189]: audit 2024-09-12T23:42:25.257286+0000 mon.a (mon.0) 56 : audit [INF] from='mgr.24661 ' entity='mgr.y' 2024-09-12T23:42:25.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:25 smithi136 bash[32189]: audit 2024-09-12T23:42:25.261710+0000 mon.c (mon.2) 118 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:42:25.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:25 smithi136 bash[32189]: audit 2024-09-12T23:42:25.263724+0000 mon.c (mon.2) 119 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:42:26.325 INFO:tasks.rados.rados.0.smithi136.stdout:1277: done (2 left) 2024-09-12T23:42:26.326 INFO:tasks.rados.rados.0.smithi136.stdout:1278: read oid 6 snap -1 2024-09-12T23:42:26.326 INFO:tasks.rados.rados.0.smithi136.stdout:1278: expect (ObjNum 254 snap 69 seq_num 254) 2024-09-12T23:42:26.326 INFO:tasks.rados.rados.0.smithi136.stdout:1279: snap_remove snap 110 2024-09-12T23:42:26.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:26 smithi181 bash[24776]: cephadm 2024-09-12T23:42:25.266297+0000 mgr.y (mgr.24661) 15 : cephadm [INF] Updating smithi136:/etc/ceph/ceph.conf 2024-09-12T23:42:26.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:26 smithi181 bash[24776]: cephadm 2024-09-12T23:42:25.267056+0000 mgr.y (mgr.24661) 16 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.conf 2024-09-12T23:42:26.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:26 smithi181 bash[24776]: cluster 2024-09-12T23:42:25.320947+0000 mon.a (mon.0) 57 : cluster [DBG] osdmap e307: 8 total, 8 up, 8 in 2024-09-12T23:42:26.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:26 smithi181 bash[24776]: cephadm 2024-09-12T23:42:25.457217+0000 mgr.y (mgr.24661) 17 : cephadm [INF] Updating smithi136:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-12T23:42:26.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:26 smithi181 bash[24776]: cephadm 2024-09-12T23:42:25.460889+0000 mgr.y (mgr.24661) 18 : cephadm [INF] Updating smithi181:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-12T23:42:26.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:26 smithi181 bash[24776]: cephadm 2024-09-12T23:42:25.578856+0000 mgr.y (mgr.24661) 19 : cephadm [INF] Updating smithi136:/etc/ceph/ceph.client.admin.keyring 2024-09-12T23:42:26.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:26 smithi181 bash[24776]: cephadm 2024-09-12T23:42:25.594481+0000 mgr.y (mgr.24661) 20 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2024-09-12T23:42:26.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:26 smithi181 bash[24776]: cephadm 2024-09-12T23:42:25.709866+0000 mgr.y (mgr.24661) 21 : cephadm [INF] Updating smithi136:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-12T23:42:26.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:26 smithi181 bash[24776]: cephadm 2024-09-12T23:42:25.731189+0000 mgr.y (mgr.24661) 22 : cephadm [INF] Updating smithi181:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-12T23:42:26.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:26 smithi181 bash[24776]: audit 2024-09-12T23:42:25.846336+0000 mon.a (mon.0) 58 : audit [INF] from='mgr.24661 ' entity='mgr.y' 2024-09-12T23:42:26.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:26 smithi181 bash[24776]: audit 2024-09-12T23:42:25.856540+0000 mon.a (mon.0) 59 : audit [INF] from='mgr.24661 ' entity='mgr.y' 2024-09-12T23:42:26.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:26 smithi181 bash[24776]: audit 2024-09-12T23:42:25.871262+0000 mon.a (mon.0) 60 : audit [INF] from='mgr.24661 ' entity='mgr.y' 2024-09-12T23:42:26.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:26 smithi181 bash[24776]: audit 2024-09-12T23:42:25.881081+0000 mon.a (mon.0) 61 : audit [INF] from='mgr.24661 ' entity='mgr.y' 2024-09-12T23:42:26.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:26 smithi181 bash[24776]: audit 2024-09-12T23:42:25.889778+0000 mon.a (mon.0) 62 : audit [INF] from='mgr.24661 ' entity='mgr.y' 2024-09-12T23:42:26.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:26 smithi181 bash[24776]: audit 2024-09-12T23:42:25.970368+0000 mon.c (mon.2) 120 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:42:26.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:26 smithi181 bash[24776]: audit 2024-09-12T23:42:25.974042+0000 mon.c (mon.2) 121 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:42:26.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:26 smithi181 bash[24776]: audit 2024-09-12T23:42:25.975475+0000 mon.c (mon.2) 122 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-12T23:42:26.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:26 smithi181 bash[24776]: audit 2024-09-12T23:42:25.976283+0000 mon.c (mon.2) 123 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-09-12T23:42:26.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:26 smithi136 bash[75164]: cephadm 2024-09-12T23:42:25.266297+0000 mgr.y (mgr.24661) 15 : cephadm [INF] Updating smithi136:/etc/ceph/ceph.conf 2024-09-12T23:42:26.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:26 smithi136 bash[75164]: cephadm 2024-09-12T23:42:25.267056+0000 mgr.y (mgr.24661) 16 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.conf 2024-09-12T23:42:26.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:26 smithi136 bash[75164]: cluster 2024-09-12T23:42:25.320947+0000 mon.a (mon.0) 57 : cluster [DBG] osdmap e307: 8 total, 8 up, 8 in 2024-09-12T23:42:26.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:26 smithi136 bash[75164]: cephadm 2024-09-12T23:42:25.457217+0000 mgr.y (mgr.24661) 17 : cephadm [INF] Updating smithi136:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-12T23:42:26.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:26 smithi136 bash[75164]: cephadm 2024-09-12T23:42:25.460889+0000 mgr.y (mgr.24661) 18 : cephadm [INF] Updating smithi181:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-12T23:42:26.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:26 smithi136 bash[75164]: cephadm 2024-09-12T23:42:25.578856+0000 mgr.y (mgr.24661) 19 : cephadm [INF] Updating smithi136:/etc/ceph/ceph.client.admin.keyring 2024-09-12T23:42:26.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:26 smithi136 bash[75164]: cephadm 2024-09-12T23:42:25.594481+0000 mgr.y (mgr.24661) 20 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2024-09-12T23:42:26.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:26 smithi136 bash[75164]: cephadm 2024-09-12T23:42:25.709866+0000 mgr.y (mgr.24661) 21 : cephadm [INF] Updating smithi136:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-12T23:42:26.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:26 smithi136 bash[75164]: cephadm 2024-09-12T23:42:25.731189+0000 mgr.y (mgr.24661) 22 : cephadm [INF] Updating smithi181:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-12T23:42:26.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:26 smithi136 bash[75164]: audit 2024-09-12T23:42:25.846336+0000 mon.a (mon.0) 58 : audit [INF] from='mgr.24661 ' entity='mgr.y' 2024-09-12T23:42:26.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:26 smithi136 bash[75164]: audit 2024-09-12T23:42:25.856540+0000 mon.a (mon.0) 59 : audit [INF] from='mgr.24661 ' entity='mgr.y' 2024-09-12T23:42:26.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:26 smithi136 bash[75164]: audit 2024-09-12T23:42:25.871262+0000 mon.a (mon.0) 60 : audit [INF] from='mgr.24661 ' entity='mgr.y' 2024-09-12T23:42:26.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:26 smithi136 bash[75164]: audit 2024-09-12T23:42:25.881081+0000 mon.a (mon.0) 61 : audit [INF] from='mgr.24661 ' entity='mgr.y' 2024-09-12T23:42:26.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:26 smithi136 bash[75164]: audit 2024-09-12T23:42:25.889778+0000 mon.a (mon.0) 62 : audit [INF] from='mgr.24661 ' entity='mgr.y' 2024-09-12T23:42:26.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:26 smithi136 bash[75164]: audit 2024-09-12T23:42:25.970368+0000 mon.c (mon.2) 120 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:42:26.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:26 smithi136 bash[75164]: audit 2024-09-12T23:42:25.974042+0000 mon.c (mon.2) 121 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:42:26.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:26 smithi136 bash[75164]: audit 2024-09-12T23:42:25.975475+0000 mon.c (mon.2) 122 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-12T23:42:26.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:26 smithi136 bash[75164]: audit 2024-09-12T23:42:25.976283+0000 mon.c (mon.2) 123 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-09-12T23:42:26.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:26 smithi136 bash[32189]: cephadm 2024-09-12T23:42:25.266297+0000 mgr.y (mgr.24661) 15 : cephadm [INF] Updating smithi136:/etc/ceph/ceph.conf 2024-09-12T23:42:26.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:26 smithi136 bash[32189]: cephadm 2024-09-12T23:42:25.267056+0000 mgr.y (mgr.24661) 16 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.conf 2024-09-12T23:42:26.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:26 smithi136 bash[32189]: cluster 2024-09-12T23:42:25.320947+0000 mon.a (mon.0) 57 : cluster [DBG] osdmap e307: 8 total, 8 up, 8 in 2024-09-12T23:42:26.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:26 smithi136 bash[32189]: cephadm 2024-09-12T23:42:25.457217+0000 mgr.y (mgr.24661) 17 : cephadm [INF] Updating smithi136:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-12T23:42:26.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:26 smithi136 bash[32189]: cephadm 2024-09-12T23:42:25.460889+0000 mgr.y (mgr.24661) 18 : cephadm [INF] Updating smithi181:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-12T23:42:26.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:26 smithi136 bash[32189]: cephadm 2024-09-12T23:42:25.578856+0000 mgr.y (mgr.24661) 19 : cephadm [INF] Updating smithi136:/etc/ceph/ceph.client.admin.keyring 2024-09-12T23:42:26.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:26 smithi136 bash[32189]: cephadm 2024-09-12T23:42:25.594481+0000 mgr.y (mgr.24661) 20 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2024-09-12T23:42:26.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:26 smithi136 bash[32189]: cephadm 2024-09-12T23:42:25.709866+0000 mgr.y (mgr.24661) 21 : cephadm [INF] Updating smithi136:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-12T23:42:26.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:26 smithi136 bash[32189]: cephadm 2024-09-12T23:42:25.731189+0000 mgr.y (mgr.24661) 22 : cephadm [INF] Updating smithi181:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-12T23:42:26.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:26 smithi136 bash[32189]: audit 2024-09-12T23:42:25.846336+0000 mon.a (mon.0) 58 : audit [INF] from='mgr.24661 ' entity='mgr.y' 2024-09-12T23:42:26.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:26 smithi136 bash[32189]: audit 2024-09-12T23:42:25.856540+0000 mon.a (mon.0) 59 : audit [INF] from='mgr.24661 ' entity='mgr.y' 2024-09-12T23:42:26.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:26 smithi136 bash[32189]: audit 2024-09-12T23:42:25.871262+0000 mon.a (mon.0) 60 : audit [INF] from='mgr.24661 ' entity='mgr.y' 2024-09-12T23:42:26.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:26 smithi136 bash[32189]: audit 2024-09-12T23:42:25.881081+0000 mon.a (mon.0) 61 : audit [INF] from='mgr.24661 ' entity='mgr.y' 2024-09-12T23:42:26.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:26 smithi136 bash[32189]: audit 2024-09-12T23:42:25.889778+0000 mon.a (mon.0) 62 : audit [INF] from='mgr.24661 ' entity='mgr.y' 2024-09-12T23:42:26.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:26 smithi136 bash[32189]: audit 2024-09-12T23:42:25.970368+0000 mon.c (mon.2) 120 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:42:26.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:26 smithi136 bash[32189]: audit 2024-09-12T23:42:25.974042+0000 mon.c (mon.2) 121 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:42:26.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:26 smithi136 bash[32189]: audit 2024-09-12T23:42:25.975475+0000 mon.c (mon.2) 122 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-12T23:42:26.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:26 smithi136 bash[32189]: audit 2024-09-12T23:42:25.976283+0000 mon.c (mon.2) 123 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-09-12T23:42:27.332 INFO:tasks.rados.rados.0.smithi136.stdout:1279: done (3 left) 2024-09-12T23:42:27.332 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 19 current snap is 122 2024-09-12T23:42:27.333 INFO:tasks.rados.rados.0.smithi136.stdout:1280: seq_num 418 ranges {5603328=606208,6209536=737280,6946816=434176,7380992=434176} 2024-09-12T23:42:27.370 INFO:tasks.rados.rados.0.smithi136.stdout:1280: writing smithi13667081-19 from 5603328 to 6209536 tid 1 2024-09-12T23:42:27.375 INFO:tasks.rados.rados.0.smithi136.stdout:1280: writing smithi13667081-19 from 6209536 to 6946816 tid 2 2024-09-12T23:42:27.377 INFO:tasks.rados.rados.0.smithi136.stdout:1280: writing smithi13667081-19 from 6946816 to 7380992 tid 3 2024-09-12T23:42:27.380 INFO:tasks.rados.rados.0.smithi136.stdout:1280: writing smithi13667081-19 from 7380992 to 7815168 tid 4 2024-09-12T23:42:27.380 INFO:tasks.rados.rados.0.smithi136.stdout:1281: setattr oid 11 current snap is 122 2024-09-12T23:42:27.381 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 40 v 861 (ObjNum 412 snap 120 seq_num 412) dirty exists 2024-09-12T23:42:27.381 INFO:tasks.rados.rados.0.smithi136.stdout:1278: expect (ObjNum 254 snap 69 seq_num 254) 2024-09-12T23:42:27.542 INFO:tasks.rados.rados.0.smithi136.stdout:1280: finishing write tid 1 to smithi13667081-19 2024-09-12T23:42:27.542 INFO:tasks.rados.rados.0.smithi136.stdout:1280: finishing write tid 2 to smithi13667081-19 2024-09-12T23:42:27.542 INFO:tasks.rados.rados.0.smithi136.stdout:1280: finishing write tid 3 to smithi13667081-19 2024-09-12T23:42:27.542 INFO:tasks.rados.rados.0.smithi136.stdout:1280: finishing write tid 4 to smithi13667081-19 2024-09-12T23:42:27.542 INFO:tasks.rados.rados.0.smithi136.stdout:1280: finishing write tid 5 to smithi13667081-19 2024-09-12T23:42:27.542 INFO:tasks.rados.rados.0.smithi136.stdout:1280: finishing write tid 6 to smithi13667081-19 2024-09-12T23:42:27.542 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 19 v 603 (ObjNum 418 snap 122 seq_num 418) dirty exists 2024-09-12T23:42:27.543 INFO:tasks.rados.rados.0.smithi136.stdout:1280: left oid 19 (ObjNum 418 snap 122 seq_num 418) 2024-09-12T23:42:27.543 INFO:tasks.rados.rados.0.smithi136.stdout:1275: done (4 left) 2024-09-12T23:42:27.543 INFO:tasks.rados.rados.0.smithi136.stdout:1276: done (3 left) 2024-09-12T23:42:27.543 INFO:tasks.rados.rados.0.smithi136.stdout:1278: done (2 left) 2024-09-12T23:42:27.543 INFO:tasks.rados.rados.0.smithi136.stdout:1280: done (1 left) 2024-09-12T23:42:27.543 INFO:tasks.rados.rados.0.smithi136.stdout:1282: rollback oid 9 current snap is 122 2024-09-12T23:42:27.543 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 9 to 118 2024-09-12T23:42:27.543 INFO:tasks.rados.rados.0.smithi136.stdout:1283: snap_remove snap 109 2024-09-12T23:42:27.653 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:27 smithi136 bash[75164]: cephadm 2024-09-12T23:42:25.976935+0000 mgr.y (mgr.24661) 23 : cephadm [INF] Upgrade: It appears safe to stop mon.c 2024-09-12T23:42:27.653 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:27 smithi136 bash[75164]: cluster 2024-09-12T23:42:26.327574+0000 mon.a (mon.0) 63 : cluster [DBG] osdmap e308: 8 total, 8 up, 8 in 2024-09-12T23:42:27.653 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:27 smithi136 bash[75164]: cluster 2024-09-12T23:42:26.647010+0000 mgr.y (mgr.24661) 24 : cluster [DBG] pgmap v29: 121 pgs: 121 active+clean; 258 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.9 MiB/s rd, 4.3 MiB/s wr, 20 op/s 2024-09-12T23:42:27.653 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:27 smithi136 bash[75164]: cluster 2024-09-12T23:42:27.323857+0000 mon.a (mon.0) 64 : cluster [DBG] osdmap e309: 8 total, 8 up, 8 in 2024-09-12T23:42:27.653 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:27 smithi136 bash[32189]: cephadm 2024-09-12T23:42:25.976935+0000 mgr.y (mgr.24661) 23 : cephadm [INF] Upgrade: It appears safe to stop mon.c 2024-09-12T23:42:27.654 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:27 smithi136 bash[32189]: cluster 2024-09-12T23:42:26.327574+0000 mon.a (mon.0) 63 : cluster [DBG] osdmap e308: 8 total, 8 up, 8 in 2024-09-12T23:42:27.654 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:27 smithi136 bash[32189]: cluster 2024-09-12T23:42:26.647010+0000 mgr.y (mgr.24661) 24 : cluster [DBG] pgmap v29: 121 pgs: 121 active+clean; 258 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.9 MiB/s rd, 4.3 MiB/s wr, 20 op/s 2024-09-12T23:42:27.654 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:27 smithi136 bash[32189]: cluster 2024-09-12T23:42:27.323857+0000 mon.a (mon.0) 64 : cluster [DBG] osdmap e309: 8 total, 8 up, 8 in 2024-09-12T23:42:27.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:27 smithi181 bash[24776]: cephadm 2024-09-12T23:42:25.976935+0000 mgr.y (mgr.24661) 23 : cephadm [INF] Upgrade: It appears safe to stop mon.c 2024-09-12T23:42:27.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:27 smithi181 bash[24776]: cluster 2024-09-12T23:42:26.327574+0000 mon.a (mon.0) 63 : cluster [DBG] osdmap e308: 8 total, 8 up, 8 in 2024-09-12T23:42:27.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:27 smithi181 bash[24776]: cluster 2024-09-12T23:42:26.647010+0000 mgr.y (mgr.24661) 24 : cluster [DBG] pgmap v29: 121 pgs: 121 active+clean; 258 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.9 MiB/s rd, 4.3 MiB/s wr, 20 op/s 2024-09-12T23:42:27.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:27 smithi181 bash[24776]: cluster 2024-09-12T23:42:27.323857+0000 mon.a (mon.0) 64 : cluster [DBG] osdmap e309: 8 total, 8 up, 8 in 2024-09-12T23:42:28.363 INFO:tasks.rados.rados.0.smithi136.stdout:1282: finishing rollback tid 1 to smithi13667081-9 2024-09-12T23:42:28.363 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 9 v 583 (ObjNum 65 snap 0 seq_num 1869768531) dirty dne 2024-09-12T23:42:28.364 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 11 v 875 (ObjNum 379 snap 109 seq_num 379) dirty exists 2024-09-12T23:42:28.364 INFO:tasks.rados.rados.0.smithi136.stdout:1281: done (2 left) 2024-09-12T23:42:28.364 INFO:tasks.rados.rados.0.smithi136.stdout:1282: done (1 left) 2024-09-12T23:42:28.364 INFO:tasks.rados.rados.0.smithi136.stdout:1283: done (0 left) 2024-09-12T23:42:28.364 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 5 current snap is 122 2024-09-12T23:42:28.364 INFO:tasks.rados.rados.0.smithi136.stdout:1284: seq_num 419 ranges {3678208=622592,4300800=671744,4972544=647168,5619712=286720} 2024-09-12T23:42:28.388 INFO:tasks.rados.rados.0.smithi136.stdout:1284: writing smithi13667081-5 from 3678208 to 4300800 tid 1 2024-09-12T23:42:28.393 INFO:tasks.rados.rados.0.smithi136.stdout:1284: writing smithi13667081-5 from 4300800 to 4972544 tid 2 2024-09-12T23:42:28.398 INFO:tasks.rados.rados.0.smithi136.stdout:1284: writing smithi13667081-5 from 4972544 to 5619712 tid 3 2024-09-12T23:42:28.400 INFO:tasks.rados.rados.0.smithi136.stdout:1284: writing smithi13667081-5 from 5619712 to 5906432 tid 4 2024-09-12T23:42:28.400 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 6 current snap is 122 2024-09-12T23:42:28.400 INFO:tasks.rados.rados.0.smithi136.stdout:1285: seq_num 420 ranges {3710976=638976,4349952=557056,4907008=450560,5357568=598016} 2024-09-12T23:42:28.425 INFO:tasks.rados.rados.0.smithi136.stdout:1285: writing smithi13667081-6 from 3710976 to 4349952 tid 1 2024-09-12T23:42:28.428 INFO:tasks.rados.rados.0.smithi136.stdout:1285: writing smithi13667081-6 from 4349952 to 4907008 tid 2 2024-09-12T23:42:28.430 INFO:tasks.rados.rados.0.smithi136.stdout:1285: writing smithi13667081-6 from 4907008 to 5357568 tid 3 2024-09-12T23:42:28.435 INFO:tasks.rados.rados.0.smithi136.stdout:1285: writing smithi13667081-6 from 5357568 to 5955584 tid 4 2024-09-12T23:42:28.435 INFO:tasks.rados.rados.0.smithi136.stdout:1286: read oid 28 snap -1 2024-09-12T23:42:28.435 INFO:tasks.rados.rados.0.smithi136.stdout:1286: expect (ObjNum 396 snap 115 seq_num 396) 2024-09-12T23:42:28.435 INFO:tasks.rados.rados.0.smithi136.stdout:1287: copy_from oid 18 from oid 39 current snap is 122 2024-09-12T23:42:28.435 INFO:tasks.rados.rados.0.smithi136.stdout:1284: finishing write tid 1 to smithi13667081-5 2024-09-12T23:42:28.436 INFO:tasks.rados.rados.0.smithi136.stdout:1288: read oid 41 snap -1 2024-09-12T23:42:28.436 INFO:tasks.rados.rados.0.smithi136.stdout:1288: expect (ObjNum 262 snap 71 seq_num 262) 2024-09-12T23:42:28.436 INFO:tasks.rados.rados.0.smithi136.stdout:1289: rollback oid 19 current snap is 122 2024-09-12T23:42:28.436 INFO:tasks.rados.rados.0.smithi136.stdout:1284: finishing write tid 2 to smithi13667081-5 2024-09-12T23:42:28.436 INFO:tasks.rados.rados.0.smithi136.stdout:1284: finishing write tid 3 to smithi13667081-5 2024-09-12T23:42:28.436 INFO:tasks.rados.rados.0.smithi136.stdout:1284: finishing write tid 4 to smithi13667081-5 2024-09-12T23:42:28.436 INFO:tasks.rados.rados.0.smithi136.stdout:1284: finishing write tid 5 to smithi13667081-5 2024-09-12T23:42:28.436 INFO:tasks.rados.rados.0.smithi136.stdout:1284: finishing write tid 6 to smithi13667081-5 2024-09-12T23:42:28.436 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 5 v 889 (ObjNum 419 snap 122 seq_num 419) dirty exists 2024-09-12T23:42:28.436 INFO:tasks.rados.rados.0.smithi136.stdout:1284: left oid 5 (ObjNum 419 snap 122 seq_num 419) 2024-09-12T23:42:28.436 INFO:tasks.rados.rados.0.smithi136.stdout:1285: finishing write tid 1 to smithi13667081-6 2024-09-12T23:42:28.436 INFO:tasks.rados.rados.0.smithi136.stdout:1285: finishing write tid 2 to smithi13667081-6 2024-09-12T23:42:28.436 INFO:tasks.rados.rados.0.smithi136.stdout:1285: finishing write tid 3 to smithi13667081-6 2024-09-12T23:42:28.437 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 19 to 118 2024-09-12T23:42:28.437 INFO:tasks.rados.rados.0.smithi136.stdout:1284: done (5 left) 2024-09-12T23:42:28.437 INFO:tasks.rados.rados.0.smithi136.stdout:1290: rmattr oid 20 current snap is 122 2024-09-12T23:42:28.437 INFO:tasks.rados.rados.0.smithi136.stdout:1290: done (5 left) 2024-09-12T23:42:28.437 INFO:tasks.rados.rados.0.smithi136.stdout:1291: read oid 31 snap -1 2024-09-12T23:42:28.437 INFO:tasks.rados.rados.0.smithi136.stdout:1291: expect (ObjNum 262 snap 71 seq_num 262) 2024-09-12T23:42:28.437 INFO:tasks.rados.rados.0.smithi136.stdout:1292: rmattr oid 46 current snap is 122 2024-09-12T23:42:28.437 INFO:tasks.rados.rados.0.smithi136.stdout:1292: done (6 left) 2024-09-12T23:42:28.437 INFO:tasks.rados.rados.0.smithi136.stdout:1293: rmattr oid 9 current snap is 122 2024-09-12T23:42:28.437 INFO:tasks.rados.rados.0.smithi136.stdout:1293: done (6 left) 2024-09-12T23:42:28.437 INFO:tasks.rados.rados.0.smithi136.stdout:1294: copy_from oid 2 from oid 24 current snap is 122 2024-09-12T23:42:28.437 INFO:tasks.rados.rados.0.smithi136.stdout:1295: snap_create 2024-09-12T23:42:28.438 INFO:tasks.rados.rados.0.smithi136.stdout:1289: finishing rollback tid 0 to smithi13667081-19 2024-09-12T23:42:28.440 INFO:tasks.rados.rados.0.smithi136.stdout:1289: finishing rollback tid 1 to smithi13667081-19 2024-09-12T23:42:28.440 INFO:tasks.rados.rados.0.smithi136.stdout:1289: finishing rollback tid 2 to smithi13667081-19 2024-09-12T23:42:28.440 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 19 v 606 (ObjNum 391 snap 113 seq_num 391) dirty exists 2024-09-12T23:42:28.440 INFO:tasks.rados.rados.0.smithi136.stdout:1285: finishing write tid 4 to smithi13667081-6 2024-09-12T23:42:28.441 INFO:tasks.rados.rados.0.smithi136.stdout:1285: finishing write tid 5 to smithi13667081-6 2024-09-12T23:42:28.442 INFO:tasks.rados.rados.0.smithi136.stdout:1285: finishing write tid 6 to smithi13667081-6 2024-09-12T23:42:28.442 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 6 v 293 (ObjNum 420 snap 122 seq_num 420) dirty exists 2024-09-12T23:42:28.442 INFO:tasks.rados.rados.0.smithi136.stdout:1285: left oid 6 (ObjNum 420 snap 122 seq_num 420) 2024-09-12T23:42:28.458 INFO:tasks.rados.rados.0.smithi136.stdout:1288: expect (ObjNum 262 snap 71 seq_num 262) 2024-09-12T23:42:28.618 INFO:tasks.rados.rados.0.smithi136.stdout:1286: expect (ObjNum 396 snap 115 seq_num 396) 2024-09-12T23:42:28.694 INFO:tasks.rados.rados.0.smithi136.stdout:1291: expect (ObjNum 262 snap 71 seq_num 262) 2024-09-12T23:42:28.848 INFO:tasks.rados.rados.0.smithi136.stdout:1287: finishing copy_from racing read to smithi13667081-18 2024-09-12T23:42:28.849 INFO:tasks.rados.rados.0.smithi136.stdout:1287: finishing copy_from to smithi13667081-18 2024-09-12T23:42:28.849 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 18 v 199 (ObjNum 399 snap 116 seq_num 399) dirty exists 2024-09-12T23:42:28.849 INFO:tasks.rados.rados.0.smithi136.stdout:1294: finishing copy_from racing read to smithi13667081-2 2024-09-12T23:42:28.849 INFO:tasks.rados.rados.0.smithi136.stdout:1294: finishing copy_from to smithi13667081-2 2024-09-12T23:42:28.849 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 2 v 402 (ObjNum 416 snap 120 seq_num 416) dirty exists 2024-09-12T23:42:29.094 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:42:28 smithi181 bash[60471]: ts=2024-09-12T23:42:28.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:42:29.362 INFO:tasks.rados.rados.0.smithi136.stdout:1285: done (7 left) 2024-09-12T23:42:29.362 INFO:tasks.rados.rados.0.smithi136.stdout:1286: done (6 left) 2024-09-12T23:42:29.362 INFO:tasks.rados.rados.0.smithi136.stdout:1287: done (5 left) 2024-09-12T23:42:29.362 INFO:tasks.rados.rados.0.smithi136.stdout:1288: done (4 left) 2024-09-12T23:42:29.362 INFO:tasks.rados.rados.0.smithi136.stdout:1289: done (3 left) 2024-09-12T23:42:29.362 INFO:tasks.rados.rados.0.smithi136.stdout:1291: done (2 left) 2024-09-12T23:42:29.362 INFO:tasks.rados.rados.0.smithi136.stdout:1294: done (1 left) 2024-09-12T23:42:29.362 INFO:tasks.rados.rados.0.smithi136.stdout:1295: done (0 left) 2024-09-12T23:42:29.362 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 28 current snap is 123 2024-09-12T23:42:29.363 INFO:tasks.rados.rados.0.smithi136.stdout:1296: seq_num 421 ranges {2523136=655360,3178496=442368,3620864=663552,4284416=499712} 2024-09-12T23:42:29.380 INFO:tasks.rados.rados.0.smithi136.stdout:1296: writing smithi13667081-28 from 2523136 to 3178496 tid 1 2024-09-12T23:42:29.382 INFO:tasks.rados.rados.0.smithi136.stdout:1296: writing smithi13667081-28 from 3178496 to 3620864 tid 2 2024-09-12T23:42:29.386 INFO:tasks.rados.rados.0.smithi136.stdout:1296: writing smithi13667081-28 from 3620864 to 4284416 tid 3 2024-09-12T23:42:29.390 INFO:tasks.rados.rados.0.smithi136.stdout:1296: writing smithi13667081-28 from 4284416 to 4784128 tid 4 2024-09-12T23:42:29.390 INFO:tasks.rados.rados.0.smithi136.stdout:1297: rmattr oid 39 current snap is 123 2024-09-12T23:42:29.390 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 4 current snap is 123 2024-09-12T23:42:29.390 INFO:tasks.rados.rados.0.smithi136.stdout:1298: seq_num 422 ranges {0=671744,671744=729088,1400832=417792,1818624=458752} 2024-09-12T23:42:29.394 INFO:tasks.rados.rados.0.smithi136.stdout:1298: writing smithi13667081-4 from 0 to 671744 tid 1 2024-09-12T23:42:29.399 INFO:tasks.rados.rados.0.smithi136.stdout:1298: writing smithi13667081-4 from 671744 to 1400832 tid 2 2024-09-12T23:42:29.401 INFO:tasks.rados.rados.0.smithi136.stdout:1298: writing smithi13667081-4 from 1400832 to 1818624 tid 3 2024-09-12T23:42:29.404 INFO:tasks.rados.rados.0.smithi136.stdout:1298: writing smithi13667081-4 from 1818624 to 2277376 tid 4 2024-09-12T23:42:29.404 INFO:tasks.rados.rados.0.smithi136.stdout:1299: copy_from oid 37 from oid 50 current snap is 123 2024-09-12T23:42:29.404 INFO:tasks.rados.rados.0.smithi136.stdout:1296: finishing write tid 1 to smithi13667081-28 2024-09-12T23:42:29.404 INFO:tasks.rados.rados.0.smithi136.stdout:1296: finishing write tid 2 to smithi13667081-28 2024-09-12T23:42:29.404 INFO:tasks.rados.rados.0.smithi136.stdout:1296: finishing write tid 3 to smithi13667081-28 2024-09-12T23:42:29.404 INFO:tasks.rados.rados.0.smithi136.stdout:1296: finishing write tid 4 to smithi13667081-28 2024-09-12T23:42:29.404 INFO:tasks.rados.rados.0.smithi136.stdout:1296: finishing write tid 5 to smithi13667081-28 2024-09-12T23:42:29.405 INFO:tasks.rados.rados.0.smithi136.stdout:1300: copy_from oid 10 from oid 14 current snap is 123 2024-09-12T23:42:29.405 INFO:tasks.rados.rados.0.smithi136.stdout:1301: snap_create 2024-09-12T23:42:29.405 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 39 v 225 (ObjNum 399 snap 116 seq_num 399) dirty exists 2024-09-12T23:42:29.406 INFO:tasks.rados.rados.0.smithi136.stdout:1296: finishing write tid 6 to smithi13667081-28 2024-09-12T23:42:29.406 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 28 v 750 (ObjNum 421 snap 123 seq_num 421) dirty exists 2024-09-12T23:42:29.406 INFO:tasks.rados.rados.0.smithi136.stdout:1296: left oid 28 (ObjNum 421 snap 123 seq_num 421) 2024-09-12T23:42:29.414 INFO:tasks.rados.rados.0.smithi136.stdout:1298: finishing write tid 1 to smithi13667081-4 2024-09-12T23:42:29.414 INFO:tasks.rados.rados.0.smithi136.stdout:1298: finishing write tid 2 to smithi13667081-4 2024-09-12T23:42:29.414 INFO:tasks.rados.rados.0.smithi136.stdout:1299: finishing copy_from racing read to smithi13667081-37 2024-09-12T23:42:29.417 INFO:tasks.rados.rados.0.smithi136.stdout:1299: finishing copy_from to smithi13667081-37 2024-09-12T23:42:29.417 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 37 v 209 (ObjNum 292 snap 79 seq_num 292) dirty exists 2024-09-12T23:42:29.417 INFO:tasks.rados.rados.0.smithi136.stdout:1298: finishing write tid 3 to smithi13667081-4 2024-09-12T23:42:29.419 INFO:tasks.rados.rados.0.smithi136.stdout:1298: finishing write tid 4 to smithi13667081-4 2024-09-12T23:42:29.420 INFO:tasks.rados.rados.0.smithi136.stdout:1298: finishing write tid 5 to smithi13667081-4 2024-09-12T23:42:29.423 INFO:tasks.rados.rados.0.smithi136.stdout:1298: finishing write tid 6 to smithi13667081-4 2024-09-12T23:42:29.423 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 4 v 724 (ObjNum 422 snap 123 seq_num 422) dirty exists 2024-09-12T23:42:29.423 INFO:tasks.rados.rados.0.smithi136.stdout:1298: left oid 4 (ObjNum 422 snap 123 seq_num 422) 2024-09-12T23:42:29.449 INFO:tasks.rados.rados.0.smithi136.stdout:1300: finishing copy_from racing read to smithi13667081-10 2024-09-12T23:42:29.449 INFO:tasks.rados.rados.0.smithi136.stdout:1300: finishing copy_from to smithi13667081-10 2024-09-12T23:42:29.449 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 10 v 685 (ObjNum 398 snap 116 seq_num 398) dirty exists 2024-09-12T23:42:29.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:42:29] "GET /metrics HTTP/1.1" 200 36135 "" "Prometheus/2.43.0" 2024-09-12T23:42:29.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:29 smithi136 bash[75164]: cluster 2024-09-12T23:42:28.363318+0000 mon.a (mon.0) 65 : cluster [DBG] osdmap e310: 8 total, 8 up, 8 in 2024-09-12T23:42:29.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:29 smithi136 bash[75164]: cephadm 2024-09-12T23:42:28.547477+0000 mgr.y (mgr.24661) 25 : cephadm [INF] Upgrade: Updating mon.c 2024-09-12T23:42:29.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:29 smithi136 bash[75164]: audit 2024-09-12T23:42:28.552603+0000 mon.a (mon.0) 66 : audit [INF] from='mgr.24661 ' entity='mgr.y' 2024-09-12T23:42:29.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:29 smithi136 bash[75164]: audit 2024-09-12T23:42:28.554118+0000 mon.c (mon.2) 124 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-12T23:42:29.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:29 smithi136 bash[75164]: audit 2024-09-12T23:42:28.554933+0000 mon.c (mon.2) 125 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-12T23:42:29.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:29 smithi136 bash[75164]: audit 2024-09-12T23:42:28.555619+0000 mon.c (mon.2) 126 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:42:29.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:29 smithi136 bash[75164]: cephadm 2024-09-12T23:42:28.556616+0000 mgr.y (mgr.24661) 26 : cephadm [INF] Deploying daemon mon.c on smithi136 2024-09-12T23:42:29.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:29 smithi136 bash[75164]: cluster 2024-09-12T23:42:28.647521+0000 mgr.y (mgr.24661) 27 : cluster [DBG] pgmap v32: 121 pgs: 121 active+clean; 258 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.8 MiB/s rd, 536 KiB/s wr, 3 op/s 2024-09-12T23:42:29.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:29 smithi136 bash[32189]: cluster 2024-09-12T23:42:28.363318+0000 mon.a (mon.0) 65 : cluster [DBG] osdmap e310: 8 total, 8 up, 8 in 2024-09-12T23:42:29.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:29 smithi136 bash[32189]: cephadm 2024-09-12T23:42:28.547477+0000 mgr.y (mgr.24661) 25 : cephadm [INF] Upgrade: Updating mon.c 2024-09-12T23:42:29.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:29 smithi136 bash[32189]: audit 2024-09-12T23:42:28.552603+0000 mon.a (mon.0) 66 : audit [INF] from='mgr.24661 ' entity='mgr.y' 2024-09-12T23:42:29.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:29 smithi136 bash[32189]: audit 2024-09-12T23:42:28.554118+0000 mon.c (mon.2) 124 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-12T23:42:29.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:29 smithi136 bash[32189]: audit 2024-09-12T23:42:28.554933+0000 mon.c (mon.2) 125 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-12T23:42:29.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:29 smithi136 bash[32189]: audit 2024-09-12T23:42:28.555619+0000 mon.c (mon.2) 126 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:42:29.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:29 smithi136 bash[32189]: cephadm 2024-09-12T23:42:28.556616+0000 mgr.y (mgr.24661) 26 : cephadm [INF] Deploying daemon mon.c on smithi136 2024-09-12T23:42:29.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:29 smithi136 bash[32189]: cluster 2024-09-12T23:42:28.647521+0000 mgr.y (mgr.24661) 27 : cluster [DBG] pgmap v32: 121 pgs: 121 active+clean; 258 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.8 MiB/s rd, 536 KiB/s wr, 3 op/s 2024-09-12T23:42:29.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:29 smithi181 bash[24776]: cluster 2024-09-12T23:42:28.363318+0000 mon.a (mon.0) 65 : cluster [DBG] osdmap e310: 8 total, 8 up, 8 in 2024-09-12T23:42:29.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:29 smithi181 bash[24776]: cephadm 2024-09-12T23:42:28.547477+0000 mgr.y (mgr.24661) 25 : cephadm [INF] Upgrade: Updating mon.c 2024-09-12T23:42:29.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:29 smithi181 bash[24776]: audit 2024-09-12T23:42:28.552603+0000 mon.a (mon.0) 66 : audit [INF] from='mgr.24661 ' entity='mgr.y' 2024-09-12T23:42:29.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:29 smithi181 bash[24776]: audit 2024-09-12T23:42:28.554118+0000 mon.c (mon.2) 124 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-12T23:42:29.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:29 smithi181 bash[24776]: audit 2024-09-12T23:42:28.554933+0000 mon.c (mon.2) 125 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-12T23:42:29.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:29 smithi181 bash[24776]: audit 2024-09-12T23:42:28.555619+0000 mon.c (mon.2) 126 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:42:29.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:29 smithi181 bash[24776]: cephadm 2024-09-12T23:42:28.556616+0000 mgr.y (mgr.24661) 26 : cephadm [INF] Deploying daemon mon.c on smithi136 2024-09-12T23:42:29.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:29 smithi181 bash[24776]: cluster 2024-09-12T23:42:28.647521+0000 mgr.y (mgr.24661) 27 : cluster [DBG] pgmap v32: 121 pgs: 121 active+clean; 258 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.8 MiB/s rd, 536 KiB/s wr, 3 op/s 2024-09-12T23:42:30.375 INFO:tasks.rados.rados.0.smithi136.stdout:1296: done (5 left) 2024-09-12T23:42:30.376 INFO:tasks.rados.rados.0.smithi136.stdout:1297: done (4 left) 2024-09-12T23:42:30.376 INFO:tasks.rados.rados.0.smithi136.stdout:1298: done (3 left) 2024-09-12T23:42:30.376 INFO:tasks.rados.rados.0.smithi136.stdout:1299: done (2 left) 2024-09-12T23:42:30.376 INFO:tasks.rados.rados.0.smithi136.stdout:1300: done (1 left) 2024-09-12T23:42:30.376 INFO:tasks.rados.rados.0.smithi136.stdout:1301: done (0 left) 2024-09-12T23:42:30.376 INFO:tasks.rados.rados.0.smithi136.stdout:1302: snap_create 2024-09-12T23:42:30.665 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:30 smithi136 bash[32189]: cluster 2024-09-12T23:42:29.364670+0000 mon.a (mon.0) 67 : cluster [DBG] osdmap e311: 8 total, 8 up, 8 in 2024-09-12T23:42:30.665 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:30 smithi136 bash[75164]: cluster 2024-09-12T23:42:29.364670+0000 mon.a (mon.0) 67 : cluster [DBG] osdmap e311: 8 total, 8 up, 8 in 2024-09-12T23:42:30.835 INFO:teuthology.orchestra.run.smithi136.stdout:true 2024-09-12T23:42:30.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:30 smithi181 bash[24776]: cluster 2024-09-12T23:42:29.364670+0000 mon.a (mon.0) 67 : cluster [DBG] osdmap e311: 8 total, 8 up, 8 in 2024-09-12T23:42:31.005 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:30 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:42:31.006 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 12 23:42:30 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:42:31.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:30 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:42:31.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:30 smithi136 systemd[1]: Stopping Ceph mon.c for da58a47c-715d-11ef-bceb-c7b262605968... 2024-09-12T23:42:31.007 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 12 23:42:30 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:42:31.007 INFO:journalctl@ceph.mds.a.smithi136.stdout:Sep 12 23:42:30 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:42:31.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:30 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:42:31.007 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 12 23:42:30 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:42:31.008 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 12 23:42:30 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:42:31.008 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:42:30 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:42:31.008 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:42:30 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:42:31.143 INFO:teuthology.orchestra.run.smithi136.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-12T23:42:31.143 INFO:teuthology.orchestra.run.smithi136.stdout:alertmanager.a smithi136 *:9093,9094 running (118s) 16s ago 8m 13.6M - 0.25.0 c8568f914cd2 b5197f020eeb 2024-09-12T23:42:31.143 INFO:teuthology.orchestra.run.smithi136.stdout:grafana.a smithi181 *:3000 running (106s) 16s ago 7m 74.9M - 9.4.7 954c08fa6188 256c181be984 2024-09-12T23:42:31.144 INFO:teuthology.orchestra.run.smithi136.stdout:mds.a smithi136 running (9m) 16s ago 9m 14.3M - 17.2.7-1627-gbbdadfa3 214985049bdb 713b9795ad19 2024-09-12T23:42:31.144 INFO:teuthology.orchestra.run.smithi136.stdout:mds.b smithi181 running (9m) 16s ago 9m 17.3M - 17.2.7-1627-gbbdadfa3 214985049bdb ad13179f0868 2024-09-12T23:42:31.144 INFO:teuthology.orchestra.run.smithi136.stdout:mgr.x smithi181 *:8443,9283,8765 running (54s) 16s ago 16m 423M - 18.2.4-837-g7a853b17 f21ba1eea74e cc664dc337f7 2024-09-12T23:42:31.144 INFO:teuthology.orchestra.run.smithi136.stdout:mgr.y smithi136 *:8443,9283,8765 running (99s) 16s ago 18m 460M - 18.2.4-837-g7a853b17 f21ba1eea74e 272f10b42fc3 2024-09-12T23:42:31.144 INFO:teuthology.orchestra.run.smithi136.stdout:mon.a smithi136 running (31s) 16s ago 19m 33.0M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e ad8d21d58ad9 2024-09-12T23:42:31.145 INFO:teuthology.orchestra.run.smithi136.stdout:mon.b smithi181 running (16m) 16s ago 16m 53.6M 2048M 17.2.7-1627-gbbdadfa3 214985049bdb 8b3d784af845 2024-09-12T23:42:31.145 INFO:teuthology.orchestra.run.smithi136.stdout:mon.c smithi136 running (16m) 16s ago 16m 48.7M 2048M 17.2.7-1627-gbbdadfa3 214985049bdb 647b028ea963 2024-09-12T23:42:31.145 INFO:teuthology.orchestra.run.smithi136.stdout:node-exporter.a smithi136 *:9100 running (2m) 16s ago 8m 7975k - 1.5.0 0da6a335fe13 ccfc37c96620 2024-09-12T23:42:31.145 INFO:teuthology.orchestra.run.smithi136.stdout:node-exporter.b smithi181 *:9100 running (111s) 16s ago 8m 7516k - 1.5.0 0da6a335fe13 80bf0e383230 2024-09-12T23:42:31.145 INFO:teuthology.orchestra.run.smithi136.stdout:osd.0 smithi136 running (15m) 16s ago 15m 186M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb cbc0b7df0115 2024-09-12T23:42:31.146 INFO:teuthology.orchestra.run.smithi136.stdout:osd.1 smithi136 running (14m) 16s ago 14m 189M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb c4daee7da836 2024-09-12T23:42:31.146 INFO:teuthology.orchestra.run.smithi136.stdout:osd.2 smithi136 running (14m) 16s ago 14m 187M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 4ab8fd79cdb4 2024-09-12T23:42:31.146 INFO:teuthology.orchestra.run.smithi136.stdout:osd.3 smithi136 running (13m) 16s ago 13m 144M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 9ceb14a27521 2024-09-12T23:42:31.146 INFO:teuthology.orchestra.run.smithi136.stdout:osd.4 smithi181 running (12m) 16s ago 12m 198M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb a35cdca79055 2024-09-12T23:42:31.146 INFO:teuthology.orchestra.run.smithi136.stdout:osd.5 smithi181 running (11m) 16s ago 11m 167M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 8f2e8d494fef 2024-09-12T23:42:31.147 INFO:teuthology.orchestra.run.smithi136.stdout:osd.6 smithi181 running (10m) 16s ago 10m 97.5M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 9c6bec406408 2024-09-12T23:42:31.147 INFO:teuthology.orchestra.run.smithi136.stdout:osd.7 smithi181 running (9m) 16s ago 9m 136M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb bc0632ae2145 2024-09-12T23:42:31.147 INFO:teuthology.orchestra.run.smithi136.stdout:prometheus.a smithi181 *:9095 running (60s) 16s ago 8m 46.0M - 2.43.0 a07b618ecd1d 3853d003a5cc 2024-09-12T23:42:31.418 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:31 smithi136 bash[68787]: [12/Sep/2024:23:42:31] ENGINE Bus STOPPING 2024-09-12T23:42:31.419 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:31 smithi136 bash[32189]: debug 2024-09-12T23:42:31.009+0000 7f9c37057640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-09-12T23:42:31.419 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:31 smithi136 bash[32189]: debug 2024-09-12T23:42:31.009+0000 7f9c37057640 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2024-09-12T23:42:31.479 INFO:teuthology.orchestra.run.smithi136.stdout:{ 2024-09-12T23:42:31.479 INFO:teuthology.orchestra.run.smithi136.stdout: "mon": { 2024-09-12T23:42:31.479 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2, 2024-09-12T23:42:31.479 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 1 2024-09-12T23:42:31.479 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:42:31.479 INFO:teuthology.orchestra.run.smithi136.stdout: "mgr": { 2024-09-12T23:42:31.480 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 2 2024-09-12T23:42:31.480 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:42:31.480 INFO:teuthology.orchestra.run.smithi136.stdout: "osd": { 2024-09-12T23:42:31.480 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 8 2024-09-12T23:42:31.480 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:42:31.480 INFO:teuthology.orchestra.run.smithi136.stdout: "mds": { 2024-09-12T23:42:31.480 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-12T23:42:31.480 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:42:31.480 INFO:teuthology.orchestra.run.smithi136.stdout: "overall": { 2024-09-12T23:42:31.480 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 12, 2024-09-12T23:42:31.480 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 3 2024-09-12T23:42:31.480 INFO:teuthology.orchestra.run.smithi136.stdout: } 2024-09-12T23:42:31.480 INFO:teuthology.orchestra.run.smithi136.stdout:} 2024-09-12T23:42:31.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:31 smithi136 bash[76219]: ceph-da58a47c-715d-11ef-bceb-c7b262605968-mon-c 2024-09-12T23:42:31.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:31 smithi136 bash[68787]: [12/Sep/2024:23:42:31] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-12T23:42:31.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:31 smithi136 bash[68787]: [12/Sep/2024:23:42:31] ENGINE Bus STOPPED 2024-09-12T23:42:31.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:31 smithi136 bash[68787]: [12/Sep/2024:23:42:31] ENGINE Bus STARTING 2024-09-12T23:42:31.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:31 smithi136 bash[68787]: [12/Sep/2024:23:42:31] ENGINE Serving on http://:::9283 2024-09-12T23:42:31.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:31 smithi136 bash[68787]: [12/Sep/2024:23:42:31] ENGINE Bus STARTED 2024-09-12T23:42:32.134 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:31 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:42:32.135 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:31 smithi136 bash[76315]: Error response from daemon: No such container: ceph-da58a47c-715d-11ef-bceb-c7b262605968-mon-c 2024-09-12T23:42:32.135 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:31 smithi136 systemd[1]: ceph-da58a47c-715d-11ef-bceb-c7b262605968@mon.c.service: Deactivated successfully. 2024-09-12T23:42:32.135 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:31 smithi136 systemd[1]: Stopped Ceph mon.c for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:42:32.135 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:31 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:42:32.135 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:32 smithi136 systemd[1]: Started Ceph mon.c for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:42:32.135 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 12 23:42:31 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:42:32.136 INFO:journalctl@ceph.mds.a.smithi136.stdout:Sep 12 23:42:31 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:42:32.136 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:42:31 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:42:32.136 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:42:31 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:42:32.137 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:31 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:42:32.137 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 12 23:42:31 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:42:32.137 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 12 23:42:31 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:42:32.138 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 12 23:42:31 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:42:32.594 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:42:32 smithi181 bash[60471]: ts=2024-09-12T23:42:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:42:33.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.317+0000 7f502e8b1b00 0 set uid:gid to 167:167 (ceph:ceph) 2024-09-12T23:42:33.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.317+0000 7f502e8b1b00 0 ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable), process ceph-mon, pid 7 2024-09-12T23:42:33.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.317+0000 7f502e8b1b00 0 pidfile_write: ignore empty --pid-file 2024-09-12T23:42:33.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.321+0000 7f502e8b1b00 0 load: jerasure load: lrc 2024-09-12T23:42:33.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: RocksDB version: 7.9.2 2024-09-12T23:42:33.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Git sha 0 2024-09-12T23:42:33.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Compile date 2024-09-10 15:58:06 2024-09-12T23:42:33.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: DB SUMMARY 2024-09-12T23:42:33.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: DB Session ID: 7X6QNO661XK3UD9PB65S 2024-09-12T23:42:33.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: CURRENT file: CURRENT 2024-09-12T23:42:33.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: IDENTITY file: IDENTITY 2024-09-12T23:42:33.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: MANIFEST file: MANIFEST-000009 size: 1651 Bytes 2024-09-12T23:42:33.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000036.sst 2024-09-12T23:42:33.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000034.log size: 3323123 ; 2024-09-12T23:42:33.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.error_if_exists: 0 2024-09-12T23:42:33.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.create_if_missing: 0 2024-09-12T23:42:33.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.paranoid_checks: 1 2024-09-12T23:42:33.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.flush_verify_memtable_count: 1 2024-09-12T23:42:33.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-12T23:42:33.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-12T23:42:33.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.env: 0x55ab5b02dbc0 2024-09-12T23:42:33.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.fs: PosixFileSystem 2024-09-12T23:42:33.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.info_log: 0x55ab5baaabe0 2024-09-12T23:42:33.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.max_file_opening_threads: 16 2024-09-12T23:42:33.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.statistics: (nil) 2024-09-12T23:42:33.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.use_fsync: 0 2024-09-12T23:42:33.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.max_log_file_size: 0 2024-09-12T23:42:33.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-12T23:42:33.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.log_file_time_to_roll: 0 2024-09-12T23:42:33.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.keep_log_file_num: 1000 2024-09-12T23:42:33.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.recycle_log_file_num: 0 2024-09-12T23:42:33.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.allow_fallocate: 1 2024-09-12T23:42:33.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.allow_mmap_reads: 0 2024-09-12T23:42:33.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.allow_mmap_writes: 0 2024-09-12T23:42:33.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.use_direct_reads: 0 2024-09-12T23:42:33.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-12T23:42:33.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.create_missing_column_families: 0 2024-09-12T23:42:33.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.db_log_dir: 2024-09-12T23:42:33.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.wal_dir: 2024-09-12T23:42:33.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.table_cache_numshardbits: 6 2024-09-12T23:42:33.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-09-12T23:42:33.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-09-12T23:42:33.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-12T23:42:33.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-12T23:42:33.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-09-12T23:42:33.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.advise_random_on_open: 1 2024-09-12T23:42:33.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.db_write_buffer_size: 0 2024-09-12T23:42:33.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.write_buffer_manager: 0x55ab5bab8b40 2024-09-12T23:42:33.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-12T23:42:33.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-12T23:42:33.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.use_adaptive_mutex: 0 2024-09-12T23:42:33.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.rate_limiter: (nil) 2024-09-12T23:42:33.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-12T23:42:33.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.wal_recovery_mode: 2 2024-09-12T23:42:33.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.enable_thread_tracking: 0 2024-09-12T23:42:33.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.enable_pipelined_write: 0 2024-09-12T23:42:33.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.unordered_write: 0 2024-09-12T23:42:33.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-12T23:42:33.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-12T23:42:33.512 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-12T23:42:33.512 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-12T23:42:33.512 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.row_cache: None 2024-09-12T23:42:33.512 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.wal_filter: None 2024-09-12T23:42:33.512 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-12T23:42:33.512 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.allow_ingest_behind: 0 2024-09-12T23:42:33.512 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.two_write_queues: 0 2024-09-12T23:42:33.512 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.manual_wal_flush: 0 2024-09-12T23:42:33.512 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.wal_compression: 0 2024-09-12T23:42:33.512 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.atomic_flush: 0 2024-09-12T23:42:33.513 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-12T23:42:33.513 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.persist_stats_to_disk: 0 2024-09-12T23:42:33.513 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-09-12T23:42:33.513 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.log_readahead_size: 0 2024-09-12T23:42:33.513 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-12T23:42:33.513 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.best_efforts_recovery: 0 2024-09-12T23:42:33.513 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-12T23:42:33.513 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-12T23:42:33.513 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.allow_data_in_errors: 0 2024-09-12T23:42:33.513 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.db_host_id: __hostname__ 2024-09-12T23:42:33.513 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.enforce_single_del_contracts: true 2024-09-12T23:42:33.513 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.max_background_jobs: 2 2024-09-12T23:42:33.513 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.max_background_compactions: -1 2024-09-12T23:42:33.513 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.max_subcompactions: 1 2024-09-12T23:42:33.514 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-12T23:42:33.514 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-12T23:42:33.514 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.delayed_write_rate : 16777216 2024-09-12T23:42:33.514 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.max_total_wal_size: 0 2024-09-12T23:42:33.514 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-12T23:42:33.514 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.stats_dump_period_sec: 600 2024-09-12T23:42:33.514 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.stats_persist_period_sec: 600 2024-09-12T23:42:33.514 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-12T23:42:33.514 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.max_open_files: -1 2024-09-12T23:42:33.514 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.bytes_per_sync: 0 2024-09-12T23:42:33.514 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-09-12T23:42:33.514 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-09-12T23:42:33.514 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.compaction_readahead_size: 0 2024-09-12T23:42:33.515 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.max_background_flushes: -1 2024-09-12T23:42:33.515 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Compression algorithms supported: 2024-09-12T23:42:33.515 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: kZSTD supported: 0 2024-09-12T23:42:33.515 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: kXpressCompression supported: 0 2024-09-12T23:42:33.515 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: kBZip2Compression supported: 0 2024-09-12T23:42:33.515 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-12T23:42:33.515 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: kLZ4Compression supported: 1 2024-09-12T23:42:33.515 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: kZlibCompression supported: 1 2024-09-12T23:42:33.515 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: kLZ4HCCompression supported: 1 2024-09-12T23:42:33.515 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: kSnappyCompression supported: 1 2024-09-12T23:42:33.515 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-09-12T23:42:33.515 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: DMutex implementation: pthread_mutex_t 2024-09-12T23:42:33.515 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2024-09-12T23:42:33.516 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-12T23:42:33.516 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-12T23:42:33.516 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.merge_operator: 2024-09-12T23:42:33.516 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.compaction_filter: None 2024-09-12T23:42:33.516 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.compaction_filter_factory: None 2024-09-12T23:42:33.516 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.sst_partitioner_factory: None 2024-09-12T23:42:33.516 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-09-12T23:42:33.516 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.table_factory: BlockBasedTable 2024-09-12T23:42:33.516 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55ab5baaa7e0) 2024-09-12T23:42:33.516 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: cache_index_and_filter_blocks: 1 2024-09-12T23:42:33.516 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-12T23:42:33.516 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-12T23:42:33.516 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: pin_top_level_index_and_filter: 1 2024-09-12T23:42:33.516 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: index_type: 0 2024-09-12T23:42:33.517 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: data_block_index_type: 0 2024-09-12T23:42:33.517 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: index_shortening: 1 2024-09-12T23:42:33.517 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: data_block_hash_table_util_ratio: 0.750000 2024-09-12T23:42:33.517 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: checksum: 4 2024-09-12T23:42:33.517 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: no_block_cache: 0 2024-09-12T23:42:33.517 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: block_cache: 0x55ab5baa11f0 2024-09-12T23:42:33.517 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: block_cache_name: BinnedLRUCache 2024-09-12T23:42:33.517 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: block_cache_options: 2024-09-12T23:42:33.517 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: capacity : 536870912 2024-09-12T23:42:33.517 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: num_shard_bits : 4 2024-09-12T23:42:33.517 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: strict_capacity_limit : 0 2024-09-12T23:42:33.517 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: high_pri_pool_ratio: 0.000 2024-09-12T23:42:33.517 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: block_cache_compressed: (nil) 2024-09-12T23:42:33.517 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: persistent_cache: (nil) 2024-09-12T23:42:33.518 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: block_size: 4096 2024-09-12T23:42:33.518 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: block_size_deviation: 10 2024-09-12T23:42:33.518 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: block_restart_interval: 16 2024-09-12T23:42:33.518 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: index_block_restart_interval: 1 2024-09-12T23:42:33.518 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: metadata_block_size: 4096 2024-09-12T23:42:33.518 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: partition_filters: 0 2024-09-12T23:42:33.518 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: use_delta_encoding: 1 2024-09-12T23:42:33.518 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: filter_policy: bloomfilter 2024-09-12T23:42:33.518 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: whole_key_filtering: 1 2024-09-12T23:42:33.518 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: verify_compression: 0 2024-09-12T23:42:33.518 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: read_amp_bytes_per_bit: 0 2024-09-12T23:42:33.518 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: format_version: 5 2024-09-12T23:42:33.518 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: enable_index_compression: 1 2024-09-12T23:42:33.518 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: block_align: 0 2024-09-12T23:42:33.519 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: max_auto_readahead_size: 262144 2024-09-12T23:42:33.519 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: prepopulate_block_cache: 0 2024-09-12T23:42:33.519 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: initial_auto_readahead_size: 8192 2024-09-12T23:42:33.519 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: num_file_reads_for_auto_readahead: 2 2024-09-12T23:42:33.519 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.write_buffer_size: 33554432 2024-09-12T23:42:33.519 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.max_write_buffer_number: 2 2024-09-12T23:42:33.519 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.compression: NoCompression 2024-09-12T23:42:33.519 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.bottommost_compression: Disabled 2024-09-12T23:42:33.519 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.prefix_extractor: nullptr 2024-09-12T23:42:33.519 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-12T23:42:33.519 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.num_levels: 7 2024-09-12T23:42:33.519 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-12T23:42:33.519 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-12T23:42:33.520 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-12T23:42:33.520 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-12T23:42:33.520 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-12T23:42:33.520 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-12T23:42:33.520 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-12T23:42:33.520 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-12T23:42:33.520 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-12T23:42:33.520 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-12T23:42:33.520 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-12T23:42:33.520 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-12T23:42:33.520 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.compression_opts.window_bits: -14 2024-09-12T23:42:33.520 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.compression_opts.level: 32767 2024-09-12T23:42:33.520 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.compression_opts.strategy: 0 2024-09-12T23:42:33.520 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-12T23:42:33.521 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-12T23:42:33.521 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-12T23:42:33.521 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-12T23:42:33.521 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.compression_opts.enabled: false 2024-09-12T23:42:33.521 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-12T23:42:33.521 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-12T23:42:33.521 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-12T23:42:33.521 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-12T23:42:33.521 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.target_file_size_base: 67108864 2024-09-12T23:42:33.521 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.target_file_size_multiplier: 1 2024-09-12T23:42:33.521 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-12T23:42:33.521 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-12T23:42:33.521 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-12T23:42:33.522 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-12T23:42:33.522 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-12T23:42:33.522 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-12T23:42:33.522 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-12T23:42:33.522 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-12T23:42:33.522 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-12T23:42:33.522 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-12T23:42:33.522 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-12T23:42:33.522 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-12T23:42:33.522 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-12T23:42:33.522 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.arena_block_size: 1048576 2024-09-12T23:42:33.522 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-12T23:42:33.522 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-12T23:42:33.522 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.disable_auto_compactions: 0 2024-09-12T23:42:33.522 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-12T23:42:33.523 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-12T23:42:33.523 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-12T23:42:33.523 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-12T23:42:33.523 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-12T23:42:33.524 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-12T23:42:33.524 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-12T23:42:33.524 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-12T23:42:33.524 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-12T23:42:33.524 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-12T23:42:33.524 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-12T23:42:33.524 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.inplace_update_support: 0 2024-09-12T23:42:33.524 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-09-12T23:42:33.524 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-12T23:42:33.524 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-12T23:42:33.525 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.memtable_huge_page_size: 0 2024-09-12T23:42:33.525 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.bloom_locality: 0 2024-09-12T23:42:33.525 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.max_successive_merges: 0 2024-09-12T23:42:33.525 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-09-12T23:42:33.525 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.paranoid_file_checks: 0 2024-09-12T23:42:33.525 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.force_consistency_checks: 1 2024-09-12T23:42:33.525 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.report_bg_io_stats: 0 2024-09-12T23:42:33.525 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.ttl: 2592000 2024-09-12T23:42:33.525 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-09-12T23:42:33.525 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-12T23:42:33.525 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-12T23:42:33.525 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.enable_blob_files: false 2024-09-12T23:42:33.525 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.min_blob_size: 0 2024-09-12T23:42:33.526 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.blob_file_size: 268435456 2024-09-12T23:42:33.526 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.blob_compression_type: NoCompression 2024-09-12T23:42:33.526 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.enable_blob_garbage_collection: false 2024-09-12T23:42:33.526 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-12T23:42:33.526 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-12T23:42:33.526 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-12T23:42:33.526 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.blob_file_starting_level: 0 2024-09-12T23:42:33.526 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-12T23:42:33.526 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502b68a640 3 rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 36.sst 2024-09-12T23:42:33.526 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-12T23:42:33.526 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 38, last_sequence is 20738, log_number is 34,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-12T23:42:33.526 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 34 2024-09-12T23:42:33.526 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: eaf9438c-dfa2-4677-b545-e6b896130802 2024-09-12T23:42:33.526 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726184553330920, "job": 1, "event": "recovery_started", "wal_files": [34]} 2024-09-12T23:42:33.527 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.325+0000 7f502e8b1b00 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #34 mode 2 2024-09-12T23:42:33.527 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.337+0000 7f502e8b1b00 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726184553343689, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 39, "file_size": 2142752, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 20743, "largest_seqno": 21565, "table_properties": {"data_size": 2137940, "index_size": 2808, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 965, "raw_key_size": 9174, "raw_average_key_size": 25, "raw_value_size": 2129650, "raw_average_value_size": 5818, "num_data_blocks": 123, "num_entries": 366, "num_filter_entries": 366, "num_deletions": 17, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726184553, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "eaf9438c-dfa2-4677-b545-e6b896130802", "db_session_id": "7X6QNO661XK3UD9PB65S", "orig_file_number": 39, "seqno_to_time_mapping": "N/A"}} 2024-09-12T23:42:33.527 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.337+0000 7f502e8b1b00 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726184553343880, "job": 1, "event": "recovery_finished"} 2024-09-12T23:42:33.527 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.337+0000 7f502e8b1b00 4 rocksdb: [db/version_set.cc:5047] Creating manifest 41 2024-09-12T23:42:33.527 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.337+0000 7f502e8b1b00 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-12T23:42:33.527 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.341+0000 7f502e8b1b00 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000034.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-12T23:42:33.527 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.341+0000 7f502e8b1b00 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55ab5bacae00 2024-09-12T23:42:33.527 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.341+0000 7f502e8b1b00 4 rocksdb: DB pointer 0x55ab5bbc2000 2024-09-12T23:42:33.527 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.341+0000 7f502467c640 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-12T23:42:33.527 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.341+0000 7f502467c640 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-12T23:42:33.527 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: ** DB Stats ** 2024-09-12T23:42:33.527 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: Uptime(secs): 0.0 total, 0.0 interval 2024-09-12T23:42:33.527 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-12T23:42:33.528 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-12T23:42:33.528 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-12T23:42:33.528 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-12T23:42:33.528 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-12T23:42:33.528 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-12T23:42:33.528 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: ** Compaction Stats [default] ** 2024-09-12T23:42:33.528 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-12T23:42:33.528 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-12T23:42:33.528 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: L0 1/0 2.04 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 427.2 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-09-12T23:42:33.528 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: L6 1/0 11.49 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-12T23:42:33.528 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: Sum 2/0 13.53 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 427.2 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-09-12T23:42:33.528 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 427.2 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-09-12T23:42:33.528 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: ** Compaction Stats [default] ** 2024-09-12T23:42:33.528 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-12T23:42:33.529 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-12T23:42:33.529 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 427.2 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-09-12T23:42:33.529 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-12T23:42:33.529 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: Uptime(secs): 0.0 total, 0.0 interval 2024-09-12T23:42:33.529 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: Flush(GB): cumulative 0.002, interval 0.002 2024-09-12T23:42:33.529 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-12T23:42:33.529 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: AddFile(Total Files): cumulative 0, interval 0 2024-09-12T23:42:33.529 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: AddFile(L0 Files): cumulative 0, interval 0 2024-09-12T23:42:33.530 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: AddFile(Keys): cumulative 0, interval 0 2024-09-12T23:42:33.530 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: Cumulative compaction: 0.00 GB write, 121.17 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-12T23:42:33.530 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: Interval compaction: 0.00 GB write, 121.17 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-12T23:42:33.530 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-12T23:42:33.530 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: Block cache BinnedLRUCache@0x55ab5baa11f0#7 capacity: 512.00 MB usage: 3.86 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.3e-05 secs_since: 0 2024-09-12T23:42:33.530 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: Block cache entry stats(count,size,portion): FilterBlock(1,1.03 KB,0.000196695%) IndexBlock(1,2.83 KB,0.000539422%) Misc(1,0.00 KB,0%) 2024-09-12T23:42:33.530 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: ** File Read Latency Histogram By Level [default] ** 2024-09-12T23:42:33.530 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.341+0000 7f502e8b1b00 0 starting mon.c rank 2 at public addrs [v2:172.21.15.136:3301/0,v1:172.21.15.136:6790/0] at bind addrs [v2:172.21.15.136:3301/0,v1:172.21.15.136:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid da58a47c-715d-11ef-bceb-c7b262605968 2024-09-12T23:42:33.530 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.345+0000 7f502e8b1b00 1 mon.c@-1(???) e3 preinit fsid da58a47c-715d-11ef-bceb-c7b262605968 2024-09-12T23:42:33.531 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.345+0000 7f502e8b1b00 0 mon.c@-1(???).mds e7 new map 2024-09-12T23:42:33.531 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.345+0000 7f502e8b1b00 0 mon.c@-1(???).mds e7 print_map 2024-09-12T23:42:33.531 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: e7 2024-09-12T23:42:33.531 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: enable_multiple, ever_enabled_multiple: 1,1 2024-09-12T23:42:33.531 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-12T23:42:33.531 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: legacy client fscid: 1 2024-09-12T23:42:33.531 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: 2024-09-12T23:42:33.531 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: Filesystem 'cephfs' (1) 2024-09-12T23:42:33.531 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: fs_name cephfs 2024-09-12T23:42:33.531 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: epoch 7 2024-09-12T23:42:33.531 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-12T23:42:33.531 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: created 2024-09-12T23:33:29.192396+0000 2024-09-12T23:42:33.531 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: modified 2024-09-12T23:33:32.368306+0000 2024-09-12T23:42:33.531 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: tableserver 0 2024-09-12T23:42:33.531 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: root 0 2024-09-12T23:42:33.532 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: session_timeout 60 2024-09-12T23:42:33.532 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: session_autoclose 300 2024-09-12T23:42:33.532 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: max_file_size 1099511627776 2024-09-12T23:42:33.532 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: max_xattr_size 65536 2024-09-12T23:42:33.532 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: required_client_features {} 2024-09-12T23:42:33.532 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: last_failure 0 2024-09-12T23:42:33.532 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: last_failure_osd_epoch 0 2024-09-12T23:42:33.532 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-12T23:42:33.532 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: max_mds 1 2024-09-12T23:42:33.532 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: in 0 2024-09-12T23:42:33.532 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: up {0=24316} 2024-09-12T23:42:33.532 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: failed 2024-09-12T23:42:33.532 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: damaged 2024-09-12T23:42:33.533 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: stopped 2024-09-12T23:42:33.533 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: data_pools [3] 2024-09-12T23:42:33.533 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: metadata_pool 2 2024-09-12T23:42:33.533 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: inline_data disabled 2024-09-12T23:42:33.533 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: balancer 2024-09-12T23:42:33.533 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: bal_rank_mask -1 2024-09-12T23:42:33.533 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: standby_count_wanted 0 2024-09-12T23:42:33.533 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: [mds.b{0:24316} state up:active seq 3 addr [v2:172.21.15.181:6832/2077070284,v1:172.21.15.181:6833/2077070284] compat {c=[1],r=[1],i=[7ff]}] 2024-09-12T23:42:33.533 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: 2024-09-12T23:42:33.534 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: 2024-09-12T23:42:33.534 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: Standby daemons: 2024-09-12T23:42:33.534 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: 2024-09-12T23:42:33.534 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: [mds.a{-1:14391} state up:standby seq 1 addr [v2:172.21.15.136:6834/3940972569,v1:172.21.15.136:6835/3940972569] compat {c=[1],r=[1],i=[7ff]}] 2024-09-12T23:42:33.534 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.345+0000 7f502e8b1b00 0 mon.c@-1(???).osd e312 crush map has features 3314933069573799936, adjusting msgr requires 2024-09-12T23:42:33.534 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.345+0000 7f502e8b1b00 0 mon.c@-1(???).osd e312 crush map has features 432629308056666112, adjusting msgr requires 2024-09-12T23:42:33.534 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.345+0000 7f502e8b1b00 0 mon.c@-1(???).osd e312 crush map has features 432629308056666112, adjusting msgr requires 2024-09-12T23:42:33.534 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.345+0000 7f502e8b1b00 0 mon.c@-1(???).osd e312 crush map has features 432629308056666112, adjusting msgr requires 2024-09-12T23:42:33.534 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:33 smithi136 bash[76371]: debug 2024-09-12T23:42:33.345+0000 7f502e8b1b00 1 mon.c@-1(???).paxosservice(auth 1..21) refresh upgraded, format 0 -> 3 2024-09-12T23:42:33.844 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:42:33 smithi181 bash[60471]: ts=2024-09-12T23:42:33.509Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi136\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:42:34.844 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:42:34 smithi181 bash[60471]: ts=2024-09-12T23:42:34.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:42:34.844 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:42:34 smithi181 bash[60471]: ts=2024-09-12T23:42:34.456Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:42:39.094 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:42:38 smithi181 bash[60471]: ts=2024-09-12T23:42:38.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:42:39.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:42:39] "GET /metrics HTTP/1.1" 200 36196 "" "Prometheus/2.43.0" 2024-09-12T23:42:39.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[76371]: audit 2024-09-12T23:42:31.134428+0000 mgr.y (mgr.24661) 30 : audit [DBG] from='client.34164 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:42:39.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[76371]: audit 2024-09-12T23:42:31.483042+0000 mon.a (mon.0) 80 : audit [DBG] from='client.? 172.21.15.136:0/1269568884' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:42:39.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[76371]: cluster 2024-09-12T23:42:32.648849+0000 mgr.y (mgr.24661) 31 : cluster [DBG] pgmap v36: 121 pgs: 121 active+clean; 266 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.0 MiB/s rd, 5.2 MiB/s wr, 20 op/s 2024-09-12T23:42:39.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[76371]: cluster 2024-09-12T23:42:33.565395+0000 mon.b (mon.1) 189 : cluster [INF] mon.b calling monitor election 2024-09-12T23:42:39.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[76371]: cluster 2024-09-12T23:42:33.566338+0000 mon.a (mon.0) 81 : cluster [INF] mon.a calling monitor election 2024-09-12T23:42:39.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[76371]: cluster 2024-09-12T23:42:34.650064+0000 mgr.y (mgr.24661) 32 : cluster [DBG] pgmap v37: 121 pgs: 121 active+clean; 266 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.7 MiB/s rd, 4.4 MiB/s wr, 17 op/s 2024-09-12T23:42:39.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[76371]: cluster 2024-09-12T23:42:36.651143+0000 mgr.y (mgr.24661) 33 : cluster [DBG] pgmap v38: 121 pgs: 121 active+clean; 266 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.3 MiB/s rd, 3.5 MiB/s wr, 13 op/s 2024-09-12T23:42:39.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[76371]: cluster 2024-09-12T23:42:38.569054+0000 mon.a (mon.0) 82 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-12T23:42:39.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[76371]: cluster 2024-09-12T23:42:38.579439+0000 mon.a (mon.0) 83 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.136:3300/0,v1:172.21.15.136:6789/0],b=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0],c=[v2:172.21.15.136:3301/0,v1:172.21.15.136:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-12T23:42:39.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[76371]: cluster 2024-09-12T23:42:38.597708+0000 mon.a (mon.0) 84 : cluster [DBG] fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-12T23:42:39.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[76371]: cluster 2024-09-12T23:42:38.597769+0000 mon.a (mon.0) 85 : cluster [DBG] osdmap e313: 8 total, 8 up, 8 in 2024-09-12T23:42:39.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[76371]: cluster 2024-09-12T23:42:38.598362+0000 mon.a (mon.0) 86 : cluster [DBG] mgrmap e42: y(active, since 29s), standbys: x 2024-09-12T23:42:39.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[76371]: cluster 2024-09-12T23:42:38.598678+0000 mon.a (mon.0) 87 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2024-09-12T23:42:39.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[76371]: cluster 2024-09-12T23:42:38.609961+0000 mon.a (mon.0) 88 : cluster [WRN] Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2024-09-12T23:42:39.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[76371]: cluster 2024-09-12T23:42:38.609992+0000 mon.a (mon.0) 89 : cluster [WRN] [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2024-09-12T23:42:39.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[76371]: cluster 2024-09-12T23:42:38.610013+0000 mon.a (mon.0) 90 : cluster [WRN] mon.c (rank 2) addr [v2:172.21.15.136:3301/0,v1:172.21.15.136:6790/0] is down (out of quorum) 2024-09-12T23:42:39.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[76371]: audit 2024-09-12T23:42:38.615320+0000 mon.a (mon.0) 91 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:42:39.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[76371]: audit 2024-09-12T23:42:38.626900+0000 mon.a (mon.0) 92 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:42:39.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[76371]: audit 2024-09-12T23:42:38.628811+0000 mon.a (mon.0) 93 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:42:39.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[76371]: audit 2024-09-12T23:42:38.628943+0000 mon.a (mon.0) 94 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:42:39.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[76371]: audit 2024-09-12T23:42:38.629564+0000 mon.a (mon.0) 95 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:42:39.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[75164]: audit 2024-09-12T23:42:31.134428+0000 mgr.y (mgr.24661) 30 : audit [DBG] from='client.34164 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:42:39.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[75164]: audit 2024-09-12T23:42:31.483042+0000 mon.a (mon.0) 80 : audit [DBG] from='client.? 172.21.15.136:0/1269568884' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:42:39.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[75164]: cluster 2024-09-12T23:42:32.648849+0000 mgr.y (mgr.24661) 31 : cluster [DBG] pgmap v36: 121 pgs: 121 active+clean; 266 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.0 MiB/s rd, 5.2 MiB/s wr, 20 op/s 2024-09-12T23:42:39.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[75164]: cluster 2024-09-12T23:42:33.565395+0000 mon.b (mon.1) 189 : cluster [INF] mon.b calling monitor election 2024-09-12T23:42:39.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[75164]: cluster 2024-09-12T23:42:33.566338+0000 mon.a (mon.0) 81 : cluster [INF] mon.a calling monitor election 2024-09-12T23:42:39.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[75164]: cluster 2024-09-12T23:42:34.650064+0000 mgr.y (mgr.24661) 32 : cluster [DBG] pgmap v37: 121 pgs: 121 active+clean; 266 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.7 MiB/s rd, 4.4 MiB/s wr, 17 op/s 2024-09-12T23:42:39.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[75164]: cluster 2024-09-12T23:42:36.651143+0000 mgr.y (mgr.24661) 33 : cluster [DBG] pgmap v38: 121 pgs: 121 active+clean; 266 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.3 MiB/s rd, 3.5 MiB/s wr, 13 op/s 2024-09-12T23:42:39.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[75164]: cluster 2024-09-12T23:42:38.569054+0000 mon.a (mon.0) 82 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-12T23:42:39.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[75164]: cluster 2024-09-12T23:42:38.579439+0000 mon.a (mon.0) 83 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.136:3300/0,v1:172.21.15.136:6789/0],b=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0],c=[v2:172.21.15.136:3301/0,v1:172.21.15.136:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-12T23:42:39.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[75164]: cluster 2024-09-12T23:42:38.597708+0000 mon.a (mon.0) 84 : cluster [DBG] fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-12T23:42:39.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[75164]: cluster 2024-09-12T23:42:38.597769+0000 mon.a (mon.0) 85 : cluster [DBG] osdmap e313: 8 total, 8 up, 8 in 2024-09-12T23:42:39.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[75164]: cluster 2024-09-12T23:42:38.598362+0000 mon.a (mon.0) 86 : cluster [DBG] mgrmap e42: y(active, since 29s), standbys: x 2024-09-12T23:42:39.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[75164]: cluster 2024-09-12T23:42:38.598678+0000 mon.a (mon.0) 87 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2024-09-12T23:42:39.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[75164]: cluster 2024-09-12T23:42:38.609961+0000 mon.a (mon.0) 88 : cluster [WRN] Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2024-09-12T23:42:39.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[75164]: cluster 2024-09-12T23:42:38.609992+0000 mon.a (mon.0) 89 : cluster [WRN] [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2024-09-12T23:42:39.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[75164]: cluster 2024-09-12T23:42:38.610013+0000 mon.a (mon.0) 90 : cluster [WRN] mon.c (rank 2) addr [v2:172.21.15.136:3301/0,v1:172.21.15.136:6790/0] is down (out of quorum) 2024-09-12T23:42:39.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[75164]: audit 2024-09-12T23:42:38.615320+0000 mon.a (mon.0) 91 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:42:39.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[75164]: audit 2024-09-12T23:42:38.626900+0000 mon.a (mon.0) 92 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:42:39.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[75164]: audit 2024-09-12T23:42:38.628811+0000 mon.a (mon.0) 93 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:42:39.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[75164]: audit 2024-09-12T23:42:38.628943+0000 mon.a (mon.0) 94 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:42:39.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[75164]: audit 2024-09-12T23:42:38.629564+0000 mon.a (mon.0) 95 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:42:39.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[76371]: audit 2024-09-12T23:42:38.630091+0000 mon.a (mon.0) 96 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:42:39.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[76371]: audit 2024-09-12T23:42:38.630580+0000 mon.a (mon.0) 97 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:42:39.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[76371]: audit 2024-09-12T23:42:38.631027+0000 mon.a (mon.0) 98 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:42:39.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[76371]: audit 2024-09-12T23:42:38.635247+0000 mon.a (mon.0) 99 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:42:39.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[76371]: audit 2024-09-12T23:42:38.635933+0000 mon.b (mon.1) 190 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:42:39.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[76371]: audit 2024-09-12T23:42:38.636059+0000 mon.b (mon.1) 191 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:42:39.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[76371]: audit 2024-09-12T23:42:38.636228+0000 mon.a (mon.0) 100 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:42:39.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[76371]: audit 2024-09-12T23:42:38.636689+0000 mon.a (mon.0) 101 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:42:39.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[76371]: cluster 2024-09-12T23:42:38.651997+0000 mgr.y (mgr.24661) 34 : cluster [DBG] pgmap v40: 121 pgs: 121 active+clean; 266 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:42:39.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[76371]: audit 2024-09-12T23:42:38.728122+0000 mon.a (mon.0) 102 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:42:39.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[76371]: audit 2024-09-12T23:42:38.729564+0000 mon.a (mon.0) 103 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:42:39.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[75164]: audit 2024-09-12T23:42:38.630091+0000 mon.a (mon.0) 96 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:42:39.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[75164]: audit 2024-09-12T23:42:38.630580+0000 mon.a (mon.0) 97 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:42:39.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[75164]: audit 2024-09-12T23:42:38.631027+0000 mon.a (mon.0) 98 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:42:39.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[75164]: audit 2024-09-12T23:42:38.635247+0000 mon.a (mon.0) 99 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:42:39.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[75164]: audit 2024-09-12T23:42:38.635933+0000 mon.b (mon.1) 190 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:42:39.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[75164]: audit 2024-09-12T23:42:38.636059+0000 mon.b (mon.1) 191 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:42:39.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[75164]: audit 2024-09-12T23:42:38.636228+0000 mon.a (mon.0) 100 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:42:39.762 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[75164]: audit 2024-09-12T23:42:38.636689+0000 mon.a (mon.0) 101 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:42:39.762 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[75164]: cluster 2024-09-12T23:42:38.651997+0000 mgr.y (mgr.24661) 34 : cluster [DBG] pgmap v40: 121 pgs: 121 active+clean; 266 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:42:39.762 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[75164]: audit 2024-09-12T23:42:38.728122+0000 mon.a (mon.0) 102 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:42:39.762 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:39 smithi136 bash[75164]: audit 2024-09-12T23:42:38.729564+0000 mon.a (mon.0) 103 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:42:39.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:39 smithi181 bash[24776]: audit 2024-09-12T23:42:31.134428+0000 mgr.y (mgr.24661) 30 : audit [DBG] from='client.34164 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:42:39.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:39 smithi181 bash[24776]: audit 2024-09-12T23:42:31.483042+0000 mon.a (mon.0) 80 : audit [DBG] from='client.? 172.21.15.136:0/1269568884' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:42:39.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:39 smithi181 bash[24776]: cluster 2024-09-12T23:42:32.648849+0000 mgr.y (mgr.24661) 31 : cluster [DBG] pgmap v36: 121 pgs: 121 active+clean; 266 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.0 MiB/s rd, 5.2 MiB/s wr, 20 op/s 2024-09-12T23:42:39.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:39 smithi181 bash[24776]: cluster 2024-09-12T23:42:33.565395+0000 mon.b (mon.1) 189 : cluster [INF] mon.b calling monitor election 2024-09-12T23:42:39.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:39 smithi181 bash[24776]: cluster 2024-09-12T23:42:33.566338+0000 mon.a (mon.0) 81 : cluster [INF] mon.a calling monitor election 2024-09-12T23:42:39.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:39 smithi181 bash[24776]: cluster 2024-09-12T23:42:34.650064+0000 mgr.y (mgr.24661) 32 : cluster [DBG] pgmap v37: 121 pgs: 121 active+clean; 266 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.7 MiB/s rd, 4.4 MiB/s wr, 17 op/s 2024-09-12T23:42:39.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:39 smithi181 bash[24776]: cluster 2024-09-12T23:42:36.651143+0000 mgr.y (mgr.24661) 33 : cluster [DBG] pgmap v38: 121 pgs: 121 active+clean; 266 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.3 MiB/s rd, 3.5 MiB/s wr, 13 op/s 2024-09-12T23:42:39.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:39 smithi181 bash[24776]: cluster 2024-09-12T23:42:38.569054+0000 mon.a (mon.0) 82 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-12T23:42:39.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:39 smithi181 bash[24776]: cluster 2024-09-12T23:42:38.579439+0000 mon.a (mon.0) 83 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.136:3300/0,v1:172.21.15.136:6789/0],b=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0],c=[v2:172.21.15.136:3301/0,v1:172.21.15.136:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-12T23:42:39.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:39 smithi181 bash[24776]: cluster 2024-09-12T23:42:38.597708+0000 mon.a (mon.0) 84 : cluster [DBG] fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-12T23:42:39.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:39 smithi181 bash[24776]: cluster 2024-09-12T23:42:38.597769+0000 mon.a (mon.0) 85 : cluster [DBG] osdmap e313: 8 total, 8 up, 8 in 2024-09-12T23:42:39.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:39 smithi181 bash[24776]: cluster 2024-09-12T23:42:38.598362+0000 mon.a (mon.0) 86 : cluster [DBG] mgrmap e42: y(active, since 29s), standbys: x 2024-09-12T23:42:39.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:39 smithi181 bash[24776]: cluster 2024-09-12T23:42:38.598678+0000 mon.a (mon.0) 87 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2024-09-12T23:42:39.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:39 smithi181 bash[24776]: cluster 2024-09-12T23:42:38.609961+0000 mon.a (mon.0) 88 : cluster [WRN] Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2024-09-12T23:42:39.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:39 smithi181 bash[24776]: cluster 2024-09-12T23:42:38.609992+0000 mon.a (mon.0) 89 : cluster [WRN] [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2024-09-12T23:42:39.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:39 smithi181 bash[24776]: cluster 2024-09-12T23:42:38.610013+0000 mon.a (mon.0) 90 : cluster [WRN] mon.c (rank 2) addr [v2:172.21.15.136:3301/0,v1:172.21.15.136:6790/0] is down (out of quorum) 2024-09-12T23:42:39.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:39 smithi181 bash[24776]: audit 2024-09-12T23:42:38.615320+0000 mon.a (mon.0) 91 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:42:39.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:39 smithi181 bash[24776]: audit 2024-09-12T23:42:38.626900+0000 mon.a (mon.0) 92 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:42:39.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:39 smithi181 bash[24776]: audit 2024-09-12T23:42:38.628811+0000 mon.a (mon.0) 93 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:42:39.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:39 smithi181 bash[24776]: audit 2024-09-12T23:42:38.628943+0000 mon.a (mon.0) 94 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:42:39.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:39 smithi181 bash[24776]: audit 2024-09-12T23:42:38.629564+0000 mon.a (mon.0) 95 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:42:39.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:39 smithi181 bash[24776]: audit 2024-09-12T23:42:38.630091+0000 mon.a (mon.0) 96 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:42:39.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:39 smithi181 bash[24776]: audit 2024-09-12T23:42:38.630580+0000 mon.a (mon.0) 97 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:42:39.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:39 smithi181 bash[24776]: audit 2024-09-12T23:42:38.631027+0000 mon.a (mon.0) 98 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:42:39.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:39 smithi181 bash[24776]: audit 2024-09-12T23:42:38.635247+0000 mon.a (mon.0) 99 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-12T23:42:39.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:39 smithi181 bash[24776]: audit 2024-09-12T23:42:38.635933+0000 mon.b (mon.1) 190 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-12T23:42:39.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:39 smithi181 bash[24776]: audit 2024-09-12T23:42:38.636059+0000 mon.b (mon.1) 191 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-12T23:42:39.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:39 smithi181 bash[24776]: audit 2024-09-12T23:42:38.636228+0000 mon.a (mon.0) 100 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-12T23:42:39.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:39 smithi181 bash[24776]: audit 2024-09-12T23:42:38.636689+0000 mon.a (mon.0) 101 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-12T23:42:39.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:39 smithi181 bash[24776]: cluster 2024-09-12T23:42:38.651997+0000 mgr.y (mgr.24661) 34 : cluster [DBG] pgmap v40: 121 pgs: 121 active+clean; 266 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:42:39.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:39 smithi181 bash[24776]: audit 2024-09-12T23:42:38.728122+0000 mon.a (mon.0) 102 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:42:39.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:39 smithi181 bash[24776]: audit 2024-09-12T23:42:38.729564+0000 mon.a (mon.0) 103 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:42:40.344 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:42:40 smithi181 bash[60471]: ts=2024-09-12T23:42:40.088Z caller=alerting.go:366 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephMonDown: template: __alert_CephMonDown:1:427: executing \"__alert_CephMonDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"mon.a\"} on the right hand-side of the operation: [{ceph_daemon=\"mon.a\", ceph_version=\"ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", public_addr=\"172.21.15.136\", rank=\"0\"}, {ceph_daemon=\"mon.a\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", public_addr=\"172.21.15.136\", rank=\"0\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-09-12T23:42:40.639 INFO:tasks.rados.rados.0.smithi136.stdout:1302: done (0 left) 2024-09-12T23:42:40.639 INFO:tasks.rados.rados.0.smithi136.stdout:1303: snap_remove snap 120 2024-09-12T23:42:41.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:40 smithi136 bash[75164]: cluster 2024-09-12T23:42:33.562907+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2024-09-12T23:42:41.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:40 smithi136 bash[75164]: cluster 2024-09-12T23:42:39.584560+0000 mon.a (mon.0) 104 : cluster [INF] mon.a calling monitor election 2024-09-12T23:42:41.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:40 smithi136 bash[75164]: cluster 2024-09-12T23:42:39.587633+0000 mon.b (mon.1) 192 : cluster [INF] mon.b calling monitor election 2024-09-12T23:42:41.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:40 smithi136 bash[75164]: cluster 2024-09-12T23:42:39.609687+0000 mon.a (mon.0) 105 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-12T23:42:41.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:40 smithi136 bash[75164]: cluster 2024-09-12T23:42:39.632293+0000 mon.a (mon.0) 106 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.136:3300/0,v1:172.21.15.136:6789/0],b=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0],c=[v2:172.21.15.136:3301/0,v1:172.21.15.136:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-12T23:42:41.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:40 smithi136 bash[75164]: cluster 2024-09-12T23:42:39.653589+0000 mon.a (mon.0) 107 : cluster [DBG] fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-12T23:42:41.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:40 smithi136 bash[75164]: cluster 2024-09-12T23:42:39.653648+0000 mon.a (mon.0) 108 : cluster [DBG] osdmap e314: 8 total, 8 up, 8 in 2024-09-12T23:42:41.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:40 smithi136 bash[75164]: cluster 2024-09-12T23:42:39.654288+0000 mon.a (mon.0) 109 : cluster [DBG] mgrmap e42: y(active, since 31s), standbys: x 2024-09-12T23:42:41.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:40 smithi136 bash[75164]: cluster 2024-09-12T23:42:39.654592+0000 mon.a (mon.0) 110 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-09-12T23:42:41.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:40 smithi136 bash[75164]: cluster 2024-09-12T23:42:39.654622+0000 mon.a (mon.0) 111 : cluster [INF] Cluster is now healthy 2024-09-12T23:42:41.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:40 smithi136 bash[75164]: cluster 2024-09-12T23:42:39.663211+0000 mon.a (mon.0) 112 : cluster [INF] overall HEALTH_OK 2024-09-12T23:42:41.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:40 smithi136 bash[76371]: cluster 2024-09-12T23:42:33.562907+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2024-09-12T23:42:41.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:40 smithi136 bash[76371]: cluster 2024-09-12T23:42:39.584560+0000 mon.a (mon.0) 104 : cluster [INF] mon.a calling monitor election 2024-09-12T23:42:41.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:40 smithi136 bash[76371]: cluster 2024-09-12T23:42:39.587633+0000 mon.b (mon.1) 192 : cluster [INF] mon.b calling monitor election 2024-09-12T23:42:41.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:40 smithi136 bash[76371]: cluster 2024-09-12T23:42:39.609687+0000 mon.a (mon.0) 105 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-12T23:42:41.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:40 smithi136 bash[76371]: cluster 2024-09-12T23:42:39.632293+0000 mon.a (mon.0) 106 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.136:3300/0,v1:172.21.15.136:6789/0],b=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0],c=[v2:172.21.15.136:3301/0,v1:172.21.15.136:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-12T23:42:41.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:40 smithi136 bash[76371]: cluster 2024-09-12T23:42:39.653589+0000 mon.a (mon.0) 107 : cluster [DBG] fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-12T23:42:41.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:40 smithi136 bash[76371]: cluster 2024-09-12T23:42:39.653648+0000 mon.a (mon.0) 108 : cluster [DBG] osdmap e314: 8 total, 8 up, 8 in 2024-09-12T23:42:41.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:40 smithi136 bash[76371]: cluster 2024-09-12T23:42:39.654288+0000 mon.a (mon.0) 109 : cluster [DBG] mgrmap e42: y(active, since 31s), standbys: x 2024-09-12T23:42:41.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:40 smithi136 bash[76371]: cluster 2024-09-12T23:42:39.654592+0000 mon.a (mon.0) 110 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-09-12T23:42:41.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:40 smithi136 bash[76371]: cluster 2024-09-12T23:42:39.654622+0000 mon.a (mon.0) 111 : cluster [INF] Cluster is now healthy 2024-09-12T23:42:41.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:40 smithi136 bash[76371]: cluster 2024-09-12T23:42:39.663211+0000 mon.a (mon.0) 112 : cluster [INF] overall HEALTH_OK 2024-09-12T23:42:41.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:40 smithi181 bash[24776]: cluster 2024-09-12T23:42:33.562907+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2024-09-12T23:42:41.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:40 smithi181 bash[24776]: cluster 2024-09-12T23:42:39.584560+0000 mon.a (mon.0) 104 : cluster [INF] mon.a calling monitor election 2024-09-12T23:42:41.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:40 smithi181 bash[24776]: cluster 2024-09-12T23:42:39.587633+0000 mon.b (mon.1) 192 : cluster [INF] mon.b calling monitor election 2024-09-12T23:42:41.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:40 smithi181 bash[24776]: cluster 2024-09-12T23:42:39.609687+0000 mon.a (mon.0) 105 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-12T23:42:41.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:40 smithi181 bash[24776]: cluster 2024-09-12T23:42:39.632293+0000 mon.a (mon.0) 106 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.136:3300/0,v1:172.21.15.136:6789/0],b=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0],c=[v2:172.21.15.136:3301/0,v1:172.21.15.136:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-12T23:42:41.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:40 smithi181 bash[24776]: cluster 2024-09-12T23:42:39.653589+0000 mon.a (mon.0) 107 : cluster [DBG] fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-12T23:42:41.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:40 smithi181 bash[24776]: cluster 2024-09-12T23:42:39.653648+0000 mon.a (mon.0) 108 : cluster [DBG] osdmap e314: 8 total, 8 up, 8 in 2024-09-12T23:42:41.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:40 smithi181 bash[24776]: cluster 2024-09-12T23:42:39.654288+0000 mon.a (mon.0) 109 : cluster [DBG] mgrmap e42: y(active, since 31s), standbys: x 2024-09-12T23:42:41.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:40 smithi181 bash[24776]: cluster 2024-09-12T23:42:39.654592+0000 mon.a (mon.0) 110 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-09-12T23:42:41.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:40 smithi181 bash[24776]: cluster 2024-09-12T23:42:39.654622+0000 mon.a (mon.0) 111 : cluster [INF] Cluster is now healthy 2024-09-12T23:42:41.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:40 smithi181 bash[24776]: cluster 2024-09-12T23:42:39.663211+0000 mon.a (mon.0) 112 : cluster [INF] overall HEALTH_OK 2024-09-12T23:42:41.659 INFO:tasks.rados.rados.0.smithi136.stdout:1303: done (0 left) 2024-09-12T23:42:41.660 INFO:tasks.rados.rados.0.smithi136.stdout:1304: rollback oid 5 current snap is 125 2024-09-12T23:42:41.660 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 5 to 118 2024-09-12T23:42:41.660 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 29 current snap is 125 2024-09-12T23:42:41.660 INFO:tasks.rados.rados.0.smithi136.stdout:1305: seq_num 423 ranges {3874816=688128,4562944=614400,5177344=622592,5799936=368640} 2024-09-12T23:42:41.694 INFO:tasks.rados.rados.0.smithi136.stdout:1305: writing smithi13667081-29 from 3874816 to 4562944 tid 1 2024-09-12T23:42:41.699 INFO:tasks.rados.rados.0.smithi136.stdout:1305: writing smithi13667081-29 from 4562944 to 5177344 tid 2 2024-09-12T23:42:41.704 INFO:tasks.rados.rados.0.smithi136.stdout:1305: writing smithi13667081-29 from 5177344 to 5799936 tid 3 2024-09-12T23:42:41.708 INFO:tasks.rados.rados.0.smithi136.stdout:1305: writing smithi13667081-29 from 5799936 to 6168576 tid 4 2024-09-12T23:42:41.708 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 10 current snap is 125 2024-09-12T23:42:41.708 INFO:tasks.rados.rados.0.smithi136.stdout:1306: seq_num 424 ranges {4775936=704512,5480448=491520,5971968=434176,6406144=679936} 2024-09-12T23:42:41.742 INFO:tasks.rados.rados.0.smithi136.stdout:1306: writing smithi13667081-10 from 4775936 to 5480448 tid 1 2024-09-12T23:42:41.745 INFO:tasks.rados.rados.0.smithi136.stdout:1306: writing smithi13667081-10 from 5480448 to 5971968 tid 2 2024-09-12T23:42:41.748 INFO:tasks.rados.rados.0.smithi136.stdout:1306: writing smithi13667081-10 from 5971968 to 6406144 tid 3 2024-09-12T23:42:41.752 INFO:tasks.rados.rados.0.smithi136.stdout:1306: writing smithi13667081-10 from 6406144 to 7086080 tid 4 2024-09-12T23:42:41.752 INFO:tasks.rados.rados.0.smithi136.stdout:1307: delete oid 26 current snap is 125 2024-09-12T23:42:41.752 INFO:tasks.rados.rados.0.smithi136.stdout:1304: finishing rollback tid 0 to smithi13667081-5 2024-09-12T23:42:41.753 INFO:tasks.rados.rados.0.smithi136.stdout:1304: finishing rollback tid 1 to smithi13667081-5 2024-09-12T23:42:41.753 INFO:tasks.rados.rados.0.smithi136.stdout:1304: finishing rollback tid 2 to smithi13667081-5 2024-09-12T23:42:41.753 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 5 v 893 (ObjNum 372 snap 108 seq_num 372) dirty exists 2024-09-12T23:42:41.753 INFO:tasks.rados.rados.0.smithi136.stdout:1305: finishing write tid 1 to smithi13667081-29 2024-09-12T23:42:41.753 INFO:tasks.rados.rados.0.smithi136.stdout:1305: finishing write tid 2 to smithi13667081-29 2024-09-12T23:42:41.753 INFO:tasks.rados.rados.0.smithi136.stdout:1305: finishing write tid 3 to smithi13667081-29 2024-09-12T23:42:41.753 INFO:tasks.rados.rados.0.smithi136.stdout:1305: finishing write tid 4 to smithi13667081-29 2024-09-12T23:42:41.753 INFO:tasks.rados.rados.0.smithi136.stdout:1305: finishing write tid 5 to smithi13667081-29 2024-09-12T23:42:41.753 INFO:tasks.rados.rados.0.smithi136.stdout:1305: finishing write tid 6 to smithi13667081-29 2024-09-12T23:42:41.753 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 29 v 551 (ObjNum 423 snap 125 seq_num 423) dirty exists 2024-09-12T23:42:41.753 INFO:tasks.rados.rados.0.smithi136.stdout:1305: left oid 29 (ObjNum 423 snap 125 seq_num 423) 2024-09-12T23:42:41.754 INFO:tasks.rados.rados.0.smithi136.stdout:1306: finishing write tid 1 to smithi13667081-10 2024-09-12T23:42:41.755 INFO:tasks.rados.rados.0.smithi136.stdout:1304: done (3 left) 2024-09-12T23:42:41.755 INFO:tasks.rados.rados.0.smithi136.stdout:1305: done (2 left) 2024-09-12T23:42:41.755 INFO:tasks.rados.rados.0.smithi136.stdout:1307: done (1 left) 2024-09-12T23:42:41.755 INFO:tasks.rados.rados.0.smithi136.stdout:1308: read oid 39 snap -1 2024-09-12T23:42:41.755 INFO:tasks.rados.rados.0.smithi136.stdout:1308: expect (ObjNum 399 snap 116 seq_num 399) 2024-09-12T23:42:41.755 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 15 current snap is 125 2024-09-12T23:42:41.756 INFO:tasks.rados.rados.0.smithi136.stdout:1309: seq_num 426 ranges {6135808=737280,6873088=712704,7585792=802816,8388608=90112} 2024-09-12T23:42:41.794 INFO:tasks.rados.rados.0.smithi136.stdout:1309: writing smithi13667081-15 from 6135808 to 6873088 tid 1 2024-09-12T23:42:41.798 INFO:tasks.rados.rados.0.smithi136.stdout:1309: writing smithi13667081-15 from 6873088 to 7585792 tid 2 2024-09-12T23:42:41.802 INFO:tasks.rados.rados.0.smithi136.stdout:1309: writing smithi13667081-15 from 7585792 to 8388608 tid 3 2024-09-12T23:42:41.804 INFO:tasks.rados.rados.0.smithi136.stdout:1309: writing smithi13667081-15 from 8388608 to 8478720 tid 4 2024-09-12T23:42:41.804 INFO:tasks.rados.rados.0.smithi136.stdout:1310: snap_create 2024-09-12T23:42:41.804 INFO:tasks.rados.rados.0.smithi136.stdout:1306: finishing write tid 2 to smithi13667081-10 2024-09-12T23:42:41.804 INFO:tasks.rados.rados.0.smithi136.stdout:1306: finishing write tid 3 to smithi13667081-10 2024-09-12T23:42:41.804 INFO:tasks.rados.rados.0.smithi136.stdout:1306: finishing write tid 4 to smithi13667081-10 2024-09-12T23:42:41.804 INFO:tasks.rados.rados.0.smithi136.stdout:1306: finishing write tid 5 to smithi13667081-10 2024-09-12T23:42:41.804 INFO:tasks.rados.rados.0.smithi136.stdout:1306: finishing write tid 6 to smithi13667081-10 2024-09-12T23:42:41.804 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 10 v 690 (ObjNum 424 snap 125 seq_num 424) dirty exists 2024-09-12T23:42:41.804 INFO:tasks.rados.rados.0.smithi136.stdout:1306: left oid 10 (ObjNum 424 snap 125 seq_num 424) 2024-09-12T23:42:41.804 INFO:tasks.rados.rados.0.smithi136.stdout:1308: expect (ObjNum 399 snap 116 seq_num 399) 2024-09-12T23:42:42.005 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:41 smithi136 bash[76371]: cluster 2024-09-12T23:42:40.643240+0000 mon.a (mon.0) 113 : cluster [DBG] osdmap e315: 8 total, 8 up, 8 in 2024-09-12T23:42:42.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:41 smithi136 bash[76371]: cluster 2024-09-12T23:42:40.652734+0000 mgr.y (mgr.24661) 35 : cluster [DBG] pgmap v43: 121 pgs: 121 active+clean; 266 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:42:42.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:41 smithi136 bash[75164]: cluster 2024-09-12T23:42:40.643240+0000 mon.a (mon.0) 113 : cluster [DBG] osdmap e315: 8 total, 8 up, 8 in 2024-09-12T23:42:42.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:41 smithi136 bash[75164]: cluster 2024-09-12T23:42:40.652734+0000 mgr.y (mgr.24661) 35 : cluster [DBG] pgmap v43: 121 pgs: 121 active+clean; 266 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:42:42.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:41 smithi181 bash[24776]: cluster 2024-09-12T23:42:40.643240+0000 mon.a (mon.0) 113 : cluster [DBG] osdmap e315: 8 total, 8 up, 8 in 2024-09-12T23:42:42.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:41 smithi181 bash[24776]: cluster 2024-09-12T23:42:40.652734+0000 mgr.y (mgr.24661) 35 : cluster [DBG] pgmap v43: 121 pgs: 121 active+clean; 266 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:42:42.100 INFO:tasks.rados.rados.0.smithi136.stdout:1309: finishing write tid 1 to smithi13667081-15 2024-09-12T23:42:42.100 INFO:tasks.rados.rados.0.smithi136.stdout:1309: finishing write tid 2 to smithi13667081-15 2024-09-12T23:42:42.100 INFO:tasks.rados.rados.0.smithi136.stdout:1309: finishing write tid 3 to smithi13667081-15 2024-09-12T23:42:42.100 INFO:tasks.rados.rados.0.smithi136.stdout:1309: finishing write tid 4 to smithi13667081-15 2024-09-12T23:42:42.100 INFO:tasks.rados.rados.0.smithi136.stdout:1309: finishing write tid 5 to smithi13667081-15 2024-09-12T23:42:42.101 INFO:tasks.rados.rados.0.smithi136.stdout:1309: finishing write tid 6 to smithi13667081-15 2024-09-12T23:42:42.101 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 15 v 629 (ObjNum 426 snap 125 seq_num 426) dirty exists 2024-09-12T23:42:42.101 INFO:tasks.rados.rados.0.smithi136.stdout:1309: left oid 15 (ObjNum 426 snap 125 seq_num 426) 2024-09-12T23:42:42.594 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:42:42 smithi181 bash[60471]: ts=2024-09-12T23:42:42.313Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:42:42.654 INFO:tasks.rados.rados.0.smithi136.stdout:1306: done (3 left) 2024-09-12T23:42:42.654 INFO:tasks.rados.rados.0.smithi136.stdout:1308: done (2 left) 2024-09-12T23:42:42.655 INFO:tasks.rados.rados.0.smithi136.stdout:1309: done (1 left) 2024-09-12T23:42:42.655 INFO:tasks.rados.rados.0.smithi136.stdout:1310: done (0 left) 2024-09-12T23:42:42.655 INFO:tasks.rados.rados.0.smithi136.stdout:1311: snap_create 2024-09-12T23:42:43.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:42 smithi136 bash[75164]: cluster 2024-09-12T23:42:41.653910+0000 mon.a (mon.0) 114 : cluster [DBG] osdmap e316: 8 total, 8 up, 8 in 2024-09-12T23:42:43.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:42 smithi136 bash[76371]: cluster 2024-09-12T23:42:41.653910+0000 mon.a (mon.0) 114 : cluster [DBG] osdmap e316: 8 total, 8 up, 8 in 2024-09-12T23:42:43.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:42 smithi181 bash[24776]: cluster 2024-09-12T23:42:41.653910+0000 mon.a (mon.0) 114 : cluster [DBG] osdmap e316: 8 total, 8 up, 8 in 2024-09-12T23:42:43.655 INFO:tasks.rados.rados.0.smithi136.stdout:1311: done (0 left) 2024-09-12T23:42:43.656 INFO:tasks.rados.rados.0.smithi136.stdout:1312: read oid 2 snap 116 2024-09-12T23:42:43.656 INFO:tasks.rados.rados.0.smithi136.stdout:1312: expect deleted 2024-09-12T23:42:43.656 INFO:tasks.rados.rados.0.smithi136.stdout:1313: setattr oid 7 current snap is 127 2024-09-12T23:42:43.657 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 16 current snap is 127 2024-09-12T23:42:43.657 INFO:tasks.rados.rados.0.smithi136.stdout:1314: seq_num 427 ranges {1425408=761856,2187264=598016,2785280=606208,3391488=401408} 2024-09-12T23:42:43.665 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:43 smithi136 bash[68787]: debug 2024-09-12T23:42:43.352+0000 7f5e4b0ad640 -1 mgr.server handle_report got status from non-daemon mon.c 2024-09-12T23:42:43.670 INFO:tasks.rados.rados.0.smithi136.stdout:1314: writing smithi13667081-16 from 1425408 to 2187264 tid 1 2024-09-12T23:42:43.673 INFO:tasks.rados.rados.0.smithi136.stdout:1314: writing smithi13667081-16 from 2187264 to 2785280 tid 2 2024-09-12T23:42:43.677 INFO:tasks.rados.rados.0.smithi136.stdout:1314: writing smithi13667081-16 from 2785280 to 3391488 tid 3 2024-09-12T23:42:43.680 INFO:tasks.rados.rados.0.smithi136.stdout:1314: writing smithi13667081-16 from 3391488 to 3792896 tid 4 2024-09-12T23:42:43.680 INFO:tasks.rados.rados.0.smithi136.stdout:1315: read oid 20 snap -1 2024-09-12T23:42:43.680 INFO:tasks.rados.rados.0.smithi136.stdout:1315: expect deleted 2024-09-12T23:42:43.680 INFO:tasks.rados.rados.0.smithi136.stdout:1316: delete oid 4 current snap is 127 2024-09-12T23:42:43.680 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 7 v 613 (ObjNum 350 snap 95 seq_num 350) dirty exists 2024-09-12T23:42:43.680 INFO:tasks.rados.rados.0.smithi136.stdout:1314: finishing write tid 1 to smithi13667081-16 2024-09-12T23:42:43.680 INFO:tasks.rados.rados.0.smithi136.stdout:1314: finishing write tid 2 to smithi13667081-16 2024-09-12T23:42:43.683 INFO:tasks.rados.rados.0.smithi136.stdout:1314: finishing write tid 3 to smithi13667081-16 2024-09-12T23:42:43.684 INFO:tasks.rados.rados.0.smithi136.stdout:1312: done (4 left) 2024-09-12T23:42:43.684 INFO:tasks.rados.rados.0.smithi136.stdout:1313: done (3 left) 2024-09-12T23:42:43.684 INFO:tasks.rados.rados.0.smithi136.stdout:1315: done (2 left) 2024-09-12T23:42:43.684 INFO:tasks.rados.rados.0.smithi136.stdout:1316: done (1 left) 2024-09-12T23:42:43.684 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 10 current snap is 127 2024-09-12T23:42:43.684 INFO:tasks.rados.rados.0.smithi136.stdout:1314: finishing write tid 4 to smithi13667081-16 2024-09-12T23:42:43.684 INFO:tasks.rados.rados.0.smithi136.stdout:1314: finishing write tid 5 to smithi13667081-16 2024-09-12T23:42:43.685 INFO:tasks.rados.rados.0.smithi136.stdout:1317: seq_num 429 ranges {7086080=794624,7880704=770048,8650752=622592,9273344=212992} 2024-09-12T23:42:43.728 INFO:tasks.rados.rados.0.smithi136.stdout:1317: writing smithi13667081-10 from 7086080 to 7880704 tid 1 2024-09-12T23:42:43.733 INFO:tasks.rados.rados.0.smithi136.stdout:1317: writing smithi13667081-10 from 7880704 to 8650752 tid 2 2024-09-12T23:42:43.736 INFO:tasks.rados.rados.0.smithi136.stdout:1317: writing smithi13667081-10 from 8650752 to 9273344 tid 3 2024-09-12T23:42:43.738 INFO:tasks.rados.rados.0.smithi136.stdout:1317: writing smithi13667081-10 from 9273344 to 9486336 tid 4 2024-09-12T23:42:43.738 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 17 current snap is 127 2024-09-12T23:42:43.739 INFO:tasks.rados.rados.0.smithi136.stdout:1314: finishing write tid 6 to smithi13667081-16 2024-09-12T23:42:43.739 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 16 v 758 (ObjNum 427 snap 127 seq_num 427) dirty exists 2024-09-12T23:42:43.739 INFO:tasks.rados.rados.0.smithi136.stdout:1314: left oid 16 (ObjNum 427 snap 127 seq_num 427) 2024-09-12T23:42:43.739 INFO:tasks.rados.rados.0.smithi136.stdout:1318: seq_num 430 ranges {0=8192} 2024-09-12T23:42:43.739 INFO:tasks.rados.rados.0.smithi136.stdout:1318: writing smithi13667081-17 from 0 to 8192 tid 1 2024-09-12T23:42:43.739 INFO:tasks.rados.rados.0.smithi136.stdout:1314: done (2 left) 2024-09-12T23:42:43.739 INFO:tasks.rados.rados.0.smithi136.stdout:1319: read oid 26 snap -1 2024-09-12T23:42:43.739 INFO:tasks.rados.rados.0.smithi136.stdout:1319: expect deleted 2024-09-12T23:42:43.739 INFO:tasks.rados.rados.0.smithi136.stdout:1320: read oid 46 snap -1 2024-09-12T23:42:43.739 INFO:tasks.rados.rados.0.smithi136.stdout:1320: expect (ObjNum 385 snap 111 seq_num 385) 2024-09-12T23:42:43.739 INFO:tasks.rados.rados.0.smithi136.stdout:1321: snap_remove snap 121 2024-09-12T23:42:43.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:43 smithi181 bash[24776]: cluster 2024-09-12T23:42:42.653520+0000 mgr.y (mgr.24661) 36 : cluster [DBG] pgmap v46: 121 pgs: 121 active+clean; 266 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:42:43.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:43 smithi181 bash[24776]: cluster 2024-09-12T23:42:42.656680+0000 mon.a (mon.0) 115 : cluster [DBG] osdmap e317: 8 total, 8 up, 8 in 2024-09-12T23:42:43.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:43 smithi181 bash[24776]: cluster 2024-09-12T23:42:43.660100+0000 mon.a (mon.0) 116 : cluster [DBG] osdmap e318: 8 total, 8 up, 8 in 2024-09-12T23:42:43.844 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:42:43 smithi181 bash[60471]: ts=2024-09-12T23:42:43.509Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi136\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:42:44.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:43 smithi136 bash[75164]: cluster 2024-09-12T23:42:42.653520+0000 mgr.y (mgr.24661) 36 : cluster [DBG] pgmap v46: 121 pgs: 121 active+clean; 266 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:42:44.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:43 smithi136 bash[75164]: cluster 2024-09-12T23:42:42.656680+0000 mon.a (mon.0) 115 : cluster [DBG] osdmap e317: 8 total, 8 up, 8 in 2024-09-12T23:42:44.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:43 smithi136 bash[75164]: cluster 2024-09-12T23:42:43.660100+0000 mon.a (mon.0) 116 : cluster [DBG] osdmap e318: 8 total, 8 up, 8 in 2024-09-12T23:42:44.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:43 smithi136 bash[76371]: cluster 2024-09-12T23:42:42.653520+0000 mgr.y (mgr.24661) 36 : cluster [DBG] pgmap v46: 121 pgs: 121 active+clean; 266 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:42:44.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:43 smithi136 bash[76371]: cluster 2024-09-12T23:42:42.656680+0000 mon.a (mon.0) 115 : cluster [DBG] osdmap e317: 8 total, 8 up, 8 in 2024-09-12T23:42:44.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:43 smithi136 bash[76371]: cluster 2024-09-12T23:42:43.660100+0000 mon.a (mon.0) 116 : cluster [DBG] osdmap e318: 8 total, 8 up, 8 in 2024-09-12T23:42:44.675 INFO:tasks.rados.rados.0.smithi136.stdout:1321: done (4 left) 2024-09-12T23:42:44.675 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 9 current snap is 127 2024-09-12T23:42:44.675 INFO:tasks.rados.rados.0.smithi136.stdout:1322: seq_num 431 ranges {0=24576} 2024-09-12T23:42:44.675 INFO:tasks.rados.rados.0.smithi136.stdout:1322: writing smithi13667081-9 from 0 to 24576 tid 1 2024-09-12T23:42:44.675 INFO:tasks.rados.rados.0.smithi136.stdout:1323: snap_create 2024-09-12T23:42:44.675 INFO:tasks.rados.rados.0.smithi136.stdout:1317: finishing write tid 1 to smithi13667081-10 2024-09-12T23:42:44.675 INFO:tasks.rados.rados.0.smithi136.stdout:1318: finishing write tid 1 to smithi13667081-17 2024-09-12T23:42:44.675 INFO:tasks.rados.rados.0.smithi136.stdout:1317: finishing write tid 2 to smithi13667081-10 2024-09-12T23:42:44.675 INFO:tasks.rados.rados.0.smithi136.stdout:1317: finishing write tid 3 to smithi13667081-10 2024-09-12T23:42:44.675 INFO:tasks.rados.rados.0.smithi136.stdout:1318: finishing write tid 2 to smithi13667081-17 2024-09-12T23:42:44.676 INFO:tasks.rados.rados.0.smithi136.stdout:1317: finishing write tid 4 to smithi13667081-10 2024-09-12T23:42:44.676 INFO:tasks.rados.rados.0.smithi136.stdout:1318: finishing write tid 3 to smithi13667081-17 2024-09-12T23:42:44.676 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 17 v 901 (ObjNum 430 snap 127 seq_num 430) dirty exists 2024-09-12T23:42:44.676 INFO:tasks.rados.rados.0.smithi136.stdout:1318: left oid 17 (ObjNum 430 snap 127 seq_num 430) 2024-09-12T23:42:44.676 INFO:tasks.rados.rados.0.smithi136.stdout:1317: finishing write tid 5 to smithi13667081-10 2024-09-12T23:42:44.676 INFO:tasks.rados.rados.0.smithi136.stdout:1320: expect (ObjNum 385 snap 111 seq_num 385) 2024-09-12T23:42:44.726 INFO:tasks.rados.rados.0.smithi136.stdout:1317: finishing write tid 6 to smithi13667081-10 2024-09-12T23:42:44.727 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 10 v 695 (ObjNum 429 snap 127 seq_num 429) dirty exists 2024-09-12T23:42:44.727 INFO:tasks.rados.rados.0.smithi136.stdout:1317: left oid 10 (ObjNum 429 snap 127 seq_num 429) 2024-09-12T23:42:44.727 INFO:tasks.rados.rados.0.smithi136.stdout:1322: finishing write tid 1 to smithi13667081-9 2024-09-12T23:42:44.727 INFO:tasks.rados.rados.0.smithi136.stdout:1322: finishing write tid 2 to smithi13667081-9 2024-09-12T23:42:44.727 INFO:tasks.rados.rados.0.smithi136.stdout:1322: finishing write tid 3 to smithi13667081-9 2024-09-12T23:42:44.727 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 9 v 608 (ObjNum 431 snap 127 seq_num 431) dirty exists 2024-09-12T23:42:44.727 INFO:tasks.rados.rados.0.smithi136.stdout:1322: left oid 9 (ObjNum 431 snap 127 seq_num 431) 2024-09-12T23:42:44.844 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:42:44 smithi181 bash[60471]: ts=2024-09-12T23:42:44.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:42:44.844 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:42:44 smithi181 bash[60471]: ts=2024-09-12T23:42:44.456Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:42:45.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:45 smithi136 bash[76371]: audit 2024-09-12T23:42:44.252011+0000 mon.a (mon.0) 117 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:42:45.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:45 smithi136 bash[76371]: audit 2024-09-12T23:42:44.257161+0000 mon.a (mon.0) 118 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:42:45.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:45 smithi136 bash[76371]: cluster 2024-09-12T23:42:44.654358+0000 mgr.y (mgr.24661) 37 : cluster [DBG] pgmap v48: 121 pgs: 121 active+clean; 277 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 1.2 MiB/s rd, 2.8 MiB/s wr, 9 op/s 2024-09-12T23:42:45.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:45 smithi136 bash[76371]: cluster 2024-09-12T23:42:44.677727+0000 mon.a (mon.0) 119 : cluster [DBG] osdmap e319: 8 total, 8 up, 8 in 2024-09-12T23:42:45.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:45 smithi136 bash[76371]: audit 2024-09-12T23:42:44.972884+0000 mon.a (mon.0) 120 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:42:45.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:45 smithi136 bash[76371]: audit 2024-09-12T23:42:44.978192+0000 mon.a (mon.0) 121 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:42:45.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:45 smithi136 bash[75164]: audit 2024-09-12T23:42:44.252011+0000 mon.a (mon.0) 117 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:42:45.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:45 smithi136 bash[75164]: audit 2024-09-12T23:42:44.257161+0000 mon.a (mon.0) 118 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:42:45.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:45 smithi136 bash[75164]: cluster 2024-09-12T23:42:44.654358+0000 mgr.y (mgr.24661) 37 : cluster [DBG] pgmap v48: 121 pgs: 121 active+clean; 277 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 1.2 MiB/s rd, 2.8 MiB/s wr, 9 op/s 2024-09-12T23:42:45.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:45 smithi136 bash[75164]: cluster 2024-09-12T23:42:44.677727+0000 mon.a (mon.0) 119 : cluster [DBG] osdmap e319: 8 total, 8 up, 8 in 2024-09-12T23:42:45.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:45 smithi136 bash[75164]: audit 2024-09-12T23:42:44.972884+0000 mon.a (mon.0) 120 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:42:45.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:45 smithi136 bash[75164]: audit 2024-09-12T23:42:44.978192+0000 mon.a (mon.0) 121 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:42:45.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:45 smithi181 bash[24776]: audit 2024-09-12T23:42:44.252011+0000 mon.a (mon.0) 117 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:42:45.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:45 smithi181 bash[24776]: audit 2024-09-12T23:42:44.257161+0000 mon.a (mon.0) 118 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:42:45.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:45 smithi181 bash[24776]: cluster 2024-09-12T23:42:44.654358+0000 mgr.y (mgr.24661) 37 : cluster [DBG] pgmap v48: 121 pgs: 121 active+clean; 277 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 1.2 MiB/s rd, 2.8 MiB/s wr, 9 op/s 2024-09-12T23:42:45.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:45 smithi181 bash[24776]: cluster 2024-09-12T23:42:44.677727+0000 mon.a (mon.0) 119 : cluster [DBG] osdmap e319: 8 total, 8 up, 8 in 2024-09-12T23:42:45.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:45 smithi181 bash[24776]: audit 2024-09-12T23:42:44.972884+0000 mon.a (mon.0) 120 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:42:45.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:45 smithi181 bash[24776]: audit 2024-09-12T23:42:44.978192+0000 mon.a (mon.0) 121 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:42:45.677 INFO:tasks.rados.rados.0.smithi136.stdout:1317: done (5 left) 2024-09-12T23:42:45.678 INFO:tasks.rados.rados.0.smithi136.stdout:1318: done (4 left) 2024-09-12T23:42:45.678 INFO:tasks.rados.rados.0.smithi136.stdout:1319: done (3 left) 2024-09-12T23:42:45.678 INFO:tasks.rados.rados.0.smithi136.stdout:1320: done (2 left) 2024-09-12T23:42:45.678 INFO:tasks.rados.rados.0.smithi136.stdout:1322: done (1 left) 2024-09-12T23:42:45.678 INFO:tasks.rados.rados.0.smithi136.stdout:1323: done (0 left) 2024-09-12T23:42:45.678 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 34 current snap is 128 2024-09-12T23:42:45.678 INFO:tasks.rados.rados.0.smithi136.stdout:1324: seq_num 432 ranges {1409024=40960} 2024-09-12T23:42:45.686 INFO:tasks.rados.rados.0.smithi136.stdout:1324: writing smithi13667081-34 from 1409024 to 1449984 tid 1 2024-09-12T23:42:45.686 INFO:tasks.rados.rados.0.smithi136.stdout:1325: read oid 37 snap -1 2024-09-12T23:42:45.686 INFO:tasks.rados.rados.0.smithi136.stdout:1325: expect (ObjNum 292 snap 79 seq_num 292) 2024-09-12T23:42:45.686 INFO:tasks.rados.rados.0.smithi136.stdout:1326: snap_create 2024-09-12T23:42:45.694 INFO:tasks.rados.rados.0.smithi136.stdout:1324: finishing write tid 1 to smithi13667081-34 2024-09-12T23:42:45.695 INFO:tasks.rados.rados.0.smithi136.stdout:1324: finishing write tid 2 to smithi13667081-34 2024-09-12T23:42:45.697 INFO:tasks.rados.rados.0.smithi136.stdout:1325: expect (ObjNum 292 snap 79 seq_num 292) 2024-09-12T23:42:45.701 INFO:tasks.rados.rados.0.smithi136.stdout:1324: finishing write tid 3 to smithi13667081-34 2024-09-12T23:42:45.701 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 34 v 647 (ObjNum 432 snap 128 seq_num 432) dirty exists 2024-09-12T23:42:45.701 INFO:tasks.rados.rados.0.smithi136.stdout:1324: left oid 34 (ObjNum 432 snap 128 seq_num 432) 2024-09-12T23:42:46.284 INFO:tasks.rados.rados.0.smithi136.stdout:1324: done (2 left) 2024-09-12T23:42:46.285 INFO:tasks.rados.rados.0.smithi136.stdout:1325: done (1 left) 2024-09-12T23:42:46.285 INFO:tasks.rados.rados.0.smithi136.stdout:1326: done (0 left) 2024-09-12T23:42:46.285 INFO:tasks.rados.rados.0.smithi136.stdout:1327: snap_remove snap 126 2024-09-12T23:42:47.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:46 smithi136 bash[75164]: cluster 2024-09-12T23:42:45.680362+0000 mon.a (mon.0) 122 : cluster [DBG] osdmap e320: 8 total, 8 up, 8 in 2024-09-12T23:42:47.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:46 smithi136 bash[75164]: cluster 2024-09-12T23:42:46.287948+0000 mon.a (mon.0) 123 : cluster [DBG] osdmap e321: 8 total, 8 up, 8 in 2024-09-12T23:42:47.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:46 smithi136 bash[76371]: cluster 2024-09-12T23:42:45.680362+0000 mon.a (mon.0) 122 : cluster [DBG] osdmap e320: 8 total, 8 up, 8 in 2024-09-12T23:42:47.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:46 smithi136 bash[76371]: cluster 2024-09-12T23:42:46.287948+0000 mon.a (mon.0) 123 : cluster [DBG] osdmap e321: 8 total, 8 up, 8 in 2024-09-12T23:42:47.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:46 smithi181 bash[24776]: cluster 2024-09-12T23:42:45.680362+0000 mon.a (mon.0) 122 : cluster [DBG] osdmap e320: 8 total, 8 up, 8 in 2024-09-12T23:42:47.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:46 smithi181 bash[24776]: cluster 2024-09-12T23:42:46.287948+0000 mon.a (mon.0) 123 : cluster [DBG] osdmap e321: 8 total, 8 up, 8 in 2024-09-12T23:42:47.294 INFO:tasks.rados.rados.0.smithi136.stdout:1327: done (0 left) 2024-09-12T23:42:47.294 INFO:tasks.rados.rados.0.smithi136.stdout:1328: read oid 9 snap -1 2024-09-12T23:42:47.294 INFO:tasks.rados.rados.0.smithi136.stdout:1328: expect (ObjNum 431 snap 127 seq_num 431) 2024-09-12T23:42:47.294 INFO:tasks.rados.rados.0.smithi136.stdout:1329: read oid 41 snap 122 2024-09-12T23:42:47.295 INFO:tasks.rados.rados.0.smithi136.stdout:1329: expect (ObjNum 262 snap 71 seq_num 262) 2024-09-12T23:42:47.295 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 28 current snap is 129 2024-09-12T23:42:47.295 INFO:tasks.rados.rados.0.smithi136.stdout:1330: seq_num 433 ranges {4784128=57344} 2024-09-12T23:42:47.326 INFO:tasks.rados.rados.0.smithi136.stdout:1330: writing smithi13667081-28 from 4784128 to 4841472 tid 1 2024-09-12T23:42:47.326 INFO:tasks.rados.rados.0.smithi136.stdout:1331: copy_from oid 8 from oid 14 current snap is 129 2024-09-12T23:42:47.326 INFO:tasks.rados.rados.0.smithi136.stdout:1328: expect (ObjNum 431 snap 127 seq_num 431) 2024-09-12T23:42:47.327 INFO:tasks.rados.rados.0.smithi136.stdout:1329: expect (ObjNum 262 snap 71 seq_num 262) 2024-09-12T23:42:47.500 INFO:tasks.rados.rados.0.smithi136.stdout:1330: finishing write tid 1 to smithi13667081-28 2024-09-12T23:42:47.501 INFO:tasks.rados.rados.0.smithi136.stdout:1330: finishing write tid 2 to smithi13667081-28 2024-09-12T23:42:47.501 INFO:tasks.rados.rados.0.smithi136.stdout:1330: finishing write tid 3 to smithi13667081-28 2024-09-12T23:42:47.501 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 28 v 767 (ObjNum 433 snap 129 seq_num 433) dirty exists 2024-09-12T23:42:47.501 INFO:tasks.rados.rados.0.smithi136.stdout:1330: left oid 28 (ObjNum 433 snap 129 seq_num 433) 2024-09-12T23:42:47.501 INFO:tasks.rados.rados.0.smithi136.stdout:1331: finishing copy_from racing read to smithi13667081-8 2024-09-12T23:42:47.501 INFO:tasks.rados.rados.0.smithi136.stdout:1331: finishing copy_from to smithi13667081-8 2024-09-12T23:42:47.501 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 8 v 685 (ObjNum 398 snap 116 seq_num 398) dirty exists 2024-09-12T23:42:47.501 INFO:tasks.rados.rados.0.smithi136.stdout:1328: done (3 left) 2024-09-12T23:42:47.501 INFO:tasks.rados.rados.0.smithi136.stdout:1329: done (2 left) 2024-09-12T23:42:47.501 INFO:tasks.rados.rados.0.smithi136.stdout:1330: done (1 left) 2024-09-12T23:42:47.501 INFO:tasks.rados.rados.0.smithi136.stdout:1331: done (0 left) 2024-09-12T23:42:47.501 INFO:tasks.rados.rados.0.smithi136.stdout:1332: rollback oid 26 current snap is 129 2024-09-12T23:42:47.501 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 26 to 127 2024-09-12T23:42:47.501 INFO:tasks.rados.rados.0.smithi136.stdout:1333: copy_from oid 24 from oid 43 current snap is 129 2024-09-12T23:42:47.502 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 28 current snap is 129 2024-09-12T23:42:47.502 INFO:tasks.rados.rados.0.smithi136.stdout:1334: seq_num 434 ranges {4841472=73728} 2024-09-12T23:42:47.542 INFO:tasks.rados.rados.0.smithi136.stdout:1334: writing smithi13667081-28 from 4841472 to 4915200 tid 1 2024-09-12T23:42:47.542 INFO:tasks.rados.rados.0.smithi136.stdout:1335: snap_create 2024-09-12T23:42:47.542 INFO:tasks.rados.rados.0.smithi136.stdout:1332: finishing rollback tid 1 to smithi13667081-26 2024-09-12T23:42:47.542 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 26 v 397 (ObjNum 129 snap 0 seq_num 4243015024) dirty dne 2024-09-12T23:42:47.542 INFO:tasks.rados.rados.0.smithi136.stdout:1333: finishing copy_from racing read to smithi13667081-24 2024-09-12T23:42:47.542 INFO:tasks.rados.rados.0.smithi136.stdout:1333: finishing copy_from to smithi13667081-24 2024-09-12T23:42:47.542 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 24 v 199 (ObjNum 399 snap 116 seq_num 399) dirty exists 2024-09-12T23:42:47.544 INFO:tasks.rados.rados.0.smithi136.stdout:1334: finishing write tid 1 to smithi13667081-28 2024-09-12T23:42:47.545 INFO:tasks.rados.rados.0.smithi136.stdout:1334: finishing write tid 2 to smithi13667081-28 2024-09-12T23:42:47.546 INFO:tasks.rados.rados.0.smithi136.stdout:1334: finishing write tid 3 to smithi13667081-28 2024-09-12T23:42:47.547 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 28 v 769 (ObjNum 434 snap 129 seq_num 434) dirty exists 2024-09-12T23:42:47.547 INFO:tasks.rados.rados.0.smithi136.stdout:1334: left oid 28 (ObjNum 434 snap 129 seq_num 434) 2024-09-12T23:42:48.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:47 smithi136 bash[75164]: cluster 2024-09-12T23:42:46.655011+0000 mgr.y (mgr.24661) 38 : cluster [DBG] pgmap v52: 121 pgs: 121 active+clean; 277 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 4.9 MiB/s rd, 2.8 MiB/s wr, 15 op/s 2024-09-12T23:42:48.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:47 smithi136 bash[75164]: cluster 2024-09-12T23:42:47.297705+0000 mon.a (mon.0) 124 : cluster [DBG] osdmap e322: 8 total, 8 up, 8 in 2024-09-12T23:42:48.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:47 smithi136 bash[76371]: cluster 2024-09-12T23:42:46.655011+0000 mgr.y (mgr.24661) 38 : cluster [DBG] pgmap v52: 121 pgs: 121 active+clean; 277 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 4.9 MiB/s rd, 2.8 MiB/s wr, 15 op/s 2024-09-12T23:42:48.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:47 smithi136 bash[76371]: cluster 2024-09-12T23:42:47.297705+0000 mon.a (mon.0) 124 : cluster [DBG] osdmap e322: 8 total, 8 up, 8 in 2024-09-12T23:42:48.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:47 smithi181 bash[24776]: cluster 2024-09-12T23:42:46.655011+0000 mgr.y (mgr.24661) 38 : cluster [DBG] pgmap v52: 121 pgs: 121 active+clean; 277 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 4.9 MiB/s rd, 2.8 MiB/s wr, 15 op/s 2024-09-12T23:42:48.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:47 smithi181 bash[24776]: cluster 2024-09-12T23:42:47.297705+0000 mon.a (mon.0) 124 : cluster [DBG] osdmap e322: 8 total, 8 up, 8 in 2024-09-12T23:42:48.300 INFO:tasks.rados.rados.0.smithi136.stdout:1332: done (3 left) 2024-09-12T23:42:48.300 INFO:tasks.rados.rados.0.smithi136.stdout:1333: done (2 left) 2024-09-12T23:42:48.300 INFO:tasks.rados.rados.0.smithi136.stdout:1334: done (1 left) 2024-09-12T23:42:48.300 INFO:tasks.rados.rados.0.smithi136.stdout:1335: done (0 left) 2024-09-12T23:42:48.300 INFO:tasks.rados.rados.0.smithi136.stdout:1336: copy_from oid 31 from oid 47 current snap is 130 2024-09-12T23:42:48.300 INFO:tasks.rados.rados.0.smithi136.stdout:1337: read oid 12 snap 127 2024-09-12T23:42:48.301 INFO:tasks.rados.rados.0.smithi136.stdout:1337: expect (ObjNum 297 snap 81 seq_num 297) 2024-09-12T23:42:48.301 INFO:tasks.rados.rados.0.smithi136.stdout:1338: delete oid 32 current snap is 130 2024-09-12T23:42:48.305 INFO:tasks.rados.rados.0.smithi136.stdout:1336: finishing copy_from to smithi13667081-31 2024-09-12T23:42:48.305 INFO:tasks.rados.rados.0.smithi136.stdout:1336: got expected ENOENT (src dne) 2024-09-12T23:42:48.305 INFO:tasks.rados.rados.0.smithi136.stdout:1336: finishing copy_from racing read to smithi13667081-31 2024-09-12T23:42:48.310 INFO:tasks.rados.rados.0.smithi136.stdout:1336: done (2 left) 2024-09-12T23:42:48.310 INFO:tasks.rados.rados.0.smithi136.stdout:1338: done (1 left) 2024-09-12T23:42:48.310 INFO:tasks.rados.rados.0.smithi136.stdout:1339: snap_create 2024-09-12T23:42:48.329 INFO:tasks.rados.rados.0.smithi136.stdout:1337: expect (ObjNum 297 snap 81 seq_num 297) 2024-09-12T23:42:49.094 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:42:48 smithi181 bash[60471]: ts=2024-09-12T23:42:48.662Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:42:49.319 INFO:tasks.rados.rados.0.smithi136.stdout:1337: done (1 left) 2024-09-12T23:42:49.319 INFO:tasks.rados.rados.0.smithi136.stdout:1339: done (0 left) 2024-09-12T23:42:49.319 INFO:tasks.rados.rados.0.smithi136.stdout:1340: copy_from oid 38 from oid 8 current snap is 131 2024-09-12T23:42:49.319 INFO:tasks.rados.rados.0.smithi136.stdout:1341: delete oid 42 current snap is 131 2024-09-12T23:42:49.322 INFO:tasks.rados.rados.0.smithi136.stdout:1341: done (1 left) 2024-09-12T23:42:49.322 INFO:tasks.rados.rados.0.smithi136.stdout:1342: delete oid 29 current snap is 131 2024-09-12T23:42:49.331 INFO:tasks.rados.rados.0.smithi136.stdout:1342: done (1 left) 2024-09-12T23:42:49.331 INFO:tasks.rados.rados.0.smithi136.stdout:1343: rollback oid 27 current snap is 131 2024-09-12T23:42:49.331 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 27 to 127 2024-09-12T23:42:49.331 INFO:tasks.rados.rados.0.smithi136.stdout:1344: snap_remove snap 124 2024-09-12T23:42:49.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:49 smithi181 bash[24776]: cluster 2024-09-12T23:42:48.303499+0000 mon.a (mon.0) 125 : cluster [DBG] osdmap e323: 8 total, 8 up, 8 in 2024-09-12T23:42:49.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:49 smithi181 bash[24776]: cluster 2024-09-12T23:42:48.655781+0000 mgr.y (mgr.24661) 39 : cluster [DBG] pgmap v55: 121 pgs: 121 active+clean; 277 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:42:49.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:42:49] "GET /metrics HTTP/1.1" 200 36196 "" "Prometheus/2.43.0" 2024-09-12T23:42:49.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:49 smithi136 bash[75164]: cluster 2024-09-12T23:42:48.303499+0000 mon.a (mon.0) 125 : cluster [DBG] osdmap e323: 8 total, 8 up, 8 in 2024-09-12T23:42:49.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:49 smithi136 bash[75164]: cluster 2024-09-12T23:42:48.655781+0000 mgr.y (mgr.24661) 39 : cluster [DBG] pgmap v55: 121 pgs: 121 active+clean; 277 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:42:49.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:49 smithi136 bash[76371]: cluster 2024-09-12T23:42:48.303499+0000 mon.a (mon.0) 125 : cluster [DBG] osdmap e323: 8 total, 8 up, 8 in 2024-09-12T23:42:49.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:49 smithi136 bash[76371]: cluster 2024-09-12T23:42:48.655781+0000 mgr.y (mgr.24661) 39 : cluster [DBG] pgmap v55: 121 pgs: 121 active+clean; 277 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:42:50.321 INFO:tasks.rados.rados.0.smithi136.stdout:1344: done (2 left) 2024-09-12T23:42:50.321 INFO:tasks.rados.rados.0.smithi136.stdout:1345: copy_from oid 3 from oid 44 current snap is 131 2024-09-12T23:42:50.321 INFO:tasks.rados.rados.0.smithi136.stdout:1346: rollback oid 6 current snap is 131 2024-09-12T23:42:50.321 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 6 to 125 2024-09-12T23:42:50.321 INFO:tasks.rados.rados.0.smithi136.stdout:1347: rollback oid 29 current snap is 131 2024-09-12T23:42:50.321 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 29 to 127 2024-09-12T23:42:50.321 INFO:tasks.rados.rados.0.smithi136.stdout:1348: snap_remove snap 123 2024-09-12T23:42:50.343 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:50 smithi181 bash[24776]: cluster 2024-09-12T23:42:49.316143+0000 mon.a (mon.0) 126 : cluster [DBG] osdmap e324: 8 total, 8 up, 8 in 2024-09-12T23:42:50.344 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:42:50 smithi181 bash[60471]: ts=2024-09-12T23:42:50.088Z caller=alerting.go:366 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephMonDown: template: __alert_CephMonDown:1:427: executing \"__alert_CephMonDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"mon.a\"} on the right hand-side of the operation: [{ceph_daemon=\"mon.a\", ceph_version=\"ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", public_addr=\"172.21.15.136\", rank=\"0\"}, {ceph_daemon=\"mon.a\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", public_addr=\"172.21.15.136\", rank=\"0\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-09-12T23:42:50.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:50 smithi136 bash[75164]: cluster 2024-09-12T23:42:49.316143+0000 mon.a (mon.0) 126 : cluster [DBG] osdmap e324: 8 total, 8 up, 8 in 2024-09-12T23:42:50.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:50 smithi136 bash[76371]: cluster 2024-09-12T23:42:49.316143+0000 mon.a (mon.0) 126 : cluster [DBG] osdmap e324: 8 total, 8 up, 8 in 2024-09-12T23:42:51.325 INFO:tasks.rados.rados.0.smithi136.stdout:1348: done (5 left) 2024-09-12T23:42:51.325 INFO:tasks.rados.rados.0.smithi136.stdout:1349: copy_from oid 42 from oid 32 current snap is 131 2024-09-12T23:42:51.325 INFO:tasks.rados.rados.0.smithi136.stdout:1343: finishing rollback tid 0 to smithi13667081-27 2024-09-12T23:42:51.325 INFO:tasks.rados.rados.0.smithi136.stdout:1343: finishing rollback tid 1 to smithi13667081-27 2024-09-12T23:42:51.325 INFO:tasks.rados.rados.0.smithi136.stdout:1343: finishing rollback tid 2 to smithi13667081-27 2024-09-12T23:42:51.325 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 27 v 911 (ObjNum 308 snap 86 seq_num 308) dirty exists 2024-09-12T23:42:51.325 INFO:tasks.rados.rados.0.smithi136.stdout:1343: done (5 left) 2024-09-12T23:42:51.325 INFO:tasks.rados.rados.0.smithi136.stdout:1350: snap_create 2024-09-12T23:42:51.325 INFO:tasks.rados.rados.0.smithi136.stdout:1340: finishing copy_from racing read to smithi13667081-38 2024-09-12T23:42:51.325 INFO:tasks.rados.rados.0.smithi136.stdout:1340: finishing copy_from to smithi13667081-38 2024-09-12T23:42:51.325 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 38 v 685 (ObjNum 398 snap 116 seq_num 398) dirty exists 2024-09-12T23:42:51.326 INFO:tasks.rados.rados.0.smithi136.stdout:1347: finishing rollback tid 1 to smithi13667081-29 2024-09-12T23:42:51.326 INFO:tasks.rados.rados.0.smithi136.stdout:1347: finishing rollback tid 2 to smithi13667081-29 2024-09-12T23:42:51.326 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 29 v 687 (ObjNum 423 snap 125 seq_num 423) dirty exists 2024-09-12T23:42:51.326 INFO:tasks.rados.rados.0.smithi136.stdout:1346: finishing rollback tid 0 to smithi13667081-6 2024-09-12T23:42:51.326 INFO:tasks.rados.rados.0.smithi136.stdout:1346: finishing rollback tid 1 to smithi13667081-6 2024-09-12T23:42:51.326 INFO:tasks.rados.rados.0.smithi136.stdout:1346: finishing rollback tid 2 to smithi13667081-6 2024-09-12T23:42:51.326 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 6 v 296 (ObjNum 420 snap 122 seq_num 420) dirty exists 2024-09-12T23:42:51.326 INFO:tasks.rados.rados.0.smithi136.stdout:1345: finishing copy_from racing read to smithi13667081-3 2024-09-12T23:42:51.326 INFO:tasks.rados.rados.0.smithi136.stdout:1345: finishing copy_from to smithi13667081-3 2024-09-12T23:42:51.326 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 3 v 328 (ObjNum 323 snap 88 seq_num 323) dirty exists 2024-09-12T23:42:51.330 INFO:tasks.rados.rados.0.smithi136.stdout:1349: finishing copy_from to smithi13667081-42 2024-09-12T23:42:51.330 INFO:tasks.rados.rados.0.smithi136.stdout:1349: got expected ENOENT (src dne) 2024-09-12T23:42:51.330 INFO:tasks.rados.rados.0.smithi136.stdout:1349: finishing copy_from racing read to smithi13667081-42 2024-09-12T23:42:51.389 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:51 smithi136 bash[75164]: cluster 2024-09-12T23:42:50.320489+0000 mon.a (mon.0) 127 : cluster [DBG] osdmap e325: 8 total, 8 up, 8 in 2024-09-12T23:42:51.389 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:51 smithi136 bash[75164]: cluster 2024-09-12T23:42:50.656635+0000 mgr.y (mgr.24661) 40 : cluster [DBG] pgmap v58: 121 pgs: 121 active+clean; 292 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 6.7 MiB/s rd, 3.6 MiB/s wr, 20 op/s 2024-09-12T23:42:51.389 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:51 smithi136 bash[76371]: cluster 2024-09-12T23:42:50.320489+0000 mon.a (mon.0) 127 : cluster [DBG] osdmap e325: 8 total, 8 up, 8 in 2024-09-12T23:42:51.390 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:51 smithi136 bash[76371]: cluster 2024-09-12T23:42:50.656635+0000 mgr.y (mgr.24661) 40 : cluster [DBG] pgmap v58: 121 pgs: 121 active+clean; 292 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 6.7 MiB/s rd, 3.6 MiB/s wr, 20 op/s 2024-09-12T23:42:51.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:51 smithi181 bash[24776]: cluster 2024-09-12T23:42:50.320489+0000 mon.a (mon.0) 127 : cluster [DBG] osdmap e325: 8 total, 8 up, 8 in 2024-09-12T23:42:51.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:51 smithi181 bash[24776]: cluster 2024-09-12T23:42:50.656635+0000 mgr.y (mgr.24661) 40 : cluster [DBG] pgmap v58: 121 pgs: 121 active+clean; 292 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 6.7 MiB/s rd, 3.6 MiB/s wr, 20 op/s 2024-09-12T23:42:52.334 INFO:tasks.rados.rados.0.smithi136.stdout:1340: done (5 left) 2024-09-12T23:42:52.335 INFO:tasks.rados.rados.0.smithi136.stdout:1345: done (4 left) 2024-09-12T23:42:52.335 INFO:tasks.rados.rados.0.smithi136.stdout:1346: done (3 left) 2024-09-12T23:42:52.335 INFO:tasks.rados.rados.0.smithi136.stdout:1347: done (2 left) 2024-09-12T23:42:52.335 INFO:tasks.rados.rados.0.smithi136.stdout:1349: done (1 left) 2024-09-12T23:42:52.335 INFO:tasks.rados.rados.0.smithi136.stdout:1350: done (0 left) 2024-09-12T23:42:52.335 INFO:tasks.rados.rados.0.smithi136.stdout:1351: copy_from oid 35 from oid 22 current snap is 132 2024-09-12T23:42:52.335 INFO:tasks.rados.rados.0.smithi136.stdout:1352: rollback oid 20 current snap is 132 2024-09-12T23:42:52.335 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 20 to 116 2024-09-12T23:42:52.335 INFO:tasks.rados.rados.0.smithi136.stdout:1353: delete oid 27 current snap is 132 2024-09-12T23:42:52.336 INFO:tasks.rados.rados.0.smithi136.stdout:1352: finishing rollback tid 1 to smithi13667081-20 2024-09-12T23:42:52.336 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 20 v 0 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-12T23:42:52.346 INFO:tasks.rados.rados.0.smithi136.stdout:1351: finishing copy_from to smithi13667081-35 2024-09-12T23:42:52.346 INFO:tasks.rados.rados.0.smithi136.stdout:1351: got expected ENOENT (src dne) 2024-09-12T23:42:52.346 INFO:tasks.rados.rados.0.smithi136.stdout:1351: finishing copy_from racing read to smithi13667081-35 2024-09-12T23:42:52.348 INFO:tasks.rados.rados.0.smithi136.stdout:1351: done (2 left) 2024-09-12T23:42:52.348 INFO:tasks.rados.rados.0.smithi136.stdout:1352: done (1 left) 2024-09-12T23:42:52.348 INFO:tasks.rados.rados.0.smithi136.stdout:1353: done (0 left) 2024-09-12T23:42:52.349 INFO:tasks.rados.rados.0.smithi136.stdout:1354: rollback oid 5 current snap is 132 2024-09-12T23:42:52.349 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 5 to 128 2024-09-12T23:42:52.349 INFO:tasks.rados.rados.0.smithi136.stdout:1355: delete oid 50 current snap is 132 2024-09-12T23:42:52.355 INFO:tasks.rados.rados.0.smithi136.stdout:1355: done (1 left) 2024-09-12T23:42:52.355 INFO:tasks.rados.rados.0.smithi136.stdout:1356: copy_from oid 1 from oid 47 current snap is 132 2024-09-12T23:42:52.356 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 49 current snap is 132 2024-09-12T23:42:52.356 INFO:tasks.rados.rados.0.smithi136.stdout:1357: seq_num 440 ranges {2146304=172032} 2024-09-12T23:42:52.391 INFO:tasks.rados.rados.0.smithi136.stdout:1357: writing smithi13667081-49 from 2146304 to 2318336 tid 1 2024-09-12T23:42:52.392 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 29 current snap is 132 2024-09-12T23:42:52.392 INFO:tasks.rados.rados.0.smithi136.stdout:1358: seq_num 441 ranges {6168576=188416} 2024-09-12T23:42:52.446 INFO:tasks.rados.rados.0.smithi136.stdout:1358: writing smithi13667081-29 from 6168576 to 6356992 tid 1 2024-09-12T23:42:52.446 INFO:tasks.rados.rados.0.smithi136.stdout:1359: rollback oid 13 current snap is 132 2024-09-12T23:42:52.446 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 13 to 125 2024-09-12T23:42:52.446 INFO:tasks.rados.rados.0.smithi136.stdout:1356: finishing copy_from to smithi13667081-1 2024-09-12T23:42:52.446 INFO:tasks.rados.rados.0.smithi136.stdout:1356: got expected ENOENT (src dne) 2024-09-12T23:42:52.446 INFO:tasks.rados.rados.0.smithi136.stdout:1356: finishing copy_from racing read to smithi13667081-1 2024-09-12T23:42:52.446 INFO:tasks.rados.rados.0.smithi136.stdout:1354: finishing rollback tid 0 to smithi13667081-5 2024-09-12T23:42:52.446 INFO:tasks.rados.rados.0.smithi136.stdout:1354: finishing rollback tid 1 to smithi13667081-5 2024-09-12T23:42:52.446 INFO:tasks.rados.rados.0.smithi136.stdout:1354: finishing rollback tid 2 to smithi13667081-5 2024-09-12T23:42:52.447 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 5 v 925 (ObjNum 372 snap 108 seq_num 372) dirty exists 2024-09-12T23:42:52.447 INFO:tasks.rados.rados.0.smithi136.stdout:1357: finishing write tid 1 to smithi13667081-49 2024-09-12T23:42:52.447 INFO:tasks.rados.rados.0.smithi136.stdout:1357: finishing write tid 2 to smithi13667081-49 2024-09-12T23:42:52.447 INFO:tasks.rados.rados.0.smithi136.stdout:1357: finishing write tid 3 to smithi13667081-49 2024-09-12T23:42:52.447 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 49 v 642 (ObjNum 440 snap 132 seq_num 440) dirty exists 2024-09-12T23:42:52.447 INFO:tasks.rados.rados.0.smithi136.stdout:1357: left oid 49 (ObjNum 440 snap 132 seq_num 440) 2024-09-12T23:42:52.447 INFO:tasks.rados.rados.0.smithi136.stdout:1354: done (4 left) 2024-09-12T23:42:52.447 INFO:tasks.rados.rados.0.smithi136.stdout:1356: done (3 left) 2024-09-12T23:42:52.447 INFO:tasks.rados.rados.0.smithi136.stdout:1357: done (2 left) 2024-09-12T23:42:52.447 INFO:tasks.rados.rados.0.smithi136.stdout:1360: read oid 10 snap -1 2024-09-12T23:42:52.447 INFO:tasks.rados.rados.0.smithi136.stdout:1360: expect (ObjNum 429 snap 127 seq_num 429) 2024-09-12T23:42:52.447 INFO:tasks.rados.rados.0.smithi136.stdout:1361: rmattr oid 12 current snap is 132 2024-09-12T23:42:52.447 INFO:tasks.rados.rados.0.smithi136.stdout:1361: done (3 left) 2024-09-12T23:42:52.448 INFO:tasks.rados.rados.0.smithi136.stdout:1362: delete oid 20 current snap is 132 2024-09-12T23:42:52.448 INFO:tasks.rados.rados.0.smithi136.stdout:1362: done (3 left) 2024-09-12T23:42:52.448 INFO:tasks.rados.rados.0.smithi136.stdout:1363: snap_create 2024-09-12T23:42:52.459 INFO:tasks.rados.rados.0.smithi136.stdout:1358: finishing write tid 1 to smithi13667081-29 2024-09-12T23:42:52.459 INFO:tasks.rados.rados.0.smithi136.stdout:1358: finishing write tid 2 to smithi13667081-29 2024-09-12T23:42:52.460 INFO:tasks.rados.rados.0.smithi136.stdout:1359: finishing rollback tid 0 to smithi13667081-13 2024-09-12T23:42:52.462 INFO:tasks.rados.rados.0.smithi136.stdout:1358: finishing write tid 3 to smithi13667081-29 2024-09-12T23:42:52.462 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 29 v 690 (ObjNum 441 snap 132 seq_num 441) dirty exists 2024-09-12T23:42:52.462 INFO:tasks.rados.rados.0.smithi136.stdout:1358: left oid 29 (ObjNum 441 snap 132 seq_num 441) 2024-09-12T23:42:52.490 INFO:tasks.rados.rados.0.smithi136.stdout:1359: finishing rollback tid 1 to smithi13667081-13 2024-09-12T23:42:52.490 INFO:tasks.rados.rados.0.smithi136.stdout:1359: finishing rollback tid 2 to smithi13667081-13 2024-09-12T23:42:52.490 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 13 v 693 (ObjNum 262 snap 71 seq_num 262) dirty exists 2024-09-12T23:42:52.516 INFO:tasks.rados.rados.0.smithi136.stdout:1360: expect (ObjNum 429 snap 127 seq_num 429) 2024-09-12T23:42:52.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:52 smithi181 bash[24776]: cluster 2024-09-12T23:42:51.328021+0000 mon.a (mon.0) 128 : cluster [DBG] osdmap e326: 8 total, 8 up, 8 in 2024-09-12T23:42:52.594 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:42:52 smithi181 bash[60471]: ts=2024-09-12T23:42:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:42:52.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:52 smithi136 bash[75164]: cluster 2024-09-12T23:42:51.328021+0000 mon.a (mon.0) 128 : cluster [DBG] osdmap e326: 8 total, 8 up, 8 in 2024-09-12T23:42:52.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:52 smithi136 bash[76371]: cluster 2024-09-12T23:42:51.328021+0000 mon.a (mon.0) 128 : cluster [DBG] osdmap e326: 8 total, 8 up, 8 in 2024-09-12T23:42:53.343 INFO:tasks.rados.rados.0.smithi136.stdout:1358: done (3 left) 2024-09-12T23:42:53.343 INFO:tasks.rados.rados.0.smithi136.stdout:1359: done (2 left) 2024-09-12T23:42:53.343 INFO:tasks.rados.rados.0.smithi136.stdout:1360: done (1 left) 2024-09-12T23:42:53.343 INFO:tasks.rados.rados.0.smithi136.stdout:1363: done (0 left) 2024-09-12T23:42:53.343 INFO:tasks.rados.rados.0.smithi136.stdout:1364: read oid 10 snap -1 2024-09-12T23:42:53.343 INFO:tasks.rados.rados.0.smithi136.stdout:1364: expect (ObjNum 429 snap 127 seq_num 429) 2024-09-12T23:42:53.343 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 15 current snap is 133 2024-09-12T23:42:53.343 INFO:tasks.rados.rados.0.smithi136.stdout:1365: seq_num 443 ranges {8478720=221184} 2024-09-12T23:42:53.416 INFO:tasks.rados.rados.0.smithi136.stdout:1365: writing smithi13667081-15 from 8478720 to 8699904 tid 1 2024-09-12T23:42:53.416 INFO:tasks.rados.rados.0.smithi136.stdout:1366: snap_create 2024-09-12T23:42:53.417 INFO:tasks.rados.rados.0.smithi136.stdout:1364: expect (ObjNum 429 snap 127 seq_num 429) 2024-09-12T23:42:53.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:53 smithi181 bash[24776]: cluster 2024-09-12T23:42:52.338312+0000 mon.a (mon.0) 129 : cluster [DBG] osdmap e327: 8 total, 8 up, 8 in 2024-09-12T23:42:53.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:53 smithi181 bash[24776]: cluster 2024-09-12T23:42:52.657340+0000 mgr.y (mgr.24661) 41 : cluster [DBG] pgmap v61: 121 pgs: 121 active+clean; 292 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 6.7 MiB/s rd, 3.6 MiB/s wr, 20 op/s 2024-09-12T23:42:53.594 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:42:53 smithi181 bash[60471]: ts=2024-09-12T23:42:53.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi136\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:42:53.755 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:53 smithi136 bash[76371]: cluster 2024-09-12T23:42:52.338312+0000 mon.a (mon.0) 129 : cluster [DBG] osdmap e327: 8 total, 8 up, 8 in 2024-09-12T23:42:53.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:53 smithi136 bash[76371]: cluster 2024-09-12T23:42:52.657340+0000 mgr.y (mgr.24661) 41 : cluster [DBG] pgmap v61: 121 pgs: 121 active+clean; 292 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 6.7 MiB/s rd, 3.6 MiB/s wr, 20 op/s 2024-09-12T23:42:53.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:53 smithi136 bash[75164]: cluster 2024-09-12T23:42:52.338312+0000 mon.a (mon.0) 129 : cluster [DBG] osdmap e327: 8 total, 8 up, 8 in 2024-09-12T23:42:53.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:53 smithi136 bash[75164]: cluster 2024-09-12T23:42:52.657340+0000 mgr.y (mgr.24661) 41 : cluster [DBG] pgmap v61: 121 pgs: 121 active+clean; 292 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 6.7 MiB/s rd, 3.6 MiB/s wr, 20 op/s 2024-09-12T23:42:53.914 INFO:tasks.rados.rados.0.smithi136.stdout:1365: finishing write tid 1 to smithi13667081-15 2024-09-12T23:42:53.915 INFO:tasks.rados.rados.0.smithi136.stdout:1365: finishing write tid 2 to smithi13667081-15 2024-09-12T23:42:53.915 INFO:tasks.rados.rados.0.smithi136.stdout:1365: finishing write tid 3 to smithi13667081-15 2024-09-12T23:42:53.915 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 15 v 645 (ObjNum 443 snap 133 seq_num 443) dirty exists 2024-09-12T23:42:53.915 INFO:tasks.rados.rados.0.smithi136.stdout:1365: left oid 15 (ObjNum 443 snap 133 seq_num 443) 2024-09-12T23:42:54.359 INFO:tasks.rados.rados.0.smithi136.stdout:1364: done (2 left) 2024-09-12T23:42:54.362 INFO:tasks.rados.rados.0.smithi136.stdout:1365: done (1 left) 2024-09-12T23:42:54.362 INFO:tasks.rados.rados.0.smithi136.stdout:1366: done (0 left) 2024-09-12T23:42:54.362 INFO:tasks.rados.rados.0.smithi136.stdout:1367: snap_create 2024-09-12T23:42:54.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:54 smithi181 bash[24776]: cluster 2024-09-12T23:42:53.346468+0000 mon.a (mon.0) 130 : cluster [DBG] osdmap e328: 8 total, 8 up, 8 in 2024-09-12T23:42:54.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:54 smithi181 bash[24776]: audit 2024-09-12T23:42:53.726848+0000 mon.a (mon.0) 131 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:42:54.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:54 smithi181 bash[24776]: audit 2024-09-12T23:42:53.727739+0000 mon.a (mon.0) 132 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:42:54.594 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:42:54 smithi181 bash[60471]: ts=2024-09-12T23:42:54.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:42:54.595 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:42:54 smithi181 bash[60471]: ts=2024-09-12T23:42:54.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:42:54.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:54 smithi136 bash[75164]: cluster 2024-09-12T23:42:53.346468+0000 mon.a (mon.0) 130 : cluster [DBG] osdmap e328: 8 total, 8 up, 8 in 2024-09-12T23:42:54.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:54 smithi136 bash[75164]: audit 2024-09-12T23:42:53.726848+0000 mon.a (mon.0) 131 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:42:54.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:54 smithi136 bash[75164]: audit 2024-09-12T23:42:53.727739+0000 mon.a (mon.0) 132 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:42:54.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:54 smithi136 bash[76371]: cluster 2024-09-12T23:42:53.346468+0000 mon.a (mon.0) 130 : cluster [DBG] osdmap e328: 8 total, 8 up, 8 in 2024-09-12T23:42:54.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:54 smithi136 bash[76371]: audit 2024-09-12T23:42:53.726848+0000 mon.a (mon.0) 131 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:42:54.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:54 smithi136 bash[76371]: audit 2024-09-12T23:42:53.727739+0000 mon.a (mon.0) 132 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:42:55.361 INFO:tasks.rados.rados.0.smithi136.stdout:1367: done (0 left) 2024-09-12T23:42:55.361 INFO:tasks.rados.rados.0.smithi136.stdout:1368: read oid 17 snap 127 2024-09-12T23:42:55.361 INFO:tasks.rados.rados.0.smithi136.stdout:1368: expect (ObjNum 430 snap 127 seq_num 430) 2024-09-12T23:42:55.361 INFO:tasks.rados.rados.0.smithi136.stdout:1369: delete oid 15 current snap is 135 2024-09-12T23:42:55.371 INFO:tasks.rados.rados.0.smithi136.stdout:1369: done (1 left) 2024-09-12T23:42:55.372 INFO:tasks.rados.rados.0.smithi136.stdout:1370: read oid 2 snap 131 2024-09-12T23:42:55.372 INFO:tasks.rados.rados.0.smithi136.stdout:1370: expect (ObjNum 416 snap 120 seq_num 416) 2024-09-12T23:42:55.372 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 8 current snap is 135 2024-09-12T23:42:55.372 INFO:tasks.rados.rados.0.smithi136.stdout:1371: seq_num 445 ranges {4775936=253952} 2024-09-12T23:42:55.426 INFO:tasks.rados.rados.0.smithi136.stdout:1371: writing smithi13667081-8 from 4775936 to 5029888 tid 1 2024-09-12T23:42:55.426 INFO:tasks.rados.rados.0.smithi136.stdout:1372: rollback oid 35 current snap is 135 2024-09-12T23:42:55.426 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 35 to 118 2024-09-12T23:42:55.426 INFO:tasks.rados.rados.0.smithi136.stdout:1368: expect (ObjNum 430 snap 127 seq_num 430) 2024-09-12T23:42:55.426 INFO:tasks.rados.rados.0.smithi136.stdout:1370: expect (ObjNum 416 snap 120 seq_num 416) 2024-09-12T23:42:55.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:55 smithi181 bash[24776]: cluster 2024-09-12T23:42:54.362522+0000 mon.a (mon.0) 133 : cluster [DBG] osdmap e329: 8 total, 8 up, 8 in 2024-09-12T23:42:55.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:55 smithi181 bash[24776]: audit 2024-09-12T23:42:54.631770+0000 mon.a (mon.0) 134 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:42:55.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:55 smithi181 bash[24776]: audit 2024-09-12T23:42:54.641898+0000 mon.a (mon.0) 135 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:42:55.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:55 smithi181 bash[24776]: audit 2024-09-12T23:42:54.643873+0000 mon.a (mon.0) 136 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:42:55.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:55 smithi181 bash[24776]: audit 2024-09-12T23:42:54.645416+0000 mon.a (mon.0) 137 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:42:55.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:55 smithi181 bash[24776]: audit 2024-09-12T23:42:54.656030+0000 mon.a (mon.0) 138 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:42:55.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:55 smithi181 bash[24776]: cluster 2024-09-12T23:42:54.658936+0000 mgr.y (mgr.24661) 42 : cluster [DBG] pgmap v64: 121 pgs: 121 active+clean; 300 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 14 MiB/s rd, 652 KiB/s wr, 7 op/s 2024-09-12T23:42:55.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:55 smithi181 bash[24776]: audit 2024-09-12T23:42:54.809818+0000 mon.a (mon.0) 139 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:42:55.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:55 smithi181 bash[24776]: audit 2024-09-12T23:42:54.814126+0000 mon.a (mon.0) 140 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:42:55.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:55 smithi181 bash[24776]: audit 2024-09-12T23:42:54.816261+0000 mon.a (mon.0) 141 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-12T23:42:55.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:55 smithi181 bash[24776]: audit 2024-09-12T23:42:54.817461+0000 mon.a (mon.0) 142 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-09-12T23:42:55.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:55 smithi181 bash[24776]: cephadm 2024-09-12T23:42:54.818290+0000 mgr.y (mgr.24661) 43 : cephadm [INF] Upgrade: It appears safe to stop mon.b 2024-09-12T23:42:55.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:55 smithi136 bash[75164]: cluster 2024-09-12T23:42:54.362522+0000 mon.a (mon.0) 133 : cluster [DBG] osdmap e329: 8 total, 8 up, 8 in 2024-09-12T23:42:55.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:55 smithi136 bash[75164]: audit 2024-09-12T23:42:54.631770+0000 mon.a (mon.0) 134 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:42:55.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:55 smithi136 bash[75164]: audit 2024-09-12T23:42:54.641898+0000 mon.a (mon.0) 135 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:42:55.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:55 smithi136 bash[75164]: audit 2024-09-12T23:42:54.643873+0000 mon.a (mon.0) 136 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:42:55.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:55 smithi136 bash[75164]: audit 2024-09-12T23:42:54.645416+0000 mon.a (mon.0) 137 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:42:55.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:55 smithi136 bash[75164]: audit 2024-09-12T23:42:54.656030+0000 mon.a (mon.0) 138 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:42:55.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:55 smithi136 bash[75164]: cluster 2024-09-12T23:42:54.658936+0000 mgr.y (mgr.24661) 42 : cluster [DBG] pgmap v64: 121 pgs: 121 active+clean; 300 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 14 MiB/s rd, 652 KiB/s wr, 7 op/s 2024-09-12T23:42:55.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:55 smithi136 bash[75164]: audit 2024-09-12T23:42:54.809818+0000 mon.a (mon.0) 139 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:42:55.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:55 smithi136 bash[75164]: audit 2024-09-12T23:42:54.814126+0000 mon.a (mon.0) 140 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:42:55.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:55 smithi136 bash[75164]: audit 2024-09-12T23:42:54.816261+0000 mon.a (mon.0) 141 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-12T23:42:55.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:55 smithi136 bash[75164]: audit 2024-09-12T23:42:54.817461+0000 mon.a (mon.0) 142 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-09-12T23:42:55.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:55 smithi136 bash[75164]: cephadm 2024-09-12T23:42:54.818290+0000 mgr.y (mgr.24661) 43 : cephadm [INF] Upgrade: It appears safe to stop mon.b 2024-09-12T23:42:55.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:55 smithi136 bash[76371]: cluster 2024-09-12T23:42:54.362522+0000 mon.a (mon.0) 133 : cluster [DBG] osdmap e329: 8 total, 8 up, 8 in 2024-09-12T23:42:55.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:55 smithi136 bash[76371]: audit 2024-09-12T23:42:54.631770+0000 mon.a (mon.0) 134 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:42:55.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:55 smithi136 bash[76371]: audit 2024-09-12T23:42:54.641898+0000 mon.a (mon.0) 135 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:42:55.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:55 smithi136 bash[76371]: audit 2024-09-12T23:42:54.643873+0000 mon.a (mon.0) 136 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:42:55.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:55 smithi136 bash[76371]: audit 2024-09-12T23:42:54.645416+0000 mon.a (mon.0) 137 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:42:55.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:55 smithi136 bash[76371]: audit 2024-09-12T23:42:54.656030+0000 mon.a (mon.0) 138 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:42:55.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:55 smithi136 bash[76371]: cluster 2024-09-12T23:42:54.658936+0000 mgr.y (mgr.24661) 42 : cluster [DBG] pgmap v64: 121 pgs: 121 active+clean; 300 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 14 MiB/s rd, 652 KiB/s wr, 7 op/s 2024-09-12T23:42:55.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:55 smithi136 bash[76371]: audit 2024-09-12T23:42:54.809818+0000 mon.a (mon.0) 139 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:42:55.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:55 smithi136 bash[76371]: audit 2024-09-12T23:42:54.814126+0000 mon.a (mon.0) 140 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:42:55.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:55 smithi136 bash[76371]: audit 2024-09-12T23:42:54.816261+0000 mon.a (mon.0) 141 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-12T23:42:55.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:55 smithi136 bash[76371]: audit 2024-09-12T23:42:54.817461+0000 mon.a (mon.0) 142 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-09-12T23:42:55.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:55 smithi136 bash[76371]: cephadm 2024-09-12T23:42:54.818290+0000 mgr.y (mgr.24661) 43 : cephadm [INF] Upgrade: It appears safe to stop mon.b 2024-09-12T23:42:55.837 INFO:tasks.rados.rados.0.smithi136.stdout:1372: finishing rollback tid 0 to smithi13667081-35 2024-09-12T23:42:55.838 INFO:tasks.rados.rados.0.smithi136.stdout:1371: finishing write tid 1 to smithi13667081-8 2024-09-12T23:42:55.838 INFO:tasks.rados.rados.0.smithi136.stdout:1371: finishing write tid 2 to smithi13667081-8 2024-09-12T23:42:55.838 INFO:tasks.rados.rados.0.smithi136.stdout:1372: finishing rollback tid 1 to smithi13667081-35 2024-09-12T23:42:55.838 INFO:tasks.rados.rados.0.smithi136.stdout:1372: finishing rollback tid 2 to smithi13667081-35 2024-09-12T23:42:55.838 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 35 v 929 (ObjNum 373 snap 108 seq_num 373) dirty exists 2024-09-12T23:42:55.838 INFO:tasks.rados.rados.0.smithi136.stdout:1371: finishing write tid 3 to smithi13667081-8 2024-09-12T23:42:55.838 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 8 v 687 (ObjNum 445 snap 135 seq_num 445) dirty exists 2024-09-12T23:42:55.838 INFO:tasks.rados.rados.0.smithi136.stdout:1371: left oid 8 (ObjNum 445 snap 135 seq_num 445) 2024-09-12T23:42:55.838 INFO:tasks.rados.rados.0.smithi136.stdout:1368: done (3 left) 2024-09-12T23:42:55.838 INFO:tasks.rados.rados.0.smithi136.stdout:1370: done (2 left) 2024-09-12T23:42:55.838 INFO:tasks.rados.rados.0.smithi136.stdout:1371: done (1 left) 2024-09-12T23:42:55.838 INFO:tasks.rados.rados.0.smithi136.stdout:1372: done (0 left) 2024-09-12T23:42:55.839 INFO:tasks.rados.rados.0.smithi136.stdout:1373: rmattr oid 35 current snap is 135 2024-09-12T23:42:55.839 INFO:tasks.rados.rados.0.smithi136.stdout:1373: done (0 left) 2024-09-12T23:42:55.839 INFO:tasks.rados.rados.0.smithi136.stdout:1374: setattr oid 19 current snap is 135 2024-09-12T23:42:55.840 INFO:tasks.rados.rados.0.smithi136.stdout:1375: snap_remove snap 87 2024-09-12T23:42:56.292 INFO:tasks.rados.rados.0.smithi136.stdout:1375: done (1 left) 2024-09-12T23:42:56.293 INFO:tasks.rados.rados.0.smithi136.stdout:1376: rmattr oid 44 current snap is 135 2024-09-12T23:42:56.293 INFO:tasks.rados.rados.0.smithi136.stdout:1377: snap_remove snap 116 2024-09-12T23:42:56.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:56 smithi181 bash[24776]: cluster 2024-09-12T23:42:55.364720+0000 mon.a (mon.0) 143 : cluster [DBG] osdmap e330: 8 total, 8 up, 8 in 2024-09-12T23:42:56.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:56 smithi181 bash[24776]: cluster 2024-09-12T23:42:56.294949+0000 mon.a (mon.0) 144 : cluster [DBG] osdmap e331: 8 total, 8 up, 8 in 2024-09-12T23:42:56.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:56 smithi136 bash[75164]: cluster 2024-09-12T23:42:55.364720+0000 mon.a (mon.0) 143 : cluster [DBG] osdmap e330: 8 total, 8 up, 8 in 2024-09-12T23:42:56.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:56 smithi136 bash[75164]: cluster 2024-09-12T23:42:56.294949+0000 mon.a (mon.0) 144 : cluster [DBG] osdmap e331: 8 total, 8 up, 8 in 2024-09-12T23:42:56.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:56 smithi136 bash[76371]: cluster 2024-09-12T23:42:55.364720+0000 mon.a (mon.0) 143 : cluster [DBG] osdmap e330: 8 total, 8 up, 8 in 2024-09-12T23:42:56.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:56 smithi136 bash[76371]: cluster 2024-09-12T23:42:56.294949+0000 mon.a (mon.0) 144 : cluster [DBG] osdmap e331: 8 total, 8 up, 8 in 2024-09-12T23:42:57.303 INFO:tasks.rados.rados.0.smithi136.stdout:1377: done (2 left) 2024-09-12T23:42:57.303 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 27 current snap is 135 2024-09-12T23:42:57.303 INFO:tasks.rados.rados.0.smithi136.stdout:1378: seq_num 446 ranges {0=278528} 2024-09-12T23:42:57.305 INFO:tasks.rados.rados.0.smithi136.stdout:1378: writing smithi13667081-27 from 0 to 278528 tid 1 2024-09-12T23:42:57.305 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 17 current snap is 135 2024-09-12T23:42:57.305 INFO:tasks.rados.rados.0.smithi136.stdout:1379: seq_num 447 ranges {8192=294912} 2024-09-12T23:42:57.308 INFO:tasks.rados.rados.0.smithi136.stdout:1379: writing smithi13667081-17 from 8192 to 303104 tid 1 2024-09-12T23:42:57.308 INFO:tasks.rados.rados.0.smithi136.stdout:1380: rollback oid 16 current snap is 135 2024-09-12T23:42:57.308 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 16 to 128 2024-09-12T23:42:57.308 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 19 v 609 (ObjNum 391 snap 113 seq_num 391) dirty exists 2024-09-12T23:42:57.308 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 44 v 779 (ObjNum 323 snap 88 seq_num 323) dirty exists 2024-09-12T23:42:57.308 INFO:tasks.rados.rados.0.smithi136.stdout:1374: done (4 left) 2024-09-12T23:42:57.309 INFO:tasks.rados.rados.0.smithi136.stdout:1376: done (3 left) 2024-09-12T23:42:57.309 INFO:tasks.rados.rados.0.smithi136.stdout:1381: rollback oid 8 current snap is 135 2024-09-12T23:42:57.309 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 8 to 93 2024-09-12T23:42:57.309 INFO:tasks.rados.rados.0.smithi136.stdout:1382: rmattr oid 35 current snap is 135 2024-09-12T23:42:57.309 INFO:tasks.rados.rados.0.smithi136.stdout:1383: rmattr oid 12 current snap is 135 2024-09-12T23:42:57.309 INFO:tasks.rados.rados.0.smithi136.stdout:1383: done (5 left) 2024-09-12T23:42:57.309 INFO:tasks.rados.rados.0.smithi136.stdout:1384: delete oid 6 current snap is 135 2024-09-12T23:42:57.316 INFO:tasks.rados.rados.0.smithi136.stdout:1384: done (5 left) 2024-09-12T23:42:57.316 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 1 current snap is 135 2024-09-12T23:42:57.316 INFO:tasks.rados.rados.0.smithi136.stdout:1385: seq_num 449 ranges {3514368=327680} 2024-09-12T23:42:57.365 INFO:tasks.rados.rados.0.smithi136.stdout:1385: writing smithi13667081-1 from 3514368 to 3842048 tid 1 2024-09-12T23:42:57.366 INFO:tasks.rados.rados.0.smithi136.stdout:1386: snap_remove snap 133 2024-09-12T23:42:57.366 INFO:tasks.rados.rados.0.smithi136.stdout:1381: finishing rollback tid 0 to smithi13667081-8 2024-09-12T23:42:57.366 INFO:tasks.rados.rados.0.smithi136.stdout:1381: finishing rollback tid 1 to smithi13667081-8 2024-09-12T23:42:57.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:57 smithi136 bash[75164]: cluster 2024-09-12T23:42:56.659713+0000 mgr.y (mgr.24661) 44 : cluster [DBG] pgmap v67: 121 pgs: 121 active+clean; 300 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 14 MiB/s rd, 652 KiB/s wr, 7 op/s 2024-09-12T23:42:57.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:57 smithi136 bash[75164]: audit 2024-09-12T23:42:57.107924+0000 mon.a (mon.0) 145 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:42:57.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:57 smithi136 bash[75164]: audit 2024-09-12T23:42:57.109855+0000 mon.a (mon.0) 146 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-12T23:42:57.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:57 smithi136 bash[75164]: audit 2024-09-12T23:42:57.111375+0000 mon.a (mon.0) 147 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-12T23:42:57.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:57 smithi136 bash[75164]: audit 2024-09-12T23:42:57.112884+0000 mon.a (mon.0) 148 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:42:57.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:57 smithi136 bash[75164]: cluster 2024-09-12T23:42:57.305372+0000 mon.a (mon.0) 149 : cluster [DBG] osdmap e332: 8 total, 8 up, 8 in 2024-09-12T23:42:57.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:57 smithi136 bash[76371]: cluster 2024-09-12T23:42:56.659713+0000 mgr.y (mgr.24661) 44 : cluster [DBG] pgmap v67: 121 pgs: 121 active+clean; 300 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 14 MiB/s rd, 652 KiB/s wr, 7 op/s 2024-09-12T23:42:57.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:57 smithi136 bash[76371]: audit 2024-09-12T23:42:57.107924+0000 mon.a (mon.0) 145 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:42:57.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:57 smithi136 bash[76371]: audit 2024-09-12T23:42:57.109855+0000 mon.a (mon.0) 146 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-12T23:42:57.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:57 smithi136 bash[76371]: audit 2024-09-12T23:42:57.111375+0000 mon.a (mon.0) 147 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-12T23:42:57.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:57 smithi136 bash[76371]: audit 2024-09-12T23:42:57.112884+0000 mon.a (mon.0) 148 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:42:57.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:57 smithi136 bash[76371]: cluster 2024-09-12T23:42:57.305372+0000 mon.a (mon.0) 149 : cluster [DBG] osdmap e332: 8 total, 8 up, 8 in 2024-09-12T23:42:57.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:57 smithi181 bash[24776]: cluster 2024-09-12T23:42:56.659713+0000 mgr.y (mgr.24661) 44 : cluster [DBG] pgmap v67: 121 pgs: 121 active+clean; 300 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 14 MiB/s rd, 652 KiB/s wr, 7 op/s 2024-09-12T23:42:57.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:57 smithi181 bash[24776]: audit 2024-09-12T23:42:57.107924+0000 mon.a (mon.0) 145 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:42:57.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:57 smithi181 bash[24776]: audit 2024-09-12T23:42:57.109855+0000 mon.a (mon.0) 146 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-12T23:42:57.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:57 smithi181 bash[24776]: audit 2024-09-12T23:42:57.111375+0000 mon.a (mon.0) 147 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-12T23:42:57.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:57 smithi181 bash[24776]: audit 2024-09-12T23:42:57.112884+0000 mon.a (mon.0) 148 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:42:57.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:57 smithi181 bash[24776]: cluster 2024-09-12T23:42:57.305372+0000 mon.a (mon.0) 149 : cluster [DBG] osdmap e332: 8 total, 8 up, 8 in 2024-09-12T23:42:58.300 INFO:tasks.rados.rados.0.smithi136.stdout:1386: done (6 left) 2024-09-12T23:42:58.300 INFO:tasks.rados.rados.0.smithi136.stdout:1387: read oid 34 snap -1 2024-09-12T23:42:58.300 INFO:tasks.rados.rados.0.smithi136.stdout:1387: expect (ObjNum 432 snap 128 seq_num 432) 2024-09-12T23:42:58.300 INFO:tasks.rados.rados.0.smithi136.stdout:1381: finishing rollback tid 2 to smithi13667081-8 2024-09-12T23:42:58.300 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 8 v 690 (ObjNum 286 snap 78 seq_num 286) dirty exists 2024-09-12T23:42:58.300 INFO:tasks.rados.rados.0.smithi136.stdout:1380: finishing rollback tid 0 to smithi13667081-16 2024-09-12T23:42:58.300 INFO:tasks.rados.rados.0.smithi136.stdout:1380: finishing rollback tid 1 to smithi13667081-16 2024-09-12T23:42:58.300 INFO:tasks.rados.rados.0.smithi136.stdout:1380: finishing rollback tid 2 to smithi13667081-16 2024-09-12T23:42:58.300 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 16 v 791 (ObjNum 427 snap 127 seq_num 427) dirty exists 2024-09-12T23:42:58.301 INFO:tasks.rados.rados.0.smithi136.stdout:1378: finishing write tid 1 to smithi13667081-27 2024-09-12T23:42:58.301 INFO:tasks.rados.rados.0.smithi136.stdout:1378: finishing write tid 2 to smithi13667081-27 2024-09-12T23:42:58.301 INFO:tasks.rados.rados.0.smithi136.stdout:1379: finishing write tid 1 to smithi13667081-17 2024-09-12T23:42:58.301 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 35 v 0 (ObjNum 373 snap 108 seq_num 373) dirty exists 2024-09-12T23:42:58.301 INFO:tasks.rados.rados.0.smithi136.stdout:1378: finishing write tid 3 to smithi13667081-27 2024-09-12T23:42:58.301 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 27 v 949 (ObjNum 446 snap 135 seq_num 446) dirty exists 2024-09-12T23:42:58.301 INFO:tasks.rados.rados.0.smithi136.stdout:1378: left oid 27 (ObjNum 446 snap 135 seq_num 446) 2024-09-12T23:42:58.301 INFO:tasks.rados.rados.0.smithi136.stdout:1379: finishing write tid 2 to smithi13667081-17 2024-09-12T23:42:58.301 INFO:tasks.rados.rados.0.smithi136.stdout:1379: finishing write tid 3 to smithi13667081-17 2024-09-12T23:42:58.301 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 17 v 952 (ObjNum 447 snap 135 seq_num 447) dirty exists 2024-09-12T23:42:58.301 INFO:tasks.rados.rados.0.smithi136.stdout:1379: left oid 17 (ObjNum 447 snap 135 seq_num 447) 2024-09-12T23:42:58.301 INFO:tasks.rados.rados.0.smithi136.stdout:1385: finishing write tid 1 to smithi13667081-1 2024-09-12T23:42:58.301 INFO:tasks.rados.rados.0.smithi136.stdout:1385: finishing write tid 2 to smithi13667081-1 2024-09-12T23:42:58.301 INFO:tasks.rados.rados.0.smithi136.stdout:1385: finishing write tid 3 to smithi13667081-1 2024-09-12T23:42:58.302 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 1 v 728 (ObjNum 449 snap 135 seq_num 449) dirty exists 2024-09-12T23:42:58.302 INFO:tasks.rados.rados.0.smithi136.stdout:1385: left oid 1 (ObjNum 449 snap 135 seq_num 449) 2024-09-12T23:42:58.302 INFO:tasks.rados.rados.0.smithi136.stdout:1378: done (6 left) 2024-09-12T23:42:58.302 INFO:tasks.rados.rados.0.smithi136.stdout:1379: done (5 left) 2024-09-12T23:42:58.302 INFO:tasks.rados.rados.0.smithi136.stdout:1380: done (4 left) 2024-09-12T23:42:58.302 INFO:tasks.rados.rados.0.smithi136.stdout:1381: done (3 left) 2024-09-12T23:42:58.302 INFO:tasks.rados.rados.0.smithi136.stdout:1382: done (2 left) 2024-09-12T23:42:58.302 INFO:tasks.rados.rados.0.smithi136.stdout:1385: done (1 left) 2024-09-12T23:42:58.302 INFO:tasks.rados.rados.0.smithi136.stdout:1388: copy_from oid 30 from oid 16 current snap is 135 2024-09-12T23:42:58.302 INFO:tasks.rados.rados.0.smithi136.stdout:1389: delete oid 46 current snap is 135 2024-09-12T23:42:58.318 INFO:tasks.rados.rados.0.smithi136.stdout:1389: done (2 left) 2024-09-12T23:42:58.318 INFO:tasks.rados.rados.0.smithi136.stdout:1390: copy_from oid 24 from oid 26 current snap is 135 2024-09-12T23:42:58.318 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 41 current snap is 135 2024-09-12T23:42:58.318 INFO:tasks.rados.rados.0.smithi136.stdout:1391: seq_num 451 ranges {4300800=360448} 2024-09-12T23:42:58.353 INFO:tasks.rados.rados.0.smithi136.stdout:1391: writing smithi13667081-41 from 4300800 to 4661248 tid 1 2024-09-12T23:42:58.353 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 38 current snap is 135 2024-09-12T23:42:58.353 INFO:tasks.rados.rados.0.smithi136.stdout:1392: seq_num 452 ranges {4775936=376832} 2024-09-12T23:42:58.388 INFO:tasks.rados.rados.0.smithi136.stdout:1392: writing smithi13667081-38 from 4775936 to 5152768 tid 1 2024-09-12T23:42:58.388 INFO:tasks.rados.rados.0.smithi136.stdout:1393: read oid 19 snap -1 2024-09-12T23:42:58.388 INFO:tasks.rados.rados.0.smithi136.stdout:1393: expect (ObjNum 391 snap 113 seq_num 391) 2024-09-12T23:42:58.388 INFO:tasks.rados.rados.0.smithi136.stdout:1394: setattr oid 48 current snap is 135 2024-09-12T23:42:58.388 INFO:tasks.rados.rados.0.smithi136.stdout:1387: expect (ObjNum 432 snap 128 seq_num 432) 2024-09-12T23:42:58.480 INFO:tasks.rados.rados.0.smithi136.stdout:1390: finishing copy_from to smithi13667081-24 2024-09-12T23:42:58.480 INFO:tasks.rados.rados.0.smithi136.stdout:1390: got expected ENOENT (src dne) 2024-09-12T23:42:58.480 INFO:tasks.rados.rados.0.smithi136.stdout:1390: finishing copy_from racing read to smithi13667081-24 2024-09-12T23:42:58.480 INFO:tasks.rados.rados.0.smithi136.stdout:1388: finishing copy_from racing read to smithi13667081-30 2024-09-12T23:42:58.480 INFO:tasks.rados.rados.0.smithi136.stdout:1388: finishing copy_from to smithi13667081-30 2024-09-12T23:42:58.480 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 30 v 791 (ObjNum 427 snap 127 seq_num 427) dirty exists 2024-09-12T23:42:58.480 INFO:tasks.rados.rados.0.smithi136.stdout:1391: finishing write tid 1 to smithi13667081-41 2024-09-12T23:42:58.480 INFO:tasks.rados.rados.0.smithi136.stdout:1391: finishing write tid 2 to smithi13667081-41 2024-09-12T23:42:58.481 INFO:tasks.rados.rados.0.smithi136.stdout:1391: finishing write tid 3 to smithi13667081-41 2024-09-12T23:42:58.481 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 41 v 670 (ObjNum 451 snap 135 seq_num 451) dirty exists 2024-09-12T23:42:58.481 INFO:tasks.rados.rados.0.smithi136.stdout:1391: left oid 41 (ObjNum 451 snap 135 seq_num 451) 2024-09-12T23:42:58.481 INFO:tasks.rados.rados.0.smithi136.stdout:1392: finishing write tid 1 to smithi13667081-38 2024-09-12T23:42:58.481 INFO:tasks.rados.rados.0.smithi136.stdout:1392: finishing write tid 2 to smithi13667081-38 2024-09-12T23:42:58.481 INFO:tasks.rados.rados.0.smithi136.stdout:1392: finishing write tid 3 to smithi13667081-38 2024-09-12T23:42:58.481 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 38 v 695 (ObjNum 452 snap 135 seq_num 452) dirty exists 2024-09-12T23:42:58.481 INFO:tasks.rados.rados.0.smithi136.stdout:1392: left oid 38 (ObjNum 452 snap 135 seq_num 452) 2024-09-12T23:42:58.481 INFO:tasks.rados.rados.0.smithi136.stdout:1393: expect (ObjNum 391 snap 113 seq_num 391) 2024-09-12T23:42:58.656 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:58 smithi181 bash[24776]: cephadm 2024-09-12T23:42:57.095982+0000 mgr.y (mgr.24661) 45 : cephadm [INF] Upgrade: Updating mon.b 2024-09-12T23:42:58.656 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:58 smithi181 bash[24776]: cephadm 2024-09-12T23:42:57.114697+0000 mgr.y (mgr.24661) 46 : cephadm [INF] Deploying daemon mon.b on smithi181 2024-09-12T23:42:58.656 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:58 smithi181 bash[24776]: cluster 2024-09-12T23:42:58.302452+0000 mon.a (mon.0) 150 : cluster [DBG] osdmap e333: 8 total, 8 up, 8 in 2024-09-12T23:42:58.721 INFO:tasks.rados.rados.0.smithi136.stdout:1387: done (6 left) 2024-09-12T23:42:58.722 INFO:tasks.rados.rados.0.smithi136.stdout:1388: done (5 left) 2024-09-12T23:42:58.722 INFO:tasks.rados.rados.0.smithi136.stdout:1390: done (4 left) 2024-09-12T23:42:58.722 INFO:tasks.rados.rados.0.smithi136.stdout:1391: done (3 left) 2024-09-12T23:42:58.722 INFO:tasks.rados.rados.0.smithi136.stdout:1392: done (2 left) 2024-09-12T23:42:58.722 INFO:tasks.rados.rados.0.smithi136.stdout:1393: done (1 left) 2024-09-12T23:42:58.722 INFO:tasks.rados.rados.0.smithi136.stdout:1395: delete oid 46 current snap is 135 2024-09-12T23:42:58.724 INFO:tasks.rados.rados.0.smithi136.stdout:1395: done (1 left) 2024-09-12T23:42:58.724 INFO:tasks.rados.rados.0.smithi136.stdout:1396: delete oid 10 current snap is 135 2024-09-12T23:42:58.726 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 48 v 806 (ObjNum 289 snap 0 seq_num 1) dirty exists 2024-09-12T23:42:58.730 INFO:tasks.rados.rados.0.smithi136.stdout:1394: done (1 left) 2024-09-12T23:42:58.730 INFO:tasks.rados.rados.0.smithi136.stdout:1396: done (0 left) 2024-09-12T23:42:58.730 INFO:tasks.rados.rados.0.smithi136.stdout:1397: copy_from oid 21 from oid 29 current snap is 135 2024-09-12T23:42:58.731 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 3 current snap is 135 2024-09-12T23:42:58.731 INFO:tasks.rados.rados.0.smithi136.stdout:1398: seq_num 455 ranges {2088960=425984} 2024-09-12T23:42:58.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:58 smithi136 bash[75164]: cephadm 2024-09-12T23:42:57.095982+0000 mgr.y (mgr.24661) 45 : cephadm [INF] Upgrade: Updating mon.b 2024-09-12T23:42:58.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:58 smithi136 bash[75164]: cephadm 2024-09-12T23:42:57.114697+0000 mgr.y (mgr.24661) 46 : cephadm [INF] Deploying daemon mon.b on smithi181 2024-09-12T23:42:58.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:58 smithi136 bash[75164]: cluster 2024-09-12T23:42:58.302452+0000 mon.a (mon.0) 150 : cluster [DBG] osdmap e333: 8 total, 8 up, 8 in 2024-09-12T23:42:58.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:58 smithi136 bash[76371]: cephadm 2024-09-12T23:42:57.095982+0000 mgr.y (mgr.24661) 45 : cephadm [INF] Upgrade: Updating mon.b 2024-09-12T23:42:58.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:58 smithi136 bash[76371]: cephadm 2024-09-12T23:42:57.114697+0000 mgr.y (mgr.24661) 46 : cephadm [INF] Deploying daemon mon.b on smithi181 2024-09-12T23:42:58.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:58 smithi136 bash[76371]: cluster 2024-09-12T23:42:58.302452+0000 mon.a (mon.0) 150 : cluster [DBG] osdmap e333: 8 total, 8 up, 8 in 2024-09-12T23:42:58.767 INFO:tasks.rados.rados.0.smithi136.stdout:1398: writing smithi13667081-3 from 2088960 to 2514944 tid 1 2024-09-12T23:42:58.767 INFO:tasks.rados.rados.0.smithi136.stdout:1399: copy_from oid 35 from oid 34 current snap is 135 2024-09-12T23:42:58.767 INFO:tasks.rados.rados.0.smithi136.stdout:1397: finishing copy_from racing read to smithi13667081-21 2024-09-12T23:42:58.767 INFO:tasks.rados.rados.0.smithi136.stdout:1400: read oid 19 snap -1 2024-09-12T23:42:58.767 INFO:tasks.rados.rados.0.smithi136.stdout:1400: expect (ObjNum 391 snap 113 seq_num 391) 2024-09-12T23:42:58.767 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 41 current snap is 135 2024-09-12T23:42:58.767 INFO:tasks.rados.rados.0.smithi136.stdout:1401: seq_num 456 ranges {4661248=442368} 2024-09-12T23:42:58.818 INFO:tasks.rados.rados.0.smithi136.stdout:1401: writing smithi13667081-41 from 4661248 to 5103616 tid 1 2024-09-12T23:42:58.818 INFO:tasks.rados.rados.0.smithi136.stdout:1402: copy_from oid 31 from oid 36 current snap is 135 2024-09-12T23:42:58.818 INFO:tasks.rados.rados.0.smithi136.stdout:1398: finishing write tid 1 to smithi13667081-3 2024-09-12T23:42:58.818 INFO:tasks.rados.rados.0.smithi136.stdout:1397: finishing copy_from to smithi13667081-21 2024-09-12T23:42:58.818 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 21 v 690 (ObjNum 441 snap 132 seq_num 441) dirty exists 2024-09-12T23:42:58.819 INFO:tasks.rados.rados.0.smithi136.stdout:1399: finishing copy_from racing read to smithi13667081-35 2024-09-12T23:42:58.819 INFO:tasks.rados.rados.0.smithi136.stdout:1398: finishing write tid 2 to smithi13667081-3 2024-09-12T23:42:58.819 INFO:tasks.rados.rados.0.smithi136.stdout:1398: finishing write tid 3 to smithi13667081-3 2024-09-12T23:42:58.819 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 3 v 654 (ObjNum 455 snap 135 seq_num 455) dirty exists 2024-09-12T23:42:58.819 INFO:tasks.rados.rados.0.smithi136.stdout:1398: left oid 3 (ObjNum 455 snap 135 seq_num 455) 2024-09-12T23:42:58.819 INFO:tasks.rados.rados.0.smithi136.stdout:1399: finishing copy_from to smithi13667081-35 2024-09-12T23:42:58.819 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 35 v 647 (ObjNum 432 snap 128 seq_num 432) dirty exists 2024-09-12T23:42:58.819 INFO:tasks.rados.rados.0.smithi136.stdout:1400: expect (ObjNum 391 snap 113 seq_num 391) 2024-09-12T23:42:59.076 INFO:tasks.rados.rados.0.smithi136.stdout:1402: finishing copy_from to smithi13667081-31 2024-09-12T23:42:59.076 INFO:tasks.rados.rados.0.smithi136.stdout:1402: got expected ENOENT (src dne) 2024-09-12T23:42:59.076 INFO:tasks.rados.rados.0.smithi136.stdout:1402: finishing copy_from racing read to smithi13667081-31 2024-09-12T23:42:59.076 INFO:tasks.rados.rados.0.smithi136.stdout:1401: finishing write tid 1 to smithi13667081-41 2024-09-12T23:42:59.076 INFO:tasks.rados.rados.0.smithi136.stdout:1401: finishing write tid 2 to smithi13667081-41 2024-09-12T23:42:59.076 INFO:tasks.rados.rados.0.smithi136.stdout:1401: finishing write tid 3 to smithi13667081-41 2024-09-12T23:42:59.076 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 41 v 672 (ObjNum 456 snap 135 seq_num 456) dirty exists 2024-09-12T23:42:59.076 INFO:tasks.rados.rados.0.smithi136.stdout:1401: left oid 41 (ObjNum 456 snap 135 seq_num 456) 2024-09-12T23:42:59.076 INFO:tasks.rados.rados.0.smithi136.stdout:1397: done (5 left) 2024-09-12T23:42:59.076 INFO:tasks.rados.rados.0.smithi136.stdout:1398: done (4 left) 2024-09-12T23:42:59.076 INFO:tasks.rados.rados.0.smithi136.stdout:1399: done (3 left) 2024-09-12T23:42:59.077 INFO:tasks.rados.rados.0.smithi136.stdout:1400: done (2 left) 2024-09-12T23:42:59.077 INFO:tasks.rados.rados.0.smithi136.stdout:1401: done (1 left) 2024-09-12T23:42:59.077 INFO:tasks.rados.rados.0.smithi136.stdout:1402: done (0 left) 2024-09-12T23:42:59.077 INFO:tasks.rados.rados.0.smithi136.stdout:1403: rmattr oid 14 current snap is 135 2024-09-12T23:42:59.077 INFO:tasks.rados.rados.0.smithi136.stdout:1404: snap_remove snap 118 2024-09-12T23:42:59.094 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:42:58 smithi181 bash[60471]: ts=2024-09-12T23:42:58.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:42:59.398 INFO:tasks.rados.rados.0.smithi136.stdout:1404: done (1 left) 2024-09-12T23:42:59.398 INFO:tasks.rados.rados.0.smithi136.stdout:1405: copy_from oid 9 from oid 43 current snap is 135 2024-09-12T23:42:59.398 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 14 v 808 (ObjNum 398 snap 116 seq_num 398) dirty exists 2024-09-12T23:42:59.398 INFO:tasks.rados.rados.0.smithi136.stdout:1403: done (1 left) 2024-09-12T23:42:59.398 INFO:tasks.rados.rados.0.smithi136.stdout:1406: snap_create 2024-09-12T23:42:59.443 INFO:tasks.rados.rados.0.smithi136.stdout:1405: finishing copy_from racing read to smithi13667081-9 2024-09-12T23:42:59.446 INFO:tasks.rados.rados.0.smithi136.stdout:1405: finishing copy_from to smithi13667081-9 2024-09-12T23:42:59.446 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 9 v 199 (ObjNum 399 snap 116 seq_num 399) dirty exists 2024-09-12T23:42:59.489 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:59 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:42:59.489 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:59 smithi181 bash[24776]: cluster 2024-09-12T23:42:58.660370+0000 mgr.y (mgr.24661) 47 : cluster [DBG] pgmap v70: 121 pgs: 121 active+clean; 300 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:42:59.489 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:42:59 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:42:59.490 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 12 23:42:59 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:42:59.490 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 12 23:42:59 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:42:59.490 INFO:journalctl@ceph.mds.b.smithi181.stdout:Sep 12 23:42:59 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:42:59.491 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:42:59 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:42:59.491 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:42:59 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:42:59.491 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:42:59 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:42:59.492 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:42:59 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:42:59.492 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:42:59 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:42:59.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:42:59 smithi136 bash[75164]: cluster 2024-09-12T23:42:58.660370+0000 mgr.y (mgr.24661) 47 : cluster [DBG] pgmap v70: 121 pgs: 121 active+clean; 300 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:42:59.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:42:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:42:59] "GET /metrics HTTP/1.1" 200 36199 "" "Prometheus/2.43.0" 2024-09-12T23:42:59.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:42:59 smithi136 bash[76371]: cluster 2024-09-12T23:42:58.660370+0000 mgr.y (mgr.24661) 47 : cluster [DBG] pgmap v70: 121 pgs: 121 active+clean; 300 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:42:59.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:59 smithi181 systemd[1]: Stopping Ceph mon.b for da58a47c-715d-11ef-bceb-c7b262605968... 2024-09-12T23:42:59.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:59 smithi181 bash[24776]: debug 2024-09-12T23:42:59.599+0000 7f1addb4a640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-09-12T23:42:59.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:42:59 smithi181 bash[24776]: debug 2024-09-12T23:42:59.599+0000 7f1addb4a640 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2024-09-12T23:43:00.593 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:00 smithi181 bash[67473]: ceph-da58a47c-715d-11ef-bceb-c7b262605968-mon-b 2024-09-12T23:43:00.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:00 smithi181 bash[67533]: Error response from daemon: No such container: ceph-da58a47c-715d-11ef-bceb-c7b262605968-mon-b 2024-09-12T23:43:00.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:00 smithi181 systemd[1]: ceph-da58a47c-715d-11ef-bceb-c7b262605968@mon.b.service: Deactivated successfully. 2024-09-12T23:43:00.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:00 smithi181 systemd[1]: Stopped Ceph mon.b for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:43:01.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:00 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:43:01.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:00 smithi181 systemd[1]: Started Ceph mon.b for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:43:01.094 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:43:00 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:43:01.095 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:43:00 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:43:01.095 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:43:00 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:43:01.095 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 12 23:43:00 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:43:01.096 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 12 23:43:00 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:43:01.096 INFO:journalctl@ceph.mds.b.smithi181.stdout:Sep 12 23:43:00 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:43:01.096 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:43:00 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:43:01.096 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:43:00 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:43:01.097 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:43:00 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:43:01.799 INFO:teuthology.orchestra.run.smithi136.stdout:true 2024-09-12T23:43:02.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.819+0000 7fba27323b00 0 set uid:gid to 167:167 (ceph:ceph) 2024-09-12T23:43:02.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.819+0000 7fba27323b00 0 ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable), process ceph-mon, pid 7 2024-09-12T23:43:02.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.819+0000 7fba27323b00 0 pidfile_write: ignore empty --pid-file 2024-09-12T23:43:02.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 0 load: jerasure load: lrc 2024-09-12T23:43:02.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: RocksDB version: 7.9.2 2024-09-12T23:43:02.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Git sha 0 2024-09-12T23:43:02.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Compile date 2024-09-10 15:58:06 2024-09-12T23:43:02.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: DB SUMMARY 2024-09-12T23:43:02.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: DB Session ID: RG1UV67VF5OG74HWS2G9 2024-09-12T23:43:02.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: CURRENT file: CURRENT 2024-09-12T23:43:02.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: IDENTITY file: IDENTITY 2024-09-12T23:43:02.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: MANIFEST file: MANIFEST-000009 size: 1842 Bytes 2024-09-12T23:43:02.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000039.sst 2024-09-12T23:43:02.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000037.log size: 750488 ; 2024-09-12T23:43:02.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.error_if_exists: 0 2024-09-12T23:43:02.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.create_if_missing: 0 2024-09-12T23:43:02.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.paranoid_checks: 1 2024-09-12T23:43:02.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.flush_verify_memtable_count: 1 2024-09-12T23:43:02.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-12T23:43:02.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-12T23:43:02.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.env: 0x556fa0230bc0 2024-09-12T23:43:02.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.fs: PosixFileSystem 2024-09-12T23:43:02.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.info_log: 0x556fa16e8be0 2024-09-12T23:43:02.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.max_file_opening_threads: 16 2024-09-12T23:43:02.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.statistics: (nil) 2024-09-12T23:43:02.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.use_fsync: 0 2024-09-12T23:43:02.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.max_log_file_size: 0 2024-09-12T23:43:02.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-12T23:43:02.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.log_file_time_to_roll: 0 2024-09-12T23:43:02.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.keep_log_file_num: 1000 2024-09-12T23:43:02.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.recycle_log_file_num: 0 2024-09-12T23:43:02.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.allow_fallocate: 1 2024-09-12T23:43:02.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.allow_mmap_reads: 0 2024-09-12T23:43:02.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.allow_mmap_writes: 0 2024-09-12T23:43:02.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.use_direct_reads: 0 2024-09-12T23:43:02.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-12T23:43:02.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.create_missing_column_families: 0 2024-09-12T23:43:02.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.db_log_dir: 2024-09-12T23:43:02.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.wal_dir: 2024-09-12T23:43:02.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.table_cache_numshardbits: 6 2024-09-12T23:43:02.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-09-12T23:43:02.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-09-12T23:43:02.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-12T23:43:02.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-12T23:43:02.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-09-12T23:43:02.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.advise_random_on_open: 1 2024-09-12T23:43:02.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.db_write_buffer_size: 0 2024-09-12T23:43:02.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.write_buffer_manager: 0x556fa16f6b40 2024-09-12T23:43:02.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-12T23:43:02.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-12T23:43:02.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.use_adaptive_mutex: 0 2024-09-12T23:43:02.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.rate_limiter: (nil) 2024-09-12T23:43:02.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-12T23:43:02.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.wal_recovery_mode: 2 2024-09-12T23:43:02.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.enable_thread_tracking: 0 2024-09-12T23:43:02.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.enable_pipelined_write: 0 2024-09-12T23:43:02.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.unordered_write: 0 2024-09-12T23:43:02.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-12T23:43:02.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-12T23:43:02.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-12T23:43:02.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-12T23:43:02.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.row_cache: None 2024-09-12T23:43:02.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.wal_filter: None 2024-09-12T23:43:02.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-12T23:43:02.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.allow_ingest_behind: 0 2024-09-12T23:43:02.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.two_write_queues: 0 2024-09-12T23:43:02.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.manual_wal_flush: 0 2024-09-12T23:43:02.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.wal_compression: 0 2024-09-12T23:43:02.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.atomic_flush: 0 2024-09-12T23:43:02.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-12T23:43:02.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.persist_stats_to_disk: 0 2024-09-12T23:43:02.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-09-12T23:43:02.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.log_readahead_size: 0 2024-09-12T23:43:02.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-12T23:43:02.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.best_efforts_recovery: 0 2024-09-12T23:43:02.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-12T23:43:02.102 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-12T23:43:02.102 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.allow_data_in_errors: 0 2024-09-12T23:43:02.102 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.db_host_id: __hostname__ 2024-09-12T23:43:02.102 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.enforce_single_del_contracts: true 2024-09-12T23:43:02.102 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.max_background_jobs: 2 2024-09-12T23:43:02.102 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.max_background_compactions: -1 2024-09-12T23:43:02.102 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.max_subcompactions: 1 2024-09-12T23:43:02.102 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-12T23:43:02.102 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-12T23:43:02.102 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.delayed_write_rate : 16777216 2024-09-12T23:43:02.102 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.max_total_wal_size: 0 2024-09-12T23:43:02.102 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-12T23:43:02.102 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.stats_dump_period_sec: 600 2024-09-12T23:43:02.103 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.stats_persist_period_sec: 600 2024-09-12T23:43:02.103 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-12T23:43:02.103 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.max_open_files: -1 2024-09-12T23:43:02.103 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.bytes_per_sync: 0 2024-09-12T23:43:02.103 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-09-12T23:43:02.103 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-09-12T23:43:02.103 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.compaction_readahead_size: 0 2024-09-12T23:43:02.103 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.max_background_flushes: -1 2024-09-12T23:43:02.103 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Compression algorithms supported: 2024-09-12T23:43:02.103 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: kZSTD supported: 0 2024-09-12T23:43:02.103 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: kXpressCompression supported: 0 2024-09-12T23:43:02.103 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: kBZip2Compression supported: 0 2024-09-12T23:43:02.103 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-12T23:43:02.104 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: kLZ4Compression supported: 1 2024-09-12T23:43:02.104 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: kZlibCompression supported: 1 2024-09-12T23:43:02.104 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: kLZ4HCCompression supported: 1 2024-09-12T23:43:02.104 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: kSnappyCompression supported: 1 2024-09-12T23:43:02.104 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-09-12T23:43:02.104 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: DMutex implementation: pthread_mutex_t 2024-09-12T23:43:02.104 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2024-09-12T23:43:02.105 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-12T23:43:02.106 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-12T23:43:02.106 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.merge_operator: 2024-09-12T23:43:02.106 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.compaction_filter: None 2024-09-12T23:43:02.106 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.compaction_filter_factory: None 2024-09-12T23:43:02.106 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.sst_partitioner_factory: None 2024-09-12T23:43:02.106 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-09-12T23:43:02.106 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.table_factory: BlockBasedTable 2024-09-12T23:43:02.106 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x556fa16e87e0) 2024-09-12T23:43:02.106 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: cache_index_and_filter_blocks: 1 2024-09-12T23:43:02.106 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-12T23:43:02.106 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-12T23:43:02.106 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: pin_top_level_index_and_filter: 1 2024-09-12T23:43:02.107 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: index_type: 0 2024-09-12T23:43:02.107 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: data_block_index_type: 0 2024-09-12T23:43:02.107 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: index_shortening: 1 2024-09-12T23:43:02.107 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: data_block_hash_table_util_ratio: 0.750000 2024-09-12T23:43:02.107 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: checksum: 4 2024-09-12T23:43:02.107 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: no_block_cache: 0 2024-09-12T23:43:02.107 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: block_cache: 0x556fa16df350 2024-09-12T23:43:02.107 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: block_cache_name: BinnedLRUCache 2024-09-12T23:43:02.107 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: block_cache_options: 2024-09-12T23:43:02.107 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: capacity : 536870912 2024-09-12T23:43:02.107 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: num_shard_bits : 4 2024-09-12T23:43:02.107 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: strict_capacity_limit : 0 2024-09-12T23:43:02.107 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: high_pri_pool_ratio: 0.000 2024-09-12T23:43:02.108 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: block_cache_compressed: (nil) 2024-09-12T23:43:02.108 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: persistent_cache: (nil) 2024-09-12T23:43:02.108 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: block_size: 4096 2024-09-12T23:43:02.108 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: block_size_deviation: 10 2024-09-12T23:43:02.108 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: block_restart_interval: 16 2024-09-12T23:43:02.108 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: index_block_restart_interval: 1 2024-09-12T23:43:02.108 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: metadata_block_size: 4096 2024-09-12T23:43:02.108 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: partition_filters: 0 2024-09-12T23:43:02.108 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: use_delta_encoding: 1 2024-09-12T23:43:02.108 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: filter_policy: bloomfilter 2024-09-12T23:43:02.108 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: whole_key_filtering: 1 2024-09-12T23:43:02.108 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: verify_compression: 0 2024-09-12T23:43:02.109 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: read_amp_bytes_per_bit: 0 2024-09-12T23:43:02.109 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: format_version: 5 2024-09-12T23:43:02.109 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: enable_index_compression: 1 2024-09-12T23:43:02.109 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: block_align: 0 2024-09-12T23:43:02.109 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: max_auto_readahead_size: 262144 2024-09-12T23:43:02.109 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: prepopulate_block_cache: 0 2024-09-12T23:43:02.109 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: initial_auto_readahead_size: 8192 2024-09-12T23:43:02.109 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: num_file_reads_for_auto_readahead: 2 2024-09-12T23:43:02.109 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.write_buffer_size: 33554432 2024-09-12T23:43:02.109 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.max_write_buffer_number: 2 2024-09-12T23:43:02.109 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.compression: NoCompression 2024-09-12T23:43:02.109 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.bottommost_compression: Disabled 2024-09-12T23:43:02.109 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.prefix_extractor: nullptr 2024-09-12T23:43:02.110 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-12T23:43:02.110 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.num_levels: 7 2024-09-12T23:43:02.110 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-12T23:43:02.110 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-12T23:43:02.110 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-12T23:43:02.110 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-12T23:43:02.110 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-12T23:43:02.110 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-12T23:43:02.110 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-12T23:43:02.110 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-12T23:43:02.110 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-12T23:43:02.110 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-12T23:43:02.111 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-12T23:43:02.111 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-12T23:43:02.111 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.compression_opts.window_bits: -14 2024-09-12T23:43:02.111 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.compression_opts.level: 32767 2024-09-12T23:43:02.111 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.compression_opts.strategy: 0 2024-09-12T23:43:02.111 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-12T23:43:02.111 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-12T23:43:02.111 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-12T23:43:02.111 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-12T23:43:02.111 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.compression_opts.enabled: false 2024-09-12T23:43:02.111 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-12T23:43:02.111 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-12T23:43:02.111 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-12T23:43:02.111 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-12T23:43:02.112 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.target_file_size_base: 67108864 2024-09-12T23:43:02.112 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.target_file_size_multiplier: 1 2024-09-12T23:43:02.112 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-12T23:43:02.112 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-12T23:43:02.112 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-12T23:43:02.112 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-12T23:43:02.112 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-12T23:43:02.112 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-12T23:43:02.112 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-12T23:43:02.112 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-12T23:43:02.112 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-12T23:43:02.112 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-12T23:43:02.112 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-12T23:43:02.113 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-12T23:43:02.113 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-12T23:43:02.113 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.arena_block_size: 1048576 2024-09-12T23:43:02.113 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-12T23:43:02.113 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-12T23:43:02.113 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.disable_auto_compactions: 0 2024-09-12T23:43:02.113 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-12T23:43:02.113 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-12T23:43:02.113 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-12T23:43:02.113 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-12T23:43:02.113 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-12T23:43:02.113 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-12T23:43:02.113 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-12T23:43:02.114 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-12T23:43:02.114 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-12T23:43:02.114 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-12T23:43:02.114 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-12T23:43:02.114 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.inplace_update_support: 0 2024-09-12T23:43:02.114 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-09-12T23:43:02.114 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-12T23:43:02.114 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-12T23:43:02.114 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.memtable_huge_page_size: 0 2024-09-12T23:43:02.114 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.bloom_locality: 0 2024-09-12T23:43:02.114 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.max_successive_merges: 0 2024-09-12T23:43:02.114 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-09-12T23:43:02.114 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.paranoid_file_checks: 0 2024-09-12T23:43:02.115 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.force_consistency_checks: 1 2024-09-12T23:43:02.115 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.report_bg_io_stats: 0 2024-09-12T23:43:02.115 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.ttl: 2592000 2024-09-12T23:43:02.115 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-09-12T23:43:02.115 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-12T23:43:02.115 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-12T23:43:02.115 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.enable_blob_files: false 2024-09-12T23:43:02.115 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.min_blob_size: 0 2024-09-12T23:43:02.115 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.blob_file_size: 268435456 2024-09-12T23:43:02.115 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.blob_compression_type: NoCompression 2024-09-12T23:43:02.115 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.enable_blob_garbage_collection: false 2024-09-12T23:43:02.115 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-12T23:43:02.115 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-12T23:43:02.116 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-12T23:43:02.116 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.blob_file_starting_level: 0 2024-09-12T23:43:02.116 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-12T23:43:02.116 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba238fb640 3 rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 39.sst 2024-09-12T23:43:02.116 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-12T23:43:02.116 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 41, last_sequence is 22154, log_number is 37,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-12T23:43:02.116 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 37 2024-09-12T23:43:02.116 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: ebd3e9c1-ca35-43b2-9a78-ef88d46dad0e 2024-09-12T23:43:02.116 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726184581831016, "job": 1, "event": "recovery_started", "wal_files": [37]} 2024-09-12T23:43:02.116 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.823+0000 7fba27323b00 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #37 mode 2 2024-09-12T23:43:02.116 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.827+0000 7fba27323b00 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726184581834208, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 42, "file_size": 494714, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 22159, "largest_seqno": 22572, "table_properties": {"data_size": 491859, "index_size": 1367, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 453, "raw_key_size": 4416, "raw_average_key_size": 25, "raw_value_size": 487686, "raw_average_value_size": 2786, "num_data_blocks": 59, "num_entries": 175, "num_filter_entries": 175, "num_deletions": 7, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726184581, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "ebd3e9c1-ca35-43b2-9a78-ef88d46dad0e", "db_session_id": "RG1UV67VF5OG74HWS2G9", "orig_file_number": 42, "seqno_to_time_mapping": "N/A"}} 2024-09-12T23:43:02.116 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.827+0000 7fba27323b00 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726184581834348, "job": 1, "event": "recovery_finished"} 2024-09-12T23:43:02.116 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.827+0000 7fba27323b00 4 rocksdb: [db/version_set.cc:5047] Creating manifest 44 2024-09-12T23:43:02.117 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.827+0000 7fba27323b00 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-12T23:43:02.117 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.831+0000 7fba27323b00 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000037.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-12T23:43:02.117 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.831+0000 7fba27323b00 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x556fa1708e00 2024-09-12T23:43:02.117 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.831+0000 7fba27323b00 4 rocksdb: DB pointer 0x556fa1800000 2024-09-12T23:43:02.117 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.831+0000 7fba1d0ee640 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-12T23:43:02.117 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.831+0000 7fba1d0ee640 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-12T23:43:02.117 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: ** DB Stats ** 2024-09-12T23:43:02.117 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: Uptime(secs): 0.0 total, 0.0 interval 2024-09-12T23:43:02.117 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-12T23:43:02.117 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-12T23:43:02.117 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-12T23:43:02.117 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-12T23:43:02.118 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-12T23:43:02.118 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-12T23:43:02.118 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: ** Compaction Stats [default] ** 2024-09-12T23:43:02.118 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-12T23:43:02.118 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-12T23:43:02.118 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: L0 1/0 483.12 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 258.9 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-09-12T23:43:02.118 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: L6 1/0 12.39 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-12T23:43:02.118 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: Sum 2/0 12.86 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 258.9 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-09-12T23:43:02.118 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 258.9 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-09-12T23:43:02.118 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: ** Compaction Stats [default] ** 2024-09-12T23:43:02.118 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-12T23:43:02.118 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-12T23:43:02.118 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 258.9 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-09-12T23:43:02.119 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-12T23:43:02.119 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: Uptime(secs): 0.0 total, 0.0 interval 2024-09-12T23:43:02.119 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: Flush(GB): cumulative 0.000, interval 0.000 2024-09-12T23:43:02.119 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-12T23:43:02.119 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: AddFile(Total Files): cumulative 0, interval 0 2024-09-12T23:43:02.119 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: AddFile(L0 Files): cumulative 0, interval 0 2024-09-12T23:43:02.119 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: AddFile(Keys): cumulative 0, interval 0 2024-09-12T23:43:02.119 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: Cumulative compaction: 0.00 GB write, 74.95 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-12T23:43:02.119 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: Interval compaction: 0.00 GB write, 74.95 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-12T23:43:02.119 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-12T23:43:02.119 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: Block cache BinnedLRUCache@0x556fa16df350#7 capacity: 512.00 MB usage: 1.95 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.9e-05 secs_since: 0 2024-09-12T23:43:02.119 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: Block cache entry stats(count,size,portion): FilterBlock(1,0.50 KB,9.53674e-05%) IndexBlock(1,1.45 KB,0.000277162%) Misc(1,0.00 KB,0%) 2024-09-12T23:43:02.119 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: ** File Read Latency Histogram By Level [default] ** 2024-09-12T23:43:02.120 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.831+0000 7fba27323b00 0 starting mon.b rank 1 at public addrs [v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0] at bind addrs [v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid da58a47c-715d-11ef-bceb-c7b262605968 2024-09-12T23:43:02.120 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.831+0000 7fba27323b00 1 mon.b@-1(???) e3 preinit fsid da58a47c-715d-11ef-bceb-c7b262605968 2024-09-12T23:43:02.120 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.831+0000 7fba27323b00 0 mon.b@-1(???).mds e7 new map 2024-09-12T23:43:02.120 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.831+0000 7fba27323b00 0 mon.b@-1(???).mds e7 print_map 2024-09-12T23:43:02.120 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: e7 2024-09-12T23:43:02.120 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: enable_multiple, ever_enabled_multiple: 1,1 2024-09-12T23:43:02.120 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-12T23:43:02.120 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: legacy client fscid: 1 2024-09-12T23:43:02.120 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: 2024-09-12T23:43:02.120 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: Filesystem 'cephfs' (1) 2024-09-12T23:43:02.120 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: fs_name cephfs 2024-09-12T23:43:02.120 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: epoch 7 2024-09-12T23:43:02.120 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-12T23:43:02.120 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: created 2024-09-12T23:33:29.192396+0000 2024-09-12T23:43:02.121 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: modified 2024-09-12T23:33:32.368306+0000 2024-09-12T23:43:02.121 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: tableserver 0 2024-09-12T23:43:02.121 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: root 0 2024-09-12T23:43:02.121 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: session_timeout 60 2024-09-12T23:43:02.121 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: session_autoclose 300 2024-09-12T23:43:02.121 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: max_file_size 1099511627776 2024-09-12T23:43:02.121 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: max_xattr_size 65536 2024-09-12T23:43:02.121 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: required_client_features {} 2024-09-12T23:43:02.121 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: last_failure 0 2024-09-12T23:43:02.121 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: last_failure_osd_epoch 0 2024-09-12T23:43:02.121 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-12T23:43:02.121 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: max_mds 1 2024-09-12T23:43:02.121 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: in 0 2024-09-12T23:43:02.122 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: up {0=24316} 2024-09-12T23:43:02.122 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: failed 2024-09-12T23:43:02.122 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: damaged 2024-09-12T23:43:02.122 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: stopped 2024-09-12T23:43:02.122 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: data_pools [3] 2024-09-12T23:43:02.122 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: metadata_pool 2 2024-09-12T23:43:02.122 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: inline_data disabled 2024-09-12T23:43:02.122 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: balancer 2024-09-12T23:43:02.122 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: bal_rank_mask -1 2024-09-12T23:43:02.122 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: standby_count_wanted 0 2024-09-12T23:43:02.122 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: [mds.b{0:24316} state up:active seq 3 addr [v2:172.21.15.181:6832/2077070284,v1:172.21.15.181:6833/2077070284] compat {c=[1],r=[1],i=[7ff]}] 2024-09-12T23:43:02.122 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: 2024-09-12T23:43:02.122 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: 2024-09-12T23:43:02.122 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: Standby daemons: 2024-09-12T23:43:02.123 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: 2024-09-12T23:43:02.123 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: [mds.a{-1:14391} state up:standby seq 1 addr [v2:172.21.15.136:6834/3940972569,v1:172.21.15.136:6835/3940972569] compat {c=[1],r=[1],i=[7ff]}] 2024-09-12T23:43:02.123 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.831+0000 7fba27323b00 0 mon.b@-1(???).osd e334 crush map has features 3314933069573799936, adjusting msgr requires 2024-09-12T23:43:02.123 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.831+0000 7fba27323b00 0 mon.b@-1(???).osd e334 crush map has features 432629308056666112, adjusting msgr requires 2024-09-12T23:43:02.123 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.831+0000 7fba27323b00 0 mon.b@-1(???).osd e334 crush map has features 432629308056666112, adjusting msgr requires 2024-09-12T23:43:02.123 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.831+0000 7fba27323b00 0 mon.b@-1(???).osd e334 crush map has features 432629308056666112, adjusting msgr requires 2024-09-12T23:43:02.123 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:01 smithi181 bash[67587]: debug 2024-09-12T23:43:01.831+0000 7fba27323b00 1 mon.b@-1(???).paxosservice(auth 1..22) refresh upgraded, format 0 -> 3 2024-09-12T23:43:02.123 INFO:teuthology.orchestra.run.smithi136.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-12T23:43:02.123 INFO:teuthology.orchestra.run.smithi136.stdout:alertmanager.a smithi136 *:9093,9094 running (2m) 17s ago 8m 13.7M - 0.25.0 c8568f914cd2 b5197f020eeb 2024-09-12T23:43:02.124 INFO:teuthology.orchestra.run.smithi136.stdout:grafana.a smithi181 *:3000 running (2m) 47s ago 7m 74.9M - 9.4.7 954c08fa6188 256c181be984 2024-09-12T23:43:02.124 INFO:teuthology.orchestra.run.smithi136.stdout:mds.a smithi136 running (9m) 17s ago 9m 14.5M - 17.2.7-1627-gbbdadfa3 214985049bdb 713b9795ad19 2024-09-12T23:43:02.124 INFO:teuthology.orchestra.run.smithi136.stdout:mds.b smithi181 running (9m) 47s ago 9m 17.3M - 17.2.7-1627-gbbdadfa3 214985049bdb ad13179f0868 2024-09-12T23:43:02.124 INFO:teuthology.orchestra.run.smithi136.stdout:mgr.x smithi181 *:8443,9283,8765 running (85s) 47s ago 16m 423M - 18.2.4-837-g7a853b17 f21ba1eea74e cc664dc337f7 2024-09-12T23:43:02.124 INFO:teuthology.orchestra.run.smithi136.stdout:mgr.y smithi136 *:8443,9283,8765 running (2m) 17s ago 19m 482M - 18.2.4-837-g7a853b17 f21ba1eea74e 272f10b42fc3 2024-09-12T23:43:02.124 INFO:teuthology.orchestra.run.smithi136.stdout:mon.a smithi136 running (61s) 17s ago 19m 39.3M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e ad8d21d58ad9 2024-09-12T23:43:02.124 INFO:teuthology.orchestra.run.smithi136.stdout:mon.b smithi181 starting - - - 2048M 2024-09-12T23:43:02.124 INFO:teuthology.orchestra.run.smithi136.stdout:mon.c smithi136 running (30s) 17s ago 17m 19.2M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e 44d1be055a5e 2024-09-12T23:43:02.124 INFO:teuthology.orchestra.run.smithi136.stdout:node-exporter.a smithi136 *:9100 running (2m) 17s ago 8m 7996k - 1.5.0 0da6a335fe13 ccfc37c96620 2024-09-12T23:43:02.124 INFO:teuthology.orchestra.run.smithi136.stdout:node-exporter.b smithi181 *:9100 running (2m) 47s ago 8m 7516k - 1.5.0 0da6a335fe13 80bf0e383230 2024-09-12T23:43:02.124 INFO:teuthology.orchestra.run.smithi136.stdout:osd.0 smithi136 running (16m) 17s ago 16m 187M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb cbc0b7df0115 2024-09-12T23:43:02.124 INFO:teuthology.orchestra.run.smithi136.stdout:osd.1 smithi136 running (15m) 17s ago 15m 197M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb c4daee7da836 2024-09-12T23:43:02.125 INFO:teuthology.orchestra.run.smithi136.stdout:osd.2 smithi136 running (14m) 17s ago 14m 210M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 4ab8fd79cdb4 2024-09-12T23:43:02.125 INFO:teuthology.orchestra.run.smithi136.stdout:osd.3 smithi136 running (13m) 17s ago 13m 152M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 9ceb14a27521 2024-09-12T23:43:02.125 INFO:teuthology.orchestra.run.smithi136.stdout:osd.4 smithi181 running (12m) 47s ago 12m 198M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb a35cdca79055 2024-09-12T23:43:02.125 INFO:teuthology.orchestra.run.smithi136.stdout:osd.5 smithi181 running (12m) 47s ago 12m 167M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 8f2e8d494fef 2024-09-12T23:43:02.125 INFO:teuthology.orchestra.run.smithi136.stdout:osd.6 smithi181 running (11m) 47s ago 11m 97.5M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 9c6bec406408 2024-09-12T23:43:02.125 INFO:teuthology.orchestra.run.smithi136.stdout:osd.7 smithi181 running (10m) 47s ago 10m 136M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb bc0632ae2145 2024-09-12T23:43:02.125 INFO:teuthology.orchestra.run.smithi136.stdout:prometheus.a smithi181 *:9095 running (91s) 47s ago 9m 46.0M - 2.43.0 a07b618ecd1d 3853d003a5cc 2024-09-12T23:43:02.436 INFO:teuthology.orchestra.run.smithi136.stdout:{ 2024-09-12T23:43:02.436 INFO:teuthology.orchestra.run.smithi136.stdout: "mon": { 2024-09-12T23:43:02.436 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 1, 2024-09-12T23:43:02.436 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 2 2024-09-12T23:43:02.436 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:43:02.436 INFO:teuthology.orchestra.run.smithi136.stdout: "mgr": { 2024-09-12T23:43:02.436 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 2 2024-09-12T23:43:02.436 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:43:02.436 INFO:teuthology.orchestra.run.smithi136.stdout: "osd": { 2024-09-12T23:43:02.436 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 8 2024-09-12T23:43:02.437 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:43:02.437 INFO:teuthology.orchestra.run.smithi136.stdout: "mds": { 2024-09-12T23:43:02.437 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-12T23:43:02.437 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:43:02.437 INFO:teuthology.orchestra.run.smithi136.stdout: "overall": { 2024-09-12T23:43:02.437 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 11, 2024-09-12T23:43:02.437 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 4 2024-09-12T23:43:02.437 INFO:teuthology.orchestra.run.smithi136.stdout: } 2024-09-12T23:43:02.437 INFO:teuthology.orchestra.run.smithi136.stdout:} 2024-09-12T23:43:02.594 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:43:02 smithi181 bash[60471]: ts=2024-09-12T23:43:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:43:03.844 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:43:03 smithi181 bash[60471]: ts=2024-09-12T23:43:03.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi136\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:43:04.844 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:43:04 smithi181 bash[60471]: ts=2024-09-12T23:43:04.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:43:04.844 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:43:04 smithi181 bash[60471]: ts=2024-09-12T23:43:04.456Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:43:07.256 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:43:06 smithi136 bash[68787]: debug 2024-09-12T23:43:06.844+0000 7f5e4b0ad640 -1 mgr.server handle_report got status from non-daemon mon.b 2024-09-12T23:43:09.031 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:43:08 smithi181 bash[60471]: ts=2024-09-12T23:43:08.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:43:09.048 INFO:tasks.rados.rados.0.smithi136.stdout:1405: done (1 left) 2024-09-12T23:43:09.048 INFO:tasks.rados.rados.0.smithi136.stdout:1406: done (0 left) 2024-09-12T23:43:09.048 INFO:tasks.rados.rados.0.smithi136.stdout:1407: read oid 31 snap 125 2024-09-12T23:43:09.048 INFO:tasks.rados.rados.0.smithi136.stdout:1407: expect (ObjNum 262 snap 71 seq_num 262) 2024-09-12T23:43:09.048 INFO:tasks.rados.rados.0.smithi136.stdout:1408: delete oid 4 current snap is 136 2024-09-12T23:43:09.058 INFO:tasks.rados.rados.0.smithi136.stdout:1408: done (1 left) 2024-09-12T23:43:09.058 INFO:tasks.rados.rados.0.smithi136.stdout:1409: snap_remove snap 134 2024-09-12T23:43:09.344 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:09 smithi181 bash[67587]: cluster 2024-09-12T23:43:02.981361+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2024-09-12T23:43:09.345 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:09 smithi181 bash[67587]: cluster 2024-09-12T23:43:02.985100+0000 mon.c (mon.2) 2 : cluster [INF] mon.c calling monitor election 2024-09-12T23:43:09.345 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:09 smithi181 bash[67587]: cluster 2024-09-12T23:43:07.989694+0000 mon.a (mon.0) 165 : cluster [DBG] monmap e4: 3 mons at {a=[v2:172.21.15.136:3300/0,v1:172.21.15.136:6789/0],b=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0],c=[v2:172.21.15.136:3301/0,v1:172.21.15.136:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-12T23:43:09.345 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:09 smithi181 bash[67587]: cluster 2024-09-12T23:43:07.990819+0000 mon.a (mon.0) 166 : cluster [DBG] fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-12T23:43:09.345 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:09 smithi181 bash[67587]: cluster 2024-09-12T23:43:07.990908+0000 mon.a (mon.0) 167 : cluster [DBG] osdmap e335: 8 total, 8 up, 8 in 2024-09-12T23:43:09.345 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:09 smithi181 bash[67587]: cluster 2024-09-12T23:43:07.991540+0000 mon.a (mon.0) 168 : cluster [DBG] mgrmap e42: y(active, since 59s), standbys: x 2024-09-12T23:43:09.345 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:09 smithi181 bash[67587]: cluster 2024-09-12T23:43:07.991891+0000 mon.a (mon.0) 169 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2024-09-12T23:43:09.345 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:09 smithi181 bash[67587]: cluster 2024-09-12T23:43:07.996064+0000 mon.a (mon.0) 170 : cluster [INF] overall HEALTH_OK 2024-09-12T23:43:09.345 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:09 smithi181 bash[67587]: cluster 2024-09-12T23:43:07.996438+0000 mon.a (mon.0) 171 : cluster [INF] mon.a calling monitor election 2024-09-12T23:43:09.345 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:09 smithi181 bash[67587]: cluster 2024-09-12T23:43:08.005402+0000 mon.b (mon.1) 3 : cluster [INF] mon.b calling monitor election 2024-09-12T23:43:09.345 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:09 smithi181 bash[67587]: cluster 2024-09-12T23:43:08.025767+0000 mon.a (mon.0) 172 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-12T23:43:09.345 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:09 smithi181 bash[67587]: cluster 2024-09-12T23:43:08.034806+0000 mon.a (mon.0) 173 : cluster [DBG] monmap e4: 3 mons at {a=[v2:172.21.15.136:3300/0,v1:172.21.15.136:6789/0],b=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0],c=[v2:172.21.15.136:3301/0,v1:172.21.15.136:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-12T23:43:09.345 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:09 smithi181 bash[67587]: cluster 2024-09-12T23:43:08.053586+0000 mon.a (mon.0) 174 : cluster [DBG] fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-12T23:43:09.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:09 smithi181 bash[67587]: cluster 2024-09-12T23:43:08.053639+0000 mon.a (mon.0) 175 : cluster [DBG] osdmap e335: 8 total, 8 up, 8 in 2024-09-12T23:43:09.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:09 smithi181 bash[67587]: cluster 2024-09-12T23:43:08.054204+0000 mon.a (mon.0) 176 : cluster [DBG] mgrmap e42: y(active, since 59s), standbys: x 2024-09-12T23:43:09.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:09 smithi181 bash[67587]: cluster 2024-09-12T23:43:08.054497+0000 mon.a (mon.0) 177 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-09-12T23:43:09.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:09 smithi181 bash[67587]: cluster 2024-09-12T23:43:08.054533+0000 mon.a (mon.0) 178 : cluster [INF] Cluster is now healthy 2024-09-12T23:43:09.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:09 smithi181 bash[67587]: cluster 2024-09-12T23:43:08.061664+0000 mon.a (mon.0) 179 : cluster [INF] overall HEALTH_OK 2024-09-12T23:43:09.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:09 smithi181 bash[67587]: audit 2024-09-12T23:43:08.068759+0000 mon.a (mon.0) 180 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:09.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:09 smithi181 bash[67587]: audit 2024-09-12T23:43:08.080391+0000 mon.a (mon.0) 181 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:09.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:09 smithi181 bash[67587]: audit 2024-09-12T23:43:08.083385+0000 mon.a (mon.0) 182 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:43:09.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:09 smithi181 bash[67587]: cluster 2024-09-12T23:43:08.665417+0000 mgr.y (mgr.24661) 54 : cluster [DBG] pgmap v77: 121 pgs: 121 active+clean; 284 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 6.3 MiB/s rd, 2.0 MiB/s wr, 12 op/s 2024-09-12T23:43:09.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:09 smithi181 bash[67587]: audit 2024-09-12T23:43:08.741987+0000 mon.a (mon.0) 183 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:43:09.409 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:09 smithi136 bash[76371]: cluster 2024-09-12T23:43:02.981361+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2024-09-12T23:43:09.409 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:09 smithi136 bash[76371]: cluster 2024-09-12T23:43:02.985100+0000 mon.c (mon.2) 2 : cluster [INF] mon.c calling monitor election 2024-09-12T23:43:09.409 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:09 smithi136 bash[76371]: cluster 2024-09-12T23:43:07.989694+0000 mon.a (mon.0) 165 : cluster [DBG] monmap e4: 3 mons at {a=[v2:172.21.15.136:3300/0,v1:172.21.15.136:6789/0],b=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0],c=[v2:172.21.15.136:3301/0,v1:172.21.15.136:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-12T23:43:09.409 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:09 smithi136 bash[76371]: cluster 2024-09-12T23:43:07.990819+0000 mon.a (mon.0) 166 : cluster [DBG] fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-12T23:43:09.409 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:09 smithi136 bash[76371]: cluster 2024-09-12T23:43:07.990908+0000 mon.a (mon.0) 167 : cluster [DBG] osdmap e335: 8 total, 8 up, 8 in 2024-09-12T23:43:09.409 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:09 smithi136 bash[76371]: cluster 2024-09-12T23:43:07.991540+0000 mon.a (mon.0) 168 : cluster [DBG] mgrmap e42: y(active, since 59s), standbys: x 2024-09-12T23:43:09.410 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:09 smithi136 bash[76371]: cluster 2024-09-12T23:43:07.991891+0000 mon.a (mon.0) 169 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2024-09-12T23:43:09.410 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:09 smithi136 bash[76371]: cluster 2024-09-12T23:43:07.996064+0000 mon.a (mon.0) 170 : cluster [INF] overall HEALTH_OK 2024-09-12T23:43:09.410 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:09 smithi136 bash[76371]: cluster 2024-09-12T23:43:07.996438+0000 mon.a (mon.0) 171 : cluster [INF] mon.a calling monitor election 2024-09-12T23:43:09.410 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:09 smithi136 bash[76371]: cluster 2024-09-12T23:43:08.005402+0000 mon.b (mon.1) 3 : cluster [INF] mon.b calling monitor election 2024-09-12T23:43:09.410 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:09 smithi136 bash[76371]: cluster 2024-09-12T23:43:08.025767+0000 mon.a (mon.0) 172 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-12T23:43:09.410 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:09 smithi136 bash[76371]: cluster 2024-09-12T23:43:08.034806+0000 mon.a (mon.0) 173 : cluster [DBG] monmap e4: 3 mons at {a=[v2:172.21.15.136:3300/0,v1:172.21.15.136:6789/0],b=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0],c=[v2:172.21.15.136:3301/0,v1:172.21.15.136:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-12T23:43:09.410 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:09 smithi136 bash[76371]: cluster 2024-09-12T23:43:08.053586+0000 mon.a (mon.0) 174 : cluster [DBG] fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-12T23:43:09.410 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:09 smithi136 bash[76371]: cluster 2024-09-12T23:43:08.053639+0000 mon.a (mon.0) 175 : cluster [DBG] osdmap e335: 8 total, 8 up, 8 in 2024-09-12T23:43:09.410 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:09 smithi136 bash[76371]: cluster 2024-09-12T23:43:08.054204+0000 mon.a (mon.0) 176 : cluster [DBG] mgrmap e42: y(active, since 59s), standbys: x 2024-09-12T23:43:09.410 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:09 smithi136 bash[76371]: cluster 2024-09-12T23:43:08.054497+0000 mon.a (mon.0) 177 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-09-12T23:43:09.410 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:09 smithi136 bash[76371]: cluster 2024-09-12T23:43:08.054533+0000 mon.a (mon.0) 178 : cluster [INF] Cluster is now healthy 2024-09-12T23:43:09.410 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:09 smithi136 bash[76371]: cluster 2024-09-12T23:43:08.061664+0000 mon.a (mon.0) 179 : cluster [INF] overall HEALTH_OK 2024-09-12T23:43:09.411 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:09 smithi136 bash[76371]: audit 2024-09-12T23:43:08.068759+0000 mon.a (mon.0) 180 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:09.411 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:09 smithi136 bash[76371]: audit 2024-09-12T23:43:08.080391+0000 mon.a (mon.0) 181 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:09.411 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:09 smithi136 bash[76371]: audit 2024-09-12T23:43:08.083385+0000 mon.a (mon.0) 182 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:43:09.411 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:09 smithi136 bash[76371]: cluster 2024-09-12T23:43:08.665417+0000 mgr.y (mgr.24661) 54 : cluster [DBG] pgmap v77: 121 pgs: 121 active+clean; 284 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 6.3 MiB/s rd, 2.0 MiB/s wr, 12 op/s 2024-09-12T23:43:09.411 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:09 smithi136 bash[76371]: audit 2024-09-12T23:43:08.741987+0000 mon.a (mon.0) 183 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:43:09.411 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:09 smithi136 bash[75164]: cluster 2024-09-12T23:43:02.981361+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2024-09-12T23:43:09.411 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:09 smithi136 bash[75164]: cluster 2024-09-12T23:43:02.985100+0000 mon.c (mon.2) 2 : cluster [INF] mon.c calling monitor election 2024-09-12T23:43:09.411 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:09 smithi136 bash[75164]: cluster 2024-09-12T23:43:07.989694+0000 mon.a (mon.0) 165 : cluster [DBG] monmap e4: 3 mons at {a=[v2:172.21.15.136:3300/0,v1:172.21.15.136:6789/0],b=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0],c=[v2:172.21.15.136:3301/0,v1:172.21.15.136:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-12T23:43:09.412 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:09 smithi136 bash[75164]: cluster 2024-09-12T23:43:07.990819+0000 mon.a (mon.0) 166 : cluster [DBG] fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-12T23:43:09.412 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:09 smithi136 bash[75164]: cluster 2024-09-12T23:43:07.990908+0000 mon.a (mon.0) 167 : cluster [DBG] osdmap e335: 8 total, 8 up, 8 in 2024-09-12T23:43:09.412 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:09 smithi136 bash[75164]: cluster 2024-09-12T23:43:07.991540+0000 mon.a (mon.0) 168 : cluster [DBG] mgrmap e42: y(active, since 59s), standbys: x 2024-09-12T23:43:09.412 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:09 smithi136 bash[75164]: cluster 2024-09-12T23:43:07.991891+0000 mon.a (mon.0) 169 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2024-09-12T23:43:09.412 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:09 smithi136 bash[75164]: cluster 2024-09-12T23:43:07.996064+0000 mon.a (mon.0) 170 : cluster [INF] overall HEALTH_OK 2024-09-12T23:43:09.412 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:09 smithi136 bash[75164]: cluster 2024-09-12T23:43:07.996438+0000 mon.a (mon.0) 171 : cluster [INF] mon.a calling monitor election 2024-09-12T23:43:09.412 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:09 smithi136 bash[75164]: cluster 2024-09-12T23:43:08.005402+0000 mon.b (mon.1) 3 : cluster [INF] mon.b calling monitor election 2024-09-12T23:43:09.412 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:09 smithi136 bash[75164]: cluster 2024-09-12T23:43:08.025767+0000 mon.a (mon.0) 172 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-12T23:43:09.412 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:09 smithi136 bash[75164]: cluster 2024-09-12T23:43:08.034806+0000 mon.a (mon.0) 173 : cluster [DBG] monmap e4: 3 mons at {a=[v2:172.21.15.136:3300/0,v1:172.21.15.136:6789/0],b=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0],c=[v2:172.21.15.136:3301/0,v1:172.21.15.136:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-12T23:43:09.412 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:09 smithi136 bash[75164]: cluster 2024-09-12T23:43:08.053586+0000 mon.a (mon.0) 174 : cluster [DBG] fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-12T23:43:09.412 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:09 smithi136 bash[75164]: cluster 2024-09-12T23:43:08.053639+0000 mon.a (mon.0) 175 : cluster [DBG] osdmap e335: 8 total, 8 up, 8 in 2024-09-12T23:43:09.412 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:09 smithi136 bash[75164]: cluster 2024-09-12T23:43:08.054204+0000 mon.a (mon.0) 176 : cluster [DBG] mgrmap e42: y(active, since 59s), standbys: x 2024-09-12T23:43:09.412 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:09 smithi136 bash[75164]: cluster 2024-09-12T23:43:08.054497+0000 mon.a (mon.0) 177 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-09-12T23:43:09.412 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:09 smithi136 bash[75164]: cluster 2024-09-12T23:43:08.054533+0000 mon.a (mon.0) 178 : cluster [INF] Cluster is now healthy 2024-09-12T23:43:09.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:09 smithi136 bash[75164]: cluster 2024-09-12T23:43:08.061664+0000 mon.a (mon.0) 179 : cluster [INF] overall HEALTH_OK 2024-09-12T23:43:09.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:09 smithi136 bash[75164]: audit 2024-09-12T23:43:08.068759+0000 mon.a (mon.0) 180 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:09.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:09 smithi136 bash[75164]: audit 2024-09-12T23:43:08.080391+0000 mon.a (mon.0) 181 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:09.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:09 smithi136 bash[75164]: audit 2024-09-12T23:43:08.083385+0000 mon.a (mon.0) 182 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:43:09.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:09 smithi136 bash[75164]: cluster 2024-09-12T23:43:08.665417+0000 mgr.y (mgr.24661) 54 : cluster [DBG] pgmap v77: 121 pgs: 121 active+clean; 284 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 6.3 MiB/s rd, 2.0 MiB/s wr, 12 op/s 2024-09-12T23:43:09.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:09 smithi136 bash[75164]: audit 2024-09-12T23:43:08.741987+0000 mon.a (mon.0) 183 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:43:09.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:43:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:43:09] "GET /metrics HTTP/1.1" 200 36195 "" "Prometheus/2.43.0" 2024-09-12T23:43:10.061 INFO:tasks.rados.rados.0.smithi136.stdout:1409: done (1 left) 2024-09-12T23:43:10.061 INFO:tasks.rados.rados.0.smithi136.stdout:1410: read oid 9 snap -1 2024-09-12T23:43:10.061 INFO:tasks.rados.rados.0.smithi136.stdout:1410: expect (ObjNum 399 snap 116 seq_num 399) 2024-09-12T23:43:10.061 INFO:tasks.rados.rados.0.smithi136.stdout:1411: copy_from oid 46 from oid 7 current snap is 136 2024-09-12T23:43:10.061 INFO:tasks.rados.rados.0.smithi136.stdout:1412: read oid 39 snap -1 2024-09-12T23:43:10.061 INFO:tasks.rados.rados.0.smithi136.stdout:1412: expect (ObjNum 399 snap 116 seq_num 399) 2024-09-12T23:43:10.061 INFO:tasks.rados.rados.0.smithi136.stdout:1413: read oid 42 snap 127 2024-09-12T23:43:10.061 INFO:tasks.rados.rados.0.smithi136.stdout:1413: expect deleted 2024-09-12T23:43:10.061 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 18 current snap is 136 2024-09-12T23:43:10.062 INFO:tasks.rados.rados.0.smithi136.stdout:1414: seq_num 458 ranges {5226496=475136} 2024-09-12T23:43:10.094 INFO:tasks.rados.rados.0.smithi136.stdout:1414: writing smithi13667081-18 from 5226496 to 5701632 tid 1 2024-09-12T23:43:10.094 INFO:tasks.rados.rados.0.smithi136.stdout:1415: rollback oid 29 current snap is 136 2024-09-12T23:43:10.094 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 29 to 130 2024-09-12T23:43:10.095 INFO:tasks.rados.rados.0.smithi136.stdout:1407: expect (ObjNum 262 snap 71 seq_num 262) 2024-09-12T23:43:10.269 INFO:tasks.rados.rados.0.smithi136.stdout:1415: finishing rollback tid 0 to smithi13667081-29 2024-09-12T23:43:10.269 INFO:tasks.rados.rados.0.smithi136.stdout:1415: finishing rollback tid 1 to smithi13667081-29 2024-09-12T23:43:10.269 INFO:tasks.rados.rados.0.smithi136.stdout:1415: finishing rollback tid 2 to smithi13667081-29 2024-09-12T23:43:10.269 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 29 v 698 (ObjNum 423 snap 125 seq_num 423) dirty exists 2024-09-12T23:43:10.269 INFO:tasks.rados.rados.0.smithi136.stdout:1411: finishing copy_from racing read to smithi13667081-46 2024-09-12T23:43:10.269 INFO:tasks.rados.rados.0.smithi136.stdout:1412: expect (ObjNum 399 snap 116 seq_num 399) 2024-09-12T23:43:10.344 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:10 smithi181 bash[67587]: cluster 2024-09-12T23:43:09.051475+0000 mon.a (mon.0) 184 : cluster [DBG] osdmap e336: 8 total, 8 up, 8 in 2024-09-12T23:43:10.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:10 smithi136 bash[76371]: cluster 2024-09-12T23:43:09.051475+0000 mon.a (mon.0) 184 : cluster [DBG] osdmap e336: 8 total, 8 up, 8 in 2024-09-12T23:43:10.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:10 smithi136 bash[75164]: cluster 2024-09-12T23:43:09.051475+0000 mon.a (mon.0) 184 : cluster [DBG] osdmap e336: 8 total, 8 up, 8 in 2024-09-12T23:43:10.513 INFO:tasks.rados.rados.0.smithi136.stdout:1414: finishing write tid 1 to smithi13667081-18 2024-09-12T23:43:10.513 INFO:tasks.rados.rados.0.smithi136.stdout:1414: finishing write tid 2 to smithi13667081-18 2024-09-12T23:43:10.513 INFO:tasks.rados.rados.0.smithi136.stdout:1414: finishing write tid 3 to smithi13667081-18 2024-09-12T23:43:10.514 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 18 v 615 (ObjNum 458 snap 136 seq_num 458) dirty exists 2024-09-12T23:43:10.514 INFO:tasks.rados.rados.0.smithi136.stdout:1414: left oid 18 (ObjNum 458 snap 136 seq_num 458) 2024-09-12T23:43:10.514 INFO:tasks.rados.rados.0.smithi136.stdout:1411: finishing copy_from to smithi13667081-46 2024-09-12T23:43:10.514 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 46 v 613 (ObjNum 350 snap 95 seq_num 350) dirty exists 2024-09-12T23:43:10.514 INFO:tasks.rados.rados.0.smithi136.stdout:1410: expect (ObjNum 399 snap 116 seq_num 399) 2024-09-12T23:43:10.757 INFO:tasks.rados.rados.0.smithi136.stdout:1407: done (6 left) 2024-09-12T23:43:10.757 INFO:tasks.rados.rados.0.smithi136.stdout:1410: done (5 left) 2024-09-12T23:43:10.758 INFO:tasks.rados.rados.0.smithi136.stdout:1411: done (4 left) 2024-09-12T23:43:10.758 INFO:tasks.rados.rados.0.smithi136.stdout:1412: done (3 left) 2024-09-12T23:43:10.758 INFO:tasks.rados.rados.0.smithi136.stdout:1413: done (2 left) 2024-09-12T23:43:10.758 INFO:tasks.rados.rados.0.smithi136.stdout:1414: done (1 left) 2024-09-12T23:43:10.758 INFO:tasks.rados.rados.0.smithi136.stdout:1415: done (0 left) 2024-09-12T23:43:10.758 INFO:tasks.rados.rados.0.smithi136.stdout:1416: delete oid 12 current snap is 136 2024-09-12T23:43:10.761 INFO:tasks.rados.rados.0.smithi136.stdout:1416: done (0 left) 2024-09-12T23:43:10.761 INFO:tasks.rados.rados.0.smithi136.stdout:1417: snap_remove snap 129 2024-09-12T23:43:11.068 INFO:tasks.rados.rados.0.smithi136.stdout:1417: done (0 left) 2024-09-12T23:43:11.068 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 47 current snap is 136 2024-09-12T23:43:11.069 INFO:tasks.rados.rados.0.smithi136.stdout:1418: seq_num 460 ranges {0=507904} 2024-09-12T23:43:11.078 INFO:tasks.rados.rados.0.smithi136.stdout:1418: writing smithi13667081-47 from 0 to 507904 tid 1 2024-09-12T23:43:11.079 INFO:tasks.rados.rados.0.smithi136.stdout:1419: rmattr oid 13 current snap is 136 2024-09-12T23:43:11.080 INFO:tasks.rados.rados.0.smithi136.stdout:1419: done (1 left) 2024-09-12T23:43:11.080 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 20 current snap is 136 2024-09-12T23:43:11.080 INFO:tasks.rados.rados.0.smithi136.stdout:1420: seq_num 461 ranges {0=524288} 2024-09-12T23:43:11.091 INFO:tasks.rados.rados.0.smithi136.stdout:1420: writing smithi13667081-20 from 0 to 524288 tid 1 2024-09-12T23:43:11.091 INFO:tasks.rados.rados.0.smithi136.stdout:1421: copy_from oid 40 from oid 36 current snap is 136 2024-09-12T23:43:11.092 INFO:tasks.rados.rados.0.smithi136.stdout:1422: read oid 50 snap -1 2024-09-12T23:43:11.092 INFO:tasks.rados.rados.0.smithi136.stdout:1422: expect deleted 2024-09-12T23:43:11.092 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 3 current snap is 136 2024-09-12T23:43:11.092 INFO:tasks.rados.rados.0.smithi136.stdout:1423: seq_num 462 ranges {2514944=540672} 2024-09-12T23:43:11.132 INFO:tasks.rados.rados.0.smithi136.stdout:1423: writing smithi13667081-3 from 2514944 to 3055616 tid 1 2024-09-12T23:43:11.132 INFO:tasks.rados.rados.0.smithi136.stdout:1424: snap_create 2024-09-12T23:43:11.132 INFO:tasks.rados.rados.0.smithi136.stdout:1421: finishing copy_from to smithi13667081-40 2024-09-12T23:43:11.132 INFO:tasks.rados.rados.0.smithi136.stdout:1421: got expected ENOENT (src dne) 2024-09-12T23:43:11.132 INFO:tasks.rados.rados.0.smithi136.stdout:1421: finishing copy_from racing read to smithi13667081-40 2024-09-12T23:43:11.132 INFO:tasks.rados.rados.0.smithi136.stdout:1418: finishing write tid 1 to smithi13667081-47 2024-09-12T23:43:11.132 INFO:tasks.rados.rados.0.smithi136.stdout:1418: finishing write tid 2 to smithi13667081-47 2024-09-12T23:43:11.132 INFO:tasks.rados.rados.0.smithi136.stdout:1418: finishing write tid 3 to smithi13667081-47 2024-09-12T23:43:11.132 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 47 v 611 (ObjNum 460 snap 136 seq_num 460) dirty exists 2024-09-12T23:43:11.133 INFO:tasks.rados.rados.0.smithi136.stdout:1418: left oid 47 (ObjNum 460 snap 136 seq_num 460) 2024-09-12T23:43:11.133 INFO:tasks.rados.rados.0.smithi136.stdout:1420: finishing write tid 1 to smithi13667081-20 2024-09-12T23:43:11.133 INFO:tasks.rados.rados.0.smithi136.stdout:1420: finishing write tid 2 to smithi13667081-20 2024-09-12T23:43:11.133 INFO:tasks.rados.rados.0.smithi136.stdout:1420: finishing write tid 3 to smithi13667081-20 2024-09-12T23:43:11.133 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 20 v 299 (ObjNum 461 snap 136 seq_num 461) dirty exists 2024-09-12T23:43:11.133 INFO:tasks.rados.rados.0.smithi136.stdout:1420: left oid 20 (ObjNum 461 snap 136 seq_num 461) 2024-09-12T23:43:11.135 INFO:tasks.rados.rados.0.smithi136.stdout:1423: finishing write tid 1 to smithi13667081-3 2024-09-12T23:43:11.135 INFO:tasks.rados.rados.0.smithi136.stdout:1423: finishing write tid 2 to smithi13667081-3 2024-09-12T23:43:11.136 INFO:tasks.rados.rados.0.smithi136.stdout:1423: finishing write tid 3 to smithi13667081-3 2024-09-12T23:43:11.136 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 3 v 673 (ObjNum 462 snap 136 seq_num 462) dirty exists 2024-09-12T23:43:11.136 INFO:tasks.rados.rados.0.smithi136.stdout:1423: left oid 3 (ObjNum 462 snap 136 seq_num 462) 2024-09-12T23:43:11.293 INFO:tasks.rados.rados.0.smithi136.stdout:1418: done (5 left) 2024-09-12T23:43:11.293 INFO:tasks.rados.rados.0.smithi136.stdout:1420: done (4 left) 2024-09-12T23:43:11.293 INFO:tasks.rados.rados.0.smithi136.stdout:1421: done (3 left) 2024-09-12T23:43:11.293 INFO:tasks.rados.rados.0.smithi136.stdout:1422: done (2 left) 2024-09-12T23:43:11.294 INFO:tasks.rados.rados.0.smithi136.stdout:1423: done (1 left) 2024-09-12T23:43:11.294 INFO:tasks.rados.rados.0.smithi136.stdout:1424: done (0 left) 2024-09-12T23:43:11.294 INFO:tasks.rados.rados.0.smithi136.stdout:1425: copy_from oid 49 from oid 24 current snap is 137 2024-09-12T23:43:11.294 INFO:tasks.rados.rados.0.smithi136.stdout:1426: rollback oid 15 current snap is 137 2024-09-12T23:43:11.294 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 15 to 128 2024-09-12T23:43:11.294 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 35 current snap is 137 2024-09-12T23:43:11.294 INFO:tasks.rados.rados.0.smithi136.stdout:1427: seq_num 463 ranges {1449984=557056} 2024-09-12T23:43:11.310 INFO:tasks.rados.rados.0.smithi136.stdout:1427: writing smithi13667081-35 from 1449984 to 2007040 tid 1 2024-09-12T23:43:11.310 INFO:tasks.rados.rados.0.smithi136.stdout:1428: snap_create 2024-09-12T23:43:11.312 INFO:tasks.rados.rados.0.smithi136.stdout:1426: finishing rollback tid 1 to smithi13667081-15 2024-09-12T23:43:11.312 INFO:tasks.rados.rados.0.smithi136.stdout:1426: finishing rollback tid 2 to smithi13667081-15 2024-09-12T23:43:11.312 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 15 v 675 (ObjNum 426 snap 125 seq_num 426) dirty exists 2024-09-12T23:43:11.327 INFO:tasks.rados.rados.0.smithi136.stdout:1427: finishing write tid 1 to smithi13667081-35 2024-09-12T23:43:11.327 INFO:tasks.rados.rados.0.smithi136.stdout:1427: finishing write tid 2 to smithi13667081-35 2024-09-12T23:43:11.330 INFO:tasks.rados.rados.0.smithi136.stdout:1427: finishing write tid 3 to smithi13667081-35 2024-09-12T23:43:11.330 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 35 v 992 (ObjNum 463 snap 137 seq_num 463) dirty exists 2024-09-12T23:43:11.330 INFO:tasks.rados.rados.0.smithi136.stdout:1427: left oid 35 (ObjNum 463 snap 137 seq_num 463) 2024-09-12T23:43:11.333 INFO:tasks.rados.rados.0.smithi136.stdout:1425: finishing copy_from racing read to smithi13667081-49 2024-09-12T23:43:11.339 INFO:tasks.rados.rados.0.smithi136.stdout:1425: finishing copy_from to smithi13667081-49 2024-09-12T23:43:11.339 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 49 v 199 (ObjNum 399 snap 116 seq_num 399) dirty exists 2024-09-12T23:43:11.344 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:11 smithi181 bash[67587]: cluster 2024-09-12T23:43:10.061057+0000 mon.a (mon.0) 185 : cluster [DBG] osdmap e337: 8 total, 8 up, 8 in 2024-09-12T23:43:11.344 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:11 smithi181 bash[67587]: cluster 2024-09-12T23:43:10.665946+0000 mgr.y (mgr.24661) 55 : cluster [DBG] pgmap v80: 121 pgs: 121 active+clean; 284 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 1.6 MiB/s rd, 0 op/s 2024-09-12T23:43:11.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:11 smithi136 bash[75164]: cluster 2024-09-12T23:43:10.061057+0000 mon.a (mon.0) 185 : cluster [DBG] osdmap e337: 8 total, 8 up, 8 in 2024-09-12T23:43:11.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:11 smithi136 bash[75164]: cluster 2024-09-12T23:43:10.665946+0000 mgr.y (mgr.24661) 55 : cluster [DBG] pgmap v80: 121 pgs: 121 active+clean; 284 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 1.6 MiB/s rd, 0 op/s 2024-09-12T23:43:11.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:11 smithi136 bash[76371]: cluster 2024-09-12T23:43:10.061057+0000 mon.a (mon.0) 185 : cluster [DBG] osdmap e337: 8 total, 8 up, 8 in 2024-09-12T23:43:11.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:11 smithi136 bash[76371]: cluster 2024-09-12T23:43:10.665946+0000 mgr.y (mgr.24661) 55 : cluster [DBG] pgmap v80: 121 pgs: 121 active+clean; 284 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 1.6 MiB/s rd, 0 op/s 2024-09-12T23:43:12.296 INFO:tasks.rados.rados.0.smithi136.stdout:1425: done (3 left) 2024-09-12T23:43:12.296 INFO:tasks.rados.rados.0.smithi136.stdout:1426: done (2 left) 2024-09-12T23:43:12.296 INFO:tasks.rados.rados.0.smithi136.stdout:1427: done (1 left) 2024-09-12T23:43:12.296 INFO:tasks.rados.rados.0.smithi136.stdout:1428: done (0 left) 2024-09-12T23:43:12.296 INFO:tasks.rados.rados.0.smithi136.stdout:1429: rollback oid 1 current snap is 138 2024-09-12T23:43:12.297 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 1 to 135 2024-09-12T23:43:12.297 INFO:tasks.rados.rados.0.smithi136.stdout:1430: rollback oid 36 current snap is 138 2024-09-12T23:43:12.297 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 36 to 122 2024-09-12T23:43:12.297 INFO:tasks.rados.rados.0.smithi136.stdout:1431: snap_remove snap 93 2024-09-12T23:43:12.344 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:12 smithi181 bash[67587]: cluster 2024-09-12T23:43:11.071939+0000 mon.a (mon.0) 186 : cluster [DBG] osdmap e338: 8 total, 8 up, 8 in 2024-09-12T23:43:12.344 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:12 smithi181 bash[67587]: cluster 2024-09-12T23:43:11.296514+0000 mon.a (mon.0) 187 : cluster [DBG] osdmap e339: 8 total, 8 up, 8 in 2024-09-12T23:43:12.344 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:43:12 smithi181 bash[60471]: ts=2024-09-12T23:43:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:43:12.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:12 smithi136 bash[75164]: cluster 2024-09-12T23:43:11.071939+0000 mon.a (mon.0) 186 : cluster [DBG] osdmap e338: 8 total, 8 up, 8 in 2024-09-12T23:43:12.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:12 smithi136 bash[75164]: cluster 2024-09-12T23:43:11.296514+0000 mon.a (mon.0) 187 : cluster [DBG] osdmap e339: 8 total, 8 up, 8 in 2024-09-12T23:43:12.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:12 smithi136 bash[76371]: cluster 2024-09-12T23:43:11.071939+0000 mon.a (mon.0) 186 : cluster [DBG] osdmap e338: 8 total, 8 up, 8 in 2024-09-12T23:43:12.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:12 smithi136 bash[76371]: cluster 2024-09-12T23:43:11.296514+0000 mon.a (mon.0) 187 : cluster [DBG] osdmap e339: 8 total, 8 up, 8 in 2024-09-12T23:43:13.313 INFO:tasks.rados.rados.0.smithi136.stdout:1430: finishing rollback tid 1 to smithi13667081-36 2024-09-12T23:43:13.313 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 36 v 542 (ObjNum 113 snap 0 seq_num 4243602832) dirty dne 2024-09-12T23:43:13.313 INFO:tasks.rados.rados.0.smithi136.stdout:1429: finishing rollback tid 0 to smithi13667081-1 2024-09-12T23:43:13.313 INFO:tasks.rados.rados.0.smithi136.stdout:1429: finishing rollback tid 1 to smithi13667081-1 2024-09-12T23:43:13.313 INFO:tasks.rados.rados.0.smithi136.stdout:1429: finishing rollback tid 2 to smithi13667081-1 2024-09-12T23:43:13.313 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 1 v 731 (ObjNum 449 snap 135 seq_num 449) dirty exists 2024-09-12T23:43:13.313 INFO:tasks.rados.rados.0.smithi136.stdout:1429: done (2 left) 2024-09-12T23:43:13.314 INFO:tasks.rados.rados.0.smithi136.stdout:1430: done (1 left) 2024-09-12T23:43:13.314 INFO:tasks.rados.rados.0.smithi136.stdout:1431: done (0 left) 2024-09-12T23:43:13.314 INFO:tasks.rados.rados.0.smithi136.stdout:1432: snap_remove snap 128 2024-09-12T23:43:13.594 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:43:13 smithi181 bash[60471]: ts=2024-09-12T23:43:13.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi136\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:43:13.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:13 smithi181 bash[67587]: cluster 2024-09-12T23:43:12.299705+0000 mon.a (mon.0) 188 : cluster [DBG] osdmap e340: 8 total, 8 up, 8 in 2024-09-12T23:43:13.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:13 smithi181 bash[67587]: cluster 2024-09-12T23:43:12.666729+0000 mgr.y (mgr.24661) 56 : cluster [DBG] pgmap v84: 121 pgs: 121 active+clean; 284 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 3.4 MiB/s rd, 1 op/s 2024-09-12T23:43:13.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:13 smithi136 bash[75164]: cluster 2024-09-12T23:43:12.299705+0000 mon.a (mon.0) 188 : cluster [DBG] osdmap e340: 8 total, 8 up, 8 in 2024-09-12T23:43:13.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:13 smithi136 bash[75164]: cluster 2024-09-12T23:43:12.666729+0000 mgr.y (mgr.24661) 56 : cluster [DBG] pgmap v84: 121 pgs: 121 active+clean; 284 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 3.4 MiB/s rd, 1 op/s 2024-09-12T23:43:13.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:13 smithi136 bash[76371]: cluster 2024-09-12T23:43:12.299705+0000 mon.a (mon.0) 188 : cluster [DBG] osdmap e340: 8 total, 8 up, 8 in 2024-09-12T23:43:13.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:13 smithi136 bash[76371]: cluster 2024-09-12T23:43:12.666729+0000 mgr.y (mgr.24661) 56 : cluster [DBG] pgmap v84: 121 pgs: 121 active+clean; 284 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 3.4 MiB/s rd, 1 op/s 2024-09-12T23:43:14.315 INFO:tasks.rados.rados.0.smithi136.stdout:1432: done (0 left) 2024-09-12T23:43:14.316 INFO:tasks.rados.rados.0.smithi136.stdout:1433: snap_remove snap 135 2024-09-12T23:43:14.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:14 smithi181 bash[67587]: cluster 2024-09-12T23:43:13.312485+0000 mon.a (mon.0) 189 : cluster [DBG] osdmap e341: 8 total, 8 up, 8 in 2024-09-12T23:43:14.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:14 smithi181 bash[67587]: audit 2024-09-12T23:43:13.945297+0000 mon.a (mon.0) 190 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:14.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:14 smithi181 bash[67587]: audit 2024-09-12T23:43:13.957453+0000 mon.a (mon.0) 191 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:14.594 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:43:14 smithi181 bash[60471]: ts=2024-09-12T23:43:14.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:43:14.595 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:43:14 smithi181 bash[60471]: ts=2024-09-12T23:43:14.456Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:43:14.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:14 smithi136 bash[76371]: cluster 2024-09-12T23:43:13.312485+0000 mon.a (mon.0) 189 : cluster [DBG] osdmap e341: 8 total, 8 up, 8 in 2024-09-12T23:43:14.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:14 smithi136 bash[76371]: audit 2024-09-12T23:43:13.945297+0000 mon.a (mon.0) 190 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:14.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:14 smithi136 bash[76371]: audit 2024-09-12T23:43:13.957453+0000 mon.a (mon.0) 191 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:14.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:14 smithi136 bash[75164]: cluster 2024-09-12T23:43:13.312485+0000 mon.a (mon.0) 189 : cluster [DBG] osdmap e341: 8 total, 8 up, 8 in 2024-09-12T23:43:14.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:14 smithi136 bash[75164]: audit 2024-09-12T23:43:13.945297+0000 mon.a (mon.0) 190 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:14.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:14 smithi136 bash[75164]: audit 2024-09-12T23:43:13.957453+0000 mon.a (mon.0) 191 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:15.327 INFO:tasks.rados.rados.0.smithi136.stdout:1433: done (0 left) 2024-09-12T23:43:15.327 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 31 current snap is 138 2024-09-12T23:43:15.327 INFO:tasks.rados.rados.0.smithi136.stdout:1434: seq_num 464 ranges {4300800=573440} 2024-09-12T23:43:15.398 INFO:tasks.rados.rados.0.smithi136.stdout:1434: writing smithi13667081-31 from 4300800 to 4874240 tid 1 2024-09-12T23:43:15.398 INFO:tasks.rados.rados.0.smithi136.stdout:1435: snap_remove snap 131 2024-09-12T23:43:15.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:15 smithi181 bash[67587]: cluster 2024-09-12T23:43:14.317486+0000 mon.a (mon.0) 192 : cluster [DBG] osdmap e342: 8 total, 8 up, 8 in 2024-09-12T23:43:15.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:15 smithi181 bash[67587]: cluster 2024-09-12T23:43:14.668094+0000 mgr.y (mgr.24661) 57 : cluster [DBG] pgmap v87: 121 pgs: 121 active+clean; 246 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.4 MiB/s rd, 3.0 MiB/s wr, 9 op/s 2024-09-12T23:43:15.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:15 smithi181 bash[67587]: audit 2024-09-12T23:43:14.735706+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:15.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:15 smithi181 bash[67587]: audit 2024-09-12T23:43:14.744483+0000 mon.a (mon.0) 194 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:15.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:15 smithi136 bash[76371]: cluster 2024-09-12T23:43:14.317486+0000 mon.a (mon.0) 192 : cluster [DBG] osdmap e342: 8 total, 8 up, 8 in 2024-09-12T23:43:15.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:15 smithi136 bash[76371]: cluster 2024-09-12T23:43:14.668094+0000 mgr.y (mgr.24661) 57 : cluster [DBG] pgmap v87: 121 pgs: 121 active+clean; 246 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.4 MiB/s rd, 3.0 MiB/s wr, 9 op/s 2024-09-12T23:43:15.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:15 smithi136 bash[76371]: audit 2024-09-12T23:43:14.735706+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:15.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:15 smithi136 bash[76371]: audit 2024-09-12T23:43:14.744483+0000 mon.a (mon.0) 194 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:15.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:15 smithi136 bash[75164]: cluster 2024-09-12T23:43:14.317486+0000 mon.a (mon.0) 192 : cluster [DBG] osdmap e342: 8 total, 8 up, 8 in 2024-09-12T23:43:15.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:15 smithi136 bash[75164]: cluster 2024-09-12T23:43:14.668094+0000 mgr.y (mgr.24661) 57 : cluster [DBG] pgmap v87: 121 pgs: 121 active+clean; 246 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.4 MiB/s rd, 3.0 MiB/s wr, 9 op/s 2024-09-12T23:43:15.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:15 smithi136 bash[75164]: audit 2024-09-12T23:43:14.735706+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:15.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:15 smithi136 bash[75164]: audit 2024-09-12T23:43:14.744483+0000 mon.a (mon.0) 194 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:16.300 INFO:tasks.rados.rados.0.smithi136.stdout:1435: done (1 left) 2024-09-12T23:43:16.300 INFO:tasks.rados.rados.0.smithi136.stdout:1436: delete oid 3 current snap is 138 2024-09-12T23:43:16.300 INFO:tasks.rados.rados.0.smithi136.stdout:1434: finishing write tid 1 to smithi13667081-31 2024-09-12T23:43:16.300 INFO:tasks.rados.rados.0.smithi136.stdout:1434: finishing write tid 2 to smithi13667081-31 2024-09-12T23:43:16.301 INFO:tasks.rados.rados.0.smithi136.stdout:1434: finishing write tid 3 to smithi13667081-31 2024-09-12T23:43:16.301 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 31 v 694 (ObjNum 464 snap 138 seq_num 464) dirty exists 2024-09-12T23:43:16.301 INFO:tasks.rados.rados.0.smithi136.stdout:1434: left oid 31 (ObjNum 464 snap 138 seq_num 464) 2024-09-12T23:43:16.324 INFO:tasks.rados.rados.0.smithi136.stdout:1434: done (1 left) 2024-09-12T23:43:16.325 INFO:tasks.rados.rados.0.smithi136.stdout:1436: done (0 left) 2024-09-12T23:43:16.325 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 15 current snap is 138 2024-09-12T23:43:16.325 INFO:tasks.rados.rados.0.smithi136.stdout:1437: seq_num 466 ranges {8478720=614400} 2024-09-12T23:43:16.394 INFO:tasks.rados.rados.0.smithi136.stdout:1437: writing smithi13667081-15 from 8478720 to 9093120 tid 1 2024-09-12T23:43:16.395 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 22 current snap is 138 2024-09-12T23:43:16.395 INFO:tasks.rados.rados.0.smithi136.stdout:1438: seq_num 467 ranges {0=630784} 2024-09-12T23:43:16.398 INFO:tasks.rados.rados.0.smithi136.stdout:1438: writing smithi13667081-22 from 0 to 630784 tid 1 2024-09-12T23:43:16.399 INFO:tasks.rados.rados.0.smithi136.stdout:1439: read oid 30 snap -1 2024-09-12T23:43:16.399 INFO:tasks.rados.rados.0.smithi136.stdout:1439: expect (ObjNum 427 snap 127 seq_num 427) 2024-09-12T23:43:16.399 INFO:tasks.rados.rados.0.smithi136.stdout:1440: read oid 50 snap -1 2024-09-12T23:43:16.399 INFO:tasks.rados.rados.0.smithi136.stdout:1440: expect deleted 2024-09-12T23:43:16.399 INFO:tasks.rados.rados.0.smithi136.stdout:1441: snap_create 2024-09-12T23:43:16.405 INFO:tasks.rados.rados.0.smithi136.stdout:1438: finishing write tid 1 to smithi13667081-22 2024-09-12T23:43:16.405 INFO:tasks.rados.rados.0.smithi136.stdout:1438: finishing write tid 2 to smithi13667081-22 2024-09-12T23:43:16.408 INFO:tasks.rados.rados.0.smithi136.stdout:1438: finishing write tid 3 to smithi13667081-22 2024-09-12T23:43:16.408 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 22 v 733 (ObjNum 467 snap 138 seq_num 467) dirty exists 2024-09-12T23:43:16.408 INFO:tasks.rados.rados.0.smithi136.stdout:1438: left oid 22 (ObjNum 467 snap 138 seq_num 467) 2024-09-12T23:43:16.408 INFO:tasks.rados.rados.0.smithi136.stdout:1437: finishing write tid 1 to smithi13667081-15 2024-09-12T23:43:16.409 INFO:tasks.rados.rados.0.smithi136.stdout:1437: finishing write tid 2 to smithi13667081-15 2024-09-12T23:43:16.411 INFO:tasks.rados.rados.0.smithi136.stdout:1437: finishing write tid 3 to smithi13667081-15 2024-09-12T23:43:16.411 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 15 v 715 (ObjNum 466 snap 138 seq_num 466) dirty exists 2024-09-12T23:43:16.411 INFO:tasks.rados.rados.0.smithi136.stdout:1437: left oid 15 (ObjNum 466 snap 138 seq_num 466) 2024-09-12T23:43:16.426 INFO:tasks.rados.rados.0.smithi136.stdout:1439: expect (ObjNum 427 snap 127 seq_num 427) 2024-09-12T23:43:16.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:16 smithi181 bash[67587]: cluster 2024-09-12T23:43:15.330201+0000 mon.a (mon.0) 195 : cluster [DBG] osdmap e343: 8 total, 8 up, 8 in 2024-09-12T23:43:16.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:16 smithi181 bash[67587]: cluster 2024-09-12T23:43:16.303621+0000 mon.a (mon.0) 196 : cluster [DBG] osdmap e344: 8 total, 8 up, 8 in 2024-09-12T23:43:16.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:16 smithi136 bash[75164]: cluster 2024-09-12T23:43:15.330201+0000 mon.a (mon.0) 195 : cluster [DBG] osdmap e343: 8 total, 8 up, 8 in 2024-09-12T23:43:16.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:16 smithi136 bash[75164]: cluster 2024-09-12T23:43:16.303621+0000 mon.a (mon.0) 196 : cluster [DBG] osdmap e344: 8 total, 8 up, 8 in 2024-09-12T23:43:16.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:16 smithi136 bash[76371]: cluster 2024-09-12T23:43:15.330201+0000 mon.a (mon.0) 195 : cluster [DBG] osdmap e343: 8 total, 8 up, 8 in 2024-09-12T23:43:16.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:16 smithi136 bash[76371]: cluster 2024-09-12T23:43:16.303621+0000 mon.a (mon.0) 196 : cluster [DBG] osdmap e344: 8 total, 8 up, 8 in 2024-09-12T23:43:17.343 INFO:tasks.rados.rados.0.smithi136.stdout:1437: done (4 left) 2024-09-12T23:43:17.343 INFO:tasks.rados.rados.0.smithi136.stdout:1438: done (3 left) 2024-09-12T23:43:17.343 INFO:tasks.rados.rados.0.smithi136.stdout:1439: done (2 left) 2024-09-12T23:43:17.343 INFO:tasks.rados.rados.0.smithi136.stdout:1440: done (1 left) 2024-09-12T23:43:17.343 INFO:tasks.rados.rados.0.smithi136.stdout:1441: done (0 left) 2024-09-12T23:43:17.343 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 17 current snap is 139 2024-09-12T23:43:17.344 INFO:tasks.rados.rados.0.smithi136.stdout:1442: seq_num 468 ranges {303104=647168} 2024-09-12T23:43:17.349 INFO:tasks.rados.rados.0.smithi136.stdout:1442: writing smithi13667081-17 from 303104 to 950272 tid 1 2024-09-12T23:43:17.349 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 35 current snap is 139 2024-09-12T23:43:17.349 INFO:tasks.rados.rados.0.smithi136.stdout:1443: seq_num 469 ranges {2007040=663552} 2024-09-12T23:43:17.367 INFO:tasks.rados.rados.0.smithi136.stdout:1443: writing smithi13667081-35 from 2007040 to 2670592 tid 1 2024-09-12T23:43:17.367 INFO:tasks.rados.rados.0.smithi136.stdout:1444: snap_create 2024-09-12T23:43:17.367 INFO:tasks.rados.rados.0.smithi136.stdout:1442: finishing write tid 1 to smithi13667081-17 2024-09-12T23:43:17.367 INFO:tasks.rados.rados.0.smithi136.stdout:1442: finishing write tid 2 to smithi13667081-17 2024-09-12T23:43:17.367 INFO:tasks.rados.rados.0.smithi136.stdout:1442: finishing write tid 3 to smithi13667081-17 2024-09-12T23:43:17.367 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 17 v 1027 (ObjNum 468 snap 139 seq_num 468) dirty exists 2024-09-12T23:43:17.367 INFO:tasks.rados.rados.0.smithi136.stdout:1442: left oid 17 (ObjNum 468 snap 139 seq_num 468) 2024-09-12T23:43:17.372 INFO:tasks.rados.rados.0.smithi136.stdout:1443: finishing write tid 1 to smithi13667081-35 2024-09-12T23:43:17.372 INFO:tasks.rados.rados.0.smithi136.stdout:1443: finishing write tid 2 to smithi13667081-35 2024-09-12T23:43:17.375 INFO:tasks.rados.rados.0.smithi136.stdout:1443: finishing write tid 3 to smithi13667081-35 2024-09-12T23:43:17.375 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 35 v 1030 (ObjNum 469 snap 139 seq_num 469) dirty exists 2024-09-12T23:43:17.375 INFO:tasks.rados.rados.0.smithi136.stdout:1443: left oid 35 (ObjNum 469 snap 139 seq_num 469) 2024-09-12T23:43:17.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:17 smithi181 bash[67587]: cluster 2024-09-12T23:43:16.668796+0000 mgr.y (mgr.24661) 58 : cluster [DBG] pgmap v90: 121 pgs: 121 active+clean; 246 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 2.7 MiB/s wr, 8 op/s 2024-09-12T23:43:17.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:17 smithi136 bash[75164]: cluster 2024-09-12T23:43:16.668796+0000 mgr.y (mgr.24661) 58 : cluster [DBG] pgmap v90: 121 pgs: 121 active+clean; 246 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 2.7 MiB/s wr, 8 op/s 2024-09-12T23:43:17.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:17 smithi136 bash[76371]: cluster 2024-09-12T23:43:16.668796+0000 mgr.y (mgr.24661) 58 : cluster [DBG] pgmap v90: 121 pgs: 121 active+clean; 246 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 2.7 MiB/s wr, 8 op/s 2024-09-12T23:43:18.348 INFO:tasks.rados.rados.0.smithi136.stdout:1442: done (2 left) 2024-09-12T23:43:18.349 INFO:tasks.rados.rados.0.smithi136.stdout:1443: done (1 left) 2024-09-12T23:43:18.349 INFO:tasks.rados.rados.0.smithi136.stdout:1444: done (0 left) 2024-09-12T23:43:18.349 INFO:tasks.rados.rados.0.smithi136.stdout:1445: delete oid 44 current snap is 140 2024-09-12T23:43:18.356 INFO:tasks.rados.rados.0.smithi136.stdout:1445: done (0 left) 2024-09-12T23:43:18.356 INFO:tasks.rados.rados.0.smithi136.stdout:1446: rmattr oid 21 current snap is 140 2024-09-12T23:43:18.356 INFO:tasks.rados.rados.0.smithi136.stdout:1447: read oid 35 snap -1 2024-09-12T23:43:18.356 INFO:tasks.rados.rados.0.smithi136.stdout:1447: expect (ObjNum 469 snap 139 seq_num 469) 2024-09-12T23:43:18.356 INFO:tasks.rados.rados.0.smithi136.stdout:1448: read oid 47 snap -1 2024-09-12T23:43:18.356 INFO:tasks.rados.rados.0.smithi136.stdout:1448: expect (ObjNum 460 snap 136 seq_num 460) 2024-09-12T23:43:18.356 INFO:tasks.rados.rados.0.smithi136.stdout:1449: snap_create 2024-09-12T23:43:18.370 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 21 v 734 (ObjNum 441 snap 132 seq_num 441) dirty exists 2024-09-12T23:43:18.372 INFO:tasks.rados.rados.0.smithi136.stdout:1448: expect (ObjNum 460 snap 136 seq_num 460) 2024-09-12T23:43:18.386 INFO:tasks.rados.rados.0.smithi136.stdout:1447: expect (ObjNum 469 snap 139 seq_num 469) 2024-09-12T23:43:18.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:18 smithi181 bash[67587]: cluster 2024-09-12T23:43:17.346701+0000 mon.a (mon.0) 197 : cluster [DBG] osdmap e345: 8 total, 8 up, 8 in 2024-09-12T23:43:18.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:18 smithi136 bash[75164]: cluster 2024-09-12T23:43:17.346701+0000 mon.a (mon.0) 197 : cluster [DBG] osdmap e345: 8 total, 8 up, 8 in 2024-09-12T23:43:18.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:18 smithi136 bash[76371]: cluster 2024-09-12T23:43:17.346701+0000 mon.a (mon.0) 197 : cluster [DBG] osdmap e345: 8 total, 8 up, 8 in 2024-09-12T23:43:19.094 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:43:18 smithi181 bash[60471]: ts=2024-09-12T23:43:18.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:43:19.361 INFO:tasks.rados.rados.0.smithi136.stdout:1446: done (3 left) 2024-09-12T23:43:19.362 INFO:tasks.rados.rados.0.smithi136.stdout:1447: done (2 left) 2024-09-12T23:43:19.362 INFO:tasks.rados.rados.0.smithi136.stdout:1448: done (1 left) 2024-09-12T23:43:19.362 INFO:tasks.rados.rados.0.smithi136.stdout:1449: done (0 left) 2024-09-12T23:43:19.362 INFO:tasks.rados.rados.0.smithi136.stdout:1450: read oid 40 snap 132 2024-09-12T23:43:19.362 INFO:tasks.rados.rados.0.smithi136.stdout:1450: expect (ObjNum 412 snap 120 seq_num 412) 2024-09-12T23:43:19.362 INFO:tasks.rados.rados.0.smithi136.stdout:1451: delete oid 11 current snap is 141 2024-09-12T23:43:19.400 INFO:tasks.rados.rados.0.smithi136.stdout:1450: expect (ObjNum 412 snap 120 seq_num 412) 2024-09-12T23:43:19.622 INFO:tasks.rados.rados.0.smithi136.stdout:1450: done (1 left) 2024-09-12T23:43:19.622 INFO:tasks.rados.rados.0.smithi136.stdout:1451: done (0 left) 2024-09-12T23:43:19.622 INFO:tasks.rados.rados.0.smithi136.stdout:1452: delete oid 5 current snap is 141 2024-09-12T23:43:19.626 INFO:tasks.rados.rados.0.smithi136.stdout:1452: done (0 left) 2024-09-12T23:43:19.626 INFO:tasks.rados.rados.0.smithi136.stdout:1453: setattr oid 12 current snap is 141 2024-09-12T23:43:19.629 INFO:tasks.rados.rados.0.smithi136.stdout:1454: read oid 1 snap -1 2024-09-12T23:43:19.629 INFO:tasks.rados.rados.0.smithi136.stdout:1454: expect (ObjNum 449 snap 135 seq_num 449) 2024-09-12T23:43:19.629 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 33 current snap is 141 2024-09-12T23:43:19.629 INFO:tasks.rados.rados.0.smithi136.stdout:1455: seq_num 473 ranges {2228224=729088} 2024-09-12T23:43:19.678 INFO:tasks.rados.rados.0.smithi136.stdout:1455: writing smithi13667081-33 from 2228224 to 2957312 tid 1 2024-09-12T23:43:19.678 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 39 current snap is 141 2024-09-12T23:43:19.678 INFO:tasks.rados.rados.0.smithi136.stdout:1456: seq_num 474 ranges {5226496=745472} 2024-09-12T23:43:19.731 INFO:tasks.rados.rados.0.smithi136.stdout:1456: writing smithi13667081-39 from 5226496 to 5971968 tid 1 2024-09-12T23:43:19.732 INFO:tasks.rados.rados.0.smithi136.stdout:1457: rollback oid 28 current snap is 141 2024-09-12T23:43:19.732 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 28 to 125 2024-09-12T23:43:19.732 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 12 v 716 (ObjNum 289 snap 0 seq_num 1) dirty exists 2024-09-12T23:43:19.732 INFO:tasks.rados.rados.0.smithi136.stdout:1454: expect (ObjNum 449 snap 135 seq_num 449) 2024-09-12T23:43:19.756 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:19 smithi181 bash[67587]: cluster 2024-09-12T23:43:18.352347+0000 mon.a (mon.0) 198 : cluster [DBG] osdmap e346: 8 total, 8 up, 8 in 2024-09-12T23:43:19.756 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:19 smithi181 bash[67587]: cluster 2024-09-12T23:43:18.669569+0000 mgr.y (mgr.24661) 59 : cluster [DBG] pgmap v93: 121 pgs: 121 active+clean; 246 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:43:19.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:19 smithi136 bash[76371]: cluster 2024-09-12T23:43:18.352347+0000 mon.a (mon.0) 198 : cluster [DBG] osdmap e346: 8 total, 8 up, 8 in 2024-09-12T23:43:19.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:19 smithi136 bash[76371]: cluster 2024-09-12T23:43:18.669569+0000 mgr.y (mgr.24661) 59 : cluster [DBG] pgmap v93: 121 pgs: 121 active+clean; 246 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:43:19.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:19 smithi136 bash[75164]: cluster 2024-09-12T23:43:18.352347+0000 mon.a (mon.0) 198 : cluster [DBG] osdmap e346: 8 total, 8 up, 8 in 2024-09-12T23:43:19.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:19 smithi136 bash[75164]: cluster 2024-09-12T23:43:18.669569+0000 mgr.y (mgr.24661) 59 : cluster [DBG] pgmap v93: 121 pgs: 121 active+clean; 246 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:43:19.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:43:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:43:19] "GET /metrics HTTP/1.1" 200 36195 "" "Prometheus/2.43.0" 2024-09-12T23:43:19.995 INFO:tasks.rados.rados.0.smithi136.stdout:1455: finishing write tid 1 to smithi13667081-33 2024-09-12T23:43:19.995 INFO:tasks.rados.rados.0.smithi136.stdout:1455: finishing write tid 2 to smithi13667081-33 2024-09-12T23:43:19.995 INFO:tasks.rados.rados.0.smithi136.stdout:1455: finishing write tid 3 to smithi13667081-33 2024-09-12T23:43:19.996 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 33 v 699 (ObjNum 473 snap 141 seq_num 473) dirty exists 2024-09-12T23:43:19.996 INFO:tasks.rados.rados.0.smithi136.stdout:1455: left oid 33 (ObjNum 473 snap 141 seq_num 473) 2024-09-12T23:43:19.996 INFO:tasks.rados.rados.0.smithi136.stdout:1457: finishing rollback tid 0 to smithi13667081-28 2024-09-12T23:43:19.996 INFO:tasks.rados.rados.0.smithi136.stdout:1457: finishing rollback tid 1 to smithi13667081-28 2024-09-12T23:43:19.996 INFO:tasks.rados.rados.0.smithi136.stdout:1457: finishing rollback tid 2 to smithi13667081-28 2024-09-12T23:43:19.996 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 28 v 860 (ObjNum 421 snap 123 seq_num 421) dirty exists 2024-09-12T23:43:19.996 INFO:tasks.rados.rados.0.smithi136.stdout:1456: finishing write tid 1 to smithi13667081-39 2024-09-12T23:43:19.996 INFO:tasks.rados.rados.0.smithi136.stdout:1456: finishing write tid 2 to smithi13667081-39 2024-09-12T23:43:19.996 INFO:tasks.rados.rados.0.smithi136.stdout:1456: finishing write tid 3 to smithi13667081-39 2024-09-12T23:43:19.996 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 39 v 244 (ObjNum 474 snap 141 seq_num 474) dirty exists 2024-09-12T23:43:19.996 INFO:tasks.rados.rados.0.smithi136.stdout:1456: left oid 39 (ObjNum 474 snap 141 seq_num 474) 2024-09-12T23:43:19.996 INFO:tasks.rados.rados.0.smithi136.stdout:1453: done (4 left) 2024-09-12T23:43:19.997 INFO:tasks.rados.rados.0.smithi136.stdout:1454: done (3 left) 2024-09-12T23:43:19.997 INFO:tasks.rados.rados.0.smithi136.stdout:1455: done (2 left) 2024-09-12T23:43:19.997 INFO:tasks.rados.rados.0.smithi136.stdout:1456: done (1 left) 2024-09-12T23:43:19.997 INFO:tasks.rados.rados.0.smithi136.stdout:1457: done (0 left) 2024-09-12T23:43:19.997 INFO:tasks.rados.rados.0.smithi136.stdout:1458: rollback oid 5 current snap is 141 2024-09-12T23:43:19.997 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 5 to 132 2024-09-12T23:43:19.997 INFO:tasks.rados.rados.0.smithi136.stdout:1459: snap_create 2024-09-12T23:43:19.998 INFO:tasks.rados.rados.0.smithi136.stdout:1458: finishing rollback tid 1 to smithi13667081-5 2024-09-12T23:43:19.999 INFO:tasks.rados.rados.0.smithi136.stdout:1458: finishing rollback tid 2 to smithi13667081-5 2024-09-12T23:43:19.999 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 5 v 1036 (ObjNum 372 snap 108 seq_num 372) dirty exists 2024-09-12T23:43:20.379 INFO:tasks.rados.rados.0.smithi136.stdout:1458: done (1 left) 2024-09-12T23:43:20.379 INFO:tasks.rados.rados.0.smithi136.stdout:1459: done (0 left) 2024-09-12T23:43:20.379 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 5 current snap is 142 2024-09-12T23:43:20.379 INFO:tasks.rados.rados.0.smithi136.stdout:1460: seq_num 475 ranges {3678208=761856} 2024-09-12T23:43:20.417 INFO:tasks.rados.rados.0.smithi136.stdout:1460: writing smithi13667081-5 from 3678208 to 4440064 tid 1 2024-09-12T23:43:20.417 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 10 current snap is 142 2024-09-12T23:43:20.417 INFO:tasks.rados.rados.0.smithi136.stdout:1461: seq_num 476 ranges {0=778240} 2024-09-12T23:43:20.424 INFO:tasks.rados.rados.0.smithi136.stdout:1461: writing smithi13667081-10 from 0 to 778240 tid 1 2024-09-12T23:43:20.424 INFO:tasks.rados.rados.0.smithi136.stdout:1462: rmattr oid 30 current snap is 142 2024-09-12T23:43:20.424 INFO:tasks.rados.rados.0.smithi136.stdout:1462: done (2 left) 2024-09-12T23:43:20.425 INFO:tasks.rados.rados.0.smithi136.stdout:1463: copy_from oid 33 from oid 50 current snap is 142 2024-09-12T23:43:20.425 INFO:tasks.rados.rados.0.smithi136.stdout:1464: read oid 28 snap -1 2024-09-12T23:43:20.425 INFO:tasks.rados.rados.0.smithi136.stdout:1464: expect (ObjNum 421 snap 123 seq_num 421) 2024-09-12T23:43:20.425 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 22 current snap is 142 2024-09-12T23:43:20.425 INFO:tasks.rados.rados.0.smithi136.stdout:1465: seq_num 477 ranges {630784=794624} 2024-09-12T23:43:20.436 INFO:tasks.rados.rados.0.smithi136.stdout:1465: writing smithi13667081-22 from 630784 to 1425408 tid 1 2024-09-12T23:43:20.436 INFO:tasks.rados.rados.0.smithi136.stdout:1466: copy_from oid 41 from oid 25 current snap is 142 2024-09-12T23:43:20.437 INFO:tasks.rados.rados.0.smithi136.stdout:1463: finishing copy_from to smithi13667081-33 2024-09-12T23:43:20.437 INFO:tasks.rados.rados.0.smithi136.stdout:1463: got expected ENOENT (src dne) 2024-09-12T23:43:20.437 INFO:tasks.rados.rados.0.smithi136.stdout:1463: finishing copy_from racing read to smithi13667081-33 2024-09-12T23:43:20.437 INFO:tasks.rados.rados.0.smithi136.stdout:1460: finishing write tid 1 to smithi13667081-5 2024-09-12T23:43:20.437 INFO:tasks.rados.rados.0.smithi136.stdout:1463: done (5 left) 2024-09-12T23:43:20.437 INFO:tasks.rados.rados.0.smithi136.stdout:1467: snap_remove snap 136 2024-09-12T23:43:20.712 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:20 smithi181 bash[67587]: cluster 2024-09-12T23:43:19.365022+0000 mon.a (mon.0) 199 : cluster [DBG] osdmap e347: 8 total, 8 up, 8 in 2024-09-12T23:43:20.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:20 smithi136 bash[75164]: cluster 2024-09-12T23:43:19.365022+0000 mon.a (mon.0) 199 : cluster [DBG] osdmap e347: 8 total, 8 up, 8 in 2024-09-12T23:43:20.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:20 smithi136 bash[76371]: cluster 2024-09-12T23:43:19.365022+0000 mon.a (mon.0) 199 : cluster [DBG] osdmap e347: 8 total, 8 up, 8 in 2024-09-12T23:43:21.385 INFO:tasks.rados.rados.0.smithi136.stdout:1467: done (5 left) 2024-09-12T23:43:21.385 INFO:tasks.rados.rados.0.smithi136.stdout:1468: snap_remove snap 130 2024-09-12T23:43:21.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:21 smithi136 bash[76371]: cluster 2024-09-12T23:43:20.383047+0000 mon.a (mon.0) 200 : cluster [DBG] osdmap e348: 8 total, 8 up, 8 in 2024-09-12T23:43:21.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:21 smithi136 bash[76371]: cluster 2024-09-12T23:43:20.670352+0000 mgr.y (mgr.24661) 60 : cluster [DBG] pgmap v96: 121 pgs: 121 active+clean; 249 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 764 KiB/s wr, 16 op/s 2024-09-12T23:43:21.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:21 smithi136 bash[75164]: cluster 2024-09-12T23:43:20.383047+0000 mon.a (mon.0) 200 : cluster [DBG] osdmap e348: 8 total, 8 up, 8 in 2024-09-12T23:43:21.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:21 smithi136 bash[75164]: cluster 2024-09-12T23:43:20.670352+0000 mgr.y (mgr.24661) 60 : cluster [DBG] pgmap v96: 121 pgs: 121 active+clean; 249 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 764 KiB/s wr, 16 op/s 2024-09-12T23:43:21.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:21 smithi181 bash[67587]: cluster 2024-09-12T23:43:20.383047+0000 mon.a (mon.0) 200 : cluster [DBG] osdmap e348: 8 total, 8 up, 8 in 2024-09-12T23:43:21.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:21 smithi181 bash[67587]: cluster 2024-09-12T23:43:20.670352+0000 mgr.y (mgr.24661) 60 : cluster [DBG] pgmap v96: 121 pgs: 121 active+clean; 249 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 764 KiB/s wr, 16 op/s 2024-09-12T23:43:22.397 INFO:tasks.rados.rados.0.smithi136.stdout:1468: done (5 left) 2024-09-12T23:43:22.397 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 32 current snap is 142 2024-09-12T23:43:22.397 INFO:tasks.rados.rados.0.smithi136.stdout:1460: finishing write tid 2 to smithi13667081-5 2024-09-12T23:43:22.397 INFO:tasks.rados.rados.0.smithi136.stdout:1461: finishing write tid 1 to smithi13667081-10 2024-09-12T23:43:22.397 INFO:tasks.rados.rados.0.smithi136.stdout:1461: finishing write tid 2 to smithi13667081-10 2024-09-12T23:43:22.397 INFO:tasks.rados.rados.0.smithi136.stdout:1461: finishing write tid 3 to smithi13667081-10 2024-09-12T23:43:22.397 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 10 v 794 (ObjNum 476 snap 142 seq_num 476) dirty exists 2024-09-12T23:43:22.398 INFO:tasks.rados.rados.0.smithi136.stdout:1461: left oid 10 (ObjNum 476 snap 142 seq_num 476) 2024-09-12T23:43:22.398 INFO:tasks.rados.rados.0.smithi136.stdout:1460: finishing write tid 3 to smithi13667081-5 2024-09-12T23:43:22.398 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 5 v 1039 (ObjNum 475 snap 142 seq_num 475) dirty exists 2024-09-12T23:43:22.398 INFO:tasks.rados.rados.0.smithi136.stdout:1460: left oid 5 (ObjNum 475 snap 142 seq_num 475) 2024-09-12T23:43:22.398 INFO:tasks.rados.rados.0.smithi136.stdout:1465: finishing write tid 1 to smithi13667081-22 2024-09-12T23:43:22.398 INFO:tasks.rados.rados.0.smithi136.stdout:1465: finishing write tid 2 to smithi13667081-22 2024-09-12T23:43:22.398 INFO:tasks.rados.rados.0.smithi136.stdout:1465: finishing write tid 3 to smithi13667081-22 2024-09-12T23:43:22.398 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 22 v 736 (ObjNum 477 snap 142 seq_num 477) dirty exists 2024-09-12T23:43:22.398 INFO:tasks.rados.rados.0.smithi136.stdout:1465: left oid 22 (ObjNum 477 snap 142 seq_num 477) 2024-09-12T23:43:22.398 INFO:tasks.rados.rados.0.smithi136.stdout:1464: expect (ObjNum 421 snap 123 seq_num 421) 2024-09-12T23:43:22.564 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:22 smithi181 bash[67587]: cluster 2024-09-12T23:43:21.387975+0000 mon.a (mon.0) 201 : cluster [DBG] osdmap e349: 8 total, 8 up, 8 in 2024-09-12T23:43:22.564 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:43:22 smithi181 bash[60471]: ts=2024-09-12T23:43:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:43:22.581 INFO:tasks.rados.rados.0.smithi136.stdout:1466: finishing copy_from racing read to smithi13667081-41 2024-09-12T23:43:22.581 INFO:tasks.rados.rados.0.smithi136.stdout:1466: finishing copy_from to smithi13667081-41 2024-09-12T23:43:22.581 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 41 v 211 (ObjNum 417 snap 120 seq_num 417) dirty exists 2024-09-12T23:43:22.581 INFO:tasks.rados.rados.0.smithi136.stdout:1469: seq_num 478 ranges {0=409600,409600=401408} 2024-09-12T23:43:22.585 INFO:tasks.rados.rados.0.smithi136.stdout:1469: writing smithi13667081-32 from 0 to 409600 tid 1 2024-09-12T23:43:22.589 INFO:tasks.rados.rados.0.smithi136.stdout:1469: writing smithi13667081-32 from 409600 to 811008 tid 2 2024-09-12T23:43:22.589 INFO:tasks.rados.rados.0.smithi136.stdout:1460: done (5 left) 2024-09-12T23:43:22.589 INFO:tasks.rados.rados.0.smithi136.stdout:1461: done (4 left) 2024-09-12T23:43:22.589 INFO:tasks.rados.rados.0.smithi136.stdout:1464: done (3 left) 2024-09-12T23:43:22.589 INFO:tasks.rados.rados.0.smithi136.stdout:1465: done (2 left) 2024-09-12T23:43:22.589 INFO:tasks.rados.rados.0.smithi136.stdout:1466: done (1 left) 2024-09-12T23:43:22.589 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 3 current snap is 142 2024-09-12T23:43:22.589 INFO:tasks.rados.rados.0.smithi136.stdout:1470: seq_num 479 ranges {0=425984,425984=401408} 2024-09-12T23:43:22.593 INFO:tasks.rados.rados.0.smithi136.stdout:1470: writing smithi13667081-3 from 0 to 425984 tid 1 2024-09-12T23:43:22.598 INFO:tasks.rados.rados.0.smithi136.stdout:1470: writing smithi13667081-3 from 425984 to 827392 tid 2 2024-09-12T23:43:22.598 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 48 current snap is 142 2024-09-12T23:43:22.598 INFO:tasks.rados.rados.0.smithi136.stdout:1471: seq_num 480 ranges {0=442368,442368=401408} 2024-09-12T23:43:22.602 INFO:tasks.rados.rados.0.smithi136.stdout:1471: writing smithi13667081-48 from 0 to 442368 tid 1 2024-09-12T23:43:22.608 INFO:tasks.rados.rados.0.smithi136.stdout:1471: writing smithi13667081-48 from 442368 to 843776 tid 2 2024-09-12T23:43:22.608 INFO:tasks.rados.rados.0.smithi136.stdout:1472: copy_from oid 38 from oid 8 current snap is 142 2024-09-12T23:43:22.608 INFO:tasks.rados.rados.0.smithi136.stdout:1469: finishing write tid 1 to smithi13667081-32 2024-09-12T23:43:22.608 INFO:tasks.rados.rados.0.smithi136.stdout:1469: finishing write tid 2 to smithi13667081-32 2024-09-12T23:43:22.608 INFO:tasks.rados.rados.0.smithi136.stdout:1469: finishing write tid 3 to smithi13667081-32 2024-09-12T23:43:22.609 INFO:tasks.rados.rados.0.smithi136.stdout:1469: finishing write tid 4 to smithi13667081-32 2024-09-12T23:43:22.609 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 32 v 729 (ObjNum 478 snap 142 seq_num 478) dirty exists 2024-09-12T23:43:22.609 INFO:tasks.rados.rados.0.smithi136.stdout:1469: left oid 32 (ObjNum 478 snap 142 seq_num 478) 2024-09-12T23:43:22.609 INFO:tasks.rados.rados.0.smithi136.stdout:1470: finishing write tid 1 to smithi13667081-3 2024-09-12T23:43:22.609 INFO:tasks.rados.rados.0.smithi136.stdout:1470: finishing write tid 2 to smithi13667081-3 2024-09-12T23:43:22.609 INFO:tasks.rados.rados.0.smithi136.stdout:1470: finishing write tid 3 to smithi13667081-3 2024-09-12T23:43:22.609 INFO:tasks.rados.rados.0.smithi136.stdout:1470: finishing write tid 4 to smithi13667081-3 2024-09-12T23:43:22.609 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 3 v 735 (ObjNum 479 snap 142 seq_num 479) dirty exists 2024-09-12T23:43:22.609 INFO:tasks.rados.rados.0.smithi136.stdout:1470: left oid 3 (ObjNum 479 snap 142 seq_num 479) 2024-09-12T23:43:22.609 INFO:tasks.rados.rados.0.smithi136.stdout:1469: done (3 left) 2024-09-12T23:43:22.609 INFO:tasks.rados.rados.0.smithi136.stdout:1470: done (2 left) 2024-09-12T23:43:22.609 INFO:tasks.rados.rados.0.smithi136.stdout:1473: copy_from oid 45 from oid 36 current snap is 142 2024-09-12T23:43:22.610 INFO:tasks.rados.rados.0.smithi136.stdout:1474: rollback oid 34 current snap is 142 2024-09-12T23:43:22.610 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 34 to 127 2024-09-12T23:43:22.610 INFO:tasks.rados.rados.0.smithi136.stdout:1475: read oid 40 snap -1 2024-09-12T23:43:22.610 INFO:tasks.rados.rados.0.smithi136.stdout:1475: expect (ObjNum 412 snap 120 seq_num 412) 2024-09-12T23:43:22.610 INFO:tasks.rados.rados.0.smithi136.stdout:1476: read oid 2 snap -1 2024-09-12T23:43:22.610 INFO:tasks.rados.rados.0.smithi136.stdout:1476: expect (ObjNum 416 snap 120 seq_num 416) 2024-09-12T23:43:22.610 INFO:tasks.rados.rados.0.smithi136.stdout:1477: read oid 21 snap -1 2024-09-12T23:43:22.610 INFO:tasks.rados.rados.0.smithi136.stdout:1477: expect (ObjNum 441 snap 132 seq_num 441) 2024-09-12T23:43:22.610 INFO:tasks.rados.rados.0.smithi136.stdout:1478: copy_from oid 10 from oid 37 current snap is 142 2024-09-12T23:43:22.610 INFO:tasks.rados.rados.0.smithi136.stdout:1479: setattr oid 22 current snap is 142 2024-09-12T23:43:22.612 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 23 current snap is 142 2024-09-12T23:43:22.612 INFO:tasks.rados.rados.0.smithi136.stdout:1480: seq_num 481 ranges {10330112=458752,10788864=401408} 2024-09-12T23:43:22.694 INFO:tasks.rados.rados.0.smithi136.stdout:1480: writing smithi13667081-23 from 10330112 to 10788864 tid 1 2024-09-12T23:43:22.697 INFO:tasks.rados.rados.0.smithi136.stdout:1480: writing smithi13667081-23 from 10788864 to 11190272 tid 2 2024-09-12T23:43:22.697 INFO:tasks.rados.rados.0.smithi136.stdout:1481: rollback oid 7 current snap is 142 2024-09-12T23:43:22.697 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 7 to 132 2024-09-12T23:43:22.697 INFO:tasks.rados.rados.0.smithi136.stdout:1473: finishing copy_from to smithi13667081-45 2024-09-12T23:43:22.697 INFO:tasks.rados.rados.0.smithi136.stdout:1473: got expected ENOENT (src dne) 2024-09-12T23:43:22.698 INFO:tasks.rados.rados.0.smithi136.stdout:1473: finishing copy_from racing read to smithi13667081-45 2024-09-12T23:43:22.698 INFO:tasks.rados.rados.0.smithi136.stdout:1474: finishing rollback tid 0 to smithi13667081-34 2024-09-12T23:43:22.698 INFO:tasks.rados.rados.0.smithi136.stdout:1472: finishing copy_from racing read to smithi13667081-38 2024-09-12T23:43:22.698 INFO:tasks.rados.rados.0.smithi136.stdout:1472: finishing copy_from to smithi13667081-38 2024-09-12T23:43:22.698 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 38 v 690 (ObjNum 286 snap 78 seq_num 286) dirty exists 2024-09-12T23:43:22.698 INFO:tasks.rados.rados.0.smithi136.stdout:1471: finishing write tid 1 to smithi13667081-48 2024-09-12T23:43:22.698 INFO:tasks.rados.rados.0.smithi136.stdout:1471: finishing write tid 2 to smithi13667081-48 2024-09-12T23:43:22.698 INFO:tasks.rados.rados.0.smithi136.stdout:1474: finishing rollback tid 1 to smithi13667081-34 2024-09-12T23:43:22.698 INFO:tasks.rados.rados.0.smithi136.stdout:1471: finishing write tid 3 to smithi13667081-48 2024-09-12T23:43:22.698 INFO:tasks.rados.rados.0.smithi136.stdout:1474: finishing rollback tid 2 to smithi13667081-34 2024-09-12T23:43:22.698 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 34 v 713 (ObjNum 370 snap 108 seq_num 370) dirty exists 2024-09-12T23:43:22.698 INFO:tasks.rados.rados.0.smithi136.stdout:1475: expect (ObjNum 412 snap 120 seq_num 412) 2024-09-12T23:43:22.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:22 smithi136 bash[76371]: cluster 2024-09-12T23:43:21.387975+0000 mon.a (mon.0) 201 : cluster [DBG] osdmap e349: 8 total, 8 up, 8 in 2024-09-12T23:43:22.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:22 smithi136 bash[75164]: cluster 2024-09-12T23:43:21.387975+0000 mon.a (mon.0) 201 : cluster [DBG] osdmap e349: 8 total, 8 up, 8 in 2024-09-12T23:43:22.916 INFO:tasks.rados.rados.0.smithi136.stdout:1471: finishing write tid 4 to smithi13667081-48 2024-09-12T23:43:22.916 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 48 v 876 (ObjNum 480 snap 142 seq_num 480) dirty exists 2024-09-12T23:43:22.916 INFO:tasks.rados.rados.0.smithi136.stdout:1471: left oid 48 (ObjNum 480 snap 142 seq_num 480) 2024-09-12T23:43:22.916 INFO:tasks.rados.rados.0.smithi136.stdout:1478: finishing copy_from racing read to smithi13667081-10 2024-09-12T23:43:22.917 INFO:tasks.rados.rados.0.smithi136.stdout:1478: finishing copy_from to smithi13667081-10 2024-09-12T23:43:22.917 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 10 v 209 (ObjNum 292 snap 79 seq_num 292) dirty exists 2024-09-12T23:43:22.917 INFO:tasks.rados.rados.0.smithi136.stdout:1477: expect (ObjNum 441 snap 132 seq_num 441) 2024-09-12T23:43:23.306 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 22 v 737 (ObjNum 477 snap 142 seq_num 477) dirty exists 2024-09-12T23:43:23.306 INFO:tasks.rados.rados.0.smithi136.stdout:1476: expect (ObjNum 416 snap 120 seq_num 416) 2024-09-12T23:43:23.657 INFO:tasks.rados.rados.0.smithi136.stdout:1480: finishing write tid 1 to smithi13667081-23 2024-09-12T23:43:23.657 INFO:tasks.rados.rados.0.smithi136.stdout:1480: finishing write tid 2 to smithi13667081-23 2024-09-12T23:43:23.657 INFO:tasks.rados.rados.0.smithi136.stdout:1481: finishing rollback tid 0 to smithi13667081-7 2024-09-12T23:43:23.657 INFO:tasks.rados.rados.0.smithi136.stdout:1480: finishing write tid 3 to smithi13667081-23 2024-09-12T23:43:23.658 INFO:tasks.rados.rados.0.smithi136.stdout:1481: finishing rollback tid 1 to smithi13667081-7 2024-09-12T23:43:23.658 INFO:tasks.rados.rados.0.smithi136.stdout:1481: finishing rollback tid 2 to smithi13667081-7 2024-09-12T23:43:23.658 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 7 v 618 (ObjNum 350 snap 95 seq_num 350) dirty exists 2024-09-12T23:43:23.658 INFO:tasks.rados.rados.0.smithi136.stdout:1480: finishing write tid 4 to smithi13667081-23 2024-09-12T23:43:23.658 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 23 v 1061 (ObjNum 481 snap 142 seq_num 481) dirty exists 2024-09-12T23:43:23.658 INFO:tasks.rados.rados.0.smithi136.stdout:1480: left oid 23 (ObjNum 481 snap 142 seq_num 481) 2024-09-12T23:43:23.658 INFO:tasks.rados.rados.0.smithi136.stdout:1471: done (10 left) 2024-09-12T23:43:23.658 INFO:tasks.rados.rados.0.smithi136.stdout:1472: done (9 left) 2024-09-12T23:43:23.658 INFO:tasks.rados.rados.0.smithi136.stdout:1473: done (8 left) 2024-09-12T23:43:23.658 INFO:tasks.rados.rados.0.smithi136.stdout:1474: done (7 left) 2024-09-12T23:43:23.658 INFO:tasks.rados.rados.0.smithi136.stdout:1475: done (6 left) 2024-09-12T23:43:23.658 INFO:tasks.rados.rados.0.smithi136.stdout:1476: done (5 left) 2024-09-12T23:43:23.658 INFO:tasks.rados.rados.0.smithi136.stdout:1477: done (4 left) 2024-09-12T23:43:23.659 INFO:tasks.rados.rados.0.smithi136.stdout:1478: done (3 left) 2024-09-12T23:43:23.659 INFO:tasks.rados.rados.0.smithi136.stdout:1479: done (2 left) 2024-09-12T23:43:23.659 INFO:tasks.rados.rados.0.smithi136.stdout:1480: done (1 left) 2024-09-12T23:43:23.659 INFO:tasks.rados.rados.0.smithi136.stdout:1481: done (0 left) 2024-09-12T23:43:23.659 INFO:tasks.rados.rados.0.smithi136.stdout:1482: read oid 6 snap -1 2024-09-12T23:43:23.659 INFO:tasks.rados.rados.0.smithi136.stdout:1482: expect deleted 2024-09-12T23:43:23.659 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 11 current snap is 142 2024-09-12T23:43:23.659 INFO:tasks.rados.rados.0.smithi136.stdout:1483: seq_num 482 ranges {0=475136,475136=401408} 2024-09-12T23:43:23.661 INFO:tasks.rados.rados.0.smithi136.stdout:1483: writing smithi13667081-11 from 0 to 475136 tid 1 2024-09-12T23:43:23.663 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:23 smithi181 bash[67587]: cluster 2024-09-12T23:43:22.389187+0000 mon.a (mon.0) 202 : cluster [DBG] osdmap e350: 8 total, 8 up, 8 in 2024-09-12T23:43:23.663 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:23 smithi181 bash[67587]: cluster 2024-09-12T23:43:22.670782+0000 mgr.y (mgr.24661) 61 : cluster [DBG] pgmap v99: 121 pgs: 121 active+clean; 249 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 764 KiB/s wr, 16 op/s 2024-09-12T23:43:23.664 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:43:23 smithi181 bash[60471]: ts=2024-09-12T23:43:23.506Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi136\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:43:23.664 INFO:tasks.rados.rados.0.smithi136.stdout:1483: writing smithi13667081-11 from 475136 to 876544 tid 2 2024-09-12T23:43:23.664 INFO:tasks.rados.rados.0.smithi136.stdout:1484: snap_remove snap 132 2024-09-12T23:43:23.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:23 smithi136 bash[76371]: cluster 2024-09-12T23:43:22.389187+0000 mon.a (mon.0) 202 : cluster [DBG] osdmap e350: 8 total, 8 up, 8 in 2024-09-12T23:43:23.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:23 smithi136 bash[76371]: cluster 2024-09-12T23:43:22.670782+0000 mgr.y (mgr.24661) 61 : cluster [DBG] pgmap v99: 121 pgs: 121 active+clean; 249 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 764 KiB/s wr, 16 op/s 2024-09-12T23:43:23.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:23 smithi136 bash[75164]: cluster 2024-09-12T23:43:22.389187+0000 mon.a (mon.0) 202 : cluster [DBG] osdmap e350: 8 total, 8 up, 8 in 2024-09-12T23:43:23.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:23 smithi136 bash[75164]: cluster 2024-09-12T23:43:22.670782+0000 mgr.y (mgr.24661) 61 : cluster [DBG] pgmap v99: 121 pgs: 121 active+clean; 249 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 764 KiB/s wr, 16 op/s 2024-09-12T23:43:24.434 INFO:tasks.rados.rados.0.smithi136.stdout:1483: finishing write tid 1 to smithi13667081-11 2024-09-12T23:43:24.434 INFO:tasks.rados.rados.0.smithi136.stdout:1483: finishing write tid 2 to smithi13667081-11 2024-09-12T23:43:24.434 INFO:tasks.rados.rados.0.smithi136.stdout:1483: finishing write tid 3 to smithi13667081-11 2024-09-12T23:43:24.434 INFO:tasks.rados.rados.0.smithi136.stdout:1483: finishing write tid 4 to smithi13667081-11 2024-09-12T23:43:24.434 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 11 v 1064 (ObjNum 482 snap 142 seq_num 482) dirty exists 2024-09-12T23:43:24.434 INFO:tasks.rados.rados.0.smithi136.stdout:1483: left oid 11 (ObjNum 482 snap 142 seq_num 482) 2024-09-12T23:43:24.440 INFO:tasks.rados.rados.0.smithi136.stdout:1482: done (2 left) 2024-09-12T23:43:24.440 INFO:tasks.rados.rados.0.smithi136.stdout:1483: done (1 left) 2024-09-12T23:43:24.440 INFO:tasks.rados.rados.0.smithi136.stdout:1484: done (0 left) 2024-09-12T23:43:24.440 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 5 current snap is 142 2024-09-12T23:43:24.440 INFO:tasks.rados.rados.0.smithi136.stdout:1485: seq_num 483 ranges {4440064=491520,4931584=401408} 2024-09-12T23:43:24.473 INFO:tasks.rados.rados.0.smithi136.stdout:1485: writing smithi13667081-5 from 4440064 to 4931584 tid 1 2024-09-12T23:43:24.477 INFO:tasks.rados.rados.0.smithi136.stdout:1485: writing smithi13667081-5 from 4931584 to 5332992 tid 2 2024-09-12T23:43:24.477 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 39 current snap is 142 2024-09-12T23:43:24.477 INFO:tasks.rados.rados.0.smithi136.stdout:1486: seq_num 484 ranges {5971968=507904,6479872=401408} 2024-09-12T23:43:24.516 INFO:tasks.rados.rados.0.smithi136.stdout:1486: writing smithi13667081-39 from 5971968 to 6479872 tid 1 2024-09-12T23:43:24.519 INFO:tasks.rados.rados.0.smithi136.stdout:1486: writing smithi13667081-39 from 6479872 to 6881280 tid 2 2024-09-12T23:43:24.519 INFO:tasks.rados.rados.0.smithi136.stdout:1487: copy_from oid 18 from oid 48 current snap is 142 2024-09-12T23:43:24.519 INFO:tasks.rados.rados.0.smithi136.stdout:1485: finishing write tid 1 to smithi13667081-5 2024-09-12T23:43:24.519 INFO:tasks.rados.rados.0.smithi136.stdout:1485: finishing write tid 2 to smithi13667081-5 2024-09-12T23:43:24.519 INFO:tasks.rados.rados.0.smithi136.stdout:1485: finishing write tid 3 to smithi13667081-5 2024-09-12T23:43:24.519 INFO:tasks.rados.rados.0.smithi136.stdout:1485: finishing write tid 4 to smithi13667081-5 2024-09-12T23:43:24.519 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 5 v 1075 (ObjNum 483 snap 142 seq_num 483) dirty exists 2024-09-12T23:43:24.519 INFO:tasks.rados.rados.0.smithi136.stdout:1485: left oid 5 (ObjNum 483 snap 142 seq_num 483) 2024-09-12T23:43:24.519 INFO:tasks.rados.rados.0.smithi136.stdout:1485: done (2 left) 2024-09-12T23:43:24.519 INFO:tasks.rados.rados.0.smithi136.stdout:1488: read oid 5 snap 125 2024-09-12T23:43:24.520 INFO:tasks.rados.rados.0.smithi136.stdout:1488: expect (ObjNum 372 snap 108 seq_num 372) 2024-09-12T23:43:24.520 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 20 current snap is 142 2024-09-12T23:43:24.520 INFO:tasks.rados.rados.0.smithi136.stdout:1489: seq_num 485 ranges {524288=532480,1056768=401408} 2024-09-12T23:43:24.525 INFO:tasks.rados.rados.0.smithi136.stdout:1489: writing smithi13667081-20 from 524288 to 1056768 tid 1 2024-09-12T23:43:24.531 INFO:tasks.rados.rados.0.smithi136.stdout:1489: writing smithi13667081-20 from 1056768 to 1458176 tid 2 2024-09-12T23:43:24.531 INFO:tasks.rados.rados.0.smithi136.stdout:1490: setattr oid 41 current snap is 142 2024-09-12T23:43:24.531 INFO:tasks.rados.rados.0.smithi136.stdout:1486: finishing write tid 1 to smithi13667081-39 2024-09-12T23:43:24.531 INFO:tasks.rados.rados.0.smithi136.stdout:1486: finishing write tid 2 to smithi13667081-39 2024-09-12T23:43:24.531 INFO:tasks.rados.rados.0.smithi136.stdout:1486: finishing write tid 3 to smithi13667081-39 2024-09-12T23:43:24.531 INFO:tasks.rados.rados.0.smithi136.stdout:1491: snap_create 2024-09-12T23:43:24.532 INFO:tasks.rados.rados.0.smithi136.stdout:1489: finishing write tid 1 to smithi13667081-20 2024-09-12T23:43:24.532 INFO:tasks.rados.rados.0.smithi136.stdout:1486: finishing write tid 4 to smithi13667081-39 2024-09-12T23:43:24.532 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 39 v 254 (ObjNum 484 snap 142 seq_num 484) dirty exists 2024-09-12T23:43:24.532 INFO:tasks.rados.rados.0.smithi136.stdout:1486: left oid 39 (ObjNum 484 snap 142 seq_num 484) 2024-09-12T23:43:24.533 INFO:tasks.rados.rados.0.smithi136.stdout:1489: finishing write tid 2 to smithi13667081-20 2024-09-12T23:43:24.533 INFO:tasks.rados.rados.0.smithi136.stdout:1489: finishing write tid 3 to smithi13667081-20 2024-09-12T23:43:24.534 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 41 v 720 (ObjNum 417 snap 120 seq_num 417) dirty exists 2024-09-12T23:43:24.535 INFO:tasks.rados.rados.0.smithi136.stdout:1487: finishing copy_from racing read to smithi13667081-18 2024-09-12T23:43:24.536 INFO:tasks.rados.rados.0.smithi136.stdout:1488: expect (ObjNum 372 snap 108 seq_num 372) 2024-09-12T23:43:24.684 INFO:tasks.rados.rados.0.smithi136.stdout:1489: finishing write tid 4 to smithi13667081-20 2024-09-12T23:43:24.684 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 20 v 302 (ObjNum 485 snap 142 seq_num 485) dirty exists 2024-09-12T23:43:24.684 INFO:tasks.rados.rados.0.smithi136.stdout:1489: left oid 20 (ObjNum 485 snap 142 seq_num 485) 2024-09-12T23:43:24.684 INFO:tasks.rados.rados.0.smithi136.stdout:1487: finishing copy_from to smithi13667081-18 2024-09-12T23:43:24.685 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 18 v 876 (ObjNum 480 snap 142 seq_num 480) dirty exists 2024-09-12T23:43:24.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:24 smithi136 bash[75164]: audit 2024-09-12T23:43:23.742245+0000 mon.a (mon.0) 203 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:43:24.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:24 smithi136 bash[75164]: audit 2024-09-12T23:43:24.338475+0000 mon.a (mon.0) 204 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:24.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:24 smithi136 bash[75164]: audit 2024-09-12T23:43:24.348706+0000 mon.a (mon.0) 205 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:24.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:24 smithi136 bash[75164]: audit 2024-09-12T23:43:24.350674+0000 mon.a (mon.0) 206 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:43:24.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:24 smithi136 bash[75164]: audit 2024-09-12T23:43:24.352260+0000 mon.a (mon.0) 207 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:43:24.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:24 smithi136 bash[75164]: audit 2024-09-12T23:43:24.363951+0000 mon.a (mon.0) 208 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:24.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:24 smithi136 bash[76371]: audit 2024-09-12T23:43:23.742245+0000 mon.a (mon.0) 203 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:43:24.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:24 smithi136 bash[76371]: audit 2024-09-12T23:43:24.338475+0000 mon.a (mon.0) 204 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:24.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:24 smithi136 bash[76371]: audit 2024-09-12T23:43:24.348706+0000 mon.a (mon.0) 205 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:24.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:24 smithi136 bash[76371]: audit 2024-09-12T23:43:24.350674+0000 mon.a (mon.0) 206 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:43:24.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:24 smithi136 bash[76371]: audit 2024-09-12T23:43:24.352260+0000 mon.a (mon.0) 207 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:43:24.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:24 smithi136 bash[76371]: audit 2024-09-12T23:43:24.363951+0000 mon.a (mon.0) 208 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:24.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:24 smithi181 bash[67587]: audit 2024-09-12T23:43:23.742245+0000 mon.a (mon.0) 203 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:43:24.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:24 smithi181 bash[67587]: audit 2024-09-12T23:43:24.338475+0000 mon.a (mon.0) 204 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:24.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:24 smithi181 bash[67587]: audit 2024-09-12T23:43:24.348706+0000 mon.a (mon.0) 205 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:24.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:24 smithi181 bash[67587]: audit 2024-09-12T23:43:24.350674+0000 mon.a (mon.0) 206 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:43:24.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:24 smithi181 bash[67587]: audit 2024-09-12T23:43:24.352260+0000 mon.a (mon.0) 207 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:43:24.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:24 smithi181 bash[67587]: audit 2024-09-12T23:43:24.363951+0000 mon.a (mon.0) 208 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:24.845 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:43:24 smithi181 bash[60471]: ts=2024-09-12T23:43:24.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:43:24.845 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:43:24 smithi181 bash[60471]: ts=2024-09-12T23:43:24.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:43:25.440 INFO:tasks.rados.rados.0.smithi136.stdout:1486: done (5 left) 2024-09-12T23:43:25.440 INFO:tasks.rados.rados.0.smithi136.stdout:1487: done (4 left) 2024-09-12T23:43:25.440 INFO:tasks.rados.rados.0.smithi136.stdout:1488: done (3 left) 2024-09-12T23:43:25.440 INFO:tasks.rados.rados.0.smithi136.stdout:1489: done (2 left) 2024-09-12T23:43:25.440 INFO:tasks.rados.rados.0.smithi136.stdout:1490: done (1 left) 2024-09-12T23:43:25.440 INFO:tasks.rados.rados.0.smithi136.stdout:1491: done (0 left) 2024-09-12T23:43:25.440 INFO:tasks.rados.rados.0.smithi136.stdout:1492: rmattr oid 41 current snap is 143 2024-09-12T23:43:25.441 INFO:tasks.rados.rados.0.smithi136.stdout:1493: snap_remove snap 141 2024-09-12T23:43:25.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:25 smithi136 bash[76371]: cephadm 2024-09-12T23:43:24.431992+0000 mgr.y (mgr.24661) 62 : cephadm [INF] Reconfiguring mon.a (monmap changed)... 2024-09-12T23:43:25.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:25 smithi136 bash[76371]: cluster 2024-09-12T23:43:24.436843+0000 mon.a (mon.0) 209 : cluster [DBG] osdmap e351: 8 total, 8 up, 8 in 2024-09-12T23:43:25.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:25 smithi136 bash[76371]: audit 2024-09-12T23:43:24.437712+0000 mon.a (mon.0) 210 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-12T23:43:25.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:25 smithi136 bash[76371]: audit 2024-09-12T23:43:24.440777+0000 mon.a (mon.0) 211 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-12T23:43:25.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:25 smithi136 bash[76371]: audit 2024-09-12T23:43:24.442851+0000 mon.a (mon.0) 212 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:43:25.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:25 smithi136 bash[76371]: cephadm 2024-09-12T23:43:24.445310+0000 mgr.y (mgr.24661) 63 : cephadm [INF] Reconfiguring daemon mon.a on smithi136 2024-09-12T23:43:25.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:25 smithi136 bash[76371]: cluster 2024-09-12T23:43:24.672039+0000 mgr.y (mgr.24661) 64 : cluster [DBG] pgmap v101: 121 pgs: 121 active+clean; 258 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 3.9 MiB/s wr, 28 op/s 2024-09-12T23:43:25.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:25 smithi136 bash[75164]: cephadm 2024-09-12T23:43:24.431992+0000 mgr.y (mgr.24661) 62 : cephadm [INF] Reconfiguring mon.a (monmap changed)... 2024-09-12T23:43:25.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:25 smithi136 bash[75164]: cluster 2024-09-12T23:43:24.436843+0000 mon.a (mon.0) 209 : cluster [DBG] osdmap e351: 8 total, 8 up, 8 in 2024-09-12T23:43:25.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:25 smithi136 bash[75164]: audit 2024-09-12T23:43:24.437712+0000 mon.a (mon.0) 210 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-12T23:43:25.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:25 smithi136 bash[75164]: audit 2024-09-12T23:43:24.440777+0000 mon.a (mon.0) 211 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-12T23:43:25.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:25 smithi136 bash[75164]: audit 2024-09-12T23:43:24.442851+0000 mon.a (mon.0) 212 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:43:25.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:25 smithi136 bash[75164]: cephadm 2024-09-12T23:43:24.445310+0000 mgr.y (mgr.24661) 63 : cephadm [INF] Reconfiguring daemon mon.a on smithi136 2024-09-12T23:43:25.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:25 smithi136 bash[75164]: cluster 2024-09-12T23:43:24.672039+0000 mgr.y (mgr.24661) 64 : cluster [DBG] pgmap v101: 121 pgs: 121 active+clean; 258 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 3.9 MiB/s wr, 28 op/s 2024-09-12T23:43:25.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:25 smithi181 bash[67587]: cephadm 2024-09-12T23:43:24.431992+0000 mgr.y (mgr.24661) 62 : cephadm [INF] Reconfiguring mon.a (monmap changed)... 2024-09-12T23:43:25.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:25 smithi181 bash[67587]: cluster 2024-09-12T23:43:24.436843+0000 mon.a (mon.0) 209 : cluster [DBG] osdmap e351: 8 total, 8 up, 8 in 2024-09-12T23:43:25.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:25 smithi181 bash[67587]: audit 2024-09-12T23:43:24.437712+0000 mon.a (mon.0) 210 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-12T23:43:25.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:25 smithi181 bash[67587]: audit 2024-09-12T23:43:24.440777+0000 mon.a (mon.0) 211 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-12T23:43:25.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:25 smithi181 bash[67587]: audit 2024-09-12T23:43:24.442851+0000 mon.a (mon.0) 212 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:43:25.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:25 smithi181 bash[67587]: cephadm 2024-09-12T23:43:24.445310+0000 mgr.y (mgr.24661) 63 : cephadm [INF] Reconfiguring daemon mon.a on smithi136 2024-09-12T23:43:25.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:25 smithi181 bash[67587]: cluster 2024-09-12T23:43:24.672039+0000 mgr.y (mgr.24661) 64 : cluster [DBG] pgmap v101: 121 pgs: 121 active+clean; 258 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 3.9 MiB/s wr, 28 op/s 2024-09-12T23:43:26.296 INFO:tasks.rados.rados.0.smithi136.stdout:1493: done (1 left) 2024-09-12T23:43:26.297 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 43 current snap is 143 2024-09-12T23:43:26.297 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 41 v 722 (ObjNum 417 snap 120 seq_num 417) dirty exists 2024-09-12T23:43:26.297 INFO:tasks.rados.rados.0.smithi136.stdout:1494: seq_num 486 ranges {5226496=548864,5775360=401408} 2024-09-12T23:43:26.328 INFO:tasks.rados.rados.0.smithi136.stdout:1494: writing smithi13667081-43 from 5226496 to 5775360 tid 1 2024-09-12T23:43:26.331 INFO:tasks.rados.rados.0.smithi136.stdout:1494: writing smithi13667081-43 from 5775360 to 6176768 tid 2 2024-09-12T23:43:26.331 INFO:tasks.rados.rados.0.smithi136.stdout:1492: done (1 left) 2024-09-12T23:43:26.331 INFO:tasks.rados.rados.0.smithi136.stdout:1495: delete oid 26 current snap is 143 2024-09-12T23:43:26.335 INFO:tasks.rados.rados.0.smithi136.stdout:1495: done (1 left) 2024-09-12T23:43:26.335 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 18 current snap is 143 2024-09-12T23:43:26.335 INFO:tasks.rados.rados.0.smithi136.stdout:1496: seq_num 488 ranges {843776=581632,1425408=401408} 2024-09-12T23:43:26.344 INFO:tasks.rados.rados.0.smithi136.stdout:1496: writing smithi13667081-18 from 843776 to 1425408 tid 1 2024-09-12T23:43:26.347 INFO:tasks.rados.rados.0.smithi136.stdout:1496: writing smithi13667081-18 from 1425408 to 1826816 tid 2 2024-09-12T23:43:26.347 INFO:tasks.rados.rados.0.smithi136.stdout:1497: delete oid 1 current snap is 143 2024-09-12T23:43:26.349 INFO:tasks.rados.rados.0.smithi136.stdout:1494: finishing write tid 1 to smithi13667081-43 2024-09-12T23:43:26.350 INFO:tasks.rados.rados.0.smithi136.stdout:1494: finishing write tid 2 to smithi13667081-43 2024-09-12T23:43:26.350 INFO:tasks.rados.rados.0.smithi136.stdout:1494: finishing write tid 3 to smithi13667081-43 2024-09-12T23:43:26.354 INFO:tasks.rados.rados.0.smithi136.stdout:1497: done (2 left) 2024-09-12T23:43:26.354 INFO:tasks.rados.rados.0.smithi136.stdout:1498: copy_from oid 22 from oid 16 current snap is 143 2024-09-12T23:43:26.354 INFO:tasks.rados.rados.0.smithi136.stdout:1494: finishing write tid 4 to smithi13667081-43 2024-09-12T23:43:26.354 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 43 v 888 (ObjNum 486 snap 143 seq_num 486) dirty exists 2024-09-12T23:43:26.354 INFO:tasks.rados.rados.0.smithi136.stdout:1494: left oid 43 (ObjNum 486 snap 143 seq_num 486) 2024-09-12T23:43:26.354 INFO:tasks.rados.rados.0.smithi136.stdout:1494: done (2 left) 2024-09-12T23:43:26.354 INFO:tasks.rados.rados.0.smithi136.stdout:1499: snap_create 2024-09-12T23:43:26.362 INFO:tasks.rados.rados.0.smithi136.stdout:1496: finishing write tid 1 to smithi13667081-18 2024-09-12T23:43:26.362 INFO:tasks.rados.rados.0.smithi136.stdout:1496: finishing write tid 2 to smithi13667081-18 2024-09-12T23:43:26.365 INFO:tasks.rados.rados.0.smithi136.stdout:1496: finishing write tid 3 to smithi13667081-18 2024-09-12T23:43:26.367 INFO:tasks.rados.rados.0.smithi136.stdout:1496: finishing write tid 4 to smithi13667081-18 2024-09-12T23:43:26.367 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 18 v 879 (ObjNum 488 snap 143 seq_num 488) dirty exists 2024-09-12T23:43:26.367 INFO:tasks.rados.rados.0.smithi136.stdout:1496: left oid 18 (ObjNum 488 snap 143 seq_num 488) 2024-09-12T23:43:26.372 INFO:tasks.rados.rados.0.smithi136.stdout:1498: finishing copy_from racing read to smithi13667081-22 2024-09-12T23:43:26.375 INFO:tasks.rados.rados.0.smithi136.stdout:1498: finishing copy_from to smithi13667081-22 2024-09-12T23:43:26.376 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 22 v 791 (ObjNum 427 snap 127 seq_num 427) dirty exists 2024-09-12T23:43:26.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:26 smithi136 bash[75164]: cluster 2024-09-12T23:43:25.442896+0000 mon.a (mon.0) 213 : cluster [DBG] osdmap e352: 8 total, 8 up, 8 in 2024-09-12T23:43:26.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:26 smithi136 bash[75164]: cluster 2024-09-12T23:43:26.300850+0000 mon.a (mon.0) 214 : cluster [DBG] osdmap e353: 8 total, 8 up, 8 in 2024-09-12T23:43:26.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:26 smithi136 bash[76371]: cluster 2024-09-12T23:43:25.442896+0000 mon.a (mon.0) 213 : cluster [DBG] osdmap e352: 8 total, 8 up, 8 in 2024-09-12T23:43:26.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:26 smithi136 bash[76371]: cluster 2024-09-12T23:43:26.300850+0000 mon.a (mon.0) 214 : cluster [DBG] osdmap e353: 8 total, 8 up, 8 in 2024-09-12T23:43:26.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:26 smithi181 bash[67587]: cluster 2024-09-12T23:43:25.442896+0000 mon.a (mon.0) 213 : cluster [DBG] osdmap e352: 8 total, 8 up, 8 in 2024-09-12T23:43:26.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:26 smithi181 bash[67587]: cluster 2024-09-12T23:43:26.300850+0000 mon.a (mon.0) 214 : cluster [DBG] osdmap e353: 8 total, 8 up, 8 in 2024-09-12T23:43:27.297 INFO:tasks.rados.rados.0.smithi136.stdout:1496: done (2 left) 2024-09-12T23:43:27.297 INFO:tasks.rados.rados.0.smithi136.stdout:1498: done (1 left) 2024-09-12T23:43:27.297 INFO:tasks.rados.rados.0.smithi136.stdout:1499: done (0 left) 2024-09-12T23:43:27.297 INFO:tasks.rados.rados.0.smithi136.stdout:1500: setattr oid 25 current snap is 144 2024-09-12T23:43:27.298 INFO:tasks.rados.rados.0.smithi136.stdout:1501: setattr oid 47 current snap is 144 2024-09-12T23:43:27.300 INFO:tasks.rados.rados.0.smithi136.stdout:1502: snap_create 2024-09-12T23:43:27.307 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 47 v 612 (ObjNum 460 snap 136 seq_num 460) dirty exists 2024-09-12T23:43:27.308 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 25 v 258 (ObjNum 417 snap 120 seq_num 417) dirty exists 2024-09-12T23:43:27.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:27 smithi136 bash[75164]: cluster 2024-09-12T23:43:26.672770+0000 mgr.y (mgr.24661) 65 : cluster [DBG] pgmap v104: 121 pgs: 121 active+clean; 258 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 3.9 MiB/s wr, 28 op/s 2024-09-12T23:43:27.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:27 smithi136 bash[75164]: audit 2024-09-12T23:43:26.990719+0000 mon.a (mon.0) 215 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:27.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:27 smithi136 bash[75164]: audit 2024-09-12T23:43:26.999319+0000 mon.a (mon.0) 216 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:27.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:27 smithi136 bash[75164]: audit 2024-09-12T23:43:27.000848+0000 mon.a (mon.0) 217 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-12T23:43:27.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:27 smithi136 bash[75164]: audit 2024-09-12T23:43:27.002060+0000 mon.a (mon.0) 218 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-12T23:43:27.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:27 smithi136 bash[75164]: audit 2024-09-12T23:43:27.003145+0000 mon.a (mon.0) 219 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:43:27.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:27 smithi136 bash[75164]: cluster 2024-09-12T23:43:27.301253+0000 mon.a (mon.0) 220 : cluster [DBG] osdmap e354: 8 total, 8 up, 8 in 2024-09-12T23:43:27.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:27 smithi136 bash[76371]: cluster 2024-09-12T23:43:26.672770+0000 mgr.y (mgr.24661) 65 : cluster [DBG] pgmap v104: 121 pgs: 121 active+clean; 258 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 3.9 MiB/s wr, 28 op/s 2024-09-12T23:43:27.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:27 smithi136 bash[76371]: audit 2024-09-12T23:43:26.990719+0000 mon.a (mon.0) 215 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:27.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:27 smithi136 bash[76371]: audit 2024-09-12T23:43:26.999319+0000 mon.a (mon.0) 216 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:27.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:27 smithi136 bash[76371]: audit 2024-09-12T23:43:27.000848+0000 mon.a (mon.0) 217 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-12T23:43:27.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:27 smithi136 bash[76371]: audit 2024-09-12T23:43:27.002060+0000 mon.a (mon.0) 218 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-12T23:43:27.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:27 smithi136 bash[76371]: audit 2024-09-12T23:43:27.003145+0000 mon.a (mon.0) 219 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:43:27.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:27 smithi136 bash[76371]: cluster 2024-09-12T23:43:27.301253+0000 mon.a (mon.0) 220 : cluster [DBG] osdmap e354: 8 total, 8 up, 8 in 2024-09-12T23:43:27.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:27 smithi181 bash[67587]: cluster 2024-09-12T23:43:26.672770+0000 mgr.y (mgr.24661) 65 : cluster [DBG] pgmap v104: 121 pgs: 121 active+clean; 258 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 3.9 MiB/s wr, 28 op/s 2024-09-12T23:43:27.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:27 smithi181 bash[67587]: audit 2024-09-12T23:43:26.990719+0000 mon.a (mon.0) 215 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:27.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:27 smithi181 bash[67587]: audit 2024-09-12T23:43:26.999319+0000 mon.a (mon.0) 216 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:27.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:27 smithi181 bash[67587]: audit 2024-09-12T23:43:27.000848+0000 mon.a (mon.0) 217 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-12T23:43:27.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:27 smithi181 bash[67587]: audit 2024-09-12T23:43:27.002060+0000 mon.a (mon.0) 218 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-12T23:43:27.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:27 smithi181 bash[67587]: audit 2024-09-12T23:43:27.003145+0000 mon.a (mon.0) 219 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:43:27.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:27 smithi181 bash[67587]: cluster 2024-09-12T23:43:27.301253+0000 mon.a (mon.0) 220 : cluster [DBG] osdmap e354: 8 total, 8 up, 8 in 2024-09-12T23:43:28.302 INFO:tasks.rados.rados.0.smithi136.stdout:1500: done (2 left) 2024-09-12T23:43:28.303 INFO:tasks.rados.rados.0.smithi136.stdout:1501: done (1 left) 2024-09-12T23:43:28.303 INFO:tasks.rados.rados.0.smithi136.stdout:1502: done (0 left) 2024-09-12T23:43:28.303 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 7 current snap is 145 2024-09-12T23:43:28.303 INFO:tasks.rados.rados.0.smithi136.stdout:1503: seq_num 490 ranges {3514368=614400,4128768=401408} 2024-09-12T23:43:28.341 INFO:tasks.rados.rados.0.smithi136.stdout:1503: writing smithi13667081-7 from 3514368 to 4128768 tid 1 2024-09-12T23:43:28.345 INFO:tasks.rados.rados.0.smithi136.stdout:1503: writing smithi13667081-7 from 4128768 to 4530176 tid 2 2024-09-12T23:43:28.345 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 32 current snap is 145 2024-09-12T23:43:28.345 INFO:tasks.rados.rados.0.smithi136.stdout:1504: seq_num 491 ranges {811008=630784,1441792=401408} 2024-09-12T23:43:28.359 INFO:tasks.rados.rados.0.smithi136.stdout:1504: writing smithi13667081-32 from 811008 to 1441792 tid 1 2024-09-12T23:43:28.362 INFO:tasks.rados.rados.0.smithi136.stdout:1504: writing smithi13667081-32 from 1441792 to 1843200 tid 2 2024-09-12T23:43:28.362 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 34 current snap is 145 2024-09-12T23:43:28.362 INFO:tasks.rados.rados.0.smithi136.stdout:1503: finishing write tid 1 to smithi13667081-7 2024-09-12T23:43:28.362 INFO:tasks.rados.rados.0.smithi136.stdout:1503: finishing write tid 2 to smithi13667081-7 2024-09-12T23:43:28.363 INFO:tasks.rados.rados.0.smithi136.stdout:1503: finishing write tid 3 to smithi13667081-7 2024-09-12T23:43:28.363 INFO:tasks.rados.rados.0.smithi136.stdout:1503: finishing write tid 4 to smithi13667081-7 2024-09-12T23:43:28.363 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 7 v 882 (ObjNum 490 snap 145 seq_num 490) dirty exists 2024-09-12T23:43:28.363 INFO:tasks.rados.rados.0.smithi136.stdout:1503: left oid 7 (ObjNum 490 snap 145 seq_num 490) 2024-09-12T23:43:28.363 INFO:tasks.rados.rados.0.smithi136.stdout:1505: seq_num 492 ranges {1409024=647168,2056192=401408} 2024-09-12T23:43:28.379 INFO:tasks.rados.rados.0.smithi136.stdout:1505: writing smithi13667081-34 from 1409024 to 2056192 tid 1 2024-09-12T23:43:28.383 INFO:tasks.rados.rados.0.smithi136.stdout:1505: writing smithi13667081-34 from 2056192 to 2457600 tid 2 2024-09-12T23:43:28.383 INFO:tasks.rados.rados.0.smithi136.stdout:1503: done (2 left) 2024-09-12T23:43:28.383 INFO:tasks.rados.rados.0.smithi136.stdout:1506: snap_create 2024-09-12T23:43:28.383 INFO:tasks.rados.rados.0.smithi136.stdout:1504: finishing write tid 1 to smithi13667081-32 2024-09-12T23:43:28.383 INFO:tasks.rados.rados.0.smithi136.stdout:1504: finishing write tid 2 to smithi13667081-32 2024-09-12T23:43:28.383 INFO:tasks.rados.rados.0.smithi136.stdout:1504: finishing write tid 3 to smithi13667081-32 2024-09-12T23:43:28.384 INFO:tasks.rados.rados.0.smithi136.stdout:1504: finishing write tid 4 to smithi13667081-32 2024-09-12T23:43:28.384 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 32 v 732 (ObjNum 491 snap 145 seq_num 491) dirty exists 2024-09-12T23:43:28.384 INFO:tasks.rados.rados.0.smithi136.stdout:1504: left oid 32 (ObjNum 491 snap 145 seq_num 491) 2024-09-12T23:43:28.388 INFO:tasks.rados.rados.0.smithi136.stdout:1505: finishing write tid 1 to smithi13667081-34 2024-09-12T23:43:28.388 INFO:tasks.rados.rados.0.smithi136.stdout:1505: finishing write tid 2 to smithi13667081-34 2024-09-12T23:43:28.388 INFO:tasks.rados.rados.0.smithi136.stdout:1505: finishing write tid 3 to smithi13667081-34 2024-09-12T23:43:28.389 INFO:tasks.rados.rados.0.smithi136.stdout:1505: finishing write tid 4 to smithi13667081-34 2024-09-12T23:43:28.390 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 34 v 730 (ObjNum 492 snap 145 seq_num 492) dirty exists 2024-09-12T23:43:28.390 INFO:tasks.rados.rados.0.smithi136.stdout:1505: left oid 34 (ObjNum 492 snap 145 seq_num 492) 2024-09-12T23:43:28.749 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:28 smithi136 bash[76371]: cephadm 2024-09-12T23:43:27.000485+0000 mgr.y (mgr.24661) 66 : cephadm [INF] Reconfiguring mgr.y (monmap changed)... 2024-09-12T23:43:28.750 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:28 smithi136 bash[76371]: cephadm 2024-09-12T23:43:27.004263+0000 mgr.y (mgr.24661) 67 : cephadm [INF] Reconfiguring daemon mgr.y on smithi136 2024-09-12T23:43:28.750 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:28 smithi136 bash[76371]: cluster 2024-09-12T23:43:28.306477+0000 mon.a (mon.0) 221 : cluster [DBG] osdmap e355: 8 total, 8 up, 8 in 2024-09-12T23:43:28.750 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:28 smithi136 bash[75164]: cephadm 2024-09-12T23:43:27.000485+0000 mgr.y (mgr.24661) 66 : cephadm [INF] Reconfiguring mgr.y (monmap changed)... 2024-09-12T23:43:28.750 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:28 smithi136 bash[75164]: cephadm 2024-09-12T23:43:27.004263+0000 mgr.y (mgr.24661) 67 : cephadm [INF] Reconfiguring daemon mgr.y on smithi136 2024-09-12T23:43:28.750 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:28 smithi136 bash[75164]: cluster 2024-09-12T23:43:28.306477+0000 mon.a (mon.0) 221 : cluster [DBG] osdmap e355: 8 total, 8 up, 8 in 2024-09-12T23:43:28.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:28 smithi181 bash[67587]: cephadm 2024-09-12T23:43:27.000485+0000 mgr.y (mgr.24661) 66 : cephadm [INF] Reconfiguring mgr.y (monmap changed)... 2024-09-12T23:43:28.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:28 smithi181 bash[67587]: cephadm 2024-09-12T23:43:27.004263+0000 mgr.y (mgr.24661) 67 : cephadm [INF] Reconfiguring daemon mgr.y on smithi136 2024-09-12T23:43:28.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:28 smithi181 bash[67587]: cluster 2024-09-12T23:43:28.306477+0000 mon.a (mon.0) 221 : cluster [DBG] osdmap e355: 8 total, 8 up, 8 in 2024-09-12T23:43:28.845 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:43:28 smithi181 bash[60471]: ts=2024-09-12T23:43:28.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:43:29.306 INFO:tasks.rados.rados.0.smithi136.stdout:1504: done (2 left) 2024-09-12T23:43:29.306 INFO:tasks.rados.rados.0.smithi136.stdout:1505: done (1 left) 2024-09-12T23:43:29.306 INFO:tasks.rados.rados.0.smithi136.stdout:1506: done (0 left) 2024-09-12T23:43:29.306 INFO:tasks.rados.rados.0.smithi136.stdout:1507: delete oid 40 current snap is 146 2024-09-12T23:43:29.311 INFO:tasks.rados.rados.0.smithi136.stdout:1507: done (0 left) 2024-09-12T23:43:29.311 INFO:tasks.rados.rados.0.smithi136.stdout:1508: snap_remove snap 143 2024-09-12T23:43:29.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:43:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:43:29] "GET /metrics HTTP/1.1" 200 36197 "" "Prometheus/2.43.0" 2024-09-12T23:43:29.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:29 smithi136 bash[75164]: cluster 2024-09-12T23:43:28.673427+0000 mgr.y (mgr.24661) 68 : cluster [DBG] pgmap v107: 121 pgs: 121 active+clean; 258 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:43:29.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:29 smithi136 bash[75164]: audit 2024-09-12T23:43:29.250300+0000 mon.a (mon.0) 222 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:29.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:29 smithi136 bash[75164]: audit 2024-09-12T23:43:29.259247+0000 mon.a (mon.0) 223 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:29.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:29 smithi136 bash[75164]: audit 2024-09-12T23:43:29.260892+0000 mon.a (mon.0) 224 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-12T23:43:29.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:29 smithi136 bash[75164]: audit 2024-09-12T23:43:29.261804+0000 mon.a (mon.0) 225 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-12T23:43:29.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:29 smithi136 bash[75164]: audit 2024-09-12T23:43:29.262572+0000 mon.a (mon.0) 226 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:43:29.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:29 smithi136 bash[75164]: cluster 2024-09-12T23:43:29.309824+0000 mon.a (mon.0) 227 : cluster [DBG] osdmap e356: 8 total, 8 up, 8 in 2024-09-12T23:43:29.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:29 smithi136 bash[76371]: cluster 2024-09-12T23:43:28.673427+0000 mgr.y (mgr.24661) 68 : cluster [DBG] pgmap v107: 121 pgs: 121 active+clean; 258 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:43:29.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:29 smithi136 bash[76371]: audit 2024-09-12T23:43:29.250300+0000 mon.a (mon.0) 222 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:29.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:29 smithi136 bash[76371]: audit 2024-09-12T23:43:29.259247+0000 mon.a (mon.0) 223 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:29.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:29 smithi136 bash[76371]: audit 2024-09-12T23:43:29.260892+0000 mon.a (mon.0) 224 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-12T23:43:29.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:29 smithi136 bash[76371]: audit 2024-09-12T23:43:29.261804+0000 mon.a (mon.0) 225 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-12T23:43:29.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:29 smithi136 bash[76371]: audit 2024-09-12T23:43:29.262572+0000 mon.a (mon.0) 226 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:43:29.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:29 smithi136 bash[76371]: cluster 2024-09-12T23:43:29.309824+0000 mon.a (mon.0) 227 : cluster [DBG] osdmap e356: 8 total, 8 up, 8 in 2024-09-12T23:43:29.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:29 smithi181 bash[67587]: cluster 2024-09-12T23:43:28.673427+0000 mgr.y (mgr.24661) 68 : cluster [DBG] pgmap v107: 121 pgs: 121 active+clean; 258 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:43:29.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:29 smithi181 bash[67587]: audit 2024-09-12T23:43:29.250300+0000 mon.a (mon.0) 222 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:29.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:29 smithi181 bash[67587]: audit 2024-09-12T23:43:29.259247+0000 mon.a (mon.0) 223 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:29.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:29 smithi181 bash[67587]: audit 2024-09-12T23:43:29.260892+0000 mon.a (mon.0) 224 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-12T23:43:29.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:29 smithi181 bash[67587]: audit 2024-09-12T23:43:29.261804+0000 mon.a (mon.0) 225 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-12T23:43:29.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:29 smithi181 bash[67587]: audit 2024-09-12T23:43:29.262572+0000 mon.a (mon.0) 226 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:43:29.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:29 smithi181 bash[67587]: cluster 2024-09-12T23:43:29.309824+0000 mon.a (mon.0) 227 : cluster [DBG] osdmap e356: 8 total, 8 up, 8 in 2024-09-12T23:43:30.310 INFO:tasks.rados.rados.0.smithi136.stdout:1508: done (0 left) 2024-09-12T23:43:30.310 INFO:tasks.rados.rados.0.smithi136.stdout:1509: snap_remove snap 137 2024-09-12T23:43:30.710 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:30 smithi136 bash[76371]: cephadm 2024-09-12T23:43:29.260565+0000 mgr.y (mgr.24661) 69 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2024-09-12T23:43:30.710 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:30 smithi136 bash[76371]: cephadm 2024-09-12T23:43:29.263523+0000 mgr.y (mgr.24661) 70 : cephadm [INF] Reconfiguring daemon mon.c on smithi136 2024-09-12T23:43:30.710 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:30 smithi136 bash[76371]: cluster 2024-09-12T23:43:30.313527+0000 mon.a (mon.0) 228 : cluster [DBG] osdmap e357: 8 total, 8 up, 8 in 2024-09-12T23:43:30.710 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:30 smithi136 bash[75164]: cephadm 2024-09-12T23:43:29.260565+0000 mgr.y (mgr.24661) 69 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2024-09-12T23:43:30.710 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:30 smithi136 bash[75164]: cephadm 2024-09-12T23:43:29.263523+0000 mgr.y (mgr.24661) 70 : cephadm [INF] Reconfiguring daemon mon.c on smithi136 2024-09-12T23:43:30.710 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:30 smithi136 bash[75164]: cluster 2024-09-12T23:43:30.313527+0000 mon.a (mon.0) 228 : cluster [DBG] osdmap e357: 8 total, 8 up, 8 in 2024-09-12T23:43:30.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:30 smithi181 bash[67587]: cephadm 2024-09-12T23:43:29.260565+0000 mgr.y (mgr.24661) 69 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2024-09-12T23:43:30.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:30 smithi181 bash[67587]: cephadm 2024-09-12T23:43:29.263523+0000 mgr.y (mgr.24661) 70 : cephadm [INF] Reconfiguring daemon mon.c on smithi136 2024-09-12T23:43:30.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:30 smithi181 bash[67587]: cluster 2024-09-12T23:43:30.313527+0000 mon.a (mon.0) 228 : cluster [DBG] osdmap e357: 8 total, 8 up, 8 in 2024-09-12T23:43:31.313 INFO:tasks.rados.rados.0.smithi136.stdout:1509: done (0 left) 2024-09-12T23:43:31.314 INFO:tasks.rados.rados.0.smithi136.stdout:1510: snap_remove snap 122 2024-09-12T23:43:31.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:31 smithi136 bash[75164]: cluster 2024-09-12T23:43:30.674089+0000 mgr.y (mgr.24661) 71 : cluster [DBG] pgmap v110: 121 pgs: 121 active+clean; 267 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 1.2 KiB/s rd, 2.1 MiB/s wr, 16 op/s 2024-09-12T23:43:31.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:31 smithi136 bash[75164]: cluster 2024-09-12T23:43:31.317315+0000 mon.a (mon.0) 229 : cluster [DBG] osdmap e358: 8 total, 8 up, 8 in 2024-09-12T23:43:31.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:31 smithi136 bash[76371]: cluster 2024-09-12T23:43:30.674089+0000 mgr.y (mgr.24661) 71 : cluster [DBG] pgmap v110: 121 pgs: 121 active+clean; 267 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 1.2 KiB/s rd, 2.1 MiB/s wr, 16 op/s 2024-09-12T23:43:31.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:31 smithi136 bash[76371]: cluster 2024-09-12T23:43:31.317315+0000 mon.a (mon.0) 229 : cluster [DBG] osdmap e358: 8 total, 8 up, 8 in 2024-09-12T23:43:31.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:31 smithi181 bash[67587]: cluster 2024-09-12T23:43:30.674089+0000 mgr.y (mgr.24661) 71 : cluster [DBG] pgmap v110: 121 pgs: 121 active+clean; 267 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 1.2 KiB/s rd, 2.1 MiB/s wr, 16 op/s 2024-09-12T23:43:31.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:31 smithi181 bash[67587]: cluster 2024-09-12T23:43:31.317315+0000 mon.a (mon.0) 229 : cluster [DBG] osdmap e358: 8 total, 8 up, 8 in 2024-09-12T23:43:32.314 INFO:tasks.rados.rados.0.smithi136.stdout:1510: done (0 left) 2024-09-12T23:43:32.314 INFO:tasks.rados.rados.0.smithi136.stdout:1511: delete oid 34 current snap is 146 2024-09-12T23:43:32.344 INFO:tasks.rados.rados.0.smithi136.stdout:1511: done (0 left) 2024-09-12T23:43:32.344 INFO:tasks.rados.rados.0.smithi136.stdout:1512: read oid 33 snap -1 2024-09-12T23:43:32.344 INFO:tasks.rados.rados.0.smithi136.stdout:1512: expect (ObjNum 473 snap 141 seq_num 473) 2024-09-12T23:43:32.344 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 48 current snap is 146 2024-09-12T23:43:32.344 INFO:tasks.rados.rados.0.smithi136.stdout:1513: seq_num 495 ranges {843776=696320,1540096=401408} 2024-09-12T23:43:32.357 INFO:tasks.rados.rados.0.smithi136.stdout:1513: writing smithi13667081-48 from 843776 to 1540096 tid 1 2024-09-12T23:43:32.362 INFO:tasks.rados.rados.0.smithi136.stdout:1513: writing smithi13667081-48 from 1540096 to 1941504 tid 2 2024-09-12T23:43:32.362 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 5 current snap is 146 2024-09-12T23:43:32.362 INFO:tasks.rados.rados.0.smithi136.stdout:1514: seq_num 496 ranges {5332992=712704,6045696=401408} 2024-09-12T23:43:32.404 INFO:tasks.rados.rados.0.smithi136.stdout:1514: writing smithi13667081-5 from 5332992 to 6045696 tid 1 2024-09-12T23:43:32.406 INFO:tasks.rados.rados.0.smithi136.stdout:1514: writing smithi13667081-5 from 6045696 to 6447104 tid 2 2024-09-12T23:43:32.407 INFO:tasks.rados.rados.0.smithi136.stdout:1515: read oid 42 snap -1 2024-09-12T23:43:32.407 INFO:tasks.rados.rados.0.smithi136.stdout:1515: expect deleted 2024-09-12T23:43:32.407 INFO:tasks.rados.rados.0.smithi136.stdout:1516: delete oid 15 current snap is 146 2024-09-12T23:43:32.407 INFO:tasks.rados.rados.0.smithi136.stdout:1513: finishing write tid 1 to smithi13667081-48 2024-09-12T23:43:32.407 INFO:tasks.rados.rados.0.smithi136.stdout:1513: finishing write tid 2 to smithi13667081-48 2024-09-12T23:43:32.407 INFO:tasks.rados.rados.0.smithi136.stdout:1512: expect (ObjNum 473 snap 141 seq_num 473) 2024-09-12T23:43:32.577 INFO:tasks.rados.rados.0.smithi136.stdout:1513: finishing write tid 3 to smithi13667081-48 2024-09-12T23:43:32.577 INFO:tasks.rados.rados.0.smithi136.stdout:1513: finishing write tid 4 to smithi13667081-48 2024-09-12T23:43:32.577 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 48 v 912 (ObjNum 495 snap 146 seq_num 495) dirty exists 2024-09-12T23:43:32.578 INFO:tasks.rados.rados.0.smithi136.stdout:1513: left oid 48 (ObjNum 495 snap 146 seq_num 495) 2024-09-12T23:43:32.578 INFO:tasks.rados.rados.0.smithi136.stdout:1512: done (4 left) 2024-09-12T23:43:32.578 INFO:tasks.rados.rados.0.smithi136.stdout:1513: done (3 left) 2024-09-12T23:43:32.578 INFO:tasks.rados.rados.0.smithi136.stdout:1516: done (2 left) 2024-09-12T23:43:32.578 INFO:tasks.rados.rados.0.smithi136.stdout:1517: rollback oid 47 current snap is 146 2024-09-12T23:43:32.578 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 47 to 139 2024-09-12T23:43:32.578 INFO:tasks.rados.rados.0.smithi136.stdout:1514: finishing write tid 1 to smithi13667081-5 2024-09-12T23:43:32.578 INFO:tasks.rados.rados.0.smithi136.stdout:1514: finishing write tid 2 to smithi13667081-5 2024-09-12T23:43:32.578 INFO:tasks.rados.rados.0.smithi136.stdout:1514: finishing write tid 3 to smithi13667081-5 2024-09-12T23:43:32.578 INFO:tasks.rados.rados.0.smithi136.stdout:1514: finishing write tid 4 to smithi13667081-5 2024-09-12T23:43:32.578 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 5 v 1113 (ObjNum 496 snap 146 seq_num 496) dirty exists 2024-09-12T23:43:32.578 INFO:tasks.rados.rados.0.smithi136.stdout:1514: left oid 5 (ObjNum 496 snap 146 seq_num 496) 2024-09-12T23:43:32.579 INFO:tasks.rados.rados.0.smithi136.stdout:1514: done (2 left) 2024-09-12T23:43:32.579 INFO:tasks.rados.rados.0.smithi136.stdout:1515: done (1 left) 2024-09-12T23:43:32.579 INFO:tasks.rados.rados.0.smithi136.stdout:1518: read oid 30 snap 140 2024-09-12T23:43:32.579 INFO:tasks.rados.rados.0.smithi136.stdout:1518: expect (ObjNum 427 snap 127 seq_num 427) 2024-09-12T23:43:32.579 INFO:tasks.rados.rados.0.smithi136.stdout:1519: rmattr oid 24 current snap is 146 2024-09-12T23:43:32.579 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 8 current snap is 146 2024-09-12T23:43:32.579 INFO:tasks.rados.rados.0.smithi136.stdout:1520: seq_num 498 ranges {2400256=745472,3145728=401408} 2024-09-12T23:43:32.583 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:43:32 smithi181 bash[60471]: ts=2024-09-12T23:43:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:43:32.606 INFO:tasks.rados.rados.0.smithi136.stdout:1520: writing smithi13667081-8 from 2400256 to 3145728 tid 1 2024-09-12T23:43:32.609 INFO:tasks.rados.rados.0.smithi136.stdout:1520: writing smithi13667081-8 from 3145728 to 3547136 tid 2 2024-09-12T23:43:32.609 INFO:tasks.rados.rados.0.smithi136.stdout:1521: snap_remove snap 127 2024-09-12T23:43:32.764 INFO:teuthology.orchestra.run.smithi136.stdout:true 2024-09-12T23:43:32.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:32 smithi181 bash[67587]: audit 2024-09-12T23:43:31.587198+0000 mon.a (mon.0) 230 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:32.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:32 smithi181 bash[67587]: audit 2024-09-12T23:43:31.594043+0000 mon.a (mon.0) 231 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:32.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:32 smithi181 bash[67587]: cephadm 2024-09-12T23:43:31.595547+0000 mgr.y (mgr.24661) 72 : cephadm [INF] Reconfiguring osd.0 (monmap changed)... 2024-09-12T23:43:32.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:32 smithi181 bash[67587]: audit 2024-09-12T23:43:31.595869+0000 mon.a (mon.0) 232 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-12T23:43:32.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:32 smithi181 bash[67587]: audit 2024-09-12T23:43:31.596936+0000 mon.a (mon.0) 233 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:43:32.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:32 smithi181 bash[67587]: cephadm 2024-09-12T23:43:31.600157+0000 mgr.y (mgr.24661) 73 : cephadm [INF] Reconfiguring daemon osd.0 on smithi136 2024-09-12T23:43:32.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:32 smithi181 bash[67587]: cluster 2024-09-12T23:43:32.317781+0000 mon.a (mon.0) 234 : cluster [DBG] osdmap e359: 8 total, 8 up, 8 in 2024-09-12T23:43:33.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:32 smithi136 bash[75164]: audit 2024-09-12T23:43:31.587198+0000 mon.a (mon.0) 230 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:33.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:32 smithi136 bash[75164]: audit 2024-09-12T23:43:31.594043+0000 mon.a (mon.0) 231 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:33.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:32 smithi136 bash[75164]: cephadm 2024-09-12T23:43:31.595547+0000 mgr.y (mgr.24661) 72 : cephadm [INF] Reconfiguring osd.0 (monmap changed)... 2024-09-12T23:43:33.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:32 smithi136 bash[75164]: audit 2024-09-12T23:43:31.595869+0000 mon.a (mon.0) 232 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-12T23:43:33.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:32 smithi136 bash[75164]: audit 2024-09-12T23:43:31.596936+0000 mon.a (mon.0) 233 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:43:33.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:32 smithi136 bash[75164]: cephadm 2024-09-12T23:43:31.600157+0000 mgr.y (mgr.24661) 73 : cephadm [INF] Reconfiguring daemon osd.0 on smithi136 2024-09-12T23:43:33.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:32 smithi136 bash[75164]: cluster 2024-09-12T23:43:32.317781+0000 mon.a (mon.0) 234 : cluster [DBG] osdmap e359: 8 total, 8 up, 8 in 2024-09-12T23:43:33.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:32 smithi136 bash[76371]: audit 2024-09-12T23:43:31.587198+0000 mon.a (mon.0) 230 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:33.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:32 smithi136 bash[76371]: audit 2024-09-12T23:43:31.594043+0000 mon.a (mon.0) 231 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:33.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:32 smithi136 bash[76371]: cephadm 2024-09-12T23:43:31.595547+0000 mgr.y (mgr.24661) 72 : cephadm [INF] Reconfiguring osd.0 (monmap changed)... 2024-09-12T23:43:33.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:32 smithi136 bash[76371]: audit 2024-09-12T23:43:31.595869+0000 mon.a (mon.0) 232 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-12T23:43:33.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:32 smithi136 bash[76371]: audit 2024-09-12T23:43:31.596936+0000 mon.a (mon.0) 233 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:43:33.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:32 smithi136 bash[76371]: cephadm 2024-09-12T23:43:31.600157+0000 mgr.y (mgr.24661) 73 : cephadm [INF] Reconfiguring daemon osd.0 on smithi136 2024-09-12T23:43:33.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:32 smithi136 bash[76371]: cluster 2024-09-12T23:43:32.317781+0000 mon.a (mon.0) 234 : cluster [DBG] osdmap e359: 8 total, 8 up, 8 in 2024-09-12T23:43:33.059 INFO:teuthology.orchestra.run.smithi136.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-12T23:43:33.059 INFO:teuthology.orchestra.run.smithi136.stdout:alertmanager.a smithi136 *:9093,9094 running (3m) 48s ago 9m 13.7M - 0.25.0 c8568f914cd2 b5197f020eeb 2024-09-12T23:43:33.059 INFO:teuthology.orchestra.run.smithi136.stdout:grafana.a smithi181 *:3000 running (2m) 19s ago 8m 75.3M - 9.4.7 954c08fa6188 256c181be984 2024-09-12T23:43:33.059 INFO:teuthology.orchestra.run.smithi136.stdout:mds.a smithi136 running (10m) 48s ago 10m 14.5M - 17.2.7-1627-gbbdadfa3 214985049bdb 713b9795ad19 2024-09-12T23:43:33.059 INFO:teuthology.orchestra.run.smithi136.stdout:mds.b smithi181 running (10m) 19s ago 10m 17.8M - 17.2.7-1627-gbbdadfa3 214985049bdb ad13179f0868 2024-09-12T23:43:33.060 INFO:teuthology.orchestra.run.smithi136.stdout:mgr.x smithi181 *:8443,9283,8765 running (116s) 19s ago 17m 425M - 18.2.4-837-g7a853b17 f21ba1eea74e cc664dc337f7 2024-09-12T23:43:33.060 INFO:teuthology.orchestra.run.smithi136.stdout:mgr.y smithi136 *:8443,9283,8765 running (2m) 48s ago 19m 482M - 18.2.4-837-g7a853b17 f21ba1eea74e 272f10b42fc3 2024-09-12T23:43:33.060 INFO:teuthology.orchestra.run.smithi136.stdout:mon.a smithi136 running (92s) 48s ago 20m 39.3M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e ad8d21d58ad9 2024-09-12T23:43:33.060 INFO:teuthology.orchestra.run.smithi136.stdout:mon.b smithi181 running (32s) 19s ago 18m 19.5M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e a70ec8315934 2024-09-12T23:43:33.060 INFO:teuthology.orchestra.run.smithi136.stdout:mon.c smithi136 running (60s) 48s ago 17m 19.2M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e 44d1be055a5e 2024-09-12T23:43:33.060 INFO:teuthology.orchestra.run.smithi136.stdout:node-exporter.a smithi136 *:9100 running (3m) 48s ago 9m 7996k - 1.5.0 0da6a335fe13 ccfc37c96620 2024-09-12T23:43:33.060 INFO:teuthology.orchestra.run.smithi136.stdout:node-exporter.b smithi181 *:9100 running (2m) 19s ago 9m 7847k - 1.5.0 0da6a335fe13 80bf0e383230 2024-09-12T23:43:33.060 INFO:teuthology.orchestra.run.smithi136.stdout:osd.0 smithi136 running (16m) 48s ago 16m 187M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb cbc0b7df0115 2024-09-12T23:43:33.060 INFO:teuthology.orchestra.run.smithi136.stdout:osd.1 smithi136 running (15m) 48s ago 15m 197M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb c4daee7da836 2024-09-12T23:43:33.060 INFO:teuthology.orchestra.run.smithi136.stdout:osd.2 smithi136 running (15m) 48s ago 15m 210M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 4ab8fd79cdb4 2024-09-12T23:43:33.060 INFO:teuthology.orchestra.run.smithi136.stdout:osd.3 smithi136 running (14m) 48s ago 14m 152M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 9ceb14a27521 2024-09-12T23:43:33.060 INFO:teuthology.orchestra.run.smithi136.stdout:osd.4 smithi181 running (13m) 19s ago 13m 230M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb a35cdca79055 2024-09-12T23:43:33.061 INFO:teuthology.orchestra.run.smithi136.stdout:osd.5 smithi181 running (12m) 19s ago 12m 192M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 8f2e8d494fef 2024-09-12T23:43:33.061 INFO:teuthology.orchestra.run.smithi136.stdout:osd.6 smithi181 running (11m) 19s ago 11m 106M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 9c6bec406408 2024-09-12T23:43:33.061 INFO:teuthology.orchestra.run.smithi136.stdout:osd.7 smithi181 running (10m) 19s ago 10m 174M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb bc0632ae2145 2024-09-12T23:43:33.061 INFO:teuthology.orchestra.run.smithi136.stdout:prometheus.a smithi181 *:9095 running (2m) 19s ago 9m 51.7M - 2.43.0 a07b618ecd1d 3853d003a5cc 2024-09-12T23:43:33.406 INFO:teuthology.orchestra.run.smithi136.stdout:{ 2024-09-12T23:43:33.406 INFO:teuthology.orchestra.run.smithi136.stdout: "mon": { 2024-09-12T23:43:33.406 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 3 2024-09-12T23:43:33.406 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:43:33.406 INFO:teuthology.orchestra.run.smithi136.stdout: "mgr": { 2024-09-12T23:43:33.406 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 2 2024-09-12T23:43:33.407 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:43:33.407 INFO:teuthology.orchestra.run.smithi136.stdout: "osd": { 2024-09-12T23:43:33.407 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 8 2024-09-12T23:43:33.407 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:43:33.407 INFO:teuthology.orchestra.run.smithi136.stdout: "mds": { 2024-09-12T23:43:33.407 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-12T23:43:33.407 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:43:33.407 INFO:teuthology.orchestra.run.smithi136.stdout: "overall": { 2024-09-12T23:43:33.407 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 10, 2024-09-12T23:43:33.407 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 5 2024-09-12T23:43:33.407 INFO:teuthology.orchestra.run.smithi136.stdout: } 2024-09-12T23:43:33.408 INFO:teuthology.orchestra.run.smithi136.stdout:} 2024-09-12T23:43:33.620 INFO:tasks.rados.rados.0.smithi136.stdout:1517: finishing rollback tid 0 to smithi13667081-47 2024-09-12T23:43:33.620 INFO:tasks.rados.rados.0.smithi136.stdout:1517: finishing rollback tid 1 to smithi13667081-47 2024-09-12T23:43:33.620 INFO:tasks.rados.rados.0.smithi136.stdout:1517: finishing rollback tid 2 to smithi13667081-47 2024-09-12T23:43:33.620 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 47 v 615 (ObjNum 460 snap 136 seq_num 460) dirty exists 2024-09-12T23:43:33.620 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 24 v 614 (ObjNum 399 snap 116 seq_num 399) dirty exists 2024-09-12T23:43:33.620 INFO:tasks.rados.rados.0.smithi136.stdout:1518: expect (ObjNum 427 snap 127 seq_num 427) 2024-09-12T23:43:33.746 INFO:tasks.rados.rados.0.smithi136.stdout:1520: finishing write tid 1 to smithi13667081-8 2024-09-12T23:43:33.746 INFO:tasks.rados.rados.0.smithi136.stdout:1520: finishing write tid 2 to smithi13667081-8 2024-09-12T23:43:33.746 INFO:tasks.rados.rados.0.smithi136.stdout:1520: finishing write tid 3 to smithi13667081-8 2024-09-12T23:43:33.747 INFO:tasks.rados.rados.0.smithi136.stdout:1517: done (4 left) 2024-09-12T23:43:33.747 INFO:tasks.rados.rados.0.smithi136.stdout:1518: done (3 left) 2024-09-12T23:43:33.747 INFO:tasks.rados.rados.0.smithi136.stdout:1519: done (2 left) 2024-09-12T23:43:33.747 INFO:tasks.rados.rados.0.smithi136.stdout:1521: done (1 left) 2024-09-12T23:43:33.747 INFO:tasks.rados.rados.0.smithi136.stdout:1522: read oid 2 snap -1 2024-09-12T23:43:33.747 INFO:tasks.rados.rados.0.smithi136.stdout:1522: expect (ObjNum 416 snap 120 seq_num 416) 2024-09-12T23:43:33.747 INFO:tasks.rados.rados.0.smithi136.stdout:1523: snap_create 2024-09-12T23:43:33.747 INFO:tasks.rados.rados.0.smithi136.stdout:1520: finishing write tid 4 to smithi13667081-8 2024-09-12T23:43:33.747 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 8 v 693 (ObjNum 498 snap 146 seq_num 498) dirty exists 2024-09-12T23:43:33.747 INFO:tasks.rados.rados.0.smithi136.stdout:1520: left oid 8 (ObjNum 498 snap 146 seq_num 498) 2024-09-12T23:43:33.755 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:33 smithi136 bash[76371]: cluster 2024-09-12T23:43:32.674566+0000 mgr.y (mgr.24661) 74 : cluster [DBG] pgmap v113: 121 pgs: 121 active+clean; 267 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 1.2 KiB/s rd, 2.1 MiB/s wr, 16 op/s 2024-09-12T23:43:33.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:33 smithi136 bash[76371]: audit 2024-09-12T23:43:32.756139+0000 mgr.y (mgr.24661) 75 : audit [DBG] from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:43:33.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:33 smithi136 bash[76371]: audit 2024-09-12T23:43:33.409993+0000 mon.a (mon.0) 235 : audit [DBG] from='client.? 172.21.15.136:0/321022530' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:43:33.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:33 smithi136 bash[75164]: cluster 2024-09-12T23:43:32.674566+0000 mgr.y (mgr.24661) 74 : cluster [DBG] pgmap v113: 121 pgs: 121 active+clean; 267 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 1.2 KiB/s rd, 2.1 MiB/s wr, 16 op/s 2024-09-12T23:43:33.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:33 smithi136 bash[75164]: audit 2024-09-12T23:43:32.756139+0000 mgr.y (mgr.24661) 75 : audit [DBG] from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:43:33.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:33 smithi136 bash[75164]: audit 2024-09-12T23:43:33.409993+0000 mon.a (mon.0) 235 : audit [DBG] from='client.? 172.21.15.136:0/321022530' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:43:33.780 INFO:tasks.rados.rados.0.smithi136.stdout:1522: expect (ObjNum 416 snap 120 seq_num 416) 2024-09-12T23:43:33.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:33 smithi181 bash[67587]: cluster 2024-09-12T23:43:32.674566+0000 mgr.y (mgr.24661) 74 : cluster [DBG] pgmap v113: 121 pgs: 121 active+clean; 267 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 1.2 KiB/s rd, 2.1 MiB/s wr, 16 op/s 2024-09-12T23:43:33.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:33 smithi181 bash[67587]: audit 2024-09-12T23:43:32.756139+0000 mgr.y (mgr.24661) 75 : audit [DBG] from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:43:33.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:33 smithi181 bash[67587]: audit 2024-09-12T23:43:33.409993+0000 mon.a (mon.0) 235 : audit [DBG] from='client.? 172.21.15.136:0/321022530' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:43:33.844 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:43:33 smithi181 bash[60471]: ts=2024-09-12T23:43:33.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi136\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:43:34.600 INFO:tasks.rados.rados.0.smithi136.stdout:1520: done (2 left) 2024-09-12T23:43:34.600 INFO:tasks.rados.rados.0.smithi136.stdout:1522: done (1 left) 2024-09-12T23:43:34.600 INFO:tasks.rados.rados.0.smithi136.stdout:1523: done (0 left) 2024-09-12T23:43:34.600 INFO:tasks.rados.rados.0.smithi136.stdout:1524: rollback oid 42 current snap is 147 2024-09-12T23:43:34.600 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 42 to 144 2024-09-12T23:43:34.601 INFO:tasks.rados.rados.0.smithi136.stdout:1525: read oid 22 snap -1 2024-09-12T23:43:34.601 INFO:tasks.rados.rados.0.smithi136.stdout:1525: expect (ObjNum 427 snap 127 seq_num 427) 2024-09-12T23:43:34.601 INFO:tasks.rados.rados.0.smithi136.stdout:1526: snap_remove snap 139 2024-09-12T23:43:34.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:34 smithi181 bash[67587]: audit 2024-09-12T23:43:33.050442+0000 mgr.y (mgr.24661) 76 : audit [DBG] from='client.44120 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:43:34.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:34 smithi181 bash[67587]: cluster 2024-09-12T23:43:33.612711+0000 mon.a (mon.0) 236 : cluster [DBG] osdmap e360: 8 total, 8 up, 8 in 2024-09-12T23:43:34.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:34 smithi181 bash[67587]: audit 2024-09-12T23:43:33.906328+0000 mon.a (mon.0) 237 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:34.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:34 smithi181 bash[67587]: audit 2024-09-12T23:43:33.912294+0000 mon.a (mon.0) 238 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:34.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:34 smithi181 bash[67587]: audit 2024-09-12T23:43:33.913626+0000 mon.a (mon.0) 239 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-12T23:43:34.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:34 smithi181 bash[67587]: audit 2024-09-12T23:43:33.914496+0000 mon.a (mon.0) 240 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:43:34.846 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:43:34 smithi181 bash[60471]: ts=2024-09-12T23:43:34.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:43:34.846 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:43:34 smithi181 bash[60471]: ts=2024-09-12T23:43:34.456Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:43:35.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:34 smithi136 bash[76371]: audit 2024-09-12T23:43:33.050442+0000 mgr.y (mgr.24661) 76 : audit [DBG] from='client.44120 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:43:35.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:34 smithi136 bash[76371]: cluster 2024-09-12T23:43:33.612711+0000 mon.a (mon.0) 236 : cluster [DBG] osdmap e360: 8 total, 8 up, 8 in 2024-09-12T23:43:35.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:34 smithi136 bash[76371]: audit 2024-09-12T23:43:33.906328+0000 mon.a (mon.0) 237 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:35.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:34 smithi136 bash[76371]: audit 2024-09-12T23:43:33.912294+0000 mon.a (mon.0) 238 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:35.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:34 smithi136 bash[76371]: audit 2024-09-12T23:43:33.913626+0000 mon.a (mon.0) 239 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-12T23:43:35.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:34 smithi136 bash[76371]: audit 2024-09-12T23:43:33.914496+0000 mon.a (mon.0) 240 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:43:35.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:34 smithi136 bash[75164]: audit 2024-09-12T23:43:33.050442+0000 mgr.y (mgr.24661) 76 : audit [DBG] from='client.44120 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:43:35.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:34 smithi136 bash[75164]: cluster 2024-09-12T23:43:33.612711+0000 mon.a (mon.0) 236 : cluster [DBG] osdmap e360: 8 total, 8 up, 8 in 2024-09-12T23:43:35.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:34 smithi136 bash[75164]: audit 2024-09-12T23:43:33.906328+0000 mon.a (mon.0) 237 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:35.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:34 smithi136 bash[75164]: audit 2024-09-12T23:43:33.912294+0000 mon.a (mon.0) 238 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:35.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:34 smithi136 bash[75164]: audit 2024-09-12T23:43:33.913626+0000 mon.a (mon.0) 239 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-12T23:43:35.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:34 smithi136 bash[75164]: audit 2024-09-12T23:43:33.914496+0000 mon.a (mon.0) 240 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:43:35.599 INFO:tasks.rados.rados.0.smithi136.stdout:1526: done (2 left) 2024-09-12T23:43:35.599 INFO:tasks.rados.rados.0.smithi136.stdout:1527: setattr oid 33 current snap is 147 2024-09-12T23:43:35.600 INFO:tasks.rados.rados.0.smithi136.stdout:1524: finishing rollback tid 1 to smithi13667081-42 2024-09-12T23:43:35.600 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 42 v 0 (ObjNum 65 snap 0 seq_num 4243056592) dirty dne 2024-09-12T23:43:35.600 INFO:tasks.rados.rados.0.smithi136.stdout:1525: expect (ObjNum 427 snap 127 seq_num 427) 2024-09-12T23:43:35.723 INFO:tasks.rados.rados.0.smithi136.stdout:1524: done (2 left) 2024-09-12T23:43:35.723 INFO:tasks.rados.rados.0.smithi136.stdout:1525: done (1 left) 2024-09-12T23:43:35.723 INFO:tasks.rados.rados.0.smithi136.stdout:1528: copy_from oid 12 from oid 18 current snap is 147 2024-09-12T23:43:35.723 INFO:tasks.rados.rados.0.smithi136.stdout:1529: read oid 24 snap -1 2024-09-12T23:43:35.723 INFO:tasks.rados.rados.0.smithi136.stdout:1529: expect (ObjNum 399 snap 116 seq_num 399) 2024-09-12T23:43:35.723 INFO:tasks.rados.rados.0.smithi136.stdout:1530: read oid 8 snap -1 2024-09-12T23:43:35.723 INFO:tasks.rados.rados.0.smithi136.stdout:1530: expect (ObjNum 498 snap 146 seq_num 498) 2024-09-12T23:43:35.723 INFO:tasks.rados.rados.0.smithi136.stdout:1531: snap_create 2024-09-12T23:43:35.726 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 33 v 762 (ObjNum 473 snap 141 seq_num 473) dirty exists 2024-09-12T23:43:35.742 INFO:tasks.rados.rados.0.smithi136.stdout:1528: finishing copy_from racing read to smithi13667081-12 2024-09-12T23:43:35.747 INFO:tasks.rados.rados.0.smithi136.stdout:1528: finishing copy_from to smithi13667081-12 2024-09-12T23:43:35.747 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 12 v 879 (ObjNum 488 snap 143 seq_num 488) dirty exists 2024-09-12T23:43:35.750 INFO:tasks.rados.rados.0.smithi136.stdout:1530: expect (ObjNum 498 snap 146 seq_num 498) 2024-09-12T23:43:35.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:35 smithi136 bash[76371]: cephadm 2024-09-12T23:43:33.913304+0000 mgr.y (mgr.24661) 77 : cephadm [INF] Reconfiguring osd.1 (monmap changed)... 2024-09-12T23:43:35.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:35 smithi136 bash[76371]: cephadm 2024-09-12T23:43:33.916913+0000 mgr.y (mgr.24661) 78 : cephadm [INF] Reconfiguring daemon osd.1 on smithi136 2024-09-12T23:43:35.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:35 smithi136 bash[76371]: cluster 2024-09-12T23:43:34.603225+0000 mon.a (mon.0) 241 : cluster [DBG] osdmap e361: 8 total, 8 up, 8 in 2024-09-12T23:43:35.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:35 smithi136 bash[76371]: cluster 2024-09-12T23:43:34.675906+0000 mgr.y (mgr.24661) 79 : cluster [DBG] pgmap v116: 121 pgs: 121 active+clean; 252 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 820 KiB/s wr, 10 op/s 2024-09-12T23:43:35.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:35 smithi136 bash[76371]: cluster 2024-09-12T23:43:35.603531+0000 mon.a (mon.0) 242 : cluster [DBG] osdmap e362: 8 total, 8 up, 8 in 2024-09-12T23:43:35.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:35 smithi136 bash[75164]: cephadm 2024-09-12T23:43:33.913304+0000 mgr.y (mgr.24661) 77 : cephadm [INF] Reconfiguring osd.1 (monmap changed)... 2024-09-12T23:43:35.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:35 smithi136 bash[75164]: cephadm 2024-09-12T23:43:33.916913+0000 mgr.y (mgr.24661) 78 : cephadm [INF] Reconfiguring daemon osd.1 on smithi136 2024-09-12T23:43:35.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:35 smithi136 bash[75164]: cluster 2024-09-12T23:43:34.603225+0000 mon.a (mon.0) 241 : cluster [DBG] osdmap e361: 8 total, 8 up, 8 in 2024-09-12T23:43:35.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:35 smithi136 bash[75164]: cluster 2024-09-12T23:43:34.675906+0000 mgr.y (mgr.24661) 79 : cluster [DBG] pgmap v116: 121 pgs: 121 active+clean; 252 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 820 KiB/s wr, 10 op/s 2024-09-12T23:43:35.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:35 smithi136 bash[75164]: cluster 2024-09-12T23:43:35.603531+0000 mon.a (mon.0) 242 : cluster [DBG] osdmap e362: 8 total, 8 up, 8 in 2024-09-12T23:43:35.881 INFO:tasks.rados.rados.0.smithi136.stdout:1529: expect (ObjNum 399 snap 116 seq_num 399) 2024-09-12T23:43:36.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:35 smithi181 bash[67587]: cephadm 2024-09-12T23:43:33.913304+0000 mgr.y (mgr.24661) 77 : cephadm [INF] Reconfiguring osd.1 (monmap changed)... 2024-09-12T23:43:36.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:35 smithi181 bash[67587]: cephadm 2024-09-12T23:43:33.916913+0000 mgr.y (mgr.24661) 78 : cephadm [INF] Reconfiguring daemon osd.1 on smithi136 2024-09-12T23:43:36.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:35 smithi181 bash[67587]: cluster 2024-09-12T23:43:34.603225+0000 mon.a (mon.0) 241 : cluster [DBG] osdmap e361: 8 total, 8 up, 8 in 2024-09-12T23:43:36.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:35 smithi181 bash[67587]: cluster 2024-09-12T23:43:34.675906+0000 mgr.y (mgr.24661) 79 : cluster [DBG] pgmap v116: 121 pgs: 121 active+clean; 252 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 820 KiB/s wr, 10 op/s 2024-09-12T23:43:36.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:35 smithi181 bash[67587]: cluster 2024-09-12T23:43:35.603531+0000 mon.a (mon.0) 242 : cluster [DBG] osdmap e362: 8 total, 8 up, 8 in 2024-09-12T23:43:36.305 INFO:tasks.rados.rados.0.smithi136.stdout:1527: done (4 left) 2024-09-12T23:43:36.305 INFO:tasks.rados.rados.0.smithi136.stdout:1528: done (3 left) 2024-09-12T23:43:36.305 INFO:tasks.rados.rados.0.smithi136.stdout:1529: done (2 left) 2024-09-12T23:43:36.305 INFO:tasks.rados.rados.0.smithi136.stdout:1530: done (1 left) 2024-09-12T23:43:36.305 INFO:tasks.rados.rados.0.smithi136.stdout:1531: done (0 left) 2024-09-12T23:43:36.305 INFO:tasks.rados.rados.0.smithi136.stdout:1532: copy_from oid 45 from oid 50 current snap is 148 2024-09-12T23:43:36.306 INFO:tasks.rados.rados.0.smithi136.stdout:1533: copy_from oid 17 from oid 29 current snap is 148 2024-09-12T23:43:36.306 INFO:tasks.rados.rados.0.smithi136.stdout:1534: read oid 30 snap -1 2024-09-12T23:43:36.306 INFO:tasks.rados.rados.0.smithi136.stdout:1534: expect (ObjNum 427 snap 127 seq_num 427) 2024-09-12T23:43:36.306 INFO:tasks.rados.rados.0.smithi136.stdout:1535: read oid 41 snap -1 2024-09-12T23:43:36.306 INFO:tasks.rados.rados.0.smithi136.stdout:1535: expect (ObjNum 417 snap 120 seq_num 417) 2024-09-12T23:43:36.306 INFO:tasks.rados.rados.0.smithi136.stdout:1536: snap_remove snap 146 2024-09-12T23:43:37.309 INFO:tasks.rados.rados.0.smithi136.stdout:1536: done (4 left) 2024-09-12T23:43:37.309 INFO:tasks.rados.rados.0.smithi136.stdout:1537: copy_from oid 5 from oid 28 current snap is 148 2024-09-12T23:43:37.309 INFO:tasks.rados.rados.0.smithi136.stdout:1532: finishing copy_from to smithi13667081-45 2024-09-12T23:43:37.309 INFO:tasks.rados.rados.0.smithi136.stdout:1532: got expected ENOENT (src dne) 2024-09-12T23:43:37.309 INFO:tasks.rados.rados.0.smithi136.stdout:1532: finishing copy_from racing read to smithi13667081-45 2024-09-12T23:43:37.309 INFO:tasks.rados.rados.0.smithi136.stdout:1535: expect (ObjNum 417 snap 120 seq_num 417) 2024-09-12T23:43:37.437 INFO:tasks.rados.rados.0.smithi136.stdout:1534: expect (ObjNum 427 snap 127 seq_num 427) 2024-09-12T23:43:37.558 INFO:tasks.rados.rados.0.smithi136.stdout:1533: finishing copy_from racing read to smithi13667081-17 2024-09-12T23:43:37.558 INFO:tasks.rados.rados.0.smithi136.stdout:1533: finishing copy_from to smithi13667081-17 2024-09-12T23:43:37.558 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 17 v 698 (ObjNum 423 snap 125 seq_num 423) dirty exists 2024-09-12T23:43:37.558 INFO:tasks.rados.rados.0.smithi136.stdout:1537: finishing copy_from racing read to smithi13667081-5 2024-09-12T23:43:37.558 INFO:tasks.rados.rados.0.smithi136.stdout:1537: finishing copy_from to smithi13667081-5 2024-09-12T23:43:37.558 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 5 v 860 (ObjNum 421 snap 123 seq_num 421) dirty exists 2024-09-12T23:43:37.558 INFO:tasks.rados.rados.0.smithi136.stdout:1532: done (4 left) 2024-09-12T23:43:37.558 INFO:tasks.rados.rados.0.smithi136.stdout:1533: done (3 left) 2024-09-12T23:43:37.558 INFO:tasks.rados.rados.0.smithi136.stdout:1534: done (2 left) 2024-09-12T23:43:37.558 INFO:tasks.rados.rados.0.smithi136.stdout:1535: done (1 left) 2024-09-12T23:43:37.559 INFO:tasks.rados.rados.0.smithi136.stdout:1537: done (0 left) 2024-09-12T23:43:37.559 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 25 current snap is 148 2024-09-12T23:43:37.559 INFO:tasks.rados.rados.0.smithi136.stdout:1538: seq_num 499 ranges {3604480=761856,4366336=401408} 2024-09-12T23:43:37.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:37 smithi181 bash[67587]: cluster 2024-09-12T23:43:36.309036+0000 mon.a (mon.0) 243 : cluster [DBG] osdmap e363: 8 total, 8 up, 8 in 2024-09-12T23:43:37.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:37 smithi181 bash[67587]: audit 2024-09-12T23:43:36.383349+0000 mon.a (mon.0) 244 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:37.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:37 smithi181 bash[67587]: audit 2024-09-12T23:43:36.391748+0000 mon.a (mon.0) 245 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:37.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:37 smithi181 bash[67587]: cephadm 2024-09-12T23:43:36.392906+0000 mgr.y (mgr.24661) 80 : cephadm [INF] Reconfiguring osd.2 (monmap changed)... 2024-09-12T23:43:37.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:37 smithi181 bash[67587]: audit 2024-09-12T23:43:36.393309+0000 mon.a (mon.0) 246 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-12T23:43:37.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:37 smithi181 bash[67587]: audit 2024-09-12T23:43:36.394480+0000 mon.a (mon.0) 247 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:43:37.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:37 smithi181 bash[67587]: cephadm 2024-09-12T23:43:36.396737+0000 mgr.y (mgr.24661) 81 : cephadm [INF] Reconfiguring daemon osd.2 on smithi136 2024-09-12T23:43:37.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:37 smithi181 bash[67587]: cluster 2024-09-12T23:43:36.676549+0000 mgr.y (mgr.24661) 82 : cluster [DBG] pgmap v119: 121 pgs: 121 active+clean; 252 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 820 KiB/s wr, 10 op/s 2024-09-12T23:43:37.622 INFO:tasks.rados.rados.0.smithi136.stdout:1538: writing smithi13667081-25 from 3604480 to 4366336 tid 1 2024-09-12T23:43:37.627 INFO:tasks.rados.rados.0.smithi136.stdout:1538: writing smithi13667081-25 from 4366336 to 4767744 tid 2 2024-09-12T23:43:37.627 INFO:tasks.rados.rados.0.smithi136.stdout:1539: read oid 13 snap 140 2024-09-12T23:43:37.627 INFO:tasks.rados.rados.0.smithi136.stdout:1539: expect (ObjNum 262 snap 71 seq_num 262) 2024-09-12T23:43:37.627 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 7 current snap is 148 2024-09-12T23:43:37.627 INFO:tasks.rados.rados.0.smithi136.stdout:1540: seq_num 500 ranges {4530176=778240,5308416=401408} 2024-09-12T23:43:37.667 INFO:tasks.rados.rados.0.smithi136.stdout:1540: writing smithi13667081-7 from 4530176 to 5308416 tid 1 2024-09-12T23:43:37.670 INFO:tasks.rados.rados.0.smithi136.stdout:1540: writing smithi13667081-7 from 5308416 to 5709824 tid 2 2024-09-12T23:43:37.670 INFO:tasks.rados.rados.0.smithi136.stdout:1541: setattr oid 45 current snap is 148 2024-09-12T23:43:37.670 INFO:tasks.rados.rados.0.smithi136.stdout:1538: finishing write tid 1 to smithi13667081-25 2024-09-12T23:43:37.670 INFO:tasks.rados.rados.0.smithi136.stdout:1538: finishing write tid 2 to smithi13667081-25 2024-09-12T23:43:37.670 INFO:tasks.rados.rados.0.smithi136.stdout:1538: finishing write tid 3 to smithi13667081-25 2024-09-12T23:43:37.670 INFO:tasks.rados.rados.0.smithi136.stdout:1538: finishing write tid 4 to smithi13667081-25 2024-09-12T23:43:37.670 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 25 v 280 (ObjNum 499 snap 148 seq_num 499) dirty exists 2024-09-12T23:43:37.671 INFO:tasks.rados.rados.0.smithi136.stdout:1538: left oid 25 (ObjNum 499 snap 148 seq_num 499) 2024-09-12T23:43:37.671 INFO:tasks.rados.rados.0.smithi136.stdout:1539: expect (ObjNum 262 snap 71 seq_num 262) 2024-09-12T23:43:37.696 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:37 smithi136 bash[75164]: cluster 2024-09-12T23:43:36.309036+0000 mon.a (mon.0) 243 : cluster [DBG] osdmap e363: 8 total, 8 up, 8 in 2024-09-12T23:43:37.696 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:37 smithi136 bash[75164]: audit 2024-09-12T23:43:36.383349+0000 mon.a (mon.0) 244 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:37.696 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:37 smithi136 bash[75164]: audit 2024-09-12T23:43:36.391748+0000 mon.a (mon.0) 245 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:37.696 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:37 smithi136 bash[75164]: cephadm 2024-09-12T23:43:36.392906+0000 mgr.y (mgr.24661) 80 : cephadm [INF] Reconfiguring osd.2 (monmap changed)... 2024-09-12T23:43:37.696 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:37 smithi136 bash[75164]: audit 2024-09-12T23:43:36.393309+0000 mon.a (mon.0) 246 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-12T23:43:37.697 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:37 smithi136 bash[75164]: audit 2024-09-12T23:43:36.394480+0000 mon.a (mon.0) 247 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:43:37.697 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:37 smithi136 bash[75164]: cephadm 2024-09-12T23:43:36.396737+0000 mgr.y (mgr.24661) 81 : cephadm [INF] Reconfiguring daemon osd.2 on smithi136 2024-09-12T23:43:37.697 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:37 smithi136 bash[75164]: cluster 2024-09-12T23:43:36.676549+0000 mgr.y (mgr.24661) 82 : cluster [DBG] pgmap v119: 121 pgs: 121 active+clean; 252 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 820 KiB/s wr, 10 op/s 2024-09-12T23:43:37.697 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:37 smithi136 bash[76371]: cluster 2024-09-12T23:43:36.309036+0000 mon.a (mon.0) 243 : cluster [DBG] osdmap e363: 8 total, 8 up, 8 in 2024-09-12T23:43:37.697 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:37 smithi136 bash[76371]: audit 2024-09-12T23:43:36.383349+0000 mon.a (mon.0) 244 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:37.697 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:37 smithi136 bash[76371]: audit 2024-09-12T23:43:36.391748+0000 mon.a (mon.0) 245 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:37.697 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:37 smithi136 bash[76371]: cephadm 2024-09-12T23:43:36.392906+0000 mgr.y (mgr.24661) 80 : cephadm [INF] Reconfiguring osd.2 (monmap changed)... 2024-09-12T23:43:37.697 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:37 smithi136 bash[76371]: audit 2024-09-12T23:43:36.393309+0000 mon.a (mon.0) 246 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-12T23:43:37.697 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:37 smithi136 bash[76371]: audit 2024-09-12T23:43:36.394480+0000 mon.a (mon.0) 247 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:43:37.697 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:37 smithi136 bash[76371]: cephadm 2024-09-12T23:43:36.396737+0000 mgr.y (mgr.24661) 81 : cephadm [INF] Reconfiguring daemon osd.2 on smithi136 2024-09-12T23:43:37.697 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:37 smithi136 bash[76371]: cluster 2024-09-12T23:43:36.676549+0000 mgr.y (mgr.24661) 82 : cluster [DBG] pgmap v119: 121 pgs: 121 active+clean; 252 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 820 KiB/s wr, 10 op/s 2024-09-12T23:43:37.880 INFO:tasks.rados.rados.0.smithi136.stdout:1540: finishing write tid 1 to smithi13667081-7 2024-09-12T23:43:37.880 INFO:tasks.rados.rados.0.smithi136.stdout:1540: finishing write tid 2 to smithi13667081-7 2024-09-12T23:43:37.880 INFO:tasks.rados.rados.0.smithi136.stdout:1540: finishing write tid 3 to smithi13667081-7 2024-09-12T23:43:37.880 INFO:tasks.rados.rados.0.smithi136.stdout:1540: finishing write tid 4 to smithi13667081-7 2024-09-12T23:43:37.880 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 7 v 885 (ObjNum 500 snap 148 seq_num 500) dirty exists 2024-09-12T23:43:37.880 INFO:tasks.rados.rados.0.smithi136.stdout:1540: left oid 7 (ObjNum 500 snap 148 seq_num 500) 2024-09-12T23:43:37.880 INFO:tasks.rados.rados.0.smithi136.stdout:1538: done (3 left) 2024-09-12T23:43:37.880 INFO:tasks.rados.rados.0.smithi136.stdout:1539: done (2 left) 2024-09-12T23:43:37.880 INFO:tasks.rados.rados.0.smithi136.stdout:1540: done (1 left) 2024-09-12T23:43:37.880 INFO:tasks.rados.rados.0.smithi136.stdout:1542: read oid 36 snap -1 2024-09-12T23:43:37.880 INFO:tasks.rados.rados.0.smithi136.stdout:1542: expect deleted 2024-09-12T23:43:37.880 INFO:tasks.rados.rados.0.smithi136.stdout:1543: read oid 12 snap -1 2024-09-12T23:43:37.881 INFO:tasks.rados.rados.0.smithi136.stdout:1543: expect (ObjNum 488 snap 143 seq_num 488) 2024-09-12T23:43:37.881 INFO:tasks.rados.rados.0.smithi136.stdout:1544: snap_remove snap 144 2024-09-12T23:43:38.393 INFO:tasks.rados.rados.0.smithi136.stdout:1544: done (3 left) 2024-09-12T23:43:38.393 INFO:tasks.rados.rados.0.smithi136.stdout:1545: snap_create 2024-09-12T23:43:38.393 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 45 v 694 (ObjNum 351 snap 96 seq_num 351) dirty exists 2024-09-12T23:43:38.393 INFO:tasks.rados.rados.0.smithi136.stdout:1543: expect (ObjNum 488 snap 143 seq_num 488) 2024-09-12T23:43:38.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:38 smithi181 bash[67587]: cluster 2024-09-12T23:43:37.312006+0000 mon.a (mon.0) 248 : cluster [DBG] osdmap e364: 8 total, 8 up, 8 in 2024-09-12T23:43:38.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:38 smithi136 bash[76371]: cluster 2024-09-12T23:43:37.312006+0000 mon.a (mon.0) 248 : cluster [DBG] osdmap e364: 8 total, 8 up, 8 in 2024-09-12T23:43:38.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:38 smithi136 bash[75164]: cluster 2024-09-12T23:43:37.312006+0000 mon.a (mon.0) 248 : cluster [DBG] osdmap e364: 8 total, 8 up, 8 in 2024-09-12T23:43:39.094 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:43:38 smithi181 bash[60471]: ts=2024-09-12T23:43:38.662Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:43:39.407 INFO:tasks.rados.rados.0.smithi136.stdout:1541: done (3 left) 2024-09-12T23:43:39.407 INFO:tasks.rados.rados.0.smithi136.stdout:1542: done (2 left) 2024-09-12T23:43:39.407 INFO:tasks.rados.rados.0.smithi136.stdout:1543: done (1 left) 2024-09-12T23:43:39.407 INFO:tasks.rados.rados.0.smithi136.stdout:1545: done (0 left) 2024-09-12T23:43:39.407 INFO:tasks.rados.rados.0.smithi136.stdout:1546: copy_from oid 36 from oid 33 current snap is 149 2024-09-12T23:43:39.407 INFO:tasks.rados.rados.0.smithi136.stdout:1547: rollback oid 44 current snap is 149 2024-09-12T23:43:39.407 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 44 to 148 2024-09-12T23:43:39.407 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 8 current snap is 149 2024-09-12T23:43:39.407 INFO:tasks.rados.rados.0.smithi136.stdout:1548: seq_num 501 ranges {3547136=794624,4341760=401408} 2024-09-12T23:43:39.449 INFO:tasks.rados.rados.0.smithi136.stdout:1548: writing smithi13667081-8 from 3547136 to 4341760 tid 1 2024-09-12T23:43:39.453 INFO:tasks.rados.rados.0.smithi136.stdout:1548: writing smithi13667081-8 from 4341760 to 4743168 tid 2 2024-09-12T23:43:39.453 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 9 current snap is 149 2024-09-12T23:43:39.453 INFO:tasks.rados.rados.0.smithi136.stdout:1549: seq_num 502 ranges {5226496=409600,5636096=417792,6053888=385024} 2024-09-12T23:43:39.495 INFO:tasks.rados.rados.0.smithi136.stdout:1549: writing smithi13667081-9 from 5226496 to 5636096 tid 1 2024-09-12T23:43:39.497 INFO:tasks.rados.rados.0.smithi136.stdout:1549: writing smithi13667081-9 from 5636096 to 6053888 tid 2 2024-09-12T23:43:39.500 INFO:tasks.rados.rados.0.smithi136.stdout:1549: writing smithi13667081-9 from 6053888 to 6438912 tid 3 2024-09-12T23:43:39.500 INFO:tasks.rados.rados.0.smithi136.stdout:1550: setattr oid 29 current snap is 149 2024-09-12T23:43:39.500 INFO:tasks.rados.rados.0.smithi136.stdout:1547: finishing rollback tid 1 to smithi13667081-44 2024-09-12T23:43:39.501 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 44 v 856 (ObjNum 65 snap 0 seq_num 1699701866) dirty dne 2024-09-12T23:43:39.501 INFO:tasks.rados.rados.0.smithi136.stdout:1546: finishing copy_from racing read to smithi13667081-36 2024-09-12T23:43:39.501 INFO:tasks.rados.rados.0.smithi136.stdout:1546: finishing copy_from to smithi13667081-36 2024-09-12T23:43:39.501 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 36 v 762 (ObjNum 473 snap 141 seq_num 473) dirty exists 2024-09-12T23:43:39.501 INFO:tasks.rados.rados.0.smithi136.stdout:1548: finishing write tid 1 to smithi13667081-8 2024-09-12T23:43:39.501 INFO:tasks.rados.rados.0.smithi136.stdout:1548: finishing write tid 2 to smithi13667081-8 2024-09-12T23:43:39.501 INFO:tasks.rados.rados.0.smithi136.stdout:1548: finishing write tid 3 to smithi13667081-8 2024-09-12T23:43:39.501 INFO:tasks.rados.rados.0.smithi136.stdout:1548: finishing write tid 4 to smithi13667081-8 2024-09-12T23:43:39.502 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 8 v 765 (ObjNum 501 snap 149 seq_num 501) dirty exists 2024-09-12T23:43:39.502 INFO:tasks.rados.rados.0.smithi136.stdout:1548: left oid 8 (ObjNum 501 snap 149 seq_num 501) 2024-09-12T23:43:39.502 INFO:tasks.rados.rados.0.smithi136.stdout:1546: done (4 left) 2024-09-12T23:43:39.502 INFO:tasks.rados.rados.0.smithi136.stdout:1547: done (3 left) 2024-09-12T23:43:39.502 INFO:tasks.rados.rados.0.smithi136.stdout:1548: done (2 left) 2024-09-12T23:43:39.502 INFO:tasks.rados.rados.0.smithi136.stdout:1551: snap_remove snap 138 2024-09-12T23:43:39.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:43:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:43:39] "GET /metrics HTTP/1.1" 200 36186 "" "Prometheus/2.43.0" 2024-09-12T23:43:39.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:39 smithi136 bash[75164]: cluster 2024-09-12T23:43:38.395772+0000 mon.a (mon.0) 249 : cluster [DBG] osdmap e365: 8 total, 8 up, 8 in 2024-09-12T23:43:39.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:39 smithi136 bash[75164]: audit 2024-09-12T23:43:38.669837+0000 mon.a (mon.0) 250 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:39.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:39 smithi136 bash[75164]: cluster 2024-09-12T23:43:38.677169+0000 mgr.y (mgr.24661) 83 : cluster [DBG] pgmap v122: 121 pgs: 121 active+clean; 252 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:43:39.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:39 smithi136 bash[75164]: audit 2024-09-12T23:43:38.679729+0000 mon.a (mon.0) 251 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:39.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:39 smithi136 bash[75164]: cephadm 2024-09-12T23:43:38.681252+0000 mgr.y (mgr.24661) 84 : cephadm [INF] Reconfiguring osd.3 (monmap changed)... 2024-09-12T23:43:39.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:39 smithi136 bash[75164]: audit 2024-09-12T23:43:38.681708+0000 mon.a (mon.0) 252 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-12T23:43:39.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:39 smithi136 bash[75164]: audit 2024-09-12T23:43:38.683021+0000 mon.a (mon.0) 253 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:43:39.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:39 smithi136 bash[75164]: cephadm 2024-09-12T23:43:38.685985+0000 mgr.y (mgr.24661) 85 : cephadm [INF] Reconfiguring daemon osd.3 on smithi136 2024-09-12T23:43:39.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:39 smithi136 bash[75164]: audit 2024-09-12T23:43:38.742973+0000 mon.a (mon.0) 254 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:43:39.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:39 smithi136 bash[76371]: cluster 2024-09-12T23:43:38.395772+0000 mon.a (mon.0) 249 : cluster [DBG] osdmap e365: 8 total, 8 up, 8 in 2024-09-12T23:43:39.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:39 smithi136 bash[76371]: audit 2024-09-12T23:43:38.669837+0000 mon.a (mon.0) 250 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:39.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:39 smithi136 bash[76371]: cluster 2024-09-12T23:43:38.677169+0000 mgr.y (mgr.24661) 83 : cluster [DBG] pgmap v122: 121 pgs: 121 active+clean; 252 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:43:39.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:39 smithi136 bash[76371]: audit 2024-09-12T23:43:38.679729+0000 mon.a (mon.0) 251 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:39.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:39 smithi136 bash[76371]: cephadm 2024-09-12T23:43:38.681252+0000 mgr.y (mgr.24661) 84 : cephadm [INF] Reconfiguring osd.3 (monmap changed)... 2024-09-12T23:43:39.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:39 smithi136 bash[76371]: audit 2024-09-12T23:43:38.681708+0000 mon.a (mon.0) 252 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-12T23:43:39.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:39 smithi136 bash[76371]: audit 2024-09-12T23:43:38.683021+0000 mon.a (mon.0) 253 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:43:39.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:39 smithi136 bash[76371]: cephadm 2024-09-12T23:43:38.685985+0000 mgr.y (mgr.24661) 85 : cephadm [INF] Reconfiguring daemon osd.3 on smithi136 2024-09-12T23:43:39.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:39 smithi136 bash[76371]: audit 2024-09-12T23:43:38.742973+0000 mon.a (mon.0) 254 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:43:39.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:39 smithi181 bash[67587]: cluster 2024-09-12T23:43:38.395772+0000 mon.a (mon.0) 249 : cluster [DBG] osdmap e365: 8 total, 8 up, 8 in 2024-09-12T23:43:39.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:39 smithi181 bash[67587]: audit 2024-09-12T23:43:38.669837+0000 mon.a (mon.0) 250 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:39.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:39 smithi181 bash[67587]: cluster 2024-09-12T23:43:38.677169+0000 mgr.y (mgr.24661) 83 : cluster [DBG] pgmap v122: 121 pgs: 121 active+clean; 252 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:43:39.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:39 smithi181 bash[67587]: audit 2024-09-12T23:43:38.679729+0000 mon.a (mon.0) 251 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:39.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:39 smithi181 bash[67587]: cephadm 2024-09-12T23:43:38.681252+0000 mgr.y (mgr.24661) 84 : cephadm [INF] Reconfiguring osd.3 (monmap changed)... 2024-09-12T23:43:39.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:39 smithi181 bash[67587]: audit 2024-09-12T23:43:38.681708+0000 mon.a (mon.0) 252 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-12T23:43:39.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:39 smithi181 bash[67587]: audit 2024-09-12T23:43:38.683021+0000 mon.a (mon.0) 253 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:43:39.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:39 smithi181 bash[67587]: cephadm 2024-09-12T23:43:38.685985+0000 mgr.y (mgr.24661) 85 : cephadm [INF] Reconfiguring daemon osd.3 on smithi136 2024-09-12T23:43:39.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:39 smithi181 bash[67587]: audit 2024-09-12T23:43:38.742973+0000 mon.a (mon.0) 254 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:43:40.412 INFO:tasks.rados.rados.0.smithi136.stdout:1551: done (2 left) 2024-09-12T23:43:40.412 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 38 current snap is 149 2024-09-12T23:43:40.412 INFO:tasks.rados.rados.0.smithi136.stdout:1549: finishing write tid 1 to smithi13667081-9 2024-09-12T23:43:40.412 INFO:tasks.rados.rados.0.smithi136.stdout:1549: finishing write tid 2 to smithi13667081-9 2024-09-12T23:43:40.412 INFO:tasks.rados.rados.0.smithi136.stdout:1552: seq_num 503 ranges {2400256=425984,2826240=704512,3530752=98304} 2024-09-12T23:43:40.428 INFO:tasks.rados.rados.0.smithi136.stdout:1552: writing smithi13667081-38 from 2400256 to 2826240 tid 1 2024-09-12T23:43:40.433 INFO:tasks.rados.rados.0.smithi136.stdout:1552: writing smithi13667081-38 from 2826240 to 3530752 tid 2 2024-09-12T23:43:40.434 INFO:tasks.rados.rados.0.smithi136.stdout:1552: writing smithi13667081-38 from 3530752 to 3629056 tid 3 2024-09-12T23:43:40.434 INFO:tasks.rados.rados.0.smithi136.stdout:1553: setattr oid 22 current snap is 149 2024-09-12T23:43:40.434 INFO:tasks.rados.rados.0.smithi136.stdout:1549: finishing write tid 3 to smithi13667081-9 2024-09-12T23:43:40.434 INFO:tasks.rados.rados.0.smithi136.stdout:1549: finishing write tid 4 to smithi13667081-9 2024-09-12T23:43:40.434 INFO:tasks.rados.rados.0.smithi136.stdout:1549: finishing write tid 5 to smithi13667081-9 2024-09-12T23:43:40.435 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 9 v 619 (ObjNum 502 snap 149 seq_num 502) dirty exists 2024-09-12T23:43:40.435 INFO:tasks.rados.rados.0.smithi136.stdout:1549: left oid 9 (ObjNum 502 snap 149 seq_num 502) 2024-09-12T23:43:40.435 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 29 v 699 (ObjNum 423 snap 125 seq_num 423) dirty exists 2024-09-12T23:43:40.438 INFO:tasks.rados.rados.0.smithi136.stdout:1549: done (3 left) 2024-09-12T23:43:40.438 INFO:tasks.rados.rados.0.smithi136.stdout:1550: done (2 left) 2024-09-12T23:43:40.438 INFO:tasks.rados.rados.0.smithi136.stdout:1554: snap_remove snap 125 2024-09-12T23:43:40.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:40 smithi136 bash[75164]: cluster 2024-09-12T23:43:39.409888+0000 mon.a (mon.0) 255 : cluster [DBG] osdmap e366: 8 total, 8 up, 8 in 2024-09-12T23:43:40.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:40 smithi136 bash[75164]: cluster 2024-09-12T23:43:39.706613+0000 mon.a (mon.0) 256 : cluster [DBG] mgrmap e43: y(active, since 91s), standbys: x 2024-09-12T23:43:40.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:40 smithi136 bash[76371]: cluster 2024-09-12T23:43:39.409888+0000 mon.a (mon.0) 255 : cluster [DBG] osdmap e366: 8 total, 8 up, 8 in 2024-09-12T23:43:40.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:40 smithi136 bash[76371]: cluster 2024-09-12T23:43:39.706613+0000 mon.a (mon.0) 256 : cluster [DBG] mgrmap e43: y(active, since 91s), standbys: x 2024-09-12T23:43:40.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:40 smithi181 bash[67587]: cluster 2024-09-12T23:43:39.409888+0000 mon.a (mon.0) 255 : cluster [DBG] osdmap e366: 8 total, 8 up, 8 in 2024-09-12T23:43:40.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:40 smithi181 bash[67587]: cluster 2024-09-12T23:43:39.706613+0000 mon.a (mon.0) 256 : cluster [DBG] mgrmap e43: y(active, since 91s), standbys: x 2024-09-12T23:43:41.449 INFO:tasks.rados.rados.0.smithi136.stdout:1554: done (2 left) 2024-09-12T23:43:41.449 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 50 current snap is 149 2024-09-12T23:43:41.449 INFO:tasks.rados.rados.0.smithi136.stdout:1555: seq_num 504 ranges {0=450560,450560=589824,1040384=212992} 2024-09-12T23:43:41.451 INFO:tasks.rados.rados.0.smithi136.stdout:1555: writing smithi13667081-50 from 0 to 450560 tid 1 2024-09-12T23:43:41.456 INFO:tasks.rados.rados.0.smithi136.stdout:1555: writing smithi13667081-50 from 450560 to 1040384 tid 2 2024-09-12T23:43:41.458 INFO:tasks.rados.rados.0.smithi136.stdout:1555: writing smithi13667081-50 from 1040384 to 1253376 tid 3 2024-09-12T23:43:41.458 INFO:tasks.rados.rados.0.smithi136.stdout:1556: copy_from oid 28 from oid 19 current snap is 149 2024-09-12T23:43:41.458 INFO:tasks.rados.rados.0.smithi136.stdout:1552: finishing write tid 1 to smithi13667081-38 2024-09-12T23:43:41.458 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 22 v 792 (ObjNum 427 snap 127 seq_num 427) dirty exists 2024-09-12T23:43:41.459 INFO:tasks.rados.rados.0.smithi136.stdout:1552: finishing write tid 2 to smithi13667081-38 2024-09-12T23:43:41.459 INFO:tasks.rados.rados.0.smithi136.stdout:1552: finishing write tid 3 to smithi13667081-38 2024-09-12T23:43:41.459 INFO:tasks.rados.rados.0.smithi136.stdout:1552: finishing write tid 4 to smithi13667081-38 2024-09-12T23:43:41.459 INFO:tasks.rados.rados.0.smithi136.stdout:1552: finishing write tid 5 to smithi13667081-38 2024-09-12T23:43:41.459 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 38 v 703 (ObjNum 503 snap 149 seq_num 503) dirty exists 2024-09-12T23:43:41.459 INFO:tasks.rados.rados.0.smithi136.stdout:1552: left oid 38 (ObjNum 503 snap 149 seq_num 503) 2024-09-12T23:43:41.459 INFO:tasks.rados.rados.0.smithi136.stdout:1552: done (3 left) 2024-09-12T23:43:41.459 INFO:tasks.rados.rados.0.smithi136.stdout:1553: done (2 left) 2024-09-12T23:43:41.459 INFO:tasks.rados.rados.0.smithi136.stdout:1557: setattr oid 25 current snap is 149 2024-09-12T23:43:41.461 INFO:tasks.rados.rados.0.smithi136.stdout:1558: rollback oid 45 current snap is 149 2024-09-12T23:43:41.461 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 45 to 145 2024-09-12T23:43:41.461 INFO:tasks.rados.rados.0.smithi136.stdout:1559: rollback oid 6 current snap is 149 2024-09-12T23:43:41.461 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 6 to 145 2024-09-12T23:43:41.461 INFO:tasks.rados.rados.0.smithi136.stdout:1560: delete oid 7 current snap is 149 2024-09-12T23:43:41.461 INFO:tasks.rados.rados.0.smithi136.stdout:1559: finishing rollback tid 1 to smithi13667081-6 2024-09-12T23:43:41.461 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 6 v 0 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-12T23:43:41.473 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 25 v 292 (ObjNum 499 snap 148 seq_num 499) dirty exists 2024-09-12T23:43:41.481 INFO:tasks.rados.rados.0.smithi136.stdout:1558: finishing rollback tid 0 to smithi13667081-45 2024-09-12T23:43:41.481 INFO:tasks.rados.rados.0.smithi136.stdout:1558: finishing rollback tid 1 to smithi13667081-45 2024-09-12T23:43:41.483 INFO:tasks.rados.rados.0.smithi136.stdout:1557: done (5 left) 2024-09-12T23:43:41.483 INFO:tasks.rados.rados.0.smithi136.stdout:1559: done (4 left) 2024-09-12T23:43:41.483 INFO:tasks.rados.rados.0.smithi136.stdout:1560: done (3 left) 2024-09-12T23:43:41.483 INFO:tasks.rados.rados.0.smithi136.stdout:1561: read oid 3 snap -1 2024-09-12T23:43:41.483 INFO:tasks.rados.rados.0.smithi136.stdout:1561: expect (ObjNum 479 snap 142 seq_num 479) 2024-09-12T23:43:41.483 INFO:tasks.rados.rados.0.smithi136.stdout:1562: read oid 49 snap 148 2024-09-12T23:43:41.483 INFO:tasks.rados.rados.0.smithi136.stdout:1562: expect (ObjNum 399 snap 116 seq_num 399) 2024-09-12T23:43:41.483 INFO:tasks.rados.rados.0.smithi136.stdout:1563: delete oid 14 current snap is 149 2024-09-12T23:43:41.483 INFO:tasks.rados.rados.0.smithi136.stdout:1558: finishing rollback tid 2 to smithi13667081-45 2024-09-12T23:43:41.483 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 45 v 768 (ObjNum 351 snap 96 seq_num 351) dirty exists 2024-09-12T23:43:41.484 INFO:tasks.rados.rados.0.smithi136.stdout:1555: finishing write tid 1 to smithi13667081-50 2024-09-12T23:43:41.485 INFO:tasks.rados.rados.0.smithi136.stdout:1555: finishing write tid 2 to smithi13667081-50 2024-09-12T23:43:41.485 INFO:tasks.rados.rados.0.smithi136.stdout:1555: finishing write tid 3 to smithi13667081-50 2024-09-12T23:43:41.485 INFO:tasks.rados.rados.0.smithi136.stdout:1555: finishing write tid 4 to smithi13667081-50 2024-09-12T23:43:41.487 INFO:tasks.rados.rados.0.smithi136.stdout:1555: finishing write tid 5 to smithi13667081-50 2024-09-12T23:43:41.488 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 50 v 707 (ObjNum 504 snap 149 seq_num 504) dirty exists 2024-09-12T23:43:41.488 INFO:tasks.rados.rados.0.smithi136.stdout:1555: left oid 50 (ObjNum 504 snap 149 seq_num 504) 2024-09-12T23:43:41.492 INFO:tasks.rados.rados.0.smithi136.stdout:1561: expect (ObjNum 479 snap 142 seq_num 479) 2024-09-12T23:43:41.517 INFO:tasks.rados.rados.0.smithi136.stdout:1556: finishing copy_from racing read to smithi13667081-28 2024-09-12T23:43:41.517 INFO:tasks.rados.rados.0.smithi136.stdout:1556: finishing copy_from to smithi13667081-28 2024-09-12T23:43:41.517 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 28 v 609 (ObjNum 391 snap 113 seq_num 391) dirty exists 2024-09-12T23:43:41.517 INFO:tasks.rados.rados.0.smithi136.stdout:1562: expect (ObjNum 399 snap 116 seq_num 399) 2024-09-12T23:43:41.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:41 smithi136 bash[75164]: cluster 2024-09-12T23:43:40.413916+0000 mon.a (mon.0) 257 : cluster [DBG] osdmap e367: 8 total, 8 up, 8 in 2024-09-12T23:43:41.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:41 smithi136 bash[75164]: cluster 2024-09-12T23:43:40.677886+0000 mgr.y (mgr.24661) 86 : cluster [DBG] pgmap v125: 121 pgs: 121 active+clean; 272 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 16 MiB/s rd, 4.9 MiB/s wr, 28 op/s 2024-09-12T23:43:41.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:41 smithi136 bash[75164]: audit 2024-09-12T23:43:41.081000+0000 mon.a (mon.0) 258 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:41.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:41 smithi136 bash[75164]: audit 2024-09-12T23:43:41.086308+0000 mon.a (mon.0) 259 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:41.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:41 smithi136 bash[75164]: audit 2024-09-12T23:43:41.087649+0000 mon.a (mon.0) 260 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-12T23:43:41.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:41 smithi136 bash[75164]: audit 2024-09-12T23:43:41.088462+0000 mon.a (mon.0) 261 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:43:41.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:41 smithi136 bash[76371]: cluster 2024-09-12T23:43:40.413916+0000 mon.a (mon.0) 257 : cluster [DBG] osdmap e367: 8 total, 8 up, 8 in 2024-09-12T23:43:41.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:41 smithi136 bash[76371]: cluster 2024-09-12T23:43:40.677886+0000 mgr.y (mgr.24661) 86 : cluster [DBG] pgmap v125: 121 pgs: 121 active+clean; 272 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 16 MiB/s rd, 4.9 MiB/s wr, 28 op/s 2024-09-12T23:43:41.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:41 smithi136 bash[76371]: audit 2024-09-12T23:43:41.081000+0000 mon.a (mon.0) 258 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:41.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:41 smithi136 bash[76371]: audit 2024-09-12T23:43:41.086308+0000 mon.a (mon.0) 259 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:41.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:41 smithi136 bash[76371]: audit 2024-09-12T23:43:41.087649+0000 mon.a (mon.0) 260 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-12T23:43:41.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:41 smithi136 bash[76371]: audit 2024-09-12T23:43:41.088462+0000 mon.a (mon.0) 261 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:43:41.764 INFO:tasks.rados.rados.0.smithi136.stdout:1555: done (5 left) 2024-09-12T23:43:41.765 INFO:tasks.rados.rados.0.smithi136.stdout:1556: done (4 left) 2024-09-12T23:43:41.765 INFO:tasks.rados.rados.0.smithi136.stdout:1558: done (3 left) 2024-09-12T23:43:41.765 INFO:tasks.rados.rados.0.smithi136.stdout:1561: done (2 left) 2024-09-12T23:43:41.765 INFO:tasks.rados.rados.0.smithi136.stdout:1562: done (1 left) 2024-09-12T23:43:41.765 INFO:tasks.rados.rados.0.smithi136.stdout:1563: done (0 left) 2024-09-12T23:43:41.765 INFO:tasks.rados.rados.0.smithi136.stdout:1564: read oid 40 snap 140 2024-09-12T23:43:41.765 INFO:tasks.rados.rados.0.smithi136.stdout:1564: expect (ObjNum 412 snap 120 seq_num 412) 2024-09-12T23:43:41.765 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 43 current snap is 149 2024-09-12T23:43:41.765 INFO:tasks.rados.rados.0.smithi136.stdout:1565: seq_num 507 ranges {6176768=499712,6676480=638976,7315456=163840} 2024-09-12T23:43:41.801 INFO:tasks.rados.rados.0.smithi136.stdout:1565: writing smithi13667081-43 from 6176768 to 6676480 tid 1 2024-09-12T23:43:41.804 INFO:tasks.rados.rados.0.smithi136.stdout:1565: writing smithi13667081-43 from 6676480 to 7315456 tid 2 2024-09-12T23:43:41.805 INFO:tasks.rados.rados.0.smithi136.stdout:1565: writing smithi13667081-43 from 7315456 to 7479296 tid 3 2024-09-12T23:43:41.806 INFO:tasks.rados.rados.0.smithi136.stdout:1566: setattr oid 44 current snap is 149 2024-09-12T23:43:41.806 INFO:tasks.rados.rados.0.smithi136.stdout:1564: expect (ObjNum 412 snap 120 seq_num 412) 2024-09-12T23:43:41.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:41 smithi181 bash[67587]: cluster 2024-09-12T23:43:40.413916+0000 mon.a (mon.0) 257 : cluster [DBG] osdmap e367: 8 total, 8 up, 8 in 2024-09-12T23:43:41.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:41 smithi181 bash[67587]: cluster 2024-09-12T23:43:40.677886+0000 mgr.y (mgr.24661) 86 : cluster [DBG] pgmap v125: 121 pgs: 121 active+clean; 272 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 16 MiB/s rd, 4.9 MiB/s wr, 28 op/s 2024-09-12T23:43:41.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:41 smithi181 bash[67587]: audit 2024-09-12T23:43:41.081000+0000 mon.a (mon.0) 258 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:41.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:41 smithi181 bash[67587]: audit 2024-09-12T23:43:41.086308+0000 mon.a (mon.0) 259 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:41.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:41 smithi181 bash[67587]: audit 2024-09-12T23:43:41.087649+0000 mon.a (mon.0) 260 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-12T23:43:41.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:41 smithi181 bash[67587]: audit 2024-09-12T23:43:41.088462+0000 mon.a (mon.0) 261 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:43:42.014 INFO:tasks.rados.rados.0.smithi136.stdout:1565: finishing write tid 1 to smithi13667081-43 2024-09-12T23:43:42.014 INFO:tasks.rados.rados.0.smithi136.stdout:1565: finishing write tid 2 to smithi13667081-43 2024-09-12T23:43:42.015 INFO:tasks.rados.rados.0.smithi136.stdout:1565: finishing write tid 3 to smithi13667081-43 2024-09-12T23:43:42.015 INFO:tasks.rados.rados.0.smithi136.stdout:1565: finishing write tid 4 to smithi13667081-43 2024-09-12T23:43:42.015 INFO:tasks.rados.rados.0.smithi136.stdout:1565: finishing write tid 5 to smithi13667081-43 2024-09-12T23:43:42.015 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 43 v 960 (ObjNum 507 snap 149 seq_num 507) dirty exists 2024-09-12T23:43:42.015 INFO:tasks.rados.rados.0.smithi136.stdout:1565: left oid 43 (ObjNum 507 snap 149 seq_num 507) 2024-09-12T23:43:42.015 INFO:tasks.rados.rados.0.smithi136.stdout:1564: done (2 left) 2024-09-12T23:43:42.015 INFO:tasks.rados.rados.0.smithi136.stdout:1565: done (1 left) 2024-09-12T23:43:42.015 INFO:tasks.rados.rados.0.smithi136.stdout:1567: delete oid 38 current snap is 149 2024-09-12T23:43:42.018 INFO:tasks.rados.rados.0.smithi136.stdout:1567: done (1 left) 2024-09-12T23:43:42.018 INFO:tasks.rados.rados.0.smithi136.stdout:1568: rollback oid 40 current snap is 149 2024-09-12T23:43:42.018 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 40 to 140 2024-09-12T23:43:42.018 INFO:tasks.rados.rados.0.smithi136.stdout:1569: delete oid 13 current snap is 149 2024-09-12T23:43:42.021 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 44 v 961 (ObjNum 65 snap 0 seq_num 1599168097) dirty exists 2024-09-12T23:43:42.022 INFO:tasks.rados.rados.0.smithi136.stdout:1566: done (2 left) 2024-09-12T23:43:42.023 INFO:tasks.rados.rados.0.smithi136.stdout:1569: done (1 left) 2024-09-12T23:43:42.023 INFO:tasks.rados.rados.0.smithi136.stdout:1570: delete oid 22 current snap is 149 2024-09-12T23:43:42.024 INFO:tasks.rados.rados.0.smithi136.stdout:1568: finishing rollback tid 1 to smithi13667081-40 2024-09-12T23:43:42.024 INFO:tasks.rados.rados.0.smithi136.stdout:1568: finishing rollback tid 2 to smithi13667081-40 2024-09-12T23:43:42.024 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 40 v 1173 (ObjNum 412 snap 120 seq_num 412) dirty exists 2024-09-12T23:43:42.026 INFO:tasks.rados.rados.0.smithi136.stdout:1568: done (1 left) 2024-09-12T23:43:42.026 INFO:tasks.rados.rados.0.smithi136.stdout:1570: done (0 left) 2024-09-12T23:43:42.026 INFO:tasks.rados.rados.0.smithi136.stdout:1571: read oid 32 snap -1 2024-09-12T23:43:42.026 INFO:tasks.rados.rados.0.smithi136.stdout:1571: expect (ObjNum 491 snap 145 seq_num 491) 2024-09-12T23:43:42.027 INFO:tasks.rados.rados.0.smithi136.stdout:1572: setattr oid 19 current snap is 149 2024-09-12T23:43:42.027 INFO:tasks.rados.rados.0.smithi136.stdout:1573: snap_remove snap 145 2024-09-12T23:43:42.440 INFO:tasks.rados.rados.0.smithi136.stdout:1573: done (2 left) 2024-09-12T23:43:42.440 INFO:tasks.rados.rados.0.smithi136.stdout:1574: read oid 10 snap -1 2024-09-12T23:43:42.440 INFO:tasks.rados.rados.0.smithi136.stdout:1574: expect (ObjNum 292 snap 79 seq_num 292) 2024-09-12T23:43:42.441 INFO:tasks.rados.rados.0.smithi136.stdout:1575: rmattr oid 50 current snap is 149 2024-09-12T23:43:42.441 INFO:tasks.rados.rados.0.smithi136.stdout:1575: done (3 left) 2024-09-12T23:43:42.441 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 1 current snap is 149 2024-09-12T23:43:42.441 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 19 v 620 (ObjNum 391 snap 113 seq_num 391) dirty exists 2024-09-12T23:43:42.441 INFO:tasks.rados.rados.0.smithi136.stdout:1571: expect (ObjNum 491 snap 145 seq_num 491) 2024-09-12T23:43:42.514 INFO:tasks.rados.rados.0.smithi136.stdout:1574: expect (ObjNum 292 snap 79 seq_num 292) 2024-09-12T23:43:42.518 INFO:tasks.rados.rados.0.smithi136.stdout:1576: seq_num 511 ranges {0=565248,565248=630784,1196032=172032} 2024-09-12T23:43:42.521 INFO:tasks.rados.rados.0.smithi136.stdout:1576: writing smithi13667081-1 from 0 to 565248 tid 1 2024-09-12T23:43:42.524 INFO:tasks.rados.rados.0.smithi136.stdout:1576: writing smithi13667081-1 from 565248 to 1196032 tid 2 2024-09-12T23:43:42.526 INFO:tasks.rados.rados.0.smithi136.stdout:1576: writing smithi13667081-1 from 1196032 to 1368064 tid 3 2024-09-12T23:43:42.526 INFO:tasks.rados.rados.0.smithi136.stdout:1571: done (3 left) 2024-09-12T23:43:42.527 INFO:tasks.rados.rados.0.smithi136.stdout:1572: done (2 left) 2024-09-12T23:43:42.527 INFO:tasks.rados.rados.0.smithi136.stdout:1574: done (1 left) 2024-09-12T23:43:42.527 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 12 current snap is 149 2024-09-12T23:43:42.527 INFO:tasks.rados.rados.0.smithi136.stdout:1577: seq_num 512 ranges {1826816=581632,2408448=516096,2924544=286720} 2024-09-12T23:43:42.540 INFO:tasks.rados.rados.0.smithi136.stdout:1577: writing smithi13667081-12 from 1826816 to 2408448 tid 1 2024-09-12T23:43:42.543 INFO:tasks.rados.rados.0.smithi136.stdout:1577: writing smithi13667081-12 from 2408448 to 2924544 tid 2 2024-09-12T23:43:42.545 INFO:tasks.rados.rados.0.smithi136.stdout:1577: writing smithi13667081-12 from 2924544 to 3211264 tid 3 2024-09-12T23:43:42.545 INFO:tasks.rados.rados.0.smithi136.stdout:1578: snap_create 2024-09-12T23:43:42.545 INFO:tasks.rados.rados.0.smithi136.stdout:1576: finishing write tid 1 to smithi13667081-1 2024-09-12T23:43:42.545 INFO:tasks.rados.rados.0.smithi136.stdout:1576: finishing write tid 2 to smithi13667081-1 2024-09-12T23:43:42.545 INFO:tasks.rados.rados.0.smithi136.stdout:1576: finishing write tid 3 to smithi13667081-1 2024-09-12T23:43:42.546 INFO:tasks.rados.rados.0.smithi136.stdout:1576: finishing write tid 4 to smithi13667081-1 2024-09-12T23:43:42.546 INFO:tasks.rados.rados.0.smithi136.stdout:1576: finishing write tid 5 to smithi13667081-1 2024-09-12T23:43:42.546 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 1 v 797 (ObjNum 511 snap 149 seq_num 511) dirty exists 2024-09-12T23:43:42.546 INFO:tasks.rados.rados.0.smithi136.stdout:1576: left oid 1 (ObjNum 511 snap 149 seq_num 511) 2024-09-12T23:43:42.546 INFO:tasks.rados.rados.0.smithi136.stdout:1577: finishing write tid 1 to smithi13667081-12 2024-09-12T23:43:42.548 INFO:tasks.rados.rados.0.smithi136.stdout:1577: finishing write tid 2 to smithi13667081-12 2024-09-12T23:43:42.548 INFO:tasks.rados.rados.0.smithi136.stdout:1577: finishing write tid 3 to smithi13667081-12 2024-09-12T23:43:42.549 INFO:tasks.rados.rados.0.smithi136.stdout:1577: finishing write tid 4 to smithi13667081-12 2024-09-12T23:43:42.550 INFO:tasks.rados.rados.0.smithi136.stdout:1577: finishing write tid 5 to smithi13667081-12 2024-09-12T23:43:42.550 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 12 v 883 (ObjNum 512 snap 149 seq_num 512) dirty exists 2024-09-12T23:43:42.550 INFO:tasks.rados.rados.0.smithi136.stdout:1577: left oid 12 (ObjNum 512 snap 149 seq_num 512) 2024-09-12T23:43:42.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:42 smithi181 bash[67587]: cephadm 2024-09-12T23:43:41.087362+0000 mgr.y (mgr.24661) 87 : cephadm [INF] Reconfiguring mds.a (monmap changed)... 2024-09-12T23:43:42.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:42 smithi181 bash[67587]: cephadm 2024-09-12T23:43:41.089621+0000 mgr.y (mgr.24661) 88 : cephadm [INF] Reconfiguring daemon mds.a on smithi136 2024-09-12T23:43:42.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:42 smithi181 bash[67587]: cluster 2024-09-12T23:43:41.446657+0000 mon.a (mon.0) 262 : cluster [DBG] osdmap e368: 8 total, 8 up, 8 in 2024-09-12T23:43:42.594 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:43:42 smithi181 bash[60471]: ts=2024-09-12T23:43:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:43:42.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:42 smithi136 bash[75164]: cephadm 2024-09-12T23:43:41.087362+0000 mgr.y (mgr.24661) 87 : cephadm [INF] Reconfiguring mds.a (monmap changed)... 2024-09-12T23:43:42.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:42 smithi136 bash[75164]: cephadm 2024-09-12T23:43:41.089621+0000 mgr.y (mgr.24661) 88 : cephadm [INF] Reconfiguring daemon mds.a on smithi136 2024-09-12T23:43:42.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:42 smithi136 bash[75164]: cluster 2024-09-12T23:43:41.446657+0000 mon.a (mon.0) 262 : cluster [DBG] osdmap e368: 8 total, 8 up, 8 in 2024-09-12T23:43:42.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:42 smithi136 bash[76371]: cephadm 2024-09-12T23:43:41.087362+0000 mgr.y (mgr.24661) 87 : cephadm [INF] Reconfiguring mds.a (monmap changed)... 2024-09-12T23:43:42.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:42 smithi136 bash[76371]: cephadm 2024-09-12T23:43:41.089621+0000 mgr.y (mgr.24661) 88 : cephadm [INF] Reconfiguring daemon mds.a on smithi136 2024-09-12T23:43:42.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:42 smithi136 bash[76371]: cluster 2024-09-12T23:43:41.446657+0000 mon.a (mon.0) 262 : cluster [DBG] osdmap e368: 8 total, 8 up, 8 in 2024-09-12T23:43:43.468 INFO:tasks.rados.rados.0.smithi136.stdout:1576: done (2 left) 2024-09-12T23:43:43.469 INFO:tasks.rados.rados.0.smithi136.stdout:1577: done (1 left) 2024-09-12T23:43:43.469 INFO:tasks.rados.rados.0.smithi136.stdout:1578: done (0 left) 2024-09-12T23:43:43.469 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 9 current snap is 150 2024-09-12T23:43:43.469 INFO:tasks.rados.rados.0.smithi136.stdout:1579: seq_num 513 ranges {6438912=598016,7036928=794624,7831552=8192} 2024-09-12T23:43:43.531 INFO:tasks.rados.rados.0.smithi136.stdout:1579: writing smithi13667081-9 from 6438912 to 7036928 tid 1 2024-09-12T23:43:43.537 INFO:tasks.rados.rados.0.smithi136.stdout:1579: writing smithi13667081-9 from 7036928 to 7831552 tid 2 2024-09-12T23:43:43.537 INFO:tasks.rados.rados.0.smithi136.stdout:1579: writing smithi13667081-9 from 7831552 to 7839744 tid 3 2024-09-12T23:43:43.537 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 10 current snap is 150 2024-09-12T23:43:43.537 INFO:tasks.rados.rados.0.smithi136.stdout:1580: seq_num 514 ranges {122880=614400,737280=679936,1417216=122880} 2024-09-12T23:43:43.542 INFO:tasks.rados.rados.0.smithi136.stdout:1580: writing smithi13667081-10 from 122880 to 737280 tid 1 2024-09-12T23:43:43.547 INFO:tasks.rados.rados.0.smithi136.stdout:1580: writing smithi13667081-10 from 737280 to 1417216 tid 2 2024-09-12T23:43:43.548 INFO:tasks.rados.rados.0.smithi136.stdout:1580: writing smithi13667081-10 from 1417216 to 1540096 tid 3 2024-09-12T23:43:43.548 INFO:tasks.rados.rados.0.smithi136.stdout:1581: copy_from oid 6 from oid 7 current snap is 150 2024-09-12T23:43:43.548 INFO:tasks.rados.rados.0.smithi136.stdout:1579: finishing write tid 1 to smithi13667081-9 2024-09-12T23:43:43.548 INFO:tasks.rados.rados.0.smithi136.stdout:1582: rmattr oid 30 current snap is 150 2024-09-12T23:43:43.548 INFO:tasks.rados.rados.0.smithi136.stdout:1582: done (3 left) 2024-09-12T23:43:43.548 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 4 current snap is 150 2024-09-12T23:43:43.548 INFO:tasks.rados.rados.0.smithi136.stdout:1583: seq_num 515 ranges {0=630784,630784=565248,1196032=237568} 2024-09-12T23:43:43.552 INFO:tasks.rados.rados.0.smithi136.stdout:1583: writing smithi13667081-4 from 0 to 630784 tid 1 2024-09-12T23:43:43.555 INFO:tasks.rados.rados.0.smithi136.stdout:1583: writing smithi13667081-4 from 630784 to 1196032 tid 2 2024-09-12T23:43:43.557 INFO:tasks.rados.rados.0.smithi136.stdout:1583: writing smithi13667081-4 from 1196032 to 1433600 tid 3 2024-09-12T23:43:43.557 INFO:tasks.rados.rados.0.smithi136.stdout:1584: delete oid 15 current snap is 150 2024-09-12T23:43:43.557 INFO:tasks.rados.rados.0.smithi136.stdout:1579: finishing write tid 2 to smithi13667081-9 2024-09-12T23:43:43.558 INFO:tasks.rados.rados.0.smithi136.stdout:1579: finishing write tid 3 to smithi13667081-9 2024-09-12T23:43:43.558 INFO:tasks.rados.rados.0.smithi136.stdout:1581: finishing copy_from to smithi13667081-6 2024-09-12T23:43:43.558 INFO:tasks.rados.rados.0.smithi136.stdout:1581: got expected ENOENT (src dne) 2024-09-12T23:43:43.558 INFO:tasks.rados.rados.0.smithi136.stdout:1581: finishing copy_from racing read to smithi13667081-6 2024-09-12T23:43:43.558 INFO:tasks.rados.rados.0.smithi136.stdout:1579: finishing write tid 4 to smithi13667081-9 2024-09-12T23:43:43.558 INFO:tasks.rados.rados.0.smithi136.stdout:1580: finishing write tid 1 to smithi13667081-10 2024-09-12T23:43:43.558 INFO:tasks.rados.rados.0.smithi136.stdout:1580: finishing write tid 2 to smithi13667081-10 2024-09-12T23:43:43.558 INFO:tasks.rados.rados.0.smithi136.stdout:1580: finishing write tid 3 to smithi13667081-10 2024-09-12T23:43:43.558 INFO:tasks.rados.rados.0.smithi136.stdout:1580: finishing write tid 4 to smithi13667081-10 2024-09-12T23:43:43.558 INFO:tasks.rados.rados.0.smithi136.stdout:1579: finishing write tid 5 to smithi13667081-9 2024-09-12T23:43:43.558 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 9 v 624 (ObjNum 513 snap 150 seq_num 513) dirty exists 2024-09-12T23:43:43.558 INFO:tasks.rados.rados.0.smithi136.stdout:1579: left oid 9 (ObjNum 513 snap 150 seq_num 513) 2024-09-12T23:43:43.558 INFO:tasks.rados.rados.0.smithi136.stdout:1580: finishing write tid 5 to smithi13667081-10 2024-09-12T23:43:43.558 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 10 v 798 (ObjNum 514 snap 150 seq_num 514) dirty exists 2024-09-12T23:43:43.559 INFO:tasks.rados.rados.0.smithi136.stdout:1580: left oid 10 (ObjNum 514 snap 150 seq_num 514) 2024-09-12T23:43:43.561 INFO:tasks.rados.rados.0.smithi136.stdout:1583: finishing write tid 1 to smithi13667081-4 2024-09-12T23:43:43.563 INFO:tasks.rados.rados.0.smithi136.stdout:1579: done (4 left) 2024-09-12T23:43:43.563 INFO:tasks.rados.rados.0.smithi136.stdout:1580: done (3 left) 2024-09-12T23:43:43.563 INFO:tasks.rados.rados.0.smithi136.stdout:1581: done (2 left) 2024-09-12T23:43:43.563 INFO:tasks.rados.rados.0.smithi136.stdout:1584: done (1 left) 2024-09-12T23:43:43.563 INFO:tasks.rados.rados.0.smithi136.stdout:1585: read oid 11 snap -1 2024-09-12T23:43:43.563 INFO:tasks.rados.rados.0.smithi136.stdout:1585: expect (ObjNum 482 snap 142 seq_num 482) 2024-09-12T23:43:43.564 INFO:tasks.rados.rados.0.smithi136.stdout:1586: snap_create 2024-09-12T23:43:43.564 INFO:tasks.rados.rados.0.smithi136.stdout:1583: finishing write tid 2 to smithi13667081-4 2024-09-12T23:43:43.565 INFO:tasks.rados.rados.0.smithi136.stdout:1583: finishing write tid 3 to smithi13667081-4 2024-09-12T23:43:43.565 INFO:tasks.rados.rados.0.smithi136.stdout:1583: finishing write tid 4 to smithi13667081-4 2024-09-12T23:43:43.567 INFO:tasks.rados.rados.0.smithi136.stdout:1583: finishing write tid 5 to smithi13667081-4 2024-09-12T23:43:43.567 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 4 v 736 (ObjNum 515 snap 150 seq_num 515) dirty exists 2024-09-12T23:43:43.567 INFO:tasks.rados.rados.0.smithi136.stdout:1583: left oid 4 (ObjNum 515 snap 150 seq_num 515) 2024-09-12T23:43:43.569 INFO:tasks.rados.rados.0.smithi136.stdout:1585: expect (ObjNum 482 snap 142 seq_num 482) 2024-09-12T23:43:43.740 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:43:43 smithi181 bash[60471]: ts=2024-09-12T23:43:43.513Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi136\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:43:43.741 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:43 smithi181 bash[67587]: cluster 2024-09-12T23:43:42.443586+0000 mon.a (mon.0) 263 : cluster [DBG] osdmap e369: 8 total, 8 up, 8 in 2024-09-12T23:43:43.741 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:43 smithi181 bash[67587]: cluster 2024-09-12T23:43:42.678388+0000 mgr.y (mgr.24661) 89 : cluster [DBG] pgmap v128: 121 pgs: 121 active+clean; 272 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 16 MiB/s rd, 4.9 MiB/s wr, 28 op/s 2024-09-12T23:43:43.741 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:43 smithi181 bash[67587]: audit 2024-09-12T23:43:43.353352+0000 mon.a (mon.0) 264 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:43.741 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:43 smithi181 bash[67587]: audit 2024-09-12T23:43:43.362943+0000 mon.a (mon.0) 265 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:43.741 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:43 smithi181 bash[67587]: audit 2024-09-12T23:43:43.364881+0000 mon.a (mon.0) 266 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-12T23:43:43.741 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:43 smithi181 bash[67587]: audit 2024-09-12T23:43:43.365829+0000 mon.a (mon.0) 267 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-12T23:43:43.741 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:43 smithi181 bash[67587]: audit 2024-09-12T23:43:43.366717+0000 mon.a (mon.0) 268 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:43:43.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:43 smithi136 bash[75164]: cluster 2024-09-12T23:43:42.443586+0000 mon.a (mon.0) 263 : cluster [DBG] osdmap e369: 8 total, 8 up, 8 in 2024-09-12T23:43:43.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:43 smithi136 bash[75164]: cluster 2024-09-12T23:43:42.678388+0000 mgr.y (mgr.24661) 89 : cluster [DBG] pgmap v128: 121 pgs: 121 active+clean; 272 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 16 MiB/s rd, 4.9 MiB/s wr, 28 op/s 2024-09-12T23:43:43.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:43 smithi136 bash[75164]: audit 2024-09-12T23:43:43.353352+0000 mon.a (mon.0) 264 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:43.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:43 smithi136 bash[75164]: audit 2024-09-12T23:43:43.362943+0000 mon.a (mon.0) 265 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:43.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:43 smithi136 bash[75164]: audit 2024-09-12T23:43:43.364881+0000 mon.a (mon.0) 266 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-12T23:43:43.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:43 smithi136 bash[75164]: audit 2024-09-12T23:43:43.365829+0000 mon.a (mon.0) 267 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-12T23:43:43.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:43 smithi136 bash[75164]: audit 2024-09-12T23:43:43.366717+0000 mon.a (mon.0) 268 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:43:43.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:43 smithi136 bash[76371]: cluster 2024-09-12T23:43:42.443586+0000 mon.a (mon.0) 263 : cluster [DBG] osdmap e369: 8 total, 8 up, 8 in 2024-09-12T23:43:43.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:43 smithi136 bash[76371]: cluster 2024-09-12T23:43:42.678388+0000 mgr.y (mgr.24661) 89 : cluster [DBG] pgmap v128: 121 pgs: 121 active+clean; 272 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 16 MiB/s rd, 4.9 MiB/s wr, 28 op/s 2024-09-12T23:43:43.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:43 smithi136 bash[76371]: audit 2024-09-12T23:43:43.353352+0000 mon.a (mon.0) 264 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:43.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:43 smithi136 bash[76371]: audit 2024-09-12T23:43:43.362943+0000 mon.a (mon.0) 265 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:43.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:43 smithi136 bash[76371]: audit 2024-09-12T23:43:43.364881+0000 mon.a (mon.0) 266 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-12T23:43:43.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:43 smithi136 bash[76371]: audit 2024-09-12T23:43:43.365829+0000 mon.a (mon.0) 267 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-12T23:43:43.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:43 smithi136 bash[76371]: audit 2024-09-12T23:43:43.366717+0000 mon.a (mon.0) 268 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:43:44.481 INFO:tasks.rados.rados.0.smithi136.stdout:1583: done (2 left) 2024-09-12T23:43:44.481 INFO:tasks.rados.rados.0.smithi136.stdout:1585: done (1 left) 2024-09-12T23:43:44.481 INFO:tasks.rados.rados.0.smithi136.stdout:1586: done (0 left) 2024-09-12T23:43:44.481 INFO:tasks.rados.rados.0.smithi136.stdout:1587: delete oid 24 current snap is 151 2024-09-12T23:43:44.491 INFO:tasks.rados.rados.0.smithi136.stdout:1587: done (0 left) 2024-09-12T23:43:44.491 INFO:tasks.rados.rados.0.smithi136.stdout:1588: snap_create 2024-09-12T23:43:44.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:44 smithi136 bash[75164]: cephadm 2024-09-12T23:43:43.364529+0000 mgr.y (mgr.24661) 90 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2024-09-12T23:43:44.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:44 smithi136 bash[75164]: cephadm 2024-09-12T23:43:43.367808+0000 mgr.y (mgr.24661) 91 : cephadm [INF] Reconfiguring daemon mon.b on smithi181 2024-09-12T23:43:44.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:44 smithi136 bash[75164]: cluster 2024-09-12T23:43:43.471779+0000 mon.a (mon.0) 269 : cluster [DBG] osdmap e370: 8 total, 8 up, 8 in 2024-09-12T23:43:44.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:44 smithi136 bash[76371]: cephadm 2024-09-12T23:43:43.364529+0000 mgr.y (mgr.24661) 90 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2024-09-12T23:43:44.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:44 smithi136 bash[76371]: cephadm 2024-09-12T23:43:43.367808+0000 mgr.y (mgr.24661) 91 : cephadm [INF] Reconfiguring daemon mon.b on smithi181 2024-09-12T23:43:44.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:44 smithi136 bash[76371]: cluster 2024-09-12T23:43:43.471779+0000 mon.a (mon.0) 269 : cluster [DBG] osdmap e370: 8 total, 8 up, 8 in 2024-09-12T23:43:44.816 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:43:44 smithi181 bash[60471]: ts=2024-09-12T23:43:44.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:43:44.816 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:43:44 smithi181 bash[60471]: ts=2024-09-12T23:43:44.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:43:44.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:44 smithi181 bash[67587]: cephadm 2024-09-12T23:43:43.364529+0000 mgr.y (mgr.24661) 90 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2024-09-12T23:43:44.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:44 smithi181 bash[67587]: cephadm 2024-09-12T23:43:43.367808+0000 mgr.y (mgr.24661) 91 : cephadm [INF] Reconfiguring daemon mon.b on smithi181 2024-09-12T23:43:44.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:44 smithi181 bash[67587]: cluster 2024-09-12T23:43:43.471779+0000 mon.a (mon.0) 269 : cluster [DBG] osdmap e370: 8 total, 8 up, 8 in 2024-09-12T23:43:45.492 INFO:tasks.rados.rados.0.smithi136.stdout:1588: done (0 left) 2024-09-12T23:43:45.493 INFO:tasks.rados.rados.0.smithi136.stdout:1589: snap_remove snap 149 2024-09-12T23:43:45.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:45 smithi136 bash[75164]: cluster 2024-09-12T23:43:44.484536+0000 mon.a (mon.0) 270 : cluster [DBG] osdmap e371: 8 total, 8 up, 8 in 2024-09-12T23:43:45.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:45 smithi136 bash[75164]: cluster 2024-09-12T23:43:44.679778+0000 mgr.y (mgr.24661) 92 : cluster [DBG] pgmap v131: 121 pgs: 121 active+clean; 222 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 3.9 MiB/s wr, 39 op/s 2024-09-12T23:43:45.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:45 smithi136 bash[76371]: cluster 2024-09-12T23:43:44.484536+0000 mon.a (mon.0) 270 : cluster [DBG] osdmap e371: 8 total, 8 up, 8 in 2024-09-12T23:43:45.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:45 smithi136 bash[76371]: cluster 2024-09-12T23:43:44.679778+0000 mgr.y (mgr.24661) 92 : cluster [DBG] pgmap v131: 121 pgs: 121 active+clean; 222 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 3.9 MiB/s wr, 39 op/s 2024-09-12T23:43:45.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:45 smithi181 bash[67587]: cluster 2024-09-12T23:43:44.484536+0000 mon.a (mon.0) 270 : cluster [DBG] osdmap e371: 8 total, 8 up, 8 in 2024-09-12T23:43:45.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:45 smithi181 bash[67587]: cluster 2024-09-12T23:43:44.679778+0000 mgr.y (mgr.24661) 92 : cluster [DBG] pgmap v131: 121 pgs: 121 active+clean; 222 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 3.9 MiB/s wr, 39 op/s 2024-09-12T23:43:46.309 INFO:tasks.rados.rados.0.smithi136.stdout:1589: done (0 left) 2024-09-12T23:43:46.309 INFO:tasks.rados.rados.0.smithi136.stdout:1590: snap_create 2024-09-12T23:43:46.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:46 smithi136 bash[75164]: cluster 2024-09-12T23:43:45.495828+0000 mon.a (mon.0) 271 : cluster [DBG] osdmap e372: 8 total, 8 up, 8 in 2024-09-12T23:43:46.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:46 smithi136 bash[75164]: audit 2024-09-12T23:43:45.643136+0000 mon.a (mon.0) 272 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:46.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:46 smithi136 bash[75164]: audit 2024-09-12T23:43:45.654192+0000 mon.a (mon.0) 273 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:46.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:46 smithi136 bash[75164]: cephadm 2024-09-12T23:43:45.656067+0000 mgr.y (mgr.24661) 93 : cephadm [INF] Reconfiguring mgr.x (monmap changed)... 2024-09-12T23:43:46.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:46 smithi136 bash[75164]: audit 2024-09-12T23:43:45.656752+0000 mon.a (mon.0) 274 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-12T23:43:46.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:46 smithi136 bash[75164]: audit 2024-09-12T23:43:45.658637+0000 mon.a (mon.0) 275 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-12T23:43:46.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:46 smithi136 bash[75164]: audit 2024-09-12T23:43:45.660380+0000 mon.a (mon.0) 276 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:43:46.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:46 smithi136 bash[75164]: cephadm 2024-09-12T23:43:45.662182+0000 mgr.y (mgr.24661) 94 : cephadm [INF] Reconfiguring daemon mgr.x on smithi181 2024-09-12T23:43:46.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:46 smithi136 bash[75164]: cluster 2024-09-12T23:43:46.310385+0000 mon.a (mon.0) 277 : cluster [DBG] osdmap e373: 8 total, 8 up, 8 in 2024-09-12T23:43:46.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:46 smithi136 bash[76371]: cluster 2024-09-12T23:43:45.495828+0000 mon.a (mon.0) 271 : cluster [DBG] osdmap e372: 8 total, 8 up, 8 in 2024-09-12T23:43:46.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:46 smithi136 bash[76371]: audit 2024-09-12T23:43:45.643136+0000 mon.a (mon.0) 272 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:46.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:46 smithi136 bash[76371]: audit 2024-09-12T23:43:45.654192+0000 mon.a (mon.0) 273 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:46.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:46 smithi136 bash[76371]: cephadm 2024-09-12T23:43:45.656067+0000 mgr.y (mgr.24661) 93 : cephadm [INF] Reconfiguring mgr.x (monmap changed)... 2024-09-12T23:43:46.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:46 smithi136 bash[76371]: audit 2024-09-12T23:43:45.656752+0000 mon.a (mon.0) 274 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-12T23:43:46.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:46 smithi136 bash[76371]: audit 2024-09-12T23:43:45.658637+0000 mon.a (mon.0) 275 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-12T23:43:46.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:46 smithi136 bash[76371]: audit 2024-09-12T23:43:45.660380+0000 mon.a (mon.0) 276 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:43:46.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:46 smithi136 bash[76371]: cephadm 2024-09-12T23:43:45.662182+0000 mgr.y (mgr.24661) 94 : cephadm [INF] Reconfiguring daemon mgr.x on smithi181 2024-09-12T23:43:46.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:46 smithi136 bash[76371]: cluster 2024-09-12T23:43:46.310385+0000 mon.a (mon.0) 277 : cluster [DBG] osdmap e373: 8 total, 8 up, 8 in 2024-09-12T23:43:46.841 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:46 smithi181 bash[67587]: cluster 2024-09-12T23:43:45.495828+0000 mon.a (mon.0) 271 : cluster [DBG] osdmap e372: 8 total, 8 up, 8 in 2024-09-12T23:43:46.841 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:46 smithi181 bash[67587]: audit 2024-09-12T23:43:45.643136+0000 mon.a (mon.0) 272 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:46.841 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:46 smithi181 bash[67587]: audit 2024-09-12T23:43:45.654192+0000 mon.a (mon.0) 273 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:46.842 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:46 smithi181 bash[67587]: cephadm 2024-09-12T23:43:45.656067+0000 mgr.y (mgr.24661) 93 : cephadm [INF] Reconfiguring mgr.x (monmap changed)... 2024-09-12T23:43:46.842 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:46 smithi181 bash[67587]: audit 2024-09-12T23:43:45.656752+0000 mon.a (mon.0) 274 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-12T23:43:46.842 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:46 smithi181 bash[67587]: audit 2024-09-12T23:43:45.658637+0000 mon.a (mon.0) 275 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-12T23:43:46.842 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:46 smithi181 bash[67587]: audit 2024-09-12T23:43:45.660380+0000 mon.a (mon.0) 276 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:43:46.842 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:46 smithi181 bash[67587]: cephadm 2024-09-12T23:43:45.662182+0000 mgr.y (mgr.24661) 94 : cephadm [INF] Reconfiguring daemon mgr.x on smithi181 2024-09-12T23:43:46.842 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:46 smithi181 bash[67587]: cluster 2024-09-12T23:43:46.310385+0000 mon.a (mon.0) 277 : cluster [DBG] osdmap e373: 8 total, 8 up, 8 in 2024-09-12T23:43:47.309 INFO:tasks.rados.rados.0.smithi136.stdout:1590: done (0 left) 2024-09-12T23:43:47.309 INFO:tasks.rados.rados.0.smithi136.stdout:1591: read oid 11 snap -1 2024-09-12T23:43:47.309 INFO:tasks.rados.rados.0.smithi136.stdout:1591: expect (ObjNum 482 snap 142 seq_num 482) 2024-09-12T23:43:47.309 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 24 current snap is 153 2024-09-12T23:43:47.309 INFO:tasks.rados.rados.0.smithi136.stdout:1592: seq_num 518 ranges {0=679936,679936=671744,1351680=131072} 2024-09-12T23:43:47.321 INFO:tasks.rados.rados.0.smithi136.stdout:1592: writing smithi13667081-24 from 0 to 679936 tid 1 2024-09-12T23:43:47.332 INFO:tasks.rados.rados.0.smithi136.stdout:1592: writing smithi13667081-24 from 679936 to 1351680 tid 2 2024-09-12T23:43:47.335 INFO:tasks.rados.rados.0.smithi136.stdout:1592: writing smithi13667081-24 from 1351680 to 1482752 tid 3 2024-09-12T23:43:47.335 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 32 current snap is 153 2024-09-12T23:43:47.335 INFO:tasks.rados.rados.0.smithi136.stdout:1593: seq_num 519 ranges {1843200=696320,2539520=548864,3088384=253952} 2024-09-12T23:43:47.372 INFO:tasks.rados.rados.0.smithi136.stdout:1593: writing smithi13667081-32 from 1843200 to 2539520 tid 1 2024-09-12T23:43:47.378 INFO:tasks.rados.rados.0.smithi136.stdout:1593: writing smithi13667081-32 from 2539520 to 3088384 tid 2 2024-09-12T23:43:47.381 INFO:tasks.rados.rados.0.smithi136.stdout:1593: writing smithi13667081-32 from 3088384 to 3342336 tid 3 2024-09-12T23:43:47.381 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 36 current snap is 153 2024-09-12T23:43:47.382 INFO:tasks.rados.rados.0.smithi136.stdout:1594: seq_num 520 ranges {2957312=712704,3670016=434176,4104192=368640} 2024-09-12T23:43:47.410 INFO:tasks.rados.rados.0.smithi136.stdout:1594: writing smithi13667081-36 from 2957312 to 3670016 tid 1 2024-09-12T23:43:47.413 INFO:tasks.rados.rados.0.smithi136.stdout:1594: writing smithi13667081-36 from 3670016 to 4104192 tid 2 2024-09-12T23:43:47.415 INFO:tasks.rados.rados.0.smithi136.stdout:1594: writing smithi13667081-36 from 4104192 to 4472832 tid 3 2024-09-12T23:43:47.415 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 22 current snap is 153 2024-09-12T23:43:47.415 INFO:tasks.rados.rados.0.smithi136.stdout:1595: seq_num 521 ranges {0=729088,729088=720896,1449984=81920} 2024-09-12T23:43:47.420 INFO:tasks.rados.rados.0.smithi136.stdout:1595: writing smithi13667081-22 from 0 to 729088 tid 1 2024-09-12T23:43:47.424 INFO:tasks.rados.rados.0.smithi136.stdout:1595: writing smithi13667081-22 from 729088 to 1449984 tid 2 2024-09-12T23:43:47.425 INFO:tasks.rados.rados.0.smithi136.stdout:1595: writing smithi13667081-22 from 1449984 to 1531904 tid 3 2024-09-12T23:43:47.425 INFO:tasks.rados.rados.0.smithi136.stdout:1596: snap_remove snap 148 2024-09-12T23:43:47.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:47 smithi136 bash[75164]: cluster 2024-09-12T23:43:46.680490+0000 mgr.y (mgr.24661) 95 : cluster [DBG] pgmap v134: 121 pgs: 121 active+clean; 222 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 3.9 MiB/s wr, 39 op/s 2024-09-12T23:43:47.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:47 smithi136 bash[75164]: cluster 2024-09-12T23:43:47.312648+0000 mon.a (mon.0) 278 : cluster [DBG] osdmap e374: 8 total, 8 up, 8 in 2024-09-12T23:43:47.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:47 smithi136 bash[76371]: cluster 2024-09-12T23:43:46.680490+0000 mgr.y (mgr.24661) 95 : cluster [DBG] pgmap v134: 121 pgs: 121 active+clean; 222 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 3.9 MiB/s wr, 39 op/s 2024-09-12T23:43:47.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:47 smithi136 bash[76371]: cluster 2024-09-12T23:43:47.312648+0000 mon.a (mon.0) 278 : cluster [DBG] osdmap e374: 8 total, 8 up, 8 in 2024-09-12T23:43:47.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:47 smithi181 bash[67587]: cluster 2024-09-12T23:43:46.680490+0000 mgr.y (mgr.24661) 95 : cluster [DBG] pgmap v134: 121 pgs: 121 active+clean; 222 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 3.9 MiB/s wr, 39 op/s 2024-09-12T23:43:47.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:47 smithi181 bash[67587]: cluster 2024-09-12T23:43:47.312648+0000 mon.a (mon.0) 278 : cluster [DBG] osdmap e374: 8 total, 8 up, 8 in 2024-09-12T23:43:48.314 INFO:tasks.rados.rados.0.smithi136.stdout:1596: done (5 left) 2024-09-12T23:43:48.314 INFO:tasks.rados.rados.0.smithi136.stdout:1597: rollback oid 34 current snap is 153 2024-09-12T23:43:48.314 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 34 to 142 2024-09-12T23:43:48.314 INFO:tasks.rados.rados.0.smithi136.stdout:1591: expect (ObjNum 482 snap 142 seq_num 482) 2024-09-12T23:43:48.359 INFO:tasks.rados.rados.0.smithi136.stdout:1592: finishing write tid 1 to smithi13667081-24 2024-09-12T23:43:48.359 INFO:tasks.rados.rados.0.smithi136.stdout:1592: finishing write tid 2 to smithi13667081-24 2024-09-12T23:43:48.360 INFO:tasks.rados.rados.0.smithi136.stdout:1592: finishing write tid 3 to smithi13667081-24 2024-09-12T23:43:48.360 INFO:tasks.rados.rados.0.smithi136.stdout:1592: finishing write tid 4 to smithi13667081-24 2024-09-12T23:43:48.360 INFO:tasks.rados.rados.0.smithi136.stdout:1591: done (5 left) 2024-09-12T23:43:48.360 INFO:tasks.rados.rados.0.smithi136.stdout:1598: snap_remove snap 152 2024-09-12T23:43:48.942 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:43:48 smithi181 bash[60471]: ts=2024-09-12T23:43:48.662Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:43:48.942 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:48 smithi181 bash[67587]: audit 2024-09-12T23:43:47.661984+0000 mon.a (mon.0) 279 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:48.942 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:48 smithi181 bash[67587]: audit 2024-09-12T23:43:47.674052+0000 mon.a (mon.0) 280 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:48.942 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:48 smithi181 bash[67587]: cephadm 2024-09-12T23:43:47.676207+0000 mgr.y (mgr.24661) 96 : cephadm [INF] Reconfiguring osd.4 (monmap changed)... 2024-09-12T23:43:48.942 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:48 smithi181 bash[67587]: audit 2024-09-12T23:43:47.676785+0000 mon.a (mon.0) 281 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-12T23:43:48.942 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:48 smithi181 bash[67587]: audit 2024-09-12T23:43:47.678574+0000 mon.a (mon.0) 282 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:43:48.942 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:48 smithi181 bash[67587]: cephadm 2024-09-12T23:43:47.683522+0000 mgr.y (mgr.24661) 97 : cephadm [INF] Reconfiguring daemon osd.4 on smithi181 2024-09-12T23:43:48.942 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:48 smithi181 bash[67587]: cluster 2024-09-12T23:43:48.317616+0000 mon.a (mon.0) 283 : cluster [DBG] osdmap e375: 8 total, 8 up, 8 in 2024-09-12T23:43:49.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:48 smithi136 bash[76371]: audit 2024-09-12T23:43:47.661984+0000 mon.a (mon.0) 279 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:49.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:48 smithi136 bash[76371]: audit 2024-09-12T23:43:47.674052+0000 mon.a (mon.0) 280 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:49.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:48 smithi136 bash[76371]: cephadm 2024-09-12T23:43:47.676207+0000 mgr.y (mgr.24661) 96 : cephadm [INF] Reconfiguring osd.4 (monmap changed)... 2024-09-12T23:43:49.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:48 smithi136 bash[76371]: audit 2024-09-12T23:43:47.676785+0000 mon.a (mon.0) 281 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-12T23:43:49.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:48 smithi136 bash[76371]: audit 2024-09-12T23:43:47.678574+0000 mon.a (mon.0) 282 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:43:49.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:48 smithi136 bash[76371]: cephadm 2024-09-12T23:43:47.683522+0000 mgr.y (mgr.24661) 97 : cephadm [INF] Reconfiguring daemon osd.4 on smithi181 2024-09-12T23:43:49.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:48 smithi136 bash[76371]: cluster 2024-09-12T23:43:48.317616+0000 mon.a (mon.0) 283 : cluster [DBG] osdmap e375: 8 total, 8 up, 8 in 2024-09-12T23:43:49.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:48 smithi136 bash[75164]: audit 2024-09-12T23:43:47.661984+0000 mon.a (mon.0) 279 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:49.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:48 smithi136 bash[75164]: audit 2024-09-12T23:43:47.674052+0000 mon.a (mon.0) 280 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:49.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:48 smithi136 bash[75164]: cephadm 2024-09-12T23:43:47.676207+0000 mgr.y (mgr.24661) 96 : cephadm [INF] Reconfiguring osd.4 (monmap changed)... 2024-09-12T23:43:49.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:48 smithi136 bash[75164]: audit 2024-09-12T23:43:47.676785+0000 mon.a (mon.0) 281 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-12T23:43:49.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:48 smithi136 bash[75164]: audit 2024-09-12T23:43:47.678574+0000 mon.a (mon.0) 282 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:43:49.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:48 smithi136 bash[75164]: cephadm 2024-09-12T23:43:47.683522+0000 mgr.y (mgr.24661) 97 : cephadm [INF] Reconfiguring daemon osd.4 on smithi181 2024-09-12T23:43:49.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:48 smithi136 bash[75164]: cluster 2024-09-12T23:43:48.317616+0000 mon.a (mon.0) 283 : cluster [DBG] osdmap e375: 8 total, 8 up, 8 in 2024-09-12T23:43:49.317 INFO:tasks.rados.rados.0.smithi136.stdout:1598: done (5 left) 2024-09-12T23:43:49.318 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 20 current snap is 153 2024-09-12T23:43:49.318 INFO:tasks.rados.rados.0.smithi136.stdout:1592: finishing write tid 5 to smithi13667081-24 2024-09-12T23:43:49.318 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 24 v 619 (ObjNum 518 snap 153 seq_num 518) dirty exists 2024-09-12T23:43:49.318 INFO:tasks.rados.rados.0.smithi136.stdout:1592: left oid 24 (ObjNum 518 snap 153 seq_num 518) 2024-09-12T23:43:49.318 INFO:tasks.rados.rados.0.smithi136.stdout:1593: finishing write tid 1 to smithi13667081-32 2024-09-12T23:43:49.318 INFO:tasks.rados.rados.0.smithi136.stdout:1593: finishing write tid 2 to smithi13667081-32 2024-09-12T23:43:49.318 INFO:tasks.rados.rados.0.smithi136.stdout:1593: finishing write tid 3 to smithi13667081-32 2024-09-12T23:43:49.318 INFO:tasks.rados.rados.0.smithi136.stdout:1593: finishing write tid 4 to smithi13667081-32 2024-09-12T23:43:49.318 INFO:tasks.rados.rados.0.smithi136.stdout:1593: finishing write tid 5 to smithi13667081-32 2024-09-12T23:43:49.318 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 32 v 740 (ObjNum 519 snap 153 seq_num 519) dirty exists 2024-09-12T23:43:49.318 INFO:tasks.rados.rados.0.smithi136.stdout:1593: left oid 32 (ObjNum 519 snap 153 seq_num 519) 2024-09-12T23:43:49.318 INFO:tasks.rados.rados.0.smithi136.stdout:1594: finishing write tid 1 to smithi13667081-36 2024-09-12T23:43:49.318 INFO:tasks.rados.rados.0.smithi136.stdout:1594: finishing write tid 2 to smithi13667081-36 2024-09-12T23:43:49.319 INFO:tasks.rados.rados.0.smithi136.stdout:1594: finishing write tid 3 to smithi13667081-36 2024-09-12T23:43:49.319 INFO:tasks.rados.rados.0.smithi136.stdout:1594: finishing write tid 4 to smithi13667081-36 2024-09-12T23:43:49.319 INFO:tasks.rados.rados.0.smithi136.stdout:1594: finishing write tid 5 to smithi13667081-36 2024-09-12T23:43:49.319 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 36 v 772 (ObjNum 520 snap 153 seq_num 520) dirty exists 2024-09-12T23:43:49.319 INFO:tasks.rados.rados.0.smithi136.stdout:1594: left oid 36 (ObjNum 520 snap 153 seq_num 520) 2024-09-12T23:43:49.319 INFO:tasks.rados.rados.0.smithi136.stdout:1595: finishing write tid 1 to smithi13667081-22 2024-09-12T23:43:49.319 INFO:tasks.rados.rados.0.smithi136.stdout:1595: finishing write tid 2 to smithi13667081-22 2024-09-12T23:43:49.319 INFO:tasks.rados.rados.0.smithi136.stdout:1595: finishing write tid 3 to smithi13667081-22 2024-09-12T23:43:49.319 INFO:tasks.rados.rados.0.smithi136.stdout:1599: seq_num 522 ranges {1458176=745472,2203648=606208,2809856=196608} 2024-09-12T23:43:49.339 INFO:tasks.rados.rados.0.smithi136.stdout:1599: writing smithi13667081-20 from 1458176 to 2203648 tid 1 2024-09-12T23:43:49.345 INFO:tasks.rados.rados.0.smithi136.stdout:1599: writing smithi13667081-20 from 2203648 to 2809856 tid 2 2024-09-12T23:43:49.349 INFO:tasks.rados.rados.0.smithi136.stdout:1599: writing smithi13667081-20 from 2809856 to 3006464 tid 3 2024-09-12T23:43:49.349 INFO:tasks.rados.rados.0.smithi136.stdout:1592: done (5 left) 2024-09-12T23:43:49.349 INFO:tasks.rados.rados.0.smithi136.stdout:1593: done (4 left) 2024-09-12T23:43:49.349 INFO:tasks.rados.rados.0.smithi136.stdout:1594: done (3 left) 2024-09-12T23:43:49.349 INFO:tasks.rados.rados.0.smithi136.stdout:1600: read oid 43 snap 142 2024-09-12T23:43:49.349 INFO:tasks.rados.rados.0.smithi136.stdout:1600: expect (ObjNum 399 snap 116 seq_num 399) 2024-09-12T23:43:49.349 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 8 current snap is 153 2024-09-12T23:43:49.349 INFO:tasks.rados.rados.0.smithi136.stdout:1595: finishing write tid 4 to smithi13667081-22 2024-09-12T23:43:49.349 INFO:tasks.rados.rados.0.smithi136.stdout:1595: finishing write tid 5 to smithi13667081-22 2024-09-12T23:43:49.349 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 22 v 801 (ObjNum 521 snap 153 seq_num 521) dirty exists 2024-09-12T23:43:49.349 INFO:tasks.rados.rados.0.smithi136.stdout:1595: left oid 22 (ObjNum 521 snap 153 seq_num 521) 2024-09-12T23:43:49.349 INFO:tasks.rados.rados.0.smithi136.stdout:1597: finishing rollback tid 1 to smithi13667081-34 2024-09-12T23:43:49.349 INFO:tasks.rados.rados.0.smithi136.stdout:1597: finishing rollback tid 2 to smithi13667081-34 2024-09-12T23:43:49.350 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 34 v 786 (ObjNum 370 snap 108 seq_num 370) dirty exists 2024-09-12T23:43:49.350 INFO:tasks.rados.rados.0.smithi136.stdout:1601: seq_num 523 ranges {4743168=770048,5513216=491520,6004736=311296} 2024-09-12T23:43:49.399 INFO:tasks.rados.rados.0.smithi136.stdout:1601: writing smithi13667081-8 from 4743168 to 5513216 tid 1 2024-09-12T23:43:49.403 INFO:tasks.rados.rados.0.smithi136.stdout:1601: writing smithi13667081-8 from 5513216 to 6004736 tid 2 2024-09-12T23:43:49.405 INFO:tasks.rados.rados.0.smithi136.stdout:1601: writing smithi13667081-8 from 6004736 to 6316032 tid 3 2024-09-12T23:43:49.405 INFO:tasks.rados.rados.0.smithi136.stdout:1595: done (4 left) 2024-09-12T23:43:49.406 INFO:tasks.rados.rados.0.smithi136.stdout:1597: done (3 left) 2024-09-12T23:43:49.406 INFO:tasks.rados.rados.0.smithi136.stdout:1602: read oid 30 snap -1 2024-09-12T23:43:49.406 INFO:tasks.rados.rados.0.smithi136.stdout:1602: expect (ObjNum 427 snap 127 seq_num 427) 2024-09-12T23:43:49.406 INFO:tasks.rados.rados.0.smithi136.stdout:1603: rmattr oid 44 current snap is 153 2024-09-12T23:43:49.406 INFO:tasks.rados.rados.0.smithi136.stdout:1599: finishing write tid 1 to smithi13667081-20 2024-09-12T23:43:49.406 INFO:tasks.rados.rados.0.smithi136.stdout:1604: read oid 19 snap -1 2024-09-12T23:43:49.406 INFO:tasks.rados.rados.0.smithi136.stdout:1604: expect (ObjNum 391 snap 113 seq_num 391) 2024-09-12T23:43:49.406 INFO:tasks.rados.rados.0.smithi136.stdout:1605: snap_remove snap 140 2024-09-12T23:43:49.662 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:43:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:43:49] "GET /metrics HTTP/1.1" 200 36186 "" "Prometheus/2.43.0" 2024-09-12T23:43:50.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:49 smithi136 bash[75164]: cluster 2024-09-12T23:43:48.681180+0000 mgr.y (mgr.24661) 98 : cluster [DBG] pgmap v137: 121 pgs: 121 active+clean; 222 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:43:50.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:49 smithi136 bash[75164]: cluster 2024-09-12T23:43:49.320724+0000 mon.a (mon.0) 284 : cluster [DBG] osdmap e376: 8 total, 8 up, 8 in 2024-09-12T23:43:50.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:49 smithi136 bash[76371]: cluster 2024-09-12T23:43:48.681180+0000 mgr.y (mgr.24661) 98 : cluster [DBG] pgmap v137: 121 pgs: 121 active+clean; 222 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:43:50.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:49 smithi136 bash[76371]: cluster 2024-09-12T23:43:49.320724+0000 mon.a (mon.0) 284 : cluster [DBG] osdmap e376: 8 total, 8 up, 8 in 2024-09-12T23:43:50.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:49 smithi181 bash[67587]: cluster 2024-09-12T23:43:48.681180+0000 mgr.y (mgr.24661) 98 : cluster [DBG] pgmap v137: 121 pgs: 121 active+clean; 222 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:43:50.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:49 smithi181 bash[67587]: cluster 2024-09-12T23:43:49.320724+0000 mon.a (mon.0) 284 : cluster [DBG] osdmap e376: 8 total, 8 up, 8 in 2024-09-12T23:43:50.319 INFO:tasks.rados.rados.0.smithi136.stdout:1605: done (6 left) 2024-09-12T23:43:50.319 INFO:tasks.rados.rados.0.smithi136.stdout:1606: copy_from oid 13 from oid 45 current snap is 153 2024-09-12T23:43:50.319 INFO:tasks.rados.rados.0.smithi136.stdout:1607: rollback oid 21 current snap is 153 2024-09-12T23:43:50.319 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 21 to 150 2024-09-12T23:43:50.320 INFO:tasks.rados.rados.0.smithi136.stdout:1608: snap_create 2024-09-12T23:43:50.320 INFO:tasks.rados.rados.0.smithi136.stdout:1599: finishing write tid 2 to smithi13667081-20 2024-09-12T23:43:50.320 INFO:tasks.rados.rados.0.smithi136.stdout:1599: finishing write tid 3 to smithi13667081-20 2024-09-12T23:43:50.320 INFO:tasks.rados.rados.0.smithi136.stdout:1599: finishing write tid 4 to smithi13667081-20 2024-09-12T23:43:50.320 INFO:tasks.rados.rados.0.smithi136.stdout:1599: finishing write tid 5 to smithi13667081-20 2024-09-12T23:43:50.321 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 20 v 306 (ObjNum 522 snap 153 seq_num 522) dirty exists 2024-09-12T23:43:50.321 INFO:tasks.rados.rados.0.smithi136.stdout:1599: left oid 20 (ObjNum 522 snap 153 seq_num 522) 2024-09-12T23:43:50.321 INFO:tasks.rados.rados.0.smithi136.stdout:1600: expect (ObjNum 399 snap 116 seq_num 399) 2024-09-12T23:43:50.592 INFO:tasks.rados.rados.0.smithi136.stdout:1601: finishing write tid 1 to smithi13667081-8 2024-09-12T23:43:50.592 INFO:tasks.rados.rados.0.smithi136.stdout:1601: finishing write tid 2 to smithi13667081-8 2024-09-12T23:43:50.592 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 44 v 977 (ObjNum 161 snap 0 seq_num 0) dirty exists 2024-09-12T23:43:50.592 INFO:tasks.rados.rados.0.smithi136.stdout:1601: finishing write tid 3 to smithi13667081-8 2024-09-12T23:43:50.592 INFO:tasks.rados.rados.0.smithi136.stdout:1601: finishing write tid 4 to smithi13667081-8 2024-09-12T23:43:50.592 INFO:tasks.rados.rados.0.smithi136.stdout:1601: finishing write tid 5 to smithi13667081-8 2024-09-12T23:43:50.592 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 8 v 776 (ObjNum 523 snap 153 seq_num 523) dirty exists 2024-09-12T23:43:50.592 INFO:tasks.rados.rados.0.smithi136.stdout:1601: left oid 8 (ObjNum 523 snap 153 seq_num 523) 2024-09-12T23:43:50.592 INFO:tasks.rados.rados.0.smithi136.stdout:1602: expect (ObjNum 427 snap 127 seq_num 427) 2024-09-12T23:43:50.711 INFO:tasks.rados.rados.0.smithi136.stdout:1604: expect (ObjNum 391 snap 113 seq_num 391) 2024-09-12T23:43:50.953 INFO:tasks.rados.rados.0.smithi136.stdout:1607: finishing rollback tid 0 to smithi13667081-21 2024-09-12T23:43:50.953 INFO:tasks.rados.rados.0.smithi136.stdout:1607: finishing rollback tid 1 to smithi13667081-21 2024-09-12T23:43:50.953 INFO:tasks.rados.rados.0.smithi136.stdout:1607: finishing rollback tid 2 to smithi13667081-21 2024-09-12T23:43:50.954 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 21 v 804 (ObjNum 441 snap 132 seq_num 441) dirty exists 2024-09-12T23:43:50.954 INFO:tasks.rados.rados.0.smithi136.stdout:1606: finishing copy_from racing read to smithi13667081-13 2024-09-12T23:43:50.954 INFO:tasks.rados.rados.0.smithi136.stdout:1606: finishing copy_from to smithi13667081-13 2024-09-12T23:43:50.954 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 13 v 768 (ObjNum 351 snap 96 seq_num 351) dirty exists 2024-09-12T23:43:51.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:50 smithi181 bash[67587]: audit 2024-09-12T23:43:49.835402+0000 mon.a (mon.0) 285 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:51.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:50 smithi181 bash[67587]: audit 2024-09-12T23:43:49.845619+0000 mon.a (mon.0) 286 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:51.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:50 smithi181 bash[67587]: audit 2024-09-12T23:43:49.848181+0000 mon.a (mon.0) 287 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-12T23:43:51.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:50 smithi181 bash[67587]: audit 2024-09-12T23:43:49.849887+0000 mon.a (mon.0) 288 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:43:51.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:50 smithi181 bash[67587]: cluster 2024-09-12T23:43:50.322483+0000 mon.a (mon.0) 289 : cluster [DBG] osdmap e377: 8 total, 8 up, 8 in 2024-09-12T23:43:51.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:50 smithi136 bash[75164]: audit 2024-09-12T23:43:49.835402+0000 mon.a (mon.0) 285 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:51.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:50 smithi136 bash[75164]: audit 2024-09-12T23:43:49.845619+0000 mon.a (mon.0) 286 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:51.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:50 smithi136 bash[75164]: audit 2024-09-12T23:43:49.848181+0000 mon.a (mon.0) 287 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-12T23:43:51.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:50 smithi136 bash[75164]: audit 2024-09-12T23:43:49.849887+0000 mon.a (mon.0) 288 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:43:51.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:50 smithi136 bash[75164]: cluster 2024-09-12T23:43:50.322483+0000 mon.a (mon.0) 289 : cluster [DBG] osdmap e377: 8 total, 8 up, 8 in 2024-09-12T23:43:51.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:50 smithi136 bash[76371]: audit 2024-09-12T23:43:49.835402+0000 mon.a (mon.0) 285 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:51.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:50 smithi136 bash[76371]: audit 2024-09-12T23:43:49.845619+0000 mon.a (mon.0) 286 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:51.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:50 smithi136 bash[76371]: audit 2024-09-12T23:43:49.848181+0000 mon.a (mon.0) 287 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-12T23:43:51.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:50 smithi136 bash[76371]: audit 2024-09-12T23:43:49.849887+0000 mon.a (mon.0) 288 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:43:51.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:50 smithi136 bash[76371]: cluster 2024-09-12T23:43:50.322483+0000 mon.a (mon.0) 289 : cluster [DBG] osdmap e377: 8 total, 8 up, 8 in 2024-09-12T23:43:51.329 INFO:tasks.rados.rados.0.smithi136.stdout:1599: done (8 left) 2024-09-12T23:43:51.330 INFO:tasks.rados.rados.0.smithi136.stdout:1600: done (7 left) 2024-09-12T23:43:51.330 INFO:tasks.rados.rados.0.smithi136.stdout:1601: done (6 left) 2024-09-12T23:43:51.330 INFO:tasks.rados.rados.0.smithi136.stdout:1602: done (5 left) 2024-09-12T23:43:51.330 INFO:tasks.rados.rados.0.smithi136.stdout:1603: done (4 left) 2024-09-12T23:43:51.330 INFO:tasks.rados.rados.0.smithi136.stdout:1604: done (3 left) 2024-09-12T23:43:51.330 INFO:tasks.rados.rados.0.smithi136.stdout:1606: done (2 left) 2024-09-12T23:43:51.330 INFO:tasks.rados.rados.0.smithi136.stdout:1607: done (1 left) 2024-09-12T23:43:51.330 INFO:tasks.rados.rados.0.smithi136.stdout:1608: done (0 left) 2024-09-12T23:43:51.330 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 25 current snap is 154 2024-09-12T23:43:51.330 INFO:tasks.rados.rados.0.smithi136.stdout:1609: seq_num 524 ranges {4767744=786432,5554176=778240,6332416=24576} 2024-09-12T23:43:51.382 INFO:tasks.rados.rados.0.smithi136.stdout:1609: writing smithi13667081-25 from 4767744 to 5554176 tid 1 2024-09-12T23:43:51.389 INFO:tasks.rados.rados.0.smithi136.stdout:1609: writing smithi13667081-25 from 5554176 to 6332416 tid 2 2024-09-12T23:43:51.389 INFO:tasks.rados.rados.0.smithi136.stdout:1609: writing smithi13667081-25 from 6332416 to 6356992 tid 3 2024-09-12T23:43:51.389 INFO:tasks.rados.rados.0.smithi136.stdout:1610: delete oid 33 current snap is 154 2024-09-12T23:43:51.393 INFO:tasks.rados.rados.0.smithi136.stdout:1609: finishing write tid 1 to smithi13667081-25 2024-09-12T23:43:51.397 INFO:tasks.rados.rados.0.smithi136.stdout:1610: done (1 left) 2024-09-12T23:43:51.397 INFO:tasks.rados.rados.0.smithi136.stdout:1611: snap_create 2024-09-12T23:43:51.399 INFO:tasks.rados.rados.0.smithi136.stdout:1609: finishing write tid 2 to smithi13667081-25 2024-09-12T23:43:51.399 INFO:tasks.rados.rados.0.smithi136.stdout:1609: finishing write tid 3 to smithi13667081-25 2024-09-12T23:43:51.400 INFO:tasks.rados.rados.0.smithi136.stdout:1609: finishing write tid 4 to smithi13667081-25 2024-09-12T23:43:51.401 INFO:tasks.rados.rados.0.smithi136.stdout:1609: finishing write tid 5 to smithi13667081-25 2024-09-12T23:43:51.401 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 25 v 305 (ObjNum 524 snap 154 seq_num 524) dirty exists 2024-09-12T23:43:51.402 INFO:tasks.rados.rados.0.smithi136.stdout:1609: left oid 25 (ObjNum 524 snap 154 seq_num 524) 2024-09-12T23:43:52.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:51 smithi181 bash[67587]: cephadm 2024-09-12T23:43:49.847526+0000 mgr.y (mgr.24661) 99 : cephadm [INF] Reconfiguring osd.5 (monmap changed)... 2024-09-12T23:43:52.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:51 smithi181 bash[67587]: cephadm 2024-09-12T23:43:49.854693+0000 mgr.y (mgr.24661) 100 : cephadm [INF] Reconfiguring daemon osd.5 on smithi181 2024-09-12T23:43:52.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:51 smithi181 bash[67587]: cluster 2024-09-12T23:43:50.681850+0000 mgr.y (mgr.24661) 101 : cluster [DBG] pgmap v140: 121 pgs: 121 active+clean; 231 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.2 MiB/s wr, 13 op/s 2024-09-12T23:43:52.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:51 smithi181 bash[67587]: cluster 2024-09-12T23:43:51.332940+0000 mon.a (mon.0) 290 : cluster [DBG] osdmap e378: 8 total, 8 up, 8 in 2024-09-12T23:43:52.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:51 smithi136 bash[75164]: cephadm 2024-09-12T23:43:49.847526+0000 mgr.y (mgr.24661) 99 : cephadm [INF] Reconfiguring osd.5 (monmap changed)... 2024-09-12T23:43:52.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:51 smithi136 bash[75164]: cephadm 2024-09-12T23:43:49.854693+0000 mgr.y (mgr.24661) 100 : cephadm [INF] Reconfiguring daemon osd.5 on smithi181 2024-09-12T23:43:52.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:51 smithi136 bash[75164]: cluster 2024-09-12T23:43:50.681850+0000 mgr.y (mgr.24661) 101 : cluster [DBG] pgmap v140: 121 pgs: 121 active+clean; 231 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.2 MiB/s wr, 13 op/s 2024-09-12T23:43:52.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:51 smithi136 bash[75164]: cluster 2024-09-12T23:43:51.332940+0000 mon.a (mon.0) 290 : cluster [DBG] osdmap e378: 8 total, 8 up, 8 in 2024-09-12T23:43:52.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:51 smithi136 bash[76371]: cephadm 2024-09-12T23:43:49.847526+0000 mgr.y (mgr.24661) 99 : cephadm [INF] Reconfiguring osd.5 (monmap changed)... 2024-09-12T23:43:52.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:51 smithi136 bash[76371]: cephadm 2024-09-12T23:43:49.854693+0000 mgr.y (mgr.24661) 100 : cephadm [INF] Reconfiguring daemon osd.5 on smithi181 2024-09-12T23:43:52.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:51 smithi136 bash[76371]: cluster 2024-09-12T23:43:50.681850+0000 mgr.y (mgr.24661) 101 : cluster [DBG] pgmap v140: 121 pgs: 121 active+clean; 231 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.2 MiB/s wr, 13 op/s 2024-09-12T23:43:52.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:51 smithi136 bash[76371]: cluster 2024-09-12T23:43:51.332940+0000 mon.a (mon.0) 290 : cluster [DBG] osdmap e378: 8 total, 8 up, 8 in 2024-09-12T23:43:52.334 INFO:tasks.rados.rados.0.smithi136.stdout:1609: done (1 left) 2024-09-12T23:43:52.335 INFO:tasks.rados.rados.0.smithi136.stdout:1611: done (0 left) 2024-09-12T23:43:52.335 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 5 current snap is 155 2024-09-12T23:43:52.335 INFO:tasks.rados.rados.0.smithi136.stdout:1612: seq_num 526 ranges {4784128=417792,5201920=589824,5791744=614400} 2024-09-12T23:43:52.375 INFO:tasks.rados.rados.0.smithi136.stdout:1612: writing smithi13667081-5 from 4784128 to 5201920 tid 1 2024-09-12T23:43:52.380 INFO:tasks.rados.rados.0.smithi136.stdout:1612: writing smithi13667081-5 from 5201920 to 5791744 tid 2 2024-09-12T23:43:52.387 INFO:tasks.rados.rados.0.smithi136.stdout:1612: writing smithi13667081-5 from 5791744 to 6406144 tid 3 2024-09-12T23:43:52.387 INFO:tasks.rados.rados.0.smithi136.stdout:1613: rollback oid 11 current snap is 155 2024-09-12T23:43:52.387 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 11 to 150 2024-09-12T23:43:52.387 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 4 current snap is 155 2024-09-12T23:43:52.387 INFO:tasks.rados.rados.0.smithi136.stdout:1614: seq_num 527 ranges {1433600=434176,1867776=475136,2342912=565248,2908160=163840} 2024-09-12T23:43:52.401 INFO:tasks.rados.rados.0.smithi136.stdout:1614: writing smithi13667081-4 from 1433600 to 1867776 tid 1 2024-09-12T23:43:52.404 INFO:tasks.rados.rados.0.smithi136.stdout:1614: writing smithi13667081-4 from 1867776 to 2342912 tid 2 2024-09-12T23:43:52.408 INFO:tasks.rados.rados.0.smithi136.stdout:1614: writing smithi13667081-4 from 2342912 to 2908160 tid 3 2024-09-12T23:43:52.410 INFO:tasks.rados.rados.0.smithi136.stdout:1614: writing smithi13667081-4 from 2908160 to 3072000 tid 4 2024-09-12T23:43:52.410 INFO:tasks.rados.rados.0.smithi136.stdout:1615: delete oid 30 current snap is 155 2024-09-12T23:43:52.410 INFO:tasks.rados.rados.0.smithi136.stdout:1612: finishing write tid 1 to smithi13667081-5 2024-09-12T23:43:52.410 INFO:tasks.rados.rados.0.smithi136.stdout:1612: finishing write tid 2 to smithi13667081-5 2024-09-12T23:43:52.410 INFO:tasks.rados.rados.0.smithi136.stdout:1612: finishing write tid 3 to smithi13667081-5 2024-09-12T23:43:52.410 INFO:tasks.rados.rados.0.smithi136.stdout:1612: finishing write tid 4 to smithi13667081-5 2024-09-12T23:43:52.410 INFO:tasks.rados.rados.0.smithi136.stdout:1612: finishing write tid 5 to smithi13667081-5 2024-09-12T23:43:52.410 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 5 v 1194 (ObjNum 526 snap 155 seq_num 526) dirty exists 2024-09-12T23:43:52.410 INFO:tasks.rados.rados.0.smithi136.stdout:1612: left oid 5 (ObjNum 526 snap 155 seq_num 526) 2024-09-12T23:43:52.410 INFO:tasks.rados.rados.0.smithi136.stdout:1613: finishing rollback tid 0 to smithi13667081-11 2024-09-12T23:43:52.410 INFO:tasks.rados.rados.0.smithi136.stdout:1613: finishing rollback tid 1 to smithi13667081-11 2024-09-12T23:43:52.410 INFO:tasks.rados.rados.0.smithi136.stdout:1613: finishing rollback tid 2 to smithi13667081-11 2024-09-12T23:43:52.410 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 11 v 1198 (ObjNum 482 snap 142 seq_num 482) dirty exists 2024-09-12T23:43:52.410 INFO:tasks.rados.rados.0.smithi136.stdout:1614: finishing write tid 1 to smithi13667081-4 2024-09-12T23:43:52.411 INFO:tasks.rados.rados.0.smithi136.stdout:1614: finishing write tid 2 to smithi13667081-4 2024-09-12T23:43:52.412 INFO:tasks.rados.rados.0.smithi136.stdout:1612: done (3 left) 2024-09-12T23:43:52.412 INFO:tasks.rados.rados.0.smithi136.stdout:1613: done (2 left) 2024-09-12T23:43:52.412 INFO:tasks.rados.rados.0.smithi136.stdout:1615: done (1 left) 2024-09-12T23:43:52.412 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 15 current snap is 155 2024-09-12T23:43:52.412 INFO:tasks.rados.rados.0.smithi136.stdout:1616: seq_num 529 ranges {0=466944,466944=647168,1114112=532480,1646592=24576} 2024-09-12T23:43:52.415 INFO:tasks.rados.rados.0.smithi136.stdout:1616: writing smithi13667081-15 from 0 to 466944 tid 1 2024-09-12T23:43:52.419 INFO:tasks.rados.rados.0.smithi136.stdout:1616: writing smithi13667081-15 from 466944 to 1114112 tid 2 2024-09-12T23:43:52.423 INFO:tasks.rados.rados.0.smithi136.stdout:1616: writing smithi13667081-15 from 1114112 to 1646592 tid 3 2024-09-12T23:43:52.423 INFO:tasks.rados.rados.0.smithi136.stdout:1616: writing smithi13667081-15 from 1646592 to 1671168 tid 4 2024-09-12T23:43:52.424 INFO:tasks.rados.rados.0.smithi136.stdout:1617: delete oid 34 current snap is 155 2024-09-12T23:43:52.424 INFO:tasks.rados.rados.0.smithi136.stdout:1614: finishing write tid 3 to smithi13667081-4 2024-09-12T23:43:52.424 INFO:tasks.rados.rados.0.smithi136.stdout:1614: finishing write tid 4 to smithi13667081-4 2024-09-12T23:43:52.424 INFO:tasks.rados.rados.0.smithi136.stdout:1614: finishing write tid 5 to smithi13667081-4 2024-09-12T23:43:52.424 INFO:tasks.rados.rados.0.smithi136.stdout:1614: finishing write tid 6 to smithi13667081-4 2024-09-12T23:43:52.424 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 4 v 745 (ObjNum 527 snap 155 seq_num 527) dirty exists 2024-09-12T23:43:52.424 INFO:tasks.rados.rados.0.smithi136.stdout:1614: left oid 4 (ObjNum 527 snap 155 seq_num 527) 2024-09-12T23:43:52.424 INFO:tasks.rados.rados.0.smithi136.stdout:1616: finishing write tid 1 to smithi13667081-15 2024-09-12T23:43:52.424 INFO:tasks.rados.rados.0.smithi136.stdout:1616: finishing write tid 2 to smithi13667081-15 2024-09-12T23:43:52.427 INFO:tasks.rados.rados.0.smithi136.stdout:1616: finishing write tid 3 to smithi13667081-15 2024-09-12T23:43:52.427 INFO:tasks.rados.rados.0.smithi136.stdout:1614: done (2 left) 2024-09-12T23:43:52.427 INFO:tasks.rados.rados.0.smithi136.stdout:1617: done (1 left) 2024-09-12T23:43:52.427 INFO:tasks.rados.rados.0.smithi136.stdout:1618: rollback oid 13 current snap is 155 2024-09-12T23:43:52.428 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 13 to 154 2024-09-12T23:43:52.428 INFO:tasks.rados.rados.0.smithi136.stdout:1616: finishing write tid 4 to smithi13667081-15 2024-09-12T23:43:52.428 INFO:tasks.rados.rados.0.smithi136.stdout:1619: snap_create 2024-09-12T23:43:52.428 INFO:tasks.rados.rados.0.smithi136.stdout:1616: finishing write tid 5 to smithi13667081-15 2024-09-12T23:43:52.430 INFO:tasks.rados.rados.0.smithi136.stdout:1616: finishing write tid 6 to smithi13667081-15 2024-09-12T23:43:52.430 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 15 v 888 (ObjNum 529 snap 155 seq_num 529) dirty exists 2024-09-12T23:43:52.430 INFO:tasks.rados.rados.0.smithi136.stdout:1616: left oid 15 (ObjNum 529 snap 155 seq_num 529) 2024-09-12T23:43:52.432 INFO:tasks.rados.rados.0.smithi136.stdout:1618: finishing rollback tid 0 to smithi13667081-13 2024-09-12T23:43:52.433 INFO:tasks.rados.rados.0.smithi136.stdout:1618: finishing rollback tid 1 to smithi13667081-13 2024-09-12T23:43:52.433 INFO:tasks.rados.rados.0.smithi136.stdout:1618: finishing rollback tid 2 to smithi13667081-13 2024-09-12T23:43:52.433 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 13 v 771 (ObjNum 351 snap 96 seq_num 351) dirty exists 2024-09-12T23:43:52.594 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:43:52 smithi181 bash[60471]: ts=2024-09-12T23:43:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:43:53.344 INFO:tasks.rados.rados.0.smithi136.stdout:1616: done (2 left) 2024-09-12T23:43:53.345 INFO:tasks.rados.rados.0.smithi136.stdout:1618: done (1 left) 2024-09-12T23:43:53.345 INFO:tasks.rados.rados.0.smithi136.stdout:1619: done (0 left) 2024-09-12T23:43:53.345 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 38 current snap is 156 2024-09-12T23:43:53.345 INFO:tasks.rados.rados.0.smithi136.stdout:1620: seq_num 531 ranges {0=499712,499712=417792,917504=548864,1466368=237568} 2024-09-12T23:43:53.345 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:53 smithi181 bash[67587]: audit 2024-09-12T23:43:52.034621+0000 mon.a (mon.0) 291 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:53.345 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:53 smithi181 bash[67587]: audit 2024-09-12T23:43:52.046288+0000 mon.a (mon.0) 292 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:53.345 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:53 smithi181 bash[67587]: cephadm 2024-09-12T23:43:52.048283+0000 mgr.y (mgr.24661) 102 : cephadm [INF] Reconfiguring osd.6 (monmap changed)... 2024-09-12T23:43:53.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:53 smithi181 bash[67587]: audit 2024-09-12T23:43:52.048884+0000 mon.a (mon.0) 293 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-12T23:43:53.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:53 smithi181 bash[67587]: audit 2024-09-12T23:43:52.050455+0000 mon.a (mon.0) 294 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:43:53.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:53 smithi181 bash[67587]: cephadm 2024-09-12T23:43:52.055177+0000 mgr.y (mgr.24661) 103 : cephadm [INF] Reconfiguring daemon osd.6 on smithi181 2024-09-12T23:43:53.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:53 smithi181 bash[67587]: cluster 2024-09-12T23:43:52.338227+0000 mon.a (mon.0) 295 : cluster [DBG] osdmap e379: 8 total, 8 up, 8 in 2024-09-12T23:43:53.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:53 smithi181 bash[67587]: cluster 2024-09-12T23:43:52.682567+0000 mgr.y (mgr.24661) 104 : cluster [DBG] pgmap v143: 121 pgs: 121 active+clean; 231 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.2 MiB/s wr, 13 op/s 2024-09-12T23:43:53.349 INFO:tasks.rados.rados.0.smithi136.stdout:1620: writing smithi13667081-38 from 0 to 499712 tid 1 2024-09-12T23:43:53.352 INFO:tasks.rados.rados.0.smithi136.stdout:1620: writing smithi13667081-38 from 499712 to 917504 tid 2 2024-09-12T23:43:53.356 INFO:tasks.rados.rados.0.smithi136.stdout:1620: writing smithi13667081-38 from 917504 to 1466368 tid 3 2024-09-12T23:43:53.358 INFO:tasks.rados.rados.0.smithi136.stdout:1620: writing smithi13667081-38 from 1466368 to 1703936 tid 4 2024-09-12T23:43:53.358 INFO:tasks.rados.rados.0.smithi136.stdout:1621: rollback oid 50 current snap is 156 2024-09-12T23:43:53.358 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 50 to 142 2024-09-12T23:43:53.358 INFO:tasks.rados.rados.0.smithi136.stdout:1622: setattr oid 41 current snap is 156 2024-09-12T23:43:53.359 INFO:tasks.rados.rados.0.smithi136.stdout:1620: finishing write tid 1 to smithi13667081-38 2024-09-12T23:43:53.359 INFO:tasks.rados.rados.0.smithi136.stdout:1623: rollback oid 7 current snap is 156 2024-09-12T23:43:53.360 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 7 to 142 2024-09-12T23:43:53.360 INFO:tasks.rados.rados.0.smithi136.stdout:1624: snap_remove snap 154 2024-09-12T23:43:53.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:53 smithi136 bash[75164]: audit 2024-09-12T23:43:52.034621+0000 mon.a (mon.0) 291 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:53.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:53 smithi136 bash[75164]: audit 2024-09-12T23:43:52.046288+0000 mon.a (mon.0) 292 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:53.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:53 smithi136 bash[75164]: cephadm 2024-09-12T23:43:52.048283+0000 mgr.y (mgr.24661) 102 : cephadm [INF] Reconfiguring osd.6 (monmap changed)... 2024-09-12T23:43:53.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:53 smithi136 bash[75164]: audit 2024-09-12T23:43:52.048884+0000 mon.a (mon.0) 293 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-12T23:43:53.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:53 smithi136 bash[75164]: audit 2024-09-12T23:43:52.050455+0000 mon.a (mon.0) 294 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:43:53.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:53 smithi136 bash[75164]: cephadm 2024-09-12T23:43:52.055177+0000 mgr.y (mgr.24661) 103 : cephadm [INF] Reconfiguring daemon osd.6 on smithi181 2024-09-12T23:43:53.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:53 smithi136 bash[75164]: cluster 2024-09-12T23:43:52.338227+0000 mon.a (mon.0) 295 : cluster [DBG] osdmap e379: 8 total, 8 up, 8 in 2024-09-12T23:43:53.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:53 smithi136 bash[75164]: cluster 2024-09-12T23:43:52.682567+0000 mgr.y (mgr.24661) 104 : cluster [DBG] pgmap v143: 121 pgs: 121 active+clean; 231 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.2 MiB/s wr, 13 op/s 2024-09-12T23:43:53.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:53 smithi136 bash[76371]: audit 2024-09-12T23:43:52.034621+0000 mon.a (mon.0) 291 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:53.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:53 smithi136 bash[76371]: audit 2024-09-12T23:43:52.046288+0000 mon.a (mon.0) 292 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:53.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:53 smithi136 bash[76371]: cephadm 2024-09-12T23:43:52.048283+0000 mgr.y (mgr.24661) 102 : cephadm [INF] Reconfiguring osd.6 (monmap changed)... 2024-09-12T23:43:53.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:53 smithi136 bash[76371]: audit 2024-09-12T23:43:52.048884+0000 mon.a (mon.0) 293 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-12T23:43:53.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:53 smithi136 bash[76371]: audit 2024-09-12T23:43:52.050455+0000 mon.a (mon.0) 294 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:43:53.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:53 smithi136 bash[76371]: cephadm 2024-09-12T23:43:52.055177+0000 mgr.y (mgr.24661) 103 : cephadm [INF] Reconfiguring daemon osd.6 on smithi181 2024-09-12T23:43:53.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:53 smithi136 bash[76371]: cluster 2024-09-12T23:43:52.338227+0000 mon.a (mon.0) 295 : cluster [DBG] osdmap e379: 8 total, 8 up, 8 in 2024-09-12T23:43:53.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:53 smithi136 bash[76371]: cluster 2024-09-12T23:43:52.682567+0000 mgr.y (mgr.24661) 104 : cluster [DBG] pgmap v143: 121 pgs: 121 active+clean; 231 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.2 MiB/s wr, 13 op/s 2024-09-12T23:43:53.715 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:43:53 smithi181 bash[60471]: ts=2024-09-12T23:43:53.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi136\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:43:54.356 INFO:tasks.rados.rados.0.smithi136.stdout:1624: done (4 left) 2024-09-12T23:43:54.356 INFO:tasks.rados.rados.0.smithi136.stdout:1625: rollback oid 14 current snap is 156 2024-09-12T23:43:54.356 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 14 to 147 2024-09-12T23:43:54.356 INFO:tasks.rados.rados.0.smithi136.stdout:1620: finishing write tid 2 to smithi13667081-38 2024-09-12T23:43:54.356 INFO:tasks.rados.rados.0.smithi136.stdout:1623: finishing rollback tid 1 to smithi13667081-7 2024-09-12T23:43:54.356 INFO:tasks.rados.rados.0.smithi136.stdout:1623: finishing rollback tid 2 to smithi13667081-7 2024-09-12T23:43:54.356 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 7 v 888 (ObjNum 350 snap 95 seq_num 350) dirty exists 2024-09-12T23:43:54.356 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 41 v 798 (ObjNum 417 snap 120 seq_num 417) dirty exists 2024-09-12T23:43:54.356 INFO:tasks.rados.rados.0.smithi136.stdout:1620: finishing write tid 3 to smithi13667081-38 2024-09-12T23:43:54.357 INFO:tasks.rados.rados.0.smithi136.stdout:1620: finishing write tid 4 to smithi13667081-38 2024-09-12T23:43:54.357 INFO:tasks.rados.rados.0.smithi136.stdout:1620: finishing write tid 5 to smithi13667081-38 2024-09-12T23:43:54.357 INFO:tasks.rados.rados.0.smithi136.stdout:1621: finishing rollback tid 0 to smithi13667081-50 2024-09-12T23:43:54.357 INFO:tasks.rados.rados.0.smithi136.stdout:1622: done (4 left) 2024-09-12T23:43:54.357 INFO:tasks.rados.rados.0.smithi136.stdout:1623: done (3 left) 2024-09-12T23:43:54.357 INFO:tasks.rados.rados.0.smithi136.stdout:1626: delete oid 3 current snap is 156 2024-09-12T23:43:54.357 INFO:tasks.rados.rados.0.smithi136.stdout:1621: finishing rollback tid 1 to smithi13667081-50 2024-09-12T23:43:54.357 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 50 v 778 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-12T23:43:54.357 INFO:tasks.rados.rados.0.smithi136.stdout:1620: finishing write tid 6 to smithi13667081-38 2024-09-12T23:43:54.357 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 38 v 776 (ObjNum 531 snap 156 seq_num 531) dirty exists 2024-09-12T23:43:54.357 INFO:tasks.rados.rados.0.smithi136.stdout:1620: left oid 38 (ObjNum 531 snap 156 seq_num 531) 2024-09-12T23:43:54.377 INFO:tasks.rados.rados.0.smithi136.stdout:1620: done (3 left) 2024-09-12T23:43:54.377 INFO:tasks.rados.rados.0.smithi136.stdout:1621: done (2 left) 2024-09-12T23:43:54.377 INFO:tasks.rados.rados.0.smithi136.stdout:1626: done (1 left) 2024-09-12T23:43:54.377 INFO:tasks.rados.rados.0.smithi136.stdout:1627: snap_remove snap 150 2024-09-12T23:43:54.594 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:43:54 smithi181 bash[60471]: ts=2024-09-12T23:43:54.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:43:54.594 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:43:54 smithi181 bash[60471]: ts=2024-09-12T23:43:54.456Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:43:54.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:54 smithi181 bash[67587]: cluster 2024-09-12T23:43:53.348061+0000 mon.a (mon.0) 296 : cluster [DBG] osdmap e380: 8 total, 8 up, 8 in 2024-09-12T23:43:54.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:54 smithi181 bash[67587]: audit 2024-09-12T23:43:53.743234+0000 mon.a (mon.0) 297 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:43:54.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:54 smithi181 bash[67587]: audit 2024-09-12T23:43:54.226907+0000 mon.a (mon.0) 298 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:54.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:54 smithi181 bash[67587]: audit 2024-09-12T23:43:54.237137+0000 mon.a (mon.0) 299 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:54.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:54 smithi181 bash[67587]: audit 2024-09-12T23:43:54.239614+0000 mon.a (mon.0) 300 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-12T23:43:54.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:54 smithi181 bash[67587]: audit 2024-09-12T23:43:54.241471+0000 mon.a (mon.0) 301 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:43:54.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:54 smithi136 bash[76371]: cluster 2024-09-12T23:43:53.348061+0000 mon.a (mon.0) 296 : cluster [DBG] osdmap e380: 8 total, 8 up, 8 in 2024-09-12T23:43:54.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:54 smithi136 bash[76371]: audit 2024-09-12T23:43:53.743234+0000 mon.a (mon.0) 297 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:43:54.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:54 smithi136 bash[76371]: audit 2024-09-12T23:43:54.226907+0000 mon.a (mon.0) 298 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:54.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:54 smithi136 bash[76371]: audit 2024-09-12T23:43:54.237137+0000 mon.a (mon.0) 299 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:54.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:54 smithi136 bash[76371]: audit 2024-09-12T23:43:54.239614+0000 mon.a (mon.0) 300 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-12T23:43:54.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:54 smithi136 bash[76371]: audit 2024-09-12T23:43:54.241471+0000 mon.a (mon.0) 301 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:43:54.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:54 smithi136 bash[75164]: cluster 2024-09-12T23:43:53.348061+0000 mon.a (mon.0) 296 : cluster [DBG] osdmap e380: 8 total, 8 up, 8 in 2024-09-12T23:43:54.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:54 smithi136 bash[75164]: audit 2024-09-12T23:43:53.743234+0000 mon.a (mon.0) 297 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:43:54.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:54 smithi136 bash[75164]: audit 2024-09-12T23:43:54.226907+0000 mon.a (mon.0) 298 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:54.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:54 smithi136 bash[75164]: audit 2024-09-12T23:43:54.237137+0000 mon.a (mon.0) 299 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:54.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:54 smithi136 bash[75164]: audit 2024-09-12T23:43:54.239614+0000 mon.a (mon.0) 300 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-12T23:43:54.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:54 smithi136 bash[75164]: audit 2024-09-12T23:43:54.241471+0000 mon.a (mon.0) 301 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:43:55.367 INFO:tasks.rados.rados.0.smithi136.stdout:1627: done (1 left) 2024-09-12T23:43:55.367 INFO:tasks.rados.rados.0.smithi136.stdout:1628: read oid 23 snap 151 2024-09-12T23:43:55.367 INFO:tasks.rados.rados.0.smithi136.stdout:1628: expect (ObjNum 481 snap 142 seq_num 481) 2024-09-12T23:43:55.367 INFO:tasks.rados.rados.0.smithi136.stdout:1629: read oid 46 snap -1 2024-09-12T23:43:55.367 INFO:tasks.rados.rados.0.smithi136.stdout:1629: expect (ObjNum 350 snap 95 seq_num 350) 2024-09-12T23:43:55.367 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 4 current snap is 156 2024-09-12T23:43:55.368 INFO:tasks.rados.rados.0.smithi136.stdout:1625: finishing rollback tid 1 to smithi13667081-14 2024-09-12T23:43:55.368 INFO:tasks.rados.rados.0.smithi136.stdout:1625: finishing rollback tid 2 to smithi13667081-14 2024-09-12T23:43:55.368 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 14 v 987 (ObjNum 398 snap 116 seq_num 398) dirty exists 2024-09-12T23:43:55.368 INFO:tasks.rados.rados.0.smithi136.stdout:1630: seq_num 533 ranges {3072000=532480,3604480=630784,4235264=516096,4751360=57344} 2024-09-12T23:43:55.394 INFO:tasks.rados.rados.0.smithi136.stdout:1630: writing smithi13667081-4 from 3072000 to 3604480 tid 1 2024-09-12T23:43:55.400 INFO:tasks.rados.rados.0.smithi136.stdout:1630: writing smithi13667081-4 from 3604480 to 4235264 tid 2 2024-09-12T23:43:55.404 INFO:tasks.rados.rados.0.smithi136.stdout:1630: writing smithi13667081-4 from 4235264 to 4751360 tid 3 2024-09-12T23:43:55.405 INFO:tasks.rados.rados.0.smithi136.stdout:1630: writing smithi13667081-4 from 4751360 to 4808704 tid 4 2024-09-12T23:43:55.405 INFO:tasks.rados.rados.0.smithi136.stdout:1625: done (3 left) 2024-09-12T23:43:55.405 INFO:tasks.rados.rados.0.smithi136.stdout:1631: setattr oid 43 current snap is 156 2024-09-12T23:43:55.409 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 16 current snap is 156 2024-09-12T23:43:55.409 INFO:tasks.rados.rados.0.smithi136.stdout:1632: seq_num 534 ranges {3792896=548864,4341760=516096,4857856=688128} 2024-09-12T23:43:55.434 INFO:tasks.rados.rados.0.smithi136.stdout:1632: writing smithi13667081-16 from 3792896 to 4341760 tid 1 2024-09-12T23:43:55.438 INFO:tasks.rados.rados.0.smithi136.stdout:1632: writing smithi13667081-16 from 4341760 to 4857856 tid 2 2024-09-12T23:43:55.442 INFO:tasks.rados.rados.0.smithi136.stdout:1632: writing smithi13667081-16 from 4857856 to 5545984 tid 3 2024-09-12T23:43:55.443 INFO:tasks.rados.rados.0.smithi136.stdout:1633: snap_remove snap 142 2024-09-12T23:43:55.615 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:55 smithi181 bash[67587]: cephadm 2024-09-12T23:43:54.239028+0000 mgr.y (mgr.24661) 105 : cephadm [INF] Reconfiguring osd.7 (monmap changed)... 2024-09-12T23:43:55.616 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:55 smithi181 bash[67587]: cephadm 2024-09-12T23:43:54.246398+0000 mgr.y (mgr.24661) 106 : cephadm [INF] Reconfiguring daemon osd.7 on smithi181 2024-09-12T23:43:55.616 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:55 smithi181 bash[67587]: cluster 2024-09-12T23:43:54.358596+0000 mon.a (mon.0) 302 : cluster [DBG] osdmap e381: 8 total, 8 up, 8 in 2024-09-12T23:43:55.616 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:55 smithi181 bash[67587]: cluster 2024-09-12T23:43:54.684053+0000 mgr.y (mgr.24661) 107 : cluster [DBG] pgmap v146: 121 pgs: 121 active+clean; 231 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s rd, 3.7 MiB/s wr, 16 op/s 2024-09-12T23:43:55.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:55 smithi136 bash[75164]: cephadm 2024-09-12T23:43:54.239028+0000 mgr.y (mgr.24661) 105 : cephadm [INF] Reconfiguring osd.7 (monmap changed)... 2024-09-12T23:43:55.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:55 smithi136 bash[75164]: cephadm 2024-09-12T23:43:54.246398+0000 mgr.y (mgr.24661) 106 : cephadm [INF] Reconfiguring daemon osd.7 on smithi181 2024-09-12T23:43:55.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:55 smithi136 bash[75164]: cluster 2024-09-12T23:43:54.358596+0000 mon.a (mon.0) 302 : cluster [DBG] osdmap e381: 8 total, 8 up, 8 in 2024-09-12T23:43:55.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:55 smithi136 bash[75164]: cluster 2024-09-12T23:43:54.684053+0000 mgr.y (mgr.24661) 107 : cluster [DBG] pgmap v146: 121 pgs: 121 active+clean; 231 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s rd, 3.7 MiB/s wr, 16 op/s 2024-09-12T23:43:55.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:55 smithi136 bash[76371]: cephadm 2024-09-12T23:43:54.239028+0000 mgr.y (mgr.24661) 105 : cephadm [INF] Reconfiguring osd.7 (monmap changed)... 2024-09-12T23:43:55.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:55 smithi136 bash[76371]: cephadm 2024-09-12T23:43:54.246398+0000 mgr.y (mgr.24661) 106 : cephadm [INF] Reconfiguring daemon osd.7 on smithi181 2024-09-12T23:43:55.762 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:55 smithi136 bash[76371]: cluster 2024-09-12T23:43:54.358596+0000 mon.a (mon.0) 302 : cluster [DBG] osdmap e381: 8 total, 8 up, 8 in 2024-09-12T23:43:55.762 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:55 smithi136 bash[76371]: cluster 2024-09-12T23:43:54.684053+0000 mgr.y (mgr.24661) 107 : cluster [DBG] pgmap v146: 121 pgs: 121 active+clean; 231 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s rd, 3.7 MiB/s wr, 16 op/s 2024-09-12T23:43:56.310 INFO:tasks.rados.rados.0.smithi136.stdout:1633: done (5 left) 2024-09-12T23:43:56.311 INFO:tasks.rados.rados.0.smithi136.stdout:1634: rollback oid 18 current snap is 156 2024-09-12T23:43:56.311 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 18 to 151 2024-09-12T23:43:56.311 INFO:tasks.rados.rados.0.smithi136.stdout:1630: finishing write tid 1 to smithi13667081-4 2024-09-12T23:43:56.311 INFO:tasks.rados.rados.0.smithi136.stdout:1629: expect (ObjNum 350 snap 95 seq_num 350) 2024-09-12T23:43:56.516 INFO:tasks.rados.rados.0.smithi136.stdout:1630: finishing write tid 2 to smithi13667081-4 2024-09-12T23:43:56.516 INFO:tasks.rados.rados.0.smithi136.stdout:1630: finishing write tid 3 to smithi13667081-4 2024-09-12T23:43:56.516 INFO:tasks.rados.rados.0.smithi136.stdout:1630: finishing write tid 4 to smithi13667081-4 2024-09-12T23:43:56.516 INFO:tasks.rados.rados.0.smithi136.stdout:1630: finishing write tid 5 to smithi13667081-4 2024-09-12T23:43:56.516 INFO:tasks.rados.rados.0.smithi136.stdout:1630: finishing write tid 6 to smithi13667081-4 2024-09-12T23:43:56.516 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 4 v 750 (ObjNum 533 snap 156 seq_num 533) dirty exists 2024-09-12T23:43:56.516 INFO:tasks.rados.rados.0.smithi136.stdout:1630: left oid 4 (ObjNum 533 snap 156 seq_num 533) 2024-09-12T23:43:56.516 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 43 v 991 (ObjNum 507 snap 149 seq_num 507) dirty exists 2024-09-12T23:43:56.516 INFO:tasks.rados.rados.0.smithi136.stdout:1632: finishing write tid 1 to smithi13667081-16 2024-09-12T23:43:56.517 INFO:tasks.rados.rados.0.smithi136.stdout:1632: finishing write tid 2 to smithi13667081-16 2024-09-12T23:43:56.517 INFO:tasks.rados.rados.0.smithi136.stdout:1632: finishing write tid 3 to smithi13667081-16 2024-09-12T23:43:56.517 INFO:tasks.rados.rados.0.smithi136.stdout:1632: finishing write tid 4 to smithi13667081-16 2024-09-12T23:43:56.517 INFO:tasks.rados.rados.0.smithi136.stdout:1632: finishing write tid 5 to smithi13667081-16 2024-09-12T23:43:56.517 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 16 v 996 (ObjNum 534 snap 156 seq_num 534) dirty exists 2024-09-12T23:43:56.517 INFO:tasks.rados.rados.0.smithi136.stdout:1632: left oid 16 (ObjNum 534 snap 156 seq_num 534) 2024-09-12T23:43:56.517 INFO:tasks.rados.rados.0.smithi136.stdout:1628: expect (ObjNum 481 snap 142 seq_num 481) 2024-09-12T23:43:56.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:56 smithi136 bash[75164]: cluster 2024-09-12T23:43:55.369545+0000 mon.a (mon.0) 303 : cluster [DBG] osdmap e382: 8 total, 8 up, 8 in 2024-09-12T23:43:56.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:56 smithi136 bash[75164]: cluster 2024-09-12T23:43:56.314145+0000 mon.a (mon.0) 304 : cluster [DBG] osdmap e383: 8 total, 8 up, 8 in 2024-09-12T23:43:56.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:56 smithi136 bash[76371]: cluster 2024-09-12T23:43:55.369545+0000 mon.a (mon.0) 303 : cluster [DBG] osdmap e382: 8 total, 8 up, 8 in 2024-09-12T23:43:56.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:56 smithi136 bash[76371]: cluster 2024-09-12T23:43:56.314145+0000 mon.a (mon.0) 304 : cluster [DBG] osdmap e383: 8 total, 8 up, 8 in 2024-09-12T23:43:56.815 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:56 smithi181 bash[67587]: cluster 2024-09-12T23:43:55.369545+0000 mon.a (mon.0) 303 : cluster [DBG] osdmap e382: 8 total, 8 up, 8 in 2024-09-12T23:43:56.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:56 smithi181 bash[67587]: cluster 2024-09-12T23:43:56.314145+0000 mon.a (mon.0) 304 : cluster [DBG] osdmap e383: 8 total, 8 up, 8 in 2024-09-12T23:43:57.493 INFO:tasks.rados.rados.0.smithi136.stdout:1634: finishing rollback tid 0 to smithi13667081-18 2024-09-12T23:43:57.494 INFO:tasks.rados.rados.0.smithi136.stdout:1634: finishing rollback tid 1 to smithi13667081-18 2024-09-12T23:43:57.494 INFO:tasks.rados.rados.0.smithi136.stdout:1634: finishing rollback tid 2 to smithi13667081-18 2024-09-12T23:43:57.494 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 18 v 891 (ObjNum 488 snap 143 seq_num 488) dirty exists 2024-09-12T23:43:57.494 INFO:tasks.rados.rados.0.smithi136.stdout:1628: done (5 left) 2024-09-12T23:43:57.495 INFO:tasks.rados.rados.0.smithi136.stdout:1629: done (4 left) 2024-09-12T23:43:57.495 INFO:tasks.rados.rados.0.smithi136.stdout:1630: done (3 left) 2024-09-12T23:43:57.495 INFO:tasks.rados.rados.0.smithi136.stdout:1631: done (2 left) 2024-09-12T23:43:57.495 INFO:tasks.rados.rados.0.smithi136.stdout:1632: done (1 left) 2024-09-12T23:43:57.495 INFO:tasks.rados.rados.0.smithi136.stdout:1634: done (0 left) 2024-09-12T23:43:57.495 INFO:tasks.rados.rados.0.smithi136.stdout:1635: read oid 7 snap -1 2024-09-12T23:43:57.495 INFO:tasks.rados.rados.0.smithi136.stdout:1635: expect (ObjNum 350 snap 95 seq_num 350) 2024-09-12T23:43:57.495 INFO:tasks.rados.rados.0.smithi136.stdout:1636: setattr oid 23 current snap is 156 2024-09-12T23:43:57.496 INFO:tasks.rados.rados.0.smithi136.stdout:1637: copy_from oid 34 from oid 27 current snap is 156 2024-09-12T23:43:57.496 INFO:tasks.rados.rados.0.smithi136.stdout:1638: read oid 10 snap -1 2024-09-12T23:43:57.496 INFO:tasks.rados.rados.0.smithi136.stdout:1638: expect (ObjNum 514 snap 150 seq_num 514) 2024-09-12T23:43:57.496 INFO:tasks.rados.rados.0.smithi136.stdout:1639: read oid 42 snap -1 2024-09-12T23:43:57.497 INFO:tasks.rados.rados.0.smithi136.stdout:1639: expect deleted 2024-09-12T23:43:57.497 INFO:tasks.rados.rados.0.smithi136.stdout:1640: rollback oid 1 current snap is 156 2024-09-12T23:43:57.497 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 1 to 147 2024-09-12T23:43:57.497 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 32 current snap is 156 2024-09-12T23:43:57.497 INFO:tasks.rados.rados.0.smithi136.stdout:1641: seq_num 535 ranges {3342336=565248,3907584=802816,4710400=401408} 2024-09-12T23:43:57.526 INFO:tasks.rados.rados.0.smithi136.stdout:1641: writing smithi13667081-32 from 3342336 to 3907584 tid 1 2024-09-12T23:43:57.532 INFO:tasks.rados.rados.0.smithi136.stdout:1641: writing smithi13667081-32 from 3907584 to 4710400 tid 2 2024-09-12T23:43:57.535 INFO:tasks.rados.rados.0.smithi136.stdout:1641: writing smithi13667081-32 from 4710400 to 5111808 tid 3 2024-09-12T23:43:57.535 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 3 current snap is 156 2024-09-12T23:43:57.536 INFO:tasks.rados.rados.0.smithi136.stdout:1642: seq_num 536 ranges {0=581632,581632=688128,1269760=516096} 2024-09-12T23:43:57.540 INFO:tasks.rados.rados.0.smithi136.stdout:1642: writing smithi13667081-3 from 0 to 581632 tid 1 2024-09-12T23:43:57.546 INFO:tasks.rados.rados.0.smithi136.stdout:1642: writing smithi13667081-3 from 581632 to 1269760 tid 2 2024-09-12T23:43:57.550 INFO:tasks.rados.rados.0.smithi136.stdout:1642: writing smithi13667081-3 from 1269760 to 1785856 tid 3 2024-09-12T23:43:57.550 INFO:tasks.rados.rados.0.smithi136.stdout:1643: copy_from oid 13 from oid 21 current snap is 156 2024-09-12T23:43:57.551 INFO:tasks.rados.rados.0.smithi136.stdout:1640: finishing rollback tid 0 to smithi13667081-1 2024-09-12T23:43:57.551 INFO:tasks.rados.rados.0.smithi136.stdout:1640: finishing rollback tid 1 to smithi13667081-1 2024-09-12T23:43:57.551 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 1 v 806 (ObjNum 129 snap 0 seq_num 4243538608) dirty dne 2024-09-12T23:43:57.551 INFO:tasks.rados.rados.0.smithi136.stdout:1637: finishing copy_from racing read to smithi13667081-34 2024-09-12T23:43:57.551 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 23 v 1216 (ObjNum 481 snap 142 seq_num 481) dirty exists 2024-09-12T23:43:57.551 INFO:tasks.rados.rados.0.smithi136.stdout:1637: finishing copy_from to smithi13667081-34 2024-09-12T23:43:57.551 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 34 v 949 (ObjNum 446 snap 135 seq_num 446) dirty exists 2024-09-12T23:43:57.551 INFO:tasks.rados.rados.0.smithi136.stdout:1636: done (8 left) 2024-09-12T23:43:57.551 INFO:tasks.rados.rados.0.smithi136.stdout:1637: done (7 left) 2024-09-12T23:43:57.551 INFO:tasks.rados.rados.0.smithi136.stdout:1639: done (6 left) 2024-09-12T23:43:57.551 INFO:tasks.rados.rados.0.smithi136.stdout:1640: done (5 left) 2024-09-12T23:43:57.551 INFO:tasks.rados.rados.0.smithi136.stdout:1644: snap_remove snap 147 2024-09-12T23:43:57.756 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:57 smithi181 bash[67587]: audit 2024-09-12T23:43:56.448183+0000 mon.a (mon.0) 305 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:57.756 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:57 smithi181 bash[67587]: audit 2024-09-12T23:43:56.460078+0000 mon.a (mon.0) 306 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:57.756 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:57 smithi181 bash[67587]: cephadm 2024-09-12T23:43:56.462052+0000 mgr.y (mgr.24661) 108 : cephadm [INF] Reconfiguring mds.b (monmap changed)... 2024-09-12T23:43:57.756 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:57 smithi181 bash[67587]: audit 2024-09-12T23:43:56.462595+0000 mon.a (mon.0) 307 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-12T23:43:57.756 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:57 smithi181 bash[67587]: audit 2024-09-12T23:43:56.464169+0000 mon.a (mon.0) 308 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:43:57.756 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:57 smithi181 bash[67587]: cephadm 2024-09-12T23:43:56.466056+0000 mgr.y (mgr.24661) 109 : cephadm [INF] Reconfiguring daemon mds.b on smithi181 2024-09-12T23:43:57.756 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:57 smithi181 bash[67587]: cluster 2024-09-12T23:43:56.684778+0000 mgr.y (mgr.24661) 110 : cluster [DBG] pgmap v149: 121 pgs: 121 active+clean; 231 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s rd, 3.7 MiB/s wr, 16 op/s 2024-09-12T23:43:57.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:57 smithi136 bash[75164]: audit 2024-09-12T23:43:56.448183+0000 mon.a (mon.0) 305 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:57.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:57 smithi136 bash[75164]: audit 2024-09-12T23:43:56.460078+0000 mon.a (mon.0) 306 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:57.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:57 smithi136 bash[75164]: cephadm 2024-09-12T23:43:56.462052+0000 mgr.y (mgr.24661) 108 : cephadm [INF] Reconfiguring mds.b (monmap changed)... 2024-09-12T23:43:57.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:57 smithi136 bash[75164]: audit 2024-09-12T23:43:56.462595+0000 mon.a (mon.0) 307 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-12T23:43:57.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:57 smithi136 bash[75164]: audit 2024-09-12T23:43:56.464169+0000 mon.a (mon.0) 308 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:43:57.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:57 smithi136 bash[75164]: cephadm 2024-09-12T23:43:56.466056+0000 mgr.y (mgr.24661) 109 : cephadm [INF] Reconfiguring daemon mds.b on smithi181 2024-09-12T23:43:57.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:57 smithi136 bash[75164]: cluster 2024-09-12T23:43:56.684778+0000 mgr.y (mgr.24661) 110 : cluster [DBG] pgmap v149: 121 pgs: 121 active+clean; 231 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s rd, 3.7 MiB/s wr, 16 op/s 2024-09-12T23:43:57.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:57 smithi136 bash[76371]: audit 2024-09-12T23:43:56.448183+0000 mon.a (mon.0) 305 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:57.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:57 smithi136 bash[76371]: audit 2024-09-12T23:43:56.460078+0000 mon.a (mon.0) 306 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:57.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:57 smithi136 bash[76371]: cephadm 2024-09-12T23:43:56.462052+0000 mgr.y (mgr.24661) 108 : cephadm [INF] Reconfiguring mds.b (monmap changed)... 2024-09-12T23:43:57.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:57 smithi136 bash[76371]: audit 2024-09-12T23:43:56.462595+0000 mon.a (mon.0) 307 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-12T23:43:57.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:57 smithi136 bash[76371]: audit 2024-09-12T23:43:56.464169+0000 mon.a (mon.0) 308 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:43:57.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:57 smithi136 bash[76371]: cephadm 2024-09-12T23:43:56.466056+0000 mgr.y (mgr.24661) 109 : cephadm [INF] Reconfiguring daemon mds.b on smithi181 2024-09-12T23:43:57.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:57 smithi136 bash[76371]: cluster 2024-09-12T23:43:56.684778+0000 mgr.y (mgr.24661) 110 : cluster [DBG] pgmap v149: 121 pgs: 121 active+clean; 231 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s rd, 3.7 MiB/s wr, 16 op/s 2024-09-12T23:43:58.474 INFO:tasks.rados.rados.0.smithi136.stdout:1644: done (5 left) 2024-09-12T23:43:58.474 INFO:tasks.rados.rados.0.smithi136.stdout:1645: snap_create 2024-09-12T23:43:58.474 INFO:tasks.rados.rados.0.smithi136.stdout:1638: expect (ObjNum 514 snap 150 seq_num 514) 2024-09-12T23:43:58.545 INFO:tasks.rados.rados.0.smithi136.stdout:1635: expect (ObjNum 350 snap 95 seq_num 350) 2024-09-12T23:43:58.707 INFO:tasks.rados.rados.0.smithi136.stdout:1641: finishing write tid 1 to smithi13667081-32 2024-09-12T23:43:58.707 INFO:tasks.rados.rados.0.smithi136.stdout:1641: finishing write tid 2 to smithi13667081-32 2024-09-12T23:43:58.707 INFO:tasks.rados.rados.0.smithi136.stdout:1641: finishing write tid 3 to smithi13667081-32 2024-09-12T23:43:58.707 INFO:tasks.rados.rados.0.smithi136.stdout:1641: finishing write tid 4 to smithi13667081-32 2024-09-12T23:43:58.707 INFO:tasks.rados.rados.0.smithi136.stdout:1641: finishing write tid 5 to smithi13667081-32 2024-09-12T23:43:58.707 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 32 v 754 (ObjNum 535 snap 156 seq_num 535) dirty exists 2024-09-12T23:43:58.707 INFO:tasks.rados.rados.0.smithi136.stdout:1641: left oid 32 (ObjNum 535 snap 156 seq_num 535) 2024-09-12T23:43:58.707 INFO:tasks.rados.rados.0.smithi136.stdout:1642: finishing write tid 1 to smithi13667081-3 2024-09-12T23:43:58.708 INFO:tasks.rados.rados.0.smithi136.stdout:1642: finishing write tid 2 to smithi13667081-3 2024-09-12T23:43:58.708 INFO:tasks.rados.rados.0.smithi136.stdout:1642: finishing write tid 3 to smithi13667081-3 2024-09-12T23:43:58.708 INFO:tasks.rados.rados.0.smithi136.stdout:1642: finishing write tid 4 to smithi13667081-3 2024-09-12T23:43:58.708 INFO:tasks.rados.rados.0.smithi136.stdout:1642: finishing write tid 5 to smithi13667081-3 2024-09-12T23:43:58.708 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 3 v 893 (ObjNum 536 snap 156 seq_num 536) dirty exists 2024-09-12T23:43:58.708 INFO:tasks.rados.rados.0.smithi136.stdout:1642: left oid 3 (ObjNum 536 snap 156 seq_num 536) 2024-09-12T23:43:58.708 INFO:tasks.rados.rados.0.smithi136.stdout:1643: finishing copy_from racing read to smithi13667081-13 2024-09-12T23:43:58.708 INFO:tasks.rados.rados.0.smithi136.stdout:1643: finishing copy_from to smithi13667081-13 2024-09-12T23:43:58.708 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 13 v 804 (ObjNum 441 snap 132 seq_num 441) dirty exists 2024-09-12T23:43:59.094 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:43:58 smithi181 bash[60471]: ts=2024-09-12T23:43:58.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:43:59.475 INFO:tasks.rados.rados.0.smithi136.stdout:1635: done (5 left) 2024-09-12T23:43:59.475 INFO:tasks.rados.rados.0.smithi136.stdout:1638: done (4 left) 2024-09-12T23:43:59.475 INFO:tasks.rados.rados.0.smithi136.stdout:1641: done (3 left) 2024-09-12T23:43:59.475 INFO:tasks.rados.rados.0.smithi136.stdout:1642: done (2 left) 2024-09-12T23:43:59.475 INFO:tasks.rados.rados.0.smithi136.stdout:1643: done (1 left) 2024-09-12T23:43:59.475 INFO:tasks.rados.rados.0.smithi136.stdout:1645: done (0 left) 2024-09-12T23:43:59.475 INFO:tasks.rados.rados.0.smithi136.stdout:1646: delete oid 39 current snap is 157 2024-09-12T23:43:59.485 INFO:tasks.rados.rados.0.smithi136.stdout:1646: done (0 left) 2024-09-12T23:43:59.486 INFO:tasks.rados.rados.0.smithi136.stdout:1647: snap_remove snap 153 2024-09-12T23:43:59.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:43:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:43:59] "GET /metrics HTTP/1.1" 200 36197 "" "Prometheus/2.43.0" 2024-09-12T23:43:59.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:59 smithi136 bash[75164]: audit 2024-09-12T23:43:58.465225+0000 mon.a (mon.0) 309 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:59.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:59 smithi136 bash[75164]: cluster 2024-09-12T23:43:58.473870+0000 mon.a (mon.0) 310 : cluster [DBG] osdmap e384: 8 total, 8 up, 8 in 2024-09-12T23:43:59.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:59 smithi136 bash[75164]: audit 2024-09-12T23:43:58.489289+0000 mon.a (mon.0) 311 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:59.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:59 smithi136 bash[75164]: audit 2024-09-12T23:43:58.559356+0000 mon.a (mon.0) 312 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:43:59.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:59 smithi136 bash[75164]: audit 2024-09-12T23:43:58.562503+0000 mon.a (mon.0) 313 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:43:59.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:59 smithi136 bash[75164]: audit 2024-09-12T23:43:58.564281+0000 mon.a (mon.0) 314 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:43:59.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:59 smithi136 bash[75164]: cephadm 2024-09-12T23:43:58.565112+0000 mgr.y (mgr.24661) 111 : cephadm [INF] Upgrade: Setting container_image for all mon 2024-09-12T23:43:59.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:59 smithi136 bash[75164]: audit 2024-09-12T23:43:58.570986+0000 mon.a (mon.0) 315 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:59.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:59 smithi136 bash[75164]: audit 2024-09-12T23:43:58.572103+0000 mon.a (mon.0) 316 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-12T23:43:59.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:59 smithi136 bash[75164]: audit 2024-09-12T23:43:58.577361+0000 mon.a (mon.0) 317 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-12T23:43:59.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:59 smithi136 bash[75164]: audit 2024-09-12T23:43:58.578676+0000 mon.a (mon.0) 318 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-12T23:43:59.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:59 smithi136 bash[75164]: audit 2024-09-12T23:43:58.585542+0000 mon.a (mon.0) 319 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-12T23:43:59.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:59 smithi136 bash[75164]: audit 2024-09-12T23:43:58.586928+0000 mon.a (mon.0) 320 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-12T23:43:59.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:59 smithi136 bash[75164]: audit 2024-09-12T23:43:58.593561+0000 mon.a (mon.0) 321 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-12T23:43:59.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:59 smithi136 bash[75164]: audit 2024-09-12T23:43:58.595757+0000 mon.a (mon.0) 322 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:43:59.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:59 smithi136 bash[75164]: cephadm 2024-09-12T23:43:58.597183+0000 mgr.y (mgr.24661) 112 : cephadm [INF] Upgrade: Setting container_image for all crash 2024-09-12T23:43:59.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:59 smithi136 bash[75164]: audit 2024-09-12T23:43:58.602905+0000 mon.a (mon.0) 323 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:59.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:59 smithi136 bash[75164]: audit 2024-09-12T23:43:58.605325+0000 mon.a (mon.0) 324 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-12T23:43:59.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:59 smithi136 bash[75164]: audit 2024-09-12T23:43:58.605602+0000 mgr.y (mgr.24661) 113 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-12T23:43:59.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:59 smithi136 bash[75164]: cephadm 2024-09-12T23:43:58.607204+0000 mgr.y (mgr.24661) 114 : cephadm [INF] Upgrade: osd.0 is safe to restart 2024-09-12T23:43:59.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:43:59 smithi136 bash[75164]: cluster 2024-09-12T23:43:58.685273+0000 mgr.y (mgr.24661) 115 : cluster [DBG] pgmap v151: 121 pgs: 121 active+clean; 231 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s rd, 3.4 MiB/s wr, 15 op/s 2024-09-12T23:43:59.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:59 smithi136 bash[76371]: audit 2024-09-12T23:43:58.465225+0000 mon.a (mon.0) 309 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:59.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:59 smithi136 bash[76371]: cluster 2024-09-12T23:43:58.473870+0000 mon.a (mon.0) 310 : cluster [DBG] osdmap e384: 8 total, 8 up, 8 in 2024-09-12T23:43:59.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:59 smithi136 bash[76371]: audit 2024-09-12T23:43:58.489289+0000 mon.a (mon.0) 311 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:59.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:59 smithi136 bash[76371]: audit 2024-09-12T23:43:58.559356+0000 mon.a (mon.0) 312 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:43:59.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:59 smithi136 bash[76371]: audit 2024-09-12T23:43:58.562503+0000 mon.a (mon.0) 313 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:43:59.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:59 smithi136 bash[76371]: audit 2024-09-12T23:43:58.564281+0000 mon.a (mon.0) 314 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:43:59.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:59 smithi136 bash[76371]: cephadm 2024-09-12T23:43:58.565112+0000 mgr.y (mgr.24661) 111 : cephadm [INF] Upgrade: Setting container_image for all mon 2024-09-12T23:43:59.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:59 smithi136 bash[76371]: audit 2024-09-12T23:43:58.570986+0000 mon.a (mon.0) 315 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:59.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:59 smithi136 bash[76371]: audit 2024-09-12T23:43:58.572103+0000 mon.a (mon.0) 316 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-12T23:43:59.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:59 smithi136 bash[76371]: audit 2024-09-12T23:43:58.577361+0000 mon.a (mon.0) 317 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-12T23:43:59.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:59 smithi136 bash[76371]: audit 2024-09-12T23:43:58.578676+0000 mon.a (mon.0) 318 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-12T23:43:59.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:59 smithi136 bash[76371]: audit 2024-09-12T23:43:58.585542+0000 mon.a (mon.0) 319 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-12T23:43:59.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:59 smithi136 bash[76371]: audit 2024-09-12T23:43:58.586928+0000 mon.a (mon.0) 320 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-12T23:43:59.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:59 smithi136 bash[76371]: audit 2024-09-12T23:43:58.593561+0000 mon.a (mon.0) 321 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-12T23:43:59.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:59 smithi136 bash[76371]: audit 2024-09-12T23:43:58.595757+0000 mon.a (mon.0) 322 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:43:59.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:59 smithi136 bash[76371]: cephadm 2024-09-12T23:43:58.597183+0000 mgr.y (mgr.24661) 112 : cephadm [INF] Upgrade: Setting container_image for all crash 2024-09-12T23:43:59.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:59 smithi136 bash[76371]: audit 2024-09-12T23:43:58.602905+0000 mon.a (mon.0) 323 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:59.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:59 smithi136 bash[76371]: audit 2024-09-12T23:43:58.605325+0000 mon.a (mon.0) 324 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-12T23:43:59.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:59 smithi136 bash[76371]: audit 2024-09-12T23:43:58.605602+0000 mgr.y (mgr.24661) 113 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-12T23:43:59.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:59 smithi136 bash[76371]: cephadm 2024-09-12T23:43:58.607204+0000 mgr.y (mgr.24661) 114 : cephadm [INF] Upgrade: osd.0 is safe to restart 2024-09-12T23:43:59.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:43:59 smithi136 bash[76371]: cluster 2024-09-12T23:43:58.685273+0000 mgr.y (mgr.24661) 115 : cluster [DBG] pgmap v151: 121 pgs: 121 active+clean; 231 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s rd, 3.4 MiB/s wr, 15 op/s 2024-09-12T23:43:59.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:59 smithi181 bash[67587]: audit 2024-09-12T23:43:58.465225+0000 mon.a (mon.0) 309 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:59.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:59 smithi181 bash[67587]: cluster 2024-09-12T23:43:58.473870+0000 mon.a (mon.0) 310 : cluster [DBG] osdmap e384: 8 total, 8 up, 8 in 2024-09-12T23:43:59.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:59 smithi181 bash[67587]: audit 2024-09-12T23:43:58.489289+0000 mon.a (mon.0) 311 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:59.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:59 smithi181 bash[67587]: audit 2024-09-12T23:43:58.559356+0000 mon.a (mon.0) 312 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:43:59.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:59 smithi181 bash[67587]: audit 2024-09-12T23:43:58.562503+0000 mon.a (mon.0) 313 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:43:59.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:59 smithi181 bash[67587]: audit 2024-09-12T23:43:58.564281+0000 mon.a (mon.0) 314 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:43:59.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:59 smithi181 bash[67587]: cephadm 2024-09-12T23:43:58.565112+0000 mgr.y (mgr.24661) 111 : cephadm [INF] Upgrade: Setting container_image for all mon 2024-09-12T23:43:59.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:59 smithi181 bash[67587]: audit 2024-09-12T23:43:58.570986+0000 mon.a (mon.0) 315 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:59.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:59 smithi181 bash[67587]: audit 2024-09-12T23:43:58.572103+0000 mon.a (mon.0) 316 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-12T23:43:59.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:59 smithi181 bash[67587]: audit 2024-09-12T23:43:58.577361+0000 mon.a (mon.0) 317 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-12T23:43:59.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:59 smithi181 bash[67587]: audit 2024-09-12T23:43:58.578676+0000 mon.a (mon.0) 318 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-12T23:43:59.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:59 smithi181 bash[67587]: audit 2024-09-12T23:43:58.585542+0000 mon.a (mon.0) 319 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-12T23:43:59.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:59 smithi181 bash[67587]: audit 2024-09-12T23:43:58.586928+0000 mon.a (mon.0) 320 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-12T23:43:59.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:59 smithi181 bash[67587]: audit 2024-09-12T23:43:58.593561+0000 mon.a (mon.0) 321 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-12T23:43:59.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:59 smithi181 bash[67587]: audit 2024-09-12T23:43:58.595757+0000 mon.a (mon.0) 322 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:43:59.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:59 smithi181 bash[67587]: cephadm 2024-09-12T23:43:58.597183+0000 mgr.y (mgr.24661) 112 : cephadm [INF] Upgrade: Setting container_image for all crash 2024-09-12T23:43:59.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:59 smithi181 bash[67587]: audit 2024-09-12T23:43:58.602905+0000 mon.a (mon.0) 323 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:43:59.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:59 smithi181 bash[67587]: audit 2024-09-12T23:43:58.605325+0000 mon.a (mon.0) 324 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-12T23:43:59.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:59 smithi181 bash[67587]: audit 2024-09-12T23:43:58.605602+0000 mgr.y (mgr.24661) 113 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-12T23:43:59.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:59 smithi181 bash[67587]: cephadm 2024-09-12T23:43:58.607204+0000 mgr.y (mgr.24661) 114 : cephadm [INF] Upgrade: osd.0 is safe to restart 2024-09-12T23:43:59.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:43:59 smithi181 bash[67587]: cluster 2024-09-12T23:43:58.685273+0000 mgr.y (mgr.24661) 115 : cluster [DBG] pgmap v151: 121 pgs: 121 active+clean; 231 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s rd, 3.4 MiB/s wr, 15 op/s 2024-09-12T23:44:00.480 INFO:tasks.rados.rados.0.smithi136.stdout:1647: done (0 left) 2024-09-12T23:44:00.480 INFO:tasks.rados.rados.0.smithi136.stdout:1648: snap_remove snap 155 2024-09-12T23:44:00.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:00 smithi136 bash[75164]: cluster 2024-09-12T23:43:59.478461+0000 mon.a (mon.0) 325 : cluster [DBG] osdmap e385: 8 total, 8 up, 8 in 2024-09-12T23:44:00.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:00 smithi136 bash[76371]: cluster 2024-09-12T23:43:59.478461+0000 mon.a (mon.0) 325 : cluster [DBG] osdmap e385: 8 total, 8 up, 8 in 2024-09-12T23:44:00.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:00 smithi181 bash[67587]: cluster 2024-09-12T23:43:59.478461+0000 mon.a (mon.0) 325 : cluster [DBG] osdmap e385: 8 total, 8 up, 8 in 2024-09-12T23:44:01.501 INFO:tasks.rados.rados.0.smithi136.stdout:1648: done (0 left) 2024-09-12T23:44:01.501 INFO:tasks.rados.rados.0.smithi136.stdout:1649: rollback oid 19 current snap is 157 2024-09-12T23:44:01.501 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 19 to 156 2024-09-12T23:44:01.501 INFO:tasks.rados.rados.0.smithi136.stdout:1650: rmattr oid 33 current snap is 157 2024-09-12T23:44:01.501 INFO:tasks.rados.rados.0.smithi136.stdout:1650: done (1 left) 2024-09-12T23:44:01.502 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 23 current snap is 157 2024-09-12T23:44:01.502 INFO:tasks.rados.rados.0.smithi136.stdout:1651: seq_num 538 ranges {11190272=614400,11804672=450560,12255232=704512,12959744=49152} 2024-09-12T23:44:01.579 INFO:tasks.rados.rados.0.smithi136.stdout:1651: writing smithi13667081-23 from 11190272 to 11804672 tid 1 2024-09-12T23:44:01.582 INFO:tasks.rados.rados.0.smithi136.stdout:1651: writing smithi13667081-23 from 11804672 to 12255232 tid 2 2024-09-12T23:44:01.587 INFO:tasks.rados.rados.0.smithi136.stdout:1651: writing smithi13667081-23 from 12255232 to 12959744 tid 3 2024-09-12T23:44:01.588 INFO:tasks.rados.rados.0.smithi136.stdout:1651: writing smithi13667081-23 from 12959744 to 13008896 tid 4 2024-09-12T23:44:01.588 INFO:tasks.rados.rados.0.smithi136.stdout:1652: rmattr oid 37 current snap is 157 2024-09-12T23:44:01.588 INFO:tasks.rados.rados.0.smithi136.stdout:1652: done (2 left) 2024-09-12T23:44:01.588 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 6 current snap is 157 2024-09-12T23:44:01.588 INFO:tasks.rados.rados.0.smithi136.stdout:1649: finishing rollback tid 0 to smithi13667081-19 2024-09-12T23:44:01.588 INFO:tasks.rados.rados.0.smithi136.stdout:1653: seq_num 539 ranges {0=630784,630784=737280,1368064=466944} 2024-09-12T23:44:01.591 INFO:tasks.rados.rados.0.smithi136.stdout:1653: writing smithi13667081-6 from 0 to 630784 tid 1 2024-09-12T23:44:01.596 INFO:tasks.rados.rados.0.smithi136.stdout:1653: writing smithi13667081-6 from 630784 to 1368064 tid 2 2024-09-12T23:44:01.599 INFO:tasks.rados.rados.0.smithi136.stdout:1653: writing smithi13667081-6 from 1368064 to 1835008 tid 3 2024-09-12T23:44:01.599 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 18 current snap is 157 2024-09-12T23:44:01.599 INFO:tasks.rados.rados.0.smithi136.stdout:1649: finishing rollback tid 1 to smithi13667081-19 2024-09-12T23:44:01.600 INFO:tasks.rados.rados.0.smithi136.stdout:1649: finishing rollback tid 2 to smithi13667081-19 2024-09-12T23:44:01.600 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 19 v 627 (ObjNum 391 snap 113 seq_num 391) dirty exists 2024-09-12T23:44:01.600 INFO:tasks.rados.rados.0.smithi136.stdout:1651: finishing write tid 1 to smithi13667081-23 2024-09-12T23:44:01.600 INFO:tasks.rados.rados.0.smithi136.stdout:1651: finishing write tid 2 to smithi13667081-23 2024-09-12T23:44:01.600 INFO:tasks.rados.rados.0.smithi136.stdout:1651: finishing write tid 3 to smithi13667081-23 2024-09-12T23:44:01.600 INFO:tasks.rados.rados.0.smithi136.stdout:1651: finishing write tid 4 to smithi13667081-23 2024-09-12T23:44:01.600 INFO:tasks.rados.rados.0.smithi136.stdout:1653: finishing write tid 1 to smithi13667081-6 2024-09-12T23:44:01.600 INFO:tasks.rados.rados.0.smithi136.stdout:1654: seq_num 540 ranges {1826816=647168,2473984=671744,3145728=532480} 2024-09-12T23:44:01.613 INFO:tasks.rados.rados.0.smithi136.stdout:1654: writing smithi13667081-18 from 1826816 to 2473984 tid 1 2024-09-12T23:44:01.617 INFO:tasks.rados.rados.0.smithi136.stdout:1654: writing smithi13667081-18 from 2473984 to 3145728 tid 2 2024-09-12T23:44:01.620 INFO:tasks.rados.rados.0.smithi136.stdout:1654: writing smithi13667081-18 from 3145728 to 3678208 tid 3 2024-09-12T23:44:01.620 INFO:tasks.rados.rados.0.smithi136.stdout:1649: done (3 left) 2024-09-12T23:44:01.620 INFO:tasks.rados.rados.0.smithi136.stdout:1655: read oid 33 snap -1 2024-09-12T23:44:01.621 INFO:tasks.rados.rados.0.smithi136.stdout:1655: expect deleted 2024-09-12T23:44:01.621 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 7 current snap is 157 2024-09-12T23:44:01.621 INFO:tasks.rados.rados.0.smithi136.stdout:1651: finishing write tid 5 to smithi13667081-23 2024-09-12T23:44:01.621 INFO:tasks.rados.rados.0.smithi136.stdout:1653: finishing write tid 2 to smithi13667081-6 2024-09-12T23:44:01.621 INFO:tasks.rados.rados.0.smithi136.stdout:1653: finishing write tid 3 to smithi13667081-6 2024-09-12T23:44:01.621 INFO:tasks.rados.rados.0.smithi136.stdout:1651: finishing write tid 6 to smithi13667081-23 2024-09-12T23:44:01.621 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 23 v 1238 (ObjNum 538 snap 157 seq_num 538) dirty exists 2024-09-12T23:44:01.621 INFO:tasks.rados.rados.0.smithi136.stdout:1651: left oid 23 (ObjNum 538 snap 157 seq_num 538) 2024-09-12T23:44:01.621 INFO:tasks.rados.rados.0.smithi136.stdout:1653: finishing write tid 4 to smithi13667081-6 2024-09-12T23:44:01.621 INFO:tasks.rados.rados.0.smithi136.stdout:1653: finishing write tid 5 to smithi13667081-6 2024-09-12T23:44:01.621 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 6 v 310 (ObjNum 539 snap 157 seq_num 539) dirty exists 2024-09-12T23:44:01.621 INFO:tasks.rados.rados.0.smithi136.stdout:1653: left oid 6 (ObjNum 539 snap 157 seq_num 539) 2024-09-12T23:44:01.622 INFO:tasks.rados.rados.0.smithi136.stdout:1656: seq_num 541 ranges {3514368=663552,4177920=557056,4734976=475136,5210112=172032} 2024-09-12T23:44:01.643 INFO:tasks.rados.rados.0.smithi136.stdout:1656: writing smithi13667081-7 from 3514368 to 4177920 tid 1 2024-09-12T23:44:01.646 INFO:tasks.rados.rados.0.smithi136.stdout:1656: writing smithi13667081-7 from 4177920 to 4734976 tid 2 2024-09-12T23:44:01.649 INFO:tasks.rados.rados.0.smithi136.stdout:1656: writing smithi13667081-7 from 4734976 to 5210112 tid 3 2024-09-12T23:44:01.650 INFO:tasks.rados.rados.0.smithi136.stdout:1656: writing smithi13667081-7 from 5210112 to 5382144 tid 4 2024-09-12T23:44:01.650 INFO:tasks.rados.rados.0.smithi136.stdout:1651: done (4 left) 2024-09-12T23:44:01.651 INFO:tasks.rados.rados.0.smithi136.stdout:1653: done (3 left) 2024-09-12T23:44:01.651 INFO:tasks.rados.rados.0.smithi136.stdout:1657: snap_remove snap 151 2024-09-12T23:44:01.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:01 smithi136 bash[75164]: cluster 2024-09-12T23:44:00.483004+0000 mon.a (mon.0) 326 : cluster [DBG] osdmap e386: 8 total, 8 up, 8 in 2024-09-12T23:44:01.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:01 smithi136 bash[75164]: cluster 2024-09-12T23:44:00.685965+0000 mgr.y (mgr.24661) 116 : cluster [DBG] pgmap v154: 121 pgs: 121 active+clean; 189 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 3.0 MiB/s wr, 19 op/s 2024-09-12T23:44:01.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:01 smithi136 bash[75164]: audit 2024-09-12T23:44:01.058685+0000 mon.a (mon.0) 327 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:01.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:01 smithi136 bash[75164]: audit 2024-09-12T23:44:01.060014+0000 mon.a (mon.0) 328 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-12T23:44:01.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:01 smithi136 bash[75164]: audit 2024-09-12T23:44:01.060937+0000 mon.a (mon.0) 329 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:44:01.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:01 smithi136 bash[76371]: cluster 2024-09-12T23:44:00.483004+0000 mon.a (mon.0) 326 : cluster [DBG] osdmap e386: 8 total, 8 up, 8 in 2024-09-12T23:44:01.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:01 smithi136 bash[76371]: cluster 2024-09-12T23:44:00.685965+0000 mgr.y (mgr.24661) 116 : cluster [DBG] pgmap v154: 121 pgs: 121 active+clean; 189 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 3.0 MiB/s wr, 19 op/s 2024-09-12T23:44:01.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:01 smithi136 bash[76371]: audit 2024-09-12T23:44:01.058685+0000 mon.a (mon.0) 327 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:01.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:01 smithi136 bash[76371]: audit 2024-09-12T23:44:01.060014+0000 mon.a (mon.0) 328 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-12T23:44:01.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:01 smithi136 bash[76371]: audit 2024-09-12T23:44:01.060937+0000 mon.a (mon.0) 329 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:44:01.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:01 smithi181 bash[67587]: cluster 2024-09-12T23:44:00.483004+0000 mon.a (mon.0) 326 : cluster [DBG] osdmap e386: 8 total, 8 up, 8 in 2024-09-12T23:44:01.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:01 smithi181 bash[67587]: cluster 2024-09-12T23:44:00.685965+0000 mgr.y (mgr.24661) 116 : cluster [DBG] pgmap v154: 121 pgs: 121 active+clean; 189 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 3.0 MiB/s wr, 19 op/s 2024-09-12T23:44:01.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:01 smithi181 bash[67587]: audit 2024-09-12T23:44:01.058685+0000 mon.a (mon.0) 327 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:01.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:01 smithi181 bash[67587]: audit 2024-09-12T23:44:01.060014+0000 mon.a (mon.0) 328 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-12T23:44:01.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:01 smithi181 bash[67587]: audit 2024-09-12T23:44:01.060937+0000 mon.a (mon.0) 329 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:44:02.498 INFO:tasks.rados.rados.0.smithi136.stdout:1657: done (3 left) 2024-09-12T23:44:02.498 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 40 current snap is 157 2024-09-12T23:44:02.498 INFO:tasks.rados.rados.0.smithi136.stdout:1658: seq_num 542 ranges {5070848=679936,5750784=442368,6193152=679936,6873088=81920} 2024-09-12T23:44:02.533 INFO:tasks.rados.rados.0.smithi136.stdout:1658: writing smithi13667081-40 from 5070848 to 5750784 tid 1 2024-09-12T23:44:02.535 INFO:tasks.rados.rados.0.smithi136.stdout:1658: writing smithi13667081-40 from 5750784 to 6193152 tid 2 2024-09-12T23:44:02.540 INFO:tasks.rados.rados.0.smithi136.stdout:1658: writing smithi13667081-40 from 6193152 to 6873088 tid 3 2024-09-12T23:44:02.541 INFO:tasks.rados.rados.0.smithi136.stdout:1658: writing smithi13667081-40 from 6873088 to 6955008 tid 4 2024-09-12T23:44:02.541 INFO:tasks.rados.rados.0.smithi136.stdout:1659: snap_create 2024-09-12T23:44:02.541 INFO:tasks.rados.rados.0.smithi136.stdout:1654: finishing write tid 1 to smithi13667081-18 2024-09-12T23:44:02.541 INFO:tasks.rados.rados.0.smithi136.stdout:1654: finishing write tid 2 to smithi13667081-18 2024-09-12T23:44:02.541 INFO:tasks.rados.rados.0.smithi136.stdout:1654: finishing write tid 3 to smithi13667081-18 2024-09-12T23:44:02.541 INFO:tasks.rados.rados.0.smithi136.stdout:1654: finishing write tid 4 to smithi13667081-18 2024-09-12T23:44:02.542 INFO:tasks.rados.rados.0.smithi136.stdout:1654: finishing write tid 5 to smithi13667081-18 2024-09-12T23:44:02.542 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 18 v 895 (ObjNum 540 snap 157 seq_num 540) dirty exists 2024-09-12T23:44:02.542 INFO:tasks.rados.rados.0.smithi136.stdout:1654: left oid 18 (ObjNum 540 snap 157 seq_num 540) 2024-09-12T23:44:02.542 INFO:tasks.rados.rados.0.smithi136.stdout:1656: finishing write tid 1 to smithi13667081-7 2024-09-12T23:44:02.542 INFO:tasks.rados.rados.0.smithi136.stdout:1656: finishing write tid 2 to smithi13667081-7 2024-09-12T23:44:02.542 INFO:tasks.rados.rados.0.smithi136.stdout:1656: finishing write tid 3 to smithi13667081-7 2024-09-12T23:44:02.542 INFO:tasks.rados.rados.0.smithi136.stdout:1656: finishing write tid 4 to smithi13667081-7 2024-09-12T23:44:02.542 INFO:tasks.rados.rados.0.smithi136.stdout:1656: finishing write tid 5 to smithi13667081-7 2024-09-12T23:44:02.542 INFO:tasks.rados.rados.0.smithi136.stdout:1656: finishing write tid 6 to smithi13667081-7 2024-09-12T23:44:02.542 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 7 v 900 (ObjNum 541 snap 157 seq_num 541) dirty exists 2024-09-12T23:44:02.542 INFO:tasks.rados.rados.0.smithi136.stdout:1656: left oid 7 (ObjNum 541 snap 157 seq_num 541) 2024-09-12T23:44:02.544 INFO:tasks.rados.rados.0.smithi136.stdout:1658: finishing write tid 1 to smithi13667081-40 2024-09-12T23:44:02.547 INFO:tasks.rados.rados.0.smithi136.stdout:1658: finishing write tid 2 to smithi13667081-40 2024-09-12T23:44:02.548 INFO:tasks.rados.rados.0.smithi136.stdout:1658: finishing write tid 3 to smithi13667081-40 2024-09-12T23:44:02.549 INFO:tasks.rados.rados.0.smithi136.stdout:1658: finishing write tid 4 to smithi13667081-40 2024-09-12T23:44:02.549 INFO:tasks.rados.rados.0.smithi136.stdout:1658: finishing write tid 5 to smithi13667081-40 2024-09-12T23:44:02.551 INFO:tasks.rados.rados.0.smithi136.stdout:1658: finishing write tid 6 to smithi13667081-40 2024-09-12T23:44:02.551 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 40 v 1250 (ObjNum 542 snap 157 seq_num 542) dirty exists 2024-09-12T23:44:02.551 INFO:tasks.rados.rados.0.smithi136.stdout:1658: left oid 40 (ObjNum 542 snap 157 seq_num 542) 2024-09-12T23:44:02.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:02 smithi181 bash[67587]: cephadm 2024-09-12T23:44:01.051984+0000 mgr.y (mgr.24661) 117 : cephadm [INF] Upgrade: Updating osd.0 2024-09-12T23:44:02.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:02 smithi181 bash[67587]: cephadm 2024-09-12T23:44:01.065449+0000 mgr.y (mgr.24661) 118 : cephadm [INF] Deploying daemon osd.0 on smithi136 2024-09-12T23:44:02.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:02 smithi181 bash[67587]: cluster 2024-09-12T23:44:01.504151+0000 mon.a (mon.0) 330 : cluster [DBG] osdmap e387: 8 total, 8 up, 8 in 2024-09-12T23:44:02.594 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:44:02 smithi181 bash[60471]: ts=2024-09-12T23:44:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:44:02.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:02 smithi136 bash[75164]: cephadm 2024-09-12T23:44:01.051984+0000 mgr.y (mgr.24661) 117 : cephadm [INF] Upgrade: Updating osd.0 2024-09-12T23:44:02.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:02 smithi136 bash[75164]: cephadm 2024-09-12T23:44:01.065449+0000 mgr.y (mgr.24661) 118 : cephadm [INF] Deploying daemon osd.0 on smithi136 2024-09-12T23:44:02.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:02 smithi136 bash[75164]: cluster 2024-09-12T23:44:01.504151+0000 mon.a (mon.0) 330 : cluster [DBG] osdmap e387: 8 total, 8 up, 8 in 2024-09-12T23:44:02.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:02 smithi136 bash[76371]: cephadm 2024-09-12T23:44:01.051984+0000 mgr.y (mgr.24661) 117 : cephadm [INF] Upgrade: Updating osd.0 2024-09-12T23:44:02.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:02 smithi136 bash[76371]: cephadm 2024-09-12T23:44:01.065449+0000 mgr.y (mgr.24661) 118 : cephadm [INF] Deploying daemon osd.0 on smithi136 2024-09-12T23:44:02.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:02 smithi136 bash[76371]: cluster 2024-09-12T23:44:01.504151+0000 mon.a (mon.0) 330 : cluster [DBG] osdmap e387: 8 total, 8 up, 8 in 2024-09-12T23:44:03.517 INFO:tasks.rados.rados.0.smithi136.stdout:1654: done (4 left) 2024-09-12T23:44:03.518 INFO:tasks.rados.rados.0.smithi136.stdout:1655: done (3 left) 2024-09-12T23:44:03.518 INFO:tasks.rados.rados.0.smithi136.stdout:1656: done (2 left) 2024-09-12T23:44:03.518 INFO:tasks.rados.rados.0.smithi136.stdout:1658: done (1 left) 2024-09-12T23:44:03.518 INFO:tasks.rados.rados.0.smithi136.stdout:1659: done (0 left) 2024-09-12T23:44:03.518 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 44 current snap is 158 2024-09-12T23:44:03.518 INFO:tasks.rados.rados.0.smithi136.stdout:1660: seq_num 543 ranges {0=704512,704512=729088,1433600=475136} 2024-09-12T23:44:03.521 INFO:tasks.rados.rados.0.smithi136.stdout:1660: writing smithi13667081-44 from 0 to 704512 tid 1 2024-09-12T23:44:03.526 INFO:tasks.rados.rados.0.smithi136.stdout:1660: writing smithi13667081-44 from 704512 to 1433600 tid 2 2024-09-12T23:44:03.529 INFO:tasks.rados.rados.0.smithi136.stdout:1660: writing smithi13667081-44 from 1433600 to 1908736 tid 3 2024-09-12T23:44:03.530 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 48 current snap is 158 2024-09-12T23:44:03.530 INFO:tasks.rados.rados.0.smithi136.stdout:1661: seq_num 544 ranges {1941504=720896,2662400=614400,3276800=589824} 2024-09-12T23:44:03.545 INFO:tasks.rados.rados.0.smithi136.stdout:1661: writing smithi13667081-48 from 1941504 to 2662400 tid 1 2024-09-12T23:44:03.548 INFO:tasks.rados.rados.0.smithi136.stdout:1661: writing smithi13667081-48 from 2662400 to 3276800 tid 2 2024-09-12T23:44:03.552 INFO:tasks.rados.rados.0.smithi136.stdout:1661: writing smithi13667081-48 from 3276800 to 3866624 tid 3 2024-09-12T23:44:03.552 INFO:tasks.rados.rados.0.smithi136.stdout:1662: delete oid 25 current snap is 158 2024-09-12T23:44:03.552 INFO:tasks.rados.rados.0.smithi136.stdout:1660: finishing write tid 1 to smithi13667081-44 2024-09-12T23:44:03.552 INFO:tasks.rados.rados.0.smithi136.stdout:1660: finishing write tid 2 to smithi13667081-44 2024-09-12T23:44:03.552 INFO:tasks.rados.rados.0.smithi136.stdout:1660: finishing write tid 3 to smithi13667081-44 2024-09-12T23:44:03.553 INFO:tasks.rados.rados.0.smithi136.stdout:1660: finishing write tid 4 to smithi13667081-44 2024-09-12T23:44:03.553 INFO:tasks.rados.rados.0.smithi136.stdout:1660: finishing write tid 5 to smithi13667081-44 2024-09-12T23:44:03.553 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 44 v 1033 (ObjNum 543 snap 158 seq_num 543) dirty exists 2024-09-12T23:44:03.553 INFO:tasks.rados.rados.0.smithi136.stdout:1660: left oid 44 (ObjNum 543 snap 158 seq_num 543) 2024-09-12T23:44:03.553 INFO:tasks.rados.rados.0.smithi136.stdout:1661: finishing write tid 1 to smithi13667081-48 2024-09-12T23:44:03.553 INFO:tasks.rados.rados.0.smithi136.stdout:1661: finishing write tid 2 to smithi13667081-48 2024-09-12T23:44:03.558 INFO:tasks.rados.rados.0.smithi136.stdout:1661: finishing write tid 3 to smithi13667081-48 2024-09-12T23:44:03.558 INFO:tasks.rados.rados.0.smithi136.stdout:1661: finishing write tid 4 to smithi13667081-48 2024-09-12T23:44:03.560 INFO:tasks.rados.rados.0.smithi136.stdout:1661: finishing write tid 5 to smithi13667081-48 2024-09-12T23:44:03.561 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 48 v 1038 (ObjNum 544 snap 158 seq_num 544) dirty exists 2024-09-12T23:44:03.561 INFO:tasks.rados.rados.0.smithi136.stdout:1661: left oid 48 (ObjNum 544 snap 158 seq_num 544) 2024-09-12T23:44:03.563 INFO:tasks.rados.rados.0.smithi136.stdout:1660: done (2 left) 2024-09-12T23:44:03.563 INFO:tasks.rados.rados.0.smithi136.stdout:1661: done (1 left) 2024-09-12T23:44:03.563 INFO:tasks.rados.rados.0.smithi136.stdout:1662: done (0 left) 2024-09-12T23:44:03.564 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 20 current snap is 158 2024-09-12T23:44:03.564 INFO:tasks.rados.rados.0.smithi136.stdout:1663: seq_num 546 ranges {3006464=753664,3760128=778240,4538368=425984} 2024-09-12T23:44:03.584 INFO:tasks.rados.rados.0.smithi136.stdout:1663: writing smithi13667081-20 from 3006464 to 3760128 tid 1 2024-09-12T23:44:03.589 INFO:tasks.rados.rados.0.smithi136.stdout:1663: writing smithi13667081-20 from 3760128 to 4538368 tid 2 2024-09-12T23:44:03.592 INFO:tasks.rados.rados.0.smithi136.stdout:1663: writing smithi13667081-20 from 4538368 to 4964352 tid 3 2024-09-12T23:44:03.592 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 14 current snap is 158 2024-09-12T23:44:03.592 INFO:tasks.rados.rados.0.smithi136.stdout:1663: finishing write tid 1 to smithi13667081-20 2024-09-12T23:44:03.592 INFO:tasks.rados.rados.0.smithi136.stdout:1664: seq_num 547 ranges {4775936=770048,5545984=663552,6209536=466944,6676480=73728} 2024-09-12T23:44:03.623 INFO:tasks.rados.rados.0.smithi136.stdout:1664: writing smithi13667081-14 from 4775936 to 5545984 tid 1 2024-09-12T23:44:03.627 INFO:tasks.rados.rados.0.smithi136.stdout:1664: writing smithi13667081-14 from 5545984 to 6209536 tid 2 2024-09-12T23:44:03.630 INFO:tasks.rados.rados.0.smithi136.stdout:1664: writing smithi13667081-14 from 6209536 to 6676480 tid 3 2024-09-12T23:44:03.631 INFO:tasks.rados.rados.0.smithi136.stdout:1664: writing smithi13667081-14 from 6676480 to 6750208 tid 4 2024-09-12T23:44:03.631 INFO:tasks.rados.rados.0.smithi136.stdout:1665: read oid 4 snap 156 2024-09-12T23:44:03.631 INFO:tasks.rados.rados.0.smithi136.stdout:1665: expect (ObjNum 533 snap 156 seq_num 533) 2024-09-12T23:44:03.631 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 22 current snap is 158 2024-09-12T23:44:03.631 INFO:tasks.rados.rados.0.smithi136.stdout:1663: finishing write tid 2 to smithi13667081-20 2024-09-12T23:44:03.631 INFO:tasks.rados.rados.0.smithi136.stdout:1663: finishing write tid 3 to smithi13667081-20 2024-09-12T23:44:03.631 INFO:tasks.rados.rados.0.smithi136.stdout:1666: seq_num 548 ranges {1531904=786432,2318336=598016,2916352=606208} 2024-09-12T23:44:03.644 INFO:tasks.rados.rados.0.smithi136.stdout:1666: writing smithi13667081-22 from 1531904 to 2318336 tid 1 2024-09-12T23:44:03.648 INFO:tasks.rados.rados.0.smithi136.stdout:1666: writing smithi13667081-22 from 2318336 to 2916352 tid 2 2024-09-12T23:44:03.652 INFO:tasks.rados.rados.0.smithi136.stdout:1666: writing smithi13667081-22 from 2916352 to 3522560 tid 3 2024-09-12T23:44:03.652 INFO:tasks.rados.rados.0.smithi136.stdout:1667: delete oid 41 current snap is 158 2024-09-12T23:44:03.652 INFO:tasks.rados.rados.0.smithi136.stdout:1663: finishing write tid 4 to smithi13667081-20 2024-09-12T23:44:03.652 INFO:tasks.rados.rados.0.smithi136.stdout:1663: finishing write tid 5 to smithi13667081-20 2024-09-12T23:44:03.652 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 20 v 314 (ObjNum 546 snap 158 seq_num 546) dirty exists 2024-09-12T23:44:03.652 INFO:tasks.rados.rados.0.smithi136.stdout:1663: left oid 20 (ObjNum 546 snap 158 seq_num 546) 2024-09-12T23:44:03.652 INFO:tasks.rados.rados.0.smithi136.stdout:1664: finishing write tid 1 to smithi13667081-14 2024-09-12T23:44:03.652 INFO:tasks.rados.rados.0.smithi136.stdout:1664: finishing write tid 2 to smithi13667081-14 2024-09-12T23:44:03.653 INFO:tasks.rados.rados.0.smithi136.stdout:1664: finishing write tid 3 to smithi13667081-14 2024-09-12T23:44:03.653 INFO:tasks.rados.rados.0.smithi136.stdout:1664: finishing write tid 4 to smithi13667081-14 2024-09-12T23:44:03.653 INFO:tasks.rados.rados.0.smithi136.stdout:1664: finishing write tid 5 to smithi13667081-14 2024-09-12T23:44:03.653 INFO:tasks.rados.rados.0.smithi136.stdout:1664: finishing write tid 6 to smithi13667081-14 2024-09-12T23:44:03.653 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 14 v 1044 (ObjNum 547 snap 158 seq_num 547) dirty exists 2024-09-12T23:44:03.653 INFO:tasks.rados.rados.0.smithi136.stdout:1664: left oid 14 (ObjNum 547 snap 158 seq_num 547) 2024-09-12T23:44:03.659 INFO:tasks.rados.rados.0.smithi136.stdout:1663: done (4 left) 2024-09-12T23:44:03.659 INFO:tasks.rados.rados.0.smithi136.stdout:1664: done (3 left) 2024-09-12T23:44:03.659 INFO:tasks.rados.rados.0.smithi136.stdout:1667: done (2 left) 2024-09-12T23:44:03.659 INFO:tasks.rados.rados.0.smithi136.stdout:1668: setattr oid 20 current snap is 158 2024-09-12T23:44:03.661 INFO:tasks.rados.rados.0.smithi136.stdout:1669: snap_remove snap 157 2024-09-12T23:44:03.743 INFO:teuthology.orchestra.run.smithi136.stdout:true 2024-09-12T23:44:03.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:03 smithi136 bash[75164]: cluster 2024-09-12T23:44:02.501306+0000 mon.a (mon.0) 331 : cluster [DBG] osdmap e388: 8 total, 8 up, 8 in 2024-09-12T23:44:03.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:03 smithi136 bash[75164]: cluster 2024-09-12T23:44:02.686434+0000 mgr.y (mgr.24661) 119 : cluster [DBG] pgmap v157: 121 pgs: 121 active+clean; 189 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 3.3 MiB/s wr, 21 op/s 2024-09-12T23:44:03.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:03 smithi136 bash[76371]: cluster 2024-09-12T23:44:02.501306+0000 mon.a (mon.0) 331 : cluster [DBG] osdmap e388: 8 total, 8 up, 8 in 2024-09-12T23:44:03.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:03 smithi136 bash[76371]: cluster 2024-09-12T23:44:02.686434+0000 mgr.y (mgr.24661) 119 : cluster [DBG] pgmap v157: 121 pgs: 121 active+clean; 189 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 3.3 MiB/s wr, 21 op/s 2024-09-12T23:44:03.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:03 smithi181 bash[67587]: cluster 2024-09-12T23:44:02.501306+0000 mon.a (mon.0) 331 : cluster [DBG] osdmap e388: 8 total, 8 up, 8 in 2024-09-12T23:44:03.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:03 smithi181 bash[67587]: cluster 2024-09-12T23:44:02.686434+0000 mgr.y (mgr.24661) 119 : cluster [DBG] pgmap v157: 121 pgs: 121 active+clean; 189 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 3.3 MiB/s wr, 21 op/s 2024-09-12T23:44:03.844 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:44:03 smithi181 bash[60471]: ts=2024-09-12T23:44:03.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi136\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:44:04.051 INFO:teuthology.orchestra.run.smithi136.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-12T23:44:04.051 INFO:teuthology.orchestra.run.smithi136.stdout:alertmanager.a smithi136 *:9093,9094 running (3m) 79s ago 9m 13.7M - 0.25.0 c8568f914cd2 b5197f020eeb 2024-09-12T23:44:04.051 INFO:teuthology.orchestra.run.smithi136.stdout:grafana.a smithi181 *:3000 running (3m) 50s ago 9m 75.3M - 9.4.7 954c08fa6188 256c181be984 2024-09-12T23:44:04.051 INFO:teuthology.orchestra.run.smithi136.stdout:mds.a smithi136 running (10m) 79s ago 10m 14.5M - 17.2.7-1627-gbbdadfa3 214985049bdb 713b9795ad19 2024-09-12T23:44:04.052 INFO:teuthology.orchestra.run.smithi136.stdout:mds.b smithi181 running (10m) 50s ago 10m 17.8M - 17.2.7-1627-gbbdadfa3 214985049bdb ad13179f0868 2024-09-12T23:44:04.052 INFO:teuthology.orchestra.run.smithi136.stdout:mgr.x smithi181 *:8443,9283,8765 running (2m) 50s ago 18m 425M - 18.2.4-837-g7a853b17 f21ba1eea74e cc664dc337f7 2024-09-12T23:44:04.052 INFO:teuthology.orchestra.run.smithi136.stdout:mgr.y smithi136 *:8443,9283,8765 running (3m) 79s ago 20m 482M - 18.2.4-837-g7a853b17 f21ba1eea74e 272f10b42fc3 2024-09-12T23:44:04.052 INFO:teuthology.orchestra.run.smithi136.stdout:mon.a smithi136 running (2m) 79s ago 20m 39.3M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e ad8d21d58ad9 2024-09-12T23:44:04.052 INFO:teuthology.orchestra.run.smithi136.stdout:mon.b smithi181 running (63s) 50s ago 18m 19.5M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e a70ec8315934 2024-09-12T23:44:04.052 INFO:teuthology.orchestra.run.smithi136.stdout:mon.c smithi136 running (91s) 79s ago 18m 19.2M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e 44d1be055a5e 2024-09-12T23:44:04.052 INFO:teuthology.orchestra.run.smithi136.stdout:node-exporter.a smithi136 *:9100 running (3m) 79s ago 9m 7996k - 1.5.0 0da6a335fe13 ccfc37c96620 2024-09-12T23:44:04.052 INFO:teuthology.orchestra.run.smithi136.stdout:node-exporter.b smithi181 *:9100 running (3m) 50s ago 9m 7847k - 1.5.0 0da6a335fe13 80bf0e383230 2024-09-12T23:44:04.052 INFO:teuthology.orchestra.run.smithi136.stdout:osd.0 smithi136 running (17m) 79s ago 17m 187M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb cbc0b7df0115 2024-09-12T23:44:04.052 INFO:teuthology.orchestra.run.smithi136.stdout:osd.1 smithi136 running (16m) 79s ago 16m 197M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb c4daee7da836 2024-09-12T23:44:04.052 INFO:teuthology.orchestra.run.smithi136.stdout:osd.2 smithi136 running (15m) 79s ago 15m 210M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 4ab8fd79cdb4 2024-09-12T23:44:04.052 INFO:teuthology.orchestra.run.smithi136.stdout:osd.3 smithi136 running (14m) 79s ago 14m 152M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 9ceb14a27521 2024-09-12T23:44:04.053 INFO:teuthology.orchestra.run.smithi136.stdout:osd.4 smithi181 running (13m) 50s ago 13m 230M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb a35cdca79055 2024-09-12T23:44:04.053 INFO:teuthology.orchestra.run.smithi136.stdout:osd.5 smithi181 running (13m) 50s ago 13m 192M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 8f2e8d494fef 2024-09-12T23:44:04.053 INFO:teuthology.orchestra.run.smithi136.stdout:osd.6 smithi181 running (12m) 50s ago 12m 106M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 9c6bec406408 2024-09-12T23:44:04.053 INFO:teuthology.orchestra.run.smithi136.stdout:osd.7 smithi181 running (11m) 50s ago 11m 174M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb bc0632ae2145 2024-09-12T23:44:04.053 INFO:teuthology.orchestra.run.smithi136.stdout:prometheus.a smithi181 *:9095 running (2m) 50s ago 10m 51.7M - 2.43.0 a07b618ecd1d 3853d003a5cc 2024-09-12T23:44:04.401 INFO:teuthology.orchestra.run.smithi136.stdout:{ 2024-09-12T23:44:04.401 INFO:teuthology.orchestra.run.smithi136.stdout: "mon": { 2024-09-12T23:44:04.401 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 3 2024-09-12T23:44:04.401 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:44:04.401 INFO:teuthology.orchestra.run.smithi136.stdout: "mgr": { 2024-09-12T23:44:04.401 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 2 2024-09-12T23:44:04.401 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:44:04.401 INFO:teuthology.orchestra.run.smithi136.stdout: "osd": { 2024-09-12T23:44:04.401 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 8 2024-09-12T23:44:04.401 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:44:04.401 INFO:teuthology.orchestra.run.smithi136.stdout: "mds": { 2024-09-12T23:44:04.402 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-12T23:44:04.402 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:44:04.402 INFO:teuthology.orchestra.run.smithi136.stdout: "overall": { 2024-09-12T23:44:04.402 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 10, 2024-09-12T23:44:04.402 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 5 2024-09-12T23:44:04.402 INFO:teuthology.orchestra.run.smithi136.stdout: } 2024-09-12T23:44:04.402 INFO:teuthology.orchestra.run.smithi136.stdout:} 2024-09-12T23:44:04.532 INFO:tasks.rados.rados.0.smithi136.stdout:1669: done (3 left) 2024-09-12T23:44:04.532 INFO:tasks.rados.rados.0.smithi136.stdout:1670: setattr oid 9 current snap is 158 2024-09-12T23:44:04.532 INFO:tasks.rados.rados.0.smithi136.stdout:1666: finishing write tid 1 to smithi13667081-22 2024-09-12T23:44:04.532 INFO:tasks.rados.rados.0.smithi136.stdout:1666: finishing write tid 2 to smithi13667081-22 2024-09-12T23:44:04.532 INFO:tasks.rados.rados.0.smithi136.stdout:1666: finishing write tid 3 to smithi13667081-22 2024-09-12T23:44:04.532 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 20 v 315 (ObjNum 546 snap 158 seq_num 546) dirty exists 2024-09-12T23:44:04.532 INFO:tasks.rados.rados.0.smithi136.stdout:1666: finishing write tid 4 to smithi13667081-22 2024-09-12T23:44:04.532 INFO:tasks.rados.rados.0.smithi136.stdout:1665: expect (ObjNum 533 snap 156 seq_num 533) 2024-09-12T23:44:04.725 INFO:tasks.rados.rados.0.smithi136.stdout:1666: finishing write tid 5 to smithi13667081-22 2024-09-12T23:44:04.726 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 22 v 810 (ObjNum 548 snap 158 seq_num 548) dirty exists 2024-09-12T23:44:04.726 INFO:tasks.rados.rados.0.smithi136.stdout:1666: left oid 22 (ObjNum 548 snap 158 seq_num 548) 2024-09-12T23:44:04.726 INFO:tasks.rados.rados.0.smithi136.stdout:1665: done (3 left) 2024-09-12T23:44:04.726 INFO:tasks.rados.rados.0.smithi136.stdout:1666: done (2 left) 2024-09-12T23:44:04.726 INFO:tasks.rados.rados.0.smithi136.stdout:1668: done (1 left) 2024-09-12T23:44:04.726 INFO:tasks.rados.rados.0.smithi136.stdout:1671: rollback oid 32 current snap is 158 2024-09-12T23:44:04.726 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 32 to 156 2024-09-12T23:44:04.726 INFO:tasks.rados.rados.0.smithi136.stdout:1672: copy_from oid 19 from oid 46 current snap is 158 2024-09-12T23:44:04.726 INFO:tasks.rados.rados.0.smithi136.stdout:1673: delete oid 44 current snap is 158 2024-09-12T23:44:04.733 INFO:tasks.rados.rados.0.smithi136.stdout:1671: finishing rollback tid 0 to smithi13667081-32 2024-09-12T23:44:04.733 INFO:tasks.rados.rados.0.smithi136.stdout:1671: finishing rollback tid 1 to smithi13667081-32 2024-09-12T23:44:04.733 INFO:tasks.rados.rados.0.smithi136.stdout:1671: finishing rollback tid 2 to smithi13667081-32 2024-09-12T23:44:04.733 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 32 v 757 (ObjNum 535 snap 156 seq_num 535) dirty exists 2024-09-12T23:44:04.735 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 9 v 628 (ObjNum 513 snap 150 seq_num 513) dirty exists 2024-09-12T23:44:04.740 INFO:tasks.rados.rados.0.smithi136.stdout:1670: done (3 left) 2024-09-12T23:44:04.740 INFO:tasks.rados.rados.0.smithi136.stdout:1671: done (2 left) 2024-09-12T23:44:04.740 INFO:tasks.rados.rados.0.smithi136.stdout:1673: done (1 left) 2024-09-12T23:44:04.740 INFO:tasks.rados.rados.0.smithi136.stdout:1674: read oid 7 snap -1 2024-09-12T23:44:04.740 INFO:tasks.rados.rados.0.smithi136.stdout:1674: expect (ObjNum 541 snap 157 seq_num 541) 2024-09-12T23:44:04.740 INFO:tasks.rados.rados.0.smithi136.stdout:1675: snap_remove snap 156 2024-09-12T23:44:04.788 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:04 smithi136 bash[75164]: cluster 2024-09-12T23:44:03.521804+0000 mon.a (mon.0) 332 : cluster [DBG] osdmap e389: 8 total, 8 up, 8 in 2024-09-12T23:44:04.788 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:04 smithi136 bash[75164]: audit 2024-09-12T23:44:03.735359+0000 mgr.y (mgr.24661) 120 : audit [DBG] from='client.34218 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:44:04.788 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:04 smithi136 bash[75164]: audit 2024-09-12T23:44:04.404702+0000 mon.c (mon.2) 3 : audit [DBG] from='client.? 172.21.15.136:0/2867190007' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:44:04.789 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:04 smithi136 bash[76371]: cluster 2024-09-12T23:44:03.521804+0000 mon.a (mon.0) 332 : cluster [DBG] osdmap e389: 8 total, 8 up, 8 in 2024-09-12T23:44:04.789 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:04 smithi136 bash[76371]: audit 2024-09-12T23:44:03.735359+0000 mgr.y (mgr.24661) 120 : audit [DBG] from='client.34218 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:44:04.789 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:04 smithi136 bash[76371]: audit 2024-09-12T23:44:04.404702+0000 mon.c (mon.2) 3 : audit [DBG] from='client.? 172.21.15.136:0/2867190007' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:44:04.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:04 smithi181 bash[67587]: cluster 2024-09-12T23:44:03.521804+0000 mon.a (mon.0) 332 : cluster [DBG] osdmap e389: 8 total, 8 up, 8 in 2024-09-12T23:44:04.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:04 smithi181 bash[67587]: audit 2024-09-12T23:44:03.735359+0000 mgr.y (mgr.24661) 120 : audit [DBG] from='client.34218 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:44:04.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:04 smithi181 bash[67587]: audit 2024-09-12T23:44:04.404702+0000 mon.c (mon.2) 3 : audit [DBG] from='client.? 172.21.15.136:0/2867190007' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:44:04.844 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:44:04 smithi181 bash[60471]: ts=2024-09-12T23:44:04.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:44:04.844 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:44:04 smithi181 bash[60471]: ts=2024-09-12T23:44:04.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:44:05.561 INFO:tasks.rados.rados.0.smithi136.stdout:1675: done (2 left) 2024-09-12T23:44:05.562 INFO:tasks.rados.rados.0.smithi136.stdout:1676: copy_from oid 26 from oid 6 current snap is 158 2024-09-12T23:44:05.562 INFO:tasks.rados.rados.0.smithi136.stdout:1677: snap_create 2024-09-12T23:44:05.562 INFO:tasks.rados.rados.0.smithi136.stdout:1672: finishing copy_from racing read to smithi13667081-19 2024-09-12T23:44:05.562 INFO:tasks.rados.rados.0.smithi136.stdout:1672: finishing copy_from to smithi13667081-19 2024-09-12T23:44:05.562 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 19 v 613 (ObjNum 350 snap 95 seq_num 350) dirty exists 2024-09-12T23:44:05.562 INFO:tasks.rados.rados.0.smithi136.stdout:1674: expect (ObjNum 541 snap 157 seq_num 541) 2024-09-12T23:44:05.828 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:05 smithi136 bash[75164]: audit 2024-09-12T23:44:04.020234+0000 mgr.y (mgr.24661) 121 : audit [DBG] from='client.34224 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:44:05.828 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:05 smithi136 bash[75164]: cluster 2024-09-12T23:44:04.535055+0000 mon.a (mon.0) 333 : cluster [DBG] osdmap e390: 8 total, 8 up, 8 in 2024-09-12T23:44:05.828 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:05 smithi136 bash[75164]: cluster 2024-09-12T23:44:04.687727+0000 mgr.y (mgr.24661) 122 : cluster [DBG] pgmap v160: 121 pgs: 121 active+clean; 189 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 4.5 MiB/s wr, 20 op/s 2024-09-12T23:44:05.828 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:05 smithi136 bash[76371]: audit 2024-09-12T23:44:04.020234+0000 mgr.y (mgr.24661) 121 : audit [DBG] from='client.34224 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:44:05.829 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:05 smithi136 bash[76371]: cluster 2024-09-12T23:44:04.535055+0000 mon.a (mon.0) 333 : cluster [DBG] osdmap e390: 8 total, 8 up, 8 in 2024-09-12T23:44:05.829 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:05 smithi136 bash[76371]: cluster 2024-09-12T23:44:04.687727+0000 mgr.y (mgr.24661) 122 : cluster [DBG] pgmap v160: 121 pgs: 121 active+clean; 189 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 4.5 MiB/s wr, 20 op/s 2024-09-12T23:44:05.834 INFO:tasks.rados.rados.0.smithi136.stdout:1676: finishing copy_from racing read to smithi13667081-26 2024-09-12T23:44:05.834 INFO:tasks.rados.rados.0.smithi136.stdout:1676: finishing copy_from to smithi13667081-26 2024-09-12T23:44:05.834 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 26 v 310 (ObjNum 539 snap 157 seq_num 539) dirty exists 2024-09-12T23:44:05.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:05 smithi181 bash[67587]: audit 2024-09-12T23:44:04.020234+0000 mgr.y (mgr.24661) 121 : audit [DBG] from='client.34224 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:44:05.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:05 smithi181 bash[67587]: cluster 2024-09-12T23:44:04.535055+0000 mon.a (mon.0) 333 : cluster [DBG] osdmap e390: 8 total, 8 up, 8 in 2024-09-12T23:44:05.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:05 smithi181 bash[67587]: cluster 2024-09-12T23:44:04.687727+0000 mgr.y (mgr.24661) 122 : cluster [DBG] pgmap v160: 121 pgs: 121 active+clean; 189 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 4.5 MiB/s wr, 20 op/s 2024-09-12T23:44:06.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:05 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:44:06.256 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:44:05 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:44:06.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:05 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:44:06.257 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 12 23:44:05 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:44:06.257 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 12 23:44:05 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:44:06.258 INFO:journalctl@ceph.mds.a.smithi136.stdout:Sep 12 23:44:05 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:44:06.258 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:44:05 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:44:06.258 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 12 23:44:05 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:44:06.259 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:44:05 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:44:06.259 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 12 23:44:05 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:44:06.259 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 12 23:44:05 smithi136 systemd[1]: Stopping Ceph osd.0 for da58a47c-715d-11ef-bceb-c7b262605968... 2024-09-12T23:44:06.259 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 12 23:44:06 smithi136 bash[35199]: debug 2024-09-12T23:44:06.062+0000 7f249106e640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-12T23:44:06.259 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 12 23:44:06 smithi136 bash[35199]: debug 2024-09-12T23:44:06.062+0000 7f249106e640 -1 osd.0 391 *** Got signal Terminated *** 2024-09-12T23:44:06.259 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 12 23:44:06 smithi136 bash[35199]: debug 2024-09-12T23:44:06.062+0000 7f249106e640 -1 osd.0 391 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-12T23:44:06.332 INFO:tasks.rados.rados.0.smithi136.stdout:1672: done (3 left) 2024-09-12T23:44:06.332 INFO:tasks.rados.rados.0.smithi136.stdout:1674: done (2 left) 2024-09-12T23:44:06.332 INFO:tasks.rados.rados.0.smithi136.stdout:1676: done (1 left) 2024-09-12T23:44:06.332 INFO:tasks.rados.rados.0.smithi136.stdout:1677: done (0 left) 2024-09-12T23:44:06.332 INFO:tasks.rados.rados.0.smithi136.stdout:1678: read oid 44 snap 158 2024-09-12T23:44:06.333 INFO:tasks.rados.rados.0.smithi136.stdout:1678: expect deleted 2024-09-12T23:44:06.333 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 50 current snap is 159 2024-09-12T23:44:06.333 INFO:tasks.rados.rados.0.smithi136.stdout:1679: seq_num 551 ranges {0=434176,434176=647168,1081344=450560,1531904=507904} 2024-09-12T23:44:06.335 INFO:tasks.rados.rados.0.smithi136.stdout:1679: writing smithi13667081-50 from 0 to 434176 tid 1 2024-09-12T23:44:06.346 INFO:tasks.rados.rados.0.smithi136.stdout:1679: writing smithi13667081-50 from 434176 to 1081344 tid 2 2024-09-12T23:44:06.353 INFO:tasks.rados.rados.0.smithi136.stdout:1679: writing smithi13667081-50 from 1081344 to 1531904 tid 3 2024-09-12T23:44:06.360 INFO:tasks.rados.rados.0.smithi136.stdout:1679: writing smithi13667081-50 from 1531904 to 2039808 tid 4 2024-09-12T23:44:06.360 INFO:tasks.rados.rados.0.smithi136.stdout:1680: read oid 12 snap -1 2024-09-12T23:44:06.360 INFO:tasks.rados.rados.0.smithi136.stdout:1680: expect (ObjNum 512 snap 149 seq_num 512) 2024-09-12T23:44:06.360 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 43 current snap is 159 2024-09-12T23:44:06.360 INFO:tasks.rados.rados.0.smithi136.stdout:1681: seq_num 552 ranges {7479296=450560,7929856=532480,8462336=655360,9117696=417792} 2024-09-12T23:44:06.406 INFO:tasks.rados.rados.0.smithi136.stdout:1681: writing smithi13667081-43 from 7479296 to 7929856 tid 1 2024-09-12T23:44:06.409 INFO:tasks.rados.rados.0.smithi136.stdout:1681: writing smithi13667081-43 from 7929856 to 8462336 tid 2 2024-09-12T23:44:06.414 INFO:tasks.rados.rados.0.smithi136.stdout:1681: writing smithi13667081-43 from 8462336 to 9117696 tid 3 2024-09-12T23:44:06.417 INFO:tasks.rados.rados.0.smithi136.stdout:1681: writing smithi13667081-43 from 9117696 to 9535488 tid 4 2024-09-12T23:44:06.417 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 21 current snap is 159 2024-09-12T23:44:06.417 INFO:tasks.rados.rados.0.smithi136.stdout:1682: seq_num 553 ranges {6356992=466944,6823936=417792,7241728=417792,7659520=516096,8175616=253952} 2024-09-12T23:44:06.455 INFO:tasks.rados.rados.0.smithi136.stdout:1682: writing smithi13667081-21 from 6356992 to 6823936 tid 1 2024-09-12T23:44:06.457 INFO:tasks.rados.rados.0.smithi136.stdout:1682: writing smithi13667081-21 from 6823936 to 7241728 tid 2 2024-09-12T23:44:06.459 INFO:tasks.rados.rados.0.smithi136.stdout:1682: writing smithi13667081-21 from 7241728 to 7659520 tid 3 2024-09-12T23:44:06.462 INFO:tasks.rados.rados.0.smithi136.stdout:1682: writing smithi13667081-21 from 7659520 to 8175616 tid 4 2024-09-12T23:44:06.465 INFO:tasks.rados.rados.0.smithi136.stdout:1682: writing smithi13667081-21 from 8175616 to 8429568 tid 5 2024-09-12T23:44:06.465 INFO:tasks.rados.rados.0.smithi136.stdout:1683: rmattr oid 8 current snap is 159 2024-09-12T23:44:06.465 INFO:tasks.rados.rados.0.smithi136.stdout:1683: done (5 left) 2024-09-12T23:44:06.465 INFO:tasks.rados.rados.0.smithi136.stdout:1684: read oid 49 snap -1 2024-09-12T23:44:06.465 INFO:tasks.rados.rados.0.smithi136.stdout:1684: expect (ObjNum 399 snap 116 seq_num 399) 2024-09-12T23:44:06.465 INFO:tasks.rados.rados.0.smithi136.stdout:1685: delete oid 16 current snap is 159 2024-09-12T23:44:06.465 INFO:tasks.rados.rados.0.smithi136.stdout:1679: finishing write tid 1 to smithi13667081-50 2024-09-12T23:44:06.465 INFO:tasks.rados.rados.0.smithi136.stdout:1679: finishing write tid 2 to smithi13667081-50 2024-09-12T23:44:06.465 INFO:tasks.rados.rados.0.smithi136.stdout:1679: finishing write tid 3 to smithi13667081-50 2024-09-12T23:44:06.466 INFO:tasks.rados.rados.0.smithi136.stdout:1679: finishing write tid 4 to smithi13667081-50 2024-09-12T23:44:06.466 INFO:tasks.rados.rados.0.smithi136.stdout:1679: finishing write tid 5 to smithi13667081-50 2024-09-12T23:44:06.466 INFO:tasks.rados.rados.0.smithi136.stdout:1679: finishing write tid 6 to smithi13667081-50 2024-09-12T23:44:06.466 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 50 v 809 (ObjNum 551 snap 159 seq_num 551) dirty exists 2024-09-12T23:44:06.466 INFO:tasks.rados.rados.0.smithi136.stdout:1679: left oid 50 (ObjNum 551 snap 159 seq_num 551) 2024-09-12T23:44:06.466 INFO:tasks.rados.rados.0.smithi136.stdout:1680: expect (ObjNum 512 snap 149 seq_num 512) 2024-09-12T23:44:06.595 INFO:tasks.rados.rados.0.smithi136.stdout:1681: finishing write tid 1 to smithi13667081-43 2024-09-12T23:44:06.596 INFO:tasks.rados.rados.0.smithi136.stdout:1681: finishing write tid 2 to smithi13667081-43 2024-09-12T23:44:06.596 INFO:tasks.rados.rados.0.smithi136.stdout:1681: finishing write tid 3 to smithi13667081-43 2024-09-12T23:44:06.596 INFO:tasks.rados.rados.0.smithi136.stdout:1681: finishing write tid 4 to smithi13667081-43 2024-09-12T23:44:06.596 INFO:tasks.rados.rados.0.smithi136.stdout:1681: finishing write tid 5 to smithi13667081-43 2024-09-12T23:44:06.596 INFO:tasks.rados.rados.0.smithi136.stdout:1681: finishing write tid 6 to smithi13667081-43 2024-09-12T23:44:06.596 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 43 v 1063 (ObjNum 552 snap 159 seq_num 552) dirty exists 2024-09-12T23:44:06.596 INFO:tasks.rados.rados.0.smithi136.stdout:1681: left oid 43 (ObjNum 552 snap 159 seq_num 552) 2024-09-12T23:44:06.596 INFO:tasks.rados.rados.0.smithi136.stdout:1684: expect (ObjNum 399 snap 116 seq_num 399) 2024-09-12T23:44:06.843 INFO:tasks.rados.rados.0.smithi136.stdout:1678: done (6 left) 2024-09-12T23:44:06.843 INFO:tasks.rados.rados.0.smithi136.stdout:1679: done (5 left) 2024-09-12T23:44:06.843 INFO:tasks.rados.rados.0.smithi136.stdout:1680: done (4 left) 2024-09-12T23:44:06.843 INFO:tasks.rados.rados.0.smithi136.stdout:1681: done (3 left) 2024-09-12T23:44:06.843 INFO:tasks.rados.rados.0.smithi136.stdout:1684: done (2 left) 2024-09-12T23:44:06.843 INFO:tasks.rados.rados.0.smithi136.stdout:1685: done (1 left) 2024-09-12T23:44:06.843 INFO:tasks.rados.rados.0.smithi136.stdout:1686: copy_from oid 30 from oid 19 current snap is 159 2024-09-12T23:44:06.843 INFO:tasks.rados.rados.0.smithi136.stdout:1687: read oid 10 snap -1 2024-09-12T23:44:06.843 INFO:tasks.rados.rados.0.smithi136.stdout:1687: expect (ObjNum 514 snap 150 seq_num 514) 2024-09-12T23:44:06.844 INFO:tasks.rados.rados.0.smithi136.stdout:1688: rmattr oid 20 current snap is 159 2024-09-12T23:44:06.844 INFO:tasks.rados.rados.0.smithi136.stdout:1689: delete oid 32 current snap is 159 2024-09-12T23:44:06.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:06 smithi181 bash[67587]: cluster 2024-09-12T23:44:05.546342+0000 mon.a (mon.0) 334 : cluster [DBG] osdmap e391: 8 total, 8 up, 8 in 2024-09-12T23:44:06.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:06 smithi181 bash[67587]: cluster 2024-09-12T23:44:06.067142+0000 mon.a (mon.0) 335 : cluster [INF] osd.0 marked itself down and dead 2024-09-12T23:44:06.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:06 smithi181 bash[67587]: cluster 2024-09-12T23:44:06.311069+0000 mon.a (mon.0) 336 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-12T23:44:06.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:06 smithi181 bash[67587]: cluster 2024-09-12T23:44:06.322235+0000 mon.a (mon.0) 337 : cluster [DBG] osdmap e392: 8 total, 7 up, 8 in 2024-09-12T23:44:06.847 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 20 v 316 (ObjNum 546 snap 158 seq_num 546) dirty exists 2024-09-12T23:44:06.905 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:06 smithi136 bash[75164]: cluster 2024-09-12T23:44:05.546342+0000 mon.a (mon.0) 334 : cluster [DBG] osdmap e391: 8 total, 8 up, 8 in 2024-09-12T23:44:06.905 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:06 smithi136 bash[75164]: cluster 2024-09-12T23:44:06.067142+0000 mon.a (mon.0) 335 : cluster [INF] osd.0 marked itself down and dead 2024-09-12T23:44:06.905 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:06 smithi136 bash[75164]: cluster 2024-09-12T23:44:06.311069+0000 mon.a (mon.0) 336 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-12T23:44:06.905 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:06 smithi136 bash[75164]: cluster 2024-09-12T23:44:06.322235+0000 mon.a (mon.0) 337 : cluster [DBG] osdmap e392: 8 total, 7 up, 8 in 2024-09-12T23:44:06.905 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:06 smithi136 bash[76371]: cluster 2024-09-12T23:44:05.546342+0000 mon.a (mon.0) 334 : cluster [DBG] osdmap e391: 8 total, 8 up, 8 in 2024-09-12T23:44:06.905 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:06 smithi136 bash[76371]: cluster 2024-09-12T23:44:06.067142+0000 mon.a (mon.0) 335 : cluster [INF] osd.0 marked itself down and dead 2024-09-12T23:44:06.905 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:06 smithi136 bash[76371]: cluster 2024-09-12T23:44:06.311069+0000 mon.a (mon.0) 336 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-12T23:44:06.905 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:06 smithi136 bash[76371]: cluster 2024-09-12T23:44:06.322235+0000 mon.a (mon.0) 337 : cluster [DBG] osdmap e392: 8 total, 7 up, 8 in 2024-09-12T23:44:07.355 INFO:tasks.rados.rados.0.smithi136.stdout:1682: finishing write tid 1 to smithi13667081-21 2024-09-12T23:44:07.355 INFO:tasks.rados.rados.0.smithi136.stdout:1682: finishing write tid 2 to smithi13667081-21 2024-09-12T23:44:07.355 INFO:tasks.rados.rados.0.smithi136.stdout:1682: finishing write tid 3 to smithi13667081-21 2024-09-12T23:44:07.356 INFO:tasks.rados.rados.0.smithi136.stdout:1688: done (4 left) 2024-09-12T23:44:07.356 INFO:tasks.rados.rados.0.smithi136.stdout:1689: done (3 left) 2024-09-12T23:44:07.356 INFO:tasks.rados.rados.0.smithi136.stdout:1690: read oid 26 snap -1 2024-09-12T23:44:07.356 INFO:tasks.rados.rados.0.smithi136.stdout:1690: expect (ObjNum 539 snap 157 seq_num 539) 2024-09-12T23:44:07.356 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 36 current snap is 159 2024-09-12T23:44:07.356 INFO:tasks.rados.rados.0.smithi136.stdout:1682: finishing write tid 4 to smithi13667081-21 2024-09-12T23:44:07.356 INFO:tasks.rados.rados.0.smithi136.stdout:1691: seq_num 556 ranges {4472832=516096,4988928=524288,5513216=638976,6152192=442368} 2024-09-12T23:44:07.400 INFO:tasks.rados.rados.0.smithi136.stdout:1691: writing smithi13667081-36 from 4472832 to 4988928 tid 1 2024-09-12T23:44:07.404 INFO:tasks.rados.rados.0.smithi136.stdout:1691: writing smithi13667081-36 from 4988928 to 5513216 tid 2 2024-09-12T23:44:07.410 INFO:tasks.rados.rados.0.smithi136.stdout:1691: writing smithi13667081-36 from 5513216 to 6152192 tid 3 2024-09-12T23:44:07.413 INFO:tasks.rados.rados.0.smithi136.stdout:1691: writing smithi13667081-36 from 6152192 to 6594560 tid 4 2024-09-12T23:44:07.413 INFO:tasks.rados.rados.0.smithi136.stdout:1692: copy_from oid 27 from oid 14 current snap is 159 2024-09-12T23:44:07.413 INFO:tasks.rados.rados.0.smithi136.stdout:1682: finishing write tid 5 to smithi13667081-21 2024-09-12T23:44:07.414 INFO:tasks.rados.rados.0.smithi136.stdout:1682: finishing write tid 6 to smithi13667081-21 2024-09-12T23:44:07.414 INFO:tasks.rados.rados.0.smithi136.stdout:1682: finishing write tid 7 to smithi13667081-21 2024-09-12T23:44:07.414 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 21 v 816 (ObjNum 553 snap 159 seq_num 553) dirty exists 2024-09-12T23:44:07.414 INFO:tasks.rados.rados.0.smithi136.stdout:1682: left oid 21 (ObjNum 553 snap 159 seq_num 553) 2024-09-12T23:44:07.414 INFO:tasks.rados.rados.0.smithi136.stdout:1687: expect (ObjNum 514 snap 150 seq_num 514) 2024-09-12T23:44:07.466 INFO:tasks.rados.rados.0.smithi136.stdout:1690: expect (ObjNum 539 snap 157 seq_num 539) 2024-09-12T23:44:07.514 INFO:tasks.rados.rados.0.smithi136.stdout:1686: finishing copy_from racing read to smithi13667081-30 2024-09-12T23:44:07.514 INFO:tasks.rados.rados.0.smithi136.stdout:1686: finishing copy_from to smithi13667081-30 2024-09-12T23:44:07.514 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 30 v 613 (ObjNum 350 snap 95 seq_num 350) dirty exists 2024-09-12T23:44:07.514 INFO:tasks.rados.rados.0.smithi136.stdout:1691: finishing write tid 1 to smithi13667081-36 2024-09-12T23:44:07.515 INFO:tasks.rados.rados.0.smithi136.stdout:1691: finishing write tid 2 to smithi13667081-36 2024-09-12T23:44:07.515 INFO:tasks.rados.rados.0.smithi136.stdout:1682: done (5 left) 2024-09-12T23:44:07.515 INFO:tasks.rados.rados.0.smithi136.stdout:1686: done (4 left) 2024-09-12T23:44:07.515 INFO:tasks.rados.rados.0.smithi136.stdout:1687: done (3 left) 2024-09-12T23:44:07.515 INFO:tasks.rados.rados.0.smithi136.stdout:1690: done (2 left) 2024-09-12T23:44:07.515 INFO:tasks.rados.rados.0.smithi136.stdout:1693: rollback oid 49 current snap is 159 2024-09-12T23:44:07.515 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 49 to 158 2024-09-12T23:44:07.515 INFO:tasks.rados.rados.0.smithi136.stdout:1691: finishing write tid 3 to smithi13667081-36 2024-09-12T23:44:07.515 INFO:tasks.rados.rados.0.smithi136.stdout:1691: finishing write tid 4 to smithi13667081-36 2024-09-12T23:44:07.515 INFO:tasks.rados.rados.0.smithi136.stdout:1691: finishing write tid 5 to smithi13667081-36 2024-09-12T23:44:07.515 INFO:tasks.rados.rados.0.smithi136.stdout:1691: finishing write tid 6 to smithi13667081-36 2024-09-12T23:44:07.515 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 36 v 781 (ObjNum 556 snap 159 seq_num 556) dirty exists 2024-09-12T23:44:07.515 INFO:tasks.rados.rados.0.smithi136.stdout:1691: left oid 36 (ObjNum 556 snap 159 seq_num 556) 2024-09-12T23:44:07.515 INFO:tasks.rados.rados.0.smithi136.stdout:1692: finishing copy_from racing read to smithi13667081-27 2024-09-12T23:44:07.516 INFO:tasks.rados.rados.0.smithi136.stdout:1692: finishing copy_from to smithi13667081-27 2024-09-12T23:44:07.516 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 27 v 1044 (ObjNum 547 snap 158 seq_num 547) dirty exists 2024-09-12T23:44:07.516 INFO:tasks.rados.rados.0.smithi136.stdout:1691: done (2 left) 2024-09-12T23:44:07.516 INFO:tasks.rados.rados.0.smithi136.stdout:1692: done (1 left) 2024-09-12T23:44:07.516 INFO:tasks.rados.rados.0.smithi136.stdout:1694: rollback oid 3 current snap is 159 2024-09-12T23:44:07.516 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 3 to 158 2024-09-12T23:44:07.516 INFO:tasks.rados.rados.0.smithi136.stdout:1695: read oid 13 snap -1 2024-09-12T23:44:07.516 INFO:tasks.rados.rados.0.smithi136.stdout:1695: expect (ObjNum 441 snap 132 seq_num 441) 2024-09-12T23:44:07.516 INFO:tasks.rados.rados.0.smithi136.stdout:1696: snap_create 2024-09-12T23:44:07.523 INFO:tasks.rados.rados.0.smithi136.stdout:1693: finishing rollback tid 0 to smithi13667081-49 2024-09-12T23:44:07.523 INFO:tasks.rados.rados.0.smithi136.stdout:1693: finishing rollback tid 1 to smithi13667081-49 2024-09-12T23:44:07.523 INFO:tasks.rados.rados.0.smithi136.stdout:1693: finishing rollback tid 2 to smithi13667081-49 2024-09-12T23:44:07.523 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 49 v 896 (ObjNum 399 snap 116 seq_num 399) dirty exists 2024-09-12T23:44:07.523 INFO:tasks.rados.rados.0.smithi136.stdout:1694: finishing rollback tid 0 to smithi13667081-3 2024-09-12T23:44:07.523 INFO:tasks.rados.rados.0.smithi136.stdout:1694: finishing rollback tid 1 to smithi13667081-3 2024-09-12T23:44:07.524 INFO:tasks.rados.rados.0.smithi136.stdout:1694: finishing rollback tid 2 to smithi13667081-3 2024-09-12T23:44:07.524 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 3 v 899 (ObjNum 536 snap 156 seq_num 536) dirty exists 2024-09-12T23:44:07.560 INFO:tasks.rados.rados.0.smithi136.stdout:1695: expect (ObjNum 441 snap 132 seq_num 441) 2024-09-12T23:44:07.756 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 12 23:44:07 smithi136 bash[80361]: ceph-da58a47c-715d-11ef-bceb-c7b262605968-osd-0 2024-09-12T23:44:07.756 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 12 23:44:07 smithi136 bash[80427]: Error response from daemon: No such container: ceph-da58a47c-715d-11ef-bceb-c7b262605968-osd-0 2024-09-12T23:44:07.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:07 smithi136 bash[75164]: cluster 2024-09-12T23:44:06.688235+0000 mgr.y (mgr.24661) 123 : cluster [DBG] pgmap v163: 121 pgs: 16 stale+active+clean, 105 active+clean; 189 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 4.5 MiB/s wr, 20 op/s 2024-09-12T23:44:07.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:07 smithi136 bash[75164]: cluster 2024-09-12T23:44:07.323893+0000 mon.a (mon.0) 338 : cluster [DBG] osdmap e393: 8 total, 7 up, 8 in 2024-09-12T23:44:07.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:07 smithi136 bash[76371]: cluster 2024-09-12T23:44:06.688235+0000 mgr.y (mgr.24661) 123 : cluster [DBG] pgmap v163: 121 pgs: 16 stale+active+clean, 105 active+clean; 189 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 4.5 MiB/s wr, 20 op/s 2024-09-12T23:44:07.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:07 smithi136 bash[76371]: cluster 2024-09-12T23:44:07.323893+0000 mon.a (mon.0) 338 : cluster [DBG] osdmap e393: 8 total, 7 up, 8 in 2024-09-12T23:44:07.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:07 smithi181 bash[67587]: cluster 2024-09-12T23:44:06.688235+0000 mgr.y (mgr.24661) 123 : cluster [DBG] pgmap v163: 121 pgs: 16 stale+active+clean, 105 active+clean; 189 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 4.5 MiB/s wr, 20 op/s 2024-09-12T23:44:07.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:07 smithi181 bash[67587]: cluster 2024-09-12T23:44:07.323893+0000 mon.a (mon.0) 338 : cluster [DBG] osdmap e393: 8 total, 7 up, 8 in 2024-09-12T23:44:08.323 INFO:tasks.rados.rados.0.smithi136.stdout:1693: done (3 left) 2024-09-12T23:44:08.323 INFO:tasks.rados.rados.0.smithi136.stdout:1694: done (2 left) 2024-09-12T23:44:08.323 INFO:tasks.rados.rados.0.smithi136.stdout:1695: done (1 left) 2024-09-12T23:44:08.323 INFO:tasks.rados.rados.0.smithi136.stdout:1696: done (0 left) 2024-09-12T23:44:08.323 INFO:tasks.rados.rados.0.smithi136.stdout:1697: rollback oid 40 current snap is 160 2024-09-12T23:44:08.323 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 40 to 158 2024-09-12T23:44:08.323 INFO:tasks.rados.rados.0.smithi136.stdout:1698: read oid 42 snap -1 2024-09-12T23:44:08.324 INFO:tasks.rados.rados.0.smithi136.stdout:1698: expect deleted 2024-09-12T23:44:08.324 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 47 current snap is 160 2024-09-12T23:44:08.324 INFO:tasks.rados.rados.0.smithi136.stdout:1699: seq_num 557 ranges {507904=532480,1040384=802816,1843200=794624,2637824=8192} 2024-09-12T23:44:08.337 INFO:tasks.rados.rados.0.smithi136.stdout:1699: writing smithi13667081-47 from 507904 to 1040384 tid 1 2024-09-12T23:44:08.347 INFO:tasks.rados.rados.0.smithi136.stdout:1699: writing smithi13667081-47 from 1040384 to 1843200 tid 2 2024-09-12T23:44:08.359 INFO:tasks.rados.rados.0.smithi136.stdout:1699: writing smithi13667081-47 from 1843200 to 2637824 tid 3 2024-09-12T23:44:08.359 INFO:tasks.rados.rados.0.smithi136.stdout:1699: writing smithi13667081-47 from 2637824 to 2646016 tid 4 2024-09-12T23:44:08.359 INFO:tasks.rados.rados.0.smithi136.stdout:1700: read oid 9 snap -1 2024-09-12T23:44:08.359 INFO:tasks.rados.rados.0.smithi136.stdout:1700: expect (ObjNum 513 snap 150 seq_num 513) 2024-09-12T23:44:08.359 INFO:tasks.rados.rados.0.smithi136.stdout:1701: read oid 33 snap -1 2024-09-12T23:44:08.359 INFO:tasks.rados.rados.0.smithi136.stdout:1701: expect deleted 2024-09-12T23:44:08.359 INFO:tasks.rados.rados.0.smithi136.stdout:1702: rollback oid 43 current snap is 160 2024-09-12T23:44:08.359 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 43 to 159 2024-09-12T23:44:08.359 INFO:tasks.rados.rados.0.smithi136.stdout:1697: finishing rollback tid 0 to smithi13667081-40 2024-09-12T23:44:08.359 INFO:tasks.rados.rados.0.smithi136.stdout:1697: finishing rollback tid 1 to smithi13667081-40 2024-09-12T23:44:08.359 INFO:tasks.rados.rados.0.smithi136.stdout:1697: finishing rollback tid 2 to smithi13667081-40 2024-09-12T23:44:08.360 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 40 v 1260 (ObjNum 542 snap 157 seq_num 542) dirty exists 2024-09-12T23:44:08.360 INFO:tasks.rados.rados.0.smithi136.stdout:1699: finishing write tid 1 to smithi13667081-47 2024-09-12T23:44:08.360 INFO:tasks.rados.rados.0.smithi136.stdout:1699: finishing write tid 2 to smithi13667081-47 2024-09-12T23:44:08.360 INFO:tasks.rados.rados.0.smithi136.stdout:1697: done (5 left) 2024-09-12T23:44:08.360 INFO:tasks.rados.rados.0.smithi136.stdout:1698: done (4 left) 2024-09-12T23:44:08.360 INFO:tasks.rados.rados.0.smithi136.stdout:1703: delete oid 46 current snap is 160 2024-09-12T23:44:08.370 INFO:tasks.rados.rados.0.smithi136.stdout:1701: done (4 left) 2024-09-12T23:44:08.370 INFO:tasks.rados.rados.0.smithi136.stdout:1703: done (3 left) 2024-09-12T23:44:08.370 INFO:tasks.rados.rados.0.smithi136.stdout:1704: read oid 8 snap 158 2024-09-12T23:44:08.371 INFO:tasks.rados.rados.0.smithi136.stdout:1704: expect (ObjNum 523 snap 153 seq_num 523) 2024-09-12T23:44:08.371 INFO:tasks.rados.rados.0.smithi136.stdout:1705: read oid 28 snap -1 2024-09-12T23:44:08.371 INFO:tasks.rados.rados.0.smithi136.stdout:1705: expect (ObjNum 391 snap 113 seq_num 391) 2024-09-12T23:44:08.371 INFO:tasks.rados.rados.0.smithi136.stdout:1706: copy_from oid 20 from oid 11 current snap is 160 2024-09-12T23:44:08.371 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 6 current snap is 160 2024-09-12T23:44:08.371 INFO:tasks.rados.rados.0.smithi136.stdout:1707: seq_num 559 ranges {1835008=565248,2400256=573440,2973696=409600,3383296=622592} 2024-09-12T23:44:08.401 INFO:tasks.rados.rados.0.smithi136.stdout:1707: writing smithi13667081-6 from 1835008 to 2400256 tid 1 2024-09-12T23:44:08.407 INFO:tasks.rados.rados.0.smithi136.stdout:1707: writing smithi13667081-6 from 2400256 to 2973696 tid 2 2024-09-12T23:44:08.411 INFO:tasks.rados.rados.0.smithi136.stdout:1707: writing smithi13667081-6 from 2973696 to 3383296 tid 3 2024-09-12T23:44:08.417 INFO:tasks.rados.rados.0.smithi136.stdout:1707: writing smithi13667081-6 from 3383296 to 4005888 tid 4 2024-09-12T23:44:08.418 INFO:tasks.rados.rados.0.smithi136.stdout:1708: setattr oid 46 current snap is 160 2024-09-12T23:44:08.418 INFO:tasks.rados.rados.0.smithi136.stdout:1699: finishing write tid 3 to smithi13667081-47 2024-09-12T23:44:08.418 INFO:tasks.rados.rados.0.smithi136.stdout:1699: finishing write tid 4 to smithi13667081-47 2024-09-12T23:44:08.418 INFO:tasks.rados.rados.0.smithi136.stdout:1702: finishing rollback tid 0 to smithi13667081-43 2024-09-12T23:44:08.418 INFO:tasks.rados.rados.0.smithi136.stdout:1702: finishing rollback tid 1 to smithi13667081-43 2024-09-12T23:44:08.418 INFO:tasks.rados.rados.0.smithi136.stdout:1702: finishing rollback tid 2 to smithi13667081-43 2024-09-12T23:44:08.418 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 43 v 1069 (ObjNum 552 snap 159 seq_num 552) dirty exists 2024-09-12T23:44:08.418 INFO:tasks.rados.rados.0.smithi136.stdout:1699: finishing write tid 5 to smithi13667081-47 2024-09-12T23:44:08.418 INFO:tasks.rados.rados.0.smithi136.stdout:1706: finishing copy_from racing read to smithi13667081-20 2024-09-12T23:44:08.418 INFO:tasks.rados.rados.0.smithi136.stdout:1706: finishing copy_from to smithi13667081-20 2024-09-12T23:44:08.418 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 20 v 1198 (ObjNum 482 snap 142 seq_num 482) dirty exists 2024-09-12T23:44:08.418 INFO:tasks.rados.rados.0.smithi136.stdout:1707: finishing write tid 1 to smithi13667081-6 2024-09-12T23:44:08.418 INFO:tasks.rados.rados.0.smithi136.stdout:1707: finishing write tid 2 to smithi13667081-6 2024-09-12T23:44:08.419 INFO:tasks.rados.rados.0.smithi136.stdout:1700: expect (ObjNum 513 snap 150 seq_num 513) 2024-09-12T23:44:08.944 INFO:tasks.rados.rados.0.smithi136.stdout:1699: finishing write tid 6 to smithi13667081-47 2024-09-12T23:44:08.944 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 47 v 634 (ObjNum 557 snap 160 seq_num 557) dirty exists 2024-09-12T23:44:08.944 INFO:tasks.rados.rados.0.smithi136.stdout:1699: left oid 47 (ObjNum 557 snap 160 seq_num 557) 2024-09-12T23:44:08.944 INFO:tasks.rados.rados.0.smithi136.stdout:1707: finishing write tid 3 to smithi13667081-6 2024-09-12T23:44:08.944 INFO:tasks.rados.rados.0.smithi136.stdout:1705: expect (ObjNum 391 snap 113 seq_num 391) 2024-09-12T23:44:09.094 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:44:08 smithi181 bash[60471]: ts=2024-09-12T23:44:08.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:44:09.184 INFO:tasks.rados.rados.0.smithi136.stdout:1707: finishing write tid 4 to smithi13667081-6 2024-09-12T23:44:09.184 INFO:tasks.rados.rados.0.smithi136.stdout:1707: finishing write tid 5 to smithi13667081-6 2024-09-12T23:44:09.184 INFO:tasks.rados.rados.0.smithi136.stdout:1707: finishing write tid 6 to smithi13667081-6 2024-09-12T23:44:09.185 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 6 v 1203 (ObjNum 559 snap 160 seq_num 559) dirty exists 2024-09-12T23:44:09.185 INFO:tasks.rados.rados.0.smithi136.stdout:1707: left oid 6 (ObjNum 559 snap 160 seq_num 559) 2024-09-12T23:44:09.185 INFO:tasks.rados.rados.0.smithi136.stdout:1704: expect (ObjNum 523 snap 153 seq_num 523) 2024-09-12T23:44:09.504 INFO:tasks.rados.rados.0.smithi136.stdout:1699: done (7 left) 2024-09-12T23:44:09.504 INFO:tasks.rados.rados.0.smithi136.stdout:1700: done (6 left) 2024-09-12T23:44:09.504 INFO:tasks.rados.rados.0.smithi136.stdout:1702: done (5 left) 2024-09-12T23:44:09.504 INFO:tasks.rados.rados.0.smithi136.stdout:1704: done (4 left) 2024-09-12T23:44:09.504 INFO:tasks.rados.rados.0.smithi136.stdout:1705: done (3 left) 2024-09-12T23:44:09.504 INFO:tasks.rados.rados.0.smithi136.stdout:1706: done (2 left) 2024-09-12T23:44:09.504 INFO:tasks.rados.rados.0.smithi136.stdout:1707: done (1 left) 2024-09-12T23:44:09.505 INFO:tasks.rados.rados.0.smithi136.stdout:1709: snap_create 2024-09-12T23:44:09.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:09 smithi181 bash[67587]: cluster 2024-09-12T23:44:08.325787+0000 mon.a (mon.0) 339 : cluster [DBG] osdmap e394: 8 total, 7 up, 8 in 2024-09-12T23:44:09.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:09 smithi181 bash[67587]: cluster 2024-09-12T23:44:08.688623+0000 mgr.y (mgr.24661) 124 : cluster [DBG] pgmap v166: 121 pgs: 16 stale+active+clean, 105 active+clean; 189 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:44:09.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:09 smithi181 bash[67587]: audit 2024-09-12T23:44:08.690779+0000 mon.a (mon.0) 340 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.3", "id": [7, 0]}]: dispatch 2024-09-12T23:44:09.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:09 smithi181 bash[67587]: audit 2024-09-12T23:44:08.691371+0000 mon.a (mon.0) 341 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.a", "id": [7, 0]}]: dispatch 2024-09-12T23:44:09.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:09 smithi181 bash[67587]: audit 2024-09-12T23:44:08.691870+0000 mon.a (mon.0) 342 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1a", "id": [7, 0]}]: dispatch 2024-09-12T23:44:09.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:09 smithi181 bash[67587]: audit 2024-09-12T23:44:08.692286+0000 mon.a (mon.0) 343 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1d", "id": [7, 4]}]: dispatch 2024-09-12T23:44:09.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:09 smithi181 bash[67587]: audit 2024-09-12T23:44:08.692629+0000 mon.a (mon.0) 344 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [7, 2]}]: dispatch 2024-09-12T23:44:09.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:09 smithi181 bash[67587]: audit 2024-09-12T23:44:08.693001+0000 mon.a (mon.0) 345 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 0]}]: dispatch 2024-09-12T23:44:09.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:09 smithi181 bash[67587]: audit 2024-09-12T23:44:08.693327+0000 mon.a (mon.0) 346 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.11", "id": [7, 0]}]: dispatch 2024-09-12T23:44:09.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:09 smithi181 bash[67587]: audit 2024-09-12T23:44:08.693647+0000 mon.a (mon.0) 347 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3e", "id": [6, 2]}]: dispatch 2024-09-12T23:44:09.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:09 smithi181 bash[67587]: audit 2024-09-12T23:44:08.693970+0000 mon.a (mon.0) 348 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.d", "id": [5, 0]}]: dispatch 2024-09-12T23:44:09.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:09 smithi181 bash[67587]: audit 2024-09-12T23:44:08.694276+0000 mon.a (mon.0) 349 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "3.34"}]: dispatch 2024-09-12T23:44:09.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:09 smithi181 bash[67587]: audit 2024-09-12T23:44:08.751565+0000 mon.a (mon.0) 350 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:09.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:09 smithi181 bash[67587]: audit 2024-09-12T23:44:08.753897+0000 mon.a (mon.0) 351 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:44:09.740 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:44:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:44:09] "GET /metrics HTTP/1.1" 200 36254 "" "Prometheus/2.43.0" 2024-09-12T23:44:09.740 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:09 smithi136 bash[76371]: cluster 2024-09-12T23:44:08.325787+0000 mon.a (mon.0) 339 : cluster [DBG] osdmap e394: 8 total, 7 up, 8 in 2024-09-12T23:44:09.740 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:09 smithi136 bash[76371]: cluster 2024-09-12T23:44:08.688623+0000 mgr.y (mgr.24661) 124 : cluster [DBG] pgmap v166: 121 pgs: 16 stale+active+clean, 105 active+clean; 189 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:44:09.741 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:09 smithi136 bash[76371]: audit 2024-09-12T23:44:08.690779+0000 mon.a (mon.0) 340 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.3", "id": [7, 0]}]: dispatch 2024-09-12T23:44:09.741 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:09 smithi136 bash[76371]: audit 2024-09-12T23:44:08.691371+0000 mon.a (mon.0) 341 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.a", "id": [7, 0]}]: dispatch 2024-09-12T23:44:09.741 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:09 smithi136 bash[76371]: audit 2024-09-12T23:44:08.691870+0000 mon.a (mon.0) 342 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1a", "id": [7, 0]}]: dispatch 2024-09-12T23:44:09.741 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:09 smithi136 bash[76371]: audit 2024-09-12T23:44:08.692286+0000 mon.a (mon.0) 343 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1d", "id": [7, 4]}]: dispatch 2024-09-12T23:44:09.741 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:09 smithi136 bash[76371]: audit 2024-09-12T23:44:08.692629+0000 mon.a (mon.0) 344 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [7, 2]}]: dispatch 2024-09-12T23:44:09.741 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:09 smithi136 bash[76371]: audit 2024-09-12T23:44:08.693001+0000 mon.a (mon.0) 345 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 0]}]: dispatch 2024-09-12T23:44:09.741 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:09 smithi136 bash[76371]: audit 2024-09-12T23:44:08.693327+0000 mon.a (mon.0) 346 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.11", "id": [7, 0]}]: dispatch 2024-09-12T23:44:09.741 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:09 smithi136 bash[76371]: audit 2024-09-12T23:44:08.693647+0000 mon.a (mon.0) 347 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3e", "id": [6, 2]}]: dispatch 2024-09-12T23:44:09.741 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:09 smithi136 bash[76371]: audit 2024-09-12T23:44:08.693970+0000 mon.a (mon.0) 348 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.d", "id": [5, 0]}]: dispatch 2024-09-12T23:44:09.741 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:09 smithi136 bash[76371]: audit 2024-09-12T23:44:08.694276+0000 mon.a (mon.0) 349 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "3.34"}]: dispatch 2024-09-12T23:44:09.741 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:09 smithi136 bash[76371]: audit 2024-09-12T23:44:08.751565+0000 mon.a (mon.0) 350 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:09.741 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:09 smithi136 bash[76371]: audit 2024-09-12T23:44:08.753897+0000 mon.a (mon.0) 351 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:44:09.742 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:09 smithi136 bash[75164]: cluster 2024-09-12T23:44:08.325787+0000 mon.a (mon.0) 339 : cluster [DBG] osdmap e394: 8 total, 7 up, 8 in 2024-09-12T23:44:09.742 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:09 smithi136 bash[75164]: cluster 2024-09-12T23:44:08.688623+0000 mgr.y (mgr.24661) 124 : cluster [DBG] pgmap v166: 121 pgs: 16 stale+active+clean, 105 active+clean; 189 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:44:09.742 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:09 smithi136 bash[75164]: audit 2024-09-12T23:44:08.690779+0000 mon.a (mon.0) 340 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.3", "id": [7, 0]}]: dispatch 2024-09-12T23:44:09.742 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:09 smithi136 bash[75164]: audit 2024-09-12T23:44:08.691371+0000 mon.a (mon.0) 341 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.a", "id": [7, 0]}]: dispatch 2024-09-12T23:44:09.742 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:09 smithi136 bash[75164]: audit 2024-09-12T23:44:08.691870+0000 mon.a (mon.0) 342 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1a", "id": [7, 0]}]: dispatch 2024-09-12T23:44:09.742 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:09 smithi136 bash[75164]: audit 2024-09-12T23:44:08.692286+0000 mon.a (mon.0) 343 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1d", "id": [7, 4]}]: dispatch 2024-09-12T23:44:09.742 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:09 smithi136 bash[75164]: audit 2024-09-12T23:44:08.692629+0000 mon.a (mon.0) 344 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [7, 2]}]: dispatch 2024-09-12T23:44:09.743 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:09 smithi136 bash[75164]: audit 2024-09-12T23:44:08.693001+0000 mon.a (mon.0) 345 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 0]}]: dispatch 2024-09-12T23:44:09.743 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:09 smithi136 bash[75164]: audit 2024-09-12T23:44:08.693327+0000 mon.a (mon.0) 346 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.11", "id": [7, 0]}]: dispatch 2024-09-12T23:44:09.743 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:09 smithi136 bash[75164]: audit 2024-09-12T23:44:08.693647+0000 mon.a (mon.0) 347 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3e", "id": [6, 2]}]: dispatch 2024-09-12T23:44:09.743 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:09 smithi136 bash[75164]: audit 2024-09-12T23:44:08.693970+0000 mon.a (mon.0) 348 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.d", "id": [5, 0]}]: dispatch 2024-09-12T23:44:09.743 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:09 smithi136 bash[75164]: audit 2024-09-12T23:44:08.694276+0000 mon.a (mon.0) 349 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "3.34"}]: dispatch 2024-09-12T23:44:09.743 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:09 smithi136 bash[75164]: audit 2024-09-12T23:44:08.751565+0000 mon.a (mon.0) 350 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:09.743 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:09 smithi136 bash[75164]: audit 2024-09-12T23:44:08.753897+0000 mon.a (mon.0) 351 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:44:10.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:09 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:44:10.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:09 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:44:10.006 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 12 23:44:09 smithi136 systemd[1]: ceph-da58a47c-715d-11ef-bceb-c7b262605968@osd.0.service: Deactivated successfully. 2024-09-12T23:44:10.006 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 12 23:44:09 smithi136 systemd[1]: Stopped Ceph osd.0 for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:44:10.007 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 12 23:44:09 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:44:10.007 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 12 23:44:09 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:44:10.007 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 12 23:44:09 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:44:10.007 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:44:09 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:44:10.008 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:44:09 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:44:10.008 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:44:09 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:44:10.008 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 12 23:44:09 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:44:10.008 INFO:journalctl@ceph.mds.a.smithi136.stdout:Sep 12 23:44:09 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:44:10.329 INFO:tasks.rados.rados.0.smithi136.stdout:1709: done (1 left) 2024-09-12T23:44:10.330 INFO:tasks.rados.rados.0.smithi136.stdout:1710: delete oid 42 current snap is 161 2024-09-12T23:44:10.336 INFO:tasks.rados.rados.0.smithi136.stdout:1710: done (1 left) 2024-09-12T23:44:10.336 INFO:tasks.rados.rados.0.smithi136.stdout:1711: snap_create 2024-09-12T23:44:10.338 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 12 23:44:10 smithi136 systemd[1]: Started Ceph osd.0 for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:44:10.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:10 smithi136 bash[75164]: audit 2024-09-12T23:44:09.338368+0000 mon.a (mon.0) 352 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.3", "id": [7, 0]}]': finished 2024-09-12T23:44:10.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:10 smithi136 bash[75164]: audit 2024-09-12T23:44:09.338557+0000 mon.a (mon.0) 353 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.a", "id": [7, 0]}]': finished 2024-09-12T23:44:10.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:10 smithi136 bash[75164]: audit 2024-09-12T23:44:09.338724+0000 mon.a (mon.0) 354 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1a", "id": [7, 0]}]': finished 2024-09-12T23:44:10.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:10 smithi136 bash[75164]: audit 2024-09-12T23:44:09.338900+0000 mon.a (mon.0) 355 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1d", "id": [7, 4]}]': finished 2024-09-12T23:44:10.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:10 smithi136 bash[75164]: audit 2024-09-12T23:44:09.339064+0000 mon.a (mon.0) 356 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [7, 2]}]': finished 2024-09-12T23:44:10.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:10 smithi136 bash[75164]: audit 2024-09-12T23:44:09.339220+0000 mon.a (mon.0) 357 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 0]}]': finished 2024-09-12T23:44:10.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:10 smithi136 bash[75164]: audit 2024-09-12T23:44:09.339388+0000 mon.a (mon.0) 358 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.11", "id": [7, 0]}]': finished 2024-09-12T23:44:10.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:10 smithi136 bash[75164]: audit 2024-09-12T23:44:09.339547+0000 mon.a (mon.0) 359 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3e", "id": [6, 2]}]': finished 2024-09-12T23:44:10.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:10 smithi136 bash[75164]: audit 2024-09-12T23:44:09.339701+0000 mon.a (mon.0) 360 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.d", "id": [5, 0]}]': finished 2024-09-12T23:44:10.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:10 smithi136 bash[75164]: audit 2024-09-12T23:44:09.339846+0000 mon.a (mon.0) 361 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "3.34"}]': finished 2024-09-12T23:44:10.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:10 smithi136 bash[75164]: cluster 2024-09-12T23:44:09.339968+0000 mon.a (mon.0) 362 : cluster [DBG] osdmap e395: 8 total, 7 up, 8 in 2024-09-12T23:44:10.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:10 smithi136 bash[75164]: audit 2024-09-12T23:44:10.047550+0000 mon.a (mon.0) 363 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:10.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:10 smithi136 bash[75164]: audit 2024-09-12T23:44:10.053574+0000 mon.a (mon.0) 364 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:10.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:10 smithi136 bash[75164]: audit 2024-09-12T23:44:10.054522+0000 mon.a (mon.0) 365 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:44:10.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:10 smithi136 bash[76371]: audit 2024-09-12T23:44:09.338368+0000 mon.a (mon.0) 352 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.3", "id": [7, 0]}]': finished 2024-09-12T23:44:10.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:10 smithi136 bash[76371]: audit 2024-09-12T23:44:09.338557+0000 mon.a (mon.0) 353 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.a", "id": [7, 0]}]': finished 2024-09-12T23:44:10.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:10 smithi136 bash[76371]: audit 2024-09-12T23:44:09.338724+0000 mon.a (mon.0) 354 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1a", "id": [7, 0]}]': finished 2024-09-12T23:44:10.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:10 smithi136 bash[76371]: audit 2024-09-12T23:44:09.338900+0000 mon.a (mon.0) 355 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1d", "id": [7, 4]}]': finished 2024-09-12T23:44:10.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:10 smithi136 bash[76371]: audit 2024-09-12T23:44:09.339064+0000 mon.a (mon.0) 356 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [7, 2]}]': finished 2024-09-12T23:44:10.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:10 smithi136 bash[76371]: audit 2024-09-12T23:44:09.339220+0000 mon.a (mon.0) 357 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 0]}]': finished 2024-09-12T23:44:10.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:10 smithi136 bash[76371]: audit 2024-09-12T23:44:09.339388+0000 mon.a (mon.0) 358 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.11", "id": [7, 0]}]': finished 2024-09-12T23:44:10.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:10 smithi136 bash[76371]: audit 2024-09-12T23:44:09.339547+0000 mon.a (mon.0) 359 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3e", "id": [6, 2]}]': finished 2024-09-12T23:44:10.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:10 smithi136 bash[76371]: audit 2024-09-12T23:44:09.339701+0000 mon.a (mon.0) 360 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.d", "id": [5, 0]}]': finished 2024-09-12T23:44:10.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:10 smithi136 bash[76371]: audit 2024-09-12T23:44:09.339846+0000 mon.a (mon.0) 361 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "3.34"}]': finished 2024-09-12T23:44:10.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:10 smithi136 bash[76371]: cluster 2024-09-12T23:44:09.339968+0000 mon.a (mon.0) 362 : cluster [DBG] osdmap e395: 8 total, 7 up, 8 in 2024-09-12T23:44:10.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:10 smithi136 bash[76371]: audit 2024-09-12T23:44:10.047550+0000 mon.a (mon.0) 363 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:10.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:10 smithi136 bash[76371]: audit 2024-09-12T23:44:10.053574+0000 mon.a (mon.0) 364 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:10.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:10 smithi136 bash[76371]: audit 2024-09-12T23:44:10.054522+0000 mon.a (mon.0) 365 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:44:10.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:10 smithi181 bash[67587]: audit 2024-09-12T23:44:09.338368+0000 mon.a (mon.0) 352 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.3", "id": [7, 0]}]': finished 2024-09-12T23:44:10.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:10 smithi181 bash[67587]: audit 2024-09-12T23:44:09.338557+0000 mon.a (mon.0) 353 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.a", "id": [7, 0]}]': finished 2024-09-12T23:44:10.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:10 smithi181 bash[67587]: audit 2024-09-12T23:44:09.338724+0000 mon.a (mon.0) 354 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1a", "id": [7, 0]}]': finished 2024-09-12T23:44:10.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:10 smithi181 bash[67587]: audit 2024-09-12T23:44:09.338900+0000 mon.a (mon.0) 355 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1d", "id": [7, 4]}]': finished 2024-09-12T23:44:10.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:10 smithi181 bash[67587]: audit 2024-09-12T23:44:09.339064+0000 mon.a (mon.0) 356 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [7, 2]}]': finished 2024-09-12T23:44:10.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:10 smithi181 bash[67587]: audit 2024-09-12T23:44:09.339220+0000 mon.a (mon.0) 357 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 0]}]': finished 2024-09-12T23:44:10.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:10 smithi181 bash[67587]: audit 2024-09-12T23:44:09.339388+0000 mon.a (mon.0) 358 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.11", "id": [7, 0]}]': finished 2024-09-12T23:44:10.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:10 smithi181 bash[67587]: audit 2024-09-12T23:44:09.339547+0000 mon.a (mon.0) 359 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3e", "id": [6, 2]}]': finished 2024-09-12T23:44:10.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:10 smithi181 bash[67587]: audit 2024-09-12T23:44:09.339701+0000 mon.a (mon.0) 360 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.d", "id": [5, 0]}]': finished 2024-09-12T23:44:10.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:10 smithi181 bash[67587]: audit 2024-09-12T23:44:09.339846+0000 mon.a (mon.0) 361 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "3.34"}]': finished 2024-09-12T23:44:10.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:10 smithi181 bash[67587]: cluster 2024-09-12T23:44:09.339968+0000 mon.a (mon.0) 362 : cluster [DBG] osdmap e395: 8 total, 7 up, 8 in 2024-09-12T23:44:10.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:10 smithi181 bash[67587]: audit 2024-09-12T23:44:10.047550+0000 mon.a (mon.0) 363 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:10.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:10 smithi181 bash[67587]: audit 2024-09-12T23:44:10.053574+0000 mon.a (mon.0) 364 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:10.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:10 smithi181 bash[67587]: audit 2024-09-12T23:44:10.054522+0000 mon.a (mon.0) 365 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:44:11.332 INFO:tasks.rados.rados.0.smithi136.stdout:1711: done (1 left) 2024-09-12T23:44:11.332 INFO:tasks.rados.rados.0.smithi136.stdout:1712: snap_remove snap 158 2024-09-12T23:44:11.727 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:11 smithi136 bash[75164]: cluster 2024-09-12T23:44:10.333367+0000 mon.a (mon.0) 366 : cluster [DBG] osdmap e396: 8 total, 7 up, 8 in 2024-09-12T23:44:11.727 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:11 smithi136 bash[75164]: cluster 2024-09-12T23:44:10.689670+0000 mgr.y (mgr.24661) 125 : cluster [DBG] pgmap v169: 121 pgs: 8 peering, 38 active+undersized, 1 stale+active+clean, 12 active+undersized+degraded, 62 active+clean; 193 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 31 MiB/s rd, 6.9 MiB/s wr, 31 op/s; 36/252 objects degraded (14.286%) 2024-09-12T23:44:11.727 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:11 smithi136 bash[75164]: cluster 2024-09-12T23:44:11.335135+0000 mon.a (mon.0) 367 : cluster [DBG] osdmap e397: 8 total, 7 up, 8 in 2024-09-12T23:44:11.727 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:11 smithi136 bash[76371]: cluster 2024-09-12T23:44:10.333367+0000 mon.a (mon.0) 366 : cluster [DBG] osdmap e396: 8 total, 7 up, 8 in 2024-09-12T23:44:11.727 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:11 smithi136 bash[76371]: cluster 2024-09-12T23:44:10.689670+0000 mgr.y (mgr.24661) 125 : cluster [DBG] pgmap v169: 121 pgs: 8 peering, 38 active+undersized, 1 stale+active+clean, 12 active+undersized+degraded, 62 active+clean; 193 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 31 MiB/s rd, 6.9 MiB/s wr, 31 op/s; 36/252 objects degraded (14.286%) 2024-09-12T23:44:11.727 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:11 smithi136 bash[76371]: cluster 2024-09-12T23:44:11.335135+0000 mon.a (mon.0) 367 : cluster [DBG] osdmap e397: 8 total, 7 up, 8 in 2024-09-12T23:44:11.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:11 smithi181 bash[67587]: cluster 2024-09-12T23:44:10.333367+0000 mon.a (mon.0) 366 : cluster [DBG] osdmap e396: 8 total, 7 up, 8 in 2024-09-12T23:44:11.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:11 smithi181 bash[67587]: cluster 2024-09-12T23:44:10.689670+0000 mgr.y (mgr.24661) 125 : cluster [DBG] pgmap v169: 121 pgs: 8 peering, 38 active+undersized, 1 stale+active+clean, 12 active+undersized+degraded, 62 active+clean; 193 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 31 MiB/s rd, 6.9 MiB/s wr, 31 op/s; 36/252 objects degraded (14.286%) 2024-09-12T23:44:11.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:11 smithi181 bash[67587]: cluster 2024-09-12T23:44:11.335135+0000 mon.a (mon.0) 367 : cluster [DBG] osdmap e397: 8 total, 7 up, 8 in 2024-09-12T23:44:12.006 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 12 23:44:11 smithi136 bash[80576]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-12T23:44:12.006 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 12 23:44:11 smithi136 bash[80576]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-12T23:44:12.006 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 12 23:44:11 smithi136 bash[80576]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-12T23:44:12.006 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 12 23:44:11 smithi136 bash[80576]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-12T23:44:12.006 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 12 23:44:11 smithi136 bash[80576]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-12T23:44:12.006 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 12 23:44:11 smithi136 bash[80576]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-12T23:44:12.007 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 12 23:44:11 smithi136 bash[80576]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-12T23:44:12.339 INFO:tasks.rados.rados.0.smithi136.stdout:1712: done (1 left) 2024-09-12T23:44:12.339 INFO:tasks.rados.rados.0.smithi136.stdout:1713: delete oid 17 current snap is 162 2024-09-12T23:44:12.339 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 46 v 621 (ObjNum 81 snap 0 seq_num 2877405187) dirty exists 2024-09-12T23:44:12.350 INFO:tasks.rados.rados.0.smithi136.stdout:1708: done (1 left) 2024-09-12T23:44:12.350 INFO:tasks.rados.rados.0.smithi136.stdout:1713: done (0 left) 2024-09-12T23:44:12.350 INFO:tasks.rados.rados.0.smithi136.stdout:1714: delete oid 21 current snap is 162 2024-09-12T23:44:12.362 INFO:tasks.rados.rados.0.smithi136.stdout:1714: done (0 left) 2024-09-12T23:44:12.362 INFO:tasks.rados.rados.0.smithi136.stdout:1715: snap_remove snap 161 2024-09-12T23:44:12.594 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:44:12 smithi181 bash[60471]: ts=2024-09-12T23:44:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:44:12.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:12 smithi181 bash[67587]: cluster 2024-09-12T23:44:11.343374+0000 mon.a (mon.0) 368 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-12T23:44:12.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:12 smithi181 bash[67587]: cluster 2024-09-12T23:44:11.343410+0000 mon.a (mon.0) 369 : cluster [WRN] Health check failed: Degraded data redundancy: 36/252 objects degraded (14.286%), 12 pgs degraded (PG_DEGRADED) 2024-09-12T23:44:12.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:12 smithi181 bash[67587]: cluster 2024-09-12T23:44:12.342364+0000 mon.a (mon.0) 370 : cluster [DBG] osdmap e398: 8 total, 7 up, 8 in 2024-09-12T23:44:12.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:12 smithi136 bash[75164]: cluster 2024-09-12T23:44:11.343374+0000 mon.a (mon.0) 368 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-12T23:44:12.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:12 smithi136 bash[75164]: cluster 2024-09-12T23:44:11.343410+0000 mon.a (mon.0) 369 : cluster [WRN] Health check failed: Degraded data redundancy: 36/252 objects degraded (14.286%), 12 pgs degraded (PG_DEGRADED) 2024-09-12T23:44:12.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:12 smithi136 bash[75164]: cluster 2024-09-12T23:44:12.342364+0000 mon.a (mon.0) 370 : cluster [DBG] osdmap e398: 8 total, 7 up, 8 in 2024-09-12T23:44:12.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:12 smithi136 bash[76371]: cluster 2024-09-12T23:44:11.343374+0000 mon.a (mon.0) 368 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-12T23:44:12.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:12 smithi136 bash[76371]: cluster 2024-09-12T23:44:11.343410+0000 mon.a (mon.0) 369 : cluster [WRN] Health check failed: Degraded data redundancy: 36/252 objects degraded (14.286%), 12 pgs degraded (PG_DEGRADED) 2024-09-12T23:44:12.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:12 smithi136 bash[76371]: cluster 2024-09-12T23:44:12.342364+0000 mon.a (mon.0) 370 : cluster [DBG] osdmap e398: 8 total, 7 up, 8 in 2024-09-12T23:44:13.361 INFO:tasks.rados.rados.0.smithi136.stdout:1715: done (0 left) 2024-09-12T23:44:13.361 INFO:tasks.rados.rados.0.smithi136.stdout:1716: snap_remove snap 160 2024-09-12T23:44:13.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:13 smithi136 bash[75164]: cluster 2024-09-12T23:44:12.690664+0000 mgr.y (mgr.24661) 126 : cluster [DBG] pgmap v172: 121 pgs: 8 peering, 38 active+undersized, 1 stale+active+clean, 12 active+undersized+degraded, 62 active+clean; 193 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 31 MiB/s rd, 6.9 MiB/s wr, 31 op/s; 36/252 objects degraded (14.286%) 2024-09-12T23:44:13.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:13 smithi136 bash[76371]: cluster 2024-09-12T23:44:12.690664+0000 mgr.y (mgr.24661) 126 : cluster [DBG] pgmap v172: 121 pgs: 8 peering, 38 active+undersized, 1 stale+active+clean, 12 active+undersized+degraded, 62 active+clean; 193 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 31 MiB/s rd, 6.9 MiB/s wr, 31 op/s; 36/252 objects degraded (14.286%) 2024-09-12T23:44:13.757 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 12 23:44:13 smithi136 bash[80792]: debug 2024-09-12T23:44:13.526+0000 7f046b2b6640 1 -- 172.21.15.136:0/1071884736 <== mon.0 v2:172.21.15.136:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x56483b8c91e0 con 0x56483b889c00 2024-09-12T23:44:13.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:13 smithi181 bash[67587]: cluster 2024-09-12T23:44:12.690664+0000 mgr.y (mgr.24661) 126 : cluster [DBG] pgmap v172: 121 pgs: 8 peering, 38 active+undersized, 1 stale+active+clean, 12 active+undersized+degraded, 62 active+clean; 193 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 31 MiB/s rd, 6.9 MiB/s wr, 31 op/s; 36/252 objects degraded (14.286%) 2024-09-12T23:44:13.844 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:44:13 smithi181 bash[60471]: ts=2024-09-12T23:44:13.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi136\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:44:14.358 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 12 23:44:14 smithi136 bash[80792]: debug 2024-09-12T23:44:14.074+0000 7f046db1c640 -1 Falling back to public interface 2024-09-12T23:44:14.375 INFO:tasks.rados.rados.0.smithi136.stdout:1716: done (0 left) 2024-09-12T23:44:14.375 INFO:tasks.rados.rados.0.smithi136.stdout:1717: copy_from oid 11 from oid 16 current snap is 162 2024-09-12T23:44:14.375 INFO:tasks.rados.rados.0.smithi136.stdout:1718: rollback oid 40 current snap is 162 2024-09-12T23:44:14.375 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 40 to 159 2024-09-12T23:44:14.380 INFO:tasks.rados.rados.0.smithi136.stdout:1719: read oid 17 snap -1 2024-09-12T23:44:14.380 INFO:tasks.rados.rados.0.smithi136.stdout:1719: expect deleted 2024-09-12T23:44:14.380 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 3 current snap is 162 2024-09-12T23:44:14.380 INFO:tasks.rados.rados.0.smithi136.stdout:1720: seq_num 563 ranges {1785856=638976,2424832=565248,2990080=794624,3784704=245760} 2024-09-12T23:44:14.397 INFO:tasks.rados.rados.0.smithi136.stdout:1720: writing smithi13667081-3 from 1785856 to 2424832 tid 1 2024-09-12T23:44:14.401 INFO:tasks.rados.rados.0.smithi136.stdout:1720: writing smithi13667081-3 from 2424832 to 2990080 tid 2 2024-09-12T23:44:14.407 INFO:tasks.rados.rados.0.smithi136.stdout:1720: writing smithi13667081-3 from 2990080 to 3784704 tid 3 2024-09-12T23:44:14.410 INFO:tasks.rados.rados.0.smithi136.stdout:1720: writing smithi13667081-3 from 3784704 to 4030464 tid 4 2024-09-12T23:44:14.410 INFO:tasks.rados.rados.0.smithi136.stdout:1721: copy_from oid 41 from oid 24 current snap is 162 2024-09-12T23:44:14.410 INFO:tasks.rados.rados.0.smithi136.stdout:1717: finishing copy_from to smithi13667081-11 2024-09-12T23:44:14.410 INFO:tasks.rados.rados.0.smithi136.stdout:1717: got expected ENOENT (src dne) 2024-09-12T23:44:14.410 INFO:tasks.rados.rados.0.smithi136.stdout:1717: finishing copy_from racing read to smithi13667081-11 2024-09-12T23:44:14.410 INFO:tasks.rados.rados.0.smithi136.stdout:1718: finishing rollback tid 0 to smithi13667081-40 2024-09-12T23:44:14.410 INFO:tasks.rados.rados.0.smithi136.stdout:1718: finishing rollback tid 1 to smithi13667081-40 2024-09-12T23:44:14.410 INFO:tasks.rados.rados.0.smithi136.stdout:1718: finishing rollback tid 2 to smithi13667081-40 2024-09-12T23:44:14.410 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 40 v 1265 (ObjNum 542 snap 157 seq_num 542) dirty exists 2024-09-12T23:44:14.410 INFO:tasks.rados.rados.0.smithi136.stdout:1720: finishing write tid 1 to smithi13667081-3 2024-09-12T23:44:14.410 INFO:tasks.rados.rados.0.smithi136.stdout:1720: finishing write tid 2 to smithi13667081-3 2024-09-12T23:44:14.410 INFO:tasks.rados.rados.0.smithi136.stdout:1717: done (4 left) 2024-09-12T23:44:14.411 INFO:tasks.rados.rados.0.smithi136.stdout:1718: done (3 left) 2024-09-12T23:44:14.411 INFO:tasks.rados.rados.0.smithi136.stdout:1719: done (2 left) 2024-09-12T23:44:14.411 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 44 current snap is 162 2024-09-12T23:44:14.411 INFO:tasks.rados.rados.0.smithi136.stdout:1722: seq_num 564 ranges {0=655360,655360=442368,1097728=598016,1695744=565248} 2024-09-12T23:44:14.415 INFO:tasks.rados.rados.0.smithi136.stdout:1722: writing smithi13667081-44 from 0 to 655360 tid 1 2024-09-12T23:44:14.418 INFO:tasks.rados.rados.0.smithi136.stdout:1722: writing smithi13667081-44 from 655360 to 1097728 tid 2 2024-09-12T23:44:14.423 INFO:tasks.rados.rados.0.smithi136.stdout:1722: writing smithi13667081-44 from 1097728 to 1695744 tid 3 2024-09-12T23:44:14.429 INFO:tasks.rados.rados.0.smithi136.stdout:1722: writing smithi13667081-44 from 1695744 to 2260992 tid 4 2024-09-12T23:44:14.429 INFO:tasks.rados.rados.0.smithi136.stdout:1723: copy_from oid 20 from oid 35 current snap is 162 2024-09-12T23:44:14.429 INFO:tasks.rados.rados.0.smithi136.stdout:1720: finishing write tid 3 to smithi13667081-3 2024-09-12T23:44:14.429 INFO:tasks.rados.rados.0.smithi136.stdout:1720: finishing write tid 4 to smithi13667081-3 2024-09-12T23:44:14.429 INFO:tasks.rados.rados.0.smithi136.stdout:1720: finishing write tid 5 to smithi13667081-3 2024-09-12T23:44:14.430 INFO:tasks.rados.rados.0.smithi136.stdout:1720: finishing write tid 6 to smithi13667081-3 2024-09-12T23:44:14.430 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 3 v 904 (ObjNum 563 snap 162 seq_num 563) dirty exists 2024-09-12T23:44:14.430 INFO:tasks.rados.rados.0.smithi136.stdout:1720: left oid 3 (ObjNum 563 snap 162 seq_num 563) 2024-09-12T23:44:14.430 INFO:tasks.rados.rados.0.smithi136.stdout:1720: done (3 left) 2024-09-12T23:44:14.430 INFO:tasks.rados.rados.0.smithi136.stdout:1724: delete oid 1 current snap is 162 2024-09-12T23:44:14.430 INFO:tasks.rados.rados.0.smithi136.stdout:1721: finishing copy_from racing read to smithi13667081-41 2024-09-12T23:44:14.430 INFO:tasks.rados.rados.0.smithi136.stdout:1722: finishing write tid 1 to smithi13667081-44 2024-09-12T23:44:14.430 INFO:tasks.rados.rados.0.smithi136.stdout:1721: finishing copy_from to smithi13667081-41 2024-09-12T23:44:14.430 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 41 v 619 (ObjNum 518 snap 153 seq_num 518) dirty exists 2024-09-12T23:44:14.430 INFO:tasks.rados.rados.0.smithi136.stdout:1722: finishing write tid 2 to smithi13667081-44 2024-09-12T23:44:14.430 INFO:tasks.rados.rados.0.smithi136.stdout:1722: finishing write tid 3 to smithi13667081-44 2024-09-12T23:44:14.432 INFO:tasks.rados.rados.0.smithi136.stdout:1721: done (3 left) 2024-09-12T23:44:14.432 INFO:tasks.rados.rados.0.smithi136.stdout:1724: done (2 left) 2024-09-12T23:44:14.432 INFO:tasks.rados.rados.0.smithi136.stdout:1725: snap_remove snap 159 2024-09-12T23:44:14.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:14 smithi136 bash[75164]: cluster 2024-09-12T23:44:13.364804+0000 mon.a (mon.0) 371 : cluster [DBG] osdmap e399: 8 total, 7 up, 8 in 2024-09-12T23:44:14.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:14 smithi136 bash[76371]: cluster 2024-09-12T23:44:13.364804+0000 mon.a (mon.0) 371 : cluster [DBG] osdmap e399: 8 total, 7 up, 8 in 2024-09-12T23:44:14.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:14 smithi181 bash[67587]: cluster 2024-09-12T23:44:13.364804+0000 mon.a (mon.0) 371 : cluster [DBG] osdmap e399: 8 total, 7 up, 8 in 2024-09-12T23:44:14.844 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:44:14 smithi181 bash[60471]: ts=2024-09-12T23:44:14.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:44:14.845 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:44:14 smithi181 bash[60471]: ts=2024-09-12T23:44:14.456Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:44:15.390 INFO:tasks.rados.rados.0.smithi136.stdout:1725: done (2 left) 2024-09-12T23:44:15.390 INFO:tasks.rados.rados.0.smithi136.stdout:1726: read oid 1 snap -1 2024-09-12T23:44:15.390 INFO:tasks.rados.rados.0.smithi136.stdout:1726: expect deleted 2024-09-12T23:44:15.390 INFO:tasks.rados.rados.0.smithi136.stdout:1727: rmattr oid 38 current snap is 162 2024-09-12T23:44:15.390 INFO:tasks.rados.rados.0.smithi136.stdout:1727: done (3 left) 2024-09-12T23:44:15.390 INFO:tasks.rados.rados.0.smithi136.stdout:1728: read oid 32 snap -1 2024-09-12T23:44:15.390 INFO:tasks.rados.rados.0.smithi136.stdout:1728: expect deleted 2024-09-12T23:44:15.390 INFO:tasks.rados.rados.0.smithi136.stdout:1729: copy_from oid 25 from oid 8 current snap is 162 2024-09-12T23:44:15.390 INFO:tasks.rados.rados.0.smithi136.stdout:1722: finishing write tid 4 to smithi13667081-44 2024-09-12T23:44:15.391 INFO:tasks.rados.rados.0.smithi136.stdout:1722: finishing write tid 5 to smithi13667081-44 2024-09-12T23:44:15.391 INFO:tasks.rados.rados.0.smithi136.stdout:1722: finishing write tid 6 to smithi13667081-44 2024-09-12T23:44:15.391 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 44 v 1078 (ObjNum 564 snap 162 seq_num 564) dirty exists 2024-09-12T23:44:15.391 INFO:tasks.rados.rados.0.smithi136.stdout:1722: left oid 44 (ObjNum 564 snap 162 seq_num 564) 2024-09-12T23:44:15.391 INFO:tasks.rados.rados.0.smithi136.stdout:1723: finishing copy_from racing read to smithi13667081-20 2024-09-12T23:44:15.391 INFO:tasks.rados.rados.0.smithi136.stdout:1723: finishing copy_from to smithi13667081-20 2024-09-12T23:44:15.391 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 20 v 1030 (ObjNum 469 snap 139 seq_num 469) dirty exists 2024-09-12T23:44:15.391 INFO:tasks.rados.rados.0.smithi136.stdout:1722: done (4 left) 2024-09-12T23:44:15.391 INFO:tasks.rados.rados.0.smithi136.stdout:1723: done (3 left) 2024-09-12T23:44:15.391 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 42 current snap is 162 2024-09-12T23:44:15.391 INFO:tasks.rados.rados.0.smithi136.stdout:1730: seq_num 566 ranges {0=688128,688128=614400,1302528=565248,1867776=425984} 2024-09-12T23:44:15.399 INFO:tasks.rados.rados.0.smithi136.stdout:1730: writing smithi13667081-42 from 0 to 688128 tid 1 2024-09-12T23:44:15.407 INFO:tasks.rados.rados.0.smithi136.stdout:1730: writing smithi13667081-42 from 688128 to 1302528 tid 2 2024-09-12T23:44:15.414 INFO:tasks.rados.rados.0.smithi136.stdout:1730: writing smithi13667081-42 from 1302528 to 1867776 tid 3 2024-09-12T23:44:15.421 INFO:tasks.rados.rados.0.smithi136.stdout:1730: writing smithi13667081-42 from 1867776 to 2293760 tid 4 2024-09-12T23:44:15.421 INFO:tasks.rados.rados.0.smithi136.stdout:1731: setattr oid 9 current snap is 162 2024-09-12T23:44:15.421 INFO:tasks.rados.rados.0.smithi136.stdout:1730: finishing write tid 1 to smithi13667081-42 2024-09-12T23:44:15.421 INFO:tasks.rados.rados.0.smithi136.stdout:1730: finishing write tid 2 to smithi13667081-42 2024-09-12T23:44:15.421 INFO:tasks.rados.rados.0.smithi136.stdout:1730: finishing write tid 3 to smithi13667081-42 2024-09-12T23:44:15.423 INFO:tasks.rados.rados.0.smithi136.stdout:1726: done (4 left) 2024-09-12T23:44:15.423 INFO:tasks.rados.rados.0.smithi136.stdout:1728: done (3 left) 2024-09-12T23:44:15.423 INFO:tasks.rados.rados.0.smithi136.stdout:1732: read oid 32 snap -1 2024-09-12T23:44:15.423 INFO:tasks.rados.rados.0.smithi136.stdout:1732: expect deleted 2024-09-12T23:44:15.423 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 4 current snap is 162 2024-09-12T23:44:15.423 INFO:tasks.rados.rados.0.smithi136.stdout:1733: seq_num 567 ranges {4808704=704512,5513216=499712,6012928=770048,6782976=335872} 2024-09-12T23:44:15.473 INFO:tasks.rados.rados.0.smithi136.stdout:1733: writing smithi13667081-4 from 4808704 to 5513216 tid 1 2024-09-12T23:44:15.475 INFO:tasks.rados.rados.0.smithi136.stdout:1733: writing smithi13667081-4 from 5513216 to 6012928 tid 2 2024-09-12T23:44:15.480 INFO:tasks.rados.rados.0.smithi136.stdout:1733: writing smithi13667081-4 from 6012928 to 6782976 tid 3 2024-09-12T23:44:15.483 INFO:tasks.rados.rados.0.smithi136.stdout:1733: writing smithi13667081-4 from 6782976 to 7118848 tid 4 2024-09-12T23:44:15.483 INFO:tasks.rados.rados.0.smithi136.stdout:1734: read oid 5 snap -1 2024-09-12T23:44:15.483 INFO:tasks.rados.rados.0.smithi136.stdout:1734: expect (ObjNum 526 snap 155 seq_num 526) 2024-09-12T23:44:15.483 INFO:tasks.rados.rados.0.smithi136.stdout:1735: read oid 50 snap -1 2024-09-12T23:44:15.484 INFO:tasks.rados.rados.0.smithi136.stdout:1735: expect (ObjNum 551 snap 159 seq_num 551) 2024-09-12T23:44:15.484 INFO:tasks.rados.rados.0.smithi136.stdout:1736: delete oid 30 current snap is 162 2024-09-12T23:44:15.484 INFO:tasks.rados.rados.0.smithi136.stdout:1730: finishing write tid 4 to smithi13667081-42 2024-09-12T23:44:15.484 INFO:tasks.rados.rados.0.smithi136.stdout:1730: finishing write tid 5 to smithi13667081-42 2024-09-12T23:44:15.484 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 9 v 635 (ObjNum 513 snap 150 seq_num 513) dirty exists 2024-09-12T23:44:15.484 INFO:tasks.rados.rados.0.smithi136.stdout:1730: finishing write tid 6 to smithi13667081-42 2024-09-12T23:44:15.484 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 42 v 822 (ObjNum 566 snap 162 seq_num 566) dirty exists 2024-09-12T23:44:15.484 INFO:tasks.rados.rados.0.smithi136.stdout:1730: left oid 42 (ObjNum 566 snap 162 seq_num 566) 2024-09-12T23:44:15.484 INFO:tasks.rados.rados.0.smithi136.stdout:1729: finishing copy_from racing read to smithi13667081-25 2024-09-12T23:44:15.484 INFO:tasks.rados.rados.0.smithi136.stdout:1729: finishing copy_from to smithi13667081-25 2024-09-12T23:44:15.484 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 25 v 776 (ObjNum 523 snap 153 seq_num 523) dirty exists 2024-09-12T23:44:15.485 INFO:tasks.rados.rados.0.smithi136.stdout:1733: finishing write tid 1 to smithi13667081-4 2024-09-12T23:44:15.485 INFO:tasks.rados.rados.0.smithi136.stdout:1733: finishing write tid 2 to smithi13667081-4 2024-09-12T23:44:15.488 INFO:tasks.rados.rados.0.smithi136.stdout:1729: done (7 left) 2024-09-12T23:44:15.488 INFO:tasks.rados.rados.0.smithi136.stdout:1730: done (6 left) 2024-09-12T23:44:15.489 INFO:tasks.rados.rados.0.smithi136.stdout:1731: done (5 left) 2024-09-12T23:44:15.489 INFO:tasks.rados.rados.0.smithi136.stdout:1732: done (4 left) 2024-09-12T23:44:15.489 INFO:tasks.rados.rados.0.smithi136.stdout:1736: done (3 left) 2024-09-12T23:44:15.489 INFO:tasks.rados.rados.0.smithi136.stdout:1737: read oid 1 snap -1 2024-09-12T23:44:15.489 INFO:tasks.rados.rados.0.smithi136.stdout:1737: expect deleted 2024-09-12T23:44:15.489 INFO:tasks.rados.rados.0.smithi136.stdout:1738: snap_create 2024-09-12T23:44:15.492 INFO:tasks.rados.rados.0.smithi136.stdout:1733: finishing write tid 3 to smithi13667081-4 2024-09-12T23:44:15.493 INFO:tasks.rados.rados.0.smithi136.stdout:1733: finishing write tid 4 to smithi13667081-4 2024-09-12T23:44:15.493 INFO:tasks.rados.rados.0.smithi136.stdout:1733: finishing write tid 5 to smithi13667081-4 2024-09-12T23:44:15.496 INFO:tasks.rados.rados.0.smithi136.stdout:1733: finishing write tid 6 to smithi13667081-4 2024-09-12T23:44:15.496 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 4 v 763 (ObjNum 567 snap 162 seq_num 567) dirty exists 2024-09-12T23:44:15.496 INFO:tasks.rados.rados.0.smithi136.stdout:1733: left oid 4 (ObjNum 567 snap 162 seq_num 567) 2024-09-12T23:44:15.504 INFO:tasks.rados.rados.0.smithi136.stdout:1735: expect (ObjNum 551 snap 159 seq_num 551) 2024-09-12T23:44:15.573 INFO:tasks.rados.rados.0.smithi136.stdout:1734: expect (ObjNum 526 snap 155 seq_num 526) 2024-09-12T23:44:15.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:15 smithi136 bash[76371]: cluster 2024-09-12T23:44:14.378610+0000 mon.a (mon.0) 372 : cluster [DBG] osdmap e400: 8 total, 7 up, 8 in 2024-09-12T23:44:15.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:15 smithi136 bash[76371]: cluster 2024-09-12T23:44:14.691786+0000 mgr.y (mgr.24661) 127 : cluster [DBG] pgmap v175: 121 pgs: 10 active+clean+remapped, 38 active+undersized, 12 active+undersized+degraded, 61 active+clean; 196 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 2.1 MiB/s wr, 10 op/s; 39/246 objects degraded (15.854%); 7/246 objects misplaced (2.846%); 142 B/s, 0 objects/s recovering 2024-09-12T23:44:15.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:15 smithi136 bash[75164]: cluster 2024-09-12T23:44:14.378610+0000 mon.a (mon.0) 372 : cluster [DBG] osdmap e400: 8 total, 7 up, 8 in 2024-09-12T23:44:15.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:15 smithi136 bash[75164]: cluster 2024-09-12T23:44:14.691786+0000 mgr.y (mgr.24661) 127 : cluster [DBG] pgmap v175: 121 pgs: 10 active+clean+remapped, 38 active+undersized, 12 active+undersized+degraded, 61 active+clean; 196 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 2.1 MiB/s wr, 10 op/s; 39/246 objects degraded (15.854%); 7/246 objects misplaced (2.846%); 142 B/s, 0 objects/s recovering 2024-09-12T23:44:15.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:15 smithi181 bash[67587]: cluster 2024-09-12T23:44:14.378610+0000 mon.a (mon.0) 372 : cluster [DBG] osdmap e400: 8 total, 7 up, 8 in 2024-09-12T23:44:15.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:15 smithi181 bash[67587]: cluster 2024-09-12T23:44:14.691786+0000 mgr.y (mgr.24661) 127 : cluster [DBG] pgmap v175: 121 pgs: 10 active+clean+remapped, 38 active+undersized, 12 active+undersized+degraded, 61 active+clean; 196 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 2.1 MiB/s wr, 10 op/s; 39/246 objects degraded (15.854%); 7/246 objects misplaced (2.846%); 142 B/s, 0 objects/s recovering 2024-09-12T23:44:16.315 INFO:tasks.rados.rados.0.smithi136.stdout:1733: done (4 left) 2024-09-12T23:44:16.316 INFO:tasks.rados.rados.0.smithi136.stdout:1734: done (3 left) 2024-09-12T23:44:16.316 INFO:tasks.rados.rados.0.smithi136.stdout:1735: done (2 left) 2024-09-12T23:44:16.316 INFO:tasks.rados.rados.0.smithi136.stdout:1737: done (1 left) 2024-09-12T23:44:16.316 INFO:tasks.rados.rados.0.smithi136.stdout:1738: done (0 left) 2024-09-12T23:44:16.316 INFO:tasks.rados.rados.0.smithi136.stdout:1739: delete oid 13 current snap is 163 2024-09-12T23:44:16.327 INFO:tasks.rados.rados.0.smithi136.stdout:1739: done (0 left) 2024-09-12T23:44:16.327 INFO:tasks.rados.rados.0.smithi136.stdout:1740: delete oid 34 current snap is 163 2024-09-12T23:44:16.331 INFO:tasks.rados.rados.0.smithi136.stdout:1740: done (0 left) 2024-09-12T23:44:16.331 INFO:tasks.rados.rados.0.smithi136.stdout:1741: snap_create 2024-09-12T23:44:16.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:16 smithi136 bash[75164]: cluster 2024-09-12T23:44:15.371287+0000 mon.a (mon.0) 373 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-12T23:44:16.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:16 smithi136 bash[75164]: cluster 2024-09-12T23:44:15.392447+0000 mon.a (mon.0) 374 : cluster [DBG] osdmap e401: 8 total, 7 up, 8 in 2024-09-12T23:44:16.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:16 smithi136 bash[75164]: cluster 2024-09-12T23:44:16.318562+0000 mon.a (mon.0) 375 : cluster [DBG] osdmap e402: 8 total, 7 up, 8 in 2024-09-12T23:44:16.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:16 smithi136 bash[76371]: cluster 2024-09-12T23:44:15.371287+0000 mon.a (mon.0) 373 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-12T23:44:16.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:16 smithi136 bash[76371]: cluster 2024-09-12T23:44:15.392447+0000 mon.a (mon.0) 374 : cluster [DBG] osdmap e401: 8 total, 7 up, 8 in 2024-09-12T23:44:16.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:16 smithi136 bash[76371]: cluster 2024-09-12T23:44:16.318562+0000 mon.a (mon.0) 375 : cluster [DBG] osdmap e402: 8 total, 7 up, 8 in 2024-09-12T23:44:16.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:16 smithi181 bash[67587]: cluster 2024-09-12T23:44:15.371287+0000 mon.a (mon.0) 373 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-12T23:44:16.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:16 smithi181 bash[67587]: cluster 2024-09-12T23:44:15.392447+0000 mon.a (mon.0) 374 : cluster [DBG] osdmap e401: 8 total, 7 up, 8 in 2024-09-12T23:44:16.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:16 smithi181 bash[67587]: cluster 2024-09-12T23:44:16.318562+0000 mon.a (mon.0) 375 : cluster [DBG] osdmap e402: 8 total, 7 up, 8 in 2024-09-12T23:44:17.316 INFO:tasks.rados.rados.0.smithi136.stdout:1741: done (0 left) 2024-09-12T23:44:17.316 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 49 current snap is 164 2024-09-12T23:44:17.316 INFO:tasks.rados.rados.0.smithi136.stdout:1742: seq_num 571 ranges {5226496=770048,5996544=483328,6479872=753664,7233536=368640} 2024-09-12T23:44:17.349 INFO:tasks.rados.rados.0.smithi136.stdout:1742: writing smithi13667081-49 from 5226496 to 5996544 tid 1 2024-09-12T23:44:17.352 INFO:tasks.rados.rados.0.smithi136.stdout:1742: writing smithi13667081-49 from 5996544 to 6479872 tid 2 2024-09-12T23:44:17.357 INFO:tasks.rados.rados.0.smithi136.stdout:1742: writing smithi13667081-49 from 6479872 to 7233536 tid 3 2024-09-12T23:44:17.360 INFO:tasks.rados.rados.0.smithi136.stdout:1742: writing smithi13667081-49 from 7233536 to 7602176 tid 4 2024-09-12T23:44:17.360 INFO:tasks.rados.rados.0.smithi136.stdout:1743: setattr oid 31 current snap is 164 2024-09-12T23:44:17.362 INFO:tasks.rados.rados.0.smithi136.stdout:1744: read oid 35 snap -1 2024-09-12T23:44:17.362 INFO:tasks.rados.rados.0.smithi136.stdout:1744: expect (ObjNum 469 snap 139 seq_num 469) 2024-09-12T23:44:17.362 INFO:tasks.rados.rados.0.smithi136.stdout:1745: snap_remove snap 163 2024-09-12T23:44:17.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:17 smithi181 bash[67587]: audit 2024-09-12T23:44:16.578164+0000 mon.a (mon.0) 376 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:17.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:17 smithi181 bash[67587]: audit 2024-09-12T23:44:16.583060+0000 mon.a (mon.0) 377 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:17.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:17 smithi181 bash[67587]: cluster 2024-09-12T23:44:16.692359+0000 mgr.y (mgr.24661) 128 : cluster [DBG] pgmap v178: 121 pgs: 10 active+clean+remapped, 38 active+undersized, 12 active+undersized+degraded, 61 active+clean; 196 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 2.1 MiB/s wr, 10 op/s; 39/246 objects degraded (15.854%); 7/246 objects misplaced (2.846%); 142 B/s, 0 objects/s recovering 2024-09-12T23:44:17.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:17 smithi181 bash[67587]: audit 2024-09-12T23:44:17.258877+0000 mon.a (mon.0) 378 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:17.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:17 smithi181 bash[67587]: audit 2024-09-12T23:44:17.264183+0000 mon.a (mon.0) 379 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:17.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:17 smithi181 bash[67587]: cluster 2024-09-12T23:44:17.320102+0000 mon.a (mon.0) 380 : cluster [DBG] osdmap e403: 8 total, 7 up, 8 in 2024-09-12T23:44:17.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:17 smithi181 bash[67587]: audit 2024-09-12T23:44:17.570111+0000 mon.a (mon.0) 381 : audit [INF] from='osd.0 [v2:172.21.15.136:6802/1696528169,v1:172.21.15.136:6803/1696528169]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-12T23:44:18.008 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 12 23:44:17 smithi136 bash[80792]: debug 2024-09-12T23:44:17.562+0000 7f046db1c640 -1 osd.0 391 log_to_monitors true 2024-09-12T23:44:18.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:17 smithi136 bash[75164]: audit 2024-09-12T23:44:16.578164+0000 mon.a (mon.0) 376 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:18.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:17 smithi136 bash[75164]: audit 2024-09-12T23:44:16.583060+0000 mon.a (mon.0) 377 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:18.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:17 smithi136 bash[75164]: cluster 2024-09-12T23:44:16.692359+0000 mgr.y (mgr.24661) 128 : cluster [DBG] pgmap v178: 121 pgs: 10 active+clean+remapped, 38 active+undersized, 12 active+undersized+degraded, 61 active+clean; 196 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 2.1 MiB/s wr, 10 op/s; 39/246 objects degraded (15.854%); 7/246 objects misplaced (2.846%); 142 B/s, 0 objects/s recovering 2024-09-12T23:44:18.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:17 smithi136 bash[75164]: audit 2024-09-12T23:44:17.258877+0000 mon.a (mon.0) 378 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:18.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:17 smithi136 bash[75164]: audit 2024-09-12T23:44:17.264183+0000 mon.a (mon.0) 379 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:18.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:17 smithi136 bash[75164]: cluster 2024-09-12T23:44:17.320102+0000 mon.a (mon.0) 380 : cluster [DBG] osdmap e403: 8 total, 7 up, 8 in 2024-09-12T23:44:18.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:17 smithi136 bash[75164]: audit 2024-09-12T23:44:17.570111+0000 mon.a (mon.0) 381 : audit [INF] from='osd.0 [v2:172.21.15.136:6802/1696528169,v1:172.21.15.136:6803/1696528169]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-12T23:44:18.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:17 smithi136 bash[76371]: audit 2024-09-12T23:44:16.578164+0000 mon.a (mon.0) 376 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:18.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:17 smithi136 bash[76371]: audit 2024-09-12T23:44:16.583060+0000 mon.a (mon.0) 377 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:18.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:17 smithi136 bash[76371]: cluster 2024-09-12T23:44:16.692359+0000 mgr.y (mgr.24661) 128 : cluster [DBG] pgmap v178: 121 pgs: 10 active+clean+remapped, 38 active+undersized, 12 active+undersized+degraded, 61 active+clean; 196 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 2.1 MiB/s wr, 10 op/s; 39/246 objects degraded (15.854%); 7/246 objects misplaced (2.846%); 142 B/s, 0 objects/s recovering 2024-09-12T23:44:18.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:17 smithi136 bash[76371]: audit 2024-09-12T23:44:17.258877+0000 mon.a (mon.0) 378 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:18.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:17 smithi136 bash[76371]: audit 2024-09-12T23:44:17.264183+0000 mon.a (mon.0) 379 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:18.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:17 smithi136 bash[76371]: cluster 2024-09-12T23:44:17.320102+0000 mon.a (mon.0) 380 : cluster [DBG] osdmap e403: 8 total, 7 up, 8 in 2024-09-12T23:44:18.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:17 smithi136 bash[76371]: audit 2024-09-12T23:44:17.570111+0000 mon.a (mon.0) 381 : audit [INF] from='osd.0 [v2:172.21.15.136:6802/1696528169,v1:172.21.15.136:6803/1696528169]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-12T23:44:18.323 INFO:tasks.rados.rados.0.smithi136.stdout:1745: done (3 left) 2024-09-12T23:44:18.324 INFO:tasks.rados.rados.0.smithi136.stdout:1746: setattr oid 18 current snap is 164 2024-09-12T23:44:18.324 INFO:tasks.rados.rados.0.smithi136.stdout:1742: finishing write tid 1 to smithi13667081-49 2024-09-12T23:44:18.324 INFO:tasks.rados.rados.0.smithi136.stdout:1742: finishing write tid 2 to smithi13667081-49 2024-09-12T23:44:18.324 INFO:tasks.rados.rados.0.smithi136.stdout:1742: finishing write tid 3 to smithi13667081-49 2024-09-12T23:44:18.324 INFO:tasks.rados.rados.0.smithi136.stdout:1742: finishing write tid 4 to smithi13667081-49 2024-09-12T23:44:18.324 INFO:tasks.rados.rados.0.smithi136.stdout:1742: finishing write tid 5 to smithi13667081-49 2024-09-12T23:44:18.324 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 31 v 910 (ObjNum 464 snap 138 seq_num 464) dirty exists 2024-09-12T23:44:18.324 INFO:tasks.rados.rados.0.smithi136.stdout:1742: finishing write tid 6 to smithi13667081-49 2024-09-12T23:44:18.324 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 49 v 909 (ObjNum 571 snap 164 seq_num 571) dirty exists 2024-09-12T23:44:18.324 INFO:tasks.rados.rados.0.smithi136.stdout:1742: left oid 49 (ObjNum 571 snap 164 seq_num 571) 2024-09-12T23:44:18.324 INFO:tasks.rados.rados.0.smithi136.stdout:1744: expect (ObjNum 469 snap 139 seq_num 469) 2024-09-12T23:44:18.502 INFO:tasks.rados.rados.0.smithi136.stdout:1742: done (3 left) 2024-09-12T23:44:18.502 INFO:tasks.rados.rados.0.smithi136.stdout:1743: done (2 left) 2024-09-12T23:44:18.502 INFO:tasks.rados.rados.0.smithi136.stdout:1744: done (1 left) 2024-09-12T23:44:18.502 INFO:tasks.rados.rados.0.smithi136.stdout:1747: snap_create 2024-09-12T23:44:18.515 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 18 v 901 (ObjNum 540 snap 157 seq_num 540) dirty exists 2024-09-12T23:44:18.756 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 12 23:44:18 smithi136 bash[80792]: debug 2024-09-12T23:44:18.358+0000 7f04658c6640 -1 osd.0 391 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-12T23:44:19.094 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:44:18 smithi181 bash[60471]: ts=2024-09-12T23:44:18.661Z caller=alerting.go:366 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-09-12T23:44:19.094 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:44:18 smithi181 bash[60471]: ts=2024-09-12T23:44:18.664Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:44:19.329 INFO:tasks.rados.rados.0.smithi136.stdout:1746: done (1 left) 2024-09-12T23:44:19.329 INFO:tasks.rados.rados.0.smithi136.stdout:1747: done (0 left) 2024-09-12T23:44:19.329 INFO:tasks.rados.rados.0.smithi136.stdout:1748: snap_create 2024-09-12T23:44:19.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:19 smithi181 bash[67587]: audit 2024-09-12T23:44:18.326965+0000 mon.a (mon.0) 382 : audit [INF] from='osd.0 [v2:172.21.15.136:6802/1696528169,v1:172.21.15.136:6803/1696528169]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-12T23:44:19.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:19 smithi181 bash[67587]: cluster 2024-09-12T23:44:18.327085+0000 mon.a (mon.0) 383 : cluster [DBG] osdmap e404: 8 total, 7 up, 8 in 2024-09-12T23:44:19.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:19 smithi181 bash[67587]: audit 2024-09-12T23:44:18.328065+0000 mon.a (mon.0) 384 : audit [INF] from='osd.0 [v2:172.21.15.136:6802/1696528169,v1:172.21.15.136:6803/1696528169]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-12T23:44:19.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:19 smithi181 bash[67587]: cluster 2024-09-12T23:44:18.693343+0000 mgr.y (mgr.24661) 129 : cluster [DBG] pgmap v181: 121 pgs: 10 active+clean+remapped, 38 active+undersized, 12 active+undersized+degraded, 61 active+clean; 196 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 39/246 objects degraded (15.854%); 7/246 objects misplaced (2.846%) 2024-09-12T23:44:19.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:44:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:44:19] "GET /metrics HTTP/1.1" 200 36254 "" "Prometheus/2.43.0" 2024-09-12T23:44:19.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:19 smithi136 bash[76371]: audit 2024-09-12T23:44:18.326965+0000 mon.a (mon.0) 382 : audit [INF] from='osd.0 [v2:172.21.15.136:6802/1696528169,v1:172.21.15.136:6803/1696528169]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-12T23:44:19.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:19 smithi136 bash[76371]: cluster 2024-09-12T23:44:18.327085+0000 mon.a (mon.0) 383 : cluster [DBG] osdmap e404: 8 total, 7 up, 8 in 2024-09-12T23:44:19.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:19 smithi136 bash[76371]: audit 2024-09-12T23:44:18.328065+0000 mon.a (mon.0) 384 : audit [INF] from='osd.0 [v2:172.21.15.136:6802/1696528169,v1:172.21.15.136:6803/1696528169]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-12T23:44:19.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:19 smithi136 bash[76371]: cluster 2024-09-12T23:44:18.693343+0000 mgr.y (mgr.24661) 129 : cluster [DBG] pgmap v181: 121 pgs: 10 active+clean+remapped, 38 active+undersized, 12 active+undersized+degraded, 61 active+clean; 196 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 39/246 objects degraded (15.854%); 7/246 objects misplaced (2.846%) 2024-09-12T23:44:19.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:19 smithi136 bash[75164]: audit 2024-09-12T23:44:18.326965+0000 mon.a (mon.0) 382 : audit [INF] from='osd.0 [v2:172.21.15.136:6802/1696528169,v1:172.21.15.136:6803/1696528169]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-12T23:44:19.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:19 smithi136 bash[75164]: cluster 2024-09-12T23:44:18.327085+0000 mon.a (mon.0) 383 : cluster [DBG] osdmap e404: 8 total, 7 up, 8 in 2024-09-12T23:44:19.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:19 smithi136 bash[75164]: audit 2024-09-12T23:44:18.328065+0000 mon.a (mon.0) 384 : audit [INF] from='osd.0 [v2:172.21.15.136:6802/1696528169,v1:172.21.15.136:6803/1696528169]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-12T23:44:19.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:19 smithi136 bash[75164]: cluster 2024-09-12T23:44:18.693343+0000 mgr.y (mgr.24661) 129 : cluster [DBG] pgmap v181: 121 pgs: 10 active+clean+remapped, 38 active+undersized, 12 active+undersized+degraded, 61 active+clean; 196 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 39/246 objects degraded (15.854%); 7/246 objects misplaced (2.846%) 2024-09-12T23:44:20.344 INFO:tasks.rados.rados.0.smithi136.stdout:1748: done (0 left) 2024-09-12T23:44:20.344 INFO:tasks.rados.rados.0.smithi136.stdout:1749: snap_remove snap 164 2024-09-12T23:44:20.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:20 smithi136 bash[75164]: cluster 2024-09-12T23:44:19.322694+0000 mon.a (mon.0) 385 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-12T23:44:20.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:20 smithi136 bash[75164]: cluster 2024-09-12T23:44:19.330843+0000 mon.a (mon.0) 386 : cluster [INF] osd.0 [v2:172.21.15.136:6802/1696528169,v1:172.21.15.136:6803/1696528169] boot 2024-09-12T23:44:20.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:20 smithi136 bash[75164]: cluster 2024-09-12T23:44:19.331061+0000 mon.a (mon.0) 387 : cluster [DBG] osdmap e405: 8 total, 8 up, 8 in 2024-09-12T23:44:20.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:20 smithi136 bash[75164]: audit 2024-09-12T23:44:19.332123+0000 mon.a (mon.0) 388 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-12T23:44:20.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:20 smithi136 bash[76371]: cluster 2024-09-12T23:44:19.322694+0000 mon.a (mon.0) 385 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-12T23:44:20.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:20 smithi136 bash[76371]: cluster 2024-09-12T23:44:19.330843+0000 mon.a (mon.0) 386 : cluster [INF] osd.0 [v2:172.21.15.136:6802/1696528169,v1:172.21.15.136:6803/1696528169] boot 2024-09-12T23:44:20.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:20 smithi136 bash[76371]: cluster 2024-09-12T23:44:19.331061+0000 mon.a (mon.0) 387 : cluster [DBG] osdmap e405: 8 total, 8 up, 8 in 2024-09-12T23:44:20.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:20 smithi136 bash[76371]: audit 2024-09-12T23:44:19.332123+0000 mon.a (mon.0) 388 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-12T23:44:20.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:20 smithi181 bash[67587]: cluster 2024-09-12T23:44:19.322694+0000 mon.a (mon.0) 385 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-12T23:44:20.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:20 smithi181 bash[67587]: cluster 2024-09-12T23:44:19.330843+0000 mon.a (mon.0) 386 : cluster [INF] osd.0 [v2:172.21.15.136:6802/1696528169,v1:172.21.15.136:6803/1696528169] boot 2024-09-12T23:44:20.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:20 smithi181 bash[67587]: cluster 2024-09-12T23:44:19.331061+0000 mon.a (mon.0) 387 : cluster [DBG] osdmap e405: 8 total, 8 up, 8 in 2024-09-12T23:44:20.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:20 smithi181 bash[67587]: audit 2024-09-12T23:44:19.332123+0000 mon.a (mon.0) 388 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-12T23:44:21.340 INFO:tasks.rados.rados.0.smithi136.stdout:1749: done (0 left) 2024-09-12T23:44:21.340 INFO:tasks.rados.rados.0.smithi136.stdout:1750: rollback oid 7 current snap is 166 2024-09-12T23:44:21.340 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 7 to 162 2024-09-12T23:44:21.340 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 31 current snap is 166 2024-09-12T23:44:21.340 INFO:tasks.rados.rados.0.smithi136.stdout:1751: seq_num 572 ranges {4874240=786432,5660672=770048,6430720=557056,6987776=278528} 2024-09-12T23:44:21.367 INFO:tasks.rados.rados.0.smithi136.stdout:1751: writing smithi13667081-31 from 4874240 to 5660672 tid 1 2024-09-12T23:44:21.372 INFO:tasks.rados.rados.0.smithi136.stdout:1751: writing smithi13667081-31 from 5660672 to 6430720 tid 2 2024-09-12T23:44:21.376 INFO:tasks.rados.rados.0.smithi136.stdout:1751: writing smithi13667081-31 from 6430720 to 6987776 tid 3 2024-09-12T23:44:21.378 INFO:tasks.rados.rados.0.smithi136.stdout:1751: writing smithi13667081-31 from 6987776 to 7266304 tid 4 2024-09-12T23:44:21.378 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 50 current snap is 166 2024-09-12T23:44:21.379 INFO:tasks.rados.rados.0.smithi136.stdout:1752: seq_num 573 ranges {2039808=802816,2842624=655360,3497984=720896,4218880=229376} 2024-09-12T23:44:21.394 INFO:tasks.rados.rados.0.smithi136.stdout:1752: writing smithi13667081-50 from 2039808 to 2842624 tid 1 2024-09-12T23:44:21.398 INFO:tasks.rados.rados.0.smithi136.stdout:1752: writing smithi13667081-50 from 2842624 to 3497984 tid 2 2024-09-12T23:44:21.403 INFO:tasks.rados.rados.0.smithi136.stdout:1752: writing smithi13667081-50 from 3497984 to 4218880 tid 3 2024-09-12T23:44:21.405 INFO:tasks.rados.rados.0.smithi136.stdout:1752: writing smithi13667081-50 from 4218880 to 4448256 tid 4 2024-09-12T23:44:21.405 INFO:tasks.rados.rados.0.smithi136.stdout:1753: read oid 9 snap -1 2024-09-12T23:44:21.405 INFO:tasks.rados.rados.0.smithi136.stdout:1753: expect (ObjNum 513 snap 150 seq_num 513) 2024-09-12T23:44:21.405 INFO:tasks.rados.rados.0.smithi136.stdout:1754: snap_create 2024-09-12T23:44:21.405 INFO:tasks.rados.rados.0.smithi136.stdout:1750: finishing rollback tid 0 to smithi13667081-7 2024-09-12T23:44:21.406 INFO:tasks.rados.rados.0.smithi136.stdout:1750: finishing rollback tid 1 to smithi13667081-7 2024-09-12T23:44:21.406 INFO:tasks.rados.rados.0.smithi136.stdout:1750: finishing rollback tid 2 to smithi13667081-7 2024-09-12T23:44:21.406 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 7 v 904 (ObjNum 541 snap 157 seq_num 541) dirty exists 2024-09-12T23:44:21.406 INFO:tasks.rados.rados.0.smithi136.stdout:1751: finishing write tid 1 to smithi13667081-31 2024-09-12T23:44:21.406 INFO:tasks.rados.rados.0.smithi136.stdout:1751: finishing write tid 2 to smithi13667081-31 2024-09-12T23:44:21.406 INFO:tasks.rados.rados.0.smithi136.stdout:1751: finishing write tid 3 to smithi13667081-31 2024-09-12T23:44:21.406 INFO:tasks.rados.rados.0.smithi136.stdout:1751: finishing write tid 4 to smithi13667081-31 2024-09-12T23:44:21.406 INFO:tasks.rados.rados.0.smithi136.stdout:1751: finishing write tid 5 to smithi13667081-31 2024-09-12T23:44:21.406 INFO:tasks.rados.rados.0.smithi136.stdout:1751: finishing write tid 6 to smithi13667081-31 2024-09-12T23:44:21.406 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 31 v 915 (ObjNum 572 snap 166 seq_num 572) dirty exists 2024-09-12T23:44:21.406 INFO:tasks.rados.rados.0.smithi136.stdout:1751: left oid 31 (ObjNum 572 snap 166 seq_num 572) 2024-09-12T23:44:21.406 INFO:tasks.rados.rados.0.smithi136.stdout:1752: finishing write tid 1 to smithi13667081-50 2024-09-12T23:44:21.406 INFO:tasks.rados.rados.0.smithi136.stdout:1752: finishing write tid 2 to smithi13667081-50 2024-09-12T23:44:21.412 INFO:tasks.rados.rados.0.smithi136.stdout:1752: finishing write tid 3 to smithi13667081-50 2024-09-12T23:44:21.414 INFO:tasks.rados.rados.0.smithi136.stdout:1752: finishing write tid 4 to smithi13667081-50 2024-09-12T23:44:21.414 INFO:tasks.rados.rados.0.smithi136.stdout:1752: finishing write tid 5 to smithi13667081-50 2024-09-12T23:44:21.416 INFO:tasks.rados.rados.0.smithi136.stdout:1752: finishing write tid 6 to smithi13667081-50 2024-09-12T23:44:21.416 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 50 v 815 (ObjNum 573 snap 166 seq_num 573) dirty exists 2024-09-12T23:44:21.416 INFO:tasks.rados.rados.0.smithi136.stdout:1752: left oid 50 (ObjNum 573 snap 166 seq_num 573) 2024-09-12T23:44:21.441 INFO:tasks.rados.rados.0.smithi136.stdout:1753: expect (ObjNum 513 snap 150 seq_num 513) 2024-09-12T23:44:21.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:21 smithi136 bash[75164]: cluster 2024-09-12T23:44:20.335886+0000 mon.a (mon.0) 389 : cluster [DBG] osdmap e406: 8 total, 8 up, 8 in 2024-09-12T23:44:21.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:21 smithi136 bash[75164]: cluster 2024-09-12T23:44:20.693976+0000 mgr.y (mgr.24661) 130 : cluster [DBG] pgmap v184: 121 pgs: 10 remapped+peering, 35 peering, 11 active+undersized, 4 active+undersized+degraded, 61 active+clean; 195 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 2.6 MiB/s wr, 20 op/s; 17/246 objects degraded (6.911%) 2024-09-12T23:44:21.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:21 smithi136 bash[75164]: cluster 2024-09-12T23:44:21.314741+0000 mon.a (mon.0) 390 : cluster [WRN] Health check update: Degraded data redundancy: 39/246 objects degraded (15.854%), 12 pgs degraded (PG_DEGRADED) 2024-09-12T23:44:21.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:21 smithi136 bash[75164]: cluster 2024-09-12T23:44:21.337640+0000 mon.a (mon.0) 391 : cluster [DBG] osdmap e407: 8 total, 8 up, 8 in 2024-09-12T23:44:21.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:21 smithi136 bash[76371]: cluster 2024-09-12T23:44:20.335886+0000 mon.a (mon.0) 389 : cluster [DBG] osdmap e406: 8 total, 8 up, 8 in 2024-09-12T23:44:21.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:21 smithi136 bash[76371]: cluster 2024-09-12T23:44:20.693976+0000 mgr.y (mgr.24661) 130 : cluster [DBG] pgmap v184: 121 pgs: 10 remapped+peering, 35 peering, 11 active+undersized, 4 active+undersized+degraded, 61 active+clean; 195 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 2.6 MiB/s wr, 20 op/s; 17/246 objects degraded (6.911%) 2024-09-12T23:44:21.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:21 smithi136 bash[76371]: cluster 2024-09-12T23:44:21.314741+0000 mon.a (mon.0) 390 : cluster [WRN] Health check update: Degraded data redundancy: 39/246 objects degraded (15.854%), 12 pgs degraded (PG_DEGRADED) 2024-09-12T23:44:21.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:21 smithi136 bash[76371]: cluster 2024-09-12T23:44:21.337640+0000 mon.a (mon.0) 391 : cluster [DBG] osdmap e407: 8 total, 8 up, 8 in 2024-09-12T23:44:21.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:21 smithi181 bash[67587]: cluster 2024-09-12T23:44:20.335886+0000 mon.a (mon.0) 389 : cluster [DBG] osdmap e406: 8 total, 8 up, 8 in 2024-09-12T23:44:21.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:21 smithi181 bash[67587]: cluster 2024-09-12T23:44:20.693976+0000 mgr.y (mgr.24661) 130 : cluster [DBG] pgmap v184: 121 pgs: 10 remapped+peering, 35 peering, 11 active+undersized, 4 active+undersized+degraded, 61 active+clean; 195 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 2.6 MiB/s wr, 20 op/s; 17/246 objects degraded (6.911%) 2024-09-12T23:44:21.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:21 smithi181 bash[67587]: cluster 2024-09-12T23:44:21.314741+0000 mon.a (mon.0) 390 : cluster [WRN] Health check update: Degraded data redundancy: 39/246 objects degraded (15.854%), 12 pgs degraded (PG_DEGRADED) 2024-09-12T23:44:21.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:21 smithi181 bash[67587]: cluster 2024-09-12T23:44:21.337640+0000 mon.a (mon.0) 391 : cluster [DBG] osdmap e407: 8 total, 8 up, 8 in 2024-09-12T23:44:22.386 INFO:tasks.rados.rados.0.smithi136.stdout:1750: done (4 left) 2024-09-12T23:44:22.387 INFO:tasks.rados.rados.0.smithi136.stdout:1751: done (3 left) 2024-09-12T23:44:22.387 INFO:tasks.rados.rados.0.smithi136.stdout:1752: done (2 left) 2024-09-12T23:44:22.387 INFO:tasks.rados.rados.0.smithi136.stdout:1753: done (1 left) 2024-09-12T23:44:22.387 INFO:tasks.rados.rados.0.smithi136.stdout:1754: done (0 left) 2024-09-12T23:44:22.387 INFO:tasks.rados.rados.0.smithi136.stdout:1755: rmattr oid 2 current snap is 167 2024-09-12T23:44:22.387 INFO:tasks.rados.rados.0.smithi136.stdout:1755: done (0 left) 2024-09-12T23:44:22.387 INFO:tasks.rados.rados.0.smithi136.stdout:1756: snap_create 2024-09-12T23:44:22.594 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:44:22 smithi181 bash[60471]: ts=2024-09-12T23:44:22.313Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:44:23.393 INFO:tasks.rados.rados.0.smithi136.stdout:1756: done (0 left) 2024-09-12T23:44:23.393 INFO:tasks.rados.rados.0.smithi136.stdout:1757: snap_remove snap 166 2024-09-12T23:44:23.657 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:23 smithi136 bash[75164]: cluster 2024-09-12T23:44:22.389804+0000 mon.a (mon.0) 392 : cluster [DBG] osdmap e408: 8 total, 8 up, 8 in 2024-09-12T23:44:23.657 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:23 smithi136 bash[75164]: cluster 2024-09-12T23:44:22.694740+0000 mgr.y (mgr.24661) 131 : cluster [DBG] pgmap v187: 121 pgs: 10 remapped+peering, 35 peering, 11 active+undersized, 4 active+undersized+degraded, 61 active+clean; 195 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 2.6 MiB/s wr, 20 op/s; 17/246 objects degraded (6.911%) 2024-09-12T23:44:23.658 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:23 smithi136 bash[76371]: cluster 2024-09-12T23:44:22.389804+0000 mon.a (mon.0) 392 : cluster [DBG] osdmap e408: 8 total, 8 up, 8 in 2024-09-12T23:44:23.658 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:23 smithi136 bash[76371]: cluster 2024-09-12T23:44:22.694740+0000 mgr.y (mgr.24661) 131 : cluster [DBG] pgmap v187: 121 pgs: 10 remapped+peering, 35 peering, 11 active+undersized, 4 active+undersized+degraded, 61 active+clean; 195 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 2.6 MiB/s wr, 20 op/s; 17/246 objects degraded (6.911%) 2024-09-12T23:44:23.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:23 smithi181 bash[67587]: cluster 2024-09-12T23:44:22.389804+0000 mon.a (mon.0) 392 : cluster [DBG] osdmap e408: 8 total, 8 up, 8 in 2024-09-12T23:44:23.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:23 smithi181 bash[67587]: cluster 2024-09-12T23:44:22.694740+0000 mgr.y (mgr.24661) 131 : cluster [DBG] pgmap v187: 121 pgs: 10 remapped+peering, 35 peering, 11 active+undersized, 4 active+undersized+degraded, 61 active+clean; 195 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 2.6 MiB/s wr, 20 op/s; 17/246 objects degraded (6.911%) 2024-09-12T23:44:23.845 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:44:23 smithi181 bash[60471]: ts=2024-09-12T23:44:23.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi136\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:44:24.403 INFO:tasks.rados.rados.0.smithi136.stdout:1757: done (0 left) 2024-09-12T23:44:24.403 INFO:tasks.rados.rados.0.smithi136.stdout:1758: delete oid 49 current snap is 168 2024-09-12T23:44:24.416 INFO:tasks.rados.rados.0.smithi136.stdout:1758: done (0 left) 2024-09-12T23:44:24.416 INFO:tasks.rados.rados.0.smithi136.stdout:1759: delete oid 40 current snap is 168 2024-09-12T23:44:24.423 INFO:tasks.rados.rados.0.smithi136.stdout:1759: done (0 left) 2024-09-12T23:44:24.423 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 15 current snap is 168 2024-09-12T23:44:24.423 INFO:tasks.rados.rados.0.smithi136.stdout:1760: seq_num 576 ranges {1671168=49152} 2024-09-12T23:44:24.440 INFO:tasks.rados.rados.0.smithi136.stdout:1760: writing smithi13667081-15 from 1671168 to 1720320 tid 1 2024-09-12T23:44:24.440 INFO:tasks.rados.rados.0.smithi136.stdout:1761: delete oid 41 current snap is 168 2024-09-12T23:44:24.443 INFO:tasks.rados.rados.0.smithi136.stdout:1761: done (1 left) 2024-09-12T23:44:24.443 INFO:tasks.rados.rados.0.smithi136.stdout:1762: read oid 43 snap -1 2024-09-12T23:44:24.443 INFO:tasks.rados.rados.0.smithi136.stdout:1762: expect (ObjNum 552 snap 159 seq_num 552) 2024-09-12T23:44:24.443 INFO:tasks.rados.rados.0.smithi136.stdout:1763: snap_create 2024-09-12T23:44:24.445 INFO:tasks.rados.rados.0.smithi136.stdout:1760: finishing write tid 1 to smithi13667081-15 2024-09-12T23:44:24.445 INFO:tasks.rados.rados.0.smithi136.stdout:1760: finishing write tid 2 to smithi13667081-15 2024-09-12T23:44:24.446 INFO:tasks.rados.rados.0.smithi136.stdout:1760: finishing write tid 3 to smithi13667081-15 2024-09-12T23:44:24.446 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 15 v 918 (ObjNum 576 snap 168 seq_num 576) dirty exists 2024-09-12T23:44:24.446 INFO:tasks.rados.rados.0.smithi136.stdout:1760: left oid 15 (ObjNum 576 snap 168 seq_num 576) 2024-09-12T23:44:24.484 INFO:tasks.rados.rados.0.smithi136.stdout:1762: expect (ObjNum 552 snap 159 seq_num 552) 2024-09-12T23:44:24.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:24 smithi136 bash[76371]: cluster 2024-09-12T23:44:23.397249+0000 mon.a (mon.0) 393 : cluster [DBG] osdmap e409: 8 total, 8 up, 8 in 2024-09-12T23:44:24.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:24 smithi136 bash[76371]: audit 2024-09-12T23:44:23.747022+0000 mon.a (mon.0) 394 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:24.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:24 smithi136 bash[76371]: audit 2024-09-12T23:44:23.747946+0000 mon.a (mon.0) 395 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:44:24.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:24 smithi136 bash[75164]: cluster 2024-09-12T23:44:23.397249+0000 mon.a (mon.0) 393 : cluster [DBG] osdmap e409: 8 total, 8 up, 8 in 2024-09-12T23:44:24.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:24 smithi136 bash[75164]: audit 2024-09-12T23:44:23.747022+0000 mon.a (mon.0) 394 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:24.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:24 smithi136 bash[75164]: audit 2024-09-12T23:44:23.747946+0000 mon.a (mon.0) 395 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:44:24.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:24 smithi181 bash[67587]: cluster 2024-09-12T23:44:23.397249+0000 mon.a (mon.0) 393 : cluster [DBG] osdmap e409: 8 total, 8 up, 8 in 2024-09-12T23:44:24.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:24 smithi181 bash[67587]: audit 2024-09-12T23:44:23.747022+0000 mon.a (mon.0) 394 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:24.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:24 smithi181 bash[67587]: audit 2024-09-12T23:44:23.747946+0000 mon.a (mon.0) 395 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:44:24.845 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:44:24 smithi181 bash[60471]: ts=2024-09-12T23:44:24.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:44:24.845 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:44:24 smithi181 bash[60471]: ts=2024-09-12T23:44:24.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:44:25.422 INFO:tasks.rados.rados.0.smithi136.stdout:1760: done (2 left) 2024-09-12T23:44:25.422 INFO:tasks.rados.rados.0.smithi136.stdout:1762: done (1 left) 2024-09-12T23:44:25.422 INFO:tasks.rados.rados.0.smithi136.stdout:1763: done (0 left) 2024-09-12T23:44:25.422 INFO:tasks.rados.rados.0.smithi136.stdout:1764: read oid 28 snap 168 2024-09-12T23:44:25.422 INFO:tasks.rados.rados.0.smithi136.stdout:1764: expect (ObjNum 391 snap 113 seq_num 391) 2024-09-12T23:44:25.422 INFO:tasks.rados.rados.0.smithi136.stdout:1765: snap_remove snap 162 2024-09-12T23:44:25.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:25 smithi136 bash[75164]: cluster 2024-09-12T23:44:24.403810+0000 mon.a (mon.0) 396 : cluster [DBG] osdmap e410: 8 total, 8 up, 8 in 2024-09-12T23:44:25.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:25 smithi136 bash[75164]: cluster 2024-09-12T23:44:24.696024+0000 mgr.y (mgr.24661) 132 : cluster [DBG] pgmap v190: 121 pgs: 121 active+clean; 182 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 1.7 MiB/s wr, 10 op/s; 17 MiB/s, 13 objects/s recovering 2024-09-12T23:44:25.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:25 smithi136 bash[76371]: cluster 2024-09-12T23:44:24.403810+0000 mon.a (mon.0) 396 : cluster [DBG] osdmap e410: 8 total, 8 up, 8 in 2024-09-12T23:44:25.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:25 smithi136 bash[76371]: cluster 2024-09-12T23:44:24.696024+0000 mgr.y (mgr.24661) 132 : cluster [DBG] pgmap v190: 121 pgs: 121 active+clean; 182 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 1.7 MiB/s wr, 10 op/s; 17 MiB/s, 13 objects/s recovering 2024-09-12T23:44:25.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:25 smithi181 bash[67587]: cluster 2024-09-12T23:44:24.403810+0000 mon.a (mon.0) 396 : cluster [DBG] osdmap e410: 8 total, 8 up, 8 in 2024-09-12T23:44:25.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:25 smithi181 bash[67587]: cluster 2024-09-12T23:44:24.696024+0000 mgr.y (mgr.24661) 132 : cluster [DBG] pgmap v190: 121 pgs: 121 active+clean; 182 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 1.7 MiB/s wr, 10 op/s; 17 MiB/s, 13 objects/s recovering 2024-09-12T23:44:26.318 INFO:tasks.rados.rados.0.smithi136.stdout:1765: done (1 left) 2024-09-12T23:44:26.318 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 18 current snap is 169 2024-09-12T23:44:26.318 INFO:tasks.rados.rados.0.smithi136.stdout:1766: seq_num 578 ranges {3678208=81920} 2024-09-12T23:44:26.339 INFO:tasks.rados.rados.0.smithi136.stdout:1766: writing smithi13667081-18 from 3678208 to 3760128 tid 1 2024-09-12T23:44:26.340 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 29 current snap is 169 2024-09-12T23:44:26.340 INFO:tasks.rados.rados.0.smithi136.stdout:1767: seq_num 579 ranges {6168576=98304} 2024-09-12T23:44:26.375 INFO:tasks.rados.rados.0.smithi136.stdout:1767: writing smithi13667081-29 from 6168576 to 6266880 tid 1 2024-09-12T23:44:26.375 INFO:tasks.rados.rados.0.smithi136.stdout:1768: copy_from oid 7 from oid 15 current snap is 169 2024-09-12T23:44:26.375 INFO:tasks.rados.rados.0.smithi136.stdout:1764: expect (ObjNum 391 snap 113 seq_num 391) 2024-09-12T23:44:26.626 INFO:tasks.rados.rados.0.smithi136.stdout:1766: finishing write tid 1 to smithi13667081-18 2024-09-12T23:44:26.626 INFO:tasks.rados.rados.0.smithi136.stdout:1766: finishing write tid 2 to smithi13667081-18 2024-09-12T23:44:26.626 INFO:tasks.rados.rados.0.smithi136.stdout:1766: finishing write tid 3 to smithi13667081-18 2024-09-12T23:44:26.626 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 18 v 906 (ObjNum 578 snap 169 seq_num 578) dirty exists 2024-09-12T23:44:26.626 INFO:tasks.rados.rados.0.smithi136.stdout:1766: left oid 18 (ObjNum 578 snap 169 seq_num 578) 2024-09-12T23:44:26.626 INFO:tasks.rados.rados.0.smithi136.stdout:1767: finishing write tid 1 to smithi13667081-29 2024-09-12T23:44:26.626 INFO:tasks.rados.rados.0.smithi136.stdout:1767: finishing write tid 2 to smithi13667081-29 2024-09-12T23:44:26.626 INFO:tasks.rados.rados.0.smithi136.stdout:1767: finishing write tid 3 to smithi13667081-29 2024-09-12T23:44:26.626 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 29 v 817 (ObjNum 579 snap 169 seq_num 579) dirty exists 2024-09-12T23:44:26.627 INFO:tasks.rados.rados.0.smithi136.stdout:1767: left oid 29 (ObjNum 579 snap 169 seq_num 579) 2024-09-12T23:44:26.627 INFO:tasks.rados.rados.0.smithi136.stdout:1768: finishing copy_from racing read to smithi13667081-7 2024-09-12T23:44:26.627 INFO:tasks.rados.rados.0.smithi136.stdout:1768: finishing copy_from to smithi13667081-7 2024-09-12T23:44:26.627 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 7 v 918 (ObjNum 576 snap 168 seq_num 576) dirty exists 2024-09-12T23:44:26.627 INFO:tasks.rados.rados.0.smithi136.stdout:1764: done (3 left) 2024-09-12T23:44:26.627 INFO:tasks.rados.rados.0.smithi136.stdout:1766: done (2 left) 2024-09-12T23:44:26.627 INFO:tasks.rados.rados.0.smithi136.stdout:1767: done (1 left) 2024-09-12T23:44:26.627 INFO:tasks.rados.rados.0.smithi136.stdout:1768: done (0 left) 2024-09-12T23:44:26.627 INFO:tasks.rados.rados.0.smithi136.stdout:1769: snap_remove snap 168 2024-09-12T23:44:26.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:26 smithi136 bash[75164]: cluster 2024-09-12T23:44:25.411023+0000 mon.a (mon.0) 397 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 17/246 objects degraded (6.911%), 4 pgs degraded) 2024-09-12T23:44:26.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:26 smithi136 bash[75164]: cluster 2024-09-12T23:44:25.411080+0000 mon.a (mon.0) 398 : cluster [INF] Cluster is now healthy 2024-09-12T23:44:26.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:26 smithi136 bash[75164]: cluster 2024-09-12T23:44:25.425791+0000 mon.a (mon.0) 399 : cluster [DBG] osdmap e411: 8 total, 8 up, 8 in 2024-09-12T23:44:26.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:26 smithi136 bash[75164]: cluster 2024-09-12T23:44:26.321773+0000 mon.a (mon.0) 400 : cluster [DBG] osdmap e412: 8 total, 8 up, 8 in 2024-09-12T23:44:26.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:26 smithi136 bash[76371]: cluster 2024-09-12T23:44:25.411023+0000 mon.a (mon.0) 397 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 17/246 objects degraded (6.911%), 4 pgs degraded) 2024-09-12T23:44:26.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:26 smithi136 bash[76371]: cluster 2024-09-12T23:44:25.411080+0000 mon.a (mon.0) 398 : cluster [INF] Cluster is now healthy 2024-09-12T23:44:26.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:26 smithi136 bash[76371]: cluster 2024-09-12T23:44:25.425791+0000 mon.a (mon.0) 399 : cluster [DBG] osdmap e411: 8 total, 8 up, 8 in 2024-09-12T23:44:26.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:26 smithi136 bash[76371]: cluster 2024-09-12T23:44:26.321773+0000 mon.a (mon.0) 400 : cluster [DBG] osdmap e412: 8 total, 8 up, 8 in 2024-09-12T23:44:26.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:26 smithi181 bash[67587]: cluster 2024-09-12T23:44:25.411023+0000 mon.a (mon.0) 397 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 17/246 objects degraded (6.911%), 4 pgs degraded) 2024-09-12T23:44:26.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:26 smithi181 bash[67587]: cluster 2024-09-12T23:44:25.411080+0000 mon.a (mon.0) 398 : cluster [INF] Cluster is now healthy 2024-09-12T23:44:26.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:26 smithi181 bash[67587]: cluster 2024-09-12T23:44:25.425791+0000 mon.a (mon.0) 399 : cluster [DBG] osdmap e411: 8 total, 8 up, 8 in 2024-09-12T23:44:26.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:26 smithi181 bash[67587]: cluster 2024-09-12T23:44:26.321773+0000 mon.a (mon.0) 400 : cluster [DBG] osdmap e412: 8 total, 8 up, 8 in 2024-09-12T23:44:27.433 INFO:tasks.rados.rados.0.smithi136.stdout:1769: done (0 left) 2024-09-12T23:44:27.433 INFO:tasks.rados.rados.0.smithi136.stdout:1770: read oid 31 snap -1 2024-09-12T23:44:27.433 INFO:tasks.rados.rados.0.smithi136.stdout:1770: expect (ObjNum 572 snap 166 seq_num 572) 2024-09-12T23:44:27.433 INFO:tasks.rados.rados.0.smithi136.stdout:1771: delete oid 16 current snap is 169 2024-09-12T23:44:27.445 INFO:tasks.rados.rados.0.smithi136.stdout:1771: done (1 left) 2024-09-12T23:44:27.446 INFO:tasks.rados.rados.0.smithi136.stdout:1772: delete oid 21 current snap is 169 2024-09-12T23:44:27.456 INFO:tasks.rados.rados.0.smithi136.stdout:1772: done (1 left) 2024-09-12T23:44:27.456 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 42 current snap is 169 2024-09-12T23:44:27.456 INFO:tasks.rados.rados.0.smithi136.stdout:1773: seq_num 582 ranges {2293760=155648} 2024-09-12T23:44:27.484 INFO:tasks.rados.rados.0.smithi136.stdout:1773: writing smithi13667081-42 from 2293760 to 2449408 tid 1 2024-09-12T23:44:27.484 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 14 current snap is 169 2024-09-12T23:44:27.484 INFO:tasks.rados.rados.0.smithi136.stdout:1774: seq_num 583 ranges {6750208=172032} 2024-09-12T23:44:27.538 INFO:tasks.rados.rados.0.smithi136.stdout:1774: writing smithi13667081-14 from 6750208 to 6922240 tid 1 2024-09-12T23:44:27.538 INFO:tasks.rados.rados.0.smithi136.stdout:1775: read oid 16 snap -1 2024-09-12T23:44:27.538 INFO:tasks.rados.rados.0.smithi136.stdout:1775: expect deleted 2024-09-12T23:44:27.538 INFO:tasks.rados.rados.0.smithi136.stdout:1776: snap_create 2024-09-12T23:44:27.538 INFO:tasks.rados.rados.0.smithi136.stdout:1770: expect (ObjNum 572 snap 166 seq_num 572) 2024-09-12T23:44:27.721 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:27 smithi136 bash[76371]: cluster 2024-09-12T23:44:26.696759+0000 mgr.y (mgr.24661) 133 : cluster [DBG] pgmap v193: 121 pgs: 121 active+clean; 182 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 1.7 MiB/s wr, 10 op/s; 17 MiB/s, 13 objects/s recovering 2024-09-12T23:44:27.721 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:27 smithi136 bash[76371]: audit 2024-09-12T23:44:27.159723+0000 mon.a (mon.0) 401 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:27.721 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:27 smithi136 bash[76371]: audit 2024-09-12T23:44:27.169376+0000 mon.a (mon.0) 402 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:27.721 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:27 smithi136 bash[76371]: audit 2024-09-12T23:44:27.171317+0000 mon.a (mon.0) 403 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:44:27.721 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:27 smithi136 bash[76371]: audit 2024-09-12T23:44:27.173071+0000 mon.a (mon.0) 404 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:44:27.721 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:27 smithi136 bash[76371]: audit 2024-09-12T23:44:27.182540+0000 mon.a (mon.0) 405 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:27.721 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:27 smithi136 bash[76371]: audit 2024-09-12T23:44:27.335456+0000 mon.a (mon.0) 406 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:44:27.722 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:27 smithi136 bash[76371]: audit 2024-09-12T23:44:27.340006+0000 mon.a (mon.0) 407 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:44:27.722 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:27 smithi136 bash[76371]: audit 2024-09-12T23:44:27.343402+0000 mon.a (mon.0) 408 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:44:27.722 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:27 smithi136 bash[76371]: audit 2024-09-12T23:44:27.345937+0000 mon.a (mon.0) 409 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:44:27.722 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:27 smithi136 bash[76371]: audit 2024-09-12T23:44:27.349163+0000 mon.a (mon.0) 410 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-12T23:44:27.722 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:27 smithi136 bash[75164]: cluster 2024-09-12T23:44:26.696759+0000 mgr.y (mgr.24661) 133 : cluster [DBG] pgmap v193: 121 pgs: 121 active+clean; 182 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 1.7 MiB/s wr, 10 op/s; 17 MiB/s, 13 objects/s recovering 2024-09-12T23:44:27.722 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:27 smithi136 bash[75164]: audit 2024-09-12T23:44:27.159723+0000 mon.a (mon.0) 401 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:27.722 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:27 smithi136 bash[75164]: audit 2024-09-12T23:44:27.169376+0000 mon.a (mon.0) 402 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:27.722 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:27 smithi136 bash[75164]: audit 2024-09-12T23:44:27.171317+0000 mon.a (mon.0) 403 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:44:27.722 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:27 smithi136 bash[75164]: audit 2024-09-12T23:44:27.173071+0000 mon.a (mon.0) 404 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:44:27.722 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:27 smithi136 bash[75164]: audit 2024-09-12T23:44:27.182540+0000 mon.a (mon.0) 405 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:27.723 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:27 smithi136 bash[75164]: audit 2024-09-12T23:44:27.335456+0000 mon.a (mon.0) 406 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:44:27.723 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:27 smithi136 bash[75164]: audit 2024-09-12T23:44:27.340006+0000 mon.a (mon.0) 407 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:44:27.723 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:27 smithi136 bash[75164]: audit 2024-09-12T23:44:27.343402+0000 mon.a (mon.0) 408 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:44:27.723 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:27 smithi136 bash[75164]: audit 2024-09-12T23:44:27.345937+0000 mon.a (mon.0) 409 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:44:27.723 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:27 smithi136 bash[75164]: audit 2024-09-12T23:44:27.349163+0000 mon.a (mon.0) 410 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-12T23:44:27.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:27 smithi181 bash[67587]: cluster 2024-09-12T23:44:26.696759+0000 mgr.y (mgr.24661) 133 : cluster [DBG] pgmap v193: 121 pgs: 121 active+clean; 182 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 1.7 MiB/s wr, 10 op/s; 17 MiB/s, 13 objects/s recovering 2024-09-12T23:44:27.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:27 smithi181 bash[67587]: audit 2024-09-12T23:44:27.159723+0000 mon.a (mon.0) 401 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:27.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:27 smithi181 bash[67587]: audit 2024-09-12T23:44:27.169376+0000 mon.a (mon.0) 402 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:27.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:27 smithi181 bash[67587]: audit 2024-09-12T23:44:27.171317+0000 mon.a (mon.0) 403 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:44:27.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:27 smithi181 bash[67587]: audit 2024-09-12T23:44:27.173071+0000 mon.a (mon.0) 404 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:44:27.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:27 smithi181 bash[67587]: audit 2024-09-12T23:44:27.182540+0000 mon.a (mon.0) 405 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:27.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:27 smithi181 bash[67587]: audit 2024-09-12T23:44:27.335456+0000 mon.a (mon.0) 406 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:44:27.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:27 smithi181 bash[67587]: audit 2024-09-12T23:44:27.340006+0000 mon.a (mon.0) 407 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:44:27.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:27 smithi181 bash[67587]: audit 2024-09-12T23:44:27.343402+0000 mon.a (mon.0) 408 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:44:27.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:27 smithi181 bash[67587]: audit 2024-09-12T23:44:27.345937+0000 mon.a (mon.0) 409 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:44:27.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:27 smithi181 bash[67587]: audit 2024-09-12T23:44:27.349163+0000 mon.a (mon.0) 410 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-12T23:44:27.926 INFO:tasks.rados.rados.0.smithi136.stdout:1773: finishing write tid 1 to smithi13667081-42 2024-09-12T23:44:27.926 INFO:tasks.rados.rados.0.smithi136.stdout:1773: finishing write tid 2 to smithi13667081-42 2024-09-12T23:44:27.926 INFO:tasks.rados.rados.0.smithi136.stdout:1773: finishing write tid 3 to smithi13667081-42 2024-09-12T23:44:27.926 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 42 v 824 (ObjNum 582 snap 169 seq_num 582) dirty exists 2024-09-12T23:44:27.926 INFO:tasks.rados.rados.0.smithi136.stdout:1773: left oid 42 (ObjNum 582 snap 169 seq_num 582) 2024-09-12T23:44:27.926 INFO:tasks.rados.rados.0.smithi136.stdout:1774: finishing write tid 1 to smithi13667081-14 2024-09-12T23:44:27.926 INFO:tasks.rados.rados.0.smithi136.stdout:1774: finishing write tid 2 to smithi13667081-14 2024-09-12T23:44:27.926 INFO:tasks.rados.rados.0.smithi136.stdout:1774: finishing write tid 3 to smithi13667081-14 2024-09-12T23:44:27.926 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 14 v 1084 (ObjNum 583 snap 169 seq_num 583) dirty exists 2024-09-12T23:44:27.926 INFO:tasks.rados.rados.0.smithi136.stdout:1774: left oid 14 (ObjNum 583 snap 169 seq_num 583) 2024-09-12T23:44:28.429 INFO:tasks.rados.rados.0.smithi136.stdout:1770: done (4 left) 2024-09-12T23:44:28.430 INFO:tasks.rados.rados.0.smithi136.stdout:1773: done (3 left) 2024-09-12T23:44:28.430 INFO:tasks.rados.rados.0.smithi136.stdout:1774: done (2 left) 2024-09-12T23:44:28.430 INFO:tasks.rados.rados.0.smithi136.stdout:1775: done (1 left) 2024-09-12T23:44:28.430 INFO:tasks.rados.rados.0.smithi136.stdout:1776: done (0 left) 2024-09-12T23:44:28.430 INFO:tasks.rados.rados.0.smithi136.stdout:1777: delete oid 3 current snap is 170 2024-09-12T23:44:28.446 INFO:tasks.rados.rados.0.smithi136.stdout:1777: done (0 left) 2024-09-12T23:44:28.447 INFO:tasks.rados.rados.0.smithi136.stdout:1778: read oid 39 snap 165 2024-09-12T23:44:28.447 INFO:tasks.rados.rados.0.smithi136.stdout:1778: expect deleted 2024-09-12T23:44:28.447 INFO:tasks.rados.rados.0.smithi136.stdout:1779: delete oid 50 current snap is 170 2024-09-12T23:44:28.455 INFO:tasks.rados.rados.0.smithi136.stdout:1778: done (1 left) 2024-09-12T23:44:28.455 INFO:tasks.rados.rados.0.smithi136.stdout:1779: done (0 left) 2024-09-12T23:44:28.455 INFO:tasks.rados.rados.0.smithi136.stdout:1780: rollback oid 6 current snap is 170 2024-09-12T23:44:28.455 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 6 to 169 2024-09-12T23:44:28.455 INFO:tasks.rados.rados.0.smithi136.stdout:1781: read oid 17 snap -1 2024-09-12T23:44:28.455 INFO:tasks.rados.rados.0.smithi136.stdout:1781: expect deleted 2024-09-12T23:44:28.455 INFO:tasks.rados.rados.0.smithi136.stdout:1782: snap_create 2024-09-12T23:44:28.462 INFO:tasks.rados.rados.0.smithi136.stdout:1780: finishing rollback tid 0 to smithi13667081-6 2024-09-12T23:44:28.462 INFO:tasks.rados.rados.0.smithi136.stdout:1780: finishing rollback tid 1 to smithi13667081-6 2024-09-12T23:44:28.462 INFO:tasks.rados.rados.0.smithi136.stdout:1780: finishing rollback tid 2 to smithi13667081-6 2024-09-12T23:44:28.462 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 6 v 1206 (ObjNum 559 snap 160 seq_num 559) dirty exists 2024-09-12T23:44:28.682 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:28 smithi136 bash[75164]: audit 2024-09-12T23:44:27.349563+0000 mgr.y (mgr.24661) 134 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-12T23:44:28.682 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:28 smithi136 bash[75164]: cephadm 2024-09-12T23:44:27.351672+0000 mgr.y (mgr.24661) 135 : cephadm [INF] Upgrade: osd.1 is safe to restart 2024-09-12T23:44:28.682 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:28 smithi136 bash[75164]: cluster 2024-09-12T23:44:27.434503+0000 mon.a (mon.0) 411 : cluster [DBG] osdmap e413: 8 total, 8 up, 8 in 2024-09-12T23:44:28.682 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:28 smithi136 bash[76371]: audit 2024-09-12T23:44:27.349563+0000 mgr.y (mgr.24661) 134 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-12T23:44:28.682 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:28 smithi136 bash[76371]: cephadm 2024-09-12T23:44:27.351672+0000 mgr.y (mgr.24661) 135 : cephadm [INF] Upgrade: osd.1 is safe to restart 2024-09-12T23:44:28.682 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:28 smithi136 bash[76371]: cluster 2024-09-12T23:44:27.434503+0000 mon.a (mon.0) 411 : cluster [DBG] osdmap e413: 8 total, 8 up, 8 in 2024-09-12T23:44:28.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:28 smithi181 bash[67587]: audit 2024-09-12T23:44:27.349563+0000 mgr.y (mgr.24661) 134 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-12T23:44:28.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:28 smithi181 bash[67587]: cephadm 2024-09-12T23:44:27.351672+0000 mgr.y (mgr.24661) 135 : cephadm [INF] Upgrade: osd.1 is safe to restart 2024-09-12T23:44:28.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:28 smithi181 bash[67587]: cluster 2024-09-12T23:44:27.434503+0000 mon.a (mon.0) 411 : cluster [DBG] osdmap e413: 8 total, 8 up, 8 in 2024-09-12T23:44:28.845 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:44:28 smithi181 bash[60471]: ts=2024-09-12T23:44:28.660Z caller=alerting.go:366 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-09-12T23:44:28.845 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:44:28 smithi181 bash[60471]: ts=2024-09-12T23:44:28.663Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:44:29.448 INFO:tasks.rados.rados.0.smithi136.stdout:1780: done (2 left) 2024-09-12T23:44:29.448 INFO:tasks.rados.rados.0.smithi136.stdout:1781: done (1 left) 2024-09-12T23:44:29.448 INFO:tasks.rados.rados.0.smithi136.stdout:1782: done (0 left) 2024-09-12T23:44:29.448 INFO:tasks.rados.rados.0.smithi136.stdout:1783: rollback oid 5 current snap is 171 2024-09-12T23:44:29.448 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 5 to 165 2024-09-12T23:44:29.449 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 46 current snap is 171 2024-09-12T23:44:29.449 INFO:tasks.rados.rados.0.smithi136.stdout:1784: seq_num 586 ranges {0=221184} 2024-09-12T23:44:29.451 INFO:tasks.rados.rados.0.smithi136.stdout:1784: writing smithi13667081-46 from 0 to 221184 tid 1 2024-09-12T23:44:29.451 INFO:tasks.rados.rados.0.smithi136.stdout:1785: snap_create 2024-09-12T23:44:29.464 INFO:tasks.rados.rados.0.smithi136.stdout:1783: finishing rollback tid 0 to smithi13667081-5 2024-09-12T23:44:29.464 INFO:tasks.rados.rados.0.smithi136.stdout:1783: finishing rollback tid 1 to smithi13667081-5 2024-09-12T23:44:29.465 INFO:tasks.rados.rados.0.smithi136.stdout:1784: finishing write tid 1 to smithi13667081-46 2024-09-12T23:44:29.465 INFO:tasks.rados.rados.0.smithi136.stdout:1783: finishing rollback tid 2 to smithi13667081-5 2024-09-12T23:44:29.465 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 5 v 1279 (ObjNum 526 snap 155 seq_num 526) dirty exists 2024-09-12T23:44:29.465 INFO:tasks.rados.rados.0.smithi136.stdout:1784: finishing write tid 2 to smithi13667081-46 2024-09-12T23:44:29.468 INFO:tasks.rados.rados.0.smithi136.stdout:1784: finishing write tid 3 to smithi13667081-46 2024-09-12T23:44:29.468 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 46 v 623 (ObjNum 586 snap 171 seq_num 586) dirty exists 2024-09-12T23:44:29.468 INFO:tasks.rados.rados.0.smithi136.stdout:1784: left oid 46 (ObjNum 586 snap 171 seq_num 586) 2024-09-12T23:44:29.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:44:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:44:29] "GET /metrics HTTP/1.1" 200 36343 "" "Prometheus/2.43.0" 2024-09-12T23:44:29.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:29 smithi136 bash[75164]: cluster 2024-09-12T23:44:28.431975+0000 mon.a (mon.0) 412 : cluster [DBG] osdmap e414: 8 total, 8 up, 8 in 2024-09-12T23:44:29.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:29 smithi136 bash[75164]: cluster 2024-09-12T23:44:28.697242+0000 mgr.y (mgr.24661) 136 : cluster [DBG] pgmap v196: 121 pgs: 121 active+clean; 182 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 38 KiB/s wr, 1 op/s 2024-09-12T23:44:29.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:29 smithi136 bash[76371]: cluster 2024-09-12T23:44:28.431975+0000 mon.a (mon.0) 412 : cluster [DBG] osdmap e414: 8 total, 8 up, 8 in 2024-09-12T23:44:29.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:29 smithi136 bash[76371]: cluster 2024-09-12T23:44:28.697242+0000 mgr.y (mgr.24661) 136 : cluster [DBG] pgmap v196: 121 pgs: 121 active+clean; 182 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 38 KiB/s wr, 1 op/s 2024-09-12T23:44:29.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:29 smithi181 bash[67587]: cluster 2024-09-12T23:44:28.431975+0000 mon.a (mon.0) 412 : cluster [DBG] osdmap e414: 8 total, 8 up, 8 in 2024-09-12T23:44:29.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:29 smithi181 bash[67587]: cluster 2024-09-12T23:44:28.697242+0000 mgr.y (mgr.24661) 136 : cluster [DBG] pgmap v196: 121 pgs: 121 active+clean; 182 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 38 KiB/s wr, 1 op/s 2024-09-12T23:44:30.449 INFO:tasks.rados.rados.0.smithi136.stdout:1783: done (2 left) 2024-09-12T23:44:30.449 INFO:tasks.rados.rados.0.smithi136.stdout:1784: done (1 left) 2024-09-12T23:44:30.449 INFO:tasks.rados.rados.0.smithi136.stdout:1785: done (0 left) 2024-09-12T23:44:30.449 INFO:tasks.rados.rados.0.smithi136.stdout:1786: snap_remove snap 171 2024-09-12T23:44:30.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:30 smithi136 bash[75164]: cluster 2024-09-12T23:44:29.451374+0000 mon.a (mon.0) 413 : cluster [DBG] osdmap e415: 8 total, 8 up, 8 in 2024-09-12T23:44:30.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:30 smithi136 bash[75164]: cephadm 2024-09-12T23:44:29.528102+0000 mgr.y (mgr.24661) 137 : cephadm [INF] Upgrade: Updating osd.1 2024-09-12T23:44:30.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:30 smithi136 bash[75164]: audit 2024-09-12T23:44:29.535419+0000 mon.a (mon.0) 414 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:30.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:30 smithi136 bash[75164]: audit 2024-09-12T23:44:29.536301+0000 mon.a (mon.0) 415 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-12T23:44:30.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:30 smithi136 bash[75164]: audit 2024-09-12T23:44:29.537144+0000 mon.a (mon.0) 416 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:44:30.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:30 smithi136 bash[75164]: cephadm 2024-09-12T23:44:29.539527+0000 mgr.y (mgr.24661) 138 : cephadm [INF] Deploying daemon osd.1 on smithi136 2024-09-12T23:44:30.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:30 smithi136 bash[76371]: cluster 2024-09-12T23:44:29.451374+0000 mon.a (mon.0) 413 : cluster [DBG] osdmap e415: 8 total, 8 up, 8 in 2024-09-12T23:44:30.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:30 smithi136 bash[76371]: cephadm 2024-09-12T23:44:29.528102+0000 mgr.y (mgr.24661) 137 : cephadm [INF] Upgrade: Updating osd.1 2024-09-12T23:44:30.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:30 smithi136 bash[76371]: audit 2024-09-12T23:44:29.535419+0000 mon.a (mon.0) 414 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:30.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:30 smithi136 bash[76371]: audit 2024-09-12T23:44:29.536301+0000 mon.a (mon.0) 415 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-12T23:44:30.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:30 smithi136 bash[76371]: audit 2024-09-12T23:44:29.537144+0000 mon.a (mon.0) 416 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:44:30.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:30 smithi136 bash[76371]: cephadm 2024-09-12T23:44:29.539527+0000 mgr.y (mgr.24661) 138 : cephadm [INF] Deploying daemon osd.1 on smithi136 2024-09-12T23:44:30.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:30 smithi181 bash[67587]: cluster 2024-09-12T23:44:29.451374+0000 mon.a (mon.0) 413 : cluster [DBG] osdmap e415: 8 total, 8 up, 8 in 2024-09-12T23:44:30.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:30 smithi181 bash[67587]: cephadm 2024-09-12T23:44:29.528102+0000 mgr.y (mgr.24661) 137 : cephadm [INF] Upgrade: Updating osd.1 2024-09-12T23:44:30.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:30 smithi181 bash[67587]: audit 2024-09-12T23:44:29.535419+0000 mon.a (mon.0) 414 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:30.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:30 smithi181 bash[67587]: audit 2024-09-12T23:44:29.536301+0000 mon.a (mon.0) 415 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-12T23:44:30.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:30 smithi181 bash[67587]: audit 2024-09-12T23:44:29.537144+0000 mon.a (mon.0) 416 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:44:30.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:30 smithi181 bash[67587]: cephadm 2024-09-12T23:44:29.539527+0000 mgr.y (mgr.24661) 138 : cephadm [INF] Deploying daemon osd.1 on smithi136 2024-09-12T23:44:31.326 INFO:tasks.rados.rados.0.smithi136.stdout:1786: done (0 left) 2024-09-12T23:44:31.326 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 49 current snap is 172 2024-09-12T23:44:31.326 INFO:tasks.rados.rados.0.smithi136.stdout:1787: seq_num 587 ranges {0=237568} 2024-09-12T23:44:31.329 INFO:tasks.rados.rados.0.smithi136.stdout:1787: writing smithi13667081-49 from 0 to 237568 tid 1 2024-09-12T23:44:31.329 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 25 current snap is 172 2024-09-12T23:44:31.330 INFO:tasks.rados.rados.0.smithi136.stdout:1788: seq_num 588 ranges {6316032=253952} 2024-09-12T23:44:31.403 INFO:tasks.rados.rados.0.smithi136.stdout:1788: writing smithi13667081-25 from 6316032 to 6569984 tid 1 2024-09-12T23:44:31.403 INFO:tasks.rados.rados.0.smithi136.stdout:1789: copy_from oid 30 from oid 13 current snap is 172 2024-09-12T23:44:31.403 INFO:tasks.rados.rados.0.smithi136.stdout:1787: finishing write tid 1 to smithi13667081-49 2024-09-12T23:44:31.403 INFO:tasks.rados.rados.0.smithi136.stdout:1787: finishing write tid 2 to smithi13667081-49 2024-09-12T23:44:31.403 INFO:tasks.rados.rados.0.smithi136.stdout:1787: finishing write tid 3 to smithi13667081-49 2024-09-12T23:44:31.403 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 49 v 921 (ObjNum 587 snap 172 seq_num 587) dirty exists 2024-09-12T23:44:31.403 INFO:tasks.rados.rados.0.smithi136.stdout:1787: left oid 49 (ObjNum 587 snap 172 seq_num 587) 2024-09-12T23:44:31.403 INFO:tasks.rados.rados.0.smithi136.stdout:1787: done (2 left) 2024-09-12T23:44:31.403 INFO:tasks.rados.rados.0.smithi136.stdout:1790: read oid 36 snap -1 2024-09-12T23:44:31.404 INFO:tasks.rados.rados.0.smithi136.stdout:1790: expect (ObjNum 556 snap 159 seq_num 556) 2024-09-12T23:44:31.404 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 37 current snap is 172 2024-09-12T23:44:31.404 INFO:tasks.rados.rados.0.smithi136.stdout:1791: seq_num 589 ranges {122880=270336} 2024-09-12T23:44:31.406 INFO:tasks.rados.rados.0.smithi136.stdout:1791: writing smithi13667081-37 from 122880 to 393216 tid 1 2024-09-12T23:44:31.406 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 39 current snap is 172 2024-09-12T23:44:31.406 INFO:tasks.rados.rados.0.smithi136.stdout:1792: seq_num 590 ranges {0=286720} 2024-09-12T23:44:31.408 INFO:tasks.rados.rados.0.smithi136.stdout:1792: writing smithi13667081-39 from 0 to 286720 tid 1 2024-09-12T23:44:31.408 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 27 current snap is 172 2024-09-12T23:44:31.408 INFO:tasks.rados.rados.0.smithi136.stdout:1793: seq_num 591 ranges {6750208=303104} 2024-09-12T23:44:31.451 INFO:tasks.rados.rados.0.smithi136.stdout:1793: writing smithi13667081-27 from 6750208 to 7053312 tid 1 2024-09-12T23:44:31.451 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 4 current snap is 172 2024-09-12T23:44:31.451 INFO:tasks.rados.rados.0.smithi136.stdout:1791: finishing write tid 1 to smithi13667081-37 2024-09-12T23:44:31.451 INFO:tasks.rados.rados.0.smithi136.stdout:1791: finishing write tid 2 to smithi13667081-37 2024-09-12T23:44:31.452 INFO:tasks.rados.rados.0.smithi136.stdout:1794: seq_num 592 ranges {7118848=319488} 2024-09-12T23:44:31.494 INFO:tasks.rados.rados.0.smithi136.stdout:1794: writing smithi13667081-4 from 7118848 to 7438336 tid 1 2024-09-12T23:44:31.494 INFO:tasks.rados.rados.0.smithi136.stdout:1795: delete oid 49 current snap is 172 2024-09-12T23:44:31.494 INFO:tasks.rados.rados.0.smithi136.stdout:1791: finishing write tid 3 to smithi13667081-37 2024-09-12T23:44:31.494 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 37 v 954 (ObjNum 589 snap 172 seq_num 589) dirty exists 2024-09-12T23:44:31.494 INFO:tasks.rados.rados.0.smithi136.stdout:1791: left oid 37 (ObjNum 589 snap 172 seq_num 589) 2024-09-12T23:44:31.495 INFO:tasks.rados.rados.0.smithi136.stdout:1789: finishing copy_from to smithi13667081-30 2024-09-12T23:44:31.495 INFO:tasks.rados.rados.0.smithi136.stdout:1789: got expected ENOENT (src dne) 2024-09-12T23:44:31.495 INFO:tasks.rados.rados.0.smithi136.stdout:1789: finishing copy_from racing read to smithi13667081-30 2024-09-12T23:44:31.495 INFO:tasks.rados.rados.0.smithi136.stdout:1788: finishing write tid 1 to smithi13667081-25 2024-09-12T23:44:31.495 INFO:tasks.rados.rados.0.smithi136.stdout:1788: finishing write tid 2 to smithi13667081-25 2024-09-12T23:44:31.495 INFO:tasks.rados.rados.0.smithi136.stdout:1788: finishing write tid 3 to smithi13667081-25 2024-09-12T23:44:31.495 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 25 v 778 (ObjNum 588 snap 172 seq_num 588) dirty exists 2024-09-12T23:44:31.495 INFO:tasks.rados.rados.0.smithi136.stdout:1788: left oid 25 (ObjNum 588 snap 172 seq_num 588) 2024-09-12T23:44:31.495 INFO:tasks.rados.rados.0.smithi136.stdout:1792: finishing write tid 1 to smithi13667081-39 2024-09-12T23:44:31.495 INFO:tasks.rados.rados.0.smithi136.stdout:1792: finishing write tid 2 to smithi13667081-39 2024-09-12T23:44:31.495 INFO:tasks.rados.rados.0.smithi136.stdout:1792: finishing write tid 3 to smithi13667081-39 2024-09-12T23:44:31.495 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 39 v 780 (ObjNum 590 snap 172 seq_num 590) dirty exists 2024-09-12T23:44:31.495 INFO:tasks.rados.rados.0.smithi136.stdout:1792: left oid 39 (ObjNum 590 snap 172 seq_num 590) 2024-09-12T23:44:31.496 INFO:tasks.rados.rados.0.smithi136.stdout:1790: expect (ObjNum 556 snap 159 seq_num 556) 2024-09-12T23:44:31.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:31 smithi136 bash[75164]: cluster 2024-09-12T23:44:30.452245+0000 mon.a (mon.0) 417 : cluster [DBG] osdmap e416: 8 total, 8 up, 8 in 2024-09-12T23:44:31.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:31 smithi136 bash[75164]: cluster 2024-09-12T23:44:30.697971+0000 mgr.y (mgr.24661) 139 : cluster [DBG] pgmap v199: 121 pgs: 121 active+clean; 178 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.2 MiB/s rd, 598 KiB/s wr, 7 op/s 2024-09-12T23:44:31.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:31 smithi136 bash[75164]: cluster 2024-09-12T23:44:31.329161+0000 mon.a (mon.0) 418 : cluster [DBG] osdmap e417: 8 total, 8 up, 8 in 2024-09-12T23:44:31.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:31 smithi136 bash[76371]: cluster 2024-09-12T23:44:30.452245+0000 mon.a (mon.0) 417 : cluster [DBG] osdmap e416: 8 total, 8 up, 8 in 2024-09-12T23:44:31.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:31 smithi136 bash[76371]: cluster 2024-09-12T23:44:30.697971+0000 mgr.y (mgr.24661) 139 : cluster [DBG] pgmap v199: 121 pgs: 121 active+clean; 178 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.2 MiB/s rd, 598 KiB/s wr, 7 op/s 2024-09-12T23:44:31.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:31 smithi136 bash[76371]: cluster 2024-09-12T23:44:31.329161+0000 mon.a (mon.0) 418 : cluster [DBG] osdmap e417: 8 total, 8 up, 8 in 2024-09-12T23:44:31.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:31 smithi181 bash[67587]: cluster 2024-09-12T23:44:30.452245+0000 mon.a (mon.0) 417 : cluster [DBG] osdmap e416: 8 total, 8 up, 8 in 2024-09-12T23:44:31.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:31 smithi181 bash[67587]: cluster 2024-09-12T23:44:30.697971+0000 mgr.y (mgr.24661) 139 : cluster [DBG] pgmap v199: 121 pgs: 121 active+clean; 178 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.2 MiB/s rd, 598 KiB/s wr, 7 op/s 2024-09-12T23:44:31.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:31 smithi181 bash[67587]: cluster 2024-09-12T23:44:31.329161+0000 mon.a (mon.0) 418 : cluster [DBG] osdmap e417: 8 total, 8 up, 8 in 2024-09-12T23:44:31.882 INFO:tasks.rados.rados.0.smithi136.stdout:1793: finishing write tid 1 to smithi13667081-27 2024-09-12T23:44:31.882 INFO:tasks.rados.rados.0.smithi136.stdout:1793: finishing write tid 2 to smithi13667081-27 2024-09-12T23:44:31.882 INFO:tasks.rados.rados.0.smithi136.stdout:1793: finishing write tid 3 to smithi13667081-27 2024-09-12T23:44:31.882 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 27 v 1282 (ObjNum 591 snap 172 seq_num 591) dirty exists 2024-09-12T23:44:31.882 INFO:tasks.rados.rados.0.smithi136.stdout:1793: left oid 27 (ObjNum 591 snap 172 seq_num 591) 2024-09-12T23:44:31.883 INFO:tasks.rados.rados.0.smithi136.stdout:1794: finishing write tid 1 to smithi13667081-4 2024-09-12T23:44:31.883 INFO:tasks.rados.rados.0.smithi136.stdout:1794: finishing write tid 2 to smithi13667081-4 2024-09-12T23:44:31.883 INFO:tasks.rados.rados.0.smithi136.stdout:1794: finishing write tid 3 to smithi13667081-4 2024-09-12T23:44:31.883 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 4 v 765 (ObjNum 592 snap 172 seq_num 592) dirty exists 2024-09-12T23:44:31.883 INFO:tasks.rados.rados.0.smithi136.stdout:1794: left oid 4 (ObjNum 592 snap 172 seq_num 592) 2024-09-12T23:44:31.883 INFO:tasks.rados.rados.0.smithi136.stdout:1788: done (7 left) 2024-09-12T23:44:31.883 INFO:tasks.rados.rados.0.smithi136.stdout:1789: done (6 left) 2024-09-12T23:44:31.883 INFO:tasks.rados.rados.0.smithi136.stdout:1790: done (5 left) 2024-09-12T23:44:31.883 INFO:tasks.rados.rados.0.smithi136.stdout:1791: done (4 left) 2024-09-12T23:44:31.883 INFO:tasks.rados.rados.0.smithi136.stdout:1792: done (3 left) 2024-09-12T23:44:31.883 INFO:tasks.rados.rados.0.smithi136.stdout:1793: done (2 left) 2024-09-12T23:44:31.883 INFO:tasks.rados.rados.0.smithi136.stdout:1794: done (1 left) 2024-09-12T23:44:31.883 INFO:tasks.rados.rados.0.smithi136.stdout:1795: done (0 left) 2024-09-12T23:44:31.883 INFO:tasks.rados.rados.0.smithi136.stdout:1796: rollback oid 10 current snap is 172 2024-09-12T23:44:31.884 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 10 to 170 2024-09-12T23:44:31.884 INFO:tasks.rados.rados.0.smithi136.stdout:1797: snap_remove snap 169 2024-09-12T23:44:32.469 INFO:tasks.rados.rados.0.smithi136.stdout:1797: done (1 left) 2024-09-12T23:44:32.470 INFO:tasks.rados.rados.0.smithi136.stdout:1798: rmattr oid 40 current snap is 172 2024-09-12T23:44:32.470 INFO:tasks.rados.rados.0.smithi136.stdout:1798: done (1 left) 2024-09-12T23:44:32.470 INFO:tasks.rados.rados.0.smithi136.stdout:1799: snap_create 2024-09-12T23:44:32.470 INFO:tasks.rados.rados.0.smithi136.stdout:1796: finishing rollback tid 0 to smithi13667081-10 2024-09-12T23:44:32.470 INFO:tasks.rados.rados.0.smithi136.stdout:1796: finishing rollback tid 1 to smithi13667081-10 2024-09-12T23:44:32.470 INFO:tasks.rados.rados.0.smithi136.stdout:1796: finishing rollback tid 2 to smithi13667081-10 2024-09-12T23:44:32.470 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 10 v 803 (ObjNum 514 snap 150 seq_num 514) dirty exists 2024-09-12T23:44:32.594 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:44:32 smithi181 bash[60471]: ts=2024-09-12T23:44:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:44:33.484 INFO:tasks.rados.rados.0.smithi136.stdout:1796: done (1 left) 2024-09-12T23:44:33.485 INFO:tasks.rados.rados.0.smithi136.stdout:1799: done (0 left) 2024-09-12T23:44:33.485 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 32 current snap is 173 2024-09-12T23:44:33.485 INFO:tasks.rados.rados.0.smithi136.stdout:1800: seq_num 594 ranges {0=352256} 2024-09-12T23:44:33.488 INFO:tasks.rados.rados.0.smithi136.stdout:1800: writing smithi13667081-32 from 0 to 352256 tid 1 2024-09-12T23:44:33.488 INFO:tasks.rados.rados.0.smithi136.stdout:1801: rollback oid 4 current snap is 173 2024-09-12T23:44:33.488 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 4 to 165 2024-09-12T23:44:33.488 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 17 current snap is 173 2024-09-12T23:44:33.488 INFO:tasks.rados.rados.0.smithi136.stdout:1802: seq_num 595 ranges {0=368640} 2024-09-12T23:44:33.492 INFO:tasks.rados.rados.0.smithi136.stdout:1802: writing smithi13667081-17 from 0 to 368640 tid 1 2024-09-12T23:44:33.492 INFO:tasks.rados.rados.0.smithi136.stdout:1803: rollback oid 34 current snap is 173 2024-09-12T23:44:33.492 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 34 to 167 2024-09-12T23:44:33.492 INFO:tasks.rados.rados.0.smithi136.stdout:1804: setattr oid 14 current snap is 173 2024-09-12T23:44:33.492 INFO:tasks.rados.rados.0.smithi136.stdout:1803: finishing rollback tid 1 to smithi13667081-34 2024-09-12T23:44:33.492 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 34 v 0 (ObjNum 369 snap 0 seq_num 4242521040) dirty dne 2024-09-12T23:44:33.494 INFO:tasks.rados.rados.0.smithi136.stdout:1800: finishing write tid 1 to smithi13667081-32 2024-09-12T23:44:33.497 INFO:tasks.rados.rados.0.smithi136.stdout:1803: done (4 left) 2024-09-12T23:44:33.497 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 24 current snap is 173 2024-09-12T23:44:33.498 INFO:tasks.rados.rados.0.smithi136.stdout:1805: seq_num 596 ranges {1482752=385024} 2024-09-12T23:44:33.514 INFO:tasks.rados.rados.0.smithi136.stdout:1805: writing smithi13667081-24 from 1482752 to 1867776 tid 1 2024-09-12T23:44:33.514 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 15 current snap is 173 2024-09-12T23:44:33.514 INFO:tasks.rados.rados.0.smithi136.stdout:1806: seq_num 597 ranges {1720320=401408} 2024-09-12T23:44:33.533 INFO:tasks.rados.rados.0.smithi136.stdout:1806: writing smithi13667081-15 from 1720320 to 2121728 tid 1 2024-09-12T23:44:33.533 INFO:tasks.rados.rados.0.smithi136.stdout:1807: read oid 30 snap -1 2024-09-12T23:44:33.533 INFO:tasks.rados.rados.0.smithi136.stdout:1807: expect deleted 2024-09-12T23:44:33.533 INFO:tasks.rados.rados.0.smithi136.stdout:1808: read oid 38 snap -1 2024-09-12T23:44:33.533 INFO:tasks.rados.rados.0.smithi136.stdout:1808: expect (ObjNum 531 snap 156 seq_num 531) 2024-09-12T23:44:33.533 INFO:tasks.rados.rados.0.smithi136.stdout:1809: snap_create 2024-09-12T23:44:33.533 INFO:tasks.rados.rados.0.smithi136.stdout:1802: finishing write tid 1 to smithi13667081-17 2024-09-12T23:44:33.533 INFO:tasks.rados.rados.0.smithi136.stdout:1802: finishing write tid 2 to smithi13667081-17 2024-09-12T23:44:33.533 INFO:tasks.rados.rados.0.smithi136.stdout:1800: finishing write tid 2 to smithi13667081-32 2024-09-12T23:44:33.533 INFO:tasks.rados.rados.0.smithi136.stdout:1801: finishing rollback tid 0 to smithi13667081-4 2024-09-12T23:44:33.533 INFO:tasks.rados.rados.0.smithi136.stdout:1802: finishing write tid 3 to smithi13667081-17 2024-09-12T23:44:33.533 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 17 v 1288 (ObjNum 595 snap 173 seq_num 595) dirty exists 2024-09-12T23:44:33.534 INFO:tasks.rados.rados.0.smithi136.stdout:1802: left oid 17 (ObjNum 595 snap 173 seq_num 595) 2024-09-12T23:44:33.534 INFO:tasks.rados.rados.0.smithi136.stdout:1801: finishing rollback tid 1 to smithi13667081-4 2024-09-12T23:44:33.534 INFO:tasks.rados.rados.0.smithi136.stdout:1800: finishing write tid 3 to smithi13667081-32 2024-09-12T23:44:33.534 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 32 v 767 (ObjNum 594 snap 173 seq_num 594) dirty exists 2024-09-12T23:44:33.534 INFO:tasks.rados.rados.0.smithi136.stdout:1800: left oid 32 (ObjNum 594 snap 173 seq_num 594) 2024-09-12T23:44:33.534 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 14 v 1086 (ObjNum 583 snap 169 seq_num 583) dirty exists 2024-09-12T23:44:33.534 INFO:tasks.rados.rados.0.smithi136.stdout:1801: finishing rollback tid 2 to smithi13667081-4 2024-09-12T23:44:33.534 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 4 v 770 (ObjNum 567 snap 162 seq_num 567) dirty exists 2024-09-12T23:44:33.534 INFO:tasks.rados.rados.0.smithi136.stdout:1805: finishing write tid 1 to smithi13667081-24 2024-09-12T23:44:33.534 INFO:tasks.rados.rados.0.smithi136.stdout:1805: finishing write tid 2 to smithi13667081-24 2024-09-12T23:44:33.534 INFO:tasks.rados.rados.0.smithi136.stdout:1805: finishing write tid 3 to smithi13667081-24 2024-09-12T23:44:33.534 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 24 v 625 (ObjNum 596 snap 173 seq_num 596) dirty exists 2024-09-12T23:44:33.534 INFO:tasks.rados.rados.0.smithi136.stdout:1805: left oid 24 (ObjNum 596 snap 173 seq_num 596) 2024-09-12T23:44:33.537 INFO:tasks.rados.rados.0.smithi136.stdout:1806: finishing write tid 1 to smithi13667081-15 2024-09-12T23:44:33.537 INFO:tasks.rados.rados.0.smithi136.stdout:1806: finishing write tid 2 to smithi13667081-15 2024-09-12T23:44:33.539 INFO:tasks.rados.rados.0.smithi136.stdout:1806: finishing write tid 3 to smithi13667081-15 2024-09-12T23:44:33.539 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 15 v 924 (ObjNum 597 snap 173 seq_num 597) dirty exists 2024-09-12T23:44:33.539 INFO:tasks.rados.rados.0.smithi136.stdout:1806: left oid 15 (ObjNum 597 snap 173 seq_num 597) 2024-09-12T23:44:33.546 INFO:tasks.rados.rados.0.smithi136.stdout:1808: expect (ObjNum 531 snap 156 seq_num 531) 2024-09-12T23:44:33.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:33 smithi136 bash[75164]: cluster 2024-09-12T23:44:32.473323+0000 mon.a (mon.0) 419 : cluster [DBG] osdmap e418: 8 total, 8 up, 8 in 2024-09-12T23:44:33.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:33 smithi136 bash[75164]: cluster 2024-09-12T23:44:32.698708+0000 mgr.y (mgr.24661) 140 : cluster [DBG] pgmap v202: 121 pgs: 121 active+clean; 178 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.2 MiB/s rd, 560 KiB/s wr, 6 op/s 2024-09-12T23:44:33.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:33 smithi136 bash[76371]: cluster 2024-09-12T23:44:32.473323+0000 mon.a (mon.0) 419 : cluster [DBG] osdmap e418: 8 total, 8 up, 8 in 2024-09-12T23:44:33.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:33 smithi136 bash[76371]: cluster 2024-09-12T23:44:32.698708+0000 mgr.y (mgr.24661) 140 : cluster [DBG] pgmap v202: 121 pgs: 121 active+clean; 178 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.2 MiB/s rd, 560 KiB/s wr, 6 op/s 2024-09-12T23:44:33.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:33 smithi181 bash[67587]: cluster 2024-09-12T23:44:32.473323+0000 mon.a (mon.0) 419 : cluster [DBG] osdmap e418: 8 total, 8 up, 8 in 2024-09-12T23:44:33.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:33 smithi181 bash[67587]: cluster 2024-09-12T23:44:32.698708+0000 mgr.y (mgr.24661) 140 : cluster [DBG] pgmap v202: 121 pgs: 121 active+clean; 178 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.2 MiB/s rd, 560 KiB/s wr, 6 op/s 2024-09-12T23:44:33.845 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:44:33 smithi181 bash[60471]: ts=2024-09-12T23:44:33.510Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi136\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:44:34.402 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:34 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:44:34.402 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:34 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:44:34.403 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 12 23:44:34 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:44:34.403 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 12 23:44:34 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:44:34.403 INFO:journalctl@ceph.mds.a.smithi136.stdout:Sep 12 23:44:34 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:44:34.404 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:44:34 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:44:34.404 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 12 23:44:34 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:44:34.404 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 12 23:44:34 smithi136 systemd[1]: Stopping Ceph osd.1 for da58a47c-715d-11ef-bceb-c7b262605968... 2024-09-12T23:44:34.404 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 12 23:44:34 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:44:34.405 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:44:34 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:44:34.405 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:44:34 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:44:34.482 INFO:tasks.rados.rados.0.smithi136.stdout:1800: done (8 left) 2024-09-12T23:44:34.482 INFO:tasks.rados.rados.0.smithi136.stdout:1801: done (7 left) 2024-09-12T23:44:34.482 INFO:tasks.rados.rados.0.smithi136.stdout:1802: done (6 left) 2024-09-12T23:44:34.482 INFO:tasks.rados.rados.0.smithi136.stdout:1804: done (5 left) 2024-09-12T23:44:34.482 INFO:tasks.rados.rados.0.smithi136.stdout:1805: done (4 left) 2024-09-12T23:44:34.482 INFO:tasks.rados.rados.0.smithi136.stdout:1806: done (3 left) 2024-09-12T23:44:34.482 INFO:tasks.rados.rados.0.smithi136.stdout:1807: done (2 left) 2024-09-12T23:44:34.482 INFO:tasks.rados.rados.0.smithi136.stdout:1808: done (1 left) 2024-09-12T23:44:34.483 INFO:tasks.rados.rados.0.smithi136.stdout:1809: done (0 left) 2024-09-12T23:44:34.483 INFO:tasks.rados.rados.0.smithi136.stdout:1810: read oid 24 snap -1 2024-09-12T23:44:34.483 INFO:tasks.rados.rados.0.smithi136.stdout:1810: expect (ObjNum 596 snap 173 seq_num 596) 2024-09-12T23:44:34.483 INFO:tasks.rados.rados.0.smithi136.stdout:1811: read oid 14 snap -1 2024-09-12T23:44:34.483 INFO:tasks.rados.rados.0.smithi136.stdout:1811: expect (ObjNum 583 snap 169 seq_num 583) 2024-09-12T23:44:34.483 INFO:tasks.rados.rados.0.smithi136.stdout:1812: delete oid 5 current snap is 174 2024-09-12T23:44:34.507 INFO:tasks.rados.rados.0.smithi136.stdout:1810: expect (ObjNum 596 snap 173 seq_num 596) 2024-09-12T23:44:34.597 INFO:tasks.rados.rados.0.smithi136.stdout:1811: expect (ObjNum 583 snap 169 seq_num 583) 2024-09-12T23:44:34.741 INFO:teuthology.orchestra.run.smithi136.stdout:true 2024-09-12T23:44:34.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:34 smithi136 bash[75164]: cluster 2024-09-12T23:44:33.485130+0000 mon.a (mon.0) 420 : cluster [DBG] osdmap e419: 8 total, 8 up, 8 in 2024-09-12T23:44:34.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:34 smithi136 bash[75164]: audit 2024-09-12T23:44:33.794861+0000 mon.a (mon.0) 421 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:34.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:34 smithi136 bash[75164]: cluster 2024-09-12T23:44:34.406783+0000 mon.a (mon.0) 422 : cluster [INF] osd.1 marked itself down and dead 2024-09-12T23:44:34.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:34 smithi136 bash[76371]: cluster 2024-09-12T23:44:33.485130+0000 mon.a (mon.0) 420 : cluster [DBG] osdmap e419: 8 total, 8 up, 8 in 2024-09-12T23:44:34.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:34 smithi136 bash[76371]: audit 2024-09-12T23:44:33.794861+0000 mon.a (mon.0) 421 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:34.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:34 smithi136 bash[76371]: cluster 2024-09-12T23:44:34.406783+0000 mon.a (mon.0) 422 : cluster [INF] osd.1 marked itself down and dead 2024-09-12T23:44:34.756 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 12 23:44:34 smithi136 bash[40776]: debug 2024-09-12T23:44:34.402+0000 7fd190367640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-12T23:44:34.756 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 12 23:44:34 smithi136 bash[40776]: debug 2024-09-12T23:44:34.402+0000 7fd190367640 -1 osd.1 419 *** Got signal Terminated *** 2024-09-12T23:44:34.757 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 12 23:44:34 smithi136 bash[40776]: debug 2024-09-12T23:44:34.402+0000 7fd190367640 -1 osd.1 419 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-12T23:44:34.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:34 smithi181 bash[67587]: cluster 2024-09-12T23:44:33.485130+0000 mon.a (mon.0) 420 : cluster [DBG] osdmap e419: 8 total, 8 up, 8 in 2024-09-12T23:44:34.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:34 smithi181 bash[67587]: audit 2024-09-12T23:44:33.794861+0000 mon.a (mon.0) 421 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:34.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:34 smithi181 bash[67587]: cluster 2024-09-12T23:44:34.406783+0000 mon.a (mon.0) 422 : cluster [INF] osd.1 marked itself down and dead 2024-09-12T23:44:34.846 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:44:34 smithi181 bash[60471]: ts=2024-09-12T23:44:34.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:44:34.846 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:44:34 smithi181 bash[60471]: ts=2024-09-12T23:44:34.456Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:44:35.026 INFO:teuthology.orchestra.run.smithi136.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-12T23:44:35.027 INFO:teuthology.orchestra.run.smithi136.stdout:alertmanager.a smithi136 *:9093,9094 running (4m) 18s ago 10m 13.8M - 0.25.0 c8568f914cd2 b5197f020eeb 2024-09-12T23:44:35.027 INFO:teuthology.orchestra.run.smithi136.stdout:grafana.a smithi181 *:3000 running (3m) 81s ago 9m 75.3M - 9.4.7 954c08fa6188 256c181be984 2024-09-12T23:44:35.027 INFO:teuthology.orchestra.run.smithi136.stdout:mds.a smithi136 running (11m) 18s ago 11m 14.8M - 17.2.7-1627-gbbdadfa3 214985049bdb 713b9795ad19 2024-09-12T23:44:35.027 INFO:teuthology.orchestra.run.smithi136.stdout:mds.b smithi181 running (11m) 81s ago 11m 17.8M - 17.2.7-1627-gbbdadfa3 214985049bdb ad13179f0868 2024-09-12T23:44:35.027 INFO:teuthology.orchestra.run.smithi136.stdout:mgr.x smithi181 *:8443,9283,8765 running (2m) 81s ago 18m 425M - 18.2.4-837-g7a853b17 f21ba1eea74e cc664dc337f7 2024-09-12T23:44:35.027 INFO:teuthology.orchestra.run.smithi136.stdout:mgr.y smithi136 *:8443,9283,8765 running (3m) 18s ago 20m 485M - 18.2.4-837-g7a853b17 f21ba1eea74e 272f10b42fc3 2024-09-12T23:44:35.027 INFO:teuthology.orchestra.run.smithi136.stdout:mon.a smithi136 running (2m) 18s ago 21m 49.4M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e ad8d21d58ad9 2024-09-12T23:44:35.027 INFO:teuthology.orchestra.run.smithi136.stdout:mon.b smithi181 running (94s) 81s ago 19m 19.5M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e a70ec8315934 2024-09-12T23:44:35.027 INFO:teuthology.orchestra.run.smithi136.stdout:mon.c smithi136 running (2m) 18s ago 18m 41.8M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e 44d1be055a5e 2024-09-12T23:44:35.027 INFO:teuthology.orchestra.run.smithi136.stdout:node-exporter.a smithi136 *:9100 running (4m) 18s ago 10m 8059k - 1.5.0 0da6a335fe13 ccfc37c96620 2024-09-12T23:44:35.027 INFO:teuthology.orchestra.run.smithi136.stdout:node-exporter.b smithi181 *:9100 running (3m) 81s ago 10m 7847k - 1.5.0 0da6a335fe13 80bf0e383230 2024-09-12T23:44:35.028 INFO:teuthology.orchestra.run.smithi136.stdout:osd.0 smithi136 running (22s) 18s ago 17m - 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 7da21f765721 2024-09-12T23:44:35.028 INFO:teuthology.orchestra.run.smithi136.stdout:osd.1 smithi136 running (16m) 18s ago 17m 267M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb c4daee7da836 2024-09-12T23:44:35.028 INFO:teuthology.orchestra.run.smithi136.stdout:osd.2 smithi136 running (16m) 18s ago 16m 264M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 4ab8fd79cdb4 2024-09-12T23:44:35.028 INFO:teuthology.orchestra.run.smithi136.stdout:osd.3 smithi136 running (15m) 18s ago 15m 188M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 9ceb14a27521 2024-09-12T23:44:35.028 INFO:teuthology.orchestra.run.smithi136.stdout:osd.4 smithi181 running (14m) 81s ago 14m 230M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb a35cdca79055 2024-09-12T23:44:35.028 INFO:teuthology.orchestra.run.smithi136.stdout:osd.5 smithi181 running (13m) 81s ago 13m 192M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 8f2e8d494fef 2024-09-12T23:44:35.028 INFO:teuthology.orchestra.run.smithi136.stdout:osd.6 smithi181 running (12m) 81s ago 12m 106M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 9c6bec406408 2024-09-12T23:44:35.028 INFO:teuthology.orchestra.run.smithi136.stdout:osd.7 smithi181 running (11m) 81s ago 11m 174M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb bc0632ae2145 2024-09-12T23:44:35.028 INFO:teuthology.orchestra.run.smithi136.stdout:prometheus.a smithi181 *:9095 running (3m) 81s ago 10m 51.7M - 2.43.0 a07b618ecd1d 3853d003a5cc 2024-09-12T23:44:35.379 INFO:teuthology.orchestra.run.smithi136.stdout:{ 2024-09-12T23:44:35.380 INFO:teuthology.orchestra.run.smithi136.stdout: "mon": { 2024-09-12T23:44:35.380 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 3 2024-09-12T23:44:35.380 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:44:35.380 INFO:teuthology.orchestra.run.smithi136.stdout: "mgr": { 2024-09-12T23:44:35.380 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 2 2024-09-12T23:44:35.380 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:44:35.380 INFO:teuthology.orchestra.run.smithi136.stdout: "osd": { 2024-09-12T23:44:35.380 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 6, 2024-09-12T23:44:35.380 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 1 2024-09-12T23:44:35.380 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:44:35.381 INFO:teuthology.orchestra.run.smithi136.stdout: "mds": { 2024-09-12T23:44:35.381 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-12T23:44:35.381 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:44:35.381 INFO:teuthology.orchestra.run.smithi136.stdout: "overall": { 2024-09-12T23:44:35.381 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 8, 2024-09-12T23:44:35.381 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 6 2024-09-12T23:44:35.381 INFO:teuthology.orchestra.run.smithi136.stdout: } 2024-09-12T23:44:35.381 INFO:teuthology.orchestra.run.smithi136.stdout:} 2024-09-12T23:44:35.521 INFO:tasks.rados.rados.0.smithi136.stdout:1810: done (2 left) 2024-09-12T23:44:35.521 INFO:tasks.rados.rados.0.smithi136.stdout:1811: done (1 left) 2024-09-12T23:44:35.522 INFO:tasks.rados.rados.0.smithi136.stdout:1812: done (0 left) 2024-09-12T23:44:35.522 INFO:tasks.rados.rados.0.smithi136.stdout:1813: read oid 2 snap -1 2024-09-12T23:44:35.522 INFO:tasks.rados.rados.0.smithi136.stdout:1813: expect (ObjNum 416 snap 120 seq_num 416) 2024-09-12T23:44:35.522 INFO:tasks.rados.rados.0.smithi136.stdout:1814: rollback oid 35 current snap is 174 2024-09-12T23:44:35.522 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 35 to 173 2024-09-12T23:44:35.522 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 8 current snap is 174 2024-09-12T23:44:35.522 INFO:tasks.rados.rados.0.smithi136.stdout:1815: seq_num 599 ranges {6316032=434176} 2024-09-12T23:44:35.579 INFO:tasks.rados.rados.0.smithi136.stdout:1815: writing smithi13667081-8 from 6316032 to 6750208 tid 1 2024-09-12T23:44:35.579 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 24 current snap is 174 2024-09-12T23:44:35.579 INFO:tasks.rados.rados.0.smithi136.stdout:1814: finishing rollback tid 0 to smithi13667081-35 2024-09-12T23:44:35.579 INFO:tasks.rados.rados.0.smithi136.stdout:1814: finishing rollback tid 1 to smithi13667081-35 2024-09-12T23:44:35.579 INFO:tasks.rados.rados.0.smithi136.stdout:1814: finishing rollback tid 2 to smithi13667081-35 2024-09-12T23:44:35.579 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 35 v 1294 (ObjNum 469 snap 139 seq_num 469) dirty exists 2024-09-12T23:44:35.579 INFO:tasks.rados.rados.0.smithi136.stdout:1813: expect (ObjNum 416 snap 120 seq_num 416) 2024-09-12T23:44:35.756 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 12 23:44:35 smithi136 bash[84316]: ceph-da58a47c-715d-11ef-bceb-c7b262605968-osd-1 2024-09-12T23:44:35.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:35 smithi136 bash[75164]: cluster 2024-09-12T23:44:34.477207+0000 mon.a (mon.0) 423 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-12T23:44:35.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:35 smithi136 bash[75164]: cluster 2024-09-12T23:44:34.485039+0000 mon.a (mon.0) 424 : cluster [DBG] osdmap e420: 8 total, 7 up, 8 in 2024-09-12T23:44:35.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:35 smithi136 bash[75164]: cluster 2024-09-12T23:44:34.700159+0000 mgr.y (mgr.24661) 141 : cluster [DBG] pgmap v205: 121 pgs: 38 peering, 2 stale+active+clean, 81 active+clean; 181 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.6 MiB/s rd, 686 KiB/s wr, 16 op/s 2024-09-12T23:44:35.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:35 smithi136 bash[75164]: audit 2024-09-12T23:44:34.734242+0000 mgr.y (mgr.24661) 142 : audit [DBG] from='client.44156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:44:35.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:35 smithi136 bash[75164]: audit 2024-09-12T23:44:35.383033+0000 mon.c (mon.2) 4 : audit [DBG] from='client.? 172.21.15.136:0/491455225' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:44:35.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:35 smithi136 bash[75164]: cluster 2024-09-12T23:44:35.488011+0000 mon.a (mon.0) 425 : cluster [DBG] osdmap e421: 8 total, 7 up, 8 in 2024-09-12T23:44:35.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:35 smithi136 bash[76371]: cluster 2024-09-12T23:44:34.477207+0000 mon.a (mon.0) 423 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-12T23:44:35.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:35 smithi136 bash[76371]: cluster 2024-09-12T23:44:34.485039+0000 mon.a (mon.0) 424 : cluster [DBG] osdmap e420: 8 total, 7 up, 8 in 2024-09-12T23:44:35.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:35 smithi136 bash[76371]: cluster 2024-09-12T23:44:34.700159+0000 mgr.y (mgr.24661) 141 : cluster [DBG] pgmap v205: 121 pgs: 38 peering, 2 stale+active+clean, 81 active+clean; 181 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.6 MiB/s rd, 686 KiB/s wr, 16 op/s 2024-09-12T23:44:35.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:35 smithi136 bash[76371]: audit 2024-09-12T23:44:34.734242+0000 mgr.y (mgr.24661) 142 : audit [DBG] from='client.44156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:44:35.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:35 smithi136 bash[76371]: audit 2024-09-12T23:44:35.383033+0000 mon.c (mon.2) 4 : audit [DBG] from='client.? 172.21.15.136:0/491455225' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:44:35.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:35 smithi136 bash[76371]: cluster 2024-09-12T23:44:35.488011+0000 mon.a (mon.0) 425 : cluster [DBG] osdmap e421: 8 total, 7 up, 8 in 2024-09-12T23:44:35.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:35 smithi181 bash[67587]: cluster 2024-09-12T23:44:34.477207+0000 mon.a (mon.0) 423 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-12T23:44:35.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:35 smithi181 bash[67587]: cluster 2024-09-12T23:44:34.485039+0000 mon.a (mon.0) 424 : cluster [DBG] osdmap e420: 8 total, 7 up, 8 in 2024-09-12T23:44:35.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:35 smithi181 bash[67587]: cluster 2024-09-12T23:44:34.700159+0000 mgr.y (mgr.24661) 141 : cluster [DBG] pgmap v205: 121 pgs: 38 peering, 2 stale+active+clean, 81 active+clean; 181 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.6 MiB/s rd, 686 KiB/s wr, 16 op/s 2024-09-12T23:44:35.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:35 smithi181 bash[67587]: audit 2024-09-12T23:44:34.734242+0000 mgr.y (mgr.24661) 142 : audit [DBG] from='client.44156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:44:35.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:35 smithi181 bash[67587]: audit 2024-09-12T23:44:35.383033+0000 mon.c (mon.2) 4 : audit [DBG] from='client.? 172.21.15.136:0/491455225' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:44:35.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:35 smithi181 bash[67587]: cluster 2024-09-12T23:44:35.488011+0000 mon.a (mon.0) 425 : cluster [DBG] osdmap e421: 8 total, 7 up, 8 in 2024-09-12T23:44:36.002 INFO:tasks.rados.rados.0.smithi136.stdout:1815: finishing write tid 1 to smithi13667081-8 2024-09-12T23:44:36.002 INFO:tasks.rados.rados.0.smithi136.stdout:1815: finishing write tid 2 to smithi13667081-8 2024-09-12T23:44:36.002 INFO:tasks.rados.rados.0.smithi136.stdout:1815: finishing write tid 3 to smithi13667081-8 2024-09-12T23:44:36.002 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 8 v 783 (ObjNum 599 snap 174 seq_num 599) dirty exists 2024-09-12T23:44:36.002 INFO:tasks.rados.rados.0.smithi136.stdout:1815: left oid 8 (ObjNum 599 snap 174 seq_num 599) 2024-09-12T23:44:36.002 INFO:tasks.rados.rados.0.smithi136.stdout:1816: seq_num 600 ranges {1867776=450560} 2024-09-12T23:44:36.014 INFO:tasks.rados.rados.0.smithi136.stdout:1816: writing smithi13667081-24 from 1867776 to 2318336 tid 1 2024-09-12T23:44:36.014 INFO:tasks.rados.rados.0.smithi136.stdout:1813: done (3 left) 2024-09-12T23:44:36.014 INFO:tasks.rados.rados.0.smithi136.stdout:1814: done (2 left) 2024-09-12T23:44:36.014 INFO:tasks.rados.rados.0.smithi136.stdout:1815: done (1 left) 2024-09-12T23:44:36.014 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 38 current snap is 174 2024-09-12T23:44:36.015 INFO:tasks.rados.rados.0.smithi136.stdout:1817: seq_num 601 ranges {1703936=475136} 2024-09-12T23:44:36.026 INFO:tasks.rados.rados.0.smithi136.stdout:1817: writing smithi13667081-38 from 1703936 to 2179072 tid 1 2024-09-12T23:44:36.026 INFO:tasks.rados.rados.0.smithi136.stdout:1816: finishing write tid 1 to smithi13667081-24 2024-09-12T23:44:36.027 INFO:tasks.rados.rados.0.smithi136.stdout:1816: finishing write tid 2 to smithi13667081-24 2024-09-12T23:44:36.027 INFO:tasks.rados.rados.0.smithi136.stdout:1816: finishing write tid 3 to smithi13667081-24 2024-09-12T23:44:36.027 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 24 v 627 (ObjNum 600 snap 174 seq_num 600) dirty exists 2024-09-12T23:44:36.027 INFO:tasks.rados.rados.0.smithi136.stdout:1816: left oid 24 (ObjNum 600 snap 174 seq_num 600) 2024-09-12T23:44:36.027 INFO:tasks.rados.rados.0.smithi136.stdout:1818: read oid 28 snap -1 2024-09-12T23:44:36.027 INFO:tasks.rados.rados.0.smithi136.stdout:1818: expect (ObjNum 391 snap 113 seq_num 391) 2024-09-12T23:44:36.027 INFO:tasks.rados.rados.0.smithi136.stdout:1816: done (2 left) 2024-09-12T23:44:36.027 INFO:tasks.rados.rados.0.smithi136.stdout:1819: rmattr oid 32 current snap is 174 2024-09-12T23:44:36.027 INFO:tasks.rados.rados.0.smithi136.stdout:1819: done (2 left) 2024-09-12T23:44:36.027 INFO:tasks.rados.rados.0.smithi136.stdout:1820: snap_remove snap 173 2024-09-12T23:44:36.256 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 12 23:44:35 smithi136 bash[84451]: Error response from daemon: No such container: ceph-da58a47c-715d-11ef-bceb-c7b262605968-osd-1 2024-09-12T23:44:36.323 INFO:tasks.rados.rados.0.smithi136.stdout:1820: done (2 left) 2024-09-12T23:44:36.323 INFO:tasks.rados.rados.0.smithi136.stdout:1821: snap_remove snap 165 2024-09-12T23:44:36.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:36 smithi136 bash[75164]: audit 2024-09-12T23:44:35.013103+0000 mgr.y (mgr.24661) 143 : audit [DBG] from='client.44162 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:44:36.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:36 smithi136 bash[75164]: cluster 2024-09-12T23:44:35.489975+0000 mon.a (mon.0) 426 : cluster [WRN] Health check failed: Reduced data availability: 8 pgs inactive, 12 pgs peering (PG_AVAILABILITY) 2024-09-12T23:44:36.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:36 smithi136 bash[75164]: cluster 2024-09-12T23:44:36.326989+0000 mon.a (mon.0) 427 : cluster [DBG] osdmap e422: 8 total, 7 up, 8 in 2024-09-12T23:44:36.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:36 smithi136 bash[76371]: audit 2024-09-12T23:44:35.013103+0000 mgr.y (mgr.24661) 143 : audit [DBG] from='client.44162 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:44:36.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:36 smithi136 bash[76371]: cluster 2024-09-12T23:44:35.489975+0000 mon.a (mon.0) 426 : cluster [WRN] Health check failed: Reduced data availability: 8 pgs inactive, 12 pgs peering (PG_AVAILABILITY) 2024-09-12T23:44:36.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:36 smithi136 bash[76371]: cluster 2024-09-12T23:44:36.326989+0000 mon.a (mon.0) 427 : cluster [DBG] osdmap e422: 8 total, 7 up, 8 in 2024-09-12T23:44:36.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:36 smithi181 bash[67587]: audit 2024-09-12T23:44:35.013103+0000 mgr.y (mgr.24661) 143 : audit [DBG] from='client.44162 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:44:36.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:36 smithi181 bash[67587]: cluster 2024-09-12T23:44:35.489975+0000 mon.a (mon.0) 426 : cluster [WRN] Health check failed: Reduced data availability: 8 pgs inactive, 12 pgs peering (PG_AVAILABILITY) 2024-09-12T23:44:36.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:36 smithi181 bash[67587]: cluster 2024-09-12T23:44:36.326989+0000 mon.a (mon.0) 427 : cluster [DBG] osdmap e422: 8 total, 7 up, 8 in 2024-09-12T23:44:37.328 INFO:tasks.rados.rados.0.smithi136.stdout:1821: done (2 left) 2024-09-12T23:44:37.328 INFO:tasks.rados.rados.0.smithi136.stdout:1822: rollback oid 41 current snap is 174 2024-09-12T23:44:37.328 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 41 to 172 2024-09-12T23:44:37.328 INFO:tasks.rados.rados.0.smithi136.stdout:1817: finishing write tid 1 to smithi13667081-38 2024-09-12T23:44:37.328 INFO:tasks.rados.rados.0.smithi136.stdout:1817: finishing write tid 2 to smithi13667081-38 2024-09-12T23:44:37.328 INFO:tasks.rados.rados.0.smithi136.stdout:1817: finishing write tid 3 to smithi13667081-38 2024-09-12T23:44:37.328 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 38 v 820 (ObjNum 601 snap 174 seq_num 601) dirty exists 2024-09-12T23:44:37.328 INFO:tasks.rados.rados.0.smithi136.stdout:1817: left oid 38 (ObjNum 601 snap 174 seq_num 601) 2024-09-12T23:44:37.328 INFO:tasks.rados.rados.0.smithi136.stdout:1818: expect (ObjNum 391 snap 113 seq_num 391) 2024-09-12T23:44:37.582 INFO:tasks.rados.rados.0.smithi136.stdout:1822: finishing rollback tid 1 to smithi13667081-41 2024-09-12T23:44:37.583 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 41 v 952 (ObjNum 305 snap 0 seq_num 324173362) dirty dne 2024-09-12T23:44:37.583 INFO:tasks.rados.rados.0.smithi136.stdout:1817: done (2 left) 2024-09-12T23:44:37.583 INFO:tasks.rados.rados.0.smithi136.stdout:1818: done (1 left) 2024-09-12T23:44:37.583 INFO:tasks.rados.rados.0.smithi136.stdout:1822: done (0 left) 2024-09-12T23:44:37.583 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 27 current snap is 174 2024-09-12T23:44:37.583 INFO:tasks.rados.rados.0.smithi136.stdout:1823: seq_num 602 ranges {7053312=491520} 2024-09-12T23:44:37.663 INFO:tasks.rados.rados.0.smithi136.stdout:1823: writing smithi13667081-27 from 7053312 to 7544832 tid 1 2024-09-12T23:44:37.663 INFO:tasks.rados.rados.0.smithi136.stdout:1824: rmattr oid 29 current snap is 174 2024-09-12T23:44:37.663 INFO:tasks.rados.rados.0.smithi136.stdout:1825: snap_remove snap 167 2024-09-12T23:44:37.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:37 smithi136 bash[75164]: cluster 2024-09-12T23:44:36.701159+0000 mgr.y (mgr.24661) 144 : cluster [DBG] pgmap v208: 121 pgs: 38 peering, 2 stale+active+clean, 81 active+clean; 181 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.6 MiB/s rd, 686 KiB/s wr, 16 op/s 2024-09-12T23:44:37.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:37 smithi136 bash[75164]: cluster 2024-09-12T23:44:37.331615+0000 mon.a (mon.0) 428 : cluster [DBG] osdmap e423: 8 total, 7 up, 8 in 2024-09-12T23:44:37.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:37 smithi136 bash[76371]: cluster 2024-09-12T23:44:36.701159+0000 mgr.y (mgr.24661) 144 : cluster [DBG] pgmap v208: 121 pgs: 38 peering, 2 stale+active+clean, 81 active+clean; 181 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.6 MiB/s rd, 686 KiB/s wr, 16 op/s 2024-09-12T23:44:37.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:37 smithi136 bash[76371]: cluster 2024-09-12T23:44:37.331615+0000 mon.a (mon.0) 428 : cluster [DBG] osdmap e423: 8 total, 7 up, 8 in 2024-09-12T23:44:37.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:37 smithi181 bash[67587]: cluster 2024-09-12T23:44:36.701159+0000 mgr.y (mgr.24661) 144 : cluster [DBG] pgmap v208: 121 pgs: 38 peering, 2 stale+active+clean, 81 active+clean; 181 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.6 MiB/s rd, 686 KiB/s wr, 16 op/s 2024-09-12T23:44:37.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:37 smithi181 bash[67587]: cluster 2024-09-12T23:44:37.331615+0000 mon.a (mon.0) 428 : cluster [DBG] osdmap e423: 8 total, 7 up, 8 in 2024-09-12T23:44:38.408 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:38 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:44:38.409 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 12 23:44:38 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:44:38.409 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 12 23:44:38 smithi136 systemd[1]: ceph-da58a47c-715d-11ef-bceb-c7b262605968@osd.1.service: Deactivated successfully. 2024-09-12T23:44:38.409 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 12 23:44:38 smithi136 systemd[1]: Stopped Ceph osd.1 for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:44:38.409 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 12 23:44:38 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:44:38.410 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 12 23:44:38 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:44:38.410 INFO:journalctl@ceph.mds.a.smithi136.stdout:Sep 12 23:44:38 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:44:38.410 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:44:38 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:44:38.410 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:44:38 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:44:38.411 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:38 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:44:38.411 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:44:38 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:44:38.411 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 12 23:44:38 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:44:38.514 INFO:tasks.rados.rados.0.smithi136.stdout:1825: done (2 left) 2024-09-12T23:44:38.514 INFO:tasks.rados.rados.0.smithi136.stdout:1826: copy_from oid 34 from oid 6 current snap is 174 2024-09-12T23:44:38.514 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 29 v 821 (ObjNum 579 snap 169 seq_num 579) dirty exists 2024-09-12T23:44:38.514 INFO:tasks.rados.rados.0.smithi136.stdout:1823: finishing write tid 1 to smithi13667081-27 2024-09-12T23:44:38.514 INFO:tasks.rados.rados.0.smithi136.stdout:1823: finishing write tid 2 to smithi13667081-27 2024-09-12T23:44:38.514 INFO:tasks.rados.rados.0.smithi136.stdout:1823: finishing write tid 3 to smithi13667081-27 2024-09-12T23:44:38.515 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 27 v 1297 (ObjNum 602 snap 174 seq_num 602) dirty exists 2024-09-12T23:44:38.515 INFO:tasks.rados.rados.0.smithi136.stdout:1823: left oid 27 (ObjNum 602 snap 174 seq_num 602) 2024-09-12T23:44:38.515 INFO:tasks.rados.rados.0.smithi136.stdout:1823: done (2 left) 2024-09-12T23:44:38.515 INFO:tasks.rados.rados.0.smithi136.stdout:1824: done (1 left) 2024-09-12T23:44:38.515 INFO:tasks.rados.rados.0.smithi136.stdout:1827: snap_remove snap 170 2024-09-12T23:44:38.756 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 12 23:44:38 smithi136 systemd[1]: Started Ceph osd.1 for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:44:39.094 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:44:38 smithi181 bash[60471]: ts=2024-09-12T23:44:38.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:44:39.527 INFO:tasks.rados.rados.0.smithi136.stdout:1827: done (1 left) 2024-09-12T23:44:39.527 INFO:tasks.rados.rados.0.smithi136.stdout:1828: read oid 33 snap -1 2024-09-12T23:44:39.527 INFO:tasks.rados.rados.0.smithi136.stdout:1828: expect deleted 2024-09-12T23:44:39.527 INFO:tasks.rados.rados.0.smithi136.stdout:1829: delete oid 46 current snap is 174 2024-09-12T23:44:39.527 INFO:tasks.rados.rados.0.smithi136.stdout:1826: finishing copy_from racing read to smithi13667081-34 2024-09-12T23:44:39.528 INFO:tasks.rados.rados.0.smithi136.stdout:1826: finishing copy_from to smithi13667081-34 2024-09-12T23:44:39.528 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 34 v 1206 (ObjNum 559 snap 160 seq_num 559) dirty exists 2024-09-12T23:44:39.545 INFO:tasks.rados.rados.0.smithi136.stdout:1826: done (2 left) 2024-09-12T23:44:39.545 INFO:tasks.rados.rados.0.smithi136.stdout:1828: done (1 left) 2024-09-12T23:44:39.545 INFO:tasks.rados.rados.0.smithi136.stdout:1829: done (0 left) 2024-09-12T23:44:39.545 INFO:tasks.rados.rados.0.smithi136.stdout:1830: read oid 32 snap -1 2024-09-12T23:44:39.545 INFO:tasks.rados.rados.0.smithi136.stdout:1830: expect (ObjNum 594 snap 173 seq_num 594) 2024-09-12T23:44:39.545 INFO:tasks.rados.rados.0.smithi136.stdout:1831: snap_remove snap 172 2024-09-12T23:44:39.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:44:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:44:39] "GET /metrics HTTP/1.1" 200 36401 "" "Prometheus/2.43.0" 2024-09-12T23:44:39.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:39 smithi136 bash[76371]: audit 2024-09-12T23:44:38.445292+0000 mon.a (mon.0) 429 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:39.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:39 smithi136 bash[76371]: audit 2024-09-12T23:44:38.451598+0000 mon.a (mon.0) 430 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:39.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:39 smithi136 bash[76371]: audit 2024-09-12T23:44:38.452606+0000 mon.a (mon.0) 431 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:44:39.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:39 smithi136 bash[76371]: cluster 2024-09-12T23:44:38.513151+0000 mon.a (mon.0) 432 : cluster [DBG] osdmap e424: 8 total, 7 up, 8 in 2024-09-12T23:44:39.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:39 smithi136 bash[76371]: cluster 2024-09-12T23:44:38.702107+0000 mgr.y (mgr.24661) 145 : cluster [DBG] pgmap v211: 121 pgs: 7 active+undersized, 38 peering, 1 active+undersized+degraded, 75 active+clean; 182 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 210 KiB/s wr, 1 op/s; 13/294 objects degraded (4.422%) 2024-09-12T23:44:39.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:39 smithi136 bash[76371]: audit 2024-09-12T23:44:38.747413+0000 mon.a (mon.0) 433 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:39.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:39 smithi136 bash[76371]: audit 2024-09-12T23:44:38.748243+0000 mon.a (mon.0) 434 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:44:39.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:39 smithi136 bash[76371]: audit 2024-09-12T23:44:38.776388+0000 mon.a (mon.0) 435 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:39.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:39 smithi136 bash[76371]: audit 2024-09-12T23:44:38.782962+0000 mon.a (mon.0) 436 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:39.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:39 smithi136 bash[75164]: audit 2024-09-12T23:44:38.445292+0000 mon.a (mon.0) 429 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:39.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:39 smithi136 bash[75164]: audit 2024-09-12T23:44:38.451598+0000 mon.a (mon.0) 430 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:39.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:39 smithi136 bash[75164]: audit 2024-09-12T23:44:38.452606+0000 mon.a (mon.0) 431 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:44:39.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:39 smithi136 bash[75164]: cluster 2024-09-12T23:44:38.513151+0000 mon.a (mon.0) 432 : cluster [DBG] osdmap e424: 8 total, 7 up, 8 in 2024-09-12T23:44:39.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:39 smithi136 bash[75164]: cluster 2024-09-12T23:44:38.702107+0000 mgr.y (mgr.24661) 145 : cluster [DBG] pgmap v211: 121 pgs: 7 active+undersized, 38 peering, 1 active+undersized+degraded, 75 active+clean; 182 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 210 KiB/s wr, 1 op/s; 13/294 objects degraded (4.422%) 2024-09-12T23:44:39.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:39 smithi136 bash[75164]: audit 2024-09-12T23:44:38.747413+0000 mon.a (mon.0) 433 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:39.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:39 smithi136 bash[75164]: audit 2024-09-12T23:44:38.748243+0000 mon.a (mon.0) 434 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:44:39.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:39 smithi136 bash[75164]: audit 2024-09-12T23:44:38.776388+0000 mon.a (mon.0) 435 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:39.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:39 smithi136 bash[75164]: audit 2024-09-12T23:44:38.782962+0000 mon.a (mon.0) 436 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:39.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:39 smithi181 bash[67587]: audit 2024-09-12T23:44:38.445292+0000 mon.a (mon.0) 429 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:39.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:39 smithi181 bash[67587]: audit 2024-09-12T23:44:38.451598+0000 mon.a (mon.0) 430 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:39.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:39 smithi181 bash[67587]: audit 2024-09-12T23:44:38.452606+0000 mon.a (mon.0) 431 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:44:39.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:39 smithi181 bash[67587]: cluster 2024-09-12T23:44:38.513151+0000 mon.a (mon.0) 432 : cluster [DBG] osdmap e424: 8 total, 7 up, 8 in 2024-09-12T23:44:39.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:39 smithi181 bash[67587]: cluster 2024-09-12T23:44:38.702107+0000 mgr.y (mgr.24661) 145 : cluster [DBG] pgmap v211: 121 pgs: 7 active+undersized, 38 peering, 1 active+undersized+degraded, 75 active+clean; 182 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 210 KiB/s wr, 1 op/s; 13/294 objects degraded (4.422%) 2024-09-12T23:44:39.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:39 smithi181 bash[67587]: audit 2024-09-12T23:44:38.747413+0000 mon.a (mon.0) 433 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:39.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:39 smithi181 bash[67587]: audit 2024-09-12T23:44:38.748243+0000 mon.a (mon.0) 434 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:44:39.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:39 smithi181 bash[67587]: audit 2024-09-12T23:44:38.776388+0000 mon.a (mon.0) 435 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:39.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:39 smithi181 bash[67587]: audit 2024-09-12T23:44:38.782962+0000 mon.a (mon.0) 436 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:40.233 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 12 23:44:39 smithi136 bash[84599]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-12T23:44:40.234 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 12 23:44:39 smithi136 bash[84599]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-12T23:44:40.234 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 12 23:44:39 smithi136 bash[84599]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-12T23:44:40.234 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 12 23:44:39 smithi136 bash[84599]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-12T23:44:40.234 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 12 23:44:39 smithi136 bash[84599]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-12T23:44:40.234 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 12 23:44:39 smithi136 bash[84599]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-12T23:44:40.234 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 12 23:44:39 smithi136 bash[84599]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-12T23:44:40.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:40 smithi136 bash[75164]: cluster 2024-09-12T23:44:39.509547+0000 mon.a (mon.0) 437 : cluster [WRN] Health check failed: Degraded data redundancy: 13/294 objects degraded (4.422%), 1 pg degraded (PG_DEGRADED) 2024-09-12T23:44:40.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:40 smithi136 bash[75164]: cluster 2024-09-12T23:44:39.529732+0000 mon.a (mon.0) 438 : cluster [DBG] osdmap e425: 8 total, 7 up, 8 in 2024-09-12T23:44:40.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:40 smithi136 bash[76371]: cluster 2024-09-12T23:44:39.509547+0000 mon.a (mon.0) 437 : cluster [WRN] Health check failed: Degraded data redundancy: 13/294 objects degraded (4.422%), 1 pg degraded (PG_DEGRADED) 2024-09-12T23:44:40.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:40 smithi136 bash[76371]: cluster 2024-09-12T23:44:39.529732+0000 mon.a (mon.0) 438 : cluster [DBG] osdmap e425: 8 total, 7 up, 8 in 2024-09-12T23:44:40.529 INFO:tasks.rados.rados.0.smithi136.stdout:1831: done (1 left) 2024-09-12T23:44:40.529 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 6 current snap is 174 2024-09-12T23:44:40.529 INFO:tasks.rados.rados.0.smithi136.stdout:1832: seq_num 604 ranges {4005888=524288} 2024-09-12T23:44:40.560 INFO:tasks.rados.rados.0.smithi136.stdout:1832: writing smithi13667081-6 from 4005888 to 4530176 tid 1 2024-09-12T23:44:40.561 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 8 current snap is 174 2024-09-12T23:44:40.561 INFO:tasks.rados.rados.0.smithi136.stdout:1833: seq_num 605 ranges {6750208=540672} 2024-09-12T23:44:40.611 INFO:tasks.rados.rados.0.smithi136.stdout:1833: writing smithi13667081-8 from 6750208 to 7290880 tid 1 2024-09-12T23:44:40.612 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 28 current snap is 174 2024-09-12T23:44:40.612 INFO:tasks.rados.rados.0.smithi136.stdout:1834: seq_num 606 ranges {5603328=557056} 2024-09-12T23:44:40.646 INFO:tasks.rados.rados.0.smithi136.stdout:1834: writing smithi13667081-28 from 5603328 to 6160384 tid 1 2024-09-12T23:44:40.646 INFO:tasks.rados.rados.0.smithi136.stdout:1835: snap_create 2024-09-12T23:44:40.646 INFO:tasks.rados.rados.0.smithi136.stdout:1830: expect (ObjNum 594 snap 173 seq_num 594) 2024-09-12T23:44:40.669 INFO:tasks.rados.rados.0.smithi136.stdout:1832: finishing write tid 1 to smithi13667081-6 2024-09-12T23:44:40.669 INFO:tasks.rados.rados.0.smithi136.stdout:1832: finishing write tid 2 to smithi13667081-6 2024-09-12T23:44:40.669 INFO:tasks.rados.rados.0.smithi136.stdout:1832: finishing write tid 3 to smithi13667081-6 2024-09-12T23:44:40.669 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 6 v 1208 (ObjNum 604 snap 174 seq_num 604) dirty exists 2024-09-12T23:44:40.669 INFO:tasks.rados.rados.0.smithi136.stdout:1832: left oid 6 (ObjNum 604 snap 174 seq_num 604) 2024-09-12T23:44:40.669 INFO:tasks.rados.rados.0.smithi136.stdout:1833: finishing write tid 1 to smithi13667081-8 2024-09-12T23:44:40.669 INFO:tasks.rados.rados.0.smithi136.stdout:1833: finishing write tid 2 to smithi13667081-8 2024-09-12T23:44:40.669 INFO:tasks.rados.rados.0.smithi136.stdout:1833: finishing write tid 3 to smithi13667081-8 2024-09-12T23:44:40.669 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 8 v 785 (ObjNum 605 snap 174 seq_num 605) dirty exists 2024-09-12T23:44:40.670 INFO:tasks.rados.rados.0.smithi136.stdout:1833: left oid 8 (ObjNum 605 snap 174 seq_num 605) 2024-09-12T23:44:40.670 INFO:tasks.rados.rados.0.smithi136.stdout:1834: finishing write tid 1 to smithi13667081-28 2024-09-12T23:44:40.670 INFO:tasks.rados.rados.0.smithi136.stdout:1834: finishing write tid 2 to smithi13667081-28 2024-09-12T23:44:40.670 INFO:tasks.rados.rados.0.smithi136.stdout:1834: finishing write tid 3 to smithi13667081-28 2024-09-12T23:44:40.670 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 28 v 1096 (ObjNum 606 snap 174 seq_num 606) dirty exists 2024-09-12T23:44:40.670 INFO:tasks.rados.rados.0.smithi136.stdout:1834: left oid 28 (ObjNum 606 snap 174 seq_num 606) 2024-09-12T23:44:40.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:40 smithi181 bash[67587]: cluster 2024-09-12T23:44:39.509547+0000 mon.a (mon.0) 437 : cluster [WRN] Health check failed: Degraded data redundancy: 13/294 objects degraded (4.422%), 1 pg degraded (PG_DEGRADED) 2024-09-12T23:44:40.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:40 smithi181 bash[67587]: cluster 2024-09-12T23:44:39.529732+0000 mon.a (mon.0) 438 : cluster [DBG] osdmap e425: 8 total, 7 up, 8 in 2024-09-12T23:44:41.537 INFO:tasks.rados.rados.0.smithi136.stdout:1830: done (4 left) 2024-09-12T23:44:41.537 INFO:tasks.rados.rados.0.smithi136.stdout:1832: done (3 left) 2024-09-12T23:44:41.537 INFO:tasks.rados.rados.0.smithi136.stdout:1833: done (2 left) 2024-09-12T23:44:41.537 INFO:tasks.rados.rados.0.smithi136.stdout:1834: done (1 left) 2024-09-12T23:44:41.537 INFO:tasks.rados.rados.0.smithi136.stdout:1835: done (0 left) 2024-09-12T23:44:41.538 INFO:tasks.rados.rados.0.smithi136.stdout:1836: rmattr oid 13 current snap is 175 2024-09-12T23:44:41.538 INFO:tasks.rados.rados.0.smithi136.stdout:1836: done (0 left) 2024-09-12T23:44:41.538 INFO:tasks.rados.rados.0.smithi136.stdout:1837: read oid 37 snap 174 2024-09-12T23:44:41.538 INFO:tasks.rados.rados.0.smithi136.stdout:1837: expect (ObjNum 589 snap 172 seq_num 589) 2024-09-12T23:44:41.538 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 34 current snap is 175 2024-09-12T23:44:41.538 INFO:tasks.rados.rados.0.smithi136.stdout:1838: seq_num 607 ranges {4005888=573440} 2024-09-12T23:44:41.564 INFO:tasks.rados.rados.0.smithi136.stdout:1838: writing smithi13667081-34 from 4005888 to 4579328 tid 1 2024-09-12T23:44:41.564 INFO:tasks.rados.rados.0.smithi136.stdout:1839: rollback oid 1 current snap is 175 2024-09-12T23:44:41.564 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 1 to 174 2024-09-12T23:44:41.564 INFO:tasks.rados.rados.0.smithi136.stdout:1837: expect (ObjNum 589 snap 172 seq_num 589) 2024-09-12T23:44:41.597 INFO:tasks.rados.rados.0.smithi136.stdout:1839: finishing rollback tid 1 to smithi13667081-1 2024-09-12T23:44:41.597 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 1 v 0 (ObjNum 33 snap 0 seq_num 4242797232) dirty dne 2024-09-12T23:44:41.598 INFO:tasks.rados.rados.0.smithi136.stdout:1838: finishing write tid 1 to smithi13667081-34 2024-09-12T23:44:41.598 INFO:tasks.rados.rados.0.smithi136.stdout:1838: finishing write tid 2 to smithi13667081-34 2024-09-12T23:44:41.598 INFO:tasks.rados.rados.0.smithi136.stdout:1838: finishing write tid 3 to smithi13667081-34 2024-09-12T23:44:41.598 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 34 v 1208 (ObjNum 607 snap 175 seq_num 607) dirty exists 2024-09-12T23:44:41.598 INFO:tasks.rados.rados.0.smithi136.stdout:1838: left oid 34 (ObjNum 607 snap 175 seq_num 607) 2024-09-12T23:44:41.598 INFO:tasks.rados.rados.0.smithi136.stdout:1837: done (2 left) 2024-09-12T23:44:41.598 INFO:tasks.rados.rados.0.smithi136.stdout:1838: done (1 left) 2024-09-12T23:44:41.598 INFO:tasks.rados.rados.0.smithi136.stdout:1839: done (0 left) 2024-09-12T23:44:41.598 INFO:tasks.rados.rados.0.smithi136.stdout:1840: read oid 25 snap -1 2024-09-12T23:44:41.598 INFO:tasks.rados.rados.0.smithi136.stdout:1840: expect (ObjNum 588 snap 172 seq_num 588) 2024-09-12T23:44:41.598 INFO:tasks.rados.rados.0.smithi136.stdout:1841: read oid 16 snap -1 2024-09-12T23:44:41.598 INFO:tasks.rados.rados.0.smithi136.stdout:1841: expect deleted 2024-09-12T23:44:41.598 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 24 current snap is 175 2024-09-12T23:44:41.598 INFO:tasks.rados.rados.0.smithi136.stdout:1842: seq_num 608 ranges {2318336=589824} 2024-09-12T23:44:41.614 INFO:tasks.rados.rados.0.smithi136.stdout:1842: writing smithi13667081-24 from 2318336 to 2908160 tid 1 2024-09-12T23:44:41.614 INFO:tasks.rados.rados.0.smithi136.stdout:1843: copy_from oid 10 from oid 44 current snap is 175 2024-09-12T23:44:41.614 INFO:tasks.rados.rados.0.smithi136.stdout:1841: done (3 left) 2024-09-12T23:44:41.614 INFO:tasks.rados.rados.0.smithi136.stdout:1844: rollback oid 31 current snap is 175 2024-09-12T23:44:41.614 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 31 to 174 2024-09-12T23:44:41.614 INFO:tasks.rados.rados.0.smithi136.stdout:1845: rollback oid 35 current snap is 175 2024-09-12T23:44:41.614 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 35 to 174 2024-09-12T23:44:41.615 INFO:tasks.rados.rados.0.smithi136.stdout:1846: read oid 30 snap -1 2024-09-12T23:44:41.615 INFO:tasks.rados.rados.0.smithi136.stdout:1846: expect deleted 2024-09-12T23:44:41.615 INFO:tasks.rados.rados.0.smithi136.stdout:1847: delete oid 36 current snap is 175 2024-09-12T23:44:41.624 INFO:tasks.rados.rados.0.smithi136.stdout:1845: finishing rollback tid 0 to smithi13667081-35 2024-09-12T23:44:41.625 INFO:tasks.rados.rados.0.smithi136.stdout:1845: finishing rollback tid 1 to smithi13667081-35 2024-09-12T23:44:41.626 INFO:tasks.rados.rados.0.smithi136.stdout:1845: finishing rollback tid 2 to smithi13667081-35 2024-09-12T23:44:41.626 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 35 v 1301 (ObjNum 469 snap 139 seq_num 469) dirty exists 2024-09-12T23:44:41.627 INFO:tasks.rados.rados.0.smithi136.stdout:1844: finishing rollback tid 0 to smithi13667081-31 2024-09-12T23:44:41.628 INFO:tasks.rados.rados.0.smithi136.stdout:1845: done (6 left) 2024-09-12T23:44:41.628 INFO:tasks.rados.rados.0.smithi136.stdout:1846: done (5 left) 2024-09-12T23:44:41.628 INFO:tasks.rados.rados.0.smithi136.stdout:1847: done (4 left) 2024-09-12T23:44:41.628 INFO:tasks.rados.rados.0.smithi136.stdout:1848: rollback oid 28 current snap is 175 2024-09-12T23:44:41.628 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 28 to 174 2024-09-12T23:44:41.628 INFO:tasks.rados.rados.0.smithi136.stdout:1849: rollback oid 14 current snap is 175 2024-09-12T23:44:41.628 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 14 to 174 2024-09-12T23:44:41.628 INFO:tasks.rados.rados.0.smithi136.stdout:1850: rmattr oid 29 current snap is 175 2024-09-12T23:44:41.629 INFO:tasks.rados.rados.0.smithi136.stdout:1851: rollback oid 20 current snap is 175 2024-09-12T23:44:41.629 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 20 to 174 2024-09-12T23:44:41.629 INFO:tasks.rados.rados.0.smithi136.stdout:1852: setattr oid 30 current snap is 175 2024-09-12T23:44:41.631 INFO:tasks.rados.rados.0.smithi136.stdout:1842: finishing write tid 1 to smithi13667081-24 2024-09-12T23:44:41.631 INFO:tasks.rados.rados.0.smithi136.stdout:1853: read oid 4 snap -1 2024-09-12T23:44:41.631 INFO:tasks.rados.rados.0.smithi136.stdout:1853: expect (ObjNum 567 snap 162 seq_num 567) 2024-09-12T23:44:41.631 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 36 current snap is 175 2024-09-12T23:44:41.631 INFO:tasks.rados.rados.0.smithi136.stdout:1854: seq_num 610 ranges {0=622592} 2024-09-12T23:44:41.637 INFO:tasks.rados.rados.0.smithi136.stdout:1854: writing smithi13667081-36 from 0 to 622592 tid 1 2024-09-12T23:44:41.637 INFO:tasks.rados.rados.0.smithi136.stdout:1855: copy_from oid 15 from oid 11 current snap is 175 2024-09-12T23:44:41.637 INFO:tasks.rados.rados.0.smithi136.stdout:1856: rollback oid 42 current snap is 175 2024-09-12T23:44:41.637 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 42 to 174 2024-09-12T23:44:41.638 INFO:tasks.rados.rados.0.smithi136.stdout:1857: copy_from oid 27 from oid 2 current snap is 175 2024-09-12T23:44:41.638 INFO:tasks.rados.rados.0.smithi136.stdout:1858: read oid 39 snap 174 2024-09-12T23:44:41.638 INFO:tasks.rados.rados.0.smithi136.stdout:1858: expect (ObjNum 590 snap 172 seq_num 590) 2024-09-12T23:44:41.638 INFO:tasks.rados.rados.0.smithi136.stdout:1842: finishing write tid 2 to smithi13667081-24 2024-09-12T23:44:41.638 INFO:tasks.rados.rados.0.smithi136.stdout:1859: read oid 40 snap -1 2024-09-12T23:44:41.638 INFO:tasks.rados.rados.0.smithi136.stdout:1859: expect deleted 2024-09-12T23:44:41.638 INFO:tasks.rados.rados.0.smithi136.stdout: waiting on 16 2024-09-12T23:44:41.638 INFO:tasks.rados.rados.0.smithi136.stdout:1844: finishing rollback tid 1 to smithi13667081-31 2024-09-12T23:44:41.641 INFO:tasks.rados.rados.0.smithi136.stdout:1844: finishing rollback tid 2 to smithi13667081-31 2024-09-12T23:44:41.641 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 31 v 927 (ObjNum 572 snap 166 seq_num 572) dirty exists 2024-09-12T23:44:41.641 INFO:tasks.rados.rados.0.smithi136.stdout:1844: done (15 left) 2024-09-12T23:44:41.641 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 35 current snap is 175 2024-09-12T23:44:41.641 INFO:tasks.rados.rados.0.smithi136.stdout:1860: seq_num 611 ranges {2670592=638976} 2024-09-12T23:44:41.666 INFO:tasks.rados.rados.0.smithi136.stdout:1860: writing smithi13667081-35 from 2670592 to 3309568 tid 1 2024-09-12T23:44:41.666 INFO:tasks.rados.rados.0.smithi136.stdout: waiting on 16 2024-09-12T23:44:41.666 INFO:tasks.rados.rados.0.smithi136.stdout:1851: finishing rollback tid 0 to smithi13667081-20 2024-09-12T23:44:41.667 INFO:tasks.rados.rados.0.smithi136.stdout:1851: finishing rollback tid 1 to smithi13667081-20 2024-09-12T23:44:41.667 INFO:tasks.rados.rados.0.smithi136.stdout:1851: finishing rollback tid 2 to smithi13667081-20 2024-09-12T23:44:41.667 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 20 v 1211 (ObjNum 469 snap 139 seq_num 469) dirty exists 2024-09-12T23:44:41.667 INFO:tasks.rados.rados.0.smithi136.stdout:1851: done (15 left) 2024-09-12T23:44:41.667 INFO:tasks.rados.rados.0.smithi136.stdout:1859: done (14 left) 2024-09-12T23:44:41.667 INFO:tasks.rados.rados.0.smithi136.stdout:1861: rollback oid 50 current snap is 175 2024-09-12T23:44:41.667 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 50 to 174 2024-09-12T23:44:41.667 INFO:tasks.rados.rados.0.smithi136.stdout:1856: finishing rollback tid 0 to smithi13667081-42 2024-09-12T23:44:41.667 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 30 v 804 (ObjNum 33 snap 0 seq_num 4242838944) dirty exists 2024-09-12T23:44:41.667 INFO:tasks.rados.rados.0.smithi136.stdout:1856: finishing rollback tid 1 to smithi13667081-42 2024-09-12T23:44:41.667 INFO:tasks.rados.rados.0.smithi136.stdout:1856: finishing rollback tid 2 to smithi13667081-42 2024-09-12T23:44:41.667 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 42 v 827 (ObjNum 582 snap 169 seq_num 582) dirty exists 2024-09-12T23:44:41.667 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 29 v 822 (ObjNum 579 snap 169 seq_num 579) dirty exists 2024-09-12T23:44:41.668 INFO:tasks.rados.rados.0.smithi136.stdout:1854: finishing write tid 1 to smithi13667081-36 2024-09-12T23:44:41.668 INFO:tasks.rados.rados.0.smithi136.stdout:1854: finishing write tid 2 to smithi13667081-36 2024-09-12T23:44:41.668 INFO:tasks.rados.rados.0.smithi136.stdout:1848: finishing rollback tid 0 to smithi13667081-28 2024-09-12T23:44:41.668 INFO:tasks.rados.rados.0.smithi136.stdout:1848: finishing rollback tid 1 to smithi13667081-28 2024-09-12T23:44:41.668 INFO:tasks.rados.rados.0.smithi136.stdout:1842: finishing write tid 3 to smithi13667081-24 2024-09-12T23:44:41.668 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 24 v 630 (ObjNum 608 snap 175 seq_num 608) dirty exists 2024-09-12T23:44:41.668 INFO:tasks.rados.rados.0.smithi136.stdout:1842: left oid 24 (ObjNum 608 snap 175 seq_num 608) 2024-09-12T23:44:41.668 INFO:tasks.rados.rados.0.smithi136.stdout:1848: finishing rollback tid 2 to smithi13667081-28 2024-09-12T23:44:41.668 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 28 v 1100 (ObjNum 606 snap 174 seq_num 606) dirty exists 2024-09-12T23:44:41.668 INFO:tasks.rados.rados.0.smithi136.stdout:1854: finishing write tid 3 to smithi13667081-36 2024-09-12T23:44:41.668 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 36 v 788 (ObjNum 610 snap 175 seq_num 610) dirty exists 2024-09-12T23:44:41.668 INFO:tasks.rados.rados.0.smithi136.stdout:1854: left oid 36 (ObjNum 610 snap 175 seq_num 610) 2024-09-12T23:44:41.668 INFO:tasks.rados.rados.0.smithi136.stdout:1855: finishing copy_from racing read to smithi13667081-15 2024-09-12T23:44:41.668 INFO:tasks.rados.rados.0.smithi136.stdout:1840: expect (ObjNum 588 snap 172 seq_num 588) 2024-09-12T23:44:41.799 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:41 smithi136 bash[76371]: cluster 2024-09-12T23:44:40.530793+0000 mon.a (mon.0) 439 : cluster [DBG] osdmap e426: 8 total, 7 up, 8 in 2024-09-12T23:44:41.799 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:41 smithi136 bash[76371]: cluster 2024-09-12T23:44:40.702936+0000 mgr.y (mgr.24661) 146 : cluster [DBG] pgmap v214: 121 pgs: 34 active+undersized, 12 active+undersized+degraded, 75 active+clean; 182 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.7 MiB/s rd, 1.5 MiB/s wr, 10 op/s; 53/285 objects degraded (18.596%) 2024-09-12T23:44:41.799 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:41 smithi136 bash[75164]: cluster 2024-09-12T23:44:40.530793+0000 mon.a (mon.0) 439 : cluster [DBG] osdmap e426: 8 total, 7 up, 8 in 2024-09-12T23:44:41.799 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:41 smithi136 bash[75164]: cluster 2024-09-12T23:44:40.702936+0000 mgr.y (mgr.24661) 146 : cluster [DBG] pgmap v214: 121 pgs: 34 active+undersized, 12 active+undersized+degraded, 75 active+clean; 182 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.7 MiB/s rd, 1.5 MiB/s wr, 10 op/s; 53/285 objects degraded (18.596%) 2024-09-12T23:44:41.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:41 smithi181 bash[67587]: cluster 2024-09-12T23:44:40.530793+0000 mon.a (mon.0) 439 : cluster [DBG] osdmap e426: 8 total, 7 up, 8 in 2024-09-12T23:44:41.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:41 smithi181 bash[67587]: cluster 2024-09-12T23:44:40.702936+0000 mgr.y (mgr.24661) 146 : cluster [DBG] pgmap v214: 121 pgs: 34 active+undersized, 12 active+undersized+degraded, 75 active+clean; 182 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.7 MiB/s rd, 1.5 MiB/s wr, 10 op/s; 53/285 objects degraded (18.596%) 2024-09-12T23:44:42.123 INFO:tasks.rados.rados.0.smithi136.stdout:1858: expect (ObjNum 590 snap 172 seq_num 590) 2024-09-12T23:44:42.130 INFO:tasks.rados.rados.0.smithi136.stdout:1855: finishing copy_from to smithi13667081-15 2024-09-12T23:44:42.131 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 15 v 1198 (ObjNum 482 snap 142 seq_num 482) dirty exists 2024-09-12T23:44:42.131 INFO:tasks.rados.rados.0.smithi136.stdout:1849: finishing rollback tid 0 to smithi13667081-14 2024-09-12T23:44:42.131 INFO:tasks.rados.rados.0.smithi136.stdout:1861: finishing rollback tid 1 to smithi13667081-50 2024-09-12T23:44:42.131 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 50 v 0 (ObjNum 177 snap 0 seq_num 4242575600) dirty dne 2024-09-12T23:44:42.131 INFO:tasks.rados.rados.0.smithi136.stdout:1849: finishing rollback tid 1 to smithi13667081-14 2024-09-12T23:44:42.131 INFO:tasks.rados.rados.0.smithi136.stdout:1843: finishing copy_from racing read to smithi13667081-10 2024-09-12T23:44:42.131 INFO:tasks.rados.rados.0.smithi136.stdout:1849: finishing rollback tid 2 to smithi13667081-14 2024-09-12T23:44:42.131 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 14 v 1104 (ObjNum 583 snap 169 seq_num 583) dirty exists 2024-09-12T23:44:42.131 INFO:tasks.rados.rados.0.smithi136.stdout:1843: finishing copy_from to smithi13667081-10 2024-09-12T23:44:42.131 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 10 v 1078 (ObjNum 564 snap 162 seq_num 564) dirty exists 2024-09-12T23:44:42.131 INFO:tasks.rados.rados.0.smithi136.stdout:1857: finishing copy_from racing read to smithi13667081-27 2024-09-12T23:44:42.131 INFO:tasks.rados.rados.0.smithi136.stdout:1840: done (14 left) 2024-09-12T23:44:42.131 INFO:tasks.rados.rados.0.smithi136.stdout:1842: done (13 left) 2024-09-12T23:44:42.132 INFO:tasks.rados.rados.0.smithi136.stdout:1843: done (12 left) 2024-09-12T23:44:42.132 INFO:tasks.rados.rados.0.smithi136.stdout:1848: done (11 left) 2024-09-12T23:44:42.132 INFO:tasks.rados.rados.0.smithi136.stdout:1849: done (10 left) 2024-09-12T23:44:42.132 INFO:tasks.rados.rados.0.smithi136.stdout:1850: done (9 left) 2024-09-12T23:44:42.132 INFO:tasks.rados.rados.0.smithi136.stdout:1852: done (8 left) 2024-09-12T23:44:42.132 INFO:tasks.rados.rados.0.smithi136.stdout:1854: done (7 left) 2024-09-12T23:44:42.132 INFO:tasks.rados.rados.0.smithi136.stdout:1855: done (6 left) 2024-09-12T23:44:42.132 INFO:tasks.rados.rados.0.smithi136.stdout:1856: done (5 left) 2024-09-12T23:44:42.132 INFO:tasks.rados.rados.0.smithi136.stdout:1858: done (4 left) 2024-09-12T23:44:42.132 INFO:tasks.rados.rados.0.smithi136.stdout:1861: done (3 left) 2024-09-12T23:44:42.132 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 10 current snap is 175 2024-09-12T23:44:42.132 INFO:tasks.rados.rados.0.smithi136.stdout:1860: finishing write tid 1 to smithi13667081-35 2024-09-12T23:44:42.132 INFO:tasks.rados.rados.0.smithi136.stdout:1860: finishing write tid 2 to smithi13667081-35 2024-09-12T23:44:42.132 INFO:tasks.rados.rados.0.smithi136.stdout:1853: expect (ObjNum 567 snap 162 seq_num 567) 2024-09-12T23:44:42.255 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 12 23:44:41 smithi136 bash[84852]: debug 2024-09-12T23:44:41.865+0000 7f8a23bc1640 1 -- 172.21.15.136:0/1922525837 <== mon.2 v2:172.21.15.136:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55f4793211e0 con 0x55f4792e1800 2024-09-12T23:44:42.445 INFO:tasks.rados.rados.0.smithi136.stdout:1860: finishing write tid 3 to smithi13667081-35 2024-09-12T23:44:42.446 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 35 v 1303 (ObjNum 611 snap 175 seq_num 611) dirty exists 2024-09-12T23:44:42.446 INFO:tasks.rados.rados.0.smithi136.stdout:1860: left oid 35 (ObjNum 611 snap 175 seq_num 611) 2024-09-12T23:44:42.446 INFO:tasks.rados.rados.0.smithi136.stdout:1857: finishing copy_from to smithi13667081-27 2024-09-12T23:44:42.446 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 27 v 402 (ObjNum 416 snap 120 seq_num 416) dirty exists 2024-09-12T23:44:42.446 INFO:tasks.rados.rados.0.smithi136.stdout:1862: seq_num 612 ranges {2260992=655360} 2024-09-12T23:44:42.462 INFO:tasks.rados.rados.0.smithi136.stdout:1862: writing smithi13667081-10 from 2260992 to 2916352 tid 1 2024-09-12T23:44:42.462 INFO:tasks.rados.rados.0.smithi136.stdout:1853: done (3 left) 2024-09-12T23:44:42.462 INFO:tasks.rados.rados.0.smithi136.stdout:1857: done (2 left) 2024-09-12T23:44:42.462 INFO:tasks.rados.rados.0.smithi136.stdout:1860: done (1 left) 2024-09-12T23:44:42.462 INFO:tasks.rados.rados.0.smithi136.stdout:1863: snap_create 2024-09-12T23:44:42.468 INFO:tasks.rados.rados.0.smithi136.stdout:1862: finishing write tid 1 to smithi13667081-10 2024-09-12T23:44:42.469 INFO:tasks.rados.rados.0.smithi136.stdout:1862: finishing write tid 2 to smithi13667081-10 2024-09-12T23:44:42.472 INFO:tasks.rados.rados.0.smithi136.stdout:1862: finishing write tid 3 to smithi13667081-10 2024-09-12T23:44:42.472 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 10 v 1080 (ObjNum 612 snap 175 seq_num 612) dirty exists 2024-09-12T23:44:42.472 INFO:tasks.rados.rados.0.smithi136.stdout:1862: left oid 10 (ObjNum 612 snap 175 seq_num 612) 2024-09-12T23:44:42.537 INFO:tasks.rados.rados.0.smithi136.stdout:1862: done (1 left) 2024-09-12T23:44:42.538 INFO:tasks.rados.rados.0.smithi136.stdout:1863: done (0 left) 2024-09-12T23:44:42.538 INFO:tasks.rados.rados.0.smithi136.stdout:1864: read oid 12 snap -1 2024-09-12T23:44:42.538 INFO:tasks.rados.rados.0.smithi136.stdout:1864: expect (ObjNum 512 snap 149 seq_num 512) 2024-09-12T23:44:42.538 INFO:tasks.rados.rados.0.smithi136.stdout:1865: read oid 44 snap -1 2024-09-12T23:44:42.538 INFO:tasks.rados.rados.0.smithi136.stdout:1865: expect (ObjNum 564 snap 162 seq_num 564) 2024-09-12T23:44:42.538 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 14 current snap is 176 2024-09-12T23:44:42.538 INFO:tasks.rados.rados.0.smithi136.stdout:1866: seq_num 613 ranges {6922240=671744} 2024-09-12T23:44:42.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:42 smithi181 bash[67587]: cluster 2024-09-12T23:44:41.524459+0000 mon.a (mon.0) 440 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive, 12 pgs peering) 2024-09-12T23:44:42.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:42 smithi181 bash[67587]: cluster 2024-09-12T23:44:41.540317+0000 mon.a (mon.0) 441 : cluster [DBG] osdmap e427: 8 total, 7 up, 8 in 2024-09-12T23:44:42.595 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:44:42 smithi181 bash[60471]: ts=2024-09-12T23:44:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:44:42.621 INFO:tasks.rados.rados.0.smithi136.stdout:1866: writing smithi13667081-14 from 6922240 to 7593984 tid 1 2024-09-12T23:44:42.621 INFO:tasks.rados.rados.0.smithi136.stdout:1867: rollback oid 2 current snap is 176 2024-09-12T23:44:42.621 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 2 to 175 2024-09-12T23:44:42.621 INFO:tasks.rados.rados.0.smithi136.stdout:1864: expect (ObjNum 512 snap 149 seq_num 512) 2024-09-12T23:44:42.756 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 12 23:44:42 smithi136 bash[84852]: debug 2024-09-12T23:44:42.405+0000 7f8a26427640 -1 Falling back to public interface 2024-09-12T23:44:42.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:42 smithi136 bash[75164]: cluster 2024-09-12T23:44:41.524459+0000 mon.a (mon.0) 440 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive, 12 pgs peering) 2024-09-12T23:44:42.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:42 smithi136 bash[75164]: cluster 2024-09-12T23:44:41.540317+0000 mon.a (mon.0) 441 : cluster [DBG] osdmap e427: 8 total, 7 up, 8 in 2024-09-12T23:44:42.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:42 smithi136 bash[76371]: cluster 2024-09-12T23:44:41.524459+0000 mon.a (mon.0) 440 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive, 12 pgs peering) 2024-09-12T23:44:42.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:42 smithi136 bash[76371]: cluster 2024-09-12T23:44:41.540317+0000 mon.a (mon.0) 441 : cluster [DBG] osdmap e427: 8 total, 7 up, 8 in 2024-09-12T23:44:42.808 INFO:tasks.rados.rados.0.smithi136.stdout:1865: expect (ObjNum 564 snap 162 seq_num 564) 2024-09-12T23:44:42.865 INFO:tasks.rados.rados.0.smithi136.stdout:1867: finishing rollback tid 0 to smithi13667081-2 2024-09-12T23:44:42.866 INFO:tasks.rados.rados.0.smithi136.stdout:1866: finishing write tid 1 to smithi13667081-14 2024-09-12T23:44:42.866 INFO:tasks.rados.rados.0.smithi136.stdout:1866: finishing write tid 2 to smithi13667081-14 2024-09-12T23:44:42.866 INFO:tasks.rados.rados.0.smithi136.stdout:1867: finishing rollback tid 1 to smithi13667081-2 2024-09-12T23:44:42.866 INFO:tasks.rados.rados.0.smithi136.stdout:1867: finishing rollback tid 2 to smithi13667081-2 2024-09-12T23:44:42.866 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 2 v 1211 (ObjNum 416 snap 120 seq_num 416) dirty exists 2024-09-12T23:44:42.866 INFO:tasks.rados.rados.0.smithi136.stdout:1866: finishing write tid 3 to smithi13667081-14 2024-09-12T23:44:42.866 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 14 v 1107 (ObjNum 613 snap 176 seq_num 613) dirty exists 2024-09-12T23:44:42.866 INFO:tasks.rados.rados.0.smithi136.stdout:1866: left oid 14 (ObjNum 613 snap 176 seq_num 613) 2024-09-12T23:44:42.866 INFO:tasks.rados.rados.0.smithi136.stdout:1864: done (3 left) 2024-09-12T23:44:42.866 INFO:tasks.rados.rados.0.smithi136.stdout:1865: done (2 left) 2024-09-12T23:44:42.866 INFO:tasks.rados.rados.0.smithi136.stdout:1866: done (1 left) 2024-09-12T23:44:42.866 INFO:tasks.rados.rados.0.smithi136.stdout:1867: done (0 left) 2024-09-12T23:44:42.867 INFO:tasks.rados.rados.0.smithi136.stdout:1868: snap_create 2024-09-12T23:44:43.560 INFO:tasks.rados.rados.0.smithi136.stdout:1868: done (0 left) 2024-09-12T23:44:43.561 INFO:tasks.rados.rados.0.smithi136.stdout:1869: rmattr oid 18 current snap is 177 2024-09-12T23:44:43.561 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 14 current snap is 177 2024-09-12T23:44:43.561 INFO:tasks.rados.rados.0.smithi136.stdout:1870: seq_num 614 ranges {7593984=688128} 2024-09-12T23:44:43.624 INFO:tasks.rados.rados.0.smithi136.stdout:1870: writing smithi13667081-14 from 7593984 to 8282112 tid 1 2024-09-12T23:44:43.625 INFO:tasks.rados.rados.0.smithi136.stdout:1871: snap_create 2024-09-12T23:44:43.625 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 18 v 919 (ObjNum 578 snap 169 seq_num 578) dirty exists 2024-09-12T23:44:43.639 INFO:tasks.rados.rados.0.smithi136.stdout:1870: finishing write tid 1 to smithi13667081-14 2024-09-12T23:44:43.640 INFO:tasks.rados.rados.0.smithi136.stdout:1870: finishing write tid 2 to smithi13667081-14 2024-09-12T23:44:43.642 INFO:tasks.rados.rados.0.smithi136.stdout:1870: finishing write tid 3 to smithi13667081-14 2024-09-12T23:44:43.642 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 14 v 1110 (ObjNum 614 snap 177 seq_num 614) dirty exists 2024-09-12T23:44:43.642 INFO:tasks.rados.rados.0.smithi136.stdout:1870: left oid 14 (ObjNum 614 snap 177 seq_num 614) 2024-09-12T23:44:43.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:43 smithi181 bash[67587]: cluster 2024-09-12T23:44:42.540756+0000 mon.a (mon.0) 442 : cluster [DBG] osdmap e428: 8 total, 7 up, 8 in 2024-09-12T23:44:43.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:43 smithi181 bash[67587]: cluster 2024-09-12T23:44:42.703799+0000 mgr.y (mgr.24661) 147 : cluster [DBG] pgmap v217: 121 pgs: 34 active+undersized, 12 active+undersized+degraded, 75 active+clean; 182 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.7 MiB/s rd, 1.3 MiB/s wr, 8 op/s; 53/285 objects degraded (18.596%) 2024-09-12T23:44:43.845 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:44:43 smithi181 bash[60471]: ts=2024-09-12T23:44:43.510Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi136\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:44:44.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:43 smithi136 bash[76371]: cluster 2024-09-12T23:44:42.540756+0000 mon.a (mon.0) 442 : cluster [DBG] osdmap e428: 8 total, 7 up, 8 in 2024-09-12T23:44:44.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:43 smithi136 bash[76371]: cluster 2024-09-12T23:44:42.703799+0000 mgr.y (mgr.24661) 147 : cluster [DBG] pgmap v217: 121 pgs: 34 active+undersized, 12 active+undersized+degraded, 75 active+clean; 182 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.7 MiB/s rd, 1.3 MiB/s wr, 8 op/s; 53/285 objects degraded (18.596%) 2024-09-12T23:44:44.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:43 smithi136 bash[75164]: cluster 2024-09-12T23:44:42.540756+0000 mon.a (mon.0) 442 : cluster [DBG] osdmap e428: 8 total, 7 up, 8 in 2024-09-12T23:44:44.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:43 smithi136 bash[75164]: cluster 2024-09-12T23:44:42.703799+0000 mgr.y (mgr.24661) 147 : cluster [DBG] pgmap v217: 121 pgs: 34 active+undersized, 12 active+undersized+degraded, 75 active+clean; 182 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.7 MiB/s rd, 1.3 MiB/s wr, 8 op/s; 53/285 objects degraded (18.596%) 2024-09-12T23:44:44.556 INFO:tasks.rados.rados.0.smithi136.stdout:1869: done (2 left) 2024-09-12T23:44:44.556 INFO:tasks.rados.rados.0.smithi136.stdout:1870: done (1 left) 2024-09-12T23:44:44.557 INFO:tasks.rados.rados.0.smithi136.stdout:1871: done (0 left) 2024-09-12T23:44:44.557 INFO:tasks.rados.rados.0.smithi136.stdout:1872: copy_from oid 34 from oid 41 current snap is 178 2024-09-12T23:44:44.557 INFO:tasks.rados.rados.0.smithi136.stdout:1873: rollback oid 2 current snap is 178 2024-09-12T23:44:44.557 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 2 to 176 2024-09-12T23:44:44.557 INFO:tasks.rados.rados.0.smithi136.stdout:1874: rollback oid 26 current snap is 178 2024-09-12T23:44:44.557 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 26 to 177 2024-09-12T23:44:44.557 INFO:tasks.rados.rados.0.smithi136.stdout:1875: read oid 9 snap -1 2024-09-12T23:44:44.557 INFO:tasks.rados.rados.0.smithi136.stdout:1875: expect (ObjNum 513 snap 150 seq_num 513) 2024-09-12T23:44:44.557 INFO:tasks.rados.rados.0.smithi136.stdout:1876: copy_from oid 5 from oid 4 current snap is 178 2024-09-12T23:44:44.557 INFO:tasks.rados.rados.0.smithi136.stdout:1877: snap_create 2024-09-12T23:44:44.564 INFO:tasks.rados.rados.0.smithi136.stdout:1872: finishing copy_from to smithi13667081-34 2024-09-12T23:44:44.564 INFO:tasks.rados.rados.0.smithi136.stdout:1872: got expected ENOENT (src dne) 2024-09-12T23:44:44.564 INFO:tasks.rados.rados.0.smithi136.stdout:1872: finishing copy_from racing read to smithi13667081-34 2024-09-12T23:44:44.564 INFO:tasks.rados.rados.0.smithi136.stdout:1874: finishing rollback tid 0 to smithi13667081-26 2024-09-12T23:44:44.565 INFO:tasks.rados.rados.0.smithi136.stdout:1873: finishing rollback tid 0 to smithi13667081-2 2024-09-12T23:44:44.565 INFO:tasks.rados.rados.0.smithi136.stdout:1873: finishing rollback tid 1 to smithi13667081-2 2024-09-12T23:44:44.565 INFO:tasks.rados.rados.0.smithi136.stdout:1873: finishing rollback tid 2 to smithi13667081-2 2024-09-12T23:44:44.565 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 2 v 1214 (ObjNum 416 snap 120 seq_num 416) dirty exists 2024-09-12T23:44:44.565 INFO:tasks.rados.rados.0.smithi136.stdout:1874: finishing rollback tid 1 to smithi13667081-26 2024-09-12T23:44:44.566 INFO:tasks.rados.rados.0.smithi136.stdout:1874: finishing rollback tid 2 to smithi13667081-26 2024-09-12T23:44:44.566 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 26 v 633 (ObjNum 539 snap 157 seq_num 539) dirty exists 2024-09-12T23:44:44.607 INFO:tasks.rados.rados.0.smithi136.stdout:1876: finishing copy_from racing read to smithi13667081-5 2024-09-12T23:44:44.622 INFO:tasks.rados.rados.0.smithi136.stdout:1876: finishing copy_from to smithi13667081-5 2024-09-12T23:44:44.622 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 5 v 770 (ObjNum 567 snap 162 seq_num 567) dirty exists 2024-09-12T23:44:44.625 INFO:tasks.rados.rados.0.smithi136.stdout:1875: expect (ObjNum 513 snap 150 seq_num 513) 2024-09-12T23:44:44.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:44 smithi181 bash[67587]: cluster 2024-09-12T23:44:43.563072+0000 mon.a (mon.0) 443 : cluster [DBG] osdmap e429: 8 total, 7 up, 8 in 2024-09-12T23:44:44.844 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:44:44 smithi181 bash[60471]: ts=2024-09-12T23:44:44.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:44:44.845 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:44:44 smithi181 bash[60471]: ts=2024-09-12T23:44:44.456Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:44:45.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:44 smithi136 bash[75164]: cluster 2024-09-12T23:44:43.563072+0000 mon.a (mon.0) 443 : cluster [DBG] osdmap e429: 8 total, 7 up, 8 in 2024-09-12T23:44:45.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:44 smithi136 bash[76371]: cluster 2024-09-12T23:44:43.563072+0000 mon.a (mon.0) 443 : cluster [DBG] osdmap e429: 8 total, 7 up, 8 in 2024-09-12T23:44:45.561 INFO:tasks.rados.rados.0.smithi136.stdout:1872: done (5 left) 2024-09-12T23:44:45.561 INFO:tasks.rados.rados.0.smithi136.stdout:1873: done (4 left) 2024-09-12T23:44:45.561 INFO:tasks.rados.rados.0.smithi136.stdout:1874: done (3 left) 2024-09-12T23:44:45.561 INFO:tasks.rados.rados.0.smithi136.stdout:1875: done (2 left) 2024-09-12T23:44:45.561 INFO:tasks.rados.rados.0.smithi136.stdout:1876: done (1 left) 2024-09-12T23:44:45.561 INFO:tasks.rados.rados.0.smithi136.stdout:1877: done (0 left) 2024-09-12T23:44:45.562 INFO:tasks.rados.rados.0.smithi136.stdout:1878: copy_from oid 23 from oid 48 current snap is 179 2024-09-12T23:44:45.562 INFO:tasks.rados.rados.0.smithi136.stdout:1879: delete oid 40 current snap is 179 2024-09-12T23:44:45.572 INFO:tasks.rados.rados.0.smithi136.stdout:1879: done (1 left) 2024-09-12T23:44:45.572 INFO:tasks.rados.rados.0.smithi136.stdout:1880: snap_remove snap 176 2024-09-12T23:44:45.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:45 smithi181 bash[67587]: cluster 2024-09-12T23:44:44.560101+0000 mon.a (mon.0) 444 : cluster [DBG] osdmap e430: 8 total, 7 up, 8 in 2024-09-12T23:44:45.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:45 smithi181 bash[67587]: cluster 2024-09-12T23:44:44.705363+0000 mgr.y (mgr.24661) 148 : cluster [DBG] pgmap v220: 121 pgs: 34 active+undersized, 12 active+undersized+degraded, 75 active+clean; 198 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 4.0 MiB/s wr, 31 op/s; 62/321 objects degraded (19.315%) 2024-09-12T23:44:45.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:45 smithi181 bash[67587]: audit 2024-09-12T23:44:44.972128+0000 mon.a (mon.0) 445 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:45.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:45 smithi181 bash[67587]: audit 2024-09-12T23:44:44.978475+0000 mon.a (mon.0) 446 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:45.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:45 smithi181 bash[67587]: cluster 2024-09-12T23:44:45.563807+0000 mon.a (mon.0) 447 : cluster [DBG] osdmap e431: 8 total, 7 up, 8 in 2024-09-12T23:44:45.970 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:45 smithi136 bash[75164]: cluster 2024-09-12T23:44:44.560101+0000 mon.a (mon.0) 444 : cluster [DBG] osdmap e430: 8 total, 7 up, 8 in 2024-09-12T23:44:45.970 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:45 smithi136 bash[75164]: cluster 2024-09-12T23:44:44.705363+0000 mgr.y (mgr.24661) 148 : cluster [DBG] pgmap v220: 121 pgs: 34 active+undersized, 12 active+undersized+degraded, 75 active+clean; 198 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 4.0 MiB/s wr, 31 op/s; 62/321 objects degraded (19.315%) 2024-09-12T23:44:45.970 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:45 smithi136 bash[75164]: audit 2024-09-12T23:44:44.972128+0000 mon.a (mon.0) 445 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:45.970 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:45 smithi136 bash[75164]: audit 2024-09-12T23:44:44.978475+0000 mon.a (mon.0) 446 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:45.970 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:45 smithi136 bash[75164]: cluster 2024-09-12T23:44:45.563807+0000 mon.a (mon.0) 447 : cluster [DBG] osdmap e431: 8 total, 7 up, 8 in 2024-09-12T23:44:45.970 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:45 smithi136 bash[76371]: cluster 2024-09-12T23:44:44.560101+0000 mon.a (mon.0) 444 : cluster [DBG] osdmap e430: 8 total, 7 up, 8 in 2024-09-12T23:44:45.970 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:45 smithi136 bash[76371]: cluster 2024-09-12T23:44:44.705363+0000 mgr.y (mgr.24661) 148 : cluster [DBG] pgmap v220: 121 pgs: 34 active+undersized, 12 active+undersized+degraded, 75 active+clean; 198 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 4.0 MiB/s wr, 31 op/s; 62/321 objects degraded (19.315%) 2024-09-12T23:44:45.970 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:45 smithi136 bash[76371]: audit 2024-09-12T23:44:44.972128+0000 mon.a (mon.0) 445 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:45.970 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:45 smithi136 bash[76371]: audit 2024-09-12T23:44:44.978475+0000 mon.a (mon.0) 446 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:45.971 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:45 smithi136 bash[76371]: cluster 2024-09-12T23:44:45.563807+0000 mon.a (mon.0) 447 : cluster [DBG] osdmap e431: 8 total, 7 up, 8 in 2024-09-12T23:44:46.256 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 12 23:44:45 smithi136 bash[84852]: debug 2024-09-12T23:44:45.969+0000 7f8a26427640 -1 osd.1 419 log_to_monitors true 2024-09-12T23:44:46.601 INFO:tasks.rados.rados.0.smithi136.stdout:1880: done (1 left) 2024-09-12T23:44:46.601 INFO:tasks.rados.rados.0.smithi136.stdout:1881: setattr oid 37 current snap is 179 2024-09-12T23:44:46.606 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 43 current snap is 179 2024-09-12T23:44:46.606 INFO:tasks.rados.rados.0.smithi136.stdout:1882: seq_num 616 ranges {9535488=720896} 2024-09-12T23:44:46.663 INFO:tasks.rados.rados.0.smithi136.stdout:1882: writing smithi13667081-43 from 9535488 to 10256384 tid 1 2024-09-12T23:44:46.663 INFO:tasks.rados.rados.0.smithi136.stdout:1883: delete oid 33 current snap is 179 2024-09-12T23:44:46.663 INFO:tasks.rados.rados.0.smithi136.stdout:1878: finishing copy_from racing read to smithi13667081-23 2024-09-12T23:44:46.663 INFO:tasks.rados.rados.0.smithi136.stdout:1878: finishing copy_from to smithi13667081-23 2024-09-12T23:44:46.664 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 23 v 1038 (ObjNum 544 snap 158 seq_num 544) dirty exists 2024-09-12T23:44:46.664 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 37 v 1215 (ObjNum 589 snap 172 seq_num 589) dirty exists 2024-09-12T23:44:46.666 INFO:tasks.rados.rados.0.smithi136.stdout:1878: done (3 left) 2024-09-12T23:44:46.666 INFO:tasks.rados.rados.0.smithi136.stdout:1881: done (2 left) 2024-09-12T23:44:46.666 INFO:tasks.rados.rados.0.smithi136.stdout:1883: done (1 left) 2024-09-12T23:44:46.667 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 26 current snap is 179 2024-09-12T23:44:46.667 INFO:tasks.rados.rados.0.smithi136.stdout:1884: seq_num 618 ranges {1835008=753664} 2024-09-12T23:44:46.680 INFO:tasks.rados.rados.0.smithi136.stdout:1884: writing smithi13667081-26 from 1835008 to 2588672 tid 1 2024-09-12T23:44:46.681 INFO:tasks.rados.rados.0.smithi136.stdout:1885: snap_remove snap 174 2024-09-12T23:44:46.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:46 smithi181 bash[67587]: cluster 2024-09-12T23:44:45.564495+0000 mon.a (mon.0) 448 : cluster [WRN] Health check update: Degraded data redundancy: 62/321 objects degraded (19.315%), 12 pgs degraded (PG_DEGRADED) 2024-09-12T23:44:46.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:46 smithi181 bash[67587]: audit 2024-09-12T23:44:45.657779+0000 mon.a (mon.0) 449 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:46.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:46 smithi181 bash[67587]: audit 2024-09-12T23:44:45.664365+0000 mon.a (mon.0) 450 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:46.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:46 smithi181 bash[67587]: audit 2024-09-12T23:44:45.977151+0000 mon.c (mon.2) 5 : audit [INF] from='osd.1 [v2:172.21.15.136:6810/3078861551,v1:172.21.15.136:6811/3078861551]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-12T23:44:46.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:46 smithi181 bash[67587]: audit 2024-09-12T23:44:45.977375+0000 mon.a (mon.0) 451 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-12T23:44:47.006 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 12 23:44:46 smithi136 bash[84852]: debug 2024-09-12T23:44:46.649+0000 7f8a1e1d1640 -1 osd.1 419 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-12T23:44:47.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:46 smithi136 bash[75164]: cluster 2024-09-12T23:44:45.564495+0000 mon.a (mon.0) 448 : cluster [WRN] Health check update: Degraded data redundancy: 62/321 objects degraded (19.315%), 12 pgs degraded (PG_DEGRADED) 2024-09-12T23:44:47.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:46 smithi136 bash[75164]: audit 2024-09-12T23:44:45.657779+0000 mon.a (mon.0) 449 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:47.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:46 smithi136 bash[75164]: audit 2024-09-12T23:44:45.664365+0000 mon.a (mon.0) 450 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:47.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:46 smithi136 bash[75164]: audit 2024-09-12T23:44:45.977151+0000 mon.c (mon.2) 5 : audit [INF] from='osd.1 [v2:172.21.15.136:6810/3078861551,v1:172.21.15.136:6811/3078861551]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-12T23:44:47.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:46 smithi136 bash[75164]: audit 2024-09-12T23:44:45.977375+0000 mon.a (mon.0) 451 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-12T23:44:47.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:46 smithi136 bash[76371]: cluster 2024-09-12T23:44:45.564495+0000 mon.a (mon.0) 448 : cluster [WRN] Health check update: Degraded data redundancy: 62/321 objects degraded (19.315%), 12 pgs degraded (PG_DEGRADED) 2024-09-12T23:44:47.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:46 smithi136 bash[76371]: audit 2024-09-12T23:44:45.657779+0000 mon.a (mon.0) 449 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:47.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:46 smithi136 bash[76371]: audit 2024-09-12T23:44:45.664365+0000 mon.a (mon.0) 450 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:47.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:46 smithi136 bash[76371]: audit 2024-09-12T23:44:45.977151+0000 mon.c (mon.2) 5 : audit [INF] from='osd.1 [v2:172.21.15.136:6810/3078861551,v1:172.21.15.136:6811/3078861551]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-12T23:44:47.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:46 smithi136 bash[76371]: audit 2024-09-12T23:44:45.977375+0000 mon.a (mon.0) 451 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-12T23:44:47.602 INFO:tasks.rados.rados.0.smithi136.stdout:1885: done (2 left) 2024-09-12T23:44:47.602 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 9 current snap is 179 2024-09-12T23:44:47.602 INFO:tasks.rados.rados.0.smithi136.stdout:1882: finishing write tid 1 to smithi13667081-43 2024-09-12T23:44:47.603 INFO:tasks.rados.rados.0.smithi136.stdout:1886: seq_num 619 ranges {7839744=770048} 2024-09-12T23:44:47.653 INFO:tasks.rados.rados.0.smithi136.stdout:1886: writing smithi13667081-9 from 7839744 to 8609792 tid 1 2024-09-12T23:44:47.654 INFO:tasks.rados.rados.0.smithi136.stdout:1887: snap_create 2024-09-12T23:44:47.654 INFO:tasks.rados.rados.0.smithi136.stdout:1882: finishing write tid 2 to smithi13667081-43 2024-09-12T23:44:47.654 INFO:tasks.rados.rados.0.smithi136.stdout:1882: finishing write tid 3 to smithi13667081-43 2024-09-12T23:44:47.654 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 43 v 1115 (ObjNum 616 snap 179 seq_num 616) dirty exists 2024-09-12T23:44:47.654 INFO:tasks.rados.rados.0.smithi136.stdout:1882: left oid 43 (ObjNum 616 snap 179 seq_num 616) 2024-09-12T23:44:47.654 INFO:tasks.rados.rados.0.smithi136.stdout:1884: finishing write tid 1 to smithi13667081-26 2024-09-12T23:44:47.654 INFO:tasks.rados.rados.0.smithi136.stdout:1884: finishing write tid 2 to smithi13667081-26 2024-09-12T23:44:47.654 INFO:tasks.rados.rados.0.smithi136.stdout:1884: finishing write tid 3 to smithi13667081-26 2024-09-12T23:44:47.654 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 26 v 635 (ObjNum 618 snap 179 seq_num 618) dirty exists 2024-09-12T23:44:47.654 INFO:tasks.rados.rados.0.smithi136.stdout:1884: left oid 26 (ObjNum 618 snap 179 seq_num 618) 2024-09-12T23:44:48.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:47 smithi136 bash[75164]: audit 2024-09-12T23:44:46.602567+0000 mon.a (mon.0) 452 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-12T23:44:48.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:47 smithi136 bash[75164]: cluster 2024-09-12T23:44:46.602750+0000 mon.a (mon.0) 453 : cluster [DBG] osdmap e432: 8 total, 7 up, 8 in 2024-09-12T23:44:48.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:47 smithi136 bash[75164]: audit 2024-09-12T23:44:46.608237+0000 mon.c (mon.2) 6 : audit [INF] from='osd.1 [v2:172.21.15.136:6810/3078861551,v1:172.21.15.136:6811/3078861551]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-12T23:44:48.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:47 smithi136 bash[75164]: audit 2024-09-12T23:44:46.609610+0000 mon.a (mon.0) 454 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-12T23:44:48.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:47 smithi136 bash[75164]: cluster 2024-09-12T23:44:46.706110+0000 mgr.y (mgr.24661) 149 : cluster [DBG] pgmap v223: 121 pgs: 34 active+undersized, 12 active+undersized+degraded, 75 active+clean; 198 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 4.0 MiB/s wr, 31 op/s; 62/321 objects degraded (19.315%) 2024-09-12T23:44:48.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:47 smithi136 bash[76371]: audit 2024-09-12T23:44:46.602567+0000 mon.a (mon.0) 452 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-12T23:44:48.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:47 smithi136 bash[76371]: cluster 2024-09-12T23:44:46.602750+0000 mon.a (mon.0) 453 : cluster [DBG] osdmap e432: 8 total, 7 up, 8 in 2024-09-12T23:44:48.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:47 smithi136 bash[76371]: audit 2024-09-12T23:44:46.608237+0000 mon.c (mon.2) 6 : audit [INF] from='osd.1 [v2:172.21.15.136:6810/3078861551,v1:172.21.15.136:6811/3078861551]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-12T23:44:48.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:47 smithi136 bash[76371]: audit 2024-09-12T23:44:46.609610+0000 mon.a (mon.0) 454 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-12T23:44:48.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:47 smithi136 bash[76371]: cluster 2024-09-12T23:44:46.706110+0000 mgr.y (mgr.24661) 149 : cluster [DBG] pgmap v223: 121 pgs: 34 active+undersized, 12 active+undersized+degraded, 75 active+clean; 198 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 4.0 MiB/s wr, 31 op/s; 62/321 objects degraded (19.315%) 2024-09-12T23:44:48.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:47 smithi181 bash[67587]: audit 2024-09-12T23:44:46.602567+0000 mon.a (mon.0) 452 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-12T23:44:48.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:47 smithi181 bash[67587]: cluster 2024-09-12T23:44:46.602750+0000 mon.a (mon.0) 453 : cluster [DBG] osdmap e432: 8 total, 7 up, 8 in 2024-09-12T23:44:48.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:47 smithi181 bash[67587]: audit 2024-09-12T23:44:46.608237+0000 mon.c (mon.2) 6 : audit [INF] from='osd.1 [v2:172.21.15.136:6810/3078861551,v1:172.21.15.136:6811/3078861551]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-12T23:44:48.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:47 smithi181 bash[67587]: audit 2024-09-12T23:44:46.609610+0000 mon.a (mon.0) 454 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-12T23:44:48.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:47 smithi181 bash[67587]: cluster 2024-09-12T23:44:46.706110+0000 mgr.y (mgr.24661) 149 : cluster [DBG] pgmap v223: 121 pgs: 34 active+undersized, 12 active+undersized+degraded, 75 active+clean; 198 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 4.0 MiB/s wr, 31 op/s; 62/321 objects degraded (19.315%) 2024-09-12T23:44:48.613 INFO:tasks.rados.rados.0.smithi136.stdout:1882: done (3 left) 2024-09-12T23:44:48.613 INFO:tasks.rados.rados.0.smithi136.stdout:1884: done (2 left) 2024-09-12T23:44:48.613 INFO:tasks.rados.rados.0.smithi136.stdout:1887: done (1 left) 2024-09-12T23:44:48.613 INFO:tasks.rados.rados.0.smithi136.stdout:1888: rollback oid 46 current snap is 180 2024-09-12T23:44:48.613 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 46 to 178 2024-09-12T23:44:48.613 INFO:tasks.rados.rados.0.smithi136.stdout:1889: snap_create 2024-09-12T23:44:48.614 INFO:tasks.rados.rados.0.smithi136.stdout:1888: finishing rollback tid 1 to smithi13667081-46 2024-09-12T23:44:48.614 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 46 v 0 (ObjNum 33 snap 0 seq_num 4243394096) dirty dne 2024-09-12T23:44:48.668 INFO:tasks.rados.rados.0.smithi136.stdout:1886: finishing write tid 1 to smithi13667081-9 2024-09-12T23:44:48.668 INFO:tasks.rados.rados.0.smithi136.stdout:1886: finishing write tid 2 to smithi13667081-9 2024-09-12T23:44:48.670 INFO:tasks.rados.rados.0.smithi136.stdout:1886: finishing write tid 3 to smithi13667081-9 2024-09-12T23:44:48.670 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 9 v 640 (ObjNum 619 snap 179 seq_num 619) dirty exists 2024-09-12T23:44:48.670 INFO:tasks.rados.rados.0.smithi136.stdout:1886: left oid 9 (ObjNum 619 snap 179 seq_num 619) 2024-09-12T23:44:49.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:48 smithi136 bash[75164]: cluster 2024-09-12T23:44:47.596054+0000 mon.a (mon.0) 455 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-12T23:44:49.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:48 smithi136 bash[75164]: cluster 2024-09-12T23:44:47.604628+0000 mon.a (mon.0) 456 : cluster [INF] osd.1 [v2:172.21.15.136:6810/3078861551,v1:172.21.15.136:6811/3078861551] boot 2024-09-12T23:44:49.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:48 smithi136 bash[75164]: cluster 2024-09-12T23:44:47.604815+0000 mon.a (mon.0) 457 : cluster [DBG] osdmap e433: 8 total, 8 up, 8 in 2024-09-12T23:44:49.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:48 smithi136 bash[75164]: audit 2024-09-12T23:44:47.605581+0000 mon.a (mon.0) 458 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-12T23:44:49.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:48 smithi136 bash[75164]: cluster 2024-09-12T23:44:48.610946+0000 mon.a (mon.0) 459 : cluster [DBG] osdmap e434: 8 total, 8 up, 8 in 2024-09-12T23:44:49.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:48 smithi136 bash[76371]: cluster 2024-09-12T23:44:47.596054+0000 mon.a (mon.0) 455 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-12T23:44:49.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:48 smithi136 bash[76371]: cluster 2024-09-12T23:44:47.604628+0000 mon.a (mon.0) 456 : cluster [INF] osd.1 [v2:172.21.15.136:6810/3078861551,v1:172.21.15.136:6811/3078861551] boot 2024-09-12T23:44:49.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:48 smithi136 bash[76371]: cluster 2024-09-12T23:44:47.604815+0000 mon.a (mon.0) 457 : cluster [DBG] osdmap e433: 8 total, 8 up, 8 in 2024-09-12T23:44:49.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:48 smithi136 bash[76371]: audit 2024-09-12T23:44:47.605581+0000 mon.a (mon.0) 458 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-12T23:44:49.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:48 smithi136 bash[76371]: cluster 2024-09-12T23:44:48.610946+0000 mon.a (mon.0) 459 : cluster [DBG] osdmap e434: 8 total, 8 up, 8 in 2024-09-12T23:44:49.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:48 smithi181 bash[67587]: cluster 2024-09-12T23:44:47.596054+0000 mon.a (mon.0) 455 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-12T23:44:49.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:48 smithi181 bash[67587]: cluster 2024-09-12T23:44:47.604628+0000 mon.a (mon.0) 456 : cluster [INF] osd.1 [v2:172.21.15.136:6810/3078861551,v1:172.21.15.136:6811/3078861551] boot 2024-09-12T23:44:49.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:48 smithi181 bash[67587]: cluster 2024-09-12T23:44:47.604815+0000 mon.a (mon.0) 457 : cluster [DBG] osdmap e433: 8 total, 8 up, 8 in 2024-09-12T23:44:49.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:48 smithi181 bash[67587]: audit 2024-09-12T23:44:47.605581+0000 mon.a (mon.0) 458 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-12T23:44:49.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:48 smithi181 bash[67587]: cluster 2024-09-12T23:44:48.610946+0000 mon.a (mon.0) 459 : cluster [DBG] osdmap e434: 8 total, 8 up, 8 in 2024-09-12T23:44:49.095 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:44:48 smithi181 bash[60471]: ts=2024-09-12T23:44:48.662Z caller=alerting.go:366 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-09-12T23:44:49.095 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:44:48 smithi181 bash[60471]: ts=2024-09-12T23:44:48.676Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:44:49.607 INFO:tasks.rados.rados.0.smithi136.stdout:1886: done (2 left) 2024-09-12T23:44:49.607 INFO:tasks.rados.rados.0.smithi136.stdout:1888: done (1 left) 2024-09-12T23:44:49.607 INFO:tasks.rados.rados.0.smithi136.stdout:1889: done (0 left) 2024-09-12T23:44:49.608 INFO:tasks.rados.rados.0.smithi136.stdout:1890: read oid 4 snap 180 2024-09-12T23:44:49.608 INFO:tasks.rados.rados.0.smithi136.stdout:1890: expect (ObjNum 567 snap 162 seq_num 567) 2024-09-12T23:44:49.608 INFO:tasks.rados.rados.0.smithi136.stdout:1891: snap_remove snap 177 2024-09-12T23:44:49.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:49 smithi136 bash[75164]: cluster 2024-09-12T23:44:48.706614+0000 mgr.y (mgr.24661) 150 : cluster [DBG] pgmap v226: 121 pgs: 6 peering, 29 active+undersized, 11 active+undersized+degraded, 75 active+clean; 209 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 2.6 MiB/s wr, 0 op/s; 47/321 objects degraded (14.642%) 2024-09-12T23:44:49.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:49 smithi136 bash[75164]: cluster 2024-09-12T23:44:49.610784+0000 mon.a (mon.0) 460 : cluster [DBG] osdmap e435: 8 total, 8 up, 8 in 2024-09-12T23:44:49.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:44:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:44:49] "GET /metrics HTTP/1.1" 200 36401 "" "Prometheus/2.43.0" 2024-09-12T23:44:49.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:49 smithi136 bash[76371]: cluster 2024-09-12T23:44:48.706614+0000 mgr.y (mgr.24661) 150 : cluster [DBG] pgmap v226: 121 pgs: 6 peering, 29 active+undersized, 11 active+undersized+degraded, 75 active+clean; 209 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 2.6 MiB/s wr, 0 op/s; 47/321 objects degraded (14.642%) 2024-09-12T23:44:49.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:49 smithi136 bash[76371]: cluster 2024-09-12T23:44:49.610784+0000 mon.a (mon.0) 460 : cluster [DBG] osdmap e435: 8 total, 8 up, 8 in 2024-09-12T23:44:50.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:49 smithi181 bash[67587]: cluster 2024-09-12T23:44:48.706614+0000 mgr.y (mgr.24661) 150 : cluster [DBG] pgmap v226: 121 pgs: 6 peering, 29 active+undersized, 11 active+undersized+degraded, 75 active+clean; 209 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 2.6 MiB/s wr, 0 op/s; 47/321 objects degraded (14.642%) 2024-09-12T23:44:50.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:49 smithi181 bash[67587]: cluster 2024-09-12T23:44:49.610784+0000 mon.a (mon.0) 460 : cluster [DBG] osdmap e435: 8 total, 8 up, 8 in 2024-09-12T23:44:50.608 INFO:tasks.rados.rados.0.smithi136.stdout:1891: done (1 left) 2024-09-12T23:44:50.609 INFO:tasks.rados.rados.0.smithi136.stdout:1892: setattr oid 49 current snap is 181 2024-09-12T23:44:50.609 INFO:tasks.rados.rados.0.smithi136.stdout:1890: expect (ObjNum 567 snap 162 seq_num 567) 2024-09-12T23:44:50.966 INFO:tasks.rados.rados.0.smithi136.stdout:1890: done (1 left) 2024-09-12T23:44:50.966 INFO:tasks.rados.rados.0.smithi136.stdout:1893: snap_remove snap 178 2024-09-12T23:44:51.624 INFO:tasks.rados.rados.0.smithi136.stdout:1893: done (1 left) 2024-09-12T23:44:51.624 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 41 current snap is 181 2024-09-12T23:44:51.624 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 49 v 1199 (ObjNum 465 snap 0 seq_num 2879339553) dirty exists 2024-09-12T23:44:51.624 INFO:tasks.rados.rados.0.smithi136.stdout:1894: seq_num 620 ranges {0=794624} 2024-09-12T23:44:51.640 INFO:tasks.rados.rados.0.smithi136.stdout:1894: writing smithi13667081-41 from 0 to 794624 tid 1 2024-09-12T23:44:51.640 INFO:tasks.rados.rados.0.smithi136.stdout:1892: done (1 left) 2024-09-12T23:44:51.640 INFO:tasks.rados.rados.0.smithi136.stdout:1895: copy_from oid 29 from oid 48 current snap is 181 2024-09-12T23:44:51.640 INFO:tasks.rados.rados.0.smithi136.stdout:1896: snap_remove snap 175 2024-09-12T23:44:51.964 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:51 smithi136 bash[76371]: cluster 2024-09-12T23:44:50.612582+0000 mon.a (mon.0) 461 : cluster [DBG] osdmap e436: 8 total, 8 up, 8 in 2024-09-12T23:44:51.964 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:51 smithi136 bash[76371]: cluster 2024-09-12T23:44:50.707311+0000 mgr.y (mgr.24661) 151 : cluster [DBG] pgmap v229: 121 pgs: 6 peering, 12 active+undersized, 5 active+undersized+degraded, 98 active+clean; 182 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 3.0 MiB/s wr, 8 op/s; 8/249 objects degraded (3.213%); 16 MiB/s, 3 objects/s recovering 2024-09-12T23:44:51.964 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:51 smithi136 bash[76371]: cluster 2024-09-12T23:44:51.325291+0000 mon.a (mon.0) 462 : cluster [WRN] Health check update: Degraded data redundancy: 47/321 objects degraded (14.642%), 11 pgs degraded (PG_DEGRADED) 2024-09-12T23:44:51.964 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:51 smithi136 bash[75164]: cluster 2024-09-12T23:44:50.612582+0000 mon.a (mon.0) 461 : cluster [DBG] osdmap e436: 8 total, 8 up, 8 in 2024-09-12T23:44:51.965 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:51 smithi136 bash[75164]: cluster 2024-09-12T23:44:50.707311+0000 mgr.y (mgr.24661) 151 : cluster [DBG] pgmap v229: 121 pgs: 6 peering, 12 active+undersized, 5 active+undersized+degraded, 98 active+clean; 182 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 3.0 MiB/s wr, 8 op/s; 8/249 objects degraded (3.213%); 16 MiB/s, 3 objects/s recovering 2024-09-12T23:44:51.965 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:51 smithi136 bash[75164]: cluster 2024-09-12T23:44:51.325291+0000 mon.a (mon.0) 462 : cluster [WRN] Health check update: Degraded data redundancy: 47/321 objects degraded (14.642%), 11 pgs degraded (PG_DEGRADED) 2024-09-12T23:44:52.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:51 smithi181 bash[67587]: cluster 2024-09-12T23:44:50.612582+0000 mon.a (mon.0) 461 : cluster [DBG] osdmap e436: 8 total, 8 up, 8 in 2024-09-12T23:44:52.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:51 smithi181 bash[67587]: cluster 2024-09-12T23:44:50.707311+0000 mgr.y (mgr.24661) 151 : cluster [DBG] pgmap v229: 121 pgs: 6 peering, 12 active+undersized, 5 active+undersized+degraded, 98 active+clean; 182 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 3.0 MiB/s wr, 8 op/s; 8/249 objects degraded (3.213%); 16 MiB/s, 3 objects/s recovering 2024-09-12T23:44:52.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:51 smithi181 bash[67587]: cluster 2024-09-12T23:44:51.325291+0000 mon.a (mon.0) 462 : cluster [WRN] Health check update: Degraded data redundancy: 47/321 objects degraded (14.642%), 11 pgs degraded (PG_DEGRADED) 2024-09-12T23:44:52.594 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:44:52 smithi181 bash[60471]: ts=2024-09-12T23:44:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:44:52.630 INFO:tasks.rados.rados.0.smithi136.stdout:1896: done (2 left) 2024-09-12T23:44:52.630 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 49 current snap is 181 2024-09-12T23:44:52.630 INFO:tasks.rados.rados.0.smithi136.stdout:1894: finishing write tid 1 to smithi13667081-41 2024-09-12T23:44:52.630 INFO:tasks.rados.rados.0.smithi136.stdout:1894: finishing write tid 2 to smithi13667081-41 2024-09-12T23:44:52.630 INFO:tasks.rados.rados.0.smithi136.stdout:1894: finishing write tid 3 to smithi13667081-41 2024-09-12T23:44:52.630 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 41 v 1217 (ObjNum 620 snap 181 seq_num 620) dirty exists 2024-09-12T23:44:52.630 INFO:tasks.rados.rados.0.smithi136.stdout:1894: left oid 41 (ObjNum 620 snap 181 seq_num 620) 2024-09-12T23:44:52.631 INFO:tasks.rados.rados.0.smithi136.stdout:1895: finishing copy_from racing read to smithi13667081-29 2024-09-12T23:44:52.631 INFO:tasks.rados.rados.0.smithi136.stdout:1895: finishing copy_from to smithi13667081-29 2024-09-12T23:44:52.631 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 29 v 1038 (ObjNum 544 snap 158 seq_num 544) dirty exists 2024-09-12T23:44:52.631 INFO:tasks.rados.rados.0.smithi136.stdout:1897: seq_num 621 ranges {0=409600,409600=401408} 2024-09-12T23:44:52.637 INFO:tasks.rados.rados.0.smithi136.stdout:1897: writing smithi13667081-49 from 0 to 409600 tid 1 2024-09-12T23:44:52.646 INFO:tasks.rados.rados.0.smithi136.stdout:1897: writing smithi13667081-49 from 409600 to 811008 tid 2 2024-09-12T23:44:52.646 INFO:tasks.rados.rados.0.smithi136.stdout:1894: done (2 left) 2024-09-12T23:44:52.646 INFO:tasks.rados.rados.0.smithi136.stdout:1895: done (1 left) 2024-09-12T23:44:52.646 INFO:tasks.rados.rados.0.smithi136.stdout:1898: read oid 11 snap -1 2024-09-12T23:44:52.646 INFO:tasks.rados.rados.0.smithi136.stdout:1898: expect (ObjNum 482 snap 142 seq_num 482) 2024-09-12T23:44:52.646 INFO:tasks.rados.rados.0.smithi136.stdout:1899: snap_create 2024-09-12T23:44:52.667 INFO:tasks.rados.rados.0.smithi136.stdout:1897: finishing write tid 1 to smithi13667081-49 2024-09-12T23:44:52.668 INFO:tasks.rados.rados.0.smithi136.stdout:1897: finishing write tid 2 to smithi13667081-49 2024-09-12T23:44:52.668 INFO:tasks.rados.rados.0.smithi136.stdout:1897: finishing write tid 3 to smithi13667081-49 2024-09-12T23:44:52.673 INFO:tasks.rados.rados.0.smithi136.stdout:1897: finishing write tid 4 to smithi13667081-49 2024-09-12T23:44:52.673 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 49 v 1202 (ObjNum 621 snap 181 seq_num 621) dirty exists 2024-09-12T23:44:52.673 INFO:tasks.rados.rados.0.smithi136.stdout:1897: left oid 49 (ObjNum 621 snap 181 seq_num 621) 2024-09-12T23:44:52.680 INFO:tasks.rados.rados.0.smithi136.stdout:1898: expect (ObjNum 482 snap 142 seq_num 482) 2024-09-12T23:44:53.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:52 smithi136 bash[75164]: cluster 2024-09-12T23:44:51.627781+0000 mon.a (mon.0) 463 : cluster [DBG] osdmap e437: 8 total, 8 up, 8 in 2024-09-12T23:44:53.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:52 smithi136 bash[76371]: cluster 2024-09-12T23:44:51.627781+0000 mon.a (mon.0) 463 : cluster [DBG] osdmap e437: 8 total, 8 up, 8 in 2024-09-12T23:44:53.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:52 smithi181 bash[67587]: cluster 2024-09-12T23:44:51.627781+0000 mon.a (mon.0) 463 : cluster [DBG] osdmap e437: 8 total, 8 up, 8 in 2024-09-12T23:44:53.649 INFO:tasks.rados.rados.0.smithi136.stdout:1897: done (2 left) 2024-09-12T23:44:53.649 INFO:tasks.rados.rados.0.smithi136.stdout:1898: done (1 left) 2024-09-12T23:44:53.649 INFO:tasks.rados.rados.0.smithi136.stdout:1899: done (0 left) 2024-09-12T23:44:53.649 INFO:tasks.rados.rados.0.smithi136.stdout:1900: setattr oid 37 current snap is 182 2024-09-12T23:44:53.653 INFO:tasks.rados.rados.0.smithi136.stdout:1901: copy_from oid 34 from oid 33 current snap is 182 2024-09-12T23:44:53.653 INFO:tasks.rados.rados.0.smithi136.stdout:1902: copy_from oid 16 from oid 43 current snap is 182 2024-09-12T23:44:53.653 INFO:tasks.rados.rados.0.smithi136.stdout:1903: rollback oid 13 current snap is 182 2024-09-12T23:44:53.653 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 13 to 181 2024-09-12T23:44:53.653 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 24 current snap is 182 2024-09-12T23:44:53.653 INFO:tasks.rados.rados.0.smithi136.stdout:1904: seq_num 622 ranges {2908160=425984,3334144=401408} 2024-09-12T23:44:53.705 INFO:tasks.rados.rados.0.smithi136.stdout:1904: writing smithi13667081-24 from 2908160 to 3334144 tid 1 2024-09-12T23:44:53.712 INFO:tasks.rados.rados.0.smithi136.stdout:1904: writing smithi13667081-24 from 3334144 to 3735552 tid 2 2024-09-12T23:44:53.712 INFO:tasks.rados.rados.0.smithi136.stdout:1905: copy_from oid 38 from oid 11 current snap is 182 2024-09-12T23:44:53.712 INFO:tasks.rados.rados.0.smithi136.stdout:1903: finishing rollback tid 1 to smithi13667081-13 2024-09-12T23:44:53.713 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 13 v 0 (ObjNum 209 snap 0 seq_num 283364887) dirty dne 2024-09-12T23:44:53.713 INFO:tasks.rados.rados.0.smithi136.stdout:1901: finishing copy_from to smithi13667081-34 2024-09-12T23:44:53.713 INFO:tasks.rados.rados.0.smithi136.stdout:1901: got expected ENOENT (src dne) 2024-09-12T23:44:53.713 INFO:tasks.rados.rados.0.smithi136.stdout:1901: finishing copy_from racing read to smithi13667081-34 2024-09-12T23:44:53.714 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 37 v 1218 (ObjNum 589 snap 172 seq_num 589) dirty exists 2024-09-12T23:44:53.714 INFO:tasks.rados.rados.0.smithi136.stdout:1900: done (5 left) 2024-09-12T23:44:53.714 INFO:tasks.rados.rados.0.smithi136.stdout:1901: done (4 left) 2024-09-12T23:44:53.714 INFO:tasks.rados.rados.0.smithi136.stdout:1903: done (3 left) 2024-09-12T23:44:53.714 INFO:tasks.rados.rados.0.smithi136.stdout:1906: rollback oid 19 current snap is 182 2024-09-12T23:44:53.715 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 19 to 179 2024-09-12T23:44:53.715 INFO:tasks.rados.rados.0.smithi136.stdout:1907: snap_create 2024-09-12T23:44:53.717 INFO:tasks.rados.rados.0.smithi136.stdout:1904: finishing write tid 1 to smithi13667081-24 2024-09-12T23:44:53.719 INFO:tasks.rados.rados.0.smithi136.stdout:1904: finishing write tid 2 to smithi13667081-24 2024-09-12T23:44:53.720 INFO:tasks.rados.rados.0.smithi136.stdout:1904: finishing write tid 3 to smithi13667081-24 2024-09-12T23:44:53.721 INFO:tasks.rados.rados.0.smithi136.stdout:1906: finishing rollback tid 0 to smithi13667081-19 2024-09-12T23:44:53.721 INFO:tasks.rados.rados.0.smithi136.stdout:1906: finishing rollback tid 1 to smithi13667081-19 2024-09-12T23:44:53.723 INFO:tasks.rados.rados.0.smithi136.stdout:1904: finishing write tid 4 to smithi13667081-24 2024-09-12T23:44:53.723 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 24 v 638 (ObjNum 622 snap 182 seq_num 622) dirty exists 2024-09-12T23:44:53.723 INFO:tasks.rados.rados.0.smithi136.stdout:1904: left oid 24 (ObjNum 622 snap 182 seq_num 622) 2024-09-12T23:44:53.725 INFO:tasks.rados.rados.0.smithi136.stdout:1906: finishing rollback tid 2 to smithi13667081-19 2024-09-12T23:44:53.725 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 19 v 650 (ObjNum 350 snap 95 seq_num 350) dirty exists 2024-09-12T23:44:53.727 INFO:tasks.rados.rados.0.smithi136.stdout:1905: finishing copy_from racing read to smithi13667081-38 2024-09-12T23:44:53.728 INFO:tasks.rados.rados.0.smithi136.stdout:1905: finishing copy_from to smithi13667081-38 2024-09-12T23:44:53.728 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 38 v 1198 (ObjNum 482 snap 142 seq_num 482) dirty exists 2024-09-12T23:44:53.740 INFO:tasks.rados.rados.0.smithi136.stdout:1902: finishing copy_from racing read to smithi13667081-16 2024-09-12T23:44:53.744 INFO:tasks.rados.rados.0.smithi136.stdout:1902: finishing copy_from to smithi13667081-16 2024-09-12T23:44:53.744 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 16 v 1115 (ObjNum 616 snap 179 seq_num 616) dirty exists 2024-09-12T23:44:53.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:53 smithi181 bash[67587]: cluster 2024-09-12T23:44:52.633653+0000 mon.a (mon.0) 464 : cluster [DBG] osdmap e438: 8 total, 8 up, 8 in 2024-09-12T23:44:53.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:53 smithi181 bash[67587]: cluster 2024-09-12T23:44:52.707893+0000 mgr.y (mgr.24661) 152 : cluster [DBG] pgmap v232: 121 pgs: 1 active+clean+snaptrim, 6 peering, 114 active+clean; 183 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 548 KiB/s wr, 7 op/s; 16 MiB/s, 3 objects/s recovering 2024-09-12T23:44:53.844 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:44:53 smithi181 bash[60471]: ts=2024-09-12T23:44:53.509Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi136\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:44:54.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:53 smithi136 bash[75164]: cluster 2024-09-12T23:44:52.633653+0000 mon.a (mon.0) 464 : cluster [DBG] osdmap e438: 8 total, 8 up, 8 in 2024-09-12T23:44:54.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:53 smithi136 bash[75164]: cluster 2024-09-12T23:44:52.707893+0000 mgr.y (mgr.24661) 152 : cluster [DBG] pgmap v232: 121 pgs: 1 active+clean+snaptrim, 6 peering, 114 active+clean; 183 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 548 KiB/s wr, 7 op/s; 16 MiB/s, 3 objects/s recovering 2024-09-12T23:44:54.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:53 smithi136 bash[76371]: cluster 2024-09-12T23:44:52.633653+0000 mon.a (mon.0) 464 : cluster [DBG] osdmap e438: 8 total, 8 up, 8 in 2024-09-12T23:44:54.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:53 smithi136 bash[76371]: cluster 2024-09-12T23:44:52.707893+0000 mgr.y (mgr.24661) 152 : cluster [DBG] pgmap v232: 121 pgs: 1 active+clean+snaptrim, 6 peering, 114 active+clean; 183 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 548 KiB/s wr, 7 op/s; 16 MiB/s, 3 objects/s recovering 2024-09-12T23:44:54.653 INFO:tasks.rados.rados.0.smithi136.stdout:1902: done (4 left) 2024-09-12T23:44:54.653 INFO:tasks.rados.rados.0.smithi136.stdout:1904: done (3 left) 2024-09-12T23:44:54.653 INFO:tasks.rados.rados.0.smithi136.stdout:1905: done (2 left) 2024-09-12T23:44:54.653 INFO:tasks.rados.rados.0.smithi136.stdout:1906: done (1 left) 2024-09-12T23:44:54.653 INFO:tasks.rados.rados.0.smithi136.stdout:1907: done (0 left) 2024-09-12T23:44:54.653 INFO:tasks.rados.rados.0.smithi136.stdout:1908: rmattr oid 9 current snap is 183 2024-09-12T23:44:54.653 INFO:tasks.rados.rados.0.smithi136.stdout:1909: copy_from oid 11 from oid 13 current snap is 183 2024-09-12T23:44:54.653 INFO:tasks.rados.rados.0.smithi136.stdout:1910: delete oid 35 current snap is 183 2024-09-12T23:44:54.659 INFO:tasks.rados.rados.0.smithi136.stdout:1909: finishing copy_from to smithi13667081-11 2024-09-12T23:44:54.659 INFO:tasks.rados.rados.0.smithi136.stdout:1909: got expected ENOENT (src dne) 2024-09-12T23:44:54.660 INFO:tasks.rados.rados.0.smithi136.stdout:1909: finishing copy_from racing read to smithi13667081-11 2024-09-12T23:44:54.661 INFO:tasks.rados.rados.0.smithi136.stdout:1909: done (2 left) 2024-09-12T23:44:54.661 INFO:tasks.rados.rados.0.smithi136.stdout:1910: done (1 left) 2024-09-12T23:44:54.661 INFO:tasks.rados.rados.0.smithi136.stdout:1911: rollback oid 28 current snap is 183 2024-09-12T23:44:54.661 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 28 to 182 2024-09-12T23:44:54.661 INFO:tasks.rados.rados.0.smithi136.stdout:1912: snap_create 2024-09-12T23:44:54.667 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 9 v 652 (ObjNum 619 snap 179 seq_num 619) dirty exists 2024-09-12T23:44:54.668 INFO:tasks.rados.rados.0.smithi136.stdout:1911: finishing rollback tid 0 to smithi13667081-28 2024-09-12T23:44:54.669 INFO:tasks.rados.rados.0.smithi136.stdout:1911: finishing rollback tid 1 to smithi13667081-28 2024-09-12T23:44:54.670 INFO:tasks.rados.rados.0.smithi136.stdout:1911: finishing rollback tid 2 to smithi13667081-28 2024-09-12T23:44:54.670 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 28 v 1134 (ObjNum 606 snap 174 seq_num 606) dirty exists 2024-09-12T23:44:54.844 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:44:54 smithi181 bash[60471]: ts=2024-09-12T23:44:54.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:44:54.844 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:44:54 smithi181 bash[60471]: ts=2024-09-12T23:44:54.456Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:44:54.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:54 smithi181 bash[67587]: cluster 2024-09-12T23:44:53.628600+0000 mon.a (mon.0) 465 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 8/249 objects degraded (3.213%), 5 pgs degraded) 2024-09-12T23:44:54.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:54 smithi181 bash[67587]: cluster 2024-09-12T23:44:53.628676+0000 mon.a (mon.0) 466 : cluster [INF] Cluster is now healthy 2024-09-12T23:44:54.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:54 smithi181 bash[67587]: cluster 2024-09-12T23:44:53.652781+0000 mon.a (mon.0) 467 : cluster [DBG] osdmap e439: 8 total, 8 up, 8 in 2024-09-12T23:44:54.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:54 smithi181 bash[67587]: audit 2024-09-12T23:44:53.752283+0000 mon.a (mon.0) 468 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:54.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:54 smithi181 bash[67587]: audit 2024-09-12T23:44:53.753467+0000 mon.a (mon.0) 469 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:44:54.896 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:54 smithi136 bash[75164]: cluster 2024-09-12T23:44:53.628600+0000 mon.a (mon.0) 465 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 8/249 objects degraded (3.213%), 5 pgs degraded) 2024-09-12T23:44:54.896 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:54 smithi136 bash[75164]: cluster 2024-09-12T23:44:53.628676+0000 mon.a (mon.0) 466 : cluster [INF] Cluster is now healthy 2024-09-12T23:44:54.896 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:54 smithi136 bash[75164]: cluster 2024-09-12T23:44:53.652781+0000 mon.a (mon.0) 467 : cluster [DBG] osdmap e439: 8 total, 8 up, 8 in 2024-09-12T23:44:54.896 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:54 smithi136 bash[75164]: audit 2024-09-12T23:44:53.752283+0000 mon.a (mon.0) 468 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:54.897 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:54 smithi136 bash[75164]: audit 2024-09-12T23:44:53.753467+0000 mon.a (mon.0) 469 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:44:54.897 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:54 smithi136 bash[76371]: cluster 2024-09-12T23:44:53.628600+0000 mon.a (mon.0) 465 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 8/249 objects degraded (3.213%), 5 pgs degraded) 2024-09-12T23:44:54.897 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:54 smithi136 bash[76371]: cluster 2024-09-12T23:44:53.628676+0000 mon.a (mon.0) 466 : cluster [INF] Cluster is now healthy 2024-09-12T23:44:54.897 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:54 smithi136 bash[76371]: cluster 2024-09-12T23:44:53.652781+0000 mon.a (mon.0) 467 : cluster [DBG] osdmap e439: 8 total, 8 up, 8 in 2024-09-12T23:44:54.897 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:54 smithi136 bash[76371]: audit 2024-09-12T23:44:53.752283+0000 mon.a (mon.0) 468 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:54.897 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:54 smithi136 bash[76371]: audit 2024-09-12T23:44:53.753467+0000 mon.a (mon.0) 469 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:44:55.664 INFO:tasks.rados.rados.0.smithi136.stdout:1908: done (2 left) 2024-09-12T23:44:55.664 INFO:tasks.rados.rados.0.smithi136.stdout:1911: done (1 left) 2024-09-12T23:44:55.664 INFO:tasks.rados.rados.0.smithi136.stdout:1912: done (0 left) 2024-09-12T23:44:55.664 INFO:tasks.rados.rados.0.smithi136.stdout:1913: read oid 30 snap 181 2024-09-12T23:44:55.664 INFO:tasks.rados.rados.0.smithi136.stdout:1913: expect (ObjNum -51952096 snap 21878 seq_num 4242921456) 2024-09-12T23:44:55.665 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 42 current snap is 184 2024-09-12T23:44:55.665 INFO:tasks.rados.rados.0.smithi136.stdout:1914: seq_num 624 ranges {2449408=458752,2908160=401408} 2024-09-12T23:44:55.696 INFO:tasks.rados.rados.0.smithi136.stdout:1914: writing smithi13667081-42 from 2449408 to 2908160 tid 1 2024-09-12T23:44:55.701 INFO:tasks.rados.rados.0.smithi136.stdout:1914: writing smithi13667081-42 from 2908160 to 3309568 tid 2 2024-09-12T23:44:55.701 INFO:tasks.rados.rados.0.smithi136.stdout:1915: rmattr oid 38 current snap is 184 2024-09-12T23:44:55.701 INFO:tasks.rados.rados.0.smithi136.stdout:1915: done (2 left) 2024-09-12T23:44:55.701 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 39 current snap is 184 2024-09-12T23:44:55.701 INFO:tasks.rados.rados.0.smithi136.stdout:1916: seq_num 625 ranges {286720=475136,761856=401408} 2024-09-12T23:44:55.710 INFO:tasks.rados.rados.0.smithi136.stdout:1916: writing smithi13667081-39 from 286720 to 761856 tid 1 2024-09-12T23:44:55.714 INFO:tasks.rados.rados.0.smithi136.stdout:1916: writing smithi13667081-39 from 761856 to 1163264 tid 2 2024-09-12T23:44:55.714 INFO:tasks.rados.rados.0.smithi136.stdout:1917: read oid 50 snap 180 2024-09-12T23:44:55.715 INFO:tasks.rados.rados.0.smithi136.stdout:1917: expect deleted 2024-09-12T23:44:55.715 INFO:tasks.rados.rados.0.smithi136.stdout:1918: read oid 10 snap -1 2024-09-12T23:44:55.715 INFO:tasks.rados.rados.0.smithi136.stdout:1918: expect (ObjNum 612 snap 175 seq_num 612) 2024-09-12T23:44:55.715 INFO:tasks.rados.rados.0.smithi136.stdout:1919: rmattr oid 2 current snap is 184 2024-09-12T23:44:55.715 INFO:tasks.rados.rados.0.smithi136.stdout:1919: done (5 left) 2024-09-12T23:44:55.715 INFO:tasks.rados.rados.0.smithi136.stdout:1920: rollback oid 44 current snap is 184 2024-09-12T23:44:55.715 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 44 to 179 2024-09-12T23:44:55.715 INFO:tasks.rados.rados.0.smithi136.stdout:1913: expect (ObjNum -51952096 snap 21878 seq_num 4242921456) 2024-09-12T23:44:55.721 INFO:tasks.rados.rados.0.smithi136.stdout:1914: finishing write tid 1 to smithi13667081-42 2024-09-12T23:44:55.721 INFO:tasks.rados.rados.0.smithi136.stdout:1914: finishing write tid 2 to smithi13667081-42 2024-09-12T23:44:55.721 INFO:tasks.rados.rados.0.smithi136.stdout:1914: finishing write tid 3 to smithi13667081-42 2024-09-12T23:44:55.722 INFO:tasks.rados.rados.0.smithi136.stdout:1913: done (5 left) 2024-09-12T23:44:55.722 INFO:tasks.rados.rados.0.smithi136.stdout:1917: done (4 left) 2024-09-12T23:44:55.722 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 38 current snap is 184 2024-09-12T23:44:55.722 INFO:tasks.rados.rados.0.smithi136.stdout:1921: seq_num 626 ranges {876544=491520,1368064=401408} 2024-09-12T23:44:55.735 INFO:tasks.rados.rados.0.smithi136.stdout:1921: writing smithi13667081-38 from 876544 to 1368064 tid 1 2024-09-12T23:44:55.739 INFO:tasks.rados.rados.0.smithi136.stdout:1921: writing smithi13667081-38 from 1368064 to 1769472 tid 2 2024-09-12T23:44:55.739 INFO:tasks.rados.rados.0.smithi136.stdout:1922: snap_create 2024-09-12T23:44:55.739 INFO:tasks.rados.rados.0.smithi136.stdout:1914: finishing write tid 4 to smithi13667081-42 2024-09-12T23:44:55.739 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 42 v 830 (ObjNum 624 snap 184 seq_num 624) dirty exists 2024-09-12T23:44:55.739 INFO:tasks.rados.rados.0.smithi136.stdout:1914: left oid 42 (ObjNum 624 snap 184 seq_num 624) 2024-09-12T23:44:55.740 INFO:tasks.rados.rados.0.smithi136.stdout:1920: finishing rollback tid 0 to smithi13667081-44 2024-09-12T23:44:55.740 INFO:tasks.rados.rados.0.smithi136.stdout:1920: finishing rollback tid 1 to smithi13667081-44 2024-09-12T23:44:55.740 INFO:tasks.rados.rados.0.smithi136.stdout:1920: finishing rollback tid 2 to smithi13667081-44 2024-09-12T23:44:55.740 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 44 v 1138 (ObjNum 564 snap 162 seq_num 564) dirty exists 2024-09-12T23:44:55.740 INFO:tasks.rados.rados.0.smithi136.stdout:1918: expect (ObjNum 612 snap 175 seq_num 612) 2024-09-12T23:44:55.898 INFO:tasks.rados.rados.0.smithi136.stdout:1916: finishing write tid 1 to smithi13667081-39 2024-09-12T23:44:55.898 INFO:tasks.rados.rados.0.smithi136.stdout:1916: finishing write tid 2 to smithi13667081-39 2024-09-12T23:44:55.898 INFO:tasks.rados.rados.0.smithi136.stdout:1916: finishing write tid 3 to smithi13667081-39 2024-09-12T23:44:55.898 INFO:tasks.rados.rados.0.smithi136.stdout:1916: finishing write tid 4 to smithi13667081-39 2024-09-12T23:44:55.898 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 39 v 783 (ObjNum 625 snap 184 seq_num 625) dirty exists 2024-09-12T23:44:55.898 INFO:tasks.rados.rados.0.smithi136.stdout:1916: left oid 39 (ObjNum 625 snap 184 seq_num 625) 2024-09-12T23:44:55.898 INFO:tasks.rados.rados.0.smithi136.stdout:1921: finishing write tid 1 to smithi13667081-38 2024-09-12T23:44:55.898 INFO:tasks.rados.rados.0.smithi136.stdout:1921: finishing write tid 2 to smithi13667081-38 2024-09-12T23:44:55.898 INFO:tasks.rados.rados.0.smithi136.stdout:1921: finishing write tid 3 to smithi13667081-38 2024-09-12T23:44:55.898 INFO:tasks.rados.rados.0.smithi136.stdout:1921: finishing write tid 4 to smithi13667081-38 2024-09-12T23:44:55.899 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 38 v 1201 (ObjNum 626 snap 184 seq_num 626) dirty exists 2024-09-12T23:44:55.899 INFO:tasks.rados.rados.0.smithi136.stdout:1921: left oid 38 (ObjNum 626 snap 184 seq_num 626) 2024-09-12T23:44:56.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:55 smithi136 bash[76371]: cluster 2024-09-12T23:44:54.656815+0000 mon.a (mon.0) 470 : cluster [DBG] osdmap e440: 8 total, 8 up, 8 in 2024-09-12T23:44:56.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:55 smithi136 bash[76371]: cluster 2024-09-12T23:44:54.708587+0000 mgr.y (mgr.24661) 153 : cluster [DBG] pgmap v235: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 167 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 4.3 MiB/s wr, 15 op/s; 15 MiB/s, 2 objects/s recovering 2024-09-12T23:44:56.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:55 smithi136 bash[75164]: cluster 2024-09-12T23:44:54.656815+0000 mon.a (mon.0) 470 : cluster [DBG] osdmap e440: 8 total, 8 up, 8 in 2024-09-12T23:44:56.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:55 smithi136 bash[75164]: cluster 2024-09-12T23:44:54.708587+0000 mgr.y (mgr.24661) 153 : cluster [DBG] pgmap v235: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 167 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 4.3 MiB/s wr, 15 op/s; 15 MiB/s, 2 objects/s recovering 2024-09-12T23:44:56.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:55 smithi181 bash[67587]: cluster 2024-09-12T23:44:54.656815+0000 mon.a (mon.0) 470 : cluster [DBG] osdmap e440: 8 total, 8 up, 8 in 2024-09-12T23:44:56.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:55 smithi181 bash[67587]: cluster 2024-09-12T23:44:54.708587+0000 mgr.y (mgr.24661) 153 : cluster [DBG] pgmap v235: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 167 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 4.3 MiB/s wr, 15 op/s; 15 MiB/s, 2 objects/s recovering 2024-09-12T23:44:56.331 INFO:tasks.rados.rados.0.smithi136.stdout:1914: done (5 left) 2024-09-12T23:44:56.331 INFO:tasks.rados.rados.0.smithi136.stdout:1916: done (4 left) 2024-09-12T23:44:56.332 INFO:tasks.rados.rados.0.smithi136.stdout:1918: done (3 left) 2024-09-12T23:44:56.332 INFO:tasks.rados.rados.0.smithi136.stdout:1920: done (2 left) 2024-09-12T23:44:56.332 INFO:tasks.rados.rados.0.smithi136.stdout:1921: done (1 left) 2024-09-12T23:44:56.332 INFO:tasks.rados.rados.0.smithi136.stdout:1922: done (0 left) 2024-09-12T23:44:56.332 INFO:tasks.rados.rados.0.smithi136.stdout:1923: delete oid 40 current snap is 185 2024-09-12T23:44:56.342 INFO:tasks.rados.rados.0.smithi136.stdout:1923: done (0 left) 2024-09-12T23:44:56.342 INFO:tasks.rados.rados.0.smithi136.stdout:1924: read oid 17 snap 184 2024-09-12T23:44:56.342 INFO:tasks.rados.rados.0.smithi136.stdout:1924: expect (ObjNum 595 snap 173 seq_num 595) 2024-09-12T23:44:56.343 INFO:tasks.rados.rados.0.smithi136.stdout:1925: read oid 45 snap 183 2024-09-12T23:44:56.343 INFO:tasks.rados.rados.0.smithi136.stdout:1925: expect (ObjNum 351 snap 96 seq_num 351) 2024-09-12T23:44:56.343 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 29 current snap is 185 2024-09-12T23:44:56.343 INFO:tasks.rados.rados.0.smithi136.stdout:1926: seq_num 628 ranges {3866624=524288,4390912=401408} 2024-09-12T23:44:56.388 INFO:tasks.rados.rados.0.smithi136.stdout:1926: writing smithi13667081-29 from 3866624 to 4390912 tid 1 2024-09-12T23:44:56.392 INFO:tasks.rados.rados.0.smithi136.stdout:1926: writing smithi13667081-29 from 4390912 to 4792320 tid 2 2024-09-12T23:44:56.392 INFO:tasks.rados.rados.0.smithi136.stdout:1927: snap_remove snap 181 2024-09-12T23:44:57.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:56 smithi136 bash[75164]: cluster 2024-09-12T23:44:55.667484+0000 mon.a (mon.0) 471 : cluster [DBG] osdmap e441: 8 total, 8 up, 8 in 2024-09-12T23:44:57.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:56 smithi136 bash[75164]: audit 2024-09-12T23:44:55.690918+0000 mon.a (mon.0) 472 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:57.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:56 smithi136 bash[75164]: audit 2024-09-12T23:44:55.700633+0000 mon.a (mon.0) 473 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:57.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:56 smithi136 bash[75164]: audit 2024-09-12T23:44:55.702834+0000 mon.a (mon.0) 474 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:44:57.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:56 smithi136 bash[75164]: audit 2024-09-12T23:44:55.704857+0000 mon.a (mon.0) 475 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:44:57.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:56 smithi136 bash[75164]: audit 2024-09-12T23:44:55.721664+0000 mon.a (mon.0) 476 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:57.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:56 smithi136 bash[75164]: audit 2024-09-12T23:44:55.860854+0000 mon.a (mon.0) 477 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:44:57.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:56 smithi136 bash[75164]: audit 2024-09-12T23:44:55.863695+0000 mon.a (mon.0) 478 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:44:57.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:56 smithi136 bash[75164]: audit 2024-09-12T23:44:55.865310+0000 mon.a (mon.0) 479 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:44:57.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:56 smithi136 bash[75164]: audit 2024-09-12T23:44:55.866519+0000 mon.a (mon.0) 480 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:44:57.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:56 smithi136 bash[75164]: audit 2024-09-12T23:44:55.867971+0000 mon.a (mon.0) 481 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-12T23:44:57.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:56 smithi136 bash[75164]: cluster 2024-09-12T23:44:56.335421+0000 mon.a (mon.0) 482 : cluster [DBG] osdmap e442: 8 total, 8 up, 8 in 2024-09-12T23:44:57.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:56 smithi136 bash[76371]: cluster 2024-09-12T23:44:55.667484+0000 mon.a (mon.0) 471 : cluster [DBG] osdmap e441: 8 total, 8 up, 8 in 2024-09-12T23:44:57.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:56 smithi136 bash[76371]: audit 2024-09-12T23:44:55.690918+0000 mon.a (mon.0) 472 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:57.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:56 smithi136 bash[76371]: audit 2024-09-12T23:44:55.700633+0000 mon.a (mon.0) 473 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:57.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:56 smithi136 bash[76371]: audit 2024-09-12T23:44:55.702834+0000 mon.a (mon.0) 474 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:44:57.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:56 smithi136 bash[76371]: audit 2024-09-12T23:44:55.704857+0000 mon.a (mon.0) 475 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:44:57.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:56 smithi136 bash[76371]: audit 2024-09-12T23:44:55.721664+0000 mon.a (mon.0) 476 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:57.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:56 smithi136 bash[76371]: audit 2024-09-12T23:44:55.860854+0000 mon.a (mon.0) 477 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:44:57.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:56 smithi136 bash[76371]: audit 2024-09-12T23:44:55.863695+0000 mon.a (mon.0) 478 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:44:57.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:56 smithi136 bash[76371]: audit 2024-09-12T23:44:55.865310+0000 mon.a (mon.0) 479 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:44:57.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:56 smithi136 bash[76371]: audit 2024-09-12T23:44:55.866519+0000 mon.a (mon.0) 480 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:44:57.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:56 smithi136 bash[76371]: audit 2024-09-12T23:44:55.867971+0000 mon.a (mon.0) 481 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-12T23:44:57.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:56 smithi136 bash[76371]: cluster 2024-09-12T23:44:56.335421+0000 mon.a (mon.0) 482 : cluster [DBG] osdmap e442: 8 total, 8 up, 8 in 2024-09-12T23:44:57.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:56 smithi181 bash[67587]: cluster 2024-09-12T23:44:55.667484+0000 mon.a (mon.0) 471 : cluster [DBG] osdmap e441: 8 total, 8 up, 8 in 2024-09-12T23:44:57.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:56 smithi181 bash[67587]: audit 2024-09-12T23:44:55.690918+0000 mon.a (mon.0) 472 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:57.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:56 smithi181 bash[67587]: audit 2024-09-12T23:44:55.700633+0000 mon.a (mon.0) 473 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:57.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:56 smithi181 bash[67587]: audit 2024-09-12T23:44:55.702834+0000 mon.a (mon.0) 474 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:44:57.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:56 smithi181 bash[67587]: audit 2024-09-12T23:44:55.704857+0000 mon.a (mon.0) 475 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:44:57.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:56 smithi181 bash[67587]: audit 2024-09-12T23:44:55.721664+0000 mon.a (mon.0) 476 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:57.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:56 smithi181 bash[67587]: audit 2024-09-12T23:44:55.860854+0000 mon.a (mon.0) 477 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:44:57.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:56 smithi181 bash[67587]: audit 2024-09-12T23:44:55.863695+0000 mon.a (mon.0) 478 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:44:57.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:56 smithi181 bash[67587]: audit 2024-09-12T23:44:55.865310+0000 mon.a (mon.0) 479 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:44:57.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:56 smithi181 bash[67587]: audit 2024-09-12T23:44:55.866519+0000 mon.a (mon.0) 480 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:44:57.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:56 smithi181 bash[67587]: audit 2024-09-12T23:44:55.867971+0000 mon.a (mon.0) 481 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-12T23:44:57.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:56 smithi181 bash[67587]: cluster 2024-09-12T23:44:56.335421+0000 mon.a (mon.0) 482 : cluster [DBG] osdmap e442: 8 total, 8 up, 8 in 2024-09-12T23:44:57.335 INFO:tasks.rados.rados.0.smithi136.stdout:1927: done (3 left) 2024-09-12T23:44:57.335 INFO:tasks.rados.rados.0.smithi136.stdout:1928: delete oid 3 current snap is 185 2024-09-12T23:44:57.335 INFO:tasks.rados.rados.0.smithi136.stdout:1924: expect (ObjNum 595 snap 173 seq_num 595) 2024-09-12T23:44:57.368 INFO:tasks.rados.rados.0.smithi136.stdout:1925: expect (ObjNum 351 snap 96 seq_num 351) 2024-09-12T23:44:57.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:57 smithi136 bash[75164]: audit 2024-09-12T23:44:55.868165+0000 mgr.y (mgr.24661) 154 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-12T23:44:57.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:57 smithi136 bash[75164]: cephadm 2024-09-12T23:44:55.869145+0000 mgr.y (mgr.24661) 155 : cephadm [INF] Upgrade: osd.2 is safe to restart 2024-09-12T23:44:57.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:57 smithi136 bash[75164]: cluster 2024-09-12T23:44:56.709417+0000 mgr.y (mgr.24661) 156 : cluster [DBG] pgmap v238: 121 pgs: 121 active+clean; 167 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 643 KiB/s rd, 4.2 MiB/s wr, 22 op/s; 15 MiB/s, 2 objects/s recovering 2024-09-12T23:44:57.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:57 smithi136 bash[75164]: cluster 2024-09-12T23:44:57.338548+0000 mon.a (mon.0) 483 : cluster [DBG] osdmap e443: 8 total, 8 up, 8 in 2024-09-12T23:44:57.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:57 smithi136 bash[76371]: audit 2024-09-12T23:44:55.868165+0000 mgr.y (mgr.24661) 154 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-12T23:44:57.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:57 smithi136 bash[76371]: cephadm 2024-09-12T23:44:55.869145+0000 mgr.y (mgr.24661) 155 : cephadm [INF] Upgrade: osd.2 is safe to restart 2024-09-12T23:44:57.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:57 smithi136 bash[76371]: cluster 2024-09-12T23:44:56.709417+0000 mgr.y (mgr.24661) 156 : cluster [DBG] pgmap v238: 121 pgs: 121 active+clean; 167 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 643 KiB/s rd, 4.2 MiB/s wr, 22 op/s; 15 MiB/s, 2 objects/s recovering 2024-09-12T23:44:57.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:57 smithi136 bash[76371]: cluster 2024-09-12T23:44:57.338548+0000 mon.a (mon.0) 483 : cluster [DBG] osdmap e443: 8 total, 8 up, 8 in 2024-09-12T23:44:57.778 INFO:tasks.rados.rados.0.smithi136.stdout:1926: finishing write tid 1 to smithi13667081-29 2024-09-12T23:44:57.778 INFO:tasks.rados.rados.0.smithi136.stdout:1926: finishing write tid 2 to smithi13667081-29 2024-09-12T23:44:57.778 INFO:tasks.rados.rados.0.smithi136.stdout:1926: finishing write tid 3 to smithi13667081-29 2024-09-12T23:44:57.779 INFO:tasks.rados.rados.0.smithi136.stdout:1926: finishing write tid 4 to smithi13667081-29 2024-09-12T23:44:57.779 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 29 v 1204 (ObjNum 628 snap 185 seq_num 628) dirty exists 2024-09-12T23:44:57.779 INFO:tasks.rados.rados.0.smithi136.stdout:1926: left oid 29 (ObjNum 628 snap 185 seq_num 628) 2024-09-12T23:44:57.779 INFO:tasks.rados.rados.0.smithi136.stdout:1924: done (3 left) 2024-09-12T23:44:57.779 INFO:tasks.rados.rados.0.smithi136.stdout:1925: done (2 left) 2024-09-12T23:44:57.779 INFO:tasks.rados.rados.0.smithi136.stdout:1926: done (1 left) 2024-09-12T23:44:57.779 INFO:tasks.rados.rados.0.smithi136.stdout:1928: done (0 left) 2024-09-12T23:44:57.779 INFO:tasks.rados.rados.0.smithi136.stdout:1929: setattr oid 44 current snap is 185 2024-09-12T23:44:57.783 INFO:tasks.rados.rados.0.smithi136.stdout:1930: rollback oid 18 current snap is 185 2024-09-12T23:44:57.783 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 18 to 183 2024-09-12T23:44:57.783 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 48 current snap is 185 2024-09-12T23:44:57.783 INFO:tasks.rados.rados.0.smithi136.stdout:1931: seq_num 630 ranges {3866624=557056,4423680=401408} 2024-09-12T23:44:57.837 INFO:tasks.rados.rados.0.smithi136.stdout:1931: writing smithi13667081-48 from 3866624 to 4423680 tid 1 2024-09-12T23:44:57.841 INFO:tasks.rados.rados.0.smithi136.stdout:1931: writing smithi13667081-48 from 4423680 to 4825088 tid 2 2024-09-12T23:44:57.841 INFO:tasks.rados.rados.0.smithi136.stdout:1932: read oid 1 snap -1 2024-09-12T23:44:57.841 INFO:tasks.rados.rados.0.smithi136.stdout:1932: expect deleted 2024-09-12T23:44:57.841 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 15 current snap is 185 2024-09-12T23:44:57.841 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 44 v 1144 (ObjNum 564 snap 162 seq_num 564) dirty exists 2024-09-12T23:44:57.841 INFO:tasks.rados.rados.0.smithi136.stdout:1930: finishing rollback tid 0 to smithi13667081-18 2024-09-12T23:44:57.842 INFO:tasks.rados.rados.0.smithi136.stdout:1930: finishing rollback tid 1 to smithi13667081-18 2024-09-12T23:44:57.842 INFO:tasks.rados.rados.0.smithi136.stdout:1930: finishing rollback tid 2 to smithi13667081-18 2024-09-12T23:44:57.842 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 18 v 922 (ObjNum 578 snap 169 seq_num 578) dirty exists 2024-09-12T23:44:57.842 INFO:tasks.rados.rados.0.smithi136.stdout:1933: seq_num 631 ranges {876544=573440,1449984=401408} 2024-09-12T23:44:57.854 INFO:tasks.rados.rados.0.smithi136.stdout:1933: writing smithi13667081-15 from 876544 to 1449984 tid 1 2024-09-12T23:44:57.858 INFO:tasks.rados.rados.0.smithi136.stdout:1933: writing smithi13667081-15 from 1449984 to 1851392 tid 2 2024-09-12T23:44:57.858 INFO:tasks.rados.rados.0.smithi136.stdout:1929: done (4 left) 2024-09-12T23:44:57.858 INFO:tasks.rados.rados.0.smithi136.stdout:1930: done (3 left) 2024-09-12T23:44:57.858 INFO:tasks.rados.rados.0.smithi136.stdout:1934: delete oid 35 current snap is 185 2024-09-12T23:44:57.858 INFO:tasks.rados.rados.0.smithi136.stdout:1931: finishing write tid 1 to smithi13667081-48 2024-09-12T23:44:57.858 INFO:tasks.rados.rados.0.smithi136.stdout:1931: finishing write tid 2 to smithi13667081-48 2024-09-12T23:44:57.858 INFO:tasks.rados.rados.0.smithi136.stdout:1931: finishing write tid 3 to smithi13667081-48 2024-09-12T23:44:57.858 INFO:tasks.rados.rados.0.smithi136.stdout:1931: finishing write tid 4 to smithi13667081-48 2024-09-12T23:44:57.858 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 48 v 1148 (ObjNum 630 snap 185 seq_num 630) dirty exists 2024-09-12T23:44:57.858 INFO:tasks.rados.rados.0.smithi136.stdout:1931: left oid 48 (ObjNum 630 snap 185 seq_num 630) 2024-09-12T23:44:57.860 INFO:tasks.rados.rados.0.smithi136.stdout:1931: done (3 left) 2024-09-12T23:44:57.861 INFO:tasks.rados.rados.0.smithi136.stdout:1932: done (2 left) 2024-09-12T23:44:57.861 INFO:tasks.rados.rados.0.smithi136.stdout:1934: done (1 left) 2024-09-12T23:44:57.861 INFO:tasks.rados.rados.0.smithi136.stdout:1935: snap_create 2024-09-12T23:44:57.863 INFO:tasks.rados.rados.0.smithi136.stdout:1933: finishing write tid 1 to smithi13667081-15 2024-09-12T23:44:57.863 INFO:tasks.rados.rados.0.smithi136.stdout:1933: finishing write tid 2 to smithi13667081-15 2024-09-12T23:44:57.863 INFO:tasks.rados.rados.0.smithi136.stdout:1933: finishing write tid 3 to smithi13667081-15 2024-09-12T23:44:57.865 INFO:tasks.rados.rados.0.smithi136.stdout:1933: finishing write tid 4 to smithi13667081-15 2024-09-12T23:44:57.865 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 15 v 1205 (ObjNum 631 snap 185 seq_num 631) dirty exists 2024-09-12T23:44:57.866 INFO:tasks.rados.rados.0.smithi136.stdout:1933: left oid 15 (ObjNum 631 snap 185 seq_num 631) 2024-09-12T23:44:58.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:57 smithi181 bash[67587]: audit 2024-09-12T23:44:55.868165+0000 mgr.y (mgr.24661) 154 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-12T23:44:58.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:57 smithi181 bash[67587]: cephadm 2024-09-12T23:44:55.869145+0000 mgr.y (mgr.24661) 155 : cephadm [INF] Upgrade: osd.2 is safe to restart 2024-09-12T23:44:58.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:57 smithi181 bash[67587]: cluster 2024-09-12T23:44:56.709417+0000 mgr.y (mgr.24661) 156 : cluster [DBG] pgmap v238: 121 pgs: 121 active+clean; 167 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 643 KiB/s rd, 4.2 MiB/s wr, 22 op/s; 15 MiB/s, 2 objects/s recovering 2024-09-12T23:44:58.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:57 smithi181 bash[67587]: cluster 2024-09-12T23:44:57.338548+0000 mon.a (mon.0) 483 : cluster [DBG] osdmap e443: 8 total, 8 up, 8 in 2024-09-12T23:44:58.670 INFO:tasks.rados.rados.0.smithi136.stdout:1933: done (1 left) 2024-09-12T23:44:58.670 INFO:tasks.rados.rados.0.smithi136.stdout:1935: done (0 left) 2024-09-12T23:44:58.670 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 3 current snap is 186 2024-09-12T23:44:58.670 INFO:tasks.rados.rados.0.smithi136.stdout:1936: seq_num 633 ranges {0=606208,606208=401408} 2024-09-12T23:44:58.675 INFO:tasks.rados.rados.0.smithi136.stdout:1936: writing smithi13667081-3 from 0 to 606208 tid 1 2024-09-12T23:44:58.679 INFO:tasks.rados.rados.0.smithi136.stdout:1936: writing smithi13667081-3 from 606208 to 1007616 tid 2 2024-09-12T23:44:58.680 INFO:tasks.rados.rados.0.smithi136.stdout:1937: rollback oid 11 current snap is 186 2024-09-12T23:44:58.680 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 11 to 182 2024-09-12T23:44:58.681 INFO:tasks.rados.rados.0.smithi136.stdout:1938: rollback oid 26 current snap is 186 2024-09-12T23:44:58.681 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 26 to 184 2024-09-12T23:44:58.681 INFO:tasks.rados.rados.0.smithi136.stdout:1939: rollback oid 19 current snap is 186 2024-09-12T23:44:58.681 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 19 to 183 2024-09-12T23:44:58.681 INFO:tasks.rados.rados.0.smithi136.stdout:1940: snap_create 2024-09-12T23:44:58.683 INFO:tasks.rados.rados.0.smithi136.stdout:1936: finishing write tid 1 to smithi13667081-3 2024-09-12T23:44:58.686 INFO:tasks.rados.rados.0.smithi136.stdout:1936: finishing write tid 2 to smithi13667081-3 2024-09-12T23:44:58.686 INFO:tasks.rados.rados.0.smithi136.stdout:1936: finishing write tid 3 to smithi13667081-3 2024-09-12T23:44:58.688 INFO:tasks.rados.rados.0.smithi136.stdout:1936: finishing write tid 4 to smithi13667081-3 2024-09-12T23:44:58.688 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 3 v 1208 (ObjNum 633 snap 186 seq_num 633) dirty exists 2024-09-12T23:44:58.688 INFO:tasks.rados.rados.0.smithi136.stdout:1936: left oid 3 (ObjNum 633 snap 186 seq_num 633) 2024-09-12T23:44:58.689 INFO:tasks.rados.rados.0.smithi136.stdout:1938: finishing rollback tid 0 to smithi13667081-26 2024-09-12T23:44:58.690 INFO:tasks.rados.rados.0.smithi136.stdout:1938: finishing rollback tid 1 to smithi13667081-26 2024-09-12T23:44:58.691 INFO:tasks.rados.rados.0.smithi136.stdout:1938: finishing rollback tid 2 to smithi13667081-26 2024-09-12T23:44:58.691 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 26 v 641 (ObjNum 618 snap 179 seq_num 618) dirty exists 2024-09-12T23:44:58.691 INFO:tasks.rados.rados.0.smithi136.stdout:1939: finishing rollback tid 0 to smithi13667081-19 2024-09-12T23:44:58.691 INFO:tasks.rados.rados.0.smithi136.stdout:1939: finishing rollback tid 1 to smithi13667081-19 2024-09-12T23:44:58.691 INFO:tasks.rados.rados.0.smithi136.stdout:1939: finishing rollback tid 2 to smithi13667081-19 2024-09-12T23:44:58.692 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 19 v 660 (ObjNum 350 snap 95 seq_num 350) dirty exists 2024-09-12T23:44:58.693 INFO:tasks.rados.rados.0.smithi136.stdout:1937: finishing rollback tid 0 to smithi13667081-11 2024-09-12T23:44:58.693 INFO:tasks.rados.rados.0.smithi136.stdout:1937: finishing rollback tid 1 to smithi13667081-11 2024-09-12T23:44:58.693 INFO:tasks.rados.rados.0.smithi136.stdout:1937: finishing rollback tid 2 to smithi13667081-11 2024-09-12T23:44:58.693 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 11 v 1343 (ObjNum 482 snap 142 seq_num 482) dirty exists 2024-09-12T23:44:59.094 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:44:58 smithi181 bash[60471]: ts=2024-09-12T23:44:58.660Z caller=alerting.go:366 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-09-12T23:44:59.095 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:44:58 smithi181 bash[60471]: ts=2024-09-12T23:44:58.663Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:44:59.687 INFO:tasks.rados.rados.0.smithi136.stdout:1936: done (4 left) 2024-09-12T23:44:59.687 INFO:tasks.rados.rados.0.smithi136.stdout:1937: done (3 left) 2024-09-12T23:44:59.687 INFO:tasks.rados.rados.0.smithi136.stdout:1938: done (2 left) 2024-09-12T23:44:59.687 INFO:tasks.rados.rados.0.smithi136.stdout:1939: done (1 left) 2024-09-12T23:44:59.687 INFO:tasks.rados.rados.0.smithi136.stdout:1940: done (0 left) 2024-09-12T23:44:59.688 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 38 current snap is 187 2024-09-12T23:44:59.688 INFO:tasks.rados.rados.0.smithi136.stdout:1941: seq_num 634 ranges {1769472=622592,2392064=401408} 2024-09-12T23:44:59.712 INFO:tasks.rados.rados.0.smithi136.stdout:1941: writing smithi13667081-38 from 1769472 to 2392064 tid 1 2024-09-12T23:44:59.716 INFO:tasks.rados.rados.0.smithi136.stdout:1941: writing smithi13667081-38 from 2392064 to 2793472 tid 2 2024-09-12T23:44:59.716 INFO:tasks.rados.rados.0.smithi136.stdout:1942: snap_create 2024-09-12T23:44:59.727 INFO:tasks.rados.rados.0.smithi136.stdout:1941: finishing write tid 1 to smithi13667081-38 2024-09-12T23:44:59.728 INFO:tasks.rados.rados.0.smithi136.stdout:1941: finishing write tid 2 to smithi13667081-38 2024-09-12T23:44:59.728 INFO:tasks.rados.rados.0.smithi136.stdout:1941: finishing write tid 3 to smithi13667081-38 2024-09-12T23:44:59.731 INFO:tasks.rados.rados.0.smithi136.stdout:1941: finishing write tid 4 to smithi13667081-38 2024-09-12T23:44:59.731 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 38 v 1207 (ObjNum 634 snap 187 seq_num 634) dirty exists 2024-09-12T23:44:59.731 INFO:tasks.rados.rados.0.smithi136.stdout:1941: left oid 38 (ObjNum 634 snap 187 seq_num 634) 2024-09-12T23:44:59.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:44:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:44:59] "GET /metrics HTTP/1.1" 200 36346 "" "Prometheus/2.43.0" 2024-09-12T23:44:59.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:59 smithi136 bash[76371]: cephadm 2024-09-12T23:44:58.381858+0000 mgr.y (mgr.24661) 157 : cephadm [INF] Upgrade: Updating osd.2 2024-09-12T23:44:59.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:59 smithi136 bash[76371]: audit 2024-09-12T23:44:58.391852+0000 mon.a (mon.0) 484 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:59.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:59 smithi136 bash[76371]: audit 2024-09-12T23:44:58.393388+0000 mon.a (mon.0) 485 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-12T23:44:59.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:59 smithi136 bash[76371]: audit 2024-09-12T23:44:58.394481+0000 mon.a (mon.0) 486 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:44:59.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:59 smithi136 bash[76371]: cephadm 2024-09-12T23:44:58.398474+0000 mgr.y (mgr.24661) 158 : cephadm [INF] Deploying daemon osd.2 on smithi136 2024-09-12T23:44:59.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:59 smithi136 bash[76371]: cluster 2024-09-12T23:44:58.673904+0000 mon.a (mon.0) 487 : cluster [DBG] osdmap e444: 8 total, 8 up, 8 in 2024-09-12T23:44:59.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:59 smithi136 bash[76371]: cluster 2024-09-12T23:44:58.709837+0000 mgr.y (mgr.24661) 159 : cluster [DBG] pgmap v241: 121 pgs: 121 active+clean; 168 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 270 KiB/s rd, 210 KiB/s wr, 9 op/s 2024-09-12T23:44:59.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:44:59 smithi136 bash[76371]: audit 2024-09-12T23:44:58.812513+0000 mon.a (mon.0) 488 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:59.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:59 smithi136 bash[75164]: cephadm 2024-09-12T23:44:58.381858+0000 mgr.y (mgr.24661) 157 : cephadm [INF] Upgrade: Updating osd.2 2024-09-12T23:44:59.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:59 smithi136 bash[75164]: audit 2024-09-12T23:44:58.391852+0000 mon.a (mon.0) 484 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:59.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:59 smithi136 bash[75164]: audit 2024-09-12T23:44:58.393388+0000 mon.a (mon.0) 485 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-12T23:44:59.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:59 smithi136 bash[75164]: audit 2024-09-12T23:44:58.394481+0000 mon.a (mon.0) 486 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:44:59.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:59 smithi136 bash[75164]: cephadm 2024-09-12T23:44:58.398474+0000 mgr.y (mgr.24661) 158 : cephadm [INF] Deploying daemon osd.2 on smithi136 2024-09-12T23:44:59.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:59 smithi136 bash[75164]: cluster 2024-09-12T23:44:58.673904+0000 mon.a (mon.0) 487 : cluster [DBG] osdmap e444: 8 total, 8 up, 8 in 2024-09-12T23:44:59.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:59 smithi136 bash[75164]: cluster 2024-09-12T23:44:58.709837+0000 mgr.y (mgr.24661) 159 : cluster [DBG] pgmap v241: 121 pgs: 121 active+clean; 168 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 270 KiB/s rd, 210 KiB/s wr, 9 op/s 2024-09-12T23:44:59.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:44:59 smithi136 bash[75164]: audit 2024-09-12T23:44:58.812513+0000 mon.a (mon.0) 488 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:59.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:59 smithi181 bash[67587]: cephadm 2024-09-12T23:44:58.381858+0000 mgr.y (mgr.24661) 157 : cephadm [INF] Upgrade: Updating osd.2 2024-09-12T23:44:59.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:59 smithi181 bash[67587]: audit 2024-09-12T23:44:58.391852+0000 mon.a (mon.0) 484 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:44:59.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:59 smithi181 bash[67587]: audit 2024-09-12T23:44:58.393388+0000 mon.a (mon.0) 485 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-12T23:44:59.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:59 smithi181 bash[67587]: audit 2024-09-12T23:44:58.394481+0000 mon.a (mon.0) 486 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:44:59.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:59 smithi181 bash[67587]: cephadm 2024-09-12T23:44:58.398474+0000 mgr.y (mgr.24661) 158 : cephadm [INF] Deploying daemon osd.2 on smithi136 2024-09-12T23:44:59.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:59 smithi181 bash[67587]: cluster 2024-09-12T23:44:58.673904+0000 mon.a (mon.0) 487 : cluster [DBG] osdmap e444: 8 total, 8 up, 8 in 2024-09-12T23:44:59.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:59 smithi181 bash[67587]: cluster 2024-09-12T23:44:58.709837+0000 mgr.y (mgr.24661) 159 : cluster [DBG] pgmap v241: 121 pgs: 121 active+clean; 168 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 270 KiB/s rd, 210 KiB/s wr, 9 op/s 2024-09-12T23:44:59.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:44:59 smithi181 bash[67587]: audit 2024-09-12T23:44:58.812513+0000 mon.a (mon.0) 488 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:00.689 INFO:tasks.rados.rados.0.smithi136.stdout:1941: done (1 left) 2024-09-12T23:45:00.690 INFO:tasks.rados.rados.0.smithi136.stdout:1942: done (0 left) 2024-09-12T23:45:00.690 INFO:tasks.rados.rados.0.smithi136.stdout:1943: setattr oid 29 current snap is 188 2024-09-12T23:45:00.691 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 31 current snap is 188 2024-09-12T23:45:00.691 INFO:tasks.rados.rados.0.smithi136.stdout:1944: seq_num 635 ranges {7266304=638976,7905280=401408} 2024-09-12T23:45:00.745 INFO:tasks.rados.rados.0.smithi136.stdout:1944: writing smithi13667081-31 from 7266304 to 7905280 tid 1 2024-09-12T23:45:00.747 INFO:tasks.rados.rados.0.smithi136.stdout:1944: writing smithi13667081-31 from 7905280 to 8306688 tid 2 2024-09-12T23:45:00.748 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 29 v 1208 (ObjNum 628 snap 185 seq_num 628) dirty exists 2024-09-12T23:45:00.748 INFO:tasks.rados.rados.0.smithi136.stdout:1943: done (1 left) 2024-09-12T23:45:00.748 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 46 current snap is 188 2024-09-12T23:45:00.748 INFO:tasks.rados.rados.0.smithi136.stdout:1945: seq_num 636 ranges {0=655360,655360=401408} 2024-09-12T23:45:00.753 INFO:tasks.rados.rados.0.smithi136.stdout:1945: writing smithi13667081-46 from 0 to 655360 tid 1 2024-09-12T23:45:00.756 INFO:tasks.rados.rados.0.smithi136.stdout:1945: writing smithi13667081-46 from 655360 to 1056768 tid 2 2024-09-12T23:45:00.756 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 25 current snap is 188 2024-09-12T23:45:00.757 INFO:tasks.rados.rados.0.smithi136.stdout:1944: finishing write tid 1 to smithi13667081-31 2024-09-12T23:45:00.757 INFO:tasks.rados.rados.0.smithi136.stdout:1946: seq_num 637 ranges {6569984=671744,7241728=401408} 2024-09-12T23:45:00.799 INFO:tasks.rados.rados.0.smithi136.stdout:1946: writing smithi13667081-25 from 6569984 to 7241728 tid 1 2024-09-12T23:45:00.801 INFO:tasks.rados.rados.0.smithi136.stdout:1946: writing smithi13667081-25 from 7241728 to 7643136 tid 2 2024-09-12T23:45:00.804 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 2 current snap is 188 2024-09-12T23:45:00.804 INFO:tasks.rados.rados.0.smithi136.stdout:1947: seq_num 638 ranges {7618560=688128,8306688=401408} 2024-09-12T23:45:00.847 INFO:tasks.rados.rados.0.smithi136.stdout:1947: writing smithi13667081-2 from 7618560 to 8306688 tid 1 2024-09-12T23:45:00.851 INFO:tasks.rados.rados.0.smithi136.stdout:1947: writing smithi13667081-2 from 8306688 to 8708096 tid 2 2024-09-12T23:45:00.851 INFO:tasks.rados.rados.0.smithi136.stdout:1948: rmattr oid 12 current snap is 188 2024-09-12T23:45:00.851 INFO:tasks.rados.rados.0.smithi136.stdout:1944: finishing write tid 2 to smithi13667081-31 2024-09-12T23:45:00.851 INFO:tasks.rados.rados.0.smithi136.stdout:1949: setattr oid 10 current snap is 188 2024-09-12T23:45:00.851 INFO:tasks.rados.rados.0.smithi136.stdout:1944: finishing write tid 3 to smithi13667081-31 2024-09-12T23:45:00.851 INFO:tasks.rados.rados.0.smithi136.stdout:1945: finishing write tid 1 to smithi13667081-46 2024-09-12T23:45:00.851 INFO:tasks.rados.rados.0.smithi136.stdout:1944: finishing write tid 4 to smithi13667081-31 2024-09-12T23:45:00.851 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 31 v 1211 (ObjNum 635 snap 188 seq_num 635) dirty exists 2024-09-12T23:45:00.852 INFO:tasks.rados.rados.0.smithi136.stdout:1944: left oid 31 (ObjNum 635 snap 188 seq_num 635) 2024-09-12T23:45:00.852 INFO:tasks.rados.rados.0.smithi136.stdout:1945: finishing write tid 2 to smithi13667081-46 2024-09-12T23:45:00.852 INFO:tasks.rados.rados.0.smithi136.stdout:1945: finishing write tid 3 to smithi13667081-46 2024-09-12T23:45:00.852 INFO:tasks.rados.rados.0.smithi136.stdout:1945: finishing write tid 4 to smithi13667081-46 2024-09-12T23:45:00.852 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 46 v 644 (ObjNum 636 snap 188 seq_num 636) dirty exists 2024-09-12T23:45:00.852 INFO:tasks.rados.rados.0.smithi136.stdout:1945: left oid 46 (ObjNum 636 snap 188 seq_num 636) 2024-09-12T23:45:00.852 INFO:tasks.rados.rados.0.smithi136.stdout:1946: finishing write tid 1 to smithi13667081-25 2024-09-12T23:45:00.852 INFO:tasks.rados.rados.0.smithi136.stdout:1946: finishing write tid 2 to smithi13667081-25 2024-09-12T23:45:00.852 INFO:tasks.rados.rados.0.smithi136.stdout:1946: finishing write tid 3 to smithi13667081-25 2024-09-12T23:45:00.853 INFO:tasks.rados.rados.0.smithi136.stdout:1946: finishing write tid 4 to smithi13667081-25 2024-09-12T23:45:00.853 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 25 v 786 (ObjNum 637 snap 188 seq_num 637) dirty exists 2024-09-12T23:45:00.853 INFO:tasks.rados.rados.0.smithi136.stdout:1946: left oid 25 (ObjNum 637 snap 188 seq_num 637) 2024-09-12T23:45:00.853 INFO:tasks.rados.rados.0.smithi136.stdout:1944: done (5 left) 2024-09-12T23:45:00.854 INFO:tasks.rados.rados.0.smithi136.stdout:1945: done (4 left) 2024-09-12T23:45:00.854 INFO:tasks.rados.rados.0.smithi136.stdout:1946: done (3 left) 2024-09-12T23:45:00.854 INFO:tasks.rados.rados.0.smithi136.stdout:1950: rollback oid 40 current snap is 188 2024-09-12T23:45:00.854 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 40 to 187 2024-09-12T23:45:00.854 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 8 current snap is 188 2024-09-12T23:45:00.854 INFO:tasks.rados.rados.0.smithi136.stdout:1951: seq_num 639 ranges {7290880=704512,7995392=401408} 2024-09-12T23:45:00.896 INFO:tasks.rados.rados.0.smithi136.stdout:1951: writing smithi13667081-8 from 7290880 to 7995392 tid 1 2024-09-12T23:45:00.899 INFO:tasks.rados.rados.0.smithi136.stdout:1951: writing smithi13667081-8 from 7995392 to 8396800 tid 2 2024-09-12T23:45:00.899 INFO:tasks.rados.rados.0.smithi136.stdout:1952: rollback oid 3 current snap is 188 2024-09-12T23:45:00.900 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 3 to 186 2024-09-12T23:45:00.900 INFO:tasks.rados.rados.0.smithi136.stdout:1950: finishing rollback tid 1 to smithi13667081-40 2024-09-12T23:45:00.900 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 40 v 0 (ObjNum 33 snap 0 seq_num 4242577872) dirty dne 2024-09-12T23:45:00.900 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 12 v 1212 (ObjNum 512 snap 149 seq_num 512) dirty exists 2024-09-12T23:45:00.900 INFO:tasks.rados.rados.0.smithi136.stdout:1947: finishing write tid 1 to smithi13667081-2 2024-09-12T23:45:00.900 INFO:tasks.rados.rados.0.smithi136.stdout:1947: finishing write tid 2 to smithi13667081-2 2024-09-12T23:45:00.900 INFO:tasks.rados.rados.0.smithi136.stdout:1948: done (5 left) 2024-09-12T23:45:00.900 INFO:tasks.rados.rados.0.smithi136.stdout:1950: done (4 left) 2024-09-12T23:45:00.900 INFO:tasks.rados.rados.0.smithi136.stdout:1953: copy_from oid 39 from oid 22 current snap is 188 2024-09-12T23:45:00.900 INFO:tasks.rados.rados.0.smithi136.stdout:1947: finishing write tid 3 to smithi13667081-2 2024-09-12T23:45:00.900 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 10 v 1081 (ObjNum 612 snap 175 seq_num 612) dirty exists 2024-09-12T23:45:00.900 INFO:tasks.rados.rados.0.smithi136.stdout:1947: finishing write tid 4 to smithi13667081-2 2024-09-12T23:45:00.901 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 2 v 1221 (ObjNum 638 snap 188 seq_num 638) dirty exists 2024-09-12T23:45:00.901 INFO:tasks.rados.rados.0.smithi136.stdout:1947: left oid 2 (ObjNum 638 snap 188 seq_num 638) 2024-09-12T23:45:00.901 INFO:tasks.rados.rados.0.smithi136.stdout:1947: done (4 left) 2024-09-12T23:45:00.901 INFO:tasks.rados.rados.0.smithi136.stdout:1949: done (3 left) 2024-09-12T23:45:00.901 INFO:tasks.rados.rados.0.smithi136.stdout:1954: delete oid 21 current snap is 188 2024-09-12T23:45:00.904 INFO:tasks.rados.rados.0.smithi136.stdout:1952: finishing rollback tid 0 to smithi13667081-3 2024-09-12T23:45:00.905 INFO:tasks.rados.rados.0.smithi136.stdout:1952: finishing rollback tid 1 to smithi13667081-3 2024-09-12T23:45:00.905 INFO:tasks.rados.rados.0.smithi136.stdout:1952: finishing rollback tid 2 to smithi13667081-3 2024-09-12T23:45:00.905 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 3 v 1215 (ObjNum 633 snap 186 seq_num 633) dirty exists 2024-09-12T23:45:00.907 INFO:tasks.rados.rados.0.smithi136.stdout:1952: done (3 left) 2024-09-12T23:45:00.907 INFO:tasks.rados.rados.0.smithi136.stdout:1954: done (2 left) 2024-09-12T23:45:00.907 INFO:tasks.rados.rados.0.smithi136.stdout:1955: rmattr oid 16 current snap is 188 2024-09-12T23:45:00.907 INFO:tasks.rados.rados.0.smithi136.stdout:1956: copy_from oid 48 from oid 47 current snap is 188 2024-09-12T23:45:00.907 INFO:tasks.rados.rados.0.smithi136.stdout:1957: setattr oid 25 current snap is 188 2024-09-12T23:45:00.908 INFO:tasks.rados.rados.0.smithi136.stdout:1951: finishing write tid 1 to smithi13667081-8 2024-09-12T23:45:00.908 INFO:tasks.rados.rados.0.smithi136.stdout:1958: snap_create 2024-09-12T23:45:00.909 INFO:tasks.rados.rados.0.smithi136.stdout:1951: finishing write tid 2 to smithi13667081-8 2024-09-12T23:45:00.909 INFO:tasks.rados.rados.0.smithi136.stdout:1951: finishing write tid 3 to smithi13667081-8 2024-09-12T23:45:00.915 INFO:tasks.rados.rados.0.smithi136.stdout:1951: finishing write tid 4 to smithi13667081-8 2024-09-12T23:45:00.915 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 8 v 791 (ObjNum 639 snap 188 seq_num 639) dirty exists 2024-09-12T23:45:00.915 INFO:tasks.rados.rados.0.smithi136.stdout:1951: left oid 8 (ObjNum 639 snap 188 seq_num 639) 2024-09-12T23:45:00.915 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 25 v 787 (ObjNum 637 snap 188 seq_num 637) dirty exists 2024-09-12T23:45:00.918 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 16 v 1150 (ObjNum 616 snap 179 seq_num 616) dirty exists 2024-09-12T23:45:00.932 INFO:tasks.rados.rados.0.smithi136.stdout:1953: finishing copy_from racing read to smithi13667081-39 2024-09-12T23:45:00.934 INFO:tasks.rados.rados.0.smithi136.stdout:1956: finishing copy_from racing read to smithi13667081-48 2024-09-12T23:45:00.934 INFO:tasks.rados.rados.0.smithi136.stdout:1953: finishing copy_from to smithi13667081-39 2024-09-12T23:45:00.934 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 39 v 810 (ObjNum 548 snap 158 seq_num 548) dirty exists 2024-09-12T23:45:00.934 INFO:tasks.rados.rados.0.smithi136.stdout:1956: finishing copy_from to smithi13667081-48 2024-09-12T23:45:00.934 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 48 v 634 (ObjNum 557 snap 160 seq_num 557) dirty exists 2024-09-12T23:45:01.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:00 smithi136 bash[75164]: cluster 2024-09-12T23:44:59.690667+0000 mon.a (mon.0) 489 : cluster [DBG] osdmap e445: 8 total, 8 up, 8 in 2024-09-12T23:45:01.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:00 smithi136 bash[76371]: cluster 2024-09-12T23:44:59.690667+0000 mon.a (mon.0) 489 : cluster [DBG] osdmap e445: 8 total, 8 up, 8 in 2024-09-12T23:45:01.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:00 smithi181 bash[67587]: cluster 2024-09-12T23:44:59.690667+0000 mon.a (mon.0) 489 : cluster [DBG] osdmap e445: 8 total, 8 up, 8 in 2024-09-12T23:45:01.335 INFO:tasks.rados.rados.0.smithi136.stdout:1951: done (5 left) 2024-09-12T23:45:01.336 INFO:tasks.rados.rados.0.smithi136.stdout:1953: done (4 left) 2024-09-12T23:45:01.336 INFO:tasks.rados.rados.0.smithi136.stdout:1955: done (3 left) 2024-09-12T23:45:01.336 INFO:tasks.rados.rados.0.smithi136.stdout:1956: done (2 left) 2024-09-12T23:45:01.336 INFO:tasks.rados.rados.0.smithi136.stdout:1957: done (1 left) 2024-09-12T23:45:01.336 INFO:tasks.rados.rados.0.smithi136.stdout:1958: done (0 left) 2024-09-12T23:45:01.336 INFO:tasks.rados.rados.0.smithi136.stdout:1959: delete oid 24 current snap is 189 2024-09-12T23:45:01.341 INFO:tasks.rados.rados.0.smithi136.stdout:1959: done (0 left) 2024-09-12T23:45:01.341 INFO:tasks.rados.rados.0.smithi136.stdout:1960: rmattr oid 14 current snap is 189 2024-09-12T23:45:01.341 INFO:tasks.rados.rados.0.smithi136.stdout:1961: delete oid 7 current snap is 189 2024-09-12T23:45:01.350 INFO:tasks.rados.rados.0.smithi136.stdout:1961: done (1 left) 2024-09-12T23:45:01.350 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 29 current snap is 189 2024-09-12T23:45:01.350 INFO:tasks.rados.rados.0.smithi136.stdout:1962: seq_num 643 ranges {4792320=778240,5570560=401408} 2024-09-12T23:45:01.381 INFO:tasks.rados.rados.0.smithi136.stdout:1962: writing smithi13667081-29 from 4792320 to 5570560 tid 1 2024-09-12T23:45:01.383 INFO:tasks.rados.rados.0.smithi136.stdout:1962: writing smithi13667081-29 from 5570560 to 5971968 tid 2 2024-09-12T23:45:01.384 INFO:tasks.rados.rados.0.smithi136.stdout:1963: snap_create 2024-09-12T23:45:01.384 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 14 v 1154 (ObjNum 614 snap 177 seq_num 614) dirty exists 2024-09-12T23:45:01.389 INFO:tasks.rados.rados.0.smithi136.stdout:1962: finishing write tid 1 to smithi13667081-29 2024-09-12T23:45:01.389 INFO:tasks.rados.rados.0.smithi136.stdout:1962: finishing write tid 2 to smithi13667081-29 2024-09-12T23:45:01.389 INFO:tasks.rados.rados.0.smithi136.stdout:1962: finishing write tid 3 to smithi13667081-29 2024-09-12T23:45:01.391 INFO:tasks.rados.rados.0.smithi136.stdout:1962: finishing write tid 4 to smithi13667081-29 2024-09-12T23:45:01.391 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 29 v 1211 (ObjNum 643 snap 189 seq_num 643) dirty exists 2024-09-12T23:45:01.391 INFO:tasks.rados.rados.0.smithi136.stdout:1962: left oid 29 (ObjNum 643 snap 189 seq_num 643) 2024-09-12T23:45:02.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:01 smithi136 bash[75164]: cluster 2024-09-12T23:45:00.693548+0000 mon.a (mon.0) 490 : cluster [DBG] osdmap e446: 8 total, 8 up, 8 in 2024-09-12T23:45:02.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:01 smithi136 bash[75164]: cluster 2024-09-12T23:45:00.710245+0000 mgr.y (mgr.24661) 160 : cluster [DBG] pgmap v244: 121 pgs: 121 active+clean; 174 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 1.5 MiB/s wr, 18 op/s 2024-09-12T23:45:02.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:01 smithi136 bash[75164]: cluster 2024-09-12T23:45:01.338585+0000 mon.a (mon.0) 491 : cluster [DBG] osdmap e447: 8 total, 8 up, 8 in 2024-09-12T23:45:02.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:01 smithi136 bash[76371]: cluster 2024-09-12T23:45:00.693548+0000 mon.a (mon.0) 490 : cluster [DBG] osdmap e446: 8 total, 8 up, 8 in 2024-09-12T23:45:02.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:01 smithi136 bash[76371]: cluster 2024-09-12T23:45:00.710245+0000 mgr.y (mgr.24661) 160 : cluster [DBG] pgmap v244: 121 pgs: 121 active+clean; 174 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 1.5 MiB/s wr, 18 op/s 2024-09-12T23:45:02.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:01 smithi136 bash[76371]: cluster 2024-09-12T23:45:01.338585+0000 mon.a (mon.0) 491 : cluster [DBG] osdmap e447: 8 total, 8 up, 8 in 2024-09-12T23:45:02.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:01 smithi181 bash[67587]: cluster 2024-09-12T23:45:00.693548+0000 mon.a (mon.0) 490 : cluster [DBG] osdmap e446: 8 total, 8 up, 8 in 2024-09-12T23:45:02.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:01 smithi181 bash[67587]: cluster 2024-09-12T23:45:00.710245+0000 mgr.y (mgr.24661) 160 : cluster [DBG] pgmap v244: 121 pgs: 121 active+clean; 174 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 1.5 MiB/s wr, 18 op/s 2024-09-12T23:45:02.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:01 smithi181 bash[67587]: cluster 2024-09-12T23:45:01.338585+0000 mon.a (mon.0) 491 : cluster [DBG] osdmap e447: 8 total, 8 up, 8 in 2024-09-12T23:45:02.333 INFO:tasks.rados.rados.0.smithi136.stdout:1960: done (2 left) 2024-09-12T23:45:02.333 INFO:tasks.rados.rados.0.smithi136.stdout:1962: done (1 left) 2024-09-12T23:45:02.333 INFO:tasks.rados.rados.0.smithi136.stdout:1963: done (0 left) 2024-09-12T23:45:02.333 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 45 current snap is 190 2024-09-12T23:45:02.333 INFO:tasks.rados.rados.0.smithi136.stdout:1964: seq_num 644 ranges {7135232=794624,7929856=401408} 2024-09-12T23:45:02.378 INFO:tasks.rados.rados.0.smithi136.stdout:1964: writing smithi13667081-45 from 7135232 to 7929856 tid 1 2024-09-12T23:45:02.380 INFO:tasks.rados.rados.0.smithi136.stdout:1964: writing smithi13667081-45 from 7929856 to 8331264 tid 2 2024-09-12T23:45:02.380 INFO:tasks.rados.rados.0.smithi136.stdout:1965: snap_remove snap 187 2024-09-12T23:45:02.594 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:45:02 smithi181 bash[60471]: ts=2024-09-12T23:45:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:45:03.352 INFO:tasks.rados.rados.0.smithi136.stdout:1965: done (1 left) 2024-09-12T23:45:03.352 INFO:tasks.rados.rados.0.smithi136.stdout:1966: copy_from oid 25 from oid 40 current snap is 190 2024-09-12T23:45:03.352 INFO:tasks.rados.rados.0.smithi136.stdout:1964: finishing write tid 1 to smithi13667081-45 2024-09-12T23:45:03.352 INFO:tasks.rados.rados.0.smithi136.stdout:1964: finishing write tid 2 to smithi13667081-45 2024-09-12T23:45:03.352 INFO:tasks.rados.rados.0.smithi136.stdout:1964: finishing write tid 3 to smithi13667081-45 2024-09-12T23:45:03.352 INFO:tasks.rados.rados.0.smithi136.stdout:1964: finishing write tid 4 to smithi13667081-45 2024-09-12T23:45:03.353 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 45 v 794 (ObjNum 644 snap 190 seq_num 644) dirty exists 2024-09-12T23:45:03.353 INFO:tasks.rados.rados.0.smithi136.stdout:1964: left oid 45 (ObjNum 644 snap 190 seq_num 644) 2024-09-12T23:45:03.353 INFO:tasks.rados.rados.0.smithi136.stdout:1964: done (1 left) 2024-09-12T23:45:03.353 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 32 current snap is 190 2024-09-12T23:45:03.353 INFO:tasks.rados.rados.0.smithi136.stdout:1967: seq_num 645 ranges {352256=409600,761856=778240,1540096=24576} 2024-09-12T23:45:03.356 INFO:tasks.rados.rados.0.smithi136.stdout:1967: writing smithi13667081-32 from 352256 to 761856 tid 1 2024-09-12T23:45:03.361 INFO:tasks.rados.rados.0.smithi136.stdout:1967: writing smithi13667081-32 from 761856 to 1540096 tid 2 2024-09-12T23:45:03.361 INFO:tasks.rados.rados.0.smithi136.stdout:1967: writing smithi13667081-32 from 1540096 to 1564672 tid 3 2024-09-12T23:45:03.361 INFO:tasks.rados.rados.0.smithi136.stdout:1968: snap_remove snap 186 2024-09-12T23:45:03.456 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 12 23:45:03 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:45:03.456 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:03 smithi136 bash[75164]: cluster 2024-09-12T23:45:02.336955+0000 mon.a (mon.0) 492 : cluster [DBG] osdmap e448: 8 total, 8 up, 8 in 2024-09-12T23:45:03.456 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:03 smithi136 bash[75164]: cluster 2024-09-12T23:45:02.710918+0000 mgr.y (mgr.24661) 161 : cluster [DBG] pgmap v247: 121 pgs: 121 active+clean; 174 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.3 MiB/s wr, 16 op/s 2024-09-12T23:45:03.457 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:03 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:45:03.457 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:03 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:45:03.457 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:03 smithi136 bash[76371]: cluster 2024-09-12T23:45:02.336955+0000 mon.a (mon.0) 492 : cluster [DBG] osdmap e448: 8 total, 8 up, 8 in 2024-09-12T23:45:03.457 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:03 smithi136 bash[76371]: cluster 2024-09-12T23:45:02.710918+0000 mgr.y (mgr.24661) 161 : cluster [DBG] pgmap v247: 121 pgs: 121 active+clean; 174 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.3 MiB/s wr, 16 op/s 2024-09-12T23:45:03.461 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 12 23:45:03 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:45:03.462 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 12 23:45:03 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:45:03.462 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:45:03 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:45:03.462 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:45:03 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:45:03.463 INFO:journalctl@ceph.mds.a.smithi136.stdout:Sep 12 23:45:03 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:45:03.463 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 12 23:45:03 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:45:03.463 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:45:03 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:45:03.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:03 smithi181 bash[67587]: cluster 2024-09-12T23:45:02.336955+0000 mon.a (mon.0) 492 : cluster [DBG] osdmap e448: 8 total, 8 up, 8 in 2024-09-12T23:45:03.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:03 smithi181 bash[67587]: cluster 2024-09-12T23:45:02.710918+0000 mgr.y (mgr.24661) 161 : cluster [DBG] pgmap v247: 121 pgs: 121 active+clean; 174 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.3 MiB/s wr, 16 op/s 2024-09-12T23:45:03.595 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:45:03 smithi181 bash[60471]: ts=2024-09-12T23:45:03.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi136\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:45:03.756 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 12 23:45:03 smithi136 systemd[1]: Stopping Ceph osd.2 for da58a47c-715d-11ef-bceb-c7b262605968... 2024-09-12T23:45:03.756 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 12 23:45:03 smithi136 bash[46418]: debug 2024-09-12T23:45:03.589+0000 7f8512971640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-12T23:45:03.756 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 12 23:45:03 smithi136 bash[46418]: debug 2024-09-12T23:45:03.589+0000 7f8512971640 -1 osd.2 449 *** Got signal Terminated *** 2024-09-12T23:45:03.757 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 12 23:45:03 smithi136 bash[46418]: debug 2024-09-12T23:45:03.589+0000 7f8512971640 -1 osd.2 449 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-12T23:45:04.363 INFO:tasks.rados.rados.0.smithi136.stdout:1966: finishing copy_from to smithi13667081-25 2024-09-12T23:45:04.364 INFO:tasks.rados.rados.0.smithi136.stdout:1966: got expected ENOENT (src dne) 2024-09-12T23:45:04.364 INFO:tasks.rados.rados.0.smithi136.stdout:1966: finishing copy_from racing read to smithi13667081-25 2024-09-12T23:45:04.364 INFO:tasks.rados.rados.0.smithi136.stdout:1967: finishing write tid 1 to smithi13667081-32 2024-09-12T23:45:04.364 INFO:tasks.rados.rados.0.smithi136.stdout:1967: finishing write tid 2 to smithi13667081-32 2024-09-12T23:45:04.364 INFO:tasks.rados.rados.0.smithi136.stdout:1967: finishing write tid 3 to smithi13667081-32 2024-09-12T23:45:04.364 INFO:tasks.rados.rados.0.smithi136.stdout:1967: finishing write tid 4 to smithi13667081-32 2024-09-12T23:45:04.364 INFO:tasks.rados.rados.0.smithi136.stdout:1967: finishing write tid 5 to smithi13667081-32 2024-09-12T23:45:04.364 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 32 v 774 (ObjNum 645 snap 190 seq_num 645) dirty exists 2024-09-12T23:45:04.364 INFO:tasks.rados.rados.0.smithi136.stdout:1967: left oid 32 (ObjNum 645 snap 190 seq_num 645) 2024-09-12T23:45:04.364 INFO:tasks.rados.rados.0.smithi136.stdout:1966: done (2 left) 2024-09-12T23:45:04.364 INFO:tasks.rados.rados.0.smithi136.stdout:1967: done (1 left) 2024-09-12T23:45:04.365 INFO:tasks.rados.rados.0.smithi136.stdout:1968: done (0 left) 2024-09-12T23:45:04.365 INFO:tasks.rados.rados.0.smithi136.stdout:1969: delete oid 8 current snap is 190 2024-09-12T23:45:04.408 INFO:tasks.rados.rados.0.smithi136.stdout:1969: done (0 left) 2024-09-12T23:45:04.408 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 23 current snap is 190 2024-09-12T23:45:04.408 INFO:tasks.rados.rados.0.smithi136.stdout:1970: seq_num 647 ranges {3866624=442368,4308992=589824,4898816=212992} 2024-09-12T23:45:04.437 INFO:tasks.rados.rados.0.smithi136.stdout:1970: writing smithi13667081-23 from 3866624 to 4308992 tid 1 2024-09-12T23:45:04.440 INFO:tasks.rados.rados.0.smithi136.stdout:1970: writing smithi13667081-23 from 4308992 to 4898816 tid 2 2024-09-12T23:45:04.442 INFO:tasks.rados.rados.0.smithi136.stdout:1970: writing smithi13667081-23 from 4898816 to 5111808 tid 3 2024-09-12T23:45:04.442 INFO:tasks.rados.rados.0.smithi136.stdout:1971: snap_create 2024-09-12T23:45:04.449 INFO:tasks.rados.rados.0.smithi136.stdout:1970: finishing write tid 1 to smithi13667081-23 2024-09-12T23:45:04.450 INFO:tasks.rados.rados.0.smithi136.stdout:1970: finishing write tid 2 to smithi13667081-23 2024-09-12T23:45:04.453 INFO:tasks.rados.rados.0.smithi136.stdout:1970: finishing write tid 3 to smithi13667081-23 2024-09-12T23:45:04.453 INFO:tasks.rados.rados.0.smithi136.stdout:1970: finishing write tid 4 to smithi13667081-23 2024-09-12T23:45:04.455 INFO:tasks.rados.rados.0.smithi136.stdout:1970: finishing write tid 5 to smithi13667081-23 2024-09-12T23:45:04.455 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 23 v 1348 (ObjNum 647 snap 190 seq_num 647) dirty exists 2024-09-12T23:45:04.455 INFO:tasks.rados.rados.0.smithi136.stdout:1970: left oid 23 (ObjNum 647 snap 190 seq_num 647) 2024-09-12T23:45:04.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:04 smithi136 bash[76371]: cluster 2024-09-12T23:45:03.349910+0000 mon.a (mon.0) 493 : cluster [DBG] osdmap e449: 8 total, 8 up, 8 in 2024-09-12T23:45:04.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:04 smithi136 bash[76371]: cluster 2024-09-12T23:45:03.595251+0000 mon.a (mon.0) 494 : cluster [INF] osd.2 marked itself down and dead 2024-09-12T23:45:04.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:04 smithi136 bash[75164]: cluster 2024-09-12T23:45:03.349910+0000 mon.a (mon.0) 493 : cluster [DBG] osdmap e449: 8 total, 8 up, 8 in 2024-09-12T23:45:04.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:04 smithi136 bash[75164]: cluster 2024-09-12T23:45:03.595251+0000 mon.a (mon.0) 494 : cluster [INF] osd.2 marked itself down and dead 2024-09-12T23:45:04.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:04 smithi181 bash[67587]: cluster 2024-09-12T23:45:03.349910+0000 mon.a (mon.0) 493 : cluster [DBG] osdmap e449: 8 total, 8 up, 8 in 2024-09-12T23:45:04.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:04 smithi181 bash[67587]: cluster 2024-09-12T23:45:03.595251+0000 mon.a (mon.0) 494 : cluster [INF] osd.2 marked itself down and dead 2024-09-12T23:45:04.845 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:45:04 smithi181 bash[60471]: ts=2024-09-12T23:45:04.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:45:04.845 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:45:04 smithi181 bash[60471]: ts=2024-09-12T23:45:04.457Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:45:05.377 INFO:tasks.rados.rados.0.smithi136.stdout:1970: done (1 left) 2024-09-12T23:45:05.377 INFO:tasks.rados.rados.0.smithi136.stdout:1971: done (0 left) 2024-09-12T23:45:05.377 INFO:tasks.rados.rados.0.smithi136.stdout:1972: read oid 33 snap -1 2024-09-12T23:45:05.377 INFO:tasks.rados.rados.0.smithi136.stdout:1972: expect deleted 2024-09-12T23:45:05.377 INFO:tasks.rados.rados.0.smithi136.stdout:1973: read oid 6 snap -1 2024-09-12T23:45:05.377 INFO:tasks.rados.rados.0.smithi136.stdout:1973: expect (ObjNum 604 snap 174 seq_num 604) 2024-09-12T23:45:05.377 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 2 current snap is 191 2024-09-12T23:45:05.378 INFO:tasks.rados.rados.0.smithi136.stdout:1974: seq_num 648 ranges {8708096=458752,9166848=475136,9641984=327680} 2024-09-12T23:45:05.438 INFO:tasks.rados.rados.0.smithi136.stdout:1974: writing smithi13667081-2 from 8708096 to 9166848 tid 1 2024-09-12T23:45:05.441 INFO:tasks.rados.rados.0.smithi136.stdout:1974: writing smithi13667081-2 from 9166848 to 9641984 tid 2 2024-09-12T23:45:05.444 INFO:tasks.rados.rados.0.smithi136.stdout:1974: writing smithi13667081-2 from 9641984 to 9969664 tid 3 2024-09-12T23:45:05.444 INFO:tasks.rados.rados.0.smithi136.stdout:1975: delete oid 8 current snap is 191 2024-09-12T23:45:05.444 INFO:tasks.rados.rados.0.smithi136.stdout:1973: expect (ObjNum 604 snap 174 seq_num 604) 2024-09-12T23:45:05.629 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:05 smithi136 bash[76371]: cluster 2024-09-12T23:45:04.348332+0000 mon.a (mon.0) 495 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-12T23:45:05.629 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:05 smithi136 bash[76371]: cluster 2024-09-12T23:45:04.365978+0000 mon.a (mon.0) 496 : cluster [DBG] osdmap e450: 8 total, 7 up, 8 in 2024-09-12T23:45:05.629 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:05 smithi136 bash[76371]: cluster 2024-09-12T23:45:04.712603+0000 mgr.y (mgr.24661) 162 : cluster [DBG] pgmap v250: 121 pgs: 24 peering, 4 stale+active+clean, 93 active+clean; 188 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.0 KiB/s rd, 3.6 MiB/s wr, 37 op/s 2024-09-12T23:45:05.629 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:05 smithi136 bash[75164]: cluster 2024-09-12T23:45:04.348332+0000 mon.a (mon.0) 495 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-12T23:45:05.629 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:05 smithi136 bash[75164]: cluster 2024-09-12T23:45:04.365978+0000 mon.a (mon.0) 496 : cluster [DBG] osdmap e450: 8 total, 7 up, 8 in 2024-09-12T23:45:05.629 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:05 smithi136 bash[75164]: cluster 2024-09-12T23:45:04.712603+0000 mgr.y (mgr.24661) 162 : cluster [DBG] pgmap v250: 121 pgs: 24 peering, 4 stale+active+clean, 93 active+clean; 188 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.0 KiB/s rd, 3.6 MiB/s wr, 37 op/s 2024-09-12T23:45:05.658 INFO:tasks.rados.rados.0.smithi136.stdout:1974: finishing write tid 1 to smithi13667081-2 2024-09-12T23:45:05.658 INFO:tasks.rados.rados.0.smithi136.stdout:1974: finishing write tid 2 to smithi13667081-2 2024-09-12T23:45:05.658 INFO:tasks.rados.rados.0.smithi136.stdout:1974: finishing write tid 3 to smithi13667081-2 2024-09-12T23:45:05.658 INFO:tasks.rados.rados.0.smithi136.stdout:1974: finishing write tid 4 to smithi13667081-2 2024-09-12T23:45:05.658 INFO:tasks.rados.rados.0.smithi136.stdout:1972: done (3 left) 2024-09-12T23:45:05.658 INFO:tasks.rados.rados.0.smithi136.stdout:1973: done (2 left) 2024-09-12T23:45:05.658 INFO:tasks.rados.rados.0.smithi136.stdout:1975: done (1 left) 2024-09-12T23:45:05.658 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 40 current snap is 191 2024-09-12T23:45:05.658 INFO:tasks.rados.rados.0.smithi136.stdout:1974: finishing write tid 5 to smithi13667081-2 2024-09-12T23:45:05.658 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 2 v 1225 (ObjNum 648 snap 191 seq_num 648) dirty exists 2024-09-12T23:45:05.659 INFO:tasks.rados.rados.0.smithi136.stdout:1974: left oid 2 (ObjNum 648 snap 191 seq_num 648) 2024-09-12T23:45:05.659 INFO:tasks.rados.rados.0.smithi136.stdout:1976: seq_num 650 ranges {0=491520,491520=647168,1138688=155648} 2024-09-12T23:45:05.661 INFO:tasks.rados.rados.0.smithi136.stdout:1976: writing smithi13667081-40 from 0 to 491520 tid 1 2024-09-12T23:45:05.665 INFO:tasks.rados.rados.0.smithi136.stdout:1976: writing smithi13667081-40 from 491520 to 1138688 tid 2 2024-09-12T23:45:05.667 INFO:tasks.rados.rados.0.smithi136.stdout:1976: writing smithi13667081-40 from 1138688 to 1294336 tid 3 2024-09-12T23:45:05.667 INFO:tasks.rados.rados.0.smithi136.stdout:1974: done (1 left) 2024-09-12T23:45:05.667 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 39 current snap is 191 2024-09-12T23:45:05.667 INFO:tasks.rados.rados.0.smithi136.stdout:1976: finishing write tid 1 to smithi13667081-40 2024-09-12T23:45:05.667 INFO:tasks.rados.rados.0.smithi136.stdout:1977: seq_num 651 ranges {3522560=507904,4030464=532480,4562944=270336} 2024-09-12T23:45:05.689 INFO:tasks.rados.rados.0.smithi136.stdout:1977: writing smithi13667081-39 from 3522560 to 4030464 tid 1 2024-09-12T23:45:05.692 INFO:tasks.rados.rados.0.smithi136.stdout:1977: writing smithi13667081-39 from 4030464 to 4562944 tid 2 2024-09-12T23:45:05.694 INFO:tasks.rados.rados.0.smithi136.stdout:1977: writing smithi13667081-39 from 4562944 to 4833280 tid 3 2024-09-12T23:45:05.694 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 23 current snap is 191 2024-09-12T23:45:05.694 INFO:tasks.rados.rados.0.smithi136.stdout:1976: finishing write tid 2 to smithi13667081-40 2024-09-12T23:45:05.694 INFO:tasks.rados.rados.0.smithi136.stdout:1976: finishing write tid 3 to smithi13667081-40 2024-09-12T23:45:05.694 INFO:tasks.rados.rados.0.smithi136.stdout:1976: finishing write tid 4 to smithi13667081-40 2024-09-12T23:45:05.694 INFO:tasks.rados.rados.0.smithi136.stdout:1976: finishing write tid 5 to smithi13667081-40 2024-09-12T23:45:05.694 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 40 v 1352 (ObjNum 650 snap 191 seq_num 650) dirty exists 2024-09-12T23:45:05.694 INFO:tasks.rados.rados.0.smithi136.stdout:1976: left oid 40 (ObjNum 650 snap 191 seq_num 650) 2024-09-12T23:45:05.695 INFO:tasks.rados.rados.0.smithi136.stdout:1978: seq_num 652 ranges {5111808=524288,5636096=409600,6045696=393216} 2024-09-12T23:45:05.720 INFO:teuthology.orchestra.run.smithi136.stdout:true 2024-09-12T23:45:05.725 INFO:tasks.rados.rados.0.smithi136.stdout:1978: writing smithi13667081-23 from 5111808 to 5636096 tid 1 2024-09-12T23:45:05.727 INFO:tasks.rados.rados.0.smithi136.stdout:1978: writing smithi13667081-23 from 5636096 to 6045696 tid 2 2024-09-12T23:45:05.730 INFO:tasks.rados.rados.0.smithi136.stdout:1978: writing smithi13667081-23 from 6045696 to 6438912 tid 3 2024-09-12T23:45:05.730 INFO:tasks.rados.rados.0.smithi136.stdout:1976: done (2 left) 2024-09-12T23:45:05.730 INFO:tasks.rados.rados.0.smithi136.stdout:1979: delete oid 1 current snap is 191 2024-09-12T23:45:05.730 INFO:tasks.rados.rados.0.smithi136.stdout:1977: finishing write tid 1 to smithi13667081-39 2024-09-12T23:45:05.731 INFO:tasks.rados.rados.0.smithi136.stdout:1977: finishing write tid 2 to smithi13667081-39 2024-09-12T23:45:05.731 INFO:tasks.rados.rados.0.smithi136.stdout:1977: finishing write tid 3 to smithi13667081-39 2024-09-12T23:45:05.731 INFO:tasks.rados.rados.0.smithi136.stdout:1977: finishing write tid 4 to smithi13667081-39 2024-09-12T23:45:05.731 INFO:tasks.rados.rados.0.smithi136.stdout:1977: finishing write tid 5 to smithi13667081-39 2024-09-12T23:45:05.731 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 39 v 814 (ObjNum 651 snap 191 seq_num 651) dirty exists 2024-09-12T23:45:05.731 INFO:tasks.rados.rados.0.smithi136.stdout:1977: left oid 39 (ObjNum 651 snap 191 seq_num 651) 2024-09-12T23:45:05.733 INFO:tasks.rados.rados.0.smithi136.stdout:1978: finishing write tid 1 to smithi13667081-23 2024-09-12T23:45:05.733 INFO:tasks.rados.rados.0.smithi136.stdout:1977: done (2 left) 2024-09-12T23:45:05.733 INFO:tasks.rados.rados.0.smithi136.stdout:1979: done (1 left) 2024-09-12T23:45:05.733 INFO:tasks.rados.rados.0.smithi136.stdout:1980: rmattr oid 7 current snap is 191 2024-09-12T23:45:05.733 INFO:tasks.rados.rados.0.smithi136.stdout:1980: done (1 left) 2024-09-12T23:45:05.733 INFO:tasks.rados.rados.0.smithi136.stdout:1978: finishing write tid 2 to smithi13667081-23 2024-09-12T23:45:05.733 INFO:tasks.rados.rados.0.smithi136.stdout:1981: read oid 29 snap -1 2024-09-12T23:45:05.734 INFO:tasks.rados.rados.0.smithi136.stdout:1981: expect (ObjNum 643 snap 189 seq_num 643) 2024-09-12T23:45:05.734 INFO:tasks.rados.rados.0.smithi136.stdout:1982: rollback oid 8 current snap is 191 2024-09-12T23:45:05.734 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 8 to 180 2024-09-12T23:45:05.734 INFO:tasks.rados.rados.0.smithi136.stdout:1983: snap_create 2024-09-12T23:45:05.734 INFO:tasks.rados.rados.0.smithi136.stdout:1978: finishing write tid 3 to smithi13667081-23 2024-09-12T23:45:05.735 INFO:tasks.rados.rados.0.smithi136.stdout:1978: finishing write tid 4 to smithi13667081-23 2024-09-12T23:45:05.736 INFO:tasks.rados.rados.0.smithi136.stdout:1978: finishing write tid 5 to smithi13667081-23 2024-09-12T23:45:05.736 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 23 v 1357 (ObjNum 652 snap 191 seq_num 652) dirty exists 2024-09-12T23:45:05.736 INFO:tasks.rados.rados.0.smithi136.stdout:1978: left oid 23 (ObjNum 652 snap 191 seq_num 652) 2024-09-12T23:45:05.739 INFO:tasks.rados.rados.0.smithi136.stdout:1982: finishing rollback tid 1 to smithi13667081-8 2024-09-12T23:45:05.740 INFO:tasks.rados.rados.0.smithi136.stdout:1982: finishing rollback tid 2 to smithi13667081-8 2024-09-12T23:45:05.740 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 8 v 797 (ObjNum 605 snap 174 seq_num 605) dirty exists 2024-09-12T23:45:05.770 INFO:tasks.rados.rados.0.smithi136.stdout:1981: expect (ObjNum 643 snap 189 seq_num 643) 2024-09-12T23:45:05.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:05 smithi181 bash[67587]: cluster 2024-09-12T23:45:04.348332+0000 mon.a (mon.0) 495 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-12T23:45:05.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:05 smithi181 bash[67587]: cluster 2024-09-12T23:45:04.365978+0000 mon.a (mon.0) 496 : cluster [DBG] osdmap e450: 8 total, 7 up, 8 in 2024-09-12T23:45:05.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:05 smithi181 bash[67587]: cluster 2024-09-12T23:45:04.712603+0000 mgr.y (mgr.24661) 162 : cluster [DBG] pgmap v250: 121 pgs: 24 peering, 4 stale+active+clean, 93 active+clean; 188 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.0 KiB/s rd, 3.6 MiB/s wr, 37 op/s 2024-09-12T23:45:05.953 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 12 23:45:05 smithi136 bash[88765]: ceph-da58a47c-715d-11ef-bceb-c7b262605968-osd-2 2024-09-12T23:45:05.953 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 12 23:45:05 smithi136 bash[88875]: Error response from daemon: No such container: ceph-da58a47c-715d-11ef-bceb-c7b262605968-osd-2 2024-09-12T23:45:06.025 INFO:teuthology.orchestra.run.smithi136.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-12T23:45:06.025 INFO:teuthology.orchestra.run.smithi136.stdout:alertmanager.a smithi136 *:9093,9094 running (4m) 21s ago 10m 13.1M - 0.25.0 c8568f914cd2 b5197f020eeb 2024-09-12T23:45:06.025 INFO:teuthology.orchestra.run.smithi136.stdout:grafana.a smithi181 *:3000 running (4m) 112s ago 10m 75.3M - 9.4.7 954c08fa6188 256c181be984 2024-09-12T23:45:06.025 INFO:teuthology.orchestra.run.smithi136.stdout:mds.a smithi136 running (11m) 21s ago 11m 14.9M - 17.2.7-1627-gbbdadfa3 214985049bdb 713b9795ad19 2024-09-12T23:45:06.025 INFO:teuthology.orchestra.run.smithi136.stdout:mds.b smithi181 running (11m) 112s ago 11m 17.8M - 17.2.7-1627-gbbdadfa3 214985049bdb ad13179f0868 2024-09-12T23:45:06.025 INFO:teuthology.orchestra.run.smithi136.stdout:mgr.x smithi181 *:8443,9283,8765 running (3m) 112s ago 19m 425M - 18.2.4-837-g7a853b17 f21ba1eea74e cc664dc337f7 2024-09-12T23:45:06.026 INFO:teuthology.orchestra.run.smithi136.stdout:mgr.y smithi136 *:8443,9283,8765 running (4m) 21s ago 21m 487M - 18.2.4-837-g7a853b17 f21ba1eea74e 272f10b42fc3 2024-09-12T23:45:06.026 INFO:teuthology.orchestra.run.smithi136.stdout:mon.a smithi136 running (3m) 21s ago 21m 50.8M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e ad8d21d58ad9 2024-09-12T23:45:06.026 INFO:teuthology.orchestra.run.smithi136.stdout:mon.b smithi181 running (2m) 112s ago 19m 19.5M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e a70ec8315934 2024-09-12T23:45:06.026 INFO:teuthology.orchestra.run.smithi136.stdout:mon.c smithi136 running (2m) 21s ago 19m 42.1M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e 44d1be055a5e 2024-09-12T23:45:06.026 INFO:teuthology.orchestra.run.smithi136.stdout:node-exporter.a smithi136 *:9100 running (4m) 21s ago 10m 8072k - 1.5.0 0da6a335fe13 ccfc37c96620 2024-09-12T23:45:06.026 INFO:teuthology.orchestra.run.smithi136.stdout:node-exporter.b smithi181 *:9100 running (4m) 112s ago 10m 7847k - 1.5.0 0da6a335fe13 80bf0e383230 2024-09-12T23:45:06.026 INFO:teuthology.orchestra.run.smithi136.stdout:osd.0 smithi136 running (53s) 21s ago 18m 208M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 7da21f765721 2024-09-12T23:45:06.026 INFO:teuthology.orchestra.run.smithi136.stdout:osd.1 smithi136 running (25s) 21s ago 17m - 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 5302be0a016b 2024-09-12T23:45:06.026 INFO:teuthology.orchestra.run.smithi136.stdout:osd.2 smithi136 running (16m) 21s ago 16m 272M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 4ab8fd79cdb4 2024-09-12T23:45:06.026 INFO:teuthology.orchestra.run.smithi136.stdout:osd.3 smithi136 running (15m) 21s ago 15m 199M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 9ceb14a27521 2024-09-12T23:45:06.026 INFO:teuthology.orchestra.run.smithi136.stdout:osd.4 smithi181 running (14m) 112s ago 15m 230M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb a35cdca79055 2024-09-12T23:45:06.026 INFO:teuthology.orchestra.run.smithi136.stdout:osd.5 smithi181 running (14m) 112s ago 14m 192M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 8f2e8d494fef 2024-09-12T23:45:06.026 INFO:teuthology.orchestra.run.smithi136.stdout:osd.6 smithi181 running (13m) 112s ago 13m 106M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 9c6bec406408 2024-09-12T23:45:06.027 INFO:teuthology.orchestra.run.smithi136.stdout:osd.7 smithi181 running (12m) 112s ago 12m 174M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb bc0632ae2145 2024-09-12T23:45:06.027 INFO:teuthology.orchestra.run.smithi136.stdout:prometheus.a smithi181 *:9095 running (3m) 112s ago 11m 51.7M - 2.43.0 a07b618ecd1d 3853d003a5cc 2024-09-12T23:45:06.337 INFO:tasks.rados.rados.0.smithi136.stdout:1978: done (3 left) 2024-09-12T23:45:06.337 INFO:tasks.rados.rados.0.smithi136.stdout:1981: done (2 left) 2024-09-12T23:45:06.337 INFO:tasks.rados.rados.0.smithi136.stdout:1982: done (1 left) 2024-09-12T23:45:06.337 INFO:tasks.rados.rados.0.smithi136.stdout:1983: done (0 left) 2024-09-12T23:45:06.337 INFO:tasks.rados.rados.0.smithi136.stdout:1984: snap_remove snap 185 2024-09-12T23:45:06.379 INFO:teuthology.orchestra.run.smithi136.stdout:{ 2024-09-12T23:45:06.379 INFO:teuthology.orchestra.run.smithi136.stdout: "mon": { 2024-09-12T23:45:06.379 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 3 2024-09-12T23:45:06.379 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:45:06.379 INFO:teuthology.orchestra.run.smithi136.stdout: "mgr": { 2024-09-12T23:45:06.379 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 2 2024-09-12T23:45:06.379 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:45:06.380 INFO:teuthology.orchestra.run.smithi136.stdout: "osd": { 2024-09-12T23:45:06.380 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 5, 2024-09-12T23:45:06.380 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 2 2024-09-12T23:45:06.380 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:45:06.380 INFO:teuthology.orchestra.run.smithi136.stdout: "mds": { 2024-09-12T23:45:06.380 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-12T23:45:06.380 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:45:06.380 INFO:teuthology.orchestra.run.smithi136.stdout: "overall": { 2024-09-12T23:45:06.380 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 7, 2024-09-12T23:45:06.380 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 7 2024-09-12T23:45:06.380 INFO:teuthology.orchestra.run.smithi136.stdout: } 2024-09-12T23:45:06.380 INFO:teuthology.orchestra.run.smithi136.stdout:} 2024-09-12T23:45:06.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:06 smithi136 bash[75164]: cluster 2024-09-12T23:45:05.358625+0000 mon.a (mon.0) 497 : cluster [WRN] Health check failed: Reduced data availability: 3 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2024-09-12T23:45:06.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:06 smithi136 bash[75164]: cluster 2024-09-12T23:45:05.377378+0000 mon.a (mon.0) 498 : cluster [DBG] osdmap e451: 8 total, 7 up, 8 in 2024-09-12T23:45:06.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:06 smithi136 bash[75164]: audit 2024-09-12T23:45:05.712259+0000 mgr.y (mgr.24661) 163 : audit [DBG] from='client.44180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:45:06.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:06 smithi136 bash[75164]: cluster 2024-09-12T23:45:06.340951+0000 mon.a (mon.0) 499 : cluster [DBG] osdmap e452: 8 total, 7 up, 8 in 2024-09-12T23:45:06.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:06 smithi136 bash[76371]: cluster 2024-09-12T23:45:05.358625+0000 mon.a (mon.0) 497 : cluster [WRN] Health check failed: Reduced data availability: 3 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2024-09-12T23:45:06.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:06 smithi136 bash[76371]: cluster 2024-09-12T23:45:05.377378+0000 mon.a (mon.0) 498 : cluster [DBG] osdmap e451: 8 total, 7 up, 8 in 2024-09-12T23:45:06.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:06 smithi136 bash[76371]: audit 2024-09-12T23:45:05.712259+0000 mgr.y (mgr.24661) 163 : audit [DBG] from='client.44180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:45:06.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:06 smithi136 bash[76371]: cluster 2024-09-12T23:45:06.340951+0000 mon.a (mon.0) 499 : cluster [DBG] osdmap e452: 8 total, 7 up, 8 in 2024-09-12T23:45:06.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:06 smithi181 bash[67587]: cluster 2024-09-12T23:45:05.358625+0000 mon.a (mon.0) 497 : cluster [WRN] Health check failed: Reduced data availability: 3 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2024-09-12T23:45:06.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:06 smithi181 bash[67587]: cluster 2024-09-12T23:45:05.377378+0000 mon.a (mon.0) 498 : cluster [DBG] osdmap e451: 8 total, 7 up, 8 in 2024-09-12T23:45:06.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:06 smithi181 bash[67587]: audit 2024-09-12T23:45:05.712259+0000 mgr.y (mgr.24661) 163 : audit [DBG] from='client.44180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:45:06.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:06 smithi181 bash[67587]: cluster 2024-09-12T23:45:06.340951+0000 mon.a (mon.0) 499 : cluster [DBG] osdmap e452: 8 total, 7 up, 8 in 2024-09-12T23:45:07.343 INFO:tasks.rados.rados.0.smithi136.stdout:1984: done (0 left) 2024-09-12T23:45:07.343 INFO:tasks.rados.rados.0.smithi136.stdout:1985: rmattr oid 33 current snap is 192 2024-09-12T23:45:07.343 INFO:tasks.rados.rados.0.smithi136.stdout:1985: done (0 left) 2024-09-12T23:45:07.343 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 28 current snap is 192 2024-09-12T23:45:07.343 INFO:tasks.rados.rados.0.smithi136.stdout:1986: seq_num 654 ranges {6160384=557056,6717440=630784,7348224=172032} 2024-09-12T23:45:07.372 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:07 smithi136 bash[76371]: audit 2024-09-12T23:45:06.012327+0000 mgr.y (mgr.24661) 164 : audit [DBG] from='client.34272 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:45:07.372 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:07 smithi136 bash[76371]: audit 2024-09-12T23:45:06.383393+0000 mon.a (mon.0) 500 : audit [DBG] from='client.? 172.21.15.136:0/3797025021' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:45:07.372 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:07 smithi136 bash[76371]: cluster 2024-09-12T23:45:06.713644+0000 mgr.y (mgr.24661) 165 : cluster [DBG] pgmap v253: 121 pgs: 7 active+undersized, 24 peering, 3 stale+active+clean, 1 active+undersized+degraded, 86 active+clean; 188 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.0 KiB/s rd, 3.6 MiB/s wr, 36 op/s; 4/324 objects degraded (1.235%) 2024-09-12T23:45:07.372 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:07 smithi136 bash[76371]: cluster 2024-09-12T23:45:07.344807+0000 mon.a (mon.0) 501 : cluster [DBG] osdmap e453: 8 total, 7 up, 8 in 2024-09-12T23:45:07.372 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:07 smithi136 bash[75164]: audit 2024-09-12T23:45:06.012327+0000 mgr.y (mgr.24661) 164 : audit [DBG] from='client.34272 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:45:07.372 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:07 smithi136 bash[75164]: audit 2024-09-12T23:45:06.383393+0000 mon.a (mon.0) 500 : audit [DBG] from='client.? 172.21.15.136:0/3797025021' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:45:07.372 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:07 smithi136 bash[75164]: cluster 2024-09-12T23:45:06.713644+0000 mgr.y (mgr.24661) 165 : cluster [DBG] pgmap v253: 121 pgs: 7 active+undersized, 24 peering, 3 stale+active+clean, 1 active+undersized+degraded, 86 active+clean; 188 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.0 KiB/s rd, 3.6 MiB/s wr, 36 op/s; 4/324 objects degraded (1.235%) 2024-09-12T23:45:07.373 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:07 smithi136 bash[75164]: cluster 2024-09-12T23:45:07.344807+0000 mon.a (mon.0) 501 : cluster [DBG] osdmap e453: 8 total, 7 up, 8 in 2024-09-12T23:45:07.380 INFO:tasks.rados.rados.0.smithi136.stdout:1986: writing smithi13667081-28 from 6160384 to 6717440 tid 1 2024-09-12T23:45:07.383 INFO:tasks.rados.rados.0.smithi136.stdout:1986: writing smithi13667081-28 from 6717440 to 7348224 tid 2 2024-09-12T23:45:07.385 INFO:tasks.rados.rados.0.smithi136.stdout:1986: writing smithi13667081-28 from 7348224 to 7520256 tid 3 2024-09-12T23:45:07.385 INFO:tasks.rados.rados.0.smithi136.stdout:1987: snap_remove snap 182 2024-09-12T23:45:07.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:07 smithi181 bash[67587]: audit 2024-09-12T23:45:06.012327+0000 mgr.y (mgr.24661) 164 : audit [DBG] from='client.34272 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:45:07.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:07 smithi181 bash[67587]: audit 2024-09-12T23:45:06.383393+0000 mon.a (mon.0) 500 : audit [DBG] from='client.? 172.21.15.136:0/3797025021' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:45:07.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:07 smithi181 bash[67587]: cluster 2024-09-12T23:45:06.713644+0000 mgr.y (mgr.24661) 165 : cluster [DBG] pgmap v253: 121 pgs: 7 active+undersized, 24 peering, 3 stale+active+clean, 1 active+undersized+degraded, 86 active+clean; 188 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.0 KiB/s rd, 3.6 MiB/s wr, 36 op/s; 4/324 objects degraded (1.235%) 2024-09-12T23:45:07.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:07 smithi181 bash[67587]: cluster 2024-09-12T23:45:07.344807+0000 mon.a (mon.0) 501 : cluster [DBG] osdmap e453: 8 total, 7 up, 8 in 2024-09-12T23:45:08.329 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 12 23:45:08 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:45:08.329 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 12 23:45:08 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:45:08.330 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:45:08 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:45:08.330 INFO:journalctl@ceph.mds.a.smithi136.stdout:Sep 12 23:45:08 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:45:08.330 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:08 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:45:08.331 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:08 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:45:08.331 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 12 23:45:08 smithi136 systemd[1]: ceph-da58a47c-715d-11ef-bceb-c7b262605968@osd.2.service: Deactivated successfully. 2024-09-12T23:45:08.331 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 12 23:45:08 smithi136 systemd[1]: Stopped Ceph osd.2 for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:45:08.331 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 12 23:45:08 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:45:08.331 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 12 23:45:08 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:45:08.332 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:45:08 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:45:08.332 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:45:08 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:45:08.372 INFO:tasks.rados.rados.0.smithi136.stdout:1987: done (1 left) 2024-09-12T23:45:08.372 INFO:tasks.rados.rados.0.smithi136.stdout:1988: delete oid 41 current snap is 192 2024-09-12T23:45:08.372 INFO:tasks.rados.rados.0.smithi136.stdout:1986: finishing write tid 1 to smithi13667081-28 2024-09-12T23:45:08.372 INFO:tasks.rados.rados.0.smithi136.stdout:1986: finishing write tid 2 to smithi13667081-28 2024-09-12T23:45:08.372 INFO:tasks.rados.rados.0.smithi136.stdout:1986: finishing write tid 3 to smithi13667081-28 2024-09-12T23:45:08.372 INFO:tasks.rados.rados.0.smithi136.stdout:1986: finishing write tid 4 to smithi13667081-28 2024-09-12T23:45:08.372 INFO:tasks.rados.rados.0.smithi136.stdout:1986: finishing write tid 5 to smithi13667081-28 2024-09-12T23:45:08.372 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 28 v 1165 (ObjNum 654 snap 192 seq_num 654) dirty exists 2024-09-12T23:45:08.373 INFO:tasks.rados.rados.0.smithi136.stdout:1986: left oid 28 (ObjNum 654 snap 192 seq_num 654) 2024-09-12T23:45:08.382 INFO:tasks.rados.rados.0.smithi136.stdout:1986: done (1 left) 2024-09-12T23:45:08.382 INFO:tasks.rados.rados.0.smithi136.stdout:1988: done (0 left) 2024-09-12T23:45:08.382 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 6 current snap is 192 2024-09-12T23:45:08.382 INFO:tasks.rados.rados.0.smithi136.stdout:1989: seq_num 656 ranges {4530176=589824,5120000=802816} 2024-09-12T23:45:08.411 INFO:tasks.rados.rados.0.smithi136.stdout:1989: writing smithi13667081-6 from 4530176 to 5120000 tid 1 2024-09-12T23:45:08.416 INFO:tasks.rados.rados.0.smithi136.stdout:1989: writing smithi13667081-6 from 5120000 to 5922816 tid 2 2024-09-12T23:45:08.416 INFO:tasks.rados.rados.0.smithi136.stdout:1990: snap_create 2024-09-12T23:45:08.416 INFO:tasks.rados.rados.0.smithi136.stdout:1989: finishing write tid 1 to smithi13667081-6 2024-09-12T23:45:08.419 INFO:tasks.rados.rados.0.smithi136.stdout:1989: finishing write tid 2 to smithi13667081-6 2024-09-12T23:45:08.419 INFO:tasks.rados.rados.0.smithi136.stdout:1989: finishing write tid 3 to smithi13667081-6 2024-09-12T23:45:08.420 INFO:tasks.rados.rados.0.smithi136.stdout:1989: finishing write tid 4 to smithi13667081-6 2024-09-12T23:45:08.420 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 6 v 1214 (ObjNum 656 snap 192 seq_num 656) dirty exists 2024-09-12T23:45:08.420 INFO:tasks.rados.rados.0.smithi136.stdout:1989: left oid 6 (ObjNum 656 snap 192 seq_num 656) 2024-09-12T23:45:08.658 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:08 smithi181 bash[67587]: cluster 2024-09-12T23:45:07.353377+0000 mon.a (mon.0) 502 : cluster [WRN] Health check failed: Degraded data redundancy: 4/324 objects degraded (1.235%), 1 pg degraded (PG_DEGRADED) 2024-09-12T23:45:08.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:08 smithi136 bash[75164]: cluster 2024-09-12T23:45:07.353377+0000 mon.a (mon.0) 502 : cluster [WRN] Health check failed: Degraded data redundancy: 4/324 objects degraded (1.235%), 1 pg degraded (PG_DEGRADED) 2024-09-12T23:45:08.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:08 smithi136 bash[76371]: cluster 2024-09-12T23:45:07.353377+0000 mon.a (mon.0) 502 : cluster [WRN] Health check failed: Degraded data redundancy: 4/324 objects degraded (1.235%), 1 pg degraded (PG_DEGRADED) 2024-09-12T23:45:08.756 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 12 23:45:08 smithi136 systemd[1]: Started Ceph osd.2 for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:45:09.094 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:45:08 smithi181 bash[60471]: ts=2024-09-12T23:45:08.662Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:45:09.397 INFO:tasks.rados.rados.0.smithi136.stdout:1989: done (1 left) 2024-09-12T23:45:09.397 INFO:tasks.rados.rados.0.smithi136.stdout:1990: done (0 left) 2024-09-12T23:45:09.397 INFO:tasks.rados.rados.0.smithi136.stdout:1991: rmattr oid 25 current snap is 193 2024-09-12T23:45:09.397 INFO:tasks.rados.rados.0.smithi136.stdout:1992: copy_from oid 11 from oid 23 current snap is 193 2024-09-12T23:45:09.397 INFO:tasks.rados.rados.0.smithi136.stdout:1993: rollback oid 39 current snap is 193 2024-09-12T23:45:09.397 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 39 to 192 2024-09-12T23:45:09.397 INFO:tasks.rados.rados.0.smithi136.stdout:1994: read oid 18 snap -1 2024-09-12T23:45:09.397 INFO:tasks.rados.rados.0.smithi136.stdout:1994: expect (ObjNum 578 snap 169 seq_num 578) 2024-09-12T23:45:09.397 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 48 current snap is 193 2024-09-12T23:45:09.397 INFO:tasks.rados.rados.0.smithi136.stdout:1995: seq_num 657 ranges {2646016=606208,3252224=688128,3940352=114688} 2024-09-12T23:45:09.420 INFO:tasks.rados.rados.0.smithi136.stdout:1995: writing smithi13667081-48 from 2646016 to 3252224 tid 1 2024-09-12T23:45:09.425 INFO:tasks.rados.rados.0.smithi136.stdout:1995: writing smithi13667081-48 from 3252224 to 3940352 tid 2 2024-09-12T23:45:09.426 INFO:tasks.rados.rados.0.smithi136.stdout:1995: writing smithi13667081-48 from 3940352 to 4055040 tid 3 2024-09-12T23:45:09.426 INFO:tasks.rados.rados.0.smithi136.stdout:1996: setattr oid 14 current snap is 193 2024-09-12T23:45:09.427 INFO:tasks.rados.rados.0.smithi136.stdout:1997: snap_remove snap 190 2024-09-12T23:45:09.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:45:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:45:09] "GET /metrics HTTP/1.1" 200 36399 "" "Prometheus/2.43.0" 2024-09-12T23:45:09.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:09 smithi136 bash[75164]: audit 2024-09-12T23:45:08.367681+0000 mon.a (mon.0) 503 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:09.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:09 smithi136 bash[75164]: cluster 2024-09-12T23:45:08.376005+0000 mon.a (mon.0) 504 : cluster [DBG] osdmap e454: 8 total, 7 up, 8 in 2024-09-12T23:45:09.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:09 smithi136 bash[75164]: audit 2024-09-12T23:45:08.385819+0000 mon.a (mon.0) 505 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:09.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:09 smithi136 bash[75164]: audit 2024-09-12T23:45:08.392904+0000 mon.a (mon.0) 506 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:45:09.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:09 smithi136 bash[75164]: cluster 2024-09-12T23:45:08.714486+0000 mgr.y (mgr.24661) 166 : cluster [DBG] pgmap v256: 121 pgs: 12 active+undersized, 24 peering, 3 active+undersized+degraded, 82 active+clean; 194 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.5 MiB/s wr, 5 op/s; 13/339 objects degraded (3.835%) 2024-09-12T23:45:09.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:09 smithi136 bash[75164]: audit 2024-09-12T23:45:08.750867+0000 mon.a (mon.0) 507 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:09.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:09 smithi136 bash[75164]: audit 2024-09-12T23:45:08.752639+0000 mon.a (mon.0) 508 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:45:09.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:09 smithi136 bash[76371]: audit 2024-09-12T23:45:08.367681+0000 mon.a (mon.0) 503 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:09.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:09 smithi136 bash[76371]: cluster 2024-09-12T23:45:08.376005+0000 mon.a (mon.0) 504 : cluster [DBG] osdmap e454: 8 total, 7 up, 8 in 2024-09-12T23:45:09.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:09 smithi136 bash[76371]: audit 2024-09-12T23:45:08.385819+0000 mon.a (mon.0) 505 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:09.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:09 smithi136 bash[76371]: audit 2024-09-12T23:45:08.392904+0000 mon.a (mon.0) 506 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:45:09.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:09 smithi136 bash[76371]: cluster 2024-09-12T23:45:08.714486+0000 mgr.y (mgr.24661) 166 : cluster [DBG] pgmap v256: 121 pgs: 12 active+undersized, 24 peering, 3 active+undersized+degraded, 82 active+clean; 194 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.5 MiB/s wr, 5 op/s; 13/339 objects degraded (3.835%) 2024-09-12T23:45:09.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:09 smithi136 bash[76371]: audit 2024-09-12T23:45:08.750867+0000 mon.a (mon.0) 507 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:09.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:09 smithi136 bash[76371]: audit 2024-09-12T23:45:08.752639+0000 mon.a (mon.0) 508 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:45:09.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:09 smithi181 bash[67587]: audit 2024-09-12T23:45:08.367681+0000 mon.a (mon.0) 503 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:09.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:09 smithi181 bash[67587]: cluster 2024-09-12T23:45:08.376005+0000 mon.a (mon.0) 504 : cluster [DBG] osdmap e454: 8 total, 7 up, 8 in 2024-09-12T23:45:09.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:09 smithi181 bash[67587]: audit 2024-09-12T23:45:08.385819+0000 mon.a (mon.0) 505 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:09.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:09 smithi181 bash[67587]: audit 2024-09-12T23:45:08.392904+0000 mon.a (mon.0) 506 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:45:09.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:09 smithi181 bash[67587]: cluster 2024-09-12T23:45:08.714486+0000 mgr.y (mgr.24661) 166 : cluster [DBG] pgmap v256: 121 pgs: 12 active+undersized, 24 peering, 3 active+undersized+degraded, 82 active+clean; 194 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.5 MiB/s wr, 5 op/s; 13/339 objects degraded (3.835%) 2024-09-12T23:45:09.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:09 smithi181 bash[67587]: audit 2024-09-12T23:45:08.750867+0000 mon.a (mon.0) 507 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:09.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:09 smithi181 bash[67587]: audit 2024-09-12T23:45:08.752639+0000 mon.a (mon.0) 508 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:45:10.256 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 12 23:45:09 smithi136 bash[89045]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-12T23:45:10.256 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 12 23:45:09 smithi136 bash[89045]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-12T23:45:10.256 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 12 23:45:10 smithi136 bash[89045]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-12T23:45:10.257 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 12 23:45:10 smithi136 bash[89045]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-12T23:45:10.257 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 12 23:45:10 smithi136 bash[89045]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-12T23:45:10.257 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 12 23:45:10 smithi136 bash[89045]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-12T23:45:10.257 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 12 23:45:10 smithi136 bash[89045]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-12T23:45:10.401 INFO:tasks.rados.rados.0.smithi136.stdout:1997: done (6 left) 2024-09-12T23:45:10.401 INFO:tasks.rados.rados.0.smithi136.stdout:1998: read oid 21 snap -1 2024-09-12T23:45:10.401 INFO:tasks.rados.rados.0.smithi136.stdout:1998: expect deleted 2024-09-12T23:45:10.401 INFO:tasks.rados.rados.0.smithi136.stdout:1999: copy_from oid 12 from oid 29 current snap is 193 2024-09-12T23:45:10.401 INFO:tasks.rados.rados.0.smithi136.stdout:1995: finishing write tid 1 to smithi13667081-48 2024-09-12T23:45:10.401 INFO:tasks.rados.rados.0.smithi136.stdout:1995: finishing write tid 2 to smithi13667081-48 2024-09-12T23:45:10.402 INFO:tasks.rados.rados.0.smithi136.stdout:1994: expect (ObjNum 578 snap 169 seq_num 578) 2024-09-12T23:45:10.661 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 25 v 815 (ObjNum 637 snap 188 seq_num 637) dirty exists 2024-09-12T23:45:10.662 INFO:tasks.rados.rados.0.smithi136.stdout:1993: finishing rollback tid 0 to smithi13667081-39 2024-09-12T23:45:10.662 INFO:tasks.rados.rados.0.smithi136.stdout:1993: finishing rollback tid 1 to smithi13667081-39 2024-09-12T23:45:10.662 INFO:tasks.rados.rados.0.smithi136.stdout:1993: finishing rollback tid 2 to smithi13667081-39 2024-09-12T23:45:10.662 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 39 v 818 (ObjNum 651 snap 191 seq_num 651) dirty exists 2024-09-12T23:45:10.662 INFO:tasks.rados.rados.0.smithi136.stdout:1995: finishing write tid 3 to smithi13667081-48 2024-09-12T23:45:10.662 INFO:tasks.rados.rados.0.smithi136.stdout:1995: finishing write tid 4 to smithi13667081-48 2024-09-12T23:45:10.662 INFO:tasks.rados.rados.0.smithi136.stdout:1995: finishing write tid 5 to smithi13667081-48 2024-09-12T23:45:10.662 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 48 v 1170 (ObjNum 657 snap 193 seq_num 657) dirty exists 2024-09-12T23:45:10.662 INFO:tasks.rados.rados.0.smithi136.stdout:1995: left oid 48 (ObjNum 657 snap 193 seq_num 657) 2024-09-12T23:45:10.662 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 14 v 1172 (ObjNum 614 snap 177 seq_num 614) dirty exists 2024-09-12T23:45:10.662 INFO:tasks.rados.rados.0.smithi136.stdout:1992: finishing copy_from racing read to smithi13667081-11 2024-09-12T23:45:10.662 INFO:tasks.rados.rados.0.smithi136.stdout:1992: finishing copy_from to smithi13667081-11 2024-09-12T23:45:10.663 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 11 v 1357 (ObjNum 652 snap 191 seq_num 652) dirty exists 2024-09-12T23:45:10.663 INFO:tasks.rados.rados.0.smithi136.stdout:1999: finishing copy_from racing read to smithi13667081-12 2024-09-12T23:45:10.663 INFO:tasks.rados.rados.0.smithi136.stdout:1999: finishing copy_from to smithi13667081-12 2024-09-12T23:45:10.663 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 12 v 1211 (ObjNum 643 snap 189 seq_num 643) dirty exists 2024-09-12T23:45:10.663 INFO:tasks.rados.rados.0.smithi136.stdout:1991: done (7 left) 2024-09-12T23:45:10.663 INFO:tasks.rados.rados.0.smithi136.stdout:1992: done (6 left) 2024-09-12T23:45:10.663 INFO:tasks.rados.rados.0.smithi136.stdout:1993: done (5 left) 2024-09-12T23:45:10.663 INFO:tasks.rados.rados.0.smithi136.stdout:1994: done (4 left) 2024-09-12T23:45:10.663 INFO:tasks.rados.rados.0.smithi136.stdout:1995: done (3 left) 2024-09-12T23:45:10.663 INFO:tasks.rados.rados.0.smithi136.stdout:1996: done (2 left) 2024-09-12T23:45:10.663 INFO:tasks.rados.rados.0.smithi136.stdout:1998: done (1 left) 2024-09-12T23:45:10.663 INFO:tasks.rados.rados.0.smithi136.stdout:1999: done (0 left) 2024-09-12T23:45:10.663 INFO:tasks.rados.rados.0.smithi136.stdout:2000: read oid 28 snap -1 2024-09-12T23:45:10.663 INFO:tasks.rados.rados.0.smithi136.stdout:2000: expect (ObjNum 654 snap 192 seq_num 654) 2024-09-12T23:45:10.664 INFO:tasks.rados.rados.0.smithi136.stdout:2001: snap_create 2024-09-12T23:45:10.715 INFO:tasks.rados.rados.0.smithi136.stdout:2000: expect (ObjNum 654 snap 192 seq_num 654) 2024-09-12T23:45:10.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:10 smithi136 bash[75164]: cluster 2024-09-12T23:45:09.399154+0000 mon.a (mon.0) 509 : cluster [DBG] osdmap e455: 8 total, 7 up, 8 in 2024-09-12T23:45:10.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:10 smithi136 bash[76371]: cluster 2024-09-12T23:45:09.399154+0000 mon.a (mon.0) 509 : cluster [DBG] osdmap e455: 8 total, 7 up, 8 in 2024-09-12T23:45:10.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:10 smithi181 bash[67587]: cluster 2024-09-12T23:45:09.399154+0000 mon.a (mon.0) 509 : cluster [DBG] osdmap e455: 8 total, 7 up, 8 in 2024-09-12T23:45:11.334 INFO:tasks.rados.rados.0.smithi136.stdout:2000: done (1 left) 2024-09-12T23:45:11.335 INFO:tasks.rados.rados.0.smithi136.stdout:2001: done (0 left) 2024-09-12T23:45:11.335 INFO:tasks.rados.rados.0.smithi136.stdout:2002: delete oid 50 current snap is 194 2024-09-12T23:45:11.343 INFO:tasks.rados.rados.0.smithi136.stdout:2002: done (0 left) 2024-09-12T23:45:11.343 INFO:tasks.rados.rados.0.smithi136.stdout:2003: rmattr oid 47 current snap is 194 2024-09-12T23:45:11.344 INFO:tasks.rados.rados.0.smithi136.stdout:2003: done (0 left) 2024-09-12T23:45:11.344 INFO:tasks.rados.rados.0.smithi136.stdout:2004: delete oid 50 current snap is 194 2024-09-12T23:45:11.346 INFO:tasks.rados.rados.0.smithi136.stdout:2004: done (0 left) 2024-09-12T23:45:11.346 INFO:tasks.rados.rados.0.smithi136.stdout:2005: snap_remove snap 179 2024-09-12T23:45:11.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:11 smithi136 bash[75164]: cluster 2024-09-12T23:45:10.402199+0000 mon.a (mon.0) 510 : cluster [DBG] osdmap e456: 8 total, 7 up, 8 in 2024-09-12T23:45:11.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:11 smithi136 bash[75164]: cluster 2024-09-12T23:45:10.715603+0000 mgr.y (mgr.24661) 167 : cluster [DBG] pgmap v259: 121 pgs: 28 active+undersized, 11 active+undersized+degraded, 82 active+clean; 206 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 2.8 MiB/s wr, 21 op/s; 56/357 objects degraded (15.686%) 2024-09-12T23:45:11.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:11 smithi136 bash[75164]: cluster 2024-09-12T23:45:11.338721+0000 mon.a (mon.0) 511 : cluster [DBG] osdmap e457: 8 total, 7 up, 8 in 2024-09-12T23:45:11.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:11 smithi136 bash[76371]: cluster 2024-09-12T23:45:10.402199+0000 mon.a (mon.0) 510 : cluster [DBG] osdmap e456: 8 total, 7 up, 8 in 2024-09-12T23:45:11.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:11 smithi136 bash[76371]: cluster 2024-09-12T23:45:10.715603+0000 mgr.y (mgr.24661) 167 : cluster [DBG] pgmap v259: 121 pgs: 28 active+undersized, 11 active+undersized+degraded, 82 active+clean; 206 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 2.8 MiB/s wr, 21 op/s; 56/357 objects degraded (15.686%) 2024-09-12T23:45:11.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:11 smithi136 bash[76371]: cluster 2024-09-12T23:45:11.338721+0000 mon.a (mon.0) 511 : cluster [DBG] osdmap e457: 8 total, 7 up, 8 in 2024-09-12T23:45:11.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:11 smithi181 bash[67587]: cluster 2024-09-12T23:45:10.402199+0000 mon.a (mon.0) 510 : cluster [DBG] osdmap e456: 8 total, 7 up, 8 in 2024-09-12T23:45:11.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:11 smithi181 bash[67587]: cluster 2024-09-12T23:45:10.715603+0000 mgr.y (mgr.24661) 167 : cluster [DBG] pgmap v259: 121 pgs: 28 active+undersized, 11 active+undersized+degraded, 82 active+clean; 206 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 2.8 MiB/s wr, 21 op/s; 56/357 objects degraded (15.686%) 2024-09-12T23:45:11.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:11 smithi181 bash[67587]: cluster 2024-09-12T23:45:11.338721+0000 mon.a (mon.0) 511 : cluster [DBG] osdmap e457: 8 total, 7 up, 8 in 2024-09-12T23:45:12.256 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 12 23:45:12 smithi136 bash[89273]: debug 2024-09-12T23:45:12.037+0000 7faed5afa640 1 -- 172.21.15.136:0/2733154528 <== mon.0 v2:172.21.15.136:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55ffd6a911e0 con 0x55ffd7810000 2024-09-12T23:45:12.340 INFO:tasks.rados.rados.0.smithi136.stdout:2005: done (0 left) 2024-09-12T23:45:12.340 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 23 current snap is 194 2024-09-12T23:45:12.340 INFO:tasks.rados.rados.0.smithi136.stdout:2006: seq_num 660 ranges {6438912=663552,7102464=737280,7839744=65536} 2024-09-12T23:45:12.381 INFO:tasks.rados.rados.0.smithi136.stdout:2006: writing smithi13667081-23 from 6438912 to 7102464 tid 1 2024-09-12T23:45:12.385 INFO:tasks.rados.rados.0.smithi136.stdout:2006: writing smithi13667081-23 from 7102464 to 7839744 tid 2 2024-09-12T23:45:12.385 INFO:tasks.rados.rados.0.smithi136.stdout:2006: writing smithi13667081-23 from 7839744 to 7905280 tid 3 2024-09-12T23:45:12.386 INFO:tasks.rados.rados.0.smithi136.stdout:2007: read oid 15 snap -1 2024-09-12T23:45:12.386 INFO:tasks.rados.rados.0.smithi136.stdout:2007: expect (ObjNum 631 snap 185 seq_num 631) 2024-09-12T23:45:12.386 INFO:tasks.rados.rados.0.smithi136.stdout:2008: setattr oid 36 current snap is 194 2024-09-12T23:45:12.387 INFO:tasks.rados.rados.0.smithi136.stdout:2009: snap_create 2024-09-12T23:45:12.390 INFO:tasks.rados.rados.0.smithi136.stdout:2006: finishing write tid 1 to smithi13667081-23 2024-09-12T23:45:12.391 INFO:tasks.rados.rados.0.smithi136.stdout:2006: finishing write tid 2 to smithi13667081-23 2024-09-12T23:45:12.392 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 36 v 798 (ObjNum 610 snap 175 seq_num 610) dirty exists 2024-09-12T23:45:12.392 INFO:tasks.rados.rados.0.smithi136.stdout:2006: finishing write tid 3 to smithi13667081-23 2024-09-12T23:45:12.392 INFO:tasks.rados.rados.0.smithi136.stdout:2006: finishing write tid 4 to smithi13667081-23 2024-09-12T23:45:12.394 INFO:tasks.rados.rados.0.smithi136.stdout:2006: finishing write tid 5 to smithi13667081-23 2024-09-12T23:45:12.394 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 23 v 1384 (ObjNum 660 snap 194 seq_num 660) dirty exists 2024-09-12T23:45:12.394 INFO:tasks.rados.rados.0.smithi136.stdout:2006: left oid 23 (ObjNum 660 snap 194 seq_num 660) 2024-09-12T23:45:12.402 INFO:tasks.rados.rados.0.smithi136.stdout:2007: expect (ObjNum 631 snap 185 seq_num 631) 2024-09-12T23:45:12.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:12 smithi181 bash[67587]: cluster 2024-09-12T23:45:11.406936+0000 mon.a (mon.0) 512 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 5 pgs peering) 2024-09-12T23:45:12.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:12 smithi181 bash[67587]: cluster 2024-09-12T23:45:12.343318+0000 mon.a (mon.0) 513 : cluster [DBG] osdmap e458: 8 total, 7 up, 8 in 2024-09-12T23:45:12.595 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:45:12 smithi181 bash[60471]: ts=2024-09-12T23:45:12.313Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:45:12.756 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 12 23:45:12 smithi136 bash[89273]: debug 2024-09-12T23:45:12.573+0000 7faed8360640 -1 Falling back to public interface 2024-09-12T23:45:12.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:12 smithi136 bash[75164]: cluster 2024-09-12T23:45:11.406936+0000 mon.a (mon.0) 512 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 5 pgs peering) 2024-09-12T23:45:12.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:12 smithi136 bash[75164]: cluster 2024-09-12T23:45:12.343318+0000 mon.a (mon.0) 513 : cluster [DBG] osdmap e458: 8 total, 7 up, 8 in 2024-09-12T23:45:12.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:12 smithi136 bash[76371]: cluster 2024-09-12T23:45:11.406936+0000 mon.a (mon.0) 512 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 5 pgs peering) 2024-09-12T23:45:12.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:12 smithi136 bash[76371]: cluster 2024-09-12T23:45:12.343318+0000 mon.a (mon.0) 513 : cluster [DBG] osdmap e458: 8 total, 7 up, 8 in 2024-09-12T23:45:13.345 INFO:tasks.rados.rados.0.smithi136.stdout:2006: done (3 left) 2024-09-12T23:45:13.345 INFO:tasks.rados.rados.0.smithi136.stdout:2007: done (2 left) 2024-09-12T23:45:13.345 INFO:tasks.rados.rados.0.smithi136.stdout:2008: done (1 left) 2024-09-12T23:45:13.345 INFO:tasks.rados.rados.0.smithi136.stdout:2009: done (0 left) 2024-09-12T23:45:13.345 INFO:tasks.rados.rados.0.smithi136.stdout:2010: delete oid 2 current snap is 195 2024-09-12T23:45:13.356 INFO:tasks.rados.rados.0.smithi136.stdout:2010: done (0 left) 2024-09-12T23:45:13.356 INFO:tasks.rados.rados.0.smithi136.stdout:2011: copy_from oid 29 from oid 30 current snap is 195 2024-09-12T23:45:13.356 INFO:tasks.rados.rados.0.smithi136.stdout:2012: read oid 1 snap 189 2024-09-12T23:45:13.356 INFO:tasks.rados.rados.0.smithi136.stdout:2012: expect deleted 2024-09-12T23:45:13.356 INFO:tasks.rados.rados.0.smithi136.stdout:2013: rollback oid 17 current snap is 195 2024-09-12T23:45:13.356 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 17 to 191 2024-09-12T23:45:13.356 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 49 current snap is 195 2024-09-12T23:45:13.356 INFO:tasks.rados.rados.0.smithi136.stdout:2014: seq_num 662 ranges {811008=696320,1507328=557056,2064384=245760} 2024-09-12T23:45:13.369 INFO:tasks.rados.rados.0.smithi136.stdout:2014: writing smithi13667081-49 from 811008 to 1507328 tid 1 2024-09-12T23:45:13.374 INFO:tasks.rados.rados.0.smithi136.stdout:2014: writing smithi13667081-49 from 1507328 to 2064384 tid 2 2024-09-12T23:45:13.377 INFO:tasks.rados.rados.0.smithi136.stdout:2014: writing smithi13667081-49 from 2064384 to 2310144 tid 3 2024-09-12T23:45:13.377 INFO:tasks.rados.rados.0.smithi136.stdout:2015: setattr oid 5 current snap is 195 2024-09-12T23:45:13.377 INFO:tasks.rados.rados.0.smithi136.stdout:2013: finishing rollback tid 0 to smithi13667081-17 2024-09-12T23:45:13.377 INFO:tasks.rados.rados.0.smithi136.stdout:2013: finishing rollback tid 1 to smithi13667081-17 2024-09-12T23:45:13.377 INFO:tasks.rados.rados.0.smithi136.stdout:2013: finishing rollback tid 2 to smithi13667081-17 2024-09-12T23:45:13.377 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 17 v 1388 (ObjNum 595 snap 173 seq_num 595) dirty exists 2024-09-12T23:45:13.377 INFO:tasks.rados.rados.0.smithi136.stdout:2011: finishing copy_from racing read to smithi13667081-29 2024-09-12T23:45:13.378 INFO:tasks.rados.rados.0.smithi136.stdout:2011: finishing copy_from to smithi13667081-29 2024-09-12T23:45:13.378 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 29 v 804 (ObjNum 129 snap 0 seq_num 4243405312) dirty exists 2024-09-12T23:45:13.380 INFO:tasks.rados.rados.0.smithi136.stdout:2011: done (4 left) 2024-09-12T23:45:13.380 INFO:tasks.rados.rados.0.smithi136.stdout:2012: done (3 left) 2024-09-12T23:45:13.380 INFO:tasks.rados.rados.0.smithi136.stdout:2013: done (2 left) 2024-09-12T23:45:13.380 INFO:tasks.rados.rados.0.smithi136.stdout:2016: read oid 39 snap 184 2024-09-12T23:45:13.380 INFO:tasks.rados.rados.0.smithi136.stdout:2016: expect (ObjNum 625 snap 184 seq_num 625) 2024-09-12T23:45:13.380 INFO:tasks.rados.rados.0.smithi136.stdout:2017: rmattr oid 47 current snap is 195 2024-09-12T23:45:13.380 INFO:tasks.rados.rados.0.smithi136.stdout:2017: done (3 left) 2024-09-12T23:45:13.380 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 35 current snap is 195 2024-09-12T23:45:13.381 INFO:tasks.rados.rados.0.smithi136.stdout:2018: seq_num 663 ranges {0=712704,712704=442368,1155072=360448} 2024-09-12T23:45:13.386 INFO:tasks.rados.rados.0.smithi136.stdout:2018: writing smithi13667081-35 from 0 to 712704 tid 1 2024-09-12T23:45:13.392 INFO:tasks.rados.rados.0.smithi136.stdout:2018: writing smithi13667081-35 from 712704 to 1155072 tid 2 2024-09-12T23:45:13.396 INFO:tasks.rados.rados.0.smithi136.stdout:2018: writing smithi13667081-35 from 1155072 to 1515520 tid 3 2024-09-12T23:45:13.396 INFO:tasks.rados.rados.0.smithi136.stdout:2019: rollback oid 3 current snap is 195 2024-09-12T23:45:13.396 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 3 to 191 2024-09-12T23:45:13.397 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 5 v 1390 (ObjNum 567 snap 162 seq_num 567) dirty exists 2024-09-12T23:45:13.397 INFO:tasks.rados.rados.0.smithi136.stdout:2014: finishing write tid 1 to smithi13667081-49 2024-09-12T23:45:13.397 INFO:tasks.rados.rados.0.smithi136.stdout:2016: expect (ObjNum 625 snap 184 seq_num 625) 2024-09-12T23:45:13.453 INFO:tasks.rados.rados.0.smithi136.stdout:2014: finishing write tid 2 to smithi13667081-49 2024-09-12T23:45:13.453 INFO:tasks.rados.rados.0.smithi136.stdout:2014: finishing write tid 3 to smithi13667081-49 2024-09-12T23:45:13.453 INFO:tasks.rados.rados.0.smithi136.stdout:2015: done (4 left) 2024-09-12T23:45:13.453 INFO:tasks.rados.rados.0.smithi136.stdout:2016: done (3 left) 2024-09-12T23:45:13.453 INFO:tasks.rados.rados.0.smithi136.stdout:2020: copy_from oid 11 from oid 8 current snap is 195 2024-09-12T23:45:13.453 INFO:tasks.rados.rados.0.smithi136.stdout:2014: finishing write tid 4 to smithi13667081-49 2024-09-12T23:45:13.453 INFO:tasks.rados.rados.0.smithi136.stdout:2018: finishing write tid 1 to smithi13667081-35 2024-09-12T23:45:13.453 INFO:tasks.rados.rados.0.smithi136.stdout:2018: finishing write tid 2 to smithi13667081-35 2024-09-12T23:45:13.453 INFO:tasks.rados.rados.0.smithi136.stdout:2018: finishing write tid 3 to smithi13667081-35 2024-09-12T23:45:13.453 INFO:tasks.rados.rados.0.smithi136.stdout:2018: finishing write tid 4 to smithi13667081-35 2024-09-12T23:45:13.453 INFO:tasks.rados.rados.0.smithi136.stdout:2014: finishing write tid 5 to smithi13667081-49 2024-09-12T23:45:13.453 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 49 v 1219 (ObjNum 662 snap 195 seq_num 662) dirty exists 2024-09-12T23:45:13.454 INFO:tasks.rados.rados.0.smithi136.stdout:2014: left oid 49 (ObjNum 662 snap 195 seq_num 662) 2024-09-12T23:45:13.454 INFO:tasks.rados.rados.0.smithi136.stdout:2018: finishing write tid 5 to smithi13667081-35 2024-09-12T23:45:13.454 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 35 v 1394 (ObjNum 663 snap 195 seq_num 663) dirty exists 2024-09-12T23:45:13.454 INFO:tasks.rados.rados.0.smithi136.stdout:2018: left oid 35 (ObjNum 663 snap 195 seq_num 663) 2024-09-12T23:45:13.454 INFO:tasks.rados.rados.0.smithi136.stdout:2019: finishing rollback tid 0 to smithi13667081-3 2024-09-12T23:45:13.454 INFO:tasks.rados.rados.0.smithi136.stdout:2014: done (3 left) 2024-09-12T23:45:13.454 INFO:tasks.rados.rados.0.smithi136.stdout:2018: done (2 left) 2024-09-12T23:45:13.454 INFO:tasks.rados.rados.0.smithi136.stdout:2021: rollback oid 6 current snap is 195 2024-09-12T23:45:13.454 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 6 to 194 2024-09-12T23:45:13.454 INFO:tasks.rados.rados.0.smithi136.stdout:2019: finishing rollback tid 1 to smithi13667081-3 2024-09-12T23:45:13.454 INFO:tasks.rados.rados.0.smithi136.stdout:2019: finishing rollback tid 2 to smithi13667081-3 2024-09-12T23:45:13.454 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 3 v 1222 (ObjNum 633 snap 186 seq_num 633) dirty exists 2024-09-12T23:45:13.454 INFO:tasks.rados.rados.0.smithi136.stdout:2019: done (2 left) 2024-09-12T23:45:13.454 INFO:tasks.rados.rados.0.smithi136.stdout:2022: setattr oid 20 current snap is 195 2024-09-12T23:45:13.457 INFO:tasks.rados.rados.0.smithi136.stdout:2023: read oid 5 snap -1 2024-09-12T23:45:13.457 INFO:tasks.rados.rados.0.smithi136.stdout:2023: expect (ObjNum 567 snap 162 seq_num 567) 2024-09-12T23:45:13.457 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 10 current snap is 195 2024-09-12T23:45:13.457 INFO:tasks.rados.rados.0.smithi136.stdout:2024: seq_num 664 ranges {2916352=729088,3645440=729088,4374528=73728} 2024-09-12T23:45:13.486 INFO:tasks.rados.rados.0.smithi136.stdout:2024: writing smithi13667081-10 from 2916352 to 3645440 tid 1 2024-09-12T23:45:13.493 INFO:tasks.rados.rados.0.smithi136.stdout:2024: writing smithi13667081-10 from 3645440 to 4374528 tid 2 2024-09-12T23:45:13.493 INFO:tasks.rados.rados.0.smithi136.stdout:2024: writing smithi13667081-10 from 4374528 to 4448256 tid 3 2024-09-12T23:45:13.494 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 38 current snap is 195 2024-09-12T23:45:13.494 INFO:tasks.rados.rados.0.smithi136.stdout:2025: seq_num 665 ranges {2793472=745472,3538944=606208,4145152=196608} 2024-09-12T23:45:13.518 INFO:tasks.rados.rados.0.smithi136.stdout:2025: writing smithi13667081-38 from 2793472 to 3538944 tid 1 2024-09-12T23:45:13.521 INFO:tasks.rados.rados.0.smithi136.stdout:2025: writing smithi13667081-38 from 3538944 to 4145152 tid 2 2024-09-12T23:45:13.523 INFO:tasks.rados.rados.0.smithi136.stdout:2025: writing smithi13667081-38 from 4145152 to 4341760 tid 3 2024-09-12T23:45:13.523 INFO:tasks.rados.rados.0.smithi136.stdout:2026: read oid 31 snap 180 2024-09-12T23:45:13.523 INFO:tasks.rados.rados.0.smithi136.stdout:2026: expect (ObjNum 572 snap 166 seq_num 572) 2024-09-12T23:45:13.523 INFO:tasks.rados.rados.0.smithi136.stdout:2027: rollback oid 26 current snap is 195 2024-09-12T23:45:13.524 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 26 to 188 2024-09-12T23:45:13.524 INFO:tasks.rados.rados.0.smithi136.stdout:2021: finishing rollback tid 0 to smithi13667081-6 2024-09-12T23:45:13.524 INFO:tasks.rados.rados.0.smithi136.stdout:2021: finishing rollback tid 1 to smithi13667081-6 2024-09-12T23:45:13.524 INFO:tasks.rados.rados.0.smithi136.stdout:2021: finishing rollback tid 2 to smithi13667081-6 2024-09-12T23:45:13.524 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 6 v 1217 (ObjNum 656 snap 192 seq_num 656) dirty exists 2024-09-12T23:45:13.524 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 20 v 1218 (ObjNum 469 snap 139 seq_num 469) dirty exists 2024-09-12T23:45:13.524 INFO:tasks.rados.rados.0.smithi136.stdout:2023: expect (ObjNum 567 snap 162 seq_num 567) 2024-09-12T23:45:13.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:13 smithi136 bash[75164]: cluster 2024-09-12T23:45:12.716500+0000 mgr.y (mgr.24661) 168 : cluster [DBG] pgmap v262: 121 pgs: 28 active+undersized, 11 active+undersized+degraded, 82 active+clean; 206 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 1.3 MiB/s wr, 15 op/s; 56/357 objects degraded (15.686%) 2024-09-12T23:45:13.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:13 smithi136 bash[75164]: cluster 2024-09-12T23:45:13.348803+0000 mon.a (mon.0) 514 : cluster [DBG] osdmap e459: 8 total, 7 up, 8 in 2024-09-12T23:45:13.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:13 smithi136 bash[76371]: cluster 2024-09-12T23:45:12.716500+0000 mgr.y (mgr.24661) 168 : cluster [DBG] pgmap v262: 121 pgs: 28 active+undersized, 11 active+undersized+degraded, 82 active+clean; 206 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 1.3 MiB/s wr, 15 op/s; 56/357 objects degraded (15.686%) 2024-09-12T23:45:13.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:13 smithi136 bash[76371]: cluster 2024-09-12T23:45:13.348803+0000 mon.a (mon.0) 514 : cluster [DBG] osdmap e459: 8 total, 7 up, 8 in 2024-09-12T23:45:13.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:13 smithi181 bash[67587]: cluster 2024-09-12T23:45:12.716500+0000 mgr.y (mgr.24661) 168 : cluster [DBG] pgmap v262: 121 pgs: 28 active+undersized, 11 active+undersized+degraded, 82 active+clean; 206 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 1.3 MiB/s wr, 15 op/s; 56/357 objects degraded (15.686%) 2024-09-12T23:45:13.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:13 smithi181 bash[67587]: cluster 2024-09-12T23:45:13.348803+0000 mon.a (mon.0) 514 : cluster [DBG] osdmap e459: 8 total, 7 up, 8 in 2024-09-12T23:45:13.844 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:45:13 smithi181 bash[60471]: ts=2024-09-12T23:45:13.510Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi136\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:45:13.873 INFO:tasks.rados.rados.0.smithi136.stdout:2024: finishing write tid 1 to smithi13667081-10 2024-09-12T23:45:13.873 INFO:tasks.rados.rados.0.smithi136.stdout:2020: finishing copy_from racing read to smithi13667081-11 2024-09-12T23:45:13.873 INFO:tasks.rados.rados.0.smithi136.stdout:2024: finishing write tid 2 to smithi13667081-10 2024-09-12T23:45:13.873 INFO:tasks.rados.rados.0.smithi136.stdout:2024: finishing write tid 3 to smithi13667081-10 2024-09-12T23:45:13.873 INFO:tasks.rados.rados.0.smithi136.stdout:2024: finishing write tid 4 to smithi13667081-10 2024-09-12T23:45:13.873 INFO:tasks.rados.rados.0.smithi136.stdout:2024: finishing write tid 5 to smithi13667081-10 2024-09-12T23:45:13.873 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 10 v 1085 (ObjNum 664 snap 195 seq_num 664) dirty exists 2024-09-12T23:45:13.873 INFO:tasks.rados.rados.0.smithi136.stdout:2024: left oid 10 (ObjNum 664 snap 195 seq_num 664) 2024-09-12T23:45:13.873 INFO:tasks.rados.rados.0.smithi136.stdout:2020: finishing copy_from to smithi13667081-11 2024-09-12T23:45:13.873 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 11 v 797 (ObjNum 605 snap 174 seq_num 605) dirty exists 2024-09-12T23:45:13.873 INFO:tasks.rados.rados.0.smithi136.stdout:2025: finishing write tid 1 to smithi13667081-38 2024-09-12T23:45:13.873 INFO:tasks.rados.rados.0.smithi136.stdout:2025: finishing write tid 2 to smithi13667081-38 2024-09-12T23:45:13.874 INFO:tasks.rados.rados.0.smithi136.stdout:2025: finishing write tid 3 to smithi13667081-38 2024-09-12T23:45:13.874 INFO:tasks.rados.rados.0.smithi136.stdout:2027: finishing rollback tid 0 to smithi13667081-26 2024-09-12T23:45:13.874 INFO:tasks.rados.rados.0.smithi136.stdout:2025: finishing write tid 4 to smithi13667081-38 2024-09-12T23:45:13.874 INFO:tasks.rados.rados.0.smithi136.stdout:2027: finishing rollback tid 1 to smithi13667081-26 2024-09-12T23:45:13.874 INFO:tasks.rados.rados.0.smithi136.stdout:2027: finishing rollback tid 2 to smithi13667081-26 2024-09-12T23:45:13.874 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 26 v 648 (ObjNum 618 snap 179 seq_num 618) dirty exists 2024-09-12T23:45:13.874 INFO:tasks.rados.rados.0.smithi136.stdout:2025: finishing write tid 5 to smithi13667081-38 2024-09-12T23:45:13.874 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 38 v 1215 (ObjNum 665 snap 195 seq_num 665) dirty exists 2024-09-12T23:45:13.874 INFO:tasks.rados.rados.0.smithi136.stdout:2025: left oid 38 (ObjNum 665 snap 195 seq_num 665) 2024-09-12T23:45:13.874 INFO:tasks.rados.rados.0.smithi136.stdout:2026: expect (ObjNum 572 snap 166 seq_num 572) 2024-09-12T23:45:14.252 INFO:tasks.rados.rados.0.smithi136.stdout:2020: done (7 left) 2024-09-12T23:45:14.253 INFO:tasks.rados.rados.0.smithi136.stdout:2021: done (6 left) 2024-09-12T23:45:14.253 INFO:tasks.rados.rados.0.smithi136.stdout:2022: done (5 left) 2024-09-12T23:45:14.253 INFO:tasks.rados.rados.0.smithi136.stdout:2023: done (4 left) 2024-09-12T23:45:14.253 INFO:tasks.rados.rados.0.smithi136.stdout:2024: done (3 left) 2024-09-12T23:45:14.253 INFO:tasks.rados.rados.0.smithi136.stdout:2025: done (2 left) 2024-09-12T23:45:14.253 INFO:tasks.rados.rados.0.smithi136.stdout:2026: done (1 left) 2024-09-12T23:45:14.253 INFO:tasks.rados.rados.0.smithi136.stdout:2027: done (0 left) 2024-09-12T23:45:14.253 INFO:tasks.rados.rados.0.smithi136.stdout:2028: read oid 1 snap -1 2024-09-12T23:45:14.253 INFO:tasks.rados.rados.0.smithi136.stdout:2028: expect deleted 2024-09-12T23:45:14.253 INFO:tasks.rados.rados.0.smithi136.stdout:2029: snap_remove snap 180 2024-09-12T23:45:14.428 INFO:tasks.rados.rados.0.smithi136.stdout:2029: done (1 left) 2024-09-12T23:45:14.428 INFO:tasks.rados.rados.0.smithi136.stdout:2030: copy_from oid 16 from oid 43 current snap is 195 2024-09-12T23:45:14.436 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 38 current snap is 195 2024-09-12T23:45:14.436 INFO:tasks.rados.rados.0.smithi136.stdout:2031: seq_num 666 ranges {4341760=761856,5103616=491520,5595136=311296} 2024-09-12T23:45:14.461 INFO:tasks.rados.rados.0.smithi136.stdout:2031: writing smithi13667081-38 from 4341760 to 5103616 tid 1 2024-09-12T23:45:14.465 INFO:tasks.rados.rados.0.smithi136.stdout:2031: writing smithi13667081-38 from 5103616 to 5595136 tid 2 2024-09-12T23:45:14.468 INFO:tasks.rados.rados.0.smithi136.stdout:2031: writing smithi13667081-38 from 5595136 to 5906432 tid 3 2024-09-12T23:45:14.468 INFO:tasks.rados.rados.0.smithi136.stdout:2032: read oid 46 snap -1 2024-09-12T23:45:14.468 INFO:tasks.rados.rados.0.smithi136.stdout:2032: expect (ObjNum 636 snap 188 seq_num 636) 2024-09-12T23:45:14.468 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 39 current snap is 195 2024-09-12T23:45:14.468 INFO:tasks.rados.rados.0.smithi136.stdout:2033: seq_num 667 ranges {4833280=778240,5611520=778240,6389760=24576} 2024-09-12T23:45:14.500 INFO:tasks.rados.rados.0.smithi136.stdout:2033: writing smithi13667081-39 from 4833280 to 5611520 tid 1 2024-09-12T23:45:14.505 INFO:tasks.rados.rados.0.smithi136.stdout:2033: writing smithi13667081-39 from 5611520 to 6389760 tid 2 2024-09-12T23:45:14.505 INFO:tasks.rados.rados.0.smithi136.stdout:2033: writing smithi13667081-39 from 6389760 to 6414336 tid 3 2024-09-12T23:45:14.505 INFO:tasks.rados.rados.0.smithi136.stdout:2034: rollback oid 18 current snap is 195 2024-09-12T23:45:14.505 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 18 to 193 2024-09-12T23:45:14.506 INFO:tasks.rados.rados.0.smithi136.stdout:2031: finishing write tid 1 to smithi13667081-38 2024-09-12T23:45:14.506 INFO:tasks.rados.rados.0.smithi136.stdout:2032: expect (ObjNum 636 snap 188 seq_num 636) 2024-09-12T23:45:14.555 INFO:tasks.rados.rados.0.smithi136.stdout:2031: finishing write tid 2 to smithi13667081-38 2024-09-12T23:45:14.555 INFO:tasks.rados.rados.0.smithi136.stdout:2031: finishing write tid 3 to smithi13667081-38 2024-09-12T23:45:14.556 INFO:tasks.rados.rados.0.smithi136.stdout:2028: done (5 left) 2024-09-12T23:45:14.556 INFO:tasks.rados.rados.0.smithi136.stdout:2032: done (4 left) 2024-09-12T23:45:14.556 INFO:tasks.rados.rados.0.smithi136.stdout:2035: read oid 36 snap -1 2024-09-12T23:45:14.556 INFO:tasks.rados.rados.0.smithi136.stdout:2035: expect (ObjNum 610 snap 175 seq_num 610) 2024-09-12T23:45:14.556 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 12 current snap is 195 2024-09-12T23:45:14.556 INFO:tasks.rados.rados.0.smithi136.stdout:2031: finishing write tid 4 to smithi13667081-38 2024-09-12T23:45:14.556 INFO:tasks.rados.rados.0.smithi136.stdout:2031: finishing write tid 5 to smithi13667081-38 2024-09-12T23:45:14.556 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 38 v 1219 (ObjNum 666 snap 195 seq_num 666) dirty exists 2024-09-12T23:45:14.556 INFO:tasks.rados.rados.0.smithi136.stdout:2031: left oid 38 (ObjNum 666 snap 195 seq_num 666) 2024-09-12T23:45:14.556 INFO:tasks.rados.rados.0.smithi136.stdout:2033: finishing write tid 1 to smithi13667081-39 2024-09-12T23:45:14.556 INFO:tasks.rados.rados.0.smithi136.stdout:2036: seq_num 668 ranges {5971968=794624,6766592=663552,7430144=139264} 2024-09-12T23:45:14.593 INFO:tasks.rados.rados.0.smithi136.stdout:2036: writing smithi13667081-12 from 5971968 to 6766592 tid 1 2024-09-12T23:45:14.597 INFO:tasks.rados.rados.0.smithi136.stdout:2036: writing smithi13667081-12 from 6766592 to 7430144 tid 2 2024-09-12T23:45:14.599 INFO:tasks.rados.rados.0.smithi136.stdout:2036: writing smithi13667081-12 from 7430144 to 7569408 tid 3 2024-09-12T23:45:14.599 INFO:tasks.rados.rados.0.smithi136.stdout:2031: done (5 left) 2024-09-12T23:45:14.599 INFO:tasks.rados.rados.0.smithi136.stdout:2037: rollback oid 22 current snap is 195 2024-09-12T23:45:14.599 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 22 to 189 2024-09-12T23:45:14.599 INFO:tasks.rados.rados.0.smithi136.stdout:2033: finishing write tid 2 to smithi13667081-39 2024-09-12T23:45:14.599 INFO:tasks.rados.rados.0.smithi136.stdout:2033: finishing write tid 3 to smithi13667081-39 2024-09-12T23:45:14.599 INFO:tasks.rados.rados.0.smithi136.stdout:2034: finishing rollback tid 0 to smithi13667081-18 2024-09-12T23:45:14.599 INFO:tasks.rados.rados.0.smithi136.stdout:2033: finishing write tid 4 to smithi13667081-39 2024-09-12T23:45:14.599 INFO:tasks.rados.rados.0.smithi136.stdout:2034: finishing rollback tid 1 to smithi13667081-18 2024-09-12T23:45:14.599 INFO:tasks.rados.rados.0.smithi136.stdout:2033: finishing write tid 5 to smithi13667081-39 2024-09-12T23:45:14.599 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 39 v 822 (ObjNum 667 snap 195 seq_num 667) dirty exists 2024-09-12T23:45:14.599 INFO:tasks.rados.rados.0.smithi136.stdout:2033: left oid 39 (ObjNum 667 snap 195 seq_num 667) 2024-09-12T23:45:14.599 INFO:tasks.rados.rados.0.smithi136.stdout:2034: finishing rollback tid 2 to smithi13667081-18 2024-09-12T23:45:14.600 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 18 v 926 (ObjNum 578 snap 169 seq_num 578) dirty exists 2024-09-12T23:45:14.600 INFO:tasks.rados.rados.0.smithi136.stdout:2030: finishing copy_from racing read to smithi13667081-16 2024-09-12T23:45:14.600 INFO:tasks.rados.rados.0.smithi136.stdout:2030: finishing copy_from to smithi13667081-16 2024-09-12T23:45:14.600 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 16 v 1115 (ObjNum 616 snap 179 seq_num 616) dirty exists 2024-09-12T23:45:14.600 INFO:tasks.rados.rados.0.smithi136.stdout:2030: done (5 left) 2024-09-12T23:45:14.600 INFO:tasks.rados.rados.0.smithi136.stdout:2033: done (4 left) 2024-09-12T23:45:14.600 INFO:tasks.rados.rados.0.smithi136.stdout:2034: done (3 left) 2024-09-12T23:45:14.600 INFO:tasks.rados.rados.0.smithi136.stdout:2038: read oid 35 snap -1 2024-09-12T23:45:14.600 INFO:tasks.rados.rados.0.smithi136.stdout:2038: expect (ObjNum 663 snap 195 seq_num 663) 2024-09-12T23:45:14.601 INFO:tasks.rados.rados.0.smithi136.stdout:2039: snap_create 2024-09-12T23:45:14.601 INFO:tasks.rados.rados.0.smithi136.stdout:2035: expect (ObjNum 610 snap 175 seq_num 610) 2024-09-12T23:45:14.619 INFO:tasks.rados.rados.0.smithi136.stdout:2037: finishing rollback tid 0 to smithi13667081-22 2024-09-12T23:45:14.619 INFO:tasks.rados.rados.0.smithi136.stdout:2037: finishing rollback tid 1 to smithi13667081-22 2024-09-12T23:45:14.619 INFO:tasks.rados.rados.0.smithi136.stdout:2036: finishing write tid 1 to smithi13667081-12 2024-09-12T23:45:14.620 INFO:tasks.rados.rados.0.smithi136.stdout:2038: expect (ObjNum 663 snap 195 seq_num 663) 2024-09-12T23:45:14.661 INFO:tasks.rados.rados.0.smithi136.stdout:2037: finishing rollback tid 2 to smithi13667081-22 2024-09-12T23:45:14.661 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 22 v 833 (ObjNum 548 snap 158 seq_num 548) dirty exists 2024-09-12T23:45:14.661 INFO:tasks.rados.rados.0.smithi136.stdout:2036: finishing write tid 2 to smithi13667081-12 2024-09-12T23:45:14.661 INFO:tasks.rados.rados.0.smithi136.stdout:2036: finishing write tid 3 to smithi13667081-12 2024-09-12T23:45:14.661 INFO:tasks.rados.rados.0.smithi136.stdout:2036: finishing write tid 4 to smithi13667081-12 2024-09-12T23:45:14.661 INFO:tasks.rados.rados.0.smithi136.stdout:2036: finishing write tid 5 to smithi13667081-12 2024-09-12T23:45:14.661 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 12 v 1226 (ObjNum 668 snap 195 seq_num 668) dirty exists 2024-09-12T23:45:14.661 INFO:tasks.rados.rados.0.smithi136.stdout:2036: left oid 12 (ObjNum 668 snap 195 seq_num 668) 2024-09-12T23:45:14.844 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:45:14 smithi181 bash[60471]: ts=2024-09-12T23:45:14.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:45:14.844 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:45:14 smithi181 bash[60471]: ts=2024-09-12T23:45:14.456Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:45:15.439 INFO:tasks.rados.rados.0.smithi136.stdout:2035: done (4 left) 2024-09-12T23:45:15.439 INFO:tasks.rados.rados.0.smithi136.stdout:2036: done (3 left) 2024-09-12T23:45:15.439 INFO:tasks.rados.rados.0.smithi136.stdout:2037: done (2 left) 2024-09-12T23:45:15.439 INFO:tasks.rados.rados.0.smithi136.stdout:2038: done (1 left) 2024-09-12T23:45:15.439 INFO:tasks.rados.rados.0.smithi136.stdout:2039: done (0 left) 2024-09-12T23:45:15.439 INFO:tasks.rados.rados.0.smithi136.stdout:2040: rollback oid 29 current snap is 196 2024-09-12T23:45:15.439 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 29 to 189 2024-09-12T23:45:15.439 INFO:tasks.rados.rados.0.smithi136.stdout:2041: copy_from oid 37 from oid 46 current snap is 196 2024-09-12T23:45:15.440 INFO:tasks.rados.rados.0.smithi136.stdout:2042: snap_remove snap 191 2024-09-12T23:45:15.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:15 smithi136 bash[75164]: cluster 2024-09-12T23:45:14.430620+0000 mon.a (mon.0) 515 : cluster [DBG] osdmap e460: 8 total, 7 up, 8 in 2024-09-12T23:45:15.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:15 smithi136 bash[75164]: cluster 2024-09-12T23:45:14.717281+0000 mgr.y (mgr.24661) 169 : cluster [DBG] pgmap v265: 121 pgs: 28 active+undersized, 11 active+undersized+degraded, 82 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 8.0 MiB/s wr, 29 op/s; 65/402 objects degraded (16.169%) 2024-09-12T23:45:15.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:15 smithi136 bash[75164]: audit 2024-09-12T23:45:15.010025+0000 mon.a (mon.0) 516 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:15.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:15 smithi136 bash[75164]: audit 2024-09-12T23:45:15.016480+0000 mon.a (mon.0) 517 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:15.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:15 smithi136 bash[76371]: cluster 2024-09-12T23:45:14.430620+0000 mon.a (mon.0) 515 : cluster [DBG] osdmap e460: 8 total, 7 up, 8 in 2024-09-12T23:45:15.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:15 smithi136 bash[76371]: cluster 2024-09-12T23:45:14.717281+0000 mgr.y (mgr.24661) 169 : cluster [DBG] pgmap v265: 121 pgs: 28 active+undersized, 11 active+undersized+degraded, 82 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 8.0 MiB/s wr, 29 op/s; 65/402 objects degraded (16.169%) 2024-09-12T23:45:15.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:15 smithi136 bash[76371]: audit 2024-09-12T23:45:15.010025+0000 mon.a (mon.0) 516 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:15.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:15 smithi136 bash[76371]: audit 2024-09-12T23:45:15.016480+0000 mon.a (mon.0) 517 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:15.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:15 smithi181 bash[67587]: cluster 2024-09-12T23:45:14.430620+0000 mon.a (mon.0) 515 : cluster [DBG] osdmap e460: 8 total, 7 up, 8 in 2024-09-12T23:45:15.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:15 smithi181 bash[67587]: cluster 2024-09-12T23:45:14.717281+0000 mgr.y (mgr.24661) 169 : cluster [DBG] pgmap v265: 121 pgs: 28 active+undersized, 11 active+undersized+degraded, 82 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 8.0 MiB/s wr, 29 op/s; 65/402 objects degraded (16.169%) 2024-09-12T23:45:15.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:15 smithi181 bash[67587]: audit 2024-09-12T23:45:15.010025+0000 mon.a (mon.0) 516 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:15.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:15 smithi181 bash[67587]: audit 2024-09-12T23:45:15.016480+0000 mon.a (mon.0) 517 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:16.446 INFO:tasks.rados.rados.0.smithi136.stdout:2042: done (2 left) 2024-09-12T23:45:16.446 INFO:tasks.rados.rados.0.smithi136.stdout:2043: rollback oid 50 current snap is 196 2024-09-12T23:45:16.446 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 50 to 189 2024-09-12T23:45:16.446 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 48 current snap is 196 2024-09-12T23:45:16.446 INFO:tasks.rados.rados.0.smithi136.stdout:2044: seq_num 669 ranges {4055040=409600,4464640=548864,5013504=655360} 2024-09-12T23:45:16.473 INFO:tasks.rados.rados.0.smithi136.stdout:2044: writing smithi13667081-48 from 4055040 to 4464640 tid 1 2024-09-12T23:45:16.477 INFO:tasks.rados.rados.0.smithi136.stdout:2044: writing smithi13667081-48 from 4464640 to 5013504 tid 2 2024-09-12T23:45:16.482 INFO:tasks.rados.rados.0.smithi136.stdout:2044: writing smithi13667081-48 from 5013504 to 5668864 tid 3 2024-09-12T23:45:16.482 INFO:tasks.rados.rados.0.smithi136.stdout:2045: rollback oid 1 current snap is 196 2024-09-12T23:45:16.482 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 1 to 183 2024-09-12T23:45:16.482 INFO:tasks.rados.rados.0.smithi136.stdout:2040: finishing rollback tid 0 to smithi13667081-29 2024-09-12T23:45:16.482 INFO:tasks.rados.rados.0.smithi136.stdout:2040: finishing rollback tid 1 to smithi13667081-29 2024-09-12T23:45:16.482 INFO:tasks.rados.rados.0.smithi136.stdout:2040: finishing rollback tid 2 to smithi13667081-29 2024-09-12T23:45:16.482 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 29 v 1222 (ObjNum 643 snap 189 seq_num 643) dirty exists 2024-09-12T23:45:16.483 INFO:tasks.rados.rados.0.smithi136.stdout:2041: finishing copy_from racing read to smithi13667081-37 2024-09-12T23:45:16.483 INFO:tasks.rados.rados.0.smithi136.stdout:2041: finishing copy_from to smithi13667081-37 2024-09-12T23:45:16.483 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 37 v 644 (ObjNum 636 snap 188 seq_num 636) dirty exists 2024-09-12T23:45:16.483 INFO:tasks.rados.rados.0.smithi136.stdout:2043: finishing rollback tid 1 to smithi13667081-50 2024-09-12T23:45:16.483 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 50 v 0 (ObjNum 129 snap 0 seq_num 2877685989) dirty dne 2024-09-12T23:45:16.483 INFO:tasks.rados.rados.0.smithi136.stdout:2044: finishing write tid 1 to smithi13667081-48 2024-09-12T23:45:16.483 INFO:tasks.rados.rados.0.smithi136.stdout:2040: done (4 left) 2024-09-12T23:45:16.483 INFO:tasks.rados.rados.0.smithi136.stdout:2041: done (3 left) 2024-09-12T23:45:16.483 INFO:tasks.rados.rados.0.smithi136.stdout:2043: done (2 left) 2024-09-12T23:45:16.483 INFO:tasks.rados.rados.0.smithi136.stdout:2046: delete oid 19 current snap is 196 2024-09-12T23:45:16.483 INFO:tasks.rados.rados.0.smithi136.stdout:2044: finishing write tid 2 to smithi13667081-48 2024-09-12T23:45:16.485 INFO:tasks.rados.rados.0.smithi136.stdout:2045: finishing rollback tid 1 to smithi13667081-1 2024-09-12T23:45:16.485 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 1 v 0 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-12T23:45:16.486 INFO:tasks.rados.rados.0.smithi136.stdout:2045: done (2 left) 2024-09-12T23:45:16.486 INFO:tasks.rados.rados.0.smithi136.stdout:2046: done (1 left) 2024-09-12T23:45:16.486 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 35 current snap is 196 2024-09-12T23:45:16.487 INFO:tasks.rados.rados.0.smithi136.stdout:2047: seq_num 671 ranges {1515520=442368,1957888=761856,2719744=442368} 2024-09-12T23:45:16.496 INFO:tasks.rados.rados.0.smithi136.stdout:2047: writing smithi13667081-35 from 1515520 to 1957888 tid 1 2024-09-12T23:45:16.501 INFO:tasks.rados.rados.0.smithi136.stdout:2047: writing smithi13667081-35 from 1957888 to 2719744 tid 2 2024-09-12T23:45:16.504 INFO:tasks.rados.rados.0.smithi136.stdout:2047: writing smithi13667081-35 from 2719744 to 3162112 tid 3 2024-09-12T23:45:16.504 INFO:tasks.rados.rados.0.smithi136.stdout:2048: snap_create 2024-09-12T23:45:16.504 INFO:tasks.rados.rados.0.smithi136.stdout:2044: finishing write tid 3 to smithi13667081-48 2024-09-12T23:45:16.504 INFO:tasks.rados.rados.0.smithi136.stdout:2044: finishing write tid 4 to smithi13667081-48 2024-09-12T23:45:16.505 INFO:tasks.rados.rados.0.smithi136.stdout:2044: finishing write tid 5 to smithi13667081-48 2024-09-12T23:45:16.505 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 48 v 1179 (ObjNum 669 snap 196 seq_num 669) dirty exists 2024-09-12T23:45:16.505 INFO:tasks.rados.rados.0.smithi136.stdout:2044: left oid 48 (ObjNum 669 snap 196 seq_num 669) 2024-09-12T23:45:16.505 INFO:tasks.rados.rados.0.smithi136.stdout:2047: finishing write tid 1 to smithi13667081-35 2024-09-12T23:45:16.505 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:16 smithi136 bash[76371]: cluster 2024-09-12T23:45:15.427008+0000 mon.a (mon.0) 518 : cluster [WRN] Health check update: Degraded data redundancy: 65/402 objects degraded (16.169%), 11 pgs degraded (PG_DEGRADED) 2024-09-12T23:45:16.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:16 smithi136 bash[76371]: cluster 2024-09-12T23:45:15.443178+0000 mon.a (mon.0) 519 : cluster [DBG] osdmap e461: 8 total, 7 up, 8 in 2024-09-12T23:45:16.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:16 smithi136 bash[76371]: audit 2024-09-12T23:45:15.699554+0000 mon.a (mon.0) 520 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:16.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:16 smithi136 bash[76371]: audit 2024-09-12T23:45:15.706468+0000 mon.a (mon.0) 521 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:16.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:16 smithi136 bash[76371]: audit 2024-09-12T23:45:16.238764+0000 mon.c (mon.2) 7 : audit [INF] from='osd.2 [v2:172.21.15.136:6818/439422470,v1:172.21.15.136:6819/439422470]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-12T23:45:16.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:16 smithi136 bash[76371]: audit 2024-09-12T23:45:16.239057+0000 mon.a (mon.0) 522 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-12T23:45:16.506 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 12 23:45:16 smithi136 bash[89273]: debug 2024-09-12T23:45:16.229+0000 7faed8360640 -1 osd.2 449 log_to_monitors true 2024-09-12T23:45:16.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:16 smithi136 bash[75164]: cluster 2024-09-12T23:45:15.427008+0000 mon.a (mon.0) 518 : cluster [WRN] Health check update: Degraded data redundancy: 65/402 objects degraded (16.169%), 11 pgs degraded (PG_DEGRADED) 2024-09-12T23:45:16.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:16 smithi136 bash[75164]: cluster 2024-09-12T23:45:15.443178+0000 mon.a (mon.0) 519 : cluster [DBG] osdmap e461: 8 total, 7 up, 8 in 2024-09-12T23:45:16.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:16 smithi136 bash[75164]: audit 2024-09-12T23:45:15.699554+0000 mon.a (mon.0) 520 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:16.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:16 smithi136 bash[75164]: audit 2024-09-12T23:45:15.706468+0000 mon.a (mon.0) 521 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:16.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:16 smithi136 bash[75164]: audit 2024-09-12T23:45:16.238764+0000 mon.c (mon.2) 7 : audit [INF] from='osd.2 [v2:172.21.15.136:6818/439422470,v1:172.21.15.136:6819/439422470]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-12T23:45:16.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:16 smithi136 bash[75164]: audit 2024-09-12T23:45:16.239057+0000 mon.a (mon.0) 522 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-12T23:45:16.507 INFO:tasks.rados.rados.0.smithi136.stdout:2047: finishing write tid 2 to smithi13667081-35 2024-09-12T23:45:16.509 INFO:tasks.rados.rados.0.smithi136.stdout:2047: finishing write tid 3 to smithi13667081-35 2024-09-12T23:45:16.509 INFO:tasks.rados.rados.0.smithi136.stdout:2047: finishing write tid 4 to smithi13667081-35 2024-09-12T23:45:16.511 INFO:tasks.rados.rados.0.smithi136.stdout:2047: finishing write tid 5 to smithi13667081-35 2024-09-12T23:45:16.511 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 35 v 1419 (ObjNum 671 snap 196 seq_num 671) dirty exists 2024-09-12T23:45:16.511 INFO:tasks.rados.rados.0.smithi136.stdout:2047: left oid 35 (ObjNum 671 snap 196 seq_num 671) 2024-09-12T23:45:16.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:16 smithi181 bash[67587]: cluster 2024-09-12T23:45:15.427008+0000 mon.a (mon.0) 518 : cluster [WRN] Health check update: Degraded data redundancy: 65/402 objects degraded (16.169%), 11 pgs degraded (PG_DEGRADED) 2024-09-12T23:45:16.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:16 smithi181 bash[67587]: cluster 2024-09-12T23:45:15.443178+0000 mon.a (mon.0) 519 : cluster [DBG] osdmap e461: 8 total, 7 up, 8 in 2024-09-12T23:45:16.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:16 smithi181 bash[67587]: audit 2024-09-12T23:45:15.699554+0000 mon.a (mon.0) 520 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:16.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:16 smithi181 bash[67587]: audit 2024-09-12T23:45:15.706468+0000 mon.a (mon.0) 521 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:16.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:16 smithi181 bash[67587]: audit 2024-09-12T23:45:16.238764+0000 mon.c (mon.2) 7 : audit [INF] from='osd.2 [v2:172.21.15.136:6818/439422470,v1:172.21.15.136:6819/439422470]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-12T23:45:16.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:16 smithi181 bash[67587]: audit 2024-09-12T23:45:16.239057+0000 mon.a (mon.0) 522 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-12T23:45:17.006 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 12 23:45:16 smithi136 bash[89273]: debug 2024-09-12T23:45:16.509+0000 7faed010a640 -1 osd.2 449 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-12T23:45:17.476 INFO:tasks.rados.rados.0.smithi136.stdout:2044: done (2 left) 2024-09-12T23:45:17.477 INFO:tasks.rados.rados.0.smithi136.stdout:2047: done (1 left) 2024-09-12T23:45:17.477 INFO:tasks.rados.rados.0.smithi136.stdout:2048: done (0 left) 2024-09-12T23:45:17.477 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 42 current snap is 197 2024-09-12T23:45:17.477 INFO:tasks.rados.rados.0.smithi136.stdout:2049: seq_num 672 ranges {3309568=458752,3768320=647168,4415488=466944,4882432=90112} 2024-09-12T23:45:17.498 INFO:tasks.rados.rados.0.smithi136.stdout:2049: writing smithi13667081-42 from 3309568 to 3768320 tid 1 2024-09-12T23:45:17.503 INFO:tasks.rados.rados.0.smithi136.stdout:2049: writing smithi13667081-42 from 3768320 to 4415488 tid 2 2024-09-12T23:45:17.506 INFO:tasks.rados.rados.0.smithi136.stdout:2049: writing smithi13667081-42 from 4415488 to 4882432 tid 3 2024-09-12T23:45:17.507 INFO:tasks.rados.rados.0.smithi136.stdout:2049: writing smithi13667081-42 from 4882432 to 4972544 tid 4 2024-09-12T23:45:17.507 INFO:tasks.rados.rados.0.smithi136.stdout:2050: snap_create 2024-09-12T23:45:17.510 INFO:tasks.rados.rados.0.smithi136.stdout:2049: finishing write tid 1 to smithi13667081-42 2024-09-12T23:45:17.519 INFO:tasks.rados.rados.0.smithi136.stdout:2049: finishing write tid 2 to smithi13667081-42 2024-09-12T23:45:17.521 INFO:tasks.rados.rados.0.smithi136.stdout:2049: finishing write tid 3 to smithi13667081-42 2024-09-12T23:45:17.521 INFO:tasks.rados.rados.0.smithi136.stdout:2049: finishing write tid 4 to smithi13667081-42 2024-09-12T23:45:17.521 INFO:tasks.rados.rados.0.smithi136.stdout:2049: finishing write tid 5 to smithi13667081-42 2024-09-12T23:45:17.522 INFO:tasks.rados.rados.0.smithi136.stdout:2049: finishing write tid 6 to smithi13667081-42 2024-09-12T23:45:17.522 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 42 v 838 (ObjNum 672 snap 197 seq_num 672) dirty exists 2024-09-12T23:45:17.522 INFO:tasks.rados.rados.0.smithi136.stdout:2049: left oid 42 (ObjNum 672 snap 197 seq_num 672) 2024-09-12T23:45:17.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:17 smithi136 bash[75164]: audit 2024-09-12T23:45:16.448201+0000 mon.a (mon.0) 523 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-12T23:45:17.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:17 smithi136 bash[75164]: cluster 2024-09-12T23:45:16.448389+0000 mon.a (mon.0) 524 : cluster [DBG] osdmap e462: 8 total, 7 up, 8 in 2024-09-12T23:45:17.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:17 smithi136 bash[75164]: audit 2024-09-12T23:45:16.451040+0000 mon.c (mon.2) 8 : audit [INF] from='osd.2 [v2:172.21.15.136:6818/439422470,v1:172.21.15.136:6819/439422470]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-12T23:45:17.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:17 smithi136 bash[75164]: audit 2024-09-12T23:45:16.467104+0000 mon.a (mon.0) 525 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-12T23:45:17.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:17 smithi136 bash[75164]: cluster 2024-09-12T23:45:16.718198+0000 mgr.y (mgr.24661) 170 : cluster [DBG] pgmap v268: 121 pgs: 28 active+undersized, 11 active+undersized+degraded, 82 active+clean; 230 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 8.0 MiB/s wr, 29 op/s; 65/399 objects degraded (16.291%) 2024-09-12T23:45:17.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:17 smithi136 bash[76371]: audit 2024-09-12T23:45:16.448201+0000 mon.a (mon.0) 523 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-12T23:45:17.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:17 smithi136 bash[76371]: cluster 2024-09-12T23:45:16.448389+0000 mon.a (mon.0) 524 : cluster [DBG] osdmap e462: 8 total, 7 up, 8 in 2024-09-12T23:45:17.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:17 smithi136 bash[76371]: audit 2024-09-12T23:45:16.451040+0000 mon.c (mon.2) 8 : audit [INF] from='osd.2 [v2:172.21.15.136:6818/439422470,v1:172.21.15.136:6819/439422470]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-12T23:45:17.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:17 smithi136 bash[76371]: audit 2024-09-12T23:45:16.467104+0000 mon.a (mon.0) 525 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-12T23:45:17.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:17 smithi136 bash[76371]: cluster 2024-09-12T23:45:16.718198+0000 mgr.y (mgr.24661) 170 : cluster [DBG] pgmap v268: 121 pgs: 28 active+undersized, 11 active+undersized+degraded, 82 active+clean; 230 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 8.0 MiB/s wr, 29 op/s; 65/399 objects degraded (16.291%) 2024-09-12T23:45:17.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:17 smithi181 bash[67587]: audit 2024-09-12T23:45:16.448201+0000 mon.a (mon.0) 523 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-12T23:45:17.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:17 smithi181 bash[67587]: cluster 2024-09-12T23:45:16.448389+0000 mon.a (mon.0) 524 : cluster [DBG] osdmap e462: 8 total, 7 up, 8 in 2024-09-12T23:45:17.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:17 smithi181 bash[67587]: audit 2024-09-12T23:45:16.451040+0000 mon.c (mon.2) 8 : audit [INF] from='osd.2 [v2:172.21.15.136:6818/439422470,v1:172.21.15.136:6819/439422470]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-12T23:45:17.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:17 smithi181 bash[67587]: audit 2024-09-12T23:45:16.467104+0000 mon.a (mon.0) 525 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-12T23:45:17.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:17 smithi181 bash[67587]: cluster 2024-09-12T23:45:16.718198+0000 mgr.y (mgr.24661) 170 : cluster [DBG] pgmap v268: 121 pgs: 28 active+undersized, 11 active+undersized+degraded, 82 active+clean; 230 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 8.0 MiB/s wr, 29 op/s; 65/399 objects degraded (16.291%) 2024-09-12T23:45:18.477 INFO:tasks.rados.rados.0.smithi136.stdout:2049: done (1 left) 2024-09-12T23:45:18.477 INFO:tasks.rados.rados.0.smithi136.stdout:2050: done (0 left) 2024-09-12T23:45:18.477 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 18 current snap is 198 2024-09-12T23:45:18.477 INFO:tasks.rados.rados.0.smithi136.stdout:2051: seq_num 673 ranges {3760128=475136,4235264=532480,4767744=671744} 2024-09-12T23:45:18.502 INFO:tasks.rados.rados.0.smithi136.stdout:2051: writing smithi13667081-18 from 3760128 to 4235264 tid 1 2024-09-12T23:45:18.506 INFO:tasks.rados.rados.0.smithi136.stdout:2051: writing smithi13667081-18 from 4235264 to 4767744 tid 2 2024-09-12T23:45:18.511 INFO:tasks.rados.rados.0.smithi136.stdout:2051: writing smithi13667081-18 from 4767744 to 5439488 tid 3 2024-09-12T23:45:18.512 INFO:tasks.rados.rados.0.smithi136.stdout:2052: snap_create 2024-09-12T23:45:18.513 INFO:tasks.rados.rados.0.smithi136.stdout:2051: finishing write tid 1 to smithi13667081-18 2024-09-12T23:45:18.514 INFO:tasks.rados.rados.0.smithi136.stdout:2051: finishing write tid 2 to smithi13667081-18 2024-09-12T23:45:18.523 INFO:tasks.rados.rados.0.smithi136.stdout:2051: finishing write tid 3 to smithi13667081-18 2024-09-12T23:45:18.524 INFO:tasks.rados.rados.0.smithi136.stdout:2051: finishing write tid 4 to smithi13667081-18 2024-09-12T23:45:18.525 INFO:tasks.rados.rados.0.smithi136.stdout:2051: finishing write tid 5 to smithi13667081-18 2024-09-12T23:45:18.525 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 18 v 930 (ObjNum 673 snap 198 seq_num 673) dirty exists 2024-09-12T23:45:18.526 INFO:tasks.rados.rados.0.smithi136.stdout:2051: left oid 18 (ObjNum 673 snap 198 seq_num 673) 2024-09-12T23:45:18.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:18 smithi136 bash[76371]: cluster 2024-09-12T23:45:17.454510+0000 mon.a (mon.0) 526 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-12T23:45:18.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:18 smithi136 bash[76371]: cluster 2024-09-12T23:45:17.472228+0000 mon.a (mon.0) 527 : cluster [INF] osd.2 [v2:172.21.15.136:6818/439422470,v1:172.21.15.136:6819/439422470] boot 2024-09-12T23:45:18.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:18 smithi136 bash[76371]: cluster 2024-09-12T23:45:17.472366+0000 mon.a (mon.0) 528 : cluster [DBG] osdmap e463: 8 total, 8 up, 8 in 2024-09-12T23:45:18.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:18 smithi136 bash[76371]: audit 2024-09-12T23:45:17.480219+0000 mon.a (mon.0) 529 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-12T23:45:18.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:18 smithi136 bash[75164]: cluster 2024-09-12T23:45:17.454510+0000 mon.a (mon.0) 526 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-12T23:45:18.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:18 smithi136 bash[75164]: cluster 2024-09-12T23:45:17.472228+0000 mon.a (mon.0) 527 : cluster [INF] osd.2 [v2:172.21.15.136:6818/439422470,v1:172.21.15.136:6819/439422470] boot 2024-09-12T23:45:18.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:18 smithi136 bash[75164]: cluster 2024-09-12T23:45:17.472366+0000 mon.a (mon.0) 528 : cluster [DBG] osdmap e463: 8 total, 8 up, 8 in 2024-09-12T23:45:18.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:18 smithi136 bash[75164]: audit 2024-09-12T23:45:17.480219+0000 mon.a (mon.0) 529 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-12T23:45:18.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:18 smithi181 bash[67587]: cluster 2024-09-12T23:45:17.454510+0000 mon.a (mon.0) 526 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-12T23:45:18.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:18 smithi181 bash[67587]: cluster 2024-09-12T23:45:17.472228+0000 mon.a (mon.0) 527 : cluster [INF] osd.2 [v2:172.21.15.136:6818/439422470,v1:172.21.15.136:6819/439422470] boot 2024-09-12T23:45:18.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:18 smithi181 bash[67587]: cluster 2024-09-12T23:45:17.472366+0000 mon.a (mon.0) 528 : cluster [DBG] osdmap e463: 8 total, 8 up, 8 in 2024-09-12T23:45:18.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:18 smithi181 bash[67587]: audit 2024-09-12T23:45:17.480219+0000 mon.a (mon.0) 529 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-12T23:45:18.845 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:45:18 smithi181 bash[60471]: ts=2024-09-12T23:45:18.661Z caller=alerting.go:366 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-09-12T23:45:18.845 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:45:18 smithi181 bash[60471]: ts=2024-09-12T23:45:18.664Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:45:19.498 INFO:tasks.rados.rados.0.smithi136.stdout:2051: done (1 left) 2024-09-12T23:45:19.498 INFO:tasks.rados.rados.0.smithi136.stdout:2052: done (0 left) 2024-09-12T23:45:19.498 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 17 current snap is 199 2024-09-12T23:45:19.499 INFO:tasks.rados.rados.0.smithi136.stdout:2053: seq_num 674 ranges {368640=491520,860160=417792,1277952=483328,1761280=303104} 2024-09-12T23:45:19.504 INFO:tasks.rados.rados.0.smithi136.stdout:2053: writing smithi13667081-17 from 368640 to 860160 tid 1 2024-09-12T23:45:19.507 INFO:tasks.rados.rados.0.smithi136.stdout:2053: writing smithi13667081-17 from 860160 to 1277952 tid 2 2024-09-12T23:45:19.511 INFO:tasks.rados.rados.0.smithi136.stdout:2053: writing smithi13667081-17 from 1277952 to 1761280 tid 3 2024-09-12T23:45:19.514 INFO:tasks.rados.rados.0.smithi136.stdout:2053: writing smithi13667081-17 from 1761280 to 2064384 tid 4 2024-09-12T23:45:19.514 INFO:tasks.rados.rados.0.smithi136.stdout:2054: rollback oid 22 current snap is 199 2024-09-12T23:45:19.514 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 22 to 184 2024-09-12T23:45:19.514 INFO:tasks.rados.rados.0.smithi136.stdout:2055: delete oid 39 current snap is 199 2024-09-12T23:45:19.524 INFO:tasks.rados.rados.0.smithi136.stdout:2053: finishing write tid 1 to smithi13667081-17 2024-09-12T23:45:19.524 INFO:tasks.rados.rados.0.smithi136.stdout:2055: done (2 left) 2024-09-12T23:45:19.524 INFO:tasks.rados.rados.0.smithi136.stdout:2056: snap_remove snap 189 2024-09-12T23:45:19.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:19 smithi136 bash[75164]: cluster 2024-09-12T23:45:18.471443+0000 mon.a (mon.0) 530 : cluster [DBG] osdmap e464: 8 total, 8 up, 8 in 2024-09-12T23:45:19.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:19 smithi136 bash[75164]: cluster 2024-09-12T23:45:18.718768+0000 mgr.y (mgr.24661) 171 : cluster [DBG] pgmap v271: 121 pgs: 4 peering, 25 active+undersized, 10 active+undersized+degraded, 82 active+clean; 238 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.2 MiB/s rd, 2.9 MiB/s wr, 12 op/s; 63/414 objects degraded (15.217%) 2024-09-12T23:45:19.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:45:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:45:19] "GET /metrics HTTP/1.1" 200 36399 "" "Prometheus/2.43.0" 2024-09-12T23:45:19.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:19 smithi136 bash[76371]: cluster 2024-09-12T23:45:18.471443+0000 mon.a (mon.0) 530 : cluster [DBG] osdmap e464: 8 total, 8 up, 8 in 2024-09-12T23:45:19.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:19 smithi136 bash[76371]: cluster 2024-09-12T23:45:18.718768+0000 mgr.y (mgr.24661) 171 : cluster [DBG] pgmap v271: 121 pgs: 4 peering, 25 active+undersized, 10 active+undersized+degraded, 82 active+clean; 238 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.2 MiB/s rd, 2.9 MiB/s wr, 12 op/s; 63/414 objects degraded (15.217%) 2024-09-12T23:45:19.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:19 smithi181 bash[67587]: cluster 2024-09-12T23:45:18.471443+0000 mon.a (mon.0) 530 : cluster [DBG] osdmap e464: 8 total, 8 up, 8 in 2024-09-12T23:45:19.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:19 smithi181 bash[67587]: cluster 2024-09-12T23:45:18.718768+0000 mgr.y (mgr.24661) 171 : cluster [DBG] pgmap v271: 121 pgs: 4 peering, 25 active+undersized, 10 active+undersized+degraded, 82 active+clean; 238 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.2 MiB/s rd, 2.9 MiB/s wr, 12 op/s; 63/414 objects degraded (15.217%) 2024-09-12T23:45:20.498 INFO:tasks.rados.rados.0.smithi136.stdout:2056: done (2 left) 2024-09-12T23:45:20.498 INFO:tasks.rados.rados.0.smithi136.stdout:2057: snap_create 2024-09-12T23:45:20.499 INFO:tasks.rados.rados.0.smithi136.stdout:2053: finishing write tid 2 to smithi13667081-17 2024-09-12T23:45:20.499 INFO:tasks.rados.rados.0.smithi136.stdout:2054: finishing rollback tid 0 to smithi13667081-22 2024-09-12T23:45:20.499 INFO:tasks.rados.rados.0.smithi136.stdout:2054: finishing rollback tid 1 to smithi13667081-22 2024-09-12T23:45:20.499 INFO:tasks.rados.rados.0.smithi136.stdout:2054: finishing rollback tid 2 to smithi13667081-22 2024-09-12T23:45:20.499 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 22 v 841 (ObjNum 548 snap 158 seq_num 548) dirty exists 2024-09-12T23:45:20.499 INFO:tasks.rados.rados.0.smithi136.stdout:2053: finishing write tid 3 to smithi13667081-17 2024-09-12T23:45:20.499 INFO:tasks.rados.rados.0.smithi136.stdout:2053: finishing write tid 4 to smithi13667081-17 2024-09-12T23:45:20.499 INFO:tasks.rados.rados.0.smithi136.stdout:2053: finishing write tid 5 to smithi13667081-17 2024-09-12T23:45:20.499 INFO:tasks.rados.rados.0.smithi136.stdout:2053: finishing write tid 6 to smithi13667081-17 2024-09-12T23:45:20.499 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 17 v 1425 (ObjNum 674 snap 199 seq_num 674) dirty exists 2024-09-12T23:45:20.500 INFO:tasks.rados.rados.0.smithi136.stdout:2053: left oid 17 (ObjNum 674 snap 199 seq_num 674) 2024-09-12T23:45:20.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:20 smithi136 bash[75164]: cluster 2024-09-12T23:45:19.502021+0000 mon.a (mon.0) 531 : cluster [DBG] osdmap e465: 8 total, 8 up, 8 in 2024-09-12T23:45:20.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:20 smithi136 bash[76371]: cluster 2024-09-12T23:45:19.502021+0000 mon.a (mon.0) 531 : cluster [DBG] osdmap e465: 8 total, 8 up, 8 in 2024-09-12T23:45:20.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:20 smithi181 bash[67587]: cluster 2024-09-12T23:45:19.502021+0000 mon.a (mon.0) 531 : cluster [DBG] osdmap e465: 8 total, 8 up, 8 in 2024-09-12T23:45:21.499 INFO:tasks.rados.rados.0.smithi136.stdout:2053: done (2 left) 2024-09-12T23:45:21.499 INFO:tasks.rados.rados.0.smithi136.stdout:2054: done (1 left) 2024-09-12T23:45:21.500 INFO:tasks.rados.rados.0.smithi136.stdout:2057: done (0 left) 2024-09-12T23:45:21.500 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 10 current snap is 200 2024-09-12T23:45:21.500 INFO:tasks.rados.rados.0.smithi136.stdout:2058: seq_num 676 ranges {4448256=524288,4972544=589824,5562368=450560,6012928=163840} 2024-09-12T23:45:21.542 INFO:tasks.rados.rados.0.smithi136.stdout:2058: writing smithi13667081-10 from 4448256 to 4972544 tid 1 2024-09-12T23:45:21.547 INFO:tasks.rados.rados.0.smithi136.stdout:2058: writing smithi13667081-10 from 4972544 to 5562368 tid 2 2024-09-12T23:45:21.552 INFO:tasks.rados.rados.0.smithi136.stdout:2058: writing smithi13667081-10 from 5562368 to 6012928 tid 3 2024-09-12T23:45:21.553 INFO:tasks.rados.rados.0.smithi136.stdout:2058: writing smithi13667081-10 from 6012928 to 6176768 tid 4 2024-09-12T23:45:21.553 INFO:tasks.rados.rados.0.smithi136.stdout:2059: snap_create 2024-09-12T23:45:21.553 INFO:tasks.rados.rados.0.smithi136.stdout:2058: finishing write tid 1 to smithi13667081-10 2024-09-12T23:45:21.558 INFO:tasks.rados.rados.0.smithi136.stdout:2058: finishing write tid 2 to smithi13667081-10 2024-09-12T23:45:21.562 INFO:tasks.rados.rados.0.smithi136.stdout:2058: finishing write tid 3 to smithi13667081-10 2024-09-12T23:45:21.562 INFO:tasks.rados.rados.0.smithi136.stdout:2058: finishing write tid 4 to smithi13667081-10 2024-09-12T23:45:21.563 INFO:tasks.rados.rados.0.smithi136.stdout:2058: finishing write tid 5 to smithi13667081-10 2024-09-12T23:45:21.565 INFO:tasks.rados.rados.0.smithi136.stdout:2058: finishing write tid 6 to smithi13667081-10 2024-09-12T23:45:21.566 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 10 v 1090 (ObjNum 676 snap 200 seq_num 676) dirty exists 2024-09-12T23:45:21.566 INFO:tasks.rados.rados.0.smithi136.stdout:2058: left oid 10 (ObjNum 676 snap 200 seq_num 676) 2024-09-12T23:45:21.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:21 smithi181 bash[67587]: cluster 2024-09-12T23:45:20.500943+0000 mon.a (mon.0) 532 : cluster [DBG] osdmap e466: 8 total, 8 up, 8 in 2024-09-12T23:45:21.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:21 smithi181 bash[67587]: cluster 2024-09-12T23:45:20.719590+0000 mgr.y (mgr.24661) 172 : cluster [DBG] pgmap v274: 121 pgs: 4 peering, 14 active+undersized, 7 active+undersized+degraded, 96 active+clean; 246 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.2 MiB/s rd, 3.3 MiB/s wr, 14 op/s; 52/423 objects degraded (12.293%); 8.3 MiB/s, 1 objects/s recovering 2024-09-12T23:45:21.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:21 smithi181 bash[67587]: cluster 2024-09-12T23:45:21.335769+0000 mon.a (mon.0) 533 : cluster [WRN] Health check update: Degraded data redundancy: 63/414 objects degraded (15.217%), 10 pgs degraded (PG_DEGRADED) 2024-09-12T23:45:21.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:21 smithi181 bash[67587]: cluster 2024-09-12T23:45:21.503410+0000 mon.a (mon.0) 534 : cluster [DBG] osdmap e467: 8 total, 8 up, 8 in 2024-09-12T23:45:21.898 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:21 smithi136 bash[75164]: cluster 2024-09-12T23:45:20.500943+0000 mon.a (mon.0) 532 : cluster [DBG] osdmap e466: 8 total, 8 up, 8 in 2024-09-12T23:45:21.898 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:21 smithi136 bash[75164]: cluster 2024-09-12T23:45:20.719590+0000 mgr.y (mgr.24661) 172 : cluster [DBG] pgmap v274: 121 pgs: 4 peering, 14 active+undersized, 7 active+undersized+degraded, 96 active+clean; 246 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.2 MiB/s rd, 3.3 MiB/s wr, 14 op/s; 52/423 objects degraded (12.293%); 8.3 MiB/s, 1 objects/s recovering 2024-09-12T23:45:21.899 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:21 smithi136 bash[75164]: cluster 2024-09-12T23:45:21.335769+0000 mon.a (mon.0) 533 : cluster [WRN] Health check update: Degraded data redundancy: 63/414 objects degraded (15.217%), 10 pgs degraded (PG_DEGRADED) 2024-09-12T23:45:21.899 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:21 smithi136 bash[75164]: cluster 2024-09-12T23:45:21.503410+0000 mon.a (mon.0) 534 : cluster [DBG] osdmap e467: 8 total, 8 up, 8 in 2024-09-12T23:45:21.899 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:21 smithi136 bash[76371]: cluster 2024-09-12T23:45:20.500943+0000 mon.a (mon.0) 532 : cluster [DBG] osdmap e466: 8 total, 8 up, 8 in 2024-09-12T23:45:21.899 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:21 smithi136 bash[76371]: cluster 2024-09-12T23:45:20.719590+0000 mgr.y (mgr.24661) 172 : cluster [DBG] pgmap v274: 121 pgs: 4 peering, 14 active+undersized, 7 active+undersized+degraded, 96 active+clean; 246 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.2 MiB/s rd, 3.3 MiB/s wr, 14 op/s; 52/423 objects degraded (12.293%); 8.3 MiB/s, 1 objects/s recovering 2024-09-12T23:45:21.899 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:21 smithi136 bash[76371]: cluster 2024-09-12T23:45:21.335769+0000 mon.a (mon.0) 533 : cluster [WRN] Health check update: Degraded data redundancy: 63/414 objects degraded (15.217%), 10 pgs degraded (PG_DEGRADED) 2024-09-12T23:45:21.899 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:21 smithi136 bash[76371]: cluster 2024-09-12T23:45:21.503410+0000 mon.a (mon.0) 534 : cluster [DBG] osdmap e467: 8 total, 8 up, 8 in 2024-09-12T23:45:22.522 INFO:tasks.rados.rados.0.smithi136.stdout:2058: done (1 left) 2024-09-12T23:45:22.523 INFO:tasks.rados.rados.0.smithi136.stdout:2059: done (0 left) 2024-09-12T23:45:22.523 INFO:tasks.rados.rados.0.smithi136.stdout:2060: rollback oid 27 current snap is 201 2024-09-12T23:45:22.523 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 27 to 193 2024-09-12T23:45:22.523 INFO:tasks.rados.rados.0.smithi136.stdout:2061: delete oid 39 current snap is 201 2024-09-12T23:45:22.537 INFO:tasks.rados.rados.0.smithi136.stdout:2061: done (1 left) 2024-09-12T23:45:22.538 INFO:tasks.rados.rados.0.smithi136.stdout:2062: rollback oid 5 current snap is 201 2024-09-12T23:45:22.538 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 5 to 198 2024-09-12T23:45:22.538 INFO:tasks.rados.rados.0.smithi136.stdout:2063: copy_from oid 6 from oid 24 current snap is 201 2024-09-12T23:45:22.538 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 17 current snap is 201 2024-09-12T23:45:22.538 INFO:tasks.rados.rados.0.smithi136.stdout:2064: seq_num 678 ranges {2064384=565248,2629632=802816,3432448=401408} 2024-09-12T23:45:22.559 INFO:tasks.rados.rados.0.smithi136.stdout:2064: writing smithi13667081-17 from 2064384 to 2629632 tid 1 2024-09-12T23:45:22.565 INFO:tasks.rados.rados.0.smithi136.stdout:2064: writing smithi13667081-17 from 2629632 to 3432448 tid 2 2024-09-12T23:45:22.569 INFO:tasks.rados.rados.0.smithi136.stdout:2064: writing smithi13667081-17 from 3432448 to 3833856 tid 3 2024-09-12T23:45:22.569 INFO:tasks.rados.rados.0.smithi136.stdout:2065: snap_create 2024-09-12T23:45:22.570 INFO:tasks.rados.rados.0.smithi136.stdout:2060: finishing rollback tid 0 to smithi13667081-27 2024-09-12T23:45:22.570 INFO:tasks.rados.rados.0.smithi136.stdout:2060: finishing rollback tid 1 to smithi13667081-27 2024-09-12T23:45:22.570 INFO:tasks.rados.rados.0.smithi136.stdout:2063: finishing copy_from to smithi13667081-6 2024-09-12T23:45:22.570 INFO:tasks.rados.rados.0.smithi136.stdout:2063: got expected ENOENT (src dne) 2024-09-12T23:45:22.570 INFO:tasks.rados.rados.0.smithi136.stdout:2063: finishing copy_from racing read to smithi13667081-6 2024-09-12T23:45:22.570 INFO:tasks.rados.rados.0.smithi136.stdout:2060: finishing rollback tid 2 to smithi13667081-27 2024-09-12T23:45:22.570 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 27 v 1437 (ObjNum 416 snap 120 seq_num 416) dirty exists 2024-09-12T23:45:22.570 INFO:tasks.rados.rados.0.smithi136.stdout:2062: finishing rollback tid 0 to smithi13667081-5 2024-09-12T23:45:22.570 INFO:tasks.rados.rados.0.smithi136.stdout:2062: finishing rollback tid 1 to smithi13667081-5 2024-09-12T23:45:22.570 INFO:tasks.rados.rados.0.smithi136.stdout:2062: finishing rollback tid 2 to smithi13667081-5 2024-09-12T23:45:22.570 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 5 v 1441 (ObjNum 567 snap 162 seq_num 567) dirty exists 2024-09-12T23:45:22.570 INFO:tasks.rados.rados.0.smithi136.stdout:2064: finishing write tid 1 to smithi13667081-17 2024-09-12T23:45:22.574 INFO:tasks.rados.rados.0.smithi136.stdout:2064: finishing write tid 2 to smithi13667081-17 2024-09-12T23:45:22.575 INFO:tasks.rados.rados.0.smithi136.stdout:2064: finishing write tid 3 to smithi13667081-17 2024-09-12T23:45:22.575 INFO:tasks.rados.rados.0.smithi136.stdout:2064: finishing write tid 4 to smithi13667081-17 2024-09-12T23:45:22.577 INFO:tasks.rados.rados.0.smithi136.stdout:2064: finishing write tid 5 to smithi13667081-17 2024-09-12T23:45:22.577 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 17 v 1446 (ObjNum 678 snap 201 seq_num 678) dirty exists 2024-09-12T23:45:22.578 INFO:tasks.rados.rados.0.smithi136.stdout:2064: left oid 17 (ObjNum 678 snap 201 seq_num 678) 2024-09-12T23:45:22.594 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:45:22 smithi181 bash[60471]: ts=2024-09-12T23:45:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:45:23.539 INFO:tasks.rados.rados.0.smithi136.stdout:2060: done (4 left) 2024-09-12T23:45:23.539 INFO:tasks.rados.rados.0.smithi136.stdout:2062: done (3 left) 2024-09-12T23:45:23.539 INFO:tasks.rados.rados.0.smithi136.stdout:2063: done (2 left) 2024-09-12T23:45:23.539 INFO:tasks.rados.rados.0.smithi136.stdout:2064: done (1 left) 2024-09-12T23:45:23.539 INFO:tasks.rados.rados.0.smithi136.stdout:2065: done (0 left) 2024-09-12T23:45:23.540 INFO:tasks.rados.rados.0.smithi136.stdout:2066: snap_remove snap 197 2024-09-12T23:45:23.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:23 smithi181 bash[67587]: cluster 2024-09-12T23:45:22.526414+0000 mon.a (mon.0) 535 : cluster [DBG] osdmap e468: 8 total, 8 up, 8 in 2024-09-12T23:45:23.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:23 smithi181 bash[67587]: cluster 2024-09-12T23:45:22.720161+0000 mgr.y (mgr.24661) 173 : cluster [DBG] pgmap v277: 121 pgs: 4 peering, 117 active+clean; 247 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.0 MiB/s wr, 2 op/s; 35 MiB/s, 7 objects/s recovering 2024-09-12T23:45:23.844 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:45:23 smithi181 bash[60471]: ts=2024-09-12T23:45:23.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi136\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:45:24.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:23 smithi136 bash[75164]: cluster 2024-09-12T23:45:22.526414+0000 mon.a (mon.0) 535 : cluster [DBG] osdmap e468: 8 total, 8 up, 8 in 2024-09-12T23:45:24.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:23 smithi136 bash[75164]: cluster 2024-09-12T23:45:22.720161+0000 mgr.y (mgr.24661) 173 : cluster [DBG] pgmap v277: 121 pgs: 4 peering, 117 active+clean; 247 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.0 MiB/s wr, 2 op/s; 35 MiB/s, 7 objects/s recovering 2024-09-12T23:45:24.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:23 smithi136 bash[76371]: cluster 2024-09-12T23:45:22.526414+0000 mon.a (mon.0) 535 : cluster [DBG] osdmap e468: 8 total, 8 up, 8 in 2024-09-12T23:45:24.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:23 smithi136 bash[76371]: cluster 2024-09-12T23:45:22.720161+0000 mgr.y (mgr.24661) 173 : cluster [DBG] pgmap v277: 121 pgs: 4 peering, 117 active+clean; 247 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.0 MiB/s wr, 2 op/s; 35 MiB/s, 7 objects/s recovering 2024-09-12T23:45:24.541 INFO:tasks.rados.rados.0.smithi136.stdout:2066: done (0 left) 2024-09-12T23:45:24.541 INFO:tasks.rados.rados.0.smithi136.stdout:2067: read oid 42 snap 184 2024-09-12T23:45:24.541 INFO:tasks.rados.rados.0.smithi136.stdout:2067: expect (ObjNum 624 snap 184 seq_num 624) 2024-09-12T23:45:24.541 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 33 current snap is 202 2024-09-12T23:45:24.542 INFO:tasks.rados.rados.0.smithi136.stdout:2068: seq_num 679 ranges {0=581632,581632=688128,1269760=516096} 2024-09-12T23:45:24.546 INFO:tasks.rados.rados.0.smithi136.stdout:2068: writing smithi13667081-33 from 0 to 581632 tid 1 2024-09-12T23:45:24.552 INFO:tasks.rados.rados.0.smithi136.stdout:2068: writing smithi13667081-33 from 581632 to 1269760 tid 2 2024-09-12T23:45:24.556 INFO:tasks.rados.rados.0.smithi136.stdout:2068: writing smithi13667081-33 from 1269760 to 1785856 tid 3 2024-09-12T23:45:24.556 INFO:tasks.rados.rados.0.smithi136.stdout:2069: delete oid 6 current snap is 202 2024-09-12T23:45:24.563 INFO:tasks.rados.rados.0.smithi136.stdout:2069: done (2 left) 2024-09-12T23:45:24.563 INFO:tasks.rados.rados.0.smithi136.stdout:2070: read oid 24 snap -1 2024-09-12T23:45:24.563 INFO:tasks.rados.rados.0.smithi136.stdout:2070: expect deleted 2024-09-12T23:45:24.563 INFO:tasks.rados.rados.0.smithi136.stdout:2071: read oid 39 snap -1 2024-09-12T23:45:24.563 INFO:tasks.rados.rados.0.smithi136.stdout:2071: expect deleted 2024-09-12T23:45:24.563 INFO:tasks.rados.rados.0.smithi136.stdout:2072: snap_create 2024-09-12T23:45:24.567 INFO:tasks.rados.rados.0.smithi136.stdout:2067: expect (ObjNum 624 snap 184 seq_num 624) 2024-09-12T23:45:24.731 INFO:tasks.rados.rados.0.smithi136.stdout:2068: finishing write tid 1 to smithi13667081-33 2024-09-12T23:45:24.731 INFO:tasks.rados.rados.0.smithi136.stdout:2068: finishing write tid 2 to smithi13667081-33 2024-09-12T23:45:24.731 INFO:tasks.rados.rados.0.smithi136.stdout:2068: finishing write tid 3 to smithi13667081-33 2024-09-12T23:45:24.731 INFO:tasks.rados.rados.0.smithi136.stdout:2068: finishing write tid 4 to smithi13667081-33 2024-09-12T23:45:24.731 INFO:tasks.rados.rados.0.smithi136.stdout:2068: finishing write tid 5 to smithi13667081-33 2024-09-12T23:45:24.732 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 33 v 1231 (ObjNum 679 snap 202 seq_num 679) dirty exists 2024-09-12T23:45:24.732 INFO:tasks.rados.rados.0.smithi136.stdout:2068: left oid 33 (ObjNum 679 snap 202 seq_num 679) 2024-09-12T23:45:24.844 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:45:24 smithi181 bash[60471]: ts=2024-09-12T23:45:24.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:45:24.845 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:45:24 smithi181 bash[60471]: ts=2024-09-12T23:45:24.456Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:45:24.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:24 smithi181 bash[67587]: cluster 2024-09-12T23:45:23.524015+0000 mon.a (mon.0) 536 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 52/423 objects degraded (12.293%), 7 pgs degraded) 2024-09-12T23:45:24.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:24 smithi181 bash[67587]: cluster 2024-09-12T23:45:23.524080+0000 mon.a (mon.0) 537 : cluster [INF] Cluster is now healthy 2024-09-12T23:45:24.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:24 smithi181 bash[67587]: cluster 2024-09-12T23:45:23.543211+0000 mon.a (mon.0) 538 : cluster [DBG] osdmap e469: 8 total, 8 up, 8 in 2024-09-12T23:45:24.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:24 smithi181 bash[67587]: audit 2024-09-12T23:45:23.749699+0000 mon.a (mon.0) 539 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:24.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:24 smithi181 bash[67587]: audit 2024-09-12T23:45:23.750879+0000 mon.a (mon.0) 540 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:45:24.915 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:24 smithi136 bash[75164]: cluster 2024-09-12T23:45:23.524015+0000 mon.a (mon.0) 536 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 52/423 objects degraded (12.293%), 7 pgs degraded) 2024-09-12T23:45:24.916 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:24 smithi136 bash[75164]: cluster 2024-09-12T23:45:23.524080+0000 mon.a (mon.0) 537 : cluster [INF] Cluster is now healthy 2024-09-12T23:45:24.916 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:24 smithi136 bash[75164]: cluster 2024-09-12T23:45:23.543211+0000 mon.a (mon.0) 538 : cluster [DBG] osdmap e469: 8 total, 8 up, 8 in 2024-09-12T23:45:24.916 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:24 smithi136 bash[75164]: audit 2024-09-12T23:45:23.749699+0000 mon.a (mon.0) 539 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:24.916 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:24 smithi136 bash[75164]: audit 2024-09-12T23:45:23.750879+0000 mon.a (mon.0) 540 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:45:24.916 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:24 smithi136 bash[76371]: cluster 2024-09-12T23:45:23.524015+0000 mon.a (mon.0) 536 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 52/423 objects degraded (12.293%), 7 pgs degraded) 2024-09-12T23:45:24.916 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:24 smithi136 bash[76371]: cluster 2024-09-12T23:45:23.524080+0000 mon.a (mon.0) 537 : cluster [INF] Cluster is now healthy 2024-09-12T23:45:24.916 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:24 smithi136 bash[76371]: cluster 2024-09-12T23:45:23.543211+0000 mon.a (mon.0) 538 : cluster [DBG] osdmap e469: 8 total, 8 up, 8 in 2024-09-12T23:45:24.916 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:24 smithi136 bash[76371]: audit 2024-09-12T23:45:23.749699+0000 mon.a (mon.0) 539 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:24.917 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:24 smithi136 bash[76371]: audit 2024-09-12T23:45:23.750879+0000 mon.a (mon.0) 540 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:45:25.556 INFO:tasks.rados.rados.0.smithi136.stdout:2067: done (4 left) 2024-09-12T23:45:25.556 INFO:tasks.rados.rados.0.smithi136.stdout:2068: done (3 left) 2024-09-12T23:45:25.556 INFO:tasks.rados.rados.0.smithi136.stdout:2070: done (2 left) 2024-09-12T23:45:25.556 INFO:tasks.rados.rados.0.smithi136.stdout:2071: done (1 left) 2024-09-12T23:45:25.557 INFO:tasks.rados.rados.0.smithi136.stdout:2072: done (0 left) 2024-09-12T23:45:25.557 INFO:tasks.rados.rados.0.smithi136.stdout:2073: delete oid 33 current snap is 203 2024-09-12T23:45:25.563 INFO:tasks.rados.rados.0.smithi136.stdout:2073: done (0 left) 2024-09-12T23:45:25.563 INFO:tasks.rados.rados.0.smithi136.stdout:2074: copy_from oid 29 from oid 38 current snap is 203 2024-09-12T23:45:25.563 INFO:tasks.rados.rados.0.smithi136.stdout:2075: setattr oid 45 current snap is 203 2024-09-12T23:45:25.563 INFO:tasks.rados.rados.0.smithi136.stdout:2076: delete oid 12 current snap is 203 2024-09-12T23:45:25.571 INFO:tasks.rados.rados.0.smithi136.stdout:2076: done (2 left) 2024-09-12T23:45:25.571 INFO:tasks.rados.rados.0.smithi136.stdout:2077: snap_create 2024-09-12T23:45:25.576 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 45 v 799 (ObjNum 644 snap 190 seq_num 644) dirty exists 2024-09-12T23:45:25.597 INFO:tasks.rados.rados.0.smithi136.stdout:2074: finishing copy_from racing read to smithi13667081-29 2024-09-12T23:45:25.604 INFO:tasks.rados.rados.0.smithi136.stdout:2074: finishing copy_from to smithi13667081-29 2024-09-12T23:45:25.604 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 29 v 1219 (ObjNum 666 snap 195 seq_num 666) dirty exists 2024-09-12T23:45:25.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:25 smithi181 bash[67587]: cluster 2024-09-12T23:45:24.545071+0000 mon.a (mon.0) 541 : cluster [DBG] osdmap e470: 8 total, 8 up, 8 in 2024-09-12T23:45:25.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:25 smithi181 bash[67587]: cluster 2024-09-12T23:45:24.720817+0000 mgr.y (mgr.24661) 174 : cluster [DBG] pgmap v280: 121 pgs: 121 active+clean; 252 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 1.9 MiB/s wr, 6 op/s; 26 MiB/s, 6 objects/s recovering 2024-09-12T23:45:25.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:25 smithi181 bash[67587]: audit 2024-09-12T23:45:25.485077+0000 mon.a (mon.0) 542 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:25.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:25 smithi181 bash[67587]: audit 2024-09-12T23:45:25.494539+0000 mon.a (mon.0) 543 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:25.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:25 smithi181 bash[67587]: audit 2024-09-12T23:45:25.496296+0000 mon.a (mon.0) 544 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:45:25.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:25 smithi181 bash[67587]: audit 2024-09-12T23:45:25.498083+0000 mon.a (mon.0) 545 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:45:25.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:25 smithi181 bash[67587]: audit 2024-09-12T23:45:25.508329+0000 mon.a (mon.0) 546 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:25.991 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:25 smithi136 bash[75164]: cluster 2024-09-12T23:45:24.545071+0000 mon.a (mon.0) 541 : cluster [DBG] osdmap e470: 8 total, 8 up, 8 in 2024-09-12T23:45:25.992 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:25 smithi136 bash[75164]: cluster 2024-09-12T23:45:24.720817+0000 mgr.y (mgr.24661) 174 : cluster [DBG] pgmap v280: 121 pgs: 121 active+clean; 252 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 1.9 MiB/s wr, 6 op/s; 26 MiB/s, 6 objects/s recovering 2024-09-12T23:45:25.992 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:25 smithi136 bash[75164]: audit 2024-09-12T23:45:25.485077+0000 mon.a (mon.0) 542 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:25.992 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:25 smithi136 bash[75164]: audit 2024-09-12T23:45:25.494539+0000 mon.a (mon.0) 543 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:25.992 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:25 smithi136 bash[75164]: audit 2024-09-12T23:45:25.496296+0000 mon.a (mon.0) 544 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:45:25.992 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:25 smithi136 bash[75164]: audit 2024-09-12T23:45:25.498083+0000 mon.a (mon.0) 545 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:45:25.992 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:25 smithi136 bash[75164]: audit 2024-09-12T23:45:25.508329+0000 mon.a (mon.0) 546 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:25.992 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:25 smithi136 bash[76371]: cluster 2024-09-12T23:45:24.545071+0000 mon.a (mon.0) 541 : cluster [DBG] osdmap e470: 8 total, 8 up, 8 in 2024-09-12T23:45:25.992 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:25 smithi136 bash[76371]: cluster 2024-09-12T23:45:24.720817+0000 mgr.y (mgr.24661) 174 : cluster [DBG] pgmap v280: 121 pgs: 121 active+clean; 252 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 1.9 MiB/s wr, 6 op/s; 26 MiB/s, 6 objects/s recovering 2024-09-12T23:45:25.992 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:25 smithi136 bash[76371]: audit 2024-09-12T23:45:25.485077+0000 mon.a (mon.0) 542 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:25.992 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:25 smithi136 bash[76371]: audit 2024-09-12T23:45:25.494539+0000 mon.a (mon.0) 543 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:25.993 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:25 smithi136 bash[76371]: audit 2024-09-12T23:45:25.496296+0000 mon.a (mon.0) 544 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:45:25.993 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:25 smithi136 bash[76371]: audit 2024-09-12T23:45:25.498083+0000 mon.a (mon.0) 545 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:45:25.993 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:25 smithi136 bash[76371]: audit 2024-09-12T23:45:25.508329+0000 mon.a (mon.0) 546 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:26.344 INFO:tasks.rados.rados.0.smithi136.stdout:2074: done (2 left) 2024-09-12T23:45:26.344 INFO:tasks.rados.rados.0.smithi136.stdout:2075: done (1 left) 2024-09-12T23:45:26.344 INFO:tasks.rados.rados.0.smithi136.stdout:2077: done (0 left) 2024-09-12T23:45:26.344 INFO:tasks.rados.rados.0.smithi136.stdout:2078: snap_remove snap 184 2024-09-12T23:45:26.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:26 smithi181 bash[67587]: cluster 2024-09-12T23:45:25.559902+0000 mon.a (mon.0) 547 : cluster [DBG] osdmap e471: 8 total, 8 up, 8 in 2024-09-12T23:45:26.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:26 smithi181 bash[67587]: audit 2024-09-12T23:45:25.619520+0000 mon.a (mon.0) 548 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:45:26.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:26 smithi181 bash[67587]: audit 2024-09-12T23:45:25.622481+0000 mon.a (mon.0) 549 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:45:26.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:26 smithi181 bash[67587]: audit 2024-09-12T23:45:25.624231+0000 mon.a (mon.0) 550 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:45:26.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:26 smithi181 bash[67587]: audit 2024-09-12T23:45:25.625876+0000 mon.a (mon.0) 551 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:45:26.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:26 smithi181 bash[67587]: audit 2024-09-12T23:45:25.627988+0000 mon.a (mon.0) 552 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-12T23:45:26.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:26 smithi181 bash[67587]: audit 2024-09-12T23:45:25.628347+0000 mgr.y (mgr.24661) 175 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-12T23:45:26.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:26 smithi181 bash[67587]: cephadm 2024-09-12T23:45:25.629899+0000 mgr.y (mgr.24661) 176 : cephadm [INF] Upgrade: osd.3 is safe to restart 2024-09-12T23:45:26.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:26 smithi181 bash[67587]: cluster 2024-09-12T23:45:26.347320+0000 mon.a (mon.0) 553 : cluster [DBG] osdmap e472: 8 total, 8 up, 8 in 2024-09-12T23:45:26.981 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:26 smithi136 bash[75164]: cluster 2024-09-12T23:45:25.559902+0000 mon.a (mon.0) 547 : cluster [DBG] osdmap e471: 8 total, 8 up, 8 in 2024-09-12T23:45:26.981 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:26 smithi136 bash[75164]: audit 2024-09-12T23:45:25.619520+0000 mon.a (mon.0) 548 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:45:26.981 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:26 smithi136 bash[75164]: audit 2024-09-12T23:45:25.622481+0000 mon.a (mon.0) 549 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:45:26.981 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:26 smithi136 bash[75164]: audit 2024-09-12T23:45:25.624231+0000 mon.a (mon.0) 550 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:45:26.981 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:26 smithi136 bash[75164]: audit 2024-09-12T23:45:25.625876+0000 mon.a (mon.0) 551 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:45:26.982 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:26 smithi136 bash[75164]: audit 2024-09-12T23:45:25.627988+0000 mon.a (mon.0) 552 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-12T23:45:26.982 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:26 smithi136 bash[75164]: audit 2024-09-12T23:45:25.628347+0000 mgr.y (mgr.24661) 175 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-12T23:45:26.982 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:26 smithi136 bash[75164]: cephadm 2024-09-12T23:45:25.629899+0000 mgr.y (mgr.24661) 176 : cephadm [INF] Upgrade: osd.3 is safe to restart 2024-09-12T23:45:26.982 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:26 smithi136 bash[75164]: cluster 2024-09-12T23:45:26.347320+0000 mon.a (mon.0) 553 : cluster [DBG] osdmap e472: 8 total, 8 up, 8 in 2024-09-12T23:45:26.982 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:26 smithi136 bash[76371]: cluster 2024-09-12T23:45:25.559902+0000 mon.a (mon.0) 547 : cluster [DBG] osdmap e471: 8 total, 8 up, 8 in 2024-09-12T23:45:26.982 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:26 smithi136 bash[76371]: audit 2024-09-12T23:45:25.619520+0000 mon.a (mon.0) 548 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:45:26.982 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:26 smithi136 bash[76371]: audit 2024-09-12T23:45:25.622481+0000 mon.a (mon.0) 549 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:45:26.982 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:26 smithi136 bash[76371]: audit 2024-09-12T23:45:25.624231+0000 mon.a (mon.0) 550 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:45:26.982 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:26 smithi136 bash[76371]: audit 2024-09-12T23:45:25.625876+0000 mon.a (mon.0) 551 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:45:26.982 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:26 smithi136 bash[76371]: audit 2024-09-12T23:45:25.627988+0000 mon.a (mon.0) 552 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-12T23:45:26.983 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:26 smithi136 bash[76371]: audit 2024-09-12T23:45:25.628347+0000 mgr.y (mgr.24661) 175 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-12T23:45:26.983 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:26 smithi136 bash[76371]: cephadm 2024-09-12T23:45:25.629899+0000 mgr.y (mgr.24661) 176 : cephadm [INF] Upgrade: osd.3 is safe to restart 2024-09-12T23:45:26.983 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:26 smithi136 bash[76371]: cluster 2024-09-12T23:45:26.347320+0000 mon.a (mon.0) 553 : cluster [DBG] osdmap e472: 8 total, 8 up, 8 in 2024-09-12T23:45:27.345 INFO:tasks.rados.rados.0.smithi136.stdout:2078: done (0 left) 2024-09-12T23:45:27.345 INFO:tasks.rados.rados.0.smithi136.stdout:2079: snap_remove snap 196 2024-09-12T23:45:27.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:27 smithi136 bash[75164]: cluster 2024-09-12T23:45:26.721676+0000 mgr.y (mgr.24661) 177 : cluster [DBG] pgmap v283: 121 pgs: 121 active+clean; 254 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.7 MiB/s wr, 8 op/s 2024-09-12T23:45:27.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:27 smithi136 bash[75164]: cluster 2024-09-12T23:45:27.348631+0000 mon.a (mon.0) 554 : cluster [DBG] osdmap e473: 8 total, 8 up, 8 in 2024-09-12T23:45:27.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:27 smithi136 bash[76371]: cluster 2024-09-12T23:45:26.721676+0000 mgr.y (mgr.24661) 177 : cluster [DBG] pgmap v283: 121 pgs: 121 active+clean; 254 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.7 MiB/s wr, 8 op/s 2024-09-12T23:45:27.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:27 smithi136 bash[76371]: cluster 2024-09-12T23:45:27.348631+0000 mon.a (mon.0) 554 : cluster [DBG] osdmap e473: 8 total, 8 up, 8 in 2024-09-12T23:45:27.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:27 smithi181 bash[67587]: cluster 2024-09-12T23:45:26.721676+0000 mgr.y (mgr.24661) 177 : cluster [DBG] pgmap v283: 121 pgs: 121 active+clean; 254 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.7 MiB/s wr, 8 op/s 2024-09-12T23:45:27.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:27 smithi181 bash[67587]: cluster 2024-09-12T23:45:27.348631+0000 mon.a (mon.0) 554 : cluster [DBG] osdmap e473: 8 total, 8 up, 8 in 2024-09-12T23:45:28.353 INFO:tasks.rados.rados.0.smithi136.stdout:2079: done (0 left) 2024-09-12T23:45:28.353 INFO:tasks.rados.rados.0.smithi136.stdout:2080: snap_remove snap 195 2024-09-12T23:45:29.094 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:45:28 smithi181 bash[60471]: ts=2024-09-12T23:45:28.661Z caller=alerting.go:366 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-09-12T23:45:29.094 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:45:28 smithi181 bash[60471]: ts=2024-09-12T23:45:28.664Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.136\", device_class=\"ssd\", hostname=\"smithi136\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.136\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:45:29.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:28 smithi181 bash[67587]: cephadm 2024-09-12T23:45:27.875375+0000 mgr.y (mgr.24661) 178 : cephadm [INF] Upgrade: Updating osd.3 2024-09-12T23:45:29.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:28 smithi181 bash[67587]: audit 2024-09-12T23:45:27.885540+0000 mon.a (mon.0) 555 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:29.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:28 smithi181 bash[67587]: audit 2024-09-12T23:45:27.887019+0000 mon.a (mon.0) 556 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-12T23:45:29.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:28 smithi181 bash[67587]: audit 2024-09-12T23:45:27.888265+0000 mon.a (mon.0) 557 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:45:29.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:28 smithi181 bash[67587]: cephadm 2024-09-12T23:45:27.892915+0000 mgr.y (mgr.24661) 179 : cephadm [INF] Deploying daemon osd.3 on smithi136 2024-09-12T23:45:29.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:28 smithi181 bash[67587]: cluster 2024-09-12T23:45:28.353933+0000 mon.a (mon.0) 558 : cluster [DBG] osdmap e474: 8 total, 8 up, 8 in 2024-09-12T23:45:29.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:28 smithi181 bash[67587]: cluster 2024-09-12T23:45:28.722541+0000 mgr.y (mgr.24661) 180 : cluster [DBG] pgmap v286: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 254 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 436 KiB/s wr, 2 op/s 2024-09-12T23:45:29.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:28 smithi181 bash[67587]: audit 2024-09-12T23:45:28.853596+0000 mon.a (mon.0) 559 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:29.256 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:28 smithi136 bash[76371]: cephadm 2024-09-12T23:45:27.875375+0000 mgr.y (mgr.24661) 178 : cephadm [INF] Upgrade: Updating osd.3 2024-09-12T23:45:29.256 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:28 smithi136 bash[76371]: audit 2024-09-12T23:45:27.885540+0000 mon.a (mon.0) 555 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:29.256 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:28 smithi136 bash[76371]: audit 2024-09-12T23:45:27.887019+0000 mon.a (mon.0) 556 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-12T23:45:29.256 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:28 smithi136 bash[76371]: audit 2024-09-12T23:45:27.888265+0000 mon.a (mon.0) 557 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:45:29.256 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:28 smithi136 bash[76371]: cephadm 2024-09-12T23:45:27.892915+0000 mgr.y (mgr.24661) 179 : cephadm [INF] Deploying daemon osd.3 on smithi136 2024-09-12T23:45:29.256 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:28 smithi136 bash[76371]: cluster 2024-09-12T23:45:28.353933+0000 mon.a (mon.0) 558 : cluster [DBG] osdmap e474: 8 total, 8 up, 8 in 2024-09-12T23:45:29.256 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:28 smithi136 bash[76371]: cluster 2024-09-12T23:45:28.722541+0000 mgr.y (mgr.24661) 180 : cluster [DBG] pgmap v286: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 254 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 436 KiB/s wr, 2 op/s 2024-09-12T23:45:29.256 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:28 smithi136 bash[76371]: audit 2024-09-12T23:45:28.853596+0000 mon.a (mon.0) 559 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:29.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:28 smithi136 bash[75164]: cephadm 2024-09-12T23:45:27.875375+0000 mgr.y (mgr.24661) 178 : cephadm [INF] Upgrade: Updating osd.3 2024-09-12T23:45:29.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:28 smithi136 bash[75164]: audit 2024-09-12T23:45:27.885540+0000 mon.a (mon.0) 555 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:29.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:28 smithi136 bash[75164]: audit 2024-09-12T23:45:27.887019+0000 mon.a (mon.0) 556 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-12T23:45:29.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:28 smithi136 bash[75164]: audit 2024-09-12T23:45:27.888265+0000 mon.a (mon.0) 557 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:45:29.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:28 smithi136 bash[75164]: cephadm 2024-09-12T23:45:27.892915+0000 mgr.y (mgr.24661) 179 : cephadm [INF] Deploying daemon osd.3 on smithi136 2024-09-12T23:45:29.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:28 smithi136 bash[75164]: cluster 2024-09-12T23:45:28.353933+0000 mon.a (mon.0) 558 : cluster [DBG] osdmap e474: 8 total, 8 up, 8 in 2024-09-12T23:45:29.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:28 smithi136 bash[75164]: cluster 2024-09-12T23:45:28.722541+0000 mgr.y (mgr.24661) 180 : cluster [DBG] pgmap v286: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 254 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 436 KiB/s wr, 2 op/s 2024-09-12T23:45:29.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:28 smithi136 bash[75164]: audit 2024-09-12T23:45:28.853596+0000 mon.a (mon.0) 559 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:29.353 INFO:tasks.rados.rados.0.smithi136.stdout:2080: done (0 left) 2024-09-12T23:45:29.353 INFO:tasks.rados.rados.0.smithi136.stdout:2081: delete oid 26 current snap is 204 2024-09-12T23:45:29.364 INFO:tasks.rados.rados.0.smithi136.stdout:2081: done (0 left) 2024-09-12T23:45:29.365 INFO:tasks.rados.rados.0.smithi136.stdout:2082: snap_remove snap 188 2024-09-12T23:45:29.597 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:45:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:45:29] "GET /metrics HTTP/1.1" 200 36412 "" "Prometheus/2.43.0" 2024-09-12T23:45:30.371 INFO:tasks.rados.rados.0.smithi136.stdout:2082: done (0 left) 2024-09-12T23:45:30.372 INFO:tasks.rados.rados.0.smithi136.stdout:2083: read oid 49 snap -1 2024-09-12T23:45:30.372 INFO:tasks.rados.rados.0.smithi136.stdout:2083: expect (ObjNum 662 snap 195 seq_num 662) 2024-09-12T23:45:30.372 INFO:tasks.rados.rados.0.smithi136.stdout:2084: snap_remove snap 203 2024-09-12T23:45:30.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:30 smithi136 bash[75164]: cluster 2024-09-12T23:45:29.357170+0000 mon.a (mon.0) 560 : cluster [DBG] osdmap e475: 8 total, 8 up, 8 in 2024-09-12T23:45:30.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:30 smithi136 bash[76371]: cluster 2024-09-12T23:45:29.357170+0000 mon.a (mon.0) 560 : cluster [DBG] osdmap e475: 8 total, 8 up, 8 in 2024-09-12T23:45:30.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:30 smithi181 bash[67587]: cluster 2024-09-12T23:45:29.357170+0000 mon.a (mon.0) 560 : cluster [DBG] osdmap e475: 8 total, 8 up, 8 in 2024-09-12T23:45:31.341 INFO:tasks.rados.rados.0.smithi136.stdout:2084: done (1 left) 2024-09-12T23:45:31.341 INFO:tasks.rados.rados.0.smithi136.stdout:2085: read oid 41 snap 202 2024-09-12T23:45:31.341 INFO:tasks.rados.rados.0.smithi136.stdout:2085: expect deleted 2024-09-12T23:45:31.342 INFO:tasks.rados.rados.0.smithi136.stdout:2086: read oid 28 snap 192 2024-09-12T23:45:31.342 INFO:tasks.rados.rados.0.smithi136.stdout:2086: expect (ObjNum 654 snap 192 seq_num 654) 2024-09-12T23:45:31.342 INFO:tasks.rados.rados.0.smithi136.stdout:2087: copy_from oid 47 from oid 37 current snap is 204 2024-09-12T23:45:31.342 INFO:tasks.rados.rados.0.smithi136.stdout:2083: expect (ObjNum 662 snap 195 seq_num 662) 2024-09-12T23:45:31.428 INFO:tasks.rados.rados.0.smithi136.stdout:2087: finishing copy_from racing read to smithi13667081-47 2024-09-12T23:45:31.428 INFO:tasks.rados.rados.0.smithi136.stdout:2087: finishing copy_from to smithi13667081-47 2024-09-12T23:45:31.428 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 47 v 644 (ObjNum 636 snap 188 seq_num 636) dirty exists 2024-09-12T23:45:31.428 INFO:tasks.rados.rados.0.smithi136.stdout:2086: expect (ObjNum 654 snap 192 seq_num 654) 2024-09-12T23:45:31.501 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:31 smithi136 bash[76371]: cluster 2024-09-12T23:45:30.374062+0000 mon.a (mon.0) 561 : cluster [DBG] osdmap e476: 8 total, 8 up, 8 in 2024-09-12T23:45:31.501 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:31 smithi136 bash[76371]: cluster 2024-09-12T23:45:30.723251+0000 mgr.y (mgr.24661) 181 : cluster [DBG] pgmap v289: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 259 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 1.4 MiB/s wr, 6 op/s 2024-09-12T23:45:31.501 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:31 smithi136 bash[76371]: cluster 2024-09-12T23:45:31.344902+0000 mon.a (mon.0) 562 : cluster [DBG] osdmap e477: 8 total, 8 up, 8 in 2024-09-12T23:45:31.501 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:31 smithi136 bash[75164]: cluster 2024-09-12T23:45:30.374062+0000 mon.a (mon.0) 561 : cluster [DBG] osdmap e476: 8 total, 8 up, 8 in 2024-09-12T23:45:31.501 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:31 smithi136 bash[75164]: cluster 2024-09-12T23:45:30.723251+0000 mgr.y (mgr.24661) 181 : cluster [DBG] pgmap v289: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 259 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 1.4 MiB/s wr, 6 op/s 2024-09-12T23:45:31.501 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:31 smithi136 bash[75164]: cluster 2024-09-12T23:45:31.344902+0000 mon.a (mon.0) 562 : cluster [DBG] osdmap e477: 8 total, 8 up, 8 in 2024-09-12T23:45:31.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:31 smithi181 bash[67587]: cluster 2024-09-12T23:45:30.374062+0000 mon.a (mon.0) 561 : cluster [DBG] osdmap e476: 8 total, 8 up, 8 in 2024-09-12T23:45:31.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:31 smithi181 bash[67587]: cluster 2024-09-12T23:45:30.723251+0000 mgr.y (mgr.24661) 181 : cluster [DBG] pgmap v289: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 259 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 1.4 MiB/s wr, 6 op/s 2024-09-12T23:45:31.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:31 smithi181 bash[67587]: cluster 2024-09-12T23:45:31.344902+0000 mon.a (mon.0) 562 : cluster [DBG] osdmap e477: 8 total, 8 up, 8 in 2024-09-12T23:45:31.914 INFO:tasks.rados.rados.0.smithi136.stdout:2083: done (3 left) 2024-09-12T23:45:31.915 INFO:tasks.rados.rados.0.smithi136.stdout:2085: done (2 left) 2024-09-12T23:45:31.915 INFO:tasks.rados.rados.0.smithi136.stdout:2086: done (1 left) 2024-09-12T23:45:31.915 INFO:tasks.rados.rados.0.smithi136.stdout:2087: done (0 left) 2024-09-12T23:45:31.915 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 46 current snap is 204 2024-09-12T23:45:31.915 INFO:tasks.rados.rados.0.smithi136.stdout:2088: seq_num 684 ranges {1056768=663552,1720320=507904,2228224=409600,2637824=286720} 2024-09-12T23:45:31.944 INFO:tasks.rados.rados.0.smithi136.stdout:2088: writing smithi13667081-46 from 1056768 to 1720320 tid 1 2024-09-12T23:45:31.953 INFO:tasks.rados.rados.0.smithi136.stdout:2088: writing smithi13667081-46 from 1720320 to 2228224 tid 2 2024-09-12T23:45:31.960 INFO:tasks.rados.rados.0.smithi136.stdout:2088: writing smithi13667081-46 from 2228224 to 2637824 tid 3 2024-09-12T23:45:31.965 INFO:tasks.rados.rados.0.smithi136.stdout:2088: writing smithi13667081-46 from 2637824 to 2924544 tid 4 2024-09-12T23:45:31.966 INFO:tasks.rados.rados.0.smithi136.stdout:2089: read oid 47 snap -1 2024-09-12T23:45:31.966 INFO:tasks.rados.rados.0.smithi136.stdout:2089: expect (ObjNum 636 snap 188 seq_num 636) 2024-09-12T23:45:31.966 INFO:tasks.rados.rados.0.smithi136.stdout:2090: read oid 24 snap -1 2024-09-12T23:45:31.966 INFO:tasks.rados.rados.0.smithi136.stdout:2090: expect deleted 2024-09-12T23:45:31.966 INFO:tasks.rados.rados.0.smithi136.stdout:2091: snap_remove snap 202 2024-09-12T23:45:31.966 INFO:tasks.rados.rados.0.smithi136.stdout:2088: finishing write tid 1 to smithi13667081-46 2024-09-12T23:45:32.380 INFO:tasks.rados.rados.0.smithi136.stdout:2091: done (3 left) 2024-09-12T23:45:32.380 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 26 current snap is 204 2024-09-12T23:45:32.380 INFO:tasks.rados.rados.0.smithi136.stdout:2088: finishing write tid 2 to smithi13667081-46 2024-09-12T23:45:32.380 INFO:tasks.rados.rados.0.smithi136.stdout:2088: finishing write tid 3 to smithi13667081-46 2024-09-12T23:45:32.380 INFO:tasks.rados.rados.0.smithi136.stdout:2088: finishing write tid 4 to smithi13667081-46 2024-09-12T23:45:32.381 INFO:tasks.rados.rados.0.smithi136.stdout:2088: finishing write tid 5 to smithi13667081-46 2024-09-12T23:45:32.381 INFO:tasks.rados.rados.0.smithi136.stdout:2088: finishing write tid 6 to smithi13667081-46 2024-09-12T23:45:32.381 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 46 v 654 (ObjNum 684 snap 204 seq_num 684) dirty exists 2024-09-12T23:45:32.381 INFO:tasks.rados.rados.0.smithi136.stdout:2088: left oid 46 (ObjNum 684 snap 204 seq_num 684) 2024-09-12T23:45:32.381 INFO:tasks.rados.rados.0.smithi136.stdout:2089: expect (ObjNum 636 snap 188 seq_num 636) 2024-09-12T23:45:32.426 INFO:tasks.rados.rados.0.smithi136.stdout:2092: seq_num 685 ranges {0=679936,679936=442368,1122304=622592,1744896=139264} 2024-09-12T23:45:32.430 INFO:tasks.rados.rados.0.smithi136.stdout:2092: writing smithi13667081-26 from 0 to 679936 tid 1 2024-09-12T23:45:32.434 INFO:tasks.rados.rados.0.smithi136.stdout:2092: writing smithi13667081-26 from 679936 to 1122304 tid 2 2024-09-12T23:45:32.439 INFO:tasks.rados.rados.0.smithi136.stdout:2092: writing smithi13667081-26 from 1122304 to 1744896 tid 3 2024-09-12T23:45:32.440 INFO:tasks.rados.rados.0.smithi136.stdout:2092: writing smithi13667081-26 from 1744896 to 1884160 tid 4 2024-09-12T23:45:32.440 INFO:tasks.rados.rados.0.smithi136.stdout:2088: done (3 left) 2024-09-12T23:45:32.441 INFO:tasks.rados.rados.0.smithi136.stdout:2089: done (2 left) 2024-09-12T23:45:32.441 INFO:tasks.rados.rados.0.smithi136.stdout:2090: done (1 left) 2024-09-12T23:45:32.441 INFO:tasks.rados.rados.0.smithi136.stdout:2093: read oid 42 snap -1 2024-09-12T23:45:32.441 INFO:tasks.rados.rados.0.smithi136.stdout:2093: expect (ObjNum 672 snap 197 seq_num 672) 2024-09-12T23:45:32.441 INFO:tasks.rados.rados.0.smithi136.stdout:2094: snap_remove snap 198 2024-09-12T23:45:32.594 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:45:32 smithi181 bash[60471]: ts=2024-09-12T23:45:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:45:32.605 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:32 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:45:32.606 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:45:32 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:45:32.606 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:32 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:45:32.606 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 12 23:45:32 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:45:32.607 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 12 23:45:32 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:45:32.607 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:45:32 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:45:32.607 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 12 23:45:32 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:45:32.607 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 12 23:45:32 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:45:32.608 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 12 23:45:32 smithi136 systemd[1]: Stopping Ceph osd.3 for da58a47c-715d-11ef-bceb-c7b262605968... 2024-09-12T23:45:32.608 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:45:32 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:45:32.608 INFO:journalctl@ceph.mds.a.smithi136.stdout:Sep 12 23:45:32 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:45:33.006 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 12 23:45:32 smithi136 bash[52185]: debug 2024-09-12T23:45:32.728+0000 7f1b55f67640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-12T23:45:33.006 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 12 23:45:32 smithi136 bash[52185]: debug 2024-09-12T23:45:32.728+0000 7f1b55f67640 -1 osd.3 478 *** Got signal Terminated *** 2024-09-12T23:45:33.006 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 12 23:45:32 smithi136 bash[52185]: debug 2024-09-12T23:45:32.728+0000 7f1b55f67640 -1 osd.3 478 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-12T23:45:33.408 INFO:tasks.rados.rados.0.smithi136.stdout:2094: done (2 left) 2024-09-12T23:45:33.408 INFO:tasks.rados.rados.0.smithi136.stdout:2095: snap_create 2024-09-12T23:45:33.408 INFO:tasks.rados.rados.0.smithi136.stdout:2092: finishing write tid 1 to smithi13667081-26 2024-09-12T23:45:33.408 INFO:tasks.rados.rados.0.smithi136.stdout:2092: finishing write tid 2 to smithi13667081-26 2024-09-12T23:45:33.408 INFO:tasks.rados.rados.0.smithi136.stdout:2092: finishing write tid 3 to smithi13667081-26 2024-09-12T23:45:33.408 INFO:tasks.rados.rados.0.smithi136.stdout:2092: finishing write tid 4 to smithi13667081-26 2024-09-12T23:45:33.408 INFO:tasks.rados.rados.0.smithi136.stdout:2092: finishing write tid 5 to smithi13667081-26 2024-09-12T23:45:33.409 INFO:tasks.rados.rados.0.smithi136.stdout:2092: finishing write tid 6 to smithi13667081-26 2024-09-12T23:45:33.409 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 26 v 659 (ObjNum 685 snap 204 seq_num 685) dirty exists 2024-09-12T23:45:33.409 INFO:tasks.rados.rados.0.smithi136.stdout:2092: left oid 26 (ObjNum 685 snap 204 seq_num 685) 2024-09-12T23:45:33.409 INFO:tasks.rados.rados.0.smithi136.stdout:2093: expect (ObjNum 672 snap 197 seq_num 672) 2024-09-12T23:45:33.755 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:33 smithi136 bash[75164]: cluster 2024-09-12T23:45:32.383249+0000 mon.a (mon.0) 563 : cluster [DBG] osdmap e478: 8 total, 8 up, 8 in 2024-09-12T23:45:33.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:33 smithi136 bash[75164]: cluster 2024-09-12T23:45:32.723767+0000 mgr.y (mgr.24661) 182 : cluster [DBG] pgmap v292: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 260 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.1 MiB/s rd, 1.7 MiB/s wr, 8 op/s 2024-09-12T23:45:33.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:33 smithi136 bash[75164]: cluster 2024-09-12T23:45:32.735577+0000 mon.a (mon.0) 564 : cluster [INF] osd.3 marked itself down and dead 2024-09-12T23:45:33.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:33 smithi136 bash[76371]: cluster 2024-09-12T23:45:32.383249+0000 mon.a (mon.0) 563 : cluster [DBG] osdmap e478: 8 total, 8 up, 8 in 2024-09-12T23:45:33.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:33 smithi136 bash[76371]: cluster 2024-09-12T23:45:32.723767+0000 mgr.y (mgr.24661) 182 : cluster [DBG] pgmap v292: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 260 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.1 MiB/s rd, 1.7 MiB/s wr, 8 op/s 2024-09-12T23:45:33.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:33 smithi136 bash[76371]: cluster 2024-09-12T23:45:32.735577+0000 mon.a (mon.0) 564 : cluster [INF] osd.3 marked itself down and dead 2024-09-12T23:45:33.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:33 smithi181 bash[67587]: cluster 2024-09-12T23:45:32.383249+0000 mon.a (mon.0) 563 : cluster [DBG] osdmap e478: 8 total, 8 up, 8 in 2024-09-12T23:45:33.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:33 smithi181 bash[67587]: cluster 2024-09-12T23:45:32.723767+0000 mgr.y (mgr.24661) 182 : cluster [DBG] pgmap v292: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 260 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.1 MiB/s rd, 1.7 MiB/s wr, 8 op/s 2024-09-12T23:45:33.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:33 smithi181 bash[67587]: cluster 2024-09-12T23:45:32.735577+0000 mon.a (mon.0) 564 : cluster [INF] osd.3 marked itself down and dead 2024-09-12T23:45:33.845 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:45:33 smithi181 bash[60471]: ts=2024-09-12T23:45:33.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi136\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi136\", job=\"node\", machine=\"x86_64\", nodename=\"smithi136\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:45:34.406 INFO:tasks.rados.rados.0.smithi136.stdout:2092: done (2 left) 2024-09-12T23:45:34.406 INFO:tasks.rados.rados.0.smithi136.stdout:2093: done (1 left) 2024-09-12T23:45:34.406 INFO:tasks.rados.rados.0.smithi136.stdout:2095: done (0 left) 2024-09-12T23:45:34.406 INFO:tasks.rados.rados.0.smithi136.stdout:2096: rollback oid 39 current snap is 205 2024-09-12T23:45:34.406 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 39 to 194 2024-09-12T23:45:34.406 INFO:tasks.rados.rados.0.smithi136.stdout:2097: snap_create 2024-09-12T23:45:34.433 INFO:tasks.rados.rados.0.smithi136.stdout:2096: finishing rollback tid 1 to smithi13667081-39 2024-09-12T23:45:34.434 INFO:tasks.rados.rados.0.smithi136.stdout:2096: finishing rollback tid 2 to smithi13667081-39 2024-09-12T23:45:34.434 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 39 v 825 (ObjNum 651 snap 191 seq_num 651) dirty exists 2024-09-12T23:45:34.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:34 smithi136 bash[75164]: cluster 2024-09-12T23:45:33.380065+0000 mon.a (mon.0) 565 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-12T23:45:34.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:34 smithi136 bash[75164]: cluster 2024-09-12T23:45:33.402895+0000 mon.a (mon.0) 566 : cluster [DBG] osdmap e479: 8 total, 7 up, 8 in 2024-09-12T23:45:34.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:34 smithi136 bash[76371]: cluster 2024-09-12T23:45:33.380065+0000 mon.a (mon.0) 565 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-12T23:45:34.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:34 smithi136 bash[76371]: cluster 2024-09-12T23:45:33.402895+0000 mon.a (mon.0) 566 : cluster [DBG] osdmap e479: 8 total, 7 up, 8 in 2024-09-12T23:45:34.757 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 12 23:45:34 smithi136 bash[92944]: ceph-da58a47c-715d-11ef-bceb-c7b262605968-osd-3 2024-09-12T23:45:34.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:34 smithi181 bash[67587]: cluster 2024-09-12T23:45:33.380065+0000 mon.a (mon.0) 565 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-12T23:45:34.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:34 smithi181 bash[67587]: cluster 2024-09-12T23:45:33.402895+0000 mon.a (mon.0) 566 : cluster [DBG] osdmap e479: 8 total, 7 up, 8 in 2024-09-12T23:45:34.844 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:45:34 smithi181 bash[60471]: ts=2024-09-12T23:45:34.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:45:34.845 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:45:34 smithi181 bash[60471]: ts=2024-09-12T23:45:34.456Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:45:35.256 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 12 23:45:34 smithi136 bash[93011]: Error response from daemon: No such container: ceph-da58a47c-715d-11ef-bceb-c7b262605968-osd-3 2024-09-12T23:45:35.416 INFO:tasks.rados.rados.0.smithi136.stdout:2096: done (1 left) 2024-09-12T23:45:35.417 INFO:tasks.rados.rados.0.smithi136.stdout:2097: done (0 left) 2024-09-12T23:45:35.417 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 28 current snap is 206 2024-09-12T23:45:35.417 INFO:tasks.rados.rados.0.smithi136.stdout:2098: seq_num 686 ranges {7520256=696320,8216576=729088,8945664=425984,9371648=49152} 2024-09-12T23:45:35.507 INFO:tasks.rados.rados.0.smithi136.stdout:2098: writing smithi13667081-28 from 7520256 to 8216576 tid 1 2024-09-12T23:45:35.512 INFO:tasks.rados.rados.0.smithi136.stdout:2098: writing smithi13667081-28 from 8216576 to 8945664 tid 2 2024-09-12T23:45:35.514 INFO:tasks.rados.rados.0.smithi136.stdout:2098: writing smithi13667081-28 from 8945664 to 9371648 tid 3 2024-09-12T23:45:35.515 INFO:tasks.rados.rados.0.smithi136.stdout:2098: writing smithi13667081-28 from 9371648 to 9420800 tid 4 2024-09-12T23:45:35.515 INFO:tasks.rados.rados.0.smithi136.stdout:2099: snap_create 2024-09-12T23:45:35.520 INFO:tasks.rados.rados.0.smithi136.stdout:2098: finishing write tid 1 to smithi13667081-28 2024-09-12T23:45:35.523 INFO:tasks.rados.rados.0.smithi136.stdout:2098: finishing write tid 2 to smithi13667081-28 2024-09-12T23:45:35.523 INFO:tasks.rados.rados.0.smithi136.stdout:2098: finishing write tid 3 to smithi13667081-28 2024-09-12T23:45:35.526 INFO:tasks.rados.rados.0.smithi136.stdout:2098: finishing write tid 4 to smithi13667081-28 2024-09-12T23:45:35.528 INFO:tasks.rados.rados.0.smithi136.stdout:2098: finishing write tid 5 to smithi13667081-28 2024-09-12T23:45:35.530 INFO:tasks.rados.rados.0.smithi136.stdout:2098: finishing write tid 6 to smithi13667081-28 2024-09-12T23:45:35.531 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 28 v 1233 (ObjNum 686 snap 206 seq_num 686) dirty exists 2024-09-12T23:45:35.531 INFO:tasks.rados.rados.0.smithi136.stdout:2098: left oid 28 (ObjNum 686 snap 206 seq_num 686) 2024-09-12T23:45:35.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:35 smithi136 bash[76371]: cluster 2024-09-12T23:45:34.408181+0000 mon.a (mon.0) 567 : cluster [DBG] osdmap e480: 8 total, 7 up, 8 in 2024-09-12T23:45:35.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:35 smithi136 bash[76371]: cluster 2024-09-12T23:45:34.724617+0000 mgr.y (mgr.24661) 183 : cluster [DBG] pgmap v295: 121 pgs: 18 active+undersized, 5 stale+active+clean, 2 active+undersized+degraded, 96 active+clean; 262 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 258 KiB/s wr, 4 op/s; 11/423 objects degraded (2.600%) 2024-09-12T23:45:35.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:35 smithi136 bash[75164]: cluster 2024-09-12T23:45:34.408181+0000 mon.a (mon.0) 567 : cluster [DBG] osdmap e480: 8 total, 7 up, 8 in 2024-09-12T23:45:35.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:35 smithi136 bash[75164]: cluster 2024-09-12T23:45:34.724617+0000 mgr.y (mgr.24661) 183 : cluster [DBG] pgmap v295: 121 pgs: 18 active+undersized, 5 stale+active+clean, 2 active+undersized+degraded, 96 active+clean; 262 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 258 KiB/s wr, 4 op/s; 11/423 objects degraded (2.600%) 2024-09-12T23:45:35.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:35 smithi181 bash[67587]: cluster 2024-09-12T23:45:34.408181+0000 mon.a (mon.0) 567 : cluster [DBG] osdmap e480: 8 total, 7 up, 8 in 2024-09-12T23:45:35.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:35 smithi181 bash[67587]: cluster 2024-09-12T23:45:34.724617+0000 mgr.y (mgr.24661) 183 : cluster [DBG] pgmap v295: 121 pgs: 18 active+undersized, 5 stale+active+clean, 2 active+undersized+degraded, 96 active+clean; 262 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 258 KiB/s wr, 4 op/s; 11/423 objects degraded (2.600%) 2024-09-12T23:45:36.348 INFO:tasks.rados.rados.0.smithi136.stdout:2098: done (1 left) 2024-09-12T23:45:36.349 INFO:tasks.rados.rados.0.smithi136.stdout:2099: done (0 left) 2024-09-12T23:45:36.349 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 22 current snap is 207 2024-09-12T23:45:36.349 INFO:tasks.rados.rados.0.smithi136.stdout:2100: seq_num 687 ranges {3522560=712704,4235264=614400,4849664=589824} 2024-09-12T23:45:36.374 INFO:tasks.rados.rados.0.smithi136.stdout:2100: writing smithi13667081-22 from 3522560 to 4235264 tid 1 2024-09-12T23:45:36.378 INFO:tasks.rados.rados.0.smithi136.stdout:2100: writing smithi13667081-22 from 4235264 to 4849664 tid 2 2024-09-12T23:45:36.383 INFO:tasks.rados.rados.0.smithi136.stdout:2100: writing smithi13667081-22 from 4849664 to 5439488 tid 3 2024-09-12T23:45:36.384 INFO:tasks.rados.rados.0.smithi136.stdout:2101: copy_from oid 50 from oid 15 current snap is 207 2024-09-12T23:45:36.384 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 8 current snap is 207 2024-09-12T23:45:36.384 INFO:tasks.rados.rados.0.smithi136.stdout:2102: seq_num 688 ranges {7290880=729088,8019968=499712,8519680=704512} 2024-09-12T23:45:36.432 INFO:tasks.rados.rados.0.smithi136.stdout:2102: writing smithi13667081-8 from 7290880 to 8019968 tid 1 2024-09-12T23:45:36.434 INFO:tasks.rados.rados.0.smithi136.stdout:2102: writing smithi13667081-8 from 8019968 to 8519680 tid 2 2024-09-12T23:45:36.439 INFO:tasks.rados.rados.0.smithi136.stdout:2102: writing smithi13667081-8 from 8519680 to 9224192 tid 3 2024-09-12T23:45:36.439 INFO:tasks.rados.rados.0.smithi136.stdout:2103: copy_from oid 31 from oid 27 current snap is 207 2024-09-12T23:45:36.439 INFO:tasks.rados.rados.0.smithi136.stdout:2100: finishing write tid 1 to smithi13667081-22 2024-09-12T23:45:36.439 INFO:tasks.rados.rados.0.smithi136.stdout:2100: finishing write tid 2 to smithi13667081-22 2024-09-12T23:45:36.439 INFO:tasks.rados.rados.0.smithi136.stdout:2100: finishing write tid 3 to smithi13667081-22 2024-09-12T23:45:36.440 INFO:tasks.rados.rados.0.smithi136.stdout:2100: finishing write tid 4 to smithi13667081-22 2024-09-12T23:45:36.440 INFO:tasks.rados.rados.0.smithi136.stdout:2100: finishing write tid 5 to smithi13667081-22 2024-09-12T23:45:36.440 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 22 v 845 (ObjNum 687 snap 207 seq_num 687) dirty exists 2024-09-12T23:45:36.440 INFO:tasks.rados.rados.0.smithi136.stdout:2100: left oid 22 (ObjNum 687 snap 207 seq_num 687) 2024-09-12T23:45:36.440 INFO:tasks.rados.rados.0.smithi136.stdout:2101: finishing copy_from racing read to smithi13667081-50 2024-09-12T23:45:36.440 INFO:tasks.rados.rados.0.smithi136.stdout:2101: finishing copy_from to smithi13667081-50 2024-09-12T23:45:36.440 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 50 v 1205 (ObjNum 631 snap 185 seq_num 631) dirty exists 2024-09-12T23:45:36.440 INFO:tasks.rados.rados.0.smithi136.stdout:2100: done (3 left) 2024-09-12T23:45:36.440 INFO:tasks.rados.rados.0.smithi136.stdout:2101: done (2 left) 2024-09-12T23:45:36.440 INFO:tasks.rados.rados.0.smithi136.stdout:2104: snap_create 2024-09-12T23:45:36.446 INFO:tasks.rados.rados.0.smithi136.stdout:2102: finishing write tid 1 to smithi13667081-8 2024-09-12T23:45:36.447 INFO:tasks.rados.rados.0.smithi136.stdout:2102: finishing write tid 2 to smithi13667081-8 2024-09-12T23:45:36.447 INFO:tasks.rados.rados.0.smithi136.stdout:2102: finishing write tid 3 to smithi13667081-8 2024-09-12T23:45:36.447 INFO:tasks.rados.rados.0.smithi136.stdout:2102: finishing write tid 4 to smithi13667081-8 2024-09-12T23:45:36.451 INFO:tasks.rados.rados.0.smithi136.stdout:2102: finishing write tid 5 to smithi13667081-8 2024-09-12T23:45:36.451 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 8 v 803 (ObjNum 688 snap 207 seq_num 688) dirty exists 2024-09-12T23:45:36.451 INFO:tasks.rados.rados.0.smithi136.stdout:2102: left oid 8 (ObjNum 688 snap 207 seq_num 688) 2024-09-12T23:45:36.485 INFO:tasks.rados.rados.0.smithi136.stdout:2103: finishing copy_from racing read to smithi13667081-31 2024-09-12T23:45:36.486 INFO:tasks.rados.rados.0.smithi136.stdout:2103: finishing copy_from to smithi13667081-31 2024-09-12T23:45:36.486 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 31 v 1437 (ObjNum 416 snap 120 seq_num 416) dirty exists 2024-09-12T23:45:36.505 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:36 smithi136 bash[75164]: cluster 2024-09-12T23:45:35.400992+0000 mon.a (mon.0) 568 : cluster [WRN] Health check failed: Degraded data redundancy: 11/423 objects degraded (2.600%), 2 pgs degraded (PG_DEGRADED) 2024-09-12T23:45:36.505 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:36 smithi136 bash[75164]: cluster 2024-09-12T23:45:35.420231+0000 mon.a (mon.0) 569 : cluster [DBG] osdmap e481: 8 total, 7 up, 8 in 2024-09-12T23:45:36.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:36 smithi136 bash[75164]: cluster 2024-09-12T23:45:36.352420+0000 mon.a (mon.0) 570 : cluster [DBG] osdmap e482: 8 total, 7 up, 8 in 2024-09-12T23:45:36.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:36 smithi136 bash[76371]: cluster 2024-09-12T23:45:35.400992+0000 mon.a (mon.0) 568 : cluster [WRN] Health check failed: Degraded data redundancy: 11/423 objects degraded (2.600%), 2 pgs degraded (PG_DEGRADED) 2024-09-12T23:45:36.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:36 smithi136 bash[76371]: cluster 2024-09-12T23:45:35.420231+0000 mon.a (mon.0) 569 : cluster [DBG] osdmap e481: 8 total, 7 up, 8 in 2024-09-12T23:45:36.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:36 smithi136 bash[76371]: cluster 2024-09-12T23:45:36.352420+0000 mon.a (mon.0) 570 : cluster [DBG] osdmap e482: 8 total, 7 up, 8 in 2024-09-12T23:45:36.702 INFO:teuthology.orchestra.run.smithi136.stdout:true 2024-09-12T23:45:36.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:36 smithi181 bash[67587]: cluster 2024-09-12T23:45:35.400992+0000 mon.a (mon.0) 568 : cluster [WRN] Health check failed: Degraded data redundancy: 11/423 objects degraded (2.600%), 2 pgs degraded (PG_DEGRADED) 2024-09-12T23:45:36.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:36 smithi181 bash[67587]: cluster 2024-09-12T23:45:35.420231+0000 mon.a (mon.0) 569 : cluster [DBG] osdmap e481: 8 total, 7 up, 8 in 2024-09-12T23:45:36.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:36 smithi181 bash[67587]: cluster 2024-09-12T23:45:36.352420+0000 mon.a (mon.0) 570 : cluster [DBG] osdmap e482: 8 total, 7 up, 8 in 2024-09-12T23:45:37.003 INFO:teuthology.orchestra.run.smithi136.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-12T23:45:37.004 INFO:teuthology.orchestra.run.smithi136.stdout:alertmanager.a smithi136 *:9093,9094 running (5m) 21s ago 11m 13.1M - 0.25.0 c8568f914cd2 b5197f020eeb 2024-09-12T23:45:37.004 INFO:teuthology.orchestra.run.smithi136.stdout:grafana.a smithi181 *:3000 running (4m) 2m ago 10m 75.3M - 9.4.7 954c08fa6188 256c181be984 2024-09-12T23:45:37.004 INFO:teuthology.orchestra.run.smithi136.stdout:mds.a smithi136 running (12m) 21s ago 12m 15.0M - 17.2.7-1627-gbbdadfa3 214985049bdb 713b9795ad19 2024-09-12T23:45:37.004 INFO:teuthology.orchestra.run.smithi136.stdout:mds.b smithi181 running (12m) 2m ago 12m 17.8M - 17.2.7-1627-gbbdadfa3 214985049bdb ad13179f0868 2024-09-12T23:45:37.004 INFO:teuthology.orchestra.run.smithi136.stdout:mgr.x smithi181 *:8443,9283,8765 running (4m) 2m ago 19m 425M - 18.2.4-837-g7a853b17 f21ba1eea74e cc664dc337f7 2024-09-12T23:45:37.004 INFO:teuthology.orchestra.run.smithi136.stdout:mgr.y smithi136 *:8443,9283,8765 running (4m) 21s ago 22m 488M - 18.2.4-837-g7a853b17 f21ba1eea74e 272f10b42fc3 2024-09-12T23:45:37.004 INFO:teuthology.orchestra.run.smithi136.stdout:mon.a smithi136 running (3m) 21s ago 22m 53.5M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e ad8d21d58ad9 2024-09-12T23:45:37.004 INFO:teuthology.orchestra.run.smithi136.stdout:mon.b smithi181 running (2m) 2m ago 20m 19.5M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e a70ec8315934 2024-09-12T23:45:37.004 INFO:teuthology.orchestra.run.smithi136.stdout:mon.c smithi136 running (3m) 21s ago 20m 44.6M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e 44d1be055a5e 2024-09-12T23:45:37.004 INFO:teuthology.orchestra.run.smithi136.stdout:node-exporter.a smithi136 *:9100 running (5m) 21s ago 11m 8179k - 1.5.0 0da6a335fe13 ccfc37c96620 2024-09-12T23:45:37.004 INFO:teuthology.orchestra.run.smithi136.stdout:node-exporter.b smithi181 *:9100 running (4m) 2m ago 11m 7847k - 1.5.0 0da6a335fe13 80bf0e383230 2024-09-12T23:45:37.004 INFO:teuthology.orchestra.run.smithi136.stdout:osd.0 smithi136 running (84s) 21s ago 18m 205M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 7da21f765721 2024-09-12T23:45:37.005 INFO:teuthology.orchestra.run.smithi136.stdout:osd.1 smithi136 running (56s) 21s ago 18m 235M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 5302be0a016b 2024-09-12T23:45:37.005 INFO:teuthology.orchestra.run.smithi136.stdout:osd.2 smithi136 running (26s) 21s ago 17m - 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 2e12d1a31223 2024-09-12T23:45:37.005 INFO:teuthology.orchestra.run.smithi136.stdout:osd.3 smithi136 running (16m) 21s ago 16m 214M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 9ceb14a27521 2024-09-12T23:45:37.005 INFO:teuthology.orchestra.run.smithi136.stdout:osd.4 smithi181 running (15m) 2m ago 15m 230M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb a35cdca79055 2024-09-12T23:45:37.005 INFO:teuthology.orchestra.run.smithi136.stdout:osd.5 smithi181 running (14m) 2m ago 14m 192M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 8f2e8d494fef 2024-09-12T23:45:37.005 INFO:teuthology.orchestra.run.smithi136.stdout:osd.6 smithi181 running (13m) 2m ago 13m 106M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 9c6bec406408 2024-09-12T23:45:37.005 INFO:teuthology.orchestra.run.smithi136.stdout:osd.7 smithi181 running (12m) 2m ago 13m 174M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb bc0632ae2145 2024-09-12T23:45:37.005 INFO:teuthology.orchestra.run.smithi136.stdout:prometheus.a smithi181 *:9095 running (4m) 2m ago 11m 51.7M - 2.43.0 a07b618ecd1d 3853d003a5cc 2024-09-12T23:45:37.197 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:37 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:45:37.198 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 12 23:45:37 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:45:37.198 INFO:journalctl@ceph.mds.a.smithi136.stdout:Sep 12 23:45:37 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:45:37.198 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 12 23:45:37 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:45:37.199 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:45:37 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:45:37.199 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:37 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:45:37.199 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:45:37 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:45:37.200 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 12 23:45:37 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:45:37.200 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:45:37 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:45:37.200 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 12 23:45:36 smithi136 systemd[1]: ceph-da58a47c-715d-11ef-bceb-c7b262605968@osd.3.service: Deactivated successfully. 2024-09-12T23:45:37.200 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 12 23:45:36 smithi136 systemd[1]: Stopped Ceph osd.3 for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:45:37.200 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 12 23:45:37 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:45:37.200 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 12 23:45:37 smithi136 systemd[1]: Started Ceph osd.3 for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:45:37.367 INFO:teuthology.orchestra.run.smithi136.stdout:{ 2024-09-12T23:45:37.367 INFO:teuthology.orchestra.run.smithi136.stdout: "mon": { 2024-09-12T23:45:37.367 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 3 2024-09-12T23:45:37.367 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:45:37.367 INFO:teuthology.orchestra.run.smithi136.stdout: "mgr": { 2024-09-12T23:45:37.367 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 2 2024-09-12T23:45:37.367 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:45:37.367 INFO:teuthology.orchestra.run.smithi136.stdout: "osd": { 2024-09-12T23:45:37.367 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 4, 2024-09-12T23:45:37.368 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 3 2024-09-12T23:45:37.368 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:45:37.368 INFO:teuthology.orchestra.run.smithi136.stdout: "mds": { 2024-09-12T23:45:37.368 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-12T23:45:37.368 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:45:37.368 INFO:teuthology.orchestra.run.smithi136.stdout: "overall": { 2024-09-12T23:45:37.368 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 6, 2024-09-12T23:45:37.368 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 8 2024-09-12T23:45:37.368 INFO:teuthology.orchestra.run.smithi136.stdout: } 2024-09-12T23:45:37.368 INFO:teuthology.orchestra.run.smithi136.stdout:} 2024-09-12T23:45:37.423 INFO:tasks.rados.rados.0.smithi136.stdout:2102: done (2 left) 2024-09-12T23:45:37.423 INFO:tasks.rados.rados.0.smithi136.stdout:2103: done (1 left) 2024-09-12T23:45:37.423 INFO:tasks.rados.rados.0.smithi136.stdout:2104: done (0 left) 2024-09-12T23:45:37.423 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 43 current snap is 208 2024-09-12T23:45:37.424 INFO:tasks.rados.rados.0.smithi136.stdout:2105: seq_num 689 ranges {10256384=745472,11001856=786432,11788288=417792} 2024-09-12T23:45:37.484 INFO:tasks.rados.rados.0.smithi136.stdout:2105: writing smithi13667081-43 from 10256384 to 11001856 tid 1 2024-09-12T23:45:37.489 INFO:tasks.rados.rados.0.smithi136.stdout:2105: writing smithi13667081-43 from 11001856 to 11788288 tid 2 2024-09-12T23:45:37.492 INFO:tasks.rados.rados.0.smithi136.stdout:2105: writing smithi13667081-43 from 11788288 to 12206080 tid 3 2024-09-12T23:45:37.493 INFO:tasks.rados.rados.0.smithi136.stdout:2106: delete oid 18 current snap is 208 2024-09-12T23:45:37.498 INFO:tasks.rados.rados.0.smithi136.stdout:2105: finishing write tid 1 to smithi13667081-43 2024-09-12T23:45:37.502 INFO:tasks.rados.rados.0.smithi136.stdout:2105: finishing write tid 2 to smithi13667081-43 2024-09-12T23:45:37.504 INFO:tasks.rados.rados.0.smithi136.stdout:2105: finishing write tid 3 to smithi13667081-43 2024-09-12T23:45:37.505 INFO:tasks.rados.rados.0.smithi136.stdout:2106: done (1 left) 2024-09-12T23:45:37.505 INFO:tasks.rados.rados.0.smithi136.stdout:2107: read oid 13 snap -1 2024-09-12T23:45:37.505 INFO:tasks.rados.rados.0.smithi136.stdout:2107: expect deleted 2024-09-12T23:45:37.505 INFO:tasks.rados.rados.0.smithi136.stdout:2108: snap_create 2024-09-12T23:45:37.505 INFO:tasks.rados.rados.0.smithi136.stdout:2105: finishing write tid 4 to smithi13667081-43 2024-09-12T23:45:37.505 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:37 smithi136 bash[75164]: audit 2024-09-12T23:45:36.693595+0000 mgr.y (mgr.24661) 184 : audit [DBG] from='client.34290 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:45:37.505 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:37 smithi136 bash[75164]: cluster 2024-09-12T23:45:36.725255+0000 mgr.y (mgr.24661) 185 : cluster [DBG] pgmap v298: 121 pgs: 25 active+undersized, 3 stale+active+clean, 6 active+undersized+degraded, 87 active+clean; 273 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 3.2 MiB/s wr, 7 op/s; 35/426 objects degraded (8.216%) 2024-09-12T23:45:37.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:37 smithi136 bash[75164]: audit 2024-09-12T23:45:37.234407+0000 mon.a (mon.0) 571 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:37.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:37 smithi136 bash[75164]: audit 2024-09-12T23:45:37.240370+0000 mon.a (mon.0) 572 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:37.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:37 smithi136 bash[75164]: audit 2024-09-12T23:45:37.241491+0000 mon.a (mon.0) 573 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:45:37.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:37 smithi136 bash[75164]: audit 2024-09-12T23:45:37.371114+0000 mon.c (mon.2) 9 : audit [DBG] from='client.? 172.21.15.136:0/911172712' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:45:37.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:37 smithi136 bash[76371]: audit 2024-09-12T23:45:36.693595+0000 mgr.y (mgr.24661) 184 : audit [DBG] from='client.34290 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:45:37.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:37 smithi136 bash[76371]: cluster 2024-09-12T23:45:36.725255+0000 mgr.y (mgr.24661) 185 : cluster [DBG] pgmap v298: 121 pgs: 25 active+undersized, 3 stale+active+clean, 6 active+undersized+degraded, 87 active+clean; 273 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 3.2 MiB/s wr, 7 op/s; 35/426 objects degraded (8.216%) 2024-09-12T23:45:37.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:37 smithi136 bash[76371]: audit 2024-09-12T23:45:37.234407+0000 mon.a (mon.0) 571 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:37.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:37 smithi136 bash[76371]: audit 2024-09-12T23:45:37.240370+0000 mon.a (mon.0) 572 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:37.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:37 smithi136 bash[76371]: audit 2024-09-12T23:45:37.241491+0000 mon.a (mon.0) 573 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:45:37.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:37 smithi136 bash[76371]: audit 2024-09-12T23:45:37.371114+0000 mon.c (mon.2) 9 : audit [DBG] from='client.? 172.21.15.136:0/911172712' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:45:37.507 INFO:tasks.rados.rados.0.smithi136.stdout:2105: finishing write tid 5 to smithi13667081-43 2024-09-12T23:45:37.507 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 43 v 1238 (ObjNum 689 snap 208 seq_num 689) dirty exists 2024-09-12T23:45:37.507 INFO:tasks.rados.rados.0.smithi136.stdout:2105: left oid 43 (ObjNum 689 snap 208 seq_num 689) 2024-09-12T23:45:37.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:37 smithi181 bash[67587]: audit 2024-09-12T23:45:36.693595+0000 mgr.y (mgr.24661) 184 : audit [DBG] from='client.34290 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:45:37.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:37 smithi181 bash[67587]: cluster 2024-09-12T23:45:36.725255+0000 mgr.y (mgr.24661) 185 : cluster [DBG] pgmap v298: 121 pgs: 25 active+undersized, 3 stale+active+clean, 6 active+undersized+degraded, 87 active+clean; 273 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 3.2 MiB/s wr, 7 op/s; 35/426 objects degraded (8.216%) 2024-09-12T23:45:37.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:37 smithi181 bash[67587]: audit 2024-09-12T23:45:37.234407+0000 mon.a (mon.0) 571 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:37.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:37 smithi181 bash[67587]: audit 2024-09-12T23:45:37.240370+0000 mon.a (mon.0) 572 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:37.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:37 smithi181 bash[67587]: audit 2024-09-12T23:45:37.241491+0000 mon.a (mon.0) 573 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:45:37.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:37 smithi181 bash[67587]: audit 2024-09-12T23:45:37.371114+0000 mon.c (mon.2) 9 : audit [DBG] from='client.? 172.21.15.136:0/911172712' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:45:38.430 INFO:tasks.rados.rados.0.smithi136.stdout:2105: done (2 left) 2024-09-12T23:45:38.430 INFO:tasks.rados.rados.0.smithi136.stdout:2107: done (1 left) 2024-09-12T23:45:38.430 INFO:tasks.rados.rados.0.smithi136.stdout:2108: done (0 left) 2024-09-12T23:45:38.431 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 7 current snap is 209 2024-09-12T23:45:38.431 INFO:tasks.rados.rados.0.smithi136.stdout:2109: seq_num 691 ranges {0=778240,778240=548864,1327104=614400,1941504=40960} 2024-09-12T23:45:38.434 INFO:tasks.rados.rados.0.smithi136.stdout:2109: writing smithi13667081-7 from 0 to 778240 tid 1 2024-09-12T23:45:38.437 INFO:tasks.rados.rados.0.smithi136.stdout:2109: writing smithi13667081-7 from 778240 to 1327104 tid 2 2024-09-12T23:45:38.441 INFO:tasks.rados.rados.0.smithi136.stdout:2109: writing smithi13667081-7 from 1327104 to 1941504 tid 3 2024-09-12T23:45:38.441 INFO:tasks.rados.rados.0.smithi136.stdout:2109: writing smithi13667081-7 from 1941504 to 1982464 tid 4 2024-09-12T23:45:38.442 INFO:tasks.rados.rados.0.smithi136.stdout:2110: read oid 43 snap -1 2024-09-12T23:45:38.442 INFO:tasks.rados.rados.0.smithi136.stdout:2110: expect (ObjNum 689 snap 208 seq_num 689) 2024-09-12T23:45:38.442 INFO:tasks.rados.rados.0.smithi136.stdout:2111: snap_create 2024-09-12T23:45:38.451 INFO:tasks.rados.rados.0.smithi136.stdout:2109: finishing write tid 1 to smithi13667081-7 2024-09-12T23:45:38.451 INFO:tasks.rados.rados.0.smithi136.stdout:2109: finishing write tid 2 to smithi13667081-7 2024-09-12T23:45:38.456 INFO:tasks.rados.rados.0.smithi136.stdout:2109: finishing write tid 3 to smithi13667081-7 2024-09-12T23:45:38.457 INFO:tasks.rados.rados.0.smithi136.stdout:2109: finishing write tid 4 to smithi13667081-7 2024-09-12T23:45:38.457 INFO:tasks.rados.rados.0.smithi136.stdout:2109: finishing write tid 5 to smithi13667081-7 2024-09-12T23:45:38.460 INFO:tasks.rados.rados.0.smithi136.stdout:2109: finishing write tid 6 to smithi13667081-7 2024-09-12T23:45:38.460 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 7 v 936 (ObjNum 691 snap 209 seq_num 691) dirty exists 2024-09-12T23:45:38.460 INFO:tasks.rados.rados.0.smithi136.stdout:2109: left oid 7 (ObjNum 691 snap 209 seq_num 691) 2024-09-12T23:45:38.506 INFO:tasks.rados.rados.0.smithi136.stdout:2110: expect (ObjNum 689 snap 208 seq_num 689) 2024-09-12T23:45:38.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:38 smithi136 bash[75164]: audit 2024-09-12T23:45:36.987682+0000 mgr.y (mgr.24661) 186 : audit [DBG] from='client.34296 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:45:38.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:38 smithi136 bash[75164]: cluster 2024-09-12T23:45:37.427572+0000 mon.a (mon.0) 574 : cluster [DBG] osdmap e483: 8 total, 7 up, 8 in 2024-09-12T23:45:38.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:38 smithi136 bash[75164]: audit 2024-09-12T23:45:37.581854+0000 mon.a (mon.0) 575 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:38.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:38 smithi136 bash[75164]: audit 2024-09-12T23:45:37.587044+0000 mon.a (mon.0) 576 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:38.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:38 smithi136 bash[76371]: audit 2024-09-12T23:45:36.987682+0000 mgr.y (mgr.24661) 186 : audit [DBG] from='client.34296 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:45:38.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:38 smithi136 bash[76371]: cluster 2024-09-12T23:45:37.427572+0000 mon.a (mon.0) 574 : cluster [DBG] osdmap e483: 8 total, 7 up, 8 in 2024-09-12T23:45:38.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:38 smithi136 bash[76371]: audit 2024-09-12T23:45:37.581854+0000 mon.a (mon.0) 575 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:38.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:38 smithi136 bash[76371]: audit 2024-09-12T23:45:37.587044+0000 mon.a (mon.0) 576 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:38.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:38 smithi181 bash[67587]: audit 2024-09-12T23:45:36.987682+0000 mgr.y (mgr.24661) 186 : audit [DBG] from='client.34296 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:45:38.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:38 smithi181 bash[67587]: cluster 2024-09-12T23:45:37.427572+0000 mon.a (mon.0) 574 : cluster [DBG] osdmap e483: 8 total, 7 up, 8 in 2024-09-12T23:45:38.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:38 smithi181 bash[67587]: audit 2024-09-12T23:45:37.581854+0000 mon.a (mon.0) 575 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:38.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:38 smithi181 bash[67587]: audit 2024-09-12T23:45:37.587044+0000 mon.a (mon.0) 576 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:39.079 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 12 23:45:38 smithi136 bash[93223]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-12T23:45:39.079 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 12 23:45:38 smithi136 bash[93223]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-12T23:45:39.079 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 12 23:45:38 smithi136 bash[93223]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-12T23:45:39.079 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 12 23:45:38 smithi136 bash[93223]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-12T23:45:39.079 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 12 23:45:38 smithi136 bash[93223]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-12T23:45:39.079 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 12 23:45:38 smithi136 bash[93223]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-12T23:45:39.079 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 12 23:45:38 smithi136 bash[93223]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-12T23:45:39.546 INFO:tasks.rados.rados.0.smithi136.stdout:2109: done (2 left) 2024-09-12T23:45:39.546 INFO:tasks.rados.rados.0.smithi136.stdout:2110: done (1 left) 2024-09-12T23:45:39.547 INFO:tasks.rados.rados.0.smithi136.stdout:2111: done (0 left) 2024-09-12T23:45:39.547 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 49 current snap is 210 2024-09-12T23:45:39.547 INFO:tasks.rados.rados.0.smithi136.stdout:2112: seq_num 692 ranges {2310144=794624,3104768=483328,3588096=720896} 2024-09-12T23:45:39.565 INFO:tasks.rados.rados.0.smithi136.stdout:2112: writing smithi13667081-49 from 2310144 to 3104768 tid 1 2024-09-12T23:45:39.568 INFO:tasks.rados.rados.0.smithi136.stdout:2112: writing smithi13667081-49 from 3104768 to 3588096 tid 2 2024-09-12T23:45:39.573 INFO:tasks.rados.rados.0.smithi136.stdout:2112: writing smithi13667081-49 from 3588096 to 4308992 tid 3 2024-09-12T23:45:39.573 INFO:tasks.rados.rados.0.smithi136.stdout:2113: read oid 38 snap -1 2024-09-12T23:45:39.573 INFO:tasks.rados.rados.0.smithi136.stdout:2113: expect (ObjNum 666 snap 195 seq_num 666) 2024-09-12T23:45:39.573 INFO:tasks.rados.rados.0.smithi136.stdout:2114: delete oid 33 current snap is 210 2024-09-12T23:45:39.574 INFO:tasks.rados.rados.0.smithi136.stdout:2112: finishing write tid 1 to smithi13667081-49 2024-09-12T23:45:39.575 INFO:tasks.rados.rados.0.smithi136.stdout:2114: done (2 left) 2024-09-12T23:45:39.575 INFO:tasks.rados.rados.0.smithi136.stdout:2115: snap_create 2024-09-12T23:45:39.576 INFO:tasks.rados.rados.0.smithi136.stdout:2112: finishing write tid 2 to smithi13667081-49 2024-09-12T23:45:39.581 INFO:tasks.rados.rados.0.smithi136.stdout:2112: finishing write tid 3 to smithi13667081-49 2024-09-12T23:45:39.581 INFO:tasks.rados.rados.0.smithi136.stdout:2112: finishing write tid 4 to smithi13667081-49 2024-09-12T23:45:39.583 INFO:tasks.rados.rados.0.smithi136.stdout:2112: finishing write tid 5 to smithi13667081-49 2024-09-12T23:45:39.583 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 49 v 1441 (ObjNum 692 snap 210 seq_num 692) dirty exists 2024-09-12T23:45:39.583 INFO:tasks.rados.rados.0.smithi136.stdout:2112: left oid 49 (ObjNum 692 snap 210 seq_num 692) 2024-09-12T23:45:39.618 INFO:tasks.rados.rados.0.smithi136.stdout:2113: expect (ObjNum 666 snap 195 seq_num 666) 2024-09-12T23:45:39.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:45:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:45:39] "GET /metrics HTTP/1.1" 200 36396 "" "Prometheus/2.43.0" 2024-09-12T23:45:39.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:39 smithi136 bash[75164]: cluster 2024-09-12T23:45:38.434728+0000 mon.a (mon.0) 577 : cluster [DBG] osdmap e484: 8 total, 7 up, 8 in 2024-09-12T23:45:39.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:39 smithi136 bash[75164]: cluster 2024-09-12T23:45:38.725796+0000 mgr.y (mgr.24661) 187 : cluster [DBG] pgmap v301: 121 pgs: 32 active+undersized, 8 active+undersized+degraded, 81 active+clean; 275 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 3.6 MiB/s wr, 6 op/s; 44/426 objects degraded (10.329%) 2024-09-12T23:45:39.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:39 smithi136 bash[75164]: audit 2024-09-12T23:45:38.748165+0000 mon.a (mon.0) 578 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:39.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:39 smithi136 bash[75164]: audit 2024-09-12T23:45:38.749024+0000 mon.a (mon.0) 579 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:45:39.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:39 smithi136 bash[76371]: cluster 2024-09-12T23:45:38.434728+0000 mon.a (mon.0) 577 : cluster [DBG] osdmap e484: 8 total, 7 up, 8 in 2024-09-12T23:45:39.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:39 smithi136 bash[76371]: cluster 2024-09-12T23:45:38.725796+0000 mgr.y (mgr.24661) 187 : cluster [DBG] pgmap v301: 121 pgs: 32 active+undersized, 8 active+undersized+degraded, 81 active+clean; 275 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 3.6 MiB/s wr, 6 op/s; 44/426 objects degraded (10.329%) 2024-09-12T23:45:39.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:39 smithi136 bash[76371]: audit 2024-09-12T23:45:38.748165+0000 mon.a (mon.0) 578 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:39.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:39 smithi136 bash[76371]: audit 2024-09-12T23:45:38.749024+0000 mon.a (mon.0) 579 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:45:39.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:39 smithi181 bash[67587]: cluster 2024-09-12T23:45:38.434728+0000 mon.a (mon.0) 577 : cluster [DBG] osdmap e484: 8 total, 7 up, 8 in 2024-09-12T23:45:39.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:39 smithi181 bash[67587]: cluster 2024-09-12T23:45:38.725796+0000 mgr.y (mgr.24661) 187 : cluster [DBG] pgmap v301: 121 pgs: 32 active+undersized, 8 active+undersized+degraded, 81 active+clean; 275 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 3.6 MiB/s wr, 6 op/s; 44/426 objects degraded (10.329%) 2024-09-12T23:45:39.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:39 smithi181 bash[67587]: audit 2024-09-12T23:45:38.748165+0000 mon.a (mon.0) 578 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:39.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:39 smithi181 bash[67587]: audit 2024-09-12T23:45:38.749024+0000 mon.a (mon.0) 579 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:45:40.452 INFO:tasks.rados.rados.0.smithi136.stdout:2112: done (2 left) 2024-09-12T23:45:40.452 INFO:tasks.rados.rados.0.smithi136.stdout:2113: done (1 left) 2024-09-12T23:45:40.452 INFO:tasks.rados.rados.0.smithi136.stdout:2115: done (0 left) 2024-09-12T23:45:40.453 INFO:tasks.rados.rados.0.smithi136.stdout:2116: snap_create 2024-09-12T23:45:40.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:40 smithi136 bash[75164]: cluster 2024-09-12T23:45:39.448287+0000 mon.a (mon.0) 580 : cluster [DBG] osdmap e485: 8 total, 7 up, 8 in 2024-09-12T23:45:40.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:40 smithi136 bash[76371]: cluster 2024-09-12T23:45:39.448287+0000 mon.a (mon.0) 580 : cluster [DBG] osdmap e485: 8 total, 7 up, 8 in 2024-09-12T23:45:40.756 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 12 23:45:40 smithi136 bash[93456]: debug 2024-09-12T23:45:40.604+0000 7f243e716640 1 -- 172.21.15.136:0/382318100 <== mon.2 v2:172.21.15.136:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x555d5bab91e0 con 0x555d5ba79800 2024-09-12T23:45:40.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:40 smithi181 bash[67587]: cluster 2024-09-12T23:45:39.448287+0000 mon.a (mon.0) 580 : cluster [DBG] osdmap e485: 8 total, 7 up, 8 in 2024-09-12T23:45:41.350 INFO:tasks.rados.rados.0.smithi136.stdout:2116: done (0 left) 2024-09-12T23:45:41.351 INFO:tasks.rados.rados.0.smithi136.stdout:2117: read oid 37 snap 208 2024-09-12T23:45:41.351 INFO:tasks.rados.rados.0.smithi136.stdout:2117: expect (ObjNum 636 snap 188 seq_num 636) 2024-09-12T23:45:41.351 INFO:tasks.rados.rados.0.smithi136.stdout:2118: read oid 11 snap -1 2024-09-12T23:45:41.351 INFO:tasks.rados.rados.0.smithi136.stdout:2118: expect (ObjNum 605 snap 174 seq_num 605) 2024-09-12T23:45:41.351 INFO:tasks.rados.rados.0.smithi136.stdout:2119: read oid 3 snap -1 2024-09-12T23:45:41.351 INFO:tasks.rados.rados.0.smithi136.stdout:2119: expect (ObjNum 633 snap 186 seq_num 633) 2024-09-12T23:45:41.351 INFO:tasks.rados.rados.0.smithi136.stdout:2120: snap_remove snap 193 2024-09-12T23:45:41.456 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 12 23:45:41 smithi136 bash[93456]: debug 2024-09-12T23:45:41.152+0000 7f2440f7c640 -1 Falling back to public interface 2024-09-12T23:45:41.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:41 smithi136 bash[75164]: cluster 2024-09-12T23:45:40.456311+0000 mon.a (mon.0) 581 : cluster [DBG] osdmap e486: 8 total, 7 up, 8 in 2024-09-12T23:45:41.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:41 smithi136 bash[75164]: cluster 2024-09-12T23:45:40.726581+0000 mgr.y (mgr.24661) 188 : cluster [DBG] pgmap v304: 121 pgs: 32 active+undersized, 8 active+undersized+degraded, 81 active+clean; 280 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 685 KiB/s rd, 1.8 MiB/s wr, 4 op/s; 45/435 objects degraded (10.345%) 2024-09-12T23:45:41.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:41 smithi136 bash[75164]: cluster 2024-09-12T23:45:41.343398+0000 mon.a (mon.0) 582 : cluster [WRN] Health check update: Degraded data redundancy: 44/426 objects degraded (10.329%), 8 pgs degraded (PG_DEGRADED) 2024-09-12T23:45:41.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:41 smithi136 bash[75164]: cluster 2024-09-12T23:45:41.353348+0000 mon.a (mon.0) 583 : cluster [DBG] osdmap e487: 8 total, 7 up, 8 in 2024-09-12T23:45:41.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:41 smithi136 bash[76371]: cluster 2024-09-12T23:45:40.456311+0000 mon.a (mon.0) 581 : cluster [DBG] osdmap e486: 8 total, 7 up, 8 in 2024-09-12T23:45:41.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:41 smithi136 bash[76371]: cluster 2024-09-12T23:45:40.726581+0000 mgr.y (mgr.24661) 188 : cluster [DBG] pgmap v304: 121 pgs: 32 active+undersized, 8 active+undersized+degraded, 81 active+clean; 280 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 685 KiB/s rd, 1.8 MiB/s wr, 4 op/s; 45/435 objects degraded (10.345%) 2024-09-12T23:45:41.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:41 smithi136 bash[76371]: cluster 2024-09-12T23:45:41.343398+0000 mon.a (mon.0) 582 : cluster [WRN] Health check update: Degraded data redundancy: 44/426 objects degraded (10.329%), 8 pgs degraded (PG_DEGRADED) 2024-09-12T23:45:41.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:41 smithi136 bash[76371]: cluster 2024-09-12T23:45:41.353348+0000 mon.a (mon.0) 583 : cluster [DBG] osdmap e487: 8 total, 7 up, 8 in 2024-09-12T23:45:41.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:41 smithi181 bash[67587]: cluster 2024-09-12T23:45:40.456311+0000 mon.a (mon.0) 581 : cluster [DBG] osdmap e486: 8 total, 7 up, 8 in 2024-09-12T23:45:41.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:41 smithi181 bash[67587]: cluster 2024-09-12T23:45:40.726581+0000 mgr.y (mgr.24661) 188 : cluster [DBG] pgmap v304: 121 pgs: 32 active+undersized, 8 active+undersized+degraded, 81 active+clean; 280 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 685 KiB/s rd, 1.8 MiB/s wr, 4 op/s; 45/435 objects degraded (10.345%) 2024-09-12T23:45:41.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:41 smithi181 bash[67587]: cluster 2024-09-12T23:45:41.343398+0000 mon.a (mon.0) 582 : cluster [WRN] Health check update: Degraded data redundancy: 44/426 objects degraded (10.329%), 8 pgs degraded (PG_DEGRADED) 2024-09-12T23:45:41.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:41 smithi181 bash[67587]: cluster 2024-09-12T23:45:41.353348+0000 mon.a (mon.0) 583 : cluster [DBG] osdmap e487: 8 total, 7 up, 8 in 2024-09-12T23:45:42.354 INFO:tasks.rados.rados.0.smithi136.stdout:2120: done (3 left) 2024-09-12T23:45:42.354 INFO:tasks.rados.rados.0.smithi136.stdout:2121: read oid 22 snap -1 2024-09-12T23:45:42.354 INFO:tasks.rados.rados.0.smithi136.stdout:2121: expect (ObjNum 687 snap 207 seq_num 687) 2024-09-12T23:45:42.354 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 32 current snap is 212 2024-09-12T23:45:42.354 INFO:tasks.rados.rados.0.smithi136.stdout:2122: seq_num 694 ranges {1564672=425984,1990656=655360,2646016=786432,3432448=163840} 2024-09-12T23:45:42.384 INFO:tasks.rados.rados.0.smithi136.stdout:2122: writing smithi13667081-32 from 1564672 to 1990656 tid 1 2024-09-12T23:45:42.392 INFO:tasks.rados.rados.0.smithi136.stdout:2122: writing smithi13667081-32 from 1990656 to 2646016 tid 2 2024-09-12T23:45:42.401 INFO:tasks.rados.rados.0.smithi136.stdout:2122: writing smithi13667081-32 from 2646016 to 3432448 tid 3 2024-09-12T23:45:42.403 INFO:tasks.rados.rados.0.smithi136.stdout:2122: writing smithi13667081-32 from 3432448 to 3596288 tid 4 2024-09-12T23:45:42.403 INFO:tasks.rados.rados.0.smithi136.stdout:2123: snap_create 2024-09-12T23:45:42.403 INFO:tasks.rados.rados.0.smithi136.stdout:2117: expect (ObjNum 636 snap 188 seq_num 636) 2024-09-12T23:45:42.466 INFO:tasks.rados.rados.0.smithi136.stdout:2119: expect (ObjNum 633 snap 186 seq_num 633) 2024-09-12T23:45:42.507 INFO:tasks.rados.rados.0.smithi136.stdout:2118: expect (ObjNum 605 snap 174 seq_num 605) 2024-09-12T23:45:42.594 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:45:42 smithi181 bash[60471]: ts=2024-09-12T23:45:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:45:43.090 INFO:tasks.rados.rados.0.smithi136.stdout:2122: finishing write tid 1 to smithi13667081-32 2024-09-12T23:45:43.091 INFO:tasks.rados.rados.0.smithi136.stdout:2121: expect (ObjNum 687 snap 207 seq_num 687) 2024-09-12T23:45:43.309 INFO:tasks.rados.rados.0.smithi136.stdout:2122: finishing write tid 2 to smithi13667081-32 2024-09-12T23:45:43.310 INFO:tasks.rados.rados.0.smithi136.stdout:2122: finishing write tid 3 to smithi13667081-32 2024-09-12T23:45:43.310 INFO:tasks.rados.rados.0.smithi136.stdout:2122: finishing write tid 4 to smithi13667081-32 2024-09-12T23:45:43.310 INFO:tasks.rados.rados.0.smithi136.stdout:2122: finishing write tid 5 to smithi13667081-32 2024-09-12T23:45:43.310 INFO:tasks.rados.rados.0.smithi136.stdout:2122: finishing write tid 6 to smithi13667081-32 2024-09-12T23:45:43.310 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 32 v 779 (ObjNum 694 snap 212 seq_num 694) dirty exists 2024-09-12T23:45:43.310 INFO:tasks.rados.rados.0.smithi136.stdout:2122: left oid 32 (ObjNum 694 snap 212 seq_num 694) 2024-09-12T23:45:43.359 INFO:tasks.rados.rados.0.smithi136.stdout:2117: done (5 left) 2024-09-12T23:45:43.359 INFO:tasks.rados.rados.0.smithi136.stdout:2118: done (4 left) 2024-09-12T23:45:43.359 INFO:tasks.rados.rados.0.smithi136.stdout:2119: done (3 left) 2024-09-12T23:45:43.359 INFO:tasks.rados.rados.0.smithi136.stdout:2121: done (2 left) 2024-09-12T23:45:43.359 INFO:tasks.rados.rados.0.smithi136.stdout:2122: done (1 left) 2024-09-12T23:45:43.359 INFO:tasks.rados.rados.0.smithi136.stdout:2123: done (0 left) 2024-09-12T23:45:43.360 INFO:tasks.rados.rados.0.smithi136.stdout:2124: read oid 48 snap -1 2024-09-12T23:45:43.360 INFO:tasks.rados.rados.0.smithi136.stdout:2124: expect (ObjNum 669 snap 196 seq_num 669) 2024-09-12T23:45:43.360 INFO:tasks.rados.rados.0.smithi136.stdout:2125: setattr oid 21 current snap is 213 2024-09-12T23:45:43.361 INFO:tasks.rados.rados.0.smithi136.stdout:2126: setattr oid 5 current snap is 213 2024-09-12T23:45:43.364 INFO:tasks.rados.rados.0.smithi136.stdout:2127: read oid 41 snap 200 2024-09-12T23:45:43.364 INFO:tasks.rados.rados.0.smithi136.stdout:2127: expect deleted 2024-09-12T23:45:43.364 INFO:tasks.rados.rados.0.smithi136.stdout:2128: delete oid 16 current snap is 213 2024-09-12T23:45:43.375 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 21 v 846 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-12T23:45:43.375 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 5 v 1504 (ObjNum 567 snap 162 seq_num 567) dirty exists 2024-09-12T23:45:43.385 INFO:tasks.rados.rados.0.smithi136.stdout:2124: expect (ObjNum 669 snap 196 seq_num 669) 2024-09-12T23:45:43.653 INFO:tasks.rados.rados.0.smithi136.stdout:2124: done (4 left) 2024-09-12T23:45:43.653 INFO:tasks.rados.rados.0.smithi136.stdout:2125: done (3 left) 2024-09-12T23:45:43.653 INFO:tasks.rados.rados.0.smithi136.stdout:2126: done (2 left) 2024-09-12T23:45:43.653 INFO:tasks.rados.rados.0.smithi136.stdout:2127: done (1 left) 2024-09-12T23:45:43.653 INFO:tasks.rados.rados.0.smithi136.stdout:2128: done (0 left) 2024-09-12T23:45:43.654 INFO:tasks.rados.rados.0.smithi136.stdout:2129: snap_create 2024-09-12T23:45:43.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:43 smithi136 bash[76371]: cluster 2024-09-12T23:45:42.357447+0000 mon.a (mon.0) 584 : cluster [DBG] osdmap e488: 8 total, 7 up, 8 in 2024-09-12T23:45:43.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:43 smithi136 bash[76371]: cluster 2024-09-12T23:45:42.727367+0000 mgr.y (mgr.24661) 189 : cluster [DBG] pgmap v307: 121 pgs: 32 active+undersized, 8 active+undersized+degraded, 81 active+clean; 284 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 2.3 MiB/s wr, 10 op/s; 46/441 objects degraded (10.431%) 2024-09-12T23:45:43.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:43 smithi136 bash[75164]: cluster 2024-09-12T23:45:42.357447+0000 mon.a (mon.0) 584 : cluster [DBG] osdmap e488: 8 total, 7 up, 8 in 2024-09-12T23:45:43.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:43 smithi136 bash[75164]: cluster 2024-09-12T23:45:42.727367+0000 mgr.y (mgr.24661) 189 : cluster [DBG] pgmap v307: 121 pgs: 32 active+undersized, 8 active+undersized+degraded, 81 active+clean; 284 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 2.3 MiB/s wr, 10 op/s; 46/441 objects degraded (10.431%) 2024-09-12T23:45:43.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:43 smithi181 bash[67587]: cluster 2024-09-12T23:45:42.357447+0000 mon.a (mon.0) 584 : cluster [DBG] osdmap e488: 8 total, 7 up, 8 in 2024-09-12T23:45:43.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:43 smithi181 bash[67587]: cluster 2024-09-12T23:45:42.727367+0000 mgr.y (mgr.24661) 189 : cluster [DBG] pgmap v307: 121 pgs: 32 active+undersized, 8 active+undersized+degraded, 81 active+clean; 284 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 2.3 MiB/s wr, 10 op/s; 46/441 objects degraded (10.431%) 2024-09-12T23:45:44.369 INFO:tasks.rados.rados.0.smithi136.stdout:2129: done (0 left) 2024-09-12T23:45:44.369 INFO:tasks.rados.rados.0.smithi136.stdout:2130: delete oid 8 current snap is 214 2024-09-12T23:45:44.379 INFO:tasks.rados.rados.0.smithi136.stdout:2130: done (0 left) 2024-09-12T23:45:44.379 INFO:tasks.rados.rados.0.smithi136.stdout:2131: rollback oid 13 current snap is 214 2024-09-12T23:45:44.379 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 13 to 206 2024-09-12T23:45:44.379 INFO:tasks.rados.rados.0.smithi136.stdout:2132: snap_create 2024-09-12T23:45:44.385 INFO:tasks.rados.rados.0.smithi136.stdout:2131: finishing rollback tid 1 to smithi13667081-13 2024-09-12T23:45:44.385 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 13 v 0 (ObjNum 65 snap 0 seq_num 4243457200) dirty dne 2024-09-12T23:45:44.681 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:44 smithi136 bash[75164]: cluster 2024-09-12T23:45:43.363010+0000 mon.a (mon.0) 585 : cluster [DBG] osdmap e489: 8 total, 7 up, 8 in 2024-09-12T23:45:44.682 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:44 smithi136 bash[75164]: audit 2024-09-12T23:45:43.751042+0000 mon.a (mon.0) 586 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:44.682 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:44 smithi136 bash[75164]: audit 2024-09-12T23:45:43.756387+0000 mon.a (mon.0) 587 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:44.682 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:44 smithi136 bash[76371]: cluster 2024-09-12T23:45:43.363010+0000 mon.a (mon.0) 585 : cluster [DBG] osdmap e489: 8 total, 7 up, 8 in 2024-09-12T23:45:44.682 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:44 smithi136 bash[76371]: audit 2024-09-12T23:45:43.751042+0000 mon.a (mon.0) 586 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:44.682 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:44 smithi136 bash[76371]: audit 2024-09-12T23:45:43.756387+0000 mon.a (mon.0) 587 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:44.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:44 smithi181 bash[67587]: cluster 2024-09-12T23:45:43.363010+0000 mon.a (mon.0) 585 : cluster [DBG] osdmap e489: 8 total, 7 up, 8 in 2024-09-12T23:45:44.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:44 smithi181 bash[67587]: audit 2024-09-12T23:45:43.751042+0000 mon.a (mon.0) 586 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:44.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:44 smithi181 bash[67587]: audit 2024-09-12T23:45:43.756387+0000 mon.a (mon.0) 587 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:45.006 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 12 23:45:44 smithi136 bash[93456]: debug 2024-09-12T23:45:44.684+0000 7f2440f7c640 -1 osd.3 478 log_to_monitors true 2024-09-12T23:45:45.383 INFO:tasks.rados.rados.0.smithi136.stdout:2131: done (1 left) 2024-09-12T23:45:45.384 INFO:tasks.rados.rados.0.smithi136.stdout:2132: done (0 left) 2024-09-12T23:45:45.384 INFO:tasks.rados.rados.0.smithi136.stdout:2133: snap_remove snap 200 2024-09-12T23:45:45.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:45 smithi136 bash[76371]: cluster 2024-09-12T23:45:44.373698+0000 mon.a (mon.0) 588 : cluster [DBG] osdmap e490: 8 total, 7 up, 8 in 2024-09-12T23:45:45.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:45 smithi136 bash[76371]: audit 2024-09-12T23:45:44.396245+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:45.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:45 smithi136 bash[76371]: audit 2024-09-12T23:45:44.403641+0000 mon.a (mon.0) 590 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:45.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:45 smithi136 bash[76371]: audit 2024-09-12T23:45:44.688836+0000 mon.c (mon.2) 10 : audit [INF] from='osd.3 [v2:172.21.15.136:6826/2156253618,v1:172.21.15.136:6827/2156253618]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-12T23:45:45.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:45 smithi136 bash[76371]: audit 2024-09-12T23:45:44.689130+0000 mon.a (mon.0) 591 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-12T23:45:45.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:45 smithi136 bash[76371]: cluster 2024-09-12T23:45:44.728148+0000 mgr.y (mgr.24661) 190 : cluster [DBG] pgmap v310: 121 pgs: 32 active+undersized, 8 active+undersized+degraded, 81 active+clean; 286 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 1.4 MiB/s wr, 14 op/s; 47/450 objects degraded (10.444%) 2024-09-12T23:45:45.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:45 smithi136 bash[75164]: cluster 2024-09-12T23:45:44.373698+0000 mon.a (mon.0) 588 : cluster [DBG] osdmap e490: 8 total, 7 up, 8 in 2024-09-12T23:45:45.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:45 smithi136 bash[75164]: audit 2024-09-12T23:45:44.396245+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:45.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:45 smithi136 bash[75164]: audit 2024-09-12T23:45:44.403641+0000 mon.a (mon.0) 590 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:45.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:45 smithi136 bash[75164]: audit 2024-09-12T23:45:44.688836+0000 mon.c (mon.2) 10 : audit [INF] from='osd.3 [v2:172.21.15.136:6826/2156253618,v1:172.21.15.136:6827/2156253618]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-12T23:45:45.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:45 smithi136 bash[75164]: audit 2024-09-12T23:45:44.689130+0000 mon.a (mon.0) 591 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-12T23:45:45.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:45 smithi136 bash[75164]: cluster 2024-09-12T23:45:44.728148+0000 mgr.y (mgr.24661) 190 : cluster [DBG] pgmap v310: 121 pgs: 32 active+undersized, 8 active+undersized+degraded, 81 active+clean; 286 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 1.4 MiB/s wr, 14 op/s; 47/450 objects degraded (10.444%) 2024-09-12T23:45:45.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:45 smithi181 bash[67587]: cluster 2024-09-12T23:45:44.373698+0000 mon.a (mon.0) 588 : cluster [DBG] osdmap e490: 8 total, 7 up, 8 in 2024-09-12T23:45:45.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:45 smithi181 bash[67587]: audit 2024-09-12T23:45:44.396245+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:45.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:45 smithi181 bash[67587]: audit 2024-09-12T23:45:44.403641+0000 mon.a (mon.0) 590 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:45.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:45 smithi181 bash[67587]: audit 2024-09-12T23:45:44.688836+0000 mon.c (mon.2) 10 : audit [INF] from='osd.3 [v2:172.21.15.136:6826/2156253618,v1:172.21.15.136:6827/2156253618]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-12T23:45:45.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:45 smithi181 bash[67587]: audit 2024-09-12T23:45:44.689130+0000 mon.a (mon.0) 591 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-12T23:45:45.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:45 smithi181 bash[67587]: cluster 2024-09-12T23:45:44.728148+0000 mgr.y (mgr.24661) 190 : cluster [DBG] pgmap v310: 121 pgs: 32 active+undersized, 8 active+undersized+degraded, 81 active+clean; 286 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 1.4 MiB/s wr, 14 op/s; 47/450 objects degraded (10.444%) 2024-09-12T23:45:46.256 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 12 23:45:45 smithi136 bash[93456]: debug 2024-09-12T23:45:45.784+0000 7f2434d1e640 -1 osd.3 478 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-12T23:45:46.394 INFO:tasks.rados.rados.0.smithi136.stdout:2133: done (0 left) 2024-09-12T23:45:46.395 INFO:tasks.rados.rados.0.smithi136.stdout:2134: copy_from oid 10 from oid 48 current snap is 215 2024-09-12T23:45:46.395 INFO:tasks.rados.rados.0.smithi136.stdout:2135: snap_create 2024-09-12T23:45:46.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:46 smithi136 bash[76371]: audit 2024-09-12T23:45:45.387077+0000 mon.a (mon.0) 592 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-12T23:45:46.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:46 smithi136 bash[76371]: cluster 2024-09-12T23:45:45.387271+0000 mon.a (mon.0) 593 : cluster [DBG] osdmap e491: 8 total, 7 up, 8 in 2024-09-12T23:45:46.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:46 smithi136 bash[76371]: audit 2024-09-12T23:45:45.389011+0000 mon.c (mon.2) 11 : audit [INF] from='osd.3 [v2:172.21.15.136:6826/2156253618,v1:172.21.15.136:6827/2156253618]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-12T23:45:46.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:46 smithi136 bash[76371]: audit 2024-09-12T23:45:45.389508+0000 mon.a (mon.0) 594 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-12T23:45:46.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:46 smithi136 bash[76371]: audit 2024-09-12T23:45:45.746130+0000 mon.a (mon.0) 595 : audit [INF] from='osd.3 ' entity='osd.3' 2024-09-12T23:45:46.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:46 smithi136 bash[76371]: cluster 2024-09-12T23:45:46.346343+0000 mon.a (mon.0) 596 : cluster [WRN] Health check update: Degraded data redundancy: 47/450 objects degraded (10.444%), 8 pgs degraded (PG_DEGRADED) 2024-09-12T23:45:46.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:46 smithi136 bash[75164]: audit 2024-09-12T23:45:45.387077+0000 mon.a (mon.0) 592 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-12T23:45:46.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:46 smithi136 bash[75164]: cluster 2024-09-12T23:45:45.387271+0000 mon.a (mon.0) 593 : cluster [DBG] osdmap e491: 8 total, 7 up, 8 in 2024-09-12T23:45:46.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:46 smithi136 bash[75164]: audit 2024-09-12T23:45:45.389011+0000 mon.c (mon.2) 11 : audit [INF] from='osd.3 [v2:172.21.15.136:6826/2156253618,v1:172.21.15.136:6827/2156253618]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-12T23:45:46.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:46 smithi136 bash[75164]: audit 2024-09-12T23:45:45.389508+0000 mon.a (mon.0) 594 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-12T23:45:46.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:46 smithi136 bash[75164]: audit 2024-09-12T23:45:45.746130+0000 mon.a (mon.0) 595 : audit [INF] from='osd.3 ' entity='osd.3' 2024-09-12T23:45:46.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:46 smithi136 bash[75164]: cluster 2024-09-12T23:45:46.346343+0000 mon.a (mon.0) 596 : cluster [WRN] Health check update: Degraded data redundancy: 47/450 objects degraded (10.444%), 8 pgs degraded (PG_DEGRADED) 2024-09-12T23:45:46.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:46 smithi181 bash[67587]: audit 2024-09-12T23:45:45.387077+0000 mon.a (mon.0) 592 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-12T23:45:46.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:46 smithi181 bash[67587]: cluster 2024-09-12T23:45:45.387271+0000 mon.a (mon.0) 593 : cluster [DBG] osdmap e491: 8 total, 7 up, 8 in 2024-09-12T23:45:46.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:46 smithi181 bash[67587]: audit 2024-09-12T23:45:45.389011+0000 mon.c (mon.2) 11 : audit [INF] from='osd.3 [v2:172.21.15.136:6826/2156253618,v1:172.21.15.136:6827/2156253618]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-12T23:45:46.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:46 smithi181 bash[67587]: audit 2024-09-12T23:45:45.389508+0000 mon.a (mon.0) 594 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-12T23:45:46.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:46 smithi181 bash[67587]: audit 2024-09-12T23:45:45.746130+0000 mon.a (mon.0) 595 : audit [INF] from='osd.3 ' entity='osd.3' 2024-09-12T23:45:46.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:46 smithi181 bash[67587]: cluster 2024-09-12T23:45:46.346343+0000 mon.a (mon.0) 596 : cluster [WRN] Health check update: Degraded data redundancy: 47/450 objects degraded (10.444%), 8 pgs degraded (PG_DEGRADED) 2024-09-12T23:45:47.401 INFO:tasks.rados.rados.0.smithi136.stdout:2135: done (1 left) 2024-09-12T23:45:47.401 INFO:tasks.rados.rados.0.smithi136.stdout:2136: read oid 29 snap -1 2024-09-12T23:45:47.401 INFO:tasks.rados.rados.0.smithi136.stdout:2136: expect (ObjNum 666 snap 195 seq_num 666) 2024-09-12T23:45:47.401 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 31 current snap is 216 2024-09-12T23:45:47.401 INFO:tasks.rados.rados.0.smithi136.stdout:2137: seq_num 697 ranges {7618560=475136,8093696=704512,8798208=557056,9355264=344064} 2024-09-12T23:45:47.472 INFO:tasks.rados.rados.0.smithi136.stdout:2137: writing smithi13667081-31 from 7618560 to 8093696 tid 1 2024-09-12T23:45:47.478 INFO:tasks.rados.rados.0.smithi136.stdout:2137: writing smithi13667081-31 from 8093696 to 8798208 tid 2 2024-09-12T23:45:47.482 INFO:tasks.rados.rados.0.smithi136.stdout:2137: writing smithi13667081-31 from 8798208 to 9355264 tid 3 2024-09-12T23:45:47.484 INFO:tasks.rados.rados.0.smithi136.stdout:2137: writing smithi13667081-31 from 9355264 to 9699328 tid 4 2024-09-12T23:45:47.484 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 40 current snap is 216 2024-09-12T23:45:47.484 INFO:tasks.rados.rados.0.smithi136.stdout:2138: seq_num 698 ranges {1294336=499712,1794048=589824,2383872=770048,3153920=245760} 2024-09-12T23:45:47.495 INFO:tasks.rados.rados.0.smithi136.stdout:2138: writing smithi13667081-40 from 1294336 to 1794048 tid 1 2024-09-12T23:45:47.499 INFO:tasks.rados.rados.0.smithi136.stdout:2138: writing smithi13667081-40 from 1794048 to 2383872 tid 2 2024-09-12T23:45:47.503 INFO:tasks.rados.rados.0.smithi136.stdout:2138: writing smithi13667081-40 from 2383872 to 3153920 tid 3 2024-09-12T23:45:47.505 INFO:tasks.rados.rados.0.smithi136.stdout:2138: writing smithi13667081-40 from 3153920 to 3399680 tid 4 2024-09-12T23:45:47.505 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 35 current snap is 216 2024-09-12T23:45:47.505 INFO:tasks.rados.rados.0.smithi136.stdout:2139: seq_num 699 ranges {3162112=516096,3678208=475136,4153344=573440,4726784=557056} 2024-09-12T23:45:47.526 INFO:tasks.rados.rados.0.smithi136.stdout:2139: writing smithi13667081-35 from 3162112 to 3678208 tid 1 2024-09-12T23:45:47.529 INFO:tasks.rados.rados.0.smithi136.stdout:2139: writing smithi13667081-35 from 3678208 to 4153344 tid 2 2024-09-12T23:45:47.533 INFO:tasks.rados.rados.0.smithi136.stdout:2139: writing smithi13667081-35 from 4153344 to 4726784 tid 3 2024-09-12T23:45:47.537 INFO:tasks.rados.rados.0.smithi136.stdout:2139: writing smithi13667081-35 from 4726784 to 5283840 tid 4 2024-09-12T23:45:47.537 INFO:tasks.rados.rados.0.smithi136.stdout:2140: rollback oid 16 current snap is 216 2024-09-12T23:45:47.537 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 16 to 211 2024-09-12T23:45:47.537 INFO:tasks.rados.rados.0.smithi136.stdout:2137: finishing write tid 1 to smithi13667081-31 2024-09-12T23:45:47.537 INFO:tasks.rados.rados.0.smithi136.stdout:2137: finishing write tid 2 to smithi13667081-31 2024-09-12T23:45:47.537 INFO:tasks.rados.rados.0.smithi136.stdout:2141: setattr oid 14 current snap is 216 2024-09-12T23:45:47.537 INFO:tasks.rados.rados.0.smithi136.stdout:2137: finishing write tid 3 to smithi13667081-31 2024-09-12T23:45:47.537 INFO:tasks.rados.rados.0.smithi136.stdout:2137: finishing write tid 4 to smithi13667081-31 2024-09-12T23:45:47.537 INFO:tasks.rados.rados.0.smithi136.stdout:2137: finishing write tid 5 to smithi13667081-31 2024-09-12T23:45:47.537 INFO:tasks.rados.rados.0.smithi136.stdout:2136: expect (ObjNum 666 snap 195 seq_num 666) 2024-09-12T23:45:47.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:47 smithi136 bash[76371]: cluster 2024-09-12T23:45:46.381391+0000 mon.a (mon.0) 597 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-12T23:45:47.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:47 smithi136 bash[76371]: cluster 2024-09-12T23:45:46.393408+0000 mon.a (mon.0) 598 : cluster [INF] osd.3 [v2:172.21.15.136:6826/2156253618,v1:172.21.15.136:6827/2156253618] boot 2024-09-12T23:45:47.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:47 smithi136 bash[76371]: cluster 2024-09-12T23:45:46.393565+0000 mon.a (mon.0) 599 : cluster [DBG] osdmap e492: 8 total, 8 up, 8 in 2024-09-12T23:45:47.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:47 smithi136 bash[76371]: audit 2024-09-12T23:45:46.397251+0000 mon.a (mon.0) 600 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-12T23:45:47.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:47 smithi136 bash[76371]: cluster 2024-09-12T23:45:46.729173+0000 mgr.y (mgr.24661) 191 : cluster [DBG] pgmap v313: 121 pgs: 9 peering, 26 active+undersized, 5 active+undersized+degraded, 81 active+clean; 286 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 496 KiB/s wr, 10 op/s; 28/453 objects degraded (6.181%) 2024-09-12T23:45:47.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:47 smithi136 bash[76371]: cluster 2024-09-12T23:45:47.395691+0000 mon.a (mon.0) 601 : cluster [DBG] osdmap e493: 8 total, 8 up, 8 in 2024-09-12T23:45:47.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:47 smithi136 bash[75164]: cluster 2024-09-12T23:45:46.381391+0000 mon.a (mon.0) 597 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-12T23:45:47.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:47 smithi136 bash[75164]: cluster 2024-09-12T23:45:46.393408+0000 mon.a (mon.0) 598 : cluster [INF] osd.3 [v2:172.21.15.136:6826/2156253618,v1:172.21.15.136:6827/2156253618] boot 2024-09-12T23:45:47.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:47 smithi136 bash[75164]: cluster 2024-09-12T23:45:46.393565+0000 mon.a (mon.0) 599 : cluster [DBG] osdmap e492: 8 total, 8 up, 8 in 2024-09-12T23:45:47.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:47 smithi136 bash[75164]: audit 2024-09-12T23:45:46.397251+0000 mon.a (mon.0) 600 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-12T23:45:47.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:47 smithi136 bash[75164]: cluster 2024-09-12T23:45:46.729173+0000 mgr.y (mgr.24661) 191 : cluster [DBG] pgmap v313: 121 pgs: 9 peering, 26 active+undersized, 5 active+undersized+degraded, 81 active+clean; 286 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 496 KiB/s wr, 10 op/s; 28/453 objects degraded (6.181%) 2024-09-12T23:45:47.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:47 smithi136 bash[75164]: cluster 2024-09-12T23:45:47.395691+0000 mon.a (mon.0) 601 : cluster [DBG] osdmap e493: 8 total, 8 up, 8 in 2024-09-12T23:45:47.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:47 smithi181 bash[67587]: cluster 2024-09-12T23:45:46.381391+0000 mon.a (mon.0) 597 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-12T23:45:47.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:47 smithi181 bash[67587]: cluster 2024-09-12T23:45:46.393408+0000 mon.a (mon.0) 598 : cluster [INF] osd.3 [v2:172.21.15.136:6826/2156253618,v1:172.21.15.136:6827/2156253618] boot 2024-09-12T23:45:47.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:47 smithi181 bash[67587]: cluster 2024-09-12T23:45:46.393565+0000 mon.a (mon.0) 599 : cluster [DBG] osdmap e492: 8 total, 8 up, 8 in 2024-09-12T23:45:47.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:47 smithi181 bash[67587]: audit 2024-09-12T23:45:46.397251+0000 mon.a (mon.0) 600 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-12T23:45:47.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:47 smithi181 bash[67587]: cluster 2024-09-12T23:45:46.729173+0000 mgr.y (mgr.24661) 191 : cluster [DBG] pgmap v313: 121 pgs: 9 peering, 26 active+undersized, 5 active+undersized+degraded, 81 active+clean; 286 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 496 KiB/s wr, 10 op/s; 28/453 objects degraded (6.181%) 2024-09-12T23:45:47.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:47 smithi181 bash[67587]: cluster 2024-09-12T23:45:47.395691+0000 mon.a (mon.0) 601 : cluster [DBG] osdmap e493: 8 total, 8 up, 8 in 2024-09-12T23:45:47.870 INFO:tasks.rados.rados.0.smithi136.stdout:2137: finishing write tid 6 to smithi13667081-31 2024-09-12T23:45:47.871 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 31 v 1446 (ObjNum 697 snap 216 seq_num 697) dirty exists 2024-09-12T23:45:47.871 INFO:tasks.rados.rados.0.smithi136.stdout:2137: left oid 31 (ObjNum 697 snap 216 seq_num 697) 2024-09-12T23:45:47.871 INFO:tasks.rados.rados.0.smithi136.stdout:2138: finishing write tid 1 to smithi13667081-40 2024-09-12T23:45:47.871 INFO:tasks.rados.rados.0.smithi136.stdout:2134: finishing copy_from racing read to smithi13667081-10 2024-09-12T23:45:47.871 INFO:tasks.rados.rados.0.smithi136.stdout:2134: finishing copy_from to smithi13667081-10 2024-09-12T23:45:47.871 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 10 v 1179 (ObjNum 669 snap 196 seq_num 669) dirty exists 2024-09-12T23:45:47.871 INFO:tasks.rados.rados.0.smithi136.stdout:2138: finishing write tid 2 to smithi13667081-40 2024-09-12T23:45:47.871 INFO:tasks.rados.rados.0.smithi136.stdout:2138: finishing write tid 3 to smithi13667081-40 2024-09-12T23:45:47.871 INFO:tasks.rados.rados.0.smithi136.stdout:2138: finishing write tid 4 to smithi13667081-40 2024-09-12T23:45:47.871 INFO:tasks.rados.rados.0.smithi136.stdout:2138: finishing write tid 5 to smithi13667081-40 2024-09-12T23:45:47.871 INFO:tasks.rados.rados.0.smithi136.stdout:2138: finishing write tid 6 to smithi13667081-40 2024-09-12T23:45:47.871 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 40 v 1516 (ObjNum 698 snap 216 seq_num 698) dirty exists 2024-09-12T23:45:47.871 INFO:tasks.rados.rados.0.smithi136.stdout:2138: left oid 40 (ObjNum 698 snap 216 seq_num 698) 2024-09-12T23:45:47.871 INFO:tasks.rados.rados.0.smithi136.stdout:2139: finishing write tid 1 to smithi13667081-35 2024-09-12T23:45:47.872 INFO:tasks.rados.rados.0.smithi136.stdout:2139: finishing write tid 2 to smithi13667081-35 2024-09-12T23:45:47.872 INFO:tasks.rados.rados.0.smithi136.stdout:2139: finishing write tid 3 to smithi13667081-35 2024-09-12T23:45:47.872 INFO:tasks.rados.rados.0.smithi136.stdout:2139: finishing write tid 4 to smithi13667081-35 2024-09-12T23:45:47.872 INFO:tasks.rados.rados.0.smithi136.stdout:2139: finishing write tid 5 to smithi13667081-35 2024-09-12T23:45:47.872 INFO:tasks.rados.rados.0.smithi136.stdout:2139: finishing write tid 6 to smithi13667081-35 2024-09-12T23:45:47.872 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 35 v 1522 (ObjNum 699 snap 216 seq_num 699) dirty exists 2024-09-12T23:45:47.872 INFO:tasks.rados.rados.0.smithi136.stdout:2139: left oid 35 (ObjNum 699 snap 216 seq_num 699) 2024-09-12T23:45:47.872 INFO:tasks.rados.rados.0.smithi136.stdout:2140: finishing rollback tid 1 to smithi13667081-16 2024-09-12T23:45:47.872 INFO:tasks.rados.rados.0.smithi136.stdout:2140: finishing rollback tid 2 to smithi13667081-16 2024-09-12T23:45:47.872 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 16 v 1242 (ObjNum 616 snap 179 seq_num 616) dirty exists 2024-09-12T23:45:47.872 INFO:tasks.rados.rados.0.smithi136.stdout:2134: done (6 left) 2024-09-12T23:45:47.872 INFO:tasks.rados.rados.0.smithi136.stdout:2136: done (5 left) 2024-09-12T23:45:47.872 INFO:tasks.rados.rados.0.smithi136.stdout:2137: done (4 left) 2024-09-12T23:45:47.872 INFO:tasks.rados.rados.0.smithi136.stdout:2138: done (3 left) 2024-09-12T23:45:47.873 INFO:tasks.rados.rados.0.smithi136.stdout:2139: done (2 left) 2024-09-12T23:45:47.873 INFO:tasks.rados.rados.0.smithi136.stdout:2140: done (1 left) 2024-09-12T23:45:47.873 INFO:tasks.rados.rados.0.smithi136.stdout:2142: read oid 26 snap 183 2024-09-12T23:45:47.873 INFO:tasks.rados.rados.0.smithi136.stdout:2142: expect (ObjNum 618 snap 179 seq_num 618) 2024-09-12T23:45:47.873 INFO:tasks.rados.rados.0.smithi136.stdout:2143: read oid 18 snap -1 2024-09-12T23:45:47.873 INFO:tasks.rados.rados.0.smithi136.stdout:2143: expect deleted 2024-09-12T23:45:47.873 INFO:tasks.rados.rados.0.smithi136.stdout:2144: delete oid 20 current snap is 216 2024-09-12T23:45:47.877 INFO:tasks.rados.rados.0.smithi136.stdout:2143: done (3 left) 2024-09-12T23:45:47.877 INFO:tasks.rados.rados.0.smithi136.stdout:2144: done (2 left) 2024-09-12T23:45:47.877 INFO:tasks.rados.rados.0.smithi136.stdout:2145: rollback oid 46 current snap is 216 2024-09-12T23:45:47.877 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 46 to 211 2024-09-12T23:45:47.877 INFO:tasks.rados.rados.0.smithi136.stdout:2146: read oid 13 snap -1 2024-09-12T23:45:47.877 INFO:tasks.rados.rados.0.smithi136.stdout:2146: expect deleted 2024-09-12T23:45:47.877 INFO:tasks.rados.rados.0.smithi136.stdout:2147: read oid 19 snap -1 2024-09-12T23:45:47.877 INFO:tasks.rados.rados.0.smithi136.stdout:2147: expect deleted 2024-09-12T23:45:47.877 INFO:tasks.rados.rados.0.smithi136.stdout:2148: rollback oid 35 current snap is 216 2024-09-12T23:45:47.877 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 35 to 213 2024-09-12T23:45:47.879 INFO:tasks.rados.rados.0.smithi136.stdout:2146: done (5 left) 2024-09-12T23:45:47.879 INFO:tasks.rados.rados.0.smithi136.stdout:2147: done (4 left) 2024-09-12T23:45:47.879 INFO:tasks.rados.rados.0.smithi136.stdout:2149: read oid 27 snap -1 2024-09-12T23:45:47.879 INFO:tasks.rados.rados.0.smithi136.stdout:2149: expect (ObjNum 416 snap 120 seq_num 416) 2024-09-12T23:45:47.879 INFO:tasks.rados.rados.0.smithi136.stdout:2150: delete oid 17 current snap is 216 2024-09-12T23:45:47.882 INFO:tasks.rados.rados.0.smithi136.stdout:2148: finishing rollback tid 0 to smithi13667081-35 2024-09-12T23:45:47.883 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 14 v 1254 (ObjNum 614 snap 177 seq_num 614) dirty exists 2024-09-12T23:45:47.885 INFO:tasks.rados.rados.0.smithi136.stdout:2148: finishing rollback tid 1 to smithi13667081-35 2024-09-12T23:45:47.886 INFO:tasks.rados.rados.0.smithi136.stdout:2148: finishing rollback tid 2 to smithi13667081-35 2024-09-12T23:45:47.886 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 35 v 1525 (ObjNum 671 snap 196 seq_num 671) dirty exists 2024-09-12T23:45:47.888 INFO:tasks.rados.rados.0.smithi136.stdout:2145: finishing rollback tid 0 to smithi13667081-46 2024-09-12T23:45:47.888 INFO:tasks.rados.rados.0.smithi136.stdout:2145: finishing rollback tid 1 to smithi13667081-46 2024-09-12T23:45:47.895 INFO:tasks.rados.rados.0.smithi136.stdout:2142: expect (ObjNum 618 snap 179 seq_num 618) 2024-09-12T23:45:47.992 INFO:tasks.rados.rados.0.smithi136.stdout:2145: finishing rollback tid 2 to smithi13667081-46 2024-09-12T23:45:47.992 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 46 v 662 (ObjNum 684 snap 204 seq_num 684) dirty exists 2024-09-12T23:45:47.992 INFO:tasks.rados.rados.0.smithi136.stdout:2149: expect (ObjNum 416 snap 120 seq_num 416) 2024-09-12T23:45:48.370 INFO:tasks.rados.rados.0.smithi136.stdout:2141: done (5 left) 2024-09-12T23:45:48.370 INFO:tasks.rados.rados.0.smithi136.stdout:2142: done (4 left) 2024-09-12T23:45:48.370 INFO:tasks.rados.rados.0.smithi136.stdout:2145: done (3 left) 2024-09-12T23:45:48.370 INFO:tasks.rados.rados.0.smithi136.stdout:2148: done (2 left) 2024-09-12T23:45:48.370 INFO:tasks.rados.rados.0.smithi136.stdout:2149: done (1 left) 2024-09-12T23:45:48.370 INFO:tasks.rados.rados.0.smithi136.stdout:2150: done (0 left) 2024-09-12T23:45:48.370 INFO:tasks.rados.rados.0.smithi136.stdout:2151: read oid 47 snap -1 2024-09-12T23:45:48.370 INFO:tasks.rados.rados.0.smithi136.stdout:2151: expect (ObjNum 636 snap 188 seq_num 636) 2024-09-12T23:45:48.370 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 46 current snap is 216 2024-09-12T23:45:48.371 INFO:tasks.rados.rados.0.smithi136.stdout:2152: seq_num 702 ranges {2924544=565248,3489792=581632,4071424=745472,4816896=278528} 2024-09-12T23:45:48.418 INFO:tasks.rados.rados.0.smithi136.stdout:2152: writing smithi13667081-46 from 2924544 to 3489792 tid 1 2024-09-12T23:45:48.427 INFO:tasks.rados.rados.0.smithi136.stdout:2152: writing smithi13667081-46 from 3489792 to 4071424 tid 2 2024-09-12T23:45:48.434 INFO:tasks.rados.rados.0.smithi136.stdout:2152: writing smithi13667081-46 from 4071424 to 4816896 tid 3 2024-09-12T23:45:48.437 INFO:tasks.rados.rados.0.smithi136.stdout:2152: writing smithi13667081-46 from 4816896 to 5095424 tid 4 2024-09-12T23:45:48.438 INFO:tasks.rados.rados.0.smithi136.stdout:2153: read oid 33 snap -1 2024-09-12T23:45:48.438 INFO:tasks.rados.rados.0.smithi136.stdout:2153: expect deleted 2024-09-12T23:45:48.438 INFO:tasks.rados.rados.0.smithi136.stdout:2154: rollback oid 32 current snap is 216 2024-09-12T23:45:48.438 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 32 to 210 2024-09-12T23:45:48.438 INFO:tasks.rados.rados.0.smithi136.stdout:2151: expect (ObjNum 636 snap 188 seq_num 636) 2024-09-12T23:45:48.467 INFO:tasks.rados.rados.0.smithi136.stdout:2152: finishing write tid 1 to smithi13667081-46 2024-09-12T23:45:48.467 INFO:tasks.rados.rados.0.smithi136.stdout:2152: finishing write tid 2 to smithi13667081-46 2024-09-12T23:45:48.467 INFO:tasks.rados.rados.0.smithi136.stdout:2152: finishing write tid 3 to smithi13667081-46 2024-09-12T23:45:48.467 INFO:tasks.rados.rados.0.smithi136.stdout:2152: finishing write tid 4 to smithi13667081-46 2024-09-12T23:45:48.467 INFO:tasks.rados.rados.0.smithi136.stdout:2152: finishing write tid 5 to smithi13667081-46 2024-09-12T23:45:48.467 INFO:tasks.rados.rados.0.smithi136.stdout:2152: finishing write tid 6 to smithi13667081-46 2024-09-12T23:45:48.468 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 46 v 667 (ObjNum 702 snap 216 seq_num 702) dirty exists 2024-09-12T23:45:48.468 INFO:tasks.rados.rados.0.smithi136.stdout:2152: left oid 46 (ObjNum 702 snap 216 seq_num 702) 2024-09-12T23:45:48.468 INFO:tasks.rados.rados.0.smithi136.stdout:2154: finishing rollback tid 0 to smithi13667081-32 2024-09-12T23:45:48.468 INFO:tasks.rados.rados.0.smithi136.stdout:2154: finishing rollback tid 1 to smithi13667081-32 2024-09-12T23:45:48.468 INFO:tasks.rados.rados.0.smithi136.stdout:2154: finishing rollback tid 2 to smithi13667081-32 2024-09-12T23:45:48.468 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 32 v 782 (ObjNum 645 snap 190 seq_num 645) dirty exists 2024-09-12T23:45:48.468 INFO:tasks.rados.rados.0.smithi136.stdout:2151: done (3 left) 2024-09-12T23:45:48.468 INFO:tasks.rados.rados.0.smithi136.stdout:2152: done (2 left) 2024-09-12T23:45:48.468 INFO:tasks.rados.rados.0.smithi136.stdout:2153: done (1 left) 2024-09-12T23:45:48.468 INFO:tasks.rados.rados.0.smithi136.stdout:2154: done (0 left) 2024-09-12T23:45:48.468 INFO:tasks.rados.rados.0.smithi136.stdout:2155: delete oid 15 current snap is 216 2024-09-12T23:45:48.471 INFO:tasks.rados.rados.0.smithi136.stdout:2155: done (0 left) 2024-09-12T23:45:48.471 INFO:tasks.rados.rados.0.smithi136.stdout:2156: snap_create 2024-09-12T23:45:48.531 INFO:tasks.rados.rados.0.smithi136.stdout:2156: done (0 left) 2024-09-12T23:45:48.531 INFO:tasks.rados.rados.0.smithi136.stdout:2157: copy_from oid 25 from oid 32 current snap is 217 2024-09-12T23:45:48.531 INFO:tasks.rados.rados.0.smithi136.stdout:2158: snap_create 2024-09-12T23:45:48.557 INFO:tasks.rados.rados.0.smithi136.stdout:2157: finishing copy_from racing read to smithi13667081-25 2024-09-12T23:45:48.559 INFO:tasks.rados.rados.0.smithi136.stdout:2157: finishing copy_from to smithi13667081-25 2024-09-12T23:45:48.559 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 25 v 782 (ObjNum 645 snap 190 seq_num 645) dirty exists 2024-09-12T23:45:49.544 INFO:tasks.rados.rados.0.smithi136.stdout:2157: done (1 left) 2024-09-12T23:45:49.544 INFO:tasks.rados.rados.0.smithi136.stdout:2158: done (0 left) 2024-09-12T23:45:49.544 INFO:tasks.rados.rados.0.smithi136.stdout:2159: delete oid 44 current snap is 218 2024-09-12T23:45:49.555 INFO:tasks.rados.rados.0.smithi136.stdout:2159: done (0 left) 2024-09-12T23:45:49.555 INFO:tasks.rados.rados.0.smithi136.stdout:2160: copy_from oid 39 from oid 22 current snap is 218 2024-09-12T23:45:49.555 INFO:tasks.rados.rados.0.smithi136.stdout:2161: snap_remove snap 216 2024-09-12T23:45:49.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:49 smithi136 bash[76371]: cluster 2024-09-12T23:45:48.534756+0000 mon.a (mon.0) 602 : cluster [DBG] osdmap e494: 8 total, 8 up, 8 in 2024-09-12T23:45:49.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:49 smithi136 bash[76371]: cluster 2024-09-12T23:45:48.729937+0000 mgr.y (mgr.24661) 192 : cluster [DBG] pgmap v316: 121 pgs: 9 peering, 22 active+undersized, 3 active+undersized+degraded, 87 active+clean; 288 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.5 MiB/s rd, 1.0 MiB/s wr, 11 op/s; 18/462 objects degraded (3.896%); 2.1 MiB/s, 0 objects/s recovering 2024-09-12T23:45:49.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:49 smithi136 bash[75164]: cluster 2024-09-12T23:45:48.534756+0000 mon.a (mon.0) 602 : cluster [DBG] osdmap e494: 8 total, 8 up, 8 in 2024-09-12T23:45:49.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:49 smithi136 bash[75164]: cluster 2024-09-12T23:45:48.729937+0000 mgr.y (mgr.24661) 192 : cluster [DBG] pgmap v316: 121 pgs: 9 peering, 22 active+undersized, 3 active+undersized+degraded, 87 active+clean; 288 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.5 MiB/s rd, 1.0 MiB/s wr, 11 op/s; 18/462 objects degraded (3.896%); 2.1 MiB/s, 0 objects/s recovering 2024-09-12T23:45:49.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:45:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:45:49] "GET /metrics HTTP/1.1" 200 36396 "" "Prometheus/2.43.0" 2024-09-12T23:45:49.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:49 smithi181 bash[67587]: cluster 2024-09-12T23:45:48.534756+0000 mon.a (mon.0) 602 : cluster [DBG] osdmap e494: 8 total, 8 up, 8 in 2024-09-12T23:45:49.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:49 smithi181 bash[67587]: cluster 2024-09-12T23:45:48.729937+0000 mgr.y (mgr.24661) 192 : cluster [DBG] pgmap v316: 121 pgs: 9 peering, 22 active+undersized, 3 active+undersized+degraded, 87 active+clean; 288 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.5 MiB/s rd, 1.0 MiB/s wr, 11 op/s; 18/462 objects degraded (3.896%); 2.1 MiB/s, 0 objects/s recovering 2024-09-12T23:45:50.550 INFO:tasks.rados.rados.0.smithi136.stdout:2161: done (1 left) 2024-09-12T23:45:50.550 INFO:tasks.rados.rados.0.smithi136.stdout:2162: read oid 14 snap -1 2024-09-12T23:45:50.550 INFO:tasks.rados.rados.0.smithi136.stdout:2162: expect (ObjNum 614 snap 177 seq_num 614) 2024-09-12T23:45:50.551 INFO:tasks.rados.rados.0.smithi136.stdout:2163: snap_create 2024-09-12T23:45:50.551 INFO:tasks.rados.rados.0.smithi136.stdout:2160: finishing copy_from racing read to smithi13667081-39 2024-09-12T23:45:50.551 INFO:tasks.rados.rados.0.smithi136.stdout:2160: finishing copy_from to smithi13667081-39 2024-09-12T23:45:50.551 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 39 v 845 (ObjNum 687 snap 207 seq_num 687) dirty exists 2024-09-12T23:45:50.594 INFO:tasks.rados.rados.0.smithi136.stdout:2162: expect (ObjNum 614 snap 177 seq_num 614) 2024-09-12T23:45:50.801 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:50 smithi136 bash[75164]: cluster 2024-09-12T23:45:49.547363+0000 mon.a (mon.0) 603 : cluster [DBG] osdmap e495: 8 total, 8 up, 8 in 2024-09-12T23:45:50.801 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:50 smithi136 bash[76371]: cluster 2024-09-12T23:45:49.547363+0000 mon.a (mon.0) 603 : cluster [DBG] osdmap e495: 8 total, 8 up, 8 in 2024-09-12T23:45:50.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:50 smithi181 bash[67587]: cluster 2024-09-12T23:45:49.547363+0000 mon.a (mon.0) 603 : cluster [DBG] osdmap e495: 8 total, 8 up, 8 in 2024-09-12T23:45:51.560 INFO:tasks.rados.rados.0.smithi136.stdout:2160: done (2 left) 2024-09-12T23:45:51.560 INFO:tasks.rados.rados.0.smithi136.stdout:2162: done (1 left) 2024-09-12T23:45:51.560 INFO:tasks.rados.rados.0.smithi136.stdout:2163: done (0 left) 2024-09-12T23:45:51.560 INFO:tasks.rados.rados.0.smithi136.stdout:2164: snap_remove snap 217 2024-09-12T23:45:51.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:51 smithi181 bash[67587]: cluster 2024-09-12T23:45:50.553675+0000 mon.a (mon.0) 604 : cluster [DBG] osdmap e496: 8 total, 8 up, 8 in 2024-09-12T23:45:51.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:51 smithi181 bash[67587]: cluster 2024-09-12T23:45:50.730413+0000 mgr.y (mgr.24661) 193 : cluster [DBG] pgmap v319: 121 pgs: 9 peering, 112 active+clean; 300 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.7 MiB/s rd, 4.0 MiB/s wr, 12 op/s; 2.6 MiB/s, 0 objects/s recovering 2024-09-12T23:45:51.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:51 smithi181 bash[67587]: cluster 2024-09-12T23:45:51.347452+0000 mon.a (mon.0) 605 : cluster [WRN] Health check update: Degraded data redundancy: 18/462 objects degraded (3.896%), 3 pgs degraded (PG_DEGRADED) 2024-09-12T23:45:52.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:51 smithi136 bash[75164]: cluster 2024-09-12T23:45:50.553675+0000 mon.a (mon.0) 604 : cluster [DBG] osdmap e496: 8 total, 8 up, 8 in 2024-09-12T23:45:52.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:51 smithi136 bash[75164]: cluster 2024-09-12T23:45:50.730413+0000 mgr.y (mgr.24661) 193 : cluster [DBG] pgmap v319: 121 pgs: 9 peering, 112 active+clean; 300 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.7 MiB/s rd, 4.0 MiB/s wr, 12 op/s; 2.6 MiB/s, 0 objects/s recovering 2024-09-12T23:45:52.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:51 smithi136 bash[75164]: cluster 2024-09-12T23:45:51.347452+0000 mon.a (mon.0) 605 : cluster [WRN] Health check update: Degraded data redundancy: 18/462 objects degraded (3.896%), 3 pgs degraded (PG_DEGRADED) 2024-09-12T23:45:52.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:51 smithi136 bash[76371]: cluster 2024-09-12T23:45:50.553675+0000 mon.a (mon.0) 604 : cluster [DBG] osdmap e496: 8 total, 8 up, 8 in 2024-09-12T23:45:52.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:51 smithi136 bash[76371]: cluster 2024-09-12T23:45:50.730413+0000 mgr.y (mgr.24661) 193 : cluster [DBG] pgmap v319: 121 pgs: 9 peering, 112 active+clean; 300 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.7 MiB/s rd, 4.0 MiB/s wr, 12 op/s; 2.6 MiB/s, 0 objects/s recovering 2024-09-12T23:45:52.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:51 smithi136 bash[76371]: cluster 2024-09-12T23:45:51.347452+0000 mon.a (mon.0) 605 : cluster [WRN] Health check update: Degraded data redundancy: 18/462 objects degraded (3.896%), 3 pgs degraded (PG_DEGRADED) 2024-09-12T23:45:52.557 INFO:tasks.rados.rados.0.smithi136.stdout:2164: done (0 left) 2024-09-12T23:45:52.558 INFO:tasks.rados.rados.0.smithi136.stdout:2165: setattr oid 27 current snap is 219 2024-09-12T23:45:52.559 INFO:tasks.rados.rados.0.smithi136.stdout:2166: read oid 46 snap 218 2024-09-12T23:45:52.559 INFO:tasks.rados.rados.0.smithi136.stdout:2166: expect (ObjNum 702 snap 216 seq_num 702) 2024-09-12T23:45:52.559 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 22 current snap is 219 2024-09-12T23:45:52.559 INFO:tasks.rados.rados.0.smithi136.stdout:2167: seq_num 705 ranges {5439488=614400,6053888=630784,6684672=524288,7208960=450560} 2024-09-12T23:45:52.565 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:45:52 smithi181 bash[60471]: ts=2024-09-12T23:45:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:45:52.594 INFO:tasks.rados.rados.0.smithi136.stdout:2167: writing smithi13667081-22 from 5439488 to 6053888 tid 1 2024-09-12T23:45:52.597 INFO:tasks.rados.rados.0.smithi136.stdout:2167: writing smithi13667081-22 from 6053888 to 6684672 tid 2 2024-09-12T23:45:52.600 INFO:tasks.rados.rados.0.smithi136.stdout:2167: writing smithi13667081-22 from 6684672 to 7208960 tid 3 2024-09-12T23:45:52.603 INFO:tasks.rados.rados.0.smithi136.stdout:2167: writing smithi13667081-22 from 7208960 to 7659520 tid 4 2024-09-12T23:45:52.603 INFO:tasks.rados.rados.0.smithi136.stdout:2168: copy_from oid 28 from oid 34 current snap is 219 2024-09-12T23:45:52.603 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 27 v 1529 (ObjNum 416 snap 120 seq_num 416) dirty exists 2024-09-12T23:45:52.604 INFO:tasks.rados.rados.0.smithi136.stdout:2166: expect (ObjNum 702 snap 216 seq_num 702) 2024-09-12T23:45:52.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:52 smithi136 bash[75164]: cluster 2024-09-12T23:45:51.547748+0000 mon.a (mon.0) 606 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 18/462 objects degraded (3.896%), 3 pgs degraded) 2024-09-12T23:45:52.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:52 smithi136 bash[75164]: cluster 2024-09-12T23:45:51.547801+0000 mon.a (mon.0) 607 : cluster [INF] Cluster is now healthy 2024-09-12T23:45:52.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:52 smithi136 bash[75164]: cluster 2024-09-12T23:45:51.563800+0000 mon.a (mon.0) 608 : cluster [DBG] osdmap e497: 8 total, 8 up, 8 in 2024-09-12T23:45:52.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:52 smithi136 bash[76371]: cluster 2024-09-12T23:45:51.547748+0000 mon.a (mon.0) 606 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 18/462 objects degraded (3.896%), 3 pgs degraded) 2024-09-12T23:45:52.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:52 smithi136 bash[76371]: cluster 2024-09-12T23:45:51.547801+0000 mon.a (mon.0) 607 : cluster [INF] Cluster is now healthy 2024-09-12T23:45:52.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:52 smithi136 bash[76371]: cluster 2024-09-12T23:45:51.563800+0000 mon.a (mon.0) 608 : cluster [DBG] osdmap e497: 8 total, 8 up, 8 in 2024-09-12T23:45:52.804 INFO:tasks.rados.rados.0.smithi136.stdout:2167: finishing write tid 1 to smithi13667081-22 2024-09-12T23:45:52.804 INFO:tasks.rados.rados.0.smithi136.stdout:2167: finishing write tid 2 to smithi13667081-22 2024-09-12T23:45:52.804 INFO:tasks.rados.rados.0.smithi136.stdout:2165: done (3 left) 2024-09-12T23:45:52.804 INFO:tasks.rados.rados.0.smithi136.stdout:2166: done (2 left) 2024-09-12T23:45:52.804 INFO:tasks.rados.rados.0.smithi136.stdout:2169: rollback oid 39 current snap is 219 2024-09-12T23:45:52.804 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 39 to 183 2024-09-12T23:45:52.804 INFO:tasks.rados.rados.0.smithi136.stdout:2167: finishing write tid 3 to smithi13667081-22 2024-09-12T23:45:52.805 INFO:tasks.rados.rados.0.smithi136.stdout:2167: finishing write tid 4 to smithi13667081-22 2024-09-12T23:45:52.805 INFO:tasks.rados.rados.0.smithi136.stdout:2167: finishing write tid 5 to smithi13667081-22 2024-09-12T23:45:52.805 INFO:tasks.rados.rados.0.smithi136.stdout:2167: finishing write tid 6 to smithi13667081-22 2024-09-12T23:45:52.805 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 22 v 851 (ObjNum 705 snap 219 seq_num 705) dirty exists 2024-09-12T23:45:52.805 INFO:tasks.rados.rados.0.smithi136.stdout:2167: left oid 22 (ObjNum 705 snap 219 seq_num 705) 2024-09-12T23:45:52.805 INFO:tasks.rados.rados.0.smithi136.stdout:2168: finishing copy_from racing read to smithi13667081-28 2024-09-12T23:45:52.805 INFO:tasks.rados.rados.0.smithi136.stdout:2168: finishing copy_from to smithi13667081-28 2024-09-12T23:45:52.805 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 28 v 1208 (ObjNum 607 snap 175 seq_num 607) dirty exists 2024-09-12T23:45:52.805 INFO:tasks.rados.rados.0.smithi136.stdout:2167: done (2 left) 2024-09-12T23:45:52.805 INFO:tasks.rados.rados.0.smithi136.stdout:2168: done (1 left) 2024-09-12T23:45:52.805 INFO:tasks.rados.rados.0.smithi136.stdout:2170: read oid 12 snap -1 2024-09-12T23:45:52.805 INFO:tasks.rados.rados.0.smithi136.stdout:2170: expect deleted 2024-09-12T23:45:52.805 INFO:tasks.rados.rados.0.smithi136.stdout:2171: copy_from oid 21 from oid 15 current snap is 219 2024-09-12T23:45:52.806 INFO:tasks.rados.rados.0.smithi136.stdout:2172: copy_from oid 13 from oid 36 current snap is 219 2024-09-12T23:45:52.806 INFO:tasks.rados.rados.0.smithi136.stdout:2173: snap_remove snap 199 2024-09-12T23:45:52.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:52 smithi181 bash[67587]: cluster 2024-09-12T23:45:51.547748+0000 mon.a (mon.0) 606 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 18/462 objects degraded (3.896%), 3 pgs degraded) 2024-09-12T23:45:52.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:52 smithi181 bash[67587]: cluster 2024-09-12T23:45:51.547801+0000 mon.a (mon.0) 607 : cluster [INF] Cluster is now healthy 2024-09-12T23:45:52.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:52 smithi181 bash[67587]: cluster 2024-09-12T23:45:51.563800+0000 mon.a (mon.0) 608 : cluster [DBG] osdmap e497: 8 total, 8 up, 8 in 2024-09-12T23:45:53.578 INFO:tasks.rados.rados.0.smithi136.stdout:2173: done (4 left) 2024-09-12T23:45:53.578 INFO:tasks.rados.rados.0.smithi136.stdout:2174: read oid 42 snap -1 2024-09-12T23:45:53.578 INFO:tasks.rados.rados.0.smithi136.stdout:2174: expect (ObjNum 672 snap 197 seq_num 672) 2024-09-12T23:45:53.578 INFO:tasks.rados.rados.0.smithi136.stdout:2175: read oid 47 snap 206 2024-09-12T23:45:53.578 INFO:tasks.rados.rados.0.smithi136.stdout:2175: expect (ObjNum 636 snap 188 seq_num 636) 2024-09-12T23:45:53.578 INFO:tasks.rados.rados.0.smithi136.stdout:2176: snap_create 2024-09-12T23:45:53.578 INFO:tasks.rados.rados.0.smithi136.stdout:2171: finishing copy_from to smithi13667081-21 2024-09-12T23:45:53.578 INFO:tasks.rados.rados.0.smithi136.stdout:2171: got expected ENOENT (src dne) 2024-09-12T23:45:53.578 INFO:tasks.rados.rados.0.smithi136.stdout:2171: finishing copy_from racing read to smithi13667081-21 2024-09-12T23:45:53.579 INFO:tasks.rados.rados.0.smithi136.stdout:2169: finishing rollback tid 0 to smithi13667081-39 2024-09-12T23:45:53.579 INFO:tasks.rados.rados.0.smithi136.stdout:2169: finishing rollback tid 1 to smithi13667081-39 2024-09-12T23:45:53.579 INFO:tasks.rados.rados.0.smithi136.stdout:2169: finishing rollback tid 2 to smithi13667081-39 2024-09-12T23:45:53.579 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 39 v 848 (ObjNum 590 snap 172 seq_num 590) dirty exists 2024-09-12T23:45:53.579 INFO:tasks.rados.rados.0.smithi136.stdout:2172: finishing copy_from racing read to smithi13667081-13 2024-09-12T23:45:53.579 INFO:tasks.rados.rados.0.smithi136.stdout:2172: finishing copy_from to smithi13667081-13 2024-09-12T23:45:53.579 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 13 v 798 (ObjNum 610 snap 175 seq_num 610) dirty exists 2024-09-12T23:45:53.591 INFO:tasks.rados.rados.0.smithi136.stdout:2175: expect (ObjNum 636 snap 188 seq_num 636) 2024-09-12T23:45:53.629 INFO:tasks.rados.rados.0.smithi136.stdout:2174: expect (ObjNum 672 snap 197 seq_num 672) 2024-09-12T23:45:53.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:53 smithi136 bash[75164]: cluster 2024-09-12T23:45:52.561956+0000 mon.a (mon.0) 609 : cluster [DBG] osdmap e498: 8 total, 8 up, 8 in 2024-09-12T23:45:53.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:53 smithi136 bash[75164]: cluster 2024-09-12T23:45:52.730897+0000 mgr.y (mgr.24661) 194 : cluster [DBG] pgmap v322: 121 pgs: 121 active+clean; 314 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 4.0 MiB/s wr, 15 op/s; 12 MiB/s, 1 objects/s recovering 2024-09-12T23:45:53.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:53 smithi136 bash[76371]: cluster 2024-09-12T23:45:52.561956+0000 mon.a (mon.0) 609 : cluster [DBG] osdmap e498: 8 total, 8 up, 8 in 2024-09-12T23:45:53.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:53 smithi136 bash[76371]: cluster 2024-09-12T23:45:52.730897+0000 mgr.y (mgr.24661) 194 : cluster [DBG] pgmap v322: 121 pgs: 121 active+clean; 314 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 4.0 MiB/s wr, 15 op/s; 12 MiB/s, 1 objects/s recovering 2024-09-12T23:45:53.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:53 smithi181 bash[67587]: cluster 2024-09-12T23:45:52.561956+0000 mon.a (mon.0) 609 : cluster [DBG] osdmap e498: 8 total, 8 up, 8 in 2024-09-12T23:45:53.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:53 smithi181 bash[67587]: cluster 2024-09-12T23:45:52.730897+0000 mgr.y (mgr.24661) 194 : cluster [DBG] pgmap v322: 121 pgs: 121 active+clean; 314 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 4.0 MiB/s wr, 15 op/s; 12 MiB/s, 1 objects/s recovering 2024-09-12T23:45:54.582 INFO:tasks.rados.rados.0.smithi136.stdout:2169: done (6 left) 2024-09-12T23:45:54.582 INFO:tasks.rados.rados.0.smithi136.stdout:2170: done (5 left) 2024-09-12T23:45:54.582 INFO:tasks.rados.rados.0.smithi136.stdout:2171: done (4 left) 2024-09-12T23:45:54.582 INFO:tasks.rados.rados.0.smithi136.stdout:2172: done (3 left) 2024-09-12T23:45:54.582 INFO:tasks.rados.rados.0.smithi136.stdout:2174: done (2 left) 2024-09-12T23:45:54.582 INFO:tasks.rados.rados.0.smithi136.stdout:2175: done (1 left) 2024-09-12T23:45:54.582 INFO:tasks.rados.rados.0.smithi136.stdout:2176: done (0 left) 2024-09-12T23:45:54.582 INFO:tasks.rados.rados.0.smithi136.stdout:2177: snap_remove snap 206 2024-09-12T23:45:54.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:54 smithi181 bash[67587]: cluster 2024-09-12T23:45:53.581611+0000 mon.a (mon.0) 610 : cluster [DBG] osdmap e499: 8 total, 8 up, 8 in 2024-09-12T23:45:54.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:54 smithi181 bash[67587]: audit 2024-09-12T23:45:53.749670+0000 mon.a (mon.0) 611 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:54.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:54 smithi181 bash[67587]: audit 2024-09-12T23:45:53.751064+0000 mon.a (mon.0) 612 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:45:54.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:54 smithi181 bash[67587]: audit 2024-09-12T23:45:54.260474+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:54.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:54 smithi181 bash[67587]: audit 2024-09-12T23:45:54.270882+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:54.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:54 smithi181 bash[67587]: audit 2024-09-12T23:45:54.272814+0000 mon.a (mon.0) 615 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:45:54.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:54 smithi181 bash[67587]: audit 2024-09-12T23:45:54.274526+0000 mon.a (mon.0) 616 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:45:54.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:54 smithi181 bash[67587]: audit 2024-09-12T23:45:54.284585+0000 mon.a (mon.0) 617 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:54.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:54 smithi181 bash[67587]: audit 2024-09-12T23:45:54.440424+0000 mon.a (mon.0) 618 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:45:54.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:54 smithi181 bash[67587]: audit 2024-09-12T23:45:54.446451+0000 mon.a (mon.0) 619 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:45:54.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:54 smithi181 bash[67587]: audit 2024-09-12T23:45:54.449757+0000 mon.a (mon.0) 620 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:45:54.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:54 smithi181 bash[67587]: audit 2024-09-12T23:45:54.452143+0000 mon.a (mon.0) 621 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:45:54.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:54 smithi181 bash[67587]: audit 2024-09-12T23:45:54.455627+0000 mon.a (mon.0) 622 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-12T23:45:55.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:54 smithi136 bash[76371]: cluster 2024-09-12T23:45:53.581611+0000 mon.a (mon.0) 610 : cluster [DBG] osdmap e499: 8 total, 8 up, 8 in 2024-09-12T23:45:55.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:54 smithi136 bash[76371]: audit 2024-09-12T23:45:53.749670+0000 mon.a (mon.0) 611 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:55.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:54 smithi136 bash[76371]: audit 2024-09-12T23:45:53.751064+0000 mon.a (mon.0) 612 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:45:55.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:54 smithi136 bash[76371]: audit 2024-09-12T23:45:54.260474+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:55.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:54 smithi136 bash[76371]: audit 2024-09-12T23:45:54.270882+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:55.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:54 smithi136 bash[76371]: audit 2024-09-12T23:45:54.272814+0000 mon.a (mon.0) 615 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:45:55.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:54 smithi136 bash[76371]: audit 2024-09-12T23:45:54.274526+0000 mon.a (mon.0) 616 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:45:55.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:54 smithi136 bash[76371]: audit 2024-09-12T23:45:54.284585+0000 mon.a (mon.0) 617 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:55.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:54 smithi136 bash[76371]: audit 2024-09-12T23:45:54.440424+0000 mon.a (mon.0) 618 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:45:55.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:54 smithi136 bash[76371]: audit 2024-09-12T23:45:54.446451+0000 mon.a (mon.0) 619 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:45:55.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:54 smithi136 bash[76371]: audit 2024-09-12T23:45:54.449757+0000 mon.a (mon.0) 620 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:45:55.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:54 smithi136 bash[76371]: audit 2024-09-12T23:45:54.452143+0000 mon.a (mon.0) 621 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:45:55.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:54 smithi136 bash[76371]: audit 2024-09-12T23:45:54.455627+0000 mon.a (mon.0) 622 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-12T23:45:55.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:54 smithi136 bash[75164]: cluster 2024-09-12T23:45:53.581611+0000 mon.a (mon.0) 610 : cluster [DBG] osdmap e499: 8 total, 8 up, 8 in 2024-09-12T23:45:55.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:54 smithi136 bash[75164]: audit 2024-09-12T23:45:53.749670+0000 mon.a (mon.0) 611 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:55.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:54 smithi136 bash[75164]: audit 2024-09-12T23:45:53.751064+0000 mon.a (mon.0) 612 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:45:55.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:54 smithi136 bash[75164]: audit 2024-09-12T23:45:54.260474+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:55.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:54 smithi136 bash[75164]: audit 2024-09-12T23:45:54.270882+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:55.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:54 smithi136 bash[75164]: audit 2024-09-12T23:45:54.272814+0000 mon.a (mon.0) 615 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:45:55.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:54 smithi136 bash[75164]: audit 2024-09-12T23:45:54.274526+0000 mon.a (mon.0) 616 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:45:55.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:54 smithi136 bash[75164]: audit 2024-09-12T23:45:54.284585+0000 mon.a (mon.0) 617 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:55.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:54 smithi136 bash[75164]: audit 2024-09-12T23:45:54.440424+0000 mon.a (mon.0) 618 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:45:55.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:54 smithi136 bash[75164]: audit 2024-09-12T23:45:54.446451+0000 mon.a (mon.0) 619 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:45:55.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:54 smithi136 bash[75164]: audit 2024-09-12T23:45:54.449757+0000 mon.a (mon.0) 620 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:45:55.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:54 smithi136 bash[75164]: audit 2024-09-12T23:45:54.452143+0000 mon.a (mon.0) 621 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:45:55.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:54 smithi136 bash[75164]: audit 2024-09-12T23:45:54.455627+0000 mon.a (mon.0) 622 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-12T23:45:55.598 INFO:tasks.rados.rados.0.smithi136.stdout:2177: done (0 left) 2024-09-12T23:45:55.598 INFO:tasks.rados.rados.0.smithi136.stdout:2178: copy_from oid 3 from oid 28 current snap is 220 2024-09-12T23:45:55.598 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 45 current snap is 220 2024-09-12T23:45:55.598 INFO:tasks.rados.rados.0.smithi136.stdout:2179: seq_num 706 ranges {8331264=630784,8962048=516096,9478144=729088,10207232=360448} 2024-09-12T23:45:55.652 INFO:tasks.rados.rados.0.smithi136.stdout:2179: writing smithi13667081-45 from 8331264 to 8962048 tid 1 2024-09-12T23:45:55.655 INFO:tasks.rados.rados.0.smithi136.stdout:2179: writing smithi13667081-45 from 8962048 to 9478144 tid 2 2024-09-12T23:45:55.659 INFO:tasks.rados.rados.0.smithi136.stdout:2179: writing smithi13667081-45 from 9478144 to 10207232 tid 3 2024-09-12T23:45:55.663 INFO:tasks.rados.rados.0.smithi136.stdout:2179: writing smithi13667081-45 from 10207232 to 10567680 tid 4 2024-09-12T23:45:55.663 INFO:tasks.rados.rados.0.smithi136.stdout:2180: read oid 14 snap -1 2024-09-12T23:45:55.663 INFO:tasks.rados.rados.0.smithi136.stdout:2180: expect (ObjNum 614 snap 177 seq_num 614) 2024-09-12T23:45:55.663 INFO:tasks.rados.rados.0.smithi136.stdout:2181: snap_create 2024-09-12T23:45:55.664 INFO:tasks.rados.rados.0.smithi136.stdout:2178: finishing copy_from racing read to smithi13667081-3 2024-09-12T23:45:55.673 INFO:tasks.rados.rados.0.smithi136.stdout:2179: finishing write tid 1 to smithi13667081-45 2024-09-12T23:45:55.673 INFO:tasks.rados.rados.0.smithi136.stdout:2179: finishing write tid 2 to smithi13667081-45 2024-09-12T23:45:55.673 INFO:tasks.rados.rados.0.smithi136.stdout:2179: finishing write tid 3 to smithi13667081-45 2024-09-12T23:45:55.673 INFO:tasks.rados.rados.0.smithi136.stdout:2179: finishing write tid 4 to smithi13667081-45 2024-09-12T23:45:55.673 INFO:tasks.rados.rados.0.smithi136.stdout:2179: finishing write tid 5 to smithi13667081-45 2024-09-12T23:45:55.676 INFO:tasks.rados.rados.0.smithi136.stdout:2179: finishing write tid 6 to smithi13667081-45 2024-09-12T23:45:55.676 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 45 v 809 (ObjNum 706 snap 220 seq_num 706) dirty exists 2024-09-12T23:45:55.676 INFO:tasks.rados.rados.0.smithi136.stdout:2179: left oid 45 (ObjNum 706 snap 220 seq_num 706) 2024-09-12T23:45:55.676 INFO:tasks.rados.rados.0.smithi136.stdout:2178: finishing copy_from to smithi13667081-3 2024-09-12T23:45:55.676 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 3 v 1208 (ObjNum 607 snap 175 seq_num 607) dirty exists 2024-09-12T23:45:55.698 INFO:tasks.rados.rados.0.smithi136.stdout:2180: expect (ObjNum 614 snap 177 seq_num 614) 2024-09-12T23:45:56.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:55 smithi136 bash[75164]: audit 2024-09-12T23:45:54.455975+0000 mgr.y (mgr.24661) 195 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-12T23:45:56.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:55 smithi136 bash[75164]: cephadm 2024-09-12T23:45:54.458058+0000 mgr.y (mgr.24661) 196 : cephadm [INF] Upgrade: osd.4 is safe to restart 2024-09-12T23:45:56.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:55 smithi136 bash[75164]: cluster 2024-09-12T23:45:54.585438+0000 mon.a (mon.0) 623 : cluster [DBG] osdmap e500: 8 total, 8 up, 8 in 2024-09-12T23:45:56.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:55 smithi136 bash[75164]: cluster 2024-09-12T23:45:54.731933+0000 mgr.y (mgr.24661) 197 : cluster [DBG] pgmap v325: 121 pgs: 121 active+clean; 317 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.6 MiB/s rd, 1.7 MiB/s wr, 20 op/s; 12 MiB/s, 1 objects/s recovering 2024-09-12T23:45:56.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:55 smithi136 bash[76371]: audit 2024-09-12T23:45:54.455975+0000 mgr.y (mgr.24661) 195 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-12T23:45:56.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:55 smithi136 bash[76371]: cephadm 2024-09-12T23:45:54.458058+0000 mgr.y (mgr.24661) 196 : cephadm [INF] Upgrade: osd.4 is safe to restart 2024-09-12T23:45:56.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:55 smithi136 bash[76371]: cluster 2024-09-12T23:45:54.585438+0000 mon.a (mon.0) 623 : cluster [DBG] osdmap e500: 8 total, 8 up, 8 in 2024-09-12T23:45:56.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:55 smithi136 bash[76371]: cluster 2024-09-12T23:45:54.731933+0000 mgr.y (mgr.24661) 197 : cluster [DBG] pgmap v325: 121 pgs: 121 active+clean; 317 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.6 MiB/s rd, 1.7 MiB/s wr, 20 op/s; 12 MiB/s, 1 objects/s recovering 2024-09-12T23:45:56.011 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:55 smithi181 bash[67587]: audit 2024-09-12T23:45:54.455975+0000 mgr.y (mgr.24661) 195 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-12T23:45:56.012 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:55 smithi181 bash[67587]: cephadm 2024-09-12T23:45:54.458058+0000 mgr.y (mgr.24661) 196 : cephadm [INF] Upgrade: osd.4 is safe to restart 2024-09-12T23:45:56.012 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:55 smithi181 bash[67587]: cluster 2024-09-12T23:45:54.585438+0000 mon.a (mon.0) 623 : cluster [DBG] osdmap e500: 8 total, 8 up, 8 in 2024-09-12T23:45:56.012 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:55 smithi181 bash[67587]: cluster 2024-09-12T23:45:54.731933+0000 mgr.y (mgr.24661) 197 : cluster [DBG] pgmap v325: 121 pgs: 121 active+clean; 317 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.6 MiB/s rd, 1.7 MiB/s wr, 20 op/s; 12 MiB/s, 1 objects/s recovering 2024-09-12T23:45:56.429 INFO:tasks.rados.rados.0.smithi136.stdout:2178: done (3 left) 2024-09-12T23:45:56.429 INFO:tasks.rados.rados.0.smithi136.stdout:2179: done (2 left) 2024-09-12T23:45:56.429 INFO:tasks.rados.rados.0.smithi136.stdout:2180: done (1 left) 2024-09-12T23:45:56.429 INFO:tasks.rados.rados.0.smithi136.stdout:2181: done (0 left) 2024-09-12T23:45:56.429 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 49 current snap is 221 2024-09-12T23:45:56.429 INFO:tasks.rados.rados.0.smithi136.stdout:2182: seq_num 707 ranges {4308992=647168,4956160=802816,5758976=532480,6291456=270336} 2024-09-12T23:45:56.454 INFO:tasks.rados.rados.0.smithi136.stdout:2182: writing smithi13667081-49 from 4308992 to 4956160 tid 1 2024-09-12T23:45:56.461 INFO:tasks.rados.rados.0.smithi136.stdout:2182: writing smithi13667081-49 from 4956160 to 5758976 tid 2 2024-09-12T23:45:56.464 INFO:tasks.rados.rados.0.smithi136.stdout:2182: writing smithi13667081-49 from 5758976 to 6291456 tid 3 2024-09-12T23:45:56.466 INFO:tasks.rados.rados.0.smithi136.stdout:2182: writing smithi13667081-49 from 6291456 to 6561792 tid 4 2024-09-12T23:45:56.467 INFO:tasks.rados.rados.0.smithi136.stdout:2183: delete oid 40 current snap is 221 2024-09-12T23:45:56.471 INFO:tasks.rados.rados.0.smithi136.stdout:2182: finishing write tid 1 to smithi13667081-49 2024-09-12T23:45:56.476 INFO:tasks.rados.rados.0.smithi136.stdout:2182: finishing write tid 2 to smithi13667081-49 2024-09-12T23:45:56.476 INFO:tasks.rados.rados.0.smithi136.stdout:2182: finishing write tid 3 to smithi13667081-49 2024-09-12T23:45:56.477 INFO:tasks.rados.rados.0.smithi136.stdout:2182: finishing write tid 4 to smithi13667081-49 2024-09-12T23:45:56.477 INFO:tasks.rados.rados.0.smithi136.stdout:2183: done (1 left) 2024-09-12T23:45:56.477 INFO:tasks.rados.rados.0.smithi136.stdout:2184: read oid 6 snap 207 2024-09-12T23:45:56.477 INFO:tasks.rados.rados.0.smithi136.stdout:2184: expect deleted 2024-09-12T23:45:56.477 INFO:tasks.rados.rados.0.smithi136.stdout:2185: rollback oid 3 current snap is 221 2024-09-12T23:45:56.478 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 3 to 210 2024-09-12T23:45:56.478 INFO:tasks.rados.rados.0.smithi136.stdout:2186: read oid 42 snap -1 2024-09-12T23:45:56.478 INFO:tasks.rados.rados.0.smithi136.stdout:2186: expect (ObjNum 672 snap 197 seq_num 672) 2024-09-12T23:45:56.478 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 41 current snap is 221 2024-09-12T23:45:56.478 INFO:tasks.rados.rados.0.smithi136.stdout:2187: seq_num 709 ranges {0=679936,679936=614400,1294336=499712,1794048=491520} 2024-09-12T23:45:56.481 INFO:tasks.rados.rados.0.smithi136.stdout:2187: writing smithi13667081-41 from 0 to 679936 tid 1 2024-09-12T23:45:56.485 INFO:tasks.rados.rados.0.smithi136.stdout:2187: writing smithi13667081-41 from 679936 to 1294336 tid 2 2024-09-12T23:45:56.488 INFO:tasks.rados.rados.0.smithi136.stdout:2187: writing smithi13667081-41 from 1294336 to 1794048 tid 3 2024-09-12T23:45:56.491 INFO:tasks.rados.rados.0.smithi136.stdout:2187: writing smithi13667081-41 from 1794048 to 2285568 tid 4 2024-09-12T23:45:56.491 INFO:tasks.rados.rados.0.smithi136.stdout:2188: read oid 28 snap -1 2024-09-12T23:45:56.492 INFO:tasks.rados.rados.0.smithi136.stdout:2188: expect (ObjNum 607 snap 175 seq_num 607) 2024-09-12T23:45:56.492 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 13 current snap is 221 2024-09-12T23:45:56.492 INFO:tasks.rados.rados.0.smithi136.stdout:2189: seq_num 710 ranges {622592=696320,1318912=499712,1818624=712704,2531328=393216} 2024-09-12T23:45:56.499 INFO:tasks.rados.rados.0.smithi136.stdout:2189: writing smithi13667081-13 from 622592 to 1318912 tid 1 2024-09-12T23:45:56.502 INFO:tasks.rados.rados.0.smithi136.stdout:2189: writing smithi13667081-13 from 1318912 to 1818624 tid 2 2024-09-12T23:45:56.507 INFO:tasks.rados.rados.0.smithi136.stdout:2189: writing smithi13667081-13 from 1818624 to 2531328 tid 3 2024-09-12T23:45:56.509 INFO:tasks.rados.rados.0.smithi136.stdout:2189: writing smithi13667081-13 from 2531328 to 2924544 tid 4 2024-09-12T23:45:56.509 INFO:tasks.rados.rados.0.smithi136.stdout:2190: snap_remove snap 214 2024-09-12T23:45:56.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:56 smithi181 bash[67587]: cluster 2024-09-12T23:45:55.592269+0000 mon.a (mon.0) 624 : cluster [DBG] osdmap e501: 8 total, 8 up, 8 in 2024-09-12T23:45:56.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:56 smithi181 bash[67587]: cluster 2024-09-12T23:45:56.357038+0000 mon.a (mon.0) 625 : cluster [DBG] osdmap e502: 8 total, 8 up, 8 in 2024-09-12T23:45:57.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:56 smithi136 bash[75164]: cluster 2024-09-12T23:45:55.592269+0000 mon.a (mon.0) 624 : cluster [DBG] osdmap e501: 8 total, 8 up, 8 in 2024-09-12T23:45:57.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:56 smithi136 bash[75164]: cluster 2024-09-12T23:45:56.357038+0000 mon.a (mon.0) 625 : cluster [DBG] osdmap e502: 8 total, 8 up, 8 in 2024-09-12T23:45:57.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:56 smithi136 bash[76371]: cluster 2024-09-12T23:45:55.592269+0000 mon.a (mon.0) 624 : cluster [DBG] osdmap e501: 8 total, 8 up, 8 in 2024-09-12T23:45:57.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:56 smithi136 bash[76371]: cluster 2024-09-12T23:45:56.357038+0000 mon.a (mon.0) 625 : cluster [DBG] osdmap e502: 8 total, 8 up, 8 in 2024-09-12T23:45:57.364 INFO:tasks.rados.rados.0.smithi136.stdout:2182: finishing write tid 5 to smithi13667081-49 2024-09-12T23:45:57.364 INFO:tasks.rados.rados.0.smithi136.stdout:2182: finishing write tid 6 to smithi13667081-49 2024-09-12T23:45:57.364 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 49 v 1452 (ObjNum 707 snap 221 seq_num 707) dirty exists 2024-09-12T23:45:57.364 INFO:tasks.rados.rados.0.smithi136.stdout:2182: left oid 49 (ObjNum 707 snap 221 seq_num 707) 2024-09-12T23:45:57.364 INFO:tasks.rados.rados.0.smithi136.stdout:2185: finishing rollback tid 0 to smithi13667081-3 2024-09-12T23:45:57.364 INFO:tasks.rados.rados.0.smithi136.stdout:2185: finishing rollback tid 1 to smithi13667081-3 2024-09-12T23:45:57.364 INFO:tasks.rados.rados.0.smithi136.stdout:2185: finishing rollback tid 2 to smithi13667081-3 2024-09-12T23:45:57.364 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 3 v 1455 (ObjNum 633 snap 186 seq_num 633) dirty exists 2024-09-12T23:45:57.364 INFO:tasks.rados.rados.0.smithi136.stdout:2187: finishing write tid 1 to smithi13667081-41 2024-09-12T23:45:57.364 INFO:tasks.rados.rados.0.smithi136.stdout:2187: finishing write tid 2 to smithi13667081-41 2024-09-12T23:45:57.365 INFO:tasks.rados.rados.0.smithi136.stdout:2186: expect (ObjNum 672 snap 197 seq_num 672) 2024-09-12T23:45:57.630 INFO:tasks.rados.rados.0.smithi136.stdout:2187: finishing write tid 3 to smithi13667081-41 2024-09-12T23:45:57.630 INFO:tasks.rados.rados.0.smithi136.stdout:2182: done (7 left) 2024-09-12T23:45:57.630 INFO:tasks.rados.rados.0.smithi136.stdout:2184: done (6 left) 2024-09-12T23:45:57.630 INFO:tasks.rados.rados.0.smithi136.stdout:2185: done (5 left) 2024-09-12T23:45:57.630 INFO:tasks.rados.rados.0.smithi136.stdout:2186: done (4 left) 2024-09-12T23:45:57.631 INFO:tasks.rados.rados.0.smithi136.stdout:2190: done (3 left) 2024-09-12T23:45:57.631 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 37 current snap is 221 2024-09-12T23:45:57.631 INFO:tasks.rados.rados.0.smithi136.stdout:2191: seq_num 711 ranges {1056768=712704,1769472=786432,2555904=516096,3072000=303104} 2024-09-12T23:45:57.644 INFO:tasks.rados.rados.0.smithi136.stdout:2191: writing smithi13667081-37 from 1056768 to 1769472 tid 1 2024-09-12T23:45:57.651 INFO:tasks.rados.rados.0.smithi136.stdout:2191: writing smithi13667081-37 from 1769472 to 2555904 tid 2 2024-09-12T23:45:57.655 INFO:tasks.rados.rados.0.smithi136.stdout:2191: writing smithi13667081-37 from 2555904 to 3072000 tid 3 2024-09-12T23:45:57.658 INFO:tasks.rados.rados.0.smithi136.stdout:2191: writing smithi13667081-37 from 3072000 to 3375104 tid 4 2024-09-12T23:45:57.659 INFO:tasks.rados.rados.0.smithi136.stdout:2192: read oid 47 snap -1 2024-09-12T23:45:57.659 INFO:tasks.rados.rados.0.smithi136.stdout:2192: expect (ObjNum 636 snap 188 seq_num 636) 2024-09-12T23:45:57.659 INFO:tasks.rados.rados.0.smithi136.stdout:2193: rollback oid 48 current snap is 221 2024-09-12T23:45:57.659 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 48 to 220 2024-09-12T23:45:57.659 INFO:tasks.rados.rados.0.smithi136.stdout:2187: finishing write tid 4 to smithi13667081-41 2024-09-12T23:45:57.659 INFO:tasks.rados.rados.0.smithi136.stdout:2194: read oid 11 snap 201 2024-09-12T23:45:57.659 INFO:tasks.rados.rados.0.smithi136.stdout:2194: expect (ObjNum 605 snap 174 seq_num 605) 2024-09-12T23:45:57.659 INFO:tasks.rados.rados.0.smithi136.stdout:2195: setattr oid 29 current snap is 221 2024-09-12T23:45:57.659 INFO:tasks.rados.rados.0.smithi136.stdout:2187: finishing write tid 5 to smithi13667081-41 2024-09-12T23:45:57.659 INFO:tasks.rados.rados.0.smithi136.stdout:2187: finishing write tid 6 to smithi13667081-41 2024-09-12T23:45:57.659 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 41 v 1237 (ObjNum 709 snap 221 seq_num 709) dirty exists 2024-09-12T23:45:57.659 INFO:tasks.rados.rados.0.smithi136.stdout:2187: left oid 41 (ObjNum 709 snap 221 seq_num 709) 2024-09-12T23:45:57.659 INFO:tasks.rados.rados.0.smithi136.stdout:2189: finishing write tid 1 to smithi13667081-13 2024-09-12T23:45:57.659 INFO:tasks.rados.rados.0.smithi136.stdout:2189: finishing write tid 2 to smithi13667081-13 2024-09-12T23:45:57.660 INFO:tasks.rados.rados.0.smithi136.stdout:2188: expect (ObjNum 607 snap 175 seq_num 607) 2024-09-12T23:45:57.881 INFO:tasks.rados.rados.0.smithi136.stdout:2189: finishing write tid 3 to smithi13667081-13 2024-09-12T23:45:57.881 INFO:tasks.rados.rados.0.smithi136.stdout:2189: finishing write tid 4 to smithi13667081-13 2024-09-12T23:45:57.881 INFO:tasks.rados.rados.0.smithi136.stdout:2189: finishing write tid 5 to smithi13667081-13 2024-09-12T23:45:57.881 INFO:tasks.rados.rados.0.smithi136.stdout:2189: finishing write tid 6 to smithi13667081-13 2024-09-12T23:45:57.881 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 13 v 1227 (ObjNum 710 snap 221 seq_num 710) dirty exists 2024-09-12T23:45:57.881 INFO:tasks.rados.rados.0.smithi136.stdout:2189: left oid 13 (ObjNum 710 snap 221 seq_num 710) 2024-09-12T23:45:57.881 INFO:tasks.rados.rados.0.smithi136.stdout:2191: finishing write tid 1 to smithi13667081-37 2024-09-12T23:45:57.882 INFO:tasks.rados.rados.0.smithi136.stdout:2191: finishing write tid 2 to smithi13667081-37 2024-09-12T23:45:57.882 INFO:tasks.rados.rados.0.smithi136.stdout:2191: finishing write tid 3 to smithi13667081-37 2024-09-12T23:45:57.882 INFO:tasks.rados.rados.0.smithi136.stdout:2193: finishing rollback tid 0 to smithi13667081-48 2024-09-12T23:45:57.882 INFO:tasks.rados.rados.0.smithi136.stdout:2193: finishing rollback tid 1 to smithi13667081-48 2024-09-12T23:45:57.882 INFO:tasks.rados.rados.0.smithi136.stdout:2193: finishing rollback tid 2 to smithi13667081-48 2024-09-12T23:45:57.882 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 48 v 1292 (ObjNum 669 snap 196 seq_num 669) dirty exists 2024-09-12T23:45:57.882 INFO:tasks.rados.rados.0.smithi136.stdout:2192: expect (ObjNum 636 snap 188 seq_num 636) 2024-09-12T23:45:57.908 INFO:tasks.rados.rados.0.smithi136.stdout:2191: finishing write tid 4 to smithi13667081-37 2024-09-12T23:45:57.908 INFO:tasks.rados.rados.0.smithi136.stdout:2191: finishing write tid 5 to smithi13667081-37 2024-09-12T23:45:57.908 INFO:tasks.rados.rados.0.smithi136.stdout:2191: finishing write tid 6 to smithi13667081-37 2024-09-12T23:45:57.908 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 37 v 1242 (ObjNum 711 snap 221 seq_num 711) dirty exists 2024-09-12T23:45:57.908 INFO:tasks.rados.rados.0.smithi136.stdout:2191: left oid 37 (ObjNum 711 snap 221 seq_num 711) 2024-09-12T23:45:57.908 INFO:tasks.rados.rados.0.smithi136.stdout:2194: expect (ObjNum 605 snap 174 seq_num 605) 2024-09-12T23:45:58.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:57 smithi136 bash[75164]: cluster 2024-09-12T23:45:56.732922+0000 mgr.y (mgr.24661) 198 : cluster [DBG] pgmap v328: 121 pgs: 121 active+clean; 330 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 3.8 MiB/s wr, 14 op/s 2024-09-12T23:45:58.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:57 smithi136 bash[75164]: audit 2024-09-12T23:45:56.895194+0000 mon.a (mon.0) 626 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:58.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:57 smithi136 bash[75164]: audit 2024-09-12T23:45:56.896874+0000 mon.a (mon.0) 627 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-12T23:45:58.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:57 smithi136 bash[75164]: audit 2024-09-12T23:45:56.898265+0000 mon.a (mon.0) 628 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:45:58.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:57 smithi136 bash[75164]: cluster 2024-09-12T23:45:57.362002+0000 mon.a (mon.0) 629 : cluster [DBG] osdmap e503: 8 total, 8 up, 8 in 2024-09-12T23:45:58.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:57 smithi136 bash[76371]: cluster 2024-09-12T23:45:56.732922+0000 mgr.y (mgr.24661) 198 : cluster [DBG] pgmap v328: 121 pgs: 121 active+clean; 330 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 3.8 MiB/s wr, 14 op/s 2024-09-12T23:45:58.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:57 smithi136 bash[76371]: audit 2024-09-12T23:45:56.895194+0000 mon.a (mon.0) 626 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:58.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:57 smithi136 bash[76371]: audit 2024-09-12T23:45:56.896874+0000 mon.a (mon.0) 627 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-12T23:45:58.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:57 smithi136 bash[76371]: audit 2024-09-12T23:45:56.898265+0000 mon.a (mon.0) 628 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:45:58.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:57 smithi136 bash[76371]: cluster 2024-09-12T23:45:57.362002+0000 mon.a (mon.0) 629 : cluster [DBG] osdmap e503: 8 total, 8 up, 8 in 2024-09-12T23:45:58.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:57 smithi181 bash[67587]: cluster 2024-09-12T23:45:56.732922+0000 mgr.y (mgr.24661) 198 : cluster [DBG] pgmap v328: 121 pgs: 121 active+clean; 330 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 3.8 MiB/s wr, 14 op/s 2024-09-12T23:45:58.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:57 smithi181 bash[67587]: audit 2024-09-12T23:45:56.895194+0000 mon.a (mon.0) 626 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:45:58.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:57 smithi181 bash[67587]: audit 2024-09-12T23:45:56.896874+0000 mon.a (mon.0) 627 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-12T23:45:58.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:57 smithi181 bash[67587]: audit 2024-09-12T23:45:56.898265+0000 mon.a (mon.0) 628 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:45:58.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:57 smithi181 bash[67587]: cluster 2024-09-12T23:45:57.362002+0000 mon.a (mon.0) 629 : cluster [DBG] osdmap e503: 8 total, 8 up, 8 in 2024-09-12T23:45:58.455 INFO:tasks.rados.rados.0.smithi136.stdout:2187: done (7 left) 2024-09-12T23:45:58.455 INFO:tasks.rados.rados.0.smithi136.stdout:2188: done (6 left) 2024-09-12T23:45:58.455 INFO:tasks.rados.rados.0.smithi136.stdout:2189: done (5 left) 2024-09-12T23:45:58.455 INFO:tasks.rados.rados.0.smithi136.stdout:2191: done (4 left) 2024-09-12T23:45:58.455 INFO:tasks.rados.rados.0.smithi136.stdout:2192: done (3 left) 2024-09-12T23:45:58.455 INFO:tasks.rados.rados.0.smithi136.stdout:2193: done (2 left) 2024-09-12T23:45:58.455 INFO:tasks.rados.rados.0.smithi136.stdout:2194: done (1 left) 2024-09-12T23:45:58.456 INFO:tasks.rados.rados.0.smithi136.stdout:2196: delete oid 19 current snap is 221 2024-09-12T23:45:58.458 INFO:tasks.rados.rados.0.smithi136.stdout:2196: done (1 left) 2024-09-12T23:45:58.458 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 44 current snap is 221 2024-09-12T23:45:58.458 INFO:tasks.rados.rados.0.smithi136.stdout:2197: seq_num 713 ranges {0=745472,745472=557056,1302528=483328,1785856=565248} 2024-09-12T23:45:58.470 INFO:tasks.rados.rados.0.smithi136.stdout:2197: writing smithi13667081-44 from 0 to 745472 tid 1 2024-09-12T23:45:58.480 INFO:tasks.rados.rados.0.smithi136.stdout:2197: writing smithi13667081-44 from 745472 to 1302528 tid 2 2024-09-12T23:45:58.489 INFO:tasks.rados.rados.0.smithi136.stdout:2197: writing smithi13667081-44 from 1302528 to 1785856 tid 3 2024-09-12T23:45:58.499 INFO:tasks.rados.rados.0.smithi136.stdout:2197: writing smithi13667081-44 from 1785856 to 2351104 tid 4 2024-09-12T23:45:58.499 INFO:tasks.rados.rados.0.smithi136.stdout:2198: rollback oid 37 current snap is 221 2024-09-12T23:45:58.499 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 37 to 207 2024-09-12T23:45:58.499 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 29 v 1228 (ObjNum 666 snap 195 seq_num 666) dirty exists 2024-09-12T23:45:58.499 INFO:tasks.rados.rados.0.smithi136.stdout:2197: finishing write tid 1 to smithi13667081-44 2024-09-12T23:45:58.500 INFO:tasks.rados.rados.0.smithi136.stdout:2197: finishing write tid 2 to smithi13667081-44 2024-09-12T23:45:58.500 INFO:tasks.rados.rados.0.smithi136.stdout:2197: finishing write tid 3 to smithi13667081-44 2024-09-12T23:45:58.500 INFO:tasks.rados.rados.0.smithi136.stdout:2195: done (2 left) 2024-09-12T23:45:58.500 INFO:tasks.rados.rados.0.smithi136.stdout:2199: read oid 14 snap -1 2024-09-12T23:45:58.500 INFO:tasks.rados.rados.0.smithi136.stdout:2199: expect (ObjNum 614 snap 177 seq_num 614) 2024-09-12T23:45:58.500 INFO:tasks.rados.rados.0.smithi136.stdout:2200: read oid 50 snap -1 2024-09-12T23:45:58.500 INFO:tasks.rados.rados.0.smithi136.stdout:2200: expect (ObjNum 631 snap 185 seq_num 631) 2024-09-12T23:45:58.500 INFO:tasks.rados.rados.0.smithi136.stdout:2201: snap_create 2024-09-12T23:45:58.505 INFO:tasks.rados.rados.0.smithi136.stdout:2198: finishing rollback tid 0 to smithi13667081-37 2024-09-12T23:45:58.506 INFO:tasks.rados.rados.0.smithi136.stdout:2197: finishing write tid 4 to smithi13667081-44 2024-09-12T23:45:58.510 INFO:tasks.rados.rados.0.smithi136.stdout:2198: finishing rollback tid 1 to smithi13667081-37 2024-09-12T23:45:58.511 INFO:tasks.rados.rados.0.smithi136.stdout:2198: finishing rollback tid 2 to smithi13667081-37 2024-09-12T23:45:58.511 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 37 v 1245 (ObjNum 636 snap 188 seq_num 636) dirty exists 2024-09-12T23:45:58.511 INFO:tasks.rados.rados.0.smithi136.stdout:2200: expect (ObjNum 631 snap 185 seq_num 631) 2024-09-12T23:45:58.578 INFO:tasks.rados.rados.0.smithi136.stdout:2197: finishing write tid 5 to smithi13667081-44 2024-09-12T23:45:58.578 INFO:tasks.rados.rados.0.smithi136.stdout:2199: expect (ObjNum 614 snap 177 seq_num 614) 2024-09-12T23:45:59.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:58 smithi136 bash[75164]: cephadm 2024-09-12T23:45:56.883724+0000 mgr.y (mgr.24661) 199 : cephadm [INF] Upgrade: Updating osd.4 2024-09-12T23:45:59.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:58 smithi136 bash[75164]: cephadm 2024-09-12T23:45:56.903126+0000 mgr.y (mgr.24661) 200 : cephadm [INF] Deploying daemon osd.4 on smithi181 2024-09-12T23:45:59.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:58 smithi136 bash[76371]: cephadm 2024-09-12T23:45:56.883724+0000 mgr.y (mgr.24661) 199 : cephadm [INF] Upgrade: Updating osd.4 2024-09-12T23:45:59.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:58 smithi136 bash[76371]: cephadm 2024-09-12T23:45:56.903126+0000 mgr.y (mgr.24661) 200 : cephadm [INF] Deploying daemon osd.4 on smithi181 2024-09-12T23:45:59.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:58 smithi181 bash[67587]: cephadm 2024-09-12T23:45:56.883724+0000 mgr.y (mgr.24661) 199 : cephadm [INF] Upgrade: Updating osd.4 2024-09-12T23:45:59.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:58 smithi181 bash[67587]: cephadm 2024-09-12T23:45:56.903126+0000 mgr.y (mgr.24661) 200 : cephadm [INF] Deploying daemon osd.4 on smithi181 2024-09-12T23:45:59.264 INFO:tasks.rados.rados.0.smithi136.stdout:2197: finishing write tid 6 to smithi13667081-44 2024-09-12T23:45:59.264 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 44 v 1297 (ObjNum 713 snap 221 seq_num 713) dirty exists 2024-09-12T23:45:59.264 INFO:tasks.rados.rados.0.smithi136.stdout:2197: left oid 44 (ObjNum 713 snap 221 seq_num 713) 2024-09-12T23:45:59.264 INFO:tasks.rados.rados.0.smithi136.stdout:2197: done (4 left) 2024-09-12T23:45:59.264 INFO:tasks.rados.rados.0.smithi136.stdout:2198: done (3 left) 2024-09-12T23:45:59.265 INFO:tasks.rados.rados.0.smithi136.stdout:2199: done (2 left) 2024-09-12T23:45:59.265 INFO:tasks.rados.rados.0.smithi136.stdout:2200: done (1 left) 2024-09-12T23:45:59.265 INFO:tasks.rados.rados.0.smithi136.stdout:2201: done (0 left) 2024-09-12T23:45:59.265 INFO:tasks.rados.rados.0.smithi136.stdout:2202: read oid 48 snap -1 2024-09-12T23:45:59.265 INFO:tasks.rados.rados.0.smithi136.stdout:2202: expect (ObjNum 669 snap 196 seq_num 669) 2024-09-12T23:45:59.265 INFO:tasks.rados.rados.0.smithi136.stdout:2203: read oid 22 snap -1 2024-09-12T23:45:59.265 INFO:tasks.rados.rados.0.smithi136.stdout:2203: expect (ObjNum 705 snap 219 seq_num 705) 2024-09-12T23:45:59.265 INFO:tasks.rados.rados.0.smithi136.stdout:2204: delete oid 49 current snap is 222 2024-09-12T23:45:59.270 INFO:tasks.rados.rados.0.smithi136.stdout:2204: done (2 left) 2024-09-12T23:45:59.270 INFO:tasks.rados.rados.0.smithi136.stdout:2205: read oid 8 snap -1 2024-09-12T23:45:59.270 INFO:tasks.rados.rados.0.smithi136.stdout:2205: expect deleted 2024-09-12T23:45:59.270 INFO:tasks.rados.rados.0.smithi136.stdout:2206: snap_remove snap 205 2024-09-12T23:45:59.658 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:45:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:45:59] "GET /metrics HTTP/1.1" 200 36408 "" "Prometheus/2.43.0" 2024-09-12T23:45:59.876 INFO:tasks.rados.rados.0.smithi136.stdout:2206: done (3 left) 2024-09-12T23:45:59.877 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 47 current snap is 222 2024-09-12T23:45:59.877 INFO:tasks.rados.rados.0.smithi136.stdout:2207: seq_num 715 ranges {1056768=778240,1835008=770048,2605056=499712,3104768=335872} 2024-09-12T23:45:59.910 INFO:tasks.rados.rados.0.smithi136.stdout:2207: writing smithi13667081-47 from 1056768 to 1835008 tid 1 2024-09-12T23:45:59.925 INFO:tasks.rados.rados.0.smithi136.stdout:2207: writing smithi13667081-47 from 1835008 to 2605056 tid 2 2024-09-12T23:45:59.933 INFO:tasks.rados.rados.0.smithi136.stdout:2207: writing smithi13667081-47 from 2605056 to 3104768 tid 3 2024-09-12T23:45:59.938 INFO:tasks.rados.rados.0.smithi136.stdout:2207: writing smithi13667081-47 from 3104768 to 3440640 tid 4 2024-09-12T23:45:59.938 INFO:tasks.rados.rados.0.smithi136.stdout:2208: copy_from oid 5 from oid 49 current snap is 222 2024-09-12T23:45:59.938 INFO:tasks.rados.rados.0.smithi136.stdout:2202: expect (ObjNum 669 snap 196 seq_num 669) 2024-09-12T23:46:00.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:59 smithi136 bash[75164]: cluster 2024-09-12T23:45:58.659682+0000 mon.a (mon.0) 630 : cluster [DBG] osdmap e504: 8 total, 8 up, 8 in 2024-09-12T23:46:00.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:59 smithi136 bash[75164]: cluster 2024-09-12T23:45:58.733529+0000 mgr.y (mgr.24661) 201 : cluster [DBG] pgmap v331: 121 pgs: 121 active+clean; 330 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 3.1 MiB/s wr, 9 op/s 2024-09-12T23:46:00.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:45:59 smithi136 bash[75164]: audit 2024-09-12T23:45:58.875539+0000 mon.a (mon.0) 631 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:00.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:59 smithi136 bash[76371]: cluster 2024-09-12T23:45:58.659682+0000 mon.a (mon.0) 630 : cluster [DBG] osdmap e504: 8 total, 8 up, 8 in 2024-09-12T23:46:00.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:59 smithi136 bash[76371]: cluster 2024-09-12T23:45:58.733529+0000 mgr.y (mgr.24661) 201 : cluster [DBG] pgmap v331: 121 pgs: 121 active+clean; 330 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 3.1 MiB/s wr, 9 op/s 2024-09-12T23:46:00.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:45:59 smithi136 bash[76371]: audit 2024-09-12T23:45:58.875539+0000 mon.a (mon.0) 631 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:00.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:59 smithi181 bash[67587]: cluster 2024-09-12T23:45:58.659682+0000 mon.a (mon.0) 630 : cluster [DBG] osdmap e504: 8 total, 8 up, 8 in 2024-09-12T23:46:00.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:59 smithi181 bash[67587]: cluster 2024-09-12T23:45:58.733529+0000 mgr.y (mgr.24661) 201 : cluster [DBG] pgmap v331: 121 pgs: 121 active+clean; 330 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 3.1 MiB/s wr, 9 op/s 2024-09-12T23:46:00.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:45:59 smithi181 bash[67587]: audit 2024-09-12T23:45:58.875539+0000 mon.a (mon.0) 631 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:00.184 INFO:tasks.rados.rados.0.smithi136.stdout:2203: expect (ObjNum 705 snap 219 seq_num 705) 2024-09-12T23:46:00.524 INFO:tasks.rados.rados.0.smithi136.stdout:2207: finishing write tid 1 to smithi13667081-47 2024-09-12T23:46:00.524 INFO:tasks.rados.rados.0.smithi136.stdout:2207: finishing write tid 2 to smithi13667081-47 2024-09-12T23:46:00.524 INFO:tasks.rados.rados.0.smithi136.stdout:2207: finishing write tid 3 to smithi13667081-47 2024-09-12T23:46:00.524 INFO:tasks.rados.rados.0.smithi136.stdout:2208: finishing copy_from to smithi13667081-5 2024-09-12T23:46:00.524 INFO:tasks.rados.rados.0.smithi136.stdout:2208: got expected ENOENT (src dne) 2024-09-12T23:46:00.524 INFO:tasks.rados.rados.0.smithi136.stdout:2208: finishing copy_from racing read to smithi13667081-5 2024-09-12T23:46:00.524 INFO:tasks.rados.rados.0.smithi136.stdout:2207: finishing write tid 4 to smithi13667081-47 2024-09-12T23:46:00.525 INFO:tasks.rados.rados.0.smithi136.stdout:2207: finishing write tid 5 to smithi13667081-47 2024-09-12T23:46:00.525 INFO:tasks.rados.rados.0.smithi136.stdout:2207: finishing write tid 6 to smithi13667081-47 2024-09-12T23:46:00.525 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 47 v 705 (ObjNum 715 snap 222 seq_num 715) dirty exists 2024-09-12T23:46:00.525 INFO:tasks.rados.rados.0.smithi136.stdout:2207: left oid 47 (ObjNum 715 snap 222 seq_num 715) 2024-09-12T23:46:00.525 INFO:tasks.rados.rados.0.smithi136.stdout:2202: done (4 left) 2024-09-12T23:46:00.525 INFO:tasks.rados.rados.0.smithi136.stdout:2203: done (3 left) 2024-09-12T23:46:00.525 INFO:tasks.rados.rados.0.smithi136.stdout:2205: done (2 left) 2024-09-12T23:46:00.525 INFO:tasks.rados.rados.0.smithi136.stdout:2207: done (1 left) 2024-09-12T23:46:00.525 INFO:tasks.rados.rados.0.smithi136.stdout:2208: done (0 left) 2024-09-12T23:46:00.525 INFO:tasks.rados.rados.0.smithi136.stdout:2209: snap_create 2024-09-12T23:46:00.887 INFO:tasks.rados.rados.0.smithi136.stdout:2209: done (0 left) 2024-09-12T23:46:00.887 INFO:tasks.rados.rados.0.smithi136.stdout:2210: copy_from oid 24 from oid 14 current snap is 223 2024-09-12T23:46:00.887 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 10 current snap is 223 2024-09-12T23:46:00.887 INFO:tasks.rados.rados.0.smithi136.stdout:2211: seq_num 716 ranges {5668864=794624,6463488=655360,7118848=704512,7823360=245760} 2024-09-12T23:46:00.966 INFO:tasks.rados.rados.0.smithi136.stdout:2211: writing smithi13667081-10 from 5668864 to 6463488 tid 1 2024-09-12T23:46:00.971 INFO:tasks.rados.rados.0.smithi136.stdout:2211: writing smithi13667081-10 from 6463488 to 7118848 tid 2 2024-09-12T23:46:00.976 INFO:tasks.rados.rados.0.smithi136.stdout:2211: writing smithi13667081-10 from 7118848 to 7823360 tid 3 2024-09-12T23:46:00.977 INFO:tasks.rados.rados.0.smithi136.stdout:2211: writing smithi13667081-10 from 7823360 to 8069120 tid 4 2024-09-12T23:46:00.978 INFO:tasks.rados.rados.0.smithi136.stdout:2212: snap_remove snap 218 2024-09-12T23:46:01.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:00 smithi181 bash[67587]: cluster 2024-09-12T23:45:59.880287+0000 mon.a (mon.0) 632 : cluster [DBG] osdmap e505: 8 total, 8 up, 8 in 2024-09-12T23:46:01.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:00 smithi181 bash[67587]: cluster 2024-09-12T23:46:00.734369+0000 mgr.y (mgr.24661) 202 : cluster [DBG] pgmap v333: 121 pgs: 121 active+clean; 335 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 3.9 MiB/s wr, 17 op/s 2024-09-12T23:46:01.256 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:00 smithi136 bash[76371]: cluster 2024-09-12T23:45:59.880287+0000 mon.a (mon.0) 632 : cluster [DBG] osdmap e505: 8 total, 8 up, 8 in 2024-09-12T23:46:01.256 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:00 smithi136 bash[76371]: cluster 2024-09-12T23:46:00.734369+0000 mgr.y (mgr.24661) 202 : cluster [DBG] pgmap v333: 121 pgs: 121 active+clean; 335 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 3.9 MiB/s wr, 17 op/s 2024-09-12T23:46:01.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:00 smithi136 bash[75164]: cluster 2024-09-12T23:45:59.880287+0000 mon.a (mon.0) 632 : cluster [DBG] osdmap e505: 8 total, 8 up, 8 in 2024-09-12T23:46:01.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:00 smithi136 bash[75164]: cluster 2024-09-12T23:46:00.734369+0000 mgr.y (mgr.24661) 202 : cluster [DBG] pgmap v333: 121 pgs: 121 active+clean; 335 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 3.9 MiB/s wr, 17 op/s 2024-09-12T23:46:01.358 INFO:tasks.rados.rados.0.smithi136.stdout:2212: done (2 left) 2024-09-12T23:46:01.358 INFO:tasks.rados.rados.0.smithi136.stdout:2213: read oid 36 snap -1 2024-09-12T23:46:01.358 INFO:tasks.rados.rados.0.smithi136.stdout:2213: expect (ObjNum 610 snap 175 seq_num 610) 2024-09-12T23:46:01.358 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 25 current snap is 223 2024-09-12T23:46:01.358 INFO:tasks.rados.rados.0.smithi136.stdout:2210: finishing copy_from racing read to smithi13667081-24 2024-09-12T23:46:01.358 INFO:tasks.rados.rados.0.smithi136.stdout:2210: finishing copy_from to smithi13667081-24 2024-09-12T23:46:01.358 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 24 v 1254 (ObjNum 614 snap 177 seq_num 614) dirty exists 2024-09-12T23:46:01.359 INFO:tasks.rados.rados.0.smithi136.stdout:2211: finishing write tid 1 to smithi13667081-10 2024-09-12T23:46:01.359 INFO:tasks.rados.rados.0.smithi136.stdout:2214: seq_num 717 ranges {1564672=16384} 2024-09-12T23:46:01.369 INFO:tasks.rados.rados.0.smithi136.stdout:2214: writing smithi13667081-25 from 1564672 to 1581056 tid 1 2024-09-12T23:46:01.369 INFO:tasks.rados.rados.0.smithi136.stdout:2210: done (3 left) 2024-09-12T23:46:01.369 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 48 current snap is 223 2024-09-12T23:46:01.369 INFO:tasks.rados.rados.0.smithi136.stdout:2215: seq_num 718 ranges {5668864=32768} 2024-09-12T23:46:01.409 INFO:tasks.rados.rados.0.smithi136.stdout:2215: writing smithi13667081-48 from 5668864 to 5701632 tid 1 2024-09-12T23:46:01.409 INFO:tasks.rados.rados.0.smithi136.stdout:2216: copy_from oid 45 from oid 29 current snap is 223 2024-09-12T23:46:01.409 INFO:tasks.rados.rados.0.smithi136.stdout:2211: finishing write tid 2 to smithi13667081-10 2024-09-12T23:46:01.409 INFO:tasks.rados.rados.0.smithi136.stdout:2211: finishing write tid 3 to smithi13667081-10 2024-09-12T23:46:01.409 INFO:tasks.rados.rados.0.smithi136.stdout:2211: finishing write tid 4 to smithi13667081-10 2024-09-12T23:46:01.409 INFO:tasks.rados.rados.0.smithi136.stdout:2211: finishing write tid 5 to smithi13667081-10 2024-09-12T23:46:01.409 INFO:tasks.rados.rados.0.smithi136.stdout:2211: finishing write tid 6 to smithi13667081-10 2024-09-12T23:46:01.409 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 10 v 1184 (ObjNum 716 snap 223 seq_num 716) dirty exists 2024-09-12T23:46:01.409 INFO:tasks.rados.rados.0.smithi136.stdout:2211: left oid 10 (ObjNum 716 snap 223 seq_num 716) 2024-09-12T23:46:01.409 INFO:tasks.rados.rados.0.smithi136.stdout:2213: expect (ObjNum 610 snap 175 seq_num 610) 2024-09-12T23:46:01.433 INFO:tasks.rados.rados.0.smithi136.stdout:2214: finishing write tid 1 to smithi13667081-25 2024-09-12T23:46:01.433 INFO:tasks.rados.rados.0.smithi136.stdout:2214: finishing write tid 2 to smithi13667081-25 2024-09-12T23:46:01.433 INFO:tasks.rados.rados.0.smithi136.stdout:2214: finishing write tid 3 to smithi13667081-25 2024-09-12T23:46:01.433 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 25 v 850 (ObjNum 717 snap 223 seq_num 717) dirty exists 2024-09-12T23:46:01.433 INFO:tasks.rados.rados.0.smithi136.stdout:2214: left oid 25 (ObjNum 717 snap 223 seq_num 717) 2024-09-12T23:46:01.433 INFO:tasks.rados.rados.0.smithi136.stdout:2215: finishing write tid 1 to smithi13667081-48 2024-09-12T23:46:01.433 INFO:tasks.rados.rados.0.smithi136.stdout:2215: finishing write tid 2 to smithi13667081-48 2024-09-12T23:46:01.433 INFO:tasks.rados.rados.0.smithi136.stdout:2215: finishing write tid 3 to smithi13667081-48 2024-09-12T23:46:01.433 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 48 v 1316 (ObjNum 718 snap 223 seq_num 718) dirty exists 2024-09-12T23:46:01.433 INFO:tasks.rados.rados.0.smithi136.stdout:2215: left oid 48 (ObjNum 718 snap 223 seq_num 718) 2024-09-12T23:46:01.433 INFO:tasks.rados.rados.0.smithi136.stdout:2211: done (4 left) 2024-09-12T23:46:01.433 INFO:tasks.rados.rados.0.smithi136.stdout:2213: done (3 left) 2024-09-12T23:46:01.434 INFO:tasks.rados.rados.0.smithi136.stdout:2214: done (2 left) 2024-09-12T23:46:01.434 INFO:tasks.rados.rados.0.smithi136.stdout:2215: done (1 left) 2024-09-12T23:46:01.434 INFO:tasks.rados.rados.0.smithi136.stdout:2217: snap_remove snap 207 2024-09-12T23:46:01.594 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:46:01 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:46:01.594 INFO:journalctl@ceph.mds.b.smithi181.stdout:Sep 12 23:46:01 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:46:01.594 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:46:01 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:46:01.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:01 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:46:01.595 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 12 23:46:01 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:46:01.595 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:46:01 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:46:01.596 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:46:01 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:46:01.596 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:46:01 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:46:01.596 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:46:01 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:46:01.597 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 12 23:46:01 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:46:01.881 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:46:01 smithi181 systemd[1]: Stopping Ceph osd.4 for da58a47c-715d-11ef-bceb-c7b262605968... 2024-09-12T23:46:01.881 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:46:01 smithi181 bash[28243]: debug 2024-09-12T23:46:01.700+0000 7f6d2578f640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-12T23:46:01.881 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:46:01 smithi181 bash[28243]: debug 2024-09-12T23:46:01.700+0000 7f6d2578f640 -1 osd.4 507 *** Got signal Terminated *** 2024-09-12T23:46:01.881 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:46:01 smithi181 bash[28243]: debug 2024-09-12T23:46:01.700+0000 7f6d2578f640 -1 osd.4 507 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-12T23:46:02.256 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:01 smithi136 bash[76371]: cluster 2024-09-12T23:46:00.890434+0000 mon.a (mon.0) 633 : cluster [DBG] osdmap e506: 8 total, 8 up, 8 in 2024-09-12T23:46:02.256 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:01 smithi136 bash[76371]: cluster 2024-09-12T23:46:01.360495+0000 mon.a (mon.0) 634 : cluster [DBG] osdmap e507: 8 total, 8 up, 8 in 2024-09-12T23:46:02.256 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:01 smithi136 bash[76371]: cluster 2024-09-12T23:46:01.705405+0000 mon.a (mon.0) 635 : cluster [INF] osd.4 marked itself down and dead 2024-09-12T23:46:02.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:01 smithi136 bash[75164]: cluster 2024-09-12T23:46:00.890434+0000 mon.a (mon.0) 633 : cluster [DBG] osdmap e506: 8 total, 8 up, 8 in 2024-09-12T23:46:02.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:01 smithi136 bash[75164]: cluster 2024-09-12T23:46:01.360495+0000 mon.a (mon.0) 634 : cluster [DBG] osdmap e507: 8 total, 8 up, 8 in 2024-09-12T23:46:02.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:01 smithi136 bash[75164]: cluster 2024-09-12T23:46:01.705405+0000 mon.a (mon.0) 635 : cluster [INF] osd.4 marked itself down and dead 2024-09-12T23:46:02.309 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:01 smithi181 bash[67587]: cluster 2024-09-12T23:46:00.890434+0000 mon.a (mon.0) 633 : cluster [DBG] osdmap e506: 8 total, 8 up, 8 in 2024-09-12T23:46:02.309 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:01 smithi181 bash[67587]: cluster 2024-09-12T23:46:01.360495+0000 mon.a (mon.0) 634 : cluster [DBG] osdmap e507: 8 total, 8 up, 8 in 2024-09-12T23:46:02.309 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:01 smithi181 bash[67587]: cluster 2024-09-12T23:46:01.705405+0000 mon.a (mon.0) 635 : cluster [INF] osd.4 marked itself down and dead 2024-09-12T23:46:02.365 INFO:tasks.rados.rados.0.smithi136.stdout:2216: finishing copy_from racing read to smithi13667081-45 2024-09-12T23:46:02.365 INFO:tasks.rados.rados.0.smithi136.stdout:2216: finishing copy_from to smithi13667081-45 2024-09-12T23:46:02.365 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 45 v 1228 (ObjNum 666 snap 195 seq_num 666) dirty exists 2024-09-12T23:46:02.365 INFO:tasks.rados.rados.0.smithi136.stdout:2216: done (1 left) 2024-09-12T23:46:02.365 INFO:tasks.rados.rados.0.smithi136.stdout:2217: done (0 left) 2024-09-12T23:46:02.366 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 33 current snap is 223 2024-09-12T23:46:02.366 INFO:tasks.rados.rados.0.smithi136.stdout:2218: seq_num 719 ranges {0=49152} 2024-09-12T23:46:02.366 INFO:tasks.rados.rados.0.smithi136.stdout:2218: writing smithi13667081-33 from 0 to 49152 tid 1 2024-09-12T23:46:02.366 INFO:tasks.rados.rados.0.smithi136.stdout:2219: snap_remove snap 204 2024-09-12T23:46:02.594 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:46:02 smithi181 bash[60471]: ts=2024-09-12T23:46:02.313Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:46:03.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:02 smithi181 bash[67587]: cluster 2024-09-12T23:46:02.355346+0000 mon.a (mon.0) 636 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-12T23:46:03.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:02 smithi181 bash[67587]: cluster 2024-09-12T23:46:02.364897+0000 mon.a (mon.0) 637 : cluster [DBG] osdmap e508: 8 total, 7 up, 8 in 2024-09-12T23:46:03.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:02 smithi181 bash[67587]: cluster 2024-09-12T23:46:02.735061+0000 mgr.y (mgr.24661) 203 : cluster [DBG] pgmap v337: 121 pgs: 8 stale+active+clean, 113 active+clean; 348 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 4.9 MiB/s wr, 20 op/s 2024-09-12T23:46:03.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:02 smithi136 bash[75164]: cluster 2024-09-12T23:46:02.355346+0000 mon.a (mon.0) 636 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-12T23:46:03.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:02 smithi136 bash[75164]: cluster 2024-09-12T23:46:02.364897+0000 mon.a (mon.0) 637 : cluster [DBG] osdmap e508: 8 total, 7 up, 8 in 2024-09-12T23:46:03.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:02 smithi136 bash[75164]: cluster 2024-09-12T23:46:02.735061+0000 mgr.y (mgr.24661) 203 : cluster [DBG] pgmap v337: 121 pgs: 8 stale+active+clean, 113 active+clean; 348 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 4.9 MiB/s wr, 20 op/s 2024-09-12T23:46:03.256 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:02 smithi136 bash[76371]: cluster 2024-09-12T23:46:02.355346+0000 mon.a (mon.0) 636 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-12T23:46:03.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:02 smithi136 bash[76371]: cluster 2024-09-12T23:46:02.364897+0000 mon.a (mon.0) 637 : cluster [DBG] osdmap e508: 8 total, 7 up, 8 in 2024-09-12T23:46:03.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:02 smithi136 bash[76371]: cluster 2024-09-12T23:46:02.735061+0000 mgr.y (mgr.24661) 203 : cluster [DBG] pgmap v337: 121 pgs: 8 stale+active+clean, 113 active+clean; 348 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 4.9 MiB/s wr, 20 op/s 2024-09-12T23:46:03.369 INFO:tasks.rados.rados.0.smithi136.stdout:2219: done (1 left) 2024-09-12T23:46:03.369 INFO:tasks.rados.rados.0.smithi136.stdout:2220: read oid 16 snap -1 2024-09-12T23:46:03.369 INFO:tasks.rados.rados.0.smithi136.stdout:2220: expect (ObjNum 616 snap 179 seq_num 616) 2024-09-12T23:46:03.369 INFO:tasks.rados.rados.0.smithi136.stdout:2221: rollback oid 10 current snap is 223 2024-09-12T23:46:03.369 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 10 to 222 2024-09-12T23:46:03.369 INFO:tasks.rados.rados.0.smithi136.stdout:2218: finishing write tid 1 to smithi13667081-33 2024-09-12T23:46:03.369 INFO:tasks.rados.rados.0.smithi136.stdout:2218: finishing write tid 2 to smithi13667081-33 2024-09-12T23:46:03.369 INFO:tasks.rados.rados.0.smithi136.stdout:2218: finishing write tid 3 to smithi13667081-33 2024-09-12T23:46:03.370 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 33 v 1247 (ObjNum 719 snap 223 seq_num 719) dirty exists 2024-09-12T23:46:03.370 INFO:tasks.rados.rados.0.smithi136.stdout:2218: left oid 33 (ObjNum 719 snap 223 seq_num 719) 2024-09-12T23:46:03.370 INFO:tasks.rados.rados.0.smithi136.stdout:2218: done (2 left) 2024-09-12T23:46:03.370 INFO:tasks.rados.rados.0.smithi136.stdout:2222: snap_remove snap 201 2024-09-12T23:46:03.797 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:46:03 smithi181 bash[71752]: ceph-da58a47c-715d-11ef-bceb-c7b262605968-osd-4 2024-09-12T23:46:03.797 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:46:03 smithi181 bash[71819]: Error response from daemon: No such container: ceph-da58a47c-715d-11ef-bceb-c7b262605968-osd-4 2024-09-12T23:46:04.374 INFO:tasks.rados.rados.0.smithi136.stdout:2222: done (2 left) 2024-09-12T23:46:04.374 INFO:tasks.rados.rados.0.smithi136.stdout:2223: snap_create 2024-09-12T23:46:04.374 INFO:tasks.rados.rados.0.smithi136.stdout:2221: finishing rollback tid 0 to smithi13667081-10 2024-09-12T23:46:04.374 INFO:tasks.rados.rados.0.smithi136.stdout:2221: finishing rollback tid 1 to smithi13667081-10 2024-09-12T23:46:04.374 INFO:tasks.rados.rados.0.smithi136.stdout:2221: finishing rollback tid 2 to smithi13667081-10 2024-09-12T23:46:04.374 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 10 v 1187 (ObjNum 669 snap 196 seq_num 669) dirty exists 2024-09-12T23:46:04.375 INFO:tasks.rados.rados.0.smithi136.stdout:2220: expect (ObjNum 616 snap 179 seq_num 616) 2024-09-12T23:46:04.696 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:04 smithi181 bash[67587]: cluster 2024-09-12T23:46:03.372418+0000 mon.a (mon.0) 638 : cluster [DBG] osdmap e509: 8 total, 7 up, 8 in 2024-09-12T23:46:04.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:04 smithi136 bash[75164]: cluster 2024-09-12T23:46:03.372418+0000 mon.a (mon.0) 638 : cluster [DBG] osdmap e509: 8 total, 7 up, 8 in 2024-09-12T23:46:04.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:04 smithi136 bash[76371]: cluster 2024-09-12T23:46:03.372418+0000 mon.a (mon.0) 638 : cluster [DBG] osdmap e509: 8 total, 7 up, 8 in 2024-09-12T23:46:05.382 INFO:tasks.rados.rados.0.smithi136.stdout:2220: done (2 left) 2024-09-12T23:46:05.382 INFO:tasks.rados.rados.0.smithi136.stdout:2221: done (1 left) 2024-09-12T23:46:05.382 INFO:tasks.rados.rados.0.smithi136.stdout:2223: done (0 left) 2024-09-12T23:46:05.382 INFO:tasks.rados.rados.0.smithi136.stdout:2224: read oid 42 snap 223 2024-09-12T23:46:05.382 INFO:tasks.rados.rados.0.smithi136.stdout:2224: expect (ObjNum 672 snap 197 seq_num 672) 2024-09-12T23:46:05.382 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 12 current snap is 224 2024-09-12T23:46:05.382 INFO:tasks.rados.rados.0.smithi136.stdout:2225: seq_num 720 ranges {0=65536} 2024-09-12T23:46:05.383 INFO:tasks.rados.rados.0.smithi136.stdout:2225: writing smithi13667081-12 from 0 to 65536 tid 1 2024-09-12T23:46:05.383 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 22 current snap is 224 2024-09-12T23:46:05.383 INFO:tasks.rados.rados.0.smithi136.stdout:2226: seq_num 721 ranges {7659520=81920} 2024-09-12T23:46:05.435 INFO:tasks.rados.rados.0.smithi136.stdout:2226: writing smithi13667081-22 from 7659520 to 7741440 tid 1 2024-09-12T23:46:05.436 INFO:tasks.rados.rados.0.smithi136.stdout:2227: rollback oid 35 current snap is 224 2024-09-12T23:46:05.436 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 35 to 220 2024-09-12T23:46:05.436 INFO:tasks.rados.rados.0.smithi136.stdout:2225: finishing write tid 1 to smithi13667081-12 2024-09-12T23:46:05.436 INFO:tasks.rados.rados.0.smithi136.stdout:2225: finishing write tid 2 to smithi13667081-12 2024-09-12T23:46:05.436 INFO:tasks.rados.rados.0.smithi136.stdout:2228: rollback oid 7 current snap is 224 2024-09-12T23:46:05.436 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 7 to 219 2024-09-12T23:46:05.436 INFO:tasks.rados.rados.0.smithi136.stdout:2225: finishing write tid 3 to smithi13667081-12 2024-09-12T23:46:05.436 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 12 v 1458 (ObjNum 720 snap 224 seq_num 720) dirty exists 2024-09-12T23:46:05.436 INFO:tasks.rados.rados.0.smithi136.stdout:2225: left oid 12 (ObjNum 720 snap 224 seq_num 720) 2024-09-12T23:46:05.436 INFO:tasks.rados.rados.0.smithi136.stdout:2224: expect (ObjNum 672 snap 197 seq_num 672) 2024-09-12T23:46:05.647 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:05 smithi181 bash[67587]: cluster 2024-09-12T23:46:04.376640+0000 mon.a (mon.0) 639 : cluster [DBG] osdmap e510: 8 total, 7 up, 8 in 2024-09-12T23:46:05.647 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:05 smithi181 bash[67587]: cluster 2024-09-12T23:46:04.736276+0000 mgr.y (mgr.24661) 204 : cluster [DBG] pgmap v340: 121 pgs: 1 active+clean+snaptrim, 20 active+undersized, 7 activating+undersized, 2 stale+active+clean, 4 active+undersized+degraded, 87 active+clean; 346 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 6.0 MiB/s wr, 18 op/s; 44/537 objects degraded (8.194%) 2024-09-12T23:46:05.679 INFO:tasks.rados.rados.0.smithi136.stdout:2226: finishing write tid 1 to smithi13667081-22 2024-09-12T23:46:05.680 INFO:tasks.rados.rados.0.smithi136.stdout:2227: finishing rollback tid 0 to smithi13667081-35 2024-09-12T23:46:05.680 INFO:tasks.rados.rados.0.smithi136.stdout:2226: finishing write tid 2 to smithi13667081-22 2024-09-12T23:46:05.680 INFO:tasks.rados.rados.0.smithi136.stdout:2227: finishing rollback tid 1 to smithi13667081-35 2024-09-12T23:46:05.680 INFO:tasks.rados.rados.0.smithi136.stdout:2227: finishing rollback tid 2 to smithi13667081-35 2024-09-12T23:46:05.680 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 35 v 1577 (ObjNum 671 snap 196 seq_num 671) dirty exists 2024-09-12T23:46:05.680 INFO:tasks.rados.rados.0.smithi136.stdout:2226: finishing write tid 3 to smithi13667081-22 2024-09-12T23:46:05.680 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 22 v 853 (ObjNum 721 snap 224 seq_num 721) dirty exists 2024-09-12T23:46:05.680 INFO:tasks.rados.rados.0.smithi136.stdout:2226: left oid 22 (ObjNum 721 snap 224 seq_num 721) 2024-09-12T23:46:05.680 INFO:tasks.rados.rados.0.smithi136.stdout:2228: finishing rollback tid 0 to smithi13667081-7 2024-09-12T23:46:05.680 INFO:tasks.rados.rados.0.smithi136.stdout:2228: finishing rollback tid 1 to smithi13667081-7 2024-09-12T23:46:05.680 INFO:tasks.rados.rados.0.smithi136.stdout:2228: finishing rollback tid 2 to smithi13667081-7 2024-09-12T23:46:05.680 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 7 v 939 (ObjNum 691 snap 209 seq_num 691) dirty exists 2024-09-12T23:46:05.681 INFO:tasks.rados.rados.0.smithi136.stdout:2224: done (4 left) 2024-09-12T23:46:05.681 INFO:tasks.rados.rados.0.smithi136.stdout:2225: done (3 left) 2024-09-12T23:46:05.681 INFO:tasks.rados.rados.0.smithi136.stdout:2226: done (2 left) 2024-09-12T23:46:05.681 INFO:tasks.rados.rados.0.smithi136.stdout:2227: done (1 left) 2024-09-12T23:46:05.681 INFO:tasks.rados.rados.0.smithi136.stdout:2228: done (0 left) 2024-09-12T23:46:05.681 INFO:tasks.rados.rados.0.smithi136.stdout:2229: read oid 36 snap -1 2024-09-12T23:46:05.681 INFO:tasks.rados.rados.0.smithi136.stdout:2229: expect (ObjNum 610 snap 175 seq_num 610) 2024-09-12T23:46:05.681 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 41 current snap is 224 2024-09-12T23:46:05.681 INFO:tasks.rados.rados.0.smithi136.stdout:2230: seq_num 722 ranges {2285568=98304} 2024-09-12T23:46:05.697 INFO:tasks.rados.rados.0.smithi136.stdout:2230: writing smithi13667081-41 from 2285568 to 2383872 tid 1 2024-09-12T23:46:05.697 INFO:tasks.rados.rados.0.smithi136.stdout:2231: read oid 34 snap -1 2024-09-12T23:46:05.697 INFO:tasks.rados.rados.0.smithi136.stdout:2231: expect (ObjNum 607 snap 175 seq_num 607) 2024-09-12T23:46:05.697 INFO:tasks.rados.rados.0.smithi136.stdout:2232: read oid 15 snap -1 2024-09-12T23:46:05.698 INFO:tasks.rados.rados.0.smithi136.stdout:2232: expect deleted 2024-09-12T23:46:05.698 INFO:tasks.rados.rados.0.smithi136.stdout:2233: read oid 25 snap -1 2024-09-12T23:46:05.698 INFO:tasks.rados.rados.0.smithi136.stdout:2233: expect (ObjNum 717 snap 223 seq_num 717) 2024-09-12T23:46:05.698 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 14 current snap is 224 2024-09-12T23:46:05.698 INFO:tasks.rados.rados.0.smithi136.stdout:2234: seq_num 723 ranges {8282112=114688} 2024-09-12T23:46:05.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:05 smithi136 bash[75164]: cluster 2024-09-12T23:46:04.376640+0000 mon.a (mon.0) 639 : cluster [DBG] osdmap e510: 8 total, 7 up, 8 in 2024-09-12T23:46:05.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:05 smithi136 bash[75164]: cluster 2024-09-12T23:46:04.736276+0000 mgr.y (mgr.24661) 204 : cluster [DBG] pgmap v340: 121 pgs: 1 active+clean+snaptrim, 20 active+undersized, 7 activating+undersized, 2 stale+active+clean, 4 active+undersized+degraded, 87 active+clean; 346 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 6.0 MiB/s wr, 18 op/s; 44/537 objects degraded (8.194%) 2024-09-12T23:46:05.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:05 smithi136 bash[76371]: cluster 2024-09-12T23:46:04.376640+0000 mon.a (mon.0) 639 : cluster [DBG] osdmap e510: 8 total, 7 up, 8 in 2024-09-12T23:46:05.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:05 smithi136 bash[76371]: cluster 2024-09-12T23:46:04.736276+0000 mgr.y (mgr.24661) 204 : cluster [DBG] pgmap v340: 121 pgs: 1 active+clean+snaptrim, 20 active+undersized, 7 activating+undersized, 2 stale+active+clean, 4 active+undersized+degraded, 87 active+clean; 346 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 6.0 MiB/s wr, 18 op/s; 44/537 objects degraded (8.194%) 2024-09-12T23:46:05.757 INFO:tasks.rados.rados.0.smithi136.stdout:2234: writing smithi13667081-14 from 8282112 to 8396800 tid 1 2024-09-12T23:46:05.757 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 18 current snap is 224 2024-09-12T23:46:05.757 INFO:tasks.rados.rados.0.smithi136.stdout:2235: seq_num 724 ranges {0=131072} 2024-09-12T23:46:05.757 INFO:tasks.rados.rados.0.smithi136.stdout:2235: writing smithi13667081-18 from 0 to 131072 tid 1 2024-09-12T23:46:05.757 INFO:tasks.rados.rados.0.smithi136.stdout:2236: read oid 13 snap 210 2024-09-12T23:46:05.757 INFO:tasks.rados.rados.0.smithi136.stdout:2236: expect deleted 2024-09-12T23:46:05.758 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 19 current snap is 224 2024-09-12T23:46:05.758 INFO:tasks.rados.rados.0.smithi136.stdout:2229: expect (ObjNum 610 snap 175 seq_num 610) 2024-09-12T23:46:05.796 INFO:tasks.rados.rados.0.smithi136.stdout:2233: expect (ObjNum 717 snap 223 seq_num 717) 2024-09-12T23:46:05.874 INFO:tasks.rados.rados.0.smithi136.stdout:2230: finishing write tid 1 to smithi13667081-41 2024-09-12T23:46:05.874 INFO:tasks.rados.rados.0.smithi136.stdout:2230: finishing write tid 2 to smithi13667081-41 2024-09-12T23:46:05.874 INFO:tasks.rados.rados.0.smithi136.stdout:2231: expect (ObjNum 607 snap 175 seq_num 607) 2024-09-12T23:46:05.949 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:05 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:46:05.950 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:46:05 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:46:05.950 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 12 23:46:05 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:46:05.950 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:46:05 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:46:05.950 INFO:journalctl@ceph.mds.b.smithi181.stdout:Sep 12 23:46:05 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:46:05.951 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:46:05 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:46:05.951 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 12 23:46:05 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:46:05.951 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:46:05 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:46:05.952 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:46:05 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:46:05.952 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:46:05 smithi181 systemd[1]: ceph-da58a47c-715d-11ef-bceb-c7b262605968@osd.4.service: Deactivated successfully. 2024-09-12T23:46:05.952 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:46:05 smithi181 systemd[1]: Stopped Ceph osd.4 for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:46:05.952 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:46:05 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:46:05.952 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:46:05 smithi181 systemd[1]: Started Ceph osd.4 for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:46:06.074 INFO:tasks.rados.rados.0.smithi136.stdout:2230: finishing write tid 3 to smithi13667081-41 2024-09-12T23:46:06.074 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 41 v 1249 (ObjNum 722 snap 224 seq_num 722) dirty exists 2024-09-12T23:46:06.074 INFO:tasks.rados.rados.0.smithi136.stdout:2230: left oid 41 (ObjNum 722 snap 224 seq_num 722) 2024-09-12T23:46:06.074 INFO:tasks.rados.rados.0.smithi136.stdout:2235: finishing write tid 1 to smithi13667081-18 2024-09-12T23:46:06.074 INFO:tasks.rados.rados.0.smithi136.stdout:2235: finishing write tid 2 to smithi13667081-18 2024-09-12T23:46:06.074 INFO:tasks.rados.rados.0.smithi136.stdout:2234: finishing write tid 1 to smithi13667081-14 2024-09-12T23:46:06.075 INFO:tasks.rados.rados.0.smithi136.stdout:2235: finishing write tid 3 to smithi13667081-18 2024-09-12T23:46:06.075 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 18 v 941 (ObjNum 724 snap 224 seq_num 724) dirty exists 2024-09-12T23:46:06.075 INFO:tasks.rados.rados.0.smithi136.stdout:2235: left oid 18 (ObjNum 724 snap 224 seq_num 724) 2024-09-12T23:46:06.075 INFO:tasks.rados.rados.0.smithi136.stdout:2234: finishing write tid 2 to smithi13667081-14 2024-09-12T23:46:06.075 INFO:tasks.rados.rados.0.smithi136.stdout:2234: finishing write tid 3 to smithi13667081-14 2024-09-12T23:46:06.075 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 14 v 1319 (ObjNum 723 snap 224 seq_num 723) dirty exists 2024-09-12T23:46:06.075 INFO:tasks.rados.rados.0.smithi136.stdout:2234: left oid 14 (ObjNum 723 snap 224 seq_num 723) 2024-09-12T23:46:06.075 INFO:tasks.rados.rados.0.smithi136.stdout:2237: seq_num 725 ranges {0=147456} 2024-09-12T23:46:06.076 INFO:tasks.rados.rados.0.smithi136.stdout:2237: writing smithi13667081-19 from 0 to 147456 tid 1 2024-09-12T23:46:06.076 INFO:tasks.rados.rados.0.smithi136.stdout:2229: done (8 left) 2024-09-12T23:46:06.076 INFO:tasks.rados.rados.0.smithi136.stdout:2230: done (7 left) 2024-09-12T23:46:06.076 INFO:tasks.rados.rados.0.smithi136.stdout:2231: done (6 left) 2024-09-12T23:46:06.076 INFO:tasks.rados.rados.0.smithi136.stdout:2232: done (5 left) 2024-09-12T23:46:06.076 INFO:tasks.rados.rados.0.smithi136.stdout:2233: done (4 left) 2024-09-12T23:46:06.076 INFO:tasks.rados.rados.0.smithi136.stdout:2234: done (3 left) 2024-09-12T23:46:06.076 INFO:tasks.rados.rados.0.smithi136.stdout:2235: done (2 left) 2024-09-12T23:46:06.077 INFO:tasks.rados.rados.0.smithi136.stdout:2236: done (1 left) 2024-09-12T23:46:06.077 INFO:tasks.rados.rados.0.smithi136.stdout:2238: rollback oid 22 current snap is 224 2024-09-12T23:46:06.077 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 22 to 208 2024-09-12T23:46:06.077 INFO:tasks.rados.rados.0.smithi136.stdout:2239: delete oid 14 current snap is 224 2024-09-12T23:46:06.079 INFO:tasks.rados.rados.0.smithi136.stdout:2237: finishing write tid 1 to smithi13667081-19 2024-09-12T23:46:06.079 INFO:tasks.rados.rados.0.smithi136.stdout:2237: finishing write tid 2 to smithi13667081-19 2024-09-12T23:46:06.081 INFO:tasks.rados.rados.0.smithi136.stdout:2237: finishing write tid 3 to smithi13667081-19 2024-09-12T23:46:06.081 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 19 v 709 (ObjNum 725 snap 224 seq_num 725) dirty exists 2024-09-12T23:46:06.082 INFO:tasks.rados.rados.0.smithi136.stdout:2237: left oid 19 (ObjNum 725 snap 224 seq_num 725) 2024-09-12T23:46:06.082 INFO:tasks.rados.rados.0.smithi136.stdout:2238: finishing rollback tid 0 to smithi13667081-22 2024-09-12T23:46:06.082 INFO:tasks.rados.rados.0.smithi136.stdout:2237: done (2 left) 2024-09-12T23:46:06.082 INFO:tasks.rados.rados.0.smithi136.stdout:2239: done (1 left) 2024-09-12T23:46:06.082 INFO:tasks.rados.rados.0.smithi136.stdout:2240: setattr oid 34 current snap is 224 2024-09-12T23:46:06.083 INFO:tasks.rados.rados.0.smithi136.stdout:2238: finishing rollback tid 1 to smithi13667081-22 2024-09-12T23:46:06.083 INFO:tasks.rados.rados.0.smithi136.stdout:2238: finishing rollback tid 2 to smithi13667081-22 2024-09-12T23:46:06.083 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 22 v 856 (ObjNum 687 snap 207 seq_num 687) dirty exists 2024-09-12T23:46:06.083 INFO:tasks.rados.rados.0.smithi136.stdout:2238: done (1 left) 2024-09-12T23:46:06.083 INFO:tasks.rados.rados.0.smithi136.stdout:2241: rollback oid 29 current snap is 224 2024-09-12T23:46:06.083 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 29 to 222 2024-09-12T23:46:06.083 INFO:tasks.rados.rados.0.smithi136.stdout:2242: rollback oid 20 current snap is 224 2024-09-12T23:46:06.083 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 20 to 194 2024-09-12T23:46:06.083 INFO:tasks.rados.rados.0.smithi136.stdout:2243: read oid 43 snap 210 2024-09-12T23:46:06.084 INFO:tasks.rados.rados.0.smithi136.stdout:2243: expect (ObjNum 689 snap 208 seq_num 689) 2024-09-12T23:46:06.084 INFO:tasks.rados.rados.0.smithi136.stdout:2244: read oid 3 snap 213 2024-09-12T23:46:06.084 INFO:tasks.rados.rados.0.smithi136.stdout:2244: expect (ObjNum 633 snap 186 seq_num 633) 2024-09-12T23:46:06.084 INFO:tasks.rados.rados.0.smithi136.stdout:2245: snap_remove snap 223 2024-09-12T23:46:06.358 INFO:tasks.rados.rados.0.smithi136.stdout:2245: done (5 left) 2024-09-12T23:46:06.358 INFO:tasks.rados.rados.0.smithi136.stdout:2246: snap_remove snap 208 2024-09-12T23:46:06.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:06 smithi136 bash[75164]: cluster 2024-09-12T23:46:05.384297+0000 mon.a (mon.0) 640 : cluster [DBG] osdmap e511: 8 total, 7 up, 8 in 2024-09-12T23:46:06.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:06 smithi136 bash[75164]: cluster 2024-09-12T23:46:05.384647+0000 mon.a (mon.0) 641 : cluster [WRN] Health check failed: Degraded data redundancy: 44/537 objects degraded (8.194%), 4 pgs degraded (PG_DEGRADED) 2024-09-12T23:46:06.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:06 smithi136 bash[75164]: audit 2024-09-12T23:46:05.993506+0000 mon.a (mon.0) 642 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:06.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:06 smithi136 bash[75164]: audit 2024-09-12T23:46:06.005112+0000 mon.a (mon.0) 643 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:06.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:06 smithi136 bash[75164]: audit 2024-09-12T23:46:06.007106+0000 mon.a (mon.0) 644 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:46:06.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:06 smithi136 bash[75164]: cluster 2024-09-12T23:46:06.361661+0000 mon.a (mon.0) 645 : cluster [DBG] osdmap e512: 8 total, 7 up, 8 in 2024-09-12T23:46:06.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:06 smithi136 bash[76371]: cluster 2024-09-12T23:46:05.384297+0000 mon.a (mon.0) 640 : cluster [DBG] osdmap e511: 8 total, 7 up, 8 in 2024-09-12T23:46:06.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:06 smithi136 bash[76371]: cluster 2024-09-12T23:46:05.384647+0000 mon.a (mon.0) 641 : cluster [WRN] Health check failed: Degraded data redundancy: 44/537 objects degraded (8.194%), 4 pgs degraded (PG_DEGRADED) 2024-09-12T23:46:06.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:06 smithi136 bash[76371]: audit 2024-09-12T23:46:05.993506+0000 mon.a (mon.0) 642 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:06.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:06 smithi136 bash[76371]: audit 2024-09-12T23:46:06.005112+0000 mon.a (mon.0) 643 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:06.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:06 smithi136 bash[76371]: audit 2024-09-12T23:46:06.007106+0000 mon.a (mon.0) 644 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:46:06.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:06 smithi136 bash[76371]: cluster 2024-09-12T23:46:06.361661+0000 mon.a (mon.0) 645 : cluster [DBG] osdmap e512: 8 total, 7 up, 8 in 2024-09-12T23:46:06.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:06 smithi181 bash[67587]: cluster 2024-09-12T23:46:05.384297+0000 mon.a (mon.0) 640 : cluster [DBG] osdmap e511: 8 total, 7 up, 8 in 2024-09-12T23:46:06.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:06 smithi181 bash[67587]: cluster 2024-09-12T23:46:05.384647+0000 mon.a (mon.0) 641 : cluster [WRN] Health check failed: Degraded data redundancy: 44/537 objects degraded (8.194%), 4 pgs degraded (PG_DEGRADED) 2024-09-12T23:46:06.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:06 smithi181 bash[67587]: audit 2024-09-12T23:46:05.993506+0000 mon.a (mon.0) 642 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:06.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:06 smithi181 bash[67587]: audit 2024-09-12T23:46:06.005112+0000 mon.a (mon.0) 643 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:06.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:06 smithi181 bash[67587]: audit 2024-09-12T23:46:06.007106+0000 mon.a (mon.0) 644 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:46:06.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:06 smithi181 bash[67587]: cluster 2024-09-12T23:46:06.361661+0000 mon.a (mon.0) 645 : cluster [DBG] osdmap e512: 8 total, 7 up, 8 in 2024-09-12T23:46:07.361 INFO:tasks.rados.rados.0.smithi136.stdout:2246: done (5 left) 2024-09-12T23:46:07.361 INFO:tasks.rados.rados.0.smithi136.stdout:2247: rollback oid 1 current snap is 224 2024-09-12T23:46:07.362 INFO:tasks.rados.rados.0.smithi136.stdout:2242: finishing rollback tid 1 to smithi13667081-20 2024-09-12T23:46:07.362 INFO:tasks.rados.rados.0.smithi136.stdout:2242: finishing rollback tid 2 to smithi13667081-20 2024-09-12T23:46:07.362 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 20 v 1222 (ObjNum 469 snap 139 seq_num 469) dirty exists 2024-09-12T23:46:07.362 INFO:tasks.rados.rados.0.smithi136.stdout:2241: finishing rollback tid 0 to smithi13667081-29 2024-09-12T23:46:07.362 INFO:tasks.rados.rados.0.smithi136.stdout:2241: finishing rollback tid 1 to smithi13667081-29 2024-09-12T23:46:07.362 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 34 v 1250 (ObjNum 607 snap 175 seq_num 607) dirty exists 2024-09-12T23:46:07.362 INFO:tasks.rados.rados.0.smithi136.stdout:2241: finishing rollback tid 2 to smithi13667081-29 2024-09-12T23:46:07.362 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 29 v 1231 (ObjNum 666 snap 195 seq_num 666) dirty exists 2024-09-12T23:46:07.362 INFO:tasks.rados.rados.0.smithi136.stdout:2244: expect (ObjNum 633 snap 186 seq_num 633) 2024-09-12T23:46:07.391 INFO:tasks.rados.rados.0.smithi136.stdout:2243: expect (ObjNum 689 snap 208 seq_num 689) 2024-09-12T23:46:07.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:07 smithi181 bash[67587]: cluster 2024-09-12T23:46:06.737412+0000 mgr.y (mgr.24661) 205 : cluster [DBG] pgmap v343: 121 pgs: 1 active+clean+snaptrim, 26 active+undersized, 7 activating+undersized, 12 active+undersized+degraded, 75 active+clean; 345 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 26 MiB/s rd, 2.1 MiB/s wr, 18 op/s; 97/543 objects degraded (17.864%) 2024-09-12T23:46:07.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:07 smithi181 bash[67587]: cluster 2024-09-12T23:46:07.364558+0000 mon.a (mon.0) 646 : cluster [DBG] osdmap e513: 8 total, 7 up, 8 in 2024-09-12T23:46:07.695 INFO:teuthology.orchestra.run.smithi136.stdout:true 2024-09-12T23:46:07.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:07 smithi136 bash[75164]: cluster 2024-09-12T23:46:06.737412+0000 mgr.y (mgr.24661) 205 : cluster [DBG] pgmap v343: 121 pgs: 1 active+clean+snaptrim, 26 active+undersized, 7 activating+undersized, 12 active+undersized+degraded, 75 active+clean; 345 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 26 MiB/s rd, 2.1 MiB/s wr, 18 op/s; 97/543 objects degraded (17.864%) 2024-09-12T23:46:07.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:07 smithi136 bash[75164]: cluster 2024-09-12T23:46:07.364558+0000 mon.a (mon.0) 646 : cluster [DBG] osdmap e513: 8 total, 7 up, 8 in 2024-09-12T23:46:07.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:07 smithi136 bash[76371]: cluster 2024-09-12T23:46:06.737412+0000 mgr.y (mgr.24661) 205 : cluster [DBG] pgmap v343: 121 pgs: 1 active+clean+snaptrim, 26 active+undersized, 7 activating+undersized, 12 active+undersized+degraded, 75 active+clean; 345 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 26 MiB/s rd, 2.1 MiB/s wr, 18 op/s; 97/543 objects degraded (17.864%) 2024-09-12T23:46:07.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:07 smithi136 bash[76371]: cluster 2024-09-12T23:46:07.364558+0000 mon.a (mon.0) 646 : cluster [DBG] osdmap e513: 8 total, 7 up, 8 in 2024-09-12T23:46:07.903 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:46:07 smithi181 bash[71970]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-12T23:46:07.903 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:46:07 smithi181 bash[71970]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-12T23:46:07.903 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:46:07 smithi181 bash[71970]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-12T23:46:07.903 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:46:07 smithi181 bash[71970]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-12T23:46:07.903 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:46:07 smithi181 bash[71970]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-12T23:46:07.903 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:46:07 smithi181 bash[71970]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-12T23:46:07.903 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:46:07 smithi181 bash[71970]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-12T23:46:07.999 INFO:teuthology.orchestra.run.smithi136.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-12T23:46:07.999 INFO:teuthology.orchestra.run.smithi136.stdout:alertmanager.a smithi136 *:9093,9094 running (5m) 24s ago 11m 13.1M - 0.25.0 c8568f914cd2 b5197f020eeb 2024-09-12T23:46:07.999 INFO:teuthology.orchestra.run.smithi136.stdout:grafana.a smithi181 *:3000 running (5m) 2m ago 11m 75.3M - 9.4.7 954c08fa6188 256c181be984 2024-09-12T23:46:07.999 INFO:teuthology.orchestra.run.smithi136.stdout:mds.a smithi136 running (12m) 24s ago 12m 15.1M - 17.2.7-1627-gbbdadfa3 214985049bdb 713b9795ad19 2024-09-12T23:46:07.999 INFO:teuthology.orchestra.run.smithi136.stdout:mds.b smithi181 running (12m) 2m ago 12m 17.8M - 17.2.7-1627-gbbdadfa3 214985049bdb ad13179f0868 2024-09-12T23:46:07.999 INFO:teuthology.orchestra.run.smithi136.stdout:mgr.x smithi181 *:8443,9283,8765 running (4m) 2m ago 20m 425M - 18.2.4-837-g7a853b17 f21ba1eea74e cc664dc337f7 2024-09-12T23:46:07.999 INFO:teuthology.orchestra.run.smithi136.stdout:mgr.y smithi136 *:8443,9283,8765 running (5m) 24s ago 22m 490M - 18.2.4-837-g7a853b17 f21ba1eea74e 272f10b42fc3 2024-09-12T23:46:07.999 INFO:teuthology.orchestra.run.smithi136.stdout:mon.a smithi136 running (4m) 24s ago 22m 56.4M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e ad8d21d58ad9 2024-09-12T23:46:07.999 INFO:teuthology.orchestra.run.smithi136.stdout:mon.b smithi181 running (3m) 2m ago 20m 19.5M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e a70ec8315934 2024-09-12T23:46:08.000 INFO:teuthology.orchestra.run.smithi136.stdout:mon.c smithi136 running (3m) 24s ago 20m 47.2M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e 44d1be055a5e 2024-09-12T23:46:08.000 INFO:teuthology.orchestra.run.smithi136.stdout:node-exporter.a smithi136 *:9100 running (5m) 24s ago 11m 8064k - 1.5.0 0da6a335fe13 ccfc37c96620 2024-09-12T23:46:08.000 INFO:teuthology.orchestra.run.smithi136.stdout:node-exporter.b smithi181 *:9100 running (5m) 2m ago 11m 7847k - 1.5.0 0da6a335fe13 80bf0e383230 2024-09-12T23:46:08.000 INFO:teuthology.orchestra.run.smithi136.stdout:osd.0 smithi136 running (115s) 24s ago 19m 212M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 7da21f765721 2024-09-12T23:46:08.000 INFO:teuthology.orchestra.run.smithi136.stdout:osd.1 smithi136 running (87s) 24s ago 18m 247M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 5302be0a016b 2024-09-12T23:46:08.000 INFO:teuthology.orchestra.run.smithi136.stdout:osd.2 smithi136 running (57s) 24s ago 17m 252M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 2e12d1a31223 2024-09-12T23:46:08.000 INFO:teuthology.orchestra.run.smithi136.stdout:osd.3 smithi136 running (28s) 24s ago 16m - 2524M 18.2.4-837-g7a853b17 f21ba1eea74e aa9dc124e59c 2024-09-12T23:46:08.000 INFO:teuthology.orchestra.run.smithi136.stdout:osd.4 smithi181 starting - - - 2524M 2024-09-12T23:46:08.000 INFO:teuthology.orchestra.run.smithi136.stdout:osd.5 smithi181 running (15m) 2m ago 15m 192M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 8f2e8d494fef 2024-09-12T23:46:08.000 INFO:teuthology.orchestra.run.smithi136.stdout:osd.6 smithi181 running (14m) 2m ago 14m 106M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 9c6bec406408 2024-09-12T23:46:08.000 INFO:teuthology.orchestra.run.smithi136.stdout:osd.7 smithi181 running (13m) 2m ago 13m 174M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb bc0632ae2145 2024-09-12T23:46:08.000 INFO:teuthology.orchestra.run.smithi136.stdout:prometheus.a smithi181 *:9095 running (4m) 2m ago 12m 51.7M - 2.43.0 a07b618ecd1d 3853d003a5cc 2024-09-12T23:46:08.326 INFO:teuthology.orchestra.run.smithi136.stdout:{ 2024-09-12T23:46:08.326 INFO:teuthology.orchestra.run.smithi136.stdout: "mon": { 2024-09-12T23:46:08.326 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 3 2024-09-12T23:46:08.326 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:46:08.327 INFO:teuthology.orchestra.run.smithi136.stdout: "mgr": { 2024-09-12T23:46:08.327 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 2 2024-09-12T23:46:08.327 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:46:08.327 INFO:teuthology.orchestra.run.smithi136.stdout: "osd": { 2024-09-12T23:46:08.327 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 3, 2024-09-12T23:46:08.327 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 4 2024-09-12T23:46:08.327 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:46:08.327 INFO:teuthology.orchestra.run.smithi136.stdout: "mds": { 2024-09-12T23:46:08.327 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-12T23:46:08.327 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:46:08.327 INFO:teuthology.orchestra.run.smithi136.stdout: "overall": { 2024-09-12T23:46:08.327 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 5, 2024-09-12T23:46:08.327 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 9 2024-09-12T23:46:08.327 INFO:teuthology.orchestra.run.smithi136.stdout: } 2024-09-12T23:46:08.328 INFO:teuthology.orchestra.run.smithi136.stdout:} 2024-09-12T23:46:08.439 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 1 to 211 2024-09-12T23:46:08.440 INFO:tasks.rados.rados.0.smithi136.stdout:2240: done (5 left) 2024-09-12T23:46:08.440 INFO:tasks.rados.rados.0.smithi136.stdout:2241: done (4 left) 2024-09-12T23:46:08.440 INFO:tasks.rados.rados.0.smithi136.stdout:2242: done (3 left) 2024-09-12T23:46:08.440 INFO:tasks.rados.rados.0.smithi136.stdout:2243: done (2 left) 2024-09-12T23:46:08.443 INFO:tasks.rados.rados.0.smithi136.stdout:2244: done (1 left) 2024-09-12T23:46:08.443 INFO:tasks.rados.rados.0.smithi136.stdout:2248: copy_from oid 7 from oid 25 current snap is 224 2024-09-12T23:46:08.443 INFO:tasks.rados.rados.0.smithi136.stdout:2247: finishing rollback tid 1 to smithi13667081-1 2024-09-12T23:46:08.443 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 1 v 0 (ObjNum 161 snap 0 seq_num 4243757632) dirty dne 2024-09-12T23:46:08.443 INFO:tasks.rados.rados.0.smithi136.stdout:2247: done (1 left) 2024-09-12T23:46:08.443 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 38 current snap is 224 2024-09-12T23:46:08.443 INFO:tasks.rados.rados.0.smithi136.stdout:2249: seq_num 727 ranges {5906432=180224} 2024-09-12T23:46:08.477 INFO:tasks.rados.rados.0.smithi136.stdout:2249: writing smithi13667081-38 from 5906432 to 6086656 tid 1 2024-09-12T23:46:08.477 INFO:tasks.rados.rados.0.smithi136.stdout:2250: snap_remove snap 209 2024-09-12T23:46:08.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:08 smithi136 bash[75164]: audit 2024-09-12T23:46:07.686995+0000 mgr.y (mgr.24661) 206 : audit [DBG] from='client.34314 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:46:08.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:08 smithi136 bash[75164]: audit 2024-09-12T23:46:08.330192+0000 mon.c (mon.2) 12 : audit [DBG] from='client.? 172.21.15.136:0/2117526979' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:46:08.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:08 smithi136 bash[76371]: audit 2024-09-12T23:46:07.686995+0000 mgr.y (mgr.24661) 206 : audit [DBG] from='client.34314 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:46:08.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:08 smithi136 bash[76371]: audit 2024-09-12T23:46:08.330192+0000 mon.c (mon.2) 12 : audit [DBG] from='client.? 172.21.15.136:0/2117526979' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:46:08.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:08 smithi181 bash[67587]: audit 2024-09-12T23:46:07.686995+0000 mgr.y (mgr.24661) 206 : audit [DBG] from='client.34314 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:46:08.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:08 smithi181 bash[67587]: audit 2024-09-12T23:46:08.330192+0000 mon.c (mon.2) 12 : audit [DBG] from='client.? 172.21.15.136:0/2117526979' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:46:09.428 INFO:tasks.rados.rados.0.smithi136.stdout:2250: done (2 left) 2024-09-12T23:46:09.428 INFO:tasks.rados.rados.0.smithi136.stdout:2251: read oid 17 snap -1 2024-09-12T23:46:09.428 INFO:tasks.rados.rados.0.smithi136.stdout:2251: expect deleted 2024-09-12T23:46:09.428 INFO:tasks.rados.rados.0.smithi136.stdout:2252: read oid 42 snap -1 2024-09-12T23:46:09.428 INFO:tasks.rados.rados.0.smithi136.stdout:2252: expect (ObjNum 672 snap 197 seq_num 672) 2024-09-12T23:46:09.428 INFO:tasks.rados.rados.0.smithi136.stdout:2253: delete oid 29 current snap is 224 2024-09-12T23:46:09.428 INFO:tasks.rados.rados.0.smithi136.stdout:2248: finishing copy_from racing read to smithi13667081-7 2024-09-12T23:46:09.428 INFO:tasks.rados.rados.0.smithi136.stdout:2248: finishing copy_from to smithi13667081-7 2024-09-12T23:46:09.428 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 7 v 850 (ObjNum 717 snap 223 seq_num 717) dirty exists 2024-09-12T23:46:09.429 INFO:tasks.rados.rados.0.smithi136.stdout:2249: finishing write tid 1 to smithi13667081-38 2024-09-12T23:46:09.429 INFO:tasks.rados.rados.0.smithi136.stdout:2249: finishing write tid 2 to smithi13667081-38 2024-09-12T23:46:09.429 INFO:tasks.rados.rados.0.smithi136.stdout:2249: finishing write tid 3 to smithi13667081-38 2024-09-12T23:46:09.429 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 38 v 1233 (ObjNum 727 snap 224 seq_num 727) dirty exists 2024-09-12T23:46:09.429 INFO:tasks.rados.rados.0.smithi136.stdout:2249: left oid 38 (ObjNum 727 snap 224 seq_num 727) 2024-09-12T23:46:09.437 INFO:tasks.rados.rados.0.smithi136.stdout:2248: done (4 left) 2024-09-12T23:46:09.437 INFO:tasks.rados.rados.0.smithi136.stdout:2249: done (3 left) 2024-09-12T23:46:09.437 INFO:tasks.rados.rados.0.smithi136.stdout:2251: done (2 left) 2024-09-12T23:46:09.437 INFO:tasks.rados.rados.0.smithi136.stdout:2253: done (1 left) 2024-09-12T23:46:09.437 INFO:tasks.rados.rados.0.smithi136.stdout:2254: copy_from oid 48 from oid 20 current snap is 224 2024-09-12T23:46:09.437 INFO:tasks.rados.rados.0.smithi136.stdout:2255: read oid 4 snap -1 2024-09-12T23:46:09.437 INFO:tasks.rados.rados.0.smithi136.stdout:2255: expect (ObjNum 567 snap 162 seq_num 567) 2024-09-12T23:46:09.437 INFO:tasks.rados.rados.0.smithi136.stdout:2256: delete oid 9 current snap is 224 2024-09-12T23:46:09.450 INFO:tasks.rados.rados.0.smithi136.stdout:2256: done (3 left) 2024-09-12T23:46:09.450 INFO:tasks.rados.rados.0.smithi136.stdout:2257: snap_create 2024-09-12T23:46:09.463 INFO:tasks.rados.rados.0.smithi136.stdout:2252: expect (ObjNum 672 snap 197 seq_num 672) 2024-09-12T23:46:09.730 INFO:tasks.rados.rados.0.smithi136.stdout:2254: finishing copy_from racing read to smithi13667081-48 2024-09-12T23:46:09.730 INFO:tasks.rados.rados.0.smithi136.stdout:2254: finishing copy_from to smithi13667081-48 2024-09-12T23:46:09.730 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 48 v 1222 (ObjNum 469 snap 139 seq_num 469) dirty exists 2024-09-12T23:46:09.730 INFO:tasks.rados.rados.0.smithi136.stdout:2255: expect (ObjNum 567 snap 162 seq_num 567) 2024-09-12T23:46:09.746 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:46:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:46:09] "GET /metrics HTTP/1.1" 200 36387 "" "Prometheus/2.43.0" 2024-09-12T23:46:09.747 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:46:09 smithi181 bash[72192]: debug 2024-09-12T23:46:09.507+0000 7f7a3a70c640 1 -- 172.21.15.181:0/3124739300 <== mon.1 v2:172.21.15.181:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x564749d111e0 con 0x56474aa8e000 2024-09-12T23:46:10.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:09 smithi136 bash[76371]: audit 2024-09-12T23:46:07.987089+0000 mgr.y (mgr.24661) 207 : audit [DBG] from='client.34320 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:46:10.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:09 smithi136 bash[76371]: cluster 2024-09-12T23:46:08.738478+0000 mgr.y (mgr.24661) 208 : cluster [DBG] pgmap v345: 121 pgs: 30 active+undersized, 15 active+undersized+degraded, 76 active+clean; 345 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 1.9 MiB/s wr, 18 op/s; 98/546 objects degraded (17.949%) 2024-09-12T23:46:10.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:09 smithi136 bash[76371]: audit 2024-09-12T23:46:08.752733+0000 mon.a (mon.0) 647 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:10.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:09 smithi136 bash[76371]: audit 2024-09-12T23:46:08.754769+0000 mon.a (mon.0) 648 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:46:10.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:09 smithi136 bash[76371]: cluster 2024-09-12T23:46:09.431054+0000 mon.a (mon.0) 649 : cluster [DBG] osdmap e514: 8 total, 7 up, 8 in 2024-09-12T23:46:10.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:09 smithi136 bash[75164]: audit 2024-09-12T23:46:07.987089+0000 mgr.y (mgr.24661) 207 : audit [DBG] from='client.34320 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:46:10.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:09 smithi136 bash[75164]: cluster 2024-09-12T23:46:08.738478+0000 mgr.y (mgr.24661) 208 : cluster [DBG] pgmap v345: 121 pgs: 30 active+undersized, 15 active+undersized+degraded, 76 active+clean; 345 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 1.9 MiB/s wr, 18 op/s; 98/546 objects degraded (17.949%) 2024-09-12T23:46:10.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:09 smithi136 bash[75164]: audit 2024-09-12T23:46:08.752733+0000 mon.a (mon.0) 647 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:10.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:09 smithi136 bash[75164]: audit 2024-09-12T23:46:08.754769+0000 mon.a (mon.0) 648 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:46:10.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:09 smithi136 bash[75164]: cluster 2024-09-12T23:46:09.431054+0000 mon.a (mon.0) 649 : cluster [DBG] osdmap e514: 8 total, 7 up, 8 in 2024-09-12T23:46:10.044 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:09 smithi181 bash[67587]: audit 2024-09-12T23:46:07.987089+0000 mgr.y (mgr.24661) 207 : audit [DBG] from='client.34320 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:46:10.044 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:09 smithi181 bash[67587]: cluster 2024-09-12T23:46:08.738478+0000 mgr.y (mgr.24661) 208 : cluster [DBG] pgmap v345: 121 pgs: 30 active+undersized, 15 active+undersized+degraded, 76 active+clean; 345 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 1.9 MiB/s wr, 18 op/s; 98/546 objects degraded (17.949%) 2024-09-12T23:46:10.045 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:09 smithi181 bash[67587]: audit 2024-09-12T23:46:08.752733+0000 mon.a (mon.0) 647 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:10.045 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:09 smithi181 bash[67587]: audit 2024-09-12T23:46:08.754769+0000 mon.a (mon.0) 648 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:46:10.045 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:09 smithi181 bash[67587]: cluster 2024-09-12T23:46:09.431054+0000 mon.a (mon.0) 649 : cluster [DBG] osdmap e514: 8 total, 7 up, 8 in 2024-09-12T23:46:10.344 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:46:10 smithi181 bash[72192]: debug 2024-09-12T23:46:10.043+0000 7f7a3cf72640 -1 Falling back to public interface 2024-09-12T23:46:10.427 INFO:tasks.rados.rados.0.smithi136.stdout:2252: done (3 left) 2024-09-12T23:46:10.427 INFO:tasks.rados.rados.0.smithi136.stdout:2254: done (2 left) 2024-09-12T23:46:10.427 INFO:tasks.rados.rados.0.smithi136.stdout:2255: done (1 left) 2024-09-12T23:46:10.427 INFO:tasks.rados.rados.0.smithi136.stdout:2257: done (0 left) 2024-09-12T23:46:10.427 INFO:tasks.rados.rados.0.smithi136.stdout:2258: copy_from oid 23 from oid 20 current snap is 225 2024-09-12T23:46:10.427 INFO:tasks.rados.rados.0.smithi136.stdout:2259: rmattr oid 4 current snap is 225 2024-09-12T23:46:10.427 INFO:tasks.rados.rados.0.smithi136.stdout:2259: done (1 left) 2024-09-12T23:46:10.427 INFO:tasks.rados.rados.0.smithi136.stdout:2260: snap_create 2024-09-12T23:46:10.459 INFO:tasks.rados.rados.0.smithi136.stdout:2258: finishing copy_from racing read to smithi13667081-23 2024-09-12T23:46:10.460 INFO:tasks.rados.rados.0.smithi136.stdout:2258: finishing copy_from to smithi13667081-23 2024-09-12T23:46:10.460 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 23 v 1222 (ObjNum 469 snap 139 seq_num 469) dirty exists 2024-09-12T23:46:11.360 INFO:tasks.rados.rados.0.smithi136.stdout:2258: done (1 left) 2024-09-12T23:46:11.360 INFO:tasks.rados.rados.0.smithi136.stdout:2260: done (0 left) 2024-09-12T23:46:11.360 INFO:tasks.rados.rados.0.smithi136.stdout:2261: rollback oid 37 current snap is 226 2024-09-12T23:46:11.360 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 37 to 183 2024-09-12T23:46:11.361 INFO:tasks.rados.rados.0.smithi136.stdout:2262: read oid 25 snap -1 2024-09-12T23:46:11.361 INFO:tasks.rados.rados.0.smithi136.stdout:2262: expect (ObjNum 717 snap 223 seq_num 717) 2024-09-12T23:46:11.361 INFO:tasks.rados.rados.0.smithi136.stdout:2263: read oid 28 snap -1 2024-09-12T23:46:11.361 INFO:tasks.rados.rados.0.smithi136.stdout:2263: expect (ObjNum 607 snap 175 seq_num 607) 2024-09-12T23:46:11.361 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 42 current snap is 226 2024-09-12T23:46:11.361 INFO:tasks.rados.rados.0.smithi136.stdout:2264: seq_num 730 ranges {4972544=229376} 2024-09-12T23:46:11.407 INFO:tasks.rados.rados.0.smithi136.stdout:2264: writing smithi13667081-42 from 4972544 to 5201920 tid 1 2024-09-12T23:46:11.407 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 47 current snap is 226 2024-09-12T23:46:11.407 INFO:tasks.rados.rados.0.smithi136.stdout:2265: seq_num 731 ranges {3440640=245760} 2024-09-12T23:46:11.428 INFO:tasks.rados.rados.0.smithi136.stdout:2265: writing smithi13667081-47 from 3440640 to 3686400 tid 1 2024-09-12T23:46:11.428 INFO:tasks.rados.rados.0.smithi136.stdout:2266: read oid 21 snap -1 2024-09-12T23:46:11.428 INFO:tasks.rados.rados.0.smithi136.stdout:2266: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-12T23:46:11.429 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 32 current snap is 226 2024-09-12T23:46:11.430 INFO:tasks.rados.rados.0.smithi136.stdout:2267: seq_num 732 ranges {1564672=262144} 2024-09-12T23:46:11.439 INFO:tasks.rados.rados.0.smithi136.stdout:2267: writing smithi13667081-32 from 1564672 to 1826816 tid 1 2024-09-12T23:46:11.439 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 15 current snap is 226 2024-09-12T23:46:11.439 INFO:tasks.rados.rados.0.smithi136.stdout:2262: expect (ObjNum 717 snap 223 seq_num 717) 2024-09-12T23:46:11.546 INFO:tasks.rados.rados.0.smithi136.stdout:2261: finishing rollback tid 0 to smithi13667081-37 2024-09-12T23:46:11.546 INFO:tasks.rados.rados.0.smithi136.stdout:2261: finishing rollback tid 1 to smithi13667081-37 2024-09-12T23:46:11.546 INFO:tasks.rados.rados.0.smithi136.stdout:2261: finishing rollback tid 2 to smithi13667081-37 2024-09-12T23:46:11.546 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 37 v 1253 (ObjNum 589 snap 172 seq_num 589) dirty exists 2024-09-12T23:46:11.546 INFO:tasks.rados.rados.0.smithi136.stdout:2264: finishing write tid 1 to smithi13667081-42 2024-09-12T23:46:11.546 INFO:tasks.rados.rados.0.smithi136.stdout:2264: finishing write tid 2 to smithi13667081-42 2024-09-12T23:46:11.546 INFO:tasks.rados.rados.0.smithi136.stdout:2263: expect (ObjNum 607 snap 175 seq_num 607) 2024-09-12T23:46:11.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:11 smithi136 bash[76371]: cluster 2024-09-12T23:46:10.430128+0000 mon.a (mon.0) 650 : cluster [DBG] osdmap e515: 8 total, 7 up, 8 in 2024-09-12T23:46:11.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:11 smithi136 bash[76371]: cluster 2024-09-12T23:46:10.739541+0000 mgr.y (mgr.24661) 209 : cluster [DBG] pgmap v348: 121 pgs: 30 active+undersized, 15 active+undersized+degraded, 76 active+clean; 350 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 546 KiB/s wr, 20 op/s; 100/555 objects degraded (18.018%) 2024-09-12T23:46:11.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:11 smithi136 bash[76371]: cluster 2024-09-12T23:46:11.355332+0000 mon.a (mon.0) 651 : cluster [WRN] Health check update: Degraded data redundancy: 98/546 objects degraded (17.949%), 15 pgs degraded (PG_DEGRADED) 2024-09-12T23:46:11.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:11 smithi136 bash[76371]: cluster 2024-09-12T23:46:11.363810+0000 mon.a (mon.0) 652 : cluster [DBG] osdmap e516: 8 total, 7 up, 8 in 2024-09-12T23:46:11.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:11 smithi136 bash[75164]: cluster 2024-09-12T23:46:10.430128+0000 mon.a (mon.0) 650 : cluster [DBG] osdmap e515: 8 total, 7 up, 8 in 2024-09-12T23:46:11.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:11 smithi136 bash[75164]: cluster 2024-09-12T23:46:10.739541+0000 mgr.y (mgr.24661) 209 : cluster [DBG] pgmap v348: 121 pgs: 30 active+undersized, 15 active+undersized+degraded, 76 active+clean; 350 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 546 KiB/s wr, 20 op/s; 100/555 objects degraded (18.018%) 2024-09-12T23:46:11.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:11 smithi136 bash[75164]: cluster 2024-09-12T23:46:11.355332+0000 mon.a (mon.0) 651 : cluster [WRN] Health check update: Degraded data redundancy: 98/546 objects degraded (17.949%), 15 pgs degraded (PG_DEGRADED) 2024-09-12T23:46:11.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:11 smithi136 bash[75164]: cluster 2024-09-12T23:46:11.363810+0000 mon.a (mon.0) 652 : cluster [DBG] osdmap e516: 8 total, 7 up, 8 in 2024-09-12T23:46:11.759 INFO:tasks.rados.rados.0.smithi136.stdout:2264: finishing write tid 3 to smithi13667081-42 2024-09-12T23:46:11.759 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 42 v 858 (ObjNum 730 snap 226 seq_num 730) dirty exists 2024-09-12T23:46:11.759 INFO:tasks.rados.rados.0.smithi136.stdout:2264: left oid 42 (ObjNum 730 snap 226 seq_num 730) 2024-09-12T23:46:11.759 INFO:tasks.rados.rados.0.smithi136.stdout:2266: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-12T23:46:11.762 INFO:tasks.rados.rados.0.smithi136.stdout:2265: finishing write tid 1 to smithi13667081-47 2024-09-12T23:46:11.762 INFO:tasks.rados.rados.0.smithi136.stdout:2265: finishing write tid 2 to smithi13667081-47 2024-09-12T23:46:11.762 INFO:tasks.rados.rados.0.smithi136.stdout:2265: finishing write tid 3 to smithi13667081-47 2024-09-12T23:46:11.762 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 47 v 714 (ObjNum 731 snap 226 seq_num 731) dirty exists 2024-09-12T23:46:11.762 INFO:tasks.rados.rados.0.smithi136.stdout:2265: left oid 47 (ObjNum 731 snap 226 seq_num 731) 2024-09-12T23:46:11.762 INFO:tasks.rados.rados.0.smithi136.stdout:2267: finishing write tid 1 to smithi13667081-32 2024-09-12T23:46:11.762 INFO:tasks.rados.rados.0.smithi136.stdout:2267: finishing write tid 2 to smithi13667081-32 2024-09-12T23:46:11.762 INFO:tasks.rados.rados.0.smithi136.stdout:2267: finishing write tid 3 to smithi13667081-32 2024-09-12T23:46:11.762 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 32 v 784 (ObjNum 732 snap 226 seq_num 732) dirty exists 2024-09-12T23:46:11.763 INFO:tasks.rados.rados.0.smithi136.stdout:2267: left oid 32 (ObjNum 732 snap 226 seq_num 732) 2024-09-12T23:46:11.763 INFO:tasks.rados.rados.0.smithi136.stdout:2268: seq_num 733 ranges {0=278528} 2024-09-12T23:46:11.764 INFO:tasks.rados.rados.0.smithi136.stdout:2268: writing smithi13667081-15 from 0 to 278528 tid 1 2024-09-12T23:46:11.764 INFO:tasks.rados.rados.0.smithi136.stdout:2261: done (7 left) 2024-09-12T23:46:11.764 INFO:tasks.rados.rados.0.smithi136.stdout:2262: done (6 left) 2024-09-12T23:46:11.765 INFO:tasks.rados.rados.0.smithi136.stdout:2263: done (5 left) 2024-09-12T23:46:11.765 INFO:tasks.rados.rados.0.smithi136.stdout:2264: done (4 left) 2024-09-12T23:46:11.765 INFO:tasks.rados.rados.0.smithi136.stdout:2265: done (3 left) 2024-09-12T23:46:11.765 INFO:tasks.rados.rados.0.smithi136.stdout:2266: done (2 left) 2024-09-12T23:46:11.765 INFO:tasks.rados.rados.0.smithi136.stdout:2267: done (1 left) 2024-09-12T23:46:11.765 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 30 current snap is 226 2024-09-12T23:46:11.765 INFO:tasks.rados.rados.0.smithi136.stdout:2269: seq_num 734 ranges {0=294912} 2024-09-12T23:46:11.766 INFO:tasks.rados.rados.0.smithi136.stdout:2269: writing smithi13667081-30 from 0 to 294912 tid 1 2024-09-12T23:46:11.766 INFO:tasks.rados.rados.0.smithi136.stdout:2270: rollback oid 42 current snap is 226 2024-09-12T23:46:11.766 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 42 to 210 2024-09-12T23:46:11.766 INFO:tasks.rados.rados.0.smithi136.stdout:2271: delete oid 16 current snap is 226 2024-09-12T23:46:11.770 INFO:tasks.rados.rados.0.smithi136.stdout:2270: finishing rollback tid 0 to smithi13667081-42 2024-09-12T23:46:11.772 INFO:tasks.rados.rados.0.smithi136.stdout:2270: finishing rollback tid 1 to smithi13667081-42 2024-09-12T23:46:11.772 INFO:tasks.rados.rados.0.smithi136.stdout:2270: finishing rollback tid 2 to smithi13667081-42 2024-09-12T23:46:11.772 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 42 v 861 (ObjNum 672 snap 197 seq_num 672) dirty exists 2024-09-12T23:46:11.773 INFO:tasks.rados.rados.0.smithi136.stdout:2269: finishing write tid 1 to smithi13667081-30 2024-09-12T23:46:11.774 INFO:tasks.rados.rados.0.smithi136.stdout:2268: finishing write tid 1 to smithi13667081-15 2024-09-12T23:46:11.774 INFO:tasks.rados.rados.0.smithi136.stdout:2269: finishing write tid 2 to smithi13667081-30 2024-09-12T23:46:11.774 INFO:tasks.rados.rados.0.smithi136.stdout:2268: finishing write tid 2 to smithi13667081-15 2024-09-12T23:46:11.774 INFO:tasks.rados.rados.0.smithi136.stdout:2270: done (3 left) 2024-09-12T23:46:11.774 INFO:tasks.rados.rados.0.smithi136.stdout:2271: done (2 left) 2024-09-12T23:46:11.775 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 4 current snap is 226 2024-09-12T23:46:11.775 INFO:tasks.rados.rados.0.smithi136.stdout:2272: seq_num 736 ranges {7118848=335872} 2024-09-12T23:46:11.816 INFO:tasks.rados.rados.0.smithi136.stdout:2272: writing smithi13667081-4 from 7118848 to 7454720 tid 1 2024-09-12T23:46:11.816 INFO:tasks.rados.rados.0.smithi136.stdout:2273: delete oid 13 current snap is 226 2024-09-12T23:46:11.816 INFO:tasks.rados.rados.0.smithi136.stdout:2269: finishing write tid 3 to smithi13667081-30 2024-09-12T23:46:11.816 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 30 v 1189 (ObjNum 734 snap 226 seq_num 734) dirty exists 2024-09-12T23:46:11.817 INFO:tasks.rados.rados.0.smithi136.stdout:2269: left oid 30 (ObjNum 734 snap 226 seq_num 734) 2024-09-12T23:46:11.817 INFO:tasks.rados.rados.0.smithi136.stdout:2268: finishing write tid 3 to smithi13667081-15 2024-09-12T23:46:11.817 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 15 v 1460 (ObjNum 733 snap 226 seq_num 733) dirty exists 2024-09-12T23:46:11.817 INFO:tasks.rados.rados.0.smithi136.stdout:2268: left oid 15 (ObjNum 733 snap 226 seq_num 733) 2024-09-12T23:46:11.821 INFO:tasks.rados.rados.0.smithi136.stdout:2268: done (3 left) 2024-09-12T23:46:11.821 INFO:tasks.rados.rados.0.smithi136.stdout:2269: done (2 left) 2024-09-12T23:46:11.821 INFO:tasks.rados.rados.0.smithi136.stdout:2273: done (1 left) 2024-09-12T23:46:11.821 INFO:tasks.rados.rados.0.smithi136.stdout:2274: snap_create 2024-09-12T23:46:11.824 INFO:tasks.rados.rados.0.smithi136.stdout:2272: finishing write tid 1 to smithi13667081-4 2024-09-12T23:46:11.824 INFO:tasks.rados.rados.0.smithi136.stdout:2272: finishing write tid 2 to smithi13667081-4 2024-09-12T23:46:11.827 INFO:tasks.rados.rados.0.smithi136.stdout:2272: finishing write tid 3 to smithi13667081-4 2024-09-12T23:46:11.827 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 4 v 786 (ObjNum 736 snap 226 seq_num 736) dirty exists 2024-09-12T23:46:11.827 INFO:tasks.rados.rados.0.smithi136.stdout:2272: left oid 4 (ObjNum 736 snap 226 seq_num 736) 2024-09-12T23:46:11.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:11 smithi181 bash[67587]: cluster 2024-09-12T23:46:10.430128+0000 mon.a (mon.0) 650 : cluster [DBG] osdmap e515: 8 total, 7 up, 8 in 2024-09-12T23:46:11.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:11 smithi181 bash[67587]: cluster 2024-09-12T23:46:10.739541+0000 mgr.y (mgr.24661) 209 : cluster [DBG] pgmap v348: 121 pgs: 30 active+undersized, 15 active+undersized+degraded, 76 active+clean; 350 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 546 KiB/s wr, 20 op/s; 100/555 objects degraded (18.018%) 2024-09-12T23:46:11.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:11 smithi181 bash[67587]: cluster 2024-09-12T23:46:11.355332+0000 mon.a (mon.0) 651 : cluster [WRN] Health check update: Degraded data redundancy: 98/546 objects degraded (17.949%), 15 pgs degraded (PG_DEGRADED) 2024-09-12T23:46:11.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:11 smithi181 bash[67587]: cluster 2024-09-12T23:46:11.363810+0000 mon.a (mon.0) 652 : cluster [DBG] osdmap e516: 8 total, 7 up, 8 in 2024-09-12T23:46:12.437 INFO:tasks.rados.rados.0.smithi136.stdout:2272: done (1 left) 2024-09-12T23:46:12.437 INFO:tasks.rados.rados.0.smithi136.stdout:2274: done (0 left) 2024-09-12T23:46:12.437 INFO:tasks.rados.rados.0.smithi136.stdout:2275: delete oid 1 current snap is 227 2024-09-12T23:46:12.446 INFO:tasks.rados.rados.0.smithi136.stdout:2275: done (0 left) 2024-09-12T23:46:12.446 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 40 current snap is 227 2024-09-12T23:46:12.446 INFO:tasks.rados.rados.0.smithi136.stdout:2276: seq_num 739 ranges {0=385024} 2024-09-12T23:46:12.450 INFO:tasks.rados.rados.0.smithi136.stdout:2276: writing smithi13667081-40 from 0 to 385024 tid 1 2024-09-12T23:46:12.450 INFO:tasks.rados.rados.0.smithi136.stdout:2277: copy_from oid 26 from oid 4 current snap is 227 2024-09-12T23:46:12.450 INFO:tasks.rados.rados.0.smithi136.stdout:2278: snap_create 2024-09-12T23:46:12.460 INFO:tasks.rados.rados.0.smithi136.stdout:2276: finishing write tid 1 to smithi13667081-40 2024-09-12T23:46:12.460 INFO:tasks.rados.rados.0.smithi136.stdout:2276: finishing write tid 2 to smithi13667081-40 2024-09-12T23:46:12.463 INFO:tasks.rados.rados.0.smithi136.stdout:2276: finishing write tid 3 to smithi13667081-40 2024-09-12T23:46:12.463 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 40 v 1581 (ObjNum 739 snap 227 seq_num 739) dirty exists 2024-09-12T23:46:12.463 INFO:tasks.rados.rados.0.smithi136.stdout:2276: left oid 40 (ObjNum 739 snap 227 seq_num 739) 2024-09-12T23:46:12.508 INFO:tasks.rados.rados.0.smithi136.stdout:2277: finishing copy_from racing read to smithi13667081-26 2024-09-12T23:46:12.508 INFO:tasks.rados.rados.0.smithi136.stdout:2277: finishing copy_from to smithi13667081-26 2024-09-12T23:46:12.509 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 26 v 786 (ObjNum 736 snap 226 seq_num 736) dirty exists 2024-09-12T23:46:12.594 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:46:12 smithi181 bash[60471]: ts=2024-09-12T23:46:12.313Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:46:13.453 INFO:tasks.rados.rados.0.smithi136.stdout:2276: done (2 left) 2024-09-12T23:46:13.454 INFO:tasks.rados.rados.0.smithi136.stdout:2277: done (1 left) 2024-09-12T23:46:13.454 INFO:tasks.rados.rados.0.smithi136.stdout:2278: done (0 left) 2024-09-12T23:46:13.454 INFO:tasks.rados.rados.0.smithi136.stdout:2279: snap_remove snap 213 2024-09-12T23:46:13.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:13 smithi136 bash[75164]: cluster 2024-09-12T23:46:12.440990+0000 mon.a (mon.0) 653 : cluster [DBG] osdmap e517: 8 total, 7 up, 8 in 2024-09-12T23:46:13.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:13 smithi136 bash[75164]: cluster 2024-09-12T23:46:12.740407+0000 mgr.y (mgr.24661) 210 : cluster [DBG] pgmap v351: 121 pgs: 30 active+undersized, 15 active+undersized+degraded, 76 active+clean; 353 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 1.1 MiB/s wr, 9 op/s; 102/561 objects degraded (18.182%) 2024-09-12T23:46:13.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:13 smithi136 bash[75164]: audit 2024-09-12T23:46:12.843837+0000 mon.a (mon.0) 654 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:13.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:13 smithi136 bash[75164]: audit 2024-09-12T23:46:12.854574+0000 mon.a (mon.0) 655 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:13.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:13 smithi136 bash[76371]: cluster 2024-09-12T23:46:12.440990+0000 mon.a (mon.0) 653 : cluster [DBG] osdmap e517: 8 total, 7 up, 8 in 2024-09-12T23:46:13.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:13 smithi136 bash[76371]: cluster 2024-09-12T23:46:12.740407+0000 mgr.y (mgr.24661) 210 : cluster [DBG] pgmap v351: 121 pgs: 30 active+undersized, 15 active+undersized+degraded, 76 active+clean; 353 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 1.1 MiB/s wr, 9 op/s; 102/561 objects degraded (18.182%) 2024-09-12T23:46:13.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:13 smithi136 bash[76371]: audit 2024-09-12T23:46:12.843837+0000 mon.a (mon.0) 654 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:13.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:13 smithi136 bash[76371]: audit 2024-09-12T23:46:12.854574+0000 mon.a (mon.0) 655 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:13.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:13 smithi181 bash[67587]: cluster 2024-09-12T23:46:12.440990+0000 mon.a (mon.0) 653 : cluster [DBG] osdmap e517: 8 total, 7 up, 8 in 2024-09-12T23:46:13.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:13 smithi181 bash[67587]: cluster 2024-09-12T23:46:12.740407+0000 mgr.y (mgr.24661) 210 : cluster [DBG] pgmap v351: 121 pgs: 30 active+undersized, 15 active+undersized+degraded, 76 active+clean; 353 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 1.1 MiB/s wr, 9 op/s; 102/561 objects degraded (18.182%) 2024-09-12T23:46:13.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:13 smithi181 bash[67587]: audit 2024-09-12T23:46:12.843837+0000 mon.a (mon.0) 654 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:13.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:13 smithi181 bash[67587]: audit 2024-09-12T23:46:12.854574+0000 mon.a (mon.0) 655 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:14.344 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:46:13 smithi181 bash[72192]: debug 2024-09-12T23:46:13.963+0000 7f7a3cf72640 -1 osd.4 507 log_to_monitors true 2024-09-12T23:46:14.460 INFO:tasks.rados.rados.0.smithi136.stdout:2279: done (0 left) 2024-09-12T23:46:14.460 INFO:tasks.rados.rados.0.smithi136.stdout:2280: delete oid 21 current snap is 228 2024-09-12T23:46:14.471 INFO:tasks.rados.rados.0.smithi136.stdout:2280: done (0 left) 2024-09-12T23:46:14.471 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 23 current snap is 228 2024-09-12T23:46:14.471 INFO:tasks.rados.rados.0.smithi136.stdout:2281: seq_num 741 ranges {2670592=417792} 2024-09-12T23:46:14.488 INFO:tasks.rados.rados.0.smithi136.stdout:2281: writing smithi13667081-23 from 2670592 to 3088384 tid 1 2024-09-12T23:46:14.489 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 43 current snap is 228 2024-09-12T23:46:14.489 INFO:tasks.rados.rados.0.smithi136.stdout:2282: seq_num 742 ranges {12206080=434176} 2024-09-12T23:46:14.561 INFO:tasks.rados.rados.0.smithi136.stdout:2282: writing smithi13667081-43 from 12206080 to 12640256 tid 1 2024-09-12T23:46:14.561 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 48 current snap is 228 2024-09-12T23:46:14.561 INFO:tasks.rados.rados.0.smithi136.stdout:2283: seq_num 743 ranges {2670592=450560} 2024-09-12T23:46:14.579 INFO:tasks.rados.rados.0.smithi136.stdout:2283: writing smithi13667081-48 from 2670592 to 3121152 tid 1 2024-09-12T23:46:14.579 INFO:tasks.rados.rados.0.smithi136.stdout:2284: read oid 10 snap -1 2024-09-12T23:46:14.579 INFO:tasks.rados.rados.0.smithi136.stdout:2284: expect (ObjNum 669 snap 196 seq_num 669) 2024-09-12T23:46:14.579 INFO:tasks.rados.rados.0.smithi136.stdout:2285: snap_remove snap 220 2024-09-12T23:46:14.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:14 smithi136 bash[75164]: cluster 2024-09-12T23:46:13.457124+0000 mon.a (mon.0) 656 : cluster [DBG] osdmap e518: 8 total, 7 up, 8 in 2024-09-12T23:46:14.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:14 smithi136 bash[75164]: audit 2024-09-12T23:46:13.647837+0000 mon.a (mon.0) 657 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:14.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:14 smithi136 bash[75164]: audit 2024-09-12T23:46:13.659894+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:14.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:14 smithi136 bash[75164]: audit 2024-09-12T23:46:13.972315+0000 mon.a (mon.0) 659 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-12T23:46:14.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:14 smithi136 bash[75164]: audit 2024-09-12T23:46:13.972446+0000 mon.b (mon.1) 4 : audit [INF] from='osd.4 [v2:172.21.15.181:6800/2759874318,v1:172.21.15.181:6801/2759874318]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-12T23:46:14.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:14 smithi136 bash[76371]: cluster 2024-09-12T23:46:13.457124+0000 mon.a (mon.0) 656 : cluster [DBG] osdmap e518: 8 total, 7 up, 8 in 2024-09-12T23:46:14.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:14 smithi136 bash[76371]: audit 2024-09-12T23:46:13.647837+0000 mon.a (mon.0) 657 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:14.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:14 smithi136 bash[76371]: audit 2024-09-12T23:46:13.659894+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:14.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:14 smithi136 bash[76371]: audit 2024-09-12T23:46:13.972315+0000 mon.a (mon.0) 659 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-12T23:46:14.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:14 smithi136 bash[76371]: audit 2024-09-12T23:46:13.972446+0000 mon.b (mon.1) 4 : audit [INF] from='osd.4 [v2:172.21.15.181:6800/2759874318,v1:172.21.15.181:6801/2759874318]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-12T23:46:14.844 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:46:14 smithi181 bash[72192]: debug 2024-09-12T23:46:14.503+0000 7f7a34d1c640 -1 osd.4 507 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-12T23:46:14.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:14 smithi181 bash[67587]: cluster 2024-09-12T23:46:13.457124+0000 mon.a (mon.0) 656 : cluster [DBG] osdmap e518: 8 total, 7 up, 8 in 2024-09-12T23:46:14.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:14 smithi181 bash[67587]: audit 2024-09-12T23:46:13.647837+0000 mon.a (mon.0) 657 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:14.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:14 smithi181 bash[67587]: audit 2024-09-12T23:46:13.659894+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:14.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:14 smithi181 bash[67587]: audit 2024-09-12T23:46:13.972315+0000 mon.a (mon.0) 659 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-12T23:46:14.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:14 smithi181 bash[67587]: audit 2024-09-12T23:46:13.972446+0000 mon.b (mon.1) 4 : audit [INF] from='osd.4 [v2:172.21.15.181:6800/2759874318,v1:172.21.15.181:6801/2759874318]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-12T23:46:15.483 INFO:tasks.rados.rados.0.smithi136.stdout:2285: done (4 left) 2024-09-12T23:46:15.483 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 16 current snap is 228 2024-09-12T23:46:15.483 INFO:tasks.rados.rados.0.smithi136.stdout:2286: seq_num 744 ranges {0=466944} 2024-09-12T23:46:15.487 INFO:tasks.rados.rados.0.smithi136.stdout:2286: writing smithi13667081-16 from 0 to 466944 tid 1 2024-09-12T23:46:15.487 INFO:tasks.rados.rados.0.smithi136.stdout:2287: read oid 21 snap 183 2024-09-12T23:46:15.487 INFO:tasks.rados.rados.0.smithi136.stdout:2287: expect deleted 2024-09-12T23:46:15.487 INFO:tasks.rados.rados.0.smithi136.stdout:2288: read oid 37 snap -1 2024-09-12T23:46:15.487 INFO:tasks.rados.rados.0.smithi136.stdout:2288: expect (ObjNum 589 snap 172 seq_num 589) 2024-09-12T23:46:15.487 INFO:tasks.rados.rados.0.smithi136.stdout:2289: snap_create 2024-09-12T23:46:15.487 INFO:tasks.rados.rados.0.smithi136.stdout:2281: finishing write tid 1 to smithi13667081-23 2024-09-12T23:46:15.487 INFO:tasks.rados.rados.0.smithi136.stdout:2281: finishing write tid 2 to smithi13667081-23 2024-09-12T23:46:15.487 INFO:tasks.rados.rados.0.smithi136.stdout:2281: finishing write tid 3 to smithi13667081-23 2024-09-12T23:46:15.488 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 23 v 1584 (ObjNum 741 snap 228 seq_num 741) dirty exists 2024-09-12T23:46:15.488 INFO:tasks.rados.rados.0.smithi136.stdout:2281: left oid 23 (ObjNum 741 snap 228 seq_num 741) 2024-09-12T23:46:15.488 INFO:tasks.rados.rados.0.smithi136.stdout:2282: finishing write tid 1 to smithi13667081-43 2024-09-12T23:46:15.488 INFO:tasks.rados.rados.0.smithi136.stdout:2282: finishing write tid 2 to smithi13667081-43 2024-09-12T23:46:15.488 INFO:tasks.rados.rados.0.smithi136.stdout:2282: finishing write tid 3 to smithi13667081-43 2024-09-12T23:46:15.488 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 43 v 1326 (ObjNum 742 snap 228 seq_num 742) dirty exists 2024-09-12T23:46:15.488 INFO:tasks.rados.rados.0.smithi136.stdout:2282: left oid 43 (ObjNum 742 snap 228 seq_num 742) 2024-09-12T23:46:15.488 INFO:tasks.rados.rados.0.smithi136.stdout:2283: finishing write tid 1 to smithi13667081-48 2024-09-12T23:46:15.488 INFO:tasks.rados.rados.0.smithi136.stdout:2283: finishing write tid 2 to smithi13667081-48 2024-09-12T23:46:15.488 INFO:tasks.rados.rados.0.smithi136.stdout:2283: finishing write tid 3 to smithi13667081-48 2024-09-12T23:46:15.488 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 48 v 1329 (ObjNum 743 snap 228 seq_num 743) dirty exists 2024-09-12T23:46:15.488 INFO:tasks.rados.rados.0.smithi136.stdout:2283: left oid 48 (ObjNum 743 snap 228 seq_num 743) 2024-09-12T23:46:15.488 INFO:tasks.rados.rados.0.smithi136.stdout:2284: expect (ObjNum 669 snap 196 seq_num 669) 2024-09-12T23:46:15.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:15 smithi136 bash[75164]: audit 2024-09-12T23:46:14.458699+0000 mon.a (mon.0) 660 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-12T23:46:15.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:15 smithi136 bash[75164]: cluster 2024-09-12T23:46:14.458891+0000 mon.a (mon.0) 661 : cluster [DBG] osdmap e519: 8 total, 7 up, 8 in 2024-09-12T23:46:15.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:15 smithi136 bash[75164]: audit 2024-09-12T23:46:14.461523+0000 mon.b (mon.1) 5 : audit [INF] from='osd.4 [v2:172.21.15.181:6800/2759874318,v1:172.21.15.181:6801/2759874318]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-12T23:46:15.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:15 smithi136 bash[75164]: audit 2024-09-12T23:46:14.463674+0000 mon.a (mon.0) 662 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-12T23:46:15.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:15 smithi136 bash[75164]: cluster 2024-09-12T23:46:14.741663+0000 mgr.y (mgr.24661) 211 : cluster [DBG] pgmap v354: 121 pgs: 30 active+undersized, 15 active+undersized+degraded, 76 active+clean; 356 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 1.7 MiB/s wr, 11 op/s; 104/576 objects degraded (18.056%) 2024-09-12T23:46:15.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:15 smithi136 bash[76371]: audit 2024-09-12T23:46:14.458699+0000 mon.a (mon.0) 660 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-12T23:46:15.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:15 smithi136 bash[76371]: cluster 2024-09-12T23:46:14.458891+0000 mon.a (mon.0) 661 : cluster [DBG] osdmap e519: 8 total, 7 up, 8 in 2024-09-12T23:46:15.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:15 smithi136 bash[76371]: audit 2024-09-12T23:46:14.461523+0000 mon.b (mon.1) 5 : audit [INF] from='osd.4 [v2:172.21.15.181:6800/2759874318,v1:172.21.15.181:6801/2759874318]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-12T23:46:15.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:15 smithi136 bash[76371]: audit 2024-09-12T23:46:14.463674+0000 mon.a (mon.0) 662 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-12T23:46:15.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:15 smithi136 bash[76371]: cluster 2024-09-12T23:46:14.741663+0000 mgr.y (mgr.24661) 211 : cluster [DBG] pgmap v354: 121 pgs: 30 active+undersized, 15 active+undersized+degraded, 76 active+clean; 356 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 1.7 MiB/s wr, 11 op/s; 104/576 objects degraded (18.056%) 2024-09-12T23:46:15.807 INFO:tasks.rados.rados.0.smithi136.stdout:2288: expect (ObjNum 589 snap 172 seq_num 589) 2024-09-12T23:46:15.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:15 smithi181 bash[67587]: audit 2024-09-12T23:46:14.458699+0000 mon.a (mon.0) 660 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-12T23:46:15.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:15 smithi181 bash[67587]: cluster 2024-09-12T23:46:14.458891+0000 mon.a (mon.0) 661 : cluster [DBG] osdmap e519: 8 total, 7 up, 8 in 2024-09-12T23:46:15.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:15 smithi181 bash[67587]: audit 2024-09-12T23:46:14.461523+0000 mon.b (mon.1) 5 : audit [INF] from='osd.4 [v2:172.21.15.181:6800/2759874318,v1:172.21.15.181:6801/2759874318]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-12T23:46:15.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:15 smithi181 bash[67587]: audit 2024-09-12T23:46:14.463674+0000 mon.a (mon.0) 662 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-12T23:46:15.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:15 smithi181 bash[67587]: cluster 2024-09-12T23:46:14.741663+0000 mgr.y (mgr.24661) 211 : cluster [DBG] pgmap v354: 121 pgs: 30 active+undersized, 15 active+undersized+degraded, 76 active+clean; 356 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 1.7 MiB/s wr, 11 op/s; 104/576 objects degraded (18.056%) 2024-09-12T23:46:16.504 INFO:tasks.rados.rados.0.smithi136.stdout:2281: done (7 left) 2024-09-12T23:46:16.504 INFO:tasks.rados.rados.0.smithi136.stdout:2282: done (6 left) 2024-09-12T23:46:16.504 INFO:tasks.rados.rados.0.smithi136.stdout:2283: done (5 left) 2024-09-12T23:46:16.504 INFO:tasks.rados.rados.0.smithi136.stdout:2284: done (4 left) 2024-09-12T23:46:16.505 INFO:tasks.rados.rados.0.smithi136.stdout:2287: done (3 left) 2024-09-12T23:46:16.505 INFO:tasks.rados.rados.0.smithi136.stdout:2288: done (2 left) 2024-09-12T23:46:16.505 INFO:tasks.rados.rados.0.smithi136.stdout:2289: done (1 left) 2024-09-12T23:46:16.505 INFO:tasks.rados.rados.0.smithi136.stdout:2290: setattr oid 10 current snap is 229 2024-09-12T23:46:16.506 INFO:tasks.rados.rados.0.smithi136.stdout:2291: rollback oid 39 current snap is 229 2024-09-12T23:46:16.506 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 39 to 192 2024-09-12T23:46:16.506 INFO:tasks.rados.rados.0.smithi136.stdout:2292: snap_remove snap 221 2024-09-12T23:46:16.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:16 smithi136 bash[76371]: cluster 2024-09-12T23:46:15.467755+0000 mon.a (mon.0) 663 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-12T23:46:16.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:16 smithi136 bash[76371]: cluster 2024-09-12T23:46:15.485275+0000 mon.a (mon.0) 664 : cluster [INF] osd.4 [v2:172.21.15.181:6800/2759874318,v1:172.21.15.181:6801/2759874318] boot 2024-09-12T23:46:16.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:16 smithi136 bash[76371]: cluster 2024-09-12T23:46:15.485417+0000 mon.a (mon.0) 665 : cluster [DBG] osdmap e520: 8 total, 8 up, 8 in 2024-09-12T23:46:16.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:16 smithi136 bash[76371]: audit 2024-09-12T23:46:15.486563+0000 mon.a (mon.0) 666 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-12T23:46:16.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:16 smithi136 bash[76371]: cluster 2024-09-12T23:46:16.357784+0000 mon.a (mon.0) 667 : cluster [WRN] Health check update: Degraded data redundancy: 104/576 objects degraded (18.056%), 15 pgs degraded (PG_DEGRADED) 2024-09-12T23:46:16.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:16 smithi136 bash[75164]: cluster 2024-09-12T23:46:15.467755+0000 mon.a (mon.0) 663 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-12T23:46:16.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:16 smithi136 bash[75164]: cluster 2024-09-12T23:46:15.485275+0000 mon.a (mon.0) 664 : cluster [INF] osd.4 [v2:172.21.15.181:6800/2759874318,v1:172.21.15.181:6801/2759874318] boot 2024-09-12T23:46:16.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:16 smithi136 bash[75164]: cluster 2024-09-12T23:46:15.485417+0000 mon.a (mon.0) 665 : cluster [DBG] osdmap e520: 8 total, 8 up, 8 in 2024-09-12T23:46:16.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:16 smithi136 bash[75164]: audit 2024-09-12T23:46:15.486563+0000 mon.a (mon.0) 666 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-12T23:46:16.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:16 smithi136 bash[75164]: cluster 2024-09-12T23:46:16.357784+0000 mon.a (mon.0) 667 : cluster [WRN] Health check update: Degraded data redundancy: 104/576 objects degraded (18.056%), 15 pgs degraded (PG_DEGRADED) 2024-09-12T23:46:16.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:16 smithi181 bash[67587]: cluster 2024-09-12T23:46:15.467755+0000 mon.a (mon.0) 663 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-12T23:46:16.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:16 smithi181 bash[67587]: cluster 2024-09-12T23:46:15.485275+0000 mon.a (mon.0) 664 : cluster [INF] osd.4 [v2:172.21.15.181:6800/2759874318,v1:172.21.15.181:6801/2759874318] boot 2024-09-12T23:46:16.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:16 smithi181 bash[67587]: cluster 2024-09-12T23:46:15.485417+0000 mon.a (mon.0) 665 : cluster [DBG] osdmap e520: 8 total, 8 up, 8 in 2024-09-12T23:46:16.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:16 smithi181 bash[67587]: audit 2024-09-12T23:46:15.486563+0000 mon.a (mon.0) 666 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-12T23:46:16.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:16 smithi181 bash[67587]: cluster 2024-09-12T23:46:16.357784+0000 mon.a (mon.0) 667 : cluster [WRN] Health check update: Degraded data redundancy: 104/576 objects degraded (18.056%), 15 pgs degraded (PG_DEGRADED) 2024-09-12T23:46:17.504 INFO:tasks.rados.rados.0.smithi136.stdout:2292: done (3 left) 2024-09-12T23:46:17.505 INFO:tasks.rados.rados.0.smithi136.stdout:2293: read oid 26 snap -1 2024-09-12T23:46:17.505 INFO:tasks.rados.rados.0.smithi136.stdout:2293: expect (ObjNum 736 snap 226 seq_num 736) 2024-09-12T23:46:17.505 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 32 current snap is 229 2024-09-12T23:46:17.505 INFO:tasks.rados.rados.0.smithi136.stdout:2294: seq_num 745 ranges {1826816=483328} 2024-09-12T23:46:17.519 INFO:tasks.rados.rados.0.smithi136.stdout:2294: writing smithi13667081-32 from 1826816 to 2310144 tid 1 2024-09-12T23:46:17.519 INFO:tasks.rados.rados.0.smithi136.stdout:2295: snap_remove snap 219 2024-09-12T23:46:17.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:17 smithi136 bash[75164]: cluster 2024-09-12T23:46:16.501119+0000 mon.a (mon.0) 668 : cluster [DBG] osdmap e521: 8 total, 8 up, 8 in 2024-09-12T23:46:17.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:17 smithi136 bash[75164]: cluster 2024-09-12T23:46:16.742469+0000 mgr.y (mgr.24661) 212 : cluster [DBG] pgmap v357: 121 pgs: 2 activating, 1 active+recovery_wait+degraded, 29 active+undersized, 9 active+undersized+degraded, 80 active+clean; 360 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.1 MiB/s rd, 3.0 MiB/s wr, 12 op/s; 81/588 objects degraded (13.776%) 2024-09-12T23:46:17.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:17 smithi136 bash[76371]: cluster 2024-09-12T23:46:16.501119+0000 mon.a (mon.0) 668 : cluster [DBG] osdmap e521: 8 total, 8 up, 8 in 2024-09-12T23:46:17.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:17 smithi136 bash[76371]: cluster 2024-09-12T23:46:16.742469+0000 mgr.y (mgr.24661) 212 : cluster [DBG] pgmap v357: 121 pgs: 2 activating, 1 active+recovery_wait+degraded, 29 active+undersized, 9 active+undersized+degraded, 80 active+clean; 360 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.1 MiB/s rd, 3.0 MiB/s wr, 12 op/s; 81/588 objects degraded (13.776%) 2024-09-12T23:46:17.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:17 smithi181 bash[67587]: cluster 2024-09-12T23:46:16.501119+0000 mon.a (mon.0) 668 : cluster [DBG] osdmap e521: 8 total, 8 up, 8 in 2024-09-12T23:46:17.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:17 smithi181 bash[67587]: cluster 2024-09-12T23:46:16.742469+0000 mgr.y (mgr.24661) 212 : cluster [DBG] pgmap v357: 121 pgs: 2 activating, 1 active+recovery_wait+degraded, 29 active+undersized, 9 active+undersized+degraded, 80 active+clean; 360 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.1 MiB/s rd, 3.0 MiB/s wr, 12 op/s; 81/588 objects degraded (13.776%) 2024-09-12T23:46:18.507 INFO:tasks.rados.rados.0.smithi136.stdout:2295: done (5 left) 2024-09-12T23:46:18.507 INFO:tasks.rados.rados.0.smithi136.stdout:2296: rmattr oid 42 current snap is 229 2024-09-12T23:46:18.507 INFO:tasks.rados.rados.0.smithi136.stdout:2296: done (5 left) 2024-09-12T23:46:18.507 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 29 current snap is 229 2024-09-12T23:46:18.507 INFO:tasks.rados.rados.0.smithi136.stdout:2297: seq_num 746 ranges {0=499712} 2024-09-12T23:46:18.510 INFO:tasks.rados.rados.0.smithi136.stdout:2297: writing smithi13667081-29 from 0 to 499712 tid 1 2024-09-12T23:46:18.512 INFO:tasks.rados.rados.0.smithi136.stdout:2291: finishing rollback tid 0 to smithi13667081-39 2024-09-12T23:46:18.512 INFO:tasks.rados.rados.0.smithi136.stdout:2291: finishing rollback tid 1 to smithi13667081-39 2024-09-12T23:46:18.512 INFO:tasks.rados.rados.0.smithi136.stdout:2291: finishing rollback tid 2 to smithi13667081-39 2024-09-12T23:46:18.512 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 39 v 853 (ObjNum 651 snap 191 seq_num 651) dirty exists 2024-09-12T23:46:18.512 INFO:tasks.rados.rados.0.smithi136.stdout:2286: finishing write tid 1 to smithi13667081-16 2024-09-12T23:46:18.512 INFO:tasks.rados.rados.0.smithi136.stdout:2286: finishing write tid 2 to smithi13667081-16 2024-09-12T23:46:18.512 INFO:tasks.rados.rados.0.smithi136.stdout:2286: finishing write tid 3 to smithi13667081-16 2024-09-12T23:46:18.512 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 16 v 1331 (ObjNum 744 snap 228 seq_num 744) dirty exists 2024-09-12T23:46:18.512 INFO:tasks.rados.rados.0.smithi136.stdout:2286: left oid 16 (ObjNum 744 snap 228 seq_num 744) 2024-09-12T23:46:18.512 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 10 v 1190 (ObjNum 669 snap 196 seq_num 669) dirty exists 2024-09-12T23:46:18.512 INFO:tasks.rados.rados.0.smithi136.stdout:2294: finishing write tid 1 to smithi13667081-32 2024-09-12T23:46:18.512 INFO:tasks.rados.rados.0.smithi136.stdout:2294: finishing write tid 2 to smithi13667081-32 2024-09-12T23:46:18.513 INFO:tasks.rados.rados.0.smithi136.stdout:2294: finishing write tid 3 to smithi13667081-32 2024-09-12T23:46:18.513 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 32 v 788 (ObjNum 745 snap 229 seq_num 745) dirty exists 2024-09-12T23:46:18.513 INFO:tasks.rados.rados.0.smithi136.stdout:2294: left oid 32 (ObjNum 745 snap 229 seq_num 745) 2024-09-12T23:46:18.513 INFO:tasks.rados.rados.0.smithi136.stdout:2293: expect (ObjNum 736 snap 226 seq_num 736) 2024-09-12T23:46:18.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:18 smithi181 bash[67587]: cluster 2024-09-12T23:46:17.506738+0000 mon.a (mon.0) 669 : cluster [DBG] osdmap e522: 8 total, 8 up, 8 in 2024-09-12T23:46:18.981 INFO:tasks.rados.rados.0.smithi136.stdout:2297: finishing write tid 1 to smithi13667081-29 2024-09-12T23:46:18.981 INFO:tasks.rados.rados.0.smithi136.stdout:2297: finishing write tid 2 to smithi13667081-29 2024-09-12T23:46:18.981 INFO:tasks.rados.rados.0.smithi136.stdout:2297: finishing write tid 3 to smithi13667081-29 2024-09-12T23:46:18.981 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 29 v 1237 (ObjNum 746 snap 229 seq_num 746) dirty exists 2024-09-12T23:46:18.981 INFO:tasks.rados.rados.0.smithi136.stdout:2297: left oid 29 (ObjNum 746 snap 229 seq_num 746) 2024-09-12T23:46:18.981 INFO:tasks.rados.rados.0.smithi136.stdout:2286: done (5 left) 2024-09-12T23:46:18.981 INFO:tasks.rados.rados.0.smithi136.stdout:2290: done (4 left) 2024-09-12T23:46:18.981 INFO:tasks.rados.rados.0.smithi136.stdout:2291: done (3 left) 2024-09-12T23:46:18.981 INFO:tasks.rados.rados.0.smithi136.stdout:2293: done (2 left) 2024-09-12T23:46:18.981 INFO:tasks.rados.rados.0.smithi136.stdout:2294: done (1 left) 2024-09-12T23:46:18.981 INFO:tasks.rados.rados.0.smithi136.stdout:2297: done (0 left) 2024-09-12T23:46:18.982 INFO:tasks.rados.rados.0.smithi136.stdout:2298: copy_from oid 48 from oid 7 current snap is 229 2024-09-12T23:46:18.982 INFO:tasks.rados.rados.0.smithi136.stdout:2299: delete oid 29 current snap is 229 2024-09-12T23:46:18.986 INFO:tasks.rados.rados.0.smithi136.stdout:2299: done (1 left) 2024-09-12T23:46:18.986 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 32 current snap is 229 2024-09-12T23:46:18.986 INFO:tasks.rados.rados.0.smithi136.stdout:2300: seq_num 748 ranges {2310144=532480} 2024-09-12T23:46:19.001 INFO:tasks.rados.rados.0.smithi136.stdout:2300: writing smithi13667081-32 from 2310144 to 2842624 tid 1 2024-09-12T23:46:19.002 INFO:tasks.rados.rados.0.smithi136.stdout:2301: snap_remove snap 183 2024-09-12T23:46:19.005 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:18 smithi136 bash[75164]: cluster 2024-09-12T23:46:17.506738+0000 mon.a (mon.0) 669 : cluster [DBG] osdmap e522: 8 total, 8 up, 8 in 2024-09-12T23:46:19.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:18 smithi136 bash[76371]: cluster 2024-09-12T23:46:17.506738+0000 mon.a (mon.0) 669 : cluster [DBG] osdmap e522: 8 total, 8 up, 8 in 2024-09-12T23:46:19.526 INFO:tasks.rados.rados.0.smithi136.stdout:2301: done (2 left) 2024-09-12T23:46:19.526 INFO:tasks.rados.rados.0.smithi136.stdout:2302: delete oid 42 current snap is 229 2024-09-12T23:46:19.526 INFO:tasks.rados.rados.0.smithi136.stdout:2298: finishing copy_from racing read to smithi13667081-48 2024-09-12T23:46:19.526 INFO:tasks.rados.rados.0.smithi136.stdout:2298: finishing copy_from to smithi13667081-48 2024-09-12T23:46:19.526 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 48 v 850 (ObjNum 717 snap 223 seq_num 717) dirty exists 2024-09-12T23:46:19.526 INFO:tasks.rados.rados.0.smithi136.stdout:2300: finishing write tid 1 to smithi13667081-32 2024-09-12T23:46:19.526 INFO:tasks.rados.rados.0.smithi136.stdout:2300: finishing write tid 2 to smithi13667081-32 2024-09-12T23:46:19.527 INFO:tasks.rados.rados.0.smithi136.stdout:2300: finishing write tid 3 to smithi13667081-32 2024-09-12T23:46:19.527 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 32 v 790 (ObjNum 748 snap 229 seq_num 748) dirty exists 2024-09-12T23:46:19.527 INFO:tasks.rados.rados.0.smithi136.stdout:2300: left oid 32 (ObjNum 748 snap 229 seq_num 748) 2024-09-12T23:46:19.557 INFO:tasks.rados.rados.0.smithi136.stdout:2298: done (2 left) 2024-09-12T23:46:19.558 INFO:tasks.rados.rados.0.smithi136.stdout:2300: done (1 left) 2024-09-12T23:46:19.558 INFO:tasks.rados.rados.0.smithi136.stdout:2302: done (0 left) 2024-09-12T23:46:19.558 INFO:tasks.rados.rados.0.smithi136.stdout:2303: setattr oid 22 current snap is 229 2024-09-12T23:46:19.560 INFO:tasks.rados.rados.0.smithi136.stdout:2304: snap_create 2024-09-12T23:46:19.568 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 22 v 864 (ObjNum 687 snap 207 seq_num 687) dirty exists 2024-09-12T23:46:19.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:46:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:46:19] "GET /metrics HTTP/1.1" 200 36387 "" "Prometheus/2.43.0" 2024-09-12T23:46:19.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:19 smithi136 bash[75164]: cluster 2024-09-12T23:46:18.509315+0000 mon.a (mon.0) 670 : cluster [DBG] osdmap e523: 8 total, 8 up, 8 in 2024-09-12T23:46:19.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:19 smithi136 bash[75164]: cluster 2024-09-12T23:46:18.743232+0000 mgr.y (mgr.24661) 213 : cluster [DBG] pgmap v360: 121 pgs: 2 activating, 1 active+recovery_wait+degraded, 24 active+undersized, 6 active+undersized+degraded, 88 active+clean; 361 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 335 KiB/s rd, 2.2 MiB/s wr, 5 op/s; 54/594 objects degraded (9.091%); 4.9 MiB/s, 1 objects/s recovering 2024-09-12T23:46:19.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:19 smithi136 bash[76371]: cluster 2024-09-12T23:46:18.509315+0000 mon.a (mon.0) 670 : cluster [DBG] osdmap e523: 8 total, 8 up, 8 in 2024-09-12T23:46:19.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:19 smithi136 bash[76371]: cluster 2024-09-12T23:46:18.743232+0000 mgr.y (mgr.24661) 213 : cluster [DBG] pgmap v360: 121 pgs: 2 activating, 1 active+recovery_wait+degraded, 24 active+undersized, 6 active+undersized+degraded, 88 active+clean; 361 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 335 KiB/s rd, 2.2 MiB/s wr, 5 op/s; 54/594 objects degraded (9.091%); 4.9 MiB/s, 1 objects/s recovering 2024-09-12T23:46:19.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:19 smithi181 bash[67587]: cluster 2024-09-12T23:46:18.509315+0000 mon.a (mon.0) 670 : cluster [DBG] osdmap e523: 8 total, 8 up, 8 in 2024-09-12T23:46:19.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:19 smithi181 bash[67587]: cluster 2024-09-12T23:46:18.743232+0000 mgr.y (mgr.24661) 213 : cluster [DBG] pgmap v360: 121 pgs: 2 activating, 1 active+recovery_wait+degraded, 24 active+undersized, 6 active+undersized+degraded, 88 active+clean; 361 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 335 KiB/s rd, 2.2 MiB/s wr, 5 op/s; 54/594 objects degraded (9.091%); 4.9 MiB/s, 1 objects/s recovering 2024-09-12T23:46:20.535 INFO:tasks.rados.rados.0.smithi136.stdout:2303: done (1 left) 2024-09-12T23:46:20.535 INFO:tasks.rados.rados.0.smithi136.stdout:2304: done (0 left) 2024-09-12T23:46:20.535 INFO:tasks.rados.rados.0.smithi136.stdout:2305: snap_remove snap 192 2024-09-12T23:46:20.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:20 smithi181 bash[67587]: cluster 2024-09-12T23:46:19.528040+0000 mon.a (mon.0) 671 : cluster [DBG] osdmap e524: 8 total, 8 up, 8 in 2024-09-12T23:46:21.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:20 smithi136 bash[75164]: cluster 2024-09-12T23:46:19.528040+0000 mon.a (mon.0) 671 : cluster [DBG] osdmap e524: 8 total, 8 up, 8 in 2024-09-12T23:46:21.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:20 smithi136 bash[76371]: cluster 2024-09-12T23:46:19.528040+0000 mon.a (mon.0) 671 : cluster [DBG] osdmap e524: 8 total, 8 up, 8 in 2024-09-12T23:46:21.548 INFO:tasks.rados.rados.0.smithi136.stdout:2305: done (0 left) 2024-09-12T23:46:21.548 INFO:tasks.rados.rados.0.smithi136.stdout:2306: read oid 8 snap -1 2024-09-12T23:46:21.548 INFO:tasks.rados.rados.0.smithi136.stdout:2306: expect deleted 2024-09-12T23:46:21.549 INFO:tasks.rados.rados.0.smithi136.stdout:2307: read oid 35 snap -1 2024-09-12T23:46:21.549 INFO:tasks.rados.rados.0.smithi136.stdout:2307: expect (ObjNum 671 snap 196 seq_num 671) 2024-09-12T23:46:21.549 INFO:tasks.rados.rados.0.smithi136.stdout:2308: read oid 6 snap -1 2024-09-12T23:46:21.549 INFO:tasks.rados.rados.0.smithi136.stdout:2308: expect deleted 2024-09-12T23:46:21.549 INFO:tasks.rados.rados.0.smithi136.stdout:2309: snap_remove snap 224 2024-09-12T23:46:21.835 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:21 smithi181 bash[67587]: cluster 2024-09-12T23:46:20.539170+0000 mon.a (mon.0) 672 : cluster [DBG] osdmap e525: 8 total, 8 up, 8 in 2024-09-12T23:46:21.835 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:21 smithi181 bash[67587]: cluster 2024-09-12T23:46:20.743963+0000 mgr.y (mgr.24661) 214 : cluster [DBG] pgmap v363: 121 pgs: 2 activating, 1 active+recovery_wait+degraded, 118 active+clean; 352 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 472 KiB/s wr, 7 op/s; 13/570 objects degraded (2.281%); 12 MiB/s, 3 objects/s recovering 2024-09-12T23:46:21.835 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:21 smithi181 bash[67587]: cluster 2024-09-12T23:46:21.358879+0000 mon.a (mon.0) 673 : cluster [WRN] Health check update: Degraded data redundancy: 54/594 objects degraded (9.091%), 7 pgs degraded (PG_DEGRADED) 2024-09-12T23:46:22.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:21 smithi136 bash[75164]: cluster 2024-09-12T23:46:20.539170+0000 mon.a (mon.0) 672 : cluster [DBG] osdmap e525: 8 total, 8 up, 8 in 2024-09-12T23:46:22.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:21 smithi136 bash[75164]: cluster 2024-09-12T23:46:20.743963+0000 mgr.y (mgr.24661) 214 : cluster [DBG] pgmap v363: 121 pgs: 2 activating, 1 active+recovery_wait+degraded, 118 active+clean; 352 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 472 KiB/s wr, 7 op/s; 13/570 objects degraded (2.281%); 12 MiB/s, 3 objects/s recovering 2024-09-12T23:46:22.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:21 smithi136 bash[75164]: cluster 2024-09-12T23:46:21.358879+0000 mon.a (mon.0) 673 : cluster [WRN] Health check update: Degraded data redundancy: 54/594 objects degraded (9.091%), 7 pgs degraded (PG_DEGRADED) 2024-09-12T23:46:22.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:21 smithi136 bash[76371]: cluster 2024-09-12T23:46:20.539170+0000 mon.a (mon.0) 672 : cluster [DBG] osdmap e525: 8 total, 8 up, 8 in 2024-09-12T23:46:22.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:21 smithi136 bash[76371]: cluster 2024-09-12T23:46:20.743963+0000 mgr.y (mgr.24661) 214 : cluster [DBG] pgmap v363: 121 pgs: 2 activating, 1 active+recovery_wait+degraded, 118 active+clean; 352 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 472 KiB/s wr, 7 op/s; 13/570 objects degraded (2.281%); 12 MiB/s, 3 objects/s recovering 2024-09-12T23:46:22.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:21 smithi136 bash[76371]: cluster 2024-09-12T23:46:21.358879+0000 mon.a (mon.0) 673 : cluster [WRN] Health check update: Degraded data redundancy: 54/594 objects degraded (9.091%), 7 pgs degraded (PG_DEGRADED) 2024-09-12T23:46:22.560 INFO:tasks.rados.rados.0.smithi136.stdout:2309: done (3 left) 2024-09-12T23:46:22.560 INFO:tasks.rados.rados.0.smithi136.stdout:2310: snap_create 2024-09-12T23:46:22.560 INFO:tasks.rados.rados.0.smithi136.stdout:2307: expect (ObjNum 671 snap 196 seq_num 671) 2024-09-12T23:46:22.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:22 smithi181 bash[67587]: cluster 2024-09-12T23:46:21.551518+0000 mon.a (mon.0) 674 : cluster [DBG] osdmap e526: 8 total, 8 up, 8 in 2024-09-12T23:46:22.594 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:46:22 smithi181 bash[60471]: ts=2024-09-12T23:46:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:46:23.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:22 smithi136 bash[75164]: cluster 2024-09-12T23:46:21.551518+0000 mon.a (mon.0) 674 : cluster [DBG] osdmap e526: 8 total, 8 up, 8 in 2024-09-12T23:46:23.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:22 smithi136 bash[76371]: cluster 2024-09-12T23:46:21.551518+0000 mon.a (mon.0) 674 : cluster [DBG] osdmap e526: 8 total, 8 up, 8 in 2024-09-12T23:46:23.565 INFO:tasks.rados.rados.0.smithi136.stdout:2306: done (3 left) 2024-09-12T23:46:23.566 INFO:tasks.rados.rados.0.smithi136.stdout:2307: done (2 left) 2024-09-12T23:46:23.566 INFO:tasks.rados.rados.0.smithi136.stdout:2308: done (1 left) 2024-09-12T23:46:23.566 INFO:tasks.rados.rados.0.smithi136.stdout:2310: done (0 left) 2024-09-12T23:46:23.566 INFO:tasks.rados.rados.0.smithi136.stdout:2311: snap_remove snap 226 2024-09-12T23:46:23.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:23 smithi181 bash[67587]: cluster 2024-09-12T23:46:22.560838+0000 mon.a (mon.0) 675 : cluster [DBG] osdmap e527: 8 total, 8 up, 8 in 2024-09-12T23:46:23.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:23 smithi181 bash[67587]: cluster 2024-09-12T23:46:22.744629+0000 mgr.y (mgr.24661) 215 : cluster [DBG] pgmap v366: 121 pgs: 121 active+clean; 344 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 870 KiB/s wr, 8 op/s; 23 MiB/s, 5 objects/s recovering 2024-09-12T23:46:23.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:23 smithi181 bash[67587]: audit 2024-09-12T23:46:23.534203+0000 mon.a (mon.0) 676 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:23.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:23 smithi181 bash[67587]: audit 2024-09-12T23:46:23.544561+0000 mon.a (mon.0) 677 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:23.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:23 smithi181 bash[67587]: audit 2024-09-12T23:46:23.546530+0000 mon.a (mon.0) 678 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:46:23.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:23 smithi181 bash[67587]: audit 2024-09-12T23:46:23.548257+0000 mon.a (mon.0) 679 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:46:24.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:23 smithi136 bash[75164]: cluster 2024-09-12T23:46:22.560838+0000 mon.a (mon.0) 675 : cluster [DBG] osdmap e527: 8 total, 8 up, 8 in 2024-09-12T23:46:24.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:23 smithi136 bash[75164]: cluster 2024-09-12T23:46:22.744629+0000 mgr.y (mgr.24661) 215 : cluster [DBG] pgmap v366: 121 pgs: 121 active+clean; 344 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 870 KiB/s wr, 8 op/s; 23 MiB/s, 5 objects/s recovering 2024-09-12T23:46:24.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:23 smithi136 bash[75164]: audit 2024-09-12T23:46:23.534203+0000 mon.a (mon.0) 676 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:24.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:23 smithi136 bash[75164]: audit 2024-09-12T23:46:23.544561+0000 mon.a (mon.0) 677 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:24.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:23 smithi136 bash[75164]: audit 2024-09-12T23:46:23.546530+0000 mon.a (mon.0) 678 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:46:24.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:23 smithi136 bash[75164]: audit 2024-09-12T23:46:23.548257+0000 mon.a (mon.0) 679 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:46:24.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:23 smithi136 bash[76371]: cluster 2024-09-12T23:46:22.560838+0000 mon.a (mon.0) 675 : cluster [DBG] osdmap e527: 8 total, 8 up, 8 in 2024-09-12T23:46:24.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:23 smithi136 bash[76371]: cluster 2024-09-12T23:46:22.744629+0000 mgr.y (mgr.24661) 215 : cluster [DBG] pgmap v366: 121 pgs: 121 active+clean; 344 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 870 KiB/s wr, 8 op/s; 23 MiB/s, 5 objects/s recovering 2024-09-12T23:46:24.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:23 smithi136 bash[76371]: audit 2024-09-12T23:46:23.534203+0000 mon.a (mon.0) 676 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:24.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:23 smithi136 bash[76371]: audit 2024-09-12T23:46:23.544561+0000 mon.a (mon.0) 677 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:24.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:23 smithi136 bash[76371]: audit 2024-09-12T23:46:23.546530+0000 mon.a (mon.0) 678 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:46:24.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:23 smithi136 bash[76371]: audit 2024-09-12T23:46:23.548257+0000 mon.a (mon.0) 679 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:46:24.569 INFO:tasks.rados.rados.0.smithi136.stdout:2311: done (0 left) 2024-09-12T23:46:24.569 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 6 current snap is 231 2024-09-12T23:46:24.569 INFO:tasks.rados.rados.0.smithi136.stdout:2312: seq_num 750 ranges {0=565248} 2024-09-12T23:46:24.581 INFO:tasks.rados.rados.0.smithi136.stdout:2312: writing smithi13667081-6 from 0 to 565248 tid 1 2024-09-12T23:46:24.583 INFO:tasks.rados.rados.0.smithi136.stdout:2313: copy_from oid 20 from oid 45 current snap is 231 2024-09-12T23:46:24.584 INFO:tasks.rados.rados.0.smithi136.stdout:2314: rollback oid 39 current snap is 231 2024-09-12T23:46:24.584 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 39 to 229 2024-09-12T23:46:24.585 INFO:tasks.rados.rados.0.smithi136.stdout:2315: read oid 30 snap 222 2024-09-12T23:46:24.585 INFO:tasks.rados.rados.0.smithi136.stdout:2315: expect (ObjNum -52082624 snap 21878 seq_num 4242944288) 2024-09-12T23:46:24.585 INFO:tasks.rados.rados.0.smithi136.stdout:2316: read oid 8 snap -1 2024-09-12T23:46:24.585 INFO:tasks.rados.rados.0.smithi136.stdout:2316: expect deleted 2024-09-12T23:46:24.585 INFO:tasks.rados.rados.0.smithi136.stdout:2317: read oid 11 snap -1 2024-09-12T23:46:24.585 INFO:tasks.rados.rados.0.smithi136.stdout:2317: expect (ObjNum 605 snap 174 seq_num 605) 2024-09-12T23:46:24.585 INFO:tasks.rados.rados.0.smithi136.stdout:2318: snap_create 2024-09-12T23:46:24.588 INFO:tasks.rados.rados.0.smithi136.stdout:2315: expect (ObjNum -52082624 snap 21878 seq_num 4242944288) 2024-09-12T23:46:24.591 INFO:tasks.rados.rados.0.smithi136.stdout:2314: finishing rollback tid 0 to smithi13667081-39 2024-09-12T23:46:24.592 INFO:tasks.rados.rados.0.smithi136.stdout:2314: finishing rollback tid 1 to smithi13667081-39 2024-09-12T23:46:24.592 INFO:tasks.rados.rados.0.smithi136.stdout:2314: finishing rollback tid 2 to smithi13667081-39 2024-09-12T23:46:24.592 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 39 v 856 (ObjNum 651 snap 191 seq_num 651) dirty exists 2024-09-12T23:46:24.594 INFO:tasks.rados.rados.0.smithi136.stdout:2312: finishing write tid 1 to smithi13667081-6 2024-09-12T23:46:24.599 INFO:tasks.rados.rados.0.smithi136.stdout:2312: finishing write tid 2 to smithi13667081-6 2024-09-12T23:46:24.599 INFO:tasks.rados.rados.0.smithi136.stdout:2312: finishing write tid 3 to smithi13667081-6 2024-09-12T23:46:24.600 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 6 v 1224 (ObjNum 750 snap 231 seq_num 750) dirty exists 2024-09-12T23:46:24.600 INFO:tasks.rados.rados.0.smithi136.stdout:2312: left oid 6 (ObjNum 750 snap 231 seq_num 750) 2024-09-12T23:46:24.628 INFO:tasks.rados.rados.0.smithi136.stdout:2317: expect (ObjNum 605 snap 174 seq_num 605) 2024-09-12T23:46:24.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:24 smithi181 bash[67587]: audit 2024-09-12T23:46:23.558250+0000 mon.a (mon.0) 680 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:24.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:24 smithi181 bash[67587]: cluster 2024-09-12T23:46:23.559717+0000 mon.a (mon.0) 681 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 13/570 objects degraded (2.281%), 1 pg degraded) 2024-09-12T23:46:24.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:24 smithi181 bash[67587]: cluster 2024-09-12T23:46:23.559758+0000 mon.a (mon.0) 682 : cluster [INF] Cluster is now healthy 2024-09-12T23:46:24.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:24 smithi181 bash[67587]: cluster 2024-09-12T23:46:23.567337+0000 mon.a (mon.0) 683 : cluster [DBG] osdmap e528: 8 total, 8 up, 8 in 2024-09-12T23:46:24.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:24 smithi181 bash[67587]: audit 2024-09-12T23:46:23.698427+0000 mon.a (mon.0) 684 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:46:24.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:24 smithi181 bash[67587]: audit 2024-09-12T23:46:23.703727+0000 mon.a (mon.0) 685 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:46:24.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:24 smithi181 bash[67587]: audit 2024-09-12T23:46:23.706881+0000 mon.a (mon.0) 686 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:46:24.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:24 smithi181 bash[67587]: audit 2024-09-12T23:46:23.709139+0000 mon.a (mon.0) 687 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:46:24.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:24 smithi181 bash[67587]: audit 2024-09-12T23:46:23.712028+0000 mon.a (mon.0) 688 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-12T23:46:24.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:24 smithi181 bash[67587]: audit 2024-09-12T23:46:23.712423+0000 mgr.y (mgr.24661) 216 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-12T23:46:24.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:24 smithi181 bash[67587]: cephadm 2024-09-12T23:46:23.714624+0000 mgr.y (mgr.24661) 217 : cephadm [INF] Upgrade: osd.5 is safe to restart 2024-09-12T23:46:24.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:24 smithi181 bash[67587]: audit 2024-09-12T23:46:23.753130+0000 mon.a (mon.0) 689 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:24.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:24 smithi181 bash[67587]: audit 2024-09-12T23:46:23.755285+0000 mon.a (mon.0) 690 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:46:25.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:24 smithi136 bash[75164]: audit 2024-09-12T23:46:23.558250+0000 mon.a (mon.0) 680 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:25.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:24 smithi136 bash[75164]: cluster 2024-09-12T23:46:23.559717+0000 mon.a (mon.0) 681 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 13/570 objects degraded (2.281%), 1 pg degraded) 2024-09-12T23:46:25.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:24 smithi136 bash[75164]: cluster 2024-09-12T23:46:23.559758+0000 mon.a (mon.0) 682 : cluster [INF] Cluster is now healthy 2024-09-12T23:46:25.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:24 smithi136 bash[75164]: cluster 2024-09-12T23:46:23.567337+0000 mon.a (mon.0) 683 : cluster [DBG] osdmap e528: 8 total, 8 up, 8 in 2024-09-12T23:46:25.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:24 smithi136 bash[75164]: audit 2024-09-12T23:46:23.698427+0000 mon.a (mon.0) 684 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:46:25.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:24 smithi136 bash[75164]: audit 2024-09-12T23:46:23.703727+0000 mon.a (mon.0) 685 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:46:25.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:24 smithi136 bash[75164]: audit 2024-09-12T23:46:23.706881+0000 mon.a (mon.0) 686 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:46:25.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:24 smithi136 bash[75164]: audit 2024-09-12T23:46:23.709139+0000 mon.a (mon.0) 687 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:46:25.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:24 smithi136 bash[75164]: audit 2024-09-12T23:46:23.712028+0000 mon.a (mon.0) 688 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-12T23:46:25.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:24 smithi136 bash[75164]: audit 2024-09-12T23:46:23.712423+0000 mgr.y (mgr.24661) 216 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-12T23:46:25.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:24 smithi136 bash[75164]: cephadm 2024-09-12T23:46:23.714624+0000 mgr.y (mgr.24661) 217 : cephadm [INF] Upgrade: osd.5 is safe to restart 2024-09-12T23:46:25.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:24 smithi136 bash[75164]: audit 2024-09-12T23:46:23.753130+0000 mon.a (mon.0) 689 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:25.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:24 smithi136 bash[75164]: audit 2024-09-12T23:46:23.755285+0000 mon.a (mon.0) 690 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:46:25.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:24 smithi136 bash[76371]: audit 2024-09-12T23:46:23.558250+0000 mon.a (mon.0) 680 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:25.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:24 smithi136 bash[76371]: cluster 2024-09-12T23:46:23.559717+0000 mon.a (mon.0) 681 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 13/570 objects degraded (2.281%), 1 pg degraded) 2024-09-12T23:46:25.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:24 smithi136 bash[76371]: cluster 2024-09-12T23:46:23.559758+0000 mon.a (mon.0) 682 : cluster [INF] Cluster is now healthy 2024-09-12T23:46:25.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:24 smithi136 bash[76371]: cluster 2024-09-12T23:46:23.567337+0000 mon.a (mon.0) 683 : cluster [DBG] osdmap e528: 8 total, 8 up, 8 in 2024-09-12T23:46:25.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:24 smithi136 bash[76371]: audit 2024-09-12T23:46:23.698427+0000 mon.a (mon.0) 684 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:46:25.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:24 smithi136 bash[76371]: audit 2024-09-12T23:46:23.703727+0000 mon.a (mon.0) 685 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:46:25.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:24 smithi136 bash[76371]: audit 2024-09-12T23:46:23.706881+0000 mon.a (mon.0) 686 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:46:25.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:24 smithi136 bash[76371]: audit 2024-09-12T23:46:23.709139+0000 mon.a (mon.0) 687 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:46:25.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:24 smithi136 bash[76371]: audit 2024-09-12T23:46:23.712028+0000 mon.a (mon.0) 688 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-12T23:46:25.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:24 smithi136 bash[76371]: audit 2024-09-12T23:46:23.712423+0000 mgr.y (mgr.24661) 216 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-12T23:46:25.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:24 smithi136 bash[76371]: cephadm 2024-09-12T23:46:23.714624+0000 mgr.y (mgr.24661) 217 : cephadm [INF] Upgrade: osd.5 is safe to restart 2024-09-12T23:46:25.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:24 smithi136 bash[76371]: audit 2024-09-12T23:46:23.753130+0000 mon.a (mon.0) 689 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:25.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:24 smithi136 bash[76371]: audit 2024-09-12T23:46:23.755285+0000 mon.a (mon.0) 690 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:46:25.194 INFO:tasks.rados.rados.0.smithi136.stdout:2313: finishing copy_from racing read to smithi13667081-20 2024-09-12T23:46:25.194 INFO:tasks.rados.rados.0.smithi136.stdout:2313: finishing copy_from to smithi13667081-20 2024-09-12T23:46:25.194 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 20 v 1228 (ObjNum 666 snap 195 seq_num 666) dirty exists 2024-09-12T23:46:25.593 INFO:tasks.rados.rados.0.smithi136.stdout:2312: done (6 left) 2024-09-12T23:46:25.594 INFO:tasks.rados.rados.0.smithi136.stdout:2313: done (5 left) 2024-09-12T23:46:25.594 INFO:tasks.rados.rados.0.smithi136.stdout:2314: done (4 left) 2024-09-12T23:46:25.594 INFO:tasks.rados.rados.0.smithi136.stdout:2315: done (3 left) 2024-09-12T23:46:25.594 INFO:tasks.rados.rados.0.smithi136.stdout:2316: done (2 left) 2024-09-12T23:46:25.594 INFO:tasks.rados.rados.0.smithi136.stdout:2317: done (1 left) 2024-09-12T23:46:25.594 INFO:tasks.rados.rados.0.smithi136.stdout:2318: done (0 left) 2024-09-12T23:46:25.594 INFO:tasks.rados.rados.0.smithi136.stdout:2319: read oid 15 snap 211 2024-09-12T23:46:25.594 INFO:tasks.rados.rados.0.smithi136.stdout:2319: expect (ObjNum 631 snap 185 seq_num 631) 2024-09-12T23:46:25.594 INFO:tasks.rados.rados.0.smithi136.stdout:2320: rmattr oid 42 current snap is 232 2024-09-12T23:46:25.594 INFO:tasks.rados.rados.0.smithi136.stdout:2320: done (1 left) 2024-09-12T23:46:25.594 INFO:tasks.rados.rados.0.smithi136.stdout:2321: delete oid 7 current snap is 232 2024-09-12T23:46:25.604 INFO:tasks.rados.rados.0.smithi136.stdout:2321: done (1 left) 2024-09-12T23:46:25.604 INFO:tasks.rados.rados.0.smithi136.stdout:2322: copy_from oid 28 from oid 40 current snap is 232 2024-09-12T23:46:25.604 INFO:tasks.rados.rados.0.smithi136.stdout:2323: rollback oid 45 current snap is 232 2024-09-12T23:46:25.604 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 45 to 222 2024-09-12T23:46:25.604 INFO:tasks.rados.rados.0.smithi136.stdout:2324: copy_from oid 50 from oid 30 current snap is 232 2024-09-12T23:46:25.604 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 32 current snap is 232 2024-09-12T23:46:25.604 INFO:tasks.rados.rados.0.smithi136.stdout:2325: seq_num 752 ranges {2842624=598016} 2024-09-12T23:46:25.656 INFO:tasks.rados.rados.0.smithi136.stdout:2325: writing smithi13667081-32 from 2842624 to 3440640 tid 1 2024-09-12T23:46:25.656 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 41 current snap is 232 2024-09-12T23:46:25.656 INFO:tasks.rados.rados.0.smithi136.stdout:2326: seq_num 753 ranges {2383872=614400} 2024-09-12T23:46:25.686 INFO:tasks.rados.rados.0.smithi136.stdout:2326: writing smithi13667081-41 from 2383872 to 2998272 tid 1 2024-09-12T23:46:25.686 INFO:tasks.rados.rados.0.smithi136.stdout:2327: read oid 47 snap -1 2024-09-12T23:46:25.686 INFO:tasks.rados.rados.0.smithi136.stdout:2327: expect (ObjNum 731 snap 226 seq_num 731) 2024-09-12T23:46:25.686 INFO:tasks.rados.rados.0.smithi136.stdout:2328: rmattr oid 7 current snap is 232 2024-09-12T23:46:25.686 INFO:tasks.rados.rados.0.smithi136.stdout:2328: done (7 left) 2024-09-12T23:46:25.686 INFO:tasks.rados.rados.0.smithi136.stdout:2329: snap_create 2024-09-12T23:46:25.686 INFO:tasks.rados.rados.0.smithi136.stdout:2323: finishing rollback tid 0 to smithi13667081-45 2024-09-12T23:46:25.686 INFO:tasks.rados.rados.0.smithi136.stdout:2323: finishing rollback tid 1 to smithi13667081-45 2024-09-12T23:46:25.686 INFO:tasks.rados.rados.0.smithi136.stdout:2323: finishing rollback tid 2 to smithi13667081-45 2024-09-12T23:46:25.686 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 45 v 1231 (ObjNum 706 snap 220 seq_num 706) dirty exists 2024-09-12T23:46:25.686 INFO:tasks.rados.rados.0.smithi136.stdout:2319: expect (ObjNum 631 snap 185 seq_num 631) 2024-09-12T23:46:25.766 INFO:tasks.rados.rados.0.smithi136.stdout:2324: finishing copy_from racing read to smithi13667081-50 2024-09-12T23:46:25.766 INFO:tasks.rados.rados.0.smithi136.stdout:2324: finishing copy_from to smithi13667081-50 2024-09-12T23:46:25.766 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 50 v 1189 (ObjNum 734 snap 226 seq_num 734) dirty exists 2024-09-12T23:46:25.766 INFO:tasks.rados.rados.0.smithi136.stdout:2322: finishing copy_from racing read to smithi13667081-28 2024-09-12T23:46:25.766 INFO:tasks.rados.rados.0.smithi136.stdout:2322: finishing copy_from to smithi13667081-28 2024-09-12T23:46:25.766 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 28 v 1581 (ObjNum 739 snap 227 seq_num 739) dirty exists 2024-09-12T23:46:25.766 INFO:tasks.rados.rados.0.smithi136.stdout:2325: finishing write tid 1 to smithi13667081-32 2024-09-12T23:46:25.766 INFO:tasks.rados.rados.0.smithi136.stdout:2325: finishing write tid 2 to smithi13667081-32 2024-09-12T23:46:25.766 INFO:tasks.rados.rados.0.smithi136.stdout:2325: finishing write tid 3 to smithi13667081-32 2024-09-12T23:46:25.766 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 32 v 792 (ObjNum 752 snap 232 seq_num 752) dirty exists 2024-09-12T23:46:25.767 INFO:tasks.rados.rados.0.smithi136.stdout:2325: left oid 32 (ObjNum 752 snap 232 seq_num 752) 2024-09-12T23:46:25.767 INFO:tasks.rados.rados.0.smithi136.stdout:2326: finishing write tid 1 to smithi13667081-41 2024-09-12T23:46:25.767 INFO:tasks.rados.rados.0.smithi136.stdout:2326: finishing write tid 2 to smithi13667081-41 2024-09-12T23:46:25.767 INFO:tasks.rados.rados.0.smithi136.stdout:2326: finishing write tid 3 to smithi13667081-41 2024-09-12T23:46:25.767 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 41 v 1255 (ObjNum 753 snap 232 seq_num 753) dirty exists 2024-09-12T23:46:25.767 INFO:tasks.rados.rados.0.smithi136.stdout:2326: left oid 41 (ObjNum 753 snap 232 seq_num 753) 2024-09-12T23:46:25.767 INFO:tasks.rados.rados.0.smithi136.stdout:2327: expect (ObjNum 731 snap 226 seq_num 731) 2024-09-12T23:46:25.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:25 smithi181 bash[67587]: cluster 2024-09-12T23:46:24.570730+0000 mon.a (mon.0) 691 : cluster [DBG] osdmap e529: 8 total, 8 up, 8 in 2024-09-12T23:46:25.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:25 smithi181 bash[67587]: cluster 2024-09-12T23:46:24.745628+0000 mgr.y (mgr.24661) 218 : cluster [DBG] pgmap v369: 121 pgs: 121 active+clean; 341 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.6 MiB/s rd, 500 KiB/s wr, 9 op/s; 16 MiB/s, 3 objects/s recovering 2024-09-12T23:46:26.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:25 smithi136 bash[75164]: cluster 2024-09-12T23:46:24.570730+0000 mon.a (mon.0) 691 : cluster [DBG] osdmap e529: 8 total, 8 up, 8 in 2024-09-12T23:46:26.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:25 smithi136 bash[75164]: cluster 2024-09-12T23:46:24.745628+0000 mgr.y (mgr.24661) 218 : cluster [DBG] pgmap v369: 121 pgs: 121 active+clean; 341 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.6 MiB/s rd, 500 KiB/s wr, 9 op/s; 16 MiB/s, 3 objects/s recovering 2024-09-12T23:46:26.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:25 smithi136 bash[76371]: cluster 2024-09-12T23:46:24.570730+0000 mon.a (mon.0) 691 : cluster [DBG] osdmap e529: 8 total, 8 up, 8 in 2024-09-12T23:46:26.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:25 smithi136 bash[76371]: cluster 2024-09-12T23:46:24.745628+0000 mgr.y (mgr.24661) 218 : cluster [DBG] pgmap v369: 121 pgs: 121 active+clean; 341 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.6 MiB/s rd, 500 KiB/s wr, 9 op/s; 16 MiB/s, 3 objects/s recovering 2024-09-12T23:46:26.366 INFO:tasks.rados.rados.0.smithi136.stdout:2319: done (7 left) 2024-09-12T23:46:26.366 INFO:tasks.rados.rados.0.smithi136.stdout:2322: done (6 left) 2024-09-12T23:46:26.366 INFO:tasks.rados.rados.0.smithi136.stdout:2323: done (5 left) 2024-09-12T23:46:26.366 INFO:tasks.rados.rados.0.smithi136.stdout:2324: done (4 left) 2024-09-12T23:46:26.367 INFO:tasks.rados.rados.0.smithi136.stdout:2325: done (3 left) 2024-09-12T23:46:26.367 INFO:tasks.rados.rados.0.smithi136.stdout:2326: done (2 left) 2024-09-12T23:46:26.367 INFO:tasks.rados.rados.0.smithi136.stdout:2327: done (1 left) 2024-09-12T23:46:26.367 INFO:tasks.rados.rados.0.smithi136.stdout:2329: done (0 left) 2024-09-12T23:46:26.367 INFO:tasks.rados.rados.0.smithi136.stdout:2330: read oid 44 snap 232 2024-09-12T23:46:26.367 INFO:tasks.rados.rados.0.smithi136.stdout:2330: expect (ObjNum 713 snap 221 seq_num 713) 2024-09-12T23:46:26.367 INFO:tasks.rados.rados.0.smithi136.stdout:2331: delete oid 39 current snap is 233 2024-09-12T23:46:26.374 INFO:tasks.rados.rados.0.smithi136.stdout:2331: done (1 left) 2024-09-12T23:46:26.375 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 26 current snap is 233 2024-09-12T23:46:26.375 INFO:tasks.rados.rados.0.smithi136.stdout:2332: seq_num 755 ranges {7454720=647168} 2024-09-12T23:46:26.438 INFO:tasks.rados.rados.0.smithi136.stdout:2332: writing smithi13667081-26 from 7454720 to 8101888 tid 1 2024-09-12T23:46:26.438 INFO:tasks.rados.rados.0.smithi136.stdout:2333: snap_remove snap 231 2024-09-12T23:46:26.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:26 smithi181 bash[67587]: cluster 2024-09-12T23:46:25.596879+0000 mon.a (mon.0) 692 : cluster [DBG] osdmap e530: 8 total, 8 up, 8 in 2024-09-12T23:46:26.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:26 smithi181 bash[67587]: cephadm 2024-09-12T23:46:25.785803+0000 mgr.y (mgr.24661) 219 : cephadm [INF] Upgrade: Updating osd.5 2024-09-12T23:46:26.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:26 smithi181 bash[67587]: audit 2024-09-12T23:46:25.794798+0000 mon.a (mon.0) 693 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:26.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:26 smithi181 bash[67587]: audit 2024-09-12T23:46:25.796545+0000 mon.a (mon.0) 694 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-12T23:46:26.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:26 smithi181 bash[67587]: audit 2024-09-12T23:46:25.798099+0000 mon.a (mon.0) 695 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:46:26.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:26 smithi181 bash[67587]: cephadm 2024-09-12T23:46:25.804010+0000 mgr.y (mgr.24661) 220 : cephadm [INF] Deploying daemon osd.5 on smithi181 2024-09-12T23:46:26.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:26 smithi181 bash[67587]: cluster 2024-09-12T23:46:26.369747+0000 mon.a (mon.0) 696 : cluster [DBG] osdmap e531: 8 total, 8 up, 8 in 2024-09-12T23:46:27.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:26 smithi136 bash[75164]: cluster 2024-09-12T23:46:25.596879+0000 mon.a (mon.0) 692 : cluster [DBG] osdmap e530: 8 total, 8 up, 8 in 2024-09-12T23:46:27.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:26 smithi136 bash[75164]: cephadm 2024-09-12T23:46:25.785803+0000 mgr.y (mgr.24661) 219 : cephadm [INF] Upgrade: Updating osd.5 2024-09-12T23:46:27.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:26 smithi136 bash[75164]: audit 2024-09-12T23:46:25.794798+0000 mon.a (mon.0) 693 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:27.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:26 smithi136 bash[75164]: audit 2024-09-12T23:46:25.796545+0000 mon.a (mon.0) 694 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-12T23:46:27.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:26 smithi136 bash[75164]: audit 2024-09-12T23:46:25.798099+0000 mon.a (mon.0) 695 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:46:27.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:26 smithi136 bash[75164]: cephadm 2024-09-12T23:46:25.804010+0000 mgr.y (mgr.24661) 220 : cephadm [INF] Deploying daemon osd.5 on smithi181 2024-09-12T23:46:27.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:26 smithi136 bash[75164]: cluster 2024-09-12T23:46:26.369747+0000 mon.a (mon.0) 696 : cluster [DBG] osdmap e531: 8 total, 8 up, 8 in 2024-09-12T23:46:27.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:26 smithi136 bash[76371]: cluster 2024-09-12T23:46:25.596879+0000 mon.a (mon.0) 692 : cluster [DBG] osdmap e530: 8 total, 8 up, 8 in 2024-09-12T23:46:27.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:26 smithi136 bash[76371]: cephadm 2024-09-12T23:46:25.785803+0000 mgr.y (mgr.24661) 219 : cephadm [INF] Upgrade: Updating osd.5 2024-09-12T23:46:27.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:26 smithi136 bash[76371]: audit 2024-09-12T23:46:25.794798+0000 mon.a (mon.0) 693 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:27.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:26 smithi136 bash[76371]: audit 2024-09-12T23:46:25.796545+0000 mon.a (mon.0) 694 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-12T23:46:27.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:26 smithi136 bash[76371]: audit 2024-09-12T23:46:25.798099+0000 mon.a (mon.0) 695 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:46:27.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:26 smithi136 bash[76371]: cephadm 2024-09-12T23:46:25.804010+0000 mgr.y (mgr.24661) 220 : cephadm [INF] Deploying daemon osd.5 on smithi181 2024-09-12T23:46:27.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:26 smithi136 bash[76371]: cluster 2024-09-12T23:46:26.369747+0000 mon.a (mon.0) 696 : cluster [DBG] osdmap e531: 8 total, 8 up, 8 in 2024-09-12T23:46:27.370 INFO:tasks.rados.rados.0.smithi136.stdout:2333: done (2 left) 2024-09-12T23:46:27.370 INFO:tasks.rados.rados.0.smithi136.stdout:2334: read oid 10 snap -1 2024-09-12T23:46:27.370 INFO:tasks.rados.rados.0.smithi136.stdout:2334: expect (ObjNum 669 snap 196 seq_num 669) 2024-09-12T23:46:27.371 INFO:tasks.rados.rados.0.smithi136.stdout:2335: read oid 23 snap -1 2024-09-12T23:46:27.371 INFO:tasks.rados.rados.0.smithi136.stdout:2335: expect (ObjNum 741 snap 228 seq_num 741) 2024-09-12T23:46:27.371 INFO:tasks.rados.rados.0.smithi136.stdout:2336: rmattr oid 39 current snap is 233 2024-09-12T23:46:27.371 INFO:tasks.rados.rados.0.smithi136.stdout:2330: expect (ObjNum 713 snap 221 seq_num 713) 2024-09-12T23:46:27.451 INFO:tasks.rados.rados.0.smithi136.stdout:2332: finishing write tid 1 to smithi13667081-26 2024-09-12T23:46:27.452 INFO:tasks.rados.rados.0.smithi136.stdout:2332: finishing write tid 2 to smithi13667081-26 2024-09-12T23:46:27.452 INFO:tasks.rados.rados.0.smithi136.stdout:2332: finishing write tid 3 to smithi13667081-26 2024-09-12T23:46:27.452 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 26 v 1256 (ObjNum 755 snap 233 seq_num 755) dirty exists 2024-09-12T23:46:27.452 INFO:tasks.rados.rados.0.smithi136.stdout:2332: left oid 26 (ObjNum 755 snap 233 seq_num 755) 2024-09-12T23:46:27.452 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 39 v 0 (ObjNum 129 snap 0 seq_num 4243165856) dirty dne 2024-09-12T23:46:27.452 INFO:tasks.rados.rados.0.smithi136.stdout:2334: expect (ObjNum 669 snap 196 seq_num 669) 2024-09-12T23:46:27.716 INFO:tasks.rados.rados.0.smithi136.stdout:2335: expect (ObjNum 741 snap 228 seq_num 741) 2024-09-12T23:46:27.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:27 smithi181 bash[67587]: cluster 2024-09-12T23:46:26.746545+0000 mgr.y (mgr.24661) 221 : cluster [DBG] pgmap v372: 121 pgs: 121 active+clean; 345 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.9 MiB/s rd, 1.9 MiB/s wr, 13 op/s 2024-09-12T23:46:27.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:27 smithi181 bash[67587]: cluster 2024-09-12T23:46:27.373488+0000 mon.a (mon.0) 697 : cluster [DBG] osdmap e532: 8 total, 8 up, 8 in 2024-09-12T23:46:27.919 INFO:tasks.rados.rados.0.smithi136.stdout:2330: done (4 left) 2024-09-12T23:46:27.919 INFO:tasks.rados.rados.0.smithi136.stdout:2332: done (3 left) 2024-09-12T23:46:27.919 INFO:tasks.rados.rados.0.smithi136.stdout:2334: done (2 left) 2024-09-12T23:46:27.919 INFO:tasks.rados.rados.0.smithi136.stdout:2335: done (1 left) 2024-09-12T23:46:27.919 INFO:tasks.rados.rados.0.smithi136.stdout:2336: done (0 left) 2024-09-12T23:46:27.919 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 7 current snap is 233 2024-09-12T23:46:27.920 INFO:tasks.rados.rados.0.smithi136.stdout:2337: seq_num 756 ranges {0=671744} 2024-09-12T23:46:27.925 INFO:tasks.rados.rados.0.smithi136.stdout:2337: writing smithi13667081-7 from 0 to 671744 tid 1 2024-09-12T23:46:27.925 INFO:tasks.rados.rados.0.smithi136.stdout:2338: delete oid 25 current snap is 233 2024-09-12T23:46:27.931 INFO:tasks.rados.rados.0.smithi136.stdout:2338: done (1 left) 2024-09-12T23:46:27.931 INFO:tasks.rados.rados.0.smithi136.stdout:2339: setattr oid 45 current snap is 233 2024-09-12T23:46:27.932 INFO:tasks.rados.rados.0.smithi136.stdout:2337: finishing write tid 1 to smithi13667081-7 2024-09-12T23:46:27.933 INFO:tasks.rados.rados.0.smithi136.stdout:2337: finishing write tid 2 to smithi13667081-7 2024-09-12T23:46:27.933 INFO:tasks.rados.rados.0.smithi136.stdout:2340: read oid 2 snap -1 2024-09-12T23:46:27.934 INFO:tasks.rados.rados.0.smithi136.stdout:2340: expect deleted 2024-09-12T23:46:27.934 INFO:tasks.rados.rados.0.smithi136.stdout:2341: copy_from oid 6 from oid 21 current snap is 233 2024-09-12T23:46:27.934 INFO:tasks.rados.rados.0.smithi136.stdout:2342: read oid 38 snap -1 2024-09-12T23:46:27.934 INFO:tasks.rados.rados.0.smithi136.stdout:2342: expect (ObjNum 727 snap 224 seq_num 727) 2024-09-12T23:46:27.934 INFO:tasks.rados.rados.0.smithi136.stdout:2343: snap_create 2024-09-12T23:46:27.935 INFO:tasks.rados.rados.0.smithi136.stdout:2341: finishing copy_from to smithi13667081-6 2024-09-12T23:46:27.935 INFO:tasks.rados.rados.0.smithi136.stdout:2341: got expected ENOENT (src dne) 2024-09-12T23:46:27.936 INFO:tasks.rados.rados.0.smithi136.stdout:2341: finishing copy_from racing read to smithi13667081-6 2024-09-12T23:46:27.936 INFO:tasks.rados.rados.0.smithi136.stdout:2337: finishing write tid 3 to smithi13667081-7 2024-09-12T23:46:27.936 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 7 v 944 (ObjNum 756 snap 233 seq_num 756) dirty exists 2024-09-12T23:46:27.936 INFO:tasks.rados.rados.0.smithi136.stdout:2337: left oid 7 (ObjNum 756 snap 233 seq_num 756) 2024-09-12T23:46:27.941 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 45 v 1232 (ObjNum 706 snap 220 seq_num 706) dirty exists 2024-09-12T23:46:27.972 INFO:tasks.rados.rados.0.smithi136.stdout:2342: expect (ObjNum 727 snap 224 seq_num 727) 2024-09-12T23:46:28.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:27 smithi136 bash[75164]: cluster 2024-09-12T23:46:26.746545+0000 mgr.y (mgr.24661) 221 : cluster [DBG] pgmap v372: 121 pgs: 121 active+clean; 345 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.9 MiB/s rd, 1.9 MiB/s wr, 13 op/s 2024-09-12T23:46:28.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:27 smithi136 bash[75164]: cluster 2024-09-12T23:46:27.373488+0000 mon.a (mon.0) 697 : cluster [DBG] osdmap e532: 8 total, 8 up, 8 in 2024-09-12T23:46:28.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:27 smithi136 bash[76371]: cluster 2024-09-12T23:46:26.746545+0000 mgr.y (mgr.24661) 221 : cluster [DBG] pgmap v372: 121 pgs: 121 active+clean; 345 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.9 MiB/s rd, 1.9 MiB/s wr, 13 op/s 2024-09-12T23:46:28.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:27 smithi136 bash[76371]: cluster 2024-09-12T23:46:27.373488+0000 mon.a (mon.0) 697 : cluster [DBG] osdmap e532: 8 total, 8 up, 8 in 2024-09-12T23:46:28.603 INFO:tasks.rados.rados.0.smithi136.stdout:2337: done (5 left) 2024-09-12T23:46:28.603 INFO:tasks.rados.rados.0.smithi136.stdout:2339: done (4 left) 2024-09-12T23:46:28.603 INFO:tasks.rados.rados.0.smithi136.stdout:2340: done (3 left) 2024-09-12T23:46:28.603 INFO:tasks.rados.rados.0.smithi136.stdout:2341: done (2 left) 2024-09-12T23:46:28.603 INFO:tasks.rados.rados.0.smithi136.stdout:2342: done (1 left) 2024-09-12T23:46:28.603 INFO:tasks.rados.rados.0.smithi136.stdout:2343: done (0 left) 2024-09-12T23:46:28.603 INFO:tasks.rados.rados.0.smithi136.stdout:2344: snap_create 2024-09-12T23:46:29.619 INFO:tasks.rados.rados.0.smithi136.stdout:2344: done (0 left) 2024-09-12T23:46:29.619 INFO:tasks.rados.rados.0.smithi136.stdout:2345: delete oid 45 current snap is 235 2024-09-12T23:46:29.630 INFO:tasks.rados.rados.0.smithi136.stdout:2345: done (0 left) 2024-09-12T23:46:29.630 INFO:tasks.rados.rados.0.smithi136.stdout:2346: read oid 40 snap -1 2024-09-12T23:46:29.630 INFO:tasks.rados.rados.0.smithi136.stdout:2346: expect (ObjNum 739 snap 227 seq_num 739) 2024-09-12T23:46:29.630 INFO:tasks.rados.rados.0.smithi136.stdout:2347: snap_remove snap 232 2024-09-12T23:46:29.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:46:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:46:29] "GET /metrics HTTP/1.1" 200 36400 "" "Prometheus/2.43.0" 2024-09-12T23:46:29.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:29 smithi136 bash[76371]: cluster 2024-09-12T23:46:28.606016+0000 mon.a (mon.0) 698 : cluster [DBG] osdmap e533: 8 total, 8 up, 8 in 2024-09-12T23:46:29.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:29 smithi136 bash[76371]: cluster 2024-09-12T23:46:28.747219+0000 mgr.y (mgr.24661) 222 : cluster [DBG] pgmap v375: 121 pgs: 121 active+clean; 345 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.9 MiB/s wr, 8 op/s 2024-09-12T23:46:29.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:29 smithi136 bash[76371]: audit 2024-09-12T23:46:28.900215+0000 mon.a (mon.0) 699 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:29.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:29 smithi136 bash[75164]: cluster 2024-09-12T23:46:28.606016+0000 mon.a (mon.0) 698 : cluster [DBG] osdmap e533: 8 total, 8 up, 8 in 2024-09-12T23:46:29.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:29 smithi136 bash[75164]: cluster 2024-09-12T23:46:28.747219+0000 mgr.y (mgr.24661) 222 : cluster [DBG] pgmap v375: 121 pgs: 121 active+clean; 345 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.9 MiB/s wr, 8 op/s 2024-09-12T23:46:29.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:29 smithi136 bash[75164]: audit 2024-09-12T23:46:28.900215+0000 mon.a (mon.0) 699 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:29.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:29 smithi181 bash[67587]: cluster 2024-09-12T23:46:28.606016+0000 mon.a (mon.0) 698 : cluster [DBG] osdmap e533: 8 total, 8 up, 8 in 2024-09-12T23:46:30.278 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:29 smithi181 bash[67587]: cluster 2024-09-12T23:46:28.747219+0000 mgr.y (mgr.24661) 222 : cluster [DBG] pgmap v375: 121 pgs: 121 active+clean; 345 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.9 MiB/s wr, 8 op/s 2024-09-12T23:46:30.278 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:29 smithi181 bash[67587]: audit 2024-09-12T23:46:28.900215+0000 mon.a (mon.0) 699 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:30.392 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:30 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:46:30.392 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:46:30 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:46:30.392 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:46:30 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:46:30.393 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:46:30 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:46:30.393 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 12 23:46:30 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:46:30.393 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 12 23:46:30 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:46:30.394 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:46:30 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:46:30.394 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:46:30 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:46:30.394 INFO:journalctl@ceph.mds.b.smithi181.stdout:Sep 12 23:46:30 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:46:30.394 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:46:30 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:46:30.623 INFO:tasks.rados.rados.0.smithi136.stdout:2347: done (1 left) 2024-09-12T23:46:30.624 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 36 current snap is 235 2024-09-12T23:46:30.632 INFO:tasks.rados.rados.0.smithi136.stdout:2346: expect (ObjNum 739 snap 227 seq_num 739) 2024-09-12T23:46:30.643 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:46:30 smithi181 systemd[1]: Stopping Ceph osd.5 for da58a47c-715d-11ef-bceb-c7b262605968... 2024-09-12T23:46:30.643 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:46:30 smithi181 bash[33658]: debug 2024-09-12T23:46:30.503+0000 7f908f8d1640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-12T23:46:30.643 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:46:30 smithi181 bash[33658]: debug 2024-09-12T23:46:30.503+0000 7f908f8d1640 -1 osd.5 534 *** Got signal Terminated *** 2024-09-12T23:46:30.644 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:46:30 smithi181 bash[33658]: debug 2024-09-12T23:46:30.503+0000 7f908f8d1640 -1 osd.5 534 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-12T23:46:30.663 INFO:tasks.rados.rados.0.smithi136.stdout:2348: seq_num 759 ranges {622592=720896} 2024-09-12T23:46:30.680 INFO:tasks.rados.rados.0.smithi136.stdout:2348: writing smithi13667081-36 from 622592 to 1343488 tid 1 2024-09-12T23:46:30.680 INFO:tasks.rados.rados.0.smithi136.stdout:2346: done (1 left) 2024-09-12T23:46:30.680 INFO:tasks.rados.rados.0.smithi136.stdout:2349: delete oid 20 current snap is 235 2024-09-12T23:46:30.685 INFO:tasks.rados.rados.0.smithi136.stdout:2349: done (1 left) 2024-09-12T23:46:30.685 INFO:tasks.rados.rados.0.smithi136.stdout:2350: read oid 9 snap -1 2024-09-12T23:46:30.685 INFO:tasks.rados.rados.0.smithi136.stdout:2350: expect deleted 2024-09-12T23:46:30.685 INFO:tasks.rados.rados.0.smithi136.stdout:2351: delete oid 22 current snap is 235 2024-09-12T23:46:30.689 INFO:tasks.rados.rados.0.smithi136.stdout:2350: done (2 left) 2024-09-12T23:46:30.689 INFO:tasks.rados.rados.0.smithi136.stdout:2351: done (1 left) 2024-09-12T23:46:30.689 INFO:tasks.rados.rados.0.smithi136.stdout:2352: delete oid 44 current snap is 235 2024-09-12T23:46:30.692 INFO:tasks.rados.rados.0.smithi136.stdout:2352: done (1 left) 2024-09-12T23:46:30.692 INFO:tasks.rados.rados.0.smithi136.stdout:2353: read oid 32 snap 234 2024-09-12T23:46:30.692 INFO:tasks.rados.rados.0.smithi136.stdout:2353: expect (ObjNum 752 snap 232 seq_num 752) 2024-09-12T23:46:30.692 INFO:tasks.rados.rados.0.smithi136.stdout:2354: read oid 4 snap -1 2024-09-12T23:46:30.692 INFO:tasks.rados.rados.0.smithi136.stdout:2354: expect (ObjNum 736 snap 226 seq_num 736) 2024-09-12T23:46:30.693 INFO:tasks.rados.rados.0.smithi136.stdout:2355: delete oid 17 current snap is 235 2024-09-12T23:46:30.695 INFO:tasks.rados.rados.0.smithi136.stdout:2355: done (3 left) 2024-09-12T23:46:30.695 INFO:tasks.rados.rados.0.smithi136.stdout:2356: rollback oid 48 current snap is 235 2024-09-12T23:46:30.695 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 48 to 234 2024-09-12T23:46:30.695 INFO:tasks.rados.rados.0.smithi136.stdout:2357: snap_create 2024-09-12T23:46:30.699 INFO:tasks.rados.rados.0.smithi136.stdout:2356: finishing rollback tid 0 to smithi13667081-48 2024-09-12T23:46:30.699 INFO:tasks.rados.rados.0.smithi136.stdout:2356: finishing rollback tid 1 to smithi13667081-48 2024-09-12T23:46:30.699 INFO:tasks.rados.rados.0.smithi136.stdout:2356: finishing rollback tid 2 to smithi13667081-48 2024-09-12T23:46:30.700 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 48 v 1585 (ObjNum 717 snap 223 seq_num 717) dirty exists 2024-09-12T23:46:31.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:30 smithi136 bash[75164]: cluster 2024-09-12T23:46:29.622449+0000 mon.a (mon.0) 700 : cluster [DBG] osdmap e534: 8 total, 8 up, 8 in 2024-09-12T23:46:31.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:30 smithi136 bash[75164]: cluster 2024-09-12T23:46:30.507331+0000 mon.a (mon.0) 701 : cluster [INF] osd.5 marked itself down and dead 2024-09-12T23:46:31.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:30 smithi136 bash[76371]: cluster 2024-09-12T23:46:29.622449+0000 mon.a (mon.0) 700 : cluster [DBG] osdmap e534: 8 total, 8 up, 8 in 2024-09-12T23:46:31.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:30 smithi136 bash[76371]: cluster 2024-09-12T23:46:30.507331+0000 mon.a (mon.0) 701 : cluster [INF] osd.5 marked itself down and dead 2024-09-12T23:46:31.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:30 smithi181 bash[67587]: cluster 2024-09-12T23:46:29.622449+0000 mon.a (mon.0) 700 : cluster [DBG] osdmap e534: 8 total, 8 up, 8 in 2024-09-12T23:46:31.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:30 smithi181 bash[67587]: cluster 2024-09-12T23:46:30.507331+0000 mon.a (mon.0) 701 : cluster [INF] osd.5 marked itself down and dead 2024-09-12T23:46:31.371 INFO:tasks.rados.rados.0.smithi136.stdout:2356: done (4 left) 2024-09-12T23:46:31.371 INFO:tasks.rados.rados.0.smithi136.stdout:2357: done (3 left) 2024-09-12T23:46:31.371 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 48 current snap is 236 2024-09-12T23:46:31.371 INFO:tasks.rados.rados.0.smithi136.stdout:2358: seq_num 764 ranges {1581056=802816} 2024-09-12T23:46:31.413 INFO:tasks.rados.rados.0.smithi136.stdout:2358: writing smithi13667081-48 from 1581056 to 2383872 tid 1 2024-09-12T23:46:31.414 INFO:tasks.rados.rados.0.smithi136.stdout:2359: delete oid 50 current snap is 236 2024-09-12T23:46:31.414 INFO:tasks.rados.rados.0.smithi136.stdout:2348: finishing write tid 1 to smithi13667081-36 2024-09-12T23:46:31.414 INFO:tasks.rados.rados.0.smithi136.stdout:2348: finishing write tid 2 to smithi13667081-36 2024-09-12T23:46:31.414 INFO:tasks.rados.rados.0.smithi136.stdout:2348: finishing write tid 3 to smithi13667081-36 2024-09-12T23:46:31.414 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 36 v 1235 (ObjNum 759 snap 235 seq_num 759) dirty exists 2024-09-12T23:46:31.414 INFO:tasks.rados.rados.0.smithi136.stdout:2348: left oid 36 (ObjNum 759 snap 235 seq_num 759) 2024-09-12T23:46:31.419 INFO:tasks.rados.rados.0.smithi136.stdout:2348: done (4 left) 2024-09-12T23:46:31.419 INFO:tasks.rados.rados.0.smithi136.stdout:2359: done (3 left) 2024-09-12T23:46:31.419 INFO:tasks.rados.rados.0.smithi136.stdout:2360: snap_remove snap 227 2024-09-12T23:46:31.974 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:46:31 smithi181 bash[75871]: ceph-da58a47c-715d-11ef-bceb-c7b262605968-osd-5 2024-09-12T23:46:31.974 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:31 smithi181 bash[67587]: cluster 2024-09-12T23:46:30.611758+0000 mon.a (mon.0) 702 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-12T23:46:31.974 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:31 smithi181 bash[67587]: cluster 2024-09-12T23:46:30.623220+0000 mon.a (mon.0) 703 : cluster [DBG] osdmap e535: 8 total, 7 up, 8 in 2024-09-12T23:46:31.974 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:31 smithi181 bash[67587]: cluster 2024-09-12T23:46:30.747933+0000 mgr.y (mgr.24661) 223 : cluster [DBG] pgmap v378: 121 pgs: 18 stale+active+clean, 103 active+clean; 357 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 382 KiB/s wr, 14 op/s 2024-09-12T23:46:31.974 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:31 smithi181 bash[67587]: cluster 2024-09-12T23:46:31.374004+0000 mon.a (mon.0) 704 : cluster [DBG] osdmap e536: 8 total, 7 up, 8 in 2024-09-12T23:46:32.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:31 smithi136 bash[75164]: cluster 2024-09-12T23:46:30.611758+0000 mon.a (mon.0) 702 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-12T23:46:32.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:31 smithi136 bash[75164]: cluster 2024-09-12T23:46:30.623220+0000 mon.a (mon.0) 703 : cluster [DBG] osdmap e535: 8 total, 7 up, 8 in 2024-09-12T23:46:32.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:31 smithi136 bash[75164]: cluster 2024-09-12T23:46:30.747933+0000 mgr.y (mgr.24661) 223 : cluster [DBG] pgmap v378: 121 pgs: 18 stale+active+clean, 103 active+clean; 357 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 382 KiB/s wr, 14 op/s 2024-09-12T23:46:32.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:31 smithi136 bash[75164]: cluster 2024-09-12T23:46:31.374004+0000 mon.a (mon.0) 704 : cluster [DBG] osdmap e536: 8 total, 7 up, 8 in 2024-09-12T23:46:32.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:31 smithi136 bash[76371]: cluster 2024-09-12T23:46:30.611758+0000 mon.a (mon.0) 702 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-12T23:46:32.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:31 smithi136 bash[76371]: cluster 2024-09-12T23:46:30.623220+0000 mon.a (mon.0) 703 : cluster [DBG] osdmap e535: 8 total, 7 up, 8 in 2024-09-12T23:46:32.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:31 smithi136 bash[76371]: cluster 2024-09-12T23:46:30.747933+0000 mgr.y (mgr.24661) 223 : cluster [DBG] pgmap v378: 121 pgs: 18 stale+active+clean, 103 active+clean; 357 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 382 KiB/s wr, 14 op/s 2024-09-12T23:46:32.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:31 smithi136 bash[76371]: cluster 2024-09-12T23:46:31.374004+0000 mon.a (mon.0) 704 : cluster [DBG] osdmap e536: 8 total, 7 up, 8 in 2024-09-12T23:46:32.309 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:46:31 smithi181 bash[75937]: ceph-da58a47c-715d-11ef-bceb-c7b262605968-osd-5 2024-09-12T23:46:32.373 INFO:tasks.rados.rados.0.smithi136.stdout:2360: done (3 left) 2024-09-12T23:46:32.374 INFO:tasks.rados.rados.0.smithi136.stdout:2361: read oid 44 snap 194 2024-09-12T23:46:32.374 INFO:tasks.rados.rados.0.smithi136.stdout:2361: expect (ObjNum 564 snap 162 seq_num 564) 2024-09-12T23:46:32.374 INFO:tasks.rados.rados.0.smithi136.stdout:2362: rmattr oid 1 current snap is 236 2024-09-12T23:46:32.374 INFO:tasks.rados.rados.0.smithi136.stdout:2362: done (4 left) 2024-09-12T23:46:32.374 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 27 current snap is 236 2024-09-12T23:46:32.374 INFO:tasks.rados.rados.0.smithi136.stdout:2363: seq_num 766 ranges {7618560=434176,8052736=401408} 2024-09-12T23:46:32.464 INFO:tasks.rados.rados.0.smithi136.stdout:2363: writing smithi13667081-27 from 7618560 to 8052736 tid 1 2024-09-12T23:46:32.467 INFO:tasks.rados.rados.0.smithi136.stdout:2363: writing smithi13667081-27 from 8052736 to 8454144 tid 2 2024-09-12T23:46:32.467 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 9 current snap is 236 2024-09-12T23:46:32.467 INFO:tasks.rados.rados.0.smithi136.stdout:2358: finishing write tid 1 to smithi13667081-48 2024-09-12T23:46:32.467 INFO:tasks.rados.rados.0.smithi136.stdout:2364: seq_num 767 ranges {0=450560,450560=401408} 2024-09-12T23:46:32.469 INFO:tasks.rados.rados.0.smithi136.stdout:2364: writing smithi13667081-9 from 0 to 450560 tid 1 2024-09-12T23:46:32.473 INFO:tasks.rados.rados.0.smithi136.stdout:2364: writing smithi13667081-9 from 450560 to 851968 tid 2 2024-09-12T23:46:32.473 INFO:tasks.rados.rados.0.smithi136.stdout:2365: read oid 45 snap -1 2024-09-12T23:46:32.473 INFO:tasks.rados.rados.0.smithi136.stdout:2365: expect deleted 2024-09-12T23:46:32.473 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 10 current snap is 236 2024-09-12T23:46:32.473 INFO:tasks.rados.rados.0.smithi136.stdout:2366: seq_num 768 ranges {5668864=466944,6135808=401408} 2024-09-12T23:46:32.514 INFO:tasks.rados.rados.0.smithi136.stdout:2366: writing smithi13667081-10 from 5668864 to 6135808 tid 1 2024-09-12T23:46:32.516 INFO:tasks.rados.rados.0.smithi136.stdout:2366: writing smithi13667081-10 from 6135808 to 6537216 tid 2 2024-09-12T23:46:32.517 INFO:tasks.rados.rados.0.smithi136.stdout:2367: copy_from oid 15 from oid 1 current snap is 236 2024-09-12T23:46:32.517 INFO:tasks.rados.rados.0.smithi136.stdout:2358: finishing write tid 2 to smithi13667081-48 2024-09-12T23:46:32.517 INFO:tasks.rados.rados.0.smithi136.stdout:2353: expect (ObjNum 752 snap 232 seq_num 752) 2024-09-12T23:46:32.594 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:46:32 smithi181 bash[60471]: ts=2024-09-12T23:46:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:46:32.807 INFO:tasks.rados.rados.0.smithi136.stdout:2358: finishing write tid 3 to smithi13667081-48 2024-09-12T23:46:32.807 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 48 v 1587 (ObjNum 764 snap 236 seq_num 764) dirty exists 2024-09-12T23:46:32.807 INFO:tasks.rados.rados.0.smithi136.stdout:2358: left oid 48 (ObjNum 764 snap 236 seq_num 764) 2024-09-12T23:46:32.807 INFO:tasks.rados.rados.0.smithi136.stdout:2354: expect (ObjNum 736 snap 226 seq_num 736) 2024-09-12T23:46:33.268 INFO:tasks.rados.rados.0.smithi136.stdout:2361: expect (ObjNum 564 snap 162 seq_num 564) 2024-09-12T23:46:33.326 INFO:tasks.rados.rados.0.smithi136.stdout:2363: finishing write tid 1 to smithi13667081-27 2024-09-12T23:46:33.327 INFO:tasks.rados.rados.0.smithi136.stdout:2363: finishing write tid 2 to smithi13667081-27 2024-09-12T23:46:33.327 INFO:tasks.rados.rados.0.smithi136.stdout:2363: finishing write tid 3 to smithi13667081-27 2024-09-12T23:46:33.327 INFO:tasks.rados.rados.0.smithi136.stdout:2364: finishing write tid 1 to smithi13667081-9 2024-09-12T23:46:33.327 INFO:tasks.rados.rados.0.smithi136.stdout:2364: finishing write tid 2 to smithi13667081-9 2024-09-12T23:46:33.327 INFO:tasks.rados.rados.0.smithi136.stdout:2364: finishing write tid 3 to smithi13667081-9 2024-09-12T23:46:33.328 INFO:tasks.rados.rados.0.smithi136.stdout:2363: finishing write tid 4 to smithi13667081-27 2024-09-12T23:46:33.328 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 27 v 1643 (ObjNum 766 snap 236 seq_num 766) dirty exists 2024-09-12T23:46:33.328 INFO:tasks.rados.rados.0.smithi136.stdout:2363: left oid 27 (ObjNum 766 snap 236 seq_num 766) 2024-09-12T23:46:33.328 INFO:tasks.rados.rados.0.smithi136.stdout:2364: finishing write tid 4 to smithi13667081-9 2024-09-12T23:46:33.329 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 9 v 745 (ObjNum 767 snap 236 seq_num 767) dirty exists 2024-09-12T23:46:33.329 INFO:tasks.rados.rados.0.smithi136.stdout:2364: left oid 9 (ObjNum 767 snap 236 seq_num 767) 2024-09-12T23:46:33.329 INFO:tasks.rados.rados.0.smithi136.stdout:2367: finishing copy_from to smithi13667081-15 2024-09-12T23:46:33.330 INFO:tasks.rados.rados.0.smithi136.stdout:2367: got expected ENOENT (src dne) 2024-09-12T23:46:33.330 INFO:tasks.rados.rados.0.smithi136.stdout:2367: finishing copy_from racing read to smithi13667081-15 2024-09-12T23:46:33.330 INFO:tasks.rados.rados.0.smithi136.stdout:2366: finishing write tid 1 to smithi13667081-10 2024-09-12T23:46:33.330 INFO:tasks.rados.rados.0.smithi136.stdout:2366: finishing write tid 2 to smithi13667081-10 2024-09-12T23:46:33.330 INFO:tasks.rados.rados.0.smithi136.stdout:2366: finishing write tid 3 to smithi13667081-10 2024-09-12T23:46:33.331 INFO:tasks.rados.rados.0.smithi136.stdout:2366: finishing write tid 4 to smithi13667081-10 2024-09-12T23:46:33.332 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 10 v 1193 (ObjNum 768 snap 236 seq_num 768) dirty exists 2024-09-12T23:46:33.332 INFO:tasks.rados.rados.0.smithi136.stdout:2366: left oid 10 (ObjNum 768 snap 236 seq_num 768) 2024-09-12T23:46:33.332 INFO:tasks.rados.rados.0.smithi136.stdout:2353: done (8 left) 2024-09-12T23:46:33.332 INFO:tasks.rados.rados.0.smithi136.stdout:2354: done (7 left) 2024-09-12T23:46:33.333 INFO:tasks.rados.rados.0.smithi136.stdout:2358: done (6 left) 2024-09-12T23:46:33.333 INFO:tasks.rados.rados.0.smithi136.stdout:2361: done (5 left) 2024-09-12T23:46:33.333 INFO:tasks.rados.rados.0.smithi136.stdout:2363: done (4 left) 2024-09-12T23:46:33.333 INFO:tasks.rados.rados.0.smithi136.stdout:2364: done (3 left) 2024-09-12T23:46:33.334 INFO:tasks.rados.rados.0.smithi136.stdout:2365: done (2 left) 2024-09-12T23:46:33.334 INFO:tasks.rados.rados.0.smithi136.stdout:2366: done (1 left) 2024-09-12T23:46:33.334 INFO:tasks.rados.rados.0.smithi136.stdout:2367: done (0 left) 2024-09-12T23:46:33.334 INFO:tasks.rados.rados.0.smithi136.stdout:2368: read oid 22 snap -1 2024-09-12T23:46:33.335 INFO:tasks.rados.rados.0.smithi136.stdout:2368: expect deleted 2024-09-12T23:46:33.335 INFO:tasks.rados.rados.0.smithi136.stdout:2369: snap_create 2024-09-12T23:46:33.380 INFO:tasks.rados.rados.0.smithi136.stdout:2368: done (1 left) 2024-09-12T23:46:33.381 INFO:tasks.rados.rados.0.smithi136.stdout:2369: done (0 left) 2024-09-12T23:46:33.381 INFO:tasks.rados.rados.0.smithi136.stdout:2370: delete oid 16 current snap is 237 2024-09-12T23:46:33.387 INFO:tasks.rados.rados.0.smithi136.stdout:2370: done (0 left) 2024-09-12T23:46:33.387 INFO:tasks.rados.rados.0.smithi136.stdout:2371: rollback oid 28 current snap is 237 2024-09-12T23:46:33.387 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 28 to 236 2024-09-12T23:46:33.387 INFO:tasks.rados.rados.0.smithi136.stdout:2372: copy_from oid 35 from oid 21 current snap is 237 2024-09-12T23:46:33.387 INFO:tasks.rados.rados.0.smithi136.stdout:2373: delete oid 50 current snap is 237 2024-09-12T23:46:33.389 INFO:tasks.rados.rados.0.smithi136.stdout:2373: done (2 left) 2024-09-12T23:46:33.390 INFO:tasks.rados.rados.0.smithi136.stdout:2374: setattr oid 44 current snap is 237 2024-09-12T23:46:33.391 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 26 current snap is 237 2024-09-12T23:46:33.391 INFO:tasks.rados.rados.0.smithi136.stdout:2375: seq_num 771 ranges {8101888=516096,8617984=401408} 2024-09-12T23:46:33.473 INFO:tasks.rados.rados.0.smithi136.stdout:2375: writing smithi13667081-26 from 8101888 to 8617984 tid 1 2024-09-12T23:46:33.475 INFO:tasks.rados.rados.0.smithi136.stdout:2375: writing smithi13667081-26 from 8617984 to 9019392 tid 2 2024-09-12T23:46:33.475 INFO:tasks.rados.rados.0.smithi136.stdout:2376: read oid 49 snap -1 2024-09-12T23:46:33.476 INFO:tasks.rados.rados.0.smithi136.stdout:2376: expect deleted 2024-09-12T23:46:33.476 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 13 current snap is 237 2024-09-12T23:46:33.476 INFO:tasks.rados.rados.0.smithi136.stdout:2371: finishing rollback tid 0 to smithi13667081-28 2024-09-12T23:46:33.476 INFO:tasks.rados.rados.0.smithi136.stdout:2372: finishing copy_from to smithi13667081-35 2024-09-12T23:46:33.476 INFO:tasks.rados.rados.0.smithi136.stdout:2372: got expected ENOENT (src dne) 2024-09-12T23:46:33.476 INFO:tasks.rados.rados.0.smithi136.stdout:2372: finishing copy_from racing read to smithi13667081-35 2024-09-12T23:46:33.476 INFO:tasks.rados.rados.0.smithi136.stdout:2371: finishing rollback tid 1 to smithi13667081-28 2024-09-12T23:46:33.476 INFO:tasks.rados.rados.0.smithi136.stdout:2377: seq_num 772 ranges {0=532480,532480=401408} 2024-09-12T23:46:33.479 INFO:tasks.rados.rados.0.smithi136.stdout:2377: writing smithi13667081-13 from 0 to 532480 tid 1 2024-09-12T23:46:33.482 INFO:tasks.rados.rados.0.smithi136.stdout:2377: writing smithi13667081-13 from 532480 to 933888 tid 2 2024-09-12T23:46:33.483 INFO:tasks.rados.rados.0.smithi136.stdout:2372: done (5 left) 2024-09-12T23:46:33.483 INFO:tasks.rados.rados.0.smithi136.stdout:2378: snap_create 2024-09-12T23:46:33.483 INFO:tasks.rados.rados.0.smithi136.stdout:2371: finishing rollback tid 2 to smithi13667081-28 2024-09-12T23:46:33.483 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 28 v 1591 (ObjNum 739 snap 227 seq_num 739) dirty exists 2024-09-12T23:46:33.483 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 44 v 1592 (ObjNum 4113 snap 0 seq_num 1676963906) dirty exists 2024-09-12T23:46:33.483 INFO:tasks.rados.rados.0.smithi136.stdout:2375: finishing write tid 1 to smithi13667081-26 2024-09-12T23:46:33.483 INFO:tasks.rados.rados.0.smithi136.stdout:2375: finishing write tid 2 to smithi13667081-26 2024-09-12T23:46:33.483 INFO:tasks.rados.rados.0.smithi136.stdout:2375: finishing write tid 3 to smithi13667081-26 2024-09-12T23:46:33.485 INFO:tasks.rados.rados.0.smithi136.stdout:2375: finishing write tid 4 to smithi13667081-26 2024-09-12T23:46:33.485 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 26 v 1259 (ObjNum 771 snap 237 seq_num 771) dirty exists 2024-09-12T23:46:33.485 INFO:tasks.rados.rados.0.smithi136.stdout:2375: left oid 26 (ObjNum 771 snap 237 seq_num 771) 2024-09-12T23:46:33.490 INFO:tasks.rados.rados.0.smithi136.stdout:2377: finishing write tid 1 to smithi13667081-13 2024-09-12T23:46:33.490 INFO:tasks.rados.rados.0.smithi136.stdout:2377: finishing write tid 2 to smithi13667081-13 2024-09-12T23:46:33.491 INFO:tasks.rados.rados.0.smithi136.stdout:2377: finishing write tid 3 to smithi13667081-13 2024-09-12T23:46:33.493 INFO:tasks.rados.rados.0.smithi136.stdout:2377: finishing write tid 4 to smithi13667081-13 2024-09-12T23:46:33.493 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 13 v 1242 (ObjNum 772 snap 237 seq_num 772) dirty exists 2024-09-12T23:46:33.493 INFO:tasks.rados.rados.0.smithi136.stdout:2377: left oid 13 (ObjNum 772 snap 237 seq_num 772) 2024-09-12T23:46:33.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:33 smithi181 bash[67587]: cluster 2024-09-12T23:46:32.376168+0000 mon.a (mon.0) 705 : cluster [DBG] osdmap e537: 8 total, 7 up, 8 in 2024-09-12T23:46:33.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:33 smithi181 bash[67587]: cluster 2024-09-12T23:46:32.748713+0000 mgr.y (mgr.24661) 224 : cluster [DBG] pgmap v381: 121 pgs: 8 active+undersized, 15 stale+active+clean, 5 active+undersized+degraded, 93 active+clean; 358 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 578 KiB/s wr, 15 op/s; 25/540 objects degraded (4.630%) 2024-09-12T23:46:33.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:33 smithi136 bash[75164]: cluster 2024-09-12T23:46:32.376168+0000 mon.a (mon.0) 705 : cluster [DBG] osdmap e537: 8 total, 7 up, 8 in 2024-09-12T23:46:33.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:33 smithi136 bash[75164]: cluster 2024-09-12T23:46:32.748713+0000 mgr.y (mgr.24661) 224 : cluster [DBG] pgmap v381: 121 pgs: 8 active+undersized, 15 stale+active+clean, 5 active+undersized+degraded, 93 active+clean; 358 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 578 KiB/s wr, 15 op/s; 25/540 objects degraded (4.630%) 2024-09-12T23:46:33.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:33 smithi136 bash[76371]: cluster 2024-09-12T23:46:32.376168+0000 mon.a (mon.0) 705 : cluster [DBG] osdmap e537: 8 total, 7 up, 8 in 2024-09-12T23:46:33.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:33 smithi136 bash[76371]: cluster 2024-09-12T23:46:32.748713+0000 mgr.y (mgr.24661) 224 : cluster [DBG] pgmap v381: 121 pgs: 8 active+undersized, 15 stale+active+clean, 5 active+undersized+degraded, 93 active+clean; 358 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 578 KiB/s wr, 15 op/s; 25/540 objects degraded (4.630%) 2024-09-12T23:46:34.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:34 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:46:34.270 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:46:34 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:46:34.270 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:46:34 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:46:34.271 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:46:34 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:46:34.271 INFO:journalctl@ceph.mds.b.smithi181.stdout:Sep 12 23:46:34 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:46:34.271 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 12 23:46:34 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:46:34.271 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 12 23:46:34 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:46:34.272 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:46:34 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:46:34.272 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:46:33 smithi181 systemd[1]: ceph-da58a47c-715d-11ef-bceb-c7b262605968@osd.5.service: Deactivated successfully. 2024-09-12T23:46:34.272 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:46:33 smithi181 systemd[1]: Stopped Ceph osd.5 for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:46:34.273 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:46:34 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:46:34.273 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:46:34 smithi181 systemd[1]: Started Ceph osd.5 for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:46:34.273 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:46:34 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:46:34.386 INFO:tasks.rados.rados.0.smithi136.stdout:2371: done (5 left) 2024-09-12T23:46:34.386 INFO:tasks.rados.rados.0.smithi136.stdout:2374: done (4 left) 2024-09-12T23:46:34.386 INFO:tasks.rados.rados.0.smithi136.stdout:2375: done (3 left) 2024-09-12T23:46:34.386 INFO:tasks.rados.rados.0.smithi136.stdout:2376: done (2 left) 2024-09-12T23:46:34.386 INFO:tasks.rados.rados.0.smithi136.stdout:2377: done (1 left) 2024-09-12T23:46:34.386 INFO:tasks.rados.rados.0.smithi136.stdout:2378: done (0 left) 2024-09-12T23:46:34.386 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 6 current snap is 238 2024-09-12T23:46:34.386 INFO:tasks.rados.rados.0.smithi136.stdout:2379: seq_num 773 ranges {565248=548864,1114112=401408} 2024-09-12T23:46:34.394 INFO:tasks.rados.rados.0.smithi136.stdout:2379: writing smithi13667081-6 from 565248 to 1114112 tid 1 2024-09-12T23:46:34.398 INFO:tasks.rados.rados.0.smithi136.stdout:2379: writing smithi13667081-6 from 1114112 to 1515520 tid 2 2024-09-12T23:46:34.398 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 36 current snap is 238 2024-09-12T23:46:34.398 INFO:tasks.rados.rados.0.smithi136.stdout:2380: seq_num 774 ranges {1343488=565248,1908736=401408} 2024-09-12T23:46:34.413 INFO:tasks.rados.rados.0.smithi136.stdout:2380: writing smithi13667081-36 from 1343488 to 1908736 tid 1 2024-09-12T23:46:34.416 INFO:tasks.rados.rados.0.smithi136.stdout:2380: writing smithi13667081-36 from 1908736 to 2310144 tid 2 2024-09-12T23:46:34.416 INFO:tasks.rados.rados.0.smithi136.stdout:2381: snap_create 2024-09-12T23:46:34.416 INFO:tasks.rados.rados.0.smithi136.stdout:2379: finishing write tid 1 to smithi13667081-6 2024-09-12T23:46:34.417 INFO:tasks.rados.rados.0.smithi136.stdout:2379: finishing write tid 2 to smithi13667081-6 2024-09-12T23:46:34.417 INFO:tasks.rados.rados.0.smithi136.stdout:2379: finishing write tid 3 to smithi13667081-6 2024-09-12T23:46:34.417 INFO:tasks.rados.rados.0.smithi136.stdout:2379: finishing write tid 4 to smithi13667081-6 2024-09-12T23:46:34.417 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 6 v 1232 (ObjNum 773 snap 238 seq_num 773) dirty exists 2024-09-12T23:46:34.417 INFO:tasks.rados.rados.0.smithi136.stdout:2379: left oid 6 (ObjNum 773 snap 238 seq_num 773) 2024-09-12T23:46:34.418 INFO:tasks.rados.rados.0.smithi136.stdout:2380: finishing write tid 1 to smithi13667081-36 2024-09-12T23:46:34.419 INFO:tasks.rados.rados.0.smithi136.stdout:2380: finishing write tid 2 to smithi13667081-36 2024-09-12T23:46:34.419 INFO:tasks.rados.rados.0.smithi136.stdout:2380: finishing write tid 3 to smithi13667081-36 2024-09-12T23:46:34.421 INFO:tasks.rados.rados.0.smithi136.stdout:2380: finishing write tid 4 to smithi13667081-36 2024-09-12T23:46:34.421 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 36 v 1238 (ObjNum 774 snap 238 seq_num 774) dirty exists 2024-09-12T23:46:34.421 INFO:tasks.rados.rados.0.smithi136.stdout:2380: left oid 36 (ObjNum 774 snap 238 seq_num 774) 2024-09-12T23:46:34.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:34 smithi181 bash[67587]: cluster 2024-09-12T23:46:33.369608+0000 mon.a (mon.0) 706 : cluster [WRN] Health check failed: Degraded data redundancy: 25/540 objects degraded (4.630%), 5 pgs degraded (PG_DEGRADED) 2024-09-12T23:46:34.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:34 smithi181 bash[67587]: cluster 2024-09-12T23:46:33.384380+0000 mon.a (mon.0) 707 : cluster [DBG] osdmap e538: 8 total, 7 up, 8 in 2024-09-12T23:46:34.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:34 smithi181 bash[67587]: audit 2024-09-12T23:46:34.315841+0000 mon.a (mon.0) 708 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:34.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:34 smithi181 bash[67587]: audit 2024-09-12T23:46:34.326125+0000 mon.a (mon.0) 709 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:34.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:34 smithi181 bash[67587]: audit 2024-09-12T23:46:34.328323+0000 mon.a (mon.0) 710 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:46:34.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:34 smithi136 bash[75164]: cluster 2024-09-12T23:46:33.369608+0000 mon.a (mon.0) 706 : cluster [WRN] Health check failed: Degraded data redundancy: 25/540 objects degraded (4.630%), 5 pgs degraded (PG_DEGRADED) 2024-09-12T23:46:34.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:34 smithi136 bash[75164]: cluster 2024-09-12T23:46:33.384380+0000 mon.a (mon.0) 707 : cluster [DBG] osdmap e538: 8 total, 7 up, 8 in 2024-09-12T23:46:34.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:34 smithi136 bash[75164]: audit 2024-09-12T23:46:34.315841+0000 mon.a (mon.0) 708 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:34.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:34 smithi136 bash[75164]: audit 2024-09-12T23:46:34.326125+0000 mon.a (mon.0) 709 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:34.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:34 smithi136 bash[75164]: audit 2024-09-12T23:46:34.328323+0000 mon.a (mon.0) 710 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:46:34.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:34 smithi136 bash[76371]: cluster 2024-09-12T23:46:33.369608+0000 mon.a (mon.0) 706 : cluster [WRN] Health check failed: Degraded data redundancy: 25/540 objects degraded (4.630%), 5 pgs degraded (PG_DEGRADED) 2024-09-12T23:46:34.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:34 smithi136 bash[76371]: cluster 2024-09-12T23:46:33.384380+0000 mon.a (mon.0) 707 : cluster [DBG] osdmap e538: 8 total, 7 up, 8 in 2024-09-12T23:46:34.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:34 smithi136 bash[76371]: audit 2024-09-12T23:46:34.315841+0000 mon.a (mon.0) 708 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:34.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:34 smithi136 bash[76371]: audit 2024-09-12T23:46:34.326125+0000 mon.a (mon.0) 709 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:34.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:34 smithi136 bash[76371]: audit 2024-09-12T23:46:34.328323+0000 mon.a (mon.0) 710 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:46:35.410 INFO:tasks.rados.rados.0.smithi136.stdout:2379: done (2 left) 2024-09-12T23:46:35.410 INFO:tasks.rados.rados.0.smithi136.stdout:2380: done (1 left) 2024-09-12T23:46:35.410 INFO:tasks.rados.rados.0.smithi136.stdout:2381: done (0 left) 2024-09-12T23:46:35.410 INFO:tasks.rados.rados.0.smithi136.stdout:2382: rmattr oid 11 current snap is 239 2024-09-12T23:46:35.411 INFO:tasks.rados.rados.0.smithi136.stdout:2382: done (0 left) 2024-09-12T23:46:35.411 INFO:tasks.rados.rados.0.smithi136.stdout:2383: read oid 45 snap -1 2024-09-12T23:46:35.411 INFO:tasks.rados.rados.0.smithi136.stdout:2383: expect deleted 2024-09-12T23:46:35.411 INFO:tasks.rados.rados.0.smithi136.stdout:2384: copy_from oid 30 from oid 9 current snap is 239 2024-09-12T23:46:35.411 INFO:tasks.rados.rados.0.smithi136.stdout:2385: snap_create 2024-09-12T23:46:35.431 INFO:tasks.rados.rados.0.smithi136.stdout:2384: finishing copy_from racing read to smithi13667081-30 2024-09-12T23:46:35.434 INFO:tasks.rados.rados.0.smithi136.stdout:2384: finishing copy_from to smithi13667081-30 2024-09-12T23:46:35.434 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 30 v 745 (ObjNum 767 snap 236 seq_num 767) dirty exists 2024-09-12T23:46:35.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:35 smithi136 bash[75164]: cluster 2024-09-12T23:46:34.389122+0000 mon.a (mon.0) 711 : cluster [DBG] osdmap e539: 8 total, 7 up, 8 in 2024-09-12T23:46:35.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:35 smithi136 bash[75164]: cluster 2024-09-12T23:46:34.749839+0000 mgr.y (mgr.24661) 225 : cluster [DBG] pgmap v384: 121 pgs: 27 active+undersized, 6 stale+active+clean, 11 active+undersized+degraded, 77 active+clean; 362 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.1 MiB/s rd, 1.2 MiB/s wr, 10 op/s; 62/558 objects degraded (11.111%) 2024-09-12T23:46:35.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:35 smithi136 bash[76371]: cluster 2024-09-12T23:46:34.389122+0000 mon.a (mon.0) 711 : cluster [DBG] osdmap e539: 8 total, 7 up, 8 in 2024-09-12T23:46:35.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:35 smithi136 bash[76371]: cluster 2024-09-12T23:46:34.749839+0000 mgr.y (mgr.24661) 225 : cluster [DBG] pgmap v384: 121 pgs: 27 active+undersized, 6 stale+active+clean, 11 active+undersized+degraded, 77 active+clean; 362 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.1 MiB/s rd, 1.2 MiB/s wr, 10 op/s; 62/558 objects degraded (11.111%) 2024-09-12T23:46:35.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:35 smithi181 bash[67587]: cluster 2024-09-12T23:46:34.389122+0000 mon.a (mon.0) 711 : cluster [DBG] osdmap e539: 8 total, 7 up, 8 in 2024-09-12T23:46:35.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:35 smithi181 bash[67587]: cluster 2024-09-12T23:46:34.749839+0000 mgr.y (mgr.24661) 225 : cluster [DBG] pgmap v384: 121 pgs: 27 active+undersized, 6 stale+active+clean, 11 active+undersized+degraded, 77 active+clean; 362 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.1 MiB/s rd, 1.2 MiB/s wr, 10 op/s; 62/558 objects degraded (11.111%) 2024-09-12T23:46:36.141 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:46:35 smithi181 bash[76084]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-12T23:46:36.141 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:46:35 smithi181 bash[76084]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-12T23:46:36.141 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:46:35 smithi181 bash[76084]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-12T23:46:36.141 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:46:35 smithi181 bash[76084]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-12T23:46:36.141 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:46:35 smithi181 bash[76084]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-12T23:46:36.141 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:46:35 smithi181 bash[76084]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-12T23:46:36.141 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:46:35 smithi181 bash[76084]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-12T23:46:36.371 INFO:tasks.rados.rados.0.smithi136.stdout:2383: done (2 left) 2024-09-12T23:46:36.372 INFO:tasks.rados.rados.0.smithi136.stdout:2384: done (1 left) 2024-09-12T23:46:36.372 INFO:tasks.rados.rados.0.smithi136.stdout:2385: done (0 left) 2024-09-12T23:46:36.372 INFO:tasks.rados.rados.0.smithi136.stdout:2386: rollback oid 30 current snap is 240 2024-09-12T23:46:36.372 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 30 to 233 2024-09-12T23:46:36.372 INFO:tasks.rados.rados.0.smithi136.stdout:2387: rollback oid 36 current snap is 240 2024-09-12T23:46:36.372 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 36 to 225 2024-09-12T23:46:36.372 INFO:tasks.rados.rados.0.smithi136.stdout:2388: read oid 1 snap -1 2024-09-12T23:46:36.372 INFO:tasks.rados.rados.0.smithi136.stdout:2388: expect deleted 2024-09-12T23:46:36.372 INFO:tasks.rados.rados.0.smithi136.stdout:2389: snap_remove snap 194 2024-09-12T23:46:36.698 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:36 smithi181 bash[67587]: cluster 2024-09-12T23:46:35.413188+0000 mon.a (mon.0) 712 : cluster [DBG] osdmap e540: 8 total, 7 up, 8 in 2024-09-12T23:46:36.699 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:36 smithi181 bash[67587]: cluster 2024-09-12T23:46:36.374655+0000 mon.a (mon.0) 713 : cluster [DBG] osdmap e541: 8 total, 7 up, 8 in 2024-09-12T23:46:36.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:36 smithi136 bash[75164]: cluster 2024-09-12T23:46:35.413188+0000 mon.a (mon.0) 712 : cluster [DBG] osdmap e540: 8 total, 7 up, 8 in 2024-09-12T23:46:36.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:36 smithi136 bash[75164]: cluster 2024-09-12T23:46:36.374655+0000 mon.a (mon.0) 713 : cluster [DBG] osdmap e541: 8 total, 7 up, 8 in 2024-09-12T23:46:36.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:36 smithi136 bash[76371]: cluster 2024-09-12T23:46:35.413188+0000 mon.a (mon.0) 712 : cluster [DBG] osdmap e540: 8 total, 7 up, 8 in 2024-09-12T23:46:36.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:36 smithi136 bash[76371]: cluster 2024-09-12T23:46:36.374655+0000 mon.a (mon.0) 713 : cluster [DBG] osdmap e541: 8 total, 7 up, 8 in 2024-09-12T23:46:37.380 INFO:tasks.rados.rados.0.smithi136.stdout:2389: done (3 left) 2024-09-12T23:46:37.380 INFO:tasks.rados.rados.0.smithi136.stdout:2390: snap_create 2024-09-12T23:46:37.380 INFO:tasks.rados.rados.0.smithi136.stdout:2386: finishing rollback tid 0 to smithi13667081-30 2024-09-12T23:46:37.380 INFO:tasks.rados.rados.0.smithi136.stdout:2386: finishing rollback tid 1 to smithi13667081-30 2024-09-12T23:46:37.380 INFO:tasks.rados.rados.0.smithi136.stdout:2386: finishing rollback tid 2 to smithi13667081-30 2024-09-12T23:46:37.380 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 30 v 1196 (ObjNum 734 snap 226 seq_num 734) dirty exists 2024-09-12T23:46:37.380 INFO:tasks.rados.rados.0.smithi136.stdout:2387: finishing rollback tid 0 to smithi13667081-36 2024-09-12T23:46:37.380 INFO:tasks.rados.rados.0.smithi136.stdout:2387: finishing rollback tid 1 to smithi13667081-36 2024-09-12T23:46:37.380 INFO:tasks.rados.rados.0.smithi136.stdout:2387: finishing rollback tid 2 to smithi13667081-36 2024-09-12T23:46:37.380 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 36 v 1241 (ObjNum 610 snap 175 seq_num 610) dirty exists 2024-09-12T23:46:37.662 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:37 smithi181 bash[67587]: cluster 2024-09-12T23:46:36.751018+0000 mgr.y (mgr.24661) 226 : cluster [DBG] pgmap v387: 121 pgs: 36 active+undersized, 11 active+undersized+degraded, 74 active+clean; 364 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.7 MiB/s wr, 17 op/s; 62/573 objects degraded (10.820%) 2024-09-12T23:46:37.662 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:37 smithi181 bash[67587]: cluster 2024-09-12T23:46:37.378215+0000 mon.a (mon.0) 714 : cluster [DBG] osdmap e542: 8 total, 7 up, 8 in 2024-09-12T23:46:37.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:37 smithi136 bash[75164]: cluster 2024-09-12T23:46:36.751018+0000 mgr.y (mgr.24661) 226 : cluster [DBG] pgmap v387: 121 pgs: 36 active+undersized, 11 active+undersized+degraded, 74 active+clean; 364 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.7 MiB/s wr, 17 op/s; 62/573 objects degraded (10.820%) 2024-09-12T23:46:37.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:37 smithi136 bash[75164]: cluster 2024-09-12T23:46:37.378215+0000 mon.a (mon.0) 714 : cluster [DBG] osdmap e542: 8 total, 7 up, 8 in 2024-09-12T23:46:37.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:37 smithi136 bash[76371]: cluster 2024-09-12T23:46:36.751018+0000 mgr.y (mgr.24661) 226 : cluster [DBG] pgmap v387: 121 pgs: 36 active+undersized, 11 active+undersized+degraded, 74 active+clean; 364 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.7 MiB/s wr, 17 op/s; 62/573 objects degraded (10.820%) 2024-09-12T23:46:37.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:37 smithi136 bash[76371]: cluster 2024-09-12T23:46:37.378215+0000 mon.a (mon.0) 714 : cluster [DBG] osdmap e542: 8 total, 7 up, 8 in 2024-09-12T23:46:38.094 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:46:37 smithi181 bash[76307]: debug 2024-09-12T23:46:37.663+0000 7fbb2b052640 1 -- 172.21.15.181:0/1516920226 <== mon.1 v2:172.21.15.181:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55fa76b3b1e0 con 0x55fa76afb800 2024-09-12T23:46:38.377 INFO:tasks.rados.rados.0.smithi136.stdout:2386: done (3 left) 2024-09-12T23:46:38.377 INFO:tasks.rados.rados.0.smithi136.stdout:2387: done (2 left) 2024-09-12T23:46:38.378 INFO:tasks.rados.rados.0.smithi136.stdout:2388: done (1 left) 2024-09-12T23:46:38.378 INFO:tasks.rados.rados.0.smithi136.stdout:2390: done (0 left) 2024-09-12T23:46:38.378 INFO:tasks.rados.rados.0.smithi136.stdout:2391: snap_create 2024-09-12T23:46:38.594 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:46:38 smithi181 bash[76307]: debug 2024-09-12T23:46:38.215+0000 7fbb2d8b8640 -1 Falling back to public interface 2024-09-12T23:46:38.658 INFO:teuthology.orchestra.run.smithi136.stdout:true 2024-09-12T23:46:38.956 INFO:teuthology.orchestra.run.smithi136.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-12T23:46:38.956 INFO:teuthology.orchestra.run.smithi136.stdout:alertmanager.a smithi136 *:9093,9094 running (6m) 55s ago 12m 13.1M - 0.25.0 c8568f914cd2 b5197f020eeb 2024-09-12T23:46:38.956 INFO:teuthology.orchestra.run.smithi136.stdout:grafana.a smithi181 *:3000 running (5m) 26s ago 11m 74.9M - 9.4.7 954c08fa6188 256c181be984 2024-09-12T23:46:38.957 INFO:teuthology.orchestra.run.smithi136.stdout:mds.a smithi136 running (13m) 55s ago 13m 15.1M - 17.2.7-1627-gbbdadfa3 214985049bdb 713b9795ad19 2024-09-12T23:46:38.957 INFO:teuthology.orchestra.run.smithi136.stdout:mds.b smithi181 running (13m) 26s ago 13m 19.2M - 17.2.7-1627-gbbdadfa3 214985049bdb ad13179f0868 2024-09-12T23:46:38.957 INFO:teuthology.orchestra.run.smithi136.stdout:mgr.x smithi181 *:8443,9283,8765 running (5m) 26s ago 20m 427M - 18.2.4-837-g7a853b17 f21ba1eea74e cc664dc337f7 2024-09-12T23:46:38.957 INFO:teuthology.orchestra.run.smithi136.stdout:mgr.y smithi136 *:8443,9283,8765 running (5m) 55s ago 23m 490M - 18.2.4-837-g7a853b17 f21ba1eea74e 272f10b42fc3 2024-09-12T23:46:38.957 INFO:teuthology.orchestra.run.smithi136.stdout:mon.a smithi136 running (4m) 55s ago 23m 56.4M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e ad8d21d58ad9 2024-09-12T23:46:38.957 INFO:teuthology.orchestra.run.smithi136.stdout:mon.b smithi181 running (3m) 26s ago 21m 39.6M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e a70ec8315934 2024-09-12T23:46:38.957 INFO:teuthology.orchestra.run.smithi136.stdout:mon.c smithi136 running (4m) 55s ago 21m 47.2M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e 44d1be055a5e 2024-09-12T23:46:38.957 INFO:teuthology.orchestra.run.smithi136.stdout:node-exporter.a smithi136 *:9100 running (6m) 55s ago 12m 8064k - 1.5.0 0da6a335fe13 ccfc37c96620 2024-09-12T23:46:38.957 INFO:teuthology.orchestra.run.smithi136.stdout:node-exporter.b smithi181 *:9100 running (5m) 26s ago 12m 7915k - 1.5.0 0da6a335fe13 80bf0e383230 2024-09-12T23:46:38.957 INFO:teuthology.orchestra.run.smithi136.stdout:osd.0 smithi136 running (2m) 55s ago 19m 212M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 7da21f765721 2024-09-12T23:46:38.957 INFO:teuthology.orchestra.run.smithi136.stdout:osd.1 smithi136 running (118s) 55s ago 19m 247M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 5302be0a016b 2024-09-12T23:46:38.958 INFO:teuthology.orchestra.run.smithi136.stdout:osd.2 smithi136 running (88s) 55s ago 18m 252M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 2e12d1a31223 2024-09-12T23:46:38.958 INFO:teuthology.orchestra.run.smithi136.stdout:osd.3 smithi136 running (59s) 55s ago 17m - 2524M 18.2.4-837-g7a853b17 f21ba1eea74e aa9dc124e59c 2024-09-12T23:46:38.958 INFO:teuthology.orchestra.run.smithi136.stdout:osd.4 smithi181 running (30s) 26s ago 16m - 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 4ee3b228ce7d 2024-09-12T23:46:38.958 INFO:teuthology.orchestra.run.smithi136.stdout:osd.5 smithi181 starting - - - 2524M 2024-09-12T23:46:38.958 INFO:teuthology.orchestra.run.smithi136.stdout:osd.6 smithi181 running (14m) 26s ago 14m 139M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 9c6bec406408 2024-09-12T23:46:38.958 INFO:teuthology.orchestra.run.smithi136.stdout:osd.7 smithi181 running (14m) 26s ago 14m 216M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb bc0632ae2145 2024-09-12T23:46:38.958 INFO:teuthology.orchestra.run.smithi136.stdout:prometheus.a smithi181 *:9095 running (5m) 26s ago 12m 57.0M - 2.43.0 a07b618ecd1d 3853d003a5cc 2024-09-12T23:46:39.306 INFO:teuthology.orchestra.run.smithi136.stdout:{ 2024-09-12T23:46:39.306 INFO:teuthology.orchestra.run.smithi136.stdout: "mon": { 2024-09-12T23:46:39.306 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 3 2024-09-12T23:46:39.306 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:46:39.306 INFO:teuthology.orchestra.run.smithi136.stdout: "mgr": { 2024-09-12T23:46:39.307 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 2 2024-09-12T23:46:39.307 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:46:39.307 INFO:teuthology.orchestra.run.smithi136.stdout: "osd": { 2024-09-12T23:46:39.307 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2, 2024-09-12T23:46:39.307 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 5 2024-09-12T23:46:39.307 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:46:39.307 INFO:teuthology.orchestra.run.smithi136.stdout: "mds": { 2024-09-12T23:46:39.307 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-12T23:46:39.307 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:46:39.307 INFO:teuthology.orchestra.run.smithi136.stdout: "overall": { 2024-09-12T23:46:39.307 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 4, 2024-09-12T23:46:39.307 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 10 2024-09-12T23:46:39.308 INFO:teuthology.orchestra.run.smithi136.stdout: } 2024-09-12T23:46:39.308 INFO:teuthology.orchestra.run.smithi136.stdout:} 2024-09-12T23:46:39.400 INFO:tasks.rados.rados.0.smithi136.stdout:2391: done (0 left) 2024-09-12T23:46:39.400 INFO:tasks.rados.rados.0.smithi136.stdout:2392: setattr oid 22 current snap is 242 2024-09-12T23:46:39.400 INFO:tasks.rados.rados.0.smithi136.stdout:2393: snap_remove snap 236 2024-09-12T23:46:39.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:46:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:46:39] "GET /metrics HTTP/1.1" 200 36386 "" "Prometheus/2.43.0" 2024-09-12T23:46:39.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:39 smithi136 bash[75164]: cluster 2024-09-12T23:46:38.380755+0000 mon.a (mon.0) 715 : cluster [DBG] osdmap e543: 8 total, 7 up, 8 in 2024-09-12T23:46:39.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:39 smithi136 bash[75164]: audit 2024-09-12T23:46:38.648741+0000 mgr.y (mgr.24661) 227 : audit [DBG] from='client.34335 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:46:39.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:39 smithi136 bash[75164]: audit 2024-09-12T23:46:38.751477+0000 mon.a (mon.0) 716 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:39.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:39 smithi136 bash[75164]: cluster 2024-09-12T23:46:38.751702+0000 mgr.y (mgr.24661) 228 : cluster [DBG] pgmap v390: 121 pgs: 36 active+undersized, 11 active+undersized+degraded, 74 active+clean; 358 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 664 KiB/s wr, 10 op/s; 62/555 objects degraded (11.171%) 2024-09-12T23:46:39.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:39 smithi136 bash[75164]: audit 2024-09-12T23:46:38.752562+0000 mon.a (mon.0) 717 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:46:39.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:39 smithi136 bash[75164]: audit 2024-09-12T23:46:39.309617+0000 mon.a (mon.0) 718 : audit [DBG] from='client.? 172.21.15.136:0/2674404196' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:46:39.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:39 smithi136 bash[76371]: cluster 2024-09-12T23:46:38.380755+0000 mon.a (mon.0) 715 : cluster [DBG] osdmap e543: 8 total, 7 up, 8 in 2024-09-12T23:46:39.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:39 smithi136 bash[76371]: audit 2024-09-12T23:46:38.648741+0000 mgr.y (mgr.24661) 227 : audit [DBG] from='client.34335 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:46:39.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:39 smithi136 bash[76371]: audit 2024-09-12T23:46:38.751477+0000 mon.a (mon.0) 716 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:39.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:39 smithi136 bash[76371]: cluster 2024-09-12T23:46:38.751702+0000 mgr.y (mgr.24661) 228 : cluster [DBG] pgmap v390: 121 pgs: 36 active+undersized, 11 active+undersized+degraded, 74 active+clean; 358 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 664 KiB/s wr, 10 op/s; 62/555 objects degraded (11.171%) 2024-09-12T23:46:39.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:39 smithi136 bash[76371]: audit 2024-09-12T23:46:38.752562+0000 mon.a (mon.0) 717 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:46:39.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:39 smithi136 bash[76371]: audit 2024-09-12T23:46:39.309617+0000 mon.a (mon.0) 718 : audit [DBG] from='client.? 172.21.15.136:0/2674404196' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:46:39.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:39 smithi181 bash[67587]: cluster 2024-09-12T23:46:38.380755+0000 mon.a (mon.0) 715 : cluster [DBG] osdmap e543: 8 total, 7 up, 8 in 2024-09-12T23:46:39.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:39 smithi181 bash[67587]: audit 2024-09-12T23:46:38.648741+0000 mgr.y (mgr.24661) 227 : audit [DBG] from='client.34335 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:46:39.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:39 smithi181 bash[67587]: audit 2024-09-12T23:46:38.751477+0000 mon.a (mon.0) 716 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:39.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:39 smithi181 bash[67587]: cluster 2024-09-12T23:46:38.751702+0000 mgr.y (mgr.24661) 228 : cluster [DBG] pgmap v390: 121 pgs: 36 active+undersized, 11 active+undersized+degraded, 74 active+clean; 358 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 664 KiB/s wr, 10 op/s; 62/555 objects degraded (11.171%) 2024-09-12T23:46:39.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:39 smithi181 bash[67587]: audit 2024-09-12T23:46:38.752562+0000 mon.a (mon.0) 717 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:46:39.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:39 smithi181 bash[67587]: audit 2024-09-12T23:46:39.309617+0000 mon.a (mon.0) 718 : audit [DBG] from='client.? 172.21.15.136:0/2674404196' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:46:40.399 INFO:tasks.rados.rados.0.smithi136.stdout:2393: done (1 left) 2024-09-12T23:46:40.399 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 11 current snap is 242 2024-09-12T23:46:40.400 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 22 v 866 (ObjNum 385 snap 0 seq_num 1427291728) dirty exists 2024-09-12T23:46:40.400 INFO:tasks.rados.rados.0.smithi136.stdout:2394: seq_num 775 ranges {7290880=589824,7880704=401408} 2024-09-12T23:46:40.493 INFO:tasks.rados.rados.0.smithi136.stdout:2394: writing smithi13667081-11 from 7290880 to 7880704 tid 1 2024-09-12T23:46:40.496 INFO:tasks.rados.rados.0.smithi136.stdout:2394: writing smithi13667081-11 from 7880704 to 8282112 tid 2 2024-09-12T23:46:40.496 INFO:tasks.rados.rados.0.smithi136.stdout:2392: done (1 left) 2024-09-12T23:46:40.496 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 44 current snap is 242 2024-09-12T23:46:40.496 INFO:tasks.rados.rados.0.smithi136.stdout:2395: seq_num 776 ranges {0=606208,606208=401408} 2024-09-12T23:46:40.501 INFO:tasks.rados.rados.0.smithi136.stdout:2395: writing smithi13667081-44 from 0 to 606208 tid 1 2024-09-12T23:46:40.504 INFO:tasks.rados.rados.0.smithi136.stdout:2395: writing smithi13667081-44 from 606208 to 1007616 tid 2 2024-09-12T23:46:40.504 INFO:tasks.rados.rados.0.smithi136.stdout:2396: setattr oid 23 current snap is 242 2024-09-12T23:46:40.506 INFO:tasks.rados.rados.0.smithi136.stdout:2397: snap_remove snap 229 2024-09-12T23:46:40.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:40 smithi136 bash[75164]: audit 2024-09-12T23:46:38.943895+0000 mgr.y (mgr.24661) 229 : audit [DBG] from='client.44258 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:46:40.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:40 smithi136 bash[75164]: cluster 2024-09-12T23:46:39.398970+0000 mon.a (mon.0) 719 : cluster [DBG] osdmap e544: 8 total, 7 up, 8 in 2024-09-12T23:46:40.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:40 smithi136 bash[75164]: cluster 2024-09-12T23:46:39.749296+0000 mon.a (mon.0) 720 : cluster [WRN] Health check update: Degraded data redundancy: 62/555 objects degraded (11.171%), 11 pgs degraded (PG_DEGRADED) 2024-09-12T23:46:40.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:40 smithi136 bash[76371]: audit 2024-09-12T23:46:38.943895+0000 mgr.y (mgr.24661) 229 : audit [DBG] from='client.44258 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:46:40.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:40 smithi136 bash[76371]: cluster 2024-09-12T23:46:39.398970+0000 mon.a (mon.0) 719 : cluster [DBG] osdmap e544: 8 total, 7 up, 8 in 2024-09-12T23:46:40.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:40 smithi136 bash[76371]: cluster 2024-09-12T23:46:39.749296+0000 mon.a (mon.0) 720 : cluster [WRN] Health check update: Degraded data redundancy: 62/555 objects degraded (11.171%), 11 pgs degraded (PG_DEGRADED) 2024-09-12T23:46:40.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:40 smithi181 bash[67587]: audit 2024-09-12T23:46:38.943895+0000 mgr.y (mgr.24661) 229 : audit [DBG] from='client.44258 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:46:40.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:40 smithi181 bash[67587]: cluster 2024-09-12T23:46:39.398970+0000 mon.a (mon.0) 719 : cluster [DBG] osdmap e544: 8 total, 7 up, 8 in 2024-09-12T23:46:40.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:40 smithi181 bash[67587]: cluster 2024-09-12T23:46:39.749296+0000 mon.a (mon.0) 720 : cluster [WRN] Health check update: Degraded data redundancy: 62/555 objects degraded (11.171%), 11 pgs degraded (PG_DEGRADED) 2024-09-12T23:46:41.424 INFO:tasks.rados.rados.0.smithi136.stdout:2397: done (3 left) 2024-09-12T23:46:41.424 INFO:tasks.rados.rados.0.smithi136.stdout:2398: read oid 39 snap -1 2024-09-12T23:46:41.424 INFO:tasks.rados.rados.0.smithi136.stdout:2398: expect deleted 2024-09-12T23:46:41.424 INFO:tasks.rados.rados.0.smithi136.stdout:2399: delete oid 14 current snap is 242 2024-09-12T23:46:41.425 INFO:tasks.rados.rados.0.smithi136.stdout:2394: finishing write tid 1 to smithi13667081-11 2024-09-12T23:46:41.425 INFO:tasks.rados.rados.0.smithi136.stdout:2394: finishing write tid 2 to smithi13667081-11 2024-09-12T23:46:41.425 INFO:tasks.rados.rados.0.smithi136.stdout:2394: finishing write tid 3 to smithi13667081-11 2024-09-12T23:46:41.425 INFO:tasks.rados.rados.0.smithi136.stdout:2395: finishing write tid 1 to smithi13667081-44 2024-09-12T23:46:41.425 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 23 v 1661 (ObjNum 741 snap 228 seq_num 741) dirty exists 2024-09-12T23:46:41.425 INFO:tasks.rados.rados.0.smithi136.stdout:2394: finishing write tid 4 to smithi13667081-11 2024-09-12T23:46:41.425 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 11 v 1659 (ObjNum 775 snap 242 seq_num 775) dirty exists 2024-09-12T23:46:41.425 INFO:tasks.rados.rados.0.smithi136.stdout:2394: left oid 11 (ObjNum 775 snap 242 seq_num 775) 2024-09-12T23:46:41.425 INFO:tasks.rados.rados.0.smithi136.stdout:2395: finishing write tid 2 to smithi13667081-44 2024-09-12T23:46:41.425 INFO:tasks.rados.rados.0.smithi136.stdout:2395: finishing write tid 3 to smithi13667081-44 2024-09-12T23:46:41.425 INFO:tasks.rados.rados.0.smithi136.stdout:2395: finishing write tid 4 to smithi13667081-44 2024-09-12T23:46:41.425 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 44 v 1595 (ObjNum 776 snap 242 seq_num 776) dirty exists 2024-09-12T23:46:41.425 INFO:tasks.rados.rados.0.smithi136.stdout:2395: left oid 44 (ObjNum 776 snap 242 seq_num 776) 2024-09-12T23:46:41.441 INFO:tasks.rados.rados.0.smithi136.stdout:2394: done (4 left) 2024-09-12T23:46:41.442 INFO:tasks.rados.rados.0.smithi136.stdout:2395: done (3 left) 2024-09-12T23:46:41.442 INFO:tasks.rados.rados.0.smithi136.stdout:2396: done (2 left) 2024-09-12T23:46:41.442 INFO:tasks.rados.rados.0.smithi136.stdout:2398: done (1 left) 2024-09-12T23:46:41.442 INFO:tasks.rados.rados.0.smithi136.stdout:2399: done (0 left) 2024-09-12T23:46:41.442 INFO:tasks.rados.rados.0.smithi136.stdout:2400: snap_remove snap 225 2024-09-12T23:46:41.736 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:41 smithi181 bash[67587]: cluster 2024-09-12T23:46:40.401640+0000 mon.a (mon.0) 721 : cluster [DBG] osdmap e545: 8 total, 7 up, 8 in 2024-09-12T23:46:41.737 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:41 smithi181 bash[67587]: cluster 2024-09-12T23:46:40.752858+0000 mgr.y (mgr.24661) 230 : cluster [DBG] pgmap v393: 121 pgs: 36 active+undersized, 11 active+undersized+degraded, 74 active+clean; 344 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 208 KiB/s wr, 1 op/s; 65/549 objects degraded (11.840%) 2024-09-12T23:46:41.737 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:41 smithi181 bash[67587]: audit 2024-09-12T23:46:41.173202+0000 mon.a (mon.0) 722 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:41.737 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:41 smithi181 bash[67587]: audit 2024-09-12T23:46:41.183971+0000 mon.a (mon.0) 723 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:41.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:41 smithi136 bash[75164]: cluster 2024-09-12T23:46:40.401640+0000 mon.a (mon.0) 721 : cluster [DBG] osdmap e545: 8 total, 7 up, 8 in 2024-09-12T23:46:41.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:41 smithi136 bash[75164]: cluster 2024-09-12T23:46:40.752858+0000 mgr.y (mgr.24661) 230 : cluster [DBG] pgmap v393: 121 pgs: 36 active+undersized, 11 active+undersized+degraded, 74 active+clean; 344 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 208 KiB/s wr, 1 op/s; 65/549 objects degraded (11.840%) 2024-09-12T23:46:41.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:41 smithi136 bash[75164]: audit 2024-09-12T23:46:41.173202+0000 mon.a (mon.0) 722 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:41.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:41 smithi136 bash[75164]: audit 2024-09-12T23:46:41.183971+0000 mon.a (mon.0) 723 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:41.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:41 smithi136 bash[76371]: cluster 2024-09-12T23:46:40.401640+0000 mon.a (mon.0) 721 : cluster [DBG] osdmap e545: 8 total, 7 up, 8 in 2024-09-12T23:46:41.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:41 smithi136 bash[76371]: cluster 2024-09-12T23:46:40.752858+0000 mgr.y (mgr.24661) 230 : cluster [DBG] pgmap v393: 121 pgs: 36 active+undersized, 11 active+undersized+degraded, 74 active+clean; 344 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 208 KiB/s wr, 1 op/s; 65/549 objects degraded (11.840%) 2024-09-12T23:46:41.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:41 smithi136 bash[76371]: audit 2024-09-12T23:46:41.173202+0000 mon.a (mon.0) 722 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:41.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:41 smithi136 bash[76371]: audit 2024-09-12T23:46:41.183971+0000 mon.a (mon.0) 723 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:42.094 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:46:41 smithi181 bash[76307]: debug 2024-09-12T23:46:41.739+0000 7fbb2d8b8640 -1 osd.5 534 log_to_monitors true 2024-09-12T23:46:42.427 INFO:tasks.rados.rados.0.smithi136.stdout:2400: done (0 left) 2024-09-12T23:46:42.427 INFO:tasks.rados.rados.0.smithi136.stdout:2401: rollback oid 11 current snap is 242 2024-09-12T23:46:42.427 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 11 to 215 2024-09-12T23:46:42.427 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 6 current snap is 242 2024-09-12T23:46:42.427 INFO:tasks.rados.rados.0.smithi136.stdout:2402: seq_num 778 ranges {1515520=638976,2154496=401408} 2024-09-12T23:46:42.449 INFO:tasks.rados.rados.0.smithi136.stdout:2402: writing smithi13667081-6 from 1515520 to 2154496 tid 1 2024-09-12T23:46:42.455 INFO:tasks.rados.rados.0.smithi136.stdout:2402: writing smithi13667081-6 from 2154496 to 2555904 tid 2 2024-09-12T23:46:42.455 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 37 current snap is 242 2024-09-12T23:46:42.455 INFO:tasks.rados.rados.0.smithi136.stdout:2401: finishing rollback tid 0 to smithi13667081-11 2024-09-12T23:46:42.455 INFO:tasks.rados.rados.0.smithi136.stdout:2403: seq_num 779 ranges {393216=655360,1048576=401408} 2024-09-12T23:46:42.466 INFO:tasks.rados.rados.0.smithi136.stdout:2403: writing smithi13667081-37 from 393216 to 1048576 tid 1 2024-09-12T23:46:42.471 INFO:tasks.rados.rados.0.smithi136.stdout:2403: writing smithi13667081-37 from 1048576 to 1449984 tid 2 2024-09-12T23:46:42.471 INFO:tasks.rados.rados.0.smithi136.stdout:2404: snap_create 2024-09-12T23:46:42.471 INFO:tasks.rados.rados.0.smithi136.stdout:2402: finishing write tid 1 to smithi13667081-6 2024-09-12T23:46:42.471 INFO:tasks.rados.rados.0.smithi136.stdout:2401: finishing rollback tid 1 to smithi13667081-11 2024-09-12T23:46:42.471 INFO:tasks.rados.rados.0.smithi136.stdout:2401: finishing rollback tid 2 to smithi13667081-11 2024-09-12T23:46:42.471 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 11 v 1670 (ObjNum 605 snap 174 seq_num 605) dirty exists 2024-09-12T23:46:42.472 INFO:tasks.rados.rados.0.smithi136.stdout:2402: finishing write tid 2 to smithi13667081-6 2024-09-12T23:46:42.472 INFO:tasks.rados.rados.0.smithi136.stdout:2402: finishing write tid 3 to smithi13667081-6 2024-09-12T23:46:42.472 INFO:tasks.rados.rados.0.smithi136.stdout:2402: finishing write tid 4 to smithi13667081-6 2024-09-12T23:46:42.472 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 6 v 1235 (ObjNum 778 snap 242 seq_num 778) dirty exists 2024-09-12T23:46:42.472 INFO:tasks.rados.rados.0.smithi136.stdout:2402: left oid 6 (ObjNum 778 snap 242 seq_num 778) 2024-09-12T23:46:42.481 INFO:tasks.rados.rados.0.smithi136.stdout:2403: finishing write tid 1 to smithi13667081-37 2024-09-12T23:46:42.484 INFO:tasks.rados.rados.0.smithi136.stdout:2403: finishing write tid 2 to smithi13667081-37 2024-09-12T23:46:42.484 INFO:tasks.rados.rados.0.smithi136.stdout:2403: finishing write tid 3 to smithi13667081-37 2024-09-12T23:46:42.488 INFO:tasks.rados.rados.0.smithi136.stdout:2403: finishing write tid 4 to smithi13667081-37 2024-09-12T23:46:42.488 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 37 v 1258 (ObjNum 779 snap 242 seq_num 779) dirty exists 2024-09-12T23:46:42.488 INFO:tasks.rados.rados.0.smithi136.stdout:2403: left oid 37 (ObjNum 779 snap 242 seq_num 779) 2024-09-12T23:46:42.594 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:46:42 smithi181 bash[76307]: debug 2024-09-12T23:46:42.499+0000 7fbb25662640 -1 osd.5 534 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-12T23:46:42.594 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:46:42 smithi181 bash[60471]: ts=2024-09-12T23:46:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:46:42.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:42 smithi181 bash[67587]: cluster 2024-09-12T23:46:41.425377+0000 mon.a (mon.0) 724 : cluster [DBG] osdmap e546: 8 total, 7 up, 8 in 2024-09-12T23:46:42.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:42 smithi181 bash[67587]: audit 2024-09-12T23:46:41.744745+0000 mon.a (mon.0) 725 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-12T23:46:42.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:42 smithi181 bash[67587]: audit 2024-09-12T23:46:41.745314+0000 mon.b (mon.1) 6 : audit [INF] from='osd.5 [v2:172.21.15.181:6808/3863825756,v1:172.21.15.181:6809/3863825756]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-12T23:46:42.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:42 smithi181 bash[67587]: audit 2024-09-12T23:46:41.941684+0000 mon.a (mon.0) 726 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:42.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:42 smithi181 bash[67587]: audit 2024-09-12T23:46:41.952266+0000 mon.a (mon.0) 727 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:42.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:42 smithi136 bash[75164]: cluster 2024-09-12T23:46:41.425377+0000 mon.a (mon.0) 724 : cluster [DBG] osdmap e546: 8 total, 7 up, 8 in 2024-09-12T23:46:42.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:42 smithi136 bash[75164]: audit 2024-09-12T23:46:41.744745+0000 mon.a (mon.0) 725 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-12T23:46:42.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:42 smithi136 bash[75164]: audit 2024-09-12T23:46:41.745314+0000 mon.b (mon.1) 6 : audit [INF] from='osd.5 [v2:172.21.15.181:6808/3863825756,v1:172.21.15.181:6809/3863825756]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-12T23:46:42.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:42 smithi136 bash[75164]: audit 2024-09-12T23:46:41.941684+0000 mon.a (mon.0) 726 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:42.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:42 smithi136 bash[75164]: audit 2024-09-12T23:46:41.952266+0000 mon.a (mon.0) 727 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:42.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:42 smithi136 bash[76371]: cluster 2024-09-12T23:46:41.425377+0000 mon.a (mon.0) 724 : cluster [DBG] osdmap e546: 8 total, 7 up, 8 in 2024-09-12T23:46:42.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:42 smithi136 bash[76371]: audit 2024-09-12T23:46:41.744745+0000 mon.a (mon.0) 725 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-12T23:46:42.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:42 smithi136 bash[76371]: audit 2024-09-12T23:46:41.745314+0000 mon.b (mon.1) 6 : audit [INF] from='osd.5 [v2:172.21.15.181:6808/3863825756,v1:172.21.15.181:6809/3863825756]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-12T23:46:42.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:42 smithi136 bash[76371]: audit 2024-09-12T23:46:41.941684+0000 mon.a (mon.0) 726 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:42.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:42 smithi136 bash[76371]: audit 2024-09-12T23:46:41.952266+0000 mon.a (mon.0) 727 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:43.463 INFO:tasks.rados.rados.0.smithi136.stdout:2401: done (3 left) 2024-09-12T23:46:43.463 INFO:tasks.rados.rados.0.smithi136.stdout:2402: done (2 left) 2024-09-12T23:46:43.463 INFO:tasks.rados.rados.0.smithi136.stdout:2403: done (1 left) 2024-09-12T23:46:43.463 INFO:tasks.rados.rados.0.smithi136.stdout:2404: done (0 left) 2024-09-12T23:46:43.463 INFO:tasks.rados.rados.0.smithi136.stdout:2405: rollback oid 16 current snap is 243 2024-09-12T23:46:43.463 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 16 to 233 2024-09-12T23:46:43.464 INFO:tasks.rados.rados.0.smithi136.stdout:2406: read oid 42 snap -1 2024-09-12T23:46:43.464 INFO:tasks.rados.rados.0.smithi136.stdout:2406: expect deleted 2024-09-12T23:46:43.464 INFO:tasks.rados.rados.0.smithi136.stdout:2407: snap_remove snap 242 2024-09-12T23:46:43.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:43 smithi136 bash[75164]: audit 2024-09-12T23:46:42.429105+0000 mon.a (mon.0) 728 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-12T23:46:43.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:43 smithi136 bash[75164]: cluster 2024-09-12T23:46:42.429313+0000 mon.a (mon.0) 729 : cluster [DBG] osdmap e547: 8 total, 7 up, 8 in 2024-09-12T23:46:43.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:43 smithi136 bash[75164]: audit 2024-09-12T23:46:42.435660+0000 mon.b (mon.1) 7 : audit [INF] from='osd.5 [v2:172.21.15.181:6808/3863825756,v1:172.21.15.181:6809/3863825756]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-12T23:46:43.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:43 smithi136 bash[75164]: audit 2024-09-12T23:46:42.453839+0000 mon.a (mon.0) 730 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-12T23:46:43.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:43 smithi136 bash[75164]: cluster 2024-09-12T23:46:42.753713+0000 mgr.y (mgr.24661) 231 : cluster [DBG] pgmap v396: 121 pgs: 36 active+undersized, 11 active+undersized+degraded, 74 active+clean; 317 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 454 KiB/s wr, 2 op/s; 65/519 objects degraded (12.524%) 2024-09-12T23:46:43.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:43 smithi136 bash[76371]: audit 2024-09-12T23:46:42.429105+0000 mon.a (mon.0) 728 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-12T23:46:43.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:43 smithi136 bash[76371]: cluster 2024-09-12T23:46:42.429313+0000 mon.a (mon.0) 729 : cluster [DBG] osdmap e547: 8 total, 7 up, 8 in 2024-09-12T23:46:43.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:43 smithi136 bash[76371]: audit 2024-09-12T23:46:42.435660+0000 mon.b (mon.1) 7 : audit [INF] from='osd.5 [v2:172.21.15.181:6808/3863825756,v1:172.21.15.181:6809/3863825756]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-12T23:46:43.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:43 smithi136 bash[76371]: audit 2024-09-12T23:46:42.453839+0000 mon.a (mon.0) 730 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-12T23:46:43.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:43 smithi136 bash[76371]: cluster 2024-09-12T23:46:42.753713+0000 mgr.y (mgr.24661) 231 : cluster [DBG] pgmap v396: 121 pgs: 36 active+undersized, 11 active+undersized+degraded, 74 active+clean; 317 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 454 KiB/s wr, 2 op/s; 65/519 objects degraded (12.524%) 2024-09-12T23:46:43.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:43 smithi181 bash[67587]: audit 2024-09-12T23:46:42.429105+0000 mon.a (mon.0) 728 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-12T23:46:43.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:43 smithi181 bash[67587]: cluster 2024-09-12T23:46:42.429313+0000 mon.a (mon.0) 729 : cluster [DBG] osdmap e547: 8 total, 7 up, 8 in 2024-09-12T23:46:43.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:43 smithi181 bash[67587]: audit 2024-09-12T23:46:42.435660+0000 mon.b (mon.1) 7 : audit [INF] from='osd.5 [v2:172.21.15.181:6808/3863825756,v1:172.21.15.181:6809/3863825756]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-12T23:46:43.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:43 smithi181 bash[67587]: audit 2024-09-12T23:46:42.453839+0000 mon.a (mon.0) 730 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-12T23:46:43.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:43 smithi181 bash[67587]: cluster 2024-09-12T23:46:42.753713+0000 mgr.y (mgr.24661) 231 : cluster [DBG] pgmap v396: 121 pgs: 36 active+undersized, 11 active+undersized+degraded, 74 active+clean; 317 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 454 KiB/s wr, 2 op/s; 65/519 objects degraded (12.524%) 2024-09-12T23:46:44.467 INFO:tasks.rados.rados.0.smithi136.stdout:2405: finishing rollback tid 1 to smithi13667081-16 2024-09-12T23:46:44.467 INFO:tasks.rados.rados.0.smithi136.stdout:2405: finishing rollback tid 2 to smithi13667081-16 2024-09-12T23:46:44.468 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 16 v 1597 (ObjNum 744 snap 228 seq_num 744) dirty exists 2024-09-12T23:46:44.468 INFO:tasks.rados.rados.0.smithi136.stdout:2405: done (2 left) 2024-09-12T23:46:44.468 INFO:tasks.rados.rados.0.smithi136.stdout:2406: done (1 left) 2024-09-12T23:46:44.468 INFO:tasks.rados.rados.0.smithi136.stdout:2407: done (0 left) 2024-09-12T23:46:44.468 INFO:tasks.rados.rados.0.smithi136.stdout:2408: copy_from oid 50 from oid 39 current snap is 243 2024-09-12T23:46:44.468 INFO:tasks.rados.rados.0.smithi136.stdout:2409: snap_create 2024-09-12T23:46:44.509 INFO:tasks.rados.rados.0.smithi136.stdout:2408: finishing copy_from to smithi13667081-50 2024-09-12T23:46:44.509 INFO:tasks.rados.rados.0.smithi136.stdout:2408: got expected ENOENT (src dne) 2024-09-12T23:46:44.509 INFO:tasks.rados.rados.0.smithi136.stdout:2408: finishing copy_from racing read to smithi13667081-50 2024-09-12T23:46:44.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:44 smithi136 bash[75164]: cluster 2024-09-12T23:46:43.446177+0000 mon.a (mon.0) 731 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-12T23:46:44.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:44 smithi136 bash[75164]: cluster 2024-09-12T23:46:43.465303+0000 mon.a (mon.0) 732 : cluster [INF] osd.5 [v2:172.21.15.181:6808/3863825756,v1:172.21.15.181:6809/3863825756] boot 2024-09-12T23:46:44.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:44 smithi136 bash[75164]: cluster 2024-09-12T23:46:43.465423+0000 mon.a (mon.0) 733 : cluster [DBG] osdmap e548: 8 total, 8 up, 8 in 2024-09-12T23:46:44.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:44 smithi136 bash[75164]: audit 2024-09-12T23:46:43.466016+0000 mon.a (mon.0) 734 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-12T23:46:44.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:44 smithi136 bash[76371]: cluster 2024-09-12T23:46:43.446177+0000 mon.a (mon.0) 731 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-12T23:46:44.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:44 smithi136 bash[76371]: cluster 2024-09-12T23:46:43.465303+0000 mon.a (mon.0) 732 : cluster [INF] osd.5 [v2:172.21.15.181:6808/3863825756,v1:172.21.15.181:6809/3863825756] boot 2024-09-12T23:46:44.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:44 smithi136 bash[76371]: cluster 2024-09-12T23:46:43.465423+0000 mon.a (mon.0) 733 : cluster [DBG] osdmap e548: 8 total, 8 up, 8 in 2024-09-12T23:46:44.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:44 smithi136 bash[76371]: audit 2024-09-12T23:46:43.466016+0000 mon.a (mon.0) 734 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-12T23:46:44.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:44 smithi181 bash[67587]: cluster 2024-09-12T23:46:43.446177+0000 mon.a (mon.0) 731 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-12T23:46:44.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:44 smithi181 bash[67587]: cluster 2024-09-12T23:46:43.465303+0000 mon.a (mon.0) 732 : cluster [INF] osd.5 [v2:172.21.15.181:6808/3863825756,v1:172.21.15.181:6809/3863825756] boot 2024-09-12T23:46:44.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:44 smithi181 bash[67587]: cluster 2024-09-12T23:46:43.465423+0000 mon.a (mon.0) 733 : cluster [DBG] osdmap e548: 8 total, 8 up, 8 in 2024-09-12T23:46:44.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:44 smithi181 bash[67587]: audit 2024-09-12T23:46:43.466016+0000 mon.a (mon.0) 734 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-12T23:46:45.465 INFO:tasks.rados.rados.0.smithi136.stdout:2408: done (1 left) 2024-09-12T23:46:45.465 INFO:tasks.rados.rados.0.smithi136.stdout:2409: done (0 left) 2024-09-12T23:46:45.465 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 40 current snap is 244 2024-09-12T23:46:45.465 INFO:tasks.rados.rados.0.smithi136.stdout:2410: seq_num 780 ranges {385024=671744,1056768=401408} 2024-09-12T23:46:45.482 INFO:tasks.rados.rados.0.smithi136.stdout:2410: writing smithi13667081-40 from 385024 to 1056768 tid 1 2024-09-12T23:46:45.490 INFO:tasks.rados.rados.0.smithi136.stdout:2410: writing smithi13667081-40 from 1056768 to 1458176 tid 2 2024-09-12T23:46:45.490 INFO:tasks.rados.rados.0.smithi136.stdout:2411: copy_from oid 27 from oid 9 current snap is 244 2024-09-12T23:46:45.490 INFO:tasks.rados.rados.0.smithi136.stdout:2412: snap_remove snap 212 2024-09-12T23:46:45.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:45 smithi136 bash[75164]: cluster 2024-09-12T23:46:44.466647+0000 mon.a (mon.0) 735 : cluster [DBG] osdmap e549: 8 total, 8 up, 8 in 2024-09-12T23:46:45.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:45 smithi136 bash[75164]: cluster 2024-09-12T23:46:44.754837+0000 mgr.y (mgr.24661) 232 : cluster [DBG] pgmap v399: 121 pgs: 1 activating+degraded, 1 activating, 28 active+undersized, 7 active+undersized+degraded, 84 active+clean; 313 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 488 KiB/s wr, 12 op/s; 45/516 objects degraded (8.721%) 2024-09-12T23:46:45.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:45 smithi136 bash[75164]: cluster 2024-09-12T23:46:45.468385+0000 mon.a (mon.0) 736 : cluster [DBG] osdmap e550: 8 total, 8 up, 8 in 2024-09-12T23:46:45.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:45 smithi136 bash[76371]: cluster 2024-09-12T23:46:44.466647+0000 mon.a (mon.0) 735 : cluster [DBG] osdmap e549: 8 total, 8 up, 8 in 2024-09-12T23:46:45.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:45 smithi136 bash[76371]: cluster 2024-09-12T23:46:44.754837+0000 mgr.y (mgr.24661) 232 : cluster [DBG] pgmap v399: 121 pgs: 1 activating+degraded, 1 activating, 28 active+undersized, 7 active+undersized+degraded, 84 active+clean; 313 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 488 KiB/s wr, 12 op/s; 45/516 objects degraded (8.721%) 2024-09-12T23:46:45.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:45 smithi136 bash[76371]: cluster 2024-09-12T23:46:45.468385+0000 mon.a (mon.0) 736 : cluster [DBG] osdmap e550: 8 total, 8 up, 8 in 2024-09-12T23:46:45.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:45 smithi181 bash[67587]: cluster 2024-09-12T23:46:44.466647+0000 mon.a (mon.0) 735 : cluster [DBG] osdmap e549: 8 total, 8 up, 8 in 2024-09-12T23:46:45.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:45 smithi181 bash[67587]: cluster 2024-09-12T23:46:44.754837+0000 mgr.y (mgr.24661) 232 : cluster [DBG] pgmap v399: 121 pgs: 1 activating+degraded, 1 activating, 28 active+undersized, 7 active+undersized+degraded, 84 active+clean; 313 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 488 KiB/s wr, 12 op/s; 45/516 objects degraded (8.721%) 2024-09-12T23:46:45.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:45 smithi181 bash[67587]: cluster 2024-09-12T23:46:45.468385+0000 mon.a (mon.0) 736 : cluster [DBG] osdmap e550: 8 total, 8 up, 8 in 2024-09-12T23:46:46.489 INFO:tasks.rados.rados.0.smithi136.stdout:2412: done (2 left) 2024-09-12T23:46:46.489 INFO:tasks.rados.rados.0.smithi136.stdout:2413: copy_from oid 32 from oid 2 current snap is 244 2024-09-12T23:46:46.489 INFO:tasks.rados.rados.0.smithi136.stdout:2410: finishing write tid 1 to smithi13667081-40 2024-09-12T23:46:46.489 INFO:tasks.rados.rados.0.smithi136.stdout:2410: finishing write tid 2 to smithi13667081-40 2024-09-12T23:46:46.489 INFO:tasks.rados.rados.0.smithi136.stdout:2410: finishing write tid 3 to smithi13667081-40 2024-09-12T23:46:46.489 INFO:tasks.rados.rados.0.smithi136.stdout:2410: finishing write tid 4 to smithi13667081-40 2024-09-12T23:46:46.489 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 40 v 1680 (ObjNum 780 snap 244 seq_num 780) dirty exists 2024-09-12T23:46:46.489 INFO:tasks.rados.rados.0.smithi136.stdout:2410: left oid 40 (ObjNum 780 snap 244 seq_num 780) 2024-09-12T23:46:46.490 INFO:tasks.rados.rados.0.smithi136.stdout:2411: finishing copy_from racing read to smithi13667081-27 2024-09-12T23:46:46.490 INFO:tasks.rados.rados.0.smithi136.stdout:2411: finishing copy_from to smithi13667081-27 2024-09-12T23:46:46.490 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 27 v 745 (ObjNum 767 snap 236 seq_num 767) dirty exists 2024-09-12T23:46:46.490 INFO:tasks.rados.rados.0.smithi136.stdout:2410: done (2 left) 2024-09-12T23:46:46.490 INFO:tasks.rados.rados.0.smithi136.stdout:2411: done (1 left) 2024-09-12T23:46:46.490 INFO:tasks.rados.rados.0.smithi136.stdout:2414: rmattr oid 44 current snap is 244 2024-09-12T23:46:46.490 INFO:tasks.rados.rados.0.smithi136.stdout:2415: copy_from oid 21 from oid 29 current snap is 244 2024-09-12T23:46:46.490 INFO:tasks.rados.rados.0.smithi136.stdout:2416: snap_create 2024-09-12T23:46:46.511 INFO:tasks.rados.rados.0.smithi136.stdout:2413: finishing copy_from to smithi13667081-32 2024-09-12T23:46:46.512 INFO:tasks.rados.rados.0.smithi136.stdout:2413: got expected ENOENT (src dne) 2024-09-12T23:46:46.512 INFO:tasks.rados.rados.0.smithi136.stdout:2413: finishing copy_from racing read to smithi13667081-32 2024-09-12T23:46:46.516 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 44 v 1598 (ObjNum 776 snap 242 seq_num 776) dirty exists 2024-09-12T23:46:46.521 INFO:tasks.rados.rados.0.smithi136.stdout:2415: finishing copy_from to smithi13667081-21 2024-09-12T23:46:46.521 INFO:tasks.rados.rados.0.smithi136.stdout:2415: got expected ENOENT (src dne) 2024-09-12T23:46:46.522 INFO:tasks.rados.rados.0.smithi136.stdout:2415: finishing copy_from racing read to smithi13667081-21 2024-09-12T23:46:46.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:46 smithi136 bash[75164]: cluster 2024-09-12T23:46:45.474615+0000 mon.a (mon.0) 737 : cluster [WRN] Health check update: Degraded data redundancy: 45/516 objects degraded (8.721%), 8 pgs degraded (PG_DEGRADED) 2024-09-12T23:46:46.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:46 smithi136 bash[76371]: cluster 2024-09-12T23:46:45.474615+0000 mon.a (mon.0) 737 : cluster [WRN] Health check update: Degraded data redundancy: 45/516 objects degraded (8.721%), 8 pgs degraded (PG_DEGRADED) 2024-09-12T23:46:46.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:46 smithi181 bash[67587]: cluster 2024-09-12T23:46:45.474615+0000 mon.a (mon.0) 737 : cluster [WRN] Health check update: Degraded data redundancy: 45/516 objects degraded (8.721%), 8 pgs degraded (PG_DEGRADED) 2024-09-12T23:46:47.493 INFO:tasks.rados.rados.0.smithi136.stdout:2413: done (3 left) 2024-09-12T23:46:47.493 INFO:tasks.rados.rados.0.smithi136.stdout:2414: done (2 left) 2024-09-12T23:46:47.493 INFO:tasks.rados.rados.0.smithi136.stdout:2415: done (1 left) 2024-09-12T23:46:47.493 INFO:tasks.rados.rados.0.smithi136.stdout:2416: done (0 left) 2024-09-12T23:46:47.493 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 40 current snap is 245 2024-09-12T23:46:47.493 INFO:tasks.rados.rados.0.smithi136.stdout:2417: seq_num 781 ranges {1458176=688128,2146304=401408} 2024-09-12T23:46:47.528 INFO:tasks.rados.rados.0.smithi136.stdout:2417: writing smithi13667081-40 from 1458176 to 2146304 tid 1 2024-09-12T23:46:47.536 INFO:tasks.rados.rados.0.smithi136.stdout:2417: writing smithi13667081-40 from 2146304 to 2547712 tid 2 2024-09-12T23:46:47.536 INFO:tasks.rados.rados.0.smithi136.stdout:2418: setattr oid 23 current snap is 245 2024-09-12T23:46:47.544 INFO:tasks.rados.rados.0.smithi136.stdout:2419: delete oid 21 current snap is 245 2024-09-12T23:46:47.546 INFO:tasks.rados.rados.0.smithi136.stdout:2417: finishing write tid 1 to smithi13667081-40 2024-09-12T23:46:47.547 INFO:tasks.rados.rados.0.smithi136.stdout:2417: finishing write tid 2 to smithi13667081-40 2024-09-12T23:46:47.548 INFO:tasks.rados.rados.0.smithi136.stdout:2417: finishing write tid 3 to smithi13667081-40 2024-09-12T23:46:47.549 INFO:tasks.rados.rados.0.smithi136.stdout:2419: done (2 left) 2024-09-12T23:46:47.549 INFO:tasks.rados.rados.0.smithi136.stdout:2420: read oid 41 snap -1 2024-09-12T23:46:47.549 INFO:tasks.rados.rados.0.smithi136.stdout:2420: expect (ObjNum 753 snap 232 seq_num 753) 2024-09-12T23:46:47.549 INFO:tasks.rados.rados.0.smithi136.stdout:2421: rollback oid 10 current snap is 245 2024-09-12T23:46:47.549 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 10 to 243 2024-09-12T23:46:47.549 INFO:tasks.rados.rados.0.smithi136.stdout:2422: setattr oid 42 current snap is 245 2024-09-12T23:46:47.564 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 44 current snap is 245 2024-09-12T23:46:47.564 INFO:tasks.rados.rados.0.smithi136.stdout:2423: seq_num 783 ranges {1007616=720896,1728512=401408} 2024-09-12T23:46:47.576 INFO:tasks.rados.rados.0.smithi136.stdout:2423: writing smithi13667081-44 from 1007616 to 1728512 tid 1 2024-09-12T23:46:47.582 INFO:tasks.rados.rados.0.smithi136.stdout:2423: writing smithi13667081-44 from 1728512 to 2129920 tid 2 2024-09-12T23:46:47.582 INFO:tasks.rados.rados.0.smithi136.stdout:2424: copy_from oid 25 from oid 50 current snap is 245 2024-09-12T23:46:47.582 INFO:tasks.rados.rados.0.smithi136.stdout:2417: finishing write tid 4 to smithi13667081-40 2024-09-12T23:46:47.582 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 40 v 1700 (ObjNum 781 snap 245 seq_num 781) dirty exists 2024-09-12T23:46:47.582 INFO:tasks.rados.rados.0.smithi136.stdout:2417: left oid 40 (ObjNum 781 snap 245 seq_num 781) 2024-09-12T23:46:47.582 INFO:tasks.rados.rados.0.smithi136.stdout:2417: done (6 left) 2024-09-12T23:46:47.583 INFO:tasks.rados.rados.0.smithi136.stdout:2425: snap_remove snap 222 2024-09-12T23:46:47.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:47 smithi136 bash[75164]: cluster 2024-09-12T23:46:46.491589+0000 mon.a (mon.0) 738 : cluster [DBG] osdmap e551: 8 total, 8 up, 8 in 2024-09-12T23:46:47.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:47 smithi136 bash[75164]: cluster 2024-09-12T23:46:46.755876+0000 mgr.y (mgr.24661) 233 : cluster [DBG] pgmap v402: 121 pgs: 2 active+clean+snaptrim, 1 activating+degraded, 3 active+clean+snaptrim_wait, 1 activating, 19 active+undersized, 3 active+undersized+degraded, 92 active+clean; 302 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 754 KiB/s wr, 13 op/s; 28/507 objects degraded (5.523%) 2024-09-12T23:46:47.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:47 smithi136 bash[75164]: cluster 2024-09-12T23:46:47.495524+0000 mon.a (mon.0) 739 : cluster [DBG] osdmap e552: 8 total, 8 up, 8 in 2024-09-12T23:46:47.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:47 smithi136 bash[76371]: cluster 2024-09-12T23:46:46.491589+0000 mon.a (mon.0) 738 : cluster [DBG] osdmap e551: 8 total, 8 up, 8 in 2024-09-12T23:46:47.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:47 smithi136 bash[76371]: cluster 2024-09-12T23:46:46.755876+0000 mgr.y (mgr.24661) 233 : cluster [DBG] pgmap v402: 121 pgs: 2 active+clean+snaptrim, 1 activating+degraded, 3 active+clean+snaptrim_wait, 1 activating, 19 active+undersized, 3 active+undersized+degraded, 92 active+clean; 302 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 754 KiB/s wr, 13 op/s; 28/507 objects degraded (5.523%) 2024-09-12T23:46:47.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:47 smithi136 bash[76371]: cluster 2024-09-12T23:46:47.495524+0000 mon.a (mon.0) 739 : cluster [DBG] osdmap e552: 8 total, 8 up, 8 in 2024-09-12T23:46:47.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:47 smithi181 bash[67587]: cluster 2024-09-12T23:46:46.491589+0000 mon.a (mon.0) 738 : cluster [DBG] osdmap e551: 8 total, 8 up, 8 in 2024-09-12T23:46:47.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:47 smithi181 bash[67587]: cluster 2024-09-12T23:46:46.755876+0000 mgr.y (mgr.24661) 233 : cluster [DBG] pgmap v402: 121 pgs: 2 active+clean+snaptrim, 1 activating+degraded, 3 active+clean+snaptrim_wait, 1 activating, 19 active+undersized, 3 active+undersized+degraded, 92 active+clean; 302 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 754 KiB/s wr, 13 op/s; 28/507 objects degraded (5.523%) 2024-09-12T23:46:47.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:47 smithi181 bash[67587]: cluster 2024-09-12T23:46:47.495524+0000 mon.a (mon.0) 739 : cluster [DBG] osdmap e552: 8 total, 8 up, 8 in 2024-09-12T23:46:48.515 INFO:tasks.rados.rados.0.smithi136.stdout:2425: done (6 left) 2024-09-12T23:46:48.515 INFO:tasks.rados.rados.0.smithi136.stdout:2426: snap_create 2024-09-12T23:46:48.515 INFO:tasks.rados.rados.0.smithi136.stdout:2421: finishing rollback tid 0 to smithi13667081-10 2024-09-12T23:46:48.515 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 42 v 867 (ObjNum 145 snap 0 seq_num 4242596736) dirty exists 2024-09-12T23:46:48.515 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 23 v 1702 (ObjNum 741 snap 228 seq_num 741) dirty exists 2024-09-12T23:46:48.515 INFO:tasks.rados.rados.0.smithi136.stdout:2421: finishing rollback tid 1 to smithi13667081-10 2024-09-12T23:46:48.515 INFO:tasks.rados.rados.0.smithi136.stdout:2421: finishing rollback tid 2 to smithi13667081-10 2024-09-12T23:46:48.515 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 10 v 1199 (ObjNum 768 snap 236 seq_num 768) dirty exists 2024-09-12T23:46:48.516 INFO:tasks.rados.rados.0.smithi136.stdout:2420: expect (ObjNum 753 snap 232 seq_num 753) 2024-09-12T23:46:48.734 INFO:tasks.rados.rados.0.smithi136.stdout:2424: finishing copy_from to smithi13667081-25 2024-09-12T23:46:48.734 INFO:tasks.rados.rados.0.smithi136.stdout:2424: got expected ENOENT (src dne) 2024-09-12T23:46:48.734 INFO:tasks.rados.rados.0.smithi136.stdout:2424: finishing copy_from racing read to smithi13667081-25 2024-09-12T23:46:48.734 INFO:tasks.rados.rados.0.smithi136.stdout:2423: finishing write tid 1 to smithi13667081-44 2024-09-12T23:46:48.734 INFO:tasks.rados.rados.0.smithi136.stdout:2423: finishing write tid 2 to smithi13667081-44 2024-09-12T23:46:48.734 INFO:tasks.rados.rados.0.smithi136.stdout:2423: finishing write tid 3 to smithi13667081-44 2024-09-12T23:46:48.734 INFO:tasks.rados.rados.0.smithi136.stdout:2423: finishing write tid 4 to smithi13667081-44 2024-09-12T23:46:48.734 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 44 v 1601 (ObjNum 783 snap 245 seq_num 783) dirty exists 2024-09-12T23:46:48.735 INFO:tasks.rados.rados.0.smithi136.stdout:2423: left oid 44 (ObjNum 783 snap 245 seq_num 783) 2024-09-12T23:46:49.524 INFO:tasks.rados.rados.0.smithi136.stdout:2418: done (6 left) 2024-09-12T23:46:49.524 INFO:tasks.rados.rados.0.smithi136.stdout:2420: done (5 left) 2024-09-12T23:46:49.524 INFO:tasks.rados.rados.0.smithi136.stdout:2421: done (4 left) 2024-09-12T23:46:49.524 INFO:tasks.rados.rados.0.smithi136.stdout:2422: done (3 left) 2024-09-12T23:46:49.524 INFO:tasks.rados.rados.0.smithi136.stdout:2423: done (2 left) 2024-09-12T23:46:49.525 INFO:tasks.rados.rados.0.smithi136.stdout:2424: done (1 left) 2024-09-12T23:46:49.525 INFO:tasks.rados.rados.0.smithi136.stdout:2426: done (0 left) 2024-09-12T23:46:49.525 INFO:tasks.rados.rados.0.smithi136.stdout:2427: rollback oid 29 current snap is 246 2024-09-12T23:46:49.525 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 29 to 244 2024-09-12T23:46:49.525 INFO:tasks.rados.rados.0.smithi136.stdout:2428: snap_remove snap 245 2024-09-12T23:46:49.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:46:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:46:49] "GET /metrics HTTP/1.1" 200 36386 "" "Prometheus/2.43.0" 2024-09-12T23:46:49.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:49 smithi136 bash[75164]: cluster 2024-09-12T23:46:48.518550+0000 mon.a (mon.0) 740 : cluster [DBG] osdmap e553: 8 total, 8 up, 8 in 2024-09-12T23:46:49.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:49 smithi136 bash[75164]: cluster 2024-09-12T23:46:48.756632+0000 mgr.y (mgr.24661) 234 : cluster [DBG] pgmap v405: 121 pgs: 2 active+clean+snaptrim, 1 activating+degraded, 3 active+clean+snaptrim_wait, 1 activating, 114 active+clean; 304 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.2 MiB/s wr, 13 op/s; 5/510 objects degraded (0.980%); 1.2 MiB/s, 0 objects/s recovering 2024-09-12T23:46:49.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:49 smithi136 bash[76371]: cluster 2024-09-12T23:46:48.518550+0000 mon.a (mon.0) 740 : cluster [DBG] osdmap e553: 8 total, 8 up, 8 in 2024-09-12T23:46:49.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:49 smithi136 bash[76371]: cluster 2024-09-12T23:46:48.756632+0000 mgr.y (mgr.24661) 234 : cluster [DBG] pgmap v405: 121 pgs: 2 active+clean+snaptrim, 1 activating+degraded, 3 active+clean+snaptrim_wait, 1 activating, 114 active+clean; 304 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.2 MiB/s wr, 13 op/s; 5/510 objects degraded (0.980%); 1.2 MiB/s, 0 objects/s recovering 2024-09-12T23:46:49.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:49 smithi181 bash[67587]: cluster 2024-09-12T23:46:48.518550+0000 mon.a (mon.0) 740 : cluster [DBG] osdmap e553: 8 total, 8 up, 8 in 2024-09-12T23:46:49.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:49 smithi181 bash[67587]: cluster 2024-09-12T23:46:48.756632+0000 mgr.y (mgr.24661) 234 : cluster [DBG] pgmap v405: 121 pgs: 2 active+clean+snaptrim, 1 activating+degraded, 3 active+clean+snaptrim_wait, 1 activating, 114 active+clean; 304 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.2 MiB/s wr, 13 op/s; 5/510 objects degraded (0.980%); 1.2 MiB/s, 0 objects/s recovering 2024-09-12T23:46:50.536 INFO:tasks.rados.rados.0.smithi136.stdout:2428: done (1 left) 2024-09-12T23:46:50.536 INFO:tasks.rados.rados.0.smithi136.stdout:2429: rollback oid 42 current snap is 246 2024-09-12T23:46:50.536 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 42 to 240 2024-09-12T23:46:50.536 INFO:tasks.rados.rados.0.smithi136.stdout:2427: finishing rollback tid 1 to smithi13667081-29 2024-09-12T23:46:50.536 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 29 v 1238 (ObjNum 177 snap 0 seq_num 4243240736) dirty dne 2024-09-12T23:46:50.536 INFO:tasks.rados.rados.0.smithi136.stdout:2427: done (1 left) 2024-09-12T23:46:50.536 INFO:tasks.rados.rados.0.smithi136.stdout:2430: delete oid 11 current snap is 246 2024-09-12T23:46:50.546 INFO:tasks.rados.rados.0.smithi136.stdout:2429: finishing rollback tid 0 to smithi13667081-42 2024-09-12T23:46:50.547 INFO:tasks.rados.rados.0.smithi136.stdout:2429: finishing rollback tid 1 to smithi13667081-42 2024-09-12T23:46:50.548 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 42 v 869 (ObjNum 967185 snap 0 seq_num 1427293360) dirty dne 2024-09-12T23:46:50.551 INFO:tasks.rados.rados.0.smithi136.stdout:2429: done (1 left) 2024-09-12T23:46:50.551 INFO:tasks.rados.rados.0.smithi136.stdout:2430: done (0 left) 2024-09-12T23:46:50.551 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 34 current snap is 246 2024-09-12T23:46:50.551 INFO:tasks.rados.rados.0.smithi136.stdout:2431: seq_num 785 ranges {4579328=753664,5332992=401408} 2024-09-12T23:46:50.580 INFO:tasks.rados.rados.0.smithi136.stdout:2431: writing smithi13667081-34 from 4579328 to 5332992 tid 1 2024-09-12T23:46:50.583 INFO:tasks.rados.rados.0.smithi136.stdout:2431: writing smithi13667081-34 from 5332992 to 5734400 tid 2 2024-09-12T23:46:50.583 INFO:tasks.rados.rados.0.smithi136.stdout:2432: rmattr oid 44 current snap is 246 2024-09-12T23:46:50.584 INFO:tasks.rados.rados.0.smithi136.stdout:2433: rmattr oid 10 current snap is 246 2024-09-12T23:46:50.584 INFO:tasks.rados.rados.0.smithi136.stdout:2434: copy_from oid 32 from oid 16 current snap is 246 2024-09-12T23:46:50.584 INFO:tasks.rados.rados.0.smithi136.stdout:2435: copy_from oid 13 from oid 35 current snap is 246 2024-09-12T23:46:50.584 INFO:tasks.rados.rados.0.smithi136.stdout:2436: snap_remove snap 215 2024-09-12T23:46:50.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:50 smithi181 bash[67587]: cluster 2024-09-12T23:46:49.527469+0000 mon.a (mon.0) 741 : cluster [DBG] osdmap e554: 8 total, 8 up, 8 in 2024-09-12T23:46:51.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:50 smithi136 bash[75164]: cluster 2024-09-12T23:46:49.527469+0000 mon.a (mon.0) 741 : cluster [DBG] osdmap e554: 8 total, 8 up, 8 in 2024-09-12T23:46:51.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:50 smithi136 bash[76371]: cluster 2024-09-12T23:46:49.527469+0000 mon.a (mon.0) 741 : cluster [DBG] osdmap e554: 8 total, 8 up, 8 in 2024-09-12T23:46:51.554 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 10 v 1200 (ObjNum 768 snap 236 seq_num 768) dirty exists 2024-09-12T23:46:51.554 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 44 v 1602 (ObjNum 783 snap 245 seq_num 783) dirty exists 2024-09-12T23:46:51.554 INFO:tasks.rados.rados.0.smithi136.stdout:2431: finishing write tid 1 to smithi13667081-34 2024-09-12T23:46:51.554 INFO:tasks.rados.rados.0.smithi136.stdout:2431: finishing write tid 2 to smithi13667081-34 2024-09-12T23:46:51.554 INFO:tasks.rados.rados.0.smithi136.stdout:2431: finishing write tid 3 to smithi13667081-34 2024-09-12T23:46:51.554 INFO:tasks.rados.rados.0.smithi136.stdout:2431: finishing write tid 4 to smithi13667081-34 2024-09-12T23:46:51.554 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 34 v 1261 (ObjNum 785 snap 246 seq_num 785) dirty exists 2024-09-12T23:46:51.554 INFO:tasks.rados.rados.0.smithi136.stdout:2431: left oid 34 (ObjNum 785 snap 246 seq_num 785) 2024-09-12T23:46:51.554 INFO:tasks.rados.rados.0.smithi136.stdout:2434: finishing copy_from racing read to smithi13667081-32 2024-09-12T23:46:51.554 INFO:tasks.rados.rados.0.smithi136.stdout:2434: finishing copy_from to smithi13667081-32 2024-09-12T23:46:51.554 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 32 v 1597 (ObjNum 744 snap 228 seq_num 744) dirty exists 2024-09-12T23:46:51.554 INFO:tasks.rados.rados.0.smithi136.stdout:2435: finishing copy_from racing read to smithi13667081-13 2024-09-12T23:46:51.555 INFO:tasks.rados.rados.0.smithi136.stdout:2435: finishing copy_from to smithi13667081-13 2024-09-12T23:46:51.555 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 13 v 1577 (ObjNum 671 snap 196 seq_num 671) dirty exists 2024-09-12T23:46:51.555 INFO:tasks.rados.rados.0.smithi136.stdout:2431: done (5 left) 2024-09-12T23:46:51.555 INFO:tasks.rados.rados.0.smithi136.stdout:2432: done (4 left) 2024-09-12T23:46:51.555 INFO:tasks.rados.rados.0.smithi136.stdout:2433: done (3 left) 2024-09-12T23:46:51.555 INFO:tasks.rados.rados.0.smithi136.stdout:2434: done (2 left) 2024-09-12T23:46:51.555 INFO:tasks.rados.rados.0.smithi136.stdout:2435: done (1 left) 2024-09-12T23:46:51.555 INFO:tasks.rados.rados.0.smithi136.stdout:2436: done (0 left) 2024-09-12T23:46:51.555 INFO:tasks.rados.rados.0.smithi136.stdout:2437: rollback oid 17 current snap is 246 2024-09-12T23:46:51.555 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 17 to 243 2024-09-12T23:46:51.555 INFO:tasks.rados.rados.0.smithi136.stdout:2438: rollback oid 9 current snap is 246 2024-09-12T23:46:51.555 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 9 to 243 2024-09-12T23:46:51.555 INFO:tasks.rados.rados.0.smithi136.stdout:2439: read oid 48 snap -1 2024-09-12T23:46:51.556 INFO:tasks.rados.rados.0.smithi136.stdout:2439: expect (ObjNum 764 snap 236 seq_num 764) 2024-09-12T23:46:51.556 INFO:tasks.rados.rados.0.smithi136.stdout:2440: rollback oid 47 current snap is 246 2024-09-12T23:46:51.556 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 47 to 211 2024-09-12T23:46:51.556 INFO:tasks.rados.rados.0.smithi136.stdout:2441: snap_remove snap 230 2024-09-12T23:46:51.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:51 smithi181 bash[67587]: cluster 2024-09-12T23:46:50.538180+0000 mon.a (mon.0) 742 : cluster [DBG] osdmap e555: 8 total, 8 up, 8 in 2024-09-12T23:46:51.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:51 smithi181 bash[67587]: cluster 2024-09-12T23:46:50.757352+0000 mgr.y (mgr.24661) 235 : cluster [DBG] pgmap v408: 121 pgs: 2 active+clean+snaptrim, 3 active+clean+snaptrim_wait, 116 active+clean; 302 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 736 KiB/s wr, 12 op/s; 4.4 MiB/s, 2 objects/s recovering 2024-09-12T23:46:51.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:51 smithi181 bash[67587]: cluster 2024-09-12T23:46:51.370787+0000 mon.a (mon.0) 743 : cluster [WRN] Health check update: Degraded data redundancy: 5/510 objects degraded (0.980%), 1 pg degraded (PG_DEGRADED) 2024-09-12T23:46:52.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:51 smithi136 bash[75164]: cluster 2024-09-12T23:46:50.538180+0000 mon.a (mon.0) 742 : cluster [DBG] osdmap e555: 8 total, 8 up, 8 in 2024-09-12T23:46:52.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:51 smithi136 bash[75164]: cluster 2024-09-12T23:46:50.757352+0000 mgr.y (mgr.24661) 235 : cluster [DBG] pgmap v408: 121 pgs: 2 active+clean+snaptrim, 3 active+clean+snaptrim_wait, 116 active+clean; 302 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 736 KiB/s wr, 12 op/s; 4.4 MiB/s, 2 objects/s recovering 2024-09-12T23:46:52.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:51 smithi136 bash[75164]: cluster 2024-09-12T23:46:51.370787+0000 mon.a (mon.0) 743 : cluster [WRN] Health check update: Degraded data redundancy: 5/510 objects degraded (0.980%), 1 pg degraded (PG_DEGRADED) 2024-09-12T23:46:52.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:51 smithi136 bash[76371]: cluster 2024-09-12T23:46:50.538180+0000 mon.a (mon.0) 742 : cluster [DBG] osdmap e555: 8 total, 8 up, 8 in 2024-09-12T23:46:52.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:51 smithi136 bash[76371]: cluster 2024-09-12T23:46:50.757352+0000 mgr.y (mgr.24661) 235 : cluster [DBG] pgmap v408: 121 pgs: 2 active+clean+snaptrim, 3 active+clean+snaptrim_wait, 116 active+clean; 302 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 736 KiB/s wr, 12 op/s; 4.4 MiB/s, 2 objects/s recovering 2024-09-12T23:46:52.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:51 smithi136 bash[76371]: cluster 2024-09-12T23:46:51.370787+0000 mon.a (mon.0) 743 : cluster [WRN] Health check update: Degraded data redundancy: 5/510 objects degraded (0.980%), 1 pg degraded (PG_DEGRADED) 2024-09-12T23:46:52.558 INFO:tasks.rados.rados.0.smithi136.stdout:2441: done (4 left) 2024-09-12T23:46:52.558 INFO:tasks.rados.rados.0.smithi136.stdout:2442: rollback oid 40 current snap is 246 2024-09-12T23:46:52.558 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 40 to 234 2024-09-12T23:46:52.558 INFO:tasks.rados.rados.0.smithi136.stdout:2438: finishing rollback tid 0 to smithi13667081-9 2024-09-12T23:46:52.558 INFO:tasks.rados.rados.0.smithi136.stdout:2438: finishing rollback tid 1 to smithi13667081-9 2024-09-12T23:46:52.558 INFO:tasks.rados.rados.0.smithi136.stdout:2437: finishing rollback tid 1 to smithi13667081-17 2024-09-12T23:46:52.558 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 17 v 1527 (ObjNum 177 snap 0 seq_num 0) dirty dne 2024-09-12T23:46:52.558 INFO:tasks.rados.rados.0.smithi136.stdout:2438: finishing rollback tid 2 to smithi13667081-9 2024-09-12T23:46:52.558 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 9 v 765 (ObjNum 767 snap 236 seq_num 767) dirty exists 2024-09-12T23:46:52.558 INFO:tasks.rados.rados.0.smithi136.stdout:2440: finishing rollback tid 0 to smithi13667081-47 2024-09-12T23:46:52.559 INFO:tasks.rados.rados.0.smithi136.stdout:2440: finishing rollback tid 1 to smithi13667081-47 2024-09-12T23:46:52.559 INFO:tasks.rados.rados.0.smithi136.stdout:2440: finishing rollback tid 2 to smithi13667081-47 2024-09-12T23:46:52.559 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 47 v 769 (ObjNum 636 snap 188 seq_num 636) dirty exists 2024-09-12T23:46:52.559 INFO:tasks.rados.rados.0.smithi136.stdout:2439: expect (ObjNum 764 snap 236 seq_num 764) 2024-09-12T23:46:52.594 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:46:52 smithi181 bash[60471]: ts=2024-09-12T23:46:52.313Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:46:52.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:52 smithi181 bash[67587]: cluster 2024-09-12T23:46:51.531395+0000 mon.a (mon.0) 744 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/510 objects degraded (0.980%), 1 pg degraded) 2024-09-12T23:46:52.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:52 smithi181 bash[67587]: cluster 2024-09-12T23:46:51.531435+0000 mon.a (mon.0) 745 : cluster [INF] Cluster is now healthy 2024-09-12T23:46:52.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:52 smithi181 bash[67587]: cluster 2024-09-12T23:46:51.555517+0000 mon.a (mon.0) 746 : cluster [DBG] osdmap e556: 8 total, 8 up, 8 in 2024-09-12T23:46:52.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:52 smithi181 bash[67587]: audit 2024-09-12T23:46:51.993286+0000 mon.a (mon.0) 747 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:52.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:52 smithi181 bash[67587]: audit 2024-09-12T23:46:52.003860+0000 mon.a (mon.0) 748 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:52.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:52 smithi181 bash[67587]: audit 2024-09-12T23:46:52.006437+0000 mon.a (mon.0) 749 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:46:52.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:52 smithi181 bash[67587]: audit 2024-09-12T23:46:52.008761+0000 mon.a (mon.0) 750 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:46:52.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:52 smithi181 bash[67587]: audit 2024-09-12T23:46:52.019364+0000 mon.a (mon.0) 751 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:52.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:52 smithi181 bash[67587]: audit 2024-09-12T23:46:52.175317+0000 mon.a (mon.0) 752 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:46:52.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:52 smithi181 bash[67587]: audit 2024-09-12T23:46:52.180522+0000 mon.a (mon.0) 753 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:46:52.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:52 smithi181 bash[67587]: audit 2024-09-12T23:46:52.183357+0000 mon.a (mon.0) 754 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:46:52.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:52 smithi181 bash[67587]: audit 2024-09-12T23:46:52.185529+0000 mon.a (mon.0) 755 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:46:52.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:52 smithi181 bash[67587]: audit 2024-09-12T23:46:52.188258+0000 mon.a (mon.0) 756 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-12T23:46:52.697 INFO:tasks.rados.rados.0.smithi136.stdout:2442: finishing rollback tid 0 to smithi13667081-40 2024-09-12T23:46:52.698 INFO:tasks.rados.rados.0.smithi136.stdout:2442: finishing rollback tid 1 to smithi13667081-40 2024-09-12T23:46:52.698 INFO:tasks.rados.rados.0.smithi136.stdout:2442: finishing rollback tid 2 to smithi13667081-40 2024-09-12T23:46:52.698 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 40 v 1728 (ObjNum 739 snap 227 seq_num 739) dirty exists 2024-09-12T23:46:52.698 INFO:tasks.rados.rados.0.smithi136.stdout:2437: done (4 left) 2024-09-12T23:46:52.698 INFO:tasks.rados.rados.0.smithi136.stdout:2438: done (3 left) 2024-09-12T23:46:52.698 INFO:tasks.rados.rados.0.smithi136.stdout:2439: done (2 left) 2024-09-12T23:46:52.698 INFO:tasks.rados.rados.0.smithi136.stdout:2440: done (1 left) 2024-09-12T23:46:52.698 INFO:tasks.rados.rados.0.smithi136.stdout:2442: done (0 left) 2024-09-12T23:46:52.698 INFO:tasks.rados.rados.0.smithi136.stdout:2443: delete oid 46 current snap is 246 2024-09-12T23:46:52.702 INFO:tasks.rados.rados.0.smithi136.stdout:2443: done (0 left) 2024-09-12T23:46:52.702 INFO:tasks.rados.rados.0.smithi136.stdout:2444: rollback oid 11 current snap is 246 2024-09-12T23:46:52.702 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 11 to 238 2024-09-12T23:46:52.702 INFO:tasks.rados.rados.0.smithi136.stdout:2445: snap_create 2024-09-12T23:46:52.708 INFO:tasks.rados.rados.0.smithi136.stdout:2444: finishing rollback tid 1 to smithi13667081-11 2024-09-12T23:46:52.708 INFO:tasks.rados.rados.0.smithi136.stdout:2444: finishing rollback tid 2 to smithi13667081-11 2024-09-12T23:46:52.709 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 11 v 1738 (ObjNum 605 snap 174 seq_num 605) dirty exists 2024-09-12T23:46:53.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:52 smithi136 bash[76371]: cluster 2024-09-12T23:46:51.531395+0000 mon.a (mon.0) 744 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/510 objects degraded (0.980%), 1 pg degraded) 2024-09-12T23:46:53.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:52 smithi136 bash[76371]: cluster 2024-09-12T23:46:51.531435+0000 mon.a (mon.0) 745 : cluster [INF] Cluster is now healthy 2024-09-12T23:46:53.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:52 smithi136 bash[76371]: cluster 2024-09-12T23:46:51.555517+0000 mon.a (mon.0) 746 : cluster [DBG] osdmap e556: 8 total, 8 up, 8 in 2024-09-12T23:46:53.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:52 smithi136 bash[76371]: audit 2024-09-12T23:46:51.993286+0000 mon.a (mon.0) 747 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:53.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:52 smithi136 bash[76371]: audit 2024-09-12T23:46:52.003860+0000 mon.a (mon.0) 748 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:53.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:52 smithi136 bash[76371]: audit 2024-09-12T23:46:52.006437+0000 mon.a (mon.0) 749 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:46:53.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:52 smithi136 bash[76371]: audit 2024-09-12T23:46:52.008761+0000 mon.a (mon.0) 750 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:46:53.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:52 smithi136 bash[76371]: audit 2024-09-12T23:46:52.019364+0000 mon.a (mon.0) 751 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:53.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:52 smithi136 bash[76371]: audit 2024-09-12T23:46:52.175317+0000 mon.a (mon.0) 752 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:46:53.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:52 smithi136 bash[76371]: audit 2024-09-12T23:46:52.180522+0000 mon.a (mon.0) 753 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:46:53.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:52 smithi136 bash[76371]: audit 2024-09-12T23:46:52.183357+0000 mon.a (mon.0) 754 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:46:53.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:52 smithi136 bash[76371]: audit 2024-09-12T23:46:52.185529+0000 mon.a (mon.0) 755 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:46:53.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:52 smithi136 bash[76371]: audit 2024-09-12T23:46:52.188258+0000 mon.a (mon.0) 756 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-12T23:46:53.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:52 smithi136 bash[75164]: cluster 2024-09-12T23:46:51.531395+0000 mon.a (mon.0) 744 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/510 objects degraded (0.980%), 1 pg degraded) 2024-09-12T23:46:53.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:52 smithi136 bash[75164]: cluster 2024-09-12T23:46:51.531435+0000 mon.a (mon.0) 745 : cluster [INF] Cluster is now healthy 2024-09-12T23:46:53.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:52 smithi136 bash[75164]: cluster 2024-09-12T23:46:51.555517+0000 mon.a (mon.0) 746 : cluster [DBG] osdmap e556: 8 total, 8 up, 8 in 2024-09-12T23:46:53.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:52 smithi136 bash[75164]: audit 2024-09-12T23:46:51.993286+0000 mon.a (mon.0) 747 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:53.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:52 smithi136 bash[75164]: audit 2024-09-12T23:46:52.003860+0000 mon.a (mon.0) 748 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:53.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:52 smithi136 bash[75164]: audit 2024-09-12T23:46:52.006437+0000 mon.a (mon.0) 749 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:46:53.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:52 smithi136 bash[75164]: audit 2024-09-12T23:46:52.008761+0000 mon.a (mon.0) 750 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:46:53.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:52 smithi136 bash[75164]: audit 2024-09-12T23:46:52.019364+0000 mon.a (mon.0) 751 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:53.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:52 smithi136 bash[75164]: audit 2024-09-12T23:46:52.175317+0000 mon.a (mon.0) 752 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:46:53.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:52 smithi136 bash[75164]: audit 2024-09-12T23:46:52.180522+0000 mon.a (mon.0) 753 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:46:53.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:52 smithi136 bash[75164]: audit 2024-09-12T23:46:52.183357+0000 mon.a (mon.0) 754 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:46:53.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:52 smithi136 bash[75164]: audit 2024-09-12T23:46:52.185529+0000 mon.a (mon.0) 755 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:46:53.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:52 smithi136 bash[75164]: audit 2024-09-12T23:46:52.188258+0000 mon.a (mon.0) 756 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-12T23:46:53.567 INFO:tasks.rados.rados.0.smithi136.stdout:2444: done (1 left) 2024-09-12T23:46:53.567 INFO:tasks.rados.rados.0.smithi136.stdout:2445: done (0 left) 2024-09-12T23:46:53.567 INFO:tasks.rados.rados.0.smithi136.stdout:2446: snap_create 2024-09-12T23:46:53.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:53 smithi181 bash[67587]: audit 2024-09-12T23:46:52.188639+0000 mgr.y (mgr.24661) 236 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-12T23:46:53.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:53 smithi181 bash[67587]: cephadm 2024-09-12T23:46:52.190654+0000 mgr.y (mgr.24661) 237 : cephadm [INF] Upgrade: osd.6 is safe to restart 2024-09-12T23:46:53.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:53 smithi181 bash[67587]: cluster 2024-09-12T23:46:52.561059+0000 mon.a (mon.0) 757 : cluster [DBG] osdmap e557: 8 total, 8 up, 8 in 2024-09-12T23:46:53.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:53 smithi181 bash[67587]: cluster 2024-09-12T23:46:52.758045+0000 mgr.y (mgr.24661) 238 : cluster [DBG] pgmap v411: 121 pgs: 121 active+clean; 290 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 670 KiB/s wr, 7 op/s; 3.3 MiB/s, 1 objects/s recovering 2024-09-12T23:46:54.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:53 smithi136 bash[76371]: audit 2024-09-12T23:46:52.188639+0000 mgr.y (mgr.24661) 236 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-12T23:46:54.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:53 smithi136 bash[76371]: cephadm 2024-09-12T23:46:52.190654+0000 mgr.y (mgr.24661) 237 : cephadm [INF] Upgrade: osd.6 is safe to restart 2024-09-12T23:46:54.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:53 smithi136 bash[76371]: cluster 2024-09-12T23:46:52.561059+0000 mon.a (mon.0) 757 : cluster [DBG] osdmap e557: 8 total, 8 up, 8 in 2024-09-12T23:46:54.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:53 smithi136 bash[76371]: cluster 2024-09-12T23:46:52.758045+0000 mgr.y (mgr.24661) 238 : cluster [DBG] pgmap v411: 121 pgs: 121 active+clean; 290 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 670 KiB/s wr, 7 op/s; 3.3 MiB/s, 1 objects/s recovering 2024-09-12T23:46:54.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:53 smithi136 bash[75164]: audit 2024-09-12T23:46:52.188639+0000 mgr.y (mgr.24661) 236 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-12T23:46:54.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:53 smithi136 bash[75164]: cephadm 2024-09-12T23:46:52.190654+0000 mgr.y (mgr.24661) 237 : cephadm [INF] Upgrade: osd.6 is safe to restart 2024-09-12T23:46:54.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:53 smithi136 bash[75164]: cluster 2024-09-12T23:46:52.561059+0000 mon.a (mon.0) 757 : cluster [DBG] osdmap e557: 8 total, 8 up, 8 in 2024-09-12T23:46:54.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:53 smithi136 bash[75164]: cluster 2024-09-12T23:46:52.758045+0000 mgr.y (mgr.24661) 238 : cluster [DBG] pgmap v411: 121 pgs: 121 active+clean; 290 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 670 KiB/s wr, 7 op/s; 3.3 MiB/s, 1 objects/s recovering 2024-09-12T23:46:54.570 INFO:tasks.rados.rados.0.smithi136.stdout:2446: done (0 left) 2024-09-12T23:46:54.571 INFO:tasks.rados.rados.0.smithi136.stdout:2447: snap_create 2024-09-12T23:46:54.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:54 smithi181 bash[67587]: cluster 2024-09-12T23:46:53.570580+0000 mon.a (mon.0) 758 : cluster [DBG] osdmap e558: 8 total, 8 up, 8 in 2024-09-12T23:46:54.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:54 smithi181 bash[67587]: audit 2024-09-12T23:46:53.753771+0000 mon.a (mon.0) 759 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:54.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:54 smithi181 bash[67587]: audit 2024-09-12T23:46:53.755820+0000 mon.a (mon.0) 760 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:46:54.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:54 smithi181 bash[67587]: audit 2024-09-12T23:46:54.417221+0000 mon.a (mon.0) 761 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:54.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:54 smithi181 bash[67587]: audit 2024-09-12T23:46:54.419343+0000 mon.a (mon.0) 762 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-12T23:46:54.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:54 smithi181 bash[67587]: audit 2024-09-12T23:46:54.421089+0000 mon.a (mon.0) 763 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:46:55.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:54 smithi136 bash[75164]: cluster 2024-09-12T23:46:53.570580+0000 mon.a (mon.0) 758 : cluster [DBG] osdmap e558: 8 total, 8 up, 8 in 2024-09-12T23:46:55.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:54 smithi136 bash[75164]: audit 2024-09-12T23:46:53.753771+0000 mon.a (mon.0) 759 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:55.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:54 smithi136 bash[75164]: audit 2024-09-12T23:46:53.755820+0000 mon.a (mon.0) 760 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:46:55.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:54 smithi136 bash[75164]: audit 2024-09-12T23:46:54.417221+0000 mon.a (mon.0) 761 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:55.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:54 smithi136 bash[75164]: audit 2024-09-12T23:46:54.419343+0000 mon.a (mon.0) 762 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-12T23:46:55.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:54 smithi136 bash[75164]: audit 2024-09-12T23:46:54.421089+0000 mon.a (mon.0) 763 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:46:55.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:54 smithi136 bash[76371]: cluster 2024-09-12T23:46:53.570580+0000 mon.a (mon.0) 758 : cluster [DBG] osdmap e558: 8 total, 8 up, 8 in 2024-09-12T23:46:55.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:54 smithi136 bash[76371]: audit 2024-09-12T23:46:53.753771+0000 mon.a (mon.0) 759 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:55.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:54 smithi136 bash[76371]: audit 2024-09-12T23:46:53.755820+0000 mon.a (mon.0) 760 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:46:55.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:54 smithi136 bash[76371]: audit 2024-09-12T23:46:54.417221+0000 mon.a (mon.0) 761 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:55.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:54 smithi136 bash[76371]: audit 2024-09-12T23:46:54.419343+0000 mon.a (mon.0) 762 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-12T23:46:55.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:54 smithi136 bash[76371]: audit 2024-09-12T23:46:54.421089+0000 mon.a (mon.0) 763 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:46:55.571 INFO:tasks.rados.rados.0.smithi136.stdout:2447: done (0 left) 2024-09-12T23:46:55.571 INFO:tasks.rados.rados.0.smithi136.stdout:2448: read oid 21 snap -1 2024-09-12T23:46:55.572 INFO:tasks.rados.rados.0.smithi136.stdout:2448: expect deleted 2024-09-12T23:46:55.572 INFO:tasks.rados.rados.0.smithi136.stdout:2449: delete oid 10 current snap is 249 2024-09-12T23:46:55.580 INFO:tasks.rados.rados.0.smithi136.stdout:2448: done (1 left) 2024-09-12T23:46:55.581 INFO:tasks.rados.rados.0.smithi136.stdout:2449: done (0 left) 2024-09-12T23:46:55.581 INFO:tasks.rados.rados.0.smithi136.stdout:2450: rollback oid 38 current snap is 249 2024-09-12T23:46:55.581 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 38 to 241 2024-09-12T23:46:55.581 INFO:tasks.rados.rados.0.smithi136.stdout:2451: snap_remove snap 235 2024-09-12T23:46:55.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:55 smithi181 bash[67587]: cephadm 2024-09-12T23:46:54.407885+0000 mgr.y (mgr.24661) 239 : cephadm [INF] Upgrade: Updating osd.6 2024-09-12T23:46:55.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:55 smithi181 bash[67587]: cephadm 2024-09-12T23:46:54.426201+0000 mgr.y (mgr.24661) 240 : cephadm [INF] Deploying daemon osd.6 on smithi181 2024-09-12T23:46:55.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:55 smithi181 bash[67587]: cluster 2024-09-12T23:46:54.574208+0000 mon.a (mon.0) 764 : cluster [DBG] osdmap e559: 8 total, 8 up, 8 in 2024-09-12T23:46:55.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:55 smithi181 bash[67587]: cluster 2024-09-12T23:46:54.759109+0000 mgr.y (mgr.24661) 241 : cluster [DBG] pgmap v414: 121 pgs: 121 active+clean; 293 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.4 MiB/s wr, 9 op/s 2024-09-12T23:46:55.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:55 smithi181 bash[67587]: cluster 2024-09-12T23:46:55.574923+0000 mon.a (mon.0) 765 : cluster [DBG] osdmap e560: 8 total, 8 up, 8 in 2024-09-12T23:46:56.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:55 smithi136 bash[75164]: cephadm 2024-09-12T23:46:54.407885+0000 mgr.y (mgr.24661) 239 : cephadm [INF] Upgrade: Updating osd.6 2024-09-12T23:46:56.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:55 smithi136 bash[75164]: cephadm 2024-09-12T23:46:54.426201+0000 mgr.y (mgr.24661) 240 : cephadm [INF] Deploying daemon osd.6 on smithi181 2024-09-12T23:46:56.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:55 smithi136 bash[75164]: cluster 2024-09-12T23:46:54.574208+0000 mon.a (mon.0) 764 : cluster [DBG] osdmap e559: 8 total, 8 up, 8 in 2024-09-12T23:46:56.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:55 smithi136 bash[75164]: cluster 2024-09-12T23:46:54.759109+0000 mgr.y (mgr.24661) 241 : cluster [DBG] pgmap v414: 121 pgs: 121 active+clean; 293 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.4 MiB/s wr, 9 op/s 2024-09-12T23:46:56.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:55 smithi136 bash[75164]: cluster 2024-09-12T23:46:55.574923+0000 mon.a (mon.0) 765 : cluster [DBG] osdmap e560: 8 total, 8 up, 8 in 2024-09-12T23:46:56.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:55 smithi136 bash[76371]: cephadm 2024-09-12T23:46:54.407885+0000 mgr.y (mgr.24661) 239 : cephadm [INF] Upgrade: Updating osd.6 2024-09-12T23:46:56.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:55 smithi136 bash[76371]: cephadm 2024-09-12T23:46:54.426201+0000 mgr.y (mgr.24661) 240 : cephadm [INF] Deploying daemon osd.6 on smithi181 2024-09-12T23:46:56.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:55 smithi136 bash[76371]: cluster 2024-09-12T23:46:54.574208+0000 mon.a (mon.0) 764 : cluster [DBG] osdmap e559: 8 total, 8 up, 8 in 2024-09-12T23:46:56.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:55 smithi136 bash[76371]: cluster 2024-09-12T23:46:54.759109+0000 mgr.y (mgr.24661) 241 : cluster [DBG] pgmap v414: 121 pgs: 121 active+clean; 293 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.4 MiB/s wr, 9 op/s 2024-09-12T23:46:56.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:55 smithi136 bash[76371]: cluster 2024-09-12T23:46:55.574923+0000 mon.a (mon.0) 765 : cluster [DBG] osdmap e560: 8 total, 8 up, 8 in 2024-09-12T23:46:56.378 INFO:tasks.rados.rados.0.smithi136.stdout:2451: done (1 left) 2024-09-12T23:46:56.378 INFO:tasks.rados.rados.0.smithi136.stdout:2452: snap_create 2024-09-12T23:46:56.378 INFO:tasks.rados.rados.0.smithi136.stdout:2450: finishing rollback tid 0 to smithi13667081-38 2024-09-12T23:46:56.378 INFO:tasks.rados.rados.0.smithi136.stdout:2450: finishing rollback tid 1 to smithi13667081-38 2024-09-12T23:46:56.379 INFO:tasks.rados.rados.0.smithi136.stdout:2450: finishing rollback tid 2 to smithi13667081-38 2024-09-12T23:46:56.379 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 38 v 1580 (ObjNum 727 snap 224 seq_num 727) dirty exists 2024-09-12T23:46:57.388 INFO:tasks.rados.rados.0.smithi136.stdout:2450: done (1 left) 2024-09-12T23:46:57.388 INFO:tasks.rados.rados.0.smithi136.stdout:2452: done (0 left) 2024-09-12T23:46:57.388 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 40 current snap is 250 2024-09-12T23:46:57.388 INFO:tasks.rados.rados.0.smithi136.stdout:2453: seq_num 788 ranges {385024=802816,1187840=401408} 2024-09-12T23:46:57.407 INFO:tasks.rados.rados.0.smithi136.stdout:2453: writing smithi13667081-40 from 385024 to 1187840 tid 1 2024-09-12T23:46:57.415 INFO:tasks.rados.rados.0.smithi136.stdout:2453: writing smithi13667081-40 from 1187840 to 1589248 tid 2 2024-09-12T23:46:57.415 INFO:tasks.rados.rados.0.smithi136.stdout:2454: setattr oid 18 current snap is 250 2024-09-12T23:46:57.418 INFO:tasks.rados.rados.0.smithi136.stdout:2455: setattr oid 7 current snap is 250 2024-09-12T23:46:57.423 INFO:tasks.rados.rados.0.smithi136.stdout:2456: rollback oid 12 current snap is 250 2024-09-12T23:46:57.423 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 12 to 234 2024-09-12T23:46:57.423 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 4 current snap is 250 2024-09-12T23:46:57.423 INFO:tasks.rados.rados.0.smithi136.stdout:2457: seq_num 789 ranges {7454720=417792,7872512=663552,8536064=139264} 2024-09-12T23:46:57.498 INFO:tasks.rados.rados.0.smithi136.stdout:2457: writing smithi13667081-4 from 7454720 to 7872512 tid 1 2024-09-12T23:46:57.501 INFO:tasks.rados.rados.0.smithi136.stdout:2457: writing smithi13667081-4 from 7872512 to 8536064 tid 2 2024-09-12T23:46:57.503 INFO:tasks.rados.rados.0.smithi136.stdout:2457: writing smithi13667081-4 from 8536064 to 8675328 tid 3 2024-09-12T23:46:57.503 INFO:tasks.rados.rados.0.smithi136.stdout:2458: read oid 32 snap 244 2024-09-12T23:46:57.504 INFO:tasks.rados.rados.0.smithi136.stdout:2458: expect (ObjNum 752 snap 232 seq_num 752) 2024-09-12T23:46:57.504 INFO:tasks.rados.rados.0.smithi136.stdout:2459: copy_from oid 46 from oid 20 current snap is 250 2024-09-12T23:46:57.504 INFO:tasks.rados.rados.0.smithi136.stdout:2460: read oid 17 snap 234 2024-09-12T23:46:57.504 INFO:tasks.rados.rados.0.smithi136.stdout:2460: expect deleted 2024-09-12T23:46:57.504 INFO:tasks.rados.rados.0.smithi136.stdout:2461: snap_create 2024-09-12T23:46:57.504 INFO:tasks.rados.rados.0.smithi136.stdout:2453: finishing write tid 1 to smithi13667081-40 2024-09-12T23:46:57.504 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 18 v 945 (ObjNum 724 snap 224 seq_num 724) dirty exists 2024-09-12T23:46:57.504 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 7 v 946 (ObjNum 756 snap 233 seq_num 756) dirty exists 2024-09-12T23:46:57.504 INFO:tasks.rados.rados.0.smithi136.stdout:2456: finishing rollback tid 0 to smithi13667081-12 2024-09-12T23:46:57.504 INFO:tasks.rados.rados.0.smithi136.stdout:2453: finishing write tid 2 to smithi13667081-40 2024-09-12T23:46:57.504 INFO:tasks.rados.rados.0.smithi136.stdout:2453: finishing write tid 3 to smithi13667081-40 2024-09-12T23:46:57.504 INFO:tasks.rados.rados.0.smithi136.stdout:2456: finishing rollback tid 1 to smithi13667081-12 2024-09-12T23:46:57.504 INFO:tasks.rados.rados.0.smithi136.stdout:2456: finishing rollback tid 2 to smithi13667081-12 2024-09-12T23:46:57.505 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 12 v 1463 (ObjNum 720 snap 224 seq_num 720) dirty exists 2024-09-12T23:46:57.505 INFO:tasks.rados.rados.0.smithi136.stdout:2453: finishing write tid 4 to smithi13667081-40 2024-09-12T23:46:57.505 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 40 v 1750 (ObjNum 788 snap 250 seq_num 788) dirty exists 2024-09-12T23:46:57.505 INFO:tasks.rados.rados.0.smithi136.stdout:2453: left oid 40 (ObjNum 788 snap 250 seq_num 788) 2024-09-12T23:46:57.505 INFO:tasks.rados.rados.0.smithi136.stdout:2459: finishing copy_from to smithi13667081-46 2024-09-12T23:46:57.505 INFO:tasks.rados.rados.0.smithi136.stdout:2459: got expected ENOENT (src dne) 2024-09-12T23:46:57.505 INFO:tasks.rados.rados.0.smithi136.stdout:2459: finishing copy_from racing read to smithi13667081-46 2024-09-12T23:46:57.509 INFO:tasks.rados.rados.0.smithi136.stdout:2457: finishing write tid 1 to smithi13667081-4 2024-09-12T23:46:57.511 INFO:tasks.rados.rados.0.smithi136.stdout:2457: finishing write tid 2 to smithi13667081-4 2024-09-12T23:46:57.517 INFO:tasks.rados.rados.0.smithi136.stdout:2457: finishing write tid 3 to smithi13667081-4 2024-09-12T23:46:57.517 INFO:tasks.rados.rados.0.smithi136.stdout:2457: finishing write tid 4 to smithi13667081-4 2024-09-12T23:46:57.532 INFO:tasks.rados.rados.0.smithi136.stdout:2457: finishing write tid 5 to smithi13667081-4 2024-09-12T23:46:57.533 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 4 v 1601 (ObjNum 789 snap 250 seq_num 789) dirty exists 2024-09-12T23:46:57.533 INFO:tasks.rados.rados.0.smithi136.stdout:2457: left oid 4 (ObjNum 789 snap 250 seq_num 789) 2024-09-12T23:46:57.539 INFO:tasks.rados.rados.0.smithi136.stdout:2458: expect (ObjNum 752 snap 232 seq_num 752) 2024-09-12T23:46:57.738 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:57 smithi181 bash[67587]: cluster 2024-09-12T23:46:56.381742+0000 mon.a (mon.0) 766 : cluster [DBG] osdmap e561: 8 total, 8 up, 8 in 2024-09-12T23:46:57.739 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:57 smithi181 bash[67587]: cluster 2024-09-12T23:46:56.760143+0000 mgr.y (mgr.24661) 242 : cluster [DBG] pgmap v417: 121 pgs: 121 active+clean; 293 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 772 KiB/s wr, 4 op/s 2024-09-12T23:46:57.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:57 smithi136 bash[75164]: cluster 2024-09-12T23:46:56.381742+0000 mon.a (mon.0) 766 : cluster [DBG] osdmap e561: 8 total, 8 up, 8 in 2024-09-12T23:46:57.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:57 smithi136 bash[75164]: cluster 2024-09-12T23:46:56.760143+0000 mgr.y (mgr.24661) 242 : cluster [DBG] pgmap v417: 121 pgs: 121 active+clean; 293 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 772 KiB/s wr, 4 op/s 2024-09-12T23:46:57.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:57 smithi136 bash[76371]: cluster 2024-09-12T23:46:56.381742+0000 mon.a (mon.0) 766 : cluster [DBG] osdmap e561: 8 total, 8 up, 8 in 2024-09-12T23:46:57.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:57 smithi136 bash[76371]: cluster 2024-09-12T23:46:56.760143+0000 mgr.y (mgr.24661) 242 : cluster [DBG] pgmap v417: 121 pgs: 121 active+clean; 293 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 772 KiB/s wr, 4 op/s 2024-09-12T23:46:58.400 INFO:tasks.rados.rados.0.smithi136.stdout:2453: done (8 left) 2024-09-12T23:46:58.400 INFO:tasks.rados.rados.0.smithi136.stdout:2454: done (7 left) 2024-09-12T23:46:58.400 INFO:tasks.rados.rados.0.smithi136.stdout:2455: done (6 left) 2024-09-12T23:46:58.400 INFO:tasks.rados.rados.0.smithi136.stdout:2456: done (5 left) 2024-09-12T23:46:58.400 INFO:tasks.rados.rados.0.smithi136.stdout:2457: done (4 left) 2024-09-12T23:46:58.400 INFO:tasks.rados.rados.0.smithi136.stdout:2458: done (3 left) 2024-09-12T23:46:58.400 INFO:tasks.rados.rados.0.smithi136.stdout:2459: done (2 left) 2024-09-12T23:46:58.400 INFO:tasks.rados.rados.0.smithi136.stdout:2460: done (1 left) 2024-09-12T23:46:58.400 INFO:tasks.rados.rados.0.smithi136.stdout:2461: done (0 left) 2024-09-12T23:46:58.401 INFO:tasks.rados.rados.0.smithi136.stdout:2462: copy_from oid 23 from oid 16 current snap is 251 2024-09-12T23:46:58.401 INFO:tasks.rados.rados.0.smithi136.stdout:2463: setattr oid 43 current snap is 251 2024-09-12T23:46:58.401 INFO:tasks.rados.rados.0.smithi136.stdout:2464: snap_create 2024-09-12T23:46:58.415 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 43 v 1603 (ObjNum 742 snap 228 seq_num 742) dirty exists 2024-09-12T23:46:58.421 INFO:tasks.rados.rados.0.smithi136.stdout:2462: finishing copy_from racing read to smithi13667081-23 2024-09-12T23:46:58.422 INFO:tasks.rados.rados.0.smithi136.stdout:2462: finishing copy_from to smithi13667081-23 2024-09-12T23:46:58.422 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 23 v 1597 (ObjNum 744 snap 228 seq_num 744) dirty exists 2024-09-12T23:46:58.744 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:58 smithi181 bash[67587]: cluster 2024-09-12T23:46:57.390760+0000 mon.a (mon.0) 767 : cluster [DBG] osdmap e562: 8 total, 8 up, 8 in 2024-09-12T23:46:58.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:58 smithi136 bash[75164]: cluster 2024-09-12T23:46:57.390760+0000 mon.a (mon.0) 767 : cluster [DBG] osdmap e562: 8 total, 8 up, 8 in 2024-09-12T23:46:58.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:58 smithi136 bash[76371]: cluster 2024-09-12T23:46:57.390760+0000 mon.a (mon.0) 767 : cluster [DBG] osdmap e562: 8 total, 8 up, 8 in 2024-09-12T23:46:59.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:58 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:46:59.094 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:46:58 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:46:59.095 INFO:journalctl@ceph.mds.b.smithi181.stdout:Sep 12 23:46:58 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:46:59.095 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:46:58 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:46:59.095 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:46:58 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:46:59.096 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 12 23:46:58 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:46:59.096 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:46:58 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:46:59.096 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:46:58 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:46:59.096 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:46:58 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:46:59.097 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 12 23:46:58 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:46:59.097 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 12 23:46:58 smithi181 systemd[1]: Stopping Ceph osd.6 for da58a47c-715d-11ef-bceb-c7b262605968... 2024-09-12T23:46:59.097 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 12 23:46:58 smithi181 bash[39005]: debug 2024-09-12T23:46:58.975+0000 7fa16e288640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-12T23:46:59.097 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 12 23:46:58 smithi181 bash[39005]: debug 2024-09-12T23:46:58.975+0000 7fa16e288640 -1 osd.6 563 *** Got signal Terminated *** 2024-09-12T23:46:59.097 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 12 23:46:58 smithi181 bash[39005]: debug 2024-09-12T23:46:58.975+0000 7fa16e288640 -1 osd.6 563 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-12T23:46:59.420 INFO:tasks.rados.rados.0.smithi136.stdout:2462: done (2 left) 2024-09-12T23:46:59.420 INFO:tasks.rados.rados.0.smithi136.stdout:2463: done (1 left) 2024-09-12T23:46:59.420 INFO:tasks.rados.rados.0.smithi136.stdout:2464: done (0 left) 2024-09-12T23:46:59.421 INFO:tasks.rados.rados.0.smithi136.stdout:2465: copy_from oid 9 from oid 43 current snap is 252 2024-09-12T23:46:59.421 INFO:tasks.rados.rados.0.smithi136.stdout:2466: copy_from oid 20 from oid 18 current snap is 252 2024-09-12T23:46:59.421 INFO:tasks.rados.rados.0.smithi136.stdout:2467: read oid 15 snap 247 2024-09-12T23:46:59.421 INFO:tasks.rados.rados.0.smithi136.stdout:2467: expect (ObjNum 733 snap 226 seq_num 733) 2024-09-12T23:46:59.421 INFO:tasks.rados.rados.0.smithi136.stdout:2468: read oid 44 snap -1 2024-09-12T23:46:59.421 INFO:tasks.rados.rados.0.smithi136.stdout:2468: expect (ObjNum 783 snap 245 seq_num 783) 2024-09-12T23:46:59.421 INFO:tasks.rados.rados.0.smithi136.stdout:2469: read oid 11 snap -1 2024-09-12T23:46:59.421 INFO:tasks.rados.rados.0.smithi136.stdout:2469: expect (ObjNum 605 snap 174 seq_num 605) 2024-09-12T23:46:59.421 INFO:tasks.rados.rados.0.smithi136.stdout:2470: copy_from oid 6 from oid 1 current snap is 252 2024-09-12T23:46:59.421 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 26 current snap is 252 2024-09-12T23:46:59.422 INFO:tasks.rados.rados.0.smithi136.stdout:2471: seq_num 790 ranges {9019392=434176,9453568=548864,10002432=253952} 2024-09-12T23:46:59.488 INFO:tasks.rados.rados.0.smithi136.stdout:2471: writing smithi13667081-26 from 9019392 to 9453568 tid 1 2024-09-12T23:46:59.492 INFO:tasks.rados.rados.0.smithi136.stdout:2471: writing smithi13667081-26 from 9453568 to 10002432 tid 2 2024-09-12T23:46:59.494 INFO:tasks.rados.rados.0.smithi136.stdout:2471: writing smithi13667081-26 from 10002432 to 10256384 tid 3 2024-09-12T23:46:59.494 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 24 current snap is 252 2024-09-12T23:46:59.494 INFO:tasks.rados.rados.0.smithi136.stdout:2472: seq_num 791 ranges {8282112=450560,8732672=475136,9207808=327680} 2024-09-12T23:46:59.543 INFO:tasks.rados.rados.0.smithi136.stdout:2472: writing smithi13667081-24 from 8282112 to 8732672 tid 1 2024-09-12T23:46:59.546 INFO:tasks.rados.rados.0.smithi136.stdout:2472: writing smithi13667081-24 from 8732672 to 9207808 tid 2 2024-09-12T23:46:59.548 INFO:tasks.rados.rados.0.smithi136.stdout:2472: writing smithi13667081-24 from 9207808 to 9535488 tid 3 2024-09-12T23:46:59.548 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 4 current snap is 252 2024-09-12T23:46:59.548 INFO:tasks.rados.rados.0.smithi136.stdout:2473: seq_num 792 ranges {8675328=466944,9142272=761856,9904128=40960} 2024-09-12T23:46:59.599 INFO:tasks.rados.rados.0.smithi136.stdout:2473: writing smithi13667081-4 from 8675328 to 9142272 tid 1 2024-09-12T23:46:59.604 INFO:tasks.rados.rados.0.smithi136.stdout:2473: writing smithi13667081-4 from 9142272 to 9904128 tid 2 2024-09-12T23:46:59.604 INFO:tasks.rados.rados.0.smithi136.stdout:2473: writing smithi13667081-4 from 9904128 to 9945088 tid 3 2024-09-12T23:46:59.604 INFO:tasks.rados.rados.0.smithi136.stdout:2474: rollback oid 37 current snap is 252 2024-09-12T23:46:59.604 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 37 to 210 2024-09-12T23:46:59.605 INFO:tasks.rados.rados.0.smithi136.stdout:2467: expect (ObjNum 733 snap 226 seq_num 733) 2024-09-12T23:46:59.614 INFO:tasks.rados.rados.0.smithi136.stdout:2468: expect (ObjNum 783 snap 245 seq_num 783) 2024-09-12T23:46:59.685 INFO:tasks.rados.rados.0.smithi136.stdout:2469: expect (ObjNum 605 snap 174 seq_num 605) 2024-09-12T23:46:59.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:46:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:46:59] "GET /metrics HTTP/1.1" 200 36382 "" "Prometheus/2.43.0" 2024-09-12T23:46:59.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:59 smithi136 bash[75164]: cluster 2024-09-12T23:46:58.401482+0000 mon.a (mon.0) 768 : cluster [DBG] osdmap e563: 8 total, 8 up, 8 in 2024-09-12T23:46:59.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:59 smithi136 bash[75164]: cluster 2024-09-12T23:46:58.761149+0000 mgr.y (mgr.24661) 243 : cluster [DBG] pgmap v420: 121 pgs: 121 active+clean; 294 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 294 KiB/s wr, 2 op/s 2024-09-12T23:46:59.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:59 smithi136 bash[75164]: audit 2024-09-12T23:46:58.924444+0000 mon.a (mon.0) 769 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:59.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:46:59 smithi136 bash[75164]: cluster 2024-09-12T23:46:58.976826+0000 mon.a (mon.0) 770 : cluster [INF] osd.6 marked itself down and dead 2024-09-12T23:46:59.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:59 smithi136 bash[76371]: cluster 2024-09-12T23:46:58.401482+0000 mon.a (mon.0) 768 : cluster [DBG] osdmap e563: 8 total, 8 up, 8 in 2024-09-12T23:46:59.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:59 smithi136 bash[76371]: cluster 2024-09-12T23:46:58.761149+0000 mgr.y (mgr.24661) 243 : cluster [DBG] pgmap v420: 121 pgs: 121 active+clean; 294 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 294 KiB/s wr, 2 op/s 2024-09-12T23:46:59.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:59 smithi136 bash[76371]: audit 2024-09-12T23:46:58.924444+0000 mon.a (mon.0) 769 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:59.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:46:59 smithi136 bash[76371]: cluster 2024-09-12T23:46:58.976826+0000 mon.a (mon.0) 770 : cluster [INF] osd.6 marked itself down and dead 2024-09-12T23:46:59.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:59 smithi181 bash[67587]: cluster 2024-09-12T23:46:58.401482+0000 mon.a (mon.0) 768 : cluster [DBG] osdmap e563: 8 total, 8 up, 8 in 2024-09-12T23:46:59.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:59 smithi181 bash[67587]: cluster 2024-09-12T23:46:58.761149+0000 mgr.y (mgr.24661) 243 : cluster [DBG] pgmap v420: 121 pgs: 121 active+clean; 294 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 294 KiB/s wr, 2 op/s 2024-09-12T23:46:59.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:59 smithi181 bash[67587]: audit 2024-09-12T23:46:58.924444+0000 mon.a (mon.0) 769 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:46:59.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:46:59 smithi181 bash[67587]: cluster 2024-09-12T23:46:58.976826+0000 mon.a (mon.0) 770 : cluster [INF] osd.6 marked itself down and dead 2024-09-12T23:47:00.266 INFO:tasks.rados.rados.0.smithi136.stdout:2471: finishing write tid 1 to smithi13667081-26 2024-09-12T23:47:00.266 INFO:tasks.rados.rados.0.smithi136.stdout:2471: finishing write tid 2 to smithi13667081-26 2024-09-12T23:47:00.267 INFO:tasks.rados.rados.0.smithi136.stdout:2471: finishing write tid 3 to smithi13667081-26 2024-09-12T23:47:00.267 INFO:tasks.rados.rados.0.smithi136.stdout:2471: finishing write tid 4 to smithi13667081-26 2024-09-12T23:47:00.267 INFO:tasks.rados.rados.0.smithi136.stdout:2471: finishing write tid 5 to smithi13667081-26 2024-09-12T23:47:00.267 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 26 v 1264 (ObjNum 790 snap 252 seq_num 790) dirty exists 2024-09-12T23:47:00.267 INFO:tasks.rados.rados.0.smithi136.stdout:2471: left oid 26 (ObjNum 790 snap 252 seq_num 790) 2024-09-12T23:47:00.267 INFO:tasks.rados.rados.0.smithi136.stdout:2465: finishing copy_from racing read to smithi13667081-9 2024-09-12T23:47:00.267 INFO:tasks.rados.rados.0.smithi136.stdout:2465: finishing copy_from to smithi13667081-9 2024-09-12T23:47:00.267 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 9 v 1603 (ObjNum 742 snap 228 seq_num 742) dirty exists 2024-09-12T23:47:00.267 INFO:tasks.rados.rados.0.smithi136.stdout:2472: finishing write tid 1 to smithi13667081-24 2024-09-12T23:47:00.267 INFO:tasks.rados.rados.0.smithi136.stdout:2472: finishing write tid 2 to smithi13667081-24 2024-09-12T23:47:00.267 INFO:tasks.rados.rados.0.smithi136.stdout:2472: finishing write tid 3 to smithi13667081-24 2024-09-12T23:47:00.267 INFO:tasks.rados.rados.0.smithi136.stdout:2472: finishing write tid 4 to smithi13667081-24 2024-09-12T23:47:00.267 INFO:tasks.rados.rados.0.smithi136.stdout:2472: finishing write tid 5 to smithi13667081-24 2024-09-12T23:47:00.267 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 24 v 1268 (ObjNum 791 snap 252 seq_num 791) dirty exists 2024-09-12T23:47:00.268 INFO:tasks.rados.rados.0.smithi136.stdout:2472: left oid 24 (ObjNum 791 snap 252 seq_num 791) 2024-09-12T23:47:00.268 INFO:tasks.rados.rados.0.smithi136.stdout:2473: finishing write tid 1 to smithi13667081-4 2024-09-12T23:47:00.268 INFO:tasks.rados.rados.0.smithi136.stdout:2474: finishing rollback tid 0 to smithi13667081-37 2024-09-12T23:47:00.268 INFO:tasks.rados.rados.0.smithi136.stdout:2474: finishing rollback tid 1 to smithi13667081-37 2024-09-12T23:47:00.268 INFO:tasks.rados.rados.0.smithi136.stdout:2474: finishing rollback tid 2 to smithi13667081-37 2024-09-12T23:47:00.268 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 37 v 1264 (ObjNum 636 snap 188 seq_num 636) dirty exists 2024-09-12T23:47:00.268 INFO:tasks.rados.rados.0.smithi136.stdout:2473: finishing write tid 2 to smithi13667081-4 2024-09-12T23:47:00.268 INFO:tasks.rados.rados.0.smithi136.stdout:2473: finishing write tid 3 to smithi13667081-4 2024-09-12T23:47:00.268 INFO:tasks.rados.rados.0.smithi136.stdout:2473: finishing write tid 4 to smithi13667081-4 2024-09-12T23:47:00.268 INFO:tasks.rados.rados.0.smithi136.stdout:2473: finishing write tid 5 to smithi13667081-4 2024-09-12T23:47:00.268 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 4 v 1605 (ObjNum 792 snap 252 seq_num 792) dirty exists 2024-09-12T23:47:00.268 INFO:tasks.rados.rados.0.smithi136.stdout:2473: left oid 4 (ObjNum 792 snap 252 seq_num 792) 2024-09-12T23:47:00.268 INFO:tasks.rados.rados.0.smithi136.stdout:2465: done (9 left) 2024-09-12T23:47:00.268 INFO:tasks.rados.rados.0.smithi136.stdout:2467: done (8 left) 2024-09-12T23:47:00.269 INFO:tasks.rados.rados.0.smithi136.stdout:2468: done (7 left) 2024-09-12T23:47:00.269 INFO:tasks.rados.rados.0.smithi136.stdout:2469: done (6 left) 2024-09-12T23:47:00.269 INFO:tasks.rados.rados.0.smithi136.stdout:2471: done (5 left) 2024-09-12T23:47:00.269 INFO:tasks.rados.rados.0.smithi136.stdout:2472: done (4 left) 2024-09-12T23:47:00.269 INFO:tasks.rados.rados.0.smithi136.stdout:2473: done (3 left) 2024-09-12T23:47:00.269 INFO:tasks.rados.rados.0.smithi136.stdout:2474: done (2 left) 2024-09-12T23:47:00.269 INFO:tasks.rados.rados.0.smithi136.stdout:2475: rollback oid 44 current snap is 252 2024-09-12T23:47:00.269 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 44 to 210 2024-09-12T23:47:00.269 INFO:tasks.rados.rados.0.smithi136.stdout:2476: read oid 38 snap -1 2024-09-12T23:47:00.269 INFO:tasks.rados.rados.0.smithi136.stdout:2476: expect (ObjNum 727 snap 224 seq_num 727) 2024-09-12T23:47:00.269 INFO:tasks.rados.rados.0.smithi136.stdout:2477: read oid 22 snap -1 2024-09-12T23:47:00.269 INFO:tasks.rados.rados.0.smithi136.stdout:2477: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-12T23:47:00.269 INFO:tasks.rados.rados.0.smithi136.stdout:2478: delete oid 10 current snap is 252 2024-09-12T23:47:00.272 INFO:tasks.rados.rados.0.smithi136.stdout:2478: done (5 left) 2024-09-12T23:47:00.272 INFO:tasks.rados.rados.0.smithi136.stdout:2479: snap_create 2024-09-12T23:47:00.278 INFO:tasks.rados.rados.0.smithi136.stdout:2475: finishing rollback tid 0 to smithi13667081-44 2024-09-12T23:47:00.280 INFO:tasks.rados.rados.0.smithi136.stdout:2475: finishing rollback tid 1 to smithi13667081-44 2024-09-12T23:47:00.283 INFO:tasks.rados.rados.0.smithi136.stdout:2475: finishing rollback tid 2 to smithi13667081-44 2024-09-12T23:47:00.283 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 44 v 1606 (ObjNum 564 snap 162 seq_num 564) dirty exists 2024-09-12T23:47:00.301 INFO:tasks.rados.rados.0.smithi136.stdout:2476: expect (ObjNum 727 snap 224 seq_num 727) 2024-09-12T23:47:00.697 INFO:tasks.rados.rados.0.smithi136.stdout:2470: finishing copy_from to smithi13667081-6 2024-09-12T23:47:00.698 INFO:tasks.rados.rados.0.smithi136.stdout:2470: got expected ENOENT (src dne) 2024-09-12T23:47:00.698 INFO:tasks.rados.rados.0.smithi136.stdout:2477: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-12T23:47:00.700 INFO:tasks.rados.rados.0.smithi136.stdout:2470: finishing copy_from racing read to smithi13667081-6 2024-09-12T23:47:00.700 INFO:tasks.rados.rados.0.smithi136.stdout:2466: finishing copy_from racing read to smithi13667081-20 2024-09-12T23:47:00.700 INFO:tasks.rados.rados.0.smithi136.stdout:2466: finishing copy_from to smithi13667081-20 2024-09-12T23:47:00.700 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 20 v 945 (ObjNum 724 snap 224 seq_num 724) dirty exists 2024-09-12T23:47:00.700 INFO:tasks.rados.rados.0.smithi136.stdout:2466: done (5 left) 2024-09-12T23:47:00.700 INFO:tasks.rados.rados.0.smithi136.stdout:2470: done (4 left) 2024-09-12T23:47:00.700 INFO:tasks.rados.rados.0.smithi136.stdout:2475: done (3 left) 2024-09-12T23:47:00.700 INFO:tasks.rados.rados.0.smithi136.stdout:2476: done (2 left) 2024-09-12T23:47:00.700 INFO:tasks.rados.rados.0.smithi136.stdout:2477: done (1 left) 2024-09-12T23:47:00.700 INFO:tasks.rados.rados.0.smithi136.stdout:2479: done (0 left) 2024-09-12T23:47:00.700 INFO:tasks.rados.rados.0.smithi136.stdout:2480: rmattr oid 43 current snap is 253 2024-09-12T23:47:00.700 INFO:tasks.rados.rados.0.smithi136.stdout:2481: read oid 29 snap -1 2024-09-12T23:47:00.701 INFO:tasks.rados.rados.0.smithi136.stdout:2481: expect deleted 2024-09-12T23:47:00.701 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 31 current snap is 253 2024-09-12T23:47:00.701 INFO:tasks.rados.rados.0.smithi136.stdout:2482: seq_num 794 ranges {9699328=499712,10199040=532480,10731520=270336} 2024-09-12T23:47:00.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:00 smithi136 bash[75164]: cluster 2024-09-12T23:46:59.395795+0000 mon.a (mon.0) 771 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-12T23:47:00.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:00 smithi136 bash[75164]: cluster 2024-09-12T23:46:59.418130+0000 mon.a (mon.0) 772 : cluster [DBG] osdmap e564: 8 total, 7 up, 8 in 2024-09-12T23:47:00.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:00 smithi136 bash[76371]: cluster 2024-09-12T23:46:59.395795+0000 mon.a (mon.0) 771 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-12T23:47:00.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:00 smithi136 bash[76371]: cluster 2024-09-12T23:46:59.418130+0000 mon.a (mon.0) 772 : cluster [DBG] osdmap e564: 8 total, 7 up, 8 in 2024-09-12T23:47:00.800 INFO:tasks.rados.rados.0.smithi136.stdout:2482: writing smithi13667081-31 from 9699328 to 10199040 tid 1 2024-09-12T23:47:00.803 INFO:tasks.rados.rados.0.smithi136.stdout:2482: writing smithi13667081-31 from 10199040 to 10731520 tid 2 2024-09-12T23:47:00.806 INFO:tasks.rados.rados.0.smithi136.stdout:2482: writing smithi13667081-31 from 10731520 to 11001856 tid 3 2024-09-12T23:47:00.806 INFO:tasks.rados.rados.0.smithi136.stdout:2483: read oid 24 snap -1 2024-09-12T23:47:00.806 INFO:tasks.rados.rados.0.smithi136.stdout:2483: expect (ObjNum 791 snap 252 seq_num 791) 2024-09-12T23:47:00.806 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 7 current snap is 253 2024-09-12T23:47:00.806 INFO:tasks.rados.rados.0.smithi136.stdout:2484: seq_num 795 ranges {671744=524288,1196032=417792,1613824=385024} 2024-09-12T23:47:00.813 INFO:tasks.rados.rados.0.smithi136.stdout:2484: writing smithi13667081-7 from 671744 to 1196032 tid 1 2024-09-12T23:47:00.816 INFO:tasks.rados.rados.0.smithi136.stdout:2484: writing smithi13667081-7 from 1196032 to 1613824 tid 2 2024-09-12T23:47:00.818 INFO:tasks.rados.rados.0.smithi136.stdout:2484: writing smithi13667081-7 from 1613824 to 1998848 tid 3 2024-09-12T23:47:00.818 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 34 current snap is 253 2024-09-12T23:47:00.818 INFO:tasks.rados.rados.0.smithi136.stdout:2485: seq_num 796 ranges {5734400=540672,6275072=704512,6979584=98304} 2024-09-12T23:47:00.842 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 12 23:47:00 smithi181 bash[80099]: ceph-da58a47c-715d-11ef-bceb-c7b262605968-osd-6 2024-09-12T23:47:00.842 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:00 smithi181 bash[67587]: cluster 2024-09-12T23:46:59.395795+0000 mon.a (mon.0) 771 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-12T23:47:00.842 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:00 smithi181 bash[67587]: cluster 2024-09-12T23:46:59.418130+0000 mon.a (mon.0) 772 : cluster [DBG] osdmap e564: 8 total, 7 up, 8 in 2024-09-12T23:47:00.854 INFO:tasks.rados.rados.0.smithi136.stdout:2485: writing smithi13667081-34 from 5734400 to 6275072 tid 1 2024-09-12T23:47:00.857 INFO:tasks.rados.rados.0.smithi136.stdout:2485: writing smithi13667081-34 from 6275072 to 6979584 tid 2 2024-09-12T23:47:00.858 INFO:tasks.rados.rados.0.smithi136.stdout:2485: writing smithi13667081-34 from 6979584 to 7077888 tid 3 2024-09-12T23:47:00.858 INFO:tasks.rados.rados.0.smithi136.stdout:2486: rmattr oid 8 current snap is 253 2024-09-12T23:47:00.859 INFO:tasks.rados.rados.0.smithi136.stdout:2486: done (6 left) 2024-09-12T23:47:00.859 INFO:tasks.rados.rados.0.smithi136.stdout:2487: rollback oid 32 current snap is 253 2024-09-12T23:47:00.859 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 32 to 237 2024-09-12T23:47:00.859 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 43 v 1607 (ObjNum 742 snap 228 seq_num 742) dirty exists 2024-09-12T23:47:00.859 INFO:tasks.rados.rados.0.smithi136.stdout:2482: finishing write tid 1 to smithi13667081-31 2024-09-12T23:47:00.859 INFO:tasks.rados.rados.0.smithi136.stdout:2482: finishing write tid 2 to smithi13667081-31 2024-09-12T23:47:00.859 INFO:tasks.rados.rados.0.smithi136.stdout:2482: finishing write tid 3 to smithi13667081-31 2024-09-12T23:47:00.859 INFO:tasks.rados.rados.0.smithi136.stdout:2482: finishing write tid 4 to smithi13667081-31 2024-09-12T23:47:00.859 INFO:tasks.rados.rados.0.smithi136.stdout:2482: finishing write tid 5 to smithi13667081-31 2024-09-12T23:47:00.859 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 31 v 1467 (ObjNum 794 snap 253 seq_num 794) dirty exists 2024-09-12T23:47:00.859 INFO:tasks.rados.rados.0.smithi136.stdout:2482: left oid 31 (ObjNum 794 snap 253 seq_num 794) 2024-09-12T23:47:00.859 INFO:tasks.rados.rados.0.smithi136.stdout:2484: finishing write tid 1 to smithi13667081-7 2024-09-12T23:47:00.859 INFO:tasks.rados.rados.0.smithi136.stdout:2484: finishing write tid 2 to smithi13667081-7 2024-09-12T23:47:00.859 INFO:tasks.rados.rados.0.smithi136.stdout:2484: finishing write tid 3 to smithi13667081-7 2024-09-12T23:47:00.860 INFO:tasks.rados.rados.0.smithi136.stdout:2484: finishing write tid 4 to smithi13667081-7 2024-09-12T23:47:00.860 INFO:tasks.rados.rados.0.smithi136.stdout:2480: done (6 left) 2024-09-12T23:47:00.860 INFO:tasks.rados.rados.0.smithi136.stdout:2481: done (5 left) 2024-09-12T23:47:00.860 INFO:tasks.rados.rados.0.smithi136.stdout:2482: done (4 left) 2024-09-12T23:47:00.860 INFO:tasks.rados.rados.0.smithi136.stdout:2488: delete oid 33 current snap is 253 2024-09-12T23:47:00.860 INFO:tasks.rados.rados.0.smithi136.stdout:2484: finishing write tid 5 to smithi13667081-7 2024-09-12T23:47:00.860 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 7 v 950 (ObjNum 795 snap 253 seq_num 795) dirty exists 2024-09-12T23:47:00.860 INFO:tasks.rados.rados.0.smithi136.stdout:2484: left oid 7 (ObjNum 795 snap 253 seq_num 795) 2024-09-12T23:47:00.863 INFO:tasks.rados.rados.0.smithi136.stdout:2487: finishing rollback tid 0 to smithi13667081-32 2024-09-12T23:47:00.864 INFO:tasks.rados.rados.0.smithi136.stdout:2487: finishing rollback tid 1 to smithi13667081-32 2024-09-12T23:47:00.866 INFO:tasks.rados.rados.0.smithi136.stdout:2487: finishing rollback tid 2 to smithi13667081-32 2024-09-12T23:47:00.866 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 32 v 1608 (ObjNum 752 snap 232 seq_num 752) dirty exists 2024-09-12T23:47:00.875 INFO:tasks.rados.rados.0.smithi136.stdout:2483: expect (ObjNum 791 snap 252 seq_num 791) 2024-09-12T23:47:01.344 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 12 23:47:00 smithi181 bash[80165]: Error response from daemon: No such container: ceph-da58a47c-715d-11ef-bceb-c7b262605968-osd-6 2024-09-12T23:47:01.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:01 smithi136 bash[76371]: cluster 2024-09-12T23:47:00.425576+0000 mon.a (mon.0) 773 : cluster [DBG] osdmap e565: 8 total, 7 up, 8 in 2024-09-12T23:47:01.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:01 smithi136 bash[76371]: cluster 2024-09-12T23:47:00.762045+0000 mgr.y (mgr.24661) 244 : cluster [DBG] pgmap v423: 121 pgs: 6 active+undersized, 15 peering, 7 stale+active+clean, 1 active+undersized+degraded, 92 active+clean; 294 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 326 KiB/s wr, 12 op/s; 4/495 objects degraded (0.808%) 2024-09-12T23:47:01.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:01 smithi136 bash[76371]: cluster 2024-09-12T23:47:01.383839+0000 mon.a (mon.0) 774 : cluster [DBG] osdmap e566: 8 total, 7 up, 8 in 2024-09-12T23:47:01.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:01 smithi136 bash[75164]: cluster 2024-09-12T23:47:00.425576+0000 mon.a (mon.0) 773 : cluster [DBG] osdmap e565: 8 total, 7 up, 8 in 2024-09-12T23:47:01.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:01 smithi136 bash[75164]: cluster 2024-09-12T23:47:00.762045+0000 mgr.y (mgr.24661) 244 : cluster [DBG] pgmap v423: 121 pgs: 6 active+undersized, 15 peering, 7 stale+active+clean, 1 active+undersized+degraded, 92 active+clean; 294 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 326 KiB/s wr, 12 op/s; 4/495 objects degraded (0.808%) 2024-09-12T23:47:01.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:01 smithi136 bash[75164]: cluster 2024-09-12T23:47:01.383839+0000 mon.a (mon.0) 774 : cluster [DBG] osdmap e566: 8 total, 7 up, 8 in 2024-09-12T23:47:01.783 INFO:tasks.rados.rados.0.smithi136.stdout:2485: finishing write tid 1 to smithi13667081-34 2024-09-12T23:47:01.783 INFO:tasks.rados.rados.0.smithi136.stdout:2485: finishing write tid 2 to smithi13667081-34 2024-09-12T23:47:01.783 INFO:tasks.rados.rados.0.smithi136.stdout:2485: finishing write tid 3 to smithi13667081-34 2024-09-12T23:47:01.783 INFO:tasks.rados.rados.0.smithi136.stdout:2485: finishing write tid 4 to smithi13667081-34 2024-09-12T23:47:01.783 INFO:tasks.rados.rados.0.smithi136.stdout:2485: finishing write tid 5 to smithi13667081-34 2024-09-12T23:47:01.783 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 34 v 1268 (ObjNum 796 snap 253 seq_num 796) dirty exists 2024-09-12T23:47:01.783 INFO:tasks.rados.rados.0.smithi136.stdout:2485: left oid 34 (ObjNum 796 snap 253 seq_num 796) 2024-09-12T23:47:01.783 INFO:tasks.rados.rados.0.smithi136.stdout:2483: done (4 left) 2024-09-12T23:47:01.783 INFO:tasks.rados.rados.0.smithi136.stdout:2484: done (3 left) 2024-09-12T23:47:01.784 INFO:tasks.rados.rados.0.smithi136.stdout:2485: done (2 left) 2024-09-12T23:47:01.784 INFO:tasks.rados.rados.0.smithi136.stdout:2487: done (1 left) 2024-09-12T23:47:01.784 INFO:tasks.rados.rados.0.smithi136.stdout:2488: done (0 left) 2024-09-12T23:47:01.784 INFO:tasks.rados.rados.0.smithi136.stdout:2489: setattr oid 7 current snap is 253 2024-09-12T23:47:01.784 INFO:tasks.rados.rados.0.smithi136.stdout:2490: rmattr oid 5 current snap is 253 2024-09-12T23:47:01.785 INFO:tasks.rados.rados.0.smithi136.stdout:2491: read oid 13 snap -1 2024-09-12T23:47:01.785 INFO:tasks.rados.rados.0.smithi136.stdout:2491: expect (ObjNum 671 snap 196 seq_num 671) 2024-09-12T23:47:01.785 INFO:tasks.rados.rados.0.smithi136.stdout:2492: copy_from oid 20 from oid 36 current snap is 253 2024-09-12T23:47:01.785 INFO:tasks.rados.rados.0.smithi136.stdout:2493: setattr oid 8 current snap is 253 2024-09-12T23:47:01.787 INFO:tasks.rados.rados.0.smithi136.stdout:2494: rollback oid 16 current snap is 253 2024-09-12T23:47:01.787 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 16 to 211 2024-09-12T23:47:01.787 INFO:tasks.rados.rados.0.smithi136.stdout:2495: rollback oid 37 current snap is 253 2024-09-12T23:47:01.787 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 37 to 252 2024-09-12T23:47:01.787 INFO:tasks.rados.rados.0.smithi136.stdout:2496: copy_from oid 29 from oid 15 current snap is 253 2024-09-12T23:47:01.787 INFO:tasks.rados.rados.0.smithi136.stdout:2497: read oid 33 snap 237 2024-09-12T23:47:01.787 INFO:tasks.rados.rados.0.smithi136.stdout:2497: expect (ObjNum 719 snap 223 seq_num 719) 2024-09-12T23:47:01.787 INFO:tasks.rados.rados.0.smithi136.stdout:2498: delete oid 45 current snap is 253 2024-09-12T23:47:01.795 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 7 v 951 (ObjNum 795 snap 253 seq_num 795) dirty exists 2024-09-12T23:47:01.795 INFO:tasks.rados.rados.0.smithi136.stdout:2495: finishing rollback tid 0 to smithi13667081-37 2024-09-12T23:47:01.796 INFO:tasks.rados.rados.0.smithi136.stdout:2495: finishing rollback tid 1 to smithi13667081-37 2024-09-12T23:47:01.796 INFO:tasks.rados.rados.0.smithi136.stdout:2495: finishing rollback tid 2 to smithi13667081-37 2024-09-12T23:47:01.796 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 37 v 1272 (ObjNum 636 snap 188 seq_num 636) dirty exists 2024-09-12T23:47:01.797 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 8 v 1242 (ObjNum 33 snap 0 seq_num 4244467568) dirty exists 2024-09-12T23:47:01.797 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 5 v 1754 (ObjNum 567 snap 162 seq_num 567) dirty exists 2024-09-12T23:47:01.798 INFO:tasks.rados.rados.0.smithi136.stdout:2494: finishing rollback tid 0 to smithi13667081-16 2024-09-12T23:47:01.799 INFO:tasks.rados.rados.0.smithi136.stdout:2494: finishing rollback tid 1 to smithi13667081-16 2024-09-12T23:47:01.799 INFO:tasks.rados.rados.0.smithi136.stdout:2494: finishing rollback tid 2 to smithi13667081-16 2024-09-12T23:47:01.799 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 16 v 1610 (ObjNum 616 snap 179 seq_num 616) dirty exists 2024-09-12T23:47:01.800 INFO:tasks.rados.rados.0.smithi136.stdout:2497: expect (ObjNum 719 snap 223 seq_num 719) 2024-09-12T23:47:01.804 INFO:tasks.rados.rados.0.smithi136.stdout:2489: done (9 left) 2024-09-12T23:47:01.804 INFO:tasks.rados.rados.0.smithi136.stdout:2490: done (8 left) 2024-09-12T23:47:01.804 INFO:tasks.rados.rados.0.smithi136.stdout:2493: done (7 left) 2024-09-12T23:47:01.804 INFO:tasks.rados.rados.0.smithi136.stdout:2494: done (6 left) 2024-09-12T23:47:01.804 INFO:tasks.rados.rados.0.smithi136.stdout:2495: done (5 left) 2024-09-12T23:47:01.804 INFO:tasks.rados.rados.0.smithi136.stdout:2497: done (4 left) 2024-09-12T23:47:01.804 INFO:tasks.rados.rados.0.smithi136.stdout:2498: done (3 left) 2024-09-12T23:47:01.804 INFO:tasks.rados.rados.0.smithi136.stdout:2499: rmattr oid 45 current snap is 253 2024-09-12T23:47:01.804 INFO:tasks.rados.rados.0.smithi136.stdout:2499: done (3 left) 2024-09-12T23:47:01.804 INFO:tasks.rados.rados.0.smithi136.stdout:2500: delete oid 41 current snap is 253 2024-09-12T23:47:01.807 INFO:tasks.rados.rados.0.smithi136.stdout:2500: done (3 left) 2024-09-12T23:47:01.807 INFO:tasks.rados.rados.0.smithi136.stdout:2501: copy_from oid 7 from oid 25 current snap is 253 2024-09-12T23:47:01.807 INFO:tasks.rados.rados.0.smithi136.stdout:2502: read oid 11 snap 252 2024-09-12T23:47:01.807 INFO:tasks.rados.rados.0.smithi136.stdout:2502: expect (ObjNum 605 snap 174 seq_num 605) 2024-09-12T23:47:01.807 INFO:tasks.rados.rados.0.smithi136.stdout:2503: read oid 34 snap -1 2024-09-12T23:47:01.807 INFO:tasks.rados.rados.0.smithi136.stdout:2503: expect (ObjNum 796 snap 253 seq_num 796) 2024-09-12T23:47:01.807 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 4 current snap is 253 2024-09-12T23:47:01.807 INFO:tasks.rados.rados.0.smithi136.stdout:2504: seq_num 800 ranges {9945088=606208,10551296=688128,11239424=114688} 2024-09-12T23:47:01.828 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:01 smithi181 bash[67587]: cluster 2024-09-12T23:47:00.425576+0000 mon.a (mon.0) 773 : cluster [DBG] osdmap e565: 8 total, 7 up, 8 in 2024-09-12T23:47:01.829 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:01 smithi181 bash[67587]: cluster 2024-09-12T23:47:00.762045+0000 mgr.y (mgr.24661) 244 : cluster [DBG] pgmap v423: 121 pgs: 6 active+undersized, 15 peering, 7 stale+active+clean, 1 active+undersized+degraded, 92 active+clean; 294 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 326 KiB/s wr, 12 op/s; 4/495 objects degraded (0.808%) 2024-09-12T23:47:01.829 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:01 smithi181 bash[67587]: cluster 2024-09-12T23:47:01.383839+0000 mon.a (mon.0) 774 : cluster [DBG] osdmap e566: 8 total, 7 up, 8 in 2024-09-12T23:47:01.864 INFO:tasks.rados.rados.0.smithi136.stdout:2504: writing smithi13667081-4 from 9945088 to 10551296 tid 1 2024-09-12T23:47:01.868 INFO:tasks.rados.rados.0.smithi136.stdout:2504: writing smithi13667081-4 from 10551296 to 11239424 tid 2 2024-09-12T23:47:01.869 INFO:tasks.rados.rados.0.smithi136.stdout:2504: writing smithi13667081-4 from 11239424 to 11354112 tid 3 2024-09-12T23:47:01.869 INFO:tasks.rados.rados.0.smithi136.stdout:2505: delete oid 3 current snap is 253 2024-09-12T23:47:01.869 INFO:tasks.rados.rados.0.smithi136.stdout:2501: finishing copy_from to smithi13667081-7 2024-09-12T23:47:01.869 INFO:tasks.rados.rados.0.smithi136.stdout:2501: got expected ENOENT (src dne) 2024-09-12T23:47:01.870 INFO:tasks.rados.rados.0.smithi136.stdout:2501: finishing copy_from racing read to smithi13667081-7 2024-09-12T23:47:01.870 INFO:tasks.rados.rados.0.smithi136.stdout:2491: expect (ObjNum 671 snap 196 seq_num 671) 2024-09-12T23:47:01.968 INFO:tasks.rados.rados.0.smithi136.stdout:2496: finishing copy_from racing read to smithi13667081-29 2024-09-12T23:47:01.969 INFO:tasks.rados.rados.0.smithi136.stdout:2492: finishing copy_from racing read to smithi13667081-20 2024-09-12T23:47:01.969 INFO:tasks.rados.rados.0.smithi136.stdout:2492: finishing copy_from to smithi13667081-20 2024-09-12T23:47:01.969 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 20 v 1241 (ObjNum 610 snap 175 seq_num 610) dirty exists 2024-09-12T23:47:01.969 INFO:tasks.rados.rados.0.smithi136.stdout:2496: finishing copy_from to smithi13667081-29 2024-09-12T23:47:01.969 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 29 v 1460 (ObjNum 733 snap 226 seq_num 733) dirty exists 2024-09-12T23:47:01.969 INFO:tasks.rados.rados.0.smithi136.stdout:2491: done (7 left) 2024-09-12T23:47:01.969 INFO:tasks.rados.rados.0.smithi136.stdout:2492: done (6 left) 2024-09-12T23:47:01.969 INFO:tasks.rados.rados.0.smithi136.stdout:2496: done (5 left) 2024-09-12T23:47:01.969 INFO:tasks.rados.rados.0.smithi136.stdout:2501: done (4 left) 2024-09-12T23:47:01.969 INFO:tasks.rados.rados.0.smithi136.stdout:2505: done (3 left) 2024-09-12T23:47:01.969 INFO:tasks.rados.rados.0.smithi136.stdout:2506: delete oid 32 current snap is 253 2024-09-12T23:47:01.969 INFO:tasks.rados.rados.0.smithi136.stdout:2503: expect (ObjNum 796 snap 253 seq_num 796) 2024-09-12T23:47:02.385 INFO:tasks.rados.rados.0.smithi136.stdout:2502: expect (ObjNum 605 snap 174 seq_num 605) 2024-09-12T23:47:02.424 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:47:02 smithi181 bash[60471]: ts=2024-09-12T23:47:02.313Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:47:02.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:02 smithi136 bash[76371]: cluster 2024-09-12T23:47:01.417253+0000 mon.a (mon.0) 775 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-12T23:47:02.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:02 smithi136 bash[76371]: cluster 2024-09-12T23:47:01.417305+0000 mon.a (mon.0) 776 : cluster [WRN] Health check failed: Degraded data redundancy: 4/495 objects degraded (0.808%), 1 pg degraded (PG_DEGRADED) 2024-09-12T23:47:02.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:02 smithi136 bash[75164]: cluster 2024-09-12T23:47:01.417253+0000 mon.a (mon.0) 775 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-12T23:47:02.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:02 smithi136 bash[75164]: cluster 2024-09-12T23:47:01.417305+0000 mon.a (mon.0) 776 : cluster [WRN] Health check failed: Degraded data redundancy: 4/495 objects degraded (0.808%), 1 pg degraded (PG_DEGRADED) 2024-09-12T23:47:02.778 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:02 smithi181 bash[67587]: cluster 2024-09-12T23:47:01.417253+0000 mon.a (mon.0) 775 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-12T23:47:02.778 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:02 smithi181 bash[67587]: cluster 2024-09-12T23:47:01.417305+0000 mon.a (mon.0) 776 : cluster [WRN] Health check failed: Degraded data redundancy: 4/495 objects degraded (0.808%), 1 pg degraded (PG_DEGRADED) 2024-09-12T23:47:02.925 INFO:tasks.rados.rados.0.smithi136.stdout:2504: finishing write tid 1 to smithi13667081-4 2024-09-12T23:47:02.925 INFO:tasks.rados.rados.0.smithi136.stdout:2504: finishing write tid 2 to smithi13667081-4 2024-09-12T23:47:02.925 INFO:tasks.rados.rados.0.smithi136.stdout:2504: finishing write tid 3 to smithi13667081-4 2024-09-12T23:47:02.925 INFO:tasks.rados.rados.0.smithi136.stdout:2504: finishing write tid 4 to smithi13667081-4 2024-09-12T23:47:02.925 INFO:tasks.rados.rados.0.smithi136.stdout:2504: finishing write tid 5 to smithi13667081-4 2024-09-12T23:47:02.925 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 4 v 1612 (ObjNum 800 snap 253 seq_num 800) dirty exists 2024-09-12T23:47:02.925 INFO:tasks.rados.rados.0.smithi136.stdout:2504: left oid 4 (ObjNum 800 snap 253 seq_num 800) 2024-09-12T23:47:02.925 INFO:tasks.rados.rados.0.smithi136.stdout:2502: done (3 left) 2024-09-12T23:47:02.926 INFO:tasks.rados.rados.0.smithi136.stdout:2503: done (2 left) 2024-09-12T23:47:02.926 INFO:tasks.rados.rados.0.smithi136.stdout:2504: done (1 left) 2024-09-12T23:47:02.926 INFO:tasks.rados.rados.0.smithi136.stdout:2506: done (0 left) 2024-09-12T23:47:02.926 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 8 current snap is 253 2024-09-12T23:47:02.926 INFO:tasks.rados.rados.0.smithi136.stdout:2507: seq_num 803 ranges {0=655360,655360=745472,1400832=57344} 2024-09-12T23:47:02.935 INFO:tasks.rados.rados.0.smithi136.stdout:2507: writing smithi13667081-8 from 0 to 655360 tid 1 2024-09-12T23:47:02.947 INFO:tasks.rados.rados.0.smithi136.stdout:2507: writing smithi13667081-8 from 655360 to 1400832 tid 2 2024-09-12T23:47:02.948 INFO:tasks.rados.rados.0.smithi136.stdout:2507: writing smithi13667081-8 from 1400832 to 1458176 tid 3 2024-09-12T23:47:02.948 INFO:tasks.rados.rados.0.smithi136.stdout:2508: setattr oid 49 current snap is 253 2024-09-12T23:47:02.948 INFO:tasks.rados.rados.0.smithi136.stdout:2507: finishing write tid 1 to smithi13667081-8 2024-09-12T23:47:02.953 INFO:tasks.rados.rados.0.smithi136.stdout:2507: finishing write tid 2 to smithi13667081-8 2024-09-12T23:47:02.953 INFO:tasks.rados.rados.0.smithi136.stdout:2507: finishing write tid 3 to smithi13667081-8 2024-09-12T23:47:02.953 INFO:tasks.rados.rados.0.smithi136.stdout:2507: finishing write tid 4 to smithi13667081-8 2024-09-12T23:47:02.954 INFO:tasks.rados.rados.0.smithi136.stdout:2509: snap_remove snap 237 2024-09-12T23:47:03.028 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:02 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:47:03.029 INFO:journalctl@ceph.mds.b.smithi181.stdout:Sep 12 23:47:02 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:47:03.029 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:47:02 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:47:03.029 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:47:02 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:47:03.030 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:47:02 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:47:03.030 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:47:02 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:47:03.030 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 12 23:47:02 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:47:03.031 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:47:02 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:47:03.031 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:47:02 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:47:03.031 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 12 23:47:02 smithi181 systemd[1]: ceph-da58a47c-715d-11ef-bceb-c7b262605968@osd.6.service: Deactivated successfully. 2024-09-12T23:47:03.031 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 12 23:47:02 smithi181 systemd[1]: Stopped Ceph osd.6 for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:47:03.031 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 12 23:47:02 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:47:03.031 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 12 23:47:03 smithi181 systemd[1]: Started Ceph osd.6 for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:47:03.437 INFO:tasks.rados.rados.0.smithi136.stdout:2507: finishing write tid 5 to smithi13667081-8 2024-09-12T23:47:03.437 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 8 v 1246 (ObjNum 803 snap 253 seq_num 803) dirty exists 2024-09-12T23:47:03.438 INFO:tasks.rados.rados.0.smithi136.stdout:2507: left oid 8 (ObjNum 803 snap 253 seq_num 803) 2024-09-12T23:47:03.438 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 49 v 1469 (ObjNum -52010912 snap 21878 seq_num 304) dirty exists 2024-09-12T23:47:03.438 INFO:tasks.rados.rados.0.smithi136.stdout:2507: done (2 left) 2024-09-12T23:47:03.438 INFO:tasks.rados.rados.0.smithi136.stdout:2508: done (1 left) 2024-09-12T23:47:03.438 INFO:tasks.rados.rados.0.smithi136.stdout:2509: done (0 left) 2024-09-12T23:47:03.438 INFO:tasks.rados.rados.0.smithi136.stdout:2510: delete oid 16 current snap is 253 2024-09-12T23:47:03.454 INFO:tasks.rados.rados.0.smithi136.stdout:2510: done (0 left) 2024-09-12T23:47:03.454 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 21 current snap is 253 2024-09-12T23:47:03.454 INFO:tasks.rados.rados.0.smithi136.stdout:2511: seq_num 805 ranges {0=688128,688128=507904,1196032=294912} 2024-09-12T23:47:03.457 INFO:tasks.rados.rados.0.smithi136.stdout:2511: writing smithi13667081-21 from 0 to 688128 tid 1 2024-09-12T23:47:03.460 INFO:tasks.rados.rados.0.smithi136.stdout:2511: writing smithi13667081-21 from 688128 to 1196032 tid 2 2024-09-12T23:47:03.463 INFO:tasks.rados.rados.0.smithi136.stdout:2511: writing smithi13667081-21 from 1196032 to 1490944 tid 3 2024-09-12T23:47:03.463 INFO:tasks.rados.rados.0.smithi136.stdout:2512: rollback oid 24 current snap is 253 2024-09-12T23:47:03.463 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 24 to 250 2024-09-12T23:47:03.463 INFO:tasks.rados.rados.0.smithi136.stdout:2511: finishing write tid 1 to smithi13667081-21 2024-09-12T23:47:03.463 INFO:tasks.rados.rados.0.smithi136.stdout:2513: copy_from oid 44 from oid 15 current snap is 253 2024-09-12T23:47:03.463 INFO:tasks.rados.rados.0.smithi136.stdout:2514: rmattr oid 11 current snap is 253 2024-09-12T23:47:03.463 INFO:tasks.rados.rados.0.smithi136.stdout:2514: done (3 left) 2024-09-12T23:47:03.463 INFO:tasks.rados.rados.0.smithi136.stdout:2515: rmattr oid 47 current snap is 253 2024-09-12T23:47:03.464 INFO:tasks.rados.rados.0.smithi136.stdout:2515: done (3 left) 2024-09-12T23:47:03.464 INFO:tasks.rados.rados.0.smithi136.stdout:2516: delete oid 39 current snap is 253 2024-09-12T23:47:03.467 INFO:tasks.rados.rados.0.smithi136.stdout:2511: finishing write tid 2 to smithi13667081-21 2024-09-12T23:47:03.467 INFO:tasks.rados.rados.0.smithi136.stdout:2511: finishing write tid 3 to smithi13667081-21 2024-09-12T23:47:03.467 INFO:tasks.rados.rados.0.smithi136.stdout:2511: finishing write tid 4 to smithi13667081-21 2024-09-12T23:47:03.470 INFO:tasks.rados.rados.0.smithi136.stdout:2511: finishing write tid 5 to smithi13667081-21 2024-09-12T23:47:03.470 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 21 v 873 (ObjNum 805 snap 253 seq_num 805) dirty exists 2024-09-12T23:47:03.470 INFO:tasks.rados.rados.0.smithi136.stdout:2511: left oid 21 (ObjNum 805 snap 253 seq_num 805) 2024-09-12T23:47:03.472 INFO:tasks.rados.rados.0.smithi136.stdout:2513: finishing copy_from racing read to smithi13667081-44 2024-09-12T23:47:03.473 INFO:tasks.rados.rados.0.smithi136.stdout:2512: finishing rollback tid 0 to smithi13667081-24 2024-09-12T23:47:03.473 INFO:tasks.rados.rados.0.smithi136.stdout:2512: finishing rollback tid 1 to smithi13667081-24 2024-09-12T23:47:03.473 INFO:tasks.rados.rados.0.smithi136.stdout:2511: done (3 left) 2024-09-12T23:47:03.473 INFO:tasks.rados.rados.0.smithi136.stdout:2516: done (2 left) 2024-09-12T23:47:03.473 INFO:tasks.rados.rados.0.smithi136.stdout:2517: read oid 2 snap -1 2024-09-12T23:47:03.474 INFO:tasks.rados.rados.0.smithi136.stdout:2517: expect deleted 2024-09-12T23:47:03.474 INFO:tasks.rados.rados.0.smithi136.stdout:2518: read oid 10 snap -1 2024-09-12T23:47:03.474 INFO:tasks.rados.rados.0.smithi136.stdout:2518: expect deleted 2024-09-12T23:47:03.474 INFO:tasks.rados.rados.0.smithi136.stdout:2519: read oid 47 snap -1 2024-09-12T23:47:03.474 INFO:tasks.rados.rados.0.smithi136.stdout:2519: expect (ObjNum 636 snap 188 seq_num 636) 2024-09-12T23:47:03.474 INFO:tasks.rados.rados.0.smithi136.stdout:2520: delete oid 16 current snap is 253 2024-09-12T23:47:03.474 INFO:tasks.rados.rados.0.smithi136.stdout:2512: finishing rollback tid 2 to smithi13667081-24 2024-09-12T23:47:03.475 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 24 v 1271 (ObjNum 614 snap 177 seq_num 614) dirty exists 2024-09-12T23:47:03.475 INFO:tasks.rados.rados.0.smithi136.stdout:2513: finishing copy_from to smithi13667081-44 2024-09-12T23:47:03.475 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 44 v 1460 (ObjNum 733 snap 226 seq_num 733) dirty exists 2024-09-12T23:47:03.477 INFO:tasks.rados.rados.0.smithi136.stdout:2512: done (5 left) 2024-09-12T23:47:03.477 INFO:tasks.rados.rados.0.smithi136.stdout:2513: done (4 left) 2024-09-12T23:47:03.477 INFO:tasks.rados.rados.0.smithi136.stdout:2517: done (3 left) 2024-09-12T23:47:03.477 INFO:tasks.rados.rados.0.smithi136.stdout:2518: done (2 left) 2024-09-12T23:47:03.477 INFO:tasks.rados.rados.0.smithi136.stdout:2520: done (1 left) 2024-09-12T23:47:03.478 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 20 current snap is 253 2024-09-12T23:47:03.478 INFO:tasks.rados.rados.0.smithi136.stdout:2521: seq_num 808 ranges {622592=737280,1359872=614400,1974272=188416} 2024-09-12T23:47:03.485 INFO:tasks.rados.rados.0.smithi136.stdout:2521: writing smithi13667081-20 from 622592 to 1359872 tid 1 2024-09-12T23:47:03.489 INFO:tasks.rados.rados.0.smithi136.stdout:2521: writing smithi13667081-20 from 1359872 to 1974272 tid 2 2024-09-12T23:47:03.490 INFO:tasks.rados.rados.0.smithi136.stdout:2521: writing smithi13667081-20 from 1974272 to 2162688 tid 3 2024-09-12T23:47:03.490 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 6 current snap is 253 2024-09-12T23:47:03.490 INFO:tasks.rados.rados.0.smithi136.stdout:2522: seq_num 809 ranges {2555904=753664,3309568=499712,3809280=303104} 2024-09-12T23:47:03.515 INFO:tasks.rados.rados.0.smithi136.stdout:2522: writing smithi13667081-6 from 2555904 to 3309568 tid 1 2024-09-12T23:47:03.522 INFO:tasks.rados.rados.0.smithi136.stdout:2522: writing smithi13667081-6 from 3309568 to 3809280 tid 2 2024-09-12T23:47:03.527 INFO:tasks.rados.rados.0.smithi136.stdout:2522: writing smithi13667081-6 from 3809280 to 4112384 tid 3 2024-09-12T23:47:03.527 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 7 current snap is 253 2024-09-12T23:47:03.527 INFO:tasks.rados.rados.0.smithi136.stdout:2523: seq_num 810 ranges {1998848=770048,2768896=778240,3547136=24576} 2024-09-12T23:47:03.543 INFO:tasks.rados.rados.0.smithi136.stdout:2523: writing smithi13667081-7 from 1998848 to 2768896 tid 1 2024-09-12T23:47:03.548 INFO:tasks.rados.rados.0.smithi136.stdout:2523: writing smithi13667081-7 from 2768896 to 3547136 tid 2 2024-09-12T23:47:03.549 INFO:tasks.rados.rados.0.smithi136.stdout:2523: writing smithi13667081-7 from 3547136 to 3571712 tid 3 2024-09-12T23:47:03.549 INFO:tasks.rados.rados.0.smithi136.stdout:2524: read oid 15 snap -1 2024-09-12T23:47:03.549 INFO:tasks.rados.rados.0.smithi136.stdout:2524: expect (ObjNum 733 snap 226 seq_num 733) 2024-09-12T23:47:03.549 INFO:tasks.rados.rados.0.smithi136.stdout:2525: delete oid 19 current snap is 253 2024-09-12T23:47:03.549 INFO:tasks.rados.rados.0.smithi136.stdout:2519: expect (ObjNum 636 snap 188 seq_num 636) 2024-09-12T23:47:03.576 INFO:tasks.rados.rados.0.smithi136.stdout:2521: finishing write tid 1 to smithi13667081-20 2024-09-12T23:47:03.576 INFO:tasks.rados.rados.0.smithi136.stdout:2519: done (5 left) 2024-09-12T23:47:03.576 INFO:tasks.rados.rados.0.smithi136.stdout:2525: done (4 left) 2024-09-12T23:47:03.576 INFO:tasks.rados.rados.0.smithi136.stdout:2526: rmattr oid 31 current snap is 253 2024-09-12T23:47:03.577 INFO:tasks.rados.rados.0.smithi136.stdout:2526: done (4 left) 2024-09-12T23:47:03.577 INFO:tasks.rados.rados.0.smithi136.stdout:2527: snap_remove snap 210 2024-09-12T23:47:03.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:03 smithi136 bash[75164]: cluster 2024-09-12T23:47:02.763212+0000 mgr.y (mgr.24661) 245 : cluster [DBG] pgmap v425: 121 pgs: 19 active+undersized, 15 peering, 3 stale+active+clean, 3 active+undersized+degraded, 81 active+clean; 318 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.0 MiB/s wr, 40 op/s; 15/537 objects degraded (2.793%) 2024-09-12T23:47:03.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:03 smithi136 bash[75164]: audit 2024-09-12T23:47:03.073509+0000 mon.a (mon.0) 777 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:03.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:03 smithi136 bash[75164]: audit 2024-09-12T23:47:03.083262+0000 mon.a (mon.0) 778 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:03.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:03 smithi136 bash[75164]: audit 2024-09-12T23:47:03.085233+0000 mon.a (mon.0) 779 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:47:03.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:03 smithi136 bash[76371]: cluster 2024-09-12T23:47:02.763212+0000 mgr.y (mgr.24661) 245 : cluster [DBG] pgmap v425: 121 pgs: 19 active+undersized, 15 peering, 3 stale+active+clean, 3 active+undersized+degraded, 81 active+clean; 318 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.0 MiB/s wr, 40 op/s; 15/537 objects degraded (2.793%) 2024-09-12T23:47:03.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:03 smithi136 bash[76371]: audit 2024-09-12T23:47:03.073509+0000 mon.a (mon.0) 777 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:03.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:03 smithi136 bash[76371]: audit 2024-09-12T23:47:03.083262+0000 mon.a (mon.0) 778 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:03.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:03 smithi136 bash[76371]: audit 2024-09-12T23:47:03.085233+0000 mon.a (mon.0) 779 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:47:03.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:03 smithi181 bash[67587]: cluster 2024-09-12T23:47:02.763212+0000 mgr.y (mgr.24661) 245 : cluster [DBG] pgmap v425: 121 pgs: 19 active+undersized, 15 peering, 3 stale+active+clean, 3 active+undersized+degraded, 81 active+clean; 318 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.0 MiB/s wr, 40 op/s; 15/537 objects degraded (2.793%) 2024-09-12T23:47:03.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:03 smithi181 bash[67587]: audit 2024-09-12T23:47:03.073509+0000 mon.a (mon.0) 777 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:03.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:03 smithi181 bash[67587]: audit 2024-09-12T23:47:03.083262+0000 mon.a (mon.0) 778 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:03.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:03 smithi181 bash[67587]: audit 2024-09-12T23:47:03.085233+0000 mon.a (mon.0) 779 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:47:04.456 INFO:tasks.rados.rados.0.smithi136.stdout:2527: done (4 left) 2024-09-12T23:47:04.456 INFO:tasks.rados.rados.0.smithi136.stdout:2528: snap_create 2024-09-12T23:47:04.456 INFO:tasks.rados.rados.0.smithi136.stdout:2521: finishing write tid 2 to smithi13667081-20 2024-09-12T23:47:04.457 INFO:tasks.rados.rados.0.smithi136.stdout:2521: finishing write tid 3 to smithi13667081-20 2024-09-12T23:47:04.457 INFO:tasks.rados.rados.0.smithi136.stdout:2521: finishing write tid 4 to smithi13667081-20 2024-09-12T23:47:04.457 INFO:tasks.rados.rados.0.smithi136.stdout:2521: finishing write tid 5 to smithi13667081-20 2024-09-12T23:47:04.457 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 20 v 1245 (ObjNum 808 snap 253 seq_num 808) dirty exists 2024-09-12T23:47:04.457 INFO:tasks.rados.rados.0.smithi136.stdout:2521: left oid 20 (ObjNum 808 snap 253 seq_num 808) 2024-09-12T23:47:04.457 INFO:tasks.rados.rados.0.smithi136.stdout:2522: finishing write tid 1 to smithi13667081-6 2024-09-12T23:47:04.457 INFO:tasks.rados.rados.0.smithi136.stdout:2522: finishing write tid 2 to smithi13667081-6 2024-09-12T23:47:04.457 INFO:tasks.rados.rados.0.smithi136.stdout:2522: finishing write tid 3 to smithi13667081-6 2024-09-12T23:47:04.457 INFO:tasks.rados.rados.0.smithi136.stdout:2522: finishing write tid 4 to smithi13667081-6 2024-09-12T23:47:04.457 INFO:tasks.rados.rados.0.smithi136.stdout:2522: finishing write tid 5 to smithi13667081-6 2024-09-12T23:47:04.457 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 6 v 1249 (ObjNum 809 snap 253 seq_num 809) dirty exists 2024-09-12T23:47:04.457 INFO:tasks.rados.rados.0.smithi136.stdout:2522: left oid 6 (ObjNum 809 snap 253 seq_num 809) 2024-09-12T23:47:04.457 INFO:tasks.rados.rados.0.smithi136.stdout:2524: expect (ObjNum 733 snap 226 seq_num 733) 2024-09-12T23:47:04.473 INFO:tasks.rados.rados.0.smithi136.stdout:2523: finishing write tid 1 to smithi13667081-7 2024-09-12T23:47:04.473 INFO:tasks.rados.rados.0.smithi136.stdout:2523: finishing write tid 2 to smithi13667081-7 2024-09-12T23:47:04.473 INFO:tasks.rados.rados.0.smithi136.stdout:2523: finishing write tid 3 to smithi13667081-7 2024-09-12T23:47:04.473 INFO:tasks.rados.rados.0.smithi136.stdout:2523: finishing write tid 4 to smithi13667081-7 2024-09-12T23:47:04.474 INFO:tasks.rados.rados.0.smithi136.stdout:2523: finishing write tid 5 to smithi13667081-7 2024-09-12T23:47:04.474 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 7 v 955 (ObjNum 810 snap 253 seq_num 810) dirty exists 2024-09-12T23:47:04.474 INFO:tasks.rados.rados.0.smithi136.stdout:2523: left oid 7 (ObjNum 810 snap 253 seq_num 810) 2024-09-12T23:47:04.593 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:04 smithi181 bash[67587]: cluster 2024-09-12T23:47:03.440892+0000 mon.a (mon.0) 780 : cluster [DBG] osdmap e567: 8 total, 7 up, 8 in 2024-09-12T23:47:04.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:04 smithi136 bash[75164]: cluster 2024-09-12T23:47:03.440892+0000 mon.a (mon.0) 780 : cluster [DBG] osdmap e567: 8 total, 7 up, 8 in 2024-09-12T23:47:04.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:04 smithi136 bash[76371]: cluster 2024-09-12T23:47:03.440892+0000 mon.a (mon.0) 780 : cluster [DBG] osdmap e567: 8 total, 7 up, 8 in 2024-09-12T23:47:04.844 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 12 23:47:04 smithi181 bash[80312]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-12T23:47:04.844 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 12 23:47:04 smithi181 bash[80312]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-12T23:47:04.844 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 12 23:47:04 smithi181 bash[80312]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-12T23:47:04.844 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 12 23:47:04 smithi181 bash[80312]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-12T23:47:04.844 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 12 23:47:04 smithi181 bash[80312]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-12T23:47:04.844 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 12 23:47:04 smithi181 bash[80312]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-12T23:47:04.844 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 12 23:47:04 smithi181 bash[80312]: --> ceph-volume raw activate successful for osd ID: 6 2024-09-12T23:47:05.463 INFO:tasks.rados.rados.0.smithi136.stdout:2521: done (4 left) 2024-09-12T23:47:05.463 INFO:tasks.rados.rados.0.smithi136.stdout:2522: done (3 left) 2024-09-12T23:47:05.463 INFO:tasks.rados.rados.0.smithi136.stdout:2523: done (2 left) 2024-09-12T23:47:05.463 INFO:tasks.rados.rados.0.smithi136.stdout:2524: done (1 left) 2024-09-12T23:47:05.464 INFO:tasks.rados.rados.0.smithi136.stdout:2528: done (0 left) 2024-09-12T23:47:05.464 INFO:tasks.rados.rados.0.smithi136.stdout:2529: rollback oid 30 current snap is 254 2024-09-12T23:47:05.464 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 30 to 234 2024-09-12T23:47:05.464 INFO:tasks.rados.rados.0.smithi136.stdout:2530: rollback oid 21 current snap is 254 2024-09-12T23:47:05.464 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 21 to 244 2024-09-12T23:47:05.464 INFO:tasks.rados.rados.0.smithi136.stdout:2531: delete oid 10 current snap is 254 2024-09-12T23:47:05.469 INFO:tasks.rados.rados.0.smithi136.stdout:2530: finishing rollback tid 0 to smithi13667081-21 2024-09-12T23:47:05.470 INFO:tasks.rados.rados.0.smithi136.stdout:2530: finishing rollback tid 1 to smithi13667081-21 2024-09-12T23:47:05.470 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 21 v 875 (ObjNum 49 snap 0 seq_num 4243765488) dirty dne 2024-09-12T23:47:05.477 INFO:tasks.rados.rados.0.smithi136.stdout:2529: finishing rollback tid 0 to smithi13667081-30 2024-09-12T23:47:05.479 INFO:tasks.rados.rados.0.smithi136.stdout:2529: finishing rollback tid 1 to smithi13667081-30 2024-09-12T23:47:05.479 INFO:tasks.rados.rados.0.smithi136.stdout:2529: finishing rollback tid 2 to smithi13667081-30 2024-09-12T23:47:05.479 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 30 v 1204 (ObjNum 734 snap 226 seq_num 734) dirty exists 2024-09-12T23:47:05.480 INFO:tasks.rados.rados.0.smithi136.stdout:2529: done (2 left) 2024-09-12T23:47:05.481 INFO:tasks.rados.rados.0.smithi136.stdout:2530: done (1 left) 2024-09-12T23:47:05.481 INFO:tasks.rados.rados.0.smithi136.stdout:2531: done (0 left) 2024-09-12T23:47:05.481 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 15 current snap is 254 2024-09-12T23:47:05.481 INFO:tasks.rados.rados.0.smithi136.stdout:2532: seq_num 813 ranges {278528=417792,696320=434176,1130496=442368,1572864=327680} 2024-09-12T23:47:05.486 INFO:tasks.rados.rados.0.smithi136.stdout:2532: writing smithi13667081-15 from 278528 to 696320 tid 1 2024-09-12T23:47:05.489 INFO:tasks.rados.rados.0.smithi136.stdout:2532: writing smithi13667081-15 from 696320 to 1130496 tid 2 2024-09-12T23:47:05.493 INFO:tasks.rados.rados.0.smithi136.stdout:2532: writing smithi13667081-15 from 1130496 to 1572864 tid 3 2024-09-12T23:47:05.496 INFO:tasks.rados.rados.0.smithi136.stdout:2532: writing smithi13667081-15 from 1572864 to 1900544 tid 4 2024-09-12T23:47:05.496 INFO:tasks.rados.rados.0.smithi136.stdout:2533: snap_create 2024-09-12T23:47:05.503 INFO:tasks.rados.rados.0.smithi136.stdout:2532: finishing write tid 1 to smithi13667081-15 2024-09-12T23:47:05.503 INFO:tasks.rados.rados.0.smithi136.stdout:2532: finishing write tid 2 to smithi13667081-15 2024-09-12T23:47:05.508 INFO:tasks.rados.rados.0.smithi136.stdout:2532: finishing write tid 3 to smithi13667081-15 2024-09-12T23:47:05.508 INFO:tasks.rados.rados.0.smithi136.stdout:2532: finishing write tid 4 to smithi13667081-15 2024-09-12T23:47:05.509 INFO:tasks.rados.rados.0.smithi136.stdout:2532: finishing write tid 5 to smithi13667081-15 2024-09-12T23:47:05.512 INFO:tasks.rados.rados.0.smithi136.stdout:2532: finishing write tid 6 to smithi13667081-15 2024-09-12T23:47:05.512 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 15 v 1474 (ObjNum 813 snap 254 seq_num 813) dirty exists 2024-09-12T23:47:05.512 INFO:tasks.rados.rados.0.smithi136.stdout:2532: left oid 15 (ObjNum 813 snap 254 seq_num 813) 2024-09-12T23:47:05.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:05 smithi136 bash[76371]: cluster 2024-09-12T23:47:04.452287+0000 mon.a (mon.0) 781 : cluster [DBG] osdmap e568: 8 total, 7 up, 8 in 2024-09-12T23:47:05.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:05 smithi136 bash[76371]: cluster 2024-09-12T23:47:04.764267+0000 mgr.y (mgr.24661) 246 : cluster [DBG] pgmap v428: 121 pgs: 34 active+undersized, 8 active+undersized+degraded, 79 active+clean; 321 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 26 MiB/s rd, 5.6 MiB/s wr, 45 op/s; 33/546 objects degraded (6.044%) 2024-09-12T23:47:05.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:05 smithi136 bash[75164]: cluster 2024-09-12T23:47:04.452287+0000 mon.a (mon.0) 781 : cluster [DBG] osdmap e568: 8 total, 7 up, 8 in 2024-09-12T23:47:05.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:05 smithi136 bash[75164]: cluster 2024-09-12T23:47:04.764267+0000 mgr.y (mgr.24661) 246 : cluster [DBG] pgmap v428: 121 pgs: 34 active+undersized, 8 active+undersized+degraded, 79 active+clean; 321 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 26 MiB/s rd, 5.6 MiB/s wr, 45 op/s; 33/546 objects degraded (6.044%) 2024-09-12T23:47:05.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:05 smithi181 bash[67587]: cluster 2024-09-12T23:47:04.452287+0000 mon.a (mon.0) 781 : cluster [DBG] osdmap e568: 8 total, 7 up, 8 in 2024-09-12T23:47:05.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:05 smithi181 bash[67587]: cluster 2024-09-12T23:47:04.764267+0000 mgr.y (mgr.24661) 246 : cluster [DBG] pgmap v428: 121 pgs: 34 active+undersized, 8 active+undersized+degraded, 79 active+clean; 321 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 26 MiB/s rd, 5.6 MiB/s wr, 45 op/s; 33/546 objects degraded (6.044%) 2024-09-12T23:47:06.391 INFO:tasks.rados.rados.0.smithi136.stdout:2532: done (1 left) 2024-09-12T23:47:06.391 INFO:tasks.rados.rados.0.smithi136.stdout:2533: done (0 left) 2024-09-12T23:47:06.391 INFO:tasks.rados.rados.0.smithi136.stdout:2534: rmattr oid 36 current snap is 255 2024-09-12T23:47:06.391 INFO:tasks.rados.rados.0.smithi136.stdout:2535: delete oid 14 current snap is 255 2024-09-12T23:47:06.398 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 36 v 1247 (ObjNum 610 snap 175 seq_num 610) dirty exists 2024-09-12T23:47:06.406 INFO:tasks.rados.rados.0.smithi136.stdout:2534: done (1 left) 2024-09-12T23:47:06.406 INFO:tasks.rados.rados.0.smithi136.stdout:2535: done (0 left) 2024-09-12T23:47:06.406 INFO:tasks.rados.rados.0.smithi136.stdout:2536: read oid 46 snap -1 2024-09-12T23:47:06.406 INFO:tasks.rados.rados.0.smithi136.stdout:2536: expect deleted 2024-09-12T23:47:06.407 INFO:tasks.rados.rados.0.smithi136.stdout:2537: setattr oid 5 current snap is 255 2024-09-12T23:47:06.410 INFO:tasks.rados.rados.0.smithi136.stdout:2536: done (1 left) 2024-09-12T23:47:06.411 INFO:tasks.rados.rados.0.smithi136.stdout:2538: copy_from oid 29 from oid 43 current snap is 255 2024-09-12T23:47:06.411 INFO:tasks.rados.rados.0.smithi136.stdout:2539: read oid 27 snap -1 2024-09-12T23:47:06.411 INFO:tasks.rados.rados.0.smithi136.stdout:2539: expect (ObjNum 767 snap 236 seq_num 767) 2024-09-12T23:47:06.411 INFO:tasks.rados.rados.0.smithi136.stdout:2540: snap_remove snap 248 2024-09-12T23:47:06.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:06 smithi136 bash[75164]: cluster 2024-09-12T23:47:05.444807+0000 mon.a (mon.0) 782 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2024-09-12T23:47:06.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:06 smithi136 bash[75164]: cluster 2024-09-12T23:47:05.466406+0000 mon.a (mon.0) 783 : cluster [DBG] osdmap e569: 8 total, 7 up, 8 in 2024-09-12T23:47:06.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:06 smithi136 bash[75164]: cluster 2024-09-12T23:47:06.392008+0000 mon.a (mon.0) 784 : cluster [DBG] osdmap e570: 8 total, 7 up, 8 in 2024-09-12T23:47:06.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:06 smithi136 bash[76371]: cluster 2024-09-12T23:47:05.444807+0000 mon.a (mon.0) 782 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2024-09-12T23:47:06.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:06 smithi136 bash[76371]: cluster 2024-09-12T23:47:05.466406+0000 mon.a (mon.0) 783 : cluster [DBG] osdmap e569: 8 total, 7 up, 8 in 2024-09-12T23:47:06.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:06 smithi136 bash[76371]: cluster 2024-09-12T23:47:06.392008+0000 mon.a (mon.0) 784 : cluster [DBG] osdmap e570: 8 total, 7 up, 8 in 2024-09-12T23:47:06.844 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 12 23:47:06 smithi181 bash[80534]: debug 2024-09-12T23:47:06.474+0000 7ff97f24b640 1 -- 172.21.15.181:0/515831879 <== mon.1 v2:172.21.15.181:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x560134e291e0 con 0x560135ba8000 2024-09-12T23:47:06.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:06 smithi181 bash[67587]: cluster 2024-09-12T23:47:05.444807+0000 mon.a (mon.0) 782 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2024-09-12T23:47:06.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:06 smithi181 bash[67587]: cluster 2024-09-12T23:47:05.466406+0000 mon.a (mon.0) 783 : cluster [DBG] osdmap e569: 8 total, 7 up, 8 in 2024-09-12T23:47:06.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:06 smithi181 bash[67587]: cluster 2024-09-12T23:47:06.392008+0000 mon.a (mon.0) 784 : cluster [DBG] osdmap e570: 8 total, 7 up, 8 in 2024-09-12T23:47:07.344 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 12 23:47:07 smithi181 bash[80534]: debug 2024-09-12T23:47:07.030+0000 7ff981ab1640 -1 Falling back to public interface 2024-09-12T23:47:07.412 INFO:tasks.rados.rados.0.smithi136.stdout:2540: done (3 left) 2024-09-12T23:47:07.412 INFO:tasks.rados.rados.0.smithi136.stdout:2541: rmattr oid 10 current snap is 255 2024-09-12T23:47:07.412 INFO:tasks.rados.rados.0.smithi136.stdout:2541: done (3 left) 2024-09-12T23:47:07.412 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 13 current snap is 255 2024-09-12T23:47:07.413 INFO:tasks.rados.rados.0.smithi136.stdout:2542: seq_num 815 ranges {3162112=458752,3620864=655360,4276224=409600,4685824=139264} 2024-09-12T23:47:07.455 INFO:tasks.rados.rados.0.smithi136.stdout:2542: writing smithi13667081-13 from 3162112 to 3620864 tid 1 2024-09-12T23:47:07.464 INFO:tasks.rados.rados.0.smithi136.stdout:2542: writing smithi13667081-13 from 3620864 to 4276224 tid 2 2024-09-12T23:47:07.469 INFO:tasks.rados.rados.0.smithi136.stdout:2542: writing smithi13667081-13 from 4276224 to 4685824 tid 3 2024-09-12T23:47:07.470 INFO:tasks.rados.rados.0.smithi136.stdout:2542: writing smithi13667081-13 from 4685824 to 4825088 tid 4 2024-09-12T23:47:07.471 INFO:tasks.rados.rados.0.smithi136.stdout:2543: snap_create 2024-09-12T23:47:07.471 INFO:tasks.rados.rados.0.smithi136.stdout:2539: expect (ObjNum 767 snap 236 seq_num 767) 2024-09-12T23:47:07.531 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 5 v 1780 (ObjNum 567 snap 162 seq_num 567) dirty exists 2024-09-12T23:47:07.531 INFO:tasks.rados.rados.0.smithi136.stdout:2538: finishing copy_from racing read to smithi13667081-29 2024-09-12T23:47:07.531 INFO:tasks.rados.rados.0.smithi136.stdout:2538: finishing copy_from to smithi13667081-29 2024-09-12T23:47:07.531 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 29 v 1607 (ObjNum 742 snap 228 seq_num 742) dirty exists 2024-09-12T23:47:07.531 INFO:tasks.rados.rados.0.smithi136.stdout:2542: finishing write tid 1 to smithi13667081-13 2024-09-12T23:47:07.531 INFO:tasks.rados.rados.0.smithi136.stdout:2542: finishing write tid 2 to smithi13667081-13 2024-09-12T23:47:07.531 INFO:tasks.rados.rados.0.smithi136.stdout:2542: finishing write tid 3 to smithi13667081-13 2024-09-12T23:47:07.531 INFO:tasks.rados.rados.0.smithi136.stdout:2542: finishing write tid 4 to smithi13667081-13 2024-09-12T23:47:07.531 INFO:tasks.rados.rados.0.smithi136.stdout:2542: finishing write tid 5 to smithi13667081-13 2024-09-12T23:47:07.531 INFO:tasks.rados.rados.0.smithi136.stdout:2542: finishing write tid 6 to smithi13667081-13 2024-09-12T23:47:07.531 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 13 v 1612 (ObjNum 815 snap 255 seq_num 815) dirty exists 2024-09-12T23:47:07.531 INFO:tasks.rados.rados.0.smithi136.stdout:2542: left oid 13 (ObjNum 815 snap 255 seq_num 815) 2024-09-12T23:47:07.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:07 smithi136 bash[75164]: cluster 2024-09-12T23:47:06.765341+0000 mgr.y (mgr.24661) 247 : cluster [DBG] pgmap v431: 121 pgs: 34 active+undersized, 8 active+undersized+degraded, 79 active+clean; 327 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 23 MiB/s rd, 2.2 MiB/s wr, 34 op/s; 33/576 objects degraded (5.729%) 2024-09-12T23:47:07.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:07 smithi136 bash[75164]: cluster 2024-09-12T23:47:07.413354+0000 mon.a (mon.0) 785 : cluster [DBG] osdmap e571: 8 total, 7 up, 8 in 2024-09-12T23:47:07.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:07 smithi136 bash[76371]: cluster 2024-09-12T23:47:06.765341+0000 mgr.y (mgr.24661) 247 : cluster [DBG] pgmap v431: 121 pgs: 34 active+undersized, 8 active+undersized+degraded, 79 active+clean; 327 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 23 MiB/s rd, 2.2 MiB/s wr, 34 op/s; 33/576 objects degraded (5.729%) 2024-09-12T23:47:07.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:07 smithi136 bash[76371]: cluster 2024-09-12T23:47:07.413354+0000 mon.a (mon.0) 785 : cluster [DBG] osdmap e571: 8 total, 7 up, 8 in 2024-09-12T23:47:07.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:07 smithi181 bash[67587]: cluster 2024-09-12T23:47:06.765341+0000 mgr.y (mgr.24661) 247 : cluster [DBG] pgmap v431: 121 pgs: 34 active+undersized, 8 active+undersized+degraded, 79 active+clean; 327 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 23 MiB/s rd, 2.2 MiB/s wr, 34 op/s; 33/576 objects degraded (5.729%) 2024-09-12T23:47:07.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:07 smithi181 bash[67587]: cluster 2024-09-12T23:47:07.413354+0000 mon.a (mon.0) 785 : cluster [DBG] osdmap e571: 8 total, 7 up, 8 in 2024-09-12T23:47:08.475 INFO:tasks.rados.rados.0.smithi136.stdout:2537: done (4 left) 2024-09-12T23:47:08.475 INFO:tasks.rados.rados.0.smithi136.stdout:2538: done (3 left) 2024-09-12T23:47:08.475 INFO:tasks.rados.rados.0.smithi136.stdout:2539: done (2 left) 2024-09-12T23:47:08.475 INFO:tasks.rados.rados.0.smithi136.stdout:2542: done (1 left) 2024-09-12T23:47:08.475 INFO:tasks.rados.rados.0.smithi136.stdout:2543: done (0 left) 2024-09-12T23:47:08.475 INFO:tasks.rados.rados.0.smithi136.stdout:2544: read oid 13 snap 238 2024-09-12T23:47:08.475 INFO:tasks.rados.rados.0.smithi136.stdout:2544: expect (ObjNum 772 snap 237 seq_num 772) 2024-09-12T23:47:08.476 INFO:tasks.rados.rados.0.smithi136.stdout:2545: read oid 44 snap 246 2024-09-12T23:47:08.476 INFO:tasks.rados.rados.0.smithi136.stdout:2545: expect (ObjNum 783 snap 245 seq_num 783) 2024-09-12T23:47:08.476 INFO:tasks.rados.rados.0.smithi136.stdout:2546: delete oid 15 current snap is 256 2024-09-12T23:47:08.488 INFO:tasks.rados.rados.0.smithi136.stdout:2544: expect (ObjNum 772 snap 237 seq_num 772) 2024-09-12T23:47:08.532 INFO:tasks.rados.rados.0.smithi136.stdout:2545: expect (ObjNum 783 snap 245 seq_num 783) 2024-09-12T23:47:08.613 INFO:tasks.rados.rados.0.smithi136.stdout:2544: done (2 left) 2024-09-12T23:47:08.613 INFO:tasks.rados.rados.0.smithi136.stdout:2545: done (1 left) 2024-09-12T23:47:08.613 INFO:tasks.rados.rados.0.smithi136.stdout:2546: done (0 left) 2024-09-12T23:47:08.613 INFO:tasks.rados.rados.0.smithi136.stdout:2547: snap_create 2024-09-12T23:47:08.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:08 smithi136 bash[75164]: cluster 2024-09-12T23:47:07.456847+0000 mon.a (mon.0) 786 : cluster [WRN] Health check update: Degraded data redundancy: 33/576 objects degraded (5.729%), 8 pgs degraded (PG_DEGRADED) 2024-09-12T23:47:08.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:08 smithi136 bash[76371]: cluster 2024-09-12T23:47:07.456847+0000 mon.a (mon.0) 786 : cluster [WRN] Health check update: Degraded data redundancy: 33/576 objects degraded (5.729%), 8 pgs degraded (PG_DEGRADED) 2024-09-12T23:47:08.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:08 smithi181 bash[67587]: cluster 2024-09-12T23:47:07.456847+0000 mon.a (mon.0) 786 : cluster [WRN] Health check update: Degraded data redundancy: 33/576 objects degraded (5.729%), 8 pgs degraded (PG_DEGRADED) 2024-09-12T23:47:09.498 INFO:tasks.rados.rados.0.smithi136.stdout:2547: done (0 left) 2024-09-12T23:47:09.498 INFO:tasks.rados.rados.0.smithi136.stdout:2548: rollback oid 27 current snap is 257 2024-09-12T23:47:09.498 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 27 to 243 2024-09-12T23:47:09.498 INFO:tasks.rados.rados.0.smithi136.stdout:2549: read oid 11 snap 246 2024-09-12T23:47:09.498 INFO:tasks.rados.rados.0.smithi136.stdout:2549: expect (ObjNum 605 snap 174 seq_num 605) 2024-09-12T23:47:09.498 INFO:tasks.rados.rados.0.smithi136.stdout:2550: rollback oid 15 current snap is 257 2024-09-12T23:47:09.498 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 15 to 239 2024-09-12T23:47:09.499 INFO:tasks.rados.rados.0.smithi136.stdout:2551: rollback oid 21 current snap is 257 2024-09-12T23:47:09.499 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 21 to 253 2024-09-12T23:47:09.499 INFO:tasks.rados.rados.0.smithi136.stdout:2552: rollback oid 42 current snap is 257 2024-09-12T23:47:09.499 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 42 to 246 2024-09-12T23:47:09.499 INFO:tasks.rados.rados.0.smithi136.stdout:2553: read oid 50 snap -1 2024-09-12T23:47:09.499 INFO:tasks.rados.rados.0.smithi136.stdout:2553: expect deleted 2024-09-12T23:47:09.499 INFO:tasks.rados.rados.0.smithi136.stdout:2554: rmattr oid 41 current snap is 257 2024-09-12T23:47:09.499 INFO:tasks.rados.rados.0.smithi136.stdout:2554: done (6 left) 2024-09-12T23:47:09.499 INFO:tasks.rados.rados.0.smithi136.stdout:2555: read oid 6 snap -1 2024-09-12T23:47:09.499 INFO:tasks.rados.rados.0.smithi136.stdout:2555: expect (ObjNum 809 snap 253 seq_num 809) 2024-09-12T23:47:09.499 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 41 current snap is 257 2024-09-12T23:47:09.499 INFO:tasks.rados.rados.0.smithi136.stdout:2556: seq_num 817 ranges {0=491520,491520=417792,909312=425984,1335296=360448} 2024-09-12T23:47:09.507 INFO:tasks.rados.rados.0.smithi136.stdout:2556: writing smithi13667081-41 from 0 to 491520 tid 1 2024-09-12T23:47:09.516 INFO:tasks.rados.rados.0.smithi136.stdout:2556: writing smithi13667081-41 from 491520 to 909312 tid 2 2024-09-12T23:47:09.525 INFO:tasks.rados.rados.0.smithi136.stdout:2556: writing smithi13667081-41 from 909312 to 1335296 tid 3 2024-09-12T23:47:09.532 INFO:tasks.rados.rados.0.smithi136.stdout:2556: writing smithi13667081-41 from 1335296 to 1695744 tid 4 2024-09-12T23:47:09.533 INFO:tasks.rados.rados.0.smithi136.stdout:2557: read oid 48 snap 240 2024-09-12T23:47:09.533 INFO:tasks.rados.rados.0.smithi136.stdout:2557: expect (ObjNum 764 snap 236 seq_num 764) 2024-09-12T23:47:09.533 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 13 current snap is 257 2024-09-12T23:47:09.533 INFO:tasks.rados.rados.0.smithi136.stdout:2558: seq_num 818 ranges {4825088=507904,5332992=704512,6037504=499712} 2024-09-12T23:47:09.564 INFO:tasks.rados.rados.0.smithi136.stdout:2558: writing smithi13667081-13 from 4825088 to 5332992 tid 1 2024-09-12T23:47:09.568 INFO:tasks.rados.rados.0.smithi136.stdout:2558: writing smithi13667081-13 from 5332992 to 6037504 tid 2 2024-09-12T23:47:09.571 INFO:tasks.rados.rados.0.smithi136.stdout:2558: writing smithi13667081-13 from 6037504 to 6537216 tid 3 2024-09-12T23:47:09.571 INFO:tasks.rados.rados.0.smithi136.stdout:2559: delete oid 19 current snap is 257 2024-09-12T23:47:09.571 INFO:tasks.rados.rados.0.smithi136.stdout:2551: finishing rollback tid 1 to smithi13667081-21 2024-09-12T23:47:09.571 INFO:tasks.rados.rados.0.smithi136.stdout:2548: finishing rollback tid 0 to smithi13667081-27 2024-09-12T23:47:09.571 INFO:tasks.rados.rados.0.smithi136.stdout:2548: finishing rollback tid 1 to smithi13667081-27 2024-09-12T23:47:09.571 INFO:tasks.rados.rados.0.smithi136.stdout:2548: finishing rollback tid 2 to smithi13667081-27 2024-09-12T23:47:09.572 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 27 v 1790 (ObjNum 766 snap 236 seq_num 766) dirty exists 2024-09-12T23:47:09.572 INFO:tasks.rados.rados.0.smithi136.stdout:2551: finishing rollback tid 2 to smithi13667081-21 2024-09-12T23:47:09.572 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 21 v 877 (ObjNum 805 snap 253 seq_num 805) dirty exists 2024-09-12T23:47:09.572 INFO:tasks.rados.rados.0.smithi136.stdout:2550: finishing rollback tid 1 to smithi13667081-15 2024-09-12T23:47:09.572 INFO:tasks.rados.rados.0.smithi136.stdout:2556: finishing write tid 1 to smithi13667081-41 2024-09-12T23:47:09.572 INFO:tasks.rados.rados.0.smithi136.stdout:2550: finishing rollback tid 2 to smithi13667081-15 2024-09-12T23:47:09.572 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 15 v 1477 (ObjNum 733 snap 226 seq_num 733) dirty exists 2024-09-12T23:47:09.572 INFO:tasks.rados.rados.0.smithi136.stdout:2556: finishing write tid 2 to smithi13667081-41 2024-09-12T23:47:09.572 INFO:tasks.rados.rados.0.smithi136.stdout:2552: finishing rollback tid 1 to smithi13667081-42 2024-09-12T23:47:09.572 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 42 v 869 (ObjNum 33 snap 0 seq_num 4244010272) dirty dne 2024-09-12T23:47:09.572 INFO:tasks.rados.rados.0.smithi136.stdout:2556: finishing write tid 3 to smithi13667081-41 2024-09-12T23:47:09.572 INFO:tasks.rados.rados.0.smithi136.stdout:2555: expect (ObjNum 809 snap 253 seq_num 809) 2024-09-12T23:47:09.660 INFO:teuthology.orchestra.run.smithi136.stdout:true 2024-09-12T23:47:09.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:47:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:47:09] "GET /metrics HTTP/1.1" 200 36386 "" "Prometheus/2.43.0" 2024-09-12T23:47:09.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:09 smithi136 bash[75164]: cluster 2024-09-12T23:47:08.477102+0000 mon.a (mon.0) 787 : cluster [DBG] osdmap e572: 8 total, 7 up, 8 in 2024-09-12T23:47:09.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:09 smithi136 bash[75164]: audit 2024-09-12T23:47:08.754284+0000 mon.a (mon.0) 788 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:09.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:09 smithi136 bash[75164]: audit 2024-09-12T23:47:08.756276+0000 mon.a (mon.0) 789 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:47:09.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:09 smithi136 bash[75164]: cluster 2024-09-12T23:47:08.766080+0000 mgr.y (mgr.24661) 248 : cluster [DBG] pgmap v434: 121 pgs: 34 active+undersized, 8 active+undersized+degraded, 79 active+clean; 329 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.7 MiB/s rd, 2.0 MiB/s wr, 26 op/s; 36/588 objects degraded (6.122%) 2024-09-12T23:47:09.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:09 smithi136 bash[76371]: cluster 2024-09-12T23:47:08.477102+0000 mon.a (mon.0) 787 : cluster [DBG] osdmap e572: 8 total, 7 up, 8 in 2024-09-12T23:47:09.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:09 smithi136 bash[76371]: audit 2024-09-12T23:47:08.754284+0000 mon.a (mon.0) 788 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:09.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:09 smithi136 bash[76371]: audit 2024-09-12T23:47:08.756276+0000 mon.a (mon.0) 789 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:47:09.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:09 smithi136 bash[76371]: cluster 2024-09-12T23:47:08.766080+0000 mgr.y (mgr.24661) 248 : cluster [DBG] pgmap v434: 121 pgs: 34 active+undersized, 8 active+undersized+degraded, 79 active+clean; 329 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.7 MiB/s rd, 2.0 MiB/s wr, 26 op/s; 36/588 objects degraded (6.122%) 2024-09-12T23:47:09.761 INFO:tasks.rados.rados.0.smithi136.stdout:2556: finishing write tid 4 to smithi13667081-41 2024-09-12T23:47:09.761 INFO:tasks.rados.rados.0.smithi136.stdout:2556: finishing write tid 5 to smithi13667081-41 2024-09-12T23:47:09.761 INFO:tasks.rados.rados.0.smithi136.stdout:2556: finishing write tid 6 to smithi13667081-41 2024-09-12T23:47:09.761 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 41 v 1278 (ObjNum 817 snap 257 seq_num 817) dirty exists 2024-09-12T23:47:09.761 INFO:tasks.rados.rados.0.smithi136.stdout:2556: left oid 41 (ObjNum 817 snap 257 seq_num 817) 2024-09-12T23:47:09.761 INFO:tasks.rados.rados.0.smithi136.stdout:2557: expect (ObjNum 764 snap 236 seq_num 764) 2024-09-12T23:47:09.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:09 smithi181 bash[67587]: cluster 2024-09-12T23:47:08.477102+0000 mon.a (mon.0) 787 : cluster [DBG] osdmap e572: 8 total, 7 up, 8 in 2024-09-12T23:47:09.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:09 smithi181 bash[67587]: audit 2024-09-12T23:47:08.754284+0000 mon.a (mon.0) 788 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:09.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:09 smithi181 bash[67587]: audit 2024-09-12T23:47:08.756276+0000 mon.a (mon.0) 789 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:47:09.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:09 smithi181 bash[67587]: cluster 2024-09-12T23:47:08.766080+0000 mgr.y (mgr.24661) 248 : cluster [DBG] pgmap v434: 121 pgs: 34 active+undersized, 8 active+undersized+degraded, 79 active+clean; 329 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.7 MiB/s rd, 2.0 MiB/s wr, 26 op/s; 36/588 objects degraded (6.122%) 2024-09-12T23:47:09.890 INFO:tasks.rados.rados.0.smithi136.stdout:2558: finishing write tid 1 to smithi13667081-13 2024-09-12T23:47:09.891 INFO:tasks.rados.rados.0.smithi136.stdout:2549: expect (ObjNum 605 snap 174 seq_num 605) 2024-09-12T23:47:10.015 INFO:teuthology.orchestra.run.smithi136.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-12T23:47:10.015 INFO:teuthology.orchestra.run.smithi136.stdout:alertmanager.a smithi136 *:9093,9094 running (6m) 86s ago 12m 13.1M - 0.25.0 c8568f914cd2 b5197f020eeb 2024-09-12T23:47:10.015 INFO:teuthology.orchestra.run.smithi136.stdout:grafana.a smithi181 *:3000 running (6m) 0s ago 12m 74.9M - 9.4.7 954c08fa6188 256c181be984 2024-09-12T23:47:10.015 INFO:teuthology.orchestra.run.smithi136.stdout:mds.a smithi136 running (13m) 86s ago 13m 15.1M - 17.2.7-1627-gbbdadfa3 214985049bdb 713b9795ad19 2024-09-12T23:47:10.015 INFO:teuthology.orchestra.run.smithi136.stdout:mds.b smithi181 running (13m) 0s ago 13m 19.7M - 17.2.7-1627-gbbdadfa3 214985049bdb ad13179f0868 2024-09-12T23:47:10.016 INFO:teuthology.orchestra.run.smithi136.stdout:mgr.x smithi181 *:8443,9283,8765 running (5m) 0s ago 21m 427M - 18.2.4-837-g7a853b17 f21ba1eea74e cc664dc337f7 2024-09-12T23:47:10.016 INFO:teuthology.orchestra.run.smithi136.stdout:mgr.y smithi136 *:8443,9283,8765 running (6m) 86s ago 23m 490M - 18.2.4-837-g7a853b17 f21ba1eea74e 272f10b42fc3 2024-09-12T23:47:10.016 INFO:teuthology.orchestra.run.smithi136.stdout:mon.a smithi136 running (5m) 86s ago 23m 56.4M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e ad8d21d58ad9 2024-09-12T23:47:10.016 INFO:teuthology.orchestra.run.smithi136.stdout:mon.b smithi181 running (4m) 0s ago 21m 44.3M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e a70ec8315934 2024-09-12T23:47:10.016 INFO:teuthology.orchestra.run.smithi136.stdout:mon.c smithi136 running (4m) 86s ago 21m 47.2M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e 44d1be055a5e 2024-09-12T23:47:10.016 INFO:teuthology.orchestra.run.smithi136.stdout:node-exporter.a smithi136 *:9100 running (6m) 86s ago 12m 8064k - 1.5.0 0da6a335fe13 ccfc37c96620 2024-09-12T23:47:10.016 INFO:teuthology.orchestra.run.smithi136.stdout:node-exporter.b smithi181 *:9100 running (6m) 0s ago 12m 8563k - 1.5.0 0da6a335fe13 80bf0e383230 2024-09-12T23:47:10.016 INFO:teuthology.orchestra.run.smithi136.stdout:osd.0 smithi136 running (2m) 86s ago 20m 212M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 7da21f765721 2024-09-12T23:47:10.016 INFO:teuthology.orchestra.run.smithi136.stdout:osd.1 smithi136 running (2m) 86s ago 19m 247M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 5302be0a016b 2024-09-12T23:47:10.016 INFO:teuthology.orchestra.run.smithi136.stdout:osd.2 smithi136 running (119s) 86s ago 18m 252M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 2e12d1a31223 2024-09-12T23:47:10.016 INFO:teuthology.orchestra.run.smithi136.stdout:osd.3 smithi136 running (90s) 86s ago 17m - 2524M 18.2.4-837-g7a853b17 f21ba1eea74e aa9dc124e59c 2024-09-12T23:47:10.017 INFO:teuthology.orchestra.run.smithi136.stdout:osd.4 smithi181 running (61s) 0s ago 17m 357M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 4ee3b228ce7d 2024-09-12T23:47:10.017 INFO:teuthology.orchestra.run.smithi136.stdout:osd.5 smithi181 running (33s) 0s ago 16m 187M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 3f9a09565258 2024-09-12T23:47:10.017 INFO:teuthology.orchestra.run.smithi136.stdout:osd.6 smithi181 running (4s) 0s ago 15m - 2524M 18.2.4-837-g7a853b17 f21ba1eea74e ce539f6fb854 2024-09-12T23:47:10.017 INFO:teuthology.orchestra.run.smithi136.stdout:osd.7 smithi181 running (14m) 0s ago 14m 223M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb bc0632ae2145 2024-09-12T23:47:10.017 INFO:teuthology.orchestra.run.smithi136.stdout:prometheus.a smithi181 *:9095 running (5m) 0s ago 13m 57.5M - 2.43.0 a07b618ecd1d 3853d003a5cc 2024-09-12T23:47:10.358 INFO:teuthology.orchestra.run.smithi136.stdout:{ 2024-09-12T23:47:10.358 INFO:teuthology.orchestra.run.smithi136.stdout: "mon": { 2024-09-12T23:47:10.358 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 3 2024-09-12T23:47:10.358 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:47:10.358 INFO:teuthology.orchestra.run.smithi136.stdout: "mgr": { 2024-09-12T23:47:10.359 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 2 2024-09-12T23:47:10.359 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:47:10.359 INFO:teuthology.orchestra.run.smithi136.stdout: "osd": { 2024-09-12T23:47:10.359 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 1, 2024-09-12T23:47:10.359 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 6 2024-09-12T23:47:10.359 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:47:10.359 INFO:teuthology.orchestra.run.smithi136.stdout: "mds": { 2024-09-12T23:47:10.359 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-12T23:47:10.359 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:47:10.359 INFO:teuthology.orchestra.run.smithi136.stdout: "overall": { 2024-09-12T23:47:10.359 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 3, 2024-09-12T23:47:10.359 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 11 2024-09-12T23:47:10.359 INFO:teuthology.orchestra.run.smithi136.stdout: } 2024-09-12T23:47:10.360 INFO:teuthology.orchestra.run.smithi136.stdout:} 2024-09-12T23:47:10.460 INFO:tasks.rados.rados.0.smithi136.stdout:2558: finishing write tid 2 to smithi13667081-13 2024-09-12T23:47:10.460 INFO:tasks.rados.rados.0.smithi136.stdout:2558: finishing write tid 3 to smithi13667081-13 2024-09-12T23:47:10.460 INFO:tasks.rados.rados.0.smithi136.stdout:2558: finishing write tid 4 to smithi13667081-13 2024-09-12T23:47:10.460 INFO:tasks.rados.rados.0.smithi136.stdout:2558: finishing write tid 5 to smithi13667081-13 2024-09-12T23:47:10.460 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 13 v 1616 (ObjNum 818 snap 257 seq_num 818) dirty exists 2024-09-12T23:47:10.460 INFO:tasks.rados.rados.0.smithi136.stdout:2558: left oid 13 (ObjNum 818 snap 257 seq_num 818) 2024-09-12T23:47:10.460 INFO:tasks.rados.rados.0.smithi136.stdout:2548: done (10 left) 2024-09-12T23:47:10.460 INFO:tasks.rados.rados.0.smithi136.stdout:2549: done (9 left) 2024-09-12T23:47:10.461 INFO:tasks.rados.rados.0.smithi136.stdout:2550: done (8 left) 2024-09-12T23:47:10.461 INFO:tasks.rados.rados.0.smithi136.stdout:2551: done (7 left) 2024-09-12T23:47:10.461 INFO:tasks.rados.rados.0.smithi136.stdout:2552: done (6 left) 2024-09-12T23:47:10.461 INFO:tasks.rados.rados.0.smithi136.stdout:2553: done (5 left) 2024-09-12T23:47:10.461 INFO:tasks.rados.rados.0.smithi136.stdout:2555: done (4 left) 2024-09-12T23:47:10.461 INFO:tasks.rados.rados.0.smithi136.stdout:2556: done (3 left) 2024-09-12T23:47:10.461 INFO:tasks.rados.rados.0.smithi136.stdout:2557: done (2 left) 2024-09-12T23:47:10.461 INFO:tasks.rados.rados.0.smithi136.stdout:2558: done (1 left) 2024-09-12T23:47:10.461 INFO:tasks.rados.rados.0.smithi136.stdout:2559: done (0 left) 2024-09-12T23:47:10.461 INFO:tasks.rados.rados.0.smithi136.stdout:2560: snap_create 2024-09-12T23:47:10.491 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 12 23:47:10 smithi181 bash[80534]: debug 2024-09-12T23:47:10.210+0000 7ff981ab1640 -1 osd.6 563 log_to_monitors true 2024-09-12T23:47:10.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:10 smithi136 bash[76371]: cluster 2024-09-12T23:47:09.500752+0000 mon.a (mon.0) 790 : cluster [DBG] osdmap e573: 8 total, 7 up, 8 in 2024-09-12T23:47:10.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:10 smithi136 bash[76371]: audit 2024-09-12T23:47:09.651312+0000 mgr.y (mgr.24661) 249 : audit [DBG] from='client.34356 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:47:10.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:10 smithi136 bash[76371]: audit 2024-09-12T23:47:09.900401+0000 mon.a (mon.0) 791 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:10.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:10 smithi136 bash[76371]: audit 2024-09-12T23:47:09.912028+0000 mon.a (mon.0) 792 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:10.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:10 smithi136 bash[76371]: audit 2024-09-12T23:47:10.215519+0000 mon.a (mon.0) 793 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-12T23:47:10.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:10 smithi136 bash[76371]: audit 2024-09-12T23:47:10.216340+0000 mon.b (mon.1) 8 : audit [INF] from='osd.6 [v2:172.21.15.181:6816/1329407811,v1:172.21.15.181:6817/1329407811]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-12T23:47:10.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:10 smithi136 bash[76371]: audit 2024-09-12T23:47:10.361572+0000 mon.a (mon.0) 794 : audit [DBG] from='client.? 172.21.15.136:0/2757358085' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:47:10.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:10 smithi136 bash[75164]: cluster 2024-09-12T23:47:09.500752+0000 mon.a (mon.0) 790 : cluster [DBG] osdmap e573: 8 total, 7 up, 8 in 2024-09-12T23:47:10.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:10 smithi136 bash[75164]: audit 2024-09-12T23:47:09.651312+0000 mgr.y (mgr.24661) 249 : audit [DBG] from='client.34356 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:47:10.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:10 smithi136 bash[75164]: audit 2024-09-12T23:47:09.900401+0000 mon.a (mon.0) 791 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:10.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:10 smithi136 bash[75164]: audit 2024-09-12T23:47:09.912028+0000 mon.a (mon.0) 792 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:10.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:10 smithi136 bash[75164]: audit 2024-09-12T23:47:10.215519+0000 mon.a (mon.0) 793 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-12T23:47:10.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:10 smithi136 bash[75164]: audit 2024-09-12T23:47:10.216340+0000 mon.b (mon.1) 8 : audit [INF] from='osd.6 [v2:172.21.15.181:6816/1329407811,v1:172.21.15.181:6817/1329407811]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-12T23:47:10.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:10 smithi136 bash[75164]: audit 2024-09-12T23:47:10.361572+0000 mon.a (mon.0) 794 : audit [DBG] from='client.? 172.21.15.136:0/2757358085' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:47:10.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:10 smithi181 bash[67587]: cluster 2024-09-12T23:47:09.500752+0000 mon.a (mon.0) 790 : cluster [DBG] osdmap e573: 8 total, 7 up, 8 in 2024-09-12T23:47:10.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:10 smithi181 bash[67587]: audit 2024-09-12T23:47:09.651312+0000 mgr.y (mgr.24661) 249 : audit [DBG] from='client.34356 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:47:10.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:10 smithi181 bash[67587]: audit 2024-09-12T23:47:09.900401+0000 mon.a (mon.0) 791 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:10.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:10 smithi181 bash[67587]: audit 2024-09-12T23:47:09.912028+0000 mon.a (mon.0) 792 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:10.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:10 smithi181 bash[67587]: audit 2024-09-12T23:47:10.215519+0000 mon.a (mon.0) 793 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-12T23:47:10.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:10 smithi181 bash[67587]: audit 2024-09-12T23:47:10.216340+0000 mon.b (mon.1) 8 : audit [INF] from='osd.6 [v2:172.21.15.181:6816/1329407811,v1:172.21.15.181:6817/1329407811]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-12T23:47:10.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:10 smithi181 bash[67587]: audit 2024-09-12T23:47:10.361572+0000 mon.a (mon.0) 794 : audit [DBG] from='client.? 172.21.15.136:0/2757358085' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:47:10.919 INFO:tasks.rados.rados.0.smithi136.stdout:2560: done (0 left) 2024-09-12T23:47:10.919 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 13 current snap is 258 2024-09-12T23:47:10.919 INFO:tasks.rados.rados.0.smithi136.stdout:2561: seq_num 820 ranges {6537216=540672,7077888=475136,7553024=598016,8151040=131072} 2024-09-12T23:47:10.995 INFO:tasks.rados.rados.0.smithi136.stdout:2561: writing smithi13667081-13 from 6537216 to 7077888 tid 1 2024-09-12T23:47:10.998 INFO:tasks.rados.rados.0.smithi136.stdout:2561: writing smithi13667081-13 from 7077888 to 7553024 tid 2 2024-09-12T23:47:11.002 INFO:tasks.rados.rados.0.smithi136.stdout:2561: writing smithi13667081-13 from 7553024 to 8151040 tid 3 2024-09-12T23:47:11.003 INFO:tasks.rados.rados.0.smithi136.stdout:2561: writing smithi13667081-13 from 8151040 to 8282112 tid 4 2024-09-12T23:47:11.003 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 17 current snap is 258 2024-09-12T23:47:11.003 INFO:tasks.rados.rados.0.smithi136.stdout:2562: seq_num 821 ranges {0=557056,557056=761856,1318912=442368} 2024-09-12T23:47:11.007 INFO:tasks.rados.rados.0.smithi136.stdout:2562: writing smithi13667081-17 from 0 to 557056 tid 1 2024-09-12T23:47:11.012 INFO:tasks.rados.rados.0.smithi136.stdout:2562: writing smithi13667081-17 from 557056 to 1318912 tid 2 2024-09-12T23:47:11.015 INFO:tasks.rados.rados.0.smithi136.stdout:2562: writing smithi13667081-17 from 1318912 to 1761280 tid 3 2024-09-12T23:47:11.015 INFO:tasks.rados.rados.0.smithi136.stdout:2563: read oid 30 snap -1 2024-09-12T23:47:11.015 INFO:tasks.rados.rados.0.smithi136.stdout:2563: expect (ObjNum 734 snap 226 seq_num 734) 2024-09-12T23:47:11.015 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 46 current snap is 258 2024-09-12T23:47:11.015 INFO:tasks.rados.rados.0.smithi136.stdout:2564: seq_num 822 ranges {0=573440,573440=696320,1269760=507904} 2024-09-12T23:47:11.019 INFO:tasks.rados.rados.0.smithi136.stdout:2564: writing smithi13667081-46 from 0 to 573440 tid 1 2024-09-12T23:47:11.024 INFO:tasks.rados.rados.0.smithi136.stdout:2564: writing smithi13667081-46 from 573440 to 1269760 tid 2 2024-09-12T23:47:11.027 INFO:tasks.rados.rados.0.smithi136.stdout:2564: writing smithi13667081-46 from 1269760 to 1777664 tid 3 2024-09-12T23:47:11.027 INFO:tasks.rados.rados.0.smithi136.stdout:2565: snap_remove snap 243 2024-09-12T23:47:11.844 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 12 23:47:11 smithi181 bash[80534]: debug 2024-09-12T23:47:11.486+0000 7ff975853640 -1 osd.6 563 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-12T23:47:11.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:11 smithi181 bash[67587]: audit 2024-09-12T23:47:09.988415+0000 mgr.y (mgr.24661) 250 : audit [DBG] from='client.44282 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:47:11.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:11 smithi181 bash[67587]: audit 2024-09-12T23:47:10.649550+0000 mon.a (mon.0) 795 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:11.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:11 smithi181 bash[67587]: audit 2024-09-12T23:47:10.659443+0000 mon.a (mon.0) 796 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:11.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:11 smithi181 bash[67587]: cluster 2024-09-12T23:47:10.767195+0000 mgr.y (mgr.24661) 251 : cluster [DBG] pgmap v436: 121 pgs: 34 active+undersized, 8 active+undersized+degraded, 79 active+clean; 343 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.4 MiB/s rd, 4.2 MiB/s wr, 26 op/s; 36/597 objects degraded (6.030%) 2024-09-12T23:47:11.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:11 smithi181 bash[67587]: audit 2024-09-12T23:47:10.917227+0000 mon.a (mon.0) 797 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-12T23:47:11.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:11 smithi181 bash[67587]: cluster 2024-09-12T23:47:10.917478+0000 mon.a (mon.0) 798 : cluster [DBG] osdmap e574: 8 total, 7 up, 8 in 2024-09-12T23:47:11.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:11 smithi181 bash[67587]: audit 2024-09-12T23:47:10.922378+0000 mon.a (mon.0) 799 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-12T23:47:11.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:11 smithi181 bash[67587]: audit 2024-09-12T23:47:10.922441+0000 mon.b (mon.1) 9 : audit [INF] from='osd.6 [v2:172.21.15.181:6816/1329407811,v1:172.21.15.181:6817/1329407811]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-12T23:47:11.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:11 smithi181 bash[67587]: audit 2024-09-12T23:47:11.464842+0000 mon.a (mon.0) 800 : audit [INF] from='osd.6 ' entity='osd.6' 2024-09-12T23:47:11.923 INFO:tasks.rados.rados.0.smithi136.stdout:2565: done (4 left) 2024-09-12T23:47:11.923 INFO:tasks.rados.rados.0.smithi136.stdout:2566: copy_from oid 20 from oid 28 current snap is 258 2024-09-12T23:47:11.923 INFO:tasks.rados.rados.0.smithi136.stdout:2567: copy_from oid 4 from oid 10 current snap is 258 2024-09-12T23:47:11.923 INFO:tasks.rados.rados.0.smithi136.stdout:2568: delete oid 22 current snap is 258 2024-09-12T23:47:11.923 INFO:tasks.rados.rados.0.smithi136.stdout:2563: expect (ObjNum 734 snap 226 seq_num 734) 2024-09-12T23:47:11.939 INFO:tasks.rados.rados.0.smithi136.stdout:2561: finishing write tid 1 to smithi13667081-13 2024-09-12T23:47:11.940 INFO:tasks.rados.rados.0.smithi136.stdout:2561: finishing write tid 2 to smithi13667081-13 2024-09-12T23:47:11.940 INFO:tasks.rados.rados.0.smithi136.stdout:2564: finishing write tid 1 to smithi13667081-46 2024-09-12T23:47:11.940 INFO:tasks.rados.rados.0.smithi136.stdout:2561: finishing write tid 3 to smithi13667081-13 2024-09-12T23:47:11.940 INFO:tasks.rados.rados.0.smithi136.stdout:2561: finishing write tid 4 to smithi13667081-13 2024-09-12T23:47:11.940 INFO:tasks.rados.rados.0.smithi136.stdout:2561: finishing write tid 5 to smithi13667081-13 2024-09-12T23:47:11.940 INFO:tasks.rados.rados.0.smithi136.stdout:2562: finishing write tid 1 to smithi13667081-17 2024-09-12T23:47:11.940 INFO:tasks.rados.rados.0.smithi136.stdout:2562: finishing write tid 2 to smithi13667081-17 2024-09-12T23:47:11.940 INFO:tasks.rados.rados.0.smithi136.stdout:2562: finishing write tid 3 to smithi13667081-17 2024-09-12T23:47:11.940 INFO:tasks.rados.rados.0.smithi136.stdout:2562: finishing write tid 4 to smithi13667081-17 2024-09-12T23:47:11.940 INFO:tasks.rados.rados.0.smithi136.stdout:2561: finishing write tid 6 to smithi13667081-13 2024-09-12T23:47:11.940 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 13 v 1621 (ObjNum 820 snap 258 seq_num 820) dirty exists 2024-09-12T23:47:11.941 INFO:tasks.rados.rados.0.smithi136.stdout:2561: left oid 13 (ObjNum 820 snap 258 seq_num 820) 2024-09-12T23:47:11.941 INFO:tasks.rados.rados.0.smithi136.stdout:2562: finishing write tid 5 to smithi13667081-17 2024-09-12T23:47:11.941 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 17 v 1794 (ObjNum 821 snap 258 seq_num 821) dirty exists 2024-09-12T23:47:11.941 INFO:tasks.rados.rados.0.smithi136.stdout:2562: left oid 17 (ObjNum 821 snap 258 seq_num 821) 2024-09-12T23:47:11.941 INFO:tasks.rados.rados.0.smithi136.stdout:2564: finishing write tid 2 to smithi13667081-46 2024-09-12T23:47:11.941 INFO:tasks.rados.rados.0.smithi136.stdout:2564: finishing write tid 3 to smithi13667081-46 2024-09-12T23:47:11.941 INFO:tasks.rados.rados.0.smithi136.stdout:2564: finishing write tid 4 to smithi13667081-46 2024-09-12T23:47:11.941 INFO:tasks.rados.rados.0.smithi136.stdout:2564: finishing write tid 5 to smithi13667081-46 2024-09-12T23:47:11.941 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 46 v 1275 (ObjNum 822 snap 258 seq_num 822) dirty exists 2024-09-12T23:47:11.941 INFO:tasks.rados.rados.0.smithi136.stdout:2564: left oid 46 (ObjNum 822 snap 258 seq_num 822) 2024-09-12T23:47:11.954 INFO:tasks.rados.rados.0.smithi136.stdout:2566: finishing copy_from racing read to smithi13667081-20 2024-09-12T23:47:11.958 INFO:tasks.rados.rados.0.smithi136.stdout:2567: finishing copy_from to smithi13667081-4 2024-09-12T23:47:11.958 INFO:tasks.rados.rados.0.smithi136.stdout:2567: got expected ENOENT (src dne) 2024-09-12T23:47:11.958 INFO:tasks.rados.rados.0.smithi136.stdout:2567: finishing copy_from racing read to smithi13667081-4 2024-09-12T23:47:11.963 INFO:tasks.rados.rados.0.smithi136.stdout:2566: finishing copy_from to smithi13667081-20 2024-09-12T23:47:11.963 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 20 v 1591 (ObjNum 739 snap 227 seq_num 739) dirty exists 2024-09-12T23:47:12.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:11 smithi136 bash[75164]: audit 2024-09-12T23:47:09.988415+0000 mgr.y (mgr.24661) 250 : audit [DBG] from='client.44282 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:47:12.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:11 smithi136 bash[75164]: audit 2024-09-12T23:47:10.649550+0000 mon.a (mon.0) 795 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:12.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:11 smithi136 bash[75164]: audit 2024-09-12T23:47:10.659443+0000 mon.a (mon.0) 796 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:12.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:11 smithi136 bash[75164]: cluster 2024-09-12T23:47:10.767195+0000 mgr.y (mgr.24661) 251 : cluster [DBG] pgmap v436: 121 pgs: 34 active+undersized, 8 active+undersized+degraded, 79 active+clean; 343 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.4 MiB/s rd, 4.2 MiB/s wr, 26 op/s; 36/597 objects degraded (6.030%) 2024-09-12T23:47:12.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:11 smithi136 bash[75164]: audit 2024-09-12T23:47:10.917227+0000 mon.a (mon.0) 797 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-12T23:47:12.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:11 smithi136 bash[75164]: cluster 2024-09-12T23:47:10.917478+0000 mon.a (mon.0) 798 : cluster [DBG] osdmap e574: 8 total, 7 up, 8 in 2024-09-12T23:47:12.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:11 smithi136 bash[75164]: audit 2024-09-12T23:47:10.922378+0000 mon.a (mon.0) 799 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-12T23:47:12.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:11 smithi136 bash[75164]: audit 2024-09-12T23:47:10.922441+0000 mon.b (mon.1) 9 : audit [INF] from='osd.6 [v2:172.21.15.181:6816/1329407811,v1:172.21.15.181:6817/1329407811]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-12T23:47:12.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:11 smithi136 bash[75164]: audit 2024-09-12T23:47:11.464842+0000 mon.a (mon.0) 800 : audit [INF] from='osd.6 ' entity='osd.6' 2024-09-12T23:47:12.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:11 smithi136 bash[76371]: audit 2024-09-12T23:47:09.988415+0000 mgr.y (mgr.24661) 250 : audit [DBG] from='client.44282 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:47:12.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:11 smithi136 bash[76371]: audit 2024-09-12T23:47:10.649550+0000 mon.a (mon.0) 795 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:12.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:11 smithi136 bash[76371]: audit 2024-09-12T23:47:10.659443+0000 mon.a (mon.0) 796 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:12.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:11 smithi136 bash[76371]: cluster 2024-09-12T23:47:10.767195+0000 mgr.y (mgr.24661) 251 : cluster [DBG] pgmap v436: 121 pgs: 34 active+undersized, 8 active+undersized+degraded, 79 active+clean; 343 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.4 MiB/s rd, 4.2 MiB/s wr, 26 op/s; 36/597 objects degraded (6.030%) 2024-09-12T23:47:12.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:11 smithi136 bash[76371]: audit 2024-09-12T23:47:10.917227+0000 mon.a (mon.0) 797 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-12T23:47:12.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:11 smithi136 bash[76371]: cluster 2024-09-12T23:47:10.917478+0000 mon.a (mon.0) 798 : cluster [DBG] osdmap e574: 8 total, 7 up, 8 in 2024-09-12T23:47:12.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:11 smithi136 bash[76371]: audit 2024-09-12T23:47:10.922378+0000 mon.a (mon.0) 799 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-12T23:47:12.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:11 smithi136 bash[76371]: audit 2024-09-12T23:47:10.922441+0000 mon.b (mon.1) 9 : audit [INF] from='osd.6 [v2:172.21.15.181:6816/1329407811,v1:172.21.15.181:6817/1329407811]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-12T23:47:12.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:11 smithi136 bash[76371]: audit 2024-09-12T23:47:11.464842+0000 mon.a (mon.0) 800 : audit [INF] from='osd.6 ' entity='osd.6' 2024-09-12T23:47:12.594 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:47:12 smithi181 bash[60471]: ts=2024-09-12T23:47:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:47:12.984 INFO:tasks.rados.rados.0.smithi136.stdout:2561: done (6 left) 2024-09-12T23:47:12.984 INFO:tasks.rados.rados.0.smithi136.stdout:2562: done (5 left) 2024-09-12T23:47:12.984 INFO:tasks.rados.rados.0.smithi136.stdout:2563: done (4 left) 2024-09-12T23:47:12.984 INFO:tasks.rados.rados.0.smithi136.stdout:2564: done (3 left) 2024-09-12T23:47:12.984 INFO:tasks.rados.rados.0.smithi136.stdout:2566: done (2 left) 2024-09-12T23:47:12.984 INFO:tasks.rados.rados.0.smithi136.stdout:2567: done (1 left) 2024-09-12T23:47:12.984 INFO:tasks.rados.rados.0.smithi136.stdout:2568: done (0 left) 2024-09-12T23:47:12.985 INFO:tasks.rados.rados.0.smithi136.stdout:2569: read oid 34 snap -1 2024-09-12T23:47:12.985 INFO:tasks.rados.rados.0.smithi136.stdout:2569: expect (ObjNum 796 snap 253 seq_num 796) 2024-09-12T23:47:12.985 INFO:tasks.rados.rados.0.smithi136.stdout:2570: snap_remove snap 250 2024-09-12T23:47:13.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:12 smithi136 bash[75164]: cluster 2024-09-12T23:47:11.912284+0000 mon.a (mon.0) 801 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-12T23:47:13.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:12 smithi136 bash[75164]: cluster 2024-09-12T23:47:11.923510+0000 mon.a (mon.0) 802 : cluster [INF] osd.6 [v2:172.21.15.181:6816/1329407811,v1:172.21.15.181:6817/1329407811] boot 2024-09-12T23:47:13.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:12 smithi136 bash[75164]: cluster 2024-09-12T23:47:11.923629+0000 mon.a (mon.0) 803 : cluster [DBG] osdmap e575: 8 total, 8 up, 8 in 2024-09-12T23:47:13.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:12 smithi136 bash[75164]: audit 2024-09-12T23:47:11.924614+0000 mon.a (mon.0) 804 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-12T23:47:13.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:12 smithi136 bash[76371]: cluster 2024-09-12T23:47:11.912284+0000 mon.a (mon.0) 801 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-12T23:47:13.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:12 smithi136 bash[76371]: cluster 2024-09-12T23:47:11.923510+0000 mon.a (mon.0) 802 : cluster [INF] osd.6 [v2:172.21.15.181:6816/1329407811,v1:172.21.15.181:6817/1329407811] boot 2024-09-12T23:47:13.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:12 smithi136 bash[76371]: cluster 2024-09-12T23:47:11.923629+0000 mon.a (mon.0) 803 : cluster [DBG] osdmap e575: 8 total, 8 up, 8 in 2024-09-12T23:47:13.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:12 smithi136 bash[76371]: audit 2024-09-12T23:47:11.924614+0000 mon.a (mon.0) 804 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-12T23:47:13.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:12 smithi181 bash[67587]: cluster 2024-09-12T23:47:11.912284+0000 mon.a (mon.0) 801 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-12T23:47:13.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:12 smithi181 bash[67587]: cluster 2024-09-12T23:47:11.923510+0000 mon.a (mon.0) 802 : cluster [INF] osd.6 [v2:172.21.15.181:6816/1329407811,v1:172.21.15.181:6817/1329407811] boot 2024-09-12T23:47:13.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:12 smithi181 bash[67587]: cluster 2024-09-12T23:47:11.923629+0000 mon.a (mon.0) 803 : cluster [DBG] osdmap e575: 8 total, 8 up, 8 in 2024-09-12T23:47:13.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:12 smithi181 bash[67587]: audit 2024-09-12T23:47:11.924614+0000 mon.a (mon.0) 804 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-12T23:47:13.930 INFO:tasks.rados.rados.0.smithi136.stdout:2570: done (1 left) 2024-09-12T23:47:13.930 INFO:tasks.rados.rados.0.smithi136.stdout:2571: snap_create 2024-09-12T23:47:13.930 INFO:tasks.rados.rados.0.smithi136.stdout:2569: expect (ObjNum 796 snap 253 seq_num 796) 2024-09-12T23:47:14.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:13 smithi136 bash[75164]: cluster 2024-09-12T23:47:12.768100+0000 mgr.y (mgr.24661) 252 : cluster [DBG] pgmap v439: 121 pgs: 6 peering, 29 active+undersized, 7 active+undersized+degraded, 79 active+clean; 347 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6.4 MiB/s rd, 3.9 MiB/s wr, 9 op/s; 25/600 objects degraded (4.167%) 2024-09-12T23:47:14.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:13 smithi136 bash[75164]: cluster 2024-09-12T23:47:12.925441+0000 mon.a (mon.0) 805 : cluster [DBG] osdmap e576: 8 total, 8 up, 8 in 2024-09-12T23:47:14.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:13 smithi136 bash[76371]: cluster 2024-09-12T23:47:12.768100+0000 mgr.y (mgr.24661) 252 : cluster [DBG] pgmap v439: 121 pgs: 6 peering, 29 active+undersized, 7 active+undersized+degraded, 79 active+clean; 347 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6.4 MiB/s rd, 3.9 MiB/s wr, 9 op/s; 25/600 objects degraded (4.167%) 2024-09-12T23:47:14.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:13 smithi136 bash[76371]: cluster 2024-09-12T23:47:12.925441+0000 mon.a (mon.0) 805 : cluster [DBG] osdmap e576: 8 total, 8 up, 8 in 2024-09-12T23:47:14.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:13 smithi181 bash[67587]: cluster 2024-09-12T23:47:12.768100+0000 mgr.y (mgr.24661) 252 : cluster [DBG] pgmap v439: 121 pgs: 6 peering, 29 active+undersized, 7 active+undersized+degraded, 79 active+clean; 347 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6.4 MiB/s rd, 3.9 MiB/s wr, 9 op/s; 25/600 objects degraded (4.167%) 2024-09-12T23:47:14.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:13 smithi181 bash[67587]: cluster 2024-09-12T23:47:12.925441+0000 mon.a (mon.0) 805 : cluster [DBG] osdmap e576: 8 total, 8 up, 8 in 2024-09-12T23:47:14.929 INFO:tasks.rados.rados.0.smithi136.stdout:2569: done (1 left) 2024-09-12T23:47:14.930 INFO:tasks.rados.rados.0.smithi136.stdout:2571: done (0 left) 2024-09-12T23:47:14.930 INFO:tasks.rados.rados.0.smithi136.stdout:2572: snap_remove snap 253 2024-09-12T23:47:15.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:14 smithi136 bash[75164]: cluster 2024-09-12T23:47:13.663541+0000 mon.a (mon.0) 806 : cluster [WRN] Health check update: Degraded data redundancy: 25/600 objects degraded (4.167%), 7 pgs degraded (PG_DEGRADED) 2024-09-12T23:47:15.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:14 smithi136 bash[75164]: cluster 2024-09-12T23:47:13.929805+0000 mon.a (mon.0) 807 : cluster [DBG] osdmap e577: 8 total, 8 up, 8 in 2024-09-12T23:47:15.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:14 smithi136 bash[76371]: cluster 2024-09-12T23:47:13.663541+0000 mon.a (mon.0) 806 : cluster [WRN] Health check update: Degraded data redundancy: 25/600 objects degraded (4.167%), 7 pgs degraded (PG_DEGRADED) 2024-09-12T23:47:15.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:14 smithi136 bash[76371]: cluster 2024-09-12T23:47:13.929805+0000 mon.a (mon.0) 807 : cluster [DBG] osdmap e577: 8 total, 8 up, 8 in 2024-09-12T23:47:15.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:14 smithi181 bash[67587]: cluster 2024-09-12T23:47:13.663541+0000 mon.a (mon.0) 806 : cluster [WRN] Health check update: Degraded data redundancy: 25/600 objects degraded (4.167%), 7 pgs degraded (PG_DEGRADED) 2024-09-12T23:47:15.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:14 smithi181 bash[67587]: cluster 2024-09-12T23:47:13.929805+0000 mon.a (mon.0) 807 : cluster [DBG] osdmap e577: 8 total, 8 up, 8 in 2024-09-12T23:47:15.935 INFO:tasks.rados.rados.0.smithi136.stdout:2572: done (0 left) 2024-09-12T23:47:15.935 INFO:tasks.rados.rados.0.smithi136.stdout:2573: copy_from oid 33 from oid 14 current snap is 259 2024-09-12T23:47:15.935 INFO:tasks.rados.rados.0.smithi136.stdout:2574: rmattr oid 20 current snap is 259 2024-09-12T23:47:15.935 INFO:tasks.rados.rados.0.smithi136.stdout:2574: done (1 left) 2024-09-12T23:47:15.935 INFO:tasks.rados.rados.0.smithi136.stdout:2575: snap_remove snap 244 2024-09-12T23:47:16.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:15 smithi136 bash[75164]: cluster 2024-09-12T23:47:14.769105+0000 mgr.y (mgr.24661) 253 : cluster [DBG] pgmap v442: 121 pgs: 6 peering, 22 active+undersized, 3 active+undersized+degraded, 90 active+clean; 361 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 2.1 MiB/s wr, 12 op/s; 6/612 objects degraded (0.980%); 1.7 MiB/s, 0 objects/s recovering 2024-09-12T23:47:16.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:15 smithi136 bash[75164]: cluster 2024-09-12T23:47:14.932571+0000 mon.a (mon.0) 808 : cluster [DBG] osdmap e578: 8 total, 8 up, 8 in 2024-09-12T23:47:16.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:15 smithi136 bash[76371]: cluster 2024-09-12T23:47:14.769105+0000 mgr.y (mgr.24661) 253 : cluster [DBG] pgmap v442: 121 pgs: 6 peering, 22 active+undersized, 3 active+undersized+degraded, 90 active+clean; 361 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 2.1 MiB/s wr, 12 op/s; 6/612 objects degraded (0.980%); 1.7 MiB/s, 0 objects/s recovering 2024-09-12T23:47:16.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:15 smithi136 bash[76371]: cluster 2024-09-12T23:47:14.932571+0000 mon.a (mon.0) 808 : cluster [DBG] osdmap e578: 8 total, 8 up, 8 in 2024-09-12T23:47:16.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:15 smithi181 bash[67587]: cluster 2024-09-12T23:47:14.769105+0000 mgr.y (mgr.24661) 253 : cluster [DBG] pgmap v442: 121 pgs: 6 peering, 22 active+undersized, 3 active+undersized+degraded, 90 active+clean; 361 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 2.1 MiB/s wr, 12 op/s; 6/612 objects degraded (0.980%); 1.7 MiB/s, 0 objects/s recovering 2024-09-12T23:47:16.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:15 smithi181 bash[67587]: cluster 2024-09-12T23:47:14.932571+0000 mon.a (mon.0) 808 : cluster [DBG] osdmap e578: 8 total, 8 up, 8 in 2024-09-12T23:47:16.943 INFO:tasks.rados.rados.0.smithi136.stdout:2575: done (1 left) 2024-09-12T23:47:16.943 INFO:tasks.rados.rados.0.smithi136.stdout:2576: snap_create 2024-09-12T23:47:16.943 INFO:tasks.rados.rados.0.smithi136.stdout:2573: finishing copy_from to smithi13667081-33 2024-09-12T23:47:16.943 INFO:tasks.rados.rados.0.smithi136.stdout:2573: got expected ENOENT (src dne) 2024-09-12T23:47:16.943 INFO:tasks.rados.rados.0.smithi136.stdout:2573: finishing copy_from racing read to smithi13667081-33 2024-09-12T23:47:17.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:16 smithi181 bash[67587]: cluster 2024-09-12T23:47:15.935392+0000 mon.a (mon.0) 809 : cluster [DBG] osdmap e579: 8 total, 8 up, 8 in 2024-09-12T23:47:17.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:16 smithi181 bash[67587]: cluster 2024-09-12T23:47:16.770206+0000 mgr.y (mgr.24661) 254 : cluster [DBG] pgmap v445: 121 pgs: 6 peering, 115 active+clean; 362 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.3 MiB/s wr, 8 op/s; 1.7 MiB/s, 0 objects/s recovering 2024-09-12T23:47:17.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:16 smithi136 bash[75164]: cluster 2024-09-12T23:47:15.935392+0000 mon.a (mon.0) 809 : cluster [DBG] osdmap e579: 8 total, 8 up, 8 in 2024-09-12T23:47:17.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:16 smithi136 bash[75164]: cluster 2024-09-12T23:47:16.770206+0000 mgr.y (mgr.24661) 254 : cluster [DBG] pgmap v445: 121 pgs: 6 peering, 115 active+clean; 362 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.3 MiB/s wr, 8 op/s; 1.7 MiB/s, 0 objects/s recovering 2024-09-12T23:47:17.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:16 smithi136 bash[76371]: cluster 2024-09-12T23:47:15.935392+0000 mon.a (mon.0) 809 : cluster [DBG] osdmap e579: 8 total, 8 up, 8 in 2024-09-12T23:47:17.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:16 smithi136 bash[76371]: cluster 2024-09-12T23:47:16.770206+0000 mgr.y (mgr.24661) 254 : cluster [DBG] pgmap v445: 121 pgs: 6 peering, 115 active+clean; 362 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.3 MiB/s wr, 8 op/s; 1.7 MiB/s, 0 objects/s recovering 2024-09-12T23:47:17.946 INFO:tasks.rados.rados.0.smithi136.stdout:2573: done (1 left) 2024-09-12T23:47:17.946 INFO:tasks.rados.rados.0.smithi136.stdout:2576: done (0 left) 2024-09-12T23:47:17.946 INFO:tasks.rados.rados.0.smithi136.stdout:2577: read oid 47 snap -1 2024-09-12T23:47:17.946 INFO:tasks.rados.rados.0.smithi136.stdout:2577: expect (ObjNum 636 snap 188 seq_num 636) 2024-09-12T23:47:17.946 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 12 current snap is 260 2024-09-12T23:47:17.946 INFO:tasks.rados.rados.0.smithi136.stdout:2578: seq_num 824 ranges {65536=606208,671744=458752,1130496=573440,1703936=172032} 2024-09-12T23:47:17.957 INFO:tasks.rados.rados.0.smithi136.stdout:2578: writing smithi13667081-12 from 65536 to 671744 tid 1 2024-09-12T23:47:17.964 INFO:tasks.rados.rados.0.smithi136.stdout:2578: writing smithi13667081-12 from 671744 to 1130496 tid 2 2024-09-12T23:47:17.972 INFO:tasks.rados.rados.0.smithi136.stdout:2578: writing smithi13667081-12 from 1130496 to 1703936 tid 3 2024-09-12T23:47:17.974 INFO:tasks.rados.rados.0.smithi136.stdout:2578: writing smithi13667081-12 from 1703936 to 1875968 tid 4 2024-09-12T23:47:17.975 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 8 current snap is 260 2024-09-12T23:47:17.975 INFO:tasks.rados.rados.0.smithi136.stdout:2579: seq_num 825 ranges {1458176=622592,2080768=745472,2826240=458752} 2024-09-12T23:47:18.000 INFO:tasks.rados.rados.0.smithi136.stdout:2579: writing smithi13667081-8 from 1458176 to 2080768 tid 1 2024-09-12T23:47:18.007 INFO:tasks.rados.rados.0.smithi136.stdout:2579: writing smithi13667081-8 from 2080768 to 2826240 tid 2 2024-09-12T23:47:18.011 INFO:tasks.rados.rados.0.smithi136.stdout:2579: writing smithi13667081-8 from 2826240 to 3284992 tid 3 2024-09-12T23:47:18.011 INFO:tasks.rados.rados.0.smithi136.stdout:2580: delete oid 31 current snap is 260 2024-09-12T23:47:18.011 INFO:tasks.rados.rados.0.smithi136.stdout:2577: expect (ObjNum 636 snap 188 seq_num 636) 2024-09-12T23:47:18.081 INFO:tasks.rados.rados.0.smithi136.stdout:2578: finishing write tid 1 to smithi13667081-12 2024-09-12T23:47:18.081 INFO:tasks.rados.rados.0.smithi136.stdout:2578: finishing write tid 2 to smithi13667081-12 2024-09-12T23:47:18.081 INFO:tasks.rados.rados.0.smithi136.stdout:2578: finishing write tid 3 to smithi13667081-12 2024-09-12T23:47:18.081 INFO:tasks.rados.rados.0.smithi136.stdout:2578: finishing write tid 4 to smithi13667081-12 2024-09-12T23:47:18.081 INFO:tasks.rados.rados.0.smithi136.stdout:2578: finishing write tid 5 to smithi13667081-12 2024-09-12T23:47:18.081 INFO:tasks.rados.rados.0.smithi136.stdout:2578: finishing write tid 6 to smithi13667081-12 2024-09-12T23:47:18.082 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 12 v 1482 (ObjNum 824 snap 260 seq_num 824) dirty exists 2024-09-12T23:47:18.082 INFO:tasks.rados.rados.0.smithi136.stdout:2578: left oid 12 (ObjNum 824 snap 260 seq_num 824) 2024-09-12T23:47:18.082 INFO:tasks.rados.rados.0.smithi136.stdout:2579: finishing write tid 1 to smithi13667081-8 2024-09-12T23:47:18.082 INFO:tasks.rados.rados.0.smithi136.stdout:2579: finishing write tid 2 to smithi13667081-8 2024-09-12T23:47:18.082 INFO:tasks.rados.rados.0.smithi136.stdout:2579: finishing write tid 3 to smithi13667081-8 2024-09-12T23:47:18.082 INFO:tasks.rados.rados.0.smithi136.stdout:2579: finishing write tid 4 to smithi13667081-8 2024-09-12T23:47:18.082 INFO:tasks.rados.rados.0.smithi136.stdout:2579: finishing write tid 5 to smithi13667081-8 2024-09-12T23:47:18.082 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 8 v 1251 (ObjNum 825 snap 260 seq_num 825) dirty exists 2024-09-12T23:47:18.082 INFO:tasks.rados.rados.0.smithi136.stdout:2579: left oid 8 (ObjNum 825 snap 260 seq_num 825) 2024-09-12T23:47:18.082 INFO:tasks.rados.rados.0.smithi136.stdout:2577: done (3 left) 2024-09-12T23:47:18.082 INFO:tasks.rados.rados.0.smithi136.stdout:2578: done (2 left) 2024-09-12T23:47:18.082 INFO:tasks.rados.rados.0.smithi136.stdout:2579: done (1 left) 2024-09-12T23:47:18.083 INFO:tasks.rados.rados.0.smithi136.stdout:2580: done (0 left) 2024-09-12T23:47:18.083 INFO:tasks.rados.rados.0.smithi136.stdout:2581: delete oid 43 current snap is 260 2024-09-12T23:47:18.089 INFO:tasks.rados.rados.0.smithi136.stdout:2581: done (0 left) 2024-09-12T23:47:18.089 INFO:tasks.rados.rados.0.smithi136.stdout:2582: read oid 13 snap -1 2024-09-12T23:47:18.089 INFO:tasks.rados.rados.0.smithi136.stdout:2582: expect (ObjNum 820 snap 258 seq_num 820) 2024-09-12T23:47:18.089 INFO:tasks.rados.rados.0.smithi136.stdout:2583: read oid 25 snap -1 2024-09-12T23:47:18.089 INFO:tasks.rados.rados.0.smithi136.stdout:2583: expect deleted 2024-09-12T23:47:18.089 INFO:tasks.rados.rados.0.smithi136.stdout:2584: snap_remove snap 228 2024-09-12T23:47:18.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:17 smithi181 bash[67587]: cluster 2024-09-12T23:47:16.928441+0000 mon.a (mon.0) 810 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6/612 objects degraded (0.980%), 3 pgs degraded) 2024-09-12T23:47:18.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:17 smithi181 bash[67587]: cluster 2024-09-12T23:47:16.928486+0000 mon.a (mon.0) 811 : cluster [INF] Cluster is now healthy 2024-09-12T23:47:18.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:17 smithi181 bash[67587]: cluster 2024-09-12T23:47:16.945939+0000 mon.a (mon.0) 812 : cluster [DBG] osdmap e580: 8 total, 8 up, 8 in 2024-09-12T23:47:18.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:17 smithi136 bash[75164]: cluster 2024-09-12T23:47:16.928441+0000 mon.a (mon.0) 810 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6/612 objects degraded (0.980%), 3 pgs degraded) 2024-09-12T23:47:18.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:17 smithi136 bash[75164]: cluster 2024-09-12T23:47:16.928486+0000 mon.a (mon.0) 811 : cluster [INF] Cluster is now healthy 2024-09-12T23:47:18.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:17 smithi136 bash[75164]: cluster 2024-09-12T23:47:16.945939+0000 mon.a (mon.0) 812 : cluster [DBG] osdmap e580: 8 total, 8 up, 8 in 2024-09-12T23:47:18.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:17 smithi136 bash[76371]: cluster 2024-09-12T23:47:16.928441+0000 mon.a (mon.0) 810 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6/612 objects degraded (0.980%), 3 pgs degraded) 2024-09-12T23:47:18.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:17 smithi136 bash[76371]: cluster 2024-09-12T23:47:16.928486+0000 mon.a (mon.0) 811 : cluster [INF] Cluster is now healthy 2024-09-12T23:47:18.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:17 smithi136 bash[76371]: cluster 2024-09-12T23:47:16.945939+0000 mon.a (mon.0) 812 : cluster [DBG] osdmap e580: 8 total, 8 up, 8 in 2024-09-12T23:47:18.972 INFO:tasks.rados.rados.0.smithi136.stdout:2584: done (2 left) 2024-09-12T23:47:18.972 INFO:tasks.rados.rados.0.smithi136.stdout:2585: copy_from oid 26 from oid 46 current snap is 260 2024-09-12T23:47:18.973 INFO:tasks.rados.rados.0.smithi136.stdout:2582: expect (ObjNum 820 snap 258 seq_num 820) 2024-09-12T23:47:19.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:18 smithi136 bash[75164]: cluster 2024-09-12T23:47:17.947448+0000 mon.a (mon.0) 813 : cluster [DBG] osdmap e581: 8 total, 8 up, 8 in 2024-09-12T23:47:19.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:18 smithi136 bash[75164]: cluster 2024-09-12T23:47:18.771202+0000 mgr.y (mgr.24661) 255 : cluster [DBG] pgmap v448: 121 pgs: 121 active+clean; 359 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5.1 MiB/s rd, 94 KiB/s wr, 1 op/s; 660 KiB/s, 0 objects/s recovering 2024-09-12T23:47:19.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:18 smithi136 bash[76371]: cluster 2024-09-12T23:47:17.947448+0000 mon.a (mon.0) 813 : cluster [DBG] osdmap e581: 8 total, 8 up, 8 in 2024-09-12T23:47:19.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:18 smithi136 bash[76371]: cluster 2024-09-12T23:47:18.771202+0000 mgr.y (mgr.24661) 255 : cluster [DBG] pgmap v448: 121 pgs: 121 active+clean; 359 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5.1 MiB/s rd, 94 KiB/s wr, 1 op/s; 660 KiB/s, 0 objects/s recovering 2024-09-12T23:47:19.344 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:18 smithi181 bash[67587]: cluster 2024-09-12T23:47:17.947448+0000 mon.a (mon.0) 813 : cluster [DBG] osdmap e581: 8 total, 8 up, 8 in 2024-09-12T23:47:19.344 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:18 smithi181 bash[67587]: cluster 2024-09-12T23:47:18.771202+0000 mgr.y (mgr.24661) 255 : cluster [DBG] pgmap v448: 121 pgs: 121 active+clean; 359 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5.1 MiB/s rd, 94 KiB/s wr, 1 op/s; 660 KiB/s, 0 objects/s recovering 2024-09-12T23:47:19.463 INFO:tasks.rados.rados.0.smithi136.stdout:2582: done (2 left) 2024-09-12T23:47:19.463 INFO:tasks.rados.rados.0.smithi136.stdout:2583: done (1 left) 2024-09-12T23:47:19.464 INFO:tasks.rados.rados.0.smithi136.stdout:2586: snap_create 2024-09-12T23:47:19.478 INFO:tasks.rados.rados.0.smithi136.stdout:2585: finishing copy_from racing read to smithi13667081-26 2024-09-12T23:47:19.478 INFO:tasks.rados.rados.0.smithi136.stdout:2585: finishing copy_from to smithi13667081-26 2024-09-12T23:47:19.479 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 26 v 1275 (ObjNum 822 snap 258 seq_num 822) dirty exists 2024-09-12T23:47:19.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:47:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:47:19] "GET /metrics HTTP/1.1" 200 36386 "" "Prometheus/2.43.0" 2024-09-12T23:47:19.974 INFO:tasks.rados.rados.0.smithi136.stdout:2585: done (1 left) 2024-09-12T23:47:19.975 INFO:tasks.rados.rados.0.smithi136.stdout:2586: done (0 left) 2024-09-12T23:47:19.975 INFO:tasks.rados.rados.0.smithi136.stdout:2587: setattr oid 23 current snap is 261 2024-09-12T23:47:19.985 INFO:tasks.rados.rados.0.smithi136.stdout:2588: snap_create 2024-09-12T23:47:19.992 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 23 v 1836 (ObjNum 744 snap 228 seq_num 744) dirty exists 2024-09-12T23:47:20.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:19 smithi181 bash[67587]: cluster 2024-09-12T23:47:18.975069+0000 mon.a (mon.0) 814 : cluster [DBG] osdmap e582: 8 total, 8 up, 8 in 2024-09-12T23:47:20.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:19 smithi136 bash[75164]: cluster 2024-09-12T23:47:18.975069+0000 mon.a (mon.0) 814 : cluster [DBG] osdmap e582: 8 total, 8 up, 8 in 2024-09-12T23:47:20.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:19 smithi136 bash[76371]: cluster 2024-09-12T23:47:18.975069+0000 mon.a (mon.0) 814 : cluster [DBG] osdmap e582: 8 total, 8 up, 8 in 2024-09-12T23:47:20.999 INFO:tasks.rados.rados.0.smithi136.stdout:2587: done (1 left) 2024-09-12T23:47:21.000 INFO:tasks.rados.rados.0.smithi136.stdout:2588: done (0 left) 2024-09-12T23:47:21.000 INFO:tasks.rados.rados.0.smithi136.stdout:2589: read oid 30 snap 252 2024-09-12T23:47:21.000 INFO:tasks.rados.rados.0.smithi136.stdout:2589: expect (ObjNum 734 snap 226 seq_num 734) 2024-09-12T23:47:21.000 INFO:tasks.rados.rados.0.smithi136.stdout:2590: delete oid 39 current snap is 262 2024-09-12T23:47:21.008 INFO:tasks.rados.rados.0.smithi136.stdout:2589: expect (ObjNum 734 snap 226 seq_num 734) 2024-09-12T23:47:21.021 INFO:tasks.rados.rados.0.smithi136.stdout:2589: done (1 left) 2024-09-12T23:47:21.022 INFO:tasks.rados.rados.0.smithi136.stdout:2590: done (0 left) 2024-09-12T23:47:21.022 INFO:tasks.rados.rados.0.smithi136.stdout:2591: snap_remove snap 241 2024-09-12T23:47:21.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:20 smithi136 bash[75164]: cluster 2024-09-12T23:47:19.976943+0000 mon.a (mon.0) 815 : cluster [DBG] osdmap e583: 8 total, 8 up, 8 in 2024-09-12T23:47:21.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:20 smithi136 bash[75164]: audit 2024-09-12T23:47:20.546385+0000 mon.a (mon.0) 816 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:21.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:20 smithi136 bash[75164]: audit 2024-09-12T23:47:20.556936+0000 mon.a (mon.0) 817 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:21.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:20 smithi136 bash[75164]: audit 2024-09-12T23:47:20.559208+0000 mon.a (mon.0) 818 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:47:21.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:20 smithi136 bash[75164]: audit 2024-09-12T23:47:20.561028+0000 mon.a (mon.0) 819 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:47:21.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:20 smithi136 bash[75164]: audit 2024-09-12T23:47:20.571656+0000 mon.a (mon.0) 820 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:21.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:20 smithi136 bash[75164]: audit 2024-09-12T23:47:20.726969+0000 mon.a (mon.0) 821 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:47:21.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:20 smithi136 bash[75164]: audit 2024-09-12T23:47:20.731545+0000 mon.a (mon.0) 822 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:47:21.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:20 smithi136 bash[75164]: audit 2024-09-12T23:47:20.733921+0000 mon.a (mon.0) 823 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:47:21.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:20 smithi136 bash[75164]: audit 2024-09-12T23:47:20.735784+0000 mon.a (mon.0) 824 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:47:21.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:20 smithi136 bash[75164]: audit 2024-09-12T23:47:20.738079+0000 mon.a (mon.0) 825 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-12T23:47:21.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:20 smithi136 bash[75164]: audit 2024-09-12T23:47:20.738403+0000 mgr.y (mgr.24661) 256 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-12T23:47:21.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:20 smithi136 bash[75164]: cephadm 2024-09-12T23:47:20.740061+0000 mgr.y (mgr.24661) 257 : cephadm [INF] Upgrade: osd.7 is safe to restart 2024-09-12T23:47:21.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:20 smithi136 bash[75164]: cluster 2024-09-12T23:47:20.772180+0000 mgr.y (mgr.24661) 258 : cluster [DBG] pgmap v451: 121 pgs: 121 active+clean; 353 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5.9 MiB/s rd, 0 op/s; 660 KiB/s, 0 objects/s recovering 2024-09-12T23:47:21.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:20 smithi136 bash[76371]: cluster 2024-09-12T23:47:19.976943+0000 mon.a (mon.0) 815 : cluster [DBG] osdmap e583: 8 total, 8 up, 8 in 2024-09-12T23:47:21.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:20 smithi136 bash[76371]: audit 2024-09-12T23:47:20.546385+0000 mon.a (mon.0) 816 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:21.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:20 smithi136 bash[76371]: audit 2024-09-12T23:47:20.556936+0000 mon.a (mon.0) 817 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:21.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:20 smithi136 bash[76371]: audit 2024-09-12T23:47:20.559208+0000 mon.a (mon.0) 818 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:47:21.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:20 smithi136 bash[76371]: audit 2024-09-12T23:47:20.561028+0000 mon.a (mon.0) 819 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:47:21.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:20 smithi136 bash[76371]: audit 2024-09-12T23:47:20.571656+0000 mon.a (mon.0) 820 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:21.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:20 smithi136 bash[76371]: audit 2024-09-12T23:47:20.726969+0000 mon.a (mon.0) 821 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:47:21.262 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:20 smithi136 bash[76371]: audit 2024-09-12T23:47:20.731545+0000 mon.a (mon.0) 822 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:47:21.262 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:20 smithi136 bash[76371]: audit 2024-09-12T23:47:20.733921+0000 mon.a (mon.0) 823 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:47:21.262 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:20 smithi136 bash[76371]: audit 2024-09-12T23:47:20.735784+0000 mon.a (mon.0) 824 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:47:21.262 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:20 smithi136 bash[76371]: audit 2024-09-12T23:47:20.738079+0000 mon.a (mon.0) 825 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-12T23:47:21.262 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:20 smithi136 bash[76371]: audit 2024-09-12T23:47:20.738403+0000 mgr.y (mgr.24661) 256 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-12T23:47:21.263 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:20 smithi136 bash[76371]: cephadm 2024-09-12T23:47:20.740061+0000 mgr.y (mgr.24661) 257 : cephadm [INF] Upgrade: osd.7 is safe to restart 2024-09-12T23:47:21.263 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:20 smithi136 bash[76371]: cluster 2024-09-12T23:47:20.772180+0000 mgr.y (mgr.24661) 258 : cluster [DBG] pgmap v451: 121 pgs: 121 active+clean; 353 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5.9 MiB/s rd, 0 op/s; 660 KiB/s, 0 objects/s recovering 2024-09-12T23:47:21.344 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:20 smithi181 bash[67587]: cluster 2024-09-12T23:47:19.976943+0000 mon.a (mon.0) 815 : cluster [DBG] osdmap e583: 8 total, 8 up, 8 in 2024-09-12T23:47:21.344 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:20 smithi181 bash[67587]: audit 2024-09-12T23:47:20.546385+0000 mon.a (mon.0) 816 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:21.344 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:20 smithi181 bash[67587]: audit 2024-09-12T23:47:20.556936+0000 mon.a (mon.0) 817 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:21.345 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:20 smithi181 bash[67587]: audit 2024-09-12T23:47:20.559208+0000 mon.a (mon.0) 818 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:47:21.345 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:20 smithi181 bash[67587]: audit 2024-09-12T23:47:20.561028+0000 mon.a (mon.0) 819 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:47:21.345 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:20 smithi181 bash[67587]: audit 2024-09-12T23:47:20.571656+0000 mon.a (mon.0) 820 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:21.345 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:20 smithi181 bash[67587]: audit 2024-09-12T23:47:20.726969+0000 mon.a (mon.0) 821 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:47:21.345 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:20 smithi181 bash[67587]: audit 2024-09-12T23:47:20.731545+0000 mon.a (mon.0) 822 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:47:21.345 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:20 smithi181 bash[67587]: audit 2024-09-12T23:47:20.733921+0000 mon.a (mon.0) 823 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:47:21.345 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:20 smithi181 bash[67587]: audit 2024-09-12T23:47:20.735784+0000 mon.a (mon.0) 824 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:47:21.345 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:20 smithi181 bash[67587]: audit 2024-09-12T23:47:20.738079+0000 mon.a (mon.0) 825 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-12T23:47:21.345 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:20 smithi181 bash[67587]: audit 2024-09-12T23:47:20.738403+0000 mgr.y (mgr.24661) 256 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-12T23:47:21.345 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:20 smithi181 bash[67587]: cephadm 2024-09-12T23:47:20.740061+0000 mgr.y (mgr.24661) 257 : cephadm [INF] Upgrade: osd.7 is safe to restart 2024-09-12T23:47:21.345 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:20 smithi181 bash[67587]: cluster 2024-09-12T23:47:20.772180+0000 mgr.y (mgr.24661) 258 : cluster [DBG] pgmap v451: 121 pgs: 121 active+clean; 353 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5.9 MiB/s rd, 0 op/s; 660 KiB/s, 0 objects/s recovering 2024-09-12T23:47:21.399 INFO:tasks.rados.rados.0.smithi136.stdout:2591: done (0 left) 2024-09-12T23:47:21.399 INFO:tasks.rados.rados.0.smithi136.stdout:2592: read oid 49 snap -1 2024-09-12T23:47:21.399 INFO:tasks.rados.rados.0.smithi136.stdout:2592: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-12T23:47:21.399 INFO:tasks.rados.rados.0.smithi136.stdout:2593: setattr oid 20 current snap is 262 2024-09-12T23:47:21.401 INFO:tasks.rados.rados.0.smithi136.stdout:2594: snap_create 2024-09-12T23:47:21.421 INFO:tasks.rados.rados.0.smithi136.stdout:2592: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-12T23:47:21.436 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 20 v 1592 (ObjNum 739 snap 227 seq_num 739) dirty exists 2024-09-12T23:47:22.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:21 smithi136 bash[75164]: cluster 2024-09-12T23:47:21.001928+0000 mon.a (mon.0) 826 : cluster [DBG] osdmap e584: 8 total, 8 up, 8 in 2024-09-12T23:47:22.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:21 smithi136 bash[75164]: cluster 2024-09-12T23:47:21.400300+0000 mon.a (mon.0) 827 : cluster [DBG] osdmap e585: 8 total, 8 up, 8 in 2024-09-12T23:47:22.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:21 smithi136 bash[76371]: cluster 2024-09-12T23:47:21.001928+0000 mon.a (mon.0) 826 : cluster [DBG] osdmap e584: 8 total, 8 up, 8 in 2024-09-12T23:47:22.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:21 smithi136 bash[76371]: cluster 2024-09-12T23:47:21.400300+0000 mon.a (mon.0) 827 : cluster [DBG] osdmap e585: 8 total, 8 up, 8 in 2024-09-12T23:47:22.266 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:22 smithi181 bash[67587]: cluster 2024-09-12T23:47:21.001928+0000 mon.a (mon.0) 826 : cluster [DBG] osdmap e584: 8 total, 8 up, 8 in 2024-09-12T23:47:22.266 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:22 smithi181 bash[67587]: cluster 2024-09-12T23:47:21.400300+0000 mon.a (mon.0) 827 : cluster [DBG] osdmap e585: 8 total, 8 up, 8 in 2024-09-12T23:47:22.403 INFO:tasks.rados.rados.0.smithi136.stdout:2592: done (2 left) 2024-09-12T23:47:22.403 INFO:tasks.rados.rados.0.smithi136.stdout:2593: done (1 left) 2024-09-12T23:47:22.403 INFO:tasks.rados.rados.0.smithi136.stdout:2594: done (0 left) 2024-09-12T23:47:22.403 INFO:tasks.rados.rados.0.smithi136.stdout:2595: read oid 21 snap -1 2024-09-12T23:47:22.403 INFO:tasks.rados.rados.0.smithi136.stdout:2595: expect (ObjNum 805 snap 253 seq_num 805) 2024-09-12T23:47:22.403 INFO:tasks.rados.rados.0.smithi136.stdout:2596: read oid 16 snap -1 2024-09-12T23:47:22.403 INFO:tasks.rados.rados.0.smithi136.stdout:2596: expect deleted 2024-09-12T23:47:22.403 INFO:tasks.rados.rados.0.smithi136.stdout:2597: read oid 33 snap -1 2024-09-12T23:47:22.403 INFO:tasks.rados.rados.0.smithi136.stdout:2597: expect deleted 2024-09-12T23:47:22.403 INFO:tasks.rados.rados.0.smithi136.stdout:2598: snap_create 2024-09-12T23:47:22.416 INFO:tasks.rados.rados.0.smithi136.stdout:2595: expect (ObjNum 805 snap 253 seq_num 805) 2024-09-12T23:47:22.594 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:47:22 smithi181 bash[60471]: ts=2024-09-12T23:47:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:47:23.415 INFO:tasks.rados.rados.0.smithi136.stdout:2595: done (3 left) 2024-09-12T23:47:23.415 INFO:tasks.rados.rados.0.smithi136.stdout:2596: done (2 left) 2024-09-12T23:47:23.415 INFO:tasks.rados.rados.0.smithi136.stdout:2597: done (1 left) 2024-09-12T23:47:23.415 INFO:tasks.rados.rados.0.smithi136.stdout:2598: done (0 left) 2024-09-12T23:47:23.415 INFO:tasks.rados.rados.0.smithi136.stdout:2599: copy_from oid 22 from oid 19 current snap is 264 2024-09-12T23:47:23.415 INFO:tasks.rados.rados.0.smithi136.stdout:2600: read oid 11 snap -1 2024-09-12T23:47:23.416 INFO:tasks.rados.rados.0.smithi136.stdout:2600: expect (ObjNum 605 snap 174 seq_num 605) 2024-09-12T23:47:23.416 INFO:tasks.rados.rados.0.smithi136.stdout:2601: snap_create 2024-09-12T23:47:23.441 INFO:tasks.rados.rados.0.smithi136.stdout:2599: finishing copy_from to smithi13667081-22 2024-09-12T23:47:23.441 INFO:tasks.rados.rados.0.smithi136.stdout:2599: got expected ENOENT (src dne) 2024-09-12T23:47:23.441 INFO:tasks.rados.rados.0.smithi136.stdout:2599: finishing copy_from racing read to smithi13667081-22 2024-09-12T23:47:23.455 INFO:tasks.rados.rados.0.smithi136.stdout:2600: expect (ObjNum 605 snap 174 seq_num 605) 2024-09-12T23:47:23.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:23 smithi181 bash[67587]: cluster 2024-09-12T23:47:22.404761+0000 mon.a (mon.0) 828 : cluster [DBG] osdmap e586: 8 total, 8 up, 8 in 2024-09-12T23:47:23.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:23 smithi181 bash[67587]: cluster 2024-09-12T23:47:22.773112+0000 mgr.y (mgr.24661) 259 : cluster [DBG] pgmap v455: 121 pgs: 121 active+clean; 346 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.0 MiB/s rd, 1.4 MiB/s wr, 7 op/s 2024-09-12T23:47:23.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:23 smithi181 bash[67587]: audit 2024-09-12T23:47:22.926991+0000 mon.a (mon.0) 829 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:23.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:23 smithi181 bash[67587]: audit 2024-09-12T23:47:22.929088+0000 mon.a (mon.0) 830 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-12T23:47:23.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:23 smithi181 bash[67587]: audit 2024-09-12T23:47:22.930790+0000 mon.a (mon.0) 831 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:47:23.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:23 smithi136 bash[76371]: cluster 2024-09-12T23:47:22.404761+0000 mon.a (mon.0) 828 : cluster [DBG] osdmap e586: 8 total, 8 up, 8 in 2024-09-12T23:47:23.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:23 smithi136 bash[76371]: cluster 2024-09-12T23:47:22.773112+0000 mgr.y (mgr.24661) 259 : cluster [DBG] pgmap v455: 121 pgs: 121 active+clean; 346 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.0 MiB/s rd, 1.4 MiB/s wr, 7 op/s 2024-09-12T23:47:23.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:23 smithi136 bash[76371]: audit 2024-09-12T23:47:22.926991+0000 mon.a (mon.0) 829 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:23.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:23 smithi136 bash[76371]: audit 2024-09-12T23:47:22.929088+0000 mon.a (mon.0) 830 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-12T23:47:23.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:23 smithi136 bash[76371]: audit 2024-09-12T23:47:22.930790+0000 mon.a (mon.0) 831 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:47:23.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:23 smithi136 bash[75164]: cluster 2024-09-12T23:47:22.404761+0000 mon.a (mon.0) 828 : cluster [DBG] osdmap e586: 8 total, 8 up, 8 in 2024-09-12T23:47:23.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:23 smithi136 bash[75164]: cluster 2024-09-12T23:47:22.773112+0000 mgr.y (mgr.24661) 259 : cluster [DBG] pgmap v455: 121 pgs: 121 active+clean; 346 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.0 MiB/s rd, 1.4 MiB/s wr, 7 op/s 2024-09-12T23:47:23.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:23 smithi136 bash[75164]: audit 2024-09-12T23:47:22.926991+0000 mon.a (mon.0) 829 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:23.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:23 smithi136 bash[75164]: audit 2024-09-12T23:47:22.929088+0000 mon.a (mon.0) 830 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-12T23:47:23.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:23 smithi136 bash[75164]: audit 2024-09-12T23:47:22.930790+0000 mon.a (mon.0) 831 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:47:24.415 INFO:tasks.rados.rados.0.smithi136.stdout:2599: done (2 left) 2024-09-12T23:47:24.415 INFO:tasks.rados.rados.0.smithi136.stdout:2600: done (1 left) 2024-09-12T23:47:24.416 INFO:tasks.rados.rados.0.smithi136.stdout:2601: done (0 left) 2024-09-12T23:47:24.416 INFO:tasks.rados.rados.0.smithi136.stdout:2602: rmattr oid 34 current snap is 265 2024-09-12T23:47:24.418 INFO:tasks.rados.rados.0.smithi136.stdout:2603: setattr oid 27 current snap is 265 2024-09-12T23:47:24.418 INFO:tasks.rados.rados.0.smithi136.stdout:2604: read oid 38 snap -1 2024-09-12T23:47:24.418 INFO:tasks.rados.rados.0.smithi136.stdout:2604: expect (ObjNum 727 snap 224 seq_num 727) 2024-09-12T23:47:24.418 INFO:tasks.rados.rados.0.smithi136.stdout:2605: snap_create 2024-09-12T23:47:24.427 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 34 v 1279 (ObjNum 796 snap 253 seq_num 796) dirty exists 2024-09-12T23:47:24.431 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 27 v 1848 (ObjNum 766 snap 236 seq_num 766) dirty exists 2024-09-12T23:47:24.466 INFO:tasks.rados.rados.0.smithi136.stdout:2604: expect (ObjNum 727 snap 224 seq_num 727) 2024-09-12T23:47:24.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:24 smithi181 bash[67587]: cephadm 2024-09-12T23:47:22.917089+0000 mgr.y (mgr.24661) 260 : cephadm [INF] Upgrade: Updating osd.7 2024-09-12T23:47:24.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:24 smithi181 bash[67587]: cephadm 2024-09-12T23:47:22.935848+0000 mgr.y (mgr.24661) 261 : cephadm [INF] Deploying daemon osd.7 on smithi181 2024-09-12T23:47:24.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:24 smithi181 bash[67587]: cluster 2024-09-12T23:47:23.418157+0000 mon.a (mon.0) 832 : cluster [DBG] osdmap e587: 8 total, 8 up, 8 in 2024-09-12T23:47:24.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:24 smithi181 bash[67587]: audit 2024-09-12T23:47:23.754300+0000 mon.a (mon.0) 833 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:24.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:24 smithi181 bash[67587]: audit 2024-09-12T23:47:23.756043+0000 mon.a (mon.0) 834 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:47:24.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:24 smithi181 bash[67587]: audit 2024-09-12T23:47:23.961622+0000 mon.a (mon.0) 835 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:24.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:24 smithi136 bash[75164]: cephadm 2024-09-12T23:47:22.917089+0000 mgr.y (mgr.24661) 260 : cephadm [INF] Upgrade: Updating osd.7 2024-09-12T23:47:24.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:24 smithi136 bash[75164]: cephadm 2024-09-12T23:47:22.935848+0000 mgr.y (mgr.24661) 261 : cephadm [INF] Deploying daemon osd.7 on smithi181 2024-09-12T23:47:24.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:24 smithi136 bash[75164]: cluster 2024-09-12T23:47:23.418157+0000 mon.a (mon.0) 832 : cluster [DBG] osdmap e587: 8 total, 8 up, 8 in 2024-09-12T23:47:24.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:24 smithi136 bash[75164]: audit 2024-09-12T23:47:23.754300+0000 mon.a (mon.0) 833 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:24.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:24 smithi136 bash[75164]: audit 2024-09-12T23:47:23.756043+0000 mon.a (mon.0) 834 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:47:24.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:24 smithi136 bash[75164]: audit 2024-09-12T23:47:23.961622+0000 mon.a (mon.0) 835 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:24.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:24 smithi136 bash[76371]: cephadm 2024-09-12T23:47:22.917089+0000 mgr.y (mgr.24661) 260 : cephadm [INF] Upgrade: Updating osd.7 2024-09-12T23:47:24.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:24 smithi136 bash[76371]: cephadm 2024-09-12T23:47:22.935848+0000 mgr.y (mgr.24661) 261 : cephadm [INF] Deploying daemon osd.7 on smithi181 2024-09-12T23:47:24.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:24 smithi136 bash[76371]: cluster 2024-09-12T23:47:23.418157+0000 mon.a (mon.0) 832 : cluster [DBG] osdmap e587: 8 total, 8 up, 8 in 2024-09-12T23:47:24.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:24 smithi136 bash[76371]: audit 2024-09-12T23:47:23.754300+0000 mon.a (mon.0) 833 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:24.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:24 smithi136 bash[76371]: audit 2024-09-12T23:47:23.756043+0000 mon.a (mon.0) 834 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:47:24.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:24 smithi136 bash[76371]: audit 2024-09-12T23:47:23.961622+0000 mon.a (mon.0) 835 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:25.439 INFO:tasks.rados.rados.0.smithi136.stdout:2602: done (3 left) 2024-09-12T23:47:25.439 INFO:tasks.rados.rados.0.smithi136.stdout:2603: done (2 left) 2024-09-12T23:47:25.439 INFO:tasks.rados.rados.0.smithi136.stdout:2604: done (1 left) 2024-09-12T23:47:25.439 INFO:tasks.rados.rados.0.smithi136.stdout:2605: done (0 left) 2024-09-12T23:47:25.440 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 24 current snap is 266 2024-09-12T23:47:25.440 INFO:tasks.rados.rados.0.smithi136.stdout:2606: seq_num 829 ranges {8282112=688128,8970240=729088,9699328=475136} 2024-09-12T23:47:25.536 INFO:tasks.rados.rados.0.smithi136.stdout:2606: writing smithi13667081-24 from 8282112 to 8970240 tid 1 2024-09-12T23:47:25.541 INFO:tasks.rados.rados.0.smithi136.stdout:2606: writing smithi13667081-24 from 8970240 to 9699328 tid 2 2024-09-12T23:47:25.544 INFO:tasks.rados.rados.0.smithi136.stdout:2606: writing smithi13667081-24 from 9699328 to 10174464 tid 3 2024-09-12T23:47:25.544 INFO:tasks.rados.rados.0.smithi136.stdout:2607: copy_from oid 30 from oid 12 current snap is 266 2024-09-12T23:47:25.544 INFO:tasks.rados.rados.0.smithi136.stdout:2608: copy_from oid 43 from oid 16 current snap is 266 2024-09-12T23:47:25.545 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 39 current snap is 266 2024-09-12T23:47:25.545 INFO:tasks.rados.rados.0.smithi136.stdout:2609: seq_num 830 ranges {0=704512,704512=614400,1318912=573440,1892352=16384} 2024-09-12T23:47:25.549 INFO:tasks.rados.rados.0.smithi136.stdout:2609: writing smithi13667081-39 from 0 to 704512 tid 1 2024-09-12T23:47:25.553 INFO:tasks.rados.rados.0.smithi136.stdout:2609: writing smithi13667081-39 from 704512 to 1318912 tid 2 2024-09-12T23:47:25.557 INFO:tasks.rados.rados.0.smithi136.stdout:2609: writing smithi13667081-39 from 1318912 to 1892352 tid 3 2024-09-12T23:47:25.557 INFO:tasks.rados.rados.0.smithi136.stdout:2609: writing smithi13667081-39 from 1892352 to 1908736 tid 4 2024-09-12T23:47:25.557 INFO:tasks.rados.rados.0.smithi136.stdout:2610: snap_create 2024-09-12T23:47:25.557 INFO:tasks.rados.rados.0.smithi136.stdout:2608: finishing copy_from to smithi13667081-43 2024-09-12T23:47:25.557 INFO:tasks.rados.rados.0.smithi136.stdout:2608: got expected ENOENT (src dne) 2024-09-12T23:47:25.557 INFO:tasks.rados.rados.0.smithi136.stdout:2608: finishing copy_from racing read to smithi13667081-43 2024-09-12T23:47:25.557 INFO:tasks.rados.rados.0.smithi136.stdout:2606: finishing write tid 1 to smithi13667081-24 2024-09-12T23:47:25.557 INFO:tasks.rados.rados.0.smithi136.stdout:2606: finishing write tid 2 to smithi13667081-24 2024-09-12T23:47:25.557 INFO:tasks.rados.rados.0.smithi136.stdout:2606: finishing write tid 3 to smithi13667081-24 2024-09-12T23:47:25.557 INFO:tasks.rados.rados.0.smithi136.stdout:2606: finishing write tid 4 to smithi13667081-24 2024-09-12T23:47:25.558 INFO:tasks.rados.rados.0.smithi136.stdout:2606: finishing write tid 5 to smithi13667081-24 2024-09-12T23:47:25.558 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 24 v 1279 (ObjNum 829 snap 266 seq_num 829) dirty exists 2024-09-12T23:47:25.558 INFO:tasks.rados.rados.0.smithi136.stdout:2606: left oid 24 (ObjNum 829 snap 266 seq_num 829) 2024-09-12T23:47:25.559 INFO:tasks.rados.rados.0.smithi136.stdout:2609: finishing write tid 1 to smithi13667081-39 2024-09-12T23:47:25.564 INFO:tasks.rados.rados.0.smithi136.stdout:2607: finishing copy_from racing read to smithi13667081-30 2024-09-12T23:47:25.566 INFO:tasks.rados.rados.0.smithi136.stdout:2609: finishing write tid 2 to smithi13667081-39 2024-09-12T23:47:25.566 INFO:tasks.rados.rados.0.smithi136.stdout:2609: finishing write tid 3 to smithi13667081-39 2024-09-12T23:47:25.567 INFO:tasks.rados.rados.0.smithi136.stdout:2609: finishing write tid 4 to smithi13667081-39 2024-09-12T23:47:25.567 INFO:tasks.rados.rados.0.smithi136.stdout:2607: finishing copy_from to smithi13667081-30 2024-09-12T23:47:25.567 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 30 v 1482 (ObjNum 824 snap 260 seq_num 824) dirty exists 2024-09-12T23:47:25.571 INFO:tasks.rados.rados.0.smithi136.stdout:2609: finishing write tid 5 to smithi13667081-39 2024-09-12T23:47:25.573 INFO:tasks.rados.rados.0.smithi136.stdout:2609: finishing write tid 6 to smithi13667081-39 2024-09-12T23:47:25.573 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 39 v 863 (ObjNum 830 snap 266 seq_num 830) dirty exists 2024-09-12T23:47:25.573 INFO:tasks.rados.rados.0.smithi136.stdout:2609: left oid 39 (ObjNum 830 snap 266 seq_num 830) 2024-09-12T23:47:25.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:25 smithi136 bash[75164]: cluster 2024-09-12T23:47:24.417978+0000 mon.a (mon.0) 836 : cluster [DBG] osdmap e588: 8 total, 8 up, 8 in 2024-09-12T23:47:25.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:25 smithi136 bash[75164]: cluster 2024-09-12T23:47:24.773924+0000 mgr.y (mgr.24661) 262 : cluster [DBG] pgmap v458: 121 pgs: 121 active+clean; 346 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 6.8 MiB/s rd, 1.4 MiB/s wr, 14 op/s 2024-09-12T23:47:25.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:25 smithi136 bash[76371]: cluster 2024-09-12T23:47:24.417978+0000 mon.a (mon.0) 836 : cluster [DBG] osdmap e588: 8 total, 8 up, 8 in 2024-09-12T23:47:25.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:25 smithi136 bash[76371]: cluster 2024-09-12T23:47:24.773924+0000 mgr.y (mgr.24661) 262 : cluster [DBG] pgmap v458: 121 pgs: 121 active+clean; 346 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 6.8 MiB/s rd, 1.4 MiB/s wr, 14 op/s 2024-09-12T23:47:25.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:25 smithi181 bash[67587]: cluster 2024-09-12T23:47:24.417978+0000 mon.a (mon.0) 836 : cluster [DBG] osdmap e588: 8 total, 8 up, 8 in 2024-09-12T23:47:25.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:25 smithi181 bash[67587]: cluster 2024-09-12T23:47:24.773924+0000 mgr.y (mgr.24661) 262 : cluster [DBG] pgmap v458: 121 pgs: 121 active+clean; 346 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 6.8 MiB/s rd, 1.4 MiB/s wr, 14 op/s 2024-09-12T23:47:26.403 INFO:tasks.rados.rados.0.smithi136.stdout:2606: done (4 left) 2024-09-12T23:47:26.403 INFO:tasks.rados.rados.0.smithi136.stdout:2607: done (3 left) 2024-09-12T23:47:26.404 INFO:tasks.rados.rados.0.smithi136.stdout:2608: done (2 left) 2024-09-12T23:47:26.404 INFO:tasks.rados.rados.0.smithi136.stdout:2609: done (1 left) 2024-09-12T23:47:26.404 INFO:tasks.rados.rados.0.smithi136.stdout:2610: done (0 left) 2024-09-12T23:47:26.404 INFO:tasks.rados.rados.0.smithi136.stdout:2611: read oid 8 snap -1 2024-09-12T23:47:26.404 INFO:tasks.rados.rados.0.smithi136.stdout:2611: expect (ObjNum 825 snap 260 seq_num 825) 2024-09-12T23:47:26.404 INFO:tasks.rados.rados.0.smithi136.stdout:2612: read oid 28 snap -1 2024-09-12T23:47:26.404 INFO:tasks.rados.rados.0.smithi136.stdout:2612: expect (ObjNum 739 snap 227 seq_num 739) 2024-09-12T23:47:26.404 INFO:tasks.rados.rados.0.smithi136.stdout:2613: read oid 14 snap -1 2024-09-12T23:47:26.404 INFO:tasks.rados.rados.0.smithi136.stdout:2613: expect deleted 2024-09-12T23:47:26.404 INFO:tasks.rados.rados.0.smithi136.stdout:2614: snap_remove snap 260 2024-09-12T23:47:26.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:26 smithi136 bash[75164]: cluster 2024-09-12T23:47:25.442337+0000 mon.a (mon.0) 837 : cluster [DBG] osdmap e589: 8 total, 8 up, 8 in 2024-09-12T23:47:26.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:26 smithi136 bash[75164]: cluster 2024-09-12T23:47:26.404697+0000 mon.a (mon.0) 838 : cluster [DBG] osdmap e590: 8 total, 8 up, 8 in 2024-09-12T23:47:26.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:26 smithi136 bash[76371]: cluster 2024-09-12T23:47:25.442337+0000 mon.a (mon.0) 837 : cluster [DBG] osdmap e589: 8 total, 8 up, 8 in 2024-09-12T23:47:26.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:26 smithi136 bash[76371]: cluster 2024-09-12T23:47:26.404697+0000 mon.a (mon.0) 838 : cluster [DBG] osdmap e590: 8 total, 8 up, 8 in 2024-09-12T23:47:26.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:26 smithi181 bash[67587]: cluster 2024-09-12T23:47:25.442337+0000 mon.a (mon.0) 837 : cluster [DBG] osdmap e589: 8 total, 8 up, 8 in 2024-09-12T23:47:26.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:26 smithi181 bash[67587]: cluster 2024-09-12T23:47:26.404697+0000 mon.a (mon.0) 838 : cluster [DBG] osdmap e590: 8 total, 8 up, 8 in 2024-09-12T23:47:27.308 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:47:27 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:47:27.309 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:47:27 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:47:27.309 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:47:27 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:47:27.309 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 12 23:47:27 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:47:27.309 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:27 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:47:27.310 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:47:27 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:47:27.310 INFO:journalctl@ceph.mds.b.smithi181.stdout:Sep 12 23:47:27 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:47:27.311 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 12 23:47:27 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:47:27.311 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 12 23:47:27 smithi181 systemd[1]: Stopping Ceph osd.7 for da58a47c-715d-11ef-bceb-c7b262605968... 2024-09-12T23:47:27.311 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:47:27 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:47:27.311 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:47:27 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:47:27.405 INFO:tasks.rados.rados.0.smithi136.stdout:2614: done (3 left) 2024-09-12T23:47:27.405 INFO:tasks.rados.rados.0.smithi136.stdout:2615: rmattr oid 39 current snap is 267 2024-09-12T23:47:27.405 INFO:tasks.rados.rados.0.smithi136.stdout:2615: done (3 left) 2024-09-12T23:47:27.405 INFO:tasks.rados.rados.0.smithi136.stdout:2616: delete oid 22 current snap is 267 2024-09-12T23:47:27.405 INFO:tasks.rados.rados.0.smithi136.stdout:2612: expect (ObjNum 739 snap 227 seq_num 739) 2024-09-12T23:47:27.438 INFO:tasks.rados.rados.0.smithi136.stdout:2611: expect (ObjNum 825 snap 260 seq_num 825) 2024-09-12T23:47:27.576 INFO:tasks.rados.rados.0.smithi136.stdout:2611: done (3 left) 2024-09-12T23:47:27.576 INFO:tasks.rados.rados.0.smithi136.stdout:2612: done (2 left) 2024-09-12T23:47:27.576 INFO:tasks.rados.rados.0.smithi136.stdout:2613: done (1 left) 2024-09-12T23:47:27.576 INFO:tasks.rados.rados.0.smithi136.stdout:2616: done (0 left) 2024-09-12T23:47:27.576 INFO:tasks.rados.rados.0.smithi136.stdout:2617: rollback oid 38 current snap is 267 2024-09-12T23:47:27.576 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 38 to 233 2024-09-12T23:47:27.576 INFO:tasks.rados.rados.0.smithi136.stdout:2618: rollback oid 39 current snap is 267 2024-09-12T23:47:27.576 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 39 to 261 2024-09-12T23:47:27.577 INFO:tasks.rados.rados.0.smithi136.stdout:2619: read oid 2 snap 258 2024-09-12T23:47:27.577 INFO:tasks.rados.rados.0.smithi136.stdout:2619: expect deleted 2024-09-12T23:47:27.577 INFO:tasks.rados.rados.0.smithi136.stdout:2620: read oid 37 snap 251 2024-09-12T23:47:27.577 INFO:tasks.rados.rados.0.smithi136.stdout:2620: expect (ObjNum 779 snap 242 seq_num 779) 2024-09-12T23:47:27.577 INFO:tasks.rados.rados.0.smithi136.stdout:2621: copy_from oid 34 from oid 14 current snap is 267 2024-09-12T23:47:27.577 INFO:tasks.rados.rados.0.smithi136.stdout:2622: delete oid 3 current snap is 267 2024-09-12T23:47:27.585 INFO:tasks.rados.rados.0.smithi136.stdout:2617: finishing rollback tid 0 to smithi13667081-38 2024-09-12T23:47:27.586 INFO:tasks.rados.rados.0.smithi136.stdout:2617: finishing rollback tid 1 to smithi13667081-38 2024-09-12T23:47:27.589 INFO:tasks.rados.rados.0.smithi136.stdout:2617: finishing rollback tid 2 to smithi13667081-38 2024-09-12T23:47:27.589 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 38 v 1624 (ObjNum 727 snap 224 seq_num 727) dirty exists 2024-09-12T23:47:27.590 INFO:tasks.rados.rados.0.smithi136.stdout:2618: finishing rollback tid 0 to smithi13667081-39 2024-09-12T23:47:27.590 INFO:tasks.rados.rados.0.smithi136.stdout:2621: finishing copy_from to smithi13667081-34 2024-09-12T23:47:27.590 INFO:tasks.rados.rados.0.smithi136.stdout:2621: got expected ENOENT (src dne) 2024-09-12T23:47:27.591 INFO:tasks.rados.rados.0.smithi136.stdout:2621: finishing copy_from racing read to smithi13667081-34 2024-09-12T23:47:27.591 INFO:tasks.rados.rados.0.smithi136.stdout:2617: done (5 left) 2024-09-12T23:47:27.591 INFO:tasks.rados.rados.0.smithi136.stdout:2619: done (4 left) 2024-09-12T23:47:27.591 INFO:tasks.rados.rados.0.smithi136.stdout:2621: done (3 left) 2024-09-12T23:47:27.591 INFO:tasks.rados.rados.0.smithi136.stdout:2622: done (2 left) 2024-09-12T23:47:27.591 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 42 current snap is 267 2024-09-12T23:47:27.591 INFO:tasks.rados.rados.0.smithi136.stdout:2623: seq_num 833 ranges {0=761856,761856=671744,1433600=532480} 2024-09-12T23:47:27.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:27 smithi181 bash[67587]: cluster 2024-09-12T23:47:26.774822+0000 mgr.y (mgr.24661) 263 : cluster [DBG] pgmap v461: 121 pgs: 121 active+clean; 347 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 5.9 MiB/s rd, 462 KiB/s wr, 17 op/s 2024-09-12T23:47:27.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:27 smithi181 bash[67587]: cluster 2024-09-12T23:47:27.406885+0000 mon.a (mon.0) 839 : cluster [DBG] osdmap e591: 8 total, 8 up, 8 in 2024-09-12T23:47:27.594 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 12 23:47:27 smithi181 bash[44399]: debug 2024-09-12T23:47:27.430+0000 7fa232818640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-12T23:47:27.594 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 12 23:47:27 smithi181 bash[44399]: debug 2024-09-12T23:47:27.430+0000 7fa232818640 -1 osd.7 590 *** Got signal Terminated *** 2024-09-12T23:47:27.594 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 12 23:47:27 smithi181 bash[44399]: debug 2024-09-12T23:47:27.430+0000 7fa232818640 -1 osd.7 590 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-12T23:47:27.599 INFO:tasks.rados.rados.0.smithi136.stdout:2623: writing smithi13667081-42 from 0 to 761856 tid 1 2024-09-12T23:47:27.607 INFO:tasks.rados.rados.0.smithi136.stdout:2623: writing smithi13667081-42 from 761856 to 1433600 tid 2 2024-09-12T23:47:27.613 INFO:tasks.rados.rados.0.smithi136.stdout:2623: writing smithi13667081-42 from 1433600 to 1966080 tid 3 2024-09-12T23:47:27.613 INFO:tasks.rados.rados.0.smithi136.stdout:2624: rmattr oid 28 current snap is 267 2024-09-12T23:47:27.613 INFO:tasks.rados.rados.0.smithi136.stdout:2624: done (3 left) 2024-09-12T23:47:27.613 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 20 current snap is 267 2024-09-12T23:47:27.614 INFO:tasks.rados.rados.0.smithi136.stdout:2620: expect (ObjNum 779 snap 242 seq_num 779) 2024-09-12T23:47:27.666 INFO:tasks.rados.rados.0.smithi136.stdout:2618: finishing rollback tid 1 to smithi13667081-39 2024-09-12T23:47:27.666 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 39 v 865 (ObjNum 161 snap 0 seq_num 0) dirty dne 2024-09-12T23:47:27.666 INFO:tasks.rados.rados.0.smithi136.stdout:2623: finishing write tid 1 to smithi13667081-42 2024-09-12T23:47:27.667 INFO:tasks.rados.rados.0.smithi136.stdout:2623: finishing write tid 2 to smithi13667081-42 2024-09-12T23:47:27.667 INFO:tasks.rados.rados.0.smithi136.stdout:2623: finishing write tid 3 to smithi13667081-42 2024-09-12T23:47:27.667 INFO:tasks.rados.rados.0.smithi136.stdout:2623: finishing write tid 4 to smithi13667081-42 2024-09-12T23:47:27.667 INFO:tasks.rados.rados.0.smithi136.stdout:2623: finishing write tid 5 to smithi13667081-42 2024-09-12T23:47:27.667 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 42 v 887 (ObjNum 833 snap 267 seq_num 833) dirty exists 2024-09-12T23:47:27.667 INFO:tasks.rados.rados.0.smithi136.stdout:2623: left oid 42 (ObjNum 833 snap 267 seq_num 833) 2024-09-12T23:47:27.667 INFO:tasks.rados.rados.0.smithi136.stdout:2625: seq_num 834 ranges {385024=778240,1163264=557056,1720320=548864,2269184=98304} 2024-09-12T23:47:27.682 INFO:tasks.rados.rados.0.smithi136.stdout:2625: writing smithi13667081-20 from 385024 to 1163264 tid 1 2024-09-12T23:47:27.691 INFO:tasks.rados.rados.0.smithi136.stdout:2625: writing smithi13667081-20 from 1163264 to 1720320 tid 2 2024-09-12T23:47:27.699 INFO:tasks.rados.rados.0.smithi136.stdout:2625: writing smithi13667081-20 from 1720320 to 2269184 tid 3 2024-09-12T23:47:27.701 INFO:tasks.rados.rados.0.smithi136.stdout:2625: writing smithi13667081-20 from 2269184 to 2367488 tid 4 2024-09-12T23:47:27.701 INFO:tasks.rados.rados.0.smithi136.stdout:2618: done (3 left) 2024-09-12T23:47:27.701 INFO:tasks.rados.rados.0.smithi136.stdout:2620: done (2 left) 2024-09-12T23:47:27.701 INFO:tasks.rados.rados.0.smithi136.stdout:2623: done (1 left) 2024-09-12T23:47:27.701 INFO:tasks.rados.rados.0.smithi136.stdout:2626: delete oid 50 current snap is 267 2024-09-12T23:47:27.701 INFO:tasks.rados.rados.0.smithi136.stdout:2625: finishing write tid 1 to smithi13667081-20 2024-09-12T23:47:27.701 INFO:tasks.rados.rados.0.smithi136.stdout:2625: finishing write tid 2 to smithi13667081-20 2024-09-12T23:47:27.705 INFO:tasks.rados.rados.0.smithi136.stdout:2626: done (1 left) 2024-09-12T23:47:27.705 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 6 current snap is 267 2024-09-12T23:47:27.705 INFO:tasks.rados.rados.0.smithi136.stdout:2627: seq_num 836 ranges {4112384=409600,4521984=720896,5242880=516096,5758976=368640} 2024-09-12T23:47:27.745 INFO:tasks.rados.rados.0.smithi136.stdout:2627: writing smithi13667081-6 from 4112384 to 4521984 tid 1 2024-09-12T23:47:27.751 INFO:tasks.rados.rados.0.smithi136.stdout:2627: writing smithi13667081-6 from 4521984 to 5242880 tid 2 2024-09-12T23:47:27.755 INFO:tasks.rados.rados.0.smithi136.stdout:2627: writing smithi13667081-6 from 5242880 to 5758976 tid 3 2024-09-12T23:47:27.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:27 smithi136 bash[75164]: cluster 2024-09-12T23:47:26.774822+0000 mgr.y (mgr.24661) 263 : cluster [DBG] pgmap v461: 121 pgs: 121 active+clean; 347 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 5.9 MiB/s rd, 462 KiB/s wr, 17 op/s 2024-09-12T23:47:27.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:27 smithi136 bash[75164]: cluster 2024-09-12T23:47:27.406885+0000 mon.a (mon.0) 839 : cluster [DBG] osdmap e591: 8 total, 8 up, 8 in 2024-09-12T23:47:27.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:27 smithi136 bash[76371]: cluster 2024-09-12T23:47:26.774822+0000 mgr.y (mgr.24661) 263 : cluster [DBG] pgmap v461: 121 pgs: 121 active+clean; 347 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 5.9 MiB/s rd, 462 KiB/s wr, 17 op/s 2024-09-12T23:47:27.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:27 smithi136 bash[76371]: cluster 2024-09-12T23:47:27.406885+0000 mon.a (mon.0) 839 : cluster [DBG] osdmap e591: 8 total, 8 up, 8 in 2024-09-12T23:47:27.758 INFO:tasks.rados.rados.0.smithi136.stdout:2627: writing smithi13667081-6 from 5758976 to 6127616 tid 4 2024-09-12T23:47:27.758 INFO:tasks.rados.rados.0.smithi136.stdout:2628: delete oid 21 current snap is 267 2024-09-12T23:47:27.758 INFO:tasks.rados.rados.0.smithi136.stdout:2625: finishing write tid 3 to smithi13667081-20 2024-09-12T23:47:27.758 INFO:tasks.rados.rados.0.smithi136.stdout:2625: finishing write tid 4 to smithi13667081-20 2024-09-12T23:47:27.758 INFO:tasks.rados.rados.0.smithi136.stdout:2625: finishing write tid 5 to smithi13667081-20 2024-09-12T23:47:27.758 INFO:tasks.rados.rados.0.smithi136.stdout:2625: finishing write tid 6 to smithi13667081-20 2024-09-12T23:47:27.759 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 20 v 1597 (ObjNum 834 snap 267 seq_num 834) dirty exists 2024-09-12T23:47:27.759 INFO:tasks.rados.rados.0.smithi136.stdout:2625: left oid 20 (ObjNum 834 snap 267 seq_num 834) 2024-09-12T23:47:27.759 INFO:tasks.rados.rados.0.smithi136.stdout:2627: finishing write tid 1 to smithi13667081-6 2024-09-12T23:47:27.759 INFO:tasks.rados.rados.0.smithi136.stdout:2627: finishing write tid 2 to smithi13667081-6 2024-09-12T23:47:27.761 INFO:tasks.rados.rados.0.smithi136.stdout:2627: finishing write tid 3 to smithi13667081-6 2024-09-12T23:47:27.762 INFO:tasks.rados.rados.0.smithi136.stdout:2625: done (2 left) 2024-09-12T23:47:27.762 INFO:tasks.rados.rados.0.smithi136.stdout:2628: done (1 left) 2024-09-12T23:47:27.762 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 3 current snap is 267 2024-09-12T23:47:27.762 INFO:tasks.rados.rados.0.smithi136.stdout:2629: seq_num 838 ranges {0=442368,442368=540672,983040=532480,1515520=442368,1957888=90112} 2024-09-12T23:47:27.765 INFO:tasks.rados.rados.0.smithi136.stdout:2629: writing smithi13667081-3 from 0 to 442368 tid 1 2024-09-12T23:47:27.768 INFO:tasks.rados.rados.0.smithi136.stdout:2629: writing smithi13667081-3 from 442368 to 983040 tid 2 2024-09-12T23:47:27.773 INFO:tasks.rados.rados.0.smithi136.stdout:2629: writing smithi13667081-3 from 983040 to 1515520 tid 3 2024-09-12T23:47:27.776 INFO:tasks.rados.rados.0.smithi136.stdout:2629: writing smithi13667081-3 from 1515520 to 1957888 tid 4 2024-09-12T23:47:27.777 INFO:tasks.rados.rados.0.smithi136.stdout:2629: writing smithi13667081-3 from 1957888 to 2048000 tid 5 2024-09-12T23:47:27.777 INFO:tasks.rados.rados.0.smithi136.stdout:2630: snap_create 2024-09-12T23:47:27.777 INFO:tasks.rados.rados.0.smithi136.stdout:2627: finishing write tid 4 to smithi13667081-6 2024-09-12T23:47:27.777 INFO:tasks.rados.rados.0.smithi136.stdout:2627: finishing write tid 5 to smithi13667081-6 2024-09-12T23:47:27.777 INFO:tasks.rados.rados.0.smithi136.stdout:2627: finishing write tid 6 to smithi13667081-6 2024-09-12T23:47:27.777 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 6 v 1602 (ObjNum 836 snap 267 seq_num 836) dirty exists 2024-09-12T23:47:27.777 INFO:tasks.rados.rados.0.smithi136.stdout:2627: left oid 6 (ObjNum 836 snap 267 seq_num 836) 2024-09-12T23:47:27.777 INFO:tasks.rados.rados.0.smithi136.stdout:2629: finishing write tid 1 to smithi13667081-3 2024-09-12T23:47:27.777 INFO:tasks.rados.rados.0.smithi136.stdout:2629: finishing write tid 2 to smithi13667081-3 2024-09-12T23:47:27.779 INFO:tasks.rados.rados.0.smithi136.stdout:2629: finishing write tid 3 to smithi13667081-3 2024-09-12T23:47:27.782 INFO:tasks.rados.rados.0.smithi136.stdout:2629: finishing write tid 4 to smithi13667081-3 2024-09-12T23:47:27.782 INFO:tasks.rados.rados.0.smithi136.stdout:2629: finishing write tid 5 to smithi13667081-3 2024-09-12T23:47:27.782 INFO:tasks.rados.rados.0.smithi136.stdout:2629: finishing write tid 6 to smithi13667081-3 2024-09-12T23:47:27.784 INFO:tasks.rados.rados.0.smithi136.stdout:2629: finishing write tid 7 to smithi13667081-3 2024-09-12T23:47:27.784 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 3 v 1489 (ObjNum 838 snap 267 seq_num 838) dirty exists 2024-09-12T23:47:27.784 INFO:tasks.rados.rados.0.smithi136.stdout:2629: left oid 3 (ObjNum 838 snap 267 seq_num 838) 2024-09-12T23:47:28.411 INFO:tasks.rados.rados.0.smithi136.stdout:2627: done (2 left) 2024-09-12T23:47:28.411 INFO:tasks.rados.rados.0.smithi136.stdout:2629: done (1 left) 2024-09-12T23:47:28.411 INFO:tasks.rados.rados.0.smithi136.stdout:2630: done (0 left) 2024-09-12T23:47:28.411 INFO:tasks.rados.rados.0.smithi136.stdout:2631: delete oid 47 current snap is 268 2024-09-12T23:47:28.428 INFO:tasks.rados.rados.0.smithi136.stdout:2631: done (0 left) 2024-09-12T23:47:28.428 INFO:tasks.rados.rados.0.smithi136.stdout:2632: copy_from oid 37 from oid 34 current snap is 268 2024-09-12T23:47:28.428 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 9 current snap is 268 2024-09-12T23:47:28.428 INFO:tasks.rados.rados.0.smithi136.stdout:2633: seq_num 840 ranges {12640256=475136,13115392=712704,13828096=499712,14327808=393216} 2024-09-12T23:47:28.521 INFO:tasks.rados.rados.0.smithi136.stdout:2633: writing smithi13667081-9 from 12640256 to 13115392 tid 1 2024-09-12T23:47:28.526 INFO:tasks.rados.rados.0.smithi136.stdout:2633: writing smithi13667081-9 from 13115392 to 13828096 tid 2 2024-09-12T23:47:28.529 INFO:tasks.rados.rados.0.smithi136.stdout:2633: writing smithi13667081-9 from 13828096 to 14327808 tid 3 2024-09-12T23:47:28.532 INFO:tasks.rados.rados.0.smithi136.stdout:2633: writing smithi13667081-9 from 14327808 to 14721024 tid 4 2024-09-12T23:47:28.532 INFO:tasks.rados.rados.0.smithi136.stdout:2634: rollback oid 27 current snap is 268 2024-09-12T23:47:28.532 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 27 to 259 2024-09-12T23:47:28.532 INFO:tasks.rados.rados.0.smithi136.stdout:2632: finishing copy_from racing read to smithi13667081-37 2024-09-12T23:47:28.532 INFO:tasks.rados.rados.0.smithi136.stdout:2635: delete oid 28 current snap is 268 2024-09-12T23:47:28.532 INFO:tasks.rados.rados.0.smithi136.stdout:2632: finishing copy_from to smithi13667081-37 2024-09-12T23:47:28.533 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 37 v 1279 (ObjNum 796 snap 253 seq_num 796) dirty exists 2024-09-12T23:47:28.538 INFO:tasks.rados.rados.0.smithi136.stdout:2633: finishing write tid 1 to smithi13667081-9 2024-09-12T23:47:28.540 INFO:tasks.rados.rados.0.smithi136.stdout:2633: finishing write tid 2 to smithi13667081-9 2024-09-12T23:47:28.540 INFO:tasks.rados.rados.0.smithi136.stdout:2632: done (3 left) 2024-09-12T23:47:28.540 INFO:tasks.rados.rados.0.smithi136.stdout:2635: done (2 left) 2024-09-12T23:47:28.540 INFO:tasks.rados.rados.0.smithi136.stdout:2636: delete oid 20 current snap is 268 2024-09-12T23:47:28.540 INFO:tasks.rados.rados.0.smithi136.stdout:2633: finishing write tid 3 to smithi13667081-9 2024-09-12T23:47:28.543 INFO:tasks.rados.rados.0.smithi136.stdout:2634: finishing rollback tid 0 to smithi13667081-27 2024-09-12T23:47:28.543 INFO:tasks.rados.rados.0.smithi136.stdout:2634: finishing rollback tid 1 to smithi13667081-27 2024-09-12T23:47:28.545 INFO:tasks.rados.rados.0.smithi136.stdout:2634: finishing rollback tid 2 to smithi13667081-27 2024-09-12T23:47:28.545 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 27 v 1856 (ObjNum 766 snap 236 seq_num 766) dirty exists 2024-09-12T23:47:28.545 INFO:tasks.rados.rados.0.smithi136.stdout:2634: done (2 left) 2024-09-12T23:47:28.545 INFO:tasks.rados.rados.0.smithi136.stdout:2636: done (1 left) 2024-09-12T23:47:28.545 INFO:tasks.rados.rados.0.smithi136.stdout:2637: snap_create 2024-09-12T23:47:28.548 INFO:tasks.rados.rados.0.smithi136.stdout:2633: finishing write tid 4 to smithi13667081-9 2024-09-12T23:47:28.548 INFO:tasks.rados.rados.0.smithi136.stdout:2633: finishing write tid 5 to smithi13667081-9 2024-09-12T23:47:28.551 INFO:tasks.rados.rados.0.smithi136.stdout:2633: finishing write tid 6 to smithi13667081-9 2024-09-12T23:47:28.551 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 9 v 1610 (ObjNum 840 snap 268 seq_num 840) dirty exists 2024-09-12T23:47:28.551 INFO:tasks.rados.rados.0.smithi136.stdout:2633: left oid 9 (ObjNum 840 snap 268 seq_num 840) 2024-09-12T23:47:28.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:28 smithi136 bash[75164]: cluster 2024-09-12T23:47:27.433097+0000 mon.a (mon.0) 840 : cluster [INF] osd.7 marked itself down and dead 2024-09-12T23:47:28.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:28 smithi136 bash[75164]: cluster 2024-09-12T23:47:28.400726+0000 mon.a (mon.0) 841 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-12T23:47:28.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:28 smithi136 bash[75164]: cluster 2024-09-12T23:47:28.410526+0000 mon.a (mon.0) 842 : cluster [DBG] osdmap e592: 8 total, 7 up, 8 in 2024-09-12T23:47:28.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:28 smithi136 bash[76371]: cluster 2024-09-12T23:47:27.433097+0000 mon.a (mon.0) 840 : cluster [INF] osd.7 marked itself down and dead 2024-09-12T23:47:28.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:28 smithi136 bash[76371]: cluster 2024-09-12T23:47:28.400726+0000 mon.a (mon.0) 841 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-12T23:47:28.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:28 smithi136 bash[76371]: cluster 2024-09-12T23:47:28.410526+0000 mon.a (mon.0) 842 : cluster [DBG] osdmap e592: 8 total, 7 up, 8 in 2024-09-12T23:47:28.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:28 smithi181 bash[67587]: cluster 2024-09-12T23:47:27.433097+0000 mon.a (mon.0) 840 : cluster [INF] osd.7 marked itself down and dead 2024-09-12T23:47:28.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:28 smithi181 bash[67587]: cluster 2024-09-12T23:47:28.400726+0000 mon.a (mon.0) 841 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-12T23:47:28.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:28 smithi181 bash[67587]: cluster 2024-09-12T23:47:28.410526+0000 mon.a (mon.0) 842 : cluster [DBG] osdmap e592: 8 total, 7 up, 8 in 2024-09-12T23:47:29.411 INFO:tasks.rados.rados.0.smithi136.stdout:2633: done (1 left) 2024-09-12T23:47:29.412 INFO:tasks.rados.rados.0.smithi136.stdout:2637: done (0 left) 2024-09-12T23:47:29.412 INFO:tasks.rados.rados.0.smithi136.stdout:2638: snap_remove snap 259 2024-09-12T23:47:29.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:47:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:47:29] "GET /metrics HTTP/1.1" 200 36380 "" "Prometheus/2.43.0" 2024-09-12T23:47:29.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:29 smithi136 bash[75164]: cluster 2024-09-12T23:47:28.775933+0000 mgr.y (mgr.24661) 264 : cluster [DBG] pgmap v464: 121 pgs: 8 peering, 13 stale+active+clean, 100 active+clean; 349 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 942 KiB/s wr, 14 op/s 2024-09-12T23:47:29.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:29 smithi136 bash[75164]: cluster 2024-09-12T23:47:29.413736+0000 mon.a (mon.0) 843 : cluster [DBG] osdmap e593: 8 total, 7 up, 8 in 2024-09-12T23:47:29.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:29 smithi136 bash[76371]: cluster 2024-09-12T23:47:28.775933+0000 mgr.y (mgr.24661) 264 : cluster [DBG] pgmap v464: 121 pgs: 8 peering, 13 stale+active+clean, 100 active+clean; 349 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 942 KiB/s wr, 14 op/s 2024-09-12T23:47:29.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:29 smithi136 bash[76371]: cluster 2024-09-12T23:47:29.413736+0000 mon.a (mon.0) 843 : cluster [DBG] osdmap e593: 8 total, 7 up, 8 in 2024-09-12T23:47:29.795 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 12 23:47:29 smithi181 bash[84481]: ceph-da58a47c-715d-11ef-bceb-c7b262605968-osd-7 2024-09-12T23:47:29.795 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:29 smithi181 bash[67587]: cluster 2024-09-12T23:47:28.775933+0000 mgr.y (mgr.24661) 264 : cluster [DBG] pgmap v464: 121 pgs: 8 peering, 13 stale+active+clean, 100 active+clean; 349 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 942 KiB/s wr, 14 op/s 2024-09-12T23:47:29.796 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:29 smithi181 bash[67587]: cluster 2024-09-12T23:47:29.413736+0000 mon.a (mon.0) 843 : cluster [DBG] osdmap e593: 8 total, 7 up, 8 in 2024-09-12T23:47:30.094 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 12 23:47:29 smithi181 bash[84547]: Error response from daemon: No such container: ceph-da58a47c-715d-11ef-bceb-c7b262605968-osd-7 2024-09-12T23:47:30.416 INFO:tasks.rados.rados.0.smithi136.stdout:2638: done (0 left) 2024-09-12T23:47:30.416 INFO:tasks.rados.rados.0.smithi136.stdout:2639: snap_create 2024-09-12T23:47:30.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:30 smithi136 bash[75164]: cluster 2024-09-12T23:47:29.442635+0000 mon.a (mon.0) 844 : cluster [WRN] Health check failed: Reduced data availability: 3 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-09-12T23:47:30.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:30 smithi136 bash[75164]: cluster 2024-09-12T23:47:30.416615+0000 mon.a (mon.0) 845 : cluster [DBG] osdmap e594: 8 total, 7 up, 8 in 2024-09-12T23:47:30.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:30 smithi136 bash[76371]: cluster 2024-09-12T23:47:29.442635+0000 mon.a (mon.0) 844 : cluster [WRN] Health check failed: Reduced data availability: 3 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-09-12T23:47:30.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:30 smithi136 bash[76371]: cluster 2024-09-12T23:47:30.416615+0000 mon.a (mon.0) 845 : cluster [DBG] osdmap e594: 8 total, 7 up, 8 in 2024-09-12T23:47:30.774 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:30 smithi181 bash[67587]: cluster 2024-09-12T23:47:29.442635+0000 mon.a (mon.0) 844 : cluster [WRN] Health check failed: Reduced data availability: 3 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-09-12T23:47:30.774 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:30 smithi181 bash[67587]: cluster 2024-09-12T23:47:30.416615+0000 mon.a (mon.0) 845 : cluster [DBG] osdmap e594: 8 total, 7 up, 8 in 2024-09-12T23:47:31.408 INFO:tasks.rados.rados.0.smithi136.stdout:2639: done (0 left) 2024-09-12T23:47:31.408 INFO:tasks.rados.rados.0.smithi136.stdout:2640: snap_remove snap 265 2024-09-12T23:47:31.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:31 smithi136 bash[75164]: cluster 2024-09-12T23:47:30.776989+0000 mgr.y (mgr.24661) 265 : cluster [DBG] pgmap v467: 121 pgs: 8 active+undersized, 8 peering, 9 stale+active+clean, 2 active+undersized+degraded, 94 active+clean; 355 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.9 MiB/s wr, 9 op/s; 16/603 objects degraded (2.653%) 2024-09-12T23:47:31.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:31 smithi136 bash[75164]: cluster 2024-09-12T23:47:31.409786+0000 mon.a (mon.0) 846 : cluster [DBG] osdmap e595: 8 total, 7 up, 8 in 2024-09-12T23:47:31.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:31 smithi136 bash[76371]: cluster 2024-09-12T23:47:30.776989+0000 mgr.y (mgr.24661) 265 : cluster [DBG] pgmap v467: 121 pgs: 8 active+undersized, 8 peering, 9 stale+active+clean, 2 active+undersized+degraded, 94 active+clean; 355 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.9 MiB/s wr, 9 op/s; 16/603 objects degraded (2.653%) 2024-09-12T23:47:31.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:31 smithi136 bash[76371]: cluster 2024-09-12T23:47:31.409786+0000 mon.a (mon.0) 846 : cluster [DBG] osdmap e595: 8 total, 7 up, 8 in 2024-09-12T23:47:31.809 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:31 smithi181 bash[67587]: cluster 2024-09-12T23:47:30.776989+0000 mgr.y (mgr.24661) 265 : cluster [DBG] pgmap v467: 121 pgs: 8 active+undersized, 8 peering, 9 stale+active+clean, 2 active+undersized+degraded, 94 active+clean; 355 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.9 MiB/s wr, 9 op/s; 16/603 objects degraded (2.653%) 2024-09-12T23:47:31.809 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:31 smithi181 bash[67587]: cluster 2024-09-12T23:47:31.409786+0000 mon.a (mon.0) 846 : cluster [DBG] osdmap e595: 8 total, 7 up, 8 in 2024-09-12T23:47:32.093 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:47:31 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:47:32.094 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:47:31 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:47:32.094 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:47:31 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:47:32.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:31 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:47:32.095 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:47:31 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:47:32.095 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 12 23:47:31 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:47:32.095 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:47:31 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:47:32.096 INFO:journalctl@ceph.mds.b.smithi181.stdout:Sep 12 23:47:31 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:47:32.096 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 12 23:47:31 smithi181 systemd[1]: ceph-da58a47c-715d-11ef-bceb-c7b262605968@osd.7.service: Deactivated successfully. 2024-09-12T23:47:32.096 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 12 23:47:31 smithi181 systemd[1]: Stopped Ceph osd.7 for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:47:32.096 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 12 23:47:31 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:47:32.096 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:47:31 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:47:32.413 INFO:tasks.rados.rados.0.smithi136.stdout:2640: done (0 left) 2024-09-12T23:47:32.413 INFO:tasks.rados.rados.0.smithi136.stdout:2641: setattr oid 43 current snap is 270 2024-09-12T23:47:32.415 INFO:tasks.rados.rados.0.smithi136.stdout:2642: setattr oid 29 current snap is 270 2024-09-12T23:47:32.416 INFO:tasks.rados.rados.0.smithi136.stdout:2643: rmattr oid 12 current snap is 270 2024-09-12T23:47:32.417 INFO:tasks.rados.rados.0.smithi136.stdout:2643: done (2 left) 2024-09-12T23:47:32.417 INFO:tasks.rados.rados.0.smithi136.stdout:2644: read oid 3 snap -1 2024-09-12T23:47:32.417 INFO:tasks.rados.rados.0.smithi136.stdout:2644: expect (ObjNum 838 snap 267 seq_num 838) 2024-09-12T23:47:32.417 INFO:tasks.rados.rados.0.smithi136.stdout:2645: copy_from oid 15 from oid 24 current snap is 270 2024-09-12T23:47:32.417 INFO:tasks.rados.rados.0.smithi136.stdout:2646: read oid 32 snap 263 2024-09-12T23:47:32.417 INFO:tasks.rados.rados.0.smithi136.stdout:2646: expect deleted 2024-09-12T23:47:32.418 INFO:tasks.rados.rados.0.smithi136.stdout:2647: setattr oid 26 current snap is 270 2024-09-12T23:47:32.419 INFO:tasks.rados.rados.0.smithi136.stdout:2648: delete oid 30 current snap is 270 2024-09-12T23:47:32.432 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 26 v 1280 (ObjNum 822 snap 258 seq_num 822) dirty exists 2024-09-12T23:47:32.467 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 12 23:47:32 smithi181 systemd[1]: Started Ceph osd.7 for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:47:32.467 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:47:32 smithi181 bash[60471]: ts=2024-09-12T23:47:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:47:32.470 INFO:tasks.rados.rados.0.smithi136.stdout:2646: done (6 left) 2024-09-12T23:47:32.471 INFO:tasks.rados.rados.0.smithi136.stdout:2647: done (5 left) 2024-09-12T23:47:32.471 INFO:tasks.rados.rados.0.smithi136.stdout:2648: done (4 left) 2024-09-12T23:47:32.471 INFO:tasks.rados.rados.0.smithi136.stdout:2649: snap_remove snap 264 2024-09-12T23:47:32.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:32 smithi136 bash[75164]: cluster 2024-09-12T23:47:31.454849+0000 mon.a (mon.0) 847 : cluster [WRN] Health check failed: Degraded data redundancy: 16/603 objects degraded (2.653%), 2 pgs degraded (PG_DEGRADED) 2024-09-12T23:47:32.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:32 smithi136 bash[75164]: audit 2024-09-12T23:47:32.140613+0000 mon.a (mon.0) 848 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:32.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:32 smithi136 bash[75164]: audit 2024-09-12T23:47:32.151065+0000 mon.a (mon.0) 849 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:32.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:32 smithi136 bash[75164]: audit 2024-09-12T23:47:32.153365+0000 mon.a (mon.0) 850 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:47:32.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:32 smithi136 bash[75164]: cluster 2024-09-12T23:47:32.411856+0000 mon.a (mon.0) 851 : cluster [DBG] osdmap e596: 8 total, 7 up, 8 in 2024-09-12T23:47:32.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:32 smithi136 bash[76371]: cluster 2024-09-12T23:47:31.454849+0000 mon.a (mon.0) 847 : cluster [WRN] Health check failed: Degraded data redundancy: 16/603 objects degraded (2.653%), 2 pgs degraded (PG_DEGRADED) 2024-09-12T23:47:32.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:32 smithi136 bash[76371]: audit 2024-09-12T23:47:32.140613+0000 mon.a (mon.0) 848 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:32.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:32 smithi136 bash[76371]: audit 2024-09-12T23:47:32.151065+0000 mon.a (mon.0) 849 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:32.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:32 smithi136 bash[76371]: audit 2024-09-12T23:47:32.153365+0000 mon.a (mon.0) 850 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:47:32.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:32 smithi136 bash[76371]: cluster 2024-09-12T23:47:32.411856+0000 mon.a (mon.0) 851 : cluster [DBG] osdmap e596: 8 total, 7 up, 8 in 2024-09-12T23:47:32.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:32 smithi181 bash[67587]: cluster 2024-09-12T23:47:31.454849+0000 mon.a (mon.0) 847 : cluster [WRN] Health check failed: Degraded data redundancy: 16/603 objects degraded (2.653%), 2 pgs degraded (PG_DEGRADED) 2024-09-12T23:47:32.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:32 smithi181 bash[67587]: audit 2024-09-12T23:47:32.140613+0000 mon.a (mon.0) 848 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:32.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:32 smithi181 bash[67587]: audit 2024-09-12T23:47:32.151065+0000 mon.a (mon.0) 849 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:32.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:32 smithi181 bash[67587]: audit 2024-09-12T23:47:32.153365+0000 mon.a (mon.0) 850 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:47:32.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:32 smithi181 bash[67587]: cluster 2024-09-12T23:47:32.411856+0000 mon.a (mon.0) 851 : cluster [DBG] osdmap e596: 8 total, 7 up, 8 in 2024-09-12T23:47:33.486 INFO:tasks.rados.rados.0.smithi136.stdout:2649: done (4 left) 2024-09-12T23:47:33.486 INFO:tasks.rados.rados.0.smithi136.stdout:2650: rollback oid 31 current snap is 270 2024-09-12T23:47:33.486 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 31 to 269 2024-09-12T23:47:33.486 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 29 v 1625 (ObjNum 742 snap 228 seq_num 742) dirty exists 2024-09-12T23:47:33.486 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 43 v 1614 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-12T23:47:33.486 INFO:tasks.rados.rados.0.smithi136.stdout:2644: expect (ObjNum 838 snap 267 seq_num 838) 2024-09-12T23:47:33.586 INFO:tasks.rados.rados.0.smithi136.stdout:2645: finishing copy_from racing read to smithi13667081-15 2024-09-12T23:47:33.586 INFO:tasks.rados.rados.0.smithi136.stdout:2645: finishing copy_from to smithi13667081-15 2024-09-12T23:47:33.586 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 15 v 1279 (ObjNum 829 snap 266 seq_num 829) dirty exists 2024-09-12T23:47:33.586 INFO:tasks.rados.rados.0.smithi136.stdout:2650: finishing rollback tid 1 to smithi13667081-31 2024-09-12T23:47:33.586 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 31 v 1483 (ObjNum 177 snap 0 seq_num 1427291520) dirty dne 2024-09-12T23:47:33.587 INFO:tasks.rados.rados.0.smithi136.stdout:2641: done (4 left) 2024-09-12T23:47:33.587 INFO:tasks.rados.rados.0.smithi136.stdout:2642: done (3 left) 2024-09-12T23:47:33.587 INFO:tasks.rados.rados.0.smithi136.stdout:2644: done (2 left) 2024-09-12T23:47:33.587 INFO:tasks.rados.rados.0.smithi136.stdout:2645: done (1 left) 2024-09-12T23:47:33.587 INFO:tasks.rados.rados.0.smithi136.stdout:2650: done (0 left) 2024-09-12T23:47:33.587 INFO:tasks.rados.rados.0.smithi136.stdout:2651: read oid 40 snap -1 2024-09-12T23:47:33.587 INFO:tasks.rados.rados.0.smithi136.stdout:2651: expect (ObjNum 788 snap 250 seq_num 788) 2024-09-12T23:47:33.587 INFO:tasks.rados.rados.0.smithi136.stdout:2652: snap_create 2024-09-12T23:47:33.595 INFO:tasks.rados.rados.0.smithi136.stdout:2651: expect (ObjNum 788 snap 250 seq_num 788) 2024-09-12T23:47:33.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:33 smithi136 bash[75164]: cluster 2024-09-12T23:47:32.778016+0000 mgr.y (mgr.24661) 266 : cluster [DBG] pgmap v470: 121 pgs: 28 active+undersized, 8 peering, 8 active+undersized+degraded, 77 active+clean; 367 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 4.5 MiB/s wr, 11 op/s; 42/618 objects degraded (6.796%) 2024-09-12T23:47:33.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:33 smithi136 bash[76371]: cluster 2024-09-12T23:47:32.778016+0000 mgr.y (mgr.24661) 266 : cluster [DBG] pgmap v470: 121 pgs: 28 active+undersized, 8 peering, 8 active+undersized+degraded, 77 active+clean; 367 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 4.5 MiB/s wr, 11 op/s; 42/618 objects degraded (6.796%) 2024-09-12T23:47:33.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:33 smithi181 bash[67587]: cluster 2024-09-12T23:47:32.778016+0000 mgr.y (mgr.24661) 266 : cluster [DBG] pgmap v470: 121 pgs: 28 active+undersized, 8 peering, 8 active+undersized+degraded, 77 active+clean; 367 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 4.5 MiB/s wr, 11 op/s; 42/618 objects degraded (6.796%) 2024-09-12T23:47:34.106 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 12 23:47:33 smithi181 bash[84696]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-12T23:47:34.106 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 12 23:47:33 smithi181 bash[84696]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-12T23:47:34.106 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 12 23:47:33 smithi181 bash[84696]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-12T23:47:34.106 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 12 23:47:33 smithi181 bash[84696]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-12T23:47:34.106 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 12 23:47:33 smithi181 bash[84696]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-12T23:47:34.106 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 12 23:47:33 smithi181 bash[84696]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-12T23:47:34.106 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 12 23:47:33 smithi181 bash[84696]: --> ceph-volume raw activate successful for osd ID: 7 2024-09-12T23:47:34.496 INFO:tasks.rados.rados.0.smithi136.stdout:2651: done (1 left) 2024-09-12T23:47:34.496 INFO:tasks.rados.rados.0.smithi136.stdout:2652: done (0 left) 2024-09-12T23:47:34.496 INFO:tasks.rados.rados.0.smithi136.stdout:2653: read oid 10 snap -1 2024-09-12T23:47:34.496 INFO:tasks.rados.rados.0.smithi136.stdout:2653: expect deleted 2024-09-12T23:47:34.496 INFO:tasks.rados.rados.0.smithi136.stdout:2654: snap_create 2024-09-12T23:47:34.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:34 smithi136 bash[76371]: cluster 2024-09-12T23:47:33.487331+0000 mon.a (mon.0) 852 : cluster [DBG] osdmap e597: 8 total, 7 up, 8 in 2024-09-12T23:47:34.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:34 smithi136 bash[75164]: cluster 2024-09-12T23:47:33.487331+0000 mon.a (mon.0) 852 : cluster [DBG] osdmap e597: 8 total, 7 up, 8 in 2024-09-12T23:47:34.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:34 smithi181 bash[67587]: cluster 2024-09-12T23:47:33.487331+0000 mon.a (mon.0) 852 : cluster [DBG] osdmap e597: 8 total, 7 up, 8 in 2024-09-12T23:47:35.507 INFO:tasks.rados.rados.0.smithi136.stdout:2653: done (1 left) 2024-09-12T23:47:35.508 INFO:tasks.rados.rados.0.smithi136.stdout:2654: done (0 left) 2024-09-12T23:47:35.508 INFO:tasks.rados.rados.0.smithi136.stdout:2655: read oid 28 snap 246 2024-09-12T23:47:35.508 INFO:tasks.rados.rados.0.smithi136.stdout:2655: expect (ObjNum 739 snap 227 seq_num 739) 2024-09-12T23:47:35.508 INFO:tasks.rados.rados.0.smithi136.stdout:2656: snap_remove snap 234 2024-09-12T23:47:35.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:35 smithi136 bash[75164]: cluster 2024-09-12T23:47:34.498161+0000 mon.a (mon.0) 853 : cluster [DBG] osdmap e598: 8 total, 7 up, 8 in 2024-09-12T23:47:35.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:35 smithi136 bash[75164]: cluster 2024-09-12T23:47:34.778984+0000 mgr.y (mgr.24661) 267 : cluster [DBG] pgmap v473: 121 pgs: 34 active+undersized, 10 active+undersized+degraded, 77 active+clean; 367 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 3.1 MiB/s wr, 13 op/s; 46/627 objects degraded (7.337%) 2024-09-12T23:47:35.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:35 smithi136 bash[76371]: cluster 2024-09-12T23:47:34.498161+0000 mon.a (mon.0) 853 : cluster [DBG] osdmap e598: 8 total, 7 up, 8 in 2024-09-12T23:47:35.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:35 smithi136 bash[76371]: cluster 2024-09-12T23:47:34.778984+0000 mgr.y (mgr.24661) 267 : cluster [DBG] pgmap v473: 121 pgs: 34 active+undersized, 10 active+undersized+degraded, 77 active+clean; 367 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 3.1 MiB/s wr, 13 op/s; 46/627 objects degraded (7.337%) 2024-09-12T23:47:35.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:35 smithi181 bash[67587]: cluster 2024-09-12T23:47:34.498161+0000 mon.a (mon.0) 853 : cluster [DBG] osdmap e598: 8 total, 7 up, 8 in 2024-09-12T23:47:35.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:35 smithi181 bash[67587]: cluster 2024-09-12T23:47:34.778984+0000 mgr.y (mgr.24661) 267 : cluster [DBG] pgmap v473: 121 pgs: 34 active+undersized, 10 active+undersized+degraded, 77 active+clean; 367 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 3.1 MiB/s wr, 13 op/s; 46/627 objects degraded (7.337%) 2024-09-12T23:47:35.844 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 12 23:47:35 smithi181 bash[84920]: debug 2024-09-12T23:47:35.630+0000 7f2f05287640 1 -- 172.21.15.181:0/1461815457 <== mon.1 v2:172.21.15.181:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55a235d371e0 con 0x55a236ab4000 2024-09-12T23:47:36.408 INFO:tasks.rados.rados.0.smithi136.stdout:2656: done (1 left) 2024-09-12T23:47:36.408 INFO:tasks.rados.rados.0.smithi136.stdout:2657: rmattr oid 17 current snap is 272 2024-09-12T23:47:36.408 INFO:tasks.rados.rados.0.smithi136.stdout:2657: done (1 left) 2024-09-12T23:47:36.408 INFO:tasks.rados.rados.0.smithi136.stdout:2658: copy_from oid 26 from oid 25 current snap is 272 2024-09-12T23:47:36.408 INFO:tasks.rados.rados.0.smithi136.stdout:2659: read oid 36 snap -1 2024-09-12T23:47:36.409 INFO:tasks.rados.rados.0.smithi136.stdout:2659: expect (ObjNum 610 snap 175 seq_num 610) 2024-09-12T23:47:36.409 INFO:tasks.rados.rados.0.smithi136.stdout:2660: rollback oid 7 current snap is 272 2024-09-12T23:47:36.409 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 7 to 258 2024-09-12T23:47:36.409 INFO:tasks.rados.rados.0.smithi136.stdout:2661: read oid 43 snap -1 2024-09-12T23:47:36.409 INFO:tasks.rados.rados.0.smithi136.stdout:2661: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-12T23:47:36.409 INFO:tasks.rados.rados.0.smithi136.stdout:2662: read oid 32 snap -1 2024-09-12T23:47:36.409 INFO:tasks.rados.rados.0.smithi136.stdout:2662: expect deleted 2024-09-12T23:47:36.409 INFO:tasks.rados.rados.0.smithi136.stdout:2663: rmattr oid 19 current snap is 272 2024-09-12T23:47:36.409 INFO:tasks.rados.rados.0.smithi136.stdout:2663: done (6 left) 2024-09-12T23:47:36.409 INFO:tasks.rados.rados.0.smithi136.stdout:2664: read oid 20 snap 254 2024-09-12T23:47:36.409 INFO:tasks.rados.rados.0.smithi136.stdout:2664: expect (ObjNum 808 snap 253 seq_num 808) 2024-09-12T23:47:36.409 INFO:tasks.rados.rados.0.smithi136.stdout:2665: rollback oid 37 current snap is 272 2024-09-12T23:47:36.409 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 37 to 268 2024-09-12T23:47:36.410 INFO:tasks.rados.rados.0.smithi136.stdout:2666: copy_from oid 1 from oid 5 current snap is 272 2024-09-12T23:47:36.410 INFO:tasks.rados.rados.0.smithi136.stdout:2667: snap_remove snap 262 2024-09-12T23:47:36.498 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 12 23:47:36 smithi181 bash[84920]: debug 2024-09-12T23:47:36.166+0000 7f2f07aed640 -1 Falling back to public interface 2024-09-12T23:47:36.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:36 smithi136 bash[75164]: cluster 2024-09-12T23:47:35.492132+0000 mon.a (mon.0) 854 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 4 pgs peering) 2024-09-12T23:47:36.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:36 smithi136 bash[75164]: cluster 2024-09-12T23:47:35.509933+0000 mon.a (mon.0) 855 : cluster [DBG] osdmap e599: 8 total, 7 up, 8 in 2024-09-12T23:47:36.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:36 smithi136 bash[75164]: cluster 2024-09-12T23:47:36.409791+0000 mon.a (mon.0) 856 : cluster [DBG] osdmap e600: 8 total, 7 up, 8 in 2024-09-12T23:47:36.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:36 smithi136 bash[76371]: cluster 2024-09-12T23:47:35.492132+0000 mon.a (mon.0) 854 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 4 pgs peering) 2024-09-12T23:47:36.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:36 smithi136 bash[76371]: cluster 2024-09-12T23:47:35.509933+0000 mon.a (mon.0) 855 : cluster [DBG] osdmap e599: 8 total, 7 up, 8 in 2024-09-12T23:47:36.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:36 smithi136 bash[76371]: cluster 2024-09-12T23:47:36.409791+0000 mon.a (mon.0) 856 : cluster [DBG] osdmap e600: 8 total, 7 up, 8 in 2024-09-12T23:47:36.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:36 smithi181 bash[67587]: cluster 2024-09-12T23:47:35.492132+0000 mon.a (mon.0) 854 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 4 pgs peering) 2024-09-12T23:47:36.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:36 smithi181 bash[67587]: cluster 2024-09-12T23:47:35.509933+0000 mon.a (mon.0) 855 : cluster [DBG] osdmap e599: 8 total, 7 up, 8 in 2024-09-12T23:47:36.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:36 smithi181 bash[67587]: cluster 2024-09-12T23:47:36.409791+0000 mon.a (mon.0) 856 : cluster [DBG] osdmap e600: 8 total, 7 up, 8 in 2024-09-12T23:47:37.415 INFO:tasks.rados.rados.0.smithi136.stdout:2667: done (9 left) 2024-09-12T23:47:37.415 INFO:tasks.rados.rados.0.smithi136.stdout:2668: read oid 49 snap -1 2024-09-12T23:47:37.415 INFO:tasks.rados.rados.0.smithi136.stdout:2668: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-12T23:47:37.415 INFO:tasks.rados.rados.0.smithi136.stdout:2669: rmattr oid 42 current snap is 272 2024-09-12T23:47:37.415 INFO:tasks.rados.rados.0.smithi136.stdout:2669: done (10 left) 2024-09-12T23:47:37.415 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 29 current snap is 272 2024-09-12T23:47:37.417 INFO:tasks.rados.rados.0.smithi136.stdout:2670: seq_num 844 ranges {12640256=540672,13180928=696320,13877248=475136,14352384=434176} 2024-09-12T23:47:37.502 INFO:tasks.rados.rados.0.smithi136.stdout:2670: writing smithi13667081-29 from 12640256 to 13180928 tid 1 2024-09-12T23:47:37.507 INFO:tasks.rados.rados.0.smithi136.stdout:2670: writing smithi13667081-29 from 13180928 to 13877248 tid 2 2024-09-12T23:47:37.510 INFO:tasks.rados.rados.0.smithi136.stdout:2670: writing smithi13667081-29 from 13877248 to 14352384 tid 3 2024-09-12T23:47:37.513 INFO:tasks.rados.rados.0.smithi136.stdout:2670: writing smithi13667081-29 from 14352384 to 14786560 tid 4 2024-09-12T23:47:37.513 INFO:tasks.rados.rados.0.smithi136.stdout:2671: delete oid 9 current snap is 272 2024-09-12T23:47:37.513 INFO:tasks.rados.rados.0.smithi136.stdout:2655: expect (ObjNum 739 snap 227 seq_num 739) 2024-09-12T23:47:37.542 INFO:tasks.rados.rados.0.smithi136.stdout:2658: finishing copy_from to smithi13667081-26 2024-09-12T23:47:37.542 INFO:tasks.rados.rados.0.smithi136.stdout:2658: got expected ENOENT (src dne) 2024-09-12T23:47:37.542 INFO:tasks.rados.rados.0.smithi136.stdout:2658: finishing copy_from racing read to smithi13667081-26 2024-09-12T23:47:37.542 INFO:tasks.rados.rados.0.smithi136.stdout:2661: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-12T23:47:37.545 INFO:tasks.rados.rados.0.smithi136.stdout:2659: expect (ObjNum 610 snap 175 seq_num 610) 2024-09-12T23:47:37.582 INFO:tasks.rados.rados.0.smithi136.stdout:2660: finishing rollback tid 0 to smithi13667081-7 2024-09-12T23:47:37.582 INFO:tasks.rados.rados.0.smithi136.stdout:2660: finishing rollback tid 1 to smithi13667081-7 2024-09-12T23:47:37.582 INFO:tasks.rados.rados.0.smithi136.stdout:2660: finishing rollback tid 2 to smithi13667081-7 2024-09-12T23:47:37.582 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 7 v 958 (ObjNum 810 snap 253 seq_num 810) dirty exists 2024-09-12T23:47:37.582 INFO:tasks.rados.rados.0.smithi136.stdout:2665: finishing rollback tid 0 to smithi13667081-37 2024-09-12T23:47:37.582 INFO:tasks.rados.rados.0.smithi136.stdout:2665: finishing rollback tid 1 to smithi13667081-37 2024-09-12T23:47:37.582 INFO:tasks.rados.rados.0.smithi136.stdout:2665: finishing rollback tid 2 to smithi13667081-37 2024-09-12T23:47:37.582 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 37 v 1282 (ObjNum 796 snap 253 seq_num 796) dirty exists 2024-09-12T23:47:37.582 INFO:tasks.rados.rados.0.smithi136.stdout:2664: expect (ObjNum 808 snap 253 seq_num 808) 2024-09-12T23:47:37.656 INFO:tasks.rados.rados.0.smithi136.stdout:2666: finishing copy_from racing read to smithi13667081-1 2024-09-12T23:47:37.657 INFO:tasks.rados.rados.0.smithi136.stdout:2666: finishing copy_from to smithi13667081-1 2024-09-12T23:47:37.657 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 1 v 1780 (ObjNum 567 snap 162 seq_num 567) dirty exists 2024-09-12T23:47:37.657 INFO:tasks.rados.rados.0.smithi136.stdout:2668: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-12T23:47:37.661 INFO:tasks.rados.rados.0.smithi136.stdout:2670: finishing write tid 1 to smithi13667081-29 2024-09-12T23:47:37.661 INFO:tasks.rados.rados.0.smithi136.stdout:2670: finishing write tid 2 to smithi13667081-29 2024-09-12T23:47:37.661 INFO:tasks.rados.rados.0.smithi136.stdout:2670: finishing write tid 3 to smithi13667081-29 2024-09-12T23:47:37.661 INFO:tasks.rados.rados.0.smithi136.stdout:2670: finishing write tid 4 to smithi13667081-29 2024-09-12T23:47:37.661 INFO:tasks.rados.rados.0.smithi136.stdout:2670: finishing write tid 5 to smithi13667081-29 2024-09-12T23:47:37.661 INFO:tasks.rados.rados.0.smithi136.stdout:2670: finishing write tid 6 to smithi13667081-29 2024-09-12T23:47:37.661 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 29 v 1630 (ObjNum 844 snap 272 seq_num 844) dirty exists 2024-09-12T23:47:37.661 INFO:tasks.rados.rados.0.smithi136.stdout:2670: left oid 29 (ObjNum 844 snap 272 seq_num 844) 2024-09-12T23:47:37.661 INFO:tasks.rados.rados.0.smithi136.stdout:2655: done (11 left) 2024-09-12T23:47:37.661 INFO:tasks.rados.rados.0.smithi136.stdout:2658: done (10 left) 2024-09-12T23:47:37.661 INFO:tasks.rados.rados.0.smithi136.stdout:2659: done (9 left) 2024-09-12T23:47:37.662 INFO:tasks.rados.rados.0.smithi136.stdout:2660: done (8 left) 2024-09-12T23:47:37.662 INFO:tasks.rados.rados.0.smithi136.stdout:2661: done (7 left) 2024-09-12T23:47:37.662 INFO:tasks.rados.rados.0.smithi136.stdout:2662: done (6 left) 2024-09-12T23:47:37.662 INFO:tasks.rados.rados.0.smithi136.stdout:2664: done (5 left) 2024-09-12T23:47:37.662 INFO:tasks.rados.rados.0.smithi136.stdout:2665: done (4 left) 2024-09-12T23:47:37.662 INFO:tasks.rados.rados.0.smithi136.stdout:2666: done (3 left) 2024-09-12T23:47:37.662 INFO:tasks.rados.rados.0.smithi136.stdout:2668: done (2 left) 2024-09-12T23:47:37.662 INFO:tasks.rados.rados.0.smithi136.stdout:2670: done (1 left) 2024-09-12T23:47:37.662 INFO:tasks.rados.rados.0.smithi136.stdout:2671: done (0 left) 2024-09-12T23:47:37.662 INFO:tasks.rados.rados.0.smithi136.stdout:2672: rollback oid 50 current snap is 272 2024-09-12T23:47:37.662 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 50 to 270 2024-09-12T23:47:37.662 INFO:tasks.rados.rados.0.smithi136.stdout:2673: read oid 30 snap -1 2024-09-12T23:47:37.662 INFO:tasks.rados.rados.0.smithi136.stdout:2673: expect deleted 2024-09-12T23:47:37.662 INFO:tasks.rados.rados.0.smithi136.stdout:2674: snap_remove snap 257 2024-09-12T23:47:37.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:37 smithi181 bash[67587]: cluster 2024-09-12T23:47:36.780165+0000 mgr.y (mgr.24661) 268 : cluster [DBG] pgmap v476: 121 pgs: 34 active+undersized, 10 active+undersized+degraded, 77 active+clean; 377 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 2.4 MiB/s wr, 20 op/s; 47/636 objects degraded (7.390%) 2024-09-12T23:47:37.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:37 smithi181 bash[67587]: cluster 2024-09-12T23:47:37.416065+0000 mon.a (mon.0) 857 : cluster [DBG] osdmap e601: 8 total, 7 up, 8 in 2024-09-12T23:47:38.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:37 smithi136 bash[75164]: cluster 2024-09-12T23:47:36.780165+0000 mgr.y (mgr.24661) 268 : cluster [DBG] pgmap v476: 121 pgs: 34 active+undersized, 10 active+undersized+degraded, 77 active+clean; 377 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 2.4 MiB/s wr, 20 op/s; 47/636 objects degraded (7.390%) 2024-09-12T23:47:38.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:37 smithi136 bash[75164]: cluster 2024-09-12T23:47:37.416065+0000 mon.a (mon.0) 857 : cluster [DBG] osdmap e601: 8 total, 7 up, 8 in 2024-09-12T23:47:38.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:37 smithi136 bash[76371]: cluster 2024-09-12T23:47:36.780165+0000 mgr.y (mgr.24661) 268 : cluster [DBG] pgmap v476: 121 pgs: 34 active+undersized, 10 active+undersized+degraded, 77 active+clean; 377 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 2.4 MiB/s wr, 20 op/s; 47/636 objects degraded (7.390%) 2024-09-12T23:47:38.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:37 smithi136 bash[76371]: cluster 2024-09-12T23:47:37.416065+0000 mon.a (mon.0) 857 : cluster [DBG] osdmap e601: 8 total, 7 up, 8 in 2024-09-12T23:47:38.527 INFO:tasks.rados.rados.0.smithi136.stdout:2674: done (2 left) 2024-09-12T23:47:38.528 INFO:tasks.rados.rados.0.smithi136.stdout:2675: read oid 27 snap -1 2024-09-12T23:47:38.528 INFO:tasks.rados.rados.0.smithi136.stdout:2675: expect (ObjNum 766 snap 236 seq_num 766) 2024-09-12T23:47:38.528 INFO:tasks.rados.rados.0.smithi136.stdout:2676: rmattr oid 22 current snap is 272 2024-09-12T23:47:38.528 INFO:tasks.rados.rados.0.smithi136.stdout:2677: read oid 20 snap -1 2024-09-12T23:47:38.528 INFO:tasks.rados.rados.0.smithi136.stdout:2677: expect deleted 2024-09-12T23:47:38.529 INFO:tasks.rados.rados.0.smithi136.stdout:2678: delete oid 7 current snap is 272 2024-09-12T23:47:38.529 INFO:tasks.rados.rados.0.smithi136.stdout:2672: finishing rollback tid 1 to smithi13667081-50 2024-09-12T23:47:38.529 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 50 v 1239 (ObjNum 33 snap 0 seq_num 4242595856) dirty dne 2024-09-12T23:47:38.533 INFO:tasks.rados.rados.0.smithi136.stdout:2672: done (5 left) 2024-09-12T23:47:38.534 INFO:tasks.rados.rados.0.smithi136.stdout:2673: done (4 left) 2024-09-12T23:47:38.534 INFO:tasks.rados.rados.0.smithi136.stdout:2678: done (3 left) 2024-09-12T23:47:38.534 INFO:tasks.rados.rados.0.smithi136.stdout:2679: rollback oid 41 current snap is 272 2024-09-12T23:47:38.534 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 41 to 258 2024-09-12T23:47:38.534 INFO:tasks.rados.rados.0.smithi136.stdout:2680: snap_remove snap 268 2024-09-12T23:47:38.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:38 smithi181 bash[67587]: cluster 2024-09-12T23:47:37.502246+0000 mon.a (mon.0) 858 : cluster [WRN] Health check update: Degraded data redundancy: 47/636 objects degraded (7.390%), 10 pgs degraded (PG_DEGRADED) 2024-09-12T23:47:39.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:38 smithi136 bash[75164]: cluster 2024-09-12T23:47:37.502246+0000 mon.a (mon.0) 858 : cluster [WRN] Health check update: Degraded data redundancy: 47/636 objects degraded (7.390%), 10 pgs degraded (PG_DEGRADED) 2024-09-12T23:47:39.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:38 smithi136 bash[76371]: cluster 2024-09-12T23:47:37.502246+0000 mon.a (mon.0) 858 : cluster [WRN] Health check update: Degraded data redundancy: 47/636 objects degraded (7.390%), 10 pgs degraded (PG_DEGRADED) 2024-09-12T23:47:39.540 INFO:tasks.rados.rados.0.smithi136.stdout:2680: done (4 left) 2024-09-12T23:47:39.540 INFO:tasks.rados.rados.0.smithi136.stdout:2681: copy_from oid 26 from oid 18 current snap is 272 2024-09-12T23:47:39.540 INFO:tasks.rados.rados.0.smithi136.stdout:2682: read oid 35 snap -1 2024-09-12T23:47:39.540 INFO:tasks.rados.rados.0.smithi136.stdout:2682: expect (ObjNum 671 snap 196 seq_num 671) 2024-09-12T23:47:39.541 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 37 current snap is 272 2024-09-12T23:47:39.541 INFO:tasks.rados.rados.0.smithi136.stdout:2683: seq_num 847 ranges {7077888=589824,7667712=753664,8421376=655360,9076736=196608} 2024-09-12T23:47:39.588 INFO:tasks.rados.rados.0.smithi136.stdout:2683: writing smithi13667081-37 from 7077888 to 7667712 tid 1 2024-09-12T23:47:39.593 INFO:tasks.rados.rados.0.smithi136.stdout:2683: writing smithi13667081-37 from 7667712 to 8421376 tid 2 2024-09-12T23:47:39.598 INFO:tasks.rados.rados.0.smithi136.stdout:2683: writing smithi13667081-37 from 8421376 to 9076736 tid 3 2024-09-12T23:47:39.599 INFO:tasks.rados.rados.0.smithi136.stdout:2683: writing smithi13667081-37 from 9076736 to 9273344 tid 4 2024-09-12T23:47:39.599 INFO:tasks.rados.rados.0.smithi136.stdout:2684: snap_remove snap 256 2024-09-12T23:47:39.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:47:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:47:39] "GET /metrics HTTP/1.1" 200 36383 "" "Prometheus/2.43.0" 2024-09-12T23:47:39.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:39 smithi136 bash[75164]: cluster 2024-09-12T23:47:38.529319+0000 mon.a (mon.0) 859 : cluster [DBG] osdmap e602: 8 total, 7 up, 8 in 2024-09-12T23:47:39.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:39 smithi136 bash[75164]: audit 2024-09-12T23:47:38.754898+0000 mon.a (mon.0) 860 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:39.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:39 smithi136 bash[75164]: audit 2024-09-12T23:47:38.756963+0000 mon.a (mon.0) 861 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:47:39.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:39 smithi136 bash[75164]: cluster 2024-09-12T23:47:38.781012+0000 mgr.y (mgr.24661) 269 : cluster [DBG] pgmap v479: 121 pgs: 34 active+undersized, 10 active+undersized+degraded, 77 active+clean; 384 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 4.1 MiB/s wr, 16 op/s; 47/639 objects degraded (7.355%) 2024-09-12T23:47:39.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:39 smithi136 bash[75164]: audit 2024-09-12T23:47:39.097156+0000 mon.a (mon.0) 862 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:39.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:39 smithi136 bash[75164]: audit 2024-09-12T23:47:39.107113+0000 mon.a (mon.0) 863 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:39.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:39 smithi136 bash[76371]: cluster 2024-09-12T23:47:38.529319+0000 mon.a (mon.0) 859 : cluster [DBG] osdmap e602: 8 total, 7 up, 8 in 2024-09-12T23:47:39.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:39 smithi136 bash[76371]: audit 2024-09-12T23:47:38.754898+0000 mon.a (mon.0) 860 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:39.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:39 smithi136 bash[76371]: audit 2024-09-12T23:47:38.756963+0000 mon.a (mon.0) 861 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:47:39.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:39 smithi136 bash[76371]: cluster 2024-09-12T23:47:38.781012+0000 mgr.y (mgr.24661) 269 : cluster [DBG] pgmap v479: 121 pgs: 34 active+undersized, 10 active+undersized+degraded, 77 active+clean; 384 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 4.1 MiB/s wr, 16 op/s; 47/639 objects degraded (7.355%) 2024-09-12T23:47:39.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:39 smithi136 bash[76371]: audit 2024-09-12T23:47:39.097156+0000 mon.a (mon.0) 862 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:39.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:39 smithi136 bash[76371]: audit 2024-09-12T23:47:39.107113+0000 mon.a (mon.0) 863 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:39.844 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 12 23:47:39 smithi181 bash[84920]: debug 2024-09-12T23:47:39.682+0000 7f2f07aed640 -1 osd.7 591 log_to_monitors true 2024-09-12T23:47:39.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:39 smithi181 bash[67587]: cluster 2024-09-12T23:47:38.529319+0000 mon.a (mon.0) 859 : cluster [DBG] osdmap e602: 8 total, 7 up, 8 in 2024-09-12T23:47:39.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:39 smithi181 bash[67587]: audit 2024-09-12T23:47:38.754898+0000 mon.a (mon.0) 860 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:39.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:39 smithi181 bash[67587]: audit 2024-09-12T23:47:38.756963+0000 mon.a (mon.0) 861 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:47:39.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:39 smithi181 bash[67587]: cluster 2024-09-12T23:47:38.781012+0000 mgr.y (mgr.24661) 269 : cluster [DBG] pgmap v479: 121 pgs: 34 active+undersized, 10 active+undersized+degraded, 77 active+clean; 384 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 4.1 MiB/s wr, 16 op/s; 47/639 objects degraded (7.355%) 2024-09-12T23:47:39.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:39 smithi181 bash[67587]: audit 2024-09-12T23:47:39.097156+0000 mon.a (mon.0) 862 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:39.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:39 smithi181 bash[67587]: audit 2024-09-12T23:47:39.107113+0000 mon.a (mon.0) 863 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:40.556 INFO:tasks.rados.rados.0.smithi136.stdout:2684: done (7 left) 2024-09-12T23:47:40.556 INFO:tasks.rados.rados.0.smithi136.stdout:2685: read oid 14 snap -1 2024-09-12T23:47:40.556 INFO:tasks.rados.rados.0.smithi136.stdout:2685: expect deleted 2024-09-12T23:47:40.556 INFO:tasks.rados.rados.0.smithi136.stdout:2686: read oid 5 snap -1 2024-09-12T23:47:40.556 INFO:tasks.rados.rados.0.smithi136.stdout:2686: expect (ObjNum 567 snap 162 seq_num 567) 2024-09-12T23:47:40.557 INFO:tasks.rados.rados.0.smithi136.stdout:2687: rollback oid 36 current snap is 272 2024-09-12T23:47:40.557 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 36 to 270 2024-09-12T23:47:40.557 INFO:tasks.rados.rados.0.smithi136.stdout:2688: setattr oid 6 current snap is 272 2024-09-12T23:47:40.557 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 22 v 0 (ObjNum 49 snap 0 seq_num 1953066355) dirty dne 2024-09-12T23:47:40.557 INFO:tasks.rados.rados.0.smithi136.stdout:2679: finishing rollback tid 0 to smithi13667081-41 2024-09-12T23:47:40.557 INFO:tasks.rados.rados.0.smithi136.stdout:2679: finishing rollback tid 1 to smithi13667081-41 2024-09-12T23:47:40.557 INFO:tasks.rados.rados.0.smithi136.stdout:2679: finishing rollback tid 2 to smithi13667081-41 2024-09-12T23:47:40.557 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 41 v 1285 (ObjNum 817 snap 257 seq_num 817) dirty exists 2024-09-12T23:47:40.557 INFO:tasks.rados.rados.0.smithi136.stdout:2675: expect (ObjNum 766 snap 236 seq_num 766) 2024-09-12T23:47:40.707 INFO:teuthology.orchestra.run.smithi136.stdout:true 2024-09-12T23:47:40.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:40 smithi181 bash[67587]: cluster 2024-09-12T23:47:39.539049+0000 mon.a (mon.0) 864 : cluster [DBG] osdmap e603: 8 total, 7 up, 8 in 2024-09-12T23:47:40.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:40 smithi181 bash[67587]: audit 2024-09-12T23:47:39.688065+0000 mon.a (mon.0) 865 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-12T23:47:40.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:40 smithi181 bash[67587]: audit 2024-09-12T23:47:39.688972+0000 mon.b (mon.1) 10 : audit [INF] from='osd.7 [v2:172.21.15.181:6824/2237860309,v1:172.21.15.181:6825/2237860309]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-12T23:47:40.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:40 smithi181 bash[67587]: audit 2024-09-12T23:47:39.924943+0000 mon.a (mon.0) 866 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:40.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:40 smithi181 bash[67587]: audit 2024-09-12T23:47:39.936457+0000 mon.a (mon.0) 867 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:41.003 INFO:teuthology.orchestra.run.smithi136.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-12T23:47:41.003 INFO:teuthology.orchestra.run.smithi136.stdout:alertmanager.a smithi136 *:9093,9094 running (7m) 117s ago 13m 13.1M - 0.25.0 c8568f914cd2 b5197f020eeb 2024-09-12T23:47:41.003 INFO:teuthology.orchestra.run.smithi136.stdout:grafana.a smithi181 *:3000 running (6m) 1s ago 12m 75.5M - 9.4.7 954c08fa6188 256c181be984 2024-09-12T23:47:41.003 INFO:teuthology.orchestra.run.smithi136.stdout:mds.a smithi136 running (14m) 117s ago 14m 15.1M - 17.2.7-1627-gbbdadfa3 214985049bdb 713b9795ad19 2024-09-12T23:47:41.004 INFO:teuthology.orchestra.run.smithi136.stdout:mds.b smithi181 running (14m) 1s ago 14m 19.9M - 17.2.7-1627-gbbdadfa3 214985049bdb ad13179f0868 2024-09-12T23:47:41.004 INFO:teuthology.orchestra.run.smithi136.stdout:mgr.x smithi181 *:8443,9283,8765 running (6m) 1s ago 21m 427M - 18.2.4-837-g7a853b17 f21ba1eea74e cc664dc337f7 2024-09-12T23:47:41.004 INFO:teuthology.orchestra.run.smithi136.stdout:mgr.y smithi136 *:8443,9283,8765 running (6m) 117s ago 24m 490M - 18.2.4-837-g7a853b17 f21ba1eea74e 272f10b42fc3 2024-09-12T23:47:41.004 INFO:teuthology.orchestra.run.smithi136.stdout:mon.a smithi136 running (5m) 117s ago 24m 56.4M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e ad8d21d58ad9 2024-09-12T23:47:41.004 INFO:teuthology.orchestra.run.smithi136.stdout:mon.b smithi181 running (4m) 1s ago 22m 45.1M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e a70ec8315934 2024-09-12T23:47:41.004 INFO:teuthology.orchestra.run.smithi136.stdout:mon.c smithi136 running (5m) 117s ago 22m 47.2M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e 44d1be055a5e 2024-09-12T23:47:41.004 INFO:teuthology.orchestra.run.smithi136.stdout:node-exporter.a smithi136 *:9100 running (7m) 117s ago 13m 8064k - 1.5.0 0da6a335fe13 ccfc37c96620 2024-09-12T23:47:41.004 INFO:teuthology.orchestra.run.smithi136.stdout:node-exporter.b smithi181 *:9100 running (7m) 1s ago 13m 8307k - 1.5.0 0da6a335fe13 80bf0e383230 2024-09-12T23:47:41.004 INFO:teuthology.orchestra.run.smithi136.stdout:osd.0 smithi136 running (3m) 117s ago 20m 212M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 7da21f765721 2024-09-12T23:47:41.004 INFO:teuthology.orchestra.run.smithi136.stdout:osd.1 smithi136 running (3m) 117s ago 20m 247M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 5302be0a016b 2024-09-12T23:47:41.004 INFO:teuthology.orchestra.run.smithi136.stdout:osd.2 smithi136 running (2m) 117s ago 19m 252M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 2e12d1a31223 2024-09-12T23:47:41.004 INFO:teuthology.orchestra.run.smithi136.stdout:osd.3 smithi136 running (2m) 117s ago 18m - 2524M 18.2.4-837-g7a853b17 f21ba1eea74e aa9dc124e59c 2024-09-12T23:47:41.004 INFO:teuthology.orchestra.run.smithi136.stdout:osd.4 smithi181 running (92s) 1s ago 17m 377M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 4ee3b228ce7d 2024-09-12T23:47:41.005 INFO:teuthology.orchestra.run.smithi136.stdout:osd.5 smithi181 running (64s) 1s ago 16m 194M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 3f9a09565258 2024-09-12T23:47:41.005 INFO:teuthology.orchestra.run.smithi136.stdout:osd.6 smithi181 running (35s) 1s ago 15m 115M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e ce539f6fb854 2024-09-12T23:47:41.005 INFO:teuthology.orchestra.run.smithi136.stdout:osd.7 smithi181 running (6s) 1s ago 15m - 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 6ad9ab169a7b 2024-09-12T23:47:41.005 INFO:teuthology.orchestra.run.smithi136.stdout:prometheus.a smithi181 *:9095 running (6m) 1s ago 13m 57.5M - 2.43.0 a07b618ecd1d 3853d003a5cc 2024-09-12T23:47:41.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:40 smithi136 bash[76371]: cluster 2024-09-12T23:47:39.539049+0000 mon.a (mon.0) 864 : cluster [DBG] osdmap e603: 8 total, 7 up, 8 in 2024-09-12T23:47:41.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:40 smithi136 bash[76371]: audit 2024-09-12T23:47:39.688065+0000 mon.a (mon.0) 865 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-12T23:47:41.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:40 smithi136 bash[76371]: audit 2024-09-12T23:47:39.688972+0000 mon.b (mon.1) 10 : audit [INF] from='osd.7 [v2:172.21.15.181:6824/2237860309,v1:172.21.15.181:6825/2237860309]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-12T23:47:41.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:40 smithi136 bash[76371]: audit 2024-09-12T23:47:39.924943+0000 mon.a (mon.0) 866 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:41.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:40 smithi136 bash[76371]: audit 2024-09-12T23:47:39.936457+0000 mon.a (mon.0) 867 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:41.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:40 smithi136 bash[75164]: cluster 2024-09-12T23:47:39.539049+0000 mon.a (mon.0) 864 : cluster [DBG] osdmap e603: 8 total, 7 up, 8 in 2024-09-12T23:47:41.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:40 smithi136 bash[75164]: audit 2024-09-12T23:47:39.688065+0000 mon.a (mon.0) 865 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-12T23:47:41.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:40 smithi136 bash[75164]: audit 2024-09-12T23:47:39.688972+0000 mon.b (mon.1) 10 : audit [INF] from='osd.7 [v2:172.21.15.181:6824/2237860309,v1:172.21.15.181:6825/2237860309]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-12T23:47:41.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:40 smithi136 bash[75164]: audit 2024-09-12T23:47:39.924943+0000 mon.a (mon.0) 866 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:41.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:40 smithi136 bash[75164]: audit 2024-09-12T23:47:39.936457+0000 mon.a (mon.0) 867 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:41.094 INFO:tasks.rados.rados.0.smithi136.stdout:2682: expect (ObjNum 671 snap 196 seq_num 671) 2024-09-12T23:47:41.199 INFO:tasks.rados.rados.0.smithi136.stdout:2681: finishing copy_from racing read to smithi13667081-26 2024-09-12T23:47:41.200 INFO:tasks.rados.rados.0.smithi136.stdout:2681: finishing copy_from to smithi13667081-26 2024-09-12T23:47:41.200 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 26 v 945 (ObjNum 724 snap 224 seq_num 724) dirty exists 2024-09-12T23:47:41.200 INFO:tasks.rados.rados.0.smithi136.stdout:2683: finishing write tid 1 to smithi13667081-37 2024-09-12T23:47:41.200 INFO:tasks.rados.rados.0.smithi136.stdout:2683: finishing write tid 2 to smithi13667081-37 2024-09-12T23:47:41.200 INFO:tasks.rados.rados.0.smithi136.stdout:2683: finishing write tid 3 to smithi13667081-37 2024-09-12T23:47:41.200 INFO:tasks.rados.rados.0.smithi136.stdout:2683: finishing write tid 4 to smithi13667081-37 2024-09-12T23:47:41.200 INFO:tasks.rados.rados.0.smithi136.stdout:2683: finishing write tid 5 to smithi13667081-37 2024-09-12T23:47:41.200 INFO:tasks.rados.rados.0.smithi136.stdout:2683: finishing write tid 6 to smithi13667081-37 2024-09-12T23:47:41.200 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 37 v 1290 (ObjNum 847 snap 272 seq_num 847) dirty exists 2024-09-12T23:47:41.200 INFO:tasks.rados.rados.0.smithi136.stdout:2683: left oid 37 (ObjNum 847 snap 272 seq_num 847) 2024-09-12T23:47:41.200 INFO:tasks.rados.rados.0.smithi136.stdout:2687: finishing rollback tid 0 to smithi13667081-36 2024-09-12T23:47:41.200 INFO:tasks.rados.rados.0.smithi136.stdout:2687: finishing rollback tid 1 to smithi13667081-36 2024-09-12T23:47:41.200 INFO:tasks.rados.rados.0.smithi136.stdout:2687: finishing rollback tid 2 to smithi13667081-36 2024-09-12T23:47:41.200 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 36 v 1254 (ObjNum 610 snap 175 seq_num 610) dirty exists 2024-09-12T23:47:41.201 INFO:tasks.rados.rados.0.smithi136.stdout:2686: expect (ObjNum 567 snap 162 seq_num 567) 2024-09-12T23:47:41.351 INFO:teuthology.orchestra.run.smithi136.stdout:{ 2024-09-12T23:47:41.351 INFO:teuthology.orchestra.run.smithi136.stdout: "mon": { 2024-09-12T23:47:41.351 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 3 2024-09-12T23:47:41.352 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:47:41.352 INFO:teuthology.orchestra.run.smithi136.stdout: "mgr": { 2024-09-12T23:47:41.352 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 2 2024-09-12T23:47:41.352 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:47:41.352 INFO:teuthology.orchestra.run.smithi136.stdout: "osd": { 2024-09-12T23:47:41.352 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 7 2024-09-12T23:47:41.352 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:47:41.352 INFO:teuthology.orchestra.run.smithi136.stdout: "mds": { 2024-09-12T23:47:41.352 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-12T23:47:41.352 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:47:41.352 INFO:teuthology.orchestra.run.smithi136.stdout: "overall": { 2024-09-12T23:47:41.352 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2, 2024-09-12T23:47:41.352 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 12 2024-09-12T23:47:41.353 INFO:teuthology.orchestra.run.smithi136.stdout: } 2024-09-12T23:47:41.353 INFO:teuthology.orchestra.run.smithi136.stdout:} 2024-09-12T23:47:41.522 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 6 v 1604 (ObjNum 836 snap 267 seq_num 836) dirty exists 2024-09-12T23:47:41.522 INFO:tasks.rados.rados.0.smithi136.stdout:2675: done (10 left) 2024-09-12T23:47:41.522 INFO:tasks.rados.rados.0.smithi136.stdout:2676: done (9 left) 2024-09-12T23:47:41.522 INFO:tasks.rados.rados.0.smithi136.stdout:2677: done (8 left) 2024-09-12T23:47:41.522 INFO:tasks.rados.rados.0.smithi136.stdout:2679: done (7 left) 2024-09-12T23:47:41.522 INFO:tasks.rados.rados.0.smithi136.stdout:2681: done (6 left) 2024-09-12T23:47:41.522 INFO:tasks.rados.rados.0.smithi136.stdout:2682: done (5 left) 2024-09-12T23:47:41.522 INFO:tasks.rados.rados.0.smithi136.stdout:2683: done (4 left) 2024-09-12T23:47:41.522 INFO:tasks.rados.rados.0.smithi136.stdout:2685: done (3 left) 2024-09-12T23:47:41.522 INFO:tasks.rados.rados.0.smithi136.stdout:2686: done (2 left) 2024-09-12T23:47:41.522 INFO:tasks.rados.rados.0.smithi136.stdout:2687: done (1 left) 2024-09-12T23:47:41.522 INFO:tasks.rados.rados.0.smithi136.stdout:2688: done (0 left) 2024-09-12T23:47:41.522 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 8 current snap is 272 2024-09-12T23:47:41.523 INFO:tasks.rados.rados.0.smithi136.stdout:2689: seq_num 848 ranges {3284992=606208,3891200=630784,4521984=458752,4980736=516096} 2024-09-12T23:47:41.547 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 12 23:47:41 smithi181 bash[84920]: debug 2024-09-12T23:47:41.126+0000 7f2efb88f640 -1 osd.7 591 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-12T23:47:41.552 INFO:tasks.rados.rados.0.smithi136.stdout:2689: writing smithi13667081-8 from 3284992 to 3891200 tid 1 2024-09-12T23:47:41.557 INFO:tasks.rados.rados.0.smithi136.stdout:2689: writing smithi13667081-8 from 3891200 to 4521984 tid 2 2024-09-12T23:47:41.561 INFO:tasks.rados.rados.0.smithi136.stdout:2689: writing smithi13667081-8 from 4521984 to 4980736 tid 3 2024-09-12T23:47:41.565 INFO:tasks.rados.rados.0.smithi136.stdout:2689: writing smithi13667081-8 from 4980736 to 5496832 tid 4 2024-09-12T23:47:41.565 INFO:tasks.rados.rados.0.smithi136.stdout:2690: delete oid 15 current snap is 272 2024-09-12T23:47:41.565 INFO:tasks.rados.rados.0.smithi136.stdout:2689: finishing write tid 1 to smithi13667081-8 2024-09-12T23:47:41.567 INFO:tasks.rados.rados.0.smithi136.stdout:2689: finishing write tid 2 to smithi13667081-8 2024-09-12T23:47:41.573 INFO:tasks.rados.rados.0.smithi136.stdout:2689: finishing write tid 3 to smithi13667081-8 2024-09-12T23:47:41.574 INFO:tasks.rados.rados.0.smithi136.stdout:2689: finishing write tid 4 to smithi13667081-8 2024-09-12T23:47:41.574 INFO:tasks.rados.rados.0.smithi136.stdout:2690: done (1 left) 2024-09-12T23:47:41.574 INFO:tasks.rados.rados.0.smithi136.stdout:2691: setattr oid 23 current snap is 272 2024-09-12T23:47:41.575 INFO:tasks.rados.rados.0.smithi136.stdout:2689: finishing write tid 5 to smithi13667081-8 2024-09-12T23:47:41.576 INFO:tasks.rados.rados.0.smithi136.stdout:2692: copy_from oid 3 from oid 7 current snap is 272 2024-09-12T23:47:41.576 INFO:tasks.rados.rados.0.smithi136.stdout:2693: read oid 38 snap -1 2024-09-12T23:47:41.576 INFO:tasks.rados.rados.0.smithi136.stdout:2693: expect (ObjNum 727 snap 224 seq_num 727) 2024-09-12T23:47:41.576 INFO:tasks.rados.rados.0.smithi136.stdout:2694: rollback oid 11 current snap is 272 2024-09-12T23:47:41.576 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 11 to 251 2024-09-12T23:47:41.576 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 10 current snap is 272 2024-09-12T23:47:41.576 INFO:tasks.rados.rados.0.smithi136.stdout:2695: seq_num 850 ranges {0=638976,638976=802816,1441792=475136,1916928=327680} 2024-09-12T23:47:41.582 INFO:tasks.rados.rados.0.smithi136.stdout:2695: writing smithi13667081-10 from 0 to 638976 tid 1 2024-09-12T23:47:41.590 INFO:tasks.rados.rados.0.smithi136.stdout:2695: writing smithi13667081-10 from 638976 to 1441792 tid 2 2024-09-12T23:47:41.594 INFO:tasks.rados.rados.0.smithi136.stdout:2695: writing smithi13667081-10 from 1441792 to 1916928 tid 3 2024-09-12T23:47:41.597 INFO:tasks.rados.rados.0.smithi136.stdout:2695: writing smithi13667081-10 from 1916928 to 2244608 tid 4 2024-09-12T23:47:41.597 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 41 current snap is 272 2024-09-12T23:47:41.598 INFO:tasks.rados.rados.0.smithi136.stdout:2696: seq_num 851 ranges {1695744=655360,2351104=688128,3039232=679936,3719168=237568} 2024-09-12T23:47:41.616 INFO:tasks.rados.rados.0.smithi136.stdout:2696: writing smithi13667081-41 from 1695744 to 2351104 tid 1 2024-09-12T23:47:41.621 INFO:tasks.rados.rados.0.smithi136.stdout:2696: writing smithi13667081-41 from 2351104 to 3039232 tid 2 2024-09-12T23:47:41.625 INFO:tasks.rados.rados.0.smithi136.stdout:2696: writing smithi13667081-41 from 3039232 to 3719168 tid 3 2024-09-12T23:47:41.627 INFO:tasks.rados.rados.0.smithi136.stdout:2696: writing smithi13667081-41 from 3719168 to 3956736 tid 4 2024-09-12T23:47:41.628 INFO:tasks.rados.rados.0.smithi136.stdout:2697: snap_create 2024-09-12T23:47:41.628 INFO:tasks.rados.rados.0.smithi136.stdout:2689: finishing write tid 6 to smithi13667081-8 2024-09-12T23:47:41.628 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 8 v 1259 (ObjNum 848 snap 272 seq_num 848) dirty exists 2024-09-12T23:47:41.628 INFO:tasks.rados.rados.0.smithi136.stdout:2689: left oid 8 (ObjNum 848 snap 272 seq_num 848) 2024-09-12T23:47:41.628 INFO:tasks.rados.rados.0.smithi136.stdout:2692: finishing copy_from to smithi13667081-3 2024-09-12T23:47:41.628 INFO:tasks.rados.rados.0.smithi136.stdout:2692: got expected ENOENT (src dne) 2024-09-12T23:47:41.628 INFO:tasks.rados.rados.0.smithi136.stdout:2692: finishing copy_from racing read to smithi13667081-3 2024-09-12T23:47:41.628 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 23 v 1886 (ObjNum 744 snap 228 seq_num 744) dirty exists 2024-09-12T23:47:41.628 INFO:tasks.rados.rados.0.smithi136.stdout:2694: finishing rollback tid 0 to smithi13667081-11 2024-09-12T23:47:41.628 INFO:tasks.rados.rados.0.smithi136.stdout:2694: finishing rollback tid 1 to smithi13667081-11 2024-09-12T23:47:41.628 INFO:tasks.rados.rados.0.smithi136.stdout:2694: finishing rollback tid 2 to smithi13667081-11 2024-09-12T23:47:41.628 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 11 v 1890 (ObjNum 605 snap 174 seq_num 605) dirty exists 2024-09-12T23:47:41.629 INFO:tasks.rados.rados.0.smithi136.stdout:2695: finishing write tid 1 to smithi13667081-10 2024-09-12T23:47:41.629 INFO:tasks.rados.rados.0.smithi136.stdout:2695: finishing write tid 2 to smithi13667081-10 2024-09-12T23:47:41.629 INFO:tasks.rados.rados.0.smithi136.stdout:2695: finishing write tid 3 to smithi13667081-10 2024-09-12T23:47:41.629 INFO:tasks.rados.rados.0.smithi136.stdout:2695: finishing write tid 4 to smithi13667081-10 2024-09-12T23:47:41.629 INFO:tasks.rados.rados.0.smithi136.stdout:2695: finishing write tid 5 to smithi13667081-10 2024-09-12T23:47:41.629 INFO:tasks.rados.rados.0.smithi136.stdout:2693: expect (ObjNum 727 snap 224 seq_num 727) 2024-09-12T23:47:41.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:41 smithi181 bash[67587]: audit 2024-09-12T23:47:40.549141+0000 mon.a (mon.0) 868 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-12T23:47:41.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:41 smithi181 bash[67587]: cluster 2024-09-12T23:47:40.549350+0000 mon.a (mon.0) 869 : cluster [DBG] osdmap e604: 8 total, 7 up, 8 in 2024-09-12T23:47:41.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:41 smithi181 bash[67587]: audit 2024-09-12T23:47:40.551649+0000 mon.a (mon.0) 870 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-12T23:47:41.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:41 smithi181 bash[67587]: audit 2024-09-12T23:47:40.552432+0000 mon.b (mon.1) 11 : audit [INF] from='osd.7 [v2:172.21.15.181:6824/2237860309,v1:172.21.15.181:6825/2237860309]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-12T23:47:41.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:41 smithi181 bash[67587]: audit 2024-09-12T23:47:40.699067+0000 mgr.y (mgr.24661) 270 : audit [DBG] from='client.34380 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:47:41.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:41 smithi181 bash[67587]: cluster 2024-09-12T23:47:40.781496+0000 mgr.y (mgr.24661) 271 : cluster [DBG] pgmap v482: 121 pgs: 34 active+undersized, 10 active+undersized+degraded, 77 active+clean; 386 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.2 MiB/s wr, 4 op/s; 48/642 objects degraded (7.477%) 2024-09-12T23:47:41.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:41 smithi181 bash[67587]: audit 2024-09-12T23:47:41.114452+0000 mon.a (mon.0) 871 : audit [INF] from='osd.7 ' entity='osd.7' 2024-09-12T23:47:41.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:41 smithi181 bash[67587]: audit 2024-09-12T23:47:41.354965+0000 mon.a (mon.0) 872 : audit [DBG] from='client.? 172.21.15.136:0/3369455411' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:47:42.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:41 smithi136 bash[76371]: audit 2024-09-12T23:47:40.549141+0000 mon.a (mon.0) 868 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-12T23:47:42.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:41 smithi136 bash[76371]: cluster 2024-09-12T23:47:40.549350+0000 mon.a (mon.0) 869 : cluster [DBG] osdmap e604: 8 total, 7 up, 8 in 2024-09-12T23:47:42.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:41 smithi136 bash[76371]: audit 2024-09-12T23:47:40.551649+0000 mon.a (mon.0) 870 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-12T23:47:42.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:41 smithi136 bash[76371]: audit 2024-09-12T23:47:40.552432+0000 mon.b (mon.1) 11 : audit [INF] from='osd.7 [v2:172.21.15.181:6824/2237860309,v1:172.21.15.181:6825/2237860309]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-12T23:47:42.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:41 smithi136 bash[76371]: audit 2024-09-12T23:47:40.699067+0000 mgr.y (mgr.24661) 270 : audit [DBG] from='client.34380 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:47:42.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:41 smithi136 bash[76371]: cluster 2024-09-12T23:47:40.781496+0000 mgr.y (mgr.24661) 271 : cluster [DBG] pgmap v482: 121 pgs: 34 active+undersized, 10 active+undersized+degraded, 77 active+clean; 386 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.2 MiB/s wr, 4 op/s; 48/642 objects degraded (7.477%) 2024-09-12T23:47:42.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:41 smithi136 bash[76371]: audit 2024-09-12T23:47:41.114452+0000 mon.a (mon.0) 871 : audit [INF] from='osd.7 ' entity='osd.7' 2024-09-12T23:47:42.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:41 smithi136 bash[76371]: audit 2024-09-12T23:47:41.354965+0000 mon.a (mon.0) 872 : audit [DBG] from='client.? 172.21.15.136:0/3369455411' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:47:42.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:41 smithi136 bash[75164]: audit 2024-09-12T23:47:40.549141+0000 mon.a (mon.0) 868 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-12T23:47:42.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:41 smithi136 bash[75164]: cluster 2024-09-12T23:47:40.549350+0000 mon.a (mon.0) 869 : cluster [DBG] osdmap e604: 8 total, 7 up, 8 in 2024-09-12T23:47:42.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:41 smithi136 bash[75164]: audit 2024-09-12T23:47:40.551649+0000 mon.a (mon.0) 870 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-12T23:47:42.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:41 smithi136 bash[75164]: audit 2024-09-12T23:47:40.552432+0000 mon.b (mon.1) 11 : audit [INF] from='osd.7 [v2:172.21.15.181:6824/2237860309,v1:172.21.15.181:6825/2237860309]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-12T23:47:42.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:41 smithi136 bash[75164]: audit 2024-09-12T23:47:40.699067+0000 mgr.y (mgr.24661) 270 : audit [DBG] from='client.34380 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:47:42.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:41 smithi136 bash[75164]: cluster 2024-09-12T23:47:40.781496+0000 mgr.y (mgr.24661) 271 : cluster [DBG] pgmap v482: 121 pgs: 34 active+undersized, 10 active+undersized+degraded, 77 active+clean; 386 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.2 MiB/s wr, 4 op/s; 48/642 objects degraded (7.477%) 2024-09-12T23:47:42.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:41 smithi136 bash[75164]: audit 2024-09-12T23:47:41.114452+0000 mon.a (mon.0) 871 : audit [INF] from='osd.7 ' entity='osd.7' 2024-09-12T23:47:42.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:41 smithi136 bash[75164]: audit 2024-09-12T23:47:41.354965+0000 mon.a (mon.0) 872 : audit [DBG] from='client.? 172.21.15.136:0/3369455411' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:47:42.045 INFO:tasks.rados.rados.0.smithi136.stdout:2695: finishing write tid 6 to smithi13667081-10 2024-09-12T23:47:42.045 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 10 v 1488 (ObjNum 850 snap 272 seq_num 850) dirty exists 2024-09-12T23:47:42.045 INFO:tasks.rados.rados.0.smithi136.stdout:2695: left oid 10 (ObjNum 850 snap 272 seq_num 850) 2024-09-12T23:47:42.046 INFO:tasks.rados.rados.0.smithi136.stdout:2696: finishing write tid 1 to smithi13667081-41 2024-09-12T23:47:42.046 INFO:tasks.rados.rados.0.smithi136.stdout:2696: finishing write tid 2 to smithi13667081-41 2024-09-12T23:47:42.046 INFO:tasks.rados.rados.0.smithi136.stdout:2696: finishing write tid 3 to smithi13667081-41 2024-09-12T23:47:42.046 INFO:tasks.rados.rados.0.smithi136.stdout:2696: finishing write tid 4 to smithi13667081-41 2024-09-12T23:47:42.046 INFO:tasks.rados.rados.0.smithi136.stdout:2696: finishing write tid 5 to smithi13667081-41 2024-09-12T23:47:42.046 INFO:tasks.rados.rados.0.smithi136.stdout:2696: finishing write tid 6 to smithi13667081-41 2024-09-12T23:47:42.046 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 41 v 1295 (ObjNum 851 snap 272 seq_num 851) dirty exists 2024-09-12T23:47:42.046 INFO:tasks.rados.rados.0.smithi136.stdout:2696: left oid 41 (ObjNum 851 snap 272 seq_num 851) 2024-09-12T23:47:42.125 INFO:tasks.rados.rados.0.smithi136.stdout:2689: done (7 left) 2024-09-12T23:47:42.125 INFO:tasks.rados.rados.0.smithi136.stdout:2691: done (6 left) 2024-09-12T23:47:42.125 INFO:tasks.rados.rados.0.smithi136.stdout:2692: done (5 left) 2024-09-12T23:47:42.125 INFO:tasks.rados.rados.0.smithi136.stdout:2693: done (4 left) 2024-09-12T23:47:42.125 INFO:tasks.rados.rados.0.smithi136.stdout:2694: done (3 left) 2024-09-12T23:47:42.125 INFO:tasks.rados.rados.0.smithi136.stdout:2695: done (2 left) 2024-09-12T23:47:42.125 INFO:tasks.rados.rados.0.smithi136.stdout:2696: done (1 left) 2024-09-12T23:47:42.125 INFO:tasks.rados.rados.0.smithi136.stdout:2697: done (0 left) 2024-09-12T23:47:42.126 INFO:tasks.rados.rados.0.smithi136.stdout:2698: rollback oid 45 current snap is 273 2024-09-12T23:47:42.126 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 45 to 263 2024-09-12T23:47:42.126 INFO:tasks.rados.rados.0.smithi136.stdout:2699: read oid 14 snap -1 2024-09-12T23:47:42.126 INFO:tasks.rados.rados.0.smithi136.stdout:2699: expect deleted 2024-09-12T23:47:42.126 INFO:tasks.rados.rados.0.smithi136.stdout:2700: read oid 3 snap -1 2024-09-12T23:47:42.126 INFO:tasks.rados.rados.0.smithi136.stdout:2700: expect (ObjNum 838 snap 267 seq_num 838) 2024-09-12T23:47:42.126 INFO:tasks.rados.rados.0.smithi136.stdout:2701: delete oid 25 current snap is 273 2024-09-12T23:47:42.156 INFO:tasks.rados.rados.0.smithi136.stdout:2700: expect (ObjNum 838 snap 267 seq_num 838) 2024-09-12T23:47:42.210 INFO:tasks.rados.rados.0.smithi136.stdout:2698: finishing rollback tid 1 to smithi13667081-45 2024-09-12T23:47:42.210 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 45 v 1233 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-12T23:47:42.594 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:47:42 smithi181 bash[60471]: ts=2024-09-12T23:47:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:47:42.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:42 smithi181 bash[67587]: audit 2024-09-12T23:47:40.988682+0000 mgr.y (mgr.24661) 272 : audit [DBG] from='client.44306 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:47:42.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:42 smithi181 bash[67587]: cluster 2024-09-12T23:47:42.112628+0000 mon.a (mon.0) 873 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-12T23:47:42.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:42 smithi181 bash[67587]: cluster 2024-09-12T23:47:42.122522+0000 mon.a (mon.0) 874 : cluster [INF] osd.7 [v2:172.21.15.181:6824/2237860309,v1:172.21.15.181:6825/2237860309] boot 2024-09-12T23:47:42.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:42 smithi181 bash[67587]: cluster 2024-09-12T23:47:42.122695+0000 mon.a (mon.0) 875 : cluster [DBG] osdmap e605: 8 total, 8 up, 8 in 2024-09-12T23:47:42.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:42 smithi181 bash[67587]: audit 2024-09-12T23:47:42.126583+0000 mon.a (mon.0) 876 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-12T23:47:43.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:42 smithi136 bash[75164]: audit 2024-09-12T23:47:40.988682+0000 mgr.y (mgr.24661) 272 : audit [DBG] from='client.44306 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:47:43.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:42 smithi136 bash[75164]: cluster 2024-09-12T23:47:42.112628+0000 mon.a (mon.0) 873 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-12T23:47:43.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:42 smithi136 bash[75164]: cluster 2024-09-12T23:47:42.122522+0000 mon.a (mon.0) 874 : cluster [INF] osd.7 [v2:172.21.15.181:6824/2237860309,v1:172.21.15.181:6825/2237860309] boot 2024-09-12T23:47:43.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:42 smithi136 bash[75164]: cluster 2024-09-12T23:47:42.122695+0000 mon.a (mon.0) 875 : cluster [DBG] osdmap e605: 8 total, 8 up, 8 in 2024-09-12T23:47:43.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:42 smithi136 bash[75164]: audit 2024-09-12T23:47:42.126583+0000 mon.a (mon.0) 876 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-12T23:47:43.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:42 smithi136 bash[76371]: audit 2024-09-12T23:47:40.988682+0000 mgr.y (mgr.24661) 272 : audit [DBG] from='client.44306 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:47:43.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:42 smithi136 bash[76371]: cluster 2024-09-12T23:47:42.112628+0000 mon.a (mon.0) 873 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-12T23:47:43.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:42 smithi136 bash[76371]: cluster 2024-09-12T23:47:42.122522+0000 mon.a (mon.0) 874 : cluster [INF] osd.7 [v2:172.21.15.181:6824/2237860309,v1:172.21.15.181:6825/2237860309] boot 2024-09-12T23:47:43.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:42 smithi136 bash[76371]: cluster 2024-09-12T23:47:42.122695+0000 mon.a (mon.0) 875 : cluster [DBG] osdmap e605: 8 total, 8 up, 8 in 2024-09-12T23:47:43.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:42 smithi136 bash[76371]: audit 2024-09-12T23:47:42.126583+0000 mon.a (mon.0) 876 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-12T23:47:43.181 INFO:tasks.rados.rados.0.smithi136.stdout:2698: done (3 left) 2024-09-12T23:47:43.181 INFO:tasks.rados.rados.0.smithi136.stdout:2699: done (2 left) 2024-09-12T23:47:43.181 INFO:tasks.rados.rados.0.smithi136.stdout:2700: done (1 left) 2024-09-12T23:47:43.181 INFO:tasks.rados.rados.0.smithi136.stdout:2701: done (0 left) 2024-09-12T23:47:43.181 INFO:tasks.rados.rados.0.smithi136.stdout:2702: delete oid 22 current snap is 273 2024-09-12T23:47:43.183 INFO:tasks.rados.rados.0.smithi136.stdout:2702: done (0 left) 2024-09-12T23:47:43.183 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 21 current snap is 273 2024-09-12T23:47:43.184 INFO:tasks.rados.rados.0.smithi136.stdout:2703: seq_num 854 ranges {0=712704,712704=794624,1507328=450560,1957888=360448} 2024-09-12T23:47:43.188 INFO:tasks.rados.rados.0.smithi136.stdout:2703: writing smithi13667081-21 from 0 to 712704 tid 1 2024-09-12T23:47:43.193 INFO:tasks.rados.rados.0.smithi136.stdout:2703: writing smithi13667081-21 from 712704 to 1507328 tid 2 2024-09-12T23:47:43.196 INFO:tasks.rados.rados.0.smithi136.stdout:2703: writing smithi13667081-21 from 1507328 to 1957888 tid 3 2024-09-12T23:47:43.199 INFO:tasks.rados.rados.0.smithi136.stdout:2703: writing smithi13667081-21 from 1957888 to 2318336 tid 4 2024-09-12T23:47:43.199 INFO:tasks.rados.rados.0.smithi136.stdout:2704: rollback oid 1 current snap is 273 2024-09-12T23:47:43.199 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 1 to 211 2024-09-12T23:47:43.199 INFO:tasks.rados.rados.0.smithi136.stdout:2703: finishing write tid 1 to smithi13667081-21 2024-09-12T23:47:43.199 INFO:tasks.rados.rados.0.smithi136.stdout:2705: snap_create 2024-09-12T23:47:43.199 INFO:tasks.rados.rados.0.smithi136.stdout:2703: finishing write tid 2 to smithi13667081-21 2024-09-12T23:47:43.202 INFO:tasks.rados.rados.0.smithi136.stdout:2703: finishing write tid 3 to smithi13667081-21 2024-09-12T23:47:43.203 INFO:tasks.rados.rados.0.smithi136.stdout:2703: finishing write tid 4 to smithi13667081-21 2024-09-12T23:47:43.203 INFO:tasks.rados.rados.0.smithi136.stdout:2703: finishing write tid 5 to smithi13667081-21 2024-09-12T23:47:43.207 INFO:tasks.rados.rados.0.smithi136.stdout:2703: finishing write tid 6 to smithi13667081-21 2024-09-12T23:47:43.207 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 21 v 1785 (ObjNum 854 snap 273 seq_num 854) dirty exists 2024-09-12T23:47:43.207 INFO:tasks.rados.rados.0.smithi136.stdout:2703: left oid 21 (ObjNum 854 snap 273 seq_num 854) 2024-09-12T23:47:43.209 INFO:tasks.rados.rados.0.smithi136.stdout:2704: finishing rollback tid 0 to smithi13667081-1 2024-09-12T23:47:43.210 INFO:tasks.rados.rados.0.smithi136.stdout:2704: finishing rollback tid 1 to smithi13667081-1 2024-09-12T23:47:43.210 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 1 v 1787 (ObjNum 161 snap 0 seq_num 0) dirty dne 2024-09-12T23:47:43.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:43 smithi181 bash[67587]: cluster 2024-09-12T23:47:42.782469+0000 mgr.y (mgr.24661) 273 : cluster [DBG] pgmap v484: 121 pgs: 6 peering, 28 active+undersized, 10 active+undersized+degraded, 77 active+clean; 390 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 1.5 MiB/s wr, 8 op/s; 50/660 objects degraded (7.576%) 2024-09-12T23:47:43.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:43 smithi181 bash[67587]: cluster 2024-09-12T23:47:43.125999+0000 mon.a (mon.0) 877 : cluster [DBG] osdmap e606: 8 total, 8 up, 8 in 2024-09-12T23:47:44.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:43 smithi136 bash[75164]: cluster 2024-09-12T23:47:42.782469+0000 mgr.y (mgr.24661) 273 : cluster [DBG] pgmap v484: 121 pgs: 6 peering, 28 active+undersized, 10 active+undersized+degraded, 77 active+clean; 390 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 1.5 MiB/s wr, 8 op/s; 50/660 objects degraded (7.576%) 2024-09-12T23:47:44.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:43 smithi136 bash[75164]: cluster 2024-09-12T23:47:43.125999+0000 mon.a (mon.0) 877 : cluster [DBG] osdmap e606: 8 total, 8 up, 8 in 2024-09-12T23:47:44.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:43 smithi136 bash[76371]: cluster 2024-09-12T23:47:42.782469+0000 mgr.y (mgr.24661) 273 : cluster [DBG] pgmap v484: 121 pgs: 6 peering, 28 active+undersized, 10 active+undersized+degraded, 77 active+clean; 390 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 1.5 MiB/s wr, 8 op/s; 50/660 objects degraded (7.576%) 2024-09-12T23:47:44.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:43 smithi136 bash[76371]: cluster 2024-09-12T23:47:43.125999+0000 mon.a (mon.0) 877 : cluster [DBG] osdmap e606: 8 total, 8 up, 8 in 2024-09-12T23:47:44.129 INFO:tasks.rados.rados.0.smithi136.stdout:2703: done (2 left) 2024-09-12T23:47:44.129 INFO:tasks.rados.rados.0.smithi136.stdout:2704: done (1 left) 2024-09-12T23:47:44.129 INFO:tasks.rados.rados.0.smithi136.stdout:2705: done (0 left) 2024-09-12T23:47:44.129 INFO:tasks.rados.rados.0.smithi136.stdout:2706: snap_create 2024-09-12T23:47:44.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:44 smithi181 bash[67587]: cluster 2024-09-12T23:47:43.552822+0000 mon.a (mon.0) 878 : cluster [WRN] Health check update: Degraded data redundancy: 50/660 objects degraded (7.576%), 10 pgs degraded (PG_DEGRADED) 2024-09-12T23:47:44.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:44 smithi181 bash[67587]: cluster 2024-09-12T23:47:44.131993+0000 mon.a (mon.0) 879 : cluster [DBG] osdmap e607: 8 total, 8 up, 8 in 2024-09-12T23:47:45.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:44 smithi136 bash[75164]: cluster 2024-09-12T23:47:43.552822+0000 mon.a (mon.0) 878 : cluster [WRN] Health check update: Degraded data redundancy: 50/660 objects degraded (7.576%), 10 pgs degraded (PG_DEGRADED) 2024-09-12T23:47:45.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:44 smithi136 bash[75164]: cluster 2024-09-12T23:47:44.131993+0000 mon.a (mon.0) 879 : cluster [DBG] osdmap e607: 8 total, 8 up, 8 in 2024-09-12T23:47:45.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:44 smithi136 bash[76371]: cluster 2024-09-12T23:47:43.552822+0000 mon.a (mon.0) 878 : cluster [WRN] Health check update: Degraded data redundancy: 50/660 objects degraded (7.576%), 10 pgs degraded (PG_DEGRADED) 2024-09-12T23:47:45.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:44 smithi136 bash[76371]: cluster 2024-09-12T23:47:44.131993+0000 mon.a (mon.0) 879 : cluster [DBG] osdmap e607: 8 total, 8 up, 8 in 2024-09-12T23:47:45.127 INFO:tasks.rados.rados.0.smithi136.stdout:2706: done (0 left) 2024-09-12T23:47:45.127 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 36 current snap is 275 2024-09-12T23:47:45.127 INFO:tasks.rados.rados.0.smithi136.stdout:2707: seq_num 855 ranges {622592=729088,1351680=671744,2023424=663552,2686976=270336} 2024-09-12T23:47:45.141 INFO:tasks.rados.rados.0.smithi136.stdout:2707: writing smithi13667081-36 from 622592 to 1351680 tid 1 2024-09-12T23:47:45.149 INFO:tasks.rados.rados.0.smithi136.stdout:2707: writing smithi13667081-36 from 1351680 to 2023424 tid 2 2024-09-12T23:47:45.156 INFO:tasks.rados.rados.0.smithi136.stdout:2707: writing smithi13667081-36 from 2023424 to 2686976 tid 3 2024-09-12T23:47:45.159 INFO:tasks.rados.rados.0.smithi136.stdout:2707: writing smithi13667081-36 from 2686976 to 2957312 tid 4 2024-09-12T23:47:45.159 INFO:tasks.rados.rados.0.smithi136.stdout:2708: snap_create 2024-09-12T23:47:45.163 INFO:tasks.rados.rados.0.smithi136.stdout:2707: finishing write tid 1 to smithi13667081-36 2024-09-12T23:47:45.167 INFO:tasks.rados.rados.0.smithi136.stdout:2707: finishing write tid 2 to smithi13667081-36 2024-09-12T23:47:45.167 INFO:tasks.rados.rados.0.smithi136.stdout:2707: finishing write tid 3 to smithi13667081-36 2024-09-12T23:47:45.168 INFO:tasks.rados.rados.0.smithi136.stdout:2707: finishing write tid 4 to smithi13667081-36 2024-09-12T23:47:45.169 INFO:tasks.rados.rados.0.smithi136.stdout:2707: finishing write tid 5 to smithi13667081-36 2024-09-12T23:47:45.170 INFO:tasks.rados.rados.0.smithi136.stdout:2707: finishing write tid 6 to smithi13667081-36 2024-09-12T23:47:45.170 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 36 v 1264 (ObjNum 855 snap 275 seq_num 855) dirty exists 2024-09-12T23:47:45.170 INFO:tasks.rados.rados.0.smithi136.stdout:2707: left oid 36 (ObjNum 855 snap 275 seq_num 855) 2024-09-12T23:47:45.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:45 smithi181 bash[67587]: cluster 2024-09-12T23:47:44.783398+0000 mgr.y (mgr.24661) 274 : cluster [DBG] pgmap v487: 121 pgs: 6 peering, 25 active+undersized, 7 active+undersized+degraded, 83 active+clean; 395 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 2.0 MiB/s wr, 17 op/s; 31/666 objects degraded (4.655%); 9.0 MiB/s, 0 objects/s recovering 2024-09-12T23:47:45.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:45 smithi181 bash[67587]: cluster 2024-09-12T23:47:45.129692+0000 mon.a (mon.0) 880 : cluster [DBG] osdmap e608: 8 total, 8 up, 8 in 2024-09-12T23:47:46.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:45 smithi136 bash[75164]: cluster 2024-09-12T23:47:44.783398+0000 mgr.y (mgr.24661) 274 : cluster [DBG] pgmap v487: 121 pgs: 6 peering, 25 active+undersized, 7 active+undersized+degraded, 83 active+clean; 395 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 2.0 MiB/s wr, 17 op/s; 31/666 objects degraded (4.655%); 9.0 MiB/s, 0 objects/s recovering 2024-09-12T23:47:46.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:45 smithi136 bash[75164]: cluster 2024-09-12T23:47:45.129692+0000 mon.a (mon.0) 880 : cluster [DBG] osdmap e608: 8 total, 8 up, 8 in 2024-09-12T23:47:46.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:45 smithi136 bash[76371]: cluster 2024-09-12T23:47:44.783398+0000 mgr.y (mgr.24661) 274 : cluster [DBG] pgmap v487: 121 pgs: 6 peering, 25 active+undersized, 7 active+undersized+degraded, 83 active+clean; 395 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 2.0 MiB/s wr, 17 op/s; 31/666 objects degraded (4.655%); 9.0 MiB/s, 0 objects/s recovering 2024-09-12T23:47:46.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:45 smithi136 bash[76371]: cluster 2024-09-12T23:47:45.129692+0000 mon.a (mon.0) 880 : cluster [DBG] osdmap e608: 8 total, 8 up, 8 in 2024-09-12T23:47:46.130 INFO:tasks.rados.rados.0.smithi136.stdout:2707: done (1 left) 2024-09-12T23:47:46.130 INFO:tasks.rados.rados.0.smithi136.stdout:2708: done (0 left) 2024-09-12T23:47:46.131 INFO:tasks.rados.rados.0.smithi136.stdout:2709: read oid 5 snap 274 2024-09-12T23:47:46.131 INFO:tasks.rados.rados.0.smithi136.stdout:2709: expect (ObjNum 567 snap 162 seq_num 567) 2024-09-12T23:47:46.131 INFO:tasks.rados.rados.0.smithi136.stdout:2710: setattr oid 9 current snap is 276 2024-09-12T23:47:46.133 INFO:tasks.rados.rados.0.smithi136.stdout:2711: rmattr oid 8 current snap is 276 2024-09-12T23:47:46.133 INFO:tasks.rados.rados.0.smithi136.stdout:2712: rmattr oid 13 current snap is 276 2024-09-12T23:47:46.133 INFO:tasks.rados.rados.0.smithi136.stdout:2712: done (3 left) 2024-09-12T23:47:46.133 INFO:tasks.rados.rados.0.smithi136.stdout:2713: snap_create 2024-09-12T23:47:46.139 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 8 v 1265 (ObjNum 848 snap 272 seq_num 848) dirty exists 2024-09-12T23:47:46.150 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 9 v 1612 (ObjNum 33 snap 0 seq_num 4244146944) dirty exists 2024-09-12T23:47:46.169 INFO:tasks.rados.rados.0.smithi136.stdout:2709: expect (ObjNum 567 snap 162 seq_num 567) 2024-09-12T23:47:47.141 INFO:tasks.rados.rados.0.smithi136.stdout:2709: done (3 left) 2024-09-12T23:47:47.141 INFO:tasks.rados.rados.0.smithi136.stdout:2710: done (2 left) 2024-09-12T23:47:47.141 INFO:tasks.rados.rados.0.smithi136.stdout:2711: done (1 left) 2024-09-12T23:47:47.141 INFO:tasks.rados.rados.0.smithi136.stdout:2713: done (0 left) 2024-09-12T23:47:47.141 INFO:tasks.rados.rados.0.smithi136.stdout:2714: read oid 19 snap -1 2024-09-12T23:47:47.141 INFO:tasks.rados.rados.0.smithi136.stdout:2714: expect deleted 2024-09-12T23:47:47.141 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 41 current snap is 277 2024-09-12T23:47:47.142 INFO:tasks.rados.rados.0.smithi136.stdout:2715: seq_num 856 ranges {3956736=745472,4702208=557056,5259264=417792,5677056=434176,6111232=196608} 2024-09-12T23:47:47.205 INFO:tasks.rados.rados.0.smithi136.stdout:2715: writing smithi13667081-41 from 3956736 to 4702208 tid 1 2024-09-12T23:47:47.210 INFO:tasks.rados.rados.0.smithi136.stdout:2715: writing smithi13667081-41 from 4702208 to 5259264 tid 2 2024-09-12T23:47:47.215 INFO:tasks.rados.rados.0.smithi136.stdout:2715: writing smithi13667081-41 from 5259264 to 5677056 tid 3 2024-09-12T23:47:47.218 INFO:tasks.rados.rados.0.smithi136.stdout:2715: writing smithi13667081-41 from 5677056 to 6111232 tid 4 2024-09-12T23:47:47.220 INFO:tasks.rados.rados.0.smithi136.stdout:2715: writing smithi13667081-41 from 6111232 to 6307840 tid 5 2024-09-12T23:47:47.220 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 7 current snap is 277 2024-09-12T23:47:47.220 INFO:tasks.rados.rados.0.smithi136.stdout:2716: seq_num 857 ranges {0=761856,761856=442368,1204224=630784,1835008=532480} 2024-09-12T23:47:47.226 INFO:tasks.rados.rados.0.smithi136.stdout:2716: writing smithi13667081-7 from 0 to 761856 tid 1 2024-09-12T23:47:47.229 INFO:tasks.rados.rados.0.smithi136.stdout:2716: writing smithi13667081-7 from 761856 to 1204224 tid 2 2024-09-12T23:47:47.234 INFO:tasks.rados.rados.0.smithi136.stdout:2716: writing smithi13667081-7 from 1204224 to 1835008 tid 3 2024-09-12T23:47:47.238 INFO:tasks.rados.rados.0.smithi136.stdout:2716: writing smithi13667081-7 from 1835008 to 2367488 tid 4 2024-09-12T23:47:47.238 INFO:tasks.rados.rados.0.smithi136.stdout:2717: snap_create 2024-09-12T23:47:47.238 INFO:tasks.rados.rados.0.smithi136.stdout:2715: finishing write tid 1 to smithi13667081-41 2024-09-12T23:47:47.238 INFO:tasks.rados.rados.0.smithi136.stdout:2715: finishing write tid 2 to smithi13667081-41 2024-09-12T23:47:47.238 INFO:tasks.rados.rados.0.smithi136.stdout:2715: finishing write tid 3 to smithi13667081-41 2024-09-12T23:47:47.238 INFO:tasks.rados.rados.0.smithi136.stdout:2715: finishing write tid 4 to smithi13667081-41 2024-09-12T23:47:47.238 INFO:tasks.rados.rados.0.smithi136.stdout:2715: finishing write tid 5 to smithi13667081-41 2024-09-12T23:47:47.238 INFO:tasks.rados.rados.0.smithi136.stdout:2715: finishing write tid 6 to smithi13667081-41 2024-09-12T23:47:47.239 INFO:tasks.rados.rados.0.smithi136.stdout:2715: finishing write tid 7 to smithi13667081-41 2024-09-12T23:47:47.239 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 41 v 1301 (ObjNum 856 snap 277 seq_num 856) dirty exists 2024-09-12T23:47:47.239 INFO:tasks.rados.rados.0.smithi136.stdout:2715: left oid 41 (ObjNum 856 snap 277 seq_num 856) 2024-09-12T23:47:47.244 INFO:tasks.rados.rados.0.smithi136.stdout:2716: finishing write tid 1 to smithi13667081-7 2024-09-12T23:47:47.244 INFO:tasks.rados.rados.0.smithi136.stdout:2716: finishing write tid 2 to smithi13667081-7 2024-09-12T23:47:47.248 INFO:tasks.rados.rados.0.smithi136.stdout:2716: finishing write tid 3 to smithi13667081-7 2024-09-12T23:47:47.250 INFO:tasks.rados.rados.0.smithi136.stdout:2716: finishing write tid 4 to smithi13667081-7 2024-09-12T23:47:47.251 INFO:tasks.rados.rados.0.smithi136.stdout:2716: finishing write tid 5 to smithi13667081-7 2024-09-12T23:47:47.253 INFO:tasks.rados.rados.0.smithi136.stdout:2716: finishing write tid 6 to smithi13667081-7 2024-09-12T23:47:47.253 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 7 v 964 (ObjNum 857 snap 277 seq_num 857) dirty exists 2024-09-12T23:47:47.253 INFO:tasks.rados.rados.0.smithi136.stdout:2716: left oid 7 (ObjNum 857 snap 277 seq_num 857) 2024-09-12T23:47:47.344 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:47 smithi181 bash[67587]: cluster 2024-09-12T23:47:46.133108+0000 mon.a (mon.0) 881 : cluster [DBG] osdmap e609: 8 total, 8 up, 8 in 2024-09-12T23:47:47.344 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:47 smithi181 bash[67587]: cluster 2024-09-12T23:47:46.784381+0000 mgr.y (mgr.24661) 275 : cluster [DBG] pgmap v490: 121 pgs: 6 peering, 115 active+clean; 397 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 1.6 MiB/s wr, 24 op/s; 11 MiB/s, 1 objects/s recovering 2024-09-12T23:47:47.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:47 smithi136 bash[75164]: cluster 2024-09-12T23:47:46.133108+0000 mon.a (mon.0) 881 : cluster [DBG] osdmap e609: 8 total, 8 up, 8 in 2024-09-12T23:47:47.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:47 smithi136 bash[75164]: cluster 2024-09-12T23:47:46.784381+0000 mgr.y (mgr.24661) 275 : cluster [DBG] pgmap v490: 121 pgs: 6 peering, 115 active+clean; 397 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 1.6 MiB/s wr, 24 op/s; 11 MiB/s, 1 objects/s recovering 2024-09-12T23:47:47.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:47 smithi136 bash[76371]: cluster 2024-09-12T23:47:46.133108+0000 mon.a (mon.0) 881 : cluster [DBG] osdmap e609: 8 total, 8 up, 8 in 2024-09-12T23:47:47.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:47 smithi136 bash[76371]: cluster 2024-09-12T23:47:46.784381+0000 mgr.y (mgr.24661) 275 : cluster [DBG] pgmap v490: 121 pgs: 6 peering, 115 active+clean; 397 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 1.6 MiB/s wr, 24 op/s; 11 MiB/s, 1 objects/s recovering 2024-09-12T23:47:48.151 INFO:tasks.rados.rados.0.smithi136.stdout:2714: done (3 left) 2024-09-12T23:47:48.151 INFO:tasks.rados.rados.0.smithi136.stdout:2715: done (2 left) 2024-09-12T23:47:48.151 INFO:tasks.rados.rados.0.smithi136.stdout:2716: done (1 left) 2024-09-12T23:47:48.151 INFO:tasks.rados.rados.0.smithi136.stdout:2717: done (0 left) 2024-09-12T23:47:48.151 INFO:tasks.rados.rados.0.smithi136.stdout:2718: read oid 29 snap -1 2024-09-12T23:47:48.151 INFO:tasks.rados.rados.0.smithi136.stdout:2718: expect (ObjNum 844 snap 272 seq_num 844) 2024-09-12T23:47:48.152 INFO:tasks.rados.rados.0.smithi136.stdout:2719: read oid 46 snap -1 2024-09-12T23:47:48.152 INFO:tasks.rados.rados.0.smithi136.stdout:2719: expect (ObjNum 822 snap 258 seq_num 822) 2024-09-12T23:47:48.152 INFO:tasks.rados.rados.0.smithi136.stdout:2720: read oid 15 snap -1 2024-09-12T23:47:48.152 INFO:tasks.rados.rados.0.smithi136.stdout:2720: expect deleted 2024-09-12T23:47:48.152 INFO:tasks.rados.rados.0.smithi136.stdout:2721: snap_remove snap 251 2024-09-12T23:47:48.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:48 smithi136 bash[75164]: cluster 2024-09-12T23:47:47.125465+0000 mon.a (mon.0) 882 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 31/666 objects degraded (4.655%), 7 pgs degraded) 2024-09-12T23:47:48.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:48 smithi136 bash[75164]: cluster 2024-09-12T23:47:47.125509+0000 mon.a (mon.0) 883 : cluster [INF] Cluster is now healthy 2024-09-12T23:47:48.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:48 smithi136 bash[75164]: cluster 2024-09-12T23:47:47.142839+0000 mon.a (mon.0) 884 : cluster [DBG] osdmap e610: 8 total, 8 up, 8 in 2024-09-12T23:47:48.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:48 smithi136 bash[76371]: cluster 2024-09-12T23:47:47.125465+0000 mon.a (mon.0) 882 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 31/666 objects degraded (4.655%), 7 pgs degraded) 2024-09-12T23:47:48.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:48 smithi136 bash[76371]: cluster 2024-09-12T23:47:47.125509+0000 mon.a (mon.0) 883 : cluster [INF] Cluster is now healthy 2024-09-12T23:47:48.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:48 smithi136 bash[76371]: cluster 2024-09-12T23:47:47.142839+0000 mon.a (mon.0) 884 : cluster [DBG] osdmap e610: 8 total, 8 up, 8 in 2024-09-12T23:47:48.593 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:48 smithi181 bash[67587]: cluster 2024-09-12T23:47:47.125465+0000 mon.a (mon.0) 882 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 31/666 objects degraded (4.655%), 7 pgs degraded) 2024-09-12T23:47:48.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:48 smithi181 bash[67587]: cluster 2024-09-12T23:47:47.125509+0000 mon.a (mon.0) 883 : cluster [INF] Cluster is now healthy 2024-09-12T23:47:48.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:48 smithi181 bash[67587]: cluster 2024-09-12T23:47:47.142839+0000 mon.a (mon.0) 884 : cluster [DBG] osdmap e610: 8 total, 8 up, 8 in 2024-09-12T23:47:49.163 INFO:tasks.rados.rados.0.smithi136.stdout:2721: done (3 left) 2024-09-12T23:47:49.163 INFO:tasks.rados.rados.0.smithi136.stdout:2722: delete oid 16 current snap is 278 2024-09-12T23:47:49.163 INFO:tasks.rados.rados.0.smithi136.stdout:2719: expect (ObjNum 822 snap 258 seq_num 822) 2024-09-12T23:47:49.227 INFO:tasks.rados.rados.0.smithi136.stdout:2718: expect (ObjNum 844 snap 272 seq_num 844) 2024-09-12T23:47:49.506 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:47:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:47:49] "GET /metrics HTTP/1.1" 200 36383 "" "Prometheus/2.43.0" 2024-09-12T23:47:49.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:49 smithi136 bash[75164]: cluster 2024-09-12T23:47:48.153504+0000 mon.a (mon.0) 885 : cluster [DBG] osdmap e611: 8 total, 8 up, 8 in 2024-09-12T23:47:49.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:49 smithi136 bash[75164]: cluster 2024-09-12T23:47:48.785247+0000 mgr.y (mgr.24661) 276 : cluster [DBG] pgmap v493: 121 pgs: 121 active+clean; 399 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 1.1 MiB/s wr, 17 op/s; 1.7 MiB/s, 1 objects/s recovering 2024-09-12T23:47:49.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:49 smithi136 bash[76371]: cluster 2024-09-12T23:47:48.153504+0000 mon.a (mon.0) 885 : cluster [DBG] osdmap e611: 8 total, 8 up, 8 in 2024-09-12T23:47:49.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:49 smithi136 bash[76371]: cluster 2024-09-12T23:47:48.785247+0000 mgr.y (mgr.24661) 276 : cluster [DBG] pgmap v493: 121 pgs: 121 active+clean; 399 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 1.1 MiB/s wr, 17 op/s; 1.7 MiB/s, 1 objects/s recovering 2024-09-12T23:47:49.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:49 smithi181 bash[67587]: cluster 2024-09-12T23:47:48.153504+0000 mon.a (mon.0) 885 : cluster [DBG] osdmap e611: 8 total, 8 up, 8 in 2024-09-12T23:47:49.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:49 smithi181 bash[67587]: cluster 2024-09-12T23:47:48.785247+0000 mgr.y (mgr.24661) 276 : cluster [DBG] pgmap v493: 121 pgs: 121 active+clean; 399 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 1.1 MiB/s wr, 17 op/s; 1.7 MiB/s, 1 objects/s recovering 2024-09-12T23:47:50.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[75164]: cluster 2024-09-12T23:47:49.165422+0000 mon.a (mon.0) 886 : cluster [DBG] osdmap e612: 8 total, 8 up, 8 in 2024-09-12T23:47:50.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[75164]: audit 2024-09-12T23:47:49.480424+0000 mon.a (mon.0) 887 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:50.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[75164]: audit 2024-09-12T23:47:49.489769+0000 mon.a (mon.0) 888 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:50.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[75164]: audit 2024-09-12T23:47:49.491937+0000 mon.a (mon.0) 889 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:47:50.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[75164]: audit 2024-09-12T23:47:49.493448+0000 mon.a (mon.0) 890 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:47:50.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[75164]: audit 2024-09-12T23:47:49.502786+0000 mon.a (mon.0) 891 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:50.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[75164]: audit 2024-09-12T23:47:49.611838+0000 mon.a (mon.0) 892 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:47:50.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[75164]: audit 2024-09-12T23:47:49.614978+0000 mon.a (mon.0) 893 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:47:50.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[75164]: audit 2024-09-12T23:47:49.616762+0000 mon.a (mon.0) 894 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:47:50.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[75164]: audit 2024-09-12T23:47:49.618240+0000 mon.a (mon.0) 895 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:47:50.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[75164]: audit 2024-09-12T23:47:49.620438+0000 mon.a (mon.0) 896 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:47:50.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[75164]: cephadm 2024-09-12T23:47:49.621270+0000 mgr.y (mgr.24661) 277 : cephadm [INF] Upgrade: Setting container_image for all osd 2024-09-12T23:47:50.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[75164]: audit 2024-09-12T23:47:49.629798+0000 mon.a (mon.0) 897 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:50.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[75164]: audit 2024-09-12T23:47:49.631004+0000 mon.a (mon.0) 898 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-09-12T23:47:50.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[75164]: audit 2024-09-12T23:47:49.638272+0000 mon.a (mon.0) 899 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-12T23:47:50.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[75164]: audit 2024-09-12T23:47:49.639199+0000 mon.a (mon.0) 900 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-09-12T23:47:50.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[75164]: audit 2024-09-12T23:47:49.646321+0000 mon.a (mon.0) 901 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-12T23:47:50.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[75164]: audit 2024-09-12T23:47:49.647365+0000 mon.a (mon.0) 902 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-09-12T23:47:50.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[75164]: audit 2024-09-12T23:47:49.654432+0000 mon.a (mon.0) 903 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-12T23:47:50.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[75164]: audit 2024-09-12T23:47:49.655404+0000 mon.a (mon.0) 904 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-09-12T23:47:50.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[75164]: audit 2024-09-12T23:47:49.662518+0000 mon.a (mon.0) 905 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-12T23:47:50.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[75164]: audit 2024-09-12T23:47:49.663455+0000 mon.a (mon.0) 906 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-09-12T23:47:50.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[75164]: audit 2024-09-12T23:47:49.667211+0000 mon.a (mon.0) 907 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-12T23:47:50.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[75164]: audit 2024-09-12T23:47:49.668451+0000 mon.a (mon.0) 908 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-09-12T23:47:50.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[75164]: audit 2024-09-12T23:47:49.675067+0000 mon.a (mon.0) 909 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-12T23:47:50.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[75164]: audit 2024-09-12T23:47:49.676186+0000 mon.a (mon.0) 910 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-09-12T23:47:50.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[75164]: audit 2024-09-12T23:47:49.681342+0000 mon.a (mon.0) 911 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-12T23:47:50.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[75164]: audit 2024-09-12T23:47:49.682510+0000 mon.a (mon.0) 912 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-09-12T23:47:50.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[75164]: audit 2024-09-12T23:47:49.688780+0000 mon.a (mon.0) 913 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-12T23:47:50.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[75164]: cephadm 2024-09-12T23:47:49.691809+0000 mgr.y (mgr.24661) 278 : cephadm [INF] Upgrade: Setting require_osd_release to 18 reef 2024-09-12T23:47:50.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[75164]: audit 2024-09-12T23:47:49.692107+0000 mon.a (mon.0) 914 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "reef"}]: dispatch 2024-09-12T23:47:50.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[76371]: cluster 2024-09-12T23:47:49.165422+0000 mon.a (mon.0) 886 : cluster [DBG] osdmap e612: 8 total, 8 up, 8 in 2024-09-12T23:47:50.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[76371]: audit 2024-09-12T23:47:49.480424+0000 mon.a (mon.0) 887 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:50.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[76371]: audit 2024-09-12T23:47:49.489769+0000 mon.a (mon.0) 888 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:50.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[76371]: audit 2024-09-12T23:47:49.491937+0000 mon.a (mon.0) 889 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:47:50.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[76371]: audit 2024-09-12T23:47:49.493448+0000 mon.a (mon.0) 890 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:47:50.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[76371]: audit 2024-09-12T23:47:49.502786+0000 mon.a (mon.0) 891 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:50.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[76371]: audit 2024-09-12T23:47:49.611838+0000 mon.a (mon.0) 892 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:47:50.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[76371]: audit 2024-09-12T23:47:49.614978+0000 mon.a (mon.0) 893 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:47:50.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[76371]: audit 2024-09-12T23:47:49.616762+0000 mon.a (mon.0) 894 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:47:50.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[76371]: audit 2024-09-12T23:47:49.618240+0000 mon.a (mon.0) 895 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:47:50.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[76371]: audit 2024-09-12T23:47:49.620438+0000 mon.a (mon.0) 896 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:47:50.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[76371]: cephadm 2024-09-12T23:47:49.621270+0000 mgr.y (mgr.24661) 277 : cephadm [INF] Upgrade: Setting container_image for all osd 2024-09-12T23:47:50.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[76371]: audit 2024-09-12T23:47:49.629798+0000 mon.a (mon.0) 897 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:50.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[76371]: audit 2024-09-12T23:47:49.631004+0000 mon.a (mon.0) 898 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-09-12T23:47:50.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[76371]: audit 2024-09-12T23:47:49.638272+0000 mon.a (mon.0) 899 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-12T23:47:50.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[76371]: audit 2024-09-12T23:47:49.639199+0000 mon.a (mon.0) 900 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-09-12T23:47:50.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[76371]: audit 2024-09-12T23:47:49.646321+0000 mon.a (mon.0) 901 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-12T23:47:50.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[76371]: audit 2024-09-12T23:47:49.647365+0000 mon.a (mon.0) 902 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-09-12T23:47:50.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[76371]: audit 2024-09-12T23:47:49.654432+0000 mon.a (mon.0) 903 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-12T23:47:50.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[76371]: audit 2024-09-12T23:47:49.655404+0000 mon.a (mon.0) 904 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-09-12T23:47:50.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[76371]: audit 2024-09-12T23:47:49.662518+0000 mon.a (mon.0) 905 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-12T23:47:50.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[76371]: audit 2024-09-12T23:47:49.663455+0000 mon.a (mon.0) 906 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-09-12T23:47:50.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[76371]: audit 2024-09-12T23:47:49.667211+0000 mon.a (mon.0) 907 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-12T23:47:50.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[76371]: audit 2024-09-12T23:47:49.668451+0000 mon.a (mon.0) 908 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-09-12T23:47:50.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[76371]: audit 2024-09-12T23:47:49.675067+0000 mon.a (mon.0) 909 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-12T23:47:50.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[76371]: audit 2024-09-12T23:47:49.676186+0000 mon.a (mon.0) 910 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-09-12T23:47:50.512 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[76371]: audit 2024-09-12T23:47:49.681342+0000 mon.a (mon.0) 911 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-12T23:47:50.512 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[76371]: audit 2024-09-12T23:47:49.682510+0000 mon.a (mon.0) 912 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-09-12T23:47:50.512 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[76371]: audit 2024-09-12T23:47:49.688780+0000 mon.a (mon.0) 913 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-12T23:47:50.512 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[76371]: cephadm 2024-09-12T23:47:49.691809+0000 mgr.y (mgr.24661) 278 : cephadm [INF] Upgrade: Setting require_osd_release to 18 reef 2024-09-12T23:47:50.512 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:50 smithi136 bash[76371]: audit 2024-09-12T23:47:49.692107+0000 mon.a (mon.0) 914 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "reef"}]: dispatch 2024-09-12T23:47:50.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:50 smithi181 bash[67587]: cluster 2024-09-12T23:47:49.165422+0000 mon.a (mon.0) 886 : cluster [DBG] osdmap e612: 8 total, 8 up, 8 in 2024-09-12T23:47:50.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:50 smithi181 bash[67587]: audit 2024-09-12T23:47:49.480424+0000 mon.a (mon.0) 887 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:50.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:50 smithi181 bash[67587]: audit 2024-09-12T23:47:49.489769+0000 mon.a (mon.0) 888 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:50.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:50 smithi181 bash[67587]: audit 2024-09-12T23:47:49.491937+0000 mon.a (mon.0) 889 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:47:50.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:50 smithi181 bash[67587]: audit 2024-09-12T23:47:49.493448+0000 mon.a (mon.0) 890 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:47:50.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:50 smithi181 bash[67587]: audit 2024-09-12T23:47:49.502786+0000 mon.a (mon.0) 891 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:50.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:50 smithi181 bash[67587]: audit 2024-09-12T23:47:49.611838+0000 mon.a (mon.0) 892 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:47:50.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:50 smithi181 bash[67587]: audit 2024-09-12T23:47:49.614978+0000 mon.a (mon.0) 893 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:47:50.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:50 smithi181 bash[67587]: audit 2024-09-12T23:47:49.616762+0000 mon.a (mon.0) 894 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:47:50.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:50 smithi181 bash[67587]: audit 2024-09-12T23:47:49.618240+0000 mon.a (mon.0) 895 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:47:50.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:50 smithi181 bash[67587]: audit 2024-09-12T23:47:49.620438+0000 mon.a (mon.0) 896 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:47:50.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:50 smithi181 bash[67587]: cephadm 2024-09-12T23:47:49.621270+0000 mgr.y (mgr.24661) 277 : cephadm [INF] Upgrade: Setting container_image for all osd 2024-09-12T23:47:50.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:50 smithi181 bash[67587]: audit 2024-09-12T23:47:49.629798+0000 mon.a (mon.0) 897 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:50.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:50 smithi181 bash[67587]: audit 2024-09-12T23:47:49.631004+0000 mon.a (mon.0) 898 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-09-12T23:47:50.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:50 smithi181 bash[67587]: audit 2024-09-12T23:47:49.638272+0000 mon.a (mon.0) 899 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-12T23:47:50.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:50 smithi181 bash[67587]: audit 2024-09-12T23:47:49.639199+0000 mon.a (mon.0) 900 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-09-12T23:47:50.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:50 smithi181 bash[67587]: audit 2024-09-12T23:47:49.646321+0000 mon.a (mon.0) 901 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-12T23:47:50.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:50 smithi181 bash[67587]: audit 2024-09-12T23:47:49.647365+0000 mon.a (mon.0) 902 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-09-12T23:47:50.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:50 smithi181 bash[67587]: audit 2024-09-12T23:47:49.654432+0000 mon.a (mon.0) 903 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-12T23:47:50.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:50 smithi181 bash[67587]: audit 2024-09-12T23:47:49.655404+0000 mon.a (mon.0) 904 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-09-12T23:47:50.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:50 smithi181 bash[67587]: audit 2024-09-12T23:47:49.662518+0000 mon.a (mon.0) 905 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-12T23:47:50.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:50 smithi181 bash[67587]: audit 2024-09-12T23:47:49.663455+0000 mon.a (mon.0) 906 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-09-12T23:47:50.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:50 smithi181 bash[67587]: audit 2024-09-12T23:47:49.667211+0000 mon.a (mon.0) 907 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-12T23:47:50.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:50 smithi181 bash[67587]: audit 2024-09-12T23:47:49.668451+0000 mon.a (mon.0) 908 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-09-12T23:47:50.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:50 smithi181 bash[67587]: audit 2024-09-12T23:47:49.675067+0000 mon.a (mon.0) 909 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-12T23:47:50.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:50 smithi181 bash[67587]: audit 2024-09-12T23:47:49.676186+0000 mon.a (mon.0) 910 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-09-12T23:47:50.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:50 smithi181 bash[67587]: audit 2024-09-12T23:47:49.681342+0000 mon.a (mon.0) 911 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-12T23:47:50.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:50 smithi181 bash[67587]: audit 2024-09-12T23:47:49.682510+0000 mon.a (mon.0) 912 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-09-12T23:47:50.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:50 smithi181 bash[67587]: audit 2024-09-12T23:47:49.688780+0000 mon.a (mon.0) 913 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-12T23:47:50.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:50 smithi181 bash[67587]: cephadm 2024-09-12T23:47:49.691809+0000 mgr.y (mgr.24661) 278 : cephadm [INF] Upgrade: Setting require_osd_release to 18 reef 2024-09-12T23:47:50.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:50 smithi181 bash[67587]: audit 2024-09-12T23:47:49.692107+0000 mon.a (mon.0) 914 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "reef"}]: dispatch 2024-09-12T23:47:50.736 INFO:tasks.rados.rados.0.smithi136.stdout:2718: done (3 left) 2024-09-12T23:47:50.739 INFO:tasks.rados.rados.0.smithi136.stdout:2719: done (2 left) 2024-09-12T23:47:50.739 INFO:tasks.rados.rados.0.smithi136.stdout:2720: done (1 left) 2024-09-12T23:47:50.739 INFO:tasks.rados.rados.0.smithi136.stdout:2722: done (0 left) 2024-09-12T23:47:50.739 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 29 current snap is 278 2024-09-12T23:47:50.739 INFO:tasks.rados.rados.0.smithi136.stdout:2723: seq_num 859 ranges {14786560=794624,15581184=614400,16195584=638976,16834560=352256} 2024-09-12T23:47:50.859 INFO:tasks.rados.rados.0.smithi136.stdout:2723: writing smithi13667081-29 from 14786560 to 15581184 tid 1 2024-09-12T23:47:50.863 INFO:tasks.rados.rados.0.smithi136.stdout:2723: writing smithi13667081-29 from 15581184 to 16195584 tid 2 2024-09-12T23:47:50.867 INFO:tasks.rados.rados.0.smithi136.stdout:2723: writing smithi13667081-29 from 16195584 to 16834560 tid 3 2024-09-12T23:47:50.870 INFO:tasks.rados.rados.0.smithi136.stdout:2723: writing smithi13667081-29 from 16834560 to 17186816 tid 4 2024-09-12T23:47:50.870 INFO:tasks.rados.rados.0.smithi136.stdout:2724: copy_from oid 21 from oid 20 current snap is 278 2024-09-12T23:47:50.870 INFO:tasks.rados.rados.0.smithi136.stdout:2725: snap_create 2024-09-12T23:47:50.871 INFO:tasks.rados.rados.0.smithi136.stdout:2724: finishing copy_from to smithi13667081-21 2024-09-12T23:47:50.872 INFO:tasks.rados.rados.0.smithi136.stdout:2724: got expected ENOENT (src dne) 2024-09-12T23:47:50.872 INFO:tasks.rados.rados.0.smithi136.stdout:2724: finishing copy_from racing read to smithi13667081-21 2024-09-12T23:47:50.879 INFO:tasks.rados.rados.0.smithi136.stdout:2723: finishing write tid 1 to smithi13667081-29 2024-09-12T23:47:50.879 INFO:tasks.rados.rados.0.smithi136.stdout:2723: finishing write tid 2 to smithi13667081-29 2024-09-12T23:47:50.880 INFO:tasks.rados.rados.0.smithi136.stdout:2723: finishing write tid 3 to smithi13667081-29 2024-09-12T23:47:50.881 INFO:tasks.rados.rados.0.smithi136.stdout:2723: finishing write tid 4 to smithi13667081-29 2024-09-12T23:47:50.887 INFO:tasks.rados.rados.0.smithi136.stdout:2723: finishing write tid 5 to smithi13667081-29 2024-09-12T23:47:50.890 INFO:tasks.rados.rados.0.smithi136.stdout:2723: finishing write tid 6 to smithi13667081-29 2024-09-12T23:47:50.890 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 29 v 1635 (ObjNum 859 snap 278 seq_num 859) dirty exists 2024-09-12T23:47:50.890 INFO:tasks.rados.rados.0.smithi136.stdout:2723: left oid 29 (ObjNum 859 snap 278 seq_num 859) 2024-09-12T23:47:51.413 INFO:tasks.rados.rados.0.smithi136.stdout:2723: done (2 left) 2024-09-12T23:47:51.413 INFO:tasks.rados.rados.0.smithi136.stdout:2724: done (1 left) 2024-09-12T23:47:51.413 INFO:tasks.rados.rados.0.smithi136.stdout:2725: done (0 left) 2024-09-12T23:47:51.413 INFO:tasks.rados.rados.0.smithi136.stdout:2726: setattr oid 10 current snap is 279 2024-09-12T23:47:51.415 INFO:tasks.rados.rados.0.smithi136.stdout:2727: copy_from oid 36 from oid 20 current snap is 279 2024-09-12T23:47:51.415 INFO:tasks.rados.rados.0.smithi136.stdout:2728: copy_from oid 12 from oid 48 current snap is 279 2024-09-12T23:47:51.415 INFO:tasks.rados.rados.0.smithi136.stdout:2729: snap_create 2024-09-12T23:47:51.422 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 10 v 1489 (ObjNum 850 snap 272 seq_num 850) dirty exists 2024-09-12T23:47:51.429 INFO:tasks.rados.rados.0.smithi136.stdout:2727: finishing copy_from to smithi13667081-36 2024-09-12T23:47:51.429 INFO:tasks.rados.rados.0.smithi136.stdout:2727: got expected ENOENT (src dne) 2024-09-12T23:47:51.430 INFO:tasks.rados.rados.0.smithi136.stdout:2727: finishing copy_from racing read to smithi13667081-36 2024-09-12T23:47:51.444 INFO:tasks.rados.rados.0.smithi136.stdout:2728: finishing copy_from racing read to smithi13667081-12 2024-09-12T23:47:51.445 INFO:tasks.rados.rados.0.smithi136.stdout:2728: finishing copy_from to smithi13667081-12 2024-09-12T23:47:51.445 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 12 v 1587 (ObjNum 764 snap 236 seq_num 764) dirty exists 2024-09-12T23:47:52.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:51 smithi136 bash[75164]: audit 2024-09-12T23:47:50.693912+0000 mon.a (mon.0) 915 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "reef"}]': finished 2024-09-12T23:47:52.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:51 smithi136 bash[75164]: cluster 2024-09-12T23:47:50.694319+0000 mon.a (mon.0) 916 : cluster [DBG] osdmap e613: 8 total, 8 up, 8 in 2024-09-12T23:47:52.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:51 smithi136 bash[75164]: audit 2024-09-12T23:47:50.699036+0000 mon.a (mon.0) 917 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mds ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-12T23:47:52.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:51 smithi136 bash[75164]: cephadm 2024-09-12T23:47:50.700405+0000 mgr.y (mgr.24661) 279 : cephadm [INF] Upgrade: It appears safe to stop mds.a 2024-09-12T23:47:52.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:51 smithi136 bash[75164]: cluster 2024-09-12T23:47:50.785888+0000 mgr.y (mgr.24661) 280 : cluster [DBG] pgmap v496: 121 pgs: 121 active+clean; 401 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 1.1 MiB/s wr, 7 op/s 2024-09-12T23:47:52.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:51 smithi136 bash[75164]: cluster 2024-09-12T23:47:51.414903+0000 mon.a (mon.0) 918 : cluster [DBG] osdmap e614: 8 total, 8 up, 8 in 2024-09-12T23:47:52.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:51 smithi136 bash[76371]: audit 2024-09-12T23:47:50.693912+0000 mon.a (mon.0) 915 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "reef"}]': finished 2024-09-12T23:47:52.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:51 smithi136 bash[76371]: cluster 2024-09-12T23:47:50.694319+0000 mon.a (mon.0) 916 : cluster [DBG] osdmap e613: 8 total, 8 up, 8 in 2024-09-12T23:47:52.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:51 smithi136 bash[76371]: audit 2024-09-12T23:47:50.699036+0000 mon.a (mon.0) 917 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mds ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-12T23:47:52.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:51 smithi136 bash[76371]: cephadm 2024-09-12T23:47:50.700405+0000 mgr.y (mgr.24661) 279 : cephadm [INF] Upgrade: It appears safe to stop mds.a 2024-09-12T23:47:52.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:51 smithi136 bash[76371]: cluster 2024-09-12T23:47:50.785888+0000 mgr.y (mgr.24661) 280 : cluster [DBG] pgmap v496: 121 pgs: 121 active+clean; 401 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 1.1 MiB/s wr, 7 op/s 2024-09-12T23:47:52.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:51 smithi136 bash[76371]: cluster 2024-09-12T23:47:51.414903+0000 mon.a (mon.0) 918 : cluster [DBG] osdmap e614: 8 total, 8 up, 8 in 2024-09-12T23:47:52.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:51 smithi181 bash[67587]: audit 2024-09-12T23:47:50.693912+0000 mon.a (mon.0) 915 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "reef"}]': finished 2024-09-12T23:47:52.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:51 smithi181 bash[67587]: cluster 2024-09-12T23:47:50.694319+0000 mon.a (mon.0) 916 : cluster [DBG] osdmap e613: 8 total, 8 up, 8 in 2024-09-12T23:47:52.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:51 smithi181 bash[67587]: audit 2024-09-12T23:47:50.699036+0000 mon.a (mon.0) 917 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mds ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-12T23:47:52.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:51 smithi181 bash[67587]: cephadm 2024-09-12T23:47:50.700405+0000 mgr.y (mgr.24661) 279 : cephadm [INF] Upgrade: It appears safe to stop mds.a 2024-09-12T23:47:52.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:51 smithi181 bash[67587]: cluster 2024-09-12T23:47:50.785888+0000 mgr.y (mgr.24661) 280 : cluster [DBG] pgmap v496: 121 pgs: 121 active+clean; 401 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 1.1 MiB/s wr, 7 op/s 2024-09-12T23:47:52.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:51 smithi181 bash[67587]: cluster 2024-09-12T23:47:51.414903+0000 mon.a (mon.0) 918 : cluster [DBG] osdmap e614: 8 total, 8 up, 8 in 2024-09-12T23:47:52.416 INFO:tasks.rados.rados.0.smithi136.stdout:2726: done (3 left) 2024-09-12T23:47:52.417 INFO:tasks.rados.rados.0.smithi136.stdout:2727: done (2 left) 2024-09-12T23:47:52.417 INFO:tasks.rados.rados.0.smithi136.stdout:2728: done (1 left) 2024-09-12T23:47:52.417 INFO:tasks.rados.rados.0.smithi136.stdout:2729: done (0 left) 2024-09-12T23:47:52.417 INFO:tasks.rados.rados.0.smithi136.stdout:2730: rmattr oid 47 current snap is 280 2024-09-12T23:47:52.417 INFO:tasks.rados.rados.0.smithi136.stdout:2730: done (0 left) 2024-09-12T23:47:52.417 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 50 current snap is 280 2024-09-12T23:47:52.417 INFO:tasks.rados.rados.0.smithi136.stdout:2731: seq_num 860 ranges {0=8192} 2024-09-12T23:47:52.417 INFO:tasks.rados.rados.0.smithi136.stdout:2731: writing smithi13667081-50 from 0 to 8192 tid 1 2024-09-12T23:47:52.417 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 37 current snap is 280 2024-09-12T23:47:52.417 INFO:tasks.rados.rados.0.smithi136.stdout:2732: seq_num 861 ranges {9273344=24576} 2024-09-12T23:47:52.475 INFO:tasks.rados.rados.0.smithi136.stdout:2732: writing smithi13667081-37 from 9273344 to 9297920 tid 1 2024-09-12T23:47:52.476 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 47 current snap is 280 2024-09-12T23:47:52.476 INFO:tasks.rados.rados.0.smithi136.stdout:2733: seq_num 862 ranges {0=40960} 2024-09-12T23:47:52.476 INFO:tasks.rados.rados.0.smithi136.stdout:2733: writing smithi13667081-47 from 0 to 40960 tid 1 2024-09-12T23:47:52.476 INFO:tasks.rados.rados.0.smithi136.stdout:2734: delete oid 44 current snap is 280 2024-09-12T23:47:52.476 INFO:tasks.rados.rados.0.smithi136.stdout:2731: finishing write tid 1 to smithi13667081-50 2024-09-12T23:47:52.476 INFO:tasks.rados.rados.0.smithi136.stdout:2731: finishing write tid 2 to smithi13667081-50 2024-09-12T23:47:52.476 INFO:tasks.rados.rados.0.smithi136.stdout:2731: finishing write tid 3 to smithi13667081-50 2024-09-12T23:47:52.476 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 50 v 1637 (ObjNum 860 snap 280 seq_num 860) dirty exists 2024-09-12T23:47:52.476 INFO:tasks.rados.rados.0.smithi136.stdout:2731: left oid 50 (ObjNum 860 snap 280 seq_num 860) 2024-09-12T23:47:52.481 INFO:tasks.rados.rados.0.smithi136.stdout:2733: finishing write tid 1 to smithi13667081-47 2024-09-12T23:47:52.482 INFO:tasks.rados.rados.0.smithi136.stdout:2731: done (3 left) 2024-09-12T23:47:52.482 INFO:tasks.rados.rados.0.smithi136.stdout:2734: done (2 left) 2024-09-12T23:47:52.482 INFO:tasks.rados.rados.0.smithi136.stdout:2735: snap_create 2024-09-12T23:47:52.483 INFO:tasks.rados.rados.0.smithi136.stdout:2733: finishing write tid 2 to smithi13667081-47 2024-09-12T23:47:52.483 INFO:tasks.rados.rados.0.smithi136.stdout:2732: finishing write tid 1 to smithi13667081-37 2024-09-12T23:47:52.484 INFO:tasks.rados.rados.0.smithi136.stdout:2732: finishing write tid 2 to smithi13667081-37 2024-09-12T23:47:52.487 INFO:tasks.rados.rados.0.smithi136.stdout:2732: finishing write tid 3 to smithi13667081-37 2024-09-12T23:47:52.487 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 37 v 1303 (ObjNum 861 snap 280 seq_num 861) dirty exists 2024-09-12T23:47:52.487 INFO:tasks.rados.rados.0.smithi136.stdout:2732: left oid 37 (ObjNum 861 snap 280 seq_num 861) 2024-09-12T23:47:52.490 INFO:tasks.rados.rados.0.smithi136.stdout:2733: finishing write tid 3 to smithi13667081-47 2024-09-12T23:47:52.490 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 47 v 1614 (ObjNum 862 snap 280 seq_num 862) dirty exists 2024-09-12T23:47:52.490 INFO:tasks.rados.rados.0.smithi136.stdout:2733: left oid 47 (ObjNum 862 snap 280 seq_num 862) 2024-09-12T23:47:52.594 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:47:52 smithi181 bash[60471]: ts=2024-09-12T23:47:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:47:53.427 INFO:tasks.rados.rados.0.smithi136.stdout:2732: done (2 left) 2024-09-12T23:47:53.427 INFO:tasks.rados.rados.0.smithi136.stdout:2733: done (1 left) 2024-09-12T23:47:53.427 INFO:tasks.rados.rados.0.smithi136.stdout:2735: done (0 left) 2024-09-12T23:47:53.427 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 48 current snap is 281 2024-09-12T23:47:53.427 INFO:tasks.rados.rados.0.smithi136.stdout:2736: seq_num 864 ranges {2383872=73728} 2024-09-12T23:47:53.440 INFO:tasks.rados.rados.0.smithi136.stdout:2736: writing smithi13667081-48 from 2383872 to 2457600 tid 1 2024-09-12T23:47:53.440 INFO:tasks.rados.rados.0.smithi136.stdout:2737: delete oid 9 current snap is 281 2024-09-12T23:47:53.444 INFO:tasks.rados.rados.0.smithi136.stdout:2737: done (1 left) 2024-09-12T23:47:53.444 INFO:tasks.rados.rados.0.smithi136.stdout:2738: read oid 25 snap -1 2024-09-12T23:47:53.444 INFO:tasks.rados.rados.0.smithi136.stdout:2738: expect deleted 2024-09-12T23:47:53.444 INFO:tasks.rados.rados.0.smithi136.stdout:2739: read oid 18 snap -1 2024-09-12T23:47:53.444 INFO:tasks.rados.rados.0.smithi136.stdout:2739: expect (ObjNum 724 snap 224 seq_num 724) 2024-09-12T23:47:53.445 INFO:tasks.rados.rados.0.smithi136.stdout:2740: snap_remove snap 233 2024-09-12T23:47:53.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:53 smithi136 bash[76371]: cluster 2024-09-12T23:47:52.417466+0000 mon.a (mon.0) 919 : cluster [DBG] osdmap e615: 8 total, 8 up, 8 in 2024-09-12T23:47:53.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:53 smithi136 bash[76371]: cluster 2024-09-12T23:47:52.786583+0000 mgr.y (mgr.24661) 281 : cluster [DBG] pgmap v499: 121 pgs: 121 active+clean; 406 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 1.7 MiB/s wr, 5 op/s 2024-09-12T23:47:53.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:53 smithi136 bash[76371]: audit 2024-09-12T23:47:53.386968+0000 mon.a (mon.0) 920 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:53.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:53 smithi136 bash[76371]: audit 2024-09-12T23:47:53.388980+0000 mon.a (mon.0) 921 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-12T23:47:53.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:53 smithi136 bash[76371]: audit 2024-09-12T23:47:53.390405+0000 mon.a (mon.0) 922 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:47:53.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:53 smithi136 bash[75164]: cluster 2024-09-12T23:47:52.417466+0000 mon.a (mon.0) 919 : cluster [DBG] osdmap e615: 8 total, 8 up, 8 in 2024-09-12T23:47:53.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:53 smithi136 bash[75164]: cluster 2024-09-12T23:47:52.786583+0000 mgr.y (mgr.24661) 281 : cluster [DBG] pgmap v499: 121 pgs: 121 active+clean; 406 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 1.7 MiB/s wr, 5 op/s 2024-09-12T23:47:53.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:53 smithi136 bash[75164]: audit 2024-09-12T23:47:53.386968+0000 mon.a (mon.0) 920 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:53.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:53 smithi136 bash[75164]: audit 2024-09-12T23:47:53.388980+0000 mon.a (mon.0) 921 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-12T23:47:53.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:53 smithi136 bash[75164]: audit 2024-09-12T23:47:53.390405+0000 mon.a (mon.0) 922 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:47:53.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:53 smithi181 bash[67587]: cluster 2024-09-12T23:47:52.417466+0000 mon.a (mon.0) 919 : cluster [DBG] osdmap e615: 8 total, 8 up, 8 in 2024-09-12T23:47:53.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:53 smithi181 bash[67587]: cluster 2024-09-12T23:47:52.786583+0000 mgr.y (mgr.24661) 281 : cluster [DBG] pgmap v499: 121 pgs: 121 active+clean; 406 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 1.7 MiB/s wr, 5 op/s 2024-09-12T23:47:53.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:53 smithi181 bash[67587]: audit 2024-09-12T23:47:53.386968+0000 mon.a (mon.0) 920 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:53.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:53 smithi181 bash[67587]: audit 2024-09-12T23:47:53.388980+0000 mon.a (mon.0) 921 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-12T23:47:53.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:53 smithi181 bash[67587]: audit 2024-09-12T23:47:53.390405+0000 mon.a (mon.0) 922 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:47:54.443 INFO:tasks.rados.rados.0.smithi136.stdout:2740: done (3 left) 2024-09-12T23:47:54.443 INFO:tasks.rados.rados.0.smithi136.stdout:2741: snap_remove snap 276 2024-09-12T23:47:54.740 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:54 smithi136 bash[75164]: cephadm 2024-09-12T23:47:53.378539+0000 mgr.y (mgr.24661) 282 : cephadm [INF] Upgrade: Updating mds.a 2024-09-12T23:47:54.740 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:54 smithi136 bash[75164]: cephadm 2024-09-12T23:47:53.391956+0000 mgr.y (mgr.24661) 283 : cephadm [INF] Deploying daemon mds.a on smithi136 2024-09-12T23:47:54.740 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:54 smithi136 bash[75164]: cluster 2024-09-12T23:47:53.428000+0000 mon.a (mon.0) 923 : cluster [DBG] osdmap e616: 8 total, 8 up, 8 in 2024-09-12T23:47:54.740 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:54 smithi136 bash[75164]: audit 2024-09-12T23:47:53.755502+0000 mon.a (mon.0) 924 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:54.740 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:54 smithi136 bash[75164]: audit 2024-09-12T23:47:53.757379+0000 mon.a (mon.0) 925 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:47:54.740 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:54 smithi136 bash[75164]: audit 2024-09-12T23:47:53.986788+0000 mon.a (mon.0) 926 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:54.741 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:54 smithi136 bash[76371]: cephadm 2024-09-12T23:47:53.378539+0000 mgr.y (mgr.24661) 282 : cephadm [INF] Upgrade: Updating mds.a 2024-09-12T23:47:54.741 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:54 smithi136 bash[76371]: cephadm 2024-09-12T23:47:53.391956+0000 mgr.y (mgr.24661) 283 : cephadm [INF] Deploying daemon mds.a on smithi136 2024-09-12T23:47:54.741 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:54 smithi136 bash[76371]: cluster 2024-09-12T23:47:53.428000+0000 mon.a (mon.0) 923 : cluster [DBG] osdmap e616: 8 total, 8 up, 8 in 2024-09-12T23:47:54.741 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:54 smithi136 bash[76371]: audit 2024-09-12T23:47:53.755502+0000 mon.a (mon.0) 924 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:54.741 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:54 smithi136 bash[76371]: audit 2024-09-12T23:47:53.757379+0000 mon.a (mon.0) 925 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:47:54.741 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:54 smithi136 bash[76371]: audit 2024-09-12T23:47:53.986788+0000 mon.a (mon.0) 926 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:54.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:54 smithi181 bash[67587]: cephadm 2024-09-12T23:47:53.378539+0000 mgr.y (mgr.24661) 282 : cephadm [INF] Upgrade: Updating mds.a 2024-09-12T23:47:54.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:54 smithi181 bash[67587]: cephadm 2024-09-12T23:47:53.391956+0000 mgr.y (mgr.24661) 283 : cephadm [INF] Deploying daemon mds.a on smithi136 2024-09-12T23:47:54.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:54 smithi181 bash[67587]: cluster 2024-09-12T23:47:53.428000+0000 mon.a (mon.0) 923 : cluster [DBG] osdmap e616: 8 total, 8 up, 8 in 2024-09-12T23:47:54.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:54 smithi181 bash[67587]: audit 2024-09-12T23:47:53.755502+0000 mon.a (mon.0) 924 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:54.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:54 smithi181 bash[67587]: audit 2024-09-12T23:47:53.757379+0000 mon.a (mon.0) 925 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:47:54.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:54 smithi181 bash[67587]: audit 2024-09-12T23:47:53.986788+0000 mon.a (mon.0) 926 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:55.456 INFO:tasks.rados.rados.0.smithi136.stdout:2741: done (3 left) 2024-09-12T23:47:55.456 INFO:tasks.rados.rados.0.smithi136.stdout:2742: copy_from oid 49 from oid 29 current snap is 281 2024-09-12T23:47:55.456 INFO:tasks.rados.rados.0.smithi136.stdout:2743: rollback oid 34 current snap is 281 2024-09-12T23:47:55.456 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 34 to 255 2024-09-12T23:47:55.457 INFO:tasks.rados.rados.0.smithi136.stdout:2744: delete oid 47 current snap is 281 2024-09-12T23:47:55.457 INFO:tasks.rados.rados.0.smithi136.stdout:2736: finishing write tid 1 to smithi13667081-48 2024-09-12T23:47:55.457 INFO:tasks.rados.rados.0.smithi136.stdout:2736: finishing write tid 2 to smithi13667081-48 2024-09-12T23:47:55.457 INFO:tasks.rados.rados.0.smithi136.stdout:2736: finishing write tid 3 to smithi13667081-48 2024-09-12T23:47:55.457 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 48 v 1645 (ObjNum 864 snap 281 seq_num 864) dirty exists 2024-09-12T23:47:55.457 INFO:tasks.rados.rados.0.smithi136.stdout:2736: left oid 48 (ObjNum 864 snap 281 seq_num 864) 2024-09-12T23:47:55.457 INFO:tasks.rados.rados.0.smithi136.stdout:2739: expect (ObjNum 724 snap 224 seq_num 724) 2024-09-12T23:47:55.474 INFO:tasks.rados.rados.0.smithi136.stdout:2743: finishing rollback tid 0 to smithi13667081-34 2024-09-12T23:47:55.478 INFO:tasks.rados.rados.0.smithi136.stdout:2736: done (5 left) 2024-09-12T23:47:55.479 INFO:tasks.rados.rados.0.smithi136.stdout:2738: done (4 left) 2024-09-12T23:47:55.479 INFO:tasks.rados.rados.0.smithi136.stdout:2739: done (3 left) 2024-09-12T23:47:55.479 INFO:tasks.rados.rados.0.smithi136.stdout:2744: done (2 left) 2024-09-12T23:47:55.479 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 39 current snap is 281 2024-09-12T23:47:55.479 INFO:tasks.rados.rados.0.smithi136.stdout:2745: seq_num 867 ranges {0=122880} 2024-09-12T23:47:55.480 INFO:tasks.rados.rados.0.smithi136.stdout:2745: writing smithi13667081-39 from 0 to 122880 tid 1 2024-09-12T23:47:55.483 INFO:tasks.rados.rados.0.smithi136.stdout:2746: setattr oid 9 current snap is 281 2024-09-12T23:47:55.483 INFO:tasks.rados.rados.0.smithi136.stdout:2743: finishing rollback tid 1 to smithi13667081-34 2024-09-12T23:47:55.483 INFO:tasks.rados.rados.0.smithi136.stdout:2743: finishing rollback tid 2 to smithi13667081-34 2024-09-12T23:47:55.483 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 34 v 1306 (ObjNum 796 snap 253 seq_num 796) dirty exists 2024-09-12T23:47:55.483 INFO:tasks.rados.rados.0.smithi136.stdout:2743: done (3 left) 2024-09-12T23:47:55.483 INFO:tasks.rados.rados.0.smithi136.stdout:2747: read oid 27 snap -1 2024-09-12T23:47:55.483 INFO:tasks.rados.rados.0.smithi136.stdout:2747: expect (ObjNum 766 snap 236 seq_num 766) 2024-09-12T23:47:55.483 INFO:tasks.rados.rados.0.smithi136.stdout:2748: copy_from oid 22 from oid 30 current snap is 281 2024-09-12T23:47:55.484 INFO:tasks.rados.rados.0.smithi136.stdout:2749: delete oid 40 current snap is 281 2024-09-12T23:47:55.485 INFO:tasks.rados.rados.0.smithi136.stdout:2748: finishing copy_from to smithi13667081-22 2024-09-12T23:47:55.485 INFO:tasks.rados.rados.0.smithi136.stdout:2748: got expected ENOENT (src dne) 2024-09-12T23:47:55.485 INFO:tasks.rados.rados.0.smithi136.stdout:2748: finishing copy_from racing read to smithi13667081-22 2024-09-12T23:47:55.488 INFO:tasks.rados.rados.0.smithi136.stdout:2745: finishing write tid 1 to smithi13667081-39 2024-09-12T23:47:55.489 INFO:tasks.rados.rados.0.smithi136.stdout:2745: finishing write tid 2 to smithi13667081-39 2024-09-12T23:47:55.492 INFO:tasks.rados.rados.0.smithi136.stdout:2745: finishing write tid 3 to smithi13667081-39 2024-09-12T23:47:55.493 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 39 v 867 (ObjNum 867 snap 281 seq_num 867) dirty exists 2024-09-12T23:47:55.493 INFO:tasks.rados.rados.0.smithi136.stdout:2745: left oid 39 (ObjNum 867 snap 281 seq_num 867) 2024-09-12T23:47:55.496 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 9 v 1617 (ObjNum 161 snap 0 seq_num 0) dirty exists 2024-09-12T23:47:55.532 INFO:tasks.rados.rados.0.smithi136.stdout:2747: expect (ObjNum 766 snap 236 seq_num 766) 2024-09-12T23:47:55.738 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:55 smithi136 bash[75164]: cluster 2024-09-12T23:47:54.442261+0000 mon.a (mon.0) 927 : cluster [DBG] osdmap e617: 8 total, 8 up, 8 in 2024-09-12T23:47:55.738 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:55 smithi136 bash[75164]: cluster 2024-09-12T23:47:54.787053+0000 mgr.y (mgr.24661) 284 : cluster [DBG] pgmap v502: 121 pgs: 121 active+clean; 408 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.7 MiB/s wr, 10 op/s 2024-09-12T23:47:55.738 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:55 smithi136 bash[76371]: cluster 2024-09-12T23:47:54.442261+0000 mon.a (mon.0) 927 : cluster [DBG] osdmap e617: 8 total, 8 up, 8 in 2024-09-12T23:47:55.739 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:55 smithi136 bash[76371]: cluster 2024-09-12T23:47:54.787053+0000 mgr.y (mgr.24661) 284 : cluster [DBG] pgmap v502: 121 pgs: 121 active+clean; 408 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.7 MiB/s wr, 10 op/s 2024-09-12T23:47:55.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:55 smithi181 bash[67587]: cluster 2024-09-12T23:47:54.442261+0000 mon.a (mon.0) 927 : cluster [DBG] osdmap e617: 8 total, 8 up, 8 in 2024-09-12T23:47:55.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:55 smithi181 bash[67587]: cluster 2024-09-12T23:47:54.787053+0000 mgr.y (mgr.24661) 284 : cluster [DBG] pgmap v502: 121 pgs: 121 active+clean; 408 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.7 MiB/s wr, 10 op/s 2024-09-12T23:47:56.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:55 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:47:56.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:55 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:47:56.007 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 12 23:47:55 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:47:56.007 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 12 23:47:55 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:47:56.007 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 12 23:47:55 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:47:56.008 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:47:55 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:47:56.008 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:47:55 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:47:56.008 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:47:55 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:47:56.009 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 12 23:47:55 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:47:56.009 INFO:journalctl@ceph.mds.a.smithi136.stdout:Sep 12 23:47:55 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:47:56.009 INFO:journalctl@ceph.mds.a.smithi136.stdout:Sep 12 23:47:55 smithi136 systemd[1]: Stopping Ceph mds.a for da58a47c-715d-11ef-bceb-c7b262605968... 2024-09-12T23:47:56.009 INFO:journalctl@ceph.mds.a.smithi136.stdout:Sep 12 23:47:55 smithi136 bash[56511]: debug 2024-09-12T23:47:55.972+0000 7f4e40226640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mds -n mds.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-12T23:47:56.009 INFO:journalctl@ceph.mds.a.smithi136.stdout:Sep 12 23:47:55 smithi136 bash[56511]: debug 2024-09-12T23:47:55.972+0000 7f4e40226640 -1 mds.a *** got signal Terminated *** 2024-09-12T23:47:56.088 INFO:tasks.rados.rados.0.smithi136.stdout:2742: finishing copy_from racing read to smithi13667081-49 2024-09-12T23:47:56.088 INFO:tasks.rados.rados.0.smithi136.stdout:2742: finishing copy_from to smithi13667081-49 2024-09-12T23:47:56.088 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 49 v 1635 (ObjNum 859 snap 278 seq_num 859) dirty exists 2024-09-12T23:47:56.088 INFO:tasks.rados.rados.0.smithi136.stdout:2742: done (5 left) 2024-09-12T23:47:56.088 INFO:tasks.rados.rados.0.smithi136.stdout:2745: done (4 left) 2024-09-12T23:47:56.089 INFO:tasks.rados.rados.0.smithi136.stdout:2746: done (3 left) 2024-09-12T23:47:56.089 INFO:tasks.rados.rados.0.smithi136.stdout:2747: done (2 left) 2024-09-12T23:47:56.089 INFO:tasks.rados.rados.0.smithi136.stdout:2748: done (1 left) 2024-09-12T23:47:56.089 INFO:tasks.rados.rados.0.smithi136.stdout:2749: done (0 left) 2024-09-12T23:47:56.089 INFO:tasks.rados.rados.0.smithi136.stdout:2750: snap_remove snap 269 2024-09-12T23:47:56.418 INFO:tasks.rados.rados.0.smithi136.stdout:2750: done (0 left) 2024-09-12T23:47:56.418 INFO:tasks.rados.rados.0.smithi136.stdout:2751: rollback oid 12 current snap is 281 2024-09-12T23:47:56.419 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 12 to 211 2024-09-12T23:47:56.419 INFO:tasks.rados.rados.0.smithi136.stdout:2752: snap_create 2024-09-12T23:47:56.443 INFO:tasks.rados.rados.0.smithi136.stdout:2751: finishing rollback tid 0 to smithi13667081-12 2024-09-12T23:47:56.444 INFO:tasks.rados.rados.0.smithi136.stdout:2751: finishing rollback tid 1 to smithi13667081-12 2024-09-12T23:47:56.444 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 12 v 1637 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-12T23:47:56.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:56 smithi136 bash[75164]: cluster 2024-09-12T23:47:55.449735+0000 mon.a (mon.0) 928 : cluster [DBG] osdmap e618: 8 total, 8 up, 8 in 2024-09-12T23:47:56.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:56 smithi136 bash[75164]: cluster 2024-09-12T23:47:55.984051+0000 mon.a (mon.0) 929 : cluster [DBG] osdmap e619: 8 total, 8 up, 8 in 2024-09-12T23:47:56.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:56 smithi136 bash[75164]: cluster 2024-09-12T23:47:56.418351+0000 mon.a (mon.0) 930 : cluster [DBG] osdmap e620: 8 total, 8 up, 8 in 2024-09-12T23:47:56.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:56 smithi136 bash[76371]: cluster 2024-09-12T23:47:55.449735+0000 mon.a (mon.0) 928 : cluster [DBG] osdmap e618: 8 total, 8 up, 8 in 2024-09-12T23:47:56.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:56 smithi136 bash[76371]: cluster 2024-09-12T23:47:55.984051+0000 mon.a (mon.0) 929 : cluster [DBG] osdmap e619: 8 total, 8 up, 8 in 2024-09-12T23:47:56.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:56 smithi136 bash[76371]: cluster 2024-09-12T23:47:56.418351+0000 mon.a (mon.0) 930 : cluster [DBG] osdmap e620: 8 total, 8 up, 8 in 2024-09-12T23:47:56.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:56 smithi181 bash[67587]: cluster 2024-09-12T23:47:55.449735+0000 mon.a (mon.0) 928 : cluster [DBG] osdmap e618: 8 total, 8 up, 8 in 2024-09-12T23:47:56.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:56 smithi181 bash[67587]: cluster 2024-09-12T23:47:55.984051+0000 mon.a (mon.0) 929 : cluster [DBG] osdmap e619: 8 total, 8 up, 8 in 2024-09-12T23:47:56.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:56 smithi181 bash[67587]: cluster 2024-09-12T23:47:56.418351+0000 mon.a (mon.0) 930 : cluster [DBG] osdmap e620: 8 total, 8 up, 8 in 2024-09-12T23:47:57.418 INFO:tasks.rados.rados.0.smithi136.stdout:2751: done (1 left) 2024-09-12T23:47:57.419 INFO:tasks.rados.rados.0.smithi136.stdout:2752: done (0 left) 2024-09-12T23:47:57.419 INFO:tasks.rados.rados.0.smithi136.stdout:2753: snap_remove snap 281 2024-09-12T23:47:57.732 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:57 smithi136 bash[75164]: cluster 2024-09-12T23:47:56.450790+0000 mon.a (mon.0) 931 : cluster [DBG] fsmap cephfs:1 {0=b=up:active} 2024-09-12T23:47:57.732 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:57 smithi136 bash[75164]: cluster 2024-09-12T23:47:56.788006+0000 mgr.y (mgr.24661) 285 : cluster [DBG] pgmap v506: 121 pgs: 121 active+clean; 416 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 129 KiB/s rd, 5.6 MiB/s wr, 22 op/s 2024-09-12T23:47:57.732 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:57 smithi136 bash[75164]: cluster 2024-09-12T23:47:57.419490+0000 mon.a (mon.0) 932 : cluster [DBG] osdmap e621: 8 total, 8 up, 8 in 2024-09-12T23:47:57.733 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:57 smithi136 bash[76371]: cluster 2024-09-12T23:47:56.450790+0000 mon.a (mon.0) 931 : cluster [DBG] fsmap cephfs:1 {0=b=up:active} 2024-09-12T23:47:57.733 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:57 smithi136 bash[76371]: cluster 2024-09-12T23:47:56.788006+0000 mgr.y (mgr.24661) 285 : cluster [DBG] pgmap v506: 121 pgs: 121 active+clean; 416 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 129 KiB/s rd, 5.6 MiB/s wr, 22 op/s 2024-09-12T23:47:57.733 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:57 smithi136 bash[76371]: cluster 2024-09-12T23:47:57.419490+0000 mon.a (mon.0) 932 : cluster [DBG] osdmap e621: 8 total, 8 up, 8 in 2024-09-12T23:47:57.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:57 smithi181 bash[67587]: cluster 2024-09-12T23:47:56.450790+0000 mon.a (mon.0) 931 : cluster [DBG] fsmap cephfs:1 {0=b=up:active} 2024-09-12T23:47:57.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:57 smithi181 bash[67587]: cluster 2024-09-12T23:47:56.788006+0000 mgr.y (mgr.24661) 285 : cluster [DBG] pgmap v506: 121 pgs: 121 active+clean; 416 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 129 KiB/s rd, 5.6 MiB/s wr, 22 op/s 2024-09-12T23:47:57.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:57 smithi181 bash[67587]: cluster 2024-09-12T23:47:57.419490+0000 mon.a (mon.0) 932 : cluster [DBG] osdmap e621: 8 total, 8 up, 8 in 2024-09-12T23:47:58.414 INFO:tasks.rados.rados.0.smithi136.stdout:2753: done (0 left) 2024-09-12T23:47:58.415 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 2 current snap is 282 2024-09-12T23:47:58.415 INFO:tasks.rados.rados.0.smithi136.stdout:2754: seq_num 869 ranges {0=155648} 2024-09-12T23:47:58.416 INFO:tasks.rados.rados.0.smithi136.stdout:2754: writing smithi13667081-2 from 0 to 155648 tid 1 2024-09-12T23:47:58.416 INFO:tasks.rados.rados.0.smithi136.stdout:2755: read oid 25 snap -1 2024-09-12T23:47:58.416 INFO:tasks.rados.rados.0.smithi136.stdout:2755: expect deleted 2024-09-12T23:47:58.416 INFO:tasks.rados.rados.0.smithi136.stdout:2756: read oid 48 snap 238 2024-09-12T23:47:58.416 INFO:tasks.rados.rados.0.smithi136.stdout:2756: expect (ObjNum 764 snap 236 seq_num 764) 2024-09-12T23:47:58.416 INFO:tasks.rados.rados.0.smithi136.stdout:2757: delete oid 27 current snap is 282 2024-09-12T23:47:58.423 INFO:tasks.rados.rados.0.smithi136.stdout:2754: finishing write tid 1 to smithi13667081-2 2024-09-12T23:47:58.423 INFO:tasks.rados.rados.0.smithi136.stdout:2757: done (3 left) 2024-09-12T23:47:58.423 INFO:tasks.rados.rados.0.smithi136.stdout:2758: snap_create 2024-09-12T23:47:58.424 INFO:tasks.rados.rados.0.smithi136.stdout:2754: finishing write tid 2 to smithi13667081-2 2024-09-12T23:47:58.427 INFO:tasks.rados.rados.0.smithi136.stdout:2754: finishing write tid 3 to smithi13667081-2 2024-09-12T23:47:58.428 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 2 v 1308 (ObjNum 869 snap 282 seq_num 869) dirty exists 2024-09-12T23:47:58.428 INFO:tasks.rados.rados.0.smithi136.stdout:2754: left oid 2 (ObjNum 869 snap 282 seq_num 869) 2024-09-12T23:47:58.430 INFO:tasks.rados.rados.0.smithi136.stdout:2756: expect (ObjNum 764 snap 236 seq_num 764) 2024-09-12T23:47:58.693 INFO:journalctl@ceph.mds.a.smithi136.stdout:Sep 12 23:47:58 smithi136 bash[97351]: ceph-da58a47c-715d-11ef-bceb-c7b262605968-mds-a 2024-09-12T23:47:58.693 INFO:journalctl@ceph.mds.a.smithi136.stdout:Sep 12 23:47:58 smithi136 bash[97412]: Error response from daemon: No such container: ceph-da58a47c-715d-11ef-bceb-c7b262605968-mds-a 2024-09-12T23:47:58.693 INFO:journalctl@ceph.mds.a.smithi136.stdout:Sep 12 23:47:58 smithi136 systemd[1]: ceph-da58a47c-715d-11ef-bceb-c7b262605968@mds.a.service: Deactivated successfully. 2024-09-12T23:47:58.693 INFO:journalctl@ceph.mds.a.smithi136.stdout:Sep 12 23:47:58 smithi136 systemd[1]: Stopped Ceph mds.a for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:47:59.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:58 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:47:59.007 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 12 23:47:58 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:47:59.007 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:47:58 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:47:59.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:58 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:47:59.007 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 12 23:47:58 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:47:59.008 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 12 23:47:58 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:47:59.008 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:47:58 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:47:59.008 INFO:journalctl@ceph.mds.a.smithi136.stdout:Sep 12 23:47:58 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:47:59.008 INFO:journalctl@ceph.mds.a.smithi136.stdout:Sep 12 23:47:58 smithi136 systemd[1]: Started Ceph mds.a for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:47:59.009 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 12 23:47:58 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:47:59.009 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:47:58 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:47:59.429 INFO:tasks.rados.rados.0.smithi136.stdout:2754: done (3 left) 2024-09-12T23:47:59.429 INFO:tasks.rados.rados.0.smithi136.stdout:2755: done (2 left) 2024-09-12T23:47:59.429 INFO:tasks.rados.rados.0.smithi136.stdout:2756: done (1 left) 2024-09-12T23:47:59.429 INFO:tasks.rados.rados.0.smithi136.stdout:2758: done (0 left) 2024-09-12T23:47:59.429 INFO:tasks.rados.rados.0.smithi136.stdout:2759: snap_remove snap 252 2024-09-12T23:47:59.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:47:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:47:59] "GET /metrics HTTP/1.1" 200 36377 "" "Prometheus/2.43.0" 2024-09-12T23:47:59.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:59 smithi136 bash[75164]: cluster 2024-09-12T23:47:58.418100+0000 mon.a (mon.0) 933 : cluster [DBG] osdmap e622: 8 total, 8 up, 8 in 2024-09-12T23:47:59.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:59 smithi136 bash[75164]: cluster 2024-09-12T23:47:58.788878+0000 mgr.y (mgr.24661) 286 : cluster [DBG] pgmap v509: 121 pgs: 121 active+clean; 406 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.4 MiB/s rd, 5.0 MiB/s wr, 15 op/s 2024-09-12T23:47:59.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:59 smithi136 bash[75164]: audit 2024-09-12T23:47:58.845237+0000 mon.a (mon.0) 934 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:59.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:59 smithi136 bash[75164]: audit 2024-09-12T23:47:58.851900+0000 mon.a (mon.0) 935 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:59.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:47:59 smithi136 bash[75164]: audit 2024-09-12T23:47:58.853002+0000 mon.a (mon.0) 936 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:47:59.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:59 smithi136 bash[76371]: cluster 2024-09-12T23:47:58.418100+0000 mon.a (mon.0) 933 : cluster [DBG] osdmap e622: 8 total, 8 up, 8 in 2024-09-12T23:47:59.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:59 smithi136 bash[76371]: cluster 2024-09-12T23:47:58.788878+0000 mgr.y (mgr.24661) 286 : cluster [DBG] pgmap v509: 121 pgs: 121 active+clean; 406 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.4 MiB/s rd, 5.0 MiB/s wr, 15 op/s 2024-09-12T23:47:59.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:59 smithi136 bash[76371]: audit 2024-09-12T23:47:58.845237+0000 mon.a (mon.0) 934 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:59.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:59 smithi136 bash[76371]: audit 2024-09-12T23:47:58.851900+0000 mon.a (mon.0) 935 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:59.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:47:59 smithi136 bash[76371]: audit 2024-09-12T23:47:58.853002+0000 mon.a (mon.0) 936 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:47:59.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:59 smithi181 bash[67587]: cluster 2024-09-12T23:47:58.418100+0000 mon.a (mon.0) 933 : cluster [DBG] osdmap e622: 8 total, 8 up, 8 in 2024-09-12T23:47:59.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:59 smithi181 bash[67587]: cluster 2024-09-12T23:47:58.788878+0000 mgr.y (mgr.24661) 286 : cluster [DBG] pgmap v509: 121 pgs: 121 active+clean; 406 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.4 MiB/s rd, 5.0 MiB/s wr, 15 op/s 2024-09-12T23:47:59.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:59 smithi181 bash[67587]: audit 2024-09-12T23:47:58.845237+0000 mon.a (mon.0) 934 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:59.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:59 smithi181 bash[67587]: audit 2024-09-12T23:47:58.851900+0000 mon.a (mon.0) 935 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:47:59.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:47:59 smithi181 bash[67587]: audit 2024-09-12T23:47:58.853002+0000 mon.a (mon.0) 936 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:48:00.425 INFO:journalctl@ceph.mds.a.smithi136.stdout:Sep 12 23:48:00 smithi136 bash[97471]: starting mds.a at 2024-09-12T23:48:00.452 INFO:tasks.rados.rados.0.smithi136.stdout:2759: done (0 left) 2024-09-12T23:48:00.452 INFO:tasks.rados.rados.0.smithi136.stdout:2760: rmattr oid 26 current snap is 283 2024-09-12T23:48:00.452 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 11 current snap is 283 2024-09-12T23:48:00.452 INFO:tasks.rados.rados.0.smithi136.stdout:2761: seq_num 871 ranges {7290880=188416} 2024-09-12T23:48:00.489 INFO:tasks.rados.rados.0.smithi136.stdout:2761: writing smithi13667081-11 from 7290880 to 7479296 tid 1 2024-09-12T23:48:00.489 INFO:tasks.rados.rados.0.smithi136.stdout:2762: copy_from oid 24 from oid 3 current snap is 283 2024-09-12T23:48:00.489 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 26 v 1281 (ObjNum 724 snap 224 seq_num 724) dirty exists 2024-09-12T23:48:00.489 INFO:tasks.rados.rados.0.smithi136.stdout:2760: done (2 left) 2024-09-12T23:48:00.489 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 42 current snap is 283 2024-09-12T23:48:00.490 INFO:tasks.rados.rados.0.smithi136.stdout:2763: seq_num 872 ranges {1966080=212992} 2024-09-12T23:48:00.502 INFO:tasks.rados.rados.0.smithi136.stdout:2763: writing smithi13667081-42 from 1966080 to 2179072 tid 1 2024-09-12T23:48:00.502 INFO:tasks.rados.rados.0.smithi136.stdout:2764: delete oid 39 current snap is 283 2024-09-12T23:48:00.502 INFO:tasks.rados.rados.0.smithi136.stdout:2761: finishing write tid 1 to smithi13667081-11 2024-09-12T23:48:00.502 INFO:tasks.rados.rados.0.smithi136.stdout:2761: finishing write tid 2 to smithi13667081-11 2024-09-12T23:48:00.502 INFO:tasks.rados.rados.0.smithi136.stdout:2761: finishing write tid 3 to smithi13667081-11 2024-09-12T23:48:00.502 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 11 v 1931 (ObjNum 871 snap 283 seq_num 871) dirty exists 2024-09-12T23:48:00.502 INFO:tasks.rados.rados.0.smithi136.stdout:2761: left oid 11 (ObjNum 871 snap 283 seq_num 871) 2024-09-12T23:48:00.503 INFO:tasks.rados.rados.0.smithi136.stdout:2762: finishing copy_from racing read to smithi13667081-24 2024-09-12T23:48:00.506 INFO:tasks.rados.rados.0.smithi136.stdout:2761: done (3 left) 2024-09-12T23:48:00.506 INFO:tasks.rados.rados.0.smithi136.stdout:2764: done (2 left) 2024-09-12T23:48:00.506 INFO:tasks.rados.rados.0.smithi136.stdout:2765: rollback oid 30 current snap is 283 2024-09-12T23:48:00.506 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 30 to 270 2024-09-12T23:48:00.506 INFO:tasks.rados.rados.0.smithi136.stdout:2766: rollback oid 5 current snap is 283 2024-09-12T23:48:00.506 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 5 to 254 2024-09-12T23:48:00.506 INFO:tasks.rados.rados.0.smithi136.stdout:2767: rollback oid 40 current snap is 283 2024-09-12T23:48:00.506 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 40 to 274 2024-09-12T23:48:00.506 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 2 current snap is 283 2024-09-12T23:48:00.506 INFO:tasks.rados.rados.0.smithi136.stdout:2768: seq_num 874 ranges {155648=245760} 2024-09-12T23:48:00.508 INFO:tasks.rados.rados.0.smithi136.stdout:2768: writing smithi13667081-2 from 155648 to 401408 tid 1 2024-09-12T23:48:00.508 INFO:tasks.rados.rados.0.smithi136.stdout:2769: snap_create 2024-09-12T23:48:00.508 INFO:tasks.rados.rados.0.smithi136.stdout:2763: finishing write tid 1 to smithi13667081-42 2024-09-12T23:48:00.508 INFO:tasks.rados.rados.0.smithi136.stdout:2763: finishing write tid 2 to smithi13667081-42 2024-09-12T23:48:00.510 INFO:tasks.rados.rados.0.smithi136.stdout:2765: finishing rollback tid 1 to smithi13667081-30 2024-09-12T23:48:00.510 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 30 v 1483 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-12T23:48:00.510 INFO:tasks.rados.rados.0.smithi136.stdout:2762: finishing copy_from to smithi13667081-24 2024-09-12T23:48:00.510 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 24 v 1489 (ObjNum 838 snap 267 seq_num 838) dirty exists 2024-09-12T23:48:00.511 INFO:tasks.rados.rados.0.smithi136.stdout:2763: finishing write tid 3 to smithi13667081-42 2024-09-12T23:48:00.511 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 42 v 1789 (ObjNum 872 snap 283 seq_num 872) dirty exists 2024-09-12T23:48:00.511 INFO:tasks.rados.rados.0.smithi136.stdout:2763: left oid 42 (ObjNum 872 snap 283 seq_num 872) 2024-09-12T23:48:00.514 INFO:tasks.rados.rados.0.smithi136.stdout:2768: finishing write tid 1 to smithi13667081-2 2024-09-12T23:48:00.514 INFO:tasks.rados.rados.0.smithi136.stdout:2768: finishing write tid 2 to smithi13667081-2 2024-09-12T23:48:00.514 INFO:tasks.rados.rados.0.smithi136.stdout:2766: finishing rollback tid 0 to smithi13667081-5 2024-09-12T23:48:00.514 INFO:tasks.rados.rados.0.smithi136.stdout:2766: finishing rollback tid 1 to smithi13667081-5 2024-09-12T23:48:00.515 INFO:tasks.rados.rados.0.smithi136.stdout:2768: finishing write tid 3 to smithi13667081-2 2024-09-12T23:48:00.516 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 2 v 1310 (ObjNum 874 snap 283 seq_num 874) dirty exists 2024-09-12T23:48:00.516 INFO:tasks.rados.rados.0.smithi136.stdout:2768: left oid 2 (ObjNum 874 snap 283 seq_num 874) 2024-09-12T23:48:00.516 INFO:tasks.rados.rados.0.smithi136.stdout:2766: finishing rollback tid 2 to smithi13667081-5 2024-09-12T23:48:00.516 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 5 v 1935 (ObjNum 567 snap 162 seq_num 567) dirty exists 2024-09-12T23:48:00.516 INFO:tasks.rados.rados.0.smithi136.stdout:2767: finishing rollback tid 1 to smithi13667081-40 2024-09-12T23:48:00.516 INFO:tasks.rados.rados.0.smithi136.stdout:2767: finishing rollback tid 2 to smithi13667081-40 2024-09-12T23:48:00.516 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 40 v 1937 (ObjNum 788 snap 250 seq_num 788) dirty exists 2024-09-12T23:48:00.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:00 smithi136 bash[75164]: cluster 2024-09-12T23:47:59.432483+0000 mon.a (mon.0) 937 : cluster [DBG] osdmap e623: 8 total, 8 up, 8 in 2024-09-12T23:48:00.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:00 smithi136 bash[76371]: cluster 2024-09-12T23:47:59.432483+0000 mon.a (mon.0) 937 : cluster [DBG] osdmap e623: 8 total, 8 up, 8 in 2024-09-12T23:48:00.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:00 smithi181 bash[67587]: cluster 2024-09-12T23:47:59.432483+0000 mon.a (mon.0) 937 : cluster [DBG] osdmap e623: 8 total, 8 up, 8 in 2024-09-12T23:48:01.418 INFO:tasks.rados.rados.0.smithi136.stdout:2762: done (6 left) 2024-09-12T23:48:01.419 INFO:tasks.rados.rados.0.smithi136.stdout:2763: done (5 left) 2024-09-12T23:48:01.419 INFO:tasks.rados.rados.0.smithi136.stdout:2765: done (4 left) 2024-09-12T23:48:01.419 INFO:tasks.rados.rados.0.smithi136.stdout:2766: done (3 left) 2024-09-12T23:48:01.419 INFO:tasks.rados.rados.0.smithi136.stdout:2767: done (2 left) 2024-09-12T23:48:01.419 INFO:tasks.rados.rados.0.smithi136.stdout:2768: done (1 left) 2024-09-12T23:48:01.419 INFO:tasks.rados.rados.0.smithi136.stdout:2769: done (0 left) 2024-09-12T23:48:01.419 INFO:tasks.rados.rados.0.smithi136.stdout:2770: read oid 19 snap -1 2024-09-12T23:48:01.419 INFO:tasks.rados.rados.0.smithi136.stdout:2770: expect deleted 2024-09-12T23:48:01.419 INFO:tasks.rados.rados.0.smithi136.stdout:2771: snap_remove snap 277 2024-09-12T23:48:01.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:01 smithi136 bash[75164]: cluster 2024-09-12T23:48:00.445469+0000 mon.a (mon.0) 938 : cluster [DBG] osdmap e624: 8 total, 8 up, 8 in 2024-09-12T23:48:01.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:01 smithi136 bash[75164]: cluster 2024-09-12T23:48:00.448132+0000 mon.a (mon.0) 939 : cluster [DBG] mds.? [v2:172.21.15.136:6834/496731293,v1:172.21.15.136:6835/496731293] up:boot 2024-09-12T23:48:01.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:01 smithi136 bash[75164]: cluster 2024-09-12T23:48:00.448323+0000 mon.a (mon.0) 940 : cluster [DBG] fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-12T23:48:01.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:01 smithi136 bash[75164]: audit 2024-09-12T23:48:00.448633+0000 mon.a (mon.0) 941 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-12T23:48:01.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:01 smithi136 bash[75164]: cluster 2024-09-12T23:48:00.789748+0000 mgr.y (mgr.24661) 287 : cluster [DBG] pgmap v512: 121 pgs: 121 active+clean; 406 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 6.1 MiB/s rd, 0 B/s wr, 1 op/s 2024-09-12T23:48:01.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:01 smithi136 bash[75164]: cluster 2024-09-12T23:48:01.419521+0000 mon.a (mon.0) 942 : cluster [DBG] osdmap e625: 8 total, 8 up, 8 in 2024-09-12T23:48:01.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:01 smithi136 bash[76371]: cluster 2024-09-12T23:48:00.445469+0000 mon.a (mon.0) 938 : cluster [DBG] osdmap e624: 8 total, 8 up, 8 in 2024-09-12T23:48:01.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:01 smithi136 bash[76371]: cluster 2024-09-12T23:48:00.448132+0000 mon.a (mon.0) 939 : cluster [DBG] mds.? [v2:172.21.15.136:6834/496731293,v1:172.21.15.136:6835/496731293] up:boot 2024-09-12T23:48:01.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:01 smithi136 bash[76371]: cluster 2024-09-12T23:48:00.448323+0000 mon.a (mon.0) 940 : cluster [DBG] fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-12T23:48:01.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:01 smithi136 bash[76371]: audit 2024-09-12T23:48:00.448633+0000 mon.a (mon.0) 941 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-12T23:48:01.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:01 smithi136 bash[76371]: cluster 2024-09-12T23:48:00.789748+0000 mgr.y (mgr.24661) 287 : cluster [DBG] pgmap v512: 121 pgs: 121 active+clean; 406 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 6.1 MiB/s rd, 0 B/s wr, 1 op/s 2024-09-12T23:48:01.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:01 smithi136 bash[76371]: cluster 2024-09-12T23:48:01.419521+0000 mon.a (mon.0) 942 : cluster [DBG] osdmap e625: 8 total, 8 up, 8 in 2024-09-12T23:48:01.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:01 smithi181 bash[67587]: cluster 2024-09-12T23:48:00.445469+0000 mon.a (mon.0) 938 : cluster [DBG] osdmap e624: 8 total, 8 up, 8 in 2024-09-12T23:48:01.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:01 smithi181 bash[67587]: cluster 2024-09-12T23:48:00.448132+0000 mon.a (mon.0) 939 : cluster [DBG] mds.? [v2:172.21.15.136:6834/496731293,v1:172.21.15.136:6835/496731293] up:boot 2024-09-12T23:48:01.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:01 smithi181 bash[67587]: cluster 2024-09-12T23:48:00.448323+0000 mon.a (mon.0) 940 : cluster [DBG] fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-12T23:48:01.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:01 smithi181 bash[67587]: audit 2024-09-12T23:48:00.448633+0000 mon.a (mon.0) 941 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-12T23:48:01.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:01 smithi181 bash[67587]: cluster 2024-09-12T23:48:00.789748+0000 mgr.y (mgr.24661) 287 : cluster [DBG] pgmap v512: 121 pgs: 121 active+clean; 406 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 6.1 MiB/s rd, 0 B/s wr, 1 op/s 2024-09-12T23:48:01.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:01 smithi181 bash[67587]: cluster 2024-09-12T23:48:01.419521+0000 mon.a (mon.0) 942 : cluster [DBG] osdmap e625: 8 total, 8 up, 8 in 2024-09-12T23:48:02.423 INFO:tasks.rados.rados.0.smithi136.stdout:2771: done (1 left) 2024-09-12T23:48:02.423 INFO:tasks.rados.rados.0.smithi136.stdout:2772: snap_create 2024-09-12T23:48:02.594 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:48:02 smithi181 bash[60471]: ts=2024-09-12T23:48:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:48:03.430 INFO:tasks.rados.rados.0.smithi136.stdout:2770: done (1 left) 2024-09-12T23:48:03.431 INFO:tasks.rados.rados.0.smithi136.stdout:2772: done (0 left) 2024-09-12T23:48:03.431 INFO:tasks.rados.rados.0.smithi136.stdout:2773: copy_from oid 50 from oid 43 current snap is 285 2024-09-12T23:48:03.431 INFO:tasks.rados.rados.0.smithi136.stdout:2774: snap_create 2024-09-12T23:48:03.444 INFO:tasks.rados.rados.0.smithi136.stdout:2773: finishing copy_from racing read to smithi13667081-50 2024-09-12T23:48:03.445 INFO:tasks.rados.rados.0.smithi136.stdout:2773: finishing copy_from to smithi13667081-50 2024-09-12T23:48:03.445 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 50 v 1614 (ObjNum 81 snap 0 seq_num 0) dirty exists 2024-09-12T23:48:03.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:03 smithi136 bash[76371]: cluster 2024-09-12T23:48:02.422924+0000 mon.a (mon.0) 943 : cluster [DBG] osdmap e626: 8 total, 8 up, 8 in 2024-09-12T23:48:03.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:03 smithi136 bash[76371]: cluster 2024-09-12T23:48:02.790618+0000 mgr.y (mgr.24661) 288 : cluster [DBG] pgmap v515: 121 pgs: 121 active+clean; 405 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 598 KiB/s wr, 5 op/s 2024-09-12T23:48:03.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:03 smithi136 bash[75164]: cluster 2024-09-12T23:48:02.422924+0000 mon.a (mon.0) 943 : cluster [DBG] osdmap e626: 8 total, 8 up, 8 in 2024-09-12T23:48:03.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:03 smithi136 bash[75164]: cluster 2024-09-12T23:48:02.790618+0000 mgr.y (mgr.24661) 288 : cluster [DBG] pgmap v515: 121 pgs: 121 active+clean; 405 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 598 KiB/s wr, 5 op/s 2024-09-12T23:48:03.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:03 smithi181 bash[67587]: cluster 2024-09-12T23:48:02.422924+0000 mon.a (mon.0) 943 : cluster [DBG] osdmap e626: 8 total, 8 up, 8 in 2024-09-12T23:48:03.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:03 smithi181 bash[67587]: cluster 2024-09-12T23:48:02.790618+0000 mgr.y (mgr.24661) 288 : cluster [DBG] pgmap v515: 121 pgs: 121 active+clean; 405 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 598 KiB/s wr, 5 op/s 2024-09-12T23:48:04.438 INFO:tasks.rados.rados.0.smithi136.stdout:2773: done (1 left) 2024-09-12T23:48:04.438 INFO:tasks.rados.rados.0.smithi136.stdout:2774: done (0 left) 2024-09-12T23:48:04.438 INFO:tasks.rados.rados.0.smithi136.stdout:2775: rollback oid 44 current snap is 286 2024-09-12T23:48:04.438 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 44 to 238 2024-09-12T23:48:04.438 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 42 current snap is 286 2024-09-12T23:48:04.438 INFO:tasks.rados.rados.0.smithi136.stdout:2776: seq_num 875 ranges {2179072=262144} 2024-09-12T23:48:04.459 INFO:tasks.rados.rados.0.smithi136.stdout:2776: writing smithi13667081-42 from 2179072 to 2441216 tid 1 2024-09-12T23:48:04.459 INFO:tasks.rados.rados.0.smithi136.stdout:2777: rollback oid 5 current snap is 286 2024-09-12T23:48:04.459 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 5 to 240 2024-09-12T23:48:04.459 INFO:tasks.rados.rados.0.smithi136.stdout:2775: finishing rollback tid 1 to smithi13667081-44 2024-09-12T23:48:04.459 INFO:tasks.rados.rados.0.smithi136.stdout:2775: finishing rollback tid 2 to smithi13667081-44 2024-09-12T23:48:04.459 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 44 v 1679 (ObjNum 33 snap 0 seq_num 2878255351) dirty exists 2024-09-12T23:48:04.459 INFO:tasks.rados.rados.0.smithi136.stdout:2775: done (2 left) 2024-09-12T23:48:04.459 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 37 current snap is 286 2024-09-12T23:48:04.459 INFO:tasks.rados.rados.0.smithi136.stdout:2778: seq_num 876 ranges {9297920=278528} 2024-09-12T23:48:04.529 INFO:tasks.rados.rados.0.smithi136.stdout:2778: writing smithi13667081-37 from 9297920 to 9576448 tid 1 2024-09-12T23:48:04.530 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 9 current snap is 286 2024-09-12T23:48:04.530 INFO:tasks.rados.rados.0.smithi136.stdout:2779: seq_num 877 ranges {0=294912} 2024-09-12T23:48:04.532 INFO:tasks.rados.rados.0.smithi136.stdout:2779: writing smithi13667081-9 from 0 to 294912 tid 1 2024-09-12T23:48:04.532 INFO:tasks.rados.rados.0.smithi136.stdout:2780: read oid 19 snap -1 2024-09-12T23:48:04.532 INFO:tasks.rados.rados.0.smithi136.stdout:2780: expect deleted 2024-09-12T23:48:04.532 INFO:tasks.rados.rados.0.smithi136.stdout:2781: delete oid 36 current snap is 286 2024-09-12T23:48:04.532 INFO:tasks.rados.rados.0.smithi136.stdout:2776: finishing write tid 1 to smithi13667081-42 2024-09-12T23:48:04.532 INFO:tasks.rados.rados.0.smithi136.stdout:2776: finishing write tid 2 to smithi13667081-42 2024-09-12T23:48:04.532 INFO:tasks.rados.rados.0.smithi136.stdout:2776: finishing write tid 3 to smithi13667081-42 2024-09-12T23:48:04.532 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 42 v 1791 (ObjNum 875 snap 286 seq_num 875) dirty exists 2024-09-12T23:48:04.532 INFO:tasks.rados.rados.0.smithi136.stdout:2776: left oid 42 (ObjNum 875 snap 286 seq_num 875) 2024-09-12T23:48:04.532 INFO:tasks.rados.rados.0.smithi136.stdout:2777: finishing rollback tid 0 to smithi13667081-5 2024-09-12T23:48:04.533 INFO:tasks.rados.rados.0.smithi136.stdout:2777: finishing rollback tid 1 to smithi13667081-5 2024-09-12T23:48:04.533 INFO:tasks.rados.rados.0.smithi136.stdout:2777: finishing rollback tid 2 to smithi13667081-5 2024-09-12T23:48:04.533 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 5 v 1949 (ObjNum 567 snap 162 seq_num 567) dirty exists 2024-09-12T23:48:04.537 INFO:tasks.rados.rados.0.smithi136.stdout:2778: finishing write tid 1 to smithi13667081-37 2024-09-12T23:48:04.538 INFO:tasks.rados.rados.0.smithi136.stdout:2778: finishing write tid 2 to smithi13667081-37 2024-09-12T23:48:04.539 INFO:tasks.rados.rados.0.smithi136.stdout:2779: finishing write tid 1 to smithi13667081-9 2024-09-12T23:48:04.540 INFO:tasks.rados.rados.0.smithi136.stdout:2779: finishing write tid 2 to smithi13667081-9 2024-09-12T23:48:04.541 INFO:tasks.rados.rados.0.smithi136.stdout:2778: finishing write tid 3 to smithi13667081-37 2024-09-12T23:48:04.542 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 37 v 1312 (ObjNum 876 snap 286 seq_num 876) dirty exists 2024-09-12T23:48:04.542 INFO:tasks.rados.rados.0.smithi136.stdout:2778: left oid 37 (ObjNum 876 snap 286 seq_num 876) 2024-09-12T23:48:04.542 INFO:tasks.rados.rados.0.smithi136.stdout:2779: finishing write tid 3 to smithi13667081-9 2024-09-12T23:48:04.543 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 9 v 1619 (ObjNum 877 snap 286 seq_num 877) dirty exists 2024-09-12T23:48:04.543 INFO:tasks.rados.rados.0.smithi136.stdout:2779: left oid 9 (ObjNum 877 snap 286 seq_num 877) 2024-09-12T23:48:04.544 INFO:tasks.rados.rados.0.smithi136.stdout:2776: done (5 left) 2024-09-12T23:48:04.544 INFO:tasks.rados.rados.0.smithi136.stdout:2777: done (4 left) 2024-09-12T23:48:04.544 INFO:tasks.rados.rados.0.smithi136.stdout:2778: done (3 left) 2024-09-12T23:48:04.545 INFO:tasks.rados.rados.0.smithi136.stdout:2779: done (2 left) 2024-09-12T23:48:04.545 INFO:tasks.rados.rados.0.smithi136.stdout:2780: done (1 left) 2024-09-12T23:48:04.545 INFO:tasks.rados.rados.0.smithi136.stdout:2781: done (0 left) 2024-09-12T23:48:04.545 INFO:tasks.rados.rados.0.smithi136.stdout:2782: read oid 30 snap -1 2024-09-12T23:48:04.545 INFO:tasks.rados.rados.0.smithi136.stdout:2782: expect deleted 2024-09-12T23:48:04.545 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 32 current snap is 286 2024-09-12T23:48:04.545 INFO:tasks.rados.rados.0.smithi136.stdout:2783: seq_num 879 ranges {0=327680} 2024-09-12T23:48:04.546 INFO:tasks.rados.rados.0.smithi136.stdout:2783: writing smithi13667081-32 from 0 to 327680 tid 1 2024-09-12T23:48:04.546 INFO:tasks.rados.rados.0.smithi136.stdout:2784: snap_remove snap 266 2024-09-12T23:48:04.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:04 smithi136 bash[75164]: cluster 2024-09-12T23:48:03.431904+0000 mon.a (mon.0) 944 : cluster [DBG] osdmap e627: 8 total, 8 up, 8 in 2024-09-12T23:48:04.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:04 smithi136 bash[76371]: cluster 2024-09-12T23:48:03.431904+0000 mon.a (mon.0) 944 : cluster [DBG] osdmap e627: 8 total, 8 up, 8 in 2024-09-12T23:48:04.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:04 smithi181 bash[67587]: cluster 2024-09-12T23:48:03.431904+0000 mon.a (mon.0) 944 : cluster [DBG] osdmap e627: 8 total, 8 up, 8 in 2024-09-12T23:48:05.458 INFO:tasks.rados.rados.0.smithi136.stdout:2784: done (2 left) 2024-09-12T23:48:05.458 INFO:tasks.rados.rados.0.smithi136.stdout:2785: copy_from oid 12 from oid 27 current snap is 286 2024-09-12T23:48:05.461 INFO:tasks.rados.rados.0.smithi136.stdout:2786: read oid 28 snap 282 2024-09-12T23:48:05.461 INFO:tasks.rados.rados.0.smithi136.stdout:2786: expect deleted 2024-09-12T23:48:05.461 INFO:tasks.rados.rados.0.smithi136.stdout:2787: copy_from oid 17 from oid 35 current snap is 286 2024-09-12T23:48:05.461 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 39 current snap is 286 2024-09-12T23:48:05.461 INFO:tasks.rados.rados.0.smithi136.stdout:2788: seq_num 880 ranges {0=344064} 2024-09-12T23:48:05.461 INFO:tasks.rados.rados.0.smithi136.stdout:2788: writing smithi13667081-39 from 0 to 344064 tid 1 2024-09-12T23:48:05.461 INFO:tasks.rados.rados.0.smithi136.stdout:2789: delete oid 44 current snap is 286 2024-09-12T23:48:05.461 INFO:tasks.rados.rados.0.smithi136.stdout:2783: finishing write tid 1 to smithi13667081-32 2024-09-12T23:48:05.462 INFO:tasks.rados.rados.0.smithi136.stdout:2783: finishing write tid 2 to smithi13667081-32 2024-09-12T23:48:05.462 INFO:tasks.rados.rados.0.smithi136.stdout:2783: finishing write tid 3 to smithi13667081-32 2024-09-12T23:48:05.462 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 32 v 1615 (ObjNum 879 snap 286 seq_num 879) dirty exists 2024-09-12T23:48:05.462 INFO:tasks.rados.rados.0.smithi136.stdout:2783: left oid 32 (ObjNum 879 snap 286 seq_num 879) 2024-09-12T23:48:05.473 INFO:tasks.rados.rados.0.smithi136.stdout:2782: done (6 left) 2024-09-12T23:48:05.474 INFO:tasks.rados.rados.0.smithi136.stdout:2783: done (5 left) 2024-09-12T23:48:05.474 INFO:tasks.rados.rados.0.smithi136.stdout:2786: done (4 left) 2024-09-12T23:48:05.474 INFO:tasks.rados.rados.0.smithi136.stdout:2789: done (3 left) 2024-09-12T23:48:05.474 INFO:tasks.rados.rados.0.smithi136.stdout:2790: copy_from oid 42 from oid 37 current snap is 286 2024-09-12T23:48:05.474 INFO:tasks.rados.rados.0.smithi136.stdout:2791: snap_remove snap 239 2024-09-12T23:48:05.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:05 smithi136 bash[76371]: cluster 2024-09-12T23:48:04.439963+0000 mon.a (mon.0) 945 : cluster [DBG] osdmap e628: 8 total, 8 up, 8 in 2024-09-12T23:48:05.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:05 smithi136 bash[76371]: cluster 2024-09-12T23:48:04.791116+0000 mgr.y (mgr.24661) 289 : cluster [DBG] pgmap v518: 121 pgs: 121 active+clean; 407 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 696 KiB/s wr, 8 op/s 2024-09-12T23:48:05.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:05 smithi136 bash[76371]: audit 2024-09-12T23:48:05.250971+0000 mon.a (mon.0) 946 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:05.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:05 smithi136 bash[76371]: audit 2024-09-12T23:48:05.257370+0000 mon.a (mon.0) 947 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:05.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:05 smithi136 bash[75164]: cluster 2024-09-12T23:48:04.439963+0000 mon.a (mon.0) 945 : cluster [DBG] osdmap e628: 8 total, 8 up, 8 in 2024-09-12T23:48:05.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:05 smithi136 bash[75164]: cluster 2024-09-12T23:48:04.791116+0000 mgr.y (mgr.24661) 289 : cluster [DBG] pgmap v518: 121 pgs: 121 active+clean; 407 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 696 KiB/s wr, 8 op/s 2024-09-12T23:48:05.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:05 smithi136 bash[75164]: audit 2024-09-12T23:48:05.250971+0000 mon.a (mon.0) 946 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:05.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:05 smithi136 bash[75164]: audit 2024-09-12T23:48:05.257370+0000 mon.a (mon.0) 947 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:05.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:05 smithi181 bash[67587]: cluster 2024-09-12T23:48:04.439963+0000 mon.a (mon.0) 945 : cluster [DBG] osdmap e628: 8 total, 8 up, 8 in 2024-09-12T23:48:05.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:05 smithi181 bash[67587]: cluster 2024-09-12T23:48:04.791116+0000 mgr.y (mgr.24661) 289 : cluster [DBG] pgmap v518: 121 pgs: 121 active+clean; 407 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 696 KiB/s wr, 8 op/s 2024-09-12T23:48:05.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:05 smithi181 bash[67587]: audit 2024-09-12T23:48:05.250971+0000 mon.a (mon.0) 946 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:05.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:05 smithi181 bash[67587]: audit 2024-09-12T23:48:05.257370+0000 mon.a (mon.0) 947 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:06.422 INFO:tasks.rados.rados.0.smithi136.stdout:2791: done (4 left) 2024-09-12T23:48:06.422 INFO:tasks.rados.rados.0.smithi136.stdout:2792: read oid 46 snap 270 2024-09-12T23:48:06.422 INFO:tasks.rados.rados.0.smithi136.stdout:2792: expect (ObjNum 822 snap 258 seq_num 822) 2024-09-12T23:48:06.422 INFO:tasks.rados.rados.0.smithi136.stdout:2793: rollback oid 11 current snap is 286 2024-09-12T23:48:06.422 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 11 to 271 2024-09-12T23:48:06.422 INFO:tasks.rados.rados.0.smithi136.stdout:2794: read oid 32 snap -1 2024-09-12T23:48:06.422 INFO:tasks.rados.rados.0.smithi136.stdout:2794: expect (ObjNum 879 snap 286 seq_num 879) 2024-09-12T23:48:06.423 INFO:tasks.rados.rados.0.smithi136.stdout:2795: read oid 22 snap 267 2024-09-12T23:48:06.423 INFO:tasks.rados.rados.0.smithi136.stdout:2795: expect deleted 2024-09-12T23:48:06.423 INFO:tasks.rados.rados.0.smithi136.stdout:2796: read oid 38 snap -1 2024-09-12T23:48:06.423 INFO:tasks.rados.rados.0.smithi136.stdout:2796: expect (ObjNum 727 snap 224 seq_num 727) 2024-09-12T23:48:06.423 INFO:tasks.rados.rados.0.smithi136.stdout:2797: read oid 36 snap -1 2024-09-12T23:48:06.423 INFO:tasks.rados.rados.0.smithi136.stdout:2797: expect deleted 2024-09-12T23:48:06.423 INFO:tasks.rados.rados.0.smithi136.stdout:2798: read oid 10 snap -1 2024-09-12T23:48:06.423 INFO:tasks.rados.rados.0.smithi136.stdout:2798: expect (ObjNum 850 snap 272 seq_num 850) 2024-09-12T23:48:06.423 INFO:tasks.rados.rados.0.smithi136.stdout:2799: delete oid 16 current snap is 286 2024-09-12T23:48:06.423 INFO:tasks.rados.rados.0.smithi136.stdout:2788: finishing write tid 1 to smithi13667081-39 2024-09-12T23:48:06.424 INFO:tasks.rados.rados.0.smithi136.stdout:2788: finishing write tid 2 to smithi13667081-39 2024-09-12T23:48:06.424 INFO:tasks.rados.rados.0.smithi136.stdout:2787: finishing copy_from racing read to smithi13667081-17 2024-09-12T23:48:06.424 INFO:tasks.rados.rados.0.smithi136.stdout:2788: finishing write tid 3 to smithi13667081-39 2024-09-12T23:48:06.424 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 39 v 870 (ObjNum 880 snap 286 seq_num 880) dirty exists 2024-09-12T23:48:06.424 INFO:tasks.rados.rados.0.smithi136.stdout:2788: left oid 39 (ObjNum 880 snap 286 seq_num 880) 2024-09-12T23:48:06.424 INFO:tasks.rados.rados.0.smithi136.stdout:2785: finishing copy_from to smithi13667081-12 2024-09-12T23:48:06.424 INFO:tasks.rados.rados.0.smithi136.stdout:2785: got expected ENOENT (src dne) 2024-09-12T23:48:06.424 INFO:tasks.rados.rados.0.smithi136.stdout:2785: finishing copy_from racing read to smithi13667081-12 2024-09-12T23:48:06.424 INFO:tasks.rados.rados.0.smithi136.stdout:2787: finishing copy_from to smithi13667081-17 2024-09-12T23:48:06.424 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 17 v 1577 (ObjNum 671 snap 196 seq_num 671) dirty exists 2024-09-12T23:48:06.424 INFO:tasks.rados.rados.0.smithi136.stdout:2790: finishing copy_from racing read to smithi13667081-42 2024-09-12T23:48:06.424 INFO:tasks.rados.rados.0.smithi136.stdout:2790: finishing copy_from to smithi13667081-42 2024-09-12T23:48:06.424 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 42 v 1312 (ObjNum 876 snap 286 seq_num 876) dirty exists 2024-09-12T23:48:06.443 INFO:tasks.rados.rados.0.smithi136.stdout:2785: done (11 left) 2024-09-12T23:48:06.443 INFO:tasks.rados.rados.0.smithi136.stdout:2787: done (10 left) 2024-09-12T23:48:06.443 INFO:tasks.rados.rados.0.smithi136.stdout:2788: done (9 left) 2024-09-12T23:48:06.443 INFO:tasks.rados.rados.0.smithi136.stdout:2790: done (8 left) 2024-09-12T23:48:06.443 INFO:tasks.rados.rados.0.smithi136.stdout:2795: done (7 left) 2024-09-12T23:48:06.443 INFO:tasks.rados.rados.0.smithi136.stdout:2797: done (6 left) 2024-09-12T23:48:06.443 INFO:tasks.rados.rados.0.smithi136.stdout:2799: done (5 left) 2024-09-12T23:48:06.443 INFO:tasks.rados.rados.0.smithi136.stdout:2800: rollback oid 40 current snap is 286 2024-09-12T23:48:06.444 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 40 to 255 2024-09-12T23:48:06.444 INFO:tasks.rados.rados.0.smithi136.stdout:2801: rollback oid 16 current snap is 286 2024-09-12T23:48:06.444 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 16 to 258 2024-09-12T23:48:06.444 INFO:tasks.rados.rados.0.smithi136.stdout:2802: copy_from oid 28 from oid 24 current snap is 286 2024-09-12T23:48:06.444 INFO:tasks.rados.rados.0.smithi136.stdout:2803: read oid 5 snap 249 2024-09-12T23:48:06.444 INFO:tasks.rados.rados.0.smithi136.stdout:2803: expect (ObjNum 567 snap 162 seq_num 567) 2024-09-12T23:48:06.444 INFO:tasks.rados.rados.0.smithi136.stdout:2804: delete oid 6 current snap is 286 2024-09-12T23:48:06.449 INFO:tasks.rados.rados.0.smithi136.stdout:2794: expect (ObjNum 879 snap 286 seq_num 879) 2024-09-12T23:48:06.487 INFO:tasks.rados.rados.0.smithi136.stdout:2794: done (9 left) 2024-09-12T23:48:06.487 INFO:tasks.rados.rados.0.smithi136.stdout:2804: done (8 left) 2024-09-12T23:48:06.487 INFO:tasks.rados.rados.0.smithi136.stdout:2805: copy_from oid 8 from oid 32 current snap is 286 2024-09-12T23:48:06.487 INFO:tasks.rados.rados.0.smithi136.stdout:2806: snap_create 2024-09-12T23:48:06.487 INFO:tasks.rados.rados.0.smithi136.stdout:2792: expect (ObjNum 822 snap 258 seq_num 822) 2024-09-12T23:48:06.548 INFO:tasks.rados.rados.0.smithi136.stdout:2798: expect (ObjNum 850 snap 272 seq_num 850) 2024-09-12T23:48:06.607 INFO:tasks.rados.rados.0.smithi136.stdout:2793: finishing rollback tid 0 to smithi13667081-11 2024-09-12T23:48:06.607 INFO:tasks.rados.rados.0.smithi136.stdout:2793: finishing rollback tid 1 to smithi13667081-11 2024-09-12T23:48:06.607 INFO:tasks.rados.rados.0.smithi136.stdout:2793: finishing rollback tid 2 to smithi13667081-11 2024-09-12T23:48:06.607 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 11 v 1969 (ObjNum 605 snap 174 seq_num 605) dirty exists 2024-09-12T23:48:06.607 INFO:tasks.rados.rados.0.smithi136.stdout:2800: finishing rollback tid 0 to smithi13667081-40 2024-09-12T23:48:06.607 INFO:tasks.rados.rados.0.smithi136.stdout:2802: finishing copy_from racing read to smithi13667081-28 2024-09-12T23:48:06.607 INFO:tasks.rados.rados.0.smithi136.stdout:2801: finishing rollback tid 1 to smithi13667081-16 2024-09-12T23:48:06.607 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 16 v 1611 (ObjNum 177 snap 0 seq_num 1) dirty dne 2024-09-12T23:48:06.607 INFO:tasks.rados.rados.0.smithi136.stdout:2802: finishing copy_from to smithi13667081-28 2024-09-12T23:48:06.607 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 28 v 1489 (ObjNum 838 snap 267 seq_num 838) dirty exists 2024-09-12T23:48:06.607 INFO:tasks.rados.rados.0.smithi136.stdout:2805: finishing copy_from racing read to smithi13667081-8 2024-09-12T23:48:06.607 INFO:tasks.rados.rados.0.smithi136.stdout:2805: finishing copy_from to smithi13667081-8 2024-09-12T23:48:06.608 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 8 v 1615 (ObjNum 879 snap 286 seq_num 879) dirty exists 2024-09-12T23:48:06.608 INFO:tasks.rados.rados.0.smithi136.stdout:2796: expect (ObjNum 727 snap 224 seq_num 727) 2024-09-12T23:48:06.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:06 smithi136 bash[75164]: cluster 2024-09-12T23:48:05.460631+0000 mon.a (mon.0) 948 : cluster [DBG] osdmap e629: 8 total, 8 up, 8 in 2024-09-12T23:48:06.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:06 smithi136 bash[75164]: audit 2024-09-12T23:48:05.927621+0000 mon.a (mon.0) 949 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:06.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:06 smithi136 bash[75164]: audit 2024-09-12T23:48:05.933595+0000 mon.a (mon.0) 950 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:06.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:06 smithi136 bash[75164]: cluster 2024-09-12T23:48:06.423438+0000 mon.a (mon.0) 951 : cluster [DBG] osdmap e630: 8 total, 8 up, 8 in 2024-09-12T23:48:06.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:06 smithi136 bash[76371]: cluster 2024-09-12T23:48:05.460631+0000 mon.a (mon.0) 948 : cluster [DBG] osdmap e629: 8 total, 8 up, 8 in 2024-09-12T23:48:06.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:06 smithi136 bash[76371]: audit 2024-09-12T23:48:05.927621+0000 mon.a (mon.0) 949 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:06.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:06 smithi136 bash[76371]: audit 2024-09-12T23:48:05.933595+0000 mon.a (mon.0) 950 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:06.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:06 smithi136 bash[76371]: cluster 2024-09-12T23:48:06.423438+0000 mon.a (mon.0) 951 : cluster [DBG] osdmap e630: 8 total, 8 up, 8 in 2024-09-12T23:48:06.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:06 smithi181 bash[67587]: cluster 2024-09-12T23:48:05.460631+0000 mon.a (mon.0) 948 : cluster [DBG] osdmap e629: 8 total, 8 up, 8 in 2024-09-12T23:48:06.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:06 smithi181 bash[67587]: audit 2024-09-12T23:48:05.927621+0000 mon.a (mon.0) 949 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:06.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:06 smithi181 bash[67587]: audit 2024-09-12T23:48:05.933595+0000 mon.a (mon.0) 950 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:06.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:06 smithi181 bash[67587]: cluster 2024-09-12T23:48:06.423438+0000 mon.a (mon.0) 951 : cluster [DBG] osdmap e630: 8 total, 8 up, 8 in 2024-09-12T23:48:07.021 INFO:tasks.rados.rados.0.smithi136.stdout:2800: finishing rollback tid 1 to smithi13667081-40 2024-09-12T23:48:07.021 INFO:tasks.rados.rados.0.smithi136.stdout:2800: finishing rollback tid 2 to smithi13667081-40 2024-09-12T23:48:07.021 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 40 v 1973 (ObjNum 788 snap 250 seq_num 788) dirty exists 2024-09-12T23:48:07.021 INFO:tasks.rados.rados.0.smithi136.stdout:2803: expect (ObjNum 567 snap 162 seq_num 567) 2024-09-12T23:48:07.470 INFO:tasks.rados.rados.0.smithi136.stdout:2792: done (9 left) 2024-09-12T23:48:07.470 INFO:tasks.rados.rados.0.smithi136.stdout:2793: done (8 left) 2024-09-12T23:48:07.470 INFO:tasks.rados.rados.0.smithi136.stdout:2796: done (7 left) 2024-09-12T23:48:07.470 INFO:tasks.rados.rados.0.smithi136.stdout:2798: done (6 left) 2024-09-12T23:48:07.470 INFO:tasks.rados.rados.0.smithi136.stdout:2800: done (5 left) 2024-09-12T23:48:07.470 INFO:tasks.rados.rados.0.smithi136.stdout:2801: done (4 left) 2024-09-12T23:48:07.471 INFO:tasks.rados.rados.0.smithi136.stdout:2802: done (3 left) 2024-09-12T23:48:07.471 INFO:tasks.rados.rados.0.smithi136.stdout:2803: done (2 left) 2024-09-12T23:48:07.471 INFO:tasks.rados.rados.0.smithi136.stdout:2805: done (1 left) 2024-09-12T23:48:07.471 INFO:tasks.rados.rados.0.smithi136.stdout:2806: done (0 left) 2024-09-12T23:48:07.471 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 12 current snap is 287 2024-09-12T23:48:07.471 INFO:tasks.rados.rados.0.smithi136.stdout:2807: seq_num 884 ranges {0=409600} 2024-09-12T23:48:07.476 INFO:tasks.rados.rados.0.smithi136.stdout:2807: writing smithi13667081-12 from 0 to 409600 tid 1 2024-09-12T23:48:07.477 INFO:tasks.rados.rados.0.smithi136.stdout:2808: read oid 33 snap -1 2024-09-12T23:48:07.477 INFO:tasks.rados.rados.0.smithi136.stdout:2808: expect deleted 2024-09-12T23:48:07.477 INFO:tasks.rados.rados.0.smithi136.stdout:2809: rmattr oid 38 current snap is 287 2024-09-12T23:48:07.477 INFO:tasks.rados.rados.0.smithi136.stdout:2809: done (2 left) 2024-09-12T23:48:07.477 INFO:tasks.rados.rados.0.smithi136.stdout:2810: rollback oid 44 current snap is 287 2024-09-12T23:48:07.477 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 44 to 247 2024-09-12T23:48:07.477 INFO:tasks.rados.rados.0.smithi136.stdout:2811: rollback oid 4 current snap is 287 2024-09-12T23:48:07.477 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 4 to 273 2024-09-12T23:48:07.477 INFO:tasks.rados.rados.0.smithi136.stdout:2812: rollback oid 41 current snap is 287 2024-09-12T23:48:07.477 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 41 to 280 2024-09-12T23:48:07.477 INFO:tasks.rados.rados.0.smithi136.stdout:2813: read oid 29 snap -1 2024-09-12T23:48:07.477 INFO:tasks.rados.rados.0.smithi136.stdout:2813: expect (ObjNum 859 snap 278 seq_num 859) 2024-09-12T23:48:07.477 INFO:tasks.rados.rados.0.smithi136.stdout:2814: copy_from oid 24 from oid 30 current snap is 287 2024-09-12T23:48:07.478 INFO:tasks.rados.rados.0.smithi136.stdout:2815: setattr oid 36 current snap is 287 2024-09-12T23:48:07.480 INFO:tasks.rados.rados.0.smithi136.stdout:2808: done (7 left) 2024-09-12T23:48:07.480 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 16 current snap is 287 2024-09-12T23:48:07.480 INFO:tasks.rados.rados.0.smithi136.stdout:2816: seq_num 885 ranges {0=425984} 2024-09-12T23:48:07.488 INFO:tasks.rados.rados.0.smithi136.stdout:2816: writing smithi13667081-16 from 0 to 425984 tid 1 2024-09-12T23:48:07.488 INFO:tasks.rados.rados.0.smithi136.stdout:2814: finishing copy_from to smithi13667081-24 2024-09-12T23:48:07.488 INFO:tasks.rados.rados.0.smithi136.stdout:2814: got expected ENOENT (src dne) 2024-09-12T23:48:07.488 INFO:tasks.rados.rados.0.smithi136.stdout:2814: finishing copy_from racing read to smithi13667081-24 2024-09-12T23:48:07.488 INFO:tasks.rados.rados.0.smithi136.stdout:2814: done (7 left) 2024-09-12T23:48:07.488 INFO:tasks.rados.rados.0.smithi136.stdout:2817: copy_from oid 40 from oid 19 current snap is 287 2024-09-12T23:48:07.488 INFO:tasks.rados.rados.0.smithi136.stdout:2818: snap_remove snap 246 2024-09-12T23:48:07.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:07 smithi136 bash[75164]: cluster 2024-09-12T23:48:06.791970+0000 mgr.y (mgr.24661) 290 : cluster [DBG] pgmap v521: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 408 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 822 KiB/s wr, 6 op/s 2024-09-12T23:48:07.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:07 smithi136 bash[76371]: cluster 2024-09-12T23:48:06.791970+0000 mgr.y (mgr.24661) 290 : cluster [DBG] pgmap v521: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 408 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 822 KiB/s wr, 6 op/s 2024-09-12T23:48:07.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:07 smithi181 bash[67587]: cluster 2024-09-12T23:48:06.791970+0000 mgr.y (mgr.24661) 290 : cluster [DBG] pgmap v521: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 408 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 822 KiB/s wr, 6 op/s 2024-09-12T23:48:08.485 INFO:tasks.rados.rados.0.smithi136.stdout:2818: done (8 left) 2024-09-12T23:48:08.485 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 23 current snap is 287 2024-09-12T23:48:08.485 INFO:tasks.rados.rados.0.smithi136.stdout:2819: seq_num 886 ranges {466944=442368} 2024-09-12T23:48:08.500 INFO:tasks.rados.rados.0.smithi136.stdout:2819: writing smithi13667081-23 from 466944 to 909312 tid 1 2024-09-12T23:48:08.500 INFO:tasks.rados.rados.0.smithi136.stdout:2820: delete oid 47 current snap is 287 2024-09-12T23:48:08.500 INFO:tasks.rados.rados.0.smithi136.stdout:2807: finishing write tid 1 to smithi13667081-12 2024-09-12T23:48:08.500 INFO:tasks.rados.rados.0.smithi136.stdout:2807: finishing write tid 2 to smithi13667081-12 2024-09-12T23:48:08.500 INFO:tasks.rados.rados.0.smithi136.stdout:2812: finishing rollback tid 0 to smithi13667081-41 2024-09-12T23:48:08.500 INFO:tasks.rados.rados.0.smithi136.stdout:2812: finishing rollback tid 1 to smithi13667081-41 2024-09-12T23:48:08.500 INFO:tasks.rados.rados.0.smithi136.stdout:2812: finishing rollback tid 2 to smithi13667081-41 2024-09-12T23:48:08.500 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 41 v 1315 (ObjNum 856 snap 277 seq_num 856) dirty exists 2024-09-12T23:48:08.500 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 36 v 1616 (ObjNum 33 snap 0 seq_num 4244559152) dirty exists 2024-09-12T23:48:08.500 INFO:tasks.rados.rados.0.smithi136.stdout:2810: finishing rollback tid 1 to smithi13667081-44 2024-09-12T23:48:08.500 INFO:tasks.rados.rados.0.smithi136.stdout:2810: finishing rollback tid 2 to smithi13667081-44 2024-09-12T23:48:08.501 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 44 v 1699 (ObjNum 783 snap 245 seq_num 783) dirty exists 2024-09-12T23:48:08.501 INFO:tasks.rados.rados.0.smithi136.stdout:2807: finishing write tid 3 to smithi13667081-12 2024-09-12T23:48:08.501 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 12 v 1639 (ObjNum 884 snap 287 seq_num 884) dirty exists 2024-09-12T23:48:08.501 INFO:tasks.rados.rados.0.smithi136.stdout:2807: left oid 12 (ObjNum 884 snap 287 seq_num 884) 2024-09-12T23:48:08.501 INFO:tasks.rados.rados.0.smithi136.stdout:2817: finishing copy_from to smithi13667081-40 2024-09-12T23:48:08.501 INFO:tasks.rados.rados.0.smithi136.stdout:2817: got expected ENOENT (src dne) 2024-09-12T23:48:08.501 INFO:tasks.rados.rados.0.smithi136.stdout:2811: finishing rollback tid 0 to smithi13667081-4 2024-09-12T23:48:08.501 INFO:tasks.rados.rados.0.smithi136.stdout:2817: finishing copy_from racing read to smithi13667081-40 2024-09-12T23:48:08.501 INFO:tasks.rados.rados.0.smithi136.stdout:2811: finishing rollback tid 1 to smithi13667081-4 2024-09-12T23:48:08.501 INFO:tasks.rados.rados.0.smithi136.stdout:2811: finishing rollback tid 2 to smithi13667081-4 2024-09-12T23:48:08.501 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 4 v 1618 (ObjNum 800 snap 253 seq_num 800) dirty exists 2024-09-12T23:48:08.501 INFO:tasks.rados.rados.0.smithi136.stdout:2816: finishing write tid 1 to smithi13667081-16 2024-09-12T23:48:08.501 INFO:tasks.rados.rados.0.smithi136.stdout:2816: finishing write tid 2 to smithi13667081-16 2024-09-12T23:48:08.501 INFO:tasks.rados.rados.0.smithi136.stdout:2816: finishing write tid 3 to smithi13667081-16 2024-09-12T23:48:08.502 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 16 v 1701 (ObjNum 885 snap 287 seq_num 885) dirty exists 2024-09-12T23:48:08.502 INFO:tasks.rados.rados.0.smithi136.stdout:2816: left oid 16 (ObjNum 885 snap 287 seq_num 885) 2024-09-12T23:48:08.502 INFO:tasks.rados.rados.0.smithi136.stdout:2813: expect (ObjNum 859 snap 278 seq_num 859) 2024-09-12T23:48:08.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:08 smithi136 bash[76371]: cluster 2024-09-12T23:48:07.472241+0000 mon.a (mon.0) 952 : cluster [DBG] osdmap e631: 8 total, 8 up, 8 in 2024-09-12T23:48:08.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:08 smithi136 bash[75164]: cluster 2024-09-12T23:48:07.472241+0000 mon.a (mon.0) 952 : cluster [DBG] osdmap e631: 8 total, 8 up, 8 in 2024-09-12T23:48:08.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:08 smithi181 bash[67587]: cluster 2024-09-12T23:48:07.472241+0000 mon.a (mon.0) 952 : cluster [DBG] osdmap e631: 8 total, 8 up, 8 in 2024-09-12T23:48:09.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:48:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:48:09] "GET /metrics HTTP/1.1" 200 36370 "" "Prometheus/2.43.0" 2024-09-12T23:48:09.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:09 smithi136 bash[75164]: cluster 2024-09-12T23:48:08.480743+0000 mon.a (mon.0) 953 : cluster [DBG] osdmap e632: 8 total, 8 up, 8 in 2024-09-12T23:48:09.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:09 smithi136 bash[75164]: audit 2024-09-12T23:48:08.749192+0000 mon.a (mon.0) 954 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:48:09.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:09 smithi136 bash[75164]: cluster 2024-09-12T23:48:08.792355+0000 mgr.y (mgr.24661) 291 : cluster [DBG] pgmap v524: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 422 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 4.0 MiB/s wr, 11 op/s 2024-09-12T23:48:09.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:09 smithi136 bash[76371]: cluster 2024-09-12T23:48:08.480743+0000 mon.a (mon.0) 953 : cluster [DBG] osdmap e632: 8 total, 8 up, 8 in 2024-09-12T23:48:09.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:09 smithi136 bash[76371]: audit 2024-09-12T23:48:08.749192+0000 mon.a (mon.0) 954 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:48:09.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:09 smithi136 bash[76371]: cluster 2024-09-12T23:48:08.792355+0000 mgr.y (mgr.24661) 291 : cluster [DBG] pgmap v524: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 422 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 4.0 MiB/s wr, 11 op/s 2024-09-12T23:48:09.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:09 smithi181 bash[67587]: cluster 2024-09-12T23:48:08.480743+0000 mon.a (mon.0) 953 : cluster [DBG] osdmap e632: 8 total, 8 up, 8 in 2024-09-12T23:48:09.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:09 smithi181 bash[67587]: audit 2024-09-12T23:48:08.749192+0000 mon.a (mon.0) 954 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:48:09.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:09 smithi181 bash[67587]: cluster 2024-09-12T23:48:08.792355+0000 mgr.y (mgr.24661) 291 : cluster [DBG] pgmap v524: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 422 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 4.0 MiB/s wr, 11 op/s 2024-09-12T23:48:10.289 INFO:tasks.rados.rados.0.smithi136.stdout:2819: finishing write tid 1 to smithi13667081-23 2024-09-12T23:48:10.289 INFO:tasks.rados.rados.0.smithi136.stdout:2819: finishing write tid 2 to smithi13667081-23 2024-09-12T23:48:10.289 INFO:tasks.rados.rados.0.smithi136.stdout:2819: finishing write tid 3 to smithi13667081-23 2024-09-12T23:48:10.289 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 23 v 1982 (ObjNum 886 snap 287 seq_num 886) dirty exists 2024-09-12T23:48:10.289 INFO:tasks.rados.rados.0.smithi136.stdout:2819: left oid 23 (ObjNum 886 snap 287 seq_num 886) 2024-09-12T23:48:10.289 INFO:tasks.rados.rados.0.smithi136.stdout:2807: done (9 left) 2024-09-12T23:48:10.289 INFO:tasks.rados.rados.0.smithi136.stdout:2810: done (8 left) 2024-09-12T23:48:10.289 INFO:tasks.rados.rados.0.smithi136.stdout:2811: done (7 left) 2024-09-12T23:48:10.289 INFO:tasks.rados.rados.0.smithi136.stdout:2812: done (6 left) 2024-09-12T23:48:10.289 INFO:tasks.rados.rados.0.smithi136.stdout:2813: done (5 left) 2024-09-12T23:48:10.290 INFO:tasks.rados.rados.0.smithi136.stdout:2815: done (4 left) 2024-09-12T23:48:10.290 INFO:tasks.rados.rados.0.smithi136.stdout:2816: done (3 left) 2024-09-12T23:48:10.290 INFO:tasks.rados.rados.0.smithi136.stdout:2817: done (2 left) 2024-09-12T23:48:10.291 INFO:tasks.rados.rados.0.smithi136.stdout:2819: done (1 left) 2024-09-12T23:48:10.291 INFO:tasks.rados.rados.0.smithi136.stdout:2820: done (0 left) 2024-09-12T23:48:10.291 INFO:tasks.rados.rados.0.smithi136.stdout:2821: rollback oid 50 current snap is 287 2024-09-12T23:48:10.291 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 50 to 249 2024-09-12T23:48:10.291 INFO:tasks.rados.rados.0.smithi136.stdout:2822: snap_create 2024-09-12T23:48:10.296 INFO:tasks.rados.rados.0.smithi136.stdout:2821: finishing rollback tid 0 to smithi13667081-50 2024-09-12T23:48:10.297 INFO:tasks.rados.rados.0.smithi136.stdout:2821: finishing rollback tid 1 to smithi13667081-50 2024-09-12T23:48:10.297 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 50 v 1639 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-12T23:48:10.487 INFO:tasks.rados.rados.0.smithi136.stdout:2821: done (1 left) 2024-09-12T23:48:10.487 INFO:tasks.rados.rados.0.smithi136.stdout:2822: done (0 left) 2024-09-12T23:48:10.487 INFO:tasks.rados.rados.0.smithi136.stdout:2823: delete oid 44 current snap is 288 2024-09-12T23:48:10.497 INFO:tasks.rados.rados.0.smithi136.stdout:2823: done (0 left) 2024-09-12T23:48:10.497 INFO:tasks.rados.rados.0.smithi136.stdout:2824: read oid 11 snap -1 2024-09-12T23:48:10.497 INFO:tasks.rados.rados.0.smithi136.stdout:2824: expect (ObjNum 605 snap 174 seq_num 605) 2024-09-12T23:48:10.497 INFO:tasks.rados.rados.0.smithi136.stdout:2825: copy_from oid 15 from oid 43 current snap is 288 2024-09-12T23:48:10.498 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 32 current snap is 288 2024-09-12T23:48:10.498 INFO:tasks.rados.rados.0.smithi136.stdout:2826: seq_num 889 ranges {327680=491520} 2024-09-12T23:48:10.512 INFO:tasks.rados.rados.0.smithi136.stdout:2826: writing smithi13667081-32 from 327680 to 819200 tid 1 2024-09-12T23:48:10.513 INFO:tasks.rados.rados.0.smithi136.stdout:2827: delete oid 37 current snap is 288 2024-09-12T23:48:10.513 INFO:tasks.rados.rados.0.smithi136.stdout:2825: finishing copy_from racing read to smithi13667081-15 2024-09-12T23:48:10.513 INFO:tasks.rados.rados.0.smithi136.stdout:2825: finishing copy_from to smithi13667081-15 2024-09-12T23:48:10.513 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 15 v 1614 (ObjNum 33 snap 0 seq_num 4243905560) dirty exists 2024-09-12T23:48:10.519 INFO:tasks.rados.rados.0.smithi136.stdout:2825: done (3 left) 2024-09-12T23:48:10.519 INFO:tasks.rados.rados.0.smithi136.stdout:2827: done (2 left) 2024-09-12T23:48:10.519 INFO:tasks.rados.rados.0.smithi136.stdout:2828: setattr oid 34 current snap is 288 2024-09-12T23:48:10.521 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 7 current snap is 288 2024-09-12T23:48:10.521 INFO:tasks.rados.rados.0.smithi136.stdout:2829: seq_num 891 ranges {2367488=532480} 2024-09-12T23:48:10.537 INFO:tasks.rados.rados.0.smithi136.stdout:2829: writing smithi13667081-7 from 2367488 to 2899968 tid 1 2024-09-12T23:48:10.537 INFO:tasks.rados.rados.0.smithi136.stdout:2830: setattr oid 43 current snap is 288 2024-09-12T23:48:10.537 INFO:tasks.rados.rados.0.smithi136.stdout:2826: finishing write tid 1 to smithi13667081-32 2024-09-12T23:48:10.537 INFO:tasks.rados.rados.0.smithi136.stdout:2826: finishing write tid 2 to smithi13667081-32 2024-09-12T23:48:10.537 INFO:tasks.rados.rados.0.smithi136.stdout:2826: finishing write tid 3 to smithi13667081-32 2024-09-12T23:48:10.537 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 32 v 1620 (ObjNum 889 snap 288 seq_num 889) dirty exists 2024-09-12T23:48:10.537 INFO:tasks.rados.rados.0.smithi136.stdout:2826: left oid 32 (ObjNum 889 snap 288 seq_num 889) 2024-09-12T23:48:10.538 INFO:tasks.rados.rados.0.smithi136.stdout:2826: done (4 left) 2024-09-12T23:48:10.538 INFO:tasks.rados.rados.0.smithi136.stdout:2831: rollback oid 12 current snap is 288 2024-09-12T23:48:10.539 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 12 to 267 2024-09-12T23:48:10.539 INFO:tasks.rados.rados.0.smithi136.stdout:2832: read oid 41 snap -1 2024-09-12T23:48:10.539 INFO:tasks.rados.rados.0.smithi136.stdout:2832: expect (ObjNum 856 snap 277 seq_num 856) 2024-09-12T23:48:10.539 INFO:tasks.rados.rados.0.smithi136.stdout:2833: read oid 25 snap -1 2024-09-12T23:48:10.539 INFO:tasks.rados.rados.0.smithi136.stdout:2833: expect deleted 2024-09-12T23:48:10.539 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 26 current snap is 288 2024-09-12T23:48:10.539 INFO:tasks.rados.rados.0.smithi136.stdout:2834: seq_num 892 ranges {131072=548864} 2024-09-12T23:48:10.543 INFO:tasks.rados.rados.0.smithi136.stdout:2834: writing smithi13667081-26 from 131072 to 679936 tid 1 2024-09-12T23:48:10.543 INFO:tasks.rados.rados.0.smithi136.stdout:2835: copy_from oid 10 from oid 6 current snap is 288 2024-09-12T23:48:10.543 INFO:tasks.rados.rados.0.smithi136.stdout:2836: snap_remove snap 271 2024-09-12T23:48:11.420 INFO:tasks.rados.rados.0.smithi136.stdout:2836: done (9 left) 2024-09-12T23:48:11.420 INFO:tasks.rados.rados.0.smithi136.stdout:2837: snap_remove snap 286 2024-09-12T23:48:11.676 INFO:teuthology.orchestra.run.smithi136.stdout:true 2024-09-12T23:48:11.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:11 smithi136 bash[75164]: cluster 2024-09-12T23:48:10.489858+0000 mon.a (mon.0) 955 : cluster [DBG] osdmap e633: 8 total, 8 up, 8 in 2024-09-12T23:48:11.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:11 smithi136 bash[75164]: cluster 2024-09-12T23:48:10.793289+0000 mgr.y (mgr.24661) 292 : cluster [DBG] pgmap v526: 121 pgs: 121 active+clean; 422 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 23 MiB/s rd, 3.7 MiB/s wr, 13 op/s 2024-09-12T23:48:11.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:11 smithi136 bash[75164]: cluster 2024-09-12T23:48:11.421253+0000 mon.a (mon.0) 956 : cluster [DBG] osdmap e634: 8 total, 8 up, 8 in 2024-09-12T23:48:11.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:11 smithi136 bash[76371]: cluster 2024-09-12T23:48:10.489858+0000 mon.a (mon.0) 955 : cluster [DBG] osdmap e633: 8 total, 8 up, 8 in 2024-09-12T23:48:11.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:11 smithi136 bash[76371]: cluster 2024-09-12T23:48:10.793289+0000 mgr.y (mgr.24661) 292 : cluster [DBG] pgmap v526: 121 pgs: 121 active+clean; 422 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 23 MiB/s rd, 3.7 MiB/s wr, 13 op/s 2024-09-12T23:48:11.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:11 smithi136 bash[76371]: cluster 2024-09-12T23:48:11.421253+0000 mon.a (mon.0) 956 : cluster [DBG] osdmap e634: 8 total, 8 up, 8 in 2024-09-12T23:48:11.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:11 smithi181 bash[67587]: cluster 2024-09-12T23:48:10.489858+0000 mon.a (mon.0) 955 : cluster [DBG] osdmap e633: 8 total, 8 up, 8 in 2024-09-12T23:48:11.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:11 smithi181 bash[67587]: cluster 2024-09-12T23:48:10.793289+0000 mgr.y (mgr.24661) 292 : cluster [DBG] pgmap v526: 121 pgs: 121 active+clean; 422 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 23 MiB/s rd, 3.7 MiB/s wr, 13 op/s 2024-09-12T23:48:11.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:11 smithi181 bash[67587]: cluster 2024-09-12T23:48:11.421253+0000 mon.a (mon.0) 956 : cluster [DBG] osdmap e634: 8 total, 8 up, 8 in 2024-09-12T23:48:11.969 INFO:teuthology.orchestra.run.smithi136.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-12T23:48:11.969 INFO:teuthology.orchestra.run.smithi136.stdout:alertmanager.a smithi136 *:9093,9094 running (7m) 6s ago 13m 14.2M - 0.25.0 c8568f914cd2 b5197f020eeb 2024-09-12T23:48:11.969 INFO:teuthology.orchestra.run.smithi136.stdout:grafana.a smithi181 *:3000 running (7m) 32s ago 13m 75.5M - 9.4.7 954c08fa6188 256c181be984 2024-09-12T23:48:11.969 INFO:teuthology.orchestra.run.smithi136.stdout:mds.a smithi136 running (13s) 6s ago 14m 11.6M - 18.2.4-837-g7a853b17 f21ba1eea74e a20ac830342c 2024-09-12T23:48:11.969 INFO:teuthology.orchestra.run.smithi136.stdout:mds.b smithi181 running (14m) 32s ago 14m 19.9M - 17.2.7-1627-gbbdadfa3 214985049bdb ad13179f0868 2024-09-12T23:48:11.970 INFO:teuthology.orchestra.run.smithi136.stdout:mgr.x smithi181 *:8443,9283,8765 running (6m) 32s ago 22m 427M - 18.2.4-837-g7a853b17 f21ba1eea74e cc664dc337f7 2024-09-12T23:48:11.970 INFO:teuthology.orchestra.run.smithi136.stdout:mgr.y smithi136 *:8443,9283,8765 running (7m) 6s ago 24m 495M - 18.2.4-837-g7a853b17 f21ba1eea74e 272f10b42fc3 2024-09-12T23:48:11.970 INFO:teuthology.orchestra.run.smithi136.stdout:mon.a smithi136 running (6m) 6s ago 24m 66.2M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e ad8d21d58ad9 2024-09-12T23:48:11.970 INFO:teuthology.orchestra.run.smithi136.stdout:mon.b smithi181 running (5m) 32s ago 22m 45.1M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e a70ec8315934 2024-09-12T23:48:11.970 INFO:teuthology.orchestra.run.smithi136.stdout:mon.c smithi136 running (5m) 6s ago 22m 54.9M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e 44d1be055a5e 2024-09-12T23:48:11.970 INFO:teuthology.orchestra.run.smithi136.stdout:node-exporter.a smithi136 *:9100 running (7m) 6s ago 13m 8515k - 1.5.0 0da6a335fe13 ccfc37c96620 2024-09-12T23:48:11.970 INFO:teuthology.orchestra.run.smithi136.stdout:node-exporter.b smithi181 *:9100 running (7m) 32s ago 13m 8307k - 1.5.0 0da6a335fe13 80bf0e383230 2024-09-12T23:48:11.970 INFO:teuthology.orchestra.run.smithi136.stdout:osd.0 smithi136 running (3m) 6s ago 21m 299M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 7da21f765721 2024-09-12T23:48:11.970 INFO:teuthology.orchestra.run.smithi136.stdout:osd.1 smithi136 running (3m) 6s ago 20m 315M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 5302be0a016b 2024-09-12T23:48:11.970 INFO:teuthology.orchestra.run.smithi136.stdout:osd.2 smithi136 running (3m) 6s ago 19m 323M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 2e12d1a31223 2024-09-12T23:48:11.970 INFO:teuthology.orchestra.run.smithi136.stdout:osd.3 smithi136 running (2m) 6s ago 18m 245M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e aa9dc124e59c 2024-09-12T23:48:11.970 INFO:teuthology.orchestra.run.smithi136.stdout:osd.4 smithi181 running (2m) 32s ago 18m 377M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 4ee3b228ce7d 2024-09-12T23:48:11.970 INFO:teuthology.orchestra.run.smithi136.stdout:osd.5 smithi181 running (95s) 32s ago 17m 194M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 3f9a09565258 2024-09-12T23:48:11.970 INFO:teuthology.orchestra.run.smithi136.stdout:osd.6 smithi181 running (66s) 32s ago 16m 115M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e ce539f6fb854 2024-09-12T23:48:11.971 INFO:teuthology.orchestra.run.smithi136.stdout:osd.7 smithi181 running (37s) 32s ago 15m - 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 6ad9ab169a7b 2024-09-12T23:48:11.971 INFO:teuthology.orchestra.run.smithi136.stdout:prometheus.a smithi181 *:9095 running (6m) 32s ago 14m 57.5M - 2.43.0 a07b618ecd1d 3853d003a5cc 2024-09-12T23:48:12.311 INFO:teuthology.orchestra.run.smithi136.stdout:{ 2024-09-12T23:48:12.311 INFO:teuthology.orchestra.run.smithi136.stdout: "mon": { 2024-09-12T23:48:12.311 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 3 2024-09-12T23:48:12.311 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:48:12.311 INFO:teuthology.orchestra.run.smithi136.stdout: "mgr": { 2024-09-12T23:48:12.311 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 2 2024-09-12T23:48:12.311 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:48:12.311 INFO:teuthology.orchestra.run.smithi136.stdout: "osd": { 2024-09-12T23:48:12.311 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 8 2024-09-12T23:48:12.312 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:48:12.312 INFO:teuthology.orchestra.run.smithi136.stdout: "mds": { 2024-09-12T23:48:12.312 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 1, 2024-09-12T23:48:12.312 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 1 2024-09-12T23:48:12.312 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:48:12.312 INFO:teuthology.orchestra.run.smithi136.stdout: "overall": { 2024-09-12T23:48:12.312 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 1, 2024-09-12T23:48:12.312 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 14 2024-09-12T23:48:12.312 INFO:teuthology.orchestra.run.smithi136.stdout: } 2024-09-12T23:48:12.312 INFO:teuthology.orchestra.run.smithi136.stdout:} 2024-09-12T23:48:12.428 INFO:tasks.rados.rados.0.smithi136.stdout:2824: expect (ObjNum 605 snap 174 seq_num 605) 2024-09-12T23:48:12.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:12 smithi181 bash[67587]: audit 2024-09-12T23:48:11.668325+0000 mgr.y (mgr.24661) 293 : audit [DBG] from='client.34404 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:48:12.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:12 smithi181 bash[67587]: audit 2024-09-12T23:48:12.314818+0000 mon.a (mon.0) 957 : audit [DBG] from='client.? 172.21.15.136:0/2678423016' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:12.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:12 smithi181 bash[67587]: cluster 2024-09-12T23:48:12.424645+0000 mon.a (mon.0) 958 : cluster [DBG] osdmap e635: 8 total, 8 up, 8 in 2024-09-12T23:48:12.594 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:48:12 smithi181 bash[60471]: ts=2024-09-12T23:48:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:48:12.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:12 smithi136 bash[75164]: audit 2024-09-12T23:48:11.668325+0000 mgr.y (mgr.24661) 293 : audit [DBG] from='client.34404 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:48:12.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:12 smithi136 bash[75164]: audit 2024-09-12T23:48:12.314818+0000 mon.a (mon.0) 957 : audit [DBG] from='client.? 172.21.15.136:0/2678423016' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:12.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:12 smithi136 bash[75164]: cluster 2024-09-12T23:48:12.424645+0000 mon.a (mon.0) 958 : cluster [DBG] osdmap e635: 8 total, 8 up, 8 in 2024-09-12T23:48:12.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:12 smithi136 bash[76371]: audit 2024-09-12T23:48:11.668325+0000 mgr.y (mgr.24661) 293 : audit [DBG] from='client.34404 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:48:12.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:12 smithi136 bash[76371]: audit 2024-09-12T23:48:12.314818+0000 mon.a (mon.0) 957 : audit [DBG] from='client.? 172.21.15.136:0/2678423016' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:12.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:12 smithi136 bash[76371]: cluster 2024-09-12T23:48:12.424645+0000 mon.a (mon.0) 958 : cluster [DBG] osdmap e635: 8 total, 8 up, 8 in 2024-09-12T23:48:12.989 INFO:tasks.rados.rados.0.smithi136.stdout:2835: finishing copy_from to smithi13667081-10 2024-09-12T23:48:12.989 INFO:tasks.rados.rados.0.smithi136.stdout:2835: got expected ENOENT (src dne) 2024-09-12T23:48:12.989 INFO:tasks.rados.rados.0.smithi136.stdout:2835: finishing copy_from racing read to smithi13667081-10 2024-09-12T23:48:12.989 INFO:tasks.rados.rados.0.smithi136.stdout:2831: finishing rollback tid 0 to smithi13667081-12 2024-09-12T23:48:12.989 INFO:tasks.rados.rados.0.smithi136.stdout:2831: finishing rollback tid 1 to smithi13667081-12 2024-09-12T23:48:12.990 INFO:tasks.rados.rados.0.smithi136.stdout:2824: done (9 left) 2024-09-12T23:48:12.990 INFO:tasks.rados.rados.0.smithi136.stdout:2833: done (8 left) 2024-09-12T23:48:12.990 INFO:tasks.rados.rados.0.smithi136.stdout:2835: done (7 left) 2024-09-12T23:48:12.990 INFO:tasks.rados.rados.0.smithi136.stdout:2837: done (6 left) 2024-09-12T23:48:12.990 INFO:tasks.rados.rados.0.smithi136.stdout:2838: rollback oid 5 current snap is 288 2024-09-12T23:48:12.990 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 5 to 280 2024-09-12T23:48:12.990 INFO:tasks.rados.rados.0.smithi136.stdout:2829: finishing write tid 1 to smithi13667081-7 2024-09-12T23:48:12.990 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 43 v 1715 (ObjNum 65 snap 0 seq_num 1869900386) dirty exists 2024-09-12T23:48:12.990 INFO:tasks.rados.rados.0.smithi136.stdout:2831: finishing rollback tid 2 to smithi13667081-12 2024-09-12T23:48:12.990 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 12 v 1642 (ObjNum 824 snap 260 seq_num 824) dirty exists 2024-09-12T23:48:12.990 INFO:tasks.rados.rados.0.smithi136.stdout:2829: finishing write tid 2 to smithi13667081-7 2024-09-12T23:48:12.990 INFO:tasks.rados.rados.0.smithi136.stdout:2834: finishing write tid 1 to smithi13667081-26 2024-09-12T23:48:12.990 INFO:tasks.rados.rados.0.smithi136.stdout:2834: finishing write tid 2 to smithi13667081-26 2024-09-12T23:48:12.990 INFO:tasks.rados.rados.0.smithi136.stdout:2829: finishing write tid 3 to smithi13667081-7 2024-09-12T23:48:12.991 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 7 v 966 (ObjNum 891 snap 288 seq_num 891) dirty exists 2024-09-12T23:48:12.991 INFO:tasks.rados.rados.0.smithi136.stdout:2829: left oid 7 (ObjNum 891 snap 288 seq_num 891) 2024-09-12T23:48:12.991 INFO:tasks.rados.rados.0.smithi136.stdout:2834: finishing write tid 3 to smithi13667081-26 2024-09-12T23:48:12.991 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 26 v 1491 (ObjNum 892 snap 288 seq_num 892) dirty exists 2024-09-12T23:48:12.991 INFO:tasks.rados.rados.0.smithi136.stdout:2834: left oid 26 (ObjNum 892 snap 288 seq_num 892) 2024-09-12T23:48:12.991 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 34 v 1317 (ObjNum 796 snap 253 seq_num 796) dirty exists 2024-09-12T23:48:12.991 INFO:tasks.rados.rados.0.smithi136.stdout:2832: expect (ObjNum 856 snap 277 seq_num 856) 2024-09-12T23:48:13.233 INFO:tasks.rados.rados.0.smithi136.stdout:2838: finishing rollback tid 0 to smithi13667081-5 2024-09-12T23:48:13.233 INFO:tasks.rados.rados.0.smithi136.stdout:2838: finishing rollback tid 1 to smithi13667081-5 2024-09-12T23:48:13.233 INFO:tasks.rados.rados.0.smithi136.stdout:2838: finishing rollback tid 2 to smithi13667081-5 2024-09-12T23:48:13.233 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 5 v 2010 (ObjNum 567 snap 162 seq_num 567) dirty exists 2024-09-12T23:48:13.233 INFO:tasks.rados.rados.0.smithi136.stdout:2828: done (6 left) 2024-09-12T23:48:13.233 INFO:tasks.rados.rados.0.smithi136.stdout:2829: done (5 left) 2024-09-12T23:48:13.234 INFO:tasks.rados.rados.0.smithi136.stdout:2830: done (4 left) 2024-09-12T23:48:13.234 INFO:tasks.rados.rados.0.smithi136.stdout:2831: done (3 left) 2024-09-12T23:48:13.234 INFO:tasks.rados.rados.0.smithi136.stdout:2832: done (2 left) 2024-09-12T23:48:13.234 INFO:tasks.rados.rados.0.smithi136.stdout:2834: done (1 left) 2024-09-12T23:48:13.234 INFO:tasks.rados.rados.0.smithi136.stdout:2838: done (0 left) 2024-09-12T23:48:13.234 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 13 current snap is 288 2024-09-12T23:48:13.234 INFO:tasks.rados.rados.0.smithi136.stdout:2839: seq_num 893 ranges {8282112=565248} 2024-09-12T23:48:13.285 INFO:tasks.rados.rados.0.smithi136.stdout:2839: writing smithi13667081-13 from 8282112 to 8847360 tid 1 2024-09-12T23:48:13.286 INFO:tasks.rados.rados.0.smithi136.stdout:2840: rollback oid 19 current snap is 288 2024-09-12T23:48:13.286 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 19 to 285 2024-09-12T23:48:13.286 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 33 current snap is 288 2024-09-12T23:48:13.286 INFO:tasks.rados.rados.0.smithi136.stdout:2841: seq_num 894 ranges {0=581632} 2024-09-12T23:48:13.289 INFO:tasks.rados.rados.0.smithi136.stdout:2841: writing smithi13667081-33 from 0 to 581632 tid 1 2024-09-12T23:48:13.289 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 42 current snap is 288 2024-09-12T23:48:13.289 INFO:tasks.rados.rados.0.smithi136.stdout:2840: finishing rollback tid 1 to smithi13667081-19 2024-09-12T23:48:13.289 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 19 v 1604 (ObjNum 33 snap 0 seq_num 4243914416) dirty dne 2024-09-12T23:48:13.289 INFO:tasks.rados.rados.0.smithi136.stdout:2842: seq_num 895 ranges {9576448=598016} 2024-09-12T23:48:13.347 INFO:tasks.rados.rados.0.smithi136.stdout:2842: writing smithi13667081-42 from 9576448 to 10174464 tid 1 2024-09-12T23:48:13.347 INFO:tasks.rados.rados.0.smithi136.stdout:2840: done (3 left) 2024-09-12T23:48:13.347 INFO:tasks.rados.rados.0.smithi136.stdout:2843: read oid 37 snap -1 2024-09-12T23:48:13.347 INFO:tasks.rados.rados.0.smithi136.stdout:2843: expect deleted 2024-09-12T23:48:13.347 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 22 current snap is 288 2024-09-12T23:48:13.347 INFO:tasks.rados.rados.0.smithi136.stdout:2839: finishing write tid 1 to smithi13667081-13 2024-09-12T23:48:13.347 INFO:tasks.rados.rados.0.smithi136.stdout:2839: finishing write tid 2 to smithi13667081-13 2024-09-12T23:48:13.347 INFO:tasks.rados.rados.0.smithi136.stdout:2844: seq_num 896 ranges {0=614400} 2024-09-12T23:48:13.350 INFO:tasks.rados.rados.0.smithi136.stdout:2844: writing smithi13667081-22 from 0 to 614400 tid 1 2024-09-12T23:48:13.350 INFO:tasks.rados.rados.0.smithi136.stdout:2845: copy_from oid 17 from oid 20 current snap is 288 2024-09-12T23:48:13.350 INFO:tasks.rados.rados.0.smithi136.stdout:2841: finishing write tid 1 to smithi13667081-33 2024-09-12T23:48:13.350 INFO:tasks.rados.rados.0.smithi136.stdout:2839: finishing write tid 3 to smithi13667081-13 2024-09-12T23:48:13.351 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 13 v 1641 (ObjNum 893 snap 288 seq_num 893) dirty exists 2024-09-12T23:48:13.351 INFO:tasks.rados.rados.0.smithi136.stdout:2839: left oid 13 (ObjNum 893 snap 288 seq_num 893) 2024-09-12T23:48:13.351 INFO:tasks.rados.rados.0.smithi136.stdout:2841: finishing write tid 2 to smithi13667081-33 2024-09-12T23:48:13.351 INFO:tasks.rados.rados.0.smithi136.stdout:2841: finishing write tid 3 to smithi13667081-33 2024-09-12T23:48:13.351 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 33 v 1331 (ObjNum 894 snap 288 seq_num 894) dirty exists 2024-09-12T23:48:13.351 INFO:tasks.rados.rados.0.smithi136.stdout:2841: left oid 33 (ObjNum 894 snap 288 seq_num 894) 2024-09-12T23:48:13.351 INFO:tasks.rados.rados.0.smithi136.stdout:2839: done (5 left) 2024-09-12T23:48:13.351 INFO:tasks.rados.rados.0.smithi136.stdout:2841: done (4 left) 2024-09-12T23:48:13.351 INFO:tasks.rados.rados.0.smithi136.stdout:2843: done (3 left) 2024-09-12T23:48:13.351 INFO:tasks.rados.rados.0.smithi136.stdout:2846: read oid 44 snap -1 2024-09-12T23:48:13.351 INFO:tasks.rados.rados.0.smithi136.stdout:2846: expect deleted 2024-09-12T23:48:13.351 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 14 current snap is 288 2024-09-12T23:48:13.351 INFO:tasks.rados.rados.0.smithi136.stdout:2847: seq_num 897 ranges {0=630784} 2024-09-12T23:48:13.354 INFO:tasks.rados.rados.0.smithi136.stdout:2847: writing smithi13667081-14 from 0 to 630784 tid 1 2024-09-12T23:48:13.354 INFO:tasks.rados.rados.0.smithi136.stdout:2848: snap_remove snap 240 2024-09-12T23:48:13.508 INFO:tasks.rados.rados.0.smithi136.stdout:2848: done (5 left) 2024-09-12T23:48:13.509 INFO:tasks.rados.rados.0.smithi136.stdout:2849: snap_remove snap 282 2024-09-12T23:48:13.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:13 smithi136 bash[75164]: audit 2024-09-12T23:48:11.956362+0000 mgr.y (mgr.24661) 294 : audit [DBG] from='client.34410 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:48:13.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:13 smithi136 bash[75164]: cluster 2024-09-12T23:48:12.794161+0000 mgr.y (mgr.24661) 295 : cluster [DBG] pgmap v529: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 427 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 545 KiB/s wr, 24 op/s 2024-09-12T23:48:13.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:13 smithi136 bash[76371]: audit 2024-09-12T23:48:11.956362+0000 mgr.y (mgr.24661) 294 : audit [DBG] from='client.34410 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:48:13.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:13 smithi136 bash[76371]: cluster 2024-09-12T23:48:12.794161+0000 mgr.y (mgr.24661) 295 : cluster [DBG] pgmap v529: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 427 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 545 KiB/s wr, 24 op/s 2024-09-12T23:48:13.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:13 smithi181 bash[67587]: audit 2024-09-12T23:48:11.956362+0000 mgr.y (mgr.24661) 294 : audit [DBG] from='client.34410 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:48:13.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:13 smithi181 bash[67587]: cluster 2024-09-12T23:48:12.794161+0000 mgr.y (mgr.24661) 295 : cluster [DBG] pgmap v529: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 427 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 545 KiB/s wr, 24 op/s 2024-09-12T23:48:14.518 INFO:tasks.rados.rados.0.smithi136.stdout:2845: finishing copy_from to smithi13667081-17 2024-09-12T23:48:14.518 INFO:tasks.rados.rados.0.smithi136.stdout:2845: got expected ENOENT (src dne) 2024-09-12T23:48:14.518 INFO:tasks.rados.rados.0.smithi136.stdout:2845: finishing copy_from racing read to smithi13667081-17 2024-09-12T23:48:14.518 INFO:tasks.rados.rados.0.smithi136.stdout:2842: finishing write tid 1 to smithi13667081-42 2024-09-12T23:48:14.518 INFO:tasks.rados.rados.0.smithi136.stdout:2842: finishing write tid 2 to smithi13667081-42 2024-09-12T23:48:14.518 INFO:tasks.rados.rados.0.smithi136.stdout:2844: finishing write tid 1 to smithi13667081-22 2024-09-12T23:48:14.518 INFO:tasks.rados.rados.0.smithi136.stdout:2844: finishing write tid 2 to smithi13667081-22 2024-09-12T23:48:14.518 INFO:tasks.rados.rados.0.smithi136.stdout:2847: finishing write tid 1 to smithi13667081-14 2024-09-12T23:48:14.519 INFO:tasks.rados.rados.0.smithi136.stdout:2847: finishing write tid 2 to smithi13667081-14 2024-09-12T23:48:14.519 INFO:tasks.rados.rados.0.smithi136.stdout:2842: finishing write tid 3 to smithi13667081-42 2024-09-12T23:48:14.519 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 42 v 1793 (ObjNum 895 snap 288 seq_num 895) dirty exists 2024-09-12T23:48:14.519 INFO:tasks.rados.rados.0.smithi136.stdout:2842: left oid 42 (ObjNum 895 snap 288 seq_num 895) 2024-09-12T23:48:14.519 INFO:tasks.rados.rados.0.smithi136.stdout:2844: finishing write tid 3 to smithi13667081-22 2024-09-12T23:48:14.519 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 22 v 1795 (ObjNum 896 snap 288 seq_num 896) dirty exists 2024-09-12T23:48:14.519 INFO:tasks.rados.rados.0.smithi136.stdout:2844: left oid 22 (ObjNum 896 snap 288 seq_num 896) 2024-09-12T23:48:14.519 INFO:tasks.rados.rados.0.smithi136.stdout:2847: finishing write tid 3 to smithi13667081-14 2024-09-12T23:48:14.519 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 14 v 1725 (ObjNum 897 snap 288 seq_num 897) dirty exists 2024-09-12T23:48:14.519 INFO:tasks.rados.rados.0.smithi136.stdout:2847: left oid 14 (ObjNum 897 snap 288 seq_num 897) 2024-09-12T23:48:14.519 INFO:tasks.rados.rados.0.smithi136.stdout:2842: done (5 left) 2024-09-12T23:48:14.519 INFO:tasks.rados.rados.0.smithi136.stdout:2844: done (4 left) 2024-09-12T23:48:14.519 INFO:tasks.rados.rados.0.smithi136.stdout:2845: done (3 left) 2024-09-12T23:48:14.519 INFO:tasks.rados.rados.0.smithi136.stdout:2846: done (2 left) 2024-09-12T23:48:14.520 INFO:tasks.rados.rados.0.smithi136.stdout:2847: done (1 left) 2024-09-12T23:48:14.520 INFO:tasks.rados.rados.0.smithi136.stdout:2849: done (0 left) 2024-09-12T23:48:14.520 INFO:tasks.rados.rados.0.smithi136.stdout:2850: delete oid 38 current snap is 288 2024-09-12T23:48:14.541 INFO:tasks.rados.rados.0.smithi136.stdout:2850: done (0 left) 2024-09-12T23:48:14.541 INFO:tasks.rados.rados.0.smithi136.stdout:2851: rollback oid 41 current snap is 288 2024-09-12T23:48:14.541 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 41 to 273 2024-09-12T23:48:14.541 INFO:tasks.rados.rados.0.smithi136.stdout:2852: copy_from oid 2 from oid 3 current snap is 288 2024-09-12T23:48:14.541 INFO:tasks.rados.rados.0.smithi136.stdout:2853: delete oid 12 current snap is 288 2024-09-12T23:48:14.549 INFO:tasks.rados.rados.0.smithi136.stdout:2853: done (2 left) 2024-09-12T23:48:14.550 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 46 current snap is 288 2024-09-12T23:48:14.550 INFO:tasks.rados.rados.0.smithi136.stdout:2854: seq_num 900 ranges {1777664=679936} 2024-09-12T23:48:14.569 INFO:tasks.rados.rados.0.smithi136.stdout:2854: writing smithi13667081-46 from 1777664 to 2457600 tid 1 2024-09-12T23:48:14.569 INFO:tasks.rados.rados.0.smithi136.stdout:2855: copy_from oid 30 from oid 25 current snap is 288 2024-09-12T23:48:14.569 INFO:tasks.rados.rados.0.smithi136.stdout:2851: finishing rollback tid 0 to smithi13667081-41 2024-09-12T23:48:14.569 INFO:tasks.rados.rados.0.smithi136.stdout:2851: finishing rollback tid 1 to smithi13667081-41 2024-09-12T23:48:14.569 INFO:tasks.rados.rados.0.smithi136.stdout:2851: finishing rollback tid 2 to smithi13667081-41 2024-09-12T23:48:14.569 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 41 v 1351 (ObjNum 851 snap 272 seq_num 851) dirty exists 2024-09-12T23:48:14.569 INFO:tasks.rados.rados.0.smithi136.stdout:2852: finishing copy_from racing read to smithi13667081-2 2024-09-12T23:48:14.569 INFO:tasks.rados.rados.0.smithi136.stdout:2852: finishing copy_from to smithi13667081-2 2024-09-12T23:48:14.569 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 2 v 1489 (ObjNum 838 snap 267 seq_num 838) dirty exists 2024-09-12T23:48:14.569 INFO:tasks.rados.rados.0.smithi136.stdout:2851: done (3 left) 2024-09-12T23:48:14.569 INFO:tasks.rados.rados.0.smithi136.stdout:2852: done (2 left) 2024-09-12T23:48:14.569 INFO:tasks.rados.rados.0.smithi136.stdout:2856: snap_remove snap 211 2024-09-12T23:48:14.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:14 smithi181 bash[67587]: cluster 2024-09-12T23:48:13.511043+0000 mon.a (mon.0) 959 : cluster [DBG] osdmap e636: 8 total, 8 up, 8 in 2024-09-12T23:48:15.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:14 smithi136 bash[75164]: cluster 2024-09-12T23:48:13.511043+0000 mon.a (mon.0) 959 : cluster [DBG] osdmap e636: 8 total, 8 up, 8 in 2024-09-12T23:48:15.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:14 smithi136 bash[76371]: cluster 2024-09-12T23:48:13.511043+0000 mon.a (mon.0) 959 : cluster [DBG] osdmap e636: 8 total, 8 up, 8 in 2024-09-12T23:48:15.551 INFO:tasks.rados.rados.0.smithi136.stdout:2856: done (2 left) 2024-09-12T23:48:15.552 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 40 current snap is 288 2024-09-12T23:48:15.552 INFO:tasks.rados.rados.0.smithi136.stdout:2857: seq_num 901 ranges {1589248=696320} 2024-09-12T23:48:15.580 INFO:tasks.rados.rados.0.smithi136.stdout:2857: writing smithi13667081-40 from 1589248 to 2285568 tid 1 2024-09-12T23:48:15.580 INFO:tasks.rados.rados.0.smithi136.stdout:2858: delete oid 26 current snap is 288 2024-09-12T23:48:15.580 INFO:tasks.rados.rados.0.smithi136.stdout:2855: finishing copy_from to smithi13667081-30 2024-09-12T23:48:15.580 INFO:tasks.rados.rados.0.smithi136.stdout:2855: got expected ENOENT (src dne) 2024-09-12T23:48:15.580 INFO:tasks.rados.rados.0.smithi136.stdout:2855: finishing copy_from racing read to smithi13667081-30 2024-09-12T23:48:15.580 INFO:tasks.rados.rados.0.smithi136.stdout:2854: finishing write tid 1 to smithi13667081-46 2024-09-12T23:48:15.580 INFO:tasks.rados.rados.0.smithi136.stdout:2854: finishing write tid 2 to smithi13667081-46 2024-09-12T23:48:15.581 INFO:tasks.rados.rados.0.smithi136.stdout:2854: finishing write tid 3 to smithi13667081-46 2024-09-12T23:48:15.581 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 46 v 1493 (ObjNum 900 snap 288 seq_num 900) dirty exists 2024-09-12T23:48:15.581 INFO:tasks.rados.rados.0.smithi136.stdout:2854: left oid 46 (ObjNum 900 snap 288 seq_num 900) 2024-09-12T23:48:15.599 INFO:tasks.rados.rados.0.smithi136.stdout:2857: finishing write tid 1 to smithi13667081-40 2024-09-12T23:48:15.599 INFO:tasks.rados.rados.0.smithi136.stdout:2857: finishing write tid 2 to smithi13667081-40 2024-09-12T23:48:15.601 INFO:tasks.rados.rados.0.smithi136.stdout:2854: done (3 left) 2024-09-12T23:48:15.601 INFO:tasks.rados.rados.0.smithi136.stdout:2855: done (2 left) 2024-09-12T23:48:15.601 INFO:tasks.rados.rados.0.smithi136.stdout:2858: done (1 left) 2024-09-12T23:48:15.601 INFO:tasks.rados.rados.0.smithi136.stdout:2859: delete oid 28 current snap is 288 2024-09-12T23:48:15.602 INFO:tasks.rados.rados.0.smithi136.stdout:2857: finishing write tid 3 to smithi13667081-40 2024-09-12T23:48:15.602 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 40 v 2035 (ObjNum 901 snap 288 seq_num 901) dirty exists 2024-09-12T23:48:15.603 INFO:tasks.rados.rados.0.smithi136.stdout:2857: left oid 40 (ObjNum 901 snap 288 seq_num 901) 2024-09-12T23:48:15.604 INFO:tasks.rados.rados.0.smithi136.stdout:2857: done (1 left) 2024-09-12T23:48:15.604 INFO:tasks.rados.rados.0.smithi136.stdout:2859: done (0 left) 2024-09-12T23:48:15.604 INFO:tasks.rados.rados.0.smithi136.stdout:2860: snap_remove snap 285 2024-09-12T23:48:15.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:15 smithi136 bash[75164]: cluster 2024-09-12T23:48:14.513734+0000 mon.a (mon.0) 960 : cluster [DBG] osdmap e637: 8 total, 8 up, 8 in 2024-09-12T23:48:15.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:15 smithi136 bash[75164]: cluster 2024-09-12T23:48:14.794959+0000 mgr.y (mgr.24661) 296 : cluster [DBG] pgmap v532: 121 pgs: 3 active+clean+snaptrim, 118 active+clean; 430 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 1000 KiB/s wr, 27 op/s 2024-09-12T23:48:15.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:15 smithi136 bash[76371]: cluster 2024-09-12T23:48:14.513734+0000 mon.a (mon.0) 960 : cluster [DBG] osdmap e637: 8 total, 8 up, 8 in 2024-09-12T23:48:15.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:15 smithi136 bash[76371]: cluster 2024-09-12T23:48:14.794959+0000 mgr.y (mgr.24661) 296 : cluster [DBG] pgmap v532: 121 pgs: 3 active+clean+snaptrim, 118 active+clean; 430 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 1000 KiB/s wr, 27 op/s 2024-09-12T23:48:15.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:15 smithi181 bash[67587]: cluster 2024-09-12T23:48:14.513734+0000 mon.a (mon.0) 960 : cluster [DBG] osdmap e637: 8 total, 8 up, 8 in 2024-09-12T23:48:15.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:15 smithi181 bash[67587]: cluster 2024-09-12T23:48:14.794959+0000 mgr.y (mgr.24661) 296 : cluster [DBG] pgmap v532: 121 pgs: 3 active+clean+snaptrim, 118 active+clean; 430 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 1000 KiB/s wr, 27 op/s 2024-09-12T23:48:16.433 INFO:tasks.rados.rados.0.smithi136.stdout:2860: done (0 left) 2024-09-12T23:48:16.433 INFO:tasks.rados.rados.0.smithi136.stdout:2861: snap_create 2024-09-12T23:48:16.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:16 smithi181 bash[67587]: cluster 2024-09-12T23:48:15.548039+0000 mon.a (mon.0) 961 : cluster [DBG] osdmap e638: 8 total, 8 up, 8 in 2024-09-12T23:48:16.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:16 smithi181 bash[67587]: audit 2024-09-12T23:48:16.157286+0000 mon.a (mon.0) 962 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:16.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:16 smithi181 bash[67587]: audit 2024-09-12T23:48:16.167795+0000 mon.a (mon.0) 963 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:16.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:16 smithi181 bash[67587]: audit 2024-09-12T23:48:16.169748+0000 mon.a (mon.0) 964 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:48:16.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:16 smithi181 bash[67587]: audit 2024-09-12T23:48:16.171409+0000 mon.a (mon.0) 965 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:48:16.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:16 smithi181 bash[67587]: audit 2024-09-12T23:48:16.181208+0000 mon.a (mon.0) 966 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:16.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:16 smithi181 bash[67587]: audit 2024-09-12T23:48:16.307514+0000 mon.a (mon.0) 967 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:48:16.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:16 smithi181 bash[67587]: audit 2024-09-12T23:48:16.312327+0000 mon.a (mon.0) 968 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:16.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:16 smithi181 bash[67587]: audit 2024-09-12T23:48:16.315025+0000 mon.a (mon.0) 969 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:16.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:16 smithi181 bash[67587]: audit 2024-09-12T23:48:16.316986+0000 mon.a (mon.0) 970 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:16.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:16 smithi181 bash[67587]: audit 2024-09-12T23:48:16.320014+0000 mon.a (mon.0) 971 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:16.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:16 smithi181 bash[67587]: audit 2024-09-12T23:48:16.323519+0000 mon.a (mon.0) 972 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mds ok-to-stop", "ids": ["b"]}]: dispatch 2024-09-12T23:48:16.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:16 smithi181 bash[67587]: cluster 2024-09-12T23:48:16.431602+0000 mon.a (mon.0) 973 : cluster [DBG] osdmap e639: 8 total, 8 up, 8 in 2024-09-12T23:48:17.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:16 smithi136 bash[75164]: cluster 2024-09-12T23:48:15.548039+0000 mon.a (mon.0) 961 : cluster [DBG] osdmap e638: 8 total, 8 up, 8 in 2024-09-12T23:48:17.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:16 smithi136 bash[75164]: audit 2024-09-12T23:48:16.157286+0000 mon.a (mon.0) 962 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:17.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:16 smithi136 bash[75164]: audit 2024-09-12T23:48:16.167795+0000 mon.a (mon.0) 963 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:17.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:16 smithi136 bash[75164]: audit 2024-09-12T23:48:16.169748+0000 mon.a (mon.0) 964 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:48:17.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:16 smithi136 bash[75164]: audit 2024-09-12T23:48:16.171409+0000 mon.a (mon.0) 965 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:48:17.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:16 smithi136 bash[75164]: audit 2024-09-12T23:48:16.181208+0000 mon.a (mon.0) 966 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:17.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:16 smithi136 bash[75164]: audit 2024-09-12T23:48:16.307514+0000 mon.a (mon.0) 967 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:48:17.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:16 smithi136 bash[75164]: audit 2024-09-12T23:48:16.312327+0000 mon.a (mon.0) 968 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:17.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:16 smithi136 bash[75164]: audit 2024-09-12T23:48:16.315025+0000 mon.a (mon.0) 969 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:17.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:16 smithi136 bash[75164]: audit 2024-09-12T23:48:16.316986+0000 mon.a (mon.0) 970 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:17.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:16 smithi136 bash[75164]: audit 2024-09-12T23:48:16.320014+0000 mon.a (mon.0) 971 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:17.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:16 smithi136 bash[75164]: audit 2024-09-12T23:48:16.323519+0000 mon.a (mon.0) 972 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mds ok-to-stop", "ids": ["b"]}]: dispatch 2024-09-12T23:48:17.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:16 smithi136 bash[75164]: cluster 2024-09-12T23:48:16.431602+0000 mon.a (mon.0) 973 : cluster [DBG] osdmap e639: 8 total, 8 up, 8 in 2024-09-12T23:48:17.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:16 smithi136 bash[76371]: cluster 2024-09-12T23:48:15.548039+0000 mon.a (mon.0) 961 : cluster [DBG] osdmap e638: 8 total, 8 up, 8 in 2024-09-12T23:48:17.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:16 smithi136 bash[76371]: audit 2024-09-12T23:48:16.157286+0000 mon.a (mon.0) 962 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:17.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:16 smithi136 bash[76371]: audit 2024-09-12T23:48:16.167795+0000 mon.a (mon.0) 963 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:17.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:16 smithi136 bash[76371]: audit 2024-09-12T23:48:16.169748+0000 mon.a (mon.0) 964 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:48:17.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:16 smithi136 bash[76371]: audit 2024-09-12T23:48:16.171409+0000 mon.a (mon.0) 965 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:48:17.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:16 smithi136 bash[76371]: audit 2024-09-12T23:48:16.181208+0000 mon.a (mon.0) 966 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:17.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:16 smithi136 bash[76371]: audit 2024-09-12T23:48:16.307514+0000 mon.a (mon.0) 967 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:48:17.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:16 smithi136 bash[76371]: audit 2024-09-12T23:48:16.312327+0000 mon.a (mon.0) 968 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:17.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:16 smithi136 bash[76371]: audit 2024-09-12T23:48:16.315025+0000 mon.a (mon.0) 969 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:17.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:16 smithi136 bash[76371]: audit 2024-09-12T23:48:16.316986+0000 mon.a (mon.0) 970 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:17.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:16 smithi136 bash[76371]: audit 2024-09-12T23:48:16.320014+0000 mon.a (mon.0) 971 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:17.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:16 smithi136 bash[76371]: audit 2024-09-12T23:48:16.323519+0000 mon.a (mon.0) 972 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mds ok-to-stop", "ids": ["b"]}]: dispatch 2024-09-12T23:48:17.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:16 smithi136 bash[76371]: cluster 2024-09-12T23:48:16.431602+0000 mon.a (mon.0) 973 : cluster [DBG] osdmap e639: 8 total, 8 up, 8 in 2024-09-12T23:48:17.429 INFO:tasks.rados.rados.0.smithi136.stdout:2861: done (0 left) 2024-09-12T23:48:17.430 INFO:tasks.rados.rados.0.smithi136.stdout:2862: rmattr oid 49 current snap is 289 2024-09-12T23:48:17.430 INFO:tasks.rados.rados.0.smithi136.stdout:2863: snap_remove snap 247 2024-09-12T23:48:17.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:17 smithi181 bash[67587]: cephadm 2024-09-12T23:48:16.147771+0000 mgr.y (mgr.24661) 297 : cephadm [INF] Detected new or changed devices on smithi136 2024-09-12T23:48:17.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:17 smithi181 bash[67587]: cephadm 2024-09-12T23:48:16.324485+0000 mgr.y (mgr.24661) 298 : cephadm [INF] Upgrade: It appears safe to stop mds.b 2024-09-12T23:48:17.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:17 smithi181 bash[67587]: cluster 2024-09-12T23:48:16.795942+0000 mgr.y (mgr.24661) 299 : cluster [DBG] pgmap v535: 121 pgs: 2 active+clean+snaptrim_wait, 3 active+clean+snaptrim, 116 active+clean; 365 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 1.5 MiB/s wr, 9 op/s 2024-09-12T23:48:17.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:17 smithi181 bash[67587]: cluster 2024-09-12T23:48:17.432549+0000 mon.a (mon.0) 974 : cluster [DBG] osdmap e640: 8 total, 8 up, 8 in 2024-09-12T23:48:18.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:17 smithi136 bash[75164]: cephadm 2024-09-12T23:48:16.147771+0000 mgr.y (mgr.24661) 297 : cephadm [INF] Detected new or changed devices on smithi136 2024-09-12T23:48:18.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:17 smithi136 bash[75164]: cephadm 2024-09-12T23:48:16.324485+0000 mgr.y (mgr.24661) 298 : cephadm [INF] Upgrade: It appears safe to stop mds.b 2024-09-12T23:48:18.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:17 smithi136 bash[75164]: cluster 2024-09-12T23:48:16.795942+0000 mgr.y (mgr.24661) 299 : cluster [DBG] pgmap v535: 121 pgs: 2 active+clean+snaptrim_wait, 3 active+clean+snaptrim, 116 active+clean; 365 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 1.5 MiB/s wr, 9 op/s 2024-09-12T23:48:18.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:17 smithi136 bash[75164]: cluster 2024-09-12T23:48:17.432549+0000 mon.a (mon.0) 974 : cluster [DBG] osdmap e640: 8 total, 8 up, 8 in 2024-09-12T23:48:18.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:17 smithi136 bash[76371]: cephadm 2024-09-12T23:48:16.147771+0000 mgr.y (mgr.24661) 297 : cephadm [INF] Detected new or changed devices on smithi136 2024-09-12T23:48:18.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:17 smithi136 bash[76371]: cephadm 2024-09-12T23:48:16.324485+0000 mgr.y (mgr.24661) 298 : cephadm [INF] Upgrade: It appears safe to stop mds.b 2024-09-12T23:48:18.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:17 smithi136 bash[76371]: cluster 2024-09-12T23:48:16.795942+0000 mgr.y (mgr.24661) 299 : cluster [DBG] pgmap v535: 121 pgs: 2 active+clean+snaptrim_wait, 3 active+clean+snaptrim, 116 active+clean; 365 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 1.5 MiB/s wr, 9 op/s 2024-09-12T23:48:18.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:17 smithi136 bash[76371]: cluster 2024-09-12T23:48:17.432549+0000 mon.a (mon.0) 974 : cluster [DBG] osdmap e640: 8 total, 8 up, 8 in 2024-09-12T23:48:18.431 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 49 v 1644 (ObjNum 859 snap 278 seq_num 859) dirty exists 2024-09-12T23:48:18.432 INFO:tasks.rados.rados.0.smithi136.stdout:2862: done (1 left) 2024-09-12T23:48:18.432 INFO:tasks.rados.rados.0.smithi136.stdout:2863: done (0 left) 2024-09-12T23:48:18.432 INFO:tasks.rados.rados.0.smithi136.stdout:2864: rollback oid 11 current snap is 289 2024-09-12T23:48:18.432 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 11 to 267 2024-09-12T23:48:18.432 INFO:tasks.rados.rados.0.smithi136.stdout:2865: delete oid 36 current snap is 289 2024-09-12T23:48:18.461 INFO:tasks.rados.rados.0.smithi136.stdout:2864: finishing rollback tid 0 to smithi13667081-11 2024-09-12T23:48:18.461 INFO:tasks.rados.rados.0.smithi136.stdout:2865: done (1 left) 2024-09-12T23:48:18.461 INFO:tasks.rados.rados.0.smithi136.stdout:2866: copy_from oid 1 from oid 9 current snap is 289 2024-09-12T23:48:18.461 INFO:tasks.rados.rados.0.smithi136.stdout:2867: snap_remove snap 254 2024-09-12T23:48:19.444 INFO:tasks.rados.rados.0.smithi136.stdout:2867: done (2 left) 2024-09-12T23:48:19.444 INFO:tasks.rados.rados.0.smithi136.stdout:2868: setattr oid 41 current snap is 289 2024-09-12T23:48:19.444 INFO:tasks.rados.rados.0.smithi136.stdout:2864: finishing rollback tid 1 to smithi13667081-11 2024-09-12T23:48:19.445 INFO:tasks.rados.rados.0.smithi136.stdout:2864: finishing rollback tid 2 to smithi13667081-11 2024-09-12T23:48:19.445 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 11 v 2063 (ObjNum 605 snap 174 seq_num 605) dirty exists 2024-09-12T23:48:19.445 INFO:tasks.rados.rados.0.smithi136.stdout:2866: finishing copy_from racing read to smithi13667081-1 2024-09-12T23:48:19.445 INFO:tasks.rados.rados.0.smithi136.stdout:2866: finishing copy_from to smithi13667081-1 2024-09-12T23:48:19.445 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 1 v 1619 (ObjNum 877 snap 286 seq_num 877) dirty exists 2024-09-12T23:48:19.447 INFO:tasks.rados.rados.0.smithi136.stdout:2864: done (2 left) 2024-09-12T23:48:19.447 INFO:tasks.rados.rados.0.smithi136.stdout:2866: done (1 left) 2024-09-12T23:48:19.447 INFO:tasks.rados.rados.0.smithi136.stdout:2869: snap_remove snap 249 2024-09-12T23:48:19.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:48:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:48:19] "GET /metrics HTTP/1.1" 200 36370 "" "Prometheus/2.43.0" 2024-09-12T23:48:19.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:19 smithi136 bash[75164]: cluster 2024-09-12T23:48:18.432869+0000 mon.a (mon.0) 975 : cluster [DBG] osdmap e641: 8 total, 8 up, 8 in 2024-09-12T23:48:19.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:19 smithi136 bash[75164]: cephadm 2024-09-12T23:48:18.590550+0000 mgr.y (mgr.24661) 300 : cephadm [INF] Upgrade: Updating mds.b 2024-09-12T23:48:19.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:19 smithi136 bash[75164]: audit 2024-09-12T23:48:18.600137+0000 mon.a (mon.0) 976 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:19.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:19 smithi136 bash[75164]: audit 2024-09-12T23:48:18.601965+0000 mon.a (mon.0) 977 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-12T23:48:19.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:19 smithi136 bash[75164]: audit 2024-09-12T23:48:18.603625+0000 mon.a (mon.0) 978 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:48:19.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:19 smithi136 bash[75164]: cephadm 2024-09-12T23:48:18.605385+0000 mgr.y (mgr.24661) 301 : cephadm [INF] Deploying daemon mds.b on smithi181 2024-09-12T23:48:19.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:19 smithi136 bash[75164]: cluster 2024-09-12T23:48:18.796816+0000 mgr.y (mgr.24661) 302 : cluster [DBG] pgmap v538: 121 pgs: 2 active+clean+snaptrim_wait, 4 active+clean+snaptrim, 115 active+clean; 327 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1.1 MiB/s wr, 5 op/s 2024-09-12T23:48:19.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:19 smithi136 bash[76371]: cluster 2024-09-12T23:48:18.432869+0000 mon.a (mon.0) 975 : cluster [DBG] osdmap e641: 8 total, 8 up, 8 in 2024-09-12T23:48:19.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:19 smithi136 bash[76371]: cephadm 2024-09-12T23:48:18.590550+0000 mgr.y (mgr.24661) 300 : cephadm [INF] Upgrade: Updating mds.b 2024-09-12T23:48:19.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:19 smithi136 bash[76371]: audit 2024-09-12T23:48:18.600137+0000 mon.a (mon.0) 976 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:19.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:19 smithi136 bash[76371]: audit 2024-09-12T23:48:18.601965+0000 mon.a (mon.0) 977 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-12T23:48:19.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:19 smithi136 bash[76371]: audit 2024-09-12T23:48:18.603625+0000 mon.a (mon.0) 978 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:48:19.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:19 smithi136 bash[76371]: cephadm 2024-09-12T23:48:18.605385+0000 mgr.y (mgr.24661) 301 : cephadm [INF] Deploying daemon mds.b on smithi181 2024-09-12T23:48:19.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:19 smithi136 bash[76371]: cluster 2024-09-12T23:48:18.796816+0000 mgr.y (mgr.24661) 302 : cluster [DBG] pgmap v538: 121 pgs: 2 active+clean+snaptrim_wait, 4 active+clean+snaptrim, 115 active+clean; 327 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1.1 MiB/s wr, 5 op/s 2024-09-12T23:48:19.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:19 smithi181 bash[67587]: cluster 2024-09-12T23:48:18.432869+0000 mon.a (mon.0) 975 : cluster [DBG] osdmap e641: 8 total, 8 up, 8 in 2024-09-12T23:48:19.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:19 smithi181 bash[67587]: cephadm 2024-09-12T23:48:18.590550+0000 mgr.y (mgr.24661) 300 : cephadm [INF] Upgrade: Updating mds.b 2024-09-12T23:48:19.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:19 smithi181 bash[67587]: audit 2024-09-12T23:48:18.600137+0000 mon.a (mon.0) 976 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:19.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:19 smithi181 bash[67587]: audit 2024-09-12T23:48:18.601965+0000 mon.a (mon.0) 977 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-12T23:48:19.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:19 smithi181 bash[67587]: audit 2024-09-12T23:48:18.603625+0000 mon.a (mon.0) 978 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:48:19.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:19 smithi181 bash[67587]: cephadm 2024-09-12T23:48:18.605385+0000 mgr.y (mgr.24661) 301 : cephadm [INF] Deploying daemon mds.b on smithi181 2024-09-12T23:48:19.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:19 smithi181 bash[67587]: cluster 2024-09-12T23:48:18.796816+0000 mgr.y (mgr.24661) 302 : cluster [DBG] pgmap v538: 121 pgs: 2 active+clean+snaptrim_wait, 4 active+clean+snaptrim, 115 active+clean; 327 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1.1 MiB/s wr, 5 op/s 2024-09-12T23:48:20.443 INFO:tasks.rados.rados.0.smithi136.stdout:2869: done (1 left) 2024-09-12T23:48:20.443 INFO:tasks.rados.rados.0.smithi136.stdout:2870: delete oid 35 current snap is 289 2024-09-12T23:48:20.443 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 41 v 1490 (ObjNum 851 snap 272 seq_num 851) dirty exists 2024-09-12T23:48:20.473 INFO:tasks.rados.rados.0.smithi136.stdout:2868: done (1 left) 2024-09-12T23:48:20.473 INFO:tasks.rados.rados.0.smithi136.stdout:2870: done (0 left) 2024-09-12T23:48:20.473 INFO:tasks.rados.rados.0.smithi136.stdout:2871: copy_from oid 14 from oid 22 current snap is 289 2024-09-12T23:48:20.473 INFO:tasks.rados.rados.0.smithi136.stdout:2872: delete oid 40 current snap is 289 2024-09-12T23:48:20.491 INFO:tasks.rados.rados.0.smithi136.stdout:2872: done (1 left) 2024-09-12T23:48:20.491 INFO:tasks.rados.rados.0.smithi136.stdout:2873: delete oid 21 current snap is 289 2024-09-12T23:48:20.501 INFO:tasks.rados.rados.0.smithi136.stdout:2873: done (1 left) 2024-09-12T23:48:20.501 INFO:tasks.rados.rados.0.smithi136.stdout:2874: rmattr oid 28 current snap is 289 2024-09-12T23:48:20.501 INFO:tasks.rados.rados.0.smithi136.stdout:2874: done (1 left) 2024-09-12T23:48:20.501 INFO:tasks.rados.rados.0.smithi136.stdout:2875: read oid 7 snap 283 2024-09-12T23:48:20.501 INFO:tasks.rados.rados.0.smithi136.stdout:2875: expect (ObjNum 857 snap 277 seq_num 857) 2024-09-12T23:48:20.501 INFO:tasks.rados.rados.0.smithi136.stdout:2876: copy_from oid 10 from oid 36 current snap is 289 2024-09-12T23:48:20.501 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 43 current snap is 289 2024-09-12T23:48:20.501 INFO:tasks.rados.rados.0.smithi136.stdout:2877: seq_num 908 ranges {0=409600,409600=401408} 2024-09-12T23:48:20.503 INFO:tasks.rados.rados.0.smithi136.stdout:2877: writing smithi13667081-43 from 0 to 409600 tid 1 2024-09-12T23:48:20.506 INFO:tasks.rados.rados.0.smithi136.stdout:2877: writing smithi13667081-43 from 409600 to 811008 tid 2 2024-09-12T23:48:20.506 INFO:tasks.rados.rados.0.smithi136.stdout:2878: read oid 48 snap -1 2024-09-12T23:48:20.506 INFO:tasks.rados.rados.0.smithi136.stdout:2878: expect (ObjNum 864 snap 281 seq_num 864) 2024-09-12T23:48:20.506 INFO:tasks.rados.rados.0.smithi136.stdout:2879: snap_remove snap 270 2024-09-12T23:48:20.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:20 smithi181 bash[67587]: cluster 2024-09-12T23:48:19.443656+0000 mon.a (mon.0) 979 : cluster [DBG] osdmap e642: 8 total, 8 up, 8 in 2024-09-12T23:48:20.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:20 smithi136 bash[75164]: cluster 2024-09-12T23:48:19.443656+0000 mon.a (mon.0) 979 : cluster [DBG] osdmap e642: 8 total, 8 up, 8 in 2024-09-12T23:48:20.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:20 smithi136 bash[76371]: cluster 2024-09-12T23:48:19.443656+0000 mon.a (mon.0) 979 : cluster [DBG] osdmap e642: 8 total, 8 up, 8 in 2024-09-12T23:48:21.023 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:20 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:21.024 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:48:20 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:21.024 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:48:20 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:21.024 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:48:20 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:21.025 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 12 23:48:20 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:21.025 INFO:journalctl@ceph.mds.b.smithi181.stdout:Sep 12 23:48:20 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:21.025 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:48:20 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:21.026 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:48:20 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:21.026 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 12 23:48:20 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:21.026 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:20 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:21.180 INFO:tasks.rados.rados.0.smithi136.stdout:2871: finishing copy_from racing read to smithi13667081-14 2024-09-12T23:48:21.180 INFO:tasks.rados.rados.0.smithi136.stdout:2876: finishing copy_from to smithi13667081-10 2024-09-12T23:48:21.180 INFO:tasks.rados.rados.0.smithi136.stdout:2876: got expected ENOENT (src dne) 2024-09-12T23:48:21.181 INFO:tasks.rados.rados.0.smithi136.stdout:2876: finishing copy_from racing read to smithi13667081-10 2024-09-12T23:48:21.181 INFO:tasks.rados.rados.0.smithi136.stdout:2871: finishing copy_from to smithi13667081-14 2024-09-12T23:48:21.181 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 14 v 1795 (ObjNum 896 snap 288 seq_num 896) dirty exists 2024-09-12T23:48:21.181 INFO:tasks.rados.rados.0.smithi136.stdout:2877: finishing write tid 1 to smithi13667081-43 2024-09-12T23:48:21.181 INFO:tasks.rados.rados.0.smithi136.stdout:2877: finishing write tid 2 to smithi13667081-43 2024-09-12T23:48:21.181 INFO:tasks.rados.rados.0.smithi136.stdout:2875: expect (ObjNum 857 snap 277 seq_num 857) 2024-09-12T23:48:21.262 INFO:tasks.rados.rados.0.smithi136.stdout:2877: finishing write tid 3 to smithi13667081-43 2024-09-12T23:48:21.262 INFO:tasks.rados.rados.0.smithi136.stdout:2878: expect (ObjNum 864 snap 281 seq_num 864) 2024-09-12T23:48:21.344 INFO:journalctl@ceph.mds.b.smithi181.stdout:Sep 12 23:48:21 smithi181 systemd[1]: Stopping Ceph mds.b for da58a47c-715d-11ef-bceb-c7b262605968... 2024-09-12T23:48:21.344 INFO:journalctl@ceph.mds.b.smithi181.stdout:Sep 12 23:48:21 smithi181 bash[48042]: debug 2024-09-12T23:48:21.157+0000 7fd71f582640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mds -n mds.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-12T23:48:21.344 INFO:journalctl@ceph.mds.b.smithi181.stdout:Sep 12 23:48:21 smithi181 bash[48042]: debug 2024-09-12T23:48:21.157+0000 7fd71f582640 -1 mds.b *** got signal Terminated *** 2024-09-12T23:48:21.423 INFO:tasks.rados.rados.0.smithi136.stdout:2877: finishing write tid 4 to smithi13667081-43 2024-09-12T23:48:21.423 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 43 v 1798 (ObjNum 908 snap 289 seq_num 908) dirty exists 2024-09-12T23:48:21.423 INFO:tasks.rados.rados.0.smithi136.stdout:2877: left oid 43 (ObjNum 908 snap 289 seq_num 908) 2024-09-12T23:48:21.423 INFO:tasks.rados.rados.0.smithi136.stdout:2871: done (5 left) 2024-09-12T23:48:21.423 INFO:tasks.rados.rados.0.smithi136.stdout:2875: done (4 left) 2024-09-12T23:48:21.423 INFO:tasks.rados.rados.0.smithi136.stdout:2876: done (3 left) 2024-09-12T23:48:21.423 INFO:tasks.rados.rados.0.smithi136.stdout:2877: done (2 left) 2024-09-12T23:48:21.423 INFO:tasks.rados.rados.0.smithi136.stdout:2878: done (1 left) 2024-09-12T23:48:21.423 INFO:tasks.rados.rados.0.smithi136.stdout:2879: done (0 left) 2024-09-12T23:48:21.423 INFO:tasks.rados.rados.0.smithi136.stdout:2880: copy_from oid 19 from oid 31 current snap is 289 2024-09-12T23:48:21.424 INFO:tasks.rados.rados.0.smithi136.stdout:2881: read oid 40 snap 280 2024-09-12T23:48:21.424 INFO:tasks.rados.rados.0.smithi136.stdout:2881: expect (ObjNum 788 snap 250 seq_num 788) 2024-09-12T23:48:21.424 INFO:tasks.rados.rados.0.smithi136.stdout:2882: delete oid 25 current snap is 289 2024-09-12T23:48:21.436 INFO:tasks.rados.rados.0.smithi136.stdout:2882: done (2 left) 2024-09-12T23:48:21.436 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 18 current snap is 289 2024-09-12T23:48:21.436 INFO:tasks.rados.rados.0.smithi136.stdout:2883: seq_num 910 ranges {131072=442368,573440=401408} 2024-09-12T23:48:21.436 INFO:tasks.rados.rados.0.smithi136.stdout:2883: writing smithi13667081-18 from 131072 to 573440 tid 1 2024-09-12T23:48:21.437 INFO:tasks.rados.rados.0.smithi136.stdout:2883: writing smithi13667081-18 from 573440 to 974848 tid 2 2024-09-12T23:48:21.437 INFO:tasks.rados.rados.0.smithi136.stdout:2884: delete oid 25 current snap is 289 2024-09-12T23:48:21.437 INFO:tasks.rados.rados.0.smithi136.stdout:2880: finishing copy_from to smithi13667081-19 2024-09-12T23:48:21.437 INFO:tasks.rados.rados.0.smithi136.stdout:2880: got expected ENOENT (src dne) 2024-09-12T23:48:21.437 INFO:tasks.rados.rados.0.smithi136.stdout:2880: finishing copy_from racing read to smithi13667081-19 2024-09-12T23:48:21.442 INFO:tasks.rados.rados.0.smithi136.stdout:2880: done (3 left) 2024-09-12T23:48:21.442 INFO:tasks.rados.rados.0.smithi136.stdout:2884: done (2 left) 2024-09-12T23:48:21.442 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 49 current snap is 289 2024-09-12T23:48:21.442 INFO:tasks.rados.rados.0.smithi136.stdout:2885: seq_num 912 ranges {17186816=483328,17670144=401408} 2024-09-12T23:48:21.457 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:21 smithi136 bash[75164]: debug 2024-09-12T23:48:21.160+0000 7f6a844f2640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-12T23:48:21.541 INFO:tasks.rados.rados.0.smithi136.stdout:2885: writing smithi13667081-49 from 17186816 to 17670144 tid 1 2024-09-12T23:48:21.543 INFO:tasks.rados.rados.0.smithi136.stdout:2885: writing smithi13667081-49 from 17670144 to 18071552 tid 2 2024-09-12T23:48:21.543 INFO:tasks.rados.rados.0.smithi136.stdout:2886: delete oid 37 current snap is 289 2024-09-12T23:48:21.543 INFO:tasks.rados.rados.0.smithi136.stdout:2881: expect (ObjNum 788 snap 250 seq_num 788) 2024-09-12T23:48:21.592 INFO:tasks.rados.rados.0.smithi136.stdout:2883: finishing write tid 1 to smithi13667081-18 2024-09-12T23:48:21.592 INFO:tasks.rados.rados.0.smithi136.stdout:2883: finishing write tid 2 to smithi13667081-18 2024-09-12T23:48:21.592 INFO:tasks.rados.rados.0.smithi136.stdout:2883: finishing write tid 3 to smithi13667081-18 2024-09-12T23:48:21.592 INFO:tasks.rados.rados.0.smithi136.stdout:2883: finishing write tid 4 to smithi13667081-18 2024-09-12T23:48:21.593 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 18 v 969 (ObjNum 910 snap 289 seq_num 910) dirty exists 2024-09-12T23:48:21.593 INFO:tasks.rados.rados.0.smithi136.stdout:2883: left oid 18 (ObjNum 910 snap 289 seq_num 910) 2024-09-12T23:48:21.593 INFO:tasks.rados.rados.0.smithi136.stdout:2885: finishing write tid 1 to smithi13667081-49 2024-09-12T23:48:21.593 INFO:tasks.rados.rados.0.smithi136.stdout:2885: finishing write tid 2 to smithi13667081-49 2024-09-12T23:48:21.593 INFO:tasks.rados.rados.0.smithi136.stdout:2881: done (3 left) 2024-09-12T23:48:21.593 INFO:tasks.rados.rados.0.smithi136.stdout:2883: done (2 left) 2024-09-12T23:48:21.593 INFO:tasks.rados.rados.0.smithi136.stdout:2886: done (1 left) 2024-09-12T23:48:21.593 INFO:tasks.rados.rados.0.smithi136.stdout:2887: snap_remove snap 288 2024-09-12T23:48:21.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:21 smithi136 bash[76371]: cluster 2024-09-12T23:48:20.445541+0000 mon.a (mon.0) 980 : cluster [DBG] osdmap e643: 8 total, 8 up, 8 in 2024-09-12T23:48:21.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:21 smithi136 bash[76371]: cluster 2024-09-12T23:48:20.797741+0000 mgr.y (mgr.24661) 303 : cluster [DBG] pgmap v541: 121 pgs: 2 active+clean+snaptrim_wait, 3 active+clean+snaptrim, 116 active+clean; 310 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 170 KiB/s wr, 1 op/s 2024-09-12T23:48:21.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:21 smithi136 bash[76371]: cluster 2024-09-12T23:48:21.162425+0000 mon.a (mon.0) 981 : cluster [WRN] Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-12T23:48:21.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:21 smithi136 bash[76371]: cluster 2024-09-12T23:48:21.162511+0000 mon.a (mon.0) 982 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-12T23:48:21.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:21 smithi136 bash[76371]: cluster 2024-09-12T23:48:21.170155+0000 mon.a (mon.0) 983 : cluster [DBG] osdmap e644: 8 total, 8 up, 8 in 2024-09-12T23:48:21.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:21 smithi136 bash[76371]: cluster 2024-09-12T23:48:21.182949+0000 mon.a (mon.0) 984 : cluster [INF] Standby daemon mds.a assigned to filesystem cephfs as rank 0 2024-09-12T23:48:21.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:21 smithi136 bash[76371]: cluster 2024-09-12T23:48:21.183082+0000 mon.a (mon.0) 985 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-12T23:48:21.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:21 smithi136 bash[76371]: cluster 2024-09-12T23:48:21.183506+0000 mon.a (mon.0) 986 : cluster [DBG] fsmap cephfs:0/1 1 up:standby, 1 failed 2024-09-12T23:48:21.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:21 smithi136 bash[76371]: cluster 2024-09-12T23:48:21.200229+0000 mon.a (mon.0) 987 : cluster [DBG] fsmap cephfs:1/1 {0=a=up:replay} 2024-09-12T23:48:21.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:21 smithi136 bash[76371]: cluster 2024-09-12T23:48:21.432767+0000 mon.a (mon.0) 988 : cluster [DBG] osdmap e645: 8 total, 8 up, 8 in 2024-09-12T23:48:21.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:21 smithi136 bash[75164]: cluster 2024-09-12T23:48:20.445541+0000 mon.a (mon.0) 980 : cluster [DBG] osdmap e643: 8 total, 8 up, 8 in 2024-09-12T23:48:21.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:21 smithi136 bash[75164]: cluster 2024-09-12T23:48:20.797741+0000 mgr.y (mgr.24661) 303 : cluster [DBG] pgmap v541: 121 pgs: 2 active+clean+snaptrim_wait, 3 active+clean+snaptrim, 116 active+clean; 310 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 170 KiB/s wr, 1 op/s 2024-09-12T23:48:21.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:21 smithi136 bash[75164]: cluster 2024-09-12T23:48:21.162425+0000 mon.a (mon.0) 981 : cluster [WRN] Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-12T23:48:21.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:21 smithi136 bash[75164]: cluster 2024-09-12T23:48:21.162511+0000 mon.a (mon.0) 982 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-12T23:48:21.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:21 smithi136 bash[75164]: cluster 2024-09-12T23:48:21.170155+0000 mon.a (mon.0) 983 : cluster [DBG] osdmap e644: 8 total, 8 up, 8 in 2024-09-12T23:48:21.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:21 smithi136 bash[75164]: cluster 2024-09-12T23:48:21.182949+0000 mon.a (mon.0) 984 : cluster [INF] Standby daemon mds.a assigned to filesystem cephfs as rank 0 2024-09-12T23:48:21.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:21 smithi136 bash[75164]: cluster 2024-09-12T23:48:21.183082+0000 mon.a (mon.0) 985 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-12T23:48:21.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:21 smithi136 bash[75164]: cluster 2024-09-12T23:48:21.183506+0000 mon.a (mon.0) 986 : cluster [DBG] fsmap cephfs:0/1 1 up:standby, 1 failed 2024-09-12T23:48:21.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:21 smithi136 bash[75164]: cluster 2024-09-12T23:48:21.200229+0000 mon.a (mon.0) 987 : cluster [DBG] fsmap cephfs:1/1 {0=a=up:replay} 2024-09-12T23:48:21.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:21 smithi136 bash[75164]: cluster 2024-09-12T23:48:21.432767+0000 mon.a (mon.0) 988 : cluster [DBG] osdmap e645: 8 total, 8 up, 8 in 2024-09-12T23:48:21.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:21 smithi181 bash[67587]: cluster 2024-09-12T23:48:20.445541+0000 mon.a (mon.0) 980 : cluster [DBG] osdmap e643: 8 total, 8 up, 8 in 2024-09-12T23:48:21.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:21 smithi181 bash[67587]: cluster 2024-09-12T23:48:20.797741+0000 mgr.y (mgr.24661) 303 : cluster [DBG] pgmap v541: 121 pgs: 2 active+clean+snaptrim_wait, 3 active+clean+snaptrim, 116 active+clean; 310 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 170 KiB/s wr, 1 op/s 2024-09-12T23:48:21.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:21 smithi181 bash[67587]: cluster 2024-09-12T23:48:21.162425+0000 mon.a (mon.0) 981 : cluster [WRN] Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-12T23:48:21.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:21 smithi181 bash[67587]: cluster 2024-09-12T23:48:21.162511+0000 mon.a (mon.0) 982 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-12T23:48:21.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:21 smithi181 bash[67587]: cluster 2024-09-12T23:48:21.170155+0000 mon.a (mon.0) 983 : cluster [DBG] osdmap e644: 8 total, 8 up, 8 in 2024-09-12T23:48:21.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:21 smithi181 bash[67587]: cluster 2024-09-12T23:48:21.182949+0000 mon.a (mon.0) 984 : cluster [INF] Standby daemon mds.a assigned to filesystem cephfs as rank 0 2024-09-12T23:48:21.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:21 smithi181 bash[67587]: cluster 2024-09-12T23:48:21.183082+0000 mon.a (mon.0) 985 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-12T23:48:21.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:21 smithi181 bash[67587]: cluster 2024-09-12T23:48:21.183506+0000 mon.a (mon.0) 986 : cluster [DBG] fsmap cephfs:0/1 1 up:standby, 1 failed 2024-09-12T23:48:21.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:21 smithi181 bash[67587]: cluster 2024-09-12T23:48:21.200229+0000 mon.a (mon.0) 987 : cluster [DBG] fsmap cephfs:1/1 {0=a=up:replay} 2024-09-12T23:48:21.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:21 smithi181 bash[67587]: cluster 2024-09-12T23:48:21.432767+0000 mon.a (mon.0) 988 : cluster [DBG] osdmap e645: 8 total, 8 up, 8 in 2024-09-12T23:48:22.475 INFO:tasks.rados.rados.0.smithi136.stdout:2887: done (1 left) 2024-09-12T23:48:22.475 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 8 current snap is 289 2024-09-12T23:48:22.475 INFO:tasks.rados.rados.0.smithi136.stdout:2888: seq_num 914 ranges {327680=516096,843776=401408} 2024-09-12T23:48:22.480 INFO:tasks.rados.rados.0.smithi136.stdout:2888: writing smithi13667081-8 from 327680 to 843776 tid 1 2024-09-12T23:48:22.483 INFO:tasks.rados.rados.0.smithi136.stdout:2888: writing smithi13667081-8 from 843776 to 1245184 tid 2 2024-09-12T23:48:22.483 INFO:tasks.rados.rados.0.smithi136.stdout:2889: read oid 48 snap -1 2024-09-12T23:48:22.483 INFO:tasks.rados.rados.0.smithi136.stdout:2889: expect (ObjNum 864 snap 281 seq_num 864) 2024-09-12T23:48:22.483 INFO:tasks.rados.rados.0.smithi136.stdout:2890: rollback oid 24 current snap is 289 2024-09-12T23:48:22.484 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 24 to 287 2024-09-12T23:48:22.484 INFO:tasks.rados.rados.0.smithi136.stdout:2885: finishing write tid 3 to smithi13667081-49 2024-09-12T23:48:22.484 INFO:tasks.rados.rados.0.smithi136.stdout:2885: finishing write tid 4 to smithi13667081-49 2024-09-12T23:48:22.484 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 49 v 1647 (ObjNum 912 snap 289 seq_num 912) dirty exists 2024-09-12T23:48:22.484 INFO:tasks.rados.rados.0.smithi136.stdout:2885: left oid 49 (ObjNum 912 snap 289 seq_num 912) 2024-09-12T23:48:22.484 INFO:tasks.rados.rados.0.smithi136.stdout:2885: done (3 left) 2024-09-12T23:48:22.484 INFO:tasks.rados.rados.0.smithi136.stdout:2891: read oid 15 snap -1 2024-09-12T23:48:22.484 INFO:tasks.rados.rados.0.smithi136.stdout:2891: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-12T23:48:22.484 INFO:tasks.rados.rados.0.smithi136.stdout:2892: snap_create 2024-09-12T23:48:22.489 INFO:tasks.rados.rados.0.smithi136.stdout:2891: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-12T23:48:22.496 INFO:tasks.rados.rados.0.smithi136.stdout:2890: finishing rollback tid 0 to smithi13667081-24 2024-09-12T23:48:22.498 INFO:tasks.rados.rados.0.smithi136.stdout:2890: finishing rollback tid 1 to smithi13667081-24 2024-09-12T23:48:22.498 INFO:tasks.rados.rados.0.smithi136.stdout:2890: finishing rollback tid 2 to smithi13667081-24 2024-09-12T23:48:22.498 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 24 v 1497 (ObjNum 838 snap 267 seq_num 838) dirty exists 2024-09-12T23:48:22.503 INFO:tasks.rados.rados.0.smithi136.stdout:2889: expect (ObjNum 864 snap 281 seq_num 864) 2024-09-12T23:48:22.594 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:48:22 smithi181 bash[60471]: ts=2024-09-12T23:48:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:48:22.677 INFO:tasks.rados.rados.0.smithi136.stdout:2888: finishing write tid 1 to smithi13667081-8 2024-09-12T23:48:22.677 INFO:tasks.rados.rados.0.smithi136.stdout:2888: finishing write tid 2 to smithi13667081-8 2024-09-12T23:48:22.677 INFO:tasks.rados.rados.0.smithi136.stdout:2888: finishing write tid 3 to smithi13667081-8 2024-09-12T23:48:22.677 INFO:tasks.rados.rados.0.smithi136.stdout:2888: finishing write tid 4 to smithi13667081-8 2024-09-12T23:48:22.677 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 8 v 1620 (ObjNum 914 snap 289 seq_num 914) dirty exists 2024-09-12T23:48:22.677 INFO:tasks.rados.rados.0.smithi136.stdout:2888: left oid 8 (ObjNum 914 snap 289 seq_num 914) 2024-09-12T23:48:23.482 INFO:tasks.rados.rados.0.smithi136.stdout:2888: done (4 left) 2024-09-12T23:48:23.482 INFO:tasks.rados.rados.0.smithi136.stdout:2889: done (3 left) 2024-09-12T23:48:23.483 INFO:tasks.rados.rados.0.smithi136.stdout:2890: done (2 left) 2024-09-12T23:48:23.483 INFO:tasks.rados.rados.0.smithi136.stdout:2891: done (1 left) 2024-09-12T23:48:23.483 INFO:tasks.rados.rados.0.smithi136.stdout:2892: done (0 left) 2024-09-12T23:48:23.483 INFO:tasks.rados.rados.0.smithi136.stdout:2893: read oid 33 snap -1 2024-09-12T23:48:23.483 INFO:tasks.rados.rados.0.smithi136.stdout:2893: expect (ObjNum 894 snap 288 seq_num 894) 2024-09-12T23:48:23.483 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 43 current snap is 290 2024-09-12T23:48:23.483 INFO:tasks.rados.rados.0.smithi136.stdout:2894: seq_num 915 ranges {811008=532480,1343488=401408} 2024-09-12T23:48:23.490 INFO:tasks.rados.rados.0.smithi136.stdout:2894: writing smithi13667081-43 from 811008 to 1343488 tid 1 2024-09-12T23:48:23.493 INFO:tasks.rados.rados.0.smithi136.stdout:2894: writing smithi13667081-43 from 1343488 to 1744896 tid 2 2024-09-12T23:48:23.493 INFO:tasks.rados.rados.0.smithi136.stdout:2895: read oid 8 snap -1 2024-09-12T23:48:23.493 INFO:tasks.rados.rados.0.smithi136.stdout:2895: expect (ObjNum 914 snap 289 seq_num 914) 2024-09-12T23:48:23.493 INFO:tasks.rados.rados.0.smithi136.stdout:2896: rollback oid 1 current snap is 290 2024-09-12T23:48:23.493 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 1 to 279 2024-09-12T23:48:23.493 INFO:tasks.rados.rados.0.smithi136.stdout:2897: snap_remove snap 283 2024-09-12T23:48:23.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:23 smithi136 bash[76371]: cluster 2024-09-12T23:48:22.203165+0000 mon.a (mon.0) 989 : cluster [DBG] mds.? [v2:172.21.15.136:6834/496731293,v1:172.21.15.136:6835/496731293] up:reconnect 2024-09-12T23:48:23.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:23 smithi136 bash[76371]: cluster 2024-09-12T23:48:22.203309+0000 mon.a (mon.0) 990 : cluster [DBG] fsmap cephfs:1/1 {0=a=up:reconnect} 2024-09-12T23:48:23.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:23 smithi136 bash[76371]: cluster 2024-09-12T23:48:22.471617+0000 mon.a (mon.0) 991 : cluster [DBG] osdmap e646: 8 total, 8 up, 8 in 2024-09-12T23:48:23.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:23 smithi136 bash[76371]: cluster 2024-09-12T23:48:22.798627+0000 mgr.y (mgr.24661) 304 : cluster [DBG] pgmap v545: 121 pgs: 3 active+clean+snaptrim, 118 active+clean; 296 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4.1 MiB/s rd, 414 KiB/s wr, 20 op/s 2024-09-12T23:48:23.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:23 smithi136 bash[75164]: cluster 2024-09-12T23:48:22.203165+0000 mon.a (mon.0) 989 : cluster [DBG] mds.? [v2:172.21.15.136:6834/496731293,v1:172.21.15.136:6835/496731293] up:reconnect 2024-09-12T23:48:23.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:23 smithi136 bash[75164]: cluster 2024-09-12T23:48:22.203309+0000 mon.a (mon.0) 990 : cluster [DBG] fsmap cephfs:1/1 {0=a=up:reconnect} 2024-09-12T23:48:23.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:23 smithi136 bash[75164]: cluster 2024-09-12T23:48:22.471617+0000 mon.a (mon.0) 991 : cluster [DBG] osdmap e646: 8 total, 8 up, 8 in 2024-09-12T23:48:23.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:23 smithi136 bash[75164]: cluster 2024-09-12T23:48:22.798627+0000 mgr.y (mgr.24661) 304 : cluster [DBG] pgmap v545: 121 pgs: 3 active+clean+snaptrim, 118 active+clean; 296 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4.1 MiB/s rd, 414 KiB/s wr, 20 op/s 2024-09-12T23:48:23.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:23 smithi181 bash[67587]: cluster 2024-09-12T23:48:22.203165+0000 mon.a (mon.0) 989 : cluster [DBG] mds.? [v2:172.21.15.136:6834/496731293,v1:172.21.15.136:6835/496731293] up:reconnect 2024-09-12T23:48:23.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:23 smithi181 bash[67587]: cluster 2024-09-12T23:48:22.203309+0000 mon.a (mon.0) 990 : cluster [DBG] fsmap cephfs:1/1 {0=a=up:reconnect} 2024-09-12T23:48:23.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:23 smithi181 bash[67587]: cluster 2024-09-12T23:48:22.471617+0000 mon.a (mon.0) 991 : cluster [DBG] osdmap e646: 8 total, 8 up, 8 in 2024-09-12T23:48:23.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:23 smithi181 bash[67587]: cluster 2024-09-12T23:48:22.798627+0000 mgr.y (mgr.24661) 304 : cluster [DBG] pgmap v545: 121 pgs: 3 active+clean+snaptrim, 118 active+clean; 296 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4.1 MiB/s rd, 414 KiB/s wr, 20 op/s 2024-09-12T23:48:24.490 INFO:tasks.rados.rados.0.smithi136.stdout:2897: done (4 left) 2024-09-12T23:48:24.490 INFO:tasks.rados.rados.0.smithi136.stdout:2898: snap_remove snap 258 2024-09-12T23:48:24.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:24 smithi136 bash[75164]: cluster 2024-09-12T23:48:23.205459+0000 mon.a (mon.0) 992 : cluster [DBG] mds.? [v2:172.21.15.136:6834/496731293,v1:172.21.15.136:6835/496731293] up:rejoin 2024-09-12T23:48:24.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:24 smithi136 bash[75164]: cluster 2024-09-12T23:48:23.205604+0000 mon.a (mon.0) 993 : cluster [DBG] fsmap cephfs:1/1 {0=a=up:rejoin} 2024-09-12T23:48:24.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:24 smithi136 bash[75164]: cluster 2024-09-12T23:48:23.206859+0000 mon.a (mon.0) 994 : cluster [INF] daemon mds.a is now active in filesystem cephfs as rank 0 2024-09-12T23:48:24.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:24 smithi136 bash[75164]: cluster 2024-09-12T23:48:23.483110+0000 mon.a (mon.0) 995 : cluster [DBG] osdmap e647: 8 total, 8 up, 8 in 2024-09-12T23:48:24.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:24 smithi136 bash[75164]: audit 2024-09-12T23:48:23.757586+0000 mon.a (mon.0) 996 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:24.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:24 smithi136 bash[75164]: audit 2024-09-12T23:48:23.759862+0000 mon.a (mon.0) 997 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:48:24.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:24 smithi136 bash[75164]: cluster 2024-09-12T23:48:24.200642+0000 mon.a (mon.0) 998 : cluster [INF] Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-12T23:48:24.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:24 smithi136 bash[75164]: cluster 2024-09-12T23:48:24.200720+0000 mon.a (mon.0) 999 : cluster [INF] Cluster is now healthy 2024-09-12T23:48:24.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:24 smithi136 bash[76371]: cluster 2024-09-12T23:48:23.205459+0000 mon.a (mon.0) 992 : cluster [DBG] mds.? [v2:172.21.15.136:6834/496731293,v1:172.21.15.136:6835/496731293] up:rejoin 2024-09-12T23:48:24.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:24 smithi136 bash[76371]: cluster 2024-09-12T23:48:23.205604+0000 mon.a (mon.0) 993 : cluster [DBG] fsmap cephfs:1/1 {0=a=up:rejoin} 2024-09-12T23:48:24.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:24 smithi136 bash[76371]: cluster 2024-09-12T23:48:23.206859+0000 mon.a (mon.0) 994 : cluster [INF] daemon mds.a is now active in filesystem cephfs as rank 0 2024-09-12T23:48:24.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:24 smithi136 bash[76371]: cluster 2024-09-12T23:48:23.483110+0000 mon.a (mon.0) 995 : cluster [DBG] osdmap e647: 8 total, 8 up, 8 in 2024-09-12T23:48:24.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:24 smithi136 bash[76371]: audit 2024-09-12T23:48:23.757586+0000 mon.a (mon.0) 996 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:24.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:24 smithi136 bash[76371]: audit 2024-09-12T23:48:23.759862+0000 mon.a (mon.0) 997 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:48:24.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:24 smithi136 bash[76371]: cluster 2024-09-12T23:48:24.200642+0000 mon.a (mon.0) 998 : cluster [INF] Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-12T23:48:24.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:24 smithi136 bash[76371]: cluster 2024-09-12T23:48:24.200720+0000 mon.a (mon.0) 999 : cluster [INF] Cluster is now healthy 2024-09-12T23:48:24.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:24 smithi181 bash[67587]: cluster 2024-09-12T23:48:23.205459+0000 mon.a (mon.0) 992 : cluster [DBG] mds.? [v2:172.21.15.136:6834/496731293,v1:172.21.15.136:6835/496731293] up:rejoin 2024-09-12T23:48:24.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:24 smithi181 bash[67587]: cluster 2024-09-12T23:48:23.205604+0000 mon.a (mon.0) 993 : cluster [DBG] fsmap cephfs:1/1 {0=a=up:rejoin} 2024-09-12T23:48:24.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:24 smithi181 bash[67587]: cluster 2024-09-12T23:48:23.206859+0000 mon.a (mon.0) 994 : cluster [INF] daemon mds.a is now active in filesystem cephfs as rank 0 2024-09-12T23:48:24.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:24 smithi181 bash[67587]: cluster 2024-09-12T23:48:23.483110+0000 mon.a (mon.0) 995 : cluster [DBG] osdmap e647: 8 total, 8 up, 8 in 2024-09-12T23:48:24.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:24 smithi181 bash[67587]: audit 2024-09-12T23:48:23.757586+0000 mon.a (mon.0) 996 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:24.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:24 smithi181 bash[67587]: audit 2024-09-12T23:48:23.759862+0000 mon.a (mon.0) 997 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:48:24.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:24 smithi181 bash[67587]: cluster 2024-09-12T23:48:24.200642+0000 mon.a (mon.0) 998 : cluster [INF] Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-12T23:48:24.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:24 smithi181 bash[67587]: cluster 2024-09-12T23:48:24.200720+0000 mon.a (mon.0) 999 : cluster [INF] Cluster is now healthy 2024-09-12T23:48:25.493 INFO:tasks.rados.rados.0.smithi136.stdout:2898: done (4 left) 2024-09-12T23:48:25.493 INFO:tasks.rados.rados.0.smithi136.stdout:2899: rmattr oid 27 current snap is 290 2024-09-12T23:48:25.493 INFO:tasks.rados.rados.0.smithi136.stdout:2899: done (4 left) 2024-09-12T23:48:25.493 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 47 current snap is 290 2024-09-12T23:48:25.493 INFO:tasks.rados.rados.0.smithi136.stdout:2900: seq_num 916 ranges {0=548864,548864=401408} 2024-09-12T23:48:25.499 INFO:tasks.rados.rados.0.smithi136.stdout:2900: writing smithi13667081-47 from 0 to 548864 tid 1 2024-09-12T23:48:25.504 INFO:tasks.rados.rados.0.smithi136.stdout:2900: writing smithi13667081-47 from 548864 to 950272 tid 2 2024-09-12T23:48:25.504 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 7 current snap is 290 2024-09-12T23:48:25.504 INFO:tasks.rados.rados.0.smithi136.stdout:2901: seq_num 917 ranges {2899968=565248,3465216=401408} 2024-09-12T23:48:25.506 INFO:journalctl@ceph.mds.a.smithi136.stdout:Sep 12 23:48:25 smithi136 bash[97471]: debug 2024-09-12T23:48:25.452+0000 7f90a4a1d640 -1 mds.pinger is_rank_lagging: rank=0 was never sent ping request. 2024-09-12T23:48:25.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:25 smithi136 bash[75164]: cluster 2024-09-12T23:48:24.210225+0000 mon.a (mon.0) 1000 : cluster [DBG] mds.? [v2:172.21.15.136:6834/496731293,v1:172.21.15.136:6835/496731293] up:active 2024-09-12T23:48:25.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:25 smithi136 bash[75164]: cluster 2024-09-12T23:48:24.210379+0000 mon.a (mon.0) 1001 : cluster [DBG] fsmap cephfs:1 {0=a=up:active} 2024-09-12T23:48:25.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:25 smithi136 bash[75164]: cluster 2024-09-12T23:48:24.487258+0000 mon.a (mon.0) 1002 : cluster [DBG] osdmap e648: 8 total, 8 up, 8 in 2024-09-12T23:48:25.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:25 smithi136 bash[75164]: audit 2024-09-12T23:48:24.669748+0000 mon.a (mon.0) 1003 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-12T23:48:25.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:25 smithi136 bash[75164]: cluster 2024-09-12T23:48:24.799456+0000 mgr.y (mgr.24661) 305 : cluster [DBG] pgmap v548: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 296 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.2 MiB/s rd, 462 KiB/s wr, 23 op/s 2024-09-12T23:48:25.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:25 smithi136 bash[76371]: cluster 2024-09-12T23:48:24.210225+0000 mon.a (mon.0) 1000 : cluster [DBG] mds.? [v2:172.21.15.136:6834/496731293,v1:172.21.15.136:6835/496731293] up:active 2024-09-12T23:48:25.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:25 smithi136 bash[76371]: cluster 2024-09-12T23:48:24.210379+0000 mon.a (mon.0) 1001 : cluster [DBG] fsmap cephfs:1 {0=a=up:active} 2024-09-12T23:48:25.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:25 smithi136 bash[76371]: cluster 2024-09-12T23:48:24.487258+0000 mon.a (mon.0) 1002 : cluster [DBG] osdmap e648: 8 total, 8 up, 8 in 2024-09-12T23:48:25.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:25 smithi136 bash[76371]: audit 2024-09-12T23:48:24.669748+0000 mon.a (mon.0) 1003 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-12T23:48:25.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:25 smithi136 bash[76371]: cluster 2024-09-12T23:48:24.799456+0000 mgr.y (mgr.24661) 305 : cluster [DBG] pgmap v548: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 296 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.2 MiB/s rd, 462 KiB/s wr, 23 op/s 2024-09-12T23:48:25.539 INFO:tasks.rados.rados.0.smithi136.stdout:2901: writing smithi13667081-7 from 2899968 to 3465216 tid 1 2024-09-12T23:48:25.543 INFO:tasks.rados.rados.0.smithi136.stdout:2901: writing smithi13667081-7 from 3465216 to 3866624 tid 2 2024-09-12T23:48:25.543 INFO:tasks.rados.rados.0.smithi136.stdout:2902: read oid 16 snap -1 2024-09-12T23:48:25.544 INFO:tasks.rados.rados.0.smithi136.stdout:2902: expect (ObjNum 885 snap 287 seq_num 885) 2024-09-12T23:48:25.544 INFO:tasks.rados.rados.0.smithi136.stdout:2903: snap_remove snap 274 2024-09-12T23:48:25.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:25 smithi181 bash[67587]: cluster 2024-09-12T23:48:24.210225+0000 mon.a (mon.0) 1000 : cluster [DBG] mds.? [v2:172.21.15.136:6834/496731293,v1:172.21.15.136:6835/496731293] up:active 2024-09-12T23:48:25.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:25 smithi181 bash[67587]: cluster 2024-09-12T23:48:24.210379+0000 mon.a (mon.0) 1001 : cluster [DBG] fsmap cephfs:1 {0=a=up:active} 2024-09-12T23:48:25.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:25 smithi181 bash[67587]: cluster 2024-09-12T23:48:24.487258+0000 mon.a (mon.0) 1002 : cluster [DBG] osdmap e648: 8 total, 8 up, 8 in 2024-09-12T23:48:25.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:25 smithi181 bash[67587]: audit 2024-09-12T23:48:24.669748+0000 mon.a (mon.0) 1003 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-12T23:48:25.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:25 smithi181 bash[67587]: cluster 2024-09-12T23:48:24.799456+0000 mgr.y (mgr.24661) 305 : cluster [DBG] pgmap v548: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 296 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.2 MiB/s rd, 462 KiB/s wr, 23 op/s 2024-09-12T23:48:26.433 INFO:tasks.rados.rados.0.smithi136.stdout:2903: done (7 left) 2024-09-12T23:48:26.433 INFO:tasks.rados.rados.0.smithi136.stdout:2904: setattr oid 38 current snap is 290 2024-09-12T23:48:26.434 INFO:tasks.rados.rados.0.smithi136.stdout:2893: expect (ObjNum 894 snap 288 seq_num 894) 2024-09-12T23:48:26.486 INFO:tasks.rados.rados.0.smithi136.stdout:2896: finishing rollback tid 0 to smithi13667081-1 2024-09-12T23:48:26.486 INFO:tasks.rados.rados.0.smithi136.stdout:2896: finishing rollback tid 1 to smithi13667081-1 2024-09-12T23:48:26.486 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 1 v 1798 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-12T23:48:26.486 INFO:tasks.rados.rados.0.smithi136.stdout:2894: finishing write tid 1 to smithi13667081-43 2024-09-12T23:48:26.486 INFO:tasks.rados.rados.0.smithi136.stdout:2894: finishing write tid 2 to smithi13667081-43 2024-09-12T23:48:26.486 INFO:tasks.rados.rados.0.smithi136.stdout:2894: finishing write tid 3 to smithi13667081-43 2024-09-12T23:48:26.486 INFO:tasks.rados.rados.0.smithi136.stdout:2894: finishing write tid 4 to smithi13667081-43 2024-09-12T23:48:26.486 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 43 v 1801 (ObjNum 915 snap 290 seq_num 915) dirty exists 2024-09-12T23:48:26.486 INFO:tasks.rados.rados.0.smithi136.stdout:2894: left oid 43 (ObjNum 915 snap 290 seq_num 915) 2024-09-12T23:48:26.486 INFO:tasks.rados.rados.0.smithi136.stdout:2895: expect (ObjNum 914 snap 289 seq_num 914) 2024-09-12T23:48:26.533 INFO:tasks.rados.rados.0.smithi136.stdout:2900: finishing write tid 1 to smithi13667081-47 2024-09-12T23:48:26.533 INFO:tasks.rados.rados.0.smithi136.stdout:2900: finishing write tid 2 to smithi13667081-47 2024-09-12T23:48:26.533 INFO:tasks.rados.rados.0.smithi136.stdout:2900: finishing write tid 3 to smithi13667081-47 2024-09-12T23:48:26.533 INFO:tasks.rados.rados.0.smithi136.stdout:2900: finishing write tid 4 to smithi13667081-47 2024-09-12T23:48:26.533 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 47 v 1622 (ObjNum 916 snap 290 seq_num 916) dirty exists 2024-09-12T23:48:26.533 INFO:tasks.rados.rados.0.smithi136.stdout:2900: left oid 47 (ObjNum 916 snap 290 seq_num 916) 2024-09-12T23:48:26.533 INFO:tasks.rados.rados.0.smithi136.stdout:2901: finishing write tid 1 to smithi13667081-7 2024-09-12T23:48:26.533 INFO:tasks.rados.rados.0.smithi136.stdout:2901: finishing write tid 2 to smithi13667081-7 2024-09-12T23:48:26.534 INFO:tasks.rados.rados.0.smithi136.stdout:2893: done (7 left) 2024-09-12T23:48:26.534 INFO:tasks.rados.rados.0.smithi136.stdout:2894: done (6 left) 2024-09-12T23:48:26.534 INFO:tasks.rados.rados.0.smithi136.stdout:2895: done (5 left) 2024-09-12T23:48:26.534 INFO:tasks.rados.rados.0.smithi136.stdout:2896: done (4 left) 2024-09-12T23:48:26.534 INFO:tasks.rados.rados.0.smithi136.stdout:2900: done (3 left) 2024-09-12T23:48:26.534 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 40 current snap is 290 2024-09-12T23:48:26.534 INFO:tasks.rados.rados.0.smithi136.stdout:2901: finishing write tid 3 to smithi13667081-7 2024-09-12T23:48:26.534 INFO:tasks.rados.rados.0.smithi136.stdout:2902: expect (ObjNum 885 snap 287 seq_num 885) 2024-09-12T23:48:26.543 INFO:tasks.rados.rados.0.smithi136.stdout:2901: finishing write tid 4 to smithi13667081-7 2024-09-12T23:48:26.544 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 7 v 972 (ObjNum 917 snap 290 seq_num 917) dirty exists 2024-09-12T23:48:26.544 INFO:tasks.rados.rados.0.smithi136.stdout:2901: left oid 7 (ObjNum 917 snap 290 seq_num 917) 2024-09-12T23:48:26.544 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 38 v 1643 (ObjNum 273 snap 0 seq_num 0) dirty exists 2024-09-12T23:48:26.544 INFO:tasks.rados.rados.0.smithi136.stdout:2905: seq_num 918 ranges {0=581632,581632=401408} 2024-09-12T23:48:26.553 INFO:tasks.rados.rados.0.smithi136.stdout:2905: writing smithi13667081-40 from 0 to 581632 tid 1 2024-09-12T23:48:26.561 INFO:tasks.rados.rados.0.smithi136.stdout:2905: writing smithi13667081-40 from 581632 to 983040 tid 2 2024-09-12T23:48:26.561 INFO:tasks.rados.rados.0.smithi136.stdout:2901: done (3 left) 2024-09-12T23:48:26.561 INFO:tasks.rados.rados.0.smithi136.stdout:2902: done (2 left) 2024-09-12T23:48:26.561 INFO:tasks.rados.rados.0.smithi136.stdout:2904: done (1 left) 2024-09-12T23:48:26.561 INFO:tasks.rados.rados.0.smithi136.stdout:2906: snap_remove snap 261 2024-09-12T23:48:26.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:26 smithi136 bash[75164]: cluster 2024-09-12T23:48:25.493968+0000 mon.a (mon.0) 1004 : cluster [DBG] osdmap e649: 8 total, 8 up, 8 in 2024-09-12T23:48:26.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:26 smithi136 bash[75164]: cluster 2024-09-12T23:48:26.431671+0000 mon.a (mon.0) 1005 : cluster [DBG] osdmap e650: 8 total, 8 up, 8 in 2024-09-12T23:48:26.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:26 smithi136 bash[76371]: cluster 2024-09-12T23:48:25.493968+0000 mon.a (mon.0) 1004 : cluster [DBG] osdmap e649: 8 total, 8 up, 8 in 2024-09-12T23:48:26.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:26 smithi136 bash[76371]: cluster 2024-09-12T23:48:26.431671+0000 mon.a (mon.0) 1005 : cluster [DBG] osdmap e650: 8 total, 8 up, 8 in 2024-09-12T23:48:26.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:26 smithi181 bash[67587]: cluster 2024-09-12T23:48:25.493968+0000 mon.a (mon.0) 1004 : cluster [DBG] osdmap e649: 8 total, 8 up, 8 in 2024-09-12T23:48:26.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:26 smithi181 bash[67587]: cluster 2024-09-12T23:48:26.431671+0000 mon.a (mon.0) 1005 : cluster [DBG] osdmap e650: 8 total, 8 up, 8 in 2024-09-12T23:48:27.516 INFO:tasks.rados.rados.0.smithi136.stdout:2905: finishing write tid 1 to smithi13667081-40 2024-09-12T23:48:27.516 INFO:tasks.rados.rados.0.smithi136.stdout:2905: finishing write tid 2 to smithi13667081-40 2024-09-12T23:48:27.516 INFO:tasks.rados.rados.0.smithi136.stdout:2905: finishing write tid 3 to smithi13667081-40 2024-09-12T23:48:27.516 INFO:tasks.rados.rados.0.smithi136.stdout:2905: finishing write tid 4 to smithi13667081-40 2024-09-12T23:48:27.516 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 40 v 2156 (ObjNum 918 snap 290 seq_num 918) dirty exists 2024-09-12T23:48:27.516 INFO:tasks.rados.rados.0.smithi136.stdout:2905: left oid 40 (ObjNum 918 snap 290 seq_num 918) 2024-09-12T23:48:27.516 INFO:tasks.rados.rados.0.smithi136.stdout:2905: done (1 left) 2024-09-12T23:48:27.516 INFO:tasks.rados.rados.0.smithi136.stdout:2906: done (0 left) 2024-09-12T23:48:27.517 INFO:tasks.rados.rados.0.smithi136.stdout:2907: snap_remove snap 238 2024-09-12T23:48:27.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:27 smithi136 bash[76371]: cluster 2024-09-12T23:48:26.800441+0000 mgr.y (mgr.24661) 306 : cluster [DBG] pgmap v551: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 299 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.6 MiB/s rd, 1.2 MiB/s wr, 29 op/s 2024-09-12T23:48:27.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:27 smithi136 bash[75164]: cluster 2024-09-12T23:48:26.800441+0000 mgr.y (mgr.24661) 306 : cluster [DBG] pgmap v551: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 299 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.6 MiB/s rd, 1.2 MiB/s wr, 29 op/s 2024-09-12T23:48:27.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:27 smithi181 bash[67587]: cluster 2024-09-12T23:48:26.800441+0000 mgr.y (mgr.24661) 306 : cluster [DBG] pgmap v551: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 299 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.6 MiB/s rd, 1.2 MiB/s wr, 29 op/s 2024-09-12T23:48:28.520 INFO:tasks.rados.rados.0.smithi136.stdout:2907: done (0 left) 2024-09-12T23:48:28.520 INFO:tasks.rados.rados.0.smithi136.stdout:2908: delete oid 28 current snap is 290 2024-09-12T23:48:28.536 INFO:tasks.rados.rados.0.smithi136.stdout:2908: done (0 left) 2024-09-12T23:48:28.536 INFO:tasks.rados.rados.0.smithi136.stdout:2909: delete oid 19 current snap is 290 2024-09-12T23:48:28.553 INFO:tasks.rados.rados.0.smithi136.stdout:2909: done (0 left) 2024-09-12T23:48:28.553 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 18 current snap is 290 2024-09-12T23:48:28.553 INFO:tasks.rados.rados.0.smithi136.stdout:2910: seq_num 921 ranges {974848=630784,1605632=401408} 2024-09-12T23:48:28.563 INFO:tasks.rados.rados.0.smithi136.stdout:2910: writing smithi13667081-18 from 974848 to 1605632 tid 1 2024-09-12T23:48:28.565 INFO:tasks.rados.rados.0.smithi136.stdout:2910: writing smithi13667081-18 from 1605632 to 2007040 tid 2 2024-09-12T23:48:28.565 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 14 current snap is 290 2024-09-12T23:48:28.565 INFO:tasks.rados.rados.0.smithi136.stdout:2911: seq_num 922 ranges {614400=647168,1261568=401408} 2024-09-12T23:48:28.573 INFO:tasks.rados.rados.0.smithi136.stdout:2911: writing smithi13667081-14 from 614400 to 1261568 tid 1 2024-09-12T23:48:28.577 INFO:tasks.rados.rados.0.smithi136.stdout:2911: writing smithi13667081-14 from 1261568 to 1662976 tid 2 2024-09-12T23:48:28.577 INFO:tasks.rados.rados.0.smithi136.stdout:2912: delete oid 35 current snap is 290 2024-09-12T23:48:28.577 INFO:tasks.rados.rados.0.smithi136.stdout:2910: finishing write tid 1 to smithi13667081-18 2024-09-12T23:48:28.577 INFO:tasks.rados.rados.0.smithi136.stdout:2910: finishing write tid 2 to smithi13667081-18 2024-09-12T23:48:28.577 INFO:tasks.rados.rados.0.smithi136.stdout:2910: finishing write tid 3 to smithi13667081-18 2024-09-12T23:48:28.578 INFO:tasks.rados.rados.0.smithi136.stdout:2910: finishing write tid 4 to smithi13667081-18 2024-09-12T23:48:28.578 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 18 v 975 (ObjNum 921 snap 290 seq_num 921) dirty exists 2024-09-12T23:48:28.579 INFO:tasks.rados.rados.0.smithi136.stdout:2910: left oid 18 (ObjNum 921 snap 290 seq_num 921) 2024-09-12T23:48:28.582 INFO:tasks.rados.rados.0.smithi136.stdout:2910: done (2 left) 2024-09-12T23:48:28.582 INFO:tasks.rados.rados.0.smithi136.stdout:2912: done (1 left) 2024-09-12T23:48:28.582 INFO:tasks.rados.rados.0.smithi136.stdout:2913: setattr oid 35 current snap is 290 2024-09-12T23:48:28.584 INFO:tasks.rados.rados.0.smithi136.stdout:2911: finishing write tid 1 to smithi13667081-14 2024-09-12T23:48:28.585 INFO:tasks.rados.rados.0.smithi136.stdout:2911: finishing write tid 2 to smithi13667081-14 2024-09-12T23:48:28.585 INFO:tasks.rados.rados.0.smithi136.stdout:2911: finishing write tid 3 to smithi13667081-14 2024-09-12T23:48:28.586 INFO:tasks.rados.rados.0.smithi136.stdout:2914: copy_from oid 26 from oid 1 current snap is 290 2024-09-12T23:48:28.586 INFO:tasks.rados.rados.0.smithi136.stdout:2915: copy_from oid 17 from oid 29 current snap is 290 2024-09-12T23:48:28.586 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 27 current snap is 290 2024-09-12T23:48:28.586 INFO:tasks.rados.rados.0.smithi136.stdout:2916: seq_num 924 ranges {0=679936,679936=401408} 2024-09-12T23:48:28.593 INFO:tasks.rados.rados.0.smithi136.stdout:2916: writing smithi13667081-27 from 0 to 679936 tid 1 2024-09-12T23:48:28.593 INFO:tasks.rados.rados.0.smithi136.stdout:2916: writing smithi13667081-27 from 679936 to 1081344 tid 2 2024-09-12T23:48:28.593 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 2 current snap is 290 2024-09-12T23:48:28.593 INFO:tasks.rados.rados.0.smithi136.stdout:2911: finishing write tid 4 to smithi13667081-14 2024-09-12T23:48:28.594 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 14 v 1834 (ObjNum 922 snap 290 seq_num 922) dirty exists 2024-09-12T23:48:28.594 INFO:tasks.rados.rados.0.smithi136.stdout:2911: left oid 14 (ObjNum 922 snap 290 seq_num 922) 2024-09-12T23:48:28.594 INFO:tasks.rados.rados.0.smithi136.stdout:2914: finishing copy_from to smithi13667081-26 2024-09-12T23:48:28.594 INFO:tasks.rados.rados.0.smithi136.stdout:2914: got expected ENOENT (src dne) 2024-09-12T23:48:28.594 INFO:tasks.rados.rados.0.smithi136.stdout:2914: finishing copy_from racing read to smithi13667081-26 2024-09-12T23:48:28.594 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 35 v 2184 (ObjNum 289 snap 0 seq_num 0) dirty exists 2024-09-12T23:48:28.594 INFO:tasks.rados.rados.0.smithi136.stdout:2917: seq_num 925 ranges {2048000=696320,2744320=401408} 2024-09-12T23:48:28.609 INFO:tasks.rados.rados.0.smithi136.stdout:2917: writing smithi13667081-2 from 2048000 to 2744320 tid 1 2024-09-12T23:48:28.613 INFO:tasks.rados.rados.0.smithi136.stdout:2917: writing smithi13667081-2 from 2744320 to 3145728 tid 2 2024-09-12T23:48:28.613 INFO:tasks.rados.rados.0.smithi136.stdout:2911: done (5 left) 2024-09-12T23:48:28.613 INFO:tasks.rados.rados.0.smithi136.stdout:2913: done (4 left) 2024-09-12T23:48:28.613 INFO:tasks.rados.rados.0.smithi136.stdout:2914: done (3 left) 2024-09-12T23:48:28.613 INFO:tasks.rados.rados.0.smithi136.stdout:2918: read oid 45 snap -1 2024-09-12T23:48:28.613 INFO:tasks.rados.rados.0.smithi136.stdout:2918: expect deleted 2024-09-12T23:48:28.613 INFO:tasks.rados.rados.0.smithi136.stdout:2919: rmattr oid 36 current snap is 290 2024-09-12T23:48:28.613 INFO:tasks.rados.rados.0.smithi136.stdout:2919: done (4 left) 2024-09-12T23:48:28.613 INFO:tasks.rados.rados.0.smithi136.stdout:2920: rollback oid 28 current snap is 290 2024-09-12T23:48:28.613 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 28 to 279 2024-09-12T23:48:28.613 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 4 current snap is 290 2024-09-12T23:48:28.614 INFO:tasks.rados.rados.0.smithi136.stdout:2921: seq_num 926 ranges {11354112=712704,12066816=401408} 2024-09-12T23:48:28.683 INFO:tasks.rados.rados.0.smithi136.stdout:2921: writing smithi13667081-4 from 11354112 to 12066816 tid 1 2024-09-12T23:48:28.686 INFO:tasks.rados.rados.0.smithi136.stdout:2921: writing smithi13667081-4 from 12066816 to 12468224 tid 2 2024-09-12T23:48:28.686 INFO:tasks.rados.rados.0.smithi136.stdout:2922: read oid 42 snap -1 2024-09-12T23:48:28.686 INFO:tasks.rados.rados.0.smithi136.stdout:2922: expect (ObjNum 895 snap 288 seq_num 895) 2024-09-12T23:48:28.686 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 35 current snap is 290 2024-09-12T23:48:28.686 INFO:tasks.rados.rados.0.smithi136.stdout:2916: finishing write tid 1 to smithi13667081-27 2024-09-12T23:48:28.686 INFO:tasks.rados.rados.0.smithi136.stdout:2916: finishing write tid 2 to smithi13667081-27 2024-09-12T23:48:28.686 INFO:tasks.rados.rados.0.smithi136.stdout:2916: finishing write tid 3 to smithi13667081-27 2024-09-12T23:48:28.686 INFO:tasks.rados.rados.0.smithi136.stdout:2916: finishing write tid 4 to smithi13667081-27 2024-09-12T23:48:28.686 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 27 v 2187 (ObjNum 924 snap 290 seq_num 924) dirty exists 2024-09-12T23:48:28.687 INFO:tasks.rados.rados.0.smithi136.stdout:2916: left oid 27 (ObjNum 924 snap 290 seq_num 924) 2024-09-12T23:48:28.687 INFO:tasks.rados.rados.0.smithi136.stdout:2920: finishing rollback tid 1 to smithi13667081-28 2024-09-12T23:48:28.687 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 28 v 1749 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-12T23:48:28.687 INFO:tasks.rados.rados.0.smithi136.stdout:2917: finishing write tid 1 to smithi13667081-2 2024-09-12T23:48:28.687 INFO:tasks.rados.rados.0.smithi136.stdout:2917: finishing write tid 2 to smithi13667081-2 2024-09-12T23:48:28.687 INFO:tasks.rados.rados.0.smithi136.stdout:2917: finishing write tid 3 to smithi13667081-2 2024-09-12T23:48:28.687 INFO:tasks.rados.rados.0.smithi136.stdout:2917: finishing write tid 4 to smithi13667081-2 2024-09-12T23:48:28.687 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 2 v 1493 (ObjNum 925 snap 290 seq_num 925) dirty exists 2024-09-12T23:48:28.687 INFO:tasks.rados.rados.0.smithi136.stdout:2917: left oid 2 (ObjNum 925 snap 290 seq_num 925) 2024-09-12T23:48:28.687 INFO:tasks.rados.rados.0.smithi136.stdout:2915: finishing copy_from racing read to smithi13667081-17 2024-09-12T23:48:28.687 INFO:tasks.rados.rados.0.smithi136.stdout:2915: finishing copy_from to smithi13667081-17 2024-09-12T23:48:28.687 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 17 v 1635 (ObjNum 859 snap 278 seq_num 859) dirty exists 2024-09-12T23:48:28.687 INFO:tasks.rados.rados.0.smithi136.stdout:2923: seq_num 927 ranges {0=729088,729088=401408} 2024-09-12T23:48:28.689 INFO:tasks.rados.rados.0.smithi136.stdout:2923: writing smithi13667081-35 from 0 to 729088 tid 1 2024-09-12T23:48:28.693 INFO:tasks.rados.rados.0.smithi136.stdout:2923: writing smithi13667081-35 from 729088 to 1130496 tid 2 2024-09-12T23:48:28.693 INFO:tasks.rados.rados.0.smithi136.stdout:2915: done (7 left) 2024-09-12T23:48:28.693 INFO:tasks.rados.rados.0.smithi136.stdout:2916: done (6 left) 2024-09-12T23:48:28.693 INFO:tasks.rados.rados.0.smithi136.stdout:2917: done (5 left) 2024-09-12T23:48:28.693 INFO:tasks.rados.rados.0.smithi136.stdout:2918: done (4 left) 2024-09-12T23:48:28.693 INFO:tasks.rados.rados.0.smithi136.stdout:2920: done (3 left) 2024-09-12T23:48:28.693 INFO:tasks.rados.rados.0.smithi136.stdout:2924: rollback oid 36 current snap is 290 2024-09-12T23:48:28.693 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 36 to 287 2024-09-12T23:48:28.693 INFO:tasks.rados.rados.0.smithi136.stdout:2925: snap_create 2024-09-12T23:48:28.694 INFO:tasks.rados.rados.0.smithi136.stdout:2921: finishing write tid 1 to smithi13667081-4 2024-09-12T23:48:28.694 INFO:tasks.rados.rados.0.smithi136.stdout:2921: finishing write tid 2 to smithi13667081-4 2024-09-12T23:48:28.695 INFO:tasks.rados.rados.0.smithi136.stdout:2921: finishing write tid 3 to smithi13667081-4 2024-09-12T23:48:28.698 INFO:tasks.rados.rados.0.smithi136.stdout:2923: finishing write tid 1 to smithi13667081-35 2024-09-12T23:48:28.699 INFO:tasks.rados.rados.0.smithi136.stdout:2921: finishing write tid 4 to smithi13667081-4 2024-09-12T23:48:28.699 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 4 v 1623 (ObjNum 926 snap 290 seq_num 926) dirty exists 2024-09-12T23:48:28.699 INFO:tasks.rados.rados.0.smithi136.stdout:2921: left oid 4 (ObjNum 926 snap 290 seq_num 926) 2024-09-12T23:48:28.699 INFO:tasks.rados.rados.0.smithi136.stdout:2923: finishing write tid 2 to smithi13667081-35 2024-09-12T23:48:28.701 INFO:tasks.rados.rados.0.smithi136.stdout:2924: finishing rollback tid 1 to smithi13667081-36 2024-09-12T23:48:28.701 INFO:tasks.rados.rados.0.smithi136.stdout:2924: finishing rollback tid 2 to smithi13667081-36 2024-09-12T23:48:28.702 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 36 v 1622 (ObjNum 1297 snap 0 seq_num 0) dirty exists 2024-09-12T23:48:28.702 INFO:tasks.rados.rados.0.smithi136.stdout:2923: finishing write tid 3 to smithi13667081-35 2024-09-12T23:48:28.704 INFO:tasks.rados.rados.0.smithi136.stdout:2923: finishing write tid 4 to smithi13667081-35 2024-09-12T23:48:28.704 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 35 v 2192 (ObjNum 927 snap 290 seq_num 927) dirty exists 2024-09-12T23:48:28.704 INFO:tasks.rados.rados.0.smithi136.stdout:2923: left oid 35 (ObjNum 927 snap 290 seq_num 927) 2024-09-12T23:48:28.722 INFO:tasks.rados.rados.0.smithi136.stdout:2922: expect (ObjNum 895 snap 288 seq_num 895) 2024-09-12T23:48:28.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:28 smithi136 bash[75164]: cluster 2024-09-12T23:48:27.507016+0000 mon.a (mon.0) 1006 : cluster [DBG] osdmap e651: 8 total, 8 up, 8 in 2024-09-12T23:48:28.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:28 smithi136 bash[76371]: cluster 2024-09-12T23:48:27.507016+0000 mon.a (mon.0) 1006 : cluster [DBG] osdmap e651: 8 total, 8 up, 8 in 2024-09-12T23:48:28.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:28 smithi181 bash[67587]: cluster 2024-09-12T23:48:27.507016+0000 mon.a (mon.0) 1006 : cluster [DBG] osdmap e651: 8 total, 8 up, 8 in 2024-09-12T23:48:29.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:29 smithi136 bash[75164]: cluster 2024-09-12T23:48:28.521919+0000 mon.a (mon.0) 1007 : cluster [DBG] osdmap e652: 8 total, 8 up, 8 in 2024-09-12T23:48:29.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:29 smithi136 bash[75164]: cluster 2024-09-12T23:48:28.801152+0000 mgr.y (mgr.24661) 307 : cluster [DBG] pgmap v554: 121 pgs: 121 active+clean; 301 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.5 MiB/s wr, 29 op/s 2024-09-12T23:48:29.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:48:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:48:29] "GET /metrics HTTP/1.1" 200 36204 "" "Prometheus/2.43.0" 2024-09-12T23:48:29.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:29 smithi136 bash[76371]: cluster 2024-09-12T23:48:28.521919+0000 mon.a (mon.0) 1007 : cluster [DBG] osdmap e652: 8 total, 8 up, 8 in 2024-09-12T23:48:29.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:29 smithi136 bash[76371]: cluster 2024-09-12T23:48:28.801152+0000 mgr.y (mgr.24661) 307 : cluster [DBG] pgmap v554: 121 pgs: 121 active+clean; 301 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.5 MiB/s wr, 29 op/s 2024-09-12T23:48:29.833 INFO:tasks.rados.rados.0.smithi136.stdout:2921: done (4 left) 2024-09-12T23:48:29.833 INFO:tasks.rados.rados.0.smithi136.stdout:2922: done (3 left) 2024-09-12T23:48:29.833 INFO:tasks.rados.rados.0.smithi136.stdout:2923: done (2 left) 2024-09-12T23:48:29.833 INFO:tasks.rados.rados.0.smithi136.stdout:2924: done (1 left) 2024-09-12T23:48:29.833 INFO:tasks.rados.rados.0.smithi136.stdout:2925: done (0 left) 2024-09-12T23:48:29.834 INFO:tasks.rados.rados.0.smithi136.stdout:2926: snap_create 2024-09-12T23:48:29.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:29 smithi181 bash[67587]: cluster 2024-09-12T23:48:28.521919+0000 mon.a (mon.0) 1007 : cluster [DBG] osdmap e652: 8 total, 8 up, 8 in 2024-09-12T23:48:29.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:29 smithi181 bash[67587]: cluster 2024-09-12T23:48:28.801152+0000 mgr.y (mgr.24661) 307 : cluster [DBG] pgmap v554: 121 pgs: 121 active+clean; 301 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.5 MiB/s wr, 29 op/s 2024-09-12T23:48:30.533 INFO:tasks.rados.rados.0.smithi136.stdout:2926: done (0 left) 2024-09-12T23:48:30.534 INFO:tasks.rados.rados.0.smithi136.stdout:2927: rollback oid 2 current snap is 292 2024-09-12T23:48:30.534 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 2 to 290 2024-09-12T23:48:30.534 INFO:tasks.rados.rados.0.smithi136.stdout:2928: read oid 40 snap -1 2024-09-12T23:48:30.534 INFO:tasks.rados.rados.0.smithi136.stdout:2928: expect (ObjNum 918 snap 290 seq_num 918) 2024-09-12T23:48:30.534 INFO:tasks.rados.rados.0.smithi136.stdout:2929: read oid 24 snap -1 2024-09-12T23:48:30.534 INFO:tasks.rados.rados.0.smithi136.stdout:2929: expect (ObjNum 838 snap 267 seq_num 838) 2024-09-12T23:48:30.534 INFO:tasks.rados.rados.0.smithi136.stdout:2930: read oid 50 snap -1 2024-09-12T23:48:30.534 INFO:tasks.rados.rados.0.smithi136.stdout:2930: expect deleted 2024-09-12T23:48:30.534 INFO:tasks.rados.rados.0.smithi136.stdout:2931: delete oid 17 current snap is 292 2024-09-12T23:48:30.553 INFO:tasks.rados.rados.0.smithi136.stdout:2927: finishing rollback tid 0 to smithi13667081-2 2024-09-12T23:48:30.553 INFO:tasks.rados.rados.0.smithi136.stdout:2927: finishing rollback tid 1 to smithi13667081-2 2024-09-12T23:48:30.555 INFO:tasks.rados.rados.0.smithi136.stdout:2929: expect (ObjNum 838 snap 267 seq_num 838) 2024-09-12T23:48:30.658 INFO:tasks.rados.rados.0.smithi136.stdout:2927: finishing rollback tid 2 to smithi13667081-2 2024-09-12T23:48:30.658 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 2 v 1496 (ObjNum 925 snap 290 seq_num 925) dirty exists 2024-09-12T23:48:30.658 INFO:tasks.rados.rados.0.smithi136.stdout:2928: expect (ObjNum 918 snap 290 seq_num 918) 2024-09-12T23:48:30.684 INFO:tasks.rados.rados.0.smithi136.stdout:2927: done (4 left) 2024-09-12T23:48:30.684 INFO:tasks.rados.rados.0.smithi136.stdout:2928: done (3 left) 2024-09-12T23:48:30.684 INFO:tasks.rados.rados.0.smithi136.stdout:2929: done (2 left) 2024-09-12T23:48:30.684 INFO:tasks.rados.rados.0.smithi136.stdout:2930: done (1 left) 2024-09-12T23:48:30.684 INFO:tasks.rados.rados.0.smithi136.stdout:2931: done (0 left) 2024-09-12T23:48:30.685 INFO:tasks.rados.rados.0.smithi136.stdout:2932: copy_from oid 15 from oid 29 current snap is 292 2024-09-12T23:48:30.685 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 4 current snap is 292 2024-09-12T23:48:30.685 INFO:tasks.rados.rados.0.smithi136.stdout:2933: seq_num 929 ranges {12468224=761856,13230080=401408} 2024-09-12T23:48:30.768 INFO:tasks.rados.rados.0.smithi136.stdout:2933: writing smithi13667081-4 from 12468224 to 13230080 tid 1 2024-09-12T23:48:30.770 INFO:tasks.rados.rados.0.smithi136.stdout:2933: writing smithi13667081-4 from 13230080 to 13631488 tid 2 2024-09-12T23:48:30.770 INFO:tasks.rados.rados.0.smithi136.stdout:2934: delete oid 41 current snap is 292 2024-09-12T23:48:30.776 INFO:tasks.rados.rados.0.smithi136.stdout:2934: done (2 left) 2024-09-12T23:48:30.776 INFO:tasks.rados.rados.0.smithi136.stdout:2935: read oid 17 snap 290 2024-09-12T23:48:30.776 INFO:tasks.rados.rados.0.smithi136.stdout:2935: expect (ObjNum 859 snap 278 seq_num 859) 2024-09-12T23:48:30.776 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 18 current snap is 292 2024-09-12T23:48:30.776 INFO:tasks.rados.rados.0.smithi136.stdout:2936: seq_num 931 ranges {2007040=802816,2809856=401408} 2024-09-12T23:48:30.793 INFO:tasks.rados.rados.0.smithi136.stdout:2936: writing smithi13667081-18 from 2007040 to 2809856 tid 1 2024-09-12T23:48:30.795 INFO:tasks.rados.rados.0.smithi136.stdout:2936: writing smithi13667081-18 from 2809856 to 3211264 tid 2 2024-09-12T23:48:30.796 INFO:tasks.rados.rados.0.smithi136.stdout:2937: delete oid 40 current snap is 292 2024-09-12T23:48:30.796 INFO:tasks.rados.rados.0.smithi136.stdout:2933: finishing write tid 1 to smithi13667081-4 2024-09-12T23:48:30.796 INFO:tasks.rados.rados.0.smithi136.stdout:2933: finishing write tid 2 to smithi13667081-4 2024-09-12T23:48:30.796 INFO:tasks.rados.rados.0.smithi136.stdout:2933: finishing write tid 3 to smithi13667081-4 2024-09-12T23:48:30.796 INFO:tasks.rados.rados.0.smithi136.stdout:2933: finishing write tid 4 to smithi13667081-4 2024-09-12T23:48:30.797 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 4 v 1626 (ObjNum 929 snap 292 seq_num 929) dirty exists 2024-09-12T23:48:30.797 INFO:tasks.rados.rados.0.smithi136.stdout:2933: left oid 4 (ObjNum 929 snap 292 seq_num 929) 2024-09-12T23:48:30.807 INFO:tasks.rados.rados.0.smithi136.stdout:2936: finishing write tid 1 to smithi13667081-18 2024-09-12T23:48:30.810 INFO:tasks.rados.rados.0.smithi136.stdout:2936: finishing write tid 2 to smithi13667081-18 2024-09-12T23:48:30.810 INFO:tasks.rados.rados.0.smithi136.stdout:2936: finishing write tid 3 to smithi13667081-18 2024-09-12T23:48:30.813 INFO:tasks.rados.rados.0.smithi136.stdout:2936: finishing write tid 4 to smithi13667081-18 2024-09-12T23:48:30.813 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 18 v 978 (ObjNum 931 snap 292 seq_num 931) dirty exists 2024-09-12T23:48:30.813 INFO:tasks.rados.rados.0.smithi136.stdout:2936: left oid 18 (ObjNum 931 snap 292 seq_num 931) 2024-09-12T23:48:30.815 INFO:tasks.rados.rados.0.smithi136.stdout:2932: finishing copy_from racing read to smithi13667081-15 2024-09-12T23:48:30.818 INFO:tasks.rados.rados.0.smithi136.stdout:2932: finishing copy_from to smithi13667081-15 2024-09-12T23:48:30.818 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 15 v 1635 (ObjNum 859 snap 278 seq_num 859) dirty exists 2024-09-12T23:48:30.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:30 smithi181 bash[67587]: cluster 2024-09-12T23:48:29.529495+0000 mon.a (mon.0) 1008 : cluster [DBG] osdmap e653: 8 total, 8 up, 8 in 2024-09-12T23:48:30.886 INFO:tasks.rados.rados.0.smithi136.stdout:2935: expect (ObjNum 859 snap 278 seq_num 859) 2024-09-12T23:48:31.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:30 smithi136 bash[75164]: cluster 2024-09-12T23:48:29.529495+0000 mon.a (mon.0) 1008 : cluster [DBG] osdmap e653: 8 total, 8 up, 8 in 2024-09-12T23:48:31.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:30 smithi136 bash[76371]: cluster 2024-09-12T23:48:29.529495+0000 mon.a (mon.0) 1008 : cluster [DBG] osdmap e653: 8 total, 8 up, 8 in 2024-09-12T23:48:31.782 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:31 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:31.783 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:48:31 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:31.783 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:31 smithi181 bash[67587]: cluster 2024-09-12T23:48:30.535764+0000 mon.a (mon.0) 1009 : cluster [DBG] osdmap e654: 8 total, 8 up, 8 in 2024-09-12T23:48:31.783 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:31 smithi181 bash[67587]: cluster 2024-09-12T23:48:30.802051+0000 mgr.y (mgr.24661) 308 : cluster [DBG] pgmap v557: 121 pgs: 121 active+clean; 295 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 915 KiB/s rd, 716 KiB/s wr, 10 op/s 2024-09-12T23:48:31.783 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:31 smithi181 bash[67587]: cluster 2024-09-12T23:48:31.433994+0000 mon.a (mon.0) 1010 : cluster [DBG] osdmap e655: 8 total, 8 up, 8 in 2024-09-12T23:48:31.784 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:31 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:31.784 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 12 23:48:31 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:31.784 INFO:journalctl@ceph.mds.b.smithi181.stdout:Sep 12 23:48:31 smithi181 bash[88658]: ceph-da58a47c-715d-11ef-bceb-c7b262605968-mds-b 2024-09-12T23:48:31.784 INFO:journalctl@ceph.mds.b.smithi181.stdout:Sep 12 23:48:31 smithi181 systemd[1]: ceph-da58a47c-715d-11ef-bceb-c7b262605968@mds.b.service: Deactivated successfully. 2024-09-12T23:48:31.784 INFO:journalctl@ceph.mds.b.smithi181.stdout:Sep 12 23:48:31 smithi181 systemd[1]: Stopped Ceph mds.b for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:48:31.784 INFO:journalctl@ceph.mds.b.smithi181.stdout:Sep 12 23:48:31 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:31.785 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 12 23:48:31 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:31.785 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:48:31 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:31.785 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:48:31 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:31.785 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:48:31 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:31.786 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:48:31 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:32.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:31 smithi136 bash[75164]: cluster 2024-09-12T23:48:30.535764+0000 mon.a (mon.0) 1009 : cluster [DBG] osdmap e654: 8 total, 8 up, 8 in 2024-09-12T23:48:32.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:31 smithi136 bash[75164]: cluster 2024-09-12T23:48:30.802051+0000 mgr.y (mgr.24661) 308 : cluster [DBG] pgmap v557: 121 pgs: 121 active+clean; 295 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 915 KiB/s rd, 716 KiB/s wr, 10 op/s 2024-09-12T23:48:32.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:31 smithi136 bash[75164]: cluster 2024-09-12T23:48:31.433994+0000 mon.a (mon.0) 1010 : cluster [DBG] osdmap e655: 8 total, 8 up, 8 in 2024-09-12T23:48:32.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:31 smithi136 bash[76371]: cluster 2024-09-12T23:48:30.535764+0000 mon.a (mon.0) 1009 : cluster [DBG] osdmap e654: 8 total, 8 up, 8 in 2024-09-12T23:48:32.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:31 smithi136 bash[76371]: cluster 2024-09-12T23:48:30.802051+0000 mgr.y (mgr.24661) 308 : cluster [DBG] pgmap v557: 121 pgs: 121 active+clean; 295 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 915 KiB/s rd, 716 KiB/s wr, 10 op/s 2024-09-12T23:48:32.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:31 smithi136 bash[76371]: cluster 2024-09-12T23:48:31.433994+0000 mon.a (mon.0) 1010 : cluster [DBG] osdmap e655: 8 total, 8 up, 8 in 2024-09-12T23:48:32.094 INFO:journalctl@ceph.mds.b.smithi181.stdout:Sep 12 23:48:31 smithi181 systemd[1]: Started Ceph mds.b for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:48:32.594 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:48:32 smithi181 bash[60471]: ts=2024-09-12T23:48:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:48:32.696 INFO:tasks.rados.rados.0.smithi136.stdout:2932: done (4 left) 2024-09-12T23:48:32.696 INFO:tasks.rados.rados.0.smithi136.stdout:2933: done (3 left) 2024-09-12T23:48:32.696 INFO:tasks.rados.rados.0.smithi136.stdout:2935: done (2 left) 2024-09-12T23:48:32.697 INFO:tasks.rados.rados.0.smithi136.stdout:2936: done (1 left) 2024-09-12T23:48:32.697 INFO:tasks.rados.rados.0.smithi136.stdout:2937: done (0 left) 2024-09-12T23:48:32.697 INFO:tasks.rados.rados.0.smithi136.stdout:2938: snap_create 2024-09-12T23:48:32.842 INFO:tasks.rados.rados.0.smithi136.stdout:2938: done (0 left) 2024-09-12T23:48:32.843 INFO:tasks.rados.rados.0.smithi136.stdout:2939: snap_remove snap 275 2024-09-12T23:48:33.094 INFO:journalctl@ceph.mds.b.smithi181.stdout:Sep 12 23:48:33 smithi181 bash[88772]: starting mds.b at 2024-09-12T23:48:33.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:32 smithi181 bash[67587]: audit 2024-09-12T23:48:31.826918+0000 mon.a (mon.0) 1011 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:33.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:32 smithi181 bash[67587]: audit 2024-09-12T23:48:31.836984+0000 mon.a (mon.0) 1012 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:33.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:32 smithi181 bash[67587]: audit 2024-09-12T23:48:31.839001+0000 mon.a (mon.0) 1013 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:48:33.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:32 smithi136 bash[75164]: audit 2024-09-12T23:48:31.826918+0000 mon.a (mon.0) 1011 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:33.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:32 smithi136 bash[75164]: audit 2024-09-12T23:48:31.836984+0000 mon.a (mon.0) 1012 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:33.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:32 smithi136 bash[75164]: audit 2024-09-12T23:48:31.839001+0000 mon.a (mon.0) 1013 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:48:33.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:32 smithi136 bash[76371]: audit 2024-09-12T23:48:31.826918+0000 mon.a (mon.0) 1011 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:33.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:32 smithi136 bash[76371]: audit 2024-09-12T23:48:31.836984+0000 mon.a (mon.0) 1012 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:33.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:32 smithi136 bash[76371]: audit 2024-09-12T23:48:31.839001+0000 mon.a (mon.0) 1013 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:48:33.859 INFO:tasks.rados.rados.0.smithi136.stdout:2939: done (0 left) 2024-09-12T23:48:33.860 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 31 current snap is 293 2024-09-12T23:48:33.860 INFO:tasks.rados.rados.0.smithi136.stdout:2940: seq_num 933 ranges {0=434176,434176=548864,983040=253952} 2024-09-12T23:48:33.860 INFO:tasks.rados.rados.0.smithi136.stdout:2940: writing smithi13667081-31 from 0 to 434176 tid 1 2024-09-12T23:48:33.863 INFO:tasks.rados.rados.0.smithi136.stdout:2940: writing smithi13667081-31 from 434176 to 983040 tid 2 2024-09-12T23:48:33.865 INFO:tasks.rados.rados.0.smithi136.stdout:2940: writing smithi13667081-31 from 983040 to 1236992 tid 3 2024-09-12T23:48:33.865 INFO:tasks.rados.rados.0.smithi136.stdout:2941: delete oid 49 current snap is 293 2024-09-12T23:48:33.885 INFO:tasks.rados.rados.0.smithi136.stdout:2940: finishing write tid 1 to smithi13667081-31 2024-09-12T23:48:33.887 INFO:tasks.rados.rados.0.smithi136.stdout:2940: finishing write tid 2 to smithi13667081-31 2024-09-12T23:48:33.887 INFO:tasks.rados.rados.0.smithi136.stdout:2940: finishing write tid 3 to smithi13667081-31 2024-09-12T23:48:33.887 INFO:tasks.rados.rados.0.smithi136.stdout:2940: finishing write tid 4 to smithi13667081-31 2024-09-12T23:48:33.889 INFO:tasks.rados.rados.0.smithi136.stdout:2941: done (1 left) 2024-09-12T23:48:33.889 INFO:tasks.rados.rados.0.smithi136.stdout:2942: delete oid 34 current snap is 293 2024-09-12T23:48:33.891 INFO:tasks.rados.rados.0.smithi136.stdout:2940: finishing write tid 5 to smithi13667081-31 2024-09-12T23:48:33.891 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 31 v 1651 (ObjNum 933 snap 293 seq_num 933) dirty exists 2024-09-12T23:48:33.891 INFO:tasks.rados.rados.0.smithi136.stdout:2940: left oid 31 (ObjNum 933 snap 293 seq_num 933) 2024-09-12T23:48:33.896 INFO:tasks.rados.rados.0.smithi136.stdout:2940: done (1 left) 2024-09-12T23:48:33.896 INFO:tasks.rados.rados.0.smithi136.stdout:2942: done (0 left) 2024-09-12T23:48:33.896 INFO:tasks.rados.rados.0.smithi136.stdout:2943: snap_remove snap 267 2024-09-12T23:48:34.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:33 smithi136 bash[75164]: cluster 2024-09-12T23:48:32.803075+0000 mgr.y (mgr.24661) 309 : cluster [DBG] pgmap v559: 121 pgs: 121 active+clean; 298 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 5.0 MiB/s wr, 11 op/s 2024-09-12T23:48:34.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:33 smithi136 bash[75164]: cluster 2024-09-12T23:48:32.845858+0000 mon.a (mon.0) 1014 : cluster [DBG] osdmap e656: 8 total, 8 up, 8 in 2024-09-12T23:48:34.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:33 smithi136 bash[76371]: cluster 2024-09-12T23:48:32.803075+0000 mgr.y (mgr.24661) 309 : cluster [DBG] pgmap v559: 121 pgs: 121 active+clean; 298 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 5.0 MiB/s wr, 11 op/s 2024-09-12T23:48:34.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:33 smithi136 bash[76371]: cluster 2024-09-12T23:48:32.845858+0000 mon.a (mon.0) 1014 : cluster [DBG] osdmap e656: 8 total, 8 up, 8 in 2024-09-12T23:48:34.344 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:33 smithi181 bash[67587]: cluster 2024-09-12T23:48:32.803075+0000 mgr.y (mgr.24661) 309 : cluster [DBG] pgmap v559: 121 pgs: 121 active+clean; 298 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 5.0 MiB/s wr, 11 op/s 2024-09-12T23:48:34.344 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:33 smithi181 bash[67587]: cluster 2024-09-12T23:48:32.845858+0000 mon.a (mon.0) 1014 : cluster [DBG] osdmap e656: 8 total, 8 up, 8 in 2024-09-12T23:48:34.864 INFO:tasks.rados.rados.0.smithi136.stdout:2943: done (0 left) 2024-09-12T23:48:34.864 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 45 current snap is 293 2024-09-12T23:48:34.864 INFO:tasks.rados.rados.0.smithi136.stdout:2944: seq_num 936 ranges {0=483328,483328=655360,1138688=147456} 2024-09-12T23:48:34.874 INFO:tasks.rados.rados.0.smithi136.stdout:2944: writing smithi13667081-45 from 0 to 483328 tid 1 2024-09-12T23:48:34.886 INFO:tasks.rados.rados.0.smithi136.stdout:2944: writing smithi13667081-45 from 483328 to 1138688 tid 2 2024-09-12T23:48:34.889 INFO:tasks.rados.rados.0.smithi136.stdout:2944: writing smithi13667081-45 from 1138688 to 1286144 tid 3 2024-09-12T23:48:34.889 INFO:tasks.rados.rados.0.smithi136.stdout:2945: read oid 30 snap -1 2024-09-12T23:48:34.889 INFO:tasks.rados.rados.0.smithi136.stdout:2945: expect deleted 2024-09-12T23:48:34.889 INFO:tasks.rados.rados.0.smithi136.stdout:2946: rmattr oid 41 current snap is 293 2024-09-12T23:48:34.889 INFO:tasks.rados.rados.0.smithi136.stdout:2946: done (2 left) 2024-09-12T23:48:34.889 INFO:tasks.rados.rados.0.smithi136.stdout:2947: snap_remove snap 255 2024-09-12T23:48:35.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:34 smithi136 bash[75164]: cluster 2024-09-12T23:48:33.846155+0000 mon.a (mon.0) 1015 : cluster [DBG] mds.? [v2:172.21.15.181:6832/3688985057,v1:172.21.15.181:6833/3688985057] up:boot 2024-09-12T23:48:35.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:34 smithi136 bash[75164]: cluster 2024-09-12T23:48:33.846279+0000 mon.a (mon.0) 1016 : cluster [DBG] fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-12T23:48:35.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:34 smithi136 bash[75164]: audit 2024-09-12T23:48:33.847014+0000 mon.a (mon.0) 1017 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-12T23:48:35.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:34 smithi136 bash[75164]: cluster 2024-09-12T23:48:33.857220+0000 mon.a (mon.0) 1018 : cluster [DBG] osdmap e657: 8 total, 8 up, 8 in 2024-09-12T23:48:35.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:34 smithi136 bash[76371]: cluster 2024-09-12T23:48:33.846155+0000 mon.a (mon.0) 1015 : cluster [DBG] mds.? [v2:172.21.15.181:6832/3688985057,v1:172.21.15.181:6833/3688985057] up:boot 2024-09-12T23:48:35.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:34 smithi136 bash[76371]: cluster 2024-09-12T23:48:33.846279+0000 mon.a (mon.0) 1016 : cluster [DBG] fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-12T23:48:35.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:34 smithi136 bash[76371]: audit 2024-09-12T23:48:33.847014+0000 mon.a (mon.0) 1017 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-12T23:48:35.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:34 smithi136 bash[76371]: cluster 2024-09-12T23:48:33.857220+0000 mon.a (mon.0) 1018 : cluster [DBG] osdmap e657: 8 total, 8 up, 8 in 2024-09-12T23:48:35.344 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:34 smithi181 bash[67587]: cluster 2024-09-12T23:48:33.846155+0000 mon.a (mon.0) 1015 : cluster [DBG] mds.? [v2:172.21.15.181:6832/3688985057,v1:172.21.15.181:6833/3688985057] up:boot 2024-09-12T23:48:35.344 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:34 smithi181 bash[67587]: cluster 2024-09-12T23:48:33.846279+0000 mon.a (mon.0) 1016 : cluster [DBG] fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-12T23:48:35.344 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:34 smithi181 bash[67587]: audit 2024-09-12T23:48:33.847014+0000 mon.a (mon.0) 1017 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-12T23:48:35.344 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:34 smithi181 bash[67587]: cluster 2024-09-12T23:48:33.857220+0000 mon.a (mon.0) 1018 : cluster [DBG] osdmap e657: 8 total, 8 up, 8 in 2024-09-12T23:48:35.871 INFO:tasks.rados.rados.0.smithi136.stdout:2947: done (2 left) 2024-09-12T23:48:35.871 INFO:tasks.rados.rados.0.smithi136.stdout:2948: snap_create 2024-09-12T23:48:35.871 INFO:tasks.rados.rados.0.smithi136.stdout:2944: finishing write tid 1 to smithi13667081-45 2024-09-12T23:48:35.871 INFO:tasks.rados.rados.0.smithi136.stdout:2944: finishing write tid 2 to smithi13667081-45 2024-09-12T23:48:35.871 INFO:tasks.rados.rados.0.smithi136.stdout:2944: finishing write tid 3 to smithi13667081-45 2024-09-12T23:48:35.871 INFO:tasks.rados.rados.0.smithi136.stdout:2944: finishing write tid 4 to smithi13667081-45 2024-09-12T23:48:35.871 INFO:tasks.rados.rados.0.smithi136.stdout:2944: finishing write tid 5 to smithi13667081-45 2024-09-12T23:48:35.871 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 45 v 1626 (ObjNum 936 snap 293 seq_num 936) dirty exists 2024-09-12T23:48:35.872 INFO:tasks.rados.rados.0.smithi136.stdout:2944: left oid 45 (ObjNum 936 snap 293 seq_num 936) 2024-09-12T23:48:36.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:35 smithi136 bash[75164]: cluster 2024-09-12T23:48:34.803864+0000 mgr.y (mgr.24661) 310 : cluster [DBG] pgmap v562: 121 pgs: 121 active+clean; 298 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 9.4 MiB/s wr, 18 op/s 2024-09-12T23:48:36.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:35 smithi136 bash[75164]: cluster 2024-09-12T23:48:34.865733+0000 mon.a (mon.0) 1019 : cluster [DBG] osdmap e658: 8 total, 8 up, 8 in 2024-09-12T23:48:36.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:35 smithi136 bash[76371]: cluster 2024-09-12T23:48:34.803864+0000 mgr.y (mgr.24661) 310 : cluster [DBG] pgmap v562: 121 pgs: 121 active+clean; 298 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 9.4 MiB/s wr, 18 op/s 2024-09-12T23:48:36.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:35 smithi136 bash[76371]: cluster 2024-09-12T23:48:34.865733+0000 mon.a (mon.0) 1019 : cluster [DBG] osdmap e658: 8 total, 8 up, 8 in 2024-09-12T23:48:36.344 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:35 smithi181 bash[67587]: cluster 2024-09-12T23:48:34.803864+0000 mgr.y (mgr.24661) 310 : cluster [DBG] pgmap v562: 121 pgs: 121 active+clean; 298 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 9.4 MiB/s wr, 18 op/s 2024-09-12T23:48:36.344 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:35 smithi181 bash[67587]: cluster 2024-09-12T23:48:34.865733+0000 mon.a (mon.0) 1019 : cluster [DBG] osdmap e658: 8 total, 8 up, 8 in 2024-09-12T23:48:36.434 INFO:tasks.rados.rados.0.smithi136.stdout:2944: done (2 left) 2024-09-12T23:48:36.434 INFO:tasks.rados.rados.0.smithi136.stdout:2945: done (1 left) 2024-09-12T23:48:36.434 INFO:tasks.rados.rados.0.smithi136.stdout:2948: done (0 left) 2024-09-12T23:48:36.434 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 20 current snap is 294 2024-09-12T23:48:36.434 INFO:tasks.rados.rados.0.smithi136.stdout:2949: seq_num 937 ranges {0=499712,499712=532480,1032192=270336} 2024-09-12T23:48:36.443 INFO:tasks.rados.rados.0.smithi136.stdout:2949: writing smithi13667081-20 from 0 to 499712 tid 1 2024-09-12T23:48:36.453 INFO:tasks.rados.rados.0.smithi136.stdout:2949: writing smithi13667081-20 from 499712 to 1032192 tid 2 2024-09-12T23:48:36.458 INFO:tasks.rados.rados.0.smithi136.stdout:2949: writing smithi13667081-20 from 1032192 to 1302528 tid 3 2024-09-12T23:48:36.458 INFO:tasks.rados.rados.0.smithi136.stdout:2950: read oid 9 snap 289 2024-09-12T23:48:36.458 INFO:tasks.rados.rados.0.smithi136.stdout:2950: expect (ObjNum 877 snap 286 seq_num 877) 2024-09-12T23:48:36.458 INFO:tasks.rados.rados.0.smithi136.stdout:2951: rollback oid 37 current snap is 294 2024-09-12T23:48:36.459 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 37 to 263 2024-09-12T23:48:36.459 INFO:tasks.rados.rados.0.smithi136.stdout:2949: finishing write tid 1 to smithi13667081-20 2024-09-12T23:48:36.459 INFO:tasks.rados.rados.0.smithi136.stdout:2952: read oid 16 snap -1 2024-09-12T23:48:36.459 INFO:tasks.rados.rados.0.smithi136.stdout:2952: expect (ObjNum 885 snap 287 seq_num 885) 2024-09-12T23:48:36.459 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 50 current snap is 294 2024-09-12T23:48:36.459 INFO:tasks.rados.rados.0.smithi136.stdout:2953: seq_num 938 ranges {0=516096,516096=417792,933888=385024} 2024-09-12T23:48:36.463 INFO:tasks.rados.rados.0.smithi136.stdout:2953: writing smithi13667081-50 from 0 to 516096 tid 1 2024-09-12T23:48:36.467 INFO:tasks.rados.rados.0.smithi136.stdout:2953: writing smithi13667081-50 from 516096 to 933888 tid 2 2024-09-12T23:48:36.471 INFO:tasks.rados.rados.0.smithi136.stdout:2953: writing smithi13667081-50 from 933888 to 1318912 tid 3 2024-09-12T23:48:36.471 INFO:tasks.rados.rados.0.smithi136.stdout:2954: rollback oid 2 current snap is 294 2024-09-12T23:48:36.472 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 2 to 278 2024-09-12T23:48:36.472 INFO:tasks.rados.rados.0.smithi136.stdout:2949: finishing write tid 2 to smithi13667081-20 2024-09-12T23:48:36.472 INFO:tasks.rados.rados.0.smithi136.stdout:2950: expect (ObjNum 877 snap 286 seq_num 877) 2024-09-12T23:48:36.490 INFO:tasks.rados.rados.0.smithi136.stdout:2949: finishing write tid 3 to smithi13667081-20 2024-09-12T23:48:36.490 INFO:tasks.rados.rados.0.smithi136.stdout:2949: finishing write tid 4 to smithi13667081-20 2024-09-12T23:48:36.490 INFO:tasks.rados.rados.0.smithi136.stdout:2949: finishing write tid 5 to smithi13667081-20 2024-09-12T23:48:36.490 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 20 v 1609 (ObjNum 937 snap 294 seq_num 937) dirty exists 2024-09-12T23:48:36.491 INFO:tasks.rados.rados.0.smithi136.stdout:2949: left oid 20 (ObjNum 937 snap 294 seq_num 937) 2024-09-12T23:48:36.491 INFO:tasks.rados.rados.0.smithi136.stdout:2952: expect (ObjNum 885 snap 287 seq_num 885) 2024-09-12T23:48:36.509 INFO:tasks.rados.rados.0.smithi136.stdout:2953: finishing write tid 1 to smithi13667081-50 2024-09-12T23:48:36.509 INFO:tasks.rados.rados.0.smithi136.stdout:2953: finishing write tid 2 to smithi13667081-50 2024-09-12T23:48:36.509 INFO:tasks.rados.rados.0.smithi136.stdout:2953: finishing write tid 3 to smithi13667081-50 2024-09-12T23:48:36.509 INFO:tasks.rados.rados.0.smithi136.stdout:2953: finishing write tid 4 to smithi13667081-50 2024-09-12T23:48:36.509 INFO:tasks.rados.rados.0.smithi136.stdout:2949: done (5 left) 2024-09-12T23:48:36.509 INFO:tasks.rados.rados.0.smithi136.stdout:2950: done (4 left) 2024-09-12T23:48:36.509 INFO:tasks.rados.rados.0.smithi136.stdout:2952: done (3 left) 2024-09-12T23:48:36.509 INFO:tasks.rados.rados.0.smithi136.stdout:2955: rmattr oid 17 current snap is 294 2024-09-12T23:48:36.509 INFO:tasks.rados.rados.0.smithi136.stdout:2955: done (3 left) 2024-09-12T23:48:36.509 INFO:tasks.rados.rados.0.smithi136.stdout:2956: read oid 45 snap 279 2024-09-12T23:48:36.509 INFO:tasks.rados.rados.0.smithi136.stdout:2956: expect deleted 2024-09-12T23:48:36.509 INFO:tasks.rados.rados.0.smithi136.stdout:2957: delete oid 39 current snap is 294 2024-09-12T23:48:36.510 INFO:tasks.rados.rados.0.smithi136.stdout:2951: finishing rollback tid 1 to smithi13667081-37 2024-09-12T23:48:36.510 INFO:tasks.rados.rados.0.smithi136.stdout:2951: finishing rollback tid 2 to smithi13667081-37 2024-09-12T23:48:36.510 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 37 v 1500 (ObjNum 636 snap 188 seq_num 636) dirty exists 2024-09-12T23:48:36.510 INFO:tasks.rados.rados.0.smithi136.stdout:2954: finishing rollback tid 0 to smithi13667081-2 2024-09-12T23:48:36.510 INFO:tasks.rados.rados.0.smithi136.stdout:2954: finishing rollback tid 1 to smithi13667081-2 2024-09-12T23:48:36.510 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 2 v 1502 (ObjNum 1345 snap 0 seq_num 4243060576) dirty dne 2024-09-12T23:48:36.510 INFO:tasks.rados.rados.0.smithi136.stdout:2953: finishing write tid 5 to smithi13667081-50 2024-09-12T23:48:36.510 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 50 v 1647 (ObjNum 938 snap 294 seq_num 938) dirty exists 2024-09-12T23:48:36.510 INFO:tasks.rados.rados.0.smithi136.stdout:2953: left oid 50 (ObjNum 938 snap 294 seq_num 938) 2024-09-12T23:48:36.513 INFO:tasks.rados.rados.0.smithi136.stdout:2951: done (4 left) 2024-09-12T23:48:36.513 INFO:tasks.rados.rados.0.smithi136.stdout:2953: done (3 left) 2024-09-12T23:48:36.513 INFO:tasks.rados.rados.0.smithi136.stdout:2954: done (2 left) 2024-09-12T23:48:36.513 INFO:tasks.rados.rados.0.smithi136.stdout:2956: done (1 left) 2024-09-12T23:48:36.513 INFO:tasks.rados.rados.0.smithi136.stdout:2957: done (0 left) 2024-09-12T23:48:36.513 INFO:tasks.rados.rados.0.smithi136.stdout:2958: copy_from oid 48 from oid 21 current snap is 294 2024-09-12T23:48:36.513 INFO:tasks.rados.rados.0.smithi136.stdout:2959: read oid 20 snap -1 2024-09-12T23:48:36.513 INFO:tasks.rados.rados.0.smithi136.stdout:2959: expect (ObjNum 937 snap 294 seq_num 937) 2024-09-12T23:48:36.513 INFO:tasks.rados.rados.0.smithi136.stdout:2960: read oid 15 snap -1 2024-09-12T23:48:36.514 INFO:tasks.rados.rados.0.smithi136.stdout:2960: expect (ObjNum 859 snap 278 seq_num 859) 2024-09-12T23:48:36.514 INFO:tasks.rados.rados.0.smithi136.stdout:2961: snap_create 2024-09-12T23:48:36.514 INFO:tasks.rados.rados.0.smithi136.stdout:2958: finishing copy_from to smithi13667081-48 2024-09-12T23:48:36.514 INFO:tasks.rados.rados.0.smithi136.stdout:2958: got expected ENOENT (src dne) 2024-09-12T23:48:36.514 INFO:tasks.rados.rados.0.smithi136.stdout:2958: finishing copy_from racing read to smithi13667081-48 2024-09-12T23:48:36.520 INFO:tasks.rados.rados.0.smithi136.stdout:2959: expect (ObjNum 937 snap 294 seq_num 937) 2024-09-12T23:48:36.587 INFO:tasks.rados.rados.0.smithi136.stdout:2960: expect (ObjNum 859 snap 278 seq_num 859) 2024-09-12T23:48:37.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:36 smithi136 bash[75164]: cluster 2024-09-12T23:48:35.873482+0000 mon.a (mon.0) 1020 : cluster [DBG] osdmap e659: 8 total, 8 up, 8 in 2024-09-12T23:48:37.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:36 smithi136 bash[75164]: cluster 2024-09-12T23:48:36.436598+0000 mon.a (mon.0) 1021 : cluster [DBG] osdmap e660: 8 total, 8 up, 8 in 2024-09-12T23:48:37.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:36 smithi136 bash[76371]: cluster 2024-09-12T23:48:35.873482+0000 mon.a (mon.0) 1020 : cluster [DBG] osdmap e659: 8 total, 8 up, 8 in 2024-09-12T23:48:37.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:36 smithi136 bash[76371]: cluster 2024-09-12T23:48:36.436598+0000 mon.a (mon.0) 1021 : cluster [DBG] osdmap e660: 8 total, 8 up, 8 in 2024-09-12T23:48:37.344 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:36 smithi181 bash[67587]: cluster 2024-09-12T23:48:35.873482+0000 mon.a (mon.0) 1020 : cluster [DBG] osdmap e659: 8 total, 8 up, 8 in 2024-09-12T23:48:37.344 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:36 smithi181 bash[67587]: cluster 2024-09-12T23:48:36.436598+0000 mon.a (mon.0) 1021 : cluster [DBG] osdmap e660: 8 total, 8 up, 8 in 2024-09-12T23:48:38.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:37 smithi136 bash[75164]: cluster 2024-09-12T23:48:36.804425+0000 mgr.y (mgr.24661) 311 : cluster [DBG] pgmap v566: 121 pgs: 121 active+clean; 266 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 29 MiB/s rd, 5.3 MiB/s wr, 18 op/s 2024-09-12T23:48:38.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:37 smithi136 bash[75164]: cluster 2024-09-12T23:48:37.444195+0000 mon.a (mon.0) 1022 : cluster [DBG] osdmap e661: 8 total, 8 up, 8 in 2024-09-12T23:48:38.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:37 smithi136 bash[75164]: audit 2024-09-12T23:48:37.552941+0000 mon.a (mon.0) 1023 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:38.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:37 smithi136 bash[75164]: audit 2024-09-12T23:48:37.562647+0000 mon.a (mon.0) 1024 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:38.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:37 smithi136 bash[76371]: cluster 2024-09-12T23:48:36.804425+0000 mgr.y (mgr.24661) 311 : cluster [DBG] pgmap v566: 121 pgs: 121 active+clean; 266 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 29 MiB/s rd, 5.3 MiB/s wr, 18 op/s 2024-09-12T23:48:38.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:37 smithi136 bash[76371]: cluster 2024-09-12T23:48:37.444195+0000 mon.a (mon.0) 1022 : cluster [DBG] osdmap e661: 8 total, 8 up, 8 in 2024-09-12T23:48:38.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:37 smithi136 bash[76371]: audit 2024-09-12T23:48:37.552941+0000 mon.a (mon.0) 1023 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:38.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:37 smithi136 bash[76371]: audit 2024-09-12T23:48:37.562647+0000 mon.a (mon.0) 1024 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:38.344 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:37 smithi181 bash[67587]: cluster 2024-09-12T23:48:36.804425+0000 mgr.y (mgr.24661) 311 : cluster [DBG] pgmap v566: 121 pgs: 121 active+clean; 266 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 29 MiB/s rd, 5.3 MiB/s wr, 18 op/s 2024-09-12T23:48:38.344 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:37 smithi181 bash[67587]: cluster 2024-09-12T23:48:37.444195+0000 mon.a (mon.0) 1022 : cluster [DBG] osdmap e661: 8 total, 8 up, 8 in 2024-09-12T23:48:38.344 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:37 smithi181 bash[67587]: audit 2024-09-12T23:48:37.552941+0000 mon.a (mon.0) 1023 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:38.344 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:37 smithi181 bash[67587]: audit 2024-09-12T23:48:37.562647+0000 mon.a (mon.0) 1024 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:38.366 INFO:tasks.rados.rados.0.smithi136.stdout:2958: done (3 left) 2024-09-12T23:48:38.366 INFO:tasks.rados.rados.0.smithi136.stdout:2959: done (2 left) 2024-09-12T23:48:38.366 INFO:tasks.rados.rados.0.smithi136.stdout:2960: done (1 left) 2024-09-12T23:48:38.369 INFO:tasks.rados.rados.0.smithi136.stdout:2961: done (0 left) 2024-09-12T23:48:38.370 INFO:tasks.rados.rados.0.smithi136.stdout:2962: setattr oid 17 current snap is 295 2024-09-12T23:48:38.376 INFO:tasks.rados.rados.0.smithi136.stdout:2963: read oid 6 snap -1 2024-09-12T23:48:38.377 INFO:tasks.rados.rados.0.smithi136.stdout:2963: expect deleted 2024-09-12T23:48:38.377 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 26 current snap is 295 2024-09-12T23:48:38.377 INFO:tasks.rados.rados.0.smithi136.stdout:2964: seq_num 940 ranges {0=548864,548864=589824,1138688=212992} 2024-09-12T23:48:38.384 INFO:tasks.rados.rados.0.smithi136.stdout:2964: writing smithi13667081-26 from 0 to 548864 tid 1 2024-09-12T23:48:38.391 INFO:tasks.rados.rados.0.smithi136.stdout:2964: writing smithi13667081-26 from 548864 to 1138688 tid 2 2024-09-12T23:48:38.394 INFO:tasks.rados.rados.0.smithi136.stdout:2964: writing smithi13667081-26 from 1138688 to 1351680 tid 3 2024-09-12T23:48:38.394 INFO:tasks.rados.rados.0.smithi136.stdout:2965: snap_remove snap 278 2024-09-12T23:48:39.360 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 17 v 2237 (ObjNum 1281 snap 0 seq_num 4243431232) dirty exists 2024-09-12T23:48:39.361 INFO:tasks.rados.rados.0.smithi136.stdout:2964: finishing write tid 1 to smithi13667081-26 2024-09-12T23:48:39.361 INFO:tasks.rados.rados.0.smithi136.stdout:2964: finishing write tid 2 to smithi13667081-26 2024-09-12T23:48:39.361 INFO:tasks.rados.rados.0.smithi136.stdout:2964: finishing write tid 3 to smithi13667081-26 2024-09-12T23:48:39.361 INFO:tasks.rados.rados.0.smithi136.stdout:2964: finishing write tid 4 to smithi13667081-26 2024-09-12T23:48:39.361 INFO:tasks.rados.rados.0.smithi136.stdout:2964: finishing write tid 5 to smithi13667081-26 2024-09-12T23:48:39.361 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 26 v 1501 (ObjNum 940 snap 295 seq_num 940) dirty exists 2024-09-12T23:48:39.361 INFO:tasks.rados.rados.0.smithi136.stdout:2964: left oid 26 (ObjNum 940 snap 295 seq_num 940) 2024-09-12T23:48:39.361 INFO:tasks.rados.rados.0.smithi136.stdout:2962: done (3 left) 2024-09-12T23:48:39.361 INFO:tasks.rados.rados.0.smithi136.stdout:2963: done (2 left) 2024-09-12T23:48:39.361 INFO:tasks.rados.rados.0.smithi136.stdout:2964: done (1 left) 2024-09-12T23:48:39.361 INFO:tasks.rados.rados.0.smithi136.stdout:2965: done (0 left) 2024-09-12T23:48:39.362 INFO:tasks.rados.rados.0.smithi136.stdout:2966: rollback oid 5 current snap is 295 2024-09-12T23:48:39.362 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 5 to 291 2024-09-12T23:48:39.362 INFO:tasks.rados.rados.0.smithi136.stdout:2967: read oid 13 snap -1 2024-09-12T23:48:39.362 INFO:tasks.rados.rados.0.smithi136.stdout:2967: expect (ObjNum 893 snap 288 seq_num 893) 2024-09-12T23:48:39.362 INFO:tasks.rados.rados.0.smithi136.stdout:2968: snap_create 2024-09-12T23:48:39.386 INFO:tasks.rados.rados.0.smithi136.stdout:2966: finishing rollback tid 0 to smithi13667081-5 2024-09-12T23:48:39.393 INFO:tasks.rados.rados.0.smithi136.stdout:2966: finishing rollback tid 1 to smithi13667081-5 2024-09-12T23:48:39.394 INFO:tasks.rados.rados.0.smithi136.stdout:2966: finishing rollback tid 2 to smithi13667081-5 2024-09-12T23:48:39.395 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 5 v 2245 (ObjNum 567 snap 162 seq_num 567) dirty exists 2024-09-12T23:48:39.431 INFO:tasks.rados.rados.0.smithi136.stdout:2967: expect (ObjNum 893 snap 288 seq_num 893) 2024-09-12T23:48:39.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:39 smithi181 bash[67587]: audit 2024-09-12T23:48:38.339058+0000 mon.a (mon.0) 1025 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:39.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:39 smithi181 bash[67587]: audit 2024-09-12T23:48:38.349163+0000 mon.a (mon.0) 1026 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:39.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:39 smithi181 bash[67587]: audit 2024-09-12T23:48:38.756816+0000 mon.a (mon.0) 1027 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:39.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:39 smithi181 bash[67587]: audit 2024-09-12T23:48:38.758742+0000 mon.a (mon.0) 1028 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:48:39.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:39 smithi181 bash[67587]: cluster 2024-09-12T23:48:38.805346+0000 mgr.y (mgr.24661) 312 : cluster [DBG] pgmap v568: 121 pgs: 121 active+clean; 266 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 910 KiB/s wr, 12 op/s 2024-09-12T23:48:39.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:48:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:48:39] "GET /metrics HTTP/1.1" 200 36440 "" "Prometheus/2.43.0" 2024-09-12T23:48:39.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:39 smithi136 bash[75164]: audit 2024-09-12T23:48:38.339058+0000 mon.a (mon.0) 1025 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:39.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:39 smithi136 bash[75164]: audit 2024-09-12T23:48:38.349163+0000 mon.a (mon.0) 1026 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:39.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:39 smithi136 bash[75164]: audit 2024-09-12T23:48:38.756816+0000 mon.a (mon.0) 1027 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:39.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:39 smithi136 bash[75164]: audit 2024-09-12T23:48:38.758742+0000 mon.a (mon.0) 1028 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:48:39.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:39 smithi136 bash[75164]: cluster 2024-09-12T23:48:38.805346+0000 mgr.y (mgr.24661) 312 : cluster [DBG] pgmap v568: 121 pgs: 121 active+clean; 266 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 910 KiB/s wr, 12 op/s 2024-09-12T23:48:39.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:39 smithi136 bash[76371]: audit 2024-09-12T23:48:38.339058+0000 mon.a (mon.0) 1025 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:39.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:39 smithi136 bash[76371]: audit 2024-09-12T23:48:38.349163+0000 mon.a (mon.0) 1026 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:39.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:39 smithi136 bash[76371]: audit 2024-09-12T23:48:38.756816+0000 mon.a (mon.0) 1027 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:39.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:39 smithi136 bash[76371]: audit 2024-09-12T23:48:38.758742+0000 mon.a (mon.0) 1028 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:48:39.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:39 smithi136 bash[76371]: cluster 2024-09-12T23:48:38.805346+0000 mgr.y (mgr.24661) 312 : cluster [DBG] pgmap v568: 121 pgs: 121 active+clean; 266 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 910 KiB/s wr, 12 op/s 2024-09-12T23:48:40.361 INFO:tasks.rados.rados.0.smithi136.stdout:2966: done (2 left) 2024-09-12T23:48:40.361 INFO:tasks.rados.rados.0.smithi136.stdout:2967: done (1 left) 2024-09-12T23:48:40.361 INFO:tasks.rados.rados.0.smithi136.stdout:2968: done (0 left) 2024-09-12T23:48:40.362 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 12 current snap is 296 2024-09-12T23:48:40.362 INFO:tasks.rados.rados.0.smithi136.stdout:2969: seq_num 941 ranges {0=565248,565248=475136,1040384=327680} 2024-09-12T23:48:40.370 INFO:tasks.rados.rados.0.smithi136.stdout:2969: writing smithi13667081-12 from 0 to 565248 tid 1 2024-09-12T23:48:40.379 INFO:tasks.rados.rados.0.smithi136.stdout:2969: writing smithi13667081-12 from 565248 to 1040384 tid 2 2024-09-12T23:48:40.386 INFO:tasks.rados.rados.0.smithi136.stdout:2969: writing smithi13667081-12 from 1040384 to 1368064 tid 3 2024-09-12T23:48:40.386 INFO:tasks.rados.rados.0.smithi136.stdout:2970: rollback oid 47 current snap is 296 2024-09-12T23:48:40.386 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 47 to 272 2024-09-12T23:48:40.386 INFO:tasks.rados.rados.0.smithi136.stdout:2969: finishing write tid 1 to smithi13667081-12 2024-09-12T23:48:40.386 INFO:tasks.rados.rados.0.smithi136.stdout:2969: finishing write tid 2 to smithi13667081-12 2024-09-12T23:48:40.386 INFO:tasks.rados.rados.0.smithi136.stdout:2971: snap_create 2024-09-12T23:48:40.390 INFO:tasks.rados.rados.0.smithi136.stdout:2970: finishing rollback tid 0 to smithi13667081-47 2024-09-12T23:48:40.390 INFO:tasks.rados.rados.0.smithi136.stdout:2970: finishing rollback tid 1 to smithi13667081-47 2024-09-12T23:48:40.390 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 47 v 1624 (ObjNum 33 snap 0 seq_num 4243832528) dirty dne 2024-09-12T23:48:40.393 INFO:tasks.rados.rados.0.smithi136.stdout:2969: finishing write tid 3 to smithi13667081-12 2024-09-12T23:48:40.394 INFO:tasks.rados.rados.0.smithi136.stdout:2969: finishing write tid 4 to smithi13667081-12 2024-09-12T23:48:40.397 INFO:tasks.rados.rados.0.smithi136.stdout:2969: finishing write tid 5 to smithi13667081-12 2024-09-12T23:48:40.397 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 12 v 1656 (ObjNum 941 snap 296 seq_num 941) dirty exists 2024-09-12T23:48:40.397 INFO:tasks.rados.rados.0.smithi136.stdout:2969: left oid 12 (ObjNum 941 snap 296 seq_num 941) 2024-09-12T23:48:40.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:40 smithi136 bash[75164]: cluster 2024-09-12T23:48:39.359038+0000 mon.a (mon.0) 1029 : cluster [DBG] osdmap e662: 8 total, 8 up, 8 in 2024-09-12T23:48:40.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:40 smithi136 bash[76371]: cluster 2024-09-12T23:48:39.359038+0000 mon.a (mon.0) 1029 : cluster [DBG] osdmap e662: 8 total, 8 up, 8 in 2024-09-12T23:48:40.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:40 smithi181 bash[67587]: cluster 2024-09-12T23:48:39.359038+0000 mon.a (mon.0) 1029 : cluster [DBG] osdmap e662: 8 total, 8 up, 8 in 2024-09-12T23:48:41.372 INFO:tasks.rados.rados.0.smithi136.stdout:2969: done (2 left) 2024-09-12T23:48:41.372 INFO:tasks.rados.rados.0.smithi136.stdout:2970: done (1 left) 2024-09-12T23:48:41.372 INFO:tasks.rados.rados.0.smithi136.stdout:2971: done (0 left) 2024-09-12T23:48:41.373 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 20 current snap is 297 2024-09-12T23:48:41.373 INFO:tasks.rados.rados.0.smithi136.stdout:2972: seq_num 942 ranges {1302528=581632,1884160=761856,2646016=40960} 2024-09-12T23:48:41.388 INFO:tasks.rados.rados.0.smithi136.stdout:2972: writing smithi13667081-20 from 1302528 to 1884160 tid 1 2024-09-12T23:48:41.394 INFO:tasks.rados.rados.0.smithi136.stdout:2972: writing smithi13667081-20 from 1884160 to 2646016 tid 2 2024-09-12T23:48:41.395 INFO:tasks.rados.rados.0.smithi136.stdout:2972: writing smithi13667081-20 from 2646016 to 2686976 tid 3 2024-09-12T23:48:41.395 INFO:tasks.rados.rados.0.smithi136.stdout:2973: read oid 17 snap 289 2024-09-12T23:48:41.395 INFO:tasks.rados.rados.0.smithi136.stdout:2973: expect (ObjNum 671 snap 196 seq_num 671) 2024-09-12T23:48:41.395 INFO:tasks.rados.rados.0.smithi136.stdout:2974: setattr oid 26 current snap is 297 2024-09-12T23:48:41.398 INFO:tasks.rados.rados.0.smithi136.stdout:2975: read oid 14 snap -1 2024-09-12T23:48:41.398 INFO:tasks.rados.rados.0.smithi136.stdout:2975: expect (ObjNum 922 snap 290 seq_num 922) 2024-09-12T23:48:41.398 INFO:tasks.rados.rados.0.smithi136.stdout:2976: copy_from oid 50 from oid 5 current snap is 297 2024-09-12T23:48:41.398 INFO:tasks.rados.rados.0.smithi136.stdout:2977: snap_create 2024-09-12T23:48:41.401 INFO:tasks.rados.rados.0.smithi136.stdout:2972: finishing write tid 1 to smithi13667081-20 2024-09-12T23:48:41.409 INFO:tasks.rados.rados.0.smithi136.stdout:2972: finishing write tid 2 to smithi13667081-20 2024-09-12T23:48:41.409 INFO:tasks.rados.rados.0.smithi136.stdout:2972: finishing write tid 3 to smithi13667081-20 2024-09-12T23:48:41.409 INFO:tasks.rados.rados.0.smithi136.stdout:2972: finishing write tid 4 to smithi13667081-20 2024-09-12T23:48:41.410 INFO:tasks.rados.rados.0.smithi136.stdout:2972: finishing write tid 5 to smithi13667081-20 2024-09-12T23:48:41.410 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 20 v 1613 (ObjNum 942 snap 297 seq_num 942) dirty exists 2024-09-12T23:48:41.410 INFO:tasks.rados.rados.0.smithi136.stdout:2972: left oid 20 (ObjNum 942 snap 297 seq_num 942) 2024-09-12T23:48:41.417 INFO:tasks.rados.rados.0.smithi136.stdout:2975: expect (ObjNum 922 snap 290 seq_num 922) 2024-09-12T23:48:41.498 INFO:tasks.rados.rados.0.smithi136.stdout:2973: expect (ObjNum 671 snap 196 seq_num 671) 2024-09-12T23:48:41.604 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 26 v 1502 (ObjNum 940 snap 295 seq_num 940) dirty exists 2024-09-12T23:48:41.605 INFO:tasks.rados.rados.0.smithi136.stdout:2976: finishing copy_from racing read to smithi13667081-50 2024-09-12T23:48:41.605 INFO:tasks.rados.rados.0.smithi136.stdout:2976: finishing copy_from to smithi13667081-50 2024-09-12T23:48:41.605 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 50 v 2245 (ObjNum 567 snap 162 seq_num 567) dirty exists 2024-09-12T23:48:41.605 INFO:tasks.rados.rados.0.smithi136.stdout:2972: done (5 left) 2024-09-12T23:48:41.605 INFO:tasks.rados.rados.0.smithi136.stdout:2973: done (4 left) 2024-09-12T23:48:41.605 INFO:tasks.rados.rados.0.smithi136.stdout:2974: done (3 left) 2024-09-12T23:48:41.605 INFO:tasks.rados.rados.0.smithi136.stdout:2975: done (2 left) 2024-09-12T23:48:41.605 INFO:tasks.rados.rados.0.smithi136.stdout:2976: done (1 left) 2024-09-12T23:48:41.605 INFO:tasks.rados.rados.0.smithi136.stdout:2977: done (0 left) 2024-09-12T23:48:41.605 INFO:tasks.rados.rados.0.smithi136.stdout:2978: delete oid 22 current snap is 298 2024-09-12T23:48:41.609 INFO:tasks.rados.rados.0.smithi136.stdout:2978: done (0 left) 2024-09-12T23:48:41.609 INFO:tasks.rados.rados.0.smithi136.stdout:2979: rollback oid 19 current snap is 298 2024-09-12T23:48:41.609 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 19 to 273 2024-09-12T23:48:41.609 INFO:tasks.rados.rados.0.smithi136.stdout:2980: rmattr oid 23 current snap is 298 2024-09-12T23:48:41.610 INFO:tasks.rados.rados.0.smithi136.stdout:2981: read oid 37 snap -1 2024-09-12T23:48:41.610 INFO:tasks.rados.rados.0.smithi136.stdout:2981: expect (ObjNum 636 snap 188 seq_num 636) 2024-09-12T23:48:41.610 INFO:tasks.rados.rados.0.smithi136.stdout:2982: delete oid 10 current snap is 298 2024-09-12T23:48:41.610 INFO:tasks.rados.rados.0.smithi136.stdout:2979: finishing rollback tid 1 to smithi13667081-19 2024-09-12T23:48:41.610 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 19 v 0 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-12T23:48:41.616 INFO:tasks.rados.rados.0.smithi136.stdout:2979: done (3 left) 2024-09-12T23:48:41.616 INFO:tasks.rados.rados.0.smithi136.stdout:2982: done (2 left) 2024-09-12T23:48:41.616 INFO:tasks.rados.rados.0.smithi136.stdout:2983: rmattr oid 18 current snap is 298 2024-09-12T23:48:41.617 INFO:tasks.rados.rados.0.smithi136.stdout:2984: snap_remove snap 294 2024-09-12T23:48:41.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:41 smithi136 bash[75164]: cluster 2024-09-12T23:48:40.364052+0000 mon.a (mon.0) 1030 : cluster [DBG] osdmap e663: 8 total, 8 up, 8 in 2024-09-12T23:48:41.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:41 smithi136 bash[75164]: cluster 2024-09-12T23:48:40.806367+0000 mgr.y (mgr.24661) 313 : cluster [DBG] pgmap v571: 121 pgs: 121 active+clean; 265 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 1.1 MiB/s wr, 15 op/s 2024-09-12T23:48:41.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:41 smithi136 bash[76371]: cluster 2024-09-12T23:48:40.364052+0000 mon.a (mon.0) 1030 : cluster [DBG] osdmap e663: 8 total, 8 up, 8 in 2024-09-12T23:48:41.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:41 smithi136 bash[76371]: cluster 2024-09-12T23:48:40.806367+0000 mgr.y (mgr.24661) 313 : cluster [DBG] pgmap v571: 121 pgs: 121 active+clean; 265 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 1.1 MiB/s wr, 15 op/s 2024-09-12T23:48:41.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:41 smithi181 bash[67587]: cluster 2024-09-12T23:48:40.364052+0000 mon.a (mon.0) 1030 : cluster [DBG] osdmap e663: 8 total, 8 up, 8 in 2024-09-12T23:48:41.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:41 smithi181 bash[67587]: cluster 2024-09-12T23:48:40.806367+0000 mgr.y (mgr.24661) 313 : cluster [DBG] pgmap v571: 121 pgs: 121 active+clean; 265 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 1.1 MiB/s wr, 15 op/s 2024-09-12T23:48:42.434 INFO:tasks.rados.rados.0.smithi136.stdout:2984: done (3 left) 2024-09-12T23:48:42.434 INFO:tasks.rados.rados.0.smithi136.stdout:2985: rollback oid 26 current snap is 298 2024-09-12T23:48:42.434 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 26 to 263 2024-09-12T23:48:42.434 INFO:tasks.rados.rados.0.smithi136.stdout:2981: expect (ObjNum 636 snap 188 seq_num 636) 2024-09-12T23:48:42.469 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 23 v 2257 (ObjNum 886 snap 287 seq_num 886) dirty exists 2024-09-12T23:48:42.469 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 18 v 979 (ObjNum 931 snap 292 seq_num 931) dirty exists 2024-09-12T23:48:42.469 INFO:tasks.rados.rados.0.smithi136.stdout:2985: finishing rollback tid 0 to smithi13667081-26 2024-09-12T23:48:42.469 INFO:tasks.rados.rados.0.smithi136.stdout:2985: finishing rollback tid 1 to smithi13667081-26 2024-09-12T23:48:42.469 INFO:tasks.rados.rados.0.smithi136.stdout:2985: finishing rollback tid 2 to smithi13667081-26 2024-09-12T23:48:42.469 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 26 v 1505 (ObjNum 822 snap 258 seq_num 822) dirty exists 2024-09-12T23:48:42.469 INFO:tasks.rados.rados.0.smithi136.stdout:2980: done (3 left) 2024-09-12T23:48:42.469 INFO:tasks.rados.rados.0.smithi136.stdout:2981: done (2 left) 2024-09-12T23:48:42.469 INFO:tasks.rados.rados.0.smithi136.stdout:2983: done (1 left) 2024-09-12T23:48:42.469 INFO:tasks.rados.rados.0.smithi136.stdout:2985: done (0 left) 2024-09-12T23:48:42.470 INFO:tasks.rados.rados.0.smithi136.stdout:2986: copy_from oid 12 from oid 39 current snap is 298 2024-09-12T23:48:42.470 INFO:tasks.rados.rados.0.smithi136.stdout:2987: read oid 10 snap -1 2024-09-12T23:48:42.470 INFO:tasks.rados.rados.0.smithi136.stdout:2987: expect deleted 2024-09-12T23:48:42.470 INFO:tasks.rados.rados.0.smithi136.stdout:2988: copy_from oid 23 from oid 34 current snap is 298 2024-09-12T23:48:42.470 INFO:tasks.rados.rados.0.smithi136.stdout:2989: read oid 5 snap -1 2024-09-12T23:48:42.470 INFO:tasks.rados.rados.0.smithi136.stdout:2989: expect (ObjNum 567 snap 162 seq_num 567) 2024-09-12T23:48:42.470 INFO:tasks.rados.rados.0.smithi136.stdout:2990: setattr oid 33 current snap is 298 2024-09-12T23:48:42.470 INFO:tasks.rados.rados.0.smithi136.stdout:2987: done (4 left) 2024-09-12T23:48:42.470 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 40 current snap is 298 2024-09-12T23:48:42.471 INFO:tasks.rados.rados.0.smithi136.stdout:2991: seq_num 945 ranges {0=630784,630784=458752,1089536=344064} 2024-09-12T23:48:42.474 INFO:tasks.rados.rados.0.smithi136.stdout:2991: writing smithi13667081-40 from 0 to 630784 tid 1 2024-09-12T23:48:42.477 INFO:tasks.rados.rados.0.smithi136.stdout:2991: writing smithi13667081-40 from 630784 to 1089536 tid 2 2024-09-12T23:48:42.480 INFO:tasks.rados.rados.0.smithi136.stdout:2991: writing smithi13667081-40 from 1089536 to 1433600 tid 3 2024-09-12T23:48:42.480 INFO:tasks.rados.rados.0.smithi136.stdout:2992: delete oid 43 current snap is 298 2024-09-12T23:48:42.480 INFO:tasks.rados.rados.0.smithi136.stdout:2986: finishing copy_from to smithi13667081-12 2024-09-12T23:48:42.480 INFO:tasks.rados.rados.0.smithi136.stdout:2986: got expected ENOENT (src dne) 2024-09-12T23:48:42.480 INFO:tasks.rados.rados.0.smithi136.stdout:2988: finishing copy_from to smithi13667081-23 2024-09-12T23:48:42.480 INFO:tasks.rados.rados.0.smithi136.stdout:2988: got expected ENOENT (src dne) 2024-09-12T23:48:42.480 INFO:tasks.rados.rados.0.smithi136.stdout:2986: finishing copy_from racing read to smithi13667081-12 2024-09-12T23:48:42.480 INFO:tasks.rados.rados.0.smithi136.stdout:2988: finishing copy_from racing read to smithi13667081-23 2024-09-12T23:48:42.481 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 33 v 1503 (ObjNum 894 snap 288 seq_num 894) dirty exists 2024-09-12T23:48:42.484 INFO:tasks.rados.rados.0.smithi136.stdout:2986: done (5 left) 2024-09-12T23:48:42.485 INFO:tasks.rados.rados.0.smithi136.stdout:2988: done (4 left) 2024-09-12T23:48:42.485 INFO:tasks.rados.rados.0.smithi136.stdout:2990: done (3 left) 2024-09-12T23:48:42.485 INFO:tasks.rados.rados.0.smithi136.stdout:2992: done (2 left) 2024-09-12T23:48:42.485 INFO:tasks.rados.rados.0.smithi136.stdout:2993: copy_from oid 45 from oid 32 current snap is 298 2024-09-12T23:48:42.485 INFO:tasks.rados.rados.0.smithi136.stdout:2994: rollback oid 21 current snap is 298 2024-09-12T23:48:42.485 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 21 to 293 2024-09-12T23:48:42.485 INFO:tasks.rados.rados.0.smithi136.stdout:2995: snap_create 2024-09-12T23:48:42.490 INFO:tasks.rados.rados.0.smithi136.stdout:2994: finishing rollback tid 1 to smithi13667081-21 2024-09-12T23:48:42.490 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 21 v 1796 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-12T23:48:42.494 INFO:tasks.rados.rados.0.smithi136.stdout:2989: expect (ObjNum 567 snap 162 seq_num 567) 2024-09-12T23:48:42.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:42 smithi181 bash[67587]: cluster 2024-09-12T23:48:41.371752+0000 mon.a (mon.0) 1031 : cluster [DBG] osdmap e664: 8 total, 8 up, 8 in 2024-09-12T23:48:42.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:42 smithi181 bash[67587]: cluster 2024-09-12T23:48:41.438147+0000 mon.a (mon.0) 1032 : cluster [DBG] osdmap e665: 8 total, 8 up, 8 in 2024-09-12T23:48:42.595 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:48:42 smithi181 bash[60471]: ts=2024-09-12T23:48:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:48:42.656 INFO:teuthology.orchestra.run.smithi136.stdout:true 2024-09-12T23:48:42.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:42 smithi136 bash[75164]: cluster 2024-09-12T23:48:41.371752+0000 mon.a (mon.0) 1031 : cluster [DBG] osdmap e664: 8 total, 8 up, 8 in 2024-09-12T23:48:42.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:42 smithi136 bash[75164]: cluster 2024-09-12T23:48:41.438147+0000 mon.a (mon.0) 1032 : cluster [DBG] osdmap e665: 8 total, 8 up, 8 in 2024-09-12T23:48:42.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:42 smithi136 bash[76371]: cluster 2024-09-12T23:48:41.371752+0000 mon.a (mon.0) 1031 : cluster [DBG] osdmap e664: 8 total, 8 up, 8 in 2024-09-12T23:48:42.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:42 smithi136 bash[76371]: cluster 2024-09-12T23:48:41.438147+0000 mon.a (mon.0) 1032 : cluster [DBG] osdmap e665: 8 total, 8 up, 8 in 2024-09-12T23:48:42.829 INFO:tasks.rados.rados.0.smithi136.stdout:2993: finishing copy_from racing read to smithi13667081-45 2024-09-12T23:48:42.829 INFO:tasks.rados.rados.0.smithi136.stdout:2993: finishing copy_from to smithi13667081-45 2024-09-12T23:48:42.829 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 45 v 1620 (ObjNum 889 snap 288 seq_num 889) dirty exists 2024-09-12T23:48:42.829 INFO:tasks.rados.rados.0.smithi136.stdout:2991: finishing write tid 1 to smithi13667081-40 2024-09-12T23:48:42.829 INFO:tasks.rados.rados.0.smithi136.stdout:2991: finishing write tid 2 to smithi13667081-40 2024-09-12T23:48:42.830 INFO:tasks.rados.rados.0.smithi136.stdout:2991: finishing write tid 3 to smithi13667081-40 2024-09-12T23:48:42.830 INFO:tasks.rados.rados.0.smithi136.stdout:2991: finishing write tid 4 to smithi13667081-40 2024-09-12T23:48:42.830 INFO:tasks.rados.rados.0.smithi136.stdout:2991: finishing write tid 5 to smithi13667081-40 2024-09-12T23:48:42.830 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 40 v 2265 (ObjNum 945 snap 298 seq_num 945) dirty exists 2024-09-12T23:48:42.830 INFO:tasks.rados.rados.0.smithi136.stdout:2991: left oid 40 (ObjNum 945 snap 298 seq_num 945) 2024-09-12T23:48:42.966 INFO:teuthology.orchestra.run.smithi136.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-12T23:48:42.967 INFO:teuthology.orchestra.run.smithi136.stdout:alertmanager.a smithi136 *:9093,9094 running (8m) 37s ago 14m 14.2M - 0.25.0 c8568f914cd2 b5197f020eeb 2024-09-12T23:48:42.967 INFO:teuthology.orchestra.run.smithi136.stdout:grafana.a smithi181 *:3000 running (7m) 5s ago 13m 75.6M - 9.4.7 954c08fa6188 256c181be984 2024-09-12T23:48:42.967 INFO:teuthology.orchestra.run.smithi136.stdout:mds.a smithi136 running (44s) 37s ago 15m 11.6M - 18.2.4-837-g7a853b17 f21ba1eea74e a20ac830342c 2024-09-12T23:48:42.967 INFO:teuthology.orchestra.run.smithi136.stdout:mds.b smithi181 running (11s) 5s ago 15m 11.6M - 18.2.4-837-g7a853b17 f21ba1eea74e 0dc9f76cf0b8 2024-09-12T23:48:42.967 INFO:teuthology.orchestra.run.smithi136.stdout:mgr.x smithi181 *:8443,9283,8765 running (7m) 5s ago 22m 428M - 18.2.4-837-g7a853b17 f21ba1eea74e cc664dc337f7 2024-09-12T23:48:42.968 INFO:teuthology.orchestra.run.smithi136.stdout:mgr.y smithi136 *:8443,9283,8765 running (7m) 37s ago 25m 495M - 18.2.4-837-g7a853b17 f21ba1eea74e 272f10b42fc3 2024-09-12T23:48:42.968 INFO:teuthology.orchestra.run.smithi136.stdout:mon.a smithi136 running (6m) 37s ago 25m 66.2M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e ad8d21d58ad9 2024-09-12T23:48:42.968 INFO:teuthology.orchestra.run.smithi136.stdout:mon.b smithi181 running (5m) 5s ago 23m 50.3M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e a70ec8315934 2024-09-12T23:48:42.968 INFO:teuthology.orchestra.run.smithi136.stdout:mon.c smithi136 running (6m) 37s ago 23m 54.9M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e 44d1be055a5e 2024-09-12T23:48:42.968 INFO:teuthology.orchestra.run.smithi136.stdout:node-exporter.a smithi136 *:9100 running (8m) 37s ago 14m 8515k - 1.5.0 0da6a335fe13 ccfc37c96620 2024-09-12T23:48:42.968 INFO:teuthology.orchestra.run.smithi136.stdout:node-exporter.b smithi181 *:9100 running (8m) 5s ago 14m 8152k - 1.5.0 0da6a335fe13 80bf0e383230 2024-09-12T23:48:42.968 INFO:teuthology.orchestra.run.smithi136.stdout:osd.0 smithi136 running (4m) 37s ago 21m 299M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 7da21f765721 2024-09-12T23:48:42.968 INFO:teuthology.orchestra.run.smithi136.stdout:osd.1 smithi136 running (4m) 37s ago 21m 315M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 5302be0a016b 2024-09-12T23:48:42.968 INFO:teuthology.orchestra.run.smithi136.stdout:osd.2 smithi136 running (3m) 37s ago 20m 323M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 2e12d1a31223 2024-09-12T23:48:42.968 INFO:teuthology.orchestra.run.smithi136.stdout:osd.3 smithi136 running (3m) 37s ago 19m 245M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e aa9dc124e59c 2024-09-12T23:48:42.968 INFO:teuthology.orchestra.run.smithi136.stdout:osd.4 smithi181 running (2m) 5s ago 18m 429M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 4ee3b228ce7d 2024-09-12T23:48:42.968 INFO:teuthology.orchestra.run.smithi136.stdout:osd.5 smithi181 running (2m) 5s ago 17m 211M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 3f9a09565258 2024-09-12T23:48:42.968 INFO:teuthology.orchestra.run.smithi136.stdout:osd.6 smithi181 running (97s) 5s ago 16m 131M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e ce539f6fb854 2024-09-12T23:48:42.969 INFO:teuthology.orchestra.run.smithi136.stdout:osd.7 smithi181 running (68s) 5s ago 16m 202M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 6ad9ab169a7b 2024-09-12T23:48:42.969 INFO:teuthology.orchestra.run.smithi136.stdout:prometheus.a smithi181 *:9095 running (7m) 5s ago 14m 57.7M - 2.43.0 a07b618ecd1d 3853d003a5cc 2024-09-12T23:48:43.314 INFO:teuthology.orchestra.run.smithi136.stdout:{ 2024-09-12T23:48:43.314 INFO:teuthology.orchestra.run.smithi136.stdout: "mon": { 2024-09-12T23:48:43.314 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 3 2024-09-12T23:48:43.314 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:48:43.314 INFO:teuthology.orchestra.run.smithi136.stdout: "mgr": { 2024-09-12T23:48:43.314 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 2 2024-09-12T23:48:43.314 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:48:43.315 INFO:teuthology.orchestra.run.smithi136.stdout: "osd": { 2024-09-12T23:48:43.315 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 8 2024-09-12T23:48:43.315 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:48:43.315 INFO:teuthology.orchestra.run.smithi136.stdout: "mds": { 2024-09-12T23:48:43.315 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 2 2024-09-12T23:48:43.315 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:48:43.315 INFO:teuthology.orchestra.run.smithi136.stdout: "overall": { 2024-09-12T23:48:43.315 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 15 2024-09-12T23:48:43.315 INFO:teuthology.orchestra.run.smithi136.stdout: } 2024-09-12T23:48:43.315 INFO:teuthology.orchestra.run.smithi136.stdout:} 2024-09-12T23:48:43.450 INFO:tasks.rados.rados.0.smithi136.stdout:2989: done (4 left) 2024-09-12T23:48:43.451 INFO:tasks.rados.rados.0.smithi136.stdout:2991: done (3 left) 2024-09-12T23:48:43.451 INFO:tasks.rados.rados.0.smithi136.stdout:2993: done (2 left) 2024-09-12T23:48:43.451 INFO:tasks.rados.rados.0.smithi136.stdout:2994: done (1 left) 2024-09-12T23:48:43.451 INFO:tasks.rados.rados.0.smithi136.stdout:2995: done (0 left) 2024-09-12T23:48:43.451 INFO:tasks.rados.rados.0.smithi136.stdout:2996: read oid 10 snap 290 2024-09-12T23:48:43.451 INFO:tasks.rados.rados.0.smithi136.stdout:2996: expect (ObjNum 850 snap 272 seq_num 850) 2024-09-12T23:48:43.451 INFO:tasks.rados.rados.0.smithi136.stdout:2997: read oid 15 snap -1 2024-09-12T23:48:43.451 INFO:tasks.rados.rados.0.smithi136.stdout:2997: expect (ObjNum 859 snap 278 seq_num 859) 2024-09-12T23:48:43.451 INFO:tasks.rados.rados.0.smithi136.stdout:2998: setattr oid 23 current snap is 299 2024-09-12T23:48:43.453 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 1 current snap is 299 2024-09-12T23:48:43.453 INFO:tasks.rados.rados.0.smithi136.stdout:2999: seq_num 947 ranges {0=663552,663552=630784,1294336=172032} 2024-09-12T23:48:43.466 INFO:tasks.rados.rados.0.smithi136.stdout:2999: writing smithi13667081-1 from 0 to 663552 tid 1 2024-09-12T23:48:43.479 INFO:tasks.rados.rados.0.smithi136.stdout:2999: writing smithi13667081-1 from 663552 to 1294336 tid 2 2024-09-12T23:48:43.482 INFO:tasks.rados.rados.0.smithi136.stdout:2999: writing smithi13667081-1 from 1294336 to 1466368 tid 3 2024-09-12T23:48:43.482 INFO:tasks.rados.rados.0.smithi136.stdout:3000: snap_remove snap 272 2024-09-12T23:48:43.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:43 smithi181 bash[67587]: cluster 2024-09-12T23:48:42.437856+0000 mon.a (mon.0) 1033 : cluster [DBG] osdmap e666: 8 total, 8 up, 8 in 2024-09-12T23:48:43.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:43 smithi181 bash[67587]: audit 2024-09-12T23:48:42.647291+0000 mgr.y (mgr.24661) 314 : audit [DBG] from='client.34422 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:48:43.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:43 smithi181 bash[67587]: cluster 2024-09-12T23:48:42.807099+0000 mgr.y (mgr.24661) 315 : cluster [DBG] pgmap v575: 121 pgs: 121 active+clean; 268 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 1.7 MiB/s wr, 17 op/s 2024-09-12T23:48:43.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:43 smithi181 bash[67587]: audit 2024-09-12T23:48:43.317489+0000 mon.c (mon.2) 13 : audit [DBG] from='client.? 172.21.15.136:0/1600035409' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:43.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:43 smithi136 bash[75164]: cluster 2024-09-12T23:48:42.437856+0000 mon.a (mon.0) 1033 : cluster [DBG] osdmap e666: 8 total, 8 up, 8 in 2024-09-12T23:48:43.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:43 smithi136 bash[75164]: audit 2024-09-12T23:48:42.647291+0000 mgr.y (mgr.24661) 314 : audit [DBG] from='client.34422 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:48:43.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:43 smithi136 bash[75164]: cluster 2024-09-12T23:48:42.807099+0000 mgr.y (mgr.24661) 315 : cluster [DBG] pgmap v575: 121 pgs: 121 active+clean; 268 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 1.7 MiB/s wr, 17 op/s 2024-09-12T23:48:43.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:43 smithi136 bash[75164]: audit 2024-09-12T23:48:43.317489+0000 mon.c (mon.2) 13 : audit [DBG] from='client.? 172.21.15.136:0/1600035409' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:43.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:43 smithi136 bash[76371]: cluster 2024-09-12T23:48:42.437856+0000 mon.a (mon.0) 1033 : cluster [DBG] osdmap e666: 8 total, 8 up, 8 in 2024-09-12T23:48:43.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:43 smithi136 bash[76371]: audit 2024-09-12T23:48:42.647291+0000 mgr.y (mgr.24661) 314 : audit [DBG] from='client.34422 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:48:43.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:43 smithi136 bash[76371]: cluster 2024-09-12T23:48:42.807099+0000 mgr.y (mgr.24661) 315 : cluster [DBG] pgmap v575: 121 pgs: 121 active+clean; 268 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 1.7 MiB/s wr, 17 op/s 2024-09-12T23:48:43.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:43 smithi136 bash[76371]: audit 2024-09-12T23:48:43.317489+0000 mon.c (mon.2) 13 : audit [DBG] from='client.? 172.21.15.136:0/1600035409' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:44.460 INFO:tasks.rados.rados.0.smithi136.stdout:3000: done (4 left) 2024-09-12T23:48:44.460 INFO:tasks.rados.rados.0.smithi136.stdout:3001: delete oid 5 current snap is 299 2024-09-12T23:48:44.460 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 23 v 2267 (ObjNum 886 snap 287 seq_num 886) dirty exists 2024-09-12T23:48:44.460 INFO:tasks.rados.rados.0.smithi136.stdout:2996: expect (ObjNum 850 snap 272 seq_num 850) 2024-09-12T23:48:44.539 INFO:tasks.rados.rados.0.smithi136.stdout:2999: finishing write tid 1 to smithi13667081-1 2024-09-12T23:48:44.540 INFO:tasks.rados.rados.0.smithi136.stdout:2999: finishing write tid 2 to smithi13667081-1 2024-09-12T23:48:44.540 INFO:tasks.rados.rados.0.smithi136.stdout:2999: finishing write tid 3 to smithi13667081-1 2024-09-12T23:48:44.540 INFO:tasks.rados.rados.0.smithi136.stdout:2999: finishing write tid 4 to smithi13667081-1 2024-09-12T23:48:44.540 INFO:tasks.rados.rados.0.smithi136.stdout:2999: finishing write tid 5 to smithi13667081-1 2024-09-12T23:48:44.540 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 1 v 1803 (ObjNum 947 snap 299 seq_num 947) dirty exists 2024-09-12T23:48:44.540 INFO:tasks.rados.rados.0.smithi136.stdout:2999: left oid 1 (ObjNum 947 snap 299 seq_num 947) 2024-09-12T23:48:44.540 INFO:tasks.rados.rados.0.smithi136.stdout:2997: expect (ObjNum 859 snap 278 seq_num 859) 2024-09-12T23:48:44.744 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:44 smithi181 bash[67587]: audit 2024-09-12T23:48:42.935193+0000 mgr.y (mgr.24661) 316 : audit [DBG] from='client.44348 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:48:44.744 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:44 smithi181 bash[67587]: cluster 2024-09-12T23:48:43.452857+0000 mon.a (mon.0) 1034 : cluster [DBG] osdmap e667: 8 total, 8 up, 8 in 2024-09-12T23:48:44.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:44 smithi136 bash[75164]: audit 2024-09-12T23:48:42.935193+0000 mgr.y (mgr.24661) 316 : audit [DBG] from='client.44348 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:48:44.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:44 smithi136 bash[75164]: cluster 2024-09-12T23:48:43.452857+0000 mon.a (mon.0) 1034 : cluster [DBG] osdmap e667: 8 total, 8 up, 8 in 2024-09-12T23:48:44.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:44 smithi136 bash[76371]: audit 2024-09-12T23:48:42.935193+0000 mgr.y (mgr.24661) 316 : audit [DBG] from='client.44348 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:48:44.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:44 smithi136 bash[76371]: cluster 2024-09-12T23:48:43.452857+0000 mon.a (mon.0) 1034 : cluster [DBG] osdmap e667: 8 total, 8 up, 8 in 2024-09-12T23:48:45.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:45 smithi181 bash[67587]: cluster 2024-09-12T23:48:44.461874+0000 mon.a (mon.0) 1035 : cluster [DBG] osdmap e668: 8 total, 8 up, 8 in 2024-09-12T23:48:45.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:45 smithi181 bash[67587]: cluster 2024-09-12T23:48:44.807802+0000 mgr.y (mgr.24661) 317 : cluster [DBG] pgmap v578: 121 pgs: 121 active+clean; 276 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 3.4 MiB/s wr, 21 op/s 2024-09-12T23:48:45.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:45 smithi136 bash[75164]: cluster 2024-09-12T23:48:44.461874+0000 mon.a (mon.0) 1035 : cluster [DBG] osdmap e668: 8 total, 8 up, 8 in 2024-09-12T23:48:45.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:45 smithi136 bash[75164]: cluster 2024-09-12T23:48:44.807802+0000 mgr.y (mgr.24661) 317 : cluster [DBG] pgmap v578: 121 pgs: 121 active+clean; 276 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 3.4 MiB/s wr, 21 op/s 2024-09-12T23:48:45.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:45 smithi136 bash[76371]: cluster 2024-09-12T23:48:44.461874+0000 mon.a (mon.0) 1035 : cluster [DBG] osdmap e668: 8 total, 8 up, 8 in 2024-09-12T23:48:45.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:45 smithi136 bash[76371]: cluster 2024-09-12T23:48:44.807802+0000 mgr.y (mgr.24661) 317 : cluster [DBG] pgmap v578: 121 pgs: 121 active+clean; 276 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 3.4 MiB/s wr, 21 op/s 2024-09-12T23:48:46.280 INFO:tasks.rados.rados.0.smithi136.stdout:2996: done (4 left) 2024-09-12T23:48:46.280 INFO:tasks.rados.rados.0.smithi136.stdout:2997: done (3 left) 2024-09-12T23:48:46.283 INFO:tasks.rados.rados.0.smithi136.stdout:2998: done (2 left) 2024-09-12T23:48:46.283 INFO:tasks.rados.rados.0.smithi136.stdout:2999: done (1 left) 2024-09-12T23:48:46.283 INFO:tasks.rados.rados.0.smithi136.stdout:3001: done (0 left) 2024-09-12T23:48:46.283 INFO:tasks.rados.rados.0.smithi136.stdout:3002: snap_create 2024-09-12T23:48:46.441 INFO:tasks.rados.rados.0.smithi136.stdout:3002: done (0 left) 2024-09-12T23:48:46.441 INFO:tasks.rados.rados.0.smithi136.stdout:3003: read oid 43 snap 293 2024-09-12T23:48:46.441 INFO:tasks.rados.rados.0.smithi136.stdout:3003: expect (ObjNum 915 snap 290 seq_num 915) 2024-09-12T23:48:46.442 INFO:tasks.rados.rados.0.smithi136.stdout:3004: snap_create 2024-09-12T23:48:46.458 INFO:tasks.rados.rados.0.smithi136.stdout:3003: expect (ObjNum 915 snap 290 seq_num 915) 2024-09-12T23:48:47.449 INFO:tasks.rados.rados.0.smithi136.stdout:3003: done (1 left) 2024-09-12T23:48:47.450 INFO:tasks.rados.rados.0.smithi136.stdout:3004: done (0 left) 2024-09-12T23:48:47.450 INFO:tasks.rados.rados.0.smithi136.stdout:3005: rmattr oid 44 current snap is 301 2024-09-12T23:48:47.450 INFO:tasks.rados.rados.0.smithi136.stdout:3005: done (0 left) 2024-09-12T23:48:47.450 INFO:tasks.rados.rados.0.smithi136.stdout:3006: snap_remove snap 292 2024-09-12T23:48:47.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:47 smithi181 bash[67587]: cluster 2024-09-12T23:48:46.443263+0000 mon.a (mon.0) 1036 : cluster [DBG] osdmap e669: 8 total, 8 up, 8 in 2024-09-12T23:48:47.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:47 smithi181 bash[67587]: cluster 2024-09-12T23:48:46.808792+0000 mgr.y (mgr.24661) 318 : cluster [DBG] pgmap v580: 121 pgs: 121 active+clean; 279 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 2.3 MiB/s wr, 19 op/s 2024-09-12T23:48:47.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:47 smithi136 bash[75164]: cluster 2024-09-12T23:48:46.443263+0000 mon.a (mon.0) 1036 : cluster [DBG] osdmap e669: 8 total, 8 up, 8 in 2024-09-12T23:48:47.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:47 smithi136 bash[75164]: cluster 2024-09-12T23:48:46.808792+0000 mgr.y (mgr.24661) 318 : cluster [DBG] pgmap v580: 121 pgs: 121 active+clean; 279 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 2.3 MiB/s wr, 19 op/s 2024-09-12T23:48:47.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:47 smithi136 bash[76371]: cluster 2024-09-12T23:48:46.443263+0000 mon.a (mon.0) 1036 : cluster [DBG] osdmap e669: 8 total, 8 up, 8 in 2024-09-12T23:48:47.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:47 smithi136 bash[76371]: cluster 2024-09-12T23:48:46.808792+0000 mgr.y (mgr.24661) 318 : cluster [DBG] pgmap v580: 121 pgs: 121 active+clean; 279 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 2.3 MiB/s wr, 19 op/s 2024-09-12T23:48:48.464 INFO:tasks.rados.rados.0.smithi136.stdout:3006: done (0 left) 2024-09-12T23:48:48.464 INFO:tasks.rados.rados.0.smithi136.stdout:3007: read oid 10 snap -1 2024-09-12T23:48:48.464 INFO:tasks.rados.rados.0.smithi136.stdout:3007: expect deleted 2024-09-12T23:48:48.464 INFO:tasks.rados.rados.0.smithi136.stdout:3008: copy_from oid 38 from oid 48 current snap is 301 2024-09-12T23:48:48.464 INFO:tasks.rados.rados.0.smithi136.stdout:3009: copy_from oid 2 from oid 9 current snap is 301 2024-09-12T23:48:48.464 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 4 current snap is 301 2024-09-12T23:48:48.464 INFO:tasks.rados.rados.0.smithi136.stdout:3010: seq_num 949 ranges {13631488=704512,14336000=794624,15130624=8192} 2024-09-12T23:48:48.583 INFO:tasks.rados.rados.0.smithi136.stdout:3010: writing smithi13667081-4 from 13631488 to 14336000 tid 1 2024-09-12T23:48:48.588 INFO:tasks.rados.rados.0.smithi136.stdout:3010: writing smithi13667081-4 from 14336000 to 15130624 tid 2 2024-09-12T23:48:48.588 INFO:tasks.rados.rados.0.smithi136.stdout:3010: writing smithi13667081-4 from 15130624 to 15138816 tid 3 2024-09-12T23:48:48.588 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 31 current snap is 301 2024-09-12T23:48:48.589 INFO:tasks.rados.rados.0.smithi136.stdout:3011: seq_num 950 ranges {1236992=720896,1957888=679936,2637824=122880} 2024-09-12T23:48:48.599 INFO:tasks.rados.rados.0.smithi136.stdout:3011: writing smithi13667081-31 from 1236992 to 1957888 tid 1 2024-09-12T23:48:48.603 INFO:tasks.rados.rados.0.smithi136.stdout:3011: writing smithi13667081-31 from 1957888 to 2637824 tid 2 2024-09-12T23:48:48.604 INFO:tasks.rados.rados.0.smithi136.stdout:3011: writing smithi13667081-31 from 2637824 to 2760704 tid 3 2024-09-12T23:48:48.604 INFO:tasks.rados.rados.0.smithi136.stdout:3012: snap_remove snap 279 2024-09-12T23:48:48.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[75164]: cluster 2024-09-12T23:48:47.452137+0000 mon.a (mon.0) 1037 : cluster [DBG] osdmap e670: 8 total, 8 up, 8 in 2024-09-12T23:48:48.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[75164]: audit 2024-09-12T23:48:47.963794+0000 mon.a (mon.0) 1038 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:48.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[75164]: audit 2024-09-12T23:48:47.974473+0000 mon.a (mon.0) 1039 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:48.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[75164]: audit 2024-09-12T23:48:47.976640+0000 mon.a (mon.0) 1040 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:48:48.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[75164]: audit 2024-09-12T23:48:47.978552+0000 mon.a (mon.0) 1041 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:48:48.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[75164]: audit 2024-09-12T23:48:47.989050+0000 mon.a (mon.0) 1042 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:48.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[75164]: audit 2024-09-12T23:48:48.140444+0000 mon.a (mon.0) 1043 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:48:48.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[75164]: audit 2024-09-12T23:48:48.144792+0000 mon.a (mon.0) 1044 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:48.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[75164]: audit 2024-09-12T23:48:48.147264+0000 mon.a (mon.0) 1045 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:48.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[75164]: audit 2024-09-12T23:48:48.149111+0000 mon.a (mon.0) 1046 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:48.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[75164]: audit 2024-09-12T23:48:48.152146+0000 mon.a (mon.0) 1047 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:48.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[75164]: audit 2024-09-12T23:48:48.155683+0000 mon.a (mon.0) 1048 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:48.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[75164]: audit 2024-09-12T23:48:48.162555+0000 mon.a (mon.0) 1049 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:48.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[75164]: audit 2024-09-12T23:48:48.164342+0000 mon.a (mon.0) 1050 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.a"}]: dispatch 2024-09-12T23:48:48.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[75164]: audit 2024-09-12T23:48:48.173603+0000 mon.a (mon.0) 1051 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.a"}]': finished 2024-09-12T23:48:48.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[75164]: audit 2024-09-12T23:48:48.175421+0000 mon.a (mon.0) 1052 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.b"}]: dispatch 2024-09-12T23:48:48.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[75164]: audit 2024-09-12T23:48:48.183341+0000 mon.a (mon.0) 1053 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.b"}]': finished 2024-09-12T23:48:48.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[75164]: audit 2024-09-12T23:48:48.186252+0000 mon.a (mon.0) 1054 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:48.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[75164]: audit 2024-09-12T23:48:48.196317+0000 mon.a (mon.0) 1055 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:48.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[76371]: cluster 2024-09-12T23:48:47.452137+0000 mon.a (mon.0) 1037 : cluster [DBG] osdmap e670: 8 total, 8 up, 8 in 2024-09-12T23:48:48.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[76371]: audit 2024-09-12T23:48:47.963794+0000 mon.a (mon.0) 1038 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:48.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[76371]: audit 2024-09-12T23:48:47.974473+0000 mon.a (mon.0) 1039 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:48.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[76371]: audit 2024-09-12T23:48:47.976640+0000 mon.a (mon.0) 1040 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:48:48.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[76371]: audit 2024-09-12T23:48:47.978552+0000 mon.a (mon.0) 1041 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:48:48.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[76371]: audit 2024-09-12T23:48:47.989050+0000 mon.a (mon.0) 1042 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:48.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[76371]: audit 2024-09-12T23:48:48.140444+0000 mon.a (mon.0) 1043 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:48:48.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[76371]: audit 2024-09-12T23:48:48.144792+0000 mon.a (mon.0) 1044 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:48.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[76371]: audit 2024-09-12T23:48:48.147264+0000 mon.a (mon.0) 1045 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:48.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[76371]: audit 2024-09-12T23:48:48.149111+0000 mon.a (mon.0) 1046 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:48.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[76371]: audit 2024-09-12T23:48:48.152146+0000 mon.a (mon.0) 1047 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:48.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[76371]: audit 2024-09-12T23:48:48.155683+0000 mon.a (mon.0) 1048 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:48.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[76371]: audit 2024-09-12T23:48:48.162555+0000 mon.a (mon.0) 1049 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:48.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[76371]: audit 2024-09-12T23:48:48.164342+0000 mon.a (mon.0) 1050 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.a"}]: dispatch 2024-09-12T23:48:48.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[76371]: audit 2024-09-12T23:48:48.173603+0000 mon.a (mon.0) 1051 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.a"}]': finished 2024-09-12T23:48:48.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[76371]: audit 2024-09-12T23:48:48.175421+0000 mon.a (mon.0) 1052 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.b"}]: dispatch 2024-09-12T23:48:48.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[76371]: audit 2024-09-12T23:48:48.183341+0000 mon.a (mon.0) 1053 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.b"}]': finished 2024-09-12T23:48:48.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[76371]: audit 2024-09-12T23:48:48.186252+0000 mon.a (mon.0) 1054 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:48.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[76371]: audit 2024-09-12T23:48:48.196317+0000 mon.a (mon.0) 1055 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:48.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[75164]: audit 2024-09-12T23:48:48.198991+0000 mon.a (mon.0) 1056 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:48.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[75164]: audit 2024-09-12T23:48:48.208396+0000 mon.a (mon.0) 1057 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:48.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[75164]: audit 2024-09-12T23:48:48.211078+0000 mon.a (mon.0) 1058 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:48.762 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[75164]: audit 2024-09-12T23:48:48.213651+0000 mon.a (mon.0) 1059 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:48.762 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[75164]: audit 2024-09-12T23:48:48.223721+0000 mon.a (mon.0) 1060 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:48.762 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[75164]: audit 2024-09-12T23:48:48.226931+0000 mon.a (mon.0) 1061 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:48.762 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[75164]: audit 2024-09-12T23:48:48.236566+0000 mon.a (mon.0) 1062 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:48.762 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[75164]: audit 2024-09-12T23:48:48.239606+0000 mon.a (mon.0) 1063 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:48.762 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[75164]: audit 2024-09-12T23:48:48.248990+0000 mon.a (mon.0) 1064 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:48.762 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[75164]: audit 2024-09-12T23:48:48.253904+0000 mon.a (mon.0) 1065 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:48.762 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[75164]: audit 2024-09-12T23:48:48.264201+0000 mon.a (mon.0) 1066 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:48.763 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[76371]: audit 2024-09-12T23:48:48.198991+0000 mon.a (mon.0) 1056 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:48.763 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[76371]: audit 2024-09-12T23:48:48.208396+0000 mon.a (mon.0) 1057 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:48.763 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[76371]: audit 2024-09-12T23:48:48.211078+0000 mon.a (mon.0) 1058 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:48.763 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[76371]: audit 2024-09-12T23:48:48.213651+0000 mon.a (mon.0) 1059 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:48.763 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[76371]: audit 2024-09-12T23:48:48.223721+0000 mon.a (mon.0) 1060 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:48.763 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[76371]: audit 2024-09-12T23:48:48.226931+0000 mon.a (mon.0) 1061 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:48.763 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[76371]: audit 2024-09-12T23:48:48.236566+0000 mon.a (mon.0) 1062 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:48.763 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[76371]: audit 2024-09-12T23:48:48.239606+0000 mon.a (mon.0) 1063 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:48.763 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[76371]: audit 2024-09-12T23:48:48.248990+0000 mon.a (mon.0) 1064 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:48.763 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[76371]: audit 2024-09-12T23:48:48.253904+0000 mon.a (mon.0) 1065 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:48.764 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:48 smithi136 bash[76371]: audit 2024-09-12T23:48:48.264201+0000 mon.a (mon.0) 1066 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:48.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:48 smithi181 bash[67587]: cluster 2024-09-12T23:48:47.452137+0000 mon.a (mon.0) 1037 : cluster [DBG] osdmap e670: 8 total, 8 up, 8 in 2024-09-12T23:48:48.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:48 smithi181 bash[67587]: audit 2024-09-12T23:48:47.963794+0000 mon.a (mon.0) 1038 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:48.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:48 smithi181 bash[67587]: audit 2024-09-12T23:48:47.974473+0000 mon.a (mon.0) 1039 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:48.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:48 smithi181 bash[67587]: audit 2024-09-12T23:48:47.976640+0000 mon.a (mon.0) 1040 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:48:48.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:48 smithi181 bash[67587]: audit 2024-09-12T23:48:47.978552+0000 mon.a (mon.0) 1041 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:48:48.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:48 smithi181 bash[67587]: audit 2024-09-12T23:48:47.989050+0000 mon.a (mon.0) 1042 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:48.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:48 smithi181 bash[67587]: audit 2024-09-12T23:48:48.140444+0000 mon.a (mon.0) 1043 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:48:48.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:48 smithi181 bash[67587]: audit 2024-09-12T23:48:48.144792+0000 mon.a (mon.0) 1044 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:48.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:48 smithi181 bash[67587]: audit 2024-09-12T23:48:48.147264+0000 mon.a (mon.0) 1045 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:48.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:48 smithi181 bash[67587]: audit 2024-09-12T23:48:48.149111+0000 mon.a (mon.0) 1046 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:48.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:48 smithi181 bash[67587]: audit 2024-09-12T23:48:48.152146+0000 mon.a (mon.0) 1047 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:48.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:48 smithi181 bash[67587]: audit 2024-09-12T23:48:48.155683+0000 mon.a (mon.0) 1048 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:48.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:48 smithi181 bash[67587]: audit 2024-09-12T23:48:48.162555+0000 mon.a (mon.0) 1049 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:48.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:48 smithi181 bash[67587]: audit 2024-09-12T23:48:48.164342+0000 mon.a (mon.0) 1050 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.a"}]: dispatch 2024-09-12T23:48:48.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:48 smithi181 bash[67587]: audit 2024-09-12T23:48:48.173603+0000 mon.a (mon.0) 1051 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.a"}]': finished 2024-09-12T23:48:48.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:48 smithi181 bash[67587]: audit 2024-09-12T23:48:48.175421+0000 mon.a (mon.0) 1052 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.b"}]: dispatch 2024-09-12T23:48:48.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:48 smithi181 bash[67587]: audit 2024-09-12T23:48:48.183341+0000 mon.a (mon.0) 1053 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.b"}]': finished 2024-09-12T23:48:48.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:48 smithi181 bash[67587]: audit 2024-09-12T23:48:48.186252+0000 mon.a (mon.0) 1054 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:48.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:48 smithi181 bash[67587]: audit 2024-09-12T23:48:48.196317+0000 mon.a (mon.0) 1055 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:48.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:48 smithi181 bash[67587]: audit 2024-09-12T23:48:48.198991+0000 mon.a (mon.0) 1056 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:48.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:48 smithi181 bash[67587]: audit 2024-09-12T23:48:48.208396+0000 mon.a (mon.0) 1057 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:48.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:48 smithi181 bash[67587]: audit 2024-09-12T23:48:48.211078+0000 mon.a (mon.0) 1058 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:48.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:48 smithi181 bash[67587]: audit 2024-09-12T23:48:48.213651+0000 mon.a (mon.0) 1059 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:48.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:48 smithi181 bash[67587]: audit 2024-09-12T23:48:48.223721+0000 mon.a (mon.0) 1060 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:48.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:48 smithi181 bash[67587]: audit 2024-09-12T23:48:48.226931+0000 mon.a (mon.0) 1061 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:48.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:48 smithi181 bash[67587]: audit 2024-09-12T23:48:48.236566+0000 mon.a (mon.0) 1062 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:48.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:48 smithi181 bash[67587]: audit 2024-09-12T23:48:48.239606+0000 mon.a (mon.0) 1063 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:48.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:48 smithi181 bash[67587]: audit 2024-09-12T23:48:48.248990+0000 mon.a (mon.0) 1064 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:48.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:48 smithi181 bash[67587]: audit 2024-09-12T23:48:48.253904+0000 mon.a (mon.0) 1065 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:48:48.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:48 smithi181 bash[67587]: audit 2024-09-12T23:48:48.264201+0000 mon.a (mon.0) 1066 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:49.475 INFO:tasks.rados.rados.0.smithi136.stdout:3012: done (5 left) 2024-09-12T23:48:49.475 INFO:tasks.rados.rados.0.smithi136.stdout:3013: delete oid 20 current snap is 301 2024-09-12T23:48:49.475 INFO:tasks.rados.rados.0.smithi136.stdout:3009: finishing copy_from racing read to smithi13667081-2 2024-09-12T23:48:49.475 INFO:tasks.rados.rados.0.smithi136.stdout:3009: finishing copy_from to smithi13667081-2 2024-09-12T23:48:49.475 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 2 v 1619 (ObjNum 877 snap 286 seq_num 877) dirty exists 2024-09-12T23:48:49.475 INFO:tasks.rados.rados.0.smithi136.stdout:3008: finishing copy_from racing read to smithi13667081-38 2024-09-12T23:48:49.475 INFO:tasks.rados.rados.0.smithi136.stdout:3008: finishing copy_from to smithi13667081-38 2024-09-12T23:48:49.475 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 38 v 1645 (ObjNum 864 snap 281 seq_num 864) dirty exists 2024-09-12T23:48:49.475 INFO:tasks.rados.rados.0.smithi136.stdout:3010: finishing write tid 1 to smithi13667081-4 2024-09-12T23:48:49.475 INFO:tasks.rados.rados.0.smithi136.stdout:3010: finishing write tid 2 to smithi13667081-4 2024-09-12T23:48:49.475 INFO:tasks.rados.rados.0.smithi136.stdout:3010: finishing write tid 3 to smithi13667081-4 2024-09-12T23:48:49.475 INFO:tasks.rados.rados.0.smithi136.stdout:3010: finishing write tid 4 to smithi13667081-4 2024-09-12T23:48:49.475 INFO:tasks.rados.rados.0.smithi136.stdout:3010: finishing write tid 5 to smithi13667081-4 2024-09-12T23:48:49.476 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 4 v 1630 (ObjNum 949 snap 301 seq_num 949) dirty exists 2024-09-12T23:48:49.476 INFO:tasks.rados.rados.0.smithi136.stdout:3010: left oid 4 (ObjNum 949 snap 301 seq_num 949) 2024-09-12T23:48:49.476 INFO:tasks.rados.rados.0.smithi136.stdout:3011: finishing write tid 1 to smithi13667081-31 2024-09-12T23:48:49.476 INFO:tasks.rados.rados.0.smithi136.stdout:3011: finishing write tid 2 to smithi13667081-31 2024-09-12T23:48:49.476 INFO:tasks.rados.rados.0.smithi136.stdout:3011: finishing write tid 3 to smithi13667081-31 2024-09-12T23:48:49.476 INFO:tasks.rados.rados.0.smithi136.stdout:3011: finishing write tid 4 to smithi13667081-31 2024-09-12T23:48:49.476 INFO:tasks.rados.rados.0.smithi136.stdout:3011: finishing write tid 5 to smithi13667081-31 2024-09-12T23:48:49.476 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 31 v 1660 (ObjNum 950 snap 301 seq_num 950) dirty exists 2024-09-12T23:48:49.476 INFO:tasks.rados.rados.0.smithi136.stdout:3011: left oid 31 (ObjNum 950 snap 301 seq_num 950) 2024-09-12T23:48:49.497 INFO:tasks.rados.rados.0.smithi136.stdout:3007: done (5 left) 2024-09-12T23:48:49.497 INFO:tasks.rados.rados.0.smithi136.stdout:3008: done (4 left) 2024-09-12T23:48:49.497 INFO:tasks.rados.rados.0.smithi136.stdout:3009: done (3 left) 2024-09-12T23:48:49.497 INFO:tasks.rados.rados.0.smithi136.stdout:3010: done (2 left) 2024-09-12T23:48:49.497 INFO:tasks.rados.rados.0.smithi136.stdout:3011: done (1 left) 2024-09-12T23:48:49.497 INFO:tasks.rados.rados.0.smithi136.stdout:3013: done (0 left) 2024-09-12T23:48:49.497 INFO:tasks.rados.rados.0.smithi136.stdout:3014: rollback oid 31 current snap is 301 2024-09-12T23:48:49.497 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 31 to 291 2024-09-12T23:48:49.497 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 29 current snap is 301 2024-09-12T23:48:49.498 INFO:tasks.rados.rados.0.smithi136.stdout:3015: seq_num 952 ranges {17186816=753664,17940480=499712,18440192=303104} 2024-09-12T23:48:49.598 INFO:tasks.rados.rados.0.smithi136.stdout:3015: writing smithi13667081-29 from 17186816 to 17940480 tid 1 2024-09-12T23:48:49.600 INFO:tasks.rados.rados.0.smithi136.stdout:3015: writing smithi13667081-29 from 17940480 to 18440192 tid 2 2024-09-12T23:48:49.602 INFO:tasks.rados.rados.0.smithi136.stdout:3015: writing smithi13667081-29 from 18440192 to 18743296 tid 3 2024-09-12T23:48:49.603 INFO:tasks.rados.rados.0.smithi136.stdout:3016: read oid 48 snap 289 2024-09-12T23:48:49.603 INFO:tasks.rados.rados.0.smithi136.stdout:3016: expect (ObjNum 864 snap 281 seq_num 864) 2024-09-12T23:48:49.603 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 12 current snap is 301 2024-09-12T23:48:49.603 INFO:tasks.rados.rados.0.smithi136.stdout:3017: seq_num 953 ranges {1368064=770048,2138112=786432,2924544=16384} 2024-09-12T23:48:49.615 INFO:tasks.rados.rados.0.smithi136.stdout:3017: writing smithi13667081-12 from 1368064 to 2138112 tid 1 2024-09-12T23:48:49.621 INFO:tasks.rados.rados.0.smithi136.stdout:3017: writing smithi13667081-12 from 2138112 to 2924544 tid 2 2024-09-12T23:48:49.621 INFO:tasks.rados.rados.0.smithi136.stdout:3017: writing smithi13667081-12 from 2924544 to 2940928 tid 3 2024-09-12T23:48:49.621 INFO:tasks.rados.rados.0.smithi136.stdout:3018: copy_from oid 24 from oid 39 current snap is 301 2024-09-12T23:48:49.621 INFO:tasks.rados.rados.0.smithi136.stdout:3014: finishing rollback tid 0 to smithi13667081-31 2024-09-12T23:48:49.621 INFO:tasks.rados.rados.0.smithi136.stdout:3014: finishing rollback tid 1 to smithi13667081-31 2024-09-12T23:48:49.621 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 31 v 1662 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-12T23:48:49.622 INFO:tasks.rados.rados.0.smithi136.stdout:3015: finishing write tid 1 to smithi13667081-29 2024-09-12T23:48:49.622 INFO:tasks.rados.rados.0.smithi136.stdout:3014: done (4 left) 2024-09-12T23:48:49.622 INFO:tasks.rados.rados.0.smithi136.stdout:3019: delete oid 11 current snap is 301 2024-09-12T23:48:49.622 INFO:tasks.rados.rados.0.smithi136.stdout:3015: finishing write tid 2 to smithi13667081-29 2024-09-12T23:48:49.622 INFO:tasks.rados.rados.0.smithi136.stdout:3015: finishing write tid 3 to smithi13667081-29 2024-09-12T23:48:49.622 INFO:tasks.rados.rados.0.smithi136.stdout:3016: expect (ObjNum 864 snap 281 seq_num 864) 2024-09-12T23:48:49.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:49 smithi136 bash[75164]: cephadm 2024-09-12T23:48:47.954160+0000 mgr.y (mgr.24661) 319 : cephadm [INF] Detected new or changed devices on smithi181 2024-09-12T23:48:49.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:49 smithi136 bash[75164]: cephadm 2024-09-12T23:48:48.156883+0000 mgr.y (mgr.24661) 320 : cephadm [INF] Upgrade: Setting container_image for all mds 2024-09-12T23:48:49.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:49 smithi136 bash[75164]: cephadm 2024-09-12T23:48:48.187898+0000 mgr.y (mgr.24661) 321 : cephadm [INF] Upgrade: Setting container_image for all rgw 2024-09-12T23:48:49.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:49 smithi136 bash[75164]: cephadm 2024-09-12T23:48:48.200514+0000 mgr.y (mgr.24661) 322 : cephadm [INF] Upgrade: Setting container_image for all rbd-mirror 2024-09-12T23:48:49.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:49 smithi136 bash[75164]: cephadm 2024-09-12T23:48:48.215116+0000 mgr.y (mgr.24661) 323 : cephadm [INF] Upgrade: Setting container_image for all ceph-exporter 2024-09-12T23:48:49.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:49 smithi136 bash[75164]: cephadm 2024-09-12T23:48:48.228506+0000 mgr.y (mgr.24661) 324 : cephadm [INF] Upgrade: Setting container_image for all iscsi 2024-09-12T23:48:49.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:49 smithi136 bash[75164]: cephadm 2024-09-12T23:48:48.241118+0000 mgr.y (mgr.24661) 325 : cephadm [INF] Upgrade: Setting container_image for all nfs 2024-09-12T23:48:49.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:49 smithi136 bash[75164]: cephadm 2024-09-12T23:48:48.255377+0000 mgr.y (mgr.24661) 326 : cephadm [INF] Upgrade: Setting container_image for all nvmeof 2024-09-12T23:48:49.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:49 smithi136 bash[75164]: cluster 2024-09-12T23:48:48.462377+0000 mon.a (mon.0) 1067 : cluster [DBG] osdmap e671: 8 total, 8 up, 8 in 2024-09-12T23:48:49.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:49 smithi136 bash[75164]: cluster 2024-09-12T23:48:48.809606+0000 mgr.y (mgr.24661) 327 : cluster [DBG] pgmap v583: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 274 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 2.2 MiB/s wr, 18 op/s 2024-09-12T23:48:49.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:48:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:48:49] "GET /metrics HTTP/1.1" 200 36440 "" "Prometheus/2.43.0" 2024-09-12T23:48:49.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:49 smithi136 bash[76371]: cephadm 2024-09-12T23:48:47.954160+0000 mgr.y (mgr.24661) 319 : cephadm [INF] Detected new or changed devices on smithi181 2024-09-12T23:48:49.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:49 smithi136 bash[76371]: cephadm 2024-09-12T23:48:48.156883+0000 mgr.y (mgr.24661) 320 : cephadm [INF] Upgrade: Setting container_image for all mds 2024-09-12T23:48:49.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:49 smithi136 bash[76371]: cephadm 2024-09-12T23:48:48.187898+0000 mgr.y (mgr.24661) 321 : cephadm [INF] Upgrade: Setting container_image for all rgw 2024-09-12T23:48:49.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:49 smithi136 bash[76371]: cephadm 2024-09-12T23:48:48.200514+0000 mgr.y (mgr.24661) 322 : cephadm [INF] Upgrade: Setting container_image for all rbd-mirror 2024-09-12T23:48:49.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:49 smithi136 bash[76371]: cephadm 2024-09-12T23:48:48.215116+0000 mgr.y (mgr.24661) 323 : cephadm [INF] Upgrade: Setting container_image for all ceph-exporter 2024-09-12T23:48:49.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:49 smithi136 bash[76371]: cephadm 2024-09-12T23:48:48.228506+0000 mgr.y (mgr.24661) 324 : cephadm [INF] Upgrade: Setting container_image for all iscsi 2024-09-12T23:48:49.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:49 smithi136 bash[76371]: cephadm 2024-09-12T23:48:48.241118+0000 mgr.y (mgr.24661) 325 : cephadm [INF] Upgrade: Setting container_image for all nfs 2024-09-12T23:48:49.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:49 smithi136 bash[76371]: cephadm 2024-09-12T23:48:48.255377+0000 mgr.y (mgr.24661) 326 : cephadm [INF] Upgrade: Setting container_image for all nvmeof 2024-09-12T23:48:49.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:49 smithi136 bash[76371]: cluster 2024-09-12T23:48:48.462377+0000 mon.a (mon.0) 1067 : cluster [DBG] osdmap e671: 8 total, 8 up, 8 in 2024-09-12T23:48:49.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:49 smithi136 bash[76371]: cluster 2024-09-12T23:48:48.809606+0000 mgr.y (mgr.24661) 327 : cluster [DBG] pgmap v583: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 274 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 2.2 MiB/s wr, 18 op/s 2024-09-12T23:48:49.798 INFO:tasks.rados.rados.0.smithi136.stdout:3015: finishing write tid 4 to smithi13667081-29 2024-09-12T23:48:49.798 INFO:tasks.rados.rados.0.smithi136.stdout:3015: finishing write tid 5 to smithi13667081-29 2024-09-12T23:48:49.798 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 29 v 2249 (ObjNum 952 snap 301 seq_num 952) dirty exists 2024-09-12T23:48:49.798 INFO:tasks.rados.rados.0.smithi136.stdout:3015: left oid 29 (ObjNum 952 snap 301 seq_num 952) 2024-09-12T23:48:49.798 INFO:tasks.rados.rados.0.smithi136.stdout:3018: finishing copy_from to smithi13667081-24 2024-09-12T23:48:49.798 INFO:tasks.rados.rados.0.smithi136.stdout:3018: got expected ENOENT (src dne) 2024-09-12T23:48:49.799 INFO:tasks.rados.rados.0.smithi136.stdout:3018: finishing copy_from racing read to smithi13667081-24 2024-09-12T23:48:49.799 INFO:tasks.rados.rados.0.smithi136.stdout:3017: finishing write tid 1 to smithi13667081-12 2024-09-12T23:48:49.799 INFO:tasks.rados.rados.0.smithi136.stdout:3017: finishing write tid 2 to smithi13667081-12 2024-09-12T23:48:49.799 INFO:tasks.rados.rados.0.smithi136.stdout:3017: finishing write tid 3 to smithi13667081-12 2024-09-12T23:48:49.799 INFO:tasks.rados.rados.0.smithi136.stdout:3017: finishing write tid 4 to smithi13667081-12 2024-09-12T23:48:49.799 INFO:tasks.rados.rados.0.smithi136.stdout:3017: finishing write tid 5 to smithi13667081-12 2024-09-12T23:48:49.799 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 12 v 1666 (ObjNum 953 snap 301 seq_num 953) dirty exists 2024-09-12T23:48:49.799 INFO:tasks.rados.rados.0.smithi136.stdout:3017: left oid 12 (ObjNum 953 snap 301 seq_num 953) 2024-09-12T23:48:49.799 INFO:tasks.rados.rados.0.smithi136.stdout:3015: done (4 left) 2024-09-12T23:48:49.799 INFO:tasks.rados.rados.0.smithi136.stdout:3016: done (3 left) 2024-09-12T23:48:49.799 INFO:tasks.rados.rados.0.smithi136.stdout:3017: done (2 left) 2024-09-12T23:48:49.799 INFO:tasks.rados.rados.0.smithi136.stdout:3018: done (1 left) 2024-09-12T23:48:49.799 INFO:tasks.rados.rados.0.smithi136.stdout:3019: done (0 left) 2024-09-12T23:48:49.799 INFO:tasks.rados.rados.0.smithi136.stdout:3020: read oid 30 snap -1 2024-09-12T23:48:49.800 INFO:tasks.rados.rados.0.smithi136.stdout:3020: expect deleted 2024-09-12T23:48:49.800 INFO:tasks.rados.rados.0.smithi136.stdout:3021: copy_from oid 35 from oid 28 current snap is 301 2024-09-12T23:48:49.800 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 42 current snap is 301 2024-09-12T23:48:49.800 INFO:tasks.rados.rados.0.smithi136.stdout:3022: seq_num 955 ranges {10174464=802816,10977280=557056,11534336=245760} 2024-09-12T23:48:49.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:49 smithi181 bash[67587]: cephadm 2024-09-12T23:48:47.954160+0000 mgr.y (mgr.24661) 319 : cephadm [INF] Detected new or changed devices on smithi181 2024-09-12T23:48:49.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:49 smithi181 bash[67587]: cephadm 2024-09-12T23:48:48.156883+0000 mgr.y (mgr.24661) 320 : cephadm [INF] Upgrade: Setting container_image for all mds 2024-09-12T23:48:49.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:49 smithi181 bash[67587]: cephadm 2024-09-12T23:48:48.187898+0000 mgr.y (mgr.24661) 321 : cephadm [INF] Upgrade: Setting container_image for all rgw 2024-09-12T23:48:49.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:49 smithi181 bash[67587]: cephadm 2024-09-12T23:48:48.200514+0000 mgr.y (mgr.24661) 322 : cephadm [INF] Upgrade: Setting container_image for all rbd-mirror 2024-09-12T23:48:49.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:49 smithi181 bash[67587]: cephadm 2024-09-12T23:48:48.215116+0000 mgr.y (mgr.24661) 323 : cephadm [INF] Upgrade: Setting container_image for all ceph-exporter 2024-09-12T23:48:49.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:49 smithi181 bash[67587]: cephadm 2024-09-12T23:48:48.228506+0000 mgr.y (mgr.24661) 324 : cephadm [INF] Upgrade: Setting container_image for all iscsi 2024-09-12T23:48:49.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:49 smithi181 bash[67587]: cephadm 2024-09-12T23:48:48.241118+0000 mgr.y (mgr.24661) 325 : cephadm [INF] Upgrade: Setting container_image for all nfs 2024-09-12T23:48:49.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:49 smithi181 bash[67587]: cephadm 2024-09-12T23:48:48.255377+0000 mgr.y (mgr.24661) 326 : cephadm [INF] Upgrade: Setting container_image for all nvmeof 2024-09-12T23:48:49.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:49 smithi181 bash[67587]: cluster 2024-09-12T23:48:48.462377+0000 mon.a (mon.0) 1067 : cluster [DBG] osdmap e671: 8 total, 8 up, 8 in 2024-09-12T23:48:49.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:49 smithi181 bash[67587]: cluster 2024-09-12T23:48:48.809606+0000 mgr.y (mgr.24661) 327 : cluster [DBG] pgmap v583: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 274 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 2.2 MiB/s wr, 18 op/s 2024-09-12T23:48:49.864 INFO:tasks.rados.rados.0.smithi136.stdout:3022: writing smithi13667081-42 from 10174464 to 10977280 tid 1 2024-09-12T23:48:49.868 INFO:tasks.rados.rados.0.smithi136.stdout:3022: writing smithi13667081-42 from 10977280 to 11534336 tid 2 2024-09-12T23:48:49.869 INFO:tasks.rados.rados.0.smithi136.stdout:3022: writing smithi13667081-42 from 11534336 to 11780096 tid 3 2024-09-12T23:48:49.869 INFO:tasks.rados.rados.0.smithi136.stdout:3023: delete oid 24 current snap is 301 2024-09-12T23:48:49.869 INFO:tasks.rados.rados.0.smithi136.stdout:3021: finishing copy_from to smithi13667081-35 2024-09-12T23:48:49.870 INFO:tasks.rados.rados.0.smithi136.stdout:3021: got expected ENOENT (src dne) 2024-09-12T23:48:49.870 INFO:tasks.rados.rados.0.smithi136.stdout:3021: finishing copy_from racing read to smithi13667081-35 2024-09-12T23:48:49.872 INFO:tasks.rados.rados.0.smithi136.stdout:3022: finishing write tid 1 to smithi13667081-42 2024-09-12T23:48:49.872 INFO:tasks.rados.rados.0.smithi136.stdout:3022: finishing write tid 2 to smithi13667081-42 2024-09-12T23:48:49.873 INFO:tasks.rados.rados.0.smithi136.stdout:3020: done (3 left) 2024-09-12T23:48:49.873 INFO:tasks.rados.rados.0.smithi136.stdout:3021: done (2 left) 2024-09-12T23:48:49.873 INFO:tasks.rados.rados.0.smithi136.stdout:3023: done (1 left) 2024-09-12T23:48:49.873 INFO:tasks.rados.rados.0.smithi136.stdout:3024: rollback oid 20 current snap is 301 2024-09-12T23:48:49.873 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 20 to 263 2024-09-12T23:48:49.874 INFO:tasks.rados.rados.0.smithi136.stdout:3025: copy_from oid 50 from oid 23 current snap is 301 2024-09-12T23:48:49.874 INFO:tasks.rados.rados.0.smithi136.stdout:3026: rollback oid 22 current snap is 301 2024-09-12T23:48:49.874 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 22 to 263 2024-09-12T23:48:49.874 INFO:tasks.rados.rados.0.smithi136.stdout:3027: delete oid 34 current snap is 301 2024-09-12T23:48:49.874 INFO:tasks.rados.rados.0.smithi136.stdout:3022: finishing write tid 3 to smithi13667081-42 2024-09-12T23:48:49.878 INFO:tasks.rados.rados.0.smithi136.stdout:3024: finishing rollback tid 1 to smithi13667081-20 2024-09-12T23:48:49.878 INFO:tasks.rados.rados.0.smithi136.stdout:3024: finishing rollback tid 2 to smithi13667081-20 2024-09-12T23:48:49.878 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 20 v 1616 (ObjNum 739 snap 227 seq_num 739) dirty exists 2024-09-12T23:48:49.880 INFO:tasks.rados.rados.0.smithi136.stdout:3022: finishing write tid 4 to smithi13667081-42 2024-09-12T23:48:49.880 INFO:tasks.rados.rados.0.smithi136.stdout:3026: finishing rollback tid 1 to smithi13667081-22 2024-09-12T23:48:49.880 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 22 v 1799 (ObjNum 113 snap 0 seq_num 4242589072) dirty dne 2024-09-12T23:48:49.880 INFO:tasks.rados.rados.0.smithi136.stdout:3022: finishing write tid 5 to smithi13667081-42 2024-09-12T23:48:49.880 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 42 v 1807 (ObjNum 955 snap 301 seq_num 955) dirty exists 2024-09-12T23:48:49.880 INFO:tasks.rados.rados.0.smithi136.stdout:3022: left oid 42 (ObjNum 955 snap 301 seq_num 955) 2024-09-12T23:48:49.881 INFO:tasks.rados.rados.0.smithi136.stdout:3022: done (4 left) 2024-09-12T23:48:49.881 INFO:tasks.rados.rados.0.smithi136.stdout:3024: done (3 left) 2024-09-12T23:48:49.882 INFO:tasks.rados.rados.0.smithi136.stdout:3026: done (2 left) 2024-09-12T23:48:49.882 INFO:tasks.rados.rados.0.smithi136.stdout:3027: done (1 left) 2024-09-12T23:48:49.882 INFO:tasks.rados.rados.0.smithi136.stdout:3028: read oid 45 snap -1 2024-09-12T23:48:49.882 INFO:tasks.rados.rados.0.smithi136.stdout:3028: expect (ObjNum 889 snap 288 seq_num 889) 2024-09-12T23:48:49.882 INFO:tasks.rados.rados.0.smithi136.stdout:3029: read oid 18 snap -1 2024-09-12T23:48:49.882 INFO:tasks.rados.rados.0.smithi136.stdout:3029: expect (ObjNum 931 snap 292 seq_num 931) 2024-09-12T23:48:49.882 INFO:tasks.rados.rados.0.smithi136.stdout:3030: read oid 37 snap -1 2024-09-12T23:48:49.882 INFO:tasks.rados.rados.0.smithi136.stdout:3030: expect (ObjNum 636 snap 188 seq_num 636) 2024-09-12T23:48:49.882 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 40 current snap is 301 2024-09-12T23:48:49.882 INFO:tasks.rados.rados.0.smithi136.stdout:3031: seq_num 958 ranges {1433600=450560,1884160=655360,2539520=548864} 2024-09-12T23:48:49.892 INFO:tasks.rados.rados.0.smithi136.stdout:3031: writing smithi13667081-40 from 1433600 to 1884160 tid 1 2024-09-12T23:48:49.896 INFO:tasks.rados.rados.0.smithi136.stdout:3031: writing smithi13667081-40 from 1884160 to 2539520 tid 2 2024-09-12T23:48:49.900 INFO:tasks.rados.rados.0.smithi136.stdout:3031: writing smithi13667081-40 from 2539520 to 3088384 tid 3 2024-09-12T23:48:49.901 INFO:tasks.rados.rados.0.smithi136.stdout:3032: read oid 20 snap -1 2024-09-12T23:48:49.901 INFO:tasks.rados.rados.0.smithi136.stdout:3032: expect (ObjNum 739 snap 227 seq_num 739) 2024-09-12T23:48:49.901 INFO:tasks.rados.rados.0.smithi136.stdout:3033: read oid 35 snap -1 2024-09-12T23:48:49.901 INFO:tasks.rados.rados.0.smithi136.stdout:3033: expect (ObjNum 927 snap 290 seq_num 927) 2024-09-12T23:48:49.901 INFO:tasks.rados.rados.0.smithi136.stdout:3034: delete oid 34 current snap is 301 2024-09-12T23:48:49.901 INFO:tasks.rados.rados.0.smithi136.stdout:3030: expect (ObjNum 636 snap 188 seq_num 636) 2024-09-12T23:48:49.936 INFO:tasks.rados.rados.0.smithi136.stdout:3028: expect (ObjNum 889 snap 288 seq_num 889) 2024-09-12T23:48:49.962 INFO:tasks.rados.rados.0.smithi136.stdout:3025: finishing copy_from racing read to smithi13667081-50 2024-09-12T23:48:49.962 INFO:tasks.rados.rados.0.smithi136.stdout:3031: finishing write tid 1 to smithi13667081-40 2024-09-12T23:48:49.962 INFO:tasks.rados.rados.0.smithi136.stdout:3025: finishing copy_from to smithi13667081-50 2024-09-12T23:48:49.963 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 50 v 2267 (ObjNum 886 snap 287 seq_num 886) dirty exists 2024-09-12T23:48:49.963 INFO:tasks.rados.rados.0.smithi136.stdout:3032: expect (ObjNum 739 snap 227 seq_num 739) 2024-09-12T23:48:49.973 INFO:tasks.rados.rados.0.smithi136.stdout:3031: finishing write tid 2 to smithi13667081-40 2024-09-12T23:48:49.973 INFO:tasks.rados.rados.0.smithi136.stdout:3029: expect (ObjNum 931 snap 292 seq_num 931) 2024-09-12T23:48:50.108 INFO:tasks.rados.rados.0.smithi136.stdout:3031: finishing write tid 3 to smithi13667081-40 2024-09-12T23:48:50.109 INFO:tasks.rados.rados.0.smithi136.stdout:3031: finishing write tid 4 to smithi13667081-40 2024-09-12T23:48:50.109 INFO:tasks.rados.rados.0.smithi136.stdout:3031: finishing write tid 5 to smithi13667081-40 2024-09-12T23:48:50.109 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 40 v 2308 (ObjNum 958 snap 301 seq_num 958) dirty exists 2024-09-12T23:48:50.109 INFO:tasks.rados.rados.0.smithi136.stdout:3031: left oid 40 (ObjNum 958 snap 301 seq_num 958) 2024-09-12T23:48:50.109 INFO:tasks.rados.rados.0.smithi136.stdout:3033: expect (ObjNum 927 snap 290 seq_num 927) 2024-09-12T23:48:50.140 INFO:tasks.rados.rados.0.smithi136.stdout:3025: done (7 left) 2024-09-12T23:48:50.140 INFO:tasks.rados.rados.0.smithi136.stdout:3028: done (6 left) 2024-09-12T23:48:50.140 INFO:tasks.rados.rados.0.smithi136.stdout:3029: done (5 left) 2024-09-12T23:48:50.140 INFO:tasks.rados.rados.0.smithi136.stdout:3030: done (4 left) 2024-09-12T23:48:50.140 INFO:tasks.rados.rados.0.smithi136.stdout:3031: done (3 left) 2024-09-12T23:48:50.140 INFO:tasks.rados.rados.0.smithi136.stdout:3032: done (2 left) 2024-09-12T23:48:50.141 INFO:tasks.rados.rados.0.smithi136.stdout:3033: done (1 left) 2024-09-12T23:48:50.141 INFO:tasks.rados.rados.0.smithi136.stdout:3034: done (0 left) 2024-09-12T23:48:50.141 INFO:tasks.rados.rados.0.smithi136.stdout:3035: copy_from oid 34 from oid 29 current snap is 301 2024-09-12T23:48:50.141 INFO:tasks.rados.rados.0.smithi136.stdout:3036: delete oid 14 current snap is 301 2024-09-12T23:48:50.145 INFO:tasks.rados.rados.0.smithi136.stdout:3036: done (1 left) 2024-09-12T23:48:50.145 INFO:tasks.rados.rados.0.smithi136.stdout:3037: setattr oid 36 current snap is 301 2024-09-12T23:48:50.148 INFO:tasks.rados.rados.0.smithi136.stdout:3038: rollback oid 10 current snap is 301 2024-09-12T23:48:50.148 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 10 to 299 2024-09-12T23:48:50.148 INFO:tasks.rados.rados.0.smithi136.stdout:3039: copy_from oid 46 from oid 49 current snap is 301 2024-09-12T23:48:50.148 INFO:tasks.rados.rados.0.smithi136.stdout:3040: rollback oid 24 current snap is 301 2024-09-12T23:48:50.148 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 24 to 280 2024-09-12T23:48:50.148 INFO:tasks.rados.rados.0.smithi136.stdout:3041: read oid 15 snap -1 2024-09-12T23:48:50.148 INFO:tasks.rados.rados.0.smithi136.stdout:3041: expect (ObjNum 859 snap 278 seq_num 859) 2024-09-12T23:48:50.148 INFO:tasks.rados.rados.0.smithi136.stdout:3042: snap_remove snap 289 2024-09-12T23:48:50.487 INFO:tasks.rados.rados.0.smithi136.stdout:3042: done (6 left) 2024-09-12T23:48:50.488 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 3 current snap is 301 2024-09-12T23:48:50.488 INFO:tasks.rados.rados.0.smithi136.stdout:3043: seq_num 961 ranges {2048000=499712,2547712=712704,3260416=491520} 2024-09-12T23:48:50.507 INFO:tasks.rados.rados.0.smithi136.stdout:3043: writing smithi13667081-3 from 2048000 to 2547712 tid 1 2024-09-12T23:48:50.518 INFO:tasks.rados.rados.0.smithi136.stdout:3043: writing smithi13667081-3 from 2547712 to 3260416 tid 2 2024-09-12T23:48:50.525 INFO:tasks.rados.rados.0.smithi136.stdout:3043: writing smithi13667081-3 from 3260416 to 3751936 tid 3 2024-09-12T23:48:50.525 INFO:tasks.rados.rados.0.smithi136.stdout:3044: delete oid 22 current snap is 301 2024-09-12T23:48:50.525 INFO:tasks.rados.rados.0.smithi136.stdout:3039: finishing copy_from to smithi13667081-46 2024-09-12T23:48:50.525 INFO:tasks.rados.rados.0.smithi136.stdout:3039: got expected ENOENT (src dne) 2024-09-12T23:48:50.525 INFO:tasks.rados.rados.0.smithi136.stdout:3039: finishing copy_from racing read to smithi13667081-46 2024-09-12T23:48:50.525 INFO:tasks.rados.rados.0.smithi136.stdout:3040: finishing rollback tid 1 to smithi13667081-24 2024-09-12T23:48:50.525 INFO:tasks.rados.rados.0.smithi136.stdout:3040: finishing rollback tid 2 to smithi13667081-24 2024-09-12T23:48:50.525 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 24 v 1508 (ObjNum 829 snap 266 seq_num 829) dirty exists 2024-09-12T23:48:50.525 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 36 v 1627 (ObjNum 65 snap 0 seq_num 1684369761) dirty exists 2024-09-12T23:48:50.525 INFO:tasks.rados.rados.0.smithi136.stdout:3038: finishing rollback tid 1 to smithi13667081-10 2024-09-12T23:48:50.525 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 10 v 1490 (ObjNum 33 snap 0 seq_num 4243841120) dirty dne 2024-09-12T23:48:50.526 INFO:tasks.rados.rados.0.smithi136.stdout:3041: expect (ObjNum 859 snap 278 seq_num 859) 2024-09-12T23:48:50.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:50 smithi136 bash[75164]: cluster 2024-09-12T23:48:49.476889+0000 mon.a (mon.0) 1068 : cluster [DBG] osdmap e672: 8 total, 8 up, 8 in 2024-09-12T23:48:50.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:50 smithi136 bash[76371]: cluster 2024-09-12T23:48:49.476889+0000 mon.a (mon.0) 1068 : cluster [DBG] osdmap e672: 8 total, 8 up, 8 in 2024-09-12T23:48:50.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:50 smithi181 bash[67587]: cluster 2024-09-12T23:48:49.476889+0000 mon.a (mon.0) 1068 : cluster [DBG] osdmap e672: 8 total, 8 up, 8 in 2024-09-12T23:48:51.417 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:51 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:51.418 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 12 23:48:51 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:51.418 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:48:51 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:51.418 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:51 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:51.419 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 12 23:48:51 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:51.419 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:48:51 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:51.419 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 12 23:48:51 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:51.420 INFO:journalctl@ceph.mds.a.smithi136.stdout:Sep 12 23:48:51 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:51.420 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:51 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:51.420 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:51 smithi136 systemd[1]: Stopping Ceph node-exporter.a for da58a47c-715d-11ef-bceb-c7b262605968... 2024-09-12T23:48:51.420 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 12 23:48:51 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:51.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:51 smithi136 bash[75164]: cluster 2024-09-12T23:48:50.488905+0000 mon.a (mon.0) 1069 : cluster [DBG] osdmap e673: 8 total, 8 up, 8 in 2024-09-12T23:48:51.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:51 smithi136 bash[75164]: cephadm 2024-09-12T23:48:50.702134+0000 mgr.y (mgr.24661) 328 : cephadm [INF] Upgrade: Updating node-exporter.a (1/2) 2024-09-12T23:48:51.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:51 smithi136 bash[75164]: cephadm 2024-09-12T23:48:50.702663+0000 mgr.y (mgr.24661) 329 : cephadm [INF] Deploying daemon node-exporter.a on smithi136 2024-09-12T23:48:51.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:51 smithi136 bash[75164]: cluster 2024-09-12T23:48:50.810545+0000 mgr.y (mgr.24661) 330 : cluster [DBG] pgmap v586: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 277 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 958 KiB/s wr, 9 op/s 2024-09-12T23:48:51.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:51 smithi136 bash[75164]: cluster 2024-09-12T23:48:51.438526+0000 mon.a (mon.0) 1070 : cluster [DBG] osdmap e674: 8 total, 8 up, 8 in 2024-09-12T23:48:51.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:51 smithi136 bash[76371]: cluster 2024-09-12T23:48:50.488905+0000 mon.a (mon.0) 1069 : cluster [DBG] osdmap e673: 8 total, 8 up, 8 in 2024-09-12T23:48:51.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:51 smithi136 bash[76371]: cephadm 2024-09-12T23:48:50.702134+0000 mgr.y (mgr.24661) 328 : cephadm [INF] Upgrade: Updating node-exporter.a (1/2) 2024-09-12T23:48:51.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:51 smithi136 bash[76371]: cephadm 2024-09-12T23:48:50.702663+0000 mgr.y (mgr.24661) 329 : cephadm [INF] Deploying daemon node-exporter.a on smithi136 2024-09-12T23:48:51.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:51 smithi136 bash[76371]: cluster 2024-09-12T23:48:50.810545+0000 mgr.y (mgr.24661) 330 : cluster [DBG] pgmap v586: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 277 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 958 KiB/s wr, 9 op/s 2024-09-12T23:48:51.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:51 smithi136 bash[76371]: cluster 2024-09-12T23:48:51.438526+0000 mon.a (mon.0) 1070 : cluster [DBG] osdmap e674: 8 total, 8 up, 8 in 2024-09-12T23:48:51.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:51 smithi181 bash[67587]: cluster 2024-09-12T23:48:50.488905+0000 mon.a (mon.0) 1069 : cluster [DBG] osdmap e673: 8 total, 8 up, 8 in 2024-09-12T23:48:51.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:51 smithi181 bash[67587]: cephadm 2024-09-12T23:48:50.702134+0000 mgr.y (mgr.24661) 328 : cephadm [INF] Upgrade: Updating node-exporter.a (1/2) 2024-09-12T23:48:51.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:51 smithi181 bash[67587]: cephadm 2024-09-12T23:48:50.702663+0000 mgr.y (mgr.24661) 329 : cephadm [INF] Deploying daemon node-exporter.a on smithi136 2024-09-12T23:48:51.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:51 smithi181 bash[67587]: cluster 2024-09-12T23:48:50.810545+0000 mgr.y (mgr.24661) 330 : cluster [DBG] pgmap v586: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 277 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 958 KiB/s wr, 9 op/s 2024-09-12T23:48:51.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:51 smithi181 bash[67587]: cluster 2024-09-12T23:48:51.438526+0000 mon.a (mon.0) 1070 : cluster [DBG] osdmap e674: 8 total, 8 up, 8 in 2024-09-12T23:48:52.256 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:51 smithi136 bash[100790]: ceph-da58a47c-715d-11ef-bceb-c7b262605968-node-exporter-a 2024-09-12T23:48:52.256 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:52 smithi136 systemd[1]: ceph-da58a47c-715d-11ef-bceb-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-09-12T23:48:52.256 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:52 smithi136 bash[100853]: Error response from daemon: No such container: ceph-da58a47c-715d-11ef-bceb-c7b262605968-node-exporter-a 2024-09-12T23:48:52.256 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:52 smithi136 systemd[1]: ceph-da58a47c-715d-11ef-bceb-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-09-12T23:48:52.257 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:52 smithi136 systemd[1]: Stopped Ceph node-exporter.a for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:48:52.322 INFO:tasks.rados.rados.0.smithi136.stdout:3035: finishing copy_from racing read to smithi13667081-34 2024-09-12T23:48:52.323 INFO:tasks.rados.rados.0.smithi136.stdout:3035: finishing copy_from to smithi13667081-34 2024-09-12T23:48:52.323 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 34 v 2249 (ObjNum 952 snap 301 seq_num 952) dirty exists 2024-09-12T23:48:52.323 INFO:tasks.rados.rados.0.smithi136.stdout:3043: finishing write tid 1 to smithi13667081-3 2024-09-12T23:48:52.323 INFO:tasks.rados.rados.0.smithi136.stdout:3043: finishing write tid 2 to smithi13667081-3 2024-09-12T23:48:52.323 INFO:tasks.rados.rados.0.smithi136.stdout:3043: finishing write tid 3 to smithi13667081-3 2024-09-12T23:48:52.323 INFO:tasks.rados.rados.0.smithi136.stdout:3043: finishing write tid 4 to smithi13667081-3 2024-09-12T23:48:52.323 INFO:tasks.rados.rados.0.smithi136.stdout:3043: finishing write tid 5 to smithi13667081-3 2024-09-12T23:48:52.323 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 3 v 1670 (ObjNum 961 snap 301 seq_num 961) dirty exists 2024-09-12T23:48:52.323 INFO:tasks.rados.rados.0.smithi136.stdout:3043: left oid 3 (ObjNum 961 snap 301 seq_num 961) 2024-09-12T23:48:52.323 INFO:tasks.rados.rados.0.smithi136.stdout:3035: done (7 left) 2024-09-12T23:48:52.323 INFO:tasks.rados.rados.0.smithi136.stdout:3037: done (6 left) 2024-09-12T23:48:52.323 INFO:tasks.rados.rados.0.smithi136.stdout:3038: done (5 left) 2024-09-12T23:48:52.324 INFO:tasks.rados.rados.0.smithi136.stdout:3039: done (4 left) 2024-09-12T23:48:52.324 INFO:tasks.rados.rados.0.smithi136.stdout:3040: done (3 left) 2024-09-12T23:48:52.324 INFO:tasks.rados.rados.0.smithi136.stdout:3041: done (2 left) 2024-09-12T23:48:52.325 INFO:tasks.rados.rados.0.smithi136.stdout:3043: done (1 left) 2024-09-12T23:48:52.325 INFO:tasks.rados.rados.0.smithi136.stdout:3044: done (0 left) 2024-09-12T23:48:52.325 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 9 current snap is 301 2024-09-12T23:48:52.325 INFO:tasks.rados.rados.0.smithi136.stdout:3045: seq_num 963 ranges {294912=532480,827392=475136,1302528=532480,1835008=196608} 2024-09-12T23:48:52.334 INFO:tasks.rados.rados.0.smithi136.stdout:3045: writing smithi13667081-9 from 294912 to 827392 tid 1 2024-09-12T23:48:52.339 INFO:tasks.rados.rados.0.smithi136.stdout:3045: writing smithi13667081-9 from 827392 to 1302528 tid 2 2024-09-12T23:48:52.345 INFO:tasks.rados.rados.0.smithi136.stdout:3045: writing smithi13667081-9 from 1302528 to 1835008 tid 3 2024-09-12T23:48:52.348 INFO:tasks.rados.rados.0.smithi136.stdout:3045: writing smithi13667081-9 from 1835008 to 2031616 tid 4 2024-09-12T23:48:52.348 INFO:tasks.rados.rados.0.smithi136.stdout:3046: snap_remove snap 290 2024-09-12T23:48:52.488 INFO:tasks.rados.rados.0.smithi136.stdout:3046: done (1 left) 2024-09-12T23:48:52.488 INFO:tasks.rados.rados.0.smithi136.stdout:3047: rmattr oid 26 current snap is 301 2024-09-12T23:48:52.488 INFO:tasks.rados.rados.0.smithi136.stdout:3047: done (1 left) 2024-09-12T23:48:52.489 INFO:tasks.rados.rados.0.smithi136.stdout:3048: delete oid 36 current snap is 301 2024-09-12T23:48:52.489 INFO:tasks.rados.rados.0.smithi136.stdout:3045: finishing write tid 1 to smithi13667081-9 2024-09-12T23:48:52.489 INFO:tasks.rados.rados.0.smithi136.stdout:3045: finishing write tid 2 to smithi13667081-9 2024-09-12T23:48:52.489 INFO:tasks.rados.rados.0.smithi136.stdout:3045: finishing write tid 3 to smithi13667081-9 2024-09-12T23:48:52.489 INFO:tasks.rados.rados.0.smithi136.stdout:3045: finishing write tid 4 to smithi13667081-9 2024-09-12T23:48:52.489 INFO:tasks.rados.rados.0.smithi136.stdout:3045: finishing write tid 5 to smithi13667081-9 2024-09-12T23:48:52.489 INFO:tasks.rados.rados.0.smithi136.stdout:3045: finishing write tid 6 to smithi13667081-9 2024-09-12T23:48:52.489 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 9 v 1629 (ObjNum 963 snap 301 seq_num 963) dirty exists 2024-09-12T23:48:52.489 INFO:tasks.rados.rados.0.smithi136.stdout:3045: left oid 9 (ObjNum 963 snap 301 seq_num 963) 2024-09-12T23:48:52.525 INFO:tasks.rados.rados.0.smithi136.stdout:3045: done (1 left) 2024-09-12T23:48:52.525 INFO:tasks.rados.rados.0.smithi136.stdout:3048: done (0 left) 2024-09-12T23:48:52.525 INFO:tasks.rados.rados.0.smithi136.stdout:3049: rollback oid 41 current snap is 301 2024-09-12T23:48:52.525 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 41 to 273 2024-09-12T23:48:52.525 INFO:tasks.rados.rados.0.smithi136.stdout:3050: read oid 26 snap -1 2024-09-12T23:48:52.525 INFO:tasks.rados.rados.0.smithi136.stdout:3050: expect (ObjNum 822 snap 258 seq_num 822) 2024-09-12T23:48:52.525 INFO:tasks.rados.rados.0.smithi136.stdout:3051: read oid 2 snap 280 2024-09-12T23:48:52.525 INFO:tasks.rados.rados.0.smithi136.stdout:3051: expect deleted 2024-09-12T23:48:52.525 INFO:tasks.rados.rados.0.smithi136.stdout:3052: rollback oid 40 current snap is 301 2024-09-12T23:48:52.525 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 40 to 291 2024-09-12T23:48:52.525 INFO:tasks.rados.rados.0.smithi136.stdout:3053: read oid 13 snap -1 2024-09-12T23:48:52.525 INFO:tasks.rados.rados.0.smithi136.stdout:3053: expect (ObjNum 893 snap 288 seq_num 893) 2024-09-12T23:48:52.525 INFO:tasks.rados.rados.0.smithi136.stdout:3054: copy_from oid 15 from oid 10 current snap is 301 2024-09-12T23:48:52.526 INFO:tasks.rados.rados.0.smithi136.stdout:3055: read oid 23 snap 299 2024-09-12T23:48:52.526 INFO:tasks.rados.rados.0.smithi136.stdout:3055: expect (ObjNum 886 snap 287 seq_num 886) 2024-09-12T23:48:52.526 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 36 current snap is 301 2024-09-12T23:48:52.526 INFO:tasks.rados.rados.0.smithi136.stdout:3056: seq_num 965 ranges {0=565248,565248=647168,1212416=548864,1761280=8192} 2024-09-12T23:48:52.527 INFO:tasks.rados.rados.0.smithi136.stdout:3056: writing smithi13667081-36 from 0 to 565248 tid 1 2024-09-12T23:48:52.532 INFO:tasks.rados.rados.0.smithi136.stdout:3056: writing smithi13667081-36 from 565248 to 1212416 tid 2 2024-09-12T23:48:52.537 INFO:tasks.rados.rados.0.smithi136.stdout:3056: writing smithi13667081-36 from 1212416 to 1761280 tid 3 2024-09-12T23:48:52.537 INFO:tasks.rados.rados.0.smithi136.stdout:3056: writing smithi13667081-36 from 1761280 to 1769472 tid 4 2024-09-12T23:48:52.537 INFO:tasks.rados.rados.0.smithi136.stdout:3057: read oid 46 snap -1 2024-09-12T23:48:52.537 INFO:tasks.rados.rados.0.smithi136.stdout:3057: expect (ObjNum 900 snap 288 seq_num 900) 2024-09-12T23:48:52.537 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 30 current snap is 301 2024-09-12T23:48:52.537 INFO:tasks.rados.rados.0.smithi136.stdout:3058: seq_num 966 ranges {0=581632,581632=581632,1163264=622592} 2024-09-12T23:48:52.542 INFO:tasks.rados.rados.0.smithi136.stdout:3058: writing smithi13667081-30 from 0 to 581632 tid 1 2024-09-12T23:48:52.546 INFO:tasks.rados.rados.0.smithi136.stdout:3058: writing smithi13667081-30 from 581632 to 1163264 tid 2 2024-09-12T23:48:52.549 INFO:tasks.rados.rados.0.smithi136.stdout:3058: writing smithi13667081-30 from 1163264 to 1785856 tid 3 2024-09-12T23:48:52.549 INFO:tasks.rados.rados.0.smithi136.stdout:3059: delete oid 43 current snap is 301 2024-09-12T23:48:52.550 INFO:tasks.rados.rados.0.smithi136.stdout:3054: finishing copy_from to smithi13667081-15 2024-09-12T23:48:52.550 INFO:tasks.rados.rados.0.smithi136.stdout:3054: got expected ENOENT (src dne) 2024-09-12T23:48:52.550 INFO:tasks.rados.rados.0.smithi136.stdout:3054: finishing copy_from racing read to smithi13667081-15 2024-09-12T23:48:52.550 INFO:tasks.rados.rados.0.smithi136.stdout:3052: finishing rollback tid 0 to smithi13667081-40 2024-09-12T23:48:52.550 INFO:tasks.rados.rados.0.smithi136.stdout:3052: finishing rollback tid 1 to smithi13667081-40 2024-09-12T23:48:52.550 INFO:tasks.rados.rados.0.smithi136.stdout:3055: expect (ObjNum 886 snap 287 seq_num 886) 2024-09-12T23:48:52.590 INFO:tasks.rados.rados.0.smithi136.stdout:3056: finishing write tid 1 to smithi13667081-36 2024-09-12T23:48:52.590 INFO:tasks.rados.rados.0.smithi136.stdout:3052: finishing rollback tid 2 to smithi13667081-40 2024-09-12T23:48:52.590 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 40 v 2329 (ObjNum 918 snap 290 seq_num 918) dirty exists 2024-09-12T23:48:52.590 INFO:tasks.rados.rados.0.smithi136.stdout:3049: finishing rollback tid 1 to smithi13667081-41 2024-09-12T23:48:52.591 INFO:tasks.rados.rados.0.smithi136.stdout:3049: finishing rollback tid 2 to smithi13667081-41 2024-09-12T23:48:52.591 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 41 v 2251 (ObjNum 851 snap 272 seq_num 851) dirty exists 2024-09-12T23:48:52.591 INFO:tasks.rados.rados.0.smithi136.stdout:3056: finishing write tid 2 to smithi13667081-36 2024-09-12T23:48:52.591 INFO:tasks.rados.rados.0.smithi136.stdout:3049: done (10 left) 2024-09-12T23:48:52.591 INFO:tasks.rados.rados.0.smithi136.stdout:3051: done (9 left) 2024-09-12T23:48:52.591 INFO:tasks.rados.rados.0.smithi136.stdout:3052: done (8 left) 2024-09-12T23:48:52.591 INFO:tasks.rados.rados.0.smithi136.stdout:3054: done (7 left) 2024-09-12T23:48:52.591 INFO:tasks.rados.rados.0.smithi136.stdout:3055: done (6 left) 2024-09-12T23:48:52.591 INFO:tasks.rados.rados.0.smithi136.stdout:3059: done (5 left) 2024-09-12T23:48:52.591 INFO:tasks.rados.rados.0.smithi136.stdout:3060: snap_remove snap 280 2024-09-12T23:48:52.594 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:48:52 smithi181 bash[60471]: ts=2024-09-12T23:48:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:48:52.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:52 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:52.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:48:52 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:52.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:52 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:52.758 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 12 23:48:52 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:52.758 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 12 23:48:52 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:52.758 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 12 23:48:52 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:52.758 INFO:journalctl@ceph.mds.a.smithi136.stdout:Sep 12 23:48:52 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:52.759 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 12 23:48:52 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:52.759 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:52 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:52.759 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:52 smithi136 systemd[1]: Started Ceph node-exporter.a for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:48:52.759 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:48:52 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:53.503 INFO:tasks.rados.rados.0.smithi136.stdout:3060: done (5 left) 2024-09-12T23:48:53.503 INFO:tasks.rados.rados.0.smithi136.stdout:3061: rollback oid 40 current snap is 301 2024-09-12T23:48:53.504 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 40 to 300 2024-09-12T23:48:53.504 INFO:tasks.rados.rados.0.smithi136.stdout:3050: expect (ObjNum 822 snap 258 seq_num 822) 2024-09-12T23:48:53.574 INFO:tasks.rados.rados.0.smithi136.stdout:3056: finishing write tid 3 to smithi13667081-36 2024-09-12T23:48:53.574 INFO:tasks.rados.rados.0.smithi136.stdout:3056: finishing write tid 4 to smithi13667081-36 2024-09-12T23:48:53.574 INFO:tasks.rados.rados.0.smithi136.stdout:3058: finishing write tid 1 to smithi13667081-30 2024-09-12T23:48:53.574 INFO:tasks.rados.rados.0.smithi136.stdout:3050: done (5 left) 2024-09-12T23:48:53.574 INFO:tasks.rados.rados.0.smithi136.stdout:3062: copy_from oid 9 from oid 21 current snap is 301 2024-09-12T23:48:53.574 INFO:tasks.rados.rados.0.smithi136.stdout:3056: finishing write tid 5 to smithi13667081-36 2024-09-12T23:48:53.574 INFO:tasks.rados.rados.0.smithi136.stdout:3057: expect (ObjNum 900 snap 288 seq_num 900) 2024-09-12T23:48:53.669 INFO:tasks.rados.rados.0.smithi136.stdout:3056: finishing write tid 6 to smithi13667081-36 2024-09-12T23:48:53.669 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 36 v 1633 (ObjNum 965 snap 301 seq_num 965) dirty exists 2024-09-12T23:48:53.670 INFO:tasks.rados.rados.0.smithi136.stdout:3056: left oid 36 (ObjNum 965 snap 301 seq_num 965) 2024-09-12T23:48:53.670 INFO:tasks.rados.rados.0.smithi136.stdout:3058: finishing write tid 2 to smithi13667081-30 2024-09-12T23:48:53.670 INFO:tasks.rados.rados.0.smithi136.stdout:3056: done (5 left) 2024-09-12T23:48:53.670 INFO:tasks.rados.rados.0.smithi136.stdout:3057: done (4 left) 2024-09-12T23:48:53.670 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 50 current snap is 301 2024-09-12T23:48:53.670 INFO:tasks.rados.rados.0.smithi136.stdout:3063: seq_num 968 ranges {909312=614400,1523712=745472,2269184=458752} 2024-09-12T23:48:53.678 INFO:tasks.rados.rados.0.smithi136.stdout:3063: writing smithi13667081-50 from 909312 to 1523712 tid 1 2024-09-12T23:48:53.683 INFO:tasks.rados.rados.0.smithi136.stdout:3063: writing smithi13667081-50 from 1523712 to 2269184 tid 2 2024-09-12T23:48:53.686 INFO:tasks.rados.rados.0.smithi136.stdout:3063: writing smithi13667081-50 from 2269184 to 2727936 tid 3 2024-09-12T23:48:53.686 INFO:tasks.rados.rados.0.smithi136.stdout:3064: rmattr oid 32 current snap is 301 2024-09-12T23:48:53.686 INFO:tasks.rados.rados.0.smithi136.stdout:3064: done (5 left) 2024-09-12T23:48:53.686 INFO:tasks.rados.rados.0.smithi136.stdout:3065: delete oid 33 current snap is 301 2024-09-12T23:48:53.686 INFO:tasks.rados.rados.0.smithi136.stdout:3058: finishing write tid 3 to smithi13667081-30 2024-09-12T23:48:53.686 INFO:tasks.rados.rados.0.smithi136.stdout:3058: finishing write tid 4 to smithi13667081-30 2024-09-12T23:48:53.686 INFO:tasks.rados.rados.0.smithi136.stdout:3058: finishing write tid 5 to smithi13667081-30 2024-09-12T23:48:53.686 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 30 v 1494 (ObjNum 966 snap 301 seq_num 966) dirty exists 2024-09-12T23:48:53.686 INFO:tasks.rados.rados.0.smithi136.stdout:3058: left oid 30 (ObjNum 966 snap 301 seq_num 966) 2024-09-12T23:48:53.687 INFO:tasks.rados.rados.0.smithi136.stdout:3053: expect (ObjNum 893 snap 288 seq_num 893) 2024-09-12T23:48:53.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[75164]: audit 2024-09-12T23:48:52.454890+0000 mon.a (mon.0) 1071 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:53.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[75164]: audit 2024-09-12T23:48:52.461172+0000 mon.a (mon.0) 1072 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:53.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[75164]: cluster 2024-09-12T23:48:52.491418+0000 mon.a (mon.0) 1073 : cluster [DBG] osdmap e675: 8 total, 8 up, 8 in 2024-09-12T23:48:53.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[75164]: cluster 2024-09-12T23:48:52.811401+0000 mgr.y (mgr.24661) 331 : cluster [DBG] pgmap v589: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 309 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 6.6 MiB/s wr, 19 op/s 2024-09-12T23:48:53.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[76371]: audit 2024-09-12T23:48:52.454890+0000 mon.a (mon.0) 1071 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:53.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[76371]: audit 2024-09-12T23:48:52.461172+0000 mon.a (mon.0) 1072 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:53.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[76371]: cluster 2024-09-12T23:48:52.491418+0000 mon.a (mon.0) 1073 : cluster [DBG] osdmap e675: 8 total, 8 up, 8 in 2024-09-12T23:48:53.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[76371]: cluster 2024-09-12T23:48:52.811401+0000 mgr.y (mgr.24661) 331 : cluster [DBG] pgmap v589: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 309 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 6.6 MiB/s wr, 19 op/s 2024-09-12T23:48:53.758 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[100910]: ts=2024-09-12T23:48:53.634Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-09-12T23:48:53.758 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[100910]: ts=2024-09-12T23:48:53.634Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-09-12T23:48:53.758 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[100910]: ts=2024-09-12T23:48:53.635Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-12T23:48:53.758 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[100910]: ts=2024-09-12T23:48:53.635Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-12T23:48:53.758 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[100910]: ts=2024-09-12T23:48:53.635Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-12T23:48:53.758 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[100910]: ts=2024-09-12T23:48:53.635Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-12T23:48:53.759 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[100910]: ts=2024-09-12T23:48:53.636Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-12T23:48:53.759 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[100910]: ts=2024-09-12T23:48:53.636Z caller=node_exporter.go:117 level=info collector=arp 2024-09-12T23:48:53.759 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[100910]: ts=2024-09-12T23:48:53.636Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-12T23:48:53.759 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[100910]: ts=2024-09-12T23:48:53.636Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-12T23:48:53.759 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[100910]: ts=2024-09-12T23:48:53.636Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-12T23:48:53.759 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[100910]: ts=2024-09-12T23:48:53.636Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-12T23:48:53.759 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[100910]: ts=2024-09-12T23:48:53.636Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-12T23:48:53.759 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[100910]: ts=2024-09-12T23:48:53.636Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-12T23:48:53.759 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[100910]: ts=2024-09-12T23:48:53.636Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-12T23:48:53.759 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[100910]: ts=2024-09-12T23:48:53.636Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-12T23:48:53.759 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[100910]: ts=2024-09-12T23:48:53.636Z caller=node_exporter.go:117 level=info collector=edac 2024-09-12T23:48:53.759 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[100910]: ts=2024-09-12T23:48:53.636Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-12T23:48:53.760 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[100910]: ts=2024-09-12T23:48:53.636Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-12T23:48:53.760 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[100910]: ts=2024-09-12T23:48:53.636Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-12T23:48:53.760 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[100910]: ts=2024-09-12T23:48:53.636Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-12T23:48:53.760 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[100910]: ts=2024-09-12T23:48:53.636Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-12T23:48:53.760 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[100910]: ts=2024-09-12T23:48:53.636Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-12T23:48:53.760 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[100910]: ts=2024-09-12T23:48:53.636Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-12T23:48:53.760 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[100910]: ts=2024-09-12T23:48:53.636Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-12T23:48:53.760 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[100910]: ts=2024-09-12T23:48:53.636Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-12T23:48:53.760 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[100910]: ts=2024-09-12T23:48:53.636Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-12T23:48:53.760 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[100910]: ts=2024-09-12T23:48:53.636Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-12T23:48:53.760 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[100910]: ts=2024-09-12T23:48:53.636Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-12T23:48:53.760 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[100910]: ts=2024-09-12T23:48:53.636Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-12T23:48:53.760 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[100910]: ts=2024-09-12T23:48:53.636Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-12T23:48:53.761 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[100910]: ts=2024-09-12T23:48:53.636Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-12T23:48:53.761 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[100910]: ts=2024-09-12T23:48:53.636Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-12T23:48:53.761 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[100910]: ts=2024-09-12T23:48:53.636Z caller=node_exporter.go:117 level=info collector=os 2024-09-12T23:48:53.761 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[100910]: ts=2024-09-12T23:48:53.636Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-12T23:48:53.761 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[100910]: ts=2024-09-12T23:48:53.636Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-12T23:48:53.761 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[100910]: ts=2024-09-12T23:48:53.636Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-12T23:48:53.761 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[100910]: ts=2024-09-12T23:48:53.636Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-12T23:48:53.761 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[100910]: ts=2024-09-12T23:48:53.636Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-12T23:48:53.761 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[100910]: ts=2024-09-12T23:48:53.636Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-12T23:48:53.762 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[100910]: ts=2024-09-12T23:48:53.636Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-12T23:48:53.762 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[100910]: ts=2024-09-12T23:48:53.636Z caller=node_exporter.go:117 level=info collector=stat 2024-09-12T23:48:53.762 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[100910]: ts=2024-09-12T23:48:53.636Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-12T23:48:53.762 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[100910]: ts=2024-09-12T23:48:53.636Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-12T23:48:53.762 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[100910]: ts=2024-09-12T23:48:53.636Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-12T23:48:53.762 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[100910]: ts=2024-09-12T23:48:53.636Z caller=node_exporter.go:117 level=info collector=time 2024-09-12T23:48:53.762 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[100910]: ts=2024-09-12T23:48:53.636Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-12T23:48:53.762 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[100910]: ts=2024-09-12T23:48:53.636Z caller=node_exporter.go:117 level=info collector=uname 2024-09-12T23:48:53.762 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[100910]: ts=2024-09-12T23:48:53.636Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-12T23:48:53.762 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[100910]: ts=2024-09-12T23:48:53.636Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-12T23:48:53.762 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[100910]: ts=2024-09-12T23:48:53.636Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-12T23:48:53.762 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[100910]: ts=2024-09-12T23:48:53.636Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-09-12T23:48:53.763 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:48:53 smithi136 bash[100910]: ts=2024-09-12T23:48:53.636Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-12T23:48:53.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:53 smithi181 bash[67587]: audit 2024-09-12T23:48:52.454890+0000 mon.a (mon.0) 1071 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:53.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:53 smithi181 bash[67587]: audit 2024-09-12T23:48:52.461172+0000 mon.a (mon.0) 1072 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:53.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:53 smithi181 bash[67587]: cluster 2024-09-12T23:48:52.491418+0000 mon.a (mon.0) 1073 : cluster [DBG] osdmap e675: 8 total, 8 up, 8 in 2024-09-12T23:48:53.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:53 smithi181 bash[67587]: cluster 2024-09-12T23:48:52.811401+0000 mgr.y (mgr.24661) 331 : cluster [DBG] pgmap v589: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 309 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 6.6 MiB/s wr, 19 op/s 2024-09-12T23:48:54.297 INFO:tasks.rados.rados.0.smithi136.stdout:3061: finishing rollback tid 0 to smithi13667081-40 2024-09-12T23:48:54.297 INFO:tasks.rados.rados.0.smithi136.stdout:3061: finishing rollback tid 1 to smithi13667081-40 2024-09-12T23:48:54.297 INFO:tasks.rados.rados.0.smithi136.stdout:3061: finishing rollback tid 2 to smithi13667081-40 2024-09-12T23:48:54.297 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 40 v 2336 (ObjNum 945 snap 298 seq_num 945) dirty exists 2024-09-12T23:48:54.297 INFO:tasks.rados.rados.0.smithi136.stdout:3062: finishing copy_from to smithi13667081-9 2024-09-12T23:48:54.298 INFO:tasks.rados.rados.0.smithi136.stdout:3062: got expected ENOENT (src dne) 2024-09-12T23:48:54.298 INFO:tasks.rados.rados.0.smithi136.stdout:3062: finishing copy_from racing read to smithi13667081-9 2024-09-12T23:48:54.298 INFO:tasks.rados.rados.0.smithi136.stdout:3063: finishing write tid 1 to smithi13667081-50 2024-09-12T23:48:54.298 INFO:tasks.rados.rados.0.smithi136.stdout:3063: finishing write tid 2 to smithi13667081-50 2024-09-12T23:48:54.298 INFO:tasks.rados.rados.0.smithi136.stdout:3063: finishing write tid 3 to smithi13667081-50 2024-09-12T23:48:54.298 INFO:tasks.rados.rados.0.smithi136.stdout:3063: finishing write tid 4 to smithi13667081-50 2024-09-12T23:48:54.298 INFO:tasks.rados.rados.0.smithi136.stdout:3063: finishing write tid 5 to smithi13667081-50 2024-09-12T23:48:54.298 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 50 v 2271 (ObjNum 968 snap 301 seq_num 968) dirty exists 2024-09-12T23:48:54.298 INFO:tasks.rados.rados.0.smithi136.stdout:3063: left oid 50 (ObjNum 968 snap 301 seq_num 968) 2024-09-12T23:48:54.298 INFO:tasks.rados.rados.0.smithi136.stdout:3053: done (5 left) 2024-09-12T23:48:54.298 INFO:tasks.rados.rados.0.smithi136.stdout:3058: done (4 left) 2024-09-12T23:48:54.298 INFO:tasks.rados.rados.0.smithi136.stdout:3061: done (3 left) 2024-09-12T23:48:54.298 INFO:tasks.rados.rados.0.smithi136.stdout:3062: done (2 left) 2024-09-12T23:48:54.298 INFO:tasks.rados.rados.0.smithi136.stdout:3063: done (1 left) 2024-09-12T23:48:54.299 INFO:tasks.rados.rados.0.smithi136.stdout:3065: done (0 left) 2024-09-12T23:48:54.299 INFO:tasks.rados.rados.0.smithi136.stdout:3066: snap_remove snap 293 2024-09-12T23:48:54.514 INFO:tasks.rados.rados.0.smithi136.stdout:3066: done (0 left) 2024-09-12T23:48:54.514 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 21 current snap is 301 2024-09-12T23:48:54.514 INFO:tasks.rados.rados.0.smithi136.stdout:3067: seq_num 970 ranges {0=655360,655360=516096,1171456=688128} 2024-09-12T23:48:54.518 INFO:tasks.rados.rados.0.smithi136.stdout:3067: writing smithi13667081-21 from 0 to 655360 tid 1 2024-09-12T23:48:54.521 INFO:tasks.rados.rados.0.smithi136.stdout:3067: writing smithi13667081-21 from 655360 to 1171456 tid 2 2024-09-12T23:48:54.525 INFO:tasks.rados.rados.0.smithi136.stdout:3067: writing smithi13667081-21 from 1171456 to 1859584 tid 3 2024-09-12T23:48:54.526 INFO:tasks.rados.rados.0.smithi136.stdout:3068: rmattr oid 43 current snap is 301 2024-09-12T23:48:54.526 INFO:tasks.rados.rados.0.smithi136.stdout:3068: done (1 left) 2024-09-12T23:48:54.526 INFO:tasks.rados.rados.0.smithi136.stdout:3069: rollback oid 37 current snap is 301 2024-09-12T23:48:54.526 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 37 to 296 2024-09-12T23:48:54.526 INFO:tasks.rados.rados.0.smithi136.stdout:3070: snap_remove snap 273 2024-09-12T23:48:54.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:54 smithi136 bash[75164]: cluster 2024-09-12T23:48:53.505977+0000 mon.a (mon.0) 1074 : cluster [DBG] osdmap e676: 8 total, 8 up, 8 in 2024-09-12T23:48:54.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:54 smithi136 bash[75164]: audit 2024-09-12T23:48:53.750564+0000 mon.a (mon.0) 1075 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:48:54.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:54 smithi136 bash[76371]: cluster 2024-09-12T23:48:53.505977+0000 mon.a (mon.0) 1074 : cluster [DBG] osdmap e676: 8 total, 8 up, 8 in 2024-09-12T23:48:54.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:54 smithi136 bash[76371]: audit 2024-09-12T23:48:53.750564+0000 mon.a (mon.0) 1075 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:48:54.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:54 smithi181 bash[67587]: cluster 2024-09-12T23:48:53.505977+0000 mon.a (mon.0) 1074 : cluster [DBG] osdmap e676: 8 total, 8 up, 8 in 2024-09-12T23:48:54.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:54 smithi181 bash[67587]: audit 2024-09-12T23:48:53.750564+0000 mon.a (mon.0) 1075 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:48:55.264 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:55 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:55.265 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:48:55 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:55.265 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:55 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:55.265 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:48:55 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:55.265 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 12 23:48:55 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:55.266 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:48:55 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:55.266 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:48:55 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:55.266 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:48:55 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:55.267 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 12 23:48:55 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:55.267 INFO:journalctl@ceph.mds.b.smithi181.stdout:Sep 12 23:48:55 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:55.519 INFO:tasks.rados.rados.0.smithi136.stdout:3070: done (2 left) 2024-09-12T23:48:55.519 INFO:tasks.rados.rados.0.smithi136.stdout:3071: rollback oid 44 current snap is 301 2024-09-12T23:48:55.519 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 44 to 297 2024-09-12T23:48:55.520 INFO:tasks.rados.rados.0.smithi136.stdout:3069: finishing rollback tid 0 to smithi13667081-37 2024-09-12T23:48:55.520 INFO:tasks.rados.rados.0.smithi136.stdout:3069: finishing rollback tid 1 to smithi13667081-37 2024-09-12T23:48:55.520 INFO:tasks.rados.rados.0.smithi136.stdout:3069: finishing rollback tid 2 to smithi13667081-37 2024-09-12T23:48:55.520 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 37 v 2255 (ObjNum 636 snap 188 seq_num 636) dirty exists 2024-09-12T23:48:55.520 INFO:tasks.rados.rados.0.smithi136.stdout:3067: finishing write tid 1 to smithi13667081-21 2024-09-12T23:48:55.520 INFO:tasks.rados.rados.0.smithi136.stdout:3067: finishing write tid 2 to smithi13667081-21 2024-09-12T23:48:55.520 INFO:tasks.rados.rados.0.smithi136.stdout:3067: finishing write tid 3 to smithi13667081-21 2024-09-12T23:48:55.520 INFO:tasks.rados.rados.0.smithi136.stdout:3067: finishing write tid 4 to smithi13667081-21 2024-09-12T23:48:55.520 INFO:tasks.rados.rados.0.smithi136.stdout:3067: finishing write tid 5 to smithi13667081-21 2024-09-12T23:48:55.520 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 21 v 1811 (ObjNum 970 snap 301 seq_num 970) dirty exists 2024-09-12T23:48:55.520 INFO:tasks.rados.rados.0.smithi136.stdout:3067: left oid 21 (ObjNum 970 snap 301 seq_num 970) 2024-09-12T23:48:55.520 INFO:tasks.rados.rados.0.smithi136.stdout:3067: done (2 left) 2024-09-12T23:48:55.520 INFO:tasks.rados.rados.0.smithi136.stdout:3069: done (1 left) 2024-09-12T23:48:55.521 INFO:tasks.rados.rados.0.smithi136.stdout:3072: setattr oid 20 current snap is 301 2024-09-12T23:48:55.521 INFO:tasks.rados.rados.0.smithi136.stdout:3073: delete oid 10 current snap is 301 2024-09-12T23:48:55.528 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:55 smithi181 systemd[1]: Stopping Ceph node-exporter.b for da58a47c-715d-11ef-bceb-c7b262605968... 2024-09-12T23:48:55.536 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 20 v 1617 (ObjNum 739 snap 227 seq_num 739) dirty exists 2024-09-12T23:48:55.539 INFO:tasks.rados.rados.0.smithi136.stdout:3072: done (2 left) 2024-09-12T23:48:55.539 INFO:tasks.rados.rados.0.smithi136.stdout:3073: done (1 left) 2024-09-12T23:48:55.539 INFO:tasks.rados.rados.0.smithi136.stdout:3074: copy_from oid 8 from oid 40 current snap is 301 2024-09-12T23:48:55.540 INFO:tasks.rados.rados.0.smithi136.stdout:3075: snap_remove snap 298 2024-09-12T23:48:55.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:55 smithi181 bash[67587]: cluster 2024-09-12T23:48:54.516167+0000 mon.a (mon.0) 1076 : cluster [DBG] osdmap e677: 8 total, 8 up, 8 in 2024-09-12T23:48:55.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:55 smithi181 bash[67587]: cephadm 2024-09-12T23:48:54.624234+0000 mgr.y (mgr.24661) 332 : cephadm [INF] Upgrade: Updating node-exporter.b (2/2) 2024-09-12T23:48:55.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:55 smithi181 bash[67587]: cephadm 2024-09-12T23:48:54.624935+0000 mgr.y (mgr.24661) 333 : cephadm [INF] Deploying daemon node-exporter.b on smithi181 2024-09-12T23:48:55.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:55 smithi181 bash[67587]: cluster 2024-09-12T23:48:54.812192+0000 mgr.y (mgr.24661) 334 : cluster [DBG] pgmap v592: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 317 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 8.3 MiB/s wr, 27 op/s 2024-09-12T23:48:56.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:55 smithi136 bash[75164]: cluster 2024-09-12T23:48:54.516167+0000 mon.a (mon.0) 1076 : cluster [DBG] osdmap e677: 8 total, 8 up, 8 in 2024-09-12T23:48:56.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:55 smithi136 bash[75164]: cephadm 2024-09-12T23:48:54.624234+0000 mgr.y (mgr.24661) 332 : cephadm [INF] Upgrade: Updating node-exporter.b (2/2) 2024-09-12T23:48:56.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:55 smithi136 bash[75164]: cephadm 2024-09-12T23:48:54.624935+0000 mgr.y (mgr.24661) 333 : cephadm [INF] Deploying daemon node-exporter.b on smithi181 2024-09-12T23:48:56.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:55 smithi136 bash[75164]: cluster 2024-09-12T23:48:54.812192+0000 mgr.y (mgr.24661) 334 : cluster [DBG] pgmap v592: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 317 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 8.3 MiB/s wr, 27 op/s 2024-09-12T23:48:56.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:55 smithi136 bash[76371]: cluster 2024-09-12T23:48:54.516167+0000 mon.a (mon.0) 1076 : cluster [DBG] osdmap e677: 8 total, 8 up, 8 in 2024-09-12T23:48:56.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:55 smithi136 bash[76371]: cephadm 2024-09-12T23:48:54.624234+0000 mgr.y (mgr.24661) 332 : cephadm [INF] Upgrade: Updating node-exporter.b (2/2) 2024-09-12T23:48:56.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:55 smithi136 bash[76371]: cephadm 2024-09-12T23:48:54.624935+0000 mgr.y (mgr.24661) 333 : cephadm [INF] Deploying daemon node-exporter.b on smithi181 2024-09-12T23:48:56.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:55 smithi136 bash[76371]: cluster 2024-09-12T23:48:54.812192+0000 mgr.y (mgr.24661) 334 : cluster [DBG] pgmap v592: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 317 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 8.3 MiB/s wr, 27 op/s 2024-09-12T23:48:56.344 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:56 smithi181 bash[91957]: ceph-da58a47c-715d-11ef-bceb-c7b262605968-node-exporter-b 2024-09-12T23:48:56.344 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:56 smithi181 bash[92017]: ceph-da58a47c-715d-11ef-bceb-c7b262605968-node-exporter-b 2024-09-12T23:48:56.344 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:56 smithi181 systemd[1]: ceph-da58a47c-715d-11ef-bceb-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-09-12T23:48:56.344 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:56 smithi181 systemd[1]: ceph-da58a47c-715d-11ef-bceb-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-09-12T23:48:56.344 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:56 smithi181 systemd[1]: Stopped Ceph node-exporter.b for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:48:56.445 INFO:tasks.rados.rados.0.smithi136.stdout:3075: done (2 left) 2024-09-12T23:48:56.446 INFO:tasks.rados.rados.0.smithi136.stdout:3076: snap_remove snap 287 2024-09-12T23:48:56.844 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:48:56 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:56.844 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 12 23:48:56 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:56.845 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 12 23:48:56 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:56.845 INFO:journalctl@ceph.mds.b.smithi181.stdout:Sep 12 23:48:56 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:56.845 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:48:56 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:56.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:56 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:56.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:56 smithi181 bash[67587]: cluster 2024-09-12T23:48:55.517799+0000 mon.a (mon.0) 1077 : cluster [DBG] osdmap e678: 8 total, 8 up, 8 in 2024-09-12T23:48:56.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:56 smithi181 bash[67587]: cluster 2024-09-12T23:48:56.447791+0000 mon.a (mon.0) 1078 : cluster [DBG] osdmap e679: 8 total, 8 up, 8 in 2024-09-12T23:48:56.846 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:48:56 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:56.846 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:56 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:56.847 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:56 smithi181 systemd[1]: Started Ceph node-exporter.b for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:48:56.847 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:48:56 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:56.847 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:48:56 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:48:57.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:56 smithi136 bash[75164]: cluster 2024-09-12T23:48:55.517799+0000 mon.a (mon.0) 1077 : cluster [DBG] osdmap e678: 8 total, 8 up, 8 in 2024-09-12T23:48:57.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:56 smithi136 bash[75164]: cluster 2024-09-12T23:48:56.447791+0000 mon.a (mon.0) 1078 : cluster [DBG] osdmap e679: 8 total, 8 up, 8 in 2024-09-12T23:48:57.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:56 smithi136 bash[76371]: cluster 2024-09-12T23:48:55.517799+0000 mon.a (mon.0) 1077 : cluster [DBG] osdmap e678: 8 total, 8 up, 8 in 2024-09-12T23:48:57.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:56 smithi136 bash[76371]: cluster 2024-09-12T23:48:56.447791+0000 mon.a (mon.0) 1078 : cluster [DBG] osdmap e679: 8 total, 8 up, 8 in 2024-09-12T23:48:57.448 INFO:tasks.rados.rados.0.smithi136.stdout:3076: done (2 left) 2024-09-12T23:48:57.448 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 37 current snap is 301 2024-09-12T23:48:57.449 INFO:tasks.rados.rados.0.smithi136.stdout:3077: seq_num 972 ranges {1056768=688128,1744896=688128,2433024=516096} 2024-09-12T23:48:57.459 INFO:tasks.rados.rados.0.smithi136.stdout:3077: writing smithi13667081-37 from 1056768 to 1744896 tid 1 2024-09-12T23:48:57.464 INFO:tasks.rados.rados.0.smithi136.stdout:3077: writing smithi13667081-37 from 1744896 to 2433024 tid 2 2024-09-12T23:48:57.468 INFO:tasks.rados.rados.0.smithi136.stdout:3077: writing smithi13667081-37 from 2433024 to 2949120 tid 3 2024-09-12T23:48:57.468 INFO:tasks.rados.rados.0.smithi136.stdout:3071: finishing rollback tid 1 to smithi13667081-44 2024-09-12T23:48:57.468 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 44 v 1713 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-12T23:48:57.468 INFO:tasks.rados.rados.0.smithi136.stdout:3074: finishing copy_from racing read to smithi13667081-8 2024-09-12T23:48:57.468 INFO:tasks.rados.rados.0.smithi136.stdout:3074: finishing copy_from to smithi13667081-8 2024-09-12T23:48:57.469 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 8 v 2336 (ObjNum 945 snap 298 seq_num 945) dirty exists 2024-09-12T23:48:57.469 INFO:tasks.rados.rados.0.smithi136.stdout:3071: done (2 left) 2024-09-12T23:48:57.469 INFO:tasks.rados.rados.0.smithi136.stdout:3074: done (1 left) 2024-09-12T23:48:57.469 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 29 current snap is 301 2024-09-12T23:48:57.469 INFO:tasks.rados.rados.0.smithi136.stdout:3078: seq_num 973 ranges {18743296=704512,19447808=573440,20021248=507904,20529152=122880} 2024-09-12T23:48:57.579 INFO:tasks.rados.rados.0.smithi136.stdout:3078: writing smithi13667081-29 from 18743296 to 19447808 tid 1 2024-09-12T23:48:57.582 INFO:tasks.rados.rados.0.smithi136.stdout:3078: writing smithi13667081-29 from 19447808 to 20021248 tid 2 2024-09-12T23:48:57.585 INFO:tasks.rados.rados.0.smithi136.stdout:3078: writing smithi13667081-29 from 20021248 to 20529152 tid 3 2024-09-12T23:48:57.585 INFO:tasks.rados.rados.0.smithi136.stdout:3078: writing smithi13667081-29 from 20529152 to 20652032 tid 4 2024-09-12T23:48:57.586 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 20 current snap is 301 2024-09-12T23:48:57.586 INFO:tasks.rados.rados.0.smithi136.stdout:3079: seq_num 974 ranges {385024=720896,1105920=507904,1613824=671744,2285568=24576} 2024-09-12T23:48:57.592 INFO:tasks.rados.rados.0.smithi136.stdout:3079: writing smithi13667081-20 from 385024 to 1105920 tid 1 2024-09-12T23:48:57.594 INFO:tasks.rados.rados.0.smithi136.stdout:3079: writing smithi13667081-20 from 1105920 to 1613824 tid 2 2024-09-12T23:48:57.599 INFO:tasks.rados.rados.0.smithi136.stdout:3079: writing smithi13667081-20 from 1613824 to 2285568 tid 3 2024-09-12T23:48:57.599 INFO:tasks.rados.rados.0.smithi136.stdout:3079: writing smithi13667081-20 from 2285568 to 2310144 tid 4 2024-09-12T23:48:57.599 INFO:tasks.rados.rados.0.smithi136.stdout:3080: copy_from oid 5 from oid 36 current snap is 301 2024-09-12T23:48:57.600 INFO:tasks.rados.rados.0.smithi136.stdout:3077: finishing write tid 1 to smithi13667081-37 2024-09-12T23:48:57.600 INFO:tasks.rados.rados.0.smithi136.stdout:3081: snap_create 2024-09-12T23:48:57.600 INFO:tasks.rados.rados.0.smithi136.stdout:3077: finishing write tid 2 to smithi13667081-37 2024-09-12T23:48:57.600 INFO:tasks.rados.rados.0.smithi136.stdout:3077: finishing write tid 3 to smithi13667081-37 2024-09-12T23:48:57.600 INFO:tasks.rados.rados.0.smithi136.stdout:3077: finishing write tid 4 to smithi13667081-37 2024-09-12T23:48:57.600 INFO:tasks.rados.rados.0.smithi136.stdout:3077: finishing write tid 5 to smithi13667081-37 2024-09-12T23:48:57.600 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 37 v 2259 (ObjNum 972 snap 301 seq_num 972) dirty exists 2024-09-12T23:48:57.600 INFO:tasks.rados.rados.0.smithi136.stdout:3077: left oid 37 (ObjNum 972 snap 301 seq_num 972) 2024-09-12T23:48:57.600 INFO:tasks.rados.rados.0.smithi136.stdout:3078: finishing write tid 1 to smithi13667081-29 2024-09-12T23:48:57.600 INFO:tasks.rados.rados.0.smithi136.stdout:3078: finishing write tid 2 to smithi13667081-29 2024-09-12T23:48:57.600 INFO:tasks.rados.rados.0.smithi136.stdout:3078: finishing write tid 3 to smithi13667081-29 2024-09-12T23:48:57.601 INFO:tasks.rados.rados.0.smithi136.stdout:3078: finishing write tid 4 to smithi13667081-29 2024-09-12T23:48:57.602 INFO:tasks.rados.rados.0.smithi136.stdout:3078: finishing write tid 5 to smithi13667081-29 2024-09-12T23:48:57.605 INFO:tasks.rados.rados.0.smithi136.stdout:3079: finishing write tid 1 to smithi13667081-20 2024-09-12T23:48:57.605 INFO:tasks.rados.rados.0.smithi136.stdout:3079: finishing write tid 2 to smithi13667081-20 2024-09-12T23:48:57.607 INFO:tasks.rados.rados.0.smithi136.stdout:3079: finishing write tid 3 to smithi13667081-20 2024-09-12T23:48:57.607 INFO:tasks.rados.rados.0.smithi136.stdout:3078: finishing write tid 6 to smithi13667081-29 2024-09-12T23:48:57.607 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 29 v 2276 (ObjNum 973 snap 301 seq_num 973) dirty exists 2024-09-12T23:48:57.607 INFO:tasks.rados.rados.0.smithi136.stdout:3078: left oid 29 (ObjNum 973 snap 301 seq_num 973) 2024-09-12T23:48:57.607 INFO:tasks.rados.rados.0.smithi136.stdout:3079: finishing write tid 4 to smithi13667081-20 2024-09-12T23:48:57.608 INFO:tasks.rados.rados.0.smithi136.stdout:3079: finishing write tid 5 to smithi13667081-20 2024-09-12T23:48:57.612 INFO:tasks.rados.rados.0.smithi136.stdout:3079: finishing write tid 6 to smithi13667081-20 2024-09-12T23:48:57.612 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 20 v 1622 (ObjNum 974 snap 301 seq_num 974) dirty exists 2024-09-12T23:48:57.612 INFO:tasks.rados.rados.0.smithi136.stdout:3079: left oid 20 (ObjNum 974 snap 301 seq_num 974) 2024-09-12T23:48:57.615 INFO:tasks.rados.rados.0.smithi136.stdout:3080: finishing copy_from racing read to smithi13667081-5 2024-09-12T23:48:57.618 INFO:tasks.rados.rados.0.smithi136.stdout:3080: finishing copy_from to smithi13667081-5 2024-09-12T23:48:57.618 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 5 v 1633 (ObjNum 965 snap 301 seq_num 965) dirty exists 2024-09-12T23:48:57.806 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[67587]: audit 2024-09-12T23:48:56.542038+0000 mon.a (mon.0) 1079 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:57.806 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[67587]: audit 2024-09-12T23:48:56.549654+0000 mon.a (mon.0) 1080 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:57.806 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[67587]: audit 2024-09-12T23:48:56.551372+0000 mon.a (mon.0) 1081 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:48:57.806 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[67587]: cluster 2024-09-12T23:48:56.813134+0000 mgr.y (mgr.24661) 335 : cluster [DBG] pgmap v595: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 318 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.6 MiB/s wr, 15 op/s 2024-09-12T23:48:57.806 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[67587]: cluster 2024-09-12T23:48:57.450490+0000 mon.a (mon.0) 1082 : cluster [DBG] osdmap e680: 8 total, 8 up, 8 in 2024-09-12T23:48:58.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:57 smithi136 bash[75164]: audit 2024-09-12T23:48:56.542038+0000 mon.a (mon.0) 1079 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:58.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:57 smithi136 bash[75164]: audit 2024-09-12T23:48:56.549654+0000 mon.a (mon.0) 1080 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:58.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:57 smithi136 bash[75164]: audit 2024-09-12T23:48:56.551372+0000 mon.a (mon.0) 1081 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:48:58.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:57 smithi136 bash[75164]: cluster 2024-09-12T23:48:56.813134+0000 mgr.y (mgr.24661) 335 : cluster [DBG] pgmap v595: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 318 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.6 MiB/s wr, 15 op/s 2024-09-12T23:48:58.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:57 smithi136 bash[75164]: cluster 2024-09-12T23:48:57.450490+0000 mon.a (mon.0) 1082 : cluster [DBG] osdmap e680: 8 total, 8 up, 8 in 2024-09-12T23:48:58.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:57 smithi136 bash[76371]: audit 2024-09-12T23:48:56.542038+0000 mon.a (mon.0) 1079 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:58.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:57 smithi136 bash[76371]: audit 2024-09-12T23:48:56.549654+0000 mon.a (mon.0) 1080 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:48:58.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:57 smithi136 bash[76371]: audit 2024-09-12T23:48:56.551372+0000 mon.a (mon.0) 1081 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:48:58.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:57 smithi136 bash[76371]: cluster 2024-09-12T23:48:56.813134+0000 mgr.y (mgr.24661) 335 : cluster [DBG] pgmap v595: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 318 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.6 MiB/s wr, 15 op/s 2024-09-12T23:48:58.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:57 smithi136 bash[76371]: cluster 2024-09-12T23:48:57.450490+0000 mon.a (mon.0) 1082 : cluster [DBG] osdmap e680: 8 total, 8 up, 8 in 2024-09-12T23:48:58.095 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[92072]: ts=2024-09-12T23:48:57.811Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-09-12T23:48:58.095 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[92072]: ts=2024-09-12T23:48:57.811Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-09-12T23:48:58.095 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[92072]: ts=2024-09-12T23:48:57.812Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-12T23:48:58.095 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[92072]: ts=2024-09-12T23:48:57.812Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-12T23:48:58.095 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[92072]: ts=2024-09-12T23:48:57.813Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-12T23:48:58.096 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[92072]: ts=2024-09-12T23:48:57.813Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-12T23:48:58.096 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[92072]: ts=2024-09-12T23:48:57.813Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-12T23:48:58.096 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[92072]: ts=2024-09-12T23:48:57.813Z caller=node_exporter.go:117 level=info collector=arp 2024-09-12T23:48:58.096 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[92072]: ts=2024-09-12T23:48:57.813Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-12T23:48:58.096 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[92072]: ts=2024-09-12T23:48:57.813Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-12T23:48:58.096 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[92072]: ts=2024-09-12T23:48:57.813Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-12T23:48:58.096 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[92072]: ts=2024-09-12T23:48:57.813Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-12T23:48:58.096 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[92072]: ts=2024-09-12T23:48:57.813Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-12T23:48:58.096 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[92072]: ts=2024-09-12T23:48:57.813Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-12T23:48:58.097 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[92072]: ts=2024-09-12T23:48:57.813Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-12T23:48:58.097 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[92072]: ts=2024-09-12T23:48:57.813Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-12T23:48:58.097 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[92072]: ts=2024-09-12T23:48:57.813Z caller=node_exporter.go:117 level=info collector=edac 2024-09-12T23:48:58.097 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[92072]: ts=2024-09-12T23:48:57.813Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-12T23:48:58.097 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[92072]: ts=2024-09-12T23:48:57.813Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-12T23:48:58.097 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[92072]: ts=2024-09-12T23:48:57.813Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-12T23:48:58.097 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[92072]: ts=2024-09-12T23:48:57.813Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-12T23:48:58.097 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[92072]: ts=2024-09-12T23:48:57.813Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-12T23:48:58.098 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[92072]: ts=2024-09-12T23:48:57.813Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-12T23:48:58.098 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[92072]: ts=2024-09-12T23:48:57.813Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-12T23:48:58.098 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[92072]: ts=2024-09-12T23:48:57.813Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-12T23:48:58.098 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[92072]: ts=2024-09-12T23:48:57.813Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-12T23:48:58.098 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[92072]: ts=2024-09-12T23:48:57.813Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-12T23:48:58.098 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[92072]: ts=2024-09-12T23:48:57.813Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-12T23:48:58.098 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[92072]: ts=2024-09-12T23:48:57.813Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-12T23:48:58.098 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[92072]: ts=2024-09-12T23:48:57.813Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-12T23:48:58.099 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[92072]: ts=2024-09-12T23:48:57.813Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-12T23:48:58.099 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[92072]: ts=2024-09-12T23:48:57.813Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-12T23:48:58.099 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[92072]: ts=2024-09-12T23:48:57.813Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-12T23:48:58.099 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[92072]: ts=2024-09-12T23:48:57.813Z caller=node_exporter.go:117 level=info collector=os 2024-09-12T23:48:58.099 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[92072]: ts=2024-09-12T23:48:57.813Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-12T23:48:58.099 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[92072]: ts=2024-09-12T23:48:57.813Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-12T23:48:58.099 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[92072]: ts=2024-09-12T23:48:57.813Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-12T23:48:58.099 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[92072]: ts=2024-09-12T23:48:57.813Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-12T23:48:58.099 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[92072]: ts=2024-09-12T23:48:57.813Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-12T23:48:58.099 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[92072]: ts=2024-09-12T23:48:57.813Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-12T23:48:58.099 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[92072]: ts=2024-09-12T23:48:57.813Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-12T23:48:58.099 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[92072]: ts=2024-09-12T23:48:57.813Z caller=node_exporter.go:117 level=info collector=stat 2024-09-12T23:48:58.099 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[92072]: ts=2024-09-12T23:48:57.813Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-12T23:48:58.100 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[92072]: ts=2024-09-12T23:48:57.813Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-12T23:48:58.100 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[92072]: ts=2024-09-12T23:48:57.813Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-12T23:48:58.100 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[92072]: ts=2024-09-12T23:48:57.813Z caller=node_exporter.go:117 level=info collector=time 2024-09-12T23:48:58.100 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[92072]: ts=2024-09-12T23:48:57.813Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-12T23:48:58.100 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[92072]: ts=2024-09-12T23:48:57.813Z caller=node_exporter.go:117 level=info collector=uname 2024-09-12T23:48:58.100 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[92072]: ts=2024-09-12T23:48:57.813Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-12T23:48:58.100 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[92072]: ts=2024-09-12T23:48:57.813Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-12T23:48:58.100 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[92072]: ts=2024-09-12T23:48:57.813Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-12T23:48:58.100 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[92072]: ts=2024-09-12T23:48:57.814Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-09-12T23:48:58.100 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:48:57 smithi181 bash[92072]: ts=2024-09-12T23:48:57.814Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-12T23:48:58.555 INFO:tasks.rados.rados.0.smithi136.stdout:3077: done (4 left) 2024-09-12T23:48:58.555 INFO:tasks.rados.rados.0.smithi136.stdout:3078: done (3 left) 2024-09-12T23:48:58.555 INFO:tasks.rados.rados.0.smithi136.stdout:3079: done (2 left) 2024-09-12T23:48:58.556 INFO:tasks.rados.rados.0.smithi136.stdout:3080: done (1 left) 2024-09-12T23:48:58.556 INFO:tasks.rados.rados.0.smithi136.stdout:3081: done (0 left) 2024-09-12T23:48:58.556 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 13 current snap is 302 2024-09-12T23:48:58.556 INFO:tasks.rados.rados.0.smithi136.stdout:3082: seq_num 975 ranges {8847360=737280,9584640=786432,10371072=417792} 2024-09-12T23:48:58.609 INFO:tasks.rados.rados.0.smithi136.stdout:3082: writing smithi13667081-13 from 8847360 to 9584640 tid 1 2024-09-12T23:48:58.613 INFO:tasks.rados.rados.0.smithi136.stdout:3082: writing smithi13667081-13 from 9584640 to 10371072 tid 2 2024-09-12T23:48:58.616 INFO:tasks.rados.rados.0.smithi136.stdout:3082: writing smithi13667081-13 from 10371072 to 10788864 tid 3 2024-09-12T23:48:58.617 INFO:tasks.rados.rados.0.smithi136.stdout:3083: rollback oid 43 current snap is 302 2024-09-12T23:48:58.617 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 43 to 291 2024-09-12T23:48:58.617 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 30 current snap is 302 2024-09-12T23:48:58.617 INFO:tasks.rados.rados.0.smithi136.stdout:3084: seq_num 976 ranges {1785856=753664,2539520=671744,3211264=532480} 2024-09-12T23:48:58.631 INFO:tasks.rados.rados.0.smithi136.stdout:3084: writing smithi13667081-30 from 1785856 to 2539520 tid 1 2024-09-12T23:48:58.635 INFO:tasks.rados.rados.0.smithi136.stdout:3084: writing smithi13667081-30 from 2539520 to 3211264 tid 2 2024-09-12T23:48:58.639 INFO:tasks.rados.rados.0.smithi136.stdout:3084: writing smithi13667081-30 from 3211264 to 3743744 tid 3 2024-09-12T23:48:58.639 INFO:tasks.rados.rados.0.smithi136.stdout:3085: snap_create 2024-09-12T23:48:58.639 INFO:tasks.rados.rados.0.smithi136.stdout:3083: finishing rollback tid 1 to smithi13667081-43 2024-09-12T23:48:58.639 INFO:tasks.rados.rados.0.smithi136.stdout:3083: finishing rollback tid 2 to smithi13667081-43 2024-09-12T23:48:58.639 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 43 v 1913 (ObjNum 915 snap 290 seq_num 915) dirty exists 2024-09-12T23:48:58.639 INFO:tasks.rados.rados.0.smithi136.stdout:3082: finishing write tid 1 to smithi13667081-13 2024-09-12T23:48:58.639 INFO:tasks.rados.rados.0.smithi136.stdout:3082: finishing write tid 2 to smithi13667081-13 2024-09-12T23:48:58.639 INFO:tasks.rados.rados.0.smithi136.stdout:3082: finishing write tid 3 to smithi13667081-13 2024-09-12T23:48:58.639 INFO:tasks.rados.rados.0.smithi136.stdout:3082: finishing write tid 4 to smithi13667081-13 2024-09-12T23:48:58.639 INFO:tasks.rados.rados.0.smithi136.stdout:3082: finishing write tid 5 to smithi13667081-13 2024-09-12T23:48:58.639 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 13 v 2280 (ObjNum 975 snap 302 seq_num 975) dirty exists 2024-09-12T23:48:58.639 INFO:tasks.rados.rados.0.smithi136.stdout:3082: left oid 13 (ObjNum 975 snap 302 seq_num 975) 2024-09-12T23:48:58.646 INFO:tasks.rados.rados.0.smithi136.stdout:3084: finishing write tid 1 to smithi13667081-30 2024-09-12T23:48:58.647 INFO:tasks.rados.rados.0.smithi136.stdout:3084: finishing write tid 2 to smithi13667081-30 2024-09-12T23:48:58.647 INFO:tasks.rados.rados.0.smithi136.stdout:3084: finishing write tid 3 to smithi13667081-30 2024-09-12T23:48:58.648 INFO:tasks.rados.rados.0.smithi136.stdout:3084: finishing write tid 4 to smithi13667081-30 2024-09-12T23:48:58.651 INFO:tasks.rados.rados.0.smithi136.stdout:3084: finishing write tid 5 to smithi13667081-30 2024-09-12T23:48:58.651 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 30 v 1498 (ObjNum 976 snap 302 seq_num 976) dirty exists 2024-09-12T23:48:58.651 INFO:tasks.rados.rados.0.smithi136.stdout:3084: left oid 30 (ObjNum 976 snap 302 seq_num 976) 2024-09-12T23:48:59.577 INFO:tasks.rados.rados.0.smithi136.stdout:3082: done (3 left) 2024-09-12T23:48:59.577 INFO:tasks.rados.rados.0.smithi136.stdout:3083: done (2 left) 2024-09-12T23:48:59.577 INFO:tasks.rados.rados.0.smithi136.stdout:3084: done (1 left) 2024-09-12T23:48:59.577 INFO:tasks.rados.rados.0.smithi136.stdout:3085: done (0 left) 2024-09-12T23:48:59.578 INFO:tasks.rados.rados.0.smithi136.stdout:3086: snap_remove snap 263 2024-09-12T23:48:59.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:59 smithi136 bash[75164]: cluster 2024-09-12T23:48:58.557359+0000 mon.a (mon.0) 1083 : cluster [DBG] osdmap e681: 8 total, 8 up, 8 in 2024-09-12T23:48:59.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:48:59 smithi136 bash[75164]: cluster 2024-09-12T23:48:58.813879+0000 mgr.y (mgr.24661) 336 : cluster [DBG] pgmap v598: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 321 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.0 MiB/s wr, 9 op/s 2024-09-12T23:48:59.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:48:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:48:59] "GET /metrics HTTP/1.1" 200 36441 "" "Prometheus/2.43.0" 2024-09-12T23:48:59.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:59 smithi136 bash[76371]: cluster 2024-09-12T23:48:58.557359+0000 mon.a (mon.0) 1083 : cluster [DBG] osdmap e681: 8 total, 8 up, 8 in 2024-09-12T23:48:59.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:48:59 smithi136 bash[76371]: cluster 2024-09-12T23:48:58.813879+0000 mgr.y (mgr.24661) 336 : cluster [DBG] pgmap v598: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 321 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.0 MiB/s wr, 9 op/s 2024-09-12T23:48:59.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:59 smithi181 bash[67587]: cluster 2024-09-12T23:48:58.557359+0000 mon.a (mon.0) 1083 : cluster [DBG] osdmap e681: 8 total, 8 up, 8 in 2024-09-12T23:48:59.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:48:59 smithi181 bash[67587]: cluster 2024-09-12T23:48:58.813879+0000 mgr.y (mgr.24661) 336 : cluster [DBG] pgmap v598: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 321 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.0 MiB/s wr, 9 op/s 2024-09-12T23:49:00.580 INFO:tasks.rados.rados.0.smithi136.stdout:3086: done (0 left) 2024-09-12T23:49:00.580 INFO:tasks.rados.rados.0.smithi136.stdout:3087: setattr oid 40 current snap is 303 2024-09-12T23:49:00.581 INFO:tasks.rados.rados.0.smithi136.stdout:3088: read oid 41 snap 295 2024-09-12T23:49:00.581 INFO:tasks.rados.rados.0.smithi136.stdout:3088: expect deleted 2024-09-12T23:49:00.582 INFO:tasks.rados.rados.0.smithi136.stdout:3089: snap_remove snap 301 2024-09-12T23:49:00.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:00 smithi181 bash[67587]: cluster 2024-09-12T23:48:59.567922+0000 mon.a (mon.0) 1084 : cluster [DBG] osdmap e682: 8 total, 8 up, 8 in 2024-09-12T23:49:01.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:00 smithi136 bash[75164]: cluster 2024-09-12T23:48:59.567922+0000 mon.a (mon.0) 1084 : cluster [DBG] osdmap e682: 8 total, 8 up, 8 in 2024-09-12T23:49:01.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:00 smithi136 bash[76371]: cluster 2024-09-12T23:48:59.567922+0000 mon.a (mon.0) 1084 : cluster [DBG] osdmap e682: 8 total, 8 up, 8 in 2024-09-12T23:49:01.457 INFO:tasks.rados.rados.0.smithi136.stdout:3089: done (2 left) 2024-09-12T23:49:01.457 INFO:tasks.rados.rados.0.smithi136.stdout:3090: copy_from oid 27 from oid 44 current snap is 303 2024-09-12T23:49:01.458 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 40 v 2389 (ObjNum 945 snap 298 seq_num 945) dirty exists 2024-09-12T23:49:01.458 INFO:tasks.rados.rados.0.smithi136.stdout:3087: done (2 left) 2024-09-12T23:49:01.458 INFO:tasks.rados.rados.0.smithi136.stdout:3088: done (1 left) 2024-09-12T23:49:01.458 INFO:tasks.rados.rados.0.smithi136.stdout:3091: rmattr oid 48 current snap is 303 2024-09-12T23:49:01.458 INFO:tasks.rados.rados.0.smithi136.stdout:3091: done (1 left) 2024-09-12T23:49:01.458 INFO:tasks.rados.rados.0.smithi136.stdout:3092: rmattr oid 8 current snap is 303 2024-09-12T23:49:01.458 INFO:tasks.rados.rados.0.smithi136.stdout:3092: done (1 left) 2024-09-12T23:49:01.458 INFO:tasks.rados.rados.0.smithi136.stdout:3093: read oid 40 snap 284 2024-09-12T23:49:01.458 INFO:tasks.rados.rados.0.smithi136.stdout:3093: expect (ObjNum 788 snap 250 seq_num 788) 2024-09-12T23:49:01.458 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 16 current snap is 303 2024-09-12T23:49:01.458 INFO:tasks.rados.rados.0.smithi136.stdout:3094: seq_num 977 ranges {425984=770048,1196032=557056,1753088=491520,2244608=155648} 2024-09-12T23:49:01.458 INFO:tasks.rados.rados.0.smithi136.stdout:3094: writing smithi13667081-16 from 425984 to 1196032 tid 1 2024-09-12T23:49:01.461 INFO:tasks.rados.rados.0.smithi136.stdout:3094: writing smithi13667081-16 from 1196032 to 1753088 tid 2 2024-09-12T23:49:01.464 INFO:tasks.rados.rados.0.smithi136.stdout:3094: writing smithi13667081-16 from 1753088 to 2244608 tid 3 2024-09-12T23:49:01.466 INFO:tasks.rados.rados.0.smithi136.stdout:3094: writing smithi13667081-16 from 2244608 to 2400256 tid 4 2024-09-12T23:49:01.466 INFO:tasks.rados.rados.0.smithi136.stdout:3095: copy_from oid 32 from oid 5 current snap is 303 2024-09-12T23:49:01.466 INFO:tasks.rados.rados.0.smithi136.stdout:3090: finishing copy_from to smithi13667081-27 2024-09-12T23:49:01.466 INFO:tasks.rados.rados.0.smithi136.stdout:3090: got expected ENOENT (src dne) 2024-09-12T23:49:01.466 INFO:tasks.rados.rados.0.smithi136.stdout:3090: finishing copy_from racing read to smithi13667081-27 2024-09-12T23:49:01.466 INFO:tasks.rados.rados.0.smithi136.stdout:3090: done (3 left) 2024-09-12T23:49:01.466 INFO:tasks.rados.rados.0.smithi136.stdout:3096: delete oid 21 current snap is 303 2024-09-12T23:49:01.468 INFO:tasks.rados.rados.0.smithi136.stdout:3093: expect (ObjNum 788 snap 250 seq_num 788) 2024-09-12T23:49:01.521 INFO:tasks.rados.rados.0.smithi136.stdout:3094: finishing write tid 1 to smithi13667081-16 2024-09-12T23:49:01.521 INFO:tasks.rados.rados.0.smithi136.stdout:3094: finishing write tid 2 to smithi13667081-16 2024-09-12T23:49:01.521 INFO:tasks.rados.rados.0.smithi136.stdout:3094: finishing write tid 3 to smithi13667081-16 2024-09-12T23:49:01.521 INFO:tasks.rados.rados.0.smithi136.stdout:3094: finishing write tid 4 to smithi13667081-16 2024-09-12T23:49:01.521 INFO:tasks.rados.rados.0.smithi136.stdout:3094: finishing write tid 5 to smithi13667081-16 2024-09-12T23:49:01.521 INFO:tasks.rados.rados.0.smithi136.stdout:3093: done (3 left) 2024-09-12T23:49:01.522 INFO:tasks.rados.rados.0.smithi136.stdout:3096: done (2 left) 2024-09-12T23:49:01.522 INFO:tasks.rados.rados.0.smithi136.stdout:3097: copy_from oid 20 from oid 11 current snap is 303 2024-09-12T23:49:01.522 INFO:tasks.rados.rados.0.smithi136.stdout:3094: finishing write tid 6 to smithi13667081-16 2024-09-12T23:49:01.522 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 16 v 1925 (ObjNum 977 snap 303 seq_num 977) dirty exists 2024-09-12T23:49:01.522 INFO:tasks.rados.rados.0.smithi136.stdout:3094: left oid 16 (ObjNum 977 snap 303 seq_num 977) 2024-09-12T23:49:01.522 INFO:tasks.rados.rados.0.smithi136.stdout:3095: finishing copy_from racing read to smithi13667081-32 2024-09-12T23:49:01.522 INFO:tasks.rados.rados.0.smithi136.stdout:3095: finishing copy_from to smithi13667081-32 2024-09-12T23:49:01.522 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 32 v 1633 (ObjNum 965 snap 301 seq_num 965) dirty exists 2024-09-12T23:49:01.522 INFO:tasks.rados.rados.0.smithi136.stdout:3094: done (2 left) 2024-09-12T23:49:01.522 INFO:tasks.rados.rados.0.smithi136.stdout:3095: done (1 left) 2024-09-12T23:49:01.522 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 28 current snap is 303 2024-09-12T23:49:01.522 INFO:tasks.rados.rados.0.smithi136.stdout:3098: seq_num 979 ranges {0=802816,802816=729088,1531904=458752,1990656=16384} 2024-09-12T23:49:01.525 INFO:tasks.rados.rados.0.smithi136.stdout:3098: writing smithi13667081-28 from 0 to 802816 tid 1 2024-09-12T23:49:01.529 INFO:tasks.rados.rados.0.smithi136.stdout:3098: writing smithi13667081-28 from 802816 to 1531904 tid 2 2024-09-12T23:49:01.532 INFO:tasks.rados.rados.0.smithi136.stdout:3098: writing smithi13667081-28 from 1531904 to 1990656 tid 3 2024-09-12T23:49:01.532 INFO:tasks.rados.rados.0.smithi136.stdout:3098: writing smithi13667081-28 from 1990656 to 2007040 tid 4 2024-09-12T23:49:01.532 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 12 current snap is 303 2024-09-12T23:49:01.532 INFO:tasks.rados.rados.0.smithi136.stdout:3097: finishing copy_from to smithi13667081-20 2024-09-12T23:49:01.532 INFO:tasks.rados.rados.0.smithi136.stdout:3097: got expected ENOENT (src dne) 2024-09-12T23:49:01.533 INFO:tasks.rados.rados.0.smithi136.stdout:3097: finishing copy_from racing read to smithi13667081-20 2024-09-12T23:49:01.533 INFO:tasks.rados.rados.0.smithi136.stdout:3098: finishing write tid 1 to smithi13667081-28 2024-09-12T23:49:01.533 INFO:tasks.rados.rados.0.smithi136.stdout:3099: seq_num 980 ranges {2940928=417792,3358720=614400,3973120=663552,4636672=327680} 2024-09-12T23:49:01.551 INFO:tasks.rados.rados.0.smithi136.stdout:3099: writing smithi13667081-12 from 2940928 to 3358720 tid 1 2024-09-12T23:49:01.555 INFO:tasks.rados.rados.0.smithi136.stdout:3099: writing smithi13667081-12 from 3358720 to 3973120 tid 2 2024-09-12T23:49:01.559 INFO:tasks.rados.rados.0.smithi136.stdout:3099: writing smithi13667081-12 from 3973120 to 4636672 tid 3 2024-09-12T23:49:01.562 INFO:tasks.rados.rados.0.smithi136.stdout:3099: writing smithi13667081-12 from 4636672 to 4964352 tid 4 2024-09-12T23:49:01.562 INFO:tasks.rados.rados.0.smithi136.stdout:3097: done (2 left) 2024-09-12T23:49:01.562 INFO:tasks.rados.rados.0.smithi136.stdout:3100: copy_from oid 32 from oid 47 current snap is 303 2024-09-12T23:49:01.562 INFO:tasks.rados.rados.0.smithi136.stdout:3098: finishing write tid 2 to smithi13667081-28 2024-09-12T23:49:01.562 INFO:tasks.rados.rados.0.smithi136.stdout:3098: finishing write tid 3 to smithi13667081-28 2024-09-12T23:49:01.562 INFO:tasks.rados.rados.0.smithi136.stdout:3098: finishing write tid 4 to smithi13667081-28 2024-09-12T23:49:01.562 INFO:tasks.rados.rados.0.smithi136.stdout:3098: finishing write tid 5 to smithi13667081-28 2024-09-12T23:49:01.563 INFO:tasks.rados.rados.0.smithi136.stdout:3098: finishing write tid 6 to smithi13667081-28 2024-09-12T23:49:01.563 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 28 v 1930 (ObjNum 979 snap 303 seq_num 979) dirty exists 2024-09-12T23:49:01.563 INFO:tasks.rados.rados.0.smithi136.stdout:3098: left oid 28 (ObjNum 979 snap 303 seq_num 979) 2024-09-12T23:49:01.563 INFO:tasks.rados.rados.0.smithi136.stdout:3099: finishing write tid 1 to smithi13667081-12 2024-09-12T23:49:01.563 INFO:tasks.rados.rados.0.smithi136.stdout:3099: finishing write tid 2 to smithi13667081-12 2024-09-12T23:49:01.563 INFO:tasks.rados.rados.0.smithi136.stdout:3098: done (2 left) 2024-09-12T23:49:01.563 INFO:tasks.rados.rados.0.smithi136.stdout:3101: rollback oid 24 current snap is 303 2024-09-12T23:49:01.563 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 24 to 296 2024-09-12T23:49:01.563 INFO:tasks.rados.rados.0.smithi136.stdout:3102: read oid 48 snap -1 2024-09-12T23:49:01.563 INFO:tasks.rados.rados.0.smithi136.stdout:3102: expect (ObjNum 864 snap 281 seq_num 864) 2024-09-12T23:49:01.563 INFO:tasks.rados.rados.0.smithi136.stdout:3103: rollback oid 39 current snap is 303 2024-09-12T23:49:01.563 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 39 to 297 2024-09-12T23:49:01.563 INFO:tasks.rados.rados.0.smithi136.stdout:3104: copy_from oid 2 from oid 38 current snap is 303 2024-09-12T23:49:01.563 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 7 current snap is 303 2024-09-12T23:49:01.564 INFO:tasks.rados.rados.0.smithi136.stdout:3105: seq_num 981 ranges {3866624=434176,4300800=540672,4841472=466944,5308416=524288,5832704=73728} 2024-09-12T23:49:01.587 INFO:tasks.rados.rados.0.smithi136.stdout:3105: writing smithi13667081-7 from 3866624 to 4300800 tid 1 2024-09-12T23:49:01.590 INFO:tasks.rados.rados.0.smithi136.stdout:3105: writing smithi13667081-7 from 4300800 to 4841472 tid 2 2024-09-12T23:49:01.593 INFO:tasks.rados.rados.0.smithi136.stdout:3105: writing smithi13667081-7 from 4841472 to 5308416 tid 3 2024-09-12T23:49:01.596 INFO:tasks.rados.rados.0.smithi136.stdout:3105: writing smithi13667081-7 from 5308416 to 5832704 tid 4 2024-09-12T23:49:01.597 INFO:tasks.rados.rados.0.smithi136.stdout:3105: writing smithi13667081-7 from 5832704 to 5906432 tid 5 2024-09-12T23:49:01.597 INFO:tasks.rados.rados.0.smithi136.stdout:3106: snap_remove snap 299 2024-09-12T23:49:01.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:01 smithi181 bash[67587]: cluster 2024-09-12T23:49:00.580947+0000 mon.a (mon.0) 1085 : cluster [DBG] osdmap e683: 8 total, 8 up, 8 in 2024-09-12T23:49:01.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:01 smithi181 bash[67587]: cluster 2024-09-12T23:49:00.814724+0000 mgr.y (mgr.24661) 337 : cluster [DBG] pgmap v601: 121 pgs: 121 active+clean; 325 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 3.5 MiB/s wr, 15 op/s 2024-09-12T23:49:01.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:01 smithi181 bash[67587]: cluster 2024-09-12T23:49:01.450644+0000 mon.a (mon.0) 1086 : cluster [DBG] osdmap e684: 8 total, 8 up, 8 in 2024-09-12T23:49:02.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:01 smithi136 bash[75164]: cluster 2024-09-12T23:49:00.580947+0000 mon.a (mon.0) 1085 : cluster [DBG] osdmap e683: 8 total, 8 up, 8 in 2024-09-12T23:49:02.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:01 smithi136 bash[75164]: cluster 2024-09-12T23:49:00.814724+0000 mgr.y (mgr.24661) 337 : cluster [DBG] pgmap v601: 121 pgs: 121 active+clean; 325 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 3.5 MiB/s wr, 15 op/s 2024-09-12T23:49:02.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:01 smithi136 bash[75164]: cluster 2024-09-12T23:49:01.450644+0000 mon.a (mon.0) 1086 : cluster [DBG] osdmap e684: 8 total, 8 up, 8 in 2024-09-12T23:49:02.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:01 smithi136 bash[76371]: cluster 2024-09-12T23:49:00.580947+0000 mon.a (mon.0) 1085 : cluster [DBG] osdmap e683: 8 total, 8 up, 8 in 2024-09-12T23:49:02.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:01 smithi136 bash[76371]: cluster 2024-09-12T23:49:00.814724+0000 mgr.y (mgr.24661) 337 : cluster [DBG] pgmap v601: 121 pgs: 121 active+clean; 325 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 3.5 MiB/s wr, 15 op/s 2024-09-12T23:49:02.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:01 smithi136 bash[76371]: cluster 2024-09-12T23:49:01.450644+0000 mon.a (mon.0) 1086 : cluster [DBG] osdmap e684: 8 total, 8 up, 8 in 2024-09-12T23:49:02.594 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:49:02 smithi181 bash[60471]: ts=2024-09-12T23:49:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:49:02.603 INFO:tasks.rados.rados.0.smithi136.stdout:3106: done (7 left) 2024-09-12T23:49:02.603 INFO:tasks.rados.rados.0.smithi136.stdout:3107: rmattr oid 5 current snap is 303 2024-09-12T23:49:02.603 INFO:tasks.rados.rados.0.smithi136.stdout:3107: done (7 left) 2024-09-12T23:49:02.604 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 44 current snap is 303 2024-09-12T23:49:02.604 INFO:tasks.rados.rados.0.smithi136.stdout:3108: seq_num 982 ranges {0=450560,450560=425984,876544=679936,1556480=466944,2023424=32768} 2024-09-12T23:49:02.606 INFO:tasks.rados.rados.0.smithi136.stdout:3108: writing smithi13667081-44 from 0 to 450560 tid 1 2024-09-12T23:49:02.610 INFO:tasks.rados.rados.0.smithi136.stdout:3108: writing smithi13667081-44 from 450560 to 876544 tid 2 2024-09-12T23:49:02.615 INFO:tasks.rados.rados.0.smithi136.stdout:3108: writing smithi13667081-44 from 876544 to 1556480 tid 3 2024-09-12T23:49:02.618 INFO:tasks.rados.rados.0.smithi136.stdout:3108: writing smithi13667081-44 from 1556480 to 2023424 tid 4 2024-09-12T23:49:02.619 INFO:tasks.rados.rados.0.smithi136.stdout:3108: writing smithi13667081-44 from 2023424 to 2056192 tid 5 2024-09-12T23:49:02.619 INFO:tasks.rados.rados.0.smithi136.stdout:3109: read oid 6 snap -1 2024-09-12T23:49:02.619 INFO:tasks.rados.rados.0.smithi136.stdout:3109: expect deleted 2024-09-12T23:49:02.620 INFO:tasks.rados.rados.0.smithi136.stdout:3110: copy_from oid 21 from oid 18 current snap is 303 2024-09-12T23:49:02.620 INFO:tasks.rados.rados.0.smithi136.stdout:3100: finishing copy_from to smithi13667081-32 2024-09-12T23:49:02.620 INFO:tasks.rados.rados.0.smithi136.stdout:3100: got expected ENOENT (src dne) 2024-09-12T23:49:02.620 INFO:tasks.rados.rados.0.smithi136.stdout:3100: finishing copy_from racing read to smithi13667081-32 2024-09-12T23:49:02.620 INFO:tasks.rados.rados.0.smithi136.stdout:3103: finishing rollback tid 1 to smithi13667081-39 2024-09-12T23:49:02.620 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 39 v 871 (ObjNum 33 snap 0 seq_num 4243291280) dirty dne 2024-09-12T23:49:02.620 INFO:tasks.rados.rados.0.smithi136.stdout:3099: finishing write tid 3 to smithi13667081-12 2024-09-12T23:49:02.620 INFO:tasks.rados.rados.0.smithi136.stdout:3099: finishing write tid 4 to smithi13667081-12 2024-09-12T23:49:02.620 INFO:tasks.rados.rados.0.smithi136.stdout:3100: done (9 left) 2024-09-12T23:49:02.620 INFO:tasks.rados.rados.0.smithi136.stdout:3103: done (8 left) 2024-09-12T23:49:02.620 INFO:tasks.rados.rados.0.smithi136.stdout:3111: copy_from oid 43 from oid 31 current snap is 303 2024-09-12T23:49:02.628 INFO:tasks.rados.rados.0.smithi136.stdout:3099: finishing write tid 5 to smithi13667081-12 2024-09-12T23:49:02.628 INFO:tasks.rados.rados.0.smithi136.stdout:3099: finishing write tid 6 to smithi13667081-12 2024-09-12T23:49:02.628 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 12 v 1675 (ObjNum 980 snap 303 seq_num 980) dirty exists 2024-09-12T23:49:02.628 INFO:tasks.rados.rados.0.smithi136.stdout:3099: left oid 12 (ObjNum 980 snap 303 seq_num 980) 2024-09-12T23:49:02.629 INFO:tasks.rados.rados.0.smithi136.stdout:3101: finishing rollback tid 0 to smithi13667081-24 2024-09-12T23:49:02.629 INFO:tasks.rados.rados.0.smithi136.stdout:3101: finishing rollback tid 1 to smithi13667081-24 2024-09-12T23:49:02.629 INFO:tasks.rados.rados.0.smithi136.stdout:3101: finishing rollback tid 2 to smithi13667081-24 2024-09-12T23:49:02.629 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 24 v 1511 (ObjNum 838 snap 267 seq_num 838) dirty exists 2024-09-12T23:49:02.629 INFO:tasks.rados.rados.0.smithi136.stdout:3102: expect (ObjNum 864 snap 281 seq_num 864) 2024-09-12T23:49:02.783 INFO:tasks.rados.rados.0.smithi136.stdout:3104: finishing copy_from racing read to smithi13667081-2 2024-09-12T23:49:02.784 INFO:tasks.rados.rados.0.smithi136.stdout:3099: done (8 left) 2024-09-12T23:49:02.784 INFO:tasks.rados.rados.0.smithi136.stdout:3101: done (7 left) 2024-09-12T23:49:02.784 INFO:tasks.rados.rados.0.smithi136.stdout:3102: done (6 left) 2024-09-12T23:49:02.784 INFO:tasks.rados.rados.0.smithi136.stdout:3112: snap_remove snap 284 2024-09-12T23:49:03.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:02 smithi136 bash[75164]: audit 2024-09-12T23:49:01.989748+0000 mon.a (mon.0) 1087 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:03.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:02 smithi136 bash[75164]: audit 2024-09-12T23:49:01.994947+0000 mon.a (mon.0) 1088 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:03.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:02 smithi136 bash[75164]: cluster 2024-09-12T23:49:02.606071+0000 mon.a (mon.0) 1089 : cluster [DBG] osdmap e685: 8 total, 8 up, 8 in 2024-09-12T23:49:03.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:02 smithi136 bash[75164]: audit 2024-09-12T23:49:02.693763+0000 mon.a (mon.0) 1090 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:03.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:02 smithi136 bash[75164]: audit 2024-09-12T23:49:02.698019+0000 mon.a (mon.0) 1091 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:03.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:02 smithi136 bash[75164]: cluster 2024-09-12T23:49:02.815253+0000 mgr.y (mgr.24661) 338 : cluster [DBG] pgmap v604: 121 pgs: 121 active+clean; 304 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.7 KiB/s rd, 3.9 MiB/s wr, 16 op/s 2024-09-12T23:49:03.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:02 smithi136 bash[76371]: audit 2024-09-12T23:49:01.989748+0000 mon.a (mon.0) 1087 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:03.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:02 smithi136 bash[76371]: audit 2024-09-12T23:49:01.994947+0000 mon.a (mon.0) 1088 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:03.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:02 smithi136 bash[76371]: cluster 2024-09-12T23:49:02.606071+0000 mon.a (mon.0) 1089 : cluster [DBG] osdmap e685: 8 total, 8 up, 8 in 2024-09-12T23:49:03.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:02 smithi136 bash[76371]: audit 2024-09-12T23:49:02.693763+0000 mon.a (mon.0) 1090 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:03.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:02 smithi136 bash[76371]: audit 2024-09-12T23:49:02.698019+0000 mon.a (mon.0) 1091 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:03.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:02 smithi136 bash[76371]: cluster 2024-09-12T23:49:02.815253+0000 mgr.y (mgr.24661) 338 : cluster [DBG] pgmap v604: 121 pgs: 121 active+clean; 304 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.7 KiB/s rd, 3.9 MiB/s wr, 16 op/s 2024-09-12T23:49:03.344 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:02 smithi181 bash[67587]: audit 2024-09-12T23:49:01.989748+0000 mon.a (mon.0) 1087 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:03.344 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:02 smithi181 bash[67587]: audit 2024-09-12T23:49:01.994947+0000 mon.a (mon.0) 1088 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:03.345 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:02 smithi181 bash[67587]: cluster 2024-09-12T23:49:02.606071+0000 mon.a (mon.0) 1089 : cluster [DBG] osdmap e685: 8 total, 8 up, 8 in 2024-09-12T23:49:03.345 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:02 smithi181 bash[67587]: audit 2024-09-12T23:49:02.693763+0000 mon.a (mon.0) 1090 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:03.345 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:02 smithi181 bash[67587]: audit 2024-09-12T23:49:02.698019+0000 mon.a (mon.0) 1091 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:03.345 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:02 smithi181 bash[67587]: cluster 2024-09-12T23:49:02.815253+0000 mgr.y (mgr.24661) 338 : cluster [DBG] pgmap v604: 121 pgs: 121 active+clean; 304 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.7 KiB/s rd, 3.9 MiB/s wr, 16 op/s 2024-09-12T23:49:03.714 INFO:tasks.rados.rados.0.smithi136.stdout:3112: done (6 left) 2024-09-12T23:49:03.714 INFO:tasks.rados.rados.0.smithi136.stdout:3113: rmattr oid 49 current snap is 303 2024-09-12T23:49:03.714 INFO:tasks.rados.rados.0.smithi136.stdout:3113: done (6 left) 2024-09-12T23:49:03.714 INFO:tasks.rados.rados.0.smithi136.stdout:3114: snap_remove snap 297 2024-09-12T23:49:04.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:04 smithi181 bash[67587]: audit 2024-09-12T23:49:03.265812+0000 mon.a (mon.0) 1092 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:04.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:04 smithi181 bash[67587]: audit 2024-09-12T23:49:03.275830+0000 mon.a (mon.0) 1093 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:04.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:04 smithi181 bash[67587]: cluster 2024-09-12T23:49:03.716092+0000 mon.a (mon.0) 1094 : cluster [DBG] osdmap e686: 8 total, 8 up, 8 in 2024-09-12T23:49:04.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:04 smithi181 bash[67587]: audit 2024-09-12T23:49:04.039002+0000 mon.a (mon.0) 1095 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:04.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:04 smithi181 bash[67587]: audit 2024-09-12T23:49:04.047660+0000 mon.a (mon.0) 1096 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:04.715 INFO:tasks.rados.rados.0.smithi136.stdout:3114: done (6 left) 2024-09-12T23:49:04.715 INFO:tasks.rados.rados.0.smithi136.stdout:3115: rollback oid 34 current snap is 303 2024-09-12T23:49:04.715 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 34 to 291 2024-09-12T23:49:04.717 INFO:tasks.rados.rados.0.smithi136.stdout:3104: finishing copy_from to smithi13667081-2 2024-09-12T23:49:04.778 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 2 v 1645 (ObjNum 864 snap 281 seq_num 864) dirty exists 2024-09-12T23:49:04.778 INFO:tasks.rados.rados.0.smithi136.stdout:3105: finishing write tid 1 to smithi13667081-7 2024-09-12T23:49:04.778 INFO:tasks.rados.rados.0.smithi136.stdout:3105: finishing write tid 2 to smithi13667081-7 2024-09-12T23:49:04.778 INFO:tasks.rados.rados.0.smithi136.stdout:3105: finishing write tid 3 to smithi13667081-7 2024-09-12T23:49:04.778 INFO:tasks.rados.rados.0.smithi136.stdout:3105: finishing write tid 4 to smithi13667081-7 2024-09-12T23:49:04.779 INFO:tasks.rados.rados.0.smithi136.stdout:3105: finishing write tid 5 to smithi13667081-7 2024-09-12T23:49:04.779 INFO:tasks.rados.rados.0.smithi136.stdout:3105: finishing write tid 6 to smithi13667081-7 2024-09-12T23:49:04.779 INFO:tasks.rados.rados.0.smithi136.stdout:3105: finishing write tid 7 to smithi13667081-7 2024-09-12T23:49:04.779 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 7 v 985 (ObjNum 981 snap 303 seq_num 981) dirty exists 2024-09-12T23:49:04.779 INFO:tasks.rados.rados.0.smithi136.stdout:3105: left oid 7 (ObjNum 981 snap 303 seq_num 981) 2024-09-12T23:49:04.779 INFO:tasks.rados.rados.0.smithi136.stdout:3108: finishing write tid 1 to smithi13667081-44 2024-09-12T23:49:04.779 INFO:tasks.rados.rados.0.smithi136.stdout:3108: finishing write tid 2 to smithi13667081-44 2024-09-12T23:49:04.779 INFO:tasks.rados.rados.0.smithi136.stdout:3111: finishing copy_from to smithi13667081-43 2024-09-12T23:49:04.779 INFO:tasks.rados.rados.0.smithi136.stdout:3111: got expected ENOENT (src dne) 2024-09-12T23:49:04.779 INFO:tasks.rados.rados.0.smithi136.stdout:3111: finishing copy_from racing read to smithi13667081-43 2024-09-12T23:49:04.779 INFO:tasks.rados.rados.0.smithi136.stdout:3108: finishing write tid 3 to smithi13667081-44 2024-09-12T23:49:04.779 INFO:tasks.rados.rados.0.smithi136.stdout:3108: finishing write tid 4 to smithi13667081-44 2024-09-12T23:49:04.779 INFO:tasks.rados.rados.0.smithi136.stdout:3108: finishing write tid 5 to smithi13667081-44 2024-09-12T23:49:04.780 INFO:tasks.rados.rados.0.smithi136.stdout:3108: finishing write tid 6 to smithi13667081-44 2024-09-12T23:49:04.780 INFO:tasks.rados.rados.0.smithi136.stdout:3110: finishing copy_from racing read to smithi13667081-21 2024-09-12T23:49:04.780 INFO:tasks.rados.rados.0.smithi136.stdout:3110: finishing copy_from to smithi13667081-21 2024-09-12T23:49:04.780 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 21 v 979 (ObjNum 931 snap 292 seq_num 931) dirty exists 2024-09-12T23:49:04.780 INFO:tasks.rados.rados.0.smithi136.stdout:3108: finishing write tid 7 to smithi13667081-44 2024-09-12T23:49:04.780 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 44 v 1936 (ObjNum 982 snap 303 seq_num 982) dirty exists 2024-09-12T23:49:04.780 INFO:tasks.rados.rados.0.smithi136.stdout:3108: left oid 44 (ObjNum 982 snap 303 seq_num 982) 2024-09-12T23:49:04.782 INFO:tasks.rados.rados.0.smithi136.stdout:3104: done (6 left) 2024-09-12T23:49:04.782 INFO:tasks.rados.rados.0.smithi136.stdout:3105: done (5 left) 2024-09-12T23:49:04.782 INFO:tasks.rados.rados.0.smithi136.stdout:3108: done (4 left) 2024-09-12T23:49:04.782 INFO:tasks.rados.rados.0.smithi136.stdout:3109: done (3 left) 2024-09-12T23:49:04.782 INFO:tasks.rados.rados.0.smithi136.stdout:3110: done (2 left) 2024-09-12T23:49:04.782 INFO:tasks.rados.rados.0.smithi136.stdout:3111: done (1 left) 2024-09-12T23:49:04.782 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 40 current snap is 303 2024-09-12T23:49:04.782 INFO:tasks.rados.rados.0.smithi136.stdout:3116: seq_num 983 ranges {1433600=466944,1900544=712704,2613248=434176,3047424=458752} 2024-09-12T23:49:04.782 INFO:tasks.rados.rados.0.smithi136.stdout:3116: writing smithi13667081-40 from 1433600 to 1900544 tid 1 2024-09-12T23:49:04.782 INFO:tasks.rados.rados.0.smithi136.stdout:3116: writing smithi13667081-40 from 1900544 to 2613248 tid 2 2024-09-12T23:49:04.782 INFO:tasks.rados.rados.0.smithi136.stdout:3116: writing smithi13667081-40 from 2613248 to 3047424 tid 3 2024-09-12T23:49:04.782 INFO:tasks.rados.rados.0.smithi136.stdout:3116: writing smithi13667081-40 from 3047424 to 3506176 tid 4 2024-09-12T23:49:04.783 INFO:tasks.rados.rados.0.smithi136.stdout:3117: read oid 19 snap -1 2024-09-12T23:49:04.783 INFO:tasks.rados.rados.0.smithi136.stdout:3117: expect deleted 2024-09-12T23:49:04.783 INFO:tasks.rados.rados.0.smithi136.stdout:3118: delete oid 32 current snap is 303 2024-09-12T23:49:04.783 INFO:tasks.rados.rados.0.smithi136.stdout:3115: finishing rollback tid 0 to smithi13667081-34 2024-09-12T23:49:04.783 INFO:tasks.rados.rados.0.smithi136.stdout:3117: done (3 left) 2024-09-12T23:49:04.783 INFO:tasks.rados.rados.0.smithi136.stdout:3118: done (2 left) 2024-09-12T23:49:04.783 INFO:tasks.rados.rados.0.smithi136.stdout:3119: rollback oid 32 current snap is 303 2024-09-12T23:49:04.783 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 32 to 291 2024-09-12T23:49:04.783 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 43 current snap is 303 2024-09-12T23:49:04.783 INFO:tasks.rados.rados.0.smithi136.stdout:3120: seq_num 985 ranges {1744896=499712,2244608=483328,2727936=450560,3178496=581632,3760128=90112} 2024-09-12T23:49:04.784 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:04 smithi136 bash[76371]: audit 2024-09-12T23:49:03.265812+0000 mon.a (mon.0) 1092 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:04.784 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:04 smithi136 bash[76371]: audit 2024-09-12T23:49:03.275830+0000 mon.a (mon.0) 1093 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:04.784 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:04 smithi136 bash[76371]: cluster 2024-09-12T23:49:03.716092+0000 mon.a (mon.0) 1094 : cluster [DBG] osdmap e686: 8 total, 8 up, 8 in 2024-09-12T23:49:04.784 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:04 smithi136 bash[76371]: audit 2024-09-12T23:49:04.039002+0000 mon.a (mon.0) 1095 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:04.784 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:04 smithi136 bash[76371]: audit 2024-09-12T23:49:04.047660+0000 mon.a (mon.0) 1096 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:04.784 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:04 smithi136 bash[75164]: audit 2024-09-12T23:49:03.265812+0000 mon.a (mon.0) 1092 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:04.784 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:04 smithi136 bash[75164]: audit 2024-09-12T23:49:03.275830+0000 mon.a (mon.0) 1093 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:04.784 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:04 smithi136 bash[75164]: cluster 2024-09-12T23:49:03.716092+0000 mon.a (mon.0) 1094 : cluster [DBG] osdmap e686: 8 total, 8 up, 8 in 2024-09-12T23:49:04.784 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:04 smithi136 bash[75164]: audit 2024-09-12T23:49:04.039002+0000 mon.a (mon.0) 1095 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:04.784 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:04 smithi136 bash[75164]: audit 2024-09-12T23:49:04.047660+0000 mon.a (mon.0) 1096 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:04.788 INFO:tasks.rados.rados.0.smithi136.stdout:3120: writing smithi13667081-43 from 1744896 to 2244608 tid 1 2024-09-12T23:49:04.788 INFO:tasks.rados.rados.0.smithi136.stdout:3120: writing smithi13667081-43 from 2244608 to 2727936 tid 2 2024-09-12T23:49:04.788 INFO:tasks.rados.rados.0.smithi136.stdout:3120: writing smithi13667081-43 from 2727936 to 3178496 tid 3 2024-09-12T23:49:04.788 INFO:tasks.rados.rados.0.smithi136.stdout:3120: writing smithi13667081-43 from 3178496 to 3760128 tid 4 2024-09-12T23:49:04.788 INFO:tasks.rados.rados.0.smithi136.stdout:3120: writing smithi13667081-43 from 3760128 to 3850240 tid 5 2024-09-12T23:49:04.788 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 19 current snap is 303 2024-09-12T23:49:04.788 INFO:tasks.rados.rados.0.smithi136.stdout:3121: seq_num 986 ranges {0=516096,516096=770048,1286144=655360,1941504=180224} 2024-09-12T23:49:04.789 INFO:tasks.rados.rados.0.smithi136.stdout:3121: writing smithi13667081-19 from 0 to 516096 tid 1 2024-09-12T23:49:04.789 INFO:tasks.rados.rados.0.smithi136.stdout:3121: writing smithi13667081-19 from 516096 to 1286144 tid 2 2024-09-12T23:49:04.790 INFO:tasks.rados.rados.0.smithi136.stdout:3121: writing smithi13667081-19 from 1286144 to 1941504 tid 3 2024-09-12T23:49:04.791 INFO:tasks.rados.rados.0.smithi136.stdout:3121: writing smithi13667081-19 from 1941504 to 2121728 tid 4 2024-09-12T23:49:04.791 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 30 current snap is 303 2024-09-12T23:49:04.791 INFO:tasks.rados.rados.0.smithi136.stdout:3116: finishing write tid 1 to smithi13667081-40 2024-09-12T23:49:04.791 INFO:tasks.rados.rados.0.smithi136.stdout:3122: seq_num 987 ranges {3743744=532480,4276224=647168,4923392=417792,5341184=409600,5750784=131072} 2024-09-12T23:49:04.815 INFO:tasks.rados.rados.0.smithi136.stdout:3122: writing smithi13667081-30 from 3743744 to 4276224 tid 1 2024-09-12T23:49:04.819 INFO:tasks.rados.rados.0.smithi136.stdout:3122: writing smithi13667081-30 from 4276224 to 4923392 tid 2 2024-09-12T23:49:04.821 INFO:tasks.rados.rados.0.smithi136.stdout:3122: writing smithi13667081-30 from 4923392 to 5341184 tid 3 2024-09-12T23:49:04.824 INFO:tasks.rados.rados.0.smithi136.stdout:3122: writing smithi13667081-30 from 5341184 to 5750784 tid 4 2024-09-12T23:49:04.825 INFO:tasks.rados.rados.0.smithi136.stdout:3122: writing smithi13667081-30 from 5750784 to 5881856 tid 5 2024-09-12T23:49:04.825 INFO:tasks.rados.rados.0.smithi136.stdout:3123: read oid 50 snap -1 2024-09-12T23:49:04.825 INFO:tasks.rados.rados.0.smithi136.stdout:3123: expect (ObjNum 968 snap 301 seq_num 968) 2024-09-12T23:49:04.826 INFO:tasks.rados.rados.0.smithi136.stdout:3124: rollback oid 42 current snap is 303 2024-09-12T23:49:04.826 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 42 to 295 2024-09-12T23:49:04.826 INFO:tasks.rados.rados.0.smithi136.stdout:3115: finishing rollback tid 1 to smithi13667081-34 2024-09-12T23:49:04.826 INFO:tasks.rados.rados.0.smithi136.stdout:3125: read oid 16 snap -1 2024-09-12T23:49:04.826 INFO:tasks.rados.rados.0.smithi136.stdout:3125: expect (ObjNum 977 snap 303 seq_num 977) 2024-09-12T23:49:04.826 INFO:tasks.rados.rados.0.smithi136.stdout:3126: setattr oid 2 current snap is 303 2024-09-12T23:49:04.826 INFO:tasks.rados.rados.0.smithi136.stdout:3115: finishing rollback tid 2 to smithi13667081-34 2024-09-12T23:49:04.826 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 34 v 2262 (ObjNum 796 snap 253 seq_num 796) dirty exists 2024-09-12T23:49:04.826 INFO:tasks.rados.rados.0.smithi136.stdout:3116: finishing write tid 2 to smithi13667081-40 2024-09-12T23:49:04.826 INFO:tasks.rados.rados.0.smithi136.stdout:3119: finishing rollback tid 1 to smithi13667081-32 2024-09-12T23:49:04.826 INFO:tasks.rados.rados.0.smithi136.stdout:3119: finishing rollback tid 2 to smithi13667081-32 2024-09-12T23:49:04.826 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 32 v 1636 (ObjNum 889 snap 288 seq_num 889) dirty exists 2024-09-12T23:49:04.827 INFO:tasks.rados.rados.0.smithi136.stdout:3116: finishing write tid 3 to smithi13667081-40 2024-09-12T23:49:04.827 INFO:tasks.rados.rados.0.smithi136.stdout:3116: finishing write tid 4 to smithi13667081-40 2024-09-12T23:49:04.827 INFO:tasks.rados.rados.0.smithi136.stdout:3116: finishing write tid 5 to smithi13667081-40 2024-09-12T23:49:04.827 INFO:tasks.rados.rados.0.smithi136.stdout:3120: finishing write tid 1 to smithi13667081-43 2024-09-12T23:49:04.827 INFO:tasks.rados.rados.0.smithi136.stdout:3116: finishing write tid 6 to smithi13667081-40 2024-09-12T23:49:04.827 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 40 v 2432 (ObjNum 983 snap 303 seq_num 983) dirty exists 2024-09-12T23:49:04.827 INFO:tasks.rados.rados.0.smithi136.stdout:3116: left oid 40 (ObjNum 983 snap 303 seq_num 983) 2024-09-12T23:49:04.827 INFO:tasks.rados.rados.0.smithi136.stdout:3120: finishing write tid 2 to smithi13667081-43 2024-09-12T23:49:04.827 INFO:tasks.rados.rados.0.smithi136.stdout:3120: finishing write tid 3 to smithi13667081-43 2024-09-12T23:49:04.827 INFO:tasks.rados.rados.0.smithi136.stdout:3120: finishing write tid 4 to smithi13667081-43 2024-09-12T23:49:04.827 INFO:tasks.rados.rados.0.smithi136.stdout:3120: finishing write tid 5 to smithi13667081-43 2024-09-12T23:49:04.827 INFO:tasks.rados.rados.0.smithi136.stdout:3120: finishing write tid 6 to smithi13667081-43 2024-09-12T23:49:04.827 INFO:tasks.rados.rados.0.smithi136.stdout:3121: finishing write tid 1 to smithi13667081-19 2024-09-12T23:49:04.827 INFO:tasks.rados.rados.0.smithi136.stdout:3120: finishing write tid 7 to smithi13667081-43 2024-09-12T23:49:04.828 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 43 v 1955 (ObjNum 985 snap 303 seq_num 985) dirty exists 2024-09-12T23:49:04.828 INFO:tasks.rados.rados.0.smithi136.stdout:3120: left oid 43 (ObjNum 985 snap 303 seq_num 985) 2024-09-12T23:49:04.828 INFO:tasks.rados.rados.0.smithi136.stdout:3121: finishing write tid 2 to smithi13667081-19 2024-09-12T23:49:04.828 INFO:tasks.rados.rados.0.smithi136.stdout:3121: finishing write tid 3 to smithi13667081-19 2024-09-12T23:49:04.828 INFO:tasks.rados.rados.0.smithi136.stdout:3121: finishing write tid 4 to smithi13667081-19 2024-09-12T23:49:04.828 INFO:tasks.rados.rados.0.smithi136.stdout:3121: finishing write tid 5 to smithi13667081-19 2024-09-12T23:49:04.828 INFO:tasks.rados.rados.0.smithi136.stdout:3121: finishing write tid 6 to smithi13667081-19 2024-09-12T23:49:04.828 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 19 v 1634 (ObjNum 986 snap 303 seq_num 986) dirty exists 2024-09-12T23:49:04.828 INFO:tasks.rados.rados.0.smithi136.stdout:3121: left oid 19 (ObjNum 986 snap 303 seq_num 986) 2024-09-12T23:49:04.828 INFO:tasks.rados.rados.0.smithi136.stdout:3122: finishing write tid 1 to smithi13667081-30 2024-09-12T23:49:04.829 INFO:tasks.rados.rados.0.smithi136.stdout:3115: done (9 left) 2024-09-12T23:49:04.829 INFO:tasks.rados.rados.0.smithi136.stdout:3116: done (8 left) 2024-09-12T23:49:04.829 INFO:tasks.rados.rados.0.smithi136.stdout:3119: done (7 left) 2024-09-12T23:49:04.829 INFO:tasks.rados.rados.0.smithi136.stdout:3120: done (6 left) 2024-09-12T23:49:04.829 INFO:tasks.rados.rados.0.smithi136.stdout:3121: done (5 left) 2024-09-12T23:49:04.829 INFO:tasks.rados.rados.0.smithi136.stdout:3127: read oid 22 snap -1 2024-09-12T23:49:04.829 INFO:tasks.rados.rados.0.smithi136.stdout:3127: expect deleted 2024-09-12T23:49:04.829 INFO:tasks.rados.rados.0.smithi136.stdout:3128: rollback oid 44 current snap is 303 2024-09-12T23:49:04.829 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 44 to 302 2024-09-12T23:49:04.830 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 12 current snap is 303 2024-09-12T23:49:04.830 INFO:tasks.rados.rados.0.smithi136.stdout:3129: seq_num 988 ranges {4964352=557056,5521408=532480,6053888=622592,6676480=450560} 2024-09-12T23:49:04.861 INFO:tasks.rados.rados.0.smithi136.stdout:3129: writing smithi13667081-12 from 4964352 to 5521408 tid 1 2024-09-12T23:49:04.864 INFO:tasks.rados.rados.0.smithi136.stdout:3129: writing smithi13667081-12 from 5521408 to 6053888 tid 2 2024-09-12T23:49:04.869 INFO:tasks.rados.rados.0.smithi136.stdout:3129: writing smithi13667081-12 from 6053888 to 6676480 tid 3 2024-09-12T23:49:04.871 INFO:tasks.rados.rados.0.smithi136.stdout:3129: writing smithi13667081-12 from 6676480 to 7127040 tid 4 2024-09-12T23:49:04.871 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 37 current snap is 303 2024-09-12T23:49:04.871 INFO:tasks.rados.rados.0.smithi136.stdout:3122: finishing write tid 2 to smithi13667081-30 2024-09-12T23:49:04.871 INFO:tasks.rados.rados.0.smithi136.stdout:3130: seq_num 989 ranges {2949120=573440,3522560=466944,3989504=434176,4423680=638976,5062656=65536} 2024-09-12T23:49:04.890 INFO:tasks.rados.rados.0.smithi136.stdout:3130: writing smithi13667081-37 from 2949120 to 3522560 tid 1 2024-09-12T23:49:04.893 INFO:tasks.rados.rados.0.smithi136.stdout:3130: writing smithi13667081-37 from 3522560 to 3989504 tid 2 2024-09-12T23:49:04.896 INFO:tasks.rados.rados.0.smithi136.stdout:3130: writing smithi13667081-37 from 3989504 to 4423680 tid 3 2024-09-12T23:49:04.900 INFO:tasks.rados.rados.0.smithi136.stdout:3130: writing smithi13667081-37 from 4423680 to 5062656 tid 4 2024-09-12T23:49:04.901 INFO:tasks.rados.rados.0.smithi136.stdout:3130: writing smithi13667081-37 from 5062656 to 5128192 tid 5 2024-09-12T23:49:04.901 INFO:tasks.rados.rados.0.smithi136.stdout:3131: rollback oid 26 current snap is 303 2024-09-12T23:49:04.901 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 26 to 302 2024-09-12T23:49:04.901 INFO:tasks.rados.rados.0.smithi136.stdout:3124: finishing rollback tid 0 to smithi13667081-42 2024-09-12T23:49:04.901 INFO:tasks.rados.rados.0.smithi136.stdout:3122: finishing write tid 3 to smithi13667081-30 2024-09-12T23:49:04.901 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 2 v 2263 (ObjNum 864 snap 281 seq_num 864) dirty exists 2024-09-12T23:49:04.901 INFO:tasks.rados.rados.0.smithi136.stdout:3124: finishing rollback tid 1 to smithi13667081-42 2024-09-12T23:49:04.902 INFO:tasks.rados.rados.0.smithi136.stdout:3124: finishing rollback tid 2 to smithi13667081-42 2024-09-12T23:49:04.902 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 42 v 1815 (ObjNum 895 snap 288 seq_num 895) dirty exists 2024-09-12T23:49:04.902 INFO:tasks.rados.rados.0.smithi136.stdout:3125: expect (ObjNum 977 snap 303 seq_num 977) 2024-09-12T23:49:04.999 INFO:tasks.rados.rados.0.smithi136.stdout:3128: finishing rollback tid 0 to smithi13667081-44 2024-09-12T23:49:04.999 INFO:tasks.rados.rados.0.smithi136.stdout:3128: finishing rollback tid 1 to smithi13667081-44 2024-09-12T23:49:04.999 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 44 v 1957 (ObjNum 1793 snap 0 seq_num 4244469760) dirty dne 2024-09-12T23:49:05.000 INFO:tasks.rados.rados.0.smithi136.stdout:3123: expect (ObjNum 968 snap 301 seq_num 968) 2024-09-12T23:49:05.097 INFO:tasks.rados.rados.0.smithi136.stdout:3122: finishing write tid 4 to smithi13667081-30 2024-09-12T23:49:05.097 INFO:tasks.rados.rados.0.smithi136.stdout:3122: finishing write tid 5 to smithi13667081-30 2024-09-12T23:49:05.097 INFO:tasks.rados.rados.0.smithi136.stdout:3122: finishing write tid 6 to smithi13667081-30 2024-09-12T23:49:05.097 INFO:tasks.rados.rados.0.smithi136.stdout:3122: finishing write tid 7 to smithi13667081-30 2024-09-12T23:49:05.097 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 30 v 1504 (ObjNum 987 snap 303 seq_num 987) dirty exists 2024-09-12T23:49:05.097 INFO:tasks.rados.rados.0.smithi136.stdout:3122: left oid 30 (ObjNum 987 snap 303 seq_num 987) 2024-09-12T23:49:05.098 INFO:tasks.rados.rados.0.smithi136.stdout:3129: finishing write tid 1 to smithi13667081-12 2024-09-12T23:49:05.098 INFO:tasks.rados.rados.0.smithi136.stdout:3129: finishing write tid 2 to smithi13667081-12 2024-09-12T23:49:05.098 INFO:tasks.rados.rados.0.smithi136.stdout:3129: finishing write tid 3 to smithi13667081-12 2024-09-12T23:49:05.098 INFO:tasks.rados.rados.0.smithi136.stdout:3129: finishing write tid 4 to smithi13667081-12 2024-09-12T23:49:05.098 INFO:tasks.rados.rados.0.smithi136.stdout:3129: finishing write tid 5 to smithi13667081-12 2024-09-12T23:49:05.098 INFO:tasks.rados.rados.0.smithi136.stdout:3129: finishing write tid 6 to smithi13667081-12 2024-09-12T23:49:05.098 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 12 v 1680 (ObjNum 988 snap 303 seq_num 988) dirty exists 2024-09-12T23:49:05.098 INFO:tasks.rados.rados.0.smithi136.stdout:3129: left oid 12 (ObjNum 988 snap 303 seq_num 988) 2024-09-12T23:49:05.098 INFO:tasks.rados.rados.0.smithi136.stdout:3130: finishing write tid 1 to smithi13667081-37 2024-09-12T23:49:05.098 INFO:tasks.rados.rados.0.smithi136.stdout:3130: finishing write tid 2 to smithi13667081-37 2024-09-12T23:49:05.098 INFO:tasks.rados.rados.0.smithi136.stdout:3130: finishing write tid 3 to smithi13667081-37 2024-09-12T23:49:05.098 INFO:tasks.rados.rados.0.smithi136.stdout:3130: finishing write tid 4 to smithi13667081-37 2024-09-12T23:49:05.098 INFO:tasks.rados.rados.0.smithi136.stdout:3131: finishing rollback tid 0 to smithi13667081-26 2024-09-12T23:49:05.099 INFO:tasks.rados.rados.0.smithi136.stdout:3131: finishing rollback tid 1 to smithi13667081-26 2024-09-12T23:49:05.099 INFO:tasks.rados.rados.0.smithi136.stdout:3130: finishing write tid 5 to smithi13667081-37 2024-09-12T23:49:05.099 INFO:tasks.rados.rados.0.smithi136.stdout:3131: finishing rollback tid 2 to smithi13667081-26 2024-09-12T23:49:05.099 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 26 v 1514 (ObjNum 822 snap 258 seq_num 822) dirty exists 2024-09-12T23:49:05.099 INFO:tasks.rados.rados.0.smithi136.stdout:3130: finishing write tid 6 to smithi13667081-37 2024-09-12T23:49:05.099 INFO:tasks.rados.rados.0.smithi136.stdout:3130: finishing write tid 7 to smithi13667081-37 2024-09-12T23:49:05.099 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 37 v 2269 (ObjNum 989 snap 303 seq_num 989) dirty exists 2024-09-12T23:49:05.099 INFO:tasks.rados.rados.0.smithi136.stdout:3130: left oid 37 (ObjNum 989 snap 303 seq_num 989) 2024-09-12T23:49:05.099 INFO:tasks.rados.rados.0.smithi136.stdout:3122: done (9 left) 2024-09-12T23:49:05.099 INFO:tasks.rados.rados.0.smithi136.stdout:3123: done (8 left) 2024-09-12T23:49:05.099 INFO:tasks.rados.rados.0.smithi136.stdout:3124: done (7 left) 2024-09-12T23:49:05.099 INFO:tasks.rados.rados.0.smithi136.stdout:3125: done (6 left) 2024-09-12T23:49:05.099 INFO:tasks.rados.rados.0.smithi136.stdout:3126: done (5 left) 2024-09-12T23:49:05.099 INFO:tasks.rados.rados.0.smithi136.stdout:3127: done (4 left) 2024-09-12T23:49:05.100 INFO:tasks.rados.rados.0.smithi136.stdout:3128: done (3 left) 2024-09-12T23:49:05.100 INFO:tasks.rados.rados.0.smithi136.stdout:3129: done (2 left) 2024-09-12T23:49:05.100 INFO:tasks.rados.rados.0.smithi136.stdout:3130: done (1 left) 2024-09-12T23:49:05.100 INFO:tasks.rados.rados.0.smithi136.stdout:3131: done (0 left) 2024-09-12T23:49:05.100 INFO:tasks.rados.rados.0.smithi136.stdout:3132: rollback oid 28 current snap is 303 2024-09-12T23:49:05.100 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 28 to 302 2024-09-12T23:49:05.100 INFO:tasks.rados.rados.0.smithi136.stdout:3133: rmattr oid 24 current snap is 303 2024-09-12T23:49:05.100 INFO:tasks.rados.rados.0.smithi136.stdout:3133: done (1 left) 2024-09-12T23:49:05.100 INFO:tasks.rados.rados.0.smithi136.stdout:3134: copy_from oid 48 from oid 39 current snap is 303 2024-09-12T23:49:05.100 INFO:tasks.rados.rados.0.smithi136.stdout:3135: read oid 7 snap -1 2024-09-12T23:49:05.100 INFO:tasks.rados.rados.0.smithi136.stdout:3135: expect (ObjNum 981 snap 303 seq_num 981) 2024-09-12T23:49:05.100 INFO:tasks.rados.rados.0.smithi136.stdout:3136: copy_from oid 25 from oid 40 current snap is 303 2024-09-12T23:49:05.100 INFO:tasks.rados.rados.0.smithi136.stdout:3137: copy_from oid 3 from oid 42 current snap is 303 2024-09-12T23:49:05.100 INFO:tasks.rados.rados.0.smithi136.stdout:3138: snap_remove snap 291 2024-09-12T23:49:05.733 INFO:tasks.rados.rados.0.smithi136.stdout:3134: finishing copy_from to smithi13667081-48 2024-09-12T23:49:05.733 INFO:tasks.rados.rados.0.smithi136.stdout:3134: got expected ENOENT (src dne) 2024-09-12T23:49:05.733 INFO:tasks.rados.rados.0.smithi136.stdout:3134: finishing copy_from racing read to smithi13667081-48 2024-09-12T23:49:05.733 INFO:tasks.rados.rados.0.smithi136.stdout:3132: finishing rollback tid 0 to smithi13667081-28 2024-09-12T23:49:05.733 INFO:tasks.rados.rados.0.smithi136.stdout:3132: finishing rollback tid 1 to smithi13667081-28 2024-09-12T23:49:05.733 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 28 v 1959 (ObjNum 15553 snap 0 seq_num 4244447744) dirty dne 2024-09-12T23:49:05.733 INFO:tasks.rados.rados.0.smithi136.stdout:3135: expect (ObjNum 981 snap 303 seq_num 981) 2024-09-12T23:49:06.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:05 smithi136 bash[76371]: cluster 2024-09-12T23:49:04.717086+0000 mon.a (mon.0) 1097 : cluster [DBG] osdmap e687: 8 total, 8 up, 8 in 2024-09-12T23:49:06.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:05 smithi136 bash[76371]: cluster 2024-09-12T23:49:04.815727+0000 mgr.y (mgr.24661) 339 : cluster [DBG] pgmap v607: 121 pgs: 121 active+clean; 290 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 2.8 MiB/s wr, 12 op/s 2024-09-12T23:49:06.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:05 smithi136 bash[75164]: cluster 2024-09-12T23:49:04.717086+0000 mon.a (mon.0) 1097 : cluster [DBG] osdmap e687: 8 total, 8 up, 8 in 2024-09-12T23:49:06.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:05 smithi136 bash[75164]: cluster 2024-09-12T23:49:04.815727+0000 mgr.y (mgr.24661) 339 : cluster [DBG] pgmap v607: 121 pgs: 121 active+clean; 290 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 2.8 MiB/s wr, 12 op/s 2024-09-12T23:49:06.050 INFO:tasks.rados.rados.0.smithi136.stdout:3136: finishing copy_from racing read to smithi13667081-25 2024-09-12T23:49:06.051 INFO:tasks.rados.rados.0.smithi136.stdout:3136: finishing copy_from to smithi13667081-25 2024-09-12T23:49:06.051 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 25 v 2432 (ObjNum 983 snap 303 seq_num 983) dirty exists 2024-09-12T23:49:06.051 INFO:tasks.rados.rados.0.smithi136.stdout:3137: finishing copy_from racing read to smithi13667081-3 2024-09-12T23:49:06.051 INFO:tasks.rados.rados.0.smithi136.stdout:3137: finishing copy_from to smithi13667081-3 2024-09-12T23:49:06.051 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 3 v 1815 (ObjNum 895 snap 288 seq_num 895) dirty exists 2024-09-12T23:49:06.051 INFO:tasks.rados.rados.0.smithi136.stdout:3132: done (5 left) 2024-09-12T23:49:06.051 INFO:tasks.rados.rados.0.smithi136.stdout:3134: done (4 left) 2024-09-12T23:49:06.051 INFO:tasks.rados.rados.0.smithi136.stdout:3135: done (3 left) 2024-09-12T23:49:06.051 INFO:tasks.rados.rados.0.smithi136.stdout:3136: done (2 left) 2024-09-12T23:49:06.051 INFO:tasks.rados.rados.0.smithi136.stdout:3137: done (1 left) 2024-09-12T23:49:06.051 INFO:tasks.rados.rados.0.smithi136.stdout:3138: done (0 left) 2024-09-12T23:49:06.051 INFO:tasks.rados.rados.0.smithi136.stdout:3139: read oid 27 snap -1 2024-09-12T23:49:06.052 INFO:tasks.rados.rados.0.smithi136.stdout:3139: expect (ObjNum 924 snap 290 seq_num 924) 2024-09-12T23:49:06.052 INFO:tasks.rados.rados.0.smithi136.stdout:3140: copy_from oid 19 from oid 33 current snap is 303 2024-09-12T23:49:06.052 INFO:tasks.rados.rados.0.smithi136.stdout:3141: snap_create 2024-09-12T23:49:06.052 INFO:tasks.rados.rados.0.smithi136.stdout:3140: finishing copy_from to smithi13667081-19 2024-09-12T23:49:06.052 INFO:tasks.rados.rados.0.smithi136.stdout:3140: got expected ENOENT (src dne) 2024-09-12T23:49:06.052 INFO:tasks.rados.rados.0.smithi136.stdout:3140: finishing copy_from racing read to smithi13667081-19 2024-09-12T23:49:06.057 INFO:tasks.rados.rados.0.smithi136.stdout:3139: expect (ObjNum 924 snap 290 seq_num 924) 2024-09-12T23:49:06.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:05 smithi181 bash[67587]: cluster 2024-09-12T23:49:04.717086+0000 mon.a (mon.0) 1097 : cluster [DBG] osdmap e687: 8 total, 8 up, 8 in 2024-09-12T23:49:06.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:05 smithi181 bash[67587]: cluster 2024-09-12T23:49:04.815727+0000 mgr.y (mgr.24661) 339 : cluster [DBG] pgmap v607: 121 pgs: 121 active+clean; 290 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 2.8 MiB/s wr, 12 op/s 2024-09-12T23:49:06.452 INFO:tasks.rados.rados.0.smithi136.stdout:3139: done (2 left) 2024-09-12T23:49:06.452 INFO:tasks.rados.rados.0.smithi136.stdout:3140: done (1 left) 2024-09-12T23:49:06.452 INFO:tasks.rados.rados.0.smithi136.stdout:3141: done (0 left) 2024-09-12T23:49:06.452 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 47 current snap is 304 2024-09-12T23:49:06.452 INFO:tasks.rados.rados.0.smithi136.stdout:3142: seq_num 990 ranges {0=589824,589824=753664,1343488=638976,1982464=212992} 2024-09-12T23:49:06.459 INFO:tasks.rados.rados.0.smithi136.stdout:3142: writing smithi13667081-47 from 0 to 589824 tid 1 2024-09-12T23:49:06.470 INFO:tasks.rados.rados.0.smithi136.stdout:3142: writing smithi13667081-47 from 589824 to 1343488 tid 2 2024-09-12T23:49:06.478 INFO:tasks.rados.rados.0.smithi136.stdout:3142: writing smithi13667081-47 from 1343488 to 1982464 tid 3 2024-09-12T23:49:06.481 INFO:tasks.rados.rados.0.smithi136.stdout:3142: writing smithi13667081-47 from 1982464 to 2195456 tid 4 2024-09-12T23:49:06.482 INFO:tasks.rados.rados.0.smithi136.stdout:3143: snap_remove snap 302 2024-09-12T23:49:07.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:06 smithi136 bash[75164]: cluster 2024-09-12T23:49:05.728179+0000 mon.a (mon.0) 1098 : cluster [DBG] osdmap e688: 8 total, 8 up, 8 in 2024-09-12T23:49:07.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:06 smithi136 bash[75164]: cluster 2024-09-12T23:49:06.452393+0000 mon.a (mon.0) 1099 : cluster [DBG] osdmap e689: 8 total, 8 up, 8 in 2024-09-12T23:49:07.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:06 smithi136 bash[76371]: cluster 2024-09-12T23:49:05.728179+0000 mon.a (mon.0) 1098 : cluster [DBG] osdmap e688: 8 total, 8 up, 8 in 2024-09-12T23:49:07.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:06 smithi136 bash[76371]: cluster 2024-09-12T23:49:06.452393+0000 mon.a (mon.0) 1099 : cluster [DBG] osdmap e689: 8 total, 8 up, 8 in 2024-09-12T23:49:07.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:06 smithi181 bash[67587]: cluster 2024-09-12T23:49:05.728179+0000 mon.a (mon.0) 1098 : cluster [DBG] osdmap e688: 8 total, 8 up, 8 in 2024-09-12T23:49:07.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:06 smithi181 bash[67587]: cluster 2024-09-12T23:49:06.452393+0000 mon.a (mon.0) 1099 : cluster [DBG] osdmap e689: 8 total, 8 up, 8 in 2024-09-12T23:49:07.453 INFO:tasks.rados.rados.0.smithi136.stdout:3143: done (1 left) 2024-09-12T23:49:07.453 INFO:tasks.rados.rados.0.smithi136.stdout:3144: read oid 5 snap -1 2024-09-12T23:49:07.454 INFO:tasks.rados.rados.0.smithi136.stdout:3144: expect (ObjNum 965 snap 301 seq_num 965) 2024-09-12T23:49:07.454 INFO:tasks.rados.rados.0.smithi136.stdout:3145: copy_from oid 26 from oid 30 current snap is 304 2024-09-12T23:49:07.454 INFO:tasks.rados.rados.0.smithi136.stdout:3146: delete oid 50 current snap is 304 2024-09-12T23:49:07.454 INFO:tasks.rados.rados.0.smithi136.stdout:3142: finishing write tid 1 to smithi13667081-47 2024-09-12T23:49:07.454 INFO:tasks.rados.rados.0.smithi136.stdout:3142: finishing write tid 2 to smithi13667081-47 2024-09-12T23:49:07.454 INFO:tasks.rados.rados.0.smithi136.stdout:3142: finishing write tid 3 to smithi13667081-47 2024-09-12T23:49:07.454 INFO:tasks.rados.rados.0.smithi136.stdout:3142: finishing write tid 4 to smithi13667081-47 2024-09-12T23:49:07.454 INFO:tasks.rados.rados.0.smithi136.stdout:3142: finishing write tid 5 to smithi13667081-47 2024-09-12T23:49:07.454 INFO:tasks.rados.rados.0.smithi136.stdout:3142: finishing write tid 6 to smithi13667081-47 2024-09-12T23:49:07.454 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 47 v 1639 (ObjNum 990 snap 304 seq_num 990) dirty exists 2024-09-12T23:49:07.454 INFO:tasks.rados.rados.0.smithi136.stdout:3142: left oid 47 (ObjNum 990 snap 304 seq_num 990) 2024-09-12T23:49:07.475 INFO:tasks.rados.rados.0.smithi136.stdout:3142: done (3 left) 2024-09-12T23:49:07.475 INFO:tasks.rados.rados.0.smithi136.stdout:3146: done (2 left) 2024-09-12T23:49:07.475 INFO:tasks.rados.rados.0.smithi136.stdout:3147: snap_create 2024-09-12T23:49:07.478 INFO:tasks.rados.rados.0.smithi136.stdout:3144: expect (ObjNum 965 snap 301 seq_num 965) 2024-09-12T23:49:07.532 INFO:tasks.rados.rados.0.smithi136.stdout:3145: finishing copy_from racing read to smithi13667081-26 2024-09-12T23:49:07.532 INFO:tasks.rados.rados.0.smithi136.stdout:3145: finishing copy_from to smithi13667081-26 2024-09-12T23:49:07.532 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 26 v 1504 (ObjNum 987 snap 303 seq_num 987) dirty exists 2024-09-12T23:49:08.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:07 smithi136 bash[75164]: cluster 2024-09-12T23:49:06.816649+0000 mgr.y (mgr.24661) 340 : cluster [DBG] pgmap v610: 121 pgs: 121 active+clean; 276 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.8 MiB/s rd, 8.4 MiB/s wr, 32 op/s 2024-09-12T23:49:08.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:07 smithi136 bash[75164]: cluster 2024-09-12T23:49:07.455540+0000 mon.a (mon.0) 1100 : cluster [DBG] osdmap e690: 8 total, 8 up, 8 in 2024-09-12T23:49:08.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:07 smithi136 bash[76371]: cluster 2024-09-12T23:49:06.816649+0000 mgr.y (mgr.24661) 340 : cluster [DBG] pgmap v610: 121 pgs: 121 active+clean; 276 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.8 MiB/s rd, 8.4 MiB/s wr, 32 op/s 2024-09-12T23:49:08.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:07 smithi136 bash[76371]: cluster 2024-09-12T23:49:07.455540+0000 mon.a (mon.0) 1100 : cluster [DBG] osdmap e690: 8 total, 8 up, 8 in 2024-09-12T23:49:08.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:07 smithi181 bash[67587]: cluster 2024-09-12T23:49:06.816649+0000 mgr.y (mgr.24661) 340 : cluster [DBG] pgmap v610: 121 pgs: 121 active+clean; 276 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.8 MiB/s rd, 8.4 MiB/s wr, 32 op/s 2024-09-12T23:49:08.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:07 smithi181 bash[67587]: cluster 2024-09-12T23:49:07.455540+0000 mon.a (mon.0) 1100 : cluster [DBG] osdmap e690: 8 total, 8 up, 8 in 2024-09-12T23:49:08.458 INFO:tasks.rados.rados.0.smithi136.stdout:3144: done (2 left) 2024-09-12T23:49:08.458 INFO:tasks.rados.rados.0.smithi136.stdout:3145: done (1 left) 2024-09-12T23:49:08.458 INFO:tasks.rados.rados.0.smithi136.stdout:3147: done (0 left) 2024-09-12T23:49:08.458 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 29 current snap is 305 2024-09-12T23:49:08.458 INFO:tasks.rados.rados.0.smithi136.stdout:3148: seq_num 992 ranges {20652032=622592,21274624=524288,21798912=606208,22405120=475136} 2024-09-12T23:49:08.631 INFO:tasks.rados.rados.0.smithi136.stdout:3148: writing smithi13667081-29 from 20652032 to 21274624 tid 1 2024-09-12T23:49:08.633 INFO:tasks.rados.rados.0.smithi136.stdout:3148: writing smithi13667081-29 from 21274624 to 21798912 tid 2 2024-09-12T23:49:08.637 INFO:tasks.rados.rados.0.smithi136.stdout:3148: writing smithi13667081-29 from 21798912 to 22405120 tid 3 2024-09-12T23:49:08.640 INFO:tasks.rados.rados.0.smithi136.stdout:3148: writing smithi13667081-29 from 22405120 to 22880256 tid 4 2024-09-12T23:49:08.640 INFO:tasks.rados.rados.0.smithi136.stdout:3149: read oid 15 snap -1 2024-09-12T23:49:08.641 INFO:tasks.rados.rados.0.smithi136.stdout:3149: expect (ObjNum 859 snap 278 seq_num 859) 2024-09-12T23:49:08.641 INFO:tasks.rados.rados.0.smithi136.stdout:3150: snap_create 2024-09-12T23:49:08.656 INFO:tasks.rados.rados.0.smithi136.stdout:3148: finishing write tid 1 to smithi13667081-29 2024-09-12T23:49:08.656 INFO:tasks.rados.rados.0.smithi136.stdout:3148: finishing write tid 2 to smithi13667081-29 2024-09-12T23:49:08.656 INFO:tasks.rados.rados.0.smithi136.stdout:3148: finishing write tid 3 to smithi13667081-29 2024-09-12T23:49:08.657 INFO:tasks.rados.rados.0.smithi136.stdout:3148: finishing write tid 4 to smithi13667081-29 2024-09-12T23:49:08.661 INFO:tasks.rados.rados.0.smithi136.stdout:3148: finishing write tid 5 to smithi13667081-29 2024-09-12T23:49:08.663 INFO:tasks.rados.rados.0.smithi136.stdout:3148: finishing write tid 6 to smithi13667081-29 2024-09-12T23:49:08.663 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 29 v 2286 (ObjNum 992 snap 305 seq_num 992) dirty exists 2024-09-12T23:49:08.663 INFO:tasks.rados.rados.0.smithi136.stdout:3148: left oid 29 (ObjNum 992 snap 305 seq_num 992) 2024-09-12T23:49:08.743 INFO:tasks.rados.rados.0.smithi136.stdout:3149: expect (ObjNum 859 snap 278 seq_num 859) 2024-09-12T23:49:09.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:49:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:49:09] "GET /metrics HTTP/1.1" 200 36438 "" "Prometheus/2.43.0" 2024-09-12T23:49:09.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:09 smithi136 bash[75164]: cluster 2024-09-12T23:49:08.459875+0000 mon.a (mon.0) 1101 : cluster [DBG] osdmap e691: 8 total, 8 up, 8 in 2024-09-12T23:49:09.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:09 smithi136 bash[75164]: audit 2024-09-12T23:49:08.750994+0000 mon.a (mon.0) 1102 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:49:09.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:09 smithi136 bash[75164]: cluster 2024-09-12T23:49:08.817567+0000 mgr.y (mgr.24661) 341 : cluster [DBG] pgmap v613: 121 pgs: 121 active+clean; 238 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.7 MiB/s rd, 8.1 MiB/s wr, 29 op/s 2024-09-12T23:49:09.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:09 smithi136 bash[76371]: cluster 2024-09-12T23:49:08.459875+0000 mon.a (mon.0) 1101 : cluster [DBG] osdmap e691: 8 total, 8 up, 8 in 2024-09-12T23:49:09.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:09 smithi136 bash[76371]: audit 2024-09-12T23:49:08.750994+0000 mon.a (mon.0) 1102 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:49:09.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:09 smithi136 bash[76371]: cluster 2024-09-12T23:49:08.817567+0000 mgr.y (mgr.24661) 341 : cluster [DBG] pgmap v613: 121 pgs: 121 active+clean; 238 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.7 MiB/s rd, 8.1 MiB/s wr, 29 op/s 2024-09-12T23:49:09.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:09 smithi181 bash[67587]: cluster 2024-09-12T23:49:08.459875+0000 mon.a (mon.0) 1101 : cluster [DBG] osdmap e691: 8 total, 8 up, 8 in 2024-09-12T23:49:09.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:09 smithi181 bash[67587]: audit 2024-09-12T23:49:08.750994+0000 mon.a (mon.0) 1102 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:49:09.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:09 smithi181 bash[67587]: cluster 2024-09-12T23:49:08.817567+0000 mgr.y (mgr.24661) 341 : cluster [DBG] pgmap v613: 121 pgs: 121 active+clean; 238 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.7 MiB/s rd, 8.1 MiB/s wr, 29 op/s 2024-09-12T23:49:10.535 INFO:tasks.rados.rados.0.smithi136.stdout:3148: done (2 left) 2024-09-12T23:49:10.536 INFO:tasks.rados.rados.0.smithi136.stdout:3149: done (1 left) 2024-09-12T23:49:10.536 INFO:tasks.rados.rados.0.smithi136.stdout:3150: done (0 left) 2024-09-12T23:49:10.536 INFO:tasks.rados.rados.0.smithi136.stdout:3151: snap_create 2024-09-12T23:49:10.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:10 smithi136 bash[75164]: cluster 2024-09-12T23:49:09.466961+0000 mon.a (mon.0) 1103 : cluster [DBG] osdmap e692: 8 total, 8 up, 8 in 2024-09-12T23:49:10.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:10 smithi136 bash[76371]: cluster 2024-09-12T23:49:09.466961+0000 mon.a (mon.0) 1103 : cluster [DBG] osdmap e692: 8 total, 8 up, 8 in 2024-09-12T23:49:10.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:10 smithi181 bash[67587]: cluster 2024-09-12T23:49:09.466961+0000 mon.a (mon.0) 1103 : cluster [DBG] osdmap e692: 8 total, 8 up, 8 in 2024-09-12T23:49:11.452 INFO:tasks.rados.rados.0.smithi136.stdout:3151: done (0 left) 2024-09-12T23:49:11.452 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 15 current snap is 307 2024-09-12T23:49:11.452 INFO:tasks.rados.rados.0.smithi136.stdout:3152: seq_num 993 ranges {17186816=638976,17825792=409600,18235392=409600,18644992=688128,19333120=98304} 2024-09-12T23:49:11.569 INFO:tasks.rados.rados.0.smithi136.stdout:3152: writing smithi13667081-15 from 17186816 to 17825792 tid 1 2024-09-12T23:49:11.572 INFO:tasks.rados.rados.0.smithi136.stdout:3152: writing smithi13667081-15 from 17825792 to 18235392 tid 2 2024-09-12T23:49:11.574 INFO:tasks.rados.rados.0.smithi136.stdout:3152: writing smithi13667081-15 from 18235392 to 18644992 tid 3 2024-09-12T23:49:11.579 INFO:tasks.rados.rados.0.smithi136.stdout:3152: writing smithi13667081-15 from 18644992 to 19333120 tid 4 2024-09-12T23:49:11.580 INFO:tasks.rados.rados.0.smithi136.stdout:3152: writing smithi13667081-15 from 19333120 to 19431424 tid 5 2024-09-12T23:49:11.580 INFO:tasks.rados.rados.0.smithi136.stdout:3153: rollback oid 3 current snap is 307 2024-09-12T23:49:11.580 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 3 to 295 2024-09-12T23:49:11.580 INFO:tasks.rados.rados.0.smithi136.stdout:3154: read oid 50 snap 305 2024-09-12T23:49:11.580 INFO:tasks.rados.rados.0.smithi136.stdout:3154: expect deleted 2024-09-12T23:49:11.580 INFO:tasks.rados.rados.0.smithi136.stdout:3155: snap_create 2024-09-12T23:49:11.584 INFO:tasks.rados.rados.0.smithi136.stdout:3152: finishing write tid 1 to smithi13667081-15 2024-09-12T23:49:11.584 INFO:tasks.rados.rados.0.smithi136.stdout:3152: finishing write tid 2 to smithi13667081-15 2024-09-12T23:49:11.592 INFO:tasks.rados.rados.0.smithi136.stdout:3152: finishing write tid 3 to smithi13667081-15 2024-09-12T23:49:11.592 INFO:tasks.rados.rados.0.smithi136.stdout:3152: finishing write tid 4 to smithi13667081-15 2024-09-12T23:49:11.592 INFO:tasks.rados.rados.0.smithi136.stdout:3152: finishing write tid 5 to smithi13667081-15 2024-09-12T23:49:11.593 INFO:tasks.rados.rados.0.smithi136.stdout:3152: finishing write tid 6 to smithi13667081-15 2024-09-12T23:49:11.593 INFO:tasks.rados.rados.0.smithi136.stdout:3153: finishing rollback tid 0 to smithi13667081-3 2024-09-12T23:49:11.593 INFO:tasks.rados.rados.0.smithi136.stdout:3153: finishing rollback tid 1 to smithi13667081-3 2024-09-12T23:49:11.593 INFO:tasks.rados.rados.0.smithi136.stdout:3153: finishing rollback tid 2 to smithi13667081-3 2024-09-12T23:49:11.593 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 3 v 1824 (ObjNum 838 snap 267 seq_num 838) dirty exists 2024-09-12T23:49:11.595 INFO:tasks.rados.rados.0.smithi136.stdout:3152: finishing write tid 7 to smithi13667081-15 2024-09-12T23:49:11.596 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 15 v 1821 (ObjNum 993 snap 307 seq_num 993) dirty exists 2024-09-12T23:49:11.596 INFO:tasks.rados.rados.0.smithi136.stdout:3152: left oid 15 (ObjNum 993 snap 307 seq_num 993) 2024-09-12T23:49:11.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:11 smithi136 bash[75164]: cluster 2024-09-12T23:49:10.818607+0000 mgr.y (mgr.24661) 342 : cluster [DBG] pgmap v615: 121 pgs: 121 active+clean; 242 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.9 MiB/s rd, 7.2 MiB/s wr, 28 op/s 2024-09-12T23:49:11.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:11 smithi136 bash[75164]: cluster 2024-09-12T23:49:11.450978+0000 mon.a (mon.0) 1104 : cluster [DBG] osdmap e693: 8 total, 8 up, 8 in 2024-09-12T23:49:11.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:11 smithi136 bash[76371]: cluster 2024-09-12T23:49:10.818607+0000 mgr.y (mgr.24661) 342 : cluster [DBG] pgmap v615: 121 pgs: 121 active+clean; 242 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.9 MiB/s rd, 7.2 MiB/s wr, 28 op/s 2024-09-12T23:49:11.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:11 smithi136 bash[76371]: cluster 2024-09-12T23:49:11.450978+0000 mon.a (mon.0) 1104 : cluster [DBG] osdmap e693: 8 total, 8 up, 8 in 2024-09-12T23:49:11.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:11 smithi181 bash[67587]: cluster 2024-09-12T23:49:10.818607+0000 mgr.y (mgr.24661) 342 : cluster [DBG] pgmap v615: 121 pgs: 121 active+clean; 242 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.9 MiB/s rd, 7.2 MiB/s wr, 28 op/s 2024-09-12T23:49:11.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:11 smithi181 bash[67587]: cluster 2024-09-12T23:49:11.450978+0000 mon.a (mon.0) 1104 : cluster [DBG] osdmap e693: 8 total, 8 up, 8 in 2024-09-12T23:49:12.477 INFO:tasks.rados.rados.0.smithi136.stdout:3152: done (3 left) 2024-09-12T23:49:12.478 INFO:tasks.rados.rados.0.smithi136.stdout:3153: done (2 left) 2024-09-12T23:49:12.478 INFO:tasks.rados.rados.0.smithi136.stdout:3154: done (1 left) 2024-09-12T23:49:12.478 INFO:tasks.rados.rados.0.smithi136.stdout:3155: done (0 left) 2024-09-12T23:49:12.478 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 7 current snap is 308 2024-09-12T23:49:12.478 INFO:tasks.rados.rados.0.smithi136.stdout:3156: seq_num 994 ranges {5906432=655360,6561792=688128,7249920=622592,7872512=294912} 2024-09-12T23:49:12.515 INFO:tasks.rados.rados.0.smithi136.stdout:3156: writing smithi13667081-7 from 5906432 to 6561792 tid 1 2024-09-12T23:49:12.519 INFO:tasks.rados.rados.0.smithi136.stdout:3156: writing smithi13667081-7 from 6561792 to 7249920 tid 2 2024-09-12T23:49:12.522 INFO:tasks.rados.rados.0.smithi136.stdout:3156: writing smithi13667081-7 from 7249920 to 7872512 tid 3 2024-09-12T23:49:12.524 INFO:tasks.rados.rados.0.smithi136.stdout:3156: writing smithi13667081-7 from 7872512 to 8167424 tid 4 2024-09-12T23:49:12.524 INFO:tasks.rados.rados.0.smithi136.stdout:3157: rollback oid 44 current snap is 308 2024-09-12T23:49:12.525 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 44 to 304 2024-09-12T23:49:12.525 INFO:tasks.rados.rados.0.smithi136.stdout:3156: finishing write tid 1 to smithi13667081-7 2024-09-12T23:49:12.525 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 42 current snap is 308 2024-09-12T23:49:12.525 INFO:tasks.rados.rados.0.smithi136.stdout:3156: finishing write tid 2 to smithi13667081-7 2024-09-12T23:49:12.525 INFO:tasks.rados.rados.0.smithi136.stdout:3158: seq_num 995 ranges {10174464=671744,10846208=573440,11419648=778240,12197888=253952} 2024-09-12T23:49:12.585 INFO:tasks.rados.rados.0.smithi136.stdout:3158: writing smithi13667081-42 from 10174464 to 10846208 tid 1 2024-09-12T23:49:12.589 INFO:tasks.rados.rados.0.smithi136.stdout:3158: writing smithi13667081-42 from 10846208 to 11419648 tid 2 2024-09-12T23:49:12.594 INFO:tasks.rados.rados.0.smithi136.stdout:3158: writing smithi13667081-42 from 11419648 to 12197888 tid 3 2024-09-12T23:49:12.594 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:49:12 smithi181 bash[60471]: ts=2024-09-12T23:49:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:49:12.596 INFO:tasks.rados.rados.0.smithi136.stdout:3158: writing smithi13667081-42 from 12197888 to 12451840 tid 4 2024-09-12T23:49:12.596 INFO:tasks.rados.rados.0.smithi136.stdout:3159: rollback oid 9 current snap is 308 2024-09-12T23:49:12.596 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 9 to 304 2024-09-12T23:49:12.596 INFO:tasks.rados.rados.0.smithi136.stdout:3157: finishing rollback tid 1 to smithi13667081-44 2024-09-12T23:49:12.597 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 44 v 0 (ObjNum 65 snap 0 seq_num 1750947947) dirty dne 2024-09-12T23:49:12.597 INFO:tasks.rados.rados.0.smithi136.stdout:3156: finishing write tid 3 to smithi13667081-7 2024-09-12T23:49:12.597 INFO:tasks.rados.rados.0.smithi136.stdout:3156: finishing write tid 4 to smithi13667081-7 2024-09-12T23:49:12.597 INFO:tasks.rados.rados.0.smithi136.stdout:3156: finishing write tid 5 to smithi13667081-7 2024-09-12T23:49:12.597 INFO:tasks.rados.rados.0.smithi136.stdout:3156: finishing write tid 6 to smithi13667081-7 2024-09-12T23:49:12.597 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 7 v 990 (ObjNum 994 snap 308 seq_num 994) dirty exists 2024-09-12T23:49:12.597 INFO:tasks.rados.rados.0.smithi136.stdout:3156: left oid 7 (ObjNum 994 snap 308 seq_num 994) 2024-09-12T23:49:12.597 INFO:tasks.rados.rados.0.smithi136.stdout:3158: finishing write tid 1 to smithi13667081-42 2024-09-12T23:49:12.597 INFO:tasks.rados.rados.0.smithi136.stdout:3158: finishing write tid 2 to smithi13667081-42 2024-09-12T23:49:12.597 INFO:tasks.rados.rados.0.smithi136.stdout:3156: done (3 left) 2024-09-12T23:49:12.597 INFO:tasks.rados.rados.0.smithi136.stdout:3157: done (2 left) 2024-09-12T23:49:12.597 INFO:tasks.rados.rados.0.smithi136.stdout:3160: read oid 14 snap -1 2024-09-12T23:49:12.597 INFO:tasks.rados.rados.0.smithi136.stdout:3160: expect deleted 2024-09-12T23:49:12.597 INFO:tasks.rados.rados.0.smithi136.stdout:3161: snap_create 2024-09-12T23:49:12.603 INFO:tasks.rados.rados.0.smithi136.stdout:3158: finishing write tid 3 to smithi13667081-42 2024-09-12T23:49:12.603 INFO:tasks.rados.rados.0.smithi136.stdout:3158: finishing write tid 4 to smithi13667081-42 2024-09-12T23:49:12.603 INFO:tasks.rados.rados.0.smithi136.stdout:3159: finishing rollback tid 0 to smithi13667081-9 2024-09-12T23:49:12.603 INFO:tasks.rados.rados.0.smithi136.stdout:3158: finishing write tid 5 to smithi13667081-42 2024-09-12T23:49:12.603 INFO:tasks.rados.rados.0.smithi136.stdout:3159: finishing rollback tid 1 to smithi13667081-9 2024-09-12T23:49:12.603 INFO:tasks.rados.rados.0.smithi136.stdout:3159: finishing rollback tid 2 to smithi13667081-9 2024-09-12T23:49:12.603 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 9 v 1642 (ObjNum 963 snap 301 seq_num 963) dirty exists 2024-09-12T23:49:12.605 INFO:tasks.rados.rados.0.smithi136.stdout:3158: finishing write tid 6 to smithi13667081-42 2024-09-12T23:49:12.605 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 42 v 1820 (ObjNum 995 snap 308 seq_num 995) dirty exists 2024-09-12T23:49:12.605 INFO:tasks.rados.rados.0.smithi136.stdout:3158: left oid 42 (ObjNum 995 snap 308 seq_num 995) 2024-09-12T23:49:13.483 INFO:tasks.rados.rados.0.smithi136.stdout:3158: done (3 left) 2024-09-12T23:49:13.483 INFO:tasks.rados.rados.0.smithi136.stdout:3159: done (2 left) 2024-09-12T23:49:13.483 INFO:tasks.rados.rados.0.smithi136.stdout:3160: done (1 left) 2024-09-12T23:49:13.483 INFO:tasks.rados.rados.0.smithi136.stdout:3161: done (0 left) 2024-09-12T23:49:13.483 INFO:tasks.rados.rados.0.smithi136.stdout:3162: snap_remove snap 300 2024-09-12T23:49:13.649 INFO:teuthology.orchestra.run.smithi136.stdout:true 2024-09-12T23:49:13.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:13 smithi136 bash[75164]: cluster 2024-09-12T23:49:12.479087+0000 mon.a (mon.0) 1105 : cluster [DBG] osdmap e694: 8 total, 8 up, 8 in 2024-09-12T23:49:13.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:13 smithi136 bash[75164]: audit 2024-09-12T23:49:12.625744+0000 mon.a (mon.0) 1106 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:13.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:13 smithi136 bash[75164]: audit 2024-09-12T23:49:12.632163+0000 mon.a (mon.0) 1107 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:13.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:13 smithi136 bash[75164]: cluster 2024-09-12T23:49:12.819463+0000 mgr.y (mgr.24661) 343 : cluster [DBG] pgmap v618: 121 pgs: 121 active+clean; 218 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 2.7 MiB/s wr, 9 op/s 2024-09-12T23:49:13.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:13 smithi136 bash[76371]: cluster 2024-09-12T23:49:12.479087+0000 mon.a (mon.0) 1105 : cluster [DBG] osdmap e694: 8 total, 8 up, 8 in 2024-09-12T23:49:13.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:13 smithi136 bash[76371]: audit 2024-09-12T23:49:12.625744+0000 mon.a (mon.0) 1106 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:13.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:13 smithi136 bash[76371]: audit 2024-09-12T23:49:12.632163+0000 mon.a (mon.0) 1107 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:13.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:13 smithi136 bash[76371]: cluster 2024-09-12T23:49:12.819463+0000 mgr.y (mgr.24661) 343 : cluster [DBG] pgmap v618: 121 pgs: 121 active+clean; 218 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 2.7 MiB/s wr, 9 op/s 2024-09-12T23:49:13.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:13 smithi181 bash[67587]: cluster 2024-09-12T23:49:12.479087+0000 mon.a (mon.0) 1105 : cluster [DBG] osdmap e694: 8 total, 8 up, 8 in 2024-09-12T23:49:13.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:13 smithi181 bash[67587]: audit 2024-09-12T23:49:12.625744+0000 mon.a (mon.0) 1106 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:13.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:13 smithi181 bash[67587]: audit 2024-09-12T23:49:12.632163+0000 mon.a (mon.0) 1107 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:13.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:13 smithi181 bash[67587]: cluster 2024-09-12T23:49:12.819463+0000 mgr.y (mgr.24661) 343 : cluster [DBG] pgmap v618: 121 pgs: 121 active+clean; 218 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 2.7 MiB/s wr, 9 op/s 2024-09-12T23:49:13.970 INFO:teuthology.orchestra.run.smithi136.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-12T23:49:13.970 INFO:teuthology.orchestra.run.smithi136.stdout:alertmanager.a smithi136 *:9093,9094 running (8m) 11s ago 14m 14.6M - 0.25.0 c8568f914cd2 b5197f020eeb 2024-09-12T23:49:13.970 INFO:teuthology.orchestra.run.smithi136.stdout:grafana.a smithi181 *:3000 running (8m) 10s ago 14m 75.6M - 9.4.7 954c08fa6188 256c181be984 2024-09-12T23:49:13.970 INFO:teuthology.orchestra.run.smithi136.stdout:mds.a smithi136 running (75s) 11s ago 15m 13.3M - 18.2.4-837-g7a853b17 f21ba1eea74e a20ac830342c 2024-09-12T23:49:13.970 INFO:teuthology.orchestra.run.smithi136.stdout:mds.b smithi181 running (42s) 10s ago 15m 11.7M - 18.2.4-837-g7a853b17 f21ba1eea74e 0dc9f76cf0b8 2024-09-12T23:49:13.970 INFO:teuthology.orchestra.run.smithi136.stdout:mgr.x smithi181 *:8443,9283,8765 running (7m) 10s ago 23m 428M - 18.2.4-837-g7a853b17 f21ba1eea74e cc664dc337f7 2024-09-12T23:49:13.970 INFO:teuthology.orchestra.run.smithi136.stdout:mgr.y smithi136 *:8443,9283,8765 running (8m) 11s ago 25m 496M - 18.2.4-837-g7a853b17 f21ba1eea74e 272f10b42fc3 2024-09-12T23:49:13.970 INFO:teuthology.orchestra.run.smithi136.stdout:mon.a smithi136 running (7m) 11s ago 25m 70.3M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e ad8d21d58ad9 2024-09-12T23:49:13.970 INFO:teuthology.orchestra.run.smithi136.stdout:mon.b smithi181 running (6m) 10s ago 23m 52.2M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e a70ec8315934 2024-09-12T23:49:13.970 INFO:teuthology.orchestra.run.smithi136.stdout:mon.c smithi136 running (6m) 11s ago 23m 58.9M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e 44d1be055a5e 2024-09-12T23:49:13.970 INFO:teuthology.orchestra.run.smithi136.stdout:node-exporter.a smithi136 *:9100 running (21s) 11s ago 14m 7063k - 1.5.0 0da6a335fe13 348c0ab0964c 2024-09-12T23:49:13.970 INFO:teuthology.orchestra.run.smithi136.stdout:node-exporter.b smithi181 *:9100 running (17s) 10s ago 14m 3164k - 1.5.0 0da6a335fe13 3af01e08b5af 2024-09-12T23:49:13.971 INFO:teuthology.orchestra.run.smithi136.stdout:osd.0 smithi136 running (5m) 11s ago 22m 362M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 7da21f765721 2024-09-12T23:49:13.971 INFO:teuthology.orchestra.run.smithi136.stdout:osd.1 smithi136 running (4m) 11s ago 21m 375M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 5302be0a016b 2024-09-12T23:49:13.971 INFO:teuthology.orchestra.run.smithi136.stdout:osd.2 smithi136 running (4m) 11s ago 20m 359M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 2e12d1a31223 2024-09-12T23:49:13.971 INFO:teuthology.orchestra.run.smithi136.stdout:osd.3 smithi136 running (3m) 11s ago 19m 279M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e aa9dc124e59c 2024-09-12T23:49:13.971 INFO:teuthology.orchestra.run.smithi136.stdout:osd.4 smithi181 running (3m) 10s ago 19m 457M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 4ee3b228ce7d 2024-09-12T23:49:13.971 INFO:teuthology.orchestra.run.smithi136.stdout:osd.5 smithi181 running (2m) 10s ago 18m 219M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 3f9a09565258 2024-09-12T23:49:13.971 INFO:teuthology.orchestra.run.smithi136.stdout:osd.6 smithi181 running (2m) 10s ago 17m 134M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e ce539f6fb854 2024-09-12T23:49:13.971 INFO:teuthology.orchestra.run.smithi136.stdout:osd.7 smithi181 running (99s) 10s ago 16m 216M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 6ad9ab169a7b 2024-09-12T23:49:13.971 INFO:teuthology.orchestra.run.smithi136.stdout:prometheus.a smithi181 *:9095 running (7m) 10s ago 15m 57.8M - 2.43.0 a07b618ecd1d 3853d003a5cc 2024-09-12T23:49:14.327 INFO:teuthology.orchestra.run.smithi136.stdout:{ 2024-09-12T23:49:14.327 INFO:teuthology.orchestra.run.smithi136.stdout: "mon": { 2024-09-12T23:49:14.327 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 3 2024-09-12T23:49:14.327 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:49:14.327 INFO:teuthology.orchestra.run.smithi136.stdout: "mgr": { 2024-09-12T23:49:14.328 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 2 2024-09-12T23:49:14.328 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:49:14.328 INFO:teuthology.orchestra.run.smithi136.stdout: "osd": { 2024-09-12T23:49:14.328 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 8 2024-09-12T23:49:14.328 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:49:14.328 INFO:teuthology.orchestra.run.smithi136.stdout: "mds": { 2024-09-12T23:49:14.328 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 2 2024-09-12T23:49:14.328 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:49:14.328 INFO:teuthology.orchestra.run.smithi136.stdout: "overall": { 2024-09-12T23:49:14.328 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 15 2024-09-12T23:49:14.328 INFO:teuthology.orchestra.run.smithi136.stdout: } 2024-09-12T23:49:14.328 INFO:teuthology.orchestra.run.smithi136.stdout:} 2024-09-12T23:49:14.493 INFO:tasks.rados.rados.0.smithi136.stdout:3162: done (0 left) 2024-09-12T23:49:14.493 INFO:tasks.rados.rados.0.smithi136.stdout:3163: rmattr oid 38 current snap is 309 2024-09-12T23:49:14.493 INFO:tasks.rados.rados.0.smithi136.stdout:3163: done (0 left) 2024-09-12T23:49:14.493 INFO:tasks.rados.rados.0.smithi136.stdout:3164: read oid 15 snap -1 2024-09-12T23:49:14.493 INFO:tasks.rados.rados.0.smithi136.stdout:3164: expect (ObjNum 993 snap 307 seq_num 993) 2024-09-12T23:49:14.494 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 1 current snap is 309 2024-09-12T23:49:14.494 INFO:tasks.rados.rados.0.smithi136.stdout:3165: seq_num 996 ranges {1466368=688128,2154496=507904,2662400=581632,3244032=516096} 2024-09-12T23:49:14.506 INFO:tasks.rados.rados.0.smithi136.stdout:3165: writing smithi13667081-1 from 1466368 to 2154496 tid 1 2024-09-12T23:49:14.510 INFO:tasks.rados.rados.0.smithi136.stdout:3165: writing smithi13667081-1 from 2154496 to 2662400 tid 2 2024-09-12T23:49:14.514 INFO:tasks.rados.rados.0.smithi136.stdout:3165: writing smithi13667081-1 from 2662400 to 3244032 tid 3 2024-09-12T23:49:14.518 INFO:tasks.rados.rados.0.smithi136.stdout:3165: writing smithi13667081-1 from 3244032 to 3760128 tid 4 2024-09-12T23:49:14.518 INFO:tasks.rados.rados.0.smithi136.stdout:3166: rmattr oid 17 current snap is 309 2024-09-12T23:49:14.518 INFO:tasks.rados.rados.0.smithi136.stdout:3167: rmattr oid 34 current snap is 309 2024-09-12T23:49:14.519 INFO:tasks.rados.rados.0.smithi136.stdout:3168: read oid 43 snap -1 2024-09-12T23:49:14.519 INFO:tasks.rados.rados.0.smithi136.stdout:3168: expect (ObjNum 985 snap 303 seq_num 985) 2024-09-12T23:49:14.519 INFO:tasks.rados.rados.0.smithi136.stdout:3169: copy_from oid 27 from oid 31 current snap is 309 2024-09-12T23:49:14.519 INFO:tasks.rados.rados.0.smithi136.stdout:3170: rmattr oid 19 current snap is 309 2024-09-12T23:49:14.519 INFO:tasks.rados.rados.0.smithi136.stdout:3170: done (6 left) 2024-09-12T23:49:14.520 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 24 current snap is 309 2024-09-12T23:49:14.520 INFO:tasks.rados.rados.0.smithi136.stdout:3171: seq_num 997 ranges {2048000=704512,2752512=794624,3547136=794624,4341760=16384} 2024-09-12T23:49:14.535 INFO:tasks.rados.rados.0.smithi136.stdout:3171: writing smithi13667081-24 from 2048000 to 2752512 tid 1 2024-09-12T23:49:14.541 INFO:tasks.rados.rados.0.smithi136.stdout:3171: writing smithi13667081-24 from 2752512 to 3547136 tid 2 2024-09-12T23:49:14.547 INFO:tasks.rados.rados.0.smithi136.stdout:3171: writing smithi13667081-24 from 3547136 to 4341760 tid 3 2024-09-12T23:49:14.547 INFO:tasks.rados.rados.0.smithi136.stdout:3171: writing smithi13667081-24 from 4341760 to 4358144 tid 4 2024-09-12T23:49:14.547 INFO:tasks.rados.rados.0.smithi136.stdout:3172: rollback oid 16 current snap is 309 2024-09-12T23:49:14.547 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 16 to 303 2024-09-12T23:49:14.547 INFO:tasks.rados.rados.0.smithi136.stdout:3165: finishing write tid 1 to smithi13667081-1 2024-09-12T23:49:14.547 INFO:tasks.rados.rados.0.smithi136.stdout:3173: read oid 26 snap -1 2024-09-12T23:49:14.547 INFO:tasks.rados.rados.0.smithi136.stdout:3173: expect (ObjNum 987 snap 303 seq_num 987) 2024-09-12T23:49:14.548 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 25 current snap is 309 2024-09-12T23:49:14.548 INFO:tasks.rados.rados.0.smithi136.stdout:3174: seq_num 998 ranges {3506176=720896,4227072=679936,4907008=598016,5505024=327680} 2024-09-12T23:49:14.572 INFO:tasks.rados.rados.0.smithi136.stdout:3174: writing smithi13667081-25 from 3506176 to 4227072 tid 1 2024-09-12T23:49:14.576 INFO:tasks.rados.rados.0.smithi136.stdout:3174: writing smithi13667081-25 from 4227072 to 4907008 tid 2 2024-09-12T23:49:14.580 INFO:tasks.rados.rados.0.smithi136.stdout:3174: writing smithi13667081-25 from 4907008 to 5505024 tid 3 2024-09-12T23:49:14.582 INFO:tasks.rados.rados.0.smithi136.stdout:3174: writing smithi13667081-25 from 5505024 to 5832704 tid 4 2024-09-12T23:49:14.582 INFO:tasks.rados.rados.0.smithi136.stdout:3175: copy_from oid 28 from oid 5 current snap is 309 2024-09-12T23:49:14.583 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 33 current snap is 309 2024-09-12T23:49:14.583 INFO:tasks.rados.rados.0.smithi136.stdout:3165: finishing write tid 2 to smithi13667081-1 2024-09-12T23:49:14.583 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 34 v 2270 (ObjNum 796 snap 253 seq_num 796) dirty exists 2024-09-12T23:49:14.583 INFO:tasks.rados.rados.0.smithi136.stdout:3165: finishing write tid 3 to smithi13667081-1 2024-09-12T23:49:14.583 INFO:tasks.rados.rados.0.smithi136.stdout:3165: finishing write tid 4 to smithi13667081-1 2024-09-12T23:49:14.583 INFO:tasks.rados.rados.0.smithi136.stdout:3176: seq_num 999 ranges {0=737280,737280=565248,1302528=761856,2064384=278528} 2024-09-12T23:49:14.587 INFO:tasks.rados.rados.0.smithi136.stdout:3176: writing smithi13667081-33 from 0 to 737280 tid 1 2024-09-12T23:49:14.591 INFO:tasks.rados.rados.0.smithi136.stdout:3176: writing smithi13667081-33 from 737280 to 1302528 tid 2 2024-09-12T23:49:14.596 INFO:tasks.rados.rados.0.smithi136.stdout:3176: writing smithi13667081-33 from 1302528 to 2064384 tid 3 2024-09-12T23:49:14.598 INFO:tasks.rados.rados.0.smithi136.stdout:3176: writing smithi13667081-33 from 2064384 to 2342912 tid 4 2024-09-12T23:49:14.598 INFO:tasks.rados.rados.0.smithi136.stdout:3167: done (11 left) 2024-09-12T23:49:14.598 INFO:tasks.rados.rados.0.smithi136.stdout:3177: read oid 21 snap -1 2024-09-12T23:49:14.598 INFO:tasks.rados.rados.0.smithi136.stdout:3177: expect (ObjNum 931 snap 292 seq_num 931) 2024-09-12T23:49:14.598 INFO:tasks.rados.rados.0.smithi136.stdout:3178: delete oid 30 current snap is 309 2024-09-12T23:49:14.600 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 17 v 2450 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-12T23:49:14.600 INFO:tasks.rados.rados.0.smithi136.stdout:3165: finishing write tid 5 to smithi13667081-1 2024-09-12T23:49:14.600 INFO:tasks.rados.rados.0.smithi136.stdout:3169: finishing copy_from to smithi13667081-27 2024-09-12T23:49:14.600 INFO:tasks.rados.rados.0.smithi136.stdout:3169: got expected ENOENT (src dne) 2024-09-12T23:49:14.601 INFO:tasks.rados.rados.0.smithi136.stdout:3169: finishing copy_from racing read to smithi13667081-27 2024-09-12T23:49:14.601 INFO:tasks.rados.rados.0.smithi136.stdout:3165: finishing write tid 6 to smithi13667081-1 2024-09-12T23:49:14.601 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 1 v 1825 (ObjNum 996 snap 309 seq_num 996) dirty exists 2024-09-12T23:49:14.601 INFO:tasks.rados.rados.0.smithi136.stdout:3165: left oid 1 (ObjNum 996 snap 309 seq_num 996) 2024-09-12T23:49:14.601 INFO:tasks.rados.rados.0.smithi136.stdout:3171: finishing write tid 1 to smithi13667081-24 2024-09-12T23:49:14.601 INFO:tasks.rados.rados.0.smithi136.stdout:3171: finishing write tid 2 to smithi13667081-24 2024-09-12T23:49:14.601 INFO:tasks.rados.rados.0.smithi136.stdout:3171: finishing write tid 3 to smithi13667081-24 2024-09-12T23:49:14.601 INFO:tasks.rados.rados.0.smithi136.stdout:3171: finishing write tid 4 to smithi13667081-24 2024-09-12T23:49:14.601 INFO:tasks.rados.rados.0.smithi136.stdout:3171: finishing write tid 5 to smithi13667081-24 2024-09-12T23:49:14.601 INFO:tasks.rados.rados.0.smithi136.stdout:3168: expect (ObjNum 985 snap 303 seq_num 985) 2024-09-12T23:49:14.752 INFO:tasks.rados.rados.0.smithi136.stdout:3172: finishing rollback tid 0 to smithi13667081-16 2024-09-12T23:49:14.752 INFO:tasks.rados.rados.0.smithi136.stdout:3172: finishing rollback tid 1 to smithi13667081-16 2024-09-12T23:49:14.752 INFO:tasks.rados.rados.0.smithi136.stdout:3172: finishing rollback tid 2 to smithi13667081-16 2024-09-12T23:49:14.752 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 16 v 1978 (ObjNum 977 snap 303 seq_num 977) dirty exists 2024-09-12T23:49:14.752 INFO:tasks.rados.rados.0.smithi136.stdout:3173: expect (ObjNum 987 snap 303 seq_num 987) 2024-09-12T23:49:14.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:14 smithi136 bash[75164]: cluster 2024-09-12T23:49:13.484501+0000 mon.a (mon.0) 1108 : cluster [DBG] osdmap e695: 8 total, 8 up, 8 in 2024-09-12T23:49:14.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:14 smithi136 bash[75164]: audit 2024-09-12T23:49:13.639403+0000 mgr.y (mgr.24661) 344 : audit [DBG] from='client.34440 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:49:14.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:14 smithi136 bash[75164]: audit 2024-09-12T23:49:13.662017+0000 mon.a (mon.0) 1109 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:14.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:14 smithi136 bash[75164]: audit 2024-09-12T23:49:13.669475+0000 mon.a (mon.0) 1110 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:14.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:14 smithi136 bash[75164]: audit 2024-09-12T23:49:13.670697+0000 mon.a (mon.0) 1111 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:49:14.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:14 smithi136 bash[75164]: audit 2024-09-12T23:49:13.671680+0000 mon.a (mon.0) 1112 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:49:14.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:14 smithi136 bash[75164]: audit 2024-09-12T23:49:13.678631+0000 mon.a (mon.0) 1113 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:14.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:14 smithi136 bash[75164]: audit 2024-09-12T23:49:13.756912+0000 mon.a (mon.0) 1114 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:49:14.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:14 smithi136 bash[75164]: audit 2024-09-12T23:49:13.759740+0000 mon.a (mon.0) 1115 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:14.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:14 smithi136 bash[75164]: audit 2024-09-12T23:49:13.761486+0000 mon.a (mon.0) 1116 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:14.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:14 smithi136 bash[75164]: audit 2024-09-12T23:49:13.762573+0000 mon.a (mon.0) 1117 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:14.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:14 smithi136 bash[75164]: audit 2024-09-12T23:49:13.764766+0000 mon.a (mon.0) 1118 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:14.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:14 smithi136 bash[75164]: audit 2024-09-12T23:49:13.767251+0000 mon.a (mon.0) 1119 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:14.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:14 smithi136 bash[75164]: audit 2024-09-12T23:49:13.768304+0000 mon.a (mon.0) 1120 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:14.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:14 smithi136 bash[75164]: audit 2024-09-12T23:49:13.769365+0000 mon.a (mon.0) 1121 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:14.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:14 smithi136 bash[75164]: audit 2024-09-12T23:49:13.770370+0000 mon.a (mon.0) 1122 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:14.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:14 smithi136 bash[75164]: audit 2024-09-12T23:49:13.771370+0000 mon.a (mon.0) 1123 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:14.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:14 smithi136 bash[75164]: audit 2024-09-12T23:49:13.772354+0000 mon.a (mon.0) 1124 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:14.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:14 smithi136 bash[75164]: audit 2024-09-12T23:49:13.773382+0000 mon.a (mon.0) 1125 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:14.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:14 smithi136 bash[75164]: audit 2024-09-12T23:49:13.775302+0000 mon.a (mon.0) 1126 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:14.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:14 smithi136 bash[76371]: cluster 2024-09-12T23:49:13.484501+0000 mon.a (mon.0) 1108 : cluster [DBG] osdmap e695: 8 total, 8 up, 8 in 2024-09-12T23:49:14.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:14 smithi136 bash[76371]: audit 2024-09-12T23:49:13.639403+0000 mgr.y (mgr.24661) 344 : audit [DBG] from='client.34440 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:49:14.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:14 smithi136 bash[76371]: audit 2024-09-12T23:49:13.662017+0000 mon.a (mon.0) 1109 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:14.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:14 smithi136 bash[76371]: audit 2024-09-12T23:49:13.669475+0000 mon.a (mon.0) 1110 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:14.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:14 smithi136 bash[76371]: audit 2024-09-12T23:49:13.670697+0000 mon.a (mon.0) 1111 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:49:14.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:14 smithi136 bash[76371]: audit 2024-09-12T23:49:13.671680+0000 mon.a (mon.0) 1112 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:49:14.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:14 smithi136 bash[76371]: audit 2024-09-12T23:49:13.678631+0000 mon.a (mon.0) 1113 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:14.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:14 smithi136 bash[76371]: audit 2024-09-12T23:49:13.756912+0000 mon.a (mon.0) 1114 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:49:14.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:14 smithi136 bash[76371]: audit 2024-09-12T23:49:13.759740+0000 mon.a (mon.0) 1115 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:14.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:14 smithi136 bash[76371]: audit 2024-09-12T23:49:13.761486+0000 mon.a (mon.0) 1116 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:14.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:14 smithi136 bash[76371]: audit 2024-09-12T23:49:13.762573+0000 mon.a (mon.0) 1117 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:14.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:14 smithi136 bash[76371]: audit 2024-09-12T23:49:13.764766+0000 mon.a (mon.0) 1118 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:14.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:14 smithi136 bash[76371]: audit 2024-09-12T23:49:13.767251+0000 mon.a (mon.0) 1119 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:14.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:14 smithi136 bash[76371]: audit 2024-09-12T23:49:13.768304+0000 mon.a (mon.0) 1120 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:14.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:14 smithi136 bash[76371]: audit 2024-09-12T23:49:13.769365+0000 mon.a (mon.0) 1121 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:14.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:14 smithi136 bash[76371]: audit 2024-09-12T23:49:13.770370+0000 mon.a (mon.0) 1122 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:14.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:14 smithi136 bash[76371]: audit 2024-09-12T23:49:13.771370+0000 mon.a (mon.0) 1123 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:14.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:14 smithi136 bash[76371]: audit 2024-09-12T23:49:13.772354+0000 mon.a (mon.0) 1124 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:14.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:14 smithi136 bash[76371]: audit 2024-09-12T23:49:13.773382+0000 mon.a (mon.0) 1125 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:14.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:14 smithi136 bash[76371]: audit 2024-09-12T23:49:13.775302+0000 mon.a (mon.0) 1126 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:14.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:14 smithi136 bash[76371]: audit 2024-09-12T23:49:13.776758+0000 mon.a (mon.0) 1127 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:14.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:14 smithi136 bash[76371]: audit 2024-09-12T23:49:14.330502+0000 mon.a (mon.0) 1128 : audit [DBG] from='client.? 172.21.15.136:0/1955187610' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:14.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:14 smithi136 bash[75164]: audit 2024-09-12T23:49:13.776758+0000 mon.a (mon.0) 1127 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:14.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:14 smithi136 bash[75164]: audit 2024-09-12T23:49:14.330502+0000 mon.a (mon.0) 1128 : audit [DBG] from='client.? 172.21.15.136:0/1955187610' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:14.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:14 smithi181 bash[67587]: cluster 2024-09-12T23:49:13.484501+0000 mon.a (mon.0) 1108 : cluster [DBG] osdmap e695: 8 total, 8 up, 8 in 2024-09-12T23:49:14.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:14 smithi181 bash[67587]: audit 2024-09-12T23:49:13.639403+0000 mgr.y (mgr.24661) 344 : audit [DBG] from='client.34440 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:49:14.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:14 smithi181 bash[67587]: audit 2024-09-12T23:49:13.662017+0000 mon.a (mon.0) 1109 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:14.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:14 smithi181 bash[67587]: audit 2024-09-12T23:49:13.669475+0000 mon.a (mon.0) 1110 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:14.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:14 smithi181 bash[67587]: audit 2024-09-12T23:49:13.670697+0000 mon.a (mon.0) 1111 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:49:14.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:14 smithi181 bash[67587]: audit 2024-09-12T23:49:13.671680+0000 mon.a (mon.0) 1112 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:49:14.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:14 smithi181 bash[67587]: audit 2024-09-12T23:49:13.678631+0000 mon.a (mon.0) 1113 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:14.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:14 smithi181 bash[67587]: audit 2024-09-12T23:49:13.756912+0000 mon.a (mon.0) 1114 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:49:14.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:14 smithi181 bash[67587]: audit 2024-09-12T23:49:13.759740+0000 mon.a (mon.0) 1115 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:14.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:14 smithi181 bash[67587]: audit 2024-09-12T23:49:13.761486+0000 mon.a (mon.0) 1116 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:14.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:14 smithi181 bash[67587]: audit 2024-09-12T23:49:13.762573+0000 mon.a (mon.0) 1117 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:14.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:14 smithi181 bash[67587]: audit 2024-09-12T23:49:13.764766+0000 mon.a (mon.0) 1118 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:14.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:14 smithi181 bash[67587]: audit 2024-09-12T23:49:13.767251+0000 mon.a (mon.0) 1119 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:14.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:14 smithi181 bash[67587]: audit 2024-09-12T23:49:13.768304+0000 mon.a (mon.0) 1120 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:14.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:14 smithi181 bash[67587]: audit 2024-09-12T23:49:13.769365+0000 mon.a (mon.0) 1121 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:14.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:14 smithi181 bash[67587]: audit 2024-09-12T23:49:13.770370+0000 mon.a (mon.0) 1122 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:14.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:14 smithi181 bash[67587]: audit 2024-09-12T23:49:13.771370+0000 mon.a (mon.0) 1123 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:14.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:14 smithi181 bash[67587]: audit 2024-09-12T23:49:13.772354+0000 mon.a (mon.0) 1124 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:14.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:14 smithi181 bash[67587]: audit 2024-09-12T23:49:13.773382+0000 mon.a (mon.0) 1125 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:14.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:14 smithi181 bash[67587]: audit 2024-09-12T23:49:13.775302+0000 mon.a (mon.0) 1126 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:14.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:14 smithi181 bash[67587]: audit 2024-09-12T23:49:13.776758+0000 mon.a (mon.0) 1127 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:14.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:14 smithi181 bash[67587]: audit 2024-09-12T23:49:14.330502+0000 mon.a (mon.0) 1128 : audit [DBG] from='client.? 172.21.15.136:0/1955187610' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:14.995 INFO:tasks.rados.rados.0.smithi136.stdout:3174: finishing write tid 1 to smithi13667081-25 2024-09-12T23:49:14.995 INFO:tasks.rados.rados.0.smithi136.stdout:3174: finishing write tid 2 to smithi13667081-25 2024-09-12T23:49:14.995 INFO:tasks.rados.rados.0.smithi136.stdout:3174: finishing write tid 3 to smithi13667081-25 2024-09-12T23:49:14.995 INFO:tasks.rados.rados.0.smithi136.stdout:3174: finishing write tid 4 to smithi13667081-25 2024-09-12T23:49:14.995 INFO:tasks.rados.rados.0.smithi136.stdout:3174: finishing write tid 5 to smithi13667081-25 2024-09-12T23:49:14.995 INFO:tasks.rados.rados.0.smithi136.stdout:3171: finishing write tid 6 to smithi13667081-24 2024-09-12T23:49:14.995 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 24 v 1519 (ObjNum 997 snap 309 seq_num 997) dirty exists 2024-09-12T23:49:14.995 INFO:tasks.rados.rados.0.smithi136.stdout:3171: left oid 24 (ObjNum 997 snap 309 seq_num 997) 2024-09-12T23:49:14.995 INFO:tasks.rados.rados.0.smithi136.stdout:3174: finishing write tid 6 to smithi13667081-25 2024-09-12T23:49:14.995 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 25 v 2437 (ObjNum 998 snap 309 seq_num 998) dirty exists 2024-09-12T23:49:14.995 INFO:tasks.rados.rados.0.smithi136.stdout:3174: left oid 25 (ObjNum 998 snap 309 seq_num 998) 2024-09-12T23:49:14.996 INFO:tasks.rados.rados.0.smithi136.stdout:3177: expect (ObjNum 931 snap 292 seq_num 931) 2024-09-12T23:49:15.120 INFO:tasks.rados.rados.0.smithi136.stdout:3164: expect (ObjNum 993 snap 307 seq_num 993) 2024-09-12T23:49:15.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:15 smithi136 bash[75164]: audit 2024-09-12T23:49:13.937739+0000 mgr.y (mgr.24661) 345 : audit [DBG] from='client.44366 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:49:15.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:15 smithi136 bash[75164]: cluster 2024-09-12T23:49:14.485573+0000 mon.a (mon.0) 1129 : cluster [DBG] osdmap e696: 8 total, 8 up, 8 in 2024-09-12T23:49:15.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:15 smithi136 bash[75164]: cluster 2024-09-12T23:49:14.820213+0000 mgr.y (mgr.24661) 346 : cluster [DBG] pgmap v621: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 213 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 1.9 MiB/s wr, 2 op/s 2024-09-12T23:49:15.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:15 smithi136 bash[76371]: audit 2024-09-12T23:49:13.937739+0000 mgr.y (mgr.24661) 345 : audit [DBG] from='client.44366 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:49:15.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:15 smithi136 bash[76371]: cluster 2024-09-12T23:49:14.485573+0000 mon.a (mon.0) 1129 : cluster [DBG] osdmap e696: 8 total, 8 up, 8 in 2024-09-12T23:49:15.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:15 smithi136 bash[76371]: cluster 2024-09-12T23:49:14.820213+0000 mgr.y (mgr.24661) 346 : cluster [DBG] pgmap v621: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 213 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 1.9 MiB/s wr, 2 op/s 2024-09-12T23:49:15.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:15 smithi181 bash[67587]: audit 2024-09-12T23:49:13.937739+0000 mgr.y (mgr.24661) 345 : audit [DBG] from='client.44366 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:49:15.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:15 smithi181 bash[67587]: cluster 2024-09-12T23:49:14.485573+0000 mon.a (mon.0) 1129 : cluster [DBG] osdmap e696: 8 total, 8 up, 8 in 2024-09-12T23:49:15.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:15 smithi181 bash[67587]: cluster 2024-09-12T23:49:14.820213+0000 mgr.y (mgr.24661) 346 : cluster [DBG] pgmap v621: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 213 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 1.9 MiB/s wr, 2 op/s 2024-09-12T23:49:17.169 INFO:tasks.rados.rados.0.smithi136.stdout:3175: finishing copy_from racing read to smithi13667081-28 2024-09-12T23:49:17.169 INFO:tasks.rados.rados.0.smithi136.stdout:3176: finishing write tid 1 to smithi13667081-33 2024-09-12T23:49:17.169 INFO:tasks.rados.rados.0.smithi136.stdout:3176: finishing write tid 2 to smithi13667081-33 2024-09-12T23:49:17.169 INFO:tasks.rados.rados.0.smithi136.stdout:3176: finishing write tid 3 to smithi13667081-33 2024-09-12T23:49:17.169 INFO:tasks.rados.rados.0.smithi136.stdout:3176: finishing write tid 4 to smithi13667081-33 2024-09-12T23:49:17.169 INFO:tasks.rados.rados.0.smithi136.stdout:3176: finishing write tid 5 to smithi13667081-33 2024-09-12T23:49:17.169 INFO:tasks.rados.rados.0.smithi136.stdout:3175: finishing copy_from to smithi13667081-28 2024-09-12T23:49:17.169 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 28 v 1633 (ObjNum 965 snap 301 seq_num 965) dirty exists 2024-09-12T23:49:17.169 INFO:tasks.rados.rados.0.smithi136.stdout:3176: finishing write tid 6 to smithi13667081-33 2024-09-12T23:49:17.169 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 33 v 2275 (ObjNum 999 snap 309 seq_num 999) dirty exists 2024-09-12T23:49:17.170 INFO:tasks.rados.rados.0.smithi136.stdout:3176: left oid 33 (ObjNum 999 snap 309 seq_num 999) 2024-09-12T23:49:17.170 INFO:tasks.rados.rados.0.smithi136.stdout:3164: done (12 left) 2024-09-12T23:49:17.172 INFO:tasks.rados.rados.0.smithi136.stdout:3165: done (11 left) 2024-09-12T23:49:17.172 INFO:tasks.rados.rados.0.smithi136.stdout:3166: done (10 left) 2024-09-12T23:49:17.172 INFO:tasks.rados.rados.0.smithi136.stdout:3168: done (9 left) 2024-09-12T23:49:17.172 INFO:tasks.rados.rados.0.smithi136.stdout:3169: done (8 left) 2024-09-12T23:49:17.173 INFO:tasks.rados.rados.0.smithi136.stdout:3171: done (7 left) 2024-09-12T23:49:17.173 INFO:tasks.rados.rados.0.smithi136.stdout:3172: done (6 left) 2024-09-12T23:49:17.173 INFO:tasks.rados.rados.0.smithi136.stdout:3173: done (5 left) 2024-09-12T23:49:17.173 INFO:tasks.rados.rados.0.smithi136.stdout:3174: done (4 left) 2024-09-12T23:49:17.173 INFO:tasks.rados.rados.0.smithi136.stdout:3175: done (3 left) 2024-09-12T23:49:17.173 INFO:tasks.rados.rados.0.smithi136.stdout:3176: done (2 left) 2024-09-12T23:49:17.173 INFO:tasks.rados.rados.0.smithi136.stdout:3177: done (1 left) 2024-09-12T23:49:17.173 INFO:tasks.rados.rados.0.smithi136.stdout:3178: done (0 left) 2024-09-12T23:49:17.173 INFO:tasks.rados.rados.0.smithi136.stdout:3179: read oid 4 snap -1 2024-09-12T23:49:17.173 INFO:tasks.rados.rados.0.smithi136.stdout:3179: expect (ObjNum 949 snap 301 seq_num 949) 2024-09-12T23:49:17.173 INFO:tasks.rados.rados.0.smithi136.stdout:3180: rmattr oid 49 current snap is 309 2024-09-12T23:49:17.173 INFO:tasks.rados.rados.0.smithi136.stdout:3180: done (1 left) 2024-09-12T23:49:17.173 INFO:tasks.rados.rados.0.smithi136.stdout:3181: snap_create 2024-09-12T23:49:17.267 INFO:tasks.rados.rados.0.smithi136.stdout:3179: expect (ObjNum 949 snap 301 seq_num 949) 2024-09-12T23:49:17.724 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:17 smithi181 bash[67587]: cephadm 2024-09-12T23:49:16.003223+0000 mgr.y (mgr.24661) 347 : cephadm [INF] Upgrade: Updating prometheus.a 2024-09-12T23:49:17.724 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:17 smithi181 bash[67587]: cephadm 2024-09-12T23:49:16.261041+0000 mgr.y (mgr.24661) 348 : cephadm [INF] Deploying daemon prometheus.a on smithi181 2024-09-12T23:49:17.724 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:17 smithi181 bash[67587]: cluster 2024-09-12T23:49:16.456315+0000 mon.a (mon.0) 1130 : cluster [DBG] osdmap e697: 8 total, 8 up, 8 in 2024-09-12T23:49:17.725 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:17 smithi181 bash[67587]: cluster 2024-09-12T23:49:16.821088+0000 mgr.y (mgr.24661) 349 : cluster [DBG] pgmap v623: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 227 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 3.4 MiB/s wr, 17 op/s 2024-09-12T23:49:17.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:17 smithi136 bash[75164]: cephadm 2024-09-12T23:49:16.003223+0000 mgr.y (mgr.24661) 347 : cephadm [INF] Upgrade: Updating prometheus.a 2024-09-12T23:49:17.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:17 smithi136 bash[75164]: cephadm 2024-09-12T23:49:16.261041+0000 mgr.y (mgr.24661) 348 : cephadm [INF] Deploying daemon prometheus.a on smithi181 2024-09-12T23:49:17.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:17 smithi136 bash[75164]: cluster 2024-09-12T23:49:16.456315+0000 mon.a (mon.0) 1130 : cluster [DBG] osdmap e697: 8 total, 8 up, 8 in 2024-09-12T23:49:17.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:17 smithi136 bash[75164]: cluster 2024-09-12T23:49:16.821088+0000 mgr.y (mgr.24661) 349 : cluster [DBG] pgmap v623: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 227 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 3.4 MiB/s wr, 17 op/s 2024-09-12T23:49:17.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:17 smithi136 bash[76371]: cephadm 2024-09-12T23:49:16.003223+0000 mgr.y (mgr.24661) 347 : cephadm [INF] Upgrade: Updating prometheus.a 2024-09-12T23:49:17.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:17 smithi136 bash[76371]: cephadm 2024-09-12T23:49:16.261041+0000 mgr.y (mgr.24661) 348 : cephadm [INF] Deploying daemon prometheus.a on smithi181 2024-09-12T23:49:17.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:17 smithi136 bash[76371]: cluster 2024-09-12T23:49:16.456315+0000 mon.a (mon.0) 1130 : cluster [DBG] osdmap e697: 8 total, 8 up, 8 in 2024-09-12T23:49:17.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:17 smithi136 bash[76371]: cluster 2024-09-12T23:49:16.821088+0000 mgr.y (mgr.24661) 349 : cluster [DBG] pgmap v623: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 227 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 3.4 MiB/s wr, 17 op/s 2024-09-12T23:49:18.744 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:18 smithi181 bash[67587]: cluster 2024-09-12T23:49:17.466978+0000 mon.a (mon.0) 1131 : cluster [DBG] osdmap e698: 8 total, 8 up, 8 in 2024-09-12T23:49:18.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:18 smithi136 bash[75164]: cluster 2024-09-12T23:49:17.466978+0000 mon.a (mon.0) 1131 : cluster [DBG] osdmap e698: 8 total, 8 up, 8 in 2024-09-12T23:49:18.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:18 smithi136 bash[76371]: cluster 2024-09-12T23:49:17.466978+0000 mon.a (mon.0) 1131 : cluster [DBG] osdmap e698: 8 total, 8 up, 8 in 2024-09-12T23:49:18.960 INFO:tasks.rados.rados.0.smithi136.stdout:3179: done (1 left) 2024-09-12T23:49:18.961 INFO:tasks.rados.rados.0.smithi136.stdout:3181: done (0 left) 2024-09-12T23:49:18.961 INFO:tasks.rados.rados.0.smithi136.stdout:3182: read oid 49 snap -1 2024-09-12T23:49:18.961 INFO:tasks.rados.rados.0.smithi136.stdout:3182: expect deleted 2024-09-12T23:49:18.961 INFO:tasks.rados.rados.0.smithi136.stdout:3183: delete oid 13 current snap is 310 2024-09-12T23:49:18.966 INFO:tasks.rados.rados.0.smithi136.stdout:3182: done (1 left) 2024-09-12T23:49:18.966 INFO:tasks.rados.rados.0.smithi136.stdout:3183: done (0 left) 2024-09-12T23:49:18.966 INFO:tasks.rados.rados.0.smithi136.stdout:3184: snap_create 2024-09-12T23:49:19.473 INFO:tasks.rados.rados.0.smithi136.stdout:3184: done (0 left) 2024-09-12T23:49:19.473 INFO:tasks.rados.rados.0.smithi136.stdout:3185: rollback oid 15 current snap is 311 2024-09-12T23:49:19.473 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 15 to 309 2024-09-12T23:49:19.474 INFO:tasks.rados.rados.0.smithi136.stdout:3186: read oid 40 snap -1 2024-09-12T23:49:19.474 INFO:tasks.rados.rados.0.smithi136.stdout:3186: expect (ObjNum 983 snap 303 seq_num 983) 2024-09-12T23:49:19.474 INFO:tasks.rados.rados.0.smithi136.stdout:3187: snap_create 2024-09-12T23:49:19.494 INFO:tasks.rados.rados.0.smithi136.stdout:3185: finishing rollback tid 0 to smithi13667081-15 2024-09-12T23:49:19.494 INFO:tasks.rados.rados.0.smithi136.stdout:3186: expect (ObjNum 983 snap 303 seq_num 983) 2024-09-12T23:49:19.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:19 smithi181 bash[67587]: cluster 2024-09-12T23:49:18.822010+0000 mgr.y (mgr.24661) 350 : cluster [DBG] pgmap v625: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 228 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 26 MiB/s rd, 2.8 MiB/s wr, 17 op/s 2024-09-12T23:49:19.601 INFO:tasks.rados.rados.0.smithi136.stdout:3185: finishing rollback tid 1 to smithi13667081-15 2024-09-12T23:49:19.601 INFO:tasks.rados.rados.0.smithi136.stdout:3185: finishing rollback tid 2 to smithi13667081-15 2024-09-12T23:49:19.601 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 15 v 1827 (ObjNum 993 snap 307 seq_num 993) dirty exists 2024-09-12T23:49:19.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:19 smithi136 bash[75164]: cluster 2024-09-12T23:49:18.822010+0000 mgr.y (mgr.24661) 350 : cluster [DBG] pgmap v625: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 228 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 26 MiB/s rd, 2.8 MiB/s wr, 17 op/s 2024-09-12T23:49:19.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:49:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:49:19] "GET /metrics HTTP/1.1" 200 36438 "" "Prometheus/2.43.0" 2024-09-12T23:49:19.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:19 smithi136 bash[76371]: cluster 2024-09-12T23:49:18.822010+0000 mgr.y (mgr.24661) 350 : cluster [DBG] pgmap v625: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 228 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 26 MiB/s rd, 2.8 MiB/s wr, 17 op/s 2024-09-12T23:49:20.345 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:20 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:49:20.345 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:49:20 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:49:20.345 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 12 23:49:20 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:49:20.346 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:49:20 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:49:20.346 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:49:20 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:49:20.346 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:49:20 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:49:20.346 INFO:journalctl@ceph.mds.b.smithi181.stdout:Sep 12 23:49:20 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:49:20.347 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:49:20 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:49:20.347 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:49:20 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:49:20.347 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 12 23:49:20 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:49:20.483 INFO:tasks.rados.rados.0.smithi136.stdout:3185: done (2 left) 2024-09-12T23:49:20.483 INFO:tasks.rados.rados.0.smithi136.stdout:3186: done (1 left) 2024-09-12T23:49:20.483 INFO:tasks.rados.rados.0.smithi136.stdout:3187: done (0 left) 2024-09-12T23:49:20.483 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 14 current snap is 312 2024-09-12T23:49:20.483 INFO:tasks.rados.rados.0.smithi136.stdout:3188: seq_num 1002 ranges {0=786432,786432=614400,1400832=581632,1982464=409600} 2024-09-12T23:49:20.495 INFO:tasks.rados.rados.0.smithi136.stdout:3188: writing smithi13667081-14 from 0 to 786432 tid 1 2024-09-12T23:49:20.507 INFO:tasks.rados.rados.0.smithi136.stdout:3188: writing smithi13667081-14 from 786432 to 1400832 tid 2 2024-09-12T23:49:20.518 INFO:tasks.rados.rados.0.smithi136.stdout:3188: writing smithi13667081-14 from 1400832 to 1982464 tid 3 2024-09-12T23:49:20.525 INFO:tasks.rados.rados.0.smithi136.stdout:3188: writing smithi13667081-14 from 1982464 to 2392064 tid 4 2024-09-12T23:49:20.525 INFO:tasks.rados.rados.0.smithi136.stdout:3189: rollback oid 15 current snap is 312 2024-09-12T23:49:20.525 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 15 to 309 2024-09-12T23:49:20.525 INFO:tasks.rados.rados.0.smithi136.stdout:3188: finishing write tid 1 to smithi13667081-14 2024-09-12T23:49:20.525 INFO:tasks.rados.rados.0.smithi136.stdout:3188: finishing write tid 2 to smithi13667081-14 2024-09-12T23:49:20.526 INFO:tasks.rados.rados.0.smithi136.stdout:3188: finishing write tid 3 to smithi13667081-14 2024-09-12T23:49:20.526 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 46 current snap is 312 2024-09-12T23:49:20.526 INFO:tasks.rados.rados.0.smithi136.stdout:3190: seq_num 1003 ranges {2457600=802816,3260416=499712,3760128=737280,4497408=368640} 2024-09-12T23:49:20.560 INFO:tasks.rados.rados.0.smithi136.stdout:3190: writing smithi13667081-46 from 2457600 to 3260416 tid 1 2024-09-12T23:49:20.565 INFO:tasks.rados.rados.0.smithi136.stdout:3190: writing smithi13667081-46 from 3260416 to 3760128 tid 2 2024-09-12T23:49:20.571 INFO:tasks.rados.rados.0.smithi136.stdout:3190: writing smithi13667081-46 from 3760128 to 4497408 tid 3 2024-09-12T23:49:20.575 INFO:tasks.rados.rados.0.smithi136.stdout:3190: writing smithi13667081-46 from 4497408 to 4866048 tid 4 2024-09-12T23:49:20.575 INFO:tasks.rados.rados.0.smithi136.stdout:3191: read oid 49 snap 306 2024-09-12T23:49:20.575 INFO:tasks.rados.rados.0.smithi136.stdout:3191: expect deleted 2024-09-12T23:49:20.575 INFO:tasks.rados.rados.0.smithi136.stdout:3192: rmattr oid 8 current snap is 312 2024-09-12T23:49:20.575 INFO:tasks.rados.rados.0.smithi136.stdout:3192: done (4 left) 2024-09-12T23:49:20.575 INFO:tasks.rados.rados.0.smithi136.stdout:3193: snap_create 2024-09-12T23:49:20.575 INFO:tasks.rados.rados.0.smithi136.stdout:3188: finishing write tid 4 to smithi13667081-14 2024-09-12T23:49:20.575 INFO:tasks.rados.rados.0.smithi136.stdout:3188: finishing write tid 5 to smithi13667081-14 2024-09-12T23:49:20.576 INFO:tasks.rados.rados.0.smithi136.stdout:3188: finishing write tid 6 to smithi13667081-14 2024-09-12T23:49:20.576 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 14 v 1984 (ObjNum 1002 snap 312 seq_num 1002) dirty exists 2024-09-12T23:49:20.576 INFO:tasks.rados.rados.0.smithi136.stdout:3188: left oid 14 (ObjNum 1002 snap 312 seq_num 1002) 2024-09-12T23:49:20.576 INFO:tasks.rados.rados.0.smithi136.stdout:3189: finishing rollback tid 0 to smithi13667081-15 2024-09-12T23:49:20.576 INFO:tasks.rados.rados.0.smithi136.stdout:3189: finishing rollback tid 1 to smithi13667081-15 2024-09-12T23:49:20.576 INFO:tasks.rados.rados.0.smithi136.stdout:3189: finishing rollback tid 2 to smithi13667081-15 2024-09-12T23:49:20.576 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 15 v 1830 (ObjNum 993 snap 307 seq_num 993) dirty exists 2024-09-12T23:49:20.576 INFO:tasks.rados.rados.0.smithi136.stdout:3190: finishing write tid 1 to smithi13667081-46 2024-09-12T23:49:20.576 INFO:tasks.rados.rados.0.smithi136.stdout:3190: finishing write tid 2 to smithi13667081-46 2024-09-12T23:49:20.578 INFO:tasks.rados.rados.0.smithi136.stdout:3190: finishing write tid 3 to smithi13667081-46 2024-09-12T23:49:20.578 INFO:tasks.rados.rados.0.smithi136.stdout:3190: finishing write tid 4 to smithi13667081-46 2024-09-12T23:49:20.579 INFO:tasks.rados.rados.0.smithi136.stdout:3190: finishing write tid 5 to smithi13667081-46 2024-09-12T23:49:20.580 INFO:tasks.rados.rados.0.smithi136.stdout:3190: finishing write tid 6 to smithi13667081-46 2024-09-12T23:49:20.580 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 46 v 1524 (ObjNum 1003 snap 312 seq_num 1003) dirty exists 2024-09-12T23:49:20.580 INFO:tasks.rados.rados.0.smithi136.stdout:3190: left oid 46 (ObjNum 1003 snap 312 seq_num 1003) 2024-09-12T23:49:20.629 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:20 smithi181 bash[67587]: cluster 2024-09-12T23:49:19.475769+0000 mon.a (mon.0) 1132 : cluster [DBG] osdmap e699: 8 total, 8 up, 8 in 2024-09-12T23:49:20.630 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:49:20 smithi181 systemd[1]: Stopping Ceph prometheus.a for da58a47c-715d-11ef-bceb-c7b262605968... 2024-09-12T23:49:20.630 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:49:20 smithi181 bash[60471]: ts=2024-09-12T23:49:20.462Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-12T23:49:20.630 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:49:20 smithi181 bash[60471]: ts=2024-09-12T23:49:20.462Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-12T23:49:20.630 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:49:20 smithi181 bash[60471]: ts=2024-09-12T23:49:20.462Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-12T23:49:20.630 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:49:20 smithi181 bash[60471]: ts=2024-09-12T23:49:20.462Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-12T23:49:20.630 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:49:20 smithi181 bash[60471]: ts=2024-09-12T23:49:20.462Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-12T23:49:20.630 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:49:20 smithi181 bash[60471]: ts=2024-09-12T23:49:20.462Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-12T23:49:20.630 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:49:20 smithi181 bash[60471]: ts=2024-09-12T23:49:20.462Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-12T23:49:20.631 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:49:20 smithi181 bash[60471]: ts=2024-09-12T23:49:20.462Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-12T23:49:20.631 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:49:20 smithi181 bash[60471]: ts=2024-09-12T23:49:20.462Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-12T23:49:20.631 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:49:20 smithi181 bash[60471]: ts=2024-09-12T23:49:20.463Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-12T23:49:20.631 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:49:20 smithi181 bash[60471]: ts=2024-09-12T23:49:20.463Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-12T23:49:20.631 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:49:20 smithi181 bash[60471]: ts=2024-09-12T23:49:20.463Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-12T23:49:20.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:20 smithi136 bash[75164]: cluster 2024-09-12T23:49:19.475769+0000 mon.a (mon.0) 1132 : cluster [DBG] osdmap e699: 8 total, 8 up, 8 in 2024-09-12T23:49:20.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:20 smithi136 bash[76371]: cluster 2024-09-12T23:49:19.475769+0000 mon.a (mon.0) 1132 : cluster [DBG] osdmap e699: 8 total, 8 up, 8 in 2024-09-12T23:49:21.344 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:49:21 smithi181 bash[95390]: ceph-da58a47c-715d-11ef-bceb-c7b262605968-prometheus-a 2024-09-12T23:49:21.344 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:49:21 smithi181 bash[95450]: Error response from daemon: No such container: ceph-da58a47c-715d-11ef-bceb-c7b262605968-prometheus-a 2024-09-12T23:49:21.345 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:49:21 smithi181 systemd[1]: ceph-da58a47c-715d-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-12T23:49:21.345 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:49:21 smithi181 systemd[1]: Stopped Ceph prometheus.a for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:49:21.492 INFO:tasks.rados.rados.0.smithi136.stdout:3188: done (4 left) 2024-09-12T23:49:21.492 INFO:tasks.rados.rados.0.smithi136.stdout:3189: done (3 left) 2024-09-12T23:49:21.492 INFO:tasks.rados.rados.0.smithi136.stdout:3190: done (2 left) 2024-09-12T23:49:21.492 INFO:tasks.rados.rados.0.smithi136.stdout:3191: done (1 left) 2024-09-12T23:49:21.492 INFO:tasks.rados.rados.0.smithi136.stdout:3193: done (0 left) 2024-09-12T23:49:21.492 INFO:tasks.rados.rados.0.smithi136.stdout:3194: read oid 5 snap -1 2024-09-12T23:49:21.492 INFO:tasks.rados.rados.0.smithi136.stdout:3194: expect (ObjNum 965 snap 301 seq_num 965) 2024-09-12T23:49:21.492 INFO:tasks.rados.rados.0.smithi136.stdout:3195: read oid 41 snap -1 2024-09-12T23:49:21.492 INFO:tasks.rados.rados.0.smithi136.stdout:3195: expect (ObjNum 851 snap 272 seq_num 851) 2024-09-12T23:49:21.493 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 27 current snap is 313 2024-09-12T23:49:21.493 INFO:tasks.rados.rados.0.smithi136.stdout:3196: seq_num 1004 ranges {1081344=16384} 2024-09-12T23:49:21.500 INFO:tasks.rados.rados.0.smithi136.stdout:3196: writing smithi13667081-27 from 1081344 to 1097728 tid 1 2024-09-12T23:49:21.500 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 28 current snap is 313 2024-09-12T23:49:21.501 INFO:tasks.rados.rados.0.smithi136.stdout:3197: seq_num 1005 ranges {1769472=32768} 2024-09-12T23:49:21.513 INFO:tasks.rados.rados.0.smithi136.stdout:3197: writing smithi13667081-28 from 1769472 to 1802240 tid 1 2024-09-12T23:49:21.513 INFO:tasks.rados.rados.0.smithi136.stdout:3198: rollback oid 35 current snap is 313 2024-09-12T23:49:21.513 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 35 to 295 2024-09-12T23:49:21.513 INFO:tasks.rados.rados.0.smithi136.stdout:3194: expect (ObjNum 965 snap 301 seq_num 965) 2024-09-12T23:49:21.616 INFO:tasks.rados.rados.0.smithi136.stdout:3195: expect (ObjNum 851 snap 272 seq_num 851) 2024-09-12T23:49:21.744 INFO:tasks.rados.rados.0.smithi136.stdout:3197: finishing write tid 1 to smithi13667081-28 2024-09-12T23:49:21.745 INFO:tasks.rados.rados.0.smithi136.stdout:3197: finishing write tid 2 to smithi13667081-28 2024-09-12T23:49:21.745 INFO:tasks.rados.rados.0.smithi136.stdout:3196: finishing write tid 1 to smithi13667081-27 2024-09-12T23:49:21.745 INFO:tasks.rados.rados.0.smithi136.stdout:3196: finishing write tid 2 to smithi13667081-27 2024-09-12T23:49:21.745 INFO:tasks.rados.rados.0.smithi136.stdout:3197: finishing write tid 3 to smithi13667081-28 2024-09-12T23:49:21.745 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 28 v 1987 (ObjNum 1005 snap 313 seq_num 1005) dirty exists 2024-09-12T23:49:21.745 INFO:tasks.rados.rados.0.smithi136.stdout:3197: left oid 28 (ObjNum 1005 snap 313 seq_num 1005) 2024-09-12T23:49:21.745 INFO:tasks.rados.rados.0.smithi136.stdout:3198: finishing rollback tid 0 to smithi13667081-35 2024-09-12T23:49:21.745 INFO:tasks.rados.rados.0.smithi136.stdout:3198: finishing rollback tid 1 to smithi13667081-35 2024-09-12T23:49:21.745 INFO:tasks.rados.rados.0.smithi136.stdout:3198: finishing rollback tid 2 to smithi13667081-35 2024-09-12T23:49:21.745 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 35 v 2457 (ObjNum 927 snap 290 seq_num 927) dirty exists 2024-09-12T23:49:21.745 INFO:tasks.rados.rados.0.smithi136.stdout:3196: finishing write tid 3 to smithi13667081-27 2024-09-12T23:49:21.745 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 27 v 2453 (ObjNum 1004 snap 313 seq_num 1004) dirty exists 2024-09-12T23:49:21.745 INFO:tasks.rados.rados.0.smithi136.stdout:3196: left oid 27 (ObjNum 1004 snap 313 seq_num 1004) 2024-09-12T23:49:21.745 INFO:tasks.rados.rados.0.smithi136.stdout:3194: done (4 left) 2024-09-12T23:49:21.746 INFO:tasks.rados.rados.0.smithi136.stdout:3195: done (3 left) 2024-09-12T23:49:21.746 INFO:tasks.rados.rados.0.smithi136.stdout:3196: done (2 left) 2024-09-12T23:49:21.746 INFO:tasks.rados.rados.0.smithi136.stdout:3197: done (1 left) 2024-09-12T23:49:21.746 INFO:tasks.rados.rados.0.smithi136.stdout:3198: done (0 left) 2024-09-12T23:49:21.746 INFO:tasks.rados.rados.0.smithi136.stdout:3199: read oid 27 snap -1 2024-09-12T23:49:21.746 INFO:tasks.rados.rados.0.smithi136.stdout:3199: expect (ObjNum 1004 snap 313 seq_num 1004) 2024-09-12T23:49:21.746 INFO:tasks.rados.rados.0.smithi136.stdout:3200: read oid 4 snap -1 2024-09-12T23:49:21.746 INFO:tasks.rados.rados.0.smithi136.stdout:3200: expect (ObjNum 949 snap 301 seq_num 949) 2024-09-12T23:49:21.746 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 32 current snap is 313 2024-09-12T23:49:21.746 INFO:tasks.rados.rados.0.smithi136.stdout:3201: seq_num 1006 ranges {819200=49152} 2024-09-12T23:49:21.754 INFO:tasks.rados.rados.0.smithi136.stdout:3201: writing smithi13667081-32 from 819200 to 868352 tid 1 2024-09-12T23:49:21.754 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 43 current snap is 313 2024-09-12T23:49:21.754 INFO:tasks.rados.rados.0.smithi136.stdout:3202: seq_num 1007 ranges {3850240=65536} 2024-09-12T23:49:21.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:21 smithi136 bash[76371]: cluster 2024-09-12T23:49:20.483120+0000 mon.a (mon.0) 1133 : cluster [DBG] osdmap e700: 8 total, 8 up, 8 in 2024-09-12T23:49:21.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:21 smithi136 bash[76371]: cluster 2024-09-12T23:49:20.822974+0000 mgr.y (mgr.24661) 351 : cluster [DBG] pgmap v628: 121 pgs: 121 active+clean; 228 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 31 MiB/s rd, 2.4 MiB/s wr, 19 op/s 2024-09-12T23:49:21.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:21 smithi136 bash[75164]: cluster 2024-09-12T23:49:20.483120+0000 mon.a (mon.0) 1133 : cluster [DBG] osdmap e700: 8 total, 8 up, 8 in 2024-09-12T23:49:21.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:21 smithi136 bash[75164]: cluster 2024-09-12T23:49:20.822974+0000 mgr.y (mgr.24661) 351 : cluster [DBG] pgmap v628: 121 pgs: 121 active+clean; 228 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 31 MiB/s rd, 2.4 MiB/s wr, 19 op/s 2024-09-12T23:49:21.794 INFO:tasks.rados.rados.0.smithi136.stdout:3202: writing smithi13667081-43 from 3850240 to 3915776 tid 1 2024-09-12T23:49:21.795 INFO:tasks.rados.rados.0.smithi136.stdout:3203: copy_from oid 18 from oid 36 current snap is 313 2024-09-12T23:49:21.795 INFO:tasks.rados.rados.0.smithi136.stdout:3199: expect (ObjNum 1004 snap 313 seq_num 1004) 2024-09-12T23:49:21.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:21 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:49:21.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:21 smithi181 bash[67587]: cluster 2024-09-12T23:49:20.483120+0000 mon.a (mon.0) 1133 : cluster [DBG] osdmap e700: 8 total, 8 up, 8 in 2024-09-12T23:49:21.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:21 smithi181 bash[67587]: cluster 2024-09-12T23:49:20.822974+0000 mgr.y (mgr.24661) 351 : cluster [DBG] pgmap v628: 121 pgs: 121 active+clean; 228 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 31 MiB/s rd, 2.4 MiB/s wr, 19 op/s 2024-09-12T23:49:21.845 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:49:21 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:49:21.845 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 12 23:49:21 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:49:21.846 INFO:journalctl@ceph.mds.b.smithi181.stdout:Sep 12 23:49:21 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:49:21.846 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:49:21 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:49:21.846 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:49:21 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:49:21.846 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:49:21 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:49:21.847 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 12 23:49:21 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:49:21.847 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:49:21 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:49:21.847 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:49:21 smithi181 systemd[1]: Started Ceph prometheus.a for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:49:21.847 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:49:21 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:49:21.895 INFO:tasks.rados.rados.0.smithi136.stdout:3202: finishing write tid 1 to smithi13667081-43 2024-09-12T23:49:21.895 INFO:tasks.rados.rados.0.smithi136.stdout:3202: finishing write tid 2 to smithi13667081-43 2024-09-12T23:49:21.895 INFO:tasks.rados.rados.0.smithi136.stdout:3202: finishing write tid 3 to smithi13667081-43 2024-09-12T23:49:21.895 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 43 v 1990 (ObjNum 1007 snap 313 seq_num 1007) dirty exists 2024-09-12T23:49:21.895 INFO:tasks.rados.rados.0.smithi136.stdout:3202: left oid 43 (ObjNum 1007 snap 313 seq_num 1007) 2024-09-12T23:49:21.895 INFO:tasks.rados.rados.0.smithi136.stdout:3203: finishing copy_from racing read to smithi13667081-18 2024-09-12T23:49:21.896 INFO:tasks.rados.rados.0.smithi136.stdout:3199: done (4 left) 2024-09-12T23:49:21.896 INFO:tasks.rados.rados.0.smithi136.stdout:3202: done (3 left) 2024-09-12T23:49:21.896 INFO:tasks.rados.rados.0.smithi136.stdout:3204: copy_from oid 42 from oid 35 current snap is 313 2024-09-12T23:49:21.896 INFO:tasks.rados.rados.0.smithi136.stdout:3203: finishing copy_from to smithi13667081-18 2024-09-12T23:49:21.896 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 18 v 1633 (ObjNum 965 snap 301 seq_num 965) dirty exists 2024-09-12T23:49:21.896 INFO:tasks.rados.rados.0.smithi136.stdout:3200: expect (ObjNum 949 snap 301 seq_num 949) 2024-09-12T23:49:22.738 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:22 smithi181 bash[67587]: cluster 2024-09-12T23:49:21.493968+0000 mon.a (mon.0) 1134 : cluster [DBG] osdmap e701: 8 total, 8 up, 8 in 2024-09-12T23:49:22.739 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:22 smithi181 bash[67587]: audit 2024-09-12T23:49:21.521875+0000 mon.a (mon.0) 1135 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:22.739 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:22 smithi181 bash[67587]: audit 2024-09-12T23:49:21.531452+0000 mon.a (mon.0) 1136 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:22.739 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:22 smithi181 bash[67587]: audit 2024-09-12T23:49:21.532926+0000 mon.a (mon.0) 1137 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:49:22.739 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:49:22 smithi181 bash[95505]: ts=2024-09-12T23:49:22.708Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-12T23:49:22.739 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:49:22 smithi181 bash[95505]: ts=2024-09-12T23:49:22.708Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-12T23:49:22.739 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:49:22 smithi181 bash[95505]: ts=2024-09-12T23:49:22.708Z caller=main.go:570 level=info host_details="(Linux 5.15.0-121-generic #131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024 x86_64 smithi181 (none))" 2024-09-12T23:49:22.739 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:49:22 smithi181 bash[95505]: ts=2024-09-12T23:49:22.708Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-12T23:49:22.739 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:49:22 smithi181 bash[95505]: ts=2024-09-12T23:49:22.708Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-12T23:49:22.739 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:49:22 smithi181 bash[95505]: ts=2024-09-12T23:49:22.710Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-12T23:49:22.739 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:49:22 smithi181 bash[95505]: ts=2024-09-12T23:49:22.711Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-12T23:49:22.740 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:49:22 smithi181 bash[95505]: ts=2024-09-12T23:49:22.711Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-12T23:49:22.740 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:49:22 smithi181 bash[95505]: ts=2024-09-12T23:49:22.712Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-12T23:49:22.740 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:49:22 smithi181 bash[95505]: ts=2024-09-12T23:49:22.714Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-12T23:49:22.740 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:49:22 smithi181 bash[95505]: ts=2024-09-12T23:49:22.714Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=3.339µs 2024-09-12T23:49:22.740 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:49:22 smithi181 bash[95505]: ts=2024-09-12T23:49:22.714Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-12T23:49:22.740 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:49:22 smithi181 bash[95505]: ts=2024-09-12T23:49:22.715Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=4 2024-09-12T23:49:22.740 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:49:22 smithi181 bash[95505]: ts=2024-09-12T23:49:22.736Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=4 2024-09-12T23:49:22.740 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:49:22 smithi181 bash[95505]: ts=2024-09-12T23:49:22.743Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=4 2024-09-12T23:49:22.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:22 smithi136 bash[75164]: cluster 2024-09-12T23:49:21.493968+0000 mon.a (mon.0) 1134 : cluster [DBG] osdmap e701: 8 total, 8 up, 8 in 2024-09-12T23:49:22.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:22 smithi136 bash[75164]: audit 2024-09-12T23:49:21.521875+0000 mon.a (mon.0) 1135 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:22.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:22 smithi136 bash[75164]: audit 2024-09-12T23:49:21.531452+0000 mon.a (mon.0) 1136 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:22.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:22 smithi136 bash[75164]: audit 2024-09-12T23:49:21.532926+0000 mon.a (mon.0) 1137 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:49:22.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:22 smithi136 bash[76371]: cluster 2024-09-12T23:49:21.493968+0000 mon.a (mon.0) 1134 : cluster [DBG] osdmap e701: 8 total, 8 up, 8 in 2024-09-12T23:49:22.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:22 smithi136 bash[76371]: audit 2024-09-12T23:49:21.521875+0000 mon.a (mon.0) 1135 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:22.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:22 smithi136 bash[76371]: audit 2024-09-12T23:49:21.531452+0000 mon.a (mon.0) 1136 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:22.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:22 smithi136 bash[76371]: audit 2024-09-12T23:49:21.532926+0000 mon.a (mon.0) 1137 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:49:23.095 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:49:22 smithi181 bash[95505]: ts=2024-09-12T23:49:22.752Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=4 2024-09-12T23:49:23.095 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:49:22 smithi181 bash[95505]: ts=2024-09-12T23:49:22.752Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=4 2024-09-12T23:49:23.095 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:49:22 smithi181 bash[95505]: ts=2024-09-12T23:49:22.752Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=43.411µs wal_replay_duration=38.373303ms wbl_replay_duration=121ns total_replay_duration=38.4655ms 2024-09-12T23:49:23.095 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:49:22 smithi181 bash[95505]: ts=2024-09-12T23:49:22.756Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-12T23:49:23.095 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:49:22 smithi181 bash[95505]: ts=2024-09-12T23:49:22.756Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-12T23:49:23.095 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:49:22 smithi181 bash[95505]: ts=2024-09-12T23:49:22.756Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-12T23:49:23.095 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:49:22 smithi181 bash[95505]: ts=2024-09-12T23:49:22.777Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=20.51835ms db_storage=977ns remote_storage=1.722µs web_handler=479ns query_engine=781ns scrape=282.344µs scrape_sd=121.654µs notify=19.307µs notify_sd=17.687µs rules=19.712248ms tracing=5.749µs 2024-09-12T23:49:23.096 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:49:22 smithi181 bash[95505]: ts=2024-09-12T23:49:22.777Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-12T23:49:23.096 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:49:22 smithi181 bash[95505]: ts=2024-09-12T23:49:22.777Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-12T23:49:23.555 INFO:tasks.rados.rados.0.smithi136.stdout:3201: finishing write tid 1 to smithi13667081-32 2024-09-12T23:49:23.555 INFO:tasks.rados.rados.0.smithi136.stdout:3201: finishing write tid 2 to smithi13667081-32 2024-09-12T23:49:23.555 INFO:tasks.rados.rados.0.smithi136.stdout:3201: finishing write tid 3 to smithi13667081-32 2024-09-12T23:49:23.555 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 32 v 1638 (ObjNum 1006 snap 313 seq_num 1006) dirty exists 2024-09-12T23:49:23.555 INFO:tasks.rados.rados.0.smithi136.stdout:3201: left oid 32 (ObjNum 1006 snap 313 seq_num 1006) 2024-09-12T23:49:23.556 INFO:tasks.rados.rados.0.smithi136.stdout:3204: finishing copy_from racing read to smithi13667081-42 2024-09-12T23:49:23.556 INFO:tasks.rados.rados.0.smithi136.stdout:3204: finishing copy_from to smithi13667081-42 2024-09-12T23:49:23.556 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 42 v 2457 (ObjNum 927 snap 290 seq_num 927) dirty exists 2024-09-12T23:49:23.556 INFO:tasks.rados.rados.0.smithi136.stdout:3200: done (3 left) 2024-09-12T23:49:23.556 INFO:tasks.rados.rados.0.smithi136.stdout:3201: done (2 left) 2024-09-12T23:49:23.556 INFO:tasks.rados.rados.0.smithi136.stdout:3203: done (1 left) 2024-09-12T23:49:23.556 INFO:tasks.rados.rados.0.smithi136.stdout:3204: done (0 left) 2024-09-12T23:49:23.556 INFO:tasks.rados.rados.0.smithi136.stdout:3205: snap_remove snap 306 2024-09-12T23:49:23.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:23 smithi136 bash[75164]: cluster 2024-09-12T23:49:22.823976+0000 mgr.y (mgr.24661) 352 : cluster [DBG] pgmap v630: 121 pgs: 121 active+clean; 232 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 1.3 MiB/s wr, 10 op/s 2024-09-12T23:49:23.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:23 smithi136 bash[76371]: cluster 2024-09-12T23:49:22.823976+0000 mgr.y (mgr.24661) 352 : cluster [DBG] pgmap v630: 121 pgs: 121 active+clean; 232 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 1.3 MiB/s wr, 10 op/s 2024-09-12T23:49:23.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:23 smithi181 bash[67587]: cluster 2024-09-12T23:49:22.823976+0000 mgr.y (mgr.24661) 352 : cluster [DBG] pgmap v630: 121 pgs: 121 active+clean; 232 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 1.3 MiB/s wr, 10 op/s 2024-09-12T23:49:24.513 INFO:tasks.rados.rados.0.smithi136.stdout:3205: done (0 left) 2024-09-12T23:49:24.513 INFO:tasks.rados.rados.0.smithi136.stdout:3206: copy_from oid 24 from oid 46 current snap is 313 2024-09-12T23:49:24.513 INFO:tasks.rados.rados.0.smithi136.stdout:3207: rmattr oid 42 current snap is 313 2024-09-12T23:49:24.513 INFO:tasks.rados.rados.0.smithi136.stdout:3208: rollback oid 23 current snap is 313 2024-09-12T23:49:24.514 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 23 to 296 2024-09-12T23:49:24.514 INFO:tasks.rados.rados.0.smithi136.stdout:3209: copy_from oid 1 from oid 26 current snap is 313 2024-09-12T23:49:24.514 INFO:tasks.rados.rados.0.smithi136.stdout:3210: read oid 31 snap -1 2024-09-12T23:49:24.514 INFO:tasks.rados.rados.0.smithi136.stdout:3210: expect deleted 2024-09-12T23:49:24.514 INFO:tasks.rados.rados.0.smithi136.stdout:3211: setattr oid 8 current snap is 313 2024-09-12T23:49:24.515 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 21 current snap is 313 2024-09-12T23:49:24.515 INFO:tasks.rados.rados.0.smithi136.stdout:3212: seq_num 1008 ranges {3211264=90112} 2024-09-12T23:49:24.536 INFO:tasks.rados.rados.0.smithi136.stdout:3212: writing smithi13667081-21 from 3211264 to 3301376 tid 1 2024-09-12T23:49:24.536 INFO:tasks.rados.rados.0.smithi136.stdout:3213: read oid 25 snap -1 2024-09-12T23:49:24.536 INFO:tasks.rados.rados.0.smithi136.stdout:3213: expect (ObjNum 998 snap 309 seq_num 998) 2024-09-12T23:49:24.536 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 20 current snap is 313 2024-09-12T23:49:24.536 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 8 v 2337 (ObjNum 945 snap 298 seq_num 945) dirty exists 2024-09-12T23:49:24.536 INFO:tasks.rados.rados.0.smithi136.stdout:3214: seq_num 1009 ranges {2310144=106496} 2024-09-12T23:49:24.557 INFO:tasks.rados.rados.0.smithi136.stdout:3214: writing smithi13667081-20 from 2310144 to 2416640 tid 1 2024-09-12T23:49:24.557 INFO:tasks.rados.rados.0.smithi136.stdout:3210: done (8 left) 2024-09-12T23:49:24.558 INFO:tasks.rados.rados.0.smithi136.stdout:3211: done (7 left) 2024-09-12T23:49:24.558 INFO:tasks.rados.rados.0.smithi136.stdout:3215: delete oid 19 current snap is 313 2024-09-12T23:49:24.558 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 42 v 2458 (ObjNum 927 snap 290 seq_num 927) dirty exists 2024-09-12T23:49:24.558 INFO:tasks.rados.rados.0.smithi136.stdout:3208: finishing rollback tid 0 to smithi13667081-23 2024-09-12T23:49:24.558 INFO:tasks.rados.rados.0.smithi136.stdout:3208: finishing rollback tid 1 to smithi13667081-23 2024-09-12T23:49:24.558 INFO:tasks.rados.rados.0.smithi136.stdout:3208: finishing rollback tid 2 to smithi13667081-23 2024-09-12T23:49:24.558 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 23 v 2465 (ObjNum 886 snap 287 seq_num 886) dirty exists 2024-09-12T23:49:24.558 INFO:tasks.rados.rados.0.smithi136.stdout:3212: finishing write tid 1 to smithi13667081-21 2024-09-12T23:49:24.558 INFO:tasks.rados.rados.0.smithi136.stdout:3212: finishing write tid 2 to smithi13667081-21 2024-09-12T23:49:24.559 INFO:tasks.rados.rados.0.smithi136.stdout:3212: finishing write tid 3 to smithi13667081-21 2024-09-12T23:49:24.559 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 21 v 2460 (ObjNum 1008 snap 313 seq_num 1008) dirty exists 2024-09-12T23:49:24.559 INFO:tasks.rados.rados.0.smithi136.stdout:3212: left oid 21 (ObjNum 1008 snap 313 seq_num 1008) 2024-09-12T23:49:24.565 INFO:tasks.rados.rados.0.smithi136.stdout:3207: done (7 left) 2024-09-12T23:49:24.565 INFO:tasks.rados.rados.0.smithi136.stdout:3208: done (6 left) 2024-09-12T23:49:24.565 INFO:tasks.rados.rados.0.smithi136.stdout:3212: done (5 left) 2024-09-12T23:49:24.565 INFO:tasks.rados.rados.0.smithi136.stdout:3215: done (4 left) 2024-09-12T23:49:24.565 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 21 current snap is 313 2024-09-12T23:49:24.565 INFO:tasks.rados.rados.0.smithi136.stdout:3216: seq_num 1011 ranges {3301376=139264} 2024-09-12T23:49:24.585 INFO:tasks.rados.rados.0.smithi136.stdout:3216: writing smithi13667081-21 from 3301376 to 3440640 tid 1 2024-09-12T23:49:24.585 INFO:tasks.rados.rados.0.smithi136.stdout:3217: read oid 43 snap 303 2024-09-12T23:49:24.585 INFO:tasks.rados.rados.0.smithi136.stdout:3217: expect (ObjNum 985 snap 303 seq_num 985) 2024-09-12T23:49:24.585 INFO:tasks.rados.rados.0.smithi136.stdout:3218: snap_create 2024-09-12T23:49:24.585 INFO:tasks.rados.rados.0.smithi136.stdout:3214: finishing write tid 1 to smithi13667081-20 2024-09-12T23:49:24.586 INFO:tasks.rados.rados.0.smithi136.stdout:3214: finishing write tid 2 to smithi13667081-20 2024-09-12T23:49:24.586 INFO:tasks.rados.rados.0.smithi136.stdout:3214: finishing write tid 3 to smithi13667081-20 2024-09-12T23:49:24.586 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 20 v 1624 (ObjNum 1009 snap 313 seq_num 1009) dirty exists 2024-09-12T23:49:24.586 INFO:tasks.rados.rados.0.smithi136.stdout:3214: left oid 20 (ObjNum 1009 snap 313 seq_num 1009) 2024-09-12T23:49:24.586 INFO:tasks.rados.rados.0.smithi136.stdout:3206: finishing copy_from racing read to smithi13667081-24 2024-09-12T23:49:24.586 INFO:tasks.rados.rados.0.smithi136.stdout:3213: expect (ObjNum 998 snap 309 seq_num 998) 2024-09-12T23:49:24.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:24 smithi136 bash[75164]: audit 2024-09-12T23:49:23.751330+0000 mon.a (mon.0) 1138 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:49:24.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:24 smithi136 bash[76371]: audit 2024-09-12T23:49:23.751330+0000 mon.a (mon.0) 1138 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:49:24.819 INFO:tasks.rados.rados.0.smithi136.stdout:3206: finishing copy_from to smithi13667081-24 2024-09-12T23:49:24.819 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 24 v 1524 (ObjNum 1003 snap 312 seq_num 1003) dirty exists 2024-09-12T23:49:24.819 INFO:tasks.rados.rados.0.smithi136.stdout:3209: finishing copy_from racing read to smithi13667081-1 2024-09-12T23:49:24.819 INFO:tasks.rados.rados.0.smithi136.stdout:3217: expect (ObjNum 985 snap 303 seq_num 985) 2024-09-12T23:49:24.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:24 smithi181 bash[67587]: audit 2024-09-12T23:49:23.751330+0000 mon.a (mon.0) 1138 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:49:24.955 INFO:tasks.rados.rados.0.smithi136.stdout:3209: finishing copy_from to smithi13667081-1 2024-09-12T23:49:24.955 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 1 v 1504 (ObjNum 987 snap 303 seq_num 987) dirty exists 2024-09-12T23:49:24.956 INFO:tasks.rados.rados.0.smithi136.stdout:3216: finishing write tid 1 to smithi13667081-21 2024-09-12T23:49:24.956 INFO:tasks.rados.rados.0.smithi136.stdout:3216: finishing write tid 2 to smithi13667081-21 2024-09-12T23:49:24.956 INFO:tasks.rados.rados.0.smithi136.stdout:3216: finishing write tid 3 to smithi13667081-21 2024-09-12T23:49:24.956 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 21 v 2462 (ObjNum 1011 snap 313 seq_num 1011) dirty exists 2024-09-12T23:49:24.956 INFO:tasks.rados.rados.0.smithi136.stdout:3216: left oid 21 (ObjNum 1011 snap 313 seq_num 1011) 2024-09-12T23:49:25.525 INFO:tasks.rados.rados.0.smithi136.stdout:3206: done (6 left) 2024-09-12T23:49:25.526 INFO:tasks.rados.rados.0.smithi136.stdout:3209: done (5 left) 2024-09-12T23:49:25.526 INFO:tasks.rados.rados.0.smithi136.stdout:3213: done (4 left) 2024-09-12T23:49:25.526 INFO:tasks.rados.rados.0.smithi136.stdout:3214: done (3 left) 2024-09-12T23:49:25.526 INFO:tasks.rados.rados.0.smithi136.stdout:3216: done (2 left) 2024-09-12T23:49:25.526 INFO:tasks.rados.rados.0.smithi136.stdout:3217: done (1 left) 2024-09-12T23:49:25.526 INFO:tasks.rados.rados.0.smithi136.stdout:3218: done (0 left) 2024-09-12T23:49:25.526 INFO:tasks.rados.rados.0.smithi136.stdout:3219: copy_from oid 24 from oid 15 current snap is 314 2024-09-12T23:49:25.526 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 8 current snap is 314 2024-09-12T23:49:25.526 INFO:tasks.rados.rados.0.smithi136.stdout:3220: seq_num 1012 ranges {1433600=155648} 2024-09-12T23:49:25.537 INFO:tasks.rados.rados.0.smithi136.stdout:3220: writing smithi13667081-8 from 1433600 to 1589248 tid 1 2024-09-12T23:49:25.537 INFO:tasks.rados.rados.0.smithi136.stdout:3221: delete oid 50 current snap is 314 2024-09-12T23:49:25.548 INFO:tasks.rados.rados.0.smithi136.stdout:3221: done (2 left) 2024-09-12T23:49:25.548 INFO:tasks.rados.rados.0.smithi136.stdout:3222: read oid 4 snap -1 2024-09-12T23:49:25.548 INFO:tasks.rados.rados.0.smithi136.stdout:3222: expect (ObjNum 949 snap 301 seq_num 949) 2024-09-12T23:49:25.548 INFO:tasks.rados.rados.0.smithi136.stdout:3223: copy_from oid 34 from oid 47 current snap is 314 2024-09-12T23:49:25.549 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 3 current snap is 314 2024-09-12T23:49:25.549 INFO:tasks.rados.rados.0.smithi136.stdout:3224: seq_num 1014 ranges {2048000=188416} 2024-09-12T23:49:25.566 INFO:tasks.rados.rados.0.smithi136.stdout:3224: writing smithi13667081-3 from 2048000 to 2236416 tid 1 2024-09-12T23:49:25.566 INFO:tasks.rados.rados.0.smithi136.stdout:3225: read oid 42 snap -1 2024-09-12T23:49:25.566 INFO:tasks.rados.rados.0.smithi136.stdout:3225: expect (ObjNum 927 snap 290 seq_num 927) 2024-09-12T23:49:25.566 INFO:tasks.rados.rados.0.smithi136.stdout:3226: delete oid 27 current snap is 314 2024-09-12T23:49:25.566 INFO:tasks.rados.rados.0.smithi136.stdout:3220: finishing write tid 1 to smithi13667081-8 2024-09-12T23:49:25.566 INFO:tasks.rados.rados.0.smithi136.stdout:3220: finishing write tid 2 to smithi13667081-8 2024-09-12T23:49:25.571 INFO:tasks.rados.rados.0.smithi136.stdout:3226: done (6 left) 2024-09-12T23:49:25.571 INFO:tasks.rados.rados.0.smithi136.stdout:3227: read oid 35 snap -1 2024-09-12T23:49:25.571 INFO:tasks.rados.rados.0.smithi136.stdout:3227: expect (ObjNum 927 snap 290 seq_num 927) 2024-09-12T23:49:25.571 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 11 current snap is 314 2024-09-12T23:49:25.571 INFO:tasks.rados.rados.0.smithi136.stdout:3228: seq_num 1016 ranges {0=221184} 2024-09-12T23:49:25.573 INFO:tasks.rados.rados.0.smithi136.stdout:3228: writing smithi13667081-11 from 0 to 221184 tid 1 2024-09-12T23:49:25.573 INFO:tasks.rados.rados.0.smithi136.stdout:3229: copy_from oid 37 from oid 46 current snap is 314 2024-09-12T23:49:25.573 INFO:tasks.rados.rados.0.smithi136.stdout:3230: rmattr oid 17 current snap is 314 2024-09-12T23:49:25.573 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 6 current snap is 314 2024-09-12T23:49:25.573 INFO:tasks.rados.rados.0.smithi136.stdout:3231: seq_num 1017 ranges {0=237568} 2024-09-12T23:49:25.575 INFO:tasks.rados.rados.0.smithi136.stdout:3231: writing smithi13667081-6 from 0 to 237568 tid 1 2024-09-12T23:49:25.575 INFO:tasks.rados.rados.0.smithi136.stdout:3232: snap_create 2024-09-12T23:49:25.577 INFO:tasks.rados.rados.0.smithi136.stdout:3224: finishing write tid 1 to smithi13667081-3 2024-09-12T23:49:25.580 INFO:tasks.rados.rados.0.smithi136.stdout:3225: expect (ObjNum 927 snap 290 seq_num 927) 2024-09-12T23:49:25.628 INFO:tasks.rados.rados.0.smithi136.stdout:3231: finishing write tid 1 to smithi13667081-6 2024-09-12T23:49:25.628 INFO:tasks.rados.rados.0.smithi136.stdout:3231: finishing write tid 2 to smithi13667081-6 2024-09-12T23:49:25.628 INFO:tasks.rados.rados.0.smithi136.stdout:3224: finishing write tid 2 to smithi13667081-3 2024-09-12T23:49:25.629 INFO:tasks.rados.rados.0.smithi136.stdout:3231: finishing write tid 3 to smithi13667081-6 2024-09-12T23:49:25.629 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 6 v 1626 (ObjNum 1017 snap 314 seq_num 1017) dirty exists 2024-09-12T23:49:25.629 INFO:tasks.rados.rados.0.smithi136.stdout:3231: left oid 6 (ObjNum 1017 snap 314 seq_num 1017) 2024-09-12T23:49:25.629 INFO:tasks.rados.rados.0.smithi136.stdout:3228: finishing write tid 1 to smithi13667081-11 2024-09-12T23:49:25.629 INFO:tasks.rados.rados.0.smithi136.stdout:3223: finishing copy_from racing read to smithi13667081-34 2024-09-12T23:49:25.629 INFO:tasks.rados.rados.0.smithi136.stdout:3227: expect (ObjNum 927 snap 290 seq_num 927) 2024-09-12T23:49:25.662 INFO:tasks.rados.rados.0.smithi136.stdout:3228: finishing write tid 2 to smithi13667081-11 2024-09-12T23:49:25.662 INFO:tasks.rados.rados.0.smithi136.stdout:3224: finishing write tid 3 to smithi13667081-3 2024-09-12T23:49:25.662 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 3 v 1832 (ObjNum 1014 snap 314 seq_num 1014) dirty exists 2024-09-12T23:49:25.662 INFO:tasks.rados.rados.0.smithi136.stdout:3224: left oid 3 (ObjNum 1014 snap 314 seq_num 1014) 2024-09-12T23:49:25.662 INFO:tasks.rados.rados.0.smithi136.stdout:3228: finishing write tid 3 to smithi13667081-11 2024-09-12T23:49:25.662 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 11 v 2471 (ObjNum 1016 snap 314 seq_num 1016) dirty exists 2024-09-12T23:49:25.662 INFO:tasks.rados.rados.0.smithi136.stdout:3228: left oid 11 (ObjNum 1016 snap 314 seq_num 1016) 2024-09-12T23:49:25.663 INFO:tasks.rados.rados.0.smithi136.stdout:3223: finishing copy_from to smithi13667081-34 2024-09-12T23:49:25.663 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 34 v 1639 (ObjNum 990 snap 304 seq_num 990) dirty exists 2024-09-12T23:49:25.663 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 17 v 2473 (ObjNum 49 snap 0 seq_num 1817208417) dirty exists 2024-09-12T23:49:25.663 INFO:tasks.rados.rados.0.smithi136.stdout:3220: finishing write tid 3 to smithi13667081-8 2024-09-12T23:49:25.663 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 8 v 2339 (ObjNum 1012 snap 314 seq_num 1012) dirty exists 2024-09-12T23:49:25.663 INFO:tasks.rados.rados.0.smithi136.stdout:3220: left oid 8 (ObjNum 1012 snap 314 seq_num 1012) 2024-09-12T23:49:25.663 INFO:tasks.rados.rados.0.smithi136.stdout:3229: finishing copy_from racing read to smithi13667081-37 2024-09-12T23:49:25.663 INFO:tasks.rados.rados.0.smithi136.stdout:3229: finishing copy_from to smithi13667081-37 2024-09-12T23:49:25.663 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 37 v 1524 (ObjNum 1003 snap 312 seq_num 1003) dirty exists 2024-09-12T23:49:25.663 INFO:tasks.rados.rados.0.smithi136.stdout:3222: expect (ObjNum 949 snap 301 seq_num 949) 2024-09-12T23:49:25.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:25 smithi181 bash[67587]: cluster 2024-09-12T23:49:24.512667+0000 mon.a (mon.0) 1139 : cluster [DBG] osdmap e702: 8 total, 8 up, 8 in 2024-09-12T23:49:25.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:25 smithi181 bash[67587]: cluster 2024-09-12T23:49:24.824830+0000 mgr.y (mgr.24661) 353 : cluster [DBG] pgmap v632: 121 pgs: 121 active+clean; 233 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 1.1 MiB/s wr, 9 op/s 2024-09-12T23:49:26.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:25 smithi136 bash[75164]: cluster 2024-09-12T23:49:24.512667+0000 mon.a (mon.0) 1139 : cluster [DBG] osdmap e702: 8 total, 8 up, 8 in 2024-09-12T23:49:26.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:25 smithi136 bash[75164]: cluster 2024-09-12T23:49:24.824830+0000 mgr.y (mgr.24661) 353 : cluster [DBG] pgmap v632: 121 pgs: 121 active+clean; 233 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 1.1 MiB/s wr, 9 op/s 2024-09-12T23:49:26.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:25 smithi136 bash[76371]: cluster 2024-09-12T23:49:24.512667+0000 mon.a (mon.0) 1139 : cluster [DBG] osdmap e702: 8 total, 8 up, 8 in 2024-09-12T23:49:26.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:25 smithi136 bash[76371]: cluster 2024-09-12T23:49:24.824830+0000 mgr.y (mgr.24661) 353 : cluster [DBG] pgmap v632: 121 pgs: 121 active+clean; 233 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 1.1 MiB/s wr, 9 op/s 2024-09-12T23:49:26.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:26 smithi181 bash[67587]: cluster 2024-09-12T23:49:25.526298+0000 mon.a (mon.0) 1140 : cluster [DBG] osdmap e703: 8 total, 8 up, 8 in 2024-09-12T23:49:26.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:26 smithi181 bash[67587]: cluster 2024-09-12T23:49:26.463334+0000 mon.a (mon.0) 1141 : cluster [DBG] osdmap e704: 8 total, 8 up, 8 in 2024-09-12T23:49:27.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:26 smithi136 bash[75164]: cluster 2024-09-12T23:49:25.526298+0000 mon.a (mon.0) 1140 : cluster [DBG] osdmap e703: 8 total, 8 up, 8 in 2024-09-12T23:49:27.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:26 smithi136 bash[75164]: cluster 2024-09-12T23:49:26.463334+0000 mon.a (mon.0) 1141 : cluster [DBG] osdmap e704: 8 total, 8 up, 8 in 2024-09-12T23:49:27.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:26 smithi136 bash[76371]: cluster 2024-09-12T23:49:25.526298+0000 mon.a (mon.0) 1140 : cluster [DBG] osdmap e703: 8 total, 8 up, 8 in 2024-09-12T23:49:27.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:26 smithi136 bash[76371]: cluster 2024-09-12T23:49:26.463334+0000 mon.a (mon.0) 1141 : cluster [DBG] osdmap e704: 8 total, 8 up, 8 in 2024-09-12T23:49:27.286 INFO:tasks.rados.rados.0.smithi136.stdout:3219: finishing copy_from racing read to smithi13667081-24 2024-09-12T23:49:27.286 INFO:tasks.rados.rados.0.smithi136.stdout:3219: finishing copy_from to smithi13667081-24 2024-09-12T23:49:27.286 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 24 v 1830 (ObjNum 993 snap 307 seq_num 993) dirty exists 2024-09-12T23:49:27.286 INFO:tasks.rados.rados.0.smithi136.stdout:3219: done (11 left) 2024-09-12T23:49:27.286 INFO:tasks.rados.rados.0.smithi136.stdout:3220: done (10 left) 2024-09-12T23:49:27.287 INFO:tasks.rados.rados.0.smithi136.stdout:3222: done (9 left) 2024-09-12T23:49:27.287 INFO:tasks.rados.rados.0.smithi136.stdout:3223: done (8 left) 2024-09-12T23:49:27.287 INFO:tasks.rados.rados.0.smithi136.stdout:3224: done (7 left) 2024-09-12T23:49:27.287 INFO:tasks.rados.rados.0.smithi136.stdout:3225: done (6 left) 2024-09-12T23:49:27.287 INFO:tasks.rados.rados.0.smithi136.stdout:3227: done (5 left) 2024-09-12T23:49:27.287 INFO:tasks.rados.rados.0.smithi136.stdout:3228: done (4 left) 2024-09-12T23:49:27.287 INFO:tasks.rados.rados.0.smithi136.stdout:3229: done (3 left) 2024-09-12T23:49:27.287 INFO:tasks.rados.rados.0.smithi136.stdout:3230: done (2 left) 2024-09-12T23:49:27.287 INFO:tasks.rados.rados.0.smithi136.stdout:3231: done (1 left) 2024-09-12T23:49:27.287 INFO:tasks.rados.rados.0.smithi136.stdout:3232: done (0 left) 2024-09-12T23:49:27.287 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 39 current snap is 315 2024-09-12T23:49:27.287 INFO:tasks.rados.rados.0.smithi136.stdout:3233: seq_num 1018 ranges {0=253952} 2024-09-12T23:49:27.289 INFO:tasks.rados.rados.0.smithi136.stdout:3233: writing smithi13667081-39 from 0 to 253952 tid 1 2024-09-12T23:49:27.289 INFO:tasks.rados.rados.0.smithi136.stdout:3234: read oid 17 snap -1 2024-09-12T23:49:27.289 INFO:tasks.rados.rados.0.smithi136.stdout:3234: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-12T23:49:27.289 INFO:tasks.rados.rados.0.smithi136.stdout:3235: snap_remove snap 304 2024-09-12T23:49:27.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:27 smithi181 bash[67587]: cluster 2024-09-12T23:49:26.825718+0000 mgr.y (mgr.24661) 354 : cluster [DBG] pgmap v635: 121 pgs: 121 active+clean; 265 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 7.1 MiB/s wr, 14 op/s 2024-09-12T23:49:27.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:27 smithi181 bash[67587]: audit 2024-09-12T23:49:27.265360+0000 mon.a (mon.0) 1142 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:27.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:27 smithi181 bash[67587]: audit 2024-09-12T23:49:27.276042+0000 mon.a (mon.0) 1143 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:28.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:27 smithi136 bash[75164]: cluster 2024-09-12T23:49:26.825718+0000 mgr.y (mgr.24661) 354 : cluster [DBG] pgmap v635: 121 pgs: 121 active+clean; 265 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 7.1 MiB/s wr, 14 op/s 2024-09-12T23:49:28.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:27 smithi136 bash[75164]: audit 2024-09-12T23:49:27.265360+0000 mon.a (mon.0) 1142 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:28.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:27 smithi136 bash[75164]: audit 2024-09-12T23:49:27.276042+0000 mon.a (mon.0) 1143 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:28.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:27 smithi136 bash[76371]: cluster 2024-09-12T23:49:26.825718+0000 mgr.y (mgr.24661) 354 : cluster [DBG] pgmap v635: 121 pgs: 121 active+clean; 265 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 7.1 MiB/s wr, 14 op/s 2024-09-12T23:49:28.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:27 smithi136 bash[76371]: audit 2024-09-12T23:49:27.265360+0000 mon.a (mon.0) 1142 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:28.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:27 smithi136 bash[76371]: audit 2024-09-12T23:49:27.276042+0000 mon.a (mon.0) 1143 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:28.286 INFO:tasks.rados.rados.0.smithi136.stdout:3234: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-12T23:49:28.286 INFO:tasks.rados.rados.0.smithi136.stdout:3233: finishing write tid 1 to smithi13667081-39 2024-09-12T23:49:28.286 INFO:tasks.rados.rados.0.smithi136.stdout:3233: finishing write tid 2 to smithi13667081-39 2024-09-12T23:49:28.286 INFO:tasks.rados.rados.0.smithi136.stdout:3233: finishing write tid 3 to smithi13667081-39 2024-09-12T23:49:28.286 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 39 v 2439 (ObjNum 1018 snap 315 seq_num 1018) dirty exists 2024-09-12T23:49:28.286 INFO:tasks.rados.rados.0.smithi136.stdout:3233: left oid 39 (ObjNum 1018 snap 315 seq_num 1018) 2024-09-12T23:49:28.286 INFO:tasks.rados.rados.0.smithi136.stdout:3233: done (2 left) 2024-09-12T23:49:28.287 INFO:tasks.rados.rados.0.smithi136.stdout:3234: done (1 left) 2024-09-12T23:49:28.287 INFO:tasks.rados.rados.0.smithi136.stdout:3235: done (0 left) 2024-09-12T23:49:28.287 INFO:tasks.rados.rados.0.smithi136.stdout:3236: copy_from oid 20 from oid 32 current snap is 315 2024-09-12T23:49:28.287 INFO:tasks.rados.rados.0.smithi136.stdout:3237: read oid 42 snap -1 2024-09-12T23:49:28.287 INFO:tasks.rados.rados.0.smithi136.stdout:3237: expect (ObjNum 927 snap 290 seq_num 927) 2024-09-12T23:49:28.287 INFO:tasks.rados.rados.0.smithi136.stdout:3238: copy_from oid 33 from oid 21 current snap is 315 2024-09-12T23:49:28.287 INFO:tasks.rados.rados.0.smithi136.stdout:3239: copy_from oid 8 from oid 45 current snap is 315 2024-09-12T23:49:28.287 INFO:tasks.rados.rados.0.smithi136.stdout:3240: snap_remove snap 314 2024-09-12T23:49:29.291 INFO:tasks.rados.rados.0.smithi136.stdout:3240: done (4 left) 2024-09-12T23:49:29.291 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 14 current snap is 315 2024-09-12T23:49:29.291 INFO:tasks.rados.rados.0.smithi136.stdout:3241: seq_num 1019 ranges {2392064=270336} 2024-09-12T23:49:29.306 INFO:tasks.rados.rados.0.smithi136.stdout:3241: writing smithi13667081-14 from 2392064 to 2662400 tid 1 2024-09-12T23:49:29.306 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 37 current snap is 315 2024-09-12T23:49:29.306 INFO:tasks.rados.rados.0.smithi136.stdout:3242: seq_num 1020 ranges {4866048=286720} 2024-09-12T23:49:29.336 INFO:tasks.rados.rados.0.smithi136.stdout:3242: writing smithi13667081-37 from 4866048 to 5152768 tid 1 2024-09-12T23:49:29.336 INFO:tasks.rados.rados.0.smithi136.stdout:3243: setattr oid 40 current snap is 315 2024-09-12T23:49:29.337 INFO:tasks.rados.rados.0.smithi136.stdout:3239: finishing copy_from racing read to smithi13667081-8 2024-09-12T23:49:29.337 INFO:tasks.rados.rados.0.smithi136.stdout:3239: finishing copy_from to smithi13667081-8 2024-09-12T23:49:29.337 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 8 v 1620 (ObjNum 889 snap 288 seq_num 889) dirty exists 2024-09-12T23:49:29.337 INFO:tasks.rados.rados.0.smithi136.stdout:3236: finishing copy_from racing read to smithi13667081-20 2024-09-12T23:49:29.337 INFO:tasks.rados.rados.0.smithi136.stdout:3237: expect (ObjNum 927 snap 290 seq_num 927) 2024-09-12T23:49:29.345 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:29 smithi181 bash[67587]: audit 2024-09-12T23:49:28.047890+0000 mon.a (mon.0) 1144 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:29.345 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:29 smithi181 bash[67587]: audit 2024-09-12T23:49:28.058747+0000 mon.a (mon.0) 1145 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:29.345 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:29 smithi181 bash[67587]: cluster 2024-09-12T23:49:28.282611+0000 mon.a (mon.0) 1146 : cluster [DBG] osdmap e705: 8 total, 8 up, 8 in 2024-09-12T23:49:29.345 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:29 smithi181 bash[67587]: cluster 2024-09-12T23:49:28.826553+0000 mgr.y (mgr.24661) 355 : cluster [DBG] pgmap v637: 121 pgs: 121 active+clean; 271 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 8.8 MiB/s wr, 22 op/s 2024-09-12T23:49:29.367 INFO:tasks.rados.rados.0.smithi136.stdout:3236: finishing copy_from to smithi13667081-20 2024-09-12T23:49:29.367 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 20 v 1638 (ObjNum 1006 snap 313 seq_num 1006) dirty exists 2024-09-12T23:49:29.367 INFO:tasks.rados.rados.0.smithi136.stdout:3238: finishing copy_from racing read to smithi13667081-33 2024-09-12T23:49:29.368 INFO:tasks.rados.rados.0.smithi136.stdout:3238: finishing copy_from to smithi13667081-33 2024-09-12T23:49:29.368 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 33 v 2462 (ObjNum 1011 snap 313 seq_num 1011) dirty exists 2024-09-12T23:49:29.368 INFO:tasks.rados.rados.0.smithi136.stdout:3241: finishing write tid 1 to smithi13667081-14 2024-09-12T23:49:29.368 INFO:tasks.rados.rados.0.smithi136.stdout:3241: finishing write tid 2 to smithi13667081-14 2024-09-12T23:49:29.368 INFO:tasks.rados.rados.0.smithi136.stdout:3241: finishing write tid 3 to smithi13667081-14 2024-09-12T23:49:29.368 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 14 v 2001 (ObjNum 1019 snap 315 seq_num 1019) dirty exists 2024-09-12T23:49:29.368 INFO:tasks.rados.rados.0.smithi136.stdout:3241: left oid 14 (ObjNum 1019 snap 315 seq_num 1019) 2024-09-12T23:49:29.368 INFO:tasks.rados.rados.0.smithi136.stdout:3242: finishing write tid 1 to smithi13667081-37 2024-09-12T23:49:29.368 INFO:tasks.rados.rados.0.smithi136.stdout:3242: finishing write tid 2 to smithi13667081-37 2024-09-12T23:49:29.368 INFO:tasks.rados.rados.0.smithi136.stdout:3242: finishing write tid 3 to smithi13667081-37 2024-09-12T23:49:29.368 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 37 v 2464 (ObjNum 1020 snap 315 seq_num 1020) dirty exists 2024-09-12T23:49:29.368 INFO:tasks.rados.rados.0.smithi136.stdout:3242: left oid 37 (ObjNum 1020 snap 315 seq_num 1020) 2024-09-12T23:49:29.368 INFO:tasks.rados.rados.0.smithi136.stdout:3236: done (6 left) 2024-09-12T23:49:29.369 INFO:tasks.rados.rados.0.smithi136.stdout:3237: done (5 left) 2024-09-12T23:49:29.369 INFO:tasks.rados.rados.0.smithi136.stdout:3238: done (4 left) 2024-09-12T23:49:29.369 INFO:tasks.rados.rados.0.smithi136.stdout:3239: done (3 left) 2024-09-12T23:49:29.369 INFO:tasks.rados.rados.0.smithi136.stdout:3241: done (2 left) 2024-09-12T23:49:29.369 INFO:tasks.rados.rados.0.smithi136.stdout:3242: done (1 left) 2024-09-12T23:49:29.369 INFO:tasks.rados.rados.0.smithi136.stdout:3244: snap_remove snap 305 2024-09-12T23:49:29.412 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:29 smithi136 bash[76371]: audit 2024-09-12T23:49:28.047890+0000 mon.a (mon.0) 1144 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:29.412 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:29 smithi136 bash[76371]: audit 2024-09-12T23:49:28.058747+0000 mon.a (mon.0) 1145 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:29.412 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:29 smithi136 bash[76371]: cluster 2024-09-12T23:49:28.282611+0000 mon.a (mon.0) 1146 : cluster [DBG] osdmap e705: 8 total, 8 up, 8 in 2024-09-12T23:49:29.412 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:29 smithi136 bash[76371]: cluster 2024-09-12T23:49:28.826553+0000 mgr.y (mgr.24661) 355 : cluster [DBG] pgmap v637: 121 pgs: 121 active+clean; 271 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 8.8 MiB/s wr, 22 op/s 2024-09-12T23:49:29.412 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:29 smithi136 bash[75164]: audit 2024-09-12T23:49:28.047890+0000 mon.a (mon.0) 1144 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:29.412 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:29 smithi136 bash[75164]: audit 2024-09-12T23:49:28.058747+0000 mon.a (mon.0) 1145 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:29.412 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:29 smithi136 bash[75164]: cluster 2024-09-12T23:49:28.282611+0000 mon.a (mon.0) 1146 : cluster [DBG] osdmap e705: 8 total, 8 up, 8 in 2024-09-12T23:49:29.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:29 smithi136 bash[75164]: cluster 2024-09-12T23:49:28.826553+0000 mgr.y (mgr.24661) 355 : cluster [DBG] pgmap v637: 121 pgs: 121 active+clean; 271 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 8.8 MiB/s wr, 22 op/s 2024-09-12T23:49:29.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:49:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:49:29] "GET /metrics HTTP/1.1" 200 36441 "" "Prometheus/2.43.0" 2024-09-12T23:49:30.313 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 40 v 2483 (ObjNum 983 snap 303 seq_num 983) dirty exists 2024-09-12T23:49:30.314 INFO:tasks.rados.rados.0.smithi136.stdout:3243: done (1 left) 2024-09-12T23:49:30.314 INFO:tasks.rados.rados.0.smithi136.stdout:3244: done (0 left) 2024-09-12T23:49:30.314 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 29 current snap is 315 2024-09-12T23:49:30.314 INFO:tasks.rados.rados.0.smithi136.stdout:3245: seq_num 1021 ranges {22880256=303104} 2024-09-12T23:49:30.446 INFO:tasks.rados.rados.0.smithi136.stdout:3245: writing smithi13667081-29 from 22880256 to 23183360 tid 1 2024-09-12T23:49:30.446 INFO:tasks.rados.rados.0.smithi136.stdout:3246: read oid 4 snap -1 2024-09-12T23:49:30.446 INFO:tasks.rados.rados.0.smithi136.stdout:3246: expect (ObjNum 949 snap 301 seq_num 949) 2024-09-12T23:49:30.446 INFO:tasks.rados.rados.0.smithi136.stdout:3247: setattr oid 19 current snap is 315 2024-09-12T23:49:30.447 INFO:tasks.rados.rados.0.smithi136.stdout:3248: setattr oid 49 current snap is 315 2024-09-12T23:49:30.448 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 42 current snap is 315 2024-09-12T23:49:30.448 INFO:tasks.rados.rados.0.smithi136.stdout:3249: seq_num 1022 ranges {1130496=319488} 2024-09-12T23:49:30.457 INFO:tasks.rados.rados.0.smithi136.stdout:3249: writing smithi13667081-42 from 1130496 to 1449984 tid 1 2024-09-12T23:49:30.457 INFO:tasks.rados.rados.0.smithi136.stdout:3250: delete oid 46 current snap is 315 2024-09-12T23:49:30.457 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 19 v 1644 (ObjNum 33 snap 0 seq_num 4242628448) dirty exists 2024-09-12T23:49:30.457 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 49 v 1833 (ObjNum 33 snap 0 seq_num 0) dirty exists 2024-09-12T23:49:30.463 INFO:tasks.rados.rados.0.smithi136.stdout:3249: finishing write tid 1 to smithi13667081-42 2024-09-12T23:49:30.463 INFO:tasks.rados.rados.0.smithi136.stdout:3247: done (5 left) 2024-09-12T23:49:30.463 INFO:tasks.rados.rados.0.smithi136.stdout:3248: done (4 left) 2024-09-12T23:49:30.463 INFO:tasks.rados.rados.0.smithi136.stdout:3250: done (3 left) 2024-09-12T23:49:30.463 INFO:tasks.rados.rados.0.smithi136.stdout:3251: rollback oid 3 current snap is 315 2024-09-12T23:49:30.464 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 3 to 310 2024-09-12T23:49:30.464 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 5 current snap is 315 2024-09-12T23:49:30.464 INFO:tasks.rados.rados.0.smithi136.stdout:3249: finishing write tid 2 to smithi13667081-42 2024-09-12T23:49:30.464 INFO:tasks.rados.rados.0.smithi136.stdout:3252: seq_num 1024 ranges {1769472=352256} 2024-09-12T23:49:30.476 INFO:tasks.rados.rados.0.smithi136.stdout:3252: writing smithi13667081-5 from 1769472 to 2121728 tid 1 2024-09-12T23:49:30.476 INFO:tasks.rados.rados.0.smithi136.stdout:3253: read oid 25 snap -1 2024-09-12T23:49:30.476 INFO:tasks.rados.rados.0.smithi136.stdout:3253: expect (ObjNum 998 snap 309 seq_num 998) 2024-09-12T23:49:30.476 INFO:tasks.rados.rados.0.smithi136.stdout:3254: read oid 8 snap -1 2024-09-12T23:49:30.477 INFO:tasks.rados.rados.0.smithi136.stdout:3254: expect (ObjNum 889 snap 288 seq_num 889) 2024-09-12T23:49:30.477 INFO:tasks.rados.rados.0.smithi136.stdout:3255: snap_create 2024-09-12T23:49:30.477 INFO:tasks.rados.rados.0.smithi136.stdout:3249: finishing write tid 3 to smithi13667081-42 2024-09-12T23:49:30.477 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 42 v 2464 (ObjNum 1022 snap 315 seq_num 1022) dirty exists 2024-09-12T23:49:30.477 INFO:tasks.rados.rados.0.smithi136.stdout:3249: left oid 42 (ObjNum 1022 snap 315 seq_num 1022) 2024-09-12T23:49:30.477 INFO:tasks.rados.rados.0.smithi136.stdout:3245: finishing write tid 1 to smithi13667081-29 2024-09-12T23:49:30.477 INFO:tasks.rados.rados.0.smithi136.stdout:3245: finishing write tid 2 to smithi13667081-29 2024-09-12T23:49:30.477 INFO:tasks.rados.rados.0.smithi136.stdout:3251: finishing rollback tid 0 to smithi13667081-3 2024-09-12T23:49:30.477 INFO:tasks.rados.rados.0.smithi136.stdout:3245: finishing write tid 3 to smithi13667081-29 2024-09-12T23:49:30.477 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 29 v 2289 (ObjNum 1021 snap 315 seq_num 1021) dirty exists 2024-09-12T23:49:30.477 INFO:tasks.rados.rados.0.smithi136.stdout:3245: left oid 29 (ObjNum 1021 snap 315 seq_num 1021) 2024-09-12T23:49:30.477 INFO:tasks.rados.rados.0.smithi136.stdout:3251: finishing rollback tid 1 to smithi13667081-3 2024-09-12T23:49:30.477 INFO:tasks.rados.rados.0.smithi136.stdout:3251: finishing rollback tid 2 to smithi13667081-3 2024-09-12T23:49:30.478 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 3 v 1836 (ObjNum 838 snap 267 seq_num 838) dirty exists 2024-09-12T23:49:30.487 INFO:tasks.rados.rados.0.smithi136.stdout:3252: finishing write tid 1 to smithi13667081-5 2024-09-12T23:49:30.487 INFO:tasks.rados.rados.0.smithi136.stdout:3252: finishing write tid 2 to smithi13667081-5 2024-09-12T23:49:30.490 INFO:tasks.rados.rados.0.smithi136.stdout:3252: finishing write tid 3 to smithi13667081-5 2024-09-12T23:49:30.490 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 5 v 2496 (ObjNum 1024 snap 315 seq_num 1024) dirty exists 2024-09-12T23:49:30.490 INFO:tasks.rados.rados.0.smithi136.stdout:3252: left oid 5 (ObjNum 1024 snap 315 seq_num 1024) 2024-09-12T23:49:30.524 INFO:tasks.rados.rados.0.smithi136.stdout:3253: expect (ObjNum 998 snap 309 seq_num 998) 2024-09-12T23:49:30.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:30 smithi181 bash[67587]: cluster 2024-09-12T23:49:29.292882+0000 mon.a (mon.0) 1147 : cluster [DBG] osdmap e706: 8 total, 8 up, 8 in 2024-09-12T23:49:30.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:30 smithi136 bash[75164]: cluster 2024-09-12T23:49:29.292882+0000 mon.a (mon.0) 1147 : cluster [DBG] osdmap e706: 8 total, 8 up, 8 in 2024-09-12T23:49:30.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:30 smithi136 bash[76371]: cluster 2024-09-12T23:49:29.292882+0000 mon.a (mon.0) 1147 : cluster [DBG] osdmap e706: 8 total, 8 up, 8 in 2024-09-12T23:49:30.809 INFO:tasks.rados.rados.0.smithi136.stdout:3254: expect (ObjNum 889 snap 288 seq_num 889) 2024-09-12T23:49:30.835 INFO:tasks.rados.rados.0.smithi136.stdout:3246: expect (ObjNum 949 snap 301 seq_num 949) 2024-09-12T23:49:31.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:31 smithi181 bash[67587]: cluster 2024-09-12T23:49:30.313829+0000 mon.a (mon.0) 1148 : cluster [DBG] osdmap e707: 8 total, 8 up, 8 in 2024-09-12T23:49:31.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:31 smithi181 bash[67587]: cluster 2024-09-12T23:49:30.827592+0000 mgr.y (mgr.24661) 356 : cluster [DBG] pgmap v640: 121 pgs: 121 active+clean; 273 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 8.6 MiB/s wr, 22 op/s 2024-09-12T23:49:31.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:31 smithi136 bash[75164]: cluster 2024-09-12T23:49:30.313829+0000 mon.a (mon.0) 1148 : cluster [DBG] osdmap e707: 8 total, 8 up, 8 in 2024-09-12T23:49:31.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:31 smithi136 bash[75164]: cluster 2024-09-12T23:49:30.827592+0000 mgr.y (mgr.24661) 356 : cluster [DBG] pgmap v640: 121 pgs: 121 active+clean; 273 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 8.6 MiB/s wr, 22 op/s 2024-09-12T23:49:31.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:31 smithi136 bash[76371]: cluster 2024-09-12T23:49:30.313829+0000 mon.a (mon.0) 1148 : cluster [DBG] osdmap e707: 8 total, 8 up, 8 in 2024-09-12T23:49:31.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:31 smithi136 bash[76371]: cluster 2024-09-12T23:49:30.827592+0000 mgr.y (mgr.24661) 356 : cluster [DBG] pgmap v640: 121 pgs: 121 active+clean; 273 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 8.6 MiB/s wr, 22 op/s 2024-09-12T23:49:32.468 INFO:tasks.rados.rados.0.smithi136.stdout:3245: done (7 left) 2024-09-12T23:49:32.468 INFO:tasks.rados.rados.0.smithi136.stdout:3246: done (6 left) 2024-09-12T23:49:32.468 INFO:tasks.rados.rados.0.smithi136.stdout:3249: done (5 left) 2024-09-12T23:49:32.468 INFO:tasks.rados.rados.0.smithi136.stdout:3251: done (4 left) 2024-09-12T23:49:32.469 INFO:tasks.rados.rados.0.smithi136.stdout:3252: done (3 left) 2024-09-12T23:49:32.469 INFO:tasks.rados.rados.0.smithi136.stdout:3253: done (2 left) 2024-09-12T23:49:32.469 INFO:tasks.rados.rados.0.smithi136.stdout:3254: done (1 left) 2024-09-12T23:49:32.469 INFO:tasks.rados.rados.0.smithi136.stdout:3255: done (0 left) 2024-09-12T23:49:32.469 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 42 current snap is 316 2024-09-12T23:49:32.469 INFO:tasks.rados.rados.0.smithi136.stdout:3256: seq_num 1025 ranges {1449984=368640} 2024-09-12T23:49:32.479 INFO:tasks.rados.rados.0.smithi136.stdout:3256: writing smithi13667081-42 from 1449984 to 1818624 tid 1 2024-09-12T23:49:32.479 INFO:tasks.rados.rados.0.smithi136.stdout:3257: snap_remove snap 308 2024-09-12T23:49:32.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:32 smithi181 bash[67587]: cluster 2024-09-12T23:49:31.327659+0000 mon.a (mon.0) 1149 : cluster [DBG] osdmap e708: 8 total, 8 up, 8 in 2024-09-12T23:49:32.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:32 smithi181 bash[67587]: cluster 2024-09-12T23:49:31.461269+0000 mon.a (mon.0) 1150 : cluster [DBG] osdmap e709: 8 total, 8 up, 8 in 2024-09-12T23:49:32.595 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:49:32 smithi181 bash[95505]: ts=2024-09-12T23:49:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:49:32.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:32 smithi136 bash[75164]: cluster 2024-09-12T23:49:31.327659+0000 mon.a (mon.0) 1149 : cluster [DBG] osdmap e708: 8 total, 8 up, 8 in 2024-09-12T23:49:32.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:32 smithi136 bash[75164]: cluster 2024-09-12T23:49:31.461269+0000 mon.a (mon.0) 1150 : cluster [DBG] osdmap e709: 8 total, 8 up, 8 in 2024-09-12T23:49:32.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:32 smithi136 bash[76371]: cluster 2024-09-12T23:49:31.327659+0000 mon.a (mon.0) 1149 : cluster [DBG] osdmap e708: 8 total, 8 up, 8 in 2024-09-12T23:49:32.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:32 smithi136 bash[76371]: cluster 2024-09-12T23:49:31.461269+0000 mon.a (mon.0) 1150 : cluster [DBG] osdmap e709: 8 total, 8 up, 8 in 2024-09-12T23:49:33.337 INFO:tasks.rados.rados.0.smithi136.stdout:3256: finishing write tid 1 to smithi13667081-42 2024-09-12T23:49:33.337 INFO:tasks.rados.rados.0.smithi136.stdout:3256: finishing write tid 2 to smithi13667081-42 2024-09-12T23:49:33.337 INFO:tasks.rados.rados.0.smithi136.stdout:3256: finishing write tid 3 to smithi13667081-42 2024-09-12T23:49:33.337 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 42 v 2466 (ObjNum 1025 snap 316 seq_num 1025) dirty exists 2024-09-12T23:49:33.337 INFO:tasks.rados.rados.0.smithi136.stdout:3256: left oid 42 (ObjNum 1025 snap 316 seq_num 1025) 2024-09-12T23:49:33.337 INFO:tasks.rados.rados.0.smithi136.stdout:3256: done (1 left) 2024-09-12T23:49:33.338 INFO:tasks.rados.rados.0.smithi136.stdout:3257: done (0 left) 2024-09-12T23:49:33.338 INFO:tasks.rados.rados.0.smithi136.stdout:3258: rollback oid 48 current snap is 316 2024-09-12T23:49:33.338 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 48 to 315 2024-09-12T23:49:33.338 INFO:tasks.rados.rados.0.smithi136.stdout:3259: copy_from oid 21 from oid 16 current snap is 316 2024-09-12T23:49:33.338 INFO:tasks.rados.rados.0.smithi136.stdout:3260: snap_remove snap 311 2024-09-12T23:49:33.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:33 smithi181 bash[67587]: cluster 2024-09-12T23:49:32.828728+0000 mgr.y (mgr.24661) 357 : cluster [DBG] pgmap v643: 121 pgs: 121 active+clean; 277 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 1.5 MiB/s wr, 18 op/s 2024-09-12T23:49:33.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:33 smithi136 bash[75164]: cluster 2024-09-12T23:49:32.828728+0000 mgr.y (mgr.24661) 357 : cluster [DBG] pgmap v643: 121 pgs: 121 active+clean; 277 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 1.5 MiB/s wr, 18 op/s 2024-09-12T23:49:33.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:33 smithi136 bash[76371]: cluster 2024-09-12T23:49:32.828728+0000 mgr.y (mgr.24661) 357 : cluster [DBG] pgmap v643: 121 pgs: 121 active+clean; 277 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 1.5 MiB/s wr, 18 op/s 2024-09-12T23:49:34.345 INFO:tasks.rados.rados.0.smithi136.stdout:3260: done (2 left) 2024-09-12T23:49:34.345 INFO:tasks.rados.rados.0.smithi136.stdout:3261: rollback oid 26 current snap is 316 2024-09-12T23:49:34.345 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 26 to 313 2024-09-12T23:49:34.346 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 4 current snap is 316 2024-09-12T23:49:34.346 INFO:tasks.rados.rados.0.smithi136.stdout:3262: seq_num 1026 ranges {15138816=385024} 2024-09-12T23:49:34.355 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:34 smithi181 bash[67587]: cluster 2024-09-12T23:49:33.338851+0000 mon.a (mon.0) 1151 : cluster [DBG] osdmap e710: 8 total, 8 up, 8 in 2024-09-12T23:49:34.434 INFO:tasks.rados.rados.0.smithi136.stdout:3262: writing smithi13667081-4 from 15138816 to 15523840 tid 1 2024-09-12T23:49:34.435 INFO:tasks.rados.rados.0.smithi136.stdout:3263: snap_create 2024-09-12T23:49:34.435 INFO:tasks.rados.rados.0.smithi136.stdout:3258: finishing rollback tid 0 to smithi13667081-48 2024-09-12T23:49:34.435 INFO:tasks.rados.rados.0.smithi136.stdout:3258: finishing rollback tid 1 to smithi13667081-48 2024-09-12T23:49:34.435 INFO:tasks.rados.rados.0.smithi136.stdout:3258: finishing rollback tid 2 to smithi13667081-48 2024-09-12T23:49:34.435 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 48 v 2009 (ObjNum 864 snap 281 seq_num 864) dirty exists 2024-09-12T23:49:34.435 INFO:tasks.rados.rados.0.smithi136.stdout:3259: finishing copy_from racing read to smithi13667081-21 2024-09-12T23:49:34.435 INFO:tasks.rados.rados.0.smithi136.stdout:3259: finishing copy_from to smithi13667081-21 2024-09-12T23:49:34.435 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 21 v 1978 (ObjNum 977 snap 303 seq_num 977) dirty exists 2024-09-12T23:49:34.435 INFO:tasks.rados.rados.0.smithi136.stdout:3261: finishing rollback tid 0 to smithi13667081-26 2024-09-12T23:49:34.435 INFO:tasks.rados.rados.0.smithi136.stdout:3261: finishing rollback tid 1 to smithi13667081-26 2024-09-12T23:49:34.435 INFO:tasks.rados.rados.0.smithi136.stdout:3261: finishing rollback tid 2 to smithi13667081-26 2024-09-12T23:49:34.436 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 26 v 1834 (ObjNum 987 snap 303 seq_num 987) dirty exists 2024-09-12T23:49:34.441 INFO:tasks.rados.rados.0.smithi136.stdout:3262: finishing write tid 1 to smithi13667081-4 2024-09-12T23:49:34.442 INFO:tasks.rados.rados.0.smithi136.stdout:3262: finishing write tid 2 to smithi13667081-4 2024-09-12T23:49:34.445 INFO:tasks.rados.rados.0.smithi136.stdout:3262: finishing write tid 3 to smithi13667081-4 2024-09-12T23:49:34.445 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 4 v 1640 (ObjNum 1026 snap 316 seq_num 1026) dirty exists 2024-09-12T23:49:34.445 INFO:tasks.rados.rados.0.smithi136.stdout:3262: left oid 4 (ObjNum 1026 snap 316 seq_num 1026) 2024-09-12T23:49:34.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:34 smithi136 bash[75164]: cluster 2024-09-12T23:49:33.338851+0000 mon.a (mon.0) 1151 : cluster [DBG] osdmap e710: 8 total, 8 up, 8 in 2024-09-12T23:49:34.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:34 smithi136 bash[76371]: cluster 2024-09-12T23:49:33.338851+0000 mon.a (mon.0) 1151 : cluster [DBG] osdmap e710: 8 total, 8 up, 8 in 2024-09-12T23:49:35.357 INFO:tasks.rados.rados.0.smithi136.stdout:3258: done (4 left) 2024-09-12T23:49:35.358 INFO:tasks.rados.rados.0.smithi136.stdout:3259: done (3 left) 2024-09-12T23:49:35.358 INFO:tasks.rados.rados.0.smithi136.stdout:3261: done (2 left) 2024-09-12T23:49:35.358 INFO:tasks.rados.rados.0.smithi136.stdout:3262: done (1 left) 2024-09-12T23:49:35.358 INFO:tasks.rados.rados.0.smithi136.stdout:3263: done (0 left) 2024-09-12T23:49:35.358 INFO:tasks.rados.rados.0.smithi136.stdout:3264: copy_from oid 49 from oid 13 current snap is 317 2024-09-12T23:49:35.358 INFO:tasks.rados.rados.0.smithi136.stdout:3265: copy_from oid 46 from oid 17 current snap is 317 2024-09-12T23:49:35.358 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 25 current snap is 317 2024-09-12T23:49:35.358 INFO:tasks.rados.rados.0.smithi136.stdout:3266: seq_num 1027 ranges {5832704=409600} 2024-09-12T23:49:35.395 INFO:tasks.rados.rados.0.smithi136.stdout:3266: writing smithi13667081-25 from 5832704 to 6242304 tid 1 2024-09-12T23:49:35.395 INFO:tasks.rados.rados.0.smithi136.stdout:3267: read oid 11 snap -1 2024-09-12T23:49:35.395 INFO:tasks.rados.rados.0.smithi136.stdout:3267: expect (ObjNum 1016 snap 314 seq_num 1016) 2024-09-12T23:49:35.395 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 12 current snap is 317 2024-09-12T23:49:35.395 INFO:tasks.rados.rados.0.smithi136.stdout:3264: finishing copy_from to smithi13667081-49 2024-09-12T23:49:35.395 INFO:tasks.rados.rados.0.smithi136.stdout:3264: got expected ENOENT (src dne) 2024-09-12T23:49:35.395 INFO:tasks.rados.rados.0.smithi136.stdout:3265: finishing copy_from racing read to smithi13667081-46 2024-09-12T23:49:35.395 INFO:tasks.rados.rados.0.smithi136.stdout:3264: finishing copy_from racing read to smithi13667081-49 2024-09-12T23:49:35.395 INFO:tasks.rados.rados.0.smithi136.stdout:3265: finishing copy_from to smithi13667081-46 2024-09-12T23:49:35.395 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 46 v 2473 (ObjNum 33 snap 0 seq_num 4243775024) dirty exists 2024-09-12T23:49:35.395 INFO:tasks.rados.rados.0.smithi136.stdout:3268: seq_num 1028 ranges {7127040=425984} 2024-09-12T23:49:35.437 INFO:tasks.rados.rados.0.smithi136.stdout:3268: writing smithi13667081-12 from 7127040 to 7553024 tid 1 2024-09-12T23:49:35.437 INFO:tasks.rados.rados.0.smithi136.stdout:3264: done (4 left) 2024-09-12T23:49:35.438 INFO:tasks.rados.rados.0.smithi136.stdout:3265: done (3 left) 2024-09-12T23:49:35.438 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 6 current snap is 317 2024-09-12T23:49:35.438 INFO:tasks.rados.rados.0.smithi136.stdout:3266: finishing write tid 1 to smithi13667081-25 2024-09-12T23:49:35.438 INFO:tasks.rados.rados.0.smithi136.stdout:3266: finishing write tid 2 to smithi13667081-25 2024-09-12T23:49:35.438 INFO:tasks.rados.rados.0.smithi136.stdout:3267: expect (ObjNum 1016 snap 314 seq_num 1016) 2024-09-12T23:49:35.443 INFO:tasks.rados.rados.0.smithi136.stdout:3266: finishing write tid 3 to smithi13667081-25 2024-09-12T23:49:35.656 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 25 v 2441 (ObjNum 1027 snap 317 seq_num 1027) dirty exists 2024-09-12T23:49:35.656 INFO:tasks.rados.rados.0.smithi136.stdout:3266: left oid 25 (ObjNum 1027 snap 317 seq_num 1027) 2024-09-12T23:49:35.656 INFO:tasks.rados.rados.0.smithi136.stdout:3269: seq_num 1029 ranges {237568=442368} 2024-09-12T23:49:35.658 INFO:tasks.rados.rados.0.smithi136.stdout:3269: writing smithi13667081-6 from 237568 to 679936 tid 1 2024-09-12T23:49:35.658 INFO:tasks.rados.rados.0.smithi136.stdout:3266: done (3 left) 2024-09-12T23:49:35.658 INFO:tasks.rados.rados.0.smithi136.stdout:3267: done (2 left) 2024-09-12T23:49:35.658 INFO:tasks.rados.rados.0.smithi136.stdout:3270: rollback oid 20 current snap is 317 2024-09-12T23:49:35.658 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 20 to 315 2024-09-12T23:49:35.658 INFO:tasks.rados.rados.0.smithi136.stdout:3268: finishing write tid 1 to smithi13667081-12 2024-09-12T23:49:35.658 INFO:tasks.rados.rados.0.smithi136.stdout:3271: delete oid 50 current snap is 317 2024-09-12T23:49:35.658 INFO:tasks.rados.rados.0.smithi136.stdout:3268: finishing write tid 2 to smithi13667081-12 2024-09-12T23:49:35.658 INFO:tasks.rados.rados.0.smithi136.stdout:3268: finishing write tid 3 to smithi13667081-12 2024-09-12T23:49:35.658 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 12 v 1838 (ObjNum 1028 snap 317 seq_num 1028) dirty exists 2024-09-12T23:49:35.658 INFO:tasks.rados.rados.0.smithi136.stdout:3268: left oid 12 (ObjNum 1028 snap 317 seq_num 1028) 2024-09-12T23:49:35.658 INFO:tasks.rados.rados.0.smithi136.stdout:3268: done (3 left) 2024-09-12T23:49:35.658 INFO:tasks.rados.rados.0.smithi136.stdout:3271: done (2 left) 2024-09-12T23:49:35.659 INFO:tasks.rados.rados.0.smithi136.stdout:3272: read oid 27 snap -1 2024-09-12T23:49:35.659 INFO:tasks.rados.rados.0.smithi136.stdout:3272: expect deleted 2024-09-12T23:49:35.659 INFO:tasks.rados.rados.0.smithi136.stdout:3273: read oid 45 snap 313 2024-09-12T23:49:35.659 INFO:tasks.rados.rados.0.smithi136.stdout:3273: expect (ObjNum 889 snap 288 seq_num 889) 2024-09-12T23:49:35.659 INFO:tasks.rados.rados.0.smithi136.stdout:3274: snap_remove snap 312 2024-09-12T23:49:35.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:35 smithi136 bash[75164]: cluster 2024-09-12T23:49:34.346545+0000 mon.a (mon.0) 1152 : cluster [DBG] osdmap e711: 8 total, 8 up, 8 in 2024-09-12T23:49:35.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:35 smithi136 bash[75164]: cluster 2024-09-12T23:49:34.829521+0000 mgr.y (mgr.24661) 358 : cluster [DBG] pgmap v646: 121 pgs: 121 active+clean; 281 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.0 MiB/s wr, 23 op/s 2024-09-12T23:49:35.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:35 smithi136 bash[76371]: cluster 2024-09-12T23:49:34.346545+0000 mon.a (mon.0) 1152 : cluster [DBG] osdmap e711: 8 total, 8 up, 8 in 2024-09-12T23:49:35.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:35 smithi136 bash[76371]: cluster 2024-09-12T23:49:34.829521+0000 mgr.y (mgr.24661) 358 : cluster [DBG] pgmap v646: 121 pgs: 121 active+clean; 281 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.0 MiB/s wr, 23 op/s 2024-09-12T23:49:35.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:35 smithi181 bash[67587]: cluster 2024-09-12T23:49:34.346545+0000 mon.a (mon.0) 1152 : cluster [DBG] osdmap e711: 8 total, 8 up, 8 in 2024-09-12T23:49:35.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:35 smithi181 bash[67587]: cluster 2024-09-12T23:49:34.829521+0000 mgr.y (mgr.24661) 358 : cluster [DBG] pgmap v646: 121 pgs: 121 active+clean; 281 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.0 MiB/s wr, 23 op/s 2024-09-12T23:49:36.362 INFO:tasks.rados.rados.0.smithi136.stdout:3274: done (4 left) 2024-09-12T23:49:36.362 INFO:tasks.rados.rados.0.smithi136.stdout:3275: snap_remove snap 307 2024-09-12T23:49:36.476 INFO:tasks.rados.rados.0.smithi136.stdout:3269: finishing write tid 1 to smithi13667081-6 2024-09-12T23:49:36.476 INFO:tasks.rados.rados.0.smithi136.stdout:3269: finishing write tid 2 to smithi13667081-6 2024-09-12T23:49:36.477 INFO:tasks.rados.rados.0.smithi136.stdout:3270: finishing rollback tid 0 to smithi13667081-20 2024-09-12T23:49:36.477 INFO:tasks.rados.rados.0.smithi136.stdout:3270: finishing rollback tid 1 to smithi13667081-20 2024-09-12T23:49:36.477 INFO:tasks.rados.rados.0.smithi136.stdout:3270: finishing rollback tid 2 to smithi13667081-20 2024-09-12T23:49:36.477 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 20 v 1643 (ObjNum 1006 snap 313 seq_num 1006) dirty exists 2024-09-12T23:49:36.477 INFO:tasks.rados.rados.0.smithi136.stdout:3269: finishing write tid 3 to smithi13667081-6 2024-09-12T23:49:36.477 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 6 v 1640 (ObjNum 1029 snap 317 seq_num 1029) dirty exists 2024-09-12T23:49:36.477 INFO:tasks.rados.rados.0.smithi136.stdout:3269: left oid 6 (ObjNum 1029 snap 317 seq_num 1029) 2024-09-12T23:49:36.477 INFO:tasks.rados.rados.0.smithi136.stdout:3273: expect (ObjNum 889 snap 288 seq_num 889) 2024-09-12T23:49:36.508 INFO:tasks.rados.rados.0.smithi136.stdout:3269: done (4 left) 2024-09-12T23:49:36.508 INFO:tasks.rados.rados.0.smithi136.stdout:3270: done (3 left) 2024-09-12T23:49:36.508 INFO:tasks.rados.rados.0.smithi136.stdout:3272: done (2 left) 2024-09-12T23:49:36.508 INFO:tasks.rados.rados.0.smithi136.stdout:3273: done (1 left) 2024-09-12T23:49:36.508 INFO:tasks.rados.rados.0.smithi136.stdout:3275: done (0 left) 2024-09-12T23:49:36.508 INFO:tasks.rados.rados.0.smithi136.stdout:3276: read oid 15 snap 315 2024-09-12T23:49:36.509 INFO:tasks.rados.rados.0.smithi136.stdout:3276: expect (ObjNum 993 snap 307 seq_num 993) 2024-09-12T23:49:36.509 INFO:tasks.rados.rados.0.smithi136.stdout:3277: delete oid 36 current snap is 317 2024-09-12T23:49:36.512 INFO:tasks.rados.rados.0.smithi136.stdout:3277: done (1 left) 2024-09-12T23:49:36.512 INFO:tasks.rados.rados.0.smithi136.stdout:3278: copy_from oid 16 from oid 1 current snap is 317 2024-09-12T23:49:36.512 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 5 current snap is 317 2024-09-12T23:49:36.512 INFO:tasks.rados.rados.0.smithi136.stdout:3279: seq_num 1032 ranges {2121728=491520} 2024-09-12T23:49:36.533 INFO:tasks.rados.rados.0.smithi136.stdout:3279: writing smithi13667081-5 from 2121728 to 2613248 tid 1 2024-09-12T23:49:36.533 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 18 current snap is 317 2024-09-12T23:49:36.533 INFO:tasks.rados.rados.0.smithi136.stdout:3280: seq_num 1033 ranges {1769472=507904} 2024-09-12T23:49:36.549 INFO:tasks.rados.rados.0.smithi136.stdout:3280: writing smithi13667081-18 from 1769472 to 2277376 tid 1 2024-09-12T23:49:36.549 INFO:tasks.rados.rados.0.smithi136.stdout:3281: copy_from oid 29 from oid 25 current snap is 317 2024-09-12T23:49:36.549 INFO:tasks.rados.rados.0.smithi136.stdout:3279: finishing write tid 1 to smithi13667081-5 2024-09-12T23:49:36.549 INFO:tasks.rados.rados.0.smithi136.stdout:3279: finishing write tid 2 to smithi13667081-5 2024-09-12T23:49:36.549 INFO:tasks.rados.rados.0.smithi136.stdout:3282: snap_create 2024-09-12T23:49:36.550 INFO:tasks.rados.rados.0.smithi136.stdout:3279: finishing write tid 3 to smithi13667081-5 2024-09-12T23:49:36.550 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 5 v 2547 (ObjNum 1032 snap 317 seq_num 1032) dirty exists 2024-09-12T23:49:36.550 INFO:tasks.rados.rados.0.smithi136.stdout:3279: left oid 5 (ObjNum 1032 snap 317 seq_num 1032) 2024-09-12T23:49:36.550 INFO:tasks.rados.rados.0.smithi136.stdout:3278: finishing copy_from racing read to smithi13667081-16 2024-09-12T23:49:36.553 INFO:tasks.rados.rados.0.smithi136.stdout:3280: finishing write tid 1 to smithi13667081-18 2024-09-12T23:49:36.554 INFO:tasks.rados.rados.0.smithi136.stdout:3280: finishing write tid 2 to smithi13667081-18 2024-09-12T23:49:36.557 INFO:tasks.rados.rados.0.smithi136.stdout:3280: finishing write tid 3 to smithi13667081-18 2024-09-12T23:49:36.558 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 18 v 1635 (ObjNum 1033 snap 317 seq_num 1033) dirty exists 2024-09-12T23:49:36.558 INFO:tasks.rados.rados.0.smithi136.stdout:3280: left oid 18 (ObjNum 1033 snap 317 seq_num 1033) 2024-09-12T23:49:36.565 INFO:tasks.rados.rados.0.smithi136.stdout:3278: finishing copy_from to smithi13667081-16 2024-09-12T23:49:36.565 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 16 v 1504 (ObjNum 987 snap 303 seq_num 987) dirty exists 2024-09-12T23:49:36.583 INFO:tasks.rados.rados.0.smithi136.stdout:3281: finishing copy_from racing read to smithi13667081-29 2024-09-12T23:49:36.583 INFO:tasks.rados.rados.0.smithi136.stdout:3281: finishing copy_from to smithi13667081-29 2024-09-12T23:49:36.583 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 29 v 2441 (ObjNum 1027 snap 317 seq_num 1027) dirty exists 2024-09-12T23:49:36.595 INFO:tasks.rados.rados.0.smithi136.stdout:3276: expect (ObjNum 993 snap 307 seq_num 993) 2024-09-12T23:49:36.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:36 smithi136 bash[75164]: cluster 2024-09-12T23:49:35.359755+0000 mon.a (mon.0) 1153 : cluster [DBG] osdmap e712: 8 total, 8 up, 8 in 2024-09-12T23:49:36.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:36 smithi136 bash[76371]: cluster 2024-09-12T23:49:35.359755+0000 mon.a (mon.0) 1153 : cluster [DBG] osdmap e712: 8 total, 8 up, 8 in 2024-09-12T23:49:36.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:36 smithi181 bash[67587]: cluster 2024-09-12T23:49:35.359755+0000 mon.a (mon.0) 1153 : cluster [DBG] osdmap e712: 8 total, 8 up, 8 in 2024-09-12T23:49:37.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:37 smithi136 bash[75164]: cluster 2024-09-12T23:49:36.364342+0000 mon.a (mon.0) 1154 : cluster [DBG] osdmap e713: 8 total, 8 up, 8 in 2024-09-12T23:49:37.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:37 smithi136 bash[75164]: cluster 2024-09-12T23:49:36.464195+0000 mon.a (mon.0) 1155 : cluster [DBG] osdmap e714: 8 total, 8 up, 8 in 2024-09-12T23:49:37.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:37 smithi136 bash[75164]: cluster 2024-09-12T23:49:36.830391+0000 mgr.y (mgr.24661) 359 : cluster [DBG] pgmap v650: 121 pgs: 121 active+clean; 287 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 3.0 MiB/s wr, 16 op/s 2024-09-12T23:49:37.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:37 smithi136 bash[76371]: cluster 2024-09-12T23:49:36.364342+0000 mon.a (mon.0) 1154 : cluster [DBG] osdmap e713: 8 total, 8 up, 8 in 2024-09-12T23:49:37.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:37 smithi136 bash[76371]: cluster 2024-09-12T23:49:36.464195+0000 mon.a (mon.0) 1155 : cluster [DBG] osdmap e714: 8 total, 8 up, 8 in 2024-09-12T23:49:37.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:37 smithi136 bash[76371]: cluster 2024-09-12T23:49:36.830391+0000 mgr.y (mgr.24661) 359 : cluster [DBG] pgmap v650: 121 pgs: 121 active+clean; 287 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 3.0 MiB/s wr, 16 op/s 2024-09-12T23:49:37.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:37 smithi181 bash[67587]: cluster 2024-09-12T23:49:36.364342+0000 mon.a (mon.0) 1154 : cluster [DBG] osdmap e713: 8 total, 8 up, 8 in 2024-09-12T23:49:37.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:37 smithi181 bash[67587]: cluster 2024-09-12T23:49:36.464195+0000 mon.a (mon.0) 1155 : cluster [DBG] osdmap e714: 8 total, 8 up, 8 in 2024-09-12T23:49:37.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:37 smithi181 bash[67587]: cluster 2024-09-12T23:49:36.830391+0000 mgr.y (mgr.24661) 359 : cluster [DBG] pgmap v650: 121 pgs: 121 active+clean; 287 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 3.0 MiB/s wr, 16 op/s 2024-09-12T23:49:38.704 INFO:tasks.rados.rados.0.smithi136.stdout:3276: done (5 left) 2024-09-12T23:49:38.707 INFO:tasks.rados.rados.0.smithi136.stdout:3278: done (4 left) 2024-09-12T23:49:38.707 INFO:tasks.rados.rados.0.smithi136.stdout:3279: done (3 left) 2024-09-12T23:49:38.707 INFO:tasks.rados.rados.0.smithi136.stdout:3280: done (2 left) 2024-09-12T23:49:38.707 INFO:tasks.rados.rados.0.smithi136.stdout:3281: done (1 left) 2024-09-12T23:49:38.708 INFO:tasks.rados.rados.0.smithi136.stdout:3282: done (0 left) 2024-09-12T23:49:38.708 INFO:tasks.rados.rados.0.smithi136.stdout:3283: read oid 38 snap 315 2024-09-12T23:49:38.708 INFO:tasks.rados.rados.0.smithi136.stdout:3283: expect (ObjNum 864 snap 281 seq_num 864) 2024-09-12T23:49:38.708 INFO:tasks.rados.rados.0.smithi136.stdout:3284: delete oid 20 current snap is 318 2024-09-12T23:49:38.710 INFO:tasks.rados.rados.0.smithi136.stdout:3284: done (1 left) 2024-09-12T23:49:38.710 INFO:tasks.rados.rados.0.smithi136.stdout:3285: snap_create 2024-09-12T23:49:38.725 INFO:tasks.rados.rados.0.smithi136.stdout:3283: expect (ObjNum 864 snap 281 seq_num 864) 2024-09-12T23:49:38.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:38 smithi136 bash[75164]: cluster 2024-09-12T23:49:37.473416+0000 mon.a (mon.0) 1156 : cluster [DBG] osdmap e715: 8 total, 8 up, 8 in 2024-09-12T23:49:38.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:38 smithi136 bash[75164]: audit 2024-09-12T23:49:37.614195+0000 mon.a (mon.0) 1157 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:38.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:38 smithi136 bash[75164]: audit 2024-09-12T23:49:37.625476+0000 mon.a (mon.0) 1158 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:38.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:38 smithi136 bash[75164]: audit 2024-09-12T23:49:37.627359+0000 mon.a (mon.0) 1159 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:49:38.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:38 smithi136 bash[75164]: audit 2024-09-12T23:49:37.628785+0000 mon.a (mon.0) 1160 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:49:38.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:38 smithi136 bash[75164]: audit 2024-09-12T23:49:37.639487+0000 mon.a (mon.0) 1161 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:38.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:38 smithi136 bash[75164]: audit 2024-09-12T23:49:37.708862+0000 mon.a (mon.0) 1162 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-12T23:49:38.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:38 smithi136 bash[75164]: audit 2024-09-12T23:49:37.709744+0000 mgr.y (mgr.24661) 360 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-12T23:49:38.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:38 smithi136 bash[75164]: audit 2024-09-12T23:49:37.790793+0000 mon.a (mon.0) 1163 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:49:38.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:38 smithi136 bash[75164]: audit 2024-09-12T23:49:37.794339+0000 mon.a (mon.0) 1164 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:38.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:38 smithi136 bash[75164]: audit 2024-09-12T23:49:37.796455+0000 mon.a (mon.0) 1165 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:38.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:38 smithi136 bash[75164]: audit 2024-09-12T23:49:37.797809+0000 mon.a (mon.0) 1166 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:38.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:38 smithi136 bash[75164]: audit 2024-09-12T23:49:37.800340+0000 mon.a (mon.0) 1167 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:38.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:38 smithi136 bash[75164]: audit 2024-09-12T23:49:37.803240+0000 mon.a (mon.0) 1168 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:38.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:38 smithi136 bash[75164]: audit 2024-09-12T23:49:37.804841+0000 mon.a (mon.0) 1169 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:38.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:38 smithi136 bash[75164]: audit 2024-09-12T23:49:37.806414+0000 mon.a (mon.0) 1170 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:38.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:38 smithi136 bash[75164]: audit 2024-09-12T23:49:37.808014+0000 mon.a (mon.0) 1171 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:38.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:38 smithi136 bash[75164]: audit 2024-09-12T23:49:37.809403+0000 mon.a (mon.0) 1172 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:38.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:38 smithi136 bash[75164]: audit 2024-09-12T23:49:37.810697+0000 mon.a (mon.0) 1173 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:38.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:38 smithi136 bash[75164]: audit 2024-09-12T23:49:37.812068+0000 mon.a (mon.0) 1174 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:38.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:38 smithi136 bash[75164]: audit 2024-09-12T23:49:37.816005+0000 mon.a (mon.0) 1175 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:38.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:38 smithi136 bash[75164]: audit 2024-09-12T23:49:37.818797+0000 mon.a (mon.0) 1176 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:38.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:38 smithi136 bash[75164]: audit 2024-09-12T23:49:37.821303+0000 mon.a (mon.0) 1177 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:38.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:38 smithi136 bash[76371]: cluster 2024-09-12T23:49:37.473416+0000 mon.a (mon.0) 1156 : cluster [DBG] osdmap e715: 8 total, 8 up, 8 in 2024-09-12T23:49:38.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:38 smithi136 bash[76371]: audit 2024-09-12T23:49:37.614195+0000 mon.a (mon.0) 1157 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:38.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:38 smithi136 bash[76371]: audit 2024-09-12T23:49:37.625476+0000 mon.a (mon.0) 1158 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:38.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:38 smithi136 bash[76371]: audit 2024-09-12T23:49:37.627359+0000 mon.a (mon.0) 1159 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:49:38.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:38 smithi136 bash[76371]: audit 2024-09-12T23:49:37.628785+0000 mon.a (mon.0) 1160 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:49:38.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:38 smithi136 bash[76371]: audit 2024-09-12T23:49:37.639487+0000 mon.a (mon.0) 1161 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:38.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:38 smithi136 bash[76371]: audit 2024-09-12T23:49:37.708862+0000 mon.a (mon.0) 1162 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-12T23:49:38.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:38 smithi136 bash[76371]: audit 2024-09-12T23:49:37.709744+0000 mgr.y (mgr.24661) 360 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-12T23:49:38.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:38 smithi136 bash[76371]: audit 2024-09-12T23:49:37.790793+0000 mon.a (mon.0) 1163 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:49:38.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:38 smithi136 bash[76371]: audit 2024-09-12T23:49:37.794339+0000 mon.a (mon.0) 1164 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:38.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:38 smithi136 bash[76371]: audit 2024-09-12T23:49:37.796455+0000 mon.a (mon.0) 1165 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:38.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:38 smithi136 bash[76371]: audit 2024-09-12T23:49:37.797809+0000 mon.a (mon.0) 1166 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:38.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:38 smithi136 bash[76371]: audit 2024-09-12T23:49:37.800340+0000 mon.a (mon.0) 1167 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:38.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:38 smithi136 bash[76371]: audit 2024-09-12T23:49:37.803240+0000 mon.a (mon.0) 1168 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:38.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:38 smithi136 bash[76371]: audit 2024-09-12T23:49:37.804841+0000 mon.a (mon.0) 1169 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:38.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:38 smithi136 bash[76371]: audit 2024-09-12T23:49:37.806414+0000 mon.a (mon.0) 1170 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:38.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:38 smithi136 bash[76371]: audit 2024-09-12T23:49:37.808014+0000 mon.a (mon.0) 1171 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:38.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:38 smithi136 bash[76371]: audit 2024-09-12T23:49:37.809403+0000 mon.a (mon.0) 1172 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:38.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:38 smithi136 bash[76371]: audit 2024-09-12T23:49:37.810697+0000 mon.a (mon.0) 1173 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:38.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:38 smithi136 bash[76371]: audit 2024-09-12T23:49:37.812068+0000 mon.a (mon.0) 1174 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:38.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:38 smithi136 bash[76371]: audit 2024-09-12T23:49:37.816005+0000 mon.a (mon.0) 1175 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:38.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:38 smithi136 bash[76371]: audit 2024-09-12T23:49:37.818797+0000 mon.a (mon.0) 1176 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:38.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:38 smithi136 bash[76371]: audit 2024-09-12T23:49:37.821303+0000 mon.a (mon.0) 1177 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:38.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:38 smithi181 bash[67587]: cluster 2024-09-12T23:49:37.473416+0000 mon.a (mon.0) 1156 : cluster [DBG] osdmap e715: 8 total, 8 up, 8 in 2024-09-12T23:49:38.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:38 smithi181 bash[67587]: audit 2024-09-12T23:49:37.614195+0000 mon.a (mon.0) 1157 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:38.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:38 smithi181 bash[67587]: audit 2024-09-12T23:49:37.625476+0000 mon.a (mon.0) 1158 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:38.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:38 smithi181 bash[67587]: audit 2024-09-12T23:49:37.627359+0000 mon.a (mon.0) 1159 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:49:38.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:38 smithi181 bash[67587]: audit 2024-09-12T23:49:37.628785+0000 mon.a (mon.0) 1160 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:49:38.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:38 smithi181 bash[67587]: audit 2024-09-12T23:49:37.639487+0000 mon.a (mon.0) 1161 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:38.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:38 smithi181 bash[67587]: audit 2024-09-12T23:49:37.708862+0000 mon.a (mon.0) 1162 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-12T23:49:38.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:38 smithi181 bash[67587]: audit 2024-09-12T23:49:37.709744+0000 mgr.y (mgr.24661) 360 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-12T23:49:38.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:38 smithi181 bash[67587]: audit 2024-09-12T23:49:37.790793+0000 mon.a (mon.0) 1163 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:49:38.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:38 smithi181 bash[67587]: audit 2024-09-12T23:49:37.794339+0000 mon.a (mon.0) 1164 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:38.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:38 smithi181 bash[67587]: audit 2024-09-12T23:49:37.796455+0000 mon.a (mon.0) 1165 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:38.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:38 smithi181 bash[67587]: audit 2024-09-12T23:49:37.797809+0000 mon.a (mon.0) 1166 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:38.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:38 smithi181 bash[67587]: audit 2024-09-12T23:49:37.800340+0000 mon.a (mon.0) 1167 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:38.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:38 smithi181 bash[67587]: audit 2024-09-12T23:49:37.803240+0000 mon.a (mon.0) 1168 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:38.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:38 smithi181 bash[67587]: audit 2024-09-12T23:49:37.804841+0000 mon.a (mon.0) 1169 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:38.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:38 smithi181 bash[67587]: audit 2024-09-12T23:49:37.806414+0000 mon.a (mon.0) 1170 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:38.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:38 smithi181 bash[67587]: audit 2024-09-12T23:49:37.808014+0000 mon.a (mon.0) 1171 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:38.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:38 smithi181 bash[67587]: audit 2024-09-12T23:49:37.809403+0000 mon.a (mon.0) 1172 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:38.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:38 smithi181 bash[67587]: audit 2024-09-12T23:49:37.810697+0000 mon.a (mon.0) 1173 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:38.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:38 smithi181 bash[67587]: audit 2024-09-12T23:49:37.812068+0000 mon.a (mon.0) 1174 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:38.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:38 smithi181 bash[67587]: audit 2024-09-12T23:49:37.816005+0000 mon.a (mon.0) 1175 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:38.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:38 smithi181 bash[67587]: audit 2024-09-12T23:49:37.818797+0000 mon.a (mon.0) 1176 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:38.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:38 smithi181 bash[67587]: audit 2024-09-12T23:49:37.821303+0000 mon.a (mon.0) 1177 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:39.485 INFO:tasks.rados.rados.0.smithi136.stdout:3283: done (1 left) 2024-09-12T23:49:39.485 INFO:tasks.rados.rados.0.smithi136.stdout:3285: done (0 left) 2024-09-12T23:49:39.485 INFO:tasks.rados.rados.0.smithi136.stdout:3286: rollback oid 31 current snap is 319 2024-09-12T23:49:39.485 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 31 to 313 2024-09-12T23:49:39.485 INFO:tasks.rados.rados.0.smithi136.stdout:3287: read oid 45 snap -1 2024-09-12T23:49:39.485 INFO:tasks.rados.rados.0.smithi136.stdout:3287: expect (ObjNum 889 snap 288 seq_num 889) 2024-09-12T23:49:39.485 INFO:tasks.rados.rados.0.smithi136.stdout:3288: read oid 1 snap -1 2024-09-12T23:49:39.485 INFO:tasks.rados.rados.0.smithi136.stdout:3288: expect (ObjNum 987 snap 303 seq_num 987) 2024-09-12T23:49:39.485 INFO:tasks.rados.rados.0.smithi136.stdout:3289: snap_remove snap 295 2024-09-12T23:49:39.685 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:39 smithi136 bash[76371]: audit 2024-09-12T23:49:38.751024+0000 mon.a (mon.0) 1178 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:49:39.685 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:39 smithi136 bash[76371]: cluster 2024-09-12T23:49:38.831262+0000 mgr.y (mgr.24661) 361 : cluster [DBG] pgmap v652: 121 pgs: 121 active+clean; 288 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 1.9 MiB/s wr, 9 op/s 2024-09-12T23:49:39.686 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:39 smithi136 bash[75164]: audit 2024-09-12T23:49:38.751024+0000 mon.a (mon.0) 1178 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:49:39.686 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:39 smithi136 bash[75164]: cluster 2024-09-12T23:49:38.831262+0000 mgr.y (mgr.24661) 361 : cluster [DBG] pgmap v652: 121 pgs: 121 active+clean; 288 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 1.9 MiB/s wr, 9 op/s 2024-09-12T23:49:39.686 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:49:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:49:39] "GET /metrics HTTP/1.1" 200 36439 "" "Prometheus/2.43.0" 2024-09-12T23:49:39.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:39 smithi181 bash[67587]: audit 2024-09-12T23:49:38.751024+0000 mon.a (mon.0) 1178 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:49:39.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:39 smithi181 bash[67587]: cluster 2024-09-12T23:49:38.831262+0000 mgr.y (mgr.24661) 361 : cluster [DBG] pgmap v652: 121 pgs: 121 active+clean; 288 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 1.9 MiB/s wr, 9 op/s 2024-09-12T23:49:40.487 INFO:tasks.rados.rados.0.smithi136.stdout:3289: done (3 left) 2024-09-12T23:49:40.487 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 41 current snap is 319 2024-09-12T23:49:40.487 INFO:tasks.rados.rados.0.smithi136.stdout:3290: seq_num 1035 ranges {3956736=540672} 2024-09-12T23:49:40.543 INFO:tasks.rados.rados.0.smithi136.stdout:3290: writing smithi13667081-41 from 3956736 to 4497408 tid 1 2024-09-12T23:49:40.544 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 22 current snap is 319 2024-09-12T23:49:40.544 INFO:tasks.rados.rados.0.smithi136.stdout:3291: seq_num 1036 ranges {0=557056} 2024-09-12T23:49:40.552 INFO:tasks.rados.rados.0.smithi136.stdout:3291: writing smithi13667081-22 from 0 to 557056 tid 1 2024-09-12T23:49:40.552 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 19 current snap is 319 2024-09-12T23:49:40.552 INFO:tasks.rados.rados.0.smithi136.stdout:3286: finishing rollback tid 1 to smithi13667081-31 2024-09-12T23:49:40.552 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 31 v 1662 (ObjNum 33 snap 0 seq_num 4242948448) dirty dne 2024-09-12T23:49:40.553 INFO:tasks.rados.rados.0.smithi136.stdout:3287: expect (ObjNum 889 snap 288 seq_num 889) 2024-09-12T23:49:40.586 INFO:tasks.rados.rados.0.smithi136.stdout:3288: expect (ObjNum 987 snap 303 seq_num 987) 2024-09-12T23:49:40.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:40 smithi136 bash[75164]: cluster 2024-09-12T23:49:39.484098+0000 mon.a (mon.0) 1179 : cluster [DBG] osdmap e716: 8 total, 8 up, 8 in 2024-09-12T23:49:40.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:40 smithi136 bash[76371]: cluster 2024-09-12T23:49:39.484098+0000 mon.a (mon.0) 1179 : cluster [DBG] osdmap e716: 8 total, 8 up, 8 in 2024-09-12T23:49:40.831 INFO:tasks.rados.rados.0.smithi136.stdout:3291: finishing write tid 1 to smithi13667081-22 2024-09-12T23:49:40.831 INFO:tasks.rados.rados.0.smithi136.stdout:3291: finishing write tid 2 to smithi13667081-22 2024-09-12T23:49:40.832 INFO:tasks.rados.rados.0.smithi136.stdout:3290: finishing write tid 1 to smithi13667081-41 2024-09-12T23:49:40.832 INFO:tasks.rados.rados.0.smithi136.stdout:3290: finishing write tid 2 to smithi13667081-41 2024-09-12T23:49:40.832 INFO:tasks.rados.rados.0.smithi136.stdout:3290: finishing write tid 3 to smithi13667081-41 2024-09-12T23:49:40.832 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 41 v 2466 (ObjNum 1035 snap 319 seq_num 1035) dirty exists 2024-09-12T23:49:40.832 INFO:tasks.rados.rados.0.smithi136.stdout:3290: left oid 41 (ObjNum 1035 snap 319 seq_num 1035) 2024-09-12T23:49:40.832 INFO:tasks.rados.rados.0.smithi136.stdout:3291: finishing write tid 3 to smithi13667081-22 2024-09-12T23:49:40.832 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 22 v 2468 (ObjNum 1036 snap 319 seq_num 1036) dirty exists 2024-09-12T23:49:40.832 INFO:tasks.rados.rados.0.smithi136.stdout:3291: left oid 22 (ObjNum 1036 snap 319 seq_num 1036) 2024-09-12T23:49:40.832 INFO:tasks.rados.rados.0.smithi136.stdout:3292: seq_num 1037 ranges {0=573440} 2024-09-12T23:49:40.838 INFO:tasks.rados.rados.0.smithi136.stdout:3292: writing smithi13667081-19 from 0 to 573440 tid 1 2024-09-12T23:49:40.838 INFO:tasks.rados.rados.0.smithi136.stdout:3286: done (5 left) 2024-09-12T23:49:40.838 INFO:tasks.rados.rados.0.smithi136.stdout:3287: done (4 left) 2024-09-12T23:49:40.838 INFO:tasks.rados.rados.0.smithi136.stdout:3288: done (3 left) 2024-09-12T23:49:40.838 INFO:tasks.rados.rados.0.smithi136.stdout:3290: done (2 left) 2024-09-12T23:49:40.838 INFO:tasks.rados.rados.0.smithi136.stdout:3291: done (1 left) 2024-09-12T23:49:40.838 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 1 current snap is 319 2024-09-12T23:49:40.838 INFO:tasks.rados.rados.0.smithi136.stdout:3293: seq_num 1038 ranges {5881856=589824} 2024-09-12T23:49:40.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:40 smithi181 bash[67587]: cluster 2024-09-12T23:49:39.484098+0000 mon.a (mon.0) 1179 : cluster [DBG] osdmap e716: 8 total, 8 up, 8 in 2024-09-12T23:49:40.891 INFO:tasks.rados.rados.0.smithi136.stdout:3293: writing smithi13667081-1 from 5881856 to 6471680 tid 1 2024-09-12T23:49:40.891 INFO:tasks.rados.rados.0.smithi136.stdout:3294: read oid 17 snap -1 2024-09-12T23:49:40.891 INFO:tasks.rados.rados.0.smithi136.stdout:3294: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-12T23:49:40.891 INFO:tasks.rados.rados.0.smithi136.stdout:3295: setattr oid 30 current snap is 319 2024-09-12T23:49:40.891 INFO:tasks.rados.rados.0.smithi136.stdout:3292: finishing write tid 1 to smithi13667081-19 2024-09-12T23:49:40.891 INFO:tasks.rados.rados.0.smithi136.stdout:3292: finishing write tid 2 to smithi13667081-19 2024-09-12T23:49:40.891 INFO:tasks.rados.rados.0.smithi136.stdout:3292: finishing write tid 3 to smithi13667081-19 2024-09-12T23:49:40.891 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 19 v 1646 (ObjNum 1037 snap 319 seq_num 1037) dirty exists 2024-09-12T23:49:40.891 INFO:tasks.rados.rados.0.smithi136.stdout:3292: left oid 19 (ObjNum 1037 snap 319 seq_num 1037) 2024-09-12T23:49:40.892 INFO:tasks.rados.rados.0.smithi136.stdout:3294: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-12T23:49:40.897 INFO:tasks.rados.rados.0.smithi136.stdout:3293: finishing write tid 1 to smithi13667081-1 2024-09-12T23:49:40.897 INFO:tasks.rados.rados.0.smithi136.stdout:3293: finishing write tid 2 to smithi13667081-1 2024-09-12T23:49:40.897 INFO:tasks.rados.rados.0.smithi136.stdout:3292: done (3 left) 2024-09-12T23:49:40.897 INFO:tasks.rados.rados.0.smithi136.stdout:3294: done (2 left) 2024-09-12T23:49:40.897 INFO:tasks.rados.rados.0.smithi136.stdout:3296: delete oid 33 current snap is 319 2024-09-12T23:49:40.899 INFO:tasks.rados.rados.0.smithi136.stdout:3293: finishing write tid 3 to smithi13667081-1 2024-09-12T23:49:40.899 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 1 v 2470 (ObjNum 1038 snap 319 seq_num 1038) dirty exists 2024-09-12T23:49:40.899 INFO:tasks.rados.rados.0.smithi136.stdout:3293: left oid 1 (ObjNum 1038 snap 319 seq_num 1038) 2024-09-12T23:49:40.900 INFO:tasks.rados.rados.0.smithi136.stdout:3293: done (2 left) 2024-09-12T23:49:40.900 INFO:tasks.rados.rados.0.smithi136.stdout:3296: done (1 left) 2024-09-12T23:49:40.900 INFO:tasks.rados.rados.0.smithi136.stdout:3297: snap_create 2024-09-12T23:49:40.901 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 30 v 1506 (ObjNum 129 snap 0 seq_num 4243597248) dirty exists 2024-09-12T23:49:41.465 INFO:tasks.rados.rados.0.smithi136.stdout:3295: done (1 left) 2024-09-12T23:49:41.465 INFO:tasks.rados.rados.0.smithi136.stdout:3297: done (0 left) 2024-09-12T23:49:41.465 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 45 current snap is 320 2024-09-12T23:49:41.465 INFO:tasks.rados.rados.0.smithi136.stdout:3298: seq_num 1040 ranges {819200=622592} 2024-09-12T23:49:41.477 INFO:tasks.rados.rados.0.smithi136.stdout:3298: writing smithi13667081-45 from 819200 to 1441792 tid 1 2024-09-12T23:49:41.477 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 1 current snap is 320 2024-09-12T23:49:41.478 INFO:tasks.rados.rados.0.smithi136.stdout:3299: seq_num 1041 ranges {6471680=638976} 2024-09-12T23:49:41.532 INFO:tasks.rados.rados.0.smithi136.stdout:3299: writing smithi13667081-1 from 6471680 to 7110656 tid 1 2024-09-12T23:49:41.532 INFO:tasks.rados.rados.0.smithi136.stdout:3300: snap_remove snap 313 2024-09-12T23:49:41.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:41 smithi136 bash[75164]: cluster 2024-09-12T23:49:40.489846+0000 mon.a (mon.0) 1180 : cluster [DBG] osdmap e717: 8 total, 8 up, 8 in 2024-09-12T23:49:41.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:41 smithi136 bash[75164]: cephadm 2024-09-12T23:49:40.512113+0000 mgr.y (mgr.24661) 362 : cephadm [INF] Upgrade: Updating alertmanager.a 2024-09-12T23:49:41.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:41 smithi136 bash[75164]: cephadm 2024-09-12T23:49:40.522498+0000 mgr.y (mgr.24661) 363 : cephadm [INF] Deploying daemon alertmanager.a on smithi136 2024-09-12T23:49:41.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:41 smithi136 bash[75164]: cluster 2024-09-12T23:49:40.831891+0000 mgr.y (mgr.24661) 364 : cluster [DBG] pgmap v655: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 295 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 3.0 MiB/s wr, 9 op/s 2024-09-12T23:49:41.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:41 smithi136 bash[75164]: cluster 2024-09-12T23:49:41.466929+0000 mon.a (mon.0) 1181 : cluster [DBG] osdmap e718: 8 total, 8 up, 8 in 2024-09-12T23:49:41.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:41 smithi136 bash[76371]: cluster 2024-09-12T23:49:40.489846+0000 mon.a (mon.0) 1180 : cluster [DBG] osdmap e717: 8 total, 8 up, 8 in 2024-09-12T23:49:41.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:41 smithi136 bash[76371]: cephadm 2024-09-12T23:49:40.512113+0000 mgr.y (mgr.24661) 362 : cephadm [INF] Upgrade: Updating alertmanager.a 2024-09-12T23:49:41.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:41 smithi136 bash[76371]: cephadm 2024-09-12T23:49:40.522498+0000 mgr.y (mgr.24661) 363 : cephadm [INF] Deploying daemon alertmanager.a on smithi136 2024-09-12T23:49:41.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:41 smithi136 bash[76371]: cluster 2024-09-12T23:49:40.831891+0000 mgr.y (mgr.24661) 364 : cluster [DBG] pgmap v655: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 295 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 3.0 MiB/s wr, 9 op/s 2024-09-12T23:49:41.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:41 smithi136 bash[76371]: cluster 2024-09-12T23:49:41.466929+0000 mon.a (mon.0) 1181 : cluster [DBG] osdmap e718: 8 total, 8 up, 8 in 2024-09-12T23:49:41.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:41 smithi181 bash[67587]: cluster 2024-09-12T23:49:40.489846+0000 mon.a (mon.0) 1180 : cluster [DBG] osdmap e717: 8 total, 8 up, 8 in 2024-09-12T23:49:41.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:41 smithi181 bash[67587]: cephadm 2024-09-12T23:49:40.512113+0000 mgr.y (mgr.24661) 362 : cephadm [INF] Upgrade: Updating alertmanager.a 2024-09-12T23:49:41.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:41 smithi181 bash[67587]: cephadm 2024-09-12T23:49:40.522498+0000 mgr.y (mgr.24661) 363 : cephadm [INF] Deploying daemon alertmanager.a on smithi136 2024-09-12T23:49:41.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:41 smithi181 bash[67587]: cluster 2024-09-12T23:49:40.831891+0000 mgr.y (mgr.24661) 364 : cluster [DBG] pgmap v655: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 295 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 3.0 MiB/s wr, 9 op/s 2024-09-12T23:49:41.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:41 smithi181 bash[67587]: cluster 2024-09-12T23:49:41.466929+0000 mon.a (mon.0) 1181 : cluster [DBG] osdmap e718: 8 total, 8 up, 8 in 2024-09-12T23:49:42.516 INFO:tasks.rados.rados.0.smithi136.stdout:3300: done (2 left) 2024-09-12T23:49:42.516 INFO:tasks.rados.rados.0.smithi136.stdout:3301: read oid 9 snap 303 2024-09-12T23:49:42.516 INFO:tasks.rados.rados.0.smithi136.stdout:3301: expect (ObjNum 963 snap 301 seq_num 963) 2024-09-12T23:49:42.518 INFO:tasks.rados.rados.0.smithi136.stdout:3302: read oid 10 snap 315 2024-09-12T23:49:42.518 INFO:tasks.rados.rados.0.smithi136.stdout:3302: expect deleted 2024-09-12T23:49:42.518 INFO:tasks.rados.rados.0.smithi136.stdout:3303: delete oid 16 current snap is 320 2024-09-12T23:49:42.518 INFO:tasks.rados.rados.0.smithi136.stdout:3298: finishing write tid 1 to smithi13667081-45 2024-09-12T23:49:42.518 INFO:tasks.rados.rados.0.smithi136.stdout:3298: finishing write tid 2 to smithi13667081-45 2024-09-12T23:49:42.518 INFO:tasks.rados.rados.0.smithi136.stdout:3298: finishing write tid 3 to smithi13667081-45 2024-09-12T23:49:42.518 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 45 v 2342 (ObjNum 1040 snap 320 seq_num 1040) dirty exists 2024-09-12T23:49:42.518 INFO:tasks.rados.rados.0.smithi136.stdout:3298: left oid 45 (ObjNum 1040 snap 320 seq_num 1040) 2024-09-12T23:49:42.518 INFO:tasks.rados.rados.0.smithi136.stdout:3299: finishing write tid 1 to smithi13667081-1 2024-09-12T23:49:42.519 INFO:tasks.rados.rados.0.smithi136.stdout:3299: finishing write tid 2 to smithi13667081-1 2024-09-12T23:49:42.519 INFO:tasks.rados.rados.0.smithi136.stdout:3299: finishing write tid 3 to smithi13667081-1 2024-09-12T23:49:42.519 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 1 v 2472 (ObjNum 1041 snap 320 seq_num 1041) dirty exists 2024-09-12T23:49:42.519 INFO:tasks.rados.rados.0.smithi136.stdout:3299: left oid 1 (ObjNum 1041 snap 320 seq_num 1041) 2024-09-12T23:49:42.540 INFO:tasks.rados.rados.0.smithi136.stdout:3298: done (4 left) 2024-09-12T23:49:42.540 INFO:tasks.rados.rados.0.smithi136.stdout:3299: done (3 left) 2024-09-12T23:49:42.540 INFO:tasks.rados.rados.0.smithi136.stdout:3302: done (2 left) 2024-09-12T23:49:42.541 INFO:tasks.rados.rados.0.smithi136.stdout:3303: done (1 left) 2024-09-12T23:49:42.541 INFO:tasks.rados.rados.0.smithi136.stdout:3304: read oid 16 snap 310 2024-09-12T23:49:42.541 INFO:tasks.rados.rados.0.smithi136.stdout:3304: expect (ObjNum 977 snap 303 seq_num 977) 2024-09-12T23:49:42.541 INFO:tasks.rados.rados.0.smithi136.stdout:3305: setattr oid 40 current snap is 320 2024-09-12T23:49:42.543 INFO:tasks.rados.rados.0.smithi136.stdout:3306: copy_from oid 32 from oid 4 current snap is 320 2024-09-12T23:49:42.543 INFO:tasks.rados.rados.0.smithi136.stdout:3307: delete oid 47 current snap is 320 2024-09-12T23:49:42.554 INFO:tasks.rados.rados.0.smithi136.stdout:3301: expect (ObjNum 963 snap 301 seq_num 963) 2024-09-12T23:49:42.595 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:49:42 smithi181 bash[95505]: ts=2024-09-12T23:49:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:49:42.617 INFO:tasks.rados.rados.0.smithi136.stdout:3304: expect (ObjNum 977 snap 303 seq_num 977) 2024-09-12T23:49:42.685 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 40 v 2569 (ObjNum 983 snap 303 seq_num 983) dirty exists 2024-09-12T23:49:42.685 INFO:tasks.rados.rados.0.smithi136.stdout:3306: finishing copy_from racing read to smithi13667081-32 2024-09-12T23:49:42.685 INFO:tasks.rados.rados.0.smithi136.stdout:3306: finishing copy_from to smithi13667081-32 2024-09-12T23:49:42.685 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 32 v 1640 (ObjNum 1026 snap 316 seq_num 1026) dirty exists 2024-09-12T23:49:42.685 INFO:tasks.rados.rados.0.smithi136.stdout:3301: done (4 left) 2024-09-12T23:49:42.685 INFO:tasks.rados.rados.0.smithi136.stdout:3304: done (3 left) 2024-09-12T23:49:42.685 INFO:tasks.rados.rados.0.smithi136.stdout:3305: done (2 left) 2024-09-12T23:49:42.685 INFO:tasks.rados.rados.0.smithi136.stdout:3306: done (1 left) 2024-09-12T23:49:42.685 INFO:tasks.rados.rados.0.smithi136.stdout:3307: done (0 left) 2024-09-12T23:49:42.686 INFO:tasks.rados.rados.0.smithi136.stdout:3308: snap_create 2024-09-12T23:49:43.524 INFO:tasks.rados.rados.0.smithi136.stdout:3308: done (0 left) 2024-09-12T23:49:43.525 INFO:tasks.rados.rados.0.smithi136.stdout:3309: copy_from oid 14 from oid 19 current snap is 321 2024-09-12T23:49:43.525 INFO:tasks.rados.rados.0.smithi136.stdout:3310: rmattr oid 49 current snap is 321 2024-09-12T23:49:43.525 INFO:tasks.rados.rados.0.smithi136.stdout:3311: delete oid 31 current snap is 321 2024-09-12T23:49:43.537 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 49 v 1839 (ObjNum 273 snap 0 seq_num 0) dirty exists 2024-09-12T23:49:43.539 INFO:tasks.rados.rados.0.smithi136.stdout:3310: done (2 left) 2024-09-12T23:49:43.539 INFO:tasks.rados.rados.0.smithi136.stdout:3311: done (1 left) 2024-09-12T23:49:43.539 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 39 current snap is 321 2024-09-12T23:49:43.539 INFO:tasks.rados.rados.0.smithi136.stdout:3312: seq_num 1045 ranges {253952=704512} 2024-09-12T23:49:43.556 INFO:tasks.rados.rados.0.smithi136.stdout:3312: writing smithi13667081-39 from 253952 to 958464 tid 1 2024-09-12T23:49:43.556 INFO:tasks.rados.rados.0.smithi136.stdout:3313: delete oid 5 current snap is 321 2024-09-12T23:49:43.556 INFO:tasks.rados.rados.0.smithi136.stdout:3309: finishing copy_from racing read to smithi13667081-14 2024-09-12T23:49:43.556 INFO:tasks.rados.rados.0.smithi136.stdout:3309: finishing copy_from to smithi13667081-14 2024-09-12T23:49:43.556 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 14 v 1646 (ObjNum 1037 snap 319 seq_num 1037) dirty exists 2024-09-12T23:49:43.561 INFO:tasks.rados.rados.0.smithi136.stdout:3309: done (2 left) 2024-09-12T23:49:43.561 INFO:tasks.rados.rados.0.smithi136.stdout:3313: done (1 left) 2024-09-12T23:49:43.561 INFO:tasks.rados.rados.0.smithi136.stdout:3314: rmattr oid 47 current snap is 321 2024-09-12T23:49:43.561 INFO:tasks.rados.rados.0.smithi136.stdout:3314: done (1 left) 2024-09-12T23:49:43.561 INFO:tasks.rados.rados.0.smithi136.stdout:3315: read oid 47 snap 319 2024-09-12T23:49:43.562 INFO:tasks.rados.rados.0.smithi136.stdout:3315: expect (ObjNum 990 snap 304 seq_num 990) 2024-09-12T23:49:43.562 INFO:tasks.rados.rados.0.smithi136.stdout:3316: read oid 15 snap -1 2024-09-12T23:49:43.562 INFO:tasks.rados.rados.0.smithi136.stdout:3316: expect (ObjNum 993 snap 307 seq_num 993) 2024-09-12T23:49:43.562 INFO:tasks.rados.rados.0.smithi136.stdout:3317: setattr oid 11 current snap is 321 2024-09-12T23:49:43.568 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 27 current snap is 321 2024-09-12T23:49:43.568 INFO:tasks.rados.rados.0.smithi136.stdout:3318: seq_num 1047 ranges {0=745472} 2024-09-12T23:49:43.576 INFO:tasks.rados.rados.0.smithi136.stdout:3318: writing smithi13667081-27 from 0 to 745472 tid 1 2024-09-12T23:49:43.577 INFO:tasks.rados.rados.0.smithi136.stdout:3319: copy_from oid 42 from oid 49 current snap is 321 2024-09-12T23:49:43.577 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 40 current snap is 321 2024-09-12T23:49:43.577 INFO:tasks.rados.rados.0.smithi136.stdout:3320: seq_num 1048 ranges {3506176=761856} 2024-09-12T23:49:43.602 INFO:tasks.rados.rados.0.smithi136.stdout:3320: writing smithi13667081-40 from 3506176 to 4268032 tid 1 2024-09-12T23:49:43.602 INFO:tasks.rados.rados.0.smithi136.stdout:3321: read oid 38 snap -1 2024-09-12T23:49:43.602 INFO:tasks.rados.rados.0.smithi136.stdout:3321: expect (ObjNum 864 snap 281 seq_num 864) 2024-09-12T23:49:43.602 INFO:tasks.rados.rados.0.smithi136.stdout:3322: copy_from oid 17 from oid 43 current snap is 321 2024-09-12T23:49:43.602 INFO:tasks.rados.rados.0.smithi136.stdout:3312: finishing write tid 1 to smithi13667081-39 2024-09-12T23:49:43.602 INFO:tasks.rados.rados.0.smithi136.stdout:3312: finishing write tid 2 to smithi13667081-39 2024-09-12T23:49:43.602 INFO:tasks.rados.rados.0.smithi136.stdout:3315: expect (ObjNum 990 snap 304 seq_num 990) 2024-09-12T23:49:43.672 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 11 v 2573 (ObjNum 1016 snap 314 seq_num 1016) dirty exists 2024-09-12T23:49:43.672 INFO:tasks.rados.rados.0.smithi136.stdout:3312: finishing write tid 3 to smithi13667081-39 2024-09-12T23:49:43.672 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 39 v 2443 (ObjNum 1045 snap 321 seq_num 1045) dirty exists 2024-09-12T23:49:43.672 INFO:tasks.rados.rados.0.smithi136.stdout:3312: left oid 39 (ObjNum 1045 snap 321 seq_num 1045) 2024-09-12T23:49:43.672 INFO:tasks.rados.rados.0.smithi136.stdout:3318: finishing write tid 1 to smithi13667081-27 2024-09-12T23:49:43.672 INFO:tasks.rados.rados.0.smithi136.stdout:3318: finishing write tid 2 to smithi13667081-27 2024-09-12T23:49:43.673 INFO:tasks.rados.rados.0.smithi136.stdout:3319: finishing copy_from racing read to smithi13667081-42 2024-09-12T23:49:43.673 INFO:tasks.rados.rados.0.smithi136.stdout:3318: finishing write tid 3 to smithi13667081-27 2024-09-12T23:49:43.673 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 27 v 2575 (ObjNum 1047 snap 321 seq_num 1047) dirty exists 2024-09-12T23:49:43.673 INFO:tasks.rados.rados.0.smithi136.stdout:3318: left oid 27 (ObjNum 1047 snap 321 seq_num 1047) 2024-09-12T23:49:43.673 INFO:tasks.rados.rados.0.smithi136.stdout:3319: finishing copy_from to smithi13667081-42 2024-09-12T23:49:43.673 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 42 v 1839 (ObjNum 33 snap 0 seq_num 4242638448) dirty exists 2024-09-12T23:49:43.673 INFO:tasks.rados.rados.0.smithi136.stdout:3320: finishing write tid 1 to smithi13667081-40 2024-09-12T23:49:43.673 INFO:tasks.rados.rados.0.smithi136.stdout:3320: finishing write tid 2 to smithi13667081-40 2024-09-12T23:49:43.673 INFO:tasks.rados.rados.0.smithi136.stdout:3312: done (8 left) 2024-09-12T23:49:43.673 INFO:tasks.rados.rados.0.smithi136.stdout:3315: done (7 left) 2024-09-12T23:49:43.673 INFO:tasks.rados.rados.0.smithi136.stdout:3317: done (6 left) 2024-09-12T23:49:43.673 INFO:tasks.rados.rados.0.smithi136.stdout:3318: done (5 left) 2024-09-12T23:49:43.673 INFO:tasks.rados.rados.0.smithi136.stdout:3319: done (4 left) 2024-09-12T23:49:43.674 INFO:tasks.rados.rados.0.smithi136.stdout:3323: copy_from oid 5 from oid 33 current snap is 321 2024-09-12T23:49:43.674 INFO:tasks.rados.rados.0.smithi136.stdout:3320: finishing write tid 3 to smithi13667081-40 2024-09-12T23:49:43.674 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 40 v 2578 (ObjNum 1048 snap 321 seq_num 1048) dirty exists 2024-09-12T23:49:43.674 INFO:tasks.rados.rados.0.smithi136.stdout:3320: left oid 40 (ObjNum 1048 snap 321 seq_num 1048) 2024-09-12T23:49:43.674 INFO:tasks.rados.rados.0.smithi136.stdout:3321: expect (ObjNum 864 snap 281 seq_num 864) 2024-09-12T23:49:43.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:43 smithi136 bash[75164]: cluster 2024-09-12T23:49:42.514861+0000 mon.a (mon.0) 1182 : cluster [DBG] osdmap e719: 8 total, 8 up, 8 in 2024-09-12T23:49:43.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:43 smithi136 bash[75164]: cluster 2024-09-12T23:49:42.832511+0000 mgr.y (mgr.24661) 365 : cluster [DBG] pgmap v658: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 295 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.0 MiB/s wr, 5 op/s 2024-09-12T23:49:43.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:43 smithi136 bash[76371]: cluster 2024-09-12T23:49:42.514861+0000 mon.a (mon.0) 1182 : cluster [DBG] osdmap e719: 8 total, 8 up, 8 in 2024-09-12T23:49:43.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:43 smithi136 bash[76371]: cluster 2024-09-12T23:49:42.832511+0000 mgr.y (mgr.24661) 365 : cluster [DBG] pgmap v658: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 295 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.0 MiB/s wr, 5 op/s 2024-09-12T23:49:43.832 INFO:tasks.rados.rados.0.smithi136.stdout:3322: finishing copy_from racing read to smithi13667081-17 2024-09-12T23:49:43.832 INFO:tasks.rados.rados.0.smithi136.stdout:3322: finishing copy_from to smithi13667081-17 2024-09-12T23:49:43.833 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 17 v 1990 (ObjNum 1007 snap 313 seq_num 1007) dirty exists 2024-09-12T23:49:43.833 INFO:tasks.rados.rados.0.smithi136.stdout:3316: expect (ObjNum 993 snap 307 seq_num 993) 2024-09-12T23:49:43.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:43 smithi181 bash[67587]: cluster 2024-09-12T23:49:42.514861+0000 mon.a (mon.0) 1182 : cluster [DBG] osdmap e719: 8 total, 8 up, 8 in 2024-09-12T23:49:43.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:43 smithi181 bash[67587]: cluster 2024-09-12T23:49:42.832511+0000 mgr.y (mgr.24661) 365 : cluster [DBG] pgmap v658: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 295 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.0 MiB/s wr, 5 op/s 2024-09-12T23:49:44.641 INFO:teuthology.orchestra.run.smithi136.stdout:true 2024-09-12T23:49:44.728 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:44 smithi136 bash[76371]: cluster 2024-09-12T23:49:43.526946+0000 mon.a (mon.0) 1183 : cluster [DBG] osdmap e720: 8 total, 8 up, 8 in 2024-09-12T23:49:44.728 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:44 smithi136 bash[75164]: cluster 2024-09-12T23:49:43.526946+0000 mon.a (mon.0) 1183 : cluster [DBG] osdmap e720: 8 total, 8 up, 8 in 2024-09-12T23:49:44.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:44 smithi181 bash[67587]: cluster 2024-09-12T23:49:43.526946+0000 mon.a (mon.0) 1183 : cluster [DBG] osdmap e720: 8 total, 8 up, 8 in 2024-09-12T23:49:44.932 INFO:teuthology.orchestra.run.smithi136.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-12T23:49:44.932 INFO:teuthology.orchestra.run.smithi136.stdout:alertmanager.a smithi136 *:9093,9094 running (9m) 42s ago 15m 14.6M - 0.25.0 c8568f914cd2 b5197f020eeb 2024-09-12T23:49:44.933 INFO:teuthology.orchestra.run.smithi136.stdout:grafana.a smithi181 *:3000 running (8m) 17s ago 14m 75.6M - 9.4.7 954c08fa6188 256c181be984 2024-09-12T23:49:44.933 INFO:teuthology.orchestra.run.smithi136.stdout:mds.a smithi136 running (106s) 42s ago 16m 13.3M - 18.2.4-837-g7a853b17 f21ba1eea74e a20ac830342c 2024-09-12T23:49:44.933 INFO:teuthology.orchestra.run.smithi136.stdout:mds.b smithi181 running (73s) 17s ago 16m 11.8M - 18.2.4-837-g7a853b17 f21ba1eea74e 0dc9f76cf0b8 2024-09-12T23:49:44.933 INFO:teuthology.orchestra.run.smithi136.stdout:mgr.x smithi181 *:8443,9283,8765 running (8m) 17s ago 23m 428M - 18.2.4-837-g7a853b17 f21ba1eea74e cc664dc337f7 2024-09-12T23:49:44.933 INFO:teuthology.orchestra.run.smithi136.stdout:mgr.y smithi136 *:8443,9283,8765 running (8m) 42s ago 26m 496M - 18.2.4-837-g7a853b17 f21ba1eea74e 272f10b42fc3 2024-09-12T23:49:44.933 INFO:teuthology.orchestra.run.smithi136.stdout:mon.a smithi136 running (7m) 42s ago 26m 70.3M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e ad8d21d58ad9 2024-09-12T23:49:44.933 INFO:teuthology.orchestra.run.smithi136.stdout:mon.b smithi181 running (6m) 17s ago 24m 51.2M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e a70ec8315934 2024-09-12T23:49:44.933 INFO:teuthology.orchestra.run.smithi136.stdout:mon.c smithi136 running (7m) 42s ago 24m 58.9M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e 44d1be055a5e 2024-09-12T23:49:44.933 INFO:teuthology.orchestra.run.smithi136.stdout:node-exporter.a smithi136 *:9100 running (52s) 42s ago 15m 7063k - 1.5.0 0da6a335fe13 348c0ab0964c 2024-09-12T23:49:44.933 INFO:teuthology.orchestra.run.smithi136.stdout:node-exporter.b smithi181 *:9100 running (48s) 17s ago 15m 7784k - 1.5.0 0da6a335fe13 3af01e08b5af 2024-09-12T23:49:44.934 INFO:teuthology.orchestra.run.smithi136.stdout:osd.0 smithi136 running (5m) 42s ago 23m 362M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 7da21f765721 2024-09-12T23:49:44.934 INFO:teuthology.orchestra.run.smithi136.stdout:osd.1 smithi136 running (5m) 42s ago 22m 375M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 5302be0a016b 2024-09-12T23:49:44.934 INFO:teuthology.orchestra.run.smithi136.stdout:osd.2 smithi136 running (4m) 42s ago 21m 359M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 2e12d1a31223 2024-09-12T23:49:44.934 INFO:teuthology.orchestra.run.smithi136.stdout:osd.3 smithi136 running (4m) 42s ago 20m 279M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e aa9dc124e59c 2024-09-12T23:49:44.934 INFO:teuthology.orchestra.run.smithi136.stdout:osd.4 smithi181 running (3m) 17s ago 19m 463M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 4ee3b228ce7d 2024-09-12T23:49:44.934 INFO:teuthology.orchestra.run.smithi136.stdout:osd.5 smithi181 running (3m) 17s ago 18m 228M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 3f9a09565258 2024-09-12T23:49:44.934 INFO:teuthology.orchestra.run.smithi136.stdout:osd.6 smithi181 running (2m) 17s ago 18m 136M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e ce539f6fb854 2024-09-12T23:49:44.934 INFO:teuthology.orchestra.run.smithi136.stdout:osd.7 smithi181 running (2m) 17s ago 17m 236M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 6ad9ab169a7b 2024-09-12T23:49:44.934 INFO:teuthology.orchestra.run.smithi136.stdout:prometheus.a smithi181 *:9095 running (23s) 17s ago 15m 40.6M - 2.43.0 a07b618ecd1d 302d5b6a8434 2024-09-12T23:49:45.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:44 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:49:45.006 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 12 23:49:44 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:49:45.007 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 12 23:49:44 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:49:45.007 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 12 23:49:44 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:49:45.007 INFO:journalctl@ceph.mds.a.smithi136.stdout:Sep 12 23:49:44 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:49:45.008 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:49:44 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:49:45.008 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:49:44 smithi136 systemd[1]: Stopping Ceph alertmanager.a for da58a47c-715d-11ef-bceb-c7b262605968... 2024-09-12T23:49:45.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:44 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:49:45.008 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:49:44 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:49:45.009 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 12 23:49:44 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:49:45.009 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:49:44 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:49:45.293 INFO:teuthology.orchestra.run.smithi136.stdout:{ 2024-09-12T23:49:45.293 INFO:teuthology.orchestra.run.smithi136.stdout: "mon": { 2024-09-12T23:49:45.293 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 3 2024-09-12T23:49:45.293 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:49:45.293 INFO:teuthology.orchestra.run.smithi136.stdout: "mgr": { 2024-09-12T23:49:45.293 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 2 2024-09-12T23:49:45.293 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:49:45.293 INFO:teuthology.orchestra.run.smithi136.stdout: "osd": { 2024-09-12T23:49:45.293 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 8 2024-09-12T23:49:45.294 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:49:45.294 INFO:teuthology.orchestra.run.smithi136.stdout: "mds": { 2024-09-12T23:49:45.294 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 2 2024-09-12T23:49:45.294 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:49:45.294 INFO:teuthology.orchestra.run.smithi136.stdout: "overall": { 2024-09-12T23:49:45.294 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 15 2024-09-12T23:49:45.294 INFO:teuthology.orchestra.run.smithi136.stdout: } 2024-09-12T23:49:45.294 INFO:teuthology.orchestra.run.smithi136.stdout:} 2024-09-12T23:49:45.506 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:49:45 smithi136 bash[68390]: ts=2024-09-12T23:49:45.059Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-12T23:49:45.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:45 smithi181 bash[67587]: audit 2024-09-12T23:49:44.633266+0000 mgr.y (mgr.24661) 366 : audit [DBG] from='client.44378 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:49:45.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:45 smithi181 bash[67587]: cluster 2024-09-12T23:49:44.832977+0000 mgr.y (mgr.24661) 367 : cluster [DBG] pgmap v660: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 297 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s rd, 905 KiB/s wr, 18 op/s 2024-09-12T23:49:45.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:45 smithi181 bash[67587]: audit 2024-09-12T23:49:45.296473+0000 mon.c (mon.2) 14 : audit [DBG] from='client.? 172.21.15.136:0/245939426' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:45.879 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:49:45 smithi136 bash[104461]: ceph-da58a47c-715d-11ef-bceb-c7b262605968-alertmanager-a 2024-09-12T23:49:45.879 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:45 smithi136 bash[75164]: audit 2024-09-12T23:49:44.633266+0000 mgr.y (mgr.24661) 366 : audit [DBG] from='client.44378 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:49:45.879 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:45 smithi136 bash[75164]: cluster 2024-09-12T23:49:44.832977+0000 mgr.y (mgr.24661) 367 : cluster [DBG] pgmap v660: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 297 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s rd, 905 KiB/s wr, 18 op/s 2024-09-12T23:49:45.879 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:45 smithi136 bash[75164]: audit 2024-09-12T23:49:45.296473+0000 mon.c (mon.2) 14 : audit [DBG] from='client.? 172.21.15.136:0/245939426' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:45.880 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:45 smithi136 bash[76371]: audit 2024-09-12T23:49:44.633266+0000 mgr.y (mgr.24661) 366 : audit [DBG] from='client.44378 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:49:45.880 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:45 smithi136 bash[76371]: cluster 2024-09-12T23:49:44.832977+0000 mgr.y (mgr.24661) 367 : cluster [DBG] pgmap v660: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 297 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s rd, 905 KiB/s wr, 18 op/s 2024-09-12T23:49:45.880 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:45 smithi136 bash[76371]: audit 2024-09-12T23:49:45.296473+0000 mon.c (mon.2) 14 : audit [DBG] from='client.? 172.21.15.136:0/245939426' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:49:45.964 INFO:tasks.rados.rados.0.smithi136.stdout:3323: finishing copy_from to smithi13667081-5 2024-09-12T23:49:45.964 INFO:tasks.rados.rados.0.smithi136.stdout:3323: got expected ENOENT (src dne) 2024-09-12T23:49:45.964 INFO:tasks.rados.rados.0.smithi136.stdout:3323: finishing copy_from racing read to smithi13667081-5 2024-09-12T23:49:45.964 INFO:tasks.rados.rados.0.smithi136.stdout:3316: done (4 left) 2024-09-12T23:49:45.965 INFO:tasks.rados.rados.0.smithi136.stdout:3320: done (3 left) 2024-09-12T23:49:45.965 INFO:tasks.rados.rados.0.smithi136.stdout:3321: done (2 left) 2024-09-12T23:49:45.965 INFO:tasks.rados.rados.0.smithi136.stdout:3322: done (1 left) 2024-09-12T23:49:45.966 INFO:tasks.rados.rados.0.smithi136.stdout:3323: done (0 left) 2024-09-12T23:49:45.966 INFO:tasks.rados.rados.0.smithi136.stdout:3324: setattr oid 40 current snap is 321 2024-09-12T23:49:45.966 INFO:tasks.rados.rados.0.smithi136.stdout:3325: snap_remove snap 318 2024-09-12T23:49:46.217 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:45 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:49:46.217 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:45 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:49:46.218 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 12 23:49:45 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:49:46.218 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 12 23:49:45 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:49:46.218 INFO:journalctl@ceph.mds.a.smithi136.stdout:Sep 12 23:49:45 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:49:46.219 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 12 23:49:45 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:49:46.219 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:49:45 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:49:46.219 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:49:45 smithi136 bash[104544]: Error response from daemon: No such container: ceph-da58a47c-715d-11ef-bceb-c7b262605968-alertmanager-a 2024-09-12T23:49:46.219 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:49:45 smithi136 systemd[1]: ceph-da58a47c-715d-11ef-bceb-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-09-12T23:49:46.219 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:49:45 smithi136 systemd[1]: Stopped Ceph alertmanager.a for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:49:46.219 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:49:45 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:49:46.219 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:49:46 smithi136 systemd[1]: Started Ceph alertmanager.a for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:49:46.220 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 12 23:49:45 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:49:46.220 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 12 23:49:45 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:49:46.469 INFO:tasks.rados.rados.0.smithi136.stdout:3325: done (1 left) 2024-09-12T23:49:46.469 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 23 current snap is 321 2024-09-12T23:49:46.469 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 40 v 2581 (ObjNum 1048 snap 321 seq_num 1048) dirty exists 2024-09-12T23:49:46.472 INFO:tasks.rados.rados.0.smithi136.stdout:3326: seq_num 1049 ranges {909312=778240} 2024-09-12T23:49:46.499 INFO:tasks.rados.rados.0.smithi136.stdout:3326: writing smithi13667081-23 from 909312 to 1687552 tid 1 2024-09-12T23:49:46.499 INFO:tasks.rados.rados.0.smithi136.stdout:3324: done (1 left) 2024-09-12T23:49:46.499 INFO:tasks.rados.rados.0.smithi136.stdout:3327: rollback oid 29 current snap is 321 2024-09-12T23:49:46.499 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 29 to 319 2024-09-12T23:49:46.499 INFO:tasks.rados.rados.0.smithi136.stdout:3328: snap_remove snap 320 2024-09-12T23:49:46.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:46 smithi181 bash[67587]: audit 2024-09-12T23:49:44.924313+0000 mgr.y (mgr.24661) 368 : audit [DBG] from='client.44384 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:49:46.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:46 smithi181 bash[67587]: audit 2024-09-12T23:49:46.139149+0000 mon.a (mon.0) 1184 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:46.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:46 smithi181 bash[67587]: audit 2024-09-12T23:49:46.145744+0000 mon.a (mon.0) 1185 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:46.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:46 smithi181 bash[67587]: audit 2024-09-12T23:49:46.146742+0000 mon.a (mon.0) 1186 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:49:46.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:46 smithi181 bash[67587]: cluster 2024-09-12T23:49:46.468783+0000 mon.a (mon.0) 1187 : cluster [DBG] osdmap e721: 8 total, 8 up, 8 in 2024-09-12T23:49:47.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:46 smithi136 bash[76371]: audit 2024-09-12T23:49:44.924313+0000 mgr.y (mgr.24661) 368 : audit [DBG] from='client.44384 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:49:47.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:46 smithi136 bash[76371]: audit 2024-09-12T23:49:46.139149+0000 mon.a (mon.0) 1184 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:47.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:46 smithi136 bash[76371]: audit 2024-09-12T23:49:46.145744+0000 mon.a (mon.0) 1185 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:47.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:46 smithi136 bash[76371]: audit 2024-09-12T23:49:46.146742+0000 mon.a (mon.0) 1186 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:49:47.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:46 smithi136 bash[76371]: cluster 2024-09-12T23:49:46.468783+0000 mon.a (mon.0) 1187 : cluster [DBG] osdmap e721: 8 total, 8 up, 8 in 2024-09-12T23:49:47.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:46 smithi136 bash[75164]: audit 2024-09-12T23:49:44.924313+0000 mgr.y (mgr.24661) 368 : audit [DBG] from='client.44384 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:49:47.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:46 smithi136 bash[75164]: audit 2024-09-12T23:49:46.139149+0000 mon.a (mon.0) 1184 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:47.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:46 smithi136 bash[75164]: audit 2024-09-12T23:49:46.145744+0000 mon.a (mon.0) 1185 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:47.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:46 smithi136 bash[75164]: audit 2024-09-12T23:49:46.146742+0000 mon.a (mon.0) 1186 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:49:47.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:46 smithi136 bash[75164]: cluster 2024-09-12T23:49:46.468783+0000 mon.a (mon.0) 1187 : cluster [DBG] osdmap e721: 8 total, 8 up, 8 in 2024-09-12T23:49:47.469 INFO:tasks.rados.rados.0.smithi136.stdout:3328: done (2 left) 2024-09-12T23:49:47.470 INFO:tasks.rados.rados.0.smithi136.stdout:3329: delete oid 25 current snap is 321 2024-09-12T23:49:47.470 INFO:tasks.rados.rados.0.smithi136.stdout:3327: finishing rollback tid 0 to smithi13667081-29 2024-09-12T23:49:47.470 INFO:tasks.rados.rados.0.smithi136.stdout:3326: finishing write tid 1 to smithi13667081-23 2024-09-12T23:49:47.470 INFO:tasks.rados.rados.0.smithi136.stdout:3327: finishing rollback tid 1 to smithi13667081-29 2024-09-12T23:49:47.470 INFO:tasks.rados.rados.0.smithi136.stdout:3326: finishing write tid 2 to smithi13667081-23 2024-09-12T23:49:47.470 INFO:tasks.rados.rados.0.smithi136.stdout:3327: finishing rollback tid 2 to smithi13667081-29 2024-09-12T23:49:47.470 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 29 v 2444 (ObjNum 1027 snap 317 seq_num 1027) dirty exists 2024-09-12T23:49:47.471 INFO:tasks.rados.rados.0.smithi136.stdout:3326: finishing write tid 3 to smithi13667081-23 2024-09-12T23:49:47.471 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 23 v 2592 (ObjNum 1049 snap 321 seq_num 1049) dirty exists 2024-09-12T23:49:47.471 INFO:tasks.rados.rados.0.smithi136.stdout:3326: left oid 23 (ObjNum 1049 snap 321 seq_num 1049) 2024-09-12T23:49:47.487 INFO:tasks.rados.rados.0.smithi136.stdout:3326: done (2 left) 2024-09-12T23:49:47.487 INFO:tasks.rados.rados.0.smithi136.stdout:3327: done (1 left) 2024-09-12T23:49:47.487 INFO:tasks.rados.rados.0.smithi136.stdout:3329: done (0 left) 2024-09-12T23:49:47.487 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 41 current snap is 321 2024-09-12T23:49:47.487 INFO:tasks.rados.rados.0.smithi136.stdout:3330: seq_num 1051 ranges {4497408=409600,4907008=401408} 2024-09-12T23:49:47.515 INFO:tasks.rados.rados.0.smithi136.stdout:3330: writing smithi13667081-41 from 4497408 to 4907008 tid 1 2024-09-12T23:49:47.517 INFO:tasks.rados.rados.0.smithi136.stdout:3330: writing smithi13667081-41 from 4907008 to 5308416 tid 2 2024-09-12T23:49:47.518 INFO:tasks.rados.rados.0.smithi136.stdout:3331: delete oid 48 current snap is 321 2024-09-12T23:49:47.521 INFO:tasks.rados.rados.0.smithi136.stdout:3330: finishing write tid 1 to smithi13667081-41 2024-09-12T23:49:47.521 INFO:tasks.rados.rados.0.smithi136.stdout:3331: done (1 left) 2024-09-12T23:49:47.521 INFO:tasks.rados.rados.0.smithi136.stdout:3332: copy_from oid 34 from oid 3 current snap is 321 2024-09-12T23:49:47.522 INFO:tasks.rados.rados.0.smithi136.stdout:3333: copy_from oid 25 from oid 31 current snap is 321 2024-09-12T23:49:47.522 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 4 current snap is 321 2024-09-12T23:49:47.522 INFO:tasks.rados.rados.0.smithi136.stdout:3334: seq_num 1053 ranges {15523840=442368,15966208=401408} 2024-09-12T23:49:47.611 INFO:tasks.rados.rados.0.smithi136.stdout:3334: writing smithi13667081-4 from 15523840 to 15966208 tid 1 2024-09-12T23:49:47.614 INFO:tasks.rados.rados.0.smithi136.stdout:3334: writing smithi13667081-4 from 15966208 to 16367616 tid 2 2024-09-12T23:49:47.615 INFO:tasks.rados.rados.0.smithi136.stdout:3335: rollback oid 12 current snap is 321 2024-09-12T23:49:47.615 INFO:tasks.rados.rados.0.smithi136.stdout:3330: finishing write tid 2 to smithi13667081-41 2024-09-12T23:49:47.615 INFO:tasks.rados.rados.0.smithi136.stdout:3330: finishing write tid 3 to smithi13667081-41 2024-09-12T23:49:47.615 INFO:tasks.rados.rados.0.smithi136.stdout:3330: finishing write tid 4 to smithi13667081-41 2024-09-12T23:49:47.615 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 41 v 2470 (ObjNum 1051 snap 321 seq_num 1051) dirty exists 2024-09-12T23:49:47.616 INFO:tasks.rados.rados.0.smithi136.stdout:3330: left oid 41 (ObjNum 1051 snap 321 seq_num 1051) 2024-09-12T23:49:47.616 INFO:tasks.rados.rados.0.smithi136.stdout:3333: finishing copy_from to smithi13667081-25 2024-09-12T23:49:47.616 INFO:tasks.rados.rados.0.smithi136.stdout:3333: got expected ENOENT (src dne) 2024-09-12T23:49:47.616 INFO:tasks.rados.rados.0.smithi136.stdout:3333: finishing copy_from racing read to smithi13667081-25 2024-09-12T23:49:47.616 INFO:tasks.rados.rados.0.smithi136.stdout:3332: finishing copy_from racing read to smithi13667081-34 2024-09-12T23:49:47.617 INFO:tasks.rados.rados.0.smithi136.stdout:3332: finishing copy_from to smithi13667081-34 2024-09-12T23:49:47.617 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 34 v 1836 (ObjNum 838 snap 267 seq_num 838) dirty exists 2024-09-12T23:49:47.617 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 12 to 310 2024-09-12T23:49:47.617 INFO:tasks.rados.rados.0.smithi136.stdout:3330: done (4 left) 2024-09-12T23:49:47.617 INFO:tasks.rados.rados.0.smithi136.stdout:3332: done (3 left) 2024-09-12T23:49:47.618 INFO:tasks.rados.rados.0.smithi136.stdout:3333: done (2 left) 2024-09-12T23:49:47.618 INFO:tasks.rados.rados.0.smithi136.stdout:3336: delete oid 43 current snap is 321 2024-09-12T23:49:47.620 INFO:tasks.rados.rados.0.smithi136.stdout:3336: done (2 left) 2024-09-12T23:49:47.620 INFO:tasks.rados.rados.0.smithi136.stdout:3337: copy_from oid 27 from oid 5 current snap is 321 2024-09-12T23:49:47.620 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 36 current snap is 321 2024-09-12T23:49:47.620 INFO:tasks.rados.rados.0.smithi136.stdout:3338: seq_num 1055 ranges {0=475136,475136=401408} 2024-09-12T23:49:47.623 INFO:tasks.rados.rados.0.smithi136.stdout:3338: writing smithi13667081-36 from 0 to 475136 tid 1 2024-09-12T23:49:47.626 INFO:tasks.rados.rados.0.smithi136.stdout:3338: writing smithi13667081-36 from 475136 to 876544 tid 2 2024-09-12T23:49:47.627 INFO:tasks.rados.rados.0.smithi136.stdout:3339: rollback oid 47 current snap is 321 2024-09-12T23:49:47.627 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 47 to 310 2024-09-12T23:49:47.627 INFO:tasks.rados.rados.0.smithi136.stdout:3337: finishing copy_from to smithi13667081-27 2024-09-12T23:49:47.627 INFO:tasks.rados.rados.0.smithi136.stdout:3337: got expected ENOENT (src dne) 2024-09-12T23:49:47.627 INFO:tasks.rados.rados.0.smithi136.stdout:3335: finishing rollback tid 0 to smithi13667081-12 2024-09-12T23:49:47.627 INFO:tasks.rados.rados.0.smithi136.stdout:3337: finishing copy_from racing read to smithi13667081-27 2024-09-12T23:49:47.628 INFO:tasks.rados.rados.0.smithi136.stdout:3335: finishing rollback tid 1 to smithi13667081-12 2024-09-12T23:49:47.628 INFO:tasks.rados.rados.0.smithi136.stdout:3337: done (4 left) 2024-09-12T23:49:47.628 INFO:tasks.rados.rados.0.smithi136.stdout:3340: rollback oid 23 current snap is 321 2024-09-12T23:49:47.628 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 23 to 310 2024-09-12T23:49:47.628 INFO:tasks.rados.rados.0.smithi136.stdout:3334: finishing write tid 1 to smithi13667081-4 2024-09-12T23:49:47.629 INFO:tasks.rados.rados.0.smithi136.stdout:3334: finishing write tid 2 to smithi13667081-4 2024-09-12T23:49:47.629 INFO:tasks.rados.rados.0.smithi136.stdout:3334: finishing write tid 3 to smithi13667081-4 2024-09-12T23:49:47.629 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 28 current snap is 321 2024-09-12T23:49:47.629 INFO:tasks.rados.rados.0.smithi136.stdout:3341: seq_num 1056 ranges {1802240=491520,2293760=401408} 2024-09-12T23:49:47.639 INFO:tasks.rados.rados.0.smithi136.stdout:3341: writing smithi13667081-28 from 1802240 to 2293760 tid 1 2024-09-12T23:49:47.642 INFO:tasks.rados.rados.0.smithi136.stdout:3341: writing smithi13667081-28 from 2293760 to 2695168 tid 2 2024-09-12T23:49:47.642 INFO:tasks.rados.rados.0.smithi136.stdout:3342: delete oid 49 current snap is 321 2024-09-12T23:49:47.642 INFO:tasks.rados.rados.0.smithi136.stdout:3335: finishing rollback tid 2 to smithi13667081-12 2024-09-12T23:49:47.642 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 12 v 1842 (ObjNum 988 snap 303 seq_num 988) dirty exists 2024-09-12T23:49:47.643 INFO:tasks.rados.rados.0.smithi136.stdout:3339: finishing rollback tid 1 to smithi13667081-47 2024-09-12T23:49:47.643 INFO:tasks.rados.rados.0.smithi136.stdout:3334: finishing write tid 4 to smithi13667081-4 2024-09-12T23:49:47.643 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 4 v 1643 (ObjNum 1053 snap 321 seq_num 1053) dirty exists 2024-09-12T23:49:47.643 INFO:tasks.rados.rados.0.smithi136.stdout:3334: left oid 4 (ObjNum 1053 snap 321 seq_num 1053) 2024-09-12T23:49:47.643 INFO:tasks.rados.rados.0.smithi136.stdout:3339: finishing rollback tid 2 to smithi13667081-47 2024-09-12T23:49:47.644 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 47 v 1649 (ObjNum 990 snap 304 seq_num 990) dirty exists 2024-09-12T23:49:47.644 INFO:tasks.rados.rados.0.smithi136.stdout:3338: finishing write tid 1 to smithi13667081-36 2024-09-12T23:49:47.644 INFO:tasks.rados.rados.0.smithi136.stdout:3340: finishing rollback tid 0 to smithi13667081-23 2024-09-12T23:49:47.644 INFO:tasks.rados.rados.0.smithi136.stdout:3340: finishing rollback tid 1 to smithi13667081-23 2024-09-12T23:49:47.644 INFO:tasks.rados.rados.0.smithi136.stdout:3338: finishing write tid 2 to smithi13667081-36 2024-09-12T23:49:47.645 INFO:tasks.rados.rados.0.smithi136.stdout:3338: finishing write tid 3 to smithi13667081-36 2024-09-12T23:49:47.645 INFO:tasks.rados.rados.0.smithi136.stdout:3340: finishing rollback tid 2 to smithi13667081-23 2024-09-12T23:49:47.645 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 23 v 2605 (ObjNum 886 snap 287 seq_num 886) dirty exists 2024-09-12T23:49:47.645 INFO:tasks.rados.rados.0.smithi136.stdout:3338: finishing write tid 4 to smithi13667081-36 2024-09-12T23:49:47.645 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 36 v 2345 (ObjNum 1055 snap 321 seq_num 1055) dirty exists 2024-09-12T23:49:47.645 INFO:tasks.rados.rados.0.smithi136.stdout:3338: left oid 36 (ObjNum 1055 snap 321 seq_num 1055) 2024-09-12T23:49:47.647 INFO:tasks.rados.rados.0.smithi136.stdout:3334: done (6 left) 2024-09-12T23:49:47.647 INFO:tasks.rados.rados.0.smithi136.stdout:3335: done (5 left) 2024-09-12T23:49:47.647 INFO:tasks.rados.rados.0.smithi136.stdout:3338: done (4 left) 2024-09-12T23:49:47.647 INFO:tasks.rados.rados.0.smithi136.stdout:3339: done (3 left) 2024-09-12T23:49:47.647 INFO:tasks.rados.rados.0.smithi136.stdout:3340: done (2 left) 2024-09-12T23:49:47.648 INFO:tasks.rados.rados.0.smithi136.stdout:3342: done (1 left) 2024-09-12T23:49:47.648 INFO:tasks.rados.rados.0.smithi136.stdout:3343: read oid 16 snap -1 2024-09-12T23:49:47.648 INFO:tasks.rados.rados.0.smithi136.stdout:3343: expect deleted 2024-09-12T23:49:47.648 INFO:tasks.rados.rados.0.smithi136.stdout:3344: delete oid 31 current snap is 321 2024-09-12T23:49:47.648 INFO:tasks.rados.rados.0.smithi136.stdout:3341: finishing write tid 1 to smithi13667081-28 2024-09-12T23:49:47.649 INFO:tasks.rados.rados.0.smithi136.stdout:3341: finishing write tid 2 to smithi13667081-28 2024-09-12T23:49:47.650 INFO:tasks.rados.rados.0.smithi136.stdout:3341: finishing write tid 3 to smithi13667081-28 2024-09-12T23:49:47.650 INFO:tasks.rados.rados.0.smithi136.stdout:3343: done (2 left) 2024-09-12T23:49:47.650 INFO:tasks.rados.rados.0.smithi136.stdout:3344: done (1 left) 2024-09-12T23:49:47.651 INFO:tasks.rados.rados.0.smithi136.stdout:3345: copy_from oid 47 from oid 37 current snap is 321 2024-09-12T23:49:47.651 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 1 current snap is 321 2024-09-12T23:49:47.651 INFO:tasks.rados.rados.0.smithi136.stdout:3346: seq_num 1059 ranges {7110656=540672,7651328=401408} 2024-09-12T23:49:47.690 INFO:tasks.rados.rados.0.smithi136.stdout:3346: writing smithi13667081-1 from 7110656 to 7651328 tid 1 2024-09-12T23:49:47.693 INFO:tasks.rados.rados.0.smithi136.stdout:3346: writing smithi13667081-1 from 7651328 to 8052736 tid 2 2024-09-12T23:49:47.693 INFO:tasks.rados.rados.0.smithi136.stdout:3347: read oid 50 snap -1 2024-09-12T23:49:47.693 INFO:tasks.rados.rados.0.smithi136.stdout:3347: expect deleted 2024-09-12T23:49:47.694 INFO:tasks.rados.rados.0.smithi136.stdout:3348: delete oid 18 current snap is 321 2024-09-12T23:49:47.694 INFO:tasks.rados.rados.0.smithi136.stdout:3341: finishing write tid 4 to smithi13667081-28 2024-09-12T23:49:47.694 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 28 v 2067 (ObjNum 1056 snap 321 seq_num 1056) dirty exists 2024-09-12T23:49:47.694 INFO:tasks.rados.rados.0.smithi136.stdout:3341: left oid 28 (ObjNum 1056 snap 321 seq_num 1056) 2024-09-12T23:49:47.694 INFO:tasks.rados.rados.0.smithi136.stdout:3345: finishing copy_from racing read to smithi13667081-47 2024-09-12T23:49:47.695 INFO:tasks.rados.rados.0.smithi136.stdout:3345: finishing copy_from to smithi13667081-47 2024-09-12T23:49:47.695 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 47 v 2464 (ObjNum 1020 snap 315 seq_num 1020) dirty exists 2024-09-12T23:49:47.697 INFO:tasks.rados.rados.0.smithi136.stdout:3346: finishing write tid 1 to smithi13667081-1 2024-09-12T23:49:47.697 INFO:tasks.rados.rados.0.smithi136.stdout:3346: finishing write tid 2 to smithi13667081-1 2024-09-12T23:49:47.697 INFO:tasks.rados.rados.0.smithi136.stdout:3346: finishing write tid 3 to smithi13667081-1 2024-09-12T23:49:47.698 INFO:tasks.rados.rados.0.smithi136.stdout:3341: done (4 left) 2024-09-12T23:49:47.698 INFO:tasks.rados.rados.0.smithi136.stdout:3345: done (3 left) 2024-09-12T23:49:47.698 INFO:tasks.rados.rados.0.smithi136.stdout:3347: done (2 left) 2024-09-12T23:49:47.698 INFO:tasks.rados.rados.0.smithi136.stdout:3348: done (1 left) 2024-09-12T23:49:47.699 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 43 current snap is 321 2024-09-12T23:49:47.699 INFO:tasks.rados.rados.0.smithi136.stdout:3349: seq_num 1061 ranges {0=573440,573440=401408} 2024-09-12T23:49:47.701 INFO:tasks.rados.rados.0.smithi136.stdout:3349: writing smithi13667081-43 from 0 to 573440 tid 1 2024-09-12T23:49:47.703 INFO:tasks.rados.rados.0.smithi136.stdout:3349: writing smithi13667081-43 from 573440 to 974848 tid 2 2024-09-12T23:49:47.704 INFO:tasks.rados.rados.0.smithi136.stdout:3350: rollback oid 37 current snap is 321 2024-09-12T23:49:47.704 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 37 to 316 2024-09-12T23:49:47.704 INFO:tasks.rados.rados.0.smithi136.stdout:3346: finishing write tid 4 to smithi13667081-1 2024-09-12T23:49:47.704 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 1 v 2475 (ObjNum 1059 snap 321 seq_num 1059) dirty exists 2024-09-12T23:49:47.704 INFO:tasks.rados.rados.0.smithi136.stdout:3346: left oid 1 (ObjNum 1059 snap 321 seq_num 1059) 2024-09-12T23:49:47.705 INFO:tasks.rados.rados.0.smithi136.stdout:3346: done (2 left) 2024-09-12T23:49:47.705 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 42 current snap is 321 2024-09-12T23:49:47.705 INFO:tasks.rados.rados.0.smithi136.stdout:3351: seq_num 1062 ranges {0=589824,589824=401408} 2024-09-12T23:49:47.707 INFO:tasks.rados.rados.0.smithi136.stdout:3351: writing smithi13667081-42 from 0 to 589824 tid 1 2024-09-12T23:49:47.710 INFO:tasks.rados.rados.0.smithi136.stdout:3351: writing smithi13667081-42 from 589824 to 991232 tid 2 2024-09-12T23:49:47.710 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 4 current snap is 321 2024-09-12T23:49:47.710 INFO:tasks.rados.rados.0.smithi136.stdout:3349: finishing write tid 1 to smithi13667081-43 2024-09-12T23:49:47.710 INFO:tasks.rados.rados.0.smithi136.stdout:3352: seq_num 1063 ranges {16367616=606208,16973824=401408} 2024-09-12T23:49:47.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:47 smithi136 bash[76371]: cluster 2024-09-12T23:49:46.833894+0000 mgr.y (mgr.24661) 369 : cluster [DBG] pgmap v662: 121 pgs: 121 active+clean; 291 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 873 KiB/s wr, 19 op/s 2024-09-12T23:49:47.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:47 smithi136 bash[76371]: cluster 2024-09-12T23:49:47.472544+0000 mon.a (mon.0) 1188 : cluster [DBG] osdmap e722: 8 total, 8 up, 8 in 2024-09-12T23:49:47.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:47 smithi136 bash[75164]: cluster 2024-09-12T23:49:46.833894+0000 mgr.y (mgr.24661) 369 : cluster [DBG] pgmap v662: 121 pgs: 121 active+clean; 291 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 873 KiB/s wr, 19 op/s 2024-09-12T23:49:47.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:47 smithi136 bash[75164]: cluster 2024-09-12T23:49:47.472544+0000 mon.a (mon.0) 1188 : cluster [DBG] osdmap e722: 8 total, 8 up, 8 in 2024-09-12T23:49:47.757 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:49:47 smithi136 bash[104600]: ts=2024-09-12T23:49:47.540Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-09-12T23:49:47.757 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:49:47 smithi136 bash[104600]: ts=2024-09-12T23:49:47.540Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-09-12T23:49:47.757 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:49:47 smithi136 bash[104600]: ts=2024-09-12T23:49:47.542Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.136 port=9094 2024-09-12T23:49:47.757 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:49:47 smithi136 bash[104600]: ts=2024-09-12T23:49:47.544Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-12T23:49:47.757 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:49:47 smithi136 bash[104600]: ts=2024-09-12T23:49:47.574Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-12T23:49:47.757 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:49:47 smithi136 bash[104600]: ts=2024-09-12T23:49:47.574Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-12T23:49:47.757 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:49:47 smithi136 bash[104600]: ts=2024-09-12T23:49:47.578Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-09-12T23:49:47.757 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:49:47 smithi136 bash[104600]: ts=2024-09-12T23:49:47.578Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-09-12T23:49:47.805 INFO:tasks.rados.rados.0.smithi136.stdout:3352: writing smithi13667081-4 from 16367616 to 16973824 tid 1 2024-09-12T23:49:47.807 INFO:tasks.rados.rados.0.smithi136.stdout:3352: writing smithi13667081-4 from 16973824 to 17375232 tid 2 2024-09-12T23:49:47.807 INFO:tasks.rados.rados.0.smithi136.stdout:3353: snap_create 2024-09-12T23:49:47.808 INFO:tasks.rados.rados.0.smithi136.stdout:3350: finishing rollback tid 0 to smithi13667081-37 2024-09-12T23:49:47.808 INFO:tasks.rados.rados.0.smithi136.stdout:3351: finishing write tid 1 to smithi13667081-42 2024-09-12T23:49:47.808 INFO:tasks.rados.rados.0.smithi136.stdout:3350: finishing rollback tid 1 to smithi13667081-37 2024-09-12T23:49:47.808 INFO:tasks.rados.rados.0.smithi136.stdout:3349: finishing write tid 2 to smithi13667081-43 2024-09-12T23:49:47.808 INFO:tasks.rados.rados.0.smithi136.stdout:3350: finishing rollback tid 2 to smithi13667081-37 2024-09-12T23:49:47.808 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 37 v 2473 (ObjNum 1020 snap 315 seq_num 1020) dirty exists 2024-09-12T23:49:47.808 INFO:tasks.rados.rados.0.smithi136.stdout:3349: finishing write tid 3 to smithi13667081-43 2024-09-12T23:49:47.808 INFO:tasks.rados.rados.0.smithi136.stdout:3349: finishing write tid 4 to smithi13667081-43 2024-09-12T23:49:47.808 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 43 v 2070 (ObjNum 1061 snap 321 seq_num 1061) dirty exists 2024-09-12T23:49:47.808 INFO:tasks.rados.rados.0.smithi136.stdout:3349: left oid 43 (ObjNum 1061 snap 321 seq_num 1061) 2024-09-12T23:49:47.808 INFO:tasks.rados.rados.0.smithi136.stdout:3351: finishing write tid 2 to smithi13667081-42 2024-09-12T23:49:47.808 INFO:tasks.rados.rados.0.smithi136.stdout:3351: finishing write tid 3 to smithi13667081-42 2024-09-12T23:49:47.809 INFO:tasks.rados.rados.0.smithi136.stdout:3351: finishing write tid 4 to smithi13667081-42 2024-09-12T23:49:47.809 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 42 v 2478 (ObjNum 1062 snap 321 seq_num 1062) dirty exists 2024-09-12T23:49:47.809 INFO:tasks.rados.rados.0.smithi136.stdout:3351: left oid 42 (ObjNum 1062 snap 321 seq_num 1062) 2024-09-12T23:49:47.814 INFO:tasks.rados.rados.0.smithi136.stdout:3352: finishing write tid 1 to smithi13667081-4 2024-09-12T23:49:47.814 INFO:tasks.rados.rados.0.smithi136.stdout:3352: finishing write tid 2 to smithi13667081-4 2024-09-12T23:49:47.815 INFO:tasks.rados.rados.0.smithi136.stdout:3352: finishing write tid 3 to smithi13667081-4 2024-09-12T23:49:47.817 INFO:tasks.rados.rados.0.smithi136.stdout:3352: finishing write tid 4 to smithi13667081-4 2024-09-12T23:49:47.817 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 4 v 1646 (ObjNum 1063 snap 321 seq_num 1063) dirty exists 2024-09-12T23:49:47.817 INFO:tasks.rados.rados.0.smithi136.stdout:3352: left oid 4 (ObjNum 1063 snap 321 seq_num 1063) 2024-09-12T23:49:47.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:47 smithi181 bash[67587]: cluster 2024-09-12T23:49:46.833894+0000 mgr.y (mgr.24661) 369 : cluster [DBG] pgmap v662: 121 pgs: 121 active+clean; 291 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 873 KiB/s wr, 19 op/s 2024-09-12T23:49:47.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:47 smithi181 bash[67587]: cluster 2024-09-12T23:49:47.472544+0000 mon.a (mon.0) 1188 : cluster [DBG] osdmap e722: 8 total, 8 up, 8 in 2024-09-12T23:49:48.546 INFO:tasks.rados.rados.0.smithi136.stdout:3349: done (4 left) 2024-09-12T23:49:48.547 INFO:tasks.rados.rados.0.smithi136.stdout:3350: done (3 left) 2024-09-12T23:49:48.547 INFO:tasks.rados.rados.0.smithi136.stdout:3351: done (2 left) 2024-09-12T23:49:48.547 INFO:tasks.rados.rados.0.smithi136.stdout:3352: done (1 left) 2024-09-12T23:49:48.547 INFO:tasks.rados.rados.0.smithi136.stdout:3353: done (0 left) 2024-09-12T23:49:48.547 INFO:tasks.rados.rados.0.smithi136.stdout:3354: delete oid 47 current snap is 322 2024-09-12T23:49:48.551 INFO:tasks.rados.rados.0.smithi136.stdout:3354: done (0 left) 2024-09-12T23:49:48.551 INFO:tasks.rados.rados.0.smithi136.stdout:3355: snap_create 2024-09-12T23:49:49.560 INFO:tasks.rados.rados.0.smithi136.stdout:3355: done (0 left) 2024-09-12T23:49:49.560 INFO:tasks.rados.rados.0.smithi136.stdout:3356: read oid 39 snap -1 2024-09-12T23:49:49.560 INFO:tasks.rados.rados.0.smithi136.stdout:3356: expect (ObjNum 1045 snap 321 seq_num 1045) 2024-09-12T23:49:49.560 INFO:tasks.rados.rados.0.smithi136.stdout:3357: snap_create 2024-09-12T23:49:49.574 INFO:tasks.rados.rados.0.smithi136.stdout:3356: expect (ObjNum 1045 snap 321 seq_num 1045) 2024-09-12T23:49:49.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:49:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:49:49] "GET /metrics HTTP/1.1" 200 36439 "" "Prometheus/2.43.0" 2024-09-12T23:49:49.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:49 smithi136 bash[75164]: cluster 2024-09-12T23:49:48.545048+0000 mon.a (mon.0) 1189 : cluster [DBG] osdmap e723: 8 total, 8 up, 8 in 2024-09-12T23:49:49.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:49 smithi136 bash[75164]: cluster 2024-09-12T23:49:48.834638+0000 mgr.y (mgr.24661) 370 : cluster [DBG] pgmap v665: 121 pgs: 121 active+clean; 313 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 4.5 MiB/s wr, 22 op/s 2024-09-12T23:49:49.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:49 smithi136 bash[76371]: cluster 2024-09-12T23:49:48.545048+0000 mon.a (mon.0) 1189 : cluster [DBG] osdmap e723: 8 total, 8 up, 8 in 2024-09-12T23:49:49.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:49 smithi136 bash[76371]: cluster 2024-09-12T23:49:48.834638+0000 mgr.y (mgr.24661) 370 : cluster [DBG] pgmap v665: 121 pgs: 121 active+clean; 313 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 4.5 MiB/s wr, 22 op/s 2024-09-12T23:49:49.757 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:49:49 smithi136 bash[104600]: ts=2024-09-12T23:49:49.544Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000257032s 2024-09-12T23:49:49.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:49 smithi181 bash[67587]: cluster 2024-09-12T23:49:48.545048+0000 mon.a (mon.0) 1189 : cluster [DBG] osdmap e723: 8 total, 8 up, 8 in 2024-09-12T23:49:49.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:49 smithi181 bash[67587]: cluster 2024-09-12T23:49:48.834638+0000 mgr.y (mgr.24661) 370 : cluster [DBG] pgmap v665: 121 pgs: 121 active+clean; 313 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 4.5 MiB/s wr, 22 op/s 2024-09-12T23:49:50.561 INFO:tasks.rados.rados.0.smithi136.stdout:3356: done (1 left) 2024-09-12T23:49:50.561 INFO:tasks.rados.rados.0.smithi136.stdout:3357: done (0 left) 2024-09-12T23:49:50.561 INFO:tasks.rados.rados.0.smithi136.stdout:3358: delete oid 11 current snap is 324 2024-09-12T23:49:50.569 INFO:tasks.rados.rados.0.smithi136.stdout:3358: done (0 left) 2024-09-12T23:49:50.570 INFO:tasks.rados.rados.0.smithi136.stdout:3359: delete oid 18 current snap is 324 2024-09-12T23:49:50.577 INFO:tasks.rados.rados.0.smithi136.stdout:3359: done (0 left) 2024-09-12T23:49:50.577 INFO:tasks.rados.rados.0.smithi136.stdout:3360: read oid 38 snap -1 2024-09-12T23:49:50.577 INFO:tasks.rados.rados.0.smithi136.stdout:3360: expect (ObjNum 864 snap 281 seq_num 864) 2024-09-12T23:49:50.578 INFO:tasks.rados.rados.0.smithi136.stdout:3361: read oid 31 snap -1 2024-09-12T23:49:50.578 INFO:tasks.rados.rados.0.smithi136.stdout:3361: expect deleted 2024-09-12T23:49:50.578 INFO:tasks.rados.rados.0.smithi136.stdout:3362: snap_remove snap 319 2024-09-12T23:49:50.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:50 smithi181 bash[67587]: cluster 2024-09-12T23:49:49.560351+0000 mon.a (mon.0) 1190 : cluster [DBG] osdmap e724: 8 total, 8 up, 8 in 2024-09-12T23:49:51.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:50 smithi136 bash[75164]: cluster 2024-09-12T23:49:49.560351+0000 mon.a (mon.0) 1190 : cluster [DBG] osdmap e724: 8 total, 8 up, 8 in 2024-09-12T23:49:51.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:50 smithi136 bash[76371]: cluster 2024-09-12T23:49:49.560351+0000 mon.a (mon.0) 1190 : cluster [DBG] osdmap e724: 8 total, 8 up, 8 in 2024-09-12T23:49:51.468 INFO:tasks.rados.rados.0.smithi136.stdout:3362: done (2 left) 2024-09-12T23:49:51.468 INFO:tasks.rados.rados.0.smithi136.stdout:3363: setattr oid 12 current snap is 324 2024-09-12T23:49:51.472 INFO:tasks.rados.rados.0.smithi136.stdout:3360: expect (ObjNum 864 snap 281 seq_num 864) 2024-09-12T23:49:51.647 INFO:tasks.rados.rados.0.smithi136.stdout:3360: done (2 left) 2024-09-12T23:49:51.647 INFO:tasks.rados.rados.0.smithi136.stdout:3361: done (1 left) 2024-09-12T23:49:51.647 INFO:tasks.rados.rados.0.smithi136.stdout:3364: rollback oid 38 current snap is 324 2024-09-12T23:49:51.647 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 38 to 303 2024-09-12T23:49:51.647 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 29 current snap is 324 2024-09-12T23:49:51.647 INFO:tasks.rados.rados.0.smithi136.stdout:3365: seq_num 1067 ranges {6242304=679936,6922240=401408} 2024-09-12T23:49:51.685 INFO:tasks.rados.rados.0.smithi136.stdout:3365: writing smithi13667081-29 from 6242304 to 6922240 tid 1 2024-09-12T23:49:51.689 INFO:tasks.rados.rados.0.smithi136.stdout:3365: writing smithi13667081-29 from 6922240 to 7323648 tid 2 2024-09-12T23:49:51.689 INFO:tasks.rados.rados.0.smithi136.stdout:3366: copy_from oid 40 from oid 47 current snap is 324 2024-09-12T23:49:51.689 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 12 v 1844 (ObjNum 988 snap 303 seq_num 988) dirty exists 2024-09-12T23:49:51.689 INFO:tasks.rados.rados.0.smithi136.stdout:3364: finishing rollback tid 0 to smithi13667081-38 2024-09-12T23:49:51.689 INFO:tasks.rados.rados.0.smithi136.stdout:3364: finishing rollback tid 1 to smithi13667081-38 2024-09-12T23:49:51.689 INFO:tasks.rados.rados.0.smithi136.stdout:3363: done (3 left) 2024-09-12T23:49:51.689 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 10 current snap is 324 2024-09-12T23:49:51.689 INFO:tasks.rados.rados.0.smithi136.stdout:3364: finishing rollback tid 2 to smithi13667081-38 2024-09-12T23:49:51.689 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 38 v 2447 (ObjNum 864 snap 281 seq_num 864) dirty exists 2024-09-12T23:49:51.689 INFO:tasks.rados.rados.0.smithi136.stdout:3367: seq_num 1068 ranges {0=696320,696320=401408} 2024-09-12T23:49:51.693 INFO:tasks.rados.rados.0.smithi136.stdout:3367: writing smithi13667081-10 from 0 to 696320 tid 1 2024-09-12T23:49:51.697 INFO:tasks.rados.rados.0.smithi136.stdout:3367: writing smithi13667081-10 from 696320 to 1097728 tid 2 2024-09-12T23:49:51.697 INFO:tasks.rados.rados.0.smithi136.stdout:3364: done (3 left) 2024-09-12T23:49:51.697 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 43 current snap is 324 2024-09-12T23:49:51.697 INFO:tasks.rados.rados.0.smithi136.stdout:3368: seq_num 1069 ranges {974848=712704,1687552=401408} 2024-09-12T23:49:51.708 INFO:tasks.rados.rados.0.smithi136.stdout:3368: writing smithi13667081-43 from 974848 to 1687552 tid 1 2024-09-12T23:49:51.711 INFO:tasks.rados.rados.0.smithi136.stdout:3368: writing smithi13667081-43 from 1687552 to 2088960 tid 2 2024-09-12T23:49:51.711 INFO:tasks.rados.rados.0.smithi136.stdout:3369: read oid 34 snap -1 2024-09-12T23:49:51.711 INFO:tasks.rados.rados.0.smithi136.stdout:3369: expect (ObjNum 838 snap 267 seq_num 838) 2024-09-12T23:49:51.711 INFO:tasks.rados.rados.0.smithi136.stdout:3370: read oid 17 snap 321 2024-09-12T23:49:51.711 INFO:tasks.rados.rados.0.smithi136.stdout:3370: expect (ObjNum 1007 snap 313 seq_num 1007) 2024-09-12T23:49:51.711 INFO:tasks.rados.rados.0.smithi136.stdout:3371: rollback oid 38 current snap is 324 2024-09-12T23:49:51.711 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 38 to 322 2024-09-12T23:49:51.711 INFO:tasks.rados.rados.0.smithi136.stdout:3366: finishing copy_from to smithi13667081-40 2024-09-12T23:49:51.711 INFO:tasks.rados.rados.0.smithi136.stdout:3366: got expected ENOENT (src dne) 2024-09-12T23:49:51.711 INFO:tasks.rados.rados.0.smithi136.stdout:3366: finishing copy_from racing read to smithi13667081-40 2024-09-12T23:49:51.711 INFO:tasks.rados.rados.0.smithi136.stdout:3365: finishing write tid 1 to smithi13667081-29 2024-09-12T23:49:51.712 INFO:tasks.rados.rados.0.smithi136.stdout:3365: finishing write tid 2 to smithi13667081-29 2024-09-12T23:49:51.712 INFO:tasks.rados.rados.0.smithi136.stdout:3365: finishing write tid 3 to smithi13667081-29 2024-09-12T23:49:51.712 INFO:tasks.rados.rados.0.smithi136.stdout:3367: finishing write tid 1 to smithi13667081-10 2024-09-12T23:49:51.712 INFO:tasks.rados.rados.0.smithi136.stdout:3367: finishing write tid 2 to smithi13667081-10 2024-09-12T23:49:51.712 INFO:tasks.rados.rados.0.smithi136.stdout:3367: finishing write tid 3 to smithi13667081-10 2024-09-12T23:49:51.712 INFO:tasks.rados.rados.0.smithi136.stdout:3365: finishing write tid 4 to smithi13667081-29 2024-09-12T23:49:51.713 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 29 v 2450 (ObjNum 1067 snap 324 seq_num 1067) dirty exists 2024-09-12T23:49:51.713 INFO:tasks.rados.rados.0.smithi136.stdout:3365: left oid 29 (ObjNum 1067 snap 324 seq_num 1067) 2024-09-12T23:49:51.713 INFO:tasks.rados.rados.0.smithi136.stdout:3367: finishing write tid 4 to smithi13667081-10 2024-09-12T23:49:51.713 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 10 v 1509 (ObjNum 1068 snap 324 seq_num 1068) dirty exists 2024-09-12T23:49:51.713 INFO:tasks.rados.rados.0.smithi136.stdout:3367: left oid 10 (ObjNum 1068 snap 324 seq_num 1068) 2024-09-12T23:49:51.713 INFO:tasks.rados.rados.0.smithi136.stdout:3365: done (6 left) 2024-09-12T23:49:51.713 INFO:tasks.rados.rados.0.smithi136.stdout:3366: done (5 left) 2024-09-12T23:49:51.713 INFO:tasks.rados.rados.0.smithi136.stdout:3367: done (4 left) 2024-09-12T23:49:51.713 INFO:tasks.rados.rados.0.smithi136.stdout:3372: snap_remove snap 316 2024-09-12T23:49:51.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:51 smithi181 bash[67587]: cluster 2024-09-12T23:49:50.562344+0000 mon.a (mon.0) 1191 : cluster [DBG] osdmap e725: 8 total, 8 up, 8 in 2024-09-12T23:49:51.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:51 smithi181 bash[67587]: cluster 2024-09-12T23:49:50.835468+0000 mgr.y (mgr.24661) 371 : cluster [DBG] pgmap v668: 121 pgs: 121 active+clean; 313 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 703 KiB/s rd, 5.6 MiB/s wr, 18 op/s 2024-09-12T23:49:51.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:51 smithi181 bash[67587]: cluster 2024-09-12T23:49:51.470664+0000 mon.a (mon.0) 1192 : cluster [DBG] osdmap e726: 8 total, 8 up, 8 in 2024-09-12T23:49:52.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:51 smithi136 bash[75164]: cluster 2024-09-12T23:49:50.562344+0000 mon.a (mon.0) 1191 : cluster [DBG] osdmap e725: 8 total, 8 up, 8 in 2024-09-12T23:49:52.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:51 smithi136 bash[75164]: cluster 2024-09-12T23:49:50.835468+0000 mgr.y (mgr.24661) 371 : cluster [DBG] pgmap v668: 121 pgs: 121 active+clean; 313 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 703 KiB/s rd, 5.6 MiB/s wr, 18 op/s 2024-09-12T23:49:52.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:51 smithi136 bash[75164]: cluster 2024-09-12T23:49:51.470664+0000 mon.a (mon.0) 1192 : cluster [DBG] osdmap e726: 8 total, 8 up, 8 in 2024-09-12T23:49:52.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:51 smithi136 bash[76371]: cluster 2024-09-12T23:49:50.562344+0000 mon.a (mon.0) 1191 : cluster [DBG] osdmap e725: 8 total, 8 up, 8 in 2024-09-12T23:49:52.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:51 smithi136 bash[76371]: cluster 2024-09-12T23:49:50.835468+0000 mgr.y (mgr.24661) 371 : cluster [DBG] pgmap v668: 121 pgs: 121 active+clean; 313 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 703 KiB/s rd, 5.6 MiB/s wr, 18 op/s 2024-09-12T23:49:52.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:51 smithi136 bash[76371]: cluster 2024-09-12T23:49:51.470664+0000 mon.a (mon.0) 1192 : cluster [DBG] osdmap e726: 8 total, 8 up, 8 in 2024-09-12T23:49:52.581 INFO:tasks.rados.rados.0.smithi136.stdout:3372: done (4 left) 2024-09-12T23:49:52.581 INFO:tasks.rados.rados.0.smithi136.stdout:3373: rmattr oid 45 current snap is 324 2024-09-12T23:49:52.581 INFO:tasks.rados.rados.0.smithi136.stdout:3373: done (4 left) 2024-09-12T23:49:52.581 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 44 current snap is 324 2024-09-12T23:49:52.582 INFO:tasks.rados.rados.0.smithi136.stdout:3368: finishing write tid 1 to smithi13667081-43 2024-09-12T23:49:52.582 INFO:tasks.rados.rados.0.smithi136.stdout:3368: finishing write tid 2 to smithi13667081-43 2024-09-12T23:49:52.582 INFO:tasks.rados.rados.0.smithi136.stdout:3368: finishing write tid 3 to smithi13667081-43 2024-09-12T23:49:52.582 INFO:tasks.rados.rados.0.smithi136.stdout:3374: seq_num 1070 ranges {0=729088,729088=401408} 2024-09-12T23:49:52.586 INFO:tasks.rados.rados.0.smithi136.stdout:3374: writing smithi13667081-44 from 0 to 729088 tid 1 2024-09-12T23:49:52.589 INFO:tasks.rados.rados.0.smithi136.stdout:3374: writing smithi13667081-44 from 729088 to 1130496 tid 2 2024-09-12T23:49:52.589 INFO:tasks.rados.rados.0.smithi136.stdout:3375: snap_remove snap 317 2024-09-12T23:49:52.597 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:49:52 smithi181 bash[95505]: ts=2024-09-12T23:49:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:49:53.586 INFO:tasks.rados.rados.0.smithi136.stdout:3375: done (5 left) 2024-09-12T23:49:53.586 INFO:tasks.rados.rados.0.smithi136.stdout:3376: copy_from oid 16 from oid 21 current snap is 324 2024-09-12T23:49:53.586 INFO:tasks.rados.rados.0.smithi136.stdout:3371: finishing rollback tid 0 to smithi13667081-38 2024-09-12T23:49:53.586 INFO:tasks.rados.rados.0.smithi136.stdout:3371: finishing rollback tid 1 to smithi13667081-38 2024-09-12T23:49:53.586 INFO:tasks.rados.rados.0.smithi136.stdout:3371: finishing rollback tid 2 to smithi13667081-38 2024-09-12T23:49:53.586 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 38 v 2453 (ObjNum 864 snap 281 seq_num 864) dirty exists 2024-09-12T23:49:53.586 INFO:tasks.rados.rados.0.smithi136.stdout:3368: finishing write tid 4 to smithi13667081-43 2024-09-12T23:49:53.587 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 43 v 2084 (ObjNum 1069 snap 324 seq_num 1069) dirty exists 2024-09-12T23:49:53.587 INFO:tasks.rados.rados.0.smithi136.stdout:3368: left oid 43 (ObjNum 1069 snap 324 seq_num 1069) 2024-09-12T23:49:53.587 INFO:tasks.rados.rados.0.smithi136.stdout:3369: expect (ObjNum 838 snap 267 seq_num 838) 2024-09-12T23:49:53.654 INFO:tasks.rados.rados.0.smithi136.stdout:3370: expect (ObjNum 1007 snap 313 seq_num 1007) 2024-09-12T23:49:53.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:53 smithi181 bash[67587]: audit 2024-09-12T23:49:52.554054+0000 mon.a (mon.0) 1193 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:53.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:53 smithi181 bash[67587]: audit 2024-09-12T23:49:52.558852+0000 mon.a (mon.0) 1194 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:53.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:53 smithi181 bash[67587]: cluster 2024-09-12T23:49:52.583860+0000 mon.a (mon.0) 1195 : cluster [DBG] osdmap e727: 8 total, 8 up, 8 in 2024-09-12T23:49:53.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:53 smithi181 bash[67587]: cluster 2024-09-12T23:49:52.835927+0000 mgr.y (mgr.24661) 372 : cluster [DBG] pgmap v671: 121 pgs: 121 active+clean; 325 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 703 KiB/s rd, 3.0 MiB/s wr, 11 op/s 2024-09-12T23:49:53.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:53 smithi181 bash[67587]: audit 2024-09-12T23:49:53.308588+0000 mon.a (mon.0) 1196 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:53.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:53 smithi181 bash[67587]: audit 2024-09-12T23:49:53.316648+0000 mon.a (mon.0) 1197 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:53.856 INFO:tasks.rados.rados.0.smithi136.stdout:3374: finishing write tid 1 to smithi13667081-44 2024-09-12T23:49:53.856 INFO:tasks.rados.rados.0.smithi136.stdout:3374: finishing write tid 2 to smithi13667081-44 2024-09-12T23:49:53.856 INFO:tasks.rados.rados.0.smithi136.stdout:3374: finishing write tid 3 to smithi13667081-44 2024-09-12T23:49:53.856 INFO:tasks.rados.rados.0.smithi136.stdout:3374: finishing write tid 4 to smithi13667081-44 2024-09-12T23:49:53.856 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 44 v 2087 (ObjNum 1070 snap 324 seq_num 1070) dirty exists 2024-09-12T23:49:53.856 INFO:tasks.rados.rados.0.smithi136.stdout:3374: left oid 44 (ObjNum 1070 snap 324 seq_num 1070) 2024-09-12T23:49:53.856 INFO:tasks.rados.rados.0.smithi136.stdout:3376: finishing copy_from racing read to smithi13667081-16 2024-09-12T23:49:53.856 INFO:tasks.rados.rados.0.smithi136.stdout:3376: finishing copy_from to smithi13667081-16 2024-09-12T23:49:53.856 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 16 v 1978 (ObjNum 977 snap 303 seq_num 977) dirty exists 2024-09-12T23:49:53.856 INFO:tasks.rados.rados.0.smithi136.stdout:3368: done (5 left) 2024-09-12T23:49:53.856 INFO:tasks.rados.rados.0.smithi136.stdout:3369: done (4 left) 2024-09-12T23:49:53.856 INFO:tasks.rados.rados.0.smithi136.stdout:3370: done (3 left) 2024-09-12T23:49:53.857 INFO:tasks.rados.rados.0.smithi136.stdout:3371: done (2 left) 2024-09-12T23:49:53.857 INFO:tasks.rados.rados.0.smithi136.stdout:3374: done (1 left) 2024-09-12T23:49:53.857 INFO:tasks.rados.rados.0.smithi136.stdout:3376: done (0 left) 2024-09-12T23:49:53.857 INFO:tasks.rados.rados.0.smithi136.stdout:3377: copy_from oid 39 from oid 5 current snap is 324 2024-09-12T23:49:53.857 INFO:tasks.rados.rados.0.smithi136.stdout:3378: setattr oid 21 current snap is 324 2024-09-12T23:49:53.857 INFO:tasks.rados.rados.0.smithi136.stdout:3379: delete oid 13 current snap is 324 2024-09-12T23:49:53.857 INFO:tasks.rados.rados.0.smithi136.stdout:3377: finishing copy_from to smithi13667081-39 2024-09-12T23:49:53.858 INFO:tasks.rados.rados.0.smithi136.stdout:3377: got expected ENOENT (src dne) 2024-09-12T23:49:53.858 INFO:tasks.rados.rados.0.smithi136.stdout:3377: finishing copy_from racing read to smithi13667081-39 2024-09-12T23:49:53.861 INFO:tasks.rados.rados.0.smithi136.stdout:3377: done (2 left) 2024-09-12T23:49:53.861 INFO:tasks.rados.rados.0.smithi136.stdout:3379: done (1 left) 2024-09-12T23:49:53.861 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 14 current snap is 324 2024-09-12T23:49:53.861 INFO:tasks.rados.rados.0.smithi136.stdout:3380: seq_num 1072 ranges {573440=761856,1335296=401408} 2024-09-12T23:49:53.877 INFO:tasks.rados.rados.0.smithi136.stdout:3380: writing smithi13667081-14 from 573440 to 1335296 tid 1 2024-09-12T23:49:53.883 INFO:tasks.rados.rados.0.smithi136.stdout:3380: writing smithi13667081-14 from 1335296 to 1736704 tid 2 2024-09-12T23:49:53.883 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 47 current snap is 324 2024-09-12T23:49:53.883 INFO:tasks.rados.rados.0.smithi136.stdout:3381: seq_num 1073 ranges {0=778240,778240=401408} 2024-09-12T23:49:53.893 INFO:tasks.rados.rados.0.smithi136.stdout:3381: writing smithi13667081-47 from 0 to 778240 tid 1 2024-09-12T23:49:53.899 INFO:tasks.rados.rados.0.smithi136.stdout:3381: writing smithi13667081-47 from 778240 to 1179648 tid 2 2024-09-12T23:49:53.899 INFO:tasks.rados.rados.0.smithi136.stdout:3382: setattr oid 35 current snap is 324 2024-09-12T23:49:53.899 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 21 v 2479 (ObjNum 977 snap 303 seq_num 977) dirty exists 2024-09-12T23:49:53.899 INFO:tasks.rados.rados.0.smithi136.stdout:3380: finishing write tid 1 to smithi13667081-14 2024-09-12T23:49:53.899 INFO:tasks.rados.rados.0.smithi136.stdout:3380: finishing write tid 2 to smithi13667081-14 2024-09-12T23:49:53.899 INFO:tasks.rados.rados.0.smithi136.stdout:3380: finishing write tid 3 to smithi13667081-14 2024-09-12T23:49:53.899 INFO:tasks.rados.rados.0.smithi136.stdout:3380: finishing write tid 4 to smithi13667081-14 2024-09-12T23:49:53.899 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 14 v 2112 (ObjNum 1072 snap 324 seq_num 1072) dirty exists 2024-09-12T23:49:53.899 INFO:tasks.rados.rados.0.smithi136.stdout:3380: left oid 14 (ObjNum 1072 snap 324 seq_num 1072) 2024-09-12T23:49:53.900 INFO:tasks.rados.rados.0.smithi136.stdout:3381: finishing write tid 1 to smithi13667081-47 2024-09-12T23:49:53.901 INFO:tasks.rados.rados.0.smithi136.stdout:3378: done (3 left) 2024-09-12T23:49:53.902 INFO:tasks.rados.rados.0.smithi136.stdout:3380: done (2 left) 2024-09-12T23:49:53.902 INFO:tasks.rados.rados.0.smithi136.stdout:3383: rollback oid 14 current snap is 324 2024-09-12T23:49:53.902 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 14 to 322 2024-09-12T23:49:53.902 INFO:tasks.rados.rados.0.smithi136.stdout:3384: setattr oid 22 current snap is 324 2024-09-12T23:49:53.905 INFO:tasks.rados.rados.0.smithi136.stdout:3385: snap_create 2024-09-12T23:49:53.906 INFO:tasks.rados.rados.0.smithi136.stdout:3381: finishing write tid 2 to smithi13667081-47 2024-09-12T23:49:53.906 INFO:tasks.rados.rados.0.smithi136.stdout:3381: finishing write tid 3 to smithi13667081-47 2024-09-12T23:49:53.910 INFO:tasks.rados.rados.0.smithi136.stdout:3381: finishing write tid 4 to smithi13667081-47 2024-09-12T23:49:53.911 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 47 v 2468 (ObjNum 1073 snap 324 seq_num 1073) dirty exists 2024-09-12T23:49:53.911 INFO:tasks.rados.rados.0.smithi136.stdout:3381: left oid 47 (ObjNum 1073 snap 324 seq_num 1073) 2024-09-12T23:49:53.911 INFO:tasks.rados.rados.0.smithi136.stdout:3383: finishing rollback tid 0 to smithi13667081-14 2024-09-12T23:49:53.911 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 35 v 2639 (ObjNum 927 snap 290 seq_num 927) dirty exists 2024-09-12T23:49:53.912 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 22 v 2480 (ObjNum 1036 snap 319 seq_num 1036) dirty exists 2024-09-12T23:49:53.913 INFO:tasks.rados.rados.0.smithi136.stdout:3383: finishing rollback tid 1 to smithi13667081-14 2024-09-12T23:49:53.913 INFO:tasks.rados.rados.0.smithi136.stdout:3383: finishing rollback tid 2 to smithi13667081-14 2024-09-12T23:49:53.913 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 14 v 2115 (ObjNum 1037 snap 319 seq_num 1037) dirty exists 2024-09-12T23:49:54.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:53 smithi136 bash[75164]: audit 2024-09-12T23:49:52.554054+0000 mon.a (mon.0) 1193 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:54.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:53 smithi136 bash[75164]: audit 2024-09-12T23:49:52.558852+0000 mon.a (mon.0) 1194 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:54.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:53 smithi136 bash[75164]: cluster 2024-09-12T23:49:52.583860+0000 mon.a (mon.0) 1195 : cluster [DBG] osdmap e727: 8 total, 8 up, 8 in 2024-09-12T23:49:54.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:53 smithi136 bash[75164]: cluster 2024-09-12T23:49:52.835927+0000 mgr.y (mgr.24661) 372 : cluster [DBG] pgmap v671: 121 pgs: 121 active+clean; 325 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 703 KiB/s rd, 3.0 MiB/s wr, 11 op/s 2024-09-12T23:49:54.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:53 smithi136 bash[75164]: audit 2024-09-12T23:49:53.308588+0000 mon.a (mon.0) 1196 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:54.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:53 smithi136 bash[75164]: audit 2024-09-12T23:49:53.316648+0000 mon.a (mon.0) 1197 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:54.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:53 smithi136 bash[76371]: audit 2024-09-12T23:49:52.554054+0000 mon.a (mon.0) 1193 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:54.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:53 smithi136 bash[76371]: audit 2024-09-12T23:49:52.558852+0000 mon.a (mon.0) 1194 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:54.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:53 smithi136 bash[76371]: cluster 2024-09-12T23:49:52.583860+0000 mon.a (mon.0) 1195 : cluster [DBG] osdmap e727: 8 total, 8 up, 8 in 2024-09-12T23:49:54.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:53 smithi136 bash[76371]: cluster 2024-09-12T23:49:52.835927+0000 mgr.y (mgr.24661) 372 : cluster [DBG] pgmap v671: 121 pgs: 121 active+clean; 325 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 703 KiB/s rd, 3.0 MiB/s wr, 11 op/s 2024-09-12T23:49:54.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:53 smithi136 bash[76371]: audit 2024-09-12T23:49:53.308588+0000 mon.a (mon.0) 1196 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:54.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:53 smithi136 bash[76371]: audit 2024-09-12T23:49:53.316648+0000 mon.a (mon.0) 1197 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:49:54.601 INFO:tasks.rados.rados.0.smithi136.stdout:3381: done (4 left) 2024-09-12T23:49:54.601 INFO:tasks.rados.rados.0.smithi136.stdout:3382: done (3 left) 2024-09-12T23:49:54.602 INFO:tasks.rados.rados.0.smithi136.stdout:3383: done (2 left) 2024-09-12T23:49:54.602 INFO:tasks.rados.rados.0.smithi136.stdout:3384: done (1 left) 2024-09-12T23:49:54.602 INFO:tasks.rados.rados.0.smithi136.stdout:3385: done (0 left) 2024-09-12T23:49:54.602 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 45 current snap is 325 2024-09-12T23:49:54.602 INFO:tasks.rados.rados.0.smithi136.stdout:3386: seq_num 1074 ranges {1441792=794624,2236416=401408} 2024-09-12T23:49:54.623 INFO:tasks.rados.rados.0.smithi136.stdout:3386: writing smithi13667081-45 from 1441792 to 2236416 tid 1 2024-09-12T23:49:54.627 INFO:tasks.rados.rados.0.smithi136.stdout:3386: writing smithi13667081-45 from 2236416 to 2637824 tid 2 2024-09-12T23:49:54.627 INFO:tasks.rados.rados.0.smithi136.stdout:3387: read oid 41 snap 310 2024-09-12T23:49:54.627 INFO:tasks.rados.rados.0.smithi136.stdout:3387: expect (ObjNum 851 snap 272 seq_num 851) 2024-09-12T23:49:54.627 INFO:tasks.rados.rados.0.smithi136.stdout:3388: delete oid 6 current snap is 325 2024-09-12T23:49:54.640 INFO:tasks.rados.rados.0.smithi136.stdout:3388: done (2 left) 2024-09-12T23:49:54.640 INFO:tasks.rados.rados.0.smithi136.stdout:3389: copy_from oid 5 from oid 32 current snap is 325 2024-09-12T23:49:54.640 INFO:tasks.rados.rados.0.smithi136.stdout:3390: setattr oid 24 current snap is 325 2024-09-12T23:49:54.641 INFO:tasks.rados.rados.0.smithi136.stdout:3386: finishing write tid 1 to smithi13667081-45 2024-09-12T23:49:54.641 INFO:tasks.rados.rados.0.smithi136.stdout:3386: finishing write tid 2 to smithi13667081-45 2024-09-12T23:49:54.641 INFO:tasks.rados.rados.0.smithi136.stdout:3386: finishing write tid 3 to smithi13667081-45 2024-09-12T23:49:54.643 INFO:tasks.rados.rados.0.smithi136.stdout:3391: read oid 40 snap 310 2024-09-12T23:49:54.643 INFO:tasks.rados.rados.0.smithi136.stdout:3391: expect (ObjNum 983 snap 303 seq_num 983) 2024-09-12T23:49:54.643 INFO:tasks.rados.rados.0.smithi136.stdout:3392: copy_from oid 31 from oid 44 current snap is 325 2024-09-12T23:49:54.643 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 38 current snap is 325 2024-09-12T23:49:54.643 INFO:tasks.rados.rados.0.smithi136.stdout:3393: seq_num 1076 ranges {2457600=425984,2883584=548864,3432448=253952} 2024-09-12T23:49:54.672 INFO:tasks.rados.rados.0.smithi136.stdout:3393: writing smithi13667081-38 from 2457600 to 2883584 tid 1 2024-09-12T23:49:54.675 INFO:tasks.rados.rados.0.smithi136.stdout:3393: writing smithi13667081-38 from 2883584 to 3432448 tid 2 2024-09-12T23:49:54.677 INFO:tasks.rados.rados.0.smithi136.stdout:3393: writing smithi13667081-38 from 3432448 to 3686400 tid 3 2024-09-12T23:49:54.677 INFO:tasks.rados.rados.0.smithi136.stdout:3394: rollback oid 42 current snap is 325 2024-09-12T23:49:54.678 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 42 to 322 2024-09-12T23:49:54.678 INFO:tasks.rados.rados.0.smithi136.stdout:3386: finishing write tid 4 to smithi13667081-45 2024-09-12T23:49:54.678 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 45 v 2348 (ObjNum 1074 snap 325 seq_num 1074) dirty exists 2024-09-12T23:49:54.678 INFO:tasks.rados.rados.0.smithi136.stdout:3386: left oid 45 (ObjNum 1074 snap 325 seq_num 1074) 2024-09-12T23:49:54.678 INFO:tasks.rados.rados.0.smithi136.stdout:3387: expect (ObjNum 851 snap 272 seq_num 851) 2024-09-12T23:49:54.828 INFO:tasks.rados.rados.0.smithi136.stdout:3392: finishing copy_from racing read to smithi13667081-31 2024-09-12T23:49:54.828 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 24 v 2474 (ObjNum 993 snap 307 seq_num 993) dirty exists 2024-09-12T23:49:54.828 INFO:tasks.rados.rados.0.smithi136.stdout:3392: finishing copy_from to smithi13667081-31 2024-09-12T23:49:54.828 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 31 v 2087 (ObjNum 1070 snap 324 seq_num 1070) dirty exists 2024-09-12T23:49:54.828 INFO:tasks.rados.rados.0.smithi136.stdout:3391: expect (ObjNum 983 snap 303 seq_num 983) 2024-09-12T23:49:54.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:54 smithi181 bash[67587]: cluster 2024-09-12T23:49:53.589196+0000 mon.a (mon.0) 1198 : cluster [DBG] osdmap e728: 8 total, 8 up, 8 in 2024-09-12T23:49:54.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:54 smithi181 bash[67587]: audit 2024-09-12T23:49:53.752004+0000 mon.a (mon.0) 1199 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:49:54.943 INFO:tasks.rados.rados.0.smithi136.stdout:3394: finishing rollback tid 0 to smithi13667081-42 2024-09-12T23:49:54.943 INFO:tasks.rados.rados.0.smithi136.stdout:3394: finishing rollback tid 1 to smithi13667081-42 2024-09-12T23:49:54.943 INFO:tasks.rados.rados.0.smithi136.stdout:3394: finishing rollback tid 2 to smithi13667081-42 2024-09-12T23:49:54.943 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 42 v 2483 (ObjNum 1062 snap 321 seq_num 1062) dirty exists 2024-09-12T23:49:54.943 INFO:tasks.rados.rados.0.smithi136.stdout:3393: finishing write tid 1 to smithi13667081-38 2024-09-12T23:49:54.943 INFO:tasks.rados.rados.0.smithi136.stdout:3393: finishing write tid 2 to smithi13667081-38 2024-09-12T23:49:54.943 INFO:tasks.rados.rados.0.smithi136.stdout:3393: finishing write tid 3 to smithi13667081-38 2024-09-12T23:49:54.943 INFO:tasks.rados.rados.0.smithi136.stdout:3393: finishing write tid 4 to smithi13667081-38 2024-09-12T23:49:54.943 INFO:tasks.rados.rados.0.smithi136.stdout:3393: finishing write tid 5 to smithi13667081-38 2024-09-12T23:49:54.943 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 38 v 2457 (ObjNum 1076 snap 325 seq_num 1076) dirty exists 2024-09-12T23:49:54.944 INFO:tasks.rados.rados.0.smithi136.stdout:3393: left oid 38 (ObjNum 1076 snap 325 seq_num 1076) 2024-09-12T23:49:54.944 INFO:tasks.rados.rados.0.smithi136.stdout:3389: finishing copy_from racing read to smithi13667081-5 2024-09-12T23:49:54.944 INFO:tasks.rados.rados.0.smithi136.stdout:3389: finishing copy_from to smithi13667081-5 2024-09-12T23:49:54.944 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 5 v 1640 (ObjNum 1026 snap 316 seq_num 1026) dirty exists 2024-09-12T23:49:54.944 INFO:tasks.rados.rados.0.smithi136.stdout:3386: done (7 left) 2024-09-12T23:49:54.944 INFO:tasks.rados.rados.0.smithi136.stdout:3387: done (6 left) 2024-09-12T23:49:54.944 INFO:tasks.rados.rados.0.smithi136.stdout:3389: done (5 left) 2024-09-12T23:49:54.944 INFO:tasks.rados.rados.0.smithi136.stdout:3390: done (4 left) 2024-09-12T23:49:54.944 INFO:tasks.rados.rados.0.smithi136.stdout:3391: done (3 left) 2024-09-12T23:49:54.944 INFO:tasks.rados.rados.0.smithi136.stdout:3392: done (2 left) 2024-09-12T23:49:54.944 INFO:tasks.rados.rados.0.smithi136.stdout:3393: done (1 left) 2024-09-12T23:49:54.944 INFO:tasks.rados.rados.0.smithi136.stdout:3394: done (0 left) 2024-09-12T23:49:54.944 INFO:tasks.rados.rados.0.smithi136.stdout:3395: snap_remove snap 323 2024-09-12T23:49:55.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:54 smithi136 bash[75164]: cluster 2024-09-12T23:49:53.589196+0000 mon.a (mon.0) 1198 : cluster [DBG] osdmap e728: 8 total, 8 up, 8 in 2024-09-12T23:49:55.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:54 smithi136 bash[75164]: audit 2024-09-12T23:49:53.752004+0000 mon.a (mon.0) 1199 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:49:55.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:54 smithi136 bash[76371]: cluster 2024-09-12T23:49:53.589196+0000 mon.a (mon.0) 1198 : cluster [DBG] osdmap e728: 8 total, 8 up, 8 in 2024-09-12T23:49:55.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:54 smithi136 bash[76371]: audit 2024-09-12T23:49:53.752004+0000 mon.a (mon.0) 1199 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:49:55.625 INFO:tasks.rados.rados.0.smithi136.stdout:3395: done (0 left) 2024-09-12T23:49:55.625 INFO:tasks.rados.rados.0.smithi136.stdout:3396: rollback oid 24 current snap is 325 2024-09-12T23:49:55.626 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 24 to 310 2024-09-12T23:49:55.626 INFO:tasks.rados.rados.0.smithi136.stdout:3397: read oid 2 snap -1 2024-09-12T23:49:55.626 INFO:tasks.rados.rados.0.smithi136.stdout:3397: expect (ObjNum 864 snap 281 seq_num 864) 2024-09-12T23:49:55.626 INFO:tasks.rados.rados.0.smithi136.stdout:3398: delete oid 48 current snap is 325 2024-09-12T23:49:55.636 INFO:tasks.rados.rados.0.smithi136.stdout:3398: done (2 left) 2024-09-12T23:49:55.637 INFO:tasks.rados.rados.0.smithi136.stdout:3399: rmattr oid 45 current snap is 325 2024-09-12T23:49:55.637 INFO:tasks.rados.rados.0.smithi136.stdout:3399: done (2 left) 2024-09-12T23:49:55.637 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 3 current snap is 325 2024-09-12T23:49:55.637 INFO:tasks.rados.rados.0.smithi136.stdout:3400: seq_num 1078 ranges {2048000=458752,2506752=720896,3227648=81920} 2024-09-12T23:49:55.652 INFO:tasks.rados.rados.0.smithi136.stdout:3400: writing smithi13667081-3 from 2048000 to 2506752 tid 1 2024-09-12T23:49:55.656 INFO:tasks.rados.rados.0.smithi136.stdout:3400: writing smithi13667081-3 from 2506752 to 3227648 tid 2 2024-09-12T23:49:55.657 INFO:tasks.rados.rados.0.smithi136.stdout:3400: writing smithi13667081-3 from 3227648 to 3309568 tid 3 2024-09-12T23:49:55.657 INFO:tasks.rados.rados.0.smithi136.stdout:3401: snap_create 2024-09-12T23:49:55.657 INFO:tasks.rados.rados.0.smithi136.stdout:3397: expect (ObjNum 864 snap 281 seq_num 864) 2024-09-12T23:49:55.820 INFO:tasks.rados.rados.0.smithi136.stdout:3396: finishing rollback tid 0 to smithi13667081-24 2024-09-12T23:49:55.821 INFO:tasks.rados.rados.0.smithi136.stdout:3396: finishing rollback tid 1 to smithi13667081-24 2024-09-12T23:49:55.821 INFO:tasks.rados.rados.0.smithi136.stdout:3396: finishing rollback tid 2 to smithi13667081-24 2024-09-12T23:49:55.821 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 24 v 2477 (ObjNum 997 snap 309 seq_num 997) dirty exists 2024-09-12T23:49:55.821 INFO:tasks.rados.rados.0.smithi136.stdout:3400: finishing write tid 1 to smithi13667081-3 2024-09-12T23:49:55.821 INFO:tasks.rados.rados.0.smithi136.stdout:3400: finishing write tid 2 to smithi13667081-3 2024-09-12T23:49:55.821 INFO:tasks.rados.rados.0.smithi136.stdout:3400: finishing write tid 3 to smithi13667081-3 2024-09-12T23:49:55.821 INFO:tasks.rados.rados.0.smithi136.stdout:3400: finishing write tid 4 to smithi13667081-3 2024-09-12T23:49:55.821 INFO:tasks.rados.rados.0.smithi136.stdout:3400: finishing write tid 5 to smithi13667081-3 2024-09-12T23:49:55.821 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 3 v 2091 (ObjNum 1078 snap 325 seq_num 1078) dirty exists 2024-09-12T23:49:55.821 INFO:tasks.rados.rados.0.smithi136.stdout:3400: left oid 3 (ObjNum 1078 snap 325 seq_num 1078) 2024-09-12T23:49:56.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:55 smithi136 bash[75164]: cluster 2024-09-12T23:49:54.604343+0000 mon.a (mon.0) 1200 : cluster [DBG] osdmap e729: 8 total, 8 up, 8 in 2024-09-12T23:49:56.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:55 smithi136 bash[75164]: cluster 2024-09-12T23:49:54.836362+0000 mgr.y (mgr.24661) 373 : cluster [DBG] pgmap v674: 121 pgs: 121 active+clean; 327 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 3.5 MiB/s wr, 15 op/s 2024-09-12T23:49:56.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:55 smithi136 bash[76371]: cluster 2024-09-12T23:49:54.604343+0000 mon.a (mon.0) 1200 : cluster [DBG] osdmap e729: 8 total, 8 up, 8 in 2024-09-12T23:49:56.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:55 smithi136 bash[76371]: cluster 2024-09-12T23:49:54.836362+0000 mgr.y (mgr.24661) 373 : cluster [DBG] pgmap v674: 121 pgs: 121 active+clean; 327 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 3.5 MiB/s wr, 15 op/s 2024-09-12T23:49:56.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:55 smithi181 bash[67587]: cluster 2024-09-12T23:49:54.604343+0000 mon.a (mon.0) 1200 : cluster [DBG] osdmap e729: 8 total, 8 up, 8 in 2024-09-12T23:49:56.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:55 smithi181 bash[67587]: cluster 2024-09-12T23:49:54.836362+0000 mgr.y (mgr.24661) 373 : cluster [DBG] pgmap v674: 121 pgs: 121 active+clean; 327 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 3.5 MiB/s wr, 15 op/s 2024-09-12T23:49:56.471 INFO:tasks.rados.rados.0.smithi136.stdout:3396: done (3 left) 2024-09-12T23:49:56.472 INFO:tasks.rados.rados.0.smithi136.stdout:3397: done (2 left) 2024-09-12T23:49:56.472 INFO:tasks.rados.rados.0.smithi136.stdout:3400: done (1 left) 2024-09-12T23:49:56.472 INFO:tasks.rados.rados.0.smithi136.stdout:3401: done (0 left) 2024-09-12T23:49:56.472 INFO:tasks.rados.rados.0.smithi136.stdout:3402: delete oid 25 current snap is 326 2024-09-12T23:49:56.481 INFO:tasks.rados.rados.0.smithi136.stdout:3402: done (0 left) 2024-09-12T23:49:56.481 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 45 current snap is 326 2024-09-12T23:49:56.481 INFO:tasks.rados.rados.0.smithi136.stdout:3403: seq_num 1080 ranges {2637824=491520,3129344=540672,3670016=262144} 2024-09-12T23:49:56.513 INFO:tasks.rados.rados.0.smithi136.stdout:3403: writing smithi13667081-45 from 2637824 to 3129344 tid 1 2024-09-12T23:49:56.518 INFO:tasks.rados.rados.0.smithi136.stdout:3403: writing smithi13667081-45 from 3129344 to 3670016 tid 2 2024-09-12T23:49:56.522 INFO:tasks.rados.rados.0.smithi136.stdout:3403: writing smithi13667081-45 from 3670016 to 3932160 tid 3 2024-09-12T23:49:56.522 INFO:tasks.rados.rados.0.smithi136.stdout:3404: read oid 17 snap -1 2024-09-12T23:49:56.522 INFO:tasks.rados.rados.0.smithi136.stdout:3404: expect (ObjNum 1007 snap 313 seq_num 1007) 2024-09-12T23:49:56.522 INFO:tasks.rados.rados.0.smithi136.stdout:3405: read oid 3 snap -1 2024-09-12T23:49:56.522 INFO:tasks.rados.rados.0.smithi136.stdout:3405: expect (ObjNum 1078 snap 325 seq_num 1078) 2024-09-12T23:49:56.522 INFO:tasks.rados.rados.0.smithi136.stdout:3406: copy_from oid 11 from oid 28 current snap is 326 2024-09-12T23:49:56.522 INFO:tasks.rados.rados.0.smithi136.stdout:3407: read oid 16 snap -1 2024-09-12T23:49:56.522 INFO:tasks.rados.rados.0.smithi136.stdout:3407: expect (ObjNum 977 snap 303 seq_num 977) 2024-09-12T23:49:56.522 INFO:tasks.rados.rados.0.smithi136.stdout:3408: read oid 40 snap 322 2024-09-12T23:49:56.523 INFO:tasks.rados.rados.0.smithi136.stdout:3408: expect (ObjNum 1048 snap 321 seq_num 1048) 2024-09-12T23:49:56.523 INFO:tasks.rados.rados.0.smithi136.stdout:3409: read oid 6 snap 309 2024-09-12T23:49:56.523 INFO:tasks.rados.rados.0.smithi136.stdout:3409: expect deleted 2024-09-12T23:49:56.523 INFO:tasks.rados.rados.0.smithi136.stdout:3410: setattr oid 37 current snap is 326 2024-09-12T23:49:56.525 INFO:tasks.rados.rados.0.smithi136.stdout:3403: finishing write tid 1 to smithi13667081-45 2024-09-12T23:49:56.525 INFO:tasks.rados.rados.0.smithi136.stdout:3411: read oid 5 snap -1 2024-09-12T23:49:56.525 INFO:tasks.rados.rados.0.smithi136.stdout:3411: expect (ObjNum 1026 snap 316 seq_num 1026) 2024-09-12T23:49:56.525 INFO:tasks.rados.rados.0.smithi136.stdout:3412: read oid 30 snap -1 2024-09-12T23:49:56.525 INFO:tasks.rados.rados.0.smithi136.stdout:3412: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-12T23:49:56.525 INFO:tasks.rados.rados.0.smithi136.stdout:3413: snap_remove snap 310 2024-09-12T23:49:57.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:56 smithi136 bash[75164]: cluster 2024-09-12T23:49:55.617360+0000 mon.a (mon.0) 1201 : cluster [DBG] osdmap e730: 8 total, 8 up, 8 in 2024-09-12T23:49:57.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:56 smithi136 bash[75164]: cluster 2024-09-12T23:49:56.473945+0000 mon.a (mon.0) 1202 : cluster [DBG] osdmap e731: 8 total, 8 up, 8 in 2024-09-12T23:49:57.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:56 smithi136 bash[76371]: cluster 2024-09-12T23:49:55.617360+0000 mon.a (mon.0) 1201 : cluster [DBG] osdmap e730: 8 total, 8 up, 8 in 2024-09-12T23:49:57.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:56 smithi136 bash[76371]: cluster 2024-09-12T23:49:56.473945+0000 mon.a (mon.0) 1202 : cluster [DBG] osdmap e731: 8 total, 8 up, 8 in 2024-09-12T23:49:57.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:56 smithi181 bash[67587]: cluster 2024-09-12T23:49:55.617360+0000 mon.a (mon.0) 1201 : cluster [DBG] osdmap e730: 8 total, 8 up, 8 in 2024-09-12T23:49:57.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:56 smithi181 bash[67587]: cluster 2024-09-12T23:49:56.473945+0000 mon.a (mon.0) 1202 : cluster [DBG] osdmap e731: 8 total, 8 up, 8 in 2024-09-12T23:49:57.476 INFO:tasks.rados.rados.0.smithi136.stdout:3412: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-12T23:49:57.476 INFO:tasks.rados.rados.0.smithi136.stdout:3403: finishing write tid 2 to smithi13667081-45 2024-09-12T23:49:57.477 INFO:tasks.rados.rados.0.smithi136.stdout:3403: finishing write tid 3 to smithi13667081-45 2024-09-12T23:49:57.477 INFO:tasks.rados.rados.0.smithi136.stdout:3403: finishing write tid 4 to smithi13667081-45 2024-09-12T23:49:57.477 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 37 v 2474 (ObjNum 1020 snap 315 seq_num 1020) dirty exists 2024-09-12T23:49:57.477 INFO:tasks.rados.rados.0.smithi136.stdout:3403: finishing write tid 5 to smithi13667081-45 2024-09-12T23:49:57.477 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 45 v 2352 (ObjNum 1080 snap 326 seq_num 1080) dirty exists 2024-09-12T23:49:57.477 INFO:tasks.rados.rados.0.smithi136.stdout:3403: left oid 45 (ObjNum 1080 snap 326 seq_num 1080) 2024-09-12T23:49:57.477 INFO:tasks.rados.rados.0.smithi136.stdout:3404: expect (ObjNum 1007 snap 313 seq_num 1007) 2024-09-12T23:49:57.702 INFO:tasks.rados.rados.0.smithi136.stdout:3405: expect (ObjNum 1078 snap 325 seq_num 1078) 2024-09-12T23:49:57.817 INFO:tasks.rados.rados.0.smithi136.stdout:3407: expect (ObjNum 977 snap 303 seq_num 977) 2024-09-12T23:49:57.880 INFO:tasks.rados.rados.0.smithi136.stdout:3406: finishing copy_from racing read to smithi13667081-11 2024-09-12T23:49:57.880 INFO:tasks.rados.rados.0.smithi136.stdout:3408: expect (ObjNum 1048 snap 321 seq_num 1048) 2024-09-12T23:49:58.006 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 12 23:49:57 smithi136 bash[104600]: ts=2024-09-12T23:49:57.547Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002832756s 2024-09-12T23:49:58.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:57 smithi136 bash[75164]: cluster 2024-09-12T23:49:56.837209+0000 mgr.y (mgr.24661) 374 : cluster [DBG] pgmap v677: 121 pgs: 121 active+clean; 332 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 2.8 MiB/s wr, 33 op/s 2024-09-12T23:49:58.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:57 smithi136 bash[75164]: cluster 2024-09-12T23:49:57.475936+0000 mon.a (mon.0) 1203 : cluster [DBG] osdmap e732: 8 total, 8 up, 8 in 2024-09-12T23:49:58.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:57 smithi136 bash[76371]: cluster 2024-09-12T23:49:56.837209+0000 mgr.y (mgr.24661) 374 : cluster [DBG] pgmap v677: 121 pgs: 121 active+clean; 332 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 2.8 MiB/s wr, 33 op/s 2024-09-12T23:49:58.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:57 smithi136 bash[76371]: cluster 2024-09-12T23:49:57.475936+0000 mon.a (mon.0) 1203 : cluster [DBG] osdmap e732: 8 total, 8 up, 8 in 2024-09-12T23:49:58.062 INFO:tasks.rados.rados.0.smithi136.stdout:3406: finishing copy_from to smithi13667081-11 2024-09-12T23:49:58.062 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 11 v 2067 (ObjNum 1056 snap 321 seq_num 1056) dirty exists 2024-09-12T23:49:58.062 INFO:tasks.rados.rados.0.smithi136.stdout:3411: expect (ObjNum 1026 snap 316 seq_num 1026) 2024-09-12T23:49:58.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:57 smithi181 bash[67587]: cluster 2024-09-12T23:49:56.837209+0000 mgr.y (mgr.24661) 374 : cluster [DBG] pgmap v677: 121 pgs: 121 active+clean; 332 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 2.8 MiB/s wr, 33 op/s 2024-09-12T23:49:58.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:57 smithi181 bash[67587]: cluster 2024-09-12T23:49:57.475936+0000 mon.a (mon.0) 1203 : cluster [DBG] osdmap e732: 8 total, 8 up, 8 in 2024-09-12T23:49:59.671 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:49:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:49:59] "GET /metrics HTTP/1.1" 200 36442 "" "Prometheus/2.43.0" 2024-09-12T23:49:59.929 INFO:tasks.rados.rados.0.smithi136.stdout:3403: done (10 left) 2024-09-12T23:49:59.929 INFO:tasks.rados.rados.0.smithi136.stdout:3404: done (9 left) 2024-09-12T23:49:59.929 INFO:tasks.rados.rados.0.smithi136.stdout:3405: done (8 left) 2024-09-12T23:49:59.929 INFO:tasks.rados.rados.0.smithi136.stdout:3406: done (7 left) 2024-09-12T23:49:59.929 INFO:tasks.rados.rados.0.smithi136.stdout:3407: done (6 left) 2024-09-12T23:49:59.929 INFO:tasks.rados.rados.0.smithi136.stdout:3408: done (5 left) 2024-09-12T23:49:59.929 INFO:tasks.rados.rados.0.smithi136.stdout:3409: done (4 left) 2024-09-12T23:49:59.929 INFO:tasks.rados.rados.0.smithi136.stdout:3410: done (3 left) 2024-09-12T23:49:59.929 INFO:tasks.rados.rados.0.smithi136.stdout:3411: done (2 left) 2024-09-12T23:49:59.930 INFO:tasks.rados.rados.0.smithi136.stdout:3412: done (1 left) 2024-09-12T23:49:59.930 INFO:tasks.rados.rados.0.smithi136.stdout:3413: done (0 left) 2024-09-12T23:49:59.930 INFO:tasks.rados.rados.0.smithi136.stdout:3414: rollback oid 38 current snap is 326 2024-09-12T23:49:59.930 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 38 to 324 2024-09-12T23:49:59.930 INFO:tasks.rados.rados.0.smithi136.stdout:3415: read oid 20 snap -1 2024-09-12T23:49:59.930 INFO:tasks.rados.rados.0.smithi136.stdout:3415: expect deleted 2024-09-12T23:49:59.930 INFO:tasks.rados.rados.0.smithi136.stdout:3416: copy_from oid 11 from oid 13 current snap is 326 2024-09-12T23:49:59.930 INFO:tasks.rados.rados.0.smithi136.stdout:3417: rollback oid 44 current snap is 326 2024-09-12T23:49:59.930 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 44 to 303 2024-09-12T23:49:59.930 INFO:tasks.rados.rados.0.smithi136.stdout:3418: setattr oid 27 current snap is 326 2024-09-12T23:49:59.931 INFO:tasks.rados.rados.0.smithi136.stdout:3415: done (4 left) 2024-09-12T23:49:59.931 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 24 current snap is 326 2024-09-12T23:49:59.931 INFO:tasks.rados.rados.0.smithi136.stdout:3419: seq_num 1081 ranges {4358144=507904,4866048=425984,5292032=376832} 2024-09-12T23:50:00.000 INFO:tasks.rados.rados.0.smithi136.stdout:3419: writing smithi13667081-24 from 4358144 to 4866048 tid 1 2024-09-12T23:50:00.005 INFO:tasks.rados.rados.0.smithi136.stdout:3419: writing smithi13667081-24 from 4866048 to 5292032 tid 2 2024-09-12T23:50:00.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:49:59 smithi136 bash[76371]: cluster 2024-09-12T23:49:58.838165+0000 mgr.y (mgr.24661) 375 : cluster [DBG] pgmap v679: 121 pgs: 121 active+clean; 351 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 6.8 MiB/s wr, 47 op/s 2024-09-12T23:50:00.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:49:59 smithi136 bash[75164]: cluster 2024-09-12T23:49:58.838165+0000 mgr.y (mgr.24661) 375 : cluster [DBG] pgmap v679: 121 pgs: 121 active+clean; 351 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 6.8 MiB/s wr, 47 op/s 2024-09-12T23:50:00.010 INFO:tasks.rados.rados.0.smithi136.stdout:3419: writing smithi13667081-24 from 5292032 to 5668864 tid 3 2024-09-12T23:50:00.010 INFO:tasks.rados.rados.0.smithi136.stdout:3420: read oid 45 snap -1 2024-09-12T23:50:00.010 INFO:tasks.rados.rados.0.smithi136.stdout:3420: expect (ObjNum 1080 snap 326 seq_num 1080) 2024-09-12T23:50:00.010 INFO:tasks.rados.rados.0.smithi136.stdout:3421: copy_from oid 8 from oid 7 current snap is 326 2024-09-12T23:50:00.010 INFO:tasks.rados.rados.0.smithi136.stdout:3417: finishing rollback tid 0 to smithi13667081-44 2024-09-12T23:50:00.010 INFO:tasks.rados.rados.0.smithi136.stdout:3417: finishing rollback tid 1 to smithi13667081-44 2024-09-12T23:50:00.010 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 44 v 2131 (ObjNum 113 snap 0 seq_num 1466945341) dirty dne 2024-09-12T23:50:00.010 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 27 v 2659 (ObjNum 1047 snap 321 seq_num 1047) dirty exists 2024-09-12T23:50:00.010 INFO:tasks.rados.rados.0.smithi136.stdout:3414: finishing rollback tid 0 to smithi13667081-38 2024-09-12T23:50:00.010 INFO:tasks.rados.rados.0.smithi136.stdout:3414: finishing rollback tid 1 to smithi13667081-38 2024-09-12T23:50:00.010 INFO:tasks.rados.rados.0.smithi136.stdout:3416: finishing copy_from to smithi13667081-11 2024-09-12T23:50:00.011 INFO:tasks.rados.rados.0.smithi136.stdout:3416: got expected ENOENT (src dne) 2024-09-12T23:50:00.011 INFO:tasks.rados.rados.0.smithi136.stdout:3414: finishing rollback tid 2 to smithi13667081-38 2024-09-12T23:50:00.011 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 38 v 2460 (ObjNum 864 snap 281 seq_num 864) dirty exists 2024-09-12T23:50:00.011 INFO:tasks.rados.rados.0.smithi136.stdout:3416: finishing copy_from racing read to smithi13667081-11 2024-09-12T23:50:00.011 INFO:tasks.rados.rados.0.smithi136.stdout:3419: finishing write tid 1 to smithi13667081-24 2024-09-12T23:50:00.011 INFO:tasks.rados.rados.0.smithi136.stdout:3414: done (6 left) 2024-09-12T23:50:00.011 INFO:tasks.rados.rados.0.smithi136.stdout:3416: done (5 left) 2024-09-12T23:50:00.011 INFO:tasks.rados.rados.0.smithi136.stdout:3417: done (4 left) 2024-09-12T23:50:00.011 INFO:tasks.rados.rados.0.smithi136.stdout:3418: done (3 left) 2024-09-12T23:50:00.011 INFO:tasks.rados.rados.0.smithi136.stdout:3422: read oid 46 snap -1 2024-09-12T23:50:00.011 INFO:tasks.rados.rados.0.smithi136.stdout:3422: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-12T23:50:00.011 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 4 current snap is 326 2024-09-12T23:50:00.011 INFO:tasks.rados.rados.0.smithi136.stdout:3423: seq_num 1082 ranges {17375232=524288,17899520=704512,18604032=98304} 2024-09-12T23:50:00.110 INFO:tasks.rados.rados.0.smithi136.stdout:3423: writing smithi13667081-4 from 17375232 to 17899520 tid 1 2024-09-12T23:50:00.114 INFO:tasks.rados.rados.0.smithi136.stdout:3423: writing smithi13667081-4 from 17899520 to 18604032 tid 2 2024-09-12T23:50:00.115 INFO:tasks.rados.rados.0.smithi136.stdout:3423: writing smithi13667081-4 from 18604032 to 18702336 tid 3 2024-09-12T23:50:00.115 INFO:tasks.rados.rados.0.smithi136.stdout:3424: copy_from oid 29 from oid 27 current snap is 326 2024-09-12T23:50:00.115 INFO:tasks.rados.rados.0.smithi136.stdout:3419: finishing write tid 2 to smithi13667081-24 2024-09-12T23:50:00.115 INFO:tasks.rados.rados.0.smithi136.stdout:3422: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-12T23:50:00.118 INFO:tasks.rados.rados.0.smithi136.stdout:3419: finishing write tid 3 to smithi13667081-24 2024-09-12T23:50:00.118 INFO:tasks.rados.rados.0.smithi136.stdout:3419: finishing write tid 4 to smithi13667081-24 2024-09-12T23:50:00.118 INFO:tasks.rados.rados.0.smithi136.stdout:3419: finishing write tid 5 to smithi13667081-24 2024-09-12T23:50:00.118 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 24 v 2481 (ObjNum 1081 snap 326 seq_num 1081) dirty exists 2024-09-12T23:50:00.118 INFO:tasks.rados.rados.0.smithi136.stdout:3419: left oid 24 (ObjNum 1081 snap 326 seq_num 1081) 2024-09-12T23:50:00.118 INFO:tasks.rados.rados.0.smithi136.stdout:3420: expect (ObjNum 1080 snap 326 seq_num 1080) 2024-09-12T23:50:00.307 INFO:tasks.rados.rados.0.smithi136.stdout:3421: finishing copy_from racing read to smithi13667081-8 2024-09-12T23:50:00.307 INFO:tasks.rados.rados.0.smithi136.stdout:3421: finishing copy_from to smithi13667081-8 2024-09-12T23:50:00.307 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 8 v 990 (ObjNum 994 snap 308 seq_num 994) dirty exists 2024-09-12T23:50:00.308 INFO:tasks.rados.rados.0.smithi136.stdout:3423: finishing write tid 1 to smithi13667081-4 2024-09-12T23:50:00.308 INFO:tasks.rados.rados.0.smithi136.stdout:3423: finishing write tid 2 to smithi13667081-4 2024-09-12T23:50:00.308 INFO:tasks.rados.rados.0.smithi136.stdout:3423: finishing write tid 3 to smithi13667081-4 2024-09-12T23:50:00.308 INFO:tasks.rados.rados.0.smithi136.stdout:3423: finishing write tid 4 to smithi13667081-4 2024-09-12T23:50:00.308 INFO:tasks.rados.rados.0.smithi136.stdout:3423: finishing write tid 5 to smithi13667081-4 2024-09-12T23:50:00.308 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 4 v 1650 (ObjNum 1082 snap 326 seq_num 1082) dirty exists 2024-09-12T23:50:00.308 INFO:tasks.rados.rados.0.smithi136.stdout:3423: left oid 4 (ObjNum 1082 snap 326 seq_num 1082) 2024-09-12T23:50:00.308 INFO:tasks.rados.rados.0.smithi136.stdout:3424: finishing copy_from racing read to smithi13667081-29 2024-09-12T23:50:00.308 INFO:tasks.rados.rados.0.smithi136.stdout:3424: finishing copy_from to smithi13667081-29 2024-09-12T23:50:00.308 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 29 v 2659 (ObjNum 1047 snap 321 seq_num 1047) dirty exists 2024-09-12T23:50:00.308 INFO:tasks.rados.rados.0.smithi136.stdout:3419: done (5 left) 2024-09-12T23:50:00.308 INFO:tasks.rados.rados.0.smithi136.stdout:3420: done (4 left) 2024-09-12T23:50:00.308 INFO:tasks.rados.rados.0.smithi136.stdout:3421: done (3 left) 2024-09-12T23:50:00.309 INFO:tasks.rados.rados.0.smithi136.stdout:3422: done (2 left) 2024-09-12T23:50:00.309 INFO:tasks.rados.rados.0.smithi136.stdout:3423: done (1 left) 2024-09-12T23:50:00.309 INFO:tasks.rados.rados.0.smithi136.stdout:3424: done (0 left) 2024-09-12T23:50:00.309 INFO:tasks.rados.rados.0.smithi136.stdout:3425: snap_remove snap 322 2024-09-12T23:50:00.345 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:49:59 smithi181 bash[67587]: cluster 2024-09-12T23:49:58.838165+0000 mgr.y (mgr.24661) 375 : cluster [DBG] pgmap v679: 121 pgs: 121 active+clean; 351 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 6.8 MiB/s wr, 47 op/s 2024-09-12T23:50:00.910 INFO:tasks.rados.rados.0.smithi136.stdout:3425: done (0 left) 2024-09-12T23:50:00.910 INFO:tasks.rados.rados.0.smithi136.stdout:3426: read oid 24 snap 325 2024-09-12T23:50:00.910 INFO:tasks.rados.rados.0.smithi136.stdout:3426: expect (ObjNum 997 snap 309 seq_num 997) 2024-09-12T23:50:00.910 INFO:tasks.rados.rados.0.smithi136.stdout:3427: read oid 34 snap -1 2024-09-12T23:50:00.910 INFO:tasks.rados.rados.0.smithi136.stdout:3427: expect (ObjNum 838 snap 267 seq_num 838) 2024-09-12T23:50:00.910 INFO:tasks.rados.rados.0.smithi136.stdout:3428: snap_remove snap 321 2024-09-12T23:50:01.197 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:00 smithi136 bash[75164]: cluster 2024-09-12T23:50:00.000119+0000 mon.a (mon.0) 1204 : cluster [INF] overall HEALTH_OK 2024-09-12T23:50:01.197 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:00 smithi136 bash[76371]: cluster 2024-09-12T23:50:00.000119+0000 mon.a (mon.0) 1204 : cluster [INF] overall HEALTH_OK 2024-09-12T23:50:01.345 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:00 smithi181 bash[67587]: cluster 2024-09-12T23:50:00.000119+0000 mon.a (mon.0) 1204 : cluster [INF] overall HEALTH_OK 2024-09-12T23:50:01.475 INFO:tasks.rados.rados.0.smithi136.stdout:3428: done (2 left) 2024-09-12T23:50:01.475 INFO:tasks.rados.rados.0.smithi136.stdout:3429: rmattr oid 49 current snap is 326 2024-09-12T23:50:01.475 INFO:tasks.rados.rados.0.smithi136.stdout:3429: done (2 left) 2024-09-12T23:50:01.475 INFO:tasks.rados.rados.0.smithi136.stdout:3430: snap_create 2024-09-12T23:50:01.475 INFO:tasks.rados.rados.0.smithi136.stdout:3427: expect (ObjNum 838 snap 267 seq_num 838) 2024-09-12T23:50:01.535 INFO:tasks.rados.rados.0.smithi136.stdout:3426: expect (ObjNum 997 snap 309 seq_num 997) 2024-09-12T23:50:02.213 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:01 smithi136 bash[75164]: cluster 2024-09-12T23:50:00.839157+0000 mgr.y (mgr.24661) 376 : cluster [DBG] pgmap v680: 121 pgs: 121 active+clean; 352 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 5.0 MiB/s wr, 35 op/s 2024-09-12T23:50:02.213 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:01 smithi136 bash[75164]: cluster 2024-09-12T23:50:00.910266+0000 mon.a (mon.0) 1205 : cluster [DBG] osdmap e733: 8 total, 8 up, 8 in 2024-09-12T23:50:02.213 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:01 smithi136 bash[75164]: cluster 2024-09-12T23:50:01.476649+0000 mon.a (mon.0) 1206 : cluster [DBG] osdmap e734: 8 total, 8 up, 8 in 2024-09-12T23:50:02.214 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:01 smithi136 bash[76371]: cluster 2024-09-12T23:50:00.839157+0000 mgr.y (mgr.24661) 376 : cluster [DBG] pgmap v680: 121 pgs: 121 active+clean; 352 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 5.0 MiB/s wr, 35 op/s 2024-09-12T23:50:02.214 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:01 smithi136 bash[76371]: cluster 2024-09-12T23:50:00.910266+0000 mon.a (mon.0) 1205 : cluster [DBG] osdmap e733: 8 total, 8 up, 8 in 2024-09-12T23:50:02.214 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:01 smithi136 bash[76371]: cluster 2024-09-12T23:50:01.476649+0000 mon.a (mon.0) 1206 : cluster [DBG] osdmap e734: 8 total, 8 up, 8 in 2024-09-12T23:50:02.311 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:01 smithi181 bash[67587]: cluster 2024-09-12T23:50:00.839157+0000 mgr.y (mgr.24661) 376 : cluster [DBG] pgmap v680: 121 pgs: 121 active+clean; 352 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 5.0 MiB/s wr, 35 op/s 2024-09-12T23:50:02.311 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:01 smithi181 bash[67587]: cluster 2024-09-12T23:50:00.910266+0000 mon.a (mon.0) 1205 : cluster [DBG] osdmap e733: 8 total, 8 up, 8 in 2024-09-12T23:50:02.311 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:01 smithi181 bash[67587]: cluster 2024-09-12T23:50:01.476649+0000 mon.a (mon.0) 1206 : cluster [DBG] osdmap e734: 8 total, 8 up, 8 in 2024-09-12T23:50:02.477 INFO:tasks.rados.rados.0.smithi136.stdout:3426: done (2 left) 2024-09-12T23:50:02.477 INFO:tasks.rados.rados.0.smithi136.stdout:3427: done (1 left) 2024-09-12T23:50:02.477 INFO:tasks.rados.rados.0.smithi136.stdout:3430: done (0 left) 2024-09-12T23:50:02.477 INFO:tasks.rados.rados.0.smithi136.stdout:3431: delete oid 43 current snap is 327 2024-09-12T23:50:02.487 INFO:tasks.rados.rados.0.smithi136.stdout:3431: done (0 left) 2024-09-12T23:50:02.487 INFO:tasks.rados.rados.0.smithi136.stdout:3432: snap_create 2024-09-12T23:50:02.595 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:50:02 smithi181 bash[95505]: ts=2024-09-12T23:50:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:50:03.493 INFO:tasks.rados.rados.0.smithi136.stdout:3432: done (0 left) 2024-09-12T23:50:03.493 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 33 current snap is 328 2024-09-12T23:50:03.493 INFO:tasks.rados.rados.0.smithi136.stdout:3433: seq_num 1084 ranges {0=557056,557056=475136,1032192=327680} 2024-09-12T23:50:03.496 INFO:tasks.rados.rados.0.smithi136.stdout:3433: writing smithi13667081-33 from 0 to 557056 tid 1 2024-09-12T23:50:03.499 INFO:tasks.rados.rados.0.smithi136.stdout:3433: writing smithi13667081-33 from 557056 to 1032192 tid 2 2024-09-12T23:50:03.501 INFO:tasks.rados.rados.0.smithi136.stdout:3433: writing smithi13667081-33 from 1032192 to 1359872 tid 3 2024-09-12T23:50:03.501 INFO:tasks.rados.rados.0.smithi136.stdout:3434: copy_from oid 10 from oid 40 current snap is 328 2024-09-12T23:50:03.501 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 18 current snap is 328 2024-09-12T23:50:03.501 INFO:tasks.rados.rados.0.smithi136.stdout:3435: seq_num 1085 ranges {0=581632,581632=761856,1343488=40960} 2024-09-12T23:50:03.504 INFO:tasks.rados.rados.0.smithi136.stdout:3435: writing smithi13667081-18 from 0 to 581632 tid 1 2024-09-12T23:50:03.518 INFO:tasks.rados.rados.0.smithi136.stdout:3435: writing smithi13667081-18 from 581632 to 1343488 tid 2 2024-09-12T23:50:03.519 INFO:tasks.rados.rados.0.smithi136.stdout:3435: writing smithi13667081-18 from 1343488 to 1384448 tid 3 2024-09-12T23:50:03.520 INFO:tasks.rados.rados.0.smithi136.stdout:3436: setattr oid 34 current snap is 328 2024-09-12T23:50:03.520 INFO:tasks.rados.rados.0.smithi136.stdout:3435: finishing write tid 1 to smithi13667081-18 2024-09-12T23:50:03.520 INFO:tasks.rados.rados.0.smithi136.stdout:3433: finishing write tid 1 to smithi13667081-33 2024-09-12T23:50:03.520 INFO:tasks.rados.rados.0.smithi136.stdout:3433: finishing write tid 2 to smithi13667081-33 2024-09-12T23:50:03.520 INFO:tasks.rados.rados.0.smithi136.stdout:3433: finishing write tid 3 to smithi13667081-33 2024-09-12T23:50:03.520 INFO:tasks.rados.rados.0.smithi136.stdout:3433: finishing write tid 4 to smithi13667081-33 2024-09-12T23:50:03.521 INFO:tasks.rados.rados.0.smithi136.stdout:3433: finishing write tid 5 to smithi13667081-33 2024-09-12T23:50:03.521 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 33 v 2478 (ObjNum 1084 snap 328 seq_num 1084) dirty exists 2024-09-12T23:50:03.521 INFO:tasks.rados.rados.0.smithi136.stdout:3433: left oid 33 (ObjNum 1084 snap 328 seq_num 1084) 2024-09-12T23:50:03.521 INFO:tasks.rados.rados.0.smithi136.stdout:3433: done (3 left) 2024-09-12T23:50:03.521 INFO:tasks.rados.rados.0.smithi136.stdout:3437: snap_remove snap 315 2024-09-12T23:50:03.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:03 smithi136 bash[75164]: cluster 2024-09-12T23:50:02.477372+0000 mon.a (mon.0) 1207 : cluster [DBG] osdmap e735: 8 total, 8 up, 8 in 2024-09-12T23:50:03.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:03 smithi136 bash[75164]: cluster 2024-09-12T23:50:02.839976+0000 mgr.y (mgr.24661) 377 : cluster [DBG] pgmap v684: 121 pgs: 121 active+clean; 358 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 5.8 MiB/s wr, 24 op/s 2024-09-12T23:50:03.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:03 smithi136 bash[75164]: audit 2024-09-12T23:50:03.030422+0000 mon.a (mon.0) 1208 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:50:03.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:03 smithi136 bash[75164]: audit 2024-09-12T23:50:03.039589+0000 mon.a (mon.0) 1209 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:50:03.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:03 smithi136 bash[75164]: audit 2024-09-12T23:50:03.041224+0000 mon.a (mon.0) 1210 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:50:03.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:03 smithi136 bash[75164]: audit 2024-09-12T23:50:03.042648+0000 mon.a (mon.0) 1211 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:50:03.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:03 smithi136 bash[75164]: audit 2024-09-12T23:50:03.049906+0000 mon.a (mon.0) 1212 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:50:03.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:03 smithi136 bash[75164]: audit 2024-09-12T23:50:03.124195+0000 mon.a (mon.0) 1213 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-12T23:50:03.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:03 smithi136 bash[75164]: audit 2024-09-12T23:50:03.209471+0000 mon.a (mon.0) 1214 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:50:03.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:03 smithi136 bash[75164]: audit 2024-09-12T23:50:03.214130+0000 mon.a (mon.0) 1215 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:03.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:03 smithi136 bash[75164]: audit 2024-09-12T23:50:03.216780+0000 mon.a (mon.0) 1216 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:03.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:03 smithi136 bash[75164]: audit 2024-09-12T23:50:03.219034+0000 mon.a (mon.0) 1217 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:03.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:03 smithi136 bash[75164]: audit 2024-09-12T23:50:03.222346+0000 mon.a (mon.0) 1218 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:03.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:03 smithi136 bash[75164]: audit 2024-09-12T23:50:03.226238+0000 mon.a (mon.0) 1219 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:03.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:03 smithi136 bash[75164]: audit 2024-09-12T23:50:03.228434+0000 mon.a (mon.0) 1220 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:03.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:03 smithi136 bash[75164]: audit 2024-09-12T23:50:03.230361+0000 mon.a (mon.0) 1221 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:03.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:03 smithi136 bash[75164]: audit 2024-09-12T23:50:03.232192+0000 mon.a (mon.0) 1222 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:03.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:03 smithi136 bash[75164]: audit 2024-09-12T23:50:03.234536+0000 mon.a (mon.0) 1223 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:03.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:03 smithi136 bash[75164]: audit 2024-09-12T23:50:03.236979+0000 mon.a (mon.0) 1224 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:03.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:03 smithi136 bash[75164]: audit 2024-09-12T23:50:03.239488+0000 mon.a (mon.0) 1225 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:03.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:03 smithi136 bash[76371]: cluster 2024-09-12T23:50:02.477372+0000 mon.a (mon.0) 1207 : cluster [DBG] osdmap e735: 8 total, 8 up, 8 in 2024-09-12T23:50:03.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:03 smithi136 bash[76371]: cluster 2024-09-12T23:50:02.839976+0000 mgr.y (mgr.24661) 377 : cluster [DBG] pgmap v684: 121 pgs: 121 active+clean; 358 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 5.8 MiB/s wr, 24 op/s 2024-09-12T23:50:03.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:03 smithi136 bash[76371]: audit 2024-09-12T23:50:03.030422+0000 mon.a (mon.0) 1208 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:50:03.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:03 smithi136 bash[76371]: audit 2024-09-12T23:50:03.039589+0000 mon.a (mon.0) 1209 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:50:03.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:03 smithi136 bash[76371]: audit 2024-09-12T23:50:03.041224+0000 mon.a (mon.0) 1210 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:50:03.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:03 smithi136 bash[76371]: audit 2024-09-12T23:50:03.042648+0000 mon.a (mon.0) 1211 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:50:03.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:03 smithi136 bash[76371]: audit 2024-09-12T23:50:03.049906+0000 mon.a (mon.0) 1212 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:50:03.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:03 smithi136 bash[76371]: audit 2024-09-12T23:50:03.124195+0000 mon.a (mon.0) 1213 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-12T23:50:03.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:03 smithi136 bash[76371]: audit 2024-09-12T23:50:03.209471+0000 mon.a (mon.0) 1214 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:50:03.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:03 smithi136 bash[76371]: audit 2024-09-12T23:50:03.214130+0000 mon.a (mon.0) 1215 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:03.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:03 smithi136 bash[76371]: audit 2024-09-12T23:50:03.216780+0000 mon.a (mon.0) 1216 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:03.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:03 smithi136 bash[76371]: audit 2024-09-12T23:50:03.219034+0000 mon.a (mon.0) 1217 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:03.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:03 smithi136 bash[76371]: audit 2024-09-12T23:50:03.222346+0000 mon.a (mon.0) 1218 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:03.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:03 smithi136 bash[76371]: audit 2024-09-12T23:50:03.226238+0000 mon.a (mon.0) 1219 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:03.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:03 smithi136 bash[76371]: audit 2024-09-12T23:50:03.228434+0000 mon.a (mon.0) 1220 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:03.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:03 smithi136 bash[76371]: audit 2024-09-12T23:50:03.230361+0000 mon.a (mon.0) 1221 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:03.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:03 smithi136 bash[76371]: audit 2024-09-12T23:50:03.232192+0000 mon.a (mon.0) 1222 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:03.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:03 smithi136 bash[76371]: audit 2024-09-12T23:50:03.234536+0000 mon.a (mon.0) 1223 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:03.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:03 smithi136 bash[76371]: audit 2024-09-12T23:50:03.236979+0000 mon.a (mon.0) 1224 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:03.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:03 smithi136 bash[76371]: audit 2024-09-12T23:50:03.239488+0000 mon.a (mon.0) 1225 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:03.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:03 smithi136 bash[75164]: audit 2024-09-12T23:50:03.244078+0000 mon.a (mon.0) 1226 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:03.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:03 smithi136 bash[75164]: audit 2024-09-12T23:50:03.247461+0000 mon.a (mon.0) 1227 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:03.762 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:03 smithi136 bash[75164]: audit 2024-09-12T23:50:03.250619+0000 mon.a (mon.0) 1228 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:03.762 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:03 smithi136 bash[75164]: audit 2024-09-12T23:50:03.253766+0000 mon.a (mon.0) 1229 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:03.762 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:03 smithi136 bash[76371]: audit 2024-09-12T23:50:03.244078+0000 mon.a (mon.0) 1226 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:03.762 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:03 smithi136 bash[76371]: audit 2024-09-12T23:50:03.247461+0000 mon.a (mon.0) 1227 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:03.762 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:03 smithi136 bash[76371]: audit 2024-09-12T23:50:03.250619+0000 mon.a (mon.0) 1228 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:03.762 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:03 smithi136 bash[76371]: audit 2024-09-12T23:50:03.253766+0000 mon.a (mon.0) 1229 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:03.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:03 smithi181 bash[67587]: cluster 2024-09-12T23:50:02.477372+0000 mon.a (mon.0) 1207 : cluster [DBG] osdmap e735: 8 total, 8 up, 8 in 2024-09-12T23:50:03.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:03 smithi181 bash[67587]: cluster 2024-09-12T23:50:02.839976+0000 mgr.y (mgr.24661) 377 : cluster [DBG] pgmap v684: 121 pgs: 121 active+clean; 358 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 5.8 MiB/s wr, 24 op/s 2024-09-12T23:50:03.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:03 smithi181 bash[67587]: audit 2024-09-12T23:50:03.030422+0000 mon.a (mon.0) 1208 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:50:03.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:03 smithi181 bash[67587]: audit 2024-09-12T23:50:03.039589+0000 mon.a (mon.0) 1209 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:50:03.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:03 smithi181 bash[67587]: audit 2024-09-12T23:50:03.041224+0000 mon.a (mon.0) 1210 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:50:03.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:03 smithi181 bash[67587]: audit 2024-09-12T23:50:03.042648+0000 mon.a (mon.0) 1211 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:50:03.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:03 smithi181 bash[67587]: audit 2024-09-12T23:50:03.049906+0000 mon.a (mon.0) 1212 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:50:03.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:03 smithi181 bash[67587]: audit 2024-09-12T23:50:03.124195+0000 mon.a (mon.0) 1213 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-12T23:50:03.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:03 smithi181 bash[67587]: audit 2024-09-12T23:50:03.209471+0000 mon.a (mon.0) 1214 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:50:03.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:03 smithi181 bash[67587]: audit 2024-09-12T23:50:03.214130+0000 mon.a (mon.0) 1215 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:03.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:03 smithi181 bash[67587]: audit 2024-09-12T23:50:03.216780+0000 mon.a (mon.0) 1216 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:03.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:03 smithi181 bash[67587]: audit 2024-09-12T23:50:03.219034+0000 mon.a (mon.0) 1217 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:03.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:03 smithi181 bash[67587]: audit 2024-09-12T23:50:03.222346+0000 mon.a (mon.0) 1218 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:03.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:03 smithi181 bash[67587]: audit 2024-09-12T23:50:03.226238+0000 mon.a (mon.0) 1219 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:03.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:03 smithi181 bash[67587]: audit 2024-09-12T23:50:03.228434+0000 mon.a (mon.0) 1220 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:03.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:03 smithi181 bash[67587]: audit 2024-09-12T23:50:03.230361+0000 mon.a (mon.0) 1221 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:03.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:03 smithi181 bash[67587]: audit 2024-09-12T23:50:03.232192+0000 mon.a (mon.0) 1222 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:03.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:03 smithi181 bash[67587]: audit 2024-09-12T23:50:03.234536+0000 mon.a (mon.0) 1223 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:03.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:03 smithi181 bash[67587]: audit 2024-09-12T23:50:03.236979+0000 mon.a (mon.0) 1224 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:03.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:03 smithi181 bash[67587]: audit 2024-09-12T23:50:03.239488+0000 mon.a (mon.0) 1225 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:03.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:03 smithi181 bash[67587]: audit 2024-09-12T23:50:03.244078+0000 mon.a (mon.0) 1226 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:03.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:03 smithi181 bash[67587]: audit 2024-09-12T23:50:03.247461+0000 mon.a (mon.0) 1227 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:03.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:03 smithi181 bash[67587]: audit 2024-09-12T23:50:03.250619+0000 mon.a (mon.0) 1228 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:03.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:03 smithi181 bash[67587]: audit 2024-09-12T23:50:03.253766+0000 mon.a (mon.0) 1229 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:04.498 INFO:tasks.rados.rados.0.smithi136.stdout:3437: done (3 left) 2024-09-12T23:50:04.498 INFO:tasks.rados.rados.0.smithi136.stdout:3438: snap_remove snap 303 2024-09-12T23:50:04.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:04 smithi136 bash[75164]: audit 2024-09-12T23:50:03.124893+0000 mgr.y (mgr.24661) 378 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-12T23:50:04.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:04 smithi136 bash[75164]: cluster 2024-09-12T23:50:03.493800+0000 mon.a (mon.0) 1230 : cluster [DBG] osdmap e736: 8 total, 8 up, 8 in 2024-09-12T23:50:04.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:04 smithi136 bash[76371]: audit 2024-09-12T23:50:03.124893+0000 mgr.y (mgr.24661) 378 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-12T23:50:04.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:04 smithi136 bash[76371]: cluster 2024-09-12T23:50:03.493800+0000 mon.a (mon.0) 1230 : cluster [DBG] osdmap e736: 8 total, 8 up, 8 in 2024-09-12T23:50:04.832 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:04 smithi181 bash[67587]: audit 2024-09-12T23:50:03.124893+0000 mgr.y (mgr.24661) 378 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-12T23:50:04.832 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:04 smithi181 bash[67587]: cluster 2024-09-12T23:50:03.493800+0000 mon.a (mon.0) 1230 : cluster [DBG] osdmap e736: 8 total, 8 up, 8 in 2024-09-12T23:50:05.502 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 34 v 2479 (ObjNum 838 snap 267 seq_num 838) dirty exists 2024-09-12T23:50:05.502 INFO:tasks.rados.rados.0.smithi136.stdout:3435: finishing write tid 2 to smithi13667081-18 2024-09-12T23:50:05.502 INFO:tasks.rados.rados.0.smithi136.stdout:3435: finishing write tid 3 to smithi13667081-18 2024-09-12T23:50:05.502 INFO:tasks.rados.rados.0.smithi136.stdout:3435: finishing write tid 4 to smithi13667081-18 2024-09-12T23:50:05.502 INFO:tasks.rados.rados.0.smithi136.stdout:3435: finishing write tid 5 to smithi13667081-18 2024-09-12T23:50:05.503 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 18 v 1640 (ObjNum 1085 snap 328 seq_num 1085) dirty exists 2024-09-12T23:50:05.503 INFO:tasks.rados.rados.0.smithi136.stdout:3435: left oid 18 (ObjNum 1085 snap 328 seq_num 1085) 2024-09-12T23:50:05.503 INFO:tasks.rados.rados.0.smithi136.stdout:3434: finishing copy_from racing read to smithi13667081-10 2024-09-12T23:50:05.503 INFO:tasks.rados.rados.0.smithi136.stdout:3434: finishing copy_from to smithi13667081-10 2024-09-12T23:50:05.503 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 10 v 2581 (ObjNum 1048 snap 321 seq_num 1048) dirty exists 2024-09-12T23:50:05.503 INFO:tasks.rados.rados.0.smithi136.stdout:3434: done (3 left) 2024-09-12T23:50:05.503 INFO:tasks.rados.rados.0.smithi136.stdout:3435: done (2 left) 2024-09-12T23:50:05.503 INFO:tasks.rados.rados.0.smithi136.stdout:3436: done (1 left) 2024-09-12T23:50:05.503 INFO:tasks.rados.rados.0.smithi136.stdout:3438: done (0 left) 2024-09-12T23:50:05.503 INFO:tasks.rados.rados.0.smithi136.stdout:3439: snap_remove snap 325 2024-09-12T23:50:05.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:05 smithi136 bash[75164]: cluster 2024-09-12T23:50:04.498188+0000 mon.a (mon.0) 1231 : cluster [DBG] osdmap e737: 8 total, 8 up, 8 in 2024-09-12T23:50:05.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:05 smithi136 bash[75164]: cluster 2024-09-12T23:50:04.840770+0000 mgr.y (mgr.24661) 379 : cluster [DBG] pgmap v687: 121 pgs: 121 active+clean; 362 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 3.8 MiB/s wr, 14 op/s 2024-09-12T23:50:05.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:05 smithi136 bash[76371]: cluster 2024-09-12T23:50:04.498188+0000 mon.a (mon.0) 1231 : cluster [DBG] osdmap e737: 8 total, 8 up, 8 in 2024-09-12T23:50:05.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:05 smithi136 bash[76371]: cluster 2024-09-12T23:50:04.840770+0000 mgr.y (mgr.24661) 379 : cluster [DBG] pgmap v687: 121 pgs: 121 active+clean; 362 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 3.8 MiB/s wr, 14 op/s 2024-09-12T23:50:05.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:05 smithi181 bash[67587]: cluster 2024-09-12T23:50:04.498188+0000 mon.a (mon.0) 1231 : cluster [DBG] osdmap e737: 8 total, 8 up, 8 in 2024-09-12T23:50:05.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:05 smithi181 bash[67587]: cluster 2024-09-12T23:50:04.840770+0000 mgr.y (mgr.24661) 379 : cluster [DBG] pgmap v687: 121 pgs: 121 active+clean; 362 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 3.8 MiB/s wr, 14 op/s 2024-09-12T23:50:06.477 INFO:tasks.rados.rados.0.smithi136.stdout:3439: done (0 left) 2024-09-12T23:50:06.478 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 29 current snap is 328 2024-09-12T23:50:06.478 INFO:tasks.rados.rados.0.smithi136.stdout:3440: seq_num 1086 ranges {745472=598016,1343488=647168,1990656=155648} 2024-09-12T23:50:06.489 INFO:tasks.rados.rados.0.smithi136.stdout:3440: writing smithi13667081-29 from 745472 to 1343488 tid 1 2024-09-12T23:50:06.495 INFO:tasks.rados.rados.0.smithi136.stdout:3440: writing smithi13667081-29 from 1343488 to 1990656 tid 2 2024-09-12T23:50:06.497 INFO:tasks.rados.rados.0.smithi136.stdout:3440: writing smithi13667081-29 from 1990656 to 2146304 tid 3 2024-09-12T23:50:06.497 INFO:tasks.rados.rados.0.smithi136.stdout:3441: snap_create 2024-09-12T23:50:06.508 INFO:tasks.rados.rados.0.smithi136.stdout:3440: finishing write tid 1 to smithi13667081-29 2024-09-12T23:50:06.508 INFO:tasks.rados.rados.0.smithi136.stdout:3440: finishing write tid 2 to smithi13667081-29 2024-09-12T23:50:06.513 INFO:tasks.rados.rados.0.smithi136.stdout:3440: finishing write tid 3 to smithi13667081-29 2024-09-12T23:50:06.513 INFO:tasks.rados.rados.0.smithi136.stdout:3440: finishing write tid 4 to smithi13667081-29 2024-09-12T23:50:06.515 INFO:tasks.rados.rados.0.smithi136.stdout:3440: finishing write tid 5 to smithi13667081-29 2024-09-12T23:50:06.515 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 29 v 2663 (ObjNum 1086 snap 328 seq_num 1086) dirty exists 2024-09-12T23:50:06.515 INFO:tasks.rados.rados.0.smithi136.stdout:3440: left oid 29 (ObjNum 1086 snap 328 seq_num 1086) 2024-09-12T23:50:06.765 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:06 smithi181 bash[67587]: cephadm 2024-09-12T23:50:05.289760+0000 mgr.y (mgr.24661) 380 : cephadm [INF] Upgrade: Updating grafana.a 2024-09-12T23:50:06.765 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:06 smithi181 bash[67587]: cephadm 2024-09-12T23:50:05.385894+0000 mgr.y (mgr.24661) 381 : cephadm [INF] Deploying daemon grafana.a on smithi181 2024-09-12T23:50:06.765 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:06 smithi181 bash[67587]: cluster 2024-09-12T23:50:05.502606+0000 mon.a (mon.0) 1232 : cluster [DBG] osdmap e738: 8 total, 8 up, 8 in 2024-09-12T23:50:06.766 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:06 smithi181 bash[67587]: cluster 2024-09-12T23:50:06.478742+0000 mon.a (mon.0) 1233 : cluster [DBG] osdmap e739: 8 total, 8 up, 8 in 2024-09-12T23:50:07.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:06 smithi136 bash[76371]: cephadm 2024-09-12T23:50:05.289760+0000 mgr.y (mgr.24661) 380 : cephadm [INF] Upgrade: Updating grafana.a 2024-09-12T23:50:07.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:06 smithi136 bash[76371]: cephadm 2024-09-12T23:50:05.385894+0000 mgr.y (mgr.24661) 381 : cephadm [INF] Deploying daemon grafana.a on smithi181 2024-09-12T23:50:07.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:06 smithi136 bash[76371]: cluster 2024-09-12T23:50:05.502606+0000 mon.a (mon.0) 1232 : cluster [DBG] osdmap e738: 8 total, 8 up, 8 in 2024-09-12T23:50:07.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:06 smithi136 bash[76371]: cluster 2024-09-12T23:50:06.478742+0000 mon.a (mon.0) 1233 : cluster [DBG] osdmap e739: 8 total, 8 up, 8 in 2024-09-12T23:50:07.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:06 smithi136 bash[75164]: cephadm 2024-09-12T23:50:05.289760+0000 mgr.y (mgr.24661) 380 : cephadm [INF] Upgrade: Updating grafana.a 2024-09-12T23:50:07.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:06 smithi136 bash[75164]: cephadm 2024-09-12T23:50:05.385894+0000 mgr.y (mgr.24661) 381 : cephadm [INF] Deploying daemon grafana.a on smithi181 2024-09-12T23:50:07.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:06 smithi136 bash[75164]: cluster 2024-09-12T23:50:05.502606+0000 mon.a (mon.0) 1232 : cluster [DBG] osdmap e738: 8 total, 8 up, 8 in 2024-09-12T23:50:07.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:06 smithi136 bash[75164]: cluster 2024-09-12T23:50:06.478742+0000 mon.a (mon.0) 1233 : cluster [DBG] osdmap e739: 8 total, 8 up, 8 in 2024-09-12T23:50:07.480 INFO:tasks.rados.rados.0.smithi136.stdout:3440: done (1 left) 2024-09-12T23:50:07.480 INFO:tasks.rados.rados.0.smithi136.stdout:3441: done (0 left) 2024-09-12T23:50:07.480 INFO:tasks.rados.rados.0.smithi136.stdout:3442: rmattr oid 43 current snap is 329 2024-09-12T23:50:07.480 INFO:tasks.rados.rados.0.smithi136.stdout:3442: done (0 left) 2024-09-12T23:50:07.480 INFO:tasks.rados.rados.0.smithi136.stdout:3443: read oid 26 snap 327 2024-09-12T23:50:07.480 INFO:tasks.rados.rados.0.smithi136.stdout:3443: expect (ObjNum 987 snap 303 seq_num 987) 2024-09-12T23:50:07.480 INFO:tasks.rados.rados.0.smithi136.stdout:3444: delete oid 33 current snap is 329 2024-09-12T23:50:07.493 INFO:tasks.rados.rados.0.smithi136.stdout:3444: done (1 left) 2024-09-12T23:50:07.493 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 23 current snap is 329 2024-09-12T23:50:07.493 INFO:tasks.rados.rados.0.smithi136.stdout:3445: seq_num 1088 ranges {909312=630784,1540096=466944,2007040=335872} 2024-09-12T23:50:07.519 INFO:tasks.rados.rados.0.smithi136.stdout:3445: writing smithi13667081-23 from 909312 to 1540096 tid 1 2024-09-12T23:50:07.528 INFO:tasks.rados.rados.0.smithi136.stdout:3445: writing smithi13667081-23 from 1540096 to 2007040 tid 2 2024-09-12T23:50:07.534 INFO:tasks.rados.rados.0.smithi136.stdout:3445: writing smithi13667081-23 from 2007040 to 2342912 tid 3 2024-09-12T23:50:07.534 INFO:tasks.rados.rados.0.smithi136.stdout:3446: copy_from oid 8 from oid 30 current snap is 329 2024-09-12T23:50:07.534 INFO:tasks.rados.rados.0.smithi136.stdout:3443: expect (ObjNum 987 snap 303 seq_num 987) 2024-09-12T23:50:07.788 INFO:tasks.rados.rados.0.smithi136.stdout:3445: finishing write tid 1 to smithi13667081-23 2024-09-12T23:50:07.788 INFO:tasks.rados.rados.0.smithi136.stdout:3446: finishing copy_from racing read to smithi13667081-8 2024-09-12T23:50:07.788 INFO:tasks.rados.rados.0.smithi136.stdout:3446: finishing copy_from to smithi13667081-8 2024-09-12T23:50:07.788 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 8 v 1506 (ObjNum 65 snap 0 seq_num 1633245791) dirty exists 2024-09-12T23:50:07.788 INFO:tasks.rados.rados.0.smithi136.stdout:3445: finishing write tid 2 to smithi13667081-23 2024-09-12T23:50:07.788 INFO:tasks.rados.rados.0.smithi136.stdout:3445: finishing write tid 3 to smithi13667081-23 2024-09-12T23:50:07.788 INFO:tasks.rados.rados.0.smithi136.stdout:3445: finishing write tid 4 to smithi13667081-23 2024-09-12T23:50:07.788 INFO:tasks.rados.rados.0.smithi136.stdout:3445: finishing write tid 5 to smithi13667081-23 2024-09-12T23:50:07.789 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 23 v 2702 (ObjNum 1088 snap 329 seq_num 1088) dirty exists 2024-09-12T23:50:07.789 INFO:tasks.rados.rados.0.smithi136.stdout:3445: left oid 23 (ObjNum 1088 snap 329 seq_num 1088) 2024-09-12T23:50:07.789 INFO:tasks.rados.rados.0.smithi136.stdout:3443: done (2 left) 2024-09-12T23:50:07.789 INFO:tasks.rados.rados.0.smithi136.stdout:3445: done (1 left) 2024-09-12T23:50:07.789 INFO:tasks.rados.rados.0.smithi136.stdout:3446: done (0 left) 2024-09-12T23:50:07.789 INFO:tasks.rados.rados.0.smithi136.stdout:3447: delete oid 41 current snap is 329 2024-09-12T23:50:07.792 INFO:tasks.rados.rados.0.smithi136.stdout:3447: done (0 left) 2024-09-12T23:50:07.792 INFO:tasks.rados.rados.0.smithi136.stdout:3448: read oid 12 snap 327 2024-09-12T23:50:07.792 INFO:tasks.rados.rados.0.smithi136.stdout:3448: expect (ObjNum 988 snap 303 seq_num 988) 2024-09-12T23:50:07.792 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 40 current snap is 329 2024-09-12T23:50:07.792 INFO:tasks.rados.rados.0.smithi136.stdout:3449: seq_num 1090 ranges {4268032=663552,4931584=630784,5562368=172032} 2024-09-12T23:50:07.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:07 smithi181 bash[67587]: cluster 2024-09-12T23:50:06.841685+0000 mgr.y (mgr.24661) 382 : cluster [DBG] pgmap v690: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 343 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 1.8 MiB/s wr, 14 op/s 2024-09-12T23:50:07.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:07 smithi181 bash[67587]: cluster 2024-09-12T23:50:07.480999+0000 mon.a (mon.0) 1234 : cluster [DBG] osdmap e740: 8 total, 8 up, 8 in 2024-09-12T23:50:07.864 INFO:tasks.rados.rados.0.smithi136.stdout:3449: writing smithi13667081-40 from 4268032 to 4931584 tid 1 2024-09-12T23:50:07.870 INFO:tasks.rados.rados.0.smithi136.stdout:3449: writing smithi13667081-40 from 4931584 to 5562368 tid 2 2024-09-12T23:50:07.872 INFO:tasks.rados.rados.0.smithi136.stdout:3449: writing smithi13667081-40 from 5562368 to 5734400 tid 3 2024-09-12T23:50:07.872 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 9 current snap is 329 2024-09-12T23:50:07.872 INFO:tasks.rados.rados.0.smithi136.stdout:3450: seq_num 1091 ranges {2031616=679936,2711552=516096,3227648=286720} 2024-09-12T23:50:07.893 INFO:tasks.rados.rados.0.smithi136.stdout:3450: writing smithi13667081-9 from 2031616 to 2711552 tid 1 2024-09-12T23:50:07.897 INFO:tasks.rados.rados.0.smithi136.stdout:3450: writing smithi13667081-9 from 2711552 to 3227648 tid 2 2024-09-12T23:50:07.898 INFO:tasks.rados.rados.0.smithi136.stdout:3450: writing smithi13667081-9 from 3227648 to 3514368 tid 3 2024-09-12T23:50:07.899 INFO:tasks.rados.rados.0.smithi136.stdout:3451: rollback oid 3 current snap is 329 2024-09-12T23:50:07.899 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 3 to 326 2024-09-12T23:50:07.899 INFO:tasks.rados.rados.0.smithi136.stdout:3448: expect (ObjNum 988 snap 303 seq_num 988) 2024-09-12T23:50:08.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:07 smithi136 bash[75164]: cluster 2024-09-12T23:50:06.841685+0000 mgr.y (mgr.24661) 382 : cluster [DBG] pgmap v690: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 343 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 1.8 MiB/s wr, 14 op/s 2024-09-12T23:50:08.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:07 smithi136 bash[75164]: cluster 2024-09-12T23:50:07.480999+0000 mon.a (mon.0) 1234 : cluster [DBG] osdmap e740: 8 total, 8 up, 8 in 2024-09-12T23:50:08.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:07 smithi136 bash[76371]: cluster 2024-09-12T23:50:06.841685+0000 mgr.y (mgr.24661) 382 : cluster [DBG] pgmap v690: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 343 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 1.8 MiB/s wr, 14 op/s 2024-09-12T23:50:08.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:07 smithi136 bash[76371]: cluster 2024-09-12T23:50:07.480999+0000 mon.a (mon.0) 1234 : cluster [DBG] osdmap e740: 8 total, 8 up, 8 in 2024-09-12T23:50:08.276 INFO:tasks.rados.rados.0.smithi136.stdout:3449: finishing write tid 1 to smithi13667081-40 2024-09-12T23:50:08.276 INFO:tasks.rados.rados.0.smithi136.stdout:3449: finishing write tid 2 to smithi13667081-40 2024-09-12T23:50:08.277 INFO:tasks.rados.rados.0.smithi136.stdout:3449: finishing write tid 3 to smithi13667081-40 2024-09-12T23:50:08.277 INFO:tasks.rados.rados.0.smithi136.stdout:3449: finishing write tid 4 to smithi13667081-40 2024-09-12T23:50:08.277 INFO:tasks.rados.rados.0.smithi136.stdout:3449: finishing write tid 5 to smithi13667081-40 2024-09-12T23:50:08.277 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 40 v 2707 (ObjNum 1090 snap 329 seq_num 1090) dirty exists 2024-09-12T23:50:08.277 INFO:tasks.rados.rados.0.smithi136.stdout:3449: left oid 40 (ObjNum 1090 snap 329 seq_num 1090) 2024-09-12T23:50:08.277 INFO:tasks.rados.rados.0.smithi136.stdout:3450: finishing write tid 1 to smithi13667081-9 2024-09-12T23:50:08.277 INFO:tasks.rados.rados.0.smithi136.stdout:3450: finishing write tid 2 to smithi13667081-9 2024-09-12T23:50:08.277 INFO:tasks.rados.rados.0.smithi136.stdout:3448: done (3 left) 2024-09-12T23:50:08.277 INFO:tasks.rados.rados.0.smithi136.stdout:3449: done (2 left) 2024-09-12T23:50:08.277 INFO:tasks.rados.rados.0.smithi136.stdout:3452: copy_from oid 16 from oid 46 current snap is 329 2024-09-12T23:50:08.277 INFO:tasks.rados.rados.0.smithi136.stdout:3451: finishing rollback tid 0 to smithi13667081-3 2024-09-12T23:50:08.277 INFO:tasks.rados.rados.0.smithi136.stdout:3451: finishing rollback tid 1 to smithi13667081-3 2024-09-12T23:50:08.277 INFO:tasks.rados.rados.0.smithi136.stdout:3450: finishing write tid 3 to smithi13667081-9 2024-09-12T23:50:08.278 INFO:tasks.rados.rados.0.smithi136.stdout:3451: finishing rollback tid 2 to smithi13667081-3 2024-09-12T23:50:08.278 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 3 v 2094 (ObjNum 1078 snap 325 seq_num 1078) dirty exists 2024-09-12T23:50:08.278 INFO:tasks.rados.rados.0.smithi136.stdout:3450: finishing write tid 4 to smithi13667081-9 2024-09-12T23:50:08.278 INFO:tasks.rados.rados.0.smithi136.stdout:3450: finishing write tid 5 to smithi13667081-9 2024-09-12T23:50:08.278 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 9 v 2472 (ObjNum 1091 snap 329 seq_num 1091) dirty exists 2024-09-12T23:50:08.278 INFO:tasks.rados.rados.0.smithi136.stdout:3450: left oid 9 (ObjNum 1091 snap 329 seq_num 1091) 2024-09-12T23:50:08.278 INFO:tasks.rados.rados.0.smithi136.stdout:3450: done (2 left) 2024-09-12T23:50:08.278 INFO:tasks.rados.rados.0.smithi136.stdout:3451: done (1 left) 2024-09-12T23:50:08.278 INFO:tasks.rados.rados.0.smithi136.stdout:3453: snap_create 2024-09-12T23:50:08.282 INFO:tasks.rados.rados.0.smithi136.stdout:3452: finishing copy_from racing read to smithi13667081-16 2024-09-12T23:50:08.283 INFO:tasks.rados.rados.0.smithi136.stdout:3452: finishing copy_from to smithi13667081-16 2024-09-12T23:50:08.283 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 16 v 2473 (ObjNum 289 snap 0 seq_num 0) dirty exists 2024-09-12T23:50:08.529 INFO:tasks.rados.rados.0.smithi136.stdout:3452: done (1 left) 2024-09-12T23:50:08.529 INFO:tasks.rados.rados.0.smithi136.stdout:3453: done (0 left) 2024-09-12T23:50:08.529 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 2 current snap is 330 2024-09-12T23:50:08.529 INFO:tasks.rados.rados.0.smithi136.stdout:3454: seq_num 1092 ranges {2457600=696320,3153920=802816} 2024-09-12T23:50:08.554 INFO:tasks.rados.rados.0.smithi136.stdout:3454: writing smithi13667081-2 from 2457600 to 3153920 tid 1 2024-09-12T23:50:08.560 INFO:tasks.rados.rados.0.smithi136.stdout:3454: writing smithi13667081-2 from 3153920 to 3956736 tid 2 2024-09-12T23:50:08.561 INFO:tasks.rados.rados.0.smithi136.stdout:3455: setattr oid 41 current snap is 330 2024-09-12T23:50:08.563 INFO:tasks.rados.rados.0.smithi136.stdout:3456: read oid 44 snap -1 2024-09-12T23:50:08.563 INFO:tasks.rados.rados.0.smithi136.stdout:3456: expect deleted 2024-09-12T23:50:08.564 INFO:tasks.rados.rados.0.smithi136.stdout:3457: read oid 6 snap -1 2024-09-12T23:50:08.564 INFO:tasks.rados.rados.0.smithi136.stdout:3457: expect deleted 2024-09-12T23:50:08.564 INFO:tasks.rados.rados.0.smithi136.stdout:3458: read oid 35 snap -1 2024-09-12T23:50:08.564 INFO:tasks.rados.rados.0.smithi136.stdout:3458: expect (ObjNum 927 snap 290 seq_num 927) 2024-09-12T23:50:08.564 INFO:tasks.rados.rados.0.smithi136.stdout:3459: copy_from oid 50 from oid 16 current snap is 330 2024-09-12T23:50:08.564 INFO:tasks.rados.rados.0.smithi136.stdout:3460: snap_remove snap 309 2024-09-12T23:50:09.538 INFO:tasks.rados.rados.0.smithi136.stdout:3460: done (6 left) 2024-09-12T23:50:09.538 INFO:tasks.rados.rados.0.smithi136.stdout:3461: read oid 34 snap -1 2024-09-12T23:50:09.538 INFO:tasks.rados.rados.0.smithi136.stdout:3461: expect (ObjNum 838 snap 267 seq_num 838) 2024-09-12T23:50:09.538 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 20 current snap is 330 2024-09-12T23:50:09.538 INFO:tasks.rados.rados.0.smithi136.stdout:3462: seq_num 1093 ranges {0=712704,712704=688128,1400832=114688} 2024-09-12T23:50:09.543 INFO:tasks.rados.rados.0.smithi136.stdout:3462: writing smithi13667081-20 from 0 to 712704 tid 1 2024-09-12T23:50:09.548 INFO:tasks.rados.rados.0.smithi136.stdout:3462: writing smithi13667081-20 from 712704 to 1400832 tid 2 2024-09-12T23:50:09.549 INFO:tasks.rados.rados.0.smithi136.stdout:3462: writing smithi13667081-20 from 1400832 to 1515520 tid 3 2024-09-12T23:50:09.549 INFO:tasks.rados.rados.0.smithi136.stdout:3459: finishing copy_from racing read to smithi13667081-50 2024-09-12T23:50:09.549 INFO:tasks.rados.rados.0.smithi136.stdout:3459: finishing copy_from to smithi13667081-50 2024-09-12T23:50:09.550 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 50 v 2473 (ObjNum 289 snap 0 seq_num 1) dirty exists 2024-09-12T23:50:09.550 INFO:tasks.rados.rados.0.smithi136.stdout:3458: expect (ObjNum 927 snap 290 seq_num 927) 2024-09-12T23:50:09.605 INFO:tasks.rados.rados.0.smithi136.stdout:3454: finishing write tid 1 to smithi13667081-2 2024-09-12T23:50:09.605 INFO:tasks.rados.rados.0.smithi136.stdout:3454: finishing write tid 2 to smithi13667081-2 2024-09-12T23:50:09.605 INFO:tasks.rados.rados.0.smithi136.stdout:3454: finishing write tid 3 to smithi13667081-2 2024-09-12T23:50:09.605 INFO:tasks.rados.rados.0.smithi136.stdout:3454: finishing write tid 4 to smithi13667081-2 2024-09-12T23:50:09.605 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 2 v 2484 (ObjNum 1092 snap 330 seq_num 1092) dirty exists 2024-09-12T23:50:09.605 INFO:tasks.rados.rados.0.smithi136.stdout:3454: left oid 2 (ObjNum 1092 snap 330 seq_num 1092) 2024-09-12T23:50:09.605 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 41 v 2485 (ObjNum 33 snap 0 seq_num 4244437424) dirty exists 2024-09-12T23:50:09.605 INFO:tasks.rados.rados.0.smithi136.stdout:3454: done (7 left) 2024-09-12T23:50:09.605 INFO:tasks.rados.rados.0.smithi136.stdout:3455: done (6 left) 2024-09-12T23:50:09.605 INFO:tasks.rados.rados.0.smithi136.stdout:3456: done (5 left) 2024-09-12T23:50:09.606 INFO:tasks.rados.rados.0.smithi136.stdout:3457: done (4 left) 2024-09-12T23:50:09.606 INFO:tasks.rados.rados.0.smithi136.stdout:3458: done (3 left) 2024-09-12T23:50:09.606 INFO:tasks.rados.rados.0.smithi136.stdout:3459: done (2 left) 2024-09-12T23:50:09.607 INFO:tasks.rados.rados.0.smithi136.stdout:3463: read oid 30 snap -1 2024-09-12T23:50:09.607 INFO:tasks.rados.rados.0.smithi136.stdout:3463: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-12T23:50:09.607 INFO:tasks.rados.rados.0.smithi136.stdout:3464: snap_remove snap 326 2024-09-12T23:50:09.657 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:50:09 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:50:09.657 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 12 23:50:09 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:50:09.658 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:50:09 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:50:09.658 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:50:09 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:50:09.658 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:50:09 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:50:09.658 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:50:09 smithi181 systemd[1]: Stopping Ceph grafana.a for da58a47c-715d-11ef-bceb-c7b262605968... 2024-09-12T23:50:09.659 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:50:09 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:50:09.659 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:50:09 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:50:09.659 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 12 23:50:09 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:50:09.660 INFO:journalctl@ceph.mds.b.smithi181.stdout:Sep 12 23:50:09 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:50:09.660 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:09 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:50:09.660 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:09 smithi181 bash[67587]: cluster 2024-09-12T23:50:08.531391+0000 mon.a (mon.0) 1235 : cluster [DBG] osdmap e741: 8 total, 8 up, 8 in 2024-09-12T23:50:09.660 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:09 smithi181 bash[67587]: audit 2024-09-12T23:50:08.752691+0000 mon.a (mon.0) 1236 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:50:09.660 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:09 smithi181 bash[67587]: cluster 2024-09-12T23:50:08.842366+0000 mgr.y (mgr.24661) 383 : cluster [DBG] pgmap v693: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 344 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.3 MiB/s wr, 10 op/s 2024-09-12T23:50:09.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:50:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:50:09] "GET /metrics HTTP/1.1" 200 36440 "" "Prometheus/2.43.0" 2024-09-12T23:50:09.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:09 smithi136 bash[75164]: cluster 2024-09-12T23:50:08.531391+0000 mon.a (mon.0) 1235 : cluster [DBG] osdmap e741: 8 total, 8 up, 8 in 2024-09-12T23:50:09.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:09 smithi136 bash[75164]: audit 2024-09-12T23:50:08.752691+0000 mon.a (mon.0) 1236 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:50:09.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:09 smithi136 bash[75164]: cluster 2024-09-12T23:50:08.842366+0000 mgr.y (mgr.24661) 383 : cluster [DBG] pgmap v693: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 344 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.3 MiB/s wr, 10 op/s 2024-09-12T23:50:09.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:09 smithi136 bash[76371]: cluster 2024-09-12T23:50:08.531391+0000 mon.a (mon.0) 1235 : cluster [DBG] osdmap e741: 8 total, 8 up, 8 in 2024-09-12T23:50:09.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:09 smithi136 bash[76371]: audit 2024-09-12T23:50:08.752691+0000 mon.a (mon.0) 1236 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:50:09.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:09 smithi136 bash[76371]: cluster 2024-09-12T23:50:08.842366+0000 mgr.y (mgr.24661) 383 : cluster [DBG] pgmap v693: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 344 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.3 MiB/s wr, 10 op/s 2024-09-12T23:50:10.545 INFO:tasks.rados.rados.0.smithi136.stdout:3464: done (3 left) 2024-09-12T23:50:10.545 INFO:tasks.rados.rados.0.smithi136.stdout:3465: snap_create 2024-09-12T23:50:10.545 INFO:tasks.rados.rados.0.smithi136.stdout:3462: finishing write tid 1 to smithi13667081-20 2024-09-12T23:50:10.545 INFO:tasks.rados.rados.0.smithi136.stdout:3462: finishing write tid 2 to smithi13667081-20 2024-09-12T23:50:10.545 INFO:tasks.rados.rados.0.smithi136.stdout:3462: finishing write tid 3 to smithi13667081-20 2024-09-12T23:50:10.545 INFO:tasks.rados.rados.0.smithi136.stdout:3462: finishing write tid 4 to smithi13667081-20 2024-09-12T23:50:10.546 INFO:tasks.rados.rados.0.smithi136.stdout:3461: expect (ObjNum 838 snap 267 seq_num 838) 2024-09-12T23:50:10.548 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:50:10 smithi181 bash[98704]: ceph-da58a47c-715d-11ef-bceb-c7b262605968-grafana-a 2024-09-12T23:50:10.548 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:50:10 smithi181 systemd[1]: ceph-da58a47c-715d-11ef-bceb-c7b262605968@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-09-12T23:50:10.548 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:50:10 smithi181 bash[98765]: Error response from daemon: No such container: ceph-da58a47c-715d-11ef-bceb-c7b262605968-grafana-a 2024-09-12T23:50:10.548 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:50:10 smithi181 systemd[1]: ceph-da58a47c-715d-11ef-bceb-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-09-12T23:50:10.548 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:50:10 smithi181 systemd[1]: Stopped Ceph grafana.a for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:50:10.604 INFO:tasks.rados.rados.0.smithi136.stdout:3462: finishing write tid 5 to smithi13667081-20 2024-09-12T23:50:10.604 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 20 v 1649 (ObjNum 1093 snap 330 seq_num 1093) dirty exists 2024-09-12T23:50:10.604 INFO:tasks.rados.rados.0.smithi136.stdout:3462: left oid 20 (ObjNum 1093 snap 330 seq_num 1093) 2024-09-12T23:50:10.604 INFO:tasks.rados.rados.0.smithi136.stdout:3463: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-12T23:50:10.834 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:10 smithi181 bash[67587]: cluster 2024-09-12T23:50:09.540271+0000 mon.a (mon.0) 1237 : cluster [DBG] osdmap e742: 8 total, 8 up, 8 in 2024-09-12T23:50:10.834 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:10 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:50:10.834 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 12 23:50:10 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:50:10.835 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 12 23:50:10 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:50:10.835 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 12 23:50:10 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:50:10.835 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 12 23:50:10 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:50:10.836 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 12 23:50:10 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:50:10.836 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:50:10 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:50:10.836 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:50:10 smithi181 systemd[1]: Started Ceph grafana.a for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-12T23:50:10.836 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 12 23:50:10 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:50:10.837 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:50:10 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:50:10.837 INFO:journalctl@ceph.mds.b.smithi181.stdout:Sep 12 23:50:10 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-12T23:50:11.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:10 smithi136 bash[75164]: cluster 2024-09-12T23:50:09.540271+0000 mon.a (mon.0) 1237 : cluster [DBG] osdmap e742: 8 total, 8 up, 8 in 2024-09-12T23:50:11.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:10 smithi136 bash[76371]: cluster 2024-09-12T23:50:09.540271+0000 mon.a (mon.0) 1237 : cluster [DBG] osdmap e742: 8 total, 8 up, 8 in 2024-09-12T23:50:11.484 INFO:tasks.rados.rados.0.smithi136.stdout:3461: done (3 left) 2024-09-12T23:50:11.484 INFO:tasks.rados.rados.0.smithi136.stdout:3462: done (2 left) 2024-09-12T23:50:11.484 INFO:tasks.rados.rados.0.smithi136.stdout:3463: done (1 left) 2024-09-12T23:50:11.485 INFO:tasks.rados.rados.0.smithi136.stdout:3465: done (0 left) 2024-09-12T23:50:11.485 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 46 current snap is 331 2024-09-12T23:50:11.485 INFO:tasks.rados.rados.0.smithi136.stdout:3466: seq_num 1094 ranges {0=729088,729088=573440,1302528=229376} 2024-09-12T23:50:11.485 INFO:tasks.rados.rados.0.smithi136.stdout:3466: writing smithi13667081-46 from 0 to 729088 tid 1 2024-09-12T23:50:11.493 INFO:tasks.rados.rados.0.smithi136.stdout:3466: writing smithi13667081-46 from 729088 to 1302528 tid 2 2024-09-12T23:50:11.495 INFO:tasks.rados.rados.0.smithi136.stdout:3466: writing smithi13667081-46 from 1302528 to 1531904 tid 3 2024-09-12T23:50:11.495 INFO:tasks.rados.rados.0.smithi136.stdout:3467: read oid 11 snap -1 2024-09-12T23:50:11.495 INFO:tasks.rados.rados.0.smithi136.stdout:3467: expect (ObjNum 1056 snap 321 seq_num 1056) 2024-09-12T23:50:11.496 INFO:tasks.rados.rados.0.smithi136.stdout:3468: setattr oid 38 current snap is 331 2024-09-12T23:50:11.499 INFO:tasks.rados.rados.0.smithi136.stdout:3469: snap_remove snap 324 2024-09-12T23:50:11.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:11 smithi181 bash[67587]: cluster 2024-09-12T23:50:10.541573+0000 mon.a (mon.0) 1238 : cluster [DBG] osdmap e743: 8 total, 8 up, 8 in 2024-09-12T23:50:11.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:11 smithi181 bash[67587]: audit 2024-09-12T23:50:10.770706+0000 mon.a (mon.0) 1239 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:50:11.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:11 smithi181 bash[67587]: audit 2024-09-12T23:50:10.780939+0000 mon.a (mon.0) 1240 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:50:11.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:11 smithi181 bash[67587]: audit 2024-09-12T23:50:10.783238+0000 mon.a (mon.0) 1241 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:50:11.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:11 smithi181 bash[67587]: cluster 2024-09-12T23:50:10.843202+0000 mgr.y (mgr.24661) 384 : cluster [DBG] pgmap v696: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 327 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.4 MiB/s wr, 5 op/s 2024-09-12T23:50:11.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:11 smithi181 bash[67587]: cluster 2024-09-12T23:50:11.480175+0000 mon.a (mon.0) 1242 : cluster [DBG] osdmap e744: 8 total, 8 up, 8 in 2024-09-12T23:50:12.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:11 smithi136 bash[75164]: cluster 2024-09-12T23:50:10.541573+0000 mon.a (mon.0) 1238 : cluster [DBG] osdmap e743: 8 total, 8 up, 8 in 2024-09-12T23:50:12.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:11 smithi136 bash[75164]: audit 2024-09-12T23:50:10.770706+0000 mon.a (mon.0) 1239 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:50:12.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:11 smithi136 bash[75164]: audit 2024-09-12T23:50:10.780939+0000 mon.a (mon.0) 1240 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:50:12.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:11 smithi136 bash[75164]: audit 2024-09-12T23:50:10.783238+0000 mon.a (mon.0) 1241 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:50:12.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:11 smithi136 bash[75164]: cluster 2024-09-12T23:50:10.843202+0000 mgr.y (mgr.24661) 384 : cluster [DBG] pgmap v696: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 327 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.4 MiB/s wr, 5 op/s 2024-09-12T23:50:12.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:11 smithi136 bash[75164]: cluster 2024-09-12T23:50:11.480175+0000 mon.a (mon.0) 1242 : cluster [DBG] osdmap e744: 8 total, 8 up, 8 in 2024-09-12T23:50:12.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:11 smithi136 bash[76371]: cluster 2024-09-12T23:50:10.541573+0000 mon.a (mon.0) 1238 : cluster [DBG] osdmap e743: 8 total, 8 up, 8 in 2024-09-12T23:50:12.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:11 smithi136 bash[76371]: audit 2024-09-12T23:50:10.770706+0000 mon.a (mon.0) 1239 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:50:12.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:11 smithi136 bash[76371]: audit 2024-09-12T23:50:10.780939+0000 mon.a (mon.0) 1240 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:50:12.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:11 smithi136 bash[76371]: audit 2024-09-12T23:50:10.783238+0000 mon.a (mon.0) 1241 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:50:12.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:11 smithi136 bash[76371]: cluster 2024-09-12T23:50:10.843202+0000 mgr.y (mgr.24661) 384 : cluster [DBG] pgmap v696: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 327 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.4 MiB/s wr, 5 op/s 2024-09-12T23:50:12.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:11 smithi136 bash[76371]: cluster 2024-09-12T23:50:11.480175+0000 mon.a (mon.0) 1242 : cluster [DBG] osdmap e744: 8 total, 8 up, 8 in 2024-09-12T23:50:12.485 INFO:tasks.rados.rados.0.smithi136.stdout:3469: done (3 left) 2024-09-12T23:50:12.485 INFO:tasks.rados.rados.0.smithi136.stdout:3470: snap_remove snap 328 2024-09-12T23:50:12.595 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:50:12 smithi181 bash[95505]: ts=2024-09-12T23:50:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:50:12.595 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:50:12 smithi181 bash[98820]: logger=settings t=2024-09-12T23:50:12.141601399Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-09-12T23:50:12.596 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:50:12 smithi181 bash[98820]: logger=settings t=2024-09-12T23:50:12.141833176Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-09-12T23:50:12.596 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:50:12 smithi181 bash[98820]: logger=settings t=2024-09-12T23:50:12.141863881Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-09-12T23:50:12.596 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:50:12 smithi181 bash[98820]: logger=settings t=2024-09-12T23:50:12.141882549Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-09-12T23:50:12.596 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:50:12 smithi181 bash[98820]: logger=settings t=2024-09-12T23:50:12.141896667Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-09-12T23:50:12.596 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:50:12 smithi181 bash[98820]: logger=settings t=2024-09-12T23:50:12.141911Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-09-12T23:50:12.596 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:50:12 smithi181 bash[98820]: logger=settings t=2024-09-12T23:50:12.141924566Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-09-12T23:50:12.596 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:50:12 smithi181 bash[98820]: logger=settings t=2024-09-12T23:50:12.141937926Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-09-12T23:50:12.596 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:50:12 smithi181 bash[98820]: logger=settings t=2024-09-12T23:50:12.141951699Z level=info msg="Path Home" path=/usr/share/grafana 2024-09-12T23:50:12.596 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:50:12 smithi181 bash[98820]: logger=settings t=2024-09-12T23:50:12.141965024Z level=info msg="Path Data" path=/var/lib/grafana 2024-09-12T23:50:12.596 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:50:12 smithi181 bash[98820]: logger=settings t=2024-09-12T23:50:12.141981542Z level=info msg="Path Logs" path=/var/log/grafana 2024-09-12T23:50:12.597 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:50:12 smithi181 bash[98820]: logger=settings t=2024-09-12T23:50:12.14199681Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-09-12T23:50:12.597 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:50:12 smithi181 bash[98820]: logger=settings t=2024-09-12T23:50:12.142010045Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-09-12T23:50:12.597 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:50:12 smithi181 bash[98820]: logger=settings t=2024-09-12T23:50:12.142024221Z level=info msg="App mode production" 2024-09-12T23:50:12.597 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:50:12 smithi181 bash[98820]: logger=sqlstore t=2024-09-12T23:50:12.142108885Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-09-12T23:50:12.597 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:50:12 smithi181 bash[98820]: logger=sqlstore t=2024-09-12T23:50:12.142145146Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-09-12T23:50:12.597 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:50:12 smithi181 bash[98820]: logger=migrator t=2024-09-12T23:50:12.158953503Z level=info msg="Starting DB migrations" 2024-09-12T23:50:12.597 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:50:12 smithi181 bash[98820]: logger=migrator t=2024-09-12T23:50:12.164542986Z level=info msg="migrations completed" performed=0 skipped=481 duration=509.245µs 2024-09-12T23:50:12.598 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:50:12 smithi181 bash[98820]: logger=sqlstore t=2024-09-12T23:50:12.164987501Z level=info msg="Created default organization" 2024-09-12T23:50:12.598 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:50:12 smithi181 bash[98820]: logger=plugin.loader t=2024-09-12T23:50:12.268674429Z level=info msg="Plugin registered" pluginID=input 2024-09-12T23:50:12.598 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:50:12 smithi181 bash[98820]: logger=plugin.loader t=2024-09-12T23:50:12.299358126Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-09-12T23:50:12.598 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:50:12 smithi181 bash[98820]: logger=plugin.loader t=2024-09-12T23:50:12.299393851Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-09-12T23:50:12.598 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:50:12 smithi181 bash[98820]: logger=secrets t=2024-09-12T23:50:12.299430511Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-09-12T23:50:12.598 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:50:12 smithi181 bash[98820]: logger=query_data t=2024-09-12T23:50:12.301459191Z level=info msg="Query Service initialization" 2024-09-12T23:50:12.598 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:50:12 smithi181 bash[98820]: logger=live.push_http t=2024-09-12T23:50:12.305784712Z level=info msg="Live Push Gateway initialization" 2024-09-12T23:50:13.486 INFO:tasks.rados.rados.0.smithi136.stdout:3470: done (3 left) 2024-09-12T23:50:13.486 INFO:tasks.rados.rados.0.smithi136.stdout:3471: snap_create 2024-09-12T23:50:13.487 INFO:tasks.rados.rados.0.smithi136.stdout:3466: finishing write tid 1 to smithi13667081-46 2024-09-12T23:50:13.487 INFO:tasks.rados.rados.0.smithi136.stdout:3466: finishing write tid 2 to smithi13667081-46 2024-09-12T23:50:13.487 INFO:tasks.rados.rados.0.smithi136.stdout:3466: finishing write tid 3 to smithi13667081-46 2024-09-12T23:50:13.487 INFO:tasks.rados.rados.0.smithi136.stdout:3466: finishing write tid 4 to smithi13667081-46 2024-09-12T23:50:13.487 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 38 v 2664 (ObjNum 864 snap 281 seq_num 864) dirty exists 2024-09-12T23:50:13.487 INFO:tasks.rados.rados.0.smithi136.stdout:3467: expect (ObjNum 1056 snap 321 seq_num 1056) 2024-09-12T23:50:13.625 INFO:tasks.rados.rados.0.smithi136.stdout:3466: finishing write tid 5 to smithi13667081-46 2024-09-12T23:50:13.625 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 46 v 2485 (ObjNum 1094 snap 331 seq_num 1094) dirty exists 2024-09-12T23:50:13.625 INFO:tasks.rados.rados.0.smithi136.stdout:3466: left oid 46 (ObjNum 1094 snap 331 seq_num 1094) 2024-09-12T23:50:13.689 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:13 smithi181 bash[67587]: cluster 2024-09-12T23:50:12.487520+0000 mon.a (mon.0) 1243 : cluster [DBG] osdmap e745: 8 total, 8 up, 8 in 2024-09-12T23:50:13.689 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:13 smithi181 bash[67587]: cluster 2024-09-12T23:50:12.844381+0000 mgr.y (mgr.24661) 385 : cluster [DBG] pgmap v699: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 300 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.8 MiB/s wr, 16 op/s 2024-09-12T23:50:13.689 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:50:13 smithi181 bash[98820]: logger=infra.usagestats.collector t=2024-09-12T23:50:13.400056267Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-09-12T23:50:13.690 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:50:13 smithi181 bash[98820]: logger=provisioning.datasources t=2024-09-12T23:50:13.544875389Z level=info msg="deleted correlations based on configuration" ds_name=Dashboard1 2024-09-12T23:50:13.690 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:50:13 smithi181 bash[98820]: logger=provisioning.datasources t=2024-09-12T23:50:13.544944554Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2024-09-12T23:50:13.690 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:50:13 smithi181 bash[98820]: logger=provisioning.datasources t=2024-09-12T23:50:13.556335905Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-09-12T23:50:13.690 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:50:13 smithi181 bash[98820]: logger=provisioning.alerting t=2024-09-12T23:50:13.692992589Z level=info msg="starting to provision alerting" 2024-09-12T23:50:13.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:13 smithi136 bash[75164]: cluster 2024-09-12T23:50:12.487520+0000 mon.a (mon.0) 1243 : cluster [DBG] osdmap e745: 8 total, 8 up, 8 in 2024-09-12T23:50:13.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:13 smithi136 bash[75164]: cluster 2024-09-12T23:50:12.844381+0000 mgr.y (mgr.24661) 385 : cluster [DBG] pgmap v699: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 300 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.8 MiB/s wr, 16 op/s 2024-09-12T23:50:13.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:13 smithi136 bash[76371]: cluster 2024-09-12T23:50:12.487520+0000 mon.a (mon.0) 1243 : cluster [DBG] osdmap e745: 8 total, 8 up, 8 in 2024-09-12T23:50:13.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:13 smithi136 bash[76371]: cluster 2024-09-12T23:50:12.844381+0000 mgr.y (mgr.24661) 385 : cluster [DBG] pgmap v699: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 300 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.8 MiB/s wr, 16 op/s 2024-09-12T23:50:14.096 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:50:13 smithi181 bash[98820]: logger=provisioning.alerting t=2024-09-12T23:50:13.693028827Z level=info msg="finished to provision alerting" 2024-09-12T23:50:14.096 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:50:13 smithi181 bash[98820]: logger=grafanaStorageLogger t=2024-09-12T23:50:13.693287281Z level=info msg="storage starting" 2024-09-12T23:50:14.096 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:50:13 smithi181 bash[98820]: logger=ngalert.state.manager t=2024-09-12T23:50:13.693453898Z level=info msg="Warming state cache for startup" 2024-09-12T23:50:14.096 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:50:13 smithi181 bash[98820]: logger=http.server t=2024-09-12T23:50:13.699185099Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-09-12T23:50:14.096 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:50:13 smithi181 bash[98820]: logger=ngalert.state.manager t=2024-09-12T23:50:13.737596437Z level=info msg="State cache has been initialized" states=0 duration=44.137494ms 2024-09-12T23:50:14.097 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:50:13 smithi181 bash[98820]: logger=ngalert.multiorg.alertmanager t=2024-09-12T23:50:13.737737558Z level=info msg="starting MultiOrg Alertmanager" 2024-09-12T23:50:14.097 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 12 23:50:13 smithi181 bash[98820]: logger=ticker t=2024-09-12T23:50:13.737831555Z level=info msg=starting first_tick=2024-09-12T23:50:20Z 2024-09-12T23:50:14.489 INFO:tasks.rados.rados.0.smithi136.stdout:3466: done (3 left) 2024-09-12T23:50:14.490 INFO:tasks.rados.rados.0.smithi136.stdout:3467: done (2 left) 2024-09-12T23:50:14.490 INFO:tasks.rados.rados.0.smithi136.stdout:3468: done (1 left) 2024-09-12T23:50:14.490 INFO:tasks.rados.rados.0.smithi136.stdout:3471: done (0 left) 2024-09-12T23:50:14.490 INFO:tasks.rados.rados.0.smithi136.stdout:3472: rmattr oid 20 current snap is 332 2024-09-12T23:50:14.490 INFO:tasks.rados.rados.0.smithi136.stdout:3472: done (0 left) 2024-09-12T23:50:14.490 INFO:tasks.rados.rados.0.smithi136.stdout:3473: rollback oid 4 current snap is 332 2024-09-12T23:50:14.490 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 4 to 331 2024-09-12T23:50:14.490 INFO:tasks.rados.rados.0.smithi136.stdout:3474: rmattr oid 11 current snap is 332 2024-09-12T23:50:14.490 INFO:tasks.rados.rados.0.smithi136.stdout:3474: done (1 left) 2024-09-12T23:50:14.490 INFO:tasks.rados.rados.0.smithi136.stdout:3475: read oid 23 snap 296 2024-09-12T23:50:14.490 INFO:tasks.rados.rados.0.smithi136.stdout:3475: expect (ObjNum 886 snap 287 seq_num 886) 2024-09-12T23:50:14.491 INFO:tasks.rados.rados.0.smithi136.stdout:3476: read oid 9 snap 330 2024-09-12T23:50:14.491 INFO:tasks.rados.rados.0.smithi136.stdout:3476: expect (ObjNum 1091 snap 329 seq_num 1091) 2024-09-12T23:50:14.491 INFO:tasks.rados.rados.0.smithi136.stdout:3477: delete oid 45 current snap is 332 2024-09-12T23:50:14.503 INFO:tasks.rados.rados.0.smithi136.stdout:3475: expect (ObjNum 886 snap 287 seq_num 886) 2024-09-12T23:50:14.542 INFO:tasks.rados.rados.0.smithi136.stdout:3473: finishing rollback tid 0 to smithi13667081-4 2024-09-12T23:50:14.542 INFO:tasks.rados.rados.0.smithi136.stdout:3473: finishing rollback tid 1 to smithi13667081-4 2024-09-12T23:50:14.542 INFO:tasks.rados.rados.0.smithi136.stdout:3473: finishing rollback tid 2 to smithi13667081-4 2024-09-12T23:50:14.542 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 4 v 1653 (ObjNum 1082 snap 326 seq_num 1082) dirty exists 2024-09-12T23:50:14.542 INFO:tasks.rados.rados.0.smithi136.stdout:3476: expect (ObjNum 1091 snap 329 seq_num 1091) 2024-09-12T23:50:14.673 INFO:tasks.rados.rados.0.smithi136.stdout:3473: done (3 left) 2024-09-12T23:50:14.673 INFO:tasks.rados.rados.0.smithi136.stdout:3475: done (2 left) 2024-09-12T23:50:14.673 INFO:tasks.rados.rados.0.smithi136.stdout:3476: done (1 left) 2024-09-12T23:50:14.673 INFO:tasks.rados.rados.0.smithi136.stdout:3477: done (0 left) 2024-09-12T23:50:14.673 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 4 current snap is 332 2024-09-12T23:50:14.673 INFO:tasks.rados.rados.0.smithi136.stdout:3478: seq_num 1096 ranges {18702336=761856,19464192=786432,20250624=16384} 2024-09-12T23:50:14.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:14 smithi136 bash[75164]: cluster 2024-09-12T23:50:13.487574+0000 mon.a (mon.0) 1244 : cluster [DBG] osdmap e746: 8 total, 8 up, 8 in 2024-09-12T23:50:14.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:14 smithi136 bash[75164]: cluster 2024-09-12T23:50:14.490819+0000 mon.a (mon.0) 1245 : cluster [DBG] osdmap e747: 8 total, 8 up, 8 in 2024-09-12T23:50:14.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:14 smithi136 bash[76371]: cluster 2024-09-12T23:50:13.487574+0000 mon.a (mon.0) 1244 : cluster [DBG] osdmap e746: 8 total, 8 up, 8 in 2024-09-12T23:50:14.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:14 smithi136 bash[76371]: cluster 2024-09-12T23:50:14.490819+0000 mon.a (mon.0) 1245 : cluster [DBG] osdmap e747: 8 total, 8 up, 8 in 2024-09-12T23:50:14.835 INFO:tasks.rados.rados.0.smithi136.stdout:3478: writing smithi13667081-4 from 18702336 to 19464192 tid 1 2024-09-12T23:50:14.839 INFO:tasks.rados.rados.0.smithi136.stdout:3478: writing smithi13667081-4 from 19464192 to 20250624 tid 2 2024-09-12T23:50:14.839 INFO:tasks.rados.rados.0.smithi136.stdout:3478: writing smithi13667081-4 from 20250624 to 20267008 tid 3 2024-09-12T23:50:14.839 INFO:tasks.rados.rados.0.smithi136.stdout:3479: rmattr oid 39 current snap is 332 2024-09-12T23:50:14.840 INFO:tasks.rados.rados.0.smithi136.stdout:3479: done (1 left) 2024-09-12T23:50:14.840 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 32 current snap is 332 2024-09-12T23:50:14.840 INFO:tasks.rados.rados.0.smithi136.stdout:3480: seq_num 1097 ranges {15523840=778240,16302080=671744,16973824=131072} 2024-09-12T23:50:14.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:14 smithi181 bash[67587]: cluster 2024-09-12T23:50:13.487574+0000 mon.a (mon.0) 1244 : cluster [DBG] osdmap e746: 8 total, 8 up, 8 in 2024-09-12T23:50:14.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:14 smithi181 bash[67587]: cluster 2024-09-12T23:50:14.490819+0000 mon.a (mon.0) 1245 : cluster [DBG] osdmap e747: 8 total, 8 up, 8 in 2024-09-12T23:50:14.930 INFO:tasks.rados.rados.0.smithi136.stdout:3480: writing smithi13667081-32 from 15523840 to 16302080 tid 1 2024-09-12T23:50:14.934 INFO:tasks.rados.rados.0.smithi136.stdout:3480: writing smithi13667081-32 from 16302080 to 16973824 tid 2 2024-09-12T23:50:14.935 INFO:tasks.rados.rados.0.smithi136.stdout:3480: writing smithi13667081-32 from 16973824 to 17104896 tid 3 2024-09-12T23:50:14.935 INFO:tasks.rados.rados.0.smithi136.stdout:3481: delete oid 50 current snap is 332 2024-09-12T23:50:14.935 INFO:tasks.rados.rados.0.smithi136.stdout:3478: finishing write tid 1 to smithi13667081-4 2024-09-12T23:50:14.935 INFO:tasks.rados.rados.0.smithi136.stdout:3478: finishing write tid 2 to smithi13667081-4 2024-09-12T23:50:14.935 INFO:tasks.rados.rados.0.smithi136.stdout:3478: finishing write tid 3 to smithi13667081-4 2024-09-12T23:50:14.935 INFO:tasks.rados.rados.0.smithi136.stdout:3478: finishing write tid 4 to smithi13667081-4 2024-09-12T23:50:14.935 INFO:tasks.rados.rados.0.smithi136.stdout:3478: finishing write tid 5 to smithi13667081-4 2024-09-12T23:50:14.935 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 4 v 1657 (ObjNum 1096 snap 332 seq_num 1096) dirty exists 2024-09-12T23:50:14.936 INFO:tasks.rados.rados.0.smithi136.stdout:3478: left oid 4 (ObjNum 1096 snap 332 seq_num 1096) 2024-09-12T23:50:14.942 INFO:tasks.rados.rados.0.smithi136.stdout:3480: finishing write tid 1 to smithi13667081-32 2024-09-12T23:50:14.942 INFO:tasks.rados.rados.0.smithi136.stdout:3478: done (2 left) 2024-09-12T23:50:14.942 INFO:tasks.rados.rados.0.smithi136.stdout:3481: done (1 left) 2024-09-12T23:50:14.942 INFO:tasks.rados.rados.0.smithi136.stdout:3482: copy_from oid 35 from oid 7 current snap is 332 2024-09-12T23:50:14.942 INFO:tasks.rados.rados.0.smithi136.stdout:3483: rmattr oid 43 current snap is 332 2024-09-12T23:50:14.942 INFO:tasks.rados.rados.0.smithi136.stdout:3483: done (2 left) 2024-09-12T23:50:14.942 INFO:tasks.rados.rados.0.smithi136.stdout:3484: snap_remove snap 327 2024-09-12T23:50:15.512 INFO:tasks.rados.rados.0.smithi136.stdout:3484: done (2 left) 2024-09-12T23:50:15.512 INFO:tasks.rados.rados.0.smithi136.stdout:3485: setattr oid 26 current snap is 332 2024-09-12T23:50:15.512 INFO:tasks.rados.rados.0.smithi136.stdout:3480: finishing write tid 2 to smithi13667081-32 2024-09-12T23:50:15.512 INFO:tasks.rados.rados.0.smithi136.stdout:3480: finishing write tid 3 to smithi13667081-32 2024-09-12T23:50:15.512 INFO:tasks.rados.rados.0.smithi136.stdout:3480: finishing write tid 4 to smithi13667081-32 2024-09-12T23:50:15.512 INFO:tasks.rados.rados.0.smithi136.stdout:3480: finishing write tid 5 to smithi13667081-32 2024-09-12T23:50:15.512 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 32 v 1661 (ObjNum 1097 snap 332 seq_num 1097) dirty exists 2024-09-12T23:50:15.513 INFO:tasks.rados.rados.0.smithi136.stdout:3480: left oid 32 (ObjNum 1097 snap 332 seq_num 1097) 2024-09-12T23:50:15.513 INFO:tasks.rados.rados.0.smithi136.stdout:3482: finishing copy_from racing read to smithi13667081-35 2024-09-12T23:50:15.513 INFO:tasks.rados.rados.0.smithi136.stdout:3482: finishing copy_from to smithi13667081-35 2024-09-12T23:50:15.513 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 35 v 990 (ObjNum 994 snap 308 seq_num 994) dirty exists 2024-09-12T23:50:15.514 INFO:tasks.rados.rados.0.smithi136.stdout:3480: done (2 left) 2024-09-12T23:50:15.514 INFO:tasks.rados.rados.0.smithi136.stdout:3482: done (1 left) 2024-09-12T23:50:15.514 INFO:tasks.rados.rados.0.smithi136.stdout:3486: copy_from oid 30 from oid 9 current snap is 332 2024-09-12T23:50:15.514 INFO:tasks.rados.rados.0.smithi136.stdout:3487: read oid 44 snap -1 2024-09-12T23:50:15.514 INFO:tasks.rados.rados.0.smithi136.stdout:3487: expect deleted 2024-09-12T23:50:15.514 INFO:tasks.rados.rados.0.smithi136.stdout:3488: read oid 49 snap 329 2024-09-12T23:50:15.514 INFO:tasks.rados.rados.0.smithi136.stdout:3488: expect deleted 2024-09-12T23:50:15.514 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 3 current snap is 332 2024-09-12T23:50:15.514 INFO:tasks.rados.rados.0.smithi136.stdout:3489: seq_num 1099 ranges {3309568=409600,3719168=442368,4161536=720896,4882432=40960} 2024-09-12T23:50:15.536 INFO:tasks.rados.rados.0.smithi136.stdout:3489: writing smithi13667081-3 from 3309568 to 3719168 tid 1 2024-09-12T23:50:15.539 INFO:tasks.rados.rados.0.smithi136.stdout:3489: writing smithi13667081-3 from 3719168 to 4161536 tid 2 2024-09-12T23:50:15.544 INFO:tasks.rados.rados.0.smithi136.stdout:3489: writing smithi13667081-3 from 4161536 to 4882432 tid 3 2024-09-12T23:50:15.544 INFO:tasks.rados.rados.0.smithi136.stdout:3489: writing smithi13667081-3 from 4882432 to 4923392 tid 4 2024-09-12T23:50:15.544 INFO:tasks.rados.rados.0.smithi136.stdout:3490: snap_remove snap 330 2024-09-12T23:50:15.642 INFO:teuthology.orchestra.run.smithi136.stdout:true 2024-09-12T23:50:15.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:15 smithi136 bash[75164]: cluster 2024-09-12T23:50:14.845339+0000 mgr.y (mgr.24661) 386 : cluster [DBG] pgmap v702: 121 pgs: 1 active+clean+snaptrim_wait, 120 active+clean; 277 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 1.1 MiB/s wr, 21 op/s 2024-09-12T23:50:15.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:15 smithi136 bash[76371]: cluster 2024-09-12T23:50:14.845339+0000 mgr.y (mgr.24661) 386 : cluster [DBG] pgmap v702: 121 pgs: 1 active+clean+snaptrim_wait, 120 active+clean; 277 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 1.1 MiB/s wr, 21 op/s 2024-09-12T23:50:15.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:15 smithi181 bash[67587]: cluster 2024-09-12T23:50:14.845339+0000 mgr.y (mgr.24661) 386 : cluster [DBG] pgmap v702: 121 pgs: 1 active+clean+snaptrim_wait, 120 active+clean; 277 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 1.1 MiB/s wr, 21 op/s 2024-09-12T23:50:15.949 INFO:teuthology.orchestra.run.smithi136.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-12T23:50:15.949 INFO:teuthology.orchestra.run.smithi136.stdout:alertmanager.a smithi136 *:9093,9094 running (29s) 23s ago 15m 13.8M - 0.25.0 c8568f914cd2 f36cb97f345b 2024-09-12T23:50:15.950 INFO:teuthology.orchestra.run.smithi136.stdout:grafana.a smithi181 *:3000 starting - - - - 2024-09-12T23:50:15.950 INFO:teuthology.orchestra.run.smithi136.stdout:mds.a smithi136 running (2m) 23s ago 16m 13.7M - 18.2.4-837-g7a853b17 f21ba1eea74e a20ac830342c 2024-09-12T23:50:15.950 INFO:teuthology.orchestra.run.smithi136.stdout:mds.b smithi181 running (104s) 48s ago 16m 11.8M - 18.2.4-837-g7a853b17 f21ba1eea74e 0dc9f76cf0b8 2024-09-12T23:50:15.950 INFO:teuthology.orchestra.run.smithi136.stdout:mgr.x smithi181 *:8443,9283,8765 running (8m) 48s ago 24m 428M - 18.2.4-837-g7a853b17 f21ba1eea74e cc664dc337f7 2024-09-12T23:50:15.950 INFO:teuthology.orchestra.run.smithi136.stdout:mgr.y smithi136 *:8443,9283,8765 running (9m) 23s ago 26m 498M - 18.2.4-837-g7a853b17 f21ba1eea74e 272f10b42fc3 2024-09-12T23:50:15.950 INFO:teuthology.orchestra.run.smithi136.stdout:mon.a smithi136 running (8m) 23s ago 26m 72.2M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e ad8d21d58ad9 2024-09-12T23:50:15.950 INFO:teuthology.orchestra.run.smithi136.stdout:mon.b smithi181 running (7m) 48s ago 24m 51.2M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e a70ec8315934 2024-09-12T23:50:15.950 INFO:teuthology.orchestra.run.smithi136.stdout:mon.c smithi136 running (7m) 23s ago 24m 60.4M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e 44d1be055a5e 2024-09-12T23:50:15.950 INFO:teuthology.orchestra.run.smithi136.stdout:node-exporter.a smithi136 *:9100 running (83s) 23s ago 16m 8248k - 1.5.0 0da6a335fe13 348c0ab0964c 2024-09-12T23:50:15.950 INFO:teuthology.orchestra.run.smithi136.stdout:node-exporter.b smithi181 *:9100 running (79s) 48s ago 15m 7784k - 1.5.0 0da6a335fe13 3af01e08b5af 2024-09-12T23:50:15.950 INFO:teuthology.orchestra.run.smithi136.stdout:osd.0 smithi136 running (6m) 23s ago 23m 404M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 7da21f765721 2024-09-12T23:50:15.950 INFO:teuthology.orchestra.run.smithi136.stdout:osd.1 smithi136 running (5m) 23s ago 22m 424M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 5302be0a016b 2024-09-12T23:50:15.951 INFO:teuthology.orchestra.run.smithi136.stdout:osd.2 smithi136 running (5m) 23s ago 21m 385M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 2e12d1a31223 2024-09-12T23:50:15.951 INFO:teuthology.orchestra.run.smithi136.stdout:osd.3 smithi136 running (4m) 23s ago 21m 297M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e aa9dc124e59c 2024-09-12T23:50:15.951 INFO:teuthology.orchestra.run.smithi136.stdout:osd.4 smithi181 running (4m) 48s ago 20m 463M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 4ee3b228ce7d 2024-09-12T23:50:15.951 INFO:teuthology.orchestra.run.smithi136.stdout:osd.5 smithi181 running (3m) 48s ago 19m 228M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 3f9a09565258 2024-09-12T23:50:15.951 INFO:teuthology.orchestra.run.smithi136.stdout:osd.6 smithi181 running (3m) 48s ago 18m 136M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e ce539f6fb854 2024-09-12T23:50:15.951 INFO:teuthology.orchestra.run.smithi136.stdout:osd.7 smithi181 running (2m) 48s ago 17m 236M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 6ad9ab169a7b 2024-09-12T23:50:15.951 INFO:teuthology.orchestra.run.smithi136.stdout:prometheus.a smithi181 *:9095 running (54s) 48s ago 16m 40.6M - 2.43.0 a07b618ecd1d 302d5b6a8434 2024-09-12T23:50:16.307 INFO:teuthology.orchestra.run.smithi136.stdout:{ 2024-09-12T23:50:16.307 INFO:teuthology.orchestra.run.smithi136.stdout: "mon": { 2024-09-12T23:50:16.307 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 3 2024-09-12T23:50:16.307 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:50:16.307 INFO:teuthology.orchestra.run.smithi136.stdout: "mgr": { 2024-09-12T23:50:16.307 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 2 2024-09-12T23:50:16.307 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:50:16.307 INFO:teuthology.orchestra.run.smithi136.stdout: "osd": { 2024-09-12T23:50:16.308 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 8 2024-09-12T23:50:16.308 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:50:16.308 INFO:teuthology.orchestra.run.smithi136.stdout: "mds": { 2024-09-12T23:50:16.308 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 2 2024-09-12T23:50:16.308 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:50:16.308 INFO:teuthology.orchestra.run.smithi136.stdout: "overall": { 2024-09-12T23:50:16.308 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 15 2024-09-12T23:50:16.308 INFO:teuthology.orchestra.run.smithi136.stdout: } 2024-09-12T23:50:16.308 INFO:teuthology.orchestra.run.smithi136.stdout:} 2024-09-12T23:50:16.480 INFO:tasks.rados.rados.0.smithi136.stdout:3490: done (5 left) 2024-09-12T23:50:16.480 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 34 current snap is 332 2024-09-12T23:50:16.480 INFO:tasks.rados.rados.0.smithi136.stdout:3491: seq_num 1100 ranges {2048000=425984,2473984=729088,3203072=475136} 2024-09-12T23:50:16.495 INFO:tasks.rados.rados.0.smithi136.stdout:3491: writing smithi13667081-34 from 2048000 to 2473984 tid 1 2024-09-12T23:50:16.500 INFO:tasks.rados.rados.0.smithi136.stdout:3491: writing smithi13667081-34 from 2473984 to 3203072 tid 2 2024-09-12T23:50:16.504 INFO:tasks.rados.rados.0.smithi136.stdout:3491: writing smithi13667081-34 from 3203072 to 3678208 tid 3 2024-09-12T23:50:16.504 INFO:tasks.rados.rados.0.smithi136.stdout:3492: snap_create 2024-09-12T23:50:16.506 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 26 v 2486 (ObjNum 987 snap 303 seq_num 987) dirty exists 2024-09-12T23:50:16.506 INFO:tasks.rados.rados.0.smithi136.stdout:3489: finishing write tid 1 to smithi13667081-3 2024-09-12T23:50:16.506 INFO:tasks.rados.rados.0.smithi136.stdout:3489: finishing write tid 2 to smithi13667081-3 2024-09-12T23:50:16.506 INFO:tasks.rados.rados.0.smithi136.stdout:3489: finishing write tid 3 to smithi13667081-3 2024-09-12T23:50:16.506 INFO:tasks.rados.rados.0.smithi136.stdout:3489: finishing write tid 4 to smithi13667081-3 2024-09-12T23:50:16.506 INFO:tasks.rados.rados.0.smithi136.stdout:3489: finishing write tid 5 to smithi13667081-3 2024-09-12T23:50:16.506 INFO:tasks.rados.rados.0.smithi136.stdout:3489: finishing write tid 6 to smithi13667081-3 2024-09-12T23:50:16.506 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 3 v 2099 (ObjNum 1099 snap 332 seq_num 1099) dirty exists 2024-09-12T23:50:16.506 INFO:tasks.rados.rados.0.smithi136.stdout:3489: left oid 3 (ObjNum 1099 snap 332 seq_num 1099) 2024-09-12T23:50:16.506 INFO:tasks.rados.rados.0.smithi136.stdout:3486: finishing copy_from racing read to smithi13667081-30 2024-09-12T23:50:16.506 INFO:tasks.rados.rados.0.smithi136.stdout:3486: finishing copy_from to smithi13667081-30 2024-09-12T23:50:16.506 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 30 v 2472 (ObjNum 1091 snap 329 seq_num 1091) dirty exists 2024-09-12T23:50:16.511 INFO:tasks.rados.rados.0.smithi136.stdout:3491: finishing write tid 1 to smithi13667081-34 2024-09-12T23:50:16.514 INFO:tasks.rados.rados.0.smithi136.stdout:3491: finishing write tid 2 to smithi13667081-34 2024-09-12T23:50:16.514 INFO:tasks.rados.rados.0.smithi136.stdout:3491: finishing write tid 3 to smithi13667081-34 2024-09-12T23:50:16.515 INFO:tasks.rados.rados.0.smithi136.stdout:3491: finishing write tid 4 to smithi13667081-34 2024-09-12T23:50:16.518 INFO:tasks.rados.rados.0.smithi136.stdout:3491: finishing write tid 5 to smithi13667081-34 2024-09-12T23:50:16.518 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 34 v 2489 (ObjNum 1100 snap 332 seq_num 1100) dirty exists 2024-09-12T23:50:16.518 INFO:tasks.rados.rados.0.smithi136.stdout:3491: left oid 34 (ObjNum 1100 snap 332 seq_num 1100) 2024-09-12T23:50:16.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:16 smithi181 bash[67587]: cluster 2024-09-12T23:50:15.515044+0000 mon.a (mon.0) 1246 : cluster [DBG] osdmap e748: 8 total, 8 up, 8 in 2024-09-12T23:50:16.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:16 smithi181 bash[67587]: audit 2024-09-12T23:50:15.632082+0000 mgr.y (mgr.24661) 387 : audit [DBG] from='client.34476 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:50:16.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:16 smithi181 bash[67587]: audit 2024-09-12T23:50:16.310325+0000 mon.c (mon.2) 15 : audit [DBG] from='client.? 172.21.15.136:0/3446993371' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:16.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:16 smithi181 bash[67587]: cluster 2024-09-12T23:50:16.482872+0000 mon.a (mon.0) 1247 : cluster [DBG] osdmap e749: 8 total, 8 up, 8 in 2024-09-12T23:50:16.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:16 smithi181 bash[67587]: audit 2024-09-12T23:50:16.503310+0000 mon.a (mon.0) 1248 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:50:17.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:16 smithi136 bash[75164]: cluster 2024-09-12T23:50:15.515044+0000 mon.a (mon.0) 1246 : cluster [DBG] osdmap e748: 8 total, 8 up, 8 in 2024-09-12T23:50:17.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:16 smithi136 bash[75164]: audit 2024-09-12T23:50:15.632082+0000 mgr.y (mgr.24661) 387 : audit [DBG] from='client.34476 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:50:17.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:16 smithi136 bash[75164]: audit 2024-09-12T23:50:16.310325+0000 mon.c (mon.2) 15 : audit [DBG] from='client.? 172.21.15.136:0/3446993371' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:17.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:16 smithi136 bash[75164]: cluster 2024-09-12T23:50:16.482872+0000 mon.a (mon.0) 1247 : cluster [DBG] osdmap e749: 8 total, 8 up, 8 in 2024-09-12T23:50:17.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:16 smithi136 bash[75164]: audit 2024-09-12T23:50:16.503310+0000 mon.a (mon.0) 1248 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:50:17.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:16 smithi136 bash[76371]: cluster 2024-09-12T23:50:15.515044+0000 mon.a (mon.0) 1246 : cluster [DBG] osdmap e748: 8 total, 8 up, 8 in 2024-09-12T23:50:17.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:16 smithi136 bash[76371]: audit 2024-09-12T23:50:15.632082+0000 mgr.y (mgr.24661) 387 : audit [DBG] from='client.34476 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:50:17.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:16 smithi136 bash[76371]: audit 2024-09-12T23:50:16.310325+0000 mon.c (mon.2) 15 : audit [DBG] from='client.? 172.21.15.136:0/3446993371' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:17.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:16 smithi136 bash[76371]: cluster 2024-09-12T23:50:16.482872+0000 mon.a (mon.0) 1247 : cluster [DBG] osdmap e749: 8 total, 8 up, 8 in 2024-09-12T23:50:17.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:16 smithi136 bash[76371]: audit 2024-09-12T23:50:16.503310+0000 mon.a (mon.0) 1248 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:50:17.503 INFO:tasks.rados.rados.0.smithi136.stdout:3485: done (6 left) 2024-09-12T23:50:17.503 INFO:tasks.rados.rados.0.smithi136.stdout:3486: done (5 left) 2024-09-12T23:50:17.503 INFO:tasks.rados.rados.0.smithi136.stdout:3487: done (4 left) 2024-09-12T23:50:17.503 INFO:tasks.rados.rados.0.smithi136.stdout:3488: done (3 left) 2024-09-12T23:50:17.503 INFO:tasks.rados.rados.0.smithi136.stdout:3489: done (2 left) 2024-09-12T23:50:17.503 INFO:tasks.rados.rados.0.smithi136.stdout:3491: done (1 left) 2024-09-12T23:50:17.503 INFO:tasks.rados.rados.0.smithi136.stdout:3492: done (0 left) 2024-09-12T23:50:17.503 INFO:tasks.rados.rados.0.smithi136.stdout:3493: read oid 3 snap -1 2024-09-12T23:50:17.504 INFO:tasks.rados.rados.0.smithi136.stdout:3493: expect (ObjNum 1099 snap 332 seq_num 1099) 2024-09-12T23:50:17.504 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 22 current snap is 333 2024-09-12T23:50:17.504 INFO:tasks.rados.rados.0.smithi136.stdout:3494: seq_num 1101 ranges {557056=442368,999424=606208,1605632=598016} 2024-09-12T23:50:17.509 INFO:tasks.rados.rados.0.smithi136.stdout:3494: writing smithi13667081-22 from 557056 to 999424 tid 1 2024-09-12T23:50:17.514 INFO:tasks.rados.rados.0.smithi136.stdout:3494: writing smithi13667081-22 from 999424 to 1605632 tid 2 2024-09-12T23:50:17.519 INFO:tasks.rados.rados.0.smithi136.stdout:3494: writing smithi13667081-22 from 1605632 to 2203648 tid 3 2024-09-12T23:50:17.519 INFO:tasks.rados.rados.0.smithi136.stdout:3495: read oid 37 snap -1 2024-09-12T23:50:17.519 INFO:tasks.rados.rados.0.smithi136.stdout:3495: expect (ObjNum 1020 snap 315 seq_num 1020) 2024-09-12T23:50:17.519 INFO:tasks.rados.rados.0.smithi136.stdout:3496: delete oid 18 current snap is 333 2024-09-12T23:50:17.528 INFO:tasks.rados.rados.0.smithi136.stdout:3494: finishing write tid 1 to smithi13667081-22 2024-09-12T23:50:17.530 INFO:tasks.rados.rados.0.smithi136.stdout:3496: done (3 left) 2024-09-12T23:50:17.531 INFO:tasks.rados.rados.0.smithi136.stdout:3497: read oid 11 snap -1 2024-09-12T23:50:17.531 INFO:tasks.rados.rados.0.smithi136.stdout:3497: expect (ObjNum 1056 snap 321 seq_num 1056) 2024-09-12T23:50:17.531 INFO:tasks.rados.rados.0.smithi136.stdout:3498: delete oid 27 current snap is 333 2024-09-12T23:50:17.539 INFO:tasks.rados.rados.0.smithi136.stdout:3494: finishing write tid 2 to smithi13667081-22 2024-09-12T23:50:17.539 INFO:tasks.rados.rados.0.smithi136.stdout:3494: finishing write tid 3 to smithi13667081-22 2024-09-12T23:50:17.539 INFO:tasks.rados.rados.0.smithi136.stdout:3494: finishing write tid 4 to smithi13667081-22 2024-09-12T23:50:17.551 INFO:tasks.rados.rados.0.smithi136.stdout:3494: finishing write tid 5 to smithi13667081-22 2024-09-12T23:50:17.551 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 22 v 2487 (ObjNum 1101 snap 333 seq_num 1101) dirty exists 2024-09-12T23:50:17.551 INFO:tasks.rados.rados.0.smithi136.stdout:3494: left oid 22 (ObjNum 1101 snap 333 seq_num 1101) 2024-09-12T23:50:17.554 INFO:tasks.rados.rados.0.smithi136.stdout:3493: expect (ObjNum 1099 snap 332 seq_num 1099) 2024-09-12T23:50:17.775 INFO:tasks.rados.rados.0.smithi136.stdout:3497: expect (ObjNum 1056 snap 321 seq_num 1056) 2024-09-12T23:50:17.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:17 smithi181 bash[67587]: audit 2024-09-12T23:50:15.922638+0000 mgr.y (mgr.24661) 388 : audit [DBG] from='client.34482 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:50:17.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:17 smithi181 bash[67587]: audit 2024-09-12T23:50:16.526752+0000 mon.a (mon.0) 1249 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:50:17.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:17 smithi181 bash[67587]: cluster 2024-09-12T23:50:16.846306+0000 mgr.y (mgr.24661) 389 : cluster [DBG] pgmap v705: 121 pgs: 1 active+clean+snaptrim_wait, 120 active+clean; 250 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.3 MiB/s rd, 792 KiB/s wr, 16 op/s 2024-09-12T23:50:17.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:17 smithi181 bash[67587]: audit 2024-09-12T23:50:17.290394+0000 mon.a (mon.0) 1250 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:50:17.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:17 smithi181 bash[67587]: audit 2024-09-12T23:50:17.300783+0000 mon.a (mon.0) 1251 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:50:17.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:17 smithi181 bash[67587]: cluster 2024-09-12T23:50:17.502920+0000 mon.a (mon.0) 1252 : cluster [DBG] osdmap e750: 8 total, 8 up, 8 in 2024-09-12T23:50:17.902 INFO:tasks.rados.rados.0.smithi136.stdout:3495: expect (ObjNum 1020 snap 315 seq_num 1020) 2024-09-12T23:50:18.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:17 smithi136 bash[76371]: audit 2024-09-12T23:50:15.922638+0000 mgr.y (mgr.24661) 388 : audit [DBG] from='client.34482 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:50:18.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:17 smithi136 bash[76371]: audit 2024-09-12T23:50:16.526752+0000 mon.a (mon.0) 1249 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:50:18.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:17 smithi136 bash[76371]: cluster 2024-09-12T23:50:16.846306+0000 mgr.y (mgr.24661) 389 : cluster [DBG] pgmap v705: 121 pgs: 1 active+clean+snaptrim_wait, 120 active+clean; 250 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.3 MiB/s rd, 792 KiB/s wr, 16 op/s 2024-09-12T23:50:18.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:17 smithi136 bash[76371]: audit 2024-09-12T23:50:17.290394+0000 mon.a (mon.0) 1250 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:50:18.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:17 smithi136 bash[76371]: audit 2024-09-12T23:50:17.300783+0000 mon.a (mon.0) 1251 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:50:18.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:17 smithi136 bash[76371]: cluster 2024-09-12T23:50:17.502920+0000 mon.a (mon.0) 1252 : cluster [DBG] osdmap e750: 8 total, 8 up, 8 in 2024-09-12T23:50:18.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:17 smithi136 bash[75164]: audit 2024-09-12T23:50:15.922638+0000 mgr.y (mgr.24661) 388 : audit [DBG] from='client.34482 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:50:18.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:17 smithi136 bash[75164]: audit 2024-09-12T23:50:16.526752+0000 mon.a (mon.0) 1249 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:50:18.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:17 smithi136 bash[75164]: cluster 2024-09-12T23:50:16.846306+0000 mgr.y (mgr.24661) 389 : cluster [DBG] pgmap v705: 121 pgs: 1 active+clean+snaptrim_wait, 120 active+clean; 250 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.3 MiB/s rd, 792 KiB/s wr, 16 op/s 2024-09-12T23:50:18.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:17 smithi136 bash[75164]: audit 2024-09-12T23:50:17.290394+0000 mon.a (mon.0) 1250 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:50:18.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:17 smithi136 bash[75164]: audit 2024-09-12T23:50:17.300783+0000 mon.a (mon.0) 1251 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:50:18.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:17 smithi136 bash[75164]: cluster 2024-09-12T23:50:17.502920+0000 mon.a (mon.0) 1252 : cluster [DBG] osdmap e750: 8 total, 8 up, 8 in 2024-09-12T23:50:18.184 INFO:tasks.rados.rados.0.smithi136.stdout:3493: done (4 left) 2024-09-12T23:50:18.185 INFO:tasks.rados.rados.0.smithi136.stdout:3494: done (3 left) 2024-09-12T23:50:18.185 INFO:tasks.rados.rados.0.smithi136.stdout:3495: done (2 left) 2024-09-12T23:50:18.185 INFO:tasks.rados.rados.0.smithi136.stdout:3497: done (1 left) 2024-09-12T23:50:18.185 INFO:tasks.rados.rados.0.smithi136.stdout:3498: done (0 left) 2024-09-12T23:50:18.185 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 43 current snap is 333 2024-09-12T23:50:18.185 INFO:tasks.rados.rados.0.smithi136.stdout:3499: seq_num 1104 ranges {0=499712,499712=712704,1212416=491520} 2024-09-12T23:50:18.187 INFO:tasks.rados.rados.0.smithi136.stdout:3499: writing smithi13667081-43 from 0 to 499712 tid 1 2024-09-12T23:50:18.191 INFO:tasks.rados.rados.0.smithi136.stdout:3499: writing smithi13667081-43 from 499712 to 1212416 tid 2 2024-09-12T23:50:18.194 INFO:tasks.rados.rados.0.smithi136.stdout:3499: writing smithi13667081-43 from 1212416 to 1703936 tid 3 2024-09-12T23:50:18.194 INFO:tasks.rados.rados.0.smithi136.stdout:3500: snap_create 2024-09-12T23:50:18.198 INFO:tasks.rados.rados.0.smithi136.stdout:3499: finishing write tid 1 to smithi13667081-43 2024-09-12T23:50:18.204 INFO:tasks.rados.rados.0.smithi136.stdout:3499: finishing write tid 2 to smithi13667081-43 2024-09-12T23:50:18.204 INFO:tasks.rados.rados.0.smithi136.stdout:3499: finishing write tid 3 to smithi13667081-43 2024-09-12T23:50:18.205 INFO:tasks.rados.rados.0.smithi136.stdout:3499: finishing write tid 4 to smithi13667081-43 2024-09-12T23:50:18.208 INFO:tasks.rados.rados.0.smithi136.stdout:3499: finishing write tid 5 to smithi13667081-43 2024-09-12T23:50:18.208 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 43 v 2477 (ObjNum 1104 snap 333 seq_num 1104) dirty exists 2024-09-12T23:50:18.208 INFO:tasks.rados.rados.0.smithi136.stdout:3499: left oid 43 (ObjNum 1104 snap 333 seq_num 1104) 2024-09-12T23:50:18.557 INFO:tasks.rados.rados.0.smithi136.stdout:3499: done (1 left) 2024-09-12T23:50:18.558 INFO:tasks.rados.rados.0.smithi136.stdout:3500: done (0 left) 2024-09-12T23:50:18.558 INFO:tasks.rados.rados.0.smithi136.stdout:3501: rmattr oid 21 current snap is 334 2024-09-12T23:50:18.558 INFO:tasks.rados.rados.0.smithi136.stdout:3502: snap_remove snap 332 2024-09-12T23:50:19.570 INFO:tasks.rados.rados.0.smithi136.stdout:3502: done (1 left) 2024-09-12T23:50:19.570 INFO:tasks.rados.rados.0.smithi136.stdout:3503: rmattr oid 15 current snap is 334 2024-09-12T23:50:19.570 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 21 v 2488 (ObjNum 977 snap 303 seq_num 977) dirty exists 2024-09-12T23:50:19.570 INFO:tasks.rados.rados.0.smithi136.stdout:3501: done (1 left) 2024-09-12T23:50:19.571 INFO:tasks.rados.rados.0.smithi136.stdout:3504: setattr oid 4 current snap is 334 2024-09-12T23:50:19.576 INFO:tasks.rados.rados.0.smithi136.stdout:3505: read oid 1 snap 329 2024-09-12T23:50:19.576 INFO:tasks.rados.rados.0.smithi136.stdout:3505: expect (ObjNum 1059 snap 321 seq_num 1059) 2024-09-12T23:50:19.576 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 39 current snap is 334 2024-09-12T23:50:19.576 INFO:tasks.rados.rados.0.smithi136.stdout:3506: seq_num 1105 ranges {958464=516096,1474560=598016,2072576=606208} 2024-09-12T23:50:19.602 INFO:tasks.rados.rados.0.smithi136.stdout:3506: writing smithi13667081-39 from 958464 to 1474560 tid 1 2024-09-12T23:50:19.612 INFO:tasks.rados.rados.0.smithi136.stdout:3506: writing smithi13667081-39 from 1474560 to 2072576 tid 2 2024-09-12T23:50:19.620 INFO:tasks.rados.rados.0.smithi136.stdout:3506: writing smithi13667081-39 from 2072576 to 2678784 tid 3 2024-09-12T23:50:19.620 INFO:tasks.rados.rados.0.smithi136.stdout:3507: snap_create 2024-09-12T23:50:19.620 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 15 v 2100 (ObjNum 993 snap 307 seq_num 993) dirty exists 2024-09-12T23:50:19.620 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 4 v 1662 (ObjNum 1096 snap 332 seq_num 1096) dirty exists 2024-09-12T23:50:19.620 INFO:tasks.rados.rados.0.smithi136.stdout:3506: finishing write tid 1 to smithi13667081-39 2024-09-12T23:50:19.620 INFO:tasks.rados.rados.0.smithi136.stdout:3506: finishing write tid 2 to smithi13667081-39 2024-09-12T23:50:19.627 INFO:tasks.rados.rados.0.smithi136.stdout:3506: finishing write tid 3 to smithi13667081-39 2024-09-12T23:50:19.627 INFO:tasks.rados.rados.0.smithi136.stdout:3506: finishing write tid 4 to smithi13667081-39 2024-09-12T23:50:19.628 INFO:tasks.rados.rados.0.smithi136.stdout:3505: expect (ObjNum 1059 snap 321 seq_num 1059) 2024-09-12T23:50:19.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:50:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:50:19] "GET /metrics HTTP/1.1" 200 36440 "" "Prometheus/2.43.0" 2024-09-12T23:50:19.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:19 smithi136 bash[75164]: cluster 2024-09-12T23:50:18.559120+0000 mon.a (mon.0) 1253 : cluster [DBG] osdmap e751: 8 total, 8 up, 8 in 2024-09-12T23:50:19.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:19 smithi136 bash[75164]: cluster 2024-09-12T23:50:18.846984+0000 mgr.y (mgr.24661) 390 : cluster [DBG] pgmap v708: 121 pgs: 121 active+clean; 254 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 3.9 MiB/s wr, 16 op/s 2024-09-12T23:50:19.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:19 smithi136 bash[76371]: cluster 2024-09-12T23:50:18.559120+0000 mon.a (mon.0) 1253 : cluster [DBG] osdmap e751: 8 total, 8 up, 8 in 2024-09-12T23:50:19.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:19 smithi136 bash[76371]: cluster 2024-09-12T23:50:18.846984+0000 mgr.y (mgr.24661) 390 : cluster [DBG] pgmap v708: 121 pgs: 121 active+clean; 254 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 3.9 MiB/s wr, 16 op/s 2024-09-12T23:50:19.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:19 smithi181 bash[67587]: cluster 2024-09-12T23:50:18.559120+0000 mon.a (mon.0) 1253 : cluster [DBG] osdmap e751: 8 total, 8 up, 8 in 2024-09-12T23:50:19.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:19 smithi181 bash[67587]: cluster 2024-09-12T23:50:18.846984+0000 mgr.y (mgr.24661) 390 : cluster [DBG] pgmap v708: 121 pgs: 121 active+clean; 254 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 3.9 MiB/s wr, 16 op/s 2024-09-12T23:50:20.264 INFO:tasks.rados.rados.0.smithi136.stdout:3506: finishing write tid 5 to smithi13667081-39 2024-09-12T23:50:20.264 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 39 v 2448 (ObjNum 1105 snap 334 seq_num 1105) dirty exists 2024-09-12T23:50:20.264 INFO:tasks.rados.rados.0.smithi136.stdout:3506: left oid 39 (ObjNum 1105 snap 334 seq_num 1105) 2024-09-12T23:50:20.585 INFO:tasks.rados.rados.0.smithi136.stdout:3503: done (4 left) 2024-09-12T23:50:20.585 INFO:tasks.rados.rados.0.smithi136.stdout:3504: done (3 left) 2024-09-12T23:50:20.585 INFO:tasks.rados.rados.0.smithi136.stdout:3505: done (2 left) 2024-09-12T23:50:20.585 INFO:tasks.rados.rados.0.smithi136.stdout:3506: done (1 left) 2024-09-12T23:50:20.585 INFO:tasks.rados.rados.0.smithi136.stdout:3507: done (0 left) 2024-09-12T23:50:20.586 INFO:tasks.rados.rados.0.smithi136.stdout:3508: snap_remove snap 333 2024-09-12T23:50:20.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:20 smithi181 bash[67587]: cluster 2024-09-12T23:50:19.571758+0000 mon.a (mon.0) 1254 : cluster [DBG] osdmap e752: 8 total, 8 up, 8 in 2024-09-12T23:50:21.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:20 smithi136 bash[75164]: cluster 2024-09-12T23:50:19.571758+0000 mon.a (mon.0) 1254 : cluster [DBG] osdmap e752: 8 total, 8 up, 8 in 2024-09-12T23:50:21.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:20 smithi136 bash[76371]: cluster 2024-09-12T23:50:19.571758+0000 mon.a (mon.0) 1254 : cluster [DBG] osdmap e752: 8 total, 8 up, 8 in 2024-09-12T23:50:21.487 INFO:tasks.rados.rados.0.smithi136.stdout:3508: done (0 left) 2024-09-12T23:50:21.487 INFO:tasks.rados.rados.0.smithi136.stdout:3509: snap_create 2024-09-12T23:50:21.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:21 smithi181 bash[67587]: cluster 2024-09-12T23:50:20.587556+0000 mon.a (mon.0) 1255 : cluster [DBG] osdmap e753: 8 total, 8 up, 8 in 2024-09-12T23:50:21.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:21 smithi181 bash[67587]: cluster 2024-09-12T23:50:20.847825+0000 mgr.y (mgr.24661) 391 : cluster [DBG] pgmap v711: 121 pgs: 121 active+clean; 258 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 4.3 MiB/s wr, 9 op/s 2024-09-12T23:50:21.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:21 smithi181 bash[67587]: cluster 2024-09-12T23:50:21.488882+0000 mon.a (mon.0) 1256 : cluster [DBG] osdmap e754: 8 total, 8 up, 8 in 2024-09-12T23:50:22.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:21 smithi136 bash[75164]: cluster 2024-09-12T23:50:20.587556+0000 mon.a (mon.0) 1255 : cluster [DBG] osdmap e753: 8 total, 8 up, 8 in 2024-09-12T23:50:22.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:21 smithi136 bash[75164]: cluster 2024-09-12T23:50:20.847825+0000 mgr.y (mgr.24661) 391 : cluster [DBG] pgmap v711: 121 pgs: 121 active+clean; 258 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 4.3 MiB/s wr, 9 op/s 2024-09-12T23:50:22.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:21 smithi136 bash[75164]: cluster 2024-09-12T23:50:21.488882+0000 mon.a (mon.0) 1256 : cluster [DBG] osdmap e754: 8 total, 8 up, 8 in 2024-09-12T23:50:22.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:21 smithi136 bash[76371]: cluster 2024-09-12T23:50:20.587556+0000 mon.a (mon.0) 1255 : cluster [DBG] osdmap e753: 8 total, 8 up, 8 in 2024-09-12T23:50:22.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:21 smithi136 bash[76371]: cluster 2024-09-12T23:50:20.847825+0000 mgr.y (mgr.24661) 391 : cluster [DBG] pgmap v711: 121 pgs: 121 active+clean; 258 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 4.3 MiB/s wr, 9 op/s 2024-09-12T23:50:22.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:21 smithi136 bash[76371]: cluster 2024-09-12T23:50:21.488882+0000 mon.a (mon.0) 1256 : cluster [DBG] osdmap e754: 8 total, 8 up, 8 in 2024-09-12T23:50:22.490 INFO:tasks.rados.rados.0.smithi136.stdout:3509: done (0 left) 2024-09-12T23:50:22.490 INFO:tasks.rados.rados.0.smithi136.stdout:3510: read oid 11 snap -1 2024-09-12T23:50:22.490 INFO:tasks.rados.rados.0.smithi136.stdout:3510: expect (ObjNum 1056 snap 321 seq_num 1056) 2024-09-12T23:50:22.490 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 28 current snap is 336 2024-09-12T23:50:22.490 INFO:tasks.rados.rados.0.smithi136.stdout:3511: seq_num 1106 ranges {2695168=532480,3227648=483328,3710976=475136,4186112=245760} 2024-09-12T23:50:22.539 INFO:tasks.rados.rados.0.smithi136.stdout:3511: writing smithi13667081-28 from 2695168 to 3227648 tid 1 2024-09-12T23:50:22.546 INFO:tasks.rados.rados.0.smithi136.stdout:3511: writing smithi13667081-28 from 3227648 to 3710976 tid 2 2024-09-12T23:50:22.552 INFO:tasks.rados.rados.0.smithi136.stdout:3511: writing smithi13667081-28 from 3710976 to 4186112 tid 3 2024-09-12T23:50:22.555 INFO:tasks.rados.rados.0.smithi136.stdout:3511: writing smithi13667081-28 from 4186112 to 4431872 tid 4 2024-09-12T23:50:22.555 INFO:tasks.rados.rados.0.smithi136.stdout:3512: delete oid 46 current snap is 336 2024-09-12T23:50:22.555 INFO:tasks.rados.rados.0.smithi136.stdout:3510: expect (ObjNum 1056 snap 321 seq_num 1056) 2024-09-12T23:50:22.595 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:50:22 smithi181 bash[95505]: ts=2024-09-12T23:50:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:50:22.687 INFO:tasks.rados.rados.0.smithi136.stdout:3511: finishing write tid 1 to smithi13667081-28 2024-09-12T23:50:22.687 INFO:tasks.rados.rados.0.smithi136.stdout:3511: finishing write tid 2 to smithi13667081-28 2024-09-12T23:50:22.687 INFO:tasks.rados.rados.0.smithi136.stdout:3511: finishing write tid 3 to smithi13667081-28 2024-09-12T23:50:22.687 INFO:tasks.rados.rados.0.smithi136.stdout:3511: finishing write tid 4 to smithi13667081-28 2024-09-12T23:50:22.687 INFO:tasks.rados.rados.0.smithi136.stdout:3511: finishing write tid 5 to smithi13667081-28 2024-09-12T23:50:22.687 INFO:tasks.rados.rados.0.smithi136.stdout:3511: finishing write tid 6 to smithi13667081-28 2024-09-12T23:50:22.687 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 28 v 2482 (ObjNum 1106 snap 336 seq_num 1106) dirty exists 2024-09-12T23:50:22.688 INFO:tasks.rados.rados.0.smithi136.stdout:3511: left oid 28 (ObjNum 1106 snap 336 seq_num 1106) 2024-09-12T23:50:22.688 INFO:tasks.rados.rados.0.smithi136.stdout:3510: done (2 left) 2024-09-12T23:50:22.688 INFO:tasks.rados.rados.0.smithi136.stdout:3511: done (1 left) 2024-09-12T23:50:22.688 INFO:tasks.rados.rados.0.smithi136.stdout:3512: done (0 left) 2024-09-12T23:50:22.688 INFO:tasks.rados.rados.0.smithi136.stdout:3513: read oid 9 snap -1 2024-09-12T23:50:22.688 INFO:tasks.rados.rados.0.smithi136.stdout:3513: expect (ObjNum 1091 snap 329 seq_num 1091) 2024-09-12T23:50:22.688 INFO:tasks.rados.rados.0.smithi136.stdout:3514: copy_from oid 45 from oid 27 current snap is 336 2024-09-12T23:50:22.688 INFO:tasks.rados.rados.0.smithi136.stdout:3515: rollback oid 49 current snap is 336 2024-09-12T23:50:22.688 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 49 to 329 2024-09-12T23:50:22.688 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 12 current snap is 336 2024-09-12T23:50:22.688 INFO:tasks.rados.rados.0.smithi136.stdout:3516: seq_num 1108 ranges {7127040=565248,7692288=647168,8339456=483328,8822784=73728} 2024-09-12T23:50:22.765 INFO:tasks.rados.rados.0.smithi136.stdout:3516: writing smithi13667081-12 from 7127040 to 7692288 tid 1 2024-09-12T23:50:22.770 INFO:tasks.rados.rados.0.smithi136.stdout:3516: writing smithi13667081-12 from 7692288 to 8339456 tid 2 2024-09-12T23:50:22.774 INFO:tasks.rados.rados.0.smithi136.stdout:3516: writing smithi13667081-12 from 8339456 to 8822784 tid 3 2024-09-12T23:50:22.775 INFO:tasks.rados.rados.0.smithi136.stdout:3516: writing smithi13667081-12 from 8822784 to 8896512 tid 4 2024-09-12T23:50:22.775 INFO:tasks.rados.rados.0.smithi136.stdout:3517: read oid 3 snap -1 2024-09-12T23:50:22.775 INFO:tasks.rados.rados.0.smithi136.stdout:3517: expect (ObjNum 1099 snap 332 seq_num 1099) 2024-09-12T23:50:22.775 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 1 current snap is 336 2024-09-12T23:50:22.775 INFO:tasks.rados.rados.0.smithi136.stdout:3518: seq_num 1109 ranges {8052736=581632,8634368=532480,9166848=647168,9814016=24576} 2024-09-12T23:50:22.824 INFO:tasks.rados.rados.0.smithi136.stdout:3518: writing smithi13667081-1 from 8052736 to 8634368 tid 1 2024-09-12T23:50:22.827 INFO:tasks.rados.rados.0.smithi136.stdout:3518: writing smithi13667081-1 from 8634368 to 9166848 tid 2 2024-09-12T23:50:22.831 INFO:tasks.rados.rados.0.smithi136.stdout:3518: writing smithi13667081-1 from 9166848 to 9814016 tid 3 2024-09-12T23:50:22.831 INFO:tasks.rados.rados.0.smithi136.stdout:3518: writing smithi13667081-1 from 9814016 to 9838592 tid 4 2024-09-12T23:50:22.832 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 7 current snap is 336 2024-09-12T23:50:22.832 INFO:tasks.rados.rados.0.smithi136.stdout:3519: seq_num 1110 ranges {8167424=598016,8765440=466944,9232384=450560,9682944=286720} 2024-09-12T23:50:22.881 INFO:tasks.rados.rados.0.smithi136.stdout:3519: writing smithi13667081-7 from 8167424 to 8765440 tid 1 2024-09-12T23:50:22.883 INFO:tasks.rados.rados.0.smithi136.stdout:3519: writing smithi13667081-7 from 8765440 to 9232384 tid 2 2024-09-12T23:50:22.886 INFO:tasks.rados.rados.0.smithi136.stdout:3519: writing smithi13667081-7 from 9232384 to 9682944 tid 3 2024-09-12T23:50:22.887 INFO:tasks.rados.rados.0.smithi136.stdout:3519: writing smithi13667081-7 from 9682944 to 9969664 tid 4 2024-09-12T23:50:22.888 INFO:tasks.rados.rados.0.smithi136.stdout:3520: read oid 22 snap 296 2024-09-12T23:50:22.888 INFO:tasks.rados.rados.0.smithi136.stdout:3520: expect (ObjNum 896 snap 288 seq_num 896) 2024-09-12T23:50:22.888 INFO:tasks.rados.rados.0.smithi136.stdout:3521: snap_create 2024-09-12T23:50:22.888 INFO:tasks.rados.rados.0.smithi136.stdout:3515: finishing rollback tid 1 to smithi13667081-49 2024-09-12T23:50:22.888 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 49 v 0 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-12T23:50:22.888 INFO:tasks.rados.rados.0.smithi136.stdout:3514: finishing copy_from to smithi13667081-45 2024-09-12T23:50:22.888 INFO:tasks.rados.rados.0.smithi136.stdout:3514: got expected ENOENT (src dne) 2024-09-12T23:50:22.888 INFO:tasks.rados.rados.0.smithi136.stdout:3514: finishing copy_from racing read to smithi13667081-45 2024-09-12T23:50:22.888 INFO:tasks.rados.rados.0.smithi136.stdout:3513: expect (ObjNum 1091 snap 329 seq_num 1091) 2024-09-12T23:50:23.046 INFO:tasks.rados.rados.0.smithi136.stdout:3516: finishing write tid 1 to smithi13667081-12 2024-09-12T23:50:23.046 INFO:tasks.rados.rados.0.smithi136.stdout:3516: finishing write tid 2 to smithi13667081-12 2024-09-12T23:50:23.047 INFO:tasks.rados.rados.0.smithi136.stdout:3516: finishing write tid 3 to smithi13667081-12 2024-09-12T23:50:23.047 INFO:tasks.rados.rados.0.smithi136.stdout:3516: finishing write tid 4 to smithi13667081-12 2024-09-12T23:50:23.047 INFO:tasks.rados.rados.0.smithi136.stdout:3516: finishing write tid 5 to smithi13667081-12 2024-09-12T23:50:23.047 INFO:tasks.rados.rados.0.smithi136.stdout:3516: finishing write tid 6 to smithi13667081-12 2024-09-12T23:50:23.047 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 12 v 2105 (ObjNum 1108 snap 336 seq_num 1108) dirty exists 2024-09-12T23:50:23.047 INFO:tasks.rados.rados.0.smithi136.stdout:3516: left oid 12 (ObjNum 1108 snap 336 seq_num 1108) 2024-09-12T23:50:23.047 INFO:tasks.rados.rados.0.smithi136.stdout:3517: expect (ObjNum 1099 snap 332 seq_num 1099) 2024-09-12T23:50:23.261 INFO:tasks.rados.rados.0.smithi136.stdout:3518: finishing write tid 1 to smithi13667081-1 2024-09-12T23:50:23.261 INFO:tasks.rados.rados.0.smithi136.stdout:3518: finishing write tid 2 to smithi13667081-1 2024-09-12T23:50:23.261 INFO:tasks.rados.rados.0.smithi136.stdout:3518: finishing write tid 3 to smithi13667081-1 2024-09-12T23:50:23.261 INFO:tasks.rados.rados.0.smithi136.stdout:3518: finishing write tid 4 to smithi13667081-1 2024-09-12T23:50:23.262 INFO:tasks.rados.rados.0.smithi136.stdout:3518: finishing write tid 5 to smithi13667081-1 2024-09-12T23:50:23.262 INFO:tasks.rados.rados.0.smithi136.stdout:3518: finishing write tid 6 to smithi13667081-1 2024-09-12T23:50:23.262 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 1 v 2493 (ObjNum 1109 snap 336 seq_num 1109) dirty exists 2024-09-12T23:50:23.262 INFO:tasks.rados.rados.0.smithi136.stdout:3518: left oid 1 (ObjNum 1109 snap 336 seq_num 1109) 2024-09-12T23:50:23.262 INFO:tasks.rados.rados.0.smithi136.stdout:3519: finishing write tid 1 to smithi13667081-7 2024-09-12T23:50:23.262 INFO:tasks.rados.rados.0.smithi136.stdout:3519: finishing write tid 2 to smithi13667081-7 2024-09-12T23:50:23.262 INFO:tasks.rados.rados.0.smithi136.stdout:3519: finishing write tid 3 to smithi13667081-7 2024-09-12T23:50:23.262 INFO:tasks.rados.rados.0.smithi136.stdout:3520: expect (ObjNum 896 snap 288 seq_num 896) 2024-09-12T23:50:23.277 INFO:tasks.rados.rados.0.smithi136.stdout:3519: finishing write tid 4 to smithi13667081-7 2024-09-12T23:50:23.277 INFO:tasks.rados.rados.0.smithi136.stdout:3519: finishing write tid 5 to smithi13667081-7 2024-09-12T23:50:23.277 INFO:tasks.rados.rados.0.smithi136.stdout:3519: finishing write tid 6 to smithi13667081-7 2024-09-12T23:50:23.277 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 7 v 1646 (ObjNum 1110 snap 336 seq_num 1110) dirty exists 2024-09-12T23:50:23.277 INFO:tasks.rados.rados.0.smithi136.stdout:3519: left oid 7 (ObjNum 1110 snap 336 seq_num 1110) 2024-09-12T23:50:23.498 INFO:tasks.rados.rados.0.smithi136.stdout:3513: done (8 left) 2024-09-12T23:50:23.498 INFO:tasks.rados.rados.0.smithi136.stdout:3514: done (7 left) 2024-09-12T23:50:23.499 INFO:tasks.rados.rados.0.smithi136.stdout:3515: done (6 left) 2024-09-12T23:50:23.499 INFO:tasks.rados.rados.0.smithi136.stdout:3516: done (5 left) 2024-09-12T23:50:23.499 INFO:tasks.rados.rados.0.smithi136.stdout:3517: done (4 left) 2024-09-12T23:50:23.499 INFO:tasks.rados.rados.0.smithi136.stdout:3518: done (3 left) 2024-09-12T23:50:23.499 INFO:tasks.rados.rados.0.smithi136.stdout:3519: done (2 left) 2024-09-12T23:50:23.499 INFO:tasks.rados.rados.0.smithi136.stdout:3520: done (1 left) 2024-09-12T23:50:23.499 INFO:tasks.rados.rados.0.smithi136.stdout:3521: done (0 left) 2024-09-12T23:50:23.499 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 20 current snap is 337 2024-09-12T23:50:23.499 INFO:tasks.rados.rados.0.smithi136.stdout:3522: seq_num 1111 ranges {1515520=614400,2129920=753664,2883584=450560} 2024-09-12T23:50:23.510 INFO:tasks.rados.rados.0.smithi136.stdout:3522: writing smithi13667081-20 from 1515520 to 2129920 tid 1 2024-09-12T23:50:23.515 INFO:tasks.rados.rados.0.smithi136.stdout:3522: writing smithi13667081-20 from 2129920 to 2883584 tid 2 2024-09-12T23:50:23.518 INFO:tasks.rados.rados.0.smithi136.stdout:3522: writing smithi13667081-20 from 2883584 to 3334144 tid 3 2024-09-12T23:50:23.518 INFO:tasks.rados.rados.0.smithi136.stdout:3523: read oid 7 snap -1 2024-09-12T23:50:23.518 INFO:tasks.rados.rados.0.smithi136.stdout:3523: expect (ObjNum 1110 snap 336 seq_num 1110) 2024-09-12T23:50:23.518 INFO:tasks.rados.rados.0.smithi136.stdout:3524: rollback oid 37 current snap is 337 2024-09-12T23:50:23.519 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 37 to 296 2024-09-12T23:50:23.519 INFO:tasks.rados.rados.0.smithi136.stdout:3525: snap_create 2024-09-12T23:50:23.525 INFO:tasks.rados.rados.0.smithi136.stdout:3522: finishing write tid 1 to smithi13667081-20 2024-09-12T23:50:23.526 INFO:tasks.rados.rados.0.smithi136.stdout:3524: finishing rollback tid 0 to smithi13667081-37 2024-09-12T23:50:23.527 INFO:tasks.rados.rados.0.smithi136.stdout:3524: finishing rollback tid 1 to smithi13667081-37 2024-09-12T23:50:23.527 INFO:tasks.rados.rados.0.smithi136.stdout:3522: finishing write tid 2 to smithi13667081-20 2024-09-12T23:50:23.527 INFO:tasks.rados.rados.0.smithi136.stdout:3524: finishing rollback tid 2 to smithi13667081-37 2024-09-12T23:50:23.527 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 37 v 2492 (ObjNum 636 snap 188 seq_num 636) dirty exists 2024-09-12T23:50:23.527 INFO:tasks.rados.rados.0.smithi136.stdout:3522: finishing write tid 3 to smithi13667081-20 2024-09-12T23:50:23.527 INFO:tasks.rados.rados.0.smithi136.stdout:3522: finishing write tid 4 to smithi13667081-20 2024-09-12T23:50:23.531 INFO:tasks.rados.rados.0.smithi136.stdout:3522: finishing write tid 5 to smithi13667081-20 2024-09-12T23:50:23.531 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 20 v 1653 (ObjNum 1111 snap 337 seq_num 1111) dirty exists 2024-09-12T23:50:23.531 INFO:tasks.rados.rados.0.smithi136.stdout:3522: left oid 20 (ObjNum 1111 snap 337 seq_num 1111) 2024-09-12T23:50:23.538 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:23 smithi181 bash[67587]: cluster 2024-09-12T23:50:22.490172+0000 mon.a (mon.0) 1257 : cluster [DBG] osdmap e755: 8 total, 8 up, 8 in 2024-09-12T23:50:23.538 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:23 smithi181 bash[67587]: cluster 2024-09-12T23:50:22.848380+0000 mgr.y (mgr.24661) 392 : cluster [DBG] pgmap v714: 121 pgs: 121 active+clean; 260 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.7 MiB/s wr, 17 op/s 2024-09-12T23:50:23.582 INFO:tasks.rados.rados.0.smithi136.stdout:3523: expect (ObjNum 1110 snap 336 seq_num 1110) 2024-09-12T23:50:23.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:23 smithi136 bash[76371]: cluster 2024-09-12T23:50:22.490172+0000 mon.a (mon.0) 1257 : cluster [DBG] osdmap e755: 8 total, 8 up, 8 in 2024-09-12T23:50:23.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:23 smithi136 bash[76371]: cluster 2024-09-12T23:50:22.848380+0000 mgr.y (mgr.24661) 392 : cluster [DBG] pgmap v714: 121 pgs: 121 active+clean; 260 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.7 MiB/s wr, 17 op/s 2024-09-12T23:50:23.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:23 smithi136 bash[75164]: cluster 2024-09-12T23:50:22.490172+0000 mon.a (mon.0) 1257 : cluster [DBG] osdmap e755: 8 total, 8 up, 8 in 2024-09-12T23:50:23.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:23 smithi136 bash[75164]: cluster 2024-09-12T23:50:22.848380+0000 mgr.y (mgr.24661) 392 : cluster [DBG] pgmap v714: 121 pgs: 121 active+clean; 260 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.7 MiB/s wr, 17 op/s 2024-09-12T23:50:24.511 INFO:tasks.rados.rados.0.smithi136.stdout:3522: done (3 left) 2024-09-12T23:50:24.511 INFO:tasks.rados.rados.0.smithi136.stdout:3523: done (2 left) 2024-09-12T23:50:24.512 INFO:tasks.rados.rados.0.smithi136.stdout:3524: done (1 left) 2024-09-12T23:50:24.513 INFO:tasks.rados.rados.0.smithi136.stdout:3525: done (0 left) 2024-09-12T23:50:24.513 INFO:tasks.rados.rados.0.smithi136.stdout:3526: snap_create 2024-09-12T23:50:24.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:24 smithi136 bash[75164]: cluster 2024-09-12T23:50:23.501394+0000 mon.a (mon.0) 1258 : cluster [DBG] osdmap e756: 8 total, 8 up, 8 in 2024-09-12T23:50:24.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:24 smithi136 bash[75164]: audit 2024-09-12T23:50:23.752932+0000 mon.a (mon.0) 1259 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:50:24.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:24 smithi136 bash[76371]: cluster 2024-09-12T23:50:23.501394+0000 mon.a (mon.0) 1258 : cluster [DBG] osdmap e756: 8 total, 8 up, 8 in 2024-09-12T23:50:24.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:24 smithi136 bash[76371]: audit 2024-09-12T23:50:23.752932+0000 mon.a (mon.0) 1259 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:50:24.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:24 smithi181 bash[67587]: cluster 2024-09-12T23:50:23.501394+0000 mon.a (mon.0) 1258 : cluster [DBG] osdmap e756: 8 total, 8 up, 8 in 2024-09-12T23:50:24.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:24 smithi181 bash[67587]: audit 2024-09-12T23:50:23.752932+0000 mon.a (mon.0) 1259 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:50:25.513 INFO:tasks.rados.rados.0.smithi136.stdout:3526: done (0 left) 2024-09-12T23:50:25.513 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 8 current snap is 339 2024-09-12T23:50:25.513 INFO:tasks.rados.rados.0.smithi136.stdout:3527: seq_num 1112 ranges {0=630784,630784=638976,1269760=466944,1736704=98304} 2024-09-12T23:50:25.516 INFO:tasks.rados.rados.0.smithi136.stdout:3527: writing smithi13667081-8 from 0 to 630784 tid 1 2024-09-12T23:50:25.520 INFO:tasks.rados.rados.0.smithi136.stdout:3527: writing smithi13667081-8 from 630784 to 1269760 tid 2 2024-09-12T23:50:25.523 INFO:tasks.rados.rados.0.smithi136.stdout:3527: writing smithi13667081-8 from 1269760 to 1736704 tid 3 2024-09-12T23:50:25.524 INFO:tasks.rados.rados.0.smithi136.stdout:3527: writing smithi13667081-8 from 1736704 to 1835008 tid 4 2024-09-12T23:50:25.524 INFO:tasks.rados.rados.0.smithi136.stdout:3528: rollback oid 37 current snap is 339 2024-09-12T23:50:25.524 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 37 to 334 2024-09-12T23:50:25.524 INFO:tasks.rados.rados.0.smithi136.stdout:3529: snap_create 2024-09-12T23:50:25.530 INFO:tasks.rados.rados.0.smithi136.stdout:3527: finishing write tid 1 to smithi13667081-8 2024-09-12T23:50:25.532 INFO:tasks.rados.rados.0.smithi136.stdout:3527: finishing write tid 2 to smithi13667081-8 2024-09-12T23:50:25.532 INFO:tasks.rados.rados.0.smithi136.stdout:3527: finishing write tid 3 to smithi13667081-8 2024-09-12T23:50:25.533 INFO:tasks.rados.rados.0.smithi136.stdout:3527: finishing write tid 4 to smithi13667081-8 2024-09-12T23:50:25.534 INFO:tasks.rados.rados.0.smithi136.stdout:3527: finishing write tid 5 to smithi13667081-8 2024-09-12T23:50:25.534 INFO:tasks.rados.rados.0.smithi136.stdout:3528: finishing rollback tid 0 to smithi13667081-37 2024-09-12T23:50:25.534 INFO:tasks.rados.rados.0.smithi136.stdout:3528: finishing rollback tid 1 to smithi13667081-37 2024-09-12T23:50:25.538 INFO:tasks.rados.rados.0.smithi136.stdout:3527: finishing write tid 6 to smithi13667081-8 2024-09-12T23:50:25.538 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 8 v 2358 (ObjNum 1112 snap 339 seq_num 1112) dirty exists 2024-09-12T23:50:25.538 INFO:tasks.rados.rados.0.smithi136.stdout:3527: left oid 8 (ObjNum 1112 snap 339 seq_num 1112) 2024-09-12T23:50:25.538 INFO:tasks.rados.rados.0.smithi136.stdout:3528: finishing rollback tid 2 to smithi13667081-37 2024-09-12T23:50:25.538 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 37 v 2495 (ObjNum 1020 snap 315 seq_num 1020) dirty exists 2024-09-12T23:50:25.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:25 smithi181 bash[67587]: cluster 2024-09-12T23:50:24.514278+0000 mon.a (mon.0) 1260 : cluster [DBG] osdmap e757: 8 total, 8 up, 8 in 2024-09-12T23:50:25.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:25 smithi181 bash[67587]: cluster 2024-09-12T23:50:24.849219+0000 mgr.y (mgr.24661) 393 : cluster [DBG] pgmap v717: 121 pgs: 121 active+clean; 261 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 852 KiB/s wr, 21 op/s 2024-09-12T23:50:26.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:25 smithi136 bash[75164]: cluster 2024-09-12T23:50:24.514278+0000 mon.a (mon.0) 1260 : cluster [DBG] osdmap e757: 8 total, 8 up, 8 in 2024-09-12T23:50:26.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:25 smithi136 bash[75164]: cluster 2024-09-12T23:50:24.849219+0000 mgr.y (mgr.24661) 393 : cluster [DBG] pgmap v717: 121 pgs: 121 active+clean; 261 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 852 KiB/s wr, 21 op/s 2024-09-12T23:50:26.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:25 smithi136 bash[76371]: cluster 2024-09-12T23:50:24.514278+0000 mon.a (mon.0) 1260 : cluster [DBG] osdmap e757: 8 total, 8 up, 8 in 2024-09-12T23:50:26.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:25 smithi136 bash[76371]: cluster 2024-09-12T23:50:24.849219+0000 mgr.y (mgr.24661) 393 : cluster [DBG] pgmap v717: 121 pgs: 121 active+clean; 261 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 852 KiB/s wr, 21 op/s 2024-09-12T23:50:26.490 INFO:tasks.rados.rados.0.smithi136.stdout:3527: done (2 left) 2024-09-12T23:50:26.490 INFO:tasks.rados.rados.0.smithi136.stdout:3528: done (1 left) 2024-09-12T23:50:26.490 INFO:tasks.rados.rados.0.smithi136.stdout:3529: done (0 left) 2024-09-12T23:50:26.491 INFO:tasks.rados.rados.0.smithi136.stdout:3530: setattr oid 17 current snap is 340 2024-09-12T23:50:26.491 INFO:tasks.rados.rados.0.smithi136.stdout:3531: snap_create 2024-09-12T23:50:26.500 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 17 v 2749 (ObjNum 1007 snap 313 seq_num 1007) dirty exists 2024-09-12T23:50:26.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:26 smithi181 bash[67587]: cluster 2024-09-12T23:50:25.515155+0000 mon.a (mon.0) 1261 : cluster [DBG] osdmap e758: 8 total, 8 up, 8 in 2024-09-12T23:50:26.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:26 smithi181 bash[67587]: cluster 2024-09-12T23:50:26.488264+0000 mon.a (mon.0) 1262 : cluster [DBG] osdmap e759: 8 total, 8 up, 8 in 2024-09-12T23:50:27.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:26 smithi136 bash[75164]: cluster 2024-09-12T23:50:25.515155+0000 mon.a (mon.0) 1261 : cluster [DBG] osdmap e758: 8 total, 8 up, 8 in 2024-09-12T23:50:27.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:26 smithi136 bash[75164]: cluster 2024-09-12T23:50:26.488264+0000 mon.a (mon.0) 1262 : cluster [DBG] osdmap e759: 8 total, 8 up, 8 in 2024-09-12T23:50:27.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:26 smithi136 bash[76371]: cluster 2024-09-12T23:50:25.515155+0000 mon.a (mon.0) 1261 : cluster [DBG] osdmap e758: 8 total, 8 up, 8 in 2024-09-12T23:50:27.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:26 smithi136 bash[76371]: cluster 2024-09-12T23:50:26.488264+0000 mon.a (mon.0) 1262 : cluster [DBG] osdmap e759: 8 total, 8 up, 8 in 2024-09-12T23:50:27.490 INFO:tasks.rados.rados.0.smithi136.stdout:3530: done (1 left) 2024-09-12T23:50:27.490 INFO:tasks.rados.rados.0.smithi136.stdout:3531: done (0 left) 2024-09-12T23:50:27.490 INFO:tasks.rados.rados.0.smithi136.stdout:3532: snap_create 2024-09-12T23:50:28.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:26.845576+0000 mon.a (mon.0) 1263 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:50:28.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: cluster 2024-09-12T23:50:26.850163+0000 mgr.y (mgr.24661) 394 : cluster [DBG] pgmap v720: 121 pgs: 121 active+clean; 274 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 2.1 MiB/s wr, 16 op/s 2024-09-12T23:50:28.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:26.857037+0000 mon.a (mon.0) 1264 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:50:28.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:26.859068+0000 mon.a (mon.0) 1265 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:50:28.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:26.860735+0000 mon.a (mon.0) 1266 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:50:28.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:26.871310+0000 mon.a (mon.0) 1267 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:50:28.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:26.941736+0000 mon.a (mon.0) 1268 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-12T23:50:28.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:26.993856+0000 mon.a (mon.0) 1269 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:50:28.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:26.998372+0000 mon.a (mon.0) 1270 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.000904+0000 mon.a (mon.0) 1271 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.003014+0000 mon.a (mon.0) 1272 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.006224+0000 mon.a (mon.0) 1273 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.010031+0000 mon.a (mon.0) 1274 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.011975+0000 mon.a (mon.0) 1275 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.013705+0000 mon.a (mon.0) 1276 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.015415+0000 mon.a (mon.0) 1277 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.017214+0000 mon.a (mon.0) 1278 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.019274+0000 mon.a (mon.0) 1279 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.021222+0000 mon.a (mon.0) 1280 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:26.845576+0000 mon.a (mon.0) 1263 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:50:28.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: cluster 2024-09-12T23:50:26.850163+0000 mgr.y (mgr.24661) 394 : cluster [DBG] pgmap v720: 121 pgs: 121 active+clean; 274 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 2.1 MiB/s wr, 16 op/s 2024-09-12T23:50:28.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:26.857037+0000 mon.a (mon.0) 1264 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:50:28.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:26.859068+0000 mon.a (mon.0) 1265 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:50:28.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:26.860735+0000 mon.a (mon.0) 1266 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:50:28.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:26.871310+0000 mon.a (mon.0) 1267 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:50:28.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:26.941736+0000 mon.a (mon.0) 1268 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-12T23:50:28.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:26.993856+0000 mon.a (mon.0) 1269 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:50:28.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:26.998372+0000 mon.a (mon.0) 1270 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.000904+0000 mon.a (mon.0) 1271 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.003014+0000 mon.a (mon.0) 1272 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.006224+0000 mon.a (mon.0) 1273 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.010031+0000 mon.a (mon.0) 1274 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.011975+0000 mon.a (mon.0) 1275 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.013705+0000 mon.a (mon.0) 1276 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.015415+0000 mon.a (mon.0) 1277 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.017214+0000 mon.a (mon.0) 1278 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.019274+0000 mon.a (mon.0) 1279 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.021222+0000 mon.a (mon.0) 1280 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.025065+0000 mon.a (mon.0) 1281 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.027956+0000 mon.a (mon.0) 1282 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.030438+0000 mon.a (mon.0) 1283 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.033256+0000 mon.a (mon.0) 1284 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.035655+0000 mon.a (mon.0) 1285 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.038062+0000 mon.a (mon.0) 1286 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.040099+0000 mon.a (mon.0) 1287 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.050331+0000 mon.a (mon.0) 1288 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:50:28.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.051769+0000 mon.a (mon.0) 1289 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-12T23:50:28.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.059545+0000 mon.a (mon.0) 1290 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-12T23:50:28.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.061085+0000 mon.a (mon.0) 1291 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-12T23:50:28.262 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.069339+0000 mon.a (mon.0) 1292 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-12T23:50:28.262 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.071045+0000 mon.a (mon.0) 1293 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-12T23:50:28.262 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.079023+0000 mon.a (mon.0) 1294 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-12T23:50:28.262 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.080764+0000 mon.a (mon.0) 1295 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-12T23:50:28.262 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.090851+0000 mon.a (mon.0) 1296 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-12T23:50:28.262 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.092271+0000 mon.a (mon.0) 1297 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-12T23:50:28.262 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.102361+0000 mon.a (mon.0) 1298 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-12T23:50:28.262 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.025065+0000 mon.a (mon.0) 1281 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.262 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.027956+0000 mon.a (mon.0) 1282 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.262 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.030438+0000 mon.a (mon.0) 1283 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.263 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.033256+0000 mon.a (mon.0) 1284 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.263 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.035655+0000 mon.a (mon.0) 1285 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.263 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.038062+0000 mon.a (mon.0) 1286 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.263 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.040099+0000 mon.a (mon.0) 1287 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.263 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.050331+0000 mon.a (mon.0) 1288 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:50:28.263 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.051769+0000 mon.a (mon.0) 1289 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-12T23:50:28.263 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.059545+0000 mon.a (mon.0) 1290 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-12T23:50:28.263 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.061085+0000 mon.a (mon.0) 1291 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-12T23:50:28.263 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.069339+0000 mon.a (mon.0) 1292 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-12T23:50:28.263 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.071045+0000 mon.a (mon.0) 1293 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-12T23:50:28.263 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.079023+0000 mon.a (mon.0) 1294 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-12T23:50:28.263 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.080764+0000 mon.a (mon.0) 1295 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-12T23:50:28.263 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.090851+0000 mon.a (mon.0) 1296 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-12T23:50:28.264 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.092271+0000 mon.a (mon.0) 1297 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-12T23:50:28.264 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.102361+0000 mon.a (mon.0) 1298 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-12T23:50:28.264 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.103829+0000 mon.a (mon.0) 1299 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-12T23:50:28.264 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.113957+0000 mon.a (mon.0) 1300 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-12T23:50:28.264 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.115506+0000 mon.a (mon.0) 1301 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-12T23:50:28.264 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.125958+0000 mon.a (mon.0) 1302 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-12T23:50:28.264 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.127732+0000 mon.a (mon.0) 1303 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-12T23:50:28.264 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.129172+0000 mon.a (mon.0) 1304 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-12T23:50:28.264 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.138263+0000 mon.a (mon.0) 1305 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-12T23:50:28.264 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.140001+0000 mon.a (mon.0) 1306 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-12T23:50:28.265 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.148232+0000 mon.a (mon.0) 1307 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-12T23:50:28.265 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.149969+0000 mon.a (mon.0) 1308 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-12T23:50:28.265 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.158143+0000 mon.a (mon.0) 1309 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-12T23:50:28.265 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.159780+0000 mon.a (mon.0) 1310 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-12T23:50:28.265 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.167115+0000 mon.a (mon.0) 1311 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-12T23:50:28.265 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.169058+0000 mon.a (mon.0) 1312 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-12T23:50:28.265 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.170437+0000 mon.a (mon.0) 1313 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-12T23:50:28.265 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.171755+0000 mon.a (mon.0) 1314 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-12T23:50:28.265 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.103829+0000 mon.a (mon.0) 1299 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-12T23:50:28.266 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.113957+0000 mon.a (mon.0) 1300 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-12T23:50:28.266 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.115506+0000 mon.a (mon.0) 1301 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-12T23:50:28.266 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.125958+0000 mon.a (mon.0) 1302 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-12T23:50:28.266 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.127732+0000 mon.a (mon.0) 1303 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-12T23:50:28.266 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.129172+0000 mon.a (mon.0) 1304 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-12T23:50:28.266 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.138263+0000 mon.a (mon.0) 1305 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-12T23:50:28.266 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.140001+0000 mon.a (mon.0) 1306 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-12T23:50:28.266 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.148232+0000 mon.a (mon.0) 1307 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-12T23:50:28.266 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.149969+0000 mon.a (mon.0) 1308 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-12T23:50:28.266 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.158143+0000 mon.a (mon.0) 1309 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-12T23:50:28.266 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.159780+0000 mon.a (mon.0) 1310 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-12T23:50:28.266 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.167115+0000 mon.a (mon.0) 1311 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-12T23:50:28.267 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.169058+0000 mon.a (mon.0) 1312 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-12T23:50:28.267 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.170437+0000 mon.a (mon.0) 1313 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-12T23:50:28.267 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.171755+0000 mon.a (mon.0) 1314 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-12T23:50:28.267 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.173101+0000 mon.a (mon.0) 1315 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-12T23:50:28.267 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.174406+0000 mon.a (mon.0) 1316 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-12T23:50:28.267 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.175654+0000 mon.a (mon.0) 1317 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-12T23:50:28.267 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.176981+0000 mon.a (mon.0) 1318 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]: dispatch 2024-09-12T23:50:28.267 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.184158+0000 mon.a (mon.0) 1319 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]': finished 2024-09-12T23:50:28.267 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.186806+0000 mon.a (mon.0) 1320 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-12T23:50:28.268 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.192762+0000 mon.a (mon.0) 1321 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-12T23:50:28.268 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.194025+0000 mon.a (mon.0) 1322 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:50:28.268 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.198680+0000 mon.a (mon.0) 1323 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:50:28.268 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.200366+0000 mon.a (mon.0) 1324 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:50:28.268 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.210934+0000 mon.a (mon.0) 1325 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:50:28.268 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.362164+0000 mon.a (mon.0) 1326 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:50:28.268 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.365069+0000 mon.a (mon.0) 1327 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:50:28.268 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.366483+0000 mon.a (mon.0) 1328 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:50:28.268 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: audit 2024-09-12T23:50:27.376257+0000 mon.a (mon.0) 1329 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:50:28.268 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[75164]: cluster 2024-09-12T23:50:27.492172+0000 mon.a (mon.0) 1330 : cluster [DBG] osdmap e760: 8 total, 8 up, 8 in 2024-09-12T23:50:28.268 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.173101+0000 mon.a (mon.0) 1315 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-12T23:50:28.269 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.174406+0000 mon.a (mon.0) 1316 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-12T23:50:28.269 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.175654+0000 mon.a (mon.0) 1317 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-12T23:50:28.269 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.176981+0000 mon.a (mon.0) 1318 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]: dispatch 2024-09-12T23:50:28.269 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.184158+0000 mon.a (mon.0) 1319 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]': finished 2024-09-12T23:50:28.269 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.186806+0000 mon.a (mon.0) 1320 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-12T23:50:28.269 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.192762+0000 mon.a (mon.0) 1321 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-12T23:50:28.269 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.194025+0000 mon.a (mon.0) 1322 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:50:28.269 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.198680+0000 mon.a (mon.0) 1323 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:50:28.269 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.200366+0000 mon.a (mon.0) 1324 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:50:28.269 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.210934+0000 mon.a (mon.0) 1325 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:50:28.269 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.362164+0000 mon.a (mon.0) 1326 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:50:28.269 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.365069+0000 mon.a (mon.0) 1327 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:50:28.270 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.366483+0000 mon.a (mon.0) 1328 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:50:28.270 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: audit 2024-09-12T23:50:27.376257+0000 mon.a (mon.0) 1329 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:50:28.270 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:27 smithi136 bash[76371]: cluster 2024-09-12T23:50:27.492172+0000 mon.a (mon.0) 1330 : cluster [DBG] osdmap e760: 8 total, 8 up, 8 in 2024-09-12T23:50:28.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:26.845576+0000 mon.a (mon.0) 1263 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:50:28.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: cluster 2024-09-12T23:50:26.850163+0000 mgr.y (mgr.24661) 394 : cluster [DBG] pgmap v720: 121 pgs: 121 active+clean; 274 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 2.1 MiB/s wr, 16 op/s 2024-09-12T23:50:28.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:26.857037+0000 mon.a (mon.0) 1264 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:50:28.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:26.859068+0000 mon.a (mon.0) 1265 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:50:28.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:26.860735+0000 mon.a (mon.0) 1266 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:50:28.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:26.871310+0000 mon.a (mon.0) 1267 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:50:28.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:26.941736+0000 mon.a (mon.0) 1268 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-12T23:50:28.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:26.993856+0000 mon.a (mon.0) 1269 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:50:28.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:26.998372+0000 mon.a (mon.0) 1270 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.000904+0000 mon.a (mon.0) 1271 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.003014+0000 mon.a (mon.0) 1272 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.006224+0000 mon.a (mon.0) 1273 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.010031+0000 mon.a (mon.0) 1274 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.011975+0000 mon.a (mon.0) 1275 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.013705+0000 mon.a (mon.0) 1276 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.015415+0000 mon.a (mon.0) 1277 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.017214+0000 mon.a (mon.0) 1278 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.019274+0000 mon.a (mon.0) 1279 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.021222+0000 mon.a (mon.0) 1280 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.025065+0000 mon.a (mon.0) 1281 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.027956+0000 mon.a (mon.0) 1282 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.030438+0000 mon.a (mon.0) 1283 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.033256+0000 mon.a (mon.0) 1284 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.035655+0000 mon.a (mon.0) 1285 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.038062+0000 mon.a (mon.0) 1286 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.040099+0000 mon.a (mon.0) 1287 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:50:28.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.050331+0000 mon.a (mon.0) 1288 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:50:28.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.051769+0000 mon.a (mon.0) 1289 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-12T23:50:28.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.059545+0000 mon.a (mon.0) 1290 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-12T23:50:28.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.061085+0000 mon.a (mon.0) 1291 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-12T23:50:28.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.069339+0000 mon.a (mon.0) 1292 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-12T23:50:28.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.071045+0000 mon.a (mon.0) 1293 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-12T23:50:28.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.079023+0000 mon.a (mon.0) 1294 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-12T23:50:28.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.080764+0000 mon.a (mon.0) 1295 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-12T23:50:28.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.090851+0000 mon.a (mon.0) 1296 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-12T23:50:28.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.092271+0000 mon.a (mon.0) 1297 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-12T23:50:28.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.102361+0000 mon.a (mon.0) 1298 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-12T23:50:28.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.103829+0000 mon.a (mon.0) 1299 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-12T23:50:28.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.113957+0000 mon.a (mon.0) 1300 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-12T23:50:28.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.115506+0000 mon.a (mon.0) 1301 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-12T23:50:28.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.125958+0000 mon.a (mon.0) 1302 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-12T23:50:28.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.127732+0000 mon.a (mon.0) 1303 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-12T23:50:28.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.129172+0000 mon.a (mon.0) 1304 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-12T23:50:28.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.138263+0000 mon.a (mon.0) 1305 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-12T23:50:28.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.140001+0000 mon.a (mon.0) 1306 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-12T23:50:28.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.148232+0000 mon.a (mon.0) 1307 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-12T23:50:28.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.149969+0000 mon.a (mon.0) 1308 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-12T23:50:28.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.158143+0000 mon.a (mon.0) 1309 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-12T23:50:28.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.159780+0000 mon.a (mon.0) 1310 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-12T23:50:28.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.167115+0000 mon.a (mon.0) 1311 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-12T23:50:28.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.169058+0000 mon.a (mon.0) 1312 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-12T23:50:28.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.170437+0000 mon.a (mon.0) 1313 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-12T23:50:28.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.171755+0000 mon.a (mon.0) 1314 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-12T23:50:28.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.173101+0000 mon.a (mon.0) 1315 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-12T23:50:28.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.174406+0000 mon.a (mon.0) 1316 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-12T23:50:28.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.175654+0000 mon.a (mon.0) 1317 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-12T23:50:28.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.176981+0000 mon.a (mon.0) 1318 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]: dispatch 2024-09-12T23:50:28.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.184158+0000 mon.a (mon.0) 1319 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]': finished 2024-09-12T23:50:28.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.186806+0000 mon.a (mon.0) 1320 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-12T23:50:28.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.192762+0000 mon.a (mon.0) 1321 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-12T23:50:28.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.194025+0000 mon.a (mon.0) 1322 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:50:28.352 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.198680+0000 mon.a (mon.0) 1323 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:50:28.352 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.200366+0000 mon.a (mon.0) 1324 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:50:28.352 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.210934+0000 mon.a (mon.0) 1325 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:50:28.352 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.362164+0000 mon.a (mon.0) 1326 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:50:28.352 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.365069+0000 mon.a (mon.0) 1327 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:50:28.352 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.366483+0000 mon.a (mon.0) 1328 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:50:28.352 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: audit 2024-09-12T23:50:27.376257+0000 mon.a (mon.0) 1329 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:50:28.352 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:27 smithi181 bash[67587]: cluster 2024-09-12T23:50:27.492172+0000 mon.a (mon.0) 1330 : cluster [DBG] osdmap e760: 8 total, 8 up, 8 in 2024-09-12T23:50:28.492 INFO:tasks.rados.rados.0.smithi136.stdout:3532: done (0 left) 2024-09-12T23:50:28.492 INFO:tasks.rados.rados.0.smithi136.stdout:3533: read oid 29 snap -1 2024-09-12T23:50:28.492 INFO:tasks.rados.rados.0.smithi136.stdout:3533: expect (ObjNum 1086 snap 328 seq_num 1086) 2024-09-12T23:50:28.492 INFO:tasks.rados.rados.0.smithi136.stdout:3534: copy_from oid 27 from oid 9 current snap is 342 2024-09-12T23:50:28.492 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 41 current snap is 342 2024-09-12T23:50:28.493 INFO:tasks.rados.rados.0.smithi136.stdout:3535: seq_num 1113 ranges {0=647168,647168=524288,1171456=622592,1794048=57344} 2024-09-12T23:50:28.500 INFO:tasks.rados.rados.0.smithi136.stdout:3535: writing smithi13667081-41 from 0 to 647168 tid 1 2024-09-12T23:50:28.508 INFO:tasks.rados.rados.0.smithi136.stdout:3535: writing smithi13667081-41 from 647168 to 1171456 tid 2 2024-09-12T23:50:28.514 INFO:tasks.rados.rados.0.smithi136.stdout:3535: writing smithi13667081-41 from 1171456 to 1794048 tid 3 2024-09-12T23:50:28.515 INFO:tasks.rados.rados.0.smithi136.stdout:3535: writing smithi13667081-41 from 1794048 to 1851392 tid 4 2024-09-12T23:50:28.515 INFO:tasks.rados.rados.0.smithi136.stdout:3536: read oid 18 snap 338 2024-09-12T23:50:28.515 INFO:tasks.rados.rados.0.smithi136.stdout:3536: expect deleted 2024-09-12T23:50:28.516 INFO:tasks.rados.rados.0.smithi136.stdout:3537: setattr oid 37 current snap is 342 2024-09-12T23:50:28.518 INFO:tasks.rados.rados.0.smithi136.stdout:3533: expect (ObjNum 1086 snap 328 seq_num 1086) 2024-09-12T23:50:28.590 INFO:tasks.rados.rados.0.smithi136.stdout:3535: finishing write tid 1 to smithi13667081-41 2024-09-12T23:50:28.590 INFO:tasks.rados.rados.0.smithi136.stdout:3535: finishing write tid 2 to smithi13667081-41 2024-09-12T23:50:28.590 INFO:tasks.rados.rados.0.smithi136.stdout:3535: finishing write tid 3 to smithi13667081-41 2024-09-12T23:50:28.591 INFO:tasks.rados.rados.0.smithi136.stdout:3535: finishing write tid 4 to smithi13667081-41 2024-09-12T23:50:28.591 INFO:tasks.rados.rados.0.smithi136.stdout:3534: finishing copy_from racing read to smithi13667081-27 2024-09-12T23:50:28.591 INFO:tasks.rados.rados.0.smithi136.stdout:3535: finishing write tid 5 to smithi13667081-41 2024-09-12T23:50:28.591 INFO:tasks.rados.rados.0.smithi136.stdout:3535: finishing write tid 6 to smithi13667081-41 2024-09-12T23:50:28.591 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 41 v 2500 (ObjNum 1113 snap 342 seq_num 1113) dirty exists 2024-09-12T23:50:28.591 INFO:tasks.rados.rados.0.smithi136.stdout:3535: left oid 41 (ObjNum 1113 snap 342 seq_num 1113) 2024-09-12T23:50:28.591 INFO:tasks.rados.rados.0.smithi136.stdout:3534: finishing copy_from to smithi13667081-27 2024-09-12T23:50:28.591 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 27 v 2472 (ObjNum 1091 snap 329 seq_num 1091) dirty exists 2024-09-12T23:50:28.591 INFO:tasks.rados.rados.0.smithi136.stdout:3533: done (4 left) 2024-09-12T23:50:28.591 INFO:tasks.rados.rados.0.smithi136.stdout:3534: done (3 left) 2024-09-12T23:50:28.591 INFO:tasks.rados.rados.0.smithi136.stdout:3535: done (2 left) 2024-09-12T23:50:28.591 INFO:tasks.rados.rados.0.smithi136.stdout:3536: done (1 left) 2024-09-12T23:50:28.591 INFO:tasks.rados.rados.0.smithi136.stdout:3538: delete oid 39 current snap is 342 2024-09-12T23:50:28.596 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 37 v 2501 (ObjNum 1020 snap 315 seq_num 1020) dirty exists 2024-09-12T23:50:28.597 INFO:tasks.rados.rados.0.smithi136.stdout:3537: done (1 left) 2024-09-12T23:50:28.597 INFO:tasks.rados.rados.0.smithi136.stdout:3538: done (0 left) 2024-09-12T23:50:28.597 INFO:tasks.rados.rados.0.smithi136.stdout:3539: setattr oid 12 current snap is 342 2024-09-12T23:50:28.601 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 25 current snap is 342 2024-09-12T23:50:28.601 INFO:tasks.rados.rados.0.smithi136.stdout:3540: seq_num 1115 ranges {0=679936,679936=688128,1368064=516096} 2024-09-12T23:50:28.610 INFO:tasks.rados.rados.0.smithi136.stdout:3540: writing smithi13667081-25 from 0 to 679936 tid 1 2024-09-12T23:50:28.620 INFO:tasks.rados.rados.0.smithi136.stdout:3540: writing smithi13667081-25 from 679936 to 1368064 tid 2 2024-09-12T23:50:28.628 INFO:tasks.rados.rados.0.smithi136.stdout:3540: writing smithi13667081-25 from 1368064 to 1884160 tid 3 2024-09-12T23:50:28.628 INFO:tasks.rados.rados.0.smithi136.stdout:3541: read oid 38 snap -1 2024-09-12T23:50:28.628 INFO:tasks.rados.rados.0.smithi136.stdout:3541: expect (ObjNum 864 snap 281 seq_num 864) 2024-09-12T23:50:28.628 INFO:tasks.rados.rados.0.smithi136.stdout:3542: snap_remove snap 337 2024-09-12T23:50:29.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:28 smithi136 bash[76371]: audit 2024-09-12T23:50:26.942720+0000 mgr.y (mgr.24661) 395 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-12T23:50:29.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:28 smithi136 bash[76371]: cephadm 2024-09-12T23:50:27.041216+0000 mgr.y (mgr.24661) 396 : cephadm [INF] Upgrade: Finalizing container_image settings 2024-09-12T23:50:29.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:28 smithi136 bash[76371]: cephadm 2024-09-12T23:50:27.185808+0000 mgr.y (mgr.24661) 397 : cephadm [INF] Upgrade: Complete! 2024-09-12T23:50:29.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:28 smithi136 bash[76371]: cluster 2024-09-12T23:50:28.494420+0000 mon.a (mon.0) 1331 : cluster [DBG] osdmap e761: 8 total, 8 up, 8 in 2024-09-12T23:50:29.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:28 smithi136 bash[75164]: audit 2024-09-12T23:50:26.942720+0000 mgr.y (mgr.24661) 395 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-12T23:50:29.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:28 smithi136 bash[75164]: cephadm 2024-09-12T23:50:27.041216+0000 mgr.y (mgr.24661) 396 : cephadm [INF] Upgrade: Finalizing container_image settings 2024-09-12T23:50:29.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:28 smithi136 bash[75164]: cephadm 2024-09-12T23:50:27.185808+0000 mgr.y (mgr.24661) 397 : cephadm [INF] Upgrade: Complete! 2024-09-12T23:50:29.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:28 smithi136 bash[75164]: cluster 2024-09-12T23:50:28.494420+0000 mon.a (mon.0) 1331 : cluster [DBG] osdmap e761: 8 total, 8 up, 8 in 2024-09-12T23:50:29.345 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:28 smithi181 bash[67587]: audit 2024-09-12T23:50:26.942720+0000 mgr.y (mgr.24661) 395 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-12T23:50:29.345 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:28 smithi181 bash[67587]: cephadm 2024-09-12T23:50:27.041216+0000 mgr.y (mgr.24661) 396 : cephadm [INF] Upgrade: Finalizing container_image settings 2024-09-12T23:50:29.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:28 smithi181 bash[67587]: cephadm 2024-09-12T23:50:27.185808+0000 mgr.y (mgr.24661) 397 : cephadm [INF] Upgrade: Complete! 2024-09-12T23:50:29.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:28 smithi181 bash[67587]: cluster 2024-09-12T23:50:28.494420+0000 mon.a (mon.0) 1331 : cluster [DBG] osdmap e761: 8 total, 8 up, 8 in 2024-09-12T23:50:29.502 INFO:tasks.rados.rados.0.smithi136.stdout:3542: done (3 left) 2024-09-12T23:50:29.502 INFO:tasks.rados.rados.0.smithi136.stdout:3543: delete oid 49 current snap is 342 2024-09-12T23:50:29.508 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 12 v 2106 (ObjNum 1108 snap 336 seq_num 1108) dirty exists 2024-09-12T23:50:29.508 INFO:tasks.rados.rados.0.smithi136.stdout:3540: finishing write tid 1 to smithi13667081-25 2024-09-12T23:50:29.508 INFO:tasks.rados.rados.0.smithi136.stdout:3540: finishing write tid 2 to smithi13667081-25 2024-09-12T23:50:29.508 INFO:tasks.rados.rados.0.smithi136.stdout:3540: finishing write tid 3 to smithi13667081-25 2024-09-12T23:50:29.508 INFO:tasks.rados.rados.0.smithi136.stdout:3540: finishing write tid 4 to smithi13667081-25 2024-09-12T23:50:29.508 INFO:tasks.rados.rados.0.smithi136.stdout:3540: finishing write tid 5 to smithi13667081-25 2024-09-12T23:50:29.508 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 25 v 2453 (ObjNum 1115 snap 342 seq_num 1115) dirty exists 2024-09-12T23:50:29.508 INFO:tasks.rados.rados.0.smithi136.stdout:3540: left oid 25 (ObjNum 1115 snap 342 seq_num 1115) 2024-09-12T23:50:29.509 INFO:tasks.rados.rados.0.smithi136.stdout:3541: expect (ObjNum 864 snap 281 seq_num 864) 2024-09-12T23:50:29.673 INFO:tasks.rados.rados.0.smithi136.stdout:3539: done (3 left) 2024-09-12T23:50:29.674 INFO:tasks.rados.rados.0.smithi136.stdout:3540: done (2 left) 2024-09-12T23:50:29.674 INFO:tasks.rados.rados.0.smithi136.stdout:3541: done (1 left) 2024-09-12T23:50:29.674 INFO:tasks.rados.rados.0.smithi136.stdout:3543: done (0 left) 2024-09-12T23:50:29.674 INFO:tasks.rados.rados.0.smithi136.stdout:3544: snap_remove snap 340 2024-09-12T23:50:29.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:50:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:50:29] "GET /metrics HTTP/1.1" 200 36440 "" "Prometheus/2.43.0" 2024-09-12T23:50:30.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:29 smithi136 bash[75164]: cluster 2024-09-12T23:50:28.851109+0000 mgr.y (mgr.24661) 398 : cluster [DBG] pgmap v723: 121 pgs: 121 active+clean; 276 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 2.1 MiB/s wr, 16 op/s 2024-09-12T23:50:30.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:29 smithi136 bash[75164]: audit 2024-09-12T23:50:29.024884+0000 mon.a (mon.0) 1332 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:50:30.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:29 smithi136 bash[75164]: cluster 2024-09-12T23:50:29.501742+0000 mon.a (mon.0) 1333 : cluster [DBG] osdmap e762: 8 total, 8 up, 8 in 2024-09-12T23:50:30.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:29 smithi136 bash[76371]: cluster 2024-09-12T23:50:28.851109+0000 mgr.y (mgr.24661) 398 : cluster [DBG] pgmap v723: 121 pgs: 121 active+clean; 276 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 2.1 MiB/s wr, 16 op/s 2024-09-12T23:50:30.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:29 smithi136 bash[76371]: audit 2024-09-12T23:50:29.024884+0000 mon.a (mon.0) 1332 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:50:30.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:29 smithi136 bash[76371]: cluster 2024-09-12T23:50:29.501742+0000 mon.a (mon.0) 1333 : cluster [DBG] osdmap e762: 8 total, 8 up, 8 in 2024-09-12T23:50:30.345 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:29 smithi181 bash[67587]: cluster 2024-09-12T23:50:28.851109+0000 mgr.y (mgr.24661) 398 : cluster [DBG] pgmap v723: 121 pgs: 121 active+clean; 276 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 2.1 MiB/s wr, 16 op/s 2024-09-12T23:50:30.345 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:29 smithi181 bash[67587]: audit 2024-09-12T23:50:29.024884+0000 mon.a (mon.0) 1332 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:50:30.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:29 smithi181 bash[67587]: cluster 2024-09-12T23:50:29.501742+0000 mon.a (mon.0) 1333 : cluster [DBG] osdmap e762: 8 total, 8 up, 8 in 2024-09-12T23:50:30.504 INFO:tasks.rados.rados.0.smithi136.stdout:3544: done (0 left) 2024-09-12T23:50:30.504 INFO:tasks.rados.rados.0.smithi136.stdout:3545: read oid 35 snap 341 2024-09-12T23:50:30.504 INFO:tasks.rados.rados.0.smithi136.stdout:3545: expect (ObjNum 994 snap 308 seq_num 994) 2024-09-12T23:50:30.504 INFO:tasks.rados.rados.0.smithi136.stdout:3546: copy_from oid 34 from oid 5 current snap is 342 2024-09-12T23:50:30.504 INFO:tasks.rados.rados.0.smithi136.stdout:3547: setattr oid 47 current snap is 342 2024-09-12T23:50:30.508 INFO:tasks.rados.rados.0.smithi136.stdout:3548: snap_create 2024-09-12T23:50:30.528 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 47 v 2473 (ObjNum 1073 snap 324 seq_num 1073) dirty exists 2024-09-12T23:50:30.568 INFO:tasks.rados.rados.0.smithi136.stdout:3545: expect (ObjNum 994 snap 308 seq_num 994) 2024-09-12T23:50:31.077 INFO:tasks.rados.rados.0.smithi136.stdout:3546: finishing copy_from racing read to smithi13667081-34 2024-09-12T23:50:31.077 INFO:tasks.rados.rados.0.smithi136.stdout:3546: finishing copy_from to smithi13667081-34 2024-09-12T23:50:31.077 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 34 v 1640 (ObjNum 1026 snap 316 seq_num 1026) dirty exists 2024-09-12T23:50:31.490 INFO:tasks.rados.rados.0.smithi136.stdout:3545: done (3 left) 2024-09-12T23:50:31.491 INFO:tasks.rados.rados.0.smithi136.stdout:3546: done (2 left) 2024-09-12T23:50:31.491 INFO:tasks.rados.rados.0.smithi136.stdout:3547: done (1 left) 2024-09-12T23:50:31.491 INFO:tasks.rados.rados.0.smithi136.stdout:3548: done (0 left) 2024-09-12T23:50:31.491 INFO:tasks.rados.rados.0.smithi136.stdout:3549: delete oid 34 current snap is 343 2024-09-12T23:50:31.503 INFO:tasks.rados.rados.0.smithi136.stdout:3549: done (0 left) 2024-09-12T23:50:31.503 INFO:tasks.rados.rados.0.smithi136.stdout:3550: rmattr oid 6 current snap is 343 2024-09-12T23:50:31.503 INFO:tasks.rados.rados.0.smithi136.stdout:3550: done (0 left) 2024-09-12T23:50:31.503 INFO:tasks.rados.rados.0.smithi136.stdout:3551: read oid 36 snap -1 2024-09-12T23:50:31.503 INFO:tasks.rados.rados.0.smithi136.stdout:3551: expect (ObjNum 1055 snap 321 seq_num 1055) 2024-09-12T23:50:31.503 INFO:tasks.rados.rados.0.smithi136.stdout:3552: delete oid 50 current snap is 343 2024-09-12T23:50:31.512 INFO:tasks.rados.rados.0.smithi136.stdout:3552: done (1 left) 2024-09-12T23:50:31.512 INFO:tasks.rados.rados.0.smithi136.stdout:3553: rollback oid 25 current snap is 343 2024-09-12T23:50:31.512 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 25 to 342 2024-09-12T23:50:31.512 INFO:tasks.rados.rados.0.smithi136.stdout:3554: read oid 20 snap 334 2024-09-12T23:50:31.512 INFO:tasks.rados.rados.0.smithi136.stdout:3554: expect (ObjNum 1093 snap 330 seq_num 1093) 2024-09-12T23:50:31.512 INFO:tasks.rados.rados.0.smithi136.stdout:3555: snap_remove snap 329 2024-09-12T23:50:31.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:31 smithi181 bash[67587]: cluster 2024-09-12T23:50:30.506203+0000 mon.a (mon.0) 1334 : cluster [DBG] osdmap e763: 8 total, 8 up, 8 in 2024-09-12T23:50:31.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:31 smithi181 bash[67587]: cluster 2024-09-12T23:50:30.852044+0000 mgr.y (mgr.24661) 399 : cluster [DBG] pgmap v726: 121 pgs: 121 active+clean; 278 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 908 KiB/s wr, 10 op/s 2024-09-12T23:50:31.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:31 smithi181 bash[67587]: cluster 2024-09-12T23:50:31.491501+0000 mon.a (mon.0) 1335 : cluster [DBG] osdmap e764: 8 total, 8 up, 8 in 2024-09-12T23:50:32.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:31 smithi136 bash[75164]: cluster 2024-09-12T23:50:30.506203+0000 mon.a (mon.0) 1334 : cluster [DBG] osdmap e763: 8 total, 8 up, 8 in 2024-09-12T23:50:32.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:31 smithi136 bash[75164]: cluster 2024-09-12T23:50:30.852044+0000 mgr.y (mgr.24661) 399 : cluster [DBG] pgmap v726: 121 pgs: 121 active+clean; 278 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 908 KiB/s wr, 10 op/s 2024-09-12T23:50:32.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:31 smithi136 bash[75164]: cluster 2024-09-12T23:50:31.491501+0000 mon.a (mon.0) 1335 : cluster [DBG] osdmap e764: 8 total, 8 up, 8 in 2024-09-12T23:50:32.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:31 smithi136 bash[76371]: cluster 2024-09-12T23:50:30.506203+0000 mon.a (mon.0) 1334 : cluster [DBG] osdmap e763: 8 total, 8 up, 8 in 2024-09-12T23:50:32.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:31 smithi136 bash[76371]: cluster 2024-09-12T23:50:30.852044+0000 mgr.y (mgr.24661) 399 : cluster [DBG] pgmap v726: 121 pgs: 121 active+clean; 278 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 908 KiB/s wr, 10 op/s 2024-09-12T23:50:32.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:31 smithi136 bash[76371]: cluster 2024-09-12T23:50:31.491501+0000 mon.a (mon.0) 1335 : cluster [DBG] osdmap e764: 8 total, 8 up, 8 in 2024-09-12T23:50:32.520 INFO:tasks.rados.rados.0.smithi136.stdout:3555: done (3 left) 2024-09-12T23:50:32.520 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 16 current snap is 343 2024-09-12T23:50:32.520 INFO:tasks.rados.rados.0.smithi136.stdout:3556: seq_num 1119 ranges {0=745472,745472=679936,1425408=524288} 2024-09-12T23:50:32.524 INFO:tasks.rados.rados.0.smithi136.stdout:3556: writing smithi13667081-16 from 0 to 745472 tid 1 2024-09-12T23:50:32.529 INFO:tasks.rados.rados.0.smithi136.stdout:3556: writing smithi13667081-16 from 745472 to 1425408 tid 2 2024-09-12T23:50:32.533 INFO:tasks.rados.rados.0.smithi136.stdout:3556: writing smithi13667081-16 from 1425408 to 1949696 tid 3 2024-09-12T23:50:32.533 INFO:tasks.rados.rados.0.smithi136.stdout:3557: copy_from oid 41 from oid 44 current snap is 343 2024-09-12T23:50:32.533 INFO:tasks.rados.rados.0.smithi136.stdout:3551: expect (ObjNum 1055 snap 321 seq_num 1055) 2024-09-12T23:50:32.567 INFO:tasks.rados.rados.0.smithi136.stdout:3553: finishing rollback tid 0 to smithi13667081-25 2024-09-12T23:50:32.567 INFO:tasks.rados.rados.0.smithi136.stdout:3553: finishing rollback tid 1 to smithi13667081-25 2024-09-12T23:50:32.567 INFO:tasks.rados.rados.0.smithi136.stdout:3554: expect (ObjNum 1093 snap 330 seq_num 1093) 2024-09-12T23:50:32.595 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:50:32 smithi181 bash[95505]: ts=2024-09-12T23:50:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:50:32.613 INFO:tasks.rados.rados.0.smithi136.stdout:3553: finishing rollback tid 2 to smithi13667081-25 2024-09-12T23:50:32.613 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 25 v 2456 (ObjNum 1115 snap 342 seq_num 1115) dirty exists 2024-09-12T23:50:32.613 INFO:tasks.rados.rados.0.smithi136.stdout:3557: finishing copy_from to smithi13667081-41 2024-09-12T23:50:32.614 INFO:tasks.rados.rados.0.smithi136.stdout:3557: got expected ENOENT (src dne) 2024-09-12T23:50:32.614 INFO:tasks.rados.rados.0.smithi136.stdout:3557: finishing copy_from racing read to smithi13667081-41 2024-09-12T23:50:32.614 INFO:tasks.rados.rados.0.smithi136.stdout:3556: finishing write tid 1 to smithi13667081-16 2024-09-12T23:50:32.614 INFO:tasks.rados.rados.0.smithi136.stdout:3556: finishing write tid 2 to smithi13667081-16 2024-09-12T23:50:32.614 INFO:tasks.rados.rados.0.smithi136.stdout:3556: finishing write tid 3 to smithi13667081-16 2024-09-12T23:50:32.614 INFO:tasks.rados.rados.0.smithi136.stdout:3556: finishing write tid 4 to smithi13667081-16 2024-09-12T23:50:32.614 INFO:tasks.rados.rados.0.smithi136.stdout:3551: done (4 left) 2024-09-12T23:50:32.614 INFO:tasks.rados.rados.0.smithi136.stdout:3553: done (3 left) 2024-09-12T23:50:32.614 INFO:tasks.rados.rados.0.smithi136.stdout:3554: done (2 left) 2024-09-12T23:50:32.614 INFO:tasks.rados.rados.0.smithi136.stdout:3557: done (1 left) 2024-09-12T23:50:32.614 INFO:tasks.rados.rados.0.smithi136.stdout:3558: read oid 31 snap -1 2024-09-12T23:50:32.614 INFO:tasks.rados.rados.0.smithi136.stdout:3558: expect (ObjNum 1070 snap 324 seq_num 1070) 2024-09-12T23:50:32.614 INFO:tasks.rados.rados.0.smithi136.stdout:3559: snap_remove snap 342 2024-09-12T23:50:33.531 INFO:tasks.rados.rados.0.smithi136.stdout:3559: done (2 left) 2024-09-12T23:50:33.531 INFO:tasks.rados.rados.0.smithi136.stdout:3560: snap_remove snap 339 2024-09-12T23:50:33.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:33 smithi181 bash[67587]: cluster 2024-09-12T23:50:32.521082+0000 mon.a (mon.0) 1336 : cluster [DBG] osdmap e765: 8 total, 8 up, 8 in 2024-09-12T23:50:33.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:33 smithi181 bash[67587]: cluster 2024-09-12T23:50:32.852881+0000 mgr.y (mgr.24661) 400 : cluster [DBG] pgmap v729: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 294 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 4.6 MiB/s wr, 21 op/s 2024-09-12T23:50:34.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:33 smithi136 bash[75164]: cluster 2024-09-12T23:50:32.521082+0000 mon.a (mon.0) 1336 : cluster [DBG] osdmap e765: 8 total, 8 up, 8 in 2024-09-12T23:50:34.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:33 smithi136 bash[75164]: cluster 2024-09-12T23:50:32.852881+0000 mgr.y (mgr.24661) 400 : cluster [DBG] pgmap v729: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 294 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 4.6 MiB/s wr, 21 op/s 2024-09-12T23:50:34.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:33 smithi136 bash[76371]: cluster 2024-09-12T23:50:32.521082+0000 mon.a (mon.0) 1336 : cluster [DBG] osdmap e765: 8 total, 8 up, 8 in 2024-09-12T23:50:34.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:33 smithi136 bash[76371]: cluster 2024-09-12T23:50:32.852881+0000 mgr.y (mgr.24661) 400 : cluster [DBG] pgmap v729: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 294 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 4.6 MiB/s wr, 21 op/s 2024-09-12T23:50:34.531 INFO:tasks.rados.rados.0.smithi136.stdout:3560: done (2 left) 2024-09-12T23:50:34.531 INFO:tasks.rados.rados.0.smithi136.stdout:3561: copy_from oid 40 from oid 25 current snap is 343 2024-09-12T23:50:34.532 INFO:tasks.rados.rados.0.smithi136.stdout:3562: read oid 18 snap -1 2024-09-12T23:50:34.532 INFO:tasks.rados.rados.0.smithi136.stdout:3562: expect deleted 2024-09-12T23:50:34.532 INFO:tasks.rados.rados.0.smithi136.stdout:3563: rmattr oid 35 current snap is 343 2024-09-12T23:50:34.532 INFO:tasks.rados.rados.0.smithi136.stdout:3563: done (4 left) 2024-09-12T23:50:34.532 INFO:tasks.rados.rados.0.smithi136.stdout:3564: delete oid 10 current snap is 343 2024-09-12T23:50:34.532 INFO:tasks.rados.rados.0.smithi136.stdout:3556: finishing write tid 5 to smithi13667081-16 2024-09-12T23:50:34.532 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 16 v 2486 (ObjNum 1119 snap 343 seq_num 1119) dirty exists 2024-09-12T23:50:34.532 INFO:tasks.rados.rados.0.smithi136.stdout:3556: left oid 16 (ObjNum 1119 snap 343 seq_num 1119) 2024-09-12T23:50:34.532 INFO:tasks.rados.rados.0.smithi136.stdout:3558: expect (ObjNum 1070 snap 324 seq_num 1070) 2024-09-12T23:50:34.577 INFO:tasks.rados.rados.0.smithi136.stdout:3561: finishing copy_from racing read to smithi13667081-40 2024-09-12T23:50:34.577 INFO:tasks.rados.rados.0.smithi136.stdout:3561: finishing copy_from to smithi13667081-40 2024-09-12T23:50:34.577 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 40 v 2456 (ObjNum 1115 snap 342 seq_num 1115) dirty exists 2024-09-12T23:50:34.577 INFO:tasks.rados.rados.0.smithi136.stdout:3556: done (4 left) 2024-09-12T23:50:34.577 INFO:tasks.rados.rados.0.smithi136.stdout:3558: done (3 left) 2024-09-12T23:50:34.577 INFO:tasks.rados.rados.0.smithi136.stdout:3561: done (2 left) 2024-09-12T23:50:34.577 INFO:tasks.rados.rados.0.smithi136.stdout:3562: done (1 left) 2024-09-12T23:50:34.577 INFO:tasks.rados.rados.0.smithi136.stdout:3564: done (0 left) 2024-09-12T23:50:34.578 INFO:tasks.rados.rados.0.smithi136.stdout:3565: delete oid 40 current snap is 343 2024-09-12T23:50:34.579 INFO:tasks.rados.rados.0.smithi136.stdout:3565: done (0 left) 2024-09-12T23:50:34.580 INFO:tasks.rados.rados.0.smithi136.stdout:3566: delete oid 37 current snap is 343 2024-09-12T23:50:34.583 INFO:tasks.rados.rados.0.smithi136.stdout:3566: done (0 left) 2024-09-12T23:50:34.583 INFO:tasks.rados.rados.0.smithi136.stdout:3567: rmattr oid 49 current snap is 343 2024-09-12T23:50:34.583 INFO:tasks.rados.rados.0.smithi136.stdout:3567: done (0 left) 2024-09-12T23:50:34.583 INFO:tasks.rados.rados.0.smithi136.stdout:3568: snap_remove snap 331 2024-09-12T23:50:34.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:34 smithi181 bash[67587]: cluster 2024-09-12T23:50:33.533438+0000 mon.a (mon.0) 1337 : cluster [DBG] osdmap e766: 8 total, 8 up, 8 in 2024-09-12T23:50:35.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:34 smithi136 bash[75164]: cluster 2024-09-12T23:50:33.533438+0000 mon.a (mon.0) 1337 : cluster [DBG] osdmap e766: 8 total, 8 up, 8 in 2024-09-12T23:50:35.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:34 smithi136 bash[76371]: cluster 2024-09-12T23:50:33.533438+0000 mon.a (mon.0) 1337 : cluster [DBG] osdmap e766: 8 total, 8 up, 8 in 2024-09-12T23:50:35.564 INFO:tasks.rados.rados.0.smithi136.stdout:3568: done (0 left) 2024-09-12T23:50:35.564 INFO:tasks.rados.rados.0.smithi136.stdout:3569: rmattr oid 50 current snap is 343 2024-09-12T23:50:35.564 INFO:tasks.rados.rados.0.smithi136.stdout:3569: done (0 left) 2024-09-12T23:50:35.564 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 44 current snap is 343 2024-09-12T23:50:35.564 INFO:tasks.rados.rados.0.smithi136.stdout:3570: seq_num 1123 ranges {0=409600,409600=614400,1024000=598016,1622016=393216} 2024-09-12T23:50:35.565 INFO:tasks.rados.rados.0.smithi136.stdout:3570: writing smithi13667081-44 from 0 to 409600 tid 1 2024-09-12T23:50:35.565 INFO:tasks.rados.rados.0.smithi136.stdout:3570: writing smithi13667081-44 from 409600 to 1024000 tid 2 2024-09-12T23:50:35.568 INFO:tasks.rados.rados.0.smithi136.stdout:3570: writing smithi13667081-44 from 1024000 to 1622016 tid 3 2024-09-12T23:50:35.571 INFO:tasks.rados.rados.0.smithi136.stdout:3570: writing smithi13667081-44 from 1622016 to 2015232 tid 4 2024-09-12T23:50:35.571 INFO:tasks.rados.rados.0.smithi136.stdout:3571: rollback oid 4 current snap is 343 2024-09-12T23:50:35.571 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 4 to 336 2024-09-12T23:50:35.571 INFO:tasks.rados.rados.0.smithi136.stdout:3572: read oid 2 snap -1 2024-09-12T23:50:35.571 INFO:tasks.rados.rados.0.smithi136.stdout:3572: expect (ObjNum 1092 snap 330 seq_num 1092) 2024-09-12T23:50:35.572 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 32 current snap is 343 2024-09-12T23:50:35.572 INFO:tasks.rados.rados.0.smithi136.stdout:3573: seq_num 1124 ranges {17104896=434176,17539072=499712,18038784=409600,18448384=663552,19111936=32768} 2024-09-12T23:50:35.671 INFO:tasks.rados.rados.0.smithi136.stdout:3573: writing smithi13667081-32 from 17104896 to 17539072 tid 1 2024-09-12T23:50:35.674 INFO:tasks.rados.rados.0.smithi136.stdout:3573: writing smithi13667081-32 from 17539072 to 18038784 tid 2 2024-09-12T23:50:35.677 INFO:tasks.rados.rados.0.smithi136.stdout:3573: writing smithi13667081-32 from 18038784 to 18448384 tid 3 2024-09-12T23:50:35.681 INFO:tasks.rados.rados.0.smithi136.stdout:3573: writing smithi13667081-32 from 18448384 to 19111936 tid 4 2024-09-12T23:50:35.681 INFO:tasks.rados.rados.0.smithi136.stdout:3573: writing smithi13667081-32 from 19111936 to 19144704 tid 5 2024-09-12T23:50:35.681 INFO:tasks.rados.rados.0.smithi136.stdout:3574: rmattr oid 16 current snap is 343 2024-09-12T23:50:35.682 INFO:tasks.rados.rados.0.smithi136.stdout:3574: done (4 left) 2024-09-12T23:50:35.682 INFO:tasks.rados.rados.0.smithi136.stdout:3575: rollback oid 6 current snap is 343 2024-09-12T23:50:35.682 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 6 to 335 2024-09-12T23:50:35.682 INFO:tasks.rados.rados.0.smithi136.stdout:3571: finishing rollback tid 0 to smithi13667081-4 2024-09-12T23:50:35.682 INFO:tasks.rados.rados.0.smithi136.stdout:3571: finishing rollback tid 1 to smithi13667081-4 2024-09-12T23:50:35.682 INFO:tasks.rados.rados.0.smithi136.stdout:3571: finishing rollback tid 2 to smithi13667081-4 2024-09-12T23:50:35.682 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 4 v 1665 (ObjNum 1096 snap 332 seq_num 1096) dirty exists 2024-09-12T23:50:35.682 INFO:tasks.rados.rados.0.smithi136.stdout:3572: expect (ObjNum 1092 snap 330 seq_num 1092) 2024-09-12T23:50:35.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:35 smithi181 bash[67587]: cluster 2024-09-12T23:50:34.533018+0000 mon.a (mon.0) 1338 : cluster [DBG] osdmap e767: 8 total, 8 up, 8 in 2024-09-12T23:50:35.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:35 smithi181 bash[67587]: cluster 2024-09-12T23:50:34.853581+0000 mgr.y (mgr.24661) 401 : cluster [DBG] pgmap v732: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 298 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.5 MiB/s rd, 5.0 MiB/s wr, 19 op/s 2024-09-12T23:50:35.949 INFO:tasks.rados.rados.0.smithi136.stdout:3570: finishing write tid 1 to smithi13667081-44 2024-09-12T23:50:35.949 INFO:tasks.rados.rados.0.smithi136.stdout:3570: finishing write tid 2 to smithi13667081-44 2024-09-12T23:50:35.949 INFO:tasks.rados.rados.0.smithi136.stdout:3570: finishing write tid 3 to smithi13667081-44 2024-09-12T23:50:35.949 INFO:tasks.rados.rados.0.smithi136.stdout:3570: finishing write tid 4 to smithi13667081-44 2024-09-12T23:50:35.949 INFO:tasks.rados.rados.0.smithi136.stdout:3570: finishing write tid 5 to smithi13667081-44 2024-09-12T23:50:35.949 INFO:tasks.rados.rados.0.smithi136.stdout:3570: finishing write tid 6 to smithi13667081-44 2024-09-12T23:50:35.949 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 44 v 2491 (ObjNum 1123 snap 343 seq_num 1123) dirty exists 2024-09-12T23:50:35.949 INFO:tasks.rados.rados.0.smithi136.stdout:3570: left oid 44 (ObjNum 1123 snap 343 seq_num 1123) 2024-09-12T23:50:35.949 INFO:tasks.rados.rados.0.smithi136.stdout:3575: finishing rollback tid 1 to smithi13667081-6 2024-09-12T23:50:35.950 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 6 v 0 (ObjNum 129 snap 0 seq_num 4243705368) dirty dne 2024-09-12T23:50:35.950 INFO:tasks.rados.rados.0.smithi136.stdout:3573: finishing write tid 1 to smithi13667081-32 2024-09-12T23:50:35.950 INFO:tasks.rados.rados.0.smithi136.stdout:3573: finishing write tid 2 to smithi13667081-32 2024-09-12T23:50:35.950 INFO:tasks.rados.rados.0.smithi136.stdout:3573: finishing write tid 3 to smithi13667081-32 2024-09-12T23:50:35.950 INFO:tasks.rados.rados.0.smithi136.stdout:3573: finishing write tid 4 to smithi13667081-32 2024-09-12T23:50:35.950 INFO:tasks.rados.rados.0.smithi136.stdout:3573: finishing write tid 5 to smithi13667081-32 2024-09-12T23:50:35.950 INFO:tasks.rados.rados.0.smithi136.stdout:3573: finishing write tid 6 to smithi13667081-32 2024-09-12T23:50:35.950 INFO:tasks.rados.rados.0.smithi136.stdout:3573: finishing write tid 7 to smithi13667081-32 2024-09-12T23:50:35.950 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 32 v 1671 (ObjNum 1124 snap 343 seq_num 1124) dirty exists 2024-09-12T23:50:35.950 INFO:tasks.rados.rados.0.smithi136.stdout:3573: left oid 32 (ObjNum 1124 snap 343 seq_num 1124) 2024-09-12T23:50:35.950 INFO:tasks.rados.rados.0.smithi136.stdout:3570: done (4 left) 2024-09-12T23:50:35.951 INFO:tasks.rados.rados.0.smithi136.stdout:3571: done (3 left) 2024-09-12T23:50:35.951 INFO:tasks.rados.rados.0.smithi136.stdout:3572: done (2 left) 2024-09-12T23:50:35.951 INFO:tasks.rados.rados.0.smithi136.stdout:3573: done (1 left) 2024-09-12T23:50:35.951 INFO:tasks.rados.rados.0.smithi136.stdout:3575: done (0 left) 2024-09-12T23:50:35.951 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 9 current snap is 343 2024-09-12T23:50:35.951 INFO:tasks.rados.rados.0.smithi136.stdout:3576: seq_num 1125 ranges {3514368=450560,3964928=786432,4751360=614400,5365760=204800} 2024-09-12T23:50:35.970 INFO:tasks.rados.rados.0.smithi136.stdout:3576: writing smithi13667081-9 from 3514368 to 3964928 tid 1 2024-09-12T23:50:35.975 INFO:tasks.rados.rados.0.smithi136.stdout:3576: writing smithi13667081-9 from 3964928 to 4751360 tid 2 2024-09-12T23:50:35.978 INFO:tasks.rados.rados.0.smithi136.stdout:3576: writing smithi13667081-9 from 4751360 to 5365760 tid 3 2024-09-12T23:50:35.980 INFO:tasks.rados.rados.0.smithi136.stdout:3576: writing smithi13667081-9 from 5365760 to 5570560 tid 4 2024-09-12T23:50:35.980 INFO:tasks.rados.rados.0.smithi136.stdout:3577: copy_from oid 12 from oid 28 current snap is 343 2024-09-12T23:50:35.980 INFO:tasks.rados.rados.0.smithi136.stdout:3576: finishing write tid 1 to smithi13667081-9 2024-09-12T23:50:35.980 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 50 current snap is 343 2024-09-12T23:50:35.981 INFO:tasks.rados.rados.0.smithi136.stdout:3578: seq_num 1126 ranges {0=466944,466944=720896,1187840=778240,1966080=106496} 2024-09-12T23:50:35.983 INFO:tasks.rados.rados.0.smithi136.stdout:3578: writing smithi13667081-50 from 0 to 466944 tid 1 2024-09-12T23:50:35.988 INFO:tasks.rados.rados.0.smithi136.stdout:3578: writing smithi13667081-50 from 466944 to 1187840 tid 2 2024-09-12T23:50:35.993 INFO:tasks.rados.rados.0.smithi136.stdout:3578: writing smithi13667081-50 from 1187840 to 1966080 tid 3 2024-09-12T23:50:35.994 INFO:tasks.rados.rados.0.smithi136.stdout:3578: writing smithi13667081-50 from 1966080 to 2072576 tid 4 2024-09-12T23:50:35.994 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 13 current snap is 343 2024-09-12T23:50:35.994 INFO:tasks.rados.rados.0.smithi136.stdout:3576: finishing write tid 2 to smithi13667081-9 2024-09-12T23:50:35.994 INFO:tasks.rados.rados.0.smithi136.stdout:3576: finishing write tid 3 to smithi13667081-9 2024-09-12T23:50:35.994 INFO:tasks.rados.rados.0.smithi136.stdout:3576: finishing write tid 4 to smithi13667081-9 2024-09-12T23:50:35.994 INFO:tasks.rados.rados.0.smithi136.stdout:3576: finishing write tid 5 to smithi13667081-9 2024-09-12T23:50:35.994 INFO:tasks.rados.rados.0.smithi136.stdout:3579: seq_num 1127 ranges {0=483328,483328=598016,1081344=581632,1662976=425984} 2024-09-12T23:50:35.996 INFO:tasks.rados.rados.0.smithi136.stdout:3579: writing smithi13667081-13 from 0 to 483328 tid 1 2024-09-12T23:50:36.000 INFO:tasks.rados.rados.0.smithi136.stdout:3579: writing smithi13667081-13 from 483328 to 1081344 tid 2 2024-09-12T23:50:36.003 INFO:tasks.rados.rados.0.smithi136.stdout:3579: writing smithi13667081-13 from 1081344 to 1662976 tid 3 2024-09-12T23:50:36.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:35 smithi136 bash[75164]: cluster 2024-09-12T23:50:34.533018+0000 mon.a (mon.0) 1338 : cluster [DBG] osdmap e767: 8 total, 8 up, 8 in 2024-09-12T23:50:36.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:35 smithi136 bash[75164]: cluster 2024-09-12T23:50:34.853581+0000 mgr.y (mgr.24661) 401 : cluster [DBG] pgmap v732: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 298 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.5 MiB/s rd, 5.0 MiB/s wr, 19 op/s 2024-09-12T23:50:36.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:35 smithi136 bash[76371]: cluster 2024-09-12T23:50:34.533018+0000 mon.a (mon.0) 1338 : cluster [DBG] osdmap e767: 8 total, 8 up, 8 in 2024-09-12T23:50:36.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:35 smithi136 bash[76371]: cluster 2024-09-12T23:50:34.853581+0000 mgr.y (mgr.24661) 401 : cluster [DBG] pgmap v732: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 298 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.5 MiB/s rd, 5.0 MiB/s wr, 19 op/s 2024-09-12T23:50:36.007 INFO:tasks.rados.rados.0.smithi136.stdout:3579: writing smithi13667081-13 from 1662976 to 2088960 tid 4 2024-09-12T23:50:36.007 INFO:tasks.rados.rados.0.smithi136.stdout:3580: snap_create 2024-09-12T23:50:36.007 INFO:tasks.rados.rados.0.smithi136.stdout:3576: finishing write tid 6 to smithi13667081-9 2024-09-12T23:50:36.007 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 9 v 2478 (ObjNum 1125 snap 343 seq_num 1125) dirty exists 2024-09-12T23:50:36.007 INFO:tasks.rados.rados.0.smithi136.stdout:3576: left oid 9 (ObjNum 1125 snap 343 seq_num 1125) 2024-09-12T23:50:36.007 INFO:tasks.rados.rados.0.smithi136.stdout:3578: finishing write tid 1 to smithi13667081-50 2024-09-12T23:50:36.008 INFO:tasks.rados.rados.0.smithi136.stdout:3578: finishing write tid 2 to smithi13667081-50 2024-09-12T23:50:36.008 INFO:tasks.rados.rados.0.smithi136.stdout:3578: finishing write tid 3 to smithi13667081-50 2024-09-12T23:50:36.008 INFO:tasks.rados.rados.0.smithi136.stdout:3578: finishing write tid 4 to smithi13667081-50 2024-09-12T23:50:36.008 INFO:tasks.rados.rados.0.smithi136.stdout:3577: finishing copy_from racing read to smithi13667081-12 2024-09-12T23:50:36.009 INFO:tasks.rados.rados.0.smithi136.stdout:3578: finishing write tid 5 to smithi13667081-50 2024-09-12T23:50:36.011 INFO:tasks.rados.rados.0.smithi136.stdout:3579: finishing write tid 1 to smithi13667081-13 2024-09-12T23:50:36.013 INFO:tasks.rados.rados.0.smithi136.stdout:3577: finishing copy_from to smithi13667081-12 2024-09-12T23:50:36.013 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 12 v 2482 (ObjNum 1106 snap 336 seq_num 1106) dirty exists 2024-09-12T23:50:36.013 INFO:tasks.rados.rados.0.smithi136.stdout:3578: finishing write tid 6 to smithi13667081-50 2024-09-12T23:50:36.013 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 50 v 2670 (ObjNum 1126 snap 343 seq_num 1126) dirty exists 2024-09-12T23:50:36.013 INFO:tasks.rados.rados.0.smithi136.stdout:3578: left oid 50 (ObjNum 1126 snap 343 seq_num 1126) 2024-09-12T23:50:36.023 INFO:tasks.rados.rados.0.smithi136.stdout:3579: finishing write tid 2 to smithi13667081-13 2024-09-12T23:50:36.024 INFO:tasks.rados.rados.0.smithi136.stdout:3579: finishing write tid 3 to smithi13667081-13 2024-09-12T23:50:36.025 INFO:tasks.rados.rados.0.smithi136.stdout:3579: finishing write tid 4 to smithi13667081-13 2024-09-12T23:50:36.029 INFO:tasks.rados.rados.0.smithi136.stdout:3579: finishing write tid 5 to smithi13667081-13 2024-09-12T23:50:36.032 INFO:tasks.rados.rados.0.smithi136.stdout:3579: finishing write tid 6 to smithi13667081-13 2024-09-12T23:50:36.032 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 13 v 2675 (ObjNum 1127 snap 343 seq_num 1127) dirty exists 2024-09-12T23:50:36.033 INFO:tasks.rados.rados.0.smithi136.stdout:3579: left oid 13 (ObjNum 1127 snap 343 seq_num 1127) 2024-09-12T23:50:36.497 INFO:tasks.rados.rados.0.smithi136.stdout:3576: done (4 left) 2024-09-12T23:50:36.498 INFO:tasks.rados.rados.0.smithi136.stdout:3577: done (3 left) 2024-09-12T23:50:36.498 INFO:tasks.rados.rados.0.smithi136.stdout:3578: done (2 left) 2024-09-12T23:50:36.498 INFO:tasks.rados.rados.0.smithi136.stdout:3579: done (1 left) 2024-09-12T23:50:36.498 INFO:tasks.rados.rados.0.smithi136.stdout:3580: done (0 left) 2024-09-12T23:50:36.498 INFO:tasks.rados.rados.0.smithi136.stdout:3581: read oid 35 snap 334 2024-09-12T23:50:36.498 INFO:tasks.rados.rados.0.smithi136.stdout:3581: expect (ObjNum 994 snap 308 seq_num 994) 2024-09-12T23:50:36.498 INFO:tasks.rados.rados.0.smithi136.stdout:3582: read oid 5 snap 335 2024-09-12T23:50:36.498 INFO:tasks.rados.rados.0.smithi136.stdout:3582: expect (ObjNum 1026 snap 316 seq_num 1026) 2024-09-12T23:50:36.498 INFO:tasks.rados.rados.0.smithi136.stdout:3583: rollback oid 29 current snap is 344 2024-09-12T23:50:36.498 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 29 to 335 2024-09-12T23:50:36.498 INFO:tasks.rados.rados.0.smithi136.stdout:3584: delete oid 14 current snap is 344 2024-09-12T23:50:36.520 INFO:tasks.rados.rados.0.smithi136.stdout:3584: done (3 left) 2024-09-12T23:50:36.520 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 14 current snap is 344 2024-09-12T23:50:36.520 INFO:tasks.rados.rados.0.smithi136.stdout:3585: seq_num 1129 ranges {0=516096,516096=770048,1286144=598016,1884160=237568} 2024-09-12T23:50:36.523 INFO:tasks.rados.rados.0.smithi136.stdout:3585: writing smithi13667081-14 from 0 to 516096 tid 1 2024-09-12T23:50:36.528 INFO:tasks.rados.rados.0.smithi136.stdout:3585: writing smithi13667081-14 from 516096 to 1286144 tid 2 2024-09-12T23:50:36.532 INFO:tasks.rados.rados.0.smithi136.stdout:3585: writing smithi13667081-14 from 1286144 to 1884160 tid 3 2024-09-12T23:50:36.535 INFO:tasks.rados.rados.0.smithi136.stdout:3585: writing smithi13667081-14 from 1884160 to 2121728 tid 4 2024-09-12T23:50:36.535 INFO:tasks.rados.rados.0.smithi136.stdout:3586: read oid 18 snap -1 2024-09-12T23:50:36.535 INFO:tasks.rados.rados.0.smithi136.stdout:3586: expect deleted 2024-09-12T23:50:36.535 INFO:tasks.rados.rados.0.smithi136.stdout:3587: read oid 2 snap -1 2024-09-12T23:50:36.535 INFO:tasks.rados.rados.0.smithi136.stdout:3587: expect (ObjNum 1092 snap 330 seq_num 1092) 2024-09-12T23:50:36.535 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 19 current snap is 344 2024-09-12T23:50:36.535 INFO:tasks.rados.rados.0.smithi136.stdout:3588: seq_num 1130 ranges {573440=532480,1105920=655360,1761280=753664,2514944=196608} 2024-09-12T23:50:36.542 INFO:tasks.rados.rados.0.smithi136.stdout:3588: writing smithi13667081-19 from 573440 to 1105920 tid 1 2024-09-12T23:50:36.546 INFO:tasks.rados.rados.0.smithi136.stdout:3588: writing smithi13667081-19 from 1105920 to 1761280 tid 2 2024-09-12T23:50:36.552 INFO:tasks.rados.rados.0.smithi136.stdout:3588: writing smithi13667081-19 from 1761280 to 2514944 tid 3 2024-09-12T23:50:36.554 INFO:tasks.rados.rados.0.smithi136.stdout:3588: writing smithi13667081-19 from 2514944 to 2711552 tid 4 2024-09-12T23:50:36.554 INFO:tasks.rados.rados.0.smithi136.stdout:3583: finishing rollback tid 0 to smithi13667081-29 2024-09-12T23:50:36.554 INFO:tasks.rados.rados.0.smithi136.stdout:3583: finishing rollback tid 1 to smithi13667081-29 2024-09-12T23:50:36.554 INFO:tasks.rados.rados.0.smithi136.stdout:3583: finishing rollback tid 2 to smithi13667081-29 2024-09-12T23:50:36.554 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 29 v 2678 (ObjNum 1086 snap 328 seq_num 1086) dirty exists 2024-09-12T23:50:36.554 INFO:tasks.rados.rados.0.smithi136.stdout:3583: done (6 left) 2024-09-12T23:50:36.554 INFO:tasks.rados.rados.0.smithi136.stdout:3586: done (5 left) 2024-09-12T23:50:36.554 INFO:tasks.rados.rados.0.smithi136.stdout:3589: copy_from oid 42 from oid 22 current snap is 344 2024-09-12T23:50:36.554 INFO:tasks.rados.rados.0.smithi136.stdout:3590: snap_create 2024-09-12T23:50:36.569 INFO:tasks.rados.rados.0.smithi136.stdout:3588: finishing write tid 1 to smithi13667081-19 2024-09-12T23:50:36.569 INFO:tasks.rados.rados.0.smithi136.stdout:3588: finishing write tid 2 to smithi13667081-19 2024-09-12T23:50:36.571 INFO:tasks.rados.rados.0.smithi136.stdout:3588: finishing write tid 3 to smithi13667081-19 2024-09-12T23:50:36.571 INFO:tasks.rados.rados.0.smithi136.stdout:3588: finishing write tid 4 to smithi13667081-19 2024-09-12T23:50:36.571 INFO:tasks.rados.rados.0.smithi136.stdout:3588: finishing write tid 5 to smithi13667081-19 2024-09-12T23:50:36.573 INFO:tasks.rados.rados.0.smithi136.stdout:3588: finishing write tid 6 to smithi13667081-19 2024-09-12T23:50:36.573 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 19 v 2483 (ObjNum 1130 snap 344 seq_num 1130) dirty exists 2024-09-12T23:50:36.573 INFO:tasks.rados.rados.0.smithi136.stdout:3588: left oid 19 (ObjNum 1130 snap 344 seq_num 1130) 2024-09-12T23:50:36.573 INFO:tasks.rados.rados.0.smithi136.stdout:3585: finishing write tid 1 to smithi13667081-14 2024-09-12T23:50:36.577 INFO:tasks.rados.rados.0.smithi136.stdout:3585: finishing write tid 2 to smithi13667081-14 2024-09-12T23:50:36.580 INFO:tasks.rados.rados.0.smithi136.stdout:3589: finishing copy_from racing read to smithi13667081-42 2024-09-12T23:50:36.585 INFO:tasks.rados.rados.0.smithi136.stdout:3587: expect (ObjNum 1092 snap 330 seq_num 1092) 2024-09-12T23:50:36.836 INFO:tasks.rados.rados.0.smithi136.stdout:3585: finishing write tid 3 to smithi13667081-14 2024-09-12T23:50:36.837 INFO:tasks.rados.rados.0.smithi136.stdout:3585: finishing write tid 4 to smithi13667081-14 2024-09-12T23:50:36.837 INFO:tasks.rados.rados.0.smithi136.stdout:3585: finishing write tid 5 to smithi13667081-14 2024-09-12T23:50:36.837 INFO:tasks.rados.rados.0.smithi136.stdout:3585: finishing write tid 6 to smithi13667081-14 2024-09-12T23:50:36.837 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 14 v 2497 (ObjNum 1129 snap 344 seq_num 1129) dirty exists 2024-09-12T23:50:36.837 INFO:tasks.rados.rados.0.smithi136.stdout:3585: left oid 14 (ObjNum 1129 snap 344 seq_num 1129) 2024-09-12T23:50:36.837 INFO:tasks.rados.rados.0.smithi136.stdout:3581: expect (ObjNum 994 snap 308 seq_num 994) 2024-09-12T23:50:36.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:36 smithi181 bash[67587]: cluster 2024-09-12T23:50:35.559721+0000 mon.a (mon.0) 1339 : cluster [DBG] osdmap e768: 8 total, 8 up, 8 in 2024-09-12T23:50:36.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:36 smithi181 bash[67587]: cluster 2024-09-12T23:50:36.497823+0000 mon.a (mon.0) 1340 : cluster [DBG] osdmap e769: 8 total, 8 up, 8 in 2024-09-12T23:50:37.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:36 smithi136 bash[75164]: cluster 2024-09-12T23:50:35.559721+0000 mon.a (mon.0) 1339 : cluster [DBG] osdmap e768: 8 total, 8 up, 8 in 2024-09-12T23:50:37.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:36 smithi136 bash[75164]: cluster 2024-09-12T23:50:36.497823+0000 mon.a (mon.0) 1340 : cluster [DBG] osdmap e769: 8 total, 8 up, 8 in 2024-09-12T23:50:37.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:36 smithi136 bash[76371]: cluster 2024-09-12T23:50:35.559721+0000 mon.a (mon.0) 1339 : cluster [DBG] osdmap e768: 8 total, 8 up, 8 in 2024-09-12T23:50:37.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:36 smithi136 bash[76371]: cluster 2024-09-12T23:50:36.497823+0000 mon.a (mon.0) 1340 : cluster [DBG] osdmap e769: 8 total, 8 up, 8 in 2024-09-12T23:50:37.262 INFO:tasks.rados.rados.0.smithi136.stdout:3589: finishing copy_from to smithi13667081-42 2024-09-12T23:50:37.262 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 42 v 2487 (ObjNum 1101 snap 333 seq_num 1101) dirty exists 2024-09-12T23:50:37.262 INFO:tasks.rados.rados.0.smithi136.stdout:3582: expect (ObjNum 1026 snap 316 seq_num 1026) 2024-09-12T23:50:37.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:37 smithi181 bash[67587]: cluster 2024-09-12T23:50:36.854457+0000 mgr.y (mgr.24661) 402 : cluster [DBG] pgmap v735: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 296 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 4.3 MiB/s wr, 14 op/s 2024-09-12T23:50:37.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:37 smithi181 bash[67587]: cluster 2024-09-12T23:50:37.512621+0000 mon.a (mon.0) 1341 : cluster [DBG] osdmap e770: 8 total, 8 up, 8 in 2024-09-12T23:50:38.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:37 smithi136 bash[75164]: cluster 2024-09-12T23:50:36.854457+0000 mgr.y (mgr.24661) 402 : cluster [DBG] pgmap v735: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 296 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 4.3 MiB/s wr, 14 op/s 2024-09-12T23:50:38.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:37 smithi136 bash[75164]: cluster 2024-09-12T23:50:37.512621+0000 mon.a (mon.0) 1341 : cluster [DBG] osdmap e770: 8 total, 8 up, 8 in 2024-09-12T23:50:38.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:37 smithi136 bash[76371]: cluster 2024-09-12T23:50:36.854457+0000 mgr.y (mgr.24661) 402 : cluster [DBG] pgmap v735: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 296 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 4.3 MiB/s wr, 14 op/s 2024-09-12T23:50:38.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:37 smithi136 bash[76371]: cluster 2024-09-12T23:50:37.512621+0000 mon.a (mon.0) 1341 : cluster [DBG] osdmap e770: 8 total, 8 up, 8 in 2024-09-12T23:50:39.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:38 smithi181 bash[67587]: audit 2024-09-12T23:50:38.753197+0000 mon.a (mon.0) 1342 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:50:39.099 INFO:tasks.rados.rados.0.smithi136.stdout:3581: done (6 left) 2024-09-12T23:50:39.100 INFO:tasks.rados.rados.0.smithi136.stdout:3582: done (5 left) 2024-09-12T23:50:39.100 INFO:tasks.rados.rados.0.smithi136.stdout:3585: done (4 left) 2024-09-12T23:50:39.100 INFO:tasks.rados.rados.0.smithi136.stdout:3587: done (3 left) 2024-09-12T23:50:39.100 INFO:tasks.rados.rados.0.smithi136.stdout:3588: done (2 left) 2024-09-12T23:50:39.100 INFO:tasks.rados.rados.0.smithi136.stdout:3589: done (1 left) 2024-09-12T23:50:39.100 INFO:tasks.rados.rados.0.smithi136.stdout:3590: done (0 left) 2024-09-12T23:50:39.100 INFO:tasks.rados.rados.0.smithi136.stdout:3591: delete oid 26 current snap is 345 2024-09-12T23:50:39.106 INFO:tasks.rados.rados.0.smithi136.stdout:3591: done (0 left) 2024-09-12T23:50:39.106 INFO:tasks.rados.rados.0.smithi136.stdout:3592: snap_remove snap 341 2024-09-12T23:50:39.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:38 smithi136 bash[75164]: audit 2024-09-12T23:50:38.753197+0000 mon.a (mon.0) 1342 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:50:39.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:38 smithi136 bash[76371]: audit 2024-09-12T23:50:38.753197+0000 mon.a (mon.0) 1342 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:50:39.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:50:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:50:39] "GET /metrics HTTP/1.1" 200 36437 "" "Prometheus/2.43.0" 2024-09-12T23:50:39.822 INFO:tasks.rados.rados.0.smithi136.stdout:3592: done (0 left) 2024-09-12T23:50:39.822 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 1 current snap is 345 2024-09-12T23:50:39.822 INFO:tasks.rados.rados.0.smithi136.stdout:3593: seq_num 1132 ranges {9838592=565248,10403840=425984,10829824=770048,11599872=409600} 2024-09-12T23:50:39.904 INFO:tasks.rados.rados.0.smithi136.stdout:3593: writing smithi13667081-1 from 9838592 to 10403840 tid 1 2024-09-12T23:50:39.907 INFO:tasks.rados.rados.0.smithi136.stdout:3593: writing smithi13667081-1 from 10403840 to 10829824 tid 2 2024-09-12T23:50:39.911 INFO:tasks.rados.rados.0.smithi136.stdout:3593: writing smithi13667081-1 from 10829824 to 11599872 tid 3 2024-09-12T23:50:39.914 INFO:tasks.rados.rados.0.smithi136.stdout:3593: writing smithi13667081-1 from 11599872 to 12009472 tid 4 2024-09-12T23:50:39.914 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 31 current snap is 345 2024-09-12T23:50:39.914 INFO:tasks.rados.rados.0.smithi136.stdout:3593: finishing write tid 1 to smithi13667081-1 2024-09-12T23:50:39.914 INFO:tasks.rados.rados.0.smithi136.stdout:3594: seq_num 1133 ranges {1130496=581632,1712128=753664,2465792=573440,3039232=278528} 2024-09-12T23:50:39.924 INFO:tasks.rados.rados.0.smithi136.stdout:3594: writing smithi13667081-31 from 1130496 to 1712128 tid 1 2024-09-12T23:50:39.928 INFO:tasks.rados.rados.0.smithi136.stdout:3594: writing smithi13667081-31 from 1712128 to 2465792 tid 2 2024-09-12T23:50:39.931 INFO:tasks.rados.rados.0.smithi136.stdout:3594: writing smithi13667081-31 from 2465792 to 3039232 tid 3 2024-09-12T23:50:39.934 INFO:tasks.rados.rados.0.smithi136.stdout:3594: writing smithi13667081-31 from 3039232 to 3317760 tid 4 2024-09-12T23:50:39.934 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 38 current snap is 345 2024-09-12T23:50:39.934 INFO:tasks.rados.rados.0.smithi136.stdout:3593: finishing write tid 2 to smithi13667081-1 2024-09-12T23:50:39.934 INFO:tasks.rados.rados.0.smithi136.stdout:3595: seq_num 1134 ranges {2457600=598016,3055616=638976,3694592=737280,4431872=229376} 2024-09-12T23:50:39.951 INFO:tasks.rados.rados.0.smithi136.stdout:3595: writing smithi13667081-38 from 2457600 to 3055616 tid 1 2024-09-12T23:50:39.954 INFO:tasks.rados.rados.0.smithi136.stdout:3595: writing smithi13667081-38 from 3055616 to 3694592 tid 2 2024-09-12T23:50:39.958 INFO:tasks.rados.rados.0.smithi136.stdout:3595: writing smithi13667081-38 from 3694592 to 4431872 tid 3 2024-09-12T23:50:39.960 INFO:tasks.rados.rados.0.smithi136.stdout:3595: writing smithi13667081-38 from 4431872 to 4661248 tid 4 2024-09-12T23:50:39.960 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 44 current snap is 345 2024-09-12T23:50:39.960 INFO:tasks.rados.rados.0.smithi136.stdout:3596: seq_num 1135 ranges {2015232=614400,2629632=524288,3153920=540672,3694592=483328,4177920=57344} 2024-09-12T23:50:39.974 INFO:tasks.rados.rados.0.smithi136.stdout:3596: writing smithi13667081-44 from 2015232 to 2629632 tid 1 2024-09-12T23:50:39.978 INFO:tasks.rados.rados.0.smithi136.stdout:3596: writing smithi13667081-44 from 2629632 to 3153920 tid 2 2024-09-12T23:50:39.981 INFO:tasks.rados.rados.0.smithi136.stdout:3596: writing smithi13667081-44 from 3153920 to 3694592 tid 3 2024-09-12T23:50:39.984 INFO:tasks.rados.rados.0.smithi136.stdout:3596: writing smithi13667081-44 from 3694592 to 4177920 tid 4 2024-09-12T23:50:39.985 INFO:tasks.rados.rados.0.smithi136.stdout:3596: writing smithi13667081-44 from 4177920 to 4235264 tid 5 2024-09-12T23:50:39.985 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 5 current snap is 345 2024-09-12T23:50:39.985 INFO:tasks.rados.rados.0.smithi136.stdout:3593: finishing write tid 3 to smithi13667081-1 2024-09-12T23:50:39.985 INFO:tasks.rados.rados.0.smithi136.stdout:3597: seq_num 1136 ranges {15523840=630784,16154624=409600,16564224=753664,17317888=417792,17735680=24576} 2024-09-12T23:50:40.075 INFO:tasks.rados.rados.0.smithi136.stdout:3597: writing smithi13667081-5 from 15523840 to 16154624 tid 1 2024-09-12T23:50:40.077 INFO:tasks.rados.rados.0.smithi136.stdout:3597: writing smithi13667081-5 from 16154624 to 16564224 tid 2 2024-09-12T23:50:40.081 INFO:tasks.rados.rados.0.smithi136.stdout:3597: writing smithi13667081-5 from 16564224 to 17317888 tid 3 2024-09-12T23:50:40.086 INFO:tasks.rados.rados.0.smithi136.stdout:3597: writing smithi13667081-5 from 17317888 to 17735680 tid 4 2024-09-12T23:50:40.086 INFO:tasks.rados.rados.0.smithi136.stdout:3597: writing smithi13667081-5 from 17735680 to 17760256 tid 5 2024-09-12T23:50:40.086 INFO:tasks.rados.rados.0.smithi136.stdout:3598: rmattr oid 26 current snap is 345 2024-09-12T23:50:40.086 INFO:tasks.rados.rados.0.smithi136.stdout:3598: done (5 left) 2024-09-12T23:50:40.087 INFO:tasks.rados.rados.0.smithi136.stdout:3599: rollback oid 25 current snap is 345 2024-09-12T23:50:40.087 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 25 to 296 2024-09-12T23:50:40.087 INFO:tasks.rados.rados.0.smithi136.stdout:3593: finishing write tid 4 to smithi13667081-1 2024-09-12T23:50:40.087 INFO:tasks.rados.rados.0.smithi136.stdout:3593: finishing write tid 5 to smithi13667081-1 2024-09-12T23:50:40.087 INFO:tasks.rados.rados.0.smithi136.stdout:3593: finishing write tid 6 to smithi13667081-1 2024-09-12T23:50:40.087 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 1 v 2498 (ObjNum 1132 snap 345 seq_num 1132) dirty exists 2024-09-12T23:50:40.087 INFO:tasks.rados.rados.0.smithi136.stdout:3593: left oid 1 (ObjNum 1132 snap 345 seq_num 1132) 2024-09-12T23:50:40.087 INFO:tasks.rados.rados.0.smithi136.stdout:3594: finishing write tid 1 to smithi13667081-31 2024-09-12T23:50:40.087 INFO:tasks.rados.rados.0.smithi136.stdout:3593: done (5 left) 2024-09-12T23:50:40.087 INFO:tasks.rados.rados.0.smithi136.stdout:3600: rmattr oid 49 current snap is 345 2024-09-12T23:50:40.087 INFO:tasks.rados.rados.0.smithi136.stdout:3600: done (5 left) 2024-09-12T23:50:40.087 INFO:tasks.rados.rados.0.smithi136.stdout:3601: rmattr oid 2 current snap is 345 2024-09-12T23:50:40.087 INFO:tasks.rados.rados.0.smithi136.stdout:3594: finishing write tid 2 to smithi13667081-31 2024-09-12T23:50:40.088 INFO:tasks.rados.rados.0.smithi136.stdout:3594: finishing write tid 3 to smithi13667081-31 2024-09-12T23:50:40.088 INFO:tasks.rados.rados.0.smithi136.stdout:3594: finishing write tid 4 to smithi13667081-31 2024-09-12T23:50:40.088 INFO:tasks.rados.rados.0.smithi136.stdout:3594: finishing write tid 5 to smithi13667081-31 2024-09-12T23:50:40.088 INFO:tasks.rados.rados.0.smithi136.stdout:3594: finishing write tid 6 to smithi13667081-31 2024-09-12T23:50:40.088 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 31 v 2487 (ObjNum 1133 snap 345 seq_num 1133) dirty exists 2024-09-12T23:50:40.088 INFO:tasks.rados.rados.0.smithi136.stdout:3594: left oid 31 (ObjNum 1133 snap 345 seq_num 1133) 2024-09-12T23:50:40.088 INFO:tasks.rados.rados.0.smithi136.stdout:3595: finishing write tid 1 to smithi13667081-38 2024-09-12T23:50:40.088 INFO:tasks.rados.rados.0.smithi136.stdout:3595: finishing write tid 2 to smithi13667081-38 2024-09-12T23:50:40.088 INFO:tasks.rados.rados.0.smithi136.stdout:3595: finishing write tid 3 to smithi13667081-38 2024-09-12T23:50:40.088 INFO:tasks.rados.rados.0.smithi136.stdout:3595: finishing write tid 4 to smithi13667081-38 2024-09-12T23:50:40.088 INFO:tasks.rados.rados.0.smithi136.stdout:3595: finishing write tid 5 to smithi13667081-38 2024-09-12T23:50:40.089 INFO:tasks.rados.rados.0.smithi136.stdout:3595: finishing write tid 6 to smithi13667081-38 2024-09-12T23:50:40.089 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 38 v 2683 (ObjNum 1134 snap 345 seq_num 1134) dirty exists 2024-09-12T23:50:40.089 INFO:tasks.rados.rados.0.smithi136.stdout:3595: left oid 38 (ObjNum 1134 snap 345 seq_num 1134) 2024-09-12T23:50:40.089 INFO:tasks.rados.rados.0.smithi136.stdout:3596: finishing write tid 1 to smithi13667081-44 2024-09-12T23:50:40.089 INFO:tasks.rados.rados.0.smithi136.stdout:3596: finishing write tid 2 to smithi13667081-44 2024-09-12T23:50:40.089 INFO:tasks.rados.rados.0.smithi136.stdout:3596: finishing write tid 3 to smithi13667081-44 2024-09-12T23:50:40.089 INFO:tasks.rados.rados.0.smithi136.stdout:3596: finishing write tid 4 to smithi13667081-44 2024-09-12T23:50:40.089 INFO:tasks.rados.rados.0.smithi136.stdout:3596: finishing write tid 5 to smithi13667081-44 2024-09-12T23:50:40.089 INFO:tasks.rados.rados.0.smithi136.stdout:3602: read oid 46 snap -1 2024-09-12T23:50:40.089 INFO:tasks.rados.rados.0.smithi136.stdout:3602: expect deleted 2024-09-12T23:50:40.089 INFO:tasks.rados.rados.0.smithi136.stdout:3594: done (6 left) 2024-09-12T23:50:40.089 INFO:tasks.rados.rados.0.smithi136.stdout:3595: done (5 left) 2024-09-12T23:50:40.090 INFO:tasks.rados.rados.0.smithi136.stdout:3603: rollback oid 1 current snap is 345 2024-09-12T23:50:40.090 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 1 to 335 2024-09-12T23:50:40.090 INFO:tasks.rados.rados.0.smithi136.stdout:3596: finishing write tid 6 to smithi13667081-44 2024-09-12T23:50:40.090 INFO:tasks.rados.rados.0.smithi136.stdout:3604: read oid 40 snap -1 2024-09-12T23:50:40.090 INFO:tasks.rados.rados.0.smithi136.stdout:3604: expect deleted 2024-09-12T23:50:40.090 INFO:tasks.rados.rados.0.smithi136.stdout:3605: setattr oid 7 current snap is 345 2024-09-12T23:50:40.090 INFO:tasks.rados.rados.0.smithi136.stdout:3596: finishing write tid 7 to smithi13667081-44 2024-09-12T23:50:40.090 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 44 v 2503 (ObjNum 1135 snap 345 seq_num 1135) dirty exists 2024-09-12T23:50:40.090 INFO:tasks.rados.rados.0.smithi136.stdout:3596: left oid 44 (ObjNum 1135 snap 345 seq_num 1135) 2024-09-12T23:50:40.090 INFO:tasks.rados.rados.0.smithi136.stdout:3597: finishing write tid 1 to smithi13667081-5 2024-09-12T23:50:40.090 INFO:tasks.rados.rados.0.smithi136.stdout:3597: finishing write tid 2 to smithi13667081-5 2024-09-12T23:50:40.091 INFO:tasks.rados.rados.0.smithi136.stdout:3596: done (7 left) 2024-09-12T23:50:40.091 INFO:tasks.rados.rados.0.smithi136.stdout:3602: done (6 left) 2024-09-12T23:50:40.091 INFO:tasks.rados.rados.0.smithi136.stdout:3606: rmattr oid 30 current snap is 345 2024-09-12T23:50:40.092 INFO:tasks.rados.rados.0.smithi136.stdout:3607: read oid 22 snap -1 2024-09-12T23:50:40.092 INFO:tasks.rados.rados.0.smithi136.stdout:3607: expect (ObjNum 1101 snap 333 seq_num 1101) 2024-09-12T23:50:40.092 INFO:tasks.rados.rados.0.smithi136.stdout:3608: read oid 33 snap -1 2024-09-12T23:50:40.092 INFO:tasks.rados.rados.0.smithi136.stdout:3608: expect deleted 2024-09-12T23:50:40.092 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 28 current snap is 345 2024-09-12T23:50:40.092 INFO:tasks.rados.rados.0.smithi136.stdout:3609: seq_num 1137 ranges {4431872=647168,5079040=696320,5775360=557056,6332416=352256} 2024-09-12T23:50:40.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:39 smithi181 bash[67587]: cluster 2024-09-12T23:50:38.855338+0000 mgr.y (mgr.24661) 403 : cluster [DBG] pgmap v737: 121 pgs: 121 active+clean; 295 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 4.6 MiB/s wr, 17 op/s 2024-09-12T23:50:40.118 INFO:tasks.rados.rados.0.smithi136.stdout:3609: writing smithi13667081-28 from 4431872 to 5079040 tid 1 2024-09-12T23:50:40.122 INFO:tasks.rados.rados.0.smithi136.stdout:3609: writing smithi13667081-28 from 5079040 to 5775360 tid 2 2024-09-12T23:50:40.125 INFO:tasks.rados.rados.0.smithi136.stdout:3609: writing smithi13667081-28 from 5775360 to 6332416 tid 3 2024-09-12T23:50:40.128 INFO:tasks.rados.rados.0.smithi136.stdout:3609: writing smithi13667081-28 from 6332416 to 6684672 tid 4 2024-09-12T23:50:40.128 INFO:tasks.rados.rados.0.smithi136.stdout:3610: read oid 44 snap -1 2024-09-12T23:50:40.128 INFO:tasks.rados.rados.0.smithi136.stdout:3610: expect (ObjNum 1135 snap 345 seq_num 1135) 2024-09-12T23:50:40.128 INFO:tasks.rados.rados.0.smithi136.stdout:3611: rmattr oid 10 current snap is 345 2024-09-12T23:50:40.129 INFO:tasks.rados.rados.0.smithi136.stdout:3611: done (11 left) 2024-09-12T23:50:40.129 INFO:tasks.rados.rados.0.smithi136.stdout:3612: read oid 45 snap 335 2024-09-12T23:50:40.129 INFO:tasks.rados.rados.0.smithi136.stdout:3612: expect deleted 2024-09-12T23:50:40.129 INFO:tasks.rados.rados.0.smithi136.stdout:3613: rollback oid 46 current snap is 345 2024-09-12T23:50:40.129 INFO:tasks.rados.rados.0.smithi136.stdout:3599: finishing rollback tid 0 to smithi13667081-25 2024-09-12T23:50:40.129 INFO:tasks.rados.rados.0.smithi136.stdout:3603: finishing rollback tid 0 to smithi13667081-1 2024-09-12T23:50:40.129 INFO:tasks.rados.rados.0.smithi136.stdout:3599: finishing rollback tid 1 to smithi13667081-25 2024-09-12T23:50:40.129 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 25 v 2458 (ObjNum 257 snap 0 seq_num 4243367088) dirty dne 2024-09-12T23:50:40.129 INFO:tasks.rados.rados.0.smithi136.stdout:3603: finishing rollback tid 1 to smithi13667081-1 2024-09-12T23:50:40.129 INFO:tasks.rados.rados.0.smithi136.stdout:3603: finishing rollback tid 2 to smithi13667081-1 2024-09-12T23:50:40.129 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 1 v 2501 (ObjNum 1059 snap 321 seq_num 1059) dirty exists 2024-09-12T23:50:40.129 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 30 v 2583 (ObjNum 1091 snap 329 seq_num 1091) dirty exists 2024-09-12T23:50:40.130 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 7 v 1647 (ObjNum 1110 snap 336 seq_num 1110) dirty exists 2024-09-12T23:50:40.130 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 2 v 2504 (ObjNum 1092 snap 330 seq_num 1092) dirty exists 2024-09-12T23:50:40.130 INFO:tasks.rados.rados.0.smithi136.stdout:3597: finishing write tid 3 to smithi13667081-5 2024-09-12T23:50:40.130 INFO:tasks.rados.rados.0.smithi136.stdout:3597: finishing write tid 4 to smithi13667081-5 2024-09-12T23:50:40.130 INFO:tasks.rados.rados.0.smithi136.stdout:3597: finishing write tid 5 to smithi13667081-5 2024-09-12T23:50:40.130 INFO:tasks.rados.rados.0.smithi136.stdout:3597: finishing write tid 6 to smithi13667081-5 2024-09-12T23:50:40.130 INFO:tasks.rados.rados.0.smithi136.stdout:3597: finishing write tid 7 to smithi13667081-5 2024-09-12T23:50:40.130 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 5 v 2780 (ObjNum 1136 snap 345 seq_num 1136) dirty exists 2024-09-12T23:50:40.130 INFO:tasks.rados.rados.0.smithi136.stdout:3597: left oid 5 (ObjNum 1136 snap 345 seq_num 1136) 2024-09-12T23:50:40.130 INFO:tasks.rados.rados.0.smithi136.stdout:3607: expect (ObjNum 1101 snap 333 seq_num 1101) 2024-09-12T23:50:40.200 INFO:tasks.rados.rados.0.smithi136.stdout:3609: finishing write tid 1 to smithi13667081-28 2024-09-12T23:50:40.200 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 46 to 334 2024-09-12T23:50:40.201 INFO:tasks.rados.rados.0.smithi136.stdout:3609: finishing write tid 2 to smithi13667081-28 2024-09-12T23:50:40.201 INFO:tasks.rados.rados.0.smithi136.stdout:3609: finishing write tid 3 to smithi13667081-28 2024-09-12T23:50:40.201 INFO:tasks.rados.rados.0.smithi136.stdout:3609: finishing write tid 4 to smithi13667081-28 2024-09-12T23:50:40.201 INFO:tasks.rados.rados.0.smithi136.stdout:3609: finishing write tid 5 to smithi13667081-28 2024-09-12T23:50:40.201 INFO:tasks.rados.rados.0.smithi136.stdout:3609: finishing write tid 6 to smithi13667081-28 2024-09-12T23:50:40.201 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 28 v 2508 (ObjNum 1137 snap 345 seq_num 1137) dirty exists 2024-09-12T23:50:40.201 INFO:tasks.rados.rados.0.smithi136.stdout:3609: left oid 28 (ObjNum 1137 snap 345 seq_num 1137) 2024-09-12T23:50:40.201 INFO:tasks.rados.rados.0.smithi136.stdout:3610: expect (ObjNum 1135 snap 345 seq_num 1135) 2024-09-12T23:50:40.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:39 smithi136 bash[75164]: cluster 2024-09-12T23:50:38.855338+0000 mgr.y (mgr.24661) 403 : cluster [DBG] pgmap v737: 121 pgs: 121 active+clean; 295 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 4.6 MiB/s wr, 17 op/s 2024-09-12T23:50:40.256 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:39 smithi136 bash[76371]: cluster 2024-09-12T23:50:38.855338+0000 mgr.y (mgr.24661) 403 : cluster [DBG] pgmap v737: 121 pgs: 121 active+clean; 295 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 4.6 MiB/s wr, 17 op/s 2024-09-12T23:50:40.341 INFO:tasks.rados.rados.0.smithi136.stdout:3613: finishing rollback tid 1 to smithi13667081-46 2024-09-12T23:50:40.341 INFO:tasks.rados.rados.0.smithi136.stdout:3613: finishing rollback tid 2 to smithi13667081-46 2024-09-12T23:50:40.341 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 46 v 2490 (ObjNum 1094 snap 331 seq_num 1094) dirty exists 2024-09-12T23:50:40.342 INFO:tasks.rados.rados.0.smithi136.stdout:3597: done (12 left) 2024-09-12T23:50:40.342 INFO:tasks.rados.rados.0.smithi136.stdout:3599: done (11 left) 2024-09-12T23:50:40.342 INFO:tasks.rados.rados.0.smithi136.stdout:3601: done (10 left) 2024-09-12T23:50:40.342 INFO:tasks.rados.rados.0.smithi136.stdout:3603: done (9 left) 2024-09-12T23:50:40.342 INFO:tasks.rados.rados.0.smithi136.stdout:3604: done (8 left) 2024-09-12T23:50:40.342 INFO:tasks.rados.rados.0.smithi136.stdout:3605: done (7 left) 2024-09-12T23:50:40.342 INFO:tasks.rados.rados.0.smithi136.stdout:3606: done (6 left) 2024-09-12T23:50:40.342 INFO:tasks.rados.rados.0.smithi136.stdout:3607: done (5 left) 2024-09-12T23:50:40.342 INFO:tasks.rados.rados.0.smithi136.stdout:3608: done (4 left) 2024-09-12T23:50:40.342 INFO:tasks.rados.rados.0.smithi136.stdout:3609: done (3 left) 2024-09-12T23:50:40.342 INFO:tasks.rados.rados.0.smithi136.stdout:3610: done (2 left) 2024-09-12T23:50:40.342 INFO:tasks.rados.rados.0.smithi136.stdout:3612: done (1 left) 2024-09-12T23:50:40.342 INFO:tasks.rados.rados.0.smithi136.stdout:3613: done (0 left) 2024-09-12T23:50:40.343 INFO:tasks.rados.rados.0.smithi136.stdout:3614: read oid 17 snap 336 2024-09-12T23:50:40.343 INFO:tasks.rados.rados.0.smithi136.stdout:3614: expect (ObjNum 1007 snap 313 seq_num 1007) 2024-09-12T23:50:40.343 INFO:tasks.rados.rados.0.smithi136.stdout:3615: snap_remove snap 344 2024-09-12T23:50:40.828 INFO:tasks.rados.rados.0.smithi136.stdout:3615: done (1 left) 2024-09-12T23:50:40.828 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 4 current snap is 345 2024-09-12T23:50:40.828 INFO:tasks.rados.rados.0.smithi136.stdout:3614: expect (ObjNum 1007 snap 313 seq_num 1007) 2024-09-12T23:50:41.071 INFO:tasks.rados.rados.0.smithi136.stdout:3616: seq_num 1138 ranges {20267008=663552,20930560=581632,21512192=712704,22224896=311296} 2024-09-12T23:50:41.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:40 smithi181 bash[67587]: cluster 2024-09-12T23:50:39.824193+0000 mon.a (mon.0) 1343 : cluster [DBG] osdmap e771: 8 total, 8 up, 8 in 2024-09-12T23:50:41.215 INFO:tasks.rados.rados.0.smithi136.stdout:3616: writing smithi13667081-4 from 20267008 to 20930560 tid 1 2024-09-12T23:50:41.219 INFO:tasks.rados.rados.0.smithi136.stdout:3616: writing smithi13667081-4 from 20930560 to 21512192 tid 2 2024-09-12T23:50:41.223 INFO:tasks.rados.rados.0.smithi136.stdout:3616: writing smithi13667081-4 from 21512192 to 22224896 tid 3 2024-09-12T23:50:41.225 INFO:tasks.rados.rados.0.smithi136.stdout:3616: writing smithi13667081-4 from 22224896 to 22536192 tid 4 2024-09-12T23:50:41.225 INFO:tasks.rados.rados.0.smithi136.stdout:3614: done (1 left) 2024-09-12T23:50:41.225 INFO:tasks.rados.rados.0.smithi136.stdout:3617: rollback oid 18 current snap is 345 2024-09-12T23:50:41.225 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 18 to 336 2024-09-12T23:50:41.225 INFO:tasks.rados.rados.0.smithi136.stdout:3618: snap_remove snap 343 2024-09-12T23:50:41.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:40 smithi136 bash[75164]: cluster 2024-09-12T23:50:39.824193+0000 mon.a (mon.0) 1343 : cluster [DBG] osdmap e771: 8 total, 8 up, 8 in 2024-09-12T23:50:41.256 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:40 smithi136 bash[76371]: cluster 2024-09-12T23:50:39.824193+0000 mon.a (mon.0) 1343 : cluster [DBG] osdmap e771: 8 total, 8 up, 8 in 2024-09-12T23:50:41.499 INFO:tasks.rados.rados.0.smithi136.stdout:3618: done (2 left) 2024-09-12T23:50:41.500 INFO:tasks.rados.rados.0.smithi136.stdout:3619: read oid 17 snap -1 2024-09-12T23:50:41.500 INFO:tasks.rados.rados.0.smithi136.stdout:3619: expect (ObjNum 1007 snap 313 seq_num 1007) 2024-09-12T23:50:41.501 INFO:tasks.rados.rados.0.smithi136.stdout:3617: finishing rollback tid 1 to smithi13667081-18 2024-09-12T23:50:41.501 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 18 v 1641 (ObjNum 65 snap 0 seq_num 1884973920) dirty dne 2024-09-12T23:50:41.501 INFO:tasks.rados.rados.0.smithi136.stdout:3616: finishing write tid 1 to smithi13667081-4 2024-09-12T23:50:41.501 INFO:tasks.rados.rados.0.smithi136.stdout:3616: finishing write tid 2 to smithi13667081-4 2024-09-12T23:50:41.501 INFO:tasks.rados.rados.0.smithi136.stdout:3616: finishing write tid 3 to smithi13667081-4 2024-09-12T23:50:41.501 INFO:tasks.rados.rados.0.smithi136.stdout:3616: finishing write tid 4 to smithi13667081-4 2024-09-12T23:50:41.501 INFO:tasks.rados.rados.0.smithi136.stdout:3616: finishing write tid 5 to smithi13667081-4 2024-09-12T23:50:41.501 INFO:tasks.rados.rados.0.smithi136.stdout:3616: finishing write tid 6 to smithi13667081-4 2024-09-12T23:50:41.501 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 4 v 1676 (ObjNum 1138 snap 345 seq_num 1138) dirty exists 2024-09-12T23:50:41.502 INFO:tasks.rados.rados.0.smithi136.stdout:3616: left oid 4 (ObjNum 1138 snap 345 seq_num 1138) 2024-09-12T23:50:41.502 INFO:tasks.rados.rados.0.smithi136.stdout:3616: done (2 left) 2024-09-12T23:50:41.502 INFO:tasks.rados.rados.0.smithi136.stdout:3617: done (1 left) 2024-09-12T23:50:41.502 INFO:tasks.rados.rados.0.smithi136.stdout:3620: read oid 6 snap -1 2024-09-12T23:50:41.502 INFO:tasks.rados.rados.0.smithi136.stdout:3620: expect deleted 2024-09-12T23:50:41.502 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 16 current snap is 345 2024-09-12T23:50:41.502 INFO:tasks.rados.rados.0.smithi136.stdout:3621: seq_num 1139 ranges {1949696=679936,2629632=458752,3088384=524288,3612672=540672,4153344=81920} 2024-09-12T23:50:41.516 INFO:tasks.rados.rados.0.smithi136.stdout:3621: writing smithi13667081-16 from 1949696 to 2629632 tid 1 2024-09-12T23:50:41.519 INFO:tasks.rados.rados.0.smithi136.stdout:3621: writing smithi13667081-16 from 2629632 to 3088384 tid 2 2024-09-12T23:50:41.523 INFO:tasks.rados.rados.0.smithi136.stdout:3621: writing smithi13667081-16 from 3088384 to 3612672 tid 3 2024-09-12T23:50:41.526 INFO:tasks.rados.rados.0.smithi136.stdout:3621: writing smithi13667081-16 from 3612672 to 4153344 tid 4 2024-09-12T23:50:41.527 INFO:tasks.rados.rados.0.smithi136.stdout:3621: writing smithi13667081-16 from 4153344 to 4235264 tid 5 2024-09-12T23:50:41.527 INFO:tasks.rados.rados.0.smithi136.stdout:3622: snap_create 2024-09-12T23:50:41.542 INFO:tasks.rados.rados.0.smithi136.stdout:3619: expect (ObjNum 1007 snap 313 seq_num 1007) 2024-09-12T23:50:41.757 INFO:tasks.rados.rados.0.smithi136.stdout:3621: finishing write tid 1 to smithi13667081-16 2024-09-12T23:50:41.757 INFO:tasks.rados.rados.0.smithi136.stdout:3621: finishing write tid 2 to smithi13667081-16 2024-09-12T23:50:41.757 INFO:tasks.rados.rados.0.smithi136.stdout:3621: finishing write tid 3 to smithi13667081-16 2024-09-12T23:50:41.757 INFO:tasks.rados.rados.0.smithi136.stdout:3621: finishing write tid 4 to smithi13667081-16 2024-09-12T23:50:41.757 INFO:tasks.rados.rados.0.smithi136.stdout:3621: finishing write tid 5 to smithi13667081-16 2024-09-12T23:50:41.757 INFO:tasks.rados.rados.0.smithi136.stdout:3621: finishing write tid 6 to smithi13667081-16 2024-09-12T23:50:41.757 INFO:tasks.rados.rados.0.smithi136.stdout:3621: finishing write tid 7 to smithi13667081-16 2024-09-12T23:50:41.757 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 16 v 2514 (ObjNum 1139 snap 345 seq_num 1139) dirty exists 2024-09-12T23:50:41.757 INFO:tasks.rados.rados.0.smithi136.stdout:3621: left oid 16 (ObjNum 1139 snap 345 seq_num 1139) 2024-09-12T23:50:42.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:41 smithi181 bash[67587]: cluster 2024-09-12T23:50:40.828262+0000 mon.a (mon.0) 1344 : cluster [DBG] osdmap e772: 8 total, 8 up, 8 in 2024-09-12T23:50:42.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:41 smithi181 bash[67587]: cluster 2024-09-12T23:50:40.857210+0000 mgr.y (mgr.24661) 404 : cluster [DBG] pgmap v740: 121 pgs: 121 active+clean; 299 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 5.5 MiB/s wr, 24 op/s 2024-09-12T23:50:42.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:41 smithi181 bash[67587]: cluster 2024-09-12T23:50:41.502545+0000 mon.a (mon.0) 1345 : cluster [DBG] osdmap e773: 8 total, 8 up, 8 in 2024-09-12T23:50:42.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:41 smithi136 bash[75164]: cluster 2024-09-12T23:50:40.828262+0000 mon.a (mon.0) 1344 : cluster [DBG] osdmap e772: 8 total, 8 up, 8 in 2024-09-12T23:50:42.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:41 smithi136 bash[75164]: cluster 2024-09-12T23:50:40.857210+0000 mgr.y (mgr.24661) 404 : cluster [DBG] pgmap v740: 121 pgs: 121 active+clean; 299 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 5.5 MiB/s wr, 24 op/s 2024-09-12T23:50:42.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:41 smithi136 bash[75164]: cluster 2024-09-12T23:50:41.502545+0000 mon.a (mon.0) 1345 : cluster [DBG] osdmap e773: 8 total, 8 up, 8 in 2024-09-12T23:50:42.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:41 smithi136 bash[76371]: cluster 2024-09-12T23:50:40.828262+0000 mon.a (mon.0) 1344 : cluster [DBG] osdmap e772: 8 total, 8 up, 8 in 2024-09-12T23:50:42.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:41 smithi136 bash[76371]: cluster 2024-09-12T23:50:40.857210+0000 mgr.y (mgr.24661) 404 : cluster [DBG] pgmap v740: 121 pgs: 121 active+clean; 299 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 5.5 MiB/s wr, 24 op/s 2024-09-12T23:50:42.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:41 smithi136 bash[76371]: cluster 2024-09-12T23:50:41.502545+0000 mon.a (mon.0) 1345 : cluster [DBG] osdmap e773: 8 total, 8 up, 8 in 2024-09-12T23:50:42.502 INFO:tasks.rados.rados.0.smithi136.stdout:3619: done (3 left) 2024-09-12T23:50:42.502 INFO:tasks.rados.rados.0.smithi136.stdout:3620: done (2 left) 2024-09-12T23:50:42.502 INFO:tasks.rados.rados.0.smithi136.stdout:3621: done (1 left) 2024-09-12T23:50:42.502 INFO:tasks.rados.rados.0.smithi136.stdout:3622: done (0 left) 2024-09-12T23:50:42.502 INFO:tasks.rados.rados.0.smithi136.stdout:3623: setattr oid 31 current snap is 346 2024-09-12T23:50:42.506 INFO:tasks.rados.rados.0.smithi136.stdout:3624: snap_remove snap 334 2024-09-12T23:50:42.595 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:50:42 smithi181 bash[95505]: ts=2024-09-12T23:50:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:50:43.513 INFO:tasks.rados.rados.0.smithi136.stdout:3624: done (1 left) 2024-09-12T23:50:43.513 INFO:tasks.rados.rados.0.smithi136.stdout:3625: copy_from oid 4 from oid 18 current snap is 346 2024-09-12T23:50:43.513 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 31 v 2488 (ObjNum 1133 snap 345 seq_num 1133) dirty exists 2024-09-12T23:50:43.513 INFO:tasks.rados.rados.0.smithi136.stdout:3623: done (1 left) 2024-09-12T23:50:43.513 INFO:tasks.rados.rados.0.smithi136.stdout:3626: snap_remove snap 338 2024-09-12T23:50:43.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:43 smithi136 bash[75164]: cluster 2024-09-12T23:50:42.503416+0000 mon.a (mon.0) 1346 : cluster [DBG] osdmap e774: 8 total, 8 up, 8 in 2024-09-12T23:50:43.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:43 smithi136 bash[75164]: cluster 2024-09-12T23:50:42.858042+0000 mgr.y (mgr.24661) 405 : cluster [DBG] pgmap v743: 121 pgs: 121 active+clean; 306 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s rd, 3.7 MiB/s wr, 19 op/s 2024-09-12T23:50:43.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:43 smithi136 bash[76371]: cluster 2024-09-12T23:50:42.503416+0000 mon.a (mon.0) 1346 : cluster [DBG] osdmap e774: 8 total, 8 up, 8 in 2024-09-12T23:50:43.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:43 smithi136 bash[76371]: cluster 2024-09-12T23:50:42.858042+0000 mgr.y (mgr.24661) 405 : cluster [DBG] pgmap v743: 121 pgs: 121 active+clean; 306 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s rd, 3.7 MiB/s wr, 19 op/s 2024-09-12T23:50:43.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:43 smithi181 bash[67587]: cluster 2024-09-12T23:50:42.503416+0000 mon.a (mon.0) 1346 : cluster [DBG] osdmap e774: 8 total, 8 up, 8 in 2024-09-12T23:50:43.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:43 smithi181 bash[67587]: cluster 2024-09-12T23:50:42.858042+0000 mgr.y (mgr.24661) 405 : cluster [DBG] pgmap v743: 121 pgs: 121 active+clean; 306 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s rd, 3.7 MiB/s wr, 19 op/s 2024-09-12T23:50:44.513 INFO:tasks.rados.rados.0.smithi136.stdout:3625: finishing copy_from to smithi13667081-4 2024-09-12T23:50:44.514 INFO:tasks.rados.rados.0.smithi136.stdout:3625: got expected ENOENT (src dne) 2024-09-12T23:50:44.514 INFO:tasks.rados.rados.0.smithi136.stdout:3625: finishing copy_from racing read to smithi13667081-4 2024-09-12T23:50:44.514 INFO:tasks.rados.rados.0.smithi136.stdout:3625: done (1 left) 2024-09-12T23:50:44.514 INFO:tasks.rados.rados.0.smithi136.stdout:3626: done (0 left) 2024-09-12T23:50:44.514 INFO:tasks.rados.rados.0.smithi136.stdout:3627: snap_remove snap 336 2024-09-12T23:50:44.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:44 smithi181 bash[67587]: cluster 2024-09-12T23:50:43.513621+0000 mon.a (mon.0) 1347 : cluster [DBG] osdmap e775: 8 total, 8 up, 8 in 2024-09-12T23:50:45.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:44 smithi136 bash[75164]: cluster 2024-09-12T23:50:43.513621+0000 mon.a (mon.0) 1347 : cluster [DBG] osdmap e775: 8 total, 8 up, 8 in 2024-09-12T23:50:45.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:44 smithi136 bash[76371]: cluster 2024-09-12T23:50:43.513621+0000 mon.a (mon.0) 1347 : cluster [DBG] osdmap e775: 8 total, 8 up, 8 in 2024-09-12T23:50:45.515 INFO:tasks.rados.rados.0.smithi136.stdout:3627: done (0 left) 2024-09-12T23:50:45.515 INFO:tasks.rados.rados.0.smithi136.stdout:3628: read oid 27 snap 296 2024-09-12T23:50:45.515 INFO:tasks.rados.rados.0.smithi136.stdout:3628: expect (ObjNum 924 snap 290 seq_num 924) 2024-09-12T23:50:45.516 INFO:tasks.rados.rados.0.smithi136.stdout:3629: copy_from oid 41 from oid 11 current snap is 346 2024-09-12T23:50:45.516 INFO:tasks.rados.rados.0.smithi136.stdout:3630: snap_create 2024-09-12T23:50:45.532 INFO:tasks.rados.rados.0.smithi136.stdout:3628: expect (ObjNum 924 snap 290 seq_num 924) 2024-09-12T23:50:45.579 INFO:tasks.rados.rados.0.smithi136.stdout:3629: finishing copy_from racing read to smithi13667081-41 2024-09-12T23:50:45.579 INFO:tasks.rados.rados.0.smithi136.stdout:3629: finishing copy_from to smithi13667081-41 2024-09-12T23:50:45.579 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 41 v 2067 (ObjNum 1056 snap 321 seq_num 1056) dirty exists 2024-09-12T23:50:45.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:45 smithi181 bash[67587]: cluster 2024-09-12T23:50:44.515757+0000 mon.a (mon.0) 1348 : cluster [DBG] osdmap e776: 8 total, 8 up, 8 in 2024-09-12T23:50:45.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:45 smithi181 bash[67587]: cluster 2024-09-12T23:50:44.858836+0000 mgr.y (mgr.24661) 406 : cluster [DBG] pgmap v746: 121 pgs: 121 active+clean; 311 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 4.2 MiB/s wr, 20 op/s 2024-09-12T23:50:45.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:45 smithi181 bash[67587]: cluster 2024-09-12T23:50:45.518010+0000 mon.a (mon.0) 1349 : cluster [DBG] osdmap e777: 8 total, 8 up, 8 in 2024-09-12T23:50:46.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:45 smithi136 bash[75164]: cluster 2024-09-12T23:50:44.515757+0000 mon.a (mon.0) 1348 : cluster [DBG] osdmap e776: 8 total, 8 up, 8 in 2024-09-12T23:50:46.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:45 smithi136 bash[75164]: cluster 2024-09-12T23:50:44.858836+0000 mgr.y (mgr.24661) 406 : cluster [DBG] pgmap v746: 121 pgs: 121 active+clean; 311 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 4.2 MiB/s wr, 20 op/s 2024-09-12T23:50:46.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:45 smithi136 bash[75164]: cluster 2024-09-12T23:50:45.518010+0000 mon.a (mon.0) 1349 : cluster [DBG] osdmap e777: 8 total, 8 up, 8 in 2024-09-12T23:50:46.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:45 smithi136 bash[76371]: cluster 2024-09-12T23:50:44.515757+0000 mon.a (mon.0) 1348 : cluster [DBG] osdmap e776: 8 total, 8 up, 8 in 2024-09-12T23:50:46.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:45 smithi136 bash[76371]: cluster 2024-09-12T23:50:44.858836+0000 mgr.y (mgr.24661) 406 : cluster [DBG] pgmap v746: 121 pgs: 121 active+clean; 311 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 4.2 MiB/s wr, 20 op/s 2024-09-12T23:50:46.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:45 smithi136 bash[76371]: cluster 2024-09-12T23:50:45.518010+0000 mon.a (mon.0) 1349 : cluster [DBG] osdmap e777: 8 total, 8 up, 8 in 2024-09-12T23:50:46.505 INFO:tasks.rados.rados.0.smithi136.stdout:3628: done (2 left) 2024-09-12T23:50:46.505 INFO:tasks.rados.rados.0.smithi136.stdout:3629: done (1 left) 2024-09-12T23:50:46.505 INFO:tasks.rados.rados.0.smithi136.stdout:3630: done (0 left) 2024-09-12T23:50:46.505 INFO:tasks.rados.rados.0.smithi136.stdout:3631: snap_create 2024-09-12T23:50:47.418 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid da58a47c-715d-11ef-bceb-c7b262605968 -e sha1=7a853b1706d076d58e0da7a5d3b9b9eb36276880 -- bash -c 'ceph orch ps' 2024-09-12T23:50:47.509 INFO:tasks.rados.rados.0.smithi136.stdout:3631: done (0 left) 2024-09-12T23:50:47.509 INFO:tasks.rados.rados.0.smithi136.stdout:3632: rollback oid 9 current snap is 348 2024-09-12T23:50:47.509 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 9 to 335 2024-09-12T23:50:47.509 INFO:tasks.rados.rados.0.smithi136.stdout:3633: copy_from oid 11 from oid 31 current snap is 348 2024-09-12T23:50:47.509 INFO:tasks.rados.rados.0.smithi136.stdout:3634: read oid 13 snap -1 2024-09-12T23:50:47.509 INFO:tasks.rados.rados.0.smithi136.stdout:3634: expect (ObjNum 1127 snap 343 seq_num 1127) 2024-09-12T23:50:47.510 INFO:tasks.rados.rados.0.smithi136.stdout:3635: snap_create 2024-09-12T23:50:47.522 INFO:tasks.rados.rados.0.smithi136.stdout:3632: finishing rollback tid 0 to smithi13667081-9 2024-09-12T23:50:47.527 INFO:tasks.rados.rados.0.smithi136.stdout:3632: finishing rollback tid 1 to smithi13667081-9 2024-09-12T23:50:47.527 INFO:tasks.rados.rados.0.smithi136.stdout:3632: finishing rollback tid 2 to smithi13667081-9 2024-09-12T23:50:47.527 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 9 v 2486 (ObjNum 1091 snap 329 seq_num 1091) dirty exists 2024-09-12T23:50:47.527 INFO:tasks.rados.rados.0.smithi136.stdout:3634: expect (ObjNum 1127 snap 343 seq_num 1127) 2024-09-12T23:50:47.582 INFO:tasks.rados.rados.0.smithi136.stdout:3633: finishing copy_from racing read to smithi13667081-11 2024-09-12T23:50:47.582 INFO:tasks.rados.rados.0.smithi136.stdout:3633: finishing copy_from to smithi13667081-11 2024-09-12T23:50:47.582 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 11 v 2488 (ObjNum 1133 snap 345 seq_num 1133) dirty exists 2024-09-12T23:50:47.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:47 smithi136 bash[76371]: cluster 2024-09-12T23:50:46.504420+0000 mon.a (mon.0) 1350 : cluster [DBG] osdmap e778: 8 total, 8 up, 8 in 2024-09-12T23:50:47.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:47 smithi136 bash[76371]: audit 2024-09-12T23:50:46.627250+0000 mgr.y (mgr.24661) 407 : audit [DBG] from='client.44414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:50:47.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:47 smithi136 bash[76371]: cluster 2024-09-12T23:50:46.859847+0000 mgr.y (mgr.24661) 408 : cluster [DBG] pgmap v749: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 309 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.3 MiB/s rd, 2.2 MiB/s wr, 14 op/s 2024-09-12T23:50:47.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:47 smithi136 bash[75164]: cluster 2024-09-12T23:50:46.504420+0000 mon.a (mon.0) 1350 : cluster [DBG] osdmap e778: 8 total, 8 up, 8 in 2024-09-12T23:50:47.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:47 smithi136 bash[75164]: audit 2024-09-12T23:50:46.627250+0000 mgr.y (mgr.24661) 407 : audit [DBG] from='client.44414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:50:47.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:47 smithi136 bash[75164]: cluster 2024-09-12T23:50:46.859847+0000 mgr.y (mgr.24661) 408 : cluster [DBG] pgmap v749: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 309 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.3 MiB/s rd, 2.2 MiB/s wr, 14 op/s 2024-09-12T23:50:47.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:47 smithi181 bash[67587]: cluster 2024-09-12T23:50:46.504420+0000 mon.a (mon.0) 1350 : cluster [DBG] osdmap e778: 8 total, 8 up, 8 in 2024-09-12T23:50:47.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:47 smithi181 bash[67587]: audit 2024-09-12T23:50:46.627250+0000 mgr.y (mgr.24661) 407 : audit [DBG] from='client.44414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:50:47.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:47 smithi181 bash[67587]: cluster 2024-09-12T23:50:46.859847+0000 mgr.y (mgr.24661) 408 : cluster [DBG] pgmap v749: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 309 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.3 MiB/s rd, 2.2 MiB/s wr, 14 op/s 2024-09-12T23:50:48.514 INFO:tasks.rados.rados.0.smithi136.stdout:3632: done (3 left) 2024-09-12T23:50:48.514 INFO:tasks.rados.rados.0.smithi136.stdout:3633: done (2 left) 2024-09-12T23:50:48.514 INFO:tasks.rados.rados.0.smithi136.stdout:3634: done (1 left) 2024-09-12T23:50:48.514 INFO:tasks.rados.rados.0.smithi136.stdout:3635: done (0 left) 2024-09-12T23:50:48.514 INFO:tasks.rados.rados.0.smithi136.stdout:3636: rollback oid 20 current snap is 349 2024-09-12T23:50:48.514 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 20 to 345 2024-09-12T23:50:48.514 INFO:tasks.rados.rados.0.smithi136.stdout:3637: rmattr oid 23 current snap is 349 2024-09-12T23:50:48.514 INFO:tasks.rados.rados.0.smithi136.stdout:3638: rollback oid 6 current snap is 349 2024-09-12T23:50:48.514 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 6 to 346 2024-09-12T23:50:48.514 INFO:tasks.rados.rados.0.smithi136.stdout:3639: delete oid 36 current snap is 349 2024-09-12T23:50:48.526 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 23 v 2806 (ObjNum 1088 snap 329 seq_num 1088) dirty exists 2024-09-12T23:50:48.527 INFO:tasks.rados.rados.0.smithi136.stdout:3637: done (3 left) 2024-09-12T23:50:48.527 INFO:tasks.rados.rados.0.smithi136.stdout:3639: done (2 left) 2024-09-12T23:50:48.527 INFO:tasks.rados.rados.0.smithi136.stdout:3640: delete oid 11 current snap is 349 2024-09-12T23:50:48.528 INFO:tasks.rados.rados.0.smithi136.stdout:3636: finishing rollback tid 0 to smithi13667081-20 2024-09-12T23:50:48.529 INFO:tasks.rados.rados.0.smithi136.stdout:3636: finishing rollback tid 1 to smithi13667081-20 2024-09-12T23:50:48.529 INFO:tasks.rados.rados.0.smithi136.stdout:3636: finishing rollback tid 2 to smithi13667081-20 2024-09-12T23:50:48.529 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 20 v 1656 (ObjNum 1111 snap 337 seq_num 1111) dirty exists 2024-09-12T23:50:48.530 INFO:tasks.rados.rados.0.smithi136.stdout:3638: finishing rollback tid 1 to smithi13667081-6 2024-09-12T23:50:48.530 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 6 v 0 (ObjNum 145 snap 0 seq_num 4244446440) dirty dne 2024-09-12T23:50:48.534 INFO:tasks.rados.rados.0.smithi136.stdout:3636: done (2 left) 2024-09-12T23:50:48.534 INFO:tasks.rados.rados.0.smithi136.stdout:3638: done (1 left) 2024-09-12T23:50:48.534 INFO:tasks.rados.rados.0.smithi136.stdout:3640: done (0 left) 2024-09-12T23:50:48.534 INFO:tasks.rados.rados.0.smithi136.stdout:3641: snap_remove snap 347 2024-09-12T23:50:48.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:48 smithi181 bash[67587]: cluster 2024-09-12T23:50:47.509383+0000 mon.a (mon.0) 1351 : cluster [DBG] osdmap e779: 8 total, 8 up, 8 in 2024-09-12T23:50:49.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:48 smithi136 bash[75164]: cluster 2024-09-12T23:50:47.509383+0000 mon.a (mon.0) 1351 : cluster [DBG] osdmap e779: 8 total, 8 up, 8 in 2024-09-12T23:50:49.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:48 smithi136 bash[76371]: cluster 2024-09-12T23:50:47.509383+0000 mon.a (mon.0) 1351 : cluster [DBG] osdmap e779: 8 total, 8 up, 8 in 2024-09-12T23:50:49.536 INFO:tasks.rados.rados.0.smithi136.stdout:3641: done (0 left) 2024-09-12T23:50:49.537 INFO:tasks.rados.rados.0.smithi136.stdout:3642: delete oid 17 current snap is 349 2024-09-12T23:50:49.554 INFO:tasks.rados.rados.0.smithi136.stdout:3642: done (0 left) 2024-09-12T23:50:49.554 INFO:tasks.rados.rados.0.smithi136.stdout:3643: snap_create 2024-09-12T23:50:49.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:50:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:50:49] "GET /metrics HTTP/1.1" 200 36437 "" "Prometheus/2.43.0" 2024-09-12T23:50:49.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:49 smithi136 bash[75164]: cluster 2024-09-12T23:50:48.513998+0000 mon.a (mon.0) 1352 : cluster [DBG] osdmap e780: 8 total, 8 up, 8 in 2024-09-12T23:50:49.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:49 smithi136 bash[75164]: cluster 2024-09-12T23:50:48.860614+0000 mgr.y (mgr.24661) 409 : cluster [DBG] pgmap v752: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 310 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 792 KiB/s rd, 1.4 MiB/s wr, 7 op/s 2024-09-12T23:50:49.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:49 smithi136 bash[76371]: cluster 2024-09-12T23:50:48.513998+0000 mon.a (mon.0) 1352 : cluster [DBG] osdmap e780: 8 total, 8 up, 8 in 2024-09-12T23:50:49.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:49 smithi136 bash[76371]: cluster 2024-09-12T23:50:48.860614+0000 mgr.y (mgr.24661) 409 : cluster [DBG] pgmap v752: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 310 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 792 KiB/s rd, 1.4 MiB/s wr, 7 op/s 2024-09-12T23:50:49.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:49 smithi181 bash[67587]: cluster 2024-09-12T23:50:48.513998+0000 mon.a (mon.0) 1352 : cluster [DBG] osdmap e780: 8 total, 8 up, 8 in 2024-09-12T23:50:49.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:49 smithi181 bash[67587]: cluster 2024-09-12T23:50:48.860614+0000 mgr.y (mgr.24661) 409 : cluster [DBG] pgmap v752: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 310 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 792 KiB/s rd, 1.4 MiB/s wr, 7 op/s 2024-09-12T23:50:50.548 INFO:tasks.rados.rados.0.smithi136.stdout:3643: done (0 left) 2024-09-12T23:50:50.548 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 8 current snap is 350 2024-09-12T23:50:50.548 INFO:tasks.rados.rados.0.smithi136.stdout:3644: seq_num 1143 ranges {1835008=753664,2588672=450560,3039232=507904,3547136=589824,4136960=57344} 2024-09-12T23:50:50.560 INFO:tasks.rados.rados.0.smithi136.stdout:3644: writing smithi13667081-8 from 1835008 to 2588672 tid 1 2024-09-12T23:50:50.563 INFO:tasks.rados.rados.0.smithi136.stdout:3644: writing smithi13667081-8 from 2588672 to 3039232 tid 2 2024-09-12T23:50:50.566 INFO:tasks.rados.rados.0.smithi136.stdout:3644: writing smithi13667081-8 from 3039232 to 3547136 tid 3 2024-09-12T23:50:50.569 INFO:tasks.rados.rados.0.smithi136.stdout:3644: writing smithi13667081-8 from 3547136 to 4136960 tid 4 2024-09-12T23:50:50.570 INFO:tasks.rados.rados.0.smithi136.stdout:3644: writing smithi13667081-8 from 4136960 to 4194304 tid 5 2024-09-12T23:50:50.570 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 29 current snap is 350 2024-09-12T23:50:50.570 INFO:tasks.rados.rados.0.smithi136.stdout:3645: seq_num 1144 ranges {2146304=770048,2916352=737280,3653632=712704,4366336=155648} 2024-09-12T23:50:50.586 INFO:tasks.rados.rados.0.smithi136.stdout:3645: writing smithi13667081-29 from 2146304 to 2916352 tid 1 2024-09-12T23:50:50.590 INFO:tasks.rados.rados.0.smithi136.stdout:3645: writing smithi13667081-29 from 2916352 to 3653632 tid 2 2024-09-12T23:50:50.594 INFO:tasks.rados.rados.0.smithi136.stdout:3645: writing smithi13667081-29 from 3653632 to 4366336 tid 3 2024-09-12T23:50:50.595 INFO:tasks.rados.rados.0.smithi136.stdout:3645: writing smithi13667081-29 from 4366336 to 4521984 tid 4 2024-09-12T23:50:50.595 INFO:tasks.rados.rados.0.smithi136.stdout:3646: copy_from oid 12 from oid 17 current snap is 350 2024-09-12T23:50:50.595 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 10 current snap is 350 2024-09-12T23:50:50.595 INFO:tasks.rados.rados.0.smithi136.stdout:3644: finishing write tid 1 to smithi13667081-8 2024-09-12T23:50:50.595 INFO:tasks.rados.rados.0.smithi136.stdout:3647: seq_num 1145 ranges {0=786432,786432=622592,1409024=516096,1925120=417792,2342912=49152} 2024-09-12T23:50:50.599 INFO:tasks.rados.rados.0.smithi136.stdout:3647: writing smithi13667081-10 from 0 to 786432 tid 1 2024-09-12T23:50:50.603 INFO:tasks.rados.rados.0.smithi136.stdout:3647: writing smithi13667081-10 from 786432 to 1409024 tid 2 2024-09-12T23:50:50.607 INFO:tasks.rados.rados.0.smithi136.stdout:3647: writing smithi13667081-10 from 1409024 to 1925120 tid 3 2024-09-12T23:50:50.609 INFO:tasks.rados.rados.0.smithi136.stdout:3647: writing smithi13667081-10 from 1925120 to 2342912 tid 4 2024-09-12T23:50:50.610 INFO:tasks.rados.rados.0.smithi136.stdout:3647: writing smithi13667081-10 from 2342912 to 2392064 tid 5 2024-09-12T23:50:50.610 INFO:tasks.rados.rados.0.smithi136.stdout:3648: snap_remove snap 345 2024-09-12T23:50:50.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:50 smithi181 bash[67587]: cluster 2024-09-12T23:50:49.538722+0000 mon.a (mon.0) 1353 : cluster [DBG] osdmap e781: 8 total, 8 up, 8 in 2024-09-12T23:50:51.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:50 smithi136 bash[75164]: cluster 2024-09-12T23:50:49.538722+0000 mon.a (mon.0) 1353 : cluster [DBG] osdmap e781: 8 total, 8 up, 8 in 2024-09-12T23:50:51.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:50 smithi136 bash[76371]: cluster 2024-09-12T23:50:49.538722+0000 mon.a (mon.0) 1353 : cluster [DBG] osdmap e781: 8 total, 8 up, 8 in 2024-09-12T23:50:51.507 INFO:tasks.rados.rados.0.smithi136.stdout:3644: finishing write tid 2 to smithi13667081-8 2024-09-12T23:50:51.508 INFO:tasks.rados.rados.0.smithi136.stdout:3644: finishing write tid 3 to smithi13667081-8 2024-09-12T23:50:51.508 INFO:tasks.rados.rados.0.smithi136.stdout:3644: finishing write tid 4 to smithi13667081-8 2024-09-12T23:50:51.508 INFO:tasks.rados.rados.0.smithi136.stdout:3644: finishing write tid 5 to smithi13667081-8 2024-09-12T23:50:51.508 INFO:tasks.rados.rados.0.smithi136.stdout:3644: finishing write tid 6 to smithi13667081-8 2024-09-12T23:50:51.508 INFO:tasks.rados.rados.0.smithi136.stdout:3645: finishing write tid 1 to smithi13667081-29 2024-09-12T23:50:51.508 INFO:tasks.rados.rados.0.smithi136.stdout:3644: finishing write tid 7 to smithi13667081-8 2024-09-12T23:50:51.508 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 8 v 2365 (ObjNum 1143 snap 350 seq_num 1143) dirty exists 2024-09-12T23:50:51.508 INFO:tasks.rados.rados.0.smithi136.stdout:3644: left oid 8 (ObjNum 1143 snap 350 seq_num 1143) 2024-09-12T23:50:51.508 INFO:tasks.rados.rados.0.smithi136.stdout:3646: finishing copy_from to smithi13667081-12 2024-09-12T23:50:51.508 INFO:tasks.rados.rados.0.smithi136.stdout:3646: got expected ENOENT (src dne) 2024-09-12T23:50:51.508 INFO:tasks.rados.rados.0.smithi136.stdout:3646: finishing copy_from racing read to smithi13667081-12 2024-09-12T23:50:51.508 INFO:tasks.rados.rados.0.smithi136.stdout:3647: finishing write tid 1 to smithi13667081-10 2024-09-12T23:50:51.509 INFO:tasks.rados.rados.0.smithi136.stdout:3645: finishing write tid 2 to smithi13667081-29 2024-09-12T23:50:51.509 INFO:tasks.rados.rados.0.smithi136.stdout:3645: finishing write tid 3 to smithi13667081-29 2024-09-12T23:50:51.509 INFO:tasks.rados.rados.0.smithi136.stdout:3645: finishing write tid 4 to smithi13667081-29 2024-09-12T23:50:51.509 INFO:tasks.rados.rados.0.smithi136.stdout:3645: finishing write tid 5 to smithi13667081-29 2024-09-12T23:50:51.509 INFO:tasks.rados.rados.0.smithi136.stdout:3647: finishing write tid 2 to smithi13667081-10 2024-09-12T23:50:51.509 INFO:tasks.rados.rados.0.smithi136.stdout:3645: finishing write tid 6 to smithi13667081-29 2024-09-12T23:50:51.509 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 29 v 2688 (ObjNum 1144 snap 350 seq_num 1144) dirty exists 2024-09-12T23:50:51.509 INFO:tasks.rados.rados.0.smithi136.stdout:3645: left oid 29 (ObjNum 1144 snap 350 seq_num 1144) 2024-09-12T23:50:51.509 INFO:tasks.rados.rados.0.smithi136.stdout:3647: finishing write tid 3 to smithi13667081-10 2024-09-12T23:50:51.509 INFO:tasks.rados.rados.0.smithi136.stdout:3647: finishing write tid 4 to smithi13667081-10 2024-09-12T23:50:51.509 INFO:tasks.rados.rados.0.smithi136.stdout:3647: finishing write tid 5 to smithi13667081-10 2024-09-12T23:50:51.509 INFO:tasks.rados.rados.0.smithi136.stdout:3647: finishing write tid 6 to smithi13667081-10 2024-09-12T23:50:51.509 INFO:tasks.rados.rados.0.smithi136.stdout:3647: finishing write tid 7 to smithi13667081-10 2024-09-12T23:50:51.509 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 10 v 2589 (ObjNum 1145 snap 350 seq_num 1145) dirty exists 2024-09-12T23:50:51.509 INFO:tasks.rados.rados.0.smithi136.stdout:3647: left oid 10 (ObjNum 1145 snap 350 seq_num 1145) 2024-09-12T23:50:51.510 INFO:tasks.rados.rados.0.smithi136.stdout:3644: done (4 left) 2024-09-12T23:50:51.510 INFO:tasks.rados.rados.0.smithi136.stdout:3645: done (3 left) 2024-09-12T23:50:51.510 INFO:tasks.rados.rados.0.smithi136.stdout:3646: done (2 left) 2024-09-12T23:50:51.510 INFO:tasks.rados.rados.0.smithi136.stdout:3647: done (1 left) 2024-09-12T23:50:51.510 INFO:tasks.rados.rados.0.smithi136.stdout:3648: done (0 left) 2024-09-12T23:50:51.510 INFO:tasks.rados.rados.0.smithi136.stdout:3649: rollback oid 19 current snap is 350 2024-09-12T23:50:51.510 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 19 to 349 2024-09-12T23:50:51.510 INFO:tasks.rados.rados.0.smithi136.stdout:3650: snap_remove snap 335 2024-09-12T23:50:51.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:51 smithi181 bash[67587]: cluster 2024-09-12T23:50:50.546998+0000 mon.a (mon.0) 1354 : cluster [DBG] osdmap e782: 8 total, 8 up, 8 in 2024-09-12T23:50:51.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:51 smithi181 bash[67587]: cluster 2024-09-12T23:50:50.861452+0000 mgr.y (mgr.24661) 410 : cluster [DBG] pgmap v755: 121 pgs: 121 active+clean; 310 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 810 KiB/s wr, 3 op/s 2024-09-12T23:50:51.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:51 smithi181 bash[67587]: cluster 2024-09-12T23:50:51.504701+0000 mon.a (mon.0) 1355 : cluster [DBG] osdmap e783: 8 total, 8 up, 8 in 2024-09-12T23:50:52.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:51 smithi136 bash[75164]: cluster 2024-09-12T23:50:50.546998+0000 mon.a (mon.0) 1354 : cluster [DBG] osdmap e782: 8 total, 8 up, 8 in 2024-09-12T23:50:52.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:51 smithi136 bash[75164]: cluster 2024-09-12T23:50:50.861452+0000 mgr.y (mgr.24661) 410 : cluster [DBG] pgmap v755: 121 pgs: 121 active+clean; 310 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 810 KiB/s wr, 3 op/s 2024-09-12T23:50:52.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:51 smithi136 bash[75164]: cluster 2024-09-12T23:50:51.504701+0000 mon.a (mon.0) 1355 : cluster [DBG] osdmap e783: 8 total, 8 up, 8 in 2024-09-12T23:50:52.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:51 smithi136 bash[76371]: cluster 2024-09-12T23:50:50.546998+0000 mon.a (mon.0) 1354 : cluster [DBG] osdmap e782: 8 total, 8 up, 8 in 2024-09-12T23:50:52.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:51 smithi136 bash[76371]: cluster 2024-09-12T23:50:50.861452+0000 mgr.y (mgr.24661) 410 : cluster [DBG] pgmap v755: 121 pgs: 121 active+clean; 310 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 810 KiB/s wr, 3 op/s 2024-09-12T23:50:52.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:51 smithi136 bash[76371]: cluster 2024-09-12T23:50:51.504701+0000 mon.a (mon.0) 1355 : cluster [DBG] osdmap e783: 8 total, 8 up, 8 in 2024-09-12T23:50:52.220 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:50:52.508 INFO:tasks.rados.rados.0.smithi136.stdout:3649: finishing rollback tid 0 to smithi13667081-19 2024-09-12T23:50:52.508 INFO:tasks.rados.rados.0.smithi136.stdout:3649: finishing rollback tid 1 to smithi13667081-19 2024-09-12T23:50:52.508 INFO:tasks.rados.rados.0.smithi136.stdout:3649: finishing rollback tid 2 to smithi13667081-19 2024-09-12T23:50:52.508 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 19 v 2489 (ObjNum 1130 snap 344 seq_num 1130) dirty exists 2024-09-12T23:50:52.508 INFO:tasks.rados.rados.0.smithi136.stdout:3649: done (1 left) 2024-09-12T23:50:52.508 INFO:tasks.rados.rados.0.smithi136.stdout:3650: done (0 left) 2024-09-12T23:50:52.508 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 34 current snap is 350 2024-09-12T23:50:52.508 INFO:tasks.rados.rados.0.smithi136.stdout:3651: seq_num 1146 ranges {0=802816,802816=499712,1302528=679936,1982464=425984} 2024-09-12T23:50:52.513 INFO:tasks.rados.rados.0.smithi136.stdout:3651: writing smithi13667081-34 from 0 to 802816 tid 1 2024-09-12T23:50:52.517 INFO:tasks.rados.rados.0.smithi136.stdout:3651: writing smithi13667081-34 from 802816 to 1302528 tid 2 2024-09-12T23:50:52.521 INFO:tasks.rados.rados.0.smithi136.stdout:3651: writing smithi13667081-34 from 1302528 to 1982464 tid 3 2024-09-12T23:50:52.525 INFO:tasks.rados.rados.0.smithi136.stdout:3651: writing smithi13667081-34 from 1982464 to 2408448 tid 4 2024-09-12T23:50:52.525 INFO:tasks.rados.rados.0.smithi136.stdout:3652: snap_create 2024-09-12T23:50:52.542 INFO:tasks.rados.rados.0.smithi136.stdout:3651: finishing write tid 1 to smithi13667081-34 2024-09-12T23:50:52.544 INFO:tasks.rados.rados.0.smithi136.stdout:3651: finishing write tid 2 to smithi13667081-34 2024-09-12T23:50:52.544 INFO:tasks.rados.rados.0.smithi136.stdout:3651: finishing write tid 3 to smithi13667081-34 2024-09-12T23:50:52.544 INFO:tasks.rados.rados.0.smithi136.stdout:3651: finishing write tid 4 to smithi13667081-34 2024-09-12T23:50:52.550 INFO:tasks.rados.rados.0.smithi136.stdout:3651: finishing write tid 5 to smithi13667081-34 2024-09-12T23:50:52.556 INFO:tasks.rados.rados.0.smithi136.stdout:3651: finishing write tid 6 to smithi13667081-34 2024-09-12T23:50:52.556 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 34 v 2509 (ObjNum 1146 snap 350 seq_num 1146) dirty exists 2024-09-12T23:50:52.556 INFO:tasks.rados.rados.0.smithi136.stdout:3651: left oid 34 (ObjNum 1146 snap 350 seq_num 1146) 2024-09-12T23:50:52.596 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:50:52 smithi181 bash[95505]: ts=2024-09-12T23:50:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:50:53.519 INFO:tasks.rados.rados.0.smithi136.stdout:3651: done (1 left) 2024-09-12T23:50:53.519 INFO:tasks.rados.rados.0.smithi136.stdout:3652: done (0 left) 2024-09-12T23:50:53.519 INFO:tasks.rados.rados.0.smithi136.stdout:3653: read oid 31 snap -1 2024-09-12T23:50:53.519 INFO:tasks.rados.rados.0.smithi136.stdout:3653: expect (ObjNum 1133 snap 345 seq_num 1133) 2024-09-12T23:50:53.519 INFO:tasks.rados.rados.0.smithi136.stdout:3654: delete oid 23 current snap is 351 2024-09-12T23:50:53.525 INFO:tasks.rados.rados.0.smithi136.stdout:3654: done (1 left) 2024-09-12T23:50:53.525 INFO:tasks.rados.rados.0.smithi136.stdout:3655: read oid 20 snap 296 2024-09-12T23:50:53.525 INFO:tasks.rados.rados.0.smithi136.stdout:3655: expect (ObjNum 937 snap 294 seq_num 937) 2024-09-12T23:50:53.525 INFO:tasks.rados.rados.0.smithi136.stdout:3656: delete oid 3 current snap is 351 2024-09-12T23:50:53.534 INFO:tasks.rados.rados.0.smithi136.stdout:3655: expect (ObjNum 937 snap 294 seq_num 937) 2024-09-12T23:50:53.581 INFO:tasks.rados.rados.0.smithi136.stdout:3653: expect (ObjNum 1133 snap 345 seq_num 1133) 2024-09-12T23:50:53.691 INFO:tasks.rados.rados.0.smithi136.stdout:3653: done (2 left) 2024-09-12T23:50:53.691 INFO:tasks.rados.rados.0.smithi136.stdout:3655: done (1 left) 2024-09-12T23:50:53.691 INFO:tasks.rados.rados.0.smithi136.stdout:3656: done (0 left) 2024-09-12T23:50:53.691 INFO:tasks.rados.rados.0.smithi136.stdout:3657: rollback oid 1 current snap is 351 2024-09-12T23:50:53.691 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 1 to 296 2024-09-12T23:50:53.691 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 21 current snap is 351 2024-09-12T23:50:53.691 INFO:tasks.rados.rados.0.smithi136.stdout:3658: seq_num 1149 ranges {2400256=49152} 2024-09-12T23:50:53.705 INFO:tasks.rados.rados.0.smithi136.stdout:3658: writing smithi13667081-21 from 2400256 to 2449408 tid 1 2024-09-12T23:50:53.705 INFO:tasks.rados.rados.0.smithi136.stdout:3659: snap_create 2024-09-12T23:50:53.705 INFO:tasks.rados.rados.0.smithi136.stdout:3657: finishing rollback tid 0 to smithi13667081-1 2024-09-12T23:50:53.705 INFO:tasks.rados.rados.0.smithi136.stdout:3657: finishing rollback tid 1 to smithi13667081-1 2024-09-12T23:50:53.705 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 1 v 2503 (ObjNum 1809 snap 0 seq_num 1427292560) dirty dne 2024-09-12T23:50:53.708 INFO:tasks.rados.rados.0.smithi136.stdout:3658: finishing write tid 1 to smithi13667081-21 2024-09-12T23:50:53.708 INFO:tasks.rados.rados.0.smithi136.stdout:3658: finishing write tid 2 to smithi13667081-21 2024-09-12T23:50:53.712 INFO:tasks.rados.rados.0.smithi136.stdout:3658: finishing write tid 3 to smithi13667081-21 2024-09-12T23:50:53.712 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 21 v 2505 (ObjNum 1149 snap 351 seq_num 1149) dirty exists 2024-09-12T23:50:53.712 INFO:tasks.rados.rados.0.smithi136.stdout:3658: left oid 21 (ObjNum 1149 snap 351 seq_num 1149) 2024-09-12T23:50:53.827 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:53 smithi136 bash[75164]: cluster 2024-09-12T23:50:52.508614+0000 mon.a (mon.0) 1356 : cluster [DBG] osdmap e784: 8 total, 8 up, 8 in 2024-09-12T23:50:53.827 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:53 smithi136 bash[75164]: cluster 2024-09-12T23:50:52.862344+0000 mgr.y (mgr.24661) 411 : cluster [DBG] pgmap v758: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 311 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 576 KiB/s wr, 4 op/s 2024-09-12T23:50:53.827 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:53 smithi136 bash[76371]: cluster 2024-09-12T23:50:52.508614+0000 mon.a (mon.0) 1356 : cluster [DBG] osdmap e784: 8 total, 8 up, 8 in 2024-09-12T23:50:53.827 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:53 smithi136 bash[76371]: cluster 2024-09-12T23:50:52.862344+0000 mgr.y (mgr.24661) 411 : cluster [DBG] pgmap v758: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 311 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 576 KiB/s wr, 4 op/s 2024-09-12T23:50:53.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:53 smithi181 bash[67587]: cluster 2024-09-12T23:50:52.508614+0000 mon.a (mon.0) 1356 : cluster [DBG] osdmap e784: 8 total, 8 up, 8 in 2024-09-12T23:50:53.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:53 smithi181 bash[67587]: cluster 2024-09-12T23:50:52.862344+0000 mgr.y (mgr.24661) 411 : cluster [DBG] pgmap v758: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 311 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 576 KiB/s wr, 4 op/s 2024-09-12T23:50:54.521 INFO:tasks.rados.rados.0.smithi136.stdout:3657: done (2 left) 2024-09-12T23:50:54.522 INFO:tasks.rados.rados.0.smithi136.stdout:3658: done (1 left) 2024-09-12T23:50:54.522 INFO:tasks.rados.rados.0.smithi136.stdout:3659: done (0 left) 2024-09-12T23:50:54.522 INFO:tasks.rados.rados.0.smithi136.stdout:3660: rollback oid 16 current snap is 352 2024-09-12T23:50:54.522 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 16 to 296 2024-09-12T23:50:54.522 INFO:tasks.rados.rados.0.smithi136.stdout:3661: setattr oid 43 current snap is 352 2024-09-12T23:50:54.527 INFO:tasks.rados.rados.0.smithi136.stdout:3662: delete oid 40 current snap is 352 2024-09-12T23:50:54.536 INFO:tasks.rados.rados.0.smithi136.stdout:3662: done (2 left) 2024-09-12T23:50:54.537 INFO:tasks.rados.rados.0.smithi136.stdout:3663: copy_from oid 13 from oid 49 current snap is 352 2024-09-12T23:50:54.537 INFO:tasks.rados.rados.0.smithi136.stdout:3664: setattr oid 22 current snap is 352 2024-09-12T23:50:54.537 INFO:tasks.rados.rados.0.smithi136.stdout:3663: finishing copy_from to smithi13667081-13 2024-09-12T23:50:54.537 INFO:tasks.rados.rados.0.smithi136.stdout:3663: got expected ENOENT (src dne) 2024-09-12T23:50:54.537 INFO:tasks.rados.rados.0.smithi136.stdout:3663: finishing copy_from racing read to smithi13667081-13 2024-09-12T23:50:54.537 INFO:tasks.rados.rados.0.smithi136.stdout:3663: done (3 left) 2024-09-12T23:50:54.537 INFO:tasks.rados.rados.0.smithi136.stdout:3665: copy_from oid 50 from oid 31 current snap is 352 2024-09-12T23:50:54.538 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 39 current snap is 352 2024-09-12T23:50:54.538 INFO:tasks.rados.rados.0.smithi136.stdout:3666: seq_num 1151 ranges {0=81920} 2024-09-12T23:50:54.538 INFO:tasks.rados.rados.0.smithi136.stdout:3666: writing smithi13667081-39 from 0 to 81920 tid 1 2024-09-12T23:50:54.538 INFO:tasks.rados.rados.0.smithi136.stdout:3667: snap_create 2024-09-12T23:50:54.547 INFO:tasks.rados.rados.0.smithi136.stdout:3666: finishing write tid 1 to smithi13667081-39 2024-09-12T23:50:54.552 INFO:tasks.rados.rados.0.smithi136.stdout:3666: finishing write tid 2 to smithi13667081-39 2024-09-12T23:50:54.555 INFO:tasks.rados.rados.0.smithi136.stdout:3666: finishing write tid 3 to smithi13667081-39 2024-09-12T23:50:54.555 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 39 v 2460 (ObjNum 1151 snap 352 seq_num 1151) dirty exists 2024-09-12T23:50:54.555 INFO:tasks.rados.rados.0.smithi136.stdout:3666: left oid 39 (ObjNum 1151 snap 352 seq_num 1151) 2024-09-12T23:50:54.565 INFO:tasks.rados.rados.0.smithi136.stdout:3660: finishing rollback tid 0 to smithi13667081-16 2024-09-12T23:50:54.568 INFO:tasks.rados.rados.0.smithi136.stdout:3665: finishing copy_from racing read to smithi13667081-50 2024-09-12T23:50:54.570 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 22 v 2506 (ObjNum 1101 snap 333 seq_num 1101) dirty exists 2024-09-12T23:50:54.571 INFO:tasks.rados.rados.0.smithi136.stdout:3665: finishing copy_from to smithi13667081-50 2024-09-12T23:50:54.571 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 50 v 2488 (ObjNum 1133 snap 345 seq_num 1133) dirty exists 2024-09-12T23:50:54.572 INFO:tasks.rados.rados.0.smithi136.stdout:3660: finishing rollback tid 1 to smithi13667081-16 2024-09-12T23:50:54.572 INFO:tasks.rados.rados.0.smithi136.stdout:3660: finishing rollback tid 2 to smithi13667081-16 2024-09-12T23:50:54.572 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 16 v 2517 (ObjNum 885 snap 287 seq_num 885) dirty exists 2024-09-12T23:50:54.573 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 43 v 2518 (ObjNum 1104 snap 333 seq_num 1104) dirty exists 2024-09-12T23:50:54.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:54 smithi136 bash[75164]: cluster 2024-09-12T23:50:53.521340+0000 mon.a (mon.0) 1357 : cluster [DBG] osdmap e785: 8 total, 8 up, 8 in 2024-09-12T23:50:54.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:54 smithi136 bash[75164]: audit 2024-09-12T23:50:53.753647+0000 mon.a (mon.0) 1358 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:50:54.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:54 smithi136 bash[76371]: cluster 2024-09-12T23:50:53.521340+0000 mon.a (mon.0) 1357 : cluster [DBG] osdmap e785: 8 total, 8 up, 8 in 2024-09-12T23:50:54.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:54 smithi136 bash[76371]: audit 2024-09-12T23:50:53.753647+0000 mon.a (mon.0) 1358 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:50:54.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:54 smithi181 bash[67587]: cluster 2024-09-12T23:50:53.521340+0000 mon.a (mon.0) 1357 : cluster [DBG] osdmap e785: 8 total, 8 up, 8 in 2024-09-12T23:50:54.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:54 smithi181 bash[67587]: audit 2024-09-12T23:50:53.753647+0000 mon.a (mon.0) 1358 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:50:55.534 INFO:tasks.rados.rados.0.smithi136.stdout:3660: done (5 left) 2024-09-12T23:50:55.534 INFO:tasks.rados.rados.0.smithi136.stdout:3661: done (4 left) 2024-09-12T23:50:55.534 INFO:tasks.rados.rados.0.smithi136.stdout:3664: done (3 left) 2024-09-12T23:50:55.534 INFO:tasks.rados.rados.0.smithi136.stdout:3665: done (2 left) 2024-09-12T23:50:55.534 INFO:tasks.rados.rados.0.smithi136.stdout:3666: done (1 left) 2024-09-12T23:50:55.535 INFO:tasks.rados.rados.0.smithi136.stdout:3667: done (0 left) 2024-09-12T23:50:55.535 INFO:tasks.rados.rados.0.smithi136.stdout:3668: read oid 10 snap -1 2024-09-12T23:50:55.535 INFO:tasks.rados.rados.0.smithi136.stdout:3668: expect (ObjNum 1145 snap 350 seq_num 1145) 2024-09-12T23:50:55.535 INFO:tasks.rados.rados.0.smithi136.stdout:3669: snap_create 2024-09-12T23:50:55.562 INFO:tasks.rados.rados.0.smithi136.stdout:3668: expect (ObjNum 1145 snap 350 seq_num 1145) 2024-09-12T23:50:55.732 INFO:teuthology.orchestra.run.smithi136.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-12T23:50:55.732 INFO:teuthology.orchestra.run.smithi136.stdout:alertmanager.a smithi136 *:9093,9094 running (69s) 63s ago 16m 13.8M - 0.25.0 c8568f914cd2 f36cb97f345b 2024-09-12T23:50:55.733 INFO:teuthology.orchestra.run.smithi136.stdout:grafana.a smithi181 *:3000 running (44s) 39s ago 15m 52.0M - 9.4.7 954c08fa6188 ffb746fc3d92 2024-09-12T23:50:55.733 INFO:teuthology.orchestra.run.smithi136.stdout:mds.a smithi136 running (2m) 63s ago 17m 13.7M - 18.2.4-837-g7a853b17 f21ba1eea74e a20ac830342c 2024-09-12T23:50:55.733 INFO:teuthology.orchestra.run.smithi136.stdout:mds.b smithi181 running (2m) 39s ago 17m 12.0M - 18.2.4-837-g7a853b17 f21ba1eea74e 0dc9f76cf0b8 2024-09-12T23:50:55.733 INFO:teuthology.orchestra.run.smithi136.stdout:mgr.x smithi181 *:8443,9283,8765 running (9m) 39s ago 24m 428M - 18.2.4-837-g7a853b17 f21ba1eea74e cc664dc337f7 2024-09-12T23:50:55.733 INFO:teuthology.orchestra.run.smithi136.stdout:mgr.y smithi136 *:8443,9283,8765 running (10m) 63s ago 27m 498M - 18.2.4-837-g7a853b17 f21ba1eea74e 272f10b42fc3 2024-09-12T23:50:55.733 INFO:teuthology.orchestra.run.smithi136.stdout:mon.a smithi136 running (8m) 63s ago 27m 72.2M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e ad8d21d58ad9 2024-09-12T23:50:55.733 INFO:teuthology.orchestra.run.smithi136.stdout:mon.b smithi181 running (7m) 39s ago 25m 54.4M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e a70ec8315934 2024-09-12T23:50:55.733 INFO:teuthology.orchestra.run.smithi136.stdout:mon.c smithi136 running (8m) 63s ago 25m 60.4M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e 44d1be055a5e 2024-09-12T23:50:55.733 INFO:teuthology.orchestra.run.smithi136.stdout:node-exporter.a smithi136 *:9100 running (2m) 63s ago 16m 8248k - 1.5.0 0da6a335fe13 348c0ab0964c 2024-09-12T23:50:55.733 INFO:teuthology.orchestra.run.smithi136.stdout:node-exporter.b smithi181 *:9100 running (119s) 39s ago 16m 7535k - 1.5.0 0da6a335fe13 3af01e08b5af 2024-09-12T23:50:55.734 INFO:teuthology.orchestra.run.smithi136.stdout:osd.0 smithi136 running (6m) 63s ago 24m 404M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 7da21f765721 2024-09-12T23:50:55.734 INFO:teuthology.orchestra.run.smithi136.stdout:osd.1 smithi136 running (6m) 63s ago 23m 424M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 5302be0a016b 2024-09-12T23:50:55.734 INFO:teuthology.orchestra.run.smithi136.stdout:osd.2 smithi136 running (5m) 63s ago 22m 385M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 2e12d1a31223 2024-09-12T23:50:55.734 INFO:teuthology.orchestra.run.smithi136.stdout:osd.3 smithi136 running (5m) 63s ago 21m 297M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e aa9dc124e59c 2024-09-12T23:50:55.734 INFO:teuthology.orchestra.run.smithi136.stdout:osd.4 smithi181 running (4m) 39s ago 20m 402M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 4ee3b228ce7d 2024-09-12T23:50:55.734 INFO:teuthology.orchestra.run.smithi136.stdout:osd.5 smithi181 running (4m) 39s ago 20m 238M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 3f9a09565258 2024-09-12T23:50:55.734 INFO:teuthology.orchestra.run.smithi136.stdout:osd.6 smithi181 running (3m) 39s ago 19m 143M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e ce539f6fb854 2024-09-12T23:50:55.734 INFO:teuthology.orchestra.run.smithi136.stdout:osd.7 smithi181 running (3m) 39s ago 18m 260M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 6ad9ab169a7b 2024-09-12T23:50:55.734 INFO:teuthology.orchestra.run.smithi136.stdout:prometheus.a smithi181 *:9095 running (94s) 39s ago 16m 46.2M - 2.43.0 a07b618ecd1d 302d5b6a8434 2024-09-12T23:50:55.745 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:55 smithi136 bash[75164]: cluster 2024-09-12T23:50:54.521355+0000 mon.a (mon.0) 1359 : cluster [DBG] osdmap e786: 8 total, 8 up, 8 in 2024-09-12T23:50:55.745 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:55 smithi136 bash[75164]: cluster 2024-09-12T23:50:54.863116+0000 mgr.y (mgr.24661) 412 : cluster [DBG] pgmap v761: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 308 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 2.5 MiB/s wr, 11 op/s 2024-09-12T23:50:55.745 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:55 smithi136 bash[76371]: cluster 2024-09-12T23:50:54.521355+0000 mon.a (mon.0) 1359 : cluster [DBG] osdmap e786: 8 total, 8 up, 8 in 2024-09-12T23:50:55.745 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:55 smithi136 bash[76371]: cluster 2024-09-12T23:50:54.863116+0000 mgr.y (mgr.24661) 412 : cluster [DBG] pgmap v761: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 308 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 2.5 MiB/s wr, 11 op/s 2024-09-12T23:50:55.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:55 smithi181 bash[67587]: cluster 2024-09-12T23:50:54.521355+0000 mon.a (mon.0) 1359 : cluster [DBG] osdmap e786: 8 total, 8 up, 8 in 2024-09-12T23:50:55.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:55 smithi181 bash[67587]: cluster 2024-09-12T23:50:54.863116+0000 mgr.y (mgr.24661) 412 : cluster [DBG] pgmap v761: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 308 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 2.5 MiB/s wr, 11 op/s 2024-09-12T23:50:56.458 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid da58a47c-715d-11ef-bceb-c7b262605968 -e sha1=7a853b1706d076d58e0da7a5d3b9b9eb36276880 -- bash -c 'ceph versions' 2024-09-12T23:50:56.505 INFO:tasks.rados.rados.0.smithi136.stdout:3668: done (1 left) 2024-09-12T23:50:56.505 INFO:tasks.rados.rados.0.smithi136.stdout:3669: done (0 left) 2024-09-12T23:50:56.506 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 42 current snap is 354 2024-09-12T23:50:56.506 INFO:tasks.rados.rados.0.smithi136.stdout:3670: seq_num 1152 ranges {2203648=98304} 2024-09-12T23:50:56.541 INFO:tasks.rados.rados.0.smithi136.stdout:3670: writing smithi13667081-42 from 2203648 to 2301952 tid 1 2024-09-12T23:50:56.541 INFO:tasks.rados.rados.0.smithi136.stdout:3671: setattr oid 12 current snap is 354 2024-09-12T23:50:56.547 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 25 current snap is 354 2024-09-12T23:50:56.547 INFO:tasks.rados.rados.0.smithi136.stdout:3672: seq_num 1153 ranges {0=114688} 2024-09-12T23:50:56.549 INFO:tasks.rados.rados.0.smithi136.stdout:3672: writing smithi13667081-25 from 0 to 114688 tid 1 2024-09-12T23:50:56.549 INFO:tasks.rados.rados.0.smithi136.stdout:3673: read oid 23 snap -1 2024-09-12T23:50:56.549 INFO:tasks.rados.rados.0.smithi136.stdout:3673: expect deleted 2024-09-12T23:50:56.550 INFO:tasks.rados.rados.0.smithi136.stdout:3674: delete oid 17 current snap is 354 2024-09-12T23:50:56.550 INFO:tasks.rados.rados.0.smithi136.stdout:3670: finishing write tid 1 to smithi13667081-42 2024-09-12T23:50:56.556 INFO:tasks.rados.rados.0.smithi136.stdout:3673: done (4 left) 2024-09-12T23:50:56.556 INFO:tasks.rados.rados.0.smithi136.stdout:3674: done (3 left) 2024-09-12T23:50:56.556 INFO:tasks.rados.rados.0.smithi136.stdout:3675: setattr oid 43 current snap is 354 2024-09-12T23:50:56.558 INFO:tasks.rados.rados.0.smithi136.stdout:3676: delete oid 23 current snap is 354 2024-09-12T23:50:56.559 INFO:tasks.rados.rados.0.smithi136.stdout:3670: finishing write tid 2 to smithi13667081-42 2024-09-12T23:50:56.561 INFO:tasks.rados.rados.0.smithi136.stdout:3672: finishing write tid 1 to smithi13667081-25 2024-09-12T23:50:56.563 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 12 v 2490 (ObjNum 1106 snap 336 seq_num 1106) dirty exists 2024-09-12T23:50:56.564 INFO:tasks.rados.rados.0.smithi136.stdout:3670: finishing write tid 3 to smithi13667081-42 2024-09-12T23:50:56.564 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 42 v 2508 (ObjNum 1152 snap 354 seq_num 1152) dirty exists 2024-09-12T23:50:56.564 INFO:tasks.rados.rados.0.smithi136.stdout:3670: left oid 42 (ObjNum 1152 snap 354 seq_num 1152) 2024-09-12T23:50:56.564 INFO:tasks.rados.rados.0.smithi136.stdout:3672: finishing write tid 2 to smithi13667081-25 2024-09-12T23:50:56.566 INFO:tasks.rados.rados.0.smithi136.stdout:3672: finishing write tid 3 to smithi13667081-25 2024-09-12T23:50:56.566 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 25 v 2462 (ObjNum 1153 snap 354 seq_num 1153) dirty exists 2024-09-12T23:50:56.567 INFO:tasks.rados.rados.0.smithi136.stdout:3672: left oid 25 (ObjNum 1153 snap 354 seq_num 1153) 2024-09-12T23:50:56.567 INFO:tasks.rados.rados.0.smithi136.stdout:3670: done (4 left) 2024-09-12T23:50:56.567 INFO:tasks.rados.rados.0.smithi136.stdout:3671: done (3 left) 2024-09-12T23:50:56.567 INFO:tasks.rados.rados.0.smithi136.stdout:3672: done (2 left) 2024-09-12T23:50:56.567 INFO:tasks.rados.rados.0.smithi136.stdout:3676: done (1 left) 2024-09-12T23:50:56.567 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 28 current snap is 354 2024-09-12T23:50:56.567 INFO:tasks.rados.rados.0.smithi136.stdout:3677: seq_num 1156 ranges {6684672=163840} 2024-09-12T23:50:56.617 INFO:tasks.rados.rados.0.smithi136.stdout:3677: writing smithi13667081-28 from 6684672 to 6848512 tid 1 2024-09-12T23:50:56.617 INFO:tasks.rados.rados.0.smithi136.stdout:3678: snap_remove snap 351 2024-09-12T23:50:56.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:56 smithi136 bash[75164]: cluster 2024-09-12T23:50:55.537400+0000 mon.a (mon.0) 1360 : cluster [DBG] osdmap e787: 8 total, 8 up, 8 in 2024-09-12T23:50:56.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:56 smithi136 bash[75164]: audit 2024-09-12T23:50:55.706561+0000 mgr.y (mgr.24661) 413 : audit [DBG] from='client.34500 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:50:56.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:56 smithi136 bash[75164]: cluster 2024-09-12T23:50:56.507959+0000 mon.a (mon.0) 1361 : cluster [DBG] osdmap e788: 8 total, 8 up, 8 in 2024-09-12T23:50:56.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:56 smithi136 bash[76371]: cluster 2024-09-12T23:50:55.537400+0000 mon.a (mon.0) 1360 : cluster [DBG] osdmap e787: 8 total, 8 up, 8 in 2024-09-12T23:50:56.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:56 smithi136 bash[76371]: audit 2024-09-12T23:50:55.706561+0000 mgr.y (mgr.24661) 413 : audit [DBG] from='client.34500 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:50:56.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:56 smithi136 bash[76371]: cluster 2024-09-12T23:50:56.507959+0000 mon.a (mon.0) 1361 : cluster [DBG] osdmap e788: 8 total, 8 up, 8 in 2024-09-12T23:50:56.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:56 smithi181 bash[67587]: cluster 2024-09-12T23:50:55.537400+0000 mon.a (mon.0) 1360 : cluster [DBG] osdmap e787: 8 total, 8 up, 8 in 2024-09-12T23:50:56.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:56 smithi181 bash[67587]: audit 2024-09-12T23:50:55.706561+0000 mgr.y (mgr.24661) 413 : audit [DBG] from='client.34500 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-12T23:50:56.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:56 smithi181 bash[67587]: cluster 2024-09-12T23:50:56.507959+0000 mon.a (mon.0) 1361 : cluster [DBG] osdmap e788: 8 total, 8 up, 8 in 2024-09-12T23:50:57.553 INFO:tasks.rados.rados.0.smithi136.stdout:3678: done (2 left) 2024-09-12T23:50:57.553 INFO:tasks.rados.rados.0.smithi136.stdout:3679: copy_from oid 35 from oid 27 current snap is 354 2024-09-12T23:50:57.553 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 43 v 2519 (ObjNum 1104 snap 333 seq_num 1104) dirty exists 2024-09-12T23:50:57.553 INFO:tasks.rados.rados.0.smithi136.stdout:3677: finishing write tid 1 to smithi13667081-28 2024-09-12T23:50:57.554 INFO:tasks.rados.rados.0.smithi136.stdout:3677: finishing write tid 2 to smithi13667081-28 2024-09-12T23:50:57.554 INFO:tasks.rados.rados.0.smithi136.stdout:3677: finishing write tid 3 to smithi13667081-28 2024-09-12T23:50:57.554 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 28 v 2521 (ObjNum 1156 snap 354 seq_num 1156) dirty exists 2024-09-12T23:50:57.554 INFO:tasks.rados.rados.0.smithi136.stdout:3677: left oid 28 (ObjNum 1156 snap 354 seq_num 1156) 2024-09-12T23:50:57.554 INFO:tasks.rados.rados.0.smithi136.stdout:3675: done (2 left) 2024-09-12T23:50:57.554 INFO:tasks.rados.rados.0.smithi136.stdout:3677: done (1 left) 2024-09-12T23:50:57.554 INFO:tasks.rados.rados.0.smithi136.stdout:3680: rollback oid 20 current snap is 354 2024-09-12T23:50:57.554 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 20 to 348 2024-09-12T23:50:57.554 INFO:tasks.rados.rados.0.smithi136.stdout:3681: snap_create 2024-09-12T23:50:57.565 INFO:tasks.rados.rados.0.smithi136.stdout:3680: finishing rollback tid 0 to smithi13667081-20 2024-09-12T23:50:57.566 INFO:tasks.rados.rados.0.smithi136.stdout:3680: finishing rollback tid 1 to smithi13667081-20 2024-09-12T23:50:57.566 INFO:tasks.rados.rados.0.smithi136.stdout:3680: finishing rollback tid 2 to smithi13667081-20 2024-09-12T23:50:57.566 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 20 v 1659 (ObjNum 1111 snap 337 seq_num 1111) dirty exists 2024-09-12T23:50:57.585 INFO:tasks.rados.rados.0.smithi136.stdout:3679: finishing copy_from racing read to smithi13667081-35 2024-09-12T23:50:57.590 INFO:tasks.rados.rados.0.smithi136.stdout:3679: finishing copy_from to smithi13667081-35 2024-09-12T23:50:57.590 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 35 v 2472 (ObjNum 1091 snap 329 seq_num 1091) dirty exists 2024-09-12T23:50:57.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:57 smithi181 bash[67587]: cluster 2024-09-12T23:50:56.864009+0000 mgr.y (mgr.24661) 414 : cluster [DBG] pgmap v764: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 287 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 2.5 MiB/s wr, 28 op/s 2024-09-12T23:50:58.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:57 smithi136 bash[75164]: cluster 2024-09-12T23:50:56.864009+0000 mgr.y (mgr.24661) 414 : cluster [DBG] pgmap v764: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 287 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 2.5 MiB/s wr, 28 op/s 2024-09-12T23:50:58.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:57 smithi136 bash[76371]: cluster 2024-09-12T23:50:56.864009+0000 mgr.y (mgr.24661) 414 : cluster [DBG] pgmap v764: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 287 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 2.5 MiB/s wr, 28 op/s 2024-09-12T23:50:58.558 INFO:tasks.rados.rados.0.smithi136.stdout:3679: done (2 left) 2024-09-12T23:50:58.558 INFO:tasks.rados.rados.0.smithi136.stdout:3680: done (1 left) 2024-09-12T23:50:58.558 INFO:tasks.rados.rados.0.smithi136.stdout:3681: done (0 left) 2024-09-12T23:50:58.558 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 4 current snap is 355 2024-09-12T23:50:58.558 INFO:tasks.rados.rados.0.smithi136.stdout:3682: seq_num 1157 ranges {22536192=180224} 2024-09-12T23:50:58.697 INFO:tasks.rados.rados.0.smithi136.stdout:3682: writing smithi13667081-4 from 22536192 to 22716416 tid 1 2024-09-12T23:50:58.697 INFO:tasks.rados.rados.0.smithi136.stdout:3683: snap_remove snap 352 2024-09-12T23:50:58.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:58 smithi181 bash[67587]: cluster 2024-09-12T23:50:57.555569+0000 mon.a (mon.0) 1362 : cluster [DBG] osdmap e789: 8 total, 8 up, 8 in 2024-09-12T23:50:59.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:58 smithi136 bash[75164]: cluster 2024-09-12T23:50:57.555569+0000 mon.a (mon.0) 1362 : cluster [DBG] osdmap e789: 8 total, 8 up, 8 in 2024-09-12T23:50:59.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:58 smithi136 bash[76371]: cluster 2024-09-12T23:50:57.555569+0000 mon.a (mon.0) 1362 : cluster [DBG] osdmap e789: 8 total, 8 up, 8 in 2024-09-12T23:50:59.581 INFO:tasks.rados.rados.0.smithi136.stdout:3682: finishing write tid 1 to smithi13667081-4 2024-09-12T23:50:59.581 INFO:tasks.rados.rados.0.smithi136.stdout:3682: finishing write tid 2 to smithi13667081-4 2024-09-12T23:50:59.581 INFO:tasks.rados.rados.0.smithi136.stdout:3682: finishing write tid 3 to smithi13667081-4 2024-09-12T23:50:59.581 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 4 v 1678 (ObjNum 1157 snap 355 seq_num 1157) dirty exists 2024-09-12T23:50:59.581 INFO:tasks.rados.rados.0.smithi136.stdout:3682: left oid 4 (ObjNum 1157 snap 355 seq_num 1157) 2024-09-12T23:50:59.581 INFO:tasks.rados.rados.0.smithi136.stdout:3682: done (1 left) 2024-09-12T23:50:59.581 INFO:tasks.rados.rados.0.smithi136.stdout:3683: done (0 left) 2024-09-12T23:50:59.582 INFO:tasks.rados.rados.0.smithi136.stdout:3684: snap_remove snap 346 2024-09-12T23:50:59.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:59 smithi136 bash[75164]: cluster 2024-09-12T23:50:58.558428+0000 mon.a (mon.0) 1363 : cluster [DBG] osdmap e790: 8 total, 8 up, 8 in 2024-09-12T23:50:59.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:50:59 smithi136 bash[75164]: cluster 2024-09-12T23:50:58.864631+0000 mgr.y (mgr.24661) 415 : cluster [DBG] pgmap v767: 121 pgs: 121 active+clean; 290 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.5 MiB/s wr, 27 op/s 2024-09-12T23:50:59.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:50:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:50:59] "GET /metrics HTTP/1.1" 200 36437 "" "Prometheus/2.43.0" 2024-09-12T23:50:59.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:59 smithi136 bash[76371]: cluster 2024-09-12T23:50:58.558428+0000 mon.a (mon.0) 1363 : cluster [DBG] osdmap e790: 8 total, 8 up, 8 in 2024-09-12T23:50:59.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:50:59 smithi136 bash[76371]: cluster 2024-09-12T23:50:58.864631+0000 mgr.y (mgr.24661) 415 : cluster [DBG] pgmap v767: 121 pgs: 121 active+clean; 290 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.5 MiB/s wr, 27 op/s 2024-09-12T23:50:59.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:59 smithi181 bash[67587]: cluster 2024-09-12T23:50:58.558428+0000 mon.a (mon.0) 1363 : cluster [DBG] osdmap e790: 8 total, 8 up, 8 in 2024-09-12T23:50:59.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:50:59 smithi181 bash[67587]: cluster 2024-09-12T23:50:58.864631+0000 mgr.y (mgr.24661) 415 : cluster [DBG] pgmap v767: 121 pgs: 121 active+clean; 290 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.5 MiB/s wr, 27 op/s 2024-09-12T23:51:00.587 INFO:tasks.rados.rados.0.smithi136.stdout:3684: done (0 left) 2024-09-12T23:51:00.588 INFO:tasks.rados.rados.0.smithi136.stdout:3685: delete oid 49 current snap is 355 2024-09-12T23:51:00.606 INFO:tasks.rados.rados.0.smithi136.stdout:3685: done (0 left) 2024-09-12T23:51:00.606 INFO:tasks.rados.rados.0.smithi136.stdout:3686: snap_remove snap 349 2024-09-12T23:51:00.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:00 smithi181 bash[67587]: cluster 2024-09-12T23:50:59.581968+0000 mon.a (mon.0) 1364 : cluster [DBG] osdmap e791: 8 total, 8 up, 8 in 2024-09-12T23:51:01.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:00 smithi136 bash[75164]: cluster 2024-09-12T23:50:59.581968+0000 mon.a (mon.0) 1364 : cluster [DBG] osdmap e791: 8 total, 8 up, 8 in 2024-09-12T23:51:01.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:00 smithi136 bash[76371]: cluster 2024-09-12T23:50:59.581968+0000 mon.a (mon.0) 1364 : cluster [DBG] osdmap e791: 8 total, 8 up, 8 in 2024-09-12T23:51:01.302 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:51:01.516 INFO:tasks.rados.rados.0.smithi136.stdout:3686: done (0 left) 2024-09-12T23:51:01.516 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 50 current snap is 355 2024-09-12T23:51:01.516 INFO:tasks.rados.rados.0.smithi136.stdout:3687: seq_num 1159 ranges {3317760=212992} 2024-09-12T23:51:01.530 INFO:tasks.rados.rados.0.smithi136.stdout:3687: writing smithi13667081-50 from 3317760 to 3530752 tid 1 2024-09-12T23:51:01.531 INFO:tasks.rados.rados.0.smithi136.stdout:3688: delete oid 42 current snap is 355 2024-09-12T23:51:01.537 INFO:tasks.rados.rados.0.smithi136.stdout:3688: done (1 left) 2024-09-12T23:51:01.537 INFO:tasks.rados.rados.0.smithi136.stdout:3689: snap_create 2024-09-12T23:51:01.540 INFO:tasks.rados.rados.0.smithi136.stdout:3687: finishing write tid 1 to smithi13667081-50 2024-09-12T23:51:01.541 INFO:tasks.rados.rados.0.smithi136.stdout:3687: finishing write tid 2 to smithi13667081-50 2024-09-12T23:51:01.542 INFO:tasks.rados.rados.0.smithi136.stdout:3687: finishing write tid 3 to smithi13667081-50 2024-09-12T23:51:01.543 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 50 v 2690 (ObjNum 1159 snap 355 seq_num 1159) dirty exists 2024-09-12T23:51:01.543 INFO:tasks.rados.rados.0.smithi136.stdout:3687: left oid 50 (ObjNum 1159 snap 355 seq_num 1159) 2024-09-12T23:51:01.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:01 smithi136 bash[75164]: cluster 2024-09-12T23:51:00.588986+0000 mon.a (mon.0) 1365 : cluster [DBG] osdmap e792: 8 total, 8 up, 8 in 2024-09-12T23:51:01.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:01 smithi136 bash[75164]: cluster 2024-09-12T23:51:00.865524+0000 mgr.y (mgr.24661) 416 : cluster [DBG] pgmap v770: 121 pgs: 121 active+clean; 290 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.7 MiB/s rd, 922 KiB/s wr, 9 op/s 2024-09-12T23:51:01.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:01 smithi136 bash[75164]: cluster 2024-09-12T23:51:01.511130+0000 mon.a (mon.0) 1366 : cluster [DBG] osdmap e793: 8 total, 8 up, 8 in 2024-09-12T23:51:01.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:01 smithi136 bash[76371]: cluster 2024-09-12T23:51:00.588986+0000 mon.a (mon.0) 1365 : cluster [DBG] osdmap e792: 8 total, 8 up, 8 in 2024-09-12T23:51:01.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:01 smithi136 bash[76371]: cluster 2024-09-12T23:51:00.865524+0000 mgr.y (mgr.24661) 416 : cluster [DBG] pgmap v770: 121 pgs: 121 active+clean; 290 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.7 MiB/s rd, 922 KiB/s wr, 9 op/s 2024-09-12T23:51:01.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:01 smithi136 bash[76371]: cluster 2024-09-12T23:51:01.511130+0000 mon.a (mon.0) 1366 : cluster [DBG] osdmap e793: 8 total, 8 up, 8 in 2024-09-12T23:51:01.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:01 smithi181 bash[67587]: cluster 2024-09-12T23:51:00.588986+0000 mon.a (mon.0) 1365 : cluster [DBG] osdmap e792: 8 total, 8 up, 8 in 2024-09-12T23:51:01.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:01 smithi181 bash[67587]: cluster 2024-09-12T23:51:00.865524+0000 mgr.y (mgr.24661) 416 : cluster [DBG] pgmap v770: 121 pgs: 121 active+clean; 290 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.7 MiB/s rd, 922 KiB/s wr, 9 op/s 2024-09-12T23:51:01.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:01 smithi181 bash[67587]: cluster 2024-09-12T23:51:01.511130+0000 mon.a (mon.0) 1366 : cluster [DBG] osdmap e793: 8 total, 8 up, 8 in 2024-09-12T23:51:02.514 INFO:tasks.rados.rados.0.smithi136.stdout:3687: done (1 left) 2024-09-12T23:51:02.514 INFO:tasks.rados.rados.0.smithi136.stdout:3689: done (0 left) 2024-09-12T23:51:02.514 INFO:tasks.rados.rados.0.smithi136.stdout:3690: snap_create 2024-09-12T23:51:02.596 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:51:02 smithi181 bash[95505]: ts=2024-09-12T23:51:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:51:03.519 INFO:tasks.rados.rados.0.smithi136.stdout:3690: done (0 left) 2024-09-12T23:51:03.519 INFO:tasks.rados.rados.0.smithi136.stdout:3691: rollback oid 11 current snap is 357 2024-09-12T23:51:03.519 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 11 to 353 2024-09-12T23:51:03.519 INFO:tasks.rados.rados.0.smithi136.stdout:3692: rollback oid 32 current snap is 357 2024-09-12T23:51:03.519 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 32 to 350 2024-09-12T23:51:03.519 INFO:tasks.rados.rados.0.smithi136.stdout:3693: delete oid 15 current snap is 357 2024-09-12T23:51:03.526 INFO:tasks.rados.rados.0.smithi136.stdout:3691: finishing rollback tid 1 to smithi13667081-11 2024-09-12T23:51:03.526 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 11 v 2808 (ObjNum 113 snap 0 seq_num 4243830160) dirty dne 2024-09-12T23:51:03.531 INFO:tasks.rados.rados.0.smithi136.stdout:3691: done (2 left) 2024-09-12T23:51:03.531 INFO:tasks.rados.rados.0.smithi136.stdout:3693: done (1 left) 2024-09-12T23:51:03.531 INFO:tasks.rados.rados.0.smithi136.stdout:3694: rollback oid 20 current snap is 357 2024-09-12T23:51:03.531 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 20 to 356 2024-09-12T23:51:03.531 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 16 current snap is 357 2024-09-12T23:51:03.531 INFO:tasks.rados.rados.0.smithi136.stdout:3695: seq_num 1162 ranges {425984=270336} 2024-09-12T23:51:03.535 INFO:tasks.rados.rados.0.smithi136.stdout:3695: writing smithi13667081-16 from 425984 to 696320 tid 1 2024-09-12T23:51:03.536 INFO:tasks.rados.rados.0.smithi136.stdout:3696: snap_create 2024-09-12T23:51:03.536 INFO:tasks.rados.rados.0.smithi136.stdout:3692: finishing rollback tid 0 to smithi13667081-32 2024-09-12T23:51:03.536 INFO:tasks.rados.rados.0.smithi136.stdout:3692: finishing rollback tid 1 to smithi13667081-32 2024-09-12T23:51:03.536 INFO:tasks.rados.rados.0.smithi136.stdout:3692: finishing rollback tid 2 to smithi13667081-32 2024-09-12T23:51:03.536 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 32 v 1681 (ObjNum 1124 snap 343 seq_num 1124) dirty exists 2024-09-12T23:51:03.539 INFO:tasks.rados.rados.0.smithi136.stdout:3694: finishing rollback tid 0 to smithi13667081-20 2024-09-12T23:51:03.539 INFO:tasks.rados.rados.0.smithi136.stdout:3694: finishing rollback tid 1 to smithi13667081-20 2024-09-12T23:51:03.540 INFO:tasks.rados.rados.0.smithi136.stdout:3694: finishing rollback tid 2 to smithi13667081-20 2024-09-12T23:51:03.540 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 20 v 1662 (ObjNum 1111 snap 337 seq_num 1111) dirty exists 2024-09-12T23:51:03.543 INFO:tasks.rados.rados.0.smithi136.stdout:3695: finishing write tid 1 to smithi13667081-16 2024-09-12T23:51:03.544 INFO:tasks.rados.rados.0.smithi136.stdout:3695: finishing write tid 2 to smithi13667081-16 2024-09-12T23:51:03.546 INFO:tasks.rados.rados.0.smithi136.stdout:3695: finishing write tid 3 to smithi13667081-16 2024-09-12T23:51:03.546 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 16 v 2523 (ObjNum 1162 snap 357 seq_num 1162) dirty exists 2024-09-12T23:51:03.546 INFO:tasks.rados.rados.0.smithi136.stdout:3695: left oid 16 (ObjNum 1162 snap 357 seq_num 1162) 2024-09-12T23:51:03.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:03 smithi181 bash[67587]: cluster 2024-09-12T23:51:02.514414+0000 mon.a (mon.0) 1367 : cluster [DBG] osdmap e794: 8 total, 8 up, 8 in 2024-09-12T23:51:03.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:03 smithi181 bash[67587]: cluster 2024-09-12T23:51:02.866265+0000 mgr.y (mgr.24661) 417 : cluster [DBG] pgmap v773: 121 pgs: 121 active+clean; 288 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.7 MiB/s rd, 112 KiB/s wr, 3 op/s 2024-09-12T23:51:04.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:03 smithi136 bash[76371]: cluster 2024-09-12T23:51:02.514414+0000 mon.a (mon.0) 1367 : cluster [DBG] osdmap e794: 8 total, 8 up, 8 in 2024-09-12T23:51:04.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:03 smithi136 bash[76371]: cluster 2024-09-12T23:51:02.866265+0000 mgr.y (mgr.24661) 417 : cluster [DBG] pgmap v773: 121 pgs: 121 active+clean; 288 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.7 MiB/s rd, 112 KiB/s wr, 3 op/s 2024-09-12T23:51:04.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:03 smithi136 bash[75164]: cluster 2024-09-12T23:51:02.514414+0000 mon.a (mon.0) 1367 : cluster [DBG] osdmap e794: 8 total, 8 up, 8 in 2024-09-12T23:51:04.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:03 smithi136 bash[75164]: cluster 2024-09-12T23:51:02.866265+0000 mgr.y (mgr.24661) 417 : cluster [DBG] pgmap v773: 121 pgs: 121 active+clean; 288 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.7 MiB/s rd, 112 KiB/s wr, 3 op/s 2024-09-12T23:51:04.525 INFO:tasks.rados.rados.0.smithi136.stdout:3692: done (3 left) 2024-09-12T23:51:04.525 INFO:tasks.rados.rados.0.smithi136.stdout:3694: done (2 left) 2024-09-12T23:51:04.525 INFO:tasks.rados.rados.0.smithi136.stdout:3695: done (1 left) 2024-09-12T23:51:04.526 INFO:tasks.rados.rados.0.smithi136.stdout:3696: done (0 left) 2024-09-12T23:51:04.526 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 13 current snap is 358 2024-09-12T23:51:04.526 INFO:tasks.rados.rados.0.smithi136.stdout:3697: seq_num 1163 ranges {2088960=286720} 2024-09-12T23:51:04.538 INFO:tasks.rados.rados.0.smithi136.stdout:3697: writing smithi13667081-13 from 2088960 to 2375680 tid 1 2024-09-12T23:51:04.538 INFO:tasks.rados.rados.0.smithi136.stdout:3698: rollback oid 30 current snap is 358 2024-09-12T23:51:04.539 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 30 to 348 2024-09-12T23:51:04.539 INFO:tasks.rados.rados.0.smithi136.stdout:3699: snap_create 2024-09-12T23:51:04.553 INFO:tasks.rados.rados.0.smithi136.stdout:3697: finishing write tid 1 to smithi13667081-13 2024-09-12T23:51:04.553 INFO:tasks.rados.rados.0.smithi136.stdout:3697: finishing write tid 2 to smithi13667081-13 2024-09-12T23:51:04.555 INFO:tasks.rados.rados.0.smithi136.stdout:3697: finishing write tid 3 to smithi13667081-13 2024-09-12T23:51:04.556 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 13 v 2692 (ObjNum 1163 snap 358 seq_num 1163) dirty exists 2024-09-12T23:51:04.556 INFO:tasks.rados.rados.0.smithi136.stdout:3697: left oid 13 (ObjNum 1163 snap 358 seq_num 1163) 2024-09-12T23:51:04.556 INFO:tasks.rados.rados.0.smithi136.stdout:3698: finishing rollback tid 0 to smithi13667081-30 2024-09-12T23:51:04.556 INFO:tasks.rados.rados.0.smithi136.stdout:3698: finishing rollback tid 1 to smithi13667081-30 2024-09-12T23:51:04.557 INFO:tasks.rados.rados.0.smithi136.stdout:3698: finishing rollback tid 2 to smithi13667081-30 2024-09-12T23:51:04.557 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 30 v 2592 (ObjNum 1091 snap 329 seq_num 1091) dirty exists 2024-09-12T23:51:04.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:04 smithi181 bash[67587]: cluster 2024-09-12T23:51:03.521659+0000 mon.a (mon.0) 1368 : cluster [DBG] osdmap e795: 8 total, 8 up, 8 in 2024-09-12T23:51:04.901 INFO:teuthology.orchestra.run.smithi136.stdout:{ 2024-09-12T23:51:04.901 INFO:teuthology.orchestra.run.smithi136.stdout: "mon": { 2024-09-12T23:51:04.901 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 3 2024-09-12T23:51:04.901 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:51:04.901 INFO:teuthology.orchestra.run.smithi136.stdout: "mgr": { 2024-09-12T23:51:04.901 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 2 2024-09-12T23:51:04.901 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:51:04.901 INFO:teuthology.orchestra.run.smithi136.stdout: "osd": { 2024-09-12T23:51:04.902 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 8 2024-09-12T23:51:04.902 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:51:04.902 INFO:teuthology.orchestra.run.smithi136.stdout: "mds": { 2024-09-12T23:51:04.902 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 2 2024-09-12T23:51:04.902 INFO:teuthology.orchestra.run.smithi136.stdout: }, 2024-09-12T23:51:04.902 INFO:teuthology.orchestra.run.smithi136.stdout: "overall": { 2024-09-12T23:51:04.902 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 15 2024-09-12T23:51:04.902 INFO:teuthology.orchestra.run.smithi136.stdout: } 2024-09-12T23:51:04.902 INFO:teuthology.orchestra.run.smithi136.stdout:} 2024-09-12T23:51:04.913 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:04 smithi136 bash[75164]: cluster 2024-09-12T23:51:03.521659+0000 mon.a (mon.0) 1368 : cluster [DBG] osdmap e795: 8 total, 8 up, 8 in 2024-09-12T23:51:04.913 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:04 smithi136 bash[76371]: cluster 2024-09-12T23:51:03.521659+0000 mon.a (mon.0) 1368 : cluster [DBG] osdmap e795: 8 total, 8 up, 8 in 2024-09-12T23:51:05.539 INFO:tasks.rados.rados.0.smithi136.stdout:3697: done (2 left) 2024-09-12T23:51:05.539 INFO:tasks.rados.rados.0.smithi136.stdout:3698: done (1 left) 2024-09-12T23:51:05.539 INFO:tasks.rados.rados.0.smithi136.stdout:3699: done (0 left) 2024-09-12T23:51:05.539 INFO:tasks.rados.rados.0.smithi136.stdout:3700: read oid 41 snap -1 2024-09-12T23:51:05.539 INFO:tasks.rados.rados.0.smithi136.stdout:3700: expect (ObjNum 1056 snap 321 seq_num 1056) 2024-09-12T23:51:05.539 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 8 current snap is 359 2024-09-12T23:51:05.540 INFO:tasks.rados.rados.0.smithi136.stdout:3701: seq_num 1164 ranges {4194304=303104} 2024-09-12T23:51:05.586 INFO:tasks.rados.rados.0.smithi136.stdout:3701: writing smithi13667081-8 from 4194304 to 4497408 tid 1 2024-09-12T23:51:05.586 INFO:tasks.rados.rados.0.smithi136.stdout:3702: snap_create 2024-09-12T23:51:05.586 INFO:tasks.rados.rados.0.smithi136.stdout:3700: expect (ObjNum 1056 snap 321 seq_num 1056) 2024-09-12T23:51:05.653 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid da58a47c-715d-11ef-bceb-c7b262605968 -e sha1=7a853b1706d076d58e0da7a5d3b9b9eb36276880 -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-09-12T23:51:05.757 INFO:tasks.rados.rados.0.smithi136.stdout:3701: finishing write tid 1 to smithi13667081-8 2024-09-12T23:51:05.757 INFO:tasks.rados.rados.0.smithi136.stdout:3701: finishing write tid 2 to smithi13667081-8 2024-09-12T23:51:05.757 INFO:tasks.rados.rados.0.smithi136.stdout:3701: finishing write tid 3 to smithi13667081-8 2024-09-12T23:51:05.757 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 8 v 2367 (ObjNum 1164 snap 359 seq_num 1164) dirty exists 2024-09-12T23:51:05.757 INFO:tasks.rados.rados.0.smithi136.stdout:3701: left oid 8 (ObjNum 1164 snap 359 seq_num 1164) 2024-09-12T23:51:05.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:05 smithi181 bash[67587]: cluster 2024-09-12T23:51:04.528551+0000 mon.a (mon.0) 1369 : cluster [DBG] osdmap e796: 8 total, 8 up, 8 in 2024-09-12T23:51:05.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:05 smithi181 bash[67587]: cluster 2024-09-12T23:51:04.866963+0000 mgr.y (mgr.24661) 418 : cluster [DBG] pgmap v776: 121 pgs: 121 active+clean; 286 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 206 KiB/s wr, 3 op/s 2024-09-12T23:51:05.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:05 smithi181 bash[67587]: audit 2024-09-12T23:51:04.904235+0000 mon.a (mon.0) 1370 : audit [DBG] from='client.? 172.21.15.136:0/807595569' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:51:06.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:05 smithi136 bash[75164]: cluster 2024-09-12T23:51:04.528551+0000 mon.a (mon.0) 1369 : cluster [DBG] osdmap e796: 8 total, 8 up, 8 in 2024-09-12T23:51:06.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:05 smithi136 bash[75164]: cluster 2024-09-12T23:51:04.866963+0000 mgr.y (mgr.24661) 418 : cluster [DBG] pgmap v776: 121 pgs: 121 active+clean; 286 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 206 KiB/s wr, 3 op/s 2024-09-12T23:51:06.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:05 smithi136 bash[75164]: audit 2024-09-12T23:51:04.904235+0000 mon.a (mon.0) 1370 : audit [DBG] from='client.? 172.21.15.136:0/807595569' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:51:06.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:05 smithi136 bash[76371]: cluster 2024-09-12T23:51:04.528551+0000 mon.a (mon.0) 1369 : cluster [DBG] osdmap e796: 8 total, 8 up, 8 in 2024-09-12T23:51:06.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:05 smithi136 bash[76371]: cluster 2024-09-12T23:51:04.866963+0000 mgr.y (mgr.24661) 418 : cluster [DBG] pgmap v776: 121 pgs: 121 active+clean; 286 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 206 KiB/s wr, 3 op/s 2024-09-12T23:51:06.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:05 smithi136 bash[76371]: audit 2024-09-12T23:51:04.904235+0000 mon.a (mon.0) 1370 : audit [DBG] from='client.? 172.21.15.136:0/807595569' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:51:06.511 INFO:tasks.rados.rados.0.smithi136.stdout:3700: done (2 left) 2024-09-12T23:51:06.512 INFO:tasks.rados.rados.0.smithi136.stdout:3701: done (1 left) 2024-09-12T23:51:06.512 INFO:tasks.rados.rados.0.smithi136.stdout:3702: done (0 left) 2024-09-12T23:51:06.512 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 4 current snap is 360 2024-09-12T23:51:06.512 INFO:tasks.rados.rados.0.smithi136.stdout:3703: seq_num 1165 ranges {22716416=319488} 2024-09-12T23:51:06.670 INFO:tasks.rados.rados.0.smithi136.stdout:3703: writing smithi13667081-4 from 22716416 to 23035904 tid 1 2024-09-12T23:51:06.670 INFO:tasks.rados.rados.0.smithi136.stdout:3704: copy_from oid 41 from oid 13 current snap is 360 2024-09-12T23:51:06.670 INFO:tasks.rados.rados.0.smithi136.stdout:3705: setattr oid 47 current snap is 360 2024-09-12T23:51:06.671 INFO:tasks.rados.rados.0.smithi136.stdout:3706: delete oid 37 current snap is 360 2024-09-12T23:51:06.681 INFO:tasks.rados.rados.0.smithi136.stdout:3706: done (3 left) 2024-09-12T23:51:06.681 INFO:tasks.rados.rados.0.smithi136.stdout:3707: delete oid 25 current snap is 360 2024-09-12T23:51:06.683 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 47 v 2490 (ObjNum 1073 snap 324 seq_num 1073) dirty exists 2024-09-12T23:51:06.690 INFO:tasks.rados.rados.0.smithi136.stdout:3703: finishing write tid 1 to smithi13667081-4 2024-09-12T23:51:06.690 INFO:tasks.rados.rados.0.smithi136.stdout:3705: done (3 left) 2024-09-12T23:51:06.691 INFO:tasks.rados.rados.0.smithi136.stdout:3707: done (2 left) 2024-09-12T23:51:06.691 INFO:tasks.rados.rados.0.smithi136.stdout:3708: snap_remove snap 348 2024-09-12T23:51:06.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:06 smithi181 bash[67587]: cluster 2024-09-12T23:51:05.541595+0000 mon.a (mon.0) 1371 : cluster [DBG] osdmap e797: 8 total, 8 up, 8 in 2024-09-12T23:51:06.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:06 smithi181 bash[67587]: cluster 2024-09-12T23:51:06.513165+0000 mon.a (mon.0) 1372 : cluster [DBG] osdmap e798: 8 total, 8 up, 8 in 2024-09-12T23:51:07.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:06 smithi136 bash[75164]: cluster 2024-09-12T23:51:05.541595+0000 mon.a (mon.0) 1371 : cluster [DBG] osdmap e797: 8 total, 8 up, 8 in 2024-09-12T23:51:07.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:06 smithi136 bash[75164]: cluster 2024-09-12T23:51:06.513165+0000 mon.a (mon.0) 1372 : cluster [DBG] osdmap e798: 8 total, 8 up, 8 in 2024-09-12T23:51:07.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:06 smithi136 bash[76371]: cluster 2024-09-12T23:51:05.541595+0000 mon.a (mon.0) 1371 : cluster [DBG] osdmap e797: 8 total, 8 up, 8 in 2024-09-12T23:51:07.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:06 smithi136 bash[76371]: cluster 2024-09-12T23:51:06.513165+0000 mon.a (mon.0) 1372 : cluster [DBG] osdmap e798: 8 total, 8 up, 8 in 2024-09-12T23:51:07.560 INFO:tasks.rados.rados.0.smithi136.stdout:3703: finishing write tid 2 to smithi13667081-4 2024-09-12T23:51:07.560 INFO:tasks.rados.rados.0.smithi136.stdout:3703: finishing write tid 3 to smithi13667081-4 2024-09-12T23:51:07.560 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 4 v 1683 (ObjNum 1165 snap 360 seq_num 1165) dirty exists 2024-09-12T23:51:07.560 INFO:tasks.rados.rados.0.smithi136.stdout:3703: left oid 4 (ObjNum 1165 snap 360 seq_num 1165) 2024-09-12T23:51:07.560 INFO:tasks.rados.rados.0.smithi136.stdout:3704: finishing copy_from racing read to smithi13667081-41 2024-09-12T23:51:07.560 INFO:tasks.rados.rados.0.smithi136.stdout:3704: finishing copy_from to smithi13667081-41 2024-09-12T23:51:07.560 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 41 v 2692 (ObjNum 1163 snap 358 seq_num 1163) dirty exists 2024-09-12T23:51:07.560 INFO:tasks.rados.rados.0.smithi136.stdout:3703: done (2 left) 2024-09-12T23:51:07.561 INFO:tasks.rados.rados.0.smithi136.stdout:3704: done (1 left) 2024-09-12T23:51:07.561 INFO:tasks.rados.rados.0.smithi136.stdout:3708: done (0 left) 2024-09-12T23:51:07.561 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 41 current snap is 360 2024-09-12T23:51:07.561 INFO:tasks.rados.rados.0.smithi136.stdout:3709: seq_num 1168 ranges {2375680=368640} 2024-09-12T23:51:07.576 INFO:tasks.rados.rados.0.smithi136.stdout:3709: writing smithi13667081-41 from 2375680 to 2744320 tid 1 2024-09-12T23:51:07.577 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 27 current snap is 360 2024-09-12T23:51:07.577 INFO:tasks.rados.rados.0.smithi136.stdout:3710: seq_num 1169 ranges {3514368=385024} 2024-09-12T23:51:07.600 INFO:tasks.rados.rados.0.smithi136.stdout:3710: writing smithi13667081-27 from 3514368 to 3899392 tid 1 2024-09-12T23:51:07.608 INFO:tasks.rados.rados.0.smithi136.stdout:3711: read oid 3 snap -1 2024-09-12T23:51:07.608 INFO:tasks.rados.rados.0.smithi136.stdout:3711: expect deleted 2024-09-12T23:51:07.609 INFO:tasks.rados.rados.0.smithi136.stdout:3712: read oid 6 snap -1 2024-09-12T23:51:07.609 INFO:tasks.rados.rados.0.smithi136.stdout:3712: expect deleted 2024-09-12T23:51:07.609 INFO:tasks.rados.rados.0.smithi136.stdout:3713: copy_from oid 44 from oid 36 current snap is 360 2024-09-12T23:51:07.609 INFO:tasks.rados.rados.0.smithi136.stdout:3709: finishing write tid 1 to smithi13667081-41 2024-09-12T23:51:07.609 INFO:tasks.rados.rados.0.smithi136.stdout:3714: rmattr oid 28 current snap is 360 2024-09-12T23:51:07.609 INFO:tasks.rados.rados.0.smithi136.stdout:3714: done (5 left) 2024-09-12T23:51:07.609 INFO:tasks.rados.rados.0.smithi136.stdout:3715: delete oid 28 current snap is 360 2024-09-12T23:51:07.609 INFO:tasks.rados.rados.0.smithi136.stdout:3709: finishing write tid 2 to smithi13667081-41 2024-09-12T23:51:07.610 INFO:tasks.rados.rados.0.smithi136.stdout:3709: finishing write tid 3 to smithi13667081-41 2024-09-12T23:51:07.610 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 41 v 2694 (ObjNum 1168 snap 360 seq_num 1168) dirty exists 2024-09-12T23:51:07.610 INFO:tasks.rados.rados.0.smithi136.stdout:3709: left oid 41 (ObjNum 1168 snap 360 seq_num 1168) 2024-09-12T23:51:07.610 INFO:tasks.rados.rados.0.smithi136.stdout:3713: finishing copy_from to smithi13667081-44 2024-09-12T23:51:07.610 INFO:tasks.rados.rados.0.smithi136.stdout:3713: got expected ENOENT (src dne) 2024-09-12T23:51:07.610 INFO:tasks.rados.rados.0.smithi136.stdout:3713: finishing copy_from racing read to smithi13667081-44 2024-09-12T23:51:07.610 INFO:tasks.rados.rados.0.smithi136.stdout:3710: finishing write tid 1 to smithi13667081-27 2024-09-12T23:51:07.611 INFO:tasks.rados.rados.0.smithi136.stdout:3710: finishing write tid 2 to smithi13667081-27 2024-09-12T23:51:07.611 INFO:tasks.rados.rados.0.smithi136.stdout:3709: done (5 left) 2024-09-12T23:51:07.611 INFO:tasks.rados.rados.0.smithi136.stdout:3711: done (4 left) 2024-09-12T23:51:07.611 INFO:tasks.rados.rados.0.smithi136.stdout:3712: done (3 left) 2024-09-12T23:51:07.611 INFO:tasks.rados.rados.0.smithi136.stdout:3713: done (2 left) 2024-09-12T23:51:07.611 INFO:tasks.rados.rados.0.smithi136.stdout:3715: done (1 left) 2024-09-12T23:51:07.611 INFO:tasks.rados.rados.0.smithi136.stdout:3716: delete oid 29 current snap is 360 2024-09-12T23:51:07.611 INFO:tasks.rados.rados.0.smithi136.stdout:3710: finishing write tid 3 to smithi13667081-27 2024-09-12T23:51:07.611 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 27 v 2863 (ObjNum 1169 snap 360 seq_num 1169) dirty exists 2024-09-12T23:51:07.612 INFO:tasks.rados.rados.0.smithi136.stdout:3710: left oid 27 (ObjNum 1169 snap 360 seq_num 1169) 2024-09-12T23:51:07.613 INFO:tasks.rados.rados.0.smithi136.stdout:3710: done (1 left) 2024-09-12T23:51:07.613 INFO:tasks.rados.rados.0.smithi136.stdout:3716: done (0 left) 2024-09-12T23:51:07.614 INFO:tasks.rados.rados.0.smithi136.stdout:3717: snap_create 2024-09-12T23:51:07.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:07 smithi181 bash[67587]: cluster 2024-09-12T23:51:06.867888+0000 mgr.y (mgr.24661) 419 : cluster [DBG] pgmap v779: 121 pgs: 121 active+clean; 287 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 188 KiB/s wr, 10 op/s 2024-09-12T23:51:08.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:07 smithi136 bash[75164]: cluster 2024-09-12T23:51:06.867888+0000 mgr.y (mgr.24661) 419 : cluster [DBG] pgmap v779: 121 pgs: 121 active+clean; 287 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 188 KiB/s wr, 10 op/s 2024-09-12T23:51:08.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:07 smithi136 bash[76371]: cluster 2024-09-12T23:51:06.867888+0000 mgr.y (mgr.24661) 419 : cluster [DBG] pgmap v779: 121 pgs: 121 active+clean; 287 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 188 KiB/s wr, 10 op/s 2024-09-12T23:51:08.550 INFO:tasks.rados.rados.0.smithi136.stdout:3717: done (0 left) 2024-09-12T23:51:08.550 INFO:tasks.rados.rados.0.smithi136.stdout:3718: snap_remove snap 353 2024-09-12T23:51:08.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:08 smithi181 bash[67587]: cluster 2024-09-12T23:51:07.561961+0000 mon.a (mon.0) 1373 : cluster [DBG] osdmap e799: 8 total, 8 up, 8 in 2024-09-12T23:51:09.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:08 smithi136 bash[75164]: cluster 2024-09-12T23:51:07.561961+0000 mon.a (mon.0) 1373 : cluster [DBG] osdmap e799: 8 total, 8 up, 8 in 2024-09-12T23:51:09.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:08 smithi136 bash[76371]: cluster 2024-09-12T23:51:07.561961+0000 mon.a (mon.0) 1373 : cluster [DBG] osdmap e799: 8 total, 8 up, 8 in 2024-09-12T23:51:09.556 INFO:tasks.rados.rados.0.smithi136.stdout:3718: done (0 left) 2024-09-12T23:51:09.557 INFO:tasks.rados.rados.0.smithi136.stdout:3719: snap_create 2024-09-12T23:51:09.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:09 smithi136 bash[76371]: cluster 2024-09-12T23:51:08.552591+0000 mon.a (mon.0) 1374 : cluster [DBG] osdmap e800: 8 total, 8 up, 8 in 2024-09-12T23:51:09.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:09 smithi136 bash[76371]: audit 2024-09-12T23:51:08.754129+0000 mon.a (mon.0) 1375 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:51:09.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:09 smithi136 bash[76371]: cluster 2024-09-12T23:51:08.868608+0000 mgr.y (mgr.24661) 420 : cluster [DBG] pgmap v782: 121 pgs: 121 active+clean; 281 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 312 KiB/s wr, 10 op/s 2024-09-12T23:51:09.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:51:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:51:09] "GET /metrics HTTP/1.1" 200 36443 "" "Prometheus/2.43.0" 2024-09-12T23:51:09.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:09 smithi136 bash[75164]: cluster 2024-09-12T23:51:08.552591+0000 mon.a (mon.0) 1374 : cluster [DBG] osdmap e800: 8 total, 8 up, 8 in 2024-09-12T23:51:09.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:09 smithi136 bash[75164]: audit 2024-09-12T23:51:08.754129+0000 mon.a (mon.0) 1375 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:51:09.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:09 smithi136 bash[75164]: cluster 2024-09-12T23:51:08.868608+0000 mgr.y (mgr.24661) 420 : cluster [DBG] pgmap v782: 121 pgs: 121 active+clean; 281 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 312 KiB/s wr, 10 op/s 2024-09-12T23:51:09.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:09 smithi181 bash[67587]: cluster 2024-09-12T23:51:08.552591+0000 mon.a (mon.0) 1374 : cluster [DBG] osdmap e800: 8 total, 8 up, 8 in 2024-09-12T23:51:09.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:09 smithi181 bash[67587]: audit 2024-09-12T23:51:08.754129+0000 mon.a (mon.0) 1375 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:51:09.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:09 smithi181 bash[67587]: cluster 2024-09-12T23:51:08.868608+0000 mgr.y (mgr.24661) 420 : cluster [DBG] pgmap v782: 121 pgs: 121 active+clean; 281 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 312 KiB/s wr, 10 op/s 2024-09-12T23:51:10.476 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:51:10.561 INFO:tasks.rados.rados.0.smithi136.stdout:3719: done (0 left) 2024-09-12T23:51:10.561 INFO:tasks.rados.rados.0.smithi136.stdout:3720: copy_from oid 13 from oid 23 current snap is 362 2024-09-12T23:51:10.561 INFO:tasks.rados.rados.0.smithi136.stdout:3721: delete oid 7 current snap is 362 2024-09-12T23:51:10.572 INFO:tasks.rados.rados.0.smithi136.stdout:3720: finishing copy_from to smithi13667081-13 2024-09-12T23:51:10.572 INFO:tasks.rados.rados.0.smithi136.stdout:3720: got expected ENOENT (src dne) 2024-09-12T23:51:10.572 INFO:tasks.rados.rados.0.smithi136.stdout:3720: finishing copy_from racing read to smithi13667081-13 2024-09-12T23:51:10.580 INFO:tasks.rados.rados.0.smithi136.stdout:3720: done (1 left) 2024-09-12T23:51:10.580 INFO:tasks.rados.rados.0.smithi136.stdout:3721: done (0 left) 2024-09-12T23:51:10.580 INFO:tasks.rados.rados.0.smithi136.stdout:3722: snap_create 2024-09-12T23:51:10.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:10 smithi181 bash[67587]: cluster 2024-09-12T23:51:09.558358+0000 mon.a (mon.0) 1376 : cluster [DBG] osdmap e801: 8 total, 8 up, 8 in 2024-09-12T23:51:11.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:10 smithi136 bash[75164]: cluster 2024-09-12T23:51:09.558358+0000 mon.a (mon.0) 1376 : cluster [DBG] osdmap e801: 8 total, 8 up, 8 in 2024-09-12T23:51:11.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:10 smithi136 bash[76371]: cluster 2024-09-12T23:51:09.558358+0000 mon.a (mon.0) 1376 : cluster [DBG] osdmap e801: 8 total, 8 up, 8 in 2024-09-12T23:51:11.511 INFO:tasks.rados.rados.0.smithi136.stdout:3722: done (0 left) 2024-09-12T23:51:11.511 INFO:tasks.rados.rados.0.smithi136.stdout:3723: snap_create 2024-09-12T23:51:11.841 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:11 smithi136 bash[75164]: cluster 2024-09-12T23:51:10.562367+0000 mon.a (mon.0) 1377 : cluster [DBG] osdmap e802: 8 total, 8 up, 8 in 2024-09-12T23:51:11.842 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:11 smithi136 bash[75164]: cluster 2024-09-12T23:51:10.869435+0000 mgr.y (mgr.24661) 421 : cluster [DBG] pgmap v785: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 281 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 246 KiB/s wr, 3 op/s 2024-09-12T23:51:11.842 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:11 smithi136 bash[75164]: cluster 2024-09-12T23:51:11.514779+0000 mon.a (mon.0) 1378 : cluster [DBG] osdmap e803: 8 total, 8 up, 8 in 2024-09-12T23:51:11.842 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:11 smithi136 bash[76371]: cluster 2024-09-12T23:51:10.562367+0000 mon.a (mon.0) 1377 : cluster [DBG] osdmap e802: 8 total, 8 up, 8 in 2024-09-12T23:51:11.842 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:11 smithi136 bash[76371]: cluster 2024-09-12T23:51:10.869435+0000 mgr.y (mgr.24661) 421 : cluster [DBG] pgmap v785: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 281 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 246 KiB/s wr, 3 op/s 2024-09-12T23:51:11.842 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:11 smithi136 bash[76371]: cluster 2024-09-12T23:51:11.514779+0000 mon.a (mon.0) 1378 : cluster [DBG] osdmap e803: 8 total, 8 up, 8 in 2024-09-12T23:51:11.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:11 smithi181 bash[67587]: cluster 2024-09-12T23:51:10.562367+0000 mon.a (mon.0) 1377 : cluster [DBG] osdmap e802: 8 total, 8 up, 8 in 2024-09-12T23:51:11.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:11 smithi181 bash[67587]: cluster 2024-09-12T23:51:10.869435+0000 mgr.y (mgr.24661) 421 : cluster [DBG] pgmap v785: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 281 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 246 KiB/s wr, 3 op/s 2024-09-12T23:51:11.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:11 smithi181 bash[67587]: cluster 2024-09-12T23:51:11.514779+0000 mon.a (mon.0) 1378 : cluster [DBG] osdmap e803: 8 total, 8 up, 8 in 2024-09-12T23:51:12.514 INFO:tasks.rados.rados.0.smithi136.stdout:3723: done (0 left) 2024-09-12T23:51:12.514 INFO:tasks.rados.rados.0.smithi136.stdout:3724: snap_create 2024-09-12T23:51:12.596 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:51:12 smithi181 bash[95505]: ts=2024-09-12T23:51:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:51:13.525 INFO:tasks.rados.rados.0.smithi136.stdout:3724: done (0 left) 2024-09-12T23:51:13.526 INFO:tasks.rados.rados.0.smithi136.stdout:3725: snap_create 2024-09-12T23:51:13.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:13 smithi181 bash[67587]: cluster 2024-09-12T23:51:12.515944+0000 mon.a (mon.0) 1379 : cluster [DBG] osdmap e804: 8 total, 8 up, 8 in 2024-09-12T23:51:13.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:13 smithi181 bash[67587]: cluster 2024-09-12T23:51:12.870300+0000 mgr.y (mgr.24661) 422 : cluster [DBG] pgmap v788: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 282 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 670 KiB/s wr, 2 op/s 2024-09-12T23:51:13.927 INFO:teuthology.orchestra.run.smithi136.stdout:true 2024-09-12T23:51:13.928 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:13 smithi136 bash[75164]: cluster 2024-09-12T23:51:12.515944+0000 mon.a (mon.0) 1379 : cluster [DBG] osdmap e804: 8 total, 8 up, 8 in 2024-09-12T23:51:13.928 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:13 smithi136 bash[75164]: cluster 2024-09-12T23:51:12.870300+0000 mgr.y (mgr.24661) 422 : cluster [DBG] pgmap v788: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 282 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 670 KiB/s wr, 2 op/s 2024-09-12T23:51:13.928 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:13 smithi136 bash[76371]: cluster 2024-09-12T23:51:12.515944+0000 mon.a (mon.0) 1379 : cluster [DBG] osdmap e804: 8 total, 8 up, 8 in 2024-09-12T23:51:13.929 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:13 smithi136 bash[76371]: cluster 2024-09-12T23:51:12.870300+0000 mgr.y (mgr.24661) 422 : cluster [DBG] pgmap v788: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 282 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 670 KiB/s wr, 2 op/s 2024-09-12T23:51:14.529 INFO:tasks.rados.rados.0.smithi136.stdout:3725: done (0 left) 2024-09-12T23:51:14.529 INFO:tasks.rados.rados.0.smithi136.stdout:3726: read oid 3 snap 360 2024-09-12T23:51:14.529 INFO:tasks.rados.rados.0.smithi136.stdout:3726: expect deleted 2024-09-12T23:51:14.529 INFO:tasks.rados.rados.0.smithi136.stdout:3727: delete oid 41 current snap is 366 2024-09-12T23:51:14.543 INFO:tasks.rados.rados.0.smithi136.stdout:3726: done (1 left) 2024-09-12T23:51:14.543 INFO:tasks.rados.rados.0.smithi136.stdout:3727: done (0 left) 2024-09-12T23:51:14.543 INFO:tasks.rados.rados.0.smithi136.stdout:3728: snap_remove snap 362 2024-09-12T23:51:14.636 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid da58a47c-715d-11ef-bceb-c7b262605968 -e sha1=7a853b1706d076d58e0da7a5d3b9b9eb36276880 -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-09-12T23:51:14.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:14 smithi181 bash[67587]: cluster 2024-09-12T23:51:13.526083+0000 mon.a (mon.0) 1380 : cluster [DBG] osdmap e805: 8 total, 8 up, 8 in 2024-09-12T23:51:14.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:14 smithi181 bash[67587]: audit 2024-09-12T23:51:13.920055+0000 mon.a (mon.0) 1381 : audit [DBG] from='client.? 172.21.15.136:0/3294697403' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:51:15.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:14 smithi136 bash[75164]: cluster 2024-09-12T23:51:13.526083+0000 mon.a (mon.0) 1380 : cluster [DBG] osdmap e805: 8 total, 8 up, 8 in 2024-09-12T23:51:15.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:14 smithi136 bash[75164]: audit 2024-09-12T23:51:13.920055+0000 mon.a (mon.0) 1381 : audit [DBG] from='client.? 172.21.15.136:0/3294697403' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:51:15.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:14 smithi136 bash[76371]: cluster 2024-09-12T23:51:13.526083+0000 mon.a (mon.0) 1380 : cluster [DBG] osdmap e805: 8 total, 8 up, 8 in 2024-09-12T23:51:15.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:14 smithi136 bash[76371]: audit 2024-09-12T23:51:13.920055+0000 mon.a (mon.0) 1381 : audit [DBG] from='client.? 172.21.15.136:0/3294697403' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:51:15.552 INFO:tasks.rados.rados.0.smithi136.stdout:3728: done (0 left) 2024-09-12T23:51:15.552 INFO:tasks.rados.rados.0.smithi136.stdout:3729: rollback oid 48 current snap is 366 2024-09-12T23:51:15.552 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 48 to 361 2024-09-12T23:51:15.552 INFO:tasks.rados.rados.0.smithi136.stdout:3730: copy_from oid 44 from oid 42 current snap is 366 2024-09-12T23:51:15.552 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 41 current snap is 366 2024-09-12T23:51:15.552 INFO:tasks.rados.rados.0.smithi136.stdout:3731: seq_num 1174 ranges {0=466944} 2024-09-12T23:51:15.559 INFO:tasks.rados.rados.0.smithi136.stdout:3731: writing smithi13667081-41 from 0 to 466944 tid 1 2024-09-12T23:51:15.559 INFO:tasks.rados.rados.0.smithi136.stdout:3732: read oid 1 snap -1 2024-09-12T23:51:15.559 INFO:tasks.rados.rados.0.smithi136.stdout:3732: expect deleted 2024-09-12T23:51:15.559 INFO:tasks.rados.rados.0.smithi136.stdout:3733: snap_remove snap 364 2024-09-12T23:51:15.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:15 smithi181 bash[67587]: cluster 2024-09-12T23:51:14.529467+0000 mon.a (mon.0) 1382 : cluster [DBG] osdmap e806: 8 total, 8 up, 8 in 2024-09-12T23:51:15.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:15 smithi181 bash[67587]: cluster 2024-09-12T23:51:14.870809+0000 mgr.y (mgr.24661) 423 : cluster [DBG] pgmap v791: 121 pgs: 121 active+clean; 282 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 670 KiB/s wr, 1 op/s 2024-09-12T23:51:16.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:15 smithi136 bash[75164]: cluster 2024-09-12T23:51:14.529467+0000 mon.a (mon.0) 1382 : cluster [DBG] osdmap e806: 8 total, 8 up, 8 in 2024-09-12T23:51:16.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:15 smithi136 bash[75164]: cluster 2024-09-12T23:51:14.870809+0000 mgr.y (mgr.24661) 423 : cluster [DBG] pgmap v791: 121 pgs: 121 active+clean; 282 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 670 KiB/s wr, 1 op/s 2024-09-12T23:51:16.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:15 smithi136 bash[76371]: cluster 2024-09-12T23:51:14.529467+0000 mon.a (mon.0) 1382 : cluster [DBG] osdmap e806: 8 total, 8 up, 8 in 2024-09-12T23:51:16.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:15 smithi136 bash[76371]: cluster 2024-09-12T23:51:14.870809+0000 mgr.y (mgr.24661) 423 : cluster [DBG] pgmap v791: 121 pgs: 121 active+clean; 282 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 670 KiB/s wr, 1 op/s 2024-09-12T23:51:16.517 INFO:tasks.rados.rados.0.smithi136.stdout:3730: finishing copy_from to smithi13667081-44 2024-09-12T23:51:16.517 INFO:tasks.rados.rados.0.smithi136.stdout:3730: got expected ENOENT (src dne) 2024-09-12T23:51:16.517 INFO:tasks.rados.rados.0.smithi136.stdout:3730: finishing copy_from racing read to smithi13667081-44 2024-09-12T23:51:16.517 INFO:tasks.rados.rados.0.smithi136.stdout:3729: finishing rollback tid 1 to smithi13667081-48 2024-09-12T23:51:16.517 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 48 v 2061 (ObjNum 65 snap 0 seq_num 4243273904) dirty dne 2024-09-12T23:51:16.517 INFO:tasks.rados.rados.0.smithi136.stdout:3731: finishing write tid 1 to smithi13667081-41 2024-09-12T23:51:16.517 INFO:tasks.rados.rados.0.smithi136.stdout:3731: finishing write tid 2 to smithi13667081-41 2024-09-12T23:51:16.517 INFO:tasks.rados.rados.0.smithi136.stdout:3731: finishing write tid 3 to smithi13667081-41 2024-09-12T23:51:16.517 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 41 v 2697 (ObjNum 1174 snap 366 seq_num 1174) dirty exists 2024-09-12T23:51:16.518 INFO:tasks.rados.rados.0.smithi136.stdout:3731: left oid 41 (ObjNum 1174 snap 366 seq_num 1174) 2024-09-12T23:51:16.518 INFO:tasks.rados.rados.0.smithi136.stdout:3729: done (4 left) 2024-09-12T23:51:16.518 INFO:tasks.rados.rados.0.smithi136.stdout:3730: done (3 left) 2024-09-12T23:51:16.518 INFO:tasks.rados.rados.0.smithi136.stdout:3731: done (2 left) 2024-09-12T23:51:16.518 INFO:tasks.rados.rados.0.smithi136.stdout:3732: done (1 left) 2024-09-12T23:51:16.518 INFO:tasks.rados.rados.0.smithi136.stdout:3733: done (0 left) 2024-09-12T23:51:16.518 INFO:tasks.rados.rados.0.smithi136.stdout:3734: snap_create 2024-09-12T23:51:16.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:16 smithi181 bash[67587]: cluster 2024-09-12T23:51:15.553933+0000 mon.a (mon.0) 1383 : cluster [DBG] osdmap e807: 8 total, 8 up, 8 in 2024-09-12T23:51:16.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:16 smithi181 bash[67587]: cluster 2024-09-12T23:51:16.517640+0000 mon.a (mon.0) 1384 : cluster [DBG] osdmap e808: 8 total, 8 up, 8 in 2024-09-12T23:51:17.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:16 smithi136 bash[75164]: cluster 2024-09-12T23:51:15.553933+0000 mon.a (mon.0) 1383 : cluster [DBG] osdmap e807: 8 total, 8 up, 8 in 2024-09-12T23:51:17.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:16 smithi136 bash[75164]: cluster 2024-09-12T23:51:16.517640+0000 mon.a (mon.0) 1384 : cluster [DBG] osdmap e808: 8 total, 8 up, 8 in 2024-09-12T23:51:17.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:16 smithi136 bash[76371]: cluster 2024-09-12T23:51:15.553933+0000 mon.a (mon.0) 1383 : cluster [DBG] osdmap e807: 8 total, 8 up, 8 in 2024-09-12T23:51:17.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:16 smithi136 bash[76371]: cluster 2024-09-12T23:51:16.517640+0000 mon.a (mon.0) 1384 : cluster [DBG] osdmap e808: 8 total, 8 up, 8 in 2024-09-12T23:51:17.517 INFO:tasks.rados.rados.0.smithi136.stdout:3734: done (0 left) 2024-09-12T23:51:17.517 INFO:tasks.rados.rados.0.smithi136.stdout:3735: snap_remove snap 365 2024-09-12T23:51:17.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:17 smithi181 bash[67587]: cluster 2024-09-12T23:51:16.871817+0000 mgr.y (mgr.24661) 424 : cluster [DBG] pgmap v794: 121 pgs: 121 active+clean; 283 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 114 KiB/s wr, 1 op/s 2024-09-12T23:51:17.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:17 smithi181 bash[67587]: cluster 2024-09-12T23:51:17.520539+0000 mon.a (mon.0) 1385 : cluster [DBG] osdmap e809: 8 total, 8 up, 8 in 2024-09-12T23:51:18.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:17 smithi136 bash[75164]: cluster 2024-09-12T23:51:16.871817+0000 mgr.y (mgr.24661) 424 : cluster [DBG] pgmap v794: 121 pgs: 121 active+clean; 283 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 114 KiB/s wr, 1 op/s 2024-09-12T23:51:18.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:17 smithi136 bash[75164]: cluster 2024-09-12T23:51:17.520539+0000 mon.a (mon.0) 1385 : cluster [DBG] osdmap e809: 8 total, 8 up, 8 in 2024-09-12T23:51:18.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:17 smithi136 bash[76371]: cluster 2024-09-12T23:51:16.871817+0000 mgr.y (mgr.24661) 424 : cluster [DBG] pgmap v794: 121 pgs: 121 active+clean; 283 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 114 KiB/s wr, 1 op/s 2024-09-12T23:51:18.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:17 smithi136 bash[76371]: cluster 2024-09-12T23:51:17.520539+0000 mon.a (mon.0) 1385 : cluster [DBG] osdmap e809: 8 total, 8 up, 8 in 2024-09-12T23:51:18.523 INFO:tasks.rados.rados.0.smithi136.stdout:3735: done (0 left) 2024-09-12T23:51:18.523 INFO:tasks.rados.rados.0.smithi136.stdout:3736: snap_create 2024-09-12T23:51:19.428 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:51:19.530 INFO:tasks.rados.rados.0.smithi136.stdout:3736: done (0 left) 2024-09-12T23:51:19.530 INFO:tasks.rados.rados.0.smithi136.stdout:3737: read oid 23 snap -1 2024-09-12T23:51:19.530 INFO:tasks.rados.rados.0.smithi136.stdout:3737: expect deleted 2024-09-12T23:51:19.531 INFO:tasks.rados.rados.0.smithi136.stdout:3738: delete oid 46 current snap is 368 2024-09-12T23:51:19.543 INFO:tasks.rados.rados.0.smithi136.stdout:3737: done (1 left) 2024-09-12T23:51:19.543 INFO:tasks.rados.rados.0.smithi136.stdout:3738: done (0 left) 2024-09-12T23:51:19.543 INFO:tasks.rados.rados.0.smithi136.stdout:3739: rmattr oid 8 current snap is 368 2024-09-12T23:51:19.543 INFO:tasks.rados.rados.0.smithi136.stdout:3740: rmattr oid 37 current snap is 368 2024-09-12T23:51:19.543 INFO:tasks.rados.rados.0.smithi136.stdout:3740: done (1 left) 2024-09-12T23:51:19.543 INFO:tasks.rados.rados.0.smithi136.stdout:3741: read oid 20 snap 358 2024-09-12T23:51:19.543 INFO:tasks.rados.rados.0.smithi136.stdout:3741: expect (ObjNum 1111 snap 337 seq_num 1111) 2024-09-12T23:51:19.543 INFO:tasks.rados.rados.0.smithi136.stdout:3742: read oid 42 snap 296 2024-09-12T23:51:19.543 INFO:tasks.rados.rados.0.smithi136.stdout:3742: expect (ObjNum 895 snap 288 seq_num 895) 2024-09-12T23:51:19.543 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 35 current snap is 368 2024-09-12T23:51:19.543 INFO:tasks.rados.rados.0.smithi136.stdout:3743: seq_num 1176 ranges {3514368=499712} 2024-09-12T23:51:19.566 INFO:tasks.rados.rados.0.smithi136.stdout:3743: writing smithi13667081-35 from 3514368 to 4014080 tid 1 2024-09-12T23:51:19.567 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 12 current snap is 368 2024-09-12T23:51:19.567 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 8 v 2368 (ObjNum 1164 snap 359 seq_num 1164) dirty exists 2024-09-12T23:51:19.567 INFO:tasks.rados.rados.0.smithi136.stdout:3744: seq_num 1177 ranges {4431872=516096} 2024-09-12T23:51:19.601 INFO:tasks.rados.rados.0.smithi136.stdout:3744: writing smithi13667081-12 from 4431872 to 4947968 tid 1 2024-09-12T23:51:19.601 INFO:tasks.rados.rados.0.smithi136.stdout:3739: done (4 left) 2024-09-12T23:51:19.601 INFO:tasks.rados.rados.0.smithi136.stdout:3745: snap_remove snap 366 2024-09-12T23:51:19.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:51:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:51:19] "GET /metrics HTTP/1.1" 200 36443 "" "Prometheus/2.43.0" 2024-09-12T23:51:19.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:19 smithi136 bash[75164]: cluster 2024-09-12T23:51:18.523240+0000 mon.a (mon.0) 1386 : cluster [DBG] osdmap e810: 8 total, 8 up, 8 in 2024-09-12T23:51:19.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:19 smithi136 bash[75164]: cluster 2024-09-12T23:51:18.872557+0000 mgr.y (mgr.24661) 425 : cluster [DBG] pgmap v797: 121 pgs: 121 active+clean; 283 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 114 KiB/s wr, 1 op/s 2024-09-12T23:51:19.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:19 smithi136 bash[76371]: cluster 2024-09-12T23:51:18.523240+0000 mon.a (mon.0) 1386 : cluster [DBG] osdmap e810: 8 total, 8 up, 8 in 2024-09-12T23:51:19.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:19 smithi136 bash[76371]: cluster 2024-09-12T23:51:18.872557+0000 mgr.y (mgr.24661) 425 : cluster [DBG] pgmap v797: 121 pgs: 121 active+clean; 283 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 114 KiB/s wr, 1 op/s 2024-09-12T23:51:19.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:19 smithi181 bash[67587]: cluster 2024-09-12T23:51:18.523240+0000 mon.a (mon.0) 1386 : cluster [DBG] osdmap e810: 8 total, 8 up, 8 in 2024-09-12T23:51:19.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:19 smithi181 bash[67587]: cluster 2024-09-12T23:51:18.872557+0000 mgr.y (mgr.24661) 425 : cluster [DBG] pgmap v797: 121 pgs: 121 active+clean; 283 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 114 KiB/s wr, 1 op/s 2024-09-12T23:51:20.536 INFO:tasks.rados.rados.0.smithi136.stdout:3741: expect (ObjNum 1111 snap 337 seq_num 1111) 2024-09-12T23:51:20.645 INFO:tasks.rados.rados.0.smithi136.stdout:3743: finishing write tid 1 to smithi13667081-35 2024-09-12T23:51:20.646 INFO:tasks.rados.rados.0.smithi136.stdout:3741: done (4 left) 2024-09-12T23:51:20.646 INFO:tasks.rados.rados.0.smithi136.stdout:3745: done (3 left) 2024-09-12T23:51:20.646 INFO:tasks.rados.rados.0.smithi136.stdout:3746: snap_create 2024-09-12T23:51:20.646 INFO:tasks.rados.rados.0.smithi136.stdout:3743: finishing write tid 2 to smithi13667081-35 2024-09-12T23:51:20.646 INFO:tasks.rados.rados.0.smithi136.stdout:3743: finishing write tid 3 to smithi13667081-35 2024-09-12T23:51:20.646 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 35 v 2870 (ObjNum 1176 snap 368 seq_num 1176) dirty exists 2024-09-12T23:51:20.646 INFO:tasks.rados.rados.0.smithi136.stdout:3743: left oid 35 (ObjNum 1176 snap 368 seq_num 1176) 2024-09-12T23:51:20.646 INFO:tasks.rados.rados.0.smithi136.stdout:3742: expect (ObjNum 895 snap 288 seq_num 895) 2024-09-12T23:51:20.820 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:20 smithi136 bash[75164]: cluster 2024-09-12T23:51:19.533233+0000 mon.a (mon.0) 1387 : cluster [DBG] osdmap e811: 8 total, 8 up, 8 in 2024-09-12T23:51:20.821 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:20 smithi136 bash[76371]: cluster 2024-09-12T23:51:19.533233+0000 mon.a (mon.0) 1387 : cluster [DBG] osdmap e811: 8 total, 8 up, 8 in 2024-09-12T23:51:20.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:20 smithi181 bash[67587]: cluster 2024-09-12T23:51:19.533233+0000 mon.a (mon.0) 1387 : cluster [DBG] osdmap e811: 8 total, 8 up, 8 in 2024-09-12T23:51:21.717 INFO:tasks.rados.rados.0.smithi136.stdout:3744: finishing write tid 1 to smithi13667081-12 2024-09-12T23:51:21.717 INFO:tasks.rados.rados.0.smithi136.stdout:3744: finishing write tid 2 to smithi13667081-12 2024-09-12T23:51:21.717 INFO:tasks.rados.rados.0.smithi136.stdout:3744: finishing write tid 3 to smithi13667081-12 2024-09-12T23:51:21.717 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 12 v 2493 (ObjNum 1177 snap 368 seq_num 1177) dirty exists 2024-09-12T23:51:21.717 INFO:tasks.rados.rados.0.smithi136.stdout:3744: left oid 12 (ObjNum 1177 snap 368 seq_num 1177) 2024-09-12T23:51:21.717 INFO:tasks.rados.rados.0.smithi136.stdout:3742: done (3 left) 2024-09-12T23:51:21.718 INFO:tasks.rados.rados.0.smithi136.stdout:3743: done (2 left) 2024-09-12T23:51:21.718 INFO:tasks.rados.rados.0.smithi136.stdout:3744: done (1 left) 2024-09-12T23:51:21.718 INFO:tasks.rados.rados.0.smithi136.stdout:3746: done (0 left) 2024-09-12T23:51:21.718 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 45 current snap is 369 2024-09-12T23:51:21.718 INFO:tasks.rados.rados.0.smithi136.stdout:3747: seq_num 1178 ranges {0=532480} 2024-09-12T23:51:21.724 INFO:tasks.rados.rados.0.smithi136.stdout:3747: writing smithi13667081-45 from 0 to 532480 tid 1 2024-09-12T23:51:21.724 INFO:tasks.rados.rados.0.smithi136.stdout:3748: rollback oid 38 current snap is 369 2024-09-12T23:51:21.725 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 38 to 358 2024-09-12T23:51:21.725 INFO:tasks.rados.rados.0.smithi136.stdout:3749: rollback oid 1 current snap is 369 2024-09-12T23:51:21.725 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 1 to 358 2024-09-12T23:51:21.725 INFO:tasks.rados.rados.0.smithi136.stdout:3750: setattr oid 22 current snap is 369 2024-09-12T23:51:21.727 INFO:tasks.rados.rados.0.smithi136.stdout:3751: snap_create 2024-09-12T23:51:21.729 INFO:tasks.rados.rados.0.smithi136.stdout:3749: finishing rollback tid 1 to smithi13667081-1 2024-09-12T23:51:21.729 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 1 v 2503 (ObjNum 129 snap 0 seq_num 4243273632) dirty dne 2024-09-12T23:51:21.735 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 22 v 2510 (ObjNum 1101 snap 333 seq_num 1101) dirty exists 2024-09-12T23:51:21.745 INFO:tasks.rados.rados.0.smithi136.stdout:3747: finishing write tid 1 to smithi13667081-45 2024-09-12T23:51:21.745 INFO:tasks.rados.rados.0.smithi136.stdout:3748: finishing rollback tid 0 to smithi13667081-38 2024-09-12T23:51:21.746 INFO:tasks.rados.rados.0.smithi136.stdout:3747: finishing write tid 2 to smithi13667081-45 2024-09-12T23:51:21.747 INFO:tasks.rados.rados.0.smithi136.stdout:3748: finishing rollback tid 1 to smithi13667081-38 2024-09-12T23:51:21.748 INFO:tasks.rados.rados.0.smithi136.stdout:3748: finishing rollback tid 2 to smithi13667081-38 2024-09-12T23:51:21.748 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 38 v 2696 (ObjNum 1134 snap 345 seq_num 1134) dirty exists 2024-09-12T23:51:21.750 INFO:tasks.rados.rados.0.smithi136.stdout:3747: finishing write tid 3 to smithi13667081-45 2024-09-12T23:51:21.751 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 45 v 2370 (ObjNum 1178 snap 369 seq_num 1178) dirty exists 2024-09-12T23:51:21.751 INFO:tasks.rados.rados.0.smithi136.stdout:3747: left oid 45 (ObjNum 1178 snap 369 seq_num 1178) 2024-09-12T23:51:21.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:21 smithi181 bash[67587]: cluster 2024-09-12T23:51:20.533862+0000 mon.a (mon.0) 1388 : cluster [DBG] osdmap e812: 8 total, 8 up, 8 in 2024-09-12T23:51:21.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:21 smithi181 bash[67587]: cluster 2024-09-12T23:51:20.873051+0000 mgr.y (mgr.24661) 426 : cluster [DBG] pgmap v800: 121 pgs: 121 active+clean; 283 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 0 op/s 2024-09-12T23:51:22.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:21 smithi136 bash[76371]: cluster 2024-09-12T23:51:20.533862+0000 mon.a (mon.0) 1388 : cluster [DBG] osdmap e812: 8 total, 8 up, 8 in 2024-09-12T23:51:22.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:21 smithi136 bash[76371]: cluster 2024-09-12T23:51:20.873051+0000 mgr.y (mgr.24661) 426 : cluster [DBG] pgmap v800: 121 pgs: 121 active+clean; 283 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 0 op/s 2024-09-12T23:51:22.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:21 smithi136 bash[75164]: cluster 2024-09-12T23:51:20.533862+0000 mon.a (mon.0) 1388 : cluster [DBG] osdmap e812: 8 total, 8 up, 8 in 2024-09-12T23:51:22.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:21 smithi136 bash[75164]: cluster 2024-09-12T23:51:20.873051+0000 mgr.y (mgr.24661) 426 : cluster [DBG] pgmap v800: 121 pgs: 121 active+clean; 283 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 0 op/s 2024-09-12T23:51:22.558 INFO:tasks.rados.rados.0.smithi136.stdout:3747: done (4 left) 2024-09-12T23:51:22.559 INFO:tasks.rados.rados.0.smithi136.stdout:3748: done (3 left) 2024-09-12T23:51:22.559 INFO:tasks.rados.rados.0.smithi136.stdout:3749: done (2 left) 2024-09-12T23:51:22.559 INFO:tasks.rados.rados.0.smithi136.stdout:3750: done (1 left) 2024-09-12T23:51:22.559 INFO:tasks.rados.rados.0.smithi136.stdout:3751: done (0 left) 2024-09-12T23:51:22.559 INFO:tasks.rados.rados.0.smithi136.stdout:3752: rollback oid 29 current snap is 370 2024-09-12T23:51:22.559 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 29 to 361 2024-09-12T23:51:22.559 INFO:tasks.rados.rados.0.smithi136.stdout:3753: snap_create 2024-09-12T23:51:22.568 INFO:tasks.rados.rados.0.smithi136.stdout:3752: finishing rollback tid 1 to smithi13667081-29 2024-09-12T23:51:22.569 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 29 v 2693 (ObjNum 33 snap 0 seq_num 4243493472) dirty dne 2024-09-12T23:51:22.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:22 smithi181 bash[67587]: cluster 2024-09-12T23:51:21.549219+0000 mon.a (mon.0) 1389 : cluster [DBG] osdmap e813: 8 total, 8 up, 8 in 2024-09-12T23:51:22.596 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:51:22 smithi181 bash[95505]: ts=2024-09-12T23:51:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:51:23.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:22 smithi136 bash[75164]: cluster 2024-09-12T23:51:21.549219+0000 mon.a (mon.0) 1389 : cluster [DBG] osdmap e813: 8 total, 8 up, 8 in 2024-09-12T23:51:23.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:22 smithi136 bash[76371]: cluster 2024-09-12T23:51:21.549219+0000 mon.a (mon.0) 1389 : cluster [DBG] osdmap e813: 8 total, 8 up, 8 in 2024-09-12T23:51:23.097 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)" 2024-09-12T23:51:23.568 INFO:tasks.rados.rados.0.smithi136.stdout:3752: done (1 left) 2024-09-12T23:51:23.568 INFO:tasks.rados.rados.0.smithi136.stdout:3753: done (0 left) 2024-09-12T23:51:23.568 INFO:tasks.rados.rados.0.smithi136.stdout:3754: delete oid 19 current snap is 371 2024-09-12T23:51:23.577 INFO:tasks.rados.rados.0.smithi136.stdout:3754: done (0 left) 2024-09-12T23:51:23.577 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 44 current snap is 371 2024-09-12T23:51:23.577 INFO:tasks.rados.rados.0.smithi136.stdout:3755: seq_num 1180 ranges {4235264=565248} 2024-09-12T23:51:23.645 INFO:tasks.rados.rados.0.smithi136.stdout:3755: writing smithi13667081-44 from 4235264 to 4800512 tid 1 2024-09-12T23:51:23.645 INFO:tasks.rados.rados.0.smithi136.stdout:3756: snap_create 2024-09-12T23:51:23.662 INFO:tasks.rados.rados.0.smithi136.stdout:3755: finishing write tid 1 to smithi13667081-44 2024-09-12T23:51:23.662 INFO:tasks.rados.rados.0.smithi136.stdout:3755: finishing write tid 2 to smithi13667081-44 2024-09-12T23:51:23.666 INFO:tasks.rados.rados.0.smithi136.stdout:3755: finishing write tid 3 to smithi13667081-44 2024-09-12T23:51:23.666 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 44 v 2526 (ObjNum 1180 snap 371 seq_num 1180) dirty exists 2024-09-12T23:51:23.666 INFO:tasks.rados.rados.0.smithi136.stdout:3755: left oid 44 (ObjNum 1180 snap 371 seq_num 1180) 2024-09-12T23:51:23.812 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:23 smithi136 bash[76371]: cluster 2024-09-12T23:51:22.560832+0000 mon.a (mon.0) 1390 : cluster [DBG] osdmap e814: 8 total, 8 up, 8 in 2024-09-12T23:51:23.812 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:23 smithi136 bash[76371]: cluster 2024-09-12T23:51:22.873495+0000 mgr.y (mgr.24661) 427 : cluster [DBG] pgmap v803: 121 pgs: 121 active+clean; 284 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 256 KiB/s wr, 4 op/s 2024-09-12T23:51:23.812 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:23 smithi136 bash[76371]: audit 2024-09-12T23:51:23.088490+0000 mon.c (mon.2) 16 : audit [DBG] from='client.? 172.21.15.136:0/652360991' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:51:23.812 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:23 smithi136 bash[75164]: cluster 2024-09-12T23:51:22.560832+0000 mon.a (mon.0) 1390 : cluster [DBG] osdmap e814: 8 total, 8 up, 8 in 2024-09-12T23:51:23.812 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:23 smithi136 bash[75164]: cluster 2024-09-12T23:51:22.873495+0000 mgr.y (mgr.24661) 427 : cluster [DBG] pgmap v803: 121 pgs: 121 active+clean; 284 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 256 KiB/s wr, 4 op/s 2024-09-12T23:51:23.812 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:23 smithi136 bash[75164]: audit 2024-09-12T23:51:23.088490+0000 mon.c (mon.2) 16 : audit [DBG] from='client.? 172.21.15.136:0/652360991' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:51:23.813 INFO:teuthology.task.sequential:In sequential, running task print... 2024-09-12T23:51:23.814 INFO:teuthology.task.print:**** done end upgrade, wait... 2024-09-12T23:51:23.814 DEBUG:teuthology.parallel:result is None 2024-09-12T23:51:23.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:23 smithi181 bash[67587]: cluster 2024-09-12T23:51:22.560832+0000 mon.a (mon.0) 1390 : cluster [DBG] osdmap e814: 8 total, 8 up, 8 in 2024-09-12T23:51:23.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:23 smithi181 bash[67587]: cluster 2024-09-12T23:51:22.873495+0000 mgr.y (mgr.24661) 427 : cluster [DBG] pgmap v803: 121 pgs: 121 active+clean; 284 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 256 KiB/s wr, 4 op/s 2024-09-12T23:51:23.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:23 smithi181 bash[67587]: audit 2024-09-12T23:51:23.088490+0000 mon.c (mon.2) 16 : audit [DBG] from='client.? 172.21.15.136:0/652360991' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-12T23:51:24.577 INFO:tasks.rados.rados.0.smithi136.stdout:3755: done (1 left) 2024-09-12T23:51:24.577 INFO:tasks.rados.rados.0.smithi136.stdout:3756: done (0 left) 2024-09-12T23:51:24.577 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 23 current snap is 372 2024-09-12T23:51:24.577 INFO:tasks.rados.rados.0.smithi136.stdout:3757: seq_num 1181 ranges {0=581632} 2024-09-12T23:51:24.583 INFO:tasks.rados.rados.0.smithi136.stdout:3757: writing smithi13667081-23 from 0 to 581632 tid 1 2024-09-12T23:51:24.583 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 3 current snap is 372 2024-09-12T23:51:24.583 INFO:tasks.rados.rados.0.smithi136.stdout:3758: seq_num 1182 ranges {0=606208} 2024-09-12T23:51:24.590 INFO:tasks.rados.rados.0.smithi136.stdout:3758: writing smithi13667081-3 from 0 to 606208 tid 1 2024-09-12T23:51:24.590 INFO:tasks.rados.rados.0.smithi136.stdout:3759: read oid 34 snap -1 2024-09-12T23:51:24.591 INFO:tasks.rados.rados.0.smithi136.stdout:3759: expect (ObjNum 1146 snap 350 seq_num 1146) 2024-09-12T23:51:24.591 INFO:tasks.rados.rados.0.smithi136.stdout:3760: rollback oid 19 current snap is 372 2024-09-12T23:51:24.591 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 19 to 296 2024-09-12T23:51:24.591 INFO:tasks.rados.rados.0.smithi136.stdout:3761: delete oid 30 current snap is 372 2024-09-12T23:51:24.593 INFO:tasks.rados.rados.0.smithi136.stdout:3757: finishing write tid 1 to smithi13667081-23 2024-09-12T23:51:24.599 INFO:tasks.rados.rados.0.smithi136.stdout:3757: finishing write tid 2 to smithi13667081-23 2024-09-12T23:51:24.599 INFO:tasks.rados.rados.0.smithi136.stdout:3760: finishing rollback tid 1 to smithi13667081-19 2024-09-12T23:51:24.599 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 19 v 2491 (ObjNum 33 snap 0 seq_num 4210353496) dirty dne 2024-09-12T23:51:24.599 INFO:tasks.rados.rados.0.smithi136.stdout:3760: done (4 left) 2024-09-12T23:51:24.599 INFO:tasks.rados.rados.0.smithi136.stdout:3761: done (3 left) 2024-09-12T23:51:24.599 INFO:tasks.rados.rados.0.smithi136.stdout:3762: rollback oid 44 current snap is 372 2024-09-12T23:51:24.599 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 44 to 355 2024-09-12T23:51:24.600 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 15 current snap is 372 2024-09-12T23:51:24.600 INFO:tasks.rados.rados.0.smithi136.stdout:3763: seq_num 1184 ranges {0=638976} 2024-09-12T23:51:24.608 INFO:tasks.rados.rados.0.smithi136.stdout:3763: writing smithi13667081-15 from 0 to 638976 tid 1 2024-09-12T23:51:24.609 INFO:tasks.rados.rados.0.smithi136.stdout:3764: rollback oid 40 current snap is 372 2024-09-12T23:51:24.609 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 40 to 358 2024-09-12T23:51:24.609 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 28 current snap is 372 2024-09-12T23:51:24.609 INFO:tasks.rados.rados.0.smithi136.stdout:3765: seq_num 1185 ranges {0=655360} 2024-09-12T23:51:24.617 INFO:tasks.rados.rados.0.smithi136.stdout:3765: writing smithi13667081-28 from 0 to 655360 tid 1 2024-09-12T23:51:24.617 INFO:tasks.rados.rados.0.smithi136.stdout:3766: snap_create 2024-09-12T23:51:24.617 INFO:tasks.rados.rados.0.smithi136.stdout:3764: finishing rollback tid 1 to smithi13667081-40 2024-09-12T23:51:24.617 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 40 v 0 (ObjNum 33 snap 0 seq_num 4210353496) dirty dne 2024-09-12T23:51:24.617 INFO:tasks.rados.rados.0.smithi136.stdout:3757: finishing write tid 3 to smithi13667081-23 2024-09-12T23:51:24.618 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 23 v 2874 (ObjNum 1181 snap 372 seq_num 1181) dirty exists 2024-09-12T23:51:24.618 INFO:tasks.rados.rados.0.smithi136.stdout:3757: left oid 23 (ObjNum 1181 snap 372 seq_num 1181) 2024-09-12T23:51:24.622 INFO:tasks.rados.rados.0.smithi136.stdout:3759: expect (ObjNum 1146 snap 350 seq_num 1146) 2024-09-12T23:51:24.717 INFO:tasks.rados.rados.0.smithi136.stdout:3758: finishing write tid 1 to smithi13667081-3 2024-09-12T23:51:24.717 INFO:tasks.rados.rados.0.smithi136.stdout:3758: finishing write tid 2 to smithi13667081-3 2024-09-12T23:51:24.718 INFO:tasks.rados.rados.0.smithi136.stdout:3762: finishing rollback tid 0 to smithi13667081-44 2024-09-12T23:51:24.718 INFO:tasks.rados.rados.0.smithi136.stdout:3762: finishing rollback tid 1 to smithi13667081-44 2024-09-12T23:51:24.718 INFO:tasks.rados.rados.0.smithi136.stdout:3762: finishing rollback tid 2 to smithi13667081-44 2024-09-12T23:51:24.718 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 44 v 2529 (ObjNum 1135 snap 345 seq_num 1135) dirty exists 2024-09-12T23:51:24.718 INFO:tasks.rados.rados.0.smithi136.stdout:3763: finishing write tid 1 to smithi13667081-15 2024-09-12T23:51:24.718 INFO:tasks.rados.rados.0.smithi136.stdout:3758: finishing write tid 3 to smithi13667081-3 2024-09-12T23:51:24.718 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 3 v 2495 (ObjNum 1182 snap 372 seq_num 1182) dirty exists 2024-09-12T23:51:24.718 INFO:tasks.rados.rados.0.smithi136.stdout:3758: left oid 3 (ObjNum 1182 snap 372 seq_num 1182) 2024-09-12T23:51:24.718 INFO:tasks.rados.rados.0.smithi136.stdout:3763: finishing write tid 2 to smithi13667081-15 2024-09-12T23:51:24.718 INFO:tasks.rados.rados.0.smithi136.stdout:3763: finishing write tid 3 to smithi13667081-15 2024-09-12T23:51:24.718 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 15 v 2497 (ObjNum 1184 snap 372 seq_num 1184) dirty exists 2024-09-12T23:51:24.719 INFO:tasks.rados.rados.0.smithi136.stdout:3763: left oid 15 (ObjNum 1184 snap 372 seq_num 1184) 2024-09-12T23:51:24.719 INFO:tasks.rados.rados.0.smithi136.stdout:3765: finishing write tid 1 to smithi13667081-28 2024-09-12T23:51:24.719 INFO:tasks.rados.rados.0.smithi136.stdout:3765: finishing write tid 2 to smithi13667081-28 2024-09-12T23:51:24.719 INFO:tasks.rados.rados.0.smithi136.stdout:3765: finishing write tid 3 to smithi13667081-28 2024-09-12T23:51:24.719 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 28 v 2531 (ObjNum 1185 snap 372 seq_num 1185) dirty exists 2024-09-12T23:51:24.719 INFO:tasks.rados.rados.0.smithi136.stdout:3765: left oid 28 (ObjNum 1185 snap 372 seq_num 1185) 2024-09-12T23:51:24.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:24 smithi181 bash[67587]: cluster 2024-09-12T23:51:23.570552+0000 mon.a (mon.0) 1391 : cluster [DBG] osdmap e815: 8 total, 8 up, 8 in 2024-09-12T23:51:24.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:24 smithi181 bash[67587]: audit 2024-09-12T23:51:23.754596+0000 mon.a (mon.0) 1392 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:51:25.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:24 smithi136 bash[75164]: cluster 2024-09-12T23:51:23.570552+0000 mon.a (mon.0) 1391 : cluster [DBG] osdmap e815: 8 total, 8 up, 8 in 2024-09-12T23:51:25.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:24 smithi136 bash[75164]: audit 2024-09-12T23:51:23.754596+0000 mon.a (mon.0) 1392 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:51:25.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:24 smithi136 bash[76371]: cluster 2024-09-12T23:51:23.570552+0000 mon.a (mon.0) 1391 : cluster [DBG] osdmap e815: 8 total, 8 up, 8 in 2024-09-12T23:51:25.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:24 smithi136 bash[76371]: audit 2024-09-12T23:51:23.754596+0000 mon.a (mon.0) 1392 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:51:25.587 INFO:tasks.rados.rados.0.smithi136.stdout:3757: done (7 left) 2024-09-12T23:51:25.587 INFO:tasks.rados.rados.0.smithi136.stdout:3758: done (6 left) 2024-09-12T23:51:25.587 INFO:tasks.rados.rados.0.smithi136.stdout:3759: done (5 left) 2024-09-12T23:51:25.587 INFO:tasks.rados.rados.0.smithi136.stdout:3762: done (4 left) 2024-09-12T23:51:25.587 INFO:tasks.rados.rados.0.smithi136.stdout:3763: done (3 left) 2024-09-12T23:51:25.587 INFO:tasks.rados.rados.0.smithi136.stdout:3764: done (2 left) 2024-09-12T23:51:25.587 INFO:tasks.rados.rados.0.smithi136.stdout:3765: done (1 left) 2024-09-12T23:51:25.588 INFO:tasks.rados.rados.0.smithi136.stdout:3766: done (0 left) 2024-09-12T23:51:25.588 INFO:tasks.rados.rados.0.smithi136.stdout:3767: delete oid 49 current snap is 373 2024-09-12T23:51:25.593 INFO:tasks.rados.rados.0.smithi136.stdout:3767: done (0 left) 2024-09-12T23:51:25.593 INFO:tasks.rados.rados.0.smithi136.stdout:3768: snap_remove snap 356 2024-09-12T23:51:25.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:25 smithi181 bash[67587]: cluster 2024-09-12T23:51:24.579356+0000 mon.a (mon.0) 1393 : cluster [DBG] osdmap e816: 8 total, 8 up, 8 in 2024-09-12T23:51:25.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:25 smithi181 bash[67587]: cluster 2024-09-12T23:51:24.874303+0000 mgr.y (mgr.24661) 428 : cluster [DBG] pgmap v806: 121 pgs: 121 active+clean; 284 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.3 MiB/s rd, 378 KiB/s wr, 10 op/s 2024-09-12T23:51:26.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:25 smithi136 bash[75164]: cluster 2024-09-12T23:51:24.579356+0000 mon.a (mon.0) 1393 : cluster [DBG] osdmap e816: 8 total, 8 up, 8 in 2024-09-12T23:51:26.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:25 smithi136 bash[75164]: cluster 2024-09-12T23:51:24.874303+0000 mgr.y (mgr.24661) 428 : cluster [DBG] pgmap v806: 121 pgs: 121 active+clean; 284 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.3 MiB/s rd, 378 KiB/s wr, 10 op/s 2024-09-12T23:51:26.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:25 smithi136 bash[76371]: cluster 2024-09-12T23:51:24.579356+0000 mon.a (mon.0) 1393 : cluster [DBG] osdmap e816: 8 total, 8 up, 8 in 2024-09-12T23:51:26.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:25 smithi136 bash[76371]: cluster 2024-09-12T23:51:24.874303+0000 mgr.y (mgr.24661) 428 : cluster [DBG] pgmap v806: 121 pgs: 121 active+clean; 284 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.3 MiB/s rd, 378 KiB/s wr, 10 op/s 2024-09-12T23:51:26.532 INFO:tasks.rados.rados.0.smithi136.stdout:3768: done (0 left) 2024-09-12T23:51:26.532 INFO:tasks.rados.rados.0.smithi136.stdout:3769: rollback oid 20 current snap is 373 2024-09-12T23:51:26.532 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 20 to 371 2024-09-12T23:51:26.532 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 10 current snap is 373 2024-09-12T23:51:26.532 INFO:tasks.rados.rados.0.smithi136.stdout:3770: seq_num 1187 ranges {2392064=688128} 2024-09-12T23:51:26.550 INFO:tasks.rados.rados.0.smithi136.stdout:3770: writing smithi13667081-10 from 2392064 to 3080192 tid 1 2024-09-12T23:51:26.551 INFO:tasks.rados.rados.0.smithi136.stdout:3771: copy_from oid 24 from oid 49 current snap is 373 2024-09-12T23:51:26.551 INFO:tasks.rados.rados.0.smithi136.stdout:3772: rollback oid 36 current snap is 373 2024-09-12T23:51:26.551 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 36 to 360 2024-09-12T23:51:26.551 INFO:tasks.rados.rados.0.smithi136.stdout:3773: snap_remove snap 363 2024-09-12T23:51:26.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:26 smithi181 bash[67587]: cluster 2024-09-12T23:51:25.589173+0000 mon.a (mon.0) 1394 : cluster [DBG] osdmap e817: 8 total, 8 up, 8 in 2024-09-12T23:51:26.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:26 smithi181 bash[67587]: cluster 2024-09-12T23:51:26.533543+0000 mon.a (mon.0) 1395 : cluster [DBG] osdmap e818: 8 total, 8 up, 8 in 2024-09-12T23:51:27.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:26 smithi136 bash[76371]: cluster 2024-09-12T23:51:25.589173+0000 mon.a (mon.0) 1394 : cluster [DBG] osdmap e817: 8 total, 8 up, 8 in 2024-09-12T23:51:27.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:26 smithi136 bash[76371]: cluster 2024-09-12T23:51:26.533543+0000 mon.a (mon.0) 1395 : cluster [DBG] osdmap e818: 8 total, 8 up, 8 in 2024-09-12T23:51:27.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:26 smithi136 bash[75164]: cluster 2024-09-12T23:51:25.589173+0000 mon.a (mon.0) 1394 : cluster [DBG] osdmap e817: 8 total, 8 up, 8 in 2024-09-12T23:51:27.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:26 smithi136 bash[75164]: cluster 2024-09-12T23:51:26.533543+0000 mon.a (mon.0) 1395 : cluster [DBG] osdmap e818: 8 total, 8 up, 8 in 2024-09-12T23:51:27.535 INFO:tasks.rados.rados.0.smithi136.stdout:3769: finishing rollback tid 0 to smithi13667081-20 2024-09-12T23:51:27.535 INFO:tasks.rados.rados.0.smithi136.stdout:3769: finishing rollback tid 1 to smithi13667081-20 2024-09-12T23:51:27.535 INFO:tasks.rados.rados.0.smithi136.stdout:3769: finishing rollback tid 2 to smithi13667081-20 2024-09-12T23:51:27.535 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 20 v 1665 (ObjNum 1111 snap 337 seq_num 1111) dirty exists 2024-09-12T23:51:27.535 INFO:tasks.rados.rados.0.smithi136.stdout:3771: finishing copy_from to smithi13667081-24 2024-09-12T23:51:27.536 INFO:tasks.rados.rados.0.smithi136.stdout:3771: got expected ENOENT (src dne) 2024-09-12T23:51:27.536 INFO:tasks.rados.rados.0.smithi136.stdout:3772: finishing rollback tid 1 to smithi13667081-36 2024-09-12T23:51:27.536 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 36 v 2359 (ObjNum 225 snap 0 seq_num 4243229072) dirty dne 2024-09-12T23:51:27.536 INFO:tasks.rados.rados.0.smithi136.stdout:3771: finishing copy_from racing read to smithi13667081-24 2024-09-12T23:51:27.536 INFO:tasks.rados.rados.0.smithi136.stdout:3770: finishing write tid 1 to smithi13667081-10 2024-09-12T23:51:27.536 INFO:tasks.rados.rados.0.smithi136.stdout:3770: finishing write tid 2 to smithi13667081-10 2024-09-12T23:51:27.536 INFO:tasks.rados.rados.0.smithi136.stdout:3770: finishing write tid 3 to smithi13667081-10 2024-09-12T23:51:27.536 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 10 v 2595 (ObjNum 1187 snap 373 seq_num 1187) dirty exists 2024-09-12T23:51:27.536 INFO:tasks.rados.rados.0.smithi136.stdout:3770: left oid 10 (ObjNum 1187 snap 373 seq_num 1187) 2024-09-12T23:51:27.536 INFO:tasks.rados.rados.0.smithi136.stdout:3769: done (4 left) 2024-09-12T23:51:27.536 INFO:tasks.rados.rados.0.smithi136.stdout:3770: done (3 left) 2024-09-12T23:51:27.536 INFO:tasks.rados.rados.0.smithi136.stdout:3771: done (2 left) 2024-09-12T23:51:27.536 INFO:tasks.rados.rados.0.smithi136.stdout:3772: done (1 left) 2024-09-12T23:51:27.537 INFO:tasks.rados.rados.0.smithi136.stdout:3773: done (0 left) 2024-09-12T23:51:27.537 INFO:tasks.rados.rados.0.smithi136.stdout:3774: copy_from oid 22 from oid 44 current snap is 373 2024-09-12T23:51:27.537 INFO:tasks.rados.rados.0.smithi136.stdout:3775: copy_from oid 12 from oid 2 current snap is 373 2024-09-12T23:51:27.538 INFO:tasks.rados.rados.0.smithi136.stdout:3776: read oid 43 snap -1 2024-09-12T23:51:27.538 INFO:tasks.rados.rados.0.smithi136.stdout:3776: expect (ObjNum 1104 snap 333 seq_num 1104) 2024-09-12T23:51:27.538 INFO:tasks.rados.rados.0.smithi136.stdout:3777: read oid 33 snap -1 2024-09-12T23:51:27.538 INFO:tasks.rados.rados.0.smithi136.stdout:3777: expect deleted 2024-09-12T23:51:27.538 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 31 current snap is 373 2024-09-12T23:51:27.538 INFO:tasks.rados.rados.0.smithi136.stdout:3778: seq_num 1188 ranges {3317760=704512} 2024-09-12T23:51:27.573 INFO:tasks.rados.rados.0.smithi136.stdout:3778: writing smithi13667081-31 from 3317760 to 4022272 tid 1 2024-09-12T23:51:27.573 INFO:tasks.rados.rados.0.smithi136.stdout:3779: rmattr oid 13 current snap is 373 2024-09-12T23:51:27.573 INFO:tasks.rados.rados.0.smithi136.stdout:3779: done (5 left) 2024-09-12T23:51:27.573 INFO:tasks.rados.rados.0.smithi136.stdout:3780: delete oid 21 current snap is 373 2024-09-12T23:51:27.588 INFO:tasks.rados.rados.0.smithi136.stdout:3776: expect (ObjNum 1104 snap 333 seq_num 1104) 2024-09-12T23:51:27.648 INFO:tasks.rados.rados.0.smithi136.stdout:3775: finishing copy_from racing read to smithi13667081-12 2024-09-12T23:51:27.648 INFO:tasks.rados.rados.0.smithi136.stdout:3775: finishing copy_from to smithi13667081-12 2024-09-12T23:51:27.649 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 12 v 2504 (ObjNum 1092 snap 330 seq_num 1092) dirty exists 2024-09-12T23:51:27.649 INFO:tasks.rados.rados.0.smithi136.stdout:3774: finishing copy_from racing read to smithi13667081-22 2024-09-12T23:51:27.649 INFO:tasks.rados.rados.0.smithi136.stdout:3778: finishing write tid 1 to smithi13667081-31 2024-09-12T23:51:27.649 INFO:tasks.rados.rados.0.smithi136.stdout:3775: done (5 left) 2024-09-12T23:51:27.649 INFO:tasks.rados.rados.0.smithi136.stdout:3776: done (4 left) 2024-09-12T23:51:27.649 INFO:tasks.rados.rados.0.smithi136.stdout:3777: done (3 left) 2024-09-12T23:51:27.649 INFO:tasks.rados.rados.0.smithi136.stdout:3780: done (2 left) 2024-09-12T23:51:27.649 INFO:tasks.rados.rados.0.smithi136.stdout:3781: rmattr oid 18 current snap is 373 2024-09-12T23:51:27.649 INFO:tasks.rados.rados.0.smithi136.stdout:3781: done (2 left) 2024-09-12T23:51:27.649 INFO:tasks.rados.rados.0.smithi136.stdout:3782: copy_from oid 4 from oid 13 current snap is 373 2024-09-12T23:51:27.649 INFO:tasks.rados.rados.0.smithi136.stdout:3778: finishing write tid 2 to smithi13667081-31 2024-09-12T23:51:27.650 INFO:tasks.rados.rados.0.smithi136.stdout:3778: finishing write tid 3 to smithi13667081-31 2024-09-12T23:51:27.650 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 31 v 2506 (ObjNum 1188 snap 373 seq_num 1188) dirty exists 2024-09-12T23:51:27.650 INFO:tasks.rados.rados.0.smithi136.stdout:3778: left oid 31 (ObjNum 1188 snap 373 seq_num 1188) 2024-09-12T23:51:27.650 INFO:tasks.rados.rados.0.smithi136.stdout:3774: finishing copy_from to smithi13667081-22 2024-09-12T23:51:27.650 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 22 v 2529 (ObjNum 1135 snap 345 seq_num 1135) dirty exists 2024-09-12T23:51:27.650 INFO:tasks.rados.rados.0.smithi136.stdout:3774: done (2 left) 2024-09-12T23:51:27.650 INFO:tasks.rados.rados.0.smithi136.stdout:3778: done (1 left) 2024-09-12T23:51:27.650 INFO:tasks.rados.rados.0.smithi136.stdout:3783: rollback oid 15 current snap is 373 2024-09-12T23:51:27.650 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 15 to 368 2024-09-12T23:51:27.650 INFO:tasks.rados.rados.0.smithi136.stdout:3784: delete oid 42 current snap is 373 2024-09-12T23:51:27.651 INFO:tasks.rados.rados.0.smithi136.stdout:3784: done (2 left) 2024-09-12T23:51:27.651 INFO:tasks.rados.rados.0.smithi136.stdout:3785: read oid 40 snap 357 2024-09-12T23:51:27.651 INFO:tasks.rados.rados.0.smithi136.stdout:3785: expect deleted 2024-09-12T23:51:27.651 INFO:tasks.rados.rados.0.smithi136.stdout:3786: setattr oid 43 current snap is 373 2024-09-12T23:51:27.652 INFO:tasks.rados.rados.0.smithi136.stdout:3783: finishing rollback tid 0 to smithi13667081-15 2024-09-12T23:51:27.653 INFO:tasks.rados.rados.0.smithi136.stdout:3783: finishing rollback tid 1 to smithi13667081-15 2024-09-12T23:51:27.653 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 15 v 2508 (ObjNum 1427291360 snap 32541 seq_num 304) dirty dne 2024-09-12T23:51:27.653 INFO:tasks.rados.rados.0.smithi136.stdout:3783: done (3 left) 2024-09-12T23:51:27.653 INFO:tasks.rados.rados.0.smithi136.stdout:3785: done (2 left) 2024-09-12T23:51:27.653 INFO:tasks.rados.rados.0.smithi136.stdout:3787: snap_create 2024-09-12T23:51:27.659 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 43 v 2532 (ObjNum 1104 snap 333 seq_num 1104) dirty exists 2024-09-12T23:51:27.668 INFO:tasks.rados.rados.0.smithi136.stdout:3782: finishing copy_from racing read to smithi13667081-4 2024-09-12T23:51:27.669 INFO:tasks.rados.rados.0.smithi136.stdout:3782: finishing copy_from to smithi13667081-4 2024-09-12T23:51:27.670 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 4 v 2692 (ObjNum 1163 snap 358 seq_num 1163) dirty exists 2024-09-12T23:51:27.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:27 smithi181 bash[67587]: cluster 2024-09-12T23:51:26.875340+0000 mgr.y (mgr.24661) 429 : cluster [DBG] pgmap v809: 121 pgs: 121 active+clean; 287 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.0 MiB/s rd, 724 KiB/s wr, 12 op/s 2024-09-12T23:51:27.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:27 smithi181 bash[67587]: audit 2024-09-12T23:51:27.499747+0000 mon.a (mon.0) 1396 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:51:27.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:27 smithi181 bash[67587]: cluster 2024-09-12T23:51:27.537250+0000 mon.a (mon.0) 1397 : cluster [DBG] osdmap e819: 8 total, 8 up, 8 in 2024-09-12T23:51:28.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:27 smithi136 bash[75164]: cluster 2024-09-12T23:51:26.875340+0000 mgr.y (mgr.24661) 429 : cluster [DBG] pgmap v809: 121 pgs: 121 active+clean; 287 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.0 MiB/s rd, 724 KiB/s wr, 12 op/s 2024-09-12T23:51:28.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:27 smithi136 bash[75164]: audit 2024-09-12T23:51:27.499747+0000 mon.a (mon.0) 1396 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:51:28.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:27 smithi136 bash[75164]: cluster 2024-09-12T23:51:27.537250+0000 mon.a (mon.0) 1397 : cluster [DBG] osdmap e819: 8 total, 8 up, 8 in 2024-09-12T23:51:28.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:27 smithi136 bash[76371]: cluster 2024-09-12T23:51:26.875340+0000 mgr.y (mgr.24661) 429 : cluster [DBG] pgmap v809: 121 pgs: 121 active+clean; 287 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.0 MiB/s rd, 724 KiB/s wr, 12 op/s 2024-09-12T23:51:28.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:27 smithi136 bash[76371]: audit 2024-09-12T23:51:27.499747+0000 mon.a (mon.0) 1396 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:51:28.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:27 smithi136 bash[76371]: cluster 2024-09-12T23:51:27.537250+0000 mon.a (mon.0) 1397 : cluster [DBG] osdmap e819: 8 total, 8 up, 8 in 2024-09-12T23:51:28.603 INFO:tasks.rados.rados.0.smithi136.stdout:3782: done (2 left) 2024-09-12T23:51:28.603 INFO:tasks.rados.rados.0.smithi136.stdout:3786: done (1 left) 2024-09-12T23:51:28.604 INFO:tasks.rados.rados.0.smithi136.stdout:3787: done (0 left) 2024-09-12T23:51:28.604 INFO:tasks.rados.rados.0.smithi136.stdout:3788: rmattr oid 41 current snap is 374 2024-09-12T23:51:28.604 INFO:tasks.rados.rados.0.smithi136.stdout:3788: done (0 left) 2024-09-12T23:51:28.604 INFO:tasks.rados.rados.0.smithi136.stdout:3789: read oid 41 snap -1 2024-09-12T23:51:28.604 INFO:tasks.rados.rados.0.smithi136.stdout:3789: expect (ObjNum 1174 snap 366 seq_num 1174) 2024-09-12T23:51:28.604 INFO:tasks.rados.rados.0.smithi136.stdout:3790: snap_remove snap 368 2024-09-12T23:51:29.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:28 smithi136 bash[75164]: audit 2024-09-12T23:51:28.065287+0000 mon.a (mon.0) 1398 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:51:29.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:28 smithi136 bash[75164]: audit 2024-09-12T23:51:28.066913+0000 mon.a (mon.0) 1399 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:51:29.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:28 smithi136 bash[75164]: audit 2024-09-12T23:51:28.075991+0000 mon.a (mon.0) 1400 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:51:29.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:28 smithi136 bash[76371]: audit 2024-09-12T23:51:28.065287+0000 mon.a (mon.0) 1398 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:51:29.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:28 smithi136 bash[76371]: audit 2024-09-12T23:51:28.066913+0000 mon.a (mon.0) 1399 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:51:29.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:28 smithi136 bash[76371]: audit 2024-09-12T23:51:28.075991+0000 mon.a (mon.0) 1400 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:51:29.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:28 smithi181 bash[67587]: audit 2024-09-12T23:51:28.065287+0000 mon.a (mon.0) 1398 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:51:29.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:28 smithi181 bash[67587]: audit 2024-09-12T23:51:28.066913+0000 mon.a (mon.0) 1399 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:51:29.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:28 smithi181 bash[67587]: audit 2024-09-12T23:51:28.075991+0000 mon.a (mon.0) 1400 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:51:29.628 INFO:tasks.rados.rados.0.smithi136.stdout:3789: expect (ObjNum 1174 snap 366 seq_num 1174) 2024-09-12T23:51:29.641 INFO:tasks.rados.rados.0.smithi136.stdout:3789: done (1 left) 2024-09-12T23:51:29.641 INFO:tasks.rados.rados.0.smithi136.stdout:3790: done (0 left) 2024-09-12T23:51:29.641 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 19 current snap is 374 2024-09-12T23:51:29.641 INFO:tasks.rados.rados.0.smithi136.stdout:3791: seq_num 1191 ranges {0=753664} 2024-09-12T23:51:29.654 INFO:tasks.rados.rados.0.smithi136.stdout:3791: writing smithi13667081-19 from 0 to 753664 tid 1 2024-09-12T23:51:29.654 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 2 current snap is 374 2024-09-12T23:51:29.655 INFO:tasks.rados.rados.0.smithi136.stdout:3792: seq_num 1192 ranges {3956736=770048} 2024-09-12T23:51:29.703 INFO:tasks.rados.rados.0.smithi136.stdout:3792: writing smithi13667081-2 from 3956736 to 4726784 tid 1 2024-09-12T23:51:29.704 INFO:tasks.rados.rados.0.smithi136.stdout:3793: read oid 40 snap 370 2024-09-12T23:51:29.704 INFO:tasks.rados.rados.0.smithi136.stdout:3793: expect deleted 2024-09-12T23:51:29.704 INFO:tasks.rados.rados.0.smithi136.stdout:3794: snap_create 2024-09-12T23:51:29.704 INFO:tasks.rados.rados.0.smithi136.stdout:3791: finishing write tid 1 to smithi13667081-19 2024-09-12T23:51:29.704 INFO:tasks.rados.rados.0.smithi136.stdout:3791: finishing write tid 2 to smithi13667081-19 2024-09-12T23:51:29.704 INFO:tasks.rados.rados.0.smithi136.stdout:3791: finishing write tid 3 to smithi13667081-19 2024-09-12T23:51:29.704 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 19 v 2493 (ObjNum 1191 snap 374 seq_num 1191) dirty exists 2024-09-12T23:51:29.704 INFO:tasks.rados.rados.0.smithi136.stdout:3791: left oid 19 (ObjNum 1191 snap 374 seq_num 1191) 2024-09-12T23:51:29.722 INFO:tasks.rados.rados.0.smithi136.stdout:3792: finishing write tid 1 to smithi13667081-2 2024-09-12T23:51:29.722 INFO:tasks.rados.rados.0.smithi136.stdout:3792: finishing write tid 2 to smithi13667081-2 2024-09-12T23:51:29.728 INFO:tasks.rados.rados.0.smithi136.stdout:3792: finishing write tid 3 to smithi13667081-2 2024-09-12T23:51:29.728 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 2 v 2699 (ObjNum 1192 snap 374 seq_num 1192) dirty exists 2024-09-12T23:51:29.728 INFO:tasks.rados.rados.0.smithi136.stdout:3792: left oid 2 (ObjNum 1192 snap 374 seq_num 1192) 2024-09-12T23:51:29.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:29 smithi136 bash[76371]: cluster 2024-09-12T23:51:28.605336+0000 mon.a (mon.0) 1401 : cluster [DBG] osdmap e820: 8 total, 8 up, 8 in 2024-09-12T23:51:29.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:29 smithi136 bash[76371]: cluster 2024-09-12T23:51:28.876088+0000 mgr.y (mgr.24661) 430 : cluster [DBG] pgmap v812: 121 pgs: 121 active+clean; 291 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 1.7 MiB/s rd, 1.7 MiB/s wr, 6 op/s 2024-09-12T23:51:29.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:51:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:51:29] "GET /metrics HTTP/1.1" 200 36441 "" "Prometheus/2.43.0" 2024-09-12T23:51:29.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:29 smithi136 bash[75164]: cluster 2024-09-12T23:51:28.605336+0000 mon.a (mon.0) 1401 : cluster [DBG] osdmap e820: 8 total, 8 up, 8 in 2024-09-12T23:51:29.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:29 smithi136 bash[75164]: cluster 2024-09-12T23:51:28.876088+0000 mgr.y (mgr.24661) 430 : cluster [DBG] pgmap v812: 121 pgs: 121 active+clean; 291 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 1.7 MiB/s rd, 1.7 MiB/s wr, 6 op/s 2024-09-12T23:51:30.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:29 smithi181 bash[67587]: cluster 2024-09-12T23:51:28.605336+0000 mon.a (mon.0) 1401 : cluster [DBG] osdmap e820: 8 total, 8 up, 8 in 2024-09-12T23:51:30.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:29 smithi181 bash[67587]: cluster 2024-09-12T23:51:28.876088+0000 mgr.y (mgr.24661) 430 : cluster [DBG] pgmap v812: 121 pgs: 121 active+clean; 291 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 1.7 MiB/s rd, 1.7 MiB/s wr, 6 op/s 2024-09-12T23:51:30.631 INFO:tasks.rados.rados.0.smithi136.stdout:3791: done (3 left) 2024-09-12T23:51:30.632 INFO:tasks.rados.rados.0.smithi136.stdout:3792: done (2 left) 2024-09-12T23:51:30.632 INFO:tasks.rados.rados.0.smithi136.stdout:3793: done (1 left) 2024-09-12T23:51:30.632 INFO:tasks.rados.rados.0.smithi136.stdout:3794: done (0 left) 2024-09-12T23:51:30.632 INFO:tasks.rados.rados.0.smithi136.stdout:3795: rollback oid 22 current snap is 375 2024-09-12T23:51:30.632 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 22 to 361 2024-09-12T23:51:30.632 INFO:tasks.rados.rados.0.smithi136.stdout:3796: snap_remove snap 367 2024-09-12T23:51:31.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:30 smithi136 bash[75164]: cluster 2024-09-12T23:51:29.626071+0000 mon.a (mon.0) 1402 : cluster [DBG] osdmap e821: 8 total, 8 up, 8 in 2024-09-12T23:51:31.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:30 smithi136 bash[76371]: cluster 2024-09-12T23:51:29.626071+0000 mon.a (mon.0) 1402 : cluster [DBG] osdmap e821: 8 total, 8 up, 8 in 2024-09-12T23:51:31.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:30 smithi181 bash[67587]: cluster 2024-09-12T23:51:29.626071+0000 mon.a (mon.0) 1402 : cluster [DBG] osdmap e821: 8 total, 8 up, 8 in 2024-09-12T23:51:31.539 INFO:tasks.rados.rados.0.smithi136.stdout:3796: done (1 left) 2024-09-12T23:51:31.539 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 50 current snap is 375 2024-09-12T23:51:31.539 INFO:tasks.rados.rados.0.smithi136.stdout:3797: seq_num 1193 ranges {3530752=786432} 2024-09-12T23:51:31.585 INFO:tasks.rados.rados.0.smithi136.stdout:3797: writing smithi13667081-50 from 3530752 to 4317184 tid 1 2024-09-12T23:51:31.585 INFO:tasks.rados.rados.0.smithi136.stdout:3798: delete oid 17 current snap is 375 2024-09-12T23:51:31.585 INFO:tasks.rados.rados.0.smithi136.stdout:3795: finishing rollback tid 0 to smithi13667081-22 2024-09-12T23:51:31.585 INFO:tasks.rados.rados.0.smithi136.stdout:3795: finishing rollback tid 1 to smithi13667081-22 2024-09-12T23:51:31.585 INFO:tasks.rados.rados.0.smithi136.stdout:3795: finishing rollback tid 2 to smithi13667081-22 2024-09-12T23:51:31.585 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 22 v 2532 (ObjNum 1101 snap 333 seq_num 1101) dirty exists 2024-09-12T23:51:31.591 INFO:tasks.rados.rados.0.smithi136.stdout:3795: done (2 left) 2024-09-12T23:51:31.591 INFO:tasks.rados.rados.0.smithi136.stdout:3798: done (1 left) 2024-09-12T23:51:31.591 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 14 current snap is 375 2024-09-12T23:51:31.591 INFO:tasks.rados.rados.0.smithi136.stdout:3799: seq_num 1195 ranges {2121728=417792,2539520=401408} 2024-09-12T23:51:31.615 INFO:tasks.rados.rados.0.smithi136.stdout:3799: writing smithi13667081-14 from 2121728 to 2539520 tid 1 2024-09-12T23:51:31.619 INFO:tasks.rados.rados.0.smithi136.stdout:3799: writing smithi13667081-14 from 2539520 to 2940928 tid 2 2024-09-12T23:51:31.619 INFO:tasks.rados.rados.0.smithi136.stdout:3800: read oid 44 snap -1 2024-09-12T23:51:31.620 INFO:tasks.rados.rados.0.smithi136.stdout:3800: expect (ObjNum 1135 snap 345 seq_num 1135) 2024-09-12T23:51:31.620 INFO:tasks.rados.rados.0.smithi136.stdout:3801: rollback oid 11 current snap is 375 2024-09-12T23:51:31.620 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 11 to 370 2024-09-12T23:51:31.620 INFO:tasks.rados.rados.0.smithi136.stdout:3797: finishing write tid 1 to smithi13667081-50 2024-09-12T23:51:31.620 INFO:tasks.rados.rados.0.smithi136.stdout:3797: finishing write tid 2 to smithi13667081-50 2024-09-12T23:51:31.620 INFO:tasks.rados.rados.0.smithi136.stdout:3797: finishing write tid 3 to smithi13667081-50 2024-09-12T23:51:31.620 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 50 v 2698 (ObjNum 1193 snap 375 seq_num 1193) dirty exists 2024-09-12T23:51:31.620 INFO:tasks.rados.rados.0.smithi136.stdout:3797: left oid 50 (ObjNum 1193 snap 375 seq_num 1193) 2024-09-12T23:51:31.620 INFO:tasks.rados.rados.0.smithi136.stdout:3797: done (3 left) 2024-09-12T23:51:31.621 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 13 current snap is 375 2024-09-12T23:51:31.621 INFO:tasks.rados.rados.0.smithi136.stdout:3802: seq_num 1196 ranges {2375680=434176,2809856=401408} 2024-09-12T23:51:31.644 INFO:tasks.rados.rados.0.smithi136.stdout:3802: writing smithi13667081-13 from 2375680 to 2809856 tid 1 2024-09-12T23:51:31.648 INFO:tasks.rados.rados.0.smithi136.stdout:3802: writing smithi13667081-13 from 2809856 to 3211264 tid 2 2024-09-12T23:51:31.648 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 16 current snap is 375 2024-09-12T23:51:31.648 INFO:tasks.rados.rados.0.smithi136.stdout:3803: seq_num 1197 ranges {696320=450560,1146880=401408} 2024-09-12T23:51:31.656 INFO:tasks.rados.rados.0.smithi136.stdout:3803: writing smithi13667081-16 from 696320 to 1146880 tid 1 2024-09-12T23:51:31.659 INFO:tasks.rados.rados.0.smithi136.stdout:3803: writing smithi13667081-16 from 1146880 to 1548288 tid 2 2024-09-12T23:51:31.659 INFO:tasks.rados.rados.0.smithi136.stdout:3804: rollback oid 10 current snap is 375 2024-09-12T23:51:31.660 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 10 to 359 2024-09-12T23:51:31.660 INFO:tasks.rados.rados.0.smithi136.stdout:3801: finishing rollback tid 1 to smithi13667081-11 2024-09-12T23:51:31.660 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 11 v 2808 (ObjNum 33 snap 0 seq_num 4210353496) dirty dne 2024-09-12T23:51:31.660 INFO:tasks.rados.rados.0.smithi136.stdout:3799: finishing write tid 1 to smithi13667081-14 2024-09-12T23:51:31.660 INFO:tasks.rados.rados.0.smithi136.stdout:3799: finishing write tid 2 to smithi13667081-14 2024-09-12T23:51:31.660 INFO:tasks.rados.rados.0.smithi136.stdout:3799: finishing write tid 3 to smithi13667081-14 2024-09-12T23:51:31.660 INFO:tasks.rados.rados.0.smithi136.stdout:3802: finishing write tid 1 to smithi13667081-13 2024-09-12T23:51:31.660 INFO:tasks.rados.rados.0.smithi136.stdout:3802: finishing write tid 2 to smithi13667081-13 2024-09-12T23:51:31.660 INFO:tasks.rados.rados.0.smithi136.stdout:3802: finishing write tid 3 to smithi13667081-13 2024-09-12T23:51:31.660 INFO:tasks.rados.rados.0.smithi136.stdout:3802: finishing write tid 4 to smithi13667081-13 2024-09-12T23:51:31.660 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 13 v 2701 (ObjNum 1196 snap 375 seq_num 1196) dirty exists 2024-09-12T23:51:31.660 INFO:tasks.rados.rados.0.smithi136.stdout:3802: left oid 13 (ObjNum 1196 snap 375 seq_num 1196) 2024-09-12T23:51:31.661 INFO:tasks.rados.rados.0.smithi136.stdout:3801: done (5 left) 2024-09-12T23:51:31.661 INFO:tasks.rados.rados.0.smithi136.stdout:3802: done (4 left) 2024-09-12T23:51:31.661 INFO:tasks.rados.rados.0.smithi136.stdout:3805: snap_create 2024-09-12T23:51:31.665 INFO:tasks.rados.rados.0.smithi136.stdout:3804: finishing rollback tid 0 to smithi13667081-10 2024-09-12T23:51:31.666 INFO:tasks.rados.rados.0.smithi136.stdout:3804: finishing rollback tid 1 to smithi13667081-10 2024-09-12T23:51:31.666 INFO:tasks.rados.rados.0.smithi136.stdout:3804: finishing rollback tid 2 to smithi13667081-10 2024-09-12T23:51:31.666 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 10 v 2598 (ObjNum 1145 snap 350 seq_num 1145) dirty exists 2024-09-12T23:51:31.669 INFO:tasks.rados.rados.0.smithi136.stdout:3800: expect (ObjNum 1135 snap 345 seq_num 1135) 2024-09-12T23:51:31.814 INFO:tasks.rados.rados.0.smithi136.stdout:3799: finishing write tid 4 to smithi13667081-14 2024-09-12T23:51:31.814 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 14 v 2535 (ObjNum 1195 snap 375 seq_num 1195) dirty exists 2024-09-12T23:51:31.814 INFO:tasks.rados.rados.0.smithi136.stdout:3799: left oid 14 (ObjNum 1195 snap 375 seq_num 1195) 2024-09-12T23:51:31.814 INFO:tasks.rados.rados.0.smithi136.stdout:3803: finishing write tid 1 to smithi13667081-16 2024-09-12T23:51:31.814 INFO:tasks.rados.rados.0.smithi136.stdout:3803: finishing write tid 2 to smithi13667081-16 2024-09-12T23:51:31.814 INFO:tasks.rados.rados.0.smithi136.stdout:3803: finishing write tid 3 to smithi13667081-16 2024-09-12T23:51:31.814 INFO:tasks.rados.rados.0.smithi136.stdout:3803: finishing write tid 4 to smithi13667081-16 2024-09-12T23:51:31.814 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 16 v 2538 (ObjNum 1197 snap 375 seq_num 1197) dirty exists 2024-09-12T23:51:31.815 INFO:tasks.rados.rados.0.smithi136.stdout:3803: left oid 16 (ObjNum 1197 snap 375 seq_num 1197) 2024-09-12T23:51:32.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:31 smithi136 bash[75164]: cluster 2024-09-12T23:51:30.634731+0000 mon.a (mon.0) 1403 : cluster [DBG] osdmap e822: 8 total, 8 up, 8 in 2024-09-12T23:51:32.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:31 smithi136 bash[75164]: cluster 2024-09-12T23:51:30.876901+0000 mgr.y (mgr.24661) 431 : cluster [DBG] pgmap v815: 121 pgs: 121 active+clean; 292 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 1.3 MiB/s wr, 2 op/s 2024-09-12T23:51:32.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:31 smithi136 bash[75164]: cluster 2024-09-12T23:51:31.540395+0000 mon.a (mon.0) 1404 : cluster [DBG] osdmap e823: 8 total, 8 up, 8 in 2024-09-12T23:51:32.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:31 smithi136 bash[76371]: cluster 2024-09-12T23:51:30.634731+0000 mon.a (mon.0) 1403 : cluster [DBG] osdmap e822: 8 total, 8 up, 8 in 2024-09-12T23:51:32.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:31 smithi136 bash[76371]: cluster 2024-09-12T23:51:30.876901+0000 mgr.y (mgr.24661) 431 : cluster [DBG] pgmap v815: 121 pgs: 121 active+clean; 292 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 1.3 MiB/s wr, 2 op/s 2024-09-12T23:51:32.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:31 smithi136 bash[76371]: cluster 2024-09-12T23:51:31.540395+0000 mon.a (mon.0) 1404 : cluster [DBG] osdmap e823: 8 total, 8 up, 8 in 2024-09-12T23:51:32.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:31 smithi181 bash[67587]: cluster 2024-09-12T23:51:30.634731+0000 mon.a (mon.0) 1403 : cluster [DBG] osdmap e822: 8 total, 8 up, 8 in 2024-09-12T23:51:32.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:31 smithi181 bash[67587]: cluster 2024-09-12T23:51:30.876901+0000 mgr.y (mgr.24661) 431 : cluster [DBG] pgmap v815: 121 pgs: 121 active+clean; 292 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 1.3 MiB/s wr, 2 op/s 2024-09-12T23:51:32.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:31 smithi181 bash[67587]: cluster 2024-09-12T23:51:31.540395+0000 mon.a (mon.0) 1404 : cluster [DBG] osdmap e823: 8 total, 8 up, 8 in 2024-09-12T23:51:32.596 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:51:32 smithi181 bash[95505]: ts=2024-09-12T23:51:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:51:32.650 INFO:tasks.rados.rados.0.smithi136.stdout:3799: done (4 left) 2024-09-12T23:51:32.650 INFO:tasks.rados.rados.0.smithi136.stdout:3800: done (3 left) 2024-09-12T23:51:32.650 INFO:tasks.rados.rados.0.smithi136.stdout:3803: done (2 left) 2024-09-12T23:51:32.650 INFO:tasks.rados.rados.0.smithi136.stdout:3804: done (1 left) 2024-09-12T23:51:32.650 INFO:tasks.rados.rados.0.smithi136.stdout:3805: done (0 left) 2024-09-12T23:51:32.650 INFO:tasks.rados.rados.0.smithi136.stdout:3806: rollback oid 6 current snap is 376 2024-09-12T23:51:32.650 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 6 to 369 2024-09-12T23:51:32.650 INFO:tasks.rados.rados.0.smithi136.stdout:3807: read oid 16 snap -1 2024-09-12T23:51:32.650 INFO:tasks.rados.rados.0.smithi136.stdout:3807: expect (ObjNum 1197 snap 375 seq_num 1197) 2024-09-12T23:51:32.650 INFO:tasks.rados.rados.0.smithi136.stdout:3808: snap_remove snap 357 2024-09-12T23:51:33.661 INFO:tasks.rados.rados.0.smithi136.stdout:3808: done (2 left) 2024-09-12T23:51:33.661 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 3 current snap is 376 2024-09-12T23:51:33.661 INFO:tasks.rados.rados.0.smithi136.stdout:3809: seq_num 1198 ranges {606208=466944,1073152=401408} 2024-09-12T23:51:33.668 INFO:tasks.rados.rados.0.smithi136.stdout:3809: writing smithi13667081-3 from 606208 to 1073152 tid 1 2024-09-12T23:51:33.672 INFO:tasks.rados.rados.0.smithi136.stdout:3809: writing smithi13667081-3 from 1073152 to 1474560 tid 2 2024-09-12T23:51:33.672 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 33 current snap is 376 2024-09-12T23:51:33.672 INFO:tasks.rados.rados.0.smithi136.stdout:3806: finishing rollback tid 1 to smithi13667081-6 2024-09-12T23:51:33.672 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 6 v 0 (ObjNum 177 snap 0 seq_num 4242937840) dirty dne 2024-09-12T23:51:33.673 INFO:tasks.rados.rados.0.smithi136.stdout:3807: expect (ObjNum 1197 snap 375 seq_num 1197) 2024-09-12T23:51:33.742 INFO:tasks.rados.rados.0.smithi136.stdout:3809: finishing write tid 1 to smithi13667081-3 2024-09-12T23:51:33.742 INFO:tasks.rados.rados.0.smithi136.stdout:3809: finishing write tid 2 to smithi13667081-3 2024-09-12T23:51:33.742 INFO:tasks.rados.rados.0.smithi136.stdout:3809: finishing write tid 3 to smithi13667081-3 2024-09-12T23:51:33.743 INFO:tasks.rados.rados.0.smithi136.stdout:3809: finishing write tid 4 to smithi13667081-3 2024-09-12T23:51:33.743 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 3 v 2511 (ObjNum 1198 snap 376 seq_num 1198) dirty exists 2024-09-12T23:51:33.743 INFO:tasks.rados.rados.0.smithi136.stdout:3809: left oid 3 (ObjNum 1198 snap 376 seq_num 1198) 2024-09-12T23:51:33.743 INFO:tasks.rados.rados.0.smithi136.stdout:3810: seq_num 1199 ranges {0=483328,483328=401408} 2024-09-12T23:51:33.745 INFO:tasks.rados.rados.0.smithi136.stdout:3810: writing smithi13667081-33 from 0 to 483328 tid 1 2024-09-12T23:51:33.747 INFO:tasks.rados.rados.0.smithi136.stdout:3810: writing smithi13667081-33 from 483328 to 884736 tid 2 2024-09-12T23:51:33.747 INFO:tasks.rados.rados.0.smithi136.stdout:3806: done (3 left) 2024-09-12T23:51:33.748 INFO:tasks.rados.rados.0.smithi136.stdout:3807: done (2 left) 2024-09-12T23:51:33.748 INFO:tasks.rados.rados.0.smithi136.stdout:3809: done (1 left) 2024-09-12T23:51:33.748 INFO:tasks.rados.rados.0.smithi136.stdout:3811: rollback oid 40 current snap is 376 2024-09-12T23:51:33.748 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 40 to 372 2024-09-12T23:51:33.748 INFO:tasks.rados.rados.0.smithi136.stdout:3812: snap_create 2024-09-12T23:51:33.751 INFO:tasks.rados.rados.0.smithi136.stdout:3810: finishing write tid 1 to smithi13667081-33 2024-09-12T23:51:33.752 INFO:tasks.rados.rados.0.smithi136.stdout:3810: finishing write tid 2 to smithi13667081-33 2024-09-12T23:51:33.752 INFO:tasks.rados.rados.0.smithi136.stdout:3811: finishing rollback tid 1 to smithi13667081-40 2024-09-12T23:51:33.752 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 40 v 0 (ObjNum 833 snap 0 seq_num 4244372080) dirty dne 2024-09-12T23:51:33.752 INFO:tasks.rados.rados.0.smithi136.stdout:3810: finishing write tid 3 to smithi13667081-33 2024-09-12T23:51:33.755 INFO:tasks.rados.rados.0.smithi136.stdout:3810: finishing write tid 4 to smithi13667081-33 2024-09-12T23:51:33.755 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 33 v 2702 (ObjNum 1199 snap 376 seq_num 1199) dirty exists 2024-09-12T23:51:33.755 INFO:tasks.rados.rados.0.smithi136.stdout:3810: left oid 33 (ObjNum 1199 snap 376 seq_num 1199) 2024-09-12T23:51:34.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:33 smithi136 bash[76371]: cluster 2024-09-12T23:51:32.652255+0000 mon.a (mon.0) 1405 : cluster [DBG] osdmap e824: 8 total, 8 up, 8 in 2024-09-12T23:51:34.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:33 smithi136 bash[76371]: cluster 2024-09-12T23:51:32.877599+0000 mgr.y (mgr.24661) 432 : cluster [DBG] pgmap v818: 121 pgs: 121 active+clean; 300 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 2.2 MiB/s wr, 12 op/s 2024-09-12T23:51:34.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:33 smithi136 bash[75164]: cluster 2024-09-12T23:51:32.652255+0000 mon.a (mon.0) 1405 : cluster [DBG] osdmap e824: 8 total, 8 up, 8 in 2024-09-12T23:51:34.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:33 smithi136 bash[75164]: cluster 2024-09-12T23:51:32.877599+0000 mgr.y (mgr.24661) 432 : cluster [DBG] pgmap v818: 121 pgs: 121 active+clean; 300 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 2.2 MiB/s wr, 12 op/s 2024-09-12T23:51:34.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:33 smithi181 bash[67587]: cluster 2024-09-12T23:51:32.652255+0000 mon.a (mon.0) 1405 : cluster [DBG] osdmap e824: 8 total, 8 up, 8 in 2024-09-12T23:51:34.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:33 smithi181 bash[67587]: cluster 2024-09-12T23:51:32.877599+0000 mgr.y (mgr.24661) 432 : cluster [DBG] pgmap v818: 121 pgs: 121 active+clean; 300 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 2.2 MiB/s wr, 12 op/s 2024-09-12T23:51:34.661 INFO:tasks.rados.rados.0.smithi136.stdout:3810: done (2 left) 2024-09-12T23:51:34.661 INFO:tasks.rados.rados.0.smithi136.stdout:3811: done (1 left) 2024-09-12T23:51:34.661 INFO:tasks.rados.rados.0.smithi136.stdout:3812: done (0 left) 2024-09-12T23:51:34.661 INFO:tasks.rados.rados.0.smithi136.stdout:3813: read oid 5 snap -1 2024-09-12T23:51:34.662 INFO:tasks.rados.rados.0.smithi136.stdout:3813: expect (ObjNum 1136 snap 345 seq_num 1136) 2024-09-12T23:51:34.662 INFO:tasks.rados.rados.0.smithi136.stdout:3814: copy_from oid 49 from oid 47 current snap is 377 2024-09-12T23:51:34.662 INFO:tasks.rados.rados.0.smithi136.stdout:3815: delete oid 34 current snap is 377 2024-09-12T23:51:34.677 INFO:tasks.rados.rados.0.smithi136.stdout:3815: done (2 left) 2024-09-12T23:51:34.677 INFO:tasks.rados.rados.0.smithi136.stdout:3816: delete oid 29 current snap is 377 2024-09-12T23:51:34.687 INFO:tasks.rados.rados.0.smithi136.stdout:3816: done (2 left) 2024-09-12T23:51:34.687 INFO:tasks.rados.rados.0.smithi136.stdout:3817: snap_create 2024-09-12T23:51:34.692 INFO:tasks.rados.rados.0.smithi136.stdout:3814: finishing copy_from racing read to smithi13667081-49 2024-09-12T23:51:34.694 INFO:tasks.rados.rados.0.smithi136.stdout:3814: finishing copy_from to smithi13667081-49 2024-09-12T23:51:34.694 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 49 v 2490 (ObjNum 1073 snap 324 seq_num 1073) dirty exists 2024-09-12T23:51:34.788 INFO:tasks.rados.rados.0.smithi136.stdout:3813: expect (ObjNum 1136 snap 345 seq_num 1136) 2024-09-12T23:51:35.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:34 smithi136 bash[75164]: cluster 2024-09-12T23:51:33.661300+0000 mon.a (mon.0) 1406 : cluster [DBG] osdmap e825: 8 total, 8 up, 8 in 2024-09-12T23:51:35.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:34 smithi136 bash[76371]: cluster 2024-09-12T23:51:33.661300+0000 mon.a (mon.0) 1406 : cluster [DBG] osdmap e825: 8 total, 8 up, 8 in 2024-09-12T23:51:35.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:34 smithi181 bash[67587]: cluster 2024-09-12T23:51:33.661300+0000 mon.a (mon.0) 1406 : cluster [DBG] osdmap e825: 8 total, 8 up, 8 in 2024-09-12T23:51:36.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:35 smithi136 bash[75164]: cluster 2024-09-12T23:51:34.663592+0000 mon.a (mon.0) 1407 : cluster [DBG] osdmap e826: 8 total, 8 up, 8 in 2024-09-12T23:51:36.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:35 smithi136 bash[75164]: cluster 2024-09-12T23:51:34.878282+0000 mgr.y (mgr.24661) 433 : cluster [DBG] pgmap v821: 121 pgs: 121 active+clean; 304 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 2.4 MiB/s wr, 13 op/s 2024-09-12T23:51:36.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:35 smithi136 bash[76371]: cluster 2024-09-12T23:51:34.663592+0000 mon.a (mon.0) 1407 : cluster [DBG] osdmap e826: 8 total, 8 up, 8 in 2024-09-12T23:51:36.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:35 smithi136 bash[76371]: cluster 2024-09-12T23:51:34.878282+0000 mgr.y (mgr.24661) 433 : cluster [DBG] pgmap v821: 121 pgs: 121 active+clean; 304 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 2.4 MiB/s wr, 13 op/s 2024-09-12T23:51:36.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:35 smithi181 bash[67587]: cluster 2024-09-12T23:51:34.663592+0000 mon.a (mon.0) 1407 : cluster [DBG] osdmap e826: 8 total, 8 up, 8 in 2024-09-12T23:51:36.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:35 smithi181 bash[67587]: cluster 2024-09-12T23:51:34.878282+0000 mgr.y (mgr.24661) 433 : cluster [DBG] pgmap v821: 121 pgs: 121 active+clean; 304 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 2.4 MiB/s wr, 13 op/s 2024-09-12T23:51:37.000 INFO:tasks.rados.rados.0.smithi136.stdout:3813: done (2 left) 2024-09-12T23:51:37.000 INFO:tasks.rados.rados.0.smithi136.stdout:3814: done (1 left) 2024-09-12T23:51:37.000 INFO:tasks.rados.rados.0.smithi136.stdout:3817: done (0 left) 2024-09-12T23:51:37.000 INFO:tasks.rados.rados.0.smithi136.stdout:3818: snap_remove snap 360 2024-09-12T23:51:37.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:36 smithi136 bash[75164]: cluster 2024-09-12T23:51:35.684055+0000 mon.a (mon.0) 1408 : cluster [DBG] osdmap e827: 8 total, 8 up, 8 in 2024-09-12T23:51:37.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:36 smithi136 bash[75164]: cluster 2024-09-12T23:51:36.540722+0000 mon.a (mon.0) 1409 : cluster [DBG] osdmap e828: 8 total, 8 up, 8 in 2024-09-12T23:51:37.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:36 smithi136 bash[76371]: cluster 2024-09-12T23:51:35.684055+0000 mon.a (mon.0) 1408 : cluster [DBG] osdmap e827: 8 total, 8 up, 8 in 2024-09-12T23:51:37.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:36 smithi136 bash[76371]: cluster 2024-09-12T23:51:36.540722+0000 mon.a (mon.0) 1409 : cluster [DBG] osdmap e828: 8 total, 8 up, 8 in 2024-09-12T23:51:37.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:36 smithi181 bash[67587]: cluster 2024-09-12T23:51:35.684055+0000 mon.a (mon.0) 1408 : cluster [DBG] osdmap e827: 8 total, 8 up, 8 in 2024-09-12T23:51:37.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:36 smithi181 bash[67587]: cluster 2024-09-12T23:51:36.540722+0000 mon.a (mon.0) 1409 : cluster [DBG] osdmap e828: 8 total, 8 up, 8 in 2024-09-12T23:51:37.690 INFO:tasks.rados.rados.0.smithi136.stdout:3818: done (0 left) 2024-09-12T23:51:37.690 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 7 current snap is 378 2024-09-12T23:51:37.690 INFO:tasks.rados.rados.0.smithi136.stdout:3819: seq_num 1202 ranges {0=540672,540672=401408} 2024-09-12T23:51:37.701 INFO:tasks.rados.rados.0.smithi136.stdout:3819: writing smithi13667081-7 from 0 to 540672 tid 1 2024-09-12T23:51:37.709 INFO:tasks.rados.rados.0.smithi136.stdout:3819: writing smithi13667081-7 from 540672 to 942080 tid 2 2024-09-12T23:51:37.709 INFO:tasks.rados.rados.0.smithi136.stdout:3820: snap_remove snap 371 2024-09-12T23:51:38.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:37 smithi136 bash[75164]: cluster 2024-09-12T23:51:36.879155+0000 mgr.y (mgr.24661) 434 : cluster [DBG] pgmap v824: 121 pgs: 121 active+clean; 308 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.5 MiB/s wr, 9 op/s 2024-09-12T23:51:38.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:37 smithi136 bash[76371]: cluster 2024-09-12T23:51:36.879155+0000 mgr.y (mgr.24661) 434 : cluster [DBG] pgmap v824: 121 pgs: 121 active+clean; 308 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.5 MiB/s wr, 9 op/s 2024-09-12T23:51:38.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:37 smithi181 bash[67587]: cluster 2024-09-12T23:51:36.879155+0000 mgr.y (mgr.24661) 434 : cluster [DBG] pgmap v824: 121 pgs: 121 active+clean; 308 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.5 MiB/s wr, 9 op/s 2024-09-12T23:51:38.700 INFO:tasks.rados.rados.0.smithi136.stdout:3819: finishing write tid 1 to smithi13667081-7 2024-09-12T23:51:38.700 INFO:tasks.rados.rados.0.smithi136.stdout:3819: finishing write tid 2 to smithi13667081-7 2024-09-12T23:51:38.700 INFO:tasks.rados.rados.0.smithi136.stdout:3819: finishing write tid 3 to smithi13667081-7 2024-09-12T23:51:38.700 INFO:tasks.rados.rados.0.smithi136.stdout:3819: finishing write tid 4 to smithi13667081-7 2024-09-12T23:51:38.700 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 7 v 1651 (ObjNum 1202 snap 378 seq_num 1202) dirty exists 2024-09-12T23:51:38.700 INFO:tasks.rados.rados.0.smithi136.stdout:3819: left oid 7 (ObjNum 1202 snap 378 seq_num 1202) 2024-09-12T23:51:38.701 INFO:tasks.rados.rados.0.smithi136.stdout:3819: done (1 left) 2024-09-12T23:51:38.701 INFO:tasks.rados.rados.0.smithi136.stdout:3820: done (0 left) 2024-09-12T23:51:38.701 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 28 current snap is 378 2024-09-12T23:51:38.701 INFO:tasks.rados.rados.0.smithi136.stdout:3821: seq_num 1203 ranges {655360=557056,1212416=401408} 2024-09-12T23:51:38.713 INFO:tasks.rados.rados.0.smithi136.stdout:3821: writing smithi13667081-28 from 655360 to 1212416 tid 1 2024-09-12T23:51:38.722 INFO:tasks.rados.rados.0.smithi136.stdout:3821: writing smithi13667081-28 from 1212416 to 1613824 tid 2 2024-09-12T23:51:38.722 INFO:tasks.rados.rados.0.smithi136.stdout:3822: read oid 38 snap 375 2024-09-12T23:51:38.722 INFO:tasks.rados.rados.0.smithi136.stdout:3822: expect (ObjNum 1134 snap 345 seq_num 1134) 2024-09-12T23:51:38.722 INFO:tasks.rados.rados.0.smithi136.stdout:3823: copy_from oid 22 from oid 46 current snap is 378 2024-09-12T23:51:38.722 INFO:tasks.rados.rados.0.smithi136.stdout:3824: read oid 42 snap 296 2024-09-12T23:51:38.722 INFO:tasks.rados.rados.0.smithi136.stdout:3824: expect (ObjNum 895 snap 288 seq_num 895) 2024-09-12T23:51:38.722 INFO:tasks.rados.rados.0.smithi136.stdout:3825: copy_from oid 48 from oid 24 current snap is 378 2024-09-12T23:51:38.722 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 32 current snap is 378 2024-09-12T23:51:38.723 INFO:tasks.rados.rados.0.smithi136.stdout:3826: seq_num 1204 ranges {19144704=573440,19718144=401408} 2024-09-12T23:51:38.857 INFO:tasks.rados.rados.0.smithi136.stdout:3826: writing smithi13667081-32 from 19144704 to 19718144 tid 1 2024-09-12T23:51:38.859 INFO:tasks.rados.rados.0.smithi136.stdout:3826: writing smithi13667081-32 from 19718144 to 20119552 tid 2 2024-09-12T23:51:38.859 INFO:tasks.rados.rados.0.smithi136.stdout:3827: snap_remove snap 355 2024-09-12T23:51:39.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:38 smithi136 bash[75164]: cluster 2024-09-12T23:51:37.692476+0000 mon.a (mon.0) 1410 : cluster [DBG] osdmap e829: 8 total, 8 up, 8 in 2024-09-12T23:51:39.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:38 smithi136 bash[76371]: cluster 2024-09-12T23:51:37.692476+0000 mon.a (mon.0) 1410 : cluster [DBG] osdmap e829: 8 total, 8 up, 8 in 2024-09-12T23:51:39.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:38 smithi181 bash[67587]: cluster 2024-09-12T23:51:37.692476+0000 mon.a (mon.0) 1410 : cluster [DBG] osdmap e829: 8 total, 8 up, 8 in 2024-09-12T23:51:39.703 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:51:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:51:39] "GET /metrics HTTP/1.1" 200 36441 "" "Prometheus/2.43.0" 2024-09-12T23:51:39.715 INFO:tasks.rados.rados.0.smithi136.stdout:3827: done (6 left) 2024-09-12T23:51:39.715 INFO:tasks.rados.rados.0.smithi136.stdout:3823: finishing copy_from to smithi13667081-22 2024-09-12T23:51:39.715 INFO:tasks.rados.rados.0.smithi136.stdout:3823: got expected ENOENT (src dne) 2024-09-12T23:51:39.715 INFO:tasks.rados.rados.0.smithi136.stdout:3823: finishing copy_from racing read to smithi13667081-22 2024-09-12T23:51:39.715 INFO:tasks.rados.rados.0.smithi136.stdout:3821: finishing write tid 1 to smithi13667081-28 2024-09-12T23:51:39.715 INFO:tasks.rados.rados.0.smithi136.stdout:3821: finishing write tid 2 to smithi13667081-28 2024-09-12T23:51:39.715 INFO:tasks.rados.rados.0.smithi136.stdout:3822: expect (ObjNum 1134 snap 345 seq_num 1134) 2024-09-12T23:51:39.984 INFO:tasks.rados.rados.0.smithi136.stdout:3821: finishing write tid 3 to smithi13667081-28 2024-09-12T23:51:39.984 INFO:tasks.rados.rados.0.smithi136.stdout:3821: finishing write tid 4 to smithi13667081-28 2024-09-12T23:51:39.985 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 28 v 2541 (ObjNum 1203 snap 378 seq_num 1203) dirty exists 2024-09-12T23:51:39.985 INFO:tasks.rados.rados.0.smithi136.stdout:3821: left oid 28 (ObjNum 1203 snap 378 seq_num 1203) 2024-09-12T23:51:39.985 INFO:tasks.rados.rados.0.smithi136.stdout:3824: expect (ObjNum 895 snap 288 seq_num 895) 2024-09-12T23:51:40.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:39 smithi136 bash[75164]: cluster 2024-09-12T23:51:38.702496+0000 mon.a (mon.0) 1411 : cluster [DBG] osdmap e830: 8 total, 8 up, 8 in 2024-09-12T23:51:40.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:39 smithi136 bash[75164]: audit 2024-09-12T23:51:38.755150+0000 mon.a (mon.0) 1412 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:51:40.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:39 smithi136 bash[75164]: cluster 2024-09-12T23:51:38.879877+0000 mgr.y (mgr.24661) 435 : cluster [DBG] pgmap v827: 121 pgs: 121 active+clean; 308 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 1.1 MiB/s wr, 7 op/s 2024-09-12T23:51:40.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:39 smithi136 bash[76371]: cluster 2024-09-12T23:51:38.702496+0000 mon.a (mon.0) 1411 : cluster [DBG] osdmap e830: 8 total, 8 up, 8 in 2024-09-12T23:51:40.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:39 smithi136 bash[76371]: audit 2024-09-12T23:51:38.755150+0000 mon.a (mon.0) 1412 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:51:40.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:39 smithi136 bash[76371]: cluster 2024-09-12T23:51:38.879877+0000 mgr.y (mgr.24661) 435 : cluster [DBG] pgmap v827: 121 pgs: 121 active+clean; 308 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 1.1 MiB/s wr, 7 op/s 2024-09-12T23:51:40.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:39 smithi181 bash[67587]: cluster 2024-09-12T23:51:38.702496+0000 mon.a (mon.0) 1411 : cluster [DBG] osdmap e830: 8 total, 8 up, 8 in 2024-09-12T23:51:40.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:39 smithi181 bash[67587]: audit 2024-09-12T23:51:38.755150+0000 mon.a (mon.0) 1412 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:51:40.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:39 smithi181 bash[67587]: cluster 2024-09-12T23:51:38.879877+0000 mgr.y (mgr.24661) 435 : cluster [DBG] pgmap v827: 121 pgs: 121 active+clean; 308 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 1.1 MiB/s wr, 7 op/s 2024-09-12T23:51:41.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:40 smithi136 bash[76371]: cluster 2024-09-12T23:51:39.715103+0000 mon.a (mon.0) 1413 : cluster [DBG] osdmap e831: 8 total, 8 up, 8 in 2024-09-12T23:51:41.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:40 smithi136 bash[75164]: cluster 2024-09-12T23:51:39.715103+0000 mon.a (mon.0) 1413 : cluster [DBG] osdmap e831: 8 total, 8 up, 8 in 2024-09-12T23:51:41.026 INFO:tasks.rados.rados.0.smithi136.stdout:3825: finishing copy_from racing read to smithi13667081-48 2024-09-12T23:51:41.026 INFO:tasks.rados.rados.0.smithi136.stdout:3825: finishing copy_from to smithi13667081-48 2024-09-12T23:51:41.026 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 48 v 2481 (ObjNum 1081 snap 326 seq_num 1081) dirty exists 2024-09-12T23:51:41.027 INFO:tasks.rados.rados.0.smithi136.stdout:3826: finishing write tid 1 to smithi13667081-32 2024-09-12T23:51:41.027 INFO:tasks.rados.rados.0.smithi136.stdout:3826: finishing write tid 2 to smithi13667081-32 2024-09-12T23:51:41.027 INFO:tasks.rados.rados.0.smithi136.stdout:3826: finishing write tid 3 to smithi13667081-32 2024-09-12T23:51:41.027 INFO:tasks.rados.rados.0.smithi136.stdout:3826: finishing write tid 4 to smithi13667081-32 2024-09-12T23:51:41.027 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 32 v 2695 (ObjNum 1204 snap 378 seq_num 1204) dirty exists 2024-09-12T23:51:41.027 INFO:tasks.rados.rados.0.smithi136.stdout:3826: left oid 32 (ObjNum 1204 snap 378 seq_num 1204) 2024-09-12T23:51:41.027 INFO:tasks.rados.rados.0.smithi136.stdout:3828: read oid 16 snap 376 2024-09-12T23:51:41.027 INFO:tasks.rados.rados.0.smithi136.stdout:3828: expect (ObjNum 1197 snap 375 seq_num 1197) 2024-09-12T23:51:41.027 INFO:tasks.rados.rados.0.smithi136.stdout:3821: done (6 left) 2024-09-12T23:51:41.027 INFO:tasks.rados.rados.0.smithi136.stdout:3822: done (5 left) 2024-09-12T23:51:41.027 INFO:tasks.rados.rados.0.smithi136.stdout:3823: done (4 left) 2024-09-12T23:51:41.027 INFO:tasks.rados.rados.0.smithi136.stdout:3824: done (3 left) 2024-09-12T23:51:41.027 INFO:tasks.rados.rados.0.smithi136.stdout:3825: done (2 left) 2024-09-12T23:51:41.027 INFO:tasks.rados.rados.0.smithi136.stdout:3826: done (1 left) 2024-09-12T23:51:41.028 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 50 current snap is 378 2024-09-12T23:51:41.028 INFO:tasks.rados.rados.0.smithi136.stdout:3829: seq_num 1205 ranges {4317184=589824,4907008=401408} 2024-09-12T23:51:41.052 INFO:tasks.rados.rados.0.smithi136.stdout:3829: writing smithi13667081-50 from 4317184 to 4907008 tid 1 2024-09-12T23:51:41.055 INFO:tasks.rados.rados.0.smithi136.stdout:3829: writing smithi13667081-50 from 4907008 to 5308416 tid 2 2024-09-12T23:51:41.055 INFO:tasks.rados.rados.0.smithi136.stdout:3830: snap_remove snap 358 2024-09-12T23:51:41.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:40 smithi181 bash[67587]: cluster 2024-09-12T23:51:39.715103+0000 mon.a (mon.0) 1413 : cluster [DBG] osdmap e831: 8 total, 8 up, 8 in 2024-09-12T23:51:41.547 INFO:tasks.rados.rados.0.smithi136.stdout:3828: expect (ObjNum 1197 snap 375 seq_num 1197) 2024-09-12T23:51:41.613 INFO:tasks.rados.rados.0.smithi136.stdout:3829: finishing write tid 1 to smithi13667081-50 2024-09-12T23:51:41.613 INFO:tasks.rados.rados.0.smithi136.stdout:3829: finishing write tid 2 to smithi13667081-50 2024-09-12T23:51:41.613 INFO:tasks.rados.rados.0.smithi136.stdout:3829: finishing write tid 3 to smithi13667081-50 2024-09-12T23:51:41.614 INFO:tasks.rados.rados.0.smithi136.stdout:3829: finishing write tid 4 to smithi13667081-50 2024-09-12T23:51:41.614 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 50 v 2704 (ObjNum 1205 snap 378 seq_num 1205) dirty exists 2024-09-12T23:51:41.614 INFO:tasks.rados.rados.0.smithi136.stdout:3829: left oid 50 (ObjNum 1205 snap 378 seq_num 1205) 2024-09-12T23:51:41.614 INFO:tasks.rados.rados.0.smithi136.stdout:3828: done (2 left) 2024-09-12T23:51:41.614 INFO:tasks.rados.rados.0.smithi136.stdout:3829: done (1 left) 2024-09-12T23:51:41.614 INFO:tasks.rados.rados.0.smithi136.stdout:3830: done (0 left) 2024-09-12T23:51:41.614 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 18 current snap is 378 2024-09-12T23:51:41.614 INFO:tasks.rados.rados.0.smithi136.stdout:3831: seq_num 1206 ranges {0=606208,606208=401408} 2024-09-12T23:51:41.616 INFO:tasks.rados.rados.0.smithi136.stdout:3831: writing smithi13667081-18 from 0 to 606208 tid 1 2024-09-12T23:51:41.619 INFO:tasks.rados.rados.0.smithi136.stdout:3831: writing smithi13667081-18 from 606208 to 1007616 tid 2 2024-09-12T23:51:41.619 INFO:tasks.rados.rados.0.smithi136.stdout:3832: snap_create 2024-09-12T23:51:41.625 INFO:tasks.rados.rados.0.smithi136.stdout:3831: finishing write tid 1 to smithi13667081-18 2024-09-12T23:51:41.626 INFO:tasks.rados.rados.0.smithi136.stdout:3831: finishing write tid 2 to smithi13667081-18 2024-09-12T23:51:41.626 INFO:tasks.rados.rados.0.smithi136.stdout:3831: finishing write tid 3 to smithi13667081-18 2024-09-12T23:51:41.630 INFO:tasks.rados.rados.0.smithi136.stdout:3831: finishing write tid 4 to smithi13667081-18 2024-09-12T23:51:41.631 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 18 v 1654 (ObjNum 1206 snap 378 seq_num 1206) dirty exists 2024-09-12T23:51:41.631 INFO:tasks.rados.rados.0.smithi136.stdout:3831: left oid 18 (ObjNum 1206 snap 378 seq_num 1206) 2024-09-12T23:51:42.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:41 smithi136 bash[75164]: cluster 2024-09-12T23:51:40.880546+0000 mgr.y (mgr.24661) 436 : cluster [DBG] pgmap v829: 121 pgs: 121 active+clean; 309 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-09-12T23:51:42.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:41 smithi136 bash[75164]: cluster 2024-09-12T23:51:41.541238+0000 mon.a (mon.0) 1414 : cluster [DBG] osdmap e832: 8 total, 8 up, 8 in 2024-09-12T23:51:42.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:41 smithi136 bash[76371]: cluster 2024-09-12T23:51:40.880546+0000 mgr.y (mgr.24661) 436 : cluster [DBG] pgmap v829: 121 pgs: 121 active+clean; 309 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-09-12T23:51:42.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:41 smithi136 bash[76371]: cluster 2024-09-12T23:51:41.541238+0000 mon.a (mon.0) 1414 : cluster [DBG] osdmap e832: 8 total, 8 up, 8 in 2024-09-12T23:51:42.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:41 smithi181 bash[67587]: cluster 2024-09-12T23:51:40.880546+0000 mgr.y (mgr.24661) 436 : cluster [DBG] pgmap v829: 121 pgs: 121 active+clean; 309 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-09-12T23:51:42.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:41 smithi181 bash[67587]: cluster 2024-09-12T23:51:41.541238+0000 mon.a (mon.0) 1414 : cluster [DBG] osdmap e832: 8 total, 8 up, 8 in 2024-09-12T23:51:42.542 INFO:tasks.rados.rados.0.smithi136.stdout:3831: done (1 left) 2024-09-12T23:51:42.542 INFO:tasks.rados.rados.0.smithi136.stdout:3832: done (0 left) 2024-09-12T23:51:42.542 INFO:tasks.rados.rados.0.smithi136.stdout:3833: snap_create 2024-09-12T23:51:42.596 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:51:42 smithi181 bash[95505]: ts=2024-09-12T23:51:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:51:43.557 INFO:tasks.rados.rados.0.smithi136.stdout:3833: done (0 left) 2024-09-12T23:51:43.557 INFO:tasks.rados.rados.0.smithi136.stdout:3834: rollback oid 29 current snap is 380 2024-09-12T23:51:43.557 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 29 to 361 2024-09-12T23:51:43.557 INFO:tasks.rados.rados.0.smithi136.stdout:3835: rmattr oid 15 current snap is 380 2024-09-12T23:51:43.558 INFO:tasks.rados.rados.0.smithi136.stdout:3835: done (1 left) 2024-09-12T23:51:43.558 INFO:tasks.rados.rados.0.smithi136.stdout:3836: rmattr oid 40 current snap is 380 2024-09-12T23:51:43.558 INFO:tasks.rados.rados.0.smithi136.stdout:3836: done (1 left) 2024-09-12T23:51:43.558 INFO:tasks.rados.rados.0.smithi136.stdout:3837: snap_remove snap 372 2024-09-12T23:51:43.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:43 smithi181 bash[67587]: cluster 2024-09-12T23:51:42.544326+0000 mon.a (mon.0) 1415 : cluster [DBG] osdmap e833: 8 total, 8 up, 8 in 2024-09-12T23:51:43.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:43 smithi181 bash[67587]: cluster 2024-09-12T23:51:42.881405+0000 mgr.y (mgr.24661) 437 : cluster [DBG] pgmap v832: 121 pgs: 121 active+clean; 316 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.9 MiB/s wr, 5 op/s 2024-09-12T23:51:44.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:43 smithi136 bash[75164]: cluster 2024-09-12T23:51:42.544326+0000 mon.a (mon.0) 1415 : cluster [DBG] osdmap e833: 8 total, 8 up, 8 in 2024-09-12T23:51:44.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:43 smithi136 bash[75164]: cluster 2024-09-12T23:51:42.881405+0000 mgr.y (mgr.24661) 437 : cluster [DBG] pgmap v832: 121 pgs: 121 active+clean; 316 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.9 MiB/s wr, 5 op/s 2024-09-12T23:51:44.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:43 smithi136 bash[76371]: cluster 2024-09-12T23:51:42.544326+0000 mon.a (mon.0) 1415 : cluster [DBG] osdmap e833: 8 total, 8 up, 8 in 2024-09-12T23:51:44.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:43 smithi136 bash[76371]: cluster 2024-09-12T23:51:42.881405+0000 mgr.y (mgr.24661) 437 : cluster [DBG] pgmap v832: 121 pgs: 121 active+clean; 316 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.9 MiB/s wr, 5 op/s 2024-09-12T23:51:44.557 INFO:tasks.rados.rados.0.smithi136.stdout:3834: finishing rollback tid 1 to smithi13667081-29 2024-09-12T23:51:44.557 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 29 v 2693 (ObjNum 33 snap 0 seq_num 1442768144) dirty dne 2024-09-12T23:51:44.557 INFO:tasks.rados.rados.0.smithi136.stdout:3834: done (1 left) 2024-09-12T23:51:44.557 INFO:tasks.rados.rados.0.smithi136.stdout:3837: done (0 left) 2024-09-12T23:51:44.557 INFO:tasks.rados.rados.0.smithi136.stdout:3838: delete oid 27 current snap is 380 2024-09-12T23:51:44.572 INFO:tasks.rados.rados.0.smithi136.stdout:3838: done (0 left) 2024-09-12T23:51:44.572 INFO:tasks.rados.rados.0.smithi136.stdout:3839: snap_create 2024-09-12T23:51:44.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:44 smithi181 bash[67587]: cluster 2024-09-12T23:51:43.559553+0000 mon.a (mon.0) 1416 : cluster [DBG] osdmap e834: 8 total, 8 up, 8 in 2024-09-12T23:51:45.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:44 smithi136 bash[75164]: cluster 2024-09-12T23:51:43.559553+0000 mon.a (mon.0) 1416 : cluster [DBG] osdmap e834: 8 total, 8 up, 8 in 2024-09-12T23:51:45.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:44 smithi136 bash[76371]: cluster 2024-09-12T23:51:43.559553+0000 mon.a (mon.0) 1416 : cluster [DBG] osdmap e834: 8 total, 8 up, 8 in 2024-09-12T23:51:45.592 INFO:tasks.rados.rados.0.smithi136.stdout:3839: done (0 left) 2024-09-12T23:51:45.592 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 19 current snap is 381 2024-09-12T23:51:45.592 INFO:tasks.rados.rados.0.smithi136.stdout:3840: seq_num 1208 ranges {753664=638976,1392640=401408} 2024-09-12T23:51:45.600 INFO:tasks.rados.rados.0.smithi136.stdout:3840: writing smithi13667081-19 from 753664 to 1392640 tid 1 2024-09-12T23:51:45.608 INFO:tasks.rados.rados.0.smithi136.stdout:3840: writing smithi13667081-19 from 1392640 to 1794048 tid 2 2024-09-12T23:51:45.608 INFO:tasks.rados.rados.0.smithi136.stdout:3841: copy_from oid 20 from oid 49 current snap is 381 2024-09-12T23:51:45.608 INFO:tasks.rados.rados.0.smithi136.stdout:3842: snap_remove snap 370 2024-09-12T23:51:45.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:45 smithi181 bash[67587]: cluster 2024-09-12T23:51:44.558739+0000 mon.a (mon.0) 1417 : cluster [DBG] osdmap e835: 8 total, 8 up, 8 in 2024-09-12T23:51:45.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:45 smithi181 bash[67587]: cluster 2024-09-12T23:51:44.882191+0000 mgr.y (mgr.24661) 438 : cluster [DBG] pgmap v835: 121 pgs: 121 active+clean; 317 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 8.4 MiB/s rd, 2.0 MiB/s wr, 5 op/s 2024-09-12T23:51:46.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:45 smithi136 bash[75164]: cluster 2024-09-12T23:51:44.558739+0000 mon.a (mon.0) 1417 : cluster [DBG] osdmap e835: 8 total, 8 up, 8 in 2024-09-12T23:51:46.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:45 smithi136 bash[75164]: cluster 2024-09-12T23:51:44.882191+0000 mgr.y (mgr.24661) 438 : cluster [DBG] pgmap v835: 121 pgs: 121 active+clean; 317 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 8.4 MiB/s rd, 2.0 MiB/s wr, 5 op/s 2024-09-12T23:51:46.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:45 smithi136 bash[76371]: cluster 2024-09-12T23:51:44.558739+0000 mon.a (mon.0) 1417 : cluster [DBG] osdmap e835: 8 total, 8 up, 8 in 2024-09-12T23:51:46.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:45 smithi136 bash[76371]: cluster 2024-09-12T23:51:44.882191+0000 mgr.y (mgr.24661) 438 : cluster [DBG] pgmap v835: 121 pgs: 121 active+clean; 317 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 8.4 MiB/s rd, 2.0 MiB/s wr, 5 op/s 2024-09-12T23:51:46.544 INFO:tasks.rados.rados.0.smithi136.stdout:3840: finishing write tid 1 to smithi13667081-19 2024-09-12T23:51:46.544 INFO:tasks.rados.rados.0.smithi136.stdout:3840: finishing write tid 2 to smithi13667081-19 2024-09-12T23:51:46.544 INFO:tasks.rados.rados.0.smithi136.stdout:3840: finishing write tid 3 to smithi13667081-19 2024-09-12T23:51:46.544 INFO:tasks.rados.rados.0.smithi136.stdout:3840: finishing write tid 4 to smithi13667081-19 2024-09-12T23:51:46.544 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 19 v 2496 (ObjNum 1208 snap 381 seq_num 1208) dirty exists 2024-09-12T23:51:46.544 INFO:tasks.rados.rados.0.smithi136.stdout:3840: left oid 19 (ObjNum 1208 snap 381 seq_num 1208) 2024-09-12T23:51:46.544 INFO:tasks.rados.rados.0.smithi136.stdout:3841: finishing copy_from racing read to smithi13667081-20 2024-09-12T23:51:46.544 INFO:tasks.rados.rados.0.smithi136.stdout:3841: finishing copy_from to smithi13667081-20 2024-09-12T23:51:46.544 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 20 v 2490 (ObjNum 1073 snap 324 seq_num 1073) dirty exists 2024-09-12T23:51:46.544 INFO:tasks.rados.rados.0.smithi136.stdout:3840: done (2 left) 2024-09-12T23:51:46.545 INFO:tasks.rados.rados.0.smithi136.stdout:3841: done (1 left) 2024-09-12T23:51:46.545 INFO:tasks.rados.rados.0.smithi136.stdout:3842: done (0 left) 2024-09-12T23:51:46.545 INFO:tasks.rados.rados.0.smithi136.stdout:3843: rollback oid 48 current snap is 381 2024-09-12T23:51:46.545 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 48 to 350 2024-09-12T23:51:46.545 INFO:tasks.rados.rados.0.smithi136.stdout:3844: copy_from oid 27 from oid 45 current snap is 381 2024-09-12T23:51:46.545 INFO:tasks.rados.rados.0.smithi136.stdout:3845: delete oid 39 current snap is 381 2024-09-12T23:51:46.572 INFO:tasks.rados.rados.0.smithi136.stdout:3843: finishing rollback tid 0 to smithi13667081-48 2024-09-12T23:51:46.572 INFO:tasks.rados.rados.0.smithi136.stdout:3843: finishing rollback tid 1 to smithi13667081-48 2024-09-12T23:51:46.572 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 48 v 2543 (ObjNum 33 snap 0 seq_num 4244564992) dirty dne 2024-09-12T23:51:46.580 INFO:tasks.rados.rados.0.smithi136.stdout:3843: done (2 left) 2024-09-12T23:51:46.580 INFO:tasks.rados.rados.0.smithi136.stdout:3845: done (1 left) 2024-09-12T23:51:46.580 INFO:tasks.rados.rados.0.smithi136.stdout:3846: rollback oid 43 current snap is 381 2024-09-12T23:51:46.580 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 43 to 378 2024-09-12T23:51:46.580 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 14 current snap is 381 2024-09-12T23:51:46.580 INFO:tasks.rados.rados.0.smithi136.stdout:3847: seq_num 1210 ranges {2940928=671744,3612672=401408} 2024-09-12T23:51:46.623 INFO:tasks.rados.rados.0.smithi136.stdout:3847: writing smithi13667081-14 from 2940928 to 3612672 tid 1 2024-09-12T23:51:46.627 INFO:tasks.rados.rados.0.smithi136.stdout:3847: writing smithi13667081-14 from 3612672 to 4014080 tid 2 2024-09-12T23:51:46.628 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 9 current snap is 381 2024-09-12T23:51:46.628 INFO:tasks.rados.rados.0.smithi136.stdout:3844: finishing copy_from racing read to smithi13667081-27 2024-09-12T23:51:46.628 INFO:tasks.rados.rados.0.smithi136.stdout:3844: finishing copy_from to smithi13667081-27 2024-09-12T23:51:46.628 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 27 v 2370 (ObjNum 1178 snap 369 seq_num 1178) dirty exists 2024-09-12T23:51:46.628 INFO:tasks.rados.rados.0.smithi136.stdout:3846: finishing rollback tid 0 to smithi13667081-43 2024-09-12T23:51:46.628 INFO:tasks.rados.rados.0.smithi136.stdout:3846: finishing rollback tid 1 to smithi13667081-43 2024-09-12T23:51:46.628 INFO:tasks.rados.rados.0.smithi136.stdout:3846: finishing rollback tid 2 to smithi13667081-43 2024-09-12T23:51:46.628 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 43 v 2546 (ObjNum 1104 snap 333 seq_num 1104) dirty exists 2024-09-12T23:51:46.628 INFO:tasks.rados.rados.0.smithi136.stdout:3848: seq_num 1211 ranges {3514368=688128,4202496=401408} 2024-09-12T23:51:46.668 INFO:tasks.rados.rados.0.smithi136.stdout:3848: writing smithi13667081-9 from 3514368 to 4202496 tid 1 2024-09-12T23:51:46.671 INFO:tasks.rados.rados.0.smithi136.stdout:3848: writing smithi13667081-9 from 4202496 to 4603904 tid 2 2024-09-12T23:51:46.672 INFO:tasks.rados.rados.0.smithi136.stdout:3844: done (3 left) 2024-09-12T23:51:46.672 INFO:tasks.rados.rados.0.smithi136.stdout:3846: done (2 left) 2024-09-12T23:51:46.672 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 30 current snap is 381 2024-09-12T23:51:46.672 INFO:tasks.rados.rados.0.smithi136.stdout:3849: seq_num 1212 ranges {0=704512,704512=401408} 2024-09-12T23:51:46.678 INFO:tasks.rados.rados.0.smithi136.stdout:3849: writing smithi13667081-30 from 0 to 704512 tid 1 2024-09-12T23:51:46.681 INFO:tasks.rados.rados.0.smithi136.stdout:3849: writing smithi13667081-30 from 704512 to 1105920 tid 2 2024-09-12T23:51:46.681 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 44 current snap is 381 2024-09-12T23:51:46.681 INFO:tasks.rados.rados.0.smithi136.stdout:3847: finishing write tid 1 to smithi13667081-14 2024-09-12T23:51:46.682 INFO:tasks.rados.rados.0.smithi136.stdout:3847: finishing write tid 2 to smithi13667081-14 2024-09-12T23:51:46.682 INFO:tasks.rados.rados.0.smithi136.stdout:3847: finishing write tid 3 to smithi13667081-14 2024-09-12T23:51:46.682 INFO:tasks.rados.rados.0.smithi136.stdout:3847: finishing write tid 4 to smithi13667081-14 2024-09-12T23:51:46.682 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 14 v 2549 (ObjNum 1210 snap 381 seq_num 1210) dirty exists 2024-09-12T23:51:46.682 INFO:tasks.rados.rados.0.smithi136.stdout:3847: left oid 14 (ObjNum 1210 snap 381 seq_num 1210) 2024-09-12T23:51:46.682 INFO:tasks.rados.rados.0.smithi136.stdout:3848: finishing write tid 1 to smithi13667081-9 2024-09-12T23:51:46.682 INFO:tasks.rados.rados.0.smithi136.stdout:3848: finishing write tid 2 to smithi13667081-9 2024-09-12T23:51:46.682 INFO:tasks.rados.rados.0.smithi136.stdout:3848: finishing write tid 3 to smithi13667081-9 2024-09-12T23:51:46.682 INFO:tasks.rados.rados.0.smithi136.stdout:3848: finishing write tid 4 to smithi13667081-9 2024-09-12T23:51:46.682 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 9 v 2499 (ObjNum 1211 snap 381 seq_num 1211) dirty exists 2024-09-12T23:51:46.682 INFO:tasks.rados.rados.0.smithi136.stdout:3848: left oid 9 (ObjNum 1211 snap 381 seq_num 1211) 2024-09-12T23:51:46.682 INFO:tasks.rados.rados.0.smithi136.stdout:3850: seq_num 1213 ranges {4235264=720896,4956160=401408} 2024-09-12T23:51:46.714 INFO:tasks.rados.rados.0.smithi136.stdout:3850: writing smithi13667081-44 from 4235264 to 4956160 tid 1 2024-09-12T23:51:46.717 INFO:tasks.rados.rados.0.smithi136.stdout:3850: writing smithi13667081-44 from 4956160 to 5357568 tid 2 2024-09-12T23:51:46.717 INFO:tasks.rados.rados.0.smithi136.stdout:3847: done (3 left) 2024-09-12T23:51:46.717 INFO:tasks.rados.rados.0.smithi136.stdout:3848: done (2 left) 2024-09-12T23:51:46.718 INFO:tasks.rados.rados.0.smithi136.stdout:3851: delete oid 49 current snap is 381 2024-09-12T23:51:46.718 INFO:tasks.rados.rados.0.smithi136.stdout:3849: finishing write tid 1 to smithi13667081-30 2024-09-12T23:51:46.718 INFO:tasks.rados.rados.0.smithi136.stdout:3849: finishing write tid 2 to smithi13667081-30 2024-09-12T23:51:46.718 INFO:tasks.rados.rados.0.smithi136.stdout:3849: finishing write tid 3 to smithi13667081-30 2024-09-12T23:51:46.718 INFO:tasks.rados.rados.0.smithi136.stdout:3849: finishing write tid 4 to smithi13667081-30 2024-09-12T23:51:46.718 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 30 v 2601 (ObjNum 1212 snap 381 seq_num 1212) dirty exists 2024-09-12T23:51:46.718 INFO:tasks.rados.rados.0.smithi136.stdout:3849: left oid 30 (ObjNum 1212 snap 381 seq_num 1212) 2024-09-12T23:51:46.724 INFO:tasks.rados.rados.0.smithi136.stdout:3849: done (2 left) 2024-09-12T23:51:46.724 INFO:tasks.rados.rados.0.smithi136.stdout:3851: done (1 left) 2024-09-12T23:51:46.725 INFO:tasks.rados.rados.0.smithi136.stdout:3852: delete oid 1 current snap is 381 2024-09-12T23:51:46.726 INFO:tasks.rados.rados.0.smithi136.stdout:3850: finishing write tid 1 to smithi13667081-44 2024-09-12T23:51:46.727 INFO:tasks.rados.rados.0.smithi136.stdout:3850: finishing write tid 2 to smithi13667081-44 2024-09-12T23:51:46.727 INFO:tasks.rados.rados.0.smithi136.stdout:3850: finishing write tid 3 to smithi13667081-44 2024-09-12T23:51:46.728 INFO:tasks.rados.rados.0.smithi136.stdout:3852: done (1 left) 2024-09-12T23:51:46.728 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 2 current snap is 381 2024-09-12T23:51:46.728 INFO:tasks.rados.rados.0.smithi136.stdout:3853: seq_num 1216 ranges {4726784=770048,5496832=401408} 2024-09-12T23:51:46.761 INFO:tasks.rados.rados.0.smithi136.stdout:3853: writing smithi13667081-2 from 4726784 to 5496832 tid 1 2024-09-12T23:51:46.764 INFO:tasks.rados.rados.0.smithi136.stdout:3853: writing smithi13667081-2 from 5496832 to 5898240 tid 2 2024-09-12T23:51:46.764 INFO:tasks.rados.rados.0.smithi136.stdout:3854: snap_create 2024-09-12T23:51:46.764 INFO:tasks.rados.rados.0.smithi136.stdout:3850: finishing write tid 4 to smithi13667081-44 2024-09-12T23:51:46.764 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 44 v 2552 (ObjNum 1213 snap 381 seq_num 1213) dirty exists 2024-09-12T23:51:46.764 INFO:tasks.rados.rados.0.smithi136.stdout:3850: left oid 44 (ObjNum 1213 snap 381 seq_num 1213) 2024-09-12T23:51:46.771 INFO:tasks.rados.rados.0.smithi136.stdout:3853: finishing write tid 1 to smithi13667081-2 2024-09-12T23:51:46.771 INFO:tasks.rados.rados.0.smithi136.stdout:3853: finishing write tid 2 to smithi13667081-2 2024-09-12T23:51:46.772 INFO:tasks.rados.rados.0.smithi136.stdout:3853: finishing write tid 3 to smithi13667081-2 2024-09-12T23:51:46.775 INFO:tasks.rados.rados.0.smithi136.stdout:3853: finishing write tid 4 to smithi13667081-2 2024-09-12T23:51:46.775 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 2 v 2706 (ObjNum 1216 snap 381 seq_num 1216) dirty exists 2024-09-12T23:51:46.775 INFO:tasks.rados.rados.0.smithi136.stdout:3853: left oid 2 (ObjNum 1216 snap 381 seq_num 1216) 2024-09-12T23:51:46.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:46 smithi181 bash[67587]: cluster 2024-09-12T23:51:45.595281+0000 mon.a (mon.0) 1418 : cluster [DBG] osdmap e836: 8 total, 8 up, 8 in 2024-09-12T23:51:46.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:46 smithi181 bash[67587]: cluster 2024-09-12T23:51:46.544571+0000 mon.a (mon.0) 1419 : cluster [DBG] osdmap e837: 8 total, 8 up, 8 in 2024-09-12T23:51:47.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:46 smithi136 bash[75164]: cluster 2024-09-12T23:51:45.595281+0000 mon.a (mon.0) 1418 : cluster [DBG] osdmap e836: 8 total, 8 up, 8 in 2024-09-12T23:51:47.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:46 smithi136 bash[75164]: cluster 2024-09-12T23:51:46.544571+0000 mon.a (mon.0) 1419 : cluster [DBG] osdmap e837: 8 total, 8 up, 8 in 2024-09-12T23:51:47.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:46 smithi136 bash[76371]: cluster 2024-09-12T23:51:45.595281+0000 mon.a (mon.0) 1418 : cluster [DBG] osdmap e836: 8 total, 8 up, 8 in 2024-09-12T23:51:47.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:46 smithi136 bash[76371]: cluster 2024-09-12T23:51:46.544571+0000 mon.a (mon.0) 1419 : cluster [DBG] osdmap e837: 8 total, 8 up, 8 in 2024-09-12T23:51:47.599 INFO:tasks.rados.rados.0.smithi136.stdout:3850: done (2 left) 2024-09-12T23:51:47.599 INFO:tasks.rados.rados.0.smithi136.stdout:3853: done (1 left) 2024-09-12T23:51:47.599 INFO:tasks.rados.rados.0.smithi136.stdout:3854: done (0 left) 2024-09-12T23:51:47.599 INFO:tasks.rados.rados.0.smithi136.stdout:3855: read oid 25 snap -1 2024-09-12T23:51:47.599 INFO:tasks.rados.rados.0.smithi136.stdout:3855: expect deleted 2024-09-12T23:51:47.599 INFO:tasks.rados.rados.0.smithi136.stdout:3856: rollback oid 21 current snap is 382 2024-09-12T23:51:47.599 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 21 to 361 2024-09-12T23:51:47.599 INFO:tasks.rados.rados.0.smithi136.stdout:3857: read oid 35 snap 359 2024-09-12T23:51:47.600 INFO:tasks.rados.rados.0.smithi136.stdout:3857: expect (ObjNum 1091 snap 329 seq_num 1091) 2024-09-12T23:51:47.600 INFO:tasks.rados.rados.0.smithi136.stdout:3858: read oid 18 snap -1 2024-09-12T23:51:47.600 INFO:tasks.rados.rados.0.smithi136.stdout:3858: expect (ObjNum 1206 snap 378 seq_num 1206) 2024-09-12T23:51:47.600 INFO:tasks.rados.rados.0.smithi136.stdout:3859: copy_from oid 13 from oid 33 current snap is 382 2024-09-12T23:51:47.600 INFO:tasks.rados.rados.0.smithi136.stdout:3860: read oid 38 snap 359 2024-09-12T23:51:47.600 INFO:tasks.rados.rados.0.smithi136.stdout:3860: expect (ObjNum 1134 snap 345 seq_num 1134) 2024-09-12T23:51:47.600 INFO:tasks.rados.rados.0.smithi136.stdout:3861: snap_remove snap 375 2024-09-12T23:51:47.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:47 smithi181 bash[67587]: cluster 2024-09-12T23:51:46.883126+0000 mgr.y (mgr.24661) 439 : cluster [DBG] pgmap v838: 121 pgs: 121 active+clean; 319 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 7.3 MiB/s rd, 1008 KiB/s wr, 6 op/s 2024-09-12T23:51:48.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:47 smithi136 bash[75164]: cluster 2024-09-12T23:51:46.883126+0000 mgr.y (mgr.24661) 439 : cluster [DBG] pgmap v838: 121 pgs: 121 active+clean; 319 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 7.3 MiB/s rd, 1008 KiB/s wr, 6 op/s 2024-09-12T23:51:48.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:47 smithi136 bash[76371]: cluster 2024-09-12T23:51:46.883126+0000 mgr.y (mgr.24661) 439 : cluster [DBG] pgmap v838: 121 pgs: 121 active+clean; 319 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 7.3 MiB/s rd, 1008 KiB/s wr, 6 op/s 2024-09-12T23:51:48.602 INFO:tasks.rados.rados.0.smithi136.stdout:3861: done (6 left) 2024-09-12T23:51:48.602 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 15 current snap is 382 2024-09-12T23:51:48.602 INFO:tasks.rados.rados.0.smithi136.stdout:3862: seq_num 1217 ranges {0=786432,786432=401408} 2024-09-12T23:51:48.607 INFO:tasks.rados.rados.0.smithi136.stdout:3862: writing smithi13667081-15 from 0 to 786432 tid 1 2024-09-12T23:51:48.610 INFO:tasks.rados.rados.0.smithi136.stdout:3862: writing smithi13667081-15 from 786432 to 1187840 tid 2 2024-09-12T23:51:48.610 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 28 current snap is 382 2024-09-12T23:51:48.610 INFO:tasks.rados.rados.0.smithi136.stdout:3863: seq_num 1218 ranges {1613824=802816,2416640=401408} 2024-09-12T23:51:48.625 INFO:tasks.rados.rados.0.smithi136.stdout:3863: writing smithi13667081-28 from 1613824 to 2416640 tid 1 2024-09-12T23:51:48.629 INFO:tasks.rados.rados.0.smithi136.stdout:3863: writing smithi13667081-28 from 2416640 to 2818048 tid 2 2024-09-12T23:51:48.629 INFO:tasks.rados.rados.0.smithi136.stdout:3864: rmattr oid 2 current snap is 382 2024-09-12T23:51:48.629 INFO:tasks.rados.rados.0.smithi136.stdout:3865: snap_remove snap 296 2024-09-12T23:51:49.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:48 smithi136 bash[75164]: cluster 2024-09-12T23:51:47.601208+0000 mon.a (mon.0) 1420 : cluster [DBG] osdmap e838: 8 total, 8 up, 8 in 2024-09-12T23:51:49.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:48 smithi136 bash[76371]: cluster 2024-09-12T23:51:47.601208+0000 mon.a (mon.0) 1420 : cluster [DBG] osdmap e838: 8 total, 8 up, 8 in 2024-09-12T23:51:49.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:48 smithi181 bash[67587]: cluster 2024-09-12T23:51:47.601208+0000 mon.a (mon.0) 1420 : cluster [DBG] osdmap e838: 8 total, 8 up, 8 in 2024-09-12T23:51:49.636 INFO:tasks.rados.rados.0.smithi136.stdout:3865: done (9 left) 2024-09-12T23:51:49.636 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 48 current snap is 382 2024-09-12T23:51:49.636 INFO:tasks.rados.rados.0.smithi136.stdout:3866: seq_num 1219 ranges {0=417792,417792=557056,974848=245760} 2024-09-12T23:51:49.641 INFO:tasks.rados.rados.0.smithi136.stdout:3866: writing smithi13667081-48 from 0 to 417792 tid 1 2024-09-12T23:51:49.646 INFO:tasks.rados.rados.0.smithi136.stdout:3866: writing smithi13667081-48 from 417792 to 974848 tid 2 2024-09-12T23:51:49.649 INFO:tasks.rados.rados.0.smithi136.stdout:3866: writing smithi13667081-48 from 974848 to 1220608 tid 3 2024-09-12T23:51:49.649 INFO:tasks.rados.rados.0.smithi136.stdout:3867: snap_remove snap 354 2024-09-12T23:51:49.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:49 smithi136 bash[75164]: cluster 2024-09-12T23:51:48.604146+0000 mon.a (mon.0) 1421 : cluster [DBG] osdmap e839: 8 total, 8 up, 8 in 2024-09-12T23:51:49.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:49 smithi136 bash[75164]: cluster 2024-09-12T23:51:48.883960+0000 mgr.y (mgr.24661) 440 : cluster [DBG] pgmap v841: 121 pgs: 121 active+clean; 322 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.5 MiB/s rd, 896 KiB/s wr, 5 op/s 2024-09-12T23:51:49.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:51:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:51:49] "GET /metrics HTTP/1.1" 200 36441 "" "Prometheus/2.43.0" 2024-09-12T23:51:49.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:49 smithi136 bash[76371]: cluster 2024-09-12T23:51:48.604146+0000 mon.a (mon.0) 1421 : cluster [DBG] osdmap e839: 8 total, 8 up, 8 in 2024-09-12T23:51:49.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:49 smithi136 bash[76371]: cluster 2024-09-12T23:51:48.883960+0000 mgr.y (mgr.24661) 440 : cluster [DBG] pgmap v841: 121 pgs: 121 active+clean; 322 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.5 MiB/s rd, 896 KiB/s wr, 5 op/s 2024-09-12T23:51:50.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:49 smithi181 bash[67587]: cluster 2024-09-12T23:51:48.604146+0000 mon.a (mon.0) 1421 : cluster [DBG] osdmap e839: 8 total, 8 up, 8 in 2024-09-12T23:51:50.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:49 smithi181 bash[67587]: cluster 2024-09-12T23:51:48.883960+0000 mgr.y (mgr.24661) 440 : cluster [DBG] pgmap v841: 121 pgs: 121 active+clean; 322 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.5 MiB/s rd, 896 KiB/s wr, 5 op/s 2024-09-12T23:51:50.632 INFO:tasks.rados.rados.0.smithi136.stdout:3867: done (10 left) 2024-09-12T23:51:50.633 INFO:tasks.rados.rados.0.smithi136.stdout:3868: delete oid 9 current snap is 382 2024-09-12T23:51:50.633 INFO:tasks.rados.rados.0.smithi136.stdout:3856: finishing rollback tid 1 to smithi13667081-21 2024-09-12T23:51:50.633 INFO:tasks.rados.rados.0.smithi136.stdout:3856: finishing rollback tid 2 to smithi13667081-21 2024-09-12T23:51:50.633 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 21 v 2534 (ObjNum 1149 snap 351 seq_num 1149) dirty exists 2024-09-12T23:51:50.633 INFO:tasks.rados.rados.0.smithi136.stdout:3858: expect (ObjNum 1206 snap 378 seq_num 1206) 2024-09-12T23:51:50.713 INFO:tasks.rados.rados.0.smithi136.stdout:3857: expect (ObjNum 1091 snap 329 seq_num 1091) 2024-09-12T23:51:50.848 INFO:tasks.rados.rados.0.smithi136.stdout:3860: expect (ObjNum 1134 snap 345 seq_num 1134) 2024-09-12T23:51:51.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:50 smithi136 bash[76371]: cluster 2024-09-12T23:51:49.632136+0000 mon.a (mon.0) 1422 : cluster [DBG] osdmap e840: 8 total, 8 up, 8 in 2024-09-12T23:51:51.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:50 smithi136 bash[75164]: cluster 2024-09-12T23:51:49.632136+0000 mon.a (mon.0) 1422 : cluster [DBG] osdmap e840: 8 total, 8 up, 8 in 2024-09-12T23:51:51.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:50 smithi181 bash[67587]: cluster 2024-09-12T23:51:49.632136+0000 mon.a (mon.0) 1422 : cluster [DBG] osdmap e840: 8 total, 8 up, 8 in 2024-09-12T23:51:51.109 INFO:tasks.rados.rados.0.smithi136.stdout:3859: finishing copy_from racing read to smithi13667081-13 2024-09-12T23:51:51.109 INFO:tasks.rados.rados.0.smithi136.stdout:3859: finishing copy_from to smithi13667081-13 2024-09-12T23:51:51.109 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 13 v 2702 (ObjNum 1199 snap 376 seq_num 1199) dirty exists 2024-09-12T23:51:51.109 INFO:tasks.rados.rados.0.smithi136.stdout:3862: finishing write tid 1 to smithi13667081-15 2024-09-12T23:51:51.109 INFO:tasks.rados.rados.0.smithi136.stdout:3862: finishing write tid 2 to smithi13667081-15 2024-09-12T23:51:51.109 INFO:tasks.rados.rados.0.smithi136.stdout:3862: finishing write tid 3 to smithi13667081-15 2024-09-12T23:51:51.109 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 2 v 2707 (ObjNum 1216 snap 381 seq_num 1216) dirty exists 2024-09-12T23:51:51.110 INFO:tasks.rados.rados.0.smithi136.stdout:3862: finishing write tid 4 to smithi13667081-15 2024-09-12T23:51:51.110 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 15 v 2515 (ObjNum 1217 snap 382 seq_num 1217) dirty exists 2024-09-12T23:51:51.110 INFO:tasks.rados.rados.0.smithi136.stdout:3862: left oid 15 (ObjNum 1217 snap 382 seq_num 1217) 2024-09-12T23:51:51.110 INFO:tasks.rados.rados.0.smithi136.stdout:3855: done (10 left) 2024-09-12T23:51:51.110 INFO:tasks.rados.rados.0.smithi136.stdout:3856: done (9 left) 2024-09-12T23:51:51.110 INFO:tasks.rados.rados.0.smithi136.stdout:3857: done (8 left) 2024-09-12T23:51:51.110 INFO:tasks.rados.rados.0.smithi136.stdout:3858: done (7 left) 2024-09-12T23:51:51.110 INFO:tasks.rados.rados.0.smithi136.stdout:3859: done (6 left) 2024-09-12T23:51:51.110 INFO:tasks.rados.rados.0.smithi136.stdout:3860: done (5 left) 2024-09-12T23:51:51.111 INFO:tasks.rados.rados.0.smithi136.stdout:3862: done (4 left) 2024-09-12T23:51:51.111 INFO:tasks.rados.rados.0.smithi136.stdout:3864: done (3 left) 2024-09-12T23:51:51.111 INFO:tasks.rados.rados.0.smithi136.stdout:3868: done (2 left) 2024-09-12T23:51:51.111 INFO:tasks.rados.rados.0.smithi136.stdout:3869: rollback oid 45 current snap is 382 2024-09-12T23:51:51.111 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 45 to 379 2024-09-12T23:51:51.111 INFO:tasks.rados.rados.0.smithi136.stdout:3863: finishing write tid 1 to smithi13667081-28 2024-09-12T23:51:51.111 INFO:tasks.rados.rados.0.smithi136.stdout:3863: finishing write tid 2 to smithi13667081-28 2024-09-12T23:51:51.111 INFO:tasks.rados.rados.0.smithi136.stdout:3863: finishing write tid 3 to smithi13667081-28 2024-09-12T23:51:51.111 INFO:tasks.rados.rados.0.smithi136.stdout:3863: finishing write tid 4 to smithi13667081-28 2024-09-12T23:51:51.111 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 28 v 2555 (ObjNum 1218 snap 382 seq_num 1218) dirty exists 2024-09-12T23:51:51.111 INFO:tasks.rados.rados.0.smithi136.stdout:3863: left oid 28 (ObjNum 1218 snap 382 seq_num 1218) 2024-09-12T23:51:51.111 INFO:tasks.rados.rados.0.smithi136.stdout:3866: finishing write tid 1 to smithi13667081-48 2024-09-12T23:51:51.112 INFO:tasks.rados.rados.0.smithi136.stdout:3866: finishing write tid 2 to smithi13667081-48 2024-09-12T23:51:51.112 INFO:tasks.rados.rados.0.smithi136.stdout:3866: finishing write tid 3 to smithi13667081-48 2024-09-12T23:51:51.112 INFO:tasks.rados.rados.0.smithi136.stdout:3866: finishing write tid 4 to smithi13667081-48 2024-09-12T23:51:51.112 INFO:tasks.rados.rados.0.smithi136.stdout:3866: finishing write tid 5 to smithi13667081-48 2024-09-12T23:51:51.112 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 48 v 2559 (ObjNum 1219 snap 382 seq_num 1219) dirty exists 2024-09-12T23:51:51.112 INFO:tasks.rados.rados.0.smithi136.stdout:3866: left oid 48 (ObjNum 1219 snap 382 seq_num 1219) 2024-09-12T23:51:51.112 INFO:tasks.rados.rados.0.smithi136.stdout:3863: done (2 left) 2024-09-12T23:51:51.112 INFO:tasks.rados.rados.0.smithi136.stdout:3866: done (1 left) 2024-09-12T23:51:51.112 INFO:tasks.rados.rados.0.smithi136.stdout:3870: read oid 28 snap -1 2024-09-12T23:51:51.112 INFO:tasks.rados.rados.0.smithi136.stdout:3870: expect (ObjNum 1218 snap 382 seq_num 1218) 2024-09-12T23:51:51.112 INFO:tasks.rados.rados.0.smithi136.stdout:3871: delete oid 43 current snap is 382 2024-09-12T23:51:51.116 INFO:tasks.rados.rados.0.smithi136.stdout:3869: finishing rollback tid 0 to smithi13667081-45 2024-09-12T23:51:51.117 INFO:tasks.rados.rados.0.smithi136.stdout:3869: finishing rollback tid 1 to smithi13667081-45 2024-09-12T23:51:51.118 INFO:tasks.rados.rados.0.smithi136.stdout:3869: finishing rollback tid 2 to smithi13667081-45 2024-09-12T23:51:51.118 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 45 v 2373 (ObjNum 1178 snap 369 seq_num 1178) dirty exists 2024-09-12T23:51:51.126 INFO:tasks.rados.rados.0.smithi136.stdout:3870: expect (ObjNum 1218 snap 382 seq_num 1218) 2024-09-12T23:51:51.237 INFO:tasks.rados.rados.0.smithi136.stdout:3869: done (2 left) 2024-09-12T23:51:51.237 INFO:tasks.rados.rados.0.smithi136.stdout:3870: done (1 left) 2024-09-12T23:51:51.237 INFO:tasks.rados.rados.0.smithi136.stdout:3871: done (0 left) 2024-09-12T23:51:51.237 INFO:tasks.rados.rados.0.smithi136.stdout:3872: snap_remove snap 361 2024-09-12T23:51:51.544 INFO:tasks.rados.rados.0.smithi136.stdout:3872: done (0 left) 2024-09-12T23:51:51.545 INFO:tasks.rados.rados.0.smithi136.stdout:3873: rollback oid 16 current snap is 382 2024-09-12T23:51:51.545 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 16 to 373 2024-09-12T23:51:51.545 INFO:tasks.rados.rados.0.smithi136.stdout:3874: rollback oid 7 current snap is 382 2024-09-12T23:51:51.546 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 7 to 376 2024-09-12T23:51:51.546 INFO:tasks.rados.rados.0.smithi136.stdout:3875: rollback oid 32 current snap is 382 2024-09-12T23:51:51.546 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 32 to 380 2024-09-12T23:51:51.546 INFO:tasks.rados.rados.0.smithi136.stdout:3876: delete oid 46 current snap is 382 2024-09-12T23:51:51.571 INFO:tasks.rados.rados.0.smithi136.stdout:3874: finishing rollback tid 0 to smithi13667081-7 2024-09-12T23:51:51.573 INFO:tasks.rados.rados.0.smithi136.stdout:3874: finishing rollback tid 1 to smithi13667081-7 2024-09-12T23:51:51.573 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 7 v 1656 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-12T23:51:51.577 INFO:tasks.rados.rados.0.smithi136.stdout:3874: done (3 left) 2024-09-12T23:51:51.577 INFO:tasks.rados.rados.0.smithi136.stdout:3876: done (2 left) 2024-09-12T23:51:51.577 INFO:tasks.rados.rados.0.smithi136.stdout:3877: delete oid 37 current snap is 382 2024-09-12T23:51:51.581 INFO:tasks.rados.rados.0.smithi136.stdout:3873: finishing rollback tid 0 to smithi13667081-16 2024-09-12T23:51:51.581 INFO:tasks.rados.rados.0.smithi136.stdout:3873: finishing rollback tid 1 to smithi13667081-16 2024-09-12T23:51:51.583 INFO:tasks.rados.rados.0.smithi136.stdout:3873: finishing rollback tid 2 to smithi13667081-16 2024-09-12T23:51:51.584 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 16 v 2563 (ObjNum 1162 snap 357 seq_num 1162) dirty exists 2024-09-12T23:51:51.584 INFO:tasks.rados.rados.0.smithi136.stdout:3875: finishing rollback tid 0 to smithi13667081-32 2024-09-12T23:51:51.584 INFO:tasks.rados.rados.0.smithi136.stdout:3875: finishing rollback tid 1 to smithi13667081-32 2024-09-12T23:51:51.584 INFO:tasks.rados.rados.0.smithi136.stdout:3875: finishing rollback tid 2 to smithi13667081-32 2024-09-12T23:51:51.584 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 32 v 2698 (ObjNum 1204 snap 378 seq_num 1204) dirty exists 2024-09-12T23:51:51.589 INFO:tasks.rados.rados.0.smithi136.stdout:3873: done (2 left) 2024-09-12T23:51:51.589 INFO:tasks.rados.rados.0.smithi136.stdout:3875: done (1 left) 2024-09-12T23:51:51.589 INFO:tasks.rados.rados.0.smithi136.stdout:3877: done (0 left) 2024-09-12T23:51:51.589 INFO:tasks.rados.rados.0.smithi136.stdout:3878: rmattr oid 44 current snap is 382 2024-09-12T23:51:51.589 INFO:tasks.rados.rados.0.smithi136.stdout:3878: done (0 left) 2024-09-12T23:51:51.589 INFO:tasks.rados.rados.0.smithi136.stdout:3879: read oid 7 snap -1 2024-09-12T23:51:51.589 INFO:tasks.rados.rados.0.smithi136.stdout:3879: expect deleted 2024-09-12T23:51:51.589 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 46 current snap is 382 2024-09-12T23:51:51.589 INFO:tasks.rados.rados.0.smithi136.stdout:3880: seq_num 1224 ranges {0=507904,507904=425984,933888=376832} 2024-09-12T23:51:51.595 INFO:tasks.rados.rados.0.smithi136.stdout:3880: writing smithi13667081-46 from 0 to 507904 tid 1 2024-09-12T23:51:51.600 INFO:tasks.rados.rados.0.smithi136.stdout:3880: writing smithi13667081-46 from 507904 to 933888 tid 2 2024-09-12T23:51:51.604 INFO:tasks.rados.rados.0.smithi136.stdout:3880: writing smithi13667081-46 from 933888 to 1310720 tid 3 2024-09-12T23:51:51.604 INFO:tasks.rados.rados.0.smithi136.stdout:3881: copy_from oid 36 from oid 4 current snap is 382 2024-09-12T23:51:51.604 INFO:tasks.rados.rados.0.smithi136.stdout:3880: finishing write tid 1 to smithi13667081-46 2024-09-12T23:51:51.604 INFO:tasks.rados.rados.0.smithi136.stdout:3880: finishing write tid 2 to smithi13667081-46 2024-09-12T23:51:51.605 INFO:tasks.rados.rados.0.smithi136.stdout:3879: done (2 left) 2024-09-12T23:51:51.605 INFO:tasks.rados.rados.0.smithi136.stdout:3882: read oid 12 snap -1 2024-09-12T23:51:51.605 INFO:tasks.rados.rados.0.smithi136.stdout:3882: expect (ObjNum 1092 snap 330 seq_num 1092) 2024-09-12T23:51:51.605 INFO:tasks.rados.rados.0.smithi136.stdout:3883: snap_create 2024-09-12T23:51:51.608 INFO:tasks.rados.rados.0.smithi136.stdout:3880: finishing write tid 3 to smithi13667081-46 2024-09-12T23:51:51.608 INFO:tasks.rados.rados.0.smithi136.stdout:3880: finishing write tid 4 to smithi13667081-46 2024-09-12T23:51:51.610 INFO:tasks.rados.rados.0.smithi136.stdout:3880: finishing write tid 5 to smithi13667081-46 2024-09-12T23:51:51.610 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 46 v 2495 (ObjNum 1224 snap 382 seq_num 1224) dirty exists 2024-09-12T23:51:51.610 INFO:tasks.rados.rados.0.smithi136.stdout:3880: left oid 46 (ObjNum 1224 snap 382 seq_num 1224) 2024-09-12T23:51:51.617 INFO:tasks.rados.rados.0.smithi136.stdout:3881: finishing copy_from racing read to smithi13667081-36 2024-09-12T23:51:51.619 INFO:tasks.rados.rados.0.smithi136.stdout:3881: finishing copy_from to smithi13667081-36 2024-09-12T23:51:51.619 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 36 v 2692 (ObjNum 1163 snap 358 seq_num 1163) dirty exists 2024-09-12T23:51:51.620 INFO:tasks.rados.rados.0.smithi136.stdout:3882: expect (ObjNum 1092 snap 330 seq_num 1092) 2024-09-12T23:51:52.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:51 smithi136 bash[75164]: cluster 2024-09-12T23:51:50.631842+0000 mon.a (mon.0) 1423 : cluster [DBG] osdmap e841: 8 total, 8 up, 8 in 2024-09-12T23:51:52.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:51 smithi136 bash[75164]: cluster 2024-09-12T23:51:50.884916+0000 mgr.y (mgr.24661) 441 : cluster [DBG] pgmap v844: 121 pgs: 121 active+clean; 321 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 6.6 MiB/s rd, 904 KiB/s wr, 6 op/s 2024-09-12T23:51:52.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:51 smithi136 bash[75164]: cluster 2024-09-12T23:51:51.546717+0000 mon.a (mon.0) 1424 : cluster [DBG] osdmap e842: 8 total, 8 up, 8 in 2024-09-12T23:51:52.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:51 smithi136 bash[76371]: cluster 2024-09-12T23:51:50.631842+0000 mon.a (mon.0) 1423 : cluster [DBG] osdmap e841: 8 total, 8 up, 8 in 2024-09-12T23:51:52.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:51 smithi136 bash[76371]: cluster 2024-09-12T23:51:50.884916+0000 mgr.y (mgr.24661) 441 : cluster [DBG] pgmap v844: 121 pgs: 121 active+clean; 321 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 6.6 MiB/s rd, 904 KiB/s wr, 6 op/s 2024-09-12T23:51:52.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:51 smithi136 bash[76371]: cluster 2024-09-12T23:51:51.546717+0000 mon.a (mon.0) 1424 : cluster [DBG] osdmap e842: 8 total, 8 up, 8 in 2024-09-12T23:51:52.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:51 smithi181 bash[67587]: cluster 2024-09-12T23:51:50.631842+0000 mon.a (mon.0) 1423 : cluster [DBG] osdmap e841: 8 total, 8 up, 8 in 2024-09-12T23:51:52.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:51 smithi181 bash[67587]: cluster 2024-09-12T23:51:50.884916+0000 mgr.y (mgr.24661) 441 : cluster [DBG] pgmap v844: 121 pgs: 121 active+clean; 321 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 6.6 MiB/s rd, 904 KiB/s wr, 6 op/s 2024-09-12T23:51:52.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:51 smithi181 bash[67587]: cluster 2024-09-12T23:51:51.546717+0000 mon.a (mon.0) 1424 : cluster [DBG] osdmap e842: 8 total, 8 up, 8 in 2024-09-12T23:51:52.548 INFO:tasks.rados.rados.0.smithi136.stdout:3880: done (3 left) 2024-09-12T23:51:52.548 INFO:tasks.rados.rados.0.smithi136.stdout:3881: done (2 left) 2024-09-12T23:51:52.548 INFO:tasks.rados.rados.0.smithi136.stdout:3882: done (1 left) 2024-09-12T23:51:52.548 INFO:tasks.rados.rados.0.smithi136.stdout:3883: done (0 left) 2024-09-12T23:51:52.548 INFO:tasks.rados.rados.0.smithi136.stdout:3884: delete oid 10 current snap is 383 2024-09-12T23:51:52.553 INFO:tasks.rados.rados.0.smithi136.stdout:3884: done (0 left) 2024-09-12T23:51:52.553 INFO:tasks.rados.rados.0.smithi136.stdout:3885: copy_from oid 32 from oid 4 current snap is 383 2024-09-12T23:51:52.553 INFO:tasks.rados.rados.0.smithi136.stdout:3886: read oid 44 snap -1 2024-09-12T23:51:52.553 INFO:tasks.rados.rados.0.smithi136.stdout:3886: expect (ObjNum 1213 snap 381 seq_num 1213) 2024-09-12T23:51:52.554 INFO:tasks.rados.rados.0.smithi136.stdout:3887: copy_from oid 20 from oid 2 current snap is 383 2024-09-12T23:51:52.554 INFO:tasks.rados.rados.0.smithi136.stdout:3888: snap_remove snap 380 2024-09-12T23:51:52.596 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:51:52 smithi181 bash[95505]: ts=2024-09-12T23:51:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:51:53.561 INFO:tasks.rados.rados.0.smithi136.stdout:3888: done (3 left) 2024-09-12T23:51:53.561 INFO:tasks.rados.rados.0.smithi136.stdout:3889: read oid 37 snap -1 2024-09-12T23:51:53.561 INFO:tasks.rados.rados.0.smithi136.stdout:3889: expect deleted 2024-09-12T23:51:53.561 INFO:tasks.rados.rados.0.smithi136.stdout:3890: rollback oid 7 current snap is 383 2024-09-12T23:51:53.561 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 7 to 377 2024-09-12T23:51:53.561 INFO:tasks.rados.rados.0.smithi136.stdout:3891: snap_remove snap 369 2024-09-12T23:51:53.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:53 smithi181 bash[67587]: cluster 2024-09-12T23:51:52.550783+0000 mon.a (mon.0) 1425 : cluster [DBG] osdmap e843: 8 total, 8 up, 8 in 2024-09-12T23:51:53.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:53 smithi181 bash[67587]: cluster 2024-09-12T23:51:52.885941+0000 mgr.y (mgr.24661) 442 : cluster [DBG] pgmap v847: 121 pgs: 121 active+clean; 307 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 6.1 MiB/s rd, 3.3 MiB/s wr, 16 op/s 2024-09-12T23:51:54.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:53 smithi136 bash[75164]: cluster 2024-09-12T23:51:52.550783+0000 mon.a (mon.0) 1425 : cluster [DBG] osdmap e843: 8 total, 8 up, 8 in 2024-09-12T23:51:54.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:53 smithi136 bash[75164]: cluster 2024-09-12T23:51:52.885941+0000 mgr.y (mgr.24661) 442 : cluster [DBG] pgmap v847: 121 pgs: 121 active+clean; 307 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 6.1 MiB/s rd, 3.3 MiB/s wr, 16 op/s 2024-09-12T23:51:54.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:53 smithi136 bash[76371]: cluster 2024-09-12T23:51:52.550783+0000 mon.a (mon.0) 1425 : cluster [DBG] osdmap e843: 8 total, 8 up, 8 in 2024-09-12T23:51:54.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:53 smithi136 bash[76371]: cluster 2024-09-12T23:51:52.885941+0000 mgr.y (mgr.24661) 442 : cluster [DBG] pgmap v847: 121 pgs: 121 active+clean; 307 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 6.1 MiB/s rd, 3.3 MiB/s wr, 16 op/s 2024-09-12T23:51:54.581 INFO:tasks.rados.rados.0.smithi136.stdout:3891: done (5 left) 2024-09-12T23:51:54.581 INFO:tasks.rados.rados.0.smithi136.stdout:3892: read oid 31 snap 376 2024-09-12T23:51:54.581 INFO:tasks.rados.rados.0.smithi136.stdout:3892: expect (ObjNum 1188 snap 373 seq_num 1188) 2024-09-12T23:51:54.581 INFO:tasks.rados.rados.0.smithi136.stdout:3893: read oid 36 snap -1 2024-09-12T23:51:54.581 INFO:tasks.rados.rados.0.smithi136.stdout:3893: expect (ObjNum 1163 snap 358 seq_num 1163) 2024-09-12T23:51:54.581 INFO:tasks.rados.rados.0.smithi136.stdout:3894: copy_from oid 10 from oid 26 current snap is 383 2024-09-12T23:51:54.581 INFO:tasks.rados.rados.0.smithi136.stdout:3895: snap_remove snap 373 2024-09-12T23:51:54.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:54 smithi181 bash[67587]: cluster 2024-09-12T23:51:53.562757+0000 mon.a (mon.0) 1426 : cluster [DBG] osdmap e844: 8 total, 8 up, 8 in 2024-09-12T23:51:54.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:54 smithi181 bash[67587]: audit 2024-09-12T23:51:53.754941+0000 mon.a (mon.0) 1427 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:51:55.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:54 smithi136 bash[75164]: cluster 2024-09-12T23:51:53.562757+0000 mon.a (mon.0) 1426 : cluster [DBG] osdmap e844: 8 total, 8 up, 8 in 2024-09-12T23:51:55.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:54 smithi136 bash[75164]: audit 2024-09-12T23:51:53.754941+0000 mon.a (mon.0) 1427 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:51:55.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:54 smithi136 bash[76371]: cluster 2024-09-12T23:51:53.562757+0000 mon.a (mon.0) 1426 : cluster [DBG] osdmap e844: 8 total, 8 up, 8 in 2024-09-12T23:51:55.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:54 smithi136 bash[76371]: audit 2024-09-12T23:51:53.754941+0000 mon.a (mon.0) 1427 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:51:55.584 INFO:tasks.rados.rados.0.smithi136.stdout:3895: done (8 left) 2024-09-12T23:51:55.584 INFO:tasks.rados.rados.0.smithi136.stdout:3896: read oid 38 snap 376 2024-09-12T23:51:55.584 INFO:tasks.rados.rados.0.smithi136.stdout:3896: expect (ObjNum 1134 snap 345 seq_num 1134) 2024-09-12T23:51:55.584 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 35 current snap is 383 2024-09-12T23:51:55.585 INFO:tasks.rados.rados.0.smithi136.stdout:3897: seq_num 1226 ranges {4014080=540672,4554752=598016,5152768=204800} 2024-09-12T23:51:55.606 INFO:tasks.rados.rados.0.smithi136.stdout:3897: writing smithi13667081-35 from 4014080 to 4554752 tid 1 2024-09-12T23:51:55.610 INFO:tasks.rados.rados.0.smithi136.stdout:3897: writing smithi13667081-35 from 4554752 to 5152768 tid 2 2024-09-12T23:51:55.612 INFO:tasks.rados.rados.0.smithi136.stdout:3897: writing smithi13667081-35 from 5152768 to 5357568 tid 3 2024-09-12T23:51:55.612 INFO:tasks.rados.rados.0.smithi136.stdout:3898: copy_from oid 28 from oid 14 current snap is 383 2024-09-12T23:51:55.612 INFO:tasks.rados.rados.0.smithi136.stdout:3885: finishing copy_from racing read to smithi13667081-32 2024-09-12T23:51:55.612 INFO:tasks.rados.rados.0.smithi136.stdout:3885: finishing copy_from to smithi13667081-32 2024-09-12T23:51:55.612 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 32 v 2692 (ObjNum 1163 snap 358 seq_num 1163) dirty exists 2024-09-12T23:51:55.612 INFO:tasks.rados.rados.0.smithi136.stdout:3886: expect (ObjNum 1213 snap 381 seq_num 1213) 2024-09-12T23:51:55.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:55 smithi181 bash[67587]: cluster 2024-09-12T23:51:54.571620+0000 mon.a (mon.0) 1428 : cluster [DBG] osdmap e845: 8 total, 8 up, 8 in 2024-09-12T23:51:55.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:55 smithi181 bash[67587]: cluster 2024-09-12T23:51:54.886765+0000 mgr.y (mgr.24661) 443 : cluster [DBG] pgmap v850: 121 pgs: 121 active+clean; 253 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.5 MiB/s wr, 13 op/s 2024-09-12T23:51:55.862 INFO:tasks.rados.rados.0.smithi136.stdout:3887: finishing copy_from racing read to smithi13667081-20 2024-09-12T23:51:55.862 INFO:tasks.rados.rados.0.smithi136.stdout:3887: finishing copy_from to smithi13667081-20 2024-09-12T23:51:55.862 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 20 v 2707 (ObjNum 1216 snap 381 seq_num 1216) dirty exists 2024-09-12T23:51:55.862 INFO:tasks.rados.rados.0.smithi136.stdout:3890: finishing rollback tid 1 to smithi13667081-7 2024-09-12T23:51:55.862 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 7 v 1656 (ObjNum 33 snap 0 seq_num 4242686544) dirty dne 2024-09-12T23:51:55.862 INFO:tasks.rados.rados.0.smithi136.stdout:3894: finishing copy_from to smithi13667081-10 2024-09-12T23:51:55.862 INFO:tasks.rados.rados.0.smithi136.stdout:3894: got expected ENOENT (src dne) 2024-09-12T23:51:55.862 INFO:tasks.rados.rados.0.smithi136.stdout:3894: finishing copy_from racing read to smithi13667081-10 2024-09-12T23:51:55.863 INFO:tasks.rados.rados.0.smithi136.stdout:3885: done (10 left) 2024-09-12T23:51:55.863 INFO:tasks.rados.rados.0.smithi136.stdout:3886: done (9 left) 2024-09-12T23:51:55.863 INFO:tasks.rados.rados.0.smithi136.stdout:3887: done (8 left) 2024-09-12T23:51:55.863 INFO:tasks.rados.rados.0.smithi136.stdout:3889: done (7 left) 2024-09-12T23:51:55.863 INFO:tasks.rados.rados.0.smithi136.stdout:3890: done (6 left) 2024-09-12T23:51:55.863 INFO:tasks.rados.rados.0.smithi136.stdout:3894: done (5 left) 2024-09-12T23:51:55.863 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 41 current snap is 383 2024-09-12T23:51:55.863 INFO:tasks.rados.rados.0.smithi136.stdout:3899: seq_num 1227 ranges {466944=557056,1024000=483328,1507328=319488} 2024-09-12T23:51:55.871 INFO:tasks.rados.rados.0.smithi136.stdout:3899: writing smithi13667081-41 from 466944 to 1024000 tid 1 2024-09-12T23:51:55.875 INFO:tasks.rados.rados.0.smithi136.stdout:3899: writing smithi13667081-41 from 1024000 to 1507328 tid 2 2024-09-12T23:51:55.879 INFO:tasks.rados.rados.0.smithi136.stdout:3899: writing smithi13667081-41 from 1507328 to 1826816 tid 3 2024-09-12T23:51:55.879 INFO:tasks.rados.rados.0.smithi136.stdout:3900: delete oid 1 current snap is 383 2024-09-12T23:51:55.879 INFO:tasks.rados.rados.0.smithi136.stdout:3892: expect (ObjNum 1188 snap 373 seq_num 1188) 2024-09-12T23:51:56.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:55 smithi136 bash[75164]: cluster 2024-09-12T23:51:54.571620+0000 mon.a (mon.0) 1428 : cluster [DBG] osdmap e845: 8 total, 8 up, 8 in 2024-09-12T23:51:56.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:55 smithi136 bash[75164]: cluster 2024-09-12T23:51:54.886765+0000 mgr.y (mgr.24661) 443 : cluster [DBG] pgmap v850: 121 pgs: 121 active+clean; 253 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.5 MiB/s wr, 13 op/s 2024-09-12T23:51:56.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:55 smithi136 bash[76371]: cluster 2024-09-12T23:51:54.571620+0000 mon.a (mon.0) 1428 : cluster [DBG] osdmap e845: 8 total, 8 up, 8 in 2024-09-12T23:51:56.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:55 smithi136 bash[76371]: cluster 2024-09-12T23:51:54.886765+0000 mgr.y (mgr.24661) 443 : cluster [DBG] pgmap v850: 121 pgs: 121 active+clean; 253 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.5 MiB/s wr, 13 op/s 2024-09-12T23:51:56.062 INFO:tasks.rados.rados.0.smithi136.stdout:3893: expect (ObjNum 1163 snap 358 seq_num 1163) 2024-09-12T23:51:56.157 INFO:tasks.rados.rados.0.smithi136.stdout:3897: finishing write tid 1 to smithi13667081-35 2024-09-12T23:51:56.158 INFO:tasks.rados.rados.0.smithi136.stdout:3897: finishing write tid 2 to smithi13667081-35 2024-09-12T23:51:56.158 INFO:tasks.rados.rados.0.smithi136.stdout:3892: done (6 left) 2024-09-12T23:51:56.158 INFO:tasks.rados.rados.0.smithi136.stdout:3893: done (5 left) 2024-09-12T23:51:56.158 INFO:tasks.rados.rados.0.smithi136.stdout:3900: done (4 left) 2024-09-12T23:51:56.158 INFO:tasks.rados.rados.0.smithi136.stdout:3901: rollback oid 11 current snap is 383 2024-09-12T23:51:56.158 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 11 to 379 2024-09-12T23:51:56.158 INFO:tasks.rados.rados.0.smithi136.stdout:3897: finishing write tid 3 to smithi13667081-35 2024-09-12T23:51:56.158 INFO:tasks.rados.rados.0.smithi136.stdout:3897: finishing write tid 4 to smithi13667081-35 2024-09-12T23:51:56.158 INFO:tasks.rados.rados.0.smithi136.stdout:3897: finishing write tid 5 to smithi13667081-35 2024-09-12T23:51:56.158 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 35 v 2935 (ObjNum 1226 snap 383 seq_num 1226) dirty exists 2024-09-12T23:51:56.158 INFO:tasks.rados.rados.0.smithi136.stdout:3897: left oid 35 (ObjNum 1226 snap 383 seq_num 1226) 2024-09-12T23:51:56.158 INFO:tasks.rados.rados.0.smithi136.stdout:3896: expect (ObjNum 1134 snap 345 seq_num 1134) 2024-09-12T23:51:56.416 INFO:tasks.rados.rados.0.smithi136.stdout:3898: finishing copy_from racing read to smithi13667081-28 2024-09-12T23:51:56.417 INFO:tasks.rados.rados.0.smithi136.stdout:3898: finishing copy_from to smithi13667081-28 2024-09-12T23:51:56.417 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 28 v 2549 (ObjNum 1210 snap 381 seq_num 1210) dirty exists 2024-09-12T23:51:56.417 INFO:tasks.rados.rados.0.smithi136.stdout:3899: finishing write tid 1 to smithi13667081-41 2024-09-12T23:51:56.417 INFO:tasks.rados.rados.0.smithi136.stdout:3899: finishing write tid 2 to smithi13667081-41 2024-09-12T23:51:56.417 INFO:tasks.rados.rados.0.smithi136.stdout:3899: finishing write tid 3 to smithi13667081-41 2024-09-12T23:51:56.417 INFO:tasks.rados.rados.0.smithi136.stdout:3896: done (4 left) 2024-09-12T23:51:56.417 INFO:tasks.rados.rados.0.smithi136.stdout:3897: done (3 left) 2024-09-12T23:51:56.417 INFO:tasks.rados.rados.0.smithi136.stdout:3898: done (2 left) 2024-09-12T23:51:56.417 INFO:tasks.rados.rados.0.smithi136.stdout:3902: setattr oid 20 current snap is 383 2024-09-12T23:51:56.417 INFO:tasks.rados.rados.0.smithi136.stdout:3899: finishing write tid 4 to smithi13667081-41 2024-09-12T23:51:56.417 INFO:tasks.rados.rados.0.smithi136.stdout:3899: finishing write tid 5 to smithi13667081-41 2024-09-12T23:51:56.417 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 41 v 2711 (ObjNum 1227 snap 383 seq_num 1227) dirty exists 2024-09-12T23:51:56.418 INFO:tasks.rados.rados.0.smithi136.stdout:3899: left oid 41 (ObjNum 1227 snap 383 seq_num 1227) 2024-09-12T23:51:56.418 INFO:tasks.rados.rados.0.smithi136.stdout:3901: finishing rollback tid 1 to smithi13667081-11 2024-09-12T23:51:56.418 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 11 v 0 (ObjNum 97 snap 0 seq_num 1) dirty dne 2024-09-12T23:51:56.418 INFO:tasks.rados.rados.0.smithi136.stdout:3899: done (2 left) 2024-09-12T23:51:56.418 INFO:tasks.rados.rados.0.smithi136.stdout:3901: done (1 left) 2024-09-12T23:51:56.418 INFO:tasks.rados.rados.0.smithi136.stdout:3903: read oid 8 snap -1 2024-09-12T23:51:56.418 INFO:tasks.rados.rados.0.smithi136.stdout:3903: expect (ObjNum 1164 snap 359 seq_num 1164) 2024-09-12T23:51:56.419 INFO:tasks.rados.rados.0.smithi136.stdout:3904: copy_from oid 25 from oid 35 current snap is 383 2024-09-12T23:51:56.419 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 50 current snap is 383 2024-09-12T23:51:56.419 INFO:tasks.rados.rados.0.smithi136.stdout:3905: seq_num 1229 ranges {5308416=589824,5898240=647168,6545408=155648} 2024-09-12T23:51:56.453 INFO:tasks.rados.rados.0.smithi136.stdout:3905: writing smithi13667081-50 from 5308416 to 5898240 tid 1 2024-09-12T23:51:56.457 INFO:tasks.rados.rados.0.smithi136.stdout:3905: writing smithi13667081-50 from 5898240 to 6545408 tid 2 2024-09-12T23:51:56.459 INFO:tasks.rados.rados.0.smithi136.stdout:3905: writing smithi13667081-50 from 6545408 to 6701056 tid 3 2024-09-12T23:51:56.459 INFO:tasks.rados.rados.0.smithi136.stdout:3906: snap_create 2024-09-12T23:51:56.459 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 20 v 2708 (ObjNum 1216 snap 381 seq_num 1216) dirty exists 2024-09-12T23:51:56.459 INFO:tasks.rados.rados.0.smithi136.stdout:3903: expect (ObjNum 1164 snap 359 seq_num 1164) 2024-09-12T23:51:56.744 INFO:tasks.rados.rados.0.smithi136.stdout:3904: finishing copy_from racing read to smithi13667081-25 2024-09-12T23:51:56.744 INFO:tasks.rados.rados.0.smithi136.stdout:3904: finishing copy_from to smithi13667081-25 2024-09-12T23:51:56.744 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 25 v 2935 (ObjNum 1226 snap 383 seq_num 1226) dirty exists 2024-09-12T23:51:56.744 INFO:tasks.rados.rados.0.smithi136.stdout:3905: finishing write tid 1 to smithi13667081-50 2024-09-12T23:51:56.744 INFO:tasks.rados.rados.0.smithi136.stdout:3905: finishing write tid 2 to smithi13667081-50 2024-09-12T23:51:56.744 INFO:tasks.rados.rados.0.smithi136.stdout:3905: finishing write tid 3 to smithi13667081-50 2024-09-12T23:51:56.744 INFO:tasks.rados.rados.0.smithi136.stdout:3905: finishing write tid 4 to smithi13667081-50 2024-09-12T23:51:56.744 INFO:tasks.rados.rados.0.smithi136.stdout:3905: finishing write tid 5 to smithi13667081-50 2024-09-12T23:51:56.744 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 50 v 2708 (ObjNum 1229 snap 383 seq_num 1229) dirty exists 2024-09-12T23:51:56.744 INFO:tasks.rados.rados.0.smithi136.stdout:3905: left oid 50 (ObjNum 1229 snap 383 seq_num 1229) 2024-09-12T23:51:56.744 INFO:tasks.rados.rados.0.smithi136.stdout:3902: done (4 left) 2024-09-12T23:51:56.745 INFO:tasks.rados.rados.0.smithi136.stdout:3903: done (3 left) 2024-09-12T23:51:56.745 INFO:tasks.rados.rados.0.smithi136.stdout:3904: done (2 left) 2024-09-12T23:51:56.745 INFO:tasks.rados.rados.0.smithi136.stdout:3905: done (1 left) 2024-09-12T23:51:56.745 INFO:tasks.rados.rados.0.smithi136.stdout:3906: done (0 left) 2024-09-12T23:51:56.745 INFO:tasks.rados.rados.0.smithi136.stdout:3907: copy_from oid 7 from oid 49 current snap is 384 2024-09-12T23:51:56.745 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 37 current snap is 384 2024-09-12T23:51:56.745 INFO:tasks.rados.rados.0.smithi136.stdout:3908: seq_num 1230 ranges {0=606208,606208=532480,1138688=270336} 2024-09-12T23:51:56.747 INFO:tasks.rados.rados.0.smithi136.stdout:3908: writing smithi13667081-37 from 0 to 606208 tid 1 2024-09-12T23:51:56.751 INFO:tasks.rados.rados.0.smithi136.stdout:3908: writing smithi13667081-37 from 606208 to 1138688 tid 2 2024-09-12T23:51:56.754 INFO:tasks.rados.rados.0.smithi136.stdout:3908: writing smithi13667081-37 from 1138688 to 1409024 tid 3 2024-09-12T23:51:56.755 INFO:tasks.rados.rados.0.smithi136.stdout:3909: copy_from oid 5 from oid 6 current snap is 384 2024-09-12T23:51:56.755 INFO:tasks.rados.rados.0.smithi136.stdout:3907: finishing copy_from to smithi13667081-7 2024-09-12T23:51:56.755 INFO:tasks.rados.rados.0.smithi136.stdout:3907: got expected ENOENT (src dne) 2024-09-12T23:51:56.755 INFO:tasks.rados.rados.0.smithi136.stdout:3907: finishing copy_from racing read to smithi13667081-7 2024-09-12T23:51:56.755 INFO:tasks.rados.rados.0.smithi136.stdout:3907: done (2 left) 2024-09-12T23:51:56.755 INFO:tasks.rados.rados.0.smithi136.stdout:3910: rollback oid 46 current snap is 384 2024-09-12T23:51:56.755 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 46 to 374 2024-09-12T23:51:56.755 INFO:tasks.rados.rados.0.smithi136.stdout:3911: rollback oid 33 current snap is 384 2024-09-12T23:51:56.755 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 33 to 350 2024-09-12T23:51:56.755 INFO:tasks.rados.rados.0.smithi136.stdout:3912: copy_from oid 34 from oid 48 current snap is 384 2024-09-12T23:51:56.755 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 26 current snap is 384 2024-09-12T23:51:56.755 INFO:tasks.rados.rados.0.smithi136.stdout:3913: seq_num 1231 ranges {0=622592,622592=417792,1040384=385024} 2024-09-12T23:51:56.760 INFO:tasks.rados.rados.0.smithi136.stdout:3913: writing smithi13667081-26 from 0 to 622592 tid 1 2024-09-12T23:51:56.764 INFO:tasks.rados.rados.0.smithi136.stdout:3913: writing smithi13667081-26 from 622592 to 1040384 tid 2 2024-09-12T23:51:56.767 INFO:tasks.rados.rados.0.smithi136.stdout:3913: writing smithi13667081-26 from 1040384 to 1425408 tid 3 2024-09-12T23:51:56.767 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 43 current snap is 384 2024-09-12T23:51:56.767 INFO:tasks.rados.rados.0.smithi136.stdout:3909: finishing copy_from to smithi13667081-5 2024-09-12T23:51:56.767 INFO:tasks.rados.rados.0.smithi136.stdout:3909: got expected ENOENT (src dne) 2024-09-12T23:51:56.767 INFO:tasks.rados.rados.0.smithi136.stdout:3909: finishing copy_from racing read to smithi13667081-5 2024-09-12T23:51:56.768 INFO:tasks.rados.rados.0.smithi136.stdout:3908: finishing write tid 1 to smithi13667081-37 2024-09-12T23:51:56.768 INFO:tasks.rados.rados.0.smithi136.stdout:3908: finishing write tid 2 to smithi13667081-37 2024-09-12T23:51:56.768 INFO:tasks.rados.rados.0.smithi136.stdout:3908: finishing write tid 3 to smithi13667081-37 2024-09-12T23:51:56.769 INFO:tasks.rados.rados.0.smithi136.stdout:3910: finishing rollback tid 0 to smithi13667081-46 2024-09-12T23:51:56.769 INFO:tasks.rados.rados.0.smithi136.stdout:3910: finishing rollback tid 1 to smithi13667081-46 2024-09-12T23:51:56.769 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 46 v 2497 (ObjNum 33 snap 0 seq_num 1902204758) dirty dne 2024-09-12T23:51:56.769 INFO:tasks.rados.rados.0.smithi136.stdout:3908: finishing write tid 4 to smithi13667081-37 2024-09-12T23:51:56.769 INFO:tasks.rados.rados.0.smithi136.stdout:3914: seq_num 1232 ranges {0=638976,638976=753664,1392640=49152} 2024-09-12T23:51:56.773 INFO:tasks.rados.rados.0.smithi136.stdout:3914: writing smithi13667081-43 from 0 to 638976 tid 1 2024-09-12T23:51:56.779 INFO:tasks.rados.rados.0.smithi136.stdout:3914: writing smithi13667081-43 from 638976 to 1392640 tid 2 2024-09-12T23:51:56.780 INFO:tasks.rados.rados.0.smithi136.stdout:3914: writing smithi13667081-43 from 1392640 to 1441792 tid 3 2024-09-12T23:51:56.780 INFO:tasks.rados.rados.0.smithi136.stdout:3909: done (6 left) 2024-09-12T23:51:56.780 INFO:tasks.rados.rados.0.smithi136.stdout:3910: done (5 left) 2024-09-12T23:51:56.780 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 40 current snap is 384 2024-09-12T23:51:56.780 INFO:tasks.rados.rados.0.smithi136.stdout:3911: finishing rollback tid 0 to smithi13667081-33 2024-09-12T23:51:56.780 INFO:tasks.rados.rados.0.smithi136.stdout:3913: finishing write tid 1 to smithi13667081-26 2024-09-12T23:51:56.780 INFO:tasks.rados.rados.0.smithi136.stdout:3908: finishing write tid 5 to smithi13667081-37 2024-09-12T23:51:56.781 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 37 v 2715 (ObjNum 1230 snap 384 seq_num 1230) dirty exists 2024-09-12T23:51:56.781 INFO:tasks.rados.rados.0.smithi136.stdout:3908: left oid 37 (ObjNum 1230 snap 384 seq_num 1230) 2024-09-12T23:51:56.781 INFO:tasks.rados.rados.0.smithi136.stdout:3911: finishing rollback tid 1 to smithi13667081-33 2024-09-12T23:51:56.781 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 33 v 2717 (ObjNum 65 snap 0 seq_num 1869242708) dirty dne 2024-09-12T23:51:56.781 INFO:tasks.rados.rados.0.smithi136.stdout:3913: finishing write tid 2 to smithi13667081-26 2024-09-12T23:51:56.781 INFO:tasks.rados.rados.0.smithi136.stdout:3913: finishing write tid 3 to smithi13667081-26 2024-09-12T23:51:56.781 INFO:tasks.rados.rados.0.smithi136.stdout:3913: finishing write tid 4 to smithi13667081-26 2024-09-12T23:51:56.781 INFO:tasks.rados.rados.0.smithi136.stdout:3912: finishing copy_from racing read to smithi13667081-34 2024-09-12T23:51:56.781 INFO:tasks.rados.rados.0.smithi136.stdout:3915: seq_num 1233 ranges {0=655360,655360=638976,1294336=163840} 2024-09-12T23:51:56.784 INFO:tasks.rados.rados.0.smithi136.stdout:3915: writing smithi13667081-40 from 0 to 655360 tid 1 2024-09-12T23:51:56.790 INFO:tasks.rados.rados.0.smithi136.stdout:3915: writing smithi13667081-40 from 655360 to 1294336 tid 2 2024-09-12T23:51:56.791 INFO:tasks.rados.rados.0.smithi136.stdout:3915: writing smithi13667081-40 from 1294336 to 1458176 tid 3 2024-09-12T23:51:56.791 INFO:tasks.rados.rados.0.smithi136.stdout:3908: done (5 left) 2024-09-12T23:51:56.791 INFO:tasks.rados.rados.0.smithi136.stdout:3911: done (4 left) 2024-09-12T23:51:56.791 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 2 current snap is 384 2024-09-12T23:51:56.792 INFO:tasks.rados.rados.0.smithi136.stdout:3914: finishing write tid 1 to smithi13667081-43 2024-09-12T23:51:56.792 INFO:tasks.rados.rados.0.smithi136.stdout:3916: seq_num 1234 ranges {5898240=671744,6569984=516096,7086080=286720} 2024-09-12T23:51:56.833 INFO:tasks.rados.rados.0.smithi136.stdout:3916: writing smithi13667081-2 from 5898240 to 6569984 tid 1 2024-09-12T23:51:56.836 INFO:tasks.rados.rados.0.smithi136.stdout:3916: writing smithi13667081-2 from 6569984 to 7086080 tid 2 2024-09-12T23:51:56.838 INFO:tasks.rados.rados.0.smithi136.stdout:3916: writing smithi13667081-2 from 7086080 to 7372800 tid 3 2024-09-12T23:51:56.838 INFO:tasks.rados.rados.0.smithi136.stdout:3917: read oid 16 snap -1 2024-09-12T23:51:56.838 INFO:tasks.rados.rados.0.smithi136.stdout:3917: expect (ObjNum 1162 snap 357 seq_num 1162) 2024-09-12T23:51:56.838 INFO:tasks.rados.rados.0.smithi136.stdout:3918: copy_from oid 35 from oid 36 current snap is 384 2024-09-12T23:51:56.839 INFO:tasks.rados.rados.0.smithi136.stdout:3913: finishing write tid 5 to smithi13667081-26 2024-09-12T23:51:56.839 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 26 v 2501 (ObjNum 1231 snap 384 seq_num 1231) dirty exists 2024-09-12T23:51:56.839 INFO:tasks.rados.rados.0.smithi136.stdout:3913: left oid 26 (ObjNum 1231 snap 384 seq_num 1231) 2024-09-12T23:51:56.839 INFO:tasks.rados.rados.0.smithi136.stdout:3912: finishing copy_from to smithi13667081-34 2024-09-12T23:51:56.839 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 34 v 2559 (ObjNum 1219 snap 382 seq_num 1219) dirty exists 2024-09-12T23:51:56.839 INFO:tasks.rados.rados.0.smithi136.stdout:3914: finishing write tid 2 to smithi13667081-43 2024-09-12T23:51:56.839 INFO:tasks.rados.rados.0.smithi136.stdout:3914: finishing write tid 3 to smithi13667081-43 2024-09-12T23:51:56.839 INFO:tasks.rados.rados.0.smithi136.stdout:3914: finishing write tid 4 to smithi13667081-43 2024-09-12T23:51:56.839 INFO:tasks.rados.rados.0.smithi136.stdout:3912: done (6 left) 2024-09-12T23:51:56.839 INFO:tasks.rados.rados.0.smithi136.stdout:3913: done (5 left) 2024-09-12T23:51:56.839 INFO:tasks.rados.rados.0.smithi136.stdout:3919: snap_remove snap 359 2024-09-12T23:51:56.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:56 smithi181 bash[67587]: cluster 2024-09-12T23:51:55.581031+0000 mon.a (mon.0) 1429 : cluster [DBG] osdmap e846: 8 total, 8 up, 8 in 2024-09-12T23:51:56.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:56 smithi181 bash[67587]: cluster 2024-09-12T23:51:56.550056+0000 mon.a (mon.0) 1430 : cluster [DBG] osdmap e847: 8 total, 8 up, 8 in 2024-09-12T23:51:57.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:56 smithi136 bash[75164]: cluster 2024-09-12T23:51:55.581031+0000 mon.a (mon.0) 1429 : cluster [DBG] osdmap e846: 8 total, 8 up, 8 in 2024-09-12T23:51:57.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:56 smithi136 bash[75164]: cluster 2024-09-12T23:51:56.550056+0000 mon.a (mon.0) 1430 : cluster [DBG] osdmap e847: 8 total, 8 up, 8 in 2024-09-12T23:51:57.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:56 smithi136 bash[76371]: cluster 2024-09-12T23:51:55.581031+0000 mon.a (mon.0) 1429 : cluster [DBG] osdmap e846: 8 total, 8 up, 8 in 2024-09-12T23:51:57.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:56 smithi136 bash[76371]: cluster 2024-09-12T23:51:56.550056+0000 mon.a (mon.0) 1430 : cluster [DBG] osdmap e847: 8 total, 8 up, 8 in 2024-09-12T23:51:57.599 INFO:tasks.rados.rados.0.smithi136.stdout:3919: done (5 left) 2024-09-12T23:51:57.599 INFO:tasks.rados.rados.0.smithi136.stdout:3920: copy_from oid 34 from oid 42 current snap is 384 2024-09-12T23:51:57.599 INFO:tasks.rados.rados.0.smithi136.stdout:3914: finishing write tid 5 to smithi13667081-43 2024-09-12T23:51:57.599 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 43 v 2567 (ObjNum 1232 snap 384 seq_num 1232) dirty exists 2024-09-12T23:51:57.599 INFO:tasks.rados.rados.0.smithi136.stdout:3914: left oid 43 (ObjNum 1232 snap 384 seq_num 1232) 2024-09-12T23:51:57.599 INFO:tasks.rados.rados.0.smithi136.stdout:3915: finishing write tid 1 to smithi13667081-40 2024-09-12T23:51:57.599 INFO:tasks.rados.rados.0.smithi136.stdout:3915: finishing write tid 2 to smithi13667081-40 2024-09-12T23:51:57.599 INFO:tasks.rados.rados.0.smithi136.stdout:3915: finishing write tid 3 to smithi13667081-40 2024-09-12T23:51:57.599 INFO:tasks.rados.rados.0.smithi136.stdout:3915: finishing write tid 4 to smithi13667081-40 2024-09-12T23:51:57.599 INFO:tasks.rados.rados.0.smithi136.stdout:3915: finishing write tid 5 to smithi13667081-40 2024-09-12T23:51:57.600 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 40 v 2939 (ObjNum 1233 snap 384 seq_num 1233) dirty exists 2024-09-12T23:51:57.600 INFO:tasks.rados.rados.0.smithi136.stdout:3915: left oid 40 (ObjNum 1233 snap 384 seq_num 1233) 2024-09-12T23:51:57.600 INFO:tasks.rados.rados.0.smithi136.stdout:3916: finishing write tid 1 to smithi13667081-2 2024-09-12T23:51:57.600 INFO:tasks.rados.rados.0.smithi136.stdout:3917: expect (ObjNum 1162 snap 357 seq_num 1162) 2024-09-12T23:51:57.635 INFO:tasks.rados.rados.0.smithi136.stdout:3916: finishing write tid 2 to smithi13667081-2 2024-09-12T23:51:57.636 INFO:tasks.rados.rados.0.smithi136.stdout:3916: finishing write tid 3 to smithi13667081-2 2024-09-12T23:51:57.636 INFO:tasks.rados.rados.0.smithi136.stdout:3916: finishing write tid 4 to smithi13667081-2 2024-09-12T23:51:57.636 INFO:tasks.rados.rados.0.smithi136.stdout:3916: finishing write tid 5 to smithi13667081-2 2024-09-12T23:51:57.636 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 2 v 2721 (ObjNum 1234 snap 384 seq_num 1234) dirty exists 2024-09-12T23:51:57.636 INFO:tasks.rados.rados.0.smithi136.stdout:3916: left oid 2 (ObjNum 1234 snap 384 seq_num 1234) 2024-09-12T23:51:57.636 INFO:tasks.rados.rados.0.smithi136.stdout:3918: finishing copy_from racing read to smithi13667081-35 2024-09-12T23:51:57.636 INFO:tasks.rados.rados.0.smithi136.stdout:3918: finishing copy_from to smithi13667081-35 2024-09-12T23:51:57.636 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 35 v 2692 (ObjNum 1163 snap 358 seq_num 1163) dirty exists 2024-09-12T23:51:57.636 INFO:tasks.rados.rados.0.smithi136.stdout:3920: finishing copy_from to smithi13667081-34 2024-09-12T23:51:57.636 INFO:tasks.rados.rados.0.smithi136.stdout:3920: got expected ENOENT (src dne) 2024-09-12T23:51:57.636 INFO:tasks.rados.rados.0.smithi136.stdout:3920: finishing copy_from racing read to smithi13667081-34 2024-09-12T23:51:57.636 INFO:tasks.rados.rados.0.smithi136.stdout:3914: done (5 left) 2024-09-12T23:51:57.637 INFO:tasks.rados.rados.0.smithi136.stdout:3915: done (4 left) 2024-09-12T23:51:57.637 INFO:tasks.rados.rados.0.smithi136.stdout:3916: done (3 left) 2024-09-12T23:51:57.637 INFO:tasks.rados.rados.0.smithi136.stdout:3917: done (2 left) 2024-09-12T23:51:57.637 INFO:tasks.rados.rados.0.smithi136.stdout:3918: done (1 left) 2024-09-12T23:51:57.637 INFO:tasks.rados.rados.0.smithi136.stdout:3920: done (0 left) 2024-09-12T23:51:57.637 INFO:tasks.rados.rados.0.smithi136.stdout:3921: rollback oid 9 current snap is 384 2024-09-12T23:51:57.637 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 9 to 383 2024-09-12T23:51:57.637 INFO:tasks.rados.rados.0.smithi136.stdout:3922: copy_from oid 10 from oid 12 current snap is 384 2024-09-12T23:51:57.637 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 47 current snap is 384 2024-09-12T23:51:57.637 INFO:tasks.rados.rados.0.smithi136.stdout:3923: seq_num 1235 ranges {1179648=688128,1867776=802816} 2024-09-12T23:51:57.646 INFO:tasks.rados.rados.0.smithi136.stdout:3923: writing smithi13667081-47 from 1179648 to 1867776 tid 1 2024-09-12T23:51:57.652 INFO:tasks.rados.rados.0.smithi136.stdout:3923: writing smithi13667081-47 from 1867776 to 2670592 tid 2 2024-09-12T23:51:57.653 INFO:tasks.rados.rados.0.smithi136.stdout:3924: copy_from oid 22 from oid 19 current snap is 384 2024-09-12T23:51:57.653 INFO:tasks.rados.rados.0.smithi136.stdout:3921: finishing rollback tid 1 to smithi13667081-9 2024-09-12T23:51:57.653 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 9 v 2500 (ObjNum 65 snap 0 seq_num 1399418992) dirty dne 2024-09-12T23:51:57.653 INFO:tasks.rados.rados.0.smithi136.stdout:3921: done (3 left) 2024-09-12T23:51:57.653 INFO:tasks.rados.rados.0.smithi136.stdout:3925: rollback oid 35 current snap is 384 2024-09-12T23:51:57.653 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 35 to 381 2024-09-12T23:51:57.653 INFO:tasks.rados.rados.0.smithi136.stdout:3926: snap_create 2024-09-12T23:51:57.655 INFO:tasks.rados.rados.0.smithi136.stdout:3923: finishing write tid 1 to smithi13667081-47 2024-09-12T23:51:57.656 INFO:tasks.rados.rados.0.smithi136.stdout:3925: finishing rollback tid 0 to smithi13667081-35 2024-09-12T23:51:57.659 INFO:tasks.rados.rados.0.smithi136.stdout:3925: finishing rollback tid 1 to smithi13667081-35 2024-09-12T23:51:57.660 INFO:tasks.rados.rados.0.smithi136.stdout:3925: finishing rollback tid 2 to smithi13667081-35 2024-09-12T23:51:57.660 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 35 v 2948 (ObjNum 1176 snap 368 seq_num 1176) dirty exists 2024-09-12T23:51:57.662 INFO:tasks.rados.rados.0.smithi136.stdout:3923: finishing write tid 2 to smithi13667081-47 2024-09-12T23:51:57.662 INFO:tasks.rados.rados.0.smithi136.stdout:3923: finishing write tid 3 to smithi13667081-47 2024-09-12T23:51:57.663 INFO:tasks.rados.rados.0.smithi136.stdout:3922: finishing copy_from racing read to smithi13667081-10 2024-09-12T23:51:57.664 INFO:tasks.rados.rados.0.smithi136.stdout:3923: finishing write tid 4 to smithi13667081-47 2024-09-12T23:51:57.664 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 47 v 2503 (ObjNum 1235 snap 384 seq_num 1235) dirty exists 2024-09-12T23:51:57.664 INFO:tasks.rados.rados.0.smithi136.stdout:3923: left oid 47 (ObjNum 1235 snap 384 seq_num 1235) 2024-09-12T23:51:57.664 INFO:tasks.rados.rados.0.smithi136.stdout:3922: finishing copy_from to smithi13667081-10 2024-09-12T23:51:57.664 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 10 v 2504 (ObjNum 1092 snap 330 seq_num 1092) dirty exists 2024-09-12T23:51:57.667 INFO:tasks.rados.rados.0.smithi136.stdout:3924: finishing copy_from racing read to smithi13667081-22 2024-09-12T23:51:57.670 INFO:tasks.rados.rados.0.smithi136.stdout:3924: finishing copy_from to smithi13667081-22 2024-09-12T23:51:57.670 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 22 v 2496 (ObjNum 1208 snap 381 seq_num 1208) dirty exists 2024-09-12T23:51:57.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:57 smithi181 bash[67587]: cluster 2024-09-12T23:51:56.887683+0000 mgr.y (mgr.24661) 444 : cluster [DBG] pgmap v853: 121 pgs: 121 active+clean; 263 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 9.6 MiB/s rd, 2.7 MiB/s wr, 6 op/s 2024-09-12T23:51:58.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:57 smithi136 bash[75164]: cluster 2024-09-12T23:51:56.887683+0000 mgr.y (mgr.24661) 444 : cluster [DBG] pgmap v853: 121 pgs: 121 active+clean; 263 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 9.6 MiB/s rd, 2.7 MiB/s wr, 6 op/s 2024-09-12T23:51:58.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:57 smithi136 bash[76371]: cluster 2024-09-12T23:51:56.887683+0000 mgr.y (mgr.24661) 444 : cluster [DBG] pgmap v853: 121 pgs: 121 active+clean; 263 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 9.6 MiB/s rd, 2.7 MiB/s wr, 6 op/s 2024-09-12T23:51:58.600 INFO:tasks.rados.rados.0.smithi136.stdout:3922: done (4 left) 2024-09-12T23:51:58.600 INFO:tasks.rados.rados.0.smithi136.stdout:3923: done (3 left) 2024-09-12T23:51:58.600 INFO:tasks.rados.rados.0.smithi136.stdout:3924: done (2 left) 2024-09-12T23:51:58.600 INFO:tasks.rados.rados.0.smithi136.stdout:3925: done (1 left) 2024-09-12T23:51:58.601 INFO:tasks.rados.rados.0.smithi136.stdout:3926: done (0 left) 2024-09-12T23:51:58.601 INFO:tasks.rados.rados.0.smithi136.stdout:3927: rollback oid 42 current snap is 385 2024-09-12T23:51:58.601 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 42 to 378 2024-09-12T23:51:58.601 INFO:tasks.rados.rados.0.smithi136.stdout:3928: read oid 27 snap -1 2024-09-12T23:51:58.601 INFO:tasks.rados.rados.0.smithi136.stdout:3928: expect (ObjNum 1178 snap 369 seq_num 1178) 2024-09-12T23:51:58.601 INFO:tasks.rados.rados.0.smithi136.stdout:3929: read oid 29 snap -1 2024-09-12T23:51:58.601 INFO:tasks.rados.rados.0.smithi136.stdout:3929: expect deleted 2024-09-12T23:51:58.601 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 36 current snap is 385 2024-09-12T23:51:58.601 INFO:tasks.rados.rados.0.smithi136.stdout:3930: seq_num 1236 ranges {2375680=704512,3080192=688128,3768320=114688} 2024-09-12T23:51:58.621 INFO:tasks.rados.rados.0.smithi136.stdout:3930: writing smithi13667081-36 from 2375680 to 3080192 tid 1 2024-09-12T23:51:58.625 INFO:tasks.rados.rados.0.smithi136.stdout:3930: writing smithi13667081-36 from 3080192 to 3768320 tid 2 2024-09-12T23:51:58.626 INFO:tasks.rados.rados.0.smithi136.stdout:3930: writing smithi13667081-36 from 3768320 to 3883008 tid 3 2024-09-12T23:51:58.627 INFO:tasks.rados.rados.0.smithi136.stdout:3931: read oid 47 snap -1 2024-09-12T23:51:58.627 INFO:tasks.rados.rados.0.smithi136.stdout:3931: expect (ObjNum 1235 snap 384 seq_num 1235) 2024-09-12T23:51:58.627 INFO:tasks.rados.rados.0.smithi136.stdout:3932: copy_from oid 38 from oid 8 current snap is 385 2024-09-12T23:51:58.627 INFO:tasks.rados.rados.0.smithi136.stdout:3933: read oid 45 snap -1 2024-09-12T23:51:58.627 INFO:tasks.rados.rados.0.smithi136.stdout:3933: expect (ObjNum 1178 snap 369 seq_num 1178) 2024-09-12T23:51:58.627 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 33 current snap is 385 2024-09-12T23:51:58.627 INFO:tasks.rados.rados.0.smithi136.stdout:3927: finishing rollback tid 1 to smithi13667081-42 2024-09-12T23:51:58.627 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 42 v 2509 (ObjNum 49 snap 0 seq_num 1953066355) dirty dne 2024-09-12T23:51:58.627 INFO:tasks.rados.rados.0.smithi136.stdout:3928: expect (ObjNum 1178 snap 369 seq_num 1178) 2024-09-12T23:51:58.654 INFO:tasks.rados.rados.0.smithi136.stdout:3930: finishing write tid 1 to smithi13667081-36 2024-09-12T23:51:58.654 INFO:tasks.rados.rados.0.smithi136.stdout:3931: expect (ObjNum 1235 snap 384 seq_num 1235) 2024-09-12T23:51:58.763 INFO:tasks.rados.rados.0.smithi136.stdout:3930: finishing write tid 2 to smithi13667081-36 2024-09-12T23:51:58.763 INFO:tasks.rados.rados.0.smithi136.stdout:3930: finishing write tid 3 to smithi13667081-36 2024-09-12T23:51:58.763 INFO:tasks.rados.rados.0.smithi136.stdout:3930: finishing write tid 4 to smithi13667081-36 2024-09-12T23:51:58.763 INFO:tasks.rados.rados.0.smithi136.stdout:3934: seq_num 1237 ranges {0=720896,720896=573440,1294336=229376} 2024-09-12T23:51:58.767 INFO:tasks.rados.rados.0.smithi136.stdout:3934: writing smithi13667081-33 from 0 to 720896 tid 1 2024-09-12T23:51:58.770 INFO:tasks.rados.rados.0.smithi136.stdout:3934: writing smithi13667081-33 from 720896 to 1294336 tid 2 2024-09-12T23:51:58.773 INFO:tasks.rados.rados.0.smithi136.stdout:3934: writing smithi13667081-33 from 1294336 to 1523712 tid 3 2024-09-12T23:51:58.773 INFO:tasks.rados.rados.0.smithi136.stdout:3927: done (7 left) 2024-09-12T23:51:58.773 INFO:tasks.rados.rados.0.smithi136.stdout:3928: done (6 left) 2024-09-12T23:51:58.773 INFO:tasks.rados.rados.0.smithi136.stdout:3929: done (5 left) 2024-09-12T23:51:58.773 INFO:tasks.rados.rados.0.smithi136.stdout:3931: done (4 left) 2024-09-12T23:51:58.774 INFO:tasks.rados.rados.0.smithi136.stdout:3935: snap_remove snap 377 2024-09-12T23:51:58.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:58 smithi181 bash[67587]: cluster 2024-09-12T23:51:57.596216+0000 mon.a (mon.0) 1431 : cluster [DBG] osdmap e848: 8 total, 8 up, 8 in 2024-09-12T23:51:59.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:58 smithi136 bash[75164]: cluster 2024-09-12T23:51:57.596216+0000 mon.a (mon.0) 1431 : cluster [DBG] osdmap e848: 8 total, 8 up, 8 in 2024-09-12T23:51:59.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:58 smithi136 bash[76371]: cluster 2024-09-12T23:51:57.596216+0000 mon.a (mon.0) 1431 : cluster [DBG] osdmap e848: 8 total, 8 up, 8 in 2024-09-12T23:51:59.613 INFO:tasks.rados.rados.0.smithi136.stdout:3935: done (4 left) 2024-09-12T23:51:59.613 INFO:tasks.rados.rados.0.smithi136.stdout:3936: read oid 24 snap -1 2024-09-12T23:51:59.613 INFO:tasks.rados.rados.0.smithi136.stdout:3936: expect (ObjNum 1081 snap 326 seq_num 1081) 2024-09-12T23:51:59.613 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 34 current snap is 385 2024-09-12T23:51:59.613 INFO:tasks.rados.rados.0.smithi136.stdout:3937: seq_num 1238 ranges {1220608=737280,1957888=458752,2416640=344064} 2024-09-12T23:51:59.627 INFO:tasks.rados.rados.0.smithi136.stdout:3937: writing smithi13667081-34 from 1220608 to 1957888 tid 1 2024-09-12T23:51:59.631 INFO:tasks.rados.rados.0.smithi136.stdout:3937: writing smithi13667081-34 from 1957888 to 2416640 tid 2 2024-09-12T23:51:59.634 INFO:tasks.rados.rados.0.smithi136.stdout:3937: writing smithi13667081-34 from 2416640 to 2760704 tid 3 2024-09-12T23:51:59.634 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 30 current snap is 385 2024-09-12T23:51:59.634 INFO:tasks.rados.rados.0.smithi136.stdout:3938: seq_num 1239 ranges {1105920=753664,1859584=745472,2605056=57344} 2024-09-12T23:51:59.645 INFO:tasks.rados.rados.0.smithi136.stdout:3938: writing smithi13667081-30 from 1105920 to 1859584 tid 1 2024-09-12T23:51:59.650 INFO:tasks.rados.rados.0.smithi136.stdout:3938: writing smithi13667081-30 from 1859584 to 2605056 tid 2 2024-09-12T23:51:59.651 INFO:tasks.rados.rados.0.smithi136.stdout:3938: writing smithi13667081-30 from 2605056 to 2662400 tid 3 2024-09-12T23:51:59.651 INFO:tasks.rados.rados.0.smithi136.stdout:3939: setattr oid 10 current snap is 385 2024-09-12T23:51:59.652 INFO:tasks.rados.rados.0.smithi136.stdout:3933: expect (ObjNum 1178 snap 369 seq_num 1178) 2024-09-12T23:51:59.671 INFO:tasks.rados.rados.0.smithi136.stdout:3930: finishing write tid 5 to smithi13667081-36 2024-09-12T23:51:59.671 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 36 v 2696 (ObjNum 1236 snap 385 seq_num 1236) dirty exists 2024-09-12T23:51:59.671 INFO:tasks.rados.rados.0.smithi136.stdout:3930: left oid 36 (ObjNum 1236 snap 385 seq_num 1236) 2024-09-12T23:51:59.671 INFO:tasks.rados.rados.0.smithi136.stdout:3932: finishing copy_from racing read to smithi13667081-38 2024-09-12T23:51:59.671 INFO:tasks.rados.rados.0.smithi136.stdout:3932: finishing copy_from to smithi13667081-38 2024-09-12T23:51:59.671 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 38 v 2368 (ObjNum 1164 snap 359 seq_num 1164) dirty exists 2024-09-12T23:51:59.671 INFO:tasks.rados.rados.0.smithi136.stdout:3934: finishing write tid 1 to smithi13667081-33 2024-09-12T23:51:59.672 INFO:tasks.rados.rados.0.smithi136.stdout:3934: finishing write tid 2 to smithi13667081-33 2024-09-12T23:51:59.672 INFO:tasks.rados.rados.0.smithi136.stdout:3934: finishing write tid 3 to smithi13667081-33 2024-09-12T23:51:59.672 INFO:tasks.rados.rados.0.smithi136.stdout:3934: finishing write tid 4 to smithi13667081-33 2024-09-12T23:51:59.672 INFO:tasks.rados.rados.0.smithi136.stdout:3934: finishing write tid 5 to smithi13667081-33 2024-09-12T23:51:59.672 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 33 v 2725 (ObjNum 1237 snap 385 seq_num 1237) dirty exists 2024-09-12T23:51:59.672 INFO:tasks.rados.rados.0.smithi136.stdout:3934: left oid 33 (ObjNum 1237 snap 385 seq_num 1237) 2024-09-12T23:51:59.672 INFO:tasks.rados.rados.0.smithi136.stdout:3937: finishing write tid 1 to smithi13667081-34 2024-09-12T23:51:59.672 INFO:tasks.rados.rados.0.smithi136.stdout:3937: finishing write tid 2 to smithi13667081-34 2024-09-12T23:51:59.672 INFO:tasks.rados.rados.0.smithi136.stdout:3936: expect (ObjNum 1081 snap 326 seq_num 1081) 2024-09-12T23:51:59.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:51:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:51:59] "GET /metrics HTTP/1.1" 200 36439 "" "Prometheus/2.43.0" 2024-09-12T23:51:59.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:59 smithi136 bash[75164]: cluster 2024-09-12T23:51:58.602145+0000 mon.a (mon.0) 1432 : cluster [DBG] osdmap e849: 8 total, 8 up, 8 in 2024-09-12T23:51:59.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:51:59 smithi136 bash[75164]: cluster 2024-09-12T23:51:58.888485+0000 mgr.y (mgr.24661) 445 : cluster [DBG] pgmap v856: 121 pgs: 121 active+clean; 273 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 4.9 MiB/s wr, 12 op/s 2024-09-12T23:51:59.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:59 smithi136 bash[76371]: cluster 2024-09-12T23:51:58.602145+0000 mon.a (mon.0) 1432 : cluster [DBG] osdmap e849: 8 total, 8 up, 8 in 2024-09-12T23:51:59.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:51:59 smithi136 bash[76371]: cluster 2024-09-12T23:51:58.888485+0000 mgr.y (mgr.24661) 445 : cluster [DBG] pgmap v856: 121 pgs: 121 active+clean; 273 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 4.9 MiB/s wr, 12 op/s 2024-09-12T23:51:59.922 INFO:tasks.rados.rados.0.smithi136.stdout:3938: finishing write tid 1 to smithi13667081-30 2024-09-12T23:51:59.922 INFO:tasks.rados.rados.0.smithi136.stdout:3937: finishing write tid 3 to smithi13667081-34 2024-09-12T23:51:59.922 INFO:tasks.rados.rados.0.smithi136.stdout:3937: finishing write tid 4 to smithi13667081-34 2024-09-12T23:51:59.923 INFO:tasks.rados.rados.0.smithi136.stdout:3938: finishing write tid 2 to smithi13667081-30 2024-09-12T23:51:59.923 INFO:tasks.rados.rados.0.smithi136.stdout:3938: finishing write tid 3 to smithi13667081-30 2024-09-12T23:51:59.923 INFO:tasks.rados.rados.0.smithi136.stdout:3938: finishing write tid 4 to smithi13667081-30 2024-09-12T23:51:59.923 INFO:tasks.rados.rados.0.smithi136.stdout:3937: finishing write tid 5 to smithi13667081-34 2024-09-12T23:51:59.923 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 34 v 2729 (ObjNum 1238 snap 385 seq_num 1238) dirty exists 2024-09-12T23:51:59.924 INFO:tasks.rados.rados.0.smithi136.stdout:3937: left oid 34 (ObjNum 1238 snap 385 seq_num 1238) 2024-09-12T23:51:59.924 INFO:tasks.rados.rados.0.smithi136.stdout:3938: finishing write tid 5 to smithi13667081-30 2024-09-12T23:51:59.924 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 30 v 2606 (ObjNum 1239 snap 385 seq_num 1239) dirty exists 2024-09-12T23:51:59.924 INFO:tasks.rados.rados.0.smithi136.stdout:3938: left oid 30 (ObjNum 1239 snap 385 seq_num 1239) 2024-09-12T23:51:59.924 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 10 v 2607 (ObjNum 1092 snap 330 seq_num 1092) dirty exists 2024-09-12T23:51:59.924 INFO:tasks.rados.rados.0.smithi136.stdout:3930: done (7 left) 2024-09-12T23:51:59.925 INFO:tasks.rados.rados.0.smithi136.stdout:3932: done (6 left) 2024-09-12T23:51:59.925 INFO:tasks.rados.rados.0.smithi136.stdout:3933: done (5 left) 2024-09-12T23:51:59.925 INFO:tasks.rados.rados.0.smithi136.stdout:3934: done (4 left) 2024-09-12T23:51:59.925 INFO:tasks.rados.rados.0.smithi136.stdout:3936: done (3 left) 2024-09-12T23:51:59.925 INFO:tasks.rados.rados.0.smithi136.stdout:3937: done (2 left) 2024-09-12T23:51:59.926 INFO:tasks.rados.rados.0.smithi136.stdout:3938: done (1 left) 2024-09-12T23:51:59.926 INFO:tasks.rados.rados.0.smithi136.stdout:3939: done (0 left) 2024-09-12T23:51:59.926 INFO:tasks.rados.rados.0.smithi136.stdout:3940: rollback oid 19 current snap is 385 2024-09-12T23:51:59.926 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 19 to 384 2024-09-12T23:51:59.926 INFO:tasks.rados.rados.0.smithi136.stdout:3941: snap_remove snap 378 2024-09-12T23:52:00.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:59 smithi181 bash[67587]: cluster 2024-09-12T23:51:58.602145+0000 mon.a (mon.0) 1432 : cluster [DBG] osdmap e849: 8 total, 8 up, 8 in 2024-09-12T23:52:00.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:51:59 smithi181 bash[67587]: cluster 2024-09-12T23:51:58.888485+0000 mgr.y (mgr.24661) 445 : cluster [DBG] pgmap v856: 121 pgs: 121 active+clean; 273 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 4.9 MiB/s wr, 12 op/s 2024-09-12T23:52:00.623 INFO:tasks.rados.rados.0.smithi136.stdout:3941: done (1 left) 2024-09-12T23:52:00.623 INFO:tasks.rados.rados.0.smithi136.stdout:3942: rmattr oid 2 current snap is 385 2024-09-12T23:52:00.623 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 22 current snap is 385 2024-09-12T23:52:00.623 INFO:tasks.rados.rados.0.smithi136.stdout:3943: seq_num 1240 ranges {1794048=778240,2572288=671744,3244032=131072} 2024-09-12T23:52:00.638 INFO:tasks.rados.rados.0.smithi136.stdout:3943: writing smithi13667081-22 from 1794048 to 2572288 tid 1 2024-09-12T23:52:00.643 INFO:tasks.rados.rados.0.smithi136.stdout:3943: writing smithi13667081-22 from 2572288 to 3244032 tid 2 2024-09-12T23:52:00.646 INFO:tasks.rados.rados.0.smithi136.stdout:3943: writing smithi13667081-22 from 3244032 to 3375104 tid 3 2024-09-12T23:52:00.646 INFO:tasks.rados.rados.0.smithi136.stdout:3940: finishing rollback tid 0 to smithi13667081-19 2024-09-12T23:52:00.646 INFO:tasks.rados.rados.0.smithi136.stdout:3940: finishing rollback tid 1 to smithi13667081-19 2024-09-12T23:52:00.646 INFO:tasks.rados.rados.0.smithi136.stdout:3940: finishing rollback tid 2 to smithi13667081-19 2024-09-12T23:52:00.647 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 19 v 2506 (ObjNum 1208 snap 381 seq_num 1208) dirty exists 2024-09-12T23:52:00.647 INFO:tasks.rados.rados.0.smithi136.stdout:3940: done (2 left) 2024-09-12T23:52:00.647 INFO:tasks.rados.rados.0.smithi136.stdout:3944: rmattr oid 9 current snap is 385 2024-09-12T23:52:00.647 INFO:tasks.rados.rados.0.smithi136.stdout:3944: done (2 left) 2024-09-12T23:52:00.647 INFO:tasks.rados.rados.0.smithi136.stdout:3945: read oid 26 snap 384 2024-09-12T23:52:00.648 INFO:tasks.rados.rados.0.smithi136.stdout:3945: expect (ObjNum 1231 snap 384 seq_num 1231) 2024-09-12T23:52:00.648 INFO:tasks.rados.rados.0.smithi136.stdout:3946: read oid 12 snap -1 2024-09-12T23:52:00.648 INFO:tasks.rados.rados.0.smithi136.stdout:3946: expect (ObjNum 1092 snap 330 seq_num 1092) 2024-09-12T23:52:00.648 INFO:tasks.rados.rados.0.smithi136.stdout:3947: read oid 23 snap -1 2024-09-12T23:52:00.648 INFO:tasks.rados.rados.0.smithi136.stdout:3947: expect (ObjNum 1181 snap 372 seq_num 1181) 2024-09-12T23:52:00.648 INFO:tasks.rados.rados.0.smithi136.stdout:3948: setattr oid 13 current snap is 385 2024-09-12T23:52:00.650 INFO:tasks.rados.rados.0.smithi136.stdout:3949: copy_from oid 27 from oid 49 current snap is 385 2024-09-12T23:52:00.650 INFO:tasks.rados.rados.0.smithi136.stdout:3950: read oid 6 snap -1 2024-09-12T23:52:00.650 INFO:tasks.rados.rados.0.smithi136.stdout:3950: expect deleted 2024-09-12T23:52:00.650 INFO:tasks.rados.rados.0.smithi136.stdout:3951: snap_remove snap 374 2024-09-12T23:52:01.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:00 smithi136 bash[75164]: cluster 2024-09-12T23:51:59.614133+0000 mon.a (mon.0) 1433 : cluster [DBG] osdmap e850: 8 total, 8 up, 8 in 2024-09-12T23:52:01.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:00 smithi136 bash[76371]: cluster 2024-09-12T23:51:59.614133+0000 mon.a (mon.0) 1433 : cluster [DBG] osdmap e850: 8 total, 8 up, 8 in 2024-09-12T23:52:01.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:00 smithi181 bash[67587]: cluster 2024-09-12T23:51:59.614133+0000 mon.a (mon.0) 1433 : cluster [DBG] osdmap e850: 8 total, 8 up, 8 in 2024-09-12T23:52:01.552 INFO:tasks.rados.rados.0.smithi136.stdout:3951: done (8 left) 2024-09-12T23:52:01.552 INFO:tasks.rados.rados.0.smithi136.stdout:3952: rollback oid 29 current snap is 385 2024-09-12T23:52:01.552 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 29 to 379 2024-09-12T23:52:01.552 INFO:tasks.rados.rados.0.smithi136.stdout:3953: rollback oid 15 current snap is 385 2024-09-12T23:52:01.552 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 15 to 383 2024-09-12T23:52:01.552 INFO:tasks.rados.rados.0.smithi136.stdout:3954: copy_from oid 4 from oid 43 current snap is 385 2024-09-12T23:52:01.552 INFO:tasks.rados.rados.0.smithi136.stdout:3955: setattr oid 7 current snap is 385 2024-09-12T23:52:01.560 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 13 v 2709 (ObjNum 1199 snap 376 seq_num 1199) dirty exists 2024-09-12T23:52:01.560 INFO:tasks.rados.rados.0.smithi136.stdout:3947: expect (ObjNum 1181 snap 372 seq_num 1181) 2024-09-12T23:52:01.576 INFO:tasks.rados.rados.0.smithi136.stdout:3947: done (11 left) 2024-09-12T23:52:01.576 INFO:tasks.rados.rados.0.smithi136.stdout:3948: done (10 left) 2024-09-12T23:52:01.576 INFO:tasks.rados.rados.0.smithi136.stdout:3950: done (9 left) 2024-09-12T23:52:01.576 INFO:tasks.rados.rados.0.smithi136.stdout:3956: rollback oid 6 current snap is 385 2024-09-12T23:52:01.576 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 6 to 350 2024-09-12T23:52:01.576 INFO:tasks.rados.rados.0.smithi136.stdout:3957: read oid 40 snap -1 2024-09-12T23:52:01.576 INFO:tasks.rados.rados.0.smithi136.stdout:3957: expect (ObjNum 1233 snap 384 seq_num 1233) 2024-09-12T23:52:01.577 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 39 current snap is 385 2024-09-12T23:52:01.577 INFO:tasks.rados.rados.0.smithi136.stdout:3958: seq_num 1241 ranges {0=794624,794624=557056,1351680=245760} 2024-09-12T23:52:01.581 INFO:tasks.rados.rados.0.smithi136.stdout:3958: writing smithi13667081-39 from 0 to 794624 tid 1 2024-09-12T23:52:01.585 INFO:tasks.rados.rados.0.smithi136.stdout:3958: writing smithi13667081-39 from 794624 to 1351680 tid 2 2024-09-12T23:52:01.587 INFO:tasks.rados.rados.0.smithi136.stdout:3958: writing smithi13667081-39 from 1351680 to 1597440 tid 3 2024-09-12T23:52:01.587 INFO:tasks.rados.rados.0.smithi136.stdout:3959: read oid 19 snap -1 2024-09-12T23:52:01.587 INFO:tasks.rados.rados.0.smithi136.stdout:3959: expect (ObjNum 1208 snap 381 seq_num 1208) 2024-09-12T23:52:01.587 INFO:tasks.rados.rados.0.smithi136.stdout:3960: snap_remove snap 376 2024-09-12T23:52:02.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:01 smithi136 bash[75164]: cluster 2024-09-12T23:52:00.623145+0000 mon.a (mon.0) 1434 : cluster [DBG] osdmap e851: 8 total, 8 up, 8 in 2024-09-12T23:52:02.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:01 smithi136 bash[75164]: cluster 2024-09-12T23:52:00.889422+0000 mgr.y (mgr.24661) 446 : cluster [DBG] pgmap v859: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 288 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.3 MiB/s rd, 5.9 MiB/s wr, 15 op/s 2024-09-12T23:52:02.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:01 smithi136 bash[75164]: cluster 2024-09-12T23:52:01.554723+0000 mon.a (mon.0) 1435 : cluster [DBG] osdmap e852: 8 total, 8 up, 8 in 2024-09-12T23:52:02.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:01 smithi136 bash[76371]: cluster 2024-09-12T23:52:00.623145+0000 mon.a (mon.0) 1434 : cluster [DBG] osdmap e851: 8 total, 8 up, 8 in 2024-09-12T23:52:02.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:01 smithi136 bash[76371]: cluster 2024-09-12T23:52:00.889422+0000 mgr.y (mgr.24661) 446 : cluster [DBG] pgmap v859: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 288 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.3 MiB/s rd, 5.9 MiB/s wr, 15 op/s 2024-09-12T23:52:02.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:01 smithi136 bash[76371]: cluster 2024-09-12T23:52:01.554723+0000 mon.a (mon.0) 1435 : cluster [DBG] osdmap e852: 8 total, 8 up, 8 in 2024-09-12T23:52:02.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:01 smithi181 bash[67587]: cluster 2024-09-12T23:52:00.623145+0000 mon.a (mon.0) 1434 : cluster [DBG] osdmap e851: 8 total, 8 up, 8 in 2024-09-12T23:52:02.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:01 smithi181 bash[67587]: cluster 2024-09-12T23:52:00.889422+0000 mgr.y (mgr.24661) 446 : cluster [DBG] pgmap v859: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 288 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.3 MiB/s rd, 5.9 MiB/s wr, 15 op/s 2024-09-12T23:52:02.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:01 smithi181 bash[67587]: cluster 2024-09-12T23:52:01.554723+0000 mon.a (mon.0) 1435 : cluster [DBG] osdmap e852: 8 total, 8 up, 8 in 2024-09-12T23:52:02.557 INFO:tasks.rados.rados.0.smithi136.stdout:3960: done (13 left) 2024-09-12T23:52:02.558 INFO:tasks.rados.rados.0.smithi136.stdout:3961: read oid 14 snap -1 2024-09-12T23:52:02.558 INFO:tasks.rados.rados.0.smithi136.stdout:3961: expect (ObjNum 1210 snap 381 seq_num 1210) 2024-09-12T23:52:02.558 INFO:tasks.rados.rados.0.smithi136.stdout:3962: snap_remove snap 382 2024-09-12T23:52:02.558 INFO:tasks.rados.rados.0.smithi136.stdout:3949: finishing copy_from to smithi13667081-27 2024-09-12T23:52:02.558 INFO:tasks.rados.rados.0.smithi136.stdout:3949: got expected ENOENT (src dne) 2024-09-12T23:52:02.558 INFO:tasks.rados.rados.0.smithi136.stdout:3949: finishing copy_from racing read to smithi13667081-27 2024-09-12T23:52:02.558 INFO:tasks.rados.rados.0.smithi136.stdout:3945: expect (ObjNum 1231 snap 384 seq_num 1231) 2024-09-12T23:52:02.596 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:52:02 smithi181 bash[95505]: ts=2024-09-12T23:52:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:52:02.611 INFO:tasks.rados.rados.0.smithi136.stdout:3943: finishing write tid 1 to smithi13667081-22 2024-09-12T23:52:02.611 INFO:tasks.rados.rados.0.smithi136.stdout:3943: finishing write tid 2 to smithi13667081-22 2024-09-12T23:52:03.567 INFO:tasks.rados.rados.0.smithi136.stdout:3945: done (14 left) 2024-09-12T23:52:03.567 INFO:tasks.rados.rados.0.smithi136.stdout:3949: done (13 left) 2024-09-12T23:52:03.568 INFO:tasks.rados.rados.0.smithi136.stdout:3962: done (12 left) 2024-09-12T23:52:03.568 INFO:tasks.rados.rados.0.smithi136.stdout:3963: copy_from oid 23 from oid 3 current snap is 385 2024-09-12T23:52:03.568 INFO:tasks.rados.rados.0.smithi136.stdout:3943: finishing write tid 3 to smithi13667081-22 2024-09-12T23:52:03.568 INFO:tasks.rados.rados.0.smithi136.stdout:3943: finishing write tid 4 to smithi13667081-22 2024-09-12T23:52:03.568 INFO:tasks.rados.rados.0.smithi136.stdout:3943: finishing write tid 5 to smithi13667081-22 2024-09-12T23:52:03.568 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 22 v 2538 (ObjNum 1240 snap 385 seq_num 1240) dirty exists 2024-09-12T23:52:03.568 INFO:tasks.rados.rados.0.smithi136.stdout:3943: left oid 22 (ObjNum 1240 snap 385 seq_num 1240) 2024-09-12T23:52:03.568 INFO:tasks.rados.rados.0.smithi136.stdout:3946: expect (ObjNum 1092 snap 330 seq_num 1092) 2024-09-12T23:52:03.820 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 2 v 2730 (ObjNum 1234 snap 384 seq_num 1234) dirty exists 2024-09-12T23:52:03.820 INFO:tasks.rados.rados.0.smithi136.stdout:3952: finishing rollback tid 1 to smithi13667081-29 2024-09-12T23:52:03.820 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 29 v 2693 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-12T23:52:03.820 INFO:tasks.rados.rados.0.smithi136.stdout:3956: finishing rollback tid 1 to smithi13667081-6 2024-09-12T23:52:03.820 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 6 v 0 (ObjNum 161 snap 0 seq_num 0) dirty dne 2024-09-12T23:52:03.820 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 7 v 1657 (ObjNum 81 snap 0 seq_num 1466200926) dirty exists 2024-09-12T23:52:03.820 INFO:tasks.rados.rados.0.smithi136.stdout:3953: finishing rollback tid 0 to smithi13667081-15 2024-09-12T23:52:03.820 INFO:tasks.rados.rados.0.smithi136.stdout:3957: expect (ObjNum 1233 snap 384 seq_num 1233) 2024-09-12T23:52:03.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:03 smithi181 bash[67587]: cluster 2024-09-12T23:52:02.559488+0000 mon.a (mon.0) 1436 : cluster [DBG] osdmap e853: 8 total, 8 up, 8 in 2024-09-12T23:52:03.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:03 smithi181 bash[67587]: cluster 2024-09-12T23:52:02.890313+0000 mgr.y (mgr.24661) 447 : cluster [DBG] pgmap v862: 121 pgs: 1 active+clean+snaptrim_wait, 4 active+clean+snaptrim, 116 active+clean; 301 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 7.1 MiB/s wr, 28 op/s 2024-09-12T23:52:03.856 INFO:tasks.rados.rados.0.smithi136.stdout:3958: finishing write tid 1 to smithi13667081-39 2024-09-12T23:52:03.856 INFO:tasks.rados.rados.0.smithi136.stdout:3958: finishing write tid 2 to smithi13667081-39 2024-09-12T23:52:03.856 INFO:tasks.rados.rados.0.smithi136.stdout:3958: finishing write tid 3 to smithi13667081-39 2024-09-12T23:52:03.856 INFO:tasks.rados.rados.0.smithi136.stdout:3953: finishing rollback tid 1 to smithi13667081-15 2024-09-12T23:52:03.856 INFO:tasks.rados.rados.0.smithi136.stdout:3953: finishing rollback tid 2 to smithi13667081-15 2024-09-12T23:52:03.857 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 15 v 2518 (ObjNum 1217 snap 382 seq_num 1217) dirty exists 2024-09-12T23:52:03.857 INFO:tasks.rados.rados.0.smithi136.stdout:3958: finishing write tid 4 to smithi13667081-39 2024-09-12T23:52:03.857 INFO:tasks.rados.rados.0.smithi136.stdout:3958: finishing write tid 5 to smithi13667081-39 2024-09-12T23:52:03.857 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 39 v 2939 (ObjNum 1241 snap 385 seq_num 1241) dirty exists 2024-09-12T23:52:03.857 INFO:tasks.rados.rados.0.smithi136.stdout:3958: left oid 39 (ObjNum 1241 snap 385 seq_num 1241) 2024-09-12T23:52:03.857 INFO:tasks.rados.rados.0.smithi136.stdout:3954: finishing copy_from racing read to smithi13667081-4 2024-09-12T23:52:03.857 INFO:tasks.rados.rados.0.smithi136.stdout:3959: expect (ObjNum 1208 snap 381 seq_num 1208) 2024-09-12T23:52:03.914 INFO:tasks.rados.rados.0.smithi136.stdout:3954: finishing copy_from to smithi13667081-4 2024-09-12T23:52:03.914 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 4 v 2567 (ObjNum 1232 snap 384 seq_num 1232) dirty exists 2024-09-12T23:52:03.914 INFO:tasks.rados.rados.0.smithi136.stdout:3961: expect (ObjNum 1210 snap 381 seq_num 1210) 2024-09-12T23:52:04.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:03 smithi136 bash[75164]: cluster 2024-09-12T23:52:02.559488+0000 mon.a (mon.0) 1436 : cluster [DBG] osdmap e853: 8 total, 8 up, 8 in 2024-09-12T23:52:04.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:03 smithi136 bash[75164]: cluster 2024-09-12T23:52:02.890313+0000 mgr.y (mgr.24661) 447 : cluster [DBG] pgmap v862: 121 pgs: 1 active+clean+snaptrim_wait, 4 active+clean+snaptrim, 116 active+clean; 301 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 7.1 MiB/s wr, 28 op/s 2024-09-12T23:52:04.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:03 smithi136 bash[76371]: cluster 2024-09-12T23:52:02.559488+0000 mon.a (mon.0) 1436 : cluster [DBG] osdmap e853: 8 total, 8 up, 8 in 2024-09-12T23:52:04.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:03 smithi136 bash[76371]: cluster 2024-09-12T23:52:02.890313+0000 mgr.y (mgr.24661) 447 : cluster [DBG] pgmap v862: 121 pgs: 1 active+clean+snaptrim_wait, 4 active+clean+snaptrim, 116 active+clean; 301 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 7.1 MiB/s wr, 28 op/s 2024-09-12T23:52:04.101 INFO:tasks.rados.rados.0.smithi136.stdout:3963: finishing copy_from racing read to smithi13667081-23 2024-09-12T23:52:04.101 INFO:tasks.rados.rados.0.smithi136.stdout:3963: finishing copy_from to smithi13667081-23 2024-09-12T23:52:04.101 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 23 v 2511 (ObjNum 1198 snap 376 seq_num 1198) dirty exists 2024-09-12T23:52:04.101 INFO:tasks.rados.rados.0.smithi136.stdout:3942: done (12 left) 2024-09-12T23:52:04.101 INFO:tasks.rados.rados.0.smithi136.stdout:3943: done (11 left) 2024-09-12T23:52:04.101 INFO:tasks.rados.rados.0.smithi136.stdout:3946: done (10 left) 2024-09-12T23:52:04.102 INFO:tasks.rados.rados.0.smithi136.stdout:3952: done (9 left) 2024-09-12T23:52:04.102 INFO:tasks.rados.rados.0.smithi136.stdout:3953: done (8 left) 2024-09-12T23:52:04.102 INFO:tasks.rados.rados.0.smithi136.stdout:3954: done (7 left) 2024-09-12T23:52:04.102 INFO:tasks.rados.rados.0.smithi136.stdout:3955: done (6 left) 2024-09-12T23:52:04.102 INFO:tasks.rados.rados.0.smithi136.stdout:3956: done (5 left) 2024-09-12T23:52:04.102 INFO:tasks.rados.rados.0.smithi136.stdout:3957: done (4 left) 2024-09-12T23:52:04.102 INFO:tasks.rados.rados.0.smithi136.stdout:3958: done (3 left) 2024-09-12T23:52:04.102 INFO:tasks.rados.rados.0.smithi136.stdout:3959: done (2 left) 2024-09-12T23:52:04.102 INFO:tasks.rados.rados.0.smithi136.stdout:3961: done (1 left) 2024-09-12T23:52:04.102 INFO:tasks.rados.rados.0.smithi136.stdout:3963: done (0 left) 2024-09-12T23:52:04.102 INFO:tasks.rados.rados.0.smithi136.stdout:3964: read oid 40 snap 384 2024-09-12T23:52:04.102 INFO:tasks.rados.rados.0.smithi136.stdout:3964: expect (ObjNum 1233 snap 384 seq_num 1233) 2024-09-12T23:52:04.102 INFO:tasks.rados.rados.0.smithi136.stdout:3965: snap_create 2024-09-12T23:52:04.107 INFO:tasks.rados.rados.0.smithi136.stdout:3964: expect (ObjNum 1233 snap 384 seq_num 1233) 2024-09-12T23:52:04.578 INFO:tasks.rados.rados.0.smithi136.stdout:3964: done (1 left) 2024-09-12T23:52:04.578 INFO:tasks.rados.rados.0.smithi136.stdout:3965: done (0 left) 2024-09-12T23:52:04.578 INFO:tasks.rados.rados.0.smithi136.stdout:3966: rollback oid 10 current snap is 386 2024-09-12T23:52:04.578 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 10 to 379 2024-09-12T23:52:04.578 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 46 current snap is 386 2024-09-12T23:52:04.578 INFO:tasks.rados.rados.0.smithi136.stdout:3967: seq_num 1242 ranges {0=409600,409600=442368,851968=655360,1507328=106496} 2024-09-12T23:52:04.585 INFO:tasks.rados.rados.0.smithi136.stdout:3967: writing smithi13667081-46 from 0 to 409600 tid 1 2024-09-12T23:52:04.593 INFO:tasks.rados.rados.0.smithi136.stdout:3967: writing smithi13667081-46 from 409600 to 851968 tid 2 2024-09-12T23:52:04.604 INFO:tasks.rados.rados.0.smithi136.stdout:3967: writing smithi13667081-46 from 851968 to 1507328 tid 3 2024-09-12T23:52:04.607 INFO:tasks.rados.rados.0.smithi136.stdout:3967: writing smithi13667081-46 from 1507328 to 1613824 tid 4 2024-09-12T23:52:04.607 INFO:tasks.rados.rados.0.smithi136.stdout:3968: snap_create 2024-09-12T23:52:04.607 INFO:tasks.rados.rados.0.smithi136.stdout:3966: finishing rollback tid 0 to smithi13667081-10 2024-09-12T23:52:04.607 INFO:tasks.rados.rados.0.smithi136.stdout:3966: finishing rollback tid 1 to smithi13667081-10 2024-09-12T23:52:04.607 INFO:tasks.rados.rados.0.smithi136.stdout:3966: finishing rollback tid 2 to smithi13667081-10 2024-09-12T23:52:04.607 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 10 v 2610 (ObjNum 1145 snap 350 seq_num 1145) dirty exists 2024-09-12T23:52:04.607 INFO:tasks.rados.rados.0.smithi136.stdout:3967: finishing write tid 1 to smithi13667081-46 2024-09-12T23:52:04.607 INFO:tasks.rados.rados.0.smithi136.stdout:3967: finishing write tid 2 to smithi13667081-46 2024-09-12T23:52:04.613 INFO:tasks.rados.rados.0.smithi136.stdout:3967: finishing write tid 3 to smithi13667081-46 2024-09-12T23:52:04.613 INFO:tasks.rados.rados.0.smithi136.stdout:3967: finishing write tid 4 to smithi13667081-46 2024-09-12T23:52:04.614 INFO:tasks.rados.rados.0.smithi136.stdout:3967: finishing write tid 5 to smithi13667081-46 2024-09-12T23:52:04.616 INFO:tasks.rados.rados.0.smithi136.stdout:3967: finishing write tid 6 to smithi13667081-46 2024-09-12T23:52:04.616 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 46 v 2506 (ObjNum 1242 snap 386 seq_num 1242) dirty exists 2024-09-12T23:52:04.617 INFO:tasks.rados.rados.0.smithi136.stdout:3967: left oid 46 (ObjNum 1242 snap 386 seq_num 1242) 2024-09-12T23:52:04.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:04 smithi181 bash[67587]: cluster 2024-09-12T23:52:03.569653+0000 mon.a (mon.0) 1437 : cluster [DBG] osdmap e854: 8 total, 8 up, 8 in 2024-09-12T23:52:05.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:04 smithi136 bash[75164]: cluster 2024-09-12T23:52:03.569653+0000 mon.a (mon.0) 1437 : cluster [DBG] osdmap e854: 8 total, 8 up, 8 in 2024-09-12T23:52:05.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:04 smithi136 bash[76371]: cluster 2024-09-12T23:52:03.569653+0000 mon.a (mon.0) 1437 : cluster [DBG] osdmap e854: 8 total, 8 up, 8 in 2024-09-12T23:52:05.586 INFO:tasks.rados.rados.0.smithi136.stdout:3966: done (2 left) 2024-09-12T23:52:05.586 INFO:tasks.rados.rados.0.smithi136.stdout:3967: done (1 left) 2024-09-12T23:52:05.587 INFO:tasks.rados.rados.0.smithi136.stdout:3968: done (0 left) 2024-09-12T23:52:05.587 INFO:tasks.rados.rados.0.smithi136.stdout:3969: copy_from oid 13 from oid 29 current snap is 387 2024-09-12T23:52:05.587 INFO:tasks.rados.rados.0.smithi136.stdout:3970: setattr oid 33 current snap is 387 2024-09-12T23:52:05.590 INFO:tasks.rados.rados.0.smithi136.stdout:3971: setattr oid 30 current snap is 387 2024-09-12T23:52:05.592 INFO:tasks.rados.rados.0.smithi136.stdout:3969: finishing copy_from to smithi13667081-13 2024-09-12T23:52:05.592 INFO:tasks.rados.rados.0.smithi136.stdout:3969: got expected ENOENT (src dne) 2024-09-12T23:52:05.592 INFO:tasks.rados.rados.0.smithi136.stdout:3969: finishing copy_from racing read to smithi13667081-13 2024-09-12T23:52:05.594 INFO:tasks.rados.rados.0.smithi136.stdout:3969: done (2 left) 2024-09-12T23:52:05.594 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 25 current snap is 387 2024-09-12T23:52:05.594 INFO:tasks.rados.rados.0.smithi136.stdout:3972: seq_num 1243 ranges {5357568=425984,5783552=729088,6512640=466944,6979584=8192} 2024-09-12T23:52:05.663 INFO:tasks.rados.rados.0.smithi136.stdout:3972: writing smithi13667081-25 from 5357568 to 5783552 tid 1 2024-09-12T23:52:05.668 INFO:tasks.rados.rados.0.smithi136.stdout:3972: writing smithi13667081-25 from 5783552 to 6512640 tid 2 2024-09-12T23:52:05.672 INFO:tasks.rados.rados.0.smithi136.stdout:3972: writing smithi13667081-25 from 6512640 to 6979584 tid 3 2024-09-12T23:52:05.672 INFO:tasks.rados.rados.0.smithi136.stdout:3972: writing smithi13667081-25 from 6979584 to 6987776 tid 4 2024-09-12T23:52:05.673 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 8 current snap is 387 2024-09-12T23:52:05.673 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 33 v 2731 (ObjNum 1237 snap 385 seq_num 1237) dirty exists 2024-09-12T23:52:05.673 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 30 v 2611 (ObjNum 1239 snap 385 seq_num 1239) dirty exists 2024-09-12T23:52:05.673 INFO:tasks.rados.rados.0.smithi136.stdout:3973: seq_num 1244 ranges {4497408=442368,4939776=614400,5554176=589824} 2024-09-12T23:52:05.705 INFO:tasks.rados.rados.0.smithi136.stdout:3973: writing smithi13667081-8 from 4497408 to 4939776 tid 1 2024-09-12T23:52:05.710 INFO:tasks.rados.rados.0.smithi136.stdout:3973: writing smithi13667081-8 from 4939776 to 5554176 tid 2 2024-09-12T23:52:05.714 INFO:tasks.rados.rados.0.smithi136.stdout:3973: writing smithi13667081-8 from 5554176 to 6144000 tid 3 2024-09-12T23:52:05.714 INFO:tasks.rados.rados.0.smithi136.stdout:3970: done (3 left) 2024-09-12T23:52:05.714 INFO:tasks.rados.rados.0.smithi136.stdout:3971: done (2 left) 2024-09-12T23:52:05.714 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 19 current snap is 387 2024-09-12T23:52:05.714 INFO:tasks.rados.rados.0.smithi136.stdout:3974: seq_num 1245 ranges {1794048=458752,2252800=499712,2752512=425984,3178496=278528} 2024-09-12T23:52:05.728 INFO:tasks.rados.rados.0.smithi136.stdout:3974: writing smithi13667081-19 from 1794048 to 2252800 tid 1 2024-09-12T23:52:05.731 INFO:tasks.rados.rados.0.smithi136.stdout:3974: writing smithi13667081-19 from 2252800 to 2752512 tid 2 2024-09-12T23:52:05.734 INFO:tasks.rados.rados.0.smithi136.stdout:3974: writing smithi13667081-19 from 2752512 to 3178496 tid 3 2024-09-12T23:52:05.736 INFO:tasks.rados.rados.0.smithi136.stdout:3974: writing smithi13667081-19 from 3178496 to 3457024 tid 4 2024-09-12T23:52:05.736 INFO:tasks.rados.rados.0.smithi136.stdout:3975: read oid 46 snap -1 2024-09-12T23:52:05.736 INFO:tasks.rados.rados.0.smithi136.stdout:3975: expect (ObjNum 1242 snap 386 seq_num 1242) 2024-09-12T23:52:05.736 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 49 current snap is 387 2024-09-12T23:52:05.736 INFO:tasks.rados.rados.0.smithi136.stdout:3972: finishing write tid 1 to smithi13667081-25 2024-09-12T23:52:05.736 INFO:tasks.rados.rados.0.smithi136.stdout:3976: seq_num 1246 ranges {0=475136,475136=786432,1261568=417792} 2024-09-12T23:52:05.738 INFO:tasks.rados.rados.0.smithi136.stdout:3976: writing smithi13667081-49 from 0 to 475136 tid 1 2024-09-12T23:52:05.743 INFO:tasks.rados.rados.0.smithi136.stdout:3976: writing smithi13667081-49 from 475136 to 1261568 tid 2 2024-09-12T23:52:05.746 INFO:tasks.rados.rados.0.smithi136.stdout:3976: writing smithi13667081-49 from 1261568 to 1679360 tid 3 2024-09-12T23:52:05.747 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 3 current snap is 387 2024-09-12T23:52:05.747 INFO:tasks.rados.rados.0.smithi136.stdout:3977: seq_num 1247 ranges {1474560=491520,1966080=712704,2678784=442368,3121152=49152} 2024-09-12T23:52:05.757 INFO:tasks.rados.rados.0.smithi136.stdout:3977: writing smithi13667081-3 from 1474560 to 1966080 tid 1 2024-09-12T23:52:05.762 INFO:tasks.rados.rados.0.smithi136.stdout:3977: writing smithi13667081-3 from 1966080 to 2678784 tid 2 2024-09-12T23:52:05.765 INFO:tasks.rados.rados.0.smithi136.stdout:3977: writing smithi13667081-3 from 2678784 to 3121152 tid 3 2024-09-12T23:52:05.765 INFO:tasks.rados.rados.0.smithi136.stdout:3977: writing smithi13667081-3 from 3121152 to 3170304 tid 4 2024-09-12T23:52:05.765 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 17 current snap is 387 2024-09-12T23:52:05.766 INFO:tasks.rados.rados.0.smithi136.stdout:3972: finishing write tid 2 to smithi13667081-25 2024-09-12T23:52:05.766 INFO:tasks.rados.rados.0.smithi136.stdout:3978: seq_num 1248 ranges {0=507904,507904=598016,1105920=606208} 2024-09-12T23:52:05.768 INFO:tasks.rados.rados.0.smithi136.stdout:3978: writing smithi13667081-17 from 0 to 507904 tid 1 2024-09-12T23:52:05.772 INFO:tasks.rados.rados.0.smithi136.stdout:3978: writing smithi13667081-17 from 507904 to 1105920 tid 2 2024-09-12T23:52:05.776 INFO:tasks.rados.rados.0.smithi136.stdout:3978: writing smithi13667081-17 from 1105920 to 1712128 tid 3 2024-09-12T23:52:05.776 INFO:tasks.rados.rados.0.smithi136.stdout:3979: rmattr oid 9 current snap is 387 2024-09-12T23:52:05.776 INFO:tasks.rados.rados.0.smithi136.stdout:3979: done (7 left) 2024-09-12T23:52:05.776 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 28 current snap is 387 2024-09-12T23:52:05.776 INFO:tasks.rados.rados.0.smithi136.stdout:3980: seq_num 1249 ranges {4014080=524288,4538368=483328,5021696=409600,5431296=311296} 2024-09-12T23:52:05.801 INFO:tasks.rados.rados.0.smithi136.stdout:3980: writing smithi13667081-28 from 4014080 to 4538368 tid 1 2024-09-12T23:52:05.804 INFO:tasks.rados.rados.0.smithi136.stdout:3980: writing smithi13667081-28 from 4538368 to 5021696 tid 2 2024-09-12T23:52:05.807 INFO:tasks.rados.rados.0.smithi136.stdout:3980: writing smithi13667081-28 from 5021696 to 5431296 tid 3 2024-09-12T23:52:05.810 INFO:tasks.rados.rados.0.smithi136.stdout:3980: writing smithi13667081-28 from 5431296 to 5742592 tid 4 2024-09-12T23:52:05.810 INFO:tasks.rados.rados.0.smithi136.stdout:3981: snap_create 2024-09-12T23:52:05.810 INFO:tasks.rados.rados.0.smithi136.stdout:3972: finishing write tid 3 to smithi13667081-25 2024-09-12T23:52:05.810 INFO:tasks.rados.rados.0.smithi136.stdout:3972: finishing write tid 4 to smithi13667081-25 2024-09-12T23:52:05.810 INFO:tasks.rados.rados.0.smithi136.stdout:3972: finishing write tid 5 to smithi13667081-25 2024-09-12T23:52:05.810 INFO:tasks.rados.rados.0.smithi136.stdout:3972: finishing write tid 6 to smithi13667081-25 2024-09-12T23:52:05.810 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 25 v 2944 (ObjNum 1243 snap 387 seq_num 1243) dirty exists 2024-09-12T23:52:05.810 INFO:tasks.rados.rados.0.smithi136.stdout:3972: left oid 25 (ObjNum 1243 snap 387 seq_num 1243) 2024-09-12T23:52:05.811 INFO:tasks.rados.rados.0.smithi136.stdout:3973: finishing write tid 1 to smithi13667081-8 2024-09-12T23:52:05.811 INFO:tasks.rados.rados.0.smithi136.stdout:3973: finishing write tid 2 to smithi13667081-8 2024-09-12T23:52:05.811 INFO:tasks.rados.rados.0.smithi136.stdout:3973: finishing write tid 3 to smithi13667081-8 2024-09-12T23:52:05.811 INFO:tasks.rados.rados.0.smithi136.stdout:3973: finishing write tid 4 to smithi13667081-8 2024-09-12T23:52:05.811 INFO:tasks.rados.rados.0.smithi136.stdout:3973: finishing write tid 5 to smithi13667081-8 2024-09-12T23:52:05.811 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 8 v 2700 (ObjNum 1244 snap 387 seq_num 1244) dirty exists 2024-09-12T23:52:05.811 INFO:tasks.rados.rados.0.smithi136.stdout:3973: left oid 8 (ObjNum 1244 snap 387 seq_num 1244) 2024-09-12T23:52:05.811 INFO:tasks.rados.rados.0.smithi136.stdout:3974: finishing write tid 1 to smithi13667081-19 2024-09-12T23:52:05.811 INFO:tasks.rados.rados.0.smithi136.stdout:3974: finishing write tid 2 to smithi13667081-19 2024-09-12T23:52:05.811 INFO:tasks.rados.rados.0.smithi136.stdout:3974: finishing write tid 3 to smithi13667081-19 2024-09-12T23:52:05.811 INFO:tasks.rados.rados.0.smithi136.stdout:3974: finishing write tid 4 to smithi13667081-19 2024-09-12T23:52:05.811 INFO:tasks.rados.rados.0.smithi136.stdout:3974: finishing write tid 5 to smithi13667081-19 2024-09-12T23:52:05.811 INFO:tasks.rados.rados.0.smithi136.stdout:3974: finishing write tid 6 to smithi13667081-19 2024-09-12T23:52:05.811 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 19 v 2511 (ObjNum 1245 snap 387 seq_num 1245) dirty exists 2024-09-12T23:52:05.812 INFO:tasks.rados.rados.0.smithi136.stdout:3974: left oid 19 (ObjNum 1245 snap 387 seq_num 1245) 2024-09-12T23:52:05.812 INFO:tasks.rados.rados.0.smithi136.stdout:3976: finishing write tid 1 to smithi13667081-49 2024-09-12T23:52:05.812 INFO:tasks.rados.rados.0.smithi136.stdout:3976: finishing write tid 2 to smithi13667081-49 2024-09-12T23:52:05.812 INFO:tasks.rados.rados.0.smithi136.stdout:3975: expect (ObjNum 1242 snap 386 seq_num 1242) 2024-09-12T23:52:05.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:05 smithi181 bash[67587]: cluster 2024-09-12T23:52:04.580735+0000 mon.a (mon.0) 1438 : cluster [DBG] osdmap e855: 8 total, 8 up, 8 in 2024-09-12T23:52:05.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:05 smithi181 bash[67587]: cluster 2024-09-12T23:52:04.891191+0000 mgr.y (mgr.24661) 448 : cluster [DBG] pgmap v865: 121 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 118 active+clean; 302 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 14 MiB/s rd, 4.2 MiB/s wr, 36 op/s 2024-09-12T23:52:05.885 INFO:tasks.rados.rados.0.smithi136.stdout:3976: finishing write tid 3 to smithi13667081-49 2024-09-12T23:52:05.885 INFO:tasks.rados.rados.0.smithi136.stdout:3976: finishing write tid 4 to smithi13667081-49 2024-09-12T23:52:05.885 INFO:tasks.rados.rados.0.smithi136.stdout:3976: finishing write tid 5 to smithi13667081-49 2024-09-12T23:52:05.885 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 49 v 2522 (ObjNum 1246 snap 387 seq_num 1246) dirty exists 2024-09-12T23:52:05.885 INFO:tasks.rados.rados.0.smithi136.stdout:3976: left oid 49 (ObjNum 1246 snap 387 seq_num 1246) 2024-09-12T23:52:05.886 INFO:tasks.rados.rados.0.smithi136.stdout:3977: finishing write tid 1 to smithi13667081-3 2024-09-12T23:52:05.886 INFO:tasks.rados.rados.0.smithi136.stdout:3977: finishing write tid 2 to smithi13667081-3 2024-09-12T23:52:05.886 INFO:tasks.rados.rados.0.smithi136.stdout:3977: finishing write tid 3 to smithi13667081-3 2024-09-12T23:52:05.886 INFO:tasks.rados.rados.0.smithi136.stdout:3977: finishing write tid 4 to smithi13667081-3 2024-09-12T23:52:05.886 INFO:tasks.rados.rados.0.smithi136.stdout:3977: finishing write tid 5 to smithi13667081-3 2024-09-12T23:52:05.886 INFO:tasks.rados.rados.0.smithi136.stdout:3978: finishing write tid 1 to smithi13667081-17 2024-09-12T23:52:05.886 INFO:tasks.rados.rados.0.smithi136.stdout:3978: finishing write tid 2 to smithi13667081-17 2024-09-12T23:52:05.886 INFO:tasks.rados.rados.0.smithi136.stdout:3978: finishing write tid 3 to smithi13667081-17 2024-09-12T23:52:05.886 INFO:tasks.rados.rados.0.smithi136.stdout:3978: finishing write tid 4 to smithi13667081-17 2024-09-12T23:52:05.886 INFO:tasks.rados.rados.0.smithi136.stdout:3977: finishing write tid 6 to smithi13667081-3 2024-09-12T23:52:05.886 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 3 v 2527 (ObjNum 1247 snap 387 seq_num 1247) dirty exists 2024-09-12T23:52:05.886 INFO:tasks.rados.rados.0.smithi136.stdout:3977: left oid 3 (ObjNum 1247 snap 387 seq_num 1247) 2024-09-12T23:52:05.886 INFO:tasks.rados.rados.0.smithi136.stdout:3978: finishing write tid 5 to smithi13667081-17 2024-09-12T23:52:05.886 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 17 v 2972 (ObjNum 1248 snap 387 seq_num 1248) dirty exists 2024-09-12T23:52:05.887 INFO:tasks.rados.rados.0.smithi136.stdout:3978: left oid 17 (ObjNum 1248 snap 387 seq_num 1248) 2024-09-12T23:52:05.887 INFO:tasks.rados.rados.0.smithi136.stdout:3980: finishing write tid 1 to smithi13667081-28 2024-09-12T23:52:05.887 INFO:tasks.rados.rados.0.smithi136.stdout:3980: finishing write tid 2 to smithi13667081-28 2024-09-12T23:52:05.887 INFO:tasks.rados.rados.0.smithi136.stdout:3980: finishing write tid 3 to smithi13667081-28 2024-09-12T23:52:05.887 INFO:tasks.rados.rados.0.smithi136.stdout:3980: finishing write tid 4 to smithi13667081-28 2024-09-12T23:52:05.887 INFO:tasks.rados.rados.0.smithi136.stdout:3980: finishing write tid 5 to smithi13667081-28 2024-09-12T23:52:05.887 INFO:tasks.rados.rados.0.smithi136.stdout:3980: finishing write tid 6 to smithi13667081-28 2024-09-12T23:52:05.887 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 28 v 2591 (ObjNum 1249 snap 387 seq_num 1249) dirty exists 2024-09-12T23:52:05.887 INFO:tasks.rados.rados.0.smithi136.stdout:3980: left oid 28 (ObjNum 1249 snap 387 seq_num 1249) 2024-09-12T23:52:06.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:05 smithi136 bash[76371]: cluster 2024-09-12T23:52:04.580735+0000 mon.a (mon.0) 1438 : cluster [DBG] osdmap e855: 8 total, 8 up, 8 in 2024-09-12T23:52:06.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:05 smithi136 bash[76371]: cluster 2024-09-12T23:52:04.891191+0000 mgr.y (mgr.24661) 448 : cluster [DBG] pgmap v865: 121 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 118 active+clean; 302 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 14 MiB/s rd, 4.2 MiB/s wr, 36 op/s 2024-09-12T23:52:06.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:05 smithi136 bash[75164]: cluster 2024-09-12T23:52:04.580735+0000 mon.a (mon.0) 1438 : cluster [DBG] osdmap e855: 8 total, 8 up, 8 in 2024-09-12T23:52:06.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:05 smithi136 bash[75164]: cluster 2024-09-12T23:52:04.891191+0000 mgr.y (mgr.24661) 448 : cluster [DBG] pgmap v865: 121 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 118 active+clean; 302 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 14 MiB/s rd, 4.2 MiB/s wr, 36 op/s 2024-09-12T23:52:06.554 INFO:tasks.rados.rados.0.smithi136.stdout:3972: done (8 left) 2024-09-12T23:52:06.554 INFO:tasks.rados.rados.0.smithi136.stdout:3973: done (7 left) 2024-09-12T23:52:06.554 INFO:tasks.rados.rados.0.smithi136.stdout:3974: done (6 left) 2024-09-12T23:52:06.555 INFO:tasks.rados.rados.0.smithi136.stdout:3975: done (5 left) 2024-09-12T23:52:06.555 INFO:tasks.rados.rados.0.smithi136.stdout:3976: done (4 left) 2024-09-12T23:52:06.555 INFO:tasks.rados.rados.0.smithi136.stdout:3977: done (3 left) 2024-09-12T23:52:06.555 INFO:tasks.rados.rados.0.smithi136.stdout:3978: done (2 left) 2024-09-12T23:52:06.555 INFO:tasks.rados.rados.0.smithi136.stdout:3980: done (1 left) 2024-09-12T23:52:06.555 INFO:tasks.rados.rados.0.smithi136.stdout:3981: done (0 left) 2024-09-12T23:52:06.555 INFO:tasks.rados.rados.0.smithi136.stdout:3982: snap_remove snap 379 2024-09-12T23:52:06.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:06 smithi181 bash[67587]: cluster 2024-09-12T23:52:05.588100+0000 mon.a (mon.0) 1439 : cluster [DBG] osdmap e856: 8 total, 8 up, 8 in 2024-09-12T23:52:06.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:06 smithi181 bash[67587]: cluster 2024-09-12T23:52:06.556879+0000 mon.a (mon.0) 1440 : cluster [DBG] osdmap e857: 8 total, 8 up, 8 in 2024-09-12T23:52:07.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:06 smithi136 bash[76371]: cluster 2024-09-12T23:52:05.588100+0000 mon.a (mon.0) 1439 : cluster [DBG] osdmap e856: 8 total, 8 up, 8 in 2024-09-12T23:52:07.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:06 smithi136 bash[76371]: cluster 2024-09-12T23:52:06.556879+0000 mon.a (mon.0) 1440 : cluster [DBG] osdmap e857: 8 total, 8 up, 8 in 2024-09-12T23:52:07.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:06 smithi136 bash[75164]: cluster 2024-09-12T23:52:05.588100+0000 mon.a (mon.0) 1439 : cluster [DBG] osdmap e856: 8 total, 8 up, 8 in 2024-09-12T23:52:07.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:06 smithi136 bash[75164]: cluster 2024-09-12T23:52:06.556879+0000 mon.a (mon.0) 1440 : cluster [DBG] osdmap e857: 8 total, 8 up, 8 in 2024-09-12T23:52:07.564 INFO:tasks.rados.rados.0.smithi136.stdout:3982: done (0 left) 2024-09-12T23:52:07.564 INFO:tasks.rados.rados.0.smithi136.stdout:3983: copy_from oid 49 from oid 2 current snap is 388 2024-09-12T23:52:07.564 INFO:tasks.rados.rados.0.smithi136.stdout:3984: read oid 18 snap -1 2024-09-12T23:52:07.565 INFO:tasks.rados.rados.0.smithi136.stdout:3984: expect (ObjNum 1206 snap 378 seq_num 1206) 2024-09-12T23:52:07.565 INFO:tasks.rados.rados.0.smithi136.stdout:3985: copy_from oid 12 from oid 4 current snap is 388 2024-09-12T23:52:07.565 INFO:tasks.rados.rados.0.smithi136.stdout:3986: rollback oid 25 current snap is 388 2024-09-12T23:52:07.565 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 25 to 385 2024-09-12T23:52:07.565 INFO:tasks.rados.rados.0.smithi136.stdout:3987: snap_create 2024-09-12T23:52:07.588 INFO:tasks.rados.rados.0.smithi136.stdout:3984: expect (ObjNum 1206 snap 378 seq_num 1206) 2024-09-12T23:52:07.625 INFO:tasks.rados.rados.0.smithi136.stdout:3986: finishing rollback tid 0 to smithi13667081-25 2024-09-12T23:52:07.625 INFO:tasks.rados.rados.0.smithi136.stdout:3986: finishing rollback tid 1 to smithi13667081-25 2024-09-12T23:52:07.625 INFO:tasks.rados.rados.0.smithi136.stdout:3986: finishing rollback tid 2 to smithi13667081-25 2024-09-12T23:52:07.625 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 25 v 2947 (ObjNum 1226 snap 383 seq_num 1226) dirty exists 2024-09-12T23:52:07.625 INFO:tasks.rados.rados.0.smithi136.stdout:3985: finishing copy_from racing read to smithi13667081-12 2024-09-12T23:52:07.626 INFO:tasks.rados.rados.0.smithi136.stdout:3983: finishing copy_from racing read to smithi13667081-49 2024-09-12T23:52:07.626 INFO:tasks.rados.rados.0.smithi136.stdout:3985: finishing copy_from to smithi13667081-12 2024-09-12T23:52:07.626 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 12 v 2567 (ObjNum 1232 snap 384 seq_num 1232) dirty exists 2024-09-12T23:52:07.631 INFO:tasks.rados.rados.0.smithi136.stdout:3983: finishing copy_from to smithi13667081-49 2024-09-12T23:52:07.631 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 49 v 2730 (ObjNum 1234 snap 384 seq_num 1234) dirty exists 2024-09-12T23:52:07.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:07 smithi181 bash[67587]: cluster 2024-09-12T23:52:06.892179+0000 mgr.y (mgr.24661) 449 : cluster [DBG] pgmap v868: 121 pgs: 121 active+clean; 312 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.9 MiB/s rd, 3.5 MiB/s wr, 37 op/s 2024-09-12T23:52:07.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:07 smithi181 bash[67587]: cluster 2024-09-12T23:52:07.562022+0000 mon.a (mon.0) 1441 : cluster [DBG] osdmap e858: 8 total, 8 up, 8 in 2024-09-12T23:52:08.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:07 smithi136 bash[76371]: cluster 2024-09-12T23:52:06.892179+0000 mgr.y (mgr.24661) 449 : cluster [DBG] pgmap v868: 121 pgs: 121 active+clean; 312 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.9 MiB/s rd, 3.5 MiB/s wr, 37 op/s 2024-09-12T23:52:08.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:07 smithi136 bash[76371]: cluster 2024-09-12T23:52:07.562022+0000 mon.a (mon.0) 1441 : cluster [DBG] osdmap e858: 8 total, 8 up, 8 in 2024-09-12T23:52:08.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:07 smithi136 bash[75164]: cluster 2024-09-12T23:52:06.892179+0000 mgr.y (mgr.24661) 449 : cluster [DBG] pgmap v868: 121 pgs: 121 active+clean; 312 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.9 MiB/s rd, 3.5 MiB/s wr, 37 op/s 2024-09-12T23:52:08.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:07 smithi136 bash[75164]: cluster 2024-09-12T23:52:07.562022+0000 mon.a (mon.0) 1441 : cluster [DBG] osdmap e858: 8 total, 8 up, 8 in 2024-09-12T23:52:08.561 INFO:tasks.rados.rados.0.smithi136.stdout:3983: done (4 left) 2024-09-12T23:52:08.561 INFO:tasks.rados.rados.0.smithi136.stdout:3984: done (3 left) 2024-09-12T23:52:08.561 INFO:tasks.rados.rados.0.smithi136.stdout:3985: done (2 left) 2024-09-12T23:52:08.561 INFO:tasks.rados.rados.0.smithi136.stdout:3986: done (1 left) 2024-09-12T23:52:08.561 INFO:tasks.rados.rados.0.smithi136.stdout:3987: done (0 left) 2024-09-12T23:52:08.561 INFO:tasks.rados.rados.0.smithi136.stdout:append oid 32 current snap is 389 2024-09-12T23:52:08.561 INFO:tasks.rados.rados.0.smithi136.stdout:3988: seq_num 1250 ranges {2375680=540672,2916352=770048,3686400=434176} 2024-09-12T23:52:08.578 INFO:tasks.rados.rados.0.smithi136.stdout:3988: writing smithi13667081-32 from 2375680 to 2916352 tid 1 2024-09-12T23:52:08.582 INFO:tasks.rados.rados.0.smithi136.stdout:3988: writing smithi13667081-32 from 2916352 to 3686400 tid 2 2024-09-12T23:52:08.585 INFO:tasks.rados.rados.0.smithi136.stdout:3988: writing smithi13667081-32 from 3686400 to 4120576 tid 3 2024-09-12T23:52:08.585 INFO:tasks.rados.rados.0.smithi136.stdout:3989: snap_remove snap 385 2024-09-12T23:52:09.573 INFO:tasks.rados.rados.0.smithi136.stdout:3989: done (1 left) 2024-09-12T23:52:09.574 INFO:tasks.rados.rados.0.smithi136.stdout:3990: copy_from oid 44 from oid 3 current snap is 389 2024-09-12T23:52:09.574 INFO:tasks.rados.rados.0.smithi136.stdout:3991: read oid 39 snap -1 2024-09-12T23:52:09.574 INFO:tasks.rados.rados.0.smithi136.stdout:3991: expect (ObjNum 1241 snap 385 seq_num 1241) 2024-09-12T23:52:09.574 INFO:tasks.rados.rados.0.smithi136.stdout:3992: copy_from oid 36 from oid 14 current snap is 389 2024-09-12T23:52:09.574 INFO:tasks.rados.rados.0.smithi136.stdout:3993: rmattr oid 25 current snap is 389 2024-09-12T23:52:09.575 INFO:tasks.rados.rados.0.smithi136.stdout:3988: finishing write tid 1 to smithi13667081-32 2024-09-12T23:52:09.575 INFO:tasks.rados.rados.0.smithi136.stdout:3994: read oid 5 snap -1 2024-09-12T23:52:09.575 INFO:tasks.rados.rados.0.smithi136.stdout:3994: expect (ObjNum 1136 snap 345 seq_num 1136) 2024-09-12T23:52:09.576 INFO:tasks.rados.rados.0.smithi136.stdout:3995: read oid 19 snap -1 2024-09-12T23:52:09.576 INFO:tasks.rados.rados.0.smithi136.stdout:3995: expect (ObjNum 1245 snap 387 seq_num 1245) 2024-09-12T23:52:09.576 INFO:tasks.rados.rados.0.smithi136.stdout:3996: rollback oid 38 current snap is 389 2024-09-12T23:52:09.576 INFO:tasks.rados.rados.0.smithi136.stdout:rollback oid 38 to 386 2024-09-12T23:52:09.576 INFO:tasks.rados.rados.0.smithi136.stdout:3997: rmattr oid 24 current snap is 389 2024-09-12T23:52:09.576 INFO:tasks.rados.rados.0.smithi136.stdout:3997: done (8 left) 2024-09-12T23:52:09.576 INFO:tasks.rados.rados.0.smithi136.stdout:3998: read oid 10 snap -1 2024-09-12T23:52:09.576 INFO:tasks.rados.rados.0.smithi136.stdout:3998: expect (ObjNum 1145 snap 350 seq_num 1145) 2024-09-12T23:52:09.576 INFO:tasks.rados.rados.0.smithi136.stdout:3999: copy_from oid 37 from oid 18 current snap is 389 2024-09-12T23:52:09.576 INFO:tasks.rados.rados.0.smithi136.stdout: waiting on 10 2024-09-12T23:52:09.576 INFO:tasks.rados.rados.0.smithi136.stdout:3988: finishing write tid 2 to smithi13667081-32 2024-09-12T23:52:09.577 INFO:tasks.rados.rados.0.smithi136.stdout:3988: finishing write tid 3 to smithi13667081-32 2024-09-12T23:52:09.577 INFO:tasks.rados.rados.0.smithi136.stdout:3988: finishing write tid 4 to smithi13667081-32 2024-09-12T23:52:09.577 INFO:tasks.rados.rados.0.smithi136.stdout:3988: finishing write tid 5 to smithi13667081-32 2024-09-12T23:52:09.577 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 32 v 2702 (ObjNum 1250 snap 389 seq_num 1250) dirty exists 2024-09-12T23:52:09.577 INFO:tasks.rados.rados.0.smithi136.stdout:3988: left oid 32 (ObjNum 1250 snap 389 seq_num 1250) 2024-09-12T23:52:09.577 INFO:tasks.rados.rados.0.smithi136.stdout:3988: done (9 left) 2024-09-12T23:52:09.577 INFO:tasks.rados.rados.0.smithi136.stdout: waiting on 9 2024-09-12T23:52:09.599 INFO:tasks.rados.rados.0.smithi136.stdout:3996: finishing rollback tid 0 to smithi13667081-38 2024-09-12T23:52:09.599 INFO:tasks.rados.rados.0.smithi136.stdout:3996: finishing rollback tid 1 to smithi13667081-38 2024-09-12T23:52:09.604 INFO:tasks.rados.rados.0.smithi136.stdout:3996: finishing rollback tid 2 to smithi13667081-38 2024-09-12T23:52:09.604 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 38 v 2712 (ObjNum 1164 snap 359 seq_num 1164) dirty exists 2024-09-12T23:52:09.604 INFO:tasks.rados.rados.0.smithi136.stdout:3996: done (8 left) 2024-09-12T23:52:09.604 INFO:tasks.rados.rados.0.smithi136.stdout: waiting on 8 2024-09-12T23:52:09.611 INFO:tasks.rados.rados.0.smithi136.stdout:3991: expect (ObjNum 1241 snap 385 seq_num 1241) 2024-09-12T23:52:09.674 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 25 v 2948 (ObjNum 1226 snap 383 seq_num 1226) dirty exists 2024-09-12T23:52:09.675 INFO:tasks.rados.rados.0.smithi136.stdout:3998: expect (ObjNum 1145 snap 350 seq_num 1145) 2024-09-12T23:52:09.735 INFO:tasks.rados.rados.0.smithi136.stdout:3990: finishing copy_from racing read to smithi13667081-44 2024-09-12T23:52:09.735 INFO:tasks.rados.rados.0.smithi136.stdout:3995: expect (ObjNum 1245 snap 387 seq_num 1245) 2024-09-12T23:52:09.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:52:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:52:09] "GET /metrics HTTP/1.1" 200 36438 "" "Prometheus/2.43.0" 2024-09-12T23:52:09.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:09 smithi136 bash[75164]: cluster 2024-09-12T23:52:08.563828+0000 mon.a (mon.0) 1442 : cluster [DBG] osdmap e859: 8 total, 8 up, 8 in 2024-09-12T23:52:09.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:09 smithi136 bash[75164]: audit 2024-09-12T23:52:08.755886+0000 mon.a (mon.0) 1443 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:52:09.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:09 smithi136 bash[75164]: cluster 2024-09-12T23:52:08.892867+0000 mgr.y (mgr.24661) 450 : cluster [DBG] pgmap v871: 121 pgs: 121 active+clean; 316 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.1 MiB/s rd, 3.9 MiB/s wr, 25 op/s 2024-09-12T23:52:09.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:09 smithi136 bash[76371]: cluster 2024-09-12T23:52:08.563828+0000 mon.a (mon.0) 1442 : cluster [DBG] osdmap e859: 8 total, 8 up, 8 in 2024-09-12T23:52:09.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:09 smithi136 bash[76371]: audit 2024-09-12T23:52:08.755886+0000 mon.a (mon.0) 1443 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:52:09.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:09 smithi136 bash[76371]: cluster 2024-09-12T23:52:08.892867+0000 mgr.y (mgr.24661) 450 : cluster [DBG] pgmap v871: 121 pgs: 121 active+clean; 316 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.1 MiB/s rd, 3.9 MiB/s wr, 25 op/s 2024-09-12T23:52:09.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:09 smithi181 bash[67587]: cluster 2024-09-12T23:52:08.563828+0000 mon.a (mon.0) 1442 : cluster [DBG] osdmap e859: 8 total, 8 up, 8 in 2024-09-12T23:52:09.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:09 smithi181 bash[67587]: audit 2024-09-12T23:52:08.755886+0000 mon.a (mon.0) 1443 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:52:09.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:09 smithi181 bash[67587]: cluster 2024-09-12T23:52:08.892867+0000 mgr.y (mgr.24661) 450 : cluster [DBG] pgmap v871: 121 pgs: 121 active+clean; 316 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.1 MiB/s rd, 3.9 MiB/s wr, 25 op/s 2024-09-12T23:52:09.864 INFO:tasks.rados.rados.0.smithi136.stdout:3990: finishing copy_from to smithi13667081-44 2024-09-12T23:52:09.865 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 44 v 2527 (ObjNum 1247 snap 387 seq_num 1247) dirty exists 2024-09-12T23:52:09.865 INFO:tasks.rados.rados.0.smithi136.stdout:3999: finishing copy_from racing read to smithi13667081-37 2024-09-12T23:52:09.865 INFO:tasks.rados.rados.0.smithi136.stdout:3999: finishing copy_from to smithi13667081-37 2024-09-12T23:52:09.865 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 37 v 1654 (ObjNum 1206 snap 378 seq_num 1206) dirty exists 2024-09-12T23:52:09.865 INFO:tasks.rados.rados.0.smithi136.stdout:3992: finishing copy_from racing read to smithi13667081-36 2024-09-12T23:52:09.865 INFO:tasks.rados.rados.0.smithi136.stdout:3992: finishing copy_from to smithi13667081-36 2024-09-12T23:52:09.865 INFO:tasks.rados.rados.0.smithi136.stdout:update_object_version oid 36 v 2549 (ObjNum 1210 snap 381 seq_num 1210) dirty exists 2024-09-12T23:52:09.865 INFO:tasks.rados.rados.0.smithi136.stdout:3990: done (7 left) 2024-09-12T23:52:09.865 INFO:tasks.rados.rados.0.smithi136.stdout:3991: done (6 left) 2024-09-12T23:52:09.865 INFO:tasks.rados.rados.0.smithi136.stdout:3992: done (5 left) 2024-09-12T23:52:09.865 INFO:tasks.rados.rados.0.smithi136.stdout:3993: done (4 left) 2024-09-12T23:52:09.865 INFO:tasks.rados.rados.0.smithi136.stdout:3995: done (3 left) 2024-09-12T23:52:09.865 INFO:tasks.rados.rados.0.smithi136.stdout:3998: done (2 left) 2024-09-12T23:52:09.866 INFO:tasks.rados.rados.0.smithi136.stdout:3999: done (1 left) 2024-09-12T23:52:09.866 INFO:tasks.rados.rados.0.smithi136.stdout: waiting on 1 2024-09-12T23:52:09.866 INFO:tasks.rados.rados.0.smithi136.stdout:3994: expect (ObjNum 1136 snap 345 seq_num 1136) 2024-09-12T23:52:10.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:10 smithi181 bash[67587]: cluster 2024-09-12T23:52:09.574422+0000 mon.a (mon.0) 1444 : cluster [DBG] osdmap e860: 8 total, 8 up, 8 in 2024-09-12T23:52:11.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:10 smithi136 bash[75164]: cluster 2024-09-12T23:52:09.574422+0000 mon.a (mon.0) 1444 : cluster [DBG] osdmap e860: 8 total, 8 up, 8 in 2024-09-12T23:52:11.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:10 smithi136 bash[76371]: cluster 2024-09-12T23:52:09.574422+0000 mon.a (mon.0) 1444 : cluster [DBG] osdmap e860: 8 total, 8 up, 8 in 2024-09-12T23:52:11.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:11 smithi181 bash[67587]: cluster 2024-09-12T23:52:10.893763+0000 mgr.y (mgr.24661) 451 : cluster [DBG] pgmap v873: 121 pgs: 121 active+clean; 316 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.0 MiB/s rd, 2.9 MiB/s wr, 27 op/s 2024-09-12T23:52:11.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:11 smithi181 bash[67587]: cluster 2024-09-12T23:52:11.564095+0000 mon.a (mon.0) 1445 : cluster [DBG] osdmap e861: 8 total, 8 up, 8 in 2024-09-12T23:52:12.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:11 smithi136 bash[75164]: cluster 2024-09-12T23:52:10.893763+0000 mgr.y (mgr.24661) 451 : cluster [DBG] pgmap v873: 121 pgs: 121 active+clean; 316 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.0 MiB/s rd, 2.9 MiB/s wr, 27 op/s 2024-09-12T23:52:12.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:11 smithi136 bash[75164]: cluster 2024-09-12T23:52:11.564095+0000 mon.a (mon.0) 1445 : cluster [DBG] osdmap e861: 8 total, 8 up, 8 in 2024-09-12T23:52:12.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:11 smithi136 bash[76371]: cluster 2024-09-12T23:52:10.893763+0000 mgr.y (mgr.24661) 451 : cluster [DBG] pgmap v873: 121 pgs: 121 active+clean; 316 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.0 MiB/s rd, 2.9 MiB/s wr, 27 op/s 2024-09-12T23:52:12.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:11 smithi136 bash[76371]: cluster 2024-09-12T23:52:11.564095+0000 mon.a (mon.0) 1445 : cluster [DBG] osdmap e861: 8 total, 8 up, 8 in 2024-09-12T23:52:12.021 INFO:tasks.rados.rados.0.smithi136.stdout:3994: done (0 left) 2024-09-12T23:52:12.035 INFO:tasks.rados.rados.0.smithi136.stderr:0 errors. 2024-09-12T23:52:12.035 INFO:tasks.rados.rados.0.smithi136.stderr: 2024-09-12T23:52:12.051 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-12T23:52:12.596 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:52:12 smithi181 bash[95505]: ts=2024-09-12T23:52:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:52:13.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:12 smithi136 bash[75164]: cluster 2024-09-12T23:52:12.894819+0000 mgr.y (mgr.24661) 452 : cluster [DBG] pgmap v875: 121 pgs: 121 active+clean; 327 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.3 MiB/s rd, 4.2 MiB/s wr, 15 op/s 2024-09-12T23:52:13.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:12 smithi136 bash[76371]: cluster 2024-09-12T23:52:12.894819+0000 mgr.y (mgr.24661) 452 : cluster [DBG] pgmap v875: 121 pgs: 121 active+clean; 327 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.3 MiB/s rd, 4.2 MiB/s wr, 15 op/s 2024-09-12T23:52:13.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:12 smithi181 bash[67587]: cluster 2024-09-12T23:52:12.894819+0000 mgr.y (mgr.24661) 452 : cluster [DBG] pgmap v875: 121 pgs: 121 active+clean; 327 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.3 MiB/s rd, 4.2 MiB/s wr, 15 op/s 2024-09-12T23:52:15.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:14 smithi136 bash[75164]: cluster 2024-09-12T23:52:14.895741+0000 mgr.y (mgr.24661) 453 : cluster [DBG] pgmap v876: 121 pgs: 121 active+clean; 327 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 12 MiB/s rd, 2.8 MiB/s wr, 11 op/s 2024-09-12T23:52:15.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:14 smithi136 bash[76371]: cluster 2024-09-12T23:52:14.895741+0000 mgr.y (mgr.24661) 453 : cluster [DBG] pgmap v876: 121 pgs: 121 active+clean; 327 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 12 MiB/s rd, 2.8 MiB/s wr, 11 op/s 2024-09-12T23:52:15.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:14 smithi181 bash[67587]: cluster 2024-09-12T23:52:14.895741+0000 mgr.y (mgr.24661) 453 : cluster [DBG] pgmap v876: 121 pgs: 121 active+clean; 327 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 12 MiB/s rd, 2.8 MiB/s wr, 11 op/s 2024-09-12T23:52:16.823 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:52:17.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:17 smithi181 bash[67587]: cluster 2024-09-12T23:52:16.565012+0000 mon.a (mon.0) 1446 : cluster [DBG] osdmap e862: 8 total, 8 up, 8 in 2024-09-12T23:52:17.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:17 smithi181 bash[67587]: cluster 2024-09-12T23:52:16.896693+0000 mgr.y (mgr.24661) 454 : cluster [DBG] pgmap v878: 121 pgs: 121 active+clean; 327 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 10 MiB/s rd, 2.4 MiB/s wr, 9 op/s 2024-09-12T23:52:18.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:17 smithi136 bash[76371]: cluster 2024-09-12T23:52:16.565012+0000 mon.a (mon.0) 1446 : cluster [DBG] osdmap e862: 8 total, 8 up, 8 in 2024-09-12T23:52:18.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:17 smithi136 bash[76371]: cluster 2024-09-12T23:52:16.896693+0000 mgr.y (mgr.24661) 454 : cluster [DBG] pgmap v878: 121 pgs: 121 active+clean; 327 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 10 MiB/s rd, 2.4 MiB/s wr, 9 op/s 2024-09-12T23:52:18.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:17 smithi136 bash[75164]: cluster 2024-09-12T23:52:16.565012+0000 mon.a (mon.0) 1446 : cluster [DBG] osdmap e862: 8 total, 8 up, 8 in 2024-09-12T23:52:18.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:17 smithi136 bash[75164]: cluster 2024-09-12T23:52:16.896693+0000 mgr.y (mgr.24661) 454 : cluster [DBG] pgmap v878: 121 pgs: 121 active+clean; 327 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 10 MiB/s rd, 2.4 MiB/s wr, 9 op/s 2024-09-12T23:52:19.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:18 smithi136 bash[75164]: cluster 2024-09-12T23:52:18.897682+0000 mgr.y (mgr.24661) 455 : cluster [DBG] pgmap v879: 121 pgs: 121 active+clean; 327 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.6 MiB/s rd, 2.2 MiB/s wr, 2 op/s 2024-09-12T23:52:19.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:18 smithi136 bash[76371]: cluster 2024-09-12T23:52:18.897682+0000 mgr.y (mgr.24661) 455 : cluster [DBG] pgmap v879: 121 pgs: 121 active+clean; 327 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.6 MiB/s rd, 2.2 MiB/s wr, 2 op/s 2024-09-12T23:52:19.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:18 smithi181 bash[67587]: cluster 2024-09-12T23:52:18.897682+0000 mgr.y (mgr.24661) 455 : cluster [DBG] pgmap v879: 121 pgs: 121 active+clean; 327 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.6 MiB/s rd, 2.2 MiB/s wr, 2 op/s 2024-09-12T23:52:19.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:52:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:52:19] "GET /metrics HTTP/1.1" 200 36438 "" "Prometheus/2.43.0" 2024-09-12T23:52:20.576 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-12T23:52:20.576 INFO:teuthology.orchestra.run.smithi136.stdout:{"epoch":862,"fsid":"da58a47c-715d-11ef-bceb-c7b262605968","created":"2024-09-12T23:23:29.000926+0000","modified":"2024-09-12T23:52:16.553001+0000","last_up_change":"2024-09-12T23:47:42.111391+0000","last_in_change":"2024-09-12T23:32:18.995528+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-12T23:28:33.956985+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-09-12T23:33:27.478434+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-09-12T23:33:28.566349+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"54","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"rbd","create_time":"2024-09-12T23:34:48.404257+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"61","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":61,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":3,"score_stable":3,"optimal_score":1,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":5,"pool_name":"unique_pool_0","create_time":"2024-09-12T23:38:13.804539+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":3,"size":3,"min_size":2,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"860","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":776,"snap_epoch":860,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"teuthologyprofile","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"87aedf04-695e-4a8f-80c3-ffd18883a4d5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":391,"up_from":405,"up_thru":605,"down_at":392,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6802","nonce":1696528169},{"type":"v1","addr":"172.21.15.136:6803","nonce":1696528169}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6804","nonce":1696528169},{"type":"v1","addr":"172.21.15.136:6805","nonce":1696528169}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6808","nonce":1696528169},{"type":"v1","addr":"172.21.15.136:6809","nonce":1696528169}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6806","nonce":1696528169},{"type":"v1","addr":"172.21.15.136:6807","nonce":1696528169}]},"public_addr":"172.21.15.136:6803/1696528169","cluster_addr":"172.21.15.136:6805/1696528169","heartbeat_back_addr":"172.21.15.136:6809/1696528169","heartbeat_front_addr":"172.21.15.136:6807/1696528169","state":["exists","up"]},{"osd":1,"uuid":"850d4f98-0640-4151-bd45-a351ca41d54e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":419,"up_from":433,"up_thru":605,"down_at":420,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6810","nonce":3078861551},{"type":"v1","addr":"172.21.15.136:6811","nonce":3078861551}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6812","nonce":3078861551},{"type":"v1","addr":"172.21.15.136:6813","nonce":3078861551}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6816","nonce":3078861551},{"type":"v1","addr":"172.21.15.136:6817","nonce":3078861551}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6814","nonce":3078861551},{"type":"v1","addr":"172.21.15.136:6815","nonce":3078861551}]},"public_addr":"172.21.15.136:6811/3078861551","cluster_addr":"172.21.15.136:6813/3078861551","heartbeat_back_addr":"172.21.15.136:6817/3078861551","heartbeat_front_addr":"172.21.15.136:6815/3078861551","state":["exists","up"]},{"osd":2,"uuid":"bde707d6-e027-4b63-9853-fd59212eea1e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":449,"up_from":463,"up_thru":605,"down_at":450,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6818","nonce":439422470},{"type":"v1","addr":"172.21.15.136:6819","nonce":439422470}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6820","nonce":439422470},{"type":"v1","addr":"172.21.15.136:6821","nonce":439422470}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6824","nonce":439422470},{"type":"v1","addr":"172.21.15.136:6825","nonce":439422470}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6822","nonce":439422470},{"type":"v1","addr":"172.21.15.136:6823","nonce":439422470}]},"public_addr":"172.21.15.136:6819/439422470","cluster_addr":"172.21.15.136:6821/439422470","heartbeat_back_addr":"172.21.15.136:6825/439422470","heartbeat_front_addr":"172.21.15.136:6823/439422470","state":["exists","up"]},{"osd":3,"uuid":"d61f8357-b89b-4c05-b366-65a9f788064d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":478,"up_from":492,"up_thru":605,"down_at":479,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6826","nonce":2156253618},{"type":"v1","addr":"172.21.15.136:6827","nonce":2156253618}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6828","nonce":2156253618},{"type":"v1","addr":"172.21.15.136:6829","nonce":2156253618}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6832","nonce":2156253618},{"type":"v1","addr":"172.21.15.136:6833","nonce":2156253618}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6830","nonce":2156253618},{"type":"v1","addr":"172.21.15.136:6831","nonce":2156253618}]},"public_addr":"172.21.15.136:6827/2156253618","cluster_addr":"172.21.15.136:6829/2156253618","heartbeat_back_addr":"172.21.15.136:6833/2156253618","heartbeat_front_addr":"172.21.15.136:6831/2156253618","state":["exists","up"]},{"osd":4,"uuid":"98b0e81c-824e-4dbc-9faa-2de9f199af1b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":30,"last_clean_end":507,"up_from":520,"up_thru":605,"down_at":508,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6800","nonce":2759874318},{"type":"v1","addr":"172.21.15.181:6801","nonce":2759874318}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6802","nonce":2759874318},{"type":"v1","addr":"172.21.15.181:6803","nonce":2759874318}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6806","nonce":2759874318},{"type":"v1","addr":"172.21.15.181:6807","nonce":2759874318}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6804","nonce":2759874318},{"type":"v1","addr":"172.21.15.181:6805","nonce":2759874318}]},"public_addr":"172.21.15.181:6801/2759874318","cluster_addr":"172.21.15.181:6803/2759874318","heartbeat_back_addr":"172.21.15.181:6807/2759874318","heartbeat_front_addr":"172.21.15.181:6805/2759874318","state":["exists","up"]},{"osd":5,"uuid":"5c6d2f69-7f9c-4689-95b0-bd4eb858956d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":36,"last_clean_end":534,"up_from":548,"up_thru":605,"down_at":535,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6808","nonce":3863825756},{"type":"v1","addr":"172.21.15.181:6809","nonce":3863825756}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6810","nonce":3863825756},{"type":"v1","addr":"172.21.15.181:6811","nonce":3863825756}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6814","nonce":3863825756},{"type":"v1","addr":"172.21.15.181:6815","nonce":3863825756}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6812","nonce":3863825756},{"type":"v1","addr":"172.21.15.181:6813","nonce":3863825756}]},"public_addr":"172.21.15.181:6809/3863825756","cluster_addr":"172.21.15.181:6811/3863825756","heartbeat_back_addr":"172.21.15.181:6815/3863825756","heartbeat_front_addr":"172.21.15.181:6813/3863825756","state":["exists","up"]},{"osd":6,"uuid":"532ee052-af15-433a-8e93-b0b642aaf85b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":42,"last_clean_end":563,"up_from":575,"up_thru":605,"down_at":564,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6816","nonce":1329407811},{"type":"v1","addr":"172.21.15.181:6817","nonce":1329407811}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6818","nonce":1329407811},{"type":"v1","addr":"172.21.15.181:6819","nonce":1329407811}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6822","nonce":1329407811},{"type":"v1","addr":"172.21.15.181:6823","nonce":1329407811}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6820","nonce":1329407811},{"type":"v1","addr":"172.21.15.181:6821","nonce":1329407811}]},"public_addr":"172.21.15.181:6817/1329407811","cluster_addr":"172.21.15.181:6819/1329407811","heartbeat_back_addr":"172.21.15.181:6823/1329407811","heartbeat_front_addr":"172.21.15.181:6821/1329407811","state":["exists","up"]},{"osd":7,"uuid":"d2ff80f0-6ef0-4098-877c-a891d48c687d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":48,"last_clean_end":591,"up_from":605,"up_thru":605,"down_at":592,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6824","nonce":2237860309},{"type":"v1","addr":"172.21.15.181:6825","nonce":2237860309}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6826","nonce":2237860309},{"type":"v1","addr":"172.21.15.181:6827","nonce":2237860309}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6830","nonce":2237860309},{"type":"v1","addr":"172.21.15.181:6831","nonce":2237860309}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6828","nonce":2237860309},{"type":"v1","addr":"172.21.15.181:6829","nonce":2237860309}]},"public_addr":"172.21.15.181:6825/2237860309","cluster_addr":"172.21.15.181:6827/2237860309","heartbeat_back_addr":"172.21.15.181:6831/2237860309","heartbeat_front_addr":"172.21.15.181:6829/2237860309","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:26:50.657580+0000","dead_epoch":391},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:27:40.536476+0000","dead_epoch":419},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:28:31.755012+0000","dead_epoch":449},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:29:23.585451+0000","dead_epoch":478},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:30:11.888258+0000","dead_epoch":507},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:31:01.812765+0000","dead_epoch":534},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:31:52.166696+0000","dead_epoch":563},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:32:42.696525+0000","dead_epoch":590}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.3","mappings":[{"from":7,"to":0}]},{"pgid":"2.a","mappings":[{"from":7,"to":0}]},{"pgid":"2.b","mappings":[{"from":7,"to":2}]},{"pgid":"2.1a","mappings":[{"from":7,"to":0}]},{"pgid":"2.1d","mappings":[{"from":7,"to":4}]},{"pgid":"2.1f","mappings":[{"from":7,"to":2}]},{"pgid":"3.9","mappings":[{"from":4,"to":1}]},{"pgid":"3.a","mappings":[{"from":4,"to":0}]},{"pgid":"3.10","mappings":[{"from":0,"to":1}]},{"pgid":"3.11","mappings":[{"from":7,"to":0}]},{"pgid":"3.14","mappings":[{"from":4,"to":2}]},{"pgid":"3.30","mappings":[{"from":5,"to":2}]},{"pgid":"3.31","mappings":[{"from":4,"to":2}]},{"pgid":"3.37","mappings":[{"from":5,"to":1}]},{"pgid":"3.3e","mappings":[{"from":6,"to":2}]},{"pgid":"5.d","mappings":[{"from":5,"to":0}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.136:0/3143991445":"2024-09-13T23:42:08.612521+0000","172.21.15.136:6800/821792295":"2024-09-13T23:42:08.612521+0000","172.21.15.136:0/3161590810":"2024-09-13T23:42:08.612521+0000","172.21.15.136:0/1370177551":"2024-09-13T23:42:08.612521+0000","172.21.15.136:0/1549678324":"2024-09-13T23:23:54.847045+0000","172.21.15.136:0/129641967":"2024-09-13T23:34:07.892265+0000","172.21.15.136:0/942965045":"2024-09-13T23:42:08.612521+0000","172.21.15.136:6801/821792295":"2024-09-13T23:42:08.612521+0000","172.21.15.136:0/3376519841":"2024-09-13T23:23:54.847045+0000","172.21.15.181:6833/2077070284":"2024-09-13T23:48:21.160950+0000","172.21.15.136:0/1675562321":"2024-09-13T23:40:07.531346+0000","172.21.15.136:0/3346741202":"2024-09-13T23:40:07.531346+0000","172.21.15.136:0/1890850023":"2024-09-13T23:42:08.612521+0000","172.21.15.181:0/3047046473":"2024-09-13T23:41:07.408939+0000","172.21.15.136:0/3094962794":"2024-09-13T23:42:08.612521+0000","172.21.15.136:0/4260962972":"2024-09-13T23:24:25.346823+0000","172.21.15.181:6835/1989969453":"2024-09-13T23:41:07.408939+0000","172.21.15.136:0/3924275900":"2024-09-13T23:34:07.892265+0000","172.21.15.136:6800/3939504613":"2024-09-13T23:40:07.531346+0000","172.21.15.181:6834/1989969453":"2024-09-13T23:41:07.408939+0000","172.21.15.181:0/1149753131":"2024-09-13T23:41:07.408939+0000","172.21.15.136:0/320185649":"2024-09-13T23:42:08.612521+0000","172.21.15.136:0/3642432358":"2024-09-13T23:34:07.892265+0000","172.21.15.181:0/3092067856":"2024-09-13T23:41:07.408939+0000","172.21.15.136:0/3910468981":"2024-09-13T23:23:54.847045+0000","172.21.15.136:6800/2894181186":"2024-09-13T23:23:54.847045+0000","172.21.15.136:0/1913282058":"2024-09-13T23:40:07.531346+0000","172.21.15.136:0/1964283041":"2024-09-13T23:24:25.346823+0000","172.21.15.181:0/2898677511":"2024-09-13T23:41:07.408939+0000","172.21.15.136:6800/3594272622":"2024-09-13T23:24:25.346823+0000","172.21.15.181:0/912737797":"2024-09-13T23:41:07.408939+0000","172.21.15.136:6801/3594272622":"2024-09-13T23:24:25.346823+0000","172.21.15.181:0/1647856294":"2024-09-13T23:41:07.408939+0000","172.21.15.136:6801/2894181186":"2024-09-13T23:23:54.847045+0000","172.21.15.136:0/2528576426":"2024-09-13T23:40:07.531346+0000","172.21.15.136:0/209539205":"2024-09-13T23:24:25.346823+0000","172.21.15.136:6800/1810367175":"2024-09-13T23:34:07.892265+0000","172.21.15.136:0/513809002":"2024-09-13T23:34:07.892265+0000","172.21.15.136:6801/1810367175":"2024-09-13T23:34:07.892265+0000","172.21.15.136:6801/3939504613":"2024-09-13T23:40:07.531346+0000","172.21.15.181:6832/2077070284":"2024-09-13T23:48:21.160950+0000","172.21.15.136:0/4113402351":"2024-09-13T23:40:07.531346+0000","172.21.15.181:0/1736879849":"2024-09-13T23:41:07.408939+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"},"teuthologyprofile":{"crush-device-class":"","crush-failure-domain":"osd","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van","w":"8"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":5,"snaps":[{"begin":750,"length":1},{"begin":771,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-12T23:52:21.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:20 smithi136 bash[75164]: audit 2024-09-12T23:52:20.577190+0000 mon.a (mon.0) 1447 : audit [DBG] from='client.? 172.21.15.136:0/3599080468' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-12T23:52:21.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:20 smithi136 bash[76371]: audit 2024-09-12T23:52:20.577190+0000 mon.a (mon.0) 1447 : audit [DBG] from='client.? 172.21.15.136:0/3599080468' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-12T23:52:21.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:20 smithi181 bash[67587]: audit 2024-09-12T23:52:20.577190+0000 mon.a (mon.0) 1447 : audit [DBG] from='client.? 172.21.15.136:0/3599080468' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-12T23:52:21.336 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-12T23:52:22.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:21 smithi136 bash[75164]: cluster 2024-09-12T23:52:20.898686+0000 mgr.y (mgr.24661) 456 : cluster [DBG] pgmap v880: 121 pgs: 121 active+clean; 327 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.5 MiB/s rd, 1.9 MiB/s wr, 2 op/s 2024-09-12T23:52:22.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:21 smithi136 bash[76371]: cluster 2024-09-12T23:52:20.898686+0000 mgr.y (mgr.24661) 456 : cluster [DBG] pgmap v880: 121 pgs: 121 active+clean; 327 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.5 MiB/s rd, 1.9 MiB/s wr, 2 op/s 2024-09-12T23:52:22.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:21 smithi181 bash[67587]: cluster 2024-09-12T23:52:20.898686+0000 mgr.y (mgr.24661) 456 : cluster [DBG] pgmap v880: 121 pgs: 121 active+clean; 327 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.5 MiB/s rd, 1.9 MiB/s wr, 2 op/s 2024-09-12T23:52:22.596 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:52:22 smithi181 bash[95505]: ts=2024-09-12T23:52:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:52:23.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:22 smithi136 bash[75164]: cluster 2024-09-12T23:52:22.899771+0000 mgr.y (mgr.24661) 457 : cluster [DBG] pgmap v881: 121 pgs: 121 active+clean; 327 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.1 MiB/s rd, 0 op/s 2024-09-12T23:52:23.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:22 smithi136 bash[76371]: cluster 2024-09-12T23:52:22.899771+0000 mgr.y (mgr.24661) 457 : cluster [DBG] pgmap v881: 121 pgs: 121 active+clean; 327 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.1 MiB/s rd, 0 op/s 2024-09-12T23:52:23.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:22 smithi181 bash[67587]: cluster 2024-09-12T23:52:22.899771+0000 mgr.y (mgr.24661) 457 : cluster [DBG] pgmap v881: 121 pgs: 121 active+clean; 327 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.1 MiB/s rd, 0 op/s 2024-09-12T23:52:24.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:23 smithi136 bash[75164]: audit 2024-09-12T23:52:23.756085+0000 mon.a (mon.0) 1448 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:52:24.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:23 smithi136 bash[76371]: audit 2024-09-12T23:52:23.756085+0000 mon.a (mon.0) 1448 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:52:24.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:23 smithi181 bash[67587]: audit 2024-09-12T23:52:23.756085+0000 mon.a (mon.0) 1448 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:52:25.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:25 smithi136 bash[75164]: cluster 2024-09-12T23:52:24.900627+0000 mgr.y (mgr.24661) 458 : cluster [DBG] pgmap v882: 121 pgs: 121 active+clean; 327 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:52:25.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:25 smithi136 bash[76371]: cluster 2024-09-12T23:52:24.900627+0000 mgr.y (mgr.24661) 458 : cluster [DBG] pgmap v882: 121 pgs: 121 active+clean; 327 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:52:25.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:25 smithi181 bash[67587]: cluster 2024-09-12T23:52:24.900627+0000 mgr.y (mgr.24661) 458 : cluster [DBG] pgmap v882: 121 pgs: 121 active+clean; 327 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:52:26.121 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:52:27.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:26 smithi136 bash[75164]: cluster 2024-09-12T23:52:26.901820+0000 mgr.y (mgr.24661) 459 : cluster [DBG] pgmap v883: 121 pgs: 121 active+clean; 327 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:52:27.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:26 smithi136 bash[76371]: cluster 2024-09-12T23:52:26.901820+0000 mgr.y (mgr.24661) 459 : cluster [DBG] pgmap v883: 121 pgs: 121 active+clean; 327 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:52:27.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:26 smithi181 bash[67587]: cluster 2024-09-12T23:52:26.901820+0000 mgr.y (mgr.24661) 459 : cluster [DBG] pgmap v883: 121 pgs: 121 active+clean; 327 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:52:28.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:28 smithi136 bash[75164]: audit 2024-09-12T23:52:28.154604+0000 mon.a (mon.0) 1449 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:52:28.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:28 smithi136 bash[76371]: audit 2024-09-12T23:52:28.154604+0000 mon.a (mon.0) 1449 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:52:28.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:28 smithi181 bash[67587]: audit 2024-09-12T23:52:28.154604+0000 mon.a (mon.0) 1449 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:52:29.361 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-12T23:52:29.363 INFO:teuthology.orchestra.run.smithi136.stderr:dumped all 2024-09-12T23:52:29.372 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:29 smithi136 bash[76371]: audit 2024-09-12T23:52:28.694864+0000 mon.a (mon.0) 1450 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:52:29.372 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:29 smithi136 bash[76371]: audit 2024-09-12T23:52:28.696447+0000 mon.a (mon.0) 1451 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:52:29.372 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:29 smithi136 bash[76371]: audit 2024-09-12T23:52:28.706458+0000 mon.a (mon.0) 1452 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:52:29.372 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:29 smithi136 bash[76371]: cluster 2024-09-12T23:52:28.902676+0000 mgr.y (mgr.24661) 460 : cluster [DBG] pgmap v884: 121 pgs: 121 active+clean; 327 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:52:29.372 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:29 smithi136 bash[75164]: audit 2024-09-12T23:52:28.694864+0000 mon.a (mon.0) 1450 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:52:29.373 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:29 smithi136 bash[75164]: audit 2024-09-12T23:52:28.696447+0000 mon.a (mon.0) 1451 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:52:29.373 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:29 smithi136 bash[75164]: audit 2024-09-12T23:52:28.706458+0000 mon.a (mon.0) 1452 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:52:29.373 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:29 smithi136 bash[75164]: cluster 2024-09-12T23:52:28.902676+0000 mgr.y (mgr.24661) 460 : cluster [DBG] pgmap v884: 121 pgs: 121 active+clean; 327 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:52:29.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:29 smithi181 bash[67587]: audit 2024-09-12T23:52:28.694864+0000 mon.a (mon.0) 1450 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:52:29.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:29 smithi181 bash[67587]: audit 2024-09-12T23:52:28.696447+0000 mon.a (mon.0) 1451 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:52:29.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:29 smithi181 bash[67587]: audit 2024-09-12T23:52:28.706458+0000 mon.a (mon.0) 1452 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:52:29.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:29 smithi181 bash[67587]: cluster 2024-09-12T23:52:28.902676+0000 mgr.y (mgr.24661) 460 : cluster [DBG] pgmap v884: 121 pgs: 121 active+clean; 327 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:52:29.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:52:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:52:29] "GET /metrics HTTP/1.1" 200 36438 "" "Prometheus/2.43.0" 2024-09-12T23:52:30.133 INFO:teuthology.orchestra.run.smithi136.stdout:{"pg_ready":true,"pg_map":{"version":884,"stamp":"2024-09-12T23:52:28.902062+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":343186435,"num_objects":156,"num_object_clones":82,"num_object_copies":468,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":156,"num_whiteouts":4,"num_read":4755,"num_read_kb":5748751,"num_write":8670,"num_write_kb":1832841,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":179,"num_bytes_recovered":598672474,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":21105,"ondisk_log_size":21105,"up":363,"acting":363,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":363,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":3092964,"kb_used_data":558148,"kb_used_omap":325,"kb_used_meta":2534394,"kb_avail":746868252,"statfs":{"total":767960285184,"available":764793090048,"internally_reserved":0,"allocated":571543552,"data_stored":758216173,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":333292,"internal_metadata":2595219988},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.006023"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":815,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.571443+0000","last_change":"2024-09-12T23:47:43.164416+0000","last_active":"2024-09-12T23:52:11.571443+0000","last_peered":"2024-09-12T23:52:11.571443+0000","last_clean":"2024-09-12T23:52:11.571443+0000","last_became_active":"2024-09-12T23:47:43.162359+0000","last_became_peered":"2024-09-12T23:47:43.162359+0000","last_unstale":"2024-09-12T23:52:11.571443+0000","last_undegraded":"2024-09-12T23:52:11.571443+0000","last_fullsized":"2024-09-12T23:52:11.571443+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:10:29.048706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":842,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.907310+0000","last_change":"2024-09-12T23:45:18.506695+0000","last_active":"2024-09-12T23:52:11.907310+0000","last_peered":"2024-09-12T23:52:11.907310+0000","last_clean":"2024-09-12T23:52:11.907310+0000","last_became_active":"2024-09-12T23:45:18.506080+0000","last_became_peered":"2024-09-12T23:45:18.506080+0000","last_unstale":"2024-09-12T23:52:11.907310+0000","last_undegraded":"2024-09-12T23:52:11.907310+0000","last_fullsized":"2024-09-12T23:52:11.907310+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:46:00.650293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":829,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.566011+0000","last_change":"2024-09-12T23:45:18.507207+0000","last_active":"2024-09-12T23:52:11.566011+0000","last_peered":"2024-09-12T23:52:11.566011+0000","last_clean":"2024-09-12T23:52:11.566011+0000","last_became_active":"2024-09-12T23:45:18.506855+0000","last_became_peered":"2024-09-12T23:45:18.506855+0000","last_unstale":"2024-09-12T23:52:11.566011+0000","last_undegraded":"2024-09-12T23:52:11.566011+0000","last_fullsized":"2024-09-12T23:52:11.566011+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-13T23:37:19.326662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":836,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:12.259552+0000","last_change":"2024-09-12T23:47:43.157043+0000","last_active":"2024-09-12T23:52:12.259552+0000","last_peered":"2024-09-12T23:52:12.259552+0000","last_clean":"2024-09-12T23:52:12.259552+0000","last_became_active":"2024-09-12T23:47:43.156243+0000","last_became_peered":"2024-09-12T23:47:43.156243+0000","last_unstale":"2024-09-12T23:52:12.259552+0000","last_undegraded":"2024-09-12T23:52:12.259552+0000","last_fullsized":"2024-09-12T23:52:12.259552+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:19:52.599140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":801,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.569228+0000","last_change":"2024-09-12T23:47:43.180087+0000","last_active":"2024-09-12T23:52:11.569228+0000","last_peered":"2024-09-12T23:52:11.569228+0000","last_clean":"2024-09-12T23:52:11.569228+0000","last_became_active":"2024-09-12T23:47:43.179348+0000","last_became_peered":"2024-09-12T23:47:43.179348+0000","last_unstale":"2024-09-12T23:52:11.569228+0000","last_undegraded":"2024-09-12T23:52:11.569228+0000","last_fullsized":"2024-09-12T23:52:11.569228+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:27:11.494843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":821,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:12.259349+0000","last_change":"2024-09-12T23:46:44.531744+0000","last_active":"2024-09-12T23:52:12.259349+0000","last_peered":"2024-09-12T23:52:12.259349+0000","last_clean":"2024-09-12T23:52:12.259349+0000","last_became_active":"2024-09-12T23:46:44.523906+0000","last_became_peered":"2024-09-12T23:46:44.523906+0000","last_unstale":"2024-09-12T23:52:12.259349+0000","last_undegraded":"2024-09-12T23:52:12.259349+0000","last_fullsized":"2024-09-12T23:52:12.259349+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:57:48.277817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":801,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.568887+0000","last_change":"2024-09-12T23:47:43.180987+0000","last_active":"2024-09-12T23:52:11.568887+0000","last_peered":"2024-09-12T23:52:11.568887+0000","last_clean":"2024-09-12T23:52:11.568887+0000","last_became_active":"2024-09-12T23:47:43.179817+0000","last_became_peered":"2024-09-12T23:47:43.179817+0000","last_unstale":"2024-09-12T23:52:11.568887+0000","last_undegraded":"2024-09-12T23:52:11.568887+0000","last_fullsized":"2024-09-12T23:52:11.568887+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:54:37.461736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":817,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:12.487155+0000","last_change":"2024-09-12T23:47:12.986491+0000","last_active":"2024-09-12T23:52:12.487155+0000","last_peered":"2024-09-12T23:52:12.487155+0000","last_clean":"2024-09-12T23:52:12.487155+0000","last_became_active":"2024-09-12T23:47:12.985686+0000","last_became_peered":"2024-09-12T23:47:12.985686+0000","last_unstale":"2024-09-12T23:52:12.487155+0000","last_undegraded":"2024-09-12T23:52:12.487155+0000","last_fullsized":"2024-09-12T23:52:12.487155+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:26:55.503612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":806,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.571312+0000","last_change":"2024-09-12T23:47:43.164005+0000","last_active":"2024-09-12T23:52:11.571312+0000","last_peered":"2024-09-12T23:52:11.571312+0000","last_clean":"2024-09-12T23:52:11.571312+0000","last_became_active":"2024-09-12T23:47:43.161747+0000","last_became_peered":"2024-09-12T23:47:43.161747+0000","last_unstale":"2024-09-12T23:52:11.571312+0000","last_undegraded":"2024-09-12T23:52:11.571312+0000","last_fullsized":"2024-09-12T23:52:11.571312+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:05:44.333448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":820,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.560323+0000","last_change":"2024-09-12T23:47:12.973919+0000","last_active":"2024-09-12T23:52:11.560323+0000","last_peered":"2024-09-12T23:52:11.560323+0000","last_clean":"2024-09-12T23:52:11.560323+0000","last_became_active":"2024-09-12T23:47:12.972678+0000","last_became_peered":"2024-09-12T23:47:12.972678+0000","last_unstale":"2024-09-12T23:52:11.560323+0000","last_undegraded":"2024-09-12T23:52:11.560323+0000","last_fullsized":"2024-09-12T23:52:11.560323+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:14:49.600867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":820,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.559495+0000","last_change":"2024-09-12T23:47:12.981566+0000","last_active":"2024-09-12T23:52:11.559495+0000","last_peered":"2024-09-12T23:52:11.559495+0000","last_clean":"2024-09-12T23:52:11.559495+0000","last_became_active":"2024-09-12T23:47:12.980818+0000","last_became_peered":"2024-09-12T23:47:12.980818+0000","last_unstale":"2024-09-12T23:52:11.559495+0000","last_undegraded":"2024-09-12T23:52:11.559495+0000","last_fullsized":"2024-09-12T23:52:11.559495+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-13T23:52:31.720909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":847,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:12.053667+0000","last_change":"2024-09-12T23:47:12.957046+0000","last_active":"2024-09-12T23:52:12.053667+0000","last_peered":"2024-09-12T23:52:12.053667+0000","last_clean":"2024-09-12T23:52:12.053667+0000","last_became_active":"2024-09-12T23:47:12.956299+0000","last_became_peered":"2024-09-12T23:47:12.956299+0000","last_unstale":"2024-09-12T23:52:12.053667+0000","last_undegraded":"2024-09-12T23:52:12.053667+0000","last_fullsized":"2024-09-12T23:52:12.053667+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-13T23:56:21.477900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":831,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:12.487850+0000","last_change":"2024-09-12T23:47:12.985375+0000","last_active":"2024-09-12T23:52:12.487850+0000","last_peered":"2024-09-12T23:52:12.487850+0000","last_clean":"2024-09-12T23:52:12.487850+0000","last_became_active":"2024-09-12T23:47:12.984536+0000","last_became_peered":"2024-09-12T23:47:12.984536+0000","last_unstale":"2024-09-12T23:52:12.487850+0000","last_undegraded":"2024-09-12T23:52:12.487850+0000","last_fullsized":"2024-09-12T23:52:12.487850+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:27:08.443388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":829,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.564989+0000","last_change":"2024-09-12T23:46:44.508804+0000","last_active":"2024-09-12T23:52:11.564989+0000","last_peered":"2024-09-12T23:52:11.564989+0000","last_clean":"2024-09-12T23:52:11.564989+0000","last_became_active":"2024-09-12T23:46:44.508040+0000","last_became_peered":"2024-09-12T23:46:44.508040+0000","last_unstale":"2024-09-12T23:52:11.564989+0000","last_undegraded":"2024-09-12T23:52:11.564989+0000","last_fullsized":"2024-09-12T23:52:11.564989+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:55:46.775145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":822,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.905984+0000","last_change":"2024-09-12T23:46:44.526515+0000","last_active":"2024-09-12T23:52:11.905984+0000","last_peered":"2024-09-12T23:52:11.905984+0000","last_clean":"2024-09-12T23:52:11.905984+0000","last_became_active":"2024-09-12T23:46:44.526179+0000","last_became_peered":"2024-09-12T23:46:44.526179+0000","last_unstale":"2024-09-12T23:52:11.905984+0000","last_undegraded":"2024-09-12T23:52:11.905984+0000","last_fullsized":"2024-09-12T23:52:11.905984+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:29:54.339301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":821,"reported_epoch":862,"state":"active+clean","last_fresh":"2024-09-12T23:52:16.562025+0000","last_change":"2024-09-12T23:47:43.170710+0000","last_active":"2024-09-12T23:52:16.562025+0000","last_peered":"2024-09-12T23:52:16.562025+0000","last_clean":"2024-09-12T23:52:16.562025+0000","last_became_active":"2024-09-12T23:47:43.169749+0000","last_became_peered":"2024-09-12T23:47:43.169749+0000","last_unstale":"2024-09-12T23:52:16.562025+0000","last_undegraded":"2024-09-12T23:52:16.562025+0000","last_fullsized":"2024-09-12T23:52:16.562025+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:26:13.600649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":823,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:12.259804+0000","last_change":"2024-09-12T23:47:43.156466+0000","last_active":"2024-09-12T23:52:12.259804+0000","last_peered":"2024-09-12T23:52:12.259804+0000","last_clean":"2024-09-12T23:52:12.259804+0000","last_became_active":"2024-09-12T23:47:43.155548+0000","last_became_peered":"2024-09-12T23:47:43.155548+0000","last_unstale":"2024-09-12T23:52:12.259804+0000","last_undegraded":"2024-09-12T23:52:12.259804+0000","last_fullsized":"2024-09-12T23:52:12.259804+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:13:12.389793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":805,"reported_epoch":862,"state":"active+clean","last_fresh":"2024-09-12T23:52:16.564459+0000","last_change":"2024-09-12T23:47:43.177811+0000","last_active":"2024-09-12T23:52:16.564459+0000","last_peered":"2024-09-12T23:52:16.564459+0000","last_clean":"2024-09-12T23:52:16.564459+0000","last_became_active":"2024-09-12T23:47:43.177053+0000","last_became_peered":"2024-09-12T23:47:43.177053+0000","last_unstale":"2024-09-12T23:52:16.564459+0000","last_undegraded":"2024-09-12T23:52:16.564459+0000","last_fullsized":"2024-09-12T23:52:16.564459+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:44:16.830600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":815,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.570374+0000","last_change":"2024-09-12T23:47:43.170287+0000","last_active":"2024-09-12T23:52:11.570374+0000","last_peered":"2024-09-12T23:52:11.570374+0000","last_clean":"2024-09-12T23:52:11.570374+0000","last_became_active":"2024-09-12T23:47:43.168377+0000","last_became_peered":"2024-09-12T23:47:43.168377+0000","last_unstale":"2024-09-12T23:52:11.570374+0000","last_undegraded":"2024-09-12T23:52:11.570374+0000","last_fullsized":"2024-09-12T23:52:11.570374+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:50:04.140369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":834,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.558829+0000","last_change":"2024-09-12T23:47:12.982652+0000","last_active":"2024-09-12T23:52:11.558829+0000","last_peered":"2024-09-12T23:52:11.558829+0000","last_clean":"2024-09-12T23:52:11.558829+0000","last_became_active":"2024-09-12T23:47:12.981804+0000","last_became_peered":"2024-09-12T23:47:12.981804+0000","last_unstale":"2024-09-12T23:52:11.558829+0000","last_undegraded":"2024-09-12T23:52:11.558829+0000","last_fullsized":"2024-09-12T23:52:11.558829+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:03:22.641083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":829,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.565161+0000","last_change":"2024-09-12T23:47:43.161255+0000","last_active":"2024-09-12T23:52:11.565161+0000","last_peered":"2024-09-12T23:52:11.565161+0000","last_clean":"2024-09-12T23:52:11.565161+0000","last_became_active":"2024-09-12T23:47:43.159852+0000","last_became_peered":"2024-09-12T23:47:43.159852+0000","last_unstale":"2024-09-12T23:52:11.565161+0000","last_undegraded":"2024-09-12T23:52:11.565161+0000","last_fullsized":"2024-09-12T23:52:11.565161+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:04:58.884638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":805,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:12.258890+0000","last_change":"2024-09-12T23:47:43.156036+0000","last_active":"2024-09-12T23:52:12.258890+0000","last_peered":"2024-09-12T23:52:12.258890+0000","last_clean":"2024-09-12T23:52:12.258890+0000","last_became_active":"2024-09-12T23:47:43.155231+0000","last_became_peered":"2024-09-12T23:47:43.155231+0000","last_unstale":"2024-09-12T23:52:12.258890+0000","last_undegraded":"2024-09-12T23:52:12.258890+0000","last_fullsized":"2024-09-12T23:52:12.258890+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:36:39.712086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":831,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:12.487727+0000","last_change":"2024-09-12T23:46:16.523897+0000","last_active":"2024-09-12T23:52:12.487727+0000","last_peered":"2024-09-12T23:52:12.487727+0000","last_clean":"2024-09-12T23:52:12.487727+0000","last_became_active":"2024-09-12T23:46:16.523586+0000","last_became_peered":"2024-09-12T23:46:16.523586+0000","last_unstale":"2024-09-12T23:52:12.487727+0000","last_undegraded":"2024-09-12T23:52:12.487727+0000","last_fullsized":"2024-09-12T23:52:12.487727+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:33:55.391977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":838,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:12.258433+0000","last_change":"2024-09-12T23:46:44.495169+0000","last_active":"2024-09-12T23:52:12.258433+0000","last_peered":"2024-09-12T23:52:12.258433+0000","last_clean":"2024-09-12T23:52:12.258433+0000","last_became_active":"2024-09-12T23:46:44.491582+0000","last_became_peered":"2024-09-12T23:46:44.491582+0000","last_unstale":"2024-09-12T23:52:12.258433+0000","last_undegraded":"2024-09-12T23:52:12.258433+0000","last_fullsized":"2024-09-12T23:52:12.258433+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:58:32.836261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":820,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:15.545115+0000","last_change":"2024-09-12T23:46:16.525046+0000","last_active":"2024-09-12T23:52:15.545115+0000","last_peered":"2024-09-12T23:52:15.545115+0000","last_clean":"2024-09-12T23:52:15.545115+0000","last_became_active":"2024-09-12T23:46:16.524183+0000","last_became_peered":"2024-09-12T23:46:16.524183+0000","last_unstale":"2024-09-12T23:52:15.545115+0000","last_undegraded":"2024-09-12T23:52:15.545115+0000","last_fullsized":"2024-09-12T23:52:15.545115+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:27:48.912415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":832,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.559486+0000","last_change":"2024-09-12T23:47:43.150921+0000","last_active":"2024-09-12T23:52:11.559486+0000","last_peered":"2024-09-12T23:52:11.559486+0000","last_clean":"2024-09-12T23:52:11.559486+0000","last_became_active":"2024-09-12T23:47:43.150041+0000","last_became_peered":"2024-09-12T23:47:43.150041+0000","last_unstale":"2024-09-12T23:52:11.559486+0000","last_undegraded":"2024-09-12T23:52:11.559486+0000","last_fullsized":"2024-09-12T23:52:11.559486+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:09:11.982337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":801,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.906534+0000","last_change":"2024-09-12T23:45:47.437790+0000","last_active":"2024-09-12T23:52:11.906534+0000","last_peered":"2024-09-12T23:52:11.906534+0000","last_clean":"2024-09-12T23:52:11.906534+0000","last_became_active":"2024-09-12T23:45:47.436936+0000","last_became_peered":"2024-09-12T23:45:47.436936+0000","last_unstale":"2024-09-12T23:52:11.906534+0000","last_undegraded":"2024-09-12T23:52:11.906534+0000","last_fullsized":"2024-09-12T23:52:11.906534+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:56:15.465789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":804,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:15.545062+0000","last_change":"2024-09-12T23:46:16.523588+0000","last_active":"2024-09-12T23:52:15.545062+0000","last_peered":"2024-09-12T23:52:15.545062+0000","last_clean":"2024-09-12T23:52:15.545062+0000","last_became_active":"2024-09-12T23:46:16.522743+0000","last_became_peered":"2024-09-12T23:46:16.522743+0000","last_unstale":"2024-09-12T23:52:15.545062+0000","last_undegraded":"2024-09-12T23:52:15.545062+0000","last_fullsized":"2024-09-12T23:52:15.545062+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:05:31.156834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":829,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.565378+0000","last_change":"2024-09-12T23:46:16.540542+0000","last_active":"2024-09-12T23:52:11.565378+0000","last_peered":"2024-09-12T23:52:11.565378+0000","last_clean":"2024-09-12T23:52:11.565378+0000","last_became_active":"2024-09-12T23:46:16.539446+0000","last_became_peered":"2024-09-12T23:46:16.539446+0000","last_unstale":"2024-09-12T23:52:11.565378+0000","last_undegraded":"2024-09-12T23:52:11.565378+0000","last_fullsized":"2024-09-12T23:52:11.565378+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:40:33.192069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":838,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:12.259012+0000","last_change":"2024-09-12T23:46:44.500914+0000","last_active":"2024-09-12T23:52:12.259012+0000","last_peered":"2024-09-12T23:52:12.259012+0000","last_clean":"2024-09-12T23:52:12.259012+0000","last_became_active":"2024-09-12T23:46:44.499807+0000","last_became_peered":"2024-09-12T23:46:44.499807+0000","last_unstale":"2024-09-12T23:52:12.259012+0000","last_undegraded":"2024-09-12T23:52:12.259012+0000","last_fullsized":"2024-09-12T23:52:12.259012+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:00:19.166984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":805,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.561078+0000","last_change":"2024-09-12T23:47:12.954732+0000","last_active":"2024-09-12T23:52:11.561078+0000","last_peered":"2024-09-12T23:52:11.561078+0000","last_clean":"2024-09-12T23:52:11.561078+0000","last_became_active":"2024-09-12T23:47:12.953908+0000","last_became_peered":"2024-09-12T23:47:12.953908+0000","last_unstale":"2024-09-12T23:52:11.561078+0000","last_undegraded":"2024-09-12T23:52:11.561078+0000","last_fullsized":"2024-09-12T23:52:11.561078+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-13T23:47:56.339662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":820,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.558405+0000","last_change":"2024-09-12T23:47:12.979269+0000","last_active":"2024-09-12T23:52:11.558405+0000","last_peered":"2024-09-12T23:52:11.558405+0000","last_clean":"2024-09-12T23:52:11.558405+0000","last_became_active":"2024-09-12T23:47:12.978408+0000","last_became_peered":"2024-09-12T23:47:12.978408+0000","last_unstale":"2024-09-12T23:52:11.558405+0000","last_undegraded":"2024-09-12T23:52:11.558405+0000","last_fullsized":"2024-09-12T23:52:11.558405+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-13T23:51:44.218571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":848,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:12.052550+0000","last_change":"2024-09-12T23:46:16.525054+0000","last_active":"2024-09-12T23:52:12.052550+0000","last_peered":"2024-09-12T23:52:12.052550+0000","last_clean":"2024-09-12T23:52:12.052550+0000","last_became_active":"2024-09-12T23:46:16.524781+0000","last_became_peered":"2024-09-12T23:46:16.524781+0000","last_unstale":"2024-09-12T23:52:12.052550+0000","last_undegraded":"2024-09-12T23:52:12.052550+0000","last_fullsized":"2024-09-12T23:52:12.052550+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:43:27.737083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1b","version":"644'2","reported_seq":820,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:12.488041+0000","last_change":"2024-09-12T23:47:43.170740+0000","last_active":"2024-09-12T23:52:12.488041+0000","last_peered":"2024-09-12T23:52:12.488041+0000","last_clean":"2024-09-12T23:52:12.488041+0000","last_became_active":"2024-09-12T23:47:43.169673+0000","last_became_peered":"2024-09-12T23:47:43.169673+0000","last_unstale":"2024-09-12T23:52:12.488041+0000","last_undegraded":"2024-09-12T23:52:12.488041+0000","last_fullsized":"2024-09-12T23:52:12.488041+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:33:14.780464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":829,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.565143+0000","last_change":"2024-09-12T23:47:43.162111+0000","last_active":"2024-09-12T23:52:11.565143+0000","last_peered":"2024-09-12T23:52:11.565143+0000","last_clean":"2024-09-12T23:52:11.565143+0000","last_became_active":"2024-09-12T23:47:43.160887+0000","last_became_peered":"2024-09-12T23:47:43.160887+0000","last_unstale":"2024-09-12T23:52:11.565143+0000","last_undegraded":"2024-09-12T23:52:11.565143+0000","last_fullsized":"2024-09-12T23:52:11.565143+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:44:28.770668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1a","version":"644'2","reported_seq":844,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.558668+0000","last_change":"2024-09-12T23:47:12.981329+0000","last_active":"2024-09-12T23:52:11.558668+0000","last_peered":"2024-09-12T23:52:11.558668+0000","last_clean":"2024-09-12T23:52:11.558668+0000","last_became_active":"2024-09-12T23:47:12.980631+0000","last_became_peered":"2024-09-12T23:47:12.980631+0000","last_unstale":"2024-09-12T23:52:11.558668+0000","last_undegraded":"2024-09-12T23:52:11.558668+0000","last_fullsized":"2024-09-12T23:52:11.558668+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:15:05.399298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":804,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:12.487054+0000","last_change":"2024-09-12T23:47:12.985499+0000","last_active":"2024-09-12T23:52:12.487054+0000","last_peered":"2024-09-12T23:52:12.487054+0000","last_clean":"2024-09-12T23:52:12.487054+0000","last_became_active":"2024-09-12T23:47:12.984915+0000","last_became_peered":"2024-09-12T23:47:12.984915+0000","last_unstale":"2024-09-12T23:52:12.487054+0000","last_undegraded":"2024-09-12T23:52:12.487054+0000","last_fullsized":"2024-09-12T23:52:12.487054+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:57:15.399289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":833,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:12.487105+0000","last_change":"2024-09-12T23:45:47.449193+0000","last_active":"2024-09-12T23:52:12.487105+0000","last_peered":"2024-09-12T23:52:12.487105+0000","last_clean":"2024-09-12T23:52:12.487105+0000","last_became_active":"2024-09-12T23:45:47.448772+0000","last_became_peered":"2024-09-12T23:45:47.448772+0000","last_unstale":"2024-09-12T23:52:12.487105+0000","last_undegraded":"2024-09-12T23:52:12.487105+0000","last_fullsized":"2024-09-12T23:52:12.487105+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:31:02.252604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":824,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:12.258365+0000","last_change":"2024-09-12T23:47:43.154323+0000","last_active":"2024-09-12T23:52:12.258365+0000","last_peered":"2024-09-12T23:52:12.258365+0000","last_clean":"2024-09-12T23:52:12.258365+0000","last_became_active":"2024-09-12T23:47:43.153490+0000","last_became_peered":"2024-09-12T23:47:43.153490+0000","last_unstale":"2024-09-12T23:52:12.258365+0000","last_undegraded":"2024-09-12T23:52:12.258365+0000","last_fullsized":"2024-09-12T23:52:12.258365+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:45:35.481548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":801,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.905898+0000","last_change":"2024-09-12T23:46:16.530834+0000","last_active":"2024-09-12T23:52:11.905898+0000","last_peered":"2024-09-12T23:52:11.905898+0000","last_clean":"2024-09-12T23:52:11.905898+0000","last_became_active":"2024-09-12T23:46:16.530192+0000","last_became_peered":"2024-09-12T23:46:16.530192+0000","last_unstale":"2024-09-12T23:52:11.905898+0000","last_undegraded":"2024-09-12T23:52:11.905898+0000","last_fullsized":"2024-09-12T23:52:11.905898+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:31:25.328575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":821,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.560392+0000","last_change":"2024-09-12T23:47:12.972973+0000","last_active":"2024-09-12T23:52:11.560392+0000","last_peered":"2024-09-12T23:52:11.560392+0000","last_clean":"2024-09-12T23:52:11.560392+0000","last_became_active":"2024-09-12T23:47:12.972019+0000","last_became_peered":"2024-09-12T23:47:12.972019+0000","last_unstale":"2024-09-12T23:52:11.560392+0000","last_undegraded":"2024-09-12T23:52:11.560392+0000","last_fullsized":"2024-09-12T23:52:11.560392+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:37:57.827955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":819,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:12.260612+0000","last_change":"2024-09-12T23:47:43.157269+0000","last_active":"2024-09-12T23:52:12.260612+0000","last_peered":"2024-09-12T23:52:12.260612+0000","last_clean":"2024-09-12T23:52:12.260612+0000","last_became_active":"2024-09-12T23:47:43.156520+0000","last_became_peered":"2024-09-12T23:47:43.156520+0000","last_unstale":"2024-09-12T23:52:12.260612+0000","last_undegraded":"2024-09-12T23:52:12.260612+0000","last_fullsized":"2024-09-12T23:52:12.260612+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:24:00.804142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":809,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:12.260362+0000","last_change":"2024-09-12T23:47:12.970172+0000","last_active":"2024-09-12T23:52:12.260362+0000","last_peered":"2024-09-12T23:52:12.260362+0000","last_clean":"2024-09-12T23:52:12.260362+0000","last_became_active":"2024-09-12T23:47:12.968409+0000","last_became_peered":"2024-09-12T23:47:12.968409+0000","last_unstale":"2024-09-12T23:52:12.260362+0000","last_undegraded":"2024-09-12T23:52:12.260362+0000","last_fullsized":"2024-09-12T23:52:12.260362+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:01:40.306546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":829,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.566600+0000","last_change":"2024-09-12T23:46:44.499459+0000","last_active":"2024-09-12T23:52:11.566600+0000","last_peered":"2024-09-12T23:52:11.566600+0000","last_clean":"2024-09-12T23:52:11.566600+0000","last_became_active":"2024-09-12T23:46:44.498644+0000","last_became_peered":"2024-09-12T23:46:44.498644+0000","last_unstale":"2024-09-12T23:52:11.566600+0000","last_undegraded":"2024-09-12T23:52:11.566600+0000","last_fullsized":"2024-09-12T23:52:11.566600+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:27:24.945981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"53'3","reported_seq":834,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.906918+0000","last_change":"2024-09-12T23:46:44.529120+0000","last_active":"2024-09-12T23:52:11.906918+0000","last_peered":"2024-09-12T23:52:11.906918+0000","last_clean":"2024-09-12T23:52:11.906918+0000","last_became_active":"2024-09-12T23:46:44.528558+0000","last_became_peered":"2024-09-12T23:46:44.528558+0000","last_unstale":"2024-09-12T23:52:11.906918+0000","last_undegraded":"2024-09-12T23:52:11.906918+0000","last_fullsized":"2024-09-12T23:52:11.906918+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:47:02.689409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":7,"num_read_kb":7,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":806,"reported_epoch":862,"state":"active+clean","last_fresh":"2024-09-12T23:52:16.563214+0000","last_change":"2024-09-12T23:47:43.167647+0000","last_active":"2024-09-12T23:52:16.563214+0000","last_peered":"2024-09-12T23:52:16.563214+0000","last_clean":"2024-09-12T23:52:16.563214+0000","last_became_active":"2024-09-12T23:47:43.166675+0000","last_became_peered":"2024-09-12T23:47:43.166675+0000","last_unstale":"2024-09-12T23:52:16.563214+0000","last_undegraded":"2024-09-12T23:52:16.563214+0000","last_fullsized":"2024-09-12T23:52:16.563214+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:56:15.988285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,6],"acting":[2,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.14","version":"644'29","reported_seq":856,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.559148+0000","last_change":"2024-09-12T23:47:12.974561+0000","last_active":"2024-09-12T23:52:11.559148+0000","last_peered":"2024-09-12T23:52:11.559148+0000","last_clean":"2024-09-12T23:52:11.559148+0000","last_became_active":"2024-09-12T23:47:12.974155+0000","last_became_peered":"2024-09-12T23:47:12.974155+0000","last_unstale":"2024-09-12T23:52:11.559148+0000","last_undegraded":"2024-09-12T23:52:11.559148+0000","last_fullsized":"2024-09-12T23:52:11.559148+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":29,"log_dups_size":0,"ondisk_log_size":29,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:45:59.274136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5822,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":10,"num_read_kb":14,"num_write":28,"num_write_kb":25,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":801,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.570506+0000","last_change":"2024-09-12T23:47:43.182552+0000","last_active":"2024-09-12T23:52:11.570506+0000","last_peered":"2024-09-12T23:52:11.570506+0000","last_clean":"2024-09-12T23:52:11.570506+0000","last_became_active":"2024-09-12T23:47:43.181933+0000","last_became_peered":"2024-09-12T23:47:43.181933+0000","last_unstale":"2024-09-12T23:52:11.570506+0000","last_undegraded":"2024-09-12T23:52:11.570506+0000","last_fullsized":"2024-09-12T23:52:11.570506+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:45:02.137824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.13","version":"53'2","reported_seq":834,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.565222+0000","last_change":"2024-09-12T23:46:16.540119+0000","last_active":"2024-09-12T23:52:11.565222+0000","last_peered":"2024-09-12T23:52:11.565222+0000","last_clean":"2024-09-12T23:52:11.565222+0000","last_became_active":"2024-09-12T23:46:16.538396+0000","last_became_peered":"2024-09-12T23:46:16.538396+0000","last_unstale":"2024-09-12T23:52:11.565222+0000","last_undegraded":"2024-09-12T23:52:11.565222+0000","last_fullsized":"2024-09-12T23:52:11.565222+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:36:34.146040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":7,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":829,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.564921+0000","last_change":"2024-09-12T23:47:43.160555+0000","last_active":"2024-09-12T23:52:11.564921+0000","last_peered":"2024-09-12T23:52:11.564921+0000","last_clean":"2024-09-12T23:52:11.564921+0000","last_became_active":"2024-09-12T23:47:43.159675+0000","last_became_peered":"2024-09-12T23:47:43.159675+0000","last_unstale":"2024-09-12T23:52:11.564921+0000","last_undegraded":"2024-09-12T23:52:11.564921+0000","last_fullsized":"2024-09-12T23:52:11.564921+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:00:36.421427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":837,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:12.259428+0000","last_change":"2024-09-12T23:47:43.156165+0000","last_active":"2024-09-12T23:52:12.259428+0000","last_peered":"2024-09-12T23:52:12.259428+0000","last_clean":"2024-09-12T23:52:12.259428+0000","last_became_active":"2024-09-12T23:47:43.155397+0000","last_became_peered":"2024-09-12T23:47:43.155397+0000","last_unstale":"2024-09-12T23:52:12.259428+0000","last_undegraded":"2024-09-12T23:52:12.259428+0000","last_fullsized":"2024-09-12T23:52:12.259428+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:41:20.729255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":802,"reported_epoch":862,"state":"active+clean","last_fresh":"2024-09-12T23:52:16.654856+0000","last_change":"2024-09-12T23:47:43.179668+0000","last_active":"2024-09-12T23:52:16.654856+0000","last_peered":"2024-09-12T23:52:16.654856+0000","last_clean":"2024-09-12T23:52:16.654856+0000","last_became_active":"2024-09-12T23:47:43.178900+0000","last_became_peered":"2024-09-12T23:47:43.178900+0000","last_unstale":"2024-09-12T23:52:16.654856+0000","last_undegraded":"2024-09-12T23:52:16.654856+0000","last_fullsized":"2024-09-12T23:52:16.654856+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:25:12.971259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":835,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.558674+0000","last_change":"2024-09-12T23:47:12.979665+0000","last_active":"2024-09-12T23:52:11.558674+0000","last_peered":"2024-09-12T23:52:11.558674+0000","last_clean":"2024-09-12T23:52:11.558674+0000","last_became_active":"2024-09-12T23:47:12.978616+0000","last_became_peered":"2024-09-12T23:47:12.978616+0000","last_unstale":"2024-09-12T23:52:11.558674+0000","last_undegraded":"2024-09-12T23:52:11.558674+0000","last_fullsized":"2024-09-12T23:52:11.558674+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:48:28.261076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":830,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.558406+0000","last_change":"2024-09-12T23:47:12.973510+0000","last_active":"2024-09-12T23:52:11.558406+0000","last_peered":"2024-09-12T23:52:11.558406+0000","last_clean":"2024-09-12T23:52:11.558406+0000","last_became_active":"2024-09-12T23:47:12.971987+0000","last_became_peered":"2024-09-12T23:47:12.971987+0000","last_unstale":"2024-09-12T23:52:11.558406+0000","last_undegraded":"2024-09-12T23:52:11.558406+0000","last_fullsized":"2024-09-12T23:52:11.558406+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:15:17.279306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"53'8","reported_seq":837,"reported_epoch":862,"state":"active+clean","last_fresh":"2024-09-12T23:52:16.562194+0000","last_change":"2024-09-12T23:45:18.505110+0000","last_active":"2024-09-12T23:52:16.562194+0000","last_peered":"2024-09-12T23:52:16.562194+0000","last_clean":"2024-09-12T23:52:16.562194+0000","last_became_active":"2024-09-12T23:45:18.504350+0000","last_became_peered":"2024-09-12T23:45:18.504350+0000","last_unstale":"2024-09-12T23:52:16.562194+0000","last_undegraded":"2024-09-12T23:52:16.562194+0000","last_fullsized":"2024-09-12T23:52:16.562194+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:05:22.926035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":24,"num_read_kb":24,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":494,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.561695+0000","last_change":"2024-09-12T23:47:12.958785+0000","last_active":"2024-09-12T23:52:11.561695+0000","last_peered":"2024-09-12T23:52:11.561695+0000","last_clean":"2024-09-12T23:52:11.561695+0000","last_became_active":"2024-09-12T23:47:12.957889+0000","last_became_peered":"2024-09-12T23:47:12.957889+0000","last_unstale":"2024-09-12T23:52:11.561695+0000","last_undegraded":"2024-09-12T23:52:11.561695+0000","last_fullsized":"2024-09-12T23:52:11.561695+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:19:21.067825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.8","version":"860'2076","reported_seq":4458,"reported_epoch":862,"state":"active+clean","last_fresh":"2024-09-12T23:52:16.562503+0000","last_change":"2024-09-12T23:52:09.590084+0000","last_active":"2024-09-12T23:52:16.562503+0000","last_peered":"2024-09-12T23:52:16.562503+0000","last_clean":"2024-09-12T23:52:16.562503+0000","last_became_active":"2024-09-12T23:45:18.505988+0000","last_became_peered":"2024-09-12T23:45:18.505988+0000","last_unstale":"2024-09-12T23:52:16.562503+0000","last_undegraded":"2024-09-12T23:52:16.562503+0000","last_fullsized":"2024-09-12T23:52:16.562503+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:38:14.355285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:38:14.355285+0000","last_clean_scrub_stamp":"2024-09-12T23:38:14.355285+0000","objects_scrubbed":0,"log_size":2076,"log_dups_size":0,"ondisk_log_size":2076,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:08:20.337107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.019235697999999999,"stat_sum":{"num_bytes":21889024,"num_objects":16,"num_object_clones":11,"num_object_copies":48,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":16,"num_whiteouts":0,"num_read":370,"num_read_kb":547719,"num_write":782,"num_write_kb":186944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17,"num_bytes_recovered":54910976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.f","version":"53'1","reported_seq":832,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:12.054378+0000","last_change":"2024-09-12T23:47:43.173494+0000","last_active":"2024-09-12T23:52:12.054378+0000","last_peered":"2024-09-12T23:52:12.054378+0000","last_clean":"2024-09-12T23:52:12.054378+0000","last_became_active":"2024-09-12T23:47:43.172568+0000","last_became_peered":"2024-09-12T23:47:43.172568+0000","last_unstale":"2024-09-12T23:52:12.054378+0000","last_undegraded":"2024-09-12T23:52:12.054378+0000","last_fullsized":"2024-09-12T23:52:12.054378+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:39:41.538358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":802,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.571182+0000","last_change":"2024-09-12T23:47:43.179656+0000","last_active":"2024-09-12T23:52:11.571182+0000","last_peered":"2024-09-12T23:52:11.571182+0000","last_clean":"2024-09-12T23:52:11.571182+0000","last_became_active":"2024-09-12T23:47:43.179076+0000","last_became_peered":"2024-09-12T23:47:43.179076+0000","last_unstale":"2024-09-12T23:52:11.571182+0000","last_undegraded":"2024-09-12T23:52:11.571182+0000","last_fullsized":"2024-09-12T23:52:11.571182+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:16:33.206689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.9","version":"858'850","reported_seq":3302,"reported_epoch":862,"state":"active+clean","last_fresh":"2024-09-12T23:52:16.654894+0000","last_change":"2024-09-12T23:52:09.594085+0000","last_active":"2024-09-12T23:52:16.654894+0000","last_peered":"2024-09-12T23:52:16.654894+0000","last_clean":"2024-09-12T23:52:16.654894+0000","last_became_active":"2024-09-12T23:47:43.160755+0000","last_became_peered":"2024-09-12T23:47:43.160755+0000","last_unstale":"2024-09-12T23:52:16.654894+0000","last_undegraded":"2024-09-12T23:52:16.654894+0000","last_fullsized":"2024-09-12T23:52:16.654894+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:38:14.355285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:38:14.355285+0000","last_clean_scrub_stamp":"2024-09-12T23:38:14.355285+0000","objects_scrubbed":0,"log_size":850,"log_dups_size":0,"ondisk_log_size":850,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:36:32.939588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0064653829999999999,"stat_sum":{"num_bytes":11919360,"num_objects":4,"num_object_clones":2,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":138,"num_read_kb":155752,"num_write":380,"num_write_kb":66296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":15335424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"53'2","reported_seq":810,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.563607+0000","last_change":"2024-09-12T23:47:43.179483+0000","last_active":"2024-09-12T23:52:11.563607+0000","last_peered":"2024-09-12T23:52:11.563607+0000","last_clean":"2024-09-12T23:52:11.563607+0000","last_became_active":"2024-09-12T23:47:43.178127+0000","last_became_peered":"2024-09-12T23:47:43.178127+0000","last_unstale":"2024-09-12T23:52:11.563607+0000","last_undegraded":"2024-09-12T23:52:11.563607+0000","last_fullsized":"2024-09-12T23:52:11.563607+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:22:15.876553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":802,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.571570+0000","last_change":"2024-09-12T23:47:43.182310+0000","last_active":"2024-09-12T23:52:11.571570+0000","last_peered":"2024-09-12T23:52:11.571570+0000","last_clean":"2024-09-12T23:52:11.571570+0000","last_became_active":"2024-09-12T23:47:43.181625+0000","last_became_peered":"2024-09-12T23:47:43.181625+0000","last_unstale":"2024-09-12T23:52:11.571570+0000","last_undegraded":"2024-09-12T23:52:11.571570+0000","last_fullsized":"2024-09-12T23:52:11.571570+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:20:04.443027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.a","version":"860'2607","reported_seq":4975,"reported_epoch":862,"state":"active+clean","last_fresh":"2024-09-12T23:52:16.562650+0000","last_change":"2024-09-12T23:52:09.602392+0000","last_active":"2024-09-12T23:52:16.562650+0000","last_peered":"2024-09-12T23:52:16.562650+0000","last_clean":"2024-09-12T23:52:16.562650+0000","last_became_active":"2024-09-12T23:46:16.522442+0000","last_became_peered":"2024-09-12T23:46:16.522442+0000","last_unstale":"2024-09-12T23:52:16.562650+0000","last_undegraded":"2024-09-12T23:52:16.562650+0000","last_fullsized":"2024-09-12T23:52:16.562650+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:38:14.355285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:38:14.355285+0000","last_clean_scrub_stamp":"2024-09-12T23:38:14.355285+0000","objects_scrubbed":0,"log_size":2607,"log_dups_size":0,"ondisk_log_size":2607,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:37:55.896368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.026112559,"stat_sum":{"num_bytes":36732928,"num_objects":16,"num_object_clones":10,"num_object_copies":48,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":16,"num_whiteouts":0,"num_read":470,"num_read_kb":738585,"num_write":1054,"num_write_kb":215424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23,"num_bytes_recovered":161964032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.d","version":"53'1","reported_seq":804,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.906920+0000","last_change":"2024-09-12T23:46:16.531104+0000","last_active":"2024-09-12T23:52:11.906920+0000","last_peered":"2024-09-12T23:52:11.906920+0000","last_clean":"2024-09-12T23:52:11.906920+0000","last_became_active":"2024-09-12T23:46:16.530411+0000","last_became_peered":"2024-09-12T23:46:16.530411+0000","last_unstale":"2024-09-12T23:52:11.906920+0000","last_undegraded":"2024-09-12T23:52:11.906920+0000","last_fullsized":"2024-09-12T23:52:11.906920+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:44:00.314763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":836,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:12.259825+0000","last_change":"2024-09-12T23:47:12.959179+0000","last_active":"2024-09-12T23:52:12.259825+0000","last_peered":"2024-09-12T23:52:12.259825+0000","last_clean":"2024-09-12T23:52:12.259825+0000","last_became_active":"2024-09-12T23:47:12.958368+0000","last_became_peered":"2024-09-12T23:47:12.958368+0000","last_unstale":"2024-09-12T23:52:12.259825+0000","last_undegraded":"2024-09-12T23:52:12.259825+0000","last_fullsized":"2024-09-12T23:52:12.259825+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:39:42.877253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.b","version":"0'0","reported_seq":2453,"reported_epoch":862,"state":"active+clean","last_fresh":"2024-09-12T23:52:16.563582+0000","last_change":"2024-09-12T23:52:09.575970+0000","last_active":"2024-09-12T23:52:16.563582+0000","last_peered":"2024-09-12T23:52:16.563582+0000","last_clean":"2024-09-12T23:52:16.563582+0000","last_became_active":"2024-09-12T23:46:44.504250+0000","last_became_peered":"2024-09-12T23:46:44.504250+0000","last_unstale":"2024-09-12T23:52:16.563582+0000","last_undegraded":"2024-09-12T23:52:16.563582+0000","last_fullsized":"2024-09-12T23:52:16.563582+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:38:14.355285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:38:14.355285+0000","last_clean_scrub_stamp":"2024-09-12T23:38:14.355285+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:56:25.933952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0050500950000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.c","version":"0'0","reported_seq":832,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.562417+0000","last_change":"2024-09-12T23:46:44.504058+0000","last_active":"2024-09-12T23:52:11.562417+0000","last_peered":"2024-09-12T23:52:11.562417+0000","last_clean":"2024-09-12T23:52:11.562417+0000","last_became_active":"2024-09-12T23:46:44.503196+0000","last_became_peered":"2024-09-12T23:46:44.503196+0000","last_unstale":"2024-09-12T23:52:11.562417+0000","last_undegraded":"2024-09-12T23:52:11.562417+0000","last_fullsized":"2024-09-12T23:52:11.562417+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:33:40.759838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":815,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.570618+0000","last_change":"2024-09-12T23:47:43.176092+0000","last_active":"2024-09-12T23:52:11.570618+0000","last_peered":"2024-09-12T23:52:11.570618+0000","last_clean":"2024-09-12T23:52:11.570618+0000","last_became_active":"2024-09-12T23:47:43.175575+0000","last_became_peered":"2024-09-12T23:47:43.175575+0000","last_unstale":"2024-09-12T23:52:11.570618+0000","last_undegraded":"2024-09-12T23:52:11.570618+0000","last_fullsized":"2024-09-12T23:52:11.570618+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:49:05.765288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.4","version":"860'751","reported_seq":3221,"reported_epoch":862,"state":"active+clean","last_fresh":"2024-09-12T23:52:16.655665+0000","last_change":"2024-09-12T23:52:09.619104+0000","last_active":"2024-09-12T23:52:16.655665+0000","last_peered":"2024-09-12T23:52:16.655665+0000","last_clean":"2024-09-12T23:52:16.655665+0000","last_became_active":"2024-09-12T23:47:43.168537+0000","last_became_peered":"2024-09-12T23:47:43.168537+0000","last_unstale":"2024-09-12T23:52:16.655665+0000","last_undegraded":"2024-09-12T23:52:16.655665+0000","last_fullsized":"2024-09-12T23:52:16.655665+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:38:14.355285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:38:14.355285+0000","last_clean_scrub_stamp":"2024-09-12T23:38:14.355285+0000","objects_scrubbed":0,"log_size":751,"log_dups_size":0,"ondisk_log_size":751,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:06:55.298972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.030892125999999999,"stat_sum":{"num_bytes":8585216,"num_objects":5,"num_object_clones":3,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":141,"num_read_kb":126362,"num_write":317,"num_write_kb":60304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":34889728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":833,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.559683+0000","last_change":"2024-09-12T23:47:43.169865+0000","last_active":"2024-09-12T23:52:11.559683+0000","last_peered":"2024-09-12T23:52:11.559683+0000","last_clean":"2024-09-12T23:52:11.559683+0000","last_became_active":"2024-09-12T23:47:43.169082+0000","last_became_peered":"2024-09-12T23:47:43.169082+0000","last_unstale":"2024-09-12T23:52:11.559683+0000","last_undegraded":"2024-09-12T23:52:11.559683+0000","last_fullsized":"2024-09-12T23:52:11.559683+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:34:48.454503+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:34:48.454503+0000","last_clean_scrub_stamp":"2024-09-12T23:34:48.454503+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:25:42.909256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00079141900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"319'176","reported_seq":1199,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.569015+0000","last_change":"2024-09-12T23:47:43.159977+0000","last_active":"2024-09-12T23:52:11.569015+0000","last_peered":"2024-09-12T23:52:11.569015+0000","last_clean":"2024-09-12T23:52:11.569015+0000","last_became_active":"2024-09-12T23:47:43.157530+0000","last_became_peered":"2024-09-12T23:47:43.157530+0000","last_unstale":"2024-09-12T23:52:11.569015+0000","last_undegraded":"2024-09-12T23:52:11.569015+0000","last_fullsized":"2024-09-12T23:52:11.569015+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:28:34.324093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:28:34.324093+0000","last_clean_scrub_stamp":"2024-09-12T23:28:34.324093+0000","objects_scrubbed":0,"log_size":176,"log_dups_size":0,"ondisk_log_size":176,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:05:26.015459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":984,"num_read_kb":1829,"num_write":301,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":818,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:12.259062+0000","last_change":"2024-09-12T23:46:44.498084+0000","last_active":"2024-09-12T23:52:12.259062+0000","last_peered":"2024-09-12T23:52:12.259062+0000","last_clean":"2024-09-12T23:52:12.259062+0000","last_became_active":"2024-09-12T23:46:44.497312+0000","last_became_peered":"2024-09-12T23:46:44.497312+0000","last_unstale":"2024-09-12T23:52:12.259062+0000","last_undegraded":"2024-09-12T23:52:12.259062+0000","last_fullsized":"2024-09-12T23:52:12.259062+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:03:33.741908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":818,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:12.487432+0000","last_change":"2024-09-12T23:47:12.985179+0000","last_active":"2024-09-12T23:52:12.487432+0000","last_peered":"2024-09-12T23:52:12.487432+0000","last_clean":"2024-09-12T23:52:12.487432+0000","last_became_active":"2024-09-12T23:47:12.984288+0000","last_became_peered":"2024-09-12T23:47:12.984288+0000","last_unstale":"2024-09-12T23:52:12.487432+0000","last_undegraded":"2024-09-12T23:52:12.487432+0000","last_fullsized":"2024-09-12T23:52:12.487432+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:39:49.379734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.7","version":"860'1003","reported_seq":3451,"reported_epoch":862,"state":"active+clean","last_fresh":"2024-09-12T23:52:16.562248+0000","last_change":"2024-09-12T23:52:09.590939+0000","last_active":"2024-09-12T23:52:16.562248+0000","last_peered":"2024-09-12T23:52:16.562248+0000","last_clean":"2024-09-12T23:52:16.562248+0000","last_became_active":"2024-09-12T23:46:44.522943+0000","last_became_peered":"2024-09-12T23:46:44.522943+0000","last_unstale":"2024-09-12T23:52:16.562248+0000","last_undegraded":"2024-09-12T23:52:16.562248+0000","last_fullsized":"2024-09-12T23:52:16.562248+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:38:14.355285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:38:14.355285+0000","last_clean_scrub_stamp":"2024-09-12T23:38:14.355285+0000","objects_scrubbed":0,"log_size":1003,"log_dups_size":0,"ondisk_log_size":1003,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:37:41.188569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.014112016,"stat_sum":{"num_bytes":50593792,"num_objects":7,"num_object_clones":5,"num_object_copies":21,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7,"num_whiteouts":0,"num_read":161,"num_read_kb":329068,"num_write":331,"num_write_kb":90144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":12230656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":839,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.565853+0000","last_change":"2024-09-12T23:45:18.506986+0000","last_active":"2024-09-12T23:52:11.565853+0000","last_peered":"2024-09-12T23:52:11.565853+0000","last_clean":"2024-09-12T23:52:11.565853+0000","last_became_active":"2024-09-12T23:45:18.506668+0000","last_became_peered":"2024-09-12T23:45:18.506668+0000","last_unstale":"2024-09-12T23:52:11.565853+0000","last_undegraded":"2024-09-12T23:52:11.565853+0000","last_fullsized":"2024-09-12T23:52:11.565853+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:34:48.454503+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:34:48.454503+0000","last_clean_scrub_stamp":"2024-09-12T23:34:48.454503+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:57:51.373144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":798,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.569253+0000","last_change":"2024-09-12T23:47:43.153679+0000","last_active":"2024-09-12T23:52:11.569253+0000","last_peered":"2024-09-12T23:52:11.569253+0000","last_clean":"2024-09-12T23:52:11.569253+0000","last_became_active":"2024-09-12T23:47:43.152840+0000","last_became_peered":"2024-09-12T23:47:43.152840+0000","last_unstale":"2024-09-12T23:52:11.569253+0000","last_undegraded":"2024-09-12T23:52:11.569253+0000","last_fullsized":"2024-09-12T23:52:11.569253+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-13T23:41:49.901898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":829,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.564062+0000","last_change":"2024-09-12T23:46:16.537953+0000","last_active":"2024-09-12T23:52:11.564062+0000","last_peered":"2024-09-12T23:52:11.564062+0000","last_clean":"2024-09-12T23:52:11.564062+0000","last_became_active":"2024-09-12T23:46:16.537543+0000","last_became_peered":"2024-09-12T23:46:16.537543+0000","last_unstale":"2024-09-12T23:52:11.564062+0000","last_undegraded":"2024-09-12T23:52:11.564062+0000","last_fullsized":"2024-09-12T23:52:11.564062+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:21:04.392080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.6","version":"860'2026","reported_seq":4407,"reported_epoch":862,"state":"active+clean","last_fresh":"2024-09-12T23:52:16.565068+0000","last_change":"2024-09-12T23:52:09.620082+0000","last_active":"2024-09-12T23:52:16.565068+0000","last_peered":"2024-09-12T23:52:16.565068+0000","last_clean":"2024-09-12T23:52:16.565068+0000","last_became_active":"2024-09-12T23:46:44.504576+0000","last_became_peered":"2024-09-12T23:46:44.504576+0000","last_unstale":"2024-09-12T23:52:16.565068+0000","last_undegraded":"2024-09-12T23:52:16.565068+0000","last_fullsized":"2024-09-12T23:52:16.565068+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:38:14.355285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:38:14.355285+0000","last_clean_scrub_stamp":"2024-09-12T23:38:14.355285+0000","objects_scrubbed":0,"log_size":2026,"log_dups_size":0,"ondisk_log_size":2026,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:50:48.253944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.040561187999999998,"stat_sum":{"num_bytes":51617792,"num_objects":13,"num_object_clones":8,"num_object_copies":39,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13,"num_whiteouts":0,"num_read":372,"num_read_kb":771948,"num_write":804,"num_write_kb":222704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":39419904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":828,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.906588+0000","last_change":"2024-09-12T23:46:44.535029+0000","last_active":"2024-09-12T23:52:11.906588+0000","last_peered":"2024-09-12T23:52:11.906588+0000","last_clean":"2024-09-12T23:52:11.906588+0000","last_became_active":"2024-09-12T23:46:44.534537+0000","last_became_peered":"2024-09-12T23:46:44.534537+0000","last_unstale":"2024-09-12T23:52:11.906588+0000","last_undegraded":"2024-09-12T23:52:11.906588+0000","last_fullsized":"2024-09-12T23:52:11.906588+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:34:48.454503+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:34:48.454503+0000","last_clean_scrub_stamp":"2024-09-12T23:34:48.454503+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:31:24.049302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0020296509999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":832,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.561259+0000","last_change":"2024-09-12T23:45:47.454123+0000","last_active":"2024-09-12T23:52:11.561259+0000","last_peered":"2024-09-12T23:52:11.561259+0000","last_clean":"2024-09-12T23:52:11.561259+0000","last_became_active":"2024-09-12T23:45:47.453245+0000","last_became_peered":"2024-09-12T23:45:47.453245+0000","last_unstale":"2024-09-12T23:52:11.561259+0000","last_undegraded":"2024-09-12T23:52:11.561259+0000","last_fullsized":"2024-09-12T23:52:11.561259+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:18:05.054249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":801,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.906142+0000","last_change":"2024-09-12T23:47:12.957532+0000","last_active":"2024-09-12T23:52:11.906142+0000","last_peered":"2024-09-12T23:52:11.906142+0000","last_clean":"2024-09-12T23:52:11.906142+0000","last_became_active":"2024-09-12T23:47:12.956305+0000","last_became_peered":"2024-09-12T23:47:12.956305+0000","last_unstale":"2024-09-12T23:52:11.906142+0000","last_undegraded":"2024-09-12T23:52:11.906142+0000","last_fullsized":"2024-09-12T23:52:11.906142+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-13T23:39:38.072580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.5","version":"858'2978","reported_seq":5343,"reported_epoch":862,"state":"active+clean","last_fresh":"2024-09-12T23:52:16.569574+0000","last_change":"2024-09-12T23:52:09.671768+0000","last_active":"2024-09-12T23:52:16.569574+0000","last_peered":"2024-09-12T23:52:16.569574+0000","last_clean":"2024-09-12T23:52:16.569574+0000","last_became_active":"2024-09-12T23:46:16.548366+0000","last_became_peered":"2024-09-12T23:46:16.548366+0000","last_unstale":"2024-09-12T23:52:16.569574+0000","last_undegraded":"2024-09-12T23:52:16.569574+0000","last_fullsized":"2024-09-12T23:52:16.569574+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:38:14.355285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:38:14.355285+0000","last_clean_scrub_stamp":"2024-09-12T23:38:14.355285+0000","objects_scrubbed":0,"log_size":2978,"log_dups_size":0,"ondisk_log_size":2978,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:07:44.175011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.090023704999999996,"stat_sum":{"num_bytes":46915584,"num_objects":12,"num_object_clones":6,"num_object_copies":36,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12,"num_whiteouts":0,"num_read":593,"num_read_kb":1196250,"num_write":1195,"num_write_kb":246296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":108748800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":811,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.907345+0000","last_change":"2024-09-12T23:45:47.418928+0000","last_active":"2024-09-12T23:52:11.907345+0000","last_peered":"2024-09-12T23:52:11.907345+0000","last_clean":"2024-09-12T23:52:11.907345+0000","last_became_active":"2024-09-12T23:45:47.418140+0000","last_became_peered":"2024-09-12T23:45:47.418140+0000","last_unstale":"2024-09-12T23:52:11.907345+0000","last_undegraded":"2024-09-12T23:52:11.907345+0000","last_fullsized":"2024-09-12T23:52:11.907345+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:34:48.454503+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:34:48.454503+0000","last_clean_scrub_stamp":"2024-09-12T23:34:48.454503+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-13T23:41:02.551247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":820,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:12.259245+0000","last_change":"2024-09-12T23:47:12.959312+0000","last_active":"2024-09-12T23:52:12.259245+0000","last_peered":"2024-09-12T23:52:12.259245+0000","last_clean":"2024-09-12T23:52:12.259245+0000","last_became_active":"2024-09-12T23:47:12.958445+0000","last_became_peered":"2024-09-12T23:47:12.958445+0000","last_unstale":"2024-09-12T23:52:12.259245+0000","last_undegraded":"2024-09-12T23:52:12.259245+0000","last_fullsized":"2024-09-12T23:52:12.259245+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:29:13.022968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":829,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:12.053419+0000","last_change":"2024-09-12T23:47:12.957483+0000","last_active":"2024-09-12T23:52:12.053419+0000","last_peered":"2024-09-12T23:52:12.053419+0000","last_clean":"2024-09-12T23:52:12.053419+0000","last_became_active":"2024-09-12T23:47:12.956651+0000","last_became_peered":"2024-09-12T23:47:12.956651+0000","last_unstale":"2024-09-12T23:52:12.053419+0000","last_undegraded":"2024-09-12T23:52:12.053419+0000","last_fullsized":"2024-09-12T23:52:12.053419+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-13T23:35:34.463602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.3","version":"858'1453","reported_seq":3884,"reported_epoch":862,"state":"active+clean","last_fresh":"2024-09-12T23:52:16.568705+0000","last_change":"2024-09-12T23:52:09.575687+0000","last_active":"2024-09-12T23:52:16.568705+0000","last_peered":"2024-09-12T23:52:16.568705+0000","last_clean":"2024-09-12T23:52:16.568705+0000","last_became_active":"2024-09-12T23:47:12.958432+0000","last_became_peered":"2024-09-12T23:47:12.958432+0000","last_unstale":"2024-09-12T23:52:16.568705+0000","last_undegraded":"2024-09-12T23:52:16.568705+0000","last_fullsized":"2024-09-12T23:52:16.568705+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:38:14.355285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:38:14.355285+0000","last_clean_scrub_stamp":"2024-09-12T23:38:14.355285+0000","objects_scrubbed":0,"log_size":1453,"log_dups_size":0,"ondisk_log_size":1453,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:49:57.205061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001445804,"stat_sum":{"num_bytes":22888448,"num_objects":10,"num_object_clones":6,"num_object_copies":30,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10,"num_whiteouts":2,"num_read":262,"num_read_kb":368055,"num_write":666,"num_write_kb":124160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":29024256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.2","version":"61'2","reported_seq":813,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.906462+0000","last_change":"2024-09-12T23:46:44.531852+0000","last_active":"2024-09-12T23:52:11.906462+0000","last_peered":"2024-09-12T23:52:11.906462+0000","last_clean":"2024-09-12T23:52:11.906462+0000","last_became_active":"2024-09-12T23:46:44.531342+0000","last_became_peered":"2024-09-12T23:46:44.531342+0000","last_unstale":"2024-09-12T23:52:11.906462+0000","last_undegraded":"2024-09-12T23:52:11.906462+0000","last_fullsized":"2024-09-12T23:52:11.906462+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:34:48.454503+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:34:48.454503+0000","last_clean_scrub_stamp":"2024-09-12T23:34:48.454503+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:20:37.908667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":828,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.906750+0000","last_change":"2024-09-12T23:47:43.157848+0000","last_active":"2024-09-12T23:52:11.906750+0000","last_peered":"2024-09-12T23:52:11.906750+0000","last_clean":"2024-09-12T23:52:11.906750+0000","last_became_active":"2024-09-12T23:47:43.156958+0000","last_became_peered":"2024-09-12T23:47:43.156958+0000","last_unstale":"2024-09-12T23:52:11.906750+0000","last_undegraded":"2024-09-12T23:52:11.906750+0000","last_fullsized":"2024-09-12T23:52:11.906750+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:10:47.465422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":838,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:12.259859+0000","last_change":"2024-09-12T23:46:44.501362+0000","last_active":"2024-09-12T23:52:12.259859+0000","last_peered":"2024-09-12T23:52:12.259859+0000","last_clean":"2024-09-12T23:52:12.259859+0000","last_became_active":"2024-09-12T23:46:44.500275+0000","last_became_peered":"2024-09-12T23:46:44.500275+0000","last_unstale":"2024-09-12T23:52:12.259859+0000","last_undegraded":"2024-09-12T23:52:12.259859+0000","last_fullsized":"2024-09-12T23:52:12.259859+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:20:31.204281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.2","version":"0'0","reported_seq":2453,"reported_epoch":862,"state":"active+clean","last_fresh":"2024-09-12T23:52:16.846263+0000","last_change":"2024-09-12T23:52:09.592961+0000","last_active":"2024-09-12T23:52:16.846263+0000","last_peered":"2024-09-12T23:52:16.846263+0000","last_clean":"2024-09-12T23:52:16.846263+0000","last_became_active":"2024-09-12T23:47:12.976432+0000","last_became_peered":"2024-09-12T23:47:12.976432+0000","last_unstale":"2024-09-12T23:52:16.846263+0000","last_undegraded":"2024-09-12T23:52:16.846263+0000","last_fullsized":"2024-09-12T23:52:16.846263+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:38:14.355285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:38:14.355285+0000","last_clean_scrub_stamp":"2024-09-12T23:38:14.355285+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:19:35.639092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082647600000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.3","version":"59'1","reported_seq":840,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.565298+0000","last_change":"2024-09-12T23:47:43.161691+0000","last_active":"2024-09-12T23:52:11.565298+0000","last_peered":"2024-09-12T23:52:11.565298+0000","last_clean":"2024-09-12T23:52:11.565298+0000","last_became_active":"2024-09-12T23:47:43.160222+0000","last_became_peered":"2024-09-12T23:47:43.160222+0000","last_unstale":"2024-09-12T23:52:11.565298+0000","last_undegraded":"2024-09-12T23:52:11.565298+0000","last_fullsized":"2024-09-12T23:52:11.565298+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:34:48.454503+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:34:48.454503+0000","last_clean_scrub_stamp":"2024-09-12T23:34:48.454503+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:27:04.317412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":803,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.570141+0000","last_change":"2024-09-12T23:47:43.180682+0000","last_active":"2024-09-12T23:52:11.570141+0000","last_peered":"2024-09-12T23:52:11.570141+0000","last_clean":"2024-09-12T23:52:11.570141+0000","last_became_active":"2024-09-12T23:47:43.180186+0000","last_became_peered":"2024-09-12T23:47:43.180186+0000","last_unstale":"2024-09-12T23:52:11.570141+0000","last_undegraded":"2024-09-12T23:52:11.570141+0000","last_fullsized":"2024-09-12T23:52:11.570141+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:02:09.607041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":801,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.907116+0000","last_change":"2024-09-12T23:46:44.526933+0000","last_active":"2024-09-12T23:52:11.907116+0000","last_peered":"2024-09-12T23:52:11.907116+0000","last_clean":"2024-09-12T23:52:11.907116+0000","last_became_active":"2024-09-12T23:46:44.526304+0000","last_became_peered":"2024-09-12T23:46:44.526304+0000","last_unstale":"2024-09-12T23:52:11.907116+0000","last_undegraded":"2024-09-12T23:52:11.907116+0000","last_fullsized":"2024-09-12T23:52:11.907116+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:42:47.448099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.1","version":"860'1770","reported_seq":4158,"reported_epoch":862,"state":"active+clean","last_fresh":"2024-09-12T23:52:16.847038+0000","last_change":"2024-09-12T23:52:09.569351+0000","last_active":"2024-09-12T23:52:16.847038+0000","last_peered":"2024-09-12T23:52:16.847038+0000","last_clean":"2024-09-12T23:52:16.847038+0000","last_became_active":"2024-09-12T23:47:43.170480+0000","last_became_peered":"2024-09-12T23:47:43.170480+0000","last_unstale":"2024-09-12T23:52:16.847038+0000","last_undegraded":"2024-09-12T23:52:16.847038+0000","last_fullsized":"2024-09-12T23:52:16.847038+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:38:14.355285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:38:14.355285+0000","last_clean_scrub_stamp":"2024-09-12T23:38:14.355285+0000","objects_scrubbed":0,"log_size":1770,"log_dups_size":0,"ondisk_log_size":1770,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:13:19.922039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092723800000000004,"stat_sum":{"num_bytes":26550272,"num_objects":13,"num_object_clones":9,"num_object_copies":39,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13,"num_whiteouts":1,"num_read":279,"num_read_kb":505100,"num_write":657,"num_write_kb":160768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":62742528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":805,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:12.486876+0000","last_change":"2024-09-12T23:47:43.169214+0000","last_active":"2024-09-12T23:52:12.486876+0000","last_peered":"2024-09-12T23:52:12.486876+0000","last_clean":"2024-09-12T23:52:12.486876+0000","last_became_active":"2024-09-12T23:47:43.167616+0000","last_became_peered":"2024-09-12T23:47:43.167616+0000","last_unstale":"2024-09-12T23:52:12.486876+0000","last_undegraded":"2024-09-12T23:52:12.486876+0000","last_fullsized":"2024-09-12T23:52:12.486876+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:34:48.454503+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:34:48.454503+0000","last_clean_scrub_stamp":"2024-09-12T23:34:48.454503+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:50:06.080094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001798914,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"648'2","reported_seq":806,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.907285+0000","last_change":"2024-09-12T23:47:12.970662+0000","last_active":"2024-09-12T23:52:11.907285+0000","last_peered":"2024-09-12T23:52:11.907285+0000","last_clean":"2024-09-12T23:52:11.907285+0000","last_became_active":"2024-09-12T23:47:12.969780+0000","last_became_peered":"2024-09-12T23:47:12.969780+0000","last_unstale":"2024-09-12T23:52:11.907285+0000","last_undegraded":"2024-09-12T23:52:11.907285+0000","last_fullsized":"2024-09-12T23:52:11.907285+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:23:36.870510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":2,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":804,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:15.545137+0000","last_change":"2024-09-12T23:47:43.167596+0000","last_active":"2024-09-12T23:52:15.545137+0000","last_peered":"2024-09-12T23:52:15.545137+0000","last_clean":"2024-09-12T23:52:15.545137+0000","last_became_active":"2024-09-12T23:47:43.166800+0000","last_became_peered":"2024-09-12T23:47:43.166800+0000","last_unstale":"2024-09-12T23:52:15.545137+0000","last_undegraded":"2024-09-12T23:52:15.545137+0000","last_fullsized":"2024-09-12T23:52:15.545137+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:06:12.222797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.0","version":"858'756","reported_seq":3236,"reported_epoch":862,"state":"active+clean","last_fresh":"2024-09-12T23:52:16.844492+0000","last_change":"2024-09-12T23:52:09.589481+0000","last_active":"2024-09-12T23:52:16.844492+0000","last_peered":"2024-09-12T23:52:16.844492+0000","last_clean":"2024-09-12T23:52:16.844492+0000","last_became_active":"2024-09-12T23:46:16.522384+0000","last_became_peered":"2024-09-12T23:46:16.522384+0000","last_unstale":"2024-09-12T23:52:16.844492+0000","last_undegraded":"2024-09-12T23:52:16.844492+0000","last_fullsized":"2024-09-12T23:52:16.844492+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:38:14.355285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:38:14.355285+0000","last_clean_scrub_stamp":"2024-09-12T23:38:14.355285+0000","objects_scrubbed":0,"log_size":756,"log_dups_size":0,"ondisk_log_size":756,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:19:19.180917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019116999999999999,"stat_sum":{"num_bytes":10412032,"num_objects":3,"num_object_clones":2,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":113,"num_read_kb":80632,"num_write":334,"num_write_kb":65944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":12541952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":812,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:12.052981+0000","last_change":"2024-09-12T23:47:12.974899+0000","last_active":"2024-09-12T23:52:12.052981+0000","last_peered":"2024-09-12T23:52:12.052981+0000","last_clean":"2024-09-12T23:52:12.052981+0000","last_became_active":"2024-09-12T23:47:12.974571+0000","last_became_peered":"2024-09-12T23:47:12.974571+0000","last_unstale":"2024-09-12T23:52:12.052981+0000","last_undegraded":"2024-09-12T23:52:12.052981+0000","last_fullsized":"2024-09-12T23:52:12.052981+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:34:48.454503+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:34:48.454503+0000","last_clean_scrub_stamp":"2024-09-12T23:34:48.454503+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:07:57.715043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"53'2","reported_seq":821,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.560328+0000","last_change":"2024-09-12T23:47:43.153216+0000","last_active":"2024-09-12T23:52:11.560328+0000","last_peered":"2024-09-12T23:52:11.560328+0000","last_clean":"2024-09-12T23:52:11.560328+0000","last_became_active":"2024-09-12T23:47:43.152216+0000","last_became_peered":"2024-09-12T23:47:43.152216+0000","last_unstale":"2024-09-12T23:52:11.560328+0000","last_undegraded":"2024-09-12T23:52:11.560328+0000","last_fullsized":"2024-09-12T23:52:11.560328+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:20:23.278726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":829,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.565697+0000","last_change":"2024-09-12T23:46:16.544156+0000","last_active":"2024-09-12T23:52:11.565697+0000","last_peered":"2024-09-12T23:52:11.565697+0000","last_clean":"2024-09-12T23:52:11.565697+0000","last_became_active":"2024-09-12T23:46:16.543636+0000","last_became_peered":"2024-09-12T23:46:16.543636+0000","last_unstale":"2024-09-12T23:52:11.565697+0000","last_undegraded":"2024-09-12T23:52:11.565697+0000","last_fullsized":"2024-09-12T23:52:11.565697+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:49:15.812892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.f","version":"860'1205","reported_seq":3645,"reported_epoch":862,"state":"active+clean","last_fresh":"2024-09-12T23:52:16.561967+0000","last_change":"2024-09-12T23:52:09.600662+0000","last_active":"2024-09-12T23:52:16.561967+0000","last_peered":"2024-09-12T23:52:16.561967+0000","last_clean":"2024-09-12T23:52:16.561967+0000","last_became_active":"2024-09-12T23:47:12.968032+0000","last_became_peered":"2024-09-12T23:47:12.968032+0000","last_unstale":"2024-09-12T23:52:16.561967+0000","last_undegraded":"2024-09-12T23:52:16.561967+0000","last_fullsized":"2024-09-12T23:52:16.561967+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:38:14.355285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:38:14.355285+0000","last_clean_scrub_stamp":"2024-09-12T23:38:14.355285+0000","objects_scrubbed":0,"log_size":1205,"log_dups_size":0,"ondisk_log_size":1205,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:50:41.689777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.024354997,"stat_sum":{"num_bytes":14573568,"num_objects":8,"num_object_clones":5,"num_object_copies":24,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8,"num_whiteouts":0,"num_read":207,"num_read_kb":228341,"num_write":477,"num_write_kb":109928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":7602176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.8","version":"0'0","reported_seq":798,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.569140+0000","last_change":"2024-09-12T23:47:43.153171+0000","last_active":"2024-09-12T23:52:11.569140+0000","last_peered":"2024-09-12T23:52:11.569140+0000","last_clean":"2024-09-12T23:52:11.569140+0000","last_became_active":"2024-09-12T23:47:43.152469+0000","last_became_peered":"2024-09-12T23:47:43.152469+0000","last_unstale":"2024-09-12T23:52:11.569140+0000","last_undegraded":"2024-09-12T23:52:11.569140+0000","last_fullsized":"2024-09-12T23:52:11.569140+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:29:50.253799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":802,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.906560+0000","last_change":"2024-09-12T23:47:43.158133+0000","last_active":"2024-09-12T23:52:11.906560+0000","last_peered":"2024-09-12T23:52:11.906560+0000","last_clean":"2024-09-12T23:52:11.906560+0000","last_became_active":"2024-09-12T23:47:43.157356+0000","last_became_peered":"2024-09-12T23:47:43.157356+0000","last_unstale":"2024-09-12T23:52:11.906560+0000","last_undegraded":"2024-09-12T23:52:11.906560+0000","last_fullsized":"2024-09-12T23:52:11.906560+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:41:34.867343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.e","version":"860'843","reported_seq":3327,"reported_epoch":862,"state":"active+clean","last_fresh":"2024-09-12T23:52:16.848835+0000","last_change":"2024-09-12T23:52:09.580125+0000","last_active":"2024-09-12T23:52:16.848835+0000","last_peered":"2024-09-12T23:52:16.848835+0000","last_clean":"2024-09-12T23:52:16.848835+0000","last_became_active":"2024-09-12T23:46:44.533261+0000","last_became_peered":"2024-09-12T23:46:44.533261+0000","last_unstale":"2024-09-12T23:52:16.848835+0000","last_undegraded":"2024-09-12T23:52:16.848835+0000","last_fullsized":"2024-09-12T23:52:16.848835+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:38:14.355285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:38:14.355285+0000","last_clean_scrub_stamp":"2024-09-12T23:38:14.355285+0000","objects_scrubbed":0,"log_size":843,"log_dups_size":0,"ondisk_log_size":843,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:30:02.998034+0000","scrub_duration":0,"objects_trimmed":1,"snaptrim_duration":0.0099007790000000002,"stat_sum":{"num_bytes":14917632,"num_objects":8,"num_object_clones":6,"num_object_copies":24,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8,"num_whiteouts":0,"num_read":149,"num_read_kb":166079,"num_write":328,"num_write_kb":75824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":19611648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":801,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.906279+0000","last_change":"2024-09-12T23:47:43.158096+0000","last_active":"2024-09-12T23:52:11.906279+0000","last_peered":"2024-09-12T23:52:11.906279+0000","last_clean":"2024-09-12T23:52:11.906279+0000","last_became_active":"2024-09-12T23:47:43.157249+0000","last_became_peered":"2024-09-12T23:47:43.157249+0000","last_unstale":"2024-09-12T23:52:11.906279+0000","last_undegraded":"2024-09-12T23:52:11.906279+0000","last_fullsized":"2024-09-12T23:52:11.906279+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:27:40.316190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":832,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:15.544925+0000","last_change":"2024-09-12T23:47:43.167281+0000","last_active":"2024-09-12T23:52:15.544925+0000","last_peered":"2024-09-12T23:52:15.544925+0000","last_clean":"2024-09-12T23:52:15.544925+0000","last_became_active":"2024-09-12T23:47:43.166598+0000","last_became_peered":"2024-09-12T23:47:43.166598+0000","last_unstale":"2024-09-12T23:52:15.544925+0000","last_undegraded":"2024-09-12T23:52:15.544925+0000","last_fullsized":"2024-09-12T23:52:15.544925+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:42:25.097408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.d","version":"855'1215","reported_seq":3747,"reported_epoch":862,"state":"active+clean","last_fresh":"2024-09-12T23:52:16.564343+0000","last_change":"2024-09-12T23:52:09.592241+0000","last_active":"2024-09-12T23:52:16.564343+0000","last_peered":"2024-09-12T23:52:16.564343+0000","last_clean":"2024-09-12T23:52:16.564343+0000","last_became_active":"2024-09-12T23:47:43.177671+0000","last_became_peered":"2024-09-12T23:47:43.177671+0000","last_unstale":"2024-09-12T23:52:16.564343+0000","last_undegraded":"2024-09-12T23:52:16.564343+0000","last_fullsized":"2024-09-12T23:52:16.564343+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:38:14.355285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:38:14.355285+0000","last_clean_scrub_stamp":"2024-09-12T23:38:14.355285+0000","objects_scrubbed":0,"log_size":1215,"log_dups_size":0,"ondisk_log_size":1215,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:05:30.144878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.01474265,"stat_sum":{"num_bytes":11550720,"num_objects":5,"num_object_clones":2,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":215,"num_read_kb":289907,"num_write":434,"num_write_kb":114280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":12181504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":836,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.906394+0000","last_change":"2024-09-12T23:45:47.446858+0000","last_active":"2024-09-12T23:52:11.906394+0000","last_peered":"2024-09-12T23:52:11.906394+0000","last_clean":"2024-09-12T23:52:11.906394+0000","last_became_active":"2024-09-12T23:45:47.446295+0000","last_became_peered":"2024-09-12T23:45:47.446295+0000","last_unstale":"2024-09-12T23:52:11.906394+0000","last_undegraded":"2024-09-12T23:52:11.906394+0000","last_fullsized":"2024-09-12T23:52:11.906394+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:29:21.889992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":803,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:12.488062+0000","last_change":"2024-09-12T23:46:16.543155+0000","last_active":"2024-09-12T23:52:12.488062+0000","last_peered":"2024-09-12T23:52:12.488062+0000","last_clean":"2024-09-12T23:52:12.488062+0000","last_became_active":"2024-09-12T23:46:16.542474+0000","last_became_peered":"2024-09-12T23:46:16.542474+0000","last_unstale":"2024-09-12T23:52:12.488062+0000","last_undegraded":"2024-09-12T23:52:12.488062+0000","last_fullsized":"2024-09-12T23:52:12.488062+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:24:58.837016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.c","version":"860'1325","reported_seq":3767,"reported_epoch":862,"state":"active+clean","last_fresh":"2024-09-12T23:52:16.829804+0000","last_change":"2024-09-12T23:52:09.606371+0000","last_active":"2024-09-12T23:52:16.829804+0000","last_peered":"2024-09-12T23:52:16.829804+0000","last_clean":"2024-09-12T23:52:16.829804+0000","last_became_active":"2024-09-12T23:46:16.531443+0000","last_became_peered":"2024-09-12T23:46:16.531443+0000","last_unstale":"2024-09-12T23:52:16.829804+0000","last_undegraded":"2024-09-12T23:52:16.829804+0000","last_fullsized":"2024-09-12T23:52:16.829804+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:38:14.355285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:38:14.355285+0000","last_clean_scrub_stamp":"2024-09-12T23:38:14.355285+0000","objects_scrubbed":0,"log_size":1325,"log_dups_size":0,"ondisk_log_size":1325,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:14:02.900495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.026195366000000001,"stat_sum":{"num_bytes":13443072,"num_objects":10,"num_object_clones":7,"num_object_copies":30,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10,"num_whiteouts":1,"num_read":222,"num_read_kb":243038,"num_write":540,"num_write_kb":92160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":26877952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.b","version":"644'3","reported_seq":812,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.561461+0000","last_change":"2024-09-12T23:46:44.503953+0000","last_active":"2024-09-12T23:52:11.561461+0000","last_peered":"2024-09-12T23:52:11.561461+0000","last_clean":"2024-09-12T23:52:11.561461+0000","last_became_active":"2024-09-12T23:46:44.503255+0000","last_became_peered":"2024-09-12T23:46:44.503255+0000","last_unstale":"2024-09-12T23:52:11.561461+0000","last_undegraded":"2024-09-12T23:52:11.561461+0000","last_fullsized":"2024-09-12T23:52:11.561461+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:29:07.877168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":836,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.559644+0000","last_change":"2024-09-12T23:47:12.981641+0000","last_active":"2024-09-12T23:52:11.559644+0000","last_peered":"2024-09-12T23:52:11.559644+0000","last_clean":"2024-09-12T23:52:11.559644+0000","last_became_active":"2024-09-12T23:47:12.980905+0000","last_became_peered":"2024-09-12T23:47:12.980905+0000","last_unstale":"2024-09-12T23:52:11.559644+0000","last_undegraded":"2024-09-12T23:52:11.559644+0000","last_fullsized":"2024-09-12T23:52:11.559644+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:43:57.816944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":823,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:12.259550+0000","last_change":"2024-09-12T23:47:12.958777+0000","last_active":"2024-09-12T23:52:12.259550+0000","last_peered":"2024-09-12T23:52:12.259550+0000","last_clean":"2024-09-12T23:52:12.259550+0000","last_became_active":"2024-09-12T23:47:12.957952+0000","last_became_peered":"2024-09-12T23:47:12.957952+0000","last_unstale":"2024-09-12T23:52:12.259550+0000","last_undegraded":"2024-09-12T23:52:12.259550+0000","last_fullsized":"2024-09-12T23:52:12.259550+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:32:28.365538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"53'2","reported_seq":834,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.570924+0000","last_change":"2024-09-12T23:47:43.169846+0000","last_active":"2024-09-12T23:52:11.570924+0000","last_peered":"2024-09-12T23:52:11.570924+0000","last_clean":"2024-09-12T23:52:11.570924+0000","last_became_active":"2024-09-12T23:47:43.168639+0000","last_became_peered":"2024-09-12T23:47:43.168639+0000","last_unstale":"2024-09-12T23:52:11.570924+0000","last_undegraded":"2024-09-12T23:52:11.570924+0000","last_fullsized":"2024-09-12T23:52:11.570924+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:52:46.134406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":825,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:12.259958+0000","last_change":"2024-09-12T23:46:44.521848+0000","last_active":"2024-09-12T23:52:12.259958+0000","last_peered":"2024-09-12T23:52:12.259958+0000","last_clean":"2024-09-12T23:52:12.259958+0000","last_became_active":"2024-09-12T23:46:44.521555+0000","last_became_peered":"2024-09-12T23:46:44.521555+0000","last_unstale":"2024-09-12T23:52:12.259958+0000","last_undegraded":"2024-09-12T23:52:12.259958+0000","last_fullsized":"2024-09-12T23:52:12.259958+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:17:48.930522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1d","version":"53'2","reported_seq":493,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:12.053741+0000","last_change":"2024-09-12T23:47:12.959644+0000","last_active":"2024-09-12T23:52:12.053741+0000","last_peered":"2024-09-12T23:52:12.053741+0000","last_clean":"2024-09-12T23:52:12.053741+0000","last_became_active":"2024-09-12T23:47:12.958617+0000","last_became_peered":"2024-09-12T23:47:12.958617+0000","last_unstale":"2024-09-12T23:52:12.053741+0000","last_undegraded":"2024-09-12T23:52:12.053741+0000","last_fullsized":"2024-09-12T23:52:12.053741+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:00:39.181604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":829,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.566393+0000","last_change":"2024-09-12T23:46:44.492952+0000","last_active":"2024-09-12T23:52:11.566393+0000","last_peered":"2024-09-12T23:52:11.566393+0000","last_clean":"2024-09-12T23:52:11.566393+0000","last_became_active":"2024-09-12T23:46:44.492142+0000","last_became_peered":"2024-09-12T23:46:44.492142+0000","last_unstale":"2024-09-12T23:52:11.566393+0000","last_undegraded":"2024-09-12T23:52:11.566393+0000","last_fullsized":"2024-09-12T23:52:11.566393+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:17:14.021811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1e","version":"644'2","reported_seq":807,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:12.488562+0000","last_change":"2024-09-12T23:46:44.505945+0000","last_active":"2024-09-12T23:52:12.488562+0000","last_peered":"2024-09-12T23:52:12.488562+0000","last_clean":"2024-09-12T23:52:12.488562+0000","last_became_active":"2024-09-12T23:46:44.504837+0000","last_became_peered":"2024-09-12T23:46:44.504837+0000","last_unstale":"2024-09-12T23:52:12.488562+0000","last_undegraded":"2024-09-12T23:52:12.488562+0000","last_fullsized":"2024-09-12T23:52:12.488562+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:54:42.107149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":820,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:12.488215+0000","last_change":"2024-09-12T23:47:12.976383+0000","last_active":"2024-09-12T23:52:12.488215+0000","last_peered":"2024-09-12T23:52:12.488215+0000","last_clean":"2024-09-12T23:52:12.488215+0000","last_became_active":"2024-09-12T23:47:12.975482+0000","last_became_peered":"2024-09-12T23:47:12.975482+0000","last_unstale":"2024-09-12T23:52:12.488215+0000","last_undegraded":"2024-09-12T23:52:12.488215+0000","last_fullsized":"2024-09-12T23:52:12.488215+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:28.834450+0000","last_clean_scrub_stamp":"2024-09-12T23:33:28.834450+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:04:15.840172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1f","version":"644'5","reported_seq":835,"reported_epoch":861,"state":"active+clean","last_fresh":"2024-09-12T23:52:11.566839+0000","last_change":"2024-09-12T23:46:16.541785+0000","last_active":"2024-09-12T23:52:11.566839+0000","last_peered":"2024-09-12T23:52:11.566839+0000","last_clean":"2024-09-12T23:52:11.566839+0000","last_became_active":"2024-09-12T23:46:16.540407+0000","last_became_peered":"2024-09-12T23:46:16.540407+0000","last_unstale":"2024-09-12T23:52:11.566839+0000","last_undegraded":"2024-09-12T23:52:11.566839+0000","last_fullsized":"2024-09-12T23:52:11.566839+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-12T23:33:27.830370+0000","last_clean_scrub_stamp":"2024-09-12T23:33:27.830370+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:02:09.739544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":342589440,"num_objects":130,"num_object_clones":82,"num_object_copies":390,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":130,"num_whiteouts":4,"num_read":3692,"num_read_kb":5746836,"num_write":8299,"num_write_kb":1831176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":175,"num_bytes_recovered":598081536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":497369088,"data_stored":713613312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":20858,"ondisk_log_size":20858,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":192,"acting":192,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":6608,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":79,"num_read_kb":86,"num_write":68,"num_write_kb":35,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":19824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31723,"internal_metadata":0},"log_size":68,"ondisk_log_size":68,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":984,"num_read_kb":1829,"num_write":301,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":176,"ondisk_log_size":176,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":605,"seq":2598455214138,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":232916,"kb_used_data":38024,"kb_used_omap":34,"kb_used_meta":194845,"kb_avail":93512236,"statfs":{"total":95995035648,"available":95756529664,"internally_reserved":0,"allocated":38936576,"data_stored":52002342,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35655,"internal_metadata":199521465},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 12 23:51:56 2024","interfaces":[{"interface":"back","average":{"1min":0.89700000000000002,"5min":0.90700000000000003,"15min":0.91100000000000003},"min":{"1min":0.58399999999999996,"5min":0.44,"15min":0.44},"max":{"1min":1.3999999999999999,"5min":1.575,"15min":1.575},"last":0.75600000000000001},{"interface":"front","average":{"1min":1.0609999999999999,"5min":0.97699999999999998,"15min":0.95299999999999996},"min":{"1min":0.73099999999999998,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":1.4590000000000001,"5min":5.8460000000000001,"15min":5.8460000000000001},"last":1.0860000000000001}]},{"osd":1,"last update":"Thu Sep 12 23:51:56 2024","interfaces":[{"interface":"back","average":{"1min":0.88800000000000001,"5min":0.84599999999999997,"15min":0.81899999999999995},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.091,"5min":1.6519999999999999,"15min":1.6519999999999999},"last":0.80300000000000005},{"interface":"front","average":{"1min":1.1080000000000001,"5min":1.179,"15min":1.347},"min":{"1min":0.78000000000000003,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":2.0070000000000001,"5min":9.3559999999999999,"15min":9.3559999999999999},"last":1.1739999999999999}]},{"osd":2,"last update":"Thu Sep 12 23:51:56 2024","interfaces":[{"interface":"back","average":{"1min":1.1539999999999999,"5min":1.0469999999999999,"15min":1.03},"min":{"1min":0.88200000000000001,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":1.3899999999999999,"5min":1.7849999999999999,"15min":1.7849999999999999},"last":1.244},{"interface":"front","average":{"1min":0.997,"5min":1.018,"15min":1.0409999999999999},"min":{"1min":0.53400000000000003,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":1.2549999999999999,"5min":1.716,"15min":1.716},"last":1.0429999999999999}]},{"osd":3,"last update":"Thu Sep 12 23:51:56 2024","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":0.96699999999999997,"15min":0.96499999999999997},"min":{"1min":0.70599999999999996,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":1.302,"5min":1.478,"15min":1.478},"last":0.92200000000000004},{"interface":"front","average":{"1min":1.024,"5min":1.0069999999999999,"15min":1.034},"min":{"1min":0.60499999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":1.3420000000000001,"5min":1.722,"15min":1.722},"last":1.125}]},{"osd":4,"last update":"Thu Sep 12 23:51:56 2024","interfaces":[{"interface":"back","average":{"1min":0.86099999999999999,"5min":0.91300000000000003,"15min":0.95399999999999996},"min":{"1min":0.52600000000000002,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.208,"5min":1.4790000000000001,"15min":1.4790000000000001},"last":0.88600000000000001},{"interface":"front","average":{"1min":0.83899999999999997,"5min":0.90700000000000003,"15min":0.97199999999999998},"min":{"1min":0.48799999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.2010000000000001,"5min":1.5720000000000001,"15min":1.5720000000000001},"last":0.71499999999999997}]},{"osd":5,"last update":"Thu Sep 12 23:51:56 2024","interfaces":[{"interface":"back","average":{"1min":0.93899999999999995,"5min":1.024,"15min":1.0660000000000001},"min":{"1min":0.70199999999999996,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":1.345,"5min":1.5349999999999999,"15min":1.5349999999999999},"last":0.95899999999999996},{"interface":"front","average":{"1min":0.97199999999999998,"5min":1.0149999999999999,"15min":1.0569999999999999},"min":{"1min":0.76600000000000001,"5min":0.437,"15min":0.437},"max":{"1min":1.4239999999999999,"5min":1.6379999999999999,"15min":1.6379999999999999},"last":0.85399999999999998}]},{"osd":6,"last update":"Thu Sep 12 23:51:56 2024","interfaces":[{"interface":"back","average":{"1min":1.0389999999999999,"5min":1.0569999999999999,"15min":1.1299999999999999},"min":{"1min":0.65700000000000003,"5min":0.376,"15min":0.376},"max":{"1min":1.6180000000000001,"5min":1.6739999999999999,"15min":1.6739999999999999},"last":1.3169999999999999},{"interface":"front","average":{"1min":1.155,"5min":1.141,"15min":1.171},"min":{"1min":0.88800000000000001,"5min":0.437,"15min":0.437},"max":{"1min":1.5509999999999999,"5min":1.6579999999999999,"15min":1.6579999999999999},"last":1.0149999999999999}]}]},{"osd":6,"up_from":575,"seq":2469606195264,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":127804,"kb_used_data":31280,"kb_used_omap":25,"kb_used_meta":96486,"kb_avail":93617348,"statfs":{"total":95995035648,"available":95864164352,"internally_reserved":0,"allocated":32030720,"data_stored":37334431,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26613,"internal_metadata":98801675},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 12 23:52:21 2024","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":0.93300000000000005,"15min":1.0049999999999999},"min":{"1min":0.64600000000000002,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":1.681,"5min":1.7190000000000001,"15min":1.7190000000000001},"last":1.0680000000000001},{"interface":"front","average":{"1min":0.95799999999999996,"5min":0.96499999999999997,"15min":0.92100000000000004},"min":{"1min":0.625,"5min":0.57499999999999996,"15min":0.57499999999999996},"max":{"1min":1.357,"5min":2.23,"15min":2.23},"last":0.64900000000000002}]},{"osd":1,"last update":"Thu Sep 12 23:52:21 2024","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":0.95199999999999996,"15min":1.008},"min":{"1min":0.68300000000000005,"5min":0.443,"15min":0.443},"max":{"1min":1.351,"5min":1.6899999999999999,"15min":1.6899999999999999},"last":1.048},{"interface":"front","average":{"1min":0.90400000000000003,"5min":0.91800000000000004,"15min":0.90500000000000003},"min":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":1.1990000000000001,"5min":2.2919999999999998,"15min":2.2919999999999998},"last":0.68400000000000005}]},{"osd":2,"last update":"Thu Sep 12 23:52:21 2024","interfaces":[{"interface":"back","average":{"1min":1.0640000000000001,"5min":0.98199999999999998,"15min":0.98099999999999998},"min":{"1min":0.72399999999999998,"5min":0.54100000000000004,"15min":0.54100000000000004},"max":{"1min":1.7849999999999999,"5min":1.7849999999999999,"15min":1.7849999999999999},"last":1.121},{"interface":"front","average":{"1min":0.98399999999999999,"5min":0.98299999999999998,"15min":0.96099999999999997},"min":{"1min":0.54800000000000004,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":1.29,"5min":3.794,"15min":3.794},"last":0.83499999999999996}]},{"osd":3,"last update":"Thu Sep 12 23:52:21 2024","interfaces":[{"interface":"back","average":{"1min":0.94199999999999995,"5min":0.92500000000000004,"15min":0.88600000000000001},"min":{"1min":0.64300000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"max":{"1min":1.159,"5min":1.3340000000000001,"15min":1.3340000000000001},"last":0.97599999999999998},{"interface":"front","average":{"1min":1.034,"5min":1.0820000000000001,"15min":1.0549999999999999},"min":{"1min":0.79600000000000004,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":1.5649999999999999,"5min":5.6219999999999999,"15min":5.6219999999999999},"last":0.999}]},{"osd":4,"last update":"Thu Sep 12 23:52:21 2024","interfaces":[{"interface":"back","average":{"1min":0.94399999999999995,"5min":1.395,"15min":2.6440000000000001},"min":{"1min":0.64500000000000002,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":1.2649999999999999,"5min":45.195,"15min":45.195},"last":0.92000000000000004},{"interface":"front","average":{"1min":1.004,"5min":1.4450000000000001,"15min":2.6469999999999998},"min":{"1min":0.59699999999999998,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":1.284,"5min":43.968000000000004,"15min":43.968000000000004},"last":0.94599999999999995}]},{"osd":5,"last update":"Thu Sep 12 23:52:21 2024","interfaces":[{"interface":"back","average":{"1min":0.92500000000000004,"5min":0.94999999999999996,"15min":0.94299999999999995},"min":{"1min":0.70699999999999996,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":1.161,"5min":1.339,"15min":1.339},"last":1.0229999999999999},{"interface":"front","average":{"1min":1.05,"5min":1.089,"15min":1.115},"min":{"1min":0.70599999999999996,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":1.4390000000000001,"5min":1.5640000000000001,"15min":1.5640000000000001},"last":0.78400000000000003}]},{"osd":7,"last update":"Thu Sep 12 23:51:52 2024","interfaces":[{"interface":"back","average":{"1min":1.075,"5min":1.05,"15min":1.0489999999999999},"min":{"1min":0.78800000000000003,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":1.4039999999999999,"5min":1.4379999999999999,"15min":1.4379999999999999},"last":0.89400000000000002},{"interface":"front","average":{"1min":1.25,"5min":1.163,"15min":1.1599999999999999},"min":{"1min":0.996,"5min":0.54500000000000004,"15min":0.54500000000000004},"max":{"1min":1.8919999999999999,"5min":1.8919999999999999,"15min":1.8919999999999999},"last":1.097}]}]},{"osd":5,"up_from":548,"seq":2353642078278,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":235992,"kb_used_data":76172,"kb_used_omap":40,"kb_used_meta":159767,"kb_avail":93509160,"statfs":{"total":95995035648,"available":95753379840,"internally_reserved":0,"allocated":78000128,"data_stored":106165318,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":41796,"internal_metadata":163601596},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 12 23:52:01 2024","interfaces":[{"interface":"back","average":{"1min":1.411,"5min":0.98899999999999999,"15min":0.91300000000000003},"min":{"1min":0.61699999999999999,"5min":0.56100000000000005,"15min":0.56100000000000005},"max":{"1min":12.398999999999999,"5min":12.398999999999999,"15min":12.398999999999999},"last":0.99399999999999999},{"interface":"front","average":{"1min":0.92300000000000004,"5min":0.88800000000000001,"15min":0.85599999999999998},"min":{"1min":0.58899999999999997,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":1.1719999999999999,"5min":1.2490000000000001,"15min":1.2490000000000001},"last":0.91500000000000004}]},{"osd":1,"last update":"Thu Sep 12 23:52:01 2024","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":1.02,"15min":0.96699999999999997},"min":{"1min":0.73199999999999998,"5min":0.57899999999999996,"15min":0.57899999999999996},"max":{"1min":1.6439999999999999,"5min":4.5640000000000001,"15min":4.5640000000000001},"last":1.1970000000000001},{"interface":"front","average":{"1min":0.996,"5min":0.98099999999999998,"15min":0.93000000000000005},"min":{"1min":0.67800000000000005,"5min":0.54100000000000004,"15min":0.54100000000000004},"max":{"1min":1.379,"5min":4.3929999999999998,"15min":4.3929999999999998},"last":0.96099999999999997}]},{"osd":2,"last update":"Thu Sep 12 23:52:01 2024","interfaces":[{"interface":"back","average":{"1min":1.5229999999999999,"5min":1.0609999999999999,"15min":0.97799999999999998},"min":{"1min":0.629,"5min":0.60399999999999998,"15min":0.60399999999999998},"max":{"1min":12.977,"5min":12.977,"15min":12.977},"last":1.0640000000000001},{"interface":"front","average":{"1min":1.091,"5min":1.1319999999999999,"15min":1.0449999999999999},"min":{"1min":0.83199999999999996,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":1.5089999999999999,"5min":11.619,"15min":11.619},"last":0.85799999999999998}]},{"osd":3,"last update":"Thu Sep 12 23:52:01 2024","interfaces":[{"interface":"back","average":{"1min":1.0509999999999999,"5min":1.103,"15min":0.98699999999999999},"min":{"1min":0.84299999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"max":{"1min":1.3879999999999999,"5min":10.881,"15min":10.881},"last":1.121},{"interface":"front","average":{"1min":0.97999999999999998,"5min":0.94699999999999995,"15min":0.91400000000000003},"min":{"1min":0.70299999999999996,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":1.4039999999999999,"5min":5.1870000000000003,"15min":5.1870000000000003},"last":0.81499999999999995}]},{"osd":4,"last update":"Thu Sep 12 23:52:01 2024","interfaces":[{"interface":"back","average":{"1min":0.93899999999999995,"5min":0.96399999999999997,"15min":0.89600000000000002},"min":{"1min":0.64900000000000002,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":1.347,"5min":2.6789999999999998,"15min":2.6789999999999998},"last":0.73699999999999999},{"interface":"front","average":{"1min":1.044,"5min":1.1299999999999999,"15min":0.98399999999999999},"min":{"1min":0.63400000000000001,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":1.6850000000000001,"5min":10.993,"15min":10.993},"last":1.3009999999999999}]},{"osd":6,"last update":"Thu Sep 12 23:52:19 2024","interfaces":[{"interface":"back","average":{"1min":0.876,"5min":0.91300000000000003,"15min":0.91300000000000003},"min":{"1min":0.623,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":1.2090000000000001,"5min":1.4510000000000001,"15min":1.4510000000000001},"last":1.1639999999999999},{"interface":"front","average":{"1min":1.204,"5min":1.319,"15min":1.5049999999999999},"min":{"1min":1.0069999999999999,"5min":0.54100000000000004,"15min":0.54100000000000004},"max":{"1min":1.417,"5min":11.07,"15min":11.07},"last":1.389}]},{"osd":7,"last update":"Thu Sep 12 23:51:49 2024","interfaces":[{"interface":"back","average":{"1min":1.157,"5min":1.1060000000000001,"15min":1.095},"min":{"1min":0.76900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":1.6499999999999999,"5min":2.3580000000000001,"15min":2.3580000000000001},"last":1.2729999999999999},{"interface":"front","average":{"1min":1.2130000000000001,"5min":1.4370000000000001,"15min":1.6870000000000001},"min":{"1min":0.85299999999999998,"5min":0.443,"15min":0.443},"max":{"1min":1.593,"5min":11.241,"15min":11.241},"last":1.2310000000000001}]}]},{"osd":4,"up_from":520,"seq":2233382993995,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":591824,"kb_used_data":85572,"kb_used_omap":65,"kb_used_meta":506174,"kb_avail":93153328,"statfs":{"total":95995035648,"available":95389007872,"internally_reserved":0,"allocated":87625728,"data_stored":117849787,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":66914,"internal_metadata":518322846},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 12 23:51:28 2024","interfaces":[{"interface":"back","average":{"1min":1.0049999999999999,"5min":0.93200000000000005,"15min":0.88600000000000001},"min":{"1min":0.59999999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":1.4590000000000001,"5min":1.679,"15min":1.679},"last":0.80300000000000005},{"interface":"front","average":{"1min":1.0389999999999999,"5min":0.95699999999999996,"15min":0.91700000000000004},"min":{"1min":0.68500000000000005,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.4359999999999999,"5min":2.3439999999999999,"15min":2.3439999999999999},"last":0.84499999999999997}]},{"osd":1,"last update":"Thu Sep 12 23:51:28 2024","interfaces":[{"interface":"back","average":{"1min":1.1279999999999999,"5min":0.875,"15min":0.81399999999999995},"min":{"1min":0.65600000000000003,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":4.056,"5min":4.056,"15min":4.056},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.91400000000000003,"5min":0.86099999999999999,"15min":0.79500000000000004},"min":{"1min":0.54800000000000004,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":1.3600000000000001,"5min":1.4099999999999999,"15min":1.4099999999999999},"last":0.90400000000000003}]},{"osd":2,"last update":"Thu Sep 12 23:51:28 2024","interfaces":[{"interface":"back","average":{"1min":1.105,"5min":0.91800000000000004,"15min":0.82599999999999996},"min":{"1min":0.622,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":4.2709999999999999,"5min":4.2709999999999999,"15min":4.2709999999999999},"last":0.64100000000000001},{"interface":"front","average":{"1min":1.1100000000000001,"5min":0.876,"15min":0.82699999999999996},"min":{"1min":0.75800000000000001,"5min":0.433,"15min":0.433},"max":{"1min":3.7440000000000002,"5min":3.7440000000000002,"15min":3.7440000000000002},"last":0.70099999999999996}]},{"osd":3,"last update":"Thu Sep 12 23:51:28 2024","interfaces":[{"interface":"back","average":{"1min":1.0580000000000001,"5min":0.92700000000000005,"15min":0.873},"min":{"1min":0.75700000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.425,"5min":1.6120000000000001,"15min":1.6120000000000001},"last":1.056},{"interface":"front","average":{"1min":0.97099999999999997,"5min":0.88600000000000001,"15min":0.82199999999999995},"min":{"1min":0.46700000000000003,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":1.361,"5min":2.419,"15min":2.419},"last":0.76400000000000001}]},{"osd":5,"last update":"Thu Sep 12 23:51:57 2024","interfaces":[{"interface":"back","average":{"1min":1.0409999999999999,"5min":0.86599999999999999,"15min":0.75800000000000001},"min":{"1min":0.72399999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":1.923,"5min":1.923,"15min":1.923},"last":0.98299999999999998},{"interface":"front","average":{"1min":1.0609999999999999,"5min":0.91900000000000004,"15min":0.749},"min":{"1min":0.66200000000000003,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":1.579,"5min":2.806,"15min":2.806},"last":0.93500000000000005}]},{"osd":6,"last update":"Thu Sep 12 23:51:23 2024","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":0.98699999999999999,"15min":0.91900000000000004},"min":{"1min":0.75,"5min":0.313,"15min":0.313},"max":{"1min":1.353,"5min":1.4279999999999999,"15min":1.4279999999999999},"last":1.1519999999999999},{"interface":"front","average":{"1min":1.123,"5min":1.0609999999999999,"15min":1.0029999999999999},"min":{"1min":0.73999999999999999,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":1.571,"5min":3.6520000000000001,"15min":3.6520000000000001},"last":1.0189999999999999}]},{"osd":7,"last update":"Thu Sep 12 23:51:51 2024","interfaces":[{"interface":"back","average":{"1min":1.091,"5min":0.98899999999999999,"15min":0.98599999999999999},"min":{"1min":0.68799999999999994,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":2.1160000000000001,"5min":2.1160000000000001,"15min":2.1160000000000001},"last":0.88400000000000001},{"interface":"front","average":{"1min":1.252,"5min":1.133,"15min":1.149},"min":{"1min":0.86899999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":2.2080000000000002,"5min":2.2080000000000002,"15min":2.2080000000000002},"last":1.1080000000000001}]}]},{"osd":2,"up_from":463,"seq":1988569858135,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":462644,"kb_used_data":72488,"kb_used_omap":40,"kb_used_meta":390103,"kb_avail":93282508,"statfs":{"total":95995035648,"available":95521288192,"internally_reserved":0,"allocated":74227712,"data_stored":101121578,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":41111,"internal_metadata":399466345},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 12 23:51:30 2024","interfaces":[{"interface":"back","average":{"1min":0.66100000000000003,"5min":1.1439999999999999,"15min":0.80900000000000005},"min":{"1min":0.48299999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":1.0589999999999999,"5min":36.863,"15min":36.863},"last":0.76400000000000001},{"interface":"front","average":{"1min":0.75900000000000001,"5min":0.91200000000000003,"15min":0.76500000000000001},"min":{"1min":0.52600000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.94899999999999995,"5min":5.0209999999999999,"15min":5.0209999999999999},"last":0.82299999999999995}]},{"osd":1,"last update":"Thu Sep 12 23:51:30 2024","interfaces":[{"interface":"back","average":{"1min":0.73799999999999999,"5min":0.81299999999999994,"15min":0.745},"min":{"1min":0.40799999999999997,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":1.1259999999999999,"5min":5.069,"15min":5.069},"last":1.268},{"interface":"front","average":{"1min":0.79400000000000004,"5min":0.94499999999999995,"15min":0.77000000000000002},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":1.262,"5min":7.9279999999999999,"15min":7.9279999999999999},"last":0.89700000000000002}]},{"osd":3,"last update":"Thu Sep 12 23:52:01 2024","interfaces":[{"interface":"back","average":{"1min":0.92000000000000004,"5min":0.98199999999999998,"15min":0.91800000000000004},"min":{"1min":0.65600000000000003,"5min":0.379,"15min":0.379},"max":{"1min":1.141,"5min":5.1029999999999998,"15min":5.1029999999999998},"last":1.4770000000000001},{"interface":"front","average":{"1min":0.85699999999999998,"5min":1.014,"15min":0.90800000000000003},"min":{"1min":0.59499999999999997,"5min":0.378,"15min":0.378},"max":{"1min":1.2410000000000001,"5min":8.0169999999999995,"15min":8.0169999999999995},"last":1.081}]},{"osd":4,"last update":"Thu Sep 12 23:51:30 2024","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":1.111,"15min":1.0009999999999999},"min":{"1min":0.61199999999999999,"5min":0.56899999999999995,"15min":0.56899999999999995},"max":{"1min":1.355,"5min":8.3260000000000005,"15min":8.3260000000000005},"last":1.1539999999999999},{"interface":"front","average":{"1min":0.97499999999999998,"5min":1.48,"15min":1.1279999999999999},"min":{"1min":0.73299999999999998,"5min":0.55500000000000005,"15min":0.55500000000000005},"max":{"1min":1.266,"5min":36.911999999999999,"15min":36.911999999999999},"last":1.218}]},{"osd":5,"last update":"Thu Sep 12 23:52:01 2024","interfaces":[{"interface":"back","average":{"1min":1.014,"5min":1.4059999999999999,"15min":1.143},"min":{"1min":0.73699999999999999,"5min":0.56299999999999994,"15min":0.56299999999999994},"max":{"1min":1.4430000000000001,"5min":36.935000000000002,"15min":36.935000000000002},"last":1.399},{"interface":"front","average":{"1min":1.03,"5min":1.0920000000000001,"15min":1.048},"min":{"1min":0.79100000000000004,"5min":0.69099999999999995,"15min":0.69099999999999995},"max":{"1min":1.399,"5min":5.7859999999999996,"15min":5.7859999999999996},"last":1.744}]},{"osd":6,"last update":"Thu Sep 12 23:52:14 2024","interfaces":[{"interface":"back","average":{"1min":1.0600000000000001,"5min":1.214,"15min":1.1020000000000001},"min":{"1min":0.85299999999999998,"5min":0.63800000000000001,"15min":0.63800000000000001},"max":{"1min":1.3340000000000001,"5min":9.0120000000000005,"15min":9.0120000000000005},"last":1.544},{"interface":"front","average":{"1min":1.109,"5min":1.5349999999999999,"15min":1.2470000000000001},"min":{"1min":0.876,"5min":0.66800000000000004,"15min":0.66800000000000004},"max":{"1min":1.3720000000000001,"5min":36.975000000000001,"15min":36.975000000000001},"last":1.3340000000000001}]},{"osd":7,"last update":"Thu Sep 12 23:51:55 2024","interfaces":[{"interface":"back","average":{"1min":1.109,"5min":1.1879999999999999,"15min":1.1539999999999999},"min":{"1min":0.76100000000000001,"5min":0.56599999999999995,"15min":0.56599999999999995},"max":{"1min":1.4750000000000001,"5min":5.7510000000000003,"15min":5.7510000000000003},"last":1.619},{"interface":"front","average":{"1min":1.0940000000000001,"5min":1.218,"15min":1.1279999999999999},"min":{"1min":0.873,"5min":0.57999999999999996,"15min":0.57999999999999996},"max":{"1min":1.51,"5min":8.0899999999999999,"15min":8.0899999999999999},"last":1.6870000000000001}]}]},{"osd":1,"up_from":433,"seq":1859720839261,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":554832,"kb_used_data":107844,"kb_used_omap":45,"kb_used_meta":446930,"kb_avail":93190320,"statfs":{"total":95995035648,"available":95426887680,"internally_reserved":0,"allocated":110432256,"data_stored":142402827,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":46226,"internal_metadata":457657198},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 12 23:52:06 2024","interfaces":[{"interface":"back","average":{"1min":1.1020000000000001,"5min":0.81699999999999995,"15min":0.67000000000000004},"min":{"1min":0.45600000000000002,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":9.5649999999999995,"5min":9.5649999999999995,"15min":9.5649999999999995},"last":0.97299999999999998},{"interface":"front","average":{"1min":1.028,"5min":0.82599999999999996,"15min":0.75900000000000001},"min":{"1min":0.52400000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":6.9870000000000001,"5min":6.9870000000000001,"15min":18.276},"last":0.83299999999999996}]},{"osd":2,"last update":"Thu Sep 12 23:51:29 2024","interfaces":[{"interface":"back","average":{"1min":1.167,"5min":0.86699999999999999,"15min":0.86499999999999999},"min":{"1min":0.32600000000000001,"5min":0.246,"15min":0.246},"max":{"1min":9.2859999999999996,"5min":9.2859999999999996,"15min":9.2859999999999996},"last":0.88600000000000001},{"interface":"front","average":{"1min":0.79200000000000004,"5min":0.83599999999999997,"15min":0.85699999999999998},"min":{"1min":0.45600000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":1.3380000000000001,"5min":4.8209999999999997,"15min":4.8209999999999997},"last":1.01}]},{"osd":3,"last update":"Thu Sep 12 23:52:06 2024","interfaces":[{"interface":"back","average":{"1min":1.25,"5min":0.89000000000000001,"15min":1.03},"min":{"1min":0.56799999999999995,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":10.586,"5min":10.586,"15min":10.586},"last":0.78900000000000003},{"interface":"front","average":{"1min":1.276,"5min":0.96699999999999997,"15min":0.92700000000000005},"min":{"1min":0.57999999999999996,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":9.4860000000000007,"5min":9.4860000000000007,"15min":9.4860000000000007},"last":1.1559999999999999}]},{"osd":4,"last update":"Thu Sep 12 23:52:25 2024","interfaces":[{"interface":"back","average":{"1min":1.04,"5min":1.123,"15min":1.069},"min":{"1min":0.57699999999999996,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":1.992,"5min":9.7089999999999996,"15min":9.7089999999999996},"last":1.2769999999999999},{"interface":"front","average":{"1min":1.01,"5min":1.0649999999999999,"15min":0.98199999999999998},"min":{"1min":0.71299999999999997,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":1.762,"5min":6.7930000000000001,"15min":6.7930000000000001},"last":0.93999999999999995}]},{"osd":5,"last update":"Thu Sep 12 23:51:53 2024","interfaces":[{"interface":"back","average":{"1min":1.3029999999999999,"5min":1.0640000000000001,"15min":1.0109999999999999},"min":{"1min":0.80000000000000004,"5min":0.55800000000000005,"15min":0.55800000000000005},"max":{"1min":6.9139999999999997,"5min":6.9139999999999997,"15min":6.9139999999999997},"last":1.1020000000000001},{"interface":"front","average":{"1min":1.53,"5min":1.1399999999999999,"15min":1.0980000000000001},"min":{"1min":0.82499999999999996,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":10.340999999999999,"5min":10.340999999999999,"15min":10.340999999999999},"last":1.0549999999999999}]},{"osd":6,"last update":"Thu Sep 12 23:52:25 2024","interfaces":[{"interface":"back","average":{"1min":1.0660000000000001,"5min":1.1639999999999999,"15min":1.081},"min":{"1min":0.80500000000000005,"5min":0.54100000000000004,"15min":0.54100000000000004},"max":{"1min":1.5820000000000001,"5min":9.6400000000000006,"15min":9.6400000000000006},"last":1.2330000000000001},{"interface":"front","average":{"1min":1.129,"5min":1.127,"15min":1.1299999999999999},"min":{"1min":0.67900000000000005,"5min":0.63900000000000001,"15min":0.63900000000000001},"max":{"1min":1.502,"5min":6.359,"15min":6.359},"last":1.21}]},{"osd":7,"last update":"Thu Sep 12 23:51:50 2024","interfaces":[{"interface":"back","average":{"1min":1.629,"5min":1.242,"15min":1.2490000000000001},"min":{"1min":0.72099999999999997,"5min":0.66900000000000004,"15min":0.66900000000000004},"max":{"1min":10.428000000000001,"5min":10.428000000000001,"15min":10.428000000000001},"last":1.1259999999999999},{"interface":"front","average":{"1min":1.653,"5min":1.2589999999999999,"15min":1.258},"min":{"1min":0.78000000000000003,"5min":0.55200000000000005,"15min":0.55200000000000005},"max":{"1min":10.4,"5min":10.4,"15min":10.4},"last":1.1839999999999999}]}]},{"osd":0,"up_from":405,"seq":1739461754979,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":542108,"kb_used_data":93008,"kb_used_omap":38,"kb_used_meta":449049,"kb_avail":93203044,"statfs":{"total":95995035648,"available":95439917056,"internally_reserved":0,"allocated":95240192,"data_stored":131686620,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":39803,"internal_metadata":459826309},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Sep 12 23:51:56 2024","interfaces":[{"interface":"back","average":{"1min":0.79700000000000004,"5min":0.70499999999999996,"15min":0.65100000000000002},"min":{"1min":0.44500000000000001,"5min":0.249,"15min":0.249},"max":{"1min":1.962,"5min":1.962,"15min":1.962},"last":0.70399999999999996},{"interface":"front","average":{"1min":0.94199999999999995,"5min":0.76200000000000001,"15min":0.68700000000000006},"min":{"1min":0.47799999999999998,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":4.758,"5min":4.758,"15min":4.758},"last":0.65900000000000003}]},{"osd":2,"last update":"Thu Sep 12 23:51:29 2024","interfaces":[{"interface":"back","average":{"1min":0.81799999999999995,"5min":0.79200000000000004,"15min":0.73299999999999998},"min":{"1min":0.55300000000000005,"5min":0.308,"15min":0.308},"max":{"1min":1.909,"5min":1.909,"15min":1.909},"last":0.80200000000000005},{"interface":"front","average":{"1min":0.75,"5min":0.81799999999999995,"15min":0.69399999999999995},"min":{"1min":0.39100000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.042,"5min":5.4089999999999998,"15min":5.4089999999999998},"last":0.754}]},{"osd":3,"last update":"Thu Sep 12 23:51:56 2024","interfaces":[{"interface":"back","average":{"1min":1.0549999999999999,"5min":1.0269999999999999,"15min":0.82099999999999995},"min":{"1min":0.51200000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.0750000000000002,"5min":23.811,"15min":23.811},"last":0.78600000000000003},{"interface":"front","average":{"1min":1.234,"5min":1.167,"15min":0.91300000000000003},"min":{"1min":0.42099999999999999,"5min":0.35599999999999998,"15min":0.34499999999999997},"max":{"1min":9.1400000000000006,"5min":22.481999999999999,"15min":22.481999999999999},"last":0.85099999999999998}]},{"osd":4,"last update":"Thu Sep 12 23:51:29 2024","interfaces":[{"interface":"back","average":{"1min":1.0700000000000001,"5min":1.042,"15min":1.05},"min":{"1min":0.64700000000000002,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":2.0249999999999999,"5min":3.597,"15min":3.597},"last":1.0660000000000001},{"interface":"front","average":{"1min":1.0549999999999999,"5min":1.127,"15min":1.099},"min":{"1min":0.64700000000000002,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":1.3100000000000001,"5min":3.1579999999999999,"15min":3.1579999999999999},"last":0.96799999999999997}]},{"osd":5,"last update":"Thu Sep 12 23:51:56 2024","interfaces":[{"interface":"back","average":{"1min":1.6479999999999999,"5min":1.2430000000000001,"15min":1.083},"min":{"1min":0.626,"5min":0.374,"15min":0.374},"max":{"1min":9.2149999999999999,"5min":9.2149999999999999,"15min":9.2149999999999999},"last":0.92100000000000004},{"interface":"front","average":{"1min":1.3049999999999999,"5min":1.0900000000000001,"15min":1.109},"min":{"1min":0.82599999999999996,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":4.6070000000000002,"5min":4.6070000000000002,"15min":4.6070000000000002},"last":1.0009999999999999}]},{"osd":6,"last update":"Thu Sep 12 23:52:25 2024","interfaces":[{"interface":"back","average":{"1min":1.1930000000000001,"5min":1.081,"15min":1.113},"min":{"1min":0.59999999999999998,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":4.7000000000000002,"5min":4.7000000000000002,"15min":4.7000000000000002},"last":1.2230000000000001},{"interface":"front","average":{"1min":1.2789999999999999,"5min":1.1539999999999999,"15min":1.1519999999999999},"min":{"1min":0.73399999999999999,"5min":0.437,"15min":0.437},"max":{"1min":4.6559999999999997,"5min":4.6559999999999997,"15min":4.6559999999999997},"last":1.2669999999999999}]},{"osd":7,"last update":"Thu Sep 12 23:51:46 2024","interfaces":[{"interface":"back","average":{"1min":1.3400000000000001,"5min":1.1120000000000001,"15min":1.077},"min":{"1min":0.76600000000000001,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":6.024,"5min":6.024,"15min":6.024},"last":1.24},{"interface":"front","average":{"1min":1.4370000000000001,"5min":1.331,"15min":1.484},"min":{"1min":0.85999999999999999,"5min":0.497,"15min":0.497},"max":{"1min":5.9660000000000002,"5min":10.93,"15min":10.93},"last":1.3100000000000001}]}]},{"osd":3,"up_from":492,"seq":2113123909713,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":344844,"kb_used_data":53760,"kb_used_omap":34,"kb_used_meta":291037,"kb_avail":93400308,"statfs":{"total":95995035648,"available":95641915392,"internally_reserved":0,"allocated":55050240,"data_stored":69653270,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35174,"internal_metadata":298022554},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Sep 12 23:51:58 2024","interfaces":[{"interface":"back","average":{"1min":0.71799999999999997,"5min":0.71599999999999997,"15min":0.77300000000000002},"min":{"1min":0.40300000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":1.262,"5min":2.4500000000000002,"15min":3.8010000000000002},"last":0.80200000000000005},{"interface":"front","average":{"1min":0.75800000000000001,"5min":0.88500000000000001,"15min":0.78500000000000003},"min":{"1min":0.54800000000000004,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":1.0109999999999999,"5min":13.878,"15min":13.878},"last":0.84999999999999998}]},{"osd":1,"last update":"Thu Sep 12 23:51:58 2024","interfaces":[{"interface":"back","average":{"1min":0.89400000000000002,"5min":0.78000000000000003,"15min":0.89200000000000002},"min":{"1min":0.51300000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.925,"5min":2.7040000000000002,"15min":4.8879999999999999},"last":0.754},{"interface":"front","average":{"1min":0.93400000000000005,"5min":0.90100000000000002,"15min":0.93500000000000005},"min":{"1min":0.56699999999999995,"5min":0.307,"15min":0.307},"max":{"1min":1.536,"5min":3.0779999999999998,"15min":3.0779999999999998},"last":0.94499999999999995}]},{"osd":2,"last update":"Thu Sep 12 23:51:58 2024","interfaces":[{"interface":"back","average":{"1min":0.879,"5min":0.878,"15min":0.82799999999999996},"min":{"1min":0.36499999999999999,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":1.1919999999999999,"5min":2.1240000000000001,"15min":2.1240000000000001},"last":0.69499999999999995},{"interface":"front","average":{"1min":0.875,"5min":0.85199999999999998,"15min":0.80800000000000005},"min":{"1min":0.623,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.159,"5min":1.776,"15min":1.776},"last":1.008}]},{"osd":4,"last update":"Thu Sep 12 23:51:31 2024","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":0.93899999999999995,"15min":0.997},"min":{"1min":0.68799999999999994,"5min":0.60499999999999998,"15min":0.60499999999999998},"max":{"1min":1.367,"5min":3.141,"15min":3.141},"last":0.91400000000000003},{"interface":"front","average":{"1min":1.173,"5min":1.046,"15min":1.0820000000000001},"min":{"1min":0.98799999999999999,"5min":0.54400000000000004,"15min":0.54400000000000004},"max":{"1min":1.411,"5min":1.53,"15min":1.53},"last":1.1719999999999999}]},{"osd":5,"last update":"Thu Sep 12 23:51:58 2024","interfaces":[{"interface":"back","average":{"1min":1.115,"5min":1.204,"15min":1.802},"min":{"1min":0.70099999999999996,"5min":0.64800000000000002,"15min":0.64800000000000002},"max":{"1min":1.5369999999999999,"5min":27.274999999999999,"15min":27.274999999999999},"last":1.0820000000000001},{"interface":"front","average":{"1min":1.097,"5min":1.135,"15min":1.2709999999999999},"min":{"1min":0.754,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":1.371,"5min":7.7190000000000003,"15min":7.7190000000000003},"last":1.0680000000000001}]},{"osd":6,"last update":"Thu Sep 12 23:51:25 2024","interfaces":[{"interface":"back","average":{"1min":1.1930000000000001,"5min":1.0840000000000001,"15min":1.0580000000000001},"min":{"1min":1.0149999999999999,"5min":0.59499999999999997,"15min":0.59499999999999997},"max":{"1min":1.4019999999999999,"5min":1.665,"15min":1.665},"last":1.0369999999999999},{"interface":"front","average":{"1min":1.2190000000000001,"5min":1.0980000000000001,"15min":1.073},"min":{"1min":0.80600000000000005,"5min":0.63100000000000001,"15min":0.63100000000000001},"max":{"1min":2.3319999999999999,"5min":2.3319999999999999,"15min":2.3319999999999999},"last":1.109}]},{"osd":7,"last update":"Thu Sep 12 23:51:52 2024","interfaces":[{"interface":"back","average":{"1min":1.1439999999999999,"5min":1.0580000000000001,"15min":1.052},"min":{"1min":0.80800000000000005,"5min":0.59899999999999998,"15min":0.59899999999999998},"max":{"1min":1.5740000000000001,"5min":1.5740000000000001,"15min":1.5740000000000001},"last":1.208},{"interface":"front","average":{"1min":1.2370000000000001,"5min":1.121,"15min":1.119},"min":{"1min":0.72099999999999997,"5min":0.60199999999999998,"15min":0.60199999999999998},"max":{"1min":1.611,"5min":1.611,"15min":1.611},"last":1.282}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7333,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2062,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2859,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5926,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3847,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4222,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3549,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":5822,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3702,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4149,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":85594112,"data_stored":125743104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":101388288,"data_stored":137052160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":65187840,"data_stored":95768576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":45989888,"data_stored":64294912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":78569472,"data_stored":112496640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":68947968,"data_stored":100810752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":22388736,"data_stored":31387648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":29302784,"data_stored":46059520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-12T23:52:30.141 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2024-09-12T23:52:30.141 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid da58a47c-715d-11ef-bceb-c7b262605968 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2024-09-12T23:52:31.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:30 smithi136 bash[75164]: audit 2024-09-12T23:52:29.329615+0000 mgr.y (mgr.24661) 461 : audit [DBG] from='client.34530 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-12T23:52:31.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:30 smithi136 bash[76371]: audit 2024-09-12T23:52:29.329615+0000 mgr.y (mgr.24661) 461 : audit [DBG] from='client.34530 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-12T23:52:31.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:30 smithi181 bash[67587]: audit 2024-09-12T23:52:29.329615+0000 mgr.y (mgr.24661) 461 : audit [DBG] from='client.34530 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-12T23:52:32.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:31 smithi136 bash[75164]: cluster 2024-09-12T23:52:30.903680+0000 mgr.y (mgr.24661) 462 : cluster [DBG] pgmap v885: 121 pgs: 121 active+clean; 327 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:52:32.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:31 smithi136 bash[76371]: cluster 2024-09-12T23:52:30.903680+0000 mgr.y (mgr.24661) 462 : cluster [DBG] pgmap v885: 121 pgs: 121 active+clean; 327 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:52:32.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:31 smithi181 bash[67587]: cluster 2024-09-12T23:52:30.903680+0000 mgr.y (mgr.24661) 462 : cluster [DBG] pgmap v885: 121 pgs: 121 active+clean; 327 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:52:32.597 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:52:32 smithi181 bash[95505]: ts=2024-09-12T23:52:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:52:33.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:32 smithi136 bash[75164]: cluster 2024-09-12T23:52:32.904842+0000 mgr.y (mgr.24661) 463 : cluster [DBG] pgmap v886: 121 pgs: 121 active+clean; 327 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:52:33.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:32 smithi136 bash[76371]: cluster 2024-09-12T23:52:32.904842+0000 mgr.y (mgr.24661) 463 : cluster [DBG] pgmap v886: 121 pgs: 121 active+clean; 327 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:52:33.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:32 smithi181 bash[67587]: cluster 2024-09-12T23:52:32.904842+0000 mgr.y (mgr.24661) 463 : cluster [DBG] pgmap v886: 121 pgs: 121 active+clean; 327 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:52:34.969 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/mon.a/config 2024-09-12T23:52:35.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:34 smithi136 bash[75164]: cluster 2024-09-12T23:52:34.905691+0000 mgr.y (mgr.24661) 464 : cluster [DBG] pgmap v887: 121 pgs: 121 active+clean; 327 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:52:35.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:34 smithi136 bash[76371]: cluster 2024-09-12T23:52:34.905691+0000 mgr.y (mgr.24661) 464 : cluster [DBG] pgmap v887: 121 pgs: 121 active+clean; 327 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:52:35.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:34 smithi181 bash[67587]: cluster 2024-09-12T23:52:34.905691+0000 mgr.y (mgr.24661) 464 : cluster [DBG] pgmap v887: 121 pgs: 121 active+clean; 327 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:52:37.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:36 smithi136 bash[75164]: cluster 2024-09-12T23:52:36.906721+0000 mgr.y (mgr.24661) 465 : cluster [DBG] pgmap v888: 121 pgs: 121 active+clean; 327 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:52:37.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:36 smithi136 bash[76371]: cluster 2024-09-12T23:52:36.906721+0000 mgr.y (mgr.24661) 465 : cluster [DBG] pgmap v888: 121 pgs: 121 active+clean; 327 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:52:37.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:36 smithi181 bash[67587]: cluster 2024-09-12T23:52:36.906721+0000 mgr.y (mgr.24661) 465 : cluster [DBG] pgmap v888: 121 pgs: 121 active+clean; 327 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:52:38.389 INFO:teuthology.orchestra.run.smithi136.stderr:pool 'unique_pool_0' removed 2024-09-12T23:52:38.645 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:38 smithi136 bash[75164]: audit 2024-09-12T23:52:38.324145+0000 mon.c (mon.2) 17 : audit [INF] from='client.? 172.21.15.136:0/2914550611' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2024-09-12T23:52:38.645 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:38 smithi136 bash[75164]: audit 2024-09-12T23:52:38.324881+0000 mon.a (mon.0) 1453 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2024-09-12T23:52:38.645 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:38 smithi136 bash[76371]: audit 2024-09-12T23:52:38.324145+0000 mon.c (mon.2) 17 : audit [INF] from='client.? 172.21.15.136:0/2914550611' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2024-09-12T23:52:38.645 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:38 smithi136 bash[76371]: audit 2024-09-12T23:52:38.324881+0000 mon.a (mon.0) 1453 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2024-09-12T23:52:38.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:38 smithi181 bash[67587]: audit 2024-09-12T23:52:38.324145+0000 mon.c (mon.2) 17 : audit [INF] from='client.? 172.21.15.136:0/2914550611' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2024-09-12T23:52:38.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:38 smithi181 bash[67587]: audit 2024-09-12T23:52:38.324881+0000 mon.a (mon.0) 1453 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2024-09-12T23:52:39.184 INFO:teuthology.task.full_sequential:In full_sequential, running task print... 2024-09-12T23:52:39.184 INFO:teuthology.task.print:**** done end ec-rados-default.yaml 2024-09-12T23:52:39.184 INFO:teuthology.task.full_sequential:In full_sequential, running task print... 2024-09-12T23:52:39.184 INFO:teuthology.task.print:**** done start rados_api.yaml 2024-09-12T23:52:39.185 INFO:teuthology.task.full_sequential:In full_sequential, running task workunit... 2024-09-12T23:52:39.185 INFO:tasks.workunit:Pulling workunits from ref quincy 2024-09-12T23:52:39.186 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-09-12T23:52:39.186 DEBUG:teuthology.orchestra.run.smithi136:> stat -- /home/ubuntu/cephtest/mnt.0 2024-09-12T23:52:39.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-12T23:52:39.192 INFO:teuthology.orchestra.run.smithi136.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2024-09-12T23:52:39.192 DEBUG:teuthology.orchestra.run.smithi136:> mkdir -- /home/ubuntu/cephtest/mnt.0 2024-09-12T23:52:39.242 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2024-09-12T23:52:39.242 DEBUG:teuthology.orchestra.run.smithi136:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2024-09-12T23:52:39.293 INFO:tasks.workunit:timeout=3h 2024-09-12T23:52:39.293 INFO:tasks.workunit:cleanup=True 2024-09-12T23:52:39.294 DEBUG:teuthology.orchestra.run.smithi136:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://git.ceph.com/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2024-09-12T23:52:39.344 INFO:tasks.workunit.client.0.smithi136.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-09-12T23:52:39.506 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:52:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:52:39] "GET /metrics HTTP/1.1" 200 34085 "" "Prometheus/2.43.0" 2024-09-12T23:52:39.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:39 smithi136 bash[76371]: audit 2024-09-12T23:52:38.383156+0000 mon.a (mon.0) 1454 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-09-12T23:52:39.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:39 smithi136 bash[76371]: cluster 2024-09-12T23:52:38.383337+0000 mon.a (mon.0) 1455 : cluster [DBG] osdmap e863: 8 total, 8 up, 8 in 2024-09-12T23:52:39.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:39 smithi136 bash[76371]: audit 2024-09-12T23:52:38.756811+0000 mon.a (mon.0) 1456 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:52:39.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:39 smithi136 bash[76371]: cluster 2024-09-12T23:52:38.907419+0000 mgr.y (mgr.24661) 466 : cluster [DBG] pgmap v890: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:52:39.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:39 smithi136 bash[75164]: audit 2024-09-12T23:52:38.383156+0000 mon.a (mon.0) 1454 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-09-12T23:52:39.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:39 smithi136 bash[75164]: cluster 2024-09-12T23:52:38.383337+0000 mon.a (mon.0) 1455 : cluster [DBG] osdmap e863: 8 total, 8 up, 8 in 2024-09-12T23:52:39.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:39 smithi136 bash[75164]: audit 2024-09-12T23:52:38.756811+0000 mon.a (mon.0) 1456 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:52:39.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:39 smithi136 bash[75164]: cluster 2024-09-12T23:52:38.907419+0000 mgr.y (mgr.24661) 466 : cluster [DBG] pgmap v890: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:52:39.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:39 smithi181 bash[67587]: audit 2024-09-12T23:52:38.383156+0000 mon.a (mon.0) 1454 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-09-12T23:52:39.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:39 smithi181 bash[67587]: cluster 2024-09-12T23:52:38.383337+0000 mon.a (mon.0) 1455 : cluster [DBG] osdmap e863: 8 total, 8 up, 8 in 2024-09-12T23:52:39.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:39 smithi181 bash[67587]: audit 2024-09-12T23:52:38.756811+0000 mon.a (mon.0) 1456 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:52:39.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:39 smithi181 bash[67587]: cluster 2024-09-12T23:52:38.907419+0000 mgr.y (mgr.24661) 466 : cluster [DBG] pgmap v890: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:52:41.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:40 smithi136 bash[75164]: cluster 2024-09-12T23:52:40.908389+0000 mgr.y (mgr.24661) 467 : cluster [DBG] pgmap v891: 105 pgs: 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:52:41.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:40 smithi136 bash[76371]: cluster 2024-09-12T23:52:40.908389+0000 mgr.y (mgr.24661) 467 : cluster [DBG] pgmap v891: 105 pgs: 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:52:41.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:40 smithi181 bash[67587]: cluster 2024-09-12T23:52:40.908389+0000 mgr.y (mgr.24661) 467 : cluster [DBG] pgmap v891: 105 pgs: 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:52:42.597 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:52:42 smithi181 bash[95505]: ts=2024-09-12T23:52:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:52:43.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:42 smithi136 bash[76371]: cluster 2024-09-12T23:52:42.909550+0000 mgr.y (mgr.24661) 468 : cluster [DBG] pgmap v892: 105 pgs: 105 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:52:43.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:42 smithi136 bash[75164]: cluster 2024-09-12T23:52:42.909550+0000 mgr.y (mgr.24661) 468 : cluster [DBG] pgmap v892: 105 pgs: 105 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:52:43.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:42 smithi181 bash[67587]: cluster 2024-09-12T23:52:42.909550+0000 mgr.y (mgr.24661) 468 : cluster [DBG] pgmap v892: 105 pgs: 105 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:52:45.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:44 smithi136 bash[75164]: cluster 2024-09-12T23:52:44.910177+0000 mgr.y (mgr.24661) 469 : cluster [DBG] pgmap v893: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:52:45.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:44 smithi136 bash[76371]: cluster 2024-09-12T23:52:44.910177+0000 mgr.y (mgr.24661) 469 : cluster [DBG] pgmap v893: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:52:45.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:44 smithi181 bash[67587]: cluster 2024-09-12T23:52:44.910177+0000 mgr.y (mgr.24661) 469 : cluster [DBG] pgmap v893: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:52:45.611 DEBUG:teuthology.orchestra.run.smithi136:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-09-12T23:52:45.661 INFO:tasks.workunit.client.0.smithi136.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-09-12T23:52:45.665 INFO:tasks.workunit.client.0.smithi136.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-12T23:52:45.665 INFO:tasks.workunit.client.0.smithi136.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-09-12T23:52:45.720 INFO:tasks.workunit.client.0.smithi136.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-09-12T23:52:45.775 INFO:tasks.workunit.client.0.smithi136.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-09-12T23:52:45.817 INFO:tasks.workunit.client.0.smithi136.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-12T23:52:45.818 INFO:tasks.workunit.client.0.smithi136.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-12T23:52:45.818 INFO:tasks.workunit.client.0.smithi136.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-09-12T23:52:45.861 INFO:tasks.workunit.client.0.smithi136.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-12T23:52:45.864 DEBUG:teuthology.orchestra.run.smithi136:> set -ex 2024-09-12T23:52:45.864 DEBUG:teuthology.orchestra.run.smithi136:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-09-12T23:52:45.913 INFO:tasks.workunit:Running workunits matching cls on client.0... 2024-09-12T23:52:45.915 INFO:tasks.workunit:Running workunit cls/test_cls_2pc_queue.sh... 2024-09-12T23:52:45.915 DEBUG:teuthology.orchestra.run.smithi136:workunit test cls/test_cls_2pc_queue.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_2pc_queue.sh 2024-09-12T23:52:45.973 INFO:tasks.workunit.client.0.smithi136.stdout:Running main() from gmock_main.cc 2024-09-12T23:52:45.973 INFO:tasks.workunit.client.0.smithi136.stdout:[==========] Running 22 tests from 1 test suite. 2024-09-12T23:52:45.973 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] Global test environment set-up. 2024-09-12T23:52:45.973 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 22 tests from TestCls2PCQueue 2024-09-12T23:52:45.973 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestCls2PCQueue.GetCapacity 2024-09-12T23:52:47.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:47 smithi136 bash[75164]: cluster 2024-09-12T23:52:46.489784+0000 mon.a (mon.0) 1457 : cluster [DBG] osdmap e864: 8 total, 8 up, 8 in 2024-09-12T23:52:47.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:47 smithi136 bash[75164]: audit 2024-09-12T23:52:46.495551+0000 mon.a (mon.0) 1458 : audit [INF] from='client.? 172.21.15.136:0/2924532087' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:52:47.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:47 smithi136 bash[75164]: cluster 2024-09-12T23:52:46.911056+0000 mgr.y (mgr.24661) 470 : cluster [DBG] pgmap v895: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:52:47.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:47 smithi136 bash[76371]: cluster 2024-09-12T23:52:46.489784+0000 mon.a (mon.0) 1457 : cluster [DBG] osdmap e864: 8 total, 8 up, 8 in 2024-09-12T23:52:47.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:47 smithi136 bash[76371]: audit 2024-09-12T23:52:46.495551+0000 mon.a (mon.0) 1458 : audit [INF] from='client.? 172.21.15.136:0/2924532087' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:52:47.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:47 smithi136 bash[76371]: cluster 2024-09-12T23:52:46.911056+0000 mgr.y (mgr.24661) 470 : cluster [DBG] pgmap v895: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:52:47.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:47 smithi181 bash[67587]: cluster 2024-09-12T23:52:46.489784+0000 mon.a (mon.0) 1457 : cluster [DBG] osdmap e864: 8 total, 8 up, 8 in 2024-09-12T23:52:47.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:47 smithi181 bash[67587]: audit 2024-09-12T23:52:46.495551+0000 mon.a (mon.0) 1458 : audit [INF] from='client.? 172.21.15.136:0/2924532087' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:52:47.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:47 smithi181 bash[67587]: cluster 2024-09-12T23:52:46.911056+0000 mgr.y (mgr.24661) 470 : cluster [DBG] pgmap v895: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:52:48.506 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestCls2PCQueue.GetCapacity (2532 ms) 2024-09-12T23:52:48.506 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestCls2PCQueue.AsyncGetCapacity 2024-09-12T23:52:48.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:48 smithi136 bash[75164]: cluster 2024-09-12T23:52:47.481300+0000 mon.a (mon.0) 1459 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:52:48.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:48 smithi136 bash[75164]: audit 2024-09-12T23:52:47.499091+0000 mon.a (mon.0) 1460 : audit [INF] from='client.? 172.21.15.136:0/2924532087' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:52:48.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:48 smithi136 bash[75164]: cluster 2024-09-12T23:52:47.499382+0000 mon.a (mon.0) 1461 : cluster [DBG] osdmap e865: 8 total, 8 up, 8 in 2024-09-12T23:52:48.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:48 smithi136 bash[76371]: cluster 2024-09-12T23:52:47.481300+0000 mon.a (mon.0) 1459 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:52:48.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:48 smithi136 bash[76371]: audit 2024-09-12T23:52:47.499091+0000 mon.a (mon.0) 1460 : audit [INF] from='client.? 172.21.15.136:0/2924532087' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:52:48.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:48 smithi136 bash[76371]: cluster 2024-09-12T23:52:47.499382+0000 mon.a (mon.0) 1461 : cluster [DBG] osdmap e865: 8 total, 8 up, 8 in 2024-09-12T23:52:48.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:48 smithi181 bash[67587]: cluster 2024-09-12T23:52:47.481300+0000 mon.a (mon.0) 1459 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:52:48.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:48 smithi181 bash[67587]: audit 2024-09-12T23:52:47.499091+0000 mon.a (mon.0) 1460 : audit [INF] from='client.? 172.21.15.136:0/2924532087' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:52:48.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:48 smithi181 bash[67587]: cluster 2024-09-12T23:52:47.499382+0000 mon.a (mon.0) 1461 : cluster [DBG] osdmap e865: 8 total, 8 up, 8 in 2024-09-12T23:52:49.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:52:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:52:49] "GET /metrics HTTP/1.1" 200 34085 "" "Prometheus/2.43.0" 2024-09-12T23:52:49.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:49 smithi136 bash[76371]: cluster 2024-09-12T23:52:48.503103+0000 mon.a (mon.0) 1462 : cluster [DBG] osdmap e866: 8 total, 8 up, 8 in 2024-09-12T23:52:49.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:49 smithi136 bash[76371]: cluster 2024-09-12T23:52:48.911784+0000 mgr.y (mgr.24661) 471 : cluster [DBG] pgmap v898: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:52:49.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:49 smithi136 bash[75164]: cluster 2024-09-12T23:52:48.503103+0000 mon.a (mon.0) 1462 : cluster [DBG] osdmap e866: 8 total, 8 up, 8 in 2024-09-12T23:52:49.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:49 smithi136 bash[75164]: cluster 2024-09-12T23:52:48.911784+0000 mgr.y (mgr.24661) 471 : cluster [DBG] pgmap v898: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:52:49.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:49 smithi181 bash[67587]: cluster 2024-09-12T23:52:48.503103+0000 mon.a (mon.0) 1462 : cluster [DBG] osdmap e866: 8 total, 8 up, 8 in 2024-09-12T23:52:49.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:49 smithi181 bash[67587]: cluster 2024-09-12T23:52:48.911784+0000 mgr.y (mgr.24661) 471 : cluster [DBG] pgmap v898: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:52:50.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:50 smithi181 bash[67587]: cluster 2024-09-12T23:52:49.509453+0000 mon.a (mon.0) 1463 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:52:50.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:50 smithi181 bash[67587]: cluster 2024-09-12T23:52:49.509509+0000 mon.a (mon.0) 1464 : cluster [INF] Cluster is now healthy 2024-09-12T23:52:50.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:50 smithi181 bash[67587]: cluster 2024-09-12T23:52:49.522518+0000 mon.a (mon.0) 1465 : cluster [DBG] osdmap e867: 8 total, 8 up, 8 in 2024-09-12T23:52:50.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:50 smithi181 bash[67587]: audit 2024-09-12T23:52:49.528826+0000 mon.c (mon.2) 18 : audit [INF] from='client.? 172.21.15.136:0/2350385049' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:52:50.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:50 smithi181 bash[67587]: audit 2024-09-12T23:52:49.542006+0000 mon.a (mon.0) 1466 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:52:50.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:50 smithi181 bash[67587]: audit 2024-09-12T23:52:50.523592+0000 mon.a (mon.0) 1467 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:52:50.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:50 smithi181 bash[67587]: cluster 2024-09-12T23:52:50.523909+0000 mon.a (mon.0) 1468 : cluster [DBG] osdmap e868: 8 total, 8 up, 8 in 2024-09-12T23:52:51.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:50 smithi136 bash[76371]: cluster 2024-09-12T23:52:49.509453+0000 mon.a (mon.0) 1463 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:52:51.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:50 smithi136 bash[76371]: cluster 2024-09-12T23:52:49.509509+0000 mon.a (mon.0) 1464 : cluster [INF] Cluster is now healthy 2024-09-12T23:52:51.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:50 smithi136 bash[76371]: cluster 2024-09-12T23:52:49.522518+0000 mon.a (mon.0) 1465 : cluster [DBG] osdmap e867: 8 total, 8 up, 8 in 2024-09-12T23:52:51.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:50 smithi136 bash[76371]: audit 2024-09-12T23:52:49.528826+0000 mon.c (mon.2) 18 : audit [INF] from='client.? 172.21.15.136:0/2350385049' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:52:51.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:50 smithi136 bash[76371]: audit 2024-09-12T23:52:49.542006+0000 mon.a (mon.0) 1466 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:52:51.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:50 smithi136 bash[76371]: audit 2024-09-12T23:52:50.523592+0000 mon.a (mon.0) 1467 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:52:51.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:50 smithi136 bash[76371]: cluster 2024-09-12T23:52:50.523909+0000 mon.a (mon.0) 1468 : cluster [DBG] osdmap e868: 8 total, 8 up, 8 in 2024-09-12T23:52:51.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:50 smithi136 bash[75164]: cluster 2024-09-12T23:52:49.509453+0000 mon.a (mon.0) 1463 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:52:51.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:50 smithi136 bash[75164]: cluster 2024-09-12T23:52:49.509509+0000 mon.a (mon.0) 1464 : cluster [INF] Cluster is now healthy 2024-09-12T23:52:51.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:50 smithi136 bash[75164]: cluster 2024-09-12T23:52:49.522518+0000 mon.a (mon.0) 1465 : cluster [DBG] osdmap e867: 8 total, 8 up, 8 in 2024-09-12T23:52:51.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:50 smithi136 bash[75164]: audit 2024-09-12T23:52:49.528826+0000 mon.c (mon.2) 18 : audit [INF] from='client.? 172.21.15.136:0/2350385049' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:52:51.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:50 smithi136 bash[75164]: audit 2024-09-12T23:52:49.542006+0000 mon.a (mon.0) 1466 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:52:51.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:50 smithi136 bash[75164]: audit 2024-09-12T23:52:50.523592+0000 mon.a (mon.0) 1467 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:52:51.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:50 smithi136 bash[75164]: cluster 2024-09-12T23:52:50.523909+0000 mon.a (mon.0) 1468 : cluster [DBG] osdmap e868: 8 total, 8 up, 8 in 2024-09-12T23:52:51.570 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestCls2PCQueue.AsyncGetCapacity (3064 ms) 2024-09-12T23:52:51.570 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestCls2PCQueue.Reserve 2024-09-12T23:52:51.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:51 smithi181 bash[67587]: cluster 2024-09-12T23:52:50.912833+0000 mgr.y (mgr.24661) 472 : cluster [DBG] pgmap v901: 137 pgs: 3 creating+peering, 20 unknown, 114 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:52:52.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:51 smithi136 bash[75164]: cluster 2024-09-12T23:52:50.912833+0000 mgr.y (mgr.24661) 472 : cluster [DBG] pgmap v901: 137 pgs: 3 creating+peering, 20 unknown, 114 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:52:52.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:51 smithi136 bash[76371]: cluster 2024-09-12T23:52:50.912833+0000 mgr.y (mgr.24661) 472 : cluster [DBG] pgmap v901: 137 pgs: 3 creating+peering, 20 unknown, 114 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:52:52.580 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:52:52 smithi181 bash[95505]: ts=2024-09-12T23:52:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:52:52.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:52 smithi181 bash[67587]: cluster 2024-09-12T23:52:51.563796+0000 mon.a (mon.0) 1469 : cluster [DBG] osdmap e869: 8 total, 8 up, 8 in 2024-09-12T23:52:53.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:52 smithi136 bash[75164]: cluster 2024-09-12T23:52:51.563796+0000 mon.a (mon.0) 1469 : cluster [DBG] osdmap e869: 8 total, 8 up, 8 in 2024-09-12T23:52:53.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:52 smithi136 bash[76371]: cluster 2024-09-12T23:52:51.563796+0000 mon.a (mon.0) 1469 : cluster [DBG] osdmap e869: 8 total, 8 up, 8 in 2024-09-12T23:52:53.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:53 smithi181 bash[67587]: cluster 2024-09-12T23:52:52.567427+0000 mon.a (mon.0) 1470 : cluster [DBG] osdmap e870: 8 total, 8 up, 8 in 2024-09-12T23:52:53.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:53 smithi181 bash[67587]: audit 2024-09-12T23:52:52.573004+0000 mon.c (mon.2) 19 : audit [INF] from='client.? 172.21.15.136:0/2907766871' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:52:53.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:53 smithi181 bash[67587]: audit 2024-09-12T23:52:52.590924+0000 mon.a (mon.0) 1471 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:52:53.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:53 smithi181 bash[67587]: cluster 2024-09-12T23:52:52.913711+0000 mgr.y (mgr.24661) 473 : cluster [DBG] pgmap v904: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:52:53.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:53 smithi181 bash[67587]: audit 2024-09-12T23:52:53.575399+0000 mon.a (mon.0) 1472 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:52:53.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:53 smithi181 bash[67587]: cluster 2024-09-12T23:52:53.575781+0000 mon.a (mon.0) 1473 : cluster [DBG] osdmap e871: 8 total, 8 up, 8 in 2024-09-12T23:52:54.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:53 smithi136 bash[75164]: cluster 2024-09-12T23:52:52.567427+0000 mon.a (mon.0) 1470 : cluster [DBG] osdmap e870: 8 total, 8 up, 8 in 2024-09-12T23:52:54.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:53 smithi136 bash[75164]: audit 2024-09-12T23:52:52.573004+0000 mon.c (mon.2) 19 : audit [INF] from='client.? 172.21.15.136:0/2907766871' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:52:54.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:53 smithi136 bash[75164]: audit 2024-09-12T23:52:52.590924+0000 mon.a (mon.0) 1471 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:52:54.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:53 smithi136 bash[75164]: cluster 2024-09-12T23:52:52.913711+0000 mgr.y (mgr.24661) 473 : cluster [DBG] pgmap v904: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:52:54.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:53 smithi136 bash[75164]: audit 2024-09-12T23:52:53.575399+0000 mon.a (mon.0) 1472 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:52:54.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:53 smithi136 bash[75164]: cluster 2024-09-12T23:52:53.575781+0000 mon.a (mon.0) 1473 : cluster [DBG] osdmap e871: 8 total, 8 up, 8 in 2024-09-12T23:52:54.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:53 smithi136 bash[76371]: cluster 2024-09-12T23:52:52.567427+0000 mon.a (mon.0) 1470 : cluster [DBG] osdmap e870: 8 total, 8 up, 8 in 2024-09-12T23:52:54.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:53 smithi136 bash[76371]: audit 2024-09-12T23:52:52.573004+0000 mon.c (mon.2) 19 : audit [INF] from='client.? 172.21.15.136:0/2907766871' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:52:54.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:53 smithi136 bash[76371]: audit 2024-09-12T23:52:52.590924+0000 mon.a (mon.0) 1471 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:52:54.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:53 smithi136 bash[76371]: cluster 2024-09-12T23:52:52.913711+0000 mgr.y (mgr.24661) 473 : cluster [DBG] pgmap v904: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:52:54.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:53 smithi136 bash[76371]: audit 2024-09-12T23:52:53.575399+0000 mon.a (mon.0) 1472 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:52:54.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:53 smithi136 bash[76371]: cluster 2024-09-12T23:52:53.575781+0000 mon.a (mon.0) 1473 : cluster [DBG] osdmap e871: 8 total, 8 up, 8 in 2024-09-12T23:52:54.571 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestCls2PCQueue.Reserve (3002 ms) 2024-09-12T23:52:54.571 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestCls2PCQueue.AsyncReserve 2024-09-12T23:52:54.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:54 smithi181 bash[67587]: cluster 2024-09-12T23:52:53.594757+0000 mon.a (mon.0) 1474 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:52:54.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:54 smithi181 bash[67587]: audit 2024-09-12T23:52:53.765821+0000 mon.a (mon.0) 1475 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:52:54.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:54 smithi181 bash[67587]: audit 2024-09-12T23:52:53.767773+0000 mon.a (mon.0) 1476 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:52:54.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:54 smithi181 bash[67587]: cluster 2024-09-12T23:52:54.570038+0000 mon.a (mon.0) 1477 : cluster [DBG] osdmap e872: 8 total, 8 up, 8 in 2024-09-12T23:52:55.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:54 smithi136 bash[75164]: cluster 2024-09-12T23:52:53.594757+0000 mon.a (mon.0) 1474 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:52:55.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:54 smithi136 bash[75164]: audit 2024-09-12T23:52:53.765821+0000 mon.a (mon.0) 1475 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:52:55.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:54 smithi136 bash[75164]: audit 2024-09-12T23:52:53.767773+0000 mon.a (mon.0) 1476 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:52:55.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:54 smithi136 bash[75164]: cluster 2024-09-12T23:52:54.570038+0000 mon.a (mon.0) 1477 : cluster [DBG] osdmap e872: 8 total, 8 up, 8 in 2024-09-12T23:52:55.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:54 smithi136 bash[76371]: cluster 2024-09-12T23:52:53.594757+0000 mon.a (mon.0) 1474 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:52:55.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:54 smithi136 bash[76371]: audit 2024-09-12T23:52:53.765821+0000 mon.a (mon.0) 1475 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:52:55.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:54 smithi136 bash[76371]: audit 2024-09-12T23:52:53.767773+0000 mon.a (mon.0) 1476 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:52:55.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:54 smithi136 bash[76371]: cluster 2024-09-12T23:52:54.570038+0000 mon.a (mon.0) 1477 : cluster [DBG] osdmap e872: 8 total, 8 up, 8 in 2024-09-12T23:52:56.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:55 smithi136 bash[75164]: cluster 2024-09-12T23:52:54.914353+0000 mgr.y (mgr.24661) 474 : cluster [DBG] pgmap v907: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:52:56.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:55 smithi136 bash[76371]: cluster 2024-09-12T23:52:54.914353+0000 mgr.y (mgr.24661) 474 : cluster [DBG] pgmap v907: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:52:56.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:55 smithi181 bash[67587]: cluster 2024-09-12T23:52:54.914353+0000 mgr.y (mgr.24661) 474 : cluster [DBG] pgmap v907: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:52:57.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:56 smithi136 bash[75164]: cluster 2024-09-12T23:52:55.590685+0000 mon.a (mon.0) 1478 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:52:57.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:56 smithi136 bash[75164]: cluster 2024-09-12T23:52:55.590758+0000 mon.a (mon.0) 1479 : cluster [INF] Cluster is now healthy 2024-09-12T23:52:57.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:56 smithi136 bash[75164]: cluster 2024-09-12T23:52:55.618203+0000 mon.a (mon.0) 1480 : cluster [DBG] osdmap e873: 8 total, 8 up, 8 in 2024-09-12T23:52:57.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:56 smithi136 bash[75164]: audit 2024-09-12T23:52:55.622518+0000 mon.c (mon.2) 20 : audit [INF] from='client.? 172.21.15.136:0/3122690589' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:52:57.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:56 smithi136 bash[75164]: audit 2024-09-12T23:52:55.624411+0000 mon.a (mon.0) 1481 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:52:57.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:56 smithi136 bash[76371]: cluster 2024-09-12T23:52:55.590685+0000 mon.a (mon.0) 1478 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:52:57.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:56 smithi136 bash[76371]: cluster 2024-09-12T23:52:55.590758+0000 mon.a (mon.0) 1479 : cluster [INF] Cluster is now healthy 2024-09-12T23:52:57.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:56 smithi136 bash[76371]: cluster 2024-09-12T23:52:55.618203+0000 mon.a (mon.0) 1480 : cluster [DBG] osdmap e873: 8 total, 8 up, 8 in 2024-09-12T23:52:57.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:56 smithi136 bash[76371]: audit 2024-09-12T23:52:55.622518+0000 mon.c (mon.2) 20 : audit [INF] from='client.? 172.21.15.136:0/3122690589' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:52:57.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:56 smithi136 bash[76371]: audit 2024-09-12T23:52:55.624411+0000 mon.a (mon.0) 1481 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:52:57.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:56 smithi181 bash[67587]: cluster 2024-09-12T23:52:55.590685+0000 mon.a (mon.0) 1478 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:52:57.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:56 smithi181 bash[67587]: cluster 2024-09-12T23:52:55.590758+0000 mon.a (mon.0) 1479 : cluster [INF] Cluster is now healthy 2024-09-12T23:52:57.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:56 smithi181 bash[67587]: cluster 2024-09-12T23:52:55.618203+0000 mon.a (mon.0) 1480 : cluster [DBG] osdmap e873: 8 total, 8 up, 8 in 2024-09-12T23:52:57.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:56 smithi181 bash[67587]: audit 2024-09-12T23:52:55.622518+0000 mon.c (mon.2) 20 : audit [INF] from='client.? 172.21.15.136:0/3122690589' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:52:57.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:56 smithi181 bash[67587]: audit 2024-09-12T23:52:55.624411+0000 mon.a (mon.0) 1481 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:52:57.635 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestCls2PCQueue.AsyncReserve (3064 ms) 2024-09-12T23:52:57.635 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestCls2PCQueue.Commit 2024-09-12T23:52:58.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:57 smithi136 bash[75164]: audit 2024-09-12T23:52:56.623516+0000 mon.a (mon.0) 1482 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:52:58.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:57 smithi136 bash[75164]: cluster 2024-09-12T23:52:56.623965+0000 mon.a (mon.0) 1483 : cluster [DBG] osdmap e874: 8 total, 8 up, 8 in 2024-09-12T23:52:58.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:57 smithi136 bash[75164]: cluster 2024-09-12T23:52:56.915246+0000 mgr.y (mgr.24661) 475 : cluster [DBG] pgmap v910: 137 pgs: 1 creating+activating, 8 creating+peering, 20 unknown, 108 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:52:58.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:57 smithi136 bash[76371]: audit 2024-09-12T23:52:56.623516+0000 mon.a (mon.0) 1482 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:52:58.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:57 smithi136 bash[76371]: cluster 2024-09-12T23:52:56.623965+0000 mon.a (mon.0) 1483 : cluster [DBG] osdmap e874: 8 total, 8 up, 8 in 2024-09-12T23:52:58.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:57 smithi136 bash[76371]: cluster 2024-09-12T23:52:56.915246+0000 mgr.y (mgr.24661) 475 : cluster [DBG] pgmap v910: 137 pgs: 1 creating+activating, 8 creating+peering, 20 unknown, 108 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:52:58.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:57 smithi181 bash[67587]: audit 2024-09-12T23:52:56.623516+0000 mon.a (mon.0) 1482 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:52:58.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:57 smithi181 bash[67587]: cluster 2024-09-12T23:52:56.623965+0000 mon.a (mon.0) 1483 : cluster [DBG] osdmap e874: 8 total, 8 up, 8 in 2024-09-12T23:52:58.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:57 smithi181 bash[67587]: cluster 2024-09-12T23:52:56.915246+0000 mgr.y (mgr.24661) 475 : cluster [DBG] pgmap v910: 137 pgs: 1 creating+activating, 8 creating+peering, 20 unknown, 108 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:52:59.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:58 smithi136 bash[75164]: cluster 2024-09-12T23:52:57.633546+0000 mon.a (mon.0) 1484 : cluster [DBG] osdmap e875: 8 total, 8 up, 8 in 2024-09-12T23:52:59.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:58 smithi136 bash[76371]: cluster 2024-09-12T23:52:57.633546+0000 mon.a (mon.0) 1484 : cluster [DBG] osdmap e875: 8 total, 8 up, 8 in 2024-09-12T23:52:59.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:58 smithi181 bash[67587]: cluster 2024-09-12T23:52:57.633546+0000 mon.a (mon.0) 1484 : cluster [DBG] osdmap e875: 8 total, 8 up, 8 in 2024-09-12T23:52:59.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:52:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:52:59] "GET /metrics HTTP/1.1" 200 36398 "" "Prometheus/2.43.0" 2024-09-12T23:52:59.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:59 smithi136 bash[76371]: cluster 2024-09-12T23:52:58.642365+0000 mon.a (mon.0) 1485 : cluster [DBG] osdmap e876: 8 total, 8 up, 8 in 2024-09-12T23:52:59.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:59 smithi136 bash[76371]: audit 2024-09-12T23:52:58.649616+0000 mon.c (mon.2) 21 : audit [INF] from='client.? 172.21.15.136:0/2976151181' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:52:59.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:59 smithi136 bash[76371]: audit 2024-09-12T23:52:58.651012+0000 mon.a (mon.0) 1486 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:52:59.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:52:59 smithi136 bash[76371]: cluster 2024-09-12T23:52:58.915988+0000 mgr.y (mgr.24661) 476 : cluster [DBG] pgmap v913: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:52:59.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:59 smithi136 bash[75164]: cluster 2024-09-12T23:52:58.642365+0000 mon.a (mon.0) 1485 : cluster [DBG] osdmap e876: 8 total, 8 up, 8 in 2024-09-12T23:52:59.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:59 smithi136 bash[75164]: audit 2024-09-12T23:52:58.649616+0000 mon.c (mon.2) 21 : audit [INF] from='client.? 172.21.15.136:0/2976151181' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:52:59.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:59 smithi136 bash[75164]: audit 2024-09-12T23:52:58.651012+0000 mon.a (mon.0) 1486 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:52:59.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:52:59 smithi136 bash[75164]: cluster 2024-09-12T23:52:58.915988+0000 mgr.y (mgr.24661) 476 : cluster [DBG] pgmap v913: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:00.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:59 smithi181 bash[67587]: cluster 2024-09-12T23:52:58.642365+0000 mon.a (mon.0) 1485 : cluster [DBG] osdmap e876: 8 total, 8 up, 8 in 2024-09-12T23:53:00.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:59 smithi181 bash[67587]: audit 2024-09-12T23:52:58.649616+0000 mon.c (mon.2) 21 : audit [INF] from='client.? 172.21.15.136:0/2976151181' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:00.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:59 smithi181 bash[67587]: audit 2024-09-12T23:52:58.651012+0000 mon.a (mon.0) 1486 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:00.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:52:59 smithi181 bash[67587]: cluster 2024-09-12T23:52:58.915988+0000 mgr.y (mgr.24661) 476 : cluster [DBG] pgmap v913: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:01.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:00 smithi136 bash[75164]: cluster 2024-09-12T23:52:59.636529+0000 mon.a (mon.0) 1487 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:53:01.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:00 smithi136 bash[75164]: audit 2024-09-12T23:52:59.658959+0000 mon.a (mon.0) 1488 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:53:01.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:00 smithi136 bash[75164]: cluster 2024-09-12T23:52:59.659486+0000 mon.a (mon.0) 1489 : cluster [DBG] osdmap e877: 8 total, 8 up, 8 in 2024-09-12T23:53:01.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:00 smithi136 bash[76371]: cluster 2024-09-12T23:52:59.636529+0000 mon.a (mon.0) 1487 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:53:01.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:00 smithi136 bash[76371]: audit 2024-09-12T23:52:59.658959+0000 mon.a (mon.0) 1488 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:53:01.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:00 smithi136 bash[76371]: cluster 2024-09-12T23:52:59.659486+0000 mon.a (mon.0) 1489 : cluster [DBG] osdmap e877: 8 total, 8 up, 8 in 2024-09-12T23:53:01.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:00 smithi181 bash[67587]: cluster 2024-09-12T23:52:59.636529+0000 mon.a (mon.0) 1487 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:53:01.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:00 smithi181 bash[67587]: audit 2024-09-12T23:52:59.658959+0000 mon.a (mon.0) 1488 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:53:01.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:00 smithi181 bash[67587]: cluster 2024-09-12T23:52:59.659486+0000 mon.a (mon.0) 1489 : cluster [DBG] osdmap e877: 8 total, 8 up, 8 in 2024-09-12T23:53:01.682 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestCls2PCQueue.Commit (4047 ms) 2024-09-12T23:53:01.682 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestCls2PCQueue.Abort 2024-09-12T23:53:02.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:01 smithi136 bash[75164]: cluster 2024-09-12T23:53:00.659505+0000 mon.a (mon.0) 1490 : cluster [DBG] osdmap e878: 8 total, 8 up, 8 in 2024-09-12T23:53:02.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:01 smithi136 bash[75164]: cluster 2024-09-12T23:53:00.916883+0000 mgr.y (mgr.24661) 477 : cluster [DBG] pgmap v916: 137 pgs: 5 creating+peering, 23 unknown, 109 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:02.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:01 smithi136 bash[76371]: cluster 2024-09-12T23:53:00.659505+0000 mon.a (mon.0) 1490 : cluster [DBG] osdmap e878: 8 total, 8 up, 8 in 2024-09-12T23:53:02.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:01 smithi136 bash[76371]: cluster 2024-09-12T23:53:00.916883+0000 mgr.y (mgr.24661) 477 : cluster [DBG] pgmap v916: 137 pgs: 5 creating+peering, 23 unknown, 109 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:02.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:01 smithi181 bash[67587]: cluster 2024-09-12T23:53:00.659505+0000 mon.a (mon.0) 1490 : cluster [DBG] osdmap e878: 8 total, 8 up, 8 in 2024-09-12T23:53:02.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:01 smithi181 bash[67587]: cluster 2024-09-12T23:53:00.916883+0000 mgr.y (mgr.24661) 477 : cluster [DBG] pgmap v916: 137 pgs: 5 creating+peering, 23 unknown, 109 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:02.597 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:53:02 smithi181 bash[95505]: ts=2024-09-12T23:53:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:53:03.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:02 smithi136 bash[75164]: cluster 2024-09-12T23:53:01.661840+0000 mon.a (mon.0) 1491 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:53:03.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:02 smithi136 bash[75164]: cluster 2024-09-12T23:53:01.661885+0000 mon.a (mon.0) 1492 : cluster [INF] Cluster is now healthy 2024-09-12T23:53:03.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:02 smithi136 bash[75164]: cluster 2024-09-12T23:53:01.681367+0000 mon.a (mon.0) 1493 : cluster [DBG] osdmap e879: 8 total, 8 up, 8 in 2024-09-12T23:53:03.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:02 smithi136 bash[76371]: cluster 2024-09-12T23:53:01.661840+0000 mon.a (mon.0) 1491 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:53:03.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:02 smithi136 bash[76371]: cluster 2024-09-12T23:53:01.661885+0000 mon.a (mon.0) 1492 : cluster [INF] Cluster is now healthy 2024-09-12T23:53:03.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:02 smithi136 bash[76371]: cluster 2024-09-12T23:53:01.681367+0000 mon.a (mon.0) 1493 : cluster [DBG] osdmap e879: 8 total, 8 up, 8 in 2024-09-12T23:53:03.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:02 smithi181 bash[67587]: cluster 2024-09-12T23:53:01.661840+0000 mon.a (mon.0) 1491 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:53:03.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:02 smithi181 bash[67587]: cluster 2024-09-12T23:53:01.661885+0000 mon.a (mon.0) 1492 : cluster [INF] Cluster is now healthy 2024-09-12T23:53:03.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:02 smithi181 bash[67587]: cluster 2024-09-12T23:53:01.681367+0000 mon.a (mon.0) 1493 : cluster [DBG] osdmap e879: 8 total, 8 up, 8 in 2024-09-12T23:53:04.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:03 smithi136 bash[75164]: cluster 2024-09-12T23:53:02.691175+0000 mon.a (mon.0) 1494 : cluster [DBG] osdmap e880: 8 total, 8 up, 8 in 2024-09-12T23:53:04.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:03 smithi136 bash[75164]: audit 2024-09-12T23:53:02.695488+0000 mon.c (mon.2) 22 : audit [INF] from='client.? 172.21.15.136:0/3260839818' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:04.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:03 smithi136 bash[75164]: audit 2024-09-12T23:53:02.697422+0000 mon.a (mon.0) 1495 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:04.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:03 smithi136 bash[75164]: cluster 2024-09-12T23:53:02.917570+0000 mgr.y (mgr.24661) 478 : cluster [DBG] pgmap v919: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:04.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:03 smithi136 bash[76371]: cluster 2024-09-12T23:53:02.691175+0000 mon.a (mon.0) 1494 : cluster [DBG] osdmap e880: 8 total, 8 up, 8 in 2024-09-12T23:53:04.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:03 smithi136 bash[76371]: audit 2024-09-12T23:53:02.695488+0000 mon.c (mon.2) 22 : audit [INF] from='client.? 172.21.15.136:0/3260839818' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:04.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:03 smithi136 bash[76371]: audit 2024-09-12T23:53:02.697422+0000 mon.a (mon.0) 1495 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:04.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:03 smithi136 bash[76371]: cluster 2024-09-12T23:53:02.917570+0000 mgr.y (mgr.24661) 478 : cluster [DBG] pgmap v919: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:04.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:03 smithi181 bash[67587]: cluster 2024-09-12T23:53:02.691175+0000 mon.a (mon.0) 1494 : cluster [DBG] osdmap e880: 8 total, 8 up, 8 in 2024-09-12T23:53:04.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:03 smithi181 bash[67587]: audit 2024-09-12T23:53:02.695488+0000 mon.c (mon.2) 22 : audit [INF] from='client.? 172.21.15.136:0/3260839818' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:04.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:03 smithi181 bash[67587]: audit 2024-09-12T23:53:02.697422+0000 mon.a (mon.0) 1495 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:04.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:03 smithi181 bash[67587]: cluster 2024-09-12T23:53:02.917570+0000 mgr.y (mgr.24661) 478 : cluster [DBG] pgmap v919: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:04.709 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestCls2PCQueue.Abort (3026 ms) 2024-09-12T23:53:04.709 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestCls2PCQueue.ReserveError 2024-09-12T23:53:05.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:04 smithi136 bash[75164]: cluster 2024-09-12T23:53:03.683956+0000 mon.a (mon.0) 1496 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:53:05.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:04 smithi136 bash[75164]: audit 2024-09-12T23:53:03.707203+0000 mon.a (mon.0) 1497 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:53:05.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:04 smithi136 bash[75164]: cluster 2024-09-12T23:53:03.707671+0000 mon.a (mon.0) 1498 : cluster [DBG] osdmap e881: 8 total, 8 up, 8 in 2024-09-12T23:53:05.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:04 smithi136 bash[76371]: cluster 2024-09-12T23:53:03.683956+0000 mon.a (mon.0) 1496 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:53:05.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:04 smithi136 bash[76371]: audit 2024-09-12T23:53:03.707203+0000 mon.a (mon.0) 1497 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:53:05.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:04 smithi136 bash[76371]: cluster 2024-09-12T23:53:03.707671+0000 mon.a (mon.0) 1498 : cluster [DBG] osdmap e881: 8 total, 8 up, 8 in 2024-09-12T23:53:05.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:04 smithi181 bash[67587]: cluster 2024-09-12T23:53:03.683956+0000 mon.a (mon.0) 1496 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:53:05.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:04 smithi181 bash[67587]: audit 2024-09-12T23:53:03.707203+0000 mon.a (mon.0) 1497 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:53:05.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:04 smithi181 bash[67587]: cluster 2024-09-12T23:53:03.707671+0000 mon.a (mon.0) 1498 : cluster [DBG] osdmap e881: 8 total, 8 up, 8 in 2024-09-12T23:53:06.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:05 smithi136 bash[75164]: cluster 2024-09-12T23:53:04.706681+0000 mon.a (mon.0) 1499 : cluster [DBG] osdmap e882: 8 total, 8 up, 8 in 2024-09-12T23:53:06.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:05 smithi136 bash[75164]: cluster 2024-09-12T23:53:04.918148+0000 mgr.y (mgr.24661) 479 : cluster [DBG] pgmap v922: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:06.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:05 smithi136 bash[76371]: cluster 2024-09-12T23:53:04.706681+0000 mon.a (mon.0) 1499 : cluster [DBG] osdmap e882: 8 total, 8 up, 8 in 2024-09-12T23:53:06.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:05 smithi136 bash[76371]: cluster 2024-09-12T23:53:04.918148+0000 mgr.y (mgr.24661) 479 : cluster [DBG] pgmap v922: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:06.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:05 smithi181 bash[67587]: cluster 2024-09-12T23:53:04.706681+0000 mon.a (mon.0) 1499 : cluster [DBG] osdmap e882: 8 total, 8 up, 8 in 2024-09-12T23:53:06.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:05 smithi181 bash[67587]: cluster 2024-09-12T23:53:04.918148+0000 mgr.y (mgr.24661) 479 : cluster [DBG] pgmap v922: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:07.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:06 smithi136 bash[76371]: cluster 2024-09-12T23:53:05.710945+0000 mon.a (mon.0) 1500 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:53:07.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:06 smithi136 bash[76371]: cluster 2024-09-12T23:53:05.711022+0000 mon.a (mon.0) 1501 : cluster [INF] Cluster is now healthy 2024-09-12T23:53:07.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:06 smithi136 bash[76371]: cluster 2024-09-12T23:53:05.735014+0000 mon.a (mon.0) 1502 : cluster [DBG] osdmap e883: 8 total, 8 up, 8 in 2024-09-12T23:53:07.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:06 smithi136 bash[76371]: audit 2024-09-12T23:53:05.738101+0000 mon.a (mon.0) 1503 : audit [INF] from='client.? 172.21.15.136:0/2013642231' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:07.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:06 smithi136 bash[75164]: cluster 2024-09-12T23:53:05.710945+0000 mon.a (mon.0) 1500 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:53:07.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:06 smithi136 bash[75164]: cluster 2024-09-12T23:53:05.711022+0000 mon.a (mon.0) 1501 : cluster [INF] Cluster is now healthy 2024-09-12T23:53:07.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:06 smithi136 bash[75164]: cluster 2024-09-12T23:53:05.735014+0000 mon.a (mon.0) 1502 : cluster [DBG] osdmap e883: 8 total, 8 up, 8 in 2024-09-12T23:53:07.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:06 smithi136 bash[75164]: audit 2024-09-12T23:53:05.738101+0000 mon.a (mon.0) 1503 : audit [INF] from='client.? 172.21.15.136:0/2013642231' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:07.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:06 smithi181 bash[67587]: cluster 2024-09-12T23:53:05.710945+0000 mon.a (mon.0) 1500 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:53:07.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:06 smithi181 bash[67587]: cluster 2024-09-12T23:53:05.711022+0000 mon.a (mon.0) 1501 : cluster [INF] Cluster is now healthy 2024-09-12T23:53:07.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:06 smithi181 bash[67587]: cluster 2024-09-12T23:53:05.735014+0000 mon.a (mon.0) 1502 : cluster [DBG] osdmap e883: 8 total, 8 up, 8 in 2024-09-12T23:53:07.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:06 smithi181 bash[67587]: audit 2024-09-12T23:53:05.738101+0000 mon.a (mon.0) 1503 : audit [INF] from='client.? 172.21.15.136:0/2013642231' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:07.764 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestCls2PCQueue.ReserveError (3056 ms) 2024-09-12T23:53:07.764 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestCls2PCQueue.CommitError 2024-09-12T23:53:08.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:07 smithi136 bash[75164]: audit 2024-09-12T23:53:06.743554+0000 mon.a (mon.0) 1504 : audit [INF] from='client.? 172.21.15.136:0/2013642231' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:53:08.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:07 smithi136 bash[75164]: cluster 2024-09-12T23:53:06.743965+0000 mon.a (mon.0) 1505 : cluster [DBG] osdmap e884: 8 total, 8 up, 8 in 2024-09-12T23:53:08.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:07 smithi136 bash[75164]: cluster 2024-09-12T23:53:06.918907+0000 mgr.y (mgr.24661) 480 : cluster [DBG] pgmap v925: 137 pgs: 14 creating+peering, 18 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:08.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:07 smithi136 bash[76371]: audit 2024-09-12T23:53:06.743554+0000 mon.a (mon.0) 1504 : audit [INF] from='client.? 172.21.15.136:0/2013642231' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:53:08.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:07 smithi136 bash[76371]: cluster 2024-09-12T23:53:06.743965+0000 mon.a (mon.0) 1505 : cluster [DBG] osdmap e884: 8 total, 8 up, 8 in 2024-09-12T23:53:08.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:07 smithi136 bash[76371]: cluster 2024-09-12T23:53:06.918907+0000 mgr.y (mgr.24661) 480 : cluster [DBG] pgmap v925: 137 pgs: 14 creating+peering, 18 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:08.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:07 smithi181 bash[67587]: audit 2024-09-12T23:53:06.743554+0000 mon.a (mon.0) 1504 : audit [INF] from='client.? 172.21.15.136:0/2013642231' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:53:08.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:07 smithi181 bash[67587]: cluster 2024-09-12T23:53:06.743965+0000 mon.a (mon.0) 1505 : cluster [DBG] osdmap e884: 8 total, 8 up, 8 in 2024-09-12T23:53:08.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:07 smithi181 bash[67587]: cluster 2024-09-12T23:53:06.918907+0000 mgr.y (mgr.24661) 480 : cluster [DBG] pgmap v925: 137 pgs: 14 creating+peering, 18 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:09.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:08 smithi136 bash[75164]: cluster 2024-09-12T23:53:07.760086+0000 mon.a (mon.0) 1506 : cluster [DBG] osdmap e885: 8 total, 8 up, 8 in 2024-09-12T23:53:09.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:08 smithi136 bash[76371]: cluster 2024-09-12T23:53:07.760086+0000 mon.a (mon.0) 1506 : cluster [DBG] osdmap e885: 8 total, 8 up, 8 in 2024-09-12T23:53:09.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:08 smithi181 bash[67587]: cluster 2024-09-12T23:53:07.760086+0000 mon.a (mon.0) 1506 : cluster [DBG] osdmap e885: 8 total, 8 up, 8 in 2024-09-12T23:53:09.786 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:53:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:53:09] "GET /metrics HTTP/1.1" 200 36449 "" "Prometheus/2.43.0" 2024-09-12T23:53:10.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:09 smithi181 bash[67587]: cluster 2024-09-12T23:53:08.767054+0000 mon.a (mon.0) 1507 : cluster [DBG] osdmap e886: 8 total, 8 up, 8 in 2024-09-12T23:53:10.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:09 smithi181 bash[67587]: audit 2024-09-12T23:53:08.779051+0000 mon.a (mon.0) 1508 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:53:10.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:09 smithi181 bash[67587]: audit 2024-09-12T23:53:08.779747+0000 mon.c (mon.2) 23 : audit [INF] from='client.? 172.21.15.136:0/3339899769' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:10.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:09 smithi181 bash[67587]: audit 2024-09-12T23:53:08.780746+0000 mon.a (mon.0) 1509 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:53:10.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:09 smithi181 bash[67587]: audit 2024-09-12T23:53:08.781465+0000 mon.a (mon.0) 1510 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:10.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:09 smithi181 bash[67587]: cluster 2024-09-12T23:53:08.919531+0000 mgr.y (mgr.24661) 481 : cluster [DBG] pgmap v928: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:10.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:09 smithi136 bash[76371]: cluster 2024-09-12T23:53:08.767054+0000 mon.a (mon.0) 1507 : cluster [DBG] osdmap e886: 8 total, 8 up, 8 in 2024-09-12T23:53:10.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:09 smithi136 bash[76371]: audit 2024-09-12T23:53:08.779051+0000 mon.a (mon.0) 1508 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:53:10.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:09 smithi136 bash[76371]: audit 2024-09-12T23:53:08.779747+0000 mon.c (mon.2) 23 : audit [INF] from='client.? 172.21.15.136:0/3339899769' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:10.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:09 smithi136 bash[76371]: audit 2024-09-12T23:53:08.780746+0000 mon.a (mon.0) 1509 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:53:10.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:09 smithi136 bash[76371]: audit 2024-09-12T23:53:08.781465+0000 mon.a (mon.0) 1510 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:10.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:09 smithi136 bash[76371]: cluster 2024-09-12T23:53:08.919531+0000 mgr.y (mgr.24661) 481 : cluster [DBG] pgmap v928: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:10.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:09 smithi136 bash[75164]: cluster 2024-09-12T23:53:08.767054+0000 mon.a (mon.0) 1507 : cluster [DBG] osdmap e886: 8 total, 8 up, 8 in 2024-09-12T23:53:10.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:09 smithi136 bash[75164]: audit 2024-09-12T23:53:08.779051+0000 mon.a (mon.0) 1508 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:53:10.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:09 smithi136 bash[75164]: audit 2024-09-12T23:53:08.779747+0000 mon.c (mon.2) 23 : audit [INF] from='client.? 172.21.15.136:0/3339899769' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:10.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:09 smithi136 bash[75164]: audit 2024-09-12T23:53:08.780746+0000 mon.a (mon.0) 1509 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:53:10.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:09 smithi136 bash[75164]: audit 2024-09-12T23:53:08.781465+0000 mon.a (mon.0) 1510 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:10.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:09 smithi136 bash[75164]: cluster 2024-09-12T23:53:08.919531+0000 mgr.y (mgr.24661) 481 : cluster [DBG] pgmap v928: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:10.776 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestCls2PCQueue.CommitError (3012 ms) 2024-09-12T23:53:10.777 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestCls2PCQueue.AbortError 2024-09-12T23:53:11.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:10 smithi181 bash[67587]: audit 2024-09-12T23:53:09.773346+0000 mon.a (mon.0) 1511 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:53:11.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:10 smithi181 bash[67587]: cluster 2024-09-12T23:53:09.773712+0000 mon.a (mon.0) 1512 : cluster [DBG] osdmap e887: 8 total, 8 up, 8 in 2024-09-12T23:53:11.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:10 smithi181 bash[67587]: cluster 2024-09-12T23:53:09.777540+0000 mon.a (mon.0) 1513 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:53:11.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:10 smithi181 bash[67587]: cluster 2024-09-12T23:53:10.774735+0000 mon.a (mon.0) 1514 : cluster [DBG] osdmap e888: 8 total, 8 up, 8 in 2024-09-12T23:53:11.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:10 smithi136 bash[76371]: audit 2024-09-12T23:53:09.773346+0000 mon.a (mon.0) 1511 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:53:11.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:10 smithi136 bash[76371]: cluster 2024-09-12T23:53:09.773712+0000 mon.a (mon.0) 1512 : cluster [DBG] osdmap e887: 8 total, 8 up, 8 in 2024-09-12T23:53:11.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:10 smithi136 bash[76371]: cluster 2024-09-12T23:53:09.777540+0000 mon.a (mon.0) 1513 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:53:11.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:10 smithi136 bash[76371]: cluster 2024-09-12T23:53:10.774735+0000 mon.a (mon.0) 1514 : cluster [DBG] osdmap e888: 8 total, 8 up, 8 in 2024-09-12T23:53:11.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:10 smithi136 bash[75164]: audit 2024-09-12T23:53:09.773346+0000 mon.a (mon.0) 1511 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:53:11.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:10 smithi136 bash[75164]: cluster 2024-09-12T23:53:09.773712+0000 mon.a (mon.0) 1512 : cluster [DBG] osdmap e887: 8 total, 8 up, 8 in 2024-09-12T23:53:11.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:10 smithi136 bash[75164]: cluster 2024-09-12T23:53:09.777540+0000 mon.a (mon.0) 1513 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:53:11.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:10 smithi136 bash[75164]: cluster 2024-09-12T23:53:10.774735+0000 mon.a (mon.0) 1514 : cluster [DBG] osdmap e888: 8 total, 8 up, 8 in 2024-09-12T23:53:12.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:11 smithi181 bash[67587]: cluster 2024-09-12T23:53:10.920213+0000 mgr.y (mgr.24661) 482 : cluster [DBG] pgmap v931: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:12.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:11 smithi136 bash[75164]: cluster 2024-09-12T23:53:10.920213+0000 mgr.y (mgr.24661) 482 : cluster [DBG] pgmap v931: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:12.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:11 smithi136 bash[76371]: cluster 2024-09-12T23:53:10.920213+0000 mgr.y (mgr.24661) 482 : cluster [DBG] pgmap v931: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:12.597 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:53:12 smithi181 bash[95505]: ts=2024-09-12T23:53:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:53:13.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:12 smithi181 bash[67587]: cluster 2024-09-12T23:53:11.790758+0000 mon.a (mon.0) 1515 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:53:13.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:12 smithi181 bash[67587]: cluster 2024-09-12T23:53:11.790826+0000 mon.a (mon.0) 1516 : cluster [INF] Cluster is now healthy 2024-09-12T23:53:13.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:12 smithi181 bash[67587]: cluster 2024-09-12T23:53:11.812162+0000 mon.a (mon.0) 1517 : cluster [DBG] osdmap e889: 8 total, 8 up, 8 in 2024-09-12T23:53:13.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:12 smithi181 bash[67587]: audit 2024-09-12T23:53:11.813792+0000 mon.c (mon.2) 24 : audit [INF] from='client.? 172.21.15.136:0/1658261759' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-9","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:13.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:12 smithi181 bash[67587]: audit 2024-09-12T23:53:11.817249+0000 mon.a (mon.0) 1518 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-9","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:13.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:12 smithi136 bash[75164]: cluster 2024-09-12T23:53:11.790758+0000 mon.a (mon.0) 1515 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:53:13.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:12 smithi136 bash[75164]: cluster 2024-09-12T23:53:11.790826+0000 mon.a (mon.0) 1516 : cluster [INF] Cluster is now healthy 2024-09-12T23:53:13.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:12 smithi136 bash[75164]: cluster 2024-09-12T23:53:11.812162+0000 mon.a (mon.0) 1517 : cluster [DBG] osdmap e889: 8 total, 8 up, 8 in 2024-09-12T23:53:13.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:12 smithi136 bash[75164]: audit 2024-09-12T23:53:11.813792+0000 mon.c (mon.2) 24 : audit [INF] from='client.? 172.21.15.136:0/1658261759' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-9","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:13.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:12 smithi136 bash[75164]: audit 2024-09-12T23:53:11.817249+0000 mon.a (mon.0) 1518 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-9","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:13.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:12 smithi136 bash[76371]: cluster 2024-09-12T23:53:11.790758+0000 mon.a (mon.0) 1515 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:53:13.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:12 smithi136 bash[76371]: cluster 2024-09-12T23:53:11.790826+0000 mon.a (mon.0) 1516 : cluster [INF] Cluster is now healthy 2024-09-12T23:53:13.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:12 smithi136 bash[76371]: cluster 2024-09-12T23:53:11.812162+0000 mon.a (mon.0) 1517 : cluster [DBG] osdmap e889: 8 total, 8 up, 8 in 2024-09-12T23:53:13.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:12 smithi136 bash[76371]: audit 2024-09-12T23:53:11.813792+0000 mon.c (mon.2) 24 : audit [INF] from='client.? 172.21.15.136:0/1658261759' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-9","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:13.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:12 smithi136 bash[76371]: audit 2024-09-12T23:53:11.817249+0000 mon.a (mon.0) 1518 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-9","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:13.846 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestCls2PCQueue.AbortError (3070 ms) 2024-09-12T23:53:13.846 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestCls2PCQueue.MultiReserve 2024-09-12T23:53:14.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:13 smithi181 bash[67587]: audit 2024-09-12T23:53:12.814734+0000 mon.a (mon.0) 1519 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-9","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:53:14.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:13 smithi181 bash[67587]: cluster 2024-09-12T23:53:12.815418+0000 mon.a (mon.0) 1520 : cluster [DBG] osdmap e890: 8 total, 8 up, 8 in 2024-09-12T23:53:14.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:13 smithi181 bash[67587]: cluster 2024-09-12T23:53:12.920697+0000 mgr.y (mgr.24661) 483 : cluster [DBG] pgmap v934: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:14.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:13 smithi136 bash[75164]: audit 2024-09-12T23:53:12.814734+0000 mon.a (mon.0) 1519 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-9","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:53:14.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:13 smithi136 bash[75164]: cluster 2024-09-12T23:53:12.815418+0000 mon.a (mon.0) 1520 : cluster [DBG] osdmap e890: 8 total, 8 up, 8 in 2024-09-12T23:53:14.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:13 smithi136 bash[75164]: cluster 2024-09-12T23:53:12.920697+0000 mgr.y (mgr.24661) 483 : cluster [DBG] pgmap v934: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:14.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:13 smithi136 bash[76371]: audit 2024-09-12T23:53:12.814734+0000 mon.a (mon.0) 1519 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-9","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:53:14.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:13 smithi136 bash[76371]: cluster 2024-09-12T23:53:12.815418+0000 mon.a (mon.0) 1520 : cluster [DBG] osdmap e890: 8 total, 8 up, 8 in 2024-09-12T23:53:14.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:13 smithi136 bash[76371]: cluster 2024-09-12T23:53:12.920697+0000 mgr.y (mgr.24661) 483 : cluster [DBG] pgmap v934: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:15.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:14 smithi136 bash[75164]: cluster 2024-09-12T23:53:13.845811+0000 mon.a (mon.0) 1521 : cluster [DBG] osdmap e891: 8 total, 8 up, 8 in 2024-09-12T23:53:15.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:14 smithi136 bash[76371]: cluster 2024-09-12T23:53:13.845811+0000 mon.a (mon.0) 1521 : cluster [DBG] osdmap e891: 8 total, 8 up, 8 in 2024-09-12T23:53:15.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:14 smithi181 bash[67587]: cluster 2024-09-12T23:53:13.845811+0000 mon.a (mon.0) 1521 : cluster [DBG] osdmap e891: 8 total, 8 up, 8 in 2024-09-12T23:53:16.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:15 smithi136 bash[75164]: cluster 2024-09-12T23:53:14.849344+0000 mon.a (mon.0) 1522 : cluster [DBG] osdmap e892: 8 total, 8 up, 8 in 2024-09-12T23:53:16.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:15 smithi136 bash[75164]: audit 2024-09-12T23:53:14.855959+0000 mon.a (mon.0) 1523 : audit [INF] from='client.? 172.21.15.136:0/3179352571' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:16.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:15 smithi136 bash[75164]: cluster 2024-09-12T23:53:14.921326+0000 mgr.y (mgr.24661) 484 : cluster [DBG] pgmap v937: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:16.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:15 smithi136 bash[75164]: audit 2024-09-12T23:53:15.850205+0000 mon.a (mon.0) 1524 : audit [INF] from='client.? 172.21.15.136:0/3179352571' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:53:16.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:15 smithi136 bash[75164]: cluster 2024-09-12T23:53:15.850418+0000 mon.a (mon.0) 1525 : cluster [DBG] osdmap e893: 8 total, 8 up, 8 in 2024-09-12T23:53:16.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:15 smithi136 bash[76371]: cluster 2024-09-12T23:53:14.849344+0000 mon.a (mon.0) 1522 : cluster [DBG] osdmap e892: 8 total, 8 up, 8 in 2024-09-12T23:53:16.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:15 smithi136 bash[76371]: audit 2024-09-12T23:53:14.855959+0000 mon.a (mon.0) 1523 : audit [INF] from='client.? 172.21.15.136:0/3179352571' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:16.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:15 smithi136 bash[76371]: cluster 2024-09-12T23:53:14.921326+0000 mgr.y (mgr.24661) 484 : cluster [DBG] pgmap v937: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:16.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:15 smithi136 bash[76371]: audit 2024-09-12T23:53:15.850205+0000 mon.a (mon.0) 1524 : audit [INF] from='client.? 172.21.15.136:0/3179352571' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:53:16.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:15 smithi136 bash[76371]: cluster 2024-09-12T23:53:15.850418+0000 mon.a (mon.0) 1525 : cluster [DBG] osdmap e893: 8 total, 8 up, 8 in 2024-09-12T23:53:16.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:15 smithi181 bash[67587]: cluster 2024-09-12T23:53:14.849344+0000 mon.a (mon.0) 1522 : cluster [DBG] osdmap e892: 8 total, 8 up, 8 in 2024-09-12T23:53:16.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:15 smithi181 bash[67587]: audit 2024-09-12T23:53:14.855959+0000 mon.a (mon.0) 1523 : audit [INF] from='client.? 172.21.15.136:0/3179352571' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:16.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:15 smithi181 bash[67587]: cluster 2024-09-12T23:53:14.921326+0000 mgr.y (mgr.24661) 484 : cluster [DBG] pgmap v937: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:16.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:15 smithi181 bash[67587]: audit 2024-09-12T23:53:15.850205+0000 mon.a (mon.0) 1524 : audit [INF] from='client.? 172.21.15.136:0/3179352571' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:53:16.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:15 smithi181 bash[67587]: cluster 2024-09-12T23:53:15.850418+0000 mon.a (mon.0) 1525 : cluster [DBG] osdmap e893: 8 total, 8 up, 8 in 2024-09-12T23:53:16.876 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestCls2PCQueue.MultiReserve (3030 ms) 2024-09-12T23:53:16.877 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestCls2PCQueue.MultiCommit 2024-09-12T23:53:17.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:16 smithi136 bash[75164]: cluster 2024-09-12T23:53:15.860825+0000 mon.a (mon.0) 1526 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:53:17.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:16 smithi136 bash[76371]: cluster 2024-09-12T23:53:15.860825+0000 mon.a (mon.0) 1526 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:53:17.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:16 smithi181 bash[67587]: cluster 2024-09-12T23:53:15.860825+0000 mon.a (mon.0) 1526 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:53:18.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:17 smithi136 bash[75164]: cluster 2024-09-12T23:53:16.874416+0000 mon.a (mon.0) 1527 : cluster [DBG] osdmap e894: 8 total, 8 up, 8 in 2024-09-12T23:53:18.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:17 smithi136 bash[75164]: cluster 2024-09-12T23:53:16.921749+0000 mgr.y (mgr.24661) 485 : cluster [DBG] pgmap v940: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:18.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:17 smithi136 bash[76371]: cluster 2024-09-12T23:53:16.874416+0000 mon.a (mon.0) 1527 : cluster [DBG] osdmap e894: 8 total, 8 up, 8 in 2024-09-12T23:53:18.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:17 smithi136 bash[76371]: cluster 2024-09-12T23:53:16.921749+0000 mgr.y (mgr.24661) 485 : cluster [DBG] pgmap v940: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:18.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:17 smithi181 bash[67587]: cluster 2024-09-12T23:53:16.874416+0000 mon.a (mon.0) 1527 : cluster [DBG] osdmap e894: 8 total, 8 up, 8 in 2024-09-12T23:53:18.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:17 smithi181 bash[67587]: cluster 2024-09-12T23:53:16.921749+0000 mgr.y (mgr.24661) 485 : cluster [DBG] pgmap v940: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:19.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:18 smithi136 bash[75164]: cluster 2024-09-12T23:53:17.880466+0000 mon.a (mon.0) 1528 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:53:19.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:18 smithi136 bash[75164]: cluster 2024-09-12T23:53:17.880534+0000 mon.a (mon.0) 1529 : cluster [INF] Cluster is now healthy 2024-09-12T23:53:19.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:18 smithi136 bash[75164]: cluster 2024-09-12T23:53:17.906281+0000 mon.a (mon.0) 1530 : cluster [DBG] osdmap e895: 8 total, 8 up, 8 in 2024-09-12T23:53:19.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:18 smithi136 bash[75164]: audit 2024-09-12T23:53:17.910308+0000 mon.c (mon.2) 25 : audit [INF] from='client.? 172.21.15.136:0/1618142198' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:19.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:18 smithi136 bash[75164]: audit 2024-09-12T23:53:17.914752+0000 mon.a (mon.0) 1531 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:19.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:18 smithi136 bash[76371]: cluster 2024-09-12T23:53:17.880466+0000 mon.a (mon.0) 1528 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:53:19.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:18 smithi136 bash[76371]: cluster 2024-09-12T23:53:17.880534+0000 mon.a (mon.0) 1529 : cluster [INF] Cluster is now healthy 2024-09-12T23:53:19.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:18 smithi136 bash[76371]: cluster 2024-09-12T23:53:17.906281+0000 mon.a (mon.0) 1530 : cluster [DBG] osdmap e895: 8 total, 8 up, 8 in 2024-09-12T23:53:19.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:18 smithi136 bash[76371]: audit 2024-09-12T23:53:17.910308+0000 mon.c (mon.2) 25 : audit [INF] from='client.? 172.21.15.136:0/1618142198' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:19.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:18 smithi136 bash[76371]: audit 2024-09-12T23:53:17.914752+0000 mon.a (mon.0) 1531 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:19.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:18 smithi181 bash[67587]: cluster 2024-09-12T23:53:17.880466+0000 mon.a (mon.0) 1528 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:53:19.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:18 smithi181 bash[67587]: cluster 2024-09-12T23:53:17.880534+0000 mon.a (mon.0) 1529 : cluster [INF] Cluster is now healthy 2024-09-12T23:53:19.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:18 smithi181 bash[67587]: cluster 2024-09-12T23:53:17.906281+0000 mon.a (mon.0) 1530 : cluster [DBG] osdmap e895: 8 total, 8 up, 8 in 2024-09-12T23:53:19.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:18 smithi181 bash[67587]: audit 2024-09-12T23:53:17.910308+0000 mon.c (mon.2) 25 : audit [INF] from='client.? 172.21.15.136:0/1618142198' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:19.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:18 smithi181 bash[67587]: audit 2024-09-12T23:53:17.914752+0000 mon.a (mon.0) 1531 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:19.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:53:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:53:19] "GET /metrics HTTP/1.1" 200 36449 "" "Prometheus/2.43.0" 2024-09-12T23:53:19.930 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestCls2PCQueue.MultiCommit (3053 ms) 2024-09-12T23:53:19.930 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestCls2PCQueue.MultiAbort 2024-09-12T23:53:20.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:19 smithi136 bash[75164]: audit 2024-09-12T23:53:18.917475+0000 mon.a (mon.0) 1532 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:53:20.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:19 smithi136 bash[75164]: cluster 2024-09-12T23:53:18.917891+0000 mon.a (mon.0) 1533 : cluster [DBG] osdmap e896: 8 total, 8 up, 8 in 2024-09-12T23:53:20.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:19 smithi136 bash[75164]: cluster 2024-09-12T23:53:18.922427+0000 mgr.y (mgr.24661) 486 : cluster [DBG] pgmap v943: 137 pgs: 7 creating+peering, 25 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:20.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:19 smithi136 bash[76371]: audit 2024-09-12T23:53:18.917475+0000 mon.a (mon.0) 1532 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:53:20.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:19 smithi136 bash[76371]: cluster 2024-09-12T23:53:18.917891+0000 mon.a (mon.0) 1533 : cluster [DBG] osdmap e896: 8 total, 8 up, 8 in 2024-09-12T23:53:20.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:19 smithi136 bash[76371]: cluster 2024-09-12T23:53:18.922427+0000 mgr.y (mgr.24661) 486 : cluster [DBG] pgmap v943: 137 pgs: 7 creating+peering, 25 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:20.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:19 smithi181 bash[67587]: audit 2024-09-12T23:53:18.917475+0000 mon.a (mon.0) 1532 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:53:20.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:19 smithi181 bash[67587]: cluster 2024-09-12T23:53:18.917891+0000 mon.a (mon.0) 1533 : cluster [DBG] osdmap e896: 8 total, 8 up, 8 in 2024-09-12T23:53:20.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:19 smithi181 bash[67587]: cluster 2024-09-12T23:53:18.922427+0000 mgr.y (mgr.24661) 486 : cluster [DBG] pgmap v943: 137 pgs: 7 creating+peering, 25 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:21.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:20 smithi136 bash[75164]: cluster 2024-09-12T23:53:19.927635+0000 mon.a (mon.0) 1534 : cluster [DBG] osdmap e897: 8 total, 8 up, 8 in 2024-09-12T23:53:21.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:20 smithi136 bash[76371]: cluster 2024-09-12T23:53:19.927635+0000 mon.a (mon.0) 1534 : cluster [DBG] osdmap e897: 8 total, 8 up, 8 in 2024-09-12T23:53:21.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:20 smithi181 bash[67587]: cluster 2024-09-12T23:53:19.927635+0000 mon.a (mon.0) 1534 : cluster [DBG] osdmap e897: 8 total, 8 up, 8 in 2024-09-12T23:53:22.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:21 smithi136 bash[75164]: cluster 2024-09-12T23:53:20.923327+0000 mgr.y (mgr.24661) 487 : cluster [DBG] pgmap v945: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:22.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:21 smithi136 bash[75164]: cluster 2024-09-12T23:53:20.936348+0000 mon.a (mon.0) 1535 : cluster [DBG] osdmap e898: 8 total, 8 up, 8 in 2024-09-12T23:53:22.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:21 smithi136 bash[75164]: audit 2024-09-12T23:53:20.938922+0000 mon.c (mon.2) 26 : audit [INF] from='client.? 172.21.15.136:0/4234406233' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-12","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:22.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:21 smithi136 bash[75164]: audit 2024-09-12T23:53:20.939685+0000 mon.a (mon.0) 1536 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-12","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:22.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:21 smithi136 bash[75164]: audit 2024-09-12T23:53:21.934274+0000 mon.a (mon.0) 1537 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-12","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:53:22.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:21 smithi136 bash[75164]: cluster 2024-09-12T23:53:21.934832+0000 mon.a (mon.0) 1538 : cluster [DBG] osdmap e899: 8 total, 8 up, 8 in 2024-09-12T23:53:22.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:21 smithi136 bash[76371]: cluster 2024-09-12T23:53:20.923327+0000 mgr.y (mgr.24661) 487 : cluster [DBG] pgmap v945: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:22.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:21 smithi136 bash[76371]: cluster 2024-09-12T23:53:20.936348+0000 mon.a (mon.0) 1535 : cluster [DBG] osdmap e898: 8 total, 8 up, 8 in 2024-09-12T23:53:22.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:21 smithi136 bash[76371]: audit 2024-09-12T23:53:20.938922+0000 mon.c (mon.2) 26 : audit [INF] from='client.? 172.21.15.136:0/4234406233' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-12","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:22.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:21 smithi136 bash[76371]: audit 2024-09-12T23:53:20.939685+0000 mon.a (mon.0) 1536 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-12","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:22.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:21 smithi136 bash[76371]: audit 2024-09-12T23:53:21.934274+0000 mon.a (mon.0) 1537 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-12","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:53:22.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:21 smithi136 bash[76371]: cluster 2024-09-12T23:53:21.934832+0000 mon.a (mon.0) 1538 : cluster [DBG] osdmap e899: 8 total, 8 up, 8 in 2024-09-12T23:53:22.313 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:21 smithi181 bash[67587]: cluster 2024-09-12T23:53:20.923327+0000 mgr.y (mgr.24661) 487 : cluster [DBG] pgmap v945: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:22.313 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:21 smithi181 bash[67587]: cluster 2024-09-12T23:53:20.936348+0000 mon.a (mon.0) 1535 : cluster [DBG] osdmap e898: 8 total, 8 up, 8 in 2024-09-12T23:53:22.313 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:21 smithi181 bash[67587]: audit 2024-09-12T23:53:20.938922+0000 mon.c (mon.2) 26 : audit [INF] from='client.? 172.21.15.136:0/4234406233' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-12","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:22.313 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:21 smithi181 bash[67587]: audit 2024-09-12T23:53:20.939685+0000 mon.a (mon.0) 1536 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-12","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:22.313 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:21 smithi181 bash[67587]: audit 2024-09-12T23:53:21.934274+0000 mon.a (mon.0) 1537 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-12","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:53:22.314 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:21 smithi181 bash[67587]: cluster 2024-09-12T23:53:21.934832+0000 mon.a (mon.0) 1538 : cluster [DBG] osdmap e899: 8 total, 8 up, 8 in 2024-09-12T23:53:22.597 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:53:22 smithi181 bash[95505]: ts=2024-09-12T23:53:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:53:22.980 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestCls2PCQueue.MultiAbort (3050 ms) 2024-09-12T23:53:22.980 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestCls2PCQueue.ReserveCommit 2024-09-12T23:53:23.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:22 smithi136 bash[75164]: cluster 2024-09-12T23:53:22.924195+0000 mgr.y (mgr.24661) 488 : cluster [DBG] pgmap v948: 137 pgs: 8 creating+peering, 21 unknown, 108 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:23.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:22 smithi136 bash[76371]: cluster 2024-09-12T23:53:22.924195+0000 mgr.y (mgr.24661) 488 : cluster [DBG] pgmap v948: 137 pgs: 8 creating+peering, 21 unknown, 108 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:23.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:22 smithi181 bash[67587]: cluster 2024-09-12T23:53:22.924195+0000 mgr.y (mgr.24661) 488 : cluster [DBG] pgmap v948: 137 pgs: 8 creating+peering, 21 unknown, 108 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:24.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:23 smithi136 bash[76371]: cluster 2024-09-12T23:53:22.979612+0000 mon.a (mon.0) 1539 : cluster [DBG] osdmap e900: 8 total, 8 up, 8 in 2024-09-12T23:53:24.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:23 smithi136 bash[76371]: audit 2024-09-12T23:53:23.757760+0000 mon.a (mon.0) 1540 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:53:24.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:23 smithi136 bash[75164]: cluster 2024-09-12T23:53:22.979612+0000 mon.a (mon.0) 1539 : cluster [DBG] osdmap e900: 8 total, 8 up, 8 in 2024-09-12T23:53:24.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:23 smithi136 bash[75164]: audit 2024-09-12T23:53:23.757760+0000 mon.a (mon.0) 1540 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:53:24.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:23 smithi181 bash[67587]: cluster 2024-09-12T23:53:22.979612+0000 mon.a (mon.0) 1539 : cluster [DBG] osdmap e900: 8 total, 8 up, 8 in 2024-09-12T23:53:24.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:23 smithi181 bash[67587]: audit 2024-09-12T23:53:23.757760+0000 mon.a (mon.0) 1540 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:53:25.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:25 smithi136 bash[75164]: cluster 2024-09-12T23:53:23.982010+0000 mon.a (mon.0) 1541 : cluster [DBG] osdmap e901: 8 total, 8 up, 8 in 2024-09-12T23:53:25.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:25 smithi136 bash[75164]: audit 2024-09-12T23:53:23.993500+0000 mon.c (mon.2) 27 : audit [INF] from='client.? 172.21.15.136:0/1510532693' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-13","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:25.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:25 smithi136 bash[75164]: audit 2024-09-12T23:53:23.994444+0000 mon.a (mon.0) 1542 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-13","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:25.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:25 smithi136 bash[75164]: cluster 2024-09-12T23:53:24.925012+0000 mgr.y (mgr.24661) 489 : cluster [DBG] pgmap v951: 137 pgs: 2 creating+peering, 30 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:25.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:25 smithi136 bash[76371]: cluster 2024-09-12T23:53:23.982010+0000 mon.a (mon.0) 1541 : cluster [DBG] osdmap e901: 8 total, 8 up, 8 in 2024-09-12T23:53:25.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:25 smithi136 bash[76371]: audit 2024-09-12T23:53:23.993500+0000 mon.c (mon.2) 27 : audit [INF] from='client.? 172.21.15.136:0/1510532693' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-13","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:25.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:25 smithi136 bash[76371]: audit 2024-09-12T23:53:23.994444+0000 mon.a (mon.0) 1542 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-13","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:25.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:25 smithi136 bash[76371]: cluster 2024-09-12T23:53:24.925012+0000 mgr.y (mgr.24661) 489 : cluster [DBG] pgmap v951: 137 pgs: 2 creating+peering, 30 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:25.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:25 smithi181 bash[67587]: cluster 2024-09-12T23:53:23.982010+0000 mon.a (mon.0) 1541 : cluster [DBG] osdmap e901: 8 total, 8 up, 8 in 2024-09-12T23:53:25.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:25 smithi181 bash[67587]: audit 2024-09-12T23:53:23.993500+0000 mon.c (mon.2) 27 : audit [INF] from='client.? 172.21.15.136:0/1510532693' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-13","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:25.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:25 smithi181 bash[67587]: audit 2024-09-12T23:53:23.994444+0000 mon.a (mon.0) 1542 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-13","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:25.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:25 smithi181 bash[67587]: cluster 2024-09-12T23:53:24.925012+0000 mgr.y (mgr.24661) 489 : cluster [DBG] pgmap v951: 137 pgs: 2 creating+peering, 30 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:26.016 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestCls2PCQueue.ReserveCommit (3036 ms) 2024-09-12T23:53:26.016 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestCls2PCQueue.ReserveAbort 2024-09-12T23:53:26.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:26 smithi181 bash[67587]: audit 2024-09-12T23:53:24.984142+0000 mon.a (mon.0) 1543 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-13","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:53:26.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:26 smithi181 bash[67587]: cluster 2024-09-12T23:53:24.984479+0000 mon.a (mon.0) 1544 : cluster [DBG] osdmap e902: 8 total, 8 up, 8 in 2024-09-12T23:53:26.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:26 smithi181 bash[67587]: cluster 2024-09-12T23:53:24.987174+0000 mon.a (mon.0) 1545 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:53:26.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:26 smithi136 bash[76371]: audit 2024-09-12T23:53:24.984142+0000 mon.a (mon.0) 1543 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-13","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:53:26.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:26 smithi136 bash[76371]: cluster 2024-09-12T23:53:24.984479+0000 mon.a (mon.0) 1544 : cluster [DBG] osdmap e902: 8 total, 8 up, 8 in 2024-09-12T23:53:26.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:26 smithi136 bash[76371]: cluster 2024-09-12T23:53:24.987174+0000 mon.a (mon.0) 1545 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:53:26.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:26 smithi136 bash[75164]: audit 2024-09-12T23:53:24.984142+0000 mon.a (mon.0) 1543 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-13","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:53:26.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:26 smithi136 bash[75164]: cluster 2024-09-12T23:53:24.984479+0000 mon.a (mon.0) 1544 : cluster [DBG] osdmap e902: 8 total, 8 up, 8 in 2024-09-12T23:53:26.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:26 smithi136 bash[75164]: cluster 2024-09-12T23:53:24.987174+0000 mon.a (mon.0) 1545 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:53:27.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:27 smithi181 bash[67587]: cluster 2024-09-12T23:53:26.015798+0000 mon.a (mon.0) 1546 : cluster [DBG] osdmap e903: 8 total, 8 up, 8 in 2024-09-12T23:53:27.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:27 smithi181 bash[67587]: cluster 2024-09-12T23:53:26.925866+0000 mgr.y (mgr.24661) 490 : cluster [DBG] pgmap v954: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:27.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:27 smithi136 bash[76371]: cluster 2024-09-12T23:53:26.015798+0000 mon.a (mon.0) 1546 : cluster [DBG] osdmap e903: 8 total, 8 up, 8 in 2024-09-12T23:53:27.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:27 smithi136 bash[76371]: cluster 2024-09-12T23:53:26.925866+0000 mgr.y (mgr.24661) 490 : cluster [DBG] pgmap v954: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:27.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:27 smithi136 bash[75164]: cluster 2024-09-12T23:53:26.015798+0000 mon.a (mon.0) 1546 : cluster [DBG] osdmap e903: 8 total, 8 up, 8 in 2024-09-12T23:53:27.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:27 smithi136 bash[75164]: cluster 2024-09-12T23:53:26.925866+0000 mgr.y (mgr.24661) 490 : cluster [DBG] pgmap v954: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:28.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:28 smithi181 bash[67587]: cluster 2024-09-12T23:53:27.019849+0000 mon.a (mon.0) 1547 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:53:28.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:28 smithi181 bash[67587]: cluster 2024-09-12T23:53:27.019926+0000 mon.a (mon.0) 1548 : cluster [INF] Cluster is now healthy 2024-09-12T23:53:28.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:28 smithi181 bash[67587]: cluster 2024-09-12T23:53:27.037680+0000 mon.a (mon.0) 1549 : cluster [DBG] osdmap e904: 8 total, 8 up, 8 in 2024-09-12T23:53:28.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:28 smithi181 bash[67587]: audit 2024-09-12T23:53:27.040866+0000 mon.c (mon.2) 28 : audit [INF] from='client.? 172.21.15.136:0/433214746' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-14","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:28.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:28 smithi181 bash[67587]: audit 2024-09-12T23:53:27.042375+0000 mon.a (mon.0) 1550 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-14","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:28.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:28 smithi136 bash[76371]: cluster 2024-09-12T23:53:27.019849+0000 mon.a (mon.0) 1547 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:53:28.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:28 smithi136 bash[76371]: cluster 2024-09-12T23:53:27.019926+0000 mon.a (mon.0) 1548 : cluster [INF] Cluster is now healthy 2024-09-12T23:53:28.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:28 smithi136 bash[76371]: cluster 2024-09-12T23:53:27.037680+0000 mon.a (mon.0) 1549 : cluster [DBG] osdmap e904: 8 total, 8 up, 8 in 2024-09-12T23:53:28.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:28 smithi136 bash[76371]: audit 2024-09-12T23:53:27.040866+0000 mon.c (mon.2) 28 : audit [INF] from='client.? 172.21.15.136:0/433214746' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-14","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:28.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:28 smithi136 bash[76371]: audit 2024-09-12T23:53:27.042375+0000 mon.a (mon.0) 1550 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-14","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:28.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:28 smithi136 bash[75164]: cluster 2024-09-12T23:53:27.019849+0000 mon.a (mon.0) 1547 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:53:28.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:28 smithi136 bash[75164]: cluster 2024-09-12T23:53:27.019926+0000 mon.a (mon.0) 1548 : cluster [INF] Cluster is now healthy 2024-09-12T23:53:28.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:28 smithi136 bash[75164]: cluster 2024-09-12T23:53:27.037680+0000 mon.a (mon.0) 1549 : cluster [DBG] osdmap e904: 8 total, 8 up, 8 in 2024-09-12T23:53:28.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:28 smithi136 bash[75164]: audit 2024-09-12T23:53:27.040866+0000 mon.c (mon.2) 28 : audit [INF] from='client.? 172.21.15.136:0/433214746' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-14","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:28.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:28 smithi136 bash[75164]: audit 2024-09-12T23:53:27.042375+0000 mon.a (mon.0) 1550 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-14","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:29.052 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestCls2PCQueue.ReserveAbort (3035 ms) 2024-09-12T23:53:29.052 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestCls2PCQueue.ManualCleanup 2024-09-12T23:53:29.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:29 smithi181 bash[67587]: audit 2024-09-12T23:53:28.043394+0000 mon.a (mon.0) 1551 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-14","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:53:29.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:29 smithi181 bash[67587]: cluster 2024-09-12T23:53:28.043810+0000 mon.a (mon.0) 1552 : cluster [DBG] osdmap e905: 8 total, 8 up, 8 in 2024-09-12T23:53:29.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:29 smithi181 bash[67587]: audit 2024-09-12T23:53:28.809662+0000 mon.a (mon.0) 1553 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:53:29.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:29 smithi181 bash[67587]: cluster 2024-09-12T23:53:28.926650+0000 mgr.y (mgr.24661) 491 : cluster [DBG] pgmap v957: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:29.411 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:29 smithi136 bash[75164]: audit 2024-09-12T23:53:28.043394+0000 mon.a (mon.0) 1551 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-14","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:53:29.411 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:29 smithi136 bash[75164]: cluster 2024-09-12T23:53:28.043810+0000 mon.a (mon.0) 1552 : cluster [DBG] osdmap e905: 8 total, 8 up, 8 in 2024-09-12T23:53:29.411 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:29 smithi136 bash[75164]: audit 2024-09-12T23:53:28.809662+0000 mon.a (mon.0) 1553 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:53:29.411 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:29 smithi136 bash[75164]: cluster 2024-09-12T23:53:28.926650+0000 mgr.y (mgr.24661) 491 : cluster [DBG] pgmap v957: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:29.411 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:29 smithi136 bash[76371]: audit 2024-09-12T23:53:28.043394+0000 mon.a (mon.0) 1551 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-14","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:53:29.412 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:29 smithi136 bash[76371]: cluster 2024-09-12T23:53:28.043810+0000 mon.a (mon.0) 1552 : cluster [DBG] osdmap e905: 8 total, 8 up, 8 in 2024-09-12T23:53:29.412 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:29 smithi136 bash[76371]: audit 2024-09-12T23:53:28.809662+0000 mon.a (mon.0) 1553 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:53:29.412 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:29 smithi136 bash[76371]: cluster 2024-09-12T23:53:28.926650+0000 mgr.y (mgr.24661) 491 : cluster [DBG] pgmap v957: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:29.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:53:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:53:29] "GET /metrics HTTP/1.1" 200 34084 "" "Prometheus/2.43.0" 2024-09-12T23:53:30.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:30 smithi181 bash[67587]: cluster 2024-09-12T23:53:29.050139+0000 mon.a (mon.0) 1554 : cluster [DBG] osdmap e906: 8 total, 8 up, 8 in 2024-09-12T23:53:30.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:30 smithi181 bash[67587]: audit 2024-09-12T23:53:29.511128+0000 mon.a (mon.0) 1555 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:53:30.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:30 smithi181 bash[67587]: audit 2024-09-12T23:53:29.512546+0000 mon.a (mon.0) 1556 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:53:30.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:30 smithi181 bash[67587]: audit 2024-09-12T23:53:29.523223+0000 mon.a (mon.0) 1557 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:53:30.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:30 smithi136 bash[75164]: cluster 2024-09-12T23:53:29.050139+0000 mon.a (mon.0) 1554 : cluster [DBG] osdmap e906: 8 total, 8 up, 8 in 2024-09-12T23:53:30.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:30 smithi136 bash[75164]: audit 2024-09-12T23:53:29.511128+0000 mon.a (mon.0) 1555 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:53:30.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:30 smithi136 bash[75164]: audit 2024-09-12T23:53:29.512546+0000 mon.a (mon.0) 1556 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:53:30.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:30 smithi136 bash[75164]: audit 2024-09-12T23:53:29.523223+0000 mon.a (mon.0) 1557 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:53:30.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:30 smithi136 bash[76371]: cluster 2024-09-12T23:53:29.050139+0000 mon.a (mon.0) 1554 : cluster [DBG] osdmap e906: 8 total, 8 up, 8 in 2024-09-12T23:53:30.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:30 smithi136 bash[76371]: audit 2024-09-12T23:53:29.511128+0000 mon.a (mon.0) 1555 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:53:30.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:30 smithi136 bash[76371]: audit 2024-09-12T23:53:29.512546+0000 mon.a (mon.0) 1556 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:53:30.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:30 smithi136 bash[76371]: audit 2024-09-12T23:53:29.523223+0000 mon.a (mon.0) 1557 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:53:31.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:31 smithi181 bash[67587]: cluster 2024-09-12T23:53:30.061582+0000 mon.a (mon.0) 1558 : cluster [DBG] osdmap e907: 8 total, 8 up, 8 in 2024-09-12T23:53:31.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:31 smithi181 bash[67587]: audit 2024-09-12T23:53:30.068473+0000 mon.a (mon.0) 1559 : audit [INF] from='client.? 172.21.15.136:0/4242859080' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-15","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:31.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:31 smithi181 bash[67587]: cluster 2024-09-12T23:53:30.927624+0000 mgr.y (mgr.24661) 492 : cluster [DBG] pgmap v960: 137 pgs: 11 creating+peering, 21 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:31.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:31 smithi136 bash[76371]: cluster 2024-09-12T23:53:30.061582+0000 mon.a (mon.0) 1558 : cluster [DBG] osdmap e907: 8 total, 8 up, 8 in 2024-09-12T23:53:31.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:31 smithi136 bash[76371]: audit 2024-09-12T23:53:30.068473+0000 mon.a (mon.0) 1559 : audit [INF] from='client.? 172.21.15.136:0/4242859080' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-15","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:31.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:31 smithi136 bash[76371]: cluster 2024-09-12T23:53:30.927624+0000 mgr.y (mgr.24661) 492 : cluster [DBG] pgmap v960: 137 pgs: 11 creating+peering, 21 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:31.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:31 smithi136 bash[75164]: cluster 2024-09-12T23:53:30.061582+0000 mon.a (mon.0) 1558 : cluster [DBG] osdmap e907: 8 total, 8 up, 8 in 2024-09-12T23:53:31.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:31 smithi136 bash[75164]: audit 2024-09-12T23:53:30.068473+0000 mon.a (mon.0) 1559 : audit [INF] from='client.? 172.21.15.136:0/4242859080' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-15","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:31.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:31 smithi136 bash[75164]: cluster 2024-09-12T23:53:30.927624+0000 mgr.y (mgr.24661) 492 : cluster [DBG] pgmap v960: 137 pgs: 11 creating+peering, 21 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:32.085 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestCls2PCQueue.ManualCleanup (3033 ms) 2024-09-12T23:53:32.085 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestCls2PCQueue.Cleanup 2024-09-12T23:53:32.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:32 smithi181 bash[67587]: cluster 2024-09-12T23:53:31.053865+0000 mon.a (mon.0) 1560 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:53:32.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:32 smithi181 bash[67587]: audit 2024-09-12T23:53:31.074536+0000 mon.a (mon.0) 1561 : audit [INF] from='client.? 172.21.15.136:0/4242859080' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-15","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:53:32.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:32 smithi181 bash[67587]: cluster 2024-09-12T23:53:31.074918+0000 mon.a (mon.0) 1562 : cluster [DBG] osdmap e908: 8 total, 8 up, 8 in 2024-09-12T23:53:32.348 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:53:32 smithi181 bash[95505]: ts=2024-09-12T23:53:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:53:32.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:32 smithi136 bash[75164]: cluster 2024-09-12T23:53:31.053865+0000 mon.a (mon.0) 1560 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:53:32.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:32 smithi136 bash[75164]: audit 2024-09-12T23:53:31.074536+0000 mon.a (mon.0) 1561 : audit [INF] from='client.? 172.21.15.136:0/4242859080' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-15","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:53:32.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:32 smithi136 bash[75164]: cluster 2024-09-12T23:53:31.074918+0000 mon.a (mon.0) 1562 : cluster [DBG] osdmap e908: 8 total, 8 up, 8 in 2024-09-12T23:53:32.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:32 smithi136 bash[76371]: cluster 2024-09-12T23:53:31.053865+0000 mon.a (mon.0) 1560 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:53:32.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:32 smithi136 bash[76371]: audit 2024-09-12T23:53:31.074536+0000 mon.a (mon.0) 1561 : audit [INF] from='client.? 172.21.15.136:0/4242859080' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-15","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:53:32.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:32 smithi136 bash[76371]: cluster 2024-09-12T23:53:31.074918+0000 mon.a (mon.0) 1562 : cluster [DBG] osdmap e908: 8 total, 8 up, 8 in 2024-09-12T23:53:33.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:33 smithi181 bash[67587]: cluster 2024-09-12T23:53:32.081813+0000 mon.a (mon.0) 1563 : cluster [DBG] osdmap e909: 8 total, 8 up, 8 in 2024-09-12T23:53:33.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:33 smithi181 bash[67587]: cluster 2024-09-12T23:53:32.928423+0000 mgr.y (mgr.24661) 493 : cluster [DBG] pgmap v963: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:33.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:33 smithi136 bash[75164]: cluster 2024-09-12T23:53:32.081813+0000 mon.a (mon.0) 1563 : cluster [DBG] osdmap e909: 8 total, 8 up, 8 in 2024-09-12T23:53:33.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:33 smithi136 bash[75164]: cluster 2024-09-12T23:53:32.928423+0000 mgr.y (mgr.24661) 493 : cluster [DBG] pgmap v963: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:33.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:33 smithi136 bash[76371]: cluster 2024-09-12T23:53:32.081813+0000 mon.a (mon.0) 1563 : cluster [DBG] osdmap e909: 8 total, 8 up, 8 in 2024-09-12T23:53:33.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:33 smithi136 bash[76371]: cluster 2024-09-12T23:53:32.928423+0000 mgr.y (mgr.24661) 493 : cluster [DBG] pgmap v963: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:34.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:34 smithi181 bash[67587]: cluster 2024-09-12T23:53:33.075194+0000 mon.a (mon.0) 1564 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:53:34.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:34 smithi181 bash[67587]: cluster 2024-09-12T23:53:33.075257+0000 mon.a (mon.0) 1565 : cluster [INF] Cluster is now healthy 2024-09-12T23:53:34.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:34 smithi181 bash[67587]: cluster 2024-09-12T23:53:33.094372+0000 mon.a (mon.0) 1566 : cluster [DBG] osdmap e910: 8 total, 8 up, 8 in 2024-09-12T23:53:34.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:34 smithi181 bash[67587]: audit 2024-09-12T23:53:33.098791+0000 mon.c (mon.2) 29 : audit [INF] from='client.? 172.21.15.136:0/1239446395' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-16","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:34.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:34 smithi181 bash[67587]: audit 2024-09-12T23:53:33.100505+0000 mon.a (mon.0) 1567 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-16","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:34.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:34 smithi136 bash[76371]: cluster 2024-09-12T23:53:33.075194+0000 mon.a (mon.0) 1564 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:53:34.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:34 smithi136 bash[76371]: cluster 2024-09-12T23:53:33.075257+0000 mon.a (mon.0) 1565 : cluster [INF] Cluster is now healthy 2024-09-12T23:53:34.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:34 smithi136 bash[76371]: cluster 2024-09-12T23:53:33.094372+0000 mon.a (mon.0) 1566 : cluster [DBG] osdmap e910: 8 total, 8 up, 8 in 2024-09-12T23:53:34.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:34 smithi136 bash[76371]: audit 2024-09-12T23:53:33.098791+0000 mon.c (mon.2) 29 : audit [INF] from='client.? 172.21.15.136:0/1239446395' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-16","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:34.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:34 smithi136 bash[76371]: audit 2024-09-12T23:53:33.100505+0000 mon.a (mon.0) 1567 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-16","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:34.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:34 smithi136 bash[75164]: cluster 2024-09-12T23:53:33.075194+0000 mon.a (mon.0) 1564 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:53:34.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:34 smithi136 bash[75164]: cluster 2024-09-12T23:53:33.075257+0000 mon.a (mon.0) 1565 : cluster [INF] Cluster is now healthy 2024-09-12T23:53:34.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:34 smithi136 bash[75164]: cluster 2024-09-12T23:53:33.094372+0000 mon.a (mon.0) 1566 : cluster [DBG] osdmap e910: 8 total, 8 up, 8 in 2024-09-12T23:53:34.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:34 smithi136 bash[75164]: audit 2024-09-12T23:53:33.098791+0000 mon.c (mon.2) 29 : audit [INF] from='client.? 172.21.15.136:0/1239446395' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-16","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:34.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:34 smithi136 bash[75164]: audit 2024-09-12T23:53:33.100505+0000 mon.a (mon.0) 1567 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-16","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:35.121 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestCls2PCQueue.Cleanup (3037 ms) 2024-09-12T23:53:35.121 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestCls2PCQueue.MultiProducer 2024-09-12T23:53:35.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:35 smithi136 bash[76371]: audit 2024-09-12T23:53:34.105251+0000 mon.a (mon.0) 1568 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-16","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:53:35.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:35 smithi136 bash[76371]: cluster 2024-09-12T23:53:34.105704+0000 mon.a (mon.0) 1569 : cluster [DBG] osdmap e911: 8 total, 8 up, 8 in 2024-09-12T23:53:35.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:35 smithi136 bash[76371]: cluster 2024-09-12T23:53:34.929276+0000 mgr.y (mgr.24661) 494 : cluster [DBG] pgmap v966: 137 pgs: 3 creating+peering, 26 unknown, 108 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:35.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:35 smithi136 bash[75164]: audit 2024-09-12T23:53:34.105251+0000 mon.a (mon.0) 1568 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-16","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:53:35.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:35 smithi136 bash[75164]: cluster 2024-09-12T23:53:34.105704+0000 mon.a (mon.0) 1569 : cluster [DBG] osdmap e911: 8 total, 8 up, 8 in 2024-09-12T23:53:35.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:35 smithi136 bash[75164]: cluster 2024-09-12T23:53:34.929276+0000 mgr.y (mgr.24661) 494 : cluster [DBG] pgmap v966: 137 pgs: 3 creating+peering, 26 unknown, 108 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:35.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:35 smithi181 bash[67587]: audit 2024-09-12T23:53:34.105251+0000 mon.a (mon.0) 1568 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-16","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:53:35.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:35 smithi181 bash[67587]: cluster 2024-09-12T23:53:34.105704+0000 mon.a (mon.0) 1569 : cluster [DBG] osdmap e911: 8 total, 8 up, 8 in 2024-09-12T23:53:35.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:35 smithi181 bash[67587]: cluster 2024-09-12T23:53:34.929276+0000 mgr.y (mgr.24661) 494 : cluster [DBG] pgmap v966: 137 pgs: 3 creating+peering, 26 unknown, 108 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:36.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:36 smithi136 bash[75164]: cluster 2024-09-12T23:53:35.107504+0000 mon.a (mon.0) 1570 : cluster [DBG] osdmap e912: 8 total, 8 up, 8 in 2024-09-12T23:53:36.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:36 smithi136 bash[76371]: cluster 2024-09-12T23:53:35.107504+0000 mon.a (mon.0) 1570 : cluster [DBG] osdmap e912: 8 total, 8 up, 8 in 2024-09-12T23:53:36.600 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:36 smithi181 bash[67587]: cluster 2024-09-12T23:53:35.107504+0000 mon.a (mon.0) 1570 : cluster [DBG] osdmap e912: 8 total, 8 up, 8 in 2024-09-12T23:53:37.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:37 smithi136 bash[75164]: cluster 2024-09-12T23:53:36.127151+0000 mon.a (mon.0) 1571 : cluster [DBG] osdmap e913: 8 total, 8 up, 8 in 2024-09-12T23:53:37.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:37 smithi136 bash[75164]: audit 2024-09-12T23:53:36.134710+0000 mon.c (mon.2) 30 : audit [INF] from='client.? 172.21.15.136:0/1176008716' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-17","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:37.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:37 smithi136 bash[75164]: audit 2024-09-12T23:53:36.146065+0000 mon.a (mon.0) 1572 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-17","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:37.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:37 smithi136 bash[75164]: cluster 2024-09-12T23:53:36.930247+0000 mgr.y (mgr.24661) 495 : cluster [DBG] pgmap v969: 137 pgs: 10 creating+peering, 22 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:37.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:37 smithi136 bash[75164]: audit 2024-09-12T23:53:37.128287+0000 mon.a (mon.0) 1573 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-17","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:53:37.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:37 smithi136 bash[75164]: cluster 2024-09-12T23:53:37.128491+0000 mon.a (mon.0) 1574 : cluster [DBG] osdmap e914: 8 total, 8 up, 8 in 2024-09-12T23:53:37.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:37 smithi136 bash[76371]: cluster 2024-09-12T23:53:36.127151+0000 mon.a (mon.0) 1571 : cluster [DBG] osdmap e913: 8 total, 8 up, 8 in 2024-09-12T23:53:37.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:37 smithi136 bash[76371]: audit 2024-09-12T23:53:36.134710+0000 mon.c (mon.2) 30 : audit [INF] from='client.? 172.21.15.136:0/1176008716' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-17","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:37.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:37 smithi136 bash[76371]: audit 2024-09-12T23:53:36.146065+0000 mon.a (mon.0) 1572 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-17","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:37.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:37 smithi136 bash[76371]: cluster 2024-09-12T23:53:36.930247+0000 mgr.y (mgr.24661) 495 : cluster [DBG] pgmap v969: 137 pgs: 10 creating+peering, 22 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:37.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:37 smithi136 bash[76371]: audit 2024-09-12T23:53:37.128287+0000 mon.a (mon.0) 1573 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-17","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:53:37.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:37 smithi136 bash[76371]: cluster 2024-09-12T23:53:37.128491+0000 mon.a (mon.0) 1574 : cluster [DBG] osdmap e914: 8 total, 8 up, 8 in 2024-09-12T23:53:37.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:37 smithi181 bash[67587]: cluster 2024-09-12T23:53:36.127151+0000 mon.a (mon.0) 1571 : cluster [DBG] osdmap e913: 8 total, 8 up, 8 in 2024-09-12T23:53:37.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:37 smithi181 bash[67587]: audit 2024-09-12T23:53:36.134710+0000 mon.c (mon.2) 30 : audit [INF] from='client.? 172.21.15.136:0/1176008716' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-17","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:37.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:37 smithi181 bash[67587]: audit 2024-09-12T23:53:36.146065+0000 mon.a (mon.0) 1572 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-17","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:53:37.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:37 smithi181 bash[67587]: cluster 2024-09-12T23:53:36.930247+0000 mgr.y (mgr.24661) 495 : cluster [DBG] pgmap v969: 137 pgs: 10 creating+peering, 22 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:37.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:37 smithi181 bash[67587]: audit 2024-09-12T23:53:37.128287+0000 mon.a (mon.0) 1573 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-17","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:53:37.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:37 smithi181 bash[67587]: cluster 2024-09-12T23:53:37.128491+0000 mon.a (mon.0) 1574 : cluster [DBG] osdmap e914: 8 total, 8 up, 8 in 2024-09-12T23:53:38.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:38 smithi136 bash[75164]: cluster 2024-09-12T23:53:37.138106+0000 mon.a (mon.0) 1575 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:53:38.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:38 smithi136 bash[76371]: cluster 2024-09-12T23:53:37.138106+0000 mon.a (mon.0) 1575 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:53:38.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:38 smithi181 bash[67587]: cluster 2024-09-12T23:53:37.138106+0000 mon.a (mon.0) 1575 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:53:39.412 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:39 smithi136 bash[76371]: cluster 2024-09-12T23:53:38.150485+0000 mon.a (mon.0) 1576 : cluster [DBG] osdmap e915: 8 total, 8 up, 8 in 2024-09-12T23:53:39.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:39 smithi136 bash[76371]: audit 2024-09-12T23:53:38.767299+0000 mon.a (mon.0) 1577 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:53:39.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:39 smithi136 bash[76371]: audit 2024-09-12T23:53:38.769717+0000 mon.a (mon.0) 1578 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:53:39.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:39 smithi136 bash[76371]: cluster 2024-09-12T23:53:38.931291+0000 mgr.y (mgr.24661) 496 : cluster [DBG] pgmap v972: 137 pgs: 10 creating+peering, 17 unknown, 110 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:39.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:39 smithi136 bash[75164]: cluster 2024-09-12T23:53:38.150485+0000 mon.a (mon.0) 1576 : cluster [DBG] osdmap e915: 8 total, 8 up, 8 in 2024-09-12T23:53:39.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:39 smithi136 bash[75164]: audit 2024-09-12T23:53:38.767299+0000 mon.a (mon.0) 1577 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:53:39.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:39 smithi136 bash[75164]: audit 2024-09-12T23:53:38.769717+0000 mon.a (mon.0) 1578 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:53:39.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:39 smithi136 bash[75164]: cluster 2024-09-12T23:53:38.931291+0000 mgr.y (mgr.24661) 496 : cluster [DBG] pgmap v972: 137 pgs: 10 creating+peering, 17 unknown, 110 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:39.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:39 smithi181 bash[67587]: cluster 2024-09-12T23:53:38.150485+0000 mon.a (mon.0) 1576 : cluster [DBG] osdmap e915: 8 total, 8 up, 8 in 2024-09-12T23:53:39.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:39 smithi181 bash[67587]: audit 2024-09-12T23:53:38.767299+0000 mon.a (mon.0) 1577 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:53:39.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:39 smithi181 bash[67587]: audit 2024-09-12T23:53:38.769717+0000 mon.a (mon.0) 1578 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:53:39.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:39 smithi181 bash[67587]: cluster 2024-09-12T23:53:38.931291+0000 mgr.y (mgr.24661) 496 : cluster [DBG] pgmap v972: 137 pgs: 10 creating+peering, 17 unknown, 110 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:39.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:53:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:53:39] "GET /metrics HTTP/1.1" 200 36456 "" "Prometheus/2.43.0" 2024-09-12T23:53:40.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:40 smithi136 bash[75164]: cluster 2024-09-12T23:53:39.761782+0000 mon.a (mon.0) 1579 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:53:40.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:40 smithi136 bash[75164]: cluster 2024-09-12T23:53:39.761840+0000 mon.a (mon.0) 1580 : cluster [INF] Cluster is now healthy 2024-09-12T23:53:40.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:40 smithi136 bash[76371]: cluster 2024-09-12T23:53:39.761782+0000 mon.a (mon.0) 1579 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:53:40.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:40 smithi136 bash[76371]: cluster 2024-09-12T23:53:39.761840+0000 mon.a (mon.0) 1580 : cluster [INF] Cluster is now healthy 2024-09-12T23:53:40.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:40 smithi181 bash[67587]: cluster 2024-09-12T23:53:39.761782+0000 mon.a (mon.0) 1579 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:53:40.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:40 smithi181 bash[67587]: cluster 2024-09-12T23:53:39.761840+0000 mon.a (mon.0) 1580 : cluster [INF] Cluster is now healthy 2024-09-12T23:53:41.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:41 smithi136 bash[75164]: cluster 2024-09-12T23:53:40.932286+0000 mgr.y (mgr.24661) 497 : cluster [DBG] pgmap v973: 137 pgs: 10 creating+peering, 3 unknown, 124 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:41.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:41 smithi136 bash[76371]: cluster 2024-09-12T23:53:40.932286+0000 mgr.y (mgr.24661) 497 : cluster [DBG] pgmap v973: 137 pgs: 10 creating+peering, 3 unknown, 124 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:41.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:41 smithi181 bash[67587]: cluster 2024-09-12T23:53:40.932286+0000 mgr.y (mgr.24661) 497 : cluster [DBG] pgmap v973: 137 pgs: 10 creating+peering, 3 unknown, 124 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-12T23:53:42.597 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:53:42 smithi181 bash[95505]: ts=2024-09-12T23:53:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:53:43.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:43 smithi136 bash[76371]: cluster 2024-09-12T23:53:42.933496+0000 mgr.y (mgr.24661) 498 : cluster [DBG] pgmap v974: 137 pgs: 137 active+clean; 1.1 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 3.5 MiB/s wr, 3.87k op/s 2024-09-12T23:53:43.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:43 smithi136 bash[75164]: cluster 2024-09-12T23:53:42.933496+0000 mgr.y (mgr.24661) 498 : cluster [DBG] pgmap v974: 137 pgs: 137 active+clean; 1.1 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 3.5 MiB/s wr, 3.87k op/s 2024-09-12T23:53:43.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:43 smithi181 bash[67587]: cluster 2024-09-12T23:53:42.933496+0000 mgr.y (mgr.24661) 498 : cluster [DBG] pgmap v974: 137 pgs: 137 active+clean; 1.1 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 3.5 MiB/s wr, 3.87k op/s 2024-09-12T23:53:45.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:45 smithi136 bash[75164]: cluster 2024-09-12T23:53:44.934392+0000 mgr.y (mgr.24661) 499 : cluster [DBG] pgmap v975: 137 pgs: 137 active+clean; 1.1 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.6 MiB/s rd, 2.9 MiB/s wr, 3.30k op/s 2024-09-12T23:53:45.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:45 smithi136 bash[76371]: cluster 2024-09-12T23:53:44.934392+0000 mgr.y (mgr.24661) 499 : cluster [DBG] pgmap v975: 137 pgs: 137 active+clean; 1.1 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.6 MiB/s rd, 2.9 MiB/s wr, 3.30k op/s 2024-09-12T23:53:45.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:45 smithi181 bash[67587]: cluster 2024-09-12T23:53:44.934392+0000 mgr.y (mgr.24661) 499 : cluster [DBG] pgmap v975: 137 pgs: 137 active+clean; 1.1 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.6 MiB/s rd, 2.9 MiB/s wr, 3.30k op/s 2024-09-12T23:53:47.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:47 smithi136 bash[76371]: cluster 2024-09-12T23:53:46.935939+0000 mgr.y (mgr.24661) 500 : cluster [DBG] pgmap v976: 137 pgs: 137 active+clean; 1.5 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.1 MiB/s rd, 4.2 MiB/s wr, 4.71k op/s 2024-09-12T23:53:47.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:47 smithi136 bash[75164]: cluster 2024-09-12T23:53:46.935939+0000 mgr.y (mgr.24661) 500 : cluster [DBG] pgmap v976: 137 pgs: 137 active+clean; 1.5 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.1 MiB/s rd, 4.2 MiB/s wr, 4.71k op/s 2024-09-12T23:53:47.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:47 smithi181 bash[67587]: cluster 2024-09-12T23:53:46.935939+0000 mgr.y (mgr.24661) 500 : cluster [DBG] pgmap v976: 137 pgs: 137 active+clean; 1.5 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.1 MiB/s rd, 4.2 MiB/s wr, 4.71k op/s 2024-09-12T23:53:49.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:49 smithi136 bash[75164]: cluster 2024-09-12T23:53:48.936936+0000 mgr.y (mgr.24661) 501 : cluster [DBG] pgmap v977: 137 pgs: 137 active+clean; 1.5 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.4 MiB/s rd, 3.8 MiB/s wr, 4.28k op/s 2024-09-12T23:53:49.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:49 smithi136 bash[76371]: cluster 2024-09-12T23:53:48.936936+0000 mgr.y (mgr.24661) 501 : cluster [DBG] pgmap v977: 137 pgs: 137 active+clean; 1.5 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.4 MiB/s rd, 3.8 MiB/s wr, 4.28k op/s 2024-09-12T23:53:49.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:49 smithi181 bash[67587]: cluster 2024-09-12T23:53:48.936936+0000 mgr.y (mgr.24661) 501 : cluster [DBG] pgmap v977: 137 pgs: 137 active+clean; 1.5 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.4 MiB/s rd, 3.8 MiB/s wr, 4.28k op/s 2024-09-12T23:53:49.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:53:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:53:49] "GET /metrics HTTP/1.1" 200 36456 "" "Prometheus/2.43.0" 2024-09-12T23:53:51.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:51 smithi136 bash[75164]: cluster 2024-09-12T23:53:50.938004+0000 mgr.y (mgr.24661) 502 : cluster [DBG] pgmap v978: 137 pgs: 137 active+clean; 1.5 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.6 MiB/s rd, 3.4 MiB/s wr, 3.85k op/s 2024-09-12T23:53:51.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:51 smithi136 bash[76371]: cluster 2024-09-12T23:53:50.938004+0000 mgr.y (mgr.24661) 502 : cluster [DBG] pgmap v978: 137 pgs: 137 active+clean; 1.5 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.6 MiB/s rd, 3.4 MiB/s wr, 3.85k op/s 2024-09-12T23:53:51.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:51 smithi181 bash[67587]: cluster 2024-09-12T23:53:50.938004+0000 mgr.y (mgr.24661) 502 : cluster [DBG] pgmap v978: 137 pgs: 137 active+clean; 1.5 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.6 MiB/s rd, 3.4 MiB/s wr, 3.85k op/s 2024-09-12T23:53:52.597 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:53:52 smithi181 bash[95505]: ts=2024-09-12T23:53:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:53:53.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:53 smithi136 bash[75164]: cluster 2024-09-12T23:53:52.939423+0000 mgr.y (mgr.24661) 503 : cluster [DBG] pgmap v979: 137 pgs: 137 active+clean; 1.9 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 4.7 MiB/s wr, 5.27k op/s 2024-09-12T23:53:53.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:53 smithi136 bash[76371]: cluster 2024-09-12T23:53:52.939423+0000 mgr.y (mgr.24661) 503 : cluster [DBG] pgmap v979: 137 pgs: 137 active+clean; 1.9 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 4.7 MiB/s wr, 5.27k op/s 2024-09-12T23:53:53.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:53 smithi181 bash[67587]: cluster 2024-09-12T23:53:52.939423+0000 mgr.y (mgr.24661) 503 : cluster [DBG] pgmap v979: 137 pgs: 137 active+clean; 1.9 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 4.7 MiB/s wr, 5.27k op/s 2024-09-12T23:53:55.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:54 smithi181 bash[67587]: audit 2024-09-12T23:53:53.766015+0000 mon.a (mon.0) 1581 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:53:55.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:54 smithi181 bash[67587]: audit 2024-09-12T23:53:53.768133+0000 mon.a (mon.0) 1582 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:53:55.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:54 smithi136 bash[75164]: audit 2024-09-12T23:53:53.766015+0000 mon.a (mon.0) 1581 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:53:55.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:54 smithi136 bash[75164]: audit 2024-09-12T23:53:53.768133+0000 mon.a (mon.0) 1582 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:53:55.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:54 smithi136 bash[76371]: audit 2024-09-12T23:53:53.766015+0000 mon.a (mon.0) 1581 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:53:55.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:54 smithi136 bash[76371]: audit 2024-09-12T23:53:53.768133+0000 mon.a (mon.0) 1582 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:53:56.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:55 smithi181 bash[67587]: cluster 2024-09-12T23:53:54.940486+0000 mgr.y (mgr.24661) 504 : cluster [DBG] pgmap v980: 137 pgs: 137 active+clean; 1.9 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.4 MiB/s rd, 2.7 MiB/s wr, 3.07k op/s 2024-09-12T23:53:56.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:55 smithi136 bash[75164]: cluster 2024-09-12T23:53:54.940486+0000 mgr.y (mgr.24661) 504 : cluster [DBG] pgmap v980: 137 pgs: 137 active+clean; 1.9 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.4 MiB/s rd, 2.7 MiB/s wr, 3.07k op/s 2024-09-12T23:53:56.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:55 smithi136 bash[76371]: cluster 2024-09-12T23:53:54.940486+0000 mgr.y (mgr.24661) 504 : cluster [DBG] pgmap v980: 137 pgs: 137 active+clean; 1.9 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.4 MiB/s rd, 2.7 MiB/s wr, 3.07k op/s 2024-09-12T23:53:57.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:57 smithi136 bash[76371]: cluster 2024-09-12T23:53:56.941747+0000 mgr.y (mgr.24661) 505 : cluster [DBG] pgmap v981: 137 pgs: 137 active+clean; 2.2 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 3.8 MiB/s wr, 4.30k op/s 2024-09-12T23:53:57.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:57 smithi136 bash[75164]: cluster 2024-09-12T23:53:56.941747+0000 mgr.y (mgr.24661) 505 : cluster [DBG] pgmap v981: 137 pgs: 137 active+clean; 2.2 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 3.8 MiB/s wr, 4.30k op/s 2024-09-12T23:53:57.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:57 smithi181 bash[67587]: cluster 2024-09-12T23:53:56.941747+0000 mgr.y (mgr.24661) 505 : cluster [DBG] pgmap v981: 137 pgs: 137 active+clean; 2.2 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 3.8 MiB/s wr, 4.30k op/s 2024-09-12T23:53:59.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:53:59 smithi136 bash[76371]: cluster 2024-09-12T23:53:58.942664+0000 mgr.y (mgr.24661) 506 : cluster [DBG] pgmap v982: 137 pgs: 137 active+clean; 2.2 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 11 MiB/s rd, 2.4 MiB/s wr, 2.65k op/s 2024-09-12T23:53:59.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:53:59 smithi136 bash[75164]: cluster 2024-09-12T23:53:58.942664+0000 mgr.y (mgr.24661) 506 : cluster [DBG] pgmap v982: 137 pgs: 137 active+clean; 2.2 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 11 MiB/s rd, 2.4 MiB/s wr, 2.65k op/s 2024-09-12T23:53:59.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:53:59 smithi181 bash[67587]: cluster 2024-09-12T23:53:58.942664+0000 mgr.y (mgr.24661) 506 : cluster [DBG] pgmap v982: 137 pgs: 137 active+clean; 2.2 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 11 MiB/s rd, 2.4 MiB/s wr, 2.65k op/s 2024-09-12T23:53:59.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:53:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:53:59] "GET /metrics HTTP/1.1" 200 36511 "" "Prometheus/2.43.0" 2024-09-12T23:54:01.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:01 smithi136 bash[75164]: cluster 2024-09-12T23:54:00.943791+0000 mgr.y (mgr.24661) 507 : cluster [DBG] pgmap v983: 137 pgs: 137 active+clean; 2.2 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 11 MiB/s rd, 2.4 MiB/s wr, 2.65k op/s 2024-09-12T23:54:01.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:01 smithi136 bash[76371]: cluster 2024-09-12T23:54:00.943791+0000 mgr.y (mgr.24661) 507 : cluster [DBG] pgmap v983: 137 pgs: 137 active+clean; 2.2 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 11 MiB/s rd, 2.4 MiB/s wr, 2.65k op/s 2024-09-12T23:54:01.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:01 smithi181 bash[67587]: cluster 2024-09-12T23:54:00.943791+0000 mgr.y (mgr.24661) 507 : cluster [DBG] pgmap v983: 137 pgs: 137 active+clean; 2.2 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 11 MiB/s rd, 2.4 MiB/s wr, 2.65k op/s 2024-09-12T23:54:02.597 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:54:02 smithi181 bash[95505]: ts=2024-09-12T23:54:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:54:03.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:03 smithi181 bash[67587]: cluster 2024-09-12T23:54:02.944967+0000 mgr.y (mgr.24661) 508 : cluster [DBG] pgmap v984: 137 pgs: 137 active+clean; 2.3 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 58 MiB/s rd, 2.7 MiB/s wr, 3.21k op/s 2024-09-12T23:54:03.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:03 smithi136 bash[75164]: cluster 2024-09-12T23:54:02.944967+0000 mgr.y (mgr.24661) 508 : cluster [DBG] pgmap v984: 137 pgs: 137 active+clean; 2.3 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 58 MiB/s rd, 2.7 MiB/s wr, 3.21k op/s 2024-09-12T23:54:03.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:03 smithi136 bash[76371]: cluster 2024-09-12T23:54:02.944967+0000 mgr.y (mgr.24661) 508 : cluster [DBG] pgmap v984: 137 pgs: 137 active+clean; 2.3 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 58 MiB/s rd, 2.7 MiB/s wr, 3.21k op/s 2024-09-12T23:54:05.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:05 smithi181 bash[67587]: cluster 2024-09-12T23:54:04.945880+0000 mgr.y (mgr.24661) 509 : cluster [DBG] pgmap v985: 137 pgs: 137 active+clean; 2.3 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 53 MiB/s rd, 1.5 MiB/s wr, 1.79k op/s 2024-09-12T23:54:05.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:05 smithi136 bash[75164]: cluster 2024-09-12T23:54:04.945880+0000 mgr.y (mgr.24661) 509 : cluster [DBG] pgmap v985: 137 pgs: 137 active+clean; 2.3 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 53 MiB/s rd, 1.5 MiB/s wr, 1.79k op/s 2024-09-12T23:54:05.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:05 smithi136 bash[76371]: cluster 2024-09-12T23:54:04.945880+0000 mgr.y (mgr.24661) 509 : cluster [DBG] pgmap v985: 137 pgs: 137 active+clean; 2.3 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 53 MiB/s rd, 1.5 MiB/s wr, 1.79k op/s 2024-09-12T23:54:07.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:07 smithi136 bash[75164]: cluster 2024-09-12T23:54:06.947179+0000 mgr.y (mgr.24661) 510 : cluster [DBG] pgmap v986: 137 pgs: 137 active+clean; 2.3 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 87 MiB/s rd, 1.5 MiB/s wr, 2.08k op/s 2024-09-12T23:54:07.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:07 smithi136 bash[76371]: cluster 2024-09-12T23:54:06.947179+0000 mgr.y (mgr.24661) 510 : cluster [DBG] pgmap v986: 137 pgs: 137 active+clean; 2.3 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 87 MiB/s rd, 1.5 MiB/s wr, 2.08k op/s 2024-09-12T23:54:07.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:07 smithi181 bash[67587]: cluster 2024-09-12T23:54:06.947179+0000 mgr.y (mgr.24661) 510 : cluster [DBG] pgmap v986: 137 pgs: 137 active+clean; 2.3 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 87 MiB/s rd, 1.5 MiB/s wr, 2.08k op/s 2024-09-12T23:54:08.367 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestCls2PCQueue.MultiProducer (33245 ms) 2024-09-12T23:54:08.367 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestCls2PCQueue.AsyncConsumer 2024-09-12T23:54:09.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:54:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:54:09] "GET /metrics HTTP/1.1" 200 34086 "" "Prometheus/2.43.0" 2024-09-12T23:54:09.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:09 smithi136 bash[75164]: cluster 2024-09-12T23:54:08.365458+0000 mon.a (mon.0) 1583 : cluster [DBG] osdmap e916: 8 total, 8 up, 8 in 2024-09-12T23:54:09.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:09 smithi136 bash[75164]: audit 2024-09-12T23:54:08.849003+0000 mon.a (mon.0) 1584 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:54:09.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:09 smithi136 bash[75164]: cluster 2024-09-12T23:54:08.947896+0000 mgr.y (mgr.24661) 511 : cluster [DBG] pgmap v988: 105 pgs: 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:54:09.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:09 smithi136 bash[76371]: cluster 2024-09-12T23:54:08.365458+0000 mon.a (mon.0) 1583 : cluster [DBG] osdmap e916: 8 total, 8 up, 8 in 2024-09-12T23:54:09.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:09 smithi136 bash[76371]: audit 2024-09-12T23:54:08.849003+0000 mon.a (mon.0) 1584 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:54:09.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:09 smithi136 bash[76371]: cluster 2024-09-12T23:54:08.947896+0000 mgr.y (mgr.24661) 511 : cluster [DBG] pgmap v988: 105 pgs: 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:54:09.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:09 smithi181 bash[67587]: cluster 2024-09-12T23:54:08.365458+0000 mon.a (mon.0) 1583 : cluster [DBG] osdmap e916: 8 total, 8 up, 8 in 2024-09-12T23:54:09.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:09 smithi181 bash[67587]: audit 2024-09-12T23:54:08.849003+0000 mon.a (mon.0) 1584 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:54:09.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:09 smithi181 bash[67587]: cluster 2024-09-12T23:54:08.947896+0000 mgr.y (mgr.24661) 511 : cluster [DBG] pgmap v988: 105 pgs: 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:54:10.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:10 smithi136 bash[75164]: cluster 2024-09-12T23:54:09.377162+0000 mon.a (mon.0) 1585 : cluster [DBG] osdmap e917: 8 total, 8 up, 8 in 2024-09-12T23:54:10.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:10 smithi136 bash[75164]: audit 2024-09-12T23:54:09.379310+0000 mon.a (mon.0) 1586 : audit [INF] from='client.? 172.21.15.136:0/2964623572' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:54:10.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:10 smithi136 bash[76371]: cluster 2024-09-12T23:54:09.377162+0000 mon.a (mon.0) 1585 : cluster [DBG] osdmap e917: 8 total, 8 up, 8 in 2024-09-12T23:54:10.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:10 smithi136 bash[76371]: audit 2024-09-12T23:54:09.379310+0000 mon.a (mon.0) 1586 : audit [INF] from='client.? 172.21.15.136:0/2964623572' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:54:10.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:10 smithi181 bash[67587]: cluster 2024-09-12T23:54:09.377162+0000 mon.a (mon.0) 1585 : cluster [DBG] osdmap e917: 8 total, 8 up, 8 in 2024-09-12T23:54:10.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:10 smithi181 bash[67587]: audit 2024-09-12T23:54:09.379310+0000 mon.a (mon.0) 1586 : audit [INF] from='client.? 172.21.15.136:0/2964623572' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:54:11.404 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestCls2PCQueue.AsyncConsumer (3038 ms) 2024-09-12T23:54:11.404 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestCls2PCQueue.MultiProducerConsumer 2024-09-12T23:54:11.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:11 smithi136 bash[75164]: audit 2024-09-12T23:54:10.379010+0000 mon.a (mon.0) 1587 : audit [INF] from='client.? 172.21.15.136:0/2964623572' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-18","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:54:11.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:11 smithi136 bash[75164]: cluster 2024-09-12T23:54:10.379479+0000 mon.a (mon.0) 1588 : cluster [DBG] osdmap e918: 8 total, 8 up, 8 in 2024-09-12T23:54:11.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:11 smithi136 bash[75164]: cluster 2024-09-12T23:54:10.948477+0000 mgr.y (mgr.24661) 512 : cluster [DBG] pgmap v991: 137 pgs: 3 creating+peering, 20 unknown, 114 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:54:11.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:11 smithi136 bash[76371]: audit 2024-09-12T23:54:10.379010+0000 mon.a (mon.0) 1587 : audit [INF] from='client.? 172.21.15.136:0/2964623572' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-18","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:54:11.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:11 smithi136 bash[76371]: cluster 2024-09-12T23:54:10.379479+0000 mon.a (mon.0) 1588 : cluster [DBG] osdmap e918: 8 total, 8 up, 8 in 2024-09-12T23:54:11.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:11 smithi136 bash[76371]: cluster 2024-09-12T23:54:10.948477+0000 mgr.y (mgr.24661) 512 : cluster [DBG] pgmap v991: 137 pgs: 3 creating+peering, 20 unknown, 114 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:54:11.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:11 smithi181 bash[67587]: audit 2024-09-12T23:54:10.379010+0000 mon.a (mon.0) 1587 : audit [INF] from='client.? 172.21.15.136:0/2964623572' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-18","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:54:11.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:11 smithi181 bash[67587]: cluster 2024-09-12T23:54:10.379479+0000 mon.a (mon.0) 1588 : cluster [DBG] osdmap e918: 8 total, 8 up, 8 in 2024-09-12T23:54:11.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:11 smithi181 bash[67587]: cluster 2024-09-12T23:54:10.948477+0000 mgr.y (mgr.24661) 512 : cluster [DBG] pgmap v991: 137 pgs: 3 creating+peering, 20 unknown, 114 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:54:12.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:12 smithi181 bash[67587]: cluster 2024-09-12T23:54:11.403687+0000 mon.a (mon.0) 1589 : cluster [DBG] osdmap e919: 8 total, 8 up, 8 in 2024-09-12T23:54:12.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:54:12 smithi181 bash[95505]: ts=2024-09-12T23:54:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:54:12.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:12 smithi136 bash[75164]: cluster 2024-09-12T23:54:11.403687+0000 mon.a (mon.0) 1589 : cluster [DBG] osdmap e919: 8 total, 8 up, 8 in 2024-09-12T23:54:12.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:12 smithi136 bash[76371]: cluster 2024-09-12T23:54:11.403687+0000 mon.a (mon.0) 1589 : cluster [DBG] osdmap e919: 8 total, 8 up, 8 in 2024-09-12T23:54:13.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:13 smithi136 bash[76371]: cluster 2024-09-12T23:54:12.413313+0000 mon.a (mon.0) 1590 : cluster [DBG] osdmap e920: 8 total, 8 up, 8 in 2024-09-12T23:54:13.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:13 smithi136 bash[76371]: audit 2024-09-12T23:54:12.416309+0000 mon.c (mon.2) 31 : audit [INF] from='client.? 172.21.15.136:0/3416323924' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:54:13.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:13 smithi136 bash[76371]: audit 2024-09-12T23:54:12.417024+0000 mon.a (mon.0) 1591 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:54:13.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:13 smithi136 bash[76371]: cluster 2024-09-12T23:54:12.949452+0000 mgr.y (mgr.24661) 513 : cluster [DBG] pgmap v994: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:54:13.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:13 smithi136 bash[75164]: cluster 2024-09-12T23:54:12.413313+0000 mon.a (mon.0) 1590 : cluster [DBG] osdmap e920: 8 total, 8 up, 8 in 2024-09-12T23:54:13.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:13 smithi136 bash[75164]: audit 2024-09-12T23:54:12.416309+0000 mon.c (mon.2) 31 : audit [INF] from='client.? 172.21.15.136:0/3416323924' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:54:13.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:13 smithi136 bash[75164]: audit 2024-09-12T23:54:12.417024+0000 mon.a (mon.0) 1591 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:54:13.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:13 smithi136 bash[75164]: cluster 2024-09-12T23:54:12.949452+0000 mgr.y (mgr.24661) 513 : cluster [DBG] pgmap v994: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:54:13.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:13 smithi181 bash[67587]: cluster 2024-09-12T23:54:12.413313+0000 mon.a (mon.0) 1590 : cluster [DBG] osdmap e920: 8 total, 8 up, 8 in 2024-09-12T23:54:13.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:13 smithi181 bash[67587]: audit 2024-09-12T23:54:12.416309+0000 mon.c (mon.2) 31 : audit [INF] from='client.? 172.21.15.136:0/3416323924' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:54:13.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:13 smithi181 bash[67587]: audit 2024-09-12T23:54:12.417024+0000 mon.a (mon.0) 1591 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:54:13.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:13 smithi181 bash[67587]: cluster 2024-09-12T23:54:12.949452+0000 mgr.y (mgr.24661) 513 : cluster [DBG] pgmap v994: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:54:14.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:14 smithi136 bash[75164]: cluster 2024-09-12T23:54:13.406762+0000 mon.a (mon.0) 1592 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:54:14.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:14 smithi136 bash[75164]: audit 2024-09-12T23:54:13.426212+0000 mon.a (mon.0) 1593 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-19","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:54:14.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:14 smithi136 bash[75164]: cluster 2024-09-12T23:54:13.426664+0000 mon.a (mon.0) 1594 : cluster [DBG] osdmap e921: 8 total, 8 up, 8 in 2024-09-12T23:54:14.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:14 smithi136 bash[76371]: cluster 2024-09-12T23:54:13.406762+0000 mon.a (mon.0) 1592 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:54:14.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:14 smithi136 bash[76371]: audit 2024-09-12T23:54:13.426212+0000 mon.a (mon.0) 1593 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-19","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:54:14.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:14 smithi136 bash[76371]: cluster 2024-09-12T23:54:13.426664+0000 mon.a (mon.0) 1594 : cluster [DBG] osdmap e921: 8 total, 8 up, 8 in 2024-09-12T23:54:14.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:14 smithi181 bash[67587]: cluster 2024-09-12T23:54:13.406762+0000 mon.a (mon.0) 1592 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:54:14.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:14 smithi181 bash[67587]: audit 2024-09-12T23:54:13.426212+0000 mon.a (mon.0) 1593 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-19","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:54:14.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:14 smithi181 bash[67587]: cluster 2024-09-12T23:54:13.426664+0000 mon.a (mon.0) 1594 : cluster [DBG] osdmap e921: 8 total, 8 up, 8 in 2024-09-12T23:54:15.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:15 smithi136 bash[75164]: cluster 2024-09-12T23:54:14.422144+0000 mon.a (mon.0) 1595 : cluster [DBG] osdmap e922: 8 total, 8 up, 8 in 2024-09-12T23:54:15.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:15 smithi136 bash[75164]: cluster 2024-09-12T23:54:14.949999+0000 mgr.y (mgr.24661) 514 : cluster [DBG] pgmap v997: 137 pgs: 4 creating+peering, 23 unknown, 110 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:54:15.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:15 smithi136 bash[76371]: cluster 2024-09-12T23:54:14.422144+0000 mon.a (mon.0) 1595 : cluster [DBG] osdmap e922: 8 total, 8 up, 8 in 2024-09-12T23:54:15.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:15 smithi136 bash[76371]: cluster 2024-09-12T23:54:14.949999+0000 mgr.y (mgr.24661) 514 : cluster [DBG] pgmap v997: 137 pgs: 4 creating+peering, 23 unknown, 110 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:54:15.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:15 smithi181 bash[67587]: cluster 2024-09-12T23:54:14.422144+0000 mon.a (mon.0) 1595 : cluster [DBG] osdmap e922: 8 total, 8 up, 8 in 2024-09-12T23:54:15.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:15 smithi181 bash[67587]: cluster 2024-09-12T23:54:14.949999+0000 mgr.y (mgr.24661) 514 : cluster [DBG] pgmap v997: 137 pgs: 4 creating+peering, 23 unknown, 110 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:54:16.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:16 smithi136 bash[75164]: cluster 2024-09-12T23:54:15.427255+0000 mon.a (mon.0) 1596 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:54:16.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:16 smithi136 bash[75164]: cluster 2024-09-12T23:54:15.427287+0000 mon.a (mon.0) 1597 : cluster [INF] Cluster is now healthy 2024-09-12T23:54:16.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:16 smithi136 bash[76371]: cluster 2024-09-12T23:54:15.427255+0000 mon.a (mon.0) 1596 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:54:16.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:16 smithi136 bash[76371]: cluster 2024-09-12T23:54:15.427287+0000 mon.a (mon.0) 1597 : cluster [INF] Cluster is now healthy 2024-09-12T23:54:16.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:16 smithi181 bash[67587]: cluster 2024-09-12T23:54:15.427255+0000 mon.a (mon.0) 1596 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:54:16.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:16 smithi181 bash[67587]: cluster 2024-09-12T23:54:15.427287+0000 mon.a (mon.0) 1597 : cluster [INF] Cluster is now healthy 2024-09-12T23:54:17.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:17 smithi136 bash[75164]: cluster 2024-09-12T23:54:16.950996+0000 mgr.y (mgr.24661) 515 : cluster [DBG] pgmap v998: 137 pgs: 4 creating+peering, 133 active+clean; 1.0 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.0 MiB/s rd, 3.3 MiB/s wr, 3.89k op/s 2024-09-12T23:54:17.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:17 smithi136 bash[76371]: cluster 2024-09-12T23:54:16.950996+0000 mgr.y (mgr.24661) 515 : cluster [DBG] pgmap v998: 137 pgs: 4 creating+peering, 133 active+clean; 1.0 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.0 MiB/s rd, 3.3 MiB/s wr, 3.89k op/s 2024-09-12T23:54:17.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:17 smithi181 bash[67587]: cluster 2024-09-12T23:54:16.950996+0000 mgr.y (mgr.24661) 515 : cluster [DBG] pgmap v998: 137 pgs: 4 creating+peering, 133 active+clean; 1.0 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.0 MiB/s rd, 3.3 MiB/s wr, 3.89k op/s 2024-09-12T23:54:19.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:19 smithi181 bash[67587]: cluster 2024-09-12T23:54:18.951591+0000 mgr.y (mgr.24661) 516 : cluster [DBG] pgmap v999: 137 pgs: 137 active+clean; 1.0 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.6 MiB/s rd, 2.8 MiB/s wr, 3.30k op/s 2024-09-12T23:54:19.411 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:19 smithi136 bash[75164]: cluster 2024-09-12T23:54:18.951591+0000 mgr.y (mgr.24661) 516 : cluster [DBG] pgmap v999: 137 pgs: 137 active+clean; 1.0 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.6 MiB/s rd, 2.8 MiB/s wr, 3.30k op/s 2024-09-12T23:54:19.411 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:19 smithi136 bash[76371]: cluster 2024-09-12T23:54:18.951591+0000 mgr.y (mgr.24661) 516 : cluster [DBG] pgmap v999: 137 pgs: 137 active+clean; 1.0 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.6 MiB/s rd, 2.8 MiB/s wr, 3.30k op/s 2024-09-12T23:54:19.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:54:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:54:19] "GET /metrics HTTP/1.1" 200 34086 "" "Prometheus/2.43.0" 2024-09-12T23:54:21.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:21 smithi181 bash[67587]: cluster 2024-09-12T23:54:20.952294+0000 mgr.y (mgr.24661) 517 : cluster [DBG] pgmap v1000: 137 pgs: 137 active+clean; 1.5 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 27 MiB/s rd, 5.2 MiB/s wr, 6.09k op/s 2024-09-12T23:54:21.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:21 smithi136 bash[75164]: cluster 2024-09-12T23:54:20.952294+0000 mgr.y (mgr.24661) 517 : cluster [DBG] pgmap v1000: 137 pgs: 137 active+clean; 1.5 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 27 MiB/s rd, 5.2 MiB/s wr, 6.09k op/s 2024-09-12T23:54:21.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:21 smithi136 bash[76371]: cluster 2024-09-12T23:54:20.952294+0000 mgr.y (mgr.24661) 517 : cluster [DBG] pgmap v1000: 137 pgs: 137 active+clean; 1.5 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 27 MiB/s rd, 5.2 MiB/s wr, 6.09k op/s 2024-09-12T23:54:22.600 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:54:22 smithi181 bash[95505]: ts=2024-09-12T23:54:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:54:23.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:23 smithi181 bash[67587]: cluster 2024-09-12T23:54:22.953087+0000 mgr.y (mgr.24661) 518 : cluster [DBG] pgmap v1001: 137 pgs: 137 active+clean; 1.5 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 22 MiB/s rd, 4.4 MiB/s wr, 5.12k op/s 2024-09-12T23:54:23.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:23 smithi136 bash[75164]: cluster 2024-09-12T23:54:22.953087+0000 mgr.y (mgr.24661) 518 : cluster [DBG] pgmap v1001: 137 pgs: 137 active+clean; 1.5 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 22 MiB/s rd, 4.4 MiB/s wr, 5.12k op/s 2024-09-12T23:54:23.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:23 smithi136 bash[76371]: cluster 2024-09-12T23:54:22.953087+0000 mgr.y (mgr.24661) 518 : cluster [DBG] pgmap v1001: 137 pgs: 137 active+clean; 1.5 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 22 MiB/s rd, 4.4 MiB/s wr, 5.12k op/s 2024-09-12T23:54:24.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:24 smithi181 bash[67587]: audit 2024-09-12T23:54:23.758923+0000 mon.a (mon.0) 1598 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:54:24.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:24 smithi136 bash[75164]: audit 2024-09-12T23:54:23.758923+0000 mon.a (mon.0) 1598 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:54:24.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:24 smithi136 bash[76371]: audit 2024-09-12T23:54:23.758923+0000 mon.a (mon.0) 1598 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:54:25.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:25 smithi181 bash[67587]: cluster 2024-09-12T23:54:24.953606+0000 mgr.y (mgr.24661) 519 : cluster [DBG] pgmap v1002: 137 pgs: 137 active+clean; 1.5 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 20 MiB/s rd, 4.0 MiB/s wr, 4.63k op/s 2024-09-12T23:54:25.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:25 smithi136 bash[76371]: cluster 2024-09-12T23:54:24.953606+0000 mgr.y (mgr.24661) 519 : cluster [DBG] pgmap v1002: 137 pgs: 137 active+clean; 1.5 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 20 MiB/s rd, 4.0 MiB/s wr, 4.63k op/s 2024-09-12T23:54:25.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:25 smithi136 bash[75164]: cluster 2024-09-12T23:54:24.953606+0000 mgr.y (mgr.24661) 519 : cluster [DBG] pgmap v1002: 137 pgs: 137 active+clean; 1.5 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 20 MiB/s rd, 4.0 MiB/s wr, 4.63k op/s 2024-09-12T23:54:27.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:27 smithi181 bash[67587]: cluster 2024-09-12T23:54:26.954411+0000 mgr.y (mgr.24661) 520 : cluster [DBG] pgmap v1003: 137 pgs: 137 active+clean; 1.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 30 MiB/s rd, 4.6 MiB/s wr, 5.38k op/s 2024-09-12T23:54:27.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:27 smithi136 bash[75164]: cluster 2024-09-12T23:54:26.954411+0000 mgr.y (mgr.24661) 520 : cluster [DBG] pgmap v1003: 137 pgs: 137 active+clean; 1.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 30 MiB/s rd, 4.6 MiB/s wr, 5.38k op/s 2024-09-12T23:54:27.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:27 smithi136 bash[76371]: cluster 2024-09-12T23:54:26.954411+0000 mgr.y (mgr.24661) 520 : cluster [DBG] pgmap v1003: 137 pgs: 137 active+clean; 1.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 30 MiB/s rd, 4.6 MiB/s wr, 5.38k op/s 2024-09-12T23:54:29.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:29 smithi181 bash[67587]: cluster 2024-09-12T23:54:28.955001+0000 mgr.y (mgr.24661) 521 : cluster [DBG] pgmap v1004: 137 pgs: 137 active+clean; 1.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 26 MiB/s rd, 3.0 MiB/s wr, 3.58k op/s 2024-09-12T23:54:29.411 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:29 smithi136 bash[75164]: cluster 2024-09-12T23:54:28.955001+0000 mgr.y (mgr.24661) 521 : cluster [DBG] pgmap v1004: 137 pgs: 137 active+clean; 1.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 26 MiB/s rd, 3.0 MiB/s wr, 3.58k op/s 2024-09-12T23:54:29.411 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:29 smithi136 bash[76371]: cluster 2024-09-12T23:54:28.955001+0000 mgr.y (mgr.24661) 521 : cluster [DBG] pgmap v1004: 137 pgs: 137 active+clean; 1.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 26 MiB/s rd, 3.0 MiB/s wr, 3.58k op/s 2024-09-12T23:54:29.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:54:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:54:29] "GET /metrics HTTP/1.1" 200 36512 "" "Prometheus/2.43.0" 2024-09-12T23:54:30.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:30 smithi181 bash[67587]: audit 2024-09-12T23:54:29.652494+0000 mon.a (mon.0) 1599 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:54:30.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:30 smithi136 bash[75164]: audit 2024-09-12T23:54:29.652494+0000 mon.a (mon.0) 1599 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:54:30.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:30 smithi136 bash[76371]: audit 2024-09-12T23:54:29.652494+0000 mon.a (mon.0) 1599 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:54:31.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:31 smithi181 bash[67587]: audit 2024-09-12T23:54:30.226659+0000 mon.a (mon.0) 1600 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:54:31.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:31 smithi181 bash[67587]: audit 2024-09-12T23:54:30.227482+0000 mon.a (mon.0) 1601 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:54:31.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:31 smithi181 bash[67587]: audit 2024-09-12T23:54:30.233708+0000 mon.a (mon.0) 1602 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:54:31.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:31 smithi181 bash[67587]: cluster 2024-09-12T23:54:30.955720+0000 mgr.y (mgr.24661) 522 : cluster [DBG] pgmap v1005: 137 pgs: 137 active+clean; 1.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 31 MiB/s rd, 3.7 MiB/s wr, 4.41k op/s 2024-09-12T23:54:31.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:31 smithi136 bash[75164]: audit 2024-09-12T23:54:30.226659+0000 mon.a (mon.0) 1600 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:54:31.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:31 smithi136 bash[75164]: audit 2024-09-12T23:54:30.227482+0000 mon.a (mon.0) 1601 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:54:31.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:31 smithi136 bash[75164]: audit 2024-09-12T23:54:30.233708+0000 mon.a (mon.0) 1602 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:54:31.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:31 smithi136 bash[75164]: cluster 2024-09-12T23:54:30.955720+0000 mgr.y (mgr.24661) 522 : cluster [DBG] pgmap v1005: 137 pgs: 137 active+clean; 1.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 31 MiB/s rd, 3.7 MiB/s wr, 4.41k op/s 2024-09-12T23:54:31.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:31 smithi136 bash[76371]: audit 2024-09-12T23:54:30.226659+0000 mon.a (mon.0) 1600 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:54:31.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:31 smithi136 bash[76371]: audit 2024-09-12T23:54:30.227482+0000 mon.a (mon.0) 1601 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:54:31.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:31 smithi136 bash[76371]: audit 2024-09-12T23:54:30.233708+0000 mon.a (mon.0) 1602 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:54:31.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:31 smithi136 bash[76371]: cluster 2024-09-12T23:54:30.955720+0000 mgr.y (mgr.24661) 522 : cluster [DBG] pgmap v1005: 137 pgs: 137 active+clean; 1.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 31 MiB/s rd, 3.7 MiB/s wr, 4.41k op/s 2024-09-12T23:54:32.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:54:32 smithi181 bash[95505]: ts=2024-09-12T23:54:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:54:33.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:33 smithi181 bash[67587]: cluster 2024-09-12T23:54:32.956351+0000 mgr.y (mgr.24661) 523 : cluster [DBG] pgmap v1006: 137 pgs: 137 active+clean; 1.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 17 MiB/s rd, 1.8 MiB/s wr, 2.14k op/s 2024-09-12T23:54:33.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:33 smithi136 bash[76371]: cluster 2024-09-12T23:54:32.956351+0000 mgr.y (mgr.24661) 523 : cluster [DBG] pgmap v1006: 137 pgs: 137 active+clean; 1.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 17 MiB/s rd, 1.8 MiB/s wr, 2.14k op/s 2024-09-12T23:54:33.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:33 smithi136 bash[75164]: cluster 2024-09-12T23:54:32.956351+0000 mgr.y (mgr.24661) 523 : cluster [DBG] pgmap v1006: 137 pgs: 137 active+clean; 1.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 17 MiB/s rd, 1.8 MiB/s wr, 2.14k op/s 2024-09-12T23:54:35.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:35 smithi181 bash[67587]: cluster 2024-09-12T23:54:34.957121+0000 mgr.y (mgr.24661) 524 : cluster [DBG] pgmap v1007: 137 pgs: 137 active+clean; 1.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 17 MiB/s rd, 1.8 MiB/s wr, 2.14k op/s 2024-09-12T23:54:35.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:35 smithi136 bash[75164]: cluster 2024-09-12T23:54:34.957121+0000 mgr.y (mgr.24661) 524 : cluster [DBG] pgmap v1007: 137 pgs: 137 active+clean; 1.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 17 MiB/s rd, 1.8 MiB/s wr, 2.14k op/s 2024-09-12T23:54:35.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:35 smithi136 bash[76371]: cluster 2024-09-12T23:54:34.957121+0000 mgr.y (mgr.24661) 524 : cluster [DBG] pgmap v1007: 137 pgs: 137 active+clean; 1.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 17 MiB/s rd, 1.8 MiB/s wr, 2.14k op/s 2024-09-12T23:54:37.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:37 smithi181 bash[67587]: cluster 2024-09-12T23:54:36.958125+0000 mgr.y (mgr.24661) 525 : cluster [DBG] pgmap v1008: 137 pgs: 137 active+clean; 1.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 30 MiB/s rd, 2.4 MiB/s wr, 2.93k op/s 2024-09-12T23:54:37.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:37 smithi136 bash[76371]: cluster 2024-09-12T23:54:36.958125+0000 mgr.y (mgr.24661) 525 : cluster [DBG] pgmap v1008: 137 pgs: 137 active+clean; 1.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 30 MiB/s rd, 2.4 MiB/s wr, 2.93k op/s 2024-09-12T23:54:37.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:37 smithi136 bash[75164]: cluster 2024-09-12T23:54:36.958125+0000 mgr.y (mgr.24661) 525 : cluster [DBG] pgmap v1008: 137 pgs: 137 active+clean; 1.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 30 MiB/s rd, 2.4 MiB/s wr, 2.93k op/s 2024-09-12T23:54:39.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:38 smithi181 bash[67587]: audit 2024-09-12T23:54:38.759611+0000 mon.a (mon.0) 1603 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:54:39.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:38 smithi136 bash[75164]: audit 2024-09-12T23:54:38.759611+0000 mon.a (mon.0) 1603 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:54:39.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:38 smithi136 bash[76371]: audit 2024-09-12T23:54:38.759611+0000 mon.a (mon.0) 1603 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:54:39.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:54:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:54:39] "GET /metrics HTTP/1.1" 200 36511 "" "Prometheus/2.43.0" 2024-09-12T23:54:40.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:39 smithi181 bash[67587]: cluster 2024-09-12T23:54:38.958838+0000 mgr.y (mgr.24661) 526 : cluster [DBG] pgmap v1009: 137 pgs: 137 active+clean; 1.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 18 MiB/s rd, 1.3 MiB/s wr, 1.61k op/s 2024-09-12T23:54:40.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:39 smithi136 bash[75164]: cluster 2024-09-12T23:54:38.958838+0000 mgr.y (mgr.24661) 526 : cluster [DBG] pgmap v1009: 137 pgs: 137 active+clean; 1.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 18 MiB/s rd, 1.3 MiB/s wr, 1.61k op/s 2024-09-12T23:54:40.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:39 smithi136 bash[76371]: cluster 2024-09-12T23:54:38.958838+0000 mgr.y (mgr.24661) 526 : cluster [DBG] pgmap v1009: 137 pgs: 137 active+clean; 1.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 18 MiB/s rd, 1.3 MiB/s wr, 1.61k op/s 2024-09-12T23:54:41.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:41 smithi181 bash[67587]: cluster 2024-09-12T23:54:40.960030+0000 mgr.y (mgr.24661) 527 : cluster [DBG] pgmap v1010: 137 pgs: 137 active+clean; 1.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 33 MiB/s rd, 1.7 MiB/s wr, 2.29k op/s 2024-09-12T23:54:41.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:41 smithi136 bash[75164]: cluster 2024-09-12T23:54:40.960030+0000 mgr.y (mgr.24661) 527 : cluster [DBG] pgmap v1010: 137 pgs: 137 active+clean; 1.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 33 MiB/s rd, 1.7 MiB/s wr, 2.29k op/s 2024-09-12T23:54:41.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:41 smithi136 bash[76371]: cluster 2024-09-12T23:54:40.960030+0000 mgr.y (mgr.24661) 527 : cluster [DBG] pgmap v1010: 137 pgs: 137 active+clean; 1.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 33 MiB/s rd, 1.7 MiB/s wr, 2.29k op/s 2024-09-12T23:54:42.032 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestCls2PCQueue.MultiProducerConsumer (30628 ms) 2024-09-12T23:54:42.032 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestCls2PCQueue.ReserveSpillover 2024-09-12T23:54:42.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:54:42 smithi181 bash[95505]: ts=2024-09-12T23:54:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:54:43.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:43 smithi181 bash[67587]: cluster 2024-09-12T23:54:42.029242+0000 mon.a (mon.0) 1604 : cluster [DBG] osdmap e923: 8 total, 8 up, 8 in 2024-09-12T23:54:43.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:43 smithi181 bash[67587]: cluster 2024-09-12T23:54:42.960876+0000 mgr.y (mgr.24661) 528 : cluster [DBG] pgmap v1012: 105 pgs: 105 active+clean; 583 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:54:43.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:43 smithi136 bash[75164]: cluster 2024-09-12T23:54:42.029242+0000 mon.a (mon.0) 1604 : cluster [DBG] osdmap e923: 8 total, 8 up, 8 in 2024-09-12T23:54:43.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:43 smithi136 bash[75164]: cluster 2024-09-12T23:54:42.960876+0000 mgr.y (mgr.24661) 528 : cluster [DBG] pgmap v1012: 105 pgs: 105 active+clean; 583 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:54:43.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:43 smithi136 bash[76371]: cluster 2024-09-12T23:54:42.029242+0000 mon.a (mon.0) 1604 : cluster [DBG] osdmap e923: 8 total, 8 up, 8 in 2024-09-12T23:54:43.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:43 smithi136 bash[76371]: cluster 2024-09-12T23:54:42.960876+0000 mgr.y (mgr.24661) 528 : cluster [DBG] pgmap v1012: 105 pgs: 105 active+clean; 583 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:54:44.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:44 smithi181 bash[67587]: cluster 2024-09-12T23:54:43.045974+0000 mon.a (mon.0) 1605 : cluster [DBG] osdmap e924: 8 total, 8 up, 8 in 2024-09-12T23:54:44.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:44 smithi181 bash[67587]: audit 2024-09-12T23:54:43.049837+0000 mon.c (mon.2) 32 : audit [INF] from='client.? 172.21.15.136:0/10132978' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:54:44.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:44 smithi181 bash[67587]: audit 2024-09-12T23:54:43.054518+0000 mon.a (mon.0) 1606 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:54:44.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:44 smithi136 bash[75164]: cluster 2024-09-12T23:54:43.045974+0000 mon.a (mon.0) 1605 : cluster [DBG] osdmap e924: 8 total, 8 up, 8 in 2024-09-12T23:54:44.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:44 smithi136 bash[75164]: audit 2024-09-12T23:54:43.049837+0000 mon.c (mon.2) 32 : audit [INF] from='client.? 172.21.15.136:0/10132978' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:54:44.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:44 smithi136 bash[75164]: audit 2024-09-12T23:54:43.054518+0000 mon.a (mon.0) 1606 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:54:44.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:44 smithi136 bash[76371]: cluster 2024-09-12T23:54:43.045974+0000 mon.a (mon.0) 1605 : cluster [DBG] osdmap e924: 8 total, 8 up, 8 in 2024-09-12T23:54:44.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:44 smithi136 bash[76371]: audit 2024-09-12T23:54:43.049837+0000 mon.c (mon.2) 32 : audit [INF] from='client.? 172.21.15.136:0/10132978' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:54:44.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:44 smithi136 bash[76371]: audit 2024-09-12T23:54:43.054518+0000 mon.a (mon.0) 1606 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:54:45.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:45 smithi181 bash[67587]: audit 2024-09-12T23:54:44.044975+0000 mon.a (mon.0) 1607 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-20","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:54:45.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:45 smithi181 bash[67587]: cluster 2024-09-12T23:54:44.045343+0000 mon.a (mon.0) 1608 : cluster [DBG] osdmap e925: 8 total, 8 up, 8 in 2024-09-12T23:54:45.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:45 smithi181 bash[67587]: cluster 2024-09-12T23:54:44.961705+0000 mgr.y (mgr.24661) 529 : cluster [DBG] pgmap v1015: 137 pgs: 8 creating+peering, 19 unknown, 110 active+clean; 588 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:54:45.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:45 smithi136 bash[75164]: audit 2024-09-12T23:54:44.044975+0000 mon.a (mon.0) 1607 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-20","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:54:45.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:45 smithi136 bash[75164]: cluster 2024-09-12T23:54:44.045343+0000 mon.a (mon.0) 1608 : cluster [DBG] osdmap e925: 8 total, 8 up, 8 in 2024-09-12T23:54:45.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:45 smithi136 bash[75164]: cluster 2024-09-12T23:54:44.961705+0000 mgr.y (mgr.24661) 529 : cluster [DBG] pgmap v1015: 137 pgs: 8 creating+peering, 19 unknown, 110 active+clean; 588 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:54:45.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:45 smithi136 bash[76371]: audit 2024-09-12T23:54:44.044975+0000 mon.a (mon.0) 1607 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-20","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:54:45.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:45 smithi136 bash[76371]: cluster 2024-09-12T23:54:44.045343+0000 mon.a (mon.0) 1608 : cluster [DBG] osdmap e925: 8 total, 8 up, 8 in 2024-09-12T23:54:45.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:45 smithi136 bash[76371]: cluster 2024-09-12T23:54:44.961705+0000 mgr.y (mgr.24661) 529 : cluster [DBG] pgmap v1015: 137 pgs: 8 creating+peering, 19 unknown, 110 active+clean; 588 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:54:46.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:46 smithi181 bash[67587]: cluster 2024-09-12T23:54:45.074351+0000 mon.a (mon.0) 1609 : cluster [DBG] osdmap e926: 8 total, 8 up, 8 in 2024-09-12T23:54:46.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:46 smithi136 bash[75164]: cluster 2024-09-12T23:54:45.074351+0000 mon.a (mon.0) 1609 : cluster [DBG] osdmap e926: 8 total, 8 up, 8 in 2024-09-12T23:54:46.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:46 smithi136 bash[76371]: cluster 2024-09-12T23:54:45.074351+0000 mon.a (mon.0) 1609 : cluster [DBG] osdmap e926: 8 total, 8 up, 8 in 2024-09-12T23:54:48.090 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestCls2PCQueue.ReserveSpillover (6056 ms) 2024-09-12T23:54:48.090 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestCls2PCQueue.CommitSpillover 2024-09-12T23:54:48.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:48 smithi181 bash[67587]: cluster 2024-09-12T23:54:46.963009+0000 mgr.y (mgr.24661) 530 : cluster [DBG] pgmap v1017: 137 pgs: 8 creating+peering, 4 unknown, 125 active+clean; 588 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 141 KiB/s rd, 142 KiB/s wr, 97 op/s 2024-09-12T23:54:48.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:48 smithi136 bash[75164]: cluster 2024-09-12T23:54:46.963009+0000 mgr.y (mgr.24661) 530 : cluster [DBG] pgmap v1017: 137 pgs: 8 creating+peering, 4 unknown, 125 active+clean; 588 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 141 KiB/s rd, 142 KiB/s wr, 97 op/s 2024-09-12T23:54:48.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:48 smithi136 bash[76371]: cluster 2024-09-12T23:54:46.963009+0000 mgr.y (mgr.24661) 530 : cluster [DBG] pgmap v1017: 137 pgs: 8 creating+peering, 4 unknown, 125 active+clean; 588 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 141 KiB/s rd, 142 KiB/s wr, 97 op/s 2024-09-12T23:54:49.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:49 smithi181 bash[67587]: cluster 2024-09-12T23:54:48.086660+0000 mon.a (mon.0) 1610 : cluster [DBG] osdmap e927: 8 total, 8 up, 8 in 2024-09-12T23:54:49.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:49 smithi136 bash[76371]: cluster 2024-09-12T23:54:48.086660+0000 mon.a (mon.0) 1610 : cluster [DBG] osdmap e927: 8 total, 8 up, 8 in 2024-09-12T23:54:49.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:49 smithi136 bash[75164]: cluster 2024-09-12T23:54:48.086660+0000 mon.a (mon.0) 1610 : cluster [DBG] osdmap e927: 8 total, 8 up, 8 in 2024-09-12T23:54:49.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:54:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:54:49] "GET /metrics HTTP/1.1" 200 36511 "" "Prometheus/2.43.0" 2024-09-12T23:54:50.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:50 smithi136 bash[75164]: cluster 2024-09-12T23:54:48.963839+0000 mgr.y (mgr.24661) 531 : cluster [DBG] pgmap v1019: 105 pgs: 105 active+clean; 583 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:54:50.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:50 smithi136 bash[75164]: cluster 2024-09-12T23:54:49.097331+0000 mon.a (mon.0) 1611 : cluster [DBG] osdmap e928: 8 total, 8 up, 8 in 2024-09-12T23:54:50.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:50 smithi136 bash[75164]: audit 2024-09-12T23:54:49.099204+0000 mon.c (mon.2) 33 : audit [INF] from='client.? 172.21.15.136:0/2534121046' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-21","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:54:50.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:50 smithi136 bash[75164]: audit 2024-09-12T23:54:49.100113+0000 mon.a (mon.0) 1612 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-21","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:54:50.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:50 smithi136 bash[76371]: cluster 2024-09-12T23:54:48.963839+0000 mgr.y (mgr.24661) 531 : cluster [DBG] pgmap v1019: 105 pgs: 105 active+clean; 583 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:54:50.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:50 smithi136 bash[76371]: cluster 2024-09-12T23:54:49.097331+0000 mon.a (mon.0) 1611 : cluster [DBG] osdmap e928: 8 total, 8 up, 8 in 2024-09-12T23:54:50.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:50 smithi136 bash[76371]: audit 2024-09-12T23:54:49.099204+0000 mon.c (mon.2) 33 : audit [INF] from='client.? 172.21.15.136:0/2534121046' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-21","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:54:50.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:50 smithi136 bash[76371]: audit 2024-09-12T23:54:49.100113+0000 mon.a (mon.0) 1612 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-21","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:54:50.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:50 smithi181 bash[67587]: cluster 2024-09-12T23:54:48.963839+0000 mgr.y (mgr.24661) 531 : cluster [DBG] pgmap v1019: 105 pgs: 105 active+clean; 583 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:54:50.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:50 smithi181 bash[67587]: cluster 2024-09-12T23:54:49.097331+0000 mon.a (mon.0) 1611 : cluster [DBG] osdmap e928: 8 total, 8 up, 8 in 2024-09-12T23:54:50.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:50 smithi181 bash[67587]: audit 2024-09-12T23:54:49.099204+0000 mon.c (mon.2) 33 : audit [INF] from='client.? 172.21.15.136:0/2534121046' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-21","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:54:50.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:50 smithi181 bash[67587]: audit 2024-09-12T23:54:49.100113+0000 mon.a (mon.0) 1612 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-21","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:54:51.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:51 smithi136 bash[76371]: audit 2024-09-12T23:54:50.098960+0000 mon.a (mon.0) 1613 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-21","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:54:51.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:51 smithi136 bash[76371]: cluster 2024-09-12T23:54:50.099410+0000 mon.a (mon.0) 1614 : cluster [DBG] osdmap e929: 8 total, 8 up, 8 in 2024-09-12T23:54:51.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:51 smithi136 bash[75164]: audit 2024-09-12T23:54:50.098960+0000 mon.a (mon.0) 1613 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-21","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:54:51.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:51 smithi136 bash[75164]: cluster 2024-09-12T23:54:50.099410+0000 mon.a (mon.0) 1614 : cluster [DBG] osdmap e929: 8 total, 8 up, 8 in 2024-09-12T23:54:51.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:51 smithi181 bash[67587]: audit 2024-09-12T23:54:50.098960+0000 mon.a (mon.0) 1613 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-21","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:54:51.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:51 smithi181 bash[67587]: cluster 2024-09-12T23:54:50.099410+0000 mon.a (mon.0) 1614 : cluster [DBG] osdmap e929: 8 total, 8 up, 8 in 2024-09-12T23:54:52.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:52 smithi136 bash[75164]: cluster 2024-09-12T23:54:50.964758+0000 mgr.y (mgr.24661) 532 : cluster [DBG] pgmap v1022: 137 pgs: 5 creating+peering, 19 unknown, 113 active+clean; 583 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:54:52.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:52 smithi136 bash[75164]: cluster 2024-09-12T23:54:51.126055+0000 mon.a (mon.0) 1615 : cluster [DBG] osdmap e930: 8 total, 8 up, 8 in 2024-09-12T23:54:52.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:52 smithi136 bash[76371]: cluster 2024-09-12T23:54:50.964758+0000 mgr.y (mgr.24661) 532 : cluster [DBG] pgmap v1022: 137 pgs: 5 creating+peering, 19 unknown, 113 active+clean; 583 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:54:52.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:52 smithi136 bash[76371]: cluster 2024-09-12T23:54:51.126055+0000 mon.a (mon.0) 1615 : cluster [DBG] osdmap e930: 8 total, 8 up, 8 in 2024-09-12T23:54:52.597 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:54:52 smithi181 bash[95505]: ts=2024-09-12T23:54:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:54:52.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:52 smithi181 bash[67587]: cluster 2024-09-12T23:54:50.964758+0000 mgr.y (mgr.24661) 532 : cluster [DBG] pgmap v1022: 137 pgs: 5 creating+peering, 19 unknown, 113 active+clean; 583 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:54:52.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:52 smithi181 bash[67587]: cluster 2024-09-12T23:54:51.126055+0000 mon.a (mon.0) 1615 : cluster [DBG] osdmap e930: 8 total, 8 up, 8 in 2024-09-12T23:54:54.142 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestCls2PCQueue.CommitSpillover (6053 ms) 2024-09-12T23:54:54.142 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestCls2PCQueue.AbortSpillover 2024-09-12T23:54:54.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:54 smithi136 bash[75164]: cluster 2024-09-12T23:54:52.966094+0000 mgr.y (mgr.24661) 533 : cluster [DBG] pgmap v1024: 137 pgs: 5 creating+peering, 2 unknown, 130 active+clean; 626 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.1 MiB/s rd, 4.4 MiB/s wr, 1.15k op/s 2024-09-12T23:54:54.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:54 smithi136 bash[75164]: audit 2024-09-12T23:54:53.760067+0000 mon.a (mon.0) 1616 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:54:54.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:54 smithi136 bash[76371]: cluster 2024-09-12T23:54:52.966094+0000 mgr.y (mgr.24661) 533 : cluster [DBG] pgmap v1024: 137 pgs: 5 creating+peering, 2 unknown, 130 active+clean; 626 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.1 MiB/s rd, 4.4 MiB/s wr, 1.15k op/s 2024-09-12T23:54:54.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:54 smithi136 bash[76371]: audit 2024-09-12T23:54:53.760067+0000 mon.a (mon.0) 1616 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:54:54.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:54 smithi181 bash[67587]: cluster 2024-09-12T23:54:52.966094+0000 mgr.y (mgr.24661) 533 : cluster [DBG] pgmap v1024: 137 pgs: 5 creating+peering, 2 unknown, 130 active+clean; 626 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.1 MiB/s rd, 4.4 MiB/s wr, 1.15k op/s 2024-09-12T23:54:54.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:54 smithi181 bash[67587]: audit 2024-09-12T23:54:53.760067+0000 mon.a (mon.0) 1616 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:54:55.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:55 smithi136 bash[75164]: cluster 2024-09-12T23:54:54.135483+0000 mon.a (mon.0) 1617 : cluster [DBG] osdmap e931: 8 total, 8 up, 8 in 2024-09-12T23:54:55.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:55 smithi136 bash[76371]: cluster 2024-09-12T23:54:54.135483+0000 mon.a (mon.0) 1617 : cluster [DBG] osdmap e931: 8 total, 8 up, 8 in 2024-09-12T23:54:55.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:55 smithi181 bash[67587]: cluster 2024-09-12T23:54:54.135483+0000 mon.a (mon.0) 1617 : cluster [DBG] osdmap e931: 8 total, 8 up, 8 in 2024-09-12T23:54:56.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:56 smithi136 bash[75164]: cluster 2024-09-12T23:54:54.966758+0000 mgr.y (mgr.24661) 534 : cluster [DBG] pgmap v1026: 105 pgs: 105 active+clean; 583 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:54:56.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:56 smithi136 bash[75164]: cluster 2024-09-12T23:54:55.148097+0000 mon.a (mon.0) 1618 : cluster [DBG] osdmap e932: 8 total, 8 up, 8 in 2024-09-12T23:54:56.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:56 smithi136 bash[75164]: audit 2024-09-12T23:54:55.154305+0000 mon.c (mon.2) 34 : audit [INF] from='client.? 172.21.15.136:0/1357578292' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-22","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:54:56.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:56 smithi136 bash[75164]: audit 2024-09-12T23:54:55.156500+0000 mon.a (mon.0) 1619 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-22","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:54:56.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:56 smithi136 bash[76371]: cluster 2024-09-12T23:54:54.966758+0000 mgr.y (mgr.24661) 534 : cluster [DBG] pgmap v1026: 105 pgs: 105 active+clean; 583 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:54:56.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:56 smithi136 bash[76371]: cluster 2024-09-12T23:54:55.148097+0000 mon.a (mon.0) 1618 : cluster [DBG] osdmap e932: 8 total, 8 up, 8 in 2024-09-12T23:54:56.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:56 smithi136 bash[76371]: audit 2024-09-12T23:54:55.154305+0000 mon.c (mon.2) 34 : audit [INF] from='client.? 172.21.15.136:0/1357578292' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-22","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:54:56.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:56 smithi136 bash[76371]: audit 2024-09-12T23:54:55.156500+0000 mon.a (mon.0) 1619 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-22","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:54:56.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:56 smithi181 bash[67587]: cluster 2024-09-12T23:54:54.966758+0000 mgr.y (mgr.24661) 534 : cluster [DBG] pgmap v1026: 105 pgs: 105 active+clean; 583 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:54:56.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:56 smithi181 bash[67587]: cluster 2024-09-12T23:54:55.148097+0000 mon.a (mon.0) 1618 : cluster [DBG] osdmap e932: 8 total, 8 up, 8 in 2024-09-12T23:54:56.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:56 smithi181 bash[67587]: audit 2024-09-12T23:54:55.154305+0000 mon.c (mon.2) 34 : audit [INF] from='client.? 172.21.15.136:0/1357578292' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-22","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:54:56.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:56 smithi181 bash[67587]: audit 2024-09-12T23:54:55.156500+0000 mon.a (mon.0) 1619 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-22","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:54:57.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:57 smithi136 bash[76371]: audit 2024-09-12T23:54:56.151300+0000 mon.a (mon.0) 1620 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-22","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:54:57.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:57 smithi136 bash[76371]: cluster 2024-09-12T23:54:56.151607+0000 mon.a (mon.0) 1621 : cluster [DBG] osdmap e933: 8 total, 8 up, 8 in 2024-09-12T23:54:57.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:57 smithi136 bash[75164]: audit 2024-09-12T23:54:56.151300+0000 mon.a (mon.0) 1620 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-22","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:54:57.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:57 smithi136 bash[75164]: cluster 2024-09-12T23:54:56.151607+0000 mon.a (mon.0) 1621 : cluster [DBG] osdmap e933: 8 total, 8 up, 8 in 2024-09-12T23:54:57.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:57 smithi181 bash[67587]: audit 2024-09-12T23:54:56.151300+0000 mon.a (mon.0) 1620 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109136-22","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:54:57.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:57 smithi181 bash[67587]: cluster 2024-09-12T23:54:56.151607+0000 mon.a (mon.0) 1621 : cluster [DBG] osdmap e933: 8 total, 8 up, 8 in 2024-09-12T23:54:58.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:58 smithi136 bash[75164]: cluster 2024-09-12T23:54:56.967839+0000 mgr.y (mgr.24661) 535 : cluster [DBG] pgmap v1029: 137 pgs: 10 creating+peering, 15 unknown, 112 active+clean; 583 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:54:58.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:54:58 smithi136 bash[75164]: cluster 2024-09-12T23:54:57.166051+0000 mon.a (mon.0) 1622 : cluster [DBG] osdmap e934: 8 total, 8 up, 8 in 2024-09-12T23:54:58.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:58 smithi136 bash[76371]: cluster 2024-09-12T23:54:56.967839+0000 mgr.y (mgr.24661) 535 : cluster [DBG] pgmap v1029: 137 pgs: 10 creating+peering, 15 unknown, 112 active+clean; 583 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:54:58.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:54:58 smithi136 bash[76371]: cluster 2024-09-12T23:54:57.166051+0000 mon.a (mon.0) 1622 : cluster [DBG] osdmap e934: 8 total, 8 up, 8 in 2024-09-12T23:54:58.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:58 smithi181 bash[67587]: cluster 2024-09-12T23:54:56.967839+0000 mgr.y (mgr.24661) 535 : cluster [DBG] pgmap v1029: 137 pgs: 10 creating+peering, 15 unknown, 112 active+clean; 583 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:54:58.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:54:58 smithi181 bash[67587]: cluster 2024-09-12T23:54:57.166051+0000 mon.a (mon.0) 1622 : cluster [DBG] osdmap e934: 8 total, 8 up, 8 in 2024-09-12T23:54:59.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:54:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:54:59] "GET /metrics HTTP/1.1" 200 36505 "" "Prometheus/2.43.0" 2024-09-12T23:55:00.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:00 smithi136 bash[75164]: cluster 2024-09-12T23:54:58.968857+0000 mgr.y (mgr.24661) 536 : cluster [DBG] pgmap v1031: 137 pgs: 10 creating+peering, 3 unknown, 124 active+clean; 604 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.8 MiB/s wr, 473 op/s 2024-09-12T23:55:00.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:00 smithi136 bash[76371]: cluster 2024-09-12T23:54:58.968857+0000 mgr.y (mgr.24661) 536 : cluster [DBG] pgmap v1031: 137 pgs: 10 creating+peering, 3 unknown, 124 active+clean; 604 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.8 MiB/s wr, 473 op/s 2024-09-12T23:55:00.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:00 smithi181 bash[67587]: cluster 2024-09-12T23:54:58.968857+0000 mgr.y (mgr.24661) 536 : cluster [DBG] pgmap v1031: 137 pgs: 10 creating+peering, 3 unknown, 124 active+clean; 604 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.8 MiB/s wr, 473 op/s 2024-09-12T23:55:02.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:02 smithi136 bash[76371]: cluster 2024-09-12T23:55:00.969833+0000 mgr.y (mgr.24661) 537 : cluster [DBG] pgmap v1032: 137 pgs: 2 creating+peering, 135 active+clean; 604 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 1.4 MiB/s rd, 1.5 MiB/s wr, 381 op/s 2024-09-12T23:55:02.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:02 smithi136 bash[75164]: cluster 2024-09-12T23:55:00.969833+0000 mgr.y (mgr.24661) 537 : cluster [DBG] pgmap v1032: 137 pgs: 2 creating+peering, 135 active+clean; 604 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 1.4 MiB/s rd, 1.5 MiB/s wr, 381 op/s 2024-09-12T23:55:02.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:02 smithi181 bash[67587]: cluster 2024-09-12T23:55:00.969833+0000 mgr.y (mgr.24661) 537 : cluster [DBG] pgmap v1032: 137 pgs: 2 creating+peering, 135 active+clean; 604 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 1.4 MiB/s rd, 1.5 MiB/s wr, 381 op/s 2024-09-12T23:55:02.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:55:02 smithi181 bash[95505]: ts=2024-09-12T23:55:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:55:03.197 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestCls2PCQueue.AbortSpillover (9055 ms) 2024-09-12T23:55:03.198 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 22 tests from TestCls2PCQueue (137224 ms total) 2024-09-12T23:55:03.198 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-12T23:55:03.198 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] Global test environment tear-down 2024-09-12T23:55:03.198 INFO:tasks.workunit.client.0.smithi136.stdout:[==========] 22 tests from 1 test suite ran. (137224 ms total) 2024-09-12T23:55:03.198 INFO:tasks.workunit.client.0.smithi136.stdout:[ PASSED ] 22 tests. 2024-09-12T23:55:03.202 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-12T23:55:03.202 DEBUG:teuthology.orchestra.run.smithi136:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-12T23:55:03.216 INFO:tasks.workunit:Running workunit cls/test_cls_cas.sh... 2024-09-12T23:55:03.216 DEBUG:teuthology.orchestra.run.smithi136:workunit test cls/test_cls_cas.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_cas.sh 2024-09-12T23:55:03.274 INFO:tasks.workunit.client.0.smithi136.stdout:Running main() from gmock_main.cc 2024-09-12T23:55:03.275 INFO:tasks.workunit.client.0.smithi136.stdout:[==========] Running 6 tests from 2 test suites. 2024-09-12T23:55:03.275 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] Global test environment set-up. 2024-09-12T23:55:03.275 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 5 tests from cls_cas 2024-09-12T23:55:03.275 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] cls_cas.get_put 2024-09-12T23:55:04.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:04 smithi136 bash[75164]: cluster 2024-09-12T23:55:02.970980+0000 mgr.y (mgr.24661) 538 : cluster [DBG] pgmap v1033: 137 pgs: 137 active+clean; 604 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.1 MiB/s wr, 292 op/s 2024-09-12T23:55:04.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:04 smithi136 bash[75164]: cluster 2024-09-12T23:55:03.196354+0000 mon.a (mon.0) 1623 : cluster [DBG] osdmap e935: 8 total, 8 up, 8 in 2024-09-12T23:55:04.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:04 smithi136 bash[76371]: cluster 2024-09-12T23:55:02.970980+0000 mgr.y (mgr.24661) 538 : cluster [DBG] pgmap v1033: 137 pgs: 137 active+clean; 604 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.1 MiB/s wr, 292 op/s 2024-09-12T23:55:04.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:04 smithi136 bash[76371]: cluster 2024-09-12T23:55:03.196354+0000 mon.a (mon.0) 1623 : cluster [DBG] osdmap e935: 8 total, 8 up, 8 in 2024-09-12T23:55:04.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:04 smithi181 bash[67587]: cluster 2024-09-12T23:55:02.970980+0000 mgr.y (mgr.24661) 538 : cluster [DBG] pgmap v1033: 137 pgs: 137 active+clean; 604 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.1 MiB/s wr, 292 op/s 2024-09-12T23:55:04.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:04 smithi181 bash[67587]: cluster 2024-09-12T23:55:03.196354+0000 mon.a (mon.0) 1623 : cluster [DBG] osdmap e935: 8 total, 8 up, 8 in 2024-09-12T23:55:05.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:05 smithi136 bash[76371]: cluster 2024-09-12T23:55:04.208105+0000 mon.a (mon.0) 1624 : cluster [DBG] osdmap e936: 8 total, 8 up, 8 in 2024-09-12T23:55:05.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:05 smithi136 bash[76371]: audit 2024-09-12T23:55:04.213081+0000 mon.c (mon.2) 35 : audit [INF] from='client.? 172.21.15.136:0/1542414136' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109647-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:05.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:05 smithi136 bash[76371]: audit 2024-09-12T23:55:04.217073+0000 mon.a (mon.0) 1625 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109647-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:05.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:05 smithi136 bash[75164]: cluster 2024-09-12T23:55:04.208105+0000 mon.a (mon.0) 1624 : cluster [DBG] osdmap e936: 8 total, 8 up, 8 in 2024-09-12T23:55:05.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:05 smithi136 bash[75164]: audit 2024-09-12T23:55:04.213081+0000 mon.c (mon.2) 35 : audit [INF] from='client.? 172.21.15.136:0/1542414136' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109647-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:05.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:05 smithi136 bash[75164]: audit 2024-09-12T23:55:04.217073+0000 mon.a (mon.0) 1625 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109647-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:05.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:05 smithi181 bash[67587]: cluster 2024-09-12T23:55:04.208105+0000 mon.a (mon.0) 1624 : cluster [DBG] osdmap e936: 8 total, 8 up, 8 in 2024-09-12T23:55:05.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:05 smithi181 bash[67587]: audit 2024-09-12T23:55:04.213081+0000 mon.c (mon.2) 35 : audit [INF] from='client.? 172.21.15.136:0/1542414136' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109647-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:05.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:05 smithi181 bash[67587]: audit 2024-09-12T23:55:04.217073+0000 mon.a (mon.0) 1625 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109647-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:06.237 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] cls_cas.get_put (2963 ms) 2024-09-12T23:55:06.237 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] cls_cas.wrong_put 2024-09-12T23:55:06.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:06 smithi136 bash[75164]: cluster 2024-09-12T23:55:04.971772+0000 mgr.y (mgr.24661) 539 : cluster [DBG] pgmap v1036: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:06.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:06 smithi136 bash[75164]: cluster 2024-09-12T23:55:05.201434+0000 mon.a (mon.0) 1626 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:55:06.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:06 smithi136 bash[75164]: audit 2024-09-12T23:55:05.224172+0000 mon.a (mon.0) 1627 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109647-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:55:06.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:06 smithi136 bash[75164]: cluster 2024-09-12T23:55:05.224621+0000 mon.a (mon.0) 1628 : cluster [DBG] osdmap e937: 8 total, 8 up, 8 in 2024-09-12T23:55:06.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:06 smithi136 bash[76371]: cluster 2024-09-12T23:55:04.971772+0000 mgr.y (mgr.24661) 539 : cluster [DBG] pgmap v1036: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:06.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:06 smithi136 bash[76371]: cluster 2024-09-12T23:55:05.201434+0000 mon.a (mon.0) 1626 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:55:06.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:06 smithi136 bash[76371]: audit 2024-09-12T23:55:05.224172+0000 mon.a (mon.0) 1627 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109647-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:55:06.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:06 smithi136 bash[76371]: cluster 2024-09-12T23:55:05.224621+0000 mon.a (mon.0) 1628 : cluster [DBG] osdmap e937: 8 total, 8 up, 8 in 2024-09-12T23:55:06.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:06 smithi181 bash[67587]: cluster 2024-09-12T23:55:04.971772+0000 mgr.y (mgr.24661) 539 : cluster [DBG] pgmap v1036: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:06.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:06 smithi181 bash[67587]: cluster 2024-09-12T23:55:05.201434+0000 mon.a (mon.0) 1626 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:55:06.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:06 smithi181 bash[67587]: audit 2024-09-12T23:55:05.224172+0000 mon.a (mon.0) 1627 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109647-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:55:06.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:06 smithi181 bash[67587]: cluster 2024-09-12T23:55:05.224621+0000 mon.a (mon.0) 1628 : cluster [DBG] osdmap e937: 8 total, 8 up, 8 in 2024-09-12T23:55:07.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:07 smithi136 bash[75164]: cluster 2024-09-12T23:55:06.230625+0000 mon.a (mon.0) 1629 : cluster [DBG] osdmap e938: 8 total, 8 up, 8 in 2024-09-12T23:55:07.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:07 smithi136 bash[76371]: cluster 2024-09-12T23:55:06.230625+0000 mon.a (mon.0) 1629 : cluster [DBG] osdmap e938: 8 total, 8 up, 8 in 2024-09-12T23:55:07.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:07 smithi181 bash[67587]: cluster 2024-09-12T23:55:06.230625+0000 mon.a (mon.0) 1629 : cluster [DBG] osdmap e938: 8 total, 8 up, 8 in 2024-09-12T23:55:08.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:08 smithi136 bash[75164]: cluster 2024-09-12T23:55:06.972567+0000 mgr.y (mgr.24661) 540 : cluster [DBG] pgmap v1039: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:08.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:08 smithi136 bash[75164]: cluster 2024-09-12T23:55:07.223332+0000 mon.a (mon.0) 1630 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:55:08.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:08 smithi136 bash[75164]: cluster 2024-09-12T23:55:07.223391+0000 mon.a (mon.0) 1631 : cluster [INF] Cluster is now healthy 2024-09-12T23:55:08.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:08 smithi136 bash[75164]: cluster 2024-09-12T23:55:07.242388+0000 mon.a (mon.0) 1632 : cluster [DBG] osdmap e939: 8 total, 8 up, 8 in 2024-09-12T23:55:08.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:08 smithi136 bash[75164]: audit 2024-09-12T23:55:07.246509+0000 mon.a (mon.0) 1633 : audit [INF] from='client.? 172.21.15.136:0/3782977377' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109647-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:08.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:08 smithi136 bash[76371]: cluster 2024-09-12T23:55:06.972567+0000 mgr.y (mgr.24661) 540 : cluster [DBG] pgmap v1039: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:08.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:08 smithi136 bash[76371]: cluster 2024-09-12T23:55:07.223332+0000 mon.a (mon.0) 1630 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:55:08.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:08 smithi136 bash[76371]: cluster 2024-09-12T23:55:07.223391+0000 mon.a (mon.0) 1631 : cluster [INF] Cluster is now healthy 2024-09-12T23:55:08.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:08 smithi136 bash[76371]: cluster 2024-09-12T23:55:07.242388+0000 mon.a (mon.0) 1632 : cluster [DBG] osdmap e939: 8 total, 8 up, 8 in 2024-09-12T23:55:08.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:08 smithi136 bash[76371]: audit 2024-09-12T23:55:07.246509+0000 mon.a (mon.0) 1633 : audit [INF] from='client.? 172.21.15.136:0/3782977377' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109647-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:08.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:08 smithi181 bash[67587]: cluster 2024-09-12T23:55:06.972567+0000 mgr.y (mgr.24661) 540 : cluster [DBG] pgmap v1039: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:08.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:08 smithi181 bash[67587]: cluster 2024-09-12T23:55:07.223332+0000 mon.a (mon.0) 1630 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:55:08.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:08 smithi181 bash[67587]: cluster 2024-09-12T23:55:07.223391+0000 mon.a (mon.0) 1631 : cluster [INF] Cluster is now healthy 2024-09-12T23:55:08.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:08 smithi181 bash[67587]: cluster 2024-09-12T23:55:07.242388+0000 mon.a (mon.0) 1632 : cluster [DBG] osdmap e939: 8 total, 8 up, 8 in 2024-09-12T23:55:08.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:08 smithi181 bash[67587]: audit 2024-09-12T23:55:07.246509+0000 mon.a (mon.0) 1633 : audit [INF] from='client.? 172.21.15.136:0/3782977377' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109647-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:09.263 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] cls_cas.wrong_put (3025 ms) 2024-09-12T23:55:09.263 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] cls_cas.dup_get 2024-09-12T23:55:09.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:09 smithi136 bash[76371]: audit 2024-09-12T23:55:08.252075+0000 mon.a (mon.0) 1634 : audit [INF] from='client.? 172.21.15.136:0/3782977377' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109647-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:55:09.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:09 smithi136 bash[76371]: cluster 2024-09-12T23:55:08.253352+0000 mon.a (mon.0) 1635 : cluster [DBG] osdmap e940: 8 total, 8 up, 8 in 2024-09-12T23:55:09.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:09 smithi136 bash[76371]: audit 2024-09-12T23:55:08.760808+0000 mon.a (mon.0) 1636 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:55:09.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:09 smithi136 bash[75164]: audit 2024-09-12T23:55:08.252075+0000 mon.a (mon.0) 1634 : audit [INF] from='client.? 172.21.15.136:0/3782977377' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109647-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:55:09.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:09 smithi136 bash[75164]: cluster 2024-09-12T23:55:08.253352+0000 mon.a (mon.0) 1635 : cluster [DBG] osdmap e940: 8 total, 8 up, 8 in 2024-09-12T23:55:09.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:09 smithi136 bash[75164]: audit 2024-09-12T23:55:08.760808+0000 mon.a (mon.0) 1636 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:55:09.508 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:55:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:55:09] "GET /metrics HTTP/1.1" 200 36449 "" "Prometheus/2.43.0" 2024-09-12T23:55:09.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:09 smithi181 bash[67587]: audit 2024-09-12T23:55:08.252075+0000 mon.a (mon.0) 1634 : audit [INF] from='client.? 172.21.15.136:0/3782977377' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109647-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:55:09.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:09 smithi181 bash[67587]: cluster 2024-09-12T23:55:08.253352+0000 mon.a (mon.0) 1635 : cluster [DBG] osdmap e940: 8 total, 8 up, 8 in 2024-09-12T23:55:09.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:09 smithi181 bash[67587]: audit 2024-09-12T23:55:08.760808+0000 mon.a (mon.0) 1636 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:55:10.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:10 smithi136 bash[75164]: cluster 2024-09-12T23:55:08.973374+0000 mgr.y (mgr.24661) 541 : cluster [DBG] pgmap v1042: 137 pgs: 3 creating+peering, 28 unknown, 106 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:10.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:10 smithi136 bash[75164]: cluster 2024-09-12T23:55:09.262195+0000 mon.a (mon.0) 1637 : cluster [DBG] osdmap e941: 8 total, 8 up, 8 in 2024-09-12T23:55:10.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:10 smithi136 bash[76371]: cluster 2024-09-12T23:55:08.973374+0000 mgr.y (mgr.24661) 541 : cluster [DBG] pgmap v1042: 137 pgs: 3 creating+peering, 28 unknown, 106 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:10.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:10 smithi136 bash[76371]: cluster 2024-09-12T23:55:09.262195+0000 mon.a (mon.0) 1637 : cluster [DBG] osdmap e941: 8 total, 8 up, 8 in 2024-09-12T23:55:10.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:10 smithi181 bash[67587]: cluster 2024-09-12T23:55:08.973374+0000 mgr.y (mgr.24661) 541 : cluster [DBG] pgmap v1042: 137 pgs: 3 creating+peering, 28 unknown, 106 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:10.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:10 smithi181 bash[67587]: cluster 2024-09-12T23:55:09.262195+0000 mon.a (mon.0) 1637 : cluster [DBG] osdmap e941: 8 total, 8 up, 8 in 2024-09-12T23:55:11.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:11 smithi181 bash[67587]: cluster 2024-09-12T23:55:10.279687+0000 mon.a (mon.0) 1638 : cluster [DBG] osdmap e942: 8 total, 8 up, 8 in 2024-09-12T23:55:11.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:11 smithi181 bash[67587]: audit 2024-09-12T23:55:10.283257+0000 mon.a (mon.0) 1639 : audit [INF] from='client.? 172.21.15.136:0/3669224773' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109647-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:11.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:11 smithi136 bash[75164]: cluster 2024-09-12T23:55:10.279687+0000 mon.a (mon.0) 1638 : cluster [DBG] osdmap e942: 8 total, 8 up, 8 in 2024-09-12T23:55:11.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:11 smithi136 bash[75164]: audit 2024-09-12T23:55:10.283257+0000 mon.a (mon.0) 1639 : audit [INF] from='client.? 172.21.15.136:0/3669224773' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109647-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:11.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:11 smithi136 bash[76371]: cluster 2024-09-12T23:55:10.279687+0000 mon.a (mon.0) 1638 : cluster [DBG] osdmap e942: 8 total, 8 up, 8 in 2024-09-12T23:55:11.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:11 smithi136 bash[76371]: audit 2024-09-12T23:55:10.283257+0000 mon.a (mon.0) 1639 : audit [INF] from='client.? 172.21.15.136:0/3669224773' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109647-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:12.289 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] cls_cas.dup_get (3026 ms) 2024-09-12T23:55:12.290 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] cls_cas.dup_put 2024-09-12T23:55:12.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:12 smithi181 bash[67587]: cluster 2024-09-12T23:55:10.974365+0000 mgr.y (mgr.24661) 542 : cluster [DBG] pgmap v1045: 137 pgs: 13 creating+peering, 19 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:12.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:12 smithi181 bash[67587]: cluster 2024-09-12T23:55:11.267913+0000 mon.a (mon.0) 1640 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:55:12.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:12 smithi181 bash[67587]: audit 2024-09-12T23:55:11.287420+0000 mon.a (mon.0) 1641 : audit [INF] from='client.? 172.21.15.136:0/3669224773' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109647-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:55:12.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:12 smithi181 bash[67587]: cluster 2024-09-12T23:55:11.287753+0000 mon.a (mon.0) 1642 : cluster [DBG] osdmap e943: 8 total, 8 up, 8 in 2024-09-12T23:55:12.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:55:12 smithi181 bash[95505]: ts=2024-09-12T23:55:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:55:12.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:12 smithi136 bash[75164]: cluster 2024-09-12T23:55:10.974365+0000 mgr.y (mgr.24661) 542 : cluster [DBG] pgmap v1045: 137 pgs: 13 creating+peering, 19 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:12.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:12 smithi136 bash[75164]: cluster 2024-09-12T23:55:11.267913+0000 mon.a (mon.0) 1640 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:55:12.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:12 smithi136 bash[75164]: audit 2024-09-12T23:55:11.287420+0000 mon.a (mon.0) 1641 : audit [INF] from='client.? 172.21.15.136:0/3669224773' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109647-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:55:12.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:12 smithi136 bash[75164]: cluster 2024-09-12T23:55:11.287753+0000 mon.a (mon.0) 1642 : cluster [DBG] osdmap e943: 8 total, 8 up, 8 in 2024-09-12T23:55:12.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:12 smithi136 bash[76371]: cluster 2024-09-12T23:55:10.974365+0000 mgr.y (mgr.24661) 542 : cluster [DBG] pgmap v1045: 137 pgs: 13 creating+peering, 19 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:12.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:12 smithi136 bash[76371]: cluster 2024-09-12T23:55:11.267913+0000 mon.a (mon.0) 1640 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:55:12.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:12 smithi136 bash[76371]: audit 2024-09-12T23:55:11.287420+0000 mon.a (mon.0) 1641 : audit [INF] from='client.? 172.21.15.136:0/3669224773' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109647-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:55:12.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:12 smithi136 bash[76371]: cluster 2024-09-12T23:55:11.287753+0000 mon.a (mon.0) 1642 : cluster [DBG] osdmap e943: 8 total, 8 up, 8 in 2024-09-12T23:55:13.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:13 smithi181 bash[67587]: cluster 2024-09-12T23:55:12.288314+0000 mon.a (mon.0) 1643 : cluster [DBG] osdmap e944: 8 total, 8 up, 8 in 2024-09-12T23:55:13.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:13 smithi136 bash[75164]: cluster 2024-09-12T23:55:12.288314+0000 mon.a (mon.0) 1643 : cluster [DBG] osdmap e944: 8 total, 8 up, 8 in 2024-09-12T23:55:13.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:13 smithi136 bash[76371]: cluster 2024-09-12T23:55:12.288314+0000 mon.a (mon.0) 1643 : cluster [DBG] osdmap e944: 8 total, 8 up, 8 in 2024-09-12T23:55:14.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:14 smithi181 bash[67587]: cluster 2024-09-12T23:55:12.975138+0000 mgr.y (mgr.24661) 543 : cluster [DBG] pgmap v1048: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:14.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:14 smithi181 bash[67587]: cluster 2024-09-12T23:55:13.294961+0000 mon.a (mon.0) 1644 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:55:14.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:14 smithi181 bash[67587]: cluster 2024-09-12T23:55:13.295015+0000 mon.a (mon.0) 1645 : cluster [INF] Cluster is now healthy 2024-09-12T23:55:14.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:14 smithi181 bash[67587]: cluster 2024-09-12T23:55:13.320777+0000 mon.a (mon.0) 1646 : cluster [DBG] osdmap e945: 8 total, 8 up, 8 in 2024-09-12T23:55:14.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:14 smithi181 bash[67587]: audit 2024-09-12T23:55:13.323551+0000 mon.a (mon.0) 1647 : audit [INF] from='client.? 172.21.15.136:0/468661915' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109647-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:14.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:14 smithi136 bash[75164]: cluster 2024-09-12T23:55:12.975138+0000 mgr.y (mgr.24661) 543 : cluster [DBG] pgmap v1048: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:14.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:14 smithi136 bash[75164]: cluster 2024-09-12T23:55:13.294961+0000 mon.a (mon.0) 1644 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:55:14.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:14 smithi136 bash[75164]: cluster 2024-09-12T23:55:13.295015+0000 mon.a (mon.0) 1645 : cluster [INF] Cluster is now healthy 2024-09-12T23:55:14.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:14 smithi136 bash[75164]: cluster 2024-09-12T23:55:13.320777+0000 mon.a (mon.0) 1646 : cluster [DBG] osdmap e945: 8 total, 8 up, 8 in 2024-09-12T23:55:14.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:14 smithi136 bash[75164]: audit 2024-09-12T23:55:13.323551+0000 mon.a (mon.0) 1647 : audit [INF] from='client.? 172.21.15.136:0/468661915' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109647-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:14.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:14 smithi136 bash[76371]: cluster 2024-09-12T23:55:12.975138+0000 mgr.y (mgr.24661) 543 : cluster [DBG] pgmap v1048: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:14.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:14 smithi136 bash[76371]: cluster 2024-09-12T23:55:13.294961+0000 mon.a (mon.0) 1644 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:55:14.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:14 smithi136 bash[76371]: cluster 2024-09-12T23:55:13.295015+0000 mon.a (mon.0) 1645 : cluster [INF] Cluster is now healthy 2024-09-12T23:55:14.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:14 smithi136 bash[76371]: cluster 2024-09-12T23:55:13.320777+0000 mon.a (mon.0) 1646 : cluster [DBG] osdmap e945: 8 total, 8 up, 8 in 2024-09-12T23:55:14.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:14 smithi136 bash[76371]: audit 2024-09-12T23:55:13.323551+0000 mon.a (mon.0) 1647 : audit [INF] from='client.? 172.21.15.136:0/468661915' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109647-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:15.331 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] cls_cas.dup_put (3042 ms) 2024-09-12T23:55:15.332 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] cls_cas.get_wrong_data 2024-09-12T23:55:15.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:15 smithi181 bash[67587]: audit 2024-09-12T23:55:14.322669+0000 mon.a (mon.0) 1648 : audit [INF] from='client.? 172.21.15.136:0/468661915' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109647-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:55:15.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:15 smithi181 bash[67587]: cluster 2024-09-12T23:55:14.323267+0000 mon.a (mon.0) 1649 : cluster [DBG] osdmap e946: 8 total, 8 up, 8 in 2024-09-12T23:55:15.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:15 smithi136 bash[76371]: audit 2024-09-12T23:55:14.322669+0000 mon.a (mon.0) 1648 : audit [INF] from='client.? 172.21.15.136:0/468661915' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109647-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:55:15.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:15 smithi136 bash[76371]: cluster 2024-09-12T23:55:14.323267+0000 mon.a (mon.0) 1649 : cluster [DBG] osdmap e946: 8 total, 8 up, 8 in 2024-09-12T23:55:15.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:15 smithi136 bash[75164]: audit 2024-09-12T23:55:14.322669+0000 mon.a (mon.0) 1648 : audit [INF] from='client.? 172.21.15.136:0/468661915' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109647-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:55:15.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:15 smithi136 bash[75164]: cluster 2024-09-12T23:55:14.323267+0000 mon.a (mon.0) 1649 : cluster [DBG] osdmap e946: 8 total, 8 up, 8 in 2024-09-12T23:55:16.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:16 smithi181 bash[67587]: cluster 2024-09-12T23:55:14.975974+0000 mgr.y (mgr.24661) 544 : cluster [DBG] pgmap v1051: 137 pgs: 6 creating+peering, 20 unknown, 111 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:16.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:16 smithi181 bash[67587]: cluster 2024-09-12T23:55:15.331287+0000 mon.a (mon.0) 1650 : cluster [DBG] osdmap e947: 8 total, 8 up, 8 in 2024-09-12T23:55:16.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:16 smithi136 bash[75164]: cluster 2024-09-12T23:55:14.975974+0000 mgr.y (mgr.24661) 544 : cluster [DBG] pgmap v1051: 137 pgs: 6 creating+peering, 20 unknown, 111 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:16.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:16 smithi136 bash[75164]: cluster 2024-09-12T23:55:15.331287+0000 mon.a (mon.0) 1650 : cluster [DBG] osdmap e947: 8 total, 8 up, 8 in 2024-09-12T23:55:16.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:16 smithi136 bash[76371]: cluster 2024-09-12T23:55:14.975974+0000 mgr.y (mgr.24661) 544 : cluster [DBG] pgmap v1051: 137 pgs: 6 creating+peering, 20 unknown, 111 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:16.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:16 smithi136 bash[76371]: cluster 2024-09-12T23:55:15.331287+0000 mon.a (mon.0) 1650 : cluster [DBG] osdmap e947: 8 total, 8 up, 8 in 2024-09-12T23:55:17.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:17 smithi181 bash[67587]: cluster 2024-09-12T23:55:16.343212+0000 mon.a (mon.0) 1651 : cluster [DBG] osdmap e948: 8 total, 8 up, 8 in 2024-09-12T23:55:17.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:17 smithi181 bash[67587]: audit 2024-09-12T23:55:16.355388+0000 mon.c (mon.2) 36 : audit [INF] from='client.? 172.21.15.136:0/553212243' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109647-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:17.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:17 smithi181 bash[67587]: audit 2024-09-12T23:55:16.357317+0000 mon.a (mon.0) 1652 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109647-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:17.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:17 smithi136 bash[75164]: cluster 2024-09-12T23:55:16.343212+0000 mon.a (mon.0) 1651 : cluster [DBG] osdmap e948: 8 total, 8 up, 8 in 2024-09-12T23:55:17.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:17 smithi136 bash[75164]: audit 2024-09-12T23:55:16.355388+0000 mon.c (mon.2) 36 : audit [INF] from='client.? 172.21.15.136:0/553212243' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109647-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:17.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:17 smithi136 bash[75164]: audit 2024-09-12T23:55:16.357317+0000 mon.a (mon.0) 1652 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109647-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:17.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:17 smithi136 bash[76371]: cluster 2024-09-12T23:55:16.343212+0000 mon.a (mon.0) 1651 : cluster [DBG] osdmap e948: 8 total, 8 up, 8 in 2024-09-12T23:55:17.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:17 smithi136 bash[76371]: audit 2024-09-12T23:55:16.355388+0000 mon.c (mon.2) 36 : audit [INF] from='client.? 172.21.15.136:0/553212243' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109647-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:17.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:17 smithi136 bash[76371]: audit 2024-09-12T23:55:16.357317+0000 mon.a (mon.0) 1652 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109647-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:18.362 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] cls_cas.get_wrong_data (3028 ms) 2024-09-12T23:55:18.362 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 5 tests from cls_cas (15085 ms total) 2024-09-12T23:55:18.362 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-12T23:55:18.362 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 1 test from chunk_refs_t 2024-09-12T23:55:18.362 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] chunk_refs_t.size 2024-09-12T23:55:18.362 INFO:tasks.workunit.client.0.smithi136.stdout:1 60 by_object 2024-09-12T23:55:18.362 INFO:tasks.workunit.client.0.smithi136.stdout:2 103 by_object 2024-09-12T23:55:18.362 INFO:tasks.workunit.client.0.smithi136.stdout:4 189 by_object 2024-09-12T23:55:18.362 INFO:tasks.workunit.client.0.smithi136.stdout:8 361 by_object 2024-09-12T23:55:18.362 INFO:tasks.workunit.client.0.smithi136.stdout:16 112 by_hash(32 bits) 2024-09-12T23:55:18.362 INFO:tasks.workunit.client.0.smithi136.stdout:32 208 by_hash(32 bits) 2024-09-12T23:55:18.363 INFO:tasks.workunit.client.0.smithi136.stdout:64 400 by_hash(32 bits) 2024-09-12T23:55:18.363 INFO:tasks.workunit.client.0.smithi136.stdout:128 509 by_hash(10 bits) 2024-09-12T23:55:18.363 INFO:tasks.workunit.client.0.smithi136.stdout:256 499 by_hash(7 bits) 2024-09-12T23:55:18.363 INFO:tasks.workunit.client.0.smithi136.stdout:512 462 by_hash(5 bits) 2024-09-12T23:55:18.363 INFO:tasks.workunit.client.0.smithi136.stdout:1024 466 by_hash(5 bits) 2024-09-12T23:55:18.364 INFO:tasks.workunit.client.0.smithi136.stdout:2048 466 by_hash(5 bits) 2024-09-12T23:55:18.365 INFO:tasks.workunit.client.0.smithi136.stdout:4096 470 by_hash(5 bits) 2024-09-12T23:55:18.367 INFO:tasks.workunit.client.0.smithi136.stdout:8192 498 by_hash(4 bits) 2024-09-12T23:55:18.370 INFO:tasks.workunit.client.0.smithi136.stdout:16384 278 by_hash(3 bits) 2024-09-12T23:55:18.373 INFO:tasks.workunit.client.0.smithi136.stdout:32768 337 by_hash(2 bits) 2024-09-12T23:55:18.380 INFO:tasks.workunit.client.0.smithi136.stdout:65536 367 by_hash(2 bits) 2024-09-12T23:55:18.393 INFO:tasks.workunit.client.0.smithi136.stdout:131072 387 by_hash(1 bits) 2024-09-12T23:55:18.419 INFO:tasks.workunit.client.0.smithi136.stdout:262144 331 by_pool 2024-09-12T23:55:18.459 INFO:tasks.workunit.client.0.smithi136.stdout:524288 21 count 2024-09-12T23:55:18.522 INFO:tasks.workunit.client.0.smithi136.stdout:1048576 21 count 2024-09-12T23:55:18.643 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] chunk_refs_t.size (284 ms) 2024-09-12T23:55:18.643 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 1 test from chunk_refs_t (284 ms total) 2024-09-12T23:55:18.644 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-12T23:55:18.644 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] Global test environment tear-down 2024-09-12T23:55:18.644 INFO:tasks.workunit.client.0.smithi136.stdout:[==========] 6 tests from 2 test suites ran. (15369 ms total) 2024-09-12T23:55:18.644 INFO:tasks.workunit.client.0.smithi136.stdout:[ PASSED ] 6 tests. 2024-09-12T23:55:18.646 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-12T23:55:18.646 DEBUG:teuthology.orchestra.run.smithi136:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-12T23:55:18.698 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:18 smithi136 bash[75164]: cluster 2024-09-12T23:55:16.976961+0000 mgr.y (mgr.24661) 545 : cluster [DBG] pgmap v1054: 137 pgs: 10 creating+peering, 22 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:18.698 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:18 smithi136 bash[75164]: cluster 2024-09-12T23:55:17.337381+0000 mon.a (mon.0) 1653 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:55:18.698 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:18 smithi136 bash[75164]: audit 2024-09-12T23:55:17.355712+0000 mon.a (mon.0) 1654 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109647-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:55:18.698 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:18 smithi136 bash[75164]: cluster 2024-09-12T23:55:17.355998+0000 mon.a (mon.0) 1655 : cluster [DBG] osdmap e949: 8 total, 8 up, 8 in 2024-09-12T23:55:18.698 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:18 smithi136 bash[76371]: cluster 2024-09-12T23:55:16.976961+0000 mgr.y (mgr.24661) 545 : cluster [DBG] pgmap v1054: 137 pgs: 10 creating+peering, 22 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:18.698 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:18 smithi136 bash[76371]: cluster 2024-09-12T23:55:17.337381+0000 mon.a (mon.0) 1653 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:55:18.698 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:18 smithi136 bash[76371]: audit 2024-09-12T23:55:17.355712+0000 mon.a (mon.0) 1654 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109647-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:55:18.698 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:18 smithi136 bash[76371]: cluster 2024-09-12T23:55:17.355998+0000 mon.a (mon.0) 1655 : cluster [DBG] osdmap e949: 8 total, 8 up, 8 in 2024-09-12T23:55:18.700 INFO:tasks.workunit:Running workunit cls/test_cls_cmpomap.sh... 2024-09-12T23:55:18.700 DEBUG:teuthology.orchestra.run.smithi136:workunit test cls/test_cls_cmpomap.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_cmpomap.sh 2024-09-12T23:55:18.768 INFO:tasks.workunit.client.0.smithi136.stdout:Running main() from gmock_main.cc 2024-09-12T23:55:18.768 INFO:tasks.workunit.client.0.smithi136.stdout:[==========] Running 31 tests from 1 test suite. 2024-09-12T23:55:18.768 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] Global test environment set-up. 2024-09-12T23:55:18.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:18 smithi181 bash[67587]: cluster 2024-09-12T23:55:16.976961+0000 mgr.y (mgr.24661) 545 : cluster [DBG] pgmap v1054: 137 pgs: 10 creating+peering, 22 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:18.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:18 smithi181 bash[67587]: cluster 2024-09-12T23:55:17.337381+0000 mon.a (mon.0) 1653 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:55:18.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:18 smithi181 bash[67587]: audit 2024-09-12T23:55:17.355712+0000 mon.a (mon.0) 1654 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109647-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:55:18.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:18 smithi181 bash[67587]: cluster 2024-09-12T23:55:17.355998+0000 mon.a (mon.0) 1655 : cluster [DBG] osdmap e949: 8 total, 8 up, 8 in 2024-09-12T23:55:19.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:19 smithi136 bash[75164]: cluster 2024-09-12T23:55:18.356462+0000 mon.a (mon.0) 1656 : cluster [DBG] osdmap e950: 8 total, 8 up, 8 in 2024-09-12T23:55:19.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:55:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:55:19] "GET /metrics HTTP/1.1" 200 36449 "" "Prometheus/2.43.0" 2024-09-12T23:55:19.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:19 smithi136 bash[76371]: cluster 2024-09-12T23:55:18.356462+0000 mon.a (mon.0) 1656 : cluster [DBG] osdmap e950: 8 total, 8 up, 8 in 2024-09-12T23:55:19.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:19 smithi181 bash[67587]: cluster 2024-09-12T23:55:18.356462+0000 mon.a (mon.0) 1656 : cluster [DBG] osdmap e950: 8 total, 8 up, 8 in 2024-09-12T23:55:20.384 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 31 tests from CmpOmap 2024-09-12T23:55:20.385 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] CmpOmap.cmp_vals_noexist_str 2024-09-12T23:55:20.426 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] CmpOmap.cmp_vals_noexist_str (42 ms) 2024-09-12T23:55:20.427 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] CmpOmap.cmp_vals_noexist_str_default 2024-09-12T23:55:20.431 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] CmpOmap.cmp_vals_noexist_str_default (5 ms) 2024-09-12T23:55:20.431 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] CmpOmap.cmp_vals_noexist_u64 2024-09-12T23:55:20.435 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] CmpOmap.cmp_vals_noexist_u64 (4 ms) 2024-09-12T23:55:20.435 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] CmpOmap.cmp_vals_noexist_u64_default 2024-09-12T23:55:20.439 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] CmpOmap.cmp_vals_noexist_u64_default (4 ms) 2024-09-12T23:55:20.439 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] CmpOmap.cmp_vals_str 2024-09-12T23:55:20.448 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] CmpOmap.cmp_vals_str (9 ms) 2024-09-12T23:55:20.448 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] CmpOmap.cmp_vals_u64 2024-09-12T23:55:20.461 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] CmpOmap.cmp_vals_u64 (13 ms) 2024-09-12T23:55:20.461 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] CmpOmap.cmp_vals_u64_invalid_input 2024-09-12T23:55:20.466 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] CmpOmap.cmp_vals_u64_invalid_input (5 ms) 2024-09-12T23:55:20.466 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] CmpOmap.cmp_vals_u64_invalid_default 2024-09-12T23:55:20.470 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] CmpOmap.cmp_vals_u64_invalid_default (4 ms) 2024-09-12T23:55:20.470 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] CmpOmap.cmp_vals_u64_empty_default 2024-09-12T23:55:20.474 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] CmpOmap.cmp_vals_u64_empty_default (4 ms) 2024-09-12T23:55:20.474 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] CmpOmap.cmp_vals_u64_invalid_value 2024-09-12T23:55:20.480 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] CmpOmap.cmp_vals_u64_invalid_value (6 ms) 2024-09-12T23:55:20.480 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] CmpOmap.cmp_vals_at_max_keys 2024-09-12T23:55:20.481 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] CmpOmap.cmp_vals_at_max_keys (1 ms) 2024-09-12T23:55:20.481 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] CmpOmap.cmp_vals_over_max_keys 2024-09-12T23:55:20.482 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] CmpOmap.cmp_vals_over_max_keys (1 ms) 2024-09-12T23:55:20.482 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] CmpOmap.cmp_set_vals_noexist_str 2024-09-12T23:55:20.493 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] CmpOmap.cmp_set_vals_noexist_str (11 ms) 2024-09-12T23:55:20.494 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] CmpOmap.cmp_set_vals_noexist_str_default 2024-09-12T23:55:20.504 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] CmpOmap.cmp_set_vals_noexist_str_default (11 ms) 2024-09-12T23:55:20.504 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] CmpOmap.cmp_set_vals_noexist_u64 2024-09-12T23:55:20.515 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] CmpOmap.cmp_set_vals_noexist_u64 (11 ms) 2024-09-12T23:55:20.515 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] CmpOmap.cmp_set_vals_noexist_u64_default 2024-09-12T23:55:20.531 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] CmpOmap.cmp_set_vals_noexist_u64_default (16 ms) 2024-09-12T23:55:20.531 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] CmpOmap.cmp_set_vals_str 2024-09-12T23:55:20.546 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] CmpOmap.cmp_set_vals_str (15 ms) 2024-09-12T23:55:20.546 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] CmpOmap.cmp_set_vals_u64 2024-09-12T23:55:20.562 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] CmpOmap.cmp_set_vals_u64 (16 ms) 2024-09-12T23:55:20.562 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] CmpOmap.cmp_set_vals_u64_einval 2024-09-12T23:55:20.565 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] CmpOmap.cmp_set_vals_u64_einval (3 ms) 2024-09-12T23:55:20.566 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] CmpOmap.cmp_set_vals_u64_eio 2024-09-12T23:55:20.570 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] CmpOmap.cmp_set_vals_u64_eio (5 ms) 2024-09-12T23:55:20.570 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] CmpOmap.cmp_set_vals_at_max_keys 2024-09-12T23:55:20.573 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] CmpOmap.cmp_set_vals_at_max_keys (3 ms) 2024-09-12T23:55:20.573 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] CmpOmap.cmp_set_vals_over_max_keys 2024-09-12T23:55:20.575 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] CmpOmap.cmp_set_vals_over_max_keys (2 ms) 2024-09-12T23:55:20.575 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] CmpOmap.cmp_rm_keys_noexist_str 2024-09-12T23:55:20.591 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] CmpOmap.cmp_rm_keys_noexist_str (16 ms) 2024-09-12T23:55:20.591 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] CmpOmap.cmp_rm_keys_noexist_u64 2024-09-12T23:55:20.604 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] CmpOmap.cmp_rm_keys_noexist_u64 (13 ms) 2024-09-12T23:55:20.604 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] CmpOmap.cmp_rm_keys_str 2024-09-12T23:55:20.620 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] CmpOmap.cmp_rm_keys_str (16 ms) 2024-09-12T23:55:20.620 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] CmpOmap.cmp_rm_keys_u64 2024-09-12T23:55:20.638 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] CmpOmap.cmp_rm_keys_u64 (18 ms) 2024-09-12T23:55:20.638 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] CmpOmap.cmp_rm_keys_u64_einval 2024-09-12T23:55:20.644 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] CmpOmap.cmp_rm_keys_u64_einval (5 ms) 2024-09-12T23:55:20.644 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] CmpOmap.cmp_rm_keys_u64_eio 2024-09-12T23:55:20.648 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] CmpOmap.cmp_rm_keys_u64_eio (4 ms) 2024-09-12T23:55:20.648 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] CmpOmap.cmp_rm_keys_at_max_keys 2024-09-12T23:55:20.651 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] CmpOmap.cmp_rm_keys_at_max_keys (3 ms) 2024-09-12T23:55:20.651 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] CmpOmap.cmp_rm_keys_over_max_keys 2024-09-12T23:55:20.653 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] CmpOmap.cmp_rm_keys_over_max_keys (2 ms) 2024-09-12T23:55:20.653 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] CmpOmap.cmp_rm_keys_u64_empty 2024-09-12T23:55:20.672 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] CmpOmap.cmp_rm_keys_u64_empty (19 ms) 2024-09-12T23:55:20.673 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 31 tests from CmpOmap (288 ms total) 2024-09-12T23:55:20.673 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-12T23:55:20.673 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] Global test environment tear-down 2024-09-12T23:55:20.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:20 smithi136 bash[75164]: cluster 2024-09-12T23:55:18.977694+0000 mgr.y (mgr.24661) 546 : cluster [DBG] pgmap v1057: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:20.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:20 smithi136 bash[75164]: cluster 2024-09-12T23:55:19.360600+0000 mon.a (mon.0) 1657 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:55:20.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:20 smithi136 bash[75164]: cluster 2024-09-12T23:55:19.360709+0000 mon.a (mon.0) 1658 : cluster [INF] Cluster is now healthy 2024-09-12T23:55:20.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:20 smithi136 bash[75164]: cluster 2024-09-12T23:55:19.386316+0000 mon.a (mon.0) 1659 : cluster [DBG] osdmap e951: 8 total, 8 up, 8 in 2024-09-12T23:55:20.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:20 smithi136 bash[75164]: audit 2024-09-12T23:55:19.387551+0000 mon.a (mon.0) 1660 : audit [INF] from='client.? 172.21.15.136:0/2869999482' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109739-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:20.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:20 smithi136 bash[76371]: cluster 2024-09-12T23:55:18.977694+0000 mgr.y (mgr.24661) 546 : cluster [DBG] pgmap v1057: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:20.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:20 smithi136 bash[76371]: cluster 2024-09-12T23:55:19.360600+0000 mon.a (mon.0) 1657 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:55:20.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:20 smithi136 bash[76371]: cluster 2024-09-12T23:55:19.360709+0000 mon.a (mon.0) 1658 : cluster [INF] Cluster is now healthy 2024-09-12T23:55:20.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:20 smithi136 bash[76371]: cluster 2024-09-12T23:55:19.386316+0000 mon.a (mon.0) 1659 : cluster [DBG] osdmap e951: 8 total, 8 up, 8 in 2024-09-12T23:55:20.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:20 smithi136 bash[76371]: audit 2024-09-12T23:55:19.387551+0000 mon.a (mon.0) 1660 : audit [INF] from='client.? 172.21.15.136:0/2869999482' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109739-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:20.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:20 smithi181 bash[67587]: cluster 2024-09-12T23:55:18.977694+0000 mgr.y (mgr.24661) 546 : cluster [DBG] pgmap v1057: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:20.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:20 smithi181 bash[67587]: cluster 2024-09-12T23:55:19.360600+0000 mon.a (mon.0) 1657 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:55:20.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:20 smithi181 bash[67587]: cluster 2024-09-12T23:55:19.360709+0000 mon.a (mon.0) 1658 : cluster [INF] Cluster is now healthy 2024-09-12T23:55:20.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:20 smithi181 bash[67587]: cluster 2024-09-12T23:55:19.386316+0000 mon.a (mon.0) 1659 : cluster [DBG] osdmap e951: 8 total, 8 up, 8 in 2024-09-12T23:55:20.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:20 smithi181 bash[67587]: audit 2024-09-12T23:55:19.387551+0000 mon.a (mon.0) 1660 : audit [INF] from='client.? 172.21.15.136:0/2869999482' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109739-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:21.408 INFO:tasks.workunit.client.0.smithi136.stdout:[==========] 31 tests from 1 test suite ran. (2643 ms total) 2024-09-12T23:55:21.409 INFO:tasks.workunit.client.0.smithi136.stdout:[ PASSED ] 31 tests. 2024-09-12T23:55:21.412 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-12T23:55:21.412 DEBUG:teuthology.orchestra.run.smithi136:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-12T23:55:21.466 INFO:tasks.workunit:Running workunit cls/test_cls_hello.sh... 2024-09-12T23:55:21.466 DEBUG:teuthology.orchestra.run.smithi136:workunit test cls/test_cls_hello.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_hello.sh 2024-09-12T23:55:21.526 INFO:tasks.workunit.client.0.smithi136.stdout:Running main() from gmock_main.cc 2024-09-12T23:55:21.527 INFO:tasks.workunit.client.0.smithi136.stdout:[==========] Running 6 tests from 1 test suite. 2024-09-12T23:55:21.527 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] Global test environment set-up. 2024-09-12T23:55:21.527 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 6 tests from ClsHello 2024-09-12T23:55:21.527 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] ClsHello.SayHello 2024-09-12T23:55:21.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:21 smithi136 bash[75164]: audit 2024-09-12T23:55:20.384172+0000 mon.a (mon.0) 1661 : audit [INF] from='client.? 172.21.15.136:0/2869999482' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109739-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:55:21.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:21 smithi136 bash[75164]: cluster 2024-09-12T23:55:20.385568+0000 mon.a (mon.0) 1662 : cluster [DBG] osdmap e952: 8 total, 8 up, 8 in 2024-09-12T23:55:21.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:21 smithi136 bash[76371]: audit 2024-09-12T23:55:20.384172+0000 mon.a (mon.0) 1661 : audit [INF] from='client.? 172.21.15.136:0/2869999482' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109739-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:55:21.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:21 smithi136 bash[76371]: cluster 2024-09-12T23:55:20.385568+0000 mon.a (mon.0) 1662 : cluster [DBG] osdmap e952: 8 total, 8 up, 8 in 2024-09-12T23:55:21.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:21 smithi181 bash[67587]: audit 2024-09-12T23:55:20.384172+0000 mon.a (mon.0) 1661 : audit [INF] from='client.? 172.21.15.136:0/2869999482' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109739-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:55:21.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:21 smithi181 bash[67587]: cluster 2024-09-12T23:55:20.385568+0000 mon.a (mon.0) 1662 : cluster [DBG] osdmap e952: 8 total, 8 up, 8 in 2024-09-12T23:55:22.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:22 smithi181 bash[67587]: cluster 2024-09-12T23:55:20.978567+0000 mgr.y (mgr.24661) 547 : cluster [DBG] pgmap v1060: 137 pgs: 1 creating+peering, 23 unknown, 113 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 1.2 KiB/s wr, 6 op/s 2024-09-12T23:55:22.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:22 smithi181 bash[67587]: cluster 2024-09-12T23:55:21.407993+0000 mon.a (mon.0) 1663 : cluster [DBG] osdmap e953: 8 total, 8 up, 8 in 2024-09-12T23:55:22.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:55:22 smithi181 bash[95505]: ts=2024-09-12T23:55:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:55:22.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:22 smithi136 bash[75164]: cluster 2024-09-12T23:55:20.978567+0000 mgr.y (mgr.24661) 547 : cluster [DBG] pgmap v1060: 137 pgs: 1 creating+peering, 23 unknown, 113 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 1.2 KiB/s wr, 6 op/s 2024-09-12T23:55:22.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:22 smithi136 bash[75164]: cluster 2024-09-12T23:55:21.407993+0000 mon.a (mon.0) 1663 : cluster [DBG] osdmap e953: 8 total, 8 up, 8 in 2024-09-12T23:55:22.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:22 smithi136 bash[76371]: cluster 2024-09-12T23:55:20.978567+0000 mgr.y (mgr.24661) 547 : cluster [DBG] pgmap v1060: 137 pgs: 1 creating+peering, 23 unknown, 113 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 1.2 KiB/s wr, 6 op/s 2024-09-12T23:55:22.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:22 smithi136 bash[76371]: cluster 2024-09-12T23:55:21.407993+0000 mon.a (mon.0) 1663 : cluster [DBG] osdmap e953: 8 total, 8 up, 8 in 2024-09-12T23:55:23.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:23 smithi136 bash[75164]: cluster 2024-09-12T23:55:22.430418+0000 mon.a (mon.0) 1664 : cluster [DBG] osdmap e954: 8 total, 8 up, 8 in 2024-09-12T23:55:23.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:23 smithi136 bash[75164]: audit 2024-09-12T23:55:22.438780+0000 mon.c (mon.2) 37 : audit [INF] from='client.? 172.21.15.136:0/1033892669' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109763-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:23.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:23 smithi136 bash[75164]: audit 2024-09-12T23:55:22.440417+0000 mon.a (mon.0) 1665 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109763-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:23.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:23 smithi136 bash[76371]: cluster 2024-09-12T23:55:22.430418+0000 mon.a (mon.0) 1664 : cluster [DBG] osdmap e954: 8 total, 8 up, 8 in 2024-09-12T23:55:23.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:23 smithi136 bash[76371]: audit 2024-09-12T23:55:22.438780+0000 mon.c (mon.2) 37 : audit [INF] from='client.? 172.21.15.136:0/1033892669' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109763-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:23.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:23 smithi136 bash[76371]: audit 2024-09-12T23:55:22.440417+0000 mon.a (mon.0) 1665 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109763-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:23.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:23 smithi181 bash[67587]: cluster 2024-09-12T23:55:22.430418+0000 mon.a (mon.0) 1664 : cluster [DBG] osdmap e954: 8 total, 8 up, 8 in 2024-09-12T23:55:23.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:23 smithi181 bash[67587]: audit 2024-09-12T23:55:22.438780+0000 mon.c (mon.2) 37 : audit [INF] from='client.? 172.21.15.136:0/1033892669' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109763-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:23.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:23 smithi181 bash[67587]: audit 2024-09-12T23:55:22.440417+0000 mon.a (mon.0) 1665 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109763-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:24.457 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] ClsHello.SayHello (2930 ms) 2024-09-12T23:55:24.457 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] ClsHello.RecordHello 2024-09-12T23:55:24.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:24 smithi136 bash[75164]: cluster 2024-09-12T23:55:22.979431+0000 mgr.y (mgr.24661) 548 : cluster [DBG] pgmap v1063: 137 pgs: 2 creating+peering, 30 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:24.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:24 smithi136 bash[75164]: cluster 2024-09-12T23:55:23.413924+0000 mon.a (mon.0) 1666 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:55:24.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:24 smithi136 bash[75164]: audit 2024-09-12T23:55:23.424731+0000 mon.a (mon.0) 1667 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109763-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:55:24.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:24 smithi136 bash[75164]: cluster 2024-09-12T23:55:23.425240+0000 mon.a (mon.0) 1668 : cluster [DBG] osdmap e955: 8 total, 8 up, 8 in 2024-09-12T23:55:24.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:24 smithi136 bash[75164]: audit 2024-09-12T23:55:23.768209+0000 mon.a (mon.0) 1669 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:55:24.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:24 smithi136 bash[75164]: audit 2024-09-12T23:55:23.770254+0000 mon.a (mon.0) 1670 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:55:24.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:24 smithi136 bash[76371]: cluster 2024-09-12T23:55:22.979431+0000 mgr.y (mgr.24661) 548 : cluster [DBG] pgmap v1063: 137 pgs: 2 creating+peering, 30 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:24.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:24 smithi136 bash[76371]: cluster 2024-09-12T23:55:23.413924+0000 mon.a (mon.0) 1666 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:55:24.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:24 smithi136 bash[76371]: audit 2024-09-12T23:55:23.424731+0000 mon.a (mon.0) 1667 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109763-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:55:24.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:24 smithi136 bash[76371]: cluster 2024-09-12T23:55:23.425240+0000 mon.a (mon.0) 1668 : cluster [DBG] osdmap e955: 8 total, 8 up, 8 in 2024-09-12T23:55:24.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:24 smithi136 bash[76371]: audit 2024-09-12T23:55:23.768209+0000 mon.a (mon.0) 1669 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:55:24.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:24 smithi136 bash[76371]: audit 2024-09-12T23:55:23.770254+0000 mon.a (mon.0) 1670 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:55:24.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:24 smithi181 bash[67587]: cluster 2024-09-12T23:55:22.979431+0000 mgr.y (mgr.24661) 548 : cluster [DBG] pgmap v1063: 137 pgs: 2 creating+peering, 30 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:24.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:24 smithi181 bash[67587]: cluster 2024-09-12T23:55:23.413924+0000 mon.a (mon.0) 1666 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:55:24.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:24 smithi181 bash[67587]: audit 2024-09-12T23:55:23.424731+0000 mon.a (mon.0) 1667 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109763-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:55:24.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:24 smithi181 bash[67587]: cluster 2024-09-12T23:55:23.425240+0000 mon.a (mon.0) 1668 : cluster [DBG] osdmap e955: 8 total, 8 up, 8 in 2024-09-12T23:55:24.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:24 smithi181 bash[67587]: audit 2024-09-12T23:55:23.768209+0000 mon.a (mon.0) 1669 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:55:24.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:24 smithi181 bash[67587]: audit 2024-09-12T23:55:23.770254+0000 mon.a (mon.0) 1670 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:55:25.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:25 smithi136 bash[75164]: cluster 2024-09-12T23:55:24.450775+0000 mon.a (mon.0) 1671 : cluster [DBG] osdmap e956: 8 total, 8 up, 8 in 2024-09-12T23:55:25.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:25 smithi136 bash[76371]: cluster 2024-09-12T23:55:24.450775+0000 mon.a (mon.0) 1671 : cluster [DBG] osdmap e956: 8 total, 8 up, 8 in 2024-09-12T23:55:25.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:25 smithi181 bash[67587]: cluster 2024-09-12T23:55:24.450775+0000 mon.a (mon.0) 1671 : cluster [DBG] osdmap e956: 8 total, 8 up, 8 in 2024-09-12T23:55:26.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:26 smithi136 bash[75164]: cluster 2024-09-12T23:55:24.980198+0000 mgr.y (mgr.24661) 549 : cluster [DBG] pgmap v1066: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:26.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:26 smithi136 bash[75164]: cluster 2024-09-12T23:55:25.445567+0000 mon.a (mon.0) 1672 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:55:26.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:26 smithi136 bash[75164]: cluster 2024-09-12T23:55:25.445627+0000 mon.a (mon.0) 1673 : cluster [INF] Cluster is now healthy 2024-09-12T23:55:26.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:26 smithi136 bash[75164]: cluster 2024-09-12T23:55:25.466995+0000 mon.a (mon.0) 1674 : cluster [DBG] osdmap e957: 8 total, 8 up, 8 in 2024-09-12T23:55:26.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:26 smithi136 bash[75164]: audit 2024-09-12T23:55:25.470508+0000 mon.a (mon.0) 1675 : audit [INF] from='client.? 172.21.15.136:0/2018049024' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109763-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:26.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:26 smithi136 bash[76371]: cluster 2024-09-12T23:55:24.980198+0000 mgr.y (mgr.24661) 549 : cluster [DBG] pgmap v1066: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:26.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:26 smithi136 bash[76371]: cluster 2024-09-12T23:55:25.445567+0000 mon.a (mon.0) 1672 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:55:26.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:26 smithi136 bash[76371]: cluster 2024-09-12T23:55:25.445627+0000 mon.a (mon.0) 1673 : cluster [INF] Cluster is now healthy 2024-09-12T23:55:26.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:26 smithi136 bash[76371]: cluster 2024-09-12T23:55:25.466995+0000 mon.a (mon.0) 1674 : cluster [DBG] osdmap e957: 8 total, 8 up, 8 in 2024-09-12T23:55:26.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:26 smithi136 bash[76371]: audit 2024-09-12T23:55:25.470508+0000 mon.a (mon.0) 1675 : audit [INF] from='client.? 172.21.15.136:0/2018049024' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109763-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:26.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:26 smithi181 bash[67587]: cluster 2024-09-12T23:55:24.980198+0000 mgr.y (mgr.24661) 549 : cluster [DBG] pgmap v1066: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:26.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:26 smithi181 bash[67587]: cluster 2024-09-12T23:55:25.445567+0000 mon.a (mon.0) 1672 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:55:26.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:26 smithi181 bash[67587]: cluster 2024-09-12T23:55:25.445627+0000 mon.a (mon.0) 1673 : cluster [INF] Cluster is now healthy 2024-09-12T23:55:26.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:26 smithi181 bash[67587]: cluster 2024-09-12T23:55:25.466995+0000 mon.a (mon.0) 1674 : cluster [DBG] osdmap e957: 8 total, 8 up, 8 in 2024-09-12T23:55:26.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:26 smithi181 bash[67587]: audit 2024-09-12T23:55:25.470508+0000 mon.a (mon.0) 1675 : audit [INF] from='client.? 172.21.15.136:0/2018049024' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109763-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:27.489 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] ClsHello.RecordHello (3032 ms) 2024-09-12T23:55:27.489 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] ClsHello.WriteReturnData 2024-09-12T23:55:27.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:27 smithi136 bash[75164]: audit 2024-09-12T23:55:26.468716+0000 mon.a (mon.0) 1676 : audit [INF] from='client.? 172.21.15.136:0/2018049024' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109763-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:55:27.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:27 smithi136 bash[75164]: cluster 2024-09-12T23:55:26.469198+0000 mon.a (mon.0) 1677 : cluster [DBG] osdmap e958: 8 total, 8 up, 8 in 2024-09-12T23:55:27.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:27 smithi136 bash[76371]: audit 2024-09-12T23:55:26.468716+0000 mon.a (mon.0) 1676 : audit [INF] from='client.? 172.21.15.136:0/2018049024' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109763-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:55:27.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:27 smithi136 bash[76371]: cluster 2024-09-12T23:55:26.469198+0000 mon.a (mon.0) 1677 : cluster [DBG] osdmap e958: 8 total, 8 up, 8 in 2024-09-12T23:55:27.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:27 smithi181 bash[67587]: audit 2024-09-12T23:55:26.468716+0000 mon.a (mon.0) 1676 : audit [INF] from='client.? 172.21.15.136:0/2018049024' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109763-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:55:27.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:27 smithi181 bash[67587]: cluster 2024-09-12T23:55:26.469198+0000 mon.a (mon.0) 1677 : cluster [DBG] osdmap e958: 8 total, 8 up, 8 in 2024-09-12T23:55:28.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:28 smithi136 bash[75164]: cluster 2024-09-12T23:55:26.981156+0000 mgr.y (mgr.24661) 550 : cluster [DBG] pgmap v1069: 137 pgs: 13 creating+peering, 14 unknown, 110 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s 2024-09-12T23:55:28.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:28 smithi136 bash[75164]: cluster 2024-09-12T23:55:27.485406+0000 mon.a (mon.0) 1678 : cluster [DBG] osdmap e959: 8 total, 8 up, 8 in 2024-09-12T23:55:28.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:28 smithi136 bash[76371]: cluster 2024-09-12T23:55:26.981156+0000 mgr.y (mgr.24661) 550 : cluster [DBG] pgmap v1069: 137 pgs: 13 creating+peering, 14 unknown, 110 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s 2024-09-12T23:55:28.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:28 smithi136 bash[76371]: cluster 2024-09-12T23:55:27.485406+0000 mon.a (mon.0) 1678 : cluster [DBG] osdmap e959: 8 total, 8 up, 8 in 2024-09-12T23:55:28.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:28 smithi181 bash[67587]: cluster 2024-09-12T23:55:26.981156+0000 mgr.y (mgr.24661) 550 : cluster [DBG] pgmap v1069: 137 pgs: 13 creating+peering, 14 unknown, 110 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s 2024-09-12T23:55:28.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:28 smithi181 bash[67587]: cluster 2024-09-12T23:55:27.485406+0000 mon.a (mon.0) 1678 : cluster [DBG] osdmap e959: 8 total, 8 up, 8 in 2024-09-12T23:55:29.543 INFO:tasks.workunit.client.0.smithi136.stdout:require_osd_release = reef 2024-09-12T23:55:29.545 INFO:tasks.workunit.client.0.smithi136.stdout:0 2024-09-12T23:55:29.608 INFO:tasks.workunit.client.0.smithi136.stdout:1 2024-09-12T23:55:29.610 INFO:tasks.workunit.client.0.smithi136.stdout:2 2024-09-12T23:55:29.612 INFO:tasks.workunit.client.0.smithi136.stdout:3 2024-09-12T23:55:29.615 INFO:tasks.workunit.client.0.smithi136.stdout:4 2024-09-12T23:55:29.617 INFO:tasks.workunit.client.0.smithi136.stdout:5 2024-09-12T23:55:29.619 INFO:tasks.workunit.client.0.smithi136.stdout:6 2024-09-12T23:55:29.621 INFO:tasks.workunit.client.0.smithi136.stdout:7 2024-09-12T23:55:29.624 INFO:tasks.workunit.client.0.smithi136.stdout:8 2024-09-12T23:55:29.626 INFO:tasks.workunit.client.0.smithi136.stdout:9 2024-09-12T23:55:29.628 INFO:tasks.workunit.client.0.smithi136.stdout:10 2024-09-12T23:55:29.631 INFO:tasks.workunit.client.0.smithi136.stdout:11 2024-09-12T23:55:29.633 INFO:tasks.workunit.client.0.smithi136.stdout:12 2024-09-12T23:55:29.635 INFO:tasks.workunit.client.0.smithi136.stdout:13 2024-09-12T23:55:29.637 INFO:tasks.workunit.client.0.smithi136.stdout:14 2024-09-12T23:55:29.639 INFO:tasks.workunit.client.0.smithi136.stdout:15 2024-09-12T23:55:29.641 INFO:tasks.workunit.client.0.smithi136.stdout:16 2024-09-12T23:55:29.643 INFO:tasks.workunit.client.0.smithi136.stdout:17 2024-09-12T23:55:29.645 INFO:tasks.workunit.client.0.smithi136.stdout:18 2024-09-12T23:55:29.647 INFO:tasks.workunit.client.0.smithi136.stdout:19 2024-09-12T23:55:29.649 INFO:tasks.workunit.client.0.smithi136.stdout:20 2024-09-12T23:55:29.651 INFO:tasks.workunit.client.0.smithi136.stdout:21 2024-09-12T23:55:29.653 INFO:tasks.workunit.client.0.smithi136.stdout:22 2024-09-12T23:55:29.656 INFO:tasks.workunit.client.0.smithi136.stdout:23 2024-09-12T23:55:29.658 INFO:tasks.workunit.client.0.smithi136.stdout:24 2024-09-12T23:55:29.661 INFO:tasks.workunit.client.0.smithi136.stdout:25 2024-09-12T23:55:29.663 INFO:tasks.workunit.client.0.smithi136.stdout:26 2024-09-12T23:55:29.665 INFO:tasks.workunit.client.0.smithi136.stdout:27 2024-09-12T23:55:29.668 INFO:tasks.workunit.client.0.smithi136.stdout:28 2024-09-12T23:55:29.670 INFO:tasks.workunit.client.0.smithi136.stdout:29 2024-09-12T23:55:29.672 INFO:tasks.workunit.client.0.smithi136.stdout:30 2024-09-12T23:55:29.674 INFO:tasks.workunit.client.0.smithi136.stdout:31 2024-09-12T23:55:29.677 INFO:tasks.workunit.client.0.smithi136.stdout:32 2024-09-12T23:55:29.679 INFO:tasks.workunit.client.0.smithi136.stdout:33 2024-09-12T23:55:29.681 INFO:tasks.workunit.client.0.smithi136.stdout:34 2024-09-12T23:55:29.683 INFO:tasks.workunit.client.0.smithi136.stdout:35 2024-09-12T23:55:29.685 INFO:tasks.workunit.client.0.smithi136.stdout:36 2024-09-12T23:55:29.687 INFO:tasks.workunit.client.0.smithi136.stdout:37 2024-09-12T23:55:29.690 INFO:tasks.workunit.client.0.smithi136.stdout:38 2024-09-12T23:55:29.692 INFO:tasks.workunit.client.0.smithi136.stdout:39 2024-09-12T23:55:29.694 INFO:tasks.workunit.client.0.smithi136.stdout:40 2024-09-12T23:55:29.696 INFO:tasks.workunit.client.0.smithi136.stdout:41 2024-09-12T23:55:29.698 INFO:tasks.workunit.client.0.smithi136.stdout:42 2024-09-12T23:55:29.700 INFO:tasks.workunit.client.0.smithi136.stdout:43 2024-09-12T23:55:29.703 INFO:tasks.workunit.client.0.smithi136.stdout:44 2024-09-12T23:55:29.706 INFO:tasks.workunit.client.0.smithi136.stdout:45 2024-09-12T23:55:29.712 INFO:tasks.workunit.client.0.smithi136.stdout:46 2024-09-12T23:55:29.715 INFO:tasks.workunit.client.0.smithi136.stdout:47 2024-09-12T23:55:29.717 INFO:tasks.workunit.client.0.smithi136.stdout:48 2024-09-12T23:55:29.719 INFO:tasks.workunit.client.0.smithi136.stdout:49 2024-09-12T23:55:29.721 INFO:tasks.workunit.client.0.smithi136.stdout:50 2024-09-12T23:55:29.723 INFO:tasks.workunit.client.0.smithi136.stdout:51 2024-09-12T23:55:29.725 INFO:tasks.workunit.client.0.smithi136.stdout:52 2024-09-12T23:55:29.728 INFO:tasks.workunit.client.0.smithi136.stdout:53 2024-09-12T23:55:29.730 INFO:tasks.workunit.client.0.smithi136.stdout:54 2024-09-12T23:55:29.732 INFO:tasks.workunit.client.0.smithi136.stdout:55 2024-09-12T23:55:29.734 INFO:tasks.workunit.client.0.smithi136.stdout:56 2024-09-12T23:55:29.735 INFO:tasks.workunit.client.0.smithi136.stdout:57 2024-09-12T23:55:29.737 INFO:tasks.workunit.client.0.smithi136.stdout:58 2024-09-12T23:55:29.739 INFO:tasks.workunit.client.0.smithi136.stdout:59 2024-09-12T23:55:29.742 INFO:tasks.workunit.client.0.smithi136.stdout:60 2024-09-12T23:55:29.744 INFO:tasks.workunit.client.0.smithi136.stdout:61 2024-09-12T23:55:29.746 INFO:tasks.workunit.client.0.smithi136.stdout:62 2024-09-12T23:55:29.748 INFO:tasks.workunit.client.0.smithi136.stdout:63 2024-09-12T23:55:29.750 INFO:tasks.workunit.client.0.smithi136.stdout:64 2024-09-12T23:55:29.752 INFO:tasks.workunit.client.0.smithi136.stdout:65 2024-09-12T23:55:29.755 INFO:tasks.workunit.client.0.smithi136.stdout:66 2024-09-12T23:55:29.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:55:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:55:29] "GET /metrics HTTP/1.1" 200 36454 "" "Prometheus/2.43.0" 2024-09-12T23:55:29.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:29 smithi136 bash[76371]: cluster 2024-09-12T23:55:28.512390+0000 mon.a (mon.0) 1679 : cluster [DBG] osdmap e960: 8 total, 8 up, 8 in 2024-09-12T23:55:29.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:29 smithi136 bash[76371]: audit 2024-09-12T23:55:28.519697+0000 mon.c (mon.2) 38 : audit [INF] from='client.? 172.21.15.136:0/1028442975' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109763-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:29.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:29 smithi136 bash[76371]: audit 2024-09-12T23:55:28.521442+0000 mon.a (mon.0) 1680 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109763-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:29.758 INFO:tasks.workunit.client.0.smithi136.stdout:67 2024-09-12T23:55:29.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:29 smithi136 bash[75164]: cluster 2024-09-12T23:55:28.512390+0000 mon.a (mon.0) 1679 : cluster [DBG] osdmap e960: 8 total, 8 up, 8 in 2024-09-12T23:55:29.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:29 smithi136 bash[75164]: audit 2024-09-12T23:55:28.519697+0000 mon.c (mon.2) 38 : audit [INF] from='client.? 172.21.15.136:0/1028442975' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109763-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:29.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:29 smithi136 bash[75164]: audit 2024-09-12T23:55:28.521442+0000 mon.a (mon.0) 1680 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109763-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:29.759 INFO:tasks.workunit.client.0.smithi136.stdout:68 2024-09-12T23:55:29.761 INFO:tasks.workunit.client.0.smithi136.stdout:69 2024-09-12T23:55:29.763 INFO:tasks.workunit.client.0.smithi136.stdout:70 2024-09-12T23:55:29.765 INFO:tasks.workunit.client.0.smithi136.stdout:71 2024-09-12T23:55:29.767 INFO:tasks.workunit.client.0.smithi136.stdout:72 2024-09-12T23:55:29.770 INFO:tasks.workunit.client.0.smithi136.stdout:73 2024-09-12T23:55:29.772 INFO:tasks.workunit.client.0.smithi136.stdout:74 2024-09-12T23:55:29.775 INFO:tasks.workunit.client.0.smithi136.stdout:75 2024-09-12T23:55:29.777 INFO:tasks.workunit.client.0.smithi136.stdout:76 2024-09-12T23:55:29.779 INFO:tasks.workunit.client.0.smithi136.stdout:77 2024-09-12T23:55:29.781 INFO:tasks.workunit.client.0.smithi136.stdout:78 2024-09-12T23:55:29.784 INFO:tasks.workunit.client.0.smithi136.stdout:79 2024-09-12T23:55:29.785 INFO:tasks.workunit.client.0.smithi136.stdout:80 2024-09-12T23:55:29.788 INFO:tasks.workunit.client.0.smithi136.stdout:81 2024-09-12T23:55:29.790 INFO:tasks.workunit.client.0.smithi136.stdout:82 2024-09-12T23:55:29.792 INFO:tasks.workunit.client.0.smithi136.stdout:83 2024-09-12T23:55:29.795 INFO:tasks.workunit.client.0.smithi136.stdout:84 2024-09-12T23:55:29.797 INFO:tasks.workunit.client.0.smithi136.stdout:85 2024-09-12T23:55:29.799 INFO:tasks.workunit.client.0.smithi136.stdout:86 2024-09-12T23:55:29.801 INFO:tasks.workunit.client.0.smithi136.stdout:87 2024-09-12T23:55:29.803 INFO:tasks.workunit.client.0.smithi136.stdout:88 2024-09-12T23:55:29.805 INFO:tasks.workunit.client.0.smithi136.stdout:89 2024-09-12T23:55:29.807 INFO:tasks.workunit.client.0.smithi136.stdout:90 2024-09-12T23:55:29.809 INFO:tasks.workunit.client.0.smithi136.stdout:91 2024-09-12T23:55:29.812 INFO:tasks.workunit.client.0.smithi136.stdout:92 2024-09-12T23:55:29.814 INFO:tasks.workunit.client.0.smithi136.stdout:93 2024-09-12T23:55:29.816 INFO:tasks.workunit.client.0.smithi136.stdout:94 2024-09-12T23:55:29.818 INFO:tasks.workunit.client.0.smithi136.stdout:95 2024-09-12T23:55:29.820 INFO:tasks.workunit.client.0.smithi136.stdout:96 2024-09-12T23:55:29.822 INFO:tasks.workunit.client.0.smithi136.stdout:97 2024-09-12T23:55:29.824 INFO:tasks.workunit.client.0.smithi136.stdout:98 2024-09-12T23:55:29.827 INFO:tasks.workunit.client.0.smithi136.stdout:99 2024-09-12T23:55:29.829 INFO:tasks.workunit.client.0.smithi136.stdout:100 2024-09-12T23:55:29.831 INFO:tasks.workunit.client.0.smithi136.stdout:101 2024-09-12T23:55:29.833 INFO:tasks.workunit.client.0.smithi136.stdout:102 2024-09-12T23:55:29.835 INFO:tasks.workunit.client.0.smithi136.stdout:103 2024-09-12T23:55:29.837 INFO:tasks.workunit.client.0.smithi136.stdout:104 2024-09-12T23:55:29.840 INFO:tasks.workunit.client.0.smithi136.stdout:105 2024-09-12T23:55:29.842 INFO:tasks.workunit.client.0.smithi136.stdout:106 2024-09-12T23:55:29.845 INFO:tasks.workunit.client.0.smithi136.stdout:107 2024-09-12T23:55:29.847 INFO:tasks.workunit.client.0.smithi136.stdout:108 2024-09-12T23:55:29.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:29 smithi181 bash[67587]: cluster 2024-09-12T23:55:28.512390+0000 mon.a (mon.0) 1679 : cluster [DBG] osdmap e960: 8 total, 8 up, 8 in 2024-09-12T23:55:29.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:29 smithi181 bash[67587]: audit 2024-09-12T23:55:28.519697+0000 mon.c (mon.2) 38 : audit [INF] from='client.? 172.21.15.136:0/1028442975' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109763-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:29.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:29 smithi181 bash[67587]: audit 2024-09-12T23:55:28.521442+0000 mon.a (mon.0) 1680 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109763-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:29.849 INFO:tasks.workunit.client.0.smithi136.stdout:109 2024-09-12T23:55:29.852 INFO:tasks.workunit.client.0.smithi136.stdout:110 2024-09-12T23:55:29.854 INFO:tasks.workunit.client.0.smithi136.stdout:111 2024-09-12T23:55:29.857 INFO:tasks.workunit.client.0.smithi136.stdout:112 2024-09-12T23:55:29.859 INFO:tasks.workunit.client.0.smithi136.stdout:113 2024-09-12T23:55:29.862 INFO:tasks.workunit.client.0.smithi136.stdout:114 2024-09-12T23:55:29.864 INFO:tasks.workunit.client.0.smithi136.stdout:115 2024-09-12T23:55:29.866 INFO:tasks.workunit.client.0.smithi136.stdout:116 2024-09-12T23:55:29.868 INFO:tasks.workunit.client.0.smithi136.stdout:117 2024-09-12T23:55:29.871 INFO:tasks.workunit.client.0.smithi136.stdout:118 2024-09-12T23:55:29.873 INFO:tasks.workunit.client.0.smithi136.stdout:119 2024-09-12T23:55:29.876 INFO:tasks.workunit.client.0.smithi136.stdout:120 2024-09-12T23:55:29.878 INFO:tasks.workunit.client.0.smithi136.stdout:121 2024-09-12T23:55:29.881 INFO:tasks.workunit.client.0.smithi136.stdout:122 2024-09-12T23:55:29.883 INFO:tasks.workunit.client.0.smithi136.stdout:123 2024-09-12T23:55:29.885 INFO:tasks.workunit.client.0.smithi136.stdout:124 2024-09-12T23:55:29.888 INFO:tasks.workunit.client.0.smithi136.stdout:125 2024-09-12T23:55:29.892 INFO:tasks.workunit.client.0.smithi136.stdout:126 2024-09-12T23:55:29.893 INFO:tasks.workunit.client.0.smithi136.stdout:127 2024-09-12T23:55:29.895 INFO:tasks.workunit.client.0.smithi136.stdout:128 2024-09-12T23:55:29.898 INFO:tasks.workunit.client.0.smithi136.stdout:129 2024-09-12T23:55:29.900 INFO:tasks.workunit.client.0.smithi136.stdout:130 2024-09-12T23:55:29.902 INFO:tasks.workunit.client.0.smithi136.stdout:131 2024-09-12T23:55:29.905 INFO:tasks.workunit.client.0.smithi136.stdout:132 2024-09-12T23:55:29.907 INFO:tasks.workunit.client.0.smithi136.stdout:133 2024-09-12T23:55:29.909 INFO:tasks.workunit.client.0.smithi136.stdout:134 2024-09-12T23:55:29.913 INFO:tasks.workunit.client.0.smithi136.stdout:135 2024-09-12T23:55:29.916 INFO:tasks.workunit.client.0.smithi136.stdout:136 2024-09-12T23:55:29.917 INFO:tasks.workunit.client.0.smithi136.stdout:137 2024-09-12T23:55:29.919 INFO:tasks.workunit.client.0.smithi136.stdout:138 2024-09-12T23:55:29.921 INFO:tasks.workunit.client.0.smithi136.stdout:139 2024-09-12T23:55:29.924 INFO:tasks.workunit.client.0.smithi136.stdout:140 2024-09-12T23:55:29.927 INFO:tasks.workunit.client.0.smithi136.stdout:141 2024-09-12T23:55:29.929 INFO:tasks.workunit.client.0.smithi136.stdout:142 2024-09-12T23:55:29.931 INFO:tasks.workunit.client.0.smithi136.stdout:143 2024-09-12T23:55:29.934 INFO:tasks.workunit.client.0.smithi136.stdout:144 2024-09-12T23:55:29.936 INFO:tasks.workunit.client.0.smithi136.stdout:145 2024-09-12T23:55:29.939 INFO:tasks.workunit.client.0.smithi136.stdout:146 2024-09-12T23:55:29.942 INFO:tasks.workunit.client.0.smithi136.stdout:147 2024-09-12T23:55:29.944 INFO:tasks.workunit.client.0.smithi136.stdout:148 2024-09-12T23:55:29.947 INFO:tasks.workunit.client.0.smithi136.stdout:149 2024-09-12T23:55:29.949 INFO:tasks.workunit.client.0.smithi136.stdout:150 2024-09-12T23:55:29.952 INFO:tasks.workunit.client.0.smithi136.stdout:151 2024-09-12T23:55:29.954 INFO:tasks.workunit.client.0.smithi136.stdout:152 2024-09-12T23:55:29.957 INFO:tasks.workunit.client.0.smithi136.stdout:153 2024-09-12T23:55:29.960 INFO:tasks.workunit.client.0.smithi136.stdout:154 2024-09-12T23:55:29.962 INFO:tasks.workunit.client.0.smithi136.stdout:155 2024-09-12T23:55:29.965 INFO:tasks.workunit.client.0.smithi136.stdout:156 2024-09-12T23:55:29.967 INFO:tasks.workunit.client.0.smithi136.stdout:157 2024-09-12T23:55:29.970 INFO:tasks.workunit.client.0.smithi136.stdout:158 2024-09-12T23:55:29.972 INFO:tasks.workunit.client.0.smithi136.stdout:159 2024-09-12T23:55:29.975 INFO:tasks.workunit.client.0.smithi136.stdout:160 2024-09-12T23:55:29.977 INFO:tasks.workunit.client.0.smithi136.stdout:161 2024-09-12T23:55:29.979 INFO:tasks.workunit.client.0.smithi136.stdout:162 2024-09-12T23:55:29.982 INFO:tasks.workunit.client.0.smithi136.stdout:163 2024-09-12T23:55:29.984 INFO:tasks.workunit.client.0.smithi136.stdout:164 2024-09-12T23:55:29.987 INFO:tasks.workunit.client.0.smithi136.stdout:165 2024-09-12T23:55:29.990 INFO:tasks.workunit.client.0.smithi136.stdout:166 2024-09-12T23:55:29.992 INFO:tasks.workunit.client.0.smithi136.stdout:167 2024-09-12T23:55:29.995 INFO:tasks.workunit.client.0.smithi136.stdout:168 2024-09-12T23:55:29.997 INFO:tasks.workunit.client.0.smithi136.stdout:169 2024-09-12T23:55:30.000 INFO:tasks.workunit.client.0.smithi136.stdout:170 2024-09-12T23:55:30.002 INFO:tasks.workunit.client.0.smithi136.stdout:171 2024-09-12T23:55:30.005 INFO:tasks.workunit.client.0.smithi136.stdout:172 2024-09-12T23:55:30.007 INFO:tasks.workunit.client.0.smithi136.stdout:173 2024-09-12T23:55:30.010 INFO:tasks.workunit.client.0.smithi136.stdout:174 2024-09-12T23:55:30.013 INFO:tasks.workunit.client.0.smithi136.stdout:175 2024-09-12T23:55:30.016 INFO:tasks.workunit.client.0.smithi136.stdout:176 2024-09-12T23:55:30.019 INFO:tasks.workunit.client.0.smithi136.stdout:177 2024-09-12T23:55:30.021 INFO:tasks.workunit.client.0.smithi136.stdout:178 2024-09-12T23:55:30.024 INFO:tasks.workunit.client.0.smithi136.stdout:179 2024-09-12T23:55:30.027 INFO:tasks.workunit.client.0.smithi136.stdout:180 2024-09-12T23:55:30.029 INFO:tasks.workunit.client.0.smithi136.stdout:181 2024-09-12T23:55:30.032 INFO:tasks.workunit.client.0.smithi136.stdout:182 2024-09-12T23:55:30.035 INFO:tasks.workunit.client.0.smithi136.stdout:183 2024-09-12T23:55:30.037 INFO:tasks.workunit.client.0.smithi136.stdout:184 2024-09-12T23:55:30.040 INFO:tasks.workunit.client.0.smithi136.stdout:185 2024-09-12T23:55:30.042 INFO:tasks.workunit.client.0.smithi136.stdout:186 2024-09-12T23:55:30.045 INFO:tasks.workunit.client.0.smithi136.stdout:187 2024-09-12T23:55:30.048 INFO:tasks.workunit.client.0.smithi136.stdout:188 2024-09-12T23:55:30.051 INFO:tasks.workunit.client.0.smithi136.stdout:189 2024-09-12T23:55:30.053 INFO:tasks.workunit.client.0.smithi136.stdout:190 2024-09-12T23:55:30.056 INFO:tasks.workunit.client.0.smithi136.stdout:191 2024-09-12T23:55:30.058 INFO:tasks.workunit.client.0.smithi136.stdout:192 2024-09-12T23:55:30.061 INFO:tasks.workunit.client.0.smithi136.stdout:193 2024-09-12T23:55:30.063 INFO:tasks.workunit.client.0.smithi136.stdout:194 2024-09-12T23:55:30.066 INFO:tasks.workunit.client.0.smithi136.stdout:195 2024-09-12T23:55:30.069 INFO:tasks.workunit.client.0.smithi136.stdout:196 2024-09-12T23:55:30.072 INFO:tasks.workunit.client.0.smithi136.stdout:197 2024-09-12T23:55:30.075 INFO:tasks.workunit.client.0.smithi136.stdout:198 2024-09-12T23:55:30.077 INFO:tasks.workunit.client.0.smithi136.stdout:199 2024-09-12T23:55:30.080 INFO:tasks.workunit.client.0.smithi136.stdout:200 2024-09-12T23:55:30.083 INFO:tasks.workunit.client.0.smithi136.stdout:201 2024-09-12T23:55:30.086 INFO:tasks.workunit.client.0.smithi136.stdout:202 2024-09-12T23:55:30.089 INFO:tasks.workunit.client.0.smithi136.stdout:203 2024-09-12T23:55:30.091 INFO:tasks.workunit.client.0.smithi136.stdout:204 2024-09-12T23:55:30.095 INFO:tasks.workunit.client.0.smithi136.stdout:205 2024-09-12T23:55:30.096 INFO:tasks.workunit.client.0.smithi136.stdout:206 2024-09-12T23:55:30.099 INFO:tasks.workunit.client.0.smithi136.stdout:207 2024-09-12T23:55:30.101 INFO:tasks.workunit.client.0.smithi136.stdout:208 2024-09-12T23:55:30.104 INFO:tasks.workunit.client.0.smithi136.stdout:209 2024-09-12T23:55:30.106 INFO:tasks.workunit.client.0.smithi136.stdout:210 2024-09-12T23:55:30.109 INFO:tasks.workunit.client.0.smithi136.stdout:211 2024-09-12T23:55:30.111 INFO:tasks.workunit.client.0.smithi136.stdout:212 2024-09-12T23:55:30.114 INFO:tasks.workunit.client.0.smithi136.stdout:213 2024-09-12T23:55:30.117 INFO:tasks.workunit.client.0.smithi136.stdout:214 2024-09-12T23:55:30.120 INFO:tasks.workunit.client.0.smithi136.stdout:215 2024-09-12T23:55:30.122 INFO:tasks.workunit.client.0.smithi136.stdout:216 2024-09-12T23:55:30.124 INFO:tasks.workunit.client.0.smithi136.stdout:217 2024-09-12T23:55:30.127 INFO:tasks.workunit.client.0.smithi136.stdout:218 2024-09-12T23:55:30.129 INFO:tasks.workunit.client.0.smithi136.stdout:219 2024-09-12T23:55:30.132 INFO:tasks.workunit.client.0.smithi136.stdout:220 2024-09-12T23:55:30.134 INFO:tasks.workunit.client.0.smithi136.stdout:221 2024-09-12T23:55:30.137 INFO:tasks.workunit.client.0.smithi136.stdout:222 2024-09-12T23:55:30.140 INFO:tasks.workunit.client.0.smithi136.stdout:223 2024-09-12T23:55:30.143 INFO:tasks.workunit.client.0.smithi136.stdout:224 2024-09-12T23:55:30.146 INFO:tasks.workunit.client.0.smithi136.stdout:225 2024-09-12T23:55:30.149 INFO:tasks.workunit.client.0.smithi136.stdout:226 2024-09-12T23:55:30.151 INFO:tasks.workunit.client.0.smithi136.stdout:227 2024-09-12T23:55:30.154 INFO:tasks.workunit.client.0.smithi136.stdout:228 2024-09-12T23:55:30.157 INFO:tasks.workunit.client.0.smithi136.stdout:229 2024-09-12T23:55:30.160 INFO:tasks.workunit.client.0.smithi136.stdout:230 2024-09-12T23:55:30.163 INFO:tasks.workunit.client.0.smithi136.stdout:231 2024-09-12T23:55:30.166 INFO:tasks.workunit.client.0.smithi136.stdout:232 2024-09-12T23:55:30.168 INFO:tasks.workunit.client.0.smithi136.stdout:233 2024-09-12T23:55:30.170 INFO:tasks.workunit.client.0.smithi136.stdout:234 2024-09-12T23:55:30.173 INFO:tasks.workunit.client.0.smithi136.stdout:235 2024-09-12T23:55:30.176 INFO:tasks.workunit.client.0.smithi136.stdout:236 2024-09-12T23:55:30.179 INFO:tasks.workunit.client.0.smithi136.stdout:237 2024-09-12T23:55:30.182 INFO:tasks.workunit.client.0.smithi136.stdout:238 2024-09-12T23:55:30.185 INFO:tasks.workunit.client.0.smithi136.stdout:239 2024-09-12T23:55:30.188 INFO:tasks.workunit.client.0.smithi136.stdout:240 2024-09-12T23:55:30.190 INFO:tasks.workunit.client.0.smithi136.stdout:241 2024-09-12T23:55:30.193 INFO:tasks.workunit.client.0.smithi136.stdout:242 2024-09-12T23:55:30.195 INFO:tasks.workunit.client.0.smithi136.stdout:243 2024-09-12T23:55:30.198 INFO:tasks.workunit.client.0.smithi136.stdout:244 2024-09-12T23:55:30.200 INFO:tasks.workunit.client.0.smithi136.stdout:245 2024-09-12T23:55:30.203 INFO:tasks.workunit.client.0.smithi136.stdout:246 2024-09-12T23:55:30.205 INFO:tasks.workunit.client.0.smithi136.stdout:247 2024-09-12T23:55:30.207 INFO:tasks.workunit.client.0.smithi136.stdout:248 2024-09-12T23:55:30.210 INFO:tasks.workunit.client.0.smithi136.stdout:249 2024-09-12T23:55:30.212 INFO:tasks.workunit.client.0.smithi136.stdout:250 2024-09-12T23:55:30.214 INFO:tasks.workunit.client.0.smithi136.stdout:251 2024-09-12T23:55:30.217 INFO:tasks.workunit.client.0.smithi136.stdout:252 2024-09-12T23:55:30.219 INFO:tasks.workunit.client.0.smithi136.stdout:253 2024-09-12T23:55:30.221 INFO:tasks.workunit.client.0.smithi136.stdout:254 2024-09-12T23:55:30.224 INFO:tasks.workunit.client.0.smithi136.stdout:255 2024-09-12T23:55:30.227 INFO:tasks.workunit.client.0.smithi136.stdout:256 2024-09-12T23:55:30.229 INFO:tasks.workunit.client.0.smithi136.stdout:257 2024-09-12T23:55:30.232 INFO:tasks.workunit.client.0.smithi136.stdout:258 2024-09-12T23:55:30.234 INFO:tasks.workunit.client.0.smithi136.stdout:259 2024-09-12T23:55:30.237 INFO:tasks.workunit.client.0.smithi136.stdout:260 2024-09-12T23:55:30.239 INFO:tasks.workunit.client.0.smithi136.stdout:261 2024-09-12T23:55:30.242 INFO:tasks.workunit.client.0.smithi136.stdout:262 2024-09-12T23:55:30.244 INFO:tasks.workunit.client.0.smithi136.stdout:263 2024-09-12T23:55:30.246 INFO:tasks.workunit.client.0.smithi136.stdout:264 2024-09-12T23:55:30.249 INFO:tasks.workunit.client.0.smithi136.stdout:265 2024-09-12T23:55:30.252 INFO:tasks.workunit.client.0.smithi136.stdout:266 2024-09-12T23:55:30.254 INFO:tasks.workunit.client.0.smithi136.stdout:267 2024-09-12T23:55:30.257 INFO:tasks.workunit.client.0.smithi136.stdout:268 2024-09-12T23:55:30.260 INFO:tasks.workunit.client.0.smithi136.stdout:269 2024-09-12T23:55:30.262 INFO:tasks.workunit.client.0.smithi136.stdout:270 2024-09-12T23:55:30.265 INFO:tasks.workunit.client.0.smithi136.stdout:271 2024-09-12T23:55:30.267 INFO:tasks.workunit.client.0.smithi136.stdout:272 2024-09-12T23:55:30.269 INFO:tasks.workunit.client.0.smithi136.stdout:273 2024-09-12T23:55:30.272 INFO:tasks.workunit.client.0.smithi136.stdout:274 2024-09-12T23:55:30.275 INFO:tasks.workunit.client.0.smithi136.stdout:275 2024-09-12T23:55:30.277 INFO:tasks.workunit.client.0.smithi136.stdout:276 2024-09-12T23:55:30.280 INFO:tasks.workunit.client.0.smithi136.stdout:277 2024-09-12T23:55:30.282 INFO:tasks.workunit.client.0.smithi136.stdout:278 2024-09-12T23:55:30.285 INFO:tasks.workunit.client.0.smithi136.stdout:279 2024-09-12T23:55:30.288 INFO:tasks.workunit.client.0.smithi136.stdout:280 2024-09-12T23:55:30.290 INFO:tasks.workunit.client.0.smithi136.stdout:281 2024-09-12T23:55:30.293 INFO:tasks.workunit.client.0.smithi136.stdout:282 2024-09-12T23:55:30.296 INFO:tasks.workunit.client.0.smithi136.stdout:283 2024-09-12T23:55:30.298 INFO:tasks.workunit.client.0.smithi136.stdout:284 2024-09-12T23:55:30.301 INFO:tasks.workunit.client.0.smithi136.stdout:285 2024-09-12T23:55:30.305 INFO:tasks.workunit.client.0.smithi136.stdout:286 2024-09-12T23:55:30.307 INFO:tasks.workunit.client.0.smithi136.stdout:287 2024-09-12T23:55:30.309 INFO:tasks.workunit.client.0.smithi136.stdout:288 2024-09-12T23:55:30.313 INFO:tasks.workunit.client.0.smithi136.stdout:289 2024-09-12T23:55:30.315 INFO:tasks.workunit.client.0.smithi136.stdout:290 2024-09-12T23:55:30.318 INFO:tasks.workunit.client.0.smithi136.stdout:291 2024-09-12T23:55:30.320 INFO:tasks.workunit.client.0.smithi136.stdout:292 2024-09-12T23:55:30.324 INFO:tasks.workunit.client.0.smithi136.stdout:293 2024-09-12T23:55:30.326 INFO:tasks.workunit.client.0.smithi136.stdout:294 2024-09-12T23:55:30.329 INFO:tasks.workunit.client.0.smithi136.stdout:295 2024-09-12T23:55:30.331 INFO:tasks.workunit.client.0.smithi136.stdout:296 2024-09-12T23:55:30.334 INFO:tasks.workunit.client.0.smithi136.stdout:297 2024-09-12T23:55:30.336 INFO:tasks.workunit.client.0.smithi136.stdout:298 2024-09-12T23:55:30.339 INFO:tasks.workunit.client.0.smithi136.stdout:299 2024-09-12T23:55:30.341 INFO:tasks.workunit.client.0.smithi136.stdout:300 2024-09-12T23:55:30.344 INFO:tasks.workunit.client.0.smithi136.stdout:301 2024-09-12T23:55:30.346 INFO:tasks.workunit.client.0.smithi136.stdout:302 2024-09-12T23:55:30.349 INFO:tasks.workunit.client.0.smithi136.stdout:303 2024-09-12T23:55:30.351 INFO:tasks.workunit.client.0.smithi136.stdout:304 2024-09-12T23:55:30.354 INFO:tasks.workunit.client.0.smithi136.stdout:305 2024-09-12T23:55:30.357 INFO:tasks.workunit.client.0.smithi136.stdout:306 2024-09-12T23:55:30.360 INFO:tasks.workunit.client.0.smithi136.stdout:307 2024-09-12T23:55:30.362 INFO:tasks.workunit.client.0.smithi136.stdout:308 2024-09-12T23:55:30.365 INFO:tasks.workunit.client.0.smithi136.stdout:309 2024-09-12T23:55:30.368 INFO:tasks.workunit.client.0.smithi136.stdout:310 2024-09-12T23:55:30.371 INFO:tasks.workunit.client.0.smithi136.stdout:311 2024-09-12T23:55:30.374 INFO:tasks.workunit.client.0.smithi136.stdout:312 2024-09-12T23:55:30.376 INFO:tasks.workunit.client.0.smithi136.stdout:313 2024-09-12T23:55:30.379 INFO:tasks.workunit.client.0.smithi136.stdout:314 2024-09-12T23:55:30.381 INFO:tasks.workunit.client.0.smithi136.stdout:315 2024-09-12T23:55:30.384 INFO:tasks.workunit.client.0.smithi136.stdout:316 2024-09-12T23:55:30.387 INFO:tasks.workunit.client.0.smithi136.stdout:317 2024-09-12T23:55:30.388 INFO:tasks.workunit.client.0.smithi136.stdout:318 2024-09-12T23:55:30.390 INFO:tasks.workunit.client.0.smithi136.stdout:319 2024-09-12T23:55:30.392 INFO:tasks.workunit.client.0.smithi136.stdout:320 2024-09-12T23:55:30.393 INFO:tasks.workunit.client.0.smithi136.stdout:321 2024-09-12T23:55:30.395 INFO:tasks.workunit.client.0.smithi136.stdout:322 2024-09-12T23:55:30.397 INFO:tasks.workunit.client.0.smithi136.stdout:323 2024-09-12T23:55:30.398 INFO:tasks.workunit.client.0.smithi136.stdout:324 2024-09-12T23:55:30.400 INFO:tasks.workunit.client.0.smithi136.stdout:325 2024-09-12T23:55:30.401 INFO:tasks.workunit.client.0.smithi136.stdout:326 2024-09-12T23:55:30.403 INFO:tasks.workunit.client.0.smithi136.stdout:327 2024-09-12T23:55:30.405 INFO:tasks.workunit.client.0.smithi136.stdout:328 2024-09-12T23:55:30.406 INFO:tasks.workunit.client.0.smithi136.stdout:329 2024-09-12T23:55:30.408 INFO:tasks.workunit.client.0.smithi136.stdout:330 2024-09-12T23:55:30.410 INFO:tasks.workunit.client.0.smithi136.stdout:331 2024-09-12T23:55:30.411 INFO:tasks.workunit.client.0.smithi136.stdout:332 2024-09-12T23:55:30.413 INFO:tasks.workunit.client.0.smithi136.stdout:333 2024-09-12T23:55:30.414 INFO:tasks.workunit.client.0.smithi136.stdout:334 2024-09-12T23:55:30.416 INFO:tasks.workunit.client.0.smithi136.stdout:335 2024-09-12T23:55:30.417 INFO:tasks.workunit.client.0.smithi136.stdout:336 2024-09-12T23:55:30.419 INFO:tasks.workunit.client.0.smithi136.stdout:337 2024-09-12T23:55:30.420 INFO:tasks.workunit.client.0.smithi136.stdout:338 2024-09-12T23:55:30.422 INFO:tasks.workunit.client.0.smithi136.stdout:339 2024-09-12T23:55:30.423 INFO:tasks.workunit.client.0.smithi136.stdout:340 2024-09-12T23:55:30.424 INFO:tasks.workunit.client.0.smithi136.stdout:341 2024-09-12T23:55:30.426 INFO:tasks.workunit.client.0.smithi136.stdout:342 2024-09-12T23:55:30.427 INFO:tasks.workunit.client.0.smithi136.stdout:343 2024-09-12T23:55:30.429 INFO:tasks.workunit.client.0.smithi136.stdout:344 2024-09-12T23:55:30.430 INFO:tasks.workunit.client.0.smithi136.stdout:345 2024-09-12T23:55:30.432 INFO:tasks.workunit.client.0.smithi136.stdout:346 2024-09-12T23:55:30.433 INFO:tasks.workunit.client.0.smithi136.stdout:347 2024-09-12T23:55:30.435 INFO:tasks.workunit.client.0.smithi136.stdout:348 2024-09-12T23:55:30.436 INFO:tasks.workunit.client.0.smithi136.stdout:349 2024-09-12T23:55:30.437 INFO:tasks.workunit.client.0.smithi136.stdout:350 2024-09-12T23:55:30.439 INFO:tasks.workunit.client.0.smithi136.stdout:351 2024-09-12T23:55:30.440 INFO:tasks.workunit.client.0.smithi136.stdout:352 2024-09-12T23:55:30.442 INFO:tasks.workunit.client.0.smithi136.stdout:353 2024-09-12T23:55:30.443 INFO:tasks.workunit.client.0.smithi136.stdout:354 2024-09-12T23:55:30.445 INFO:tasks.workunit.client.0.smithi136.stdout:355 2024-09-12T23:55:30.446 INFO:tasks.workunit.client.0.smithi136.stdout:356 2024-09-12T23:55:30.448 INFO:tasks.workunit.client.0.smithi136.stdout:357 2024-09-12T23:55:30.450 INFO:tasks.workunit.client.0.smithi136.stdout:358 2024-09-12T23:55:30.451 INFO:tasks.workunit.client.0.smithi136.stdout:359 2024-09-12T23:55:30.453 INFO:tasks.workunit.client.0.smithi136.stdout:360 2024-09-12T23:55:30.454 INFO:tasks.workunit.client.0.smithi136.stdout:361 2024-09-12T23:55:30.456 INFO:tasks.workunit.client.0.smithi136.stdout:362 2024-09-12T23:55:30.457 INFO:tasks.workunit.client.0.smithi136.stdout:363 2024-09-12T23:55:30.458 INFO:tasks.workunit.client.0.smithi136.stdout:364 2024-09-12T23:55:30.460 INFO:tasks.workunit.client.0.smithi136.stdout:365 2024-09-12T23:55:30.461 INFO:tasks.workunit.client.0.smithi136.stdout:366 2024-09-12T23:55:30.463 INFO:tasks.workunit.client.0.smithi136.stdout:367 2024-09-12T23:55:30.465 INFO:tasks.workunit.client.0.smithi136.stdout:368 2024-09-12T23:55:30.466 INFO:tasks.workunit.client.0.smithi136.stdout:369 2024-09-12T23:55:30.468 INFO:tasks.workunit.client.0.smithi136.stdout:370 2024-09-12T23:55:30.470 INFO:tasks.workunit.client.0.smithi136.stdout:371 2024-09-12T23:55:30.471 INFO:tasks.workunit.client.0.smithi136.stdout:372 2024-09-12T23:55:30.473 INFO:tasks.workunit.client.0.smithi136.stdout:373 2024-09-12T23:55:30.474 INFO:tasks.workunit.client.0.smithi136.stdout:374 2024-09-12T23:55:30.476 INFO:tasks.workunit.client.0.smithi136.stdout:375 2024-09-12T23:55:30.477 INFO:tasks.workunit.client.0.smithi136.stdout:376 2024-09-12T23:55:30.479 INFO:tasks.workunit.client.0.smithi136.stdout:377 2024-09-12T23:55:30.480 INFO:tasks.workunit.client.0.smithi136.stdout:378 2024-09-12T23:55:30.482 INFO:tasks.workunit.client.0.smithi136.stdout:379 2024-09-12T23:55:30.483 INFO:tasks.workunit.client.0.smithi136.stdout:380 2024-09-12T23:55:30.485 INFO:tasks.workunit.client.0.smithi136.stdout:381 2024-09-12T23:55:30.486 INFO:tasks.workunit.client.0.smithi136.stdout:382 2024-09-12T23:55:30.488 INFO:tasks.workunit.client.0.smithi136.stdout:383 2024-09-12T23:55:30.489 INFO:tasks.workunit.client.0.smithi136.stdout:384 2024-09-12T23:55:30.491 INFO:tasks.workunit.client.0.smithi136.stdout:385 2024-09-12T23:55:30.492 INFO:tasks.workunit.client.0.smithi136.stdout:386 2024-09-12T23:55:30.493 INFO:tasks.workunit.client.0.smithi136.stdout:387 2024-09-12T23:55:30.495 INFO:tasks.workunit.client.0.smithi136.stdout:388 2024-09-12T23:55:30.496 INFO:tasks.workunit.client.0.smithi136.stdout:389 2024-09-12T23:55:30.498 INFO:tasks.workunit.client.0.smithi136.stdout:390 2024-09-12T23:55:30.499 INFO:tasks.workunit.client.0.smithi136.stdout:391 2024-09-12T23:55:30.501 INFO:tasks.workunit.client.0.smithi136.stdout:392 2024-09-12T23:55:30.503 INFO:tasks.workunit.client.0.smithi136.stdout:393 2024-09-12T23:55:30.504 INFO:tasks.workunit.client.0.smithi136.stdout:394 2024-09-12T23:55:30.506 INFO:tasks.workunit.client.0.smithi136.stdout:395 2024-09-12T23:55:30.510 INFO:tasks.workunit.client.0.smithi136.stdout:396 2024-09-12T23:55:30.514 INFO:tasks.workunit.client.0.smithi136.stdout:397 2024-09-12T23:55:30.517 INFO:tasks.workunit.client.0.smithi136.stdout:398 2024-09-12T23:55:30.519 INFO:tasks.workunit.client.0.smithi136.stdout:399 2024-09-12T23:55:30.520 INFO:tasks.workunit.client.0.smithi136.stdout:400 2024-09-12T23:55:30.522 INFO:tasks.workunit.client.0.smithi136.stdout:401 2024-09-12T23:55:30.524 INFO:tasks.workunit.client.0.smithi136.stdout:402 2024-09-12T23:55:30.525 INFO:tasks.workunit.client.0.smithi136.stdout:403 2024-09-12T23:55:30.528 INFO:tasks.workunit.client.0.smithi136.stdout:404 2024-09-12T23:55:30.529 INFO:tasks.workunit.client.0.smithi136.stdout:405 2024-09-12T23:55:30.530 INFO:tasks.workunit.client.0.smithi136.stdout:406 2024-09-12T23:55:30.533 INFO:tasks.workunit.client.0.smithi136.stdout:407 2024-09-12T23:55:30.535 INFO:tasks.workunit.client.0.smithi136.stdout:408 2024-09-12T23:55:30.538 INFO:tasks.workunit.client.0.smithi136.stdout:409 2024-09-12T23:55:30.539 INFO:tasks.workunit.client.0.smithi136.stdout:410 2024-09-12T23:55:30.541 INFO:tasks.workunit.client.0.smithi136.stdout:411 2024-09-12T23:55:30.543 INFO:tasks.workunit.client.0.smithi136.stdout:412 2024-09-12T23:55:30.545 INFO:tasks.workunit.client.0.smithi136.stdout:413 2024-09-12T23:55:30.546 INFO:tasks.workunit.client.0.smithi136.stdout:414 2024-09-12T23:55:30.548 INFO:tasks.workunit.client.0.smithi136.stdout:415 2024-09-12T23:55:30.550 INFO:tasks.workunit.client.0.smithi136.stdout:416 2024-09-12T23:55:30.552 INFO:tasks.workunit.client.0.smithi136.stdout:417 2024-09-12T23:55:30.554 INFO:tasks.workunit.client.0.smithi136.stdout:418 2024-09-12T23:55:30.556 INFO:tasks.workunit.client.0.smithi136.stdout:419 2024-09-12T23:55:30.557 INFO:tasks.workunit.client.0.smithi136.stdout:420 2024-09-12T23:55:30.559 INFO:tasks.workunit.client.0.smithi136.stdout:421 2024-09-12T23:55:30.561 INFO:tasks.workunit.client.0.smithi136.stdout:422 2024-09-12T23:55:30.562 INFO:tasks.workunit.client.0.smithi136.stdout:423 2024-09-12T23:55:30.564 INFO:tasks.workunit.client.0.smithi136.stdout:424 2024-09-12T23:55:30.566 INFO:tasks.workunit.client.0.smithi136.stdout:425 2024-09-12T23:55:30.567 INFO:tasks.workunit.client.0.smithi136.stdout:426 2024-09-12T23:55:30.569 INFO:tasks.workunit.client.0.smithi136.stdout:427 2024-09-12T23:55:30.571 INFO:tasks.workunit.client.0.smithi136.stdout:428 2024-09-12T23:55:30.573 INFO:tasks.workunit.client.0.smithi136.stdout:429 2024-09-12T23:55:30.575 INFO:tasks.workunit.client.0.smithi136.stdout:430 2024-09-12T23:55:30.577 INFO:tasks.workunit.client.0.smithi136.stdout:431 2024-09-12T23:55:30.578 INFO:tasks.workunit.client.0.smithi136.stdout:432 2024-09-12T23:55:30.580 INFO:tasks.workunit.client.0.smithi136.stdout:433 2024-09-12T23:55:30.582 INFO:tasks.workunit.client.0.smithi136.stdout:434 2024-09-12T23:55:30.584 INFO:tasks.workunit.client.0.smithi136.stdout:435 2024-09-12T23:55:30.585 INFO:tasks.workunit.client.0.smithi136.stdout:436 2024-09-12T23:55:30.588 INFO:tasks.workunit.client.0.smithi136.stdout:437 2024-09-12T23:55:30.590 INFO:tasks.workunit.client.0.smithi136.stdout:438 2024-09-12T23:55:30.591 INFO:tasks.workunit.client.0.smithi136.stdout:439 2024-09-12T23:55:30.594 INFO:tasks.workunit.client.0.smithi136.stdout:440 2024-09-12T23:55:30.596 INFO:tasks.workunit.client.0.smithi136.stdout:441 2024-09-12T23:55:30.598 INFO:tasks.workunit.client.0.smithi136.stdout:442 2024-09-12T23:55:30.600 INFO:tasks.workunit.client.0.smithi136.stdout:443 2024-09-12T23:55:30.602 INFO:tasks.workunit.client.0.smithi136.stdout:444 2024-09-12T23:55:30.604 INFO:tasks.workunit.client.0.smithi136.stdout:445 2024-09-12T23:55:30.605 INFO:tasks.workunit.client.0.smithi136.stdout:446 2024-09-12T23:55:30.608 INFO:tasks.workunit.client.0.smithi136.stdout:447 2024-09-12T23:55:30.610 INFO:tasks.workunit.client.0.smithi136.stdout:448 2024-09-12T23:55:30.612 INFO:tasks.workunit.client.0.smithi136.stdout:449 2024-09-12T23:55:30.614 INFO:tasks.workunit.client.0.smithi136.stdout:450 2024-09-12T23:55:30.616 INFO:tasks.workunit.client.0.smithi136.stdout:451 2024-09-12T23:55:30.618 INFO:tasks.workunit.client.0.smithi136.stdout:452 2024-09-12T23:55:30.620 INFO:tasks.workunit.client.0.smithi136.stdout:453 2024-09-12T23:55:30.622 INFO:tasks.workunit.client.0.smithi136.stdout:454 2024-09-12T23:55:30.624 INFO:tasks.workunit.client.0.smithi136.stdout:455 2024-09-12T23:55:30.626 INFO:tasks.workunit.client.0.smithi136.stdout:456 2024-09-12T23:55:30.628 INFO:tasks.workunit.client.0.smithi136.stdout:457 2024-09-12T23:55:30.630 INFO:tasks.workunit.client.0.smithi136.stdout:458 2024-09-12T23:55:30.632 INFO:tasks.workunit.client.0.smithi136.stdout:459 2024-09-12T23:55:30.633 INFO:tasks.workunit.client.0.smithi136.stdout:460 2024-09-12T23:55:30.635 INFO:tasks.workunit.client.0.smithi136.stdout:461 2024-09-12T23:55:30.637 INFO:tasks.workunit.client.0.smithi136.stdout:462 2024-09-12T23:55:30.639 INFO:tasks.workunit.client.0.smithi136.stdout:463 2024-09-12T23:55:30.641 INFO:tasks.workunit.client.0.smithi136.stdout:464 2024-09-12T23:55:30.643 INFO:tasks.workunit.client.0.smithi136.stdout:465 2024-09-12T23:55:30.645 INFO:tasks.workunit.client.0.smithi136.stdout:466 2024-09-12T23:55:30.647 INFO:tasks.workunit.client.0.smithi136.stdout:467 2024-09-12T23:55:30.649 INFO:tasks.workunit.client.0.smithi136.stdout:468 2024-09-12T23:55:30.652 INFO:tasks.workunit.client.0.smithi136.stdout:469 2024-09-12T23:55:30.654 INFO:tasks.workunit.client.0.smithi136.stdout:470 2024-09-12T23:55:30.656 INFO:tasks.workunit.client.0.smithi136.stdout:471 2024-09-12T23:55:30.659 INFO:tasks.workunit.client.0.smithi136.stdout:472 2024-09-12T23:55:30.661 INFO:tasks.workunit.client.0.smithi136.stdout:473 2024-09-12T23:55:30.663 INFO:tasks.workunit.client.0.smithi136.stdout:474 2024-09-12T23:55:30.665 INFO:tasks.workunit.client.0.smithi136.stdout:475 2024-09-12T23:55:30.668 INFO:tasks.workunit.client.0.smithi136.stdout:476 2024-09-12T23:55:30.669 INFO:tasks.workunit.client.0.smithi136.stdout:477 2024-09-12T23:55:30.670 INFO:tasks.workunit.client.0.smithi136.stdout:478 2024-09-12T23:55:30.672 INFO:tasks.workunit.client.0.smithi136.stdout:479 2024-09-12T23:55:30.673 INFO:tasks.workunit.client.0.smithi136.stdout:480 2024-09-12T23:55:30.674 INFO:tasks.workunit.client.0.smithi136.stdout:481 2024-09-12T23:55:30.676 INFO:tasks.workunit.client.0.smithi136.stdout:482 2024-09-12T23:55:30.677 INFO:tasks.workunit.client.0.smithi136.stdout:483 2024-09-12T23:55:30.679 INFO:tasks.workunit.client.0.smithi136.stdout:484 2024-09-12T23:55:30.680 INFO:tasks.workunit.client.0.smithi136.stdout:485 2024-09-12T23:55:30.681 INFO:tasks.workunit.client.0.smithi136.stdout:486 2024-09-12T23:55:30.683 INFO:tasks.workunit.client.0.smithi136.stdout:487 2024-09-12T23:55:30.684 INFO:tasks.workunit.client.0.smithi136.stdout:488 2024-09-12T23:55:30.685 INFO:tasks.workunit.client.0.smithi136.stdout:489 2024-09-12T23:55:30.687 INFO:tasks.workunit.client.0.smithi136.stdout:490 2024-09-12T23:55:30.688 INFO:tasks.workunit.client.0.smithi136.stdout:491 2024-09-12T23:55:30.689 INFO:tasks.workunit.client.0.smithi136.stdout:492 2024-09-12T23:55:30.691 INFO:tasks.workunit.client.0.smithi136.stdout:493 2024-09-12T23:55:30.692 INFO:tasks.workunit.client.0.smithi136.stdout:494 2024-09-12T23:55:30.693 INFO:tasks.workunit.client.0.smithi136.stdout:495 2024-09-12T23:55:30.695 INFO:tasks.workunit.client.0.smithi136.stdout:496 2024-09-12T23:55:30.696 INFO:tasks.workunit.client.0.smithi136.stdout:497 2024-09-12T23:55:30.698 INFO:tasks.workunit.client.0.smithi136.stdout:498 2024-09-12T23:55:30.699 INFO:tasks.workunit.client.0.smithi136.stdout:499 2024-09-12T23:55:30.700 INFO:tasks.workunit.client.0.smithi136.stdout:500 2024-09-12T23:55:30.701 INFO:tasks.workunit.client.0.smithi136.stdout:501 2024-09-12T23:55:30.703 INFO:tasks.workunit.client.0.smithi136.stdout:502 2024-09-12T23:55:30.704 INFO:tasks.workunit.client.0.smithi136.stdout:503 2024-09-12T23:55:30.705 INFO:tasks.workunit.client.0.smithi136.stdout:504 2024-09-12T23:55:30.707 INFO:tasks.workunit.client.0.smithi136.stdout:505 2024-09-12T23:55:30.708 INFO:tasks.workunit.client.0.smithi136.stdout:506 2024-09-12T23:55:30.709 INFO:tasks.workunit.client.0.smithi136.stdout:507 2024-09-12T23:55:30.711 INFO:tasks.workunit.client.0.smithi136.stdout:508 2024-09-12T23:55:30.712 INFO:tasks.workunit.client.0.smithi136.stdout:509 2024-09-12T23:55:30.713 INFO:tasks.workunit.client.0.smithi136.stdout:510 2024-09-12T23:55:30.714 INFO:tasks.workunit.client.0.smithi136.stdout:511 2024-09-12T23:55:30.716 INFO:tasks.workunit.client.0.smithi136.stdout:512 2024-09-12T23:55:30.717 INFO:tasks.workunit.client.0.smithi136.stdout:513 2024-09-12T23:55:30.718 INFO:tasks.workunit.client.0.smithi136.stdout:514 2024-09-12T23:55:30.720 INFO:tasks.workunit.client.0.smithi136.stdout:515 2024-09-12T23:55:30.721 INFO:tasks.workunit.client.0.smithi136.stdout:516 2024-09-12T23:55:30.722 INFO:tasks.workunit.client.0.smithi136.stdout:517 2024-09-12T23:55:30.724 INFO:tasks.workunit.client.0.smithi136.stdout:518 2024-09-12T23:55:30.725 INFO:tasks.workunit.client.0.smithi136.stdout:519 2024-09-12T23:55:30.726 INFO:tasks.workunit.client.0.smithi136.stdout:520 2024-09-12T23:55:30.727 INFO:tasks.workunit.client.0.smithi136.stdout:521 2024-09-12T23:55:30.729 INFO:tasks.workunit.client.0.smithi136.stdout:522 2024-09-12T23:55:30.730 INFO:tasks.workunit.client.0.smithi136.stdout:523 2024-09-12T23:55:30.731 INFO:tasks.workunit.client.0.smithi136.stdout:524 2024-09-12T23:55:30.732 INFO:tasks.workunit.client.0.smithi136.stdout:525 2024-09-12T23:55:30.735 INFO:tasks.workunit.client.0.smithi136.stdout:526 2024-09-12T23:55:30.738 INFO:tasks.workunit.client.0.smithi136.stdout:527 2024-09-12T23:55:30.739 INFO:tasks.workunit.client.0.smithi136.stdout:528 2024-09-12T23:55:30.740 INFO:tasks.workunit.client.0.smithi136.stdout:529 2024-09-12T23:55:30.741 INFO:tasks.workunit.client.0.smithi136.stdout:530 2024-09-12T23:55:30.743 INFO:tasks.workunit.client.0.smithi136.stdout:531 2024-09-12T23:55:30.744 INFO:tasks.workunit.client.0.smithi136.stdout:532 2024-09-12T23:55:30.745 INFO:tasks.workunit.client.0.smithi136.stdout:533 2024-09-12T23:55:30.746 INFO:tasks.workunit.client.0.smithi136.stdout:534 2024-09-12T23:55:30.747 INFO:tasks.workunit.client.0.smithi136.stdout:535 2024-09-12T23:55:30.748 INFO:tasks.workunit.client.0.smithi136.stdout:536 2024-09-12T23:55:30.749 INFO:tasks.workunit.client.0.smithi136.stdout:537 2024-09-12T23:55:30.750 INFO:tasks.workunit.client.0.smithi136.stdout:538 2024-09-12T23:55:30.751 INFO:tasks.workunit.client.0.smithi136.stdout:539 2024-09-12T23:55:30.752 INFO:tasks.workunit.client.0.smithi136.stdout:540 2024-09-12T23:55:30.753 INFO:tasks.workunit.client.0.smithi136.stdout:541 2024-09-12T23:55:30.754 INFO:tasks.workunit.client.0.smithi136.stdout:542 2024-09-12T23:55:30.755 INFO:tasks.workunit.client.0.smithi136.stdout:543 2024-09-12T23:55:30.756 INFO:tasks.workunit.client.0.smithi136.stdout:544 2024-09-12T23:55:30.757 INFO:tasks.workunit.client.0.smithi136.stdout:545 2024-09-12T23:55:30.758 INFO:tasks.workunit.client.0.smithi136.stdout:546 2024-09-12T23:55:30.759 INFO:tasks.workunit.client.0.smithi136.stdout:547 2024-09-12T23:55:30.760 INFO:tasks.workunit.client.0.smithi136.stdout:548 2024-09-12T23:55:30.761 INFO:tasks.workunit.client.0.smithi136.stdout:549 2024-09-12T23:55:30.762 INFO:tasks.workunit.client.0.smithi136.stdout:550 2024-09-12T23:55:30.763 INFO:tasks.workunit.client.0.smithi136.stdout:551 2024-09-12T23:55:30.764 INFO:tasks.workunit.client.0.smithi136.stdout:552 2024-09-12T23:55:30.765 INFO:tasks.workunit.client.0.smithi136.stdout:553 2024-09-12T23:55:30.766 INFO:tasks.workunit.client.0.smithi136.stdout:554 2024-09-12T23:55:30.767 INFO:tasks.workunit.client.0.smithi136.stdout:555 2024-09-12T23:55:30.768 INFO:tasks.workunit.client.0.smithi136.stdout:556 2024-09-12T23:55:30.769 INFO:tasks.workunit.client.0.smithi136.stdout:557 2024-09-12T23:55:30.770 INFO:tasks.workunit.client.0.smithi136.stdout:558 2024-09-12T23:55:30.771 INFO:tasks.workunit.client.0.smithi136.stdout:559 2024-09-12T23:55:30.773 INFO:tasks.workunit.client.0.smithi136.stdout:560 2024-09-12T23:55:30.773 INFO:tasks.workunit.client.0.smithi136.stdout:561 2024-09-12T23:55:30.775 INFO:tasks.workunit.client.0.smithi136.stdout:562 2024-09-12T23:55:30.776 INFO:tasks.workunit.client.0.smithi136.stdout:563 2024-09-12T23:55:30.777 INFO:tasks.workunit.client.0.smithi136.stdout:564 2024-09-12T23:55:30.778 INFO:tasks.workunit.client.0.smithi136.stdout:565 2024-09-12T23:55:30.779 INFO:tasks.workunit.client.0.smithi136.stdout:566 2024-09-12T23:55:30.780 INFO:tasks.workunit.client.0.smithi136.stdout:567 2024-09-12T23:55:30.781 INFO:tasks.workunit.client.0.smithi136.stdout:568 2024-09-12T23:55:30.782 INFO:tasks.workunit.client.0.smithi136.stdout:569 2024-09-12T23:55:30.783 INFO:tasks.workunit.client.0.smithi136.stdout:570 2024-09-12T23:55:30.784 INFO:tasks.workunit.client.0.smithi136.stdout:571 2024-09-12T23:55:30.785 INFO:tasks.workunit.client.0.smithi136.stdout:572 2024-09-12T23:55:30.786 INFO:tasks.workunit.client.0.smithi136.stdout:573 2024-09-12T23:55:30.787 INFO:tasks.workunit.client.0.smithi136.stdout:574 2024-09-12T23:55:30.788 INFO:tasks.workunit.client.0.smithi136.stdout:575 2024-09-12T23:55:30.789 INFO:tasks.workunit.client.0.smithi136.stdout:576 2024-09-12T23:55:30.791 INFO:tasks.workunit.client.0.smithi136.stdout:577 2024-09-12T23:55:30.792 INFO:tasks.workunit.client.0.smithi136.stdout:578 2024-09-12T23:55:30.793 INFO:tasks.workunit.client.0.smithi136.stdout:579 2024-09-12T23:55:30.794 INFO:tasks.workunit.client.0.smithi136.stdout:580 2024-09-12T23:55:30.795 INFO:tasks.workunit.client.0.smithi136.stdout:581 2024-09-12T23:55:30.796 INFO:tasks.workunit.client.0.smithi136.stdout:582 2024-09-12T23:55:30.797 INFO:tasks.workunit.client.0.smithi136.stdout:583 2024-09-12T23:55:30.798 INFO:tasks.workunit.client.0.smithi136.stdout:584 2024-09-12T23:55:30.799 INFO:tasks.workunit.client.0.smithi136.stdout:585 2024-09-12T23:55:30.800 INFO:tasks.workunit.client.0.smithi136.stdout:586 2024-09-12T23:55:30.801 INFO:tasks.workunit.client.0.smithi136.stdout:587 2024-09-12T23:55:30.802 INFO:tasks.workunit.client.0.smithi136.stdout:588 2024-09-12T23:55:30.803 INFO:tasks.workunit.client.0.smithi136.stdout:589 2024-09-12T23:55:30.804 INFO:tasks.workunit.client.0.smithi136.stdout:590 2024-09-12T23:55:30.805 INFO:tasks.workunit.client.0.smithi136.stdout:591 2024-09-12T23:55:30.806 INFO:tasks.workunit.client.0.smithi136.stdout:592 2024-09-12T23:55:30.807 INFO:tasks.workunit.client.0.smithi136.stdout:593 2024-09-12T23:55:30.808 INFO:tasks.workunit.client.0.smithi136.stdout:594 2024-09-12T23:55:30.809 INFO:tasks.workunit.client.0.smithi136.stdout:595 2024-09-12T23:55:30.810 INFO:tasks.workunit.client.0.smithi136.stdout:596 2024-09-12T23:55:30.812 INFO:tasks.workunit.client.0.smithi136.stdout:597 2024-09-12T23:55:30.813 INFO:tasks.workunit.client.0.smithi136.stdout:598 2024-09-12T23:55:30.814 INFO:tasks.workunit.client.0.smithi136.stdout:599 2024-09-12T23:55:30.816 INFO:tasks.workunit.client.0.smithi136.stdout:600 2024-09-12T23:55:30.817 INFO:tasks.workunit.client.0.smithi136.stdout:601 2024-09-12T23:55:30.818 INFO:tasks.workunit.client.0.smithi136.stdout:602 2024-09-12T23:55:30.820 INFO:tasks.workunit.client.0.smithi136.stdout:603 2024-09-12T23:55:30.821 INFO:tasks.workunit.client.0.smithi136.stdout:604 2024-09-12T23:55:30.823 INFO:tasks.workunit.client.0.smithi136.stdout:605 2024-09-12T23:55:30.824 INFO:tasks.workunit.client.0.smithi136.stdout:606 2024-09-12T23:55:30.825 INFO:tasks.workunit.client.0.smithi136.stdout:607 2024-09-12T23:55:30.827 INFO:tasks.workunit.client.0.smithi136.stdout:608 2024-09-12T23:55:30.828 INFO:tasks.workunit.client.0.smithi136.stdout:609 2024-09-12T23:55:30.830 INFO:tasks.workunit.client.0.smithi136.stdout:610 2024-09-12T23:55:30.831 INFO:tasks.workunit.client.0.smithi136.stdout:611 2024-09-12T23:55:30.833 INFO:tasks.workunit.client.0.smithi136.stdout:612 2024-09-12T23:55:30.834 INFO:tasks.workunit.client.0.smithi136.stdout:613 2024-09-12T23:55:30.836 INFO:tasks.workunit.client.0.smithi136.stdout:614 2024-09-12T23:55:30.838 INFO:tasks.workunit.client.0.smithi136.stdout:615 2024-09-12T23:55:30.840 INFO:tasks.workunit.client.0.smithi136.stdout:616 2024-09-12T23:55:30.842 INFO:tasks.workunit.client.0.smithi136.stdout:617 2024-09-12T23:55:30.844 INFO:tasks.workunit.client.0.smithi136.stdout:618 2024-09-12T23:55:30.845 INFO:tasks.workunit.client.0.smithi136.stdout:619 2024-09-12T23:55:30.847 INFO:tasks.workunit.client.0.smithi136.stdout:620 2024-09-12T23:55:30.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:30 smithi181 bash[67587]: cluster 2024-09-12T23:55:28.982012+0000 mgr.y (mgr.24661) 551 : cluster [DBG] pgmap v1072: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:30.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:30 smithi181 bash[67587]: cluster 2024-09-12T23:55:29.506709+0000 mon.a (mon.0) 1681 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:55:30.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:30 smithi181 bash[67587]: audit 2024-09-12T23:55:29.515707+0000 mon.a (mon.0) 1682 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109763-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:55:30.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:30 smithi181 bash[67587]: cluster 2024-09-12T23:55:29.516165+0000 mon.a (mon.0) 1683 : cluster [DBG] osdmap e961: 8 total, 8 up, 8 in 2024-09-12T23:55:30.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:30 smithi181 bash[67587]: audit 2024-09-12T23:55:29.543410+0000 mon.c (mon.2) 39 : audit [DBG] from='client.? 172.21.15.136:0/1028442975' entity='client.admin' cmd=[{"prefix": "osd dump","format":"json"}]: dispatch 2024-09-12T23:55:30.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:30 smithi181 bash[67587]: audit 2024-09-12T23:55:30.325867+0000 mon.a (mon.0) 1684 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:55:30.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:30 smithi181 bash[67587]: cluster 2024-09-12T23:55:30.517174+0000 mon.a (mon.0) 1685 : cluster [DBG] osdmap e962: 8 total, 8 up, 8 in 2024-09-12T23:55:30.849 INFO:tasks.workunit.client.0.smithi136.stdout:621 2024-09-12T23:55:30.851 INFO:tasks.workunit.client.0.smithi136.stdout:622 2024-09-12T23:55:30.853 INFO:tasks.workunit.client.0.smithi136.stdout:623 2024-09-12T23:55:30.855 INFO:tasks.workunit.client.0.smithi136.stdout:624 2024-09-12T23:55:30.858 INFO:tasks.workunit.client.0.smithi136.stdout:625 2024-09-12T23:55:30.860 INFO:tasks.workunit.client.0.smithi136.stdout:626 2024-09-12T23:55:30.862 INFO:tasks.workunit.client.0.smithi136.stdout:627 2024-09-12T23:55:30.864 INFO:tasks.workunit.client.0.smithi136.stdout:628 2024-09-12T23:55:30.866 INFO:tasks.workunit.client.0.smithi136.stdout:629 2024-09-12T23:55:30.868 INFO:tasks.workunit.client.0.smithi136.stdout:630 2024-09-12T23:55:30.870 INFO:tasks.workunit.client.0.smithi136.stdout:631 2024-09-12T23:55:30.872 INFO:tasks.workunit.client.0.smithi136.stdout:632 2024-09-12T23:55:30.875 INFO:tasks.workunit.client.0.smithi136.stdout:633 2024-09-12T23:55:30.877 INFO:tasks.workunit.client.0.smithi136.stdout:634 2024-09-12T23:55:30.879 INFO:tasks.workunit.client.0.smithi136.stdout:635 2024-09-12T23:55:30.881 INFO:tasks.workunit.client.0.smithi136.stdout:636 2024-09-12T23:55:30.883 INFO:tasks.workunit.client.0.smithi136.stdout:637 2024-09-12T23:55:30.886 INFO:tasks.workunit.client.0.smithi136.stdout:638 2024-09-12T23:55:30.888 INFO:tasks.workunit.client.0.smithi136.stdout:639 2024-09-12T23:55:30.890 INFO:tasks.workunit.client.0.smithi136.stdout:640 2024-09-12T23:55:30.892 INFO:tasks.workunit.client.0.smithi136.stdout:641 2024-09-12T23:55:30.895 INFO:tasks.workunit.client.0.smithi136.stdout:642 2024-09-12T23:55:30.897 INFO:tasks.workunit.client.0.smithi136.stdout:643 2024-09-12T23:55:30.900 INFO:tasks.workunit.client.0.smithi136.stdout:644 2024-09-12T23:55:30.903 INFO:tasks.workunit.client.0.smithi136.stdout:645 2024-09-12T23:55:30.906 INFO:tasks.workunit.client.0.smithi136.stdout:646 2024-09-12T23:55:30.909 INFO:tasks.workunit.client.0.smithi136.stdout:647 2024-09-12T23:55:30.911 INFO:tasks.workunit.client.0.smithi136.stdout:648 2024-09-12T23:55:30.914 INFO:tasks.workunit.client.0.smithi136.stdout:649 2024-09-12T23:55:30.916 INFO:tasks.workunit.client.0.smithi136.stdout:650 2024-09-12T23:55:30.919 INFO:tasks.workunit.client.0.smithi136.stdout:651 2024-09-12T23:55:30.921 INFO:tasks.workunit.client.0.smithi136.stdout:652 2024-09-12T23:55:30.924 INFO:tasks.workunit.client.0.smithi136.stdout:653 2024-09-12T23:55:30.927 INFO:tasks.workunit.client.0.smithi136.stdout:654 2024-09-12T23:55:30.929 INFO:tasks.workunit.client.0.smithi136.stdout:655 2024-09-12T23:55:30.932 INFO:tasks.workunit.client.0.smithi136.stdout:656 2024-09-12T23:55:30.937 INFO:tasks.workunit.client.0.smithi136.stdout:657 2024-09-12T23:55:30.937 INFO:tasks.workunit.client.0.smithi136.stdout:658 2024-09-12T23:55:30.940 INFO:tasks.workunit.client.0.smithi136.stdout:659 2024-09-12T23:55:30.942 INFO:tasks.workunit.client.0.smithi136.stdout:660 2024-09-12T23:55:30.945 INFO:tasks.workunit.client.0.smithi136.stdout:661 2024-09-12T23:55:30.948 INFO:tasks.workunit.client.0.smithi136.stdout:662 2024-09-12T23:55:30.950 INFO:tasks.workunit.client.0.smithi136.stdout:663 2024-09-12T23:55:30.953 INFO:tasks.workunit.client.0.smithi136.stdout:664 2024-09-12T23:55:30.955 INFO:tasks.workunit.client.0.smithi136.stdout:665 2024-09-12T23:55:30.958 INFO:tasks.workunit.client.0.smithi136.stdout:666 2024-09-12T23:55:30.960 INFO:tasks.workunit.client.0.smithi136.stdout:667 2024-09-12T23:55:30.963 INFO:tasks.workunit.client.0.smithi136.stdout:668 2024-09-12T23:55:30.966 INFO:tasks.workunit.client.0.smithi136.stdout:669 2024-09-12T23:55:30.968 INFO:tasks.workunit.client.0.smithi136.stdout:670 2024-09-12T23:55:30.971 INFO:tasks.workunit.client.0.smithi136.stdout:671 2024-09-12T23:55:30.974 INFO:tasks.workunit.client.0.smithi136.stdout:672 2024-09-12T23:55:30.977 INFO:tasks.workunit.client.0.smithi136.stdout:673 2024-09-12T23:55:30.979 INFO:tasks.workunit.client.0.smithi136.stdout:674 2024-09-12T23:55:30.983 INFO:tasks.workunit.client.0.smithi136.stdout:675 2024-09-12T23:55:30.985 INFO:tasks.workunit.client.0.smithi136.stdout:676 2024-09-12T23:55:30.989 INFO:tasks.workunit.client.0.smithi136.stdout:677 2024-09-12T23:55:30.991 INFO:tasks.workunit.client.0.smithi136.stdout:678 2024-09-12T23:55:30.994 INFO:tasks.workunit.client.0.smithi136.stdout:679 2024-09-12T23:55:30.997 INFO:tasks.workunit.client.0.smithi136.stdout:680 2024-09-12T23:55:31.000 INFO:tasks.workunit.client.0.smithi136.stdout:681 2024-09-12T23:55:31.003 INFO:tasks.workunit.client.0.smithi136.stdout:682 2024-09-12T23:55:31.005 INFO:tasks.workunit.client.0.smithi136.stdout:683 2024-09-12T23:55:31.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:30 smithi136 bash[76371]: cluster 2024-09-12T23:55:28.982012+0000 mgr.y (mgr.24661) 551 : cluster [DBG] pgmap v1072: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:31.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:30 smithi136 bash[76371]: cluster 2024-09-12T23:55:29.506709+0000 mon.a (mon.0) 1681 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:55:31.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:30 smithi136 bash[76371]: audit 2024-09-12T23:55:29.515707+0000 mon.a (mon.0) 1682 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109763-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:55:31.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:30 smithi136 bash[76371]: cluster 2024-09-12T23:55:29.516165+0000 mon.a (mon.0) 1683 : cluster [DBG] osdmap e961: 8 total, 8 up, 8 in 2024-09-12T23:55:31.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:30 smithi136 bash[76371]: audit 2024-09-12T23:55:29.543410+0000 mon.c (mon.2) 39 : audit [DBG] from='client.? 172.21.15.136:0/1028442975' entity='client.admin' cmd=[{"prefix": "osd dump","format":"json"}]: dispatch 2024-09-12T23:55:31.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:30 smithi136 bash[76371]: audit 2024-09-12T23:55:30.325867+0000 mon.a (mon.0) 1684 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:55:31.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:30 smithi136 bash[76371]: cluster 2024-09-12T23:55:30.517174+0000 mon.a (mon.0) 1685 : cluster [DBG] osdmap e962: 8 total, 8 up, 8 in 2024-09-12T23:55:31.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:30 smithi136 bash[75164]: cluster 2024-09-12T23:55:28.982012+0000 mgr.y (mgr.24661) 551 : cluster [DBG] pgmap v1072: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:31.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:30 smithi136 bash[75164]: cluster 2024-09-12T23:55:29.506709+0000 mon.a (mon.0) 1681 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:55:31.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:30 smithi136 bash[75164]: audit 2024-09-12T23:55:29.515707+0000 mon.a (mon.0) 1682 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109763-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:55:31.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:30 smithi136 bash[75164]: cluster 2024-09-12T23:55:29.516165+0000 mon.a (mon.0) 1683 : cluster [DBG] osdmap e961: 8 total, 8 up, 8 in 2024-09-12T23:55:31.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:30 smithi136 bash[75164]: audit 2024-09-12T23:55:29.543410+0000 mon.c (mon.2) 39 : audit [DBG] from='client.? 172.21.15.136:0/1028442975' entity='client.admin' cmd=[{"prefix": "osd dump","format":"json"}]: dispatch 2024-09-12T23:55:31.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:30 smithi136 bash[75164]: audit 2024-09-12T23:55:30.325867+0000 mon.a (mon.0) 1684 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:55:31.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:30 smithi136 bash[75164]: cluster 2024-09-12T23:55:30.517174+0000 mon.a (mon.0) 1685 : cluster [DBG] osdmap e962: 8 total, 8 up, 8 in 2024-09-12T23:55:31.008 INFO:tasks.workunit.client.0.smithi136.stdout:684 2024-09-12T23:55:31.008 INFO:tasks.workunit.client.0.smithi136.stdout:685 2024-09-12T23:55:31.010 INFO:tasks.workunit.client.0.smithi136.stdout:686 2024-09-12T23:55:31.012 INFO:tasks.workunit.client.0.smithi136.stdout:687 2024-09-12T23:55:31.014 INFO:tasks.workunit.client.0.smithi136.stdout:688 2024-09-12T23:55:31.016 INFO:tasks.workunit.client.0.smithi136.stdout:689 2024-09-12T23:55:31.017 INFO:tasks.workunit.client.0.smithi136.stdout:690 2024-09-12T23:55:31.019 INFO:tasks.workunit.client.0.smithi136.stdout:691 2024-09-12T23:55:31.021 INFO:tasks.workunit.client.0.smithi136.stdout:692 2024-09-12T23:55:31.023 INFO:tasks.workunit.client.0.smithi136.stdout:693 2024-09-12T23:55:31.025 INFO:tasks.workunit.client.0.smithi136.stdout:694 2024-09-12T23:55:31.027 INFO:tasks.workunit.client.0.smithi136.stdout:695 2024-09-12T23:55:31.029 INFO:tasks.workunit.client.0.smithi136.stdout:696 2024-09-12T23:55:31.031 INFO:tasks.workunit.client.0.smithi136.stdout:697 2024-09-12T23:55:31.033 INFO:tasks.workunit.client.0.smithi136.stdout:698 2024-09-12T23:55:31.035 INFO:tasks.workunit.client.0.smithi136.stdout:699 2024-09-12T23:55:31.037 INFO:tasks.workunit.client.0.smithi136.stdout:700 2024-09-12T23:55:31.039 INFO:tasks.workunit.client.0.smithi136.stdout:701 2024-09-12T23:55:31.041 INFO:tasks.workunit.client.0.smithi136.stdout:702 2024-09-12T23:55:31.043 INFO:tasks.workunit.client.0.smithi136.stdout:703 2024-09-12T23:55:31.045 INFO:tasks.workunit.client.0.smithi136.stdout:704 2024-09-12T23:55:31.047 INFO:tasks.workunit.client.0.smithi136.stdout:705 2024-09-12T23:55:31.049 INFO:tasks.workunit.client.0.smithi136.stdout:706 2024-09-12T23:55:31.051 INFO:tasks.workunit.client.0.smithi136.stdout:707 2024-09-12T23:55:31.053 INFO:tasks.workunit.client.0.smithi136.stdout:708 2024-09-12T23:55:31.056 INFO:tasks.workunit.client.0.smithi136.stdout:709 2024-09-12T23:55:31.059 INFO:tasks.workunit.client.0.smithi136.stdout:710 2024-09-12T23:55:31.061 INFO:tasks.workunit.client.0.smithi136.stdout:711 2024-09-12T23:55:31.063 INFO:tasks.workunit.client.0.smithi136.stdout:712 2024-09-12T23:55:31.064 INFO:tasks.workunit.client.0.smithi136.stdout:713 2024-09-12T23:55:31.066 INFO:tasks.workunit.client.0.smithi136.stdout:714 2024-09-12T23:55:31.068 INFO:tasks.workunit.client.0.smithi136.stdout:715 2024-09-12T23:55:31.069 INFO:tasks.workunit.client.0.smithi136.stdout:716 2024-09-12T23:55:31.071 INFO:tasks.workunit.client.0.smithi136.stdout:717 2024-09-12T23:55:31.073 INFO:tasks.workunit.client.0.smithi136.stdout:718 2024-09-12T23:55:31.075 INFO:tasks.workunit.client.0.smithi136.stdout:719 2024-09-12T23:55:31.077 INFO:tasks.workunit.client.0.smithi136.stdout:720 2024-09-12T23:55:31.078 INFO:tasks.workunit.client.0.smithi136.stdout:721 2024-09-12T23:55:31.080 INFO:tasks.workunit.client.0.smithi136.stdout:722 2024-09-12T23:55:31.082 INFO:tasks.workunit.client.0.smithi136.stdout:723 2024-09-12T23:55:31.084 INFO:tasks.workunit.client.0.smithi136.stdout:724 2024-09-12T23:55:31.085 INFO:tasks.workunit.client.0.smithi136.stdout:725 2024-09-12T23:55:31.087 INFO:tasks.workunit.client.0.smithi136.stdout:726 2024-09-12T23:55:31.089 INFO:tasks.workunit.client.0.smithi136.stdout:727 2024-09-12T23:55:31.092 INFO:tasks.workunit.client.0.smithi136.stdout:728 2024-09-12T23:55:31.094 INFO:tasks.workunit.client.0.smithi136.stdout:729 2024-09-12T23:55:31.096 INFO:tasks.workunit.client.0.smithi136.stdout:730 2024-09-12T23:55:31.098 INFO:tasks.workunit.client.0.smithi136.stdout:731 2024-09-12T23:55:31.100 INFO:tasks.workunit.client.0.smithi136.stdout:732 2024-09-12T23:55:31.102 INFO:tasks.workunit.client.0.smithi136.stdout:733 2024-09-12T23:55:31.104 INFO:tasks.workunit.client.0.smithi136.stdout:734 2024-09-12T23:55:31.107 INFO:tasks.workunit.client.0.smithi136.stdout:735 2024-09-12T23:55:31.108 INFO:tasks.workunit.client.0.smithi136.stdout:736 2024-09-12T23:55:31.111 INFO:tasks.workunit.client.0.smithi136.stdout:737 2024-09-12T23:55:31.113 INFO:tasks.workunit.client.0.smithi136.stdout:738 2024-09-12T23:55:31.115 INFO:tasks.workunit.client.0.smithi136.stdout:739 2024-09-12T23:55:31.116 INFO:tasks.workunit.client.0.smithi136.stdout:740 2024-09-12T23:55:31.119 INFO:tasks.workunit.client.0.smithi136.stdout:741 2024-09-12T23:55:31.121 INFO:tasks.workunit.client.0.smithi136.stdout:742 2024-09-12T23:55:31.123 INFO:tasks.workunit.client.0.smithi136.stdout:743 2024-09-12T23:55:31.126 INFO:tasks.workunit.client.0.smithi136.stdout:744 2024-09-12T23:55:31.128 INFO:tasks.workunit.client.0.smithi136.stdout:745 2024-09-12T23:55:31.131 INFO:tasks.workunit.client.0.smithi136.stdout:746 2024-09-12T23:55:31.134 INFO:tasks.workunit.client.0.smithi136.stdout:747 2024-09-12T23:55:31.136 INFO:tasks.workunit.client.0.smithi136.stdout:748 2024-09-12T23:55:31.139 INFO:tasks.workunit.client.0.smithi136.stdout:749 2024-09-12T23:55:31.144 INFO:tasks.workunit.client.0.smithi136.stdout:750 2024-09-12T23:55:31.144 INFO:tasks.workunit.client.0.smithi136.stdout:751 2024-09-12T23:55:31.147 INFO:tasks.workunit.client.0.smithi136.stdout:752 2024-09-12T23:55:31.149 INFO:tasks.workunit.client.0.smithi136.stdout:753 2024-09-12T23:55:31.151 INFO:tasks.workunit.client.0.smithi136.stdout:754 2024-09-12T23:55:31.153 INFO:tasks.workunit.client.0.smithi136.stdout:755 2024-09-12T23:55:31.155 INFO:tasks.workunit.client.0.smithi136.stdout:756 2024-09-12T23:55:31.158 INFO:tasks.workunit.client.0.smithi136.stdout:757 2024-09-12T23:55:31.160 INFO:tasks.workunit.client.0.smithi136.stdout:758 2024-09-12T23:55:31.163 INFO:tasks.workunit.client.0.smithi136.stdout:759 2024-09-12T23:55:31.165 INFO:tasks.workunit.client.0.smithi136.stdout:760 2024-09-12T23:55:31.167 INFO:tasks.workunit.client.0.smithi136.stdout:761 2024-09-12T23:55:31.169 INFO:tasks.workunit.client.0.smithi136.stdout:762 2024-09-12T23:55:31.171 INFO:tasks.workunit.client.0.smithi136.stdout:763 2024-09-12T23:55:31.173 INFO:tasks.workunit.client.0.smithi136.stdout:764 2024-09-12T23:55:31.175 INFO:tasks.workunit.client.0.smithi136.stdout:765 2024-09-12T23:55:31.177 INFO:tasks.workunit.client.0.smithi136.stdout:766 2024-09-12T23:55:31.179 INFO:tasks.workunit.client.0.smithi136.stdout:767 2024-09-12T23:55:31.182 INFO:tasks.workunit.client.0.smithi136.stdout:768 2024-09-12T23:55:31.184 INFO:tasks.workunit.client.0.smithi136.stdout:769 2024-09-12T23:55:31.186 INFO:tasks.workunit.client.0.smithi136.stdout:770 2024-09-12T23:55:31.189 INFO:tasks.workunit.client.0.smithi136.stdout:771 2024-09-12T23:55:31.192 INFO:tasks.workunit.client.0.smithi136.stdout:772 2024-09-12T23:55:31.194 INFO:tasks.workunit.client.0.smithi136.stdout:773 2024-09-12T23:55:31.197 INFO:tasks.workunit.client.0.smithi136.stdout:774 2024-09-12T23:55:31.199 INFO:tasks.workunit.client.0.smithi136.stdout:775 2024-09-12T23:55:31.201 INFO:tasks.workunit.client.0.smithi136.stdout:776 2024-09-12T23:55:31.203 INFO:tasks.workunit.client.0.smithi136.stdout:777 2024-09-12T23:55:31.205 INFO:tasks.workunit.client.0.smithi136.stdout:778 2024-09-12T23:55:31.206 INFO:tasks.workunit.client.0.smithi136.stdout:779 2024-09-12T23:55:31.208 INFO:tasks.workunit.client.0.smithi136.stdout:780 2024-09-12T23:55:31.210 INFO:tasks.workunit.client.0.smithi136.stdout:781 2024-09-12T23:55:31.212 INFO:tasks.workunit.client.0.smithi136.stdout:782 2024-09-12T23:55:31.214 INFO:tasks.workunit.client.0.smithi136.stdout:783 2024-09-12T23:55:31.216 INFO:tasks.workunit.client.0.smithi136.stdout:784 2024-09-12T23:55:31.218 INFO:tasks.workunit.client.0.smithi136.stdout:785 2024-09-12T23:55:31.220 INFO:tasks.workunit.client.0.smithi136.stdout:786 2024-09-12T23:55:31.222 INFO:tasks.workunit.client.0.smithi136.stdout:787 2024-09-12T23:55:31.224 INFO:tasks.workunit.client.0.smithi136.stdout:788 2024-09-12T23:55:31.226 INFO:tasks.workunit.client.0.smithi136.stdout:789 2024-09-12T23:55:31.228 INFO:tasks.workunit.client.0.smithi136.stdout:790 2024-09-12T23:55:31.229 INFO:tasks.workunit.client.0.smithi136.stdout:791 2024-09-12T23:55:31.231 INFO:tasks.workunit.client.0.smithi136.stdout:792 2024-09-12T23:55:31.232 INFO:tasks.workunit.client.0.smithi136.stdout:793 2024-09-12T23:55:31.234 INFO:tasks.workunit.client.0.smithi136.stdout:794 2024-09-12T23:55:31.235 INFO:tasks.workunit.client.0.smithi136.stdout:795 2024-09-12T23:55:31.237 INFO:tasks.workunit.client.0.smithi136.stdout:796 2024-09-12T23:55:31.238 INFO:tasks.workunit.client.0.smithi136.stdout:797 2024-09-12T23:55:31.240 INFO:tasks.workunit.client.0.smithi136.stdout:798 2024-09-12T23:55:31.241 INFO:tasks.workunit.client.0.smithi136.stdout:799 2024-09-12T23:55:31.243 INFO:tasks.workunit.client.0.smithi136.stdout:800 2024-09-12T23:55:31.244 INFO:tasks.workunit.client.0.smithi136.stdout:801 2024-09-12T23:55:31.246 INFO:tasks.workunit.client.0.smithi136.stdout:802 2024-09-12T23:55:31.247 INFO:tasks.workunit.client.0.smithi136.stdout:803 2024-09-12T23:55:31.249 INFO:tasks.workunit.client.0.smithi136.stdout:804 2024-09-12T23:55:31.250 INFO:tasks.workunit.client.0.smithi136.stdout:805 2024-09-12T23:55:31.251 INFO:tasks.workunit.client.0.smithi136.stdout:806 2024-09-12T23:55:31.253 INFO:tasks.workunit.client.0.smithi136.stdout:807 2024-09-12T23:55:31.254 INFO:tasks.workunit.client.0.smithi136.stdout:808 2024-09-12T23:55:31.256 INFO:tasks.workunit.client.0.smithi136.stdout:809 2024-09-12T23:55:31.257 INFO:tasks.workunit.client.0.smithi136.stdout:810 2024-09-12T23:55:31.259 INFO:tasks.workunit.client.0.smithi136.stdout:811 2024-09-12T23:55:31.260 INFO:tasks.workunit.client.0.smithi136.stdout:812 2024-09-12T23:55:31.262 INFO:tasks.workunit.client.0.smithi136.stdout:813 2024-09-12T23:55:31.264 INFO:tasks.workunit.client.0.smithi136.stdout:814 2024-09-12T23:55:31.266 INFO:tasks.workunit.client.0.smithi136.stdout:815 2024-09-12T23:55:31.267 INFO:tasks.workunit.client.0.smithi136.stdout:816 2024-09-12T23:55:31.269 INFO:tasks.workunit.client.0.smithi136.stdout:817 2024-09-12T23:55:31.271 INFO:tasks.workunit.client.0.smithi136.stdout:818 2024-09-12T23:55:31.273 INFO:tasks.workunit.client.0.smithi136.stdout:819 2024-09-12T23:55:31.275 INFO:tasks.workunit.client.0.smithi136.stdout:820 2024-09-12T23:55:31.277 INFO:tasks.workunit.client.0.smithi136.stdout:821 2024-09-12T23:55:31.279 INFO:tasks.workunit.client.0.smithi136.stdout:822 2024-09-12T23:55:31.281 INFO:tasks.workunit.client.0.smithi136.stdout:823 2024-09-12T23:55:31.283 INFO:tasks.workunit.client.0.smithi136.stdout:824 2024-09-12T23:55:31.285 INFO:tasks.workunit.client.0.smithi136.stdout:825 2024-09-12T23:55:31.286 INFO:tasks.workunit.client.0.smithi136.stdout:826 2024-09-12T23:55:31.288 INFO:tasks.workunit.client.0.smithi136.stdout:827 2024-09-12T23:55:31.290 INFO:tasks.workunit.client.0.smithi136.stdout:828 2024-09-12T23:55:31.291 INFO:tasks.workunit.client.0.smithi136.stdout:829 2024-09-12T23:55:31.293 INFO:tasks.workunit.client.0.smithi136.stdout:830 2024-09-12T23:55:31.295 INFO:tasks.workunit.client.0.smithi136.stdout:831 2024-09-12T23:55:31.298 INFO:tasks.workunit.client.0.smithi136.stdout:832 2024-09-12T23:55:31.300 INFO:tasks.workunit.client.0.smithi136.stdout:833 2024-09-12T23:55:31.303 INFO:tasks.workunit.client.0.smithi136.stdout:834 2024-09-12T23:55:31.305 INFO:tasks.workunit.client.0.smithi136.stdout:835 2024-09-12T23:55:31.308 INFO:tasks.workunit.client.0.smithi136.stdout:836 2024-09-12T23:55:31.310 INFO:tasks.workunit.client.0.smithi136.stdout:837 2024-09-12T23:55:31.313 INFO:tasks.workunit.client.0.smithi136.stdout:838 2024-09-12T23:55:31.316 INFO:tasks.workunit.client.0.smithi136.stdout:839 2024-09-12T23:55:31.319 INFO:tasks.workunit.client.0.smithi136.stdout:840 2024-09-12T23:55:31.322 INFO:tasks.workunit.client.0.smithi136.stdout:841 2024-09-12T23:55:31.324 INFO:tasks.workunit.client.0.smithi136.stdout:842 2024-09-12T23:55:31.327 INFO:tasks.workunit.client.0.smithi136.stdout:843 2024-09-12T23:55:31.330 INFO:tasks.workunit.client.0.smithi136.stdout:844 2024-09-12T23:55:31.333 INFO:tasks.workunit.client.0.smithi136.stdout:845 2024-09-12T23:55:31.336 INFO:tasks.workunit.client.0.smithi136.stdout:846 2024-09-12T23:55:31.339 INFO:tasks.workunit.client.0.smithi136.stdout:847 2024-09-12T23:55:31.342 INFO:tasks.workunit.client.0.smithi136.stdout:848 2024-09-12T23:55:31.344 INFO:tasks.workunit.client.0.smithi136.stdout:849 2024-09-12T23:55:31.348 INFO:tasks.workunit.client.0.smithi136.stdout:850 2024-09-12T23:55:31.350 INFO:tasks.workunit.client.0.smithi136.stdout:851 2024-09-12T23:55:31.353 INFO:tasks.workunit.client.0.smithi136.stdout:852 2024-09-12T23:55:31.356 INFO:tasks.workunit.client.0.smithi136.stdout:853 2024-09-12T23:55:31.359 INFO:tasks.workunit.client.0.smithi136.stdout:854 2024-09-12T23:55:31.362 INFO:tasks.workunit.client.0.smithi136.stdout:855 2024-09-12T23:55:31.365 INFO:tasks.workunit.client.0.smithi136.stdout:856 2024-09-12T23:55:31.368 INFO:tasks.workunit.client.0.smithi136.stdout:857 2024-09-12T23:55:31.371 INFO:tasks.workunit.client.0.smithi136.stdout:858 2024-09-12T23:55:31.375 INFO:tasks.workunit.client.0.smithi136.stdout:859 2024-09-12T23:55:31.377 INFO:tasks.workunit.client.0.smithi136.stdout:860 2024-09-12T23:55:31.381 INFO:tasks.workunit.client.0.smithi136.stdout:861 2024-09-12T23:55:31.384 INFO:tasks.workunit.client.0.smithi136.stdout:862 2024-09-12T23:55:31.387 INFO:tasks.workunit.client.0.smithi136.stdout:863 2024-09-12T23:55:31.389 INFO:tasks.workunit.client.0.smithi136.stdout:864 2024-09-12T23:55:31.392 INFO:tasks.workunit.client.0.smithi136.stdout:865 2024-09-12T23:55:31.395 INFO:tasks.workunit.client.0.smithi136.stdout:866 2024-09-12T23:55:31.398 INFO:tasks.workunit.client.0.smithi136.stdout:867 2024-09-12T23:55:31.401 INFO:tasks.workunit.client.0.smithi136.stdout:868 2024-09-12T23:55:31.403 INFO:tasks.workunit.client.0.smithi136.stdout:869 2024-09-12T23:55:31.406 INFO:tasks.workunit.client.0.smithi136.stdout:870 2024-09-12T23:55:31.409 INFO:tasks.workunit.client.0.smithi136.stdout:871 2024-09-12T23:55:31.411 INFO:tasks.workunit.client.0.smithi136.stdout:872 2024-09-12T23:55:31.414 INFO:tasks.workunit.client.0.smithi136.stdout:873 2024-09-12T23:55:31.417 INFO:tasks.workunit.client.0.smithi136.stdout:874 2024-09-12T23:55:31.420 INFO:tasks.workunit.client.0.smithi136.stdout:875 2024-09-12T23:55:31.423 INFO:tasks.workunit.client.0.smithi136.stdout:876 2024-09-12T23:55:31.426 INFO:tasks.workunit.client.0.smithi136.stdout:877 2024-09-12T23:55:31.429 INFO:tasks.workunit.client.0.smithi136.stdout:878 2024-09-12T23:55:31.431 INFO:tasks.workunit.client.0.smithi136.stdout:879 2024-09-12T23:55:31.434 INFO:tasks.workunit.client.0.smithi136.stdout:880 2024-09-12T23:55:31.437 INFO:tasks.workunit.client.0.smithi136.stdout:881 2024-09-12T23:55:31.440 INFO:tasks.workunit.client.0.smithi136.stdout:882 2024-09-12T23:55:31.442 INFO:tasks.workunit.client.0.smithi136.stdout:883 2024-09-12T23:55:31.445 INFO:tasks.workunit.client.0.smithi136.stdout:884 2024-09-12T23:55:31.448 INFO:tasks.workunit.client.0.smithi136.stdout:885 2024-09-12T23:55:31.450 INFO:tasks.workunit.client.0.smithi136.stdout:886 2024-09-12T23:55:31.453 INFO:tasks.workunit.client.0.smithi136.stdout:887 2024-09-12T23:55:31.456 INFO:tasks.workunit.client.0.smithi136.stdout:888 2024-09-12T23:55:31.459 INFO:tasks.workunit.client.0.smithi136.stdout:889 2024-09-12T23:55:31.462 INFO:tasks.workunit.client.0.smithi136.stdout:890 2024-09-12T23:55:31.465 INFO:tasks.workunit.client.0.smithi136.stdout:891 2024-09-12T23:55:31.467 INFO:tasks.workunit.client.0.smithi136.stdout:892 2024-09-12T23:55:31.470 INFO:tasks.workunit.client.0.smithi136.stdout:893 2024-09-12T23:55:31.473 INFO:tasks.workunit.client.0.smithi136.stdout:894 2024-09-12T23:55:31.476 INFO:tasks.workunit.client.0.smithi136.stdout:895 2024-09-12T23:55:31.479 INFO:tasks.workunit.client.0.smithi136.stdout:896 2024-09-12T23:55:31.481 INFO:tasks.workunit.client.0.smithi136.stdout:897 2024-09-12T23:55:31.484 INFO:tasks.workunit.client.0.smithi136.stdout:898 2024-09-12T23:55:31.487 INFO:tasks.workunit.client.0.smithi136.stdout:899 2024-09-12T23:55:31.490 INFO:tasks.workunit.client.0.smithi136.stdout:900 2024-09-12T23:55:31.494 INFO:tasks.workunit.client.0.smithi136.stdout:901 2024-09-12T23:55:31.497 INFO:tasks.workunit.client.0.smithi136.stdout:902 2024-09-12T23:55:31.499 INFO:tasks.workunit.client.0.smithi136.stdout:903 2024-09-12T23:55:31.502 INFO:tasks.workunit.client.0.smithi136.stdout:904 2024-09-12T23:55:31.505 INFO:tasks.workunit.client.0.smithi136.stdout:905 2024-09-12T23:55:31.508 INFO:tasks.workunit.client.0.smithi136.stdout:906 2024-09-12T23:55:31.511 INFO:tasks.workunit.client.0.smithi136.stdout:907 2024-09-12T23:55:31.514 INFO:tasks.workunit.client.0.smithi136.stdout:908 2024-09-12T23:55:31.517 INFO:tasks.workunit.client.0.smithi136.stdout:909 2024-09-12T23:55:31.520 INFO:tasks.workunit.client.0.smithi136.stdout:910 2024-09-12T23:55:31.523 INFO:tasks.workunit.client.0.smithi136.stdout:911 2024-09-12T23:55:31.526 INFO:tasks.workunit.client.0.smithi136.stdout:912 2024-09-12T23:55:31.529 INFO:tasks.workunit.client.0.smithi136.stdout:913 2024-09-12T23:55:31.532 INFO:tasks.workunit.client.0.smithi136.stdout:914 2024-09-12T23:55:31.535 INFO:tasks.workunit.client.0.smithi136.stdout:915 2024-09-12T23:55:31.539 INFO:tasks.workunit.client.0.smithi136.stdout:916 2024-09-12T23:55:31.541 INFO:tasks.workunit.client.0.smithi136.stdout:917 2024-09-12T23:55:31.544 INFO:tasks.workunit.client.0.smithi136.stdout:918 2024-09-12T23:55:31.547 INFO:tasks.workunit.client.0.smithi136.stdout:919 2024-09-12T23:55:31.553 INFO:tasks.workunit.client.0.smithi136.stdout:920 2024-09-12T23:55:31.554 INFO:tasks.workunit.client.0.smithi136.stdout:921 2024-09-12T23:55:31.557 INFO:tasks.workunit.client.0.smithi136.stdout:922 2024-09-12T23:55:31.560 INFO:tasks.workunit.client.0.smithi136.stdout:923 2024-09-12T23:55:31.563 INFO:tasks.workunit.client.0.smithi136.stdout:924 2024-09-12T23:55:31.566 INFO:tasks.workunit.client.0.smithi136.stdout:925 2024-09-12T23:55:31.569 INFO:tasks.workunit.client.0.smithi136.stdout:926 2024-09-12T23:55:31.573 INFO:tasks.workunit.client.0.smithi136.stdout:927 2024-09-12T23:55:31.576 INFO:tasks.workunit.client.0.smithi136.stdout:928 2024-09-12T23:55:31.579 INFO:tasks.workunit.client.0.smithi136.stdout:929 2024-09-12T23:55:31.582 INFO:tasks.workunit.client.0.smithi136.stdout:930 2024-09-12T23:55:31.585 INFO:tasks.workunit.client.0.smithi136.stdout:931 2024-09-12T23:55:31.588 INFO:tasks.workunit.client.0.smithi136.stdout:932 2024-09-12T23:55:31.591 INFO:tasks.workunit.client.0.smithi136.stdout:933 2024-09-12T23:55:31.594 INFO:tasks.workunit.client.0.smithi136.stdout:934 2024-09-12T23:55:31.597 INFO:tasks.workunit.client.0.smithi136.stdout:935 2024-09-12T23:55:31.600 INFO:tasks.workunit.client.0.smithi136.stdout:936 2024-09-12T23:55:31.603 INFO:tasks.workunit.client.0.smithi136.stdout:937 2024-09-12T23:55:31.606 INFO:tasks.workunit.client.0.smithi136.stdout:938 2024-09-12T23:55:31.609 INFO:tasks.workunit.client.0.smithi136.stdout:939 2024-09-12T23:55:31.612 INFO:tasks.workunit.client.0.smithi136.stdout:940 2024-09-12T23:55:31.615 INFO:tasks.workunit.client.0.smithi136.stdout:941 2024-09-12T23:55:31.618 INFO:tasks.workunit.client.0.smithi136.stdout:942 2024-09-12T23:55:31.621 INFO:tasks.workunit.client.0.smithi136.stdout:943 2024-09-12T23:55:31.624 INFO:tasks.workunit.client.0.smithi136.stdout:944 2024-09-12T23:55:31.627 INFO:tasks.workunit.client.0.smithi136.stdout:945 2024-09-12T23:55:31.630 INFO:tasks.workunit.client.0.smithi136.stdout:946 2024-09-12T23:55:31.633 INFO:tasks.workunit.client.0.smithi136.stdout:947 2024-09-12T23:55:31.637 INFO:tasks.workunit.client.0.smithi136.stdout:948 2024-09-12T23:55:31.639 INFO:tasks.workunit.client.0.smithi136.stdout:949 2024-09-12T23:55:31.643 INFO:tasks.workunit.client.0.smithi136.stdout:950 2024-09-12T23:55:31.646 INFO:tasks.workunit.client.0.smithi136.stdout:951 2024-09-12T23:55:31.649 INFO:tasks.workunit.client.0.smithi136.stdout:952 2024-09-12T23:55:31.652 INFO:tasks.workunit.client.0.smithi136.stdout:953 2024-09-12T23:55:31.655 INFO:tasks.workunit.client.0.smithi136.stdout:954 2024-09-12T23:55:31.658 INFO:tasks.workunit.client.0.smithi136.stdout:955 2024-09-12T23:55:31.661 INFO:tasks.workunit.client.0.smithi136.stdout:956 2024-09-12T23:55:31.663 INFO:tasks.workunit.client.0.smithi136.stdout:957 2024-09-12T23:55:31.666 INFO:tasks.workunit.client.0.smithi136.stdout:958 2024-09-12T23:55:31.669 INFO:tasks.workunit.client.0.smithi136.stdout:959 2024-09-12T23:55:31.672 INFO:tasks.workunit.client.0.smithi136.stdout:960 2024-09-12T23:55:31.675 INFO:tasks.workunit.client.0.smithi136.stdout:961 2024-09-12T23:55:31.678 INFO:tasks.workunit.client.0.smithi136.stdout:962 2024-09-12T23:55:31.681 INFO:tasks.workunit.client.0.smithi136.stdout:963 2024-09-12T23:55:31.684 INFO:tasks.workunit.client.0.smithi136.stdout:964 2024-09-12T23:55:31.686 INFO:tasks.workunit.client.0.smithi136.stdout:965 2024-09-12T23:55:31.690 INFO:tasks.workunit.client.0.smithi136.stdout:966 2024-09-12T23:55:31.693 INFO:tasks.workunit.client.0.smithi136.stdout:967 2024-09-12T23:55:31.695 INFO:tasks.workunit.client.0.smithi136.stdout:968 2024-09-12T23:55:31.699 INFO:tasks.workunit.client.0.smithi136.stdout:969 2024-09-12T23:55:31.702 INFO:tasks.workunit.client.0.smithi136.stdout:970 2024-09-12T23:55:31.705 INFO:tasks.workunit.client.0.smithi136.stdout:971 2024-09-12T23:55:31.708 INFO:tasks.workunit.client.0.smithi136.stdout:972 2024-09-12T23:55:31.711 INFO:tasks.workunit.client.0.smithi136.stdout:973 2024-09-12T23:55:31.714 INFO:tasks.workunit.client.0.smithi136.stdout:974 2024-09-12T23:55:31.717 INFO:tasks.workunit.client.0.smithi136.stdout:975 2024-09-12T23:55:31.720 INFO:tasks.workunit.client.0.smithi136.stdout:976 2024-09-12T23:55:31.723 INFO:tasks.workunit.client.0.smithi136.stdout:977 2024-09-12T23:55:31.726 INFO:tasks.workunit.client.0.smithi136.stdout:978 2024-09-12T23:55:31.729 INFO:tasks.workunit.client.0.smithi136.stdout:979 2024-09-12T23:55:31.732 INFO:tasks.workunit.client.0.smithi136.stdout:980 2024-09-12T23:55:31.735 INFO:tasks.workunit.client.0.smithi136.stdout:981 2024-09-12T23:55:31.739 INFO:tasks.workunit.client.0.smithi136.stdout:982 2024-09-12T23:55:31.742 INFO:tasks.workunit.client.0.smithi136.stdout:983 2024-09-12T23:55:31.745 INFO:tasks.workunit.client.0.smithi136.stdout:984 2024-09-12T23:55:31.748 INFO:tasks.workunit.client.0.smithi136.stdout:985 2024-09-12T23:55:31.752 INFO:tasks.workunit.client.0.smithi136.stdout:986 2024-09-12T23:55:31.755 INFO:tasks.workunit.client.0.smithi136.stdout:987 2024-09-12T23:55:31.758 INFO:tasks.workunit.client.0.smithi136.stdout:988 2024-09-12T23:55:31.761 INFO:tasks.workunit.client.0.smithi136.stdout:989 2024-09-12T23:55:31.764 INFO:tasks.workunit.client.0.smithi136.stdout:990 2024-09-12T23:55:31.767 INFO:tasks.workunit.client.0.smithi136.stdout:991 2024-09-12T23:55:31.770 INFO:tasks.workunit.client.0.smithi136.stdout:992 2024-09-12T23:55:31.773 INFO:tasks.workunit.client.0.smithi136.stdout:993 2024-09-12T23:55:31.776 INFO:tasks.workunit.client.0.smithi136.stdout:994 2024-09-12T23:55:31.779 INFO:tasks.workunit.client.0.smithi136.stdout:995 2024-09-12T23:55:31.782 INFO:tasks.workunit.client.0.smithi136.stdout:996 2024-09-12T23:55:31.785 INFO:tasks.workunit.client.0.smithi136.stdout:997 2024-09-12T23:55:31.788 INFO:tasks.workunit.client.0.smithi136.stdout:998 2024-09-12T23:55:31.791 INFO:tasks.workunit.client.0.smithi136.stdout:999 2024-09-12T23:55:32.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:31 smithi136 bash[75164]: audit 2024-09-12T23:55:30.738425+0000 mon.a (mon.0) 1686 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:55:32.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:31 smithi136 bash[75164]: audit 2024-09-12T23:55:30.743671+0000 mon.a (mon.0) 1687 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:55:32.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:31 smithi136 bash[75164]: audit 2024-09-12T23:55:31.185863+0000 mon.a (mon.0) 1688 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:55:32.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:31 smithi136 bash[75164]: audit 2024-09-12T23:55:31.186841+0000 mon.a (mon.0) 1689 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:55:32.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:31 smithi136 bash[75164]: audit 2024-09-12T23:55:31.195382+0000 mon.a (mon.0) 1690 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:55:32.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:31 smithi136 bash[76371]: audit 2024-09-12T23:55:30.738425+0000 mon.a (mon.0) 1686 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:55:32.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:31 smithi136 bash[76371]: audit 2024-09-12T23:55:30.743671+0000 mon.a (mon.0) 1687 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:55:32.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:31 smithi136 bash[76371]: audit 2024-09-12T23:55:31.185863+0000 mon.a (mon.0) 1688 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:55:32.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:31 smithi136 bash[76371]: audit 2024-09-12T23:55:31.186841+0000 mon.a (mon.0) 1689 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:55:32.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:31 smithi136 bash[76371]: audit 2024-09-12T23:55:31.195382+0000 mon.a (mon.0) 1690 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:55:32.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:31 smithi181 bash[67587]: audit 2024-09-12T23:55:30.738425+0000 mon.a (mon.0) 1686 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:55:32.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:31 smithi181 bash[67587]: audit 2024-09-12T23:55:30.743671+0000 mon.a (mon.0) 1687 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:55:32.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:31 smithi181 bash[67587]: audit 2024-09-12T23:55:31.185863+0000 mon.a (mon.0) 1688 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:55:32.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:31 smithi181 bash[67587]: audit 2024-09-12T23:55:31.186841+0000 mon.a (mon.0) 1689 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:55:32.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:31 smithi181 bash[67587]: audit 2024-09-12T23:55:31.195382+0000 mon.a (mon.0) 1690 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:55:32.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:55:32 smithi181 bash[95505]: ts=2024-09-12T23:55:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:55:32.761 INFO:tasks.workunit.client.0.smithi136.stdout:00000000 79 6f 75 20 6d 69 67 68 74 20 73 65 65 20 74 68 |you might see th| 2024-09-12T23:55:32.762 INFO:tasks.workunit.client.0.smithi136.stdout:00000010 69 73 |is| 2024-09-12T23:55:32.762 INFO:tasks.workunit.client.0.smithi136.stdout:00000012 2024-09-12T23:55:32.762 INFO:tasks.workunit.client.0.smithi136.stdout:00000000 79 6f 75 20 6d 69 67 68 74 20 73 65 65 20 74 68 |you might see th| 2024-09-12T23:55:32.762 INFO:tasks.workunit.client.0.smithi136.stdout:00000010 69 73 |is| 2024-09-12T23:55:32.762 INFO:tasks.workunit.client.0.smithi136.stdout:00000012 2024-09-12T23:55:32.762 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] ClsHello.WriteReturnData (5273 ms) 2024-09-12T23:55:32.762 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] ClsHello.Loud 2024-09-12T23:55:33.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:32 smithi136 bash[75164]: cluster 2024-09-12T23:55:30.983036+0000 mgr.y (mgr.24661) 552 : cluster [DBG] pgmap v1075: 137 pgs: 4 creating+activating, 6 creating+peering, 17 unknown, 110 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:33.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:32 smithi136 bash[75164]: cluster 2024-09-12T23:55:31.746421+0000 mon.a (mon.0) 1691 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:55:33.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:32 smithi136 bash[75164]: cluster 2024-09-12T23:55:31.746488+0000 mon.a (mon.0) 1692 : cluster [INF] Cluster is now healthy 2024-09-12T23:55:33.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:32 smithi136 bash[76371]: cluster 2024-09-12T23:55:30.983036+0000 mgr.y (mgr.24661) 552 : cluster [DBG] pgmap v1075: 137 pgs: 4 creating+activating, 6 creating+peering, 17 unknown, 110 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:33.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:32 smithi136 bash[76371]: cluster 2024-09-12T23:55:31.746421+0000 mon.a (mon.0) 1691 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:55:33.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:32 smithi136 bash[76371]: cluster 2024-09-12T23:55:31.746488+0000 mon.a (mon.0) 1692 : cluster [INF] Cluster is now healthy 2024-09-12T23:55:33.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:32 smithi181 bash[67587]: cluster 2024-09-12T23:55:30.983036+0000 mgr.y (mgr.24661) 552 : cluster [DBG] pgmap v1075: 137 pgs: 4 creating+activating, 6 creating+peering, 17 unknown, 110 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:33.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:32 smithi181 bash[67587]: cluster 2024-09-12T23:55:31.746421+0000 mon.a (mon.0) 1691 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:55:33.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:32 smithi181 bash[67587]: cluster 2024-09-12T23:55:31.746488+0000 mon.a (mon.0) 1692 : cluster [INF] Cluster is now healthy 2024-09-12T23:55:34.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:33 smithi181 bash[67587]: cluster 2024-09-12T23:55:32.758029+0000 mon.a (mon.0) 1693 : cluster [DBG] osdmap e963: 8 total, 8 up, 8 in 2024-09-12T23:55:34.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:33 smithi136 bash[75164]: cluster 2024-09-12T23:55:32.758029+0000 mon.a (mon.0) 1693 : cluster [DBG] osdmap e963: 8 total, 8 up, 8 in 2024-09-12T23:55:34.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:33 smithi136 bash[76371]: cluster 2024-09-12T23:55:32.758029+0000 mon.a (mon.0) 1693 : cluster [DBG] osdmap e963: 8 total, 8 up, 8 in 2024-09-12T23:55:35.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:34 smithi181 bash[67587]: cluster 2024-09-12T23:55:32.983740+0000 mgr.y (mgr.24661) 553 : cluster [DBG] pgmap v1077: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:35.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:34 smithi181 bash[67587]: cluster 2024-09-12T23:55:33.775005+0000 mon.a (mon.0) 1694 : cluster [DBG] osdmap e964: 8 total, 8 up, 8 in 2024-09-12T23:55:35.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:34 smithi181 bash[67587]: audit 2024-09-12T23:55:33.776509+0000 mon.a (mon.0) 1695 : audit [INF] from='client.? 172.21.15.136:0/615326247' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109763-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:35.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:34 smithi136 bash[75164]: cluster 2024-09-12T23:55:32.983740+0000 mgr.y (mgr.24661) 553 : cluster [DBG] pgmap v1077: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:35.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:34 smithi136 bash[75164]: cluster 2024-09-12T23:55:33.775005+0000 mon.a (mon.0) 1694 : cluster [DBG] osdmap e964: 8 total, 8 up, 8 in 2024-09-12T23:55:35.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:34 smithi136 bash[75164]: audit 2024-09-12T23:55:33.776509+0000 mon.a (mon.0) 1695 : audit [INF] from='client.? 172.21.15.136:0/615326247' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109763-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:35.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:34 smithi136 bash[76371]: cluster 2024-09-12T23:55:32.983740+0000 mgr.y (mgr.24661) 553 : cluster [DBG] pgmap v1077: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:35.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:34 smithi136 bash[76371]: cluster 2024-09-12T23:55:33.775005+0000 mon.a (mon.0) 1694 : cluster [DBG] osdmap e964: 8 total, 8 up, 8 in 2024-09-12T23:55:35.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:34 smithi136 bash[76371]: audit 2024-09-12T23:55:33.776509+0000 mon.a (mon.0) 1695 : audit [INF] from='client.? 172.21.15.136:0/615326247' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109763-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:35.807 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] ClsHello.Loud (3044 ms) 2024-09-12T23:55:35.807 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] ClsHello.BadMethods 2024-09-12T23:55:36.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:35 smithi181 bash[67587]: audit 2024-09-12T23:55:34.775103+0000 mon.a (mon.0) 1696 : audit [INF] from='client.? 172.21.15.136:0/615326247' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109763-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:55:36.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:35 smithi181 bash[67587]: cluster 2024-09-12T23:55:34.776627+0000 mon.a (mon.0) 1697 : cluster [DBG] osdmap e965: 8 total, 8 up, 8 in 2024-09-12T23:55:36.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:35 smithi136 bash[75164]: audit 2024-09-12T23:55:34.775103+0000 mon.a (mon.0) 1696 : audit [INF] from='client.? 172.21.15.136:0/615326247' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109763-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:55:36.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:35 smithi136 bash[75164]: cluster 2024-09-12T23:55:34.776627+0000 mon.a (mon.0) 1697 : cluster [DBG] osdmap e965: 8 total, 8 up, 8 in 2024-09-12T23:55:36.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:35 smithi136 bash[76371]: audit 2024-09-12T23:55:34.775103+0000 mon.a (mon.0) 1696 : audit [INF] from='client.? 172.21.15.136:0/615326247' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109763-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:55:36.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:35 smithi136 bash[76371]: cluster 2024-09-12T23:55:34.776627+0000 mon.a (mon.0) 1697 : cluster [DBG] osdmap e965: 8 total, 8 up, 8 in 2024-09-12T23:55:37.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:36 smithi181 bash[67587]: cluster 2024-09-12T23:55:34.984421+0000 mgr.y (mgr.24661) 554 : cluster [DBG] pgmap v1080: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:37.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:36 smithi181 bash[67587]: cluster 2024-09-12T23:55:35.804609+0000 mon.a (mon.0) 1698 : cluster [DBG] osdmap e966: 8 total, 8 up, 8 in 2024-09-12T23:55:37.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:36 smithi136 bash[75164]: cluster 2024-09-12T23:55:34.984421+0000 mgr.y (mgr.24661) 554 : cluster [DBG] pgmap v1080: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:37.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:36 smithi136 bash[75164]: cluster 2024-09-12T23:55:35.804609+0000 mon.a (mon.0) 1698 : cluster [DBG] osdmap e966: 8 total, 8 up, 8 in 2024-09-12T23:55:37.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:36 smithi136 bash[76371]: cluster 2024-09-12T23:55:34.984421+0000 mgr.y (mgr.24661) 554 : cluster [DBG] pgmap v1080: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:37.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:36 smithi136 bash[76371]: cluster 2024-09-12T23:55:35.804609+0000 mon.a (mon.0) 1698 : cluster [DBG] osdmap e966: 8 total, 8 up, 8 in 2024-09-12T23:55:38.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:37 smithi181 bash[67587]: cluster 2024-09-12T23:55:36.814479+0000 mon.a (mon.0) 1699 : cluster [DBG] osdmap e967: 8 total, 8 up, 8 in 2024-09-12T23:55:38.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:37 smithi181 bash[67587]: audit 2024-09-12T23:55:36.820380+0000 mon.a (mon.0) 1700 : audit [INF] from='client.? 172.21.15.136:0/2063488042' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109763-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:38.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:37 smithi136 bash[75164]: cluster 2024-09-12T23:55:36.814479+0000 mon.a (mon.0) 1699 : cluster [DBG] osdmap e967: 8 total, 8 up, 8 in 2024-09-12T23:55:38.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:37 smithi136 bash[75164]: audit 2024-09-12T23:55:36.820380+0000 mon.a (mon.0) 1700 : audit [INF] from='client.? 172.21.15.136:0/2063488042' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109763-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:38.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:37 smithi136 bash[76371]: cluster 2024-09-12T23:55:36.814479+0000 mon.a (mon.0) 1699 : cluster [DBG] osdmap e967: 8 total, 8 up, 8 in 2024-09-12T23:55:38.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:37 smithi136 bash[76371]: audit 2024-09-12T23:55:36.820380+0000 mon.a (mon.0) 1700 : audit [INF] from='client.? 172.21.15.136:0/2063488042' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109763-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:38.258 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 12 23:55:37 smithi136 bash[93456]: debug 2024-09-12T23:55:37.861+0000 7f241e4d3640 -1 osd.3 pg_epoch: 968 pg[38.2( v 968'1 (0'0,968'1] local-lis/les=967/968 n=1 ec=967/967 lis/c=967/967 les/c/f=968/968/0 sis=967) [3,7,5] r=0 lpr=967 crt=968'1 lcod 0'0 mlcod 0'0 active+clean] method hello.bad_reader tried to read object but is not marked RD 2024-09-12T23:55:38.258 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 12 23:55:37 smithi136 bash[93456]: debug 2024-09-12T23:55:37.869+0000 7f24224db640 -1 osd.3 pg_epoch: 968 pg[38.2( v 968'2 (0'0,968'2] local-lis/les=967/968 n=1 ec=967/967 lis/c=967/967 les/c/f=968/968/0 sis=967) [3,7,5] r=0 lpr=967 crt=968'1 lcod 968'1 mlcod 968'1 active+clean] method hello.bad_writer tried to update object but is not marked WR 2024-09-12T23:55:38.828 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] ClsHello.BadMethods (3022 ms) 2024-09-12T23:55:38.828 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] ClsHello.Filter 2024-09-12T23:55:39.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:38 smithi181 bash[67587]: cluster 2024-09-12T23:55:36.985302+0000 mgr.y (mgr.24661) 555 : cluster [DBG] pgmap v1083: 137 pgs: 1 creating+peering, 31 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:39.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:38 smithi181 bash[67587]: cluster 2024-09-12T23:55:37.807593+0000 mon.a (mon.0) 1701 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:55:39.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:38 smithi181 bash[67587]: audit 2024-09-12T23:55:37.826568+0000 mon.a (mon.0) 1702 : audit [INF] from='client.? 172.21.15.136:0/2063488042' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109763-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:55:39.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:38 smithi181 bash[67587]: cluster 2024-09-12T23:55:37.827572+0000 mon.a (mon.0) 1703 : cluster [DBG] osdmap e968: 8 total, 8 up, 8 in 2024-09-12T23:55:39.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:38 smithi181 bash[67587]: audit 2024-09-12T23:55:38.761545+0000 mon.a (mon.0) 1704 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:55:39.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:38 smithi136 bash[76371]: cluster 2024-09-12T23:55:36.985302+0000 mgr.y (mgr.24661) 555 : cluster [DBG] pgmap v1083: 137 pgs: 1 creating+peering, 31 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:39.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:38 smithi136 bash[76371]: cluster 2024-09-12T23:55:37.807593+0000 mon.a (mon.0) 1701 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:55:39.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:38 smithi136 bash[76371]: audit 2024-09-12T23:55:37.826568+0000 mon.a (mon.0) 1702 : audit [INF] from='client.? 172.21.15.136:0/2063488042' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109763-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:55:39.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:38 smithi136 bash[76371]: cluster 2024-09-12T23:55:37.827572+0000 mon.a (mon.0) 1703 : cluster [DBG] osdmap e968: 8 total, 8 up, 8 in 2024-09-12T23:55:39.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:38 smithi136 bash[76371]: audit 2024-09-12T23:55:38.761545+0000 mon.a (mon.0) 1704 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:55:39.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:38 smithi136 bash[75164]: cluster 2024-09-12T23:55:36.985302+0000 mgr.y (mgr.24661) 555 : cluster [DBG] pgmap v1083: 137 pgs: 1 creating+peering, 31 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:39.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:38 smithi136 bash[75164]: cluster 2024-09-12T23:55:37.807593+0000 mon.a (mon.0) 1701 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:55:39.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:38 smithi136 bash[75164]: audit 2024-09-12T23:55:37.826568+0000 mon.a (mon.0) 1702 : audit [INF] from='client.? 172.21.15.136:0/2063488042' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109763-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:55:39.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:38 smithi136 bash[75164]: cluster 2024-09-12T23:55:37.827572+0000 mon.a (mon.0) 1703 : cluster [DBG] osdmap e968: 8 total, 8 up, 8 in 2024-09-12T23:55:39.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:38 smithi136 bash[75164]: audit 2024-09-12T23:55:38.761545+0000 mon.a (mon.0) 1704 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:55:39.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:55:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:55:39] "GET /metrics HTTP/1.1" 200 36448 "" "Prometheus/2.43.0" 2024-09-12T23:55:40.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:39 smithi136 bash[75164]: cluster 2024-09-12T23:55:38.826746+0000 mon.a (mon.0) 1705 : cluster [DBG] osdmap e969: 8 total, 8 up, 8 in 2024-09-12T23:55:40.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:39 smithi136 bash[76371]: cluster 2024-09-12T23:55:38.826746+0000 mon.a (mon.0) 1705 : cluster [DBG] osdmap e969: 8 total, 8 up, 8 in 2024-09-12T23:55:40.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:39 smithi181 bash[67587]: cluster 2024-09-12T23:55:38.826746+0000 mon.a (mon.0) 1705 : cluster [DBG] osdmap e969: 8 total, 8 up, 8 in 2024-09-12T23:55:41.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:40 smithi136 bash[75164]: cluster 2024-09-12T23:55:38.985998+0000 mgr.y (mgr.24661) 556 : cluster [DBG] pgmap v1086: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:41.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:40 smithi136 bash[75164]: cluster 2024-09-12T23:55:39.837568+0000 mon.a (mon.0) 1706 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:55:41.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:40 smithi136 bash[75164]: cluster 2024-09-12T23:55:39.837635+0000 mon.a (mon.0) 1707 : cluster [INF] Cluster is now healthy 2024-09-12T23:55:41.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:40 smithi136 bash[75164]: cluster 2024-09-12T23:55:39.861664+0000 mon.a (mon.0) 1708 : cluster [DBG] osdmap e970: 8 total, 8 up, 8 in 2024-09-12T23:55:41.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:40 smithi136 bash[75164]: audit 2024-09-12T23:55:39.866562+0000 mon.c (mon.2) 40 : audit [INF] from='client.? 172.21.15.136:0/3524963041' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109763-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:41.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:40 smithi136 bash[75164]: audit 2024-09-12T23:55:39.867885+0000 mon.a (mon.0) 1709 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109763-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:41.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:40 smithi136 bash[76371]: cluster 2024-09-12T23:55:38.985998+0000 mgr.y (mgr.24661) 556 : cluster [DBG] pgmap v1086: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:41.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:40 smithi136 bash[76371]: cluster 2024-09-12T23:55:39.837568+0000 mon.a (mon.0) 1706 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:55:41.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:40 smithi136 bash[76371]: cluster 2024-09-12T23:55:39.837635+0000 mon.a (mon.0) 1707 : cluster [INF] Cluster is now healthy 2024-09-12T23:55:41.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:40 smithi136 bash[76371]: cluster 2024-09-12T23:55:39.861664+0000 mon.a (mon.0) 1708 : cluster [DBG] osdmap e970: 8 total, 8 up, 8 in 2024-09-12T23:55:41.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:40 smithi136 bash[76371]: audit 2024-09-12T23:55:39.866562+0000 mon.c (mon.2) 40 : audit [INF] from='client.? 172.21.15.136:0/3524963041' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109763-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:41.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:40 smithi136 bash[76371]: audit 2024-09-12T23:55:39.867885+0000 mon.a (mon.0) 1709 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109763-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:41.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:40 smithi181 bash[67587]: cluster 2024-09-12T23:55:38.985998+0000 mgr.y (mgr.24661) 556 : cluster [DBG] pgmap v1086: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:41.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:40 smithi181 bash[67587]: cluster 2024-09-12T23:55:39.837568+0000 mon.a (mon.0) 1706 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:55:41.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:40 smithi181 bash[67587]: cluster 2024-09-12T23:55:39.837635+0000 mon.a (mon.0) 1707 : cluster [INF] Cluster is now healthy 2024-09-12T23:55:41.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:40 smithi181 bash[67587]: cluster 2024-09-12T23:55:39.861664+0000 mon.a (mon.0) 1708 : cluster [DBG] osdmap e970: 8 total, 8 up, 8 in 2024-09-12T23:55:41.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:40 smithi181 bash[67587]: audit 2024-09-12T23:55:39.866562+0000 mon.c (mon.2) 40 : audit [INF] from='client.? 172.21.15.136:0/3524963041' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109763-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:41.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:40 smithi181 bash[67587]: audit 2024-09-12T23:55:39.867885+0000 mon.a (mon.0) 1709 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109763-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:41.886 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] ClsHello.Filter (3058 ms) 2024-09-12T23:55:41.886 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 6 tests from ClsHello (20359 ms total) 2024-09-12T23:55:41.886 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-12T23:55:41.886 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] Global test environment tear-down 2024-09-12T23:55:41.886 INFO:tasks.workunit.client.0.smithi136.stdout:[==========] 6 tests from 1 test suite ran. (20359 ms total) 2024-09-12T23:55:41.886 INFO:tasks.workunit.client.0.smithi136.stdout:[ PASSED ] 6 tests. 2024-09-12T23:55:41.892 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-12T23:55:41.892 DEBUG:teuthology.orchestra.run.smithi136:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-12T23:55:41.945 INFO:tasks.workunit:Running workunit cls/test_cls_journal.sh... 2024-09-12T23:55:41.945 DEBUG:teuthology.orchestra.run.smithi136:workunit test cls/test_cls_journal.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_journal.sh 2024-09-12T23:55:42.008 INFO:tasks.workunit.client.0.smithi136.stdout:Running main() from gmock_main.cc 2024-09-12T23:55:42.008 INFO:tasks.workunit.client.0.smithi136.stdout:[==========] Running 27 tests from 1 test suite. 2024-09-12T23:55:42.008 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] Global test environment set-up. 2024-09-12T23:55:42.008 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 27 tests from TestClsJournal 2024-09-12T23:55:42.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:41 smithi136 bash[75164]: audit 2024-09-12T23:55:40.872623+0000 mon.a (mon.0) 1710 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109763-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:55:42.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:41 smithi136 bash[75164]: cluster 2024-09-12T23:55:40.873027+0000 mon.a (mon.0) 1711 : cluster [DBG] osdmap e971: 8 total, 8 up, 8 in 2024-09-12T23:55:42.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:41 smithi136 bash[76371]: audit 2024-09-12T23:55:40.872623+0000 mon.a (mon.0) 1710 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109763-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:55:42.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:41 smithi136 bash[76371]: cluster 2024-09-12T23:55:40.873027+0000 mon.a (mon.0) 1711 : cluster [DBG] osdmap e971: 8 total, 8 up, 8 in 2024-09-12T23:55:42.314 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:41 smithi181 bash[67587]: audit 2024-09-12T23:55:40.872623+0000 mon.a (mon.0) 1710 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109763-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:55:42.314 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:41 smithi181 bash[67587]: cluster 2024-09-12T23:55:40.873027+0000 mon.a (mon.0) 1711 : cluster [DBG] osdmap e971: 8 total, 8 up, 8 in 2024-09-12T23:55:42.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:55:42 smithi181 bash[95505]: ts=2024-09-12T23:55:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:55:43.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:42 smithi136 bash[75164]: cluster 2024-09-12T23:55:40.986727+0000 mgr.y (mgr.24661) 557 : cluster [DBG] pgmap v1089: 137 pgs: 12 creating+peering, 20 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:43.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:42 smithi136 bash[75164]: cluster 2024-09-12T23:55:41.883950+0000 mon.a (mon.0) 1712 : cluster [DBG] osdmap e972: 8 total, 8 up, 8 in 2024-09-12T23:55:43.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:42 smithi136 bash[76371]: cluster 2024-09-12T23:55:40.986727+0000 mgr.y (mgr.24661) 557 : cluster [DBG] pgmap v1089: 137 pgs: 12 creating+peering, 20 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:43.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:42 smithi136 bash[76371]: cluster 2024-09-12T23:55:41.883950+0000 mon.a (mon.0) 1712 : cluster [DBG] osdmap e972: 8 total, 8 up, 8 in 2024-09-12T23:55:43.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:42 smithi181 bash[67587]: cluster 2024-09-12T23:55:40.986727+0000 mgr.y (mgr.24661) 557 : cluster [DBG] pgmap v1089: 137 pgs: 12 creating+peering, 20 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:43.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:42 smithi181 bash[67587]: cluster 2024-09-12T23:55:41.883950+0000 mon.a (mon.0) 1712 : cluster [DBG] osdmap e972: 8 total, 8 up, 8 in 2024-09-12T23:55:43.887 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsJournal.Create 2024-09-12T23:55:43.916 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsJournal.Create (29 ms) 2024-09-12T23:55:43.916 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsJournal.MinimumSet 2024-09-12T23:55:43.938 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsJournal.MinimumSet (22 ms) 2024-09-12T23:55:43.938 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsJournal.MinimumSetStale 2024-09-12T23:55:43.945 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsJournal.MinimumSetStale (7 ms) 2024-09-12T23:55:43.945 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsJournal.MinimumSetOrderConstraint 2024-09-12T23:55:43.950 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsJournal.MinimumSetOrderConstraint (5 ms) 2024-09-12T23:55:43.950 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsJournal.ActiveSet 2024-09-12T23:55:43.954 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsJournal.ActiveSet (4 ms) 2024-09-12T23:55:43.954 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsJournal.ActiveSetStale 2024-09-12T23:55:43.958 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsJournal.ActiveSetStale (4 ms) 2024-09-12T23:55:43.959 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsJournal.CreateDuplicate 2024-09-12T23:55:43.965 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsJournal.CreateDuplicate (6 ms) 2024-09-12T23:55:43.965 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsJournal.GetClient 2024-09-12T23:55:43.969 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsJournal.GetClient (4 ms) 2024-09-12T23:55:43.969 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsJournal.ClientRegister 2024-09-12T23:55:43.974 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsJournal.ClientRegister (5 ms) 2024-09-12T23:55:43.974 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsJournal.ClientRegisterDuplicate 2024-09-12T23:55:43.980 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsJournal.ClientRegisterDuplicate (6 ms) 2024-09-12T23:55:43.980 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsJournal.ClientUpdateData 2024-09-12T23:55:43.989 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsJournal.ClientUpdateData (9 ms) 2024-09-12T23:55:43.989 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsJournal.ClientUpdateState 2024-09-12T23:55:44.000 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsJournal.ClientUpdateState (11 ms) 2024-09-12T23:55:44.000 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsJournal.ClientUnregister 2024-09-12T23:55:44.006 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsJournal.ClientUnregister (6 ms) 2024-09-12T23:55:44.006 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsJournal.ClientUnregisterDNE 2024-09-12T23:55:44.016 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsJournal.ClientUnregisterDNE (10 ms) 2024-09-12T23:55:44.016 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsJournal.ClientUnregisterPruneTags 2024-09-12T23:55:44.249 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsJournal.ClientUnregisterPruneTags (233 ms) 2024-09-12T23:55:44.249 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsJournal.ClientCommit 2024-09-12T23:55:44.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:43 smithi136 bash[75164]: cluster 2024-09-12T23:55:42.888219+0000 mon.a (mon.0) 1713 : cluster [DBG] osdmap e973: 8 total, 8 up, 8 in 2024-09-12T23:55:44.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:43 smithi136 bash[75164]: audit 2024-09-12T23:55:42.891935+0000 mon.a (mon.0) 1714 : audit [INF] from='client.? 172.21.15.136:0/3103635280' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109909-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:44.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:43 smithi136 bash[75164]: audit 2024-09-12T23:55:43.889010+0000 mon.a (mon.0) 1715 : audit [INF] from='client.? 172.21.15.136:0/3103635280' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109909-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:55:44.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:43 smithi136 bash[75164]: cluster 2024-09-12T23:55:43.890301+0000 mon.a (mon.0) 1716 : cluster [DBG] osdmap e974: 8 total, 8 up, 8 in 2024-09-12T23:55:44.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:43 smithi136 bash[76371]: cluster 2024-09-12T23:55:42.888219+0000 mon.a (mon.0) 1713 : cluster [DBG] osdmap e973: 8 total, 8 up, 8 in 2024-09-12T23:55:44.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:43 smithi136 bash[76371]: audit 2024-09-12T23:55:42.891935+0000 mon.a (mon.0) 1714 : audit [INF] from='client.? 172.21.15.136:0/3103635280' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109909-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:44.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:43 smithi136 bash[76371]: audit 2024-09-12T23:55:43.889010+0000 mon.a (mon.0) 1715 : audit [INF] from='client.? 172.21.15.136:0/3103635280' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109909-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:55:44.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:43 smithi136 bash[76371]: cluster 2024-09-12T23:55:43.890301+0000 mon.a (mon.0) 1716 : cluster [DBG] osdmap e974: 8 total, 8 up, 8 in 2024-09-12T23:55:44.258 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsJournal.ClientCommit (8 ms) 2024-09-12T23:55:44.258 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsJournal.ClientCommitInvalid 2024-09-12T23:55:44.264 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsJournal.ClientCommitInvalid (7 ms) 2024-09-12T23:55:44.264 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsJournal.ClientCommitDNE 2024-09-12T23:55:44.266 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsJournal.ClientCommitDNE (2 ms) 2024-09-12T23:55:44.266 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsJournal.ClientList 2024-09-12T23:55:44.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:43 smithi181 bash[67587]: cluster 2024-09-12T23:55:42.888219+0000 mon.a (mon.0) 1713 : cluster [DBG] osdmap e973: 8 total, 8 up, 8 in 2024-09-12T23:55:44.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:43 smithi181 bash[67587]: audit 2024-09-12T23:55:42.891935+0000 mon.a (mon.0) 1714 : audit [INF] from='client.? 172.21.15.136:0/3103635280' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109909-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:44.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:43 smithi181 bash[67587]: audit 2024-09-12T23:55:43.889010+0000 mon.a (mon.0) 1715 : audit [INF] from='client.? 172.21.15.136:0/3103635280' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109909-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:55:44.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:43 smithi181 bash[67587]: cluster 2024-09-12T23:55:43.890301+0000 mon.a (mon.0) 1716 : cluster [DBG] osdmap e974: 8 total, 8 up, 8 in 2024-09-12T23:55:44.369 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsJournal.ClientList (103 ms) 2024-09-12T23:55:44.369 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsJournal.GetNextTagTid 2024-09-12T23:55:44.380 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsJournal.GetNextTagTid (11 ms) 2024-09-12T23:55:44.380 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsJournal.TagCreate 2024-09-12T23:55:44.402 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsJournal.TagCreate (22 ms) 2024-09-12T23:55:44.402 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsJournal.TagCreatePrunesTags 2024-09-12T23:55:44.419 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsJournal.TagCreatePrunesTags (17 ms) 2024-09-12T23:55:44.419 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsJournal.TagList 2024-09-12T23:55:44.696 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsJournal.TagList (277 ms) 2024-09-12T23:55:44.697 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsJournal.GuardAppend 2024-09-12T23:55:44.701 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsJournal.GuardAppend (5 ms) 2024-09-12T23:55:44.701 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsJournal.GuardAppendDNE 2024-09-12T23:55:44.704 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsJournal.GuardAppendDNE (3 ms) 2024-09-12T23:55:44.704 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsJournal.GuardAppendOverflow 2024-09-12T23:55:44.709 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsJournal.GuardAppendOverflow (5 ms) 2024-09-12T23:55:44.709 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsJournal.Append 2024-09-12T23:55:44.732 INFO:tasks.workunit.client.0.smithi136.stdout:padding is not enabled 2024-09-12T23:55:44.732 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsJournal.Append (22 ms) 2024-09-12T23:55:44.911 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 27 tests from TestClsJournal (844 ms total) 2024-09-12T23:55:44.911 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-12T23:55:44.911 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] Global test environment tear-down 2024-09-12T23:55:44.911 INFO:tasks.workunit.client.0.smithi136.stdout:[==========] 27 tests from 1 test suite ran. (2903 ms total) 2024-09-12T23:55:44.911 INFO:tasks.workunit.client.0.smithi136.stdout:[ PASSED ] 27 tests. 2024-09-12T23:55:44.915 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-12T23:55:44.915 DEBUG:teuthology.orchestra.run.smithi136:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-12T23:55:44.932 INFO:tasks.workunit:Running workunit cls/test_cls_lock.sh... 2024-09-12T23:55:44.932 DEBUG:teuthology.orchestra.run.smithi136:workunit test cls/test_cls_lock.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_lock.sh 2024-09-12T23:55:44.992 INFO:tasks.workunit.client.0.smithi136.stdout:Running main() from gmock_main.cc 2024-09-12T23:55:44.993 INFO:tasks.workunit.client.0.smithi136.stdout:[==========] Running 11 tests from 1 test suite. 2024-09-12T23:55:44.993 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] Global test environment set-up. 2024-09-12T23:55:44.993 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 11 tests from ClsLock 2024-09-12T23:55:44.993 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] ClsLock.TestMultiLocking 2024-09-12T23:55:45.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:44 smithi136 bash[75164]: cluster 2024-09-12T23:55:42.987487+0000 mgr.y (mgr.24661) 558 : cluster [DBG] pgmap v1092: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:45.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:44 smithi136 bash[75164]: cluster 2024-09-12T23:55:43.892848+0000 mon.a (mon.0) 1717 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:55:45.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:44 smithi136 bash[76371]: cluster 2024-09-12T23:55:42.987487+0000 mgr.y (mgr.24661) 558 : cluster [DBG] pgmap v1092: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:45.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:44 smithi136 bash[76371]: cluster 2024-09-12T23:55:43.892848+0000 mon.a (mon.0) 1717 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:55:45.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:44 smithi181 bash[67587]: cluster 2024-09-12T23:55:42.987487+0000 mgr.y (mgr.24661) 558 : cluster [DBG] pgmap v1092: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:45.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:44 smithi181 bash[67587]: cluster 2024-09-12T23:55:43.892848+0000 mon.a (mon.0) 1717 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:55:46.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:45 smithi136 bash[75164]: cluster 2024-09-12T23:55:44.908880+0000 mon.a (mon.0) 1718 : cluster [DBG] osdmap e975: 8 total, 8 up, 8 in 2024-09-12T23:55:46.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:45 smithi136 bash[76371]: cluster 2024-09-12T23:55:44.908880+0000 mon.a (mon.0) 1718 : cluster [DBG] osdmap e975: 8 total, 8 up, 8 in 2024-09-12T23:55:46.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:45 smithi181 bash[67587]: cluster 2024-09-12T23:55:44.908880+0000 mon.a (mon.0) 1718 : cluster [DBG] osdmap e975: 8 total, 8 up, 8 in 2024-09-12T23:55:46.996 INFO:tasks.workunit.client.0.smithi136.stdout:lock: mylock 2024-09-12T23:55:46.997 INFO:tasks.workunit.client.0.smithi136.stdout: lock_type: exclusive 2024-09-12T23:55:46.997 INFO:tasks.workunit.client.0.smithi136.stdout: tag: 2024-09-12T23:55:46.997 INFO:tasks.workunit.client.0.smithi136.stdout: lockers: 2024-09-12T23:55:46.997 INFO:tasks.workunit.client.0.smithi136.stdout: client.44672 expiration=2024-09-12T23:57:46.991514+0000 addr=v1:172.21.15.136:0/2080813635 cookie= 2024-09-12T23:55:47.003 INFO:tasks.workunit.client.0.smithi136.stdout:lock: mylock 2024-09-12T23:55:47.003 INFO:tasks.workunit.client.0.smithi136.stdout: lock_type: shared 2024-09-12T23:55:47.004 INFO:tasks.workunit.client.0.smithi136.stdout: tag: 2024-09-12T23:55:47.004 INFO:tasks.workunit.client.0.smithi136.stdout: lockers: 2024-09-12T23:55:47.004 INFO:tasks.workunit.client.0.smithi136.stdout: client.44672 expiration=2024-09-12T23:57:47.003282+0000 addr=v1:172.21.15.136:0/2080813635 cookie= 2024-09-12T23:55:47.004 INFO:tasks.workunit.client.0.smithi136.stdout: client.44675 expiration=0.000000 addr=v1:172.21.15.136:0/4256479254 cookie= 2024-09-12T23:55:47.006 INFO:tasks.workunit.client.0.smithi136.stdout:lock: mylock 2024-09-12T23:55:47.006 INFO:tasks.workunit.client.0.smithi136.stdout: lock_type: shared 2024-09-12T23:55:47.006 INFO:tasks.workunit.client.0.smithi136.stdout: tag: 2024-09-12T23:55:47.006 INFO:tasks.workunit.client.0.smithi136.stdout: lockers: 2024-09-12T23:55:47.006 INFO:tasks.workunit.client.0.smithi136.stdout: client.44675 expiration=0.000000 addr=v1:172.21.15.136:0/4256479254 cookie= 2024-09-12T23:55:47.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:46 smithi136 bash[75164]: cluster 2024-09-12T23:55:44.988003+0000 mgr.y (mgr.24661) 559 : cluster [DBG] pgmap v1095: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:47.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:46 smithi136 bash[75164]: cluster 2024-09-12T23:55:45.916444+0000 mon.a (mon.0) 1719 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:55:47.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:46 smithi136 bash[75164]: cluster 2024-09-12T23:55:45.916510+0000 mon.a (mon.0) 1720 : cluster [INF] Cluster is now healthy 2024-09-12T23:55:47.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:46 smithi136 bash[75164]: cluster 2024-09-12T23:55:45.936309+0000 mon.a (mon.0) 1721 : cluster [DBG] osdmap e976: 8 total, 8 up, 8 in 2024-09-12T23:55:47.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:46 smithi136 bash[75164]: audit 2024-09-12T23:55:45.941048+0000 mon.c (mon.2) 41 : audit [INF] from='client.? 172.21.15.136:0/2080813635' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:47.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:46 smithi136 bash[75164]: audit 2024-09-12T23:55:45.942469+0000 mon.a (mon.0) 1722 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:47.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:46 smithi136 bash[76371]: cluster 2024-09-12T23:55:44.988003+0000 mgr.y (mgr.24661) 559 : cluster [DBG] pgmap v1095: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:47.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:46 smithi136 bash[76371]: cluster 2024-09-12T23:55:45.916444+0000 mon.a (mon.0) 1719 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:55:47.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:46 smithi136 bash[76371]: cluster 2024-09-12T23:55:45.916510+0000 mon.a (mon.0) 1720 : cluster [INF] Cluster is now healthy 2024-09-12T23:55:47.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:46 smithi136 bash[76371]: cluster 2024-09-12T23:55:45.936309+0000 mon.a (mon.0) 1721 : cluster [DBG] osdmap e976: 8 total, 8 up, 8 in 2024-09-12T23:55:47.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:46 smithi136 bash[76371]: audit 2024-09-12T23:55:45.941048+0000 mon.c (mon.2) 41 : audit [INF] from='client.? 172.21.15.136:0/2080813635' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:47.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:46 smithi136 bash[76371]: audit 2024-09-12T23:55:45.942469+0000 mon.a (mon.0) 1722 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:47.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:46 smithi181 bash[67587]: cluster 2024-09-12T23:55:44.988003+0000 mgr.y (mgr.24661) 559 : cluster [DBG] pgmap v1095: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:47.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:46 smithi181 bash[67587]: cluster 2024-09-12T23:55:45.916444+0000 mon.a (mon.0) 1719 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:55:47.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:46 smithi181 bash[67587]: cluster 2024-09-12T23:55:45.916510+0000 mon.a (mon.0) 1720 : cluster [INF] Cluster is now healthy 2024-09-12T23:55:47.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:46 smithi181 bash[67587]: cluster 2024-09-12T23:55:45.936309+0000 mon.a (mon.0) 1721 : cluster [DBG] osdmap e976: 8 total, 8 up, 8 in 2024-09-12T23:55:47.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:46 smithi181 bash[67587]: audit 2024-09-12T23:55:45.941048+0000 mon.c (mon.2) 41 : audit [INF] from='client.? 172.21.15.136:0/2080813635' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:47.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:46 smithi181 bash[67587]: audit 2024-09-12T23:55:45.942469+0000 mon.a (mon.0) 1722 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:47.966 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] ClsLock.TestMultiLocking (2974 ms) 2024-09-12T23:55:47.967 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] ClsLock.TestMeta 2024-09-12T23:55:48.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:47 smithi136 bash[75164]: audit 2024-09-12T23:55:46.946259+0000 mon.a (mon.0) 1723 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:55:48.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:47 smithi136 bash[75164]: cluster 2024-09-12T23:55:46.946640+0000 mon.a (mon.0) 1724 : cluster [DBG] osdmap e977: 8 total, 8 up, 8 in 2024-09-12T23:55:48.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:47 smithi136 bash[76371]: audit 2024-09-12T23:55:46.946259+0000 mon.a (mon.0) 1723 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:55:48.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:47 smithi136 bash[76371]: cluster 2024-09-12T23:55:46.946640+0000 mon.a (mon.0) 1724 : cluster [DBG] osdmap e977: 8 total, 8 up, 8 in 2024-09-12T23:55:48.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:47 smithi181 bash[67587]: audit 2024-09-12T23:55:46.946259+0000 mon.a (mon.0) 1723 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:55:48.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:47 smithi181 bash[67587]: cluster 2024-09-12T23:55:46.946640+0000 mon.a (mon.0) 1724 : cluster [DBG] osdmap e977: 8 total, 8 up, 8 in 2024-09-12T23:55:49.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:48 smithi136 bash[75164]: cluster 2024-09-12T23:55:46.988482+0000 mgr.y (mgr.24661) 560 : cluster [DBG] pgmap v1098: 137 pgs: 9 creating+peering, 23 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:49.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:48 smithi136 bash[75164]: cluster 2024-09-12T23:55:47.957261+0000 mon.a (mon.0) 1725 : cluster [DBG] osdmap e978: 8 total, 8 up, 8 in 2024-09-12T23:55:49.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:48 smithi136 bash[76371]: cluster 2024-09-12T23:55:46.988482+0000 mgr.y (mgr.24661) 560 : cluster [DBG] pgmap v1098: 137 pgs: 9 creating+peering, 23 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:49.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:48 smithi136 bash[76371]: cluster 2024-09-12T23:55:47.957261+0000 mon.a (mon.0) 1725 : cluster [DBG] osdmap e978: 8 total, 8 up, 8 in 2024-09-12T23:55:49.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:48 smithi181 bash[67587]: cluster 2024-09-12T23:55:46.988482+0000 mgr.y (mgr.24661) 560 : cluster [DBG] pgmap v1098: 137 pgs: 9 creating+peering, 23 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:49.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:48 smithi181 bash[67587]: cluster 2024-09-12T23:55:47.957261+0000 mon.a (mon.0) 1725 : cluster [DBG] osdmap e978: 8 total, 8 up, 8 in 2024-09-12T23:55:49.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:55:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:55:49] "GET /metrics HTTP/1.1" 200 36448 "" "Prometheus/2.43.0" 2024-09-12T23:55:50.035 INFO:tasks.workunit.client.0.smithi136.stdout:lock: mylock 2024-09-12T23:55:50.035 INFO:tasks.workunit.client.0.smithi136.stdout: lock_type: shared 2024-09-12T23:55:50.035 INFO:tasks.workunit.client.0.smithi136.stdout: tag: 2024-09-12T23:55:50.035 INFO:tasks.workunit.client.0.smithi136.stdout: lockers: 2024-09-12T23:55:50.035 INFO:tasks.workunit.client.0.smithi136.stdout: client.44687 expiration=0.000000 addr=v1:172.21.15.136:0/2338279420 cookie= 2024-09-12T23:55:50.039 INFO:tasks.workunit.client.0.smithi136.stdout:lock: mylock 2024-09-12T23:55:50.039 INFO:tasks.workunit.client.0.smithi136.stdout: lock_type: exclusive 2024-09-12T23:55:50.039 INFO:tasks.workunit.client.0.smithi136.stdout: tag: new_tag 2024-09-12T23:55:50.039 INFO:tasks.workunit.client.0.smithi136.stdout: lockers: 2024-09-12T23:55:50.039 INFO:tasks.workunit.client.0.smithi136.stdout: client.44681 expiration=0.000000 addr=v1:172.21.15.136:0/1948035254 cookie= 2024-09-12T23:55:50.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:49 smithi136 bash[75164]: cluster 2024-09-12T23:55:48.963633+0000 mon.a (mon.0) 1726 : cluster [DBG] osdmap e979: 8 total, 8 up, 8 in 2024-09-12T23:55:50.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:49 smithi136 bash[75164]: audit 2024-09-12T23:55:48.967664+0000 mon.c (mon.2) 42 : audit [INF] from='client.? 172.21.15.136:0/1948035254' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:50.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:49 smithi136 bash[75164]: audit 2024-09-12T23:55:48.972011+0000 mon.a (mon.0) 1727 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:50.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:49 smithi136 bash[75164]: audit 2024-09-12T23:55:49.964421+0000 mon.a (mon.0) 1728 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:55:50.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:49 smithi136 bash[75164]: cluster 2024-09-12T23:55:49.964995+0000 mon.a (mon.0) 1729 : cluster [DBG] osdmap e980: 8 total, 8 up, 8 in 2024-09-12T23:55:50.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:50 smithi136 bash[76371]: cluster 2024-09-12T23:55:48.963633+0000 mon.a (mon.0) 1726 : cluster [DBG] osdmap e979: 8 total, 8 up, 8 in 2024-09-12T23:55:50.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:50 smithi136 bash[76371]: audit 2024-09-12T23:55:48.967664+0000 mon.c (mon.2) 42 : audit [INF] from='client.? 172.21.15.136:0/1948035254' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:50.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:50 smithi136 bash[76371]: audit 2024-09-12T23:55:48.972011+0000 mon.a (mon.0) 1727 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:50.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:50 smithi136 bash[76371]: audit 2024-09-12T23:55:49.964421+0000 mon.a (mon.0) 1728 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:55:50.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:50 smithi136 bash[76371]: cluster 2024-09-12T23:55:49.964995+0000 mon.a (mon.0) 1729 : cluster [DBG] osdmap e980: 8 total, 8 up, 8 in 2024-09-12T23:55:50.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:50 smithi181 bash[67587]: cluster 2024-09-12T23:55:48.963633+0000 mon.a (mon.0) 1726 : cluster [DBG] osdmap e979: 8 total, 8 up, 8 in 2024-09-12T23:55:50.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:50 smithi181 bash[67587]: audit 2024-09-12T23:55:48.967664+0000 mon.c (mon.2) 42 : audit [INF] from='client.? 172.21.15.136:0/1948035254' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:50.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:50 smithi181 bash[67587]: audit 2024-09-12T23:55:48.972011+0000 mon.a (mon.0) 1727 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:50.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:50 smithi181 bash[67587]: audit 2024-09-12T23:55:49.964421+0000 mon.a (mon.0) 1728 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:55:50.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:50 smithi181 bash[67587]: cluster 2024-09-12T23:55:49.964995+0000 mon.a (mon.0) 1729 : cluster [DBG] osdmap e980: 8 total, 8 up, 8 in 2024-09-12T23:55:50.969 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] ClsLock.TestMeta (3002 ms) 2024-09-12T23:55:50.969 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] ClsLock.TestCookie 2024-09-12T23:55:51.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:51 smithi136 bash[75164]: cluster 2024-09-12T23:55:48.989161+0000 mgr.y (mgr.24661) 561 : cluster [DBG] pgmap v1101: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:51.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:51 smithi136 bash[75164]: cluster 2024-09-12T23:55:49.974883+0000 mon.a (mon.0) 1730 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:55:51.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:51 smithi136 bash[75164]: cluster 2024-09-12T23:55:50.963823+0000 mon.a (mon.0) 1731 : cluster [DBG] osdmap e981: 8 total, 8 up, 8 in 2024-09-12T23:55:51.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:51 smithi136 bash[76371]: cluster 2024-09-12T23:55:48.989161+0000 mgr.y (mgr.24661) 561 : cluster [DBG] pgmap v1101: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:51.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:51 smithi136 bash[76371]: cluster 2024-09-12T23:55:49.974883+0000 mon.a (mon.0) 1730 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:55:51.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:51 smithi136 bash[76371]: cluster 2024-09-12T23:55:50.963823+0000 mon.a (mon.0) 1731 : cluster [DBG] osdmap e981: 8 total, 8 up, 8 in 2024-09-12T23:55:51.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:51 smithi181 bash[67587]: cluster 2024-09-12T23:55:48.989161+0000 mgr.y (mgr.24661) 561 : cluster [DBG] pgmap v1101: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:51.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:51 smithi181 bash[67587]: cluster 2024-09-12T23:55:49.974883+0000 mon.a (mon.0) 1730 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:55:51.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:51 smithi181 bash[67587]: cluster 2024-09-12T23:55:50.963823+0000 mon.a (mon.0) 1731 : cluster [DBG] osdmap e981: 8 total, 8 up, 8 in 2024-09-12T23:55:52.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:55:52 smithi181 bash[95505]: ts=2024-09-12T23:55:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:55:52.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:52 smithi181 bash[67587]: cluster 2024-09-12T23:55:50.989601+0000 mgr.y (mgr.24661) 562 : cluster [DBG] pgmap v1104: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:52.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:52 smithi181 bash[67587]: cluster 2024-09-12T23:55:51.957541+0000 mon.a (mon.0) 1732 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:55:52.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:52 smithi181 bash[67587]: cluster 2024-09-12T23:55:51.957613+0000 mon.a (mon.0) 1733 : cluster [INF] Cluster is now healthy 2024-09-12T23:55:52.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:52 smithi181 bash[67587]: cluster 2024-09-12T23:55:51.974508+0000 mon.a (mon.0) 1734 : cluster [DBG] osdmap e982: 8 total, 8 up, 8 in 2024-09-12T23:55:52.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:52 smithi181 bash[67587]: audit 2024-09-12T23:55:51.981803+0000 mon.c (mon.2) 43 : audit [INF] from='client.? 172.21.15.136:0/2193283391' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:52.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:52 smithi181 bash[67587]: audit 2024-09-12T23:55:51.984601+0000 mon.a (mon.0) 1735 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:52.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:52 smithi136 bash[75164]: cluster 2024-09-12T23:55:50.989601+0000 mgr.y (mgr.24661) 562 : cluster [DBG] pgmap v1104: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:52.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:52 smithi136 bash[75164]: cluster 2024-09-12T23:55:51.957541+0000 mon.a (mon.0) 1732 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:55:52.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:52 smithi136 bash[75164]: cluster 2024-09-12T23:55:51.957613+0000 mon.a (mon.0) 1733 : cluster [INF] Cluster is now healthy 2024-09-12T23:55:52.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:52 smithi136 bash[75164]: cluster 2024-09-12T23:55:51.974508+0000 mon.a (mon.0) 1734 : cluster [DBG] osdmap e982: 8 total, 8 up, 8 in 2024-09-12T23:55:52.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:52 smithi136 bash[75164]: audit 2024-09-12T23:55:51.981803+0000 mon.c (mon.2) 43 : audit [INF] from='client.? 172.21.15.136:0/2193283391' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:52.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:52 smithi136 bash[75164]: audit 2024-09-12T23:55:51.984601+0000 mon.a (mon.0) 1735 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:52.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:52 smithi136 bash[76371]: cluster 2024-09-12T23:55:50.989601+0000 mgr.y (mgr.24661) 562 : cluster [DBG] pgmap v1104: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:52.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:52 smithi136 bash[76371]: cluster 2024-09-12T23:55:51.957541+0000 mon.a (mon.0) 1732 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:55:52.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:52 smithi136 bash[76371]: cluster 2024-09-12T23:55:51.957613+0000 mon.a (mon.0) 1733 : cluster [INF] Cluster is now healthy 2024-09-12T23:55:52.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:52 smithi136 bash[76371]: cluster 2024-09-12T23:55:51.974508+0000 mon.a (mon.0) 1734 : cluster [DBG] osdmap e982: 8 total, 8 up, 8 in 2024-09-12T23:55:52.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:52 smithi136 bash[76371]: audit 2024-09-12T23:55:51.981803+0000 mon.c (mon.2) 43 : audit [INF] from='client.? 172.21.15.136:0/2193283391' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:52.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:52 smithi136 bash[76371]: audit 2024-09-12T23:55:51.984601+0000 mon.a (mon.0) 1735 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:53.027 INFO:tasks.workunit.client.0.smithi136.stdout:lock: mylock 2024-09-12T23:55:53.027 INFO:tasks.workunit.client.0.smithi136.stdout: lock_type: exclusive 2024-09-12T23:55:53.027 INFO:tasks.workunit.client.0.smithi136.stdout: tag: 2024-09-12T23:55:53.027 INFO:tasks.workunit.client.0.smithi136.stdout: lockers: 2024-09-12T23:55:53.032 INFO:tasks.workunit.client.0.smithi136.stdout:lock: mylock 2024-09-12T23:55:53.032 INFO:tasks.workunit.client.0.smithi136.stdout: lock_type: shared 2024-09-12T23:55:53.032 INFO:tasks.workunit.client.0.smithi136.stdout: tag: 2024-09-12T23:55:53.032 INFO:tasks.workunit.client.0.smithi136.stdout: lockers: 2024-09-12T23:55:53.032 INFO:tasks.workunit.client.0.smithi136.stdout: client.44693 expiration=0.000000 addr=v1:172.21.15.136:0/2193283391 cookie= 2024-09-12T23:55:53.032 INFO:tasks.workunit.client.0.smithi136.stdout: client.44693 expiration=0.000000 addr=v1:172.21.15.136:0/2193283391 cookie=new cookie 2024-09-12T23:55:53.999 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] ClsLock.TestCookie (3031 ms) 2024-09-12T23:55:53.999 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] ClsLock.TestMultipleLocks 2024-09-12T23:55:54.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:53 smithi136 bash[76371]: audit 2024-09-12T23:55:52.977988+0000 mon.a (mon.0) 1736 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:55:54.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:53 smithi136 bash[76371]: cluster 2024-09-12T23:55:52.978304+0000 mon.a (mon.0) 1737 : cluster [DBG] osdmap e983: 8 total, 8 up, 8 in 2024-09-12T23:55:54.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:53 smithi136 bash[76371]: audit 2024-09-12T23:55:53.769357+0000 mon.a (mon.0) 1738 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:55:54.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:53 smithi136 bash[76371]: audit 2024-09-12T23:55:53.771156+0000 mon.a (mon.0) 1739 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:55:54.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:53 smithi136 bash[75164]: audit 2024-09-12T23:55:52.977988+0000 mon.a (mon.0) 1736 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:55:54.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:53 smithi136 bash[75164]: cluster 2024-09-12T23:55:52.978304+0000 mon.a (mon.0) 1737 : cluster [DBG] osdmap e983: 8 total, 8 up, 8 in 2024-09-12T23:55:54.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:53 smithi136 bash[75164]: audit 2024-09-12T23:55:53.769357+0000 mon.a (mon.0) 1738 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:55:54.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:53 smithi136 bash[75164]: audit 2024-09-12T23:55:53.771156+0000 mon.a (mon.0) 1739 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:55:54.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:53 smithi181 bash[67587]: audit 2024-09-12T23:55:52.977988+0000 mon.a (mon.0) 1736 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:55:54.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:53 smithi181 bash[67587]: cluster 2024-09-12T23:55:52.978304+0000 mon.a (mon.0) 1737 : cluster [DBG] osdmap e983: 8 total, 8 up, 8 in 2024-09-12T23:55:54.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:53 smithi181 bash[67587]: audit 2024-09-12T23:55:53.769357+0000 mon.a (mon.0) 1738 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:55:54.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:53 smithi181 bash[67587]: audit 2024-09-12T23:55:53.771156+0000 mon.a (mon.0) 1739 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:55:55.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:54 smithi136 bash[76371]: cluster 2024-09-12T23:55:52.990760+0000 mgr.y (mgr.24661) 563 : cluster [DBG] pgmap v1107: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:55.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:54 smithi136 bash[76371]: cluster 2024-09-12T23:55:53.996217+0000 mon.a (mon.0) 1740 : cluster [DBG] osdmap e984: 8 total, 8 up, 8 in 2024-09-12T23:55:55.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:54 smithi136 bash[75164]: cluster 2024-09-12T23:55:52.990760+0000 mgr.y (mgr.24661) 563 : cluster [DBG] pgmap v1107: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:55.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:54 smithi136 bash[75164]: cluster 2024-09-12T23:55:53.996217+0000 mon.a (mon.0) 1740 : cluster [DBG] osdmap e984: 8 total, 8 up, 8 in 2024-09-12T23:55:55.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:54 smithi181 bash[67587]: cluster 2024-09-12T23:55:52.990760+0000 mgr.y (mgr.24661) 563 : cluster [DBG] pgmap v1107: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:55.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:54 smithi181 bash[67587]: cluster 2024-09-12T23:55:53.996217+0000 mon.a (mon.0) 1740 : cluster [DBG] osdmap e984: 8 total, 8 up, 8 in 2024-09-12T23:55:56.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:56 smithi136 bash[75164]: cluster 2024-09-12T23:55:54.991520+0000 mgr.y (mgr.24661) 564 : cluster [DBG] pgmap v1109: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:56.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:56 smithi136 bash[75164]: cluster 2024-09-12T23:55:55.006560+0000 mon.a (mon.0) 1741 : cluster [DBG] osdmap e985: 8 total, 8 up, 8 in 2024-09-12T23:55:56.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:56 smithi136 bash[75164]: audit 2024-09-12T23:55:55.009353+0000 mon.c (mon.2) 44 : audit [INF] from='client.? 172.21.15.136:0/3667861514' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:56.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:56 smithi136 bash[75164]: audit 2024-09-12T23:55:55.010869+0000 mon.a (mon.0) 1742 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:56.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:56 smithi136 bash[76371]: cluster 2024-09-12T23:55:54.991520+0000 mgr.y (mgr.24661) 564 : cluster [DBG] pgmap v1109: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:56.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:56 smithi136 bash[76371]: cluster 2024-09-12T23:55:55.006560+0000 mon.a (mon.0) 1741 : cluster [DBG] osdmap e985: 8 total, 8 up, 8 in 2024-09-12T23:55:56.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:56 smithi136 bash[76371]: audit 2024-09-12T23:55:55.009353+0000 mon.c (mon.2) 44 : audit [INF] from='client.? 172.21.15.136:0/3667861514' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:56.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:56 smithi136 bash[76371]: audit 2024-09-12T23:55:55.010869+0000 mon.a (mon.0) 1742 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:56.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:55 smithi181 bash[67587]: cluster 2024-09-12T23:55:54.991520+0000 mgr.y (mgr.24661) 564 : cluster [DBG] pgmap v1109: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:56.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:55 smithi181 bash[67587]: cluster 2024-09-12T23:55:55.006560+0000 mon.a (mon.0) 1741 : cluster [DBG] osdmap e985: 8 total, 8 up, 8 in 2024-09-12T23:55:56.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:55 smithi181 bash[67587]: audit 2024-09-12T23:55:55.009353+0000 mon.c (mon.2) 44 : audit [INF] from='client.? 172.21.15.136:0/3667861514' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:56.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:55 smithi181 bash[67587]: audit 2024-09-12T23:55:55.010869+0000 mon.a (mon.0) 1742 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:57.035 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] ClsLock.TestMultipleLocks (3035 ms) 2024-09-12T23:55:57.035 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] ClsLock.TestLockDuration 2024-09-12T23:55:57.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:57 smithi181 bash[67587]: audit 2024-09-12T23:55:56.018738+0000 mon.a (mon.0) 1743 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:55:57.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:57 smithi181 bash[67587]: cluster 2024-09-12T23:55:56.019311+0000 mon.a (mon.0) 1744 : cluster [DBG] osdmap e986: 8 total, 8 up, 8 in 2024-09-12T23:55:57.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:57 smithi136 bash[75164]: audit 2024-09-12T23:55:56.018738+0000 mon.a (mon.0) 1743 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:55:57.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:57 smithi136 bash[75164]: cluster 2024-09-12T23:55:56.019311+0000 mon.a (mon.0) 1744 : cluster [DBG] osdmap e986: 8 total, 8 up, 8 in 2024-09-12T23:55:57.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:57 smithi136 bash[76371]: audit 2024-09-12T23:55:56.018738+0000 mon.a (mon.0) 1743 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:55:57.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:57 smithi136 bash[76371]: cluster 2024-09-12T23:55:56.019311+0000 mon.a (mon.0) 1744 : cluster [DBG] osdmap e986: 8 total, 8 up, 8 in 2024-09-12T23:55:58.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:58 smithi181 bash[67587]: cluster 2024-09-12T23:55:56.992684+0000 mgr.y (mgr.24661) 565 : cluster [DBG] pgmap v1112: 137 pgs: 9 creating+peering, 12 unknown, 116 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:58.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:58 smithi181 bash[67587]: cluster 2024-09-12T23:55:57.028115+0000 mon.a (mon.0) 1745 : cluster [DBG] osdmap e987: 8 total, 8 up, 8 in 2024-09-12T23:55:58.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:58 smithi136 bash[75164]: cluster 2024-09-12T23:55:56.992684+0000 mgr.y (mgr.24661) 565 : cluster [DBG] pgmap v1112: 137 pgs: 9 creating+peering, 12 unknown, 116 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:58.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:58 smithi136 bash[75164]: cluster 2024-09-12T23:55:57.028115+0000 mon.a (mon.0) 1745 : cluster [DBG] osdmap e987: 8 total, 8 up, 8 in 2024-09-12T23:55:58.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:58 smithi136 bash[76371]: cluster 2024-09-12T23:55:56.992684+0000 mgr.y (mgr.24661) 565 : cluster [DBG] pgmap v1112: 137 pgs: 9 creating+peering, 12 unknown, 116 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:55:58.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:58 smithi136 bash[76371]: cluster 2024-09-12T23:55:57.028115+0000 mon.a (mon.0) 1745 : cluster [DBG] osdmap e987: 8 total, 8 up, 8 in 2024-09-12T23:55:59.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:59 smithi181 bash[67587]: cluster 2024-09-12T23:55:58.036595+0000 mon.a (mon.0) 1746 : cluster [DBG] osdmap e988: 8 total, 8 up, 8 in 2024-09-12T23:55:59.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:59 smithi181 bash[67587]: audit 2024-09-12T23:55:58.039980+0000 mon.c (mon.2) 45 : audit [INF] from='client.? 172.21.15.136:0/2504418163' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:59.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:55:59 smithi181 bash[67587]: audit 2024-09-12T23:55:58.041174+0000 mon.a (mon.0) 1747 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:59.412 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:59 smithi136 bash[75164]: cluster 2024-09-12T23:55:58.036595+0000 mon.a (mon.0) 1746 : cluster [DBG] osdmap e988: 8 total, 8 up, 8 in 2024-09-12T23:55:59.412 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:59 smithi136 bash[75164]: audit 2024-09-12T23:55:58.039980+0000 mon.c (mon.2) 45 : audit [INF] from='client.? 172.21.15.136:0/2504418163' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:59.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:55:59 smithi136 bash[75164]: audit 2024-09-12T23:55:58.041174+0000 mon.a (mon.0) 1747 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:59.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:59 smithi136 bash[76371]: cluster 2024-09-12T23:55:58.036595+0000 mon.a (mon.0) 1746 : cluster [DBG] osdmap e988: 8 total, 8 up, 8 in 2024-09-12T23:55:59.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:59 smithi136 bash[76371]: audit 2024-09-12T23:55:58.039980+0000 mon.c (mon.2) 45 : audit [INF] from='client.? 172.21.15.136:0/2504418163' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:59.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:55:59 smithi136 bash[76371]: audit 2024-09-12T23:55:58.041174+0000 mon.a (mon.0) 1747 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:55:59.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:55:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:55:59] "GET /metrics HTTP/1.1" 200 36450 "" "Prometheus/2.43.0" 2024-09-12T23:56:00.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:00 smithi181 bash[67587]: cluster 2024-09-12T23:55:58.993674+0000 mgr.y (mgr.24661) 566 : cluster [DBG] pgmap v1115: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:56:00.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:00 smithi181 bash[67587]: cluster 2024-09-12T23:55:59.029553+0000 mon.a (mon.0) 1748 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:56:00.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:00 smithi181 bash[67587]: audit 2024-09-12T23:55:59.050400+0000 mon.a (mon.0) 1749 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:56:00.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:00 smithi181 bash[67587]: cluster 2024-09-12T23:55:59.051236+0000 mon.a (mon.0) 1750 : cluster [DBG] osdmap e989: 8 total, 8 up, 8 in 2024-09-12T23:56:00.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:00 smithi136 bash[75164]: cluster 2024-09-12T23:55:58.993674+0000 mgr.y (mgr.24661) 566 : cluster [DBG] pgmap v1115: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:56:00.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:00 smithi136 bash[75164]: cluster 2024-09-12T23:55:59.029553+0000 mon.a (mon.0) 1748 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:56:00.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:00 smithi136 bash[75164]: audit 2024-09-12T23:55:59.050400+0000 mon.a (mon.0) 1749 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:56:00.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:00 smithi136 bash[75164]: cluster 2024-09-12T23:55:59.051236+0000 mon.a (mon.0) 1750 : cluster [DBG] osdmap e989: 8 total, 8 up, 8 in 2024-09-12T23:56:00.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:00 smithi136 bash[76371]: cluster 2024-09-12T23:55:58.993674+0000 mgr.y (mgr.24661) 566 : cluster [DBG] pgmap v1115: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:56:00.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:00 smithi136 bash[76371]: cluster 2024-09-12T23:55:59.029553+0000 mon.a (mon.0) 1748 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:56:00.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:00 smithi136 bash[76371]: audit 2024-09-12T23:55:59.050400+0000 mon.a (mon.0) 1749 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:56:00.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:00 smithi136 bash[76371]: cluster 2024-09-12T23:55:59.051236+0000 mon.a (mon.0) 1750 : cluster [DBG] osdmap e989: 8 total, 8 up, 8 in 2024-09-12T23:56:01.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:01 smithi181 bash[67587]: cluster 2024-09-12T23:56:00.055138+0000 mon.a (mon.0) 1751 : cluster [DBG] osdmap e990: 8 total, 8 up, 8 in 2024-09-12T23:56:01.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:01 smithi136 bash[75164]: cluster 2024-09-12T23:56:00.055138+0000 mon.a (mon.0) 1751 : cluster [DBG] osdmap e990: 8 total, 8 up, 8 in 2024-09-12T23:56:01.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:01 smithi136 bash[76371]: cluster 2024-09-12T23:56:00.055138+0000 mon.a (mon.0) 1751 : cluster [DBG] osdmap e990: 8 total, 8 up, 8 in 2024-09-12T23:56:02.314 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:02 smithi181 bash[67587]: cluster 2024-09-12T23:56:00.994634+0000 mgr.y (mgr.24661) 567 : cluster [DBG] pgmap v1118: 137 pgs: 3 creating+peering, 19 unknown, 115 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-12T23:56:02.314 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:02 smithi181 bash[67587]: cluster 2024-09-12T23:56:01.049185+0000 mon.a (mon.0) 1752 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:56:02.314 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:02 smithi181 bash[67587]: cluster 2024-09-12T23:56:01.049253+0000 mon.a (mon.0) 1753 : cluster [INF] Cluster is now healthy 2024-09-12T23:56:02.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:02 smithi136 bash[75164]: cluster 2024-09-12T23:56:00.994634+0000 mgr.y (mgr.24661) 567 : cluster [DBG] pgmap v1118: 137 pgs: 3 creating+peering, 19 unknown, 115 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-12T23:56:02.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:02 smithi136 bash[75164]: cluster 2024-09-12T23:56:01.049185+0000 mon.a (mon.0) 1752 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:56:02.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:02 smithi136 bash[75164]: cluster 2024-09-12T23:56:01.049253+0000 mon.a (mon.0) 1753 : cluster [INF] Cluster is now healthy 2024-09-12T23:56:02.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:02 smithi136 bash[76371]: cluster 2024-09-12T23:56:00.994634+0000 mgr.y (mgr.24661) 567 : cluster [DBG] pgmap v1118: 137 pgs: 3 creating+peering, 19 unknown, 115 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-12T23:56:02.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:02 smithi136 bash[76371]: cluster 2024-09-12T23:56:01.049185+0000 mon.a (mon.0) 1752 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:56:02.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:02 smithi136 bash[76371]: cluster 2024-09-12T23:56:01.049253+0000 mon.a (mon.0) 1753 : cluster [INF] Cluster is now healthy 2024-09-12T23:56:02.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:56:02 smithi181 bash[95505]: ts=2024-09-12T23:56:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:56:04.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:04 smithi181 bash[67587]: cluster 2024-09-12T23:56:02.995839+0000 mgr.y (mgr.24661) 568 : cluster [DBG] pgmap v1119: 137 pgs: 3 creating+peering, 134 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-12T23:56:04.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:04 smithi136 bash[75164]: cluster 2024-09-12T23:56:02.995839+0000 mgr.y (mgr.24661) 568 : cluster [DBG] pgmap v1119: 137 pgs: 3 creating+peering, 134 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-12T23:56:04.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:04 smithi136 bash[76371]: cluster 2024-09-12T23:56:02.995839+0000 mgr.y (mgr.24661) 568 : cluster [DBG] pgmap v1119: 137 pgs: 3 creating+peering, 134 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-12T23:56:05.085 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] ClsLock.TestLockDuration (8049 ms) 2024-09-12T23:56:05.085 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] ClsLock.TestAssertLocked 2024-09-12T23:56:06.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:06 smithi181 bash[67587]: cluster 2024-09-12T23:56:04.996895+0000 mgr.y (mgr.24661) 569 : cluster [DBG] pgmap v1120: 137 pgs: 137 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-12T23:56:06.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:06 smithi181 bash[67587]: cluster 2024-09-12T23:56:05.078708+0000 mon.a (mon.0) 1754 : cluster [DBG] osdmap e991: 8 total, 8 up, 8 in 2024-09-12T23:56:06.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:06 smithi136 bash[76371]: cluster 2024-09-12T23:56:04.996895+0000 mgr.y (mgr.24661) 569 : cluster [DBG] pgmap v1120: 137 pgs: 137 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-12T23:56:06.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:06 smithi136 bash[76371]: cluster 2024-09-12T23:56:05.078708+0000 mon.a (mon.0) 1754 : cluster [DBG] osdmap e991: 8 total, 8 up, 8 in 2024-09-12T23:56:06.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:06 smithi136 bash[75164]: cluster 2024-09-12T23:56:04.996895+0000 mgr.y (mgr.24661) 569 : cluster [DBG] pgmap v1120: 137 pgs: 137 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-12T23:56:06.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:06 smithi136 bash[75164]: cluster 2024-09-12T23:56:05.078708+0000 mon.a (mon.0) 1754 : cluster [DBG] osdmap e991: 8 total, 8 up, 8 in 2024-09-12T23:56:07.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:07 smithi181 bash[67587]: cluster 2024-09-12T23:56:06.093452+0000 mon.a (mon.0) 1755 : cluster [DBG] osdmap e992: 8 total, 8 up, 8 in 2024-09-12T23:56:07.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:07 smithi181 bash[67587]: audit 2024-09-12T23:56:06.098910+0000 mon.c (mon.2) 46 : audit [INF] from='client.? 172.21.15.136:0/3304747957' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:56:07.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:07 smithi181 bash[67587]: audit 2024-09-12T23:56:06.099748+0000 mon.a (mon.0) 1756 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:56:07.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:07 smithi136 bash[76371]: cluster 2024-09-12T23:56:06.093452+0000 mon.a (mon.0) 1755 : cluster [DBG] osdmap e992: 8 total, 8 up, 8 in 2024-09-12T23:56:07.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:07 smithi136 bash[76371]: audit 2024-09-12T23:56:06.098910+0000 mon.c (mon.2) 46 : audit [INF] from='client.? 172.21.15.136:0/3304747957' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:56:07.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:07 smithi136 bash[76371]: audit 2024-09-12T23:56:06.099748+0000 mon.a (mon.0) 1756 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:56:07.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:07 smithi136 bash[75164]: cluster 2024-09-12T23:56:06.093452+0000 mon.a (mon.0) 1755 : cluster [DBG] osdmap e992: 8 total, 8 up, 8 in 2024-09-12T23:56:07.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:07 smithi136 bash[75164]: audit 2024-09-12T23:56:06.098910+0000 mon.c (mon.2) 46 : audit [INF] from='client.? 172.21.15.136:0/3304747957' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:56:07.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:07 smithi136 bash[75164]: audit 2024-09-12T23:56:06.099748+0000 mon.a (mon.0) 1756 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:56:08.121 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] ClsLock.TestAssertLocked (3036 ms) 2024-09-12T23:56:08.122 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] ClsLock.TestSetCookie 2024-09-12T23:56:08.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:08 smithi181 bash[67587]: cluster 2024-09-12T23:56:06.997943+0000 mgr.y (mgr.24661) 570 : cluster [DBG] pgmap v1123: 137 pgs: 18 creating+peering, 14 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:56:08.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:08 smithi181 bash[67587]: cluster 2024-09-12T23:56:07.085839+0000 mon.a (mon.0) 1757 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:56:08.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:08 smithi181 bash[67587]: audit 2024-09-12T23:56:07.104815+0000 mon.a (mon.0) 1758 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:56:08.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:08 smithi181 bash[67587]: cluster 2024-09-12T23:56:07.105291+0000 mon.a (mon.0) 1759 : cluster [DBG] osdmap e993: 8 total, 8 up, 8 in 2024-09-12T23:56:08.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:08 smithi136 bash[75164]: cluster 2024-09-12T23:56:06.997943+0000 mgr.y (mgr.24661) 570 : cluster [DBG] pgmap v1123: 137 pgs: 18 creating+peering, 14 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:56:08.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:08 smithi136 bash[75164]: cluster 2024-09-12T23:56:07.085839+0000 mon.a (mon.0) 1757 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:56:08.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:08 smithi136 bash[75164]: audit 2024-09-12T23:56:07.104815+0000 mon.a (mon.0) 1758 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:56:08.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:08 smithi136 bash[75164]: cluster 2024-09-12T23:56:07.105291+0000 mon.a (mon.0) 1759 : cluster [DBG] osdmap e993: 8 total, 8 up, 8 in 2024-09-12T23:56:08.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:08 smithi136 bash[76371]: cluster 2024-09-12T23:56:06.997943+0000 mgr.y (mgr.24661) 570 : cluster [DBG] pgmap v1123: 137 pgs: 18 creating+peering, 14 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:56:08.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:08 smithi136 bash[76371]: cluster 2024-09-12T23:56:07.085839+0000 mon.a (mon.0) 1757 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:56:08.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:08 smithi136 bash[76371]: audit 2024-09-12T23:56:07.104815+0000 mon.a (mon.0) 1758 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:56:08.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:08 smithi136 bash[76371]: cluster 2024-09-12T23:56:07.105291+0000 mon.a (mon.0) 1759 : cluster [DBG] osdmap e993: 8 total, 8 up, 8 in 2024-09-12T23:56:09.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:09 smithi136 bash[75164]: cluster 2024-09-12T23:56:08.115174+0000 mon.a (mon.0) 1760 : cluster [DBG] osdmap e994: 8 total, 8 up, 8 in 2024-09-12T23:56:09.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:09 smithi136 bash[75164]: audit 2024-09-12T23:56:08.770801+0000 mon.a (mon.0) 1761 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:56:09.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:09 smithi136 bash[75164]: audit 2024-09-12T23:56:08.772567+0000 mon.a (mon.0) 1762 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:56:09.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:09 smithi136 bash[76371]: cluster 2024-09-12T23:56:08.115174+0000 mon.a (mon.0) 1760 : cluster [DBG] osdmap e994: 8 total, 8 up, 8 in 2024-09-12T23:56:09.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:09 smithi136 bash[76371]: audit 2024-09-12T23:56:08.770801+0000 mon.a (mon.0) 1761 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:56:09.414 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:09 smithi136 bash[76371]: audit 2024-09-12T23:56:08.772567+0000 mon.a (mon.0) 1762 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:56:09.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:09 smithi181 bash[67587]: cluster 2024-09-12T23:56:08.115174+0000 mon.a (mon.0) 1760 : cluster [DBG] osdmap e994: 8 total, 8 up, 8 in 2024-09-12T23:56:09.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:09 smithi181 bash[67587]: audit 2024-09-12T23:56:08.770801+0000 mon.a (mon.0) 1761 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:56:09.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:09 smithi181 bash[67587]: audit 2024-09-12T23:56:08.772567+0000 mon.a (mon.0) 1762 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:56:09.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:56:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:56:09] "GET /metrics HTTP/1.1" 200 34084 "" "Prometheus/2.43.0" 2024-09-12T23:56:10.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:10 smithi136 bash[76371]: cluster 2024-09-12T23:56:08.998690+0000 mgr.y (mgr.24661) 571 : cluster [DBG] pgmap v1126: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:56:10.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:10 smithi136 bash[76371]: cluster 2024-09-12T23:56:09.119901+0000 mon.a (mon.0) 1763 : cluster [DBG] osdmap e995: 8 total, 8 up, 8 in 2024-09-12T23:56:10.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:10 smithi136 bash[76371]: audit 2024-09-12T23:56:09.138154+0000 mon.c (mon.2) 47 : audit [INF] from='client.? 172.21.15.136:0/1735815692' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:56:10.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:10 smithi136 bash[76371]: audit 2024-09-12T23:56:09.139934+0000 mon.a (mon.0) 1764 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:56:10.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:10 smithi136 bash[76371]: cluster 2024-09-12T23:56:09.764539+0000 mon.a (mon.0) 1765 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:56:10.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:10 smithi136 bash[76371]: cluster 2024-09-12T23:56:09.764591+0000 mon.a (mon.0) 1766 : cluster [INF] Cluster is now healthy 2024-09-12T23:56:10.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:10 smithi136 bash[76371]: audit 2024-09-12T23:56:10.122341+0000 mon.a (mon.0) 1767 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:56:10.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:10 smithi136 bash[76371]: cluster 2024-09-12T23:56:10.123021+0000 mon.a (mon.0) 1768 : cluster [DBG] osdmap e996: 8 total, 8 up, 8 in 2024-09-12T23:56:10.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:10 smithi136 bash[75164]: cluster 2024-09-12T23:56:08.998690+0000 mgr.y (mgr.24661) 571 : cluster [DBG] pgmap v1126: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:56:10.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:10 smithi136 bash[75164]: cluster 2024-09-12T23:56:09.119901+0000 mon.a (mon.0) 1763 : cluster [DBG] osdmap e995: 8 total, 8 up, 8 in 2024-09-12T23:56:10.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:10 smithi136 bash[75164]: audit 2024-09-12T23:56:09.138154+0000 mon.c (mon.2) 47 : audit [INF] from='client.? 172.21.15.136:0/1735815692' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:56:10.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:10 smithi136 bash[75164]: audit 2024-09-12T23:56:09.139934+0000 mon.a (mon.0) 1764 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:56:10.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:10 smithi136 bash[75164]: cluster 2024-09-12T23:56:09.764539+0000 mon.a (mon.0) 1765 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:56:10.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:10 smithi136 bash[75164]: cluster 2024-09-12T23:56:09.764591+0000 mon.a (mon.0) 1766 : cluster [INF] Cluster is now healthy 2024-09-12T23:56:10.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:10 smithi136 bash[75164]: audit 2024-09-12T23:56:10.122341+0000 mon.a (mon.0) 1767 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:56:10.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:10 smithi136 bash[75164]: cluster 2024-09-12T23:56:10.123021+0000 mon.a (mon.0) 1768 : cluster [DBG] osdmap e996: 8 total, 8 up, 8 in 2024-09-12T23:56:10.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:10 smithi181 bash[67587]: cluster 2024-09-12T23:56:08.998690+0000 mgr.y (mgr.24661) 571 : cluster [DBG] pgmap v1126: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:56:10.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:10 smithi181 bash[67587]: cluster 2024-09-12T23:56:09.119901+0000 mon.a (mon.0) 1763 : cluster [DBG] osdmap e995: 8 total, 8 up, 8 in 2024-09-12T23:56:10.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:10 smithi181 bash[67587]: audit 2024-09-12T23:56:09.138154+0000 mon.c (mon.2) 47 : audit [INF] from='client.? 172.21.15.136:0/1735815692' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:56:10.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:10 smithi181 bash[67587]: audit 2024-09-12T23:56:09.139934+0000 mon.a (mon.0) 1764 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:56:10.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:10 smithi181 bash[67587]: cluster 2024-09-12T23:56:09.764539+0000 mon.a (mon.0) 1765 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:56:10.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:10 smithi181 bash[67587]: cluster 2024-09-12T23:56:09.764591+0000 mon.a (mon.0) 1766 : cluster [INF] Cluster is now healthy 2024-09-12T23:56:10.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:10 smithi181 bash[67587]: audit 2024-09-12T23:56:10.122341+0000 mon.a (mon.0) 1767 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:56:10.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:10 smithi181 bash[67587]: cluster 2024-09-12T23:56:10.123021+0000 mon.a (mon.0) 1768 : cluster [DBG] osdmap e996: 8 total, 8 up, 8 in 2024-09-12T23:56:11.154 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] ClsLock.TestSetCookie (3033 ms) 2024-09-12T23:56:11.155 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] ClsLock.TestRenew 2024-09-12T23:56:12.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:12 smithi136 bash[75164]: cluster 2024-09-12T23:56:10.999592+0000 mgr.y (mgr.24661) 572 : cluster [DBG] pgmap v1129: 137 pgs: 5 creating+peering, 18 unknown, 114 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:56:12.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:12 smithi136 bash[75164]: cluster 2024-09-12T23:56:11.152824+0000 mon.a (mon.0) 1769 : cluster [DBG] osdmap e997: 8 total, 8 up, 8 in 2024-09-12T23:56:12.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:12 smithi136 bash[76371]: cluster 2024-09-12T23:56:10.999592+0000 mgr.y (mgr.24661) 572 : cluster [DBG] pgmap v1129: 137 pgs: 5 creating+peering, 18 unknown, 114 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:56:12.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:12 smithi136 bash[76371]: cluster 2024-09-12T23:56:11.152824+0000 mon.a (mon.0) 1769 : cluster [DBG] osdmap e997: 8 total, 8 up, 8 in 2024-09-12T23:56:12.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:12 smithi181 bash[67587]: cluster 2024-09-12T23:56:10.999592+0000 mgr.y (mgr.24661) 572 : cluster [DBG] pgmap v1129: 137 pgs: 5 creating+peering, 18 unknown, 114 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:56:12.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:12 smithi181 bash[67587]: cluster 2024-09-12T23:56:11.152824+0000 mon.a (mon.0) 1769 : cluster [DBG] osdmap e997: 8 total, 8 up, 8 in 2024-09-12T23:56:12.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:56:12 smithi181 bash[95505]: ts=2024-09-12T23:56:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:56:13.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:13 smithi136 bash[75164]: cluster 2024-09-12T23:56:12.166914+0000 mon.a (mon.0) 1770 : cluster [DBG] osdmap e998: 8 total, 8 up, 8 in 2024-09-12T23:56:13.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:13 smithi136 bash[75164]: audit 2024-09-12T23:56:12.173446+0000 mon.c (mon.2) 48 : audit [INF] from='client.? 172.21.15.136:0/2891069671' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:56:13.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:13 smithi136 bash[75164]: audit 2024-09-12T23:56:12.179444+0000 mon.a (mon.0) 1771 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:56:13.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:13 smithi136 bash[76371]: cluster 2024-09-12T23:56:12.166914+0000 mon.a (mon.0) 1770 : cluster [DBG] osdmap e998: 8 total, 8 up, 8 in 2024-09-12T23:56:13.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:13 smithi136 bash[76371]: audit 2024-09-12T23:56:12.173446+0000 mon.c (mon.2) 48 : audit [INF] from='client.? 172.21.15.136:0/2891069671' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:56:13.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:13 smithi136 bash[76371]: audit 2024-09-12T23:56:12.179444+0000 mon.a (mon.0) 1771 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:56:13.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:13 smithi181 bash[67587]: cluster 2024-09-12T23:56:12.166914+0000 mon.a (mon.0) 1770 : cluster [DBG] osdmap e998: 8 total, 8 up, 8 in 2024-09-12T23:56:13.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:13 smithi181 bash[67587]: audit 2024-09-12T23:56:12.173446+0000 mon.c (mon.2) 48 : audit [INF] from='client.? 172.21.15.136:0/2891069671' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:56:13.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:13 smithi181 bash[67587]: audit 2024-09-12T23:56:12.179444+0000 mon.a (mon.0) 1771 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:56:14.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:14 smithi136 bash[75164]: cluster 2024-09-12T23:56:13.000478+0000 mgr.y (mgr.24661) 573 : cluster [DBG] pgmap v1132: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:56:14.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:14 smithi136 bash[75164]: cluster 2024-09-12T23:56:13.157394+0000 mon.a (mon.0) 1772 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:56:14.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:14 smithi136 bash[75164]: audit 2024-09-12T23:56:13.178543+0000 mon.a (mon.0) 1773 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:56:14.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:14 smithi136 bash[75164]: cluster 2024-09-12T23:56:13.178780+0000 mon.a (mon.0) 1774 : cluster [DBG] osdmap e999: 8 total, 8 up, 8 in 2024-09-12T23:56:14.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:14 smithi136 bash[76371]: cluster 2024-09-12T23:56:13.000478+0000 mgr.y (mgr.24661) 573 : cluster [DBG] pgmap v1132: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:56:14.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:14 smithi136 bash[76371]: cluster 2024-09-12T23:56:13.157394+0000 mon.a (mon.0) 1772 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:56:14.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:14 smithi136 bash[76371]: audit 2024-09-12T23:56:13.178543+0000 mon.a (mon.0) 1773 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:56:14.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:14 smithi136 bash[76371]: cluster 2024-09-12T23:56:13.178780+0000 mon.a (mon.0) 1774 : cluster [DBG] osdmap e999: 8 total, 8 up, 8 in 2024-09-12T23:56:14.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:14 smithi181 bash[67587]: cluster 2024-09-12T23:56:13.000478+0000 mgr.y (mgr.24661) 573 : cluster [DBG] pgmap v1132: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:56:14.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:14 smithi181 bash[67587]: cluster 2024-09-12T23:56:13.157394+0000 mon.a (mon.0) 1772 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:56:14.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:14 smithi181 bash[67587]: audit 2024-09-12T23:56:13.178543+0000 mon.a (mon.0) 1773 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:56:14.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:14 smithi181 bash[67587]: cluster 2024-09-12T23:56:13.178780+0000 mon.a (mon.0) 1774 : cluster [DBG] osdmap e999: 8 total, 8 up, 8 in 2024-09-12T23:56:15.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:15 smithi136 bash[75164]: cluster 2024-09-12T23:56:14.182102+0000 mon.a (mon.0) 1775 : cluster [DBG] osdmap e1000: 8 total, 8 up, 8 in 2024-09-12T23:56:15.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:15 smithi136 bash[76371]: cluster 2024-09-12T23:56:14.182102+0000 mon.a (mon.0) 1775 : cluster [DBG] osdmap e1000: 8 total, 8 up, 8 in 2024-09-12T23:56:15.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:15 smithi181 bash[67587]: cluster 2024-09-12T23:56:14.182102+0000 mon.a (mon.0) 1775 : cluster [DBG] osdmap e1000: 8 total, 8 up, 8 in 2024-09-12T23:56:16.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:16 smithi136 bash[75164]: cluster 2024-09-12T23:56:15.001372+0000 mgr.y (mgr.24661) 574 : cluster [DBG] pgmap v1135: 137 pgs: 4 creating+peering, 22 unknown, 111 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-12T23:56:16.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:16 smithi136 bash[75164]: cluster 2024-09-12T23:56:15.187400+0000 mon.a (mon.0) 1776 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:56:16.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:16 smithi136 bash[75164]: cluster 2024-09-12T23:56:15.187442+0000 mon.a (mon.0) 1777 : cluster [INF] Cluster is now healthy 2024-09-12T23:56:16.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:16 smithi136 bash[76371]: cluster 2024-09-12T23:56:15.001372+0000 mgr.y (mgr.24661) 574 : cluster [DBG] pgmap v1135: 137 pgs: 4 creating+peering, 22 unknown, 111 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-12T23:56:16.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:16 smithi136 bash[76371]: cluster 2024-09-12T23:56:15.187400+0000 mon.a (mon.0) 1776 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:56:16.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:16 smithi136 bash[76371]: cluster 2024-09-12T23:56:15.187442+0000 mon.a (mon.0) 1777 : cluster [INF] Cluster is now healthy 2024-09-12T23:56:16.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:16 smithi181 bash[67587]: cluster 2024-09-12T23:56:15.001372+0000 mgr.y (mgr.24661) 574 : cluster [DBG] pgmap v1135: 137 pgs: 4 creating+peering, 22 unknown, 111 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-12T23:56:16.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:16 smithi181 bash[67587]: cluster 2024-09-12T23:56:15.187400+0000 mon.a (mon.0) 1776 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:56:16.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:16 smithi181 bash[67587]: cluster 2024-09-12T23:56:15.187442+0000 mon.a (mon.0) 1777 : cluster [INF] Cluster is now healthy 2024-09-12T23:56:18.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:18 smithi136 bash[75164]: cluster 2024-09-12T23:56:17.002825+0000 mgr.y (mgr.24661) 575 : cluster [DBG] pgmap v1136: 137 pgs: 4 creating+peering, 133 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-12T23:56:18.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:18 smithi136 bash[76371]: cluster 2024-09-12T23:56:17.002825+0000 mgr.y (mgr.24661) 575 : cluster [DBG] pgmap v1136: 137 pgs: 4 creating+peering, 133 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-12T23:56:18.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:18 smithi181 bash[67587]: cluster 2024-09-12T23:56:17.002825+0000 mgr.y (mgr.24661) 575 : cluster [DBG] pgmap v1136: 137 pgs: 4 creating+peering, 133 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-12T23:56:19.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:56:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:56:19] "GET /metrics HTTP/1.1" 200 34084 "" "Prometheus/2.43.0" 2024-09-12T23:56:20.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:20 smithi136 bash[75164]: cluster 2024-09-12T23:56:19.003899+0000 mgr.y (mgr.24661) 576 : cluster [DBG] pgmap v1137: 137 pgs: 137 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 149 B/s rd, 0 B/s wr, 0 op/s 2024-09-12T23:56:20.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:20 smithi136 bash[76371]: cluster 2024-09-12T23:56:19.003899+0000 mgr.y (mgr.24661) 576 : cluster [DBG] pgmap v1137: 137 pgs: 137 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 149 B/s rd, 0 B/s wr, 0 op/s 2024-09-12T23:56:20.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:20 smithi181 bash[67587]: cluster 2024-09-12T23:56:19.003899+0000 mgr.y (mgr.24661) 576 : cluster [DBG] pgmap v1137: 137 pgs: 137 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 149 B/s rd, 0 B/s wr, 0 op/s 2024-09-12T23:56:22.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:22 smithi136 bash[75164]: cluster 2024-09-12T23:56:21.005023+0000 mgr.y (mgr.24661) 577 : cluster [DBG] pgmap v1138: 137 pgs: 137 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 127 B/s rd, 0 B/s wr, 0 op/s 2024-09-12T23:56:22.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:22 smithi136 bash[76371]: cluster 2024-09-12T23:56:21.005023+0000 mgr.y (mgr.24661) 577 : cluster [DBG] pgmap v1138: 137 pgs: 137 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 127 B/s rd, 0 B/s wr, 0 op/s 2024-09-12T23:56:22.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:22 smithi181 bash[67587]: cluster 2024-09-12T23:56:21.005023+0000 mgr.y (mgr.24661) 577 : cluster [DBG] pgmap v1138: 137 pgs: 137 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 127 B/s rd, 0 B/s wr, 0 op/s 2024-09-12T23:56:22.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:56:22 smithi181 bash[95505]: ts=2024-09-12T23:56:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:56:25.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:24 smithi181 bash[67587]: cluster 2024-09-12T23:56:23.006156+0000 mgr.y (mgr.24661) 578 : cluster [DBG] pgmap v1139: 137 pgs: 137 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 104 B/s rd, 0 B/s wr, 0 op/s 2024-09-12T23:56:25.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:24 smithi181 bash[67587]: audit 2024-09-12T23:56:23.769139+0000 mon.a (mon.0) 1778 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:56:25.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:24 smithi181 bash[67587]: audit 2024-09-12T23:56:23.790962+0000 mon.a (mon.0) 1779 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:56:25.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:24 smithi136 bash[76371]: cluster 2024-09-12T23:56:23.006156+0000 mgr.y (mgr.24661) 578 : cluster [DBG] pgmap v1139: 137 pgs: 137 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 104 B/s rd, 0 B/s wr, 0 op/s 2024-09-12T23:56:25.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:24 smithi136 bash[76371]: audit 2024-09-12T23:56:23.769139+0000 mon.a (mon.0) 1778 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:56:25.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:24 smithi136 bash[76371]: audit 2024-09-12T23:56:23.790962+0000 mon.a (mon.0) 1779 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:56:25.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:24 smithi136 bash[75164]: cluster 2024-09-12T23:56:23.006156+0000 mgr.y (mgr.24661) 578 : cluster [DBG] pgmap v1139: 137 pgs: 137 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 104 B/s rd, 0 B/s wr, 0 op/s 2024-09-12T23:56:25.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:24 smithi136 bash[75164]: audit 2024-09-12T23:56:23.769139+0000 mon.a (mon.0) 1778 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:56:25.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:24 smithi136 bash[75164]: audit 2024-09-12T23:56:23.790962+0000 mon.a (mon.0) 1779 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:56:27.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:26 smithi181 bash[67587]: cluster 2024-09-12T23:56:25.007150+0000 mgr.y (mgr.24661) 579 : cluster [DBG] pgmap v1140: 137 pgs: 137 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 283 B/s rd, 0 B/s wr, 0 op/s 2024-09-12T23:56:27.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:26 smithi136 bash[75164]: cluster 2024-09-12T23:56:25.007150+0000 mgr.y (mgr.24661) 579 : cluster [DBG] pgmap v1140: 137 pgs: 137 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 283 B/s rd, 0 B/s wr, 0 op/s 2024-09-12T23:56:27.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:26 smithi136 bash[76371]: cluster 2024-09-12T23:56:25.007150+0000 mgr.y (mgr.24661) 579 : cluster [DBG] pgmap v1140: 137 pgs: 137 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 283 B/s rd, 0 B/s wr, 0 op/s 2024-09-12T23:56:29.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:28 smithi181 bash[67587]: cluster 2024-09-12T23:56:27.008510+0000 mgr.y (mgr.24661) 580 : cluster [DBG] pgmap v1141: 137 pgs: 137 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 B/s wr, 0 op/s 2024-09-12T23:56:29.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:28 smithi136 bash[76371]: cluster 2024-09-12T23:56:27.008510+0000 mgr.y (mgr.24661) 580 : cluster [DBG] pgmap v1141: 137 pgs: 137 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 B/s wr, 0 op/s 2024-09-12T23:56:29.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:28 smithi136 bash[75164]: cluster 2024-09-12T23:56:27.008510+0000 mgr.y (mgr.24661) 580 : cluster [DBG] pgmap v1141: 137 pgs: 137 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 B/s wr, 0 op/s 2024-09-12T23:56:29.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:56:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:56:29] "GET /metrics HTTP/1.1" 200 36455 "" "Prometheus/2.43.0" 2024-09-12T23:56:31.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:30 smithi181 bash[67587]: cluster 2024-09-12T23:56:29.009599+0000 mgr.y (mgr.24661) 581 : cluster [DBG] pgmap v1142: 137 pgs: 137 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 B/s wr, 0 op/s 2024-09-12T23:56:31.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:30 smithi136 bash[75164]: cluster 2024-09-12T23:56:29.009599+0000 mgr.y (mgr.24661) 581 : cluster [DBG] pgmap v1142: 137 pgs: 137 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 B/s wr, 0 op/s 2024-09-12T23:56:31.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:30 smithi136 bash[76371]: cluster 2024-09-12T23:56:29.009599+0000 mgr.y (mgr.24661) 581 : cluster [DBG] pgmap v1142: 137 pgs: 137 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 B/s wr, 0 op/s 2024-09-12T23:56:31.792 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] ClsLock.TestRenew (20638 ms) 2024-09-12T23:56:31.792 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] ClsLock.TestExclusiveEphemeralBasic 2024-09-12T23:56:32.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:31 smithi181 bash[67587]: audit 2024-09-12T23:56:31.317937+0000 mon.a (mon.0) 1780 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:56:32.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:31 smithi181 bash[67587]: audit 2024-09-12T23:56:31.688559+0000 mon.a (mon.0) 1781 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:56:32.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:31 smithi181 bash[67587]: audit 2024-09-12T23:56:31.693456+0000 mon.a (mon.0) 1782 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:56:32.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:31 smithi136 bash[75164]: audit 2024-09-12T23:56:31.317937+0000 mon.a (mon.0) 1780 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:56:32.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:31 smithi136 bash[75164]: audit 2024-09-12T23:56:31.688559+0000 mon.a (mon.0) 1781 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:56:32.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:31 smithi136 bash[75164]: audit 2024-09-12T23:56:31.693456+0000 mon.a (mon.0) 1782 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:56:32.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:31 smithi136 bash[76371]: audit 2024-09-12T23:56:31.317937+0000 mon.a (mon.0) 1780 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:56:32.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:31 smithi136 bash[76371]: audit 2024-09-12T23:56:31.688559+0000 mon.a (mon.0) 1781 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:56:32.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:31 smithi136 bash[76371]: audit 2024-09-12T23:56:31.693456+0000 mon.a (mon.0) 1782 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:56:32.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:56:32 smithi181 bash[95505]: ts=2024-09-12T23:56:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:56:33.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:32 smithi181 bash[67587]: cluster 2024-09-12T23:56:31.010766+0000 mgr.y (mgr.24661) 582 : cluster [DBG] pgmap v1143: 137 pgs: 137 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 B/s wr, 0 op/s 2024-09-12T23:56:33.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:32 smithi181 bash[67587]: cluster 2024-09-12T23:56:31.791409+0000 mon.a (mon.0) 1783 : cluster [DBG] osdmap e1001: 8 total, 8 up, 8 in 2024-09-12T23:56:33.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:32 smithi181 bash[67587]: audit 2024-09-12T23:56:32.248511+0000 mon.a (mon.0) 1784 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:56:33.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:32 smithi181 bash[67587]: audit 2024-09-12T23:56:32.250007+0000 mon.a (mon.0) 1785 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:56:33.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:32 smithi181 bash[67587]: audit 2024-09-12T23:56:32.260094+0000 mon.a (mon.0) 1786 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:56:33.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:32 smithi136 bash[75164]: cluster 2024-09-12T23:56:31.010766+0000 mgr.y (mgr.24661) 582 : cluster [DBG] pgmap v1143: 137 pgs: 137 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 B/s wr, 0 op/s 2024-09-12T23:56:33.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:32 smithi136 bash[75164]: cluster 2024-09-12T23:56:31.791409+0000 mon.a (mon.0) 1783 : cluster [DBG] osdmap e1001: 8 total, 8 up, 8 in 2024-09-12T23:56:33.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:32 smithi136 bash[75164]: audit 2024-09-12T23:56:32.248511+0000 mon.a (mon.0) 1784 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:56:33.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:32 smithi136 bash[75164]: audit 2024-09-12T23:56:32.250007+0000 mon.a (mon.0) 1785 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:56:33.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:32 smithi136 bash[75164]: audit 2024-09-12T23:56:32.260094+0000 mon.a (mon.0) 1786 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:56:33.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:32 smithi136 bash[76371]: cluster 2024-09-12T23:56:31.010766+0000 mgr.y (mgr.24661) 582 : cluster [DBG] pgmap v1143: 137 pgs: 137 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 B/s wr, 0 op/s 2024-09-12T23:56:33.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:32 smithi136 bash[76371]: cluster 2024-09-12T23:56:31.791409+0000 mon.a (mon.0) 1783 : cluster [DBG] osdmap e1001: 8 total, 8 up, 8 in 2024-09-12T23:56:33.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:32 smithi136 bash[76371]: audit 2024-09-12T23:56:32.248511+0000 mon.a (mon.0) 1784 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:56:33.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:32 smithi136 bash[76371]: audit 2024-09-12T23:56:32.250007+0000 mon.a (mon.0) 1785 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:56:33.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:32 smithi136 bash[76371]: audit 2024-09-12T23:56:32.260094+0000 mon.a (mon.0) 1786 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:56:34.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:33 smithi181 bash[67587]: cluster 2024-09-12T23:56:32.806721+0000 mon.a (mon.0) 1787 : cluster [DBG] osdmap e1002: 8 total, 8 up, 8 in 2024-09-12T23:56:34.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:33 smithi181 bash[67587]: audit 2024-09-12T23:56:32.811528+0000 mon.a (mon.0) 1788 : audit [INF] from='client.? 172.21.15.136:0/264360188' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-9","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:56:34.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:33 smithi136 bash[75164]: cluster 2024-09-12T23:56:32.806721+0000 mon.a (mon.0) 1787 : cluster [DBG] osdmap e1002: 8 total, 8 up, 8 in 2024-09-12T23:56:34.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:33 smithi136 bash[75164]: audit 2024-09-12T23:56:32.811528+0000 mon.a (mon.0) 1788 : audit [INF] from='client.? 172.21.15.136:0/264360188' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-9","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:56:34.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:33 smithi136 bash[76371]: cluster 2024-09-12T23:56:32.806721+0000 mon.a (mon.0) 1787 : cluster [DBG] osdmap e1002: 8 total, 8 up, 8 in 2024-09-12T23:56:34.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:33 smithi136 bash[76371]: audit 2024-09-12T23:56:32.811528+0000 mon.a (mon.0) 1788 : audit [INF] from='client.? 172.21.15.136:0/264360188' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-9","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:56:35.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:34 smithi181 bash[67587]: cluster 2024-09-12T23:56:33.011570+0000 mgr.y (mgr.24661) 583 : cluster [DBG] pgmap v1146: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:56:35.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:34 smithi181 bash[67587]: cluster 2024-09-12T23:56:33.799161+0000 mon.a (mon.0) 1789 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:56:35.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:34 smithi181 bash[67587]: audit 2024-09-12T23:56:33.818717+0000 mon.a (mon.0) 1790 : audit [INF] from='client.? 172.21.15.136:0/264360188' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-9","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:56:35.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:34 smithi181 bash[67587]: cluster 2024-09-12T23:56:33.819162+0000 mon.a (mon.0) 1791 : cluster [DBG] osdmap e1003: 8 total, 8 up, 8 in 2024-09-12T23:56:35.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:34 smithi136 bash[75164]: cluster 2024-09-12T23:56:33.011570+0000 mgr.y (mgr.24661) 583 : cluster [DBG] pgmap v1146: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:56:35.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:34 smithi136 bash[75164]: cluster 2024-09-12T23:56:33.799161+0000 mon.a (mon.0) 1789 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:56:35.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:34 smithi136 bash[75164]: audit 2024-09-12T23:56:33.818717+0000 mon.a (mon.0) 1790 : audit [INF] from='client.? 172.21.15.136:0/264360188' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-9","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:56:35.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:34 smithi136 bash[75164]: cluster 2024-09-12T23:56:33.819162+0000 mon.a (mon.0) 1791 : cluster [DBG] osdmap e1003: 8 total, 8 up, 8 in 2024-09-12T23:56:35.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:34 smithi136 bash[76371]: cluster 2024-09-12T23:56:33.011570+0000 mgr.y (mgr.24661) 583 : cluster [DBG] pgmap v1146: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:56:35.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:34 smithi136 bash[76371]: cluster 2024-09-12T23:56:33.799161+0000 mon.a (mon.0) 1789 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:56:35.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:34 smithi136 bash[76371]: audit 2024-09-12T23:56:33.818717+0000 mon.a (mon.0) 1790 : audit [INF] from='client.? 172.21.15.136:0/264360188' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-9","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:56:35.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:34 smithi136 bash[76371]: cluster 2024-09-12T23:56:33.819162+0000 mon.a (mon.0) 1791 : cluster [DBG] osdmap e1003: 8 total, 8 up, 8 in 2024-09-12T23:56:36.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:35 smithi181 bash[67587]: cluster 2024-09-12T23:56:34.818833+0000 mon.a (mon.0) 1792 : cluster [DBG] osdmap e1004: 8 total, 8 up, 8 in 2024-09-12T23:56:36.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:35 smithi136 bash[75164]: cluster 2024-09-12T23:56:34.818833+0000 mon.a (mon.0) 1792 : cluster [DBG] osdmap e1004: 8 total, 8 up, 8 in 2024-09-12T23:56:36.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:35 smithi136 bash[76371]: cluster 2024-09-12T23:56:34.818833+0000 mon.a (mon.0) 1792 : cluster [DBG] osdmap e1004: 8 total, 8 up, 8 in 2024-09-12T23:56:37.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:36 smithi181 bash[67587]: cluster 2024-09-12T23:56:35.012393+0000 mgr.y (mgr.24661) 584 : cluster [DBG] pgmap v1149: 137 pgs: 1 creating+peering, 26 unknown, 110 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:56:37.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:36 smithi181 bash[67587]: cluster 2024-09-12T23:56:35.824026+0000 mon.a (mon.0) 1793 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:56:37.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:36 smithi181 bash[67587]: cluster 2024-09-12T23:56:35.824074+0000 mon.a (mon.0) 1794 : cluster [INF] Cluster is now healthy 2024-09-12T23:56:37.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:36 smithi136 bash[76371]: cluster 2024-09-12T23:56:35.012393+0000 mgr.y (mgr.24661) 584 : cluster [DBG] pgmap v1149: 137 pgs: 1 creating+peering, 26 unknown, 110 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:56:37.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:36 smithi136 bash[76371]: cluster 2024-09-12T23:56:35.824026+0000 mon.a (mon.0) 1793 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:56:37.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:36 smithi136 bash[76371]: cluster 2024-09-12T23:56:35.824074+0000 mon.a (mon.0) 1794 : cluster [INF] Cluster is now healthy 2024-09-12T23:56:37.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:36 smithi136 bash[75164]: cluster 2024-09-12T23:56:35.012393+0000 mgr.y (mgr.24661) 584 : cluster [DBG] pgmap v1149: 137 pgs: 1 creating+peering, 26 unknown, 110 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:56:37.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:36 smithi136 bash[75164]: cluster 2024-09-12T23:56:35.824026+0000 mon.a (mon.0) 1793 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:56:37.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:36 smithi136 bash[75164]: cluster 2024-09-12T23:56:35.824074+0000 mon.a (mon.0) 1794 : cluster [INF] Cluster is now healthy 2024-09-12T23:56:38.855 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] ClsLock.TestExclusiveEphemeralBasic (7063 ms) 2024-09-12T23:56:38.855 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] ClsLock.TestExclusiveEphemeralStealEphemeral 2024-09-12T23:56:39.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:38 smithi181 bash[67587]: cluster 2024-09-12T23:56:37.013688+0000 mgr.y (mgr.24661) 585 : cluster [DBG] pgmap v1150: 137 pgs: 1 creating+peering, 3 unknown, 133 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:56:39.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:38 smithi181 bash[67587]: audit 2024-09-12T23:56:38.763128+0000 mon.a (mon.0) 1795 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:56:39.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:38 smithi136 bash[75164]: cluster 2024-09-12T23:56:37.013688+0000 mgr.y (mgr.24661) 585 : cluster [DBG] pgmap v1150: 137 pgs: 1 creating+peering, 3 unknown, 133 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:56:39.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:38 smithi136 bash[75164]: audit 2024-09-12T23:56:38.763128+0000 mon.a (mon.0) 1795 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:56:39.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:38 smithi136 bash[76371]: cluster 2024-09-12T23:56:37.013688+0000 mgr.y (mgr.24661) 585 : cluster [DBG] pgmap v1150: 137 pgs: 1 creating+peering, 3 unknown, 133 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:56:39.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:38 smithi136 bash[76371]: audit 2024-09-12T23:56:38.763128+0000 mon.a (mon.0) 1795 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:56:39.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:56:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:56:39] "GET /metrics HTTP/1.1" 200 36451 "" "Prometheus/2.43.0" 2024-09-12T23:56:40.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:39 smithi136 bash[75164]: cluster 2024-09-12T23:56:38.853952+0000 mon.a (mon.0) 1796 : cluster [DBG] osdmap e1005: 8 total, 8 up, 8 in 2024-09-12T23:56:40.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:39 smithi136 bash[76371]: cluster 2024-09-12T23:56:38.853952+0000 mon.a (mon.0) 1796 : cluster [DBG] osdmap e1005: 8 total, 8 up, 8 in 2024-09-12T23:56:40.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:39 smithi181 bash[67587]: cluster 2024-09-12T23:56:38.853952+0000 mon.a (mon.0) 1796 : cluster [DBG] osdmap e1005: 8 total, 8 up, 8 in 2024-09-12T23:56:41.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:40 smithi136 bash[75164]: cluster 2024-09-12T23:56:39.014373+0000 mgr.y (mgr.24661) 586 : cluster [DBG] pgmap v1152: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:56:41.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:40 smithi136 bash[75164]: cluster 2024-09-12T23:56:39.867596+0000 mon.a (mon.0) 1797 : cluster [DBG] osdmap e1006: 8 total, 8 up, 8 in 2024-09-12T23:56:41.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:40 smithi136 bash[75164]: audit 2024-09-12T23:56:39.870260+0000 mon.a (mon.0) 1798 : audit [INF] from='client.? 172.21.15.136:0/2372297853' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:56:41.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:40 smithi136 bash[76371]: cluster 2024-09-12T23:56:39.014373+0000 mgr.y (mgr.24661) 586 : cluster [DBG] pgmap v1152: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:56:41.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:40 smithi136 bash[76371]: cluster 2024-09-12T23:56:39.867596+0000 mon.a (mon.0) 1797 : cluster [DBG] osdmap e1006: 8 total, 8 up, 8 in 2024-09-12T23:56:41.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:40 smithi136 bash[76371]: audit 2024-09-12T23:56:39.870260+0000 mon.a (mon.0) 1798 : audit [INF] from='client.? 172.21.15.136:0/2372297853' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:56:41.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:40 smithi181 bash[67587]: cluster 2024-09-12T23:56:39.014373+0000 mgr.y (mgr.24661) 586 : cluster [DBG] pgmap v1152: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:56:41.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:40 smithi181 bash[67587]: cluster 2024-09-12T23:56:39.867596+0000 mon.a (mon.0) 1797 : cluster [DBG] osdmap e1006: 8 total, 8 up, 8 in 2024-09-12T23:56:41.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:40 smithi181 bash[67587]: audit 2024-09-12T23:56:39.870260+0000 mon.a (mon.0) 1798 : audit [INF] from='client.? 172.21.15.136:0/2372297853' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:56:42.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:41 smithi136 bash[75164]: audit 2024-09-12T23:56:40.882388+0000 mon.a (mon.0) 1799 : audit [INF] from='client.? 172.21.15.136:0/2372297853' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:56:42.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:41 smithi136 bash[75164]: cluster 2024-09-12T23:56:40.883357+0000 mon.a (mon.0) 1800 : cluster [DBG] osdmap e1007: 8 total, 8 up, 8 in 2024-09-12T23:56:42.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:41 smithi136 bash[76371]: audit 2024-09-12T23:56:40.882388+0000 mon.a (mon.0) 1799 : audit [INF] from='client.? 172.21.15.136:0/2372297853' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:56:42.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:41 smithi136 bash[76371]: cluster 2024-09-12T23:56:40.883357+0000 mon.a (mon.0) 1800 : cluster [DBG] osdmap e1007: 8 total, 8 up, 8 in 2024-09-12T23:56:42.314 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:41 smithi181 bash[67587]: audit 2024-09-12T23:56:40.882388+0000 mon.a (mon.0) 1799 : audit [INF] from='client.? 172.21.15.136:0/2372297853' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:56:42.314 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:41 smithi181 bash[67587]: cluster 2024-09-12T23:56:40.883357+0000 mon.a (mon.0) 1800 : cluster [DBG] osdmap e1007: 8 total, 8 up, 8 in 2024-09-12T23:56:42.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:56:42 smithi181 bash[95505]: ts=2024-09-12T23:56:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:56:43.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:42 smithi136 bash[75164]: cluster 2024-09-12T23:56:41.015042+0000 mgr.y (mgr.24661) 587 : cluster [DBG] pgmap v1155: 137 pgs: 8 creating+peering, 24 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:56:43.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:42 smithi136 bash[75164]: cluster 2024-09-12T23:56:41.880606+0000 mon.a (mon.0) 1801 : cluster [DBG] osdmap e1008: 8 total, 8 up, 8 in 2024-09-12T23:56:43.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:42 smithi136 bash[76371]: cluster 2024-09-12T23:56:41.015042+0000 mgr.y (mgr.24661) 587 : cluster [DBG] pgmap v1155: 137 pgs: 8 creating+peering, 24 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:56:43.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:42 smithi136 bash[76371]: cluster 2024-09-12T23:56:41.880606+0000 mon.a (mon.0) 1801 : cluster [DBG] osdmap e1008: 8 total, 8 up, 8 in 2024-09-12T23:56:43.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:42 smithi181 bash[67587]: cluster 2024-09-12T23:56:41.015042+0000 mgr.y (mgr.24661) 587 : cluster [DBG] pgmap v1155: 137 pgs: 8 creating+peering, 24 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:56:43.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:42 smithi181 bash[67587]: cluster 2024-09-12T23:56:41.880606+0000 mon.a (mon.0) 1801 : cluster [DBG] osdmap e1008: 8 total, 8 up, 8 in 2024-09-12T23:56:45.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:44 smithi136 bash[75164]: cluster 2024-09-12T23:56:43.015892+0000 mgr.y (mgr.24661) 588 : cluster [DBG] pgmap v1157: 137 pgs: 8 creating+peering, 6 unknown, 123 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-12T23:56:45.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:44 smithi136 bash[76371]: cluster 2024-09-12T23:56:43.015892+0000 mgr.y (mgr.24661) 588 : cluster [DBG] pgmap v1157: 137 pgs: 8 creating+peering, 6 unknown, 123 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-12T23:56:45.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:44 smithi181 bash[67587]: cluster 2024-09-12T23:56:43.015892+0000 mgr.y (mgr.24661) 588 : cluster [DBG] pgmap v1157: 137 pgs: 8 creating+peering, 6 unknown, 123 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-12T23:56:46.911 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] ClsLock.TestExclusiveEphemeralStealEphemeral (8056 ms) 2024-09-12T23:56:46.912 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] ClsLock.TestExclusiveEphemeralStealExclusive 2024-09-12T23:56:47.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:46 smithi136 bash[75164]: cluster 2024-09-12T23:56:45.016828+0000 mgr.y (mgr.24661) 589 : cluster [DBG] pgmap v1158: 137 pgs: 8 creating+peering, 129 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-12T23:56:47.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:46 smithi136 bash[76371]: cluster 2024-09-12T23:56:45.016828+0000 mgr.y (mgr.24661) 589 : cluster [DBG] pgmap v1158: 137 pgs: 8 creating+peering, 129 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-12T23:56:47.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:46 smithi181 bash[67587]: cluster 2024-09-12T23:56:45.016828+0000 mgr.y (mgr.24661) 589 : cluster [DBG] pgmap v1158: 137 pgs: 8 creating+peering, 129 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-12T23:56:48.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:47 smithi136 bash[75164]: cluster 2024-09-12T23:56:46.912088+0000 mon.a (mon.0) 1802 : cluster [DBG] osdmap e1009: 8 total, 8 up, 8 in 2024-09-12T23:56:48.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:47 smithi136 bash[76371]: cluster 2024-09-12T23:56:46.912088+0000 mon.a (mon.0) 1802 : cluster [DBG] osdmap e1009: 8 total, 8 up, 8 in 2024-09-12T23:56:48.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:47 smithi181 bash[67587]: cluster 2024-09-12T23:56:46.912088+0000 mon.a (mon.0) 1802 : cluster [DBG] osdmap e1009: 8 total, 8 up, 8 in 2024-09-12T23:56:49.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:48 smithi136 bash[75164]: cluster 2024-09-12T23:56:47.017362+0000 mgr.y (mgr.24661) 590 : cluster [DBG] pgmap v1160: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:56:49.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:48 smithi136 bash[75164]: cluster 2024-09-12T23:56:47.922901+0000 mon.a (mon.0) 1803 : cluster [DBG] osdmap e1010: 8 total, 8 up, 8 in 2024-09-12T23:56:49.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:48 smithi136 bash[75164]: audit 2024-09-12T23:56:47.926597+0000 mon.c (mon.2) 49 : audit [INF] from='client.? 172.21.15.136:0/3073400919' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:56:49.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:48 smithi136 bash[75164]: audit 2024-09-12T23:56:47.927472+0000 mon.a (mon.0) 1804 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:56:49.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:48 smithi136 bash[76371]: cluster 2024-09-12T23:56:47.017362+0000 mgr.y (mgr.24661) 590 : cluster [DBG] pgmap v1160: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:56:49.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:48 smithi136 bash[76371]: cluster 2024-09-12T23:56:47.922901+0000 mon.a (mon.0) 1803 : cluster [DBG] osdmap e1010: 8 total, 8 up, 8 in 2024-09-12T23:56:49.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:48 smithi136 bash[76371]: audit 2024-09-12T23:56:47.926597+0000 mon.c (mon.2) 49 : audit [INF] from='client.? 172.21.15.136:0/3073400919' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:56:49.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:48 smithi136 bash[76371]: audit 2024-09-12T23:56:47.927472+0000 mon.a (mon.0) 1804 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:56:49.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:48 smithi181 bash[67587]: cluster 2024-09-12T23:56:47.017362+0000 mgr.y (mgr.24661) 590 : cluster [DBG] pgmap v1160: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:56:49.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:48 smithi181 bash[67587]: cluster 2024-09-12T23:56:47.922901+0000 mon.a (mon.0) 1803 : cluster [DBG] osdmap e1010: 8 total, 8 up, 8 in 2024-09-12T23:56:49.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:48 smithi181 bash[67587]: audit 2024-09-12T23:56:47.926597+0000 mon.c (mon.2) 49 : audit [INF] from='client.? 172.21.15.136:0/3073400919' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:56:49.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:48 smithi181 bash[67587]: audit 2024-09-12T23:56:47.927472+0000 mon.a (mon.0) 1804 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:56:49.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:56:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:56:49] "GET /metrics HTTP/1.1" 200 36451 "" "Prometheus/2.43.0" 2024-09-12T23:56:50.256 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:49 smithi136 bash[76371]: audit 2024-09-12T23:56:48.925551+0000 mon.a (mon.0) 1805 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:56:50.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:49 smithi136 bash[76371]: cluster 2024-09-12T23:56:48.926004+0000 mon.a (mon.0) 1806 : cluster [DBG] osdmap e1011: 8 total, 8 up, 8 in 2024-09-12T23:56:50.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:49 smithi136 bash[75164]: audit 2024-09-12T23:56:48.925551+0000 mon.a (mon.0) 1805 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:56:50.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:49 smithi136 bash[75164]: cluster 2024-09-12T23:56:48.926004+0000 mon.a (mon.0) 1806 : cluster [DBG] osdmap e1011: 8 total, 8 up, 8 in 2024-09-12T23:56:50.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:49 smithi181 bash[67587]: audit 2024-09-12T23:56:48.925551+0000 mon.a (mon.0) 1805 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-109933-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:56:50.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:49 smithi181 bash[67587]: cluster 2024-09-12T23:56:48.926004+0000 mon.a (mon.0) 1806 : cluster [DBG] osdmap e1011: 8 total, 8 up, 8 in 2024-09-12T23:56:51.256 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:50 smithi136 bash[76371]: cluster 2024-09-12T23:56:49.018026+0000 mgr.y (mgr.24661) 591 : cluster [DBG] pgmap v1163: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:56:51.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:50 smithi136 bash[76371]: cluster 2024-09-12T23:56:49.952108+0000 mon.a (mon.0) 1807 : cluster [DBG] osdmap e1012: 8 total, 8 up, 8 in 2024-09-12T23:56:51.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:50 smithi136 bash[75164]: cluster 2024-09-12T23:56:49.018026+0000 mgr.y (mgr.24661) 591 : cluster [DBG] pgmap v1163: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:56:51.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:50 smithi136 bash[75164]: cluster 2024-09-12T23:56:49.952108+0000 mon.a (mon.0) 1807 : cluster [DBG] osdmap e1012: 8 total, 8 up, 8 in 2024-09-12T23:56:51.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:50 smithi181 bash[67587]: cluster 2024-09-12T23:56:49.018026+0000 mgr.y (mgr.24661) 591 : cluster [DBG] pgmap v1163: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:56:51.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:50 smithi181 bash[67587]: cluster 2024-09-12T23:56:49.952108+0000 mon.a (mon.0) 1807 : cluster [DBG] osdmap e1012: 8 total, 8 up, 8 in 2024-09-12T23:56:52.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:56:52 smithi181 bash[95505]: ts=2024-09-12T23:56:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:56:53.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:52 smithi136 bash[76371]: cluster 2024-09-12T23:56:51.019078+0000 mgr.y (mgr.24661) 592 : cluster [DBG] pgmap v1165: 137 pgs: 5 creating+peering, 10 unknown, 122 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:56:53.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:52 smithi136 bash[75164]: cluster 2024-09-12T23:56:51.019078+0000 mgr.y (mgr.24661) 592 : cluster [DBG] pgmap v1165: 137 pgs: 5 creating+peering, 10 unknown, 122 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:56:53.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:52 smithi181 bash[67587]: cluster 2024-09-12T23:56:51.019078+0000 mgr.y (mgr.24661) 592 : cluster [DBG] pgmap v1165: 137 pgs: 5 creating+peering, 10 unknown, 122 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:56:54.256 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:53 smithi136 bash[76371]: audit 2024-09-12T23:56:53.763696+0000 mon.a (mon.0) 1808 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:56:54.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:53 smithi136 bash[75164]: audit 2024-09-12T23:56:53.763696+0000 mon.a (mon.0) 1808 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:56:54.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:53 smithi181 bash[67587]: audit 2024-09-12T23:56:53.763696+0000 mon.a (mon.0) 1808 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:56:54.991 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] ClsLock.TestExclusiveEphemeralStealExclusive (8079 ms) 2024-09-12T23:56:54.991 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 11 tests from ClsLock (69998 ms total) 2024-09-12T23:56:54.991 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-12T23:56:54.991 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] Global test environment tear-down 2024-09-12T23:56:54.991 INFO:tasks.workunit.client.0.smithi136.stdout:[==========] 11 tests from 1 test suite ran. (69998 ms total) 2024-09-12T23:56:54.991 INFO:tasks.workunit.client.0.smithi136.stdout:[ PASSED ] 11 tests. 2024-09-12T23:56:54.997 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-12T23:56:54.997 DEBUG:teuthology.orchestra.run.smithi136:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-12T23:56:55.053 INFO:tasks.workunit:Running workunit cls/test_cls_log.sh... 2024-09-12T23:56:55.054 DEBUG:teuthology.orchestra.run.smithi136:workunit test cls/test_cls_log.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_log.sh 2024-09-12T23:56:55.115 INFO:tasks.workunit.client.0.smithi136.stdout:Running main() from gmock_main.cc 2024-09-12T23:56:55.115 INFO:tasks.workunit.client.0.smithi136.stdout:[==========] Running 4 tests from 1 test suite. 2024-09-12T23:56:55.115 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] Global test environment set-up. 2024-09-12T23:56:55.116 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 4 tests from cls_log 2024-09-12T23:56:55.116 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] cls_log.test_log_add_same_time 2024-09-12T23:56:55.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:54 smithi136 bash[75164]: cluster 2024-09-12T23:56:53.020218+0000 mgr.y (mgr.24661) 593 : cluster [DBG] pgmap v1166: 137 pgs: 5 creating+peering, 132 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:56:55.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:54 smithi136 bash[76371]: cluster 2024-09-12T23:56:53.020218+0000 mgr.y (mgr.24661) 593 : cluster [DBG] pgmap v1166: 137 pgs: 5 creating+peering, 132 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:56:55.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:54 smithi181 bash[67587]: cluster 2024-09-12T23:56:53.020218+0000 mgr.y (mgr.24661) 593 : cluster [DBG] pgmap v1166: 137 pgs: 5 creating+peering, 132 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:56:56.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:55 smithi136 bash[75164]: cluster 2024-09-12T23:56:54.982653+0000 mon.a (mon.0) 1809 : cluster [DBG] osdmap e1013: 8 total, 8 up, 8 in 2024-09-12T23:56:56.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:55 smithi136 bash[76371]: cluster 2024-09-12T23:56:54.982653+0000 mon.a (mon.0) 1809 : cluster [DBG] osdmap e1013: 8 total, 8 up, 8 in 2024-09-12T23:56:56.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:55 smithi181 bash[67587]: cluster 2024-09-12T23:56:54.982653+0000 mon.a (mon.0) 1809 : cluster [DBG] osdmap e1013: 8 total, 8 up, 8 in 2024-09-12T23:56:57.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:56 smithi136 bash[75164]: cluster 2024-09-12T23:56:55.020875+0000 mgr.y (mgr.24661) 594 : cluster [DBG] pgmap v1168: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:56:57.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:56 smithi136 bash[75164]: cluster 2024-09-12T23:56:56.003789+0000 mon.a (mon.0) 1810 : cluster [DBG] osdmap e1014: 8 total, 8 up, 8 in 2024-09-12T23:56:57.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:56 smithi136 bash[75164]: audit 2024-09-12T23:56:56.014135+0000 mon.c (mon.2) 50 : audit [INF] from='client.? 172.21.15.136:0/1701484149' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110170-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:56:57.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:56 smithi136 bash[75164]: audit 2024-09-12T23:56:56.016188+0000 mon.a (mon.0) 1811 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110170-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:56:57.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:56 smithi136 bash[76371]: cluster 2024-09-12T23:56:55.020875+0000 mgr.y (mgr.24661) 594 : cluster [DBG] pgmap v1168: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:56:57.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:56 smithi136 bash[76371]: cluster 2024-09-12T23:56:56.003789+0000 mon.a (mon.0) 1810 : cluster [DBG] osdmap e1014: 8 total, 8 up, 8 in 2024-09-12T23:56:57.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:56 smithi136 bash[76371]: audit 2024-09-12T23:56:56.014135+0000 mon.c (mon.2) 50 : audit [INF] from='client.? 172.21.15.136:0/1701484149' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110170-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:56:57.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:56 smithi136 bash[76371]: audit 2024-09-12T23:56:56.016188+0000 mon.a (mon.0) 1811 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110170-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:56:57.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:56 smithi181 bash[67587]: cluster 2024-09-12T23:56:55.020875+0000 mgr.y (mgr.24661) 594 : cluster [DBG] pgmap v1168: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:56:57.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:56 smithi181 bash[67587]: cluster 2024-09-12T23:56:56.003789+0000 mon.a (mon.0) 1810 : cluster [DBG] osdmap e1014: 8 total, 8 up, 8 in 2024-09-12T23:56:57.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:56 smithi181 bash[67587]: audit 2024-09-12T23:56:56.014135+0000 mon.c (mon.2) 50 : audit [INF] from='client.? 172.21.15.136:0/1701484149' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110170-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:56:57.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:56 smithi181 bash[67587]: audit 2024-09-12T23:56:56.016188+0000 mon.a (mon.0) 1811 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110170-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:56:58.013 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] cls_log.test_log_add_same_time (2898 ms) 2024-09-12T23:56:58.013 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] cls_log.test_log_add_different_time 2024-09-12T23:56:58.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:58 smithi181 bash[67587]: audit 2024-09-12T23:56:57.009349+0000 mon.a (mon.0) 1812 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110170-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:56:58.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:58 smithi181 bash[67587]: cluster 2024-09-12T23:56:57.009546+0000 mon.a (mon.0) 1813 : cluster [DBG] osdmap e1015: 8 total, 8 up, 8 in 2024-09-12T23:56:58.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:58 smithi181 bash[67587]: cluster 2024-09-12T23:56:57.021398+0000 mgr.y (mgr.24661) 595 : cluster [DBG] pgmap v1171: 137 pgs: 11 creating+peering, 21 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:56:58.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:58 smithi136 bash[75164]: audit 2024-09-12T23:56:57.009349+0000 mon.a (mon.0) 1812 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110170-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:56:58.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:58 smithi136 bash[75164]: cluster 2024-09-12T23:56:57.009546+0000 mon.a (mon.0) 1813 : cluster [DBG] osdmap e1015: 8 total, 8 up, 8 in 2024-09-12T23:56:58.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:58 smithi136 bash[75164]: cluster 2024-09-12T23:56:57.021398+0000 mgr.y (mgr.24661) 595 : cluster [DBG] pgmap v1171: 137 pgs: 11 creating+peering, 21 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:56:58.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:58 smithi136 bash[76371]: audit 2024-09-12T23:56:57.009349+0000 mon.a (mon.0) 1812 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110170-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:56:58.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:58 smithi136 bash[76371]: cluster 2024-09-12T23:56:57.009546+0000 mon.a (mon.0) 1813 : cluster [DBG] osdmap e1015: 8 total, 8 up, 8 in 2024-09-12T23:56:58.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:58 smithi136 bash[76371]: cluster 2024-09-12T23:56:57.021398+0000 mgr.y (mgr.24661) 595 : cluster [DBG] pgmap v1171: 137 pgs: 11 creating+peering, 21 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:56:59.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:56:59 smithi181 bash[67587]: cluster 2024-09-12T23:56:58.011504+0000 mon.a (mon.0) 1814 : cluster [DBG] osdmap e1016: 8 total, 8 up, 8 in 2024-09-12T23:56:59.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:56:59 smithi136 bash[75164]: cluster 2024-09-12T23:56:58.011504+0000 mon.a (mon.0) 1814 : cluster [DBG] osdmap e1016: 8 total, 8 up, 8 in 2024-09-12T23:56:59.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:56:59 smithi136 bash[76371]: cluster 2024-09-12T23:56:58.011504+0000 mon.a (mon.0) 1814 : cluster [DBG] osdmap e1016: 8 total, 8 up, 8 in 2024-09-12T23:56:59.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:56:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:56:59] "GET /metrics HTTP/1.1" 200 36452 "" "Prometheus/2.43.0" 2024-09-12T23:57:00.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:00 smithi181 bash[67587]: cluster 2024-09-12T23:56:59.022186+0000 mgr.y (mgr.24661) 596 : cluster [DBG] pgmap v1174: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:00.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:00 smithi181 bash[67587]: cluster 2024-09-12T23:56:59.027750+0000 mon.a (mon.0) 1815 : cluster [DBG] osdmap e1017: 8 total, 8 up, 8 in 2024-09-12T23:57:00.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:00 smithi181 bash[67587]: audit 2024-09-12T23:56:59.031188+0000 mon.c (mon.2) 51 : audit [INF] from='client.? 172.21.15.136:0/1343447269' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110170-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:00.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:00 smithi181 bash[67587]: audit 2024-09-12T23:56:59.041625+0000 mon.a (mon.0) 1816 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110170-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:00.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:00 smithi181 bash[67587]: cluster 2024-09-12T23:57:00.020063+0000 mon.a (mon.0) 1817 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:57:00.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:00 smithi181 bash[67587]: audit 2024-09-12T23:57:00.031582+0000 mon.a (mon.0) 1818 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110170-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:57:00.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:00 smithi181 bash[67587]: cluster 2024-09-12T23:57:00.031987+0000 mon.a (mon.0) 1819 : cluster [DBG] osdmap e1018: 8 total, 8 up, 8 in 2024-09-12T23:57:00.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:00 smithi136 bash[75164]: cluster 2024-09-12T23:56:59.022186+0000 mgr.y (mgr.24661) 596 : cluster [DBG] pgmap v1174: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:00.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:00 smithi136 bash[75164]: cluster 2024-09-12T23:56:59.027750+0000 mon.a (mon.0) 1815 : cluster [DBG] osdmap e1017: 8 total, 8 up, 8 in 2024-09-12T23:57:00.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:00 smithi136 bash[75164]: audit 2024-09-12T23:56:59.031188+0000 mon.c (mon.2) 51 : audit [INF] from='client.? 172.21.15.136:0/1343447269' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110170-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:00.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:00 smithi136 bash[75164]: audit 2024-09-12T23:56:59.041625+0000 mon.a (mon.0) 1816 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110170-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:00.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:00 smithi136 bash[75164]: cluster 2024-09-12T23:57:00.020063+0000 mon.a (mon.0) 1817 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:57:00.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:00 smithi136 bash[75164]: audit 2024-09-12T23:57:00.031582+0000 mon.a (mon.0) 1818 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110170-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:57:00.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:00 smithi136 bash[75164]: cluster 2024-09-12T23:57:00.031987+0000 mon.a (mon.0) 1819 : cluster [DBG] osdmap e1018: 8 total, 8 up, 8 in 2024-09-12T23:57:00.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:00 smithi136 bash[76371]: cluster 2024-09-12T23:56:59.022186+0000 mgr.y (mgr.24661) 596 : cluster [DBG] pgmap v1174: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:00.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:00 smithi136 bash[76371]: cluster 2024-09-12T23:56:59.027750+0000 mon.a (mon.0) 1815 : cluster [DBG] osdmap e1017: 8 total, 8 up, 8 in 2024-09-12T23:57:00.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:00 smithi136 bash[76371]: audit 2024-09-12T23:56:59.031188+0000 mon.c (mon.2) 51 : audit [INF] from='client.? 172.21.15.136:0/1343447269' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110170-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:00.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:00 smithi136 bash[76371]: audit 2024-09-12T23:56:59.041625+0000 mon.a (mon.0) 1816 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110170-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:00.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:00 smithi136 bash[76371]: cluster 2024-09-12T23:57:00.020063+0000 mon.a (mon.0) 1817 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:57:00.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:00 smithi136 bash[76371]: audit 2024-09-12T23:57:00.031582+0000 mon.a (mon.0) 1818 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110170-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:57:00.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:00 smithi136 bash[76371]: cluster 2024-09-12T23:57:00.031987+0000 mon.a (mon.0) 1819 : cluster [DBG] osdmap e1018: 8 total, 8 up, 8 in 2024-09-12T23:57:01.034 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] cls_log.test_log_add_different_time (3021 ms) 2024-09-12T23:57:01.034 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] cls_log.trim_by_time 2024-09-12T23:57:02.314 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:02 smithi181 bash[67587]: cluster 2024-09-12T23:57:01.023288+0000 mgr.y (mgr.24661) 597 : cluster [DBG] pgmap v1176: 137 pgs: 3 creating+peering, 24 unknown, 110 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:02.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:02 smithi181 bash[67587]: cluster 2024-09-12T23:57:01.032379+0000 mon.a (mon.0) 1820 : cluster [DBG] osdmap e1019: 8 total, 8 up, 8 in 2024-09-12T23:57:02.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:02 smithi181 bash[67587]: cluster 2024-09-12T23:57:01.067239+0000 mon.a (mon.0) 1821 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:57:02.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:02 smithi181 bash[67587]: cluster 2024-09-12T23:57:01.067262+0000 mon.a (mon.0) 1822 : cluster [INF] Cluster is now healthy 2024-09-12T23:57:02.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:02 smithi136 bash[75164]: cluster 2024-09-12T23:57:01.023288+0000 mgr.y (mgr.24661) 597 : cluster [DBG] pgmap v1176: 137 pgs: 3 creating+peering, 24 unknown, 110 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:02.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:02 smithi136 bash[75164]: cluster 2024-09-12T23:57:01.032379+0000 mon.a (mon.0) 1820 : cluster [DBG] osdmap e1019: 8 total, 8 up, 8 in 2024-09-12T23:57:02.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:02 smithi136 bash[75164]: cluster 2024-09-12T23:57:01.067239+0000 mon.a (mon.0) 1821 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:57:02.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:02 smithi136 bash[75164]: cluster 2024-09-12T23:57:01.067262+0000 mon.a (mon.0) 1822 : cluster [INF] Cluster is now healthy 2024-09-12T23:57:02.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:02 smithi136 bash[76371]: cluster 2024-09-12T23:57:01.023288+0000 mgr.y (mgr.24661) 597 : cluster [DBG] pgmap v1176: 137 pgs: 3 creating+peering, 24 unknown, 110 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:02.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:02 smithi136 bash[76371]: cluster 2024-09-12T23:57:01.032379+0000 mon.a (mon.0) 1820 : cluster [DBG] osdmap e1019: 8 total, 8 up, 8 in 2024-09-12T23:57:02.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:02 smithi136 bash[76371]: cluster 2024-09-12T23:57:01.067239+0000 mon.a (mon.0) 1821 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:57:02.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:02 smithi136 bash[76371]: cluster 2024-09-12T23:57:01.067262+0000 mon.a (mon.0) 1822 : cluster [INF] Cluster is now healthy 2024-09-12T23:57:02.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:57:02 smithi181 bash[95505]: ts=2024-09-12T23:57:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:57:03.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:03 smithi181 bash[67587]: cluster 2024-09-12T23:57:02.043878+0000 mon.a (mon.0) 1823 : cluster [DBG] osdmap e1020: 8 total, 8 up, 8 in 2024-09-12T23:57:03.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:03 smithi181 bash[67587]: audit 2024-09-12T23:57:02.046661+0000 mon.a (mon.0) 1824 : audit [INF] from='client.? 172.21.15.136:0/3458624051' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110170-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:03.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:03 smithi136 bash[75164]: cluster 2024-09-12T23:57:02.043878+0000 mon.a (mon.0) 1823 : cluster [DBG] osdmap e1020: 8 total, 8 up, 8 in 2024-09-12T23:57:03.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:03 smithi136 bash[75164]: audit 2024-09-12T23:57:02.046661+0000 mon.a (mon.0) 1824 : audit [INF] from='client.? 172.21.15.136:0/3458624051' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110170-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:03.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:03 smithi136 bash[76371]: cluster 2024-09-12T23:57:02.043878+0000 mon.a (mon.0) 1823 : cluster [DBG] osdmap e1020: 8 total, 8 up, 8 in 2024-09-12T23:57:03.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:03 smithi136 bash[76371]: audit 2024-09-12T23:57:02.046661+0000 mon.a (mon.0) 1824 : audit [INF] from='client.? 172.21.15.136:0/3458624051' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110170-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:04.073 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] cls_log.trim_by_time (3039 ms) 2024-09-12T23:57:04.073 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] cls_log.trim_by_marker 2024-09-12T23:57:04.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:04 smithi181 bash[67587]: cluster 2024-09-12T23:57:03.024173+0000 mgr.y (mgr.24661) 598 : cluster [DBG] pgmap v1179: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:04.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:04 smithi181 bash[67587]: cluster 2024-09-12T23:57:03.038497+0000 mon.a (mon.0) 1825 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:57:04.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:04 smithi181 bash[67587]: audit 2024-09-12T23:57:03.047513+0000 mon.a (mon.0) 1826 : audit [INF] from='client.? 172.21.15.136:0/3458624051' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110170-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:57:04.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:04 smithi181 bash[67587]: cluster 2024-09-12T23:57:03.048342+0000 mon.a (mon.0) 1827 : cluster [DBG] osdmap e1021: 8 total, 8 up, 8 in 2024-09-12T23:57:04.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:04 smithi136 bash[75164]: cluster 2024-09-12T23:57:03.024173+0000 mgr.y (mgr.24661) 598 : cluster [DBG] pgmap v1179: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:04.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:04 smithi136 bash[75164]: cluster 2024-09-12T23:57:03.038497+0000 mon.a (mon.0) 1825 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:57:04.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:04 smithi136 bash[75164]: audit 2024-09-12T23:57:03.047513+0000 mon.a (mon.0) 1826 : audit [INF] from='client.? 172.21.15.136:0/3458624051' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110170-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:57:04.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:04 smithi136 bash[75164]: cluster 2024-09-12T23:57:03.048342+0000 mon.a (mon.0) 1827 : cluster [DBG] osdmap e1021: 8 total, 8 up, 8 in 2024-09-12T23:57:04.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:04 smithi136 bash[76371]: cluster 2024-09-12T23:57:03.024173+0000 mgr.y (mgr.24661) 598 : cluster [DBG] pgmap v1179: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:04.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:04 smithi136 bash[76371]: cluster 2024-09-12T23:57:03.038497+0000 mon.a (mon.0) 1825 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:57:04.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:04 smithi136 bash[76371]: audit 2024-09-12T23:57:03.047513+0000 mon.a (mon.0) 1826 : audit [INF] from='client.? 172.21.15.136:0/3458624051' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110170-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:57:04.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:04 smithi136 bash[76371]: cluster 2024-09-12T23:57:03.048342+0000 mon.a (mon.0) 1827 : cluster [DBG] osdmap e1021: 8 total, 8 up, 8 in 2024-09-12T23:57:05.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:05 smithi181 bash[67587]: cluster 2024-09-12T23:57:04.072648+0000 mon.a (mon.0) 1828 : cluster [DBG] osdmap e1022: 8 total, 8 up, 8 in 2024-09-12T23:57:05.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:05 smithi136 bash[76371]: cluster 2024-09-12T23:57:04.072648+0000 mon.a (mon.0) 1828 : cluster [DBG] osdmap e1022: 8 total, 8 up, 8 in 2024-09-12T23:57:05.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:05 smithi136 bash[75164]: cluster 2024-09-12T23:57:04.072648+0000 mon.a (mon.0) 1828 : cluster [DBG] osdmap e1022: 8 total, 8 up, 8 in 2024-09-12T23:57:06.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:06 smithi136 bash[75164]: cluster 2024-09-12T23:57:05.024935+0000 mgr.y (mgr.24661) 599 : cluster [DBG] pgmap v1182: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:06.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:06 smithi136 bash[75164]: cluster 2024-09-12T23:57:05.066465+0000 mon.a (mon.0) 1829 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:57:06.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:06 smithi136 bash[75164]: cluster 2024-09-12T23:57:05.066518+0000 mon.a (mon.0) 1830 : cluster [INF] Cluster is now healthy 2024-09-12T23:57:06.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:06 smithi136 bash[75164]: cluster 2024-09-12T23:57:05.097227+0000 mon.a (mon.0) 1831 : cluster [DBG] osdmap e1023: 8 total, 8 up, 8 in 2024-09-12T23:57:06.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:06 smithi136 bash[75164]: audit 2024-09-12T23:57:05.099999+0000 mon.c (mon.2) 52 : audit [INF] from='client.? 172.21.15.136:0/3730900361' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110170-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:06.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:06 smithi136 bash[75164]: audit 2024-09-12T23:57:05.102025+0000 mon.a (mon.0) 1832 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110170-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:06.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:06 smithi136 bash[76371]: cluster 2024-09-12T23:57:05.024935+0000 mgr.y (mgr.24661) 599 : cluster [DBG] pgmap v1182: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:06.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:06 smithi136 bash[76371]: cluster 2024-09-12T23:57:05.066465+0000 mon.a (mon.0) 1829 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:57:06.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:06 smithi136 bash[76371]: cluster 2024-09-12T23:57:05.066518+0000 mon.a (mon.0) 1830 : cluster [INF] Cluster is now healthy 2024-09-12T23:57:06.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:06 smithi136 bash[76371]: cluster 2024-09-12T23:57:05.097227+0000 mon.a (mon.0) 1831 : cluster [DBG] osdmap e1023: 8 total, 8 up, 8 in 2024-09-12T23:57:06.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:06 smithi136 bash[76371]: audit 2024-09-12T23:57:05.099999+0000 mon.c (mon.2) 52 : audit [INF] from='client.? 172.21.15.136:0/3730900361' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110170-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:06.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:06 smithi136 bash[76371]: audit 2024-09-12T23:57:05.102025+0000 mon.a (mon.0) 1832 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110170-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:06.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:06 smithi181 bash[67587]: cluster 2024-09-12T23:57:05.024935+0000 mgr.y (mgr.24661) 599 : cluster [DBG] pgmap v1182: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:06.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:06 smithi181 bash[67587]: cluster 2024-09-12T23:57:05.066465+0000 mon.a (mon.0) 1829 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:57:06.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:06 smithi181 bash[67587]: cluster 2024-09-12T23:57:05.066518+0000 mon.a (mon.0) 1830 : cluster [INF] Cluster is now healthy 2024-09-12T23:57:06.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:06 smithi181 bash[67587]: cluster 2024-09-12T23:57:05.097227+0000 mon.a (mon.0) 1831 : cluster [DBG] osdmap e1023: 8 total, 8 up, 8 in 2024-09-12T23:57:06.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:06 smithi181 bash[67587]: audit 2024-09-12T23:57:05.099999+0000 mon.c (mon.2) 52 : audit [INF] from='client.? 172.21.15.136:0/3730900361' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110170-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:06.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:06 smithi181 bash[67587]: audit 2024-09-12T23:57:05.102025+0000 mon.a (mon.0) 1832 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110170-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:07.122 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] cls_log.trim_by_marker (3049 ms) 2024-09-12T23:57:07.122 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 4 tests from cls_log (12007 ms total) 2024-09-12T23:57:07.122 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-12T23:57:07.122 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] Global test environment tear-down 2024-09-12T23:57:07.123 INFO:tasks.workunit.client.0.smithi136.stdout:[==========] 4 tests from 1 test suite ran. (12007 ms total) 2024-09-12T23:57:07.123 INFO:tasks.workunit.client.0.smithi136.stdout:[ PASSED ] 4 tests. 2024-09-12T23:57:07.125 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-12T23:57:07.126 DEBUG:teuthology.orchestra.run.smithi136:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-12T23:57:07.185 INFO:tasks.workunit:Running workunit cls/test_cls_numops.sh... 2024-09-12T23:57:07.185 DEBUG:teuthology.orchestra.run.smithi136:workunit test cls/test_cls_numops.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_numops.sh 2024-09-12T23:57:07.247 INFO:tasks.workunit.client.0.smithi136.stdout:Running main() from gmock_main.cc 2024-09-12T23:57:07.247 INFO:tasks.workunit.client.0.smithi136.stdout:[==========] Running 4 tests from 1 test suite. 2024-09-12T23:57:07.247 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] Global test environment set-up. 2024-09-12T23:57:07.247 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 4 tests from ClsNumOps 2024-09-12T23:57:07.247 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] ClsNumOps.Add 2024-09-12T23:57:07.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:07 smithi136 bash[75164]: audit 2024-09-12T23:57:06.092298+0000 mon.a (mon.0) 1833 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110170-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:57:07.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:07 smithi136 bash[75164]: cluster 2024-09-12T23:57:06.092831+0000 mon.a (mon.0) 1834 : cluster [DBG] osdmap e1024: 8 total, 8 up, 8 in 2024-09-12T23:57:07.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:07 smithi136 bash[76371]: audit 2024-09-12T23:57:06.092298+0000 mon.a (mon.0) 1833 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110170-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:57:07.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:07 smithi136 bash[76371]: cluster 2024-09-12T23:57:06.092831+0000 mon.a (mon.0) 1834 : cluster [DBG] osdmap e1024: 8 total, 8 up, 8 in 2024-09-12T23:57:07.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:07 smithi181 bash[67587]: audit 2024-09-12T23:57:06.092298+0000 mon.a (mon.0) 1833 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110170-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:57:07.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:07 smithi181 bash[67587]: cluster 2024-09-12T23:57:06.092831+0000 mon.a (mon.0) 1834 : cluster [DBG] osdmap e1024: 8 total, 8 up, 8 in 2024-09-12T23:57:08.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:08 smithi136 bash[75164]: cluster 2024-09-12T23:57:07.025932+0000 mgr.y (mgr.24661) 600 : cluster [DBG] pgmap v1185: 137 pgs: 1 creating+activating, 14 creating+peering, 10 unknown, 112 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:08.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:08 smithi136 bash[75164]: cluster 2024-09-12T23:57:07.121451+0000 mon.a (mon.0) 1835 : cluster [DBG] osdmap e1025: 8 total, 8 up, 8 in 2024-09-12T23:57:08.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:08 smithi136 bash[76371]: cluster 2024-09-12T23:57:07.025932+0000 mgr.y (mgr.24661) 600 : cluster [DBG] pgmap v1185: 137 pgs: 1 creating+activating, 14 creating+peering, 10 unknown, 112 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:08.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:08 smithi136 bash[76371]: cluster 2024-09-12T23:57:07.121451+0000 mon.a (mon.0) 1835 : cluster [DBG] osdmap e1025: 8 total, 8 up, 8 in 2024-09-12T23:57:08.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:08 smithi181 bash[67587]: cluster 2024-09-12T23:57:07.025932+0000 mgr.y (mgr.24661) 600 : cluster [DBG] pgmap v1185: 137 pgs: 1 creating+activating, 14 creating+peering, 10 unknown, 112 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:08.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:08 smithi181 bash[67587]: cluster 2024-09-12T23:57:07.121451+0000 mon.a (mon.0) 1835 : cluster [DBG] osdmap e1025: 8 total, 8 up, 8 in 2024-09-12T23:57:09.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:09 smithi136 bash[76371]: cluster 2024-09-12T23:57:08.133203+0000 mon.a (mon.0) 1836 : cluster [DBG] osdmap e1026: 8 total, 8 up, 8 in 2024-09-12T23:57:09.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:09 smithi136 bash[76371]: audit 2024-09-12T23:57:08.135764+0000 mon.a (mon.0) 1837 : audit [INF] from='client.? 172.21.15.136:0/1601097153' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110245-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:09.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:09 smithi136 bash[76371]: audit 2024-09-12T23:57:08.763979+0000 mon.a (mon.0) 1838 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:57:09.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:09 smithi136 bash[75164]: cluster 2024-09-12T23:57:08.133203+0000 mon.a (mon.0) 1836 : cluster [DBG] osdmap e1026: 8 total, 8 up, 8 in 2024-09-12T23:57:09.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:09 smithi136 bash[75164]: audit 2024-09-12T23:57:08.135764+0000 mon.a (mon.0) 1837 : audit [INF] from='client.? 172.21.15.136:0/1601097153' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110245-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:09.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:09 smithi136 bash[75164]: audit 2024-09-12T23:57:08.763979+0000 mon.a (mon.0) 1838 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:57:09.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:09 smithi181 bash[67587]: cluster 2024-09-12T23:57:08.133203+0000 mon.a (mon.0) 1836 : cluster [DBG] osdmap e1026: 8 total, 8 up, 8 in 2024-09-12T23:57:09.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:09 smithi181 bash[67587]: audit 2024-09-12T23:57:08.135764+0000 mon.a (mon.0) 1837 : audit [INF] from='client.? 172.21.15.136:0/1601097153' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110245-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:09.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:09 smithi181 bash[67587]: audit 2024-09-12T23:57:08.763979+0000 mon.a (mon.0) 1838 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:57:09.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:57:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:57:09] "GET /metrics HTTP/1.1" 200 36453 "" "Prometheus/2.43.0" 2024-09-12T23:57:10.160 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] ClsNumOps.Add (2913 ms) 2024-09-12T23:57:10.160 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] ClsNumOps.Sub 2024-09-12T23:57:10.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:10 smithi136 bash[75164]: cluster 2024-09-12T23:57:09.026797+0000 mgr.y (mgr.24661) 601 : cluster [DBG] pgmap v1188: 137 pgs: 8 creating+peering, 24 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:10.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:10 smithi136 bash[75164]: cluster 2024-09-12T23:57:09.125186+0000 mon.a (mon.0) 1839 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:57:10.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:10 smithi136 bash[75164]: audit 2024-09-12T23:57:09.147094+0000 mon.a (mon.0) 1840 : audit [INF] from='client.? 172.21.15.136:0/1601097153' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110245-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:57:10.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:10 smithi136 bash[75164]: cluster 2024-09-12T23:57:09.147910+0000 mon.a (mon.0) 1841 : cluster [DBG] osdmap e1027: 8 total, 8 up, 8 in 2024-09-12T23:57:10.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:10 smithi136 bash[76371]: cluster 2024-09-12T23:57:09.026797+0000 mgr.y (mgr.24661) 601 : cluster [DBG] pgmap v1188: 137 pgs: 8 creating+peering, 24 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:10.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:10 smithi136 bash[76371]: cluster 2024-09-12T23:57:09.125186+0000 mon.a (mon.0) 1839 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:57:10.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:10 smithi136 bash[76371]: audit 2024-09-12T23:57:09.147094+0000 mon.a (mon.0) 1840 : audit [INF] from='client.? 172.21.15.136:0/1601097153' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110245-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:57:10.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:10 smithi136 bash[76371]: cluster 2024-09-12T23:57:09.147910+0000 mon.a (mon.0) 1841 : cluster [DBG] osdmap e1027: 8 total, 8 up, 8 in 2024-09-12T23:57:10.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:10 smithi181 bash[67587]: cluster 2024-09-12T23:57:09.026797+0000 mgr.y (mgr.24661) 601 : cluster [DBG] pgmap v1188: 137 pgs: 8 creating+peering, 24 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:10.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:10 smithi181 bash[67587]: cluster 2024-09-12T23:57:09.125186+0000 mon.a (mon.0) 1839 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:57:10.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:10 smithi181 bash[67587]: audit 2024-09-12T23:57:09.147094+0000 mon.a (mon.0) 1840 : audit [INF] from='client.? 172.21.15.136:0/1601097153' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110245-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:57:10.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:10 smithi181 bash[67587]: cluster 2024-09-12T23:57:09.147910+0000 mon.a (mon.0) 1841 : cluster [DBG] osdmap e1027: 8 total, 8 up, 8 in 2024-09-12T23:57:11.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:11 smithi136 bash[75164]: cluster 2024-09-12T23:57:10.149359+0000 mon.a (mon.0) 1842 : cluster [DBG] osdmap e1028: 8 total, 8 up, 8 in 2024-09-12T23:57:11.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:11 smithi136 bash[76371]: cluster 2024-09-12T23:57:10.149359+0000 mon.a (mon.0) 1842 : cluster [DBG] osdmap e1028: 8 total, 8 up, 8 in 2024-09-12T23:57:11.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:11 smithi181 bash[67587]: cluster 2024-09-12T23:57:10.149359+0000 mon.a (mon.0) 1842 : cluster [DBG] osdmap e1028: 8 total, 8 up, 8 in 2024-09-12T23:57:12.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:12 smithi136 bash[75164]: cluster 2024-09-12T23:57:11.027673+0000 mgr.y (mgr.24661) 602 : cluster [DBG] pgmap v1191: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:12.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:12 smithi136 bash[75164]: cluster 2024-09-12T23:57:11.164379+0000 mon.a (mon.0) 1843 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:57:12.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:12 smithi136 bash[75164]: cluster 2024-09-12T23:57:11.164433+0000 mon.a (mon.0) 1844 : cluster [INF] Cluster is now healthy 2024-09-12T23:57:12.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:12 smithi136 bash[75164]: cluster 2024-09-12T23:57:11.182942+0000 mon.a (mon.0) 1845 : cluster [DBG] osdmap e1029: 8 total, 8 up, 8 in 2024-09-12T23:57:12.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:12 smithi136 bash[75164]: audit 2024-09-12T23:57:11.189276+0000 mon.c (mon.2) 53 : audit [INF] from='client.? 172.21.15.136:0/2568872400' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110245-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:12.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:12 smithi136 bash[75164]: audit 2024-09-12T23:57:11.190334+0000 mon.a (mon.0) 1846 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110245-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:12.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:12 smithi136 bash[76371]: cluster 2024-09-12T23:57:11.027673+0000 mgr.y (mgr.24661) 602 : cluster [DBG] pgmap v1191: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:12.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:12 smithi136 bash[76371]: cluster 2024-09-12T23:57:11.164379+0000 mon.a (mon.0) 1843 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:57:12.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:12 smithi136 bash[76371]: cluster 2024-09-12T23:57:11.164433+0000 mon.a (mon.0) 1844 : cluster [INF] Cluster is now healthy 2024-09-12T23:57:12.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:12 smithi136 bash[76371]: cluster 2024-09-12T23:57:11.182942+0000 mon.a (mon.0) 1845 : cluster [DBG] osdmap e1029: 8 total, 8 up, 8 in 2024-09-12T23:57:12.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:12 smithi136 bash[76371]: audit 2024-09-12T23:57:11.189276+0000 mon.c (mon.2) 53 : audit [INF] from='client.? 172.21.15.136:0/2568872400' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110245-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:12.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:12 smithi136 bash[76371]: audit 2024-09-12T23:57:11.190334+0000 mon.a (mon.0) 1846 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110245-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:12.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:57:12 smithi181 bash[95505]: ts=2024-09-12T23:57:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:57:12.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:12 smithi181 bash[67587]: cluster 2024-09-12T23:57:11.027673+0000 mgr.y (mgr.24661) 602 : cluster [DBG] pgmap v1191: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:12.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:12 smithi181 bash[67587]: cluster 2024-09-12T23:57:11.164379+0000 mon.a (mon.0) 1843 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:57:12.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:12 smithi181 bash[67587]: cluster 2024-09-12T23:57:11.164433+0000 mon.a (mon.0) 1844 : cluster [INF] Cluster is now healthy 2024-09-12T23:57:12.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:12 smithi181 bash[67587]: cluster 2024-09-12T23:57:11.182942+0000 mon.a (mon.0) 1845 : cluster [DBG] osdmap e1029: 8 total, 8 up, 8 in 2024-09-12T23:57:12.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:12 smithi181 bash[67587]: audit 2024-09-12T23:57:11.189276+0000 mon.c (mon.2) 53 : audit [INF] from='client.? 172.21.15.136:0/2568872400' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110245-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:12.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:12 smithi181 bash[67587]: audit 2024-09-12T23:57:11.190334+0000 mon.a (mon.0) 1846 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110245-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:13.200 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] ClsNumOps.Sub (3040 ms) 2024-09-12T23:57:13.201 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] ClsNumOps.Mul 2024-09-12T23:57:13.528 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:13 smithi136 bash[75164]: audit 2024-09-12T23:57:12.186786+0000 mon.a (mon.0) 1847 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110245-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:57:13.528 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:13 smithi136 bash[75164]: cluster 2024-09-12T23:57:12.187283+0000 mon.a (mon.0) 1848 : cluster [DBG] osdmap e1030: 8 total, 8 up, 8 in 2024-09-12T23:57:13.528 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:13 smithi136 bash[76371]: audit 2024-09-12T23:57:12.186786+0000 mon.a (mon.0) 1847 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110245-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:57:13.528 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:13 smithi136 bash[76371]: cluster 2024-09-12T23:57:12.187283+0000 mon.a (mon.0) 1848 : cluster [DBG] osdmap e1030: 8 total, 8 up, 8 in 2024-09-12T23:57:13.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:13 smithi181 bash[67587]: audit 2024-09-12T23:57:12.186786+0000 mon.a (mon.0) 1847 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110245-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:57:13.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:13 smithi181 bash[67587]: cluster 2024-09-12T23:57:12.187283+0000 mon.a (mon.0) 1848 : cluster [DBG] osdmap e1030: 8 total, 8 up, 8 in 2024-09-12T23:57:14.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:14 smithi136 bash[76371]: cluster 2024-09-12T23:57:13.028503+0000 mgr.y (mgr.24661) 603 : cluster [DBG] pgmap v1194: 137 pgs: 8 creating+peering, 20 unknown, 109 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:14.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:14 smithi136 bash[76371]: cluster 2024-09-12T23:57:13.197160+0000 mon.a (mon.0) 1849 : cluster [DBG] osdmap e1031: 8 total, 8 up, 8 in 2024-09-12T23:57:14.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:14 smithi136 bash[75164]: cluster 2024-09-12T23:57:13.028503+0000 mgr.y (mgr.24661) 603 : cluster [DBG] pgmap v1194: 137 pgs: 8 creating+peering, 20 unknown, 109 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:14.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:14 smithi136 bash[75164]: cluster 2024-09-12T23:57:13.197160+0000 mon.a (mon.0) 1849 : cluster [DBG] osdmap e1031: 8 total, 8 up, 8 in 2024-09-12T23:57:14.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:14 smithi181 bash[67587]: cluster 2024-09-12T23:57:13.028503+0000 mgr.y (mgr.24661) 603 : cluster [DBG] pgmap v1194: 137 pgs: 8 creating+peering, 20 unknown, 109 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:14.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:14 smithi181 bash[67587]: cluster 2024-09-12T23:57:13.197160+0000 mon.a (mon.0) 1849 : cluster [DBG] osdmap e1031: 8 total, 8 up, 8 in 2024-09-12T23:57:15.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:15 smithi136 bash[76371]: cluster 2024-09-12T23:57:14.212835+0000 mon.a (mon.0) 1850 : cluster [DBG] osdmap e1032: 8 total, 8 up, 8 in 2024-09-12T23:57:15.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:15 smithi136 bash[76371]: audit 2024-09-12T23:57:14.217733+0000 mon.c (mon.2) 54 : audit [INF] from='client.? 172.21.15.136:0/4254898829' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110245-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:15.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:15 smithi136 bash[76371]: audit 2024-09-12T23:57:14.219157+0000 mon.a (mon.0) 1851 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110245-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:15.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:15 smithi136 bash[75164]: cluster 2024-09-12T23:57:14.212835+0000 mon.a (mon.0) 1850 : cluster [DBG] osdmap e1032: 8 total, 8 up, 8 in 2024-09-12T23:57:15.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:15 smithi136 bash[75164]: audit 2024-09-12T23:57:14.217733+0000 mon.c (mon.2) 54 : audit [INF] from='client.? 172.21.15.136:0/4254898829' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110245-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:15.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:15 smithi136 bash[75164]: audit 2024-09-12T23:57:14.219157+0000 mon.a (mon.0) 1851 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110245-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:15.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:15 smithi181 bash[67587]: cluster 2024-09-12T23:57:14.212835+0000 mon.a (mon.0) 1850 : cluster [DBG] osdmap e1032: 8 total, 8 up, 8 in 2024-09-12T23:57:15.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:15 smithi181 bash[67587]: audit 2024-09-12T23:57:14.217733+0000 mon.c (mon.2) 54 : audit [INF] from='client.? 172.21.15.136:0/4254898829' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110245-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:15.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:15 smithi181 bash[67587]: audit 2024-09-12T23:57:14.219157+0000 mon.a (mon.0) 1851 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110245-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:16.229 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] ClsNumOps.Mul (3029 ms) 2024-09-12T23:57:16.229 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] ClsNumOps.Div 2024-09-12T23:57:16.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:16 smithi136 bash[75164]: cluster 2024-09-12T23:57:15.029351+0000 mgr.y (mgr.24661) 604 : cluster [DBG] pgmap v1197: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:16.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:16 smithi136 bash[75164]: cluster 2024-09-12T23:57:15.205711+0000 mon.a (mon.0) 1852 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:57:16.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:16 smithi136 bash[75164]: audit 2024-09-12T23:57:15.224551+0000 mon.a (mon.0) 1853 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110245-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:57:16.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:16 smithi136 bash[75164]: cluster 2024-09-12T23:57:15.225009+0000 mon.a (mon.0) 1854 : cluster [DBG] osdmap e1033: 8 total, 8 up, 8 in 2024-09-12T23:57:16.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:16 smithi136 bash[76371]: cluster 2024-09-12T23:57:15.029351+0000 mgr.y (mgr.24661) 604 : cluster [DBG] pgmap v1197: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:16.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:16 smithi136 bash[76371]: cluster 2024-09-12T23:57:15.205711+0000 mon.a (mon.0) 1852 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:57:16.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:16 smithi136 bash[76371]: audit 2024-09-12T23:57:15.224551+0000 mon.a (mon.0) 1853 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110245-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:57:16.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:16 smithi136 bash[76371]: cluster 2024-09-12T23:57:15.225009+0000 mon.a (mon.0) 1854 : cluster [DBG] osdmap e1033: 8 total, 8 up, 8 in 2024-09-12T23:57:16.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:16 smithi181 bash[67587]: cluster 2024-09-12T23:57:15.029351+0000 mgr.y (mgr.24661) 604 : cluster [DBG] pgmap v1197: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:16.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:16 smithi181 bash[67587]: cluster 2024-09-12T23:57:15.205711+0000 mon.a (mon.0) 1852 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:57:16.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:16 smithi181 bash[67587]: audit 2024-09-12T23:57:15.224551+0000 mon.a (mon.0) 1853 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110245-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:57:16.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:16 smithi181 bash[67587]: cluster 2024-09-12T23:57:15.225009+0000 mon.a (mon.0) 1854 : cluster [DBG] osdmap e1033: 8 total, 8 up, 8 in 2024-09-12T23:57:17.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:17 smithi136 bash[75164]: cluster 2024-09-12T23:57:16.225796+0000 mon.a (mon.0) 1855 : cluster [DBG] osdmap e1034: 8 total, 8 up, 8 in 2024-09-12T23:57:17.530 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:17 smithi136 bash[76371]: cluster 2024-09-12T23:57:16.225796+0000 mon.a (mon.0) 1855 : cluster [DBG] osdmap e1034: 8 total, 8 up, 8 in 2024-09-12T23:57:17.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:17 smithi181 bash[67587]: cluster 2024-09-12T23:57:16.225796+0000 mon.a (mon.0) 1855 : cluster [DBG] osdmap e1034: 8 total, 8 up, 8 in 2024-09-12T23:57:18.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:18 smithi136 bash[75164]: cluster 2024-09-12T23:57:17.030322+0000 mgr.y (mgr.24661) 605 : cluster [DBG] pgmap v1200: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:18.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:18 smithi136 bash[75164]: cluster 2024-09-12T23:57:17.234824+0000 mon.a (mon.0) 1856 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:57:18.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:18 smithi136 bash[75164]: cluster 2024-09-12T23:57:17.234884+0000 mon.a (mon.0) 1857 : cluster [INF] Cluster is now healthy 2024-09-12T23:57:18.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:18 smithi136 bash[75164]: cluster 2024-09-12T23:57:17.257760+0000 mon.a (mon.0) 1858 : cluster [DBG] osdmap e1035: 8 total, 8 up, 8 in 2024-09-12T23:57:18.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:18 smithi136 bash[75164]: audit 2024-09-12T23:57:17.261055+0000 mon.a (mon.0) 1859 : audit [INF] from='client.? 172.21.15.136:0/4219174515' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110245-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:18.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:18 smithi136 bash[76371]: cluster 2024-09-12T23:57:17.030322+0000 mgr.y (mgr.24661) 605 : cluster [DBG] pgmap v1200: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:18.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:18 smithi136 bash[76371]: cluster 2024-09-12T23:57:17.234824+0000 mon.a (mon.0) 1856 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:57:18.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:18 smithi136 bash[76371]: cluster 2024-09-12T23:57:17.234884+0000 mon.a (mon.0) 1857 : cluster [INF] Cluster is now healthy 2024-09-12T23:57:18.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:18 smithi136 bash[76371]: cluster 2024-09-12T23:57:17.257760+0000 mon.a (mon.0) 1858 : cluster [DBG] osdmap e1035: 8 total, 8 up, 8 in 2024-09-12T23:57:18.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:18 smithi136 bash[76371]: audit 2024-09-12T23:57:17.261055+0000 mon.a (mon.0) 1859 : audit [INF] from='client.? 172.21.15.136:0/4219174515' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110245-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:18.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:18 smithi181 bash[67587]: cluster 2024-09-12T23:57:17.030322+0000 mgr.y (mgr.24661) 605 : cluster [DBG] pgmap v1200: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:18.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:18 smithi181 bash[67587]: cluster 2024-09-12T23:57:17.234824+0000 mon.a (mon.0) 1856 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:57:18.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:18 smithi181 bash[67587]: cluster 2024-09-12T23:57:17.234884+0000 mon.a (mon.0) 1857 : cluster [INF] Cluster is now healthy 2024-09-12T23:57:18.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:18 smithi181 bash[67587]: cluster 2024-09-12T23:57:17.257760+0000 mon.a (mon.0) 1858 : cluster [DBG] osdmap e1035: 8 total, 8 up, 8 in 2024-09-12T23:57:18.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:18 smithi181 bash[67587]: audit 2024-09-12T23:57:17.261055+0000 mon.a (mon.0) 1859 : audit [INF] from='client.? 172.21.15.136:0/4219174515' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110245-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:19.280 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] ClsNumOps.Div (3051 ms) 2024-09-12T23:57:19.280 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 4 tests from ClsNumOps (12033 ms total) 2024-09-12T23:57:19.280 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-12T23:57:19.280 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] Global test environment tear-down 2024-09-12T23:57:19.280 INFO:tasks.workunit.client.0.smithi136.stdout:[==========] 4 tests from 1 test suite ran. (12033 ms total) 2024-09-12T23:57:19.280 INFO:tasks.workunit.client.0.smithi136.stdout:[ PASSED ] 4 tests. 2024-09-12T23:57:19.283 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-12T23:57:19.284 DEBUG:teuthology.orchestra.run.smithi136:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-12T23:57:19.340 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2024-09-12T23:57:19.340 DEBUG:teuthology.orchestra.run.smithi136:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2024-09-12T23:57:19.403 INFO:tasks.workunit.client.0.smithi136.stdout:Running main() from gmock_main.cc 2024-09-12T23:57:19.403 INFO:tasks.workunit.client.0.smithi136.stdout:[==========] Running 70 tests from 1 test suite. 2024-09-12T23:57:19.403 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] Global test environment set-up. 2024-09-12T23:57:19.403 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 70 tests from TestClsRbd 2024-09-12T23:57:19.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:19 smithi181 bash[67587]: audit 2024-09-12T23:57:18.265729+0000 mon.a (mon.0) 1860 : audit [INF] from='client.? 172.21.15.136:0/4219174515' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110245-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:57:19.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:19 smithi181 bash[67587]: cluster 2024-09-12T23:57:18.265989+0000 mon.a (mon.0) 1861 : cluster [DBG] osdmap e1036: 8 total, 8 up, 8 in 2024-09-12T23:57:19.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:19 smithi136 bash[75164]: audit 2024-09-12T23:57:18.265729+0000 mon.a (mon.0) 1860 : audit [INF] from='client.? 172.21.15.136:0/4219174515' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110245-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:57:19.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:19 smithi136 bash[75164]: cluster 2024-09-12T23:57:18.265989+0000 mon.a (mon.0) 1861 : cluster [DBG] osdmap e1036: 8 total, 8 up, 8 in 2024-09-12T23:57:19.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:57:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:57:19] "GET /metrics HTTP/1.1" 200 36453 "" "Prometheus/2.43.0" 2024-09-12T23:57:19.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:19 smithi136 bash[76371]: audit 2024-09-12T23:57:18.265729+0000 mon.a (mon.0) 1860 : audit [INF] from='client.? 172.21.15.136:0/4219174515' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110245-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:57:19.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:19 smithi136 bash[76371]: cluster 2024-09-12T23:57:18.265989+0000 mon.a (mon.0) 1861 : cluster [DBG] osdmap e1036: 8 total, 8 up, 8 in 2024-09-12T23:57:20.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:20 smithi181 bash[67587]: cluster 2024-09-12T23:57:19.031119+0000 mgr.y (mgr.24661) 606 : cluster [DBG] pgmap v1203: 137 pgs: 4 creating+peering, 24 unknown, 109 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:20.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:20 smithi181 bash[67587]: cluster 2024-09-12T23:57:19.276900+0000 mon.a (mon.0) 1862 : cluster [DBG] osdmap e1037: 8 total, 8 up, 8 in 2024-09-12T23:57:20.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:20 smithi136 bash[75164]: cluster 2024-09-12T23:57:19.031119+0000 mgr.y (mgr.24661) 606 : cluster [DBG] pgmap v1203: 137 pgs: 4 creating+peering, 24 unknown, 109 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:20.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:20 smithi136 bash[75164]: cluster 2024-09-12T23:57:19.276900+0000 mon.a (mon.0) 1862 : cluster [DBG] osdmap e1037: 8 total, 8 up, 8 in 2024-09-12T23:57:20.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:20 smithi136 bash[76371]: cluster 2024-09-12T23:57:19.031119+0000 mgr.y (mgr.24661) 606 : cluster [DBG] pgmap v1203: 137 pgs: 4 creating+peering, 24 unknown, 109 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:20.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:20 smithi136 bash[76371]: cluster 2024-09-12T23:57:19.276900+0000 mon.a (mon.0) 1862 : cluster [DBG] osdmap e1037: 8 total, 8 up, 8 in 2024-09-12T23:57:21.281 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.get_all_features 2024-09-12T23:57:21.321 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.get_all_features (40 ms) 2024-09-12T23:57:21.321 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.copyup 2024-09-12T23:57:21.539 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.copyup (218 ms) 2024-09-12T23:57:21.539 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.sparse_copyup 2024-09-12T23:57:21.568 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.sparse_copyup (29 ms) 2024-09-12T23:57:21.568 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.get_and_set_id 2024-09-12T23:57:21.580 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.get_and_set_id (13 ms) 2024-09-12T23:57:21.581 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.add_remove_child 2024-09-12T23:57:21.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:21 smithi181 bash[67587]: cluster 2024-09-12T23:57:20.281920+0000 mon.a (mon.0) 1863 : cluster [DBG] osdmap e1038: 8 total, 8 up, 8 in 2024-09-12T23:57:21.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:21 smithi181 bash[67587]: audit 2024-09-12T23:57:20.284813+0000 mon.c (mon.2) 55 : audit [INF] from='client.? 172.21.15.136:0/765302567' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110320-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:21.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:21 smithi181 bash[67587]: audit 2024-09-12T23:57:20.286122+0000 mon.a (mon.0) 1864 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110320-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:21.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:21 smithi181 bash[67587]: audit 2024-09-12T23:57:21.283170+0000 mon.a (mon.0) 1865 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110320-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:57:21.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:21 smithi181 bash[67587]: cluster 2024-09-12T23:57:21.283879+0000 mon.a (mon.0) 1866 : cluster [DBG] osdmap e1039: 8 total, 8 up, 8 in 2024-09-12T23:57:21.610 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.add_remove_child (29 ms) 2024-09-12T23:57:21.610 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.directory_methods 2024-09-12T23:57:21.684 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.directory_methods (74 ms) 2024-09-12T23:57:21.684 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.create 2024-09-12T23:57:21.722 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.create (38 ms) 2024-09-12T23:57:21.722 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.get_features 2024-09-12T23:57:21.726 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.get_features (4 ms) 2024-09-12T23:57:21.726 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.get_object_prefix 2024-09-12T23:57:21.730 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.get_object_prefix (4 ms) 2024-09-12T23:57:21.730 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2024-09-12T23:57:21.735 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.get_create_timestamp (5 ms) 2024-09-12T23:57:21.735 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2024-09-12T23:57:21.740 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.get_access_timestamp (5 ms) 2024-09-12T23:57:21.740 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2024-09-12T23:57:21.743 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.get_modify_timestamp (3 ms) 2024-09-12T23:57:21.743 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.get_data_pool 2024-09-12T23:57:21.753 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.get_data_pool (10 ms) 2024-09-12T23:57:21.753 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.get_size 2024-09-12T23:57:21.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:21 smithi136 bash[76371]: cluster 2024-09-12T23:57:20.281920+0000 mon.a (mon.0) 1863 : cluster [DBG] osdmap e1038: 8 total, 8 up, 8 in 2024-09-12T23:57:21.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:21 smithi136 bash[76371]: audit 2024-09-12T23:57:20.284813+0000 mon.c (mon.2) 55 : audit [INF] from='client.? 172.21.15.136:0/765302567' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110320-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:21.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:21 smithi136 bash[76371]: audit 2024-09-12T23:57:20.286122+0000 mon.a (mon.0) 1864 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110320-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:21.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:21 smithi136 bash[76371]: audit 2024-09-12T23:57:21.283170+0000 mon.a (mon.0) 1865 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110320-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:57:21.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:21 smithi136 bash[76371]: cluster 2024-09-12T23:57:21.283879+0000 mon.a (mon.0) 1866 : cluster [DBG] osdmap e1039: 8 total, 8 up, 8 in 2024-09-12T23:57:21.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:21 smithi136 bash[75164]: cluster 2024-09-12T23:57:20.281920+0000 mon.a (mon.0) 1863 : cluster [DBG] osdmap e1038: 8 total, 8 up, 8 in 2024-09-12T23:57:21.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:21 smithi136 bash[75164]: audit 2024-09-12T23:57:20.284813+0000 mon.c (mon.2) 55 : audit [INF] from='client.? 172.21.15.136:0/765302567' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110320-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:21.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:21 smithi136 bash[75164]: audit 2024-09-12T23:57:20.286122+0000 mon.a (mon.0) 1864 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110320-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:21.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:21 smithi136 bash[75164]: audit 2024-09-12T23:57:21.283170+0000 mon.a (mon.0) 1865 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110320-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:57:21.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:21 smithi136 bash[75164]: cluster 2024-09-12T23:57:21.283879+0000 mon.a (mon.0) 1866 : cluster [DBG] osdmap e1039: 8 total, 8 up, 8 in 2024-09-12T23:57:21.763 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.get_size (11 ms) 2024-09-12T23:57:21.764 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.set_size 2024-09-12T23:57:21.778 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.set_size (14 ms) 2024-09-12T23:57:21.778 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.protection_status 2024-09-12T23:57:21.826 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.protection_status (48 ms) 2024-09-12T23:57:21.826 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.snapshot_limits 2024-09-12T23:57:21.847 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.snapshot_limits (21 ms) 2024-09-12T23:57:21.847 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.parents_v1 2024-09-12T23:57:21.963 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.parents_v1 (116 ms) 2024-09-12T23:57:21.963 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.parents_v2 2024-09-12T23:57:22.047 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.parents_v2 (84 ms) 2024-09-12T23:57:22.048 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.snapshots 2024-09-12T23:57:22.107 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.snapshots (60 ms) 2024-09-12T23:57:22.107 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.snapid_race 2024-09-12T23:57:22.119 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.snapid_race (12 ms) 2024-09-12T23:57:22.119 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.stripingv2 2024-09-12T23:57:22.144 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.stripingv2 (25 ms) 2024-09-12T23:57:22.144 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.object_map_save 2024-09-12T23:57:22.148 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.object_map_save (4 ms) 2024-09-12T23:57:22.148 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.object_map_resize 2024-09-12T23:57:22.162 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.object_map_resize (14 ms) 2024-09-12T23:57:22.162 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.object_map_update 2024-09-12T23:57:22.176 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.object_map_update (14 ms) 2024-09-12T23:57:22.176 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2024-09-12T23:57:22.177 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.object_map_load_enoent (1 ms) 2024-09-12T23:57:22.177 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2024-09-12T23:57:22.188 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.object_map_snap_add (11 ms) 2024-09-12T23:57:22.188 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2024-09-12T23:57:22.199 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.object_map_snap_remove (11 ms) 2024-09-12T23:57:22.199 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.flags 2024-09-12T23:57:22.211 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.flags (13 ms) 2024-09-12T23:57:22.212 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.metadata 2024-09-12T23:57:22.276 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.metadata (64 ms) 2024-09-12T23:57:22.276 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.set_features 2024-09-12T23:57:22.295 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.set_features (19 ms) 2024-09-12T23:57:22.295 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.mirror 2024-09-12T23:57:22.388 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.mirror (93 ms) 2024-09-12T23:57:22.388 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.mirror_image 2024-09-12T23:57:22.426 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.mirror_image (38 ms) 2024-09-12T23:57:22.427 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.mirror_image_status 2024-09-12T23:57:22.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:22 smithi181 bash[67587]: cluster 2024-09-12T23:57:21.032073+0000 mgr.y (mgr.24661) 607 : cluster [DBG] pgmap v1206: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:22.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:22 smithi181 bash[67587]: cluster 2024-09-12T23:57:21.292159+0000 mon.a (mon.0) 1867 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:57:22.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:57:22 smithi181 bash[95505]: ts=2024-09-12T23:57:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:57:22.756 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:22 smithi136 bash[76371]: cluster 2024-09-12T23:57:21.032073+0000 mgr.y (mgr.24661) 607 : cluster [DBG] pgmap v1206: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:22.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:22 smithi136 bash[76371]: cluster 2024-09-12T23:57:21.292159+0000 mon.a (mon.0) 1867 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:57:22.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:22 smithi136 bash[75164]: cluster 2024-09-12T23:57:21.032073+0000 mgr.y (mgr.24661) 607 : cluster [DBG] pgmap v1206: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:22.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:22 smithi136 bash[75164]: cluster 2024-09-12T23:57:21.292159+0000 mon.a (mon.0) 1867 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:57:23.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:23 smithi181 bash[67587]: cluster 2024-09-12T23:57:22.308869+0000 mon.a (mon.0) 1868 : cluster [DBG] osdmap e1040: 8 total, 8 up, 8 in 2024-09-12T23:57:23.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:23 smithi136 bash[75164]: cluster 2024-09-12T23:57:22.308869+0000 mon.a (mon.0) 1868 : cluster [DBG] osdmap e1040: 8 total, 8 up, 8 in 2024-09-12T23:57:23.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:23 smithi136 bash[76371]: cluster 2024-09-12T23:57:22.308869+0000 mon.a (mon.0) 1868 : cluster [DBG] osdmap e1040: 8 total, 8 up, 8 in 2024-09-12T23:57:24.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:24 smithi181 bash[67587]: cluster 2024-09-12T23:57:23.032928+0000 mgr.y (mgr.24661) 608 : cluster [DBG] pgmap v1209: 137 pgs: 6 creating+peering, 15 unknown, 116 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 1023 B/s rd, 511 B/s wr, 1 op/s 2024-09-12T23:57:24.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:24 smithi181 bash[67587]: cluster 2024-09-12T23:57:23.316188+0000 mon.a (mon.0) 1869 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:57:24.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:24 smithi181 bash[67587]: cluster 2024-09-12T23:57:23.316255+0000 mon.a (mon.0) 1870 : cluster [INF] Cluster is now healthy 2024-09-12T23:57:24.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:24 smithi181 bash[67587]: audit 2024-09-12T23:57:23.764374+0000 mon.a (mon.0) 1871 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:57:24.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:24 smithi136 bash[75164]: cluster 2024-09-12T23:57:23.032928+0000 mgr.y (mgr.24661) 608 : cluster [DBG] pgmap v1209: 137 pgs: 6 creating+peering, 15 unknown, 116 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 1023 B/s rd, 511 B/s wr, 1 op/s 2024-09-12T23:57:24.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:24 smithi136 bash[75164]: cluster 2024-09-12T23:57:23.316188+0000 mon.a (mon.0) 1869 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:57:24.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:24 smithi136 bash[75164]: cluster 2024-09-12T23:57:23.316255+0000 mon.a (mon.0) 1870 : cluster [INF] Cluster is now healthy 2024-09-12T23:57:24.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:24 smithi136 bash[75164]: audit 2024-09-12T23:57:23.764374+0000 mon.a (mon.0) 1871 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:57:24.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:24 smithi136 bash[76371]: cluster 2024-09-12T23:57:23.032928+0000 mgr.y (mgr.24661) 608 : cluster [DBG] pgmap v1209: 137 pgs: 6 creating+peering, 15 unknown, 116 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 1023 B/s rd, 511 B/s wr, 1 op/s 2024-09-12T23:57:24.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:24 smithi136 bash[76371]: cluster 2024-09-12T23:57:23.316188+0000 mon.a (mon.0) 1869 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:57:24.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:24 smithi136 bash[76371]: cluster 2024-09-12T23:57:23.316255+0000 mon.a (mon.0) 1870 : cluster [INF] Cluster is now healthy 2024-09-12T23:57:24.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:24 smithi136 bash[76371]: audit 2024-09-12T23:57:23.764374+0000 mon.a (mon.0) 1871 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:57:26.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:26 smithi181 bash[67587]: cluster 2024-09-12T23:57:25.034004+0000 mgr.y (mgr.24661) 609 : cluster [DBG] pgmap v1210: 137 pgs: 6 creating+peering, 131 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 50 KiB/s rd, 16 KiB/s wr, 61 op/s 2024-09-12T23:57:26.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:26 smithi136 bash[75164]: cluster 2024-09-12T23:57:25.034004+0000 mgr.y (mgr.24661) 609 : cluster [DBG] pgmap v1210: 137 pgs: 6 creating+peering, 131 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 50 KiB/s rd, 16 KiB/s wr, 61 op/s 2024-09-12T23:57:26.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:26 smithi136 bash[76371]: cluster 2024-09-12T23:57:25.034004+0000 mgr.y (mgr.24661) 609 : cluster [DBG] pgmap v1210: 137 pgs: 6 creating+peering, 131 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 50 KiB/s rd, 16 KiB/s wr, 61 op/s 2024-09-12T23:57:28.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:28 smithi181 bash[67587]: cluster 2024-09-12T23:57:27.035379+0000 mgr.y (mgr.24661) 610 : cluster [DBG] pgmap v1211: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 1.7 MiB/s rd, 956 KiB/s wr, 920 op/s 2024-09-12T23:57:28.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:28 smithi136 bash[75164]: cluster 2024-09-12T23:57:27.035379+0000 mgr.y (mgr.24661) 610 : cluster [DBG] pgmap v1211: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 1.7 MiB/s rd, 956 KiB/s wr, 920 op/s 2024-09-12T23:57:28.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:28 smithi136 bash[76371]: cluster 2024-09-12T23:57:27.035379+0000 mgr.y (mgr.24661) 610 : cluster [DBG] pgmap v1211: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 1.7 MiB/s rd, 956 KiB/s wr, 920 op/s 2024-09-12T23:57:28.943 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.mirror_image_status (6516 ms) 2024-09-12T23:57:28.943 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.mirror_image_map 2024-09-12T23:57:29.157 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.mirror_image_map (214 ms) 2024-09-12T23:57:29.157 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.mirror_instances 2024-09-12T23:57:29.178 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.mirror_instances (21 ms) 2024-09-12T23:57:29.179 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2024-09-12T23:57:29.212 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.mirror_snapshot (35 ms) 2024-09-12T23:57:29.213 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.group_dir_list 2024-09-12T23:57:29.219 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.group_dir_list (6 ms) 2024-09-12T23:57:29.219 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.group_dir_add 2024-09-12T23:57:29.224 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.group_dir_add (5 ms) 2024-09-12T23:57:29.224 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2024-09-12T23:57:29.232 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.dir_add_already_existing (8 ms) 2024-09-12T23:57:29.232 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.group_dir_rename 2024-09-12T23:57:29.250 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.group_dir_rename (18 ms) 2024-09-12T23:57:29.250 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.group_dir_remove 2024-09-12T23:57:29.260 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.group_dir_remove (10 ms) 2024-09-12T23:57:29.260 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2024-09-12T23:57:29.272 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (13 ms) 2024-09-12T23:57:29.273 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.group_image_add 2024-09-12T23:57:29.280 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.group_image_add (5 ms) 2024-09-12T23:57:29.280 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.group_image_remove 2024-09-12T23:57:29.286 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.group_image_remove (9 ms) 2024-09-12T23:57:29.287 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.group_image_list 2024-09-12T23:57:29.294 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.group_image_list (7 ms) 2024-09-12T23:57:29.294 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.group_image_clean 2024-09-12T23:57:29.307 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.group_image_clean (13 ms) 2024-09-12T23:57:29.307 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.image_group_add 2024-09-12T23:57:29.313 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.image_group_add (6 ms) 2024-09-12T23:57:29.313 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.image_group_remove 2024-09-12T23:57:29.322 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.image_group_remove (9 ms) 2024-09-12T23:57:29.322 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.image_group_get 2024-09-12T23:57:29.328 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.image_group_get (6 ms) 2024-09-12T23:57:29.328 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2024-09-12T23:57:29.333 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (5 ms) 2024-09-12T23:57:29.333 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2024-09-12T23:57:29.338 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (5 ms) 2024-09-12T23:57:29.338 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2024-09-12T23:57:29.345 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (7 ms) 2024-09-12T23:57:29.345 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2024-09-12T23:57:29.352 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (7 ms) 2024-09-12T23:57:29.352 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.group_snap_set 2024-09-12T23:57:29.358 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.group_snap_set (6 ms) 2024-09-12T23:57:29.358 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.group_snap_list 2024-09-12T23:57:29.366 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.group_snap_list (8 ms) 2024-09-12T23:57:29.366 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2024-09-12T23:57:29.409 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (43 ms) 2024-09-12T23:57:29.409 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2024-09-12T23:57:29.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:57:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:57:29] "GET /metrics HTTP/1.1" 200 36482 "" "Prometheus/2.43.0" 2024-09-12T23:57:29.958 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (548 ms) 2024-09-12T23:57:29.958 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.group_snap_remove 2024-09-12T23:57:29.966 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.group_snap_remove (9 ms) 2024-09-12T23:57:29.966 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2024-09-12T23:57:29.971 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (5 ms) 2024-09-12T23:57:29.972 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.trash_methods 2024-09-12T23:57:29.992 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.trash_methods (21 ms) 2024-09-12T23:57:29.992 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.op_features 2024-09-12T23:57:30.010 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.op_features (18 ms) 2024-09-12T23:57:30.010 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.clone_parent 2024-09-12T23:57:30.065 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.clone_parent (55 ms) 2024-09-12T23:57:30.066 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2024-09-12T23:57:30.090 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.clone_parent_ns (25 ms) 2024-09-12T23:57:30.090 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.clone_child 2024-09-12T23:57:30.122 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.clone_child (32 ms) 2024-09-12T23:57:30.123 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.namespace_methods 2024-09-12T23:57:30.140 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.namespace_methods (18 ms) 2024-09-12T23:57:30.140 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.migration 2024-09-12T23:57:30.168 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.migration (27 ms) 2024-09-12T23:57:30.168 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.migration_v1 2024-09-12T23:57:30.193 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.migration_v1 (26 ms) 2024-09-12T23:57:30.193 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2024-09-12T23:57:30.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:30 smithi136 bash[75164]: cluster 2024-09-12T23:57:29.036516+0000 mgr.y (mgr.24661) 611 : cluster [DBG] pgmap v1212: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 1.5 MiB/s rd, 807 KiB/s wr, 777 op/s 2024-09-12T23:57:30.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:30 smithi136 bash[76371]: cluster 2024-09-12T23:57:29.036516+0000 mgr.y (mgr.24661) 611 : cluster [DBG] pgmap v1212: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 1.5 MiB/s rd, 807 KiB/s wr, 777 op/s 2024-09-12T23:57:30.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:30 smithi181 bash[67587]: cluster 2024-09-12T23:57:29.036516+0000 mgr.y (mgr.24661) 611 : cluster [DBG] pgmap v1212: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 1.5 MiB/s rd, 807 KiB/s wr, 777 op/s 2024-09-12T23:57:31.373 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1180 ms) 2024-09-12T23:57:31.373 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestClsRbd.sparsify 2024-09-12T23:57:31.455 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestClsRbd.sparsify (82 ms) 2024-09-12T23:57:31.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:31 smithi136 bash[75164]: cluster 2024-09-12T23:57:30.347137+0000 mon.a (mon.0) 1872 : cluster [DBG] osdmap e1041: 8 total, 8 up, 8 in 2024-09-12T23:57:31.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:31 smithi136 bash[76371]: cluster 2024-09-12T23:57:30.347137+0000 mon.a (mon.0) 1872 : cluster [DBG] osdmap e1041: 8 total, 8 up, 8 in 2024-09-12T23:57:31.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:31 smithi181 bash[67587]: cluster 2024-09-12T23:57:30.347137+0000 mon.a (mon.0) 1872 : cluster [DBG] osdmap e1041: 8 total, 8 up, 8 in 2024-09-12T23:57:32.380 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 70 tests from TestClsRbd (10174 ms total) 2024-09-12T23:57:32.380 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-12T23:57:32.380 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] Global test environment tear-down 2024-09-12T23:57:32.381 INFO:tasks.workunit.client.0.smithi136.stdout:[==========] 70 tests from 1 test suite ran. (12974 ms total) 2024-09-12T23:57:32.381 INFO:tasks.workunit.client.0.smithi136.stdout:[ PASSED ] 70 tests. 2024-09-12T23:57:32.383 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-12T23:57:32.383 DEBUG:teuthology.orchestra.run.smithi136:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-12T23:57:32.435 INFO:tasks.workunit:Running workunit cls/test_cls_refcount.sh... 2024-09-12T23:57:32.435 DEBUG:teuthology.orchestra.run.smithi136:workunit test cls/test_cls_refcount.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_refcount.sh 2024-09-12T23:57:32.494 INFO:tasks.workunit.client.0.smithi136.stdout:Running main() from gmock_main.cc 2024-09-12T23:57:32.494 INFO:tasks.workunit.client.0.smithi136.stdout:[==========] Running 10 tests from 1 test suite. 2024-09-12T23:57:32.494 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] Global test environment set-up. 2024-09-12T23:57:32.494 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 10 tests from cls_rgw 2024-09-12T23:57:32.494 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] cls_rgw.test_implicit 2024-09-12T23:57:32.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:57:32 smithi181 bash[95505]: ts=2024-09-12T23:57:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:57:32.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:32 smithi181 bash[67587]: cluster 2024-09-12T23:57:31.037567+0000 mgr.y (mgr.24661) 612 : cluster [DBG] pgmap v1214: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 1.3 MiB/s rd, 740 KiB/s wr, 714 op/s 2024-09-12T23:57:32.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:32 smithi181 bash[67587]: cluster 2024-09-12T23:57:31.374963+0000 mon.a (mon.0) 1873 : cluster [DBG] osdmap e1042: 8 total, 8 up, 8 in 2024-09-12T23:57:32.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:32 smithi181 bash[67587]: audit 2024-09-12T23:57:32.349929+0000 mon.a (mon.0) 1874 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:57:32.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:32 smithi136 bash[75164]: cluster 2024-09-12T23:57:31.037567+0000 mgr.y (mgr.24661) 612 : cluster [DBG] pgmap v1214: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 1.3 MiB/s rd, 740 KiB/s wr, 714 op/s 2024-09-12T23:57:32.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:32 smithi136 bash[75164]: cluster 2024-09-12T23:57:31.374963+0000 mon.a (mon.0) 1873 : cluster [DBG] osdmap e1042: 8 total, 8 up, 8 in 2024-09-12T23:57:32.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:32 smithi136 bash[75164]: audit 2024-09-12T23:57:32.349929+0000 mon.a (mon.0) 1874 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:57:32.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:32 smithi136 bash[76371]: cluster 2024-09-12T23:57:31.037567+0000 mgr.y (mgr.24661) 612 : cluster [DBG] pgmap v1214: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 1.3 MiB/s rd, 740 KiB/s wr, 714 op/s 2024-09-12T23:57:32.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:32 smithi136 bash[76371]: cluster 2024-09-12T23:57:31.374963+0000 mon.a (mon.0) 1873 : cluster [DBG] osdmap e1042: 8 total, 8 up, 8 in 2024-09-12T23:57:32.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:32 smithi136 bash[76371]: audit 2024-09-12T23:57:32.349929+0000 mon.a (mon.0) 1874 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:57:33.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:33 smithi136 bash[75164]: cluster 2024-09-12T23:57:32.374579+0000 mon.a (mon.0) 1875 : cluster [DBG] osdmap e1043: 8 total, 8 up, 8 in 2024-09-12T23:57:33.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:33 smithi136 bash[75164]: audit 2024-09-12T23:57:32.949156+0000 mon.a (mon.0) 1876 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:57:33.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:33 smithi136 bash[75164]: audit 2024-09-12T23:57:32.950662+0000 mon.a (mon.0) 1877 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:57:33.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:33 smithi136 bash[75164]: audit 2024-09-12T23:57:32.961197+0000 mon.a (mon.0) 1878 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:57:33.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:33 smithi136 bash[76371]: cluster 2024-09-12T23:57:32.374579+0000 mon.a (mon.0) 1875 : cluster [DBG] osdmap e1043: 8 total, 8 up, 8 in 2024-09-12T23:57:33.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:33 smithi136 bash[76371]: audit 2024-09-12T23:57:32.949156+0000 mon.a (mon.0) 1876 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:57:33.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:33 smithi136 bash[76371]: audit 2024-09-12T23:57:32.950662+0000 mon.a (mon.0) 1877 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:57:33.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:33 smithi136 bash[76371]: audit 2024-09-12T23:57:32.961197+0000 mon.a (mon.0) 1878 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:57:33.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:33 smithi181 bash[67587]: cluster 2024-09-12T23:57:32.374579+0000 mon.a (mon.0) 1875 : cluster [DBG] osdmap e1043: 8 total, 8 up, 8 in 2024-09-12T23:57:33.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:33 smithi181 bash[67587]: audit 2024-09-12T23:57:32.949156+0000 mon.a (mon.0) 1876 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:57:33.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:33 smithi181 bash[67587]: audit 2024-09-12T23:57:32.950662+0000 mon.a (mon.0) 1877 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:57:33.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:33 smithi181 bash[67587]: audit 2024-09-12T23:57:32.961197+0000 mon.a (mon.0) 1878 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:57:34.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:34 smithi136 bash[75164]: cluster 2024-09-12T23:57:33.038341+0000 mgr.y (mgr.24661) 613 : cluster [DBG] pgmap v1217: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:34.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:34 smithi136 bash[75164]: cluster 2024-09-12T23:57:33.398593+0000 mon.a (mon.0) 1879 : cluster [DBG] osdmap e1044: 8 total, 8 up, 8 in 2024-09-12T23:57:34.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:34 smithi136 bash[75164]: audit 2024-09-12T23:57:33.402955+0000 mon.c (mon.2) 56 : audit [INF] from='client.? 172.21.15.136:0/3223847618' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:34.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:34 smithi136 bash[75164]: audit 2024-09-12T23:57:33.403894+0000 mon.a (mon.0) 1880 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:34.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:34 smithi136 bash[76371]: cluster 2024-09-12T23:57:33.038341+0000 mgr.y (mgr.24661) 613 : cluster [DBG] pgmap v1217: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:34.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:34 smithi136 bash[76371]: cluster 2024-09-12T23:57:33.398593+0000 mon.a (mon.0) 1879 : cluster [DBG] osdmap e1044: 8 total, 8 up, 8 in 2024-09-12T23:57:34.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:34 smithi136 bash[76371]: audit 2024-09-12T23:57:33.402955+0000 mon.c (mon.2) 56 : audit [INF] from='client.? 172.21.15.136:0/3223847618' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:34.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:34 smithi136 bash[76371]: audit 2024-09-12T23:57:33.403894+0000 mon.a (mon.0) 1880 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:34.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:34 smithi181 bash[67587]: cluster 2024-09-12T23:57:33.038341+0000 mgr.y (mgr.24661) 613 : cluster [DBG] pgmap v1217: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:34.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:34 smithi181 bash[67587]: cluster 2024-09-12T23:57:33.398593+0000 mon.a (mon.0) 1879 : cluster [DBG] osdmap e1044: 8 total, 8 up, 8 in 2024-09-12T23:57:34.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:34 smithi181 bash[67587]: audit 2024-09-12T23:57:33.402955+0000 mon.c (mon.2) 56 : audit [INF] from='client.? 172.21.15.136:0/3223847618' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:34.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:34 smithi181 bash[67587]: audit 2024-09-12T23:57:33.403894+0000 mon.a (mon.0) 1880 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:35.429 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] cls_rgw.test_implicit (2935 ms) 2024-09-12T23:57:35.430 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] cls_rgw.test_implicit_idempotent 2024-09-12T23:57:35.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:35 smithi136 bash[75164]: audit 2024-09-12T23:57:34.401672+0000 mon.a (mon.0) 1881 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:57:35.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:35 smithi136 bash[75164]: cluster 2024-09-12T23:57:34.402239+0000 mon.a (mon.0) 1882 : cluster [DBG] osdmap e1045: 8 total, 8 up, 8 in 2024-09-12T23:57:35.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:35 smithi136 bash[76371]: audit 2024-09-12T23:57:34.401672+0000 mon.a (mon.0) 1881 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:57:35.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:35 smithi136 bash[76371]: cluster 2024-09-12T23:57:34.402239+0000 mon.a (mon.0) 1882 : cluster [DBG] osdmap e1045: 8 total, 8 up, 8 in 2024-09-12T23:57:35.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:35 smithi181 bash[67587]: audit 2024-09-12T23:57:34.401672+0000 mon.a (mon.0) 1881 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:57:35.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:35 smithi181 bash[67587]: cluster 2024-09-12T23:57:34.402239+0000 mon.a (mon.0) 1882 : cluster [DBG] osdmap e1045: 8 total, 8 up, 8 in 2024-09-12T23:57:36.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:36 smithi136 bash[75164]: cluster 2024-09-12T23:57:35.039078+0000 mgr.y (mgr.24661) 614 : cluster [DBG] pgmap v1220: 137 pgs: 3 creating+peering, 28 unknown, 106 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:36.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:36 smithi136 bash[75164]: cluster 2024-09-12T23:57:35.426083+0000 mon.a (mon.0) 1883 : cluster [DBG] osdmap e1046: 8 total, 8 up, 8 in 2024-09-12T23:57:36.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:36 smithi136 bash[76371]: cluster 2024-09-12T23:57:35.039078+0000 mgr.y (mgr.24661) 614 : cluster [DBG] pgmap v1220: 137 pgs: 3 creating+peering, 28 unknown, 106 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:36.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:36 smithi136 bash[76371]: cluster 2024-09-12T23:57:35.426083+0000 mon.a (mon.0) 1883 : cluster [DBG] osdmap e1046: 8 total, 8 up, 8 in 2024-09-12T23:57:36.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:36 smithi181 bash[67587]: cluster 2024-09-12T23:57:35.039078+0000 mgr.y (mgr.24661) 614 : cluster [DBG] pgmap v1220: 137 pgs: 3 creating+peering, 28 unknown, 106 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:36.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:36 smithi181 bash[67587]: cluster 2024-09-12T23:57:35.426083+0000 mon.a (mon.0) 1883 : cluster [DBG] osdmap e1046: 8 total, 8 up, 8 in 2024-09-12T23:57:37.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:37 smithi136 bash[75164]: cluster 2024-09-12T23:57:36.458899+0000 mon.a (mon.0) 1884 : cluster [DBG] osdmap e1047: 8 total, 8 up, 8 in 2024-09-12T23:57:37.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:37 smithi136 bash[75164]: audit 2024-09-12T23:57:36.464412+0000 mon.c (mon.2) 57 : audit [INF] from='client.? 172.21.15.136:0/1344656936' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:37.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:37 smithi136 bash[75164]: audit 2024-09-12T23:57:36.465409+0000 mon.a (mon.0) 1885 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:37.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:37 smithi136 bash[76371]: cluster 2024-09-12T23:57:36.458899+0000 mon.a (mon.0) 1884 : cluster [DBG] osdmap e1047: 8 total, 8 up, 8 in 2024-09-12T23:57:37.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:37 smithi136 bash[76371]: audit 2024-09-12T23:57:36.464412+0000 mon.c (mon.2) 57 : audit [INF] from='client.? 172.21.15.136:0/1344656936' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:37.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:37 smithi136 bash[76371]: audit 2024-09-12T23:57:36.465409+0000 mon.a (mon.0) 1885 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:37.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:37 smithi181 bash[67587]: cluster 2024-09-12T23:57:36.458899+0000 mon.a (mon.0) 1884 : cluster [DBG] osdmap e1047: 8 total, 8 up, 8 in 2024-09-12T23:57:37.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:37 smithi181 bash[67587]: audit 2024-09-12T23:57:36.464412+0000 mon.c (mon.2) 57 : audit [INF] from='client.? 172.21.15.136:0/1344656936' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:37.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:37 smithi181 bash[67587]: audit 2024-09-12T23:57:36.465409+0000 mon.a (mon.0) 1885 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:38.475 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] cls_rgw.test_implicit_idempotent (3046 ms) 2024-09-12T23:57:38.475 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] cls_rgw.test_put_snap 2024-09-12T23:57:38.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:38 smithi136 bash[75164]: cluster 2024-09-12T23:57:37.039994+0000 mgr.y (mgr.24661) 615 : cluster [DBG] pgmap v1223: 137 pgs: 7 creating+peering, 25 unknown, 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:38.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:38 smithi136 bash[75164]: cluster 2024-09-12T23:57:37.451045+0000 mon.a (mon.0) 1886 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:57:38.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:38 smithi136 bash[75164]: audit 2024-09-12T23:57:37.469999+0000 mon.a (mon.0) 1887 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:57:38.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:38 smithi136 bash[75164]: cluster 2024-09-12T23:57:37.470786+0000 mon.a (mon.0) 1888 : cluster [DBG] osdmap e1048: 8 total, 8 up, 8 in 2024-09-12T23:57:38.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:38 smithi136 bash[76371]: cluster 2024-09-12T23:57:37.039994+0000 mgr.y (mgr.24661) 615 : cluster [DBG] pgmap v1223: 137 pgs: 7 creating+peering, 25 unknown, 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:38.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:38 smithi136 bash[76371]: cluster 2024-09-12T23:57:37.451045+0000 mon.a (mon.0) 1886 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:57:38.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:38 smithi136 bash[76371]: audit 2024-09-12T23:57:37.469999+0000 mon.a (mon.0) 1887 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:57:38.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:38 smithi136 bash[76371]: cluster 2024-09-12T23:57:37.470786+0000 mon.a (mon.0) 1888 : cluster [DBG] osdmap e1048: 8 total, 8 up, 8 in 2024-09-12T23:57:38.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:38 smithi181 bash[67587]: cluster 2024-09-12T23:57:37.039994+0000 mgr.y (mgr.24661) 615 : cluster [DBG] pgmap v1223: 137 pgs: 7 creating+peering, 25 unknown, 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:38.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:38 smithi181 bash[67587]: cluster 2024-09-12T23:57:37.451045+0000 mon.a (mon.0) 1886 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:57:38.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:38 smithi181 bash[67587]: audit 2024-09-12T23:57:37.469999+0000 mon.a (mon.0) 1887 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:57:38.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:38 smithi181 bash[67587]: cluster 2024-09-12T23:57:37.470786+0000 mon.a (mon.0) 1888 : cluster [DBG] osdmap e1048: 8 total, 8 up, 8 in 2024-09-12T23:57:39.756 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:39 smithi136 bash[75164]: cluster 2024-09-12T23:57:38.469170+0000 mon.a (mon.0) 1889 : cluster [DBG] osdmap e1049: 8 total, 8 up, 8 in 2024-09-12T23:57:39.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:39 smithi136 bash[75164]: audit 2024-09-12T23:57:38.940387+0000 mon.a (mon.0) 1890 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:57:39.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:39 smithi136 bash[75164]: audit 2024-09-12T23:57:38.941531+0000 mon.a (mon.0) 1891 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:57:39.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:57:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:57:39] "GET /metrics HTTP/1.1" 200 34086 "" "Prometheus/2.43.0" 2024-09-12T23:57:39.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:39 smithi136 bash[76371]: cluster 2024-09-12T23:57:38.469170+0000 mon.a (mon.0) 1889 : cluster [DBG] osdmap e1049: 8 total, 8 up, 8 in 2024-09-12T23:57:39.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:39 smithi136 bash[76371]: audit 2024-09-12T23:57:38.940387+0000 mon.a (mon.0) 1890 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:57:39.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:39 smithi136 bash[76371]: audit 2024-09-12T23:57:38.941531+0000 mon.a (mon.0) 1891 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:57:39.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:39 smithi181 bash[67587]: cluster 2024-09-12T23:57:38.469170+0000 mon.a (mon.0) 1889 : cluster [DBG] osdmap e1049: 8 total, 8 up, 8 in 2024-09-12T23:57:39.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:39 smithi181 bash[67587]: audit 2024-09-12T23:57:38.940387+0000 mon.a (mon.0) 1890 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:57:39.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:39 smithi181 bash[67587]: audit 2024-09-12T23:57:38.941531+0000 mon.a (mon.0) 1891 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:57:40.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:40 smithi136 bash[75164]: cluster 2024-09-12T23:57:39.040730+0000 mgr.y (mgr.24661) 616 : cluster [DBG] pgmap v1226: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:40.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:40 smithi136 bash[75164]: cluster 2024-09-12T23:57:39.495165+0000 mon.a (mon.0) 1892 : cluster [DBG] osdmap e1050: 8 total, 8 up, 8 in 2024-09-12T23:57:40.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:40 smithi136 bash[75164]: audit 2024-09-12T23:57:39.505398+0000 mon.c (mon.2) 58 : audit [INF] from='client.? 172.21.15.136:0/149192092' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:40.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:40 smithi136 bash[75164]: audit 2024-09-12T23:57:39.506203+0000 mon.a (mon.0) 1893 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:40.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:40 smithi136 bash[75164]: cluster 2024-09-12T23:57:39.935031+0000 mon.a (mon.0) 1894 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:57:40.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:40 smithi136 bash[75164]: cluster 2024-09-12T23:57:39.935083+0000 mon.a (mon.0) 1895 : cluster [INF] Cluster is now healthy 2024-09-12T23:57:40.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:40 smithi136 bash[76371]: cluster 2024-09-12T23:57:39.040730+0000 mgr.y (mgr.24661) 616 : cluster [DBG] pgmap v1226: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:40.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:40 smithi136 bash[76371]: cluster 2024-09-12T23:57:39.495165+0000 mon.a (mon.0) 1892 : cluster [DBG] osdmap e1050: 8 total, 8 up, 8 in 2024-09-12T23:57:40.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:40 smithi136 bash[76371]: audit 2024-09-12T23:57:39.505398+0000 mon.c (mon.2) 58 : audit [INF] from='client.? 172.21.15.136:0/149192092' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:40.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:40 smithi136 bash[76371]: audit 2024-09-12T23:57:39.506203+0000 mon.a (mon.0) 1893 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:40.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:40 smithi136 bash[76371]: cluster 2024-09-12T23:57:39.935031+0000 mon.a (mon.0) 1894 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:57:40.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:40 smithi136 bash[76371]: cluster 2024-09-12T23:57:39.935083+0000 mon.a (mon.0) 1895 : cluster [INF] Cluster is now healthy 2024-09-12T23:57:40.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:40 smithi181 bash[67587]: cluster 2024-09-12T23:57:39.040730+0000 mgr.y (mgr.24661) 616 : cluster [DBG] pgmap v1226: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:40.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:40 smithi181 bash[67587]: cluster 2024-09-12T23:57:39.495165+0000 mon.a (mon.0) 1892 : cluster [DBG] osdmap e1050: 8 total, 8 up, 8 in 2024-09-12T23:57:40.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:40 smithi181 bash[67587]: audit 2024-09-12T23:57:39.505398+0000 mon.c (mon.2) 58 : audit [INF] from='client.? 172.21.15.136:0/149192092' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:40.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:40 smithi181 bash[67587]: audit 2024-09-12T23:57:39.506203+0000 mon.a (mon.0) 1893 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:40.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:40 smithi181 bash[67587]: cluster 2024-09-12T23:57:39.935031+0000 mon.a (mon.0) 1894 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:57:40.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:40 smithi181 bash[67587]: cluster 2024-09-12T23:57:39.935083+0000 mon.a (mon.0) 1895 : cluster [INF] Cluster is now healthy 2024-09-12T23:57:41.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:41 smithi181 bash[67587]: audit 2024-09-12T23:57:40.507587+0000 mon.a (mon.0) 1896 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:57:41.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:41 smithi181 bash[67587]: cluster 2024-09-12T23:57:40.507921+0000 mon.a (mon.0) 1897 : cluster [DBG] osdmap e1051: 8 total, 8 up, 8 in 2024-09-12T23:57:42.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:41 smithi136 bash[75164]: audit 2024-09-12T23:57:40.507587+0000 mon.a (mon.0) 1896 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:57:42.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:41 smithi136 bash[75164]: cluster 2024-09-12T23:57:40.507921+0000 mon.a (mon.0) 1897 : cluster [DBG] osdmap e1051: 8 total, 8 up, 8 in 2024-09-12T23:57:42.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:41 smithi136 bash[76371]: audit 2024-09-12T23:57:40.507587+0000 mon.a (mon.0) 1896 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:57:42.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:41 smithi136 bash[76371]: cluster 2024-09-12T23:57:40.507921+0000 mon.a (mon.0) 1897 : cluster [DBG] osdmap e1051: 8 total, 8 up, 8 in 2024-09-12T23:57:42.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:42 smithi181 bash[67587]: cluster 2024-09-12T23:57:41.041651+0000 mgr.y (mgr.24661) 617 : cluster [DBG] pgmap v1229: 137 pgs: 5 creating+peering, 22 unknown, 110 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:42.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:42 smithi181 bash[67587]: cluster 2024-09-12T23:57:41.518630+0000 mon.a (mon.0) 1898 : cluster [DBG] osdmap e1052: 8 total, 8 up, 8 in 2024-09-12T23:57:42.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:57:42 smithi181 bash[95505]: ts=2024-09-12T23:57:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:57:43.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:42 smithi136 bash[75164]: cluster 2024-09-12T23:57:41.041651+0000 mgr.y (mgr.24661) 617 : cluster [DBG] pgmap v1229: 137 pgs: 5 creating+peering, 22 unknown, 110 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:43.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:42 smithi136 bash[75164]: cluster 2024-09-12T23:57:41.518630+0000 mon.a (mon.0) 1898 : cluster [DBG] osdmap e1052: 8 total, 8 up, 8 in 2024-09-12T23:57:43.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:42 smithi136 bash[76371]: cluster 2024-09-12T23:57:41.041651+0000 mgr.y (mgr.24661) 617 : cluster [DBG] pgmap v1229: 137 pgs: 5 creating+peering, 22 unknown, 110 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:43.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:42 smithi136 bash[76371]: cluster 2024-09-12T23:57:41.518630+0000 mon.a (mon.0) 1898 : cluster [DBG] osdmap e1052: 8 total, 8 up, 8 in 2024-09-12T23:57:44.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:44 smithi181 bash[67587]: cluster 2024-09-12T23:57:43.042692+0000 mgr.y (mgr.24661) 618 : cluster [DBG] pgmap v1231: 137 pgs: 5 creating+peering, 132 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 224 B/s wr, 0 op/s 2024-09-12T23:57:45.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:44 smithi136 bash[75164]: cluster 2024-09-12T23:57:43.042692+0000 mgr.y (mgr.24661) 618 : cluster [DBG] pgmap v1231: 137 pgs: 5 creating+peering, 132 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 224 B/s wr, 0 op/s 2024-09-12T23:57:45.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:44 smithi136 bash[76371]: cluster 2024-09-12T23:57:43.042692+0000 mgr.y (mgr.24661) 618 : cluster [DBG] pgmap v1231: 137 pgs: 5 creating+peering, 132 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 224 B/s wr, 0 op/s 2024-09-12T23:57:45.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:45 smithi181 bash[67587]: cluster 2024-09-12T23:57:44.529914+0000 mon.a (mon.0) 1899 : cluster [DBG] osdmap e1053: 8 total, 8 up, 8 in 2024-09-12T23:57:46.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:45 smithi136 bash[75164]: cluster 2024-09-12T23:57:44.529914+0000 mon.a (mon.0) 1899 : cluster [DBG] osdmap e1053: 8 total, 8 up, 8 in 2024-09-12T23:57:46.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:45 smithi136 bash[76371]: cluster 2024-09-12T23:57:44.529914+0000 mon.a (mon.0) 1899 : cluster [DBG] osdmap e1053: 8 total, 8 up, 8 in 2024-09-12T23:57:46.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:46 smithi181 bash[67587]: cluster 2024-09-12T23:57:45.043537+0000 mgr.y (mgr.24661) 619 : cluster [DBG] pgmap v1233: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 184 B/s wr, 0 op/s 2024-09-12T23:57:46.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:46 smithi181 bash[67587]: cluster 2024-09-12T23:57:45.544110+0000 mon.a (mon.0) 1900 : cluster [DBG] osdmap e1054: 8 total, 8 up, 8 in 2024-09-12T23:57:47.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:46 smithi136 bash[75164]: cluster 2024-09-12T23:57:45.043537+0000 mgr.y (mgr.24661) 619 : cluster [DBG] pgmap v1233: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 184 B/s wr, 0 op/s 2024-09-12T23:57:47.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:46 smithi136 bash[75164]: cluster 2024-09-12T23:57:45.544110+0000 mon.a (mon.0) 1900 : cluster [DBG] osdmap e1054: 8 total, 8 up, 8 in 2024-09-12T23:57:47.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:46 smithi136 bash[76371]: cluster 2024-09-12T23:57:45.043537+0000 mgr.y (mgr.24661) 619 : cluster [DBG] pgmap v1233: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 184 B/s wr, 0 op/s 2024-09-12T23:57:47.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:46 smithi136 bash[76371]: cluster 2024-09-12T23:57:45.544110+0000 mon.a (mon.0) 1900 : cluster [DBG] osdmap e1054: 8 total, 8 up, 8 in 2024-09-12T23:57:47.569 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] cls_rgw.test_put_snap (9094 ms) 2024-09-12T23:57:47.569 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] cls_rgw.test_explicit 2024-09-12T23:57:47.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:47 smithi181 bash[67587]: cluster 2024-09-12T23:57:46.545652+0000 mon.a (mon.0) 1901 : cluster [DBG] osdmap e1055: 8 total, 8 up, 8 in 2024-09-12T23:57:48.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:47 smithi136 bash[75164]: cluster 2024-09-12T23:57:46.545652+0000 mon.a (mon.0) 1901 : cluster [DBG] osdmap e1055: 8 total, 8 up, 8 in 2024-09-12T23:57:48.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:47 smithi136 bash[76371]: cluster 2024-09-12T23:57:46.545652+0000 mon.a (mon.0) 1901 : cluster [DBG] osdmap e1055: 8 total, 8 up, 8 in 2024-09-12T23:57:48.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:48 smithi181 bash[67587]: cluster 2024-09-12T23:57:47.044469+0000 mgr.y (mgr.24661) 620 : cluster [DBG] pgmap v1236: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 185 B/s wr, 0 op/s 2024-09-12T23:57:48.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:48 smithi181 bash[67587]: cluster 2024-09-12T23:57:47.568311+0000 mon.a (mon.0) 1902 : cluster [DBG] osdmap e1056: 8 total, 8 up, 8 in 2024-09-12T23:57:49.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:48 smithi136 bash[75164]: cluster 2024-09-12T23:57:47.044469+0000 mgr.y (mgr.24661) 620 : cluster [DBG] pgmap v1236: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 185 B/s wr, 0 op/s 2024-09-12T23:57:49.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:48 smithi136 bash[75164]: cluster 2024-09-12T23:57:47.568311+0000 mon.a (mon.0) 1902 : cluster [DBG] osdmap e1056: 8 total, 8 up, 8 in 2024-09-12T23:57:49.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:48 smithi136 bash[76371]: cluster 2024-09-12T23:57:47.044469+0000 mgr.y (mgr.24661) 620 : cluster [DBG] pgmap v1236: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 185 B/s wr, 0 op/s 2024-09-12T23:57:49.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:48 smithi136 bash[76371]: cluster 2024-09-12T23:57:47.568311+0000 mon.a (mon.0) 1902 : cluster [DBG] osdmap e1056: 8 total, 8 up, 8 in 2024-09-12T23:57:49.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:57:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:57:49] "GET /metrics HTTP/1.1" 200 34086 "" "Prometheus/2.43.0" 2024-09-12T23:57:49.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:49 smithi136 bash[75164]: cluster 2024-09-12T23:57:48.581050+0000 mon.a (mon.0) 1903 : cluster [DBG] osdmap e1057: 8 total, 8 up, 8 in 2024-09-12T23:57:49.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:49 smithi136 bash[75164]: audit 2024-09-12T23:57:48.593755+0000 mon.c (mon.2) 59 : audit [INF] from='client.? 172.21.15.136:0/3086470535' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:49.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:49 smithi136 bash[75164]: audit 2024-09-12T23:57:48.595329+0000 mon.a (mon.0) 1904 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:49.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:49 smithi136 bash[76371]: cluster 2024-09-12T23:57:48.581050+0000 mon.a (mon.0) 1903 : cluster [DBG] osdmap e1057: 8 total, 8 up, 8 in 2024-09-12T23:57:49.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:49 smithi136 bash[76371]: audit 2024-09-12T23:57:48.593755+0000 mon.c (mon.2) 59 : audit [INF] from='client.? 172.21.15.136:0/3086470535' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:49.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:49 smithi136 bash[76371]: audit 2024-09-12T23:57:48.595329+0000 mon.a (mon.0) 1904 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:49.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:49 smithi181 bash[67587]: cluster 2024-09-12T23:57:48.581050+0000 mon.a (mon.0) 1903 : cluster [DBG] osdmap e1057: 8 total, 8 up, 8 in 2024-09-12T23:57:49.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:49 smithi181 bash[67587]: audit 2024-09-12T23:57:48.593755+0000 mon.c (mon.2) 59 : audit [INF] from='client.? 172.21.15.136:0/3086470535' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:49.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:49 smithi181 bash[67587]: audit 2024-09-12T23:57:48.595329+0000 mon.a (mon.0) 1904 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:50.595 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] cls_rgw.test_explicit (3026 ms) 2024-09-12T23:57:50.595 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] cls_rgw.set 2024-09-12T23:57:51.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:50 smithi136 bash[75164]: cluster 2024-09-12T23:57:49.045452+0000 mgr.y (mgr.24661) 621 : cluster [DBG] pgmap v1239: 137 pgs: 2 creating+peering, 30 unknown, 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:51.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:50 smithi136 bash[75164]: cluster 2024-09-12T23:57:49.572792+0000 mon.a (mon.0) 1905 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:57:51.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:50 smithi136 bash[75164]: audit 2024-09-12T23:57:49.596934+0000 mon.a (mon.0) 1906 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:57:51.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:50 smithi136 bash[75164]: cluster 2024-09-12T23:57:49.597651+0000 mon.a (mon.0) 1907 : cluster [DBG] osdmap e1058: 8 total, 8 up, 8 in 2024-09-12T23:57:51.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:50 smithi136 bash[76371]: cluster 2024-09-12T23:57:49.045452+0000 mgr.y (mgr.24661) 621 : cluster [DBG] pgmap v1239: 137 pgs: 2 creating+peering, 30 unknown, 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:51.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:50 smithi136 bash[76371]: cluster 2024-09-12T23:57:49.572792+0000 mon.a (mon.0) 1905 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:57:51.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:50 smithi136 bash[76371]: audit 2024-09-12T23:57:49.596934+0000 mon.a (mon.0) 1906 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:57:51.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:50 smithi136 bash[76371]: cluster 2024-09-12T23:57:49.597651+0000 mon.a (mon.0) 1907 : cluster [DBG] osdmap e1058: 8 total, 8 up, 8 in 2024-09-12T23:57:51.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:50 smithi181 bash[67587]: cluster 2024-09-12T23:57:49.045452+0000 mgr.y (mgr.24661) 621 : cluster [DBG] pgmap v1239: 137 pgs: 2 creating+peering, 30 unknown, 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:51.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:50 smithi181 bash[67587]: cluster 2024-09-12T23:57:49.572792+0000 mon.a (mon.0) 1905 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:57:51.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:50 smithi181 bash[67587]: audit 2024-09-12T23:57:49.596934+0000 mon.a (mon.0) 1906 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:57:51.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:50 smithi181 bash[67587]: cluster 2024-09-12T23:57:49.597651+0000 mon.a (mon.0) 1907 : cluster [DBG] osdmap e1058: 8 total, 8 up, 8 in 2024-09-12T23:57:52.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:51 smithi136 bash[75164]: cluster 2024-09-12T23:57:50.594120+0000 mon.a (mon.0) 1908 : cluster [DBG] osdmap e1059: 8 total, 8 up, 8 in 2024-09-12T23:57:52.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:51 smithi136 bash[76371]: cluster 2024-09-12T23:57:50.594120+0000 mon.a (mon.0) 1908 : cluster [DBG] osdmap e1059: 8 total, 8 up, 8 in 2024-09-12T23:57:52.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:51 smithi181 bash[67587]: cluster 2024-09-12T23:57:50.594120+0000 mon.a (mon.0) 1908 : cluster [DBG] osdmap e1059: 8 total, 8 up, 8 in 2024-09-12T23:57:52.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:57:52 smithi181 bash[95505]: ts=2024-09-12T23:57:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:57:53.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:52 smithi136 bash[75164]: cluster 2024-09-12T23:57:51.046390+0000 mgr.y (mgr.24661) 622 : cluster [DBG] pgmap v1242: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:53.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:52 smithi136 bash[75164]: cluster 2024-09-12T23:57:51.602428+0000 mon.a (mon.0) 1909 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:57:53.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:52 smithi136 bash[75164]: cluster 2024-09-12T23:57:51.602481+0000 mon.a (mon.0) 1910 : cluster [INF] Cluster is now healthy 2024-09-12T23:57:53.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:52 smithi136 bash[75164]: cluster 2024-09-12T23:57:51.621698+0000 mon.a (mon.0) 1911 : cluster [DBG] osdmap e1060: 8 total, 8 up, 8 in 2024-09-12T23:57:53.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:52 smithi136 bash[75164]: audit 2024-09-12T23:57:51.624857+0000 mon.c (mon.2) 60 : audit [INF] from='client.? 172.21.15.136:0/3351523301' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:53.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:52 smithi136 bash[75164]: audit 2024-09-12T23:57:51.626028+0000 mon.a (mon.0) 1912 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:53.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:52 smithi136 bash[76371]: cluster 2024-09-12T23:57:51.046390+0000 mgr.y (mgr.24661) 622 : cluster [DBG] pgmap v1242: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:53.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:52 smithi136 bash[76371]: cluster 2024-09-12T23:57:51.602428+0000 mon.a (mon.0) 1909 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:57:53.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:52 smithi136 bash[76371]: cluster 2024-09-12T23:57:51.602481+0000 mon.a (mon.0) 1910 : cluster [INF] Cluster is now healthy 2024-09-12T23:57:53.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:52 smithi136 bash[76371]: cluster 2024-09-12T23:57:51.621698+0000 mon.a (mon.0) 1911 : cluster [DBG] osdmap e1060: 8 total, 8 up, 8 in 2024-09-12T23:57:53.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:52 smithi136 bash[76371]: audit 2024-09-12T23:57:51.624857+0000 mon.c (mon.2) 60 : audit [INF] from='client.? 172.21.15.136:0/3351523301' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:53.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:52 smithi136 bash[76371]: audit 2024-09-12T23:57:51.626028+0000 mon.a (mon.0) 1912 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:53.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:52 smithi181 bash[67587]: cluster 2024-09-12T23:57:51.046390+0000 mgr.y (mgr.24661) 622 : cluster [DBG] pgmap v1242: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:53.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:52 smithi181 bash[67587]: cluster 2024-09-12T23:57:51.602428+0000 mon.a (mon.0) 1909 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:57:53.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:52 smithi181 bash[67587]: cluster 2024-09-12T23:57:51.602481+0000 mon.a (mon.0) 1910 : cluster [INF] Cluster is now healthy 2024-09-12T23:57:53.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:52 smithi181 bash[67587]: cluster 2024-09-12T23:57:51.621698+0000 mon.a (mon.0) 1911 : cluster [DBG] osdmap e1060: 8 total, 8 up, 8 in 2024-09-12T23:57:53.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:52 smithi181 bash[67587]: audit 2024-09-12T23:57:51.624857+0000 mon.c (mon.2) 60 : audit [INF] from='client.? 172.21.15.136:0/3351523301' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:53.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:52 smithi181 bash[67587]: audit 2024-09-12T23:57:51.626028+0000 mon.a (mon.0) 1912 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:53.668 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] cls_rgw.set (3072 ms) 2024-09-12T23:57:53.668 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] cls_rgw.test_implicit_ec 2024-09-12T23:57:54.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:53 smithi136 bash[75164]: audit 2024-09-12T23:57:52.624110+0000 mon.a (mon.0) 1913 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:57:54.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:53 smithi136 bash[75164]: cluster 2024-09-12T23:57:52.624609+0000 mon.a (mon.0) 1914 : cluster [DBG] osdmap e1061: 8 total, 8 up, 8 in 2024-09-12T23:57:54.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:53 smithi136 bash[76371]: audit 2024-09-12T23:57:52.624110+0000 mon.a (mon.0) 1913 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:57:54.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:53 smithi136 bash[76371]: cluster 2024-09-12T23:57:52.624609+0000 mon.a (mon.0) 1914 : cluster [DBG] osdmap e1061: 8 total, 8 up, 8 in 2024-09-12T23:57:54.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:53 smithi181 bash[67587]: audit 2024-09-12T23:57:52.624110+0000 mon.a (mon.0) 1913 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:57:54.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:53 smithi181 bash[67587]: cluster 2024-09-12T23:57:52.624609+0000 mon.a (mon.0) 1914 : cluster [DBG] osdmap e1061: 8 total, 8 up, 8 in 2024-09-12T23:57:55.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:54 smithi136 bash[75164]: cluster 2024-09-12T23:57:53.047279+0000 mgr.y (mgr.24661) 623 : cluster [DBG] pgmap v1245: 137 pgs: 2 creating+activating, 5 creating+peering, 25 unknown, 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:55.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:54 smithi136 bash[75164]: cluster 2024-09-12T23:57:53.663290+0000 mon.a (mon.0) 1915 : cluster [DBG] osdmap e1062: 8 total, 8 up, 8 in 2024-09-12T23:57:55.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:54 smithi136 bash[75164]: audit 2024-09-12T23:57:53.705425+0000 mon.a (mon.0) 1916 : audit [INF] from='client.? 172.21.15.136:0/151953135' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-6"}]: dispatch 2024-09-12T23:57:55.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:54 smithi136 bash[75164]: audit 2024-09-12T23:57:53.706346+0000 mon.a (mon.0) 1917 : audit [INF] from='client.? 172.21.15.136:0/151953135' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-6"}]: dispatch 2024-09-12T23:57:55.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:54 smithi136 bash[75164]: audit 2024-09-12T23:57:53.706761+0000 mon.a (mon.0) 1918 : audit [INF] from='client.? 172.21.15.136:0/151953135' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi136-110348-6", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-12T23:57:55.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:54 smithi136 bash[75164]: audit 2024-09-12T23:57:53.771324+0000 mon.a (mon.0) 1919 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:57:55.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:54 smithi136 bash[75164]: audit 2024-09-12T23:57:53.773427+0000 mon.a (mon.0) 1920 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:57:55.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:54 smithi136 bash[76371]: cluster 2024-09-12T23:57:53.047279+0000 mgr.y (mgr.24661) 623 : cluster [DBG] pgmap v1245: 137 pgs: 2 creating+activating, 5 creating+peering, 25 unknown, 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:55.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:54 smithi136 bash[76371]: cluster 2024-09-12T23:57:53.663290+0000 mon.a (mon.0) 1915 : cluster [DBG] osdmap e1062: 8 total, 8 up, 8 in 2024-09-12T23:57:55.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:54 smithi136 bash[76371]: audit 2024-09-12T23:57:53.705425+0000 mon.a (mon.0) 1916 : audit [INF] from='client.? 172.21.15.136:0/151953135' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-6"}]: dispatch 2024-09-12T23:57:55.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:54 smithi136 bash[76371]: audit 2024-09-12T23:57:53.706346+0000 mon.a (mon.0) 1917 : audit [INF] from='client.? 172.21.15.136:0/151953135' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-6"}]: dispatch 2024-09-12T23:57:55.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:54 smithi136 bash[76371]: audit 2024-09-12T23:57:53.706761+0000 mon.a (mon.0) 1918 : audit [INF] from='client.? 172.21.15.136:0/151953135' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi136-110348-6", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-12T23:57:55.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:54 smithi136 bash[76371]: audit 2024-09-12T23:57:53.771324+0000 mon.a (mon.0) 1919 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:57:55.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:54 smithi136 bash[76371]: audit 2024-09-12T23:57:53.773427+0000 mon.a (mon.0) 1920 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:57:55.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:54 smithi181 bash[67587]: cluster 2024-09-12T23:57:53.047279+0000 mgr.y (mgr.24661) 623 : cluster [DBG] pgmap v1245: 137 pgs: 2 creating+activating, 5 creating+peering, 25 unknown, 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:55.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:54 smithi181 bash[67587]: cluster 2024-09-12T23:57:53.663290+0000 mon.a (mon.0) 1915 : cluster [DBG] osdmap e1062: 8 total, 8 up, 8 in 2024-09-12T23:57:55.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:54 smithi181 bash[67587]: audit 2024-09-12T23:57:53.705425+0000 mon.a (mon.0) 1916 : audit [INF] from='client.? 172.21.15.136:0/151953135' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-6"}]: dispatch 2024-09-12T23:57:55.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:54 smithi181 bash[67587]: audit 2024-09-12T23:57:53.706346+0000 mon.a (mon.0) 1917 : audit [INF] from='client.? 172.21.15.136:0/151953135' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-6"}]: dispatch 2024-09-12T23:57:55.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:54 smithi181 bash[67587]: audit 2024-09-12T23:57:53.706761+0000 mon.a (mon.0) 1918 : audit [INF] from='client.? 172.21.15.136:0/151953135' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi136-110348-6", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-12T23:57:55.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:54 smithi181 bash[67587]: audit 2024-09-12T23:57:53.771324+0000 mon.a (mon.0) 1919 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:57:55.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:54 smithi181 bash[67587]: audit 2024-09-12T23:57:53.773427+0000 mon.a (mon.0) 1920 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:57:56.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:56 smithi136 bash[75164]: audit 2024-09-12T23:57:54.659601+0000 mon.a (mon.0) 1921 : audit [INF] from='client.? 172.21.15.136:0/151953135' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi136-110348-6", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2024-09-12T23:57:56.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:56 smithi136 bash[75164]: cluster 2024-09-12T23:57:54.659772+0000 mon.a (mon.0) 1922 : cluster [DBG] osdmap e1063: 8 total, 8 up, 8 in 2024-09-12T23:57:56.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:56 smithi136 bash[75164]: audit 2024-09-12T23:57:54.661682+0000 mon.a (mon.0) 1923 : audit [INF] from='client.? 172.21.15.136:0/151953135' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi136-110348-6", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi136-110348-6"}]: dispatch 2024-09-12T23:57:56.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:56 smithi136 bash[76371]: audit 2024-09-12T23:57:54.659601+0000 mon.a (mon.0) 1921 : audit [INF] from='client.? 172.21.15.136:0/151953135' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi136-110348-6", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2024-09-12T23:57:56.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:56 smithi136 bash[76371]: cluster 2024-09-12T23:57:54.659772+0000 mon.a (mon.0) 1922 : cluster [DBG] osdmap e1063: 8 total, 8 up, 8 in 2024-09-12T23:57:56.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:56 smithi136 bash[76371]: audit 2024-09-12T23:57:54.661682+0000 mon.a (mon.0) 1923 : audit [INF] from='client.? 172.21.15.136:0/151953135' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi136-110348-6", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi136-110348-6"}]: dispatch 2024-09-12T23:57:56.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:56 smithi181 bash[67587]: audit 2024-09-12T23:57:54.659601+0000 mon.a (mon.0) 1921 : audit [INF] from='client.? 172.21.15.136:0/151953135' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi136-110348-6", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2024-09-12T23:57:56.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:56 smithi181 bash[67587]: cluster 2024-09-12T23:57:54.659772+0000 mon.a (mon.0) 1922 : cluster [DBG] osdmap e1063: 8 total, 8 up, 8 in 2024-09-12T23:57:56.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:56 smithi181 bash[67587]: audit 2024-09-12T23:57:54.661682+0000 mon.a (mon.0) 1923 : audit [INF] from='client.? 172.21.15.136:0/151953135' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi136-110348-6", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi136-110348-6"}]: dispatch 2024-09-12T23:57:57.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:57 smithi136 bash[75164]: cluster 2024-09-12T23:57:55.048020+0000 mgr.y (mgr.24661) 624 : cluster [DBG] pgmap v1248: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:57.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:57 smithi136 bash[75164]: cluster 2024-09-12T23:57:56.102846+0000 mon.a (mon.0) 1924 : cluster [DBG] osdmap e1064: 8 total, 8 up, 8 in 2024-09-12T23:57:57.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:57 smithi136 bash[75164]: audit 2024-09-12T23:57:56.663664+0000 mon.a (mon.0) 1925 : audit [INF] from='client.? 172.21.15.136:0/151953135' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi136-110348-6", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi136-110348-6"}]': finished 2024-09-12T23:57:57.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:57 smithi136 bash[75164]: cluster 2024-09-12T23:57:56.663856+0000 mon.a (mon.0) 1926 : cluster [DBG] osdmap e1065: 8 total, 8 up, 8 in 2024-09-12T23:57:57.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:57 smithi136 bash[75164]: audit 2024-09-12T23:57:56.666634+0000 mon.a (mon.0) 1927 : audit [INF] from='client.? 172.21.15.136:0/151953135' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:57.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:57 smithi136 bash[76371]: cluster 2024-09-12T23:57:55.048020+0000 mgr.y (mgr.24661) 624 : cluster [DBG] pgmap v1248: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:57.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:57 smithi136 bash[76371]: cluster 2024-09-12T23:57:56.102846+0000 mon.a (mon.0) 1924 : cluster [DBG] osdmap e1064: 8 total, 8 up, 8 in 2024-09-12T23:57:57.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:57 smithi136 bash[76371]: audit 2024-09-12T23:57:56.663664+0000 mon.a (mon.0) 1925 : audit [INF] from='client.? 172.21.15.136:0/151953135' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi136-110348-6", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi136-110348-6"}]': finished 2024-09-12T23:57:57.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:57 smithi136 bash[76371]: cluster 2024-09-12T23:57:56.663856+0000 mon.a (mon.0) 1926 : cluster [DBG] osdmap e1065: 8 total, 8 up, 8 in 2024-09-12T23:57:57.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:57 smithi136 bash[76371]: audit 2024-09-12T23:57:56.666634+0000 mon.a (mon.0) 1927 : audit [INF] from='client.? 172.21.15.136:0/151953135' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:57.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:57 smithi181 bash[67587]: cluster 2024-09-12T23:57:55.048020+0000 mgr.y (mgr.24661) 624 : cluster [DBG] pgmap v1248: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:57.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:57 smithi181 bash[67587]: cluster 2024-09-12T23:57:56.102846+0000 mon.a (mon.0) 1924 : cluster [DBG] osdmap e1064: 8 total, 8 up, 8 in 2024-09-12T23:57:57.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:57 smithi181 bash[67587]: audit 2024-09-12T23:57:56.663664+0000 mon.a (mon.0) 1925 : audit [INF] from='client.? 172.21.15.136:0/151953135' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi136-110348-6", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi136-110348-6"}]': finished 2024-09-12T23:57:57.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:57 smithi181 bash[67587]: cluster 2024-09-12T23:57:56.663856+0000 mon.a (mon.0) 1926 : cluster [DBG] osdmap e1065: 8 total, 8 up, 8 in 2024-09-12T23:57:57.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:57 smithi181 bash[67587]: audit 2024-09-12T23:57:56.666634+0000 mon.a (mon.0) 1927 : audit [INF] from='client.? 172.21.15.136:0/151953135' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:57:58.506 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:58 smithi136 bash[76371]: cluster 2024-09-12T23:57:57.048983+0000 mgr.y (mgr.24661) 625 : cluster [DBG] pgmap v1251: 113 pgs: 2 creating+peering, 6 unknown, 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:58.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:58 smithi136 bash[76371]: cluster 2024-09-12T23:57:57.656838+0000 mon.a (mon.0) 1928 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:57:58.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:58 smithi136 bash[76371]: audit 2024-09-12T23:57:57.673029+0000 mon.a (mon.0) 1929 : audit [INF] from='client.? 172.21.15.136:0/151953135' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:57:58.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:58 smithi136 bash[76371]: cluster 2024-09-12T23:57:57.673835+0000 mon.a (mon.0) 1930 : cluster [DBG] osdmap e1066: 8 total, 8 up, 8 in 2024-09-12T23:57:58.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:58 smithi136 bash[75164]: cluster 2024-09-12T23:57:57.048983+0000 mgr.y (mgr.24661) 625 : cluster [DBG] pgmap v1251: 113 pgs: 2 creating+peering, 6 unknown, 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:58.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:58 smithi136 bash[75164]: cluster 2024-09-12T23:57:57.656838+0000 mon.a (mon.0) 1928 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:57:58.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:58 smithi136 bash[75164]: audit 2024-09-12T23:57:57.673029+0000 mon.a (mon.0) 1929 : audit [INF] from='client.? 172.21.15.136:0/151953135' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:57:58.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:58 smithi136 bash[75164]: cluster 2024-09-12T23:57:57.673835+0000 mon.a (mon.0) 1930 : cluster [DBG] osdmap e1066: 8 total, 8 up, 8 in 2024-09-12T23:57:58.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:58 smithi181 bash[67587]: cluster 2024-09-12T23:57:57.048983+0000 mgr.y (mgr.24661) 625 : cluster [DBG] pgmap v1251: 113 pgs: 2 creating+peering, 6 unknown, 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:57:58.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:58 smithi181 bash[67587]: cluster 2024-09-12T23:57:57.656838+0000 mon.a (mon.0) 1928 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:57:58.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:58 smithi181 bash[67587]: audit 2024-09-12T23:57:57.673029+0000 mon.a (mon.0) 1929 : audit [INF] from='client.? 172.21.15.136:0/151953135' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:57:58.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:58 smithi181 bash[67587]: cluster 2024-09-12T23:57:57.673835+0000 mon.a (mon.0) 1930 : cluster [DBG] osdmap e1066: 8 total, 8 up, 8 in 2024-09-12T23:57:59.682 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:57:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:57:59] "GET /metrics HTTP/1.1" 200 34080 "" "Prometheus/2.43.0" 2024-09-12T23:58:00.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:59 smithi136 bash[75164]: cluster 2024-09-12T23:57:58.678621+0000 mon.a (mon.0) 1931 : cluster [DBG] osdmap e1067: 8 total, 8 up, 8 in 2024-09-12T23:58:00.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:57:59 smithi136 bash[75164]: audit 2024-09-12T23:57:58.680280+0000 mon.a (mon.0) 1932 : audit [INF] from='client.? 172.21.15.136:0/151953135' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-6"}]: dispatch 2024-09-12T23:58:00.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:59 smithi136 bash[76371]: cluster 2024-09-12T23:57:58.678621+0000 mon.a (mon.0) 1931 : cluster [DBG] osdmap e1067: 8 total, 8 up, 8 in 2024-09-12T23:58:00.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:57:59 smithi136 bash[76371]: audit 2024-09-12T23:57:58.680280+0000 mon.a (mon.0) 1932 : audit [INF] from='client.? 172.21.15.136:0/151953135' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-6"}]: dispatch 2024-09-12T23:58:00.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:59 smithi181 bash[67587]: cluster 2024-09-12T23:57:58.678621+0000 mon.a (mon.0) 1931 : cluster [DBG] osdmap e1067: 8 total, 8 up, 8 in 2024-09-12T23:58:00.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:57:59 smithi181 bash[67587]: audit 2024-09-12T23:57:58.680280+0000 mon.a (mon.0) 1932 : audit [INF] from='client.? 172.21.15.136:0/151953135' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-6"}]: dispatch 2024-09-12T23:58:00.709 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] cls_rgw.test_implicit_ec (7041 ms) 2024-09-12T23:58:00.709 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] cls_rgw.test_implicit_idempotent_ec 2024-09-12T23:58:01.006 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:00 smithi136 bash[76371]: cluster 2024-09-12T23:57:59.049654+0000 mgr.y (mgr.24661) 626 : cluster [DBG] pgmap v1254: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:01.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:00 smithi136 bash[76371]: cluster 2024-09-12T23:57:59.671567+0000 mon.a (mon.0) 1933 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:58:01.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:00 smithi136 bash[76371]: cluster 2024-09-12T23:57:59.671641+0000 mon.a (mon.0) 1934 : cluster [INF] Cluster is now healthy 2024-09-12T23:58:01.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:00 smithi136 bash[76371]: audit 2024-09-12T23:57:59.693248+0000 mon.a (mon.0) 1935 : audit [INF] from='client.? 172.21.15.136:0/151953135' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-6"}]': finished 2024-09-12T23:58:01.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:00 smithi136 bash[76371]: cluster 2024-09-12T23:57:59.693410+0000 mon.a (mon.0) 1936 : cluster [DBG] osdmap e1068: 8 total, 8 up, 8 in 2024-09-12T23:58:01.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:00 smithi136 bash[76371]: audit 2024-09-12T23:57:59.694693+0000 mon.a (mon.0) 1937 : audit [INF] from='client.? 172.21.15.136:0/151953135' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-6"}]: dispatch 2024-09-12T23:58:01.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:00 smithi136 bash[75164]: cluster 2024-09-12T23:57:59.049654+0000 mgr.y (mgr.24661) 626 : cluster [DBG] pgmap v1254: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:01.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:00 smithi136 bash[75164]: cluster 2024-09-12T23:57:59.671567+0000 mon.a (mon.0) 1933 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:58:01.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:00 smithi136 bash[75164]: cluster 2024-09-12T23:57:59.671641+0000 mon.a (mon.0) 1934 : cluster [INF] Cluster is now healthy 2024-09-12T23:58:01.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:00 smithi136 bash[75164]: audit 2024-09-12T23:57:59.693248+0000 mon.a (mon.0) 1935 : audit [INF] from='client.? 172.21.15.136:0/151953135' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-6"}]': finished 2024-09-12T23:58:01.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:00 smithi136 bash[75164]: cluster 2024-09-12T23:57:59.693410+0000 mon.a (mon.0) 1936 : cluster [DBG] osdmap e1068: 8 total, 8 up, 8 in 2024-09-12T23:58:01.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:00 smithi136 bash[75164]: audit 2024-09-12T23:57:59.694693+0000 mon.a (mon.0) 1937 : audit [INF] from='client.? 172.21.15.136:0/151953135' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-6"}]: dispatch 2024-09-12T23:58:01.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:00 smithi181 bash[67587]: cluster 2024-09-12T23:57:59.049654+0000 mgr.y (mgr.24661) 626 : cluster [DBG] pgmap v1254: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:01.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:00 smithi181 bash[67587]: cluster 2024-09-12T23:57:59.671567+0000 mon.a (mon.0) 1933 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:58:01.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:00 smithi181 bash[67587]: cluster 2024-09-12T23:57:59.671641+0000 mon.a (mon.0) 1934 : cluster [INF] Cluster is now healthy 2024-09-12T23:58:01.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:00 smithi181 bash[67587]: audit 2024-09-12T23:57:59.693248+0000 mon.a (mon.0) 1935 : audit [INF] from='client.? 172.21.15.136:0/151953135' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-6"}]': finished 2024-09-12T23:58:01.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:00 smithi181 bash[67587]: cluster 2024-09-12T23:57:59.693410+0000 mon.a (mon.0) 1936 : cluster [DBG] osdmap e1068: 8 total, 8 up, 8 in 2024-09-12T23:58:01.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:00 smithi181 bash[67587]: audit 2024-09-12T23:57:59.694693+0000 mon.a (mon.0) 1937 : audit [INF] from='client.? 172.21.15.136:0/151953135' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-6"}]: dispatch 2024-09-12T23:58:02.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:01 smithi136 bash[75164]: audit 2024-09-12T23:58:00.708914+0000 mon.a (mon.0) 1938 : audit [INF] from='client.? 172.21.15.136:0/151953135' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-6"}]': finished 2024-09-12T23:58:02.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:01 smithi136 bash[75164]: cluster 2024-09-12T23:58:00.709256+0000 mon.a (mon.0) 1939 : cluster [DBG] osdmap e1069: 8 total, 8 up, 8 in 2024-09-12T23:58:02.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:01 smithi136 bash[75164]: audit 2024-09-12T23:58:00.750386+0000 mon.c (mon.2) 61 : audit [INF] from='client.? 172.21.15.136:0/1780199075' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-7"}]: dispatch 2024-09-12T23:58:02.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:01 smithi136 bash[75164]: audit 2024-09-12T23:58:00.750820+0000 mon.a (mon.0) 1940 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-7"}]: dispatch 2024-09-12T23:58:02.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:01 smithi136 bash[75164]: audit 2024-09-12T23:58:00.751503+0000 mon.c (mon.2) 62 : audit [INF] from='client.? 172.21.15.136:0/1780199075' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-7"}]: dispatch 2024-09-12T23:58:02.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:01 smithi136 bash[75164]: audit 2024-09-12T23:58:00.751868+0000 mon.a (mon.0) 1941 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-7"}]: dispatch 2024-09-12T23:58:02.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:01 smithi136 bash[75164]: audit 2024-09-12T23:58:00.752420+0000 mon.c (mon.2) 63 : audit [INF] from='client.? 172.21.15.136:0/1780199075' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi136-110348-7", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-12T23:58:02.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:01 smithi136 bash[75164]: audit 2024-09-12T23:58:00.752812+0000 mon.a (mon.0) 1942 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi136-110348-7", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-12T23:58:02.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:01 smithi136 bash[76371]: audit 2024-09-12T23:58:00.708914+0000 mon.a (mon.0) 1938 : audit [INF] from='client.? 172.21.15.136:0/151953135' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-6"}]': finished 2024-09-12T23:58:02.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:01 smithi136 bash[76371]: cluster 2024-09-12T23:58:00.709256+0000 mon.a (mon.0) 1939 : cluster [DBG] osdmap e1069: 8 total, 8 up, 8 in 2024-09-12T23:58:02.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:01 smithi136 bash[76371]: audit 2024-09-12T23:58:00.750386+0000 mon.c (mon.2) 61 : audit [INF] from='client.? 172.21.15.136:0/1780199075' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-7"}]: dispatch 2024-09-12T23:58:02.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:01 smithi136 bash[76371]: audit 2024-09-12T23:58:00.750820+0000 mon.a (mon.0) 1940 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-7"}]: dispatch 2024-09-12T23:58:02.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:01 smithi136 bash[76371]: audit 2024-09-12T23:58:00.751503+0000 mon.c (mon.2) 62 : audit [INF] from='client.? 172.21.15.136:0/1780199075' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-7"}]: dispatch 2024-09-12T23:58:02.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:01 smithi136 bash[76371]: audit 2024-09-12T23:58:00.751868+0000 mon.a (mon.0) 1941 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-7"}]: dispatch 2024-09-12T23:58:02.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:01 smithi136 bash[76371]: audit 2024-09-12T23:58:00.752420+0000 mon.c (mon.2) 63 : audit [INF] from='client.? 172.21.15.136:0/1780199075' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi136-110348-7", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-12T23:58:02.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:01 smithi136 bash[76371]: audit 2024-09-12T23:58:00.752812+0000 mon.a (mon.0) 1942 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi136-110348-7", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-12T23:58:02.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:01 smithi181 bash[67587]: audit 2024-09-12T23:58:00.708914+0000 mon.a (mon.0) 1938 : audit [INF] from='client.? 172.21.15.136:0/151953135' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-6"}]': finished 2024-09-12T23:58:02.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:01 smithi181 bash[67587]: cluster 2024-09-12T23:58:00.709256+0000 mon.a (mon.0) 1939 : cluster [DBG] osdmap e1069: 8 total, 8 up, 8 in 2024-09-12T23:58:02.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:01 smithi181 bash[67587]: audit 2024-09-12T23:58:00.750386+0000 mon.c (mon.2) 61 : audit [INF] from='client.? 172.21.15.136:0/1780199075' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-7"}]: dispatch 2024-09-12T23:58:02.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:01 smithi181 bash[67587]: audit 2024-09-12T23:58:00.750820+0000 mon.a (mon.0) 1940 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-7"}]: dispatch 2024-09-12T23:58:02.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:01 smithi181 bash[67587]: audit 2024-09-12T23:58:00.751503+0000 mon.c (mon.2) 62 : audit [INF] from='client.? 172.21.15.136:0/1780199075' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-7"}]: dispatch 2024-09-12T23:58:02.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:01 smithi181 bash[67587]: audit 2024-09-12T23:58:00.751868+0000 mon.a (mon.0) 1941 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-7"}]: dispatch 2024-09-12T23:58:02.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:01 smithi181 bash[67587]: audit 2024-09-12T23:58:00.752420+0000 mon.c (mon.2) 63 : audit [INF] from='client.? 172.21.15.136:0/1780199075' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi136-110348-7", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-12T23:58:02.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:01 smithi181 bash[67587]: audit 2024-09-12T23:58:00.752812+0000 mon.a (mon.0) 1942 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi136-110348-7", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-12T23:58:02.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:58:02 smithi181 bash[95505]: ts=2024-09-12T23:58:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:58:03.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:03 smithi136 bash[75164]: cluster 2024-09-12T23:58:01.050524+0000 mgr.y (mgr.24661) 627 : cluster [DBG] pgmap v1257: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:03.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:03 smithi136 bash[75164]: audit 2024-09-12T23:58:01.720337+0000 mon.a (mon.0) 1943 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi136-110348-7", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2024-09-12T23:58:03.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:03 smithi136 bash[75164]: cluster 2024-09-12T23:58:01.720551+0000 mon.a (mon.0) 1944 : cluster [DBG] osdmap e1070: 8 total, 8 up, 8 in 2024-09-12T23:58:03.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:03 smithi136 bash[75164]: audit 2024-09-12T23:58:01.723301+0000 mon.c (mon.2) 64 : audit [INF] from='client.? 172.21.15.136:0/1780199075' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi136-110348-7", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi136-110348-7"}]: dispatch 2024-09-12T23:58:03.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:03 smithi136 bash[75164]: audit 2024-09-12T23:58:01.723748+0000 mon.a (mon.0) 1945 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi136-110348-7", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi136-110348-7"}]: dispatch 2024-09-12T23:58:03.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:03 smithi136 bash[76371]: cluster 2024-09-12T23:58:01.050524+0000 mgr.y (mgr.24661) 627 : cluster [DBG] pgmap v1257: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:03.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:03 smithi136 bash[76371]: audit 2024-09-12T23:58:01.720337+0000 mon.a (mon.0) 1943 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi136-110348-7", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2024-09-12T23:58:03.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:03 smithi136 bash[76371]: cluster 2024-09-12T23:58:01.720551+0000 mon.a (mon.0) 1944 : cluster [DBG] osdmap e1070: 8 total, 8 up, 8 in 2024-09-12T23:58:03.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:03 smithi136 bash[76371]: audit 2024-09-12T23:58:01.723301+0000 mon.c (mon.2) 64 : audit [INF] from='client.? 172.21.15.136:0/1780199075' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi136-110348-7", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi136-110348-7"}]: dispatch 2024-09-12T23:58:03.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:03 smithi136 bash[76371]: audit 2024-09-12T23:58:01.723748+0000 mon.a (mon.0) 1945 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi136-110348-7", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi136-110348-7"}]: dispatch 2024-09-12T23:58:03.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:03 smithi181 bash[67587]: cluster 2024-09-12T23:58:01.050524+0000 mgr.y (mgr.24661) 627 : cluster [DBG] pgmap v1257: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:03.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:03 smithi181 bash[67587]: audit 2024-09-12T23:58:01.720337+0000 mon.a (mon.0) 1943 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi136-110348-7", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2024-09-12T23:58:03.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:03 smithi181 bash[67587]: cluster 2024-09-12T23:58:01.720551+0000 mon.a (mon.0) 1944 : cluster [DBG] osdmap e1070: 8 total, 8 up, 8 in 2024-09-12T23:58:03.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:03 smithi181 bash[67587]: audit 2024-09-12T23:58:01.723301+0000 mon.c (mon.2) 64 : audit [INF] from='client.? 172.21.15.136:0/1780199075' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi136-110348-7", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi136-110348-7"}]: dispatch 2024-09-12T23:58:03.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:03 smithi181 bash[67587]: audit 2024-09-12T23:58:01.723748+0000 mon.a (mon.0) 1945 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi136-110348-7", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi136-110348-7"}]: dispatch 2024-09-12T23:58:04.506 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:04 smithi136 bash[75164]: cluster 2024-09-12T23:58:03.051156+0000 mgr.y (mgr.24661) 628 : cluster [DBG] pgmap v1260: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:04.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:04 smithi136 bash[75164]: cluster 2024-09-12T23:58:03.158926+0000 mon.a (mon.0) 1946 : cluster [DBG] osdmap e1071: 8 total, 8 up, 8 in 2024-09-12T23:58:04.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:04 smithi136 bash[75164]: audit 2024-09-12T23:58:03.724141+0000 mon.a (mon.0) 1947 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi136-110348-7", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi136-110348-7"}]': finished 2024-09-12T23:58:04.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:04 smithi136 bash[75164]: cluster 2024-09-12T23:58:03.724382+0000 mon.a (mon.0) 1948 : cluster [DBG] osdmap e1072: 8 total, 8 up, 8 in 2024-09-12T23:58:04.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:04 smithi136 bash[75164]: audit 2024-09-12T23:58:03.733563+0000 mon.c (mon.2) 65 : audit [INF] from='client.? 172.21.15.136:0/1780199075' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:58:04.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:04 smithi136 bash[75164]: audit 2024-09-12T23:58:03.734418+0000 mon.a (mon.0) 1949 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:58:04.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:04 smithi136 bash[76371]: cluster 2024-09-12T23:58:03.051156+0000 mgr.y (mgr.24661) 628 : cluster [DBG] pgmap v1260: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:04.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:04 smithi136 bash[76371]: cluster 2024-09-12T23:58:03.158926+0000 mon.a (mon.0) 1946 : cluster [DBG] osdmap e1071: 8 total, 8 up, 8 in 2024-09-12T23:58:04.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:04 smithi136 bash[76371]: audit 2024-09-12T23:58:03.724141+0000 mon.a (mon.0) 1947 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi136-110348-7", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi136-110348-7"}]': finished 2024-09-12T23:58:04.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:04 smithi136 bash[76371]: cluster 2024-09-12T23:58:03.724382+0000 mon.a (mon.0) 1948 : cluster [DBG] osdmap e1072: 8 total, 8 up, 8 in 2024-09-12T23:58:04.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:04 smithi136 bash[76371]: audit 2024-09-12T23:58:03.733563+0000 mon.c (mon.2) 65 : audit [INF] from='client.? 172.21.15.136:0/1780199075' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:58:04.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:04 smithi136 bash[76371]: audit 2024-09-12T23:58:03.734418+0000 mon.a (mon.0) 1949 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:58:04.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:04 smithi181 bash[67587]: cluster 2024-09-12T23:58:03.051156+0000 mgr.y (mgr.24661) 628 : cluster [DBG] pgmap v1260: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:04.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:04 smithi181 bash[67587]: cluster 2024-09-12T23:58:03.158926+0000 mon.a (mon.0) 1946 : cluster [DBG] osdmap e1071: 8 total, 8 up, 8 in 2024-09-12T23:58:04.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:04 smithi181 bash[67587]: audit 2024-09-12T23:58:03.724141+0000 mon.a (mon.0) 1947 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi136-110348-7", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi136-110348-7"}]': finished 2024-09-12T23:58:04.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:04 smithi181 bash[67587]: cluster 2024-09-12T23:58:03.724382+0000 mon.a (mon.0) 1948 : cluster [DBG] osdmap e1072: 8 total, 8 up, 8 in 2024-09-12T23:58:04.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:04 smithi181 bash[67587]: audit 2024-09-12T23:58:03.733563+0000 mon.c (mon.2) 65 : audit [INF] from='client.? 172.21.15.136:0/1780199075' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:58:04.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:04 smithi181 bash[67587]: audit 2024-09-12T23:58:03.734418+0000 mon.a (mon.0) 1949 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:58:06.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:05 smithi136 bash[75164]: audit 2024-09-12T23:58:04.726702+0000 mon.a (mon.0) 1950 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:58:06.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:05 smithi136 bash[75164]: cluster 2024-09-12T23:58:04.727045+0000 mon.a (mon.0) 1951 : cluster [DBG] osdmap e1073: 8 total, 8 up, 8 in 2024-09-12T23:58:06.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:05 smithi136 bash[76371]: audit 2024-09-12T23:58:04.726702+0000 mon.a (mon.0) 1950 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:58:06.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:05 smithi136 bash[76371]: cluster 2024-09-12T23:58:04.727045+0000 mon.a (mon.0) 1951 : cluster [DBG] osdmap e1073: 8 total, 8 up, 8 in 2024-09-12T23:58:06.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:05 smithi181 bash[67587]: audit 2024-09-12T23:58:04.726702+0000 mon.a (mon.0) 1950 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:58:06.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:05 smithi181 bash[67587]: cluster 2024-09-12T23:58:04.727045+0000 mon.a (mon.0) 1951 : cluster [DBG] osdmap e1073: 8 total, 8 up, 8 in 2024-09-12T23:58:07.006 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:06 smithi136 bash[75164]: cluster 2024-09-12T23:58:05.051827+0000 mgr.y (mgr.24661) 629 : cluster [DBG] pgmap v1263: 113 pgs: 2 creating+peering, 6 unknown, 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:07.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:06 smithi136 bash[75164]: cluster 2024-09-12T23:58:05.737357+0000 mon.a (mon.0) 1952 : cluster [DBG] osdmap e1074: 8 total, 8 up, 8 in 2024-09-12T23:58:07.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:06 smithi136 bash[75164]: audit 2024-09-12T23:58:05.743047+0000 mon.c (mon.2) 66 : audit [INF] from='client.? 172.21.15.136:0/1780199075' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-7"}]: dispatch 2024-09-12T23:58:07.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:06 smithi136 bash[75164]: audit 2024-09-12T23:58:05.743866+0000 mon.a (mon.0) 1953 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-7"}]: dispatch 2024-09-12T23:58:07.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:06 smithi136 bash[76371]: cluster 2024-09-12T23:58:05.051827+0000 mgr.y (mgr.24661) 629 : cluster [DBG] pgmap v1263: 113 pgs: 2 creating+peering, 6 unknown, 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:07.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:06 smithi136 bash[76371]: cluster 2024-09-12T23:58:05.737357+0000 mon.a (mon.0) 1952 : cluster [DBG] osdmap e1074: 8 total, 8 up, 8 in 2024-09-12T23:58:07.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:06 smithi136 bash[76371]: audit 2024-09-12T23:58:05.743047+0000 mon.c (mon.2) 66 : audit [INF] from='client.? 172.21.15.136:0/1780199075' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-7"}]: dispatch 2024-09-12T23:58:07.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:06 smithi136 bash[76371]: audit 2024-09-12T23:58:05.743866+0000 mon.a (mon.0) 1953 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-7"}]: dispatch 2024-09-12T23:58:07.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:06 smithi181 bash[67587]: cluster 2024-09-12T23:58:05.051827+0000 mgr.y (mgr.24661) 629 : cluster [DBG] pgmap v1263: 113 pgs: 2 creating+peering, 6 unknown, 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:07.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:06 smithi181 bash[67587]: cluster 2024-09-12T23:58:05.737357+0000 mon.a (mon.0) 1952 : cluster [DBG] osdmap e1074: 8 total, 8 up, 8 in 2024-09-12T23:58:07.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:06 smithi181 bash[67587]: audit 2024-09-12T23:58:05.743047+0000 mon.c (mon.2) 66 : audit [INF] from='client.? 172.21.15.136:0/1780199075' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-7"}]: dispatch 2024-09-12T23:58:07.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:06 smithi181 bash[67587]: audit 2024-09-12T23:58:05.743866+0000 mon.a (mon.0) 1953 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-7"}]: dispatch 2024-09-12T23:58:07.759 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] cls_rgw.test_implicit_idempotent_ec (7050 ms) 2024-09-12T23:58:07.759 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] cls_rgw.test_put_snap_ec 2024-09-12T23:58:08.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:07 smithi136 bash[75164]: audit 2024-09-12T23:58:06.738739+0000 mon.a (mon.0) 1954 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-7"}]': finished 2024-09-12T23:58:08.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:07 smithi136 bash[75164]: cluster 2024-09-12T23:58:06.738993+0000 mon.a (mon.0) 1955 : cluster [DBG] osdmap e1075: 8 total, 8 up, 8 in 2024-09-12T23:58:08.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:07 smithi136 bash[75164]: audit 2024-09-12T23:58:06.740760+0000 mon.c (mon.2) 67 : audit [INF] from='client.? 172.21.15.136:0/1780199075' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-7"}]: dispatch 2024-09-12T23:58:08.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:07 smithi136 bash[75164]: audit 2024-09-12T23:58:06.748493+0000 mon.a (mon.0) 1956 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-7"}]: dispatch 2024-09-12T23:58:08.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:07 smithi136 bash[76371]: audit 2024-09-12T23:58:06.738739+0000 mon.a (mon.0) 1954 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-7"}]': finished 2024-09-12T23:58:08.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:07 smithi136 bash[76371]: cluster 2024-09-12T23:58:06.738993+0000 mon.a (mon.0) 1955 : cluster [DBG] osdmap e1075: 8 total, 8 up, 8 in 2024-09-12T23:58:08.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:07 smithi136 bash[76371]: audit 2024-09-12T23:58:06.740760+0000 mon.c (mon.2) 67 : audit [INF] from='client.? 172.21.15.136:0/1780199075' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-7"}]: dispatch 2024-09-12T23:58:08.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:07 smithi136 bash[76371]: audit 2024-09-12T23:58:06.748493+0000 mon.a (mon.0) 1956 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-7"}]: dispatch 2024-09-12T23:58:08.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:07 smithi181 bash[67587]: audit 2024-09-12T23:58:06.738739+0000 mon.a (mon.0) 1954 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-7"}]': finished 2024-09-12T23:58:08.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:07 smithi181 bash[67587]: cluster 2024-09-12T23:58:06.738993+0000 mon.a (mon.0) 1955 : cluster [DBG] osdmap e1075: 8 total, 8 up, 8 in 2024-09-12T23:58:08.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:07 smithi181 bash[67587]: audit 2024-09-12T23:58:06.740760+0000 mon.c (mon.2) 67 : audit [INF] from='client.? 172.21.15.136:0/1780199075' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-7"}]: dispatch 2024-09-12T23:58:08.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:07 smithi181 bash[67587]: audit 2024-09-12T23:58:06.748493+0000 mon.a (mon.0) 1956 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-7"}]: dispatch 2024-09-12T23:58:09.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:08 smithi181 bash[67587]: cluster 2024-09-12T23:58:07.052776+0000 mgr.y (mgr.24661) 630 : cluster [DBG] pgmap v1266: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:09.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:08 smithi181 bash[67587]: audit 2024-09-12T23:58:07.758711+0000 mon.a (mon.0) 1957 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-7"}]': finished 2024-09-12T23:58:09.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:08 smithi181 bash[67587]: cluster 2024-09-12T23:58:07.758903+0000 mon.a (mon.0) 1958 : cluster [DBG] osdmap e1076: 8 total, 8 up, 8 in 2024-09-12T23:58:09.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:08 smithi181 bash[67587]: audit 2024-09-12T23:58:07.785934+0000 mon.c (mon.2) 68 : audit [INF] from='client.? 172.21.15.136:0/3073703616' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-8"}]: dispatch 2024-09-12T23:58:09.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:08 smithi181 bash[67587]: audit 2024-09-12T23:58:07.786287+0000 mon.a (mon.0) 1959 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-8"}]: dispatch 2024-09-12T23:58:09.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:08 smithi181 bash[67587]: audit 2024-09-12T23:58:07.787010+0000 mon.c (mon.2) 69 : audit [INF] from='client.? 172.21.15.136:0/3073703616' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-8"}]: dispatch 2024-09-12T23:58:09.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:08 smithi181 bash[67587]: audit 2024-09-12T23:58:07.787283+0000 mon.a (mon.0) 1960 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-8"}]: dispatch 2024-09-12T23:58:09.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:08 smithi181 bash[67587]: audit 2024-09-12T23:58:07.787736+0000 mon.c (mon.2) 70 : audit [INF] from='client.? 172.21.15.136:0/3073703616' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi136-110348-8", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-12T23:58:09.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:08 smithi181 bash[67587]: audit 2024-09-12T23:58:07.788002+0000 mon.a (mon.0) 1961 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi136-110348-8", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-12T23:58:09.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:08 smithi136 bash[75164]: cluster 2024-09-12T23:58:07.052776+0000 mgr.y (mgr.24661) 630 : cluster [DBG] pgmap v1266: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:09.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:08 smithi136 bash[75164]: audit 2024-09-12T23:58:07.758711+0000 mon.a (mon.0) 1957 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-7"}]': finished 2024-09-12T23:58:09.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:08 smithi136 bash[75164]: cluster 2024-09-12T23:58:07.758903+0000 mon.a (mon.0) 1958 : cluster [DBG] osdmap e1076: 8 total, 8 up, 8 in 2024-09-12T23:58:09.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:08 smithi136 bash[75164]: audit 2024-09-12T23:58:07.785934+0000 mon.c (mon.2) 68 : audit [INF] from='client.? 172.21.15.136:0/3073703616' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-8"}]: dispatch 2024-09-12T23:58:09.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:08 smithi136 bash[75164]: audit 2024-09-12T23:58:07.786287+0000 mon.a (mon.0) 1959 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-8"}]: dispatch 2024-09-12T23:58:09.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:08 smithi136 bash[75164]: audit 2024-09-12T23:58:07.787010+0000 mon.c (mon.2) 69 : audit [INF] from='client.? 172.21.15.136:0/3073703616' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-8"}]: dispatch 2024-09-12T23:58:09.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:08 smithi136 bash[75164]: audit 2024-09-12T23:58:07.787283+0000 mon.a (mon.0) 1960 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-8"}]: dispatch 2024-09-12T23:58:09.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:08 smithi136 bash[75164]: audit 2024-09-12T23:58:07.787736+0000 mon.c (mon.2) 70 : audit [INF] from='client.? 172.21.15.136:0/3073703616' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi136-110348-8", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-12T23:58:09.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:08 smithi136 bash[75164]: audit 2024-09-12T23:58:07.788002+0000 mon.a (mon.0) 1961 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi136-110348-8", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-12T23:58:09.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:08 smithi136 bash[76371]: cluster 2024-09-12T23:58:07.052776+0000 mgr.y (mgr.24661) 630 : cluster [DBG] pgmap v1266: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:09.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:08 smithi136 bash[76371]: audit 2024-09-12T23:58:07.758711+0000 mon.a (mon.0) 1957 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-7"}]': finished 2024-09-12T23:58:09.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:08 smithi136 bash[76371]: cluster 2024-09-12T23:58:07.758903+0000 mon.a (mon.0) 1958 : cluster [DBG] osdmap e1076: 8 total, 8 up, 8 in 2024-09-12T23:58:09.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:08 smithi136 bash[76371]: audit 2024-09-12T23:58:07.785934+0000 mon.c (mon.2) 68 : audit [INF] from='client.? 172.21.15.136:0/3073703616' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-8"}]: dispatch 2024-09-12T23:58:09.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:08 smithi136 bash[76371]: audit 2024-09-12T23:58:07.786287+0000 mon.a (mon.0) 1959 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-8"}]: dispatch 2024-09-12T23:58:09.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:08 smithi136 bash[76371]: audit 2024-09-12T23:58:07.787010+0000 mon.c (mon.2) 69 : audit [INF] from='client.? 172.21.15.136:0/3073703616' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-8"}]: dispatch 2024-09-12T23:58:09.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:08 smithi136 bash[76371]: audit 2024-09-12T23:58:07.787283+0000 mon.a (mon.0) 1960 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-8"}]: dispatch 2024-09-12T23:58:09.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:08 smithi136 bash[76371]: audit 2024-09-12T23:58:07.787736+0000 mon.c (mon.2) 70 : audit [INF] from='client.? 172.21.15.136:0/3073703616' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi136-110348-8", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-12T23:58:09.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:08 smithi136 bash[76371]: audit 2024-09-12T23:58:07.788002+0000 mon.a (mon.0) 1961 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi136-110348-8", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-12T23:58:09.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:58:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:58:09] "GET /metrics HTTP/1.1" 200 34081 "" "Prometheus/2.43.0" 2024-09-12T23:58:10.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:09 smithi181 bash[67587]: audit 2024-09-12T23:58:08.759865+0000 mon.a (mon.0) 1962 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi136-110348-8", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2024-09-12T23:58:10.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:09 smithi181 bash[67587]: cluster 2024-09-12T23:58:08.760082+0000 mon.a (mon.0) 1963 : cluster [DBG] osdmap e1077: 8 total, 8 up, 8 in 2024-09-12T23:58:10.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:09 smithi181 bash[67587]: audit 2024-09-12T23:58:08.764230+0000 mon.c (mon.2) 71 : audit [INF] from='client.? 172.21.15.136:0/3073703616' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi136-110348-8", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi136-110348-8"}]: dispatch 2024-09-12T23:58:10.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:09 smithi181 bash[67587]: audit 2024-09-12T23:58:08.769652+0000 mon.a (mon.0) 1964 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:58:10.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:09 smithi181 bash[67587]: audit 2024-09-12T23:58:08.772208+0000 mon.a (mon.0) 1965 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi136-110348-8", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi136-110348-8"}]: dispatch 2024-09-12T23:58:10.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:09 smithi136 bash[75164]: audit 2024-09-12T23:58:08.759865+0000 mon.a (mon.0) 1962 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi136-110348-8", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2024-09-12T23:58:10.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:09 smithi136 bash[75164]: cluster 2024-09-12T23:58:08.760082+0000 mon.a (mon.0) 1963 : cluster [DBG] osdmap e1077: 8 total, 8 up, 8 in 2024-09-12T23:58:10.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:09 smithi136 bash[75164]: audit 2024-09-12T23:58:08.764230+0000 mon.c (mon.2) 71 : audit [INF] from='client.? 172.21.15.136:0/3073703616' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi136-110348-8", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi136-110348-8"}]: dispatch 2024-09-12T23:58:10.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:09 smithi136 bash[75164]: audit 2024-09-12T23:58:08.769652+0000 mon.a (mon.0) 1964 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:58:10.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:09 smithi136 bash[75164]: audit 2024-09-12T23:58:08.772208+0000 mon.a (mon.0) 1965 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi136-110348-8", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi136-110348-8"}]: dispatch 2024-09-12T23:58:10.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:09 smithi136 bash[76371]: audit 2024-09-12T23:58:08.759865+0000 mon.a (mon.0) 1962 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi136-110348-8", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2024-09-12T23:58:10.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:09 smithi136 bash[76371]: cluster 2024-09-12T23:58:08.760082+0000 mon.a (mon.0) 1963 : cluster [DBG] osdmap e1077: 8 total, 8 up, 8 in 2024-09-12T23:58:10.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:09 smithi136 bash[76371]: audit 2024-09-12T23:58:08.764230+0000 mon.c (mon.2) 71 : audit [INF] from='client.? 172.21.15.136:0/3073703616' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi136-110348-8", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi136-110348-8"}]: dispatch 2024-09-12T23:58:10.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:09 smithi136 bash[76371]: audit 2024-09-12T23:58:08.769652+0000 mon.a (mon.0) 1964 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:58:10.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:09 smithi136 bash[76371]: audit 2024-09-12T23:58:08.772208+0000 mon.a (mon.0) 1965 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi136-110348-8", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi136-110348-8"}]: dispatch 2024-09-12T23:58:11.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:10 smithi181 bash[67587]: cluster 2024-09-12T23:58:09.053410+0000 mgr.y (mgr.24661) 631 : cluster [DBG] pgmap v1269: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:11.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:10 smithi181 bash[67587]: cluster 2024-09-12T23:58:10.227001+0000 mon.a (mon.0) 1966 : cluster [DBG] osdmap e1078: 8 total, 8 up, 8 in 2024-09-12T23:58:11.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:10 smithi136 bash[75164]: cluster 2024-09-12T23:58:09.053410+0000 mgr.y (mgr.24661) 631 : cluster [DBG] pgmap v1269: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:11.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:10 smithi136 bash[75164]: cluster 2024-09-12T23:58:10.227001+0000 mon.a (mon.0) 1966 : cluster [DBG] osdmap e1078: 8 total, 8 up, 8 in 2024-09-12T23:58:11.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:10 smithi136 bash[76371]: cluster 2024-09-12T23:58:09.053410+0000 mgr.y (mgr.24661) 631 : cluster [DBG] pgmap v1269: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:11.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:10 smithi136 bash[76371]: cluster 2024-09-12T23:58:10.227001+0000 mon.a (mon.0) 1966 : cluster [DBG] osdmap e1078: 8 total, 8 up, 8 in 2024-09-12T23:58:12.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:11 smithi181 bash[67587]: audit 2024-09-12T23:58:10.797484+0000 mon.a (mon.0) 1967 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi136-110348-8", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi136-110348-8"}]': finished 2024-09-12T23:58:12.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:11 smithi181 bash[67587]: cluster 2024-09-12T23:58:10.797772+0000 mon.a (mon.0) 1968 : cluster [DBG] osdmap e1079: 8 total, 8 up, 8 in 2024-09-12T23:58:12.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:11 smithi181 bash[67587]: audit 2024-09-12T23:58:10.801379+0000 mon.c (mon.2) 72 : audit [INF] from='client.? 172.21.15.136:0/3073703616' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:58:12.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:11 smithi181 bash[67587]: audit 2024-09-12T23:58:10.802718+0000 mon.a (mon.0) 1969 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:58:12.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:11 smithi136 bash[75164]: audit 2024-09-12T23:58:10.797484+0000 mon.a (mon.0) 1967 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi136-110348-8", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi136-110348-8"}]': finished 2024-09-12T23:58:12.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:11 smithi136 bash[75164]: cluster 2024-09-12T23:58:10.797772+0000 mon.a (mon.0) 1968 : cluster [DBG] osdmap e1079: 8 total, 8 up, 8 in 2024-09-12T23:58:12.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:11 smithi136 bash[75164]: audit 2024-09-12T23:58:10.801379+0000 mon.c (mon.2) 72 : audit [INF] from='client.? 172.21.15.136:0/3073703616' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:58:12.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:11 smithi136 bash[75164]: audit 2024-09-12T23:58:10.802718+0000 mon.a (mon.0) 1969 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:58:12.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:11 smithi136 bash[76371]: audit 2024-09-12T23:58:10.797484+0000 mon.a (mon.0) 1967 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi136-110348-8", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi136-110348-8"}]': finished 2024-09-12T23:58:12.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:11 smithi136 bash[76371]: cluster 2024-09-12T23:58:10.797772+0000 mon.a (mon.0) 1968 : cluster [DBG] osdmap e1079: 8 total, 8 up, 8 in 2024-09-12T23:58:12.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:11 smithi136 bash[76371]: audit 2024-09-12T23:58:10.801379+0000 mon.c (mon.2) 72 : audit [INF] from='client.? 172.21.15.136:0/3073703616' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:58:12.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:11 smithi136 bash[76371]: audit 2024-09-12T23:58:10.802718+0000 mon.a (mon.0) 1969 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:58:12.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:58:12 smithi181 bash[95505]: ts=2024-09-12T23:58:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:58:13.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:12 smithi181 bash[67587]: cluster 2024-09-12T23:58:11.054148+0000 mgr.y (mgr.24661) 632 : cluster [DBG] pgmap v1272: 113 pgs: 8 unknown, 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:13.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:12 smithi181 bash[67587]: cluster 2024-09-12T23:58:11.789788+0000 mon.a (mon.0) 1970 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:58:13.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:12 smithi181 bash[67587]: audit 2024-09-12T23:58:11.808132+0000 mon.a (mon.0) 1971 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:58:13.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:12 smithi181 bash[67587]: cluster 2024-09-12T23:58:11.808467+0000 mon.a (mon.0) 1972 : cluster [DBG] osdmap e1080: 8 total, 8 up, 8 in 2024-09-12T23:58:13.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:12 smithi136 bash[75164]: cluster 2024-09-12T23:58:11.054148+0000 mgr.y (mgr.24661) 632 : cluster [DBG] pgmap v1272: 113 pgs: 8 unknown, 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:13.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:12 smithi136 bash[75164]: cluster 2024-09-12T23:58:11.789788+0000 mon.a (mon.0) 1970 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:58:13.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:12 smithi136 bash[75164]: audit 2024-09-12T23:58:11.808132+0000 mon.a (mon.0) 1971 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:58:13.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:12 smithi136 bash[75164]: cluster 2024-09-12T23:58:11.808467+0000 mon.a (mon.0) 1972 : cluster [DBG] osdmap e1080: 8 total, 8 up, 8 in 2024-09-12T23:58:13.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:12 smithi136 bash[76371]: cluster 2024-09-12T23:58:11.054148+0000 mgr.y (mgr.24661) 632 : cluster [DBG] pgmap v1272: 113 pgs: 8 unknown, 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:13.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:12 smithi136 bash[76371]: cluster 2024-09-12T23:58:11.789788+0000 mon.a (mon.0) 1970 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:58:13.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:12 smithi136 bash[76371]: audit 2024-09-12T23:58:11.808132+0000 mon.a (mon.0) 1971 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:58:13.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:12 smithi136 bash[76371]: cluster 2024-09-12T23:58:11.808467+0000 mon.a (mon.0) 1972 : cluster [DBG] osdmap e1080: 8 total, 8 up, 8 in 2024-09-12T23:58:14.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:13 smithi181 bash[67587]: cluster 2024-09-12T23:58:12.818136+0000 mon.a (mon.0) 1973 : cluster [DBG] osdmap e1081: 8 total, 8 up, 8 in 2024-09-12T23:58:14.256 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:13 smithi136 bash[75164]: cluster 2024-09-12T23:58:12.818136+0000 mon.a (mon.0) 1973 : cluster [DBG] osdmap e1081: 8 total, 8 up, 8 in 2024-09-12T23:58:14.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:13 smithi136 bash[76371]: cluster 2024-09-12T23:58:12.818136+0000 mon.a (mon.0) 1973 : cluster [DBG] osdmap e1081: 8 total, 8 up, 8 in 2024-09-12T23:58:15.103 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:14 smithi181 bash[67587]: cluster 2024-09-12T23:58:13.054878+0000 mgr.y (mgr.24661) 633 : cluster [DBG] pgmap v1275: 113 pgs: 1 creating+peering, 5 unknown, 107 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:15.103 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:14 smithi181 bash[67587]: cluster 2024-09-12T23:58:13.810456+0000 mon.a (mon.0) 1974 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:58:15.104 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:14 smithi181 bash[67587]: cluster 2024-09-12T23:58:13.810511+0000 mon.a (mon.0) 1975 : cluster [INF] Cluster is now healthy 2024-09-12T23:58:15.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:14 smithi136 bash[75164]: cluster 2024-09-12T23:58:13.054878+0000 mgr.y (mgr.24661) 633 : cluster [DBG] pgmap v1275: 113 pgs: 1 creating+peering, 5 unknown, 107 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:15.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:14 smithi136 bash[75164]: cluster 2024-09-12T23:58:13.810456+0000 mon.a (mon.0) 1974 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:58:15.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:14 smithi136 bash[75164]: cluster 2024-09-12T23:58:13.810511+0000 mon.a (mon.0) 1975 : cluster [INF] Cluster is now healthy 2024-09-12T23:58:15.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:14 smithi136 bash[76371]: cluster 2024-09-12T23:58:13.054878+0000 mgr.y (mgr.24661) 633 : cluster [DBG] pgmap v1275: 113 pgs: 1 creating+peering, 5 unknown, 107 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:15.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:14 smithi136 bash[76371]: cluster 2024-09-12T23:58:13.810456+0000 mon.a (mon.0) 1974 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:58:15.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:14 smithi136 bash[76371]: cluster 2024-09-12T23:58:13.810511+0000 mon.a (mon.0) 1975 : cluster [INF] Cluster is now healthy 2024-09-12T23:58:17.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:16 smithi181 bash[67587]: cluster 2024-09-12T23:58:15.055758+0000 mgr.y (mgr.24661) 634 : cluster [DBG] pgmap v1276: 113 pgs: 1 creating+peering, 2 unknown, 110 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:17.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:16 smithi181 bash[67587]: cluster 2024-09-12T23:58:15.839122+0000 mon.a (mon.0) 1976 : cluster [DBG] osdmap e1082: 8 total, 8 up, 8 in 2024-09-12T23:58:17.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:16 smithi136 bash[75164]: cluster 2024-09-12T23:58:15.055758+0000 mgr.y (mgr.24661) 634 : cluster [DBG] pgmap v1276: 113 pgs: 1 creating+peering, 2 unknown, 110 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:17.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:16 smithi136 bash[75164]: cluster 2024-09-12T23:58:15.839122+0000 mon.a (mon.0) 1976 : cluster [DBG] osdmap e1082: 8 total, 8 up, 8 in 2024-09-12T23:58:17.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:16 smithi136 bash[76371]: cluster 2024-09-12T23:58:15.055758+0000 mgr.y (mgr.24661) 634 : cluster [DBG] pgmap v1276: 113 pgs: 1 creating+peering, 2 unknown, 110 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:17.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:16 smithi136 bash[76371]: cluster 2024-09-12T23:58:15.839122+0000 mon.a (mon.0) 1976 : cluster [DBG] osdmap e1082: 8 total, 8 up, 8 in 2024-09-12T23:58:18.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:17 smithi136 bash[76371]: cluster 2024-09-12T23:58:16.852072+0000 mon.a (mon.0) 1977 : cluster [DBG] osdmap e1083: 8 total, 8 up, 8 in 2024-09-12T23:58:18.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:17 smithi136 bash[75164]: cluster 2024-09-12T23:58:16.852072+0000 mon.a (mon.0) 1977 : cluster [DBG] osdmap e1083: 8 total, 8 up, 8 in 2024-09-12T23:58:18.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:17 smithi181 bash[67587]: cluster 2024-09-12T23:58:16.852072+0000 mon.a (mon.0) 1977 : cluster [DBG] osdmap e1083: 8 total, 8 up, 8 in 2024-09-12T23:58:19.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:18 smithi136 bash[76371]: cluster 2024-09-12T23:58:17.056606+0000 mgr.y (mgr.24661) 635 : cluster [DBG] pgmap v1279: 113 pgs: 113 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 194 B/s wr, 0 op/s 2024-09-12T23:58:19.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:18 smithi136 bash[76371]: cluster 2024-09-12T23:58:17.859834+0000 mon.a (mon.0) 1978 : cluster [DBG] osdmap e1084: 8 total, 8 up, 8 in 2024-09-12T23:58:19.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:18 smithi136 bash[75164]: cluster 2024-09-12T23:58:17.056606+0000 mgr.y (mgr.24661) 635 : cluster [DBG] pgmap v1279: 113 pgs: 113 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 194 B/s wr, 0 op/s 2024-09-12T23:58:19.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:18 smithi136 bash[75164]: cluster 2024-09-12T23:58:17.859834+0000 mon.a (mon.0) 1978 : cluster [DBG] osdmap e1084: 8 total, 8 up, 8 in 2024-09-12T23:58:19.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:18 smithi181 bash[67587]: cluster 2024-09-12T23:58:17.056606+0000 mgr.y (mgr.24661) 635 : cluster [DBG] pgmap v1279: 113 pgs: 113 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 194 B/s wr, 0 op/s 2024-09-12T23:58:19.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:18 smithi181 bash[67587]: cluster 2024-09-12T23:58:17.859834+0000 mon.a (mon.0) 1978 : cluster [DBG] osdmap e1084: 8 total, 8 up, 8 in 2024-09-12T23:58:19.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:58:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:58:19] "GET /metrics HTTP/1.1" 200 34081 "" "Prometheus/2.43.0" 2024-09-12T23:58:20.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:19 smithi136 bash[76371]: cluster 2024-09-12T23:58:18.860739+0000 mon.a (mon.0) 1979 : cluster [DBG] osdmap e1085: 8 total, 8 up, 8 in 2024-09-12T23:58:20.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:19 smithi136 bash[76371]: audit 2024-09-12T23:58:18.866152+0000 mon.c (mon.2) 73 : audit [INF] from='client.? 172.21.15.136:0/3073703616' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-8"}]: dispatch 2024-09-12T23:58:20.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:19 smithi136 bash[76371]: audit 2024-09-12T23:58:18.877655+0000 mon.a (mon.0) 1980 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-8"}]: dispatch 2024-09-12T23:58:20.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:19 smithi136 bash[75164]: cluster 2024-09-12T23:58:18.860739+0000 mon.a (mon.0) 1979 : cluster [DBG] osdmap e1085: 8 total, 8 up, 8 in 2024-09-12T23:58:20.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:19 smithi136 bash[75164]: audit 2024-09-12T23:58:18.866152+0000 mon.c (mon.2) 73 : audit [INF] from='client.? 172.21.15.136:0/3073703616' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-8"}]: dispatch 2024-09-12T23:58:20.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:19 smithi136 bash[75164]: audit 2024-09-12T23:58:18.877655+0000 mon.a (mon.0) 1980 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-8"}]: dispatch 2024-09-12T23:58:20.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:19 smithi181 bash[67587]: cluster 2024-09-12T23:58:18.860739+0000 mon.a (mon.0) 1979 : cluster [DBG] osdmap e1085: 8 total, 8 up, 8 in 2024-09-12T23:58:20.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:19 smithi181 bash[67587]: audit 2024-09-12T23:58:18.866152+0000 mon.c (mon.2) 73 : audit [INF] from='client.? 172.21.15.136:0/3073703616' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-8"}]: dispatch 2024-09-12T23:58:20.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:19 smithi181 bash[67587]: audit 2024-09-12T23:58:18.877655+0000 mon.a (mon.0) 1980 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-8"}]: dispatch 2024-09-12T23:58:20.897 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] cls_rgw.test_put_snap_ec (13138 ms) 2024-09-12T23:58:20.897 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] cls_rgw.test_explicit_ec 2024-09-12T23:58:21.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:20 smithi136 bash[76371]: cluster 2024-09-12T23:58:19.057230+0000 mgr.y (mgr.24661) 636 : cluster [DBG] pgmap v1282: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:21.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:20 smithi136 bash[76371]: audit 2024-09-12T23:58:19.886576+0000 mon.a (mon.0) 1981 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-8"}]': finished 2024-09-12T23:58:21.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:20 smithi136 bash[76371]: cluster 2024-09-12T23:58:19.886770+0000 mon.a (mon.0) 1982 : cluster [DBG] osdmap e1086: 8 total, 8 up, 8 in 2024-09-12T23:58:21.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:20 smithi136 bash[76371]: audit 2024-09-12T23:58:19.888701+0000 mon.c (mon.2) 74 : audit [INF] from='client.? 172.21.15.136:0/3073703616' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-8"}]: dispatch 2024-09-12T23:58:21.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:20 smithi136 bash[76371]: audit 2024-09-12T23:58:19.889415+0000 mon.a (mon.0) 1983 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-8"}]: dispatch 2024-09-12T23:58:21.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:20 smithi136 bash[75164]: cluster 2024-09-12T23:58:19.057230+0000 mgr.y (mgr.24661) 636 : cluster [DBG] pgmap v1282: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:21.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:20 smithi136 bash[75164]: audit 2024-09-12T23:58:19.886576+0000 mon.a (mon.0) 1981 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-8"}]': finished 2024-09-12T23:58:21.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:20 smithi136 bash[75164]: cluster 2024-09-12T23:58:19.886770+0000 mon.a (mon.0) 1982 : cluster [DBG] osdmap e1086: 8 total, 8 up, 8 in 2024-09-12T23:58:21.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:20 smithi136 bash[75164]: audit 2024-09-12T23:58:19.888701+0000 mon.c (mon.2) 74 : audit [INF] from='client.? 172.21.15.136:0/3073703616' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-8"}]: dispatch 2024-09-12T23:58:21.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:20 smithi136 bash[75164]: audit 2024-09-12T23:58:19.889415+0000 mon.a (mon.0) 1983 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-8"}]: dispatch 2024-09-12T23:58:21.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:20 smithi181 bash[67587]: cluster 2024-09-12T23:58:19.057230+0000 mgr.y (mgr.24661) 636 : cluster [DBG] pgmap v1282: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:21.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:20 smithi181 bash[67587]: audit 2024-09-12T23:58:19.886576+0000 mon.a (mon.0) 1981 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-8"}]': finished 2024-09-12T23:58:21.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:20 smithi181 bash[67587]: cluster 2024-09-12T23:58:19.886770+0000 mon.a (mon.0) 1982 : cluster [DBG] osdmap e1086: 8 total, 8 up, 8 in 2024-09-12T23:58:21.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:20 smithi181 bash[67587]: audit 2024-09-12T23:58:19.888701+0000 mon.c (mon.2) 74 : audit [INF] from='client.? 172.21.15.136:0/3073703616' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-8"}]: dispatch 2024-09-12T23:58:21.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:20 smithi181 bash[67587]: audit 2024-09-12T23:58:19.889415+0000 mon.a (mon.0) 1983 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-8"}]: dispatch 2024-09-12T23:58:22.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:21 smithi136 bash[75164]: audit 2024-09-12T23:58:20.886736+0000 mon.a (mon.0) 1984 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-8"}]': finished 2024-09-12T23:58:22.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:21 smithi136 bash[75164]: cluster 2024-09-12T23:58:20.886884+0000 mon.a (mon.0) 1985 : cluster [DBG] osdmap e1087: 8 total, 8 up, 8 in 2024-09-12T23:58:22.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:21 smithi136 bash[75164]: audit 2024-09-12T23:58:20.937757+0000 mon.c (mon.2) 75 : audit [INF] from='client.? 172.21.15.136:0/4165645521' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-9"}]: dispatch 2024-09-12T23:58:22.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:21 smithi136 bash[75164]: audit 2024-09-12T23:58:20.938139+0000 mon.a (mon.0) 1986 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-9"}]: dispatch 2024-09-12T23:58:22.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:21 smithi136 bash[75164]: audit 2024-09-12T23:58:20.938849+0000 mon.c (mon.2) 76 : audit [INF] from='client.? 172.21.15.136:0/4165645521' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-9"}]: dispatch 2024-09-12T23:58:22.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:21 smithi136 bash[75164]: audit 2024-09-12T23:58:20.939240+0000 mon.a (mon.0) 1987 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-9"}]: dispatch 2024-09-12T23:58:22.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:21 smithi136 bash[75164]: audit 2024-09-12T23:58:20.939734+0000 mon.c (mon.2) 77 : audit [INF] from='client.? 172.21.15.136:0/4165645521' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi136-110348-9", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-12T23:58:22.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:21 smithi136 bash[75164]: audit 2024-09-12T23:58:20.940070+0000 mon.a (mon.0) 1988 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi136-110348-9", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-12T23:58:22.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:21 smithi136 bash[76371]: audit 2024-09-12T23:58:20.886736+0000 mon.a (mon.0) 1984 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-8"}]': finished 2024-09-12T23:58:22.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:21 smithi136 bash[76371]: cluster 2024-09-12T23:58:20.886884+0000 mon.a (mon.0) 1985 : cluster [DBG] osdmap e1087: 8 total, 8 up, 8 in 2024-09-12T23:58:22.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:21 smithi136 bash[76371]: audit 2024-09-12T23:58:20.937757+0000 mon.c (mon.2) 75 : audit [INF] from='client.? 172.21.15.136:0/4165645521' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-9"}]: dispatch 2024-09-12T23:58:22.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:21 smithi136 bash[76371]: audit 2024-09-12T23:58:20.938139+0000 mon.a (mon.0) 1986 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-9"}]: dispatch 2024-09-12T23:58:22.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:21 smithi136 bash[76371]: audit 2024-09-12T23:58:20.938849+0000 mon.c (mon.2) 76 : audit [INF] from='client.? 172.21.15.136:0/4165645521' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-9"}]: dispatch 2024-09-12T23:58:22.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:21 smithi136 bash[76371]: audit 2024-09-12T23:58:20.939240+0000 mon.a (mon.0) 1987 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-9"}]: dispatch 2024-09-12T23:58:22.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:21 smithi136 bash[76371]: audit 2024-09-12T23:58:20.939734+0000 mon.c (mon.2) 77 : audit [INF] from='client.? 172.21.15.136:0/4165645521' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi136-110348-9", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-12T23:58:22.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:21 smithi136 bash[76371]: audit 2024-09-12T23:58:20.940070+0000 mon.a (mon.0) 1988 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi136-110348-9", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-12T23:58:22.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:21 smithi181 bash[67587]: audit 2024-09-12T23:58:20.886736+0000 mon.a (mon.0) 1984 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-8"}]': finished 2024-09-12T23:58:22.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:21 smithi181 bash[67587]: cluster 2024-09-12T23:58:20.886884+0000 mon.a (mon.0) 1985 : cluster [DBG] osdmap e1087: 8 total, 8 up, 8 in 2024-09-12T23:58:22.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:21 smithi181 bash[67587]: audit 2024-09-12T23:58:20.937757+0000 mon.c (mon.2) 75 : audit [INF] from='client.? 172.21.15.136:0/4165645521' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-9"}]: dispatch 2024-09-12T23:58:22.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:21 smithi181 bash[67587]: audit 2024-09-12T23:58:20.938139+0000 mon.a (mon.0) 1986 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-9"}]: dispatch 2024-09-12T23:58:22.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:21 smithi181 bash[67587]: audit 2024-09-12T23:58:20.938849+0000 mon.c (mon.2) 76 : audit [INF] from='client.? 172.21.15.136:0/4165645521' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-9"}]: dispatch 2024-09-12T23:58:22.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:21 smithi181 bash[67587]: audit 2024-09-12T23:58:20.939240+0000 mon.a (mon.0) 1987 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-9"}]: dispatch 2024-09-12T23:58:22.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:21 smithi181 bash[67587]: audit 2024-09-12T23:58:20.939734+0000 mon.c (mon.2) 77 : audit [INF] from='client.? 172.21.15.136:0/4165645521' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi136-110348-9", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-12T23:58:22.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:21 smithi181 bash[67587]: audit 2024-09-12T23:58:20.940070+0000 mon.a (mon.0) 1988 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi136-110348-9", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-12T23:58:22.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:58:22 smithi181 bash[95505]: ts=2024-09-12T23:58:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:58:23.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:23 smithi136 bash[75164]: cluster 2024-09-12T23:58:21.057956+0000 mgr.y (mgr.24661) 637 : cluster [DBG] pgmap v1285: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:23.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:23 smithi136 bash[75164]: audit 2024-09-12T23:58:21.907371+0000 mon.a (mon.0) 1989 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi136-110348-9", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2024-09-12T23:58:23.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:23 smithi136 bash[75164]: cluster 2024-09-12T23:58:21.907578+0000 mon.a (mon.0) 1990 : cluster [DBG] osdmap e1088: 8 total, 8 up, 8 in 2024-09-12T23:58:23.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:23 smithi136 bash[75164]: audit 2024-09-12T23:58:21.909572+0000 mon.c (mon.2) 78 : audit [INF] from='client.? 172.21.15.136:0/4165645521' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi136-110348-9", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi136-110348-9"}]: dispatch 2024-09-12T23:58:23.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:23 smithi136 bash[75164]: audit 2024-09-12T23:58:21.910272+0000 mon.a (mon.0) 1991 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi136-110348-9", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi136-110348-9"}]: dispatch 2024-09-12T23:58:23.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:23 smithi136 bash[76371]: cluster 2024-09-12T23:58:21.057956+0000 mgr.y (mgr.24661) 637 : cluster [DBG] pgmap v1285: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:23.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:23 smithi136 bash[76371]: audit 2024-09-12T23:58:21.907371+0000 mon.a (mon.0) 1989 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi136-110348-9", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2024-09-12T23:58:23.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:23 smithi136 bash[76371]: cluster 2024-09-12T23:58:21.907578+0000 mon.a (mon.0) 1990 : cluster [DBG] osdmap e1088: 8 total, 8 up, 8 in 2024-09-12T23:58:23.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:23 smithi136 bash[76371]: audit 2024-09-12T23:58:21.909572+0000 mon.c (mon.2) 78 : audit [INF] from='client.? 172.21.15.136:0/4165645521' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi136-110348-9", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi136-110348-9"}]: dispatch 2024-09-12T23:58:23.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:23 smithi136 bash[76371]: audit 2024-09-12T23:58:21.910272+0000 mon.a (mon.0) 1991 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi136-110348-9", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi136-110348-9"}]: dispatch 2024-09-12T23:58:23.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:23 smithi181 bash[67587]: cluster 2024-09-12T23:58:21.057956+0000 mgr.y (mgr.24661) 637 : cluster [DBG] pgmap v1285: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:23.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:23 smithi181 bash[67587]: audit 2024-09-12T23:58:21.907371+0000 mon.a (mon.0) 1989 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi136-110348-9", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2024-09-12T23:58:23.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:23 smithi181 bash[67587]: cluster 2024-09-12T23:58:21.907578+0000 mon.a (mon.0) 1990 : cluster [DBG] osdmap e1088: 8 total, 8 up, 8 in 2024-09-12T23:58:23.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:23 smithi181 bash[67587]: audit 2024-09-12T23:58:21.909572+0000 mon.c (mon.2) 78 : audit [INF] from='client.? 172.21.15.136:0/4165645521' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi136-110348-9", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi136-110348-9"}]: dispatch 2024-09-12T23:58:23.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:23 smithi181 bash[67587]: audit 2024-09-12T23:58:21.910272+0000 mon.a (mon.0) 1991 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi136-110348-9", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi136-110348-9"}]: dispatch 2024-09-12T23:58:24.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:24 smithi136 bash[75164]: cluster 2024-09-12T23:58:23.058582+0000 mgr.y (mgr.24661) 638 : cluster [DBG] pgmap v1288: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:24.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:24 smithi136 bash[75164]: cluster 2024-09-12T23:58:23.351049+0000 mon.a (mon.0) 1992 : cluster [DBG] osdmap e1089: 8 total, 8 up, 8 in 2024-09-12T23:58:24.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:24 smithi136 bash[75164]: audit 2024-09-12T23:58:23.766013+0000 mon.a (mon.0) 1993 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:58:24.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:24 smithi136 bash[75164]: audit 2024-09-12T23:58:23.918844+0000 mon.a (mon.0) 1994 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi136-110348-9", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi136-110348-9"}]': finished 2024-09-12T23:58:24.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:24 smithi136 bash[75164]: cluster 2024-09-12T23:58:23.919088+0000 mon.a (mon.0) 1995 : cluster [DBG] osdmap e1090: 8 total, 8 up, 8 in 2024-09-12T23:58:24.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:24 smithi136 bash[75164]: audit 2024-09-12T23:58:23.924977+0000 mon.c (mon.2) 79 : audit [INF] from='client.? 172.21.15.136:0/4165645521' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-9","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:58:24.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:24 smithi136 bash[75164]: audit 2024-09-12T23:58:23.925796+0000 mon.a (mon.0) 1996 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-9","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:58:24.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:24 smithi136 bash[76371]: cluster 2024-09-12T23:58:23.058582+0000 mgr.y (mgr.24661) 638 : cluster [DBG] pgmap v1288: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:24.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:24 smithi136 bash[76371]: cluster 2024-09-12T23:58:23.351049+0000 mon.a (mon.0) 1992 : cluster [DBG] osdmap e1089: 8 total, 8 up, 8 in 2024-09-12T23:58:24.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:24 smithi136 bash[76371]: audit 2024-09-12T23:58:23.766013+0000 mon.a (mon.0) 1993 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:58:24.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:24 smithi136 bash[76371]: audit 2024-09-12T23:58:23.918844+0000 mon.a (mon.0) 1994 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi136-110348-9", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi136-110348-9"}]': finished 2024-09-12T23:58:24.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:24 smithi136 bash[76371]: cluster 2024-09-12T23:58:23.919088+0000 mon.a (mon.0) 1995 : cluster [DBG] osdmap e1090: 8 total, 8 up, 8 in 2024-09-12T23:58:24.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:24 smithi136 bash[76371]: audit 2024-09-12T23:58:23.924977+0000 mon.c (mon.2) 79 : audit [INF] from='client.? 172.21.15.136:0/4165645521' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-9","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:58:24.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:24 smithi136 bash[76371]: audit 2024-09-12T23:58:23.925796+0000 mon.a (mon.0) 1996 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-9","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:58:24.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:24 smithi181 bash[67587]: cluster 2024-09-12T23:58:23.058582+0000 mgr.y (mgr.24661) 638 : cluster [DBG] pgmap v1288: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:24.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:24 smithi181 bash[67587]: cluster 2024-09-12T23:58:23.351049+0000 mon.a (mon.0) 1992 : cluster [DBG] osdmap e1089: 8 total, 8 up, 8 in 2024-09-12T23:58:24.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:24 smithi181 bash[67587]: audit 2024-09-12T23:58:23.766013+0000 mon.a (mon.0) 1993 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:58:24.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:24 smithi181 bash[67587]: audit 2024-09-12T23:58:23.918844+0000 mon.a (mon.0) 1994 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi136-110348-9", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi136-110348-9"}]': finished 2024-09-12T23:58:24.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:24 smithi181 bash[67587]: cluster 2024-09-12T23:58:23.919088+0000 mon.a (mon.0) 1995 : cluster [DBG] osdmap e1090: 8 total, 8 up, 8 in 2024-09-12T23:58:24.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:24 smithi181 bash[67587]: audit 2024-09-12T23:58:23.924977+0000 mon.c (mon.2) 79 : audit [INF] from='client.? 172.21.15.136:0/4165645521' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-9","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:58:24.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:24 smithi181 bash[67587]: audit 2024-09-12T23:58:23.925796+0000 mon.a (mon.0) 1996 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-9","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:58:26.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:25 smithi136 bash[75164]: audit 2024-09-12T23:58:24.923250+0000 mon.a (mon.0) 1997 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-9","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:58:26.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:25 smithi136 bash[75164]: cluster 2024-09-12T23:58:24.923646+0000 mon.a (mon.0) 1998 : cluster [DBG] osdmap e1091: 8 total, 8 up, 8 in 2024-09-12T23:58:26.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:25 smithi136 bash[76371]: audit 2024-09-12T23:58:24.923250+0000 mon.a (mon.0) 1997 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-9","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:58:26.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:25 smithi136 bash[76371]: cluster 2024-09-12T23:58:24.923646+0000 mon.a (mon.0) 1998 : cluster [DBG] osdmap e1091: 8 total, 8 up, 8 in 2024-09-12T23:58:26.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:25 smithi181 bash[67587]: audit 2024-09-12T23:58:24.923250+0000 mon.a (mon.0) 1997 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-9","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:58:26.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:25 smithi181 bash[67587]: cluster 2024-09-12T23:58:24.923646+0000 mon.a (mon.0) 1998 : cluster [DBG] osdmap e1091: 8 total, 8 up, 8 in 2024-09-12T23:58:27.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:26 smithi136 bash[75164]: cluster 2024-09-12T23:58:25.059346+0000 mgr.y (mgr.24661) 639 : cluster [DBG] pgmap v1291: 113 pgs: 1 creating+peering, 7 unknown, 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:27.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:26 smithi136 bash[75164]: cluster 2024-09-12T23:58:25.933426+0000 mon.a (mon.0) 1999 : cluster [DBG] osdmap e1092: 8 total, 8 up, 8 in 2024-09-12T23:58:27.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:26 smithi136 bash[75164]: audit 2024-09-12T23:58:25.937059+0000 mon.c (mon.2) 80 : audit [INF] from='client.? 172.21.15.136:0/4165645521' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-9"}]: dispatch 2024-09-12T23:58:27.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:26 smithi136 bash[75164]: audit 2024-09-12T23:58:25.937876+0000 mon.a (mon.0) 2000 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-9"}]: dispatch 2024-09-12T23:58:27.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:26 smithi136 bash[76371]: cluster 2024-09-12T23:58:25.059346+0000 mgr.y (mgr.24661) 639 : cluster [DBG] pgmap v1291: 113 pgs: 1 creating+peering, 7 unknown, 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:27.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:26 smithi136 bash[76371]: cluster 2024-09-12T23:58:25.933426+0000 mon.a (mon.0) 1999 : cluster [DBG] osdmap e1092: 8 total, 8 up, 8 in 2024-09-12T23:58:27.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:26 smithi136 bash[76371]: audit 2024-09-12T23:58:25.937059+0000 mon.c (mon.2) 80 : audit [INF] from='client.? 172.21.15.136:0/4165645521' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-9"}]: dispatch 2024-09-12T23:58:27.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:26 smithi136 bash[76371]: audit 2024-09-12T23:58:25.937876+0000 mon.a (mon.0) 2000 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-9"}]: dispatch 2024-09-12T23:58:27.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:26 smithi181 bash[67587]: cluster 2024-09-12T23:58:25.059346+0000 mgr.y (mgr.24661) 639 : cluster [DBG] pgmap v1291: 113 pgs: 1 creating+peering, 7 unknown, 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:27.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:26 smithi181 bash[67587]: cluster 2024-09-12T23:58:25.933426+0000 mon.a (mon.0) 1999 : cluster [DBG] osdmap e1092: 8 total, 8 up, 8 in 2024-09-12T23:58:27.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:26 smithi181 bash[67587]: audit 2024-09-12T23:58:25.937059+0000 mon.c (mon.2) 80 : audit [INF] from='client.? 172.21.15.136:0/4165645521' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-9"}]: dispatch 2024-09-12T23:58:27.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:26 smithi181 bash[67587]: audit 2024-09-12T23:58:25.937876+0000 mon.a (mon.0) 2000 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-9"}]: dispatch 2024-09-12T23:58:27.947 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] cls_rgw.test_explicit_ec (7050 ms) 2024-09-12T23:58:27.948 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] cls_rgw.set_ec 2024-09-12T23:58:28.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:27 smithi136 bash[75164]: audit 2024-09-12T23:58:26.935343+0000 mon.a (mon.0) 2001 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-9"}]': finished 2024-09-12T23:58:28.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:27 smithi136 bash[75164]: cluster 2024-09-12T23:58:26.935732+0000 mon.a (mon.0) 2002 : cluster [DBG] osdmap e1093: 8 total, 8 up, 8 in 2024-09-12T23:58:28.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:27 smithi136 bash[75164]: audit 2024-09-12T23:58:26.937497+0000 mon.c (mon.2) 81 : audit [INF] from='client.? 172.21.15.136:0/4165645521' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-9"}]: dispatch 2024-09-12T23:58:28.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:27 smithi136 bash[75164]: audit 2024-09-12T23:58:26.938427+0000 mon.a (mon.0) 2003 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-9"}]: dispatch 2024-09-12T23:58:28.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:27 smithi136 bash[76371]: audit 2024-09-12T23:58:26.935343+0000 mon.a (mon.0) 2001 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-9"}]': finished 2024-09-12T23:58:28.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:27 smithi136 bash[76371]: cluster 2024-09-12T23:58:26.935732+0000 mon.a (mon.0) 2002 : cluster [DBG] osdmap e1093: 8 total, 8 up, 8 in 2024-09-12T23:58:28.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:27 smithi136 bash[76371]: audit 2024-09-12T23:58:26.937497+0000 mon.c (mon.2) 81 : audit [INF] from='client.? 172.21.15.136:0/4165645521' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-9"}]: dispatch 2024-09-12T23:58:28.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:27 smithi136 bash[76371]: audit 2024-09-12T23:58:26.938427+0000 mon.a (mon.0) 2003 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-9"}]: dispatch 2024-09-12T23:58:28.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:27 smithi181 bash[67587]: audit 2024-09-12T23:58:26.935343+0000 mon.a (mon.0) 2001 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-9"}]': finished 2024-09-12T23:58:28.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:27 smithi181 bash[67587]: cluster 2024-09-12T23:58:26.935732+0000 mon.a (mon.0) 2002 : cluster [DBG] osdmap e1093: 8 total, 8 up, 8 in 2024-09-12T23:58:28.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:27 smithi181 bash[67587]: audit 2024-09-12T23:58:26.937497+0000 mon.c (mon.2) 81 : audit [INF] from='client.? 172.21.15.136:0/4165645521' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-9"}]: dispatch 2024-09-12T23:58:28.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:27 smithi181 bash[67587]: audit 2024-09-12T23:58:26.938427+0000 mon.a (mon.0) 2003 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-9"}]: dispatch 2024-09-12T23:58:29.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:28 smithi136 bash[75164]: cluster 2024-09-12T23:58:27.060039+0000 mgr.y (mgr.24661) 640 : cluster [DBG] pgmap v1294: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:29.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:28 smithi136 bash[75164]: audit 2024-09-12T23:58:27.937526+0000 mon.a (mon.0) 2004 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-9"}]': finished 2024-09-12T23:58:29.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:28 smithi136 bash[75164]: cluster 2024-09-12T23:58:27.937718+0000 mon.a (mon.0) 2005 : cluster [DBG] osdmap e1094: 8 total, 8 up, 8 in 2024-09-12T23:58:29.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:28 smithi136 bash[75164]: audit 2024-09-12T23:58:27.972591+0000 mon.c (mon.2) 82 : audit [INF] from='client.? 172.21.15.136:0/3753062870' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-10"}]: dispatch 2024-09-12T23:58:29.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:28 smithi136 bash[75164]: audit 2024-09-12T23:58:27.972937+0000 mon.a (mon.0) 2006 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-10"}]: dispatch 2024-09-12T23:58:29.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:28 smithi136 bash[75164]: audit 2024-09-12T23:58:27.973629+0000 mon.c (mon.2) 83 : audit [INF] from='client.? 172.21.15.136:0/3753062870' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-10"}]: dispatch 2024-09-12T23:58:29.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:28 smithi136 bash[75164]: audit 2024-09-12T23:58:27.973984+0000 mon.a (mon.0) 2007 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-10"}]: dispatch 2024-09-12T23:58:29.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:28 smithi136 bash[75164]: audit 2024-09-12T23:58:27.974413+0000 mon.c (mon.2) 84 : audit [INF] from='client.? 172.21.15.136:0/3753062870' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi136-110348-10", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-12T23:58:29.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:28 smithi136 bash[75164]: audit 2024-09-12T23:58:27.974683+0000 mon.a (mon.0) 2008 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi136-110348-10", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-12T23:58:29.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:28 smithi136 bash[76371]: cluster 2024-09-12T23:58:27.060039+0000 mgr.y (mgr.24661) 640 : cluster [DBG] pgmap v1294: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:29.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:28 smithi136 bash[76371]: audit 2024-09-12T23:58:27.937526+0000 mon.a (mon.0) 2004 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-9"}]': finished 2024-09-12T23:58:29.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:28 smithi136 bash[76371]: cluster 2024-09-12T23:58:27.937718+0000 mon.a (mon.0) 2005 : cluster [DBG] osdmap e1094: 8 total, 8 up, 8 in 2024-09-12T23:58:29.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:28 smithi136 bash[76371]: audit 2024-09-12T23:58:27.972591+0000 mon.c (mon.2) 82 : audit [INF] from='client.? 172.21.15.136:0/3753062870' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-10"}]: dispatch 2024-09-12T23:58:29.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:28 smithi136 bash[76371]: audit 2024-09-12T23:58:27.972937+0000 mon.a (mon.0) 2006 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-10"}]: dispatch 2024-09-12T23:58:29.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:28 smithi136 bash[76371]: audit 2024-09-12T23:58:27.973629+0000 mon.c (mon.2) 83 : audit [INF] from='client.? 172.21.15.136:0/3753062870' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-10"}]: dispatch 2024-09-12T23:58:29.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:28 smithi136 bash[76371]: audit 2024-09-12T23:58:27.973984+0000 mon.a (mon.0) 2007 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-10"}]: dispatch 2024-09-12T23:58:29.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:28 smithi136 bash[76371]: audit 2024-09-12T23:58:27.974413+0000 mon.c (mon.2) 84 : audit [INF] from='client.? 172.21.15.136:0/3753062870' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi136-110348-10", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-12T23:58:29.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:28 smithi136 bash[76371]: audit 2024-09-12T23:58:27.974683+0000 mon.a (mon.0) 2008 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi136-110348-10", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-12T23:58:29.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:28 smithi181 bash[67587]: cluster 2024-09-12T23:58:27.060039+0000 mgr.y (mgr.24661) 640 : cluster [DBG] pgmap v1294: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:29.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:28 smithi181 bash[67587]: audit 2024-09-12T23:58:27.937526+0000 mon.a (mon.0) 2004 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-9"}]': finished 2024-09-12T23:58:29.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:28 smithi181 bash[67587]: cluster 2024-09-12T23:58:27.937718+0000 mon.a (mon.0) 2005 : cluster [DBG] osdmap e1094: 8 total, 8 up, 8 in 2024-09-12T23:58:29.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:28 smithi181 bash[67587]: audit 2024-09-12T23:58:27.972591+0000 mon.c (mon.2) 82 : audit [INF] from='client.? 172.21.15.136:0/3753062870' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-10"}]: dispatch 2024-09-12T23:58:29.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:28 smithi181 bash[67587]: audit 2024-09-12T23:58:27.972937+0000 mon.a (mon.0) 2006 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-10"}]: dispatch 2024-09-12T23:58:29.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:28 smithi181 bash[67587]: audit 2024-09-12T23:58:27.973629+0000 mon.c (mon.2) 83 : audit [INF] from='client.? 172.21.15.136:0/3753062870' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-10"}]: dispatch 2024-09-12T23:58:29.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:28 smithi181 bash[67587]: audit 2024-09-12T23:58:27.973984+0000 mon.a (mon.0) 2007 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-10"}]: dispatch 2024-09-12T23:58:29.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:28 smithi181 bash[67587]: audit 2024-09-12T23:58:27.974413+0000 mon.c (mon.2) 84 : audit [INF] from='client.? 172.21.15.136:0/3753062870' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi136-110348-10", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-12T23:58:29.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:28 smithi181 bash[67587]: audit 2024-09-12T23:58:27.974683+0000 mon.a (mon.0) 2008 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi136-110348-10", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-12T23:58:29.756 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:58:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:58:29] "GET /metrics HTTP/1.1" 200 34083 "" "Prometheus/2.43.0" 2024-09-12T23:58:30.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:30 smithi136 bash[75164]: audit 2024-09-12T23:58:28.959095+0000 mon.a (mon.0) 2009 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi136-110348-10", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2024-09-12T23:58:30.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:30 smithi136 bash[75164]: cluster 2024-09-12T23:58:28.959307+0000 mon.a (mon.0) 2010 : cluster [DBG] osdmap e1095: 8 total, 8 up, 8 in 2024-09-12T23:58:30.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:30 smithi136 bash[75164]: audit 2024-09-12T23:58:28.963892+0000 mon.c (mon.2) 85 : audit [INF] from='client.? 172.21.15.136:0/3753062870' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi136-110348-10", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi136-110348-10"}]: dispatch 2024-09-12T23:58:30.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:30 smithi136 bash[75164]: audit 2024-09-12T23:58:28.972919+0000 mon.a (mon.0) 2011 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi136-110348-10", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi136-110348-10"}]: dispatch 2024-09-12T23:58:30.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:30 smithi136 bash[76371]: audit 2024-09-12T23:58:28.959095+0000 mon.a (mon.0) 2009 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi136-110348-10", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2024-09-12T23:58:30.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:30 smithi136 bash[76371]: cluster 2024-09-12T23:58:28.959307+0000 mon.a (mon.0) 2010 : cluster [DBG] osdmap e1095: 8 total, 8 up, 8 in 2024-09-12T23:58:30.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:30 smithi136 bash[76371]: audit 2024-09-12T23:58:28.963892+0000 mon.c (mon.2) 85 : audit [INF] from='client.? 172.21.15.136:0/3753062870' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi136-110348-10", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi136-110348-10"}]: dispatch 2024-09-12T23:58:30.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:30 smithi136 bash[76371]: audit 2024-09-12T23:58:28.972919+0000 mon.a (mon.0) 2011 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi136-110348-10", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi136-110348-10"}]: dispatch 2024-09-12T23:58:30.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:30 smithi181 bash[67587]: audit 2024-09-12T23:58:28.959095+0000 mon.a (mon.0) 2009 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi136-110348-10", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2024-09-12T23:58:30.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:30 smithi181 bash[67587]: cluster 2024-09-12T23:58:28.959307+0000 mon.a (mon.0) 2010 : cluster [DBG] osdmap e1095: 8 total, 8 up, 8 in 2024-09-12T23:58:30.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:30 smithi181 bash[67587]: audit 2024-09-12T23:58:28.963892+0000 mon.c (mon.2) 85 : audit [INF] from='client.? 172.21.15.136:0/3753062870' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi136-110348-10", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi136-110348-10"}]: dispatch 2024-09-12T23:58:30.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:30 smithi181 bash[67587]: audit 2024-09-12T23:58:28.972919+0000 mon.a (mon.0) 2011 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi136-110348-10", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi136-110348-10"}]: dispatch 2024-09-12T23:58:31.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:31 smithi136 bash[75164]: cluster 2024-09-12T23:58:29.060745+0000 mgr.y (mgr.24661) 641 : cluster [DBG] pgmap v1297: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:31.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:31 smithi136 bash[75164]: cluster 2024-09-12T23:58:30.415025+0000 mon.a (mon.0) 2012 : cluster [DBG] osdmap e1096: 8 total, 8 up, 8 in 2024-09-12T23:58:31.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:31 smithi136 bash[75164]: audit 2024-09-12T23:58:30.978472+0000 mon.a (mon.0) 2013 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi136-110348-10", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi136-110348-10"}]': finished 2024-09-12T23:58:31.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:31 smithi136 bash[75164]: cluster 2024-09-12T23:58:30.978740+0000 mon.a (mon.0) 2014 : cluster [DBG] osdmap e1097: 8 total, 8 up, 8 in 2024-09-12T23:58:31.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:31 smithi136 bash[75164]: audit 2024-09-12T23:58:30.982409+0000 mon.c (mon.2) 86 : audit [INF] from='client.? 172.21.15.136:0/3753062870' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:58:31.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:31 smithi136 bash[75164]: audit 2024-09-12T23:58:30.983124+0000 mon.a (mon.0) 2015 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:58:31.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:31 smithi136 bash[76371]: cluster 2024-09-12T23:58:29.060745+0000 mgr.y (mgr.24661) 641 : cluster [DBG] pgmap v1297: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:31.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:31 smithi136 bash[76371]: cluster 2024-09-12T23:58:30.415025+0000 mon.a (mon.0) 2012 : cluster [DBG] osdmap e1096: 8 total, 8 up, 8 in 2024-09-12T23:58:31.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:31 smithi136 bash[76371]: audit 2024-09-12T23:58:30.978472+0000 mon.a (mon.0) 2013 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi136-110348-10", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi136-110348-10"}]': finished 2024-09-12T23:58:31.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:31 smithi136 bash[76371]: cluster 2024-09-12T23:58:30.978740+0000 mon.a (mon.0) 2014 : cluster [DBG] osdmap e1097: 8 total, 8 up, 8 in 2024-09-12T23:58:31.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:31 smithi136 bash[76371]: audit 2024-09-12T23:58:30.982409+0000 mon.c (mon.2) 86 : audit [INF] from='client.? 172.21.15.136:0/3753062870' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:58:31.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:31 smithi136 bash[76371]: audit 2024-09-12T23:58:30.983124+0000 mon.a (mon.0) 2015 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:58:31.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:31 smithi181 bash[67587]: cluster 2024-09-12T23:58:29.060745+0000 mgr.y (mgr.24661) 641 : cluster [DBG] pgmap v1297: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:31.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:31 smithi181 bash[67587]: cluster 2024-09-12T23:58:30.415025+0000 mon.a (mon.0) 2012 : cluster [DBG] osdmap e1096: 8 total, 8 up, 8 in 2024-09-12T23:58:31.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:31 smithi181 bash[67587]: audit 2024-09-12T23:58:30.978472+0000 mon.a (mon.0) 2013 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi136-110348-10", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi136-110348-10"}]': finished 2024-09-12T23:58:31.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:31 smithi181 bash[67587]: cluster 2024-09-12T23:58:30.978740+0000 mon.a (mon.0) 2014 : cluster [DBG] osdmap e1097: 8 total, 8 up, 8 in 2024-09-12T23:58:31.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:31 smithi181 bash[67587]: audit 2024-09-12T23:58:30.982409+0000 mon.c (mon.2) 86 : audit [INF] from='client.? 172.21.15.136:0/3753062870' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:58:31.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:31 smithi181 bash[67587]: audit 2024-09-12T23:58:30.983124+0000 mon.a (mon.0) 2015 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:58:32.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:58:32 smithi181 bash[95505]: ts=2024-09-12T23:58:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:58:32.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:32 smithi181 bash[67587]: cluster 2024-09-12T23:58:31.061399+0000 mgr.y (mgr.24661) 642 : cluster [DBG] pgmap v1300: 113 pgs: 8 unknown, 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:32.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:32 smithi181 bash[67587]: cluster 2024-09-12T23:58:31.970839+0000 mon.a (mon.0) 2016 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:58:32.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:32 smithi181 bash[67587]: audit 2024-09-12T23:58:31.993721+0000 mon.a (mon.0) 2017 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:58:32.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:32 smithi181 bash[67587]: cluster 2024-09-12T23:58:31.994116+0000 mon.a (mon.0) 2018 : cluster [DBG] osdmap e1098: 8 total, 8 up, 8 in 2024-09-12T23:58:32.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:32 smithi136 bash[75164]: cluster 2024-09-12T23:58:31.061399+0000 mgr.y (mgr.24661) 642 : cluster [DBG] pgmap v1300: 113 pgs: 8 unknown, 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:32.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:32 smithi136 bash[75164]: cluster 2024-09-12T23:58:31.970839+0000 mon.a (mon.0) 2016 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:58:32.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:32 smithi136 bash[75164]: audit 2024-09-12T23:58:31.993721+0000 mon.a (mon.0) 2017 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:58:32.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:32 smithi136 bash[75164]: cluster 2024-09-12T23:58:31.994116+0000 mon.a (mon.0) 2018 : cluster [DBG] osdmap e1098: 8 total, 8 up, 8 in 2024-09-12T23:58:32.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:32 smithi136 bash[76371]: cluster 2024-09-12T23:58:31.061399+0000 mgr.y (mgr.24661) 642 : cluster [DBG] pgmap v1300: 113 pgs: 8 unknown, 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:32.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:32 smithi136 bash[76371]: cluster 2024-09-12T23:58:31.970839+0000 mon.a (mon.0) 2016 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-12T23:58:32.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:32 smithi136 bash[76371]: audit 2024-09-12T23:58:31.993721+0000 mon.a (mon.0) 2017 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110348-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:58:32.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:32 smithi136 bash[76371]: cluster 2024-09-12T23:58:31.994116+0000 mon.a (mon.0) 2018 : cluster [DBG] osdmap e1098: 8 total, 8 up, 8 in 2024-09-12T23:58:34.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:33 smithi136 bash[75164]: cluster 2024-09-12T23:58:32.995300+0000 mon.a (mon.0) 2019 : cluster [DBG] osdmap e1099: 8 total, 8 up, 8 in 2024-09-12T23:58:34.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:33 smithi136 bash[75164]: audit 2024-09-12T23:58:33.000179+0000 mon.c (mon.2) 87 : audit [INF] from='client.? 172.21.15.136:0/3753062870' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-10"}]: dispatch 2024-09-12T23:58:34.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:33 smithi136 bash[75164]: audit 2024-09-12T23:58:33.001060+0000 mon.a (mon.0) 2020 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-10"}]: dispatch 2024-09-12T23:58:34.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:33 smithi136 bash[75164]: audit 2024-09-12T23:58:33.045368+0000 mon.a (mon.0) 2021 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:58:34.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:33 smithi136 bash[75164]: audit 2024-09-12T23:58:33.584512+0000 mon.a (mon.0) 2022 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:58:34.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:33 smithi136 bash[75164]: audit 2024-09-12T23:58:33.586164+0000 mon.a (mon.0) 2023 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:58:34.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:33 smithi136 bash[75164]: audit 2024-09-12T23:58:33.595738+0000 mon.a (mon.0) 2024 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:58:34.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:34 smithi136 bash[76371]: cluster 2024-09-12T23:58:32.995300+0000 mon.a (mon.0) 2019 : cluster [DBG] osdmap e1099: 8 total, 8 up, 8 in 2024-09-12T23:58:34.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:34 smithi136 bash[76371]: audit 2024-09-12T23:58:33.000179+0000 mon.c (mon.2) 87 : audit [INF] from='client.? 172.21.15.136:0/3753062870' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-10"}]: dispatch 2024-09-12T23:58:34.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:34 smithi136 bash[76371]: audit 2024-09-12T23:58:33.001060+0000 mon.a (mon.0) 2020 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-10"}]: dispatch 2024-09-12T23:58:34.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:34 smithi136 bash[76371]: audit 2024-09-12T23:58:33.045368+0000 mon.a (mon.0) 2021 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:58:34.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:34 smithi136 bash[76371]: audit 2024-09-12T23:58:33.584512+0000 mon.a (mon.0) 2022 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:58:34.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:34 smithi136 bash[76371]: audit 2024-09-12T23:58:33.586164+0000 mon.a (mon.0) 2023 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:58:34.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:34 smithi136 bash[76371]: audit 2024-09-12T23:58:33.595738+0000 mon.a (mon.0) 2024 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:58:34.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:33 smithi181 bash[67587]: cluster 2024-09-12T23:58:32.995300+0000 mon.a (mon.0) 2019 : cluster [DBG] osdmap e1099: 8 total, 8 up, 8 in 2024-09-12T23:58:34.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:33 smithi181 bash[67587]: audit 2024-09-12T23:58:33.000179+0000 mon.c (mon.2) 87 : audit [INF] from='client.? 172.21.15.136:0/3753062870' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-10"}]: dispatch 2024-09-12T23:58:34.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:33 smithi181 bash[67587]: audit 2024-09-12T23:58:33.001060+0000 mon.a (mon.0) 2020 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-10"}]: dispatch 2024-09-12T23:58:34.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:33 smithi181 bash[67587]: audit 2024-09-12T23:58:33.045368+0000 mon.a (mon.0) 2021 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:58:34.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:33 smithi181 bash[67587]: audit 2024-09-12T23:58:33.584512+0000 mon.a (mon.0) 2022 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:58:34.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:33 smithi181 bash[67587]: audit 2024-09-12T23:58:33.586164+0000 mon.a (mon.0) 2023 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:58:34.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:33 smithi181 bash[67587]: audit 2024-09-12T23:58:33.595738+0000 mon.a (mon.0) 2024 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:58:35.008 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] cls_rgw.set_ec (7060 ms) 2024-09-12T23:58:35.008 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 10 tests from cls_rgw (62513 ms total) 2024-09-12T23:58:35.008 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-12T23:58:35.008 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] Global test environment tear-down 2024-09-12T23:58:35.008 INFO:tasks.workunit.client.0.smithi136.stdout:[==========] 10 tests from 1 test suite ran. (62513 ms total) 2024-09-12T23:58:35.008 INFO:tasks.workunit.client.0.smithi136.stdout:[ PASSED ] 10 tests. 2024-09-12T23:58:35.012 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-12T23:58:35.012 DEBUG:teuthology.orchestra.run.smithi136:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-12T23:58:35.069 INFO:tasks.workunit:Running workunit cls/test_cls_rgw.sh... 2024-09-12T23:58:35.069 DEBUG:teuthology.orchestra.run.smithi136:workunit test cls/test_cls_rgw.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rgw.sh 2024-09-12T23:58:35.132 INFO:tasks.workunit.client.0.smithi136.stdout:Running main() from gmock_main.cc 2024-09-12T23:58:35.132 INFO:tasks.workunit.client.0.smithi136.stdout:[==========] Running 16 tests from 1 test suite. 2024-09-12T23:58:35.132 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] Global test environment set-up. 2024-09-12T23:58:35.132 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 16 tests from cls_rgw 2024-09-12T23:58:35.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:35 smithi136 bash[75164]: cluster 2024-09-12T23:58:33.062182+0000 mgr.y (mgr.24661) 643 : cluster [DBG] pgmap v1303: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:35.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:35 smithi136 bash[75164]: cluster 2024-09-12T23:58:33.984878+0000 mon.a (mon.0) 2025 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:58:35.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:35 smithi136 bash[75164]: cluster 2024-09-12T23:58:33.984936+0000 mon.a (mon.0) 2026 : cluster [INF] Cluster is now healthy 2024-09-12T23:58:35.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:35 smithi136 bash[75164]: audit 2024-09-12T23:58:34.007244+0000 mon.a (mon.0) 2027 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-10"}]': finished 2024-09-12T23:58:35.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:35 smithi136 bash[75164]: cluster 2024-09-12T23:58:34.007416+0000 mon.a (mon.0) 2028 : cluster [DBG] osdmap e1100: 8 total, 8 up, 8 in 2024-09-12T23:58:35.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:35 smithi136 bash[75164]: audit 2024-09-12T23:58:34.008693+0000 mon.c (mon.2) 88 : audit [INF] from='client.? 172.21.15.136:0/3753062870' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-10"}]: dispatch 2024-09-12T23:58:35.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:35 smithi136 bash[75164]: audit 2024-09-12T23:58:34.009451+0000 mon.a (mon.0) 2029 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-10"}]: dispatch 2024-09-12T23:58:35.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:35 smithi136 bash[76371]: cluster 2024-09-12T23:58:33.062182+0000 mgr.y (mgr.24661) 643 : cluster [DBG] pgmap v1303: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:35.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:35 smithi136 bash[76371]: cluster 2024-09-12T23:58:33.984878+0000 mon.a (mon.0) 2025 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:58:35.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:35 smithi136 bash[76371]: cluster 2024-09-12T23:58:33.984936+0000 mon.a (mon.0) 2026 : cluster [INF] Cluster is now healthy 2024-09-12T23:58:35.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:35 smithi136 bash[76371]: audit 2024-09-12T23:58:34.007244+0000 mon.a (mon.0) 2027 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-10"}]': finished 2024-09-12T23:58:35.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:35 smithi136 bash[76371]: cluster 2024-09-12T23:58:34.007416+0000 mon.a (mon.0) 2028 : cluster [DBG] osdmap e1100: 8 total, 8 up, 8 in 2024-09-12T23:58:35.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:35 smithi136 bash[76371]: audit 2024-09-12T23:58:34.008693+0000 mon.c (mon.2) 88 : audit [INF] from='client.? 172.21.15.136:0/3753062870' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-10"}]: dispatch 2024-09-12T23:58:35.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:35 smithi136 bash[76371]: audit 2024-09-12T23:58:34.009451+0000 mon.a (mon.0) 2029 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-10"}]: dispatch 2024-09-12T23:58:35.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:35 smithi181 bash[67587]: cluster 2024-09-12T23:58:33.062182+0000 mgr.y (mgr.24661) 643 : cluster [DBG] pgmap v1303: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:35.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:35 smithi181 bash[67587]: cluster 2024-09-12T23:58:33.984878+0000 mon.a (mon.0) 2025 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-12T23:58:35.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:35 smithi181 bash[67587]: cluster 2024-09-12T23:58:33.984936+0000 mon.a (mon.0) 2026 : cluster [INF] Cluster is now healthy 2024-09-12T23:58:35.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:35 smithi181 bash[67587]: audit 2024-09-12T23:58:34.007244+0000 mon.a (mon.0) 2027 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi136-110348-10"}]': finished 2024-09-12T23:58:35.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:35 smithi181 bash[67587]: cluster 2024-09-12T23:58:34.007416+0000 mon.a (mon.0) 2028 : cluster [DBG] osdmap e1100: 8 total, 8 up, 8 in 2024-09-12T23:58:35.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:35 smithi181 bash[67587]: audit 2024-09-12T23:58:34.008693+0000 mon.c (mon.2) 88 : audit [INF] from='client.? 172.21.15.136:0/3753062870' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-10"}]: dispatch 2024-09-12T23:58:35.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:35 smithi181 bash[67587]: audit 2024-09-12T23:58:34.009451+0000 mon.a (mon.0) 2029 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-10"}]: dispatch 2024-09-12T23:58:36.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:36 smithi181 bash[67587]: audit 2024-09-12T23:58:35.006077+0000 mon.a (mon.0) 2030 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-10"}]': finished 2024-09-12T23:58:36.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:36 smithi181 bash[67587]: cluster 2024-09-12T23:58:35.006278+0000 mon.a (mon.0) 2031 : cluster [DBG] osdmap e1101: 8 total, 8 up, 8 in 2024-09-12T23:58:36.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:36 smithi181 bash[67587]: cluster 2024-09-12T23:58:35.062691+0000 mgr.y (mgr.24661) 644 : cluster [DBG] pgmap v1306: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:36.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:36 smithi136 bash[75164]: audit 2024-09-12T23:58:35.006077+0000 mon.a (mon.0) 2030 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-10"}]': finished 2024-09-12T23:58:36.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:36 smithi136 bash[75164]: cluster 2024-09-12T23:58:35.006278+0000 mon.a (mon.0) 2031 : cluster [DBG] osdmap e1101: 8 total, 8 up, 8 in 2024-09-12T23:58:36.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:36 smithi136 bash[75164]: cluster 2024-09-12T23:58:35.062691+0000 mgr.y (mgr.24661) 644 : cluster [DBG] pgmap v1306: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:36.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:36 smithi136 bash[76371]: audit 2024-09-12T23:58:35.006077+0000 mon.a (mon.0) 2030 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi136-110348-10"}]': finished 2024-09-12T23:58:36.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:36 smithi136 bash[76371]: cluster 2024-09-12T23:58:35.006278+0000 mon.a (mon.0) 2031 : cluster [DBG] osdmap e1101: 8 total, 8 up, 8 in 2024-09-12T23:58:36.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:36 smithi136 bash[76371]: cluster 2024-09-12T23:58:35.062691+0000 mgr.y (mgr.24661) 644 : cluster [DBG] pgmap v1306: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:37.031 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] cls_rgw.index_basic 2024-09-12T23:58:37.105 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] cls_rgw.index_basic (74 ms) 2024-09-12T23:58:37.105 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] cls_rgw.index_multiple_obj_writers 2024-09-12T23:58:37.164 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] cls_rgw.index_multiple_obj_writers (59 ms) 2024-09-12T23:58:37.164 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] cls_rgw.index_remove_object 2024-09-12T23:58:37.252 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] cls_rgw.index_remove_object (87 ms) 2024-09-12T23:58:37.252 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] cls_rgw.index_suggest 2024-09-12T23:58:37.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:37 smithi181 bash[67587]: cluster 2024-09-12T23:58:36.030049+0000 mon.a (mon.0) 2032 : cluster [DBG] osdmap e1102: 8 total, 8 up, 8 in 2024-09-12T23:58:37.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:37 smithi181 bash[67587]: audit 2024-09-12T23:58:36.032541+0000 mon.c (mon.2) 89 : audit [INF] from='client.? 172.21.15.136:0/1451094137' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110551-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:58:37.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:37 smithi181 bash[67587]: audit 2024-09-12T23:58:36.033963+0000 mon.a (mon.0) 2033 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110551-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:58:37.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:37 smithi136 bash[75164]: cluster 2024-09-12T23:58:36.030049+0000 mon.a (mon.0) 2032 : cluster [DBG] osdmap e1102: 8 total, 8 up, 8 in 2024-09-12T23:58:37.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:37 smithi136 bash[75164]: audit 2024-09-12T23:58:36.032541+0000 mon.c (mon.2) 89 : audit [INF] from='client.? 172.21.15.136:0/1451094137' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110551-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:58:37.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:37 smithi136 bash[75164]: audit 2024-09-12T23:58:36.033963+0000 mon.a (mon.0) 2033 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110551-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:58:37.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:37 smithi136 bash[76371]: cluster 2024-09-12T23:58:36.030049+0000 mon.a (mon.0) 2032 : cluster [DBG] osdmap e1102: 8 total, 8 up, 8 in 2024-09-12T23:58:37.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:37 smithi136 bash[76371]: audit 2024-09-12T23:58:36.032541+0000 mon.c (mon.2) 89 : audit [INF] from='client.? 172.21.15.136:0/1451094137' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110551-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:58:37.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:37 smithi136 bash[76371]: audit 2024-09-12T23:58:36.033963+0000 mon.a (mon.0) 2033 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110551-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-12T23:58:38.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:38 smithi181 bash[67587]: audit 2024-09-12T23:58:37.031183+0000 mon.a (mon.0) 2034 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110551-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:58:38.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:38 smithi181 bash[67587]: cluster 2024-09-12T23:58:37.031927+0000 mon.a (mon.0) 2035 : cluster [DBG] osdmap e1103: 8 total, 8 up, 8 in 2024-09-12T23:58:38.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:38 smithi181 bash[67587]: cluster 2024-09-12T23:58:37.063368+0000 mgr.y (mgr.24661) 645 : cluster [DBG] pgmap v1309: 137 pgs: 14 creating+peering, 18 unknown, 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:38.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:38 smithi136 bash[75164]: audit 2024-09-12T23:58:37.031183+0000 mon.a (mon.0) 2034 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110551-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:58:38.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:38 smithi136 bash[75164]: cluster 2024-09-12T23:58:37.031927+0000 mon.a (mon.0) 2035 : cluster [DBG] osdmap e1103: 8 total, 8 up, 8 in 2024-09-12T23:58:38.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:38 smithi136 bash[75164]: cluster 2024-09-12T23:58:37.063368+0000 mgr.y (mgr.24661) 645 : cluster [DBG] pgmap v1309: 137 pgs: 14 creating+peering, 18 unknown, 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:38.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:38 smithi136 bash[76371]: audit 2024-09-12T23:58:37.031183+0000 mon.a (mon.0) 2034 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110551-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-12T23:58:38.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:38 smithi136 bash[76371]: cluster 2024-09-12T23:58:37.031927+0000 mon.a (mon.0) 2035 : cluster [DBG] osdmap e1103: 8 total, 8 up, 8 in 2024-09-12T23:58:38.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:38 smithi136 bash[76371]: cluster 2024-09-12T23:58:37.063368+0000 mgr.y (mgr.24661) 645 : cluster [DBG] pgmap v1309: 137 pgs: 14 creating+peering, 18 unknown, 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-12T23:58:39.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:39 smithi181 bash[67587]: cluster 2024-09-12T23:58:38.067795+0000 mon.a (mon.0) 2036 : cluster [DBG] osdmap e1104: 8 total, 8 up, 8 in 2024-09-12T23:58:39.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:39 smithi181 bash[67587]: audit 2024-09-12T23:58:38.766527+0000 mon.a (mon.0) 2037 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:58:39.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:39 smithi136 bash[75164]: cluster 2024-09-12T23:58:38.067795+0000 mon.a (mon.0) 2036 : cluster [DBG] osdmap e1104: 8 total, 8 up, 8 in 2024-09-12T23:58:39.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:39 smithi136 bash[75164]: audit 2024-09-12T23:58:38.766527+0000 mon.a (mon.0) 2037 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:58:39.414 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:39 smithi136 bash[76371]: cluster 2024-09-12T23:58:38.067795+0000 mon.a (mon.0) 2036 : cluster [DBG] osdmap e1104: 8 total, 8 up, 8 in 2024-09-12T23:58:39.414 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:39 smithi136 bash[76371]: audit 2024-09-12T23:58:38.766527+0000 mon.a (mon.0) 2037 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:58:39.512 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] cls_rgw.index_suggest (2260 ms) 2024-09-12T23:58:39.512 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] cls_rgw.index_suggest_complete 2024-09-12T23:58:39.526 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] cls_rgw.index_suggest_complete (14 ms) 2024-09-12T23:58:39.526 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] cls_rgw.index_list 2024-09-12T23:58:39.571 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] cls_rgw.index_list (44 ms) 2024-09-12T23:58:39.571 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] cls_rgw.index_list_delimited 2024-09-12T23:58:39.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:58:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:58:39] "GET /metrics HTTP/1.1" 200 36465 "" "Prometheus/2.43.0" 2024-09-12T23:58:40.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:40 smithi181 bash[67587]: cluster 2024-09-12T23:58:39.064203+0000 mgr.y (mgr.24661) 646 : cluster [DBG] pgmap v1311: 137 pgs: 14 creating+peering, 11 unknown, 112 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 22 KiB/s rd, 13 KiB/s wr, 40 op/s 2024-09-12T23:58:40.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:40 smithi136 bash[75164]: cluster 2024-09-12T23:58:39.064203+0000 mgr.y (mgr.24661) 646 : cluster [DBG] pgmap v1311: 137 pgs: 14 creating+peering, 11 unknown, 112 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 22 KiB/s rd, 13 KiB/s wr, 40 op/s 2024-09-12T23:58:40.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:40 smithi136 bash[76371]: cluster 2024-09-12T23:58:39.064203+0000 mgr.y (mgr.24661) 646 : cluster [DBG] pgmap v1311: 137 pgs: 14 creating+peering, 11 unknown, 112 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 22 KiB/s rd, 13 KiB/s wr, 40 op/s 2024-09-12T23:58:42.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:42 smithi136 bash[75164]: cluster 2024-09-12T23:58:41.065288+0000 mgr.y (mgr.24661) 647 : cluster [DBG] pgmap v1312: 137 pgs: 14 creating+peering, 123 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 176 KiB/s rd, 88 KiB/s wr, 297 op/s 2024-09-12T23:58:42.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:42 smithi136 bash[76371]: cluster 2024-09-12T23:58:41.065288+0000 mgr.y (mgr.24661) 647 : cluster [DBG] pgmap v1312: 137 pgs: 14 creating+peering, 123 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 176 KiB/s rd, 88 KiB/s wr, 297 op/s 2024-09-12T23:58:42.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:42 smithi181 bash[67587]: cluster 2024-09-12T23:58:41.065288+0000 mgr.y (mgr.24661) 647 : cluster [DBG] pgmap v1312: 137 pgs: 14 creating+peering, 123 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 176 KiB/s rd, 88 KiB/s wr, 297 op/s 2024-09-12T23:58:42.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:58:42 smithi181 bash[95505]: ts=2024-09-12T23:58:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:58:44.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:44 smithi136 bash[75164]: cluster 2024-09-12T23:58:43.066452+0000 mgr.y (mgr.24661) 648 : cluster [DBG] pgmap v1313: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 347 KiB/s rd, 207 KiB/s wr, 642 op/s 2024-09-12T23:58:44.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:44 smithi136 bash[76371]: cluster 2024-09-12T23:58:43.066452+0000 mgr.y (mgr.24661) 648 : cluster [DBG] pgmap v1313: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 347 KiB/s rd, 207 KiB/s wr, 642 op/s 2024-09-12T23:58:44.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:44 smithi181 bash[67587]: cluster 2024-09-12T23:58:43.066452+0000 mgr.y (mgr.24661) 648 : cluster [DBG] pgmap v1313: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 347 KiB/s rd, 207 KiB/s wr, 642 op/s 2024-09-12T23:58:46.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:46 smithi136 bash[75164]: cluster 2024-09-12T23:58:45.067465+0000 mgr.y (mgr.24661) 649 : cluster [DBG] pgmap v1314: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 304 KiB/s rd, 181 KiB/s wr, 562 op/s 2024-09-12T23:58:46.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:46 smithi136 bash[76371]: cluster 2024-09-12T23:58:45.067465+0000 mgr.y (mgr.24661) 649 : cluster [DBG] pgmap v1314: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 304 KiB/s rd, 181 KiB/s wr, 562 op/s 2024-09-12T23:58:46.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:46 smithi181 bash[67587]: cluster 2024-09-12T23:58:45.067465+0000 mgr.y (mgr.24661) 649 : cluster [DBG] pgmap v1314: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 304 KiB/s rd, 181 KiB/s wr, 562 op/s 2024-09-12T23:58:48.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:48 smithi136 bash[75164]: cluster 2024-09-12T23:58:47.069101+0000 mgr.y (mgr.24661) 650 : cluster [DBG] pgmap v1315: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 483 KiB/s rd, 304 KiB/s wr, 928 op/s 2024-09-12T23:58:48.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:48 smithi136 bash[76371]: cluster 2024-09-12T23:58:47.069101+0000 mgr.y (mgr.24661) 650 : cluster [DBG] pgmap v1315: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 483 KiB/s rd, 304 KiB/s wr, 928 op/s 2024-09-12T23:58:48.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:48 smithi181 bash[67587]: cluster 2024-09-12T23:58:47.069101+0000 mgr.y (mgr.24661) 650 : cluster [DBG] pgmap v1315: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 483 KiB/s rd, 304 KiB/s wr, 928 op/s 2024-09-12T23:58:49.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:58:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:58:49] "GET /metrics HTTP/1.1" 200 36465 "" "Prometheus/2.43.0" 2024-09-12T23:58:50.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:50 smithi136 bash[75164]: cluster 2024-09-12T23:58:49.070068+0000 mgr.y (mgr.24661) 651 : cluster [DBG] pgmap v1316: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 430 KiB/s rd, 272 KiB/s wr, 828 op/s 2024-09-12T23:58:50.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:50 smithi136 bash[76371]: cluster 2024-09-12T23:58:49.070068+0000 mgr.y (mgr.24661) 651 : cluster [DBG] pgmap v1316: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 430 KiB/s rd, 272 KiB/s wr, 828 op/s 2024-09-12T23:58:50.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:50 smithi181 bash[67587]: cluster 2024-09-12T23:58:49.070068+0000 mgr.y (mgr.24661) 651 : cluster [DBG] pgmap v1316: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 430 KiB/s rd, 272 KiB/s wr, 828 op/s 2024-09-12T23:58:52.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:52 smithi136 bash[75164]: cluster 2024-09-12T23:58:51.071285+0000 mgr.y (mgr.24661) 652 : cluster [DBG] pgmap v1317: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 395 KiB/s rd, 249 KiB/s wr, 759 op/s 2024-09-12T23:58:52.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:52 smithi136 bash[76371]: cluster 2024-09-12T23:58:51.071285+0000 mgr.y (mgr.24661) 652 : cluster [DBG] pgmap v1317: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 395 KiB/s rd, 249 KiB/s wr, 759 op/s 2024-09-12T23:58:52.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:52 smithi181 bash[67587]: cluster 2024-09-12T23:58:51.071285+0000 mgr.y (mgr.24661) 652 : cluster [DBG] pgmap v1317: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 395 KiB/s rd, 249 KiB/s wr, 759 op/s 2024-09-12T23:58:52.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:58:52 smithi181 bash[95505]: ts=2024-09-12T23:58:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:58:54.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:54 smithi136 bash[75164]: cluster 2024-09-12T23:58:53.072430+0000 mgr.y (mgr.24661) 653 : cluster [DBG] pgmap v1318: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 484 KiB/s rd, 323 KiB/s wr, 964 op/s 2024-09-12T23:58:54.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:54 smithi136 bash[75164]: audit 2024-09-12T23:58:53.767226+0000 mon.a (mon.0) 2038 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:58:54.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:54 smithi136 bash[76371]: cluster 2024-09-12T23:58:53.072430+0000 mgr.y (mgr.24661) 653 : cluster [DBG] pgmap v1318: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 484 KiB/s rd, 323 KiB/s wr, 964 op/s 2024-09-12T23:58:54.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:54 smithi136 bash[76371]: audit 2024-09-12T23:58:53.767226+0000 mon.a (mon.0) 2038 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:58:54.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:54 smithi181 bash[67587]: cluster 2024-09-12T23:58:53.072430+0000 mgr.y (mgr.24661) 653 : cluster [DBG] pgmap v1318: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 484 KiB/s rd, 323 KiB/s wr, 964 op/s 2024-09-12T23:58:54.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:54 smithi181 bash[67587]: audit 2024-09-12T23:58:53.767226+0000 mon.a (mon.0) 2038 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:58:56.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:56 smithi136 bash[75164]: cluster 2024-09-12T23:58:55.073284+0000 mgr.y (mgr.24661) 654 : cluster [DBG] pgmap v1319: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 368 KiB/s rd, 245 KiB/s wr, 736 op/s 2024-09-12T23:58:56.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:56 smithi136 bash[76371]: cluster 2024-09-12T23:58:55.073284+0000 mgr.y (mgr.24661) 654 : cluster [DBG] pgmap v1319: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 368 KiB/s rd, 245 KiB/s wr, 736 op/s 2024-09-12T23:58:56.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:56 smithi181 bash[67587]: cluster 2024-09-12T23:58:55.073284+0000 mgr.y (mgr.24661) 654 : cluster [DBG] pgmap v1319: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 368 KiB/s rd, 245 KiB/s wr, 736 op/s 2024-09-12T23:58:58.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:58:58 smithi136 bash[76371]: cluster 2024-09-12T23:58:57.074537+0000 mgr.y (mgr.24661) 655 : cluster [DBG] pgmap v1320: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 517 KiB/s rd, 345 KiB/s wr, 1.03k op/s 2024-09-12T23:58:58.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:58:58 smithi136 bash[75164]: cluster 2024-09-12T23:58:57.074537+0000 mgr.y (mgr.24661) 655 : cluster [DBG] pgmap v1320: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 517 KiB/s rd, 345 KiB/s wr, 1.03k op/s 2024-09-12T23:58:58.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:58:58 smithi181 bash[67587]: cluster 2024-09-12T23:58:57.074537+0000 mgr.y (mgr.24661) 655 : cluster [DBG] pgmap v1320: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 517 KiB/s rd, 345 KiB/s wr, 1.03k op/s 2024-09-12T23:58:59.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:58:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:58:59] "GET /metrics HTTP/1.1" 200 36473 "" "Prometheus/2.43.0" 2024-09-12T23:59:00.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:59:00 smithi136 bash[75164]: cluster 2024-09-12T23:58:59.075558+0000 mgr.y (mgr.24661) 656 : cluster [DBG] pgmap v1321: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 319 KiB/s rd, 213 KiB/s wr, 637 op/s 2024-09-12T23:59:00.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:59:00 smithi136 bash[76371]: cluster 2024-09-12T23:58:59.075558+0000 mgr.y (mgr.24661) 656 : cluster [DBG] pgmap v1321: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 319 KiB/s rd, 213 KiB/s wr, 637 op/s 2024-09-12T23:59:00.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:59:00 smithi181 bash[67587]: cluster 2024-09-12T23:58:59.075558+0000 mgr.y (mgr.24661) 656 : cluster [DBG] pgmap v1321: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 319 KiB/s rd, 213 KiB/s wr, 637 op/s 2024-09-12T23:59:02.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:59:02 smithi181 bash[67587]: cluster 2024-09-12T23:59:01.076478+0000 mgr.y (mgr.24661) 657 : cluster [DBG] pgmap v1322: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 319 KiB/s rd, 213 KiB/s wr, 637 op/s 2024-09-12T23:59:02.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:59:02 smithi181 bash[95505]: ts=2024-09-12T23:59:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:59:02.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:59:02 smithi136 bash[76371]: cluster 2024-09-12T23:59:01.076478+0000 mgr.y (mgr.24661) 657 : cluster [DBG] pgmap v1322: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 319 KiB/s rd, 213 KiB/s wr, 637 op/s 2024-09-12T23:59:02.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:59:02 smithi136 bash[75164]: cluster 2024-09-12T23:59:01.076478+0000 mgr.y (mgr.24661) 657 : cluster [DBG] pgmap v1322: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 319 KiB/s rd, 213 KiB/s wr, 637 op/s 2024-09-12T23:59:04.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:59:04 smithi136 bash[76371]: cluster 2024-09-12T23:59:03.077898+0000 mgr.y (mgr.24661) 658 : cluster [DBG] pgmap v1323: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 455 KiB/s rd, 303 KiB/s wr, 909 op/s 2024-09-12T23:59:04.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:59:04 smithi136 bash[75164]: cluster 2024-09-12T23:59:03.077898+0000 mgr.y (mgr.24661) 658 : cluster [DBG] pgmap v1323: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 455 KiB/s rd, 303 KiB/s wr, 909 op/s 2024-09-12T23:59:04.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:59:04 smithi181 bash[67587]: cluster 2024-09-12T23:59:03.077898+0000 mgr.y (mgr.24661) 658 : cluster [DBG] pgmap v1323: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 455 KiB/s rd, 303 KiB/s wr, 909 op/s 2024-09-12T23:59:06.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:59:06 smithi136 bash[76371]: cluster 2024-09-12T23:59:05.078836+0000 mgr.y (mgr.24661) 659 : cluster [DBG] pgmap v1324: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 285 KiB/s rd, 190 KiB/s wr, 570 op/s 2024-09-12T23:59:06.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:59:06 smithi136 bash[75164]: cluster 2024-09-12T23:59:05.078836+0000 mgr.y (mgr.24661) 659 : cluster [DBG] pgmap v1324: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 285 KiB/s rd, 190 KiB/s wr, 570 op/s 2024-09-12T23:59:06.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:59:06 smithi181 bash[67587]: cluster 2024-09-12T23:59:05.078836+0000 mgr.y (mgr.24661) 659 : cluster [DBG] pgmap v1324: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 285 KiB/s rd, 190 KiB/s wr, 570 op/s 2024-09-12T23:59:08.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:59:08 smithi136 bash[75164]: cluster 2024-09-12T23:59:07.080044+0000 mgr.y (mgr.24661) 660 : cluster [DBG] pgmap v1325: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 408 KiB/s rd, 272 KiB/s wr, 815 op/s 2024-09-12T23:59:08.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:59:08 smithi136 bash[76371]: cluster 2024-09-12T23:59:07.080044+0000 mgr.y (mgr.24661) 660 : cluster [DBG] pgmap v1325: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 408 KiB/s rd, 272 KiB/s wr, 815 op/s 2024-09-12T23:59:08.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:59:08 smithi181 bash[67587]: cluster 2024-09-12T23:59:07.080044+0000 mgr.y (mgr.24661) 660 : cluster [DBG] pgmap v1325: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 408 KiB/s rd, 272 KiB/s wr, 815 op/s 2024-09-12T23:59:09.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:59:09 smithi136 bash[75164]: audit 2024-09-12T23:59:08.767208+0000 mon.a (mon.0) 2039 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:59:09.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:59:09 smithi136 bash[76371]: audit 2024-09-12T23:59:08.767208+0000 mon.a (mon.0) 2039 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:59:09.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:59:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:59:09] "GET /metrics HTTP/1.1" 200 36471 "" "Prometheus/2.43.0" 2024-09-12T23:59:09.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:59:09 smithi181 bash[67587]: audit 2024-09-12T23:59:08.767208+0000 mon.a (mon.0) 2039 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:59:10.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:59:10 smithi136 bash[75164]: cluster 2024-09-12T23:59:09.081059+0000 mgr.y (mgr.24661) 661 : cluster [DBG] pgmap v1326: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 258 KiB/s rd, 172 KiB/s wr, 516 op/s 2024-09-12T23:59:10.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:59:10 smithi136 bash[76371]: cluster 2024-09-12T23:59:09.081059+0000 mgr.y (mgr.24661) 661 : cluster [DBG] pgmap v1326: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 258 KiB/s rd, 172 KiB/s wr, 516 op/s 2024-09-12T23:59:10.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:59:10 smithi181 bash[67587]: cluster 2024-09-12T23:59:09.081059+0000 mgr.y (mgr.24661) 661 : cluster [DBG] pgmap v1326: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 258 KiB/s rd, 172 KiB/s wr, 516 op/s 2024-09-12T23:59:12.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:59:12 smithi181 bash[67587]: cluster 2024-09-12T23:59:11.082179+0000 mgr.y (mgr.24661) 662 : cluster [DBG] pgmap v1327: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 258 KiB/s rd, 172 KiB/s wr, 516 op/s 2024-09-12T23:59:12.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:59:12 smithi181 bash[95505]: ts=2024-09-12T23:59:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:59:12.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:59:12 smithi136 bash[75164]: cluster 2024-09-12T23:59:11.082179+0000 mgr.y (mgr.24661) 662 : cluster [DBG] pgmap v1327: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 258 KiB/s rd, 172 KiB/s wr, 516 op/s 2024-09-12T23:59:12.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:59:12 smithi136 bash[76371]: cluster 2024-09-12T23:59:11.082179+0000 mgr.y (mgr.24661) 662 : cluster [DBG] pgmap v1327: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 258 KiB/s rd, 172 KiB/s wr, 516 op/s 2024-09-12T23:59:14.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:59:14 smithi136 bash[75164]: cluster 2024-09-12T23:59:13.083229+0000 mgr.y (mgr.24661) 663 : cluster [DBG] pgmap v1328: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 378 KiB/s rd, 252 KiB/s wr, 755 op/s 2024-09-12T23:59:14.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:59:14 smithi136 bash[76371]: cluster 2024-09-12T23:59:13.083229+0000 mgr.y (mgr.24661) 663 : cluster [DBG] pgmap v1328: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 378 KiB/s rd, 252 KiB/s wr, 755 op/s 2024-09-12T23:59:14.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:59:14 smithi181 bash[67587]: cluster 2024-09-12T23:59:13.083229+0000 mgr.y (mgr.24661) 663 : cluster [DBG] pgmap v1328: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 378 KiB/s rd, 252 KiB/s wr, 755 op/s 2024-09-12T23:59:16.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:59:16 smithi136 bash[75164]: cluster 2024-09-12T23:59:15.084224+0000 mgr.y (mgr.24661) 664 : cluster [DBG] pgmap v1329: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 242 KiB/s rd, 161 KiB/s wr, 483 op/s 2024-09-12T23:59:16.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:59:16 smithi136 bash[76371]: cluster 2024-09-12T23:59:15.084224+0000 mgr.y (mgr.24661) 664 : cluster [DBG] pgmap v1329: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 242 KiB/s rd, 161 KiB/s wr, 483 op/s 2024-09-12T23:59:16.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:59:16 smithi181 bash[67587]: cluster 2024-09-12T23:59:15.084224+0000 mgr.y (mgr.24661) 664 : cluster [DBG] pgmap v1329: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 242 KiB/s rd, 161 KiB/s wr, 483 op/s 2024-09-12T23:59:18.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:59:18 smithi136 bash[76371]: cluster 2024-09-12T23:59:17.085328+0000 mgr.y (mgr.24661) 665 : cluster [DBG] pgmap v1330: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 361 KiB/s rd, 241 KiB/s wr, 722 op/s 2024-09-12T23:59:18.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:59:18 smithi136 bash[75164]: cluster 2024-09-12T23:59:17.085328+0000 mgr.y (mgr.24661) 665 : cluster [DBG] pgmap v1330: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 361 KiB/s rd, 241 KiB/s wr, 722 op/s 2024-09-12T23:59:18.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:59:18 smithi181 bash[67587]: cluster 2024-09-12T23:59:17.085328+0000 mgr.y (mgr.24661) 665 : cluster [DBG] pgmap v1330: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 361 KiB/s rd, 241 KiB/s wr, 722 op/s 2024-09-12T23:59:19.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:59:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:59:19] "GET /metrics HTTP/1.1" 200 36471 "" "Prometheus/2.43.0" 2024-09-12T23:59:20.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:59:20 smithi136 bash[75164]: cluster 2024-09-12T23:59:19.086591+0000 mgr.y (mgr.24661) 666 : cluster [DBG] pgmap v1331: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 239 KiB/s rd, 159 KiB/s wr, 477 op/s 2024-09-12T23:59:20.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:59:20 smithi136 bash[76371]: cluster 2024-09-12T23:59:19.086591+0000 mgr.y (mgr.24661) 666 : cluster [DBG] pgmap v1331: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 239 KiB/s rd, 159 KiB/s wr, 477 op/s 2024-09-12T23:59:20.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:59:20 smithi181 bash[67587]: cluster 2024-09-12T23:59:19.086591+0000 mgr.y (mgr.24661) 666 : cluster [DBG] pgmap v1331: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 239 KiB/s rd, 159 KiB/s wr, 477 op/s 2024-09-12T23:59:22.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:59:22 smithi181 bash[67587]: cluster 2024-09-12T23:59:21.087778+0000 mgr.y (mgr.24661) 667 : cluster [DBG] pgmap v1332: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 239 KiB/s rd, 159 KiB/s wr, 477 op/s 2024-09-12T23:59:22.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:59:22 smithi181 bash[95505]: ts=2024-09-12T23:59:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:59:22.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:59:22 smithi136 bash[75164]: cluster 2024-09-12T23:59:21.087778+0000 mgr.y (mgr.24661) 667 : cluster [DBG] pgmap v1332: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 239 KiB/s rd, 159 KiB/s wr, 477 op/s 2024-09-12T23:59:22.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:59:22 smithi136 bash[76371]: cluster 2024-09-12T23:59:21.087778+0000 mgr.y (mgr.24661) 667 : cluster [DBG] pgmap v1332: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 239 KiB/s rd, 159 KiB/s wr, 477 op/s 2024-09-12T23:59:24.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:59:24 smithi136 bash[76371]: cluster 2024-09-12T23:59:23.089060+0000 mgr.y (mgr.24661) 668 : cluster [DBG] pgmap v1333: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 358 KiB/s rd, 239 KiB/s wr, 716 op/s 2024-09-12T23:59:24.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:59:24 smithi136 bash[76371]: audit 2024-09-12T23:59:23.767493+0000 mon.a (mon.0) 2040 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:59:24.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:59:24 smithi136 bash[75164]: cluster 2024-09-12T23:59:23.089060+0000 mgr.y (mgr.24661) 668 : cluster [DBG] pgmap v1333: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 358 KiB/s rd, 239 KiB/s wr, 716 op/s 2024-09-12T23:59:24.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:59:24 smithi136 bash[75164]: audit 2024-09-12T23:59:23.767493+0000 mon.a (mon.0) 2040 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:59:24.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:59:24 smithi181 bash[67587]: cluster 2024-09-12T23:59:23.089060+0000 mgr.y (mgr.24661) 668 : cluster [DBG] pgmap v1333: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 358 KiB/s rd, 239 KiB/s wr, 716 op/s 2024-09-12T23:59:24.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:59:24 smithi181 bash[67587]: audit 2024-09-12T23:59:23.767493+0000 mon.a (mon.0) 2040 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:59:26.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:59:26 smithi136 bash[75164]: cluster 2024-09-12T23:59:25.090035+0000 mgr.y (mgr.24661) 669 : cluster [DBG] pgmap v1334: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 239 KiB/s rd, 159 KiB/s wr, 477 op/s 2024-09-12T23:59:26.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:59:26 smithi136 bash[76371]: cluster 2024-09-12T23:59:25.090035+0000 mgr.y (mgr.24661) 669 : cluster [DBG] pgmap v1334: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 239 KiB/s rd, 159 KiB/s wr, 477 op/s 2024-09-12T23:59:26.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:59:26 smithi181 bash[67587]: cluster 2024-09-12T23:59:25.090035+0000 mgr.y (mgr.24661) 669 : cluster [DBG] pgmap v1334: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 239 KiB/s rd, 159 KiB/s wr, 477 op/s 2024-09-12T23:59:28.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:59:28 smithi136 bash[75164]: cluster 2024-09-12T23:59:27.091432+0000 mgr.y (mgr.24661) 670 : cluster [DBG] pgmap v1335: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 358 KiB/s rd, 239 KiB/s wr, 716 op/s 2024-09-12T23:59:28.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:59:28 smithi136 bash[76371]: cluster 2024-09-12T23:59:27.091432+0000 mgr.y (mgr.24661) 670 : cluster [DBG] pgmap v1335: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 358 KiB/s rd, 239 KiB/s wr, 716 op/s 2024-09-12T23:59:28.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:59:28 smithi181 bash[67587]: cluster 2024-09-12T23:59:27.091432+0000 mgr.y (mgr.24661) 670 : cluster [DBG] pgmap v1335: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 358 KiB/s rd, 239 KiB/s wr, 716 op/s 2024-09-12T23:59:29.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:59:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:59:29] "GET /metrics HTTP/1.1" 200 36474 "" "Prometheus/2.43.0" 2024-09-12T23:59:30.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:59:30 smithi136 bash[75164]: cluster 2024-09-12T23:59:29.092339+0000 mgr.y (mgr.24661) 671 : cluster [DBG] pgmap v1336: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 239 KiB/s rd, 159 KiB/s wr, 478 op/s 2024-09-12T23:59:30.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:59:30 smithi136 bash[76371]: cluster 2024-09-12T23:59:29.092339+0000 mgr.y (mgr.24661) 671 : cluster [DBG] pgmap v1336: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 239 KiB/s rd, 159 KiB/s wr, 478 op/s 2024-09-12T23:59:30.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:59:30 smithi181 bash[67587]: cluster 2024-09-12T23:59:29.092339+0000 mgr.y (mgr.24661) 671 : cluster [DBG] pgmap v1336: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 239 KiB/s rd, 159 KiB/s wr, 478 op/s 2024-09-12T23:59:32.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:59:32 smithi181 bash[67587]: cluster 2024-09-12T23:59:31.093518+0000 mgr.y (mgr.24661) 672 : cluster [DBG] pgmap v1337: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 239 KiB/s rd, 159 KiB/s wr, 478 op/s 2024-09-12T23:59:32.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:59:32 smithi181 bash[95505]: ts=2024-09-12T23:59:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:59:32.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:59:32 smithi136 bash[75164]: cluster 2024-09-12T23:59:31.093518+0000 mgr.y (mgr.24661) 672 : cluster [DBG] pgmap v1337: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 239 KiB/s rd, 159 KiB/s wr, 478 op/s 2024-09-12T23:59:32.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:59:32 smithi136 bash[76371]: cluster 2024-09-12T23:59:31.093518+0000 mgr.y (mgr.24661) 672 : cluster [DBG] pgmap v1337: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 239 KiB/s rd, 159 KiB/s wr, 478 op/s 2024-09-12T23:59:34.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:59:34 smithi136 bash[76371]: cluster 2024-09-12T23:59:33.094730+0000 mgr.y (mgr.24661) 673 : cluster [DBG] pgmap v1338: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 356 KiB/s rd, 237 KiB/s wr, 712 op/s 2024-09-12T23:59:34.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:59:34 smithi136 bash[76371]: audit 2024-09-12T23:59:33.725106+0000 mon.a (mon.0) 2041 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:59:34.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:59:34 smithi136 bash[76371]: audit 2024-09-12T23:59:34.281368+0000 mon.a (mon.0) 2042 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:59:34.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:59:34 smithi136 bash[76371]: audit 2024-09-12T23:59:34.282310+0000 mon.a (mon.0) 2043 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:59:34.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:59:34 smithi136 bash[76371]: audit 2024-09-12T23:59:34.287862+0000 mon.a (mon.0) 2044 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:59:34.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:59:34 smithi136 bash[75164]: cluster 2024-09-12T23:59:33.094730+0000 mgr.y (mgr.24661) 673 : cluster [DBG] pgmap v1338: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 356 KiB/s rd, 237 KiB/s wr, 712 op/s 2024-09-12T23:59:34.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:59:34 smithi136 bash[75164]: audit 2024-09-12T23:59:33.725106+0000 mon.a (mon.0) 2041 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:59:34.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:59:34 smithi136 bash[75164]: audit 2024-09-12T23:59:34.281368+0000 mon.a (mon.0) 2042 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:59:34.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:59:34 smithi136 bash[75164]: audit 2024-09-12T23:59:34.282310+0000 mon.a (mon.0) 2043 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:59:34.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:59:34 smithi136 bash[75164]: audit 2024-09-12T23:59:34.287862+0000 mon.a (mon.0) 2044 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:59:34.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:59:34 smithi181 bash[67587]: cluster 2024-09-12T23:59:33.094730+0000 mgr.y (mgr.24661) 673 : cluster [DBG] pgmap v1338: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 356 KiB/s rd, 237 KiB/s wr, 712 op/s 2024-09-12T23:59:34.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:59:34 smithi181 bash[67587]: audit 2024-09-12T23:59:33.725106+0000 mon.a (mon.0) 2041 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-12T23:59:34.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:59:34 smithi181 bash[67587]: audit 2024-09-12T23:59:34.281368+0000 mon.a (mon.0) 2042 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-12T23:59:34.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:59:34 smithi181 bash[67587]: audit 2024-09-12T23:59:34.282310+0000 mon.a (mon.0) 2043 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-12T23:59:34.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:59:34 smithi181 bash[67587]: audit 2024-09-12T23:59:34.287862+0000 mon.a (mon.0) 2044 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-12T23:59:36.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:59:36 smithi136 bash[75164]: cluster 2024-09-12T23:59:35.095667+0000 mgr.y (mgr.24661) 674 : cluster [DBG] pgmap v1339: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 237 KiB/s rd, 158 KiB/s wr, 473 op/s 2024-09-12T23:59:36.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:59:36 smithi136 bash[76371]: cluster 2024-09-12T23:59:35.095667+0000 mgr.y (mgr.24661) 674 : cluster [DBG] pgmap v1339: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 237 KiB/s rd, 158 KiB/s wr, 473 op/s 2024-09-12T23:59:36.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:59:36 smithi181 bash[67587]: cluster 2024-09-12T23:59:35.095667+0000 mgr.y (mgr.24661) 674 : cluster [DBG] pgmap v1339: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 237 KiB/s rd, 158 KiB/s wr, 473 op/s 2024-09-12T23:59:38.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:59:38 smithi136 bash[75164]: cluster 2024-09-12T23:59:37.097075+0000 mgr.y (mgr.24661) 675 : cluster [DBG] pgmap v1340: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 358 KiB/s rd, 238 KiB/s wr, 715 op/s 2024-09-12T23:59:38.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:59:38 smithi136 bash[76371]: cluster 2024-09-12T23:59:37.097075+0000 mgr.y (mgr.24661) 675 : cluster [DBG] pgmap v1340: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 358 KiB/s rd, 238 KiB/s wr, 715 op/s 2024-09-12T23:59:38.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:59:38 smithi181 bash[67587]: cluster 2024-09-12T23:59:37.097075+0000 mgr.y (mgr.24661) 675 : cluster [DBG] pgmap v1340: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 358 KiB/s rd, 238 KiB/s wr, 715 op/s 2024-09-12T23:59:39.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:59:39 smithi136 bash[75164]: audit 2024-09-12T23:59:38.768814+0000 mon.a (mon.0) 2045 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:59:39.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:59:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:59:39] "GET /metrics HTTP/1.1" 200 36476 "" "Prometheus/2.43.0" 2024-09-12T23:59:39.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:59:39 smithi136 bash[76371]: audit 2024-09-12T23:59:38.768814+0000 mon.a (mon.0) 2045 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:59:39.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:59:39 smithi181 bash[67587]: audit 2024-09-12T23:59:38.768814+0000 mon.a (mon.0) 2045 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:59:40.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:59:40 smithi136 bash[75164]: cluster 2024-09-12T23:59:39.098088+0000 mgr.y (mgr.24661) 676 : cluster [DBG] pgmap v1341: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 238 KiB/s rd, 159 KiB/s wr, 476 op/s 2024-09-12T23:59:40.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:59:40 smithi136 bash[76371]: cluster 2024-09-12T23:59:39.098088+0000 mgr.y (mgr.24661) 676 : cluster [DBG] pgmap v1341: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 238 KiB/s rd, 159 KiB/s wr, 476 op/s 2024-09-12T23:59:40.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:59:40 smithi181 bash[67587]: cluster 2024-09-12T23:59:39.098088+0000 mgr.y (mgr.24661) 676 : cluster [DBG] pgmap v1341: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 238 KiB/s rd, 159 KiB/s wr, 476 op/s 2024-09-12T23:59:42.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:59:42 smithi181 bash[67587]: cluster 2024-09-12T23:59:41.098936+0000 mgr.y (mgr.24661) 677 : cluster [DBG] pgmap v1342: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 238 KiB/s rd, 159 KiB/s wr, 476 op/s 2024-09-12T23:59:42.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:59:42 smithi181 bash[95505]: ts=2024-09-12T23:59:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:59:42.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:59:42 smithi136 bash[75164]: cluster 2024-09-12T23:59:41.098936+0000 mgr.y (mgr.24661) 677 : cluster [DBG] pgmap v1342: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 238 KiB/s rd, 159 KiB/s wr, 476 op/s 2024-09-12T23:59:42.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:59:42 smithi136 bash[76371]: cluster 2024-09-12T23:59:41.098936+0000 mgr.y (mgr.24661) 677 : cluster [DBG] pgmap v1342: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 238 KiB/s rd, 159 KiB/s wr, 476 op/s 2024-09-12T23:59:44.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:59:44 smithi136 bash[76371]: cluster 2024-09-12T23:59:43.100042+0000 mgr.y (mgr.24661) 678 : cluster [DBG] pgmap v1343: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 355 KiB/s rd, 237 KiB/s wr, 709 op/s 2024-09-12T23:59:44.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:59:44 smithi136 bash[75164]: cluster 2024-09-12T23:59:43.100042+0000 mgr.y (mgr.24661) 678 : cluster [DBG] pgmap v1343: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 355 KiB/s rd, 237 KiB/s wr, 709 op/s 2024-09-12T23:59:44.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:59:44 smithi181 bash[67587]: cluster 2024-09-12T23:59:43.100042+0000 mgr.y (mgr.24661) 678 : cluster [DBG] pgmap v1343: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 355 KiB/s rd, 237 KiB/s wr, 709 op/s 2024-09-12T23:59:46.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:59:46 smithi136 bash[75164]: cluster 2024-09-12T23:59:45.100808+0000 mgr.y (mgr.24661) 679 : cluster [DBG] pgmap v1344: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 238 KiB/s rd, 158 KiB/s wr, 475 op/s 2024-09-12T23:59:46.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:59:46 smithi136 bash[76371]: cluster 2024-09-12T23:59:45.100808+0000 mgr.y (mgr.24661) 679 : cluster [DBG] pgmap v1344: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 238 KiB/s rd, 158 KiB/s wr, 475 op/s 2024-09-12T23:59:46.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:59:46 smithi181 bash[67587]: cluster 2024-09-12T23:59:45.100808+0000 mgr.y (mgr.24661) 679 : cluster [DBG] pgmap v1344: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 238 KiB/s rd, 158 KiB/s wr, 475 op/s 2024-09-12T23:59:48.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:59:48 smithi136 bash[76371]: cluster 2024-09-12T23:59:47.102167+0000 mgr.y (mgr.24661) 680 : cluster [DBG] pgmap v1345: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 352 KiB/s rd, 235 KiB/s wr, 704 op/s 2024-09-12T23:59:48.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:59:48 smithi136 bash[75164]: cluster 2024-09-12T23:59:47.102167+0000 mgr.y (mgr.24661) 680 : cluster [DBG] pgmap v1345: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 352 KiB/s rd, 235 KiB/s wr, 704 op/s 2024-09-12T23:59:48.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:59:48 smithi181 bash[67587]: cluster 2024-09-12T23:59:47.102167+0000 mgr.y (mgr.24661) 680 : cluster [DBG] pgmap v1345: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 352 KiB/s rd, 235 KiB/s wr, 704 op/s 2024-09-12T23:59:49.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:59:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:59:49] "GET /metrics HTTP/1.1" 200 36476 "" "Prometheus/2.43.0" 2024-09-12T23:59:50.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:59:50 smithi136 bash[76371]: cluster 2024-09-12T23:59:49.102918+0000 mgr.y (mgr.24661) 681 : cluster [DBG] pgmap v1346: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 231 KiB/s rd, 154 KiB/s wr, 462 op/s 2024-09-12T23:59:50.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:59:50 smithi136 bash[75164]: cluster 2024-09-12T23:59:49.102918+0000 mgr.y (mgr.24661) 681 : cluster [DBG] pgmap v1346: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 231 KiB/s rd, 154 KiB/s wr, 462 op/s 2024-09-12T23:59:50.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:59:50 smithi181 bash[67587]: cluster 2024-09-12T23:59:49.102918+0000 mgr.y (mgr.24661) 681 : cluster [DBG] pgmap v1346: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 231 KiB/s rd, 154 KiB/s wr, 462 op/s 2024-09-12T23:59:52.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:59:52 smithi181 bash[67587]: cluster 2024-09-12T23:59:51.103922+0000 mgr.y (mgr.24661) 682 : cluster [DBG] pgmap v1347: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 231 KiB/s rd, 154 KiB/s wr, 462 op/s 2024-09-12T23:59:52.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 12 23:59:52 smithi181 bash[95505]: ts=2024-09-12T23:59:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-12T23:59:52.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:59:52 smithi136 bash[75164]: cluster 2024-09-12T23:59:51.103922+0000 mgr.y (mgr.24661) 682 : cluster [DBG] pgmap v1347: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 231 KiB/s rd, 154 KiB/s wr, 462 op/s 2024-09-12T23:59:52.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:59:52 smithi136 bash[76371]: cluster 2024-09-12T23:59:51.103922+0000 mgr.y (mgr.24661) 682 : cluster [DBG] pgmap v1347: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 231 KiB/s rd, 154 KiB/s wr, 462 op/s 2024-09-12T23:59:54.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:59:54 smithi136 bash[76371]: cluster 2024-09-12T23:59:53.105426+0000 mgr.y (mgr.24661) 683 : cluster [DBG] pgmap v1348: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 347 KiB/s rd, 232 KiB/s wr, 694 op/s 2024-09-12T23:59:54.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:59:54 smithi136 bash[76371]: audit 2024-09-12T23:59:53.768754+0000 mon.a (mon.0) 2046 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:59:54.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:59:54 smithi136 bash[75164]: cluster 2024-09-12T23:59:53.105426+0000 mgr.y (mgr.24661) 683 : cluster [DBG] pgmap v1348: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 347 KiB/s rd, 232 KiB/s wr, 694 op/s 2024-09-12T23:59:54.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:59:54 smithi136 bash[75164]: audit 2024-09-12T23:59:53.768754+0000 mon.a (mon.0) 2046 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:59:54.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:59:54 smithi181 bash[67587]: cluster 2024-09-12T23:59:53.105426+0000 mgr.y (mgr.24661) 683 : cluster [DBG] pgmap v1348: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 347 KiB/s rd, 232 KiB/s wr, 694 op/s 2024-09-12T23:59:54.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:59:54 smithi181 bash[67587]: audit 2024-09-12T23:59:53.768754+0000 mon.a (mon.0) 2046 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-12T23:59:56.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:59:56 smithi136 bash[75164]: cluster 2024-09-12T23:59:55.106425+0000 mgr.y (mgr.24661) 684 : cluster [DBG] pgmap v1349: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 231 KiB/s rd, 154 KiB/s wr, 461 op/s 2024-09-12T23:59:56.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:59:56 smithi136 bash[76371]: cluster 2024-09-12T23:59:55.106425+0000 mgr.y (mgr.24661) 684 : cluster [DBG] pgmap v1349: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 231 KiB/s rd, 154 KiB/s wr, 461 op/s 2024-09-12T23:59:56.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:59:56 smithi181 bash[67587]: cluster 2024-09-12T23:59:55.106425+0000 mgr.y (mgr.24661) 684 : cluster [DBG] pgmap v1349: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 231 KiB/s rd, 154 KiB/s wr, 461 op/s 2024-09-12T23:59:58.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 12 23:59:58 smithi136 bash[75164]: cluster 2024-09-12T23:59:57.107548+0000 mgr.y (mgr.24661) 685 : cluster [DBG] pgmap v1350: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 346 KiB/s rd, 231 KiB/s wr, 691 op/s 2024-09-12T23:59:58.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 12 23:59:58 smithi136 bash[76371]: cluster 2024-09-12T23:59:57.107548+0000 mgr.y (mgr.24661) 685 : cluster [DBG] pgmap v1350: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 346 KiB/s rd, 231 KiB/s wr, 691 op/s 2024-09-12T23:59:58.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 12 23:59:58 smithi181 bash[67587]: cluster 2024-09-12T23:59:57.107548+0000 mgr.y (mgr.24661) 685 : cluster [DBG] pgmap v1350: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 346 KiB/s rd, 231 KiB/s wr, 691 op/s 2024-09-12T23:59:59.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 12 23:59:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [12/Sep/2024:23:59:59] "GET /metrics HTTP/1.1" 200 36472 "" "Prometheus/2.43.0" 2024-09-13T00:00:00.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:00 smithi136 bash[75164]: cluster 2024-09-12T23:59:59.108454+0000 mgr.y (mgr.24661) 686 : cluster [DBG] pgmap v1351: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 231 KiB/s rd, 154 KiB/s wr, 462 op/s 2024-09-13T00:00:00.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:00 smithi136 bash[75164]: cluster 2024-09-13T00:00:00.000118+0000 mon.a (mon.0) 2047 : cluster [INF] overall HEALTH_OK 2024-09-13T00:00:00.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:00 smithi136 bash[76371]: cluster 2024-09-12T23:59:59.108454+0000 mgr.y (mgr.24661) 686 : cluster [DBG] pgmap v1351: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 231 KiB/s rd, 154 KiB/s wr, 462 op/s 2024-09-13T00:00:00.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:00 smithi136 bash[76371]: cluster 2024-09-13T00:00:00.000118+0000 mon.a (mon.0) 2047 : cluster [INF] overall HEALTH_OK 2024-09-13T00:00:00.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:00 smithi181 bash[67587]: cluster 2024-09-12T23:59:59.108454+0000 mgr.y (mgr.24661) 686 : cluster [DBG] pgmap v1351: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 231 KiB/s rd, 154 KiB/s wr, 462 op/s 2024-09-13T00:00:00.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:00 smithi181 bash[67587]: cluster 2024-09-13T00:00:00.000118+0000 mon.a (mon.0) 2047 : cluster [INF] overall HEALTH_OK 2024-09-13T00:00:02.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:02 smithi181 bash[67587]: cluster 2024-09-13T00:00:01.109413+0000 mgr.y (mgr.24661) 687 : cluster [DBG] pgmap v1352: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 231 KiB/s rd, 154 KiB/s wr, 462 op/s 2024-09-13T00:00:02.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:00:02 smithi181 bash[95505]: ts=2024-09-13T00:00:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:00:02.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:02 smithi136 bash[76371]: cluster 2024-09-13T00:00:01.109413+0000 mgr.y (mgr.24661) 687 : cluster [DBG] pgmap v1352: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 231 KiB/s rd, 154 KiB/s wr, 462 op/s 2024-09-13T00:00:02.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:02 smithi136 bash[75164]: cluster 2024-09-13T00:00:01.109413+0000 mgr.y (mgr.24661) 687 : cluster [DBG] pgmap v1352: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 231 KiB/s rd, 154 KiB/s wr, 462 op/s 2024-09-13T00:00:04.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:04 smithi136 bash[75164]: cluster 2024-09-13T00:00:03.110052+0000 mgr.y (mgr.24661) 688 : cluster [DBG] pgmap v1353: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 348 KiB/s rd, 232 KiB/s wr, 696 op/s 2024-09-13T00:00:04.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:04 smithi136 bash[76371]: cluster 2024-09-13T00:00:03.110052+0000 mgr.y (mgr.24661) 688 : cluster [DBG] pgmap v1353: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 348 KiB/s rd, 232 KiB/s wr, 696 op/s 2024-09-13T00:00:04.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:04 smithi181 bash[67587]: cluster 2024-09-13T00:00:03.110052+0000 mgr.y (mgr.24661) 688 : cluster [DBG] pgmap v1353: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 348 KiB/s rd, 232 KiB/s wr, 696 op/s 2024-09-13T00:00:06.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:06 smithi181 bash[67587]: cluster 2024-09-13T00:00:05.110916+0000 mgr.y (mgr.24661) 689 : cluster [DBG] pgmap v1354: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 232 KiB/s rd, 155 KiB/s wr, 463 op/s 2024-09-13T00:00:07.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:06 smithi136 bash[76371]: cluster 2024-09-13T00:00:05.110916+0000 mgr.y (mgr.24661) 689 : cluster [DBG] pgmap v1354: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 232 KiB/s rd, 155 KiB/s wr, 463 op/s 2024-09-13T00:00:07.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:06 smithi136 bash[75164]: cluster 2024-09-13T00:00:05.110916+0000 mgr.y (mgr.24661) 689 : cluster [DBG] pgmap v1354: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 232 KiB/s rd, 155 KiB/s wr, 463 op/s 2024-09-13T00:00:08.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:08 smithi181 bash[67587]: cluster 2024-09-13T00:00:07.112171+0000 mgr.y (mgr.24661) 690 : cluster [DBG] pgmap v1355: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 348 KiB/s rd, 232 KiB/s wr, 696 op/s 2024-09-13T00:00:09.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:08 smithi136 bash[76371]: cluster 2024-09-13T00:00:07.112171+0000 mgr.y (mgr.24661) 690 : cluster [DBG] pgmap v1355: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 348 KiB/s rd, 232 KiB/s wr, 696 op/s 2024-09-13T00:00:09.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:08 smithi136 bash[75164]: cluster 2024-09-13T00:00:07.112171+0000 mgr.y (mgr.24661) 690 : cluster [DBG] pgmap v1355: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 348 KiB/s rd, 232 KiB/s wr, 696 op/s 2024-09-13T00:00:09.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:09 smithi136 bash[75164]: audit 2024-09-13T00:00:08.769030+0000 mon.a (mon.0) 2048 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:00:09.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:09 smithi136 bash[76371]: audit 2024-09-13T00:00:08.769030+0000 mon.a (mon.0) 2048 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:00:09.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:00:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:00:09] "GET /metrics HTTP/1.1" 200 36478 "" "Prometheus/2.43.0" 2024-09-13T00:00:09.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:09 smithi181 bash[67587]: audit 2024-09-13T00:00:08.769030+0000 mon.a (mon.0) 2048 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:00:10.696 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:10 smithi181 bash[67587]: cluster 2024-09-13T00:00:09.113325+0000 mgr.y (mgr.24661) 691 : cluster [DBG] pgmap v1356: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 233 KiB/s rd, 155 KiB/s wr, 465 op/s 2024-09-13T00:00:11.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:10 smithi136 bash[76371]: cluster 2024-09-13T00:00:09.113325+0000 mgr.y (mgr.24661) 691 : cluster [DBG] pgmap v1356: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 233 KiB/s rd, 155 KiB/s wr, 465 op/s 2024-09-13T00:00:11.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:10 smithi136 bash[75164]: cluster 2024-09-13T00:00:09.113325+0000 mgr.y (mgr.24661) 691 : cluster [DBG] pgmap v1356: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 233 KiB/s rd, 155 KiB/s wr, 465 op/s 2024-09-13T00:00:12.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:12 smithi181 bash[67587]: cluster 2024-09-13T00:00:11.114427+0000 mgr.y (mgr.24661) 692 : cluster [DBG] pgmap v1357: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 233 KiB/s rd, 155 KiB/s wr, 465 op/s 2024-09-13T00:00:12.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:00:12 smithi181 bash[95505]: ts=2024-09-13T00:00:12.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:00:13.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:12 smithi136 bash[75164]: cluster 2024-09-13T00:00:11.114427+0000 mgr.y (mgr.24661) 692 : cluster [DBG] pgmap v1357: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 233 KiB/s rd, 155 KiB/s wr, 465 op/s 2024-09-13T00:00:13.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:12 smithi136 bash[76371]: cluster 2024-09-13T00:00:11.114427+0000 mgr.y (mgr.24661) 692 : cluster [DBG] pgmap v1357: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 233 KiB/s rd, 155 KiB/s wr, 465 op/s 2024-09-13T00:00:14.098 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 13 00:00:13 smithi181 bash[98820]: logger=sqlstore.transactions t=2024-09-13T00:00:13.705914456Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-13T00:00:14.098 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 13 00:00:13 smithi181 bash[98820]: logger=cleanup t=2024-09-13T00:00:13.762553873Z level=info msg="Completed cleanup jobs" duration=68.132402ms 2024-09-13T00:00:14.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:14 smithi181 bash[67587]: cluster 2024-09-13T00:00:13.115570+0000 mgr.y (mgr.24661) 693 : cluster [DBG] pgmap v1358: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 351 KiB/s rd, 234 KiB/s wr, 702 op/s 2024-09-13T00:00:15.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:14 smithi136 bash[75164]: cluster 2024-09-13T00:00:13.115570+0000 mgr.y (mgr.24661) 693 : cluster [DBG] pgmap v1358: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 351 KiB/s rd, 234 KiB/s wr, 702 op/s 2024-09-13T00:00:15.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:14 smithi136 bash[76371]: cluster 2024-09-13T00:00:13.115570+0000 mgr.y (mgr.24661) 693 : cluster [DBG] pgmap v1358: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 351 KiB/s rd, 234 KiB/s wr, 702 op/s 2024-09-13T00:00:16.689 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] cls_rgw.index_list_delimited (97119 ms) 2024-09-13T00:00:16.689 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] cls_rgw.bi_list 2024-09-13T00:00:16.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:16 smithi181 bash[67587]: cluster 2024-09-13T00:00:15.116549+0000 mgr.y (mgr.24661) 694 : cluster [DBG] pgmap v1359: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 235 KiB/s rd, 157 KiB/s wr, 469 op/s 2024-09-13T00:00:16.878 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] cls_rgw.bi_list (188 ms) 2024-09-13T00:00:16.878 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] cls_rgw.gc_set 2024-09-13T00:00:16.942 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] cls_rgw.gc_set (64 ms) 2024-09-13T00:00:16.942 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] cls_rgw.gc_list 2024-09-13T00:00:17.000 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] cls_rgw.gc_list (58 ms) 2024-09-13T00:00:17.000 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] cls_rgw.gc_defer 2024-09-13T00:00:17.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:16 smithi136 bash[76371]: cluster 2024-09-13T00:00:15.116549+0000 mgr.y (mgr.24661) 694 : cluster [DBG] pgmap v1359: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 235 KiB/s rd, 157 KiB/s wr, 469 op/s 2024-09-13T00:00:17.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:16 smithi136 bash[75164]: cluster 2024-09-13T00:00:15.116549+0000 mgr.y (mgr.24661) 694 : cluster [DBG] pgmap v1359: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 235 KiB/s rd, 157 KiB/s wr, 469 op/s 2024-09-13T00:00:18.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:18 smithi181 bash[67587]: cluster 2024-09-13T00:00:17.117721+0000 mgr.y (mgr.24661) 695 : cluster [DBG] pgmap v1360: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 343 KiB/s rd, 228 KiB/s wr, 685 op/s 2024-09-13T00:00:18.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:18 smithi181 bash[67587]: cluster 2024-09-13T00:00:17.556238+0000 mon.a (mon.0) 2049 : cluster [DBG] osdmap e1105: 8 total, 8 up, 8 in 2024-09-13T00:00:18.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:18 smithi181 bash[67587]: audit 2024-09-13T00:00:17.564996+0000 mon.a (mon.0) 2050 : audit [INF] from='client.? 172.21.15.136:0/2126584691' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110551-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:00:19.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:18 smithi136 bash[75164]: cluster 2024-09-13T00:00:17.117721+0000 mgr.y (mgr.24661) 695 : cluster [DBG] pgmap v1360: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 343 KiB/s rd, 228 KiB/s wr, 685 op/s 2024-09-13T00:00:19.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:18 smithi136 bash[75164]: cluster 2024-09-13T00:00:17.556238+0000 mon.a (mon.0) 2049 : cluster [DBG] osdmap e1105: 8 total, 8 up, 8 in 2024-09-13T00:00:19.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:18 smithi136 bash[75164]: audit 2024-09-13T00:00:17.564996+0000 mon.a (mon.0) 2050 : audit [INF] from='client.? 172.21.15.136:0/2126584691' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110551-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:00:19.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:18 smithi136 bash[76371]: cluster 2024-09-13T00:00:17.117721+0000 mgr.y (mgr.24661) 695 : cluster [DBG] pgmap v1360: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 343 KiB/s rd, 228 KiB/s wr, 685 op/s 2024-09-13T00:00:19.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:18 smithi136 bash[76371]: cluster 2024-09-13T00:00:17.556238+0000 mon.a (mon.0) 2049 : cluster [DBG] osdmap e1105: 8 total, 8 up, 8 in 2024-09-13T00:00:19.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:18 smithi136 bash[76371]: audit 2024-09-13T00:00:17.564996+0000 mon.a (mon.0) 2050 : audit [INF] from='client.? 172.21.15.136:0/2126584691' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110551-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:00:19.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:00:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:00:19] "GET /metrics HTTP/1.1" 200 36478 "" "Prometheus/2.43.0" 2024-09-13T00:00:19.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:19 smithi136 bash[75164]: audit 2024-09-13T00:00:18.563414+0000 mon.a (mon.0) 2051 : audit [INF] from='client.? 172.21.15.136:0/2126584691' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110551-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:00:19.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:19 smithi136 bash[75164]: cluster 2024-09-13T00:00:18.563649+0000 mon.a (mon.0) 2052 : cluster [DBG] osdmap e1106: 8 total, 8 up, 8 in 2024-09-13T00:00:19.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:19 smithi136 bash[76371]: audit 2024-09-13T00:00:18.563414+0000 mon.a (mon.0) 2051 : audit [INF] from='client.? 172.21.15.136:0/2126584691' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110551-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:00:19.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:19 smithi136 bash[76371]: cluster 2024-09-13T00:00:18.563649+0000 mon.a (mon.0) 2052 : cluster [DBG] osdmap e1106: 8 total, 8 up, 8 in 2024-09-13T00:00:19.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:19 smithi181 bash[67587]: audit 2024-09-13T00:00:18.563414+0000 mon.a (mon.0) 2051 : audit [INF] from='client.? 172.21.15.136:0/2126584691' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110551-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:00:19.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:19 smithi181 bash[67587]: cluster 2024-09-13T00:00:18.563649+0000 mon.a (mon.0) 2052 : cluster [DBG] osdmap e1106: 8 total, 8 up, 8 in 2024-09-13T00:00:20.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:20 smithi181 bash[67587]: cluster 2024-09-13T00:00:19.118594+0000 mgr.y (mgr.24661) 696 : cluster [DBG] pgmap v1363: 169 pgs: 5 creating+peering, 27 unknown, 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 340 KiB/s rd, 226 KiB/s wr, 679 op/s 2024-09-13T00:00:20.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:20 smithi181 bash[67587]: cluster 2024-09-13T00:00:19.586846+0000 mon.a (mon.0) 2053 : cluster [DBG] osdmap e1107: 8 total, 8 up, 8 in 2024-09-13T00:00:21.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:20 smithi136 bash[75164]: cluster 2024-09-13T00:00:19.118594+0000 mgr.y (mgr.24661) 696 : cluster [DBG] pgmap v1363: 169 pgs: 5 creating+peering, 27 unknown, 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 340 KiB/s rd, 226 KiB/s wr, 679 op/s 2024-09-13T00:00:21.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:20 smithi136 bash[75164]: cluster 2024-09-13T00:00:19.586846+0000 mon.a (mon.0) 2053 : cluster [DBG] osdmap e1107: 8 total, 8 up, 8 in 2024-09-13T00:00:21.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:20 smithi136 bash[76371]: cluster 2024-09-13T00:00:19.118594+0000 mgr.y (mgr.24661) 696 : cluster [DBG] pgmap v1363: 169 pgs: 5 creating+peering, 27 unknown, 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 340 KiB/s rd, 226 KiB/s wr, 679 op/s 2024-09-13T00:00:21.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:20 smithi136 bash[76371]: cluster 2024-09-13T00:00:19.586846+0000 mon.a (mon.0) 2053 : cluster [DBG] osdmap e1107: 8 total, 8 up, 8 in 2024-09-13T00:00:22.595 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:00:22 smithi181 bash[95505]: ts=2024-09-13T00:00:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:00:22.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:22 smithi181 bash[67587]: cluster 2024-09-13T00:00:21.119779+0000 mgr.y (mgr.24661) 697 : cluster [DBG] pgmap v1365: 169 pgs: 5 creating+peering, 13 unknown, 151 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 216 KiB/s rd, 144 KiB/s wr, 431 op/s 2024-09-13T00:00:23.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:22 smithi136 bash[75164]: cluster 2024-09-13T00:00:21.119779+0000 mgr.y (mgr.24661) 697 : cluster [DBG] pgmap v1365: 169 pgs: 5 creating+peering, 13 unknown, 151 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 216 KiB/s rd, 144 KiB/s wr, 431 op/s 2024-09-13T00:00:23.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:22 smithi136 bash[76371]: cluster 2024-09-13T00:00:21.119779+0000 mgr.y (mgr.24661) 697 : cluster [DBG] pgmap v1365: 169 pgs: 5 creating+peering, 13 unknown, 151 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 216 KiB/s rd, 144 KiB/s wr, 431 op/s 2024-09-13T00:00:24.619 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] cls_rgw.gc_defer (7619 ms) 2024-09-13T00:00:24.619 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] cls_rgw.usage_basic 2024-09-13T00:00:24.793 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] cls_rgw.usage_basic (173 ms) 2024-09-13T00:00:24.793 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] cls_rgw.usage_clear_no_obj 2024-09-13T00:00:24.797 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] cls_rgw.usage_clear_no_obj (4 ms) 2024-09-13T00:00:24.797 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] cls_rgw.usage_clear 2024-09-13T00:00:24.973 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] cls_rgw.usage_clear (177 ms) 2024-09-13T00:00:24.974 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] cls_rgw.bi_log_trim 2024-09-13T00:00:25.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:24 smithi136 bash[75164]: cluster 2024-09-13T00:00:23.121094+0000 mgr.y (mgr.24661) 698 : cluster [DBG] pgmap v1366: 169 pgs: 2 creating+peering, 167 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 283 KiB/s rd, 30 KiB/s wr, 82 op/s 2024-09-13T00:00:25.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:24 smithi136 bash[75164]: audit 2024-09-13T00:00:23.769159+0000 mon.a (mon.0) 2054 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:00:25.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:24 smithi136 bash[76371]: cluster 2024-09-13T00:00:23.121094+0000 mgr.y (mgr.24661) 698 : cluster [DBG] pgmap v1366: 169 pgs: 2 creating+peering, 167 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 283 KiB/s rd, 30 KiB/s wr, 82 op/s 2024-09-13T00:00:25.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:24 smithi136 bash[76371]: audit 2024-09-13T00:00:23.769159+0000 mon.a (mon.0) 2054 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:00:25.096 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] cls_rgw.bi_log_trim (123 ms) 2024-09-13T00:00:25.096 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] cls_rgw.index_racing_removes 2024-09-13T00:00:25.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:24 smithi181 bash[67587]: cluster 2024-09-13T00:00:23.121094+0000 mgr.y (mgr.24661) 698 : cluster [DBG] pgmap v1366: 169 pgs: 2 creating+peering, 167 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 283 KiB/s rd, 30 KiB/s wr, 82 op/s 2024-09-13T00:00:25.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:24 smithi181 bash[67587]: audit 2024-09-13T00:00:23.769159+0000 mon.a (mon.0) 2054 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:00:25.123 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] cls_rgw.index_racing_removes (27 ms) 2024-09-13T00:00:25.628 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 16 tests from cls_rgw (108092 ms total) 2024-09-13T00:00:25.628 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:00:25.628 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] Global test environment tear-down 2024-09-13T00:00:25.628 INFO:tasks.workunit.client.0.smithi136.stdout:[==========] 16 tests from 1 test suite ran. (110496 ms total) 2024-09-13T00:00:25.629 INFO:tasks.workunit.client.0.smithi136.stdout:[ PASSED ] 16 tests. 2024-09-13T00:00:25.633 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-13T00:00:25.633 DEBUG:teuthology.orchestra.run.smithi136:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-13T00:00:25.689 INFO:tasks.workunit:Running workunit cls/test_cls_rgw_gc.sh... 2024-09-13T00:00:25.690 DEBUG:teuthology.orchestra.run.smithi136:workunit test cls/test_cls_rgw_gc.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rgw_gc.sh 2024-09-13T00:00:25.752 INFO:tasks.workunit.client.0.smithi136.stdout:Running main() from gmock_main.cc 2024-09-13T00:00:25.752 INFO:tasks.workunit.client.0.smithi136.stdout:[==========] Running 9 tests from 1 test suite. 2024-09-13T00:00:25.752 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] Global test environment set-up. 2024-09-13T00:00:25.752 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 9 tests from cls_rgw_gc 2024-09-13T00:00:25.752 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] cls_rgw_gc.init 2024-09-13T00:00:26.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:25 smithi136 bash[75164]: cluster 2024-09-13T00:00:24.617492+0000 mon.a (mon.0) 2055 : cluster [DBG] osdmap e1108: 8 total, 8 up, 8 in 2024-09-13T00:00:26.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:25 smithi136 bash[76371]: cluster 2024-09-13T00:00:24.617492+0000 mon.a (mon.0) 2055 : cluster [DBG] osdmap e1108: 8 total, 8 up, 8 in 2024-09-13T00:00:26.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:25 smithi181 bash[67587]: cluster 2024-09-13T00:00:24.617492+0000 mon.a (mon.0) 2055 : cluster [DBG] osdmap e1108: 8 total, 8 up, 8 in 2024-09-13T00:00:27.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:26 smithi136 bash[75164]: cluster 2024-09-13T00:00:25.122080+0000 mgr.y (mgr.24661) 699 : cluster [DBG] pgmap v1368: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 258 KiB/s rd, 27 KiB/s wr, 74 op/s 2024-09-13T00:00:27.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:26 smithi136 bash[75164]: cluster 2024-09-13T00:00:25.626185+0000 mon.a (mon.0) 2056 : cluster [DBG] osdmap e1109: 8 total, 8 up, 8 in 2024-09-13T00:00:27.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:26 smithi136 bash[76371]: cluster 2024-09-13T00:00:25.122080+0000 mgr.y (mgr.24661) 699 : cluster [DBG] pgmap v1368: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 258 KiB/s rd, 27 KiB/s wr, 74 op/s 2024-09-13T00:00:27.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:26 smithi136 bash[76371]: cluster 2024-09-13T00:00:25.626185+0000 mon.a (mon.0) 2056 : cluster [DBG] osdmap e1109: 8 total, 8 up, 8 in 2024-09-13T00:00:27.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:26 smithi181 bash[67587]: cluster 2024-09-13T00:00:25.122080+0000 mgr.y (mgr.24661) 699 : cluster [DBG] pgmap v1368: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 258 KiB/s rd, 27 KiB/s wr, 74 op/s 2024-09-13T00:00:27.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:26 smithi181 bash[67587]: cluster 2024-09-13T00:00:25.626185+0000 mon.a (mon.0) 2056 : cluster [DBG] osdmap e1109: 8 total, 8 up, 8 in 2024-09-13T00:00:27.663 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] cls_rgw_gc.init (1911 ms) 2024-09-13T00:00:27.663 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] cls_rgw_gc.gc_queue_ops1 2024-09-13T00:00:27.721 INFO:tasks.workunit.client.0.smithi136.stderr:[ ] list info tag = chain-0 2024-09-13T00:00:27.721 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] cls_rgw_gc.gc_queue_ops1 (58 ms) 2024-09-13T00:00:27.721 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] cls_rgw_gc.gc_queue_ops2 2024-09-13T00:00:27.727 INFO:tasks.workunit.client.0.smithi136.stderr:[ ] next_marker is: = 0/4263 2024-09-13T00:00:27.728 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] cls_rgw_gc.gc_queue_ops2 (7 ms) 2024-09-13T00:00:27.728 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] cls_rgw_gc.gc_queue_ops5 2024-09-13T00:00:27.743 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] cls_rgw_gc.gc_queue_ops5 (11 ms) 2024-09-13T00:00:27.743 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] cls_rgw_gc.gc_queue_ops6 2024-09-13T00:00:27.747 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] cls_rgw_gc.gc_queue_ops6 (8 ms) 2024-09-13T00:00:27.747 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] cls_rgw_gc.gc_queue_ops7 2024-09-13T00:00:27.758 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] cls_rgw_gc.gc_queue_ops7 (12 ms) 2024-09-13T00:00:27.759 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] cls_rgw_gc.gc_queue_ops8 2024-09-13T00:00:27.771 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] cls_rgw_gc.gc_queue_ops8 (12 ms) 2024-09-13T00:00:27.771 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] cls_rgw_gc.gc_queue_ops9 2024-09-13T00:00:27.779 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] cls_rgw_gc.gc_queue_ops9 (9 ms) 2024-09-13T00:00:27.779 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] cls_rgw_gc.finalize 2024-09-13T00:00:28.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:27 smithi136 bash[75164]: cluster 2024-09-13T00:00:26.641992+0000 mon.a (mon.0) 2057 : cluster [DBG] osdmap e1110: 8 total, 8 up, 8 in 2024-09-13T00:00:28.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:27 smithi136 bash[75164]: audit 2024-09-13T00:00:26.653982+0000 mon.c (mon.2) 90 : audit [INF] from='client.? 172.21.15.136:0/1115451059' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110672-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:00:28.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:27 smithi136 bash[75164]: audit 2024-09-13T00:00:26.655714+0000 mon.a (mon.0) 2058 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110672-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:00:28.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:27 smithi136 bash[76371]: cluster 2024-09-13T00:00:26.641992+0000 mon.a (mon.0) 2057 : cluster [DBG] osdmap e1110: 8 total, 8 up, 8 in 2024-09-13T00:00:28.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:27 smithi136 bash[76371]: audit 2024-09-13T00:00:26.653982+0000 mon.c (mon.2) 90 : audit [INF] from='client.? 172.21.15.136:0/1115451059' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110672-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:00:28.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:27 smithi136 bash[76371]: audit 2024-09-13T00:00:26.655714+0000 mon.a (mon.0) 2058 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110672-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:00:28.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:27 smithi181 bash[67587]: cluster 2024-09-13T00:00:26.641992+0000 mon.a (mon.0) 2057 : cluster [DBG] osdmap e1110: 8 total, 8 up, 8 in 2024-09-13T00:00:28.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:27 smithi181 bash[67587]: audit 2024-09-13T00:00:26.653982+0000 mon.c (mon.2) 90 : audit [INF] from='client.? 172.21.15.136:0/1115451059' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110672-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:00:28.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:27 smithi181 bash[67587]: audit 2024-09-13T00:00:26.655714+0000 mon.a (mon.0) 2058 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110672-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:00:28.656 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] cls_rgw_gc.finalize (875 ms) 2024-09-13T00:00:28.656 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 9 tests from cls_rgw_gc (2904 ms total) 2024-09-13T00:00:28.656 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:00:28.656 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] Global test environment tear-down 2024-09-13T00:00:28.656 INFO:tasks.workunit.client.0.smithi136.stdout:[==========] 9 tests from 1 test suite ran. (2904 ms total) 2024-09-13T00:00:28.656 INFO:tasks.workunit.client.0.smithi136.stdout:[ PASSED ] 9 tests. 2024-09-13T00:00:28.661 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-13T00:00:28.661 DEBUG:teuthology.orchestra.run.smithi136:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-13T00:00:28.718 INFO:tasks.workunit:Running workunit cls/test_cls_rgw_stats.sh... 2024-09-13T00:00:28.719 DEBUG:teuthology.orchestra.run.smithi136:workunit test cls/test_cls_rgw_stats.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rgw_stats.sh 2024-09-13T00:00:28.792 INFO:tasks.workunit.client.0.smithi136.stdout:[==========] Running 1 test from 1 test suite. 2024-09-13T00:00:28.793 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] Global test environment set-up. 2024-09-13T00:00:29.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:28 smithi136 bash[75164]: cluster 2024-09-13T00:00:27.122990+0000 mgr.y (mgr.24661) 700 : cluster [DBG] pgmap v1371: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:00:29.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:28 smithi136 bash[75164]: cluster 2024-09-13T00:00:27.632146+0000 mon.a (mon.0) 2059 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:00:29.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:28 smithi136 bash[75164]: audit 2024-09-13T00:00:27.658493+0000 mon.a (mon.0) 2060 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110672-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:00:29.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:28 smithi136 bash[75164]: cluster 2024-09-13T00:00:27.658677+0000 mon.a (mon.0) 2061 : cluster [DBG] osdmap e1111: 8 total, 8 up, 8 in 2024-09-13T00:00:29.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:28 smithi136 bash[76371]: cluster 2024-09-13T00:00:27.122990+0000 mgr.y (mgr.24661) 700 : cluster [DBG] pgmap v1371: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:00:29.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:28 smithi136 bash[76371]: cluster 2024-09-13T00:00:27.632146+0000 mon.a (mon.0) 2059 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:00:29.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:28 smithi136 bash[76371]: audit 2024-09-13T00:00:27.658493+0000 mon.a (mon.0) 2060 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110672-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:00:29.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:28 smithi136 bash[76371]: cluster 2024-09-13T00:00:27.658677+0000 mon.a (mon.0) 2061 : cluster [DBG] osdmap e1111: 8 total, 8 up, 8 in 2024-09-13T00:00:29.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:28 smithi181 bash[67587]: cluster 2024-09-13T00:00:27.122990+0000 mgr.y (mgr.24661) 700 : cluster [DBG] pgmap v1371: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:00:29.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:28 smithi181 bash[67587]: cluster 2024-09-13T00:00:27.632146+0000 mon.a (mon.0) 2059 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:00:29.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:28 smithi181 bash[67587]: audit 2024-09-13T00:00:27.658493+0000 mon.a (mon.0) 2060 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110672-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:00:29.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:28 smithi181 bash[67587]: cluster 2024-09-13T00:00:27.658677+0000 mon.a (mon.0) 2061 : cluster [DBG] osdmap e1111: 8 total, 8 up, 8 in 2024-09-13T00:00:29.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:29 smithi136 bash[75164]: cluster 2024-09-13T00:00:28.652780+0000 mon.a (mon.0) 2062 : cluster [DBG] osdmap e1112: 8 total, 8 up, 8 in 2024-09-13T00:00:29.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:00:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:00:29] "GET /metrics HTTP/1.1" 200 38844 "" "Prometheus/2.43.0" 2024-09-13T00:00:29.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:29 smithi136 bash[76371]: cluster 2024-09-13T00:00:28.652780+0000 mon.a (mon.0) 2062 : cluster [DBG] osdmap e1112: 8 total, 8 up, 8 in 2024-09-13T00:00:30.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:29 smithi181 bash[67587]: cluster 2024-09-13T00:00:28.652780+0000 mon.a (mon.0) 2062 : cluster [DBG] osdmap e1112: 8 total, 8 up, 8 in 2024-09-13T00:00:30.684 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 1 test from cls_rgw_stats 2024-09-13T00:00:30.684 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] cls_rgw_stats.simulate 2024-09-13T00:00:30.723 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.722+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~8DSSHI60ZREU.0 size=1048576 2024-09-13T00:00:30.725 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.726+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~8DSSHI60ZREU.1 size=739801 2024-09-13T00:00:30.725 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.726+0000 7fb8f2ce0640 1 > prepared multipart upload key=7OX5OJZ8GXU5 upload=8DSSHI60ZREU tag=PLAWUT17AI31 type=write size=2836953 2024-09-13T00:00:30.727 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.726+0000 7fb8f2ce0640 1 > prepared operation key=44EDYFT9VF8Y tag=1BB0XV4OV6H6 type=del size=2382756 2024-09-13T00:00:30.730 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.730+0000 7fb8f2ce0640 1 > prepared operation key=NX26SI4F2ZMB tag=3C4T3WWQIQ9N type=del size=3450566 2024-09-13T00:00:30.733 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.734+0000 7fb8f2ce0640 1 > prepared operation key=44EDYFT9VF8Y tag=AGDBQ8JNCVAF type=del size=2987172 2024-09-13T00:00:30.736 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.734+0000 7fb8f2ce0640 1 > prepared operation key=W8XZ26311RJX tag=8YMTUWXXWSMQ type=del size=649748 2024-09-13T00:00:30.738 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.738+0000 7fb8f2ce0640 1 > prepared operation key=W8XZ26311RJX tag=8ATG2VBGZ8LS type=del size=2539559 2024-09-13T00:00:30.741 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.742+0000 7fb8f2ce0640 1 > prepared operation key=85YEW8N43FTA tag=ULQTNWZ4PYB0 type=del size=4049857 2024-09-13T00:00:30.744 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.742+0000 7fb8f2ce0640 1 > prepared operation key=H9AU4HD247YU tag=2YYRCB0UYW4P type=del size=2258296 2024-09-13T00:00:30.747 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.746+0000 7fb8f2ce0640 1 > prepared operation key=EUE3LT0PDIXV tag=68PCDXRD5P1K type=del size=2279573 2024-09-13T00:00:30.752 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.750+0000 7fb8f2ce0640 1 > prepared part key=_multipart_QMZREZ1QEDNJ.2~3NZKLS52PP32.0 size=62923 2024-09-13T00:00:30.753 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.750+0000 7fb8f2ce0640 1 > prepared multipart upload key=QMZREZ1QEDNJ upload=3NZKLS52PP32 tag=DNAUC56F5E6D type=write size=1111499 2024-09-13T00:00:30.755 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.754+0000 7fb8f2ce0640 1 > prepared operation key=F0QCVR9E08WE tag=8QJOTLRU5UKH type=del size=25046 2024-09-13T00:00:30.758 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.758+0000 7fb8f2ce0640 1 > prepared operation key=5GM88ERM89KT tag=BK9AZZAUQHAX type=del size=2387651 2024-09-13T00:00:30.760 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.758+0000 7fb8f2ce0640 1 > prepared operation key=ZTK9AYBSLCOZ tag=V49S03T5YEKF type=del size=3245261 2024-09-13T00:00:30.765 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.766+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2TWD9YLMSYJI.2~8NJE6SPJ5OWS.0 size=367489 2024-09-13T00:00:30.765 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.766+0000 7fb8f2ce0640 1 > prepared multipart upload key=2TWD9YLMSYJI upload=8NJE6SPJ5OWS tag=4PHT3WG3R5J7 type=write size=1416065 2024-09-13T00:00:30.768 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.766+0000 7fb8f2ce0640 1 > prepared operation key=H5EIP0E9CBWX tag=93O7TZCC2J4I type=del size=3065726 2024-09-13T00:00:30.771 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.770+0000 7fb8f2ce0640 1 > prepared operation key=EJN6HR8I7XQF tag=NN9Z6BMJ7PQW type=del size=77313 2024-09-13T00:00:30.775 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.774+0000 7fb8f2ce0640 1 < completed part key=_multipart_7OX5OJZ8GXU5.2~8DSSHI60ZREU.0 size=1048576 2024-09-13T00:00:30.775 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.774+0000 7fb8f2ce0640 1 < canceled part key=_multipart_7OX5OJZ8GXU5.2~8DSSHI60ZREU.1 size=739801 2024-09-13T00:00:30.780 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.778+0000 7fb8f2ce0640 1 < canceled multipart upload key=7OX5OJZ8GXU5 upload=8DSSHI60ZREU tag=PLAWUT17AI31 type=write size=2836953 2024-09-13T00:00:30.785 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.786+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~EVCWFHKZ5ELU.0 size=1048576 2024-09-13T00:00:30.788 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.786+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~EVCWFHKZ5ELU.1 size=950777 2024-09-13T00:00:30.788 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.786+0000 7fb8f2ce0640 1 > prepared multipart upload key=FX0NCGLK0KPJ upload=EVCWFHKZ5ELU tag=CNX3XRK8369N type=write size=3047929 2024-09-13T00:00:30.791 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.790+0000 7fb8f2ce0640 1 < completed delete operation key=44EDYFT9VF8Y tag=1BB0XV4OV6H6 type=del 2024-09-13T00:00:30.797 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.798+0000 7fb8f2ce0640 1 > prepared part key=_multipart_QMZREZ1QEDNJ.2~K77TT8VCRVHS.0 size=1048576 2024-09-13T00:00:30.799 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.798+0000 7fb8f2ce0640 1 > prepared part key=_multipart_QMZREZ1QEDNJ.2~K77TT8VCRVHS.1 size=1048576 2024-09-13T00:00:30.801 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.802+0000 7fb8f2ce0640 1 > prepared part key=_multipart_QMZREZ1QEDNJ.2~K77TT8VCRVHS.2 size=760275 2024-09-13T00:00:30.801 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.802+0000 7fb8f2ce0640 1 > prepared multipart upload key=QMZREZ1QEDNJ upload=K77TT8VCRVHS tag=JS9ZP7A4DHCH type=write size=3906003 2024-09-13T00:00:30.808 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.806+0000 7fb8f2ce0640 1 < suggested update operation key=NX26SI4F2ZMB tag=3C4T3WWQIQ9N type=del size=3450566 2024-09-13T00:00:30.812 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.810+0000 7fb8f2ce0640 1 > prepared operation key=N3Z5XIG3TZRQ tag=NT2VNQG7MKWU type=del size=2466064 2024-09-13T00:00:30.815 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.814+0000 7fb8f2ce0640 1 < completed delete operation key=44EDYFT9VF8Y tag=AGDBQ8JNCVAF type=del 2024-09-13T00:00:30.819 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.818+0000 7fb8f2ce0640 1 > prepared operation key=EJN6HR8I7XQF tag=6GH0LQ3GQRZ3 type=del size=2205020 2024-09-13T00:00:30.823 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.822+0000 7fb8f2ce0640 1 < completed delete operation key=W8XZ26311RJX tag=8YMTUWXXWSMQ type=del 2024-09-13T00:00:30.827 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.826+0000 7fb8f2ce0640 1 > prepared operation key=P8OYS5ZE1NX0 tag=O7FC7YG4345D type=del size=1018114 2024-09-13T00:00:30.831 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.830+0000 7fb8f2ce0640 1 < completed delete operation key=W8XZ26311RJX tag=8ATG2VBGZ8LS type=del 2024-09-13T00:00:30.836 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.834+0000 7fb8f2ce0640 1 > prepared operation key=H5EIP0E9CBWX tag=VKA8D16H9AZY type=del size=2385820 2024-09-13T00:00:30.841 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.842+0000 7fb8f2ce0640 1 < completed delete operation key=85YEW8N43FTA tag=ULQTNWZ4PYB0 type=del 2024-09-13T00:00:30.845 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.846+0000 7fb8f2ce0640 1 > prepared operation key=NQJQC8CEAC15 tag=58K8WUGWJYP6 type=del size=703611 2024-09-13T00:00:30.849 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.850+0000 7fb8f2ce0640 1 < completed delete operation key=H9AU4HD247YU tag=2YYRCB0UYW4P type=del 2024-09-13T00:00:30.852 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.850+0000 7fb8f2ce0640 1 > prepared operation key=O4MZ4UWNXME9 tag=WKIZETZFWJ9U type=write size=341146 2024-09-13T00:00:30.855 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.854+0000 7fb8f2ce0640 1 < completed delete operation key=EUE3LT0PDIXV tag=68PCDXRD5P1K type=del 2024-09-13T00:00:30.861 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.862+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~47QYZAI6N1EB.0 size=1048576 2024-09-13T00:00:30.863 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.862+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~47QYZAI6N1EB.1 size=177557 2024-09-13T00:00:30.863 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.862+0000 7fb8f2ce0640 1 > prepared multipart upload key=PVYZQLGYT3XU upload=47QYZAI6N1EB tag=F1SRZAT2DXB2 type=write size=2274709 2024-09-13T00:00:30.866 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.866+0000 7fb8f2ce0640 1 < completed part key=_multipart_QMZREZ1QEDNJ.2~3NZKLS52PP32.0 size=62923 2024-09-13T00:00:30.872 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.870+0000 7fb8f2ce0640 1 < completed multipart upload key=QMZREZ1QEDNJ upload=3NZKLS52PP32 tag=DNAUC56F5E6D type=write size=1111499 2024-09-13T00:00:30.878 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.878+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~N0BLZEPGJECX.0 size=848441 2024-09-13T00:00:30.878 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.878+0000 7fb8f2ce0640 1 > prepared multipart upload key=8Z6U7OOA0GBH upload=N0BLZEPGJECX tag=VDYFDIPHBUC7 type=write size=1897017 2024-09-13T00:00:30.881 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.882+0000 7fb8f2ce0640 1 < completed delete operation key=F0QCVR9E08WE tag=8QJOTLRU5UKH type=del 2024-09-13T00:00:30.884 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.882+0000 7fb8f2ce0640 1 > prepared operation key=PVYZQLGYT3XU tag=IZNWZYIAFLWA type=del size=3110723 2024-09-13T00:00:30.887 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.886+0000 7fb8f2ce0640 1 < completed delete operation key=5GM88ERM89KT tag=BK9AZZAUQHAX type=del 2024-09-13T00:00:30.890 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.890+0000 7fb8f2ce0640 1 > prepared operation key=NQJQC8CEAC15 tag=6SDR6T8ICSXU type=del size=1268230 2024-09-13T00:00:30.893 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.894+0000 7fb8f2ce0640 1 < completed delete operation key=ZTK9AYBSLCOZ tag=V49S03T5YEKF type=del 2024-09-13T00:00:30.898 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.898+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~7J2IL6E1QOA1.0 size=126665 2024-09-13T00:00:30.898 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.898+0000 7fb8f2ce0640 1 > prepared multipart upload key=8Z6U7OOA0GBH upload=7J2IL6E1QOA1 tag=VSNZXLCLA586 type=write size=1175241 2024-09-13T00:00:30.901 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.902+0000 7fb8f2ce0640 1 < completed part key=_multipart_2TWD9YLMSYJI.2~8NJE6SPJ5OWS.0 size=367489 2024-09-13T00:00:30.906 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.906+0000 7fb8f2ce0640 1 < completed multipart upload key=2TWD9YLMSYJI upload=8NJE6SPJ5OWS tag=4PHT3WG3R5J7 type=write size=1416065 2024-09-13T00:00:30.909 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.910+0000 7fb8f2ce0640 1 > prepared operation key=ZTK9AYBSLCOZ tag=1M1A1LJR5HMM type=del size=1444252 2024-09-13T00:00:30.912 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.910+0000 7fb8f2ce0640 1 < completed delete operation key=H5EIP0E9CBWX tag=93O7TZCC2J4I type=del 2024-09-13T00:00:30.918 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.918+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~ADZ5INLHM3MM.0 size=395259 2024-09-13T00:00:30.918 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.918+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZTK9AYBSLCOZ upload=ADZ5INLHM3MM tag=33T4PRXVGAOU type=write size=1443835 2024-09-13T00:00:30.924 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.922+0000 7fb8f2ce0640 1 < suggested remove operation key=EJN6HR8I7XQF tag=NN9Z6BMJ7PQW type=del 2024-09-13T00:00:30.928 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.926+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~62LMI3XQA45B.0 size=848853 2024-09-13T00:00:30.928 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.926+0000 7fb8f2ce0640 1 > prepared multipart upload key=8MPUVD37EDN9 upload=62LMI3XQA45B tag=4650RH25F7KC type=write size=1897429 2024-09-13T00:00:30.931 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.930+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~EVCWFHKZ5ELU.0 size=1048576 2024-09-13T00:00:30.933 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.934+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~EVCWFHKZ5ELU.1 size=950777 2024-09-13T00:00:30.937 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.938+0000 7fb8f2ce0640 1 < completed multipart upload key=FX0NCGLK0KPJ upload=EVCWFHKZ5ELU tag=CNX3XRK8369N type=write size=3047929 2024-09-13T00:00:30.940 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.938+0000 7fb8f2ce0640 1 > prepared operation key=ZFHN2SNTZ5W7 tag=A84MIQ1774F0 type=del size=2736334 2024-09-13T00:00:30.942 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.942+0000 7fb8f2ce0640 1 < completed part key=_multipart_QMZREZ1QEDNJ.2~K77TT8VCRVHS.0 size=1048576 2024-09-13T00:00:30.945 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.946+0000 7fb8f2ce0640 1 < completed part key=_multipart_QMZREZ1QEDNJ.2~K77TT8VCRVHS.1 size=1048576 2024-09-13T00:00:30.947 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.946+0000 7fb8f2ce0640 1 < completed part key=_multipart_QMZREZ1QEDNJ.2~K77TT8VCRVHS.2 size=760275 2024-09-13T00:00:30.952 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.950+0000 7fb8f2ce0640 1 < completed multipart upload key=QMZREZ1QEDNJ upload=K77TT8VCRVHS tag=JS9ZP7A4DHCH type=write size=3906003 2024-09-13T00:00:30.956 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.954+0000 7fb8f2ce0640 1 > prepared operation key=0WC6P8YNL0H2 tag=WK2XGNU3WGMK type=write size=388045 2024-09-13T00:00:30.959 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.958+0000 7fb8f2ce0640 1 < completed delete operation key=N3Z5XIG3TZRQ tag=NT2VNQG7MKWU type=del 2024-09-13T00:00:30.962 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.962+0000 7fb8f2ce0640 1 > prepared operation key=4K76U69GZ9AE tag=F3E32M6FWMUK type=del size=1675124 2024-09-13T00:00:30.964 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.962+0000 7fb8f2ce0640 -1 < no bi entry to suggest for operation key=EJN6HR8I7XQF tag=6GH0LQ3GQRZ3 type=del size=2205020: (2) No such file or directory 2024-09-13T00:00:30.968 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.966+0000 7fb8f2ce0640 1 > prepared operation key=2AHYRZUDFHP4 tag=UM0VHD82NCJ4 type=del size=3906424 2024-09-13T00:00:30.970 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.970+0000 7fb8f2ce0640 1 < completed delete operation key=P8OYS5ZE1NX0 tag=O7FC7YG4345D type=del 2024-09-13T00:00:30.974 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.974+0000 7fb8f2ce0640 1 > prepared operation key=NQJQC8CEAC15 tag=J6PSNJZVUMK6 type=del size=403996 2024-09-13T00:00:30.977 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.978+0000 7fb8f2ce0640 1 < completed delete operation key=H5EIP0E9CBWX tag=VKA8D16H9AZY type=del 2024-09-13T00:00:30.984 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.982+0000 7fb8f2ce0640 1 > prepared part key=_multipart_J8NGDSF9XRTP.2~BYVS7X6YCRPO.0 size=1048576 2024-09-13T00:00:30.986 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.986+0000 7fb8f2ce0640 1 > prepared part key=_multipart_J8NGDSF9XRTP.2~BYVS7X6YCRPO.1 size=1048576 2024-09-13T00:00:30.989 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.990+0000 7fb8f2ce0640 1 > prepared part key=_multipart_J8NGDSF9XRTP.2~BYVS7X6YCRPO.2 size=173314 2024-09-13T00:00:30.989 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.990+0000 7fb8f2ce0640 1 > prepared multipart upload key=J8NGDSF9XRTP upload=BYVS7X6YCRPO tag=NF2DSM52CJ0F type=write size=3319042 2024-09-13T00:00:30.993 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.990+0000 7fb8f2ce0640 1 < completed delete operation key=NQJQC8CEAC15 tag=58K8WUGWJYP6 type=del 2024-09-13T00:00:30.999 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:30.998+0000 7fb8f2ce0640 1 > prepared part key=_multipart_5GM88ERM89KT.2~U1ZDYF8Q7J7U.0 size=1048576 2024-09-13T00:00:31.002 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.002+0000 7fb8f2ce0640 1 > prepared part key=_multipart_5GM88ERM89KT.2~U1ZDYF8Q7J7U.1 size=1048576 2024-09-13T00:00:31.004 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.002+0000 7fb8f2ce0640 1 > prepared part key=_multipart_5GM88ERM89KT.2~U1ZDYF8Q7J7U.2 size=259514 2024-09-13T00:00:31.004 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.002+0000 7fb8f2ce0640 1 > prepared multipart upload key=5GM88ERM89KT upload=U1ZDYF8Q7J7U tag=X327B2MPXCCK type=write size=3405242 2024-09-13T00:00:31.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:30 smithi136 bash[75164]: cluster 2024-09-13T00:00:29.123626+0000 mgr.y (mgr.24661) 701 : cluster [DBG] pgmap v1374: 105 pgs: 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:00:31.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:30 smithi136 bash[75164]: cluster 2024-09-13T00:00:29.658692+0000 mon.a (mon.0) 2063 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:00:31.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:30 smithi136 bash[75164]: cluster 2024-09-13T00:00:29.658760+0000 mon.a (mon.0) 2064 : cluster [INF] Cluster is now healthy 2024-09-13T00:00:31.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:30 smithi136 bash[75164]: cluster 2024-09-13T00:00:29.681013+0000 mon.a (mon.0) 2065 : cluster [DBG] osdmap e1113: 8 total, 8 up, 8 in 2024-09-13T00:00:31.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:30 smithi136 bash[75164]: audit 2024-09-13T00:00:29.683142+0000 mon.c (mon.2) 91 : audit [INF] from='client.? 172.21.15.136:0/1613412818' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110696-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:00:31.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:30 smithi136 bash[75164]: audit 2024-09-13T00:00:29.684037+0000 mon.a (mon.0) 2066 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110696-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:00:31.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:30 smithi136 bash[76371]: cluster 2024-09-13T00:00:29.123626+0000 mgr.y (mgr.24661) 701 : cluster [DBG] pgmap v1374: 105 pgs: 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:00:31.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:30 smithi136 bash[76371]: cluster 2024-09-13T00:00:29.658692+0000 mon.a (mon.0) 2063 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:00:31.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:30 smithi136 bash[76371]: cluster 2024-09-13T00:00:29.658760+0000 mon.a (mon.0) 2064 : cluster [INF] Cluster is now healthy 2024-09-13T00:00:31.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:30 smithi136 bash[76371]: cluster 2024-09-13T00:00:29.681013+0000 mon.a (mon.0) 2065 : cluster [DBG] osdmap e1113: 8 total, 8 up, 8 in 2024-09-13T00:00:31.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:30 smithi136 bash[76371]: audit 2024-09-13T00:00:29.683142+0000 mon.c (mon.2) 91 : audit [INF] from='client.? 172.21.15.136:0/1613412818' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110696-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:00:31.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:30 smithi136 bash[76371]: audit 2024-09-13T00:00:29.684037+0000 mon.a (mon.0) 2066 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110696-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:00:31.009 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.006+0000 7fb8f2ce0640 1 < completed write operation key=O4MZ4UWNXME9 tag=WKIZETZFWJ9U type=write size=341146 2024-09-13T00:00:31.011 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.010+0000 7fb8f2ce0640 1 > prepared operation key=1HX06CCZJ1DD tag=AUOO3F6WWRYM type=del size=3990579 2024-09-13T00:00:31.015 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.014+0000 7fb8f2ce0640 1 < completed part key=_multipart_PVYZQLGYT3XU.2~47QYZAI6N1EB.0 size=1048576 2024-09-13T00:00:31.017 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.018+0000 7fb8f2ce0640 1 < completed part key=_multipart_PVYZQLGYT3XU.2~47QYZAI6N1EB.1 size=177557 2024-09-13T00:00:31.022 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.022+0000 7fb8f2ce0640 1 < completed multipart upload key=PVYZQLGYT3XU upload=47QYZAI6N1EB tag=F1SRZAT2DXB2 type=write size=2274709 2024-09-13T00:00:31.029 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.030+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2TWD9YLMSYJI.2~F64J6C7Y837P.0 size=1048576 2024-09-13T00:00:31.032 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.030+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2TWD9YLMSYJI.2~F64J6C7Y837P.1 size=1048576 2024-09-13T00:00:31.034 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.034+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2TWD9YLMSYJI.2~F64J6C7Y837P.2 size=911776 2024-09-13T00:00:31.034 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.034+0000 7fb8f2ce0640 1 > prepared multipart upload key=2TWD9YLMSYJI upload=F64J6C7Y837P tag=KCTNJELOTC4V type=write size=4057504 2024-09-13T00:00:31.036 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.034+0000 7fb8f2ce0640 1 < canceled part key=_multipart_8Z6U7OOA0GBH.2~N0BLZEPGJECX.0 size=848441 2024-09-13T00:00:31.041 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.042+0000 7fb8f2ce0640 1 < canceled multipart upload key=8Z6U7OOA0GBH upload=N0BLZEPGJECX tag=VDYFDIPHBUC7 type=write size=1897017 2024-09-13T00:00:31.045 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.046+0000 7fb8f2ce0640 1 > prepared operation key=EJN6HR8I7XQF tag=DOULONN59EKP type=del size=2219480 2024-09-13T00:00:31.050 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.050+0000 7fb8f2ce0640 1 < completed delete operation key=PVYZQLGYT3XU tag=IZNWZYIAFLWA type=del 2024-09-13T00:00:31.054 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.054+0000 7fb8f2ce0640 1 > prepared operation key=8Z6U7OOA0GBH tag=PCRB7DU3QL1F type=del size=3787214 2024-09-13T00:00:31.062 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.062+0000 7fb8f2ce0640 1 < suggested update operation key=NQJQC8CEAC15 tag=6SDR6T8ICSXU type=del size=1268230 2024-09-13T00:00:31.069 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.066+0000 7fb8f2ce0640 1 > prepared part key=_multipart_85YEW8N43FTA.2~79J6S7CBZIKY.0 size=578235 2024-09-13T00:00:31.069 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.066+0000 7fb8f2ce0640 1 > prepared multipart upload key=85YEW8N43FTA upload=79J6S7CBZIKY tag=GMNVVG8N7HHE type=write size=1626811 2024-09-13T00:00:31.070 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.070+0000 7fb8f2ce0640 1 < canceled part key=_multipart_8Z6U7OOA0GBH.2~7J2IL6E1QOA1.0 size=126665 2024-09-13T00:00:31.075 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.074+0000 7fb8f2ce0640 1 < canceled multipart upload key=8Z6U7OOA0GBH upload=7J2IL6E1QOA1 tag=VSNZXLCLA586 type=write size=1175241 2024-09-13T00:00:31.081 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.078+0000 7fb8f2ce0640 1 > prepared part key=_multipart_F0QCVR9E08WE.2~8B1Z6SWZMNWI.0 size=1048576 2024-09-13T00:00:31.083 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.082+0000 7fb8f2ce0640 1 > prepared part key=_multipart_F0QCVR9E08WE.2~8B1Z6SWZMNWI.1 size=995579 2024-09-13T00:00:31.083 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.082+0000 7fb8f2ce0640 1 > prepared multipart upload key=F0QCVR9E08WE upload=8B1Z6SWZMNWI tag=TL8F565ESM3B type=write size=3092731 2024-09-13T00:00:31.087 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.086+0000 7fb8f2ce0640 1 < completed delete operation key=ZTK9AYBSLCOZ tag=1M1A1LJR5HMM type=del 2024-09-13T00:00:31.093 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.094+0000 7fb8f2ce0640 1 > prepared part key=_multipart_44EDYFT9VF8Y.2~MSJ3IJ70XF7K.0 size=961927 2024-09-13T00:00:31.094 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.094+0000 7fb8f2ce0640 1 > prepared multipart upload key=44EDYFT9VF8Y upload=MSJ3IJ70XF7K tag=88HHQ23VG2ZG type=write size=2010503 2024-09-13T00:00:31.097 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.098+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~ADZ5INLHM3MM.0 size=395259 2024-09-13T00:00:31.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:30 smithi181 bash[67587]: cluster 2024-09-13T00:00:29.123626+0000 mgr.y (mgr.24661) 701 : cluster [DBG] pgmap v1374: 105 pgs: 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:00:31.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:30 smithi181 bash[67587]: cluster 2024-09-13T00:00:29.658692+0000 mon.a (mon.0) 2063 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:00:31.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:30 smithi181 bash[67587]: cluster 2024-09-13T00:00:29.658760+0000 mon.a (mon.0) 2064 : cluster [INF] Cluster is now healthy 2024-09-13T00:00:31.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:30 smithi181 bash[67587]: cluster 2024-09-13T00:00:29.681013+0000 mon.a (mon.0) 2065 : cluster [DBG] osdmap e1113: 8 total, 8 up, 8 in 2024-09-13T00:00:31.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:30 smithi181 bash[67587]: audit 2024-09-13T00:00:29.683142+0000 mon.c (mon.2) 91 : audit [INF] from='client.? 172.21.15.136:0/1613412818' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110696-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:00:31.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:30 smithi181 bash[67587]: audit 2024-09-13T00:00:29.684037+0000 mon.a (mon.0) 2066 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110696-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:00:31.102 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.102+0000 7fb8f2ce0640 1 < completed multipart upload key=ZTK9AYBSLCOZ upload=ADZ5INLHM3MM tag=33T4PRXVGAOU type=write size=1443835 2024-09-13T00:00:31.106 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.106+0000 7fb8f2ce0640 1 > prepared operation key=FX0NCGLK0KPJ tag=MW6G933GNCPO type=del size=889460 2024-09-13T00:00:31.109 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.110+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~62LMI3XQA45B.0 size=848853 2024-09-13T00:00:31.114 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.114+0000 7fb8f2ce0640 1 < completed multipart upload key=8MPUVD37EDN9 upload=62LMI3XQA45B tag=4650RH25F7KC type=write size=1897429 2024-09-13T00:00:31.118 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.118+0000 7fb8f2ce0640 1 > prepared operation key=8Z6U7OOA0GBH tag=JXZQ3ZGMZ1F7 type=del size=2877194 2024-09-13T00:00:31.121 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.122+0000 7fb8f2ce0640 1 < completed delete operation key=ZFHN2SNTZ5W7 tag=A84MIQ1774F0 type=del 2024-09-13T00:00:31.127 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.126+0000 7fb8f2ce0640 1 > prepared part key=_multipart_0WC6P8YNL0H2.2~9TMUJ9IFJOOK.0 size=1048576 2024-09-13T00:00:31.130 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.130+0000 7fb8f2ce0640 1 > prepared part key=_multipart_0WC6P8YNL0H2.2~9TMUJ9IFJOOK.1 size=412108 2024-09-13T00:00:31.130 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.130+0000 7fb8f2ce0640 1 > prepared multipart upload key=0WC6P8YNL0H2 upload=9TMUJ9IFJOOK tag=K0492XMWO6J6 type=write size=2509260 2024-09-13T00:00:31.136 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.134+0000 7fb8f2ce0640 1 < suggested remove operation key=0WC6P8YNL0H2 tag=WK2XGNU3WGMK type=write 2024-09-13T00:00:31.142 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.142+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H4M1TCN5W6XU.2~XBRVI3G23SZI.0 size=184726 2024-09-13T00:00:31.142 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.142+0000 7fb8f2ce0640 1 > prepared multipart upload key=H4M1TCN5W6XU upload=XBRVI3G23SZI tag=J273ETP17P73 type=write size=1233302 2024-09-13T00:00:31.146 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.146+0000 7fb8f2ce0640 1 < completed delete operation key=4K76U69GZ9AE tag=F3E32M6FWMUK type=del 2024-09-13T00:00:31.152 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.150+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EUE3LT0PDIXV.2~POAFTIABJX15.0 size=1048576 2024-09-13T00:00:31.154 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.154+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EUE3LT0PDIXV.2~POAFTIABJX15.1 size=568551 2024-09-13T00:00:31.154 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.154+0000 7fb8f2ce0640 1 > prepared multipart upload key=EUE3LT0PDIXV upload=POAFTIABJX15 tag=VZI8BSMX0SAN type=write size=2665703 2024-09-13T00:00:31.157 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.158+0000 7fb8f2ce0640 1 < completed delete operation key=2AHYRZUDFHP4 tag=UM0VHD82NCJ4 type=del 2024-09-13T00:00:31.161 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.162+0000 7fb8f2ce0640 1 > prepared operation key=8Z6U7OOA0GBH tag=5PBHUCWBVODY type=del size=2744102 2024-09-13T00:00:31.164 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.162+0000 7fb8f2ce0640 -1 < failed to complete operation key=NQJQC8CEAC15 tag=J6PSNJZVUMK6 type=del size=403996: (22) Invalid argument 2024-09-13T00:00:31.168 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.166+0000 7fb8f2ce0640 1 > prepared operation key=5GM88ERM89KT tag=0HB5KOMU6JGV type=del size=1787573 2024-09-13T00:00:31.172 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.170+0000 7fb8f2ce0640 1 < completed part key=_multipart_J8NGDSF9XRTP.2~BYVS7X6YCRPO.0 size=1048576 2024-09-13T00:00:31.175 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.174+0000 7fb8f2ce0640 1 < completed part key=_multipart_J8NGDSF9XRTP.2~BYVS7X6YCRPO.1 size=1048576 2024-09-13T00:00:31.177 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.178+0000 7fb8f2ce0640 1 < completed part key=_multipart_J8NGDSF9XRTP.2~BYVS7X6YCRPO.2 size=173314 2024-09-13T00:00:31.183 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.182+0000 7fb8f2ce0640 1 < completed multipart upload key=J8NGDSF9XRTP upload=BYVS7X6YCRPO tag=NF2DSM52CJ0F type=write size=3319042 2024-09-13T00:00:31.192 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.190+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NX26SI4F2ZMB.2~QCUXMS4HK53E.0 size=974828 2024-09-13T00:00:31.192 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.190+0000 7fb8f2ce0640 1 > prepared multipart upload key=NX26SI4F2ZMB upload=QCUXMS4HK53E tag=2UMNITRXQ73Z type=write size=2023404 2024-09-13T00:00:31.196 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.194+0000 7fb8f2ce0640 1 < completed part key=_multipart_5GM88ERM89KT.2~U1ZDYF8Q7J7U.0 size=1048576 2024-09-13T00:00:31.198 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.198+0000 7fb8f2ce0640 1 < completed part key=_multipart_5GM88ERM89KT.2~U1ZDYF8Q7J7U.1 size=1048576 2024-09-13T00:00:31.201 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.202+0000 7fb8f2ce0640 1 < completed part key=_multipart_5GM88ERM89KT.2~U1ZDYF8Q7J7U.2 size=259514 2024-09-13T00:00:31.205 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.206+0000 7fb8f2ce0640 1 < completed multipart upload key=5GM88ERM89KT upload=U1ZDYF8Q7J7U tag=X327B2MPXCCK type=write size=3405242 2024-09-13T00:00:31.209 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.210+0000 7fb8f2ce0640 1 > prepared operation key=W8XZ26311RJX tag=4IJLSM2EVXIY type=del size=1745819 2024-09-13T00:00:31.212 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.210+0000 7fb8f2ce0640 1 < canceled operation key=1HX06CCZJ1DD tag=AUOO3F6WWRYM type=del size=3990579 2024-09-13T00:00:31.217 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.214+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~BVF4M7RFP5IF.0 size=1048576 2024-09-13T00:00:31.219 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.218+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~BVF4M7RFP5IF.1 size=1048576 2024-09-13T00:00:31.221 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.222+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~BVF4M7RFP5IF.2 size=637072 2024-09-13T00:00:31.221 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.222+0000 7fb8f2ce0640 1 > prepared multipart upload key=7OX5OJZ8GXU5 upload=BVF4M7RFP5IF tag=OGJNL1DRCVIN type=write size=3782800 2024-09-13T00:00:31.224 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.222+0000 7fb8f2ce0640 1 < completed part key=_multipart_2TWD9YLMSYJI.2~F64J6C7Y837P.0 size=1048576 2024-09-13T00:00:31.227 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.226+0000 7fb8f2ce0640 1 < completed part key=_multipart_2TWD9YLMSYJI.2~F64J6C7Y837P.1 size=1048576 2024-09-13T00:00:31.232 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.230+0000 7fb8f2ce0640 1 < completed part key=_multipart_2TWD9YLMSYJI.2~F64J6C7Y837P.2 size=911776 2024-09-13T00:00:31.234 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.234+0000 7fb8f2ce0640 1 < completed multipart upload key=2TWD9YLMSYJI upload=F64J6C7Y837P tag=KCTNJELOTC4V type=write size=4057504 2024-09-13T00:00:31.238 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.238+0000 7fb8f2ce0640 1 > prepared operation key=W8XZ26311RJX tag=GU5YLUP75115 type=del size=2173735 2024-09-13T00:00:31.241 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.238+0000 7fb8f2ce0640 1 < completed delete operation key=EJN6HR8I7XQF tag=DOULONN59EKP type=del 2024-09-13T00:00:31.243 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.242+0000 7fb8f2ce0640 1 > prepared operation key=W8XZ26311RJX tag=BVXAROL54RGW type=del size=1160518 2024-09-13T00:00:31.249 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.250+0000 7fb8f2ce0640 1 < suggested remove operation key=8Z6U7OOA0GBH tag=PCRB7DU3QL1F type=del 2024-09-13T00:00:31.253 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.254+0000 7fb8f2ce0640 1 > prepared operation key=4K76U69GZ9AE tag=JY1PQNRSMPG0 type=del size=951159 2024-09-13T00:00:31.256 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.254+0000 7fb8f2ce0640 1 < completed part key=_multipart_85YEW8N43FTA.2~79J6S7CBZIKY.0 size=578235 2024-09-13T00:00:31.261 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.262+0000 7fb8f2ce0640 1 < completed multipart upload key=85YEW8N43FTA upload=79J6S7CBZIKY tag=GMNVVG8N7HHE type=write size=1626811 2024-09-13T00:00:31.265 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.266+0000 7fb8f2ce0640 1 > prepared operation key=N3Z5XIG3TZRQ tag=S0SRPC8PAQ09 type=del size=1450878 2024-09-13T00:00:31.268 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.266+0000 7fb8f2ce0640 1 < completed part key=_multipart_F0QCVR9E08WE.2~8B1Z6SWZMNWI.0 size=1048576 2024-09-13T00:00:31.271 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.270+0000 7fb8f2ce0640 1 < completed part key=_multipart_F0QCVR9E08WE.2~8B1Z6SWZMNWI.1 size=995579 2024-09-13T00:00:31.276 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.274+0000 7fb8f2ce0640 1 < completed multipart upload key=F0QCVR9E08WE upload=8B1Z6SWZMNWI tag=TL8F565ESM3B type=write size=3092731 2024-09-13T00:00:31.282 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.282+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~MC0PQKELMIDW.0 size=411147 2024-09-13T00:00:31.282 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.282+0000 7fb8f2ce0640 1 > prepared multipart upload key=2AHYRZUDFHP4 upload=MC0PQKELMIDW tag=DNVA0OGMQSP8 type=write size=1459723 2024-09-13T00:00:31.285 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.286+0000 7fb8f2ce0640 1 < completed part key=_multipart_44EDYFT9VF8Y.2~MSJ3IJ70XF7K.0 size=961927 2024-09-13T00:00:31.289 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.290+0000 7fb8f2ce0640 1 < completed multipart upload key=44EDYFT9VF8Y upload=MSJ3IJ70XF7K tag=88HHQ23VG2ZG type=write size=2010503 2024-09-13T00:00:31.292 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.290+0000 7fb8f2ce0640 1 > prepared operation key=2TWD9YLMSYJI tag=HBHCDHZD2D64 type=del size=1513936 2024-09-13T00:00:31.296 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.294+0000 7fb8f2ce0640 1 < completed delete operation key=FX0NCGLK0KPJ tag=MW6G933GNCPO type=del 2024-09-13T00:00:31.301 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.302+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~E7GVRSSS1VH1.0 size=571724 2024-09-13T00:00:31.301 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.302+0000 7fb8f2ce0640 1 > prepared multipart upload key=63FJDWBO2T9S upload=E7GVRSSS1VH1 tag=5YBLUCX4LU7O type=write size=1620300 2024-09-13T00:00:31.305 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.302+0000 7fb8f2ce0640 -1 < failed to complete operation key=8Z6U7OOA0GBH tag=JXZQ3ZGMZ1F7 type=del size=2877194: (22) Invalid argument 2024-09-13T00:00:31.308 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.306+0000 7fb8f2ce0640 1 > prepared operation key=2TWD9YLMSYJI tag=H2N0GZV1O0DQ type=del size=3580779 2024-09-13T00:00:31.311 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.310+0000 7fb8f2ce0640 1 < completed part key=_multipart_0WC6P8YNL0H2.2~9TMUJ9IFJOOK.0 size=1048576 2024-09-13T00:00:31.314 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.314+0000 7fb8f2ce0640 1 < completed part key=_multipart_0WC6P8YNL0H2.2~9TMUJ9IFJOOK.1 size=412108 2024-09-13T00:00:31.320 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.318+0000 7fb8f2ce0640 1 < completed multipart upload key=0WC6P8YNL0H2 upload=9TMUJ9IFJOOK tag=K0492XMWO6J6 type=write size=2509260 2024-09-13T00:00:31.322 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.322+0000 7fb8f2ce0640 1 > prepared operation key=NQJQC8CEAC15 tag=US6MYF7RHEN0 type=del size=2587426 2024-09-13T00:00:31.326 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.326+0000 7fb8f2ce0640 1 < completed part key=_multipart_H4M1TCN5W6XU.2~XBRVI3G23SZI.0 size=184726 2024-09-13T00:00:31.332 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.330+0000 7fb8f2ce0640 1 < completed multipart upload key=H4M1TCN5W6XU upload=XBRVI3G23SZI tag=J273ETP17P73 type=write size=1233302 2024-09-13T00:00:31.334 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.334+0000 7fb8f2ce0640 1 > prepared operation key=1PX7DBO9M3YI tag=YGT7Q7PNFXG2 type=write size=571544 2024-09-13T00:00:31.337 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.338+0000 7fb8f2ce0640 1 < completed part key=_multipart_EUE3LT0PDIXV.2~POAFTIABJX15.0 size=1048576 2024-09-13T00:00:31.339 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.338+0000 7fb8f2ce0640 1 < completed part key=_multipart_EUE3LT0PDIXV.2~POAFTIABJX15.1 size=568551 2024-09-13T00:00:31.345 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.342+0000 7fb8f2ce0640 1 < completed multipart upload key=EUE3LT0PDIXV upload=POAFTIABJX15 tag=VZI8BSMX0SAN type=write size=2665703 2024-09-13T00:00:31.350 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.350+0000 7fb8f2ce0640 1 > prepared part key=_multipart_44EDYFT9VF8Y.2~JBT3KFIBI9OJ.0 size=992862 2024-09-13T00:00:31.351 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.350+0000 7fb8f2ce0640 1 > prepared multipart upload key=44EDYFT9VF8Y upload=JBT3KFIBI9OJ tag=17821NF0QISQ type=write size=2041438 2024-09-13T00:00:31.352 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.350+0000 7fb8f2ce0640 -1 < no bi entry to suggest for operation key=8Z6U7OOA0GBH tag=5PBHUCWBVODY type=del size=2744102: (2) No such file or directory 2024-09-13T00:00:31.358 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.358+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~UBZ5H3A006VY.0 size=1048576 2024-09-13T00:00:31.361 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.362+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~UBZ5H3A006VY.1 size=872811 2024-09-13T00:00:31.361 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.362+0000 7fb8f2ce0640 1 > prepared multipart upload key=1PX7DBO9M3YI upload=UBZ5H3A006VY tag=EU29JV3KQVO3 type=write size=2969963 2024-09-13T00:00:31.364 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.362+0000 7fb8f2ce0640 1 < completed delete operation key=5GM88ERM89KT tag=0HB5KOMU6JGV type=del 2024-09-13T00:00:31.368 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.366+0000 7fb8f2ce0640 1 > prepared operation key=4K76U69GZ9AE tag=55SK5BI7MVCJ type=del size=1273065 2024-09-13T00:00:31.371 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.370+0000 7fb8f2ce0640 1 < completed part key=_multipart_NX26SI4F2ZMB.2~QCUXMS4HK53E.0 size=974828 2024-09-13T00:00:31.378 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.378+0000 7fb8f2ce0640 1 < completed multipart upload key=NX26SI4F2ZMB upload=QCUXMS4HK53E tag=2UMNITRXQ73Z type=write size=2023404 2024-09-13T00:00:31.381 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.382+0000 7fb8f2ce0640 1 > prepared operation key=N3Z5XIG3TZRQ tag=7JO2TFMC87K6 type=del size=327357 2024-09-13T00:00:31.389 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.390+0000 7fb8f2ce0640 1 < suggested update operation key=W8XZ26311RJX tag=4IJLSM2EVXIY type=del size=1745819 2024-09-13T00:00:31.393 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.394+0000 7fb8f2ce0640 1 > prepared operation key=44EDYFT9VF8Y tag=JY2R319N5TAP type=del size=3182814 2024-09-13T00:00:31.396 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.394+0000 7fb8f2ce0640 1 < completed part key=_multipart_7OX5OJZ8GXU5.2~BVF4M7RFP5IF.0 size=1048576 2024-09-13T00:00:31.399 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.398+0000 7fb8f2ce0640 1 < completed part key=_multipart_7OX5OJZ8GXU5.2~BVF4M7RFP5IF.1 size=1048576 2024-09-13T00:00:31.402 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.402+0000 7fb8f2ce0640 1 < completed part key=_multipart_7OX5OJZ8GXU5.2~BVF4M7RFP5IF.2 size=637072 2024-09-13T00:00:31.408 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.406+0000 7fb8f2ce0640 1 < completed multipart upload key=7OX5OJZ8GXU5 upload=BVF4M7RFP5IF tag=OGJNL1DRCVIN type=write size=3782800 2024-09-13T00:00:31.415 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.414+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~UYMP5OPTLDOW.0 size=1048576 2024-09-13T00:00:31.418 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.418+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~UYMP5OPTLDOW.1 size=102789 2024-09-13T00:00:31.418 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.418+0000 7fb8f2ce0640 1 > prepared multipart upload key=63FJDWBO2T9S upload=UYMP5OPTLDOW tag=NJ4GCGCX2GR9 type=write size=2199941 2024-09-13T00:00:31.426 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.426+0000 7fb8f2ce0640 1 < suggested update operation key=W8XZ26311RJX tag=GU5YLUP75115 type=del size=2173735 2024-09-13T00:00:31.432 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.430+0000 7fb8f2ce0640 1 > prepared part key=_multipart_44EDYFT9VF8Y.2~4QD1VVAYX1DO.0 size=1048576 2024-09-13T00:00:31.435 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.434+0000 7fb8f2ce0640 1 > prepared part key=_multipart_44EDYFT9VF8Y.2~4QD1VVAYX1DO.1 size=1048576 2024-09-13T00:00:31.439 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.438+0000 7fb8f2ce0640 1 > prepared part key=_multipart_44EDYFT9VF8Y.2~4QD1VVAYX1DO.2 size=326633 2024-09-13T00:00:31.439 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.438+0000 7fb8f2ce0640 1 > prepared multipart upload key=44EDYFT9VF8Y upload=4QD1VVAYX1DO tag=UTHU2VK58TC1 type=write size=3472361 2024-09-13T00:00:31.441 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.442+0000 7fb8f2ce0640 -1 < failed to complete operation key=W8XZ26311RJX tag=BVXAROL54RGW type=del size=1160518: (22) Invalid argument 2024-09-13T00:00:31.445 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.446+0000 7fb8f2ce0640 1 > prepared operation key=70F7D20HI504 tag=IRL1G6Y472D6 type=write size=260989 2024-09-13T00:00:31.449 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.446+0000 7fb8f2ce0640 1 < completed delete operation key=4K76U69GZ9AE tag=JY1PQNRSMPG0 type=del 2024-09-13T00:00:31.452 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.450+0000 7fb8f2ce0640 1 > prepared operation key=H9AU4HD247YU tag=IJDVPG61RME8 type=write size=928152 2024-09-13T00:00:31.456 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.454+0000 7fb8f2ce0640 1 < completed delete operation key=N3Z5XIG3TZRQ tag=S0SRPC8PAQ09 type=del 2024-09-13T00:00:31.458 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.458+0000 7fb8f2ce0640 1 > prepared operation key=H4M1TCN5W6XU tag=XQ6TBFREFLWT type=del size=2436416 2024-09-13T00:00:31.462 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.462+0000 7fb8f2ce0640 1 < completed part key=_multipart_2AHYRZUDFHP4.2~MC0PQKELMIDW.0 size=411147 2024-09-13T00:00:31.467 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.466+0000 7fb8f2ce0640 1 < completed multipart upload key=2AHYRZUDFHP4 upload=MC0PQKELMIDW tag=DNVA0OGMQSP8 type=write size=1459723 2024-09-13T00:00:31.470 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.470+0000 7fb8f2ce0640 1 > prepared operation key=H9AU4HD247YU tag=36CN9Z744PSW type=del size=772619 2024-09-13T00:00:31.474 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.474+0000 7fb8f2ce0640 1 < completed delete operation key=2TWD9YLMSYJI tag=HBHCDHZD2D64 type=del 2024-09-13T00:00:31.480 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.478+0000 7fb8f2ce0640 1 > prepared part key=_multipart_0WC6P8YNL0H2.2~SAVGXOJ25TTF.0 size=1048576 2024-09-13T00:00:31.483 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.482+0000 7fb8f2ce0640 1 > prepared part key=_multipart_0WC6P8YNL0H2.2~SAVGXOJ25TTF.1 size=1048576 2024-09-13T00:00:31.486 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.486+0000 7fb8f2ce0640 1 > prepared part key=_multipart_0WC6P8YNL0H2.2~SAVGXOJ25TTF.2 size=104899 2024-09-13T00:00:31.486 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.486+0000 7fb8f2ce0640 1 > prepared multipart upload key=0WC6P8YNL0H2 upload=SAVGXOJ25TTF tag=645MXSHOTQH0 type=write size=3250627 2024-09-13T00:00:31.489 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.490+0000 7fb8f2ce0640 1 < completed part key=_multipart_63FJDWBO2T9S.2~E7GVRSSS1VH1.0 size=571724 2024-09-13T00:00:31.495 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.494+0000 7fb8f2ce0640 1 < completed multipart upload key=63FJDWBO2T9S upload=E7GVRSSS1VH1 tag=5YBLUCX4LU7O type=write size=1620300 2024-09-13T00:00:31.501 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.498+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EJN6HR8I7XQF.2~FZO82NIHZLZO.0 size=1048576 2024-09-13T00:00:31.503 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.502+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EJN6HR8I7XQF.2~FZO82NIHZLZO.1 size=1048576 2024-09-13T00:00:31.505 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.506+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EJN6HR8I7XQF.2~FZO82NIHZLZO.2 size=1003097 2024-09-13T00:00:31.506 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.506+0000 7fb8f2ce0640 1 > prepared multipart upload key=EJN6HR8I7XQF upload=FZO82NIHZLZO tag=U8C954IW2XDN type=write size=4148825 2024-09-13T00:00:31.512 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.510+0000 7fb8f2ce0640 1 < suggested remove operation key=2TWD9YLMSYJI tag=H2N0GZV1O0DQ type=del 2024-09-13T00:00:31.518 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.518+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~MVR6B1A2JHV1.0 size=660977 2024-09-13T00:00:31.518 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.518+0000 7fb8f2ce0640 1 > prepared multipart upload key=W8XZ26311RJX upload=MVR6B1A2JHV1 tag=NN35SESNGLY3 type=write size=1709553 2024-09-13T00:00:31.521 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.522+0000 7fb8f2ce0640 1 < completed delete operation key=NQJQC8CEAC15 tag=US6MYF7RHEN0 type=del 2024-09-13T00:00:31.528 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.526+0000 7fb8f2ce0640 1 > prepared part key=_multipart_5GM88ERM89KT.2~KFJ5LA46246P.0 size=259194 2024-09-13T00:00:31.528 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.526+0000 7fb8f2ce0640 1 > prepared multipart upload key=5GM88ERM89KT upload=KFJ5LA46246P tag=JJ1V3ASYMPEW type=write size=1307770 2024-09-13T00:00:31.534 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.534+0000 7fb8f2ce0640 1 < suggested update operation key=1PX7DBO9M3YI tag=YGT7Q7PNFXG2 type=write size=571544 2024-09-13T00:00:31.540 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.538+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H5EIP0E9CBWX.2~4AIGF8QTRZTA.0 size=1048576 2024-09-13T00:00:31.542 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.542+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H5EIP0E9CBWX.2~4AIGF8QTRZTA.1 size=398826 2024-09-13T00:00:31.542 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.542+0000 7fb8f2ce0640 1 > prepared multipart upload key=H5EIP0E9CBWX upload=4AIGF8QTRZTA tag=LFPDLC0JO7OQ type=write size=2495978 2024-09-13T00:00:31.543 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.542+0000 7fb8f2ce0640 1 < canceled part key=_multipart_44EDYFT9VF8Y.2~JBT3KFIBI9OJ.0 size=992862 2024-09-13T00:00:31.548 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.546+0000 7fb8f2ce0640 1 < canceled multipart upload key=44EDYFT9VF8Y upload=JBT3KFIBI9OJ tag=17821NF0QISQ type=write size=2041438 2024-09-13T00:00:31.552 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.550+0000 7fb8f2ce0640 1 > prepared operation key=QMZREZ1QEDNJ tag=8HLT60DS79T3 type=del size=3626105 2024-09-13T00:00:31.555 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.554+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~UBZ5H3A006VY.0 size=1048576 2024-09-13T00:00:31.558 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.558+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~UBZ5H3A006VY.1 size=872811 2024-09-13T00:00:31.563 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.562+0000 7fb8f2ce0640 1 < completed multipart upload key=1PX7DBO9M3YI upload=UBZ5H3A006VY tag=EU29JV3KQVO3 type=write size=2969963 2024-09-13T00:00:31.569 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.566+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~49TBKPK0VLNZ.0 size=765060 2024-09-13T00:00:31.569 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.566+0000 7fb8f2ce0640 1 > prepared multipart upload key=W8XZ26311RJX upload=49TBKPK0VLNZ tag=TGD3A0KVXXQJ type=write size=1813636 2024-09-13T00:00:31.576 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.574+0000 7fb8f2ce0640 1 < suggested remove operation key=4K76U69GZ9AE tag=55SK5BI7MVCJ type=del 2024-09-13T00:00:31.579 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.578+0000 7fb8f2ce0640 1 > prepared operation key=7OX5OJZ8GXU5 tag=UTUS9IBLQQHD type=del size=4120581 2024-09-13T00:00:31.583 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.582+0000 7fb8f2ce0640 1 < completed delete operation key=N3Z5XIG3TZRQ tag=7JO2TFMC87K6 type=del 2024-09-13T00:00:31.587 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.586+0000 7fb8f2ce0640 1 > prepared operation key=PVYZQLGYT3XU tag=S091RN1LN9LJ type=write size=321436 2024-09-13T00:00:31.591 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.590+0000 7fb8f2ce0640 1 < completed delete operation key=44EDYFT9VF8Y tag=JY2R319N5TAP type=del 2024-09-13T00:00:31.597 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.598+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H5EIP0E9CBWX.2~N5G64DZEPHBO.0 size=249587 2024-09-13T00:00:31.598 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.598+0000 7fb8f2ce0640 1 > prepared multipart upload key=H5EIP0E9CBWX upload=N5G64DZEPHBO tag=CFLDGRB24K3R type=write size=1298163 2024-09-13T00:00:31.601 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.602+0000 7fb8f2ce0640 1 < completed part key=_multipart_63FJDWBO2T9S.2~UYMP5OPTLDOW.0 size=1048576 2024-09-13T00:00:31.604 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.602+0000 7fb8f2ce0640 1 < completed part key=_multipart_63FJDWBO2T9S.2~UYMP5OPTLDOW.1 size=102789 2024-09-13T00:00:31.610 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.610+0000 7fb8f2ce0640 1 < completed multipart upload key=63FJDWBO2T9S upload=UYMP5OPTLDOW tag=NJ4GCGCX2GR9 type=write size=2199941 2024-09-13T00:00:31.613 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.614+0000 7fb8f2ce0640 1 > prepared operation key=2AHYRZUDFHP4 tag=20QGAK26U5Q2 type=del size=528359 2024-09-13T00:00:31.614 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.614+0000 7fb8f2ce0640 1 < canceled part key=_multipart_44EDYFT9VF8Y.2~4QD1VVAYX1DO.0 size=1048576 2024-09-13T00:00:31.617 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.618+0000 7fb8f2ce0640 1 < completed part key=_multipart_44EDYFT9VF8Y.2~4QD1VVAYX1DO.1 size=1048576 2024-09-13T00:00:31.617 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.618+0000 7fb8f2ce0640 1 < canceled part key=_multipart_44EDYFT9VF8Y.2~4QD1VVAYX1DO.2 size=326633 2024-09-13T00:00:31.622 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.622+0000 7fb8f2ce0640 1 < canceled multipart upload key=44EDYFT9VF8Y upload=4QD1VVAYX1DO tag=UTHU2VK58TC1 type=write size=3472361 2024-09-13T00:00:31.626 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.626+0000 7fb8f2ce0640 1 > prepared operation key=2AHYRZUDFHP4 tag=H9HJUP3XOBHR type=del size=75433 2024-09-13T00:00:31.630 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.630+0000 7fb8f2ce0640 1 < completed write operation key=70F7D20HI504 tag=IRL1G6Y472D6 type=write size=260989 2024-09-13T00:00:31.634 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.634+0000 7fb8f2ce0640 1 > prepared operation key=H9AU4HD247YU tag=RA0KTR0WQNJ4 type=del size=3920247 2024-09-13T00:00:31.640 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.638+0000 7fb8f2ce0640 1 < suggested update operation key=H9AU4HD247YU tag=IJDVPG61RME8 type=write size=928152 2024-09-13T00:00:31.647 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.646+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~579S39F1PF67.0 size=1048576 2024-09-13T00:00:31.650 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.650+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~579S39F1PF67.1 size=1048576 2024-09-13T00:00:31.652 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.650+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~579S39F1PF67.2 size=658170 2024-09-13T00:00:31.652 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.650+0000 7fb8f2ce0640 1 > prepared multipart upload key=1PX7DBO9M3YI upload=579S39F1PF67 tag=OYN0417HI5N1 type=write size=3803898 2024-09-13T00:00:31.655 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.654+0000 7fb8f2ce0640 1 < completed delete operation key=H4M1TCN5W6XU tag=XQ6TBFREFLWT type=del 2024-09-13T00:00:31.658 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.658+0000 7fb8f2ce0640 1 > prepared operation key=NQJQC8CEAC15 tag=DDCUCFK31SOD type=del size=1599207 2024-09-13T00:00:31.662 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.662+0000 7fb8f2ce0640 -1 < failed to complete operation key=H9AU4HD247YU tag=36CN9Z744PSW type=del size=772619: (22) Invalid argument 2024-09-13T00:00:31.667 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.666+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~1VAKN31CVV9L.0 size=1048576 2024-09-13T00:00:31.670 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.670+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~1VAKN31CVV9L.1 size=268510 2024-09-13T00:00:31.670 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.670+0000 7fb8f2ce0640 1 > prepared multipart upload key=NQJQC8CEAC15 upload=1VAKN31CVV9L tag=EOBLCE73W6EX type=write size=2365662 2024-09-13T00:00:31.673 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.674+0000 7fb8f2ce0640 1 < completed part key=_multipart_0WC6P8YNL0H2.2~SAVGXOJ25TTF.0 size=1048576 2024-09-13T00:00:31.674 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.674+0000 7fb8f2ce0640 1 < canceled part key=_multipart_0WC6P8YNL0H2.2~SAVGXOJ25TTF.1 size=1048576 2024-09-13T00:00:31.676 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.674+0000 7fb8f2ce0640 1 < completed part key=_multipart_0WC6P8YNL0H2.2~SAVGXOJ25TTF.2 size=104899 2024-09-13T00:00:31.682 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.682+0000 7fb8f2ce0640 1 < canceled multipart upload key=0WC6P8YNL0H2 upload=SAVGXOJ25TTF tag=645MXSHOTQH0 type=write size=3250627 2024-09-13T00:00:31.698 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.698+0000 7fb8f2ce0640 1 > prepared part key=_multipart_4K76U69GZ9AE.2~XEU8KHF8X7LC.0 size=104482 2024-09-13T00:00:31.698 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.698+0000 7fb8f2ce0640 1 > prepared multipart upload key=4K76U69GZ9AE upload=XEU8KHF8X7LC tag=PP0JWAXOYIUA type=write size=1153058 2024-09-13T00:00:31.701 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.702+0000 7fb8f2ce0640 1 < completed part key=_multipart_EJN6HR8I7XQF.2~FZO82NIHZLZO.0 size=1048576 2024-09-13T00:00:31.705 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.706+0000 7fb8f2ce0640 1 < completed part key=_multipart_EJN6HR8I7XQF.2~FZO82NIHZLZO.1 size=1048576 2024-09-13T00:00:31.708 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.706+0000 7fb8f2ce0640 1 < completed part key=_multipart_EJN6HR8I7XQF.2~FZO82NIHZLZO.2 size=1003097 2024-09-13T00:00:31.713 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.714+0000 7fb8f2ce0640 1 < completed multipart upload key=EJN6HR8I7XQF upload=FZO82NIHZLZO tag=U8C954IW2XDN type=write size=4148825 2024-09-13T00:00:31.716 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.714+0000 7fb8f2ce0640 1 > prepared operation key=NX26SI4F2ZMB tag=T4S0O9UJQZNM type=del size=1701303 2024-09-13T00:00:31.719 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.718+0000 7fb8f2ce0640 1 < completed part key=_multipart_W8XZ26311RJX.2~MVR6B1A2JHV1.0 size=660977 2024-09-13T00:00:31.724 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.722+0000 7fb8f2ce0640 1 < completed multipart upload key=W8XZ26311RJX upload=MVR6B1A2JHV1 tag=NN35SESNGLY3 type=write size=1709553 2024-09-13T00:00:31.728 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.726+0000 7fb8f2ce0640 1 > prepared operation key=8Z6U7OOA0GBH tag=1M3FVQAQ2LJM type=del size=1399958 2024-09-13T00:00:31.731 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.730+0000 7fb8f2ce0640 1 < completed part key=_multipart_5GM88ERM89KT.2~KFJ5LA46246P.0 size=259194 2024-09-13T00:00:31.736 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.734+0000 7fb8f2ce0640 1 < completed multipart upload key=5GM88ERM89KT upload=KFJ5LA46246P tag=JJ1V3ASYMPEW type=write size=1307770 2024-09-13T00:00:31.742 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.742+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H4M1TCN5W6XU.2~57L0EPITFVQL.0 size=1048576 2024-09-13T00:00:31.745 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.742+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H4M1TCN5W6XU.2~57L0EPITFVQL.1 size=623401 2024-09-13T00:00:31.745 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.746+0000 7fb8f2ce0640 1 > prepared multipart upload key=H4M1TCN5W6XU upload=57L0EPITFVQL tag=BM3H0HT3EIKX type=write size=2720553 2024-09-13T00:00:31.748 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.746+0000 7fb8f2ce0640 1 < completed part key=_multipart_H5EIP0E9CBWX.2~4AIGF8QTRZTA.0 size=1048576 2024-09-13T00:00:31.751 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.750+0000 7fb8f2ce0640 1 < completed part key=_multipart_H5EIP0E9CBWX.2~4AIGF8QTRZTA.1 size=398826 2024-09-13T00:00:31.756 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.754+0000 7fb8f2ce0640 1 < completed multipart upload key=H5EIP0E9CBWX upload=4AIGF8QTRZTA tag=LFPDLC0JO7OQ type=write size=2495978 2024-09-13T00:00:31.760 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.758+0000 7fb8f2ce0640 1 > prepared operation key=5GM88ERM89KT tag=7PA6U2XICHGF type=write size=395728 2024-09-13T00:00:31.763 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.762+0000 7fb8f2ce0640 1 < completed delete operation key=QMZREZ1QEDNJ tag=8HLT60DS79T3 type=del 2024-09-13T00:00:31.766 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.766+0000 7fb8f2ce0640 1 > prepared operation key=QMZREZ1QEDNJ tag=WIDFU3HRHW4E type=del size=3493426 2024-09-13T00:00:31.769 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.770+0000 7fb8f2ce0640 1 < completed part key=_multipart_W8XZ26311RJX.2~49TBKPK0VLNZ.0 size=765060 2024-09-13T00:00:31.775 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.774+0000 7fb8f2ce0640 1 < completed multipart upload key=W8XZ26311RJX upload=49TBKPK0VLNZ tag=TGD3A0KVXXQJ type=write size=1813636 2024-09-13T00:00:31.778 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.778+0000 7fb8f2ce0640 1 > prepared operation key=7OX5OJZ8GXU5 tag=EE5Z304GV0R8 type=del size=439033 2024-09-13T00:00:31.782 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.782+0000 7fb8f2ce0640 1 < completed delete operation key=7OX5OJZ8GXU5 tag=UTUS9IBLQQHD type=del 2024-09-13T00:00:31.788 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.786+0000 7fb8f2ce0640 1 > prepared part key=_multipart_0WC6P8YNL0H2.2~SQDJS32EB7IR.0 size=1048576 2024-09-13T00:00:31.791 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.790+0000 7fb8f2ce0640 1 > prepared part key=_multipart_0WC6P8YNL0H2.2~SQDJS32EB7IR.1 size=518825 2024-09-13T00:00:31.791 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.790+0000 7fb8f2ce0640 1 > prepared multipart upload key=0WC6P8YNL0H2 upload=SQDJS32EB7IR tag=ZQ4I4W7HWT76 type=write size=2615977 2024-09-13T00:00:31.795 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.794+0000 7fb8f2ce0640 1 < canceled operation key=PVYZQLGYT3XU tag=S091RN1LN9LJ type=write size=321436 2024-09-13T00:00:31.799 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.798+0000 7fb8f2ce0640 1 > prepared operation key=ZFHN2SNTZ5W7 tag=YR3MFF1MVDZ8 type=del size=3043850 2024-09-13T00:00:31.803 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.802+0000 7fb8f2ce0640 1 < completed part key=_multipart_H5EIP0E9CBWX.2~N5G64DZEPHBO.0 size=249587 2024-09-13T00:00:31.809 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.806+0000 7fb8f2ce0640 1 < completed multipart upload key=H5EIP0E9CBWX upload=N5G64DZEPHBO tag=CFLDGRB24K3R type=write size=1298163 2024-09-13T00:00:31.813 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.810+0000 7fb8f2ce0640 1 > prepared operation key=2AHYRZUDFHP4 tag=BFEPN36Q9D4H type=del size=3165310 2024-09-13T00:00:31.817 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.818+0000 7fb8f2ce0640 1 < completed delete operation key=2AHYRZUDFHP4 tag=20QGAK26U5Q2 type=del 2024-09-13T00:00:31.824 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.822+0000 7fb8f2ce0640 1 > prepared part key=_multipart_QMZREZ1QEDNJ.2~K8D1LQ4QC61Y.0 size=1048576 2024-09-13T00:00:31.826 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.826+0000 7fb8f2ce0640 1 > prepared part key=_multipart_QMZREZ1QEDNJ.2~K8D1LQ4QC61Y.1 size=175438 2024-09-13T00:00:31.826 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.826+0000 7fb8f2ce0640 1 > prepared multipart upload key=QMZREZ1QEDNJ upload=K8D1LQ4QC61Y tag=WK8TTITZF9I5 type=write size=2272590 2024-09-13T00:00:31.830 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.830+0000 7fb8f2ce0640 1 < completed delete operation key=2AHYRZUDFHP4 tag=H9HJUP3XOBHR type=del 2024-09-13T00:00:31.836 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.834+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~J9P34E00L1AV.0 size=1048576 2024-09-13T00:00:31.839 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.838+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~J9P34E00L1AV.1 size=675871 2024-09-13T00:00:31.839 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.838+0000 7fb8f2ce0640 1 > prepared multipart upload key=1PX7DBO9M3YI upload=J9P34E00L1AV tag=G05R6BAD31ME type=write size=2773023 2024-09-13T00:00:31.842 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.842+0000 7fb8f2ce0640 -1 < failed to complete operation key=H9AU4HD247YU tag=RA0KTR0WQNJ4 type=del size=3920247: (22) Invalid argument 2024-09-13T00:00:31.846 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.846+0000 7fb8f2ce0640 1 > prepared operation key=7OX5OJZ8GXU5 tag=78ICI8WTYACZ type=del size=957842 2024-09-13T00:00:31.850 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.850+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~579S39F1PF67.0 size=1048576 2024-09-13T00:00:31.853 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.854+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~579S39F1PF67.1 size=1048576 2024-09-13T00:00:31.856 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.854+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~579S39F1PF67.2 size=658170 2024-09-13T00:00:31.862 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.862+0000 7fb8f2ce0640 1 < completed multipart upload key=1PX7DBO9M3YI upload=579S39F1PF67 tag=OYN0417HI5N1 type=write size=3803898 2024-09-13T00:00:31.868 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.866+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~7MFQGOSP1SRF.0 size=542448 2024-09-13T00:00:31.868 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.866+0000 7fb8f2ce0640 1 > prepared multipart upload key=W8XZ26311RJX upload=7MFQGOSP1SRF tag=T316LZXKO6F7 type=write size=1591024 2024-09-13T00:00:31.871 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.870+0000 7fb8f2ce0640 1 < completed delete operation key=NQJQC8CEAC15 tag=DDCUCFK31SOD type=del 2024-09-13T00:00:31.876 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.874+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H4M1TCN5W6XU.2~UBCXPMN7I462.0 size=1048576 2024-09-13T00:00:31.879 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.878+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H4M1TCN5W6XU.2~UBCXPMN7I462.1 size=1048576 2024-09-13T00:00:31.881 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.882+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H4M1TCN5W6XU.2~UBCXPMN7I462.2 size=923637 2024-09-13T00:00:31.882 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.882+0000 7fb8f2ce0640 1 > prepared multipart upload key=H4M1TCN5W6XU upload=UBCXPMN7I462 tag=4O9B5B3AM7KK type=write size=4069365 2024-09-13T00:00:31.885 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.886+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~1VAKN31CVV9L.0 size=1048576 2024-09-13T00:00:31.885 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.886+0000 7fb8f2ce0640 1 < canceled part key=_multipart_NQJQC8CEAC15.2~1VAKN31CVV9L.1 size=268510 2024-09-13T00:00:31.890 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.890+0000 7fb8f2ce0640 1 < canceled multipart upload key=NQJQC8CEAC15 upload=1VAKN31CVV9L tag=EOBLCE73W6EX type=write size=2365662 2024-09-13T00:00:31.896 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.894+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NX26SI4F2ZMB.2~GW108S4EOK7R.0 size=1048576 2024-09-13T00:00:31.899 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.898+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NX26SI4F2ZMB.2~GW108S4EOK7R.1 size=1048576 2024-09-13T00:00:31.901 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.902+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NX26SI4F2ZMB.2~GW108S4EOK7R.2 size=983109 2024-09-13T00:00:31.901 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.902+0000 7fb8f2ce0640 1 > prepared multipart upload key=NX26SI4F2ZMB upload=GW108S4EOK7R tag=I40XDT260391 type=write size=4128837 2024-09-13T00:00:31.905 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.902+0000 7fb8f2ce0640 1 < completed part key=_multipart_4K76U69GZ9AE.2~XEU8KHF8X7LC.0 size=104482 2024-09-13T00:00:31.910 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.910+0000 7fb8f2ce0640 1 < completed multipart upload key=4K76U69GZ9AE upload=XEU8KHF8X7LC tag=PP0JWAXOYIUA type=write size=1153058 2024-09-13T00:00:31.914 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.914+0000 7fb8f2ce0640 1 > prepared operation key=QMZREZ1QEDNJ tag=2WQ0X2USYMG6 type=del size=1349033 2024-09-13T00:00:31.921 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.922+0000 7fb8f2ce0640 1 < suggested update operation key=NX26SI4F2ZMB tag=T4S0O9UJQZNM type=del size=1701303 2024-09-13T00:00:31.925 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.922+0000 7fb8f2ce0640 1 > prepared operation key=63FJDWBO2T9S tag=NEUXV3M1P1JR type=write size=513535 2024-09-13T00:00:31.928 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.926+0000 7fb8f2ce0640 1 < completed delete operation key=8Z6U7OOA0GBH tag=1M3FVQAQ2LJM type=del 2024-09-13T00:00:31.932 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.930+0000 7fb8f2ce0640 1 > prepared operation key=2AHYRZUDFHP4 tag=FD2SOS3BXDP5 type=del size=2356102 2024-09-13T00:00:31.936 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.934+0000 7fb8f2ce0640 1 < completed part key=_multipart_H4M1TCN5W6XU.2~57L0EPITFVQL.0 size=1048576 2024-09-13T00:00:31.938 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.938+0000 7fb8f2ce0640 1 < completed part key=_multipart_H4M1TCN5W6XU.2~57L0EPITFVQL.1 size=623401 2024-09-13T00:00:31.943 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.942+0000 7fb8f2ce0640 1 < completed multipart upload key=H4M1TCN5W6XU upload=57L0EPITFVQL tag=BM3H0HT3EIKX type=write size=2720553 2024-09-13T00:00:31.950 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.950+0000 7fb8f2ce0640 1 > prepared part key=_multipart_F0QCVR9E08WE.2~7FOIGLWXBPIX.0 size=216994 2024-09-13T00:00:31.950 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.950+0000 7fb8f2ce0640 1 > prepared multipart upload key=F0QCVR9E08WE upload=7FOIGLWXBPIX tag=W609F8EBKFTH type=write size=1265570 2024-09-13T00:00:31.953 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.954+0000 7fb8f2ce0640 1 < completed write operation key=5GM88ERM89KT tag=7PA6U2XICHGF type=write size=395728 2024-09-13T00:00:31.959 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.958+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H4M1TCN5W6XU.2~3WGK6N72UXFS.0 size=1048576 2024-09-13T00:00:31.962 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.962+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H4M1TCN5W6XU.2~3WGK6N72UXFS.1 size=1048576 2024-09-13T00:00:31.965 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.962+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H4M1TCN5W6XU.2~3WGK6N72UXFS.2 size=858629 2024-09-13T00:00:31.965 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.962+0000 7fb8f2ce0640 1 > prepared multipart upload key=H4M1TCN5W6XU upload=3WGK6N72UXFS tag=1MC7F1UKLFVT type=write size=4004357 2024-09-13T00:00:31.968 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.966+0000 7fb8f2ce0640 1 < completed delete operation key=QMZREZ1QEDNJ tag=WIDFU3HRHW4E type=del 2024-09-13T00:00:31.972 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.970+0000 7fb8f2ce0640 1 > prepared operation key=H9AU4HD247YU tag=XY7J4RFS1HA5 type=del size=1528590 2024-09-13T00:00:31.979 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.978+0000 7fb8f2ce0640 1 < suggested remove operation key=7OX5OJZ8GXU5 tag=EE5Z304GV0R8 type=del 2024-09-13T00:00:31.986 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.986+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~9H9RW08EBOX0.0 size=1048576 2024-09-13T00:00:31.988 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.986+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~9H9RW08EBOX0.1 size=346477 2024-09-13T00:00:31.989 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.986+0000 7fb8f2ce0640 1 > prepared multipart upload key=70F7D20HI504 upload=9H9RW08EBOX0 tag=LDXE6ZXLD2UE type=write size=2443629 2024-09-13T00:00:31.993 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.990+0000 7fb8f2ce0640 1 < completed part key=_multipart_0WC6P8YNL0H2.2~SQDJS32EB7IR.0 size=1048576 2024-09-13T00:00:31.995 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.994+0000 7fb8f2ce0640 1 < completed part key=_multipart_0WC6P8YNL0H2.2~SQDJS32EB7IR.1 size=518825 2024-09-13T00:00:32.001 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:31.998+0000 7fb8f2ce0640 1 < completed multipart upload key=0WC6P8YNL0H2 upload=SQDJS32EB7IR tag=ZQ4I4W7HWT76 type=write size=2615977 2024-09-13T00:00:32.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:31 smithi136 bash[75164]: audit 2024-09-13T00:00:30.684114+0000 mon.a (mon.0) 2067 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110696-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:00:32.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:31 smithi136 bash[75164]: cluster 2024-09-13T00:00:30.684626+0000 mon.a (mon.0) 2068 : cluster [DBG] osdmap e1114: 8 total, 8 up, 8 in 2024-09-13T00:00:32.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:31 smithi136 bash[76371]: audit 2024-09-13T00:00:30.684114+0000 mon.a (mon.0) 2067 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110696-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:00:32.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:31 smithi136 bash[76371]: cluster 2024-09-13T00:00:30.684626+0000 mon.a (mon.0) 2068 : cluster [DBG] osdmap e1114: 8 total, 8 up, 8 in 2024-09-13T00:00:32.007 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.006+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~0EQZSLX14Q5T.0 size=1048576 2024-09-13T00:00:32.010 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.010+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~0EQZSLX14Q5T.1 size=649825 2024-09-13T00:00:32.010 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.010+0000 7fb8f2ce0640 1 > prepared multipart upload key=70F7D20HI504 upload=0EQZSLX14Q5T tag=IPX61WJUZGI0 type=write size=2746977 2024-09-13T00:00:32.013 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.014+0000 7fb8f2ce0640 1 < completed delete operation key=ZFHN2SNTZ5W7 tag=YR3MFF1MVDZ8 type=del 2024-09-13T00:00:32.020 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.018+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~S5XEHMSP8GPF.0 size=1048576 2024-09-13T00:00:32.022 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.022+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~S5XEHMSP8GPF.1 size=377851 2024-09-13T00:00:32.022 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.022+0000 7fb8f2ce0640 1 > prepared multipart upload key=7OX5OJZ8GXU5 upload=S5XEHMSP8GPF tag=PR3RI2BXBGA4 type=write size=2475003 2024-09-13T00:00:32.029 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.030+0000 7fb8f2ce0640 1 < suggested remove operation key=2AHYRZUDFHP4 tag=BFEPN36Q9D4H type=del 2024-09-13T00:00:32.032 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.030+0000 7fb8f2ce0640 1 > prepared operation key=85YEW8N43FTA tag=BJGV3RLZ0W0W type=write size=396819 2024-09-13T00:00:32.036 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.034+0000 7fb8f2ce0640 1 < completed part key=_multipart_QMZREZ1QEDNJ.2~K8D1LQ4QC61Y.0 size=1048576 2024-09-13T00:00:32.036 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.034+0000 7fb8f2ce0640 1 < canceled part key=_multipart_QMZREZ1QEDNJ.2~K8D1LQ4QC61Y.1 size=175438 2024-09-13T00:00:32.042 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.042+0000 7fb8f2ce0640 1 < canceled multipart upload key=QMZREZ1QEDNJ upload=K8D1LQ4QC61Y tag=WK8TTITZF9I5 type=write size=2272590 2024-09-13T00:00:32.045 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.046+0000 7fb8f2ce0640 1 > prepared operation key=NQJQC8CEAC15 tag=4JWUARGP3J3E type=del size=1593750 2024-09-13T00:00:32.052 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.050+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~J9P34E00L1AV.0 size=1048576 2024-09-13T00:00:32.053 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.050+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~J9P34E00L1AV.1 size=675871 2024-09-13T00:00:32.058 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.058+0000 7fb8f2ce0640 1 < completed multipart upload key=1PX7DBO9M3YI upload=J9P34E00L1AV tag=G05R6BAD31ME type=write size=2773023 2024-09-13T00:00:32.062 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.062+0000 7fb8f2ce0640 1 > prepared operation key=P8OYS5ZE1NX0 tag=217I2D3UD9FG type=write size=992259 2024-09-13T00:00:32.066 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.066+0000 7fb8f2ce0640 -1 < failed to complete operation key=7OX5OJZ8GXU5 tag=78ICI8WTYACZ type=del size=957842: (22) Invalid argument 2024-09-13T00:00:32.071 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.070+0000 7fb8f2ce0640 1 > prepared operation key=1PX7DBO9M3YI tag=4BWJU4DYLNT0 type=del size=2662642 2024-09-13T00:00:32.075 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.074+0000 7fb8f2ce0640 1 < completed part key=_multipart_W8XZ26311RJX.2~7MFQGOSP1SRF.0 size=542448 2024-09-13T00:00:32.080 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.078+0000 7fb8f2ce0640 1 < completed multipart upload key=W8XZ26311RJX upload=7MFQGOSP1SRF tag=T316LZXKO6F7 type=write size=1591024 2024-09-13T00:00:32.084 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.082+0000 7fb8f2ce0640 1 > prepared operation key=N3Z5XIG3TZRQ tag=IWZGO4X056UB type=del size=3807735 2024-09-13T00:00:32.087 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.086+0000 7fb8f2ce0640 1 < completed part key=_multipart_H4M1TCN5W6XU.2~UBCXPMN7I462.0 size=1048576 2024-09-13T00:00:32.090 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.090+0000 7fb8f2ce0640 1 < completed part key=_multipart_H4M1TCN5W6XU.2~UBCXPMN7I462.1 size=1048576 2024-09-13T00:00:32.093 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.094+0000 7fb8f2ce0640 1 < completed part key=_multipart_H4M1TCN5W6XU.2~UBCXPMN7I462.2 size=923637 2024-09-13T00:00:32.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:31 smithi181 bash[67587]: audit 2024-09-13T00:00:30.684114+0000 mon.a (mon.0) 2067 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-110696-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:00:32.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:31 smithi181 bash[67587]: cluster 2024-09-13T00:00:30.684626+0000 mon.a (mon.0) 2068 : cluster [DBG] osdmap e1114: 8 total, 8 up, 8 in 2024-09-13T00:00:32.099 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.098+0000 7fb8f2ce0640 1 < completed multipart upload key=H4M1TCN5W6XU upload=UBCXPMN7I462 tag=4O9B5B3AM7KK type=write size=4069365 2024-09-13T00:00:32.102 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.102+0000 7fb8f2ce0640 1 > prepared operation key=H9AU4HD247YU tag=LAH78YJRB1TA type=del size=2449249 2024-09-13T00:00:32.106 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.106+0000 7fb8f2ce0640 1 < completed part key=_multipart_NX26SI4F2ZMB.2~GW108S4EOK7R.0 size=1048576 2024-09-13T00:00:32.109 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.106+0000 7fb8f2ce0640 1 < completed part key=_multipart_NX26SI4F2ZMB.2~GW108S4EOK7R.1 size=1048576 2024-09-13T00:00:32.111 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.110+0000 7fb8f2ce0640 1 < completed part key=_multipart_NX26SI4F2ZMB.2~GW108S4EOK7R.2 size=983109 2024-09-13T00:00:32.117 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.118+0000 7fb8f2ce0640 1 < completed multipart upload key=NX26SI4F2ZMB upload=GW108S4EOK7R tag=I40XDT260391 type=write size=4128837 2024-09-13T00:00:32.123 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.122+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EUE3LT0PDIXV.2~2UVNR7B3L4P9.0 size=633906 2024-09-13T00:00:32.123 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.122+0000 7fb8f2ce0640 1 > prepared multipart upload key=EUE3LT0PDIXV upload=2UVNR7B3L4P9 tag=W8AQPIJ1AE1N type=write size=1682482 2024-09-13T00:00:32.127 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.126+0000 7fb8f2ce0640 1 < completed delete operation key=QMZREZ1QEDNJ tag=2WQ0X2USYMG6 type=del 2024-09-13T00:00:32.133 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.134+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~MYV82IQPQUPF.0 size=294549 2024-09-13T00:00:32.133 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.134+0000 7fb8f2ce0640 1 > prepared multipart upload key=FX0NCGLK0KPJ upload=MYV82IQPQUPF tag=HE3UQM1IG9AC type=write size=1343125 2024-09-13T00:00:32.137 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.138+0000 7fb8f2ce0640 1 < completed write operation key=63FJDWBO2T9S tag=NEUXV3M1P1JR type=write size=513535 2024-09-13T00:00:32.143 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.142+0000 7fb8f2ce0640 1 > prepared part key=_multipart_N3Z5XIG3TZRQ.2~M5N7D25CSQCB.0 size=1048576 2024-09-13T00:00:32.146 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.146+0000 7fb8f2ce0640 1 > prepared part key=_multipart_N3Z5XIG3TZRQ.2~M5N7D25CSQCB.1 size=516892 2024-09-13T00:00:32.146 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.146+0000 7fb8f2ce0640 1 > prepared multipart upload key=N3Z5XIG3TZRQ upload=M5N7D25CSQCB tag=RAWXYKAZNVV1 type=write size=2614044 2024-09-13T00:00:32.147 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.146+0000 7fb8f2ce0640 -1 < no bi entry to suggest for operation key=2AHYRZUDFHP4 tag=FD2SOS3BXDP5 type=del size=2356102: (2) No such file or directory 2024-09-13T00:00:32.151 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.150+0000 7fb8f2ce0640 1 > prepared operation key=H9AU4HD247YU tag=BRMG8SWXPSA1 type=del size=3812635 2024-09-13T00:00:32.155 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.154+0000 7fb8f2ce0640 1 < completed part key=_multipart_F0QCVR9E08WE.2~7FOIGLWXBPIX.0 size=216994 2024-09-13T00:00:32.161 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.158+0000 7fb8f2ce0640 1 < completed multipart upload key=F0QCVR9E08WE upload=7FOIGLWXBPIX tag=W609F8EBKFTH type=write size=1265570 2024-09-13T00:00:32.165 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.162+0000 7fb8f2ce0640 1 > prepared operation key=1PX7DBO9M3YI tag=2J6RO3X16DTS type=write size=33991 2024-09-13T00:00:32.168 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.166+0000 7fb8f2ce0640 1 < completed part key=_multipart_H4M1TCN5W6XU.2~3WGK6N72UXFS.0 size=1048576 2024-09-13T00:00:32.171 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.170+0000 7fb8f2ce0640 1 < completed part key=_multipart_H4M1TCN5W6XU.2~3WGK6N72UXFS.1 size=1048576 2024-09-13T00:00:32.174 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.174+0000 7fb8f2ce0640 1 < completed part key=_multipart_H4M1TCN5W6XU.2~3WGK6N72UXFS.2 size=858629 2024-09-13T00:00:32.180 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.178+0000 7fb8f2ce0640 1 < completed multipart upload key=H4M1TCN5W6XU upload=3WGK6N72UXFS tag=1MC7F1UKLFVT type=write size=4004357 2024-09-13T00:00:32.184 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.182+0000 7fb8f2ce0640 1 > prepared operation key=F0QCVR9E08WE tag=8SDMJYPUNNG1 type=del size=2728827 2024-09-13T00:00:32.187 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.186+0000 7fb8f2ce0640 1 < completed delete operation key=H9AU4HD247YU tag=XY7J4RFS1HA5 type=del 2024-09-13T00:00:32.191 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.190+0000 7fb8f2ce0640 1 > prepared operation key=7OX5OJZ8GXU5 tag=WBSB397YVDW7 type=del size=1301487 2024-09-13T00:00:32.192 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.190+0000 7fb8f2ce0640 1 < canceled part key=_multipart_70F7D20HI504.2~9H9RW08EBOX0.0 size=1048576 2024-09-13T00:00:32.193 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.190+0000 7fb8f2ce0640 1 < canceled part key=_multipart_70F7D20HI504.2~9H9RW08EBOX0.1 size=346477 2024-09-13T00:00:32.198 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.198+0000 7fb8f2ce0640 1 < canceled multipart upload key=70F7D20HI504 upload=9H9RW08EBOX0 tag=LDXE6ZXLD2UE type=write size=2443629 2024-09-13T00:00:32.202 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.202+0000 7fb8f2ce0640 1 > prepared operation key=EJN6HR8I7XQF tag=BP7ZTO9P6V4G type=del size=3781623 2024-09-13T00:00:32.206 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.206+0000 7fb8f2ce0640 1 < completed part key=_multipart_70F7D20HI504.2~0EQZSLX14Q5T.0 size=1048576 2024-09-13T00:00:32.209 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.210+0000 7fb8f2ce0640 1 < completed part key=_multipart_70F7D20HI504.2~0EQZSLX14Q5T.1 size=649825 2024-09-13T00:00:32.215 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.214+0000 7fb8f2ce0640 1 < completed multipart upload key=70F7D20HI504 upload=0EQZSLX14Q5T tag=IPX61WJUZGI0 type=write size=2746977 2024-09-13T00:00:32.219 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.218+0000 7fb8f2ce0640 1 > prepared operation key=N3Z5XIG3TZRQ tag=HSX33KXNY4RH type=del size=1146953 2024-09-13T00:00:32.223 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.222+0000 7fb8f2ce0640 1 < completed part key=_multipart_7OX5OJZ8GXU5.2~S5XEHMSP8GPF.0 size=1048576 2024-09-13T00:00:32.227 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.226+0000 7fb8f2ce0640 1 < completed part key=_multipart_7OX5OJZ8GXU5.2~S5XEHMSP8GPF.1 size=377851 2024-09-13T00:00:32.233 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.234+0000 7fb8f2ce0640 1 < completed multipart upload key=7OX5OJZ8GXU5 upload=S5XEHMSP8GPF tag=PR3RI2BXBGA4 type=write size=2475003 2024-09-13T00:00:32.239 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.238+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~JXQCDEUCWAXC.0 size=1048576 2024-09-13T00:00:32.242 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.242+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~JXQCDEUCWAXC.1 size=1048576 2024-09-13T00:00:32.245 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.242+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~JXQCDEUCWAXC.2 size=203204 2024-09-13T00:00:32.245 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.242+0000 7fb8f2ce0640 1 > prepared multipart upload key=2AHYRZUDFHP4 upload=JXQCDEUCWAXC tag=AX66EYRSI0TM type=write size=3348932 2024-09-13T00:00:32.252 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.250+0000 7fb8f2ce0640 1 < suggested remove operation key=85YEW8N43FTA tag=BJGV3RLZ0W0W type=write 2024-09-13T00:00:32.256 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.254+0000 7fb8f2ce0640 1 > prepared operation key=5GM88ERM89KT tag=39CV84J685JA type=write size=257227 2024-09-13T00:00:32.260 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.258+0000 7fb8f2ce0640 1 < completed delete operation key=NQJQC8CEAC15 tag=4JWUARGP3J3E type=del 2024-09-13T00:00:32.266 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.266+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NX26SI4F2ZMB.2~597XWHLSNJ8N.0 size=323795 2024-09-13T00:00:32.266 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.266+0000 7fb8f2ce0640 1 > prepared multipart upload key=NX26SI4F2ZMB upload=597XWHLSNJ8N tag=CQX70JGO7KUK type=write size=1372371 2024-09-13T00:00:32.270 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.270+0000 7fb8f2ce0640 1 < completed write operation key=P8OYS5ZE1NX0 tag=217I2D3UD9FG type=write size=992259 2024-09-13T00:00:32.277 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.278+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZFHN2SNTZ5W7.2~6M0SHEPF6X3U.0 size=157794 2024-09-13T00:00:32.277 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.278+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZFHN2SNTZ5W7 upload=6M0SHEPF6X3U tag=M6TF53VYL4IQ type=write size=1206370 2024-09-13T00:00:32.282 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.282+0000 7fb8f2ce0640 1 < completed delete operation key=1PX7DBO9M3YI tag=4BWJU4DYLNT0 type=del 2024-09-13T00:00:32.286 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.286+0000 7fb8f2ce0640 1 > prepared operation key=H5EIP0E9CBWX tag=5YAQ8HEI3EV7 type=del size=816056 2024-09-13T00:00:32.291 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.290+0000 7fb8f2ce0640 1 < completed delete operation key=N3Z5XIG3TZRQ tag=IWZGO4X056UB type=del 2024-09-13T00:00:32.295 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.294+0000 7fb8f2ce0640 1 > prepared operation key=EJN6HR8I7XQF tag=ILXKKPVGPQLR type=del size=1160205 2024-09-13T00:00:32.299 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.298+0000 7fb8f2ce0640 1 < completed delete operation key=H9AU4HD247YU tag=LAH78YJRB1TA type=del 2024-09-13T00:00:32.305 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.306+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~P7MY0JQ41J7R.0 size=1048576 2024-09-13T00:00:32.308 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.306+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~P7MY0JQ41J7R.1 size=790606 2024-09-13T00:00:32.308 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.306+0000 7fb8f2ce0640 1 > prepared multipart upload key=W8XZ26311RJX upload=P7MY0JQ41J7R tag=QRSB8E46HYOS type=write size=2887758 2024-09-13T00:00:32.312 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.310+0000 7fb8f2ce0640 1 < completed part key=_multipart_EUE3LT0PDIXV.2~2UVNR7B3L4P9.0 size=633906 2024-09-13T00:00:32.317 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.318+0000 7fb8f2ce0640 1 < completed multipart upload key=EUE3LT0PDIXV upload=2UVNR7B3L4P9 tag=W8AQPIJ1AE1N type=write size=1682482 2024-09-13T00:00:32.325 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.326+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EJN6HR8I7XQF.2~6VOSXQ27PDUL.0 size=1048576 2024-09-13T00:00:32.328 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.326+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EJN6HR8I7XQF.2~6VOSXQ27PDUL.1 size=1048576 2024-09-13T00:00:32.330 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.330+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EJN6HR8I7XQF.2~6VOSXQ27PDUL.2 size=370402 2024-09-13T00:00:32.330 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.330+0000 7fb8f2ce0640 1 > prepared multipart upload key=EJN6HR8I7XQF upload=6VOSXQ27PDUL tag=CK2Z0E64JZ4N type=write size=3516130 2024-09-13T00:00:32.334 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.334+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~MYV82IQPQUPF.0 size=294549 2024-09-13T00:00:32.339 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.338+0000 7fb8f2ce0640 1 < completed multipart upload key=FX0NCGLK0KPJ upload=MYV82IQPQUPF tag=HE3UQM1IG9AC type=write size=1343125 2024-09-13T00:00:32.346 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.346+0000 7fb8f2ce0640 1 > prepared part key=_multipart_F0QCVR9E08WE.2~3058E2KUARXO.0 size=1048576 2024-09-13T00:00:32.349 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.350+0000 7fb8f2ce0640 1 > prepared part key=_multipart_F0QCVR9E08WE.2~3058E2KUARXO.1 size=1041435 2024-09-13T00:00:32.349 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.350+0000 7fb8f2ce0640 1 > prepared multipart upload key=F0QCVR9E08WE upload=3058E2KUARXO tag=ZXUYBKZ3DG39 type=write size=3138587 2024-09-13T00:00:32.350 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.350+0000 7fb8f2ce0640 1 < canceled part key=_multipart_N3Z5XIG3TZRQ.2~M5N7D25CSQCB.0 size=1048576 2024-09-13T00:00:32.354 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.354+0000 7fb8f2ce0640 1 < completed part key=_multipart_N3Z5XIG3TZRQ.2~M5N7D25CSQCB.1 size=516892 2024-09-13T00:00:32.360 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.358+0000 7fb8f2ce0640 1 < canceled multipart upload key=N3Z5XIG3TZRQ upload=M5N7D25CSQCB tag=RAWXYKAZNVV1 type=write size=2614044 2024-09-13T00:00:32.363 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.362+0000 7fb8f2ce0640 1 > prepared operation key=ZTK9AYBSLCOZ tag=PX7J3SYMWEOU type=del size=3007176 2024-09-13T00:00:32.367 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.366+0000 7fb8f2ce0640 1 < completed delete operation key=H9AU4HD247YU tag=BRMG8SWXPSA1 type=del 2024-09-13T00:00:32.375 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.374+0000 7fb8f2ce0640 1 > prepared part key=_multipart_O4MZ4UWNXME9.2~2EMJJZAJEU2E.0 size=1048576 2024-09-13T00:00:32.378 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.378+0000 7fb8f2ce0640 1 > prepared part key=_multipart_O4MZ4UWNXME9.2~2EMJJZAJEU2E.1 size=557796 2024-09-13T00:00:32.378 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.378+0000 7fb8f2ce0640 1 > prepared multipart upload key=O4MZ4UWNXME9 upload=2EMJJZAJEU2E tag=0K2JPOCX1N81 type=write size=2654948 2024-09-13T00:00:32.382 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.382+0000 7fb8f2ce0640 1 < completed write operation key=1PX7DBO9M3YI tag=2J6RO3X16DTS type=write size=33991 2024-09-13T00:00:32.390 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.390+0000 7fb8f2ce0640 1 > prepared part key=_multipart_44EDYFT9VF8Y.2~DPPW4FUXIK8U.0 size=884390 2024-09-13T00:00:32.390 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.390+0000 7fb8f2ce0640 1 > prepared multipart upload key=44EDYFT9VF8Y upload=DPPW4FUXIK8U tag=16NRDE1CJFM3 type=write size=1932966 2024-09-13T00:00:32.393 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.394+0000 7fb8f2ce0640 1 < completed delete operation key=F0QCVR9E08WE tag=8SDMJYPUNNG1 type=del 2024-09-13T00:00:32.397 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.398+0000 7fb8f2ce0640 1 > prepared operation key=1HX06CCZJ1DD tag=2IYE31LVM6RF type=del size=2503346 2024-09-13T00:00:32.401 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.402+0000 7fb8f2ce0640 1 < completed delete operation key=7OX5OJZ8GXU5 tag=WBSB397YVDW7 type=del 2024-09-13T00:00:32.405 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.406+0000 7fb8f2ce0640 1 > prepared operation key=70F7D20HI504 tag=6USMA7S73VKW type=del size=3382512 2024-09-13T00:00:32.408 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.406+0000 7fb8f2ce0640 1 < completed delete operation key=EJN6HR8I7XQF tag=BP7ZTO9P6V4G type=del 2024-09-13T00:00:32.412 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.410+0000 7fb8f2ce0640 1 > prepared operation key=NX26SI4F2ZMB tag=MH41LMBF095T type=del size=3878841 2024-09-13T00:00:32.415 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.414+0000 7fb8f2ce0640 1 < completed delete operation key=N3Z5XIG3TZRQ tag=HSX33KXNY4RH type=del 2024-09-13T00:00:32.422 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.422+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EJN6HR8I7XQF.2~FL2KBNK1EKPK.0 size=1048576 2024-09-13T00:00:32.425 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.426+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EJN6HR8I7XQF.2~FL2KBNK1EKPK.1 size=1048576 2024-09-13T00:00:32.429 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.426+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EJN6HR8I7XQF.2~FL2KBNK1EKPK.2 size=763091 2024-09-13T00:00:32.429 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.426+0000 7fb8f2ce0640 1 > prepared multipart upload key=EJN6HR8I7XQF upload=FL2KBNK1EKPK tag=XDVLUG10W29H type=write size=3908819 2024-09-13T00:00:32.432 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.430+0000 7fb8f2ce0640 1 < completed part key=_multipart_2AHYRZUDFHP4.2~JXQCDEUCWAXC.0 size=1048576 2024-09-13T00:00:32.435 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.434+0000 7fb8f2ce0640 1 < completed part key=_multipart_2AHYRZUDFHP4.2~JXQCDEUCWAXC.1 size=1048576 2024-09-13T00:00:32.438 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.438+0000 7fb8f2ce0640 1 < completed part key=_multipart_2AHYRZUDFHP4.2~JXQCDEUCWAXC.2 size=203204 2024-09-13T00:00:32.444 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.442+0000 7fb8f2ce0640 1 < completed multipart upload key=2AHYRZUDFHP4 upload=JXQCDEUCWAXC tag=AX66EYRSI0TM type=write size=3348932 2024-09-13T00:00:32.448 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.446+0000 7fb8f2ce0640 1 > prepared operation key=2TWD9YLMSYJI tag=2ULCR2YDHN65 type=del size=3634270 2024-09-13T00:00:32.455 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.454+0000 7fb8f2ce0640 1 < completed write operation key=5GM88ERM89KT tag=39CV84J685JA type=write size=257227 2024-09-13T00:00:32.458 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.458+0000 7fb8f2ce0640 1 > prepared operation key=QMZREZ1QEDNJ tag=DA97F5MQB94X type=del size=3409749 2024-09-13T00:00:32.462 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.462+0000 7fb8f2ce0640 1 < completed part key=_multipart_NX26SI4F2ZMB.2~597XWHLSNJ8N.0 size=323795 2024-09-13T00:00:32.468 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.466+0000 7fb8f2ce0640 1 < completed multipart upload key=NX26SI4F2ZMB upload=597XWHLSNJ8N tag=CQX70JGO7KUK type=write size=1372371 2024-09-13T00:00:32.474 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.474+0000 7fb8f2ce0640 1 > prepared part key=_multipart_0WC6P8YNL0H2.2~3R32RCAQPQWC.0 size=503230 2024-09-13T00:00:32.474 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.474+0000 7fb8f2ce0640 1 > prepared multipart upload key=0WC6P8YNL0H2 upload=3R32RCAQPQWC tag=CMG9AZB71RRP type=write size=1551806 2024-09-13T00:00:32.475 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.474+0000 7fb8f2ce0640 1 < canceled part key=_multipart_ZFHN2SNTZ5W7.2~6M0SHEPF6X3U.0 size=157794 2024-09-13T00:00:32.480 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.478+0000 7fb8f2ce0640 1 < canceled multipart upload key=ZFHN2SNTZ5W7 upload=6M0SHEPF6X3U tag=M6TF53VYL4IQ type=write size=1206370 2024-09-13T00:00:32.484 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.482+0000 7fb8f2ce0640 1 > prepared operation key=J8NGDSF9XRTP tag=IJ31WLRJ36TK type=del size=1737620 2024-09-13T00:00:32.491 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.490+0000 7fb8f2ce0640 1 < suggested remove operation key=H5EIP0E9CBWX tag=5YAQ8HEI3EV7 type=del 2024-09-13T00:00:32.497 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.498+0000 7fb8f2ce0640 1 > prepared part key=_multipart_85YEW8N43FTA.2~G7ZSEEXHY0EO.0 size=718724 2024-09-13T00:00:32.497 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.498+0000 7fb8f2ce0640 1 > prepared multipart upload key=85YEW8N43FTA upload=G7ZSEEXHY0EO tag=QWR5KLQHZJE1 type=write size=1767300 2024-09-13T00:00:32.501 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.502+0000 7fb8f2ce0640 1 < completed delete operation key=EJN6HR8I7XQF tag=ILXKKPVGPQLR type=del 2024-09-13T00:00:32.508 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.506+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~RTHLOEZ83ZMJ.0 size=1048576 2024-09-13T00:00:32.510 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.510+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~RTHLOEZ83ZMJ.1 size=932776 2024-09-13T00:00:32.511 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.510+0000 7fb8f2ce0640 1 > prepared multipart upload key=1HX06CCZJ1DD upload=RTHLOEZ83ZMJ tag=9PBBY94G7DKR type=write size=3029928 2024-09-13T00:00:32.515 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.514+0000 7fb8f2ce0640 1 < completed part key=_multipart_W8XZ26311RJX.2~P7MY0JQ41J7R.0 size=1048576 2024-09-13T00:00:32.515 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.514+0000 7fb8f2ce0640 1 < canceled part key=_multipart_W8XZ26311RJX.2~P7MY0JQ41J7R.1 size=790606 2024-09-13T00:00:32.520 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.518+0000 7fb8f2ce0640 1 < canceled multipart upload key=W8XZ26311RJX upload=P7MY0JQ41J7R tag=QRSB8E46HYOS type=write size=2887758 2024-09-13T00:00:32.524 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.522+0000 7fb8f2ce0640 1 > prepared operation key=ZFHN2SNTZ5W7 tag=X2C0CKXD2M05 type=del size=1857390 2024-09-13T00:00:32.528 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.526+0000 7fb8f2ce0640 1 < completed part key=_multipart_EJN6HR8I7XQF.2~6VOSXQ27PDUL.0 size=1048576 2024-09-13T00:00:32.531 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.530+0000 7fb8f2ce0640 1 < completed part key=_multipart_EJN6HR8I7XQF.2~6VOSXQ27PDUL.1 size=1048576 2024-09-13T00:00:32.534 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.534+0000 7fb8f2ce0640 1 < completed part key=_multipart_EJN6HR8I7XQF.2~6VOSXQ27PDUL.2 size=370402 2024-09-13T00:00:32.540 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.538+0000 7fb8f2ce0640 1 < completed multipart upload key=EJN6HR8I7XQF upload=6VOSXQ27PDUL tag=CK2Z0E64JZ4N type=write size=3516130 2024-09-13T00:00:32.544 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.542+0000 7fb8f2ce0640 1 > prepared operation key=H5EIP0E9CBWX tag=L7P3CK05LVQH type=del size=3257113 2024-09-13T00:00:32.545 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.542+0000 7fb8f2ce0640 1 < canceled part key=_multipart_F0QCVR9E08WE.2~3058E2KUARXO.0 size=1048576 2024-09-13T00:00:32.548 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.546+0000 7fb8f2ce0640 1 < completed part key=_multipart_F0QCVR9E08WE.2~3058E2KUARXO.1 size=1041435 2024-09-13T00:00:32.553 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.554+0000 7fb8f2ce0640 1 < canceled multipart upload key=F0QCVR9E08WE upload=3058E2KUARXO tag=ZXUYBKZ3DG39 type=write size=3138587 2024-09-13T00:00:32.561 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.562+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~1WCDEW361W5K.0 size=31198 2024-09-13T00:00:32.561 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.562+0000 7fb8f2ce0640 1 > prepared multipart upload key=NQJQC8CEAC15 upload=1WCDEW361W5K tag=JKLWOEN8UBMO type=write size=1079774 2024-09-13T00:00:32.564 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.562+0000 7fb8f2ce0640 1 < completed delete operation key=ZTK9AYBSLCOZ tag=PX7J3SYMWEOU type=del 2024-09-13T00:00:32.568 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.566+0000 7fb8f2ce0640 1 > prepared operation key=PVYZQLGYT3XU tag=6UW2NR96AS9P type=del size=1743296 2024-09-13T00:00:32.572 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.570+0000 7fb8f2ce0640 1 < completed part key=_multipart_O4MZ4UWNXME9.2~2EMJJZAJEU2E.0 size=1048576 2024-09-13T00:00:32.574 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.574+0000 7fb8f2ce0640 1 < completed part key=_multipart_O4MZ4UWNXME9.2~2EMJJZAJEU2E.1 size=557796 2024-09-13T00:00:32.581 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.578+0000 7fb8f2ce0640 1 < completed multipart upload key=O4MZ4UWNXME9 upload=2EMJJZAJEU2E tag=0K2JPOCX1N81 type=write size=2654948 2024-09-13T00:00:32.585 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.582+0000 7fb8f2ce0640 1 > prepared operation key=44EDYFT9VF8Y tag=1LXDYJUX158J type=write size=78334 2024-09-13T00:00:32.588 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.586+0000 7fb8f2ce0640 1 < completed part key=_multipart_44EDYFT9VF8Y.2~DPPW4FUXIK8U.0 size=884390 2024-09-13T00:00:32.593 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.594+0000 7fb8f2ce0640 1 < completed multipart upload key=44EDYFT9VF8Y upload=DPPW4FUXIK8U tag=16NRDE1CJFM3 type=write size=1932966 2024-09-13T00:00:32.597 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.598+0000 7fb8f2ce0640 1 > prepared operation key=J8NGDSF9XRTP tag=0AG9CVVDQHH9 type=del size=2593040 2024-09-13T00:00:32.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:00:32 smithi181 bash[95505]: ts=2024-09-13T00:00:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:00:32.603 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.602+0000 7fb8f2ce0640 1 < suggested remove operation key=1HX06CCZJ1DD tag=2IYE31LVM6RF type=del 2024-09-13T00:00:32.607 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.606+0000 7fb8f2ce0640 1 > prepared operation key=44EDYFT9VF8Y tag=0QD4WIC3A9NF type=del size=3645072 2024-09-13T00:00:32.611 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.610+0000 7fb8f2ce0640 1 < completed delete operation key=70F7D20HI504 tag=6USMA7S73VKW type=del 2024-09-13T00:00:32.615 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.614+0000 7fb8f2ce0640 1 > prepared operation key=2TWD9YLMSYJI tag=0PZJW9CG2Z98 type=del size=3557979 2024-09-13T00:00:32.618 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.618+0000 7fb8f2ce0640 1 < completed delete operation key=NX26SI4F2ZMB tag=MH41LMBF095T type=del 2024-09-13T00:00:32.625 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.622+0000 7fb8f2ce0640 1 > prepared part key=_multipart_44EDYFT9VF8Y.2~71GFZ093AQBF.0 size=1048576 2024-09-13T00:00:32.627 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.626+0000 7fb8f2ce0640 1 > prepared part key=_multipart_44EDYFT9VF8Y.2~71GFZ093AQBF.1 size=925201 2024-09-13T00:00:32.627 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.626+0000 7fb8f2ce0640 1 > prepared multipart upload key=44EDYFT9VF8Y upload=71GFZ093AQBF tag=1MFGQWUFYSRW type=write size=3022353 2024-09-13T00:00:32.631 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.630+0000 7fb8f2ce0640 1 < completed part key=_multipart_EJN6HR8I7XQF.2~FL2KBNK1EKPK.0 size=1048576 2024-09-13T00:00:32.634 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.634+0000 7fb8f2ce0640 1 < completed part key=_multipart_EJN6HR8I7XQF.2~FL2KBNK1EKPK.1 size=1048576 2024-09-13T00:00:32.637 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.638+0000 7fb8f2ce0640 1 < completed part key=_multipart_EJN6HR8I7XQF.2~FL2KBNK1EKPK.2 size=763091 2024-09-13T00:00:32.643 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.642+0000 7fb8f2ce0640 1 < completed multipart upload key=EJN6HR8I7XQF upload=FL2KBNK1EKPK tag=XDVLUG10W29H type=write size=3908819 2024-09-13T00:00:32.650 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.650+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~E30T6RTT8KXL.0 size=949328 2024-09-13T00:00:32.650 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.650+0000 7fb8f2ce0640 1 > prepared multipart upload key=PVYZQLGYT3XU upload=E30T6RTT8KXL tag=7I7H6G8A9F28 type=write size=1997904 2024-09-13T00:00:32.658 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.658+0000 7fb8f2ce0640 1 < suggested update operation key=2TWD9YLMSYJI tag=2ULCR2YDHN65 type=del size=3634270 2024-09-13T00:00:32.663 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.662+0000 7fb8f2ce0640 1 > prepared part key=_multipart_F0QCVR9E08WE.2~98GBFYCNPOS9.0 size=1048576 2024-09-13T00:00:32.666 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.666+0000 7fb8f2ce0640 1 > prepared part key=_multipart_F0QCVR9E08WE.2~98GBFYCNPOS9.1 size=680388 2024-09-13T00:00:32.666 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.666+0000 7fb8f2ce0640 1 > prepared multipart upload key=F0QCVR9E08WE upload=98GBFYCNPOS9 tag=BT8A64LJM7J2 type=write size=2777540 2024-09-13T00:00:32.674 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.674+0000 7fb8f2ce0640 1 < suggested update operation key=QMZREZ1QEDNJ tag=DA97F5MQB94X type=del size=3409749 2024-09-13T00:00:32.678 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.678+0000 7fb8f2ce0640 1 > prepared operation key=7OX5OJZ8GXU5 tag=TNCK16J06BP1 type=del size=3237685 2024-09-13T00:00:32.682 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.682+0000 7fb8f2ce0640 1 < completed part key=_multipart_0WC6P8YNL0H2.2~3R32RCAQPQWC.0 size=503230 2024-09-13T00:00:32.687 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.686+0000 7fb8f2ce0640 1 < completed multipart upload key=0WC6P8YNL0H2 upload=3R32RCAQPQWC tag=CMG9AZB71RRP type=write size=1551806 2024-09-13T00:00:32.692 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.690+0000 7fb8f2ce0640 1 > prepared operation key=70F7D20HI504 tag=0F3IO07NARYZ type=del size=3137511 2024-09-13T00:00:32.695 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.694+0000 7fb8f2ce0640 1 < completed delete operation key=J8NGDSF9XRTP tag=IJ31WLRJ36TK type=del 2024-09-13T00:00:32.699 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.698+0000 7fb8f2ce0640 1 > prepared operation key=7OX5OJZ8GXU5 tag=P5Q5Z1XB202U type=del size=413523 2024-09-13T00:00:32.702 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.702+0000 7fb8f2ce0640 1 < completed part key=_multipart_85YEW8N43FTA.2~G7ZSEEXHY0EO.0 size=718724 2024-09-13T00:00:32.708 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.706+0000 7fb8f2ce0640 1 < completed multipart upload key=85YEW8N43FTA upload=G7ZSEEXHY0EO tag=QWR5KLQHZJE1 type=write size=1767300 2024-09-13T00:00:32.714 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.714+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EUE3LT0PDIXV.2~2I24CX5S70HL.0 size=1048576 2024-09-13T00:00:32.717 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.718+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EUE3LT0PDIXV.2~2I24CX5S70HL.1 size=588735 2024-09-13T00:00:32.717 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.718+0000 7fb8f2ce0640 1 > prepared multipart upload key=EUE3LT0PDIXV upload=2I24CX5S70HL tag=R035J63GCW0O type=write size=2685887 2024-09-13T00:00:32.721 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.722+0000 7fb8f2ce0640 1 < completed part key=_multipart_1HX06CCZJ1DD.2~RTHLOEZ83ZMJ.0 size=1048576 2024-09-13T00:00:32.724 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.722+0000 7fb8f2ce0640 1 < completed part key=_multipart_1HX06CCZJ1DD.2~RTHLOEZ83ZMJ.1 size=932776 2024-09-13T00:00:32.730 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.730+0000 7fb8f2ce0640 1 < completed multipart upload key=1HX06CCZJ1DD upload=RTHLOEZ83ZMJ tag=9PBBY94G7DKR type=write size=3029928 2024-09-13T00:00:32.738 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.738+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~4QBLR4DCEB8H.0 size=443062 2024-09-13T00:00:32.738 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.738+0000 7fb8f2ce0640 1 > prepared multipart upload key=1HX06CCZJ1DD upload=4QBLR4DCEB8H tag=CX60H3B1W6I6 type=write size=1491638 2024-09-13T00:00:32.740 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.738+0000 7fb8f2ce0640 1 < completed delete operation key=ZFHN2SNTZ5W7 tag=X2C0CKXD2M05 type=del 2024-09-13T00:00:32.744 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.742+0000 7fb8f2ce0640 1 > prepared operation key=PVYZQLGYT3XU tag=VO3WS5SAO5V1 type=write size=633598 2024-09-13T00:00:32.748 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.746+0000 7fb8f2ce0640 1 < completed delete operation key=H5EIP0E9CBWX tag=L7P3CK05LVQH type=del 2024-09-13T00:00:32.752 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.750+0000 7fb8f2ce0640 1 > prepared operation key=8MPUVD37EDN9 tag=KGFD1YS7M5ZE type=write size=50011 2024-09-13T00:00:32.753 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.750+0000 7fb8f2ce0640 1 < canceled part key=_multipart_NQJQC8CEAC15.2~1WCDEW361W5K.0 size=31198 2024-09-13T00:00:32.758 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.758+0000 7fb8f2ce0640 1 < canceled multipart upload key=NQJQC8CEAC15 upload=1WCDEW361W5K tag=JKLWOEN8UBMO type=write size=1079774 2024-09-13T00:00:32.765 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.766+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~1U1Y1I82UOP1.0 size=1048576 2024-09-13T00:00:32.768 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.766+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~1U1Y1I82UOP1.1 size=1037170 2024-09-13T00:00:32.768 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.766+0000 7fb8f2ce0640 1 > prepared multipart upload key=1PX7DBO9M3YI upload=1U1Y1I82UOP1 tag=RRM8QK94EUK2 type=write size=3134322 2024-09-13T00:00:32.772 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.770+0000 7fb8f2ce0640 1 < canceled operation key=PVYZQLGYT3XU tag=6UW2NR96AS9P type=del size=1743296 2024-09-13T00:00:32.775 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.774+0000 7fb8f2ce0640 1 > prepared operation key=H9AU4HD247YU tag=7OK05ZA4CRO9 type=write size=386561 2024-09-13T00:00:32.779 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.778+0000 7fb8f2ce0640 1 < completed write operation key=44EDYFT9VF8Y tag=1LXDYJUX158J type=write size=78334 2024-09-13T00:00:32.785 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.786+0000 7fb8f2ce0640 1 > prepared part key=_multipart_4K76U69GZ9AE.2~K9U8G79D34FG.0 size=1048576 2024-09-13T00:00:32.788 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.786+0000 7fb8f2ce0640 1 > prepared part key=_multipart_4K76U69GZ9AE.2~K9U8G79D34FG.1 size=1048576 2024-09-13T00:00:32.791 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.790+0000 7fb8f2ce0640 1 > prepared part key=_multipart_4K76U69GZ9AE.2~K9U8G79D34FG.2 size=688329 2024-09-13T00:00:32.791 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.790+0000 7fb8f2ce0640 1 > prepared multipart upload key=4K76U69GZ9AE upload=K9U8G79D34FG tag=CYSNDS7CJGRL type=write size=3834057 2024-09-13T00:00:32.795 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.794+0000 7fb8f2ce0640 1 < completed delete operation key=J8NGDSF9XRTP tag=0AG9CVVDQHH9 type=del 2024-09-13T00:00:32.801 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.802+0000 7fb8f2ce0640 1 > prepared part key=_multipart_44EDYFT9VF8Y.2~31OS6ZT8QQJW.0 size=219344 2024-09-13T00:00:32.802 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.802+0000 7fb8f2ce0640 1 > prepared multipart upload key=44EDYFT9VF8Y upload=31OS6ZT8QQJW tag=CK18K9Q4CLGD type=write size=1267920 2024-09-13T00:00:32.805 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.806+0000 7fb8f2ce0640 1 < completed delete operation key=44EDYFT9VF8Y tag=0QD4WIC3A9NF type=del 2024-09-13T00:00:32.809 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.810+0000 7fb8f2ce0640 1 > prepared operation key=8MPUVD37EDN9 tag=PZXT9EANKX0D type=del size=291141 2024-09-13T00:00:32.813 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.814+0000 7fb8f2ce0640 -1 < failed to complete operation key=2TWD9YLMSYJI tag=0PZJW9CG2Z98 type=del size=3557979: (22) Invalid argument 2024-09-13T00:00:32.817 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.818+0000 7fb8f2ce0640 1 > prepared operation key=NQJQC8CEAC15 tag=KYG2POC2X75D type=del size=2435414 2024-09-13T00:00:32.820 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.818+0000 7fb8f2ce0640 1 < completed part key=_multipart_44EDYFT9VF8Y.2~71GFZ093AQBF.0 size=1048576 2024-09-13T00:00:32.823 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.822+0000 7fb8f2ce0640 1 < completed part key=_multipart_44EDYFT9VF8Y.2~71GFZ093AQBF.1 size=925201 2024-09-13T00:00:32.829 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.830+0000 7fb8f2ce0640 1 < completed multipart upload key=44EDYFT9VF8Y upload=71GFZ093AQBF tag=1MFGQWUFYSRW type=write size=3022353 2024-09-13T00:00:32.835 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.834+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~9B5QZ2RMJ4YC.0 size=1048576 2024-09-13T00:00:32.838 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.838+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~9B5QZ2RMJ4YC.1 size=610288 2024-09-13T00:00:32.838 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.838+0000 7fb8f2ce0640 1 > prepared multipart upload key=7OX5OJZ8GXU5 upload=9B5QZ2RMJ4YC tag=151FE1XCJDQP type=write size=2707440 2024-09-13T00:00:32.843 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.842+0000 7fb8f2ce0640 1 < completed part key=_multipart_PVYZQLGYT3XU.2~E30T6RTT8KXL.0 size=949328 2024-09-13T00:00:32.848 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.846+0000 7fb8f2ce0640 1 < completed multipart upload key=PVYZQLGYT3XU upload=E30T6RTT8KXL tag=7I7H6G8A9F28 type=write size=1997904 2024-09-13T00:00:32.852 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.850+0000 7fb8f2ce0640 1 > prepared operation key=5GM88ERM89KT tag=JUQ6SGCJIQXS type=del size=1563239 2024-09-13T00:00:32.856 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.854+0000 7fb8f2ce0640 1 < completed part key=_multipart_F0QCVR9E08WE.2~98GBFYCNPOS9.0 size=1048576 2024-09-13T00:00:32.858 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.858+0000 7fb8f2ce0640 1 < completed part key=_multipart_F0QCVR9E08WE.2~98GBFYCNPOS9.1 size=680388 2024-09-13T00:00:32.864 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.862+0000 7fb8f2ce0640 1 < completed multipart upload key=F0QCVR9E08WE upload=98GBFYCNPOS9 tag=BT8A64LJM7J2 type=write size=2777540 2024-09-13T00:00:32.868 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.866+0000 7fb8f2ce0640 1 > prepared operation key=1HX06CCZJ1DD tag=1HPB8ODCDIHD type=del size=1268536 2024-09-13T00:00:32.875 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.874+0000 7fb8f2ce0640 1 < suggested remove operation key=7OX5OJZ8GXU5 tag=TNCK16J06BP1 type=del 2024-09-13T00:00:32.879 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.878+0000 7fb8f2ce0640 1 > prepared operation key=H5EIP0E9CBWX tag=6S5FLJIV8C5F type=del size=1242848 2024-09-13T00:00:32.884 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.882+0000 7fb8f2ce0640 1 < completed delete operation key=70F7D20HI504 tag=0F3IO07NARYZ type=del 2024-09-13T00:00:32.887 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.886+0000 7fb8f2ce0640 1 > prepared operation key=NX26SI4F2ZMB tag=8KPPLUB50XFL type=del size=789727 2024-09-13T00:00:32.891 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.890+0000 7fb8f2ce0640 -1 < failed to complete operation key=7OX5OJZ8GXU5 tag=P5Q5Z1XB202U type=del size=413523: (22) Invalid argument 2024-09-13T00:00:32.895 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.894+0000 7fb8f2ce0640 1 > prepared operation key=EJN6HR8I7XQF tag=ZH99M1J033BS type=del size=861080 2024-09-13T00:00:32.898 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.898+0000 7fb8f2ce0640 1 < completed part key=_multipart_EUE3LT0PDIXV.2~2I24CX5S70HL.0 size=1048576 2024-09-13T00:00:32.901 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.902+0000 7fb8f2ce0640 1 < completed part key=_multipart_EUE3LT0PDIXV.2~2I24CX5S70HL.1 size=588735 2024-09-13T00:00:32.907 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.906+0000 7fb8f2ce0640 1 < completed multipart upload key=EUE3LT0PDIXV upload=2I24CX5S70HL tag=R035J63GCW0O type=write size=2685887 2024-09-13T00:00:32.915 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.914+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~OQHN87X308YK.0 size=4654 2024-09-13T00:00:32.915 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.914+0000 7fb8f2ce0640 1 > prepared multipart upload key=2AHYRZUDFHP4 upload=OQHN87X308YK tag=751GQF7WRBW7 type=write size=1053230 2024-09-13T00:00:32.918 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.918+0000 7fb8f2ce0640 1 < completed part key=_multipart_1HX06CCZJ1DD.2~4QBLR4DCEB8H.0 size=443062 2024-09-13T00:00:32.925 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.922+0000 7fb8f2ce0640 1 < completed multipart upload key=1HX06CCZJ1DD upload=4QBLR4DCEB8H tag=CX60H3B1W6I6 type=write size=1491638 2024-09-13T00:00:32.929 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.930+0000 7fb8f2ce0640 1 > prepared operation key=FX0NCGLK0KPJ tag=VM4FG5NTQ2AZ type=write size=985745 2024-09-13T00:00:32.937 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.938+0000 7fb8f2ce0640 1 < suggested remove operation key=PVYZQLGYT3XU tag=VO3WS5SAO5V1 type=write 2024-09-13T00:00:32.942 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.942+0000 7fb8f2ce0640 1 > prepared operation key=85YEW8N43FTA tag=EJ6IKNW2ZODK type=write size=1045762 2024-09-13T00:00:32.950 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.950+0000 7fb8f2ce0640 1 < suggested remove operation key=8MPUVD37EDN9 tag=KGFD1YS7M5ZE type=write 2024-09-13T00:00:32.957 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.958+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~OH3AW02D2EU1.0 size=972272 2024-09-13T00:00:32.957 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.958+0000 7fb8f2ce0640 1 > prepared multipart upload key=63FJDWBO2T9S upload=OH3AW02D2EU1 tag=7STUGZ9RU2H8 type=write size=2020848 2024-09-13T00:00:32.961 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.962+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~1U1Y1I82UOP1.0 size=1048576 2024-09-13T00:00:32.964 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.962+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~1U1Y1I82UOP1.1 size=1037170 2024-09-13T00:00:32.970 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.970+0000 7fb8f2ce0640 1 < completed multipart upload key=1PX7DBO9M3YI upload=1U1Y1I82UOP1 tag=RRM8QK94EUK2 type=write size=3134322 2024-09-13T00:00:32.974 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.974+0000 7fb8f2ce0640 1 > prepared operation key=F0QCVR9E08WE tag=BKTXSDNE5XGQ type=del size=2232262 2024-09-13T00:00:32.977 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.978+0000 7fb8f2ce0640 1 < completed write operation key=H9AU4HD247YU tag=7OK05ZA4CRO9 type=write size=386561 2024-09-13T00:00:32.981 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.982+0000 7fb8f2ce0640 1 > prepared operation key=W8XZ26311RJX tag=SOVJFX5ODZ5W type=del size=1826344 2024-09-13T00:00:32.982 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.982+0000 7fb8f2ce0640 1 < canceled part key=_multipart_4K76U69GZ9AE.2~K9U8G79D34FG.0 size=1048576 2024-09-13T00:00:32.985 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.986+0000 7fb8f2ce0640 1 < completed part key=_multipart_4K76U69GZ9AE.2~K9U8G79D34FG.1 size=1048576 2024-09-13T00:00:32.988 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.986+0000 7fb8f2ce0640 1 < completed part key=_multipart_4K76U69GZ9AE.2~K9U8G79D34FG.2 size=688329 2024-09-13T00:00:32.995 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.994+0000 7fb8f2ce0640 1 < canceled multipart upload key=4K76U69GZ9AE upload=K9U8G79D34FG tag=CYSNDS7CJGRL type=write size=3834057 2024-09-13T00:00:32.999 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:32.998+0000 7fb8f2ce0640 1 > prepared operation key=1HX06CCZJ1DD tag=VUKTH43SDQ4Y type=del size=1979152 2024-09-13T00:00:33.003 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.002+0000 7fb8f2ce0640 1 < completed part key=_multipart_44EDYFT9VF8Y.2~31OS6ZT8QQJW.0 size=219344 2024-09-13T00:00:33.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:32 smithi136 bash[75164]: cluster 2024-09-13T00:00:31.124486+0000 mgr.y (mgr.24661) 702 : cluster [DBG] pgmap v1377: 137 pgs: 6 creating+peering, 18 unknown, 113 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 6.5 KiB/s wr, 24 op/s 2024-09-13T00:00:33.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:32 smithi136 bash[75164]: cluster 2024-09-13T00:00:31.700435+0000 mon.a (mon.0) 2069 : cluster [DBG] osdmap e1115: 8 total, 8 up, 8 in 2024-09-13T00:00:33.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:32 smithi136 bash[76371]: cluster 2024-09-13T00:00:31.124486+0000 mgr.y (mgr.24661) 702 : cluster [DBG] pgmap v1377: 137 pgs: 6 creating+peering, 18 unknown, 113 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 6.5 KiB/s wr, 24 op/s 2024-09-13T00:00:33.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:32 smithi136 bash[76371]: cluster 2024-09-13T00:00:31.700435+0000 mon.a (mon.0) 2069 : cluster [DBG] osdmap e1115: 8 total, 8 up, 8 in 2024-09-13T00:00:33.011 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.010+0000 7fb8f2ce0640 1 < completed multipart upload key=44EDYFT9VF8Y upload=31OS6ZT8QQJW tag=CK18K9Q4CLGD type=write size=1267920 2024-09-13T00:00:33.016 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.014+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~G8PJFQZRQOZA.0 size=1048576 2024-09-13T00:00:33.019 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.018+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~G8PJFQZRQOZA.1 size=1048576 2024-09-13T00:00:33.021 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.022+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~G8PJFQZRQOZA.2 size=572990 2024-09-13T00:00:33.021 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.022+0000 7fb8f2ce0640 1 > prepared multipart upload key=7OX5OJZ8GXU5 upload=G8PJFQZRQOZA tag=TJ3F7V894UZZ type=write size=3718718 2024-09-13T00:00:33.022 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.022+0000 7fb8f2ce0640 -1 < no bi entry to suggest for operation key=8MPUVD37EDN9 tag=PZXT9EANKX0D type=del size=291141: (2) No such file or directory 2024-09-13T00:00:33.028 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.026+0000 7fb8f2ce0640 1 > prepared part key=_multipart_J8NGDSF9XRTP.2~0E4XZHWH35Y5.0 size=776614 2024-09-13T00:00:33.029 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.026+0000 7fb8f2ce0640 1 > prepared multipart upload key=J8NGDSF9XRTP upload=0E4XZHWH35Y5 tag=VD3AQQV7IMCS type=write size=1825190 2024-09-13T00:00:33.032 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.030+0000 7fb8f2ce0640 1 < completed delete operation key=NQJQC8CEAC15 tag=KYG2POC2X75D type=del 2024-09-13T00:00:33.035 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.034+0000 7fb8f2ce0640 1 > prepared operation key=F0QCVR9E08WE tag=RXKYB4L9I2DQ type=write size=783254 2024-09-13T00:00:33.039 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.038+0000 7fb8f2ce0640 1 < completed part key=_multipart_7OX5OJZ8GXU5.2~9B5QZ2RMJ4YC.0 size=1048576 2024-09-13T00:00:33.042 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.042+0000 7fb8f2ce0640 1 < completed part key=_multipart_7OX5OJZ8GXU5.2~9B5QZ2RMJ4YC.1 size=610288 2024-09-13T00:00:33.048 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.046+0000 7fb8f2ce0640 1 < completed multipart upload key=7OX5OJZ8GXU5 upload=9B5QZ2RMJ4YC tag=151FE1XCJDQP type=write size=2707440 2024-09-13T00:00:33.052 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.050+0000 7fb8f2ce0640 1 > prepared operation key=EJN6HR8I7XQF tag=M481OB4JCCXI type=del size=3796790 2024-09-13T00:00:33.057 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.058+0000 7fb8f2ce0640 1 < completed delete operation key=5GM88ERM89KT tag=JUQ6SGCJIQXS type=del 2024-09-13T00:00:33.064 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.062+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~374VMQVZ7K69.0 size=1015463 2024-09-13T00:00:33.064 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.062+0000 7fb8f2ce0640 1 > prepared multipart upload key=NQJQC8CEAC15 upload=374VMQVZ7K69 tag=GXK04XE3LNV4 type=write size=2064039 2024-09-13T00:00:33.067 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.066+0000 7fb8f2ce0640 1 < completed delete operation key=1HX06CCZJ1DD tag=1HPB8ODCDIHD type=del 2024-09-13T00:00:33.073 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.074+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~X12SQSPEKG9A.0 size=1048576 2024-09-13T00:00:33.076 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.074+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~X12SQSPEKG9A.1 size=703104 2024-09-13T00:00:33.076 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.074+0000 7fb8f2ce0640 1 > prepared multipart upload key=FX0NCGLK0KPJ upload=X12SQSPEKG9A tag=ZPSWU1ENNG51 type=write size=2800256 2024-09-13T00:00:33.079 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.078+0000 7fb8f2ce0640 1 < completed delete operation key=H5EIP0E9CBWX tag=6S5FLJIV8C5F type=del 2024-09-13T00:00:33.086 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.086+0000 7fb8f2ce0640 1 > prepared part key=_multipart_5GM88ERM89KT.2~V6I1QW2G2KJB.0 size=706736 2024-09-13T00:00:33.206 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.086+0000 7fb8f2ce0640 1 > prepared multipart upload key=5GM88ERM89KT upload=V6I1QW2G2KJB tag=LNOPI81S2TCM type=write size=1755312 2024-09-13T00:00:33.207 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:32 smithi181 bash[67587]: cluster 2024-09-13T00:00:31.124486+0000 mgr.y (mgr.24661) 702 : cluster [DBG] pgmap v1377: 137 pgs: 6 creating+peering, 18 unknown, 113 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 6.5 KiB/s wr, 24 op/s 2024-09-13T00:00:33.207 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:32 smithi181 bash[67587]: cluster 2024-09-13T00:00:31.700435+0000 mon.a (mon.0) 2069 : cluster [DBG] osdmap e1115: 8 total, 8 up, 8 in 2024-09-13T00:00:33.209 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.090+0000 7fb8f2ce0640 1 < completed delete operation key=NX26SI4F2ZMB tag=8KPPLUB50XFL type=del 2024-09-13T00:00:33.210 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.094+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~AOW802TXPBR1.0 size=1048576 2024-09-13T00:00:33.210 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.098+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~AOW802TXPBR1.1 size=1048576 2024-09-13T00:00:33.210 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.098+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~AOW802TXPBR1.2 size=513377 2024-09-13T00:00:33.210 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.098+0000 7fb8f2ce0640 1 > prepared multipart upload key=1HX06CCZJ1DD upload=AOW802TXPBR1 tag=63PGFO1QED7M type=write size=3659105 2024-09-13T00:00:33.210 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.102+0000 7fb8f2ce0640 1 < completed delete operation key=EJN6HR8I7XQF tag=ZH99M1J033BS type=del 2024-09-13T00:00:33.210 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.106+0000 7fb8f2ce0640 1 > prepared operation key=H4M1TCN5W6XU tag=UFQIJLBCX2X2 type=del size=576013 2024-09-13T00:00:33.210 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.110+0000 7fb8f2ce0640 1 < completed part key=_multipart_2AHYRZUDFHP4.2~OQHN87X308YK.0 size=4654 2024-09-13T00:00:33.210 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.114+0000 7fb8f2ce0640 1 < completed multipart upload key=2AHYRZUDFHP4 upload=OQHN87X308YK tag=751GQF7WRBW7 type=write size=1053230 2024-09-13T00:00:33.210 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.118+0000 7fb8f2ce0640 1 > prepared operation key=O4MZ4UWNXME9 tag=SJ413CG9JQP2 type=del size=808448 2024-09-13T00:00:33.210 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.126+0000 7fb8f2ce0640 1 < suggested remove operation key=FX0NCGLK0KPJ tag=VM4FG5NTQ2AZ type=write 2024-09-13T00:00:33.210 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.130+0000 7fb8f2ce0640 1 > prepared operation key=ZFHN2SNTZ5W7 tag=I6Y26EYKNYD0 type=del size=1733516 2024-09-13T00:00:33.210 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.134+0000 7fb8f2ce0640 1 < completed write operation key=85YEW8N43FTA tag=EJ6IKNW2ZODK type=write size=1045762 2024-09-13T00:00:33.211 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.142+0000 7fb8f2ce0640 1 > prepared part key=_multipart_4K76U69GZ9AE.2~JM2D0MFWFT8J.0 size=1048576 2024-09-13T00:00:33.211 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.146+0000 7fb8f2ce0640 1 > prepared part key=_multipart_4K76U69GZ9AE.2~JM2D0MFWFT8J.1 size=760239 2024-09-13T00:00:33.211 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.146+0000 7fb8f2ce0640 1 > prepared multipart upload key=4K76U69GZ9AE upload=JM2D0MFWFT8J tag=JD0YIJUZ4RT1 type=write size=2857391 2024-09-13T00:00:33.211 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.150+0000 7fb8f2ce0640 1 < completed part key=_multipart_63FJDWBO2T9S.2~OH3AW02D2EU1.0 size=972272 2024-09-13T00:00:33.211 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.154+0000 7fb8f2ce0640 1 < completed multipart upload key=63FJDWBO2T9S upload=OH3AW02D2EU1 tag=7STUGZ9RU2H8 type=write size=2020848 2024-09-13T00:00:33.211 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.162+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~GCJEWAIRKRNU.0 size=1048576 2024-09-13T00:00:33.211 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.166+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~GCJEWAIRKRNU.1 size=1048576 2024-09-13T00:00:33.211 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.166+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~GCJEWAIRKRNU.2 size=675814 2024-09-13T00:00:33.211 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.166+0000 7fb8f2ce0640 1 > prepared multipart upload key=FX0NCGLK0KPJ upload=GCJEWAIRKRNU tag=OHEQPWJ3HO2P type=write size=3821542 2024-09-13T00:00:33.211 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.170+0000 7fb8f2ce0640 1 < completed delete operation key=F0QCVR9E08WE tag=BKTXSDNE5XGQ type=del 2024-09-13T00:00:33.211 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.178+0000 7fb8f2ce0640 1 > prepared part key=_multipart_N3Z5XIG3TZRQ.2~9932U720TQUC.0 size=1008273 2024-09-13T00:00:33.211 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.178+0000 7fb8f2ce0640 1 > prepared multipart upload key=N3Z5XIG3TZRQ upload=9932U720TQUC tag=GI1R0RC63EGL type=write size=2056849 2024-09-13T00:00:33.211 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.182+0000 7fb8f2ce0640 1 < completed delete operation key=W8XZ26311RJX tag=SOVJFX5ODZ5W type=del 2024-09-13T00:00:33.211 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.186+0000 7fb8f2ce0640 1 > prepared operation key=H5EIP0E9CBWX tag=GP79FVIONQSJ type=del size=3660772 2024-09-13T00:00:33.212 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.190+0000 7fb8f2ce0640 1 < completed delete operation key=1HX06CCZJ1DD tag=VUKTH43SDQ4Y type=del 2024-09-13T00:00:33.212 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.190+0000 7fb8f2ce0640 1 > prepared operation key=2AHYRZUDFHP4 tag=SEOH2A8TT4WT type=del size=2924336 2024-09-13T00:00:33.212 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.194+0000 7fb8f2ce0640 1 < completed part key=_multipart_7OX5OJZ8GXU5.2~G8PJFQZRQOZA.0 size=1048576 2024-09-13T00:00:33.212 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.194+0000 7fb8f2ce0640 1 < canceled part key=_multipart_7OX5OJZ8GXU5.2~G8PJFQZRQOZA.1 size=1048576 2024-09-13T00:00:33.212 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.198+0000 7fb8f2ce0640 1 < completed part key=_multipart_7OX5OJZ8GXU5.2~G8PJFQZRQOZA.2 size=572990 2024-09-13T00:00:33.212 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.206+0000 7fb8f2ce0640 1 < canceled multipart upload key=7OX5OJZ8GXU5 upload=G8PJFQZRQOZA tag=TJ3F7V894UZZ type=write size=3718718 2024-09-13T00:00:33.212 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.210+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H9AU4HD247YU.2~H3VTO37X6HQU.0 size=1048576 2024-09-13T00:00:33.214 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.214+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H9AU4HD247YU.2~H3VTO37X6HQU.1 size=560880 2024-09-13T00:00:33.214 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.214+0000 7fb8f2ce0640 1 > prepared multipart upload key=H9AU4HD247YU upload=H3VTO37X6HQU tag=Z12SR58P7DE4 type=write size=2658032 2024-09-13T00:00:33.218 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.218+0000 7fb8f2ce0640 1 < completed part key=_multipart_J8NGDSF9XRTP.2~0E4XZHWH35Y5.0 size=776614 2024-09-13T00:00:33.224 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.222+0000 7fb8f2ce0640 1 < completed multipart upload key=J8NGDSF9XRTP upload=0E4XZHWH35Y5 tag=VD3AQQV7IMCS type=write size=1825190 2024-09-13T00:00:33.230 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.230+0000 7fb8f2ce0640 1 > prepared part key=_multipart_J8NGDSF9XRTP.2~ITZS3H2BIRLM.0 size=1048576 2024-09-13T00:00:33.233 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.234+0000 7fb8f2ce0640 1 > prepared part key=_multipart_J8NGDSF9XRTP.2~ITZS3H2BIRLM.1 size=810633 2024-09-13T00:00:33.233 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.234+0000 7fb8f2ce0640 1 > prepared multipart upload key=J8NGDSF9XRTP upload=ITZS3H2BIRLM tag=GKQ9HSACUXKT type=write size=2907785 2024-09-13T00:00:33.237 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.238+0000 7fb8f2ce0640 1 < canceled operation key=F0QCVR9E08WE tag=RXKYB4L9I2DQ type=write size=783254 2024-09-13T00:00:33.241 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.242+0000 7fb8f2ce0640 1 > prepared operation key=8Z6U7OOA0GBH tag=6HVQTGQ021KX type=del size=1482935 2024-09-13T00:00:33.245 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.242+0000 7fb8f2ce0640 1 < completed delete operation key=EJN6HR8I7XQF tag=M481OB4JCCXI type=del 2024-09-13T00:00:33.248 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.246+0000 7fb8f2ce0640 1 > prepared operation key=F0QCVR9E08WE tag=Y5LX1J960PM3 type=del size=1886964 2024-09-13T00:00:33.252 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.250+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~374VMQVZ7K69.0 size=1015463 2024-09-13T00:00:33.258 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.258+0000 7fb8f2ce0640 1 < completed multipart upload key=NQJQC8CEAC15 upload=374VMQVZ7K69 tag=GXK04XE3LNV4 type=write size=2064039 2024-09-13T00:00:33.262 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.262+0000 7fb8f2ce0640 1 > prepared operation key=0WC6P8YNL0H2 tag=7QVFSAEN83P5 type=del size=3524131 2024-09-13T00:00:33.266 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.266+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~X12SQSPEKG9A.0 size=1048576 2024-09-13T00:00:33.269 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.270+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~X12SQSPEKG9A.1 size=703104 2024-09-13T00:00:33.275 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.274+0000 7fb8f2ce0640 1 < completed multipart upload key=FX0NCGLK0KPJ upload=X12SQSPEKG9A tag=ZPSWU1ENNG51 type=write size=2800256 2024-09-13T00:00:33.279 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.278+0000 7fb8f2ce0640 1 > prepared operation key=1PX7DBO9M3YI tag=2VLVNHSWEKU2 type=write size=345974 2024-09-13T00:00:33.283 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.282+0000 7fb8f2ce0640 1 < completed part key=_multipart_5GM88ERM89KT.2~V6I1QW2G2KJB.0 size=706736 2024-09-13T00:00:33.289 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.286+0000 7fb8f2ce0640 1 < completed multipart upload key=5GM88ERM89KT upload=V6I1QW2G2KJB tag=LNOPI81S2TCM type=write size=1755312 2024-09-13T00:00:33.293 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.290+0000 7fb8f2ce0640 1 > prepared operation key=85YEW8N43FTA tag=5KX8LUV8TE3J type=del size=1980869 2024-09-13T00:00:33.293 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.294+0000 7fb8f2ce0640 1 < canceled part key=_multipart_1HX06CCZJ1DD.2~AOW802TXPBR1.0 size=1048576 2024-09-13T00:00:33.296 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.294+0000 7fb8f2ce0640 1 < completed part key=_multipart_1HX06CCZJ1DD.2~AOW802TXPBR1.1 size=1048576 2024-09-13T00:00:33.300 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.298+0000 7fb8f2ce0640 1 < completed part key=_multipart_1HX06CCZJ1DD.2~AOW802TXPBR1.2 size=513377 2024-09-13T00:00:33.306 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.306+0000 7fb8f2ce0640 1 < canceled multipart upload key=1HX06CCZJ1DD upload=AOW802TXPBR1 tag=63PGFO1QED7M type=write size=3659105 2024-09-13T00:00:33.313 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.314+0000 7fb8f2ce0640 1 > prepared part key=_multipart_P8OYS5ZE1NX0.2~VX5I3FB14XSS.0 size=706656 2024-09-13T00:00:33.313 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.314+0000 7fb8f2ce0640 1 > prepared multipart upload key=P8OYS5ZE1NX0 upload=VX5I3FB14XSS tag=VY8DLAKASYAP type=write size=1755232 2024-09-13T00:00:33.317 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.318+0000 7fb8f2ce0640 1 < completed delete operation key=H4M1TCN5W6XU tag=UFQIJLBCX2X2 type=del 2024-09-13T00:00:33.323 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.322+0000 7fb8f2ce0640 1 > prepared part key=_multipart_4K76U69GZ9AE.2~3X456E0FPNWR.0 size=1048576 2024-09-13T00:00:33.326 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.326+0000 7fb8f2ce0640 1 > prepared part key=_multipart_4K76U69GZ9AE.2~3X456E0FPNWR.1 size=1048576 2024-09-13T00:00:33.329 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.330+0000 7fb8f2ce0640 1 > prepared part key=_multipart_4K76U69GZ9AE.2~3X456E0FPNWR.2 size=795005 2024-09-13T00:00:33.329 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.330+0000 7fb8f2ce0640 1 > prepared multipart upload key=4K76U69GZ9AE upload=3X456E0FPNWR tag=E1JYFGOMTS7C type=write size=3940733 2024-09-13T00:00:33.333 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.334+0000 7fb8f2ce0640 1 < canceled operation key=O4MZ4UWNXME9 tag=SJ413CG9JQP2 type=del size=808448 2024-09-13T00:00:33.337 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.338+0000 7fb8f2ce0640 1 > prepared operation key=70F7D20HI504 tag=R7SJT1GN8UIQ type=write size=688634 2024-09-13T00:00:33.341 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.338+0000 7fb8f2ce0640 1 < canceled operation key=ZFHN2SNTZ5W7 tag=I6Y26EYKNYD0 type=del size=1733516 2024-09-13T00:00:33.344 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.342+0000 7fb8f2ce0640 1 > prepared operation key=FX0NCGLK0KPJ tag=EODDWI19K5CU type=del size=2262629 2024-09-13T00:00:33.348 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.346+0000 7fb8f2ce0640 1 < completed part key=_multipart_4K76U69GZ9AE.2~JM2D0MFWFT8J.0 size=1048576 2024-09-13T00:00:33.352 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.350+0000 7fb8f2ce0640 1 < completed part key=_multipart_4K76U69GZ9AE.2~JM2D0MFWFT8J.1 size=760239 2024-09-13T00:00:33.358 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.358+0000 7fb8f2ce0640 1 < completed multipart upload key=4K76U69GZ9AE upload=JM2D0MFWFT8J tag=JD0YIJUZ4RT1 type=write size=2857391 2024-09-13T00:00:33.361 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.362+0000 7fb8f2ce0640 1 > prepared operation key=4K76U69GZ9AE tag=3JZ042E6NN47 type=del size=3370661 2024-09-13T00:00:33.365 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.366+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~GCJEWAIRKRNU.0 size=1048576 2024-09-13T00:00:33.368 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.366+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~GCJEWAIRKRNU.1 size=1048576 2024-09-13T00:00:33.370 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.370+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~GCJEWAIRKRNU.2 size=675814 2024-09-13T00:00:33.376 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.374+0000 7fb8f2ce0640 1 < completed multipart upload key=FX0NCGLK0KPJ upload=GCJEWAIRKRNU tag=OHEQPWJ3HO2P type=write size=3821542 2024-09-13T00:00:33.382 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.382+0000 7fb8f2ce0640 1 > prepared part key=_multipart_N3Z5XIG3TZRQ.2~8S0N97CXGQX6.0 size=1048576 2024-09-13T00:00:33.385 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.386+0000 7fb8f2ce0640 1 > prepared part key=_multipart_N3Z5XIG3TZRQ.2~8S0N97CXGQX6.1 size=782563 2024-09-13T00:00:33.385 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.386+0000 7fb8f2ce0640 1 > prepared multipart upload key=N3Z5XIG3TZRQ upload=8S0N97CXGQX6 tag=PLO98C6IESSC type=write size=2879715 2024-09-13T00:00:33.389 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.386+0000 7fb8f2ce0640 1 < completed part key=_multipart_N3Z5XIG3TZRQ.2~9932U720TQUC.0 size=1008273 2024-09-13T00:00:33.395 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.394+0000 7fb8f2ce0640 1 < completed multipart upload key=N3Z5XIG3TZRQ upload=9932U720TQUC tag=GI1R0RC63EGL type=write size=2056849 2024-09-13T00:00:33.398 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.398+0000 7fb8f2ce0640 1 > prepared operation key=H9AU4HD247YU tag=R30OY6919S3B type=del size=2959263 2024-09-13T00:00:33.402 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.402+0000 7fb8f2ce0640 1 < completed delete operation key=H5EIP0E9CBWX tag=GP79FVIONQSJ type=del 2024-09-13T00:00:33.406 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.406+0000 7fb8f2ce0640 1 > prepared operation key=8MPUVD37EDN9 tag=B64T1ELWLNP2 type=del size=3469104 2024-09-13T00:00:33.410 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.410+0000 7fb8f2ce0640 1 < completed delete operation key=2AHYRZUDFHP4 tag=SEOH2A8TT4WT type=del 2024-09-13T00:00:33.414 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.414+0000 7fb8f2ce0640 1 > prepared operation key=N3Z5XIG3TZRQ tag=RBHPUJSSW78K type=del size=791860 2024-09-13T00:00:33.417 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.418+0000 7fb8f2ce0640 1 < completed part key=_multipart_H9AU4HD247YU.2~H3VTO37X6HQU.0 size=1048576 2024-09-13T00:00:33.420 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.418+0000 7fb8f2ce0640 1 < completed part key=_multipart_H9AU4HD247YU.2~H3VTO37X6HQU.1 size=560880 2024-09-13T00:00:33.426 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.426+0000 7fb8f2ce0640 1 < completed multipart upload key=H9AU4HD247YU upload=H3VTO37X6HQU tag=Z12SR58P7DE4 type=write size=2658032 2024-09-13T00:00:33.430 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.430+0000 7fb8f2ce0640 1 > prepared operation key=85YEW8N43FTA tag=V9SS27WW2ZS7 type=del size=3773472 2024-09-13T00:00:33.434 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.434+0000 7fb8f2ce0640 1 < completed part key=_multipart_J8NGDSF9XRTP.2~ITZS3H2BIRLM.0 size=1048576 2024-09-13T00:00:33.438 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.438+0000 7fb8f2ce0640 1 < completed part key=_multipart_J8NGDSF9XRTP.2~ITZS3H2BIRLM.1 size=810633 2024-09-13T00:00:33.443 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.442+0000 7fb8f2ce0640 1 < completed multipart upload key=J8NGDSF9XRTP upload=ITZS3H2BIRLM tag=GKQ9HSACUXKT type=write size=2907785 2024-09-13T00:00:33.447 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.446+0000 7fb8f2ce0640 1 > prepared operation key=NX26SI4F2ZMB tag=D574JC3592CD type=del size=2115225 2024-09-13T00:00:33.450 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.450+0000 7fb8f2ce0640 1 < completed delete operation key=8Z6U7OOA0GBH tag=6HVQTGQ021KX type=del 2024-09-13T00:00:33.457 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.458+0000 7fb8f2ce0640 1 > prepared part key=_multipart_5GM88ERM89KT.2~4CQ378MLAL6M.0 size=1048576 2024-09-13T00:00:33.460 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.458+0000 7fb8f2ce0640 1 > prepared part key=_multipart_5GM88ERM89KT.2~4CQ378MLAL6M.1 size=834203 2024-09-13T00:00:33.460 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.458+0000 7fb8f2ce0640 1 > prepared multipart upload key=5GM88ERM89KT upload=4CQ378MLAL6M tag=3MG8358G7YGA type=write size=2931355 2024-09-13T00:00:33.468 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.466+0000 7fb8f2ce0640 1 < suggested remove operation key=F0QCVR9E08WE tag=Y5LX1J960PM3 type=del 2024-09-13T00:00:33.474 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.474+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~RDBM0EJHXPYM.0 size=1048576 2024-09-13T00:00:33.477 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.478+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~RDBM0EJHXPYM.1 size=1048576 2024-09-13T00:00:33.480 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.478+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~RDBM0EJHXPYM.2 size=309312 2024-09-13T00:00:33.480 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.478+0000 7fb8f2ce0640 1 > prepared multipart upload key=1PX7DBO9M3YI upload=RDBM0EJHXPYM tag=U92PMCB98BN3 type=write size=3455040 2024-09-13T00:00:33.484 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.482+0000 7fb8f2ce0640 1 < completed delete operation key=0WC6P8YNL0H2 tag=7QVFSAEN83P5 type=del 2024-09-13T00:00:33.491 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.490+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~NBTJZYPV9ZZS.0 size=434173 2024-09-13T00:00:33.491 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.490+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZTK9AYBSLCOZ upload=NBTJZYPV9ZZS tag=Q6PII29AQ32E type=write size=1482749 2024-09-13T00:00:33.495 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.494+0000 7fb8f2ce0640 1 < completed write operation key=1PX7DBO9M3YI tag=2VLVNHSWEKU2 type=write size=345974 2024-09-13T00:00:33.502 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.502+0000 7fb8f2ce0640 1 > prepared part key=_multipart_4K76U69GZ9AE.2~0OMU7KGBML2R.0 size=1048576 2024-09-13T00:00:33.505 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.502+0000 7fb8f2ce0640 1 > prepared part key=_multipart_4K76U69GZ9AE.2~0OMU7KGBML2R.1 size=847746 2024-09-13T00:00:33.505 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.502+0000 7fb8f2ce0640 1 > prepared multipart upload key=4K76U69GZ9AE upload=0OMU7KGBML2R tag=R4DM8SAVLKM3 type=write size=2944898 2024-09-13T00:00:33.509 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.506+0000 7fb8f2ce0640 1 < completed delete operation key=85YEW8N43FTA tag=5KX8LUV8TE3J type=del 2024-09-13T00:00:33.516 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.514+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~74I8U3WBQ7VZ.0 size=1048576 2024-09-13T00:00:33.519 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.518+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~74I8U3WBQ7VZ.1 size=836060 2024-09-13T00:00:33.519 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.518+0000 7fb8f2ce0640 1 > prepared multipart upload key=FX0NCGLK0KPJ upload=74I8U3WBQ7VZ tag=D3CXYNXQEC3H type=write size=2933212 2024-09-13T00:00:33.524 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.522+0000 7fb8f2ce0640 1 < completed part key=_multipart_P8OYS5ZE1NX0.2~VX5I3FB14XSS.0 size=706656 2024-09-13T00:00:33.530 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.530+0000 7fb8f2ce0640 1 < completed multipart upload key=P8OYS5ZE1NX0 upload=VX5I3FB14XSS tag=VY8DLAKASYAP type=write size=1755232 2024-09-13T00:00:33.533 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.534+0000 7fb8f2ce0640 1 > prepared operation key=85YEW8N43FTA tag=CKUYCLDOWQXY type=del size=1293541 2024-09-13T00:00:33.537 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.538+0000 7fb8f2ce0640 1 < completed part key=_multipart_4K76U69GZ9AE.2~3X456E0FPNWR.0 size=1048576 2024-09-13T00:00:33.540 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.538+0000 7fb8f2ce0640 1 < completed part key=_multipart_4K76U69GZ9AE.2~3X456E0FPNWR.1 size=1048576 2024-09-13T00:00:33.543 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.542+0000 7fb8f2ce0640 1 < completed part key=_multipart_4K76U69GZ9AE.2~3X456E0FPNWR.2 size=795005 2024-09-13T00:00:33.550 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.550+0000 7fb8f2ce0640 1 < completed multipart upload key=4K76U69GZ9AE upload=3X456E0FPNWR tag=E1JYFGOMTS7C type=write size=3940733 2024-09-13T00:00:33.554 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.554+0000 7fb8f2ce0640 1 > prepared operation key=0WC6P8YNL0H2 tag=PDJS0PRZ2KZL type=del size=132716 2024-09-13T00:00:33.559 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.558+0000 7fb8f2ce0640 1 < completed write operation key=70F7D20HI504 tag=R7SJT1GN8UIQ type=write size=688634 2024-09-13T00:00:33.566 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.566+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~SLFYH90X72RD.0 size=1048576 2024-09-13T00:00:33.569 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.570+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~SLFYH90X72RD.1 size=1048576 2024-09-13T00:00:33.572 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.570+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~SLFYH90X72RD.2 size=890814 2024-09-13T00:00:33.572 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.570+0000 7fb8f2ce0640 1 > prepared multipart upload key=8Z6U7OOA0GBH upload=SLFYH90X72RD tag=1WNGFAQKYV1I type=write size=4036542 2024-09-13T00:00:33.579 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.578+0000 7fb8f2ce0640 1 < suggested remove operation key=FX0NCGLK0KPJ tag=EODDWI19K5CU type=del 2024-09-13T00:00:33.583 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.582+0000 7fb8f2ce0640 1 > prepared operation key=8Z6U7OOA0GBH tag=Y93DBYHO6GYU type=write size=824206 2024-09-13T00:00:33.590 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.590+0000 7fb8f2ce0640 1 < suggested remove operation key=4K76U69GZ9AE tag=3JZ042E6NN47 type=del 2024-09-13T00:00:33.594 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.594+0000 7fb8f2ce0640 1 > prepared operation key=F0QCVR9E08WE tag=XHRB6ELXHWZA type=write size=437389 2024-09-13T00:00:33.598 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.598+0000 7fb8f2ce0640 1 < completed part key=_multipart_N3Z5XIG3TZRQ.2~8S0N97CXGQX6.0 size=1048576 2024-09-13T00:00:33.598 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.598+0000 7fb8f2ce0640 1 < canceled part key=_multipart_N3Z5XIG3TZRQ.2~8S0N97CXGQX6.1 size=782563 2024-09-13T00:00:33.604 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.602+0000 7fb8f2ce0640 1 < canceled multipart upload key=N3Z5XIG3TZRQ upload=8S0N97CXGQX6 tag=PLO98C6IESSC type=write size=2879715 2024-09-13T00:00:33.611 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.610+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~QNKVHGH6NYH9.0 size=184915 2024-09-13T00:00:33.611 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.610+0000 7fb8f2ce0640 1 > prepared multipart upload key=7OX5OJZ8GXU5 upload=QNKVHGH6NYH9 tag=MIN5170TS6XB type=write size=1233491 2024-09-13T00:00:33.615 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.614+0000 7fb8f2ce0640 1 < canceled operation key=H9AU4HD247YU tag=R30OY6919S3B type=del size=2959263 2024-09-13T00:00:33.619 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.618+0000 7fb8f2ce0640 1 > prepared operation key=8MPUVD37EDN9 tag=0LC9HWAVJ2O0 type=del size=1403751 2024-09-13T00:00:33.623 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.622+0000 7fb8f2ce0640 1 < completed delete operation key=8MPUVD37EDN9 tag=B64T1ELWLNP2 type=del 2024-09-13T00:00:33.629 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.626+0000 7fb8f2ce0640 1 > prepared part key=_multipart_QMZREZ1QEDNJ.2~OMOBYOQ4NBHU.0 size=1048576 2024-09-13T00:00:33.631 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.630+0000 7fb8f2ce0640 1 > prepared part key=_multipart_QMZREZ1QEDNJ.2~OMOBYOQ4NBHU.1 size=568833 2024-09-13T00:00:33.631 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.630+0000 7fb8f2ce0640 1 > prepared multipart upload key=QMZREZ1QEDNJ upload=OMOBYOQ4NBHU tag=QLH3Y1B13UML type=write size=2665985 2024-09-13T00:00:33.634 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.634+0000 7fb8f2ce0640 1 < canceled operation key=N3Z5XIG3TZRQ tag=RBHPUJSSW78K type=del size=791860 2024-09-13T00:00:33.638 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.638+0000 7fb8f2ce0640 1 > prepared operation key=O4MZ4UWNXME9 tag=9ZB277W6IHBT type=del size=3258200 2024-09-13T00:00:33.641 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.642+0000 7fb8f2ce0640 1 < completed delete operation key=85YEW8N43FTA tag=V9SS27WW2ZS7 type=del 2024-09-13T00:00:33.648 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.646+0000 7fb8f2ce0640 1 > prepared part key=_multipart_0WC6P8YNL0H2.2~9P9SB8L4XCWD.0 size=1048576 2024-09-13T00:00:33.651 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.650+0000 7fb8f2ce0640 1 > prepared part key=_multipart_0WC6P8YNL0H2.2~9P9SB8L4XCWD.1 size=1048576 2024-09-13T00:00:33.654 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.654+0000 7fb8f2ce0640 1 > prepared part key=_multipart_0WC6P8YNL0H2.2~9P9SB8L4XCWD.2 size=224504 2024-09-13T00:00:33.654 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.654+0000 7fb8f2ce0640 1 > prepared multipart upload key=0WC6P8YNL0H2 upload=9P9SB8L4XCWD tag=TIML8ZYLNRDC type=write size=3370232 2024-09-13T00:00:33.657 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.658+0000 7fb8f2ce0640 1 < completed delete operation key=NX26SI4F2ZMB tag=D574JC3592CD type=del 2024-09-13T00:00:33.661 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.662+0000 7fb8f2ce0640 1 > prepared operation key=63FJDWBO2T9S tag=U6T40K1A39N6 type=del size=1704201 2024-09-13T00:00:33.665 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.666+0000 7fb8f2ce0640 1 < completed part key=_multipart_5GM88ERM89KT.2~4CQ378MLAL6M.0 size=1048576 2024-09-13T00:00:33.668 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.666+0000 7fb8f2ce0640 1 < completed part key=_multipart_5GM88ERM89KT.2~4CQ378MLAL6M.1 size=834203 2024-09-13T00:00:33.674 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.674+0000 7fb8f2ce0640 1 < completed multipart upload key=5GM88ERM89KT upload=4CQ378MLAL6M tag=3MG8358G7YGA type=write size=2931355 2024-09-13T00:00:33.677 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.678+0000 7fb8f2ce0640 1 > prepared operation key=8MPUVD37EDN9 tag=FILNAUH1C535 type=del size=2399219 2024-09-13T00:00:33.681 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.682+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~RDBM0EJHXPYM.0 size=1048576 2024-09-13T00:00:33.681 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.682+0000 7fb8f2ce0640 1 < canceled part key=_multipart_1PX7DBO9M3YI.2~RDBM0EJHXPYM.1 size=1048576 2024-09-13T00:00:33.684 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.682+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~RDBM0EJHXPYM.2 size=309312 2024-09-13T00:00:33.690 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.690+0000 7fb8f2ce0640 1 < canceled multipart upload key=1PX7DBO9M3YI upload=RDBM0EJHXPYM tag=U92PMCB98BN3 type=write size=3455040 2024-09-13T00:00:33.694 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.694+0000 7fb8f2ce0640 1 > prepared operation key=H9AU4HD247YU tag=6DGK8TFUFNX9 type=del size=3521432 2024-09-13T00:00:33.698 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.698+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~NBTJZYPV9ZZS.0 size=434173 2024-09-13T00:00:33.703 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.702+0000 7fb8f2ce0640 1 < completed multipart upload key=ZTK9AYBSLCOZ upload=NBTJZYPV9ZZS tag=Q6PII29AQ32E type=write size=1482749 2024-09-13T00:00:33.712 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.710+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZFHN2SNTZ5W7.2~C29NICGPH7SU.0 size=1048576 2024-09-13T00:00:33.713 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.714+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZFHN2SNTZ5W7.2~C29NICGPH7SU.1 size=1048576 2024-09-13T00:00:33.716 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.714+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZFHN2SNTZ5W7.2~C29NICGPH7SU.2 size=774416 2024-09-13T00:00:33.716 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.714+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZFHN2SNTZ5W7 upload=C29NICGPH7SU tag=AVLPUAX9WJ89 type=write size=3920144 2024-09-13T00:00:33.720 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.718+0000 7fb8f2ce0640 1 < completed part key=_multipart_4K76U69GZ9AE.2~0OMU7KGBML2R.0 size=1048576 2024-09-13T00:00:33.723 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.722+0000 7fb8f2ce0640 1 < completed part key=_multipart_4K76U69GZ9AE.2~0OMU7KGBML2R.1 size=847746 2024-09-13T00:00:33.728 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.726+0000 7fb8f2ce0640 1 < completed multipart upload key=4K76U69GZ9AE upload=0OMU7KGBML2R tag=R4DM8SAVLKM3 type=write size=2944898 2024-09-13T00:00:33.734 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.734+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZFHN2SNTZ5W7.2~WGJGUCII7M1D.0 size=1048576 2024-09-13T00:00:33.737 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.738+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZFHN2SNTZ5W7.2~WGJGUCII7M1D.1 size=1048576 2024-09-13T00:00:33.740 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.738+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZFHN2SNTZ5W7.2~WGJGUCII7M1D.2 size=253694 2024-09-13T00:00:33.740 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.738+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZFHN2SNTZ5W7 upload=WGJGUCII7M1D tag=KDQTP6YJCDNY type=write size=3399422 2024-09-13T00:00:33.744 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.742+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~74I8U3WBQ7VZ.0 size=1048576 2024-09-13T00:00:33.747 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.746+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~74I8U3WBQ7VZ.1 size=836060 2024-09-13T00:00:33.754 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.754+0000 7fb8f2ce0640 1 < completed multipart upload key=FX0NCGLK0KPJ upload=74I8U3WBQ7VZ tag=D3CXYNXQEC3H type=write size=2933212 2024-09-13T00:00:33.757 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.758+0000 7fb8f2ce0640 1 > prepared operation key=NQJQC8CEAC15 tag=HWNKN0NO93EN type=write size=68466 2024-09-13T00:00:33.762 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.762+0000 7fb8f2ce0640 1 < completed delete operation key=85YEW8N43FTA tag=CKUYCLDOWQXY type=del 2024-09-13T00:00:33.766 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.766+0000 7fb8f2ce0640 1 > prepared operation key=63FJDWBO2T9S tag=MMQXZZI7IQYC type=del size=1529202 2024-09-13T00:00:33.770 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.770+0000 7fb8f2ce0640 1 < completed delete operation key=0WC6P8YNL0H2 tag=PDJS0PRZ2KZL type=del 2024-09-13T00:00:33.774 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.774+0000 7fb8f2ce0640 1 > prepared operation key=ZFHN2SNTZ5W7 tag=RETP15JF68TF type=del size=1757904 2024-09-13T00:00:33.778 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.778+0000 7fb8f2ce0640 1 < completed part key=_multipart_8Z6U7OOA0GBH.2~SLFYH90X72RD.0 size=1048576 2024-09-13T00:00:33.781 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.782+0000 7fb8f2ce0640 1 < completed part key=_multipart_8Z6U7OOA0GBH.2~SLFYH90X72RD.1 size=1048576 2024-09-13T00:00:33.784 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.782+0000 7fb8f2ce0640 1 < completed part key=_multipart_8Z6U7OOA0GBH.2~SLFYH90X72RD.2 size=890814 2024-09-13T00:00:33.791 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.790+0000 7fb8f2ce0640 1 < completed multipart upload key=8Z6U7OOA0GBH upload=SLFYH90X72RD tag=1WNGFAQKYV1I type=write size=4036542 2024-09-13T00:00:33.798 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.798+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~46I4KB7IVXOO.0 size=1048576 2024-09-13T00:00:33.801 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.802+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~46I4KB7IVXOO.1 size=1048576 2024-09-13T00:00:33.804 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.802+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~46I4KB7IVXOO.2 size=478483 2024-09-13T00:00:33.804 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.802+0000 7fb8f2ce0640 1 > prepared multipart upload key=NQJQC8CEAC15 upload=46I4KB7IVXOO tag=OQBE227J5QY8 type=write size=3624211 2024-09-13T00:00:33.811 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.810+0000 7fb8f2ce0640 1 < suggested remove operation key=8Z6U7OOA0GBH tag=Y93DBYHO6GYU type=write 2024-09-13T00:00:33.816 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.814+0000 7fb8f2ce0640 1 > prepared operation key=5GM88ERM89KT tag=SZUUUC6HICVM type=write size=300533 2024-09-13T00:00:33.820 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.818+0000 7fb8f2ce0640 1 < completed write operation key=F0QCVR9E08WE tag=XHRB6ELXHWZA type=write size=437389 2024-09-13T00:00:33.827 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.826+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~826L6YEVF0YK.0 size=1048576 2024-09-13T00:00:33.830 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.830+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~826L6YEVF0YK.1 size=1048576 2024-09-13T00:00:33.833 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.830+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~826L6YEVF0YK.2 size=235249 2024-09-13T00:00:33.833 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.830+0000 7fb8f2ce0640 1 > prepared multipart upload key=1HX06CCZJ1DD upload=826L6YEVF0YK tag=FZJ8XMS3Y9Z7 type=write size=3380977 2024-09-13T00:00:33.836 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.834+0000 7fb8f2ce0640 1 < completed part key=_multipart_7OX5OJZ8GXU5.2~QNKVHGH6NYH9.0 size=184915 2024-09-13T00:00:33.844 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.842+0000 7fb8f2ce0640 1 < completed multipart upload key=7OX5OJZ8GXU5 upload=QNKVHGH6NYH9 tag=MIN5170TS6XB type=write size=1233491 2024-09-13T00:00:33.850 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.850+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~XVT3Z8KE0YG4.0 size=499394 2024-09-13T00:00:33.850 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.850+0000 7fb8f2ce0640 1 > prepared multipart upload key=1HX06CCZJ1DD upload=XVT3Z8KE0YG4 tag=94SEXM3U2NQA type=write size=1547970 2024-09-13T00:00:33.854 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.854+0000 7fb8f2ce0640 1 < completed delete operation key=8MPUVD37EDN9 tag=0LC9HWAVJ2O0 type=del 2024-09-13T00:00:33.860 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.858+0000 7fb8f2ce0640 1 > prepared part key=_multipart_O4MZ4UWNXME9.2~F45V9KKRI3QB.0 size=1048576 2024-09-13T00:00:33.862 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.862+0000 7fb8f2ce0640 1 > prepared part key=_multipart_O4MZ4UWNXME9.2~F45V9KKRI3QB.1 size=1048576 2024-09-13T00:00:33.865 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.866+0000 7fb8f2ce0640 1 > prepared part key=_multipart_O4MZ4UWNXME9.2~F45V9KKRI3QB.2 size=317574 2024-09-13T00:00:33.865 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.866+0000 7fb8f2ce0640 1 > prepared multipart upload key=O4MZ4UWNXME9 upload=F45V9KKRI3QB tag=LZUFHDBIFX9M type=write size=3463302 2024-09-13T00:00:33.869 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.866+0000 7fb8f2ce0640 1 < completed part key=_multipart_QMZREZ1QEDNJ.2~OMOBYOQ4NBHU.0 size=1048576 2024-09-13T00:00:33.871 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.870+0000 7fb8f2ce0640 1 < completed part key=_multipart_QMZREZ1QEDNJ.2~OMOBYOQ4NBHU.1 size=568833 2024-09-13T00:00:33.877 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.878+0000 7fb8f2ce0640 1 < completed multipart upload key=QMZREZ1QEDNJ upload=OMOBYOQ4NBHU tag=QLH3Y1B13UML type=write size=2665985 2024-09-13T00:00:33.884 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.882+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~75O03OOZ5609.0 size=605909 2024-09-13T00:00:33.884 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.882+0000 7fb8f2ce0640 1 > prepared multipart upload key=8Z6U7OOA0GBH upload=75O03OOZ5609 tag=499KCTAF3P0S type=write size=1654485 2024-09-13T00:00:33.887 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.886+0000 7fb8f2ce0640 1 < completed delete operation key=O4MZ4UWNXME9 tag=9ZB277W6IHBT type=del 2024-09-13T00:00:33.891 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.890+0000 7fb8f2ce0640 1 > prepared operation key=63FJDWBO2T9S tag=67MJ2RJLPS9S type=del size=337552 2024-09-13T00:00:33.895 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.894+0000 7fb8f2ce0640 1 < completed part key=_multipart_0WC6P8YNL0H2.2~9P9SB8L4XCWD.0 size=1048576 2024-09-13T00:00:33.898 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.898+0000 7fb8f2ce0640 1 < completed part key=_multipart_0WC6P8YNL0H2.2~9P9SB8L4XCWD.1 size=1048576 2024-09-13T00:00:33.901 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.902+0000 7fb8f2ce0640 1 < completed part key=_multipart_0WC6P8YNL0H2.2~9P9SB8L4XCWD.2 size=224504 2024-09-13T00:00:33.907 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.906+0000 7fb8f2ce0640 1 < completed multipart upload key=0WC6P8YNL0H2 upload=9P9SB8L4XCWD tag=TIML8ZYLNRDC type=write size=3370232 2024-09-13T00:00:33.911 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.910+0000 7fb8f2ce0640 1 > prepared operation key=70F7D20HI504 tag=CNUVHNC1VLNU type=del size=3275364 2024-09-13T00:00:33.916 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.914+0000 7fb8f2ce0640 1 < canceled operation key=63FJDWBO2T9S tag=U6T40K1A39N6 type=del size=1704201 2024-09-13T00:00:33.919 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.918+0000 7fb8f2ce0640 1 > prepared operation key=PVYZQLGYT3XU tag=BB5LU9E2UUC7 type=del size=1121455 2024-09-13T00:00:33.923 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.922+0000 7fb8f2ce0640 1 < canceled operation key=8MPUVD37EDN9 tag=FILNAUH1C535 type=del size=2399219 2024-09-13T00:00:33.926 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.926+0000 7fb8f2ce0640 1 > prepared operation key=1HX06CCZJ1DD tag=Z3MVELU077EZ type=del size=3483713 2024-09-13T00:00:33.930 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.930+0000 7fb8f2ce0640 1 < completed delete operation key=H9AU4HD247YU tag=6DGK8TFUFNX9 type=del 2024-09-13T00:00:33.936 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.934+0000 7fb8f2ce0640 1 > prepared part key=_multipart_O4MZ4UWNXME9.2~0AY6KTSUQ6QW.0 size=382523 2024-09-13T00:00:33.936 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.934+0000 7fb8f2ce0640 1 > prepared multipart upload key=O4MZ4UWNXME9 upload=0AY6KTSUQ6QW tag=IIF4K2U9U7QQ type=write size=1431099 2024-09-13T00:00:33.941 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.938+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZFHN2SNTZ5W7.2~C29NICGPH7SU.0 size=1048576 2024-09-13T00:00:33.943 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.942+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZFHN2SNTZ5W7.2~C29NICGPH7SU.1 size=1048576 2024-09-13T00:00:33.946 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.946+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZFHN2SNTZ5W7.2~C29NICGPH7SU.2 size=774416 2024-09-13T00:00:33.952 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.950+0000 7fb8f2ce0640 1 < completed multipart upload key=ZFHN2SNTZ5W7 upload=C29NICGPH7SU tag=AVLPUAX9WJ89 type=write size=3920144 2024-09-13T00:00:33.956 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.954+0000 7fb8f2ce0640 1 > prepared operation key=5GM88ERM89KT tag=YB4IG7MZAM9Z type=del size=2045315 2024-09-13T00:00:33.960 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.958+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZFHN2SNTZ5W7.2~WGJGUCII7M1D.0 size=1048576 2024-09-13T00:00:33.963 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.962+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZFHN2SNTZ5W7.2~WGJGUCII7M1D.1 size=1048576 2024-09-13T00:00:33.966 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.966+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZFHN2SNTZ5W7.2~WGJGUCII7M1D.2 size=253694 2024-09-13T00:00:33.972 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.970+0000 7fb8f2ce0640 1 < completed multipart upload key=ZFHN2SNTZ5W7 upload=WGJGUCII7M1D tag=KDQTP6YJCDNY type=write size=3399422 2024-09-13T00:00:33.976 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.974+0000 7fb8f2ce0640 1 > prepared operation key=85YEW8N43FTA tag=LTIXHYD28MFI type=del size=735022 2024-09-13T00:00:33.980 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.978+0000 7fb8f2ce0640 1 < completed write operation key=NQJQC8CEAC15 tag=HWNKN0NO93EN type=write size=68466 2024-09-13T00:00:33.984 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.982+0000 7fb8f2ce0640 1 > prepared operation key=EJN6HR8I7XQF tag=2QMFUISJ1I76 type=del size=3015280 2024-09-13T00:00:33.987 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.986+0000 7fb8f2ce0640 1 < completed delete operation key=63FJDWBO2T9S tag=MMQXZZI7IQYC type=del 2024-09-13T00:00:33.991 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.990+0000 7fb8f2ce0640 1 > prepared operation key=H9AU4HD247YU tag=XBTMK6I5S8X2 type=write size=607579 2024-09-13T00:00:33.995 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:33.994+0000 7fb8f2ce0640 1 < completed delete operation key=ZFHN2SNTZ5W7 tag=RETP15JF68TF type=del 2024-09-13T00:00:34.001 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.002+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~2RLYAHCRCKRE.0 size=1048576 2024-09-13T00:00:34.004 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.002+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~2RLYAHCRCKRE.1 size=1048576 2024-09-13T00:00:34.007 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.006+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~2RLYAHCRCKRE.2 size=275525 2024-09-13T00:00:34.007 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.006+0000 7fb8f2ce0640 1 > prepared multipart upload key=7OX5OJZ8GXU5 upload=2RLYAHCRCKRE tag=GI5C5QGKJ3HR type=write size=3421253 2024-09-13T00:00:34.011 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.010+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~46I4KB7IVXOO.0 size=1048576 2024-09-13T00:00:34.015 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.014+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~46I4KB7IVXOO.1 size=1048576 2024-09-13T00:00:34.017 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.018+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~46I4KB7IVXOO.2 size=478483 2024-09-13T00:00:34.025 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.022+0000 7fb8f2ce0640 1 < completed multipart upload key=NQJQC8CEAC15 upload=46I4KB7IVXOO tag=OQBE227J5QY8 type=write size=3624211 2024-09-13T00:00:34.029 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.030+0000 7fb8f2ce0640 1 > prepared operation key=ZTK9AYBSLCOZ tag=3MCLAEYKL7C0 type=del size=1629411 2024-09-13T00:00:34.033 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.034+0000 7fb8f2ce0640 1 < completed write operation key=5GM88ERM89KT tag=SZUUUC6HICVM type=write size=300533 2024-09-13T00:00:34.037 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.038+0000 7fb8f2ce0640 1 > prepared operation key=EUE3LT0PDIXV tag=WBWBDN8MJOWP type=write size=412898 2024-09-13T00:00:34.041 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.038+0000 7fb8f2ce0640 1 < completed part key=_multipart_1HX06CCZJ1DD.2~826L6YEVF0YK.0 size=1048576 2024-09-13T00:00:34.044 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.042+0000 7fb8f2ce0640 1 < completed part key=_multipart_1HX06CCZJ1DD.2~826L6YEVF0YK.1 size=1048576 2024-09-13T00:00:34.047 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.046+0000 7fb8f2ce0640 1 < completed part key=_multipart_1HX06CCZJ1DD.2~826L6YEVF0YK.2 size=235249 2024-09-13T00:00:34.054 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.054+0000 7fb8f2ce0640 1 < completed multipart upload key=1HX06CCZJ1DD upload=826L6YEVF0YK tag=FZJ8XMS3Y9Z7 type=write size=3380977 2024-09-13T00:00:34.061 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.062+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EJN6HR8I7XQF.2~T9XUSRYYK1DU.0 size=1048576 2024-09-13T00:00:34.064 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.062+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EJN6HR8I7XQF.2~T9XUSRYYK1DU.1 size=1048576 2024-09-13T00:00:34.067 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.066+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EJN6HR8I7XQF.2~T9XUSRYYK1DU.2 size=411474 2024-09-13T00:00:34.068 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.066+0000 7fb8f2ce0640 1 > prepared multipart upload key=EJN6HR8I7XQF upload=T9XUSRYYK1DU tag=8Q037IVDGKPL type=write size=3557202 2024-09-13T00:00:34.071 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.070+0000 7fb8f2ce0640 1 < completed part key=_multipart_1HX06CCZJ1DD.2~XVT3Z8KE0YG4.0 size=499394 2024-09-13T00:00:34.077 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.078+0000 7fb8f2ce0640 1 < completed multipart upload key=1HX06CCZJ1DD upload=XVT3Z8KE0YG4 tag=94SEXM3U2NQA type=write size=1547970 2024-09-13T00:00:34.081 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.082+0000 7fb8f2ce0640 1 > prepared operation key=QMZREZ1QEDNJ tag=KWLCL112CW93 type=del size=1869011 2024-09-13T00:00:34.085 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.086+0000 7fb8f2ce0640 1 < completed part key=_multipart_O4MZ4UWNXME9.2~F45V9KKRI3QB.0 size=1048576 2024-09-13T00:00:34.088 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.086+0000 7fb8f2ce0640 1 < completed part key=_multipart_O4MZ4UWNXME9.2~F45V9KKRI3QB.1 size=1048576 2024-09-13T00:00:34.090 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.090+0000 7fb8f2ce0640 1 < completed part key=_multipart_O4MZ4UWNXME9.2~F45V9KKRI3QB.2 size=317574 2024-09-13T00:00:34.097 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.094+0000 7fb8f2ce0640 1 < completed multipart upload key=O4MZ4UWNXME9 upload=F45V9KKRI3QB tag=LZUFHDBIFX9M type=write size=3463302 2024-09-13T00:00:34.100 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.098+0000 7fb8f2ce0640 1 > prepared operation key=44EDYFT9VF8Y tag=30V22KRLRQ46 type=del size=1387547 2024-09-13T00:00:34.104 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.102+0000 7fb8f2ce0640 1 < completed part key=_multipart_8Z6U7OOA0GBH.2~75O03OOZ5609.0 size=605909 2024-09-13T00:00:34.110 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.110+0000 7fb8f2ce0640 1 < completed multipart upload key=8Z6U7OOA0GBH upload=75O03OOZ5609 tag=499KCTAF3P0S type=write size=1654485 2024-09-13T00:00:34.114 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.114+0000 7fb8f2ce0640 1 > prepared operation key=2TWD9YLMSYJI tag=CDRP6P7GIYOE type=write size=158786 2024-09-13T00:00:34.120 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.118+0000 7fb8f2ce0640 1 < canceled operation key=63FJDWBO2T9S tag=67MJ2RJLPS9S type=del size=337552 2024-09-13T00:00:34.124 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.122+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~QGP15P9CP9Q4.0 size=986467 2024-09-13T00:00:34.124 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.122+0000 7fb8f2ce0640 1 > prepared multipart upload key=PVYZQLGYT3XU upload=QGP15P9CP9Q4 tag=T6RN416QHKZV type=write size=2035043 2024-09-13T00:00:34.127 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.126+0000 7fb8f2ce0640 1 < completed delete operation key=70F7D20HI504 tag=CNUVHNC1VLNU type=del 2024-09-13T00:00:34.131 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.130+0000 7fb8f2ce0640 1 > prepared operation key=ZFHN2SNTZ5W7 tag=0VC0LZETVZPK type=del size=3890937 2024-09-13T00:00:34.135 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.134+0000 7fb8f2ce0640 1 < completed delete operation key=PVYZQLGYT3XU tag=BB5LU9E2UUC7 type=del 2024-09-13T00:00:34.139 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.138+0000 7fb8f2ce0640 1 > prepared operation key=8Z6U7OOA0GBH tag=N0LXEVKPJB5W type=del size=1471163 2024-09-13T00:00:34.143 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.142+0000 7fb8f2ce0640 1 < completed delete operation key=1HX06CCZJ1DD tag=Z3MVELU077EZ type=del 2024-09-13T00:00:34.147 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.146+0000 7fb8f2ce0640 1 > prepared operation key=NQJQC8CEAC15 tag=AG5E8KDBPVRA type=write size=701501 2024-09-13T00:00:34.151 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.150+0000 7fb8f2ce0640 1 < completed part key=_multipart_O4MZ4UWNXME9.2~0AY6KTSUQ6QW.0 size=382523 2024-09-13T00:00:34.157 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.158+0000 7fb8f2ce0640 1 < completed multipart upload key=O4MZ4UWNXME9 upload=0AY6KTSUQ6QW tag=IIF4K2U9U7QQ type=write size=1431099 2024-09-13T00:00:34.161 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.162+0000 7fb8f2ce0640 1 > prepared operation key=N3Z5XIG3TZRQ tag=2SWKBGC1FS0K type=del size=3878511 2024-09-13T00:00:34.169 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.166+0000 7fb8f2ce0640 1 < suggested remove operation key=5GM88ERM89KT tag=YB4IG7MZAM9Z type=del 2024-09-13T00:00:34.172 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.170+0000 7fb8f2ce0640 1 > prepared operation key=F0QCVR9E08WE tag=YOOCDP50TKUB type=write size=436668 2024-09-13T00:00:34.176 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.174+0000 7fb8f2ce0640 1 < completed delete operation key=85YEW8N43FTA tag=LTIXHYD28MFI type=del 2024-09-13T00:00:34.183 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.182+0000 7fb8f2ce0640 1 > prepared part key=_multipart_4K76U69GZ9AE.2~2UGCGR6V8225.0 size=1048576 2024-09-13T00:00:34.186 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.186+0000 7fb8f2ce0640 1 > prepared part key=_multipart_4K76U69GZ9AE.2~2UGCGR6V8225.1 size=454827 2024-09-13T00:00:34.187 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.186+0000 7fb8f2ce0640 1 > prepared multipart upload key=4K76U69GZ9AE upload=2UGCGR6V8225 tag=RUAZJ4W12EUN type=write size=2551979 2024-09-13T00:00:34.191 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.190+0000 7fb8f2ce0640 1 < canceled operation key=EJN6HR8I7XQF tag=2QMFUISJ1I76 type=del size=3015280 2024-09-13T00:00:34.195 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.194+0000 7fb8f2ce0640 1 > prepared operation key=EJN6HR8I7XQF tag=CZMKKLQ1A442 type=del size=608841 2024-09-13T00:00:34.199 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.198+0000 7fb8f2ce0640 1 < completed write operation key=H9AU4HD247YU tag=XBTMK6I5S8X2 type=write size=607579 2024-09-13T00:00:34.202 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.202+0000 7fb8f2ce0640 1 > prepared operation key=2TWD9YLMSYJI tag=KM2PREAA3D4C type=write size=936663 2024-09-13T00:00:34.206 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.206+0000 7fb8f2ce0640 1 < completed part key=_multipart_7OX5OJZ8GXU5.2~2RLYAHCRCKRE.0 size=1048576 2024-09-13T00:00:34.210 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.210+0000 7fb8f2ce0640 1 < completed part key=_multipart_7OX5OJZ8GXU5.2~2RLYAHCRCKRE.1 size=1048576 2024-09-13T00:00:34.210 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.210+0000 7fb8f2ce0640 1 < canceled part key=_multipart_7OX5OJZ8GXU5.2~2RLYAHCRCKRE.2 size=275525 2024-09-13T00:00:34.216 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.214+0000 7fb8f2ce0640 1 < canceled multipart upload key=7OX5OJZ8GXU5 upload=2RLYAHCRCKRE tag=GI5C5QGKJ3HR type=write size=3421253 2024-09-13T00:00:34.219 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.218+0000 7fb8f2ce0640 1 > prepared operation key=ZFHN2SNTZ5W7 tag=4DLFST1E54F4 type=write size=256875 2024-09-13T00:00:34.223 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.222+0000 7fb8f2ce0640 1 < completed delete operation key=ZTK9AYBSLCOZ tag=3MCLAEYKL7C0 type=del 2024-09-13T00:00:34.227 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.226+0000 7fb8f2ce0640 1 > prepared operation key=8Z6U7OOA0GBH tag=7FMZPXCZH42P type=del size=1749898 2024-09-13T00:00:34.231 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.230+0000 7fb8f2ce0640 1 < completed write operation key=EUE3LT0PDIXV tag=WBWBDN8MJOWP type=write size=412898 2024-09-13T00:00:34.235 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.234+0000 7fb8f2ce0640 1 > prepared operation key=5GM88ERM89KT tag=CA572XMDG0SZ type=del size=3111177 2024-09-13T00:00:34.239 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.238+0000 7fb8f2ce0640 1 < completed part key=_multipart_EJN6HR8I7XQF.2~T9XUSRYYK1DU.0 size=1048576 2024-09-13T00:00:34.242 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.242+0000 7fb8f2ce0640 1 < completed part key=_multipart_EJN6HR8I7XQF.2~T9XUSRYYK1DU.1 size=1048576 2024-09-13T00:00:34.245 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.246+0000 7fb8f2ce0640 1 < completed part key=_multipart_EJN6HR8I7XQF.2~T9XUSRYYK1DU.2 size=411474 2024-09-13T00:00:34.251 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.250+0000 7fb8f2ce0640 1 < completed multipart upload key=EJN6HR8I7XQF upload=T9XUSRYYK1DU tag=8Q037IVDGKPL type=write size=3557202 2024-09-13T00:00:34.257 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.258+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~Y06NB9D6Q2CQ.0 size=1048576 2024-09-13T00:00:34.261 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.258+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~Y06NB9D6Q2CQ.1 size=207194 2024-09-13T00:00:34.261 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.258+0000 7fb8f2ce0640 1 > prepared multipart upload key=8MPUVD37EDN9 upload=Y06NB9D6Q2CQ tag=ON4SGL4NHV9O type=write size=2304346 2024-09-13T00:00:34.264 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.262+0000 7fb8f2ce0640 1 < completed delete operation key=QMZREZ1QEDNJ tag=KWLCL112CW93 type=del 2024-09-13T00:00:34.271 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.270+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H9AU4HD247YU.2~0QM38CV77GK1.0 size=1048576 2024-09-13T00:00:34.273 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.274+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H9AU4HD247YU.2~0QM38CV77GK1.1 size=392708 2024-09-13T00:00:34.273 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.274+0000 7fb8f2ce0640 1 > prepared multipart upload key=H9AU4HD247YU upload=0QM38CV77GK1 tag=W487AALGGAGQ type=write size=2489860 2024-09-13T00:00:34.277 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.274+0000 7fb8f2ce0640 1 < completed delete operation key=44EDYFT9VF8Y tag=30V22KRLRQ46 type=del 2024-09-13T00:00:34.283 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.282+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~0X5SWHWQ5AZY.0 size=1009075 2024-09-13T00:00:34.284 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.282+0000 7fb8f2ce0640 1 > prepared multipart upload key=1PX7DBO9M3YI upload=0X5SWHWQ5AZY tag=DNIHOB4JJNY3 type=write size=2057651 2024-09-13T00:00:34.291 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.290+0000 7fb8f2ce0640 1 < suggested update operation key=2TWD9YLMSYJI tag=CDRP6P7GIYOE type=write size=158786 2024-09-13T00:00:34.298 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.298+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H9AU4HD247YU.2~M17PPUFV5L3N.0 size=1048576 2024-09-13T00:00:34.301 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.302+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H9AU4HD247YU.2~M17PPUFV5L3N.1 size=636455 2024-09-13T00:00:34.301 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.302+0000 7fb8f2ce0640 1 > prepared multipart upload key=H9AU4HD247YU upload=M17PPUFV5L3N tag=VLUEMVA5CN1P type=write size=2733607 2024-09-13T00:00:34.305 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.306+0000 7fb8f2ce0640 1 < completed part key=_multipart_PVYZQLGYT3XU.2~QGP15P9CP9Q4.0 size=986467 2024-09-13T00:00:34.310 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.310+0000 7fb8f2ce0640 1 < completed multipart upload key=PVYZQLGYT3XU upload=QGP15P9CP9Q4 tag=T6RN416QHKZV type=write size=2035043 2024-09-13T00:00:34.314 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.314+0000 7fb8f2ce0640 1 > prepared operation key=44EDYFT9VF8Y tag=JUA0L26PW73N type=del size=1169653 2024-09-13T00:00:34.318 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.318+0000 7fb8f2ce0640 1 < completed delete operation key=ZFHN2SNTZ5W7 tag=0VC0LZETVZPK type=del 2024-09-13T00:00:34.324 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.322+0000 7fb8f2ce0640 1 > prepared operation key=1PX7DBO9M3YI tag=92EPTUQBDD62 type=del size=2865580 2024-09-13T00:00:34.328 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.326+0000 7fb8f2ce0640 1 < suggested remove operation key=8Z6U7OOA0GBH tag=N0LXEVKPJB5W type=del 2024-09-13T00:00:34.335 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.334+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~097O1W8MGYDG.0 size=1048576 2024-09-13T00:00:34.338 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.338+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~097O1W8MGYDG.1 size=766867 2024-09-13T00:00:34.338 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.338+0000 7fb8f2ce0640 1 > prepared multipart upload key=W8XZ26311RJX upload=097O1W8MGYDG tag=YCR7R97UI8I8 type=write size=2864019 2024-09-13T00:00:34.342 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.342+0000 7fb8f2ce0640 1 < completed write operation key=NQJQC8CEAC15 tag=AG5E8KDBPVRA type=write size=701501 2024-09-13T00:00:34.347 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.346+0000 7fb8f2ce0640 1 > prepared operation key=8Z6U7OOA0GBH tag=8WQ1M8Y7ZBWV type=del size=1880715 2024-09-13T00:00:34.351 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.350+0000 7fb8f2ce0640 1 < completed delete operation key=N3Z5XIG3TZRQ tag=2SWKBGC1FS0K type=del 2024-09-13T00:00:34.359 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.358+0000 7fb8f2ce0640 1 > prepared part key=_multipart_P8OYS5ZE1NX0.2~LCOY5H5WURUT.0 size=1048576 2024-09-13T00:00:34.362 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.362+0000 7fb8f2ce0640 1 > prepared part key=_multipart_P8OYS5ZE1NX0.2~LCOY5H5WURUT.1 size=452098 2024-09-13T00:00:34.362 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.362+0000 7fb8f2ce0640 1 > prepared multipart upload key=P8OYS5ZE1NX0 upload=LCOY5H5WURUT tag=OKO276357JMK type=write size=2549250 2024-09-13T00:00:34.369 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.370+0000 7fb8f2ce0640 1 < suggested update operation key=F0QCVR9E08WE tag=YOOCDP50TKUB type=write size=436668 2024-09-13T00:00:34.373 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.374+0000 7fb8f2ce0640 1 > prepared operation key=8MPUVD37EDN9 tag=KL3Q75WE2RFQ type=del size=2537767 2024-09-13T00:00:34.378 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.378+0000 7fb8f2ce0640 1 < completed part key=_multipart_4K76U69GZ9AE.2~2UGCGR6V8225.0 size=1048576 2024-09-13T00:00:34.381 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.382+0000 7fb8f2ce0640 1 < completed part key=_multipart_4K76U69GZ9AE.2~2UGCGR6V8225.1 size=454827 2024-09-13T00:00:34.388 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.386+0000 7fb8f2ce0640 1 < completed multipart upload key=4K76U69GZ9AE upload=2UGCGR6V8225 tag=RUAZJ4W12EUN type=write size=2551979 2024-09-13T00:00:34.391 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.390+0000 7fb8f2ce0640 1 > prepared operation key=H9AU4HD247YU tag=4OL37ZR2NVOH type=del size=2162854 2024-09-13T00:00:34.394 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.394+0000 7fb8f2ce0640 1 < completed delete operation key=EJN6HR8I7XQF tag=CZMKKLQ1A442 type=del 2024-09-13T00:00:34.400 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.398+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H5EIP0E9CBWX.2~157XQHBXG2RN.0 size=1048576 2024-09-13T00:00:34.402 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.402+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H5EIP0E9CBWX.2~157XQHBXG2RN.1 size=1048576 2024-09-13T00:00:34.405 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.406+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H5EIP0E9CBWX.2~157XQHBXG2RN.2 size=395195 2024-09-13T00:00:34.405 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.406+0000 7fb8f2ce0640 1 > prepared multipart upload key=H5EIP0E9CBWX upload=157XQHBXG2RN tag=CSQKYVP9KS85 type=write size=3540923 2024-09-13T00:00:34.411 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.410+0000 7fb8f2ce0640 1 < suggested update operation key=2TWD9YLMSYJI tag=KM2PREAA3D4C type=write size=936663 2024-09-13T00:00:34.417 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.418+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H9AU4HD247YU.2~CFNM99DQ6KNJ.0 size=1048576 2024-09-13T00:00:34.419 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.418+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H9AU4HD247YU.2~CFNM99DQ6KNJ.1 size=44471 2024-09-13T00:00:34.419 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.418+0000 7fb8f2ce0640 1 > prepared multipart upload key=H9AU4HD247YU upload=CFNM99DQ6KNJ tag=AST6P2AL7AXY type=write size=2141623 2024-09-13T00:00:34.422 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.422+0000 7fb8f2ce0640 1 < completed write operation key=ZFHN2SNTZ5W7 tag=4DLFST1E54F4 type=write size=256875 2024-09-13T00:00:34.425 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.422+0000 7fb8f2ce0640 1 > prepared operation key=70F7D20HI504 tag=99H7RYOFR6D6 type=write size=902963 2024-09-13T00:00:34.430 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.430+0000 7fb8f2ce0640 1 < suggested update operation key=8Z6U7OOA0GBH tag=7FMZPXCZH42P type=del size=1749898 2024-09-13T00:00:34.434 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.434+0000 7fb8f2ce0640 1 > prepared part key=_multipart_85YEW8N43FTA.2~UY947SC2PRYB.0 size=1048576 2024-09-13T00:00:34.436 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.434+0000 7fb8f2ce0640 1 > prepared part key=_multipart_85YEW8N43FTA.2~UY947SC2PRYB.1 size=1048576 2024-09-13T00:00:34.438 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.438+0000 7fb8f2ce0640 1 > prepared part key=_multipart_85YEW8N43FTA.2~UY947SC2PRYB.2 size=17523 2024-09-13T00:00:34.438 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.438+0000 7fb8f2ce0640 1 > prepared multipart upload key=85YEW8N43FTA upload=UY947SC2PRYB tag=KUN3VVKGJWRF type=write size=3163251 2024-09-13T00:00:34.440 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.438+0000 7fb8f2ce0640 1 < completed delete operation key=5GM88ERM89KT tag=CA572XMDG0SZ type=del 2024-09-13T00:00:34.442 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.442+0000 7fb8f2ce0640 1 > prepared operation key=PVYZQLGYT3XU tag=AHO55QEB0TO3 type=del size=568453 2024-09-13T00:00:34.444 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.442+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~Y06NB9D6Q2CQ.0 size=1048576 2024-09-13T00:00:34.446 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.446+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~Y06NB9D6Q2CQ.1 size=207194 2024-09-13T00:00:34.449 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.450+0000 7fb8f2ce0640 1 < completed multipart upload key=8MPUVD37EDN9 upload=Y06NB9D6Q2CQ tag=ON4SGL4NHV9O type=write size=2304346 2024-09-13T00:00:34.451 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.450+0000 7fb8f2ce0640 1 > prepared operation key=2TWD9YLMSYJI tag=PDNOTSUDXSLL type=del size=3919996 2024-09-13T00:00:34.454 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.454+0000 7fb8f2ce0640 1 < completed part key=_multipart_H9AU4HD247YU.2~0QM38CV77GK1.0 size=1048576 2024-09-13T00:00:34.456 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.454+0000 7fb8f2ce0640 1 < completed part key=_multipart_H9AU4HD247YU.2~0QM38CV77GK1.1 size=392708 2024-09-13T00:00:34.459 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.458+0000 7fb8f2ce0640 1 < completed multipart upload key=H9AU4HD247YU upload=0QM38CV77GK1 tag=W487AALGGAGQ type=write size=2489860 2024-09-13T00:00:34.461 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.462+0000 7fb8f2ce0640 1 > prepared operation key=8MPUVD37EDN9 tag=GQG3O1555N40 type=del size=2647280 2024-09-13T00:00:34.463 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.462+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~0X5SWHWQ5AZY.0 size=1009075 2024-09-13T00:00:34.467 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.466+0000 7fb8f2ce0640 1 < completed multipart upload key=1PX7DBO9M3YI upload=0X5SWHWQ5AZY tag=DNIHOB4JJNY3 type=write size=2057651 2024-09-13T00:00:34.469 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.470+0000 7fb8f2ce0640 1 > prepared operation key=H5EIP0E9CBWX tag=VG15W8X15C89 type=del size=2817285 2024-09-13T00:00:34.469 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.470+0000 7fb8f2ce0640 1 < canceled part key=_multipart_H9AU4HD247YU.2~M17PPUFV5L3N.0 size=1048576 2024-09-13T00:00:34.471 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.470+0000 7fb8f2ce0640 1 < completed part key=_multipart_H9AU4HD247YU.2~M17PPUFV5L3N.1 size=636455 2024-09-13T00:00:34.475 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.474+0000 7fb8f2ce0640 1 < canceled multipart upload key=H9AU4HD247YU upload=M17PPUFV5L3N tag=VLUEMVA5CN1P type=write size=2733607 2024-09-13T00:00:34.477 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.478+0000 7fb8f2ce0640 1 > prepared operation key=ZTK9AYBSLCOZ tag=CPK8DFJG20PE type=del size=91474 2024-09-13T00:00:34.479 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.478+0000 7fb8f2ce0640 1 < completed delete operation key=44EDYFT9VF8Y tag=JUA0L26PW73N type=del 2024-09-13T00:00:34.481 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.482+0000 7fb8f2ce0640 1 > prepared operation key=70F7D20HI504 tag=33R880PTIBO6 type=del size=1626116 2024-09-13T00:00:34.485 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.486+0000 7fb8f2ce0640 1 < suggested remove operation key=1PX7DBO9M3YI tag=92EPTUQBDD62 type=del 2024-09-13T00:00:34.489 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.490+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~50I7EI5J408W.0 size=1048576 2024-09-13T00:00:34.491 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.490+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~50I7EI5J408W.1 size=1048576 2024-09-13T00:00:34.492 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.490+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~50I7EI5J408W.2 size=958613 2024-09-13T00:00:34.493 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.490+0000 7fb8f2ce0640 1 > prepared multipart upload key=2AHYRZUDFHP4 upload=50I7EI5J408W tag=11TMQS63TNY0 type=write size=4104341 2024-09-13T00:00:34.495 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.494+0000 7fb8f2ce0640 1 < completed part key=_multipart_W8XZ26311RJX.2~097O1W8MGYDG.0 size=1048576 2024-09-13T00:00:34.497 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.494+0000 7fb8f2ce0640 1 < completed part key=_multipart_W8XZ26311RJX.2~097O1W8MGYDG.1 size=766867 2024-09-13T00:00:34.501 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.498+0000 7fb8f2ce0640 1 < completed multipart upload key=W8XZ26311RJX upload=097O1W8MGYDG tag=YCR7R97UI8I8 type=write size=2864019 2024-09-13T00:00:34.503 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.502+0000 7fb8f2ce0640 1 > prepared operation key=J8NGDSF9XRTP tag=5EWZGZZZBDUV type=del size=2095592 2024-09-13T00:00:34.505 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.506+0000 7fb8f2ce0640 -1 < failed to complete operation key=8Z6U7OOA0GBH tag=8WQ1M8Y7ZBWV type=del size=1880715: (22) Invalid argument 2024-09-13T00:00:34.508 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.506+0000 7fb8f2ce0640 1 > prepared operation key=P8OYS5ZE1NX0 tag=1WTOYK4AGILJ type=del size=2911979 2024-09-13T00:00:34.510 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.510+0000 7fb8f2ce0640 1 < completed part key=_multipart_P8OYS5ZE1NX0.2~LCOY5H5WURUT.0 size=1048576 2024-09-13T00:00:34.510 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.510+0000 7fb8f2ce0640 1 < canceled part key=_multipart_P8OYS5ZE1NX0.2~LCOY5H5WURUT.1 size=452098 2024-09-13T00:00:34.514 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.514+0000 7fb8f2ce0640 1 < canceled multipart upload key=P8OYS5ZE1NX0 upload=LCOY5H5WURUT tag=OKO276357JMK type=write size=2549250 2024-09-13T00:00:34.515 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.514+0000 7fb8f2ce0640 1 > prepared operation key=H5EIP0E9CBWX tag=P8YE2C474O69 type=del size=3698610 2024-09-13T00:00:34.518 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.518+0000 7fb8f2ce0640 1 < completed delete operation key=8MPUVD37EDN9 tag=KL3Q75WE2RFQ type=del 2024-09-13T00:00:34.522 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.522+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~V28NKMT2BKJF.0 size=1048576 2024-09-13T00:00:34.524 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.522+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~V28NKMT2BKJF.1 size=199447 2024-09-13T00:00:34.524 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.522+0000 7fb8f2ce0640 1 > prepared multipart upload key=7OX5OJZ8GXU5 upload=V28NKMT2BKJF tag=GZLXRYNW3EA3 type=write size=2296599 2024-09-13T00:00:34.528 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.526+0000 7fb8f2ce0640 1 < completed delete operation key=H9AU4HD247YU tag=4OL37ZR2NVOH type=del 2024-09-13T00:00:34.532 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.530+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H5EIP0E9CBWX.2~KXSTALTH8U11.0 size=1048576 2024-09-13T00:00:34.534 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.534+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H5EIP0E9CBWX.2~KXSTALTH8U11.1 size=1048576 2024-09-13T00:00:34.535 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.534+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H5EIP0E9CBWX.2~KXSTALTH8U11.2 size=957414 2024-09-13T00:00:34.536 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.534+0000 7fb8f2ce0640 1 > prepared multipart upload key=H5EIP0E9CBWX upload=KXSTALTH8U11 tag=DEA03FOG7BAK type=write size=4103142 2024-09-13T00:00:34.538 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.538+0000 7fb8f2ce0640 1 < completed part key=_multipart_H5EIP0E9CBWX.2~157XQHBXG2RN.0 size=1048576 2024-09-13T00:00:34.540 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.538+0000 7fb8f2ce0640 1 < completed part key=_multipart_H5EIP0E9CBWX.2~157XQHBXG2RN.1 size=1048576 2024-09-13T00:00:34.541 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.542+0000 7fb8f2ce0640 1 < completed part key=_multipart_H5EIP0E9CBWX.2~157XQHBXG2RN.2 size=395195 2024-09-13T00:00:34.545 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.546+0000 7fb8f2ce0640 1 < completed multipart upload key=H5EIP0E9CBWX upload=157XQHBXG2RN tag=CSQKYVP9KS85 type=write size=3540923 2024-09-13T00:00:34.549 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.550+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~39O41MYQ34EZ.0 size=801808 2024-09-13T00:00:34.549 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.550+0000 7fb8f2ce0640 1 > prepared multipart upload key=2AHYRZUDFHP4 upload=39O41MYQ34EZ tag=KKJ2HLUW8Y3Q type=write size=1850384 2024-09-13T00:00:34.551 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.550+0000 7fb8f2ce0640 1 < completed part key=_multipart_H9AU4HD247YU.2~CFNM99DQ6KNJ.0 size=1048576 2024-09-13T00:00:34.553 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.554+0000 7fb8f2ce0640 1 < completed part key=_multipart_H9AU4HD247YU.2~CFNM99DQ6KNJ.1 size=44471 2024-09-13T00:00:34.556 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.554+0000 7fb8f2ce0640 1 < completed multipart upload key=H9AU4HD247YU upload=CFNM99DQ6KNJ tag=AST6P2AL7AXY type=write size=2141623 2024-09-13T00:00:34.559 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.558+0000 7fb8f2ce0640 1 > prepared operation key=ZFHN2SNTZ5W7 tag=JFOKV0C3658I type=write size=327859 2024-09-13T00:00:34.564 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.562+0000 7fb8f2ce0640 1 < suggested update operation key=70F7D20HI504 tag=99H7RYOFR6D6 type=write size=902963 2024-09-13T00:00:34.566 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.566+0000 7fb8f2ce0640 1 > prepared operation key=2TWD9YLMSYJI tag=0N9WBSBIEEPP type=write size=369657 2024-09-13T00:00:34.569 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.570+0000 7fb8f2ce0640 1 < completed part key=_multipart_85YEW8N43FTA.2~UY947SC2PRYB.0 size=1048576 2024-09-13T00:00:34.571 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.570+0000 7fb8f2ce0640 1 < completed part key=_multipart_85YEW8N43FTA.2~UY947SC2PRYB.1 size=1048576 2024-09-13T00:00:34.571 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.570+0000 7fb8f2ce0640 1 < canceled part key=_multipart_85YEW8N43FTA.2~UY947SC2PRYB.2 size=17523 2024-09-13T00:00:34.575 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.574+0000 7fb8f2ce0640 1 < canceled multipart upload key=85YEW8N43FTA upload=UY947SC2PRYB tag=KUN3VVKGJWRF type=write size=3163251 2024-09-13T00:00:34.577 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.578+0000 7fb8f2ce0640 1 > prepared operation key=2TWD9YLMSYJI tag=C1H9SOGMGF0L type=write size=198007 2024-09-13T00:00:34.580 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.578+0000 7fb8f2ce0640 1 < completed delete operation key=PVYZQLGYT3XU tag=AHO55QEB0TO3 type=del 2024-09-13T00:00:34.585 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.586+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~8V48M7X3BZWB.0 size=1048576 2024-09-13T00:00:34.587 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.586+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~8V48M7X3BZWB.1 size=1048576 2024-09-13T00:00:34.589 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.590+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~8V48M7X3BZWB.2 size=455654 2024-09-13T00:00:34.589 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.590+0000 7fb8f2ce0640 1 > prepared multipart upload key=63FJDWBO2T9S upload=8V48M7X3BZWB tag=TBYWY7C2GLR5 type=write size=3601382 2024-09-13T00:00:34.592 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.590+0000 7fb8f2ce0640 1 < completed delete operation key=2TWD9YLMSYJI tag=PDNOTSUDXSLL type=del 2024-09-13T00:00:34.595 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.594+0000 7fb8f2ce0640 1 > prepared operation key=63FJDWBO2T9S tag=62RHQ2PUQ8ZH type=del size=1589365 2024-09-13T00:00:34.598 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.598+0000 7fb8f2ce0640 1 < completed delete operation key=8MPUVD37EDN9 tag=GQG3O1555N40 type=del 2024-09-13T00:00:34.601 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.602+0000 7fb8f2ce0640 1 > prepared operation key=O4MZ4UWNXME9 tag=S6JVPPCEHL09 type=del size=2794606 2024-09-13T00:00:34.605 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.606+0000 7fb8f2ce0640 1 < completed delete operation key=H5EIP0E9CBWX tag=VG15W8X15C89 type=del 2024-09-13T00:00:34.610 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.610+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~N39QKUEI2Q04.0 size=1048576 2024-09-13T00:00:34.613 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.614+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~N39QKUEI2Q04.1 size=1048576 2024-09-13T00:00:34.615 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.614+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~N39QKUEI2Q04.2 size=508092 2024-09-13T00:00:34.615 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.614+0000 7fb8f2ce0640 1 > prepared multipart upload key=8MPUVD37EDN9 upload=N39QKUEI2Q04 tag=ZFW743IZG8Y8 type=write size=3653820 2024-09-13T00:00:34.618 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.618+0000 7fb8f2ce0640 1 < completed delete operation key=ZTK9AYBSLCOZ tag=CPK8DFJG20PE type=del 2024-09-13T00:00:34.621 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.622+0000 7fb8f2ce0640 1 > prepared operation key=0WC6P8YNL0H2 tag=KY3JIPZCG5CT type=write size=775063 2024-09-13T00:00:34.624 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.622+0000 7fb8f2ce0640 -1 < failed to complete operation key=70F7D20HI504 tag=33R880PTIBO6 type=del size=1626116: (22) Invalid argument 2024-09-13T00:00:34.629 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.630+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~AFTFFC6J1UM7.0 size=1048576 2024-09-13T00:00:34.631 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.630+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~AFTFFC6J1UM7.1 size=1048576 2024-09-13T00:00:34.633 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.634+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~AFTFFC6J1UM7.2 size=764251 2024-09-13T00:00:34.633 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.634+0000 7fb8f2ce0640 1 > prepared multipart upload key=70F7D20HI504 upload=AFTFFC6J1UM7 tag=AFGQ1RRQUI7V type=write size=3909979 2024-09-13T00:00:34.633 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.634+0000 7fb8f2ce0640 1 < canceled part key=_multipart_2AHYRZUDFHP4.2~50I7EI5J408W.0 size=1048576 2024-09-13T00:00:34.635 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.634+0000 7fb8f2ce0640 1 < completed part key=_multipart_2AHYRZUDFHP4.2~50I7EI5J408W.1 size=1048576 2024-09-13T00:00:34.637 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.638+0000 7fb8f2ce0640 1 < completed part key=_multipart_2AHYRZUDFHP4.2~50I7EI5J408W.2 size=958613 2024-09-13T00:00:34.642 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.642+0000 7fb8f2ce0640 1 < canceled multipart upload key=2AHYRZUDFHP4 upload=50I7EI5J408W tag=11TMQS63TNY0 type=write size=4104341 2024-09-13T00:00:34.644 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.642+0000 7fb8f2ce0640 1 > prepared operation key=8MPUVD37EDN9 tag=4GDWAWNIKF21 type=del size=2530047 2024-09-13T00:00:34.647 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.646+0000 7fb8f2ce0640 1 < completed delete operation key=J8NGDSF9XRTP tag=5EWZGZZZBDUV type=del 2024-09-13T00:00:34.652 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.650+0000 7fb8f2ce0640 1 > prepared part key=_multipart_O4MZ4UWNXME9.2~5UIOJNOJGBFW.0 size=1048576 2024-09-13T00:00:34.654 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.654+0000 7fb8f2ce0640 1 > prepared part key=_multipart_O4MZ4UWNXME9.2~5UIOJNOJGBFW.1 size=33936 2024-09-13T00:00:34.654 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.654+0000 7fb8f2ce0640 1 > prepared multipart upload key=O4MZ4UWNXME9 upload=5UIOJNOJGBFW tag=4HLIRM5YUK3V type=write size=2131088 2024-09-13T00:00:34.656 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.654+0000 7fb8f2ce0640 1 < completed delete operation key=P8OYS5ZE1NX0 tag=1WTOYK4AGILJ type=del 2024-09-13T00:00:34.661 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.662+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~22M68I9FZ759.0 size=1048576 2024-09-13T00:00:34.663 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.662+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~22M68I9FZ759.1 size=735474 2024-09-13T00:00:34.663 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.662+0000 7fb8f2ce0640 1 > prepared multipart upload key=8MPUVD37EDN9 upload=22M68I9FZ759 tag=Z70XRU6W37PP type=write size=2832626 2024-09-13T00:00:34.666 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.666+0000 7fb8f2ce0640 1 < completed delete operation key=H5EIP0E9CBWX tag=P8YE2C474O69 type=del 2024-09-13T00:00:34.668 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.666+0000 7fb8f2ce0640 1 > prepared operation key=2AHYRZUDFHP4 tag=R93D8XUT1P86 type=del size=927987 2024-09-13T00:00:34.671 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.670+0000 7fb8f2ce0640 1 < completed part key=_multipart_7OX5OJZ8GXU5.2~V28NKMT2BKJF.0 size=1048576 2024-09-13T00:00:34.673 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.674+0000 7fb8f2ce0640 1 < completed part key=_multipart_7OX5OJZ8GXU5.2~V28NKMT2BKJF.1 size=199447 2024-09-13T00:00:34.677 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.678+0000 7fb8f2ce0640 1 < completed multipart upload key=7OX5OJZ8GXU5 upload=V28NKMT2BKJF tag=GZLXRYNW3EA3 type=write size=2296599 2024-09-13T00:00:34.680 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.678+0000 7fb8f2ce0640 1 > prepared operation key=85YEW8N43FTA tag=KG1G3Q23KJ8B type=del size=1843638 2024-09-13T00:00:34.682 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.682+0000 7fb8f2ce0640 1 < completed part key=_multipart_H5EIP0E9CBWX.2~KXSTALTH8U11.0 size=1048576 2024-09-13T00:00:34.684 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.682+0000 7fb8f2ce0640 1 < completed part key=_multipart_H5EIP0E9CBWX.2~KXSTALTH8U11.1 size=1048576 2024-09-13T00:00:34.686 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.686+0000 7fb8f2ce0640 1 < completed part key=_multipart_H5EIP0E9CBWX.2~KXSTALTH8U11.2 size=957414 2024-09-13T00:00:34.692 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.690+0000 7fb8f2ce0640 1 < completed multipart upload key=H5EIP0E9CBWX upload=KXSTALTH8U11 tag=DEA03FOG7BAK type=write size=4103142 2024-09-13T00:00:34.695 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.694+0000 7fb8f2ce0640 1 > prepared operation key=F0QCVR9E08WE tag=1THWV3864LP3 type=del size=3853347 2024-09-13T00:00:34.697 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.698+0000 7fb8f2ce0640 1 < completed part key=_multipart_2AHYRZUDFHP4.2~39O41MYQ34EZ.0 size=801808 2024-09-13T00:00:34.701 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.702+0000 7fb8f2ce0640 1 < completed multipart upload key=2AHYRZUDFHP4 upload=39O41MYQ34EZ tag=KKJ2HLUW8Y3Q type=write size=1850384 2024-09-13T00:00:34.705 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.706+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~DDN35HW9QUSJ.0 size=743028 2024-09-13T00:00:34.705 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.706+0000 7fb8f2ce0640 1 > prepared multipart upload key=W8XZ26311RJX upload=DDN35HW9QUSJ tag=LAARB6VQIQCT type=write size=1791604 2024-09-13T00:00:34.708 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.706+0000 7fb8f2ce0640 1 < suggested update operation key=ZFHN2SNTZ5W7 tag=JFOKV0C3658I type=write size=327859 2024-09-13T00:00:34.710 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.710+0000 7fb8f2ce0640 1 > prepared operation key=H4M1TCN5W6XU tag=NVF0CR729C2Q type=del size=1033353 2024-09-13T00:00:34.711 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.710+0000 7fb8f2ce0640 1 < completed write operation key=2TWD9YLMSYJI tag=0N9WBSBIEEPP type=write size=369657 2024-09-13T00:00:34.713 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.714+0000 7fb8f2ce0640 1 > prepared operation key=63FJDWBO2T9S tag=MFV2130LKJOZ type=del size=803047 2024-09-13T00:00:34.715 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.714+0000 7fb8f2ce0640 1 < completed write operation key=2TWD9YLMSYJI tag=C1H9SOGMGF0L type=write size=198007 2024-09-13T00:00:34.718 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.718+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H4M1TCN5W6XU.2~MP80FLK566MK.0 size=1048576 2024-09-13T00:00:34.719 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.718+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H4M1TCN5W6XU.2~MP80FLK566MK.1 size=1048576 2024-09-13T00:00:34.721 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.722+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H4M1TCN5W6XU.2~MP80FLK566MK.2 size=61657 2024-09-13T00:00:34.721 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.722+0000 7fb8f2ce0640 1 > prepared multipart upload key=H4M1TCN5W6XU upload=MP80FLK566MK tag=9KNH7SEI44DF type=write size=3207385 2024-09-13T00:00:34.722 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.722+0000 7fb8f2ce0640 1 < completed part key=_multipart_63FJDWBO2T9S.2~8V48M7X3BZWB.0 size=1048576 2024-09-13T00:00:34.724 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.722+0000 7fb8f2ce0640 1 < completed part key=_multipart_63FJDWBO2T9S.2~8V48M7X3BZWB.1 size=1048576 2024-09-13T00:00:34.725 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.726+0000 7fb8f2ce0640 1 < completed part key=_multipart_63FJDWBO2T9S.2~8V48M7X3BZWB.2 size=455654 2024-09-13T00:00:34.728 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.726+0000 7fb8f2ce0640 1 < completed multipart upload key=63FJDWBO2T9S upload=8V48M7X3BZWB tag=TBYWY7C2GLR5 type=write size=3601382 2024-09-13T00:00:34.732 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.730+0000 7fb8f2ce0640 1 > prepared operation key=H5EIP0E9CBWX tag=8UJR6RP6F0RH type=del size=2821669 2024-09-13T00:00:34.732 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.730+0000 7fb8f2ce0640 1 < completed delete operation key=63FJDWBO2T9S tag=62RHQ2PUQ8ZH type=del 2024-09-13T00:00:34.735 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.734+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EJN6HR8I7XQF.2~7NL7LZBU2TO6.0 size=134162 2024-09-13T00:00:34.735 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.734+0000 7fb8f2ce0640 1 > prepared multipart upload key=EJN6HR8I7XQF upload=7NL7LZBU2TO6 tag=RL8BU5AHUYEY type=write size=1182738 2024-09-13T00:00:34.736 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.734+0000 7fb8f2ce0640 1 < canceled operation key=O4MZ4UWNXME9 tag=S6JVPPCEHL09 type=del size=2794606 2024-09-13T00:00:34.738 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.738+0000 7fb8f2ce0640 1 > prepared operation key=H9AU4HD247YU tag=HVMNHQ2SDY4P type=del size=1763782 2024-09-13T00:00:34.739 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.738+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~N39QKUEI2Q04.0 size=1048576 2024-09-13T00:00:34.740 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.738+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~N39QKUEI2Q04.1 size=1048576 2024-09-13T00:00:34.742 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.742+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~N39QKUEI2Q04.2 size=508092 2024-09-13T00:00:34.744 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.742+0000 7fb8f2ce0640 1 < completed multipart upload key=8MPUVD37EDN9 upload=N39QKUEI2Q04 tag=ZFW743IZG8Y8 type=write size=3653820 2024-09-13T00:00:34.746 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.746+0000 7fb8f2ce0640 1 > prepared operation key=NX26SI4F2ZMB tag=MSHS9XGR03V3 type=del size=1667358 2024-09-13T00:00:34.749 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.750+0000 7fb8f2ce0640 1 < suggested remove operation key=0WC6P8YNL0H2 tag=KY3JIPZCG5CT type=write 2024-09-13T00:00:34.751 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.750+0000 7fb8f2ce0640 1 > prepared operation key=44EDYFT9VF8Y tag=TESPRJYJV6I9 type=del size=2284233 2024-09-13T00:00:34.752 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.750+0000 7fb8f2ce0640 1 < completed part key=_multipart_70F7D20HI504.2~AFTFFC6J1UM7.0 size=1048576 2024-09-13T00:00:34.754 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.754+0000 7fb8f2ce0640 1 < completed part key=_multipart_70F7D20HI504.2~AFTFFC6J1UM7.1 size=1048576 2024-09-13T00:00:34.754 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.754+0000 7fb8f2ce0640 1 < canceled part key=_multipart_70F7D20HI504.2~AFTFFC6J1UM7.2 size=764251 2024-09-13T00:00:34.757 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.758+0000 7fb8f2ce0640 1 < canceled multipart upload key=70F7D20HI504 upload=AFTFFC6J1UM7 tag=AFGQ1RRQUI7V type=write size=3909979 2024-09-13T00:00:34.758 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.758+0000 7fb8f2ce0640 1 > prepared operation key=5GM88ERM89KT tag=PIAF5WCRTU17 type=del size=3566625 2024-09-13T00:00:34.761 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.758+0000 7fb8f2ce0640 1 < completed delete operation key=8MPUVD37EDN9 tag=4GDWAWNIKF21 type=del 2024-09-13T00:00:34.761 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.762+0000 7fb8f2ce0640 1 > prepared operation key=44EDYFT9VF8Y tag=X4GXEDN07KVQ type=del size=1622324 2024-09-13T00:00:34.763 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.762+0000 7fb8f2ce0640 1 < completed part key=_multipart_O4MZ4UWNXME9.2~5UIOJNOJGBFW.0 size=1048576 2024-09-13T00:00:34.763 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.762+0000 7fb8f2ce0640 1 < canceled part key=_multipart_O4MZ4UWNXME9.2~5UIOJNOJGBFW.1 size=33936 2024-09-13T00:00:34.766 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.766+0000 7fb8f2ce0640 1 < canceled multipart upload key=O4MZ4UWNXME9 upload=5UIOJNOJGBFW tag=4HLIRM5YUK3V type=write size=2131088 2024-09-13T00:00:34.769 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.770+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~ALTEDTMYKNXS.0 size=1048576 2024-09-13T00:00:34.770 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.770+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~ALTEDTMYKNXS.1 size=518133 2024-09-13T00:00:34.770 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.770+0000 7fb8f2ce0640 1 > prepared multipart upload key=NQJQC8CEAC15 upload=ALTEDTMYKNXS tag=9NA0LJFAF4IM type=write size=2615285 2024-09-13T00:00:34.771 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.770+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~22M68I9FZ759.0 size=1048576 2024-09-13T00:00:34.773 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.774+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~22M68I9FZ759.1 size=735474 2024-09-13T00:00:34.776 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.774+0000 7fb8f2ce0640 1 < completed multipart upload key=8MPUVD37EDN9 upload=22M68I9FZ759 tag=Z70XRU6W37PP type=write size=2832626 2024-09-13T00:00:34.779 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.778+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~I6S6891BRA55.0 size=1048576 2024-09-13T00:00:34.780 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.778+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~I6S6891BRA55.1 size=1048576 2024-09-13T00:00:34.781 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.782+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~I6S6891BRA55.2 size=118665 2024-09-13T00:00:34.782 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.782+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZTK9AYBSLCOZ upload=I6S6891BRA55 tag=GMYUIDWM73JJ type=write size=3264393 2024-09-13T00:00:34.785 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.786+0000 7fb8f2ce0640 1 < suggested remove operation key=2AHYRZUDFHP4 tag=R93D8XUT1P86 type=del 2024-09-13T00:00:34.788 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.786+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~M4PTG46L3G0V.0 size=1048576 2024-09-13T00:00:34.789 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.790+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~M4PTG46L3G0V.1 size=377317 2024-09-13T00:00:34.789 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.790+0000 7fb8f2ce0640 1 > prepared multipart upload key=NQJQC8CEAC15 upload=M4PTG46L3G0V tag=UE5ZM96VALAC type=write size=2474469 2024-09-13T00:00:34.791 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.790+0000 7fb8f2ce0640 1 < canceled operation key=85YEW8N43FTA tag=KG1G3Q23KJ8B type=del size=1843638 2024-09-13T00:00:34.792 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.790+0000 7fb8f2ce0640 1 > prepared operation key=7OX5OJZ8GXU5 tag=1JIROS1H99A1 type=del size=1989564 2024-09-13T00:00:34.795 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.794+0000 7fb8f2ce0640 1 < completed delete operation key=F0QCVR9E08WE tag=1THWV3864LP3 type=del 2024-09-13T00:00:34.798 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.798+0000 7fb8f2ce0640 1 > prepared part key=_multipart_N3Z5XIG3TZRQ.2~61OKJFN5HP72.0 size=1048576 2024-09-13T00:00:34.800 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.798+0000 7fb8f2ce0640 1 > prepared part key=_multipart_N3Z5XIG3TZRQ.2~61OKJFN5HP72.1 size=1039515 2024-09-13T00:00:34.800 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.798+0000 7fb8f2ce0640 1 > prepared multipart upload key=N3Z5XIG3TZRQ upload=61OKJFN5HP72 tag=629FSXYXL742 type=write size=3136667 2024-09-13T00:00:34.802 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.802+0000 7fb8f2ce0640 1 < completed part key=_multipart_W8XZ26311RJX.2~DDN35HW9QUSJ.0 size=743028 2024-09-13T00:00:34.806 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.806+0000 7fb8f2ce0640 1 < completed multipart upload key=W8XZ26311RJX upload=DDN35HW9QUSJ tag=LAARB6VQIQCT type=write size=1791604 2024-09-13T00:00:34.809 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.810+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~53NN95ONVV37.0 size=1048576 2024-09-13T00:00:34.811 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.810+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~53NN95ONVV37.1 size=1048576 2024-09-13T00:00:34.813 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.814+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~53NN95ONVV37.2 size=535109 2024-09-13T00:00:34.814 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.814+0000 7fb8f2ce0640 1 > prepared multipart upload key=8MPUVD37EDN9 upload=53NN95ONVV37 tag=DG2TVFSWBZEW type=write size=3680837 2024-09-13T00:00:34.816 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.814+0000 7fb8f2ce0640 1 < completed delete operation key=H4M1TCN5W6XU tag=NVF0CR729C2Q type=del 2024-09-13T00:00:34.818 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.818+0000 7fb8f2ce0640 1 > prepared operation key=W8XZ26311RJX tag=WBQ02XINC60R type=del size=301029 2024-09-13T00:00:34.820 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.818+0000 7fb8f2ce0640 1 < completed delete operation key=63FJDWBO2T9S tag=MFV2130LKJOZ type=del 2024-09-13T00:00:34.823 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.822+0000 7fb8f2ce0640 1 > prepared operation key=H4M1TCN5W6XU tag=W9WI5L293OWB type=del size=3299070 2024-09-13T00:00:34.825 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.826+0000 7fb8f2ce0640 1 < completed part key=_multipart_H4M1TCN5W6XU.2~MP80FLK566MK.0 size=1048576 2024-09-13T00:00:34.827 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.826+0000 7fb8f2ce0640 1 < completed part key=_multipart_H4M1TCN5W6XU.2~MP80FLK566MK.1 size=1048576 2024-09-13T00:00:34.828 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.826+0000 7fb8f2ce0640 1 < completed part key=_multipart_H4M1TCN5W6XU.2~MP80FLK566MK.2 size=61657 2024-09-13T00:00:34.832 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.830+0000 7fb8f2ce0640 1 < completed multipart upload key=H4M1TCN5W6XU upload=MP80FLK566MK tag=9KNH7SEI44DF type=write size=3207385 2024-09-13T00:00:34.835 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.834+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~DO6BSDRP7YEE.0 size=1048576 2024-09-13T00:00:34.836 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.834+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~DO6BSDRP7YEE.1 size=507466 2024-09-13T00:00:34.837 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.834+0000 7fb8f2ce0640 1 > prepared multipart upload key=2AHYRZUDFHP4 upload=DO6BSDRP7YEE tag=BSQLXMI7N1C9 type=write size=2604618 2024-09-13T00:00:34.839 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.838+0000 7fb8f2ce0640 1 < completed delete operation key=H5EIP0E9CBWX tag=8UJR6RP6F0RH type=del 2024-09-13T00:00:34.841 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.842+0000 7fb8f2ce0640 1 > prepared operation key=ZFHN2SNTZ5W7 tag=7PPU3AJ76XMW type=del size=2769437 2024-09-13T00:00:34.843 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.842+0000 7fb8f2ce0640 1 < completed part key=_multipart_EJN6HR8I7XQF.2~7NL7LZBU2TO6.0 size=134162 2024-09-13T00:00:34.846 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.846+0000 7fb8f2ce0640 1 < completed multipart upload key=EJN6HR8I7XQF upload=7NL7LZBU2TO6 tag=RL8BU5AHUYEY type=write size=1182738 2024-09-13T00:00:34.849 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.850+0000 7fb8f2ce0640 1 > prepared operation key=2TWD9YLMSYJI tag=NX27XW9B9SRF type=del size=1405995 2024-09-13T00:00:34.851 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.850+0000 7fb8f2ce0640 1 < completed delete operation key=H9AU4HD247YU tag=HVMNHQ2SDY4P type=del 2024-09-13T00:00:34.855 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.854+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~J5Q5SJYYHE6M.0 size=1048576 2024-09-13T00:00:34.857 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.854+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~J5Q5SJYYHE6M.1 size=1048576 2024-09-13T00:00:34.858 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.858+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~J5Q5SJYYHE6M.2 size=628062 2024-09-13T00:00:34.858 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.858+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZTK9AYBSLCOZ upload=J5Q5SJYYHE6M tag=KIYNDBPRS11I type=write size=3773790 2024-09-13T00:00:34.860 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.858+0000 7fb8f2ce0640 1 < completed delete operation key=NX26SI4F2ZMB tag=MSHS9XGR03V3 type=del 2024-09-13T00:00:34.864 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.862+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~8I52E9EY341O.0 size=1048576 2024-09-13T00:00:34.865 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.866+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~8I52E9EY341O.1 size=629907 2024-09-13T00:00:34.865 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.866+0000 7fb8f2ce0640 1 > prepared multipart upload key=8MPUVD37EDN9 upload=8I52E9EY341O tag=SBUSP44ACDTX type=write size=2727059 2024-09-13T00:00:34.870 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.870+0000 7fb8f2ce0640 1 < suggested remove operation key=44EDYFT9VF8Y tag=TESPRJYJV6I9 type=del 2024-09-13T00:00:34.871 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.870+0000 7fb8f2ce0640 1 > prepared operation key=EJN6HR8I7XQF tag=33NGTJHE5ZCR type=del size=3907374 2024-09-13T00:00:34.873 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.874+0000 7fb8f2ce0640 1 < completed delete operation key=5GM88ERM89KT tag=PIAF5WCRTU17 type=del 2024-09-13T00:00:34.875 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.874+0000 7fb8f2ce0640 1 > prepared operation key=2TWD9YLMSYJI tag=4G6CSNRQAB8V type=del size=1409884 2024-09-13T00:00:34.878 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.878+0000 7fb8f2ce0640 -1 < failed to complete operation key=44EDYFT9VF8Y tag=X4GXEDN07KVQ type=del size=1622324: (22) Invalid argument 2024-09-13T00:00:34.880 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.878+0000 7fb8f2ce0640 1 > prepared operation key=NQJQC8CEAC15 tag=2Y5O97012V1A type=write size=491470 2024-09-13T00:00:34.883 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.882+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~ALTEDTMYKNXS.0 size=1048576 2024-09-13T00:00:34.884 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.882+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~ALTEDTMYKNXS.1 size=518133 2024-09-13T00:00:34.888 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.886+0000 7fb8f2ce0640 1 < completed multipart upload key=NQJQC8CEAC15 upload=ALTEDTMYKNXS tag=9NA0LJFAF4IM type=write size=2615285 2024-09-13T00:00:34.892 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.890+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~FBQWK5EL8X2H.0 size=1048576 2024-09-13T00:00:34.894 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.894+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~FBQWK5EL8X2H.1 size=1048576 2024-09-13T00:00:34.896 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.894+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~FBQWK5EL8X2H.2 size=262467 2024-09-13T00:00:34.896 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.894+0000 7fb8f2ce0640 1 > prepared multipart upload key=7OX5OJZ8GXU5 upload=FBQWK5EL8X2H tag=D54JZTLEBOSK type=write size=3408195 2024-09-13T00:00:34.899 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.898+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~I6S6891BRA55.0 size=1048576 2024-09-13T00:00:34.901 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.902+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~I6S6891BRA55.1 size=1048576 2024-09-13T00:00:34.903 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.902+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~I6S6891BRA55.2 size=118665 2024-09-13T00:00:34.908 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.906+0000 7fb8f2ce0640 1 < completed multipart upload key=ZTK9AYBSLCOZ upload=I6S6891BRA55 tag=GMYUIDWM73JJ type=write size=3264393 2024-09-13T00:00:34.911 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.910+0000 7fb8f2ce0640 1 > prepared operation key=2AHYRZUDFHP4 tag=5I9IGN1XYS1P type=del size=1124082 2024-09-13T00:00:34.915 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.914+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~M4PTG46L3G0V.0 size=1048576 2024-09-13T00:00:34.915 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.914+0000 7fb8f2ce0640 1 < canceled part key=_multipart_NQJQC8CEAC15.2~M4PTG46L3G0V.1 size=377317 2024-09-13T00:00:34.920 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.918+0000 7fb8f2ce0640 1 < canceled multipart upload key=NQJQC8CEAC15 upload=M4PTG46L3G0V tag=UE5ZM96VALAC type=write size=2474469 2024-09-13T00:00:34.923 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.922+0000 7fb8f2ce0640 1 > prepared operation key=N3Z5XIG3TZRQ tag=S7UXFO5E9F23 type=del size=631618 2024-09-13T00:00:34.926 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.926+0000 7fb8f2ce0640 1 < canceled operation key=7OX5OJZ8GXU5 tag=1JIROS1H99A1 type=del size=1989564 2024-09-13T00:00:34.929 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.930+0000 7fb8f2ce0640 1 > prepared operation key=EUE3LT0PDIXV tag=HZP06LH51SP8 type=del size=2161076 2024-09-13T00:00:34.932 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.930+0000 7fb8f2ce0640 1 < completed part key=_multipart_N3Z5XIG3TZRQ.2~61OKJFN5HP72.0 size=1048576 2024-09-13T00:00:34.932 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.930+0000 7fb8f2ce0640 1 < canceled part key=_multipart_N3Z5XIG3TZRQ.2~61OKJFN5HP72.1 size=1039515 2024-09-13T00:00:34.936 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.934+0000 7fb8f2ce0640 1 < canceled multipart upload key=N3Z5XIG3TZRQ upload=61OKJFN5HP72 tag=629FSXYXL742 type=write size=3136667 2024-09-13T00:00:34.941 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.942+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~VU1FNKB104JA.0 size=1048576 2024-09-13T00:00:34.943 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.942+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~VU1FNKB104JA.1 size=1048576 2024-09-13T00:00:34.945 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.946+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~VU1FNKB104JA.2 size=296371 2024-09-13T00:00:34.945 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.946+0000 7fb8f2ce0640 1 > prepared multipart upload key=7OX5OJZ8GXU5 upload=VU1FNKB104JA tag=X4PCJIVVQ7EF type=write size=3442099 2024-09-13T00:00:34.947 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.946+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~53NN95ONVV37.0 size=1048576 2024-09-13T00:00:34.950 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.950+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~53NN95ONVV37.1 size=1048576 2024-09-13T00:00:34.952 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.950+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~53NN95ONVV37.2 size=535109 2024-09-13T00:00:34.958 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.958+0000 7fb8f2ce0640 1 < completed multipart upload key=8MPUVD37EDN9 upload=53NN95ONVV37 tag=DG2TVFSWBZEW type=write size=3680837 2024-09-13T00:00:34.964 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.962+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~8KW6XHN85JKK.0 size=947964 2024-09-13T00:00:34.965 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.962+0000 7fb8f2ce0640 1 > prepared multipart upload key=70F7D20HI504 upload=8KW6XHN85JKK tag=6P8O6Y34NEBM type=write size=1996540 2024-09-13T00:00:34.969 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.966+0000 7fb8f2ce0640 1 < completed delete operation key=W8XZ26311RJX tag=WBQ02XINC60R type=del 2024-09-13T00:00:34.972 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.970+0000 7fb8f2ce0640 1 > prepared operation key=P8OYS5ZE1NX0 tag=WLERO3XR0P02 type=del size=724935 2024-09-13T00:00:34.981 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.978+0000 7fb8f2ce0640 1 < suggested remove operation key=H4M1TCN5W6XU tag=W9WI5L293OWB type=del 2024-09-13T00:00:34.988 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.986+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NX26SI4F2ZMB.2~AM17AABO93L2.0 size=1048576 2024-09-13T00:00:34.991 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.990+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NX26SI4F2ZMB.2~AM17AABO93L2.1 size=504889 2024-09-13T00:00:34.991 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.990+0000 7fb8f2ce0640 1 > prepared multipart upload key=NX26SI4F2ZMB upload=AM17AABO93L2 tag=4IUNBTMMDBJL type=write size=2602041 2024-09-13T00:00:34.995 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.994+0000 7fb8f2ce0640 1 < completed part key=_multipart_2AHYRZUDFHP4.2~DO6BSDRP7YEE.0 size=1048576 2024-09-13T00:00:34.999 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:34.998+0000 7fb8f2ce0640 1 < completed part key=_multipart_2AHYRZUDFHP4.2~DO6BSDRP7YEE.1 size=507466 2024-09-13T00:00:35.005 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.006+0000 7fb8f2ce0640 1 < completed multipart upload key=2AHYRZUDFHP4 upload=DO6BSDRP7YEE tag=BSQLXMI7N1C9 type=write size=2604618 2024-09-13T00:00:35.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:34 smithi136 bash[76371]: cluster 2024-09-13T00:00:33.125329+0000 mgr.y (mgr.24661) 703 : cluster [DBG] pgmap v1379: 137 pgs: 6 creating+peering, 2 unknown, 129 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 5.8 KiB/s wr, 21 op/s 2024-09-13T00:00:35.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:34 smithi136 bash[76371]: audit 2024-09-13T00:00:34.373385+0000 mon.a (mon.0) 2070 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:00:35.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:34 smithi136 bash[75164]: cluster 2024-09-13T00:00:33.125329+0000 mgr.y (mgr.24661) 703 : cluster [DBG] pgmap v1379: 137 pgs: 6 creating+peering, 2 unknown, 129 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 5.8 KiB/s wr, 21 op/s 2024-09-13T00:00:35.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:34 smithi136 bash[75164]: audit 2024-09-13T00:00:34.373385+0000 mon.a (mon.0) 2070 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:00:35.009 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.010+0000 7fb8f2ce0640 1 > prepared operation key=P8OYS5ZE1NX0 tag=SLUNQLQZMK0V type=del size=678374 2024-09-13T00:00:35.013 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.014+0000 7fb8f2ce0640 1 < canceled operation key=ZFHN2SNTZ5W7 tag=7PPU3AJ76XMW type=del size=2769437 2024-09-13T00:00:35.021 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.022+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~0IV5AQRT5KGJ.0 size=1048576 2024-09-13T00:00:35.024 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.022+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~0IV5AQRT5KGJ.1 size=251169 2024-09-13T00:00:35.024 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.022+0000 7fb8f2ce0640 1 > prepared multipart upload key=7OX5OJZ8GXU5 upload=0IV5AQRT5KGJ tag=N8BAIT6C0PL8 type=write size=2348321 2024-09-13T00:00:35.029 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.026+0000 7fb8f2ce0640 1 < completed delete operation key=2TWD9YLMSYJI tag=NX27XW9B9SRF type=del 2024-09-13T00:00:35.036 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.034+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~24SOKN9QBVKJ.0 size=1048576 2024-09-13T00:00:35.040 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.038+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~24SOKN9QBVKJ.1 size=627899 2024-09-13T00:00:35.040 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.038+0000 7fb8f2ce0640 1 > prepared multipart upload key=1HX06CCZJ1DD upload=24SOKN9QBVKJ tag=ZQUWWN42MC8S type=write size=2725051 2024-09-13T00:00:35.045 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.042+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~J5Q5SJYYHE6M.0 size=1048576 2024-09-13T00:00:35.048 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.046+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~J5Q5SJYYHE6M.1 size=1048576 2024-09-13T00:00:35.048 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.046+0000 7fb8f2ce0640 1 < canceled part key=_multipart_ZTK9AYBSLCOZ.2~J5Q5SJYYHE6M.2 size=628062 2024-09-13T00:00:35.055 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.054+0000 7fb8f2ce0640 1 < canceled multipart upload key=ZTK9AYBSLCOZ upload=J5Q5SJYYHE6M tag=KIYNDBPRS11I type=write size=3773790 2024-09-13T00:00:35.059 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.058+0000 7fb8f2ce0640 1 > prepared operation key=QMZREZ1QEDNJ tag=Y3IWVA3WCYNS type=del size=2736394 2024-09-13T00:00:35.064 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.062+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~8I52E9EY341O.0 size=1048576 2024-09-13T00:00:35.069 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.066+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~8I52E9EY341O.1 size=629907 2024-09-13T00:00:35.074 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.074+0000 7fb8f2ce0640 1 < completed multipart upload key=8MPUVD37EDN9 upload=8I52E9EY341O tag=SBUSP44ACDTX type=write size=2727059 2024-09-13T00:00:35.082 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.082+0000 7fb8f2ce0640 1 > prepared part key=_multipart_44EDYFT9VF8Y.2~1A3M0CXEXO1W.0 size=241126 2024-09-13T00:00:35.082 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.082+0000 7fb8f2ce0640 1 > prepared multipart upload key=44EDYFT9VF8Y upload=1A3M0CXEXO1W tag=WTFPVU19G4D3 type=write size=1289702 2024-09-13T00:00:35.086 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.086+0000 7fb8f2ce0640 1 < completed delete operation key=EJN6HR8I7XQF tag=33NGTJHE5ZCR type=del 2024-09-13T00:00:35.090 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.090+0000 7fb8f2ce0640 1 > prepared operation key=70F7D20HI504 tag=7K2ES6PQI8Y6 type=del size=828955 2024-09-13T00:00:35.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:34 smithi181 bash[67587]: cluster 2024-09-13T00:00:33.125329+0000 mgr.y (mgr.24661) 703 : cluster [DBG] pgmap v1379: 137 pgs: 6 creating+peering, 2 unknown, 129 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 5.8 KiB/s wr, 21 op/s 2024-09-13T00:00:35.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:34 smithi181 bash[67587]: audit 2024-09-13T00:00:34.373385+0000 mon.a (mon.0) 2070 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:00:35.098 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.098+0000 7fb8f2ce0640 1 < suggested remove operation key=2TWD9YLMSYJI tag=4G6CSNRQAB8V type=del 2024-09-13T00:00:35.102 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.102+0000 7fb8f2ce0640 1 > prepared operation key=N3Z5XIG3TZRQ tag=FUO43CMCZN05 type=del size=109583 2024-09-13T00:00:35.109 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.110+0000 7fb8f2ce0640 1 < suggested remove operation key=NQJQC8CEAC15 tag=2Y5O97012V1A type=write 2024-09-13T00:00:35.113 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.114+0000 7fb8f2ce0640 1 > prepared operation key=63FJDWBO2T9S tag=FHRPC5PFU19H type=write size=411763 2024-09-13T00:00:35.117 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.118+0000 7fb8f2ce0640 1 < completed part key=_multipart_7OX5OJZ8GXU5.2~FBQWK5EL8X2H.0 size=1048576 2024-09-13T00:00:35.120 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.118+0000 7fb8f2ce0640 1 < completed part key=_multipart_7OX5OJZ8GXU5.2~FBQWK5EL8X2H.1 size=1048576 2024-09-13T00:00:35.123 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.122+0000 7fb8f2ce0640 1 < completed part key=_multipart_7OX5OJZ8GXU5.2~FBQWK5EL8X2H.2 size=262467 2024-09-13T00:00:35.130 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.130+0000 7fb8f2ce0640 1 < completed multipart upload key=7OX5OJZ8GXU5 upload=FBQWK5EL8X2H tag=D54JZTLEBOSK type=write size=3408195 2024-09-13T00:00:35.140 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.138+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZFHN2SNTZ5W7.2~HDAW7W8PFA7I.0 size=1048576 2024-09-13T00:00:35.141 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.142+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZFHN2SNTZ5W7.2~HDAW7W8PFA7I.1 size=48659 2024-09-13T00:00:35.141 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.142+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZFHN2SNTZ5W7 upload=HDAW7W8PFA7I tag=3JCMIBGFDJCU type=write size=2145811 2024-09-13T00:00:35.146 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.146+0000 7fb8f2ce0640 1 < completed delete operation key=2AHYRZUDFHP4 tag=5I9IGN1XYS1P type=del 2024-09-13T00:00:35.152 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.150+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZFHN2SNTZ5W7.2~BC5SZ98Q624Q.0 size=1048576 2024-09-13T00:00:35.156 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.154+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZFHN2SNTZ5W7.2~BC5SZ98Q624Q.1 size=944925 2024-09-13T00:00:35.156 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.154+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZFHN2SNTZ5W7 upload=BC5SZ98Q624Q tag=F7LHRJ48D8T2 type=write size=3042077 2024-09-13T00:00:35.164 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.162+0000 7fb8f2ce0640 1 < suggested update operation key=N3Z5XIG3TZRQ tag=S7UXFO5E9F23 type=del size=631618 2024-09-13T00:00:35.168 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.166+0000 7fb8f2ce0640 1 > prepared operation key=8Z6U7OOA0GBH tag=A706Q4RI74WQ type=del size=682392 2024-09-13T00:00:35.172 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.170+0000 7fb8f2ce0640 1 < canceled operation key=EUE3LT0PDIXV tag=HZP06LH51SP8 type=del size=2161076 2024-09-13T00:00:35.180 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.178+0000 7fb8f2ce0640 1 > prepared part key=_multipart_J8NGDSF9XRTP.2~D9XXJRF1U1SB.0 size=1048576 2024-09-13T00:00:35.183 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.182+0000 7fb8f2ce0640 1 > prepared part key=_multipart_J8NGDSF9XRTP.2~D9XXJRF1U1SB.1 size=1048576 2024-09-13T00:00:35.187 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.186+0000 7fb8f2ce0640 1 > prepared part key=_multipart_J8NGDSF9XRTP.2~D9XXJRF1U1SB.2 size=653956 2024-09-13T00:00:35.187 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.186+0000 7fb8f2ce0640 1 > prepared multipart upload key=J8NGDSF9XRTP upload=D9XXJRF1U1SB tag=67U2IR92RJF3 type=write size=3799684 2024-09-13T00:00:35.190 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.190+0000 7fb8f2ce0640 1 < completed part key=_multipart_7OX5OJZ8GXU5.2~VU1FNKB104JA.0 size=1048576 2024-09-13T00:00:35.190 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.190+0000 7fb8f2ce0640 1 < canceled part key=_multipart_7OX5OJZ8GXU5.2~VU1FNKB104JA.1 size=1048576 2024-09-13T00:00:35.193 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.194+0000 7fb8f2ce0640 1 < completed part key=_multipart_7OX5OJZ8GXU5.2~VU1FNKB104JA.2 size=296371 2024-09-13T00:00:35.200 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.198+0000 7fb8f2ce0640 1 < canceled multipart upload key=7OX5OJZ8GXU5 upload=VU1FNKB104JA tag=X4PCJIVVQ7EF type=write size=3442099 2024-09-13T00:00:35.207 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.206+0000 7fb8f2ce0640 1 > prepared part key=_multipart_5GM88ERM89KT.2~6S2VH3W7YUZV.0 size=1048576 2024-09-13T00:00:35.210 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.210+0000 7fb8f2ce0640 1 > prepared part key=_multipart_5GM88ERM89KT.2~6S2VH3W7YUZV.1 size=956821 2024-09-13T00:00:35.211 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.210+0000 7fb8f2ce0640 1 > prepared multipart upload key=5GM88ERM89KT upload=6S2VH3W7YUZV tag=WWOWS1DLNR48 type=write size=3053973 2024-09-13T00:00:35.214 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.214+0000 7fb8f2ce0640 1 < completed part key=_multipart_70F7D20HI504.2~8KW6XHN85JKK.0 size=947964 2024-09-13T00:00:35.221 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.222+0000 7fb8f2ce0640 1 < completed multipart upload key=70F7D20HI504 upload=8KW6XHN85JKK tag=6P8O6Y34NEBM type=write size=1996540 2024-09-13T00:00:35.224 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.222+0000 7fb8f2ce0640 1 > prepared operation key=FX0NCGLK0KPJ tag=20ZN4ZVHG921 type=del size=2893094 2024-09-13T00:00:35.232 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.230+0000 7fb8f2ce0640 1 < suggested remove operation key=P8OYS5ZE1NX0 tag=WLERO3XR0P02 type=del 2024-09-13T00:00:35.236 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.234+0000 7fb8f2ce0640 1 > prepared operation key=FX0NCGLK0KPJ tag=O1SCLF4O0DFW type=del size=1674925 2024-09-13T00:00:35.240 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.238+0000 7fb8f2ce0640 1 < completed part key=_multipart_NX26SI4F2ZMB.2~AM17AABO93L2.0 size=1048576 2024-09-13T00:00:35.243 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.242+0000 7fb8f2ce0640 1 < completed part key=_multipart_NX26SI4F2ZMB.2~AM17AABO93L2.1 size=504889 2024-09-13T00:00:35.250 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.250+0000 7fb8f2ce0640 1 < completed multipart upload key=NX26SI4F2ZMB upload=AM17AABO93L2 tag=4IUNBTMMDBJL type=write size=2602041 2024-09-13T00:00:35.254 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.254+0000 7fb8f2ce0640 1 > prepared operation key=70F7D20HI504 tag=NEH1KSZHPFW3 type=del size=3412444 2024-09-13T00:00:35.256 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.254+0000 7fb8f2ce0640 -1 < no bi entry to suggest for operation key=P8OYS5ZE1NX0 tag=SLUNQLQZMK0V type=del size=678374: (2) No such file or directory 2024-09-13T00:00:35.263 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.262+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EJN6HR8I7XQF.2~N99C1N0TCE4U.0 size=1048576 2024-09-13T00:00:35.266 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.266+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EJN6HR8I7XQF.2~N99C1N0TCE4U.1 size=1048576 2024-09-13T00:00:35.269 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.270+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EJN6HR8I7XQF.2~N99C1N0TCE4U.2 size=645933 2024-09-13T00:00:35.269 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.270+0000 7fb8f2ce0640 1 > prepared multipart upload key=EJN6HR8I7XQF upload=N99C1N0TCE4U tag=5EWV3L2HD3B2 type=write size=3791661 2024-09-13T00:00:35.273 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.274+0000 7fb8f2ce0640 1 < completed part key=_multipart_7OX5OJZ8GXU5.2~0IV5AQRT5KGJ.0 size=1048576 2024-09-13T00:00:35.277 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.278+0000 7fb8f2ce0640 1 < completed part key=_multipart_7OX5OJZ8GXU5.2~0IV5AQRT5KGJ.1 size=251169 2024-09-13T00:00:35.283 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.282+0000 7fb8f2ce0640 1 < completed multipart upload key=7OX5OJZ8GXU5 upload=0IV5AQRT5KGJ tag=N8BAIT6C0PL8 type=write size=2348321 2024-09-13T00:00:35.291 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.290+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~WTHDNDIOIEI8.0 size=692221 2024-09-13T00:00:35.291 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.290+0000 7fb8f2ce0640 1 > prepared multipart upload key=2AHYRZUDFHP4 upload=WTHDNDIOIEI8 tag=M74LWPA3HWHQ type=write size=1740797 2024-09-13T00:00:35.295 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.294+0000 7fb8f2ce0640 1 < completed part key=_multipart_1HX06CCZJ1DD.2~24SOKN9QBVKJ.0 size=1048576 2024-09-13T00:00:35.299 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.298+0000 7fb8f2ce0640 1 < completed part key=_multipart_1HX06CCZJ1DD.2~24SOKN9QBVKJ.1 size=627899 2024-09-13T00:00:35.306 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.306+0000 7fb8f2ce0640 1 < completed multipart upload key=1HX06CCZJ1DD upload=24SOKN9QBVKJ tag=ZQUWWN42MC8S type=write size=2725051 2024-09-13T00:00:35.310 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.310+0000 7fb8f2ce0640 1 > prepared operation key=J8NGDSF9XRTP tag=9TYCXD3L8H5W type=del size=3082496 2024-09-13T00:00:35.314 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.314+0000 7fb8f2ce0640 1 < completed delete operation key=QMZREZ1QEDNJ tag=Y3IWVA3WCYNS type=del 2024-09-13T00:00:35.319 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.318+0000 7fb8f2ce0640 1 > prepared operation key=2TWD9YLMSYJI tag=W59R1RW20JUC type=del size=2820155 2024-09-13T00:00:35.323 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.322+0000 7fb8f2ce0640 1 < completed part key=_multipart_44EDYFT9VF8Y.2~1A3M0CXEXO1W.0 size=241126 2024-09-13T00:00:35.330 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.330+0000 7fb8f2ce0640 1 < completed multipart upload key=44EDYFT9VF8Y upload=1A3M0CXEXO1W tag=WTFPVU19G4D3 type=write size=1289702 2024-09-13T00:00:35.338 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.338+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZFHN2SNTZ5W7.2~JMDU0MW89WIM.0 size=1026461 2024-09-13T00:00:35.338 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.338+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZFHN2SNTZ5W7 upload=JMDU0MW89WIM tag=CE3Y33HKKO2X type=write size=2075037 2024-09-13T00:00:35.344 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.342+0000 7fb8f2ce0640 1 < completed delete operation key=70F7D20HI504 tag=7K2ES6PQI8Y6 type=del 2024-09-13T00:00:35.349 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.350+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZFHN2SNTZ5W7.2~5D4AW54O4YEO.0 size=1048576 2024-09-13T00:00:35.352 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.350+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZFHN2SNTZ5W7.2~5D4AW54O4YEO.1 size=1048576 2024-09-13T00:00:35.355 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.354+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZFHN2SNTZ5W7.2~5D4AW54O4YEO.2 size=713054 2024-09-13T00:00:35.355 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.354+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZFHN2SNTZ5W7 upload=5D4AW54O4YEO tag=KX1OIHYIAT2M type=write size=3858782 2024-09-13T00:00:35.360 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.358+0000 7fb8f2ce0640 -1 < failed to complete operation key=N3Z5XIG3TZRQ tag=FUO43CMCZN05 type=del size=109583: (22) Invalid argument 2024-09-13T00:00:35.368 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.366+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~HA2WC3MNE483.0 size=1048576 2024-09-13T00:00:35.371 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.370+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~HA2WC3MNE483.1 size=1048576 2024-09-13T00:00:35.374 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.374+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~HA2WC3MNE483.2 size=1004659 2024-09-13T00:00:35.374 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.374+0000 7fb8f2ce0640 1 > prepared multipart upload key=63FJDWBO2T9S upload=HA2WC3MNE483 tag=GHC70040ZGNE type=write size=4150387 2024-09-13T00:00:35.379 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.378+0000 7fb8f2ce0640 1 < completed write operation key=63FJDWBO2T9S tag=FHRPC5PFU19H type=write size=411763 2024-09-13T00:00:35.384 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.382+0000 7fb8f2ce0640 1 > prepared operation key=63FJDWBO2T9S tag=PFO34GEO4T9N type=del size=1748039 2024-09-13T00:00:35.389 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.390+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZFHN2SNTZ5W7.2~HDAW7W8PFA7I.0 size=1048576 2024-09-13T00:00:35.393 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.394+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZFHN2SNTZ5W7.2~HDAW7W8PFA7I.1 size=48659 2024-09-13T00:00:35.400 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.398+0000 7fb8f2ce0640 1 < completed multipart upload key=ZFHN2SNTZ5W7 upload=HDAW7W8PFA7I tag=3JCMIBGFDJCU type=write size=2145811 2024-09-13T00:00:35.409 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.406+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2TWD9YLMSYJI.2~E5WV976Q91U5.0 size=1048576 2024-09-13T00:00:35.412 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.410+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2TWD9YLMSYJI.2~E5WV976Q91U5.1 size=1048576 2024-09-13T00:00:35.415 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.414+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2TWD9YLMSYJI.2~E5WV976Q91U5.2 size=245367 2024-09-13T00:00:35.415 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.414+0000 7fb8f2ce0640 1 > prepared multipart upload key=2TWD9YLMSYJI upload=E5WV976Q91U5 tag=LG7IWWTG3005 type=write size=3391095 2024-09-13T00:00:35.420 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.418+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZFHN2SNTZ5W7.2~BC5SZ98Q624Q.0 size=1048576 2024-09-13T00:00:35.424 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.422+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZFHN2SNTZ5W7.2~BC5SZ98Q624Q.1 size=944925 2024-09-13T00:00:35.431 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.430+0000 7fb8f2ce0640 1 < completed multipart upload key=ZFHN2SNTZ5W7 upload=BC5SZ98Q624Q tag=F7LHRJ48D8T2 type=write size=3042077 2024-09-13T00:00:35.438 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.438+0000 7fb8f2ce0640 1 > prepared part key=_multipart_F0QCVR9E08WE.2~128AHGYDL94F.0 size=1048576 2024-09-13T00:00:35.442 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.442+0000 7fb8f2ce0640 1 > prepared part key=_multipart_F0QCVR9E08WE.2~128AHGYDL94F.1 size=1048576 2024-09-13T00:00:35.445 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.446+0000 7fb8f2ce0640 1 > prepared part key=_multipart_F0QCVR9E08WE.2~128AHGYDL94F.2 size=179436 2024-09-13T00:00:35.445 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.446+0000 7fb8f2ce0640 1 > prepared multipart upload key=F0QCVR9E08WE upload=128AHGYDL94F tag=VT33UZTTU1UM type=write size=3325164 2024-09-13T00:00:35.453 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.454+0000 7fb8f2ce0640 1 < suggested remove operation key=8Z6U7OOA0GBH tag=A706Q4RI74WQ type=del 2024-09-13T00:00:35.457 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.458+0000 7fb8f2ce0640 1 > prepared operation key=W8XZ26311RJX tag=O8CLF4A595GW type=write size=11670 2024-09-13T00:00:35.462 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.462+0000 7fb8f2ce0640 1 < completed part key=_multipart_J8NGDSF9XRTP.2~D9XXJRF1U1SB.0 size=1048576 2024-09-13T00:00:35.465 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.466+0000 7fb8f2ce0640 1 < completed part key=_multipart_J8NGDSF9XRTP.2~D9XXJRF1U1SB.1 size=1048576 2024-09-13T00:00:35.468 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.466+0000 7fb8f2ce0640 1 < completed part key=_multipart_J8NGDSF9XRTP.2~D9XXJRF1U1SB.2 size=653956 2024-09-13T00:00:35.475 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.474+0000 7fb8f2ce0640 1 < completed multipart upload key=J8NGDSF9XRTP upload=D9XXJRF1U1SB tag=67U2IR92RJF3 type=write size=3799684 2024-09-13T00:00:35.479 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.478+0000 7fb8f2ce0640 1 > prepared operation key=ZTK9AYBSLCOZ tag=B1FTKX1440KD type=del size=2466161 2024-09-13T00:00:35.484 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.482+0000 7fb8f2ce0640 1 < completed part key=_multipart_5GM88ERM89KT.2~6S2VH3W7YUZV.0 size=1048576 2024-09-13T00:00:35.488 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.486+0000 7fb8f2ce0640 1 < completed part key=_multipart_5GM88ERM89KT.2~6S2VH3W7YUZV.1 size=956821 2024-09-13T00:00:35.495 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.494+0000 7fb8f2ce0640 1 < completed multipart upload key=5GM88ERM89KT upload=6S2VH3W7YUZV tag=WWOWS1DLNR48 type=write size=3053973 2024-09-13T00:00:35.499 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.498+0000 7fb8f2ce0640 1 > prepared operation key=FX0NCGLK0KPJ tag=8D8154THDIMW type=write size=618163 2024-09-13T00:00:35.507 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.506+0000 7fb8f2ce0640 1 < suggested remove operation key=FX0NCGLK0KPJ tag=20ZN4ZVHG921 type=del 2024-09-13T00:00:35.512 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.510+0000 7fb8f2ce0640 1 > prepared operation key=J8NGDSF9XRTP tag=DX2QU54XNM7A type=write size=624666 2024-09-13T00:00:35.514 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.514+0000 7fb8f2ce0640 -1 < no bi entry to suggest for operation key=FX0NCGLK0KPJ tag=O1SCLF4O0DFW type=del size=1674925: (2) No such file or directory 2024-09-13T00:00:35.522 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.522+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H5EIP0E9CBWX.2~X78MDPL8ZJU6.0 size=1048576 2024-09-13T00:00:35.525 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.526+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H5EIP0E9CBWX.2~X78MDPL8ZJU6.1 size=1048576 2024-09-13T00:00:35.528 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.526+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H5EIP0E9CBWX.2~X78MDPL8ZJU6.2 size=59864 2024-09-13T00:00:35.528 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.526+0000 7fb8f2ce0640 1 > prepared multipart upload key=H5EIP0E9CBWX upload=X78MDPL8ZJU6 tag=BOKG1R8FFKW0 type=write size=3205592 2024-09-13T00:00:35.532 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.530+0000 7fb8f2ce0640 1 < canceled operation key=70F7D20HI504 tag=NEH1KSZHPFW3 type=del size=3412444 2024-09-13T00:00:35.540 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.538+0000 7fb8f2ce0640 1 > prepared part key=_multipart_QMZREZ1QEDNJ.2~RRLT8D8H96E4.0 size=1048576 2024-09-13T00:00:35.543 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.542+0000 7fb8f2ce0640 1 > prepared part key=_multipart_QMZREZ1QEDNJ.2~RRLT8D8H96E4.1 size=233110 2024-09-13T00:00:35.543 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.542+0000 7fb8f2ce0640 1 > prepared multipart upload key=QMZREZ1QEDNJ upload=RRLT8D8H96E4 tag=6106ZJL22HDV type=write size=2330262 2024-09-13T00:00:35.548 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.546+0000 7fb8f2ce0640 1 < completed part key=_multipart_EJN6HR8I7XQF.2~N99C1N0TCE4U.0 size=1048576 2024-09-13T00:00:35.552 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.550+0000 7fb8f2ce0640 1 < completed part key=_multipart_EJN6HR8I7XQF.2~N99C1N0TCE4U.1 size=1048576 2024-09-13T00:00:35.555 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.554+0000 7fb8f2ce0640 1 < completed part key=_multipart_EJN6HR8I7XQF.2~N99C1N0TCE4U.2 size=645933 2024-09-13T00:00:35.562 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.562+0000 7fb8f2ce0640 1 < completed multipart upload key=EJN6HR8I7XQF upload=N99C1N0TCE4U tag=5EWV3L2HD3B2 type=write size=3791661 2024-09-13T00:00:35.566 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.566+0000 7fb8f2ce0640 1 > prepared operation key=7OX5OJZ8GXU5 tag=IJVAYY1OKDP0 type=del size=2440571 2024-09-13T00:00:35.570 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.570+0000 7fb8f2ce0640 1 < completed part key=_multipart_2AHYRZUDFHP4.2~WTHDNDIOIEI8.0 size=692221 2024-09-13T00:00:35.577 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.578+0000 7fb8f2ce0640 1 < completed multipart upload key=2AHYRZUDFHP4 upload=WTHDNDIOIEI8 tag=M74LWPA3HWHQ type=write size=1740797 2024-09-13T00:00:35.581 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.582+0000 7fb8f2ce0640 1 > prepared operation key=P8OYS5ZE1NX0 tag=RWZ0LK2DSK7D type=write size=323261 2024-09-13T00:00:35.585 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.586+0000 7fb8f2ce0640 1 < completed delete operation key=J8NGDSF9XRTP tag=9TYCXD3L8H5W type=del 2024-09-13T00:00:35.592 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.590+0000 7fb8f2ce0640 1 > prepared part key=_multipart_N3Z5XIG3TZRQ.2~Z8AMPETZWPJ2.0 size=1048576 2024-09-13T00:00:35.595 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.594+0000 7fb8f2ce0640 1 > prepared part key=_multipart_N3Z5XIG3TZRQ.2~Z8AMPETZWPJ2.1 size=309182 2024-09-13T00:00:35.595 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.594+0000 7fb8f2ce0640 1 > prepared multipart upload key=N3Z5XIG3TZRQ upload=Z8AMPETZWPJ2 tag=9BB76ZZSLXIS type=write size=2406334 2024-09-13T00:00:35.599 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.598+0000 7fb8f2ce0640 1 < completed delete operation key=2TWD9YLMSYJI tag=W59R1RW20JUC type=del 2024-09-13T00:00:35.606 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.606+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EJN6HR8I7XQF.2~P5OB4S9ELN0D.0 size=1048576 2024-09-13T00:00:35.609 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.606+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EJN6HR8I7XQF.2~P5OB4S9ELN0D.1 size=1048576 2024-09-13T00:00:35.612 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.610+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EJN6HR8I7XQF.2~P5OB4S9ELN0D.2 size=559189 2024-09-13T00:00:35.612 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.610+0000 7fb8f2ce0640 1 > prepared multipart upload key=EJN6HR8I7XQF upload=P5OB4S9ELN0D tag=RJNJE0VQF6A1 type=write size=3704917 2024-09-13T00:00:35.616 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.614+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZFHN2SNTZ5W7.2~JMDU0MW89WIM.0 size=1026461 2024-09-13T00:00:35.622 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.622+0000 7fb8f2ce0640 1 < completed multipart upload key=ZFHN2SNTZ5W7 upload=JMDU0MW89WIM tag=CE3Y33HKKO2X type=write size=2075037 2024-09-13T00:00:35.625 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.626+0000 7fb8f2ce0640 1 > prepared operation key=8Z6U7OOA0GBH tag=K8D5VTK5XT53 type=del size=3330145 2024-09-13T00:00:35.629 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.630+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZFHN2SNTZ5W7.2~5D4AW54O4YEO.0 size=1048576 2024-09-13T00:00:35.632 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.630+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZFHN2SNTZ5W7.2~5D4AW54O4YEO.1 size=1048576 2024-09-13T00:00:35.634 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.634+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZFHN2SNTZ5W7.2~5D4AW54O4YEO.2 size=713054 2024-09-13T00:00:35.641 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.638+0000 7fb8f2ce0640 1 < completed multipart upload key=ZFHN2SNTZ5W7 upload=5D4AW54O4YEO tag=KX1OIHYIAT2M type=write size=3858782 2024-09-13T00:00:35.645 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.642+0000 7fb8f2ce0640 1 > prepared operation key=FX0NCGLK0KPJ tag=K5XX5WR6F55K type=del size=3154679 2024-09-13T00:00:35.648 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.646+0000 7fb8f2ce0640 1 < completed part key=_multipart_63FJDWBO2T9S.2~HA2WC3MNE483.0 size=1048576 2024-09-13T00:00:35.652 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.650+0000 7fb8f2ce0640 1 < completed part key=_multipart_63FJDWBO2T9S.2~HA2WC3MNE483.1 size=1048576 2024-09-13T00:00:35.654 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.654+0000 7fb8f2ce0640 1 < completed part key=_multipart_63FJDWBO2T9S.2~HA2WC3MNE483.2 size=1004659 2024-09-13T00:00:35.661 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.662+0000 7fb8f2ce0640 1 < completed multipart upload key=63FJDWBO2T9S upload=HA2WC3MNE483 tag=GHC70040ZGNE type=write size=4150387 2024-09-13T00:00:35.668 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.666+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H5EIP0E9CBWX.2~PPSZWUARRWJ4.0 size=89318 2024-09-13T00:00:35.668 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.666+0000 7fb8f2ce0640 1 > prepared multipart upload key=H5EIP0E9CBWX upload=PPSZWUARRWJ4 tag=AWIY2976TDM0 type=write size=1137894 2024-09-13T00:00:35.672 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.670+0000 7fb8f2ce0640 1 < completed delete operation key=63FJDWBO2T9S tag=PFO34GEO4T9N type=del 2024-09-13T00:00:35.679 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.678+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZFHN2SNTZ5W7.2~1KYCO9V5AWHB.0 size=874187 2024-09-13T00:00:35.679 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.678+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZFHN2SNTZ5W7 upload=1KYCO9V5AWHB tag=J5ZEGDTRDVXV type=write size=1922763 2024-09-13T00:00:35.683 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.682+0000 7fb8f2ce0640 1 < completed part key=_multipart_2TWD9YLMSYJI.2~E5WV976Q91U5.0 size=1048576 2024-09-13T00:00:35.685 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.686+0000 7fb8f2ce0640 1 < completed part key=_multipart_2TWD9YLMSYJI.2~E5WV976Q91U5.1 size=1048576 2024-09-13T00:00:35.688 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.686+0000 7fb8f2ce0640 1 < completed part key=_multipart_2TWD9YLMSYJI.2~E5WV976Q91U5.2 size=245367 2024-09-13T00:00:35.695 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.694+0000 7fb8f2ce0640 1 < completed multipart upload key=2TWD9YLMSYJI upload=E5WV976Q91U5 tag=LG7IWWTG3005 type=write size=3391095 2024-09-13T00:00:35.699 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.698+0000 7fb8f2ce0640 1 > prepared operation key=EUE3LT0PDIXV tag=IC3KYFJAGZ27 type=del size=2096387 2024-09-13T00:00:35.700 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.698+0000 7fb8f2ce0640 1 < canceled part key=_multipart_F0QCVR9E08WE.2~128AHGYDL94F.0 size=1048576 2024-09-13T00:00:35.703 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.702+0000 7fb8f2ce0640 1 < completed part key=_multipart_F0QCVR9E08WE.2~128AHGYDL94F.1 size=1048576 2024-09-13T00:00:35.707 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.706+0000 7fb8f2ce0640 1 < completed part key=_multipart_F0QCVR9E08WE.2~128AHGYDL94F.2 size=179436 2024-09-13T00:00:35.713 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.714+0000 7fb8f2ce0640 1 < canceled multipart upload key=F0QCVR9E08WE upload=128AHGYDL94F tag=VT33UZTTU1UM type=write size=3325164 2024-09-13T00:00:35.720 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.718+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~QTR86NFSCT6P.0 size=307364 2024-09-13T00:00:35.720 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.718+0000 7fb8f2ce0640 1 > prepared multipart upload key=PVYZQLGYT3XU upload=QTR86NFSCT6P tag=8JSQZM67ZTSQ type=write size=1355940 2024-09-13T00:00:35.724 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.722+0000 7fb8f2ce0640 1 < completed write operation key=W8XZ26311RJX tag=O8CLF4A595GW type=write size=11670 2024-09-13T00:00:35.731 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.730+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EJN6HR8I7XQF.2~6H2DB6ULI63O.0 size=1048576 2024-09-13T00:00:35.734 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.734+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EJN6HR8I7XQF.2~6H2DB6ULI63O.1 size=637186 2024-09-13T00:00:35.734 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.734+0000 7fb8f2ce0640 1 > prepared multipart upload key=EJN6HR8I7XQF upload=6H2DB6ULI63O tag=N5MIKCXOEA2M type=write size=2734338 2024-09-13T00:00:35.737 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.738+0000 7fb8f2ce0640 1 < completed delete operation key=ZTK9AYBSLCOZ tag=B1FTKX1440KD type=del 2024-09-13T00:00:35.743 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.742+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EJN6HR8I7XQF.2~BHY80FCW3E8C.0 size=395237 2024-09-13T00:00:35.743 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.742+0000 7fb8f2ce0640 1 > prepared multipart upload key=EJN6HR8I7XQF upload=BHY80FCW3E8C tag=BFHK9L5OVGT1 type=write size=1443813 2024-09-13T00:00:35.749 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.750+0000 7fb8f2ce0640 1 < suggested remove operation key=FX0NCGLK0KPJ tag=8D8154THDIMW type=write 2024-09-13T00:00:35.753 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.754+0000 7fb8f2ce0640 1 > prepared operation key=PVYZQLGYT3XU tag=H4AMC3NY9X49 type=del size=2423544 2024-09-13T00:00:35.756 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.754+0000 7fb8f2ce0640 1 < completed write operation key=J8NGDSF9XRTP tag=DX2QU54XNM7A type=write size=624666 2024-09-13T00:00:35.759 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.758+0000 7fb8f2ce0640 1 > prepared operation key=2AHYRZUDFHP4 tag=SAC2L2MIJ4F1 type=del size=825186 2024-09-13T00:00:35.762 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.762+0000 7fb8f2ce0640 1 < completed part key=_multipart_H5EIP0E9CBWX.2~X78MDPL8ZJU6.0 size=1048576 2024-09-13T00:00:35.764 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.762+0000 7fb8f2ce0640 1 < completed part key=_multipart_H5EIP0E9CBWX.2~X78MDPL8ZJU6.1 size=1048576 2024-09-13T00:00:35.767 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.766+0000 7fb8f2ce0640 1 < completed part key=_multipart_H5EIP0E9CBWX.2~X78MDPL8ZJU6.2 size=59864 2024-09-13T00:00:35.773 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.774+0000 7fb8f2ce0640 1 < completed multipart upload key=H5EIP0E9CBWX upload=X78MDPL8ZJU6 tag=BOKG1R8FFKW0 type=write size=3205592 2024-09-13T00:00:35.776 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.774+0000 7fb8f2ce0640 1 > prepared operation key=70F7D20HI504 tag=2OCQJD26SQAQ type=del size=2782601 2024-09-13T00:00:35.779 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.778+0000 7fb8f2ce0640 1 < completed part key=_multipart_QMZREZ1QEDNJ.2~RRLT8D8H96E4.0 size=1048576 2024-09-13T00:00:35.780 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.778+0000 7fb8f2ce0640 1 < canceled part key=_multipart_QMZREZ1QEDNJ.2~RRLT8D8H96E4.1 size=233110 2024-09-13T00:00:35.785 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.782+0000 7fb8f2ce0640 1 < canceled multipart upload key=QMZREZ1QEDNJ upload=RRLT8D8H96E4 tag=6106ZJL22HDV type=write size=2330262 2024-09-13T00:00:35.790 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.790+0000 7fb8f2ce0640 1 > prepared part key=_multipart_5GM88ERM89KT.2~FHVA022MEXZJ.0 size=450564 2024-09-13T00:00:35.791 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.790+0000 7fb8f2ce0640 1 > prepared multipart upload key=5GM88ERM89KT upload=FHVA022MEXZJ tag=1PJKMJH4VB0G type=write size=1499140 2024-09-13T00:00:35.794 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.794+0000 7fb8f2ce0640 1 < completed delete operation key=7OX5OJZ8GXU5 tag=IJVAYY1OKDP0 type=del 2024-09-13T00:00:35.800 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.798+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2TWD9YLMSYJI.2~2Z691H0NHZSD.0 size=1048576 2024-09-13T00:00:35.803 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.802+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2TWD9YLMSYJI.2~2Z691H0NHZSD.1 size=1048576 2024-09-13T00:00:35.806 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.806+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2TWD9YLMSYJI.2~2Z691H0NHZSD.2 size=463379 2024-09-13T00:00:35.806 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.806+0000 7fb8f2ce0640 1 > prepared multipart upload key=2TWD9YLMSYJI upload=2Z691H0NHZSD tag=BMUCNY1SKG6T type=write size=3609107 2024-09-13T00:00:35.809 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.810+0000 7fb8f2ce0640 1 < completed write operation key=P8OYS5ZE1NX0 tag=RWZ0LK2DSK7D type=write size=323261 2024-09-13T00:00:35.813 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.814+0000 7fb8f2ce0640 1 > prepared operation key=ZFHN2SNTZ5W7 tag=BJWVAKDXQUDY type=del size=3504273 2024-09-13T00:00:35.817 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.818+0000 7fb8f2ce0640 1 < completed part key=_multipart_N3Z5XIG3TZRQ.2~Z8AMPETZWPJ2.0 size=1048576 2024-09-13T00:00:35.817 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.818+0000 7fb8f2ce0640 1 < canceled part key=_multipart_N3Z5XIG3TZRQ.2~Z8AMPETZWPJ2.1 size=309182 2024-09-13T00:00:35.822 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.822+0000 7fb8f2ce0640 1 < canceled multipart upload key=N3Z5XIG3TZRQ upload=Z8AMPETZWPJ2 tag=9BB76ZZSLXIS type=write size=2406334 2024-09-13T00:00:35.827 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.826+0000 7fb8f2ce0640 1 > prepared operation key=70F7D20HI504 tag=A1MSS6P75661 type=del size=2889819 2024-09-13T00:00:35.832 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.830+0000 7fb8f2ce0640 1 < completed part key=_multipart_EJN6HR8I7XQF.2~P5OB4S9ELN0D.0 size=1048576 2024-09-13T00:00:35.835 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.834+0000 7fb8f2ce0640 1 < completed part key=_multipart_EJN6HR8I7XQF.2~P5OB4S9ELN0D.1 size=1048576 2024-09-13T00:00:35.838 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.838+0000 7fb8f2ce0640 1 < completed part key=_multipart_EJN6HR8I7XQF.2~P5OB4S9ELN0D.2 size=559189 2024-09-13T00:00:35.845 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.846+0000 7fb8f2ce0640 1 < completed multipart upload key=EJN6HR8I7XQF upload=P5OB4S9ELN0D tag=RJNJE0VQF6A1 type=write size=3704917 2024-09-13T00:00:35.850 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.850+0000 7fb8f2ce0640 1 > prepared operation key=0WC6P8YNL0H2 tag=WE6P61Q7A79I type=del size=3906834 2024-09-13T00:00:35.853 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.854+0000 7fb8f2ce0640 1 < completed delete operation key=8Z6U7OOA0GBH tag=K8D5VTK5XT53 type=del 2024-09-13T00:00:35.861 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.858+0000 7fb8f2ce0640 1 > prepared part key=_multipart_F0QCVR9E08WE.2~EDC6UQ885ONV.0 size=520638 2024-09-13T00:00:35.861 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.858+0000 7fb8f2ce0640 1 > prepared multipart upload key=F0QCVR9E08WE upload=EDC6UQ885ONV tag=V89JGDPATO53 type=write size=1569214 2024-09-13T00:00:35.866 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.866+0000 7fb8f2ce0640 -1 < failed to complete operation key=FX0NCGLK0KPJ tag=K5XX5WR6F55K type=del size=3154679: (22) Invalid argument 2024-09-13T00:00:35.870 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.870+0000 7fb8f2ce0640 1 > prepared operation key=PVYZQLGYT3XU tag=RYAY9M3H0Y7O type=write size=35594 2024-09-13T00:00:35.875 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.874+0000 7fb8f2ce0640 1 < completed part key=_multipart_H5EIP0E9CBWX.2~PPSZWUARRWJ4.0 size=89318 2024-09-13T00:00:35.883 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.882+0000 7fb8f2ce0640 1 < completed multipart upload key=H5EIP0E9CBWX upload=PPSZWUARRWJ4 tag=AWIY2976TDM0 type=write size=1137894 2024-09-13T00:00:35.887 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.886+0000 7fb8f2ce0640 1 > prepared operation key=2TWD9YLMSYJI tag=AYUDHT10L8W1 type=del size=4005130 2024-09-13T00:00:35.891 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.890+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZFHN2SNTZ5W7.2~1KYCO9V5AWHB.0 size=874187 2024-09-13T00:00:35.899 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.898+0000 7fb8f2ce0640 1 < completed multipart upload key=ZFHN2SNTZ5W7 upload=1KYCO9V5AWHB tag=J5ZEGDTRDVXV type=write size=1922763 2024-09-13T00:00:35.907 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.906+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2TWD9YLMSYJI.2~Z88KYSTEHJWO.0 size=1048576 2024-09-13T00:00:35.910 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.910+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2TWD9YLMSYJI.2~Z88KYSTEHJWO.1 size=1048576 2024-09-13T00:00:35.914 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.914+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2TWD9YLMSYJI.2~Z88KYSTEHJWO.2 size=711098 2024-09-13T00:00:35.914 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.914+0000 7fb8f2ce0640 1 > prepared multipart upload key=2TWD9YLMSYJI upload=Z88KYSTEHJWO tag=ZDV6ZGC6B7FT type=write size=3856826 2024-09-13T00:00:35.919 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.918+0000 7fb8f2ce0640 1 < completed delete operation key=EUE3LT0PDIXV tag=IC3KYFJAGZ27 type=del 2024-09-13T00:00:35.923 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.922+0000 7fb8f2ce0640 1 > prepared operation key=F0QCVR9E08WE tag=3H7AIOU6VB81 type=write size=915391 2024-09-13T00:00:35.927 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.926+0000 7fb8f2ce0640 1 < completed part key=_multipart_PVYZQLGYT3XU.2~QTR86NFSCT6P.0 size=307364 2024-09-13T00:00:35.933 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.934+0000 7fb8f2ce0640 1 < completed multipart upload key=PVYZQLGYT3XU upload=QTR86NFSCT6P tag=8JSQZM67ZTSQ type=write size=1355940 2024-09-13T00:00:35.938 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.938+0000 7fb8f2ce0640 1 > prepared operation key=44EDYFT9VF8Y tag=2AS9HJOLFG7Y type=del size=72251 2024-09-13T00:00:35.942 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.942+0000 7fb8f2ce0640 1 < completed part key=_multipart_EJN6HR8I7XQF.2~6H2DB6ULI63O.0 size=1048576 2024-09-13T00:00:35.946 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.946+0000 7fb8f2ce0640 1 < completed part key=_multipart_EJN6HR8I7XQF.2~6H2DB6ULI63O.1 size=637186 2024-09-13T00:00:35.953 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.954+0000 7fb8f2ce0640 1 < completed multipart upload key=EJN6HR8I7XQF upload=6H2DB6ULI63O tag=N5MIKCXOEA2M type=write size=2734338 2024-09-13T00:00:35.957 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.958+0000 7fb8f2ce0640 1 > prepared operation key=85YEW8N43FTA tag=HJS9PRJDZ2HS type=del size=2616236 2024-09-13T00:00:35.960 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.958+0000 7fb8f2ce0640 1 < canceled part key=_multipart_EJN6HR8I7XQF.2~BHY80FCW3E8C.0 size=395237 2024-09-13T00:00:35.965 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.962+0000 7fb8f2ce0640 1 < canceled multipart upload key=EJN6HR8I7XQF upload=BHY80FCW3E8C tag=BFHK9L5OVGT1 type=write size=1443813 2024-09-13T00:00:35.969 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.970+0000 7fb8f2ce0640 1 > prepared operation key=ZTK9AYBSLCOZ tag=0LOV4O0EXHEU type=del size=2350259 2024-09-13T00:00:35.973 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.974+0000 7fb8f2ce0640 1 < completed delete operation key=PVYZQLGYT3XU tag=H4AMC3NY9X49 type=del 2024-09-13T00:00:35.978 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.978+0000 7fb8f2ce0640 1 > prepared operation key=1HX06CCZJ1DD tag=QXN00VA58U4G type=del size=3893058 2024-09-13T00:00:35.982 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.982+0000 7fb8f2ce0640 1 < completed delete operation key=2AHYRZUDFHP4 tag=SAC2L2MIJ4F1 type=del 2024-09-13T00:00:35.986 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.986+0000 7fb8f2ce0640 1 > prepared operation key=W8XZ26311RJX tag=HDH6U1ZOKQ20 type=del size=3554298 2024-09-13T00:00:35.991 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.990+0000 7fb8f2ce0640 1 < completed delete operation key=70F7D20HI504 tag=2OCQJD26SQAQ type=del 2024-09-13T00:00:35.998 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:35.998+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~IXHPB9L4Q908.0 size=1048576 2024-09-13T00:00:36.001 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.002+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~IXHPB9L4Q908.1 size=1048576 2024-09-13T00:00:36.005 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.002+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~IXHPB9L4Q908.2 size=54935 2024-09-13T00:00:36.005 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.002+0000 7fb8f2ce0640 1 > prepared multipart upload key=63FJDWBO2T9S upload=IXHPB9L4Q908 tag=79DFEMJSRGXE type=write size=3200663 2024-09-13T00:00:36.009 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.010+0000 7fb8f2ce0640 1 < completed part key=_multipart_5GM88ERM89KT.2~FHVA022MEXZJ.0 size=450564 2024-09-13T00:00:36.015 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.014+0000 7fb8f2ce0640 1 < completed multipart upload key=5GM88ERM89KT upload=FHVA022MEXZJ tag=1PJKMJH4VB0G type=write size=1499140 2024-09-13T00:00:36.019 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.018+0000 7fb8f2ce0640 1 > prepared operation key=85YEW8N43FTA tag=0MXP5845EVWE type=write size=259670 2024-09-13T00:00:36.023 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.022+0000 7fb8f2ce0640 1 < completed part key=_multipart_2TWD9YLMSYJI.2~2Z691H0NHZSD.0 size=1048576 2024-09-13T00:00:36.026 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.026+0000 7fb8f2ce0640 1 < completed part key=_multipart_2TWD9YLMSYJI.2~2Z691H0NHZSD.1 size=1048576 2024-09-13T00:00:36.029 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.030+0000 7fb8f2ce0640 1 < completed part key=_multipart_2TWD9YLMSYJI.2~2Z691H0NHZSD.2 size=463379 2024-09-13T00:00:36.036 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.034+0000 7fb8f2ce0640 1 < completed multipart upload key=2TWD9YLMSYJI upload=2Z691H0NHZSD tag=BMUCNY1SKG6T type=write size=3609107 2024-09-13T00:00:36.043 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.042+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EUE3LT0PDIXV.2~T1IWPLDGO1P7.0 size=1048576 2024-09-13T00:00:36.047 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.046+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EUE3LT0PDIXV.2~T1IWPLDGO1P7.1 size=749834 2024-09-13T00:00:36.047 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.046+0000 7fb8f2ce0640 1 > prepared multipart upload key=EUE3LT0PDIXV upload=T1IWPLDGO1P7 tag=XUK2R7QX7CY6 type=write size=2846986 2024-09-13T00:00:36.051 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.050+0000 7fb8f2ce0640 1 < completed delete operation key=ZFHN2SNTZ5W7 tag=BJWVAKDXQUDY type=del 2024-09-13T00:00:36.055 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.054+0000 7fb8f2ce0640 1 > prepared operation key=1HX06CCZJ1DD tag=UYJZ31Z2LP4T type=del size=318037 2024-09-13T00:00:36.059 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.058+0000 7fb8f2ce0640 1 < completed delete operation key=70F7D20HI504 tag=A1MSS6P75661 type=del 2024-09-13T00:00:36.067 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.066+0000 7fb8f2ce0640 1 > prepared part key=_multipart_P8OYS5ZE1NX0.2~KH7NS33VW6LS.0 size=786162 2024-09-13T00:00:36.067 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.066+0000 7fb8f2ce0640 1 > prepared multipart upload key=P8OYS5ZE1NX0 upload=KH7NS33VW6LS tag=M7P9D906VSFA type=write size=1834738 2024-09-13T00:00:36.071 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.070+0000 7fb8f2ce0640 1 < completed delete operation key=0WC6P8YNL0H2 tag=WE6P61Q7A79I type=del 2024-09-13T00:00:36.075 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.074+0000 7fb8f2ce0640 1 > prepared operation key=NX26SI4F2ZMB tag=N6U1WNDIR6CE type=del size=596825 2024-09-13T00:00:36.079 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.078+0000 7fb8f2ce0640 1 < completed part key=_multipart_F0QCVR9E08WE.2~EDC6UQ885ONV.0 size=520638 2024-09-13T00:00:36.086 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.086+0000 7fb8f2ce0640 1 < completed multipart upload key=F0QCVR9E08WE upload=EDC6UQ885ONV tag=V89JGDPATO53 type=write size=1569214 2024-09-13T00:00:36.090 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.090+0000 7fb8f2ce0640 1 > prepared operation key=1HX06CCZJ1DD tag=NAYD1ZT2C63P type=del size=1165850 2024-09-13T00:00:36.093 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.094+0000 7fb8f2ce0640 1 < completed write operation key=PVYZQLGYT3XU tag=RYAY9M3H0Y7O type=write size=35594 2024-09-13T00:00:36.097 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.098+0000 7fb8f2ce0640 1 > prepared operation key=O4MZ4UWNXME9 tag=EF1104V96YU0 type=del size=1389427 2024-09-13T00:00:36.101 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.102+0000 7fb8f2ce0640 1 < completed delete operation key=2TWD9YLMSYJI tag=AYUDHT10L8W1 type=del 2024-09-13T00:00:36.105 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.102+0000 7fb8f2ce0640 1 > prepared operation key=0WC6P8YNL0H2 tag=3L2OBDA26J17 type=del size=2880016 2024-09-13T00:00:36.109 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.110+0000 7fb8f2ce0640 1 < completed part key=_multipart_2TWD9YLMSYJI.2~Z88KYSTEHJWO.0 size=1048576 2024-09-13T00:00:36.112 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.110+0000 7fb8f2ce0640 1 < completed part key=_multipart_2TWD9YLMSYJI.2~Z88KYSTEHJWO.1 size=1048576 2024-09-13T00:00:36.154 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.154+0000 7fb8f2ce0640 1 < completed part key=_multipart_2TWD9YLMSYJI.2~Z88KYSTEHJWO.2 size=711098 2024-09-13T00:00:36.164 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.162+0000 7fb8f2ce0640 1 < completed multipart upload key=2TWD9YLMSYJI upload=Z88KYSTEHJWO tag=ZDV6ZGC6B7FT type=write size=3856826 2024-09-13T00:00:36.169 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.170+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~2UGDXD2WELZ5.0 size=1048576 2024-09-13T00:00:36.172 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.170+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~2UGDXD2WELZ5.1 size=520748 2024-09-13T00:00:36.173 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.170+0000 7fb8f2ce0640 1 > prepared multipart upload key=W8XZ26311RJX upload=2UGDXD2WELZ5 tag=8HP2CQISM0S8 type=write size=2617900 2024-09-13T00:00:36.176 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.174+0000 7fb8f2ce0640 1 < completed write operation key=F0QCVR9E08WE tag=3H7AIOU6VB81 type=write size=915391 2024-09-13T00:00:36.181 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.178+0000 7fb8f2ce0640 1 > prepared operation key=8MPUVD37EDN9 tag=LME7X2OI4T0W type=del size=156898 2024-09-13T00:00:36.185 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.186+0000 7fb8f2ce0640 1 < completed delete operation key=44EDYFT9VF8Y tag=2AS9HJOLFG7Y type=del 2024-09-13T00:00:36.193 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.194+0000 7fb8f2ce0640 1 > prepared part key=_multipart_44EDYFT9VF8Y.2~42HCEIHSX0RM.0 size=633223 2024-09-13T00:00:36.193 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.194+0000 7fb8f2ce0640 1 > prepared multipart upload key=44EDYFT9VF8Y upload=42HCEIHSX0RM tag=4SRUHG04W5MI type=write size=1681799 2024-09-13T00:00:36.197 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.198+0000 7fb8f2ce0640 1 < completed delete operation key=85YEW8N43FTA tag=HJS9PRJDZ2HS type=del 2024-09-13T00:00:36.201 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.202+0000 7fb8f2ce0640 1 > prepared operation key=2AHYRZUDFHP4 tag=KGZ1N3R72ZP1 type=del size=3649485 2024-09-13T00:00:36.209 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.210+0000 7fb8f2ce0640 1 < suggested remove operation key=ZTK9AYBSLCOZ tag=0LOV4O0EXHEU type=del 2024-09-13T00:00:36.217 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.214+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~BA9ORQ7I44I0.0 size=1048576 2024-09-13T00:00:36.220 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.218+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~BA9ORQ7I44I0.1 size=1048576 2024-09-13T00:00:36.223 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.222+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~BA9ORQ7I44I0.2 size=200366 2024-09-13T00:00:36.223 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.222+0000 7fb8f2ce0640 1 > prepared multipart upload key=8Z6U7OOA0GBH upload=BA9ORQ7I44I0 tag=FAMYMXF6Q9QN type=write size=3346094 2024-09-13T00:00:36.227 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.226+0000 7fb8f2ce0640 1 < completed delete operation key=1HX06CCZJ1DD tag=QXN00VA58U4G type=del 2024-09-13T00:00:36.232 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.230+0000 7fb8f2ce0640 1 > prepared operation key=EJN6HR8I7XQF tag=P1IKMSPDBNVE type=del size=1798418 2024-09-13T00:00:36.236 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.234+0000 7fb8f2ce0640 1 < completed delete operation key=W8XZ26311RJX tag=HDH6U1ZOKQ20 type=del 2024-09-13T00:00:36.241 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.242+0000 7fb8f2ce0640 1 > prepared operation key=H4M1TCN5W6XU tag=UA13AGPMJ8QN type=del size=399584 2024-09-13T00:00:36.241 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.242+0000 7fb8f2ce0640 1 < canceled part key=_multipart_63FJDWBO2T9S.2~IXHPB9L4Q908.0 size=1048576 2024-09-13T00:00:36.245 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.246+0000 7fb8f2ce0640 1 < completed part key=_multipart_63FJDWBO2T9S.2~IXHPB9L4Q908.1 size=1048576 2024-09-13T00:00:36.248 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.246+0000 7fb8f2ce0640 1 < completed part key=_multipart_63FJDWBO2T9S.2~IXHPB9L4Q908.2 size=54935 2024-09-13T00:00:36.256 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.254+0000 7fb8f2ce0640 1 < canceled multipart upload key=63FJDWBO2T9S upload=IXHPB9L4Q908 tag=79DFEMJSRGXE type=write size=3200663 2024-09-13T00:00:36.265 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.262+0000 7fb8f2ce0640 1 > prepared part key=_multipart_44EDYFT9VF8Y.2~LBQJIDBVDHVS.0 size=1048576 2024-09-13T00:00:36.268 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.266+0000 7fb8f2ce0640 1 > prepared part key=_multipart_44EDYFT9VF8Y.2~LBQJIDBVDHVS.1 size=914845 2024-09-13T00:00:36.268 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.266+0000 7fb8f2ce0640 1 > prepared multipart upload key=44EDYFT9VF8Y upload=LBQJIDBVDHVS tag=1ON3NLV1SW5Z type=write size=3011997 2024-09-13T00:00:36.273 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.270+0000 7fb8f2ce0640 1 < completed write operation key=85YEW8N43FTA tag=0MXP5845EVWE type=write size=259670 2024-09-13T00:00:36.280 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.278+0000 7fb8f2ce0640 1 > prepared part key=_multipart_J8NGDSF9XRTP.2~6UMDIMY3JRTI.0 size=1048576 2024-09-13T00:00:36.284 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.282+0000 7fb8f2ce0640 1 > prepared part key=_multipart_J8NGDSF9XRTP.2~6UMDIMY3JRTI.1 size=614497 2024-09-13T00:00:36.284 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.282+0000 7fb8f2ce0640 1 > prepared multipart upload key=J8NGDSF9XRTP upload=6UMDIMY3JRTI tag=TJFXKT8JL097 type=write size=2711649 2024-09-13T00:00:36.289 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.286+0000 7fb8f2ce0640 1 < completed part key=_multipart_EUE3LT0PDIXV.2~T1IWPLDGO1P7.0 size=1048576 2024-09-13T00:00:36.292 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.290+0000 7fb8f2ce0640 1 < completed part key=_multipart_EUE3LT0PDIXV.2~T1IWPLDGO1P7.1 size=749834 2024-09-13T00:00:36.299 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.298+0000 7fb8f2ce0640 1 < completed multipart upload key=EUE3LT0PDIXV upload=T1IWPLDGO1P7 tag=XUK2R7QX7CY6 type=write size=2846986 2024-09-13T00:00:36.304 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.302+0000 7fb8f2ce0640 1 > prepared operation key=1HX06CCZJ1DD tag=877USLMQO5Z6 type=del size=1525258 2024-09-13T00:00:36.308 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.306+0000 7fb8f2ce0640 1 < completed delete operation key=1HX06CCZJ1DD tag=UYJZ31Z2LP4T type=del 2024-09-13T00:00:36.313 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.314+0000 7fb8f2ce0640 1 > prepared operation key=5GM88ERM89KT tag=AWK3E8ULI760 type=del size=1623562 2024-09-13T00:00:36.314 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.314+0000 7fb8f2ce0640 1 < canceled part key=_multipart_P8OYS5ZE1NX0.2~KH7NS33VW6LS.0 size=786162 2024-09-13T00:00:36.321 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.322+0000 7fb8f2ce0640 1 < canceled multipart upload key=P8OYS5ZE1NX0 upload=KH7NS33VW6LS tag=M7P9D906VSFA type=write size=1834738 2024-09-13T00:00:36.325 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.326+0000 7fb8f2ce0640 1 > prepared operation key=H4M1TCN5W6XU tag=1O9ZN3IL41M7 type=del size=1629220 2024-09-13T00:00:36.329 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.330+0000 7fb8f2ce0640 1 < completed delete operation key=NX26SI4F2ZMB tag=N6U1WNDIR6CE type=del 2024-09-13T00:00:36.333 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.334+0000 7fb8f2ce0640 1 > prepared operation key=5GM88ERM89KT tag=OSTB0T8SWQV2 type=write size=300897 2024-09-13T00:00:36.338 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.338+0000 7fb8f2ce0640 1 < completed delete operation key=1HX06CCZJ1DD tag=NAYD1ZT2C63P type=del 2024-09-13T00:00:36.342 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.342+0000 7fb8f2ce0640 1 > prepared operation key=63FJDWBO2T9S tag=7II3QPTK9HZN type=del size=471257 2024-09-13T00:00:36.347 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.346+0000 7fb8f2ce0640 1 < completed delete operation key=O4MZ4UWNXME9 tag=EF1104V96YU0 type=del 2024-09-13T00:00:36.352 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.350+0000 7fb8f2ce0640 1 > prepared operation key=EUE3LT0PDIXV tag=RBKNZUUH5X4Q type=del size=1033909 2024-09-13T00:00:36.356 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.354+0000 7fb8f2ce0640 1 < completed delete operation key=0WC6P8YNL0H2 tag=3L2OBDA26J17 type=del 2024-09-13T00:00:36.360 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.358+0000 7fb8f2ce0640 1 > prepared operation key=2TWD9YLMSYJI tag=8TYCR5ZOJZCD type=del size=887453 2024-09-13T00:00:36.368 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.366+0000 7fb8f2ce0640 1 < completed part key=_multipart_W8XZ26311RJX.2~2UGDXD2WELZ5.0 size=1048576 2024-09-13T00:00:36.369 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.366+0000 7fb8f2ce0640 1 < completed part key=_multipart_W8XZ26311RJX.2~2UGDXD2WELZ5.1 size=520748 2024-09-13T00:00:36.375 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.374+0000 7fb8f2ce0640 1 < completed multipart upload key=W8XZ26311RJX upload=2UGDXD2WELZ5 tag=8HP2CQISM0S8 type=write size=2617900 2024-09-13T00:00:36.379 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.378+0000 7fb8f2ce0640 1 > prepared operation key=EUE3LT0PDIXV tag=VKA9DW3LQXDU type=write size=156725 2024-09-13T00:00:36.383 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.382+0000 7fb8f2ce0640 1 < completed delete operation key=8MPUVD37EDN9 tag=LME7X2OI4T0W type=del 2024-09-13T00:00:36.388 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.386+0000 7fb8f2ce0640 1 > prepared operation key=PVYZQLGYT3XU tag=79CSA0A2RT7H type=write size=916389 2024-09-13T00:00:36.392 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.390+0000 7fb8f2ce0640 1 < completed part key=_multipart_44EDYFT9VF8Y.2~42HCEIHSX0RM.0 size=633223 2024-09-13T00:00:36.399 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.398+0000 7fb8f2ce0640 1 < completed multipart upload key=44EDYFT9VF8Y upload=42HCEIHSX0RM tag=4SRUHG04W5MI type=write size=1681799 2024-09-13T00:00:36.404 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.402+0000 7fb8f2ce0640 1 > prepared operation key=NQJQC8CEAC15 tag=2Z54J50PAXOL type=del size=1539461 2024-09-13T00:00:36.408 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.406+0000 7fb8f2ce0640 1 < completed delete operation key=2AHYRZUDFHP4 tag=KGZ1N3R72ZP1 type=del 2024-09-13T00:00:36.413 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.414+0000 7fb8f2ce0640 1 > prepared operation key=W8XZ26311RJX tag=5HM4UYDKA17U type=del size=3591835 2024-09-13T00:00:36.418 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.418+0000 7fb8f2ce0640 1 < completed part key=_multipart_8Z6U7OOA0GBH.2~BA9ORQ7I44I0.0 size=1048576 2024-09-13T00:00:36.421 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.422+0000 7fb8f2ce0640 1 < completed part key=_multipart_8Z6U7OOA0GBH.2~BA9ORQ7I44I0.1 size=1048576 2024-09-13T00:00:36.421 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.422+0000 7fb8f2ce0640 1 < canceled part key=_multipart_8Z6U7OOA0GBH.2~BA9ORQ7I44I0.2 size=200366 2024-09-13T00:00:36.428 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.426+0000 7fb8f2ce0640 1 < canceled multipart upload key=8Z6U7OOA0GBH upload=BA9ORQ7I44I0 tag=FAMYMXF6Q9QN type=write size=3346094 2024-09-13T00:00:36.433 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.430+0000 7fb8f2ce0640 1 > prepared operation key=44EDYFT9VF8Y tag=4KPQVOIL2RYX type=write size=146301 2024-09-13T00:00:36.437 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.438+0000 7fb8f2ce0640 1 < canceled operation key=EJN6HR8I7XQF tag=P1IKMSPDBNVE type=del size=1798418 2024-09-13T00:00:36.442 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.442+0000 7fb8f2ce0640 1 > prepared operation key=P8OYS5ZE1NX0 tag=1OMUYUKLKYXZ type=write size=1026592 2024-09-13T00:00:36.446 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.446+0000 7fb8f2ce0640 1 < completed delete operation key=H4M1TCN5W6XU tag=UA13AGPMJ8QN type=del 2024-09-13T00:00:36.450 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.450+0000 7fb8f2ce0640 1 > prepared operation key=F0QCVR9E08WE tag=62XUKZ0FA3SS type=del size=1707874 2024-09-13T00:00:36.455 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.454+0000 7fb8f2ce0640 1 < completed part key=_multipart_44EDYFT9VF8Y.2~LBQJIDBVDHVS.0 size=1048576 2024-09-13T00:00:36.458 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.458+0000 7fb8f2ce0640 1 < completed part key=_multipart_44EDYFT9VF8Y.2~LBQJIDBVDHVS.1 size=914845 2024-09-13T00:00:36.465 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.466+0000 7fb8f2ce0640 1 < completed multipart upload key=44EDYFT9VF8Y upload=LBQJIDBVDHVS tag=1ON3NLV1SW5Z type=write size=3011997 2024-09-13T00:00:36.469 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.470+0000 7fb8f2ce0640 1 > prepared operation key=QMZREZ1QEDNJ tag=1LV1YLVO8TDD type=del size=2680618 2024-09-13T00:00:36.474 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.474+0000 7fb8f2ce0640 1 < completed part key=_multipart_J8NGDSF9XRTP.2~6UMDIMY3JRTI.0 size=1048576 2024-09-13T00:00:36.477 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.478+0000 7fb8f2ce0640 1 < completed part key=_multipart_J8NGDSF9XRTP.2~6UMDIMY3JRTI.1 size=614497 2024-09-13T00:00:36.485 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.482+0000 7fb8f2ce0640 1 < completed multipart upload key=J8NGDSF9XRTP upload=6UMDIMY3JRTI tag=TJFXKT8JL097 type=write size=2711649 2024-09-13T00:00:36.493 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.494+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~BDPFHUUYLLGT.0 size=1048576 2024-09-13T00:00:36.496 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.494+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~BDPFHUUYLLGT.1 size=61096 2024-09-13T00:00:36.496 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.494+0000 7fb8f2ce0640 1 > prepared multipart upload key=NQJQC8CEAC15 upload=BDPFHUUYLLGT tag=77250EVW4VFZ type=write size=2158248 2024-09-13T00:00:36.500 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.498+0000 7fb8f2ce0640 1 < completed delete operation key=1HX06CCZJ1DD tag=877USLMQO5Z6 type=del 2024-09-13T00:00:36.504 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.502+0000 7fb8f2ce0640 1 > prepared operation key=J8NGDSF9XRTP tag=13L8G127KW3S type=del size=654966 2024-09-13T00:00:36.508 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.506+0000 7fb8f2ce0640 1 < completed delete operation key=5GM88ERM89KT tag=AWK3E8ULI760 type=del 2024-09-13T00:00:36.516 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.514+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H9AU4HD247YU.2~8DBMVB3ZD758.0 size=268111 2024-09-13T00:00:36.516 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.514+0000 7fb8f2ce0640 1 > prepared multipart upload key=H9AU4HD247YU upload=8DBMVB3ZD758 tag=GO0QLQP0U5C5 type=write size=1316687 2024-09-13T00:00:36.520 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.518+0000 7fb8f2ce0640 1 < completed delete operation key=H4M1TCN5W6XU tag=1O9ZN3IL41M7 type=del 2024-09-13T00:00:36.524 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.522+0000 7fb8f2ce0640 1 > prepared operation key=85YEW8N43FTA tag=TQW5KABRGEZS type=del size=3107458 2024-09-13T00:00:36.528 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.526+0000 7fb8f2ce0640 1 < completed write operation key=5GM88ERM89KT tag=OSTB0T8SWQV2 type=write size=300897 2024-09-13T00:00:36.532 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.530+0000 7fb8f2ce0640 1 > prepared operation key=EUE3LT0PDIXV tag=B6EOAVAPSU7D type=write size=316050 2024-09-13T00:00:36.536 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.534+0000 7fb8f2ce0640 1 < completed delete operation key=63FJDWBO2T9S tag=7II3QPTK9HZN type=del 2024-09-13T00:00:36.540 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.538+0000 7fb8f2ce0640 1 > prepared operation key=FX0NCGLK0KPJ tag=6DZGL6O5QNB1 type=del size=3960586 2024-09-13T00:00:36.545 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.542+0000 7fb8f2ce0640 1 < completed delete operation key=EUE3LT0PDIXV tag=RBKNZUUH5X4Q type=del 2024-09-13T00:00:36.549 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.550+0000 7fb8f2ce0640 1 > prepared operation key=W8XZ26311RJX tag=2OXKSMZQ97RR type=write size=880315 2024-09-13T00:00:36.553 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.554+0000 7fb8f2ce0640 1 < completed delete operation key=2TWD9YLMSYJI tag=8TYCR5ZOJZCD type=del 2024-09-13T00:00:36.557 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.558+0000 7fb8f2ce0640 1 > prepared operation key=44EDYFT9VF8Y tag=27AXG90K059S type=del size=3794326 2024-09-13T00:00:36.561 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.562+0000 7fb8f2ce0640 1 < completed write operation key=EUE3LT0PDIXV tag=VKA9DW3LQXDU type=write size=156725 2024-09-13T00:00:36.569 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.570+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~WHU9VOWM23GZ.0 size=206781 2024-09-13T00:00:36.572 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.570+0000 7fb8f2ce0640 1 > prepared multipart upload key=8MPUVD37EDN9 upload=WHU9VOWM23GZ tag=1IH6UDF14TCE type=write size=1255357 2024-09-13T00:00:36.573 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.574+0000 7fb8f2ce0640 1 < completed write operation key=PVYZQLGYT3XU tag=79CSA0A2RT7H type=write size=916389 2024-09-13T00:00:36.578 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.578+0000 7fb8f2ce0640 1 > prepared operation key=H9AU4HD247YU tag=IRER86ZW54EW type=del size=2324528 2024-09-13T00:00:36.582 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.582+0000 7fb8f2ce0640 1 < completed delete operation key=NQJQC8CEAC15 tag=2Z54J50PAXOL type=del 2024-09-13T00:00:36.587 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.586+0000 7fb8f2ce0640 1 > prepared operation key=F0QCVR9E08WE tag=C4IWDHA6RVIF type=del size=2800140 2024-09-13T00:00:36.591 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.590+0000 7fb8f2ce0640 1 < completed delete operation key=W8XZ26311RJX tag=5HM4UYDKA17U type=del 2024-09-13T00:00:36.599 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.598+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EUE3LT0PDIXV.2~F231LILZYDSN.0 size=1048576 2024-09-13T00:00:36.602 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.602+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EUE3LT0PDIXV.2~F231LILZYDSN.1 size=1048576 2024-09-13T00:00:36.605 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.606+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EUE3LT0PDIXV.2~F231LILZYDSN.2 size=329771 2024-09-13T00:00:36.606 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.606+0000 7fb8f2ce0640 1 > prepared multipart upload key=EUE3LT0PDIXV upload=F231LILZYDSN tag=NRFH732ZU3VR type=write size=3475499 2024-09-13T00:00:36.609 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.610+0000 7fb8f2ce0640 1 < completed write operation key=44EDYFT9VF8Y tag=4KPQVOIL2RYX type=write size=146301 2024-09-13T00:00:36.617 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.618+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~XGZLI75UWEG6.0 size=1048576 2024-09-13T00:00:36.620 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.618+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~XGZLI75UWEG6.1 size=50727 2024-09-13T00:00:36.621 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.618+0000 7fb8f2ce0640 1 > prepared multipart upload key=2AHYRZUDFHP4 upload=XGZLI75UWEG6 tag=YETGSBUQWCH0 type=write size=2147879 2024-09-13T00:00:36.624 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.622+0000 7fb8f2ce0640 1 < completed write operation key=P8OYS5ZE1NX0 tag=1OMUYUKLKYXZ type=write size=1026592 2024-09-13T00:00:36.629 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.630+0000 7fb8f2ce0640 1 > prepared operation key=85YEW8N43FTA tag=10CC2E0SU4CA type=del size=2603860 2024-09-13T00:00:36.633 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.634+0000 7fb8f2ce0640 1 < completed delete operation key=F0QCVR9E08WE tag=62XUKZ0FA3SS type=del 2024-09-13T00:00:36.640 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.638+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~UWXVKT520KC9.0 size=988701 2024-09-13T00:00:36.640 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.638+0000 7fb8f2ce0640 1 > prepared multipart upload key=FX0NCGLK0KPJ upload=UWXVKT520KC9 tag=BF7MK4GH84O2 type=write size=2037277 2024-09-13T00:00:36.644 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.642+0000 7fb8f2ce0640 1 < completed delete operation key=QMZREZ1QEDNJ tag=1LV1YLVO8TDD type=del 2024-09-13T00:00:36.652 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.650+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~CDDY5QHD0S4Z.0 size=338095 2024-09-13T00:00:36.652 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.650+0000 7fb8f2ce0640 1 > prepared multipart upload key=1HX06CCZJ1DD upload=CDDY5QHD0S4Z tag=0EKZS4TEJGBZ type=write size=1386671 2024-09-13T00:00:36.656 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.654+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~BDPFHUUYLLGT.0 size=1048576 2024-09-13T00:00:36.659 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.658+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~BDPFHUUYLLGT.1 size=61096 2024-09-13T00:00:36.666 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.666+0000 7fb8f2ce0640 1 < completed multipart upload key=NQJQC8CEAC15 upload=BDPFHUUYLLGT tag=77250EVW4VFZ type=write size=2158248 2024-09-13T00:00:36.673 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.674+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~F5WBBU6FN8MH.0 size=355091 2024-09-13T00:00:36.673 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.674+0000 7fb8f2ce0640 1 > prepared multipart upload key=1HX06CCZJ1DD upload=F5WBBU6FN8MH tag=10PVLOUAW6S0 type=write size=1403667 2024-09-13T00:00:36.677 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.678+0000 7fb8f2ce0640 1 < completed delete operation key=J8NGDSF9XRTP tag=13L8G127KW3S type=del 2024-09-13T00:00:36.684 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.682+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZFHN2SNTZ5W7.2~A1D6RUREUHUI.0 size=1048576 2024-09-13T00:00:36.687 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.686+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZFHN2SNTZ5W7.2~A1D6RUREUHUI.1 size=1048576 2024-09-13T00:00:36.691 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.690+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZFHN2SNTZ5W7.2~A1D6RUREUHUI.2 size=621373 2024-09-13T00:00:36.691 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.690+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZFHN2SNTZ5W7 upload=A1D6RUREUHUI tag=0FXDPNGUY1WH type=write size=3767101 2024-09-13T00:00:36.695 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.694+0000 7fb8f2ce0640 1 < completed part key=_multipart_H9AU4HD247YU.2~8DBMVB3ZD758.0 size=268111 2024-09-13T00:00:36.702 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.702+0000 7fb8f2ce0640 1 < completed multipart upload key=H9AU4HD247YU upload=8DBMVB3ZD758 tag=GO0QLQP0U5C5 type=write size=1316687 2024-09-13T00:00:36.706 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.706+0000 7fb8f2ce0640 1 > prepared operation key=NX26SI4F2ZMB tag=T7P2GD8KD08R type=del size=1463616 2024-09-13T00:00:36.710 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.710+0000 7fb8f2ce0640 1 < completed delete operation key=85YEW8N43FTA tag=TQW5KABRGEZS type=del 2024-09-13T00:00:36.715 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.714+0000 7fb8f2ce0640 1 > prepared operation key=EJN6HR8I7XQF tag=VGX0VI55NZ4M type=del size=3855753 2024-09-13T00:00:36.723 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.722+0000 7fb8f2ce0640 1 < suggested remove operation key=EUE3LT0PDIXV tag=B6EOAVAPSU7D type=write 2024-09-13T00:00:36.730 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.730+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NX26SI4F2ZMB.2~1NHPWBL6NLMT.0 size=76800 2024-09-13T00:00:36.730 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.730+0000 7fb8f2ce0640 1 > prepared multipart upload key=NX26SI4F2ZMB upload=1NHPWBL6NLMT tag=EO07BAY1XJ1W type=write size=1125376 2024-09-13T00:00:36.734 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.734+0000 7fb8f2ce0640 1 < completed delete operation key=FX0NCGLK0KPJ tag=6DZGL6O5QNB1 type=del 2024-09-13T00:00:36.741 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.742+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~5TB1ZRKMFZO3.0 size=1048576 2024-09-13T00:00:36.744 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.742+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~5TB1ZRKMFZO3.1 size=1048576 2024-09-13T00:00:36.748 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.746+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~5TB1ZRKMFZO3.2 size=429193 2024-09-13T00:00:36.748 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.746+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZTK9AYBSLCOZ upload=5TB1ZRKMFZO3 tag=FT9IINFLDCGB type=write size=3574921 2024-09-13T00:00:36.752 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.750+0000 7fb8f2ce0640 1 < canceled operation key=W8XZ26311RJX tag=2OXKSMZQ97RR type=write size=880315 2024-09-13T00:00:36.756 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.754+0000 7fb8f2ce0640 1 > prepared operation key=ZTK9AYBSLCOZ tag=V767J6KS12EC type=del size=3933210 2024-09-13T00:00:36.760 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.758+0000 7fb8f2ce0640 1 < completed delete operation key=44EDYFT9VF8Y tag=27AXG90K059S type=del 2024-09-13T00:00:36.763 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.762+0000 7fb8f2ce0640 1 > prepared operation key=H5EIP0E9CBWX tag=0RKHP640S9O0 type=del size=4023654 2024-09-13T00:00:36.766 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.766+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~WHU9VOWM23GZ.0 size=206781 2024-09-13T00:00:36.772 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.770+0000 7fb8f2ce0640 1 < completed multipart upload key=8MPUVD37EDN9 upload=WHU9VOWM23GZ tag=1IH6UDF14TCE type=write size=1255357 2024-09-13T00:00:36.777 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.778+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~31MBCG2D2I0Q.0 size=934835 2024-09-13T00:00:36.777 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.778+0000 7fb8f2ce0640 1 > prepared multipart upload key=63FJDWBO2T9S upload=31MBCG2D2I0Q tag=SASBRRJD3Q89 type=write size=1983411 2024-09-13T00:00:36.783 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.782+0000 7fb8f2ce0640 1 < suggested remove operation key=H9AU4HD247YU tag=IRER86ZW54EW type=del 2024-09-13T00:00:36.786 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.786+0000 7fb8f2ce0640 1 > prepared operation key=NX26SI4F2ZMB tag=CQREFLDJJ6HX type=del size=1081255 2024-09-13T00:00:36.792 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.790+0000 7fb8f2ce0640 1 < suggested remove operation key=F0QCVR9E08WE tag=C4IWDHA6RVIF type=del 2024-09-13T00:00:36.798 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.798+0000 7fb8f2ce0640 1 > prepared part key=_multipart_P8OYS5ZE1NX0.2~SEL21UVYEQXP.0 size=1048576 2024-09-13T00:00:36.801 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.802+0000 7fb8f2ce0640 1 > prepared part key=_multipart_P8OYS5ZE1NX0.2~SEL21UVYEQXP.1 size=645632 2024-09-13T00:00:36.801 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.802+0000 7fb8f2ce0640 1 > prepared multipart upload key=P8OYS5ZE1NX0 upload=SEL21UVYEQXP tag=HXXUB893T44R type=write size=2742784 2024-09-13T00:00:36.805 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.806+0000 7fb8f2ce0640 1 < completed part key=_multipart_EUE3LT0PDIXV.2~F231LILZYDSN.0 size=1048576 2024-09-13T00:00:36.808 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.806+0000 7fb8f2ce0640 1 < completed part key=_multipart_EUE3LT0PDIXV.2~F231LILZYDSN.1 size=1048576 2024-09-13T00:00:36.811 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.810+0000 7fb8f2ce0640 1 < completed part key=_multipart_EUE3LT0PDIXV.2~F231LILZYDSN.2 size=329771 2024-09-13T00:00:36.816 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.814+0000 7fb8f2ce0640 1 < completed multipart upload key=EUE3LT0PDIXV upload=F231LILZYDSN tag=NRFH732ZU3VR type=write size=3475499 2024-09-13T00:00:36.820 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.818+0000 7fb8f2ce0640 1 > prepared operation key=4K76U69GZ9AE tag=3LFJZUUUCFT0 type=del size=3323952 2024-09-13T00:00:36.823 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.822+0000 7fb8f2ce0640 1 < completed part key=_multipart_2AHYRZUDFHP4.2~XGZLI75UWEG6.0 size=1048576 2024-09-13T00:00:36.826 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.826+0000 7fb8f2ce0640 1 < completed part key=_multipart_2AHYRZUDFHP4.2~XGZLI75UWEG6.1 size=50727 2024-09-13T00:00:36.832 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.830+0000 7fb8f2ce0640 1 < completed multipart upload key=2AHYRZUDFHP4 upload=XGZLI75UWEG6 tag=YETGSBUQWCH0 type=write size=2147879 2024-09-13T00:00:36.839 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.838+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~CYOUCMPG7PKJ.0 size=1048576 2024-09-13T00:00:36.842 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.842+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~CYOUCMPG7PKJ.1 size=1048576 2024-09-13T00:00:36.845 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.846+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~CYOUCMPG7PKJ.2 size=472782 2024-09-13T00:00:36.845 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.846+0000 7fb8f2ce0640 1 > prepared multipart upload key=PVYZQLGYT3XU upload=CYOUCMPG7PKJ tag=X0FDARXCDUAA type=write size=3618510 2024-09-13T00:00:36.849 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.846+0000 7fb8f2ce0640 1 < completed delete operation key=85YEW8N43FTA tag=10CC2E0SU4CA type=del 2024-09-13T00:00:36.855 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.854+0000 7fb8f2ce0640 1 > prepared part key=_multipart_85YEW8N43FTA.2~ZC5U69MSVBFD.0 size=1048576 2024-09-13T00:00:36.858 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.858+0000 7fb8f2ce0640 1 > prepared part key=_multipart_85YEW8N43FTA.2~ZC5U69MSVBFD.1 size=1048576 2024-09-13T00:00:36.861 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.862+0000 7fb8f2ce0640 1 > prepared part key=_multipart_85YEW8N43FTA.2~ZC5U69MSVBFD.2 size=314005 2024-09-13T00:00:36.861 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.862+0000 7fb8f2ce0640 1 > prepared multipart upload key=85YEW8N43FTA upload=ZC5U69MSVBFD tag=B5QGGNB2M16B type=write size=3459733 2024-09-13T00:00:36.865 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.866+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~UWXVKT520KC9.0 size=988701 2024-09-13T00:00:36.871 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.870+0000 7fb8f2ce0640 1 < completed multipart upload key=FX0NCGLK0KPJ upload=UWXVKT520KC9 tag=BF7MK4GH84O2 type=write size=2037277 2024-09-13T00:00:36.875 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.874+0000 7fb8f2ce0640 1 > prepared operation key=NX26SI4F2ZMB tag=OQ48DS5H66CR type=del size=1613410 2024-09-13T00:00:36.879 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.878+0000 7fb8f2ce0640 1 < completed part key=_multipart_1HX06CCZJ1DD.2~CDDY5QHD0S4Z.0 size=338095 2024-09-13T00:00:36.885 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.882+0000 7fb8f2ce0640 1 < completed multipart upload key=1HX06CCZJ1DD upload=CDDY5QHD0S4Z tag=0EKZS4TEJGBZ type=write size=1386671 2024-09-13T00:00:36.892 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.890+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~WVWB6NV5QPKU.0 size=1048576 2024-09-13T00:00:36.895 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.894+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~WVWB6NV5QPKU.1 size=375170 2024-09-13T00:00:36.895 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.894+0000 7fb8f2ce0640 1 > prepared multipart upload key=FX0NCGLK0KPJ upload=WVWB6NV5QPKU tag=FDHGBIBXZAFW type=write size=2472322 2024-09-13T00:00:36.899 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.898+0000 7fb8f2ce0640 1 < completed part key=_multipart_1HX06CCZJ1DD.2~F5WBBU6FN8MH.0 size=355091 2024-09-13T00:00:36.905 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.906+0000 7fb8f2ce0640 1 < completed multipart upload key=1HX06CCZJ1DD upload=F5WBBU6FN8MH tag=10PVLOUAW6S0 type=write size=1403667 2024-09-13T00:00:36.909 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.910+0000 7fb8f2ce0640 1 > prepared operation key=ZFHN2SNTZ5W7 tag=KEMAH1KJ1K0F type=del size=32753 2024-09-13T00:00:36.913 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.914+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZFHN2SNTZ5W7.2~A1D6RUREUHUI.0 size=1048576 2024-09-13T00:00:36.916 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.914+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZFHN2SNTZ5W7.2~A1D6RUREUHUI.1 size=1048576 2024-09-13T00:00:36.920 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.918+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZFHN2SNTZ5W7.2~A1D6RUREUHUI.2 size=621373 2024-09-13T00:00:36.926 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.926+0000 7fb8f2ce0640 1 < completed multipart upload key=ZFHN2SNTZ5W7 upload=A1D6RUREUHUI tag=0FXDPNGUY1WH type=write size=3767101 2024-09-13T00:00:36.933 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.934+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~F21V42UJYRU8.0 size=438946 2024-09-13T00:00:36.934 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.934+0000 7fb8f2ce0640 1 > prepared multipart upload key=8Z6U7OOA0GBH upload=F21V42UJYRU8 tag=LKAPBZLZZWS0 type=write size=1487522 2024-09-13T00:00:36.937 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.938+0000 7fb8f2ce0640 1 < canceled operation key=NX26SI4F2ZMB tag=T7P2GD8KD08R type=del size=1463616 2024-09-13T00:00:36.941 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.942+0000 7fb8f2ce0640 1 > prepared operation key=0WC6P8YNL0H2 tag=F3LTOPXAG7NC type=del size=2606146 2024-09-13T00:00:36.945 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.946+0000 7fb8f2ce0640 1 < completed delete operation key=EJN6HR8I7XQF tag=VGX0VI55NZ4M type=del 2024-09-13T00:00:36.949 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.950+0000 7fb8f2ce0640 1 > prepared operation key=FX0NCGLK0KPJ tag=2LA4ARUJ7ONU type=del size=1597848 2024-09-13T00:00:36.953 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.954+0000 7fb8f2ce0640 1 < completed part key=_multipart_NX26SI4F2ZMB.2~1NHPWBL6NLMT.0 size=76800 2024-09-13T00:00:36.960 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.958+0000 7fb8f2ce0640 1 < completed multipart upload key=NX26SI4F2ZMB upload=1NHPWBL6NLMT tag=EO07BAY1XJ1W type=write size=1125376 2024-09-13T00:00:36.968 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.966+0000 7fb8f2ce0640 1 > prepared part key=_multipart_4K76U69GZ9AE.2~IKRCMQI15GUD.0 size=1048576 2024-09-13T00:00:36.971 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.970+0000 7fb8f2ce0640 1 > prepared part key=_multipart_4K76U69GZ9AE.2~IKRCMQI15GUD.1 size=594136 2024-09-13T00:00:36.971 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.970+0000 7fb8f2ce0640 1 > prepared multipart upload key=4K76U69GZ9AE upload=IKRCMQI15GUD tag=3L3GE6PKARO0 type=write size=2691288 2024-09-13T00:00:36.976 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.974+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~5TB1ZRKMFZO3.0 size=1048576 2024-09-13T00:00:36.979 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.978+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~5TB1ZRKMFZO3.1 size=1048576 2024-09-13T00:00:36.983 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.982+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~5TB1ZRKMFZO3.2 size=429193 2024-09-13T00:00:36.990 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.990+0000 7fb8f2ce0640 1 < completed multipart upload key=ZTK9AYBSLCOZ upload=5TB1ZRKMFZO3 tag=FT9IINFLDCGB type=write size=3574921 2024-09-13T00:00:36.998 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:36.998+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~MWQVO3NIUKKN.0 size=1048576 2024-09-13T00:00:37.001 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.002+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~MWQVO3NIUKKN.1 size=1048576 2024-09-13T00:00:37.004 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.002+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~MWQVO3NIUKKN.2 size=374721 2024-09-13T00:00:37.004 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.002+0000 7fb8f2ce0640 1 > prepared multipart upload key=FX0NCGLK0KPJ upload=MWQVO3NIUKKN tag=3CL8UFN9XPWJ type=write size=3520449 2024-09-13T00:00:37.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:36 smithi136 bash[75164]: cluster 2024-09-13T00:00:35.126310+0000 mgr.y (mgr.24661) 704 : cluster [DBG] pgmap v1380: 137 pgs: 3 creating+peering, 134 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 4.3 KiB/s wr, 16 op/s 2024-09-13T00:00:37.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:36 smithi136 bash[76371]: cluster 2024-09-13T00:00:35.126310+0000 mgr.y (mgr.24661) 704 : cluster [DBG] pgmap v1380: 137 pgs: 3 creating+peering, 134 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 4.3 KiB/s wr, 16 op/s 2024-09-13T00:00:37.012 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.010+0000 7fb8f2ce0640 1 < suggested remove operation key=ZTK9AYBSLCOZ tag=V767J6KS12EC type=del 2024-09-13T00:00:37.017 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.018+0000 7fb8f2ce0640 1 > prepared operation key=NX26SI4F2ZMB tag=OJLD1KOKSOW1 type=del size=607462 2024-09-13T00:00:37.021 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.022+0000 7fb8f2ce0640 1 < completed delete operation key=H5EIP0E9CBWX tag=0RKHP640S9O0 type=del 2024-09-13T00:00:37.026 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.026+0000 7fb8f2ce0640 1 > prepared operation key=QMZREZ1QEDNJ tag=05OR38RPOVGJ type=del size=1015137 2024-09-13T00:00:37.030 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.030+0000 7fb8f2ce0640 1 < completed part key=_multipart_63FJDWBO2T9S.2~31MBCG2D2I0Q.0 size=934835 2024-09-13T00:00:37.037 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.038+0000 7fb8f2ce0640 1 < completed multipart upload key=63FJDWBO2T9S upload=31MBCG2D2I0Q tag=SASBRRJD3Q89 type=write size=1983411 2024-09-13T00:00:37.041 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.042+0000 7fb8f2ce0640 1 > prepared operation key=EJN6HR8I7XQF tag=CMZLU6KOV2H0 type=del size=2094682 2024-09-13T00:00:37.050 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.050+0000 7fb8f2ce0640 1 < suggested remove operation key=NX26SI4F2ZMB tag=CQREFLDJJ6HX type=del 2024-09-13T00:00:37.058 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.058+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~XQKQI88UCT3V.0 size=1048576 2024-09-13T00:00:37.061 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.062+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~XQKQI88UCT3V.1 size=421229 2024-09-13T00:00:37.061 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.062+0000 7fb8f2ce0640 1 > prepared multipart upload key=8MPUVD37EDN9 upload=XQKQI88UCT3V tag=TXBV6F903BLF type=write size=2518381 2024-09-13T00:00:37.065 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.066+0000 7fb8f2ce0640 1 < completed part key=_multipart_P8OYS5ZE1NX0.2~SEL21UVYEQXP.0 size=1048576 2024-09-13T00:00:37.069 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.070+0000 7fb8f2ce0640 1 < completed part key=_multipart_P8OYS5ZE1NX0.2~SEL21UVYEQXP.1 size=645632 2024-09-13T00:00:37.076 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.074+0000 7fb8f2ce0640 1 < completed multipart upload key=P8OYS5ZE1NX0 upload=SEL21UVYEQXP tag=HXXUB893T44R type=write size=2742784 2024-09-13T00:00:37.080 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.078+0000 7fb8f2ce0640 1 > prepared operation key=EJN6HR8I7XQF tag=QFIUFBOMZVPK type=del size=3207137 2024-09-13T00:00:37.089 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.090+0000 7fb8f2ce0640 1 < suggested remove operation key=4K76U69GZ9AE tag=3LFJZUUUCFT0 type=del 2024-09-13T00:00:37.094 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.094+0000 7fb8f2ce0640 1 > prepared operation key=8MPUVD37EDN9 tag=CEL0SG9AC8DK type=del size=3790980 2024-09-13T00:00:37.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:36 smithi181 bash[67587]: cluster 2024-09-13T00:00:35.126310+0000 mgr.y (mgr.24661) 704 : cluster [DBG] pgmap v1380: 137 pgs: 3 creating+peering, 134 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 4.3 KiB/s wr, 16 op/s 2024-09-13T00:00:37.098 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.098+0000 7fb8f2ce0640 1 < completed part key=_multipart_PVYZQLGYT3XU.2~CYOUCMPG7PKJ.0 size=1048576 2024-09-13T00:00:37.102 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.102+0000 7fb8f2ce0640 1 < completed part key=_multipart_PVYZQLGYT3XU.2~CYOUCMPG7PKJ.1 size=1048576 2024-09-13T00:00:37.105 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.106+0000 7fb8f2ce0640 1 < completed part key=_multipart_PVYZQLGYT3XU.2~CYOUCMPG7PKJ.2 size=472782 2024-09-13T00:00:37.112 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.110+0000 7fb8f2ce0640 1 < completed multipart upload key=PVYZQLGYT3XU upload=CYOUCMPG7PKJ tag=X0FDARXCDUAA type=write size=3618510 2024-09-13T00:00:37.120 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.118+0000 7fb8f2ce0640 1 > prepared part key=_multipart_N3Z5XIG3TZRQ.2~7X1THEEGY7JS.0 size=1048576 2024-09-13T00:00:37.123 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.122+0000 7fb8f2ce0640 1 > prepared part key=_multipart_N3Z5XIG3TZRQ.2~7X1THEEGY7JS.1 size=70302 2024-09-13T00:00:37.123 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.122+0000 7fb8f2ce0640 1 > prepared multipart upload key=N3Z5XIG3TZRQ upload=7X1THEEGY7JS tag=MESI7WBH39ZT type=write size=2167454 2024-09-13T00:00:37.127 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.126+0000 7fb8f2ce0640 1 < completed part key=_multipart_85YEW8N43FTA.2~ZC5U69MSVBFD.0 size=1048576 2024-09-13T00:00:37.127 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.126+0000 7fb8f2ce0640 1 < canceled part key=_multipart_85YEW8N43FTA.2~ZC5U69MSVBFD.1 size=1048576 2024-09-13T00:00:37.131 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.130+0000 7fb8f2ce0640 1 < completed part key=_multipart_85YEW8N43FTA.2~ZC5U69MSVBFD.2 size=314005 2024-09-13T00:00:37.139 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.138+0000 7fb8f2ce0640 1 < canceled multipart upload key=85YEW8N43FTA upload=ZC5U69MSVBFD tag=B5QGGNB2M16B type=write size=3459733 2024-09-13T00:00:37.147 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.146+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H5EIP0E9CBWX.2~SPAMINLKBK1Y.0 size=1048576 2024-09-13T00:00:37.151 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.150+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H5EIP0E9CBWX.2~SPAMINLKBK1Y.1 size=1048576 2024-09-13T00:00:37.154 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.154+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H5EIP0E9CBWX.2~SPAMINLKBK1Y.2 size=15467 2024-09-13T00:00:37.154 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.154+0000 7fb8f2ce0640 1 > prepared multipart upload key=H5EIP0E9CBWX upload=SPAMINLKBK1Y tag=RHJ0GFWKHA2C type=write size=3161195 2024-09-13T00:00:37.160 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.158+0000 7fb8f2ce0640 -1 < failed to complete operation key=NX26SI4F2ZMB tag=OQ48DS5H66CR type=del size=1613410: (22) Invalid argument 2024-09-13T00:00:37.165 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.162+0000 7fb8f2ce0640 1 > prepared operation key=NQJQC8CEAC15 tag=0C1UEXNUCGJG type=write size=272187 2024-09-13T00:00:37.169 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.170+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~WVWB6NV5QPKU.0 size=1048576 2024-09-13T00:00:37.172 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.170+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~WVWB6NV5QPKU.1 size=375170 2024-09-13T00:00:37.179 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.178+0000 7fb8f2ce0640 1 < completed multipart upload key=FX0NCGLK0KPJ upload=WVWB6NV5QPKU tag=FDHGBIBXZAFW type=write size=2472322 2024-09-13T00:00:37.183 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.182+0000 7fb8f2ce0640 1 > prepared operation key=1PX7DBO9M3YI tag=KKOS00Q7WBKN type=del size=1968577 2024-09-13T00:00:37.188 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.186+0000 7fb8f2ce0640 1 < completed delete operation key=ZFHN2SNTZ5W7 tag=KEMAH1KJ1K0F type=del 2024-09-13T00:00:37.192 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.190+0000 7fb8f2ce0640 1 > prepared operation key=NX26SI4F2ZMB tag=0Z25L775YK1V type=del size=72617 2024-09-13T00:00:37.196 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.194+0000 7fb8f2ce0640 1 < completed part key=_multipart_8Z6U7OOA0GBH.2~F21V42UJYRU8.0 size=438946 2024-09-13T00:00:37.203 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.202+0000 7fb8f2ce0640 1 < completed multipart upload key=8Z6U7OOA0GBH upload=F21V42UJYRU8 tag=LKAPBZLZZWS0 type=write size=1487522 2024-09-13T00:00:37.207 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.206+0000 7fb8f2ce0640 1 > prepared operation key=EJN6HR8I7XQF tag=0EJO6G29O05V type=del size=3167604 2024-09-13T00:00:37.216 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.214+0000 7fb8f2ce0640 1 < suggested update operation key=0WC6P8YNL0H2 tag=F3LTOPXAG7NC type=del size=2606146 2024-09-13T00:00:37.220 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.218+0000 7fb8f2ce0640 1 > prepared operation key=8MPUVD37EDN9 tag=7SGETSBORAXB type=write size=120456 2024-09-13T00:00:37.225 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.226+0000 7fb8f2ce0640 1 < completed delete operation key=FX0NCGLK0KPJ tag=2LA4ARUJ7ONU type=del 2024-09-13T00:00:37.229 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.230+0000 7fb8f2ce0640 1 > prepared operation key=0WC6P8YNL0H2 tag=AFVR22RMX8XF type=del size=3561181 2024-09-13T00:00:37.233 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.234+0000 7fb8f2ce0640 1 < completed part key=_multipart_4K76U69GZ9AE.2~IKRCMQI15GUD.0 size=1048576 2024-09-13T00:00:37.237 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.238+0000 7fb8f2ce0640 1 < completed part key=_multipart_4K76U69GZ9AE.2~IKRCMQI15GUD.1 size=594136 2024-09-13T00:00:37.243 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.242+0000 7fb8f2ce0640 1 < completed multipart upload key=4K76U69GZ9AE upload=IKRCMQI15GUD tag=3L3GE6PKARO0 type=write size=2691288 2024-09-13T00:00:37.251 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.250+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~XV1ITEWNNCNL.0 size=1048576 2024-09-13T00:00:37.255 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.254+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~XV1ITEWNNCNL.1 size=1048576 2024-09-13T00:00:37.258 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.258+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~XV1ITEWNNCNL.2 size=72857 2024-09-13T00:00:37.258 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.258+0000 7fb8f2ce0640 1 > prepared multipart upload key=7OX5OJZ8GXU5 upload=XV1ITEWNNCNL tag=PHBPQ2ROW6FU type=write size=3218585 2024-09-13T00:00:37.259 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.258+0000 7fb8f2ce0640 1 < canceled part key=_multipart_FX0NCGLK0KPJ.2~MWQVO3NIUKKN.0 size=1048576 2024-09-13T00:00:37.263 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.262+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~MWQVO3NIUKKN.1 size=1048576 2024-09-13T00:00:37.266 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.266+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~MWQVO3NIUKKN.2 size=374721 2024-09-13T00:00:37.273 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.274+0000 7fb8f2ce0640 1 < canceled multipart upload key=FX0NCGLK0KPJ upload=MWQVO3NIUKKN tag=3CL8UFN9XPWJ type=write size=3520449 2024-09-13T00:00:37.277 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.278+0000 7fb8f2ce0640 1 > prepared part key=_multipart_5GM88ERM89KT.2~IAJ5NVJ5C51X.0 size=1048576 2024-09-13T00:00:37.279 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.278+0000 7fb8f2ce0640 1 > prepared part key=_multipart_5GM88ERM89KT.2~IAJ5NVJ5C51X.1 size=13562 2024-09-13T00:00:37.279 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.278+0000 7fb8f2ce0640 1 > prepared multipart upload key=5GM88ERM89KT upload=IAJ5NVJ5C51X tag=WPIA04QXTSPY type=write size=2110714 2024-09-13T00:00:37.282 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.282+0000 7fb8f2ce0640 -1 < failed to complete operation key=NX26SI4F2ZMB tag=OJLD1KOKSOW1 type=del size=607462: (22) Invalid argument 2024-09-13T00:00:37.284 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.282+0000 7fb8f2ce0640 1 > prepared operation key=QMZREZ1QEDNJ tag=UT3P38RM6CUM type=write size=317206 2024-09-13T00:00:37.286 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.286+0000 7fb8f2ce0640 1 < completed delete operation key=QMZREZ1QEDNJ tag=05OR38RPOVGJ type=del 2024-09-13T00:00:37.288 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.286+0000 7fb8f2ce0640 1 > prepared operation key=5GM88ERM89KT tag=TAQD9L61AVJ7 type=del size=1277496 2024-09-13T00:00:37.291 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.290+0000 7fb8f2ce0640 1 < canceled operation key=EJN6HR8I7XQF tag=CMZLU6KOV2H0 type=del size=2094682 2024-09-13T00:00:37.294 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.294+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~NJ1RP91VO2WK.0 size=1048576 2024-09-13T00:00:37.296 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.294+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~NJ1RP91VO2WK.1 size=19439 2024-09-13T00:00:37.296 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.294+0000 7fb8f2ce0640 1 > prepared multipart upload key=8MPUVD37EDN9 upload=NJ1RP91VO2WK tag=SN15WZYRBDGO type=write size=2116591 2024-09-13T00:00:37.298 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.298+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~XQKQI88UCT3V.0 size=1048576 2024-09-13T00:00:37.300 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.298+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~XQKQI88UCT3V.1 size=421229 2024-09-13T00:00:37.303 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.302+0000 7fb8f2ce0640 1 < completed multipart upload key=8MPUVD37EDN9 upload=XQKQI88UCT3V tag=TXBV6F903BLF type=write size=2518381 2024-09-13T00:00:37.305 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.306+0000 7fb8f2ce0640 1 > prepared operation key=FX0NCGLK0KPJ tag=QQOG5JEW8U81 type=del size=1754824 2024-09-13T00:00:37.307 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.306+0000 7fb8f2ce0640 1 < canceled operation key=EJN6HR8I7XQF tag=QFIUFBOMZVPK type=del size=3207137 2024-09-13T00:00:37.308 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.306+0000 7fb8f2ce0640 1 > prepared operation key=EUE3LT0PDIXV tag=D8ANAQVAYMKF type=del size=3768005 2024-09-13T00:00:37.312 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.310+0000 7fb8f2ce0640 1 < suggested remove operation key=8MPUVD37EDN9 tag=CEL0SG9AC8DK type=del 2024-09-13T00:00:37.314 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.314+0000 7fb8f2ce0640 1 > prepared operation key=J8NGDSF9XRTP tag=CLOTDD5CUHV7 type=del size=1300784 2024-09-13T00:00:37.316 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.314+0000 7fb8f2ce0640 1 < completed part key=_multipart_N3Z5XIG3TZRQ.2~7X1THEEGY7JS.0 size=1048576 2024-09-13T00:00:37.317 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.318+0000 7fb8f2ce0640 1 < completed part key=_multipart_N3Z5XIG3TZRQ.2~7X1THEEGY7JS.1 size=70302 2024-09-13T00:00:37.320 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.318+0000 7fb8f2ce0640 1 < completed multipart upload key=N3Z5XIG3TZRQ upload=7X1THEEGY7JS tag=MESI7WBH39ZT type=write size=2167454 2024-09-13T00:00:37.322 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.322+0000 7fb8f2ce0640 1 > prepared operation key=8Z6U7OOA0GBH tag=NF9JALH3DR8X type=del size=2154064 2024-09-13T00:00:37.324 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.322+0000 7fb8f2ce0640 1 < completed part key=_multipart_H5EIP0E9CBWX.2~SPAMINLKBK1Y.0 size=1048576 2024-09-13T00:00:37.325 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.326+0000 7fb8f2ce0640 1 < completed part key=_multipart_H5EIP0E9CBWX.2~SPAMINLKBK1Y.1 size=1048576 2024-09-13T00:00:37.327 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.326+0000 7fb8f2ce0640 1 < completed part key=_multipart_H5EIP0E9CBWX.2~SPAMINLKBK1Y.2 size=15467 2024-09-13T00:00:37.330 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.330+0000 7fb8f2ce0640 1 < completed multipart upload key=H5EIP0E9CBWX upload=SPAMINLKBK1Y tag=RHJ0GFWKHA2C type=write size=3161195 2024-09-13T00:00:37.331 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.330+0000 7fb8f2ce0640 1 > prepared operation key=1HX06CCZJ1DD tag=DG60H41KLC1R type=write size=906593 2024-09-13T00:00:37.333 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.334+0000 7fb8f2ce0640 1 < completed write operation key=NQJQC8CEAC15 tag=0C1UEXNUCGJG type=write size=272187 2024-09-13T00:00:37.335 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.334+0000 7fb8f2ce0640 1 > prepared operation key=H5EIP0E9CBWX tag=2TCCH81OSA2Q type=del size=1085761 2024-09-13T00:00:37.339 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.338+0000 7fb8f2ce0640 1 < suggested remove operation key=1PX7DBO9M3YI tag=KKOS00Q7WBKN type=del 2024-09-13T00:00:37.342 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.342+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~09ZMMOJC9OEH.0 size=360015 2024-09-13T00:00:37.342 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.342+0000 7fb8f2ce0640 1 > prepared multipart upload key=8MPUVD37EDN9 upload=09ZMMOJC9OEH tag=PW1M94FU814M type=write size=1408591 2024-09-13T00:00:37.344 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.342+0000 7fb8f2ce0640 1 < completed delete operation key=NX26SI4F2ZMB tag=0Z25L775YK1V type=del 2024-09-13T00:00:37.346 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.346+0000 7fb8f2ce0640 1 > prepared operation key=1PX7DBO9M3YI tag=U90WLSD1UVAD type=del size=2958048 2024-09-13T00:00:37.348 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.346+0000 7fb8f2ce0640 1 < canceled operation key=EJN6HR8I7XQF tag=0EJO6G29O05V type=del size=3167604 2024-09-13T00:00:37.353 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.350+0000 7fb8f2ce0640 1 > prepared part key=_multipart_N3Z5XIG3TZRQ.2~40E95OYTDAQ8.0 size=1048576 2024-09-13T00:00:37.354 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.354+0000 7fb8f2ce0640 1 > prepared part key=_multipart_N3Z5XIG3TZRQ.2~40E95OYTDAQ8.1 size=485027 2024-09-13T00:00:37.354 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.354+0000 7fb8f2ce0640 1 > prepared multipart upload key=N3Z5XIG3TZRQ upload=40E95OYTDAQ8 tag=P5D92FJRF1SO type=write size=2582179 2024-09-13T00:00:37.357 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.358+0000 7fb8f2ce0640 -1 < failed to complete operation key=8MPUVD37EDN9 tag=7SGETSBORAXB type=write size=120456: (22) Invalid argument 2024-09-13T00:00:37.359 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.358+0000 7fb8f2ce0640 1 > prepared operation key=70F7D20HI504 tag=AFR23EYVRIN1 type=write size=366301 2024-09-13T00:00:37.361 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.362+0000 7fb8f2ce0640 1 < completed delete operation key=0WC6P8YNL0H2 tag=AFVR22RMX8XF type=del 2024-09-13T00:00:37.364 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.362+0000 7fb8f2ce0640 1 > prepared operation key=63FJDWBO2T9S tag=X6PQ6DYR13PM type=del size=2656570 2024-09-13T00:00:37.366 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.366+0000 7fb8f2ce0640 1 < completed part key=_multipart_7OX5OJZ8GXU5.2~XV1ITEWNNCNL.0 size=1048576 2024-09-13T00:00:37.369 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.366+0000 7fb8f2ce0640 1 < completed part key=_multipart_7OX5OJZ8GXU5.2~XV1ITEWNNCNL.1 size=1048576 2024-09-13T00:00:37.371 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.370+0000 7fb8f2ce0640 1 < completed part key=_multipart_7OX5OJZ8GXU5.2~XV1ITEWNNCNL.2 size=72857 2024-09-13T00:00:37.375 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.374+0000 7fb8f2ce0640 1 < completed multipart upload key=7OX5OJZ8GXU5 upload=XV1ITEWNNCNL tag=PHBPQ2ROW6FU type=write size=3218585 2024-09-13T00:00:37.379 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.378+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~A5KTU6HVK9GL.0 size=998102 2024-09-13T00:00:37.379 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.378+0000 7fb8f2ce0640 1 > prepared multipart upload key=PVYZQLGYT3XU upload=A5KTU6HVK9GL tag=JUPB7LOF0D22 type=write size=2046678 2024-09-13T00:00:37.382 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.382+0000 7fb8f2ce0640 1 < completed part key=_multipart_5GM88ERM89KT.2~IAJ5NVJ5C51X.0 size=1048576 2024-09-13T00:00:37.384 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.382+0000 7fb8f2ce0640 1 < completed part key=_multipart_5GM88ERM89KT.2~IAJ5NVJ5C51X.1 size=13562 2024-09-13T00:00:37.389 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.390+0000 7fb8f2ce0640 1 < completed multipart upload key=5GM88ERM89KT upload=IAJ5NVJ5C51X tag=WPIA04QXTSPY type=write size=2110714 2024-09-13T00:00:37.394 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.394+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~Z43ZY1E03G6V.0 size=937544 2024-09-13T00:00:37.394 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.394+0000 7fb8f2ce0640 1 > prepared multipart upload key=63FJDWBO2T9S upload=Z43ZY1E03G6V tag=0D8H1DR5RP3O type=write size=1986120 2024-09-13T00:00:37.397 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.398+0000 7fb8f2ce0640 1 < canceled operation key=QMZREZ1QEDNJ tag=UT3P38RM6CUM type=write size=317206 2024-09-13T00:00:37.401 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.398+0000 7fb8f2ce0640 1 > prepared operation key=2AHYRZUDFHP4 tag=2YC2PH4AX8KH type=del size=3774759 2024-09-13T00:00:37.408 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.406+0000 7fb8f2ce0640 1 < suggested remove operation key=5GM88ERM89KT tag=TAQD9L61AVJ7 type=del 2024-09-13T00:00:37.411 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.410+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~NOFM7U08NUVD.0 size=1048576 2024-09-13T00:00:37.414 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.414+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~NOFM7U08NUVD.1 size=392973 2024-09-13T00:00:37.414 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.414+0000 7fb8f2ce0640 1 > prepared multipart upload key=70F7D20HI504 upload=NOFM7U08NUVD tag=JEC3MJJD5AHG type=write size=2490125 2024-09-13T00:00:37.417 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.414+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~NJ1RP91VO2WK.0 size=1048576 2024-09-13T00:00:37.419 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.418+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~NJ1RP91VO2WK.1 size=19439 2024-09-13T00:00:37.424 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.422+0000 7fb8f2ce0640 1 < completed multipart upload key=8MPUVD37EDN9 upload=NJ1RP91VO2WK tag=SN15WZYRBDGO type=write size=2116591 2024-09-13T00:00:37.429 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.430+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~A7ZEWGP69SHU.0 size=1048576 2024-09-13T00:00:37.432 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.430+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~A7ZEWGP69SHU.1 size=1048576 2024-09-13T00:00:37.434 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.434+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~A7ZEWGP69SHU.2 size=382016 2024-09-13T00:00:37.434 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.434+0000 7fb8f2ce0640 1 > prepared multipart upload key=1HX06CCZJ1DD upload=A7ZEWGP69SHU tag=YWQOTSKWV0WZ type=write size=3527744 2024-09-13T00:00:37.437 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.438+0000 7fb8f2ce0640 1 < completed delete operation key=FX0NCGLK0KPJ tag=QQOG5JEW8U81 type=del 2024-09-13T00:00:37.442 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.442+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~0KMQTL94DSK0.0 size=438958 2024-09-13T00:00:37.442 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.442+0000 7fb8f2ce0640 1 > prepared multipart upload key=2AHYRZUDFHP4 upload=0KMQTL94DSK0 tag=QSPO345IHX9A type=write size=1487534 2024-09-13T00:00:37.446 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.446+0000 7fb8f2ce0640 1 < canceled operation key=EUE3LT0PDIXV tag=D8ANAQVAYMKF type=del size=3768005 2024-09-13T00:00:37.449 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.450+0000 7fb8f2ce0640 1 > prepared operation key=H5EIP0E9CBWX tag=V2ZTQM9RA15V type=del size=3239720 2024-09-13T00:00:37.453 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.454+0000 7fb8f2ce0640 1 < completed delete operation key=J8NGDSF9XRTP tag=CLOTDD5CUHV7 type=del 2024-09-13T00:00:37.458 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.458+0000 7fb8f2ce0640 1 > prepared operation key=J8NGDSF9XRTP tag=2W3MF6X2S1U3 type=del size=2737327 2024-09-13T00:00:37.461 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.462+0000 7fb8f2ce0640 1 < completed delete operation key=8Z6U7OOA0GBH tag=NF9JALH3DR8X type=del 2024-09-13T00:00:37.465 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.466+0000 7fb8f2ce0640 1 > prepared operation key=H5EIP0E9CBWX tag=R3D13JVCUISA type=del size=1477788 2024-09-13T00:00:37.469 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.466+0000 7fb8f2ce0640 1 < canceled operation key=1HX06CCZJ1DD tag=DG60H41KLC1R type=write size=906593 2024-09-13T00:00:37.475 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.474+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~4H0W40DRF4K9.0 size=1048576 2024-09-13T00:00:37.478 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.478+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~4H0W40DRF4K9.1 size=1048576 2024-09-13T00:00:37.481 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.482+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~4H0W40DRF4K9.2 size=203367 2024-09-13T00:00:37.481 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.482+0000 7fb8f2ce0640 1 > prepared multipart upload key=NQJQC8CEAC15 upload=4H0W40DRF4K9 tag=15N8IX13V0S6 type=write size=3349095 2024-09-13T00:00:37.485 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.486+0000 7fb8f2ce0640 1 < completed delete operation key=H5EIP0E9CBWX tag=2TCCH81OSA2Q type=del 2024-09-13T00:00:37.492 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.490+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~GDFE21AMU7BM.0 size=1048576 2024-09-13T00:00:37.495 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.494+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~GDFE21AMU7BM.1 size=49293 2024-09-13T00:00:37.496 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.494+0000 7fb8f2ce0640 1 > prepared multipart upload key=PVYZQLGYT3XU upload=GDFE21AMU7BM tag=Z0LPLBGL9T0X type=write size=2146445 2024-09-13T00:00:37.500 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.498+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~09ZMMOJC9OEH.0 size=360015 2024-09-13T00:00:37.507 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.506+0000 7fb8f2ce0640 1 < completed multipart upload key=8MPUVD37EDN9 upload=09ZMMOJC9OEH tag=PW1M94FU814M type=write size=1408591 2024-09-13T00:00:37.515 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.514+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~EMEKGK6WIXYA.0 size=1048576 2024-09-13T00:00:37.518 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.518+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~EMEKGK6WIXYA.1 size=672009 2024-09-13T00:00:37.518 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.518+0000 7fb8f2ce0640 1 > prepared multipart upload key=70F7D20HI504 upload=EMEKGK6WIXYA tag=3NHMOESCYT3L type=write size=2769161 2024-09-13T00:00:37.522 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.522+0000 7fb8f2ce0640 1 < completed delete operation key=1PX7DBO9M3YI tag=U90WLSD1UVAD type=del 2024-09-13T00:00:37.526 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.526+0000 7fb8f2ce0640 1 > prepared operation key=FX0NCGLK0KPJ tag=1M99KCII33SI type=del size=617565 2024-09-13T00:00:37.531 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.530+0000 7fb8f2ce0640 1 < completed part key=_multipart_N3Z5XIG3TZRQ.2~40E95OYTDAQ8.0 size=1048576 2024-09-13T00:00:37.534 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.534+0000 7fb8f2ce0640 1 < completed part key=_multipart_N3Z5XIG3TZRQ.2~40E95OYTDAQ8.1 size=485027 2024-09-13T00:00:37.542 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.542+0000 7fb8f2ce0640 1 < completed multipart upload key=N3Z5XIG3TZRQ upload=40E95OYTDAQ8 tag=P5D92FJRF1SO type=write size=2582179 2024-09-13T00:00:37.549 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.550+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~O9H3P29MT42D.0 size=1048576 2024-09-13T00:00:37.552 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.550+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~O9H3P29MT42D.1 size=1048576 2024-09-13T00:00:37.555 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.554+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~O9H3P29MT42D.2 size=505523 2024-09-13T00:00:37.555 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.554+0000 7fb8f2ce0640 1 > prepared multipart upload key=NQJQC8CEAC15 upload=O9H3P29MT42D tag=M3ZKGZF9I1W3 type=write size=3651251 2024-09-13T00:00:37.559 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.558+0000 7fb8f2ce0640 1 < completed write operation key=70F7D20HI504 tag=AFR23EYVRIN1 type=write size=366301 2024-09-13T00:00:37.563 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.562+0000 7fb8f2ce0640 1 > prepared operation key=8Z6U7OOA0GBH tag=UOD1IK9IVS14 type=del size=1193949 2024-09-13T00:00:37.567 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.566+0000 7fb8f2ce0640 1 < completed delete operation key=63FJDWBO2T9S tag=X6PQ6DYR13PM type=del 2024-09-13T00:00:37.575 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.574+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~HHOO46126XWY.0 size=322873 2024-09-13T00:00:37.575 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.574+0000 7fb8f2ce0640 1 > prepared multipart upload key=1PX7DBO9M3YI upload=HHOO46126XWY tag=8LH4D1OKQ243 type=write size=1371449 2024-09-13T00:00:37.579 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.578+0000 7fb8f2ce0640 1 < completed part key=_multipart_PVYZQLGYT3XU.2~A5KTU6HVK9GL.0 size=998102 2024-09-13T00:00:37.586 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.586+0000 7fb8f2ce0640 1 < completed multipart upload key=PVYZQLGYT3XU upload=A5KTU6HVK9GL tag=JUPB7LOF0D22 type=write size=2046678 2024-09-13T00:00:37.591 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.590+0000 7fb8f2ce0640 1 > prepared operation key=FX0NCGLK0KPJ tag=F84A255ZAKSZ type=del size=3946165 2024-09-13T00:00:37.596 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.594+0000 7fb8f2ce0640 1 < completed part key=_multipart_63FJDWBO2T9S.2~Z43ZY1E03G6V.0 size=937544 2024-09-13T00:00:37.602 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.602+0000 7fb8f2ce0640 1 < completed multipart upload key=63FJDWBO2T9S upload=Z43ZY1E03G6V tag=0D8H1DR5RP3O type=write size=1986120 2024-09-13T00:00:37.606 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.606+0000 7fb8f2ce0640 1 > prepared operation key=ZFHN2SNTZ5W7 tag=1XRHJKE7A23U type=del size=928734 2024-09-13T00:00:37.615 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.614+0000 7fb8f2ce0640 1 < suggested remove operation key=2AHYRZUDFHP4 tag=2YC2PH4AX8KH type=del 2024-09-13T00:00:37.620 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.618+0000 7fb8f2ce0640 1 > prepared operation key=5GM88ERM89KT tag=0NSI7TCGVLPO type=del size=3957558 2024-09-13T00:00:37.624 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.622+0000 7fb8f2ce0640 1 < completed part key=_multipart_70F7D20HI504.2~NOFM7U08NUVD.0 size=1048576 2024-09-13T00:00:37.627 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.626+0000 7fb8f2ce0640 1 < completed part key=_multipart_70F7D20HI504.2~NOFM7U08NUVD.1 size=392973 2024-09-13T00:00:37.635 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.634+0000 7fb8f2ce0640 1 < completed multipart upload key=70F7D20HI504 upload=NOFM7U08NUVD tag=JEC3MJJD5AHG type=write size=2490125 2024-09-13T00:00:37.640 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.638+0000 7fb8f2ce0640 1 > prepared operation key=ZFHN2SNTZ5W7 tag=KCVAS6KYG5D1 type=del size=696516 2024-09-13T00:00:37.645 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.642+0000 7fb8f2ce0640 1 < completed part key=_multipart_1HX06CCZJ1DD.2~A7ZEWGP69SHU.0 size=1048576 2024-09-13T00:00:37.648 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.646+0000 7fb8f2ce0640 1 < completed part key=_multipart_1HX06CCZJ1DD.2~A7ZEWGP69SHU.1 size=1048576 2024-09-13T00:00:37.648 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.646+0000 7fb8f2ce0640 1 < canceled part key=_multipart_1HX06CCZJ1DD.2~A7ZEWGP69SHU.2 size=382016 2024-09-13T00:00:37.655 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.654+0000 7fb8f2ce0640 1 < canceled multipart upload key=1HX06CCZJ1DD upload=A7ZEWGP69SHU tag=YWQOTSKWV0WZ type=write size=3527744 2024-09-13T00:00:37.663 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.662+0000 7fb8f2ce0640 1 > prepared part key=_multipart_44EDYFT9VF8Y.2~MPEXHIOUJXBE.0 size=957715 2024-09-13T00:00:37.663 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.662+0000 7fb8f2ce0640 1 > prepared multipart upload key=44EDYFT9VF8Y upload=MPEXHIOUJXBE tag=EEUVOOET0QCE type=write size=2006291 2024-09-13T00:00:37.668 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.666+0000 7fb8f2ce0640 1 < completed part key=_multipart_2AHYRZUDFHP4.2~0KMQTL94DSK0.0 size=438958 2024-09-13T00:00:37.675 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.674+0000 7fb8f2ce0640 1 < completed multipart upload key=2AHYRZUDFHP4 upload=0KMQTL94DSK0 tag=QSPO345IHX9A type=write size=1487534 2024-09-13T00:00:37.679 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.678+0000 7fb8f2ce0640 1 > prepared operation key=70F7D20HI504 tag=GS9O32745FXW type=write size=588988 2024-09-13T00:00:37.684 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.682+0000 7fb8f2ce0640 1 < completed delete operation key=H5EIP0E9CBWX tag=V2ZTQM9RA15V type=del 2024-09-13T00:00:37.691 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.690+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~HA45BS2PH8ZC.0 size=1048576 2024-09-13T00:00:37.694 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.694+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~HA45BS2PH8ZC.1 size=699193 2024-09-13T00:00:37.694 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.694+0000 7fb8f2ce0640 1 > prepared multipart upload key=NQJQC8CEAC15 upload=HA45BS2PH8ZC tag=EJ2EH964S8IJ type=write size=2796345 2024-09-13T00:00:37.699 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.698+0000 7fb8f2ce0640 1 < completed delete operation key=J8NGDSF9XRTP tag=2W3MF6X2S1U3 type=del 2024-09-13T00:00:37.703 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.702+0000 7fb8f2ce0640 1 > prepared operation key=2TWD9YLMSYJI tag=COWW30833MGM type=write size=8894 2024-09-13T00:00:37.707 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.706+0000 7fb8f2ce0640 1 < completed delete operation key=H5EIP0E9CBWX tag=R3D13JVCUISA type=del 2024-09-13T00:00:37.712 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.710+0000 7fb8f2ce0640 1 > prepared operation key=EJN6HR8I7XQF tag=Y64BUN9O1VAQ type=del size=850023 2024-09-13T00:00:37.716 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.714+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~4H0W40DRF4K9.0 size=1048576 2024-09-13T00:00:37.719 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.718+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~4H0W40DRF4K9.1 size=1048576 2024-09-13T00:00:37.723 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.722+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~4H0W40DRF4K9.2 size=203367 2024-09-13T00:00:37.730 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.730+0000 7fb8f2ce0640 1 < completed multipart upload key=NQJQC8CEAC15 upload=4H0W40DRF4K9 tag=15N8IX13V0S6 type=write size=3349095 2024-09-13T00:00:37.735 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.734+0000 7fb8f2ce0640 1 > prepared operation key=70F7D20HI504 tag=GX05ZXJD2NZE type=del size=1698177 2024-09-13T00:00:37.739 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.738+0000 7fb8f2ce0640 1 < completed part key=_multipart_PVYZQLGYT3XU.2~GDFE21AMU7BM.0 size=1048576 2024-09-13T00:00:37.742 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.742+0000 7fb8f2ce0640 1 < completed part key=_multipart_PVYZQLGYT3XU.2~GDFE21AMU7BM.1 size=49293 2024-09-13T00:00:37.750 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.750+0000 7fb8f2ce0640 1 < completed multipart upload key=PVYZQLGYT3XU upload=GDFE21AMU7BM tag=Z0LPLBGL9T0X type=write size=2146445 2024-09-13T00:00:37.757 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.758+0000 7fb8f2ce0640 1 > prepared part key=_multipart_QMZREZ1QEDNJ.2~JBUKULJ44RHA.0 size=346772 2024-09-13T00:00:37.757 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.758+0000 7fb8f2ce0640 1 > prepared multipart upload key=QMZREZ1QEDNJ upload=JBUKULJ44RHA tag=Y5C844QU6CNZ type=write size=1395348 2024-09-13T00:00:37.762 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.762+0000 7fb8f2ce0640 1 < completed part key=_multipart_70F7D20HI504.2~EMEKGK6WIXYA.0 size=1048576 2024-09-13T00:00:37.765 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.766+0000 7fb8f2ce0640 1 < completed part key=_multipart_70F7D20HI504.2~EMEKGK6WIXYA.1 size=672009 2024-09-13T00:00:37.772 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.770+0000 7fb8f2ce0640 1 < completed multipart upload key=70F7D20HI504 upload=EMEKGK6WIXYA tag=3NHMOESCYT3L type=write size=2769161 2024-09-13T00:00:37.777 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.774+0000 7fb8f2ce0640 1 > prepared operation key=2TWD9YLMSYJI tag=0ZGAH1WJTCSW type=del size=1964212 2024-09-13T00:00:37.782 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.782+0000 7fb8f2ce0640 1 < completed delete operation key=FX0NCGLK0KPJ tag=1M99KCII33SI type=del 2024-09-13T00:00:37.786 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.786+0000 7fb8f2ce0640 1 > prepared operation key=H5EIP0E9CBWX tag=3GJA2UYLETJ4 type=del size=3867850 2024-09-13T00:00:37.790 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.790+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~O9H3P29MT42D.0 size=1048576 2024-09-13T00:00:37.794 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.794+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~O9H3P29MT42D.1 size=1048576 2024-09-13T00:00:37.797 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.798+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~O9H3P29MT42D.2 size=505523 2024-09-13T00:00:37.804 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.802+0000 7fb8f2ce0640 1 < completed multipart upload key=NQJQC8CEAC15 upload=O9H3P29MT42D tag=M3ZKGZF9I1W3 type=write size=3651251 2024-09-13T00:00:37.811 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.810+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~6A2K94KWZTIR.0 size=1048576 2024-09-13T00:00:37.814 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.814+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~6A2K94KWZTIR.1 size=471637 2024-09-13T00:00:37.814 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.814+0000 7fb8f2ce0640 1 > prepared multipart upload key=2AHYRZUDFHP4 upload=6A2K94KWZTIR tag=FF0GFGJCASMR type=write size=2568789 2024-09-13T00:00:37.823 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.822+0000 7fb8f2ce0640 1 < suggested remove operation key=8Z6U7OOA0GBH tag=UOD1IK9IVS14 type=del 2024-09-13T00:00:37.827 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.826+0000 7fb8f2ce0640 1 > prepared operation key=8MPUVD37EDN9 tag=96KLBT0VQSIW type=del size=2634847 2024-09-13T00:00:37.831 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.830+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~HHOO46126XWY.0 size=322873 2024-09-13T00:00:37.838 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.838+0000 7fb8f2ce0640 1 < completed multipart upload key=1PX7DBO9M3YI upload=HHOO46126XWY tag=8LH4D1OKQ243 type=write size=1371449 2024-09-13T00:00:37.845 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.846+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~S5K2ZCYB1LIS.0 size=1048576 2024-09-13T00:00:37.849 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.846+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~S5K2ZCYB1LIS.1 size=1048576 2024-09-13T00:00:37.852 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.850+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~S5K2ZCYB1LIS.2 size=484113 2024-09-13T00:00:37.852 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.850+0000 7fb8f2ce0640 1 > prepared multipart upload key=1PX7DBO9M3YI upload=S5K2ZCYB1LIS tag=3Y9TTTO16SD3 type=write size=3629841 2024-09-13T00:00:37.856 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.854+0000 7fb8f2ce0640 1 < completed delete operation key=FX0NCGLK0KPJ tag=F84A255ZAKSZ type=del 2024-09-13T00:00:37.861 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.858+0000 7fb8f2ce0640 1 > prepared operation key=4K76U69GZ9AE tag=K8IXKYH56FYZ type=del size=2419325 2024-09-13T00:00:37.866 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.866+0000 7fb8f2ce0640 1 < completed delete operation key=ZFHN2SNTZ5W7 tag=1XRHJKE7A23U type=del 2024-09-13T00:00:37.874 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.874+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~NPAXPY4ROBRY.0 size=1048576 2024-09-13T00:00:37.877 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.878+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~NPAXPY4ROBRY.1 size=657399 2024-09-13T00:00:37.877 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.878+0000 7fb8f2ce0640 1 > prepared multipart upload key=PVYZQLGYT3XU upload=NPAXPY4ROBRY tag=HU0SNARCLQYH type=write size=2754551 2024-09-13T00:00:37.885 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.886+0000 7fb8f2ce0640 1 < suggested remove operation key=5GM88ERM89KT tag=0NSI7TCGVLPO type=del 2024-09-13T00:00:37.889 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.890+0000 7fb8f2ce0640 1 > prepared operation key=F0QCVR9E08WE tag=4AGDZ5B1RSM3 type=del size=2872755 2024-09-13T00:00:37.894 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.894+0000 7fb8f2ce0640 1 < completed delete operation key=ZFHN2SNTZ5W7 tag=KCVAS6KYG5D1 type=del 2024-09-13T00:00:37.899 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.898+0000 7fb8f2ce0640 1 > prepared operation key=8Z6U7OOA0GBH tag=C8X0ZWLB12RV type=del size=243735 2024-09-13T00:00:37.903 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.902+0000 7fb8f2ce0640 1 < completed part key=_multipart_44EDYFT9VF8Y.2~MPEXHIOUJXBE.0 size=957715 2024-09-13T00:00:37.910 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.910+0000 7fb8f2ce0640 1 < completed multipart upload key=44EDYFT9VF8Y upload=MPEXHIOUJXBE tag=EEUVOOET0QCE type=write size=2006291 2024-09-13T00:00:37.918 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.918+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~834XVMWTDA6L.0 size=1035093 2024-09-13T00:00:37.918 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.918+0000 7fb8f2ce0640 1 > prepared multipart upload key=2AHYRZUDFHP4 upload=834XVMWTDA6L tag=VTAZ7JD28N1P type=write size=2083669 2024-09-13T00:00:37.926 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.926+0000 7fb8f2ce0640 1 < suggested update operation key=70F7D20HI504 tag=GS9O32745FXW type=write size=588988 2024-09-13T00:00:37.929 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.930+0000 7fb8f2ce0640 1 > prepared operation key=QMZREZ1QEDNJ tag=4SLDJ6GDOPKA type=del size=2702161 2024-09-13T00:00:37.934 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.934+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~HA45BS2PH8ZC.0 size=1048576 2024-09-13T00:00:37.938 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.938+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~HA45BS2PH8ZC.1 size=699193 2024-09-13T00:00:37.945 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.942+0000 7fb8f2ce0640 1 < completed multipart upload key=NQJQC8CEAC15 upload=HA45BS2PH8ZC tag=EJ2EH964S8IJ type=write size=2796345 2024-09-13T00:00:37.950 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.950+0000 7fb8f2ce0640 1 > prepared operation key=ZFHN2SNTZ5W7 tag=E1KPKQPWLX7D type=write size=101439 2024-09-13T00:00:37.954 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.954+0000 7fb8f2ce0640 1 < completed write operation key=2TWD9YLMSYJI tag=COWW30833MGM type=write size=8894 2024-09-13T00:00:37.959 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.958+0000 7fb8f2ce0640 1 > prepared operation key=PVYZQLGYT3XU tag=TJHQ2SJDXA1K type=del size=2534077 2024-09-13T00:00:37.963 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.962+0000 7fb8f2ce0640 1 < completed delete operation key=EJN6HR8I7XQF tag=Y64BUN9O1VAQ type=del 2024-09-13T00:00:37.967 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.966+0000 7fb8f2ce0640 1 > prepared operation key=H4M1TCN5W6XU tag=KS60K819J3W1 type=del size=3762797 2024-09-13T00:00:37.972 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.970+0000 7fb8f2ce0640 -1 < failed to complete operation key=70F7D20HI504 tag=GX05ZXJD2NZE type=del size=1698177: (22) Invalid argument 2024-09-13T00:00:37.976 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.974+0000 7fb8f2ce0640 1 > prepared operation key=H4M1TCN5W6XU tag=5NWGV3YH6JTK type=del size=2083399 2024-09-13T00:00:37.977 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.978+0000 7fb8f2ce0640 1 < canceled part key=_multipart_QMZREZ1QEDNJ.2~JBUKULJ44RHA.0 size=346772 2024-09-13T00:00:37.984 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.982+0000 7fb8f2ce0640 1 < canceled multipart upload key=QMZREZ1QEDNJ upload=JBUKULJ44RHA tag=Y5C844QU6CNZ type=write size=1395348 2024-09-13T00:00:37.993 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.994+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~EUXBZGPU3R9U.0 size=1040122 2024-09-13T00:00:37.993 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:37.994+0000 7fb8f2ce0640 1 > prepared multipart upload key=63FJDWBO2T9S upload=EUXBZGPU3R9U tag=J2ADGCEQR5T1 type=write size=2088698 2024-09-13T00:00:38.004 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.002+0000 7fb8f2ce0640 1 < suggested remove operation key=2TWD9YLMSYJI tag=0ZGAH1WJTCSW type=del 2024-09-13T00:00:38.007 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.006+0000 7fb8f2ce0640 1 > prepared operation key=0WC6P8YNL0H2 tag=345PKKYKCL7O type=del size=2942534 2024-09-13T00:00:38.012 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.010+0000 7fb8f2ce0640 1 < completed delete operation key=H5EIP0E9CBWX tag=3GJA2UYLETJ4 type=del 2024-09-13T00:00:38.020 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.018+0000 7fb8f2ce0640 1 > prepared part key=_multipart_5GM88ERM89KT.2~Q05NDZ54TUV3.0 size=1048576 2024-09-13T00:00:38.024 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.022+0000 7fb8f2ce0640 1 > prepared part key=_multipart_5GM88ERM89KT.2~Q05NDZ54TUV3.1 size=496280 2024-09-13T00:00:38.025 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.022+0000 7fb8f2ce0640 1 > prepared multipart upload key=5GM88ERM89KT upload=Q05NDZ54TUV3 tag=XV77HUMU56EN type=write size=2593432 2024-09-13T00:00:38.029 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.030+0000 7fb8f2ce0640 1 < completed part key=_multipart_2AHYRZUDFHP4.2~6A2K94KWZTIR.0 size=1048576 2024-09-13T00:00:38.032 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.030+0000 7fb8f2ce0640 1 < completed part key=_multipart_2AHYRZUDFHP4.2~6A2K94KWZTIR.1 size=471637 2024-09-13T00:00:38.040 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.038+0000 7fb8f2ce0640 1 < completed multipart upload key=2AHYRZUDFHP4 upload=6A2K94KWZTIR tag=FF0GFGJCASMR type=write size=2568789 2024-09-13T00:00:38.044 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.042+0000 7fb8f2ce0640 1 > prepared operation key=NX26SI4F2ZMB tag=GK4C5S97LMEA type=del size=1270946 2024-09-13T00:00:38.048 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.046+0000 7fb8f2ce0640 1 < completed delete operation key=8MPUVD37EDN9 tag=96KLBT0VQSIW type=del 2024-09-13T00:00:38.052 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.050+0000 7fb8f2ce0640 1 > prepared operation key=ZTK9AYBSLCOZ tag=OLX9H7UD0C6C type=del size=2597480 2024-09-13T00:00:38.053 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.054+0000 7fb8f2ce0640 1 < canceled part key=_multipart_1PX7DBO9M3YI.2~S5K2ZCYB1LIS.0 size=1048576 2024-09-13T00:00:38.057 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.058+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~S5K2ZCYB1LIS.1 size=1048576 2024-09-13T00:00:38.061 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.062+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~S5K2ZCYB1LIS.2 size=484113 2024-09-13T00:00:38.068 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.066+0000 7fb8f2ce0640 1 < canceled multipart upload key=1PX7DBO9M3YI upload=S5K2ZCYB1LIS tag=3Y9TTTO16SD3 type=write size=3629841 2024-09-13T00:00:38.076 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.074+0000 7fb8f2ce0640 1 > prepared part key=_multipart_85YEW8N43FTA.2~3BHXRHSG420R.0 size=1048576 2024-09-13T00:00:38.079 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.078+0000 7fb8f2ce0640 1 > prepared part key=_multipart_85YEW8N43FTA.2~3BHXRHSG420R.1 size=596573 2024-09-13T00:00:38.079 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.078+0000 7fb8f2ce0640 1 > prepared multipart upload key=85YEW8N43FTA upload=3BHXRHSG420R tag=ALOW2J2RME09 type=write size=2693725 2024-09-13T00:00:38.084 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.082+0000 7fb8f2ce0640 1 < completed delete operation key=4K76U69GZ9AE tag=K8IXKYH56FYZ type=del 2024-09-13T00:00:38.088 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.086+0000 7fb8f2ce0640 1 > prepared operation key=QMZREZ1QEDNJ tag=0VBL86UQKYND type=del size=1883010 2024-09-13T00:00:38.093 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.090+0000 7fb8f2ce0640 1 < completed part key=_multipart_PVYZQLGYT3XU.2~NPAXPY4ROBRY.0 size=1048576 2024-09-13T00:00:38.096 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.094+0000 7fb8f2ce0640 1 < completed part key=_multipart_PVYZQLGYT3XU.2~NPAXPY4ROBRY.1 size=657399 2024-09-13T00:00:38.103 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.102+0000 7fb8f2ce0640 1 < completed multipart upload key=PVYZQLGYT3XU upload=NPAXPY4ROBRY tag=HU0SNARCLQYH type=write size=2754551 2024-09-13T00:00:38.107 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.106+0000 7fb8f2ce0640 1 > prepared operation key=2AHYRZUDFHP4 tag=USO4K91WD1R4 type=del size=3402354 2024-09-13T00:00:38.112 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.110+0000 7fb8f2ce0640 1 < completed delete operation key=F0QCVR9E08WE tag=4AGDZ5B1RSM3 type=del 2024-09-13T00:00:38.116 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.114+0000 7fb8f2ce0640 1 > prepared operation key=ZFHN2SNTZ5W7 tag=R7L6PQF25SGI type=del size=1814719 2024-09-13T00:00:38.121 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.122+0000 7fb8f2ce0640 1 < completed delete operation key=8Z6U7OOA0GBH tag=C8X0ZWLB12RV type=del 2024-09-13T00:00:38.125 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.126+0000 7fb8f2ce0640 1 > prepared operation key=2TWD9YLMSYJI tag=0NUG22B93ORQ type=del size=2439241 2024-09-13T00:00:38.130 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.130+0000 7fb8f2ce0640 1 < completed part key=_multipart_2AHYRZUDFHP4.2~834XVMWTDA6L.0 size=1035093 2024-09-13T00:00:38.137 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.134+0000 7fb8f2ce0640 1 < completed multipart upload key=2AHYRZUDFHP4 upload=834XVMWTDA6L tag=VTAZ7JD28N1P type=write size=2083669 2024-09-13T00:00:38.141 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.142+0000 7fb8f2ce0640 1 > prepared operation key=1HX06CCZJ1DD tag=BV3S8TOKO7UL type=del size=3237874 2024-09-13T00:00:38.150 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.150+0000 7fb8f2ce0640 1 < suggested update operation key=QMZREZ1QEDNJ tag=4SLDJ6GDOPKA type=del size=2702161 2024-09-13T00:00:38.158 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.158+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~BDWGCMY2LANU.0 size=579223 2024-09-13T00:00:38.158 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.158+0000 7fb8f2ce0640 1 > prepared multipart upload key=8Z6U7OOA0GBH upload=BDWGCMY2LANU tag=PIXM330N987N type=write size=1627799 2024-09-13T00:00:38.163 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.162+0000 7fb8f2ce0640 1 < canceled operation key=ZFHN2SNTZ5W7 tag=E1KPKQPWLX7D type=write size=101439 2024-09-13T00:00:38.167 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.166+0000 7fb8f2ce0640 1 > prepared operation key=H9AU4HD247YU tag=3H08HYPEKKA6 type=del size=3756266 2024-09-13T00:00:38.172 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.170+0000 7fb8f2ce0640 1 < completed delete operation key=PVYZQLGYT3XU tag=TJHQ2SJDXA1K type=del 2024-09-13T00:00:38.177 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.178+0000 7fb8f2ce0640 1 > prepared operation key=N3Z5XIG3TZRQ tag=RJ8GT7BL0M2B type=del size=2482092 2024-09-13T00:00:38.182 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.182+0000 7fb8f2ce0640 1 < completed delete operation key=H4M1TCN5W6XU tag=KS60K819J3W1 type=del 2024-09-13T00:00:38.190 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.190+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~D964UTJDUZ9R.0 size=1048576 2024-09-13T00:00:38.193 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.194+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~D964UTJDUZ9R.1 size=1048576 2024-09-13T00:00:38.197 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.198+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~D964UTJDUZ9R.2 size=461908 2024-09-13T00:00:38.197 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.198+0000 7fb8f2ce0640 1 > prepared multipart upload key=63FJDWBO2T9S upload=D964UTJDUZ9R tag=1ZERZ0E9F4BU type=write size=3607636 2024-09-13T00:00:38.201 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.202+0000 7fb8f2ce0640 1 < completed delete operation key=H4M1TCN5W6XU tag=5NWGV3YH6JTK type=del 2024-09-13T00:00:38.208 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.206+0000 7fb8f2ce0640 1 > prepared operation key=N3Z5XIG3TZRQ tag=25Q41PSYI0C2 type=write size=897762 2024-09-13T00:00:38.210 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.210+0000 7fb8f2ce0640 1 < completed part key=_multipart_63FJDWBO2T9S.2~EUXBZGPU3R9U.0 size=1040122 2024-09-13T00:00:38.217 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.218+0000 7fb8f2ce0640 1 < completed multipart upload key=63FJDWBO2T9S upload=EUXBZGPU3R9U tag=J2ADGCEQR5T1 type=write size=2088698 2024-09-13T00:00:38.221 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.222+0000 7fb8f2ce0640 1 > prepared operation key=2AHYRZUDFHP4 tag=DVH4NVPYZYSX type=del size=78083 2024-09-13T00:00:38.230 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.230+0000 7fb8f2ce0640 1 < suggested remove operation key=0WC6P8YNL0H2 tag=345PKKYKCL7O type=del 2024-09-13T00:00:38.238 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.238+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EUE3LT0PDIXV.2~UUNLWAOHNJ3A.0 size=125688 2024-09-13T00:00:38.238 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.238+0000 7fb8f2ce0640 1 > prepared multipart upload key=EUE3LT0PDIXV upload=UUNLWAOHNJ3A tag=LKUEPM45CS2V type=write size=1174264 2024-09-13T00:00:38.242 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.242+0000 7fb8f2ce0640 1 < completed part key=_multipart_5GM88ERM89KT.2~Q05NDZ54TUV3.0 size=1048576 2024-09-13T00:00:38.245 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.246+0000 7fb8f2ce0640 1 < completed part key=_multipart_5GM88ERM89KT.2~Q05NDZ54TUV3.1 size=496280 2024-09-13T00:00:38.253 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.250+0000 7fb8f2ce0640 1 < completed multipart upload key=5GM88ERM89KT upload=Q05NDZ54TUV3 tag=XV77HUMU56EN type=write size=2593432 2024-09-13T00:00:38.260 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.258+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2TWD9YLMSYJI.2~FP1L45U01SIJ.0 size=1048576 2024-09-13T00:00:38.264 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.262+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2TWD9YLMSYJI.2~FP1L45U01SIJ.1 size=195446 2024-09-13T00:00:38.264 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.262+0000 7fb8f2ce0640 1 > prepared multipart upload key=2TWD9YLMSYJI upload=FP1L45U01SIJ tag=4TF2WRJ2HCWR type=write size=2292598 2024-09-13T00:00:38.267 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.266+0000 7fb8f2ce0640 1 < completed delete operation key=NX26SI4F2ZMB tag=GK4C5S97LMEA type=del 2024-09-13T00:00:38.275 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.274+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NX26SI4F2ZMB.2~TKEL2XGQLXM0.0 size=1048576 2024-09-13T00:00:38.278 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.278+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NX26SI4F2ZMB.2~TKEL2XGQLXM0.1 size=906010 2024-09-13T00:00:38.278 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.278+0000 7fb8f2ce0640 1 > prepared multipart upload key=NX26SI4F2ZMB upload=TKEL2XGQLXM0 tag=KYWC202Z8Z01 type=write size=3003162 2024-09-13T00:00:38.283 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.282+0000 7fb8f2ce0640 1 < completed delete operation key=ZTK9AYBSLCOZ tag=OLX9H7UD0C6C type=del 2024-09-13T00:00:38.286 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.286+0000 7fb8f2ce0640 1 > prepared operation key=J8NGDSF9XRTP tag=C9ETJY0YLYZ4 type=del size=3793694 2024-09-13T00:00:38.288 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.286+0000 7fb8f2ce0640 1 < canceled part key=_multipart_85YEW8N43FTA.2~3BHXRHSG420R.0 size=1048576 2024-09-13T00:00:38.291 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.290+0000 7fb8f2ce0640 1 < completed part key=_multipart_85YEW8N43FTA.2~3BHXRHSG420R.1 size=596573 2024-09-13T00:00:38.297 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.298+0000 7fb8f2ce0640 1 < canceled multipart upload key=85YEW8N43FTA upload=3BHXRHSG420R tag=ALOW2J2RME09 type=write size=2693725 2024-09-13T00:00:38.302 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.302+0000 7fb8f2ce0640 1 > prepared operation key=85YEW8N43FTA tag=BV13I800M1BK type=del size=2223317 2024-09-13T00:00:38.307 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.306+0000 7fb8f2ce0640 -1 < failed to complete operation key=QMZREZ1QEDNJ tag=0VBL86UQKYND type=del size=1883010: (22) Invalid argument 2024-09-13T00:00:38.316 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.314+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EUE3LT0PDIXV.2~AZCVQ4YG8BS3.0 size=1048576 2024-09-13T00:00:38.319 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.318+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EUE3LT0PDIXV.2~AZCVQ4YG8BS3.1 size=423502 2024-09-13T00:00:38.320 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.318+0000 7fb8f2ce0640 1 > prepared multipart upload key=EUE3LT0PDIXV upload=AZCVQ4YG8BS3 tag=CI243YNCLZKH type=write size=2520654 2024-09-13T00:00:38.324 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.322+0000 7fb8f2ce0640 1 < completed delete operation key=2AHYRZUDFHP4 tag=USO4K91WD1R4 type=del 2024-09-13T00:00:38.328 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.326+0000 7fb8f2ce0640 1 > prepared operation key=NQJQC8CEAC15 tag=LI6F3MQ0UV53 type=del size=2663710 2024-09-13T00:00:38.332 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.330+0000 7fb8f2ce0640 1 < completed delete operation key=ZFHN2SNTZ5W7 tag=R7L6PQF25SGI type=del 2024-09-13T00:00:38.337 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.338+0000 7fb8f2ce0640 1 > prepared operation key=7OX5OJZ8GXU5 tag=87RUHTX56GBQ type=del size=2891381 2024-09-13T00:00:38.341 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.342+0000 7fb8f2ce0640 1 < completed delete operation key=2TWD9YLMSYJI tag=0NUG22B93ORQ type=del 2024-09-13T00:00:38.349 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.350+0000 7fb8f2ce0640 1 > prepared part key=_multipart_N3Z5XIG3TZRQ.2~3LNL7NIRVNRA.0 size=211168 2024-09-13T00:00:38.349 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.350+0000 7fb8f2ce0640 1 > prepared multipart upload key=N3Z5XIG3TZRQ upload=3LNL7NIRVNRA tag=AJQI6ZPI4O6M type=write size=1259744 2024-09-13T00:00:38.353 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.354+0000 7fb8f2ce0640 1 < canceled operation key=1HX06CCZJ1DD tag=BV3S8TOKO7UL type=del size=3237874 2024-09-13T00:00:38.361 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.362+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~BMOK65W24YGR.0 size=50572 2024-09-13T00:00:38.362 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.362+0000 7fb8f2ce0640 1 > prepared multipart upload key=PVYZQLGYT3XU upload=BMOK65W24YGR tag=TY7Y661P5E0J type=write size=1099148 2024-09-13T00:00:38.366 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.366+0000 7fb8f2ce0640 1 < completed part key=_multipart_8Z6U7OOA0GBH.2~BDWGCMY2LANU.0 size=579223 2024-09-13T00:00:38.373 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.374+0000 7fb8f2ce0640 1 < completed multipart upload key=8Z6U7OOA0GBH upload=BDWGCMY2LANU tag=PIXM330N987N type=write size=1627799 2024-09-13T00:00:38.383 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.382+0000 7fb8f2ce0640 1 > prepared part key=_multipart_5GM88ERM89KT.2~UTF71HE4QSLZ.0 size=1048576 2024-09-13T00:00:38.386 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.386+0000 7fb8f2ce0640 1 > prepared part key=_multipart_5GM88ERM89KT.2~UTF71HE4QSLZ.1 size=1048576 2024-09-13T00:00:38.390 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.390+0000 7fb8f2ce0640 1 > prepared part key=_multipart_5GM88ERM89KT.2~UTF71HE4QSLZ.2 size=576842 2024-09-13T00:00:38.390 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.390+0000 7fb8f2ce0640 1 > prepared multipart upload key=5GM88ERM89KT upload=UTF71HE4QSLZ tag=37SM4YO90ERG type=write size=3722570 2024-09-13T00:00:38.395 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.394+0000 7fb8f2ce0640 1 < completed delete operation key=H9AU4HD247YU tag=3H08HYPEKKA6 type=del 2024-09-13T00:00:38.400 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.398+0000 7fb8f2ce0640 1 > prepared operation key=EUE3LT0PDIXV tag=9L4JELRN5T4R type=write size=319152 2024-09-13T00:00:38.404 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.402+0000 7fb8f2ce0640 1 < completed delete operation key=N3Z5XIG3TZRQ tag=RJ8GT7BL0M2B type=del 2024-09-13T00:00:38.408 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.406+0000 7fb8f2ce0640 1 > prepared operation key=O4MZ4UWNXME9 tag=NUGAIW0AK0PU type=del size=2883949 2024-09-13T00:00:38.416 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.414+0000 7fb8f2ce0640 1 < completed part key=_multipart_63FJDWBO2T9S.2~D964UTJDUZ9R.0 size=1048576 2024-09-13T00:00:38.417 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.418+0000 7fb8f2ce0640 1 < completed part key=_multipart_63FJDWBO2T9S.2~D964UTJDUZ9R.1 size=1048576 2024-09-13T00:00:38.420 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.418+0000 7fb8f2ce0640 1 < completed part key=_multipart_63FJDWBO2T9S.2~D964UTJDUZ9R.2 size=461908 2024-09-13T00:00:38.427 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.426+0000 7fb8f2ce0640 1 < completed multipart upload key=63FJDWBO2T9S upload=D964UTJDUZ9R tag=1ZERZ0E9F4BU type=write size=3607636 2024-09-13T00:00:38.432 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.430+0000 7fb8f2ce0640 1 > prepared operation key=NX26SI4F2ZMB tag=M32R0TDOE6S7 type=del size=2653013 2024-09-13T00:00:38.441 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.442+0000 7fb8f2ce0640 1 < suggested update operation key=N3Z5XIG3TZRQ tag=25Q41PSYI0C2 type=write size=897762 2024-09-13T00:00:38.445 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.446+0000 7fb8f2ce0640 1 > prepared operation key=0WC6P8YNL0H2 tag=Z9EPKE1NR2BJ type=del size=409038 2024-09-13T00:00:38.450 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.450+0000 7fb8f2ce0640 1 < completed delete operation key=2AHYRZUDFHP4 tag=DVH4NVPYZYSX type=del 2024-09-13T00:00:38.458 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.458+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~JHM0UEAW3GI4.0 size=1048576 2024-09-13T00:00:38.461 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.462+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~JHM0UEAW3GI4.1 size=1048576 2024-09-13T00:00:38.466 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.466+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~JHM0UEAW3GI4.2 size=755760 2024-09-13T00:00:38.466 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.466+0000 7fb8f2ce0640 1 > prepared multipart upload key=8MPUVD37EDN9 upload=JHM0UEAW3GI4 tag=QQ7K229AIUWH type=write size=3901488 2024-09-13T00:00:38.471 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.470+0000 7fb8f2ce0640 1 < completed part key=_multipart_EUE3LT0PDIXV.2~UUNLWAOHNJ3A.0 size=125688 2024-09-13T00:00:38.480 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.478+0000 7fb8f2ce0640 1 < completed multipart upload key=EUE3LT0PDIXV upload=UUNLWAOHNJ3A tag=LKUEPM45CS2V type=write size=1174264 2024-09-13T00:00:38.486 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.486+0000 7fb8f2ce0640 1 > prepared operation key=4K76U69GZ9AE tag=SH12JVF9JSFY type=del size=990926 2024-09-13T00:00:38.490 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.490+0000 7fb8f2ce0640 1 < completed part key=_multipart_2TWD9YLMSYJI.2~FP1L45U01SIJ.0 size=1048576 2024-09-13T00:00:38.493 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.494+0000 7fb8f2ce0640 1 < completed part key=_multipart_2TWD9YLMSYJI.2~FP1L45U01SIJ.1 size=195446 2024-09-13T00:00:38.501 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.498+0000 7fb8f2ce0640 1 < completed multipart upload key=2TWD9YLMSYJI upload=FP1L45U01SIJ tag=4TF2WRJ2HCWR type=write size=2292598 2024-09-13T00:00:38.508 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.506+0000 7fb8f2ce0640 1 > prepared part key=_multipart_O4MZ4UWNXME9.2~AMSE9HSF2K5J.0 size=948040 2024-09-13T00:00:38.508 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.506+0000 7fb8f2ce0640 1 > prepared multipart upload key=O4MZ4UWNXME9 upload=AMSE9HSF2K5J tag=NM3J580FJ2OC type=write size=1996616 2024-09-13T00:00:38.509 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.510+0000 7fb8f2ce0640 1 < canceled part key=_multipart_NX26SI4F2ZMB.2~TKEL2XGQLXM0.0 size=1048576 2024-09-13T00:00:38.512 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.510+0000 7fb8f2ce0640 1 < completed part key=_multipart_NX26SI4F2ZMB.2~TKEL2XGQLXM0.1 size=906010 2024-09-13T00:00:38.520 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.518+0000 7fb8f2ce0640 1 < canceled multipart upload key=NX26SI4F2ZMB upload=TKEL2XGQLXM0 tag=KYWC202Z8Z01 type=write size=3003162 2024-09-13T00:00:38.525 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.526+0000 7fb8f2ce0640 1 > prepared operation key=0WC6P8YNL0H2 tag=CZMZRTDEZLPU type=write size=746174 2024-09-13T00:00:38.534 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.534+0000 7fb8f2ce0640 1 < suggested update operation key=J8NGDSF9XRTP tag=C9ETJY0YLYZ4 type=del size=3793694 2024-09-13T00:00:38.538 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.538+0000 7fb8f2ce0640 1 > prepared operation key=8MPUVD37EDN9 tag=Q8G90I3U5VOB type=del size=1350090 2024-09-13T00:00:38.542 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.542+0000 7fb8f2ce0640 1 < completed delete operation key=85YEW8N43FTA tag=BV13I800M1BK type=del 2024-09-13T00:00:38.546 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.546+0000 7fb8f2ce0640 1 > prepared operation key=N3Z5XIG3TZRQ tag=LCY54ZRQ4FJR type=write size=657084 2024-09-13T00:00:38.551 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.550+0000 7fb8f2ce0640 1 < completed part key=_multipart_EUE3LT0PDIXV.2~AZCVQ4YG8BS3.0 size=1048576 2024-09-13T00:00:38.555 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.554+0000 7fb8f2ce0640 1 < completed part key=_multipart_EUE3LT0PDIXV.2~AZCVQ4YG8BS3.1 size=423502 2024-09-13T00:00:38.563 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.562+0000 7fb8f2ce0640 1 < completed multipart upload key=EUE3LT0PDIXV upload=AZCVQ4YG8BS3 tag=CI243YNCLZKH type=write size=2520654 2024-09-13T00:00:38.568 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.566+0000 7fb8f2ce0640 1 > prepared operation key=J8NGDSF9XRTP tag=ABX8MZEWYFCM type=del size=3927055 2024-09-13T00:00:38.572 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.570+0000 7fb8f2ce0640 1 < completed delete operation key=NQJQC8CEAC15 tag=LI6F3MQ0UV53 type=del 2024-09-13T00:00:38.576 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.574+0000 7fb8f2ce0640 1 > prepared operation key=1HX06CCZJ1DD tag=54403J2A0UI2 type=del size=3820030 2024-09-13T00:00:38.580 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.578+0000 7fb8f2ce0640 1 < completed delete operation key=7OX5OJZ8GXU5 tag=87RUHTX56GBQ type=del 2024-09-13T00:00:38.584 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.582+0000 7fb8f2ce0640 1 > prepared operation key=O4MZ4UWNXME9 tag=4WUECJURVRZI type=del size=2555380 2024-09-13T00:00:38.588 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.586+0000 7fb8f2ce0640 1 < completed part key=_multipart_N3Z5XIG3TZRQ.2~3LNL7NIRVNRA.0 size=211168 2024-09-13T00:00:38.595 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.594+0000 7fb8f2ce0640 1 < completed multipart upload key=N3Z5XIG3TZRQ upload=3LNL7NIRVNRA tag=AJQI6ZPI4O6M type=write size=1259744 2024-09-13T00:00:38.602 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.602+0000 7fb8f2ce0640 1 > prepared part key=_multipart_J8NGDSF9XRTP.2~IB20FHDH7V4W.0 size=1048576 2024-09-13T00:00:38.606 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.606+0000 7fb8f2ce0640 1 > prepared part key=_multipart_J8NGDSF9XRTP.2~IB20FHDH7V4W.1 size=1024860 2024-09-13T00:00:38.606 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.606+0000 7fb8f2ce0640 1 > prepared multipart upload key=J8NGDSF9XRTP upload=IB20FHDH7V4W tag=RH267QJG22BB type=write size=3122012 2024-09-13T00:00:38.611 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.610+0000 7fb8f2ce0640 1 < completed part key=_multipart_PVYZQLGYT3XU.2~BMOK65W24YGR.0 size=50572 2024-09-13T00:00:38.619 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.618+0000 7fb8f2ce0640 1 < completed multipart upload key=PVYZQLGYT3XU upload=BMOK65W24YGR tag=TY7Y661P5E0J type=write size=1099148 2024-09-13T00:00:38.623 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.622+0000 7fb8f2ce0640 1 > prepared operation key=85YEW8N43FTA tag=W2JEO0DODTZX type=del size=1019989 2024-09-13T00:00:38.628 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.626+0000 7fb8f2ce0640 1 < completed part key=_multipart_5GM88ERM89KT.2~UTF71HE4QSLZ.0 size=1048576 2024-09-13T00:00:38.631 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.630+0000 7fb8f2ce0640 1 < completed part key=_multipart_5GM88ERM89KT.2~UTF71HE4QSLZ.1 size=1048576 2024-09-13T00:00:38.635 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.634+0000 7fb8f2ce0640 1 < completed part key=_multipart_5GM88ERM89KT.2~UTF71HE4QSLZ.2 size=576842 2024-09-13T00:00:38.642 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.642+0000 7fb8f2ce0640 1 < completed multipart upload key=5GM88ERM89KT upload=UTF71HE4QSLZ tag=37SM4YO90ERG type=write size=3722570 2024-09-13T00:00:38.646 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.646+0000 7fb8f2ce0640 1 > prepared operation key=0WC6P8YNL0H2 tag=C6NIX91PUXCO type=del size=284540 2024-09-13T00:00:38.650 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.650+0000 7fb8f2ce0640 1 < completed write operation key=EUE3LT0PDIXV tag=9L4JELRN5T4R type=write size=319152 2024-09-13T00:00:38.659 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.658+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~PJTFF3CSNSUH.0 size=1048576 2024-09-13T00:00:38.663 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.662+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~PJTFF3CSNSUH.1 size=1048576 2024-09-13T00:00:38.666 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.666+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~PJTFF3CSNSUH.2 size=643909 2024-09-13T00:00:38.666 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.666+0000 7fb8f2ce0640 1 > prepared multipart upload key=PVYZQLGYT3XU upload=PJTFF3CSNSUH tag=IS5LR0E95HT9 type=write size=3789637 2024-09-13T00:00:38.671 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.670+0000 7fb8f2ce0640 1 < completed delete operation key=O4MZ4UWNXME9 tag=NUGAIW0AK0PU type=del 2024-09-13T00:00:38.678 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.678+0000 7fb8f2ce0640 1 > prepared part key=_multipart_P8OYS5ZE1NX0.2~KNU8P9R8318Z.0 size=1048576 2024-09-13T00:00:38.681 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.682+0000 7fb8f2ce0640 1 > prepared part key=_multipart_P8OYS5ZE1NX0.2~KNU8P9R8318Z.1 size=1048576 2024-09-13T00:00:38.685 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.686+0000 7fb8f2ce0640 1 > prepared part key=_multipart_P8OYS5ZE1NX0.2~KNU8P9R8318Z.2 size=501242 2024-09-13T00:00:38.685 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.686+0000 7fb8f2ce0640 1 > prepared multipart upload key=P8OYS5ZE1NX0 upload=KNU8P9R8318Z tag=MSMGPIZH3QDQ type=write size=3646970 2024-09-13T00:00:38.693 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.694+0000 7fb8f2ce0640 1 < suggested remove operation key=NX26SI4F2ZMB tag=M32R0TDOE6S7 type=del 2024-09-13T00:00:38.697 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.698+0000 7fb8f2ce0640 1 > prepared operation key=70F7D20HI504 tag=ZKIDX8HCIPQA type=del size=2118380 2024-09-13T00:00:38.703 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.702+0000 7fb8f2ce0640 1 < completed delete operation key=0WC6P8YNL0H2 tag=Z9EPKE1NR2BJ type=del 2024-09-13T00:00:38.708 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.706+0000 7fb8f2ce0640 1 > prepared operation key=NQJQC8CEAC15 tag=MCUBNV4LMBFM type=del size=367880 2024-09-13T00:00:38.713 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.714+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~JHM0UEAW3GI4.0 size=1048576 2024-09-13T00:00:38.717 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.718+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~JHM0UEAW3GI4.1 size=1048576 2024-09-13T00:00:38.720 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.718+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~JHM0UEAW3GI4.2 size=755760 2024-09-13T00:00:38.726 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.726+0000 7fb8f2ce0640 1 < completed multipart upload key=8MPUVD37EDN9 upload=JHM0UEAW3GI4 tag=QQ7K229AIUWH type=write size=3901488 2024-09-13T00:00:38.730 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.730+0000 7fb8f2ce0640 1 > prepared operation key=QMZREZ1QEDNJ tag=EGNENPVFFDHP type=del size=2291882 2024-09-13T00:00:38.734 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.734+0000 7fb8f2ce0640 1 < completed delete operation key=4K76U69GZ9AE tag=SH12JVF9JSFY type=del 2024-09-13T00:00:38.741 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.742+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~J78250H7SCIO.0 size=1048576 2024-09-13T00:00:38.745 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.742+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~J78250H7SCIO.1 size=394337 2024-09-13T00:00:38.745 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.742+0000 7fb8f2ce0640 1 > prepared multipart upload key=8Z6U7OOA0GBH upload=J78250H7SCIO tag=1Q2GJ7T0EE58 type=write size=2491489 2024-09-13T00:00:38.749 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.750+0000 7fb8f2ce0640 1 < completed part key=_multipart_O4MZ4UWNXME9.2~AMSE9HSF2K5J.0 size=948040 2024-09-13T00:00:38.755 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.754+0000 7fb8f2ce0640 1 < completed multipart upload key=O4MZ4UWNXME9 upload=AMSE9HSF2K5J tag=NM3J580FJ2OC type=write size=1996616 2024-09-13T00:00:38.763 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.762+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~BI7JTQM0NMHN.0 size=1048576 2024-09-13T00:00:38.766 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.766+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~BI7JTQM0NMHN.1 size=214419 2024-09-13T00:00:38.766 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.766+0000 7fb8f2ce0640 1 > prepared multipart upload key=8Z6U7OOA0GBH upload=BI7JTQM0NMHN tag=I1D6HVW1Y51T type=write size=2311571 2024-09-13T00:00:38.770 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.770+0000 7fb8f2ce0640 1 < completed write operation key=0WC6P8YNL0H2 tag=CZMZRTDEZLPU type=write size=746174 2024-09-13T00:00:38.773 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.774+0000 7fb8f2ce0640 1 > prepared operation key=1PX7DBO9M3YI tag=JBXGXKHW5YGF type=del size=348582 2024-09-13T00:00:38.778 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.778+0000 7fb8f2ce0640 1 < canceled operation key=8MPUVD37EDN9 tag=Q8G90I3U5VOB type=del size=1350090 2024-09-13T00:00:38.785 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.786+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~PWVGO1MMBBL6.0 size=1048576 2024-09-13T00:00:38.791 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.790+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~PWVGO1MMBBL6.1 size=1048576 2024-09-13T00:00:38.794 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.794+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~PWVGO1MMBBL6.2 size=101171 2024-09-13T00:00:38.794 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.794+0000 7fb8f2ce0640 1 > prepared multipart upload key=FX0NCGLK0KPJ upload=PWVGO1MMBBL6 tag=OEVNUXW83F5B type=write size=3246899 2024-09-13T00:00:38.801 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.802+0000 7fb8f2ce0640 1 < suggested remove operation key=N3Z5XIG3TZRQ tag=LCY54ZRQ4FJR type=write 2024-09-13T00:00:38.805 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.806+0000 7fb8f2ce0640 1 > prepared operation key=PVYZQLGYT3XU tag=1UJ4DA64E71Q type=write size=623837 2024-09-13T00:00:38.808 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.806+0000 7fb8f2ce0640 1 < completed delete operation key=J8NGDSF9XRTP tag=ABX8MZEWYFCM type=del 2024-09-13T00:00:38.814 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.814+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~D9UFII9W35FJ.0 size=1048576 2024-09-13T00:00:38.817 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.818+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~D9UFII9W35FJ.1 size=1048576 2024-09-13T00:00:38.820 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.818+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~D9UFII9W35FJ.2 size=373196 2024-09-13T00:00:38.820 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.818+0000 7fb8f2ce0640 1 > prepared multipart upload key=70F7D20HI504 upload=D9UFII9W35FJ tag=EET1QGGX46BN type=write size=3518924 2024-09-13T00:00:38.825 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.826+0000 7fb8f2ce0640 1 < suggested update operation key=1HX06CCZJ1DD tag=54403J2A0UI2 type=del size=3820030 2024-09-13T00:00:38.828 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.826+0000 7fb8f2ce0640 1 > prepared operation key=ZTK9AYBSLCOZ tag=MVAUSS9YBXCP type=del size=4103716 2024-09-13T00:00:38.830 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.830+0000 7fb8f2ce0640 1 < completed delete operation key=O4MZ4UWNXME9 tag=4WUECJURVRZI type=del 2024-09-13T00:00:38.833 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.834+0000 7fb8f2ce0640 1 > prepared operation key=70F7D20HI504 tag=B4IE7O1R0G88 type=write size=633193 2024-09-13T00:00:38.836 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.834+0000 7fb8f2ce0640 1 < completed part key=_multipart_J8NGDSF9XRTP.2~IB20FHDH7V4W.0 size=1048576 2024-09-13T00:00:38.838 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.838+0000 7fb8f2ce0640 1 < completed part key=_multipart_J8NGDSF9XRTP.2~IB20FHDH7V4W.1 size=1024860 2024-09-13T00:00:38.842 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.842+0000 7fb8f2ce0640 1 < completed multipart upload key=J8NGDSF9XRTP upload=IB20FHDH7V4W tag=RH267QJG22BB type=write size=3122012 2024-09-13T00:00:38.847 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.846+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~T2WPKU5F5HLO.0 size=532614 2024-09-13T00:00:38.848 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.846+0000 7fb8f2ce0640 1 > prepared multipart upload key=2AHYRZUDFHP4 upload=T2WPKU5F5HLO tag=K0TULNTKOOWX type=write size=1581190 2024-09-13T00:00:38.850 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.850+0000 7fb8f2ce0640 1 < completed delete operation key=85YEW8N43FTA tag=W2JEO0DODTZX type=del 2024-09-13T00:00:38.853 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.854+0000 7fb8f2ce0640 1 > prepared operation key=1PX7DBO9M3YI tag=XOTZC635YY8A type=write size=196946 2024-09-13T00:00:38.856 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.854+0000 7fb8f2ce0640 1 < completed delete operation key=0WC6P8YNL0H2 tag=C6NIX91PUXCO type=del 2024-09-13T00:00:38.858 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.858+0000 7fb8f2ce0640 1 > prepared operation key=P8OYS5ZE1NX0 tag=ZXJ45A3D62FB type=write size=863794 2024-09-13T00:00:38.862 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.862+0000 7fb8f2ce0640 1 < completed part key=_multipart_PVYZQLGYT3XU.2~PJTFF3CSNSUH.0 size=1048576 2024-09-13T00:00:38.865 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.866+0000 7fb8f2ce0640 1 < completed part key=_multipart_PVYZQLGYT3XU.2~PJTFF3CSNSUH.1 size=1048576 2024-09-13T00:00:38.869 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.866+0000 7fb8f2ce0640 1 < completed part key=_multipart_PVYZQLGYT3XU.2~PJTFF3CSNSUH.2 size=643909 2024-09-13T00:00:38.875 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.874+0000 7fb8f2ce0640 1 < completed multipart upload key=PVYZQLGYT3XU upload=PJTFF3CSNSUH tag=IS5LR0E95HT9 type=write size=3789637 2024-09-13T00:00:38.881 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.882+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~9Q5RV7J9GCV4.0 size=81431 2024-09-13T00:00:38.881 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.882+0000 7fb8f2ce0640 1 > prepared multipart upload key=7OX5OJZ8GXU5 upload=9Q5RV7J9GCV4 tag=P2Q3JNZYYKKK type=write size=1130007 2024-09-13T00:00:38.884 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.882+0000 7fb8f2ce0640 1 < completed part key=_multipart_P8OYS5ZE1NX0.2~KNU8P9R8318Z.0 size=1048576 2024-09-13T00:00:38.887 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.886+0000 7fb8f2ce0640 1 < completed part key=_multipart_P8OYS5ZE1NX0.2~KNU8P9R8318Z.1 size=1048576 2024-09-13T00:00:38.890 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.890+0000 7fb8f2ce0640 1 < completed part key=_multipart_P8OYS5ZE1NX0.2~KNU8P9R8318Z.2 size=501242 2024-09-13T00:00:38.896 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.894+0000 7fb8f2ce0640 1 < completed multipart upload key=P8OYS5ZE1NX0 upload=KNU8P9R8318Z tag=MSMGPIZH3QDQ type=write size=3646970 2024-09-13T00:00:38.901 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.898+0000 7fb8f2ce0640 1 > prepared operation key=8MPUVD37EDN9 tag=P6QM842HMAJ6 type=del size=1434128 2024-09-13T00:00:38.904 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.902+0000 7fb8f2ce0640 1 < completed delete operation key=70F7D20HI504 tag=ZKIDX8HCIPQA type=del 2024-09-13T00:00:38.909 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.906+0000 7fb8f2ce0640 1 > prepared operation key=QMZREZ1QEDNJ tag=ZQ6C2ATUFP0M type=write size=24540 2024-09-13T00:00:38.916 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.914+0000 7fb8f2ce0640 1 < suggested update operation key=NQJQC8CEAC15 tag=MCUBNV4LMBFM type=del size=367880 2024-09-13T00:00:38.922 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.922+0000 7fb8f2ce0640 1 > prepared part key=_multipart_85YEW8N43FTA.2~RB73SBTBUAAR.0 size=492916 2024-09-13T00:00:38.922 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.922+0000 7fb8f2ce0640 1 > prepared multipart upload key=85YEW8N43FTA upload=RB73SBTBUAAR tag=XWYAMN0PHK71 type=write size=1541492 2024-09-13T00:00:38.926 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.926+0000 7fb8f2ce0640 1 < completed delete operation key=QMZREZ1QEDNJ tag=EGNENPVFFDHP type=del 2024-09-13T00:00:38.934 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.934+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~H38GC4S3NB3U.0 size=178259 2024-09-13T00:00:38.934 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.934+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZTK9AYBSLCOZ upload=H38GC4S3NB3U tag=K09GXQY9LH6C type=write size=1226835 2024-09-13T00:00:38.938 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.938+0000 7fb8f2ce0640 1 < completed part key=_multipart_8Z6U7OOA0GBH.2~J78250H7SCIO.0 size=1048576 2024-09-13T00:00:38.942 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.942+0000 7fb8f2ce0640 1 < completed part key=_multipart_8Z6U7OOA0GBH.2~J78250H7SCIO.1 size=394337 2024-09-13T00:00:38.949 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.950+0000 7fb8f2ce0640 1 < completed multipart upload key=8Z6U7OOA0GBH upload=J78250H7SCIO tag=1Q2GJ7T0EE58 type=write size=2491489 2024-09-13T00:00:38.953 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.954+0000 7fb8f2ce0640 1 > prepared operation key=1HX06CCZJ1DD tag=9DQ9TKJV9Y5S type=del size=651501 2024-09-13T00:00:38.958 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.958+0000 7fb8f2ce0640 1 < completed part key=_multipart_8Z6U7OOA0GBH.2~BI7JTQM0NMHN.0 size=1048576 2024-09-13T00:00:38.962 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.962+0000 7fb8f2ce0640 1 < completed part key=_multipart_8Z6U7OOA0GBH.2~BI7JTQM0NMHN.1 size=214419 2024-09-13T00:00:38.969 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.970+0000 7fb8f2ce0640 1 < completed multipart upload key=8Z6U7OOA0GBH upload=BI7JTQM0NMHN tag=I1D6HVW1Y51T type=write size=2311571 2024-09-13T00:00:38.973 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.974+0000 7fb8f2ce0640 1 > prepared operation key=2TWD9YLMSYJI tag=NLUG9HSE32AO type=del size=1285626 2024-09-13T00:00:38.977 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.978+0000 7fb8f2ce0640 1 < completed delete operation key=1PX7DBO9M3YI tag=JBXGXKHW5YGF type=del 2024-09-13T00:00:38.982 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.982+0000 7fb8f2ce0640 1 > prepared operation key=8Z6U7OOA0GBH tag=82F2BBFPNRK0 type=del size=2095370 2024-09-13T00:00:38.986 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.986+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~PWVGO1MMBBL6.0 size=1048576 2024-09-13T00:00:38.990 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.990+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~PWVGO1MMBBL6.1 size=1048576 2024-09-13T00:00:38.994 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:38.994+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~PWVGO1MMBBL6.2 size=101171 2024-09-13T00:00:39.002 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.002+0000 7fb8f2ce0640 1 < completed multipart upload key=FX0NCGLK0KPJ upload=PWVGO1MMBBL6 tag=OEVNUXW83F5B type=write size=3246899 2024-09-13T00:00:39.006 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.006+0000 7fb8f2ce0640 1 > prepared operation key=EUE3LT0PDIXV tag=LXN9763CJP9K type=del size=1373381 2024-09-13T00:00:39.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:38 smithi136 bash[75164]: cluster 2024-09-13T00:00:37.127779+0000 mgr.y (mgr.24661) 705 : cluster [DBG] pgmap v1381: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 499 KiB/s rd, 172 KiB/s wr, 883 op/s 2024-09-13T00:00:39.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:38 smithi136 bash[76371]: cluster 2024-09-13T00:00:37.127779+0000 mgr.y (mgr.24661) 705 : cluster [DBG] pgmap v1381: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 499 KiB/s rd, 172 KiB/s wr, 883 op/s 2024-09-13T00:00:39.011 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.010+0000 7fb8f2ce0640 1 < completed write operation key=PVYZQLGYT3XU tag=1UJ4DA64E71Q type=write size=623837 2024-09-13T00:00:39.019 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.018+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~WSP4TNP8H3WN.0 size=674713 2024-09-13T00:00:39.019 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.018+0000 7fb8f2ce0640 1 > prepared multipart upload key=1HX06CCZJ1DD upload=WSP4TNP8H3WN tag=F8KA5FC72OKR type=write size=1723289 2024-09-13T00:00:39.024 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.022+0000 7fb8f2ce0640 1 < completed part key=_multipart_70F7D20HI504.2~D9UFII9W35FJ.0 size=1048576 2024-09-13T00:00:39.029 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.030+0000 7fb8f2ce0640 1 < completed part key=_multipart_70F7D20HI504.2~D9UFII9W35FJ.1 size=1048576 2024-09-13T00:00:39.033 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.034+0000 7fb8f2ce0640 1 < completed part key=_multipart_70F7D20HI504.2~D9UFII9W35FJ.2 size=373196 2024-09-13T00:00:39.041 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.042+0000 7fb8f2ce0640 1 < completed multipart upload key=70F7D20HI504 upload=D9UFII9W35FJ tag=EET1QGGX46BN type=write size=3518924 2024-09-13T00:00:39.050 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.050+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H4M1TCN5W6XU.2~5VRVOUINMGM1.0 size=864625 2024-09-13T00:00:39.050 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.050+0000 7fb8f2ce0640 1 > prepared multipart upload key=H4M1TCN5W6XU upload=5VRVOUINMGM1 tag=E0K87C2MC1M1 type=write size=1913201 2024-09-13T00:00:39.055 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.054+0000 7fb8f2ce0640 1 < completed delete operation key=ZTK9AYBSLCOZ tag=MVAUSS9YBXCP type=del 2024-09-13T00:00:39.059 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.058+0000 7fb8f2ce0640 1 > prepared operation key=W8XZ26311RJX tag=3V2F47RCM5DC type=del size=2272838 2024-09-13T00:00:39.064 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.062+0000 7fb8f2ce0640 1 < completed write operation key=70F7D20HI504 tag=B4IE7O1R0G88 type=write size=633193 2024-09-13T00:00:39.069 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.070+0000 7fb8f2ce0640 1 > prepared operation key=2TWD9YLMSYJI tag=AATEPCSR9LT2 type=del size=2875856 2024-09-13T00:00:39.075 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.074+0000 7fb8f2ce0640 1 < completed part key=_multipart_2AHYRZUDFHP4.2~T2WPKU5F5HLO.0 size=532614 2024-09-13T00:00:39.082 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.082+0000 7fb8f2ce0640 1 < completed multipart upload key=2AHYRZUDFHP4 upload=T2WPKU5F5HLO tag=K0TULNTKOOWX type=write size=1581190 2024-09-13T00:00:39.087 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.086+0000 7fb8f2ce0640 1 > prepared operation key=EUE3LT0PDIXV tag=E4A9S1NAPOCW type=write size=621189 2024-09-13T00:00:39.091 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.090+0000 7fb8f2ce0640 1 < completed write operation key=1PX7DBO9M3YI tag=XOTZC635YY8A type=write size=196946 2024-09-13T00:00:39.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:38 smithi181 bash[67587]: cluster 2024-09-13T00:00:37.127779+0000 mgr.y (mgr.24661) 705 : cluster [DBG] pgmap v1381: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 499 KiB/s rd, 172 KiB/s wr, 883 op/s 2024-09-13T00:00:39.099 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.098+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~J9FP86XIZ6NC.0 size=1048576 2024-09-13T00:00:39.103 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.102+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~J9FP86XIZ6NC.1 size=1048576 2024-09-13T00:00:39.106 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.106+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~J9FP86XIZ6NC.2 size=6979 2024-09-13T00:00:39.106 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.106+0000 7fb8f2ce0640 1 > prepared multipart upload key=1PX7DBO9M3YI upload=J9FP86XIZ6NC tag=2HVHGW9S8YEC type=write size=3152707 2024-09-13T00:00:39.110 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.110+0000 7fb8f2ce0640 1 < completed write operation key=P8OYS5ZE1NX0 tag=ZXJ45A3D62FB type=write size=863794 2024-09-13T00:00:39.118 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.118+0000 7fb8f2ce0640 1 > prepared part key=_multipart_O4MZ4UWNXME9.2~AXNIA3YS7H4C.0 size=362223 2024-09-13T00:00:39.118 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.118+0000 7fb8f2ce0640 1 > prepared multipart upload key=O4MZ4UWNXME9 upload=AXNIA3YS7H4C tag=I0RWG4NZHKJQ type=write size=1410799 2024-09-13T00:00:39.122 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.122+0000 7fb8f2ce0640 1 < completed part key=_multipart_7OX5OJZ8GXU5.2~9Q5RV7J9GCV4.0 size=81431 2024-09-13T00:00:39.129 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.130+0000 7fb8f2ce0640 1 < completed multipart upload key=7OX5OJZ8GXU5 upload=9Q5RV7J9GCV4 tag=P2Q3JNZYYKKK type=write size=1130007 2024-09-13T00:00:39.134 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.134+0000 7fb8f2ce0640 1 > prepared operation key=P8OYS5ZE1NX0 tag=MO5A21RH597F type=del size=1176055 2024-09-13T00:00:39.138 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.138+0000 7fb8f2ce0640 1 < completed delete operation key=8MPUVD37EDN9 tag=P6QM842HMAJ6 type=del 2024-09-13T00:00:39.143 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.142+0000 7fb8f2ce0640 1 > prepared operation key=63FJDWBO2T9S tag=0FXXJLF570U8 type=del size=923065 2024-09-13T00:00:39.148 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.146+0000 7fb8f2ce0640 1 < completed write operation key=QMZREZ1QEDNJ tag=ZQ6C2ATUFP0M type=write size=24540 2024-09-13T00:00:39.156 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.154+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~PDMIYNSNODBS.0 size=1048576 2024-09-13T00:00:39.159 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.158+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~PDMIYNSNODBS.1 size=297809 2024-09-13T00:00:39.159 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.158+0000 7fb8f2ce0640 1 > prepared multipart upload key=NQJQC8CEAC15 upload=PDMIYNSNODBS tag=KQ034LMC5GY4 type=write size=2394961 2024-09-13T00:00:39.164 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.162+0000 7fb8f2ce0640 1 < completed part key=_multipart_85YEW8N43FTA.2~RB73SBTBUAAR.0 size=492916 2024-09-13T00:00:39.171 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.170+0000 7fb8f2ce0640 1 < completed multipart upload key=85YEW8N43FTA upload=RB73SBTBUAAR tag=XWYAMN0PHK71 type=write size=1541492 2024-09-13T00:00:39.176 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.174+0000 7fb8f2ce0640 1 > prepared operation key=H9AU4HD247YU tag=FJX0T9NUB651 type=del size=3925601 2024-09-13T00:00:39.180 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.178+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~H38GC4S3NB3U.0 size=178259 2024-09-13T00:00:39.189 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.190+0000 7fb8f2ce0640 1 < completed multipart upload key=ZTK9AYBSLCOZ upload=H38GC4S3NB3U tag=K09GXQY9LH6C type=write size=1226835 2024-09-13T00:00:39.194 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.194+0000 7fb8f2ce0640 1 > prepared operation key=85YEW8N43FTA tag=7HFAJOVYCQE4 type=del size=1944401 2024-09-13T00:00:39.202 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.202+0000 7fb8f2ce0640 1 < suggested update operation key=1HX06CCZJ1DD tag=9DQ9TKJV9Y5S type=del size=651501 2024-09-13T00:00:39.210 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.210+0000 7fb8f2ce0640 1 > prepared part key=_multipart_O4MZ4UWNXME9.2~LXR7BVTGJDHD.0 size=1042512 2024-09-13T00:00:39.210 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.210+0000 7fb8f2ce0640 1 > prepared multipart upload key=O4MZ4UWNXME9 upload=LXR7BVTGJDHD tag=0GPAAL41I3JU type=write size=2091088 2024-09-13T00:00:39.214 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.214+0000 7fb8f2ce0640 1 < completed delete operation key=2TWD9YLMSYJI tag=NLUG9HSE32AO type=del 2024-09-13T00:00:39.218 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.218+0000 7fb8f2ce0640 1 > prepared operation key=NX26SI4F2ZMB tag=JJE7FWZNT9X5 type=del size=2120567 2024-09-13T00:00:39.222 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.222+0000 7fb8f2ce0640 1 < completed delete operation key=8Z6U7OOA0GBH tag=82F2BBFPNRK0 type=del 2024-09-13T00:00:39.232 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.230+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~16NOMDAE9GRD.0 size=1048576 2024-09-13T00:00:39.232 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.230+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~16NOMDAE9GRD.1 size=935685 2024-09-13T00:00:39.232 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.230+0000 7fb8f2ce0640 1 > prepared multipart upload key=1HX06CCZJ1DD upload=16NOMDAE9GRD tag=PY0CJ28ESXW8 type=write size=3032837 2024-09-13T00:00:39.239 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.238+0000 7fb8f2ce0640 1 < suggested update operation key=EUE3LT0PDIXV tag=LXN9763CJP9K type=del size=1373381 2024-09-13T00:00:39.247 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.246+0000 7fb8f2ce0640 1 > prepared part key=_multipart_P8OYS5ZE1NX0.2~RXDED4P207VF.0 size=1048576 2024-09-13T00:00:39.250 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.250+0000 7fb8f2ce0640 1 > prepared part key=_multipart_P8OYS5ZE1NX0.2~RXDED4P207VF.1 size=1048576 2024-09-13T00:00:39.253 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.254+0000 7fb8f2ce0640 1 > prepared part key=_multipart_P8OYS5ZE1NX0.2~RXDED4P207VF.2 size=706742 2024-09-13T00:00:39.253 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.254+0000 7fb8f2ce0640 1 > prepared multipart upload key=P8OYS5ZE1NX0 upload=RXDED4P207VF tag=4BJ6GSV5RR15 type=write size=3852470 2024-09-13T00:00:39.253 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.254+0000 7fb8f2ce0640 1 < canceled part key=_multipart_1HX06CCZJ1DD.2~WSP4TNP8H3WN.0 size=674713 2024-09-13T00:00:39.260 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.258+0000 7fb8f2ce0640 1 < canceled multipart upload key=1HX06CCZJ1DD upload=WSP4TNP8H3WN tag=F8KA5FC72OKR type=write size=1723289 2024-09-13T00:00:39.268 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.266+0000 7fb8f2ce0640 1 > prepared part key=_multipart_44EDYFT9VF8Y.2~TY3NWEC6HOU4.0 size=1048576 2024-09-13T00:00:39.271 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.270+0000 7fb8f2ce0640 1 > prepared part key=_multipart_44EDYFT9VF8Y.2~TY3NWEC6HOU4.1 size=944509 2024-09-13T00:00:39.271 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.270+0000 7fb8f2ce0640 1 > prepared multipart upload key=44EDYFT9VF8Y upload=TY3NWEC6HOU4 tag=1KCM3X4MDQSB type=write size=3041661 2024-09-13T00:00:39.276 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.274+0000 7fb8f2ce0640 1 < completed part key=_multipart_H4M1TCN5W6XU.2~5VRVOUINMGM1.0 size=864625 2024-09-13T00:00:39.282 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.282+0000 7fb8f2ce0640 1 < completed multipart upload key=H4M1TCN5W6XU upload=5VRVOUINMGM1 tag=E0K87C2MC1M1 type=write size=1913201 2024-09-13T00:00:39.290 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.290+0000 7fb8f2ce0640 1 > prepared part key=_multipart_85YEW8N43FTA.2~FBZAMAKZ1RNG.0 size=1048576 2024-09-13T00:00:39.293 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.294+0000 7fb8f2ce0640 1 > prepared part key=_multipart_85YEW8N43FTA.2~FBZAMAKZ1RNG.1 size=1048576 2024-09-13T00:00:39.296 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.294+0000 7fb8f2ce0640 1 > prepared part key=_multipart_85YEW8N43FTA.2~FBZAMAKZ1RNG.2 size=45551 2024-09-13T00:00:39.296 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.294+0000 7fb8f2ce0640 1 > prepared multipart upload key=85YEW8N43FTA upload=FBZAMAKZ1RNG tag=8TRYVIVQR3ZI type=write size=3191279 2024-09-13T00:00:39.300 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.298+0000 7fb8f2ce0640 1 < completed delete operation key=W8XZ26311RJX tag=3V2F47RCM5DC type=del 2024-09-13T00:00:39.304 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.302+0000 7fb8f2ce0640 1 > prepared operation key=J8NGDSF9XRTP tag=YZXFZYKPRBEX type=del size=289849 2024-09-13T00:00:39.311 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.310+0000 7fb8f2ce0640 1 < suggested update operation key=2TWD9YLMSYJI tag=AATEPCSR9LT2 type=del size=2875856 2024-09-13T00:00:39.315 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.314+0000 7fb8f2ce0640 1 > prepared operation key=ZFHN2SNTZ5W7 tag=3OIDVUKUB1NF type=del size=4086949 2024-09-13T00:00:39.319 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.318+0000 7fb8f2ce0640 -1 < failed to complete operation key=EUE3LT0PDIXV tag=E4A9S1NAPOCW type=write size=621189: (22) Invalid argument 2024-09-13T00:00:39.322 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.322+0000 7fb8f2ce0640 1 > prepared operation key=NQJQC8CEAC15 tag=S5WF857OFT11 type=del size=285694 2024-09-13T00:00:39.325 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.326+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~J9FP86XIZ6NC.0 size=1048576 2024-09-13T00:00:39.327 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.326+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~J9FP86XIZ6NC.1 size=1048576 2024-09-13T00:00:39.329 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.330+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~J9FP86XIZ6NC.2 size=6979 2024-09-13T00:00:39.334 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.334+0000 7fb8f2ce0640 1 < completed multipart upload key=1PX7DBO9M3YI upload=J9FP86XIZ6NC tag=2HVHGW9S8YEC type=write size=3152707 2024-09-13T00:00:39.338 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.338+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~91CHB3HXTB1T.0 size=1048576 2024-09-13T00:00:39.340 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.338+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~91CHB3HXTB1T.1 size=1048576 2024-09-13T00:00:39.343 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.342+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~91CHB3HXTB1T.2 size=866536 2024-09-13T00:00:39.343 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.342+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZTK9AYBSLCOZ upload=91CHB3HXTB1T tag=96IE2RI4X4YE type=write size=4012264 2024-09-13T00:00:39.345 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.346+0000 7fb8f2ce0640 1 < completed part key=_multipart_O4MZ4UWNXME9.2~AXNIA3YS7H4C.0 size=362223 2024-09-13T00:00:39.348 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.346+0000 7fb8f2ce0640 1 < completed multipart upload key=O4MZ4UWNXME9 upload=AXNIA3YS7H4C tag=I0RWG4NZHKJQ type=write size=1410799 2024-09-13T00:00:39.353 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.354+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~H76WDH6ACY7V.0 size=1048576 2024-09-13T00:00:39.355 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.354+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~H76WDH6ACY7V.1 size=1048576 2024-09-13T00:00:39.357 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.358+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~H76WDH6ACY7V.2 size=1012001 2024-09-13T00:00:39.357 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.358+0000 7fb8f2ce0640 1 > prepared multipart upload key=W8XZ26311RJX upload=H76WDH6ACY7V tag=OCQ5ZFJB34MF type=write size=4157729 2024-09-13T00:00:39.359 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.358+0000 7fb8f2ce0640 1 < completed delete operation key=P8OYS5ZE1NX0 tag=MO5A21RH597F type=del 2024-09-13T00:00:39.361 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.362+0000 7fb8f2ce0640 1 > prepared operation key=EUE3LT0PDIXV tag=HG33SB7B5DP6 type=del size=1755293 2024-09-13T00:00:39.363 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.362+0000 7fb8f2ce0640 1 < completed delete operation key=63FJDWBO2T9S tag=0FXXJLF570U8 type=del 2024-09-13T00:00:39.365 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.366+0000 7fb8f2ce0640 1 > prepared operation key=O4MZ4UWNXME9 tag=L9XCIDK3G1DO type=del size=1747949 2024-09-13T00:00:39.368 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.366+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~PDMIYNSNODBS.0 size=1048576 2024-09-13T00:00:39.370 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.370+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~PDMIYNSNODBS.1 size=297809 2024-09-13T00:00:39.373 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.374+0000 7fb8f2ce0640 1 < completed multipart upload key=NQJQC8CEAC15 upload=PDMIYNSNODBS tag=KQ034LMC5GY4 type=write size=2394961 2024-09-13T00:00:39.375 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.374+0000 7fb8f2ce0640 1 > prepared operation key=4K76U69GZ9AE tag=8OKNRTEG29U7 type=del size=757432 2024-09-13T00:00:39.380 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.378+0000 7fb8f2ce0640 1 < suggested update operation key=H9AU4HD247YU tag=FJX0T9NUB651 type=del size=3925601 2024-09-13T00:00:39.383 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.382+0000 7fb8f2ce0640 1 > prepared operation key=70F7D20HI504 tag=TYL7QCISDDYC type=del size=1025826 2024-09-13T00:00:39.385 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.386+0000 7fb8f2ce0640 1 < completed delete operation key=85YEW8N43FTA tag=7HFAJOVYCQE4 type=del 2024-09-13T00:00:39.387 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.386+0000 7fb8f2ce0640 1 > prepared operation key=1PX7DBO9M3YI tag=RCSHHUDU7OHL type=del size=2025278 2024-09-13T00:00:39.389 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.390+0000 7fb8f2ce0640 1 < completed part key=_multipart_O4MZ4UWNXME9.2~LXR7BVTGJDHD.0 size=1042512 2024-09-13T00:00:39.392 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.390+0000 7fb8f2ce0640 1 < completed multipart upload key=O4MZ4UWNXME9 upload=LXR7BVTGJDHD tag=0GPAAL41I3JU type=write size=2091088 2024-09-13T00:00:39.394 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.394+0000 7fb8f2ce0640 1 > prepared operation key=5GM88ERM89KT tag=V3O19QUCHP3U type=del size=3349568 2024-09-13T00:00:39.396 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.394+0000 7fb8f2ce0640 1 < completed delete operation key=NX26SI4F2ZMB tag=JJE7FWZNT9X5 type=del 2024-09-13T00:00:39.400 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.398+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H4M1TCN5W6XU.2~3EVGG0V7M0RN.0 size=1048576 2024-09-13T00:00:39.402 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.402+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H4M1TCN5W6XU.2~3EVGG0V7M0RN.1 size=651298 2024-09-13T00:00:39.402 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.402+0000 7fb8f2ce0640 1 > prepared multipart upload key=H4M1TCN5W6XU upload=3EVGG0V7M0RN tag=T4YIXV307A41 type=write size=2748450 2024-09-13T00:00:39.404 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.402+0000 7fb8f2ce0640 1 < completed part key=_multipart_1HX06CCZJ1DD.2~16NOMDAE9GRD.0 size=1048576 2024-09-13T00:00:39.405 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.406+0000 7fb8f2ce0640 1 < completed part key=_multipart_1HX06CCZJ1DD.2~16NOMDAE9GRD.1 size=935685 2024-09-13T00:00:39.409 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.410+0000 7fb8f2ce0640 1 < completed multipart upload key=1HX06CCZJ1DD upload=16NOMDAE9GRD tag=PY0CJ28ESXW8 type=write size=3032837 2024-09-13T00:00:39.413 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.414+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~5NLQ7TZJQWR7.0 size=1048576 2024-09-13T00:00:39.415 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.414+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~5NLQ7TZJQWR7.1 size=1048576 2024-09-13T00:00:39.416 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.414+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~5NLQ7TZJQWR7.2 size=324348 2024-09-13T00:00:39.417 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.414+0000 7fb8f2ce0640 1 > prepared multipart upload key=63FJDWBO2T9S upload=5NLQ7TZJQWR7 tag=QS4E5DT7DH8Y type=write size=3470076 2024-09-13T00:00:39.418 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.418+0000 7fb8f2ce0640 1 < completed part key=_multipart_P8OYS5ZE1NX0.2~RXDED4P207VF.0 size=1048576 2024-09-13T00:00:39.420 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.418+0000 7fb8f2ce0640 1 < completed part key=_multipart_P8OYS5ZE1NX0.2~RXDED4P207VF.1 size=1048576 2024-09-13T00:00:39.424 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.422+0000 7fb8f2ce0640 1 < completed part key=_multipart_P8OYS5ZE1NX0.2~RXDED4P207VF.2 size=706742 2024-09-13T00:00:39.425 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.426+0000 7fb8f2ce0640 1 < completed multipart upload key=P8OYS5ZE1NX0 upload=RXDED4P207VF tag=4BJ6GSV5RR15 type=write size=3852470 2024-09-13T00:00:39.427 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.426+0000 7fb8f2ce0640 1 > prepared operation key=85YEW8N43FTA tag=L8YT620OYHEN type=del size=3906763 2024-09-13T00:00:39.429 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.430+0000 7fb8f2ce0640 1 < completed part key=_multipart_44EDYFT9VF8Y.2~TY3NWEC6HOU4.0 size=1048576 2024-09-13T00:00:39.431 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.430+0000 7fb8f2ce0640 1 < completed part key=_multipart_44EDYFT9VF8Y.2~TY3NWEC6HOU4.1 size=944509 2024-09-13T00:00:39.433 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.434+0000 7fb8f2ce0640 1 < completed multipart upload key=44EDYFT9VF8Y upload=TY3NWEC6HOU4 tag=1KCM3X4MDQSB type=write size=3041661 2024-09-13T00:00:39.435 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.434+0000 7fb8f2ce0640 1 > prepared operation key=44EDYFT9VF8Y tag=S007E5186IOZ type=del size=679367 2024-09-13T00:00:39.437 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.438+0000 7fb8f2ce0640 1 < completed part key=_multipart_85YEW8N43FTA.2~FBZAMAKZ1RNG.0 size=1048576 2024-09-13T00:00:39.439 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.438+0000 7fb8f2ce0640 1 < completed part key=_multipart_85YEW8N43FTA.2~FBZAMAKZ1RNG.1 size=1048576 2024-09-13T00:00:39.440 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.438+0000 7fb8f2ce0640 1 < completed part key=_multipart_85YEW8N43FTA.2~FBZAMAKZ1RNG.2 size=45551 2024-09-13T00:00:39.444 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.442+0000 7fb8f2ce0640 1 < completed multipart upload key=85YEW8N43FTA upload=FBZAMAKZ1RNG tag=8TRYVIVQR3ZI type=write size=3191279 2024-09-13T00:00:39.446 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.446+0000 7fb8f2ce0640 1 > prepared operation key=N3Z5XIG3TZRQ tag=E37U5Y1LTAT3 type=del size=2735997 2024-09-13T00:00:39.447 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.446+0000 7fb8f2ce0640 1 < canceled operation key=J8NGDSF9XRTP tag=YZXFZYKPRBEX type=del size=289849 2024-09-13T00:00:39.451 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.450+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~VRET0PH622P0.0 size=1048576 2024-09-13T00:00:39.453 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.454+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~VRET0PH622P0.1 size=1048576 2024-09-13T00:00:39.454 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.454+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~VRET0PH622P0.2 size=966961 2024-09-13T00:00:39.454 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.454+0000 7fb8f2ce0640 1 > prepared multipart upload key=W8XZ26311RJX upload=VRET0PH622P0 tag=EX9GD94ZE1RX type=write size=4112689 2024-09-13T00:00:39.456 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.454+0000 7fb8f2ce0640 1 < completed delete operation key=ZFHN2SNTZ5W7 tag=3OIDVUKUB1NF type=del 2024-09-13T00:00:39.458 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.458+0000 7fb8f2ce0640 1 > prepared operation key=H9AU4HD247YU tag=COXHU1MHZ6XQ type=write size=558610 2024-09-13T00:00:39.460 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.458+0000 7fb8f2ce0640 1 < canceled operation key=NQJQC8CEAC15 tag=S5WF857OFT11 type=del size=285694 2024-09-13T00:00:39.464 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.462+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H5EIP0E9CBWX.2~3HJN6AMK6P79.0 size=1048576 2024-09-13T00:00:39.466 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.466+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H5EIP0E9CBWX.2~3HJN6AMK6P79.1 size=670391 2024-09-13T00:00:39.466 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.466+0000 7fb8f2ce0640 1 > prepared multipart upload key=H5EIP0E9CBWX upload=3HJN6AMK6P79 tag=QRPPR8G6AUKR type=write size=2767543 2024-09-13T00:00:39.468 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.466+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~91CHB3HXTB1T.0 size=1048576 2024-09-13T00:00:39.469 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.470+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~91CHB3HXTB1T.1 size=1048576 2024-09-13T00:00:39.471 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.470+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~91CHB3HXTB1T.2 size=866536 2024-09-13T00:00:39.475 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.474+0000 7fb8f2ce0640 1 < completed multipart upload key=ZTK9AYBSLCOZ upload=91CHB3HXTB1T tag=96IE2RI4X4YE type=write size=4012264 2024-09-13T00:00:39.478 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.478+0000 7fb8f2ce0640 1 > prepared part key=_multipart_0WC6P8YNL0H2.2~JHM7KFSUCOOJ.0 size=958935 2024-09-13T00:00:39.478 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.478+0000 7fb8f2ce0640 1 > prepared multipart upload key=0WC6P8YNL0H2 upload=JHM7KFSUCOOJ tag=4AQ9G4QEK496 type=write size=2007511 2024-09-13T00:00:39.480 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.478+0000 7fb8f2ce0640 1 < completed part key=_multipart_W8XZ26311RJX.2~H76WDH6ACY7V.0 size=1048576 2024-09-13T00:00:39.482 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.482+0000 7fb8f2ce0640 1 < completed part key=_multipart_W8XZ26311RJX.2~H76WDH6ACY7V.1 size=1048576 2024-09-13T00:00:39.483 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.482+0000 7fb8f2ce0640 1 < completed part key=_multipart_W8XZ26311RJX.2~H76WDH6ACY7V.2 size=1012001 2024-09-13T00:00:39.487 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.486+0000 7fb8f2ce0640 1 < completed multipart upload key=W8XZ26311RJX upload=H76WDH6ACY7V tag=OCQ5ZFJB34MF type=write size=4157729 2024-09-13T00:00:39.489 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.490+0000 7fb8f2ce0640 1 > prepared operation key=44EDYFT9VF8Y tag=XDQIF8L2YQMI type=del size=3543433 2024-09-13T00:00:39.493 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.494+0000 7fb8f2ce0640 1 < suggested update operation key=EUE3LT0PDIXV tag=HG33SB7B5DP6 type=del size=1755293 2024-09-13T00:00:39.495 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.494+0000 7fb8f2ce0640 1 > prepared operation key=F0QCVR9E08WE tag=VGS38J29GPQ3 type=write size=679440 2024-09-13T00:00:39.499 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.498+0000 7fb8f2ce0640 1 < suggested remove operation key=O4MZ4UWNXME9 tag=L9XCIDK3G1DO type=del 2024-09-13T00:00:39.501 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.502+0000 7fb8f2ce0640 1 > prepared operation key=NQJQC8CEAC15 tag=EURDJ5W4LIHN type=del size=407527 2024-09-13T00:00:39.503 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.502+0000 7fb8f2ce0640 1 < completed delete operation key=4K76U69GZ9AE tag=8OKNRTEG29U7 type=del 2024-09-13T00:00:39.505 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.506+0000 7fb8f2ce0640 1 > prepared operation key=85YEW8N43FTA tag=H6CBAIL002V9 type=del size=3395226 2024-09-13T00:00:39.507 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.506+0000 7fb8f2ce0640 1 < completed delete operation key=70F7D20HI504 tag=TYL7QCISDDYC type=del 2024-09-13T00:00:39.510 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.510+0000 7fb8f2ce0640 1 > prepared operation key=N3Z5XIG3TZRQ tag=V0O6RFU4EL3O type=del size=1431198 2024-09-13T00:00:39.512 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.510+0000 7fb8f2ce0640 1 < completed delete operation key=1PX7DBO9M3YI tag=RCSHHUDU7OHL type=del 2024-09-13T00:00:39.516 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.514+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~LZEXJHGIGAWJ.0 size=1048576 2024-09-13T00:00:39.518 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.518+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~LZEXJHGIGAWJ.1 size=1048576 2024-09-13T00:00:39.520 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.518+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~LZEXJHGIGAWJ.2 size=454663 2024-09-13T00:00:39.520 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.518+0000 7fb8f2ce0640 1 > prepared multipart upload key=8MPUVD37EDN9 upload=LZEXJHGIGAWJ tag=1X8EJW92TT30 type=write size=3600391 2024-09-13T00:00:39.522 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.522+0000 7fb8f2ce0640 1 < completed delete operation key=5GM88ERM89KT tag=V3O19QUCHP3U type=del 2024-09-13T00:00:39.524 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.522+0000 7fb8f2ce0640 1 > prepared operation key=EJN6HR8I7XQF tag=ZLLG7IEYJN9D type=del size=1691238 2024-09-13T00:00:39.526 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.526+0000 7fb8f2ce0640 1 < completed part key=_multipart_H4M1TCN5W6XU.2~3EVGG0V7M0RN.0 size=1048576 2024-09-13T00:00:39.527 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.526+0000 7fb8f2ce0640 1 < completed part key=_multipart_H4M1TCN5W6XU.2~3EVGG0V7M0RN.1 size=651298 2024-09-13T00:00:39.530 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.530+0000 7fb8f2ce0640 1 < completed multipart upload key=H4M1TCN5W6XU upload=3EVGG0V7M0RN tag=T4YIXV307A41 type=write size=2748450 2024-09-13T00:00:39.533 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.534+0000 7fb8f2ce0640 1 > prepared operation key=85YEW8N43FTA tag=2985EJX0FD2L type=del size=293343 2024-09-13T00:00:39.535 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.534+0000 7fb8f2ce0640 1 < completed part key=_multipart_63FJDWBO2T9S.2~5NLQ7TZJQWR7.0 size=1048576 2024-09-13T00:00:39.537 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.538+0000 7fb8f2ce0640 1 < completed part key=_multipart_63FJDWBO2T9S.2~5NLQ7TZJQWR7.1 size=1048576 2024-09-13T00:00:39.538 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.538+0000 7fb8f2ce0640 1 < completed part key=_multipart_63FJDWBO2T9S.2~5NLQ7TZJQWR7.2 size=324348 2024-09-13T00:00:39.542 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.542+0000 7fb8f2ce0640 1 < completed multipart upload key=63FJDWBO2T9S upload=5NLQ7TZJQWR7 tag=QS4E5DT7DH8Y type=write size=3470076 2024-09-13T00:00:39.544 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.542+0000 7fb8f2ce0640 1 > prepared operation key=EJN6HR8I7XQF tag=2OY4G77YWIO0 type=del size=3697627 2024-09-13T00:00:39.546 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.546+0000 7fb8f2ce0640 1 < completed delete operation key=85YEW8N43FTA tag=L8YT620OYHEN type=del 2024-09-13T00:00:39.550 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.550+0000 7fb8f2ce0640 1 > prepared part key=_multipart_F0QCVR9E08WE.2~VKDWBWFNEYVP.0 size=1048576 2024-09-13T00:00:39.551 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.550+0000 7fb8f2ce0640 1 > prepared part key=_multipart_F0QCVR9E08WE.2~VKDWBWFNEYVP.1 size=1048576 2024-09-13T00:00:39.553 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.554+0000 7fb8f2ce0640 1 > prepared part key=_multipart_F0QCVR9E08WE.2~VKDWBWFNEYVP.2 size=891626 2024-09-13T00:00:39.553 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.554+0000 7fb8f2ce0640 1 > prepared multipart upload key=F0QCVR9E08WE upload=VKDWBWFNEYVP tag=VTNGNVMN7T0Z type=write size=4037354 2024-09-13T00:00:39.555 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.554+0000 7fb8f2ce0640 1 < completed delete operation key=44EDYFT9VF8Y tag=S007E5186IOZ type=del 2024-09-13T00:00:39.559 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.558+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~M2QYY9UFFKQ9.0 size=1048576 2024-09-13T00:00:39.561 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.562+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~M2QYY9UFFKQ9.1 size=1048576 2024-09-13T00:00:39.562 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.562+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~M2QYY9UFFKQ9.2 size=958690 2024-09-13T00:00:39.562 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.562+0000 7fb8f2ce0640 1 > prepared multipart upload key=2AHYRZUDFHP4 upload=M2QYY9UFFKQ9 tag=BXPDEPJO3WQ9 type=write size=4104418 2024-09-13T00:00:39.564 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.562+0000 7fb8f2ce0640 1 < completed delete operation key=N3Z5XIG3TZRQ tag=E37U5Y1LTAT3 type=del 2024-09-13T00:00:39.567 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.566+0000 7fb8f2ce0640 1 > prepared operation key=8MPUVD37EDN9 tag=W16U8XUIYS9B type=del size=908989 2024-09-13T00:00:39.570 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.570+0000 7fb8f2ce0640 1 < completed part key=_multipart_W8XZ26311RJX.2~VRET0PH622P0.0 size=1048576 2024-09-13T00:00:39.572 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.570+0000 7fb8f2ce0640 1 < completed part key=_multipart_W8XZ26311RJX.2~VRET0PH622P0.1 size=1048576 2024-09-13T00:00:39.574 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.574+0000 7fb8f2ce0640 1 < completed part key=_multipart_W8XZ26311RJX.2~VRET0PH622P0.2 size=966961 2024-09-13T00:00:39.578 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.578+0000 7fb8f2ce0640 1 < completed multipart upload key=W8XZ26311RJX upload=VRET0PH622P0 tag=EX9GD94ZE1RX type=write size=4112689 2024-09-13T00:00:39.580 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.578+0000 7fb8f2ce0640 1 > prepared operation key=85YEW8N43FTA tag=JT1QJ8P16AGD type=del size=875547 2024-09-13T00:00:39.582 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.582+0000 7fb8f2ce0640 1 < completed write operation key=H9AU4HD247YU tag=COXHU1MHZ6XQ type=write size=558610 2024-09-13T00:00:39.584 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.582+0000 7fb8f2ce0640 1 > prepared operation key=QMZREZ1QEDNJ tag=L2OR7SZFOKNP type=write size=445220 2024-09-13T00:00:39.586 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.586+0000 7fb8f2ce0640 1 < completed part key=_multipart_H5EIP0E9CBWX.2~3HJN6AMK6P79.0 size=1048576 2024-09-13T00:00:39.588 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.586+0000 7fb8f2ce0640 1 < completed part key=_multipart_H5EIP0E9CBWX.2~3HJN6AMK6P79.1 size=670391 2024-09-13T00:00:39.592 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.590+0000 7fb8f2ce0640 1 < completed multipart upload key=H5EIP0E9CBWX upload=3HJN6AMK6P79 tag=QRPPR8G6AUKR type=write size=2767543 2024-09-13T00:00:39.596 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.594+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EJN6HR8I7XQF.2~3F1G5KMDVAHL.0 size=1048576 2024-09-13T00:00:39.598 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.598+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EJN6HR8I7XQF.2~3F1G5KMDVAHL.1 size=1048576 2024-09-13T00:00:39.600 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.598+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EJN6HR8I7XQF.2~3F1G5KMDVAHL.2 size=152518 2024-09-13T00:00:39.600 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.598+0000 7fb8f2ce0640 1 > prepared multipart upload key=EJN6HR8I7XQF upload=3F1G5KMDVAHL tag=03439X63UTVZ type=write size=3298246 2024-09-13T00:00:39.601 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.602+0000 7fb8f2ce0640 1 < completed part key=_multipart_0WC6P8YNL0H2.2~JHM7KFSUCOOJ.0 size=958935 2024-09-13T00:00:39.605 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.606+0000 7fb8f2ce0640 1 < completed multipart upload key=0WC6P8YNL0H2 upload=JHM7KFSUCOOJ tag=4AQ9G4QEK496 type=write size=2007511 2024-09-13T00:00:39.607 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.606+0000 7fb8f2ce0640 1 > prepared operation key=H9AU4HD247YU tag=Z05F2I95GPOL type=del size=3898672 2024-09-13T00:00:39.610 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.610+0000 7fb8f2ce0640 1 < completed delete operation key=44EDYFT9VF8Y tag=XDQIF8L2YQMI type=del 2024-09-13T00:00:39.613 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.614+0000 7fb8f2ce0640 1 > prepared part key=_multipart_P8OYS5ZE1NX0.2~B89L2JPTTTWD.0 size=1048576 2024-09-13T00:00:39.615 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.614+0000 7fb8f2ce0640 1 > prepared part key=_multipart_P8OYS5ZE1NX0.2~B89L2JPTTTWD.1 size=1048576 2024-09-13T00:00:39.617 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.618+0000 7fb8f2ce0640 1 > prepared part key=_multipart_P8OYS5ZE1NX0.2~B89L2JPTTTWD.2 size=284379 2024-09-13T00:00:39.617 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.618+0000 7fb8f2ce0640 1 > prepared multipart upload key=P8OYS5ZE1NX0 upload=B89L2JPTTTWD tag=HKUFSK20X8JP type=write size=3430107 2024-09-13T00:00:39.619 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.618+0000 7fb8f2ce0640 1 < completed write operation key=F0QCVR9E08WE tag=VGS38J29GPQ3 type=write size=679440 2024-09-13T00:00:39.621 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.622+0000 7fb8f2ce0640 1 > prepared operation key=H4M1TCN5W6XU tag=JASXNNX2IYWY type=del size=4075179 2024-09-13T00:00:39.623 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.622+0000 7fb8f2ce0640 1 < completed delete operation key=NQJQC8CEAC15 tag=EURDJ5W4LIHN type=del 2024-09-13T00:00:39.625 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.622+0000 7fb8f2ce0640 1 > prepared operation key=J8NGDSF9XRTP tag=45839A558BY5 type=del size=3664044 2024-09-13T00:00:39.627 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.626+0000 7fb8f2ce0640 1 < completed delete operation key=85YEW8N43FTA tag=H6CBAIL002V9 type=del 2024-09-13T00:00:39.631 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.630+0000 7fb8f2ce0640 1 > prepared part key=_multipart_85YEW8N43FTA.2~RBA39C9BOIVO.0 size=1048576 2024-09-13T00:00:39.632 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.630+0000 7fb8f2ce0640 1 > prepared part key=_multipart_85YEW8N43FTA.2~RBA39C9BOIVO.1 size=442229 2024-09-13T00:00:39.633 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.630+0000 7fb8f2ce0640 1 > prepared multipart upload key=85YEW8N43FTA upload=RBA39C9BOIVO tag=08QIO8J3TLCT type=write size=2539381 2024-09-13T00:00:39.635 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.634+0000 7fb8f2ce0640 1 < completed delete operation key=N3Z5XIG3TZRQ tag=V0O6RFU4EL3O type=del 2024-09-13T00:00:39.637 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.638+0000 7fb8f2ce0640 1 > prepared operation key=W8XZ26311RJX tag=UH0TTKBPT6YO type=del size=231716 2024-09-13T00:00:39.639 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.638+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~LZEXJHGIGAWJ.0 size=1048576 2024-09-13T00:00:39.641 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.642+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~LZEXJHGIGAWJ.1 size=1048576 2024-09-13T00:00:39.644 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.642+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~LZEXJHGIGAWJ.2 size=454663 2024-09-13T00:00:39.646 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.646+0000 7fb8f2ce0640 1 < completed multipart upload key=8MPUVD37EDN9 upload=LZEXJHGIGAWJ tag=1X8EJW92TT30 type=write size=3600391 2024-09-13T00:00:39.648 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.646+0000 7fb8f2ce0640 1 > prepared operation key=63FJDWBO2T9S tag=1X6144804R1H type=write size=870428 2024-09-13T00:00:39.650 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.650+0000 7fb8f2ce0640 1 < canceled operation key=EJN6HR8I7XQF tag=ZLLG7IEYJN9D type=del size=1691238 2024-09-13T00:00:39.652 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.650+0000 7fb8f2ce0640 1 > prepared operation key=5GM88ERM89KT tag=0R52YIY4MDVX type=write size=186179 2024-09-13T00:00:39.654 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.654+0000 7fb8f2ce0640 1 < completed delete operation key=85YEW8N43FTA tag=2985EJX0FD2L type=del 2024-09-13T00:00:39.658 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.658+0000 7fb8f2ce0640 1 > prepared part key=_multipart_J8NGDSF9XRTP.2~OSB575U28MF5.0 size=1048576 2024-09-13T00:00:39.659 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.658+0000 7fb8f2ce0640 1 > prepared part key=_multipart_J8NGDSF9XRTP.2~OSB575U28MF5.1 size=209951 2024-09-13T00:00:39.660 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.658+0000 7fb8f2ce0640 1 > prepared multipart upload key=J8NGDSF9XRTP upload=OSB575U28MF5 tag=HUT0F50B9G9G type=write size=2307103 2024-09-13T00:00:39.663 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.662+0000 7fb8f2ce0640 1 < completed delete operation key=EJN6HR8I7XQF tag=2OY4G77YWIO0 type=del 2024-09-13T00:00:39.665 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.666+0000 7fb8f2ce0640 1 > prepared operation key=2TWD9YLMSYJI tag=3RLUYVWAYBGC type=del size=1624065 2024-09-13T00:00:39.665 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.666+0000 7fb8f2ce0640 1 < canceled part key=_multipart_F0QCVR9E08WE.2~VKDWBWFNEYVP.0 size=1048576 2024-09-13T00:00:39.667 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.666+0000 7fb8f2ce0640 1 < completed part key=_multipart_F0QCVR9E08WE.2~VKDWBWFNEYVP.1 size=1048576 2024-09-13T00:00:39.669 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.670+0000 7fb8f2ce0640 1 < completed part key=_multipart_F0QCVR9E08WE.2~VKDWBWFNEYVP.2 size=891626 2024-09-13T00:00:39.672 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.670+0000 7fb8f2ce0640 1 < canceled multipart upload key=F0QCVR9E08WE upload=VKDWBWFNEYVP tag=VTNGNVMN7T0Z type=write size=4037354 2024-09-13T00:00:39.677 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.678+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~2VE52E0XH8L5.0 size=1048576 2024-09-13T00:00:39.679 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.678+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~2VE52E0XH8L5.1 size=1048576 2024-09-13T00:00:39.681 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.682+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~2VE52E0XH8L5.2 size=491163 2024-09-13T00:00:39.682 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.682+0000 7fb8f2ce0640 1 > prepared multipart upload key=8Z6U7OOA0GBH upload=2VE52E0XH8L5 tag=ZL9MVI3Z6CEQ type=write size=3636891 2024-09-13T00:00:39.684 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.682+0000 7fb8f2ce0640 1 < completed part key=_multipart_2AHYRZUDFHP4.2~M2QYY9UFFKQ9.0 size=1048576 2024-09-13T00:00:39.685 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.686+0000 7fb8f2ce0640 1 < completed part key=_multipart_2AHYRZUDFHP4.2~M2QYY9UFFKQ9.1 size=1048576 2024-09-13T00:00:39.687 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.686+0000 7fb8f2ce0640 1 < completed part key=_multipart_2AHYRZUDFHP4.2~M2QYY9UFFKQ9.2 size=958690 2024-09-13T00:00:39.691 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.690+0000 7fb8f2ce0640 1 < completed multipart upload key=2AHYRZUDFHP4 upload=M2QYY9UFFKQ9 tag=BXPDEPJO3WQ9 type=write size=4104418 2024-09-13T00:00:39.695 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.694+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZFHN2SNTZ5W7.2~LT11GC0BR1BG.0 size=596121 2024-09-13T00:00:39.695 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.694+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZFHN2SNTZ5W7 upload=LT11GC0BR1BG tag=VYFYPAJLWTDK type=write size=1644697 2024-09-13T00:00:39.697 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.698+0000 7fb8f2ce0640 1 < completed delete operation key=8MPUVD37EDN9 tag=W16U8XUIYS9B type=del 2024-09-13T00:00:39.700 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.698+0000 7fb8f2ce0640 1 > prepared operation key=5GM88ERM89KT tag=V647E1EXR3TI type=del size=3066788 2024-09-13T00:00:39.701 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:00:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:00:39] "GET /metrics HTTP/1.1" 200 36472 "" "Prometheus/2.43.0" 2024-09-13T00:00:39.703 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.702+0000 7fb8f2ce0640 1 < completed delete operation key=85YEW8N43FTA tag=JT1QJ8P16AGD type=del 2024-09-13T00:00:39.707 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.706+0000 7fb8f2ce0640 1 > prepared part key=_multipart_0WC6P8YNL0H2.2~7C6M9CVE5YVS.0 size=512685 2024-09-13T00:00:39.707 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.706+0000 7fb8f2ce0640 1 > prepared multipart upload key=0WC6P8YNL0H2 upload=7C6M9CVE5YVS tag=HNUHZEBM88DO type=write size=1561261 2024-09-13T00:00:39.709 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.710+0000 7fb8f2ce0640 1 < completed write operation key=QMZREZ1QEDNJ tag=L2OR7SZFOKNP type=write size=445220 2024-09-13T00:00:39.712 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.710+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EUE3LT0PDIXV.2~5XEENR59GACE.0 size=1048576 2024-09-13T00:00:39.713 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.714+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EUE3LT0PDIXV.2~5XEENR59GACE.1 size=1048576 2024-09-13T00:00:39.715 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.714+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EUE3LT0PDIXV.2~5XEENR59GACE.2 size=927782 2024-09-13T00:00:39.715 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.714+0000 7fb8f2ce0640 1 > prepared multipart upload key=EUE3LT0PDIXV upload=5XEENR59GACE tag=68ZS6S7FYPOW type=write size=4073510 2024-09-13T00:00:39.715 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.714+0000 7fb8f2ce0640 1 < canceled part key=_multipart_EJN6HR8I7XQF.2~3F1G5KMDVAHL.0 size=1048576 2024-09-13T00:00:39.717 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.718+0000 7fb8f2ce0640 1 < completed part key=_multipart_EJN6HR8I7XQF.2~3F1G5KMDVAHL.1 size=1048576 2024-09-13T00:00:39.719 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.718+0000 7fb8f2ce0640 1 < completed part key=_multipart_EJN6HR8I7XQF.2~3F1G5KMDVAHL.2 size=152518 2024-09-13T00:00:39.723 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.722+0000 7fb8f2ce0640 1 < canceled multipart upload key=EJN6HR8I7XQF upload=3F1G5KMDVAHL tag=03439X63UTVZ type=write size=3298246 2024-09-13T00:00:39.726 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.726+0000 7fb8f2ce0640 1 > prepared operation key=85YEW8N43FTA tag=JTCY3JE9BNG5 type=del size=558021 2024-09-13T00:00:39.728 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.726+0000 7fb8f2ce0640 1 < completed delete operation key=H9AU4HD247YU tag=Z05F2I95GPOL type=del 2024-09-13T00:00:39.731 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.730+0000 7fb8f2ce0640 1 > prepared operation key=W8XZ26311RJX tag=VRSX0NBF3778 type=del size=2305789 2024-09-13T00:00:39.733 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.734+0000 7fb8f2ce0640 1 < completed part key=_multipart_P8OYS5ZE1NX0.2~B89L2JPTTTWD.0 size=1048576 2024-09-13T00:00:39.733 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.734+0000 7fb8f2ce0640 1 < canceled part key=_multipart_P8OYS5ZE1NX0.2~B89L2JPTTTWD.1 size=1048576 2024-09-13T00:00:39.735 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.734+0000 7fb8f2ce0640 1 < completed part key=_multipart_P8OYS5ZE1NX0.2~B89L2JPTTTWD.2 size=284379 2024-09-13T00:00:39.740 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.738+0000 7fb8f2ce0640 1 < canceled multipart upload key=P8OYS5ZE1NX0 upload=B89L2JPTTTWD tag=HKUFSK20X8JP type=write size=3430107 2024-09-13T00:00:39.746 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.746+0000 7fb8f2ce0640 1 > prepared part key=_multipart_QMZREZ1QEDNJ.2~JEV3N1ACNVAY.0 size=878800 2024-09-13T00:00:39.746 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.746+0000 7fb8f2ce0640 1 > prepared multipart upload key=QMZREZ1QEDNJ upload=JEV3N1ACNVAY tag=GECKRFDT0A1R type=write size=1927376 2024-09-13T00:00:39.751 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.750+0000 7fb8f2ce0640 1 < suggested update operation key=H4M1TCN5W6XU tag=JASXNNX2IYWY type=del size=4075179 2024-09-13T00:00:39.754 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.754+0000 7fb8f2ce0640 1 > prepared operation key=2TWD9YLMSYJI tag=JQX0CCWKR0V1 type=del size=1190522 2024-09-13T00:00:39.758 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.758+0000 7fb8f2ce0640 1 < suggested remove operation key=J8NGDSF9XRTP tag=45839A558BY5 type=del 2024-09-13T00:00:39.762 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.762+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~W47E656R53Z8.0 size=1048576 2024-09-13T00:00:39.764 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.762+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~W47E656R53Z8.1 size=1048576 2024-09-13T00:00:39.765 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.766+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~W47E656R53Z8.2 size=1010940 2024-09-13T00:00:39.765 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.766+0000 7fb8f2ce0640 1 > prepared multipart upload key=2AHYRZUDFHP4 upload=W47E656R53Z8 tag=18XQ244KX4WS type=write size=4156668 2024-09-13T00:00:39.767 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.766+0000 7fb8f2ce0640 1 < completed part key=_multipart_85YEW8N43FTA.2~RBA39C9BOIVO.0 size=1048576 2024-09-13T00:00:39.769 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.770+0000 7fb8f2ce0640 1 < completed part key=_multipart_85YEW8N43FTA.2~RBA39C9BOIVO.1 size=442229 2024-09-13T00:00:39.772 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.770+0000 7fb8f2ce0640 1 < completed multipart upload key=85YEW8N43FTA upload=RBA39C9BOIVO tag=08QIO8J3TLCT type=write size=2539381 2024-09-13T00:00:39.775 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.774+0000 7fb8f2ce0640 1 > prepared operation key=F0QCVR9E08WE tag=5ZVF1PCH5YWT type=write size=926464 2024-09-13T00:00:39.779 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.778+0000 7fb8f2ce0640 1 < suggested update operation key=W8XZ26311RJX tag=UH0TTKBPT6YO type=del size=231716 2024-09-13T00:00:39.781 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.782+0000 7fb8f2ce0640 1 > prepared operation key=H4M1TCN5W6XU tag=D2M7KYXZ6U1F type=del size=2207898 2024-09-13T00:00:39.783 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.782+0000 7fb8f2ce0640 1 < completed write operation key=63FJDWBO2T9S tag=1X6144804R1H type=write size=870428 2024-09-13T00:00:39.787 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.786+0000 7fb8f2ce0640 1 > prepared part key=_multipart_4K76U69GZ9AE.2~IBHG228WQQJI.0 size=556667 2024-09-13T00:00:39.787 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.786+0000 7fb8f2ce0640 1 > prepared multipart upload key=4K76U69GZ9AE upload=IBHG228WQQJI tag=1O1BKKO0FRUQ type=write size=1605243 2024-09-13T00:00:39.789 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.790+0000 7fb8f2ce0640 1 < completed write operation key=5GM88ERM89KT tag=0R52YIY4MDVX type=write size=186179 2024-09-13T00:00:39.792 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.790+0000 7fb8f2ce0640 1 > prepared operation key=EUE3LT0PDIXV tag=AM35DIAIKIYL type=write size=1008201 2024-09-13T00:00:39.792 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.790+0000 7fb8f2ce0640 1 < canceled part key=_multipart_J8NGDSF9XRTP.2~OSB575U28MF5.0 size=1048576 2024-09-13T00:00:39.794 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.794+0000 7fb8f2ce0640 1 < completed part key=_multipart_J8NGDSF9XRTP.2~OSB575U28MF5.1 size=209951 2024-09-13T00:00:39.797 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.798+0000 7fb8f2ce0640 1 < canceled multipart upload key=J8NGDSF9XRTP upload=OSB575U28MF5 tag=HUT0F50B9G9G type=write size=2307103 2024-09-13T00:00:39.801 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.802+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~CZ6L0Q14BDRQ.0 size=1048576 2024-09-13T00:00:39.803 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.802+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~CZ6L0Q14BDRQ.1 size=121319 2024-09-13T00:00:39.803 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.802+0000 7fb8f2ce0640 1 > prepared multipart upload key=7OX5OJZ8GXU5 upload=CZ6L0Q14BDRQ tag=14CQOE8K48WF type=write size=2218471 2024-09-13T00:00:39.807 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.806+0000 7fb8f2ce0640 1 < suggested remove operation key=2TWD9YLMSYJI tag=3RLUYVWAYBGC type=del 2024-09-13T00:00:39.809 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.810+0000 7fb8f2ce0640 1 > prepared operation key=O4MZ4UWNXME9 tag=B77BUMVFTZBQ type=del size=2932573 2024-09-13T00:00:39.811 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.810+0000 7fb8f2ce0640 1 < completed part key=_multipart_8Z6U7OOA0GBH.2~2VE52E0XH8L5.0 size=1048576 2024-09-13T00:00:39.811 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.810+0000 7fb8f2ce0640 1 < canceled part key=_multipart_8Z6U7OOA0GBH.2~2VE52E0XH8L5.1 size=1048576 2024-09-13T00:00:39.813 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.810+0000 7fb8f2ce0640 1 < completed part key=_multipart_8Z6U7OOA0GBH.2~2VE52E0XH8L5.2 size=491163 2024-09-13T00:00:39.816 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.814+0000 7fb8f2ce0640 1 < canceled multipart upload key=8Z6U7OOA0GBH upload=2VE52E0XH8L5 tag=ZL9MVI3Z6CEQ type=write size=3636891 2024-09-13T00:00:39.818 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.818+0000 7fb8f2ce0640 1 > prepared operation key=2AHYRZUDFHP4 tag=QUMR7RF1KHSJ type=del size=3640688 2024-09-13T00:00:39.819 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.818+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZFHN2SNTZ5W7.2~LT11GC0BR1BG.0 size=596121 2024-09-13T00:00:39.822 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.822+0000 7fb8f2ce0640 1 < completed multipart upload key=ZFHN2SNTZ5W7 upload=LT11GC0BR1BG tag=VYFYPAJLWTDK type=write size=1644697 2024-09-13T00:00:39.825 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.826+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~NSFILYGRJ0H6.0 size=1048576 2024-09-13T00:00:39.827 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.826+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~NSFILYGRJ0H6.1 size=1048576 2024-09-13T00:00:39.829 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.830+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~NSFILYGRJ0H6.2 size=239070 2024-09-13T00:00:39.829 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.830+0000 7fb8f2ce0640 1 > prepared multipart upload key=NQJQC8CEAC15 upload=NSFILYGRJ0H6 tag=GJWKFNZ84EX5 type=write size=3384798 2024-09-13T00:00:39.831 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.830+0000 7fb8f2ce0640 1 < completed delete operation key=5GM88ERM89KT tag=V647E1EXR3TI type=del 2024-09-13T00:00:39.833 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.834+0000 7fb8f2ce0640 1 > prepared operation key=44EDYFT9VF8Y tag=1FM20QG6GMF0 type=write size=445196 2024-09-13T00:00:39.835 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.834+0000 7fb8f2ce0640 1 < completed part key=_multipart_0WC6P8YNL0H2.2~7C6M9CVE5YVS.0 size=512685 2024-09-13T00:00:39.839 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.838+0000 7fb8f2ce0640 1 < completed multipart upload key=0WC6P8YNL0H2 upload=7C6M9CVE5YVS tag=HNUHZEBM88DO type=write size=1561261 2024-09-13T00:00:39.841 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.842+0000 7fb8f2ce0640 1 > prepared operation key=ZFHN2SNTZ5W7 tag=0FYRBQEXEOLG type=del size=2093490 2024-09-13T00:00:39.843 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.842+0000 7fb8f2ce0640 1 < completed part key=_multipart_EUE3LT0PDIXV.2~5XEENR59GACE.0 size=1048576 2024-09-13T00:00:39.844 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.842+0000 7fb8f2ce0640 1 < completed part key=_multipart_EUE3LT0PDIXV.2~5XEENR59GACE.1 size=1048576 2024-09-13T00:00:39.848 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.846+0000 7fb8f2ce0640 1 < completed part key=_multipart_EUE3LT0PDIXV.2~5XEENR59GACE.2 size=927782 2024-09-13T00:00:39.849 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.850+0000 7fb8f2ce0640 1 < completed multipart upload key=EUE3LT0PDIXV upload=5XEENR59GACE tag=68ZS6S7FYPOW type=write size=4073510 2024-09-13T00:00:39.851 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.850+0000 7fb8f2ce0640 1 > prepared operation key=NQJQC8CEAC15 tag=QYOWHVRHPG2B type=del size=3323355 2024-09-13T00:00:39.853 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.854+0000 7fb8f2ce0640 1 < canceled operation key=85YEW8N43FTA tag=JTCY3JE9BNG5 type=del size=558021 2024-09-13T00:00:39.855 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.854+0000 7fb8f2ce0640 1 > prepared operation key=EJN6HR8I7XQF tag=K6652IT41YR2 type=del size=984756 2024-09-13T00:00:39.857 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.858+0000 7fb8f2ce0640 -1 < failed to complete operation key=W8XZ26311RJX tag=VRSX0NBF3778 type=del size=2305789: (22) Invalid argument 2024-09-13T00:00:39.860 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.858+0000 7fb8f2ce0640 1 > prepared operation key=H4M1TCN5W6XU tag=CXFTU3UCHDNK type=del size=2205993 2024-09-13T00:00:39.863 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.862+0000 7fb8f2ce0640 1 < completed part key=_multipart_QMZREZ1QEDNJ.2~JEV3N1ACNVAY.0 size=878800 2024-09-13T00:00:39.867 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.866+0000 7fb8f2ce0640 1 < completed multipart upload key=QMZREZ1QEDNJ upload=JEV3N1ACNVAY tag=GECKRFDT0A1R type=write size=1927376 2024-09-13T00:00:39.870 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.870+0000 7fb8f2ce0640 1 > prepared operation key=8MPUVD37EDN9 tag=BD9FFM3F9YVP type=del size=55585 2024-09-13T00:00:39.870 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.870+0000 7fb8f2ce0640 -1 < no bi entry to suggest for operation key=2TWD9YLMSYJI tag=JQX0CCWKR0V1 type=del size=1190522: (2) No such file or directory 2024-09-13T00:00:39.872 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.870+0000 7fb8f2ce0640 1 > prepared operation key=EUE3LT0PDIXV tag=R8VJFWX7RNCB type=del size=186717 2024-09-13T00:00:39.875 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.874+0000 7fb8f2ce0640 1 < completed part key=_multipart_2AHYRZUDFHP4.2~W47E656R53Z8.0 size=1048576 2024-09-13T00:00:39.876 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.874+0000 7fb8f2ce0640 1 < completed part key=_multipart_2AHYRZUDFHP4.2~W47E656R53Z8.1 size=1048576 2024-09-13T00:00:39.878 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.878+0000 7fb8f2ce0640 1 < completed part key=_multipart_2AHYRZUDFHP4.2~W47E656R53Z8.2 size=1010940 2024-09-13T00:00:39.881 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.882+0000 7fb8f2ce0640 1 < completed multipart upload key=2AHYRZUDFHP4 upload=W47E656R53Z8 tag=18XQ244KX4WS type=write size=4156668 2024-09-13T00:00:39.886 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.886+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~CVOTV69992EN.0 size=1048576 2024-09-13T00:00:39.888 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.886+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~CVOTV69992EN.1 size=649563 2024-09-13T00:00:39.889 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.886+0000 7fb8f2ce0640 1 > prepared multipart upload key=2AHYRZUDFHP4 upload=CVOTV69992EN tag=0MXX729TJZ74 type=write size=2746715 2024-09-13T00:00:39.891 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.890+0000 7fb8f2ce0640 1 < completed write operation key=F0QCVR9E08WE tag=5ZVF1PCH5YWT type=write size=926464 2024-09-13T00:00:39.893 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.894+0000 7fb8f2ce0640 1 > prepared operation key=H5EIP0E9CBWX tag=0JE2ZYBAZKLV type=del size=3498457 2024-09-13T00:00:39.898 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.898+0000 7fb8f2ce0640 1 < suggested remove operation key=H4M1TCN5W6XU tag=D2M7KYXZ6U1F type=del 2024-09-13T00:00:39.900 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.898+0000 7fb8f2ce0640 1 > prepared operation key=8MPUVD37EDN9 tag=V0MI0SUJSN0N type=del size=4044746 2024-09-13T00:00:39.901 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.902+0000 7fb8f2ce0640 1 < canceled part key=_multipart_4K76U69GZ9AE.2~IBHG228WQQJI.0 size=556667 2024-09-13T00:00:39.904 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.902+0000 7fb8f2ce0640 1 < canceled multipart upload key=4K76U69GZ9AE upload=IBHG228WQQJI tag=1O1BKKO0FRUQ type=write size=1605243 2024-09-13T00:00:39.908 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.906+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~3IWWHUUJL2RJ.0 size=1048576 2024-09-13T00:00:39.910 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.910+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~3IWWHUUJL2RJ.1 size=1048576 2024-09-13T00:00:39.911 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.910+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~3IWWHUUJL2RJ.2 size=394451 2024-09-13T00:00:39.912 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.910+0000 7fb8f2ce0640 1 > prepared multipart upload key=63FJDWBO2T9S upload=3IWWHUUJL2RJ tag=QGSUW5U9GONE type=write size=3540179 2024-09-13T00:00:39.917 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.918+0000 7fb8f2ce0640 1 < suggested update operation key=EUE3LT0PDIXV tag=AM35DIAIKIYL type=write size=1008201 2024-09-13T00:00:39.919 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.918+0000 7fb8f2ce0640 1 > prepared operation key=H5EIP0E9CBWX tag=SK140F3QBZPH type=del size=1645340 2024-09-13T00:00:39.921 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.922+0000 7fb8f2ce0640 1 < completed part key=_multipart_7OX5OJZ8GXU5.2~CZ6L0Q14BDRQ.0 size=1048576 2024-09-13T00:00:39.923 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.922+0000 7fb8f2ce0640 1 < completed part key=_multipart_7OX5OJZ8GXU5.2~CZ6L0Q14BDRQ.1 size=121319 2024-09-13T00:00:39.926 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.926+0000 7fb8f2ce0640 1 < completed multipart upload key=7OX5OJZ8GXU5 upload=CZ6L0Q14BDRQ tag=14CQOE8K48WF type=write size=2218471 2024-09-13T00:00:39.928 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.926+0000 7fb8f2ce0640 1 > prepared operation key=8Z6U7OOA0GBH tag=97JLIO5JC6LH type=write size=753269 2024-09-13T00:00:39.930 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.930+0000 7fb8f2ce0640 1 < completed delete operation key=O4MZ4UWNXME9 tag=B77BUMVFTZBQ type=del 2024-09-13T00:00:39.933 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.934+0000 7fb8f2ce0640 1 > prepared part key=_multipart_P8OYS5ZE1NX0.2~VQN0C9V5PDY4.0 size=1048576 2024-09-13T00:00:39.935 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.934+0000 7fb8f2ce0640 1 > prepared part key=_multipart_P8OYS5ZE1NX0.2~VQN0C9V5PDY4.1 size=1048576 2024-09-13T00:00:39.937 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.934+0000 7fb8f2ce0640 1 > prepared part key=_multipart_P8OYS5ZE1NX0.2~VQN0C9V5PDY4.2 size=669419 2024-09-13T00:00:39.937 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.934+0000 7fb8f2ce0640 1 > prepared multipart upload key=P8OYS5ZE1NX0 upload=VQN0C9V5PDY4 tag=2G99YOEUN8P6 type=write size=3815147 2024-09-13T00:00:39.938 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.938+0000 7fb8f2ce0640 1 < completed delete operation key=2AHYRZUDFHP4 tag=QUMR7RF1KHSJ type=del 2024-09-13T00:00:39.942 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.942+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~TOU6SNUBTVW1.0 size=1048576 2024-09-13T00:00:39.945 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.946+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~TOU6SNUBTVW1.1 size=1048576 2024-09-13T00:00:39.947 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.946+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~TOU6SNUBTVW1.2 size=748891 2024-09-13T00:00:39.947 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.946+0000 7fb8f2ce0640 1 > prepared multipart upload key=FX0NCGLK0KPJ upload=TOU6SNUBTVW1 tag=HLJRHDLOF8H2 type=write size=3894619 2024-09-13T00:00:39.949 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.950+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~NSFILYGRJ0H6.0 size=1048576 2024-09-13T00:00:39.951 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.950+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~NSFILYGRJ0H6.1 size=1048576 2024-09-13T00:00:39.952 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.950+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~NSFILYGRJ0H6.2 size=239070 2024-09-13T00:00:39.955 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.954+0000 7fb8f2ce0640 1 < completed multipart upload key=NQJQC8CEAC15 upload=NSFILYGRJ0H6 tag=GJWKFNZ84EX5 type=write size=3384798 2024-09-13T00:00:39.957 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.958+0000 7fb8f2ce0640 1 > prepared operation key=P8OYS5ZE1NX0 tag=LPOOIRFAA0BD type=del size=400834 2024-09-13T00:00:39.959 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.958+0000 7fb8f2ce0640 1 < canceled operation key=44EDYFT9VF8Y tag=1FM20QG6GMF0 type=write size=445196 2024-09-13T00:00:39.961 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.962+0000 7fb8f2ce0640 1 > prepared operation key=5GM88ERM89KT tag=WPA7MB6II4K4 type=del size=3806125 2024-09-13T00:00:39.962 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.962+0000 7fb8f2ce0640 1 < completed delete operation key=ZFHN2SNTZ5W7 tag=0FYRBQEXEOLG type=del 2024-09-13T00:00:39.966 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.966+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EJN6HR8I7XQF.2~9F6LI6I3GWEI.0 size=1048576 2024-09-13T00:00:39.967 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.966+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EJN6HR8I7XQF.2~9F6LI6I3GWEI.1 size=519990 2024-09-13T00:00:39.967 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.966+0000 7fb8f2ce0640 1 > prepared multipart upload key=EJN6HR8I7XQF upload=9F6LI6I3GWEI tag=QMUPU2IUREVR type=write size=2617142 2024-09-13T00:00:39.969 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.970+0000 7fb8f2ce0640 1 < completed delete operation key=NQJQC8CEAC15 tag=QYOWHVRHPG2B type=del 2024-09-13T00:00:39.972 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.970+0000 7fb8f2ce0640 1 > prepared part key=_multipart_J8NGDSF9XRTP.2~OCKQLZGQKZ3W.0 size=1048576 2024-09-13T00:00:39.974 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.974+0000 7fb8f2ce0640 1 > prepared part key=_multipart_J8NGDSF9XRTP.2~OCKQLZGQKZ3W.1 size=1048576 2024-09-13T00:00:39.978 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.978+0000 7fb8f2ce0640 1 > prepared part key=_multipart_J8NGDSF9XRTP.2~OCKQLZGQKZ3W.2 size=261785 2024-09-13T00:00:39.978 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.978+0000 7fb8f2ce0640 1 > prepared multipart upload key=J8NGDSF9XRTP upload=OCKQLZGQKZ3W tag=5GPRHWTVRQP2 type=write size=3407513 2024-09-13T00:00:39.981 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.982+0000 7fb8f2ce0640 1 < completed delete operation key=EJN6HR8I7XQF tag=K6652IT41YR2 type=del 2024-09-13T00:00:39.985 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.986+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~WOVT7BWOCDQ3.0 size=1048576 2024-09-13T00:00:39.987 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.986+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~WOVT7BWOCDQ3.1 size=1048576 2024-09-13T00:00:39.988 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.986+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~WOVT7BWOCDQ3.2 size=738793 2024-09-13T00:00:39.988 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.986+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZTK9AYBSLCOZ upload=WOVT7BWOCDQ3 tag=X53NTBOZC2MZ type=write size=3884521 2024-09-13T00:00:39.989 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.990+0000 7fb8f2ce0640 -1 < no bi entry to suggest for operation key=H4M1TCN5W6XU tag=CXFTU3UCHDNK type=del size=2205993: (2) No such file or directory 2024-09-13T00:00:39.991 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.990+0000 7fb8f2ce0640 1 > prepared operation key=NX26SI4F2ZMB tag=4AZI2GASGSZY type=del size=182944 2024-09-13T00:00:39.993 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.994+0000 7fb8f2ce0640 1 < completed delete operation key=8MPUVD37EDN9 tag=BD9FFM3F9YVP type=del 2024-09-13T00:00:39.995 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.994+0000 7fb8f2ce0640 1 > prepared operation key=1PX7DBO9M3YI tag=MNYLUH1239BD type=write size=252636 2024-09-13T00:00:39.998 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:39.998+0000 7fb8f2ce0640 -1 < failed to complete operation key=EUE3LT0PDIXV tag=R8VJFWX7RNCB type=del size=186717: (22) Invalid argument 2024-09-13T00:00:40.001 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.002+0000 7fb8f2ce0640 1 > prepared operation key=63FJDWBO2T9S tag=OSUPUO7MSWL1 type=write size=52503 2024-09-13T00:00:40.004 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.002+0000 7fb8f2ce0640 1 < completed part key=_multipart_2AHYRZUDFHP4.2~CVOTV69992EN.0 size=1048576 2024-09-13T00:00:40.006 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.006+0000 7fb8f2ce0640 1 < completed part key=_multipart_2AHYRZUDFHP4.2~CVOTV69992EN.1 size=649563 2024-09-13T00:00:40.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:39 smithi136 bash[76371]: audit 2024-09-13T00:00:38.770060+0000 mon.a (mon.0) 2071 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:00:40.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:39 smithi136 bash[75164]: audit 2024-09-13T00:00:38.770060+0000 mon.a (mon.0) 2071 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:00:40.010 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.010+0000 7fb8f2ce0640 1 < completed multipart upload key=2AHYRZUDFHP4 upload=CVOTV69992EN tag=0MXX729TJZ74 type=write size=2746715 2024-09-13T00:00:40.012 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.010+0000 7fb8f2ce0640 1 > prepared operation key=EJN6HR8I7XQF tag=OQY7Y8AJBST1 type=del size=1581261 2024-09-13T00:00:40.018 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.014+0000 7fb8f2ce0640 1 < completed delete operation key=H5EIP0E9CBWX tag=0JE2ZYBAZKLV type=del 2024-09-13T00:00:40.019 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.018+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~7V159TNWXJ9Q.0 size=1048576 2024-09-13T00:00:40.021 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.022+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~7V159TNWXJ9Q.1 size=1048576 2024-09-13T00:00:40.023 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.022+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~7V159TNWXJ9Q.2 size=210202 2024-09-13T00:00:40.023 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.022+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZTK9AYBSLCOZ upload=7V159TNWXJ9Q tag=3JZZG0GZG5OO type=write size=3355930 2024-09-13T00:00:40.027 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.026+0000 7fb8f2ce0640 1 < suggested remove operation key=8MPUVD37EDN9 tag=V0MI0SUJSN0N type=del 2024-09-13T00:00:40.029 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.030+0000 7fb8f2ce0640 1 > prepared operation key=8MPUVD37EDN9 tag=J8MOVLVU70G9 type=write size=285573 2024-09-13T00:00:40.029 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.030+0000 7fb8f2ce0640 1 < canceled part key=_multipart_63FJDWBO2T9S.2~3IWWHUUJL2RJ.0 size=1048576 2024-09-13T00:00:40.031 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.030+0000 7fb8f2ce0640 1 < completed part key=_multipart_63FJDWBO2T9S.2~3IWWHUUJL2RJ.1 size=1048576 2024-09-13T00:00:40.033 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.034+0000 7fb8f2ce0640 1 < completed part key=_multipart_63FJDWBO2T9S.2~3IWWHUUJL2RJ.2 size=394451 2024-09-13T00:00:40.036 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.034+0000 7fb8f2ce0640 1 < canceled multipart upload key=63FJDWBO2T9S upload=3IWWHUUJL2RJ tag=QGSUW5U9GONE type=write size=3540179 2024-09-13T00:00:40.038 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.038+0000 7fb8f2ce0640 1 > prepared operation key=EJN6HR8I7XQF tag=JPW617D0BCLF type=del size=1865635 2024-09-13T00:00:40.046 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.046+0000 7fb8f2ce0640 1 < suggested update operation key=H5EIP0E9CBWX tag=SK140F3QBZPH type=del size=1645340 2024-09-13T00:00:40.049 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.046+0000 7fb8f2ce0640 1 > prepared operation key=NX26SI4F2ZMB tag=PVF51T3VT4W0 type=del size=2679994 2024-09-13T00:00:40.052 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.050+0000 7fb8f2ce0640 1 < canceled operation key=8Z6U7OOA0GBH tag=97JLIO5JC6LH type=write size=753269 2024-09-13T00:00:40.054 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.054+0000 7fb8f2ce0640 1 > prepared operation key=ZFHN2SNTZ5W7 tag=3YFD59XJMEJI type=del size=2624150 2024-09-13T00:00:40.057 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.058+0000 7fb8f2ce0640 1 < completed part key=_multipart_P8OYS5ZE1NX0.2~VQN0C9V5PDY4.0 size=1048576 2024-09-13T00:00:40.059 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.058+0000 7fb8f2ce0640 1 < completed part key=_multipart_P8OYS5ZE1NX0.2~VQN0C9V5PDY4.1 size=1048576 2024-09-13T00:00:40.059 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.058+0000 7fb8f2ce0640 1 < canceled part key=_multipart_P8OYS5ZE1NX0.2~VQN0C9V5PDY4.2 size=669419 2024-09-13T00:00:40.063 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.062+0000 7fb8f2ce0640 1 < canceled multipart upload key=P8OYS5ZE1NX0 upload=VQN0C9V5PDY4 tag=2G99YOEUN8P6 type=write size=3815147 2024-09-13T00:00:40.067 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.066+0000 7fb8f2ce0640 1 > prepared operation key=N3Z5XIG3TZRQ tag=DPSY64LC6IP5 type=del size=2305699 2024-09-13T00:00:40.070 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.070+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~TOU6SNUBTVW1.0 size=1048576 2024-09-13T00:00:40.072 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.070+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~TOU6SNUBTVW1.1 size=1048576 2024-09-13T00:00:40.074 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.074+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~TOU6SNUBTVW1.2 size=748891 2024-09-13T00:00:40.079 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.078+0000 7fb8f2ce0640 1 < completed multipart upload key=FX0NCGLK0KPJ upload=TOU6SNUBTVW1 tag=HLJRHDLOF8H2 type=write size=3894619 2024-09-13T00:00:40.085 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.082+0000 7fb8f2ce0640 1 > prepared part key=_multipart_QMZREZ1QEDNJ.2~4UIOEUY08PEW.0 size=1048576 2024-09-13T00:00:40.087 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.086+0000 7fb8f2ce0640 1 > prepared part key=_multipart_QMZREZ1QEDNJ.2~4UIOEUY08PEW.1 size=694481 2024-09-13T00:00:40.087 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.086+0000 7fb8f2ce0640 1 > prepared multipart upload key=QMZREZ1QEDNJ upload=4UIOEUY08PEW tag=GV93CTPWR8BN type=write size=2791633 2024-09-13T00:00:40.090 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.090+0000 7fb8f2ce0640 1 < completed delete operation key=P8OYS5ZE1NX0 tag=LPOOIRFAA0BD type=del 2024-09-13T00:00:40.093 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.094+0000 7fb8f2ce0640 1 > prepared operation key=2TWD9YLMSYJI tag=5RXM07L8XG9C type=del size=3234357 2024-09-13T00:00:40.096 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.094+0000 7fb8f2ce0640 1 < canceled operation key=5GM88ERM89KT tag=WPA7MB6II4K4 type=del size=3806125 2024-09-13T00:00:40.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:39 smithi181 bash[67587]: audit 2024-09-13T00:00:38.770060+0000 mon.a (mon.0) 2071 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:00:40.098 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.098+0000 7fb8f2ce0640 1 > prepared operation key=EJN6HR8I7XQF tag=209VOJGA3WQ1 type=del size=3263711 2024-09-13T00:00:40.101 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.102+0000 7fb8f2ce0640 1 < completed part key=_multipart_EJN6HR8I7XQF.2~9F6LI6I3GWEI.0 size=1048576 2024-09-13T00:00:40.103 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.102+0000 7fb8f2ce0640 1 < completed part key=_multipart_EJN6HR8I7XQF.2~9F6LI6I3GWEI.1 size=519990 2024-09-13T00:00:40.108 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.106+0000 7fb8f2ce0640 1 < completed multipart upload key=EJN6HR8I7XQF upload=9F6LI6I3GWEI tag=QMUPU2IUREVR type=write size=2617142 2024-09-13T00:00:40.112 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.110+0000 7fb8f2ce0640 1 > prepared operation key=FX0NCGLK0KPJ tag=H3LE82U1FZJW type=write size=363443 2024-09-13T00:00:40.114 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.114+0000 7fb8f2ce0640 1 < completed part key=_multipart_J8NGDSF9XRTP.2~OCKQLZGQKZ3W.0 size=1048576 2024-09-13T00:00:40.116 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.114+0000 7fb8f2ce0640 1 < completed part key=_multipart_J8NGDSF9XRTP.2~OCKQLZGQKZ3W.1 size=1048576 2024-09-13T00:00:40.120 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.118+0000 7fb8f2ce0640 1 < completed part key=_multipart_J8NGDSF9XRTP.2~OCKQLZGQKZ3W.2 size=261785 2024-09-13T00:00:40.125 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.126+0000 7fb8f2ce0640 1 < completed multipart upload key=J8NGDSF9XRTP upload=OCKQLZGQKZ3W tag=5GPRHWTVRQP2 type=write size=3407513 2024-09-13T00:00:40.128 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.126+0000 7fb8f2ce0640 1 > prepared operation key=J8NGDSF9XRTP tag=1Y3RGS0Q2HTM type=del size=696905 2024-09-13T00:00:40.131 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.130+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~WOVT7BWOCDQ3.0 size=1048576 2024-09-13T00:00:40.133 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.134+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~WOVT7BWOCDQ3.1 size=1048576 2024-09-13T00:00:40.137 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.138+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~WOVT7BWOCDQ3.2 size=738793 2024-09-13T00:00:40.142 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.142+0000 7fb8f2ce0640 1 < completed multipart upload key=ZTK9AYBSLCOZ upload=WOVT7BWOCDQ3 tag=X53NTBOZC2MZ type=write size=3884521 2024-09-13T00:00:40.145 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.146+0000 7fb8f2ce0640 1 > prepared operation key=85YEW8N43FTA tag=JAPR11VMG9LB type=del size=3916876 2024-09-13T00:00:40.148 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.146+0000 7fb8f2ce0640 1 < completed delete operation key=NX26SI4F2ZMB tag=4AZI2GASGSZY type=del 2024-09-13T00:00:40.150 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.150+0000 7fb8f2ce0640 1 > prepared operation key=63FJDWBO2T9S tag=8MXHXV2V1MMF type=del size=3817773 2024-09-13T00:00:40.153 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.154+0000 7fb8f2ce0640 1 < completed write operation key=1PX7DBO9M3YI tag=MNYLUH1239BD type=write size=252636 2024-09-13T00:00:40.155 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.154+0000 7fb8f2ce0640 1 > prepared operation key=J8NGDSF9XRTP tag=4BTZFD9PBV8Z type=del size=1443224 2024-09-13T00:00:40.158 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.158+0000 7fb8f2ce0640 1 < completed write operation key=63FJDWBO2T9S tag=OSUPUO7MSWL1 type=write size=52503 2024-09-13T00:00:40.160 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.158+0000 7fb8f2ce0640 1 > prepared operation key=H5EIP0E9CBWX tag=PAG3DF8LQ7DV type=write size=34307 2024-09-13T00:00:40.164 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.162+0000 7fb8f2ce0640 1 < suggested remove operation key=EJN6HR8I7XQF tag=OQY7Y8AJBST1 type=del 2024-09-13T00:00:40.168 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.166+0000 7fb8f2ce0640 1 > prepared part key=_multipart_0WC6P8YNL0H2.2~FN7HX24DVFBZ.0 size=261019 2024-09-13T00:00:40.168 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.166+0000 7fb8f2ce0640 1 > prepared multipart upload key=0WC6P8YNL0H2 upload=FN7HX24DVFBZ tag=QZ63Z1G919J0 type=write size=1309595 2024-09-13T00:00:40.170 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.170+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~7V159TNWXJ9Q.0 size=1048576 2024-09-13T00:00:40.171 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.170+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~7V159TNWXJ9Q.1 size=1048576 2024-09-13T00:00:40.172 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.170+0000 7fb8f2ce0640 1 < canceled part key=_multipart_ZTK9AYBSLCOZ.2~7V159TNWXJ9Q.2 size=210202 2024-09-13T00:00:40.175 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.174+0000 7fb8f2ce0640 1 < canceled multipart upload key=ZTK9AYBSLCOZ upload=7V159TNWXJ9Q tag=3JZZG0GZG5OO type=write size=3355930 2024-09-13T00:00:40.177 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.178+0000 7fb8f2ce0640 1 > prepared operation key=PVYZQLGYT3XU tag=MI51P3WCQ7Z2 type=del size=565066 2024-09-13T00:00:40.179 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.178+0000 7fb8f2ce0640 1 < completed write operation key=8MPUVD37EDN9 tag=J8MOVLVU70G9 type=write size=285573 2024-09-13T00:00:40.181 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.182+0000 7fb8f2ce0640 1 > prepared operation key=1PX7DBO9M3YI tag=04VN9171IOXZ type=del size=1139828 2024-09-13T00:00:40.183 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.182+0000 7fb8f2ce0640 -1 < failed to complete operation key=EJN6HR8I7XQF tag=JPW617D0BCLF type=del size=1865635: (22) Invalid argument 2024-09-13T00:00:40.188 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.186+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~4TLB6IZMCXCN.0 size=23299 2024-09-13T00:00:40.188 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.186+0000 7fb8f2ce0640 1 > prepared multipart upload key=7OX5OJZ8GXU5 upload=4TLB6IZMCXCN tag=53MJ8KZ52N9V type=write size=1071875 2024-09-13T00:00:40.190 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.190+0000 7fb8f2ce0640 1 < completed delete operation key=NX26SI4F2ZMB tag=PVF51T3VT4W0 type=del 2024-09-13T00:00:40.193 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.194+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~LC2OZ2ZGANZA.0 size=846889 2024-09-13T00:00:40.194 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.194+0000 7fb8f2ce0640 1 > prepared multipart upload key=NQJQC8CEAC15 upload=LC2OZ2ZGANZA tag=8K8CEPAEA0JA type=write size=1895465 2024-09-13T00:00:40.195 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.194+0000 7fb8f2ce0640 1 < completed delete operation key=ZFHN2SNTZ5W7 tag=3YFD59XJMEJI type=del 2024-09-13T00:00:40.197 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.198+0000 7fb8f2ce0640 1 > prepared operation key=PVYZQLGYT3XU tag=7ZMJBBD3BSY5 type=del size=2677441 2024-09-13T00:00:40.199 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.198+0000 7fb8f2ce0640 1 < completed delete operation key=N3Z5XIG3TZRQ tag=DPSY64LC6IP5 type=del 2024-09-13T00:00:40.201 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.202+0000 7fb8f2ce0640 1 > prepared operation key=85YEW8N43FTA tag=83BI0I90R0NC type=del size=1077614 2024-09-13T00:00:40.203 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.202+0000 7fb8f2ce0640 1 < completed part key=_multipart_QMZREZ1QEDNJ.2~4UIOEUY08PEW.0 size=1048576 2024-09-13T00:00:40.205 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.206+0000 7fb8f2ce0640 1 < completed part key=_multipart_QMZREZ1QEDNJ.2~4UIOEUY08PEW.1 size=694481 2024-09-13T00:00:40.209 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.210+0000 7fb8f2ce0640 1 < completed multipart upload key=QMZREZ1QEDNJ upload=4UIOEUY08PEW tag=GV93CTPWR8BN type=write size=2791633 2024-09-13T00:00:40.213 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.214+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~YBL334H02XKQ.0 size=1048576 2024-09-13T00:00:40.214 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.214+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~YBL334H02XKQ.1 size=1048576 2024-09-13T00:00:40.216 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.214+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~YBL334H02XKQ.2 size=1044776 2024-09-13T00:00:40.216 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.214+0000 7fb8f2ce0640 1 > prepared multipart upload key=1PX7DBO9M3YI upload=YBL334H02XKQ tag=CF355IV5XFZ6 type=write size=4190504 2024-09-13T00:00:40.218 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.218+0000 7fb8f2ce0640 1 < completed delete operation key=2TWD9YLMSYJI tag=5RXM07L8XG9C type=del 2024-09-13T00:00:40.222 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.222+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~O2YLZ5697GNM.0 size=312185 2024-09-13T00:00:40.222 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.222+0000 7fb8f2ce0640 1 > prepared multipart upload key=8Z6U7OOA0GBH upload=O2YLZ5697GNM tag=UL74XRMEJXEN type=write size=1360761 2024-09-13T00:00:40.224 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.222+0000 7fb8f2ce0640 -1 < failed to complete operation key=EJN6HR8I7XQF tag=209VOJGA3WQ1 type=del size=3263711: (22) Invalid argument 2024-09-13T00:00:40.226 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.226+0000 7fb8f2ce0640 1 > prepared operation key=0WC6P8YNL0H2 tag=EGXDOVWVS1ET type=del size=3516119 2024-09-13T00:00:40.228 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.226+0000 7fb8f2ce0640 1 < completed write operation key=FX0NCGLK0KPJ tag=H3LE82U1FZJW type=write size=363443 2024-09-13T00:00:40.230 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.230+0000 7fb8f2ce0640 1 > prepared operation key=2TWD9YLMSYJI tag=2HWDOODUXNT0 type=del size=307642 2024-09-13T00:00:40.233 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.234+0000 7fb8f2ce0640 1 < completed delete operation key=J8NGDSF9XRTP tag=1Y3RGS0Q2HTM type=del 2024-09-13T00:00:40.235 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.234+0000 7fb8f2ce0640 1 > prepared operation key=PVYZQLGYT3XU tag=X5VTDT9122W5 type=del size=3870787 2024-09-13T00:00:40.237 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.238+0000 7fb8f2ce0640 1 < completed delete operation key=85YEW8N43FTA tag=JAPR11VMG9LB type=del 2024-09-13T00:00:40.239 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.238+0000 7fb8f2ce0640 1 > prepared operation key=2TWD9YLMSYJI tag=HUQX8QA0V4DG type=del size=1973133 2024-09-13T00:00:40.241 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.242+0000 7fb8f2ce0640 1 < canceled operation key=63FJDWBO2T9S tag=8MXHXV2V1MMF type=del size=3817773 2024-09-13T00:00:40.245 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.246+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~OGZ00SHDE27P.0 size=1048576 2024-09-13T00:00:40.247 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.246+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~OGZ00SHDE27P.1 size=673917 2024-09-13T00:00:40.247 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.246+0000 7fb8f2ce0640 1 > prepared multipart upload key=8MPUVD37EDN9 upload=OGZ00SHDE27P tag=XBYD95691POO type=write size=2771069 2024-09-13T00:00:40.251 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.250+0000 7fb8f2ce0640 1 < suggested remove operation key=J8NGDSF9XRTP tag=4BTZFD9PBV8Z type=del 2024-09-13T00:00:40.253 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.254+0000 7fb8f2ce0640 1 > prepared operation key=44EDYFT9VF8Y tag=DX4X3KWLVART type=del size=3900512 2024-09-13T00:00:40.257 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.258+0000 7fb8f2ce0640 1 < suggested remove operation key=H5EIP0E9CBWX tag=PAG3DF8LQ7DV type=write 2024-09-13T00:00:40.259 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.258+0000 7fb8f2ce0640 1 > prepared operation key=7OX5OJZ8GXU5 tag=R0QT4CX1PF6A type=del size=1686047 2024-09-13T00:00:40.261 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.262+0000 7fb8f2ce0640 1 < completed part key=_multipart_0WC6P8YNL0H2.2~FN7HX24DVFBZ.0 size=261019 2024-09-13T00:00:40.265 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.266+0000 7fb8f2ce0640 1 < completed multipart upload key=0WC6P8YNL0H2 upload=FN7HX24DVFBZ tag=QZ63Z1G919J0 type=write size=1309595 2024-09-13T00:00:40.270 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.270+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H5EIP0E9CBWX.2~0L6NU02FPNZU.0 size=1048576 2024-09-13T00:00:40.272 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.270+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H5EIP0E9CBWX.2~0L6NU02FPNZU.1 size=1048576 2024-09-13T00:00:40.274 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.274+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H5EIP0E9CBWX.2~0L6NU02FPNZU.2 size=751927 2024-09-13T00:00:40.274 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.274+0000 7fb8f2ce0640 1 > prepared multipart upload key=H5EIP0E9CBWX upload=0L6NU02FPNZU tag=KIWJMU4Z3577 type=write size=3897655 2024-09-13T00:00:40.277 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.274+0000 7fb8f2ce0640 1 < completed delete operation key=PVYZQLGYT3XU tag=MI51P3WCQ7Z2 type=del 2024-09-13T00:00:40.279 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.278+0000 7fb8f2ce0640 1 > prepared operation key=7OX5OJZ8GXU5 tag=7HFCNY04I10S type=write size=810739 2024-09-13T00:00:40.282 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.282+0000 7fb8f2ce0640 1 < completed delete operation key=1PX7DBO9M3YI tag=04VN9171IOXZ type=del 2024-09-13T00:00:40.285 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.286+0000 7fb8f2ce0640 1 > prepared operation key=F0QCVR9E08WE tag=Z4KYY5AL82Q6 type=del size=2379953 2024-09-13T00:00:40.285 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.286+0000 7fb8f2ce0640 1 < canceled part key=_multipart_7OX5OJZ8GXU5.2~4TLB6IZMCXCN.0 size=23299 2024-09-13T00:00:40.290 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.290+0000 7fb8f2ce0640 1 < canceled multipart upload key=7OX5OJZ8GXU5 upload=4TLB6IZMCXCN tag=53MJ8KZ52N9V type=write size=1071875 2024-09-13T00:00:40.292 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.290+0000 7fb8f2ce0640 1 > prepared operation key=NQJQC8CEAC15 tag=5SKE2ATUON22 type=del size=1157099 2024-09-13T00:00:40.294 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.294+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~LC2OZ2ZGANZA.0 size=846889 2024-09-13T00:00:40.299 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.298+0000 7fb8f2ce0640 1 < completed multipart upload key=NQJQC8CEAC15 upload=LC2OZ2ZGANZA tag=8K8CEPAEA0JA type=write size=1895465 2024-09-13T00:00:40.301 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.302+0000 7fb8f2ce0640 1 > prepared operation key=EJN6HR8I7XQF tag=2LFKLDJPY0L1 type=del size=1715938 2024-09-13T00:00:40.303 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.302+0000 7fb8f2ce0640 1 < completed delete operation key=PVYZQLGYT3XU tag=7ZMJBBD3BSY5 type=del 2024-09-13T00:00:40.308 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.306+0000 7fb8f2ce0640 1 > prepared part key=_multipart_F0QCVR9E08WE.2~ORTV5JY6GR7T.0 size=1048576 2024-09-13T00:00:40.310 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.310+0000 7fb8f2ce0640 1 > prepared part key=_multipart_F0QCVR9E08WE.2~ORTV5JY6GR7T.1 size=46604 2024-09-13T00:00:40.310 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.310+0000 7fb8f2ce0640 1 > prepared multipart upload key=F0QCVR9E08WE upload=ORTV5JY6GR7T tag=0O8A1MZ4MQAZ type=write size=2143756 2024-09-13T00:00:40.312 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.310+0000 7fb8f2ce0640 1 < completed delete operation key=85YEW8N43FTA tag=83BI0I90R0NC type=del 2024-09-13T00:00:40.317 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.318+0000 7fb8f2ce0640 1 > prepared part key=_multipart_F0QCVR9E08WE.2~3Q1J7V764NH9.0 size=1048576 2024-09-13T00:00:40.318 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.318+0000 7fb8f2ce0640 1 > prepared part key=_multipart_F0QCVR9E08WE.2~3Q1J7V764NH9.1 size=916067 2024-09-13T00:00:40.318 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.318+0000 7fb8f2ce0640 1 > prepared multipart upload key=F0QCVR9E08WE upload=3Q1J7V764NH9 tag=U1EM18ZZFQHF type=write size=3013219 2024-09-13T00:00:40.320 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.318+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~YBL334H02XKQ.0 size=1048576 2024-09-13T00:00:40.322 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.322+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~YBL334H02XKQ.1 size=1048576 2024-09-13T00:00:40.324 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.322+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~YBL334H02XKQ.2 size=1044776 2024-09-13T00:00:40.328 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.326+0000 7fb8f2ce0640 1 < completed multipart upload key=1PX7DBO9M3YI upload=YBL334H02XKQ tag=CF355IV5XFZ6 type=write size=4190504 2024-09-13T00:00:40.329 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.330+0000 7fb8f2ce0640 1 > prepared operation key=4K76U69GZ9AE tag=BEY8TSIYG456 type=write size=53073 2024-09-13T00:00:40.331 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.330+0000 7fb8f2ce0640 1 < completed part key=_multipart_8Z6U7OOA0GBH.2~O2YLZ5697GNM.0 size=312185 2024-09-13T00:00:40.335 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.334+0000 7fb8f2ce0640 1 < completed multipart upload key=8Z6U7OOA0GBH upload=O2YLZ5697GNM tag=UL74XRMEJXEN type=write size=1360761 2024-09-13T00:00:40.337 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.338+0000 7fb8f2ce0640 1 > prepared operation key=P8OYS5ZE1NX0 tag=UBQDZDHZHB8S type=del size=93056 2024-09-13T00:00:40.341 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.342+0000 7fb8f2ce0640 1 < suggested update operation key=0WC6P8YNL0H2 tag=EGXDOVWVS1ET type=del size=3516119 2024-09-13T00:00:40.346 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.346+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~8IBLPTSPHWYB.0 size=1048576 2024-09-13T00:00:40.348 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.346+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~8IBLPTSPHWYB.1 size=1048576 2024-09-13T00:00:40.350 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.350+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~8IBLPTSPHWYB.2 size=186200 2024-09-13T00:00:40.350 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.350+0000 7fb8f2ce0640 1 > prepared multipart upload key=PVYZQLGYT3XU upload=8IBLPTSPHWYB tag=UWELOIEOHUE7 type=write size=3331928 2024-09-13T00:00:40.355 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.354+0000 7fb8f2ce0640 1 < suggested remove operation key=2TWD9YLMSYJI tag=2HWDOODUXNT0 type=del 2024-09-13T00:00:40.360 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.358+0000 7fb8f2ce0640 1 > prepared part key=_multipart_N3Z5XIG3TZRQ.2~2UDLIE02LBKS.0 size=1048576 2024-09-13T00:00:40.362 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.362+0000 7fb8f2ce0640 1 > prepared part key=_multipart_N3Z5XIG3TZRQ.2~2UDLIE02LBKS.1 size=1048576 2024-09-13T00:00:40.364 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.362+0000 7fb8f2ce0640 1 > prepared part key=_multipart_N3Z5XIG3TZRQ.2~2UDLIE02LBKS.2 size=59396 2024-09-13T00:00:40.365 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.362+0000 7fb8f2ce0640 1 > prepared multipart upload key=N3Z5XIG3TZRQ upload=2UDLIE02LBKS tag=FO21ZQ1OLN2L type=write size=3205124 2024-09-13T00:00:40.370 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.370+0000 7fb8f2ce0640 1 < suggested update operation key=PVYZQLGYT3XU tag=X5VTDT9122W5 type=del size=3870787 2024-09-13T00:00:40.373 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.374+0000 7fb8f2ce0640 1 > prepared operation key=F0QCVR9E08WE tag=SHHGYD497ET8 type=write size=1040509 2024-09-13T00:00:40.374 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.374+0000 7fb8f2ce0640 -1 < no bi entry to suggest for operation key=2TWD9YLMSYJI tag=HUQX8QA0V4DG type=del size=1973133: (2) No such file or directory 2024-09-13T00:00:40.376 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.374+0000 7fb8f2ce0640 1 > prepared operation key=H9AU4HD247YU tag=QMYPY42RT3KG type=write size=504106 2024-09-13T00:00:40.377 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.374+0000 7fb8f2ce0640 1 < canceled part key=_multipart_8MPUVD37EDN9.2~OGZ00SHDE27P.0 size=1048576 2024-09-13T00:00:40.379 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.378+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~OGZ00SHDE27P.1 size=673917 2024-09-13T00:00:40.384 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.382+0000 7fb8f2ce0640 1 < canceled multipart upload key=8MPUVD37EDN9 upload=OGZ00SHDE27P tag=XBYD95691POO type=write size=2771069 2024-09-13T00:00:40.387 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.386+0000 7fb8f2ce0640 1 > prepared operation key=EUE3LT0PDIXV tag=YVIU5OB99E28 type=del size=1601752 2024-09-13T00:00:40.390 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.390+0000 7fb8f2ce0640 1 < completed delete operation key=44EDYFT9VF8Y tag=DX4X3KWLVART type=del 2024-09-13T00:00:40.396 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.394+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~X0KDZ70LVQEE.0 size=1048576 2024-09-13T00:00:40.400 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.398+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~X0KDZ70LVQEE.1 size=683473 2024-09-13T00:00:40.400 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.398+0000 7fb8f2ce0640 1 > prepared multipart upload key=FX0NCGLK0KPJ upload=X0KDZ70LVQEE tag=505JV1CQVE47 type=write size=2780625 2024-09-13T00:00:40.403 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.402+0000 7fb8f2ce0640 1 < completed delete operation key=7OX5OJZ8GXU5 tag=R0QT4CX1PF6A type=del 2024-09-13T00:00:40.406 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.406+0000 7fb8f2ce0640 1 > prepared operation key=NX26SI4F2ZMB tag=COON1JJXZY9U type=del size=2796259 2024-09-13T00:00:40.407 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.406+0000 7fb8f2ce0640 1 < canceled part key=_multipart_H5EIP0E9CBWX.2~0L6NU02FPNZU.0 size=1048576 2024-09-13T00:00:40.409 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.410+0000 7fb8f2ce0640 1 < completed part key=_multipart_H5EIP0E9CBWX.2~0L6NU02FPNZU.1 size=1048576 2024-09-13T00:00:40.410 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.410+0000 7fb8f2ce0640 1 < canceled part key=_multipart_H5EIP0E9CBWX.2~0L6NU02FPNZU.2 size=751927 2024-09-13T00:00:40.415 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.414+0000 7fb8f2ce0640 1 < canceled multipart upload key=H5EIP0E9CBWX upload=0L6NU02FPNZU tag=KIWJMU4Z3577 type=write size=3897655 2024-09-13T00:00:40.422 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.422+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~WKRQUPTA51LE.0 size=1048576 2024-09-13T00:00:40.425 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.426+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~WKRQUPTA51LE.1 size=1048576 2024-09-13T00:00:40.428 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.426+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~WKRQUPTA51LE.2 size=774673 2024-09-13T00:00:40.428 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.426+0000 7fb8f2ce0640 1 > prepared multipart upload key=W8XZ26311RJX upload=WKRQUPTA51LE tag=A8XXTBPNKZ9V type=write size=3920401 2024-09-13T00:00:40.432 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.430+0000 7fb8f2ce0640 1 < completed write operation key=7OX5OJZ8GXU5 tag=7HFCNY04I10S type=write size=810739 2024-09-13T00:00:40.435 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.434+0000 7fb8f2ce0640 1 > prepared operation key=EJN6HR8I7XQF tag=H9VH0OC3A4PM type=del size=1393811 2024-09-13T00:00:40.443 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.442+0000 7fb8f2ce0640 1 < suggested remove operation key=F0QCVR9E08WE tag=Z4KYY5AL82Q6 type=del 2024-09-13T00:00:40.450 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.450+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~2OB477DNNCHW.0 size=1048576 2024-09-13T00:00:40.453 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.454+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~2OB477DNNCHW.1 size=1048576 2024-09-13T00:00:40.456 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.454+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~2OB477DNNCHW.2 size=564715 2024-09-13T00:00:40.456 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.454+0000 7fb8f2ce0640 1 > prepared multipart upload key=63FJDWBO2T9S upload=2OB477DNNCHW tag=2BRBGX9D63GW type=write size=3710443 2024-09-13T00:00:40.460 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.458+0000 7fb8f2ce0640 1 < canceled operation key=NQJQC8CEAC15 tag=5SKE2ATUON22 type=del size=1157099 2024-09-13T00:00:40.464 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.462+0000 7fb8f2ce0640 1 > prepared operation key=70F7D20HI504 tag=0SVDNTVP7ZEW type=del size=1001705 2024-09-13T00:00:40.468 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.466+0000 7fb8f2ce0640 1 < canceled operation key=EJN6HR8I7XQF tag=2LFKLDJPY0L1 type=del size=1715938 2024-09-13T00:00:40.471 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.470+0000 7fb8f2ce0640 1 > prepared operation key=2TWD9YLMSYJI tag=WMDVC30OPEQW type=del size=3857135 2024-09-13T00:00:40.475 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.474+0000 7fb8f2ce0640 1 < completed part key=_multipart_F0QCVR9E08WE.2~ORTV5JY6GR7T.0 size=1048576 2024-09-13T00:00:40.478 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.478+0000 7fb8f2ce0640 1 < completed part key=_multipart_F0QCVR9E08WE.2~ORTV5JY6GR7T.1 size=46604 2024-09-13T00:00:40.485 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.482+0000 7fb8f2ce0640 1 < completed multipart upload key=F0QCVR9E08WE upload=ORTV5JY6GR7T tag=0O8A1MZ4MQAZ type=write size=2143756 2024-09-13T00:00:40.488 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.486+0000 7fb8f2ce0640 1 > prepared operation key=PVYZQLGYT3XU tag=1A4C805PAMGX type=del size=3474126 2024-09-13T00:00:40.492 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.490+0000 7fb8f2ce0640 1 < completed part key=_multipart_F0QCVR9E08WE.2~3Q1J7V764NH9.0 size=1048576 2024-09-13T00:00:40.495 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.494+0000 7fb8f2ce0640 1 < completed part key=_multipart_F0QCVR9E08WE.2~3Q1J7V764NH9.1 size=916067 2024-09-13T00:00:40.501 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.502+0000 7fb8f2ce0640 1 < completed multipart upload key=F0QCVR9E08WE upload=3Q1J7V764NH9 tag=U1EM18ZZFQHF type=write size=3013219 2024-09-13T00:00:40.504 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.502+0000 7fb8f2ce0640 1 > prepared operation key=ZFHN2SNTZ5W7 tag=EMAFH2F71UX3 type=write size=641086 2024-09-13T00:00:40.508 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.506+0000 7fb8f2ce0640 1 < completed write operation key=4K76U69GZ9AE tag=BEY8TSIYG456 type=write size=53073 2024-09-13T00:00:40.511 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.510+0000 7fb8f2ce0640 1 > prepared operation key=1HX06CCZJ1DD tag=S84K3O1B0XCW type=del size=81033 2024-09-13T00:00:40.514 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.514+0000 7fb8f2ce0640 1 < canceled operation key=P8OYS5ZE1NX0 tag=UBQDZDHZHB8S type=del size=93056 2024-09-13T00:00:40.518 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.518+0000 7fb8f2ce0640 1 > prepared operation key=PVYZQLGYT3XU tag=IPCFSX7TE3JQ type=write size=440118 2024-09-13T00:00:40.521 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.522+0000 7fb8f2ce0640 1 < completed part key=_multipart_PVYZQLGYT3XU.2~8IBLPTSPHWYB.0 size=1048576 2024-09-13T00:00:40.524 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.522+0000 7fb8f2ce0640 1 < completed part key=_multipart_PVYZQLGYT3XU.2~8IBLPTSPHWYB.1 size=1048576 2024-09-13T00:00:40.527 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.526+0000 7fb8f2ce0640 1 < completed part key=_multipart_PVYZQLGYT3XU.2~8IBLPTSPHWYB.2 size=186200 2024-09-13T00:00:40.533 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.534+0000 7fb8f2ce0640 1 < completed multipart upload key=PVYZQLGYT3XU upload=8IBLPTSPHWYB tag=UWELOIEOHUE7 type=write size=3331928 2024-09-13T00:00:40.536 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.534+0000 7fb8f2ce0640 1 > prepared operation key=63FJDWBO2T9S tag=ZZXYMQIWR09N type=del size=2362175 2024-09-13T00:00:40.539 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.538+0000 7fb8f2ce0640 1 < completed part key=_multipart_N3Z5XIG3TZRQ.2~2UDLIE02LBKS.0 size=1048576 2024-09-13T00:00:40.542 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.542+0000 7fb8f2ce0640 1 < completed part key=_multipart_N3Z5XIG3TZRQ.2~2UDLIE02LBKS.1 size=1048576 2024-09-13T00:00:40.545 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.546+0000 7fb8f2ce0640 1 < completed part key=_multipart_N3Z5XIG3TZRQ.2~2UDLIE02LBKS.2 size=59396 2024-09-13T00:00:40.551 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.550+0000 7fb8f2ce0640 1 < completed multipart upload key=N3Z5XIG3TZRQ upload=2UDLIE02LBKS tag=FO21ZQ1OLN2L type=write size=3205124 2024-09-13T00:00:40.554 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.554+0000 7fb8f2ce0640 1 > prepared operation key=EUE3LT0PDIXV tag=0646MMQH3QOP type=del size=1510115 2024-09-13T00:00:40.559 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.558+0000 7fb8f2ce0640 -1 < failed to complete operation key=F0QCVR9E08WE tag=SHHGYD497ET8 type=write size=1040509: (22) Invalid argument 2024-09-13T00:00:40.565 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.566+0000 7fb8f2ce0640 1 > prepared part key=_multipart_N3Z5XIG3TZRQ.2~Q093DN7RWYRT.0 size=1048576 2024-09-13T00:00:40.569 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.570+0000 7fb8f2ce0640 1 > prepared part key=_multipart_N3Z5XIG3TZRQ.2~Q093DN7RWYRT.1 size=254283 2024-09-13T00:00:40.569 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.570+0000 7fb8f2ce0640 1 > prepared multipart upload key=N3Z5XIG3TZRQ upload=Q093DN7RWYRT tag=43BF230BFUOR type=write size=2351435 2024-09-13T00:00:40.573 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.574+0000 7fb8f2ce0640 1 < canceled operation key=H9AU4HD247YU tag=QMYPY42RT3KG type=write size=504106 2024-09-13T00:00:40.579 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.578+0000 7fb8f2ce0640 1 > prepared part key=_multipart_5GM88ERM89KT.2~1O1SHLGP68NB.0 size=628281 2024-09-13T00:00:40.579 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.578+0000 7fb8f2ce0640 1 > prepared multipart upload key=5GM88ERM89KT upload=1O1SHLGP68NB tag=ENFALVF6HD61 type=write size=1676857 2024-09-13T00:00:40.585 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.586+0000 7fb8f2ce0640 1 < suggested remove operation key=EUE3LT0PDIXV tag=YVIU5OB99E28 type=del 2024-09-13T00:00:40.591 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.590+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H4M1TCN5W6XU.2~VBEF6HQVEOT9.0 size=1048576 2024-09-13T00:00:40.594 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.594+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H4M1TCN5W6XU.2~VBEF6HQVEOT9.1 size=1025373 2024-09-13T00:00:40.594 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.594+0000 7fb8f2ce0640 1 > prepared multipart upload key=H4M1TCN5W6XU upload=VBEF6HQVEOT9 tag=MP82HAETZR43 type=write size=3122525 2024-09-13T00:00:40.597 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.598+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~X0KDZ70LVQEE.0 size=1048576 2024-09-13T00:00:40.600 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.598+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~X0KDZ70LVQEE.1 size=683473 2024-09-13T00:00:40.606 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.606+0000 7fb8f2ce0640 1 < completed multipart upload key=FX0NCGLK0KPJ upload=X0KDZ70LVQEE tag=505JV1CQVE47 type=write size=2780625 2024-09-13T00:00:40.612 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.610+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~IIPVG2N81MUU.0 size=1048576 2024-09-13T00:00:40.615 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.614+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~IIPVG2N81MUU.1 size=36089 2024-09-13T00:00:40.615 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.614+0000 7fb8f2ce0640 1 > prepared multipart upload key=NQJQC8CEAC15 upload=IIPVG2N81MUU tag=7MMLOS0NB4UC type=write size=2133241 2024-09-13T00:00:40.618 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.618+0000 7fb8f2ce0640 1 < completed delete operation key=NX26SI4F2ZMB tag=COON1JJXZY9U type=del 2024-09-13T00:00:40.625 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.626+0000 7fb8f2ce0640 1 > prepared part key=_multipart_85YEW8N43FTA.2~N5TPB3XGK2DW.0 size=1048576 2024-09-13T00:00:40.628 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.626+0000 7fb8f2ce0640 1 > prepared part key=_multipart_85YEW8N43FTA.2~N5TPB3XGK2DW.1 size=686114 2024-09-13T00:00:40.629 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.626+0000 7fb8f2ce0640 1 > prepared multipart upload key=85YEW8N43FTA upload=N5TPB3XGK2DW tag=JV1NO6045OOD type=write size=2783266 2024-09-13T00:00:40.629 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.626+0000 7fb8f2ce0640 1 < canceled part key=_multipart_W8XZ26311RJX.2~WKRQUPTA51LE.0 size=1048576 2024-09-13T00:00:40.629 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.626+0000 7fb8f2ce0640 1 < canceled part key=_multipart_W8XZ26311RJX.2~WKRQUPTA51LE.1 size=1048576 2024-09-13T00:00:40.631 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.630+0000 7fb8f2ce0640 1 < completed part key=_multipart_W8XZ26311RJX.2~WKRQUPTA51LE.2 size=774673 2024-09-13T00:00:40.637 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.638+0000 7fb8f2ce0640 1 < canceled multipart upload key=W8XZ26311RJX upload=WKRQUPTA51LE tag=A8XXTBPNKZ9V type=write size=3920401 2024-09-13T00:00:40.643 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.642+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~E114XBPN0PKR.0 size=1048576 2024-09-13T00:00:40.646 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.646+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~E114XBPN0PKR.1 size=293137 2024-09-13T00:00:40.646 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.646+0000 7fb8f2ce0640 1 > prepared multipart upload key=FX0NCGLK0KPJ upload=E114XBPN0PKR tag=L83VLEJRNOEX type=write size=2390289 2024-09-13T00:00:40.648 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.646+0000 7fb8f2ce0640 1 < completed delete operation key=EJN6HR8I7XQF tag=H9VH0OC3A4PM type=del 2024-09-13T00:00:40.651 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.650+0000 7fb8f2ce0640 1 > prepared operation key=2AHYRZUDFHP4 tag=DU5Q1Y6UNKHR type=del size=2933583 2024-09-13T00:00:40.655 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.654+0000 7fb8f2ce0640 1 < completed part key=_multipart_63FJDWBO2T9S.2~2OB477DNNCHW.0 size=1048576 2024-09-13T00:00:40.657 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.658+0000 7fb8f2ce0640 1 < completed part key=_multipart_63FJDWBO2T9S.2~2OB477DNNCHW.1 size=1048576 2024-09-13T00:00:40.660 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.658+0000 7fb8f2ce0640 1 < completed part key=_multipart_63FJDWBO2T9S.2~2OB477DNNCHW.2 size=564715 2024-09-13T00:00:40.666 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.666+0000 7fb8f2ce0640 1 < completed multipart upload key=63FJDWBO2T9S upload=2OB477DNNCHW tag=2BRBGX9D63GW type=write size=3710443 2024-09-13T00:00:40.668 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.666+0000 7fb8f2ce0640 1 > prepared operation key=O4MZ4UWNXME9 tag=QY5R9CY2PPLU type=del size=2951744 2024-09-13T00:00:40.673 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.674+0000 7fb8f2ce0640 1 < completed delete operation key=70F7D20HI504 tag=0SVDNTVP7ZEW type=del 2024-09-13T00:00:40.675 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.674+0000 7fb8f2ce0640 1 > prepared operation key=70F7D20HI504 tag=7CC6NBU7PJOO type=del size=3399907 2024-09-13T00:00:40.680 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.678+0000 7fb8f2ce0640 1 < suggested update operation key=2TWD9YLMSYJI tag=WMDVC30OPEQW type=del size=3857135 2024-09-13T00:00:40.683 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.682+0000 7fb8f2ce0640 1 > prepared operation key=2AHYRZUDFHP4 tag=G4F5AN0IGMVF type=del size=3245091 2024-09-13T00:00:40.685 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.686+0000 7fb8f2ce0640 1 < canceled operation key=PVYZQLGYT3XU tag=1A4C805PAMGX type=del size=3474126 2024-09-13T00:00:40.688 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.686+0000 7fb8f2ce0640 1 > prepared operation key=8Z6U7OOA0GBH tag=3AJ15OCODI8P type=del size=2235970 2024-09-13T00:00:40.692 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.690+0000 7fb8f2ce0640 1 < suggested remove operation key=ZFHN2SNTZ5W7 tag=EMAFH2F71UX3 type=write 2024-09-13T00:00:40.697 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.698+0000 7fb8f2ce0640 1 > prepared part key=_multipart_N3Z5XIG3TZRQ.2~4SNT2S1YHIUP.0 size=462152 2024-09-13T00:00:40.697 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.698+0000 7fb8f2ce0640 1 > prepared multipart upload key=N3Z5XIG3TZRQ upload=4SNT2S1YHIUP tag=SDFX5V2B9TRN type=write size=1510728 2024-09-13T00:00:40.699 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.698+0000 7fb8f2ce0640 1 < completed delete operation key=1HX06CCZJ1DD tag=S84K3O1B0XCW type=del 2024-09-13T00:00:40.702 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.702+0000 7fb8f2ce0640 1 > prepared operation key=PVYZQLGYT3XU tag=TG1GAK62FUOE type=del size=2554482 2024-09-13T00:00:40.704 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.702+0000 7fb8f2ce0640 1 < canceled operation key=PVYZQLGYT3XU tag=IPCFSX7TE3JQ type=write size=440118 2024-09-13T00:00:40.707 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.706+0000 7fb8f2ce0640 1 > prepared operation key=H4M1TCN5W6XU tag=TEC0UAZUS48D type=del size=1055007 2024-09-13T00:00:40.715 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.714+0000 7fb8f2ce0640 1 < suggested remove operation key=63FJDWBO2T9S tag=ZZXYMQIWR09N type=del 2024-09-13T00:00:40.721 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.718+0000 7fb8f2ce0640 1 > prepared part key=_multipart_J8NGDSF9XRTP.2~0DEYM1A5G8E2.0 size=1048576 2024-09-13T00:00:40.722 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.722+0000 7fb8f2ce0640 1 > prepared part key=_multipart_J8NGDSF9XRTP.2~0DEYM1A5G8E2.1 size=266417 2024-09-13T00:00:40.723 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.722+0000 7fb8f2ce0640 1 > prepared multipart upload key=J8NGDSF9XRTP upload=0DEYM1A5G8E2 tag=JWAENEX5FQOS type=write size=2363569 2024-09-13T00:00:40.726 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.726+0000 7fb8f2ce0640 -1 < failed to complete operation key=EUE3LT0PDIXV tag=0646MMQH3QOP type=del size=1510115: (22) Invalid argument 2024-09-13T00:00:40.731 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.730+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2TWD9YLMSYJI.2~41W5EOG8BH0Y.0 size=1048576 2024-09-13T00:00:40.734 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.734+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2TWD9YLMSYJI.2~41W5EOG8BH0Y.1 size=1048576 2024-09-13T00:00:40.736 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.734+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2TWD9YLMSYJI.2~41W5EOG8BH0Y.2 size=1029548 2024-09-13T00:00:40.736 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.734+0000 7fb8f2ce0640 1 > prepared multipart upload key=2TWD9YLMSYJI upload=41W5EOG8BH0Y tag=T9YMSNNRT73Q type=write size=4175276 2024-09-13T00:00:40.737 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.738+0000 7fb8f2ce0640 1 < canceled part key=_multipart_N3Z5XIG3TZRQ.2~Q093DN7RWYRT.0 size=1048576 2024-09-13T00:00:40.739 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.738+0000 7fb8f2ce0640 1 < completed part key=_multipart_N3Z5XIG3TZRQ.2~Q093DN7RWYRT.1 size=254283 2024-09-13T00:00:40.745 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.746+0000 7fb8f2ce0640 1 < canceled multipart upload key=N3Z5XIG3TZRQ upload=Q093DN7RWYRT tag=43BF230BFUOR type=write size=2351435 2024-09-13T00:00:40.752 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.750+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H9AU4HD247YU.2~JJ7UZ42RTES4.0 size=1048576 2024-09-13T00:00:40.754 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.754+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H9AU4HD247YU.2~JJ7UZ42RTES4.1 size=1048576 2024-09-13T00:00:40.757 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.758+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H9AU4HD247YU.2~JJ7UZ42RTES4.2 size=719604 2024-09-13T00:00:40.757 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.758+0000 7fb8f2ce0640 1 > prepared multipart upload key=H9AU4HD247YU upload=JJ7UZ42RTES4 tag=1GGFAYYINVET type=write size=3865332 2024-09-13T00:00:40.762 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.762+0000 7fb8f2ce0640 1 < completed part key=_multipart_5GM88ERM89KT.2~1O1SHLGP68NB.0 size=628281 2024-09-13T00:00:40.768 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.766+0000 7fb8f2ce0640 1 < completed multipart upload key=5GM88ERM89KT upload=1O1SHLGP68NB tag=ENFALVF6HD61 type=write size=1676857 2024-09-13T00:00:40.772 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.770+0000 7fb8f2ce0640 1 > prepared operation key=44EDYFT9VF8Y tag=JC9WGBVZ6I6W type=del size=2279311 2024-09-13T00:00:40.776 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.774+0000 7fb8f2ce0640 1 < completed part key=_multipart_H4M1TCN5W6XU.2~VBEF6HQVEOT9.0 size=1048576 2024-09-13T00:00:40.779 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.778+0000 7fb8f2ce0640 1 < completed part key=_multipart_H4M1TCN5W6XU.2~VBEF6HQVEOT9.1 size=1025373 2024-09-13T00:00:40.786 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.786+0000 7fb8f2ce0640 1 < completed multipart upload key=H4M1TCN5W6XU upload=VBEF6HQVEOT9 tag=MP82HAETZR43 type=write size=3122525 2024-09-13T00:00:40.795 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.794+0000 7fb8f2ce0640 1 > prepared part key=_multipart_44EDYFT9VF8Y.2~GBDAYPRB80NM.0 size=1025795 2024-09-13T00:00:40.795 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.794+0000 7fb8f2ce0640 1 > prepared multipart upload key=44EDYFT9VF8Y upload=GBDAYPRB80NM tag=UGBH4RWXEIEN type=write size=2074371 2024-09-13T00:00:40.799 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.798+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~IIPVG2N81MUU.0 size=1048576 2024-09-13T00:00:40.803 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.802+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~IIPVG2N81MUU.1 size=36089 2024-09-13T00:00:40.811 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.810+0000 7fb8f2ce0640 1 < completed multipart upload key=NQJQC8CEAC15 upload=IIPVG2N81MUU tag=7MMLOS0NB4UC type=write size=2133241 2024-09-13T00:00:40.816 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.814+0000 7fb8f2ce0640 1 > prepared operation key=2TWD9YLMSYJI tag=A7CMWUS7M3KA type=del size=127087 2024-09-13T00:00:40.821 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.822+0000 7fb8f2ce0640 1 < completed part key=_multipart_85YEW8N43FTA.2~N5TPB3XGK2DW.0 size=1048576 2024-09-13T00:00:40.825 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.826+0000 7fb8f2ce0640 1 < completed part key=_multipart_85YEW8N43FTA.2~N5TPB3XGK2DW.1 size=686114 2024-09-13T00:00:40.832 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.830+0000 7fb8f2ce0640 1 < completed multipart upload key=85YEW8N43FTA upload=N5TPB3XGK2DW tag=JV1NO6045OOD type=write size=2783266 2024-09-13T00:00:40.848 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.846+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H9AU4HD247YU.2~H20Q6OI32GVM.0 size=1048576 2024-09-13T00:00:40.852 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.850+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H9AU4HD247YU.2~H20Q6OI32GVM.1 size=665961 2024-09-13T00:00:40.852 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.850+0000 7fb8f2ce0640 1 > prepared multipart upload key=H9AU4HD247YU upload=H20Q6OI32GVM tag=CTKVQGDXN90Q type=write size=2763113 2024-09-13T00:00:40.855 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.854+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~E114XBPN0PKR.0 size=1048576 2024-09-13T00:00:40.859 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.858+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~E114XBPN0PKR.1 size=293137 2024-09-13T00:00:40.866 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.866+0000 7fb8f2ce0640 1 < completed multipart upload key=FX0NCGLK0KPJ upload=E114XBPN0PKR tag=L83VLEJRNOEX type=write size=2390289 2024-09-13T00:00:40.872 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.870+0000 7fb8f2ce0640 1 > prepared operation key=8Z6U7OOA0GBH tag=BOR3HGV4KPRD type=del size=1510468 2024-09-13T00:00:40.874 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.874+0000 7fb8f2ce0640 1 < completed delete operation key=2AHYRZUDFHP4 tag=DU5Q1Y6UNKHR type=del 2024-09-13T00:00:40.882 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.882+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~EYE1KZOTAS6M.0 size=1048576 2024-09-13T00:00:40.886 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.886+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~EYE1KZOTAS6M.1 size=1048576 2024-09-13T00:00:40.892 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.890+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~EYE1KZOTAS6M.2 size=382058 2024-09-13T00:00:40.892 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.890+0000 7fb8f2ce0640 1 > prepared multipart upload key=7OX5OJZ8GXU5 upload=EYE1KZOTAS6M tag=ZEFWYHGJ2FUD type=write size=3527786 2024-09-13T00:00:40.896 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.894+0000 7fb8f2ce0640 1 < completed delete operation key=O4MZ4UWNXME9 tag=QY5R9CY2PPLU type=del 2024-09-13T00:00:40.903 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.902+0000 7fb8f2ce0640 1 > prepared part key=_multipart_O4MZ4UWNXME9.2~3DGS4OF32X00.0 size=1048576 2024-09-13T00:00:40.907 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.906+0000 7fb8f2ce0640 1 > prepared part key=_multipart_O4MZ4UWNXME9.2~3DGS4OF32X00.1 size=242057 2024-09-13T00:00:40.907 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.906+0000 7fb8f2ce0640 1 > prepared multipart upload key=O4MZ4UWNXME9 upload=3DGS4OF32X00 tag=27AB2K0EZMVJ type=write size=2339209 2024-09-13T00:00:40.914 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.914+0000 7fb8f2ce0640 1 < suggested update operation key=70F7D20HI504 tag=7CC6NBU7PJOO type=del size=3399907 2024-09-13T00:00:40.919 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.918+0000 7fb8f2ce0640 1 > prepared operation key=EJN6HR8I7XQF tag=JZIX8AQZ3YNX type=write size=512111 2024-09-13T00:00:40.923 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.922+0000 7fb8f2ce0640 1 < canceled operation key=2AHYRZUDFHP4 tag=G4F5AN0IGMVF type=del size=3245091 2024-09-13T00:00:40.930 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.930+0000 7fb8f2ce0640 1 > prepared part key=_multipart_P8OYS5ZE1NX0.2~UCZQEE2J2KFF.0 size=1048576 2024-09-13T00:00:40.934 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.934+0000 7fb8f2ce0640 1 > prepared part key=_multipart_P8OYS5ZE1NX0.2~UCZQEE2J2KFF.1 size=31420 2024-09-13T00:00:40.934 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.934+0000 7fb8f2ce0640 1 > prepared multipart upload key=P8OYS5ZE1NX0 upload=UCZQEE2J2KFF tag=JAHFFO454MPY type=write size=2128572 2024-09-13T00:00:40.939 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.938+0000 7fb8f2ce0640 1 < completed delete operation key=8Z6U7OOA0GBH tag=3AJ15OCODI8P type=del 2024-09-13T00:00:40.943 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.942+0000 7fb8f2ce0640 1 > prepared operation key=1PX7DBO9M3YI tag=Y3Z8IE4X5FR9 type=write size=584401 2024-09-13T00:00:40.948 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.946+0000 7fb8f2ce0640 1 < completed part key=_multipart_N3Z5XIG3TZRQ.2~4SNT2S1YHIUP.0 size=462152 2024-09-13T00:00:40.954 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.954+0000 7fb8f2ce0640 1 < completed multipart upload key=N3Z5XIG3TZRQ upload=4SNT2S1YHIUP tag=SDFX5V2B9TRN type=write size=1510728 2024-09-13T00:00:40.959 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.958+0000 7fb8f2ce0640 1 > prepared operation key=0WC6P8YNL0H2 tag=GPS60HR9GW04 type=del size=3367563 2024-09-13T00:00:40.964 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.962+0000 7fb8f2ce0640 1 < completed delete operation key=PVYZQLGYT3XU tag=TG1GAK62FUOE type=del 2024-09-13T00:00:40.972 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.970+0000 7fb8f2ce0640 1 > prepared part key=_multipart_44EDYFT9VF8Y.2~L56A4CFP4E9M.0 size=1048576 2024-09-13T00:00:40.976 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.974+0000 7fb8f2ce0640 1 > prepared part key=_multipart_44EDYFT9VF8Y.2~L56A4CFP4E9M.1 size=312102 2024-09-13T00:00:40.976 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.974+0000 7fb8f2ce0640 1 > prepared multipart upload key=44EDYFT9VF8Y upload=L56A4CFP4E9M tag=ZH6GUTYCFFET type=write size=2409254 2024-09-13T00:00:40.985 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.986+0000 7fb8f2ce0640 1 < suggested remove operation key=H4M1TCN5W6XU tag=TEC0UAZUS48D type=del 2024-09-13T00:00:40.989 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.990+0000 7fb8f2ce0640 1 > prepared operation key=H4M1TCN5W6XU tag=46T0NGKUKBQJ type=del size=3949178 2024-09-13T00:00:40.995 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.994+0000 7fb8f2ce0640 1 < completed part key=_multipart_J8NGDSF9XRTP.2~0DEYM1A5G8E2.0 size=1048576 2024-09-13T00:00:40.998 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:40.998+0000 7fb8f2ce0640 1 < completed part key=_multipart_J8NGDSF9XRTP.2~0DEYM1A5G8E2.1 size=266417 2024-09-13T00:00:41.008 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.006+0000 7fb8f2ce0640 1 < completed multipart upload key=J8NGDSF9XRTP upload=0DEYM1A5G8E2 tag=JWAENEX5FQOS type=write size=2363569 2024-09-13T00:00:41.012 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.010+0000 7fb8f2ce0640 1 > prepared operation key=H4M1TCN5W6XU tag=IEUP6VH789N3 type=del size=727117 2024-09-13T00:00:41.017 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.018+0000 7fb8f2ce0640 1 < completed part key=_multipart_2TWD9YLMSYJI.2~41W5EOG8BH0Y.0 size=1048576 2024-09-13T00:00:41.021 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.022+0000 7fb8f2ce0640 1 < completed part key=_multipart_2TWD9YLMSYJI.2~41W5EOG8BH0Y.1 size=1048576 2024-09-13T00:00:41.021 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.022+0000 7fb8f2ce0640 1 < canceled part key=_multipart_2TWD9YLMSYJI.2~41W5EOG8BH0Y.2 size=1029548 2024-09-13T00:00:41.029 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.030+0000 7fb8f2ce0640 1 < canceled multipart upload key=2TWD9YLMSYJI upload=41W5EOG8BH0Y tag=T9YMSNNRT73Q type=write size=4175276 2024-09-13T00:00:41.034 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.034+0000 7fb8f2ce0640 1 > prepared operation key=ZFHN2SNTZ5W7 tag=8WQM1GJDGVOE type=del size=2353045 2024-09-13T00:00:41.039 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.038+0000 7fb8f2ce0640 1 < completed part key=_multipart_H9AU4HD247YU.2~JJ7UZ42RTES4.0 size=1048576 2024-09-13T00:00:41.043 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.042+0000 7fb8f2ce0640 1 < completed part key=_multipart_H9AU4HD247YU.2~JJ7UZ42RTES4.1 size=1048576 2024-09-13T00:00:41.047 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.046+0000 7fb8f2ce0640 1 < completed part key=_multipart_H9AU4HD247YU.2~JJ7UZ42RTES4.2 size=719604 2024-09-13T00:00:41.055 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.054+0000 7fb8f2ce0640 1 < completed multipart upload key=H9AU4HD247YU upload=JJ7UZ42RTES4 tag=1GGFAYYINVET type=write size=3865332 2024-09-13T00:00:41.060 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.058+0000 7fb8f2ce0640 1 > prepared operation key=FX0NCGLK0KPJ tag=T0Y8RJJ2PGC4 type=del size=2123438 2024-09-13T00:00:41.070 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.070+0000 7fb8f2ce0640 1 < suggested remove operation key=44EDYFT9VF8Y tag=JC9WGBVZ6I6W type=del 2024-09-13T00:00:41.079 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.078+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~7P3FJPW7JKU0.0 size=1048576 2024-09-13T00:00:41.083 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.082+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~7P3FJPW7JKU0.1 size=682742 2024-09-13T00:00:41.083 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.082+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZTK9AYBSLCOZ upload=7P3FJPW7JKU0 tag=PFTPA6R7AX4X type=write size=2779894 2024-09-13T00:00:41.088 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.086+0000 7fb8f2ce0640 1 < completed part key=_multipart_44EDYFT9VF8Y.2~GBDAYPRB80NM.0 size=1025795 2024-09-13T00:00:41.096 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.094+0000 7fb8f2ce0640 1 < completed multipart upload key=44EDYFT9VF8Y upload=GBDAYPRB80NM tag=UGBH4RWXEIEN type=write size=2074371 2024-09-13T00:00:41.101 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.102+0000 7fb8f2ce0640 1 > prepared operation key=70F7D20HI504 tag=89G8FB7LH1IK type=write size=844342 2024-09-13T00:00:41.106 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.106+0000 7fb8f2ce0640 1 < completed delete operation key=2TWD9YLMSYJI tag=A7CMWUS7M3KA type=del 2024-09-13T00:00:41.115 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.114+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EUE3LT0PDIXV.2~Y41WC346SZ9K.0 size=675628 2024-09-13T00:00:41.115 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.114+0000 7fb8f2ce0640 1 > prepared multipart upload key=EUE3LT0PDIXV upload=Y41WC346SZ9K tag=2T1XUHTVXL7W type=write size=1724204 2024-09-13T00:00:41.120 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.118+0000 7fb8f2ce0640 1 < completed part key=_multipart_H9AU4HD247YU.2~H20Q6OI32GVM.0 size=1048576 2024-09-13T00:00:41.124 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.122+0000 7fb8f2ce0640 1 < completed part key=_multipart_H9AU4HD247YU.2~H20Q6OI32GVM.1 size=665961 2024-09-13T00:00:41.133 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.134+0000 7fb8f2ce0640 1 < completed multipart upload key=H9AU4HD247YU upload=H20Q6OI32GVM tag=CTKVQGDXN90Q type=write size=2763113 2024-09-13T00:00:41.142 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.142+0000 7fb8f2ce0640 1 > prepared part key=_multipart_4K76U69GZ9AE.2~DR0JKEFC3J66.0 size=1048576 2024-09-13T00:00:41.147 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.146+0000 7fb8f2ce0640 1 > prepared part key=_multipart_4K76U69GZ9AE.2~DR0JKEFC3J66.1 size=1048576 2024-09-13T00:00:41.151 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.150+0000 7fb8f2ce0640 1 > prepared part key=_multipart_4K76U69GZ9AE.2~DR0JKEFC3J66.2 size=861973 2024-09-13T00:00:41.151 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.150+0000 7fb8f2ce0640 1 > prepared multipart upload key=4K76U69GZ9AE upload=DR0JKEFC3J66 tag=PNKW7DDX22VV type=write size=4007701 2024-09-13T00:00:41.156 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.154+0000 7fb8f2ce0640 1 < completed delete operation key=8Z6U7OOA0GBH tag=BOR3HGV4KPRD type=del 2024-09-13T00:00:41.161 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.162+0000 7fb8f2ce0640 1 > prepared operation key=EJN6HR8I7XQF tag=L1N07D8G1904 type=del size=173836 2024-09-13T00:00:41.166 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.166+0000 7fb8f2ce0640 1 < completed part key=_multipart_7OX5OJZ8GXU5.2~EYE1KZOTAS6M.0 size=1048576 2024-09-13T00:00:41.166 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.166+0000 7fb8f2ce0640 1 < canceled part key=_multipart_7OX5OJZ8GXU5.2~EYE1KZOTAS6M.1 size=1048576 2024-09-13T00:00:41.170 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.170+0000 7fb8f2ce0640 1 < completed part key=_multipart_7OX5OJZ8GXU5.2~EYE1KZOTAS6M.2 size=382058 2024-09-13T00:00:41.180 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.178+0000 7fb8f2ce0640 1 < canceled multipart upload key=7OX5OJZ8GXU5 upload=EYE1KZOTAS6M tag=ZEFWYHGJ2FUD type=write size=3527786 2024-09-13T00:00:41.185 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.186+0000 7fb8f2ce0640 1 > prepared operation key=N3Z5XIG3TZRQ tag=PJ2MC4RXPAHN type=write size=598188 2024-09-13T00:00:41.190 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.190+0000 7fb8f2ce0640 1 < completed part key=_multipart_O4MZ4UWNXME9.2~3DGS4OF32X00.0 size=1048576 2024-09-13T00:00:41.194 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.194+0000 7fb8f2ce0640 1 < completed part key=_multipart_O4MZ4UWNXME9.2~3DGS4OF32X00.1 size=242057 2024-09-13T00:00:41.202 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.202+0000 7fb8f2ce0640 1 < completed multipart upload key=O4MZ4UWNXME9 upload=3DGS4OF32X00 tag=27AB2K0EZMVJ type=write size=2339209 2024-09-13T00:00:41.211 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.210+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~GKOM0L6S7UXF.0 size=1048576 2024-09-13T00:00:41.215 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.214+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~GKOM0L6S7UXF.1 size=1048576 2024-09-13T00:00:41.219 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.218+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~GKOM0L6S7UXF.2 size=524674 2024-09-13T00:00:41.220 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.218+0000 7fb8f2ce0640 1 > prepared multipart upload key=8MPUVD37EDN9 upload=GKOM0L6S7UXF tag=OULF0D7UD424 type=write size=3670402 2024-09-13T00:00:41.224 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.222+0000 7fb8f2ce0640 1 < completed write operation key=EJN6HR8I7XQF tag=JZIX8AQZ3YNX type=write size=512111 2024-09-13T00:00:41.229 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.230+0000 7fb8f2ce0640 1 > prepared operation key=PVYZQLGYT3XU tag=D7TSXKNXTCKM type=del size=4083886 2024-09-13T00:00:41.234 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.234+0000 7fb8f2ce0640 1 < completed part key=_multipart_P8OYS5ZE1NX0.2~UCZQEE2J2KFF.0 size=1048576 2024-09-13T00:00:41.239 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.238+0000 7fb8f2ce0640 1 < completed part key=_multipart_P8OYS5ZE1NX0.2~UCZQEE2J2KFF.1 size=31420 2024-09-13T00:00:41.248 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.246+0000 7fb8f2ce0640 1 < completed multipart upload key=P8OYS5ZE1NX0 upload=UCZQEE2J2KFF tag=JAHFFO454MPY type=write size=2128572 2024-09-13T00:00:41.253 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.254+0000 7fb8f2ce0640 1 > prepared operation key=7OX5OJZ8GXU5 tag=8IJ9G64V4R5V type=del size=2613665 2024-09-13T00:00:41.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:40 smithi136 bash[76371]: cluster 2024-09-13T00:00:39.128787+0000 mgr.y (mgr.24661) 706 : cluster [DBG] pgmap v1382: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 440 KiB/s rd, 151 KiB/s wr, 778 op/s 2024-09-13T00:00:41.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:40 smithi136 bash[76371]: audit 2024-09-13T00:00:39.887727+0000 mon.a (mon.0) 2072 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:00:41.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:40 smithi136 bash[76371]: audit 2024-09-13T00:00:39.893294+0000 mon.a (mon.0) 2073 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:00:41.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:40 smithi136 bash[76371]: audit 2024-09-13T00:00:40.185431+0000 mon.a (mon.0) 2074 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:00:41.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:40 smithi136 bash[76371]: audit 2024-09-13T00:00:40.191272+0000 mon.a (mon.0) 2075 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:00:41.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:40 smithi136 bash[76371]: audit 2024-09-13T00:00:40.630517+0000 mon.a (mon.0) 2076 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:00:41.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:40 smithi136 bash[76371]: audit 2024-09-13T00:00:40.631564+0000 mon.a (mon.0) 2077 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:00:41.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:40 smithi136 bash[76371]: audit 2024-09-13T00:00:40.639877+0000 mon.a (mon.0) 2078 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:00:41.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:40 smithi136 bash[75164]: cluster 2024-09-13T00:00:39.128787+0000 mgr.y (mgr.24661) 706 : cluster [DBG] pgmap v1382: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 440 KiB/s rd, 151 KiB/s wr, 778 op/s 2024-09-13T00:00:41.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:40 smithi136 bash[75164]: audit 2024-09-13T00:00:39.887727+0000 mon.a (mon.0) 2072 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:00:41.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:40 smithi136 bash[75164]: audit 2024-09-13T00:00:39.893294+0000 mon.a (mon.0) 2073 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:00:41.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:40 smithi136 bash[75164]: audit 2024-09-13T00:00:40.185431+0000 mon.a (mon.0) 2074 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:00:41.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:40 smithi136 bash[75164]: audit 2024-09-13T00:00:40.191272+0000 mon.a (mon.0) 2075 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:00:41.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:40 smithi136 bash[75164]: audit 2024-09-13T00:00:40.630517+0000 mon.a (mon.0) 2076 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:00:41.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:40 smithi136 bash[75164]: audit 2024-09-13T00:00:40.631564+0000 mon.a (mon.0) 2077 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:00:41.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:40 smithi136 bash[75164]: audit 2024-09-13T00:00:40.639877+0000 mon.a (mon.0) 2078 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:00:41.259 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.258+0000 7fb8f2ce0640 1 < completed write operation key=1PX7DBO9M3YI tag=Y3Z8IE4X5FR9 type=write size=584401 2024-09-13T00:00:41.264 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.262+0000 7fb8f2ce0640 1 > prepared operation key=NX26SI4F2ZMB tag=QBP92JRO9MPN type=write size=915907 2024-09-13T00:00:41.270 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.270+0000 7fb8f2ce0640 1 < completed delete operation key=0WC6P8YNL0H2 tag=GPS60HR9GW04 type=del 2024-09-13T00:00:41.280 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.278+0000 7fb8f2ce0640 1 > prepared part key=_multipart_44EDYFT9VF8Y.2~Z2OQBHNAVRLZ.0 size=391323 2024-09-13T00:00:41.280 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.278+0000 7fb8f2ce0640 1 > prepared multipart upload key=44EDYFT9VF8Y upload=Z2OQBHNAVRLZ tag=PY81BOE3BHY2 type=write size=1439899 2024-09-13T00:00:41.285 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.286+0000 7fb8f2ce0640 1 < completed part key=_multipart_44EDYFT9VF8Y.2~L56A4CFP4E9M.0 size=1048576 2024-09-13T00:00:41.290 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.290+0000 7fb8f2ce0640 1 < completed part key=_multipart_44EDYFT9VF8Y.2~L56A4CFP4E9M.1 size=312102 2024-09-13T00:00:41.299 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.298+0000 7fb8f2ce0640 1 < completed multipart upload key=44EDYFT9VF8Y upload=L56A4CFP4E9M tag=ZH6GUTYCFFET type=write size=2409254 2024-09-13T00:00:41.304 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.302+0000 7fb8f2ce0640 1 > prepared operation key=5GM88ERM89KT tag=PYYPYTCEM021 type=del size=748587 2024-09-13T00:00:41.309 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.310+0000 7fb8f2ce0640 1 < completed delete operation key=H4M1TCN5W6XU tag=46T0NGKUKBQJ type=del 2024-09-13T00:00:41.314 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.314+0000 7fb8f2ce0640 1 > prepared operation key=PVYZQLGYT3XU tag=K3F3Q9P5L8N4 type=del size=1398089 2024-09-13T00:00:41.319 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.318+0000 7fb8f2ce0640 1 < completed delete operation key=H4M1TCN5W6XU tag=IEUP6VH789N3 type=del 2024-09-13T00:00:41.324 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.322+0000 7fb8f2ce0640 1 > prepared operation key=8MPUVD37EDN9 tag=7CDOXB60XF5F type=del size=1122522 2024-09-13T00:00:41.333 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.334+0000 7fb8f2ce0640 1 < suggested remove operation key=ZFHN2SNTZ5W7 tag=8WQM1GJDGVOE type=del 2024-09-13T00:00:41.338 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.338+0000 7fb8f2ce0640 1 > prepared operation key=NQJQC8CEAC15 tag=AQDR3SN91M85 type=del size=4055368 2024-09-13T00:00:41.343 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.342+0000 7fb8f2ce0640 1 < completed delete operation key=FX0NCGLK0KPJ tag=T0Y8RJJ2PGC4 type=del 2024-09-13T00:00:41.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:40 smithi181 bash[67587]: cluster 2024-09-13T00:00:39.128787+0000 mgr.y (mgr.24661) 706 : cluster [DBG] pgmap v1382: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 440 KiB/s rd, 151 KiB/s wr, 778 op/s 2024-09-13T00:00:41.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:40 smithi181 bash[67587]: audit 2024-09-13T00:00:39.887727+0000 mon.a (mon.0) 2072 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:00:41.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:40 smithi181 bash[67587]: audit 2024-09-13T00:00:39.893294+0000 mon.a (mon.0) 2073 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:00:41.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:40 smithi181 bash[67587]: audit 2024-09-13T00:00:40.185431+0000 mon.a (mon.0) 2074 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:00:41.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:40 smithi181 bash[67587]: audit 2024-09-13T00:00:40.191272+0000 mon.a (mon.0) 2075 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:00:41.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:40 smithi181 bash[67587]: audit 2024-09-13T00:00:40.630517+0000 mon.a (mon.0) 2076 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:00:41.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:40 smithi181 bash[67587]: audit 2024-09-13T00:00:40.631564+0000 mon.a (mon.0) 2077 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:00:41.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:40 smithi181 bash[67587]: audit 2024-09-13T00:00:40.639877+0000 mon.a (mon.0) 2078 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:00:41.349 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.350+0000 7fb8f2ce0640 1 > prepared operation key=85YEW8N43FTA tag=TY0F9NZFTIX6 type=del size=3754790 2024-09-13T00:00:41.355 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.354+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~7P3FJPW7JKU0.0 size=1048576 2024-09-13T00:00:41.359 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.358+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~7P3FJPW7JKU0.1 size=682742 2024-09-13T00:00:41.368 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.366+0000 7fb8f2ce0640 1 < completed multipart upload key=ZTK9AYBSLCOZ upload=7P3FJPW7JKU0 tag=PFTPA6R7AX4X type=write size=2779894 2024-09-13T00:00:41.372 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.370+0000 7fb8f2ce0640 1 > prepared operation key=8MPUVD37EDN9 tag=RT8V1N3KZ3EQ type=write size=529578 2024-09-13T00:00:41.377 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.378+0000 7fb8f2ce0640 1 < completed write operation key=70F7D20HI504 tag=89G8FB7LH1IK type=write size=844342 2024-09-13T00:00:41.386 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.386+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NX26SI4F2ZMB.2~0CGSQ3D8Y5GC.0 size=1048576 2024-09-13T00:00:41.390 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.390+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NX26SI4F2ZMB.2~0CGSQ3D8Y5GC.1 size=1048576 2024-09-13T00:00:41.395 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.394+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NX26SI4F2ZMB.2~0CGSQ3D8Y5GC.2 size=978642 2024-09-13T00:00:41.395 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.394+0000 7fb8f2ce0640 1 > prepared multipart upload key=NX26SI4F2ZMB upload=0CGSQ3D8Y5GC tag=D4BNVZXQ3M40 type=write size=4124370 2024-09-13T00:00:41.401 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.402+0000 7fb8f2ce0640 1 < completed part key=_multipart_EUE3LT0PDIXV.2~Y41WC346SZ9K.0 size=675628 2024-09-13T00:00:41.410 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.410+0000 7fb8f2ce0640 1 < completed multipart upload key=EUE3LT0PDIXV upload=Y41WC346SZ9K tag=2T1XUHTVXL7W type=write size=1724204 2024-09-13T00:00:41.419 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.418+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~PP0G457U8DTL.0 size=1048576 2024-09-13T00:00:41.423 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.422+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~PP0G457U8DTL.1 size=633045 2024-09-13T00:00:41.423 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.422+0000 7fb8f2ce0640 1 > prepared multipart upload key=70F7D20HI504 upload=PP0G457U8DTL tag=5LEN0HAYXLXB type=write size=2730197 2024-09-13T00:00:41.428 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.426+0000 7fb8f2ce0640 1 < completed part key=_multipart_4K76U69GZ9AE.2~DR0JKEFC3J66.0 size=1048576 2024-09-13T00:00:41.432 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.430+0000 7fb8f2ce0640 1 < completed part key=_multipart_4K76U69GZ9AE.2~DR0JKEFC3J66.1 size=1048576 2024-09-13T00:00:41.432 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.430+0000 7fb8f2ce0640 1 < canceled part key=_multipart_4K76U69GZ9AE.2~DR0JKEFC3J66.2 size=861973 2024-09-13T00:00:41.444 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.442+0000 7fb8f2ce0640 1 < canceled multipart upload key=4K76U69GZ9AE upload=DR0JKEFC3J66 tag=PNKW7DDX22VV type=write size=4007701 2024-09-13T00:00:41.447 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.446+0000 7fb8f2ce0640 1 > prepared operation key=N3Z5XIG3TZRQ tag=RQHS6WXCUI6B type=write size=998274 2024-09-13T00:00:41.451 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.450+0000 7fb8f2ce0640 1 < completed delete operation key=EJN6HR8I7XQF tag=L1N07D8G1904 type=del 2024-09-13T00:00:41.461 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.462+0000 7fb8f2ce0640 1 > prepared part key=_multipart_O4MZ4UWNXME9.2~DSX5GX10W7CO.0 size=321117 2024-09-13T00:00:41.461 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.462+0000 7fb8f2ce0640 1 > prepared multipart upload key=O4MZ4UWNXME9 upload=DSX5GX10W7CO tag=S1J6D56UMAUL type=write size=1369693 2024-09-13T00:00:41.467 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.466+0000 7fb8f2ce0640 1 < completed write operation key=N3Z5XIG3TZRQ tag=PJ2MC4RXPAHN type=write size=598188 2024-09-13T00:00:41.476 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.474+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZFHN2SNTZ5W7.2~GSDI74XRE69F.0 size=386554 2024-09-13T00:00:41.476 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.474+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZFHN2SNTZ5W7 upload=GSDI74XRE69F tag=9D16QIHXRHLG type=write size=1435130 2024-09-13T00:00:41.481 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.478+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~GKOM0L6S7UXF.0 size=1048576 2024-09-13T00:00:41.485 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.482+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~GKOM0L6S7UXF.1 size=1048576 2024-09-13T00:00:41.489 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.490+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~GKOM0L6S7UXF.2 size=524674 2024-09-13T00:00:41.497 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.498+0000 7fb8f2ce0640 1 < completed multipart upload key=8MPUVD37EDN9 upload=GKOM0L6S7UXF tag=OULF0D7UD424 type=write size=3670402 2024-09-13T00:00:41.506 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.506+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H9AU4HD247YU.2~APPYE1PLKL62.0 size=1048576 2024-09-13T00:00:41.510 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.510+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H9AU4HD247YU.2~APPYE1PLKL62.1 size=1048576 2024-09-13T00:00:41.514 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.514+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H9AU4HD247YU.2~APPYE1PLKL62.2 size=246534 2024-09-13T00:00:41.514 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.514+0000 7fb8f2ce0640 1 > prepared multipart upload key=H9AU4HD247YU upload=APPYE1PLKL62 tag=3IZWEYHBAJ2N type=write size=3392262 2024-09-13T00:00:41.519 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.518+0000 7fb8f2ce0640 1 < completed delete operation key=PVYZQLGYT3XU tag=D7TSXKNXTCKM type=del 2024-09-13T00:00:41.524 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.522+0000 7fb8f2ce0640 1 > prepared operation key=QMZREZ1QEDNJ tag=7T7E8BNXH6G4 type=write size=926639 2024-09-13T00:00:41.528 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.526+0000 7fb8f2ce0640 1 < completed delete operation key=7OX5OJZ8GXU5 tag=8IJ9G64V4R5V type=del 2024-09-13T00:00:41.537 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.538+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~TPYN7PTKTDEB.0 size=1048576 2024-09-13T00:00:41.541 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.542+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~TPYN7PTKTDEB.1 size=1048576 2024-09-13T00:00:41.545 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.546+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~TPYN7PTKTDEB.2 size=919857 2024-09-13T00:00:41.545 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.546+0000 7fb8f2ce0640 1 > prepared multipart upload key=1HX06CCZJ1DD upload=TPYN7PTKTDEB tag=Z6YGNPBC6R74 type=write size=4065585 2024-09-13T00:00:41.550 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.550+0000 7fb8f2ce0640 1 < completed write operation key=NX26SI4F2ZMB tag=QBP92JRO9MPN type=write size=915907 2024-09-13T00:00:41.558 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.558+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~XMHQ3UVKNRRF.0 size=1048576 2024-09-13T00:00:41.562 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.562+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~XMHQ3UVKNRRF.1 size=864314 2024-09-13T00:00:41.562 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.562+0000 7fb8f2ce0640 1 > prepared multipart upload key=8MPUVD37EDN9 upload=XMHQ3UVKNRRF tag=CFEMBE4EJW48 type=write size=2961466 2024-09-13T00:00:41.567 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.566+0000 7fb8f2ce0640 1 < completed part key=_multipart_44EDYFT9VF8Y.2~Z2OQBHNAVRLZ.0 size=391323 2024-09-13T00:00:41.575 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.574+0000 7fb8f2ce0640 1 < completed multipart upload key=44EDYFT9VF8Y upload=Z2OQBHNAVRLZ tag=PY81BOE3BHY2 type=write size=1439899 2024-09-13T00:00:41.580 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.578+0000 7fb8f2ce0640 1 > prepared operation key=PVYZQLGYT3XU tag=ZBNNDHR3OUFX type=del size=1144435 2024-09-13T00:00:41.585 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.586+0000 7fb8f2ce0640 1 < completed delete operation key=5GM88ERM89KT tag=PYYPYTCEM021 type=del 2024-09-13T00:00:41.595 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.594+0000 7fb8f2ce0640 1 > prepared part key=_multipart_85YEW8N43FTA.2~H16XBBFRPV0V.0 size=991598 2024-09-13T00:00:41.595 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.594+0000 7fb8f2ce0640 1 > prepared multipart upload key=85YEW8N43FTA upload=H16XBBFRPV0V tag=PTH66524EQMH type=write size=2040174 2024-09-13T00:00:41.606 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.606+0000 7fb8f2ce0640 1 < suggested remove operation key=PVYZQLGYT3XU tag=K3F3Q9P5L8N4 type=del 2024-09-13T00:00:41.617 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.618+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~D71XWPUZM836.0 size=689160 2024-09-13T00:00:41.617 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.618+0000 7fb8f2ce0640 1 > prepared multipart upload key=8Z6U7OOA0GBH upload=D71XWPUZM836 tag=1JAPKT2EKWTS type=write size=1737736 2024-09-13T00:00:41.622 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.622+0000 7fb8f2ce0640 1 < completed delete operation key=8MPUVD37EDN9 tag=7CDOXB60XF5F type=del 2024-09-13T00:00:41.627 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.626+0000 7fb8f2ce0640 1 > prepared operation key=1HX06CCZJ1DD tag=Q60DPHFSZJB9 type=del size=21054 2024-09-13T00:00:41.637 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.634+0000 7fb8f2ce0640 1 < suggested remove operation key=NQJQC8CEAC15 tag=AQDR3SN91M85 type=del 2024-09-13T00:00:41.646 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.646+0000 7fb8f2ce0640 1 > prepared part key=_multipart_F0QCVR9E08WE.2~0BXGEV5TL21H.0 size=1048576 2024-09-13T00:00:41.650 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.650+0000 7fb8f2ce0640 1 > prepared part key=_multipart_F0QCVR9E08WE.2~0BXGEV5TL21H.1 size=654861 2024-09-13T00:00:41.650 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.650+0000 7fb8f2ce0640 1 > prepared multipart upload key=F0QCVR9E08WE upload=0BXGEV5TL21H tag=3V9QL5LKEC4D type=write size=2752013 2024-09-13T00:00:41.656 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.654+0000 7fb8f2ce0640 1 < canceled operation key=85YEW8N43FTA tag=TY0F9NZFTIX6 type=del size=3754790 2024-09-13T00:00:41.659 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.658+0000 7fb8f2ce0640 1 > prepared operation key=NQJQC8CEAC15 tag=XT5O7QIUR9ZW type=del size=3397651 2024-09-13T00:00:41.670 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.670+0000 7fb8f2ce0640 1 < suggested update operation key=8MPUVD37EDN9 tag=RT8V1N3KZ3EQ type=write size=529578 2024-09-13T00:00:41.674 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.674+0000 7fb8f2ce0640 1 > prepared operation key=PVYZQLGYT3XU tag=TPFUVMVVVSY8 type=del size=2835888 2024-09-13T00:00:41.680 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.678+0000 7fb8f2ce0640 1 < completed part key=_multipart_NX26SI4F2ZMB.2~0CGSQ3D8Y5GC.0 size=1048576 2024-09-13T00:00:41.684 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.682+0000 7fb8f2ce0640 1 < completed part key=_multipart_NX26SI4F2ZMB.2~0CGSQ3D8Y5GC.1 size=1048576 2024-09-13T00:00:41.688 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.686+0000 7fb8f2ce0640 1 < completed part key=_multipart_NX26SI4F2ZMB.2~0CGSQ3D8Y5GC.2 size=978642 2024-09-13T00:00:41.696 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.694+0000 7fb8f2ce0640 1 < completed multipart upload key=NX26SI4F2ZMB upload=0CGSQ3D8Y5GC tag=D4BNVZXQ3M40 type=write size=4124370 2024-09-13T00:00:41.706 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.706+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H5EIP0E9CBWX.2~YZXWEFD98SDJ.0 size=1048576 2024-09-13T00:00:41.710 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.710+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H5EIP0E9CBWX.2~YZXWEFD98SDJ.1 size=1048576 2024-09-13T00:00:41.714 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.714+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H5EIP0E9CBWX.2~YZXWEFD98SDJ.2 size=1015768 2024-09-13T00:00:41.714 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.714+0000 7fb8f2ce0640 1 > prepared multipart upload key=H5EIP0E9CBWX upload=YZXWEFD98SDJ tag=RCOEONEASG8D type=write size=4161496 2024-09-13T00:00:41.719 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.718+0000 7fb8f2ce0640 1 < completed part key=_multipart_70F7D20HI504.2~PP0G457U8DTL.0 size=1048576 2024-09-13T00:00:41.723 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.722+0000 7fb8f2ce0640 1 < completed part key=_multipart_70F7D20HI504.2~PP0G457U8DTL.1 size=633045 2024-09-13T00:00:41.731 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.730+0000 7fb8f2ce0640 1 < completed multipart upload key=70F7D20HI504 upload=PP0G457U8DTL tag=5LEN0HAYXLXB type=write size=2730197 2024-09-13T00:00:41.740 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.738+0000 7fb8f2ce0640 1 > prepared part key=_multipart_44EDYFT9VF8Y.2~ZGDROA49RZNR.0 size=359315 2024-09-13T00:00:41.740 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.738+0000 7fb8f2ce0640 1 > prepared multipart upload key=44EDYFT9VF8Y upload=ZGDROA49RZNR tag=20UPKO74LFKZ type=write size=1407891 2024-09-13T00:00:41.746 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.746+0000 7fb8f2ce0640 1 < completed write operation key=N3Z5XIG3TZRQ tag=RQHS6WXCUI6B type=write size=998274 2024-09-13T00:00:41.750 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.750+0000 7fb8f2ce0640 1 > prepared operation key=63FJDWBO2T9S tag=6K1TPOW6BP0I type=del size=3870386 2024-09-13T00:00:41.756 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.754+0000 7fb8f2ce0640 1 < completed part key=_multipart_O4MZ4UWNXME9.2~DSX5GX10W7CO.0 size=321117 2024-09-13T00:00:41.764 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.762+0000 7fb8f2ce0640 1 < completed multipart upload key=O4MZ4UWNXME9 upload=DSX5GX10W7CO tag=S1J6D56UMAUL type=write size=1369693 2024-09-13T00:00:41.769 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.770+0000 7fb8f2ce0640 1 > prepared operation key=FX0NCGLK0KPJ tag=72TWREY7FR7T type=del size=113757 2024-09-13T00:00:41.775 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.774+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZFHN2SNTZ5W7.2~GSDI74XRE69F.0 size=386554 2024-09-13T00:00:41.784 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.782+0000 7fb8f2ce0640 1 < completed multipart upload key=ZFHN2SNTZ5W7 upload=GSDI74XRE69F tag=9D16QIHXRHLG type=write size=1435130 2024-09-13T00:00:41.789 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.790+0000 7fb8f2ce0640 1 > prepared operation key=8Z6U7OOA0GBH tag=PT22HNWO9WWB type=write size=453388 2024-09-13T00:00:41.795 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.794+0000 7fb8f2ce0640 1 < completed part key=_multipart_H9AU4HD247YU.2~APPYE1PLKL62.0 size=1048576 2024-09-13T00:00:41.799 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.798+0000 7fb8f2ce0640 1 < completed part key=_multipart_H9AU4HD247YU.2~APPYE1PLKL62.1 size=1048576 2024-09-13T00:00:41.803 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.802+0000 7fb8f2ce0640 1 < completed part key=_multipart_H9AU4HD247YU.2~APPYE1PLKL62.2 size=246534 2024-09-13T00:00:41.812 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.810+0000 7fb8f2ce0640 1 < completed multipart upload key=H9AU4HD247YU upload=APPYE1PLKL62 tag=3IZWEYHBAJ2N type=write size=3392262 2024-09-13T00:00:41.817 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.818+0000 7fb8f2ce0640 1 > prepared operation key=QMZREZ1QEDNJ tag=I2HK3GU0CXMF type=del size=2437476 2024-09-13T00:00:41.828 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.826+0000 7fb8f2ce0640 1 < suggested update operation key=QMZREZ1QEDNJ tag=7T7E8BNXH6G4 type=write size=926639 2024-09-13T00:00:41.833 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.834+0000 7fb8f2ce0640 1 > prepared operation key=FX0NCGLK0KPJ tag=4HO7661VDWSD type=write size=618258 2024-09-13T00:00:41.839 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.838+0000 7fb8f2ce0640 1 < completed part key=_multipart_1HX06CCZJ1DD.2~TPYN7PTKTDEB.0 size=1048576 2024-09-13T00:00:41.843 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.842+0000 7fb8f2ce0640 1 < completed part key=_multipart_1HX06CCZJ1DD.2~TPYN7PTKTDEB.1 size=1048576 2024-09-13T00:00:41.848 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.846+0000 7fb8f2ce0640 1 < completed part key=_multipart_1HX06CCZJ1DD.2~TPYN7PTKTDEB.2 size=919857 2024-09-13T00:00:41.857 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.858+0000 7fb8f2ce0640 1 < completed multipart upload key=1HX06CCZJ1DD upload=TPYN7PTKTDEB tag=Z6YGNPBC6R74 type=write size=4065585 2024-09-13T00:00:41.867 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.866+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~LFGFYODGNREH.0 size=772139 2024-09-13T00:00:41.867 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.866+0000 7fb8f2ce0640 1 > prepared multipart upload key=PVYZQLGYT3XU upload=LFGFYODGNREH tag=0VEKXCHVIXZ0 type=write size=1820715 2024-09-13T00:00:41.872 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.870+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~XMHQ3UVKNRRF.0 size=1048576 2024-09-13T00:00:41.876 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.874+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~XMHQ3UVKNRRF.1 size=864314 2024-09-13T00:00:41.885 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.886+0000 7fb8f2ce0640 1 < completed multipart upload key=8MPUVD37EDN9 upload=XMHQ3UVKNRRF tag=CFEMBE4EJW48 type=write size=2961466 2024-09-13T00:00:41.891 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.890+0000 7fb8f2ce0640 1 > prepared operation key=5GM88ERM89KT tag=TWIRHA662K8X type=del size=3703340 2024-09-13T00:00:41.898 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.898+0000 7fb8f2ce0640 -1 < failed to complete operation key=PVYZQLGYT3XU tag=ZBNNDHR3OUFX type=del size=1144435: (22) Invalid argument 2024-09-13T00:00:41.904 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.902+0000 7fb8f2ce0640 1 > prepared operation key=4K76U69GZ9AE tag=JQ71W38JGLGE type=del size=3117722 2024-09-13T00:00:41.910 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.910+0000 7fb8f2ce0640 1 < completed part key=_multipart_85YEW8N43FTA.2~H16XBBFRPV0V.0 size=991598 2024-09-13T00:00:41.919 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.918+0000 7fb8f2ce0640 1 < completed multipart upload key=85YEW8N43FTA upload=H16XBBFRPV0V tag=PTH66524EQMH type=write size=2040174 2024-09-13T00:00:41.925 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.926+0000 7fb8f2ce0640 1 > prepared operation key=ZFHN2SNTZ5W7 tag=F1WNYB33IO84 type=del size=549367 2024-09-13T00:00:41.926 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.926+0000 7fb8f2ce0640 1 < canceled part key=_multipart_8Z6U7OOA0GBH.2~D71XWPUZM836.0 size=689160 2024-09-13T00:00:41.936 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.934+0000 7fb8f2ce0640 1 < canceled multipart upload key=8Z6U7OOA0GBH upload=D71XWPUZM836 tag=1JAPKT2EKWTS type=write size=1737736 2024-09-13T00:00:41.941 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.942+0000 7fb8f2ce0640 1 > prepared operation key=P8OYS5ZE1NX0 tag=PO8J2AJ3IVM0 type=del size=2036949 2024-09-13T00:00:41.947 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.946+0000 7fb8f2ce0640 1 < completed delete operation key=1HX06CCZJ1DD tag=Q60DPHFSZJB9 type=del 2024-09-13T00:00:41.952 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.950+0000 7fb8f2ce0640 1 > prepared operation key=NQJQC8CEAC15 tag=UWX4ARLGE3PG type=del size=4154126 2024-09-13T00:00:41.958 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.958+0000 7fb8f2ce0640 1 < completed part key=_multipart_F0QCVR9E08WE.2~0BXGEV5TL21H.0 size=1048576 2024-09-13T00:00:41.962 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.962+0000 7fb8f2ce0640 1 < completed part key=_multipart_F0QCVR9E08WE.2~0BXGEV5TL21H.1 size=654861 2024-09-13T00:00:41.970 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.970+0000 7fb8f2ce0640 1 < completed multipart upload key=F0QCVR9E08WE upload=0BXGEV5TL21H tag=3V9QL5LKEC4D type=write size=2752013 2024-09-13T00:00:41.976 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.974+0000 7fb8f2ce0640 1 > prepared operation key=N3Z5XIG3TZRQ tag=T4DVMDQU48HN type=del size=1765543 2024-09-13T00:00:41.981 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.982+0000 7fb8f2ce0640 1 < completed delete operation key=NQJQC8CEAC15 tag=XT5O7QIUR9ZW type=del 2024-09-13T00:00:41.987 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.986+0000 7fb8f2ce0640 1 > prepared operation key=W8XZ26311RJX tag=4IUQ1I0M5F99 type=write size=922486 2024-09-13T00:00:41.997 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:41.998+0000 7fb8f2ce0640 1 < suggested remove operation key=PVYZQLGYT3XU tag=TPFUVMVVVSY8 type=del 2024-09-13T00:00:42.007 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.006+0000 7fb8f2ce0640 1 > prepared part key=_multipart_44EDYFT9VF8Y.2~66BDBEWMGMAQ.0 size=1048576 2024-09-13T00:00:42.011 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.010+0000 7fb8f2ce0640 1 > prepared part key=_multipart_44EDYFT9VF8Y.2~66BDBEWMGMAQ.1 size=1048576 2024-09-13T00:00:42.016 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.014+0000 7fb8f2ce0640 1 > prepared part key=_multipart_44EDYFT9VF8Y.2~66BDBEWMGMAQ.2 size=404233 2024-09-13T00:00:42.016 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.014+0000 7fb8f2ce0640 1 > prepared multipart upload key=44EDYFT9VF8Y upload=66BDBEWMGMAQ tag=I7BZ7PK136S9 type=write size=3549961 2024-09-13T00:00:42.021 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.022+0000 7fb8f2ce0640 1 < completed part key=_multipart_H5EIP0E9CBWX.2~YZXWEFD98SDJ.0 size=1048576 2024-09-13T00:00:42.025 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.026+0000 7fb8f2ce0640 1 < completed part key=_multipart_H5EIP0E9CBWX.2~YZXWEFD98SDJ.1 size=1048576 2024-09-13T00:00:42.025 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.026+0000 7fb8f2ce0640 1 < canceled part key=_multipart_H5EIP0E9CBWX.2~YZXWEFD98SDJ.2 size=1015768 2024-09-13T00:00:42.034 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.034+0000 7fb8f2ce0640 1 < canceled multipart upload key=H5EIP0E9CBWX upload=YZXWEFD98SDJ tag=RCOEONEASG8D type=write size=4161496 2024-09-13T00:00:42.044 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.042+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~HWGU7MLD4VMO.0 size=1048576 2024-09-13T00:00:42.048 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.046+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~HWGU7MLD4VMO.1 size=979047 2024-09-13T00:00:42.048 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.046+0000 7fb8f2ce0640 1 > prepared multipart upload key=2AHYRZUDFHP4 upload=HWGU7MLD4VMO tag=T9PQE9Q5RT6K type=write size=3076199 2024-09-13T00:00:42.054 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.054+0000 7fb8f2ce0640 1 < completed part key=_multipart_44EDYFT9VF8Y.2~ZGDROA49RZNR.0 size=359315 2024-09-13T00:00:42.062 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.062+0000 7fb8f2ce0640 1 < completed multipart upload key=44EDYFT9VF8Y upload=ZGDROA49RZNR tag=20UPKO74LFKZ type=write size=1407891 2024-09-13T00:00:42.067 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.066+0000 7fb8f2ce0640 1 > prepared operation key=70F7D20HI504 tag=S5VXHZSWICGJ type=del size=2586354 2024-09-13T00:00:42.072 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.070+0000 7fb8f2ce0640 1 < completed delete operation key=63FJDWBO2T9S tag=6K1TPOW6BP0I type=del 2024-09-13T00:00:42.082 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.082+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~PQ5HH2P4AVP7.0 size=1048576 2024-09-13T00:00:42.086 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.086+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~PQ5HH2P4AVP7.1 size=1048576 2024-09-13T00:00:42.090 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.090+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~PQ5HH2P4AVP7.2 size=42570 2024-09-13T00:00:42.090 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.090+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZTK9AYBSLCOZ upload=PQ5HH2P4AVP7 tag=V4ERQT8JY5UJ type=write size=3188298 2024-09-13T00:00:42.095 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.094+0000 7fb8f2ce0640 1 < completed delete operation key=FX0NCGLK0KPJ tag=72TWREY7FR7T type=del 2024-09-13T00:00:42.105 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.102+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~6UAY3EOACPSL.0 size=250873 2024-09-13T00:00:42.105 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.102+0000 7fb8f2ce0640 1 > prepared multipart upload key=2AHYRZUDFHP4 upload=6UAY3EOACPSL tag=L1VFITXYG0NO type=write size=1299449 2024-09-13T00:00:42.110 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.110+0000 7fb8f2ce0640 1 < completed write operation key=8Z6U7OOA0GBH tag=PT22HNWO9WWB type=write size=453388 2024-09-13T00:00:42.120 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.118+0000 7fb8f2ce0640 1 > prepared part key=_multipart_4K76U69GZ9AE.2~P2UDPS0MCS1B.0 size=1048576 2024-09-13T00:00:42.124 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.122+0000 7fb8f2ce0640 1 > prepared part key=_multipart_4K76U69GZ9AE.2~P2UDPS0MCS1B.1 size=553841 2024-09-13T00:00:42.124 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.122+0000 7fb8f2ce0640 1 > prepared multipart upload key=4K76U69GZ9AE upload=P2UDPS0MCS1B tag=8CGF0IHHI5BL type=write size=2650993 2024-09-13T00:00:42.130 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.130+0000 7fb8f2ce0640 -1 < failed to complete operation key=QMZREZ1QEDNJ tag=I2HK3GU0CXMF type=del size=2437476: (22) Invalid argument 2024-09-13T00:00:42.135 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.134+0000 7fb8f2ce0640 1 > prepared operation key=8Z6U7OOA0GBH tag=ZSIADS8UTJYU type=del size=1780004 2024-09-13T00:00:42.140 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.138+0000 7fb8f2ce0640 1 < completed write operation key=FX0NCGLK0KPJ tag=4HO7661VDWSD type=write size=618258 2024-09-13T00:00:42.150 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.150+0000 7fb8f2ce0640 1 > prepared part key=_multipart_F0QCVR9E08WE.2~LKDS006SGG3A.0 size=947053 2024-09-13T00:00:42.150 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.150+0000 7fb8f2ce0640 1 > prepared multipart upload key=F0QCVR9E08WE upload=LKDS006SGG3A tag=G965W96RU7V1 type=write size=1995629 2024-09-13T00:00:42.151 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.150+0000 7fb8f2ce0640 1 < canceled part key=_multipart_PVYZQLGYT3XU.2~LFGFYODGNREH.0 size=772139 2024-09-13T00:00:42.159 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.158+0000 7fb8f2ce0640 1 < canceled multipart upload key=PVYZQLGYT3XU upload=LFGFYODGNREH tag=0VEKXCHVIXZ0 type=write size=1820715 2024-09-13T00:00:42.164 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.162+0000 7fb8f2ce0640 1 > prepared operation key=70F7D20HI504 tag=4775OA3EFVAK type=del size=800798 2024-09-13T00:00:42.169 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.170+0000 7fb8f2ce0640 1 < completed delete operation key=5GM88ERM89KT tag=TWIRHA662K8X type=del 2024-09-13T00:00:42.174 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.174+0000 7fb8f2ce0640 1 > prepared operation key=5GM88ERM89KT tag=1MMHW3UW3VX3 type=del size=957667 2024-09-13T00:00:42.179 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.178+0000 7fb8f2ce0640 1 < completed delete operation key=4K76U69GZ9AE tag=JQ71W38JGLGE type=del 2024-09-13T00:00:42.188 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.186+0000 7fb8f2ce0640 1 > prepared part key=_multipart_44EDYFT9VF8Y.2~WTGK7303RCC7.0 size=1048576 2024-09-13T00:00:42.192 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.190+0000 7fb8f2ce0640 1 > prepared part key=_multipart_44EDYFT9VF8Y.2~WTGK7303RCC7.1 size=679620 2024-09-13T00:00:42.193 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.190+0000 7fb8f2ce0640 1 > prepared multipart upload key=44EDYFT9VF8Y upload=WTGK7303RCC7 tag=XLSAFIV7J6A1 type=write size=2776772 2024-09-13T00:00:42.198 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.198+0000 7fb8f2ce0640 1 < canceled operation key=ZFHN2SNTZ5W7 tag=F1WNYB33IO84 type=del size=549367 2024-09-13T00:00:42.202 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.202+0000 7fb8f2ce0640 1 > prepared operation key=O4MZ4UWNXME9 tag=U8NICHSJAV9N type=write size=1040674 2024-09-13T00:00:42.208 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.206+0000 7fb8f2ce0640 1 < completed delete operation key=P8OYS5ZE1NX0 tag=PO8J2AJ3IVM0 type=del 2024-09-13T00:00:42.213 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.210+0000 7fb8f2ce0640 1 > prepared operation key=H4M1TCN5W6XU tag=WOITC8ZAZ6C9 type=write size=801084 2024-09-13T00:00:42.217 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.218+0000 7fb8f2ce0640 1 < completed delete operation key=NQJQC8CEAC15 tag=UWX4ARLGE3PG type=del 2024-09-13T00:00:42.222 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.222+0000 7fb8f2ce0640 1 > prepared operation key=85YEW8N43FTA tag=PPYQP983JCMH type=del size=1662965 2024-09-13T00:00:42.227 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.226+0000 7fb8f2ce0640 1 < completed delete operation key=N3Z5XIG3TZRQ tag=T4DVMDQU48HN type=del 2024-09-13T00:00:42.232 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.230+0000 7fb8f2ce0640 1 > prepared operation key=P8OYS5ZE1NX0 tag=79O4RR2AQPI3 type=del size=526251 2024-09-13T00:00:42.237 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.238+0000 7fb8f2ce0640 1 < completed write operation key=W8XZ26311RJX tag=4IUQ1I0M5F99 type=write size=922486 2024-09-13T00:00:42.242 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.242+0000 7fb8f2ce0640 1 > prepared operation key=44EDYFT9VF8Y tag=0B5CRAC8IQHQ type=write size=16106 2024-09-13T00:00:42.247 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.246+0000 7fb8f2ce0640 1 < completed part key=_multipart_44EDYFT9VF8Y.2~66BDBEWMGMAQ.0 size=1048576 2024-09-13T00:00:42.252 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.250+0000 7fb8f2ce0640 1 < completed part key=_multipart_44EDYFT9VF8Y.2~66BDBEWMGMAQ.1 size=1048576 2024-09-13T00:00:42.256 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.254+0000 7fb8f2ce0640 1 < completed part key=_multipart_44EDYFT9VF8Y.2~66BDBEWMGMAQ.2 size=404233 2024-09-13T00:00:42.264 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.262+0000 7fb8f2ce0640 1 < completed multipart upload key=44EDYFT9VF8Y upload=66BDBEWMGMAQ tag=I7BZ7PK136S9 type=write size=3549961 2024-09-13T00:00:42.269 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.270+0000 7fb8f2ce0640 1 > prepared operation key=2TWD9YLMSYJI tag=3FW4XOUF58VH type=del size=1298793 2024-09-13T00:00:42.275 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.274+0000 7fb8f2ce0640 1 < completed part key=_multipart_2AHYRZUDFHP4.2~HWGU7MLD4VMO.0 size=1048576 2024-09-13T00:00:42.279 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.278+0000 7fb8f2ce0640 1 < completed part key=_multipart_2AHYRZUDFHP4.2~HWGU7MLD4VMO.1 size=979047 2024-09-13T00:00:42.288 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.286+0000 7fb8f2ce0640 1 < completed multipart upload key=2AHYRZUDFHP4 upload=HWGU7MLD4VMO tag=T9PQE9Q5RT6K type=write size=3076199 2024-09-13T00:00:42.293 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.290+0000 7fb8f2ce0640 1 > prepared operation key=P8OYS5ZE1NX0 tag=RMIOW3PEKVQY type=del size=3938219 2024-09-13T00:00:42.298 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.298+0000 7fb8f2ce0640 1 < canceled operation key=70F7D20HI504 tag=S5VXHZSWICGJ type=del size=2586354 2024-09-13T00:00:42.307 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.306+0000 7fb8f2ce0640 1 > prepared part key=_multipart_4K76U69GZ9AE.2~C3QM07IOIP6D.0 size=1048576 2024-09-13T00:00:42.311 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.310+0000 7fb8f2ce0640 1 > prepared part key=_multipart_4K76U69GZ9AE.2~C3QM07IOIP6D.1 size=1048576 2024-09-13T00:00:42.316 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.314+0000 7fb8f2ce0640 1 > prepared part key=_multipart_4K76U69GZ9AE.2~C3QM07IOIP6D.2 size=587504 2024-09-13T00:00:42.316 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.314+0000 7fb8f2ce0640 1 > prepared multipart upload key=4K76U69GZ9AE upload=C3QM07IOIP6D tag=Y0YJP28AW12N type=write size=3733232 2024-09-13T00:00:42.321 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.322+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~PQ5HH2P4AVP7.0 size=1048576 2024-09-13T00:00:42.325 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.326+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~PQ5HH2P4AVP7.1 size=1048576 2024-09-13T00:00:42.329 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.330+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~PQ5HH2P4AVP7.2 size=42570 2024-09-13T00:00:42.339 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.338+0000 7fb8f2ce0640 1 < completed multipart upload key=ZTK9AYBSLCOZ upload=PQ5HH2P4AVP7 tag=V4ERQT8JY5UJ type=write size=3188298 2024-09-13T00:00:42.348 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.346+0000 7fb8f2ce0640 1 > prepared part key=_multipart_QMZREZ1QEDNJ.2~ON10061QTIVY.0 size=78418 2024-09-13T00:00:42.348 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.346+0000 7fb8f2ce0640 1 > prepared multipart upload key=QMZREZ1QEDNJ upload=ON10061QTIVY tag=JQGOKN90NCPF type=write size=1126994 2024-09-13T00:00:42.353 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.354+0000 7fb8f2ce0640 1 < completed part key=_multipart_2AHYRZUDFHP4.2~6UAY3EOACPSL.0 size=250873 2024-09-13T00:00:42.362 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.362+0000 7fb8f2ce0640 1 < completed multipart upload key=2AHYRZUDFHP4 upload=6UAY3EOACPSL tag=L1VFITXYG0NO type=write size=1299449 2024-09-13T00:00:42.367 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.366+0000 7fb8f2ce0640 1 > prepared operation key=EJN6HR8I7XQF tag=OKVD0LJ6Q7M1 type=del size=842064 2024-09-13T00:00:42.372 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.370+0000 7fb8f2ce0640 1 < completed part key=_multipart_4K76U69GZ9AE.2~P2UDPS0MCS1B.0 size=1048576 2024-09-13T00:00:42.376 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.374+0000 7fb8f2ce0640 1 < completed part key=_multipart_4K76U69GZ9AE.2~P2UDPS0MCS1B.1 size=553841 2024-09-13T00:00:42.385 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.382+0000 7fb8f2ce0640 1 < completed multipart upload key=4K76U69GZ9AE upload=P2UDPS0MCS1B tag=8CGF0IHHI5BL type=write size=2650993 2024-09-13T00:00:42.394 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.394+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~AMAB7FF3ISVE.0 size=705014 2024-09-13T00:00:42.394 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.394+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZTK9AYBSLCOZ upload=AMAB7FF3ISVE tag=WX24XNBX5A5Q type=write size=1753590 2024-09-13T00:00:42.399 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.398+0000 7fb8f2ce0640 1 < canceled operation key=8Z6U7OOA0GBH tag=ZSIADS8UTJYU type=del size=1780004 2024-09-13T00:00:42.406 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.406+0000 7fb8f2ce0640 1 > prepared operation key=ZTK9AYBSLCOZ tag=SRSZ1S1KK212 type=del size=4024114 2024-09-13T00:00:42.411 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.410+0000 7fb8f2ce0640 1 < completed part key=_multipart_F0QCVR9E08WE.2~LKDS006SGG3A.0 size=947053 2024-09-13T00:00:42.420 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.418+0000 7fb8f2ce0640 1 < completed multipart upload key=F0QCVR9E08WE upload=LKDS006SGG3A tag=G965W96RU7V1 type=write size=1995629 2024-09-13T00:00:42.425 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.426+0000 7fb8f2ce0640 1 > prepared operation key=70F7D20HI504 tag=9OGA98A1EUEY type=del size=2712804 2024-09-13T00:00:42.435 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.434+0000 7fb8f2ce0640 1 < suggested remove operation key=70F7D20HI504 tag=4775OA3EFVAK type=del 2024-09-13T00:00:42.444 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.442+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H5EIP0E9CBWX.2~4TI2H7IEG5Y4.0 size=1048576 2024-09-13T00:00:42.448 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.446+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H5EIP0E9CBWX.2~4TI2H7IEG5Y4.1 size=1048576 2024-09-13T00:00:42.452 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.450+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H5EIP0E9CBWX.2~4TI2H7IEG5Y4.2 size=867143 2024-09-13T00:00:42.452 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.450+0000 7fb8f2ce0640 1 > prepared multipart upload key=H5EIP0E9CBWX upload=4TI2H7IEG5Y4 tag=ZVL5P1NR2Y3D type=write size=4012871 2024-09-13T00:00:42.457 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.454+0000 7fb8f2ce0640 1 < completed delete operation key=5GM88ERM89KT tag=1MMHW3UW3VX3 type=del 2024-09-13T00:00:42.461 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.462+0000 7fb8f2ce0640 1 > prepared operation key=N3Z5XIG3TZRQ tag=QMA7483307E7 type=del size=997916 2024-09-13T00:00:42.466 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.466+0000 7fb8f2ce0640 1 < completed part key=_multipart_44EDYFT9VF8Y.2~WTGK7303RCC7.0 size=1048576 2024-09-13T00:00:42.470 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.470+0000 7fb8f2ce0640 1 < completed part key=_multipart_44EDYFT9VF8Y.2~WTGK7303RCC7.1 size=679620 2024-09-13T00:00:42.479 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.478+0000 7fb8f2ce0640 1 < completed multipart upload key=44EDYFT9VF8Y upload=WTGK7303RCC7 tag=XLSAFIV7J6A1 type=write size=2776772 2024-09-13T00:00:42.488 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.486+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~EX8HUNFQQKEM.0 size=1048576 2024-09-13T00:00:42.493 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.494+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~EX8HUNFQQKEM.1 size=982239 2024-09-13T00:00:42.493 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.494+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZTK9AYBSLCOZ upload=EX8HUNFQQKEM tag=OV5OLHEG0ABF type=write size=3079391 2024-09-13T00:00:42.498 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.498+0000 7fb8f2ce0640 1 < completed write operation key=O4MZ4UWNXME9 tag=U8NICHSJAV9N type=write size=1040674 2024-09-13T00:00:42.503 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.502+0000 7fb8f2ce0640 1 > prepared operation key=P8OYS5ZE1NX0 tag=UI9MQC6U09EU type=del size=613554 2024-09-13T00:00:42.509 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.506+0000 7fb8f2ce0640 1 < completed write operation key=H4M1TCN5W6XU tag=WOITC8ZAZ6C9 type=write size=801084 2024-09-13T00:00:42.513 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.514+0000 7fb8f2ce0640 1 > prepared operation key=63FJDWBO2T9S tag=Q5ZKPAURUP4F type=del size=1665953 2024-09-13T00:00:42.518 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.518+0000 7fb8f2ce0640 1 < completed delete operation key=85YEW8N43FTA tag=PPYQP983JCMH type=del 2024-09-13T00:00:42.524 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.522+0000 7fb8f2ce0640 1 > prepared operation key=NQJQC8CEAC15 tag=RCPTR14HTJZP type=del size=1405257 2024-09-13T00:00:42.530 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.530+0000 7fb8f2ce0640 1 < canceled operation key=P8OYS5ZE1NX0 tag=79O4RR2AQPI3 type=del size=526251 2024-09-13T00:00:42.535 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.534+0000 7fb8f2ce0640 1 > prepared operation key=63FJDWBO2T9S tag=I3POWQLBKR4U type=del size=2216155 2024-09-13T00:00:42.540 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.538+0000 7fb8f2ce0640 1 < canceled operation key=44EDYFT9VF8Y tag=0B5CRAC8IQHQ type=write size=16106 2024-09-13T00:00:42.550 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.550+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~ISL3420LAYXN.0 size=1048576 2024-09-13T00:00:42.554 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.554+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~ISL3420LAYXN.1 size=1048576 2024-09-13T00:00:42.558 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.558+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~ISL3420LAYXN.2 size=407836 2024-09-13T00:00:42.559 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.558+0000 7fb8f2ce0640 1 > prepared multipart upload key=PVYZQLGYT3XU upload=ISL3420LAYXN tag=5873P6GQU5RB type=write size=3553564 2024-09-13T00:00:42.563 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.562+0000 7fb8f2ce0640 1 < completed delete operation key=2TWD9YLMSYJI tag=3FW4XOUF58VH type=del 2024-09-13T00:00:42.569 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.566+0000 7fb8f2ce0640 1 > prepared operation key=44EDYFT9VF8Y tag=IS67EQMVXTR7 type=del size=2840985 2024-09-13T00:00:42.574 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.574+0000 7fb8f2ce0640 1 < completed delete operation key=P8OYS5ZE1NX0 tag=RMIOW3PEKVQY type=del 2024-09-13T00:00:42.583 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.582+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~HJ7MT6IF79GK.0 size=642455 2024-09-13T00:00:42.584 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.582+0000 7fb8f2ce0640 1 > prepared multipart upload key=7OX5OJZ8GXU5 upload=HJ7MT6IF79GK tag=BM4EIUZLAQG4 type=write size=1691031 2024-09-13T00:00:42.588 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.586+0000 7fb8f2ce0640 1 < completed part key=_multipart_4K76U69GZ9AE.2~C3QM07IOIP6D.0 size=1048576 2024-09-13T00:00:42.592 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.590+0000 7fb8f2ce0640 1 < completed part key=_multipart_4K76U69GZ9AE.2~C3QM07IOIP6D.1 size=1048576 2024-09-13T00:00:42.596 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.594+0000 7fb8f2ce0640 1 < completed part key=_multipart_4K76U69GZ9AE.2~C3QM07IOIP6D.2 size=587504 2024-09-13T00:00:42.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:00:42 smithi181 bash[95505]: ts=2024-09-13T00:00:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:00:42.605 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.602+0000 7fb8f2ce0640 1 < completed multipart upload key=4K76U69GZ9AE upload=C3QM07IOIP6D tag=Y0YJP28AW12N type=write size=3733232 2024-09-13T00:00:42.609 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.610+0000 7fb8f2ce0640 1 > prepared operation key=5GM88ERM89KT tag=11B1KXGW1AIG type=del size=818566 2024-09-13T00:00:42.614 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.614+0000 7fb8f2ce0640 1 < completed part key=_multipart_QMZREZ1QEDNJ.2~ON10061QTIVY.0 size=78418 2024-09-13T00:00:42.622 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.622+0000 7fb8f2ce0640 1 < completed multipart upload key=QMZREZ1QEDNJ upload=ON10061QTIVY tag=JQGOKN90NCPF type=write size=1126994 2024-09-13T00:00:42.631 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.630+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~7XV8LS17LEFB.0 size=541047 2024-09-13T00:00:42.631 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.630+0000 7fb8f2ce0640 1 > prepared multipart upload key=8Z6U7OOA0GBH upload=7XV8LS17LEFB tag=5JU535H7B6E2 type=write size=1589623 2024-09-13T00:00:42.637 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.634+0000 7fb8f2ce0640 1 < completed delete operation key=EJN6HR8I7XQF tag=OKVD0LJ6Q7M1 type=del 2024-09-13T00:00:42.645 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.646+0000 7fb8f2ce0640 1 > prepared part key=_multipart_5GM88ERM89KT.2~DUX1BYWFXTIC.0 size=537270 2024-09-13T00:00:42.645 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.646+0000 7fb8f2ce0640 1 > prepared multipart upload key=5GM88ERM89KT upload=DUX1BYWFXTIC tag=O5XA3MMOSM99 type=write size=1585846 2024-09-13T00:00:42.650 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.650+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~AMAB7FF3ISVE.0 size=705014 2024-09-13T00:00:42.659 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.658+0000 7fb8f2ce0640 1 < completed multipart upload key=ZTK9AYBSLCOZ upload=AMAB7FF3ISVE tag=WX24XNBX5A5Q type=write size=1753590 2024-09-13T00:00:42.663 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.662+0000 7fb8f2ce0640 1 > prepared operation key=EUE3LT0PDIXV tag=VN8033RXQCVI type=del size=3280232 2024-09-13T00:00:42.669 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.670+0000 7fb8f2ce0640 1 < completed delete operation key=ZTK9AYBSLCOZ tag=SRSZ1S1KK212 type=del 2024-09-13T00:00:42.674 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.674+0000 7fb8f2ce0640 1 > prepared operation key=5GM88ERM89KT tag=8CLR2YJMHZU0 type=del size=3651029 2024-09-13T00:00:42.680 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.678+0000 7fb8f2ce0640 -1 < failed to complete operation key=70F7D20HI504 tag=9OGA98A1EUEY type=del size=2712804: (22) Invalid argument 2024-09-13T00:00:42.686 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.686+0000 7fb8f2ce0640 1 > prepared operation key=7OX5OJZ8GXU5 tag=3226IF28FGSH type=del size=3055324 2024-09-13T00:00:42.686 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.686+0000 7fb8f2ce0640 1 < canceled part key=_multipart_H5EIP0E9CBWX.2~4TI2H7IEG5Y4.0 size=1048576 2024-09-13T00:00:42.691 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.690+0000 7fb8f2ce0640 1 < completed part key=_multipart_H5EIP0E9CBWX.2~4TI2H7IEG5Y4.1 size=1048576 2024-09-13T00:00:42.695 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.694+0000 7fb8f2ce0640 1 < completed part key=_multipart_H5EIP0E9CBWX.2~4TI2H7IEG5Y4.2 size=867143 2024-09-13T00:00:42.703 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.702+0000 7fb8f2ce0640 1 < canceled multipart upload key=H5EIP0E9CBWX upload=4TI2H7IEG5Y4 tag=ZVL5P1NR2Y3D type=write size=4012871 2024-09-13T00:00:42.708 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.706+0000 7fb8f2ce0640 1 > prepared operation key=W8XZ26311RJX tag=ZC88KP19ORUR type=del size=3642505 2024-09-13T00:00:42.713 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.714+0000 7fb8f2ce0640 1 < completed delete operation key=N3Z5XIG3TZRQ tag=QMA7483307E7 type=del 2024-09-13T00:00:42.723 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.722+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~ZY48VWNF3FO9.0 size=221645 2024-09-13T00:00:42.723 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.722+0000 7fb8f2ce0640 1 > prepared multipart upload key=8MPUVD37EDN9 upload=ZY48VWNF3FO9 tag=MV23NEY0AQSS type=write size=1270221 2024-09-13T00:00:42.729 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.726+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~EX8HUNFQQKEM.0 size=1048576 2024-09-13T00:00:42.734 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.734+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~EX8HUNFQQKEM.1 size=982239 2024-09-13T00:00:42.742 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.742+0000 7fb8f2ce0640 1 < completed multipart upload key=ZTK9AYBSLCOZ upload=EX8HUNFQQKEM tag=OV5OLHEG0ABF type=write size=3079391 2024-09-13T00:00:42.747 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.746+0000 7fb8f2ce0640 1 > prepared operation key=2AHYRZUDFHP4 tag=BFXOEETY384Y type=write size=623298 2024-09-13T00:00:42.753 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.750+0000 7fb8f2ce0640 1 < completed delete operation key=P8OYS5ZE1NX0 tag=UI9MQC6U09EU type=del 2024-09-13T00:00:42.758 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.758+0000 7fb8f2ce0640 1 > prepared operation key=5GM88ERM89KT tag=V2S4842UZXEL type=del size=1918393 2024-09-13T00:00:42.768 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.766+0000 7fb8f2ce0640 1 < suggested update operation key=63FJDWBO2T9S tag=Q5ZKPAURUP4F type=del size=1665953 2024-09-13T00:00:42.773 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.770+0000 7fb8f2ce0640 1 > prepared operation key=W8XZ26311RJX tag=HF9YJ6FLIPOO type=del size=3399600 2024-09-13T00:00:42.778 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.778+0000 7fb8f2ce0640 1 < completed delete operation key=NQJQC8CEAC15 tag=RCPTR14HTJZP type=del 2024-09-13T00:00:42.788 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.786+0000 7fb8f2ce0640 1 > prepared part key=_multipart_85YEW8N43FTA.2~F4F7GZ0BPGPQ.0 size=1048576 2024-09-13T00:00:42.793 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.790+0000 7fb8f2ce0640 1 > prepared part key=_multipart_85YEW8N43FTA.2~F4F7GZ0BPGPQ.1 size=1048576 2024-09-13T00:00:42.797 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.798+0000 7fb8f2ce0640 1 > prepared part key=_multipart_85YEW8N43FTA.2~F4F7GZ0BPGPQ.2 size=660980 2024-09-13T00:00:42.797 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.798+0000 7fb8f2ce0640 1 > prepared multipart upload key=85YEW8N43FTA upload=F4F7GZ0BPGPQ tag=XHC2QHJS2WO4 type=write size=3806708 2024-09-13T00:00:42.804 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.802+0000 7fb8f2ce0640 -1 < failed to complete operation key=63FJDWBO2T9S tag=I3POWQLBKR4U type=del size=2216155: (22) Invalid argument 2024-09-13T00:00:42.809 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.810+0000 7fb8f2ce0640 1 > prepared operation key=63FJDWBO2T9S tag=CHRYQ7BXYS5H type=del size=411128 2024-09-13T00:00:42.815 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.814+0000 7fb8f2ce0640 1 < completed part key=_multipart_PVYZQLGYT3XU.2~ISL3420LAYXN.0 size=1048576 2024-09-13T00:00:42.820 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.818+0000 7fb8f2ce0640 1 < completed part key=_multipart_PVYZQLGYT3XU.2~ISL3420LAYXN.1 size=1048576 2024-09-13T00:00:42.825 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.822+0000 7fb8f2ce0640 1 < completed part key=_multipart_PVYZQLGYT3XU.2~ISL3420LAYXN.2 size=407836 2024-09-13T00:00:42.833 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.830+0000 7fb8f2ce0640 1 < completed multipart upload key=PVYZQLGYT3XU upload=ISL3420LAYXN tag=5873P6GQU5RB type=write size=3553564 2024-09-13T00:00:42.838 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.838+0000 7fb8f2ce0640 1 > prepared operation key=85YEW8N43FTA tag=4TCN31OVLJS3 type=del size=1895015 2024-09-13T00:00:42.843 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.842+0000 7fb8f2ce0640 1 < completed delete operation key=44EDYFT9VF8Y tag=IS67EQMVXTR7 type=del 2024-09-13T00:00:42.848 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.846+0000 7fb8f2ce0640 1 > prepared operation key=70F7D20HI504 tag=U6NI254R4138 type=write size=572550 2024-09-13T00:00:42.853 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.854+0000 7fb8f2ce0640 1 < completed part key=_multipart_7OX5OJZ8GXU5.2~HJ7MT6IF79GK.0 size=642455 2024-09-13T00:00:42.862 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.862+0000 7fb8f2ce0640 1 < completed multipart upload key=7OX5OJZ8GXU5 upload=HJ7MT6IF79GK tag=BM4EIUZLAQG4 type=write size=1691031 2024-09-13T00:00:42.872 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.870+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EUE3LT0PDIXV.2~W6OIW3VJ8IHQ.0 size=1048576 2024-09-13T00:00:42.877 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.874+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EUE3LT0PDIXV.2~W6OIW3VJ8IHQ.1 size=1048576 2024-09-13T00:00:42.881 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.882+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EUE3LT0PDIXV.2~W6OIW3VJ8IHQ.2 size=907762 2024-09-13T00:00:42.881 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.882+0000 7fb8f2ce0640 1 > prepared multipart upload key=EUE3LT0PDIXV upload=W6OIW3VJ8IHQ tag=OC2D4ZUMYKNF type=write size=4053490 2024-09-13T00:00:42.886 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.886+0000 7fb8f2ce0640 1 < canceled operation key=5GM88ERM89KT tag=11B1KXGW1AIG type=del size=818566 2024-09-13T00:00:42.892 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.890+0000 7fb8f2ce0640 1 > prepared operation key=4K76U69GZ9AE tag=51J5DUWOGWG9 type=del size=697803 2024-09-13T00:00:42.898 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.898+0000 7fb8f2ce0640 1 < completed part key=_multipart_8Z6U7OOA0GBH.2~7XV8LS17LEFB.0 size=541047 2024-09-13T00:00:42.906 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.906+0000 7fb8f2ce0640 1 < completed multipart upload key=8Z6U7OOA0GBH upload=7XV8LS17LEFB tag=5JU535H7B6E2 type=write size=1589623 2024-09-13T00:00:42.916 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.914+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~9PTJZEPCQMKG.0 size=60343 2024-09-13T00:00:42.916 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.914+0000 7fb8f2ce0640 1 > prepared multipart upload key=W8XZ26311RJX upload=9PTJZEPCQMKG tag=TOKXSV45ZBQ6 type=write size=1108919 2024-09-13T00:00:42.921 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.922+0000 7fb8f2ce0640 1 < completed part key=_multipart_5GM88ERM89KT.2~DUX1BYWFXTIC.0 size=537270 2024-09-13T00:00:42.929 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.930+0000 7fb8f2ce0640 1 < completed multipart upload key=5GM88ERM89KT upload=DUX1BYWFXTIC tag=O5XA3MMOSM99 type=write size=1585846 2024-09-13T00:00:42.938 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.938+0000 7fb8f2ce0640 1 > prepared part key=_multipart_5GM88ERM89KT.2~5TXOFSOWME7I.0 size=1048576 2024-09-13T00:00:42.943 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.942+0000 7fb8f2ce0640 1 > prepared part key=_multipart_5GM88ERM89KT.2~5TXOFSOWME7I.1 size=945673 2024-09-13T00:00:42.943 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.942+0000 7fb8f2ce0640 1 > prepared multipart upload key=5GM88ERM89KT upload=5TXOFSOWME7I tag=W4TYGZMT3662 type=write size=3042825 2024-09-13T00:00:42.948 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.946+0000 7fb8f2ce0640 1 < canceled operation key=EUE3LT0PDIXV tag=VN8033RXQCVI type=del size=3280232 2024-09-13T00:00:42.953 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.950+0000 7fb8f2ce0640 1 > prepared operation key=4K76U69GZ9AE tag=L8HSAAHR4TS1 type=del size=383280 2024-09-13T00:00:42.958 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.958+0000 7fb8f2ce0640 1 < completed delete operation key=5GM88ERM89KT tag=8CLR2YJMHZU0 type=del 2024-09-13T00:00:42.963 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.962+0000 7fb8f2ce0640 1 > prepared operation key=8MPUVD37EDN9 tag=6UZWZ0P2JRFT type=del size=1819168 2024-09-13T00:00:42.968 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.966+0000 7fb8f2ce0640 1 < completed delete operation key=7OX5OJZ8GXU5 tag=3226IF28FGSH type=del 2024-09-13T00:00:42.979 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.978+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NX26SI4F2ZMB.2~FL08RJER4LZ4.0 size=1048576 2024-09-13T00:00:42.983 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.982+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NX26SI4F2ZMB.2~FL08RJER4LZ4.1 size=1048576 2024-09-13T00:00:42.987 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.986+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NX26SI4F2ZMB.2~FL08RJER4LZ4.2 size=487347 2024-09-13T00:00:42.987 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.986+0000 7fb8f2ce0640 1 > prepared multipart upload key=NX26SI4F2ZMB upload=FL08RJER4LZ4 tag=7GAXA03DNFS2 type=write size=3633075 2024-09-13T00:00:42.992 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:42.990+0000 7fb8f2ce0640 1 < completed delete operation key=W8XZ26311RJX tag=ZC88KP19ORUR type=del 2024-09-13T00:00:43.001 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.002+0000 7fb8f2ce0640 1 > prepared part key=_multipart_F0QCVR9E08WE.2~QDACYZKIG1NN.0 size=1048576 2024-09-13T00:00:43.005 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.006+0000 7fb8f2ce0640 1 > prepared part key=_multipart_F0QCVR9E08WE.2~QDACYZKIG1NN.1 size=379358 2024-09-13T00:00:43.005 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.006+0000 7fb8f2ce0640 1 > prepared multipart upload key=F0QCVR9E08WE upload=QDACYZKIG1NN tag=S1VLW3XC2A9G type=write size=2476510 2024-09-13T00:00:43.010 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.010+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~ZY48VWNF3FO9.0 size=221645 2024-09-13T00:00:43.019 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.018+0000 7fb8f2ce0640 1 < completed multipart upload key=8MPUVD37EDN9 upload=ZY48VWNF3FO9 tag=MV23NEY0AQSS type=write size=1270221 2024-09-13T00:00:43.024 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.022+0000 7fb8f2ce0640 1 > prepared operation key=1PX7DBO9M3YI tag=G1D0XXEPZ821 type=del size=3773932 2024-09-13T00:00:43.029 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.030+0000 7fb8f2ce0640 1 < completed write operation key=2AHYRZUDFHP4 tag=BFXOEETY384Y type=write size=623298 2024-09-13T00:00:43.034 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.034+0000 7fb8f2ce0640 1 > prepared operation key=P8OYS5ZE1NX0 tag=3331C8NN3YU3 type=del size=2693332 2024-09-13T00:00:43.040 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.038+0000 7fb8f2ce0640 1 < completed delete operation key=5GM88ERM89KT tag=V2S4842UZXEL type=del 2024-09-13T00:00:43.046 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.046+0000 7fb8f2ce0640 1 > prepared operation key=ZTK9AYBSLCOZ tag=IL03ZQQ6YVJU type=del size=905483 2024-09-13T00:00:43.057 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.054+0000 7fb8f2ce0640 1 < suggested remove operation key=W8XZ26311RJX tag=HF9YJ6FLIPOO type=del 2024-09-13T00:00:43.062 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.062+0000 7fb8f2ce0640 1 > prepared operation key=FX0NCGLK0KPJ tag=VBI7O1YWGNP6 type=del size=2093293 2024-09-13T00:00:43.067 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.066+0000 7fb8f2ce0640 1 < completed part key=_multipart_85YEW8N43FTA.2~F4F7GZ0BPGPQ.0 size=1048576 2024-09-13T00:00:43.067 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.066+0000 7fb8f2ce0640 1 < canceled part key=_multipart_85YEW8N43FTA.2~F4F7GZ0BPGPQ.1 size=1048576 2024-09-13T00:00:43.071 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.070+0000 7fb8f2ce0640 1 < completed part key=_multipart_85YEW8N43FTA.2~F4F7GZ0BPGPQ.2 size=660980 2024-09-13T00:00:43.079 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.078+0000 7fb8f2ce0640 1 < canceled multipart upload key=85YEW8N43FTA upload=F4F7GZ0BPGPQ tag=XHC2QHJS2WO4 type=write size=3806708 2024-09-13T00:00:43.086 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.086+0000 7fb8f2ce0640 1 > prepared operation key=QMZREZ1QEDNJ tag=GYFGTU3LH8BD type=del size=2260538 2024-09-13T00:00:43.091 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.090+0000 7fb8f2ce0640 1 < completed delete operation key=63FJDWBO2T9S tag=CHRYQ7BXYS5H type=del 2024-09-13T00:00:43.100 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.098+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H9AU4HD247YU.2~1M520JPX2SCO.0 size=1048576 2024-09-13T00:00:43.104 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.102+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H9AU4HD247YU.2~1M520JPX2SCO.1 size=92681 2024-09-13T00:00:43.104 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.102+0000 7fb8f2ce0640 1 > prepared multipart upload key=H9AU4HD247YU upload=1M520JPX2SCO tag=YC5X5SYZLY85 type=write size=2189833 2024-09-13T00:00:43.109 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.106+0000 7fb8f2ce0640 1 < completed delete operation key=85YEW8N43FTA tag=4TCN31OVLJS3 type=del 2024-09-13T00:00:43.114 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.114+0000 7fb8f2ce0640 1 > prepared operation key=ZFHN2SNTZ5W7 tag=PDSM9TPHAMS3 type=del size=3752859 2024-09-13T00:00:43.120 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.118+0000 7fb8f2ce0640 1 < completed write operation key=70F7D20HI504 tag=U6NI254R4138 type=write size=572550 2024-09-13T00:00:43.125 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.126+0000 7fb8f2ce0640 1 > prepared operation key=1HX06CCZJ1DD tag=U721W4LK9TN7 type=del size=2704102 2024-09-13T00:00:43.130 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.130+0000 7fb8f2ce0640 1 < completed part key=_multipart_EUE3LT0PDIXV.2~W6OIW3VJ8IHQ.0 size=1048576 2024-09-13T00:00:43.134 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.134+0000 7fb8f2ce0640 1 < completed part key=_multipart_EUE3LT0PDIXV.2~W6OIW3VJ8IHQ.1 size=1048576 2024-09-13T00:00:43.138 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.138+0000 7fb8f2ce0640 1 < completed part key=_multipart_EUE3LT0PDIXV.2~W6OIW3VJ8IHQ.2 size=907762 2024-09-13T00:00:43.146 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.146+0000 7fb8f2ce0640 1 < completed multipart upload key=EUE3LT0PDIXV upload=W6OIW3VJ8IHQ tag=OC2D4ZUMYKNF type=write size=4053490 2024-09-13T00:00:43.151 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.150+0000 7fb8f2ce0640 1 > prepared operation key=1HX06CCZJ1DD tag=U7WFEEGE3M8O type=del size=1855114 2024-09-13T00:00:43.161 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.158+0000 7fb8f2ce0640 1 < suggested update operation key=4K76U69GZ9AE tag=51J5DUWOGWG9 type=del size=697803 2024-09-13T00:00:43.170 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.170+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~KHBK9SXV4JSL.0 size=95860 2024-09-13T00:00:43.170 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.170+0000 7fb8f2ce0640 1 > prepared multipart upload key=1PX7DBO9M3YI upload=KHBK9SXV4JSL tag=U1U00R5OCZ3G type=write size=1144436 2024-09-13T00:00:43.174 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.174+0000 7fb8f2ce0640 1 < completed part key=_multipart_W8XZ26311RJX.2~9PTJZEPCQMKG.0 size=60343 2024-09-13T00:00:43.183 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.182+0000 7fb8f2ce0640 1 < completed multipart upload key=W8XZ26311RJX upload=9PTJZEPCQMKG tag=TOKXSV45ZBQ6 type=write size=1108919 2024-09-13T00:00:43.187 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.186+0000 7fb8f2ce0640 1 > prepared operation key=4K76U69GZ9AE tag=O460WSXO9HRA type=del size=1672465 2024-09-13T00:00:43.192 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.190+0000 7fb8f2ce0640 1 < completed part key=_multipart_5GM88ERM89KT.2~5TXOFSOWME7I.0 size=1048576 2024-09-13T00:00:43.197 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.194+0000 7fb8f2ce0640 1 < completed part key=_multipart_5GM88ERM89KT.2~5TXOFSOWME7I.1 size=945673 2024-09-13T00:00:43.205 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.206+0000 7fb8f2ce0640 1 < completed multipart upload key=5GM88ERM89KT upload=5TXOFSOWME7I tag=W4TYGZMT3662 type=write size=3042825 2024-09-13T00:00:43.214 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.214+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~2ORUD68KP2BQ.0 size=1048576 2024-09-13T00:00:43.218 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.218+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~2ORUD68KP2BQ.1 size=1048576 2024-09-13T00:00:43.222 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.222+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~2ORUD68KP2BQ.2 size=282060 2024-09-13T00:00:43.222 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.222+0000 7fb8f2ce0640 1 > prepared multipart upload key=FX0NCGLK0KPJ upload=2ORUD68KP2BQ tag=4LVRRNAQDI0N type=write size=3427788 2024-09-13T00:00:43.228 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.226+0000 7fb8f2ce0640 -1 < failed to complete operation key=4K76U69GZ9AE tag=L8HSAAHR4TS1 type=del size=383280: (22) Invalid argument 2024-09-13T00:00:43.233 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.234+0000 7fb8f2ce0640 1 > prepared operation key=NX26SI4F2ZMB tag=8P3K6HE9Z5OZ type=write size=122533 2024-09-13T00:00:43.238 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.238+0000 7fb8f2ce0640 1 < canceled operation key=8MPUVD37EDN9 tag=6UZWZ0P2JRFT type=del size=1819168 2024-09-13T00:00:43.243 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.242+0000 7fb8f2ce0640 1 > prepared operation key=QMZREZ1QEDNJ tag=CFOKQUY8NFWM type=del size=3635138 2024-09-13T00:00:43.248 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.246+0000 7fb8f2ce0640 1 < completed part key=_multipart_NX26SI4F2ZMB.2~FL08RJER4LZ4.0 size=1048576 2024-09-13T00:00:43.253 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.250+0000 7fb8f2ce0640 1 < completed part key=_multipart_NX26SI4F2ZMB.2~FL08RJER4LZ4.1 size=1048576 2024-09-13T00:00:43.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:42 smithi136 bash[75164]: cluster 2024-09-13T00:00:41.129793+0000 mgr.y (mgr.24661) 707 : cluster [DBG] pgmap v1383: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 729 KiB/s rd, 251 KiB/s wr, 1.29k op/s 2024-09-13T00:00:43.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:42 smithi136 bash[76371]: cluster 2024-09-13T00:00:41.129793+0000 mgr.y (mgr.24661) 707 : cluster [DBG] pgmap v1383: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 729 KiB/s rd, 251 KiB/s wr, 1.29k op/s 2024-09-13T00:00:43.257 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.258+0000 7fb8f2ce0640 1 < completed part key=_multipart_NX26SI4F2ZMB.2~FL08RJER4LZ4.2 size=487347 2024-09-13T00:00:43.266 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.266+0000 7fb8f2ce0640 1 < completed multipart upload key=NX26SI4F2ZMB upload=FL08RJER4LZ4 tag=7GAXA03DNFS2 type=write size=3633075 2024-09-13T00:00:43.271 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.270+0000 7fb8f2ce0640 1 > prepared operation key=ZFHN2SNTZ5W7 tag=8MEJMYYT8UUH type=del size=3492557 2024-09-13T00:00:43.277 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.278+0000 7fb8f2ce0640 1 < completed part key=_multipart_F0QCVR9E08WE.2~QDACYZKIG1NN.0 size=1048576 2024-09-13T00:00:43.282 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.282+0000 7fb8f2ce0640 1 < completed part key=_multipart_F0QCVR9E08WE.2~QDACYZKIG1NN.1 size=379358 2024-09-13T00:00:43.291 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.290+0000 7fb8f2ce0640 1 < completed multipart upload key=F0QCVR9E08WE upload=QDACYZKIG1NN tag=S1VLW3XC2A9G type=write size=2476510 2024-09-13T00:00:43.296 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.294+0000 7fb8f2ce0640 1 > prepared operation key=F0QCVR9E08WE tag=2IDLRQHIN3ZK type=del size=2853558 2024-09-13T00:00:43.306 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.306+0000 7fb8f2ce0640 1 < suggested update operation key=1PX7DBO9M3YI tag=G1D0XXEPZ821 type=del size=3773932 2024-09-13T00:00:43.313 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.314+0000 7fb8f2ce0640 1 > prepared operation key=ZFHN2SNTZ5W7 tag=CEGI3W75VOZ6 type=write size=613222 2024-09-13T00:00:43.319 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.318+0000 7fb8f2ce0640 1 < completed delete operation key=P8OYS5ZE1NX0 tag=3331C8NN3YU3 type=del 2024-09-13T00:00:43.325 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.326+0000 7fb8f2ce0640 1 > prepared operation key=ZTK9AYBSLCOZ tag=PS62ZHXUJ4YD type=del size=1056278 2024-09-13T00:00:43.330 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.330+0000 7fb8f2ce0640 1 < completed delete operation key=ZTK9AYBSLCOZ tag=IL03ZQQ6YVJU type=del 2024-09-13T00:00:43.335 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.334+0000 7fb8f2ce0640 1 > prepared operation key=J8NGDSF9XRTP tag=Y0TI8HVPEYWM type=del size=992354 2024-09-13T00:00:43.341 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.342+0000 7fb8f2ce0640 1 < completed delete operation key=FX0NCGLK0KPJ tag=VBI7O1YWGNP6 type=del 2024-09-13T00:00:43.346 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.346+0000 7fb8f2ce0640 1 > prepared operation key=85YEW8N43FTA tag=KDFGZH4NL3EE type=del size=2309156 2024-09-13T00:00:43.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:42 smithi181 bash[67587]: cluster 2024-09-13T00:00:41.129793+0000 mgr.y (mgr.24661) 707 : cluster [DBG] pgmap v1383: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 729 KiB/s rd, 251 KiB/s wr, 1.29k op/s 2024-09-13T00:00:43.352 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.350+0000 7fb8f2ce0640 1 < completed delete operation key=QMZREZ1QEDNJ tag=GYFGTU3LH8BD type=del 2024-09-13T00:00:43.362 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.362+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2TWD9YLMSYJI.2~KQXBD7ZGTF69.0 size=1048576 2024-09-13T00:00:43.367 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.366+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2TWD9YLMSYJI.2~KQXBD7ZGTF69.1 size=1048576 2024-09-13T00:00:43.371 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.370+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2TWD9YLMSYJI.2~KQXBD7ZGTF69.2 size=15367 2024-09-13T00:00:43.371 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.370+0000 7fb8f2ce0640 1 > prepared multipart upload key=2TWD9YLMSYJI upload=KQXBD7ZGTF69 tag=5KKEU1SRS1FL type=write size=3161095 2024-09-13T00:00:43.372 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.370+0000 7fb8f2ce0640 1 < canceled part key=_multipart_H9AU4HD247YU.2~1M520JPX2SCO.0 size=1048576 2024-09-13T00:00:43.377 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.374+0000 7fb8f2ce0640 1 < completed part key=_multipart_H9AU4HD247YU.2~1M520JPX2SCO.1 size=92681 2024-09-13T00:00:43.385 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.386+0000 7fb8f2ce0640 1 < canceled multipart upload key=H9AU4HD247YU upload=1M520JPX2SCO tag=YC5X5SYZLY85 type=write size=2189833 2024-09-13T00:00:43.391 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.390+0000 7fb8f2ce0640 1 > prepared operation key=4K76U69GZ9AE tag=4K1LZ64621J2 type=write size=390686 2024-09-13T00:00:43.397 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.394+0000 7fb8f2ce0640 1 < completed delete operation key=ZFHN2SNTZ5W7 tag=PDSM9TPHAMS3 type=del 2024-09-13T00:00:43.407 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.406+0000 7fb8f2ce0640 1 > prepared part key=_multipart_44EDYFT9VF8Y.2~6ZPAFHCTR8DR.0 size=1048576 2024-09-13T00:00:43.412 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.410+0000 7fb8f2ce0640 1 > prepared part key=_multipart_44EDYFT9VF8Y.2~6ZPAFHCTR8DR.1 size=779911 2024-09-13T00:00:43.412 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.410+0000 7fb8f2ce0640 1 > prepared multipart upload key=44EDYFT9VF8Y upload=6ZPAFHCTR8DR tag=1DSPDRZUAZB1 type=write size=2877063 2024-09-13T00:00:43.423 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.422+0000 7fb8f2ce0640 1 < suggested update operation key=1HX06CCZJ1DD tag=U721W4LK9TN7 type=del size=2704102 2024-09-13T00:00:43.428 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.426+0000 7fb8f2ce0640 1 > prepared operation key=PVYZQLGYT3XU tag=CL222KN7CV7R type=del size=408493 2024-09-13T00:00:43.435 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.434+0000 7fb8f2ce0640 -1 < failed to complete operation key=1HX06CCZJ1DD tag=U7WFEEGE3M8O type=del size=1855114: (22) Invalid argument 2024-09-13T00:00:43.444 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.442+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~4938IPM5ZJQB.0 size=1048576 2024-09-13T00:00:43.449 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.446+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~4938IPM5ZJQB.1 size=186636 2024-09-13T00:00:43.449 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.446+0000 7fb8f2ce0640 1 > prepared multipart upload key=PVYZQLGYT3XU upload=4938IPM5ZJQB tag=ERC0PDWQOIBB type=write size=2283788 2024-09-13T00:00:43.453 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.454+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~KHBK9SXV4JSL.0 size=95860 2024-09-13T00:00:43.463 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.462+0000 7fb8f2ce0640 1 < completed multipart upload key=1PX7DBO9M3YI upload=KHBK9SXV4JSL tag=U1U00R5OCZ3G type=write size=1144436 2024-09-13T00:00:43.468 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.466+0000 7fb8f2ce0640 1 > prepared operation key=1HX06CCZJ1DD tag=AYHL87Z81SUG type=write size=570772 2024-09-13T00:00:43.474 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.474+0000 7fb8f2ce0640 1 < completed delete operation key=4K76U69GZ9AE tag=O460WSXO9HRA type=del 2024-09-13T00:00:43.480 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.478+0000 7fb8f2ce0640 1 > prepared operation key=1PX7DBO9M3YI tag=XGNGA1H2ANEE type=del size=2697794 2024-09-13T00:00:43.486 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.486+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~2ORUD68KP2BQ.0 size=1048576 2024-09-13T00:00:43.490 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.490+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~2ORUD68KP2BQ.1 size=1048576 2024-09-13T00:00:43.495 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.494+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~2ORUD68KP2BQ.2 size=282060 2024-09-13T00:00:43.504 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.502+0000 7fb8f2ce0640 1 < completed multipart upload key=FX0NCGLK0KPJ upload=2ORUD68KP2BQ tag=4LVRRNAQDI0N type=write size=3427788 2024-09-13T00:00:43.517 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.518+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~OXBIH5NQ2ZR0.0 size=964580 2024-09-13T00:00:43.517 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.518+0000 7fb8f2ce0640 1 > prepared multipart upload key=1PX7DBO9M3YI upload=OXBIH5NQ2ZR0 tag=RI8FJQX6F4MD type=write size=2013156 2024-09-13T00:00:43.528 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.526+0000 7fb8f2ce0640 1 < suggested update operation key=NX26SI4F2ZMB tag=8P3K6HE9Z5OZ type=write size=122533 2024-09-13T00:00:43.533 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.530+0000 7fb8f2ce0640 1 > prepared operation key=2TWD9YLMSYJI tag=QXV3Q09W11JN type=write size=410132 2024-09-13T00:00:43.543 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.542+0000 7fb8f2ce0640 1 < suggested remove operation key=QMZREZ1QEDNJ tag=CFOKQUY8NFWM type=del 2024-09-13T00:00:43.547 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.546+0000 7fb8f2ce0640 1 > prepared operation key=H5EIP0E9CBWX tag=2DWLT5ETDJJV type=del size=1485665 2024-09-13T00:00:43.553 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.550+0000 7fb8f2ce0640 1 < completed delete operation key=ZFHN2SNTZ5W7 tag=8MEJMYYT8UUH type=del 2024-09-13T00:00:43.562 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.562+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~1QKO72Y7QO0D.0 size=966718 2024-09-13T00:00:43.562 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.562+0000 7fb8f2ce0640 1 > prepared multipart upload key=FX0NCGLK0KPJ upload=1QKO72Y7QO0D tag=WQ2SWIES5NKW type=write size=2015294 2024-09-13T00:00:43.567 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.566+0000 7fb8f2ce0640 1 < completed delete operation key=F0QCVR9E08WE tag=2IDLRQHIN3ZK type=del 2024-09-13T00:00:43.572 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.570+0000 7fb8f2ce0640 1 > prepared operation key=P8OYS5ZE1NX0 tag=HJ1QR1GPRGAZ type=del size=2014530 2024-09-13T00:00:43.577 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.578+0000 7fb8f2ce0640 1 < completed write operation key=ZFHN2SNTZ5W7 tag=CEGI3W75VOZ6 type=write size=613222 2024-09-13T00:00:43.586 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.586+0000 7fb8f2ce0640 1 > prepared part key=_multipart_P8OYS5ZE1NX0.2~UKEOY7TJ8AQD.0 size=1048576 2024-09-13T00:00:43.590 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.590+0000 7fb8f2ce0640 1 > prepared part key=_multipart_P8OYS5ZE1NX0.2~UKEOY7TJ8AQD.1 size=344394 2024-09-13T00:00:43.590 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.590+0000 7fb8f2ce0640 1 > prepared multipart upload key=P8OYS5ZE1NX0 upload=UKEOY7TJ8AQD tag=0XHYPOO9QO5B type=write size=2441546 2024-09-13T00:00:43.596 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.594+0000 7fb8f2ce0640 1 < completed delete operation key=ZTK9AYBSLCOZ tag=PS62ZHXUJ4YD type=del 2024-09-13T00:00:43.606 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.606+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EUE3LT0PDIXV.2~2SPOVP4OBF0R.0 size=1048576 2024-09-13T00:00:43.611 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.610+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EUE3LT0PDIXV.2~2SPOVP4OBF0R.1 size=461299 2024-09-13T00:00:43.611 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.610+0000 7fb8f2ce0640 1 > prepared multipart upload key=EUE3LT0PDIXV upload=2SPOVP4OBF0R tag=C6OAKGWMSX7M type=write size=2558451 2024-09-13T00:00:43.616 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.614+0000 7fb8f2ce0640 1 < completed delete operation key=J8NGDSF9XRTP tag=Y0TI8HVPEYWM type=del 2024-09-13T00:00:43.625 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.626+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NX26SI4F2ZMB.2~267W57R1QH9O.0 size=137686 2024-09-13T00:00:43.626 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.626+0000 7fb8f2ce0640 1 > prepared multipart upload key=NX26SI4F2ZMB upload=267W57R1QH9O tag=BG7XF4II9C2Q type=write size=1186262 2024-09-13T00:00:43.637 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.634+0000 7fb8f2ce0640 1 < suggested update operation key=85YEW8N43FTA tag=KDFGZH4NL3EE type=del size=2309156 2024-09-13T00:00:43.641 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.642+0000 7fb8f2ce0640 1 > prepared operation key=2AHYRZUDFHP4 tag=3VIX7MD56DP0 type=del size=498480 2024-09-13T00:00:43.647 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.646+0000 7fb8f2ce0640 1 < completed part key=_multipart_2TWD9YLMSYJI.2~KQXBD7ZGTF69.0 size=1048576 2024-09-13T00:00:43.652 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.650+0000 7fb8f2ce0640 1 < completed part key=_multipart_2TWD9YLMSYJI.2~KQXBD7ZGTF69.1 size=1048576 2024-09-13T00:00:43.657 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.658+0000 7fb8f2ce0640 1 < completed part key=_multipart_2TWD9YLMSYJI.2~KQXBD7ZGTF69.2 size=15367 2024-09-13T00:00:43.666 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.666+0000 7fb8f2ce0640 1 < completed multipart upload key=2TWD9YLMSYJI upload=KQXBD7ZGTF69 tag=5KKEU1SRS1FL type=write size=3161095 2024-09-13T00:00:43.678 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.678+0000 7fb8f2ce0640 1 > prepared part key=_multipart_F0QCVR9E08WE.2~Z6GF4K441D17.0 size=754596 2024-09-13T00:00:43.678 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.678+0000 7fb8f2ce0640 1 > prepared multipart upload key=F0QCVR9E08WE upload=Z6GF4K441D17 tag=OBVZWKQNYR94 type=write size=1803172 2024-09-13T00:00:43.683 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.682+0000 7fb8f2ce0640 1 < completed write operation key=4K76U69GZ9AE tag=4K1LZ64621J2 type=write size=390686 2024-09-13T00:00:43.692 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.690+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H5EIP0E9CBWX.2~ZA5I684UQ1WV.0 size=1048576 2024-09-13T00:00:43.696 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.694+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H5EIP0E9CBWX.2~ZA5I684UQ1WV.1 size=1048576 2024-09-13T00:00:43.700 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.698+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H5EIP0E9CBWX.2~ZA5I684UQ1WV.2 size=912241 2024-09-13T00:00:43.700 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.698+0000 7fb8f2ce0640 1 > prepared multipart upload key=H5EIP0E9CBWX upload=ZA5I684UQ1WV tag=8T4WMIRXZXSN type=write size=4057969 2024-09-13T00:00:43.705 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.706+0000 7fb8f2ce0640 1 < completed part key=_multipart_44EDYFT9VF8Y.2~6ZPAFHCTR8DR.0 size=1048576 2024-09-13T00:00:43.709 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.710+0000 7fb8f2ce0640 1 < completed part key=_multipart_44EDYFT9VF8Y.2~6ZPAFHCTR8DR.1 size=779911 2024-09-13T00:00:43.718 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.718+0000 7fb8f2ce0640 1 < completed multipart upload key=44EDYFT9VF8Y upload=6ZPAFHCTR8DR tag=1DSPDRZUAZB1 type=write size=2877063 2024-09-13T00:00:43.723 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.722+0000 7fb8f2ce0640 1 > prepared operation key=F0QCVR9E08WE tag=AKFH21WX0SN5 type=write size=1044011 2024-09-13T00:00:43.736 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.734+0000 7fb8f2ce0640 1 < suggested remove operation key=PVYZQLGYT3XU tag=CL222KN7CV7R type=del 2024-09-13T00:00:43.740 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.738+0000 7fb8f2ce0640 1 > prepared operation key=H5EIP0E9CBWX tag=N8TQFHAQOUNV type=write size=304179 2024-09-13T00:00:43.746 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.746+0000 7fb8f2ce0640 1 < completed part key=_multipart_PVYZQLGYT3XU.2~4938IPM5ZJQB.0 size=1048576 2024-09-13T00:00:43.750 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.750+0000 7fb8f2ce0640 1 < completed part key=_multipart_PVYZQLGYT3XU.2~4938IPM5ZJQB.1 size=186636 2024-09-13T00:00:43.759 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.758+0000 7fb8f2ce0640 1 < completed multipart upload key=PVYZQLGYT3XU upload=4938IPM5ZJQB tag=ERC0PDWQOIBB type=write size=2283788 2024-09-13T00:00:43.768 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.766+0000 7fb8f2ce0640 1 > prepared part key=_multipart_0WC6P8YNL0H2.2~RP37G8DGIPVT.0 size=1048576 2024-09-13T00:00:43.773 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.770+0000 7fb8f2ce0640 1 > prepared part key=_multipart_0WC6P8YNL0H2.2~RP37G8DGIPVT.1 size=508783 2024-09-13T00:00:43.773 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.770+0000 7fb8f2ce0640 1 > prepared multipart upload key=0WC6P8YNL0H2 upload=RP37G8DGIPVT tag=WMF2GHMOSH0Z type=write size=2605935 2024-09-13T00:00:43.778 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.778+0000 7fb8f2ce0640 1 < completed write operation key=1HX06CCZJ1DD tag=AYHL87Z81SUG type=write size=570772 2024-09-13T00:00:43.784 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.782+0000 7fb8f2ce0640 1 > prepared operation key=EUE3LT0PDIXV tag=YHONT1LESWZD type=del size=1334552 2024-09-13T00:00:43.794 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.794+0000 7fb8f2ce0640 1 < suggested remove operation key=1PX7DBO9M3YI tag=XGNGA1H2ANEE type=del 2024-09-13T00:00:43.799 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.798+0000 7fb8f2ce0640 1 > prepared operation key=0WC6P8YNL0H2 tag=9YGM2C3JVH8S type=del size=3433599 2024-09-13T00:00:43.805 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.806+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~OXBIH5NQ2ZR0.0 size=964580 2024-09-13T00:00:43.814 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.814+0000 7fb8f2ce0640 1 < completed multipart upload key=1PX7DBO9M3YI upload=OXBIH5NQ2ZR0 tag=RI8FJQX6F4MD type=write size=2013156 2024-09-13T00:00:43.822 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.822+0000 7fb8f2ce0640 1 > prepared operation key=0WC6P8YNL0H2 tag=91UNFNW08GSK type=write size=1020214 2024-09-13T00:00:43.836 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.834+0000 7fb8f2ce0640 1 < suggested update operation key=2TWD9YLMSYJI tag=QXV3Q09W11JN type=write size=410132 2024-09-13T00:00:43.841 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.842+0000 7fb8f2ce0640 1 > prepared operation key=N3Z5XIG3TZRQ tag=6Q2NZJME68C0 type=del size=2031484 2024-09-13T00:00:43.848 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.846+0000 7fb8f2ce0640 1 < completed delete operation key=H5EIP0E9CBWX tag=2DWLT5ETDJJV type=del 2024-09-13T00:00:43.853 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.854+0000 7fb8f2ce0640 1 > prepared operation key=63FJDWBO2T9S tag=A3SKIQUV4P0R type=del size=3626641 2024-09-13T00:00:43.858 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.858+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~1QKO72Y7QO0D.0 size=966718 2024-09-13T00:00:43.866 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.866+0000 7fb8f2ce0640 1 < completed multipart upload key=FX0NCGLK0KPJ upload=1QKO72Y7QO0D tag=WQ2SWIES5NKW type=write size=2015294 2024-09-13T00:00:43.872 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.870+0000 7fb8f2ce0640 1 > prepared operation key=ZFHN2SNTZ5W7 tag=9F9PKQOU46JZ type=del size=1646698 2024-09-13T00:00:43.878 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.878+0000 7fb8f2ce0640 1 < completed delete operation key=P8OYS5ZE1NX0 tag=HJ1QR1GPRGAZ type=del 2024-09-13T00:00:43.884 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.882+0000 7fb8f2ce0640 1 > prepared operation key=N3Z5XIG3TZRQ tag=8MOIKFJNE362 type=del size=3355989 2024-09-13T00:00:43.889 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.890+0000 7fb8f2ce0640 1 < completed part key=_multipart_P8OYS5ZE1NX0.2~UKEOY7TJ8AQD.0 size=1048576 2024-09-13T00:00:43.894 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.894+0000 7fb8f2ce0640 1 < completed part key=_multipart_P8OYS5ZE1NX0.2~UKEOY7TJ8AQD.1 size=344394 2024-09-13T00:00:43.903 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.902+0000 7fb8f2ce0640 1 < completed multipart upload key=P8OYS5ZE1NX0 upload=UKEOY7TJ8AQD tag=0XHYPOO9QO5B type=write size=2441546 2024-09-13T00:00:43.909 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.910+0000 7fb8f2ce0640 1 > prepared operation key=J8NGDSF9XRTP tag=PQL8BWV0EMBP type=del size=3499359 2024-09-13T00:00:43.914 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.914+0000 7fb8f2ce0640 1 < completed part key=_multipart_EUE3LT0PDIXV.2~2SPOVP4OBF0R.0 size=1048576 2024-09-13T00:00:43.918 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.918+0000 7fb8f2ce0640 1 < completed part key=_multipart_EUE3LT0PDIXV.2~2SPOVP4OBF0R.1 size=461299 2024-09-13T00:00:43.927 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.926+0000 7fb8f2ce0640 1 < completed multipart upload key=EUE3LT0PDIXV upload=2SPOVP4OBF0R tag=C6OAKGWMSX7M type=write size=2558451 2024-09-13T00:00:43.932 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.930+0000 7fb8f2ce0640 1 > prepared operation key=0WC6P8YNL0H2 tag=GMYMFH0W8WB9 type=del size=3439063 2024-09-13T00:00:43.940 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.938+0000 7fb8f2ce0640 1 < completed part key=_multipart_NX26SI4F2ZMB.2~267W57R1QH9O.0 size=137686 2024-09-13T00:00:43.946 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.946+0000 7fb8f2ce0640 1 < completed multipart upload key=NX26SI4F2ZMB upload=267W57R1QH9O tag=BG7XF4II9C2Q type=write size=1186262 2024-09-13T00:00:43.951 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.950+0000 7fb8f2ce0640 1 > prepared operation key=8Z6U7OOA0GBH tag=8PE3N28LXHO5 type=del size=4034606 2024-09-13T00:00:43.956 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.954+0000 7fb8f2ce0640 1 < completed delete operation key=2AHYRZUDFHP4 tag=3VIX7MD56DP0 type=del 2024-09-13T00:00:43.961 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.962+0000 7fb8f2ce0640 1 > prepared operation key=8MPUVD37EDN9 tag=X8JU3LN7HN2C type=del size=1625458 2024-09-13T00:00:43.967 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.966+0000 7fb8f2ce0640 1 < completed part key=_multipart_F0QCVR9E08WE.2~Z6GF4K441D17.0 size=754596 2024-09-13T00:00:43.975 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.974+0000 7fb8f2ce0640 1 < completed multipart upload key=F0QCVR9E08WE upload=Z6GF4K441D17 tag=OBVZWKQNYR94 type=write size=1803172 2024-09-13T00:00:43.984 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.982+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H9AU4HD247YU.2~O3CBDO27SN3S.0 size=297741 2024-09-13T00:00:43.985 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.982+0000 7fb8f2ce0640 1 > prepared multipart upload key=H9AU4HD247YU upload=O3CBDO27SN3S tag=W7FLSLWMIZ6T type=write size=1346317 2024-09-13T00:00:43.990 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.990+0000 7fb8f2ce0640 1 < completed part key=_multipart_H5EIP0E9CBWX.2~ZA5I684UQ1WV.0 size=1048576 2024-09-13T00:00:43.995 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.994+0000 7fb8f2ce0640 1 < completed part key=_multipart_H5EIP0E9CBWX.2~ZA5I684UQ1WV.1 size=1048576 2024-09-13T00:00:44.000 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:43.998+0000 7fb8f2ce0640 1 < completed part key=_multipart_H5EIP0E9CBWX.2~ZA5I684UQ1WV.2 size=912241 2024-09-13T00:00:44.009 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.010+0000 7fb8f2ce0640 1 < completed multipart upload key=H5EIP0E9CBWX upload=ZA5I684UQ1WV tag=8T4WMIRXZXSN type=write size=4057969 2024-09-13T00:00:44.020 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.018+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~R72MHS1L4KPN.0 size=1048576 2024-09-13T00:00:44.024 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.022+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~R72MHS1L4KPN.1 size=204572 2024-09-13T00:00:44.024 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.022+0000 7fb8f2ce0640 1 > prepared multipart upload key=W8XZ26311RJX upload=R72MHS1L4KPN tag=8ZKPTABV9YPG type=write size=2301724 2024-09-13T00:00:44.036 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.034+0000 7fb8f2ce0640 1 < suggested remove operation key=F0QCVR9E08WE tag=AKFH21WX0SN5 type=write 2024-09-13T00:00:44.042 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.042+0000 7fb8f2ce0640 1 > prepared operation key=ZTK9AYBSLCOZ tag=CBH1IJT8S562 type=del size=2256225 2024-09-13T00:00:44.052 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.050+0000 7fb8f2ce0640 1 < suggested remove operation key=H5EIP0E9CBWX tag=N8TQFHAQOUNV type=write 2024-09-13T00:00:44.057 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.058+0000 7fb8f2ce0640 1 > prepared operation key=J8NGDSF9XRTP tag=JBX42TVMXVR4 type=write size=689483 2024-09-13T00:00:44.063 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.062+0000 7fb8f2ce0640 1 < completed part key=_multipart_0WC6P8YNL0H2.2~RP37G8DGIPVT.0 size=1048576 2024-09-13T00:00:44.069 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.070+0000 7fb8f2ce0640 1 < completed part key=_multipart_0WC6P8YNL0H2.2~RP37G8DGIPVT.1 size=508783 2024-09-13T00:00:44.077 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.078+0000 7fb8f2ce0640 1 < completed multipart upload key=0WC6P8YNL0H2 upload=RP37G8DGIPVT tag=WMF2GHMOSH0Z type=write size=2605935 2024-09-13T00:00:44.087 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.086+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H9AU4HD247YU.2~HMERS383EZ41.0 size=844723 2024-09-13T00:00:44.087 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.086+0000 7fb8f2ce0640 1 > prepared multipart upload key=H9AU4HD247YU upload=HMERS383EZ41 tag=F2VF9M2YWH5I type=write size=1893299 2024-09-13T00:00:44.092 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.090+0000 7fb8f2ce0640 1 < completed delete operation key=EUE3LT0PDIXV tag=YHONT1LESWZD type=del 2024-09-13T00:00:44.097 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.098+0000 7fb8f2ce0640 1 > prepared operation key=4K76U69GZ9AE tag=8F9SFYZQ8JSV type=del size=197696 2024-09-13T00:00:44.102 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.102+0000 7fb8f2ce0640 1 < completed delete operation key=0WC6P8YNL0H2 tag=9YGM2C3JVH8S type=del 2024-09-13T00:00:44.108 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.106+0000 7fb8f2ce0640 1 > prepared operation key=70F7D20HI504 tag=FJJI8I28LZ20 type=del size=3464490 2024-09-13T00:00:44.113 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.114+0000 7fb8f2ce0640 1 < completed write operation key=0WC6P8YNL0H2 tag=91UNFNW08GSK type=write size=1020214 2024-09-13T00:00:44.122 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.122+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~95JWM8RPC3FI.0 size=1048576 2024-09-13T00:00:44.127 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.126+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~95JWM8RPC3FI.1 size=1048576 2024-09-13T00:00:44.131 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.130+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~95JWM8RPC3FI.2 size=6355 2024-09-13T00:00:44.132 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.130+0000 7fb8f2ce0640 1 > prepared multipart upload key=63FJDWBO2T9S upload=95JWM8RPC3FI tag=PENTZI43K21J type=write size=3152083 2024-09-13T00:00:44.137 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.138+0000 7fb8f2ce0640 1 < canceled operation key=N3Z5XIG3TZRQ tag=6Q2NZJME68C0 type=del size=2031484 2024-09-13T00:00:44.146 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.146+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EJN6HR8I7XQF.2~AT0TK113NQ21.0 size=1048576 2024-09-13T00:00:44.150 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.150+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EJN6HR8I7XQF.2~AT0TK113NQ21.1 size=1048576 2024-09-13T00:00:44.155 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.154+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EJN6HR8I7XQF.2~AT0TK113NQ21.2 size=100661 2024-09-13T00:00:44.155 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.154+0000 7fb8f2ce0640 1 > prepared multipart upload key=EJN6HR8I7XQF upload=AT0TK113NQ21 tag=938ETISW2453 type=write size=3246389 2024-09-13T00:00:44.165 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.166+0000 7fb8f2ce0640 1 < suggested update operation key=63FJDWBO2T9S tag=A3SKIQUV4P0R type=del size=3626641 2024-09-13T00:00:44.171 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.170+0000 7fb8f2ce0640 1 > prepared operation key=7OX5OJZ8GXU5 tag=XTS97G3P91IE type=del size=58414 2024-09-13T00:00:44.176 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.174+0000 7fb8f2ce0640 1 < completed delete operation key=ZFHN2SNTZ5W7 tag=9F9PKQOU46JZ type=del 2024-09-13T00:00:44.186 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.186+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~2BYYRFPYW0TQ.0 size=372456 2024-09-13T00:00:44.187 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.186+0000 7fb8f2ce0640 1 > prepared multipart upload key=NQJQC8CEAC15 upload=2BYYRFPYW0TQ tag=S93W390NMOR3 type=write size=1421032 2024-09-13T00:00:44.192 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.190+0000 7fb8f2ce0640 1 < canceled operation key=N3Z5XIG3TZRQ tag=8MOIKFJNE362 type=del size=3355989 2024-09-13T00:00:44.199 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.198+0000 7fb8f2ce0640 1 > prepared operation key=8MPUVD37EDN9 tag=CX905P0H8KEX type=del size=1742837 2024-09-13T00:00:44.204 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.202+0000 7fb8f2ce0640 1 < canceled operation key=J8NGDSF9XRTP tag=PQL8BWV0EMBP type=del size=3499359 2024-09-13T00:00:44.213 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.214+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2TWD9YLMSYJI.2~XKO5EJOATKW8.0 size=983614 2024-09-13T00:00:44.213 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.214+0000 7fb8f2ce0640 1 > prepared multipart upload key=2TWD9YLMSYJI upload=XKO5EJOATKW8 tag=2V7PL9IIDFB2 type=write size=2032190 2024-09-13T00:00:44.223 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.222+0000 7fb8f2ce0640 1 < suggested remove operation key=0WC6P8YNL0H2 tag=GMYMFH0W8WB9 type=del 2024-09-13T00:00:44.228 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.226+0000 7fb8f2ce0640 1 > prepared operation key=W8XZ26311RJX tag=EU15IG7QKIZC type=del size=2539759 2024-09-13T00:00:44.239 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.238+0000 7fb8f2ce0640 1 < suggested update operation key=8Z6U7OOA0GBH tag=8PE3N28LXHO5 type=del size=4034606 2024-09-13T00:00:44.245 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.242+0000 7fb8f2ce0640 1 > prepared operation key=63FJDWBO2T9S tag=QVZN5TGHEVE6 type=del size=651876 2024-09-13T00:00:44.251 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.250+0000 7fb8f2ce0640 1 < canceled operation key=8MPUVD37EDN9 tag=X8JU3LN7HN2C type=del size=1625458 2024-09-13T00:00:44.261 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.262+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~9HP45KTRAHZ0.0 size=1048576 2024-09-13T00:00:44.265 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.266+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~9HP45KTRAHZ0.1 size=320686 2024-09-13T00:00:44.265 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.266+0000 7fb8f2ce0640 1 > prepared multipart upload key=W8XZ26311RJX upload=9HP45KTRAHZ0 tag=59UMGCC3SG2N type=write size=2417838 2024-09-13T00:00:44.270 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.270+0000 7fb8f2ce0640 1 < completed part key=_multipart_H9AU4HD247YU.2~O3CBDO27SN3S.0 size=297741 2024-09-13T00:00:44.279 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.278+0000 7fb8f2ce0640 1 < completed multipart upload key=H9AU4HD247YU upload=O3CBDO27SN3S tag=W7FLSLWMIZ6T type=write size=1346317 2024-09-13T00:00:44.285 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.286+0000 7fb8f2ce0640 1 > prepared operation key=4K76U69GZ9AE tag=0C5AD2QI3GGT type=del size=249884 2024-09-13T00:00:44.291 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.290+0000 7fb8f2ce0640 1 < completed part key=_multipart_W8XZ26311RJX.2~R72MHS1L4KPN.0 size=1048576 2024-09-13T00:00:44.295 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.294+0000 7fb8f2ce0640 1 < completed part key=_multipart_W8XZ26311RJX.2~R72MHS1L4KPN.1 size=204572 2024-09-13T00:00:44.304 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.302+0000 7fb8f2ce0640 1 < completed multipart upload key=W8XZ26311RJX upload=R72MHS1L4KPN tag=8ZKPTABV9YPG type=write size=2301724 2024-09-13T00:00:44.309 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.310+0000 7fb8f2ce0640 1 > prepared operation key=P8OYS5ZE1NX0 tag=KHCFXECNI51O type=del size=685200 2024-09-13T00:00:44.319 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.318+0000 7fb8f2ce0640 1 < suggested remove operation key=ZTK9AYBSLCOZ tag=CBH1IJT8S562 type=del 2024-09-13T00:00:44.324 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.322+0000 7fb8f2ce0640 1 > prepared operation key=H4M1TCN5W6XU tag=W9JN3OAC5732 type=del size=2464147 2024-09-13T00:00:44.329 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.326+0000 7fb8f2ce0640 1 < completed write operation key=J8NGDSF9XRTP tag=JBX42TVMXVR4 type=write size=689483 2024-09-13T00:00:44.338 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.338+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~AOVL6WFV16NG.0 size=1048576 2024-09-13T00:00:44.342 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.342+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~AOVL6WFV16NG.1 size=1048576 2024-09-13T00:00:44.346 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.346+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~AOVL6WFV16NG.2 size=832776 2024-09-13T00:00:44.347 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.346+0000 7fb8f2ce0640 1 > prepared multipart upload key=8MPUVD37EDN9 upload=AOVL6WFV16NG tag=R4QO1GUI8VLX type=write size=3978504 2024-09-13T00:00:44.352 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.350+0000 7fb8f2ce0640 1 < completed part key=_multipart_H9AU4HD247YU.2~HMERS383EZ41.0 size=844723 2024-09-13T00:00:44.362 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.362+0000 7fb8f2ce0640 1 < completed multipart upload key=H9AU4HD247YU upload=HMERS383EZ41 tag=F2VF9M2YWH5I type=write size=1893299 2024-09-13T00:00:44.372 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.370+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~UNHR0C5OCOIV.0 size=1048576 2024-09-13T00:00:44.376 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.374+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~UNHR0C5OCOIV.1 size=1048576 2024-09-13T00:00:44.380 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.378+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~UNHR0C5OCOIV.2 size=546840 2024-09-13T00:00:44.380 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.378+0000 7fb8f2ce0640 1 > prepared multipart upload key=1PX7DBO9M3YI upload=UNHR0C5OCOIV tag=9U3CC0NLP4P4 type=write size=3692568 2024-09-13T00:00:44.386 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.386+0000 7fb8f2ce0640 1 < completed delete operation key=4K76U69GZ9AE tag=8F9SFYZQ8JSV type=del 2024-09-13T00:00:44.396 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.394+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~4VEDZIV0B67F.0 size=1048576 2024-09-13T00:00:44.400 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.398+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~4VEDZIV0B67F.1 size=1048576 2024-09-13T00:00:44.404 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.402+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~4VEDZIV0B67F.2 size=558997 2024-09-13T00:00:44.404 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.402+0000 7fb8f2ce0640 1 > prepared multipart upload key=1PX7DBO9M3YI upload=4VEDZIV0B67F tag=072150E7ZO2U type=write size=3704725 2024-09-13T00:00:44.414 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.414+0000 7fb8f2ce0640 1 < suggested remove operation key=70F7D20HI504 tag=FJJI8I28LZ20 type=del 2024-09-13T00:00:44.424 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.422+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~90Z6H4D7TPMY.0 size=681560 2024-09-13T00:00:44.424 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.422+0000 7fb8f2ce0640 1 > prepared multipart upload key=PVYZQLGYT3XU upload=90Z6H4D7TPMY tag=X9SRFZBWM7T0 type=write size=1730136 2024-09-13T00:00:44.429 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.430+0000 7fb8f2ce0640 1 < completed part key=_multipart_63FJDWBO2T9S.2~95JWM8RPC3FI.0 size=1048576 2024-09-13T00:00:44.433 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.434+0000 7fb8f2ce0640 1 < completed part key=_multipart_63FJDWBO2T9S.2~95JWM8RPC3FI.1 size=1048576 2024-09-13T00:00:44.437 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.438+0000 7fb8f2ce0640 1 < completed part key=_multipart_63FJDWBO2T9S.2~95JWM8RPC3FI.2 size=6355 2024-09-13T00:00:44.446 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.446+0000 7fb8f2ce0640 1 < completed multipart upload key=63FJDWBO2T9S upload=95JWM8RPC3FI tag=PENTZI43K21J type=write size=3152083 2024-09-13T00:00:44.455 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.454+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EJN6HR8I7XQF.2~TEW41Q8SG4OC.0 size=261705 2024-09-13T00:00:44.455 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.454+0000 7fb8f2ce0640 1 > prepared multipart upload key=EJN6HR8I7XQF upload=TEW41Q8SG4OC tag=L15HZA2I9KPW type=write size=1310281 2024-09-13T00:00:44.461 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.458+0000 7fb8f2ce0640 1 < completed part key=_multipart_EJN6HR8I7XQF.2~AT0TK113NQ21.0 size=1048576 2024-09-13T00:00:44.465 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.466+0000 7fb8f2ce0640 1 < completed part key=_multipart_EJN6HR8I7XQF.2~AT0TK113NQ21.1 size=1048576 2024-09-13T00:00:44.469 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.470+0000 7fb8f2ce0640 1 < completed part key=_multipart_EJN6HR8I7XQF.2~AT0TK113NQ21.2 size=100661 2024-09-13T00:00:44.478 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.478+0000 7fb8f2ce0640 1 < completed multipart upload key=EJN6HR8I7XQF upload=AT0TK113NQ21 tag=938ETISW2453 type=write size=3246389 2024-09-13T00:00:44.483 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.482+0000 7fb8f2ce0640 1 > prepared operation key=2TWD9YLMSYJI tag=YH70YBS3RVV3 type=del size=3213215 2024-09-13T00:00:44.489 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.486+0000 7fb8f2ce0640 1 < completed delete operation key=7OX5OJZ8GXU5 tag=XTS97G3P91IE type=del 2024-09-13T00:00:44.494 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.494+0000 7fb8f2ce0640 1 > prepared operation key=2TWD9YLMSYJI tag=6200D1IH3LWK type=write size=225108 2024-09-13T00:00:44.499 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.498+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~2BYYRFPYW0TQ.0 size=372456 2024-09-13T00:00:44.507 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.506+0000 7fb8f2ce0640 1 < completed multipart upload key=NQJQC8CEAC15 upload=2BYYRFPYW0TQ tag=S93W390NMOR3 type=write size=1421032 2024-09-13T00:00:44.517 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.518+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~I0IL5BAOED0Q.0 size=1048576 2024-09-13T00:00:44.522 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.522+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~I0IL5BAOED0Q.1 size=362659 2024-09-13T00:00:44.522 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.522+0000 7fb8f2ce0640 1 > prepared multipart upload key=8Z6U7OOA0GBH upload=I0IL5BAOED0Q tag=B6F3ERI7O7LS type=write size=2459811 2024-09-13T00:00:44.526 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.526+0000 7fb8f2ce0640 1 < completed delete operation key=8MPUVD37EDN9 tag=CX905P0H8KEX type=del 2024-09-13T00:00:44.538 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.538+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~8LG08RHGBPN7.0 size=1048576 2024-09-13T00:00:44.543 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.542+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~8LG08RHGBPN7.1 size=993150 2024-09-13T00:00:44.543 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.542+0000 7fb8f2ce0640 1 > prepared multipart upload key=W8XZ26311RJX upload=8LG08RHGBPN7 tag=9XHCALB5918W type=write size=3090302 2024-09-13T00:00:44.548 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.546+0000 7fb8f2ce0640 1 < completed part key=_multipart_2TWD9YLMSYJI.2~XKO5EJOATKW8.0 size=983614 2024-09-13T00:00:44.557 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.558+0000 7fb8f2ce0640 1 < completed multipart upload key=2TWD9YLMSYJI upload=XKO5EJOATKW8 tag=2V7PL9IIDFB2 type=write size=2032190 2024-09-13T00:00:44.563 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.562+0000 7fb8f2ce0640 1 > prepared operation key=2AHYRZUDFHP4 tag=D9V7RKI8BSOA type=del size=3925962 2024-09-13T00:00:44.569 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.570+0000 7fb8f2ce0640 1 < canceled operation key=W8XZ26311RJX tag=EU15IG7QKIZC type=del size=2539759 2024-09-13T00:00:44.574 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.574+0000 7fb8f2ce0640 1 > prepared operation key=2TWD9YLMSYJI tag=L40HFWB6SO05 type=del size=151565 2024-09-13T00:00:44.585 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.582+0000 7fb8f2ce0640 1 < suggested update operation key=63FJDWBO2T9S tag=QVZN5TGHEVE6 type=del size=651876 2024-09-13T00:00:44.589 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.590+0000 7fb8f2ce0640 1 > prepared operation key=F0QCVR9E08WE tag=X15ZZ9D5KMT1 type=del size=5247 2024-09-13T00:00:44.594 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.594+0000 7fb8f2ce0640 1 < completed part key=_multipart_W8XZ26311RJX.2~9HP45KTRAHZ0.0 size=1048576 2024-09-13T00:00:44.599 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.598+0000 7fb8f2ce0640 1 < completed part key=_multipart_W8XZ26311RJX.2~9HP45KTRAHZ0.1 size=320686 2024-09-13T00:00:44.609 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.610+0000 7fb8f2ce0640 1 < completed multipart upload key=W8XZ26311RJX upload=9HP45KTRAHZ0 tag=59UMGCC3SG2N type=write size=2417838 2024-09-13T00:00:44.619 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.618+0000 7fb8f2ce0640 1 > prepared part key=_multipart_P8OYS5ZE1NX0.2~1PAJRXNMF0AQ.0 size=379127 2024-09-13T00:00:44.619 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.618+0000 7fb8f2ce0640 1 > prepared multipart upload key=P8OYS5ZE1NX0 upload=1PAJRXNMF0AQ tag=YYW6W7TTI89H type=write size=1427703 2024-09-13T00:00:44.624 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.622+0000 7fb8f2ce0640 1 < completed delete operation key=4K76U69GZ9AE tag=0C5AD2QI3GGT type=del 2024-09-13T00:00:44.629 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.630+0000 7fb8f2ce0640 1 > prepared operation key=2AHYRZUDFHP4 tag=9IQAK9GLHGPL type=del size=817376 2024-09-13T00:00:44.635 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.634+0000 7fb8f2ce0640 1 < completed delete operation key=P8OYS5ZE1NX0 tag=KHCFXECNI51O type=del 2024-09-13T00:00:44.645 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.646+0000 7fb8f2ce0640 1 > prepared part key=_multipart_5GM88ERM89KT.2~EQ7V42R4AF26.0 size=1048576 2024-09-13T00:00:44.649 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.650+0000 7fb8f2ce0640 1 > prepared part key=_multipart_5GM88ERM89KT.2~EQ7V42R4AF26.1 size=1048576 2024-09-13T00:00:44.654 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.654+0000 7fb8f2ce0640 1 > prepared part key=_multipart_5GM88ERM89KT.2~EQ7V42R4AF26.2 size=1029350 2024-09-13T00:00:44.654 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.654+0000 7fb8f2ce0640 1 > prepared multipart upload key=5GM88ERM89KT upload=EQ7V42R4AF26 tag=31GN2HABIA4Q type=write size=4175078 2024-09-13T00:00:44.665 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.662+0000 7fb8f2ce0640 1 < suggested remove operation key=H4M1TCN5W6XU tag=W9JN3OAC5732 type=del 2024-09-13T00:00:44.674 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.674+0000 7fb8f2ce0640 1 > prepared part key=_multipart_O4MZ4UWNXME9.2~SJ16NP6SFGP6.0 size=1048576 2024-09-13T00:00:44.678 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.678+0000 7fb8f2ce0640 1 > prepared part key=_multipart_O4MZ4UWNXME9.2~SJ16NP6SFGP6.1 size=789912 2024-09-13T00:00:44.678 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.678+0000 7fb8f2ce0640 1 > prepared multipart upload key=O4MZ4UWNXME9 upload=SJ16NP6SFGP6 tag=TWKVZ3W9JQGY type=write size=2887064 2024-09-13T00:00:44.684 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.682+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~AOVL6WFV16NG.0 size=1048576 2024-09-13T00:00:44.688 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.686+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~AOVL6WFV16NG.1 size=1048576 2024-09-13T00:00:44.693 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.690+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~AOVL6WFV16NG.2 size=832776 2024-09-13T00:00:44.701 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.702+0000 7fb8f2ce0640 1 < completed multipart upload key=8MPUVD37EDN9 upload=AOVL6WFV16NG tag=R4QO1GUI8VLX type=write size=3978504 2024-09-13T00:00:44.707 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.706+0000 7fb8f2ce0640 1 > prepared operation key=63FJDWBO2T9S tag=UUGTA13IP4CA type=write size=489669 2024-09-13T00:00:44.712 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.710+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~UNHR0C5OCOIV.0 size=1048576 2024-09-13T00:00:44.717 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.714+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~UNHR0C5OCOIV.1 size=1048576 2024-09-13T00:00:44.721 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.722+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~UNHR0C5OCOIV.2 size=546840 2024-09-13T00:00:44.732 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.730+0000 7fb8f2ce0640 1 < completed multipart upload key=1PX7DBO9M3YI upload=UNHR0C5OCOIV tag=9U3CC0NLP4P4 type=write size=3692568 2024-09-13T00:00:44.742 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.742+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~Y9T7M79SLQX6.0 size=1048576 2024-09-13T00:00:44.746 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.746+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~Y9T7M79SLQX6.1 size=1045027 2024-09-13T00:00:44.746 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.746+0000 7fb8f2ce0640 1 > prepared multipart upload key=7OX5OJZ8GXU5 upload=Y9T7M79SLQX6 tag=ULHG8Y0EVSRT type=write size=3142179 2024-09-13T00:00:44.752 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.750+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~4VEDZIV0B67F.0 size=1048576 2024-09-13T00:00:44.756 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.754+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~4VEDZIV0B67F.1 size=1048576 2024-09-13T00:00:44.760 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.758+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~4VEDZIV0B67F.2 size=558997 2024-09-13T00:00:44.771 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.770+0000 7fb8f2ce0640 1 < completed multipart upload key=1PX7DBO9M3YI upload=4VEDZIV0B67F tag=072150E7ZO2U type=write size=3704725 2024-09-13T00:00:44.781 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.782+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H9AU4HD247YU.2~PAM1R5T2UMFL.0 size=834202 2024-09-13T00:00:44.781 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.782+0000 7fb8f2ce0640 1 > prepared multipart upload key=H9AU4HD247YU upload=PAM1R5T2UMFL tag=ZBJ5V9Q1EQ2U type=write size=1882778 2024-09-13T00:00:44.786 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.786+0000 7fb8f2ce0640 1 < completed part key=_multipart_PVYZQLGYT3XU.2~90Z6H4D7TPMY.0 size=681560 2024-09-13T00:00:44.797 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.798+0000 7fb8f2ce0640 1 < completed multipart upload key=PVYZQLGYT3XU upload=90Z6H4D7TPMY tag=X9SRFZBWM7T0 type=write size=1730136 2024-09-13T00:00:44.807 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.806+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~U70OAHAWD7OH.0 size=1048576 2024-09-13T00:00:44.811 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.810+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~U70OAHAWD7OH.1 size=1048576 2024-09-13T00:00:44.815 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.814+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~U70OAHAWD7OH.2 size=444004 2024-09-13T00:00:44.815 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.814+0000 7fb8f2ce0640 1 > prepared multipart upload key=2AHYRZUDFHP4 upload=U70OAHAWD7OH tag=VUOWXDSEYA33 type=write size=3589732 2024-09-13T00:00:44.821 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.822+0000 7fb8f2ce0640 1 < completed part key=_multipart_EJN6HR8I7XQF.2~TEW41Q8SG4OC.0 size=261705 2024-09-13T00:00:44.830 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.830+0000 7fb8f2ce0640 1 < completed multipart upload key=EJN6HR8I7XQF upload=TEW41Q8SG4OC tag=L15HZA2I9KPW type=write size=1310281 2024-09-13T00:00:44.836 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.834+0000 7fb8f2ce0640 1 > prepared operation key=QMZREZ1QEDNJ tag=QM9NBBEJCLWU type=del size=3724334 2024-09-13T00:00:44.841 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.842+0000 7fb8f2ce0640 1 < completed delete operation key=2TWD9YLMSYJI tag=YH70YBS3RVV3 type=del 2024-09-13T00:00:44.846 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.846+0000 7fb8f2ce0640 1 > prepared operation key=EJN6HR8I7XQF tag=DFXRQH120NMZ type=del size=873651 2024-09-13T00:00:44.852 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.850+0000 7fb8f2ce0640 1 < completed write operation key=2TWD9YLMSYJI tag=6200D1IH3LWK type=write size=225108 2024-09-13T00:00:44.861 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.862+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~T9AWQMGOHHNG.0 size=1048576 2024-09-13T00:00:44.866 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.866+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~T9AWQMGOHHNG.1 size=812108 2024-09-13T00:00:44.866 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.866+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZTK9AYBSLCOZ upload=T9AWQMGOHHNG tag=UBQYVX17NCDX type=write size=2909260 2024-09-13T00:00:44.872 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.870+0000 7fb8f2ce0640 1 < completed part key=_multipart_8Z6U7OOA0GBH.2~I0IL5BAOED0Q.0 size=1048576 2024-09-13T00:00:44.876 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.874+0000 7fb8f2ce0640 1 < completed part key=_multipart_8Z6U7OOA0GBH.2~I0IL5BAOED0Q.1 size=362659 2024-09-13T00:00:44.885 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.886+0000 7fb8f2ce0640 1 < completed multipart upload key=8Z6U7OOA0GBH upload=I0IL5BAOED0Q tag=B6F3ERI7O7LS type=write size=2459811 2024-09-13T00:00:44.897 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.898+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H4M1TCN5W6XU.2~O0B6G3A74PRY.0 size=1048576 2024-09-13T00:00:44.902 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.902+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H4M1TCN5W6XU.2~O0B6G3A74PRY.1 size=1048576 2024-09-13T00:00:44.907 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.906+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H4M1TCN5W6XU.2~O0B6G3A74PRY.2 size=206265 2024-09-13T00:00:44.907 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.906+0000 7fb8f2ce0640 1 > prepared multipart upload key=H4M1TCN5W6XU upload=O0B6G3A74PRY tag=SDNF25TWGT26 type=write size=3351993 2024-09-13T00:00:44.912 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.910+0000 7fb8f2ce0640 1 < completed part key=_multipart_W8XZ26311RJX.2~8LG08RHGBPN7.0 size=1048576 2024-09-13T00:00:44.917 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.914+0000 7fb8f2ce0640 1 < completed part key=_multipart_W8XZ26311RJX.2~8LG08RHGBPN7.1 size=993150 2024-09-13T00:00:44.927 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.926+0000 7fb8f2ce0640 1 < completed multipart upload key=W8XZ26311RJX upload=8LG08RHGBPN7 tag=9XHCALB5918W type=write size=3090302 2024-09-13T00:00:44.932 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.930+0000 7fb8f2ce0640 1 > prepared operation key=ZFHN2SNTZ5W7 tag=YL3HM62FZ3JE type=del size=3933481 2024-09-13T00:00:44.938 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.938+0000 7fb8f2ce0640 1 < canceled operation key=2AHYRZUDFHP4 tag=D9V7RKI8BSOA type=del size=3925962 2024-09-13T00:00:44.943 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.942+0000 7fb8f2ce0640 1 > prepared operation key=4K76U69GZ9AE tag=XEO1KA79W9FF type=del size=3955778 2024-09-13T00:00:44.954 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.954+0000 7fb8f2ce0640 1 < suggested remove operation key=2TWD9YLMSYJI tag=L40HFWB6SO05 type=del 2024-09-13T00:00:44.960 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.958+0000 7fb8f2ce0640 1 > prepared operation key=P8OYS5ZE1NX0 tag=1K47MU6CJD4T type=del size=3194922 2024-09-13T00:00:44.965 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.966+0000 7fb8f2ce0640 1 < completed delete operation key=F0QCVR9E08WE tag=X15ZZ9D5KMT1 type=del 2024-09-13T00:00:44.976 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.974+0000 7fb8f2ce0640 1 > prepared part key=_multipart_44EDYFT9VF8Y.2~KE5YMZS83HJP.0 size=1048576 2024-09-13T00:00:44.981 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.978+0000 7fb8f2ce0640 1 > prepared part key=_multipart_44EDYFT9VF8Y.2~KE5YMZS83HJP.1 size=1048576 2024-09-13T00:00:44.985 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.986+0000 7fb8f2ce0640 1 > prepared part key=_multipart_44EDYFT9VF8Y.2~KE5YMZS83HJP.2 size=821477 2024-09-13T00:00:44.985 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.986+0000 7fb8f2ce0640 1 > prepared multipart upload key=44EDYFT9VF8Y upload=KE5YMZS83HJP tag=J62VXHK100E6 type=write size=3967205 2024-09-13T00:00:44.990 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.990+0000 7fb8f2ce0640 1 < completed part key=_multipart_P8OYS5ZE1NX0.2~1PAJRXNMF0AQ.0 size=379127 2024-09-13T00:00:45.000 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:44.998+0000 7fb8f2ce0640 1 < completed multipart upload key=P8OYS5ZE1NX0 upload=1PAJRXNMF0AQ tag=YYW6W7TTI89H type=write size=1427703 2024-09-13T00:00:45.005 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.006+0000 7fb8f2ce0640 1 > prepared operation key=70F7D20HI504 tag=YNUYIS8WATV9 type=del size=3266909 2024-09-13T00:00:45.016 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.014+0000 7fb8f2ce0640 1 < suggested remove operation key=2AHYRZUDFHP4 tag=9IQAK9GLHGPL type=del 2024-09-13T00:00:45.021 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.018+0000 7fb8f2ce0640 1 > prepared operation key=7OX5OJZ8GXU5 tag=MD2TO6WTIMWV type=del size=3993684 2024-09-13T00:00:45.026 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.026+0000 7fb8f2ce0640 1 < completed part key=_multipart_5GM88ERM89KT.2~EQ7V42R4AF26.0 size=1048576 2024-09-13T00:00:45.031 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.030+0000 7fb8f2ce0640 1 < completed part key=_multipart_5GM88ERM89KT.2~EQ7V42R4AF26.1 size=1048576 2024-09-13T00:00:45.036 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.034+0000 7fb8f2ce0640 1 < completed part key=_multipart_5GM88ERM89KT.2~EQ7V42R4AF26.2 size=1029350 2024-09-13T00:00:45.045 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.042+0000 7fb8f2ce0640 1 < completed multipart upload key=5GM88ERM89KT upload=EQ7V42R4AF26 tag=31GN2HABIA4Q type=write size=4175078 2024-09-13T00:00:45.055 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.054+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~3F596XPKAPFM.0 size=1048576 2024-09-13T00:00:45.059 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.058+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~3F596XPKAPFM.1 size=1048576 2024-09-13T00:00:45.064 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.062+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~3F596XPKAPFM.2 size=402425 2024-09-13T00:00:45.064 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.062+0000 7fb8f2ce0640 1 > prepared multipart upload key=70F7D20HI504 upload=3F596XPKAPFM tag=2YS90TNHYU7D type=write size=3548153 2024-09-13T00:00:45.065 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.066+0000 7fb8f2ce0640 1 < canceled part key=_multipart_O4MZ4UWNXME9.2~SJ16NP6SFGP6.0 size=1048576 2024-09-13T00:00:45.070 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.070+0000 7fb8f2ce0640 1 < completed part key=_multipart_O4MZ4UWNXME9.2~SJ16NP6SFGP6.1 size=789912 2024-09-13T00:00:45.080 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.078+0000 7fb8f2ce0640 1 < canceled multipart upload key=O4MZ4UWNXME9 upload=SJ16NP6SFGP6 tag=TWKVZ3W9JQGY type=write size=2887064 2024-09-13T00:00:45.089 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.090+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~A0U5EC3IOSF6.0 size=1048576 2024-09-13T00:00:45.095 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.094+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~A0U5EC3IOSF6.1 size=822284 2024-09-13T00:00:45.095 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.094+0000 7fb8f2ce0640 1 > prepared multipart upload key=63FJDWBO2T9S upload=A0U5EC3IOSF6 tag=1LX20DMX3Q9A type=write size=2919436 2024-09-13T00:00:45.100 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.098+0000 7fb8f2ce0640 1 < completed write operation key=63FJDWBO2T9S tag=UUGTA13IP4CA type=write size=489669 2024-09-13T00:00:45.105 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.106+0000 7fb8f2ce0640 1 > prepared operation key=8Z6U7OOA0GBH tag=4A7UAJQUURZT type=write size=394535 2024-09-13T00:00:45.111 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.110+0000 7fb8f2ce0640 1 < completed part key=_multipart_7OX5OJZ8GXU5.2~Y9T7M79SLQX6.0 size=1048576 2024-09-13T00:00:45.115 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.114+0000 7fb8f2ce0640 1 < completed part key=_multipart_7OX5OJZ8GXU5.2~Y9T7M79SLQX6.1 size=1045027 2024-09-13T00:00:45.125 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.122+0000 7fb8f2ce0640 1 < completed multipart upload key=7OX5OJZ8GXU5 upload=Y9T7M79SLQX6 tag=ULHG8Y0EVSRT type=write size=3142179 2024-09-13T00:00:45.134 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.134+0000 7fb8f2ce0640 1 > prepared part key=_multipart_J8NGDSF9XRTP.2~OZCWSTJZ0P5E.0 size=1048576 2024-09-13T00:00:45.139 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.138+0000 7fb8f2ce0640 1 > prepared part key=_multipart_J8NGDSF9XRTP.2~OZCWSTJZ0P5E.1 size=110278 2024-09-13T00:00:45.139 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.138+0000 7fb8f2ce0640 1 > prepared multipart upload key=J8NGDSF9XRTP upload=OZCWSTJZ0P5E tag=EWYV3E5KQ16S type=write size=2207430 2024-09-13T00:00:45.144 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.142+0000 7fb8f2ce0640 1 < completed part key=_multipart_H9AU4HD247YU.2~PAM1R5T2UMFL.0 size=834202 2024-09-13T00:00:45.154 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.154+0000 7fb8f2ce0640 1 < completed multipart upload key=H9AU4HD247YU upload=PAM1R5T2UMFL tag=ZBJ5V9Q1EQ2U type=write size=1882778 2024-09-13T00:00:45.164 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.162+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~EAKDD0QF3TWR.0 size=1048576 2024-09-13T00:00:45.168 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.166+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~EAKDD0QF3TWR.1 size=906783 2024-09-13T00:00:45.168 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.166+0000 7fb8f2ce0640 1 > prepared multipart upload key=63FJDWBO2T9S upload=EAKDD0QF3TWR tag=06EI1TEK6Y0F type=write size=3003935 2024-09-13T00:00:45.173 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.174+0000 7fb8f2ce0640 1 < completed part key=_multipart_2AHYRZUDFHP4.2~U70OAHAWD7OH.0 size=1048576 2024-09-13T00:00:45.179 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.178+0000 7fb8f2ce0640 1 < completed part key=_multipart_2AHYRZUDFHP4.2~U70OAHAWD7OH.1 size=1048576 2024-09-13T00:00:45.183 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.182+0000 7fb8f2ce0640 1 < completed part key=_multipart_2AHYRZUDFHP4.2~U70OAHAWD7OH.2 size=444004 2024-09-13T00:00:45.193 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.194+0000 7fb8f2ce0640 1 < completed multipart upload key=2AHYRZUDFHP4 upload=U70OAHAWD7OH tag=VUOWXDSEYA33 type=write size=3589732 2024-09-13T00:00:45.204 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.202+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H9AU4HD247YU.2~MJ6D2M3TX0GL.0 size=307177 2024-09-13T00:00:45.204 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.202+0000 7fb8f2ce0640 1 > prepared multipart upload key=H9AU4HD247YU upload=MJ6D2M3TX0GL tag=5FJI1G7CQWFR type=write size=1355753 2024-09-13T00:00:45.210 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.210+0000 7fb8f2ce0640 1 < canceled operation key=QMZREZ1QEDNJ tag=QM9NBBEJCLWU type=del size=3724334 2024-09-13T00:00:45.216 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.214+0000 7fb8f2ce0640 1 > prepared operation key=2AHYRZUDFHP4 tag=N2YMFL6ZWRP4 type=del size=814290 2024-09-13T00:00:45.222 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.222+0000 7fb8f2ce0640 1 < completed delete operation key=EJN6HR8I7XQF tag=DFXRQH120NMZ type=del 2024-09-13T00:00:45.228 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.226+0000 7fb8f2ce0640 1 > prepared operation key=4K76U69GZ9AE tag=P3MMMBQN3759 type=del size=1854328 2024-09-13T00:00:45.233 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.234+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~T9AWQMGOHHNG.0 size=1048576 2024-09-13T00:00:45.238 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.238+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~T9AWQMGOHHNG.1 size=812108 2024-09-13T00:00:45.247 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.246+0000 7fb8f2ce0640 1 < completed multipart upload key=ZTK9AYBSLCOZ upload=T9AWQMGOHHNG tag=UBQYVX17NCDX type=write size=2909260 2024-09-13T00:00:45.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:44 smithi136 bash[76371]: cluster 2024-09-13T00:00:43.131010+0000 mgr.y (mgr.24661) 708 : cluster [DBG] pgmap v1384: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 637 KiB/s rd, 219 KiB/s wr, 1.13k op/s 2024-09-13T00:00:45.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:44 smithi136 bash[75164]: cluster 2024-09-13T00:00:43.131010+0000 mgr.y (mgr.24661) 708 : cluster [DBG] pgmap v1384: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 637 KiB/s rd, 219 KiB/s wr, 1.13k op/s 2024-09-13T00:00:45.258 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.258+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~E5NX6J4V1ZI8.0 size=1048576 2024-09-13T00:00:45.261 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.262+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~E5NX6J4V1ZI8.1 size=589202 2024-09-13T00:00:45.261 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.262+0000 7fb8f2ce0640 1 > prepared multipart upload key=8Z6U7OOA0GBH upload=E5NX6J4V1ZI8 tag=N82H2EI2SZL9 type=write size=2686354 2024-09-13T00:00:45.266 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.266+0000 7fb8f2ce0640 1 < completed part key=_multipart_H4M1TCN5W6XU.2~O0B6G3A74PRY.0 size=1048576 2024-09-13T00:00:45.271 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.270+0000 7fb8f2ce0640 1 < completed part key=_multipart_H4M1TCN5W6XU.2~O0B6G3A74PRY.1 size=1048576 2024-09-13T00:00:45.271 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.270+0000 7fb8f2ce0640 1 < canceled part key=_multipart_H4M1TCN5W6XU.2~O0B6G3A74PRY.2 size=206265 2024-09-13T00:00:45.280 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.278+0000 7fb8f2ce0640 1 < canceled multipart upload key=H4M1TCN5W6XU upload=O0B6G3A74PRY tag=SDNF25TWGT26 type=write size=3351993 2024-09-13T00:00:45.285 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.286+0000 7fb8f2ce0640 1 > prepared operation key=NQJQC8CEAC15 tag=ONLB3AXBZLZP type=del size=3505441 2024-09-13T00:00:45.296 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.294+0000 7fb8f2ce0640 1 < suggested remove operation key=ZFHN2SNTZ5W7 tag=YL3HM62FZ3JE type=del 2024-09-13T00:00:45.301 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.302+0000 7fb8f2ce0640 1 > prepared operation key=7OX5OJZ8GXU5 tag=GQITI99J55FQ type=del size=3427728 2024-09-13T00:00:45.306 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.306+0000 7fb8f2ce0640 1 < completed delete operation key=4K76U69GZ9AE tag=XEO1KA79W9FF type=del 2024-09-13T00:00:45.312 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.310+0000 7fb8f2ce0640 1 > prepared operation key=J8NGDSF9XRTP tag=EG22DWVDVEHK type=del size=749136 2024-09-13T00:00:45.317 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.318+0000 7fb8f2ce0640 1 < completed delete operation key=P8OYS5ZE1NX0 tag=1K47MU6CJD4T type=del 2024-09-13T00:00:45.322 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.322+0000 7fb8f2ce0640 1 > prepared operation key=ZTK9AYBSLCOZ tag=B6GXC3LWI379 type=del size=802185 2024-09-13T00:00:45.328 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.326+0000 7fb8f2ce0640 1 < completed part key=_multipart_44EDYFT9VF8Y.2~KE5YMZS83HJP.0 size=1048576 2024-09-13T00:00:45.332 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.330+0000 7fb8f2ce0640 1 < completed part key=_multipart_44EDYFT9VF8Y.2~KE5YMZS83HJP.1 size=1048576 2024-09-13T00:00:45.337 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.338+0000 7fb8f2ce0640 1 < completed part key=_multipart_44EDYFT9VF8Y.2~KE5YMZS83HJP.2 size=821477 2024-09-13T00:00:45.346 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.346+0000 7fb8f2ce0640 1 < completed multipart upload key=44EDYFT9VF8Y upload=KE5YMZS83HJP tag=J62VXHK100E6 type=write size=3967205 2024-09-13T00:00:45.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:44 smithi181 bash[67587]: cluster 2024-09-13T00:00:43.131010+0000 mgr.y (mgr.24661) 708 : cluster [DBG] pgmap v1384: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 637 KiB/s rd, 219 KiB/s wr, 1.13k op/s 2024-09-13T00:00:45.351 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.350+0000 7fb8f2ce0640 1 > prepared operation key=EJN6HR8I7XQF tag=SDDBORKTHEUQ type=del size=372689 2024-09-13T00:00:45.356 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.354+0000 7fb8f2ce0640 1 < completed delete operation key=70F7D20HI504 tag=YNUYIS8WATV9 type=del 2024-09-13T00:00:45.365 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.366+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~DLD8YLRWKY1K.0 size=793730 2024-09-13T00:00:45.365 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.366+0000 7fb8f2ce0640 1 > prepared multipart upload key=70F7D20HI504 upload=DLD8YLRWKY1K tag=77ZDUJVTCNE9 type=write size=1842306 2024-09-13T00:00:45.372 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.370+0000 7fb8f2ce0640 1 < completed delete operation key=7OX5OJZ8GXU5 tag=MD2TO6WTIMWV type=del 2024-09-13T00:00:45.375 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.374+0000 7fb8f2ce0640 1 > prepared operation key=4K76U69GZ9AE tag=GH2VA9KTUH01 type=del size=158434 2024-09-13T00:00:45.381 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.382+0000 7fb8f2ce0640 1 < completed part key=_multipart_70F7D20HI504.2~3F596XPKAPFM.0 size=1048576 2024-09-13T00:00:45.386 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.386+0000 7fb8f2ce0640 1 < completed part key=_multipart_70F7D20HI504.2~3F596XPKAPFM.1 size=1048576 2024-09-13T00:00:45.390 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.390+0000 7fb8f2ce0640 1 < completed part key=_multipart_70F7D20HI504.2~3F596XPKAPFM.2 size=402425 2024-09-13T00:00:45.399 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.398+0000 7fb8f2ce0640 1 < completed multipart upload key=70F7D20HI504 upload=3F596XPKAPFM tag=2YS90TNHYU7D type=write size=3548153 2024-09-13T00:00:45.409 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.410+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~GSW0UVK84OOH.0 size=1048576 2024-09-13T00:00:45.414 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.414+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~GSW0UVK84OOH.1 size=736022 2024-09-13T00:00:45.414 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.414+0000 7fb8f2ce0640 1 > prepared multipart upload key=63FJDWBO2T9S upload=GSW0UVK84OOH tag=ZL3XKWVRWKLC type=write size=2833174 2024-09-13T00:00:45.419 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.418+0000 7fb8f2ce0640 1 < completed part key=_multipart_63FJDWBO2T9S.2~A0U5EC3IOSF6.0 size=1048576 2024-09-13T00:00:45.424 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.422+0000 7fb8f2ce0640 1 < completed part key=_multipart_63FJDWBO2T9S.2~A0U5EC3IOSF6.1 size=822284 2024-09-13T00:00:45.434 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.434+0000 7fb8f2ce0640 1 < completed multipart upload key=63FJDWBO2T9S upload=A0U5EC3IOSF6 tag=1LX20DMX3Q9A type=write size=2919436 2024-09-13T00:00:45.439 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.438+0000 7fb8f2ce0640 1 > prepared operation key=N3Z5XIG3TZRQ tag=SC3VFQR12B87 type=del size=2005702 2024-09-13T00:00:45.444 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.442+0000 7fb8f2ce0640 1 < completed write operation key=8Z6U7OOA0GBH tag=4A7UAJQUURZT type=write size=394535 2024-09-13T00:00:45.453 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.454+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~DI1CI9HYRZSE.0 size=1048576 2024-09-13T00:00:45.458 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.458+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~DI1CI9HYRZSE.1 size=1048576 2024-09-13T00:00:45.464 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.462+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~DI1CI9HYRZSE.2 size=900376 2024-09-13T00:00:45.464 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.462+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZTK9AYBSLCOZ upload=DI1CI9HYRZSE tag=Y0CRGYPLCTK8 type=write size=4046104 2024-09-13T00:00:45.470 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.470+0000 7fb8f2ce0640 1 < completed part key=_multipart_J8NGDSF9XRTP.2~OZCWSTJZ0P5E.0 size=1048576 2024-09-13T00:00:45.474 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.474+0000 7fb8f2ce0640 1 < completed part key=_multipart_J8NGDSF9XRTP.2~OZCWSTJZ0P5E.1 size=110278 2024-09-13T00:00:45.483 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.482+0000 7fb8f2ce0640 1 < completed multipart upload key=J8NGDSF9XRTP upload=OZCWSTJZ0P5E tag=EWYV3E5KQ16S type=write size=2207430 2024-09-13T00:00:45.493 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.494+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~RBR31ISUH4VR.0 size=130243 2024-09-13T00:00:45.493 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.494+0000 7fb8f2ce0640 1 > prepared multipart upload key=8MPUVD37EDN9 upload=RBR31ISUH4VR tag=LCWX92C2VTSM type=write size=1178819 2024-09-13T00:00:45.499 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.498+0000 7fb8f2ce0640 1 < completed part key=_multipart_63FJDWBO2T9S.2~EAKDD0QF3TWR.0 size=1048576 2024-09-13T00:00:45.503 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.502+0000 7fb8f2ce0640 1 < completed part key=_multipart_63FJDWBO2T9S.2~EAKDD0QF3TWR.1 size=906783 2024-09-13T00:00:45.513 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.510+0000 7fb8f2ce0640 1 < completed multipart upload key=63FJDWBO2T9S upload=EAKDD0QF3TWR tag=06EI1TEK6Y0F type=write size=3003935 2024-09-13T00:00:45.518 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.518+0000 7fb8f2ce0640 1 > prepared operation key=EUE3LT0PDIXV tag=Y0A134884EUV type=del size=3832633 2024-09-13T00:00:45.523 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.522+0000 7fb8f2ce0640 1 < completed part key=_multipart_H9AU4HD247YU.2~MJ6D2M3TX0GL.0 size=307177 2024-09-13T00:00:45.533 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.534+0000 7fb8f2ce0640 1 < completed multipart upload key=H9AU4HD247YU upload=MJ6D2M3TX0GL tag=5FJI1G7CQWFR type=write size=1355753 2024-09-13T00:00:45.545 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.542+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~3L2WCE5CXWZR.0 size=812752 2024-09-13T00:00:45.545 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.542+0000 7fb8f2ce0640 1 > prepared multipart upload key=63FJDWBO2T9S upload=3L2WCE5CXWZR tag=I6C1815H24UW type=write size=1861328 2024-09-13T00:00:45.551 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.550+0000 7fb8f2ce0640 1 < completed delete operation key=2AHYRZUDFHP4 tag=N2YMFL6ZWRP4 type=del 2024-09-13T00:00:45.556 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.554+0000 7fb8f2ce0640 1 > prepared operation key=EJN6HR8I7XQF tag=04DSTSGOV2WS type=del size=937265 2024-09-13T00:00:45.561 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.562+0000 7fb8f2ce0640 1 < completed delete operation key=4K76U69GZ9AE tag=P3MMMBQN3759 type=del 2024-09-13T00:00:45.568 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.566+0000 7fb8f2ce0640 1 > prepared operation key=ZFHN2SNTZ5W7 tag=OVEEMZ3J54P2 type=del size=3484801 2024-09-13T00:00:45.576 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.574+0000 7fb8f2ce0640 1 < completed part key=_multipart_8Z6U7OOA0GBH.2~E5NX6J4V1ZI8.0 size=1048576 2024-09-13T00:00:45.578 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.578+0000 7fb8f2ce0640 1 < completed part key=_multipart_8Z6U7OOA0GBH.2~E5NX6J4V1ZI8.1 size=589202 2024-09-13T00:00:45.587 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.586+0000 7fb8f2ce0640 1 < completed multipart upload key=8Z6U7OOA0GBH upload=E5NX6J4V1ZI8 tag=N82H2EI2SZL9 type=write size=2686354 2024-09-13T00:00:45.593 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.594+0000 7fb8f2ce0640 1 > prepared operation key=F0QCVR9E08WE tag=5I2PKRERZ35Y type=write size=124533 2024-09-13T00:00:45.598 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.598+0000 7fb8f2ce0640 1 < canceled operation key=NQJQC8CEAC15 tag=ONLB3AXBZLZP type=del size=3505441 2024-09-13T00:00:45.603 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.602+0000 7fb8f2ce0640 1 > prepared operation key=1HX06CCZJ1DD tag=QAS5E82XK9A5 type=del size=2176781 2024-09-13T00:00:45.609 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.610+0000 7fb8f2ce0640 1 < completed delete operation key=7OX5OJZ8GXU5 tag=GQITI99J55FQ type=del 2024-09-13T00:00:45.614 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.614+0000 7fb8f2ce0640 1 > prepared operation key=4K76U69GZ9AE tag=DUKX5YY8RFEK type=del size=3747926 2024-09-13T00:00:45.619 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.618+0000 7fb8f2ce0640 1 < completed delete operation key=J8NGDSF9XRTP tag=EG22DWVDVEHK type=del 2024-09-13T00:00:45.624 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.622+0000 7fb8f2ce0640 1 > prepared operation key=NQJQC8CEAC15 tag=UQXC6TEJTQW1 type=del size=3162707 2024-09-13T00:00:45.629 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.630+0000 7fb8f2ce0640 1 < completed delete operation key=ZTK9AYBSLCOZ tag=B6GXC3LWI379 type=del 2024-09-13T00:00:45.634 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.634+0000 7fb8f2ce0640 1 > prepared operation key=W8XZ26311RJX tag=M4SGZGOKPD8N type=write size=758286 2024-09-13T00:00:45.639 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.638+0000 7fb8f2ce0640 1 < completed delete operation key=EJN6HR8I7XQF tag=SDDBORKTHEUQ type=del 2024-09-13T00:00:45.644 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.642+0000 7fb8f2ce0640 1 > prepared operation key=5GM88ERM89KT tag=N8UY0EB4XWT6 type=del size=1180833 2024-09-13T00:00:45.650 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.650+0000 7fb8f2ce0640 1 < completed part key=_multipart_70F7D20HI504.2~DLD8YLRWKY1K.0 size=793730 2024-09-13T00:00:45.659 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.658+0000 7fb8f2ce0640 1 < completed multipart upload key=70F7D20HI504 upload=DLD8YLRWKY1K tag=77ZDUJVTCNE9 type=write size=1842306 2024-09-13T00:00:45.664 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.662+0000 7fb8f2ce0640 1 > prepared operation key=EUE3LT0PDIXV tag=5J2BAJDP8NAO type=del size=1190894 2024-09-13T00:00:45.669 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.670+0000 7fb8f2ce0640 1 < completed delete operation key=4K76U69GZ9AE tag=GH2VA9KTUH01 type=del 2024-09-13T00:00:45.674 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.674+0000 7fb8f2ce0640 1 > prepared operation key=W8XZ26311RJX tag=7RRDHYUGBEBP type=write size=612766 2024-09-13T00:00:45.680 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.678+0000 7fb8f2ce0640 1 < completed part key=_multipart_63FJDWBO2T9S.2~GSW0UVK84OOH.0 size=1048576 2024-09-13T00:00:45.684 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.682+0000 7fb8f2ce0640 1 < completed part key=_multipart_63FJDWBO2T9S.2~GSW0UVK84OOH.1 size=736022 2024-09-13T00:00:45.693 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.694+0000 7fb8f2ce0640 1 < completed multipart upload key=63FJDWBO2T9S upload=GSW0UVK84OOH tag=ZL3XKWVRWKLC type=write size=2833174 2024-09-13T00:00:45.698 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.698+0000 7fb8f2ce0640 1 > prepared operation key=EJN6HR8I7XQF tag=EB23C928DVOD type=del size=3373727 2024-09-13T00:00:45.704 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.702+0000 7fb8f2ce0640 1 < completed delete operation key=N3Z5XIG3TZRQ tag=SC3VFQR12B87 type=del 2024-09-13T00:00:45.709 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.710+0000 7fb8f2ce0640 1 > prepared operation key=1PX7DBO9M3YI tag=QGST3LGOVMQR type=del size=1644902 2024-09-13T00:00:45.714 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.714+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~DI1CI9HYRZSE.0 size=1048576 2024-09-13T00:00:45.719 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.718+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~DI1CI9HYRZSE.1 size=1048576 2024-09-13T00:00:45.724 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.722+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~DI1CI9HYRZSE.2 size=900376 2024-09-13T00:00:45.732 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.730+0000 7fb8f2ce0640 1 < completed multipart upload key=ZTK9AYBSLCOZ upload=DI1CI9HYRZSE tag=Y0CRGYPLCTK8 type=write size=4046104 2024-09-13T00:00:45.737 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.738+0000 7fb8f2ce0640 1 > prepared operation key=ZTK9AYBSLCOZ tag=WW56ZX8ID3DL type=del size=2377857 2024-09-13T00:00:45.743 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.742+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~RBR31ISUH4VR.0 size=130243 2024-09-13T00:00:45.755 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.750+0000 7fb8f2ce0640 1 < completed multipart upload key=8MPUVD37EDN9 upload=RBR31ISUH4VR tag=LCWX92C2VTSM type=write size=1178819 2024-09-13T00:00:45.757 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.758+0000 7fb8f2ce0640 1 > prepared operation key=1HX06CCZJ1DD tag=8CHRSPZIURYF type=write size=404376 2024-09-13T00:00:45.768 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.766+0000 7fb8f2ce0640 1 < suggested remove operation key=EUE3LT0PDIXV tag=Y0A134884EUV type=del 2024-09-13T00:00:45.778 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.778+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2TWD9YLMSYJI.2~3MDD48YXENK5.0 size=1048576 2024-09-13T00:00:45.784 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.782+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2TWD9YLMSYJI.2~3MDD48YXENK5.1 size=137808 2024-09-13T00:00:45.784 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.782+0000 7fb8f2ce0640 1 > prepared multipart upload key=2TWD9YLMSYJI upload=3MDD48YXENK5 tag=XL1XQ7NAYUZ4 type=write size=2234960 2024-09-13T00:00:45.788 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.786+0000 7fb8f2ce0640 1 < completed part key=_multipart_63FJDWBO2T9S.2~3L2WCE5CXWZR.0 size=812752 2024-09-13T00:00:45.797 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.794+0000 7fb8f2ce0640 1 < completed multipart upload key=63FJDWBO2T9S upload=3L2WCE5CXWZR tag=I6C1815H24UW type=write size=1861328 2024-09-13T00:00:45.802 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.802+0000 7fb8f2ce0640 1 > prepared operation key=1HX06CCZJ1DD tag=QJ8ZLI5HDDEK type=del size=991844 2024-09-13T00:00:45.807 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.806+0000 7fb8f2ce0640 1 < completed delete operation key=EJN6HR8I7XQF tag=04DSTSGOV2WS type=del 2024-09-13T00:00:45.812 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.810+0000 7fb8f2ce0640 1 > prepared operation key=W8XZ26311RJX tag=6QDB9MCDD28N type=del size=2948101 2024-09-13T00:00:45.824 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.822+0000 7fb8f2ce0640 1 < suggested remove operation key=ZFHN2SNTZ5W7 tag=OVEEMZ3J54P2 type=del 2024-09-13T00:00:45.830 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.830+0000 7fb8f2ce0640 1 > prepared operation key=7OX5OJZ8GXU5 tag=O8O0A4FLVY3H type=del size=3919167 2024-09-13T00:00:45.836 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.834+0000 7fb8f2ce0640 1 < completed write operation key=F0QCVR9E08WE tag=5I2PKRERZ35Y type=write size=124533 2024-09-13T00:00:45.841 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.842+0000 7fb8f2ce0640 1 > prepared operation key=1HX06CCZJ1DD tag=ZP5233P46TZ9 type=del size=421461 2024-09-13T00:00:45.847 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.846+0000 7fb8f2ce0640 1 < completed delete operation key=1HX06CCZJ1DD tag=QAS5E82XK9A5 type=del 2024-09-13T00:00:45.852 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.850+0000 7fb8f2ce0640 1 > prepared operation key=EJN6HR8I7XQF tag=NOZC6AX4MJY0 type=del size=1691915 2024-09-13T00:00:45.858 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.858+0000 7fb8f2ce0640 1 < completed delete operation key=4K76U69GZ9AE tag=DUKX5YY8RFEK type=del 2024-09-13T00:00:45.868 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.866+0000 7fb8f2ce0640 1 > prepared part key=_multipart_N3Z5XIG3TZRQ.2~BZIZPLCDGKIN.0 size=1048576 2024-09-13T00:00:45.872 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.870+0000 7fb8f2ce0640 1 > prepared part key=_multipart_N3Z5XIG3TZRQ.2~BZIZPLCDGKIN.1 size=1048576 2024-09-13T00:00:45.876 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.874+0000 7fb8f2ce0640 1 > prepared part key=_multipart_N3Z5XIG3TZRQ.2~BZIZPLCDGKIN.2 size=372168 2024-09-13T00:00:45.876 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.874+0000 7fb8f2ce0640 1 > prepared multipart upload key=N3Z5XIG3TZRQ upload=BZIZPLCDGKIN tag=SJIFFEFECI5M type=write size=3517896 2024-09-13T00:00:45.881 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.882+0000 7fb8f2ce0640 1 < canceled operation key=NQJQC8CEAC15 tag=UQXC6TEJTQW1 type=del size=3162707 2024-09-13T00:00:45.887 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.886+0000 7fb8f2ce0640 1 > prepared operation key=ZFHN2SNTZ5W7 tag=FCHZRMEBN9AE type=del size=3766816 2024-09-13T00:00:45.893 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.894+0000 7fb8f2ce0640 1 < completed write operation key=W8XZ26311RJX tag=M4SGZGOKPD8N type=write size=758286 2024-09-13T00:00:45.899 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.898+0000 7fb8f2ce0640 1 > prepared operation key=W8XZ26311RJX tag=5H4DSVML2TOX type=del size=3956038 2024-09-13T00:00:45.909 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.910+0000 7fb8f2ce0640 1 < suggested remove operation key=5GM88ERM89KT tag=N8UY0EB4XWT6 type=del 2024-09-13T00:00:45.919 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.918+0000 7fb8f2ce0640 1 > prepared part key=_multipart_44EDYFT9VF8Y.2~A4QK418H64YU.0 size=1048576 2024-09-13T00:00:45.924 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.922+0000 7fb8f2ce0640 1 > prepared part key=_multipart_44EDYFT9VF8Y.2~A4QK418H64YU.1 size=785643 2024-09-13T00:00:45.924 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.922+0000 7fb8f2ce0640 1 > prepared multipart upload key=44EDYFT9VF8Y upload=A4QK418H64YU tag=6WKZSRWNRPWC type=write size=2882795 2024-09-13T00:00:45.931 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.930+0000 7fb8f2ce0640 -1 < failed to complete operation key=EUE3LT0PDIXV tag=5J2BAJDP8NAO type=del size=1190894: (22) Invalid argument 2024-09-13T00:00:45.937 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.938+0000 7fb8f2ce0640 1 > prepared operation key=63FJDWBO2T9S tag=SG1A8OQWFXQH type=del size=1433935 2024-09-13T00:00:45.943 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.942+0000 7fb8f2ce0640 1 < completed write operation key=W8XZ26311RJX tag=7RRDHYUGBEBP type=write size=612766 2024-09-13T00:00:45.949 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.950+0000 7fb8f2ce0640 1 > prepared operation key=H4M1TCN5W6XU tag=6Q3YD0OX26ZT type=del size=2487443 2024-09-13T00:00:45.960 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.962+0000 7fb8f2ce0640 1 < completed delete operation key=EJN6HR8I7XQF tag=EB23C928DVOD type=del 2024-09-13T00:00:45.965 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.966+0000 7fb8f2ce0640 1 > prepared operation key=H5EIP0E9CBWX tag=I37LC3GW9BSV type=write size=653897 2024-09-13T00:00:45.976 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.974+0000 7fb8f2ce0640 1 < suggested update operation key=1PX7DBO9M3YI tag=QGST3LGOVMQR type=del size=1644902 2024-09-13T00:00:45.985 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.982+0000 7fb8f2ce0640 1 > prepared operation key=0WC6P8YNL0H2 tag=7UHIFGAO2RHU type=write size=928887 2024-09-13T00:00:45.986 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.986+0000 7fb8f2ce0640 1 < completed delete operation key=ZTK9AYBSLCOZ tag=WW56ZX8ID3DL type=del 2024-09-13T00:00:45.996 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:45.994+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~WZSZP2BP141T.0 size=1048576 2024-09-13T00:00:46.001 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.002+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~WZSZP2BP141T.1 size=1048576 2024-09-13T00:00:46.005 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.006+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~WZSZP2BP141T.2 size=161604 2024-09-13T00:00:46.005 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.006+0000 7fb8f2ce0640 1 > prepared multipart upload key=NQJQC8CEAC15 upload=WZSZP2BP141T tag=4GYAXMQSAWTC type=write size=3307332 2024-09-13T00:00:46.011 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.010+0000 7fb8f2ce0640 1 < completed write operation key=1HX06CCZJ1DD tag=8CHRSPZIURYF type=write size=404376 2024-09-13T00:00:46.017 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.014+0000 7fb8f2ce0640 1 > prepared operation key=H5EIP0E9CBWX tag=9L2W8MKMIBHA type=del size=2215991 2024-09-13T00:00:46.022 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.022+0000 7fb8f2ce0640 1 < completed part key=_multipart_2TWD9YLMSYJI.2~3MDD48YXENK5.0 size=1048576 2024-09-13T00:00:46.027 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.026+0000 7fb8f2ce0640 1 < completed part key=_multipart_2TWD9YLMSYJI.2~3MDD48YXENK5.1 size=137808 2024-09-13T00:00:46.036 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.034+0000 7fb8f2ce0640 1 < completed multipart upload key=2TWD9YLMSYJI upload=3MDD48YXENK5 tag=XL1XQ7NAYUZ4 type=write size=2234960 2024-09-13T00:00:46.041 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.042+0000 7fb8f2ce0640 1 > prepared operation key=P8OYS5ZE1NX0 tag=IO4P9TIL30DV type=del size=4085600 2024-09-13T00:00:46.046 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.046+0000 7fb8f2ce0640 1 < completed delete operation key=1HX06CCZJ1DD tag=QJ8ZLI5HDDEK type=del 2024-09-13T00:00:46.056 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.054+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~PFD0IYL1IKXS.0 size=1048576 2024-09-13T00:00:46.060 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.058+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~PFD0IYL1IKXS.1 size=166804 2024-09-13T00:00:46.060 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.058+0000 7fb8f2ce0640 1 > prepared multipart upload key=8MPUVD37EDN9 upload=PFD0IYL1IKXS tag=BVSE4WN1TA2G type=write size=2263956 2024-09-13T00:00:46.071 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.070+0000 7fb8f2ce0640 1 < suggested update operation key=W8XZ26311RJX tag=6QDB9MCDD28N type=del size=2948101 2024-09-13T00:00:46.077 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.078+0000 7fb8f2ce0640 1 > prepared operation key=H5EIP0E9CBWX tag=NSZ225ED81TH type=del size=504329 2024-09-13T00:00:46.082 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.082+0000 7fb8f2ce0640 1 < completed delete operation key=7OX5OJZ8GXU5 tag=O8O0A4FLVY3H type=del 2024-09-13T00:00:46.088 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.086+0000 7fb8f2ce0640 1 > prepared operation key=NQJQC8CEAC15 tag=E3GKS6Q9WQ45 type=write size=442568 2024-09-13T00:00:46.093 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.094+0000 7fb8f2ce0640 1 < completed delete operation key=1HX06CCZJ1DD tag=ZP5233P46TZ9 type=del 2024-09-13T00:00:46.099 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.098+0000 7fb8f2ce0640 1 > prepared operation key=PVYZQLGYT3XU tag=D5F4RMYPQ2OB type=del size=110847 2024-09-13T00:00:46.105 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.106+0000 7fb8f2ce0640 1 < completed delete operation key=EJN6HR8I7XQF tag=NOZC6AX4MJY0 type=del 2024-09-13T00:00:46.111 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.110+0000 7fb8f2ce0640 1 > prepared operation key=F0QCVR9E08WE tag=SHULG9KGY4JX type=del size=1514216 2024-09-13T00:00:46.116 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.114+0000 7fb8f2ce0640 1 < completed part key=_multipart_N3Z5XIG3TZRQ.2~BZIZPLCDGKIN.0 size=1048576 2024-09-13T00:00:46.121 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.122+0000 7fb8f2ce0640 1 < completed part key=_multipart_N3Z5XIG3TZRQ.2~BZIZPLCDGKIN.1 size=1048576 2024-09-13T00:00:46.125 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.126+0000 7fb8f2ce0640 1 < completed part key=_multipart_N3Z5XIG3TZRQ.2~BZIZPLCDGKIN.2 size=372168 2024-09-13T00:00:46.135 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.134+0000 7fb8f2ce0640 1 < completed multipart upload key=N3Z5XIG3TZRQ upload=BZIZPLCDGKIN tag=SJIFFEFECI5M type=write size=3517896 2024-09-13T00:00:46.140 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.138+0000 7fb8f2ce0640 1 > prepared operation key=NQJQC8CEAC15 tag=Y846MTOVDPDR type=del size=1266111 2024-09-13T00:00:46.145 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.146+0000 7fb8f2ce0640 1 < completed delete operation key=ZFHN2SNTZ5W7 tag=FCHZRMEBN9AE type=del 2024-09-13T00:00:46.150 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.150+0000 7fb8f2ce0640 1 > prepared operation key=2AHYRZUDFHP4 tag=R2VAYF68OBKA type=del size=2624426 2024-09-13T00:00:46.157 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.158+0000 7fb8f2ce0640 -1 < failed to complete operation key=W8XZ26311RJX tag=5H4DSVML2TOX type=del size=3956038: (22) Invalid argument 2024-09-13T00:00:46.163 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.162+0000 7fb8f2ce0640 1 > prepared operation key=FX0NCGLK0KPJ tag=V06A62Z8AHZJ type=del size=3412146 2024-09-13T00:00:46.169 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.170+0000 7fb8f2ce0640 1 < completed part key=_multipart_44EDYFT9VF8Y.2~A4QK418H64YU.0 size=1048576 2024-09-13T00:00:46.173 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.174+0000 7fb8f2ce0640 1 < completed part key=_multipart_44EDYFT9VF8Y.2~A4QK418H64YU.1 size=785643 2024-09-13T00:00:46.183 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.182+0000 7fb8f2ce0640 1 < completed multipart upload key=44EDYFT9VF8Y upload=A4QK418H64YU tag=6WKZSRWNRPWC type=write size=2882795 2024-09-13T00:00:46.189 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.186+0000 7fb8f2ce0640 1 > prepared operation key=1HX06CCZJ1DD tag=ZE3GGIFBTN03 type=del size=3766054 2024-09-13T00:00:46.194 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.194+0000 7fb8f2ce0640 1 < completed delete operation key=63FJDWBO2T9S tag=SG1A8OQWFXQH type=del 2024-09-13T00:00:46.204 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.202+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZFHN2SNTZ5W7.2~TAL8Q3FA125E.0 size=1048576 2024-09-13T00:00:46.208 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.206+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZFHN2SNTZ5W7.2~TAL8Q3FA125E.1 size=1048576 2024-09-13T00:00:46.213 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.214+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZFHN2SNTZ5W7.2~TAL8Q3FA125E.2 size=871580 2024-09-13T00:00:46.213 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.214+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZFHN2SNTZ5W7 upload=TAL8Q3FA125E tag=4AUMXMDO1NST type=write size=4017308 2024-09-13T00:00:46.219 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.218+0000 7fb8f2ce0640 1 < completed delete operation key=H4M1TCN5W6XU tag=6Q3YD0OX26ZT type=del 2024-09-13T00:00:46.230 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.230+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2TWD9YLMSYJI.2~VXEOHWRP3MS1.0 size=1048576 2024-09-13T00:00:46.234 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.234+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2TWD9YLMSYJI.2~VXEOHWRP3MS1.1 size=1048576 2024-09-13T00:00:46.239 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.238+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2TWD9YLMSYJI.2~VXEOHWRP3MS1.2 size=223909 2024-09-13T00:00:46.239 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.238+0000 7fb8f2ce0640 1 > prepared multipart upload key=2TWD9YLMSYJI upload=VXEOHWRP3MS1 tag=63Z0IT3ZQDZ7 type=write size=3369637 2024-09-13T00:00:46.250 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.250+0000 7fb8f2ce0640 1 < suggested update operation key=H5EIP0E9CBWX tag=I37LC3GW9BSV type=write size=653897 2024-09-13T00:00:46.255 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.254+0000 7fb8f2ce0640 1 > prepared operation key=H4M1TCN5W6XU tag=Q4KV4OPW37IL type=write size=689947 2024-09-13T00:00:46.266 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.266+0000 7fb8f2ce0640 1 < suggested remove operation key=0WC6P8YNL0H2 tag=7UHIFGAO2RHU type=write 2024-09-13T00:00:46.276 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.274+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~8A9IN1R18UAD.0 size=1048576 2024-09-13T00:00:46.281 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.278+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~8A9IN1R18UAD.1 size=1048576 2024-09-13T00:00:46.286 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.286+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~8A9IN1R18UAD.2 size=744581 2024-09-13T00:00:46.286 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.286+0000 7fb8f2ce0640 1 > prepared multipart upload key=W8XZ26311RJX upload=8A9IN1R18UAD tag=WVSFDUJ3HKNL type=write size=3890309 2024-09-13T00:00:46.291 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.290+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~WZSZP2BP141T.0 size=1048576 2024-09-13T00:00:46.296 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.294+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~WZSZP2BP141T.1 size=1048576 2024-09-13T00:00:46.300 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.298+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~WZSZP2BP141T.2 size=161604 2024-09-13T00:00:46.310 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.310+0000 7fb8f2ce0640 1 < completed multipart upload key=NQJQC8CEAC15 upload=WZSZP2BP141T tag=4GYAXMQSAWTC type=write size=3307332 2024-09-13T00:00:46.320 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.318+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~BZUXTA9LS4Z9.0 size=1048576 2024-09-13T00:00:46.325 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.322+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~BZUXTA9LS4Z9.1 size=349920 2024-09-13T00:00:46.325 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.322+0000 7fb8f2ce0640 1 > prepared multipart upload key=FX0NCGLK0KPJ upload=BZUXTA9LS4Z9 tag=TPED34229FID type=write size=2447072 2024-09-13T00:00:46.332 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.330+0000 7fb8f2ce0640 -1 < failed to complete operation key=H5EIP0E9CBWX tag=9L2W8MKMIBHA type=del size=2215991: (22) Invalid argument 2024-09-13T00:00:46.343 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.342+0000 7fb8f2ce0640 1 > prepared part key=_multipart_N3Z5XIG3TZRQ.2~DMA1V1HPK239.0 size=951120 2024-09-13T00:00:46.343 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.342+0000 7fb8f2ce0640 1 > prepared multipart upload key=N3Z5XIG3TZRQ upload=DMA1V1HPK239 tag=M98RZ2P87R1O type=write size=1999696 2024-09-13T00:00:46.348 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.346+0000 7fb8f2ce0640 1 < completed delete operation key=P8OYS5ZE1NX0 tag=IO4P9TIL30DV type=del 2024-09-13T00:00:46.358 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.358+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2TWD9YLMSYJI.2~3MRR4I6QQBCV.0 size=1048576 2024-09-13T00:00:46.362 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.362+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2TWD9YLMSYJI.2~3MRR4I6QQBCV.1 size=937820 2024-09-13T00:00:46.362 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.362+0000 7fb8f2ce0640 1 > prepared multipart upload key=2TWD9YLMSYJI upload=3MRR4I6QQBCV tag=8FWAN53Y59PU type=write size=3034972 2024-09-13T00:00:46.370 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.370+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~PFD0IYL1IKXS.0 size=1048576 2024-09-13T00:00:46.374 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.374+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~PFD0IYL1IKXS.1 size=166804 2024-09-13T00:00:46.384 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.382+0000 7fb8f2ce0640 1 < completed multipart upload key=8MPUVD37EDN9 upload=PFD0IYL1IKXS tag=BVSE4WN1TA2G type=write size=2263956 2024-09-13T00:00:46.389 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.390+0000 7fb8f2ce0640 1 > prepared operation key=NQJQC8CEAC15 tag=D4WIQ9M4SJBN type=del size=2203690 2024-09-13T00:00:46.400 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.398+0000 7fb8f2ce0640 1 < suggested update operation key=H5EIP0E9CBWX tag=NSZ225ED81TH type=del size=504329 2024-09-13T00:00:46.406 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.406+0000 7fb8f2ce0640 1 > prepared operation key=63FJDWBO2T9S tag=JLEGWR4BI4LE type=write size=647010 2024-09-13T00:00:46.412 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.410+0000 7fb8f2ce0640 1 < completed write operation key=NQJQC8CEAC15 tag=E3GKS6Q9WQ45 type=write size=442568 2024-09-13T00:00:46.424 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.422+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~ZDXSLP4S9JRY.0 size=747008 2024-09-13T00:00:46.424 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.422+0000 7fb8f2ce0640 1 > prepared multipart upload key=1HX06CCZJ1DD upload=ZDXSLP4S9JRY tag=MA467KSIV00C type=write size=1795584 2024-09-13T00:00:46.430 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.430+0000 7fb8f2ce0640 1 < canceled operation key=PVYZQLGYT3XU tag=D5F4RMYPQ2OB type=del size=110847 2024-09-13T00:00:46.440 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.438+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~BK4BV78MLYOY.0 size=134652 2024-09-13T00:00:46.440 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.438+0000 7fb8f2ce0640 1 > prepared multipart upload key=7OX5OJZ8GXU5 upload=BK4BV78MLYOY tag=OIVG95SUJ3UR type=write size=1183228 2024-09-13T00:00:46.445 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.446+0000 7fb8f2ce0640 1 < canceled operation key=F0QCVR9E08WE tag=SHULG9KGY4JX type=del size=1514216 2024-09-13T00:00:46.452 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.450+0000 7fb8f2ce0640 1 > prepared operation key=EJN6HR8I7XQF tag=W577DY1UVQRY type=del size=1261114 2024-09-13T00:00:46.457 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.458+0000 7fb8f2ce0640 1 < completed delete operation key=NQJQC8CEAC15 tag=Y846MTOVDPDR type=del 2024-09-13T00:00:46.469 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.470+0000 7fb8f2ce0640 1 > prepared part key=_multipart_85YEW8N43FTA.2~Z03NXS2ECRUB.0 size=1009872 2024-09-13T00:00:46.469 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.470+0000 7fb8f2ce0640 1 > prepared multipart upload key=85YEW8N43FTA upload=Z03NXS2ECRUB tag=79FVDUPQGV5W type=write size=2058448 2024-09-13T00:00:46.480 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.478+0000 7fb8f2ce0640 1 < suggested remove operation key=2AHYRZUDFHP4 tag=R2VAYF68OBKA type=del 2024-09-13T00:00:46.491 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.490+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~ANVWY6FAUAC9.0 size=1048576 2024-09-13T00:00:46.496 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.494+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~ANVWY6FAUAC9.1 size=1048576 2024-09-13T00:00:46.501 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.502+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~ANVWY6FAUAC9.2 size=548899 2024-09-13T00:00:46.501 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.502+0000 7fb8f2ce0640 1 > prepared multipart upload key=NQJQC8CEAC15 upload=ANVWY6FAUAC9 tag=PHV48KLZR1W2 type=write size=3694627 2024-09-13T00:00:46.507 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.506+0000 7fb8f2ce0640 1 < completed delete operation key=FX0NCGLK0KPJ tag=V06A62Z8AHZJ type=del 2024-09-13T00:00:46.518 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.518+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~FVK0GUZQQ9IH.0 size=1048576 2024-09-13T00:00:46.522 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.522+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~FVK0GUZQQ9IH.1 size=1048576 2024-09-13T00:00:46.527 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.526+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~FVK0GUZQQ9IH.2 size=46115 2024-09-13T00:00:46.527 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.526+0000 7fb8f2ce0640 1 > prepared multipart upload key=7OX5OJZ8GXU5 upload=FVK0GUZQQ9IH tag=MQ9BU21BON72 type=write size=3191843 2024-09-13T00:00:46.533 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.534+0000 7fb8f2ce0640 1 < canceled operation key=1HX06CCZJ1DD tag=ZE3GGIFBTN03 type=del size=3766054 2024-09-13T00:00:46.539 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.538+0000 7fb8f2ce0640 1 > prepared operation key=ZFHN2SNTZ5W7 tag=S8EYDN1LULHF type=del size=3159494 2024-09-13T00:00:46.545 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.546+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZFHN2SNTZ5W7.2~TAL8Q3FA125E.0 size=1048576 2024-09-13T00:00:46.550 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.550+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZFHN2SNTZ5W7.2~TAL8Q3FA125E.1 size=1048576 2024-09-13T00:00:46.555 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.554+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZFHN2SNTZ5W7.2~TAL8Q3FA125E.2 size=871580 2024-09-13T00:00:46.564 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.562+0000 7fb8f2ce0640 1 < completed multipart upload key=ZFHN2SNTZ5W7 upload=TAL8Q3FA125E tag=4AUMXMDO1NST type=write size=4017308 2024-09-13T00:00:46.576 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.574+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~10AK9BARQW0P.0 size=1048576 2024-09-13T00:00:46.581 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.582+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~10AK9BARQW0P.1 size=147892 2024-09-13T00:00:46.581 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.582+0000 7fb8f2ce0640 1 > prepared multipart upload key=FX0NCGLK0KPJ upload=10AK9BARQW0P tag=SBSNHT0S9FE2 type=write size=2245044 2024-09-13T00:00:46.587 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.586+0000 7fb8f2ce0640 1 < completed part key=_multipart_2TWD9YLMSYJI.2~VXEOHWRP3MS1.0 size=1048576 2024-09-13T00:00:46.592 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.590+0000 7fb8f2ce0640 1 < completed part key=_multipart_2TWD9YLMSYJI.2~VXEOHWRP3MS1.1 size=1048576 2024-09-13T00:00:46.592 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.590+0000 7fb8f2ce0640 1 < canceled part key=_multipart_2TWD9YLMSYJI.2~VXEOHWRP3MS1.2 size=223909 2024-09-13T00:00:46.603 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.602+0000 7fb8f2ce0640 1 < canceled multipart upload key=2TWD9YLMSYJI upload=VXEOHWRP3MS1 tag=63Z0IT3ZQDZ7 type=write size=3369637 2024-09-13T00:00:46.609 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.610+0000 7fb8f2ce0640 1 > prepared operation key=2TWD9YLMSYJI tag=T8AJA5PSO1ZA type=del size=4054390 2024-09-13T00:00:46.615 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.614+0000 7fb8f2ce0640 1 < completed write operation key=H4M1TCN5W6XU tag=Q4KV4OPW37IL type=write size=689947 2024-09-13T00:00:46.621 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.622+0000 7fb8f2ce0640 1 > prepared operation key=7OX5OJZ8GXU5 tag=D6JTRIKZ7LUA type=del size=2530803 2024-09-13T00:00:46.627 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.626+0000 7fb8f2ce0640 1 < completed part key=_multipart_W8XZ26311RJX.2~8A9IN1R18UAD.0 size=1048576 2024-09-13T00:00:46.632 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.630+0000 7fb8f2ce0640 1 < completed part key=_multipart_W8XZ26311RJX.2~8A9IN1R18UAD.1 size=1048576 2024-09-13T00:00:46.637 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.634+0000 7fb8f2ce0640 1 < completed part key=_multipart_W8XZ26311RJX.2~8A9IN1R18UAD.2 size=744581 2024-09-13T00:00:46.646 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.646+0000 7fb8f2ce0640 1 < completed multipart upload key=W8XZ26311RJX upload=8A9IN1R18UAD tag=WVSFDUJ3HKNL type=write size=3890309 2024-09-13T00:00:46.653 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.650+0000 7fb8f2ce0640 1 > prepared operation key=44EDYFT9VF8Y tag=F1DJ3N1Y9UD0 type=del size=1616083 2024-09-13T00:00:46.659 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.658+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~BZUXTA9LS4Z9.0 size=1048576 2024-09-13T00:00:46.664 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.662+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~BZUXTA9LS4Z9.1 size=349920 2024-09-13T00:00:46.674 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.674+0000 7fb8f2ce0640 1 < completed multipart upload key=FX0NCGLK0KPJ upload=BZUXTA9LS4Z9 tag=TPED34229FID type=write size=2447072 2024-09-13T00:00:46.684 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.682+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZFHN2SNTZ5W7.2~8W2P3GGXMLBP.0 size=452777 2024-09-13T00:00:46.684 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.682+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZFHN2SNTZ5W7 upload=8W2P3GGXMLBP tag=0PX2SADFKS30 type=write size=1501353 2024-09-13T00:00:46.689 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.690+0000 7fb8f2ce0640 1 < completed part key=_multipart_N3Z5XIG3TZRQ.2~DMA1V1HPK239.0 size=951120 2024-09-13T00:00:46.699 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.698+0000 7fb8f2ce0640 1 < completed multipart upload key=N3Z5XIG3TZRQ upload=DMA1V1HPK239 tag=M98RZ2P87R1O type=write size=1999696 2024-09-13T00:00:46.710 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.710+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NX26SI4F2ZMB.2~9WAD1T1XTZ66.0 size=1048576 2024-09-13T00:00:46.715 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.714+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NX26SI4F2ZMB.2~9WAD1T1XTZ66.1 size=1048576 2024-09-13T00:00:46.719 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.718+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NX26SI4F2ZMB.2~9WAD1T1XTZ66.2 size=484474 2024-09-13T00:00:46.720 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.718+0000 7fb8f2ce0640 1 > prepared multipart upload key=NX26SI4F2ZMB upload=9WAD1T1XTZ66 tag=8DC7ZDMBHEI2 type=write size=3630202 2024-09-13T00:00:46.725 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.726+0000 7fb8f2ce0640 1 < completed part key=_multipart_2TWD9YLMSYJI.2~3MRR4I6QQBCV.0 size=1048576 2024-09-13T00:00:46.730 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.730+0000 7fb8f2ce0640 1 < completed part key=_multipart_2TWD9YLMSYJI.2~3MRR4I6QQBCV.1 size=937820 2024-09-13T00:00:46.739 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.738+0000 7fb8f2ce0640 1 < completed multipart upload key=2TWD9YLMSYJI upload=3MRR4I6QQBCV tag=8FWAN53Y59PU type=write size=3034972 2024-09-13T00:00:46.744 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.742+0000 7fb8f2ce0640 1 > prepared operation key=4K76U69GZ9AE tag=4P0DHEIWBLWU type=del size=2922138 2024-09-13T00:00:46.750 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.750+0000 7fb8f2ce0640 1 < completed delete operation key=NQJQC8CEAC15 tag=D4WIQ9M4SJBN type=del 2024-09-13T00:00:46.761 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.762+0000 7fb8f2ce0640 1 > prepared part key=_multipart_J8NGDSF9XRTP.2~YJBTQ2ULZMFS.0 size=222621 2024-09-13T00:00:46.761 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.762+0000 7fb8f2ce0640 1 > prepared multipart upload key=J8NGDSF9XRTP upload=YJBTQ2ULZMFS tag=3YU26EC1IJOR type=write size=1271197 2024-09-13T00:00:46.770 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.770+0000 7fb8f2ce0640 1 < completed write operation key=63FJDWBO2T9S tag=JLEGWR4BI4LE type=write size=647010 2024-09-13T00:00:46.776 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.774+0000 7fb8f2ce0640 1 > prepared operation key=H4M1TCN5W6XU tag=XZ9VY20UW3MA type=del size=3826506 2024-09-13T00:00:46.781 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.782+0000 7fb8f2ce0640 1 < completed part key=_multipart_1HX06CCZJ1DD.2~ZDXSLP4S9JRY.0 size=747008 2024-09-13T00:00:46.791 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.790+0000 7fb8f2ce0640 1 < completed multipart upload key=1HX06CCZJ1DD upload=ZDXSLP4S9JRY tag=MA467KSIV00C type=write size=1795584 2024-09-13T00:00:46.797 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.794+0000 7fb8f2ce0640 1 > prepared operation key=8MPUVD37EDN9 tag=U0V2574SY9P0 type=del size=3002512 2024-09-13T00:00:46.804 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.802+0000 7fb8f2ce0640 1 < completed part key=_multipart_7OX5OJZ8GXU5.2~BK4BV78MLYOY.0 size=134652 2024-09-13T00:00:46.812 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.810+0000 7fb8f2ce0640 1 < completed multipart upload key=7OX5OJZ8GXU5 upload=BK4BV78MLYOY tag=OIVG95SUJ3UR type=write size=1183228 2024-09-13T00:00:46.823 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.822+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~R1REQJC9B23D.0 size=1048576 2024-09-13T00:00:46.827 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.826+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~R1REQJC9B23D.1 size=45999 2024-09-13T00:00:46.827 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.826+0000 7fb8f2ce0640 1 > prepared multipart upload key=8Z6U7OOA0GBH upload=R1REQJC9B23D tag=WX6V1O62WSB5 type=write size=2143151 2024-09-13T00:00:46.833 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.834+0000 7fb8f2ce0640 1 < canceled operation key=EJN6HR8I7XQF tag=W577DY1UVQRY type=del size=1261114 2024-09-13T00:00:46.839 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.838+0000 7fb8f2ce0640 1 > prepared operation key=85YEW8N43FTA tag=QVLT5UUECQGA type=write size=1033563 2024-09-13T00:00:46.845 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.846+0000 7fb8f2ce0640 1 < completed part key=_multipart_85YEW8N43FTA.2~Z03NXS2ECRUB.0 size=1009872 2024-09-13T00:00:46.854 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.854+0000 7fb8f2ce0640 1 < completed multipart upload key=85YEW8N43FTA upload=Z03NXS2ECRUB tag=79FVDUPQGV5W type=write size=2058448 2024-09-13T00:00:46.864 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.862+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZFHN2SNTZ5W7.2~7UPOCKPRAA4M.0 size=1048576 2024-09-13T00:00:46.869 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.870+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZFHN2SNTZ5W7.2~7UPOCKPRAA4M.1 size=1048576 2024-09-13T00:00:46.873 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.874+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZFHN2SNTZ5W7.2~7UPOCKPRAA4M.2 size=659211 2024-09-13T00:00:46.874 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.874+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZFHN2SNTZ5W7 upload=7UPOCKPRAA4M tag=T73LQEK50GZ7 type=write size=3804939 2024-09-13T00:00:46.879 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.878+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~ANVWY6FAUAC9.0 size=1048576 2024-09-13T00:00:46.879 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.878+0000 7fb8f2ce0640 1 < canceled part key=_multipart_NQJQC8CEAC15.2~ANVWY6FAUAC9.1 size=1048576 2024-09-13T00:00:46.883 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.882+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~ANVWY6FAUAC9.2 size=548899 2024-09-13T00:00:46.893 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.894+0000 7fb8f2ce0640 1 < canceled multipart upload key=NQJQC8CEAC15 upload=ANVWY6FAUAC9 tag=PHV48KLZR1W2 type=write size=3694627 2024-09-13T00:00:46.900 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.898+0000 7fb8f2ce0640 1 > prepared operation key=FX0NCGLK0KPJ tag=ZO24L3PQJOIK type=del size=3907191 2024-09-13T00:00:46.907 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.906+0000 7fb8f2ce0640 1 < completed part key=_multipart_7OX5OJZ8GXU5.2~FVK0GUZQQ9IH.0 size=1048576 2024-09-13T00:00:46.912 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.910+0000 7fb8f2ce0640 1 < completed part key=_multipart_7OX5OJZ8GXU5.2~FVK0GUZQQ9IH.1 size=1048576 2024-09-13T00:00:46.916 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.914+0000 7fb8f2ce0640 1 < completed part key=_multipart_7OX5OJZ8GXU5.2~FVK0GUZQQ9IH.2 size=46115 2024-09-13T00:00:46.925 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.926+0000 7fb8f2ce0640 1 < completed multipart upload key=7OX5OJZ8GXU5 upload=FVK0GUZQQ9IH tag=MQ9BU21BON72 type=write size=3191843 2024-09-13T00:00:46.931 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.930+0000 7fb8f2ce0640 1 > prepared operation key=PVYZQLGYT3XU tag=VWBO05R9B274 type=write size=210146 2024-09-13T00:00:46.936 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.934+0000 7fb8f2ce0640 1 < completed delete operation key=ZFHN2SNTZ5W7 tag=S8EYDN1LULHF type=del 2024-09-13T00:00:46.941 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.942+0000 7fb8f2ce0640 1 > prepared operation key=2AHYRZUDFHP4 tag=9LRIDTKW3OZ5 type=del size=478564 2024-09-13T00:00:46.946 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.946+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~10AK9BARQW0P.0 size=1048576 2024-09-13T00:00:46.951 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.950+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~10AK9BARQW0P.1 size=147892 2024-09-13T00:00:46.960 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.958+0000 7fb8f2ce0640 1 < completed multipart upload key=FX0NCGLK0KPJ upload=10AK9BARQW0P tag=SBSNHT0S9FE2 type=write size=2245044 2024-09-13T00:00:46.966 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.966+0000 7fb8f2ce0640 1 > prepared operation key=7OX5OJZ8GXU5 tag=7CL5J6JKLYY5 type=del size=1914165 2024-09-13T00:00:46.976 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.974+0000 7fb8f2ce0640 1 < suggested remove operation key=2TWD9YLMSYJI tag=T8AJA5PSO1ZA type=del 2024-09-13T00:00:46.982 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.982+0000 7fb8f2ce0640 1 > prepared operation key=4K76U69GZ9AE tag=2VBOFM9K2JQQ type=del size=2188321 2024-09-13T00:00:46.987 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.986+0000 7fb8f2ce0640 1 < completed delete operation key=7OX5OJZ8GXU5 tag=D6JTRIKZ7LUA type=del 2024-09-13T00:00:46.993 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.990+0000 7fb8f2ce0640 1 > prepared operation key=EJN6HR8I7XQF tag=9NIIH8XXXQD8 type=write size=396424 2024-09-13T00:00:46.997 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:46.998+0000 7fb8f2ce0640 1 < completed delete operation key=44EDYFT9VF8Y tag=F1DJ3N1Y9UD0 type=del 2024-09-13T00:00:47.003 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.002+0000 7fb8f2ce0640 1 > prepared operation key=EUE3LT0PDIXV tag=3KKLEXOOSME1 type=write size=661980 2024-09-13T00:00:47.009 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.006+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZFHN2SNTZ5W7.2~8W2P3GGXMLBP.0 size=452777 2024-09-13T00:00:47.017 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.018+0000 7fb8f2ce0640 1 < completed multipart upload key=ZFHN2SNTZ5W7 upload=8W2P3GGXMLBP tag=0PX2SADFKS30 type=write size=1501353 2024-09-13T00:00:47.027 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.026+0000 7fb8f2ce0640 1 > prepared part key=_multipart_85YEW8N43FTA.2~IC5ENS9FWJC6.0 size=1048576 2024-09-13T00:00:47.032 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.030+0000 7fb8f2ce0640 1 > prepared part key=_multipart_85YEW8N43FTA.2~IC5ENS9FWJC6.1 size=1048576 2024-09-13T00:00:47.036 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.034+0000 7fb8f2ce0640 1 > prepared part key=_multipart_85YEW8N43FTA.2~IC5ENS9FWJC6.2 size=374454 2024-09-13T00:00:47.036 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.034+0000 7fb8f2ce0640 1 > prepared multipart upload key=85YEW8N43FTA upload=IC5ENS9FWJC6 tag=UQKQIL82AD56 type=write size=3520182 2024-09-13T00:00:47.042 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.042+0000 7fb8f2ce0640 1 < completed part key=_multipart_NX26SI4F2ZMB.2~9WAD1T1XTZ66.0 size=1048576 2024-09-13T00:00:47.046 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.046+0000 7fb8f2ce0640 1 < completed part key=_multipart_NX26SI4F2ZMB.2~9WAD1T1XTZ66.1 size=1048576 2024-09-13T00:00:47.050 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.050+0000 7fb8f2ce0640 1 < completed part key=_multipart_NX26SI4F2ZMB.2~9WAD1T1XTZ66.2 size=484474 2024-09-13T00:00:47.059 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.058+0000 7fb8f2ce0640 1 < completed multipart upload key=NX26SI4F2ZMB upload=9WAD1T1XTZ66 tag=8DC7ZDMBHEI2 type=write size=3630202 2024-09-13T00:00:47.065 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.062+0000 7fb8f2ce0640 1 > prepared operation key=EUE3LT0PDIXV tag=8FQRTIXKO3PY type=del size=2908753 2024-09-13T00:00:47.070 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.070+0000 7fb8f2ce0640 1 < completed delete operation key=4K76U69GZ9AE tag=4P0DHEIWBLWU type=del 2024-09-13T00:00:47.075 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.074+0000 7fb8f2ce0640 1 > prepared operation key=H9AU4HD247YU tag=DVU7LXHT602T type=write size=984487 2024-09-13T00:00:47.081 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.082+0000 7fb8f2ce0640 1 < completed part key=_multipart_J8NGDSF9XRTP.2~YJBTQ2ULZMFS.0 size=222621 2024-09-13T00:00:47.091 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.090+0000 7fb8f2ce0640 1 < completed multipart upload key=J8NGDSF9XRTP upload=YJBTQ2ULZMFS tag=3YU26EC1IJOR type=write size=1271197 2024-09-13T00:00:47.097 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.094+0000 7fb8f2ce0640 1 > prepared operation key=2TWD9YLMSYJI tag=QDU05HYSL6AJ type=del size=2830814 2024-09-13T00:00:47.108 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.106+0000 7fb8f2ce0640 1 < suggested remove operation key=H4M1TCN5W6XU tag=XZ9VY20UW3MA type=del 2024-09-13T00:00:47.117 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.118+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~XCB98AG4VZC7.0 size=1048576 2024-09-13T00:00:47.122 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.122+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~XCB98AG4VZC7.1 size=948828 2024-09-13T00:00:47.122 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.122+0000 7fb8f2ce0640 1 > prepared multipart upload key=8MPUVD37EDN9 upload=XCB98AG4VZC7 tag=2L9U1M3ZCYXL type=write size=3045980 2024-09-13T00:00:47.157 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.158+0000 7fb8f2ce0640 1 < completed delete operation key=8MPUVD37EDN9 tag=U0V2574SY9P0 type=del 2024-09-13T00:00:47.162 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.162+0000 7fb8f2ce0640 1 > prepared operation key=QMZREZ1QEDNJ tag=RWFZIQKPE112 type=del size=3598323 2024-09-13T00:00:47.168 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.166+0000 7fb8f2ce0640 1 < completed part key=_multipart_8Z6U7OOA0GBH.2~R1REQJC9B23D.0 size=1048576 2024-09-13T00:00:47.175 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.174+0000 7fb8f2ce0640 1 < completed part key=_multipart_8Z6U7OOA0GBH.2~R1REQJC9B23D.1 size=45999 2024-09-13T00:00:47.186 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.186+0000 7fb8f2ce0640 1 < completed multipart upload key=8Z6U7OOA0GBH upload=R1REQJC9B23D tag=WX6V1O62WSB5 type=write size=2143151 2024-09-13T00:00:47.193 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.194+0000 7fb8f2ce0640 1 > prepared operation key=FX0NCGLK0KPJ tag=H41PD1UFPNBC type=del size=2426983 2024-09-13T00:00:47.204 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.202+0000 7fb8f2ce0640 1 < suggested update operation key=85YEW8N43FTA tag=QVLT5UUECQGA type=write size=1033563 2024-09-13T00:00:47.216 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.214+0000 7fb8f2ce0640 1 > prepared part key=_multipart_0WC6P8YNL0H2.2~6TZZ0V46WL2X.0 size=237717 2024-09-13T00:00:47.216 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.214+0000 7fb8f2ce0640 1 > prepared multipart upload key=0WC6P8YNL0H2 upload=6TZZ0V46WL2X tag=B2W206B09NPR type=write size=1286293 2024-09-13T00:00:47.221 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.218+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZFHN2SNTZ5W7.2~7UPOCKPRAA4M.0 size=1048576 2024-09-13T00:00:47.225 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.226+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZFHN2SNTZ5W7.2~7UPOCKPRAA4M.1 size=1048576 2024-09-13T00:00:47.229 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.230+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZFHN2SNTZ5W7.2~7UPOCKPRAA4M.2 size=659211 2024-09-13T00:00:47.239 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.238+0000 7fb8f2ce0640 1 < completed multipart upload key=ZFHN2SNTZ5W7 upload=7UPOCKPRAA4M tag=T73LQEK50GZ7 type=write size=3804939 2024-09-13T00:00:47.245 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.242+0000 7fb8f2ce0640 1 > prepared operation key=1HX06CCZJ1DD tag=ZDE2CW8Z5QTW type=del size=61981 2024-09-13T00:00:47.249 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.250+0000 7fb8f2ce0640 1 < completed delete operation key=FX0NCGLK0KPJ tag=ZO24L3PQJOIK type=del 2024-09-13T00:00:47.255 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.254+0000 7fb8f2ce0640 1 > prepared operation key=63FJDWBO2T9S tag=30K4W1CQFYW1 type=del size=823550 2024-09-13T00:00:47.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:46 smithi136 bash[75164]: cluster 2024-09-13T00:00:45.131930+0000 mgr.y (mgr.24661) 709 : cluster [DBG] pgmap v1385: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 607 KiB/s rd, 209 KiB/s wr, 1.07k op/s 2024-09-13T00:00:47.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:46 smithi136 bash[76371]: cluster 2024-09-13T00:00:45.131930+0000 mgr.y (mgr.24661) 709 : cluster [DBG] pgmap v1385: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 607 KiB/s rd, 209 KiB/s wr, 1.07k op/s 2024-09-13T00:00:47.261 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.262+0000 7fb8f2ce0640 1 < completed write operation key=PVYZQLGYT3XU tag=VWBO05R9B274 type=write size=210146 2024-09-13T00:00:47.267 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.266+0000 7fb8f2ce0640 1 > prepared operation key=70F7D20HI504 tag=G9MQ4WRUC6AE type=write size=287672 2024-09-13T00:00:47.272 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.270+0000 7fb8f2ce0640 1 < completed delete operation key=2AHYRZUDFHP4 tag=9LRIDTKW3OZ5 type=del 2024-09-13T00:00:47.277 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.278+0000 7fb8f2ce0640 1 > prepared operation key=N3Z5XIG3TZRQ tag=QCR50NDW0L83 type=del size=1494692 2024-09-13T00:00:47.283 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.282+0000 7fb8f2ce0640 1 < completed delete operation key=7OX5OJZ8GXU5 tag=7CL5J6JKLYY5 type=del 2024-09-13T00:00:47.294 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.294+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~K5RKNCAN18IT.0 size=1027010 2024-09-13T00:00:47.294 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.294+0000 7fb8f2ce0640 1 > prepared multipart upload key=8Z6U7OOA0GBH upload=K5RKNCAN18IT tag=ORY91ODX1H01 type=write size=2075586 2024-09-13T00:00:47.300 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.298+0000 7fb8f2ce0640 1 < completed delete operation key=4K76U69GZ9AE tag=2VBOFM9K2JQQ type=del 2024-09-13T00:00:47.310 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.310+0000 7fb8f2ce0640 1 > prepared part key=_multipart_5GM88ERM89KT.2~PMN4C0PQ6FI6.0 size=1048576 2024-09-13T00:00:47.315 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.314+0000 7fb8f2ce0640 1 > prepared part key=_multipart_5GM88ERM89KT.2~PMN4C0PQ6FI6.1 size=125051 2024-09-13T00:00:47.316 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.314+0000 7fb8f2ce0640 1 > prepared multipart upload key=5GM88ERM89KT upload=PMN4C0PQ6FI6 tag=TK3AA6KTR01U type=write size=2222203 2024-09-13T00:00:47.327 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.326+0000 7fb8f2ce0640 1 < suggested update operation key=EJN6HR8I7XQF tag=9NIIH8XXXQD8 type=write size=396424 2024-09-13T00:00:47.336 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.334+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NX26SI4F2ZMB.2~ZQEW40WZVOR0.0 size=1048576 2024-09-13T00:00:47.341 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.338+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NX26SI4F2ZMB.2~ZQEW40WZVOR0.1 size=1048576 2024-09-13T00:00:47.345 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.346+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NX26SI4F2ZMB.2~ZQEW40WZVOR0.2 size=33461 2024-09-13T00:00:47.345 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.346+0000 7fb8f2ce0640 1 > prepared multipart upload key=NX26SI4F2ZMB upload=ZQEW40WZVOR0 tag=SULH8P98HV9G type=write size=3179189 2024-09-13T00:00:47.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:46 smithi181 bash[67587]: cluster 2024-09-13T00:00:45.131930+0000 mgr.y (mgr.24661) 709 : cluster [DBG] pgmap v1385: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 607 KiB/s rd, 209 KiB/s wr, 1.07k op/s 2024-09-13T00:00:47.350 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.350+0000 7fb8f2ce0640 1 < completed write operation key=EUE3LT0PDIXV tag=3KKLEXOOSME1 type=write size=661980 2024-09-13T00:00:47.356 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.354+0000 7fb8f2ce0640 1 > prepared operation key=H9AU4HD247YU tag=AB4F6Q8MBJWW type=del size=2833314 2024-09-13T00:00:47.361 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.362+0000 7fb8f2ce0640 1 < completed part key=_multipart_85YEW8N43FTA.2~IC5ENS9FWJC6.0 size=1048576 2024-09-13T00:00:47.365 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.366+0000 7fb8f2ce0640 1 < completed part key=_multipart_85YEW8N43FTA.2~IC5ENS9FWJC6.1 size=1048576 2024-09-13T00:00:47.371 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.370+0000 7fb8f2ce0640 1 < completed part key=_multipart_85YEW8N43FTA.2~IC5ENS9FWJC6.2 size=374454 2024-09-13T00:00:47.380 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.378+0000 7fb8f2ce0640 1 < completed multipart upload key=85YEW8N43FTA upload=IC5ENS9FWJC6 tag=UQKQIL82AD56 type=write size=3520182 2024-09-13T00:00:47.385 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.386+0000 7fb8f2ce0640 1 > prepared operation key=2AHYRZUDFHP4 tag=3EUV4NETDVE2 type=write size=995023 2024-09-13T00:00:47.390 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.390+0000 7fb8f2ce0640 1 < completed delete operation key=EUE3LT0PDIXV tag=8FQRTIXKO3PY type=del 2024-09-13T00:00:47.400 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.398+0000 7fb8f2ce0640 1 > prepared part key=_multipart_J8NGDSF9XRTP.2~RTUEJI1ODO2B.0 size=1048576 2024-09-13T00:00:47.404 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.402+0000 7fb8f2ce0640 1 > prepared part key=_multipart_J8NGDSF9XRTP.2~RTUEJI1ODO2B.1 size=1048576 2024-09-13T00:00:47.410 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.410+0000 7fb8f2ce0640 1 > prepared part key=_multipart_J8NGDSF9XRTP.2~RTUEJI1ODO2B.2 size=36347 2024-09-13T00:00:47.410 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.410+0000 7fb8f2ce0640 1 > prepared multipart upload key=J8NGDSF9XRTP upload=RTUEJI1ODO2B tag=JF4RNIG9L0BR type=write size=3182075 2024-09-13T00:00:47.416 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.414+0000 7fb8f2ce0640 1 < completed write operation key=H9AU4HD247YU tag=DVU7LXHT602T type=write size=984487 2024-09-13T00:00:47.425 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.426+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZFHN2SNTZ5W7.2~XICNJ4MJR0AC.0 size=820889 2024-09-13T00:00:47.425 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.426+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZFHN2SNTZ5W7 upload=XICNJ4MJR0AC tag=G6HXPE99VFWF type=write size=1869465 2024-09-13T00:00:47.431 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.430+0000 7fb8f2ce0640 1 < completed delete operation key=2TWD9YLMSYJI tag=QDU05HYSL6AJ type=del 2024-09-13T00:00:47.437 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.438+0000 7fb8f2ce0640 1 > prepared operation key=5GM88ERM89KT tag=Y44UF4B9KSGV type=del size=3659505 2024-09-13T00:00:47.442 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.442+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~XCB98AG4VZC7.0 size=1048576 2024-09-13T00:00:47.446 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.446+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~XCB98AG4VZC7.1 size=948828 2024-09-13T00:00:47.455 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.454+0000 7fb8f2ce0640 1 < completed multipart upload key=8MPUVD37EDN9 upload=XCB98AG4VZC7 tag=2L9U1M3ZCYXL type=write size=3045980 2024-09-13T00:00:47.460 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.458+0000 7fb8f2ce0640 1 > prepared operation key=1PX7DBO9M3YI tag=V4S2VHEIYVQB type=del size=1775256 2024-09-13T00:00:47.465 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.466+0000 7fb8f2ce0640 1 < completed delete operation key=QMZREZ1QEDNJ tag=RWFZIQKPE112 type=del 2024-09-13T00:00:47.475 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.474+0000 7fb8f2ce0640 1 > prepared part key=_multipart_P8OYS5ZE1NX0.2~NCIFFX18YLJP.0 size=1048576 2024-09-13T00:00:47.479 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.478+0000 7fb8f2ce0640 1 > prepared part key=_multipart_P8OYS5ZE1NX0.2~NCIFFX18YLJP.1 size=1048576 2024-09-13T00:00:47.485 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.482+0000 7fb8f2ce0640 1 > prepared part key=_multipart_P8OYS5ZE1NX0.2~NCIFFX18YLJP.2 size=130384 2024-09-13T00:00:47.488 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.482+0000 7fb8f2ce0640 1 > prepared multipart upload key=P8OYS5ZE1NX0 upload=NCIFFX18YLJP tag=BVKFQPKAHEWD type=write size=3276112 2024-09-13T00:00:47.490 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.490+0000 7fb8f2ce0640 1 < canceled operation key=FX0NCGLK0KPJ tag=H41PD1UFPNBC type=del size=2426983 2024-09-13T00:00:47.501 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.498+0000 7fb8f2ce0640 1 > prepared part key=_multipart_J8NGDSF9XRTP.2~42E5HV21D5HB.0 size=27421 2024-09-13T00:00:47.501 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.498+0000 7fb8f2ce0640 1 > prepared multipart upload key=J8NGDSF9XRTP upload=42E5HV21D5HB tag=2HU8VXT7N4FN type=write size=1075997 2024-09-13T00:00:47.506 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.506+0000 7fb8f2ce0640 1 < completed part key=_multipart_0WC6P8YNL0H2.2~6TZZ0V46WL2X.0 size=237717 2024-09-13T00:00:47.516 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.514+0000 7fb8f2ce0640 1 < completed multipart upload key=0WC6P8YNL0H2 upload=6TZZ0V46WL2X tag=B2W206B09NPR type=write size=1286293 2024-09-13T00:00:47.521 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.522+0000 7fb8f2ce0640 1 > prepared operation key=4K76U69GZ9AE tag=7O10FSCTD80E type=del size=1726006 2024-09-13T00:00:47.526 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.526+0000 7fb8f2ce0640 1 < completed delete operation key=1HX06CCZJ1DD tag=ZDE2CW8Z5QTW type=del 2024-09-13T00:00:47.536 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.534+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~N7O6PA4X6LXI.0 size=1048576 2024-09-13T00:00:47.541 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.538+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~N7O6PA4X6LXI.1 size=785869 2024-09-13T00:00:47.541 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.538+0000 7fb8f2ce0640 1 > prepared multipart upload key=1HX06CCZJ1DD upload=N7O6PA4X6LXI tag=2KUQ708KWK75 type=write size=2883021 2024-09-13T00:00:47.546 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.546+0000 7fb8f2ce0640 1 < completed delete operation key=63FJDWBO2T9S tag=30K4W1CQFYW1 type=del 2024-09-13T00:00:47.551 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.550+0000 7fb8f2ce0640 1 > prepared operation key=QMZREZ1QEDNJ tag=JOHX6WSGY9BD type=write size=143259 2024-09-13T00:00:47.556 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.554+0000 7fb8f2ce0640 1 < completed write operation key=70F7D20HI504 tag=G9MQ4WRUC6AE type=write size=287672 2024-09-13T00:00:47.562 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.562+0000 7fb8f2ce0640 1 > prepared operation key=W8XZ26311RJX tag=694NTBVNRS04 type=del size=1760268 2024-09-13T00:00:47.567 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.566+0000 7fb8f2ce0640 1 < completed delete operation key=N3Z5XIG3TZRQ tag=QCR50NDW0L83 type=del 2024-09-13T00:00:47.573 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.570+0000 7fb8f2ce0640 1 > prepared operation key=ZFHN2SNTZ5W7 tag=554QQTBHRVD1 type=del size=157490 2024-09-13T00:00:47.579 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.578+0000 7fb8f2ce0640 1 < completed part key=_multipart_8Z6U7OOA0GBH.2~K5RKNCAN18IT.0 size=1027010 2024-09-13T00:00:47.589 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.586+0000 7fb8f2ce0640 1 < completed multipart upload key=8Z6U7OOA0GBH upload=K5RKNCAN18IT tag=ORY91ODX1H01 type=write size=2075586 2024-09-13T00:00:47.594 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.594+0000 7fb8f2ce0640 1 > prepared operation key=ZFHN2SNTZ5W7 tag=W2P5RSJUTP8Z type=write size=908684 2024-09-13T00:00:47.595 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.594+0000 7fb8f2ce0640 1 < canceled part key=_multipart_5GM88ERM89KT.2~PMN4C0PQ6FI6.0 size=1048576 2024-09-13T00:00:47.599 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.598+0000 7fb8f2ce0640 1 < completed part key=_multipart_5GM88ERM89KT.2~PMN4C0PQ6FI6.1 size=125051 2024-09-13T00:00:47.608 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.606+0000 7fb8f2ce0640 1 < canceled multipart upload key=5GM88ERM89KT upload=PMN4C0PQ6FI6 tag=TK3AA6KTR01U type=write size=2222203 2024-09-13T00:00:47.614 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.614+0000 7fb8f2ce0640 1 > prepared operation key=J8NGDSF9XRTP tag=NCL0I43TXQDR type=del size=3457627 2024-09-13T00:00:47.619 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.618+0000 7fb8f2ce0640 1 < completed part key=_multipart_NX26SI4F2ZMB.2~ZQEW40WZVOR0.0 size=1048576 2024-09-13T00:00:47.626 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.626+0000 7fb8f2ce0640 1 < completed part key=_multipart_NX26SI4F2ZMB.2~ZQEW40WZVOR0.1 size=1048576 2024-09-13T00:00:47.631 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.630+0000 7fb8f2ce0640 1 < completed part key=_multipart_NX26SI4F2ZMB.2~ZQEW40WZVOR0.2 size=33461 2024-09-13T00:00:47.641 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.642+0000 7fb8f2ce0640 1 < completed multipart upload key=NX26SI4F2ZMB upload=ZQEW40WZVOR0 tag=SULH8P98HV9G type=write size=3179189 2024-09-13T00:00:47.652 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.650+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H4M1TCN5W6XU.2~MPLTQ0VOTKES.0 size=394959 2024-09-13T00:00:47.652 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.650+0000 7fb8f2ce0640 1 > prepared multipart upload key=H4M1TCN5W6XU upload=MPLTQ0VOTKES tag=WZIO4BOXFNJG type=write size=1443535 2024-09-13T00:00:47.657 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.658+0000 7fb8f2ce0640 1 < canceled operation key=H9AU4HD247YU tag=AB4F6Q8MBJWW type=del size=2833314 2024-09-13T00:00:47.664 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.662+0000 7fb8f2ce0640 1 > prepared operation key=EJN6HR8I7XQF tag=9QSK450WSHTH type=write size=582650 2024-09-13T00:00:47.675 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.674+0000 7fb8f2ce0640 1 < suggested update operation key=2AHYRZUDFHP4 tag=3EUV4NETDVE2 type=write size=995023 2024-09-13T00:00:47.680 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.678+0000 7fb8f2ce0640 1 > prepared operation key=O4MZ4UWNXME9 tag=JSPXUE7HSLYD type=del size=704112 2024-09-13T00:00:47.685 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.686+0000 7fb8f2ce0640 1 < completed part key=_multipart_J8NGDSF9XRTP.2~RTUEJI1ODO2B.0 size=1048576 2024-09-13T00:00:47.689 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.690+0000 7fb8f2ce0640 1 < completed part key=_multipart_J8NGDSF9XRTP.2~RTUEJI1ODO2B.1 size=1048576 2024-09-13T00:00:47.693 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.694+0000 7fb8f2ce0640 1 < completed part key=_multipart_J8NGDSF9XRTP.2~RTUEJI1ODO2B.2 size=36347 2024-09-13T00:00:47.703 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.702+0000 7fb8f2ce0640 1 < completed multipart upload key=J8NGDSF9XRTP upload=RTUEJI1ODO2B tag=JF4RNIG9L0BR type=write size=3182075 2024-09-13T00:00:47.708 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.706+0000 7fb8f2ce0640 1 > prepared operation key=ZTK9AYBSLCOZ tag=HRPLOFTF0W8B type=write size=576569 2024-09-13T00:00:47.713 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.714+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZFHN2SNTZ5W7.2~XICNJ4MJR0AC.0 size=820889 2024-09-13T00:00:47.722 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.722+0000 7fb8f2ce0640 1 < completed multipart upload key=ZFHN2SNTZ5W7 upload=XICNJ4MJR0AC tag=G6HXPE99VFWF type=write size=1869465 2024-09-13T00:00:47.727 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.726+0000 7fb8f2ce0640 1 > prepared operation key=H9AU4HD247YU tag=8RVV0KMOPLLW type=del size=2159161 2024-09-13T00:00:47.732 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.730+0000 7fb8f2ce0640 1 < canceled operation key=5GM88ERM89KT tag=Y44UF4B9KSGV type=del size=3659505 2024-09-13T00:00:47.743 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.742+0000 7fb8f2ce0640 1 > prepared part key=_multipart_QMZREZ1QEDNJ.2~EFDADJY3VW7A.0 size=1048576 2024-09-13T00:00:47.747 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.746+0000 7fb8f2ce0640 1 > prepared part key=_multipart_QMZREZ1QEDNJ.2~EFDADJY3VW7A.1 size=926566 2024-09-13T00:00:47.747 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.746+0000 7fb8f2ce0640 1 > prepared multipart upload key=QMZREZ1QEDNJ upload=EFDADJY3VW7A tag=6295WDJQONGC type=write size=3023718 2024-09-13T00:00:47.753 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.754+0000 7fb8f2ce0640 1 < completed delete operation key=1PX7DBO9M3YI tag=V4S2VHEIYVQB type=del 2024-09-13T00:00:47.763 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.762+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~M23HMPVXE3OS.0 size=1048576 2024-09-13T00:00:47.768 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.766+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~M23HMPVXE3OS.1 size=1048576 2024-09-13T00:00:47.772 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.770+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~M23HMPVXE3OS.2 size=781650 2024-09-13T00:00:47.772 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.770+0000 7fb8f2ce0640 1 > prepared multipart upload key=1PX7DBO9M3YI upload=M23HMPVXE3OS tag=H7JOT4EN0QDK type=write size=3927378 2024-09-13T00:00:47.778 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.778+0000 7fb8f2ce0640 1 < completed part key=_multipart_P8OYS5ZE1NX0.2~NCIFFX18YLJP.0 size=1048576 2024-09-13T00:00:47.783 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.782+0000 7fb8f2ce0640 1 < completed part key=_multipart_P8OYS5ZE1NX0.2~NCIFFX18YLJP.1 size=1048576 2024-09-13T00:00:47.783 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.782+0000 7fb8f2ce0640 1 < canceled part key=_multipart_P8OYS5ZE1NX0.2~NCIFFX18YLJP.2 size=130384 2024-09-13T00:00:47.793 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.790+0000 7fb8f2ce0640 1 < canceled multipart upload key=P8OYS5ZE1NX0 upload=NCIFFX18YLJP tag=BVKFQPKAHEWD type=write size=3276112 2024-09-13T00:00:47.803 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.802+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~60SPOPIC1T00.0 size=1048576 2024-09-13T00:00:47.807 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.806+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~60SPOPIC1T00.1 size=1028065 2024-09-13T00:00:47.807 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.806+0000 7fb8f2ce0640 1 > prepared multipart upload key=FX0NCGLK0KPJ upload=60SPOPIC1T00 tag=YJ8A8EIJIWDE type=write size=3125217 2024-09-13T00:00:47.813 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.814+0000 7fb8f2ce0640 1 < completed part key=_multipart_J8NGDSF9XRTP.2~42E5HV21D5HB.0 size=27421 2024-09-13T00:00:47.823 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.822+0000 7fb8f2ce0640 1 < completed multipart upload key=J8NGDSF9XRTP upload=42E5HV21D5HB tag=2HU8VXT7N4FN type=write size=1075997 2024-09-13T00:00:47.828 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.826+0000 7fb8f2ce0640 1 > prepared operation key=EUE3LT0PDIXV tag=UCQCKA7U33TU type=del size=3413814 2024-09-13T00:00:47.834 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.834+0000 7fb8f2ce0640 1 < completed delete operation key=4K76U69GZ9AE tag=7O10FSCTD80E type=del 2024-09-13T00:00:47.839 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.838+0000 7fb8f2ce0640 1 > prepared operation key=F0QCVR9E08WE tag=Y7H4W53IS48Y type=del size=3298834 2024-09-13T00:00:47.845 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.846+0000 7fb8f2ce0640 1 < completed part key=_multipart_1HX06CCZJ1DD.2~N7O6PA4X6LXI.0 size=1048576 2024-09-13T00:00:47.850 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.850+0000 7fb8f2ce0640 1 < completed part key=_multipart_1HX06CCZJ1DD.2~N7O6PA4X6LXI.1 size=785869 2024-09-13T00:00:47.859 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.858+0000 7fb8f2ce0640 1 < completed multipart upload key=1HX06CCZJ1DD upload=N7O6PA4X6LXI tag=2KUQ708KWK75 type=write size=2883021 2024-09-13T00:00:47.869 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.870+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~2NGDNFV3C9Y1.0 size=1048576 2024-09-13T00:00:47.874 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.874+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~2NGDNFV3C9Y1.1 size=1048576 2024-09-13T00:00:47.879 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.878+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~2NGDNFV3C9Y1.2 size=367919 2024-09-13T00:00:47.879 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.878+0000 7fb8f2ce0640 1 > prepared multipart upload key=70F7D20HI504 upload=2NGDNFV3C9Y1 tag=GBIL7OTW24OZ type=write size=3513647 2024-09-13T00:00:47.885 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.882+0000 7fb8f2ce0640 1 < canceled operation key=QMZREZ1QEDNJ tag=JOHX6WSGY9BD type=write size=143259 2024-09-13T00:00:47.890 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.890+0000 7fb8f2ce0640 1 > prepared operation key=2TWD9YLMSYJI tag=8OVONZE932UO type=del size=3257132 2024-09-13T00:00:47.902 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.902+0000 7fb8f2ce0640 1 < suggested update operation key=W8XZ26311RJX tag=694NTBVNRS04 type=del size=1760268 2024-09-13T00:00:47.913 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.914+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~VW5KKGRKH4VW.0 size=361779 2024-09-13T00:00:47.913 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.914+0000 7fb8f2ce0640 1 > prepared multipart upload key=7OX5OJZ8GXU5 upload=VW5KKGRKH4VW tag=H77D7HKGRR8K type=write size=1410355 2024-09-13T00:00:47.919 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.918+0000 7fb8f2ce0640 1 < completed delete operation key=ZFHN2SNTZ5W7 tag=554QQTBHRVD1 type=del 2024-09-13T00:00:47.925 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.926+0000 7fb8f2ce0640 1 > prepared operation key=0WC6P8YNL0H2 tag=3XJCEDCH9A52 type=del size=2406003 2024-09-13T00:00:47.937 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.938+0000 7fb8f2ce0640 1 < suggested remove operation key=ZFHN2SNTZ5W7 tag=W2P5RSJUTP8Z type=write 2024-09-13T00:00:47.943 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.942+0000 7fb8f2ce0640 1 > prepared operation key=0WC6P8YNL0H2 tag=0BFG49BICWOU type=del size=454467 2024-09-13T00:00:47.948 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.946+0000 7fb8f2ce0640 1 < completed delete operation key=J8NGDSF9XRTP tag=NCL0I43TXQDR type=del 2024-09-13T00:00:47.958 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.958+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~Q0MZTI9ZRA5V.0 size=889029 2024-09-13T00:00:47.959 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.958+0000 7fb8f2ce0640 1 > prepared multipart upload key=2AHYRZUDFHP4 upload=Q0MZTI9ZRA5V tag=IZV70DOQ72M4 type=write size=1937605 2024-09-13T00:00:47.960 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.958+0000 7fb8f2ce0640 1 < canceled part key=_multipart_H4M1TCN5W6XU.2~MPLTQ0VOTKES.0 size=394959 2024-09-13T00:00:47.969 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.970+0000 7fb8f2ce0640 1 < canceled multipart upload key=H4M1TCN5W6XU upload=MPLTQ0VOTKES tag=WZIO4BOXFNJG type=write size=1443535 2024-09-13T00:00:47.979 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.978+0000 7fb8f2ce0640 1 > prepared part key=_multipart_O4MZ4UWNXME9.2~D9ZZMDX1XSF2.0 size=1048576 2024-09-13T00:00:47.984 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.982+0000 7fb8f2ce0640 1 > prepared part key=_multipart_O4MZ4UWNXME9.2~D9ZZMDX1XSF2.1 size=731329 2024-09-13T00:00:47.985 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.982+0000 7fb8f2ce0640 1 > prepared multipart upload key=O4MZ4UWNXME9 upload=D9ZZMDX1XSF2 tag=VEX04MXA08TP type=write size=2828481 2024-09-13T00:00:47.990 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.990+0000 7fb8f2ce0640 1 < completed write operation key=EJN6HR8I7XQF tag=9QSK450WSHTH type=write size=582650 2024-09-13T00:00:48.000 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:47.998+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~4Z3BYRVRNFCQ.0 size=1048576 2024-09-13T00:00:48.005 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.006+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~4Z3BYRVRNFCQ.1 size=269688 2024-09-13T00:00:48.005 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.006+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZTK9AYBSLCOZ upload=4Z3BYRVRNFCQ tag=SJOE9J1MW8PR type=write size=2366840 2024-09-13T00:00:48.010 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.010+0000 7fb8f2ce0640 1 < completed delete operation key=O4MZ4UWNXME9 tag=JSPXUE7HSLYD type=del 2024-09-13T00:00:48.016 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.014+0000 7fb8f2ce0640 1 > prepared operation key=QMZREZ1QEDNJ tag=3TBEZC71D79C type=del size=3309069 2024-09-13T00:00:48.021 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.022+0000 7fb8f2ce0640 1 < completed write operation key=ZTK9AYBSLCOZ tag=HRPLOFTF0W8B type=write size=576569 2024-09-13T00:00:48.027 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.026+0000 7fb8f2ce0640 1 > prepared operation key=H4M1TCN5W6XU tag=Y66LLM5YWL5K type=del size=4144116 2024-09-13T00:00:48.032 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.030+0000 7fb8f2ce0640 1 < completed delete operation key=H9AU4HD247YU tag=8RVV0KMOPLLW type=del 2024-09-13T00:00:48.042 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.042+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~3L1RGLXGSVN7.0 size=124853 2024-09-13T00:00:48.042 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.042+0000 7fb8f2ce0640 1 > prepared multipart upload key=70F7D20HI504 upload=3L1RGLXGSVN7 tag=LR83RXFRYL3W type=write size=1173429 2024-09-13T00:00:48.048 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.046+0000 7fb8f2ce0640 1 < completed part key=_multipart_QMZREZ1QEDNJ.2~EFDADJY3VW7A.0 size=1048576 2024-09-13T00:00:48.048 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.046+0000 7fb8f2ce0640 1 < canceled part key=_multipart_QMZREZ1QEDNJ.2~EFDADJY3VW7A.1 size=926566 2024-09-13T00:00:48.058 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.058+0000 7fb8f2ce0640 1 < canceled multipart upload key=QMZREZ1QEDNJ upload=EFDADJY3VW7A tag=6295WDJQONGC type=write size=3023718 2024-09-13T00:00:48.068 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.066+0000 7fb8f2ce0640 1 > prepared part key=_multipart_0WC6P8YNL0H2.2~WOFL9ULW30GH.0 size=1048576 2024-09-13T00:00:48.073 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.074+0000 7fb8f2ce0640 1 > prepared part key=_multipart_0WC6P8YNL0H2.2~WOFL9ULW30GH.1 size=323135 2024-09-13T00:00:48.073 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.074+0000 7fb8f2ce0640 1 > prepared multipart upload key=0WC6P8YNL0H2 upload=WOFL9ULW30GH tag=FBNIONWUDCE6 type=write size=2420287 2024-09-13T00:00:48.080 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.078+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~M23HMPVXE3OS.0 size=1048576 2024-09-13T00:00:48.085 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.086+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~M23HMPVXE3OS.1 size=1048576 2024-09-13T00:00:48.090 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.090+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~M23HMPVXE3OS.2 size=781650 2024-09-13T00:00:48.101 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.102+0000 7fb8f2ce0640 1 < completed multipart upload key=1PX7DBO9M3YI upload=M23HMPVXE3OS tag=H7JOT4EN0QDK type=write size=3927378 2024-09-13T00:00:48.114 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.114+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H5EIP0E9CBWX.2~PENBT3I1LAID.0 size=1048576 2024-09-13T00:00:48.118 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.118+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H5EIP0E9CBWX.2~PENBT3I1LAID.1 size=1048576 2024-09-13T00:00:48.124 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.122+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H5EIP0E9CBWX.2~PENBT3I1LAID.2 size=702070 2024-09-13T00:00:48.124 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.122+0000 7fb8f2ce0640 1 > prepared multipart upload key=H5EIP0E9CBWX upload=PENBT3I1LAID tag=55CW0ASNQ8DO type=write size=3847798 2024-09-13T00:00:48.129 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.130+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~60SPOPIC1T00.0 size=1048576 2024-09-13T00:00:48.134 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.134+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~60SPOPIC1T00.1 size=1028065 2024-09-13T00:00:48.144 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.142+0000 7fb8f2ce0640 1 < completed multipart upload key=FX0NCGLK0KPJ upload=60SPOPIC1T00 tag=YJ8A8EIJIWDE type=write size=3125217 2024-09-13T00:00:48.151 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.150+0000 7fb8f2ce0640 1 > prepared operation key=H5EIP0E9CBWX tag=XGQUHX3K0EGF type=del size=1069403 2024-09-13T00:00:48.156 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.154+0000 7fb8f2ce0640 1 < completed delete operation key=EUE3LT0PDIXV tag=UCQCKA7U33TU type=del 2024-09-13T00:00:48.162 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.162+0000 7fb8f2ce0640 1 > prepared operation key=ZTK9AYBSLCOZ tag=Q1WJ3VTGLVLA type=del size=422707 2024-09-13T00:00:48.167 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.166+0000 7fb8f2ce0640 1 < completed delete operation key=F0QCVR9E08WE tag=Y7H4W53IS48Y type=del 2024-09-13T00:00:48.177 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.178+0000 7fb8f2ce0640 1 > prepared part key=_multipart_F0QCVR9E08WE.2~CKEYKC0MSOFI.0 size=1048576 2024-09-13T00:00:48.181 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.182+0000 7fb8f2ce0640 1 > prepared part key=_multipart_F0QCVR9E08WE.2~CKEYKC0MSOFI.1 size=341903 2024-09-13T00:00:48.182 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.182+0000 7fb8f2ce0640 1 > prepared multipart upload key=F0QCVR9E08WE upload=CKEYKC0MSOFI tag=OZECQQQ9D6P2 type=write size=2439055 2024-09-13T00:00:48.187 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.186+0000 7fb8f2ce0640 1 < completed part key=_multipart_70F7D20HI504.2~2NGDNFV3C9Y1.0 size=1048576 2024-09-13T00:00:48.191 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.190+0000 7fb8f2ce0640 1 < completed part key=_multipart_70F7D20HI504.2~2NGDNFV3C9Y1.1 size=1048576 2024-09-13T00:00:48.196 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.194+0000 7fb8f2ce0640 1 < completed part key=_multipart_70F7D20HI504.2~2NGDNFV3C9Y1.2 size=367919 2024-09-13T00:00:48.206 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.206+0000 7fb8f2ce0640 1 < completed multipart upload key=70F7D20HI504 upload=2NGDNFV3C9Y1 tag=GBIL7OTW24OZ type=write size=3513647 2024-09-13T00:00:48.216 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.214+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~62ZO1HFRMOYU.0 size=1048576 2024-09-13T00:00:48.221 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.218+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~62ZO1HFRMOYU.1 size=1048576 2024-09-13T00:00:48.225 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.226+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~62ZO1HFRMOYU.2 size=293868 2024-09-13T00:00:48.225 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.226+0000 7fb8f2ce0640 1 > prepared multipart upload key=1PX7DBO9M3YI upload=62ZO1HFRMOYU tag=ILF10VVX3KX2 type=write size=3439596 2024-09-13T00:00:48.231 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.230+0000 7fb8f2ce0640 1 < completed delete operation key=2TWD9YLMSYJI tag=8OVONZE932UO type=del 2024-09-13T00:00:48.241 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.242+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~4PBQ7MK82Q5V.0 size=1048576 2024-09-13T00:00:48.246 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.246+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~4PBQ7MK82Q5V.1 size=151031 2024-09-13T00:00:48.246 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.246+0000 7fb8f2ce0640 1 > prepared multipart upload key=2AHYRZUDFHP4 upload=4PBQ7MK82Q5V tag=H2GL9ARAHJAI type=write size=2248183 2024-09-13T00:00:48.246 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.246+0000 7fb8f2ce0640 1 < canceled part key=_multipart_7OX5OJZ8GXU5.2~VW5KKGRKH4VW.0 size=361779 2024-09-13T00:00:48.256 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.254+0000 7fb8f2ce0640 1 < canceled multipart upload key=7OX5OJZ8GXU5 upload=VW5KKGRKH4VW tag=H77D7HKGRR8K type=write size=1410355 2024-09-13T00:00:48.266 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.266+0000 7fb8f2ce0640 1 > prepared part key=_multipart_4K76U69GZ9AE.2~L3OORSMT3MHT.0 size=1048576 2024-09-13T00:00:48.270 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.270+0000 7fb8f2ce0640 1 > prepared part key=_multipart_4K76U69GZ9AE.2~L3OORSMT3MHT.1 size=98615 2024-09-13T00:00:48.270 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.270+0000 7fb8f2ce0640 1 > prepared multipart upload key=4K76U69GZ9AE upload=L3OORSMT3MHT tag=UVPG5ECN3AYW type=write size=2195767 2024-09-13T00:00:48.276 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.274+0000 7fb8f2ce0640 1 < completed delete operation key=0WC6P8YNL0H2 tag=3XJCEDCH9A52 type=del 2024-09-13T00:00:48.281 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.282+0000 7fb8f2ce0640 1 > prepared operation key=1PX7DBO9M3YI tag=5VJBF4SC7AYK type=del size=714269 2024-09-13T00:00:48.291 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.290+0000 7fb8f2ce0640 1 < completed delete operation key=0WC6P8YNL0H2 tag=0BFG49BICWOU type=del 2024-09-13T00:00:48.297 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.294+0000 7fb8f2ce0640 1 > prepared operation key=85YEW8N43FTA tag=27XC245PJCX1 type=del size=4078708 2024-09-13T00:00:48.302 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.302+0000 7fb8f2ce0640 1 < completed part key=_multipart_2AHYRZUDFHP4.2~Q0MZTI9ZRA5V.0 size=889029 2024-09-13T00:00:48.313 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.310+0000 7fb8f2ce0640 1 < completed multipart upload key=2AHYRZUDFHP4 upload=Q0MZTI9ZRA5V tag=IZV70DOQ72M4 type=write size=1937605 2024-09-13T00:00:48.323 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.322+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~TIQ5AXCUNFQW.0 size=1048576 2024-09-13T00:00:48.328 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.326+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~TIQ5AXCUNFQW.1 size=400862 2024-09-13T00:00:48.328 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.326+0000 7fb8f2ce0640 1 > prepared multipart upload key=1PX7DBO9M3YI upload=TIQ5AXCUNFQW tag=ZLS7ZNPUJFFP type=write size=2498014 2024-09-13T00:00:48.333 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.334+0000 7fb8f2ce0640 1 < completed part key=_multipart_O4MZ4UWNXME9.2~D9ZZMDX1XSF2.0 size=1048576 2024-09-13T00:00:48.338 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.338+0000 7fb8f2ce0640 1 < completed part key=_multipart_O4MZ4UWNXME9.2~D9ZZMDX1XSF2.1 size=731329 2024-09-13T00:00:48.348 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.346+0000 7fb8f2ce0640 1 < completed multipart upload key=O4MZ4UWNXME9 upload=D9ZZMDX1XSF2 tag=VEX04MXA08TP type=write size=2828481 2024-09-13T00:00:48.359 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.358+0000 7fb8f2ce0640 1 > prepared part key=_multipart_0WC6P8YNL0H2.2~I4N5WOXWXOPT.0 size=1048576 2024-09-13T00:00:48.364 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.362+0000 7fb8f2ce0640 1 > prepared part key=_multipart_0WC6P8YNL0H2.2~I4N5WOXWXOPT.1 size=1048576 2024-09-13T00:00:48.369 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.370+0000 7fb8f2ce0640 1 > prepared part key=_multipart_0WC6P8YNL0H2.2~I4N5WOXWXOPT.2 size=254156 2024-09-13T00:00:48.369 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.370+0000 7fb8f2ce0640 1 > prepared multipart upload key=0WC6P8YNL0H2 upload=I4N5WOXWXOPT tag=M19LNU5U0S29 type=write size=3399884 2024-09-13T00:00:48.370 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.370+0000 7fb8f2ce0640 1 < canceled part key=_multipart_ZTK9AYBSLCOZ.2~4Z3BYRVRNFCQ.0 size=1048576 2024-09-13T00:00:48.375 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.374+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~4Z3BYRVRNFCQ.1 size=269688 2024-09-13T00:00:48.387 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.386+0000 7fb8f2ce0640 1 < canceled multipart upload key=ZTK9AYBSLCOZ upload=4Z3BYRVRNFCQ tag=SJOE9J1MW8PR type=write size=2366840 2024-09-13T00:00:48.398 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.398+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~EIJNM3TD7QPZ.0 size=1048576 2024-09-13T00:00:48.404 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.402+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~EIJNM3TD7QPZ.1 size=1048576 2024-09-13T00:00:48.409 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.406+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~EIJNM3TD7QPZ.2 size=85320 2024-09-13T00:00:48.409 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.406+0000 7fb8f2ce0640 1 > prepared multipart upload key=NQJQC8CEAC15 upload=EIJNM3TD7QPZ tag=DNWVCWKSLXPP type=write size=3231048 2024-09-13T00:00:48.414 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.414+0000 7fb8f2ce0640 1 < completed delete operation key=QMZREZ1QEDNJ tag=3TBEZC71D79C type=del 2024-09-13T00:00:48.424 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.422+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EUE3LT0PDIXV.2~GHJ50CYO7A0D.0 size=1048576 2024-09-13T00:00:48.428 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.426+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EUE3LT0PDIXV.2~GHJ50CYO7A0D.1 size=1048576 2024-09-13T00:00:48.433 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.434+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EUE3LT0PDIXV.2~GHJ50CYO7A0D.2 size=603427 2024-09-13T00:00:48.433 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.434+0000 7fb8f2ce0640 1 > prepared multipart upload key=EUE3LT0PDIXV upload=GHJ50CYO7A0D tag=PVHLDEDB8L35 type=write size=3749155 2024-09-13T00:00:48.439 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.438+0000 7fb8f2ce0640 1 < completed delete operation key=H4M1TCN5W6XU tag=Y66LLM5YWL5K type=del 2024-09-13T00:00:48.445 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.442+0000 7fb8f2ce0640 1 > prepared operation key=0WC6P8YNL0H2 tag=IH7KG07UPVI1 type=del size=3180930 2024-09-13T00:00:48.451 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.450+0000 7fb8f2ce0640 1 < completed part key=_multipart_70F7D20HI504.2~3L1RGLXGSVN7.0 size=124853 2024-09-13T00:00:48.460 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.458+0000 7fb8f2ce0640 1 < completed multipart upload key=70F7D20HI504 upload=3L1RGLXGSVN7 tag=LR83RXFRYL3W type=write size=1173429 2024-09-13T00:00:48.466 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.466+0000 7fb8f2ce0640 1 > prepared operation key=5GM88ERM89KT tag=7N3Y8YAQBA6M type=del size=524495 2024-09-13T00:00:48.472 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.470+0000 7fb8f2ce0640 1 < completed part key=_multipart_0WC6P8YNL0H2.2~WOFL9ULW30GH.0 size=1048576 2024-09-13T00:00:48.478 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.478+0000 7fb8f2ce0640 1 < completed part key=_multipart_0WC6P8YNL0H2.2~WOFL9ULW30GH.1 size=323135 2024-09-13T00:00:48.487 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.486+0000 7fb8f2ce0640 1 < completed multipart upload key=0WC6P8YNL0H2 upload=WOFL9ULW30GH tag=FBNIONWUDCE6 type=write size=2420287 2024-09-13T00:00:48.501 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.502+0000 7fb8f2ce0640 1 > prepared part key=_multipart_P8OYS5ZE1NX0.2~FDHDX5OLVOP8.0 size=1048576 2024-09-13T00:00:48.505 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.506+0000 7fb8f2ce0640 1 > prepared part key=_multipart_P8OYS5ZE1NX0.2~FDHDX5OLVOP8.1 size=1048576 2024-09-13T00:00:48.510 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.510+0000 7fb8f2ce0640 1 > prepared part key=_multipart_P8OYS5ZE1NX0.2~FDHDX5OLVOP8.2 size=738404 2024-09-13T00:00:48.510 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.510+0000 7fb8f2ce0640 1 > prepared multipart upload key=P8OYS5ZE1NX0 upload=FDHDX5OLVOP8 tag=2S66VX169G6A type=write size=3884132 2024-09-13T00:00:48.515 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.514+0000 7fb8f2ce0640 1 < completed part key=_multipart_H5EIP0E9CBWX.2~PENBT3I1LAID.0 size=1048576 2024-09-13T00:00:48.521 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.518+0000 7fb8f2ce0640 1 < completed part key=_multipart_H5EIP0E9CBWX.2~PENBT3I1LAID.1 size=1048576 2024-09-13T00:00:48.525 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.526+0000 7fb8f2ce0640 1 < completed part key=_multipart_H5EIP0E9CBWX.2~PENBT3I1LAID.2 size=702070 2024-09-13T00:00:48.535 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.534+0000 7fb8f2ce0640 1 < completed multipart upload key=H5EIP0E9CBWX upload=PENBT3I1LAID tag=55CW0ASNQ8DO type=write size=3847798 2024-09-13T00:00:48.545 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.546+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZFHN2SNTZ5W7.2~6K8CG6S770UL.0 size=1048576 2024-09-13T00:00:48.550 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.550+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZFHN2SNTZ5W7.2~6K8CG6S770UL.1 size=956666 2024-09-13T00:00:48.550 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.550+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZFHN2SNTZ5W7 upload=6K8CG6S770UL tag=HVPPSWLPC0AC type=write size=3053818 2024-09-13T00:00:48.556 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.554+0000 7fb8f2ce0640 1 < canceled operation key=H5EIP0E9CBWX tag=XGQUHX3K0EGF type=del size=1069403 2024-09-13T00:00:48.562 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.562+0000 7fb8f2ce0640 1 > prepared operation key=H5EIP0E9CBWX tag=2G3KKH6N3I52 type=del size=4126837 2024-09-13T00:00:48.568 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.566+0000 7fb8f2ce0640 1 < completed delete operation key=ZTK9AYBSLCOZ tag=Q1WJ3VTGLVLA type=del 2024-09-13T00:00:48.579 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.578+0000 7fb8f2ce0640 1 > prepared part key=_multipart_4K76U69GZ9AE.2~6E01JLZU4VGC.0 size=1048576 2024-09-13T00:00:48.583 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.582+0000 7fb8f2ce0640 1 > prepared part key=_multipart_4K76U69GZ9AE.2~6E01JLZU4VGC.1 size=1048576 2024-09-13T00:00:48.588 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.586+0000 7fb8f2ce0640 1 > prepared part key=_multipart_4K76U69GZ9AE.2~6E01JLZU4VGC.2 size=768206 2024-09-13T00:00:48.588 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.586+0000 7fb8f2ce0640 1 > prepared multipart upload key=4K76U69GZ9AE upload=6E01JLZU4VGC tag=QPD7G3MPQY9Z type=write size=3913934 2024-09-13T00:00:48.594 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.594+0000 7fb8f2ce0640 1 < completed part key=_multipart_F0QCVR9E08WE.2~CKEYKC0MSOFI.0 size=1048576 2024-09-13T00:00:48.599 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.598+0000 7fb8f2ce0640 1 < completed part key=_multipart_F0QCVR9E08WE.2~CKEYKC0MSOFI.1 size=341903 2024-09-13T00:00:48.610 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.610+0000 7fb8f2ce0640 1 < completed multipart upload key=F0QCVR9E08WE upload=CKEYKC0MSOFI tag=OZECQQQ9D6P2 type=write size=2439055 2024-09-13T00:00:48.616 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.614+0000 7fb8f2ce0640 1 > prepared operation key=70F7D20HI504 tag=6CD9KO0LTZXL type=del size=1307167 2024-09-13T00:00:48.621 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.622+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~62ZO1HFRMOYU.0 size=1048576 2024-09-13T00:00:48.621 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.622+0000 7fb8f2ce0640 1 < canceled part key=_multipart_1PX7DBO9M3YI.2~62ZO1HFRMOYU.1 size=1048576 2024-09-13T00:00:48.626 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.626+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~62ZO1HFRMOYU.2 size=293868 2024-09-13T00:00:48.636 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.634+0000 7fb8f2ce0640 1 < canceled multipart upload key=1PX7DBO9M3YI upload=62ZO1HFRMOYU tag=ILF10VVX3KX2 type=write size=3439596 2024-09-13T00:00:48.642 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.642+0000 7fb8f2ce0640 1 > prepared operation key=1PX7DBO9M3YI tag=BBK4XV01H822 type=write size=65108 2024-09-13T00:00:48.647 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.646+0000 7fb8f2ce0640 1 < completed part key=_multipart_2AHYRZUDFHP4.2~4PBQ7MK82Q5V.0 size=1048576 2024-09-13T00:00:48.648 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.646+0000 7fb8f2ce0640 1 < canceled part key=_multipart_2AHYRZUDFHP4.2~4PBQ7MK82Q5V.1 size=151031 2024-09-13T00:00:48.657 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.658+0000 7fb8f2ce0640 1 < canceled multipart upload key=2AHYRZUDFHP4 upload=4PBQ7MK82Q5V tag=H2GL9ARAHJAI type=write size=2248183 2024-09-13T00:00:48.663 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.662+0000 7fb8f2ce0640 1 > prepared operation key=O4MZ4UWNXME9 tag=T29C8AP7H5HV type=write size=941641 2024-09-13T00:00:48.669 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.666+0000 7fb8f2ce0640 1 < completed part key=_multipart_4K76U69GZ9AE.2~L3OORSMT3MHT.0 size=1048576 2024-09-13T00:00:48.673 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.674+0000 7fb8f2ce0640 1 < completed part key=_multipart_4K76U69GZ9AE.2~L3OORSMT3MHT.1 size=98615 2024-09-13T00:00:48.682 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.682+0000 7fb8f2ce0640 1 < completed multipart upload key=4K76U69GZ9AE upload=L3OORSMT3MHT tag=UVPG5ECN3AYW type=write size=2195767 2024-09-13T00:00:48.688 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.686+0000 7fb8f2ce0640 1 > prepared operation key=0WC6P8YNL0H2 tag=LTYCVTLRKS3L type=del size=460043 2024-09-13T00:00:48.694 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.694+0000 7fb8f2ce0640 1 < completed delete operation key=1PX7DBO9M3YI tag=5VJBF4SC7AYK type=del 2024-09-13T00:00:48.703 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.702+0000 7fb8f2ce0640 1 > prepared part key=_multipart_J8NGDSF9XRTP.2~QX2R0F5HEHJS.0 size=1048576 2024-09-13T00:00:48.707 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.706+0000 7fb8f2ce0640 1 > prepared part key=_multipart_J8NGDSF9XRTP.2~QX2R0F5HEHJS.1 size=1048576 2024-09-13T00:00:48.712 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.710+0000 7fb8f2ce0640 1 > prepared part key=_multipart_J8NGDSF9XRTP.2~QX2R0F5HEHJS.2 size=278310 2024-09-13T00:00:48.712 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.710+0000 7fb8f2ce0640 1 > prepared multipart upload key=J8NGDSF9XRTP upload=QX2R0F5HEHJS tag=LO6Q96062XRG type=write size=3424038 2024-09-13T00:00:48.717 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.718+0000 7fb8f2ce0640 1 < completed delete operation key=85YEW8N43FTA tag=27XC245PJCX1 type=del 2024-09-13T00:00:48.726 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.726+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~VF35W658XA27.0 size=469310 2024-09-13T00:00:48.727 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.726+0000 7fb8f2ce0640 1 > prepared multipart upload key=2AHYRZUDFHP4 upload=VF35W658XA27 tag=ECNWCUGVBR7V type=write size=1517886 2024-09-13T00:00:48.732 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.730+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~TIQ5AXCUNFQW.0 size=1048576 2024-09-13T00:00:48.737 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.734+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~TIQ5AXCUNFQW.1 size=400862 2024-09-13T00:00:48.745 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.746+0000 7fb8f2ce0640 1 < completed multipart upload key=1PX7DBO9M3YI upload=TIQ5AXCUNFQW tag=ZLS7ZNPUJFFP type=write size=2498014 2024-09-13T00:00:48.755 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.754+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~9J6KC8C7P2YV.0 size=602201 2024-09-13T00:00:48.756 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.754+0000 7fb8f2ce0640 1 > prepared multipart upload key=NQJQC8CEAC15 upload=9J6KC8C7P2YV tag=QFYQFVHQ5XCB type=write size=1650777 2024-09-13T00:00:48.761 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.762+0000 7fb8f2ce0640 1 < completed part key=_multipart_0WC6P8YNL0H2.2~I4N5WOXWXOPT.0 size=1048576 2024-09-13T00:00:48.766 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.766+0000 7fb8f2ce0640 1 < completed part key=_multipart_0WC6P8YNL0H2.2~I4N5WOXWXOPT.1 size=1048576 2024-09-13T00:00:48.771 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.770+0000 7fb8f2ce0640 1 < completed part key=_multipart_0WC6P8YNL0H2.2~I4N5WOXWXOPT.2 size=254156 2024-09-13T00:00:48.779 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.778+0000 7fb8f2ce0640 1 < completed multipart upload key=0WC6P8YNL0H2 upload=I4N5WOXWXOPT tag=M19LNU5U0S29 type=write size=3399884 2024-09-13T00:00:48.789 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.790+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~VCVB0IH5TW3W.0 size=1048576 2024-09-13T00:00:48.793 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.794+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~VCVB0IH5TW3W.1 size=569863 2024-09-13T00:00:48.793 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.794+0000 7fb8f2ce0640 1 > prepared multipart upload key=W8XZ26311RJX upload=VCVB0IH5TW3W tag=XTIG45GBO659 type=write size=2667015 2024-09-13T00:00:48.798 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.798+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~EIJNM3TD7QPZ.0 size=1048576 2024-09-13T00:00:48.803 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.802+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~EIJNM3TD7QPZ.1 size=1048576 2024-09-13T00:00:48.808 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.806+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~EIJNM3TD7QPZ.2 size=85320 2024-09-13T00:00:48.817 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.818+0000 7fb8f2ce0640 1 < completed multipart upload key=NQJQC8CEAC15 upload=EIJNM3TD7QPZ tag=DNWVCWKSLXPP type=write size=3231048 2024-09-13T00:00:48.826 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.826+0000 7fb8f2ce0640 1 > prepared part key=_multipart_O4MZ4UWNXME9.2~PL4D2S4DAJFN.0 size=1048576 2024-09-13T00:00:48.831 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.830+0000 7fb8f2ce0640 1 > prepared part key=_multipart_O4MZ4UWNXME9.2~PL4D2S4DAJFN.1 size=1048576 2024-09-13T00:00:48.836 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.834+0000 7fb8f2ce0640 1 > prepared part key=_multipart_O4MZ4UWNXME9.2~PL4D2S4DAJFN.2 size=837217 2024-09-13T00:00:48.836 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.834+0000 7fb8f2ce0640 1 > prepared multipart upload key=O4MZ4UWNXME9 upload=PL4D2S4DAJFN tag=FIA40Q91NIZ3 type=write size=3982945 2024-09-13T00:00:48.842 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.842+0000 7fb8f2ce0640 1 < completed part key=_multipart_EUE3LT0PDIXV.2~GHJ50CYO7A0D.0 size=1048576 2024-09-13T00:00:48.846 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.846+0000 7fb8f2ce0640 1 < completed part key=_multipart_EUE3LT0PDIXV.2~GHJ50CYO7A0D.1 size=1048576 2024-09-13T00:00:48.851 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.850+0000 7fb8f2ce0640 1 < completed part key=_multipart_EUE3LT0PDIXV.2~GHJ50CYO7A0D.2 size=603427 2024-09-13T00:00:48.861 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.862+0000 7fb8f2ce0640 1 < completed multipart upload key=EUE3LT0PDIXV upload=GHJ50CYO7A0D tag=PVHLDEDB8L35 type=write size=3749155 2024-09-13T00:00:48.866 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.866+0000 7fb8f2ce0640 1 > prepared operation key=H9AU4HD247YU tag=9AYANCD8HX4J type=del size=446887 2024-09-13T00:00:48.878 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.878+0000 7fb8f2ce0640 1 < suggested remove operation key=0WC6P8YNL0H2 tag=IH7KG07UPVI1 type=del 2024-09-13T00:00:48.888 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.886+0000 7fb8f2ce0640 1 > prepared part key=_multipart_5GM88ERM89KT.2~NTHJ9S9TD4DB.0 size=1048576 2024-09-13T00:00:48.893 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.894+0000 7fb8f2ce0640 1 > prepared part key=_multipart_5GM88ERM89KT.2~NTHJ9S9TD4DB.1 size=1048576 2024-09-13T00:00:48.898 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.898+0000 7fb8f2ce0640 1 > prepared part key=_multipart_5GM88ERM89KT.2~NTHJ9S9TD4DB.2 size=340321 2024-09-13T00:00:48.898 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.898+0000 7fb8f2ce0640 1 > prepared multipart upload key=5GM88ERM89KT upload=NTHJ9S9TD4DB tag=VYOM9GAK3M8O type=write size=3486049 2024-09-13T00:00:48.904 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.902+0000 7fb8f2ce0640 1 < completed delete operation key=5GM88ERM89KT tag=7N3Y8YAQBA6M type=del 2024-09-13T00:00:48.911 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.910+0000 7fb8f2ce0640 1 > prepared operation key=ZTK9AYBSLCOZ tag=LVRLJOXZDU1S type=del size=1372853 2024-09-13T00:00:48.916 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.914+0000 7fb8f2ce0640 1 < completed part key=_multipart_P8OYS5ZE1NX0.2~FDHDX5OLVOP8.0 size=1048576 2024-09-13T00:00:48.921 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.922+0000 7fb8f2ce0640 1 < completed part key=_multipart_P8OYS5ZE1NX0.2~FDHDX5OLVOP8.1 size=1048576 2024-09-13T00:00:48.926 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.926+0000 7fb8f2ce0640 1 < completed part key=_multipart_P8OYS5ZE1NX0.2~FDHDX5OLVOP8.2 size=738404 2024-09-13T00:00:48.938 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.938+0000 7fb8f2ce0640 1 < completed multipart upload key=P8OYS5ZE1NX0 upload=FDHDX5OLVOP8 tag=2S66VX169G6A type=write size=3884132 2024-09-13T00:00:48.943 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.942+0000 7fb8f2ce0640 1 > prepared operation key=EUE3LT0PDIXV tag=K2EP0O4G5ZFW type=del size=3364345 2024-09-13T00:00:48.949 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.950+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZFHN2SNTZ5W7.2~6K8CG6S770UL.0 size=1048576 2024-09-13T00:00:48.953 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.954+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZFHN2SNTZ5W7.2~6K8CG6S770UL.1 size=956666 2024-09-13T00:00:48.963 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.962+0000 7fb8f2ce0640 1 < completed multipart upload key=ZFHN2SNTZ5W7 upload=6K8CG6S770UL tag=HVPPSWLPC0AC type=write size=3053818 2024-09-13T00:00:48.969 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.970+0000 7fb8f2ce0640 1 > prepared operation key=8MPUVD37EDN9 tag=8DUF6D1RZMTD type=del size=543548 2024-09-13T00:00:48.974 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.974+0000 7fb8f2ce0640 1 < completed delete operation key=H5EIP0E9CBWX tag=2G3KKH6N3I52 type=del 2024-09-13T00:00:48.980 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.978+0000 7fb8f2ce0640 1 > prepared operation key=8Z6U7OOA0GBH tag=3JLB4EH1QI88 type=del size=3700933 2024-09-13T00:00:48.981 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.982+0000 7fb8f2ce0640 1 < canceled part key=_multipart_4K76U69GZ9AE.2~6E01JLZU4VGC.0 size=1048576 2024-09-13T00:00:48.986 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.986+0000 7fb8f2ce0640 1 < completed part key=_multipart_4K76U69GZ9AE.2~6E01JLZU4VGC.1 size=1048576 2024-09-13T00:00:48.992 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:48.990+0000 7fb8f2ce0640 1 < completed part key=_multipart_4K76U69GZ9AE.2~6E01JLZU4VGC.2 size=768206 2024-09-13T00:00:49.003 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.002+0000 7fb8f2ce0640 1 < canceled multipart upload key=4K76U69GZ9AE upload=6E01JLZU4VGC tag=QPD7G3MPQY9Z type=write size=3913934 2024-09-13T00:00:49.014 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.014+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~7TDBQH0WB9VG.0 size=1048576 2024-09-13T00:00:49.019 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.018+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~7TDBQH0WB9VG.1 size=1048576 2024-09-13T00:00:49.023 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.022+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~7TDBQH0WB9VG.2 size=27556 2024-09-13T00:00:49.023 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.022+0000 7fb8f2ce0640 1 > prepared multipart upload key=8Z6U7OOA0GBH upload=7TDBQH0WB9VG tag=OLCTNIEW4H3D type=write size=3173284 2024-09-13T00:00:49.028 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.026+0000 7fb8f2ce0640 1 < completed delete operation key=70F7D20HI504 tag=6CD9KO0LTZXL type=del 2024-09-13T00:00:49.033 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.034+0000 7fb8f2ce0640 1 > prepared operation key=H4M1TCN5W6XU tag=KC8XTPI07F76 type=del size=4106476 2024-09-13T00:00:49.039 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.038+0000 7fb8f2ce0640 1 < completed write operation key=1PX7DBO9M3YI tag=BBK4XV01H822 type=write size=65108 2024-09-13T00:00:49.045 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.042+0000 7fb8f2ce0640 1 > prepared operation key=F0QCVR9E08WE tag=H2HBVCRO2382 type=del size=895556 2024-09-13T00:00:49.050 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.050+0000 7fb8f2ce0640 1 < completed write operation key=O4MZ4UWNXME9 tag=T29C8AP7H5HV type=write size=941641 2024-09-13T00:00:49.055 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.054+0000 7fb8f2ce0640 1 > prepared operation key=7OX5OJZ8GXU5 tag=N5D7WQEXFKUO type=del size=2888447 2024-09-13T00:00:49.062 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.062+0000 7fb8f2ce0640 -1 < failed to complete operation key=0WC6P8YNL0H2 tag=LTYCVTLRKS3L type=del size=460043: (22) Invalid argument 2024-09-13T00:00:49.072 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.070+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2TWD9YLMSYJI.2~YCPOUC6F1YCO.0 size=1048576 2024-09-13T00:00:49.077 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.078+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2TWD9YLMSYJI.2~YCPOUC6F1YCO.1 size=1048576 2024-09-13T00:00:49.082 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.082+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2TWD9YLMSYJI.2~YCPOUC6F1YCO.2 size=675764 2024-09-13T00:00:49.082 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.082+0000 7fb8f2ce0640 1 > prepared multipart upload key=2TWD9YLMSYJI upload=YCPOUC6F1YCO tag=YT1X0G18SDS7 type=write size=3821492 2024-09-13T00:00:49.087 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.086+0000 7fb8f2ce0640 1 < completed part key=_multipart_J8NGDSF9XRTP.2~QX2R0F5HEHJS.0 size=1048576 2024-09-13T00:00:49.087 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.086+0000 7fb8f2ce0640 1 < canceled part key=_multipart_J8NGDSF9XRTP.2~QX2R0F5HEHJS.1 size=1048576 2024-09-13T00:00:49.092 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.090+0000 7fb8f2ce0640 1 < completed part key=_multipart_J8NGDSF9XRTP.2~QX2R0F5HEHJS.2 size=278310 2024-09-13T00:00:49.102 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.102+0000 7fb8f2ce0640 1 < canceled multipart upload key=J8NGDSF9XRTP upload=QX2R0F5HEHJS tag=LO6Q96062XRG type=write size=3424038 2024-09-13T00:00:49.107 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.106+0000 7fb8f2ce0640 1 > prepared operation key=44EDYFT9VF8Y tag=FI31RGMAMMC5 type=write size=154203 2024-09-13T00:00:49.113 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.114+0000 7fb8f2ce0640 1 < completed part key=_multipart_2AHYRZUDFHP4.2~VF35W658XA27.0 size=469310 2024-09-13T00:00:49.123 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.122+0000 7fb8f2ce0640 1 < completed multipart upload key=2AHYRZUDFHP4 upload=VF35W658XA27 tag=ECNWCUGVBR7V type=write size=1517886 2024-09-13T00:00:49.128 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.126+0000 7fb8f2ce0640 1 > prepared operation key=ZFHN2SNTZ5W7 tag=6U6M66O4R34U type=del size=1690916 2024-09-13T00:00:49.134 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.134+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~9J6KC8C7P2YV.0 size=602201 2024-09-13T00:00:49.143 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.142+0000 7fb8f2ce0640 1 < completed multipart upload key=NQJQC8CEAC15 upload=9J6KC8C7P2YV tag=QFYQFVHQ5XCB type=write size=1650777 2024-09-13T00:00:49.149 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.150+0000 7fb8f2ce0640 1 > prepared operation key=F0QCVR9E08WE tag=IKG2MO75CDCQ type=write size=305093 2024-09-13T00:00:49.154 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.154+0000 7fb8f2ce0640 1 < completed part key=_multipart_W8XZ26311RJX.2~VCVB0IH5TW3W.0 size=1048576 2024-09-13T00:00:49.159 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.158+0000 7fb8f2ce0640 1 < completed part key=_multipart_W8XZ26311RJX.2~VCVB0IH5TW3W.1 size=569863 2024-09-13T00:00:49.168 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.166+0000 7fb8f2ce0640 1 < completed multipart upload key=W8XZ26311RJX upload=VCVB0IH5TW3W tag=XTIG45GBO659 type=write size=2667015 2024-09-13T00:00:49.174 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.174+0000 7fb8f2ce0640 1 > prepared operation key=70F7D20HI504 tag=B6C70ANEIHFC type=del size=3483683 2024-09-13T00:00:49.179 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.178+0000 7fb8f2ce0640 1 < completed part key=_multipart_O4MZ4UWNXME9.2~PL4D2S4DAJFN.0 size=1048576 2024-09-13T00:00:49.184 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.182+0000 7fb8f2ce0640 1 < completed part key=_multipart_O4MZ4UWNXME9.2~PL4D2S4DAJFN.1 size=1048576 2024-09-13T00:00:49.188 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.186+0000 7fb8f2ce0640 1 < completed part key=_multipart_O4MZ4UWNXME9.2~PL4D2S4DAJFN.2 size=837217 2024-09-13T00:00:49.198 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.198+0000 7fb8f2ce0640 1 < completed multipart upload key=O4MZ4UWNXME9 upload=PL4D2S4DAJFN tag=FIA40Q91NIZ3 type=write size=3982945 2024-09-13T00:00:49.209 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.210+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~1YS0ZGT2HJ1U.0 size=1048576 2024-09-13T00:00:49.213 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.214+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~1YS0ZGT2HJ1U.1 size=1048576 2024-09-13T00:00:49.218 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.218+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~1YS0ZGT2HJ1U.2 size=158998 2024-09-13T00:00:49.218 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.218+0000 7fb8f2ce0640 1 > prepared multipart upload key=NQJQC8CEAC15 upload=1YS0ZGT2HJ1U tag=0794UW85RAT0 type=write size=3304726 2024-09-13T00:00:49.224 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.222+0000 7fb8f2ce0640 1 < completed delete operation key=H9AU4HD247YU tag=9AYANCD8HX4J type=del 2024-09-13T00:00:49.229 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.230+0000 7fb8f2ce0640 1 > prepared operation key=8Z6U7OOA0GBH tag=872U1B8Q54CA type=del size=3017043 2024-09-13T00:00:49.235 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.234+0000 7fb8f2ce0640 1 < completed part key=_multipart_5GM88ERM89KT.2~NTHJ9S9TD4DB.0 size=1048576 2024-09-13T00:00:49.240 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.238+0000 7fb8f2ce0640 1 < completed part key=_multipart_5GM88ERM89KT.2~NTHJ9S9TD4DB.1 size=1048576 2024-09-13T00:00:49.244 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.242+0000 7fb8f2ce0640 1 < completed part key=_multipart_5GM88ERM89KT.2~NTHJ9S9TD4DB.2 size=340321 2024-09-13T00:00:49.255 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.254+0000 7fb8f2ce0640 1 < completed multipart upload key=5GM88ERM89KT upload=NTHJ9S9TD4DB tag=VYOM9GAK3M8O type=write size=3486049 2024-09-13T00:00:49.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:48 smithi136 bash[75164]: cluster 2024-09-13T00:00:47.133291+0000 mgr.y (mgr.24661) 710 : cluster [DBG] pgmap v1386: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 805 KiB/s rd, 278 KiB/s wr, 1.43k op/s 2024-09-13T00:00:49.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:48 smithi136 bash[76371]: cluster 2024-09-13T00:00:47.133291+0000 mgr.y (mgr.24661) 710 : cluster [DBG] pgmap v1386: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 805 KiB/s rd, 278 KiB/s wr, 1.43k op/s 2024-09-13T00:00:49.265 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.266+0000 7fb8f2ce0640 1 > prepared part key=_multipart_QMZREZ1QEDNJ.2~O6HUPAELPRO2.0 size=570930 2024-09-13T00:00:49.265 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.266+0000 7fb8f2ce0640 1 > prepared multipart upload key=QMZREZ1QEDNJ upload=O6HUPAELPRO2 tag=YQ79MYPICP4H type=write size=1619506 2024-09-13T00:00:49.272 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.270+0000 7fb8f2ce0640 1 < completed delete operation key=ZTK9AYBSLCOZ tag=LVRLJOXZDU1S type=del 2024-09-13T00:00:49.278 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.278+0000 7fb8f2ce0640 1 > prepared operation key=0WC6P8YNL0H2 tag=E5A8CV2E3T7I type=write size=781724 2024-09-13T00:00:49.284 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.282+0000 7fb8f2ce0640 1 < completed delete operation key=EUE3LT0PDIXV tag=K2EP0O4G5ZFW type=del 2024-09-13T00:00:49.289 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.290+0000 7fb8f2ce0640 1 > prepared operation key=EJN6HR8I7XQF tag=H4WKNY8P0XHR type=del size=4116185 2024-09-13T00:00:49.295 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.294+0000 7fb8f2ce0640 1 < completed delete operation key=8MPUVD37EDN9 tag=8DUF6D1RZMTD type=del 2024-09-13T00:00:49.307 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.306+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~DLRFZ98I4980.0 size=1048576 2024-09-13T00:00:49.312 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.310+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~DLRFZ98I4980.1 size=559203 2024-09-13T00:00:49.312 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.310+0000 7fb8f2ce0640 1 > prepared multipart upload key=70F7D20HI504 upload=DLRFZ98I4980 tag=W6K2OKGL0OKE type=write size=2656355 2024-09-13T00:00:49.317 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.318+0000 7fb8f2ce0640 1 < completed delete operation key=8Z6U7OOA0GBH tag=3JLB4EH1QI88 type=del 2024-09-13T00:00:49.323 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.322+0000 7fb8f2ce0640 1 > prepared operation key=H5EIP0E9CBWX tag=5L8H5CQF8P0X type=write size=958188 2024-09-13T00:00:49.324 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.322+0000 7fb8f2ce0640 1 < canceled part key=_multipart_8Z6U7OOA0GBH.2~7TDBQH0WB9VG.0 size=1048576 2024-09-13T00:00:49.328 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.326+0000 7fb8f2ce0640 1 < completed part key=_multipart_8Z6U7OOA0GBH.2~7TDBQH0WB9VG.1 size=1048576 2024-09-13T00:00:49.333 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.334+0000 7fb8f2ce0640 1 < completed part key=_multipart_8Z6U7OOA0GBH.2~7TDBQH0WB9VG.2 size=27556 2024-09-13T00:00:49.342 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.342+0000 7fb8f2ce0640 1 < canceled multipart upload key=8Z6U7OOA0GBH upload=7TDBQH0WB9VG tag=OLCTNIEW4H3D type=write size=3173284 2024-09-13T00:00:49.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:48 smithi181 bash[67587]: cluster 2024-09-13T00:00:47.133291+0000 mgr.y (mgr.24661) 710 : cluster [DBG] pgmap v1386: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 805 KiB/s rd, 278 KiB/s wr, 1.43k op/s 2024-09-13T00:00:49.348 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.346+0000 7fb8f2ce0640 1 > prepared operation key=P8OYS5ZE1NX0 tag=214WPD39S1Z1 type=del size=1260310 2024-09-13T00:00:49.353 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.354+0000 7fb8f2ce0640 1 < completed delete operation key=H4M1TCN5W6XU tag=KC8XTPI07F76 type=del 2024-09-13T00:00:49.364 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.362+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EUE3LT0PDIXV.2~ECQ7T61YZI9I.0 size=1048576 2024-09-13T00:00:49.368 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.366+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EUE3LT0PDIXV.2~ECQ7T61YZI9I.1 size=145697 2024-09-13T00:00:49.369 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.366+0000 7fb8f2ce0640 1 > prepared multipart upload key=EUE3LT0PDIXV upload=ECQ7T61YZI9I tag=9S5SAR0Q6EXQ type=write size=2242849 2024-09-13T00:00:49.374 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.374+0000 7fb8f2ce0640 1 < completed delete operation key=F0QCVR9E08WE tag=H2HBVCRO2382 type=del 2024-09-13T00:00:49.384 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.382+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~E4GVLOB4N0U4.0 size=663217 2024-09-13T00:00:49.384 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.382+0000 7fb8f2ce0640 1 > prepared multipart upload key=63FJDWBO2T9S upload=E4GVLOB4N0U4 tag=7J87MAG2Z2IF type=write size=1711793 2024-09-13T00:00:49.390 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.390+0000 7fb8f2ce0640 1 < completed delete operation key=7OX5OJZ8GXU5 tag=N5D7WQEXFKUO type=del 2024-09-13T00:00:49.400 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.398+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~OBKKLL4H5USH.0 size=1048576 2024-09-13T00:00:49.405 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.402+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~OBKKLL4H5USH.1 size=63318 2024-09-13T00:00:49.405 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.402+0000 7fb8f2ce0640 1 > prepared multipart upload key=8Z6U7OOA0GBH upload=OBKKLL4H5USH tag=6O0O46MR8W1B type=write size=2160470 2024-09-13T00:00:49.410 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.410+0000 7fb8f2ce0640 1 < completed part key=_multipart_2TWD9YLMSYJI.2~YCPOUC6F1YCO.0 size=1048576 2024-09-13T00:00:49.416 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.414+0000 7fb8f2ce0640 1 < completed part key=_multipart_2TWD9YLMSYJI.2~YCPOUC6F1YCO.1 size=1048576 2024-09-13T00:00:49.421 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.418+0000 7fb8f2ce0640 1 < completed part key=_multipart_2TWD9YLMSYJI.2~YCPOUC6F1YCO.2 size=675764 2024-09-13T00:00:49.430 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.430+0000 7fb8f2ce0640 1 < completed multipart upload key=2TWD9YLMSYJI upload=YCPOUC6F1YCO tag=YT1X0G18SDS7 type=write size=3821492 2024-09-13T00:00:49.441 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.438+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H4M1TCN5W6XU.2~OKBCCAY5G0ZP.0 size=808539 2024-09-13T00:00:49.441 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.438+0000 7fb8f2ce0640 1 > prepared multipart upload key=H4M1TCN5W6XU upload=OKBCCAY5G0ZP tag=4AT0HY1Y1XN9 type=write size=1857115 2024-09-13T00:00:49.452 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.450+0000 7fb8f2ce0640 1 < suggested update operation key=44EDYFT9VF8Y tag=FI31RGMAMMC5 type=write size=154203 2024-09-13T00:00:49.461 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.462+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~5GNMJK2L2XYJ.0 size=1048576 2024-09-13T00:00:49.466 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.466+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~5GNMJK2L2XYJ.1 size=1048576 2024-09-13T00:00:49.472 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.470+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~5GNMJK2L2XYJ.2 size=337307 2024-09-13T00:00:49.473 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.470+0000 7fb8f2ce0640 1 > prepared multipart upload key=63FJDWBO2T9S upload=5GNMJK2L2XYJ tag=5IMQ4TYVPMNL type=write size=3483035 2024-09-13T00:00:49.477 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.478+0000 7fb8f2ce0640 1 < completed delete operation key=ZFHN2SNTZ5W7 tag=6U6M66O4R34U type=del 2024-09-13T00:00:49.488 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.486+0000 7fb8f2ce0640 1 > prepared part key=_multipart_0WC6P8YNL0H2.2~0SQ2WZEVVG3X.0 size=1048576 2024-09-13T00:00:49.492 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.490+0000 7fb8f2ce0640 1 > prepared part key=_multipart_0WC6P8YNL0H2.2~0SQ2WZEVVG3X.1 size=1048576 2024-09-13T00:00:49.498 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.498+0000 7fb8f2ce0640 1 > prepared part key=_multipart_0WC6P8YNL0H2.2~0SQ2WZEVVG3X.2 size=311504 2024-09-13T00:00:49.498 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.498+0000 7fb8f2ce0640 1 > prepared multipart upload key=0WC6P8YNL0H2 upload=0SQ2WZEVVG3X tag=VT26PT7UHAL3 type=write size=3457232 2024-09-13T00:00:49.504 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.502+0000 7fb8f2ce0640 1 < completed write operation key=F0QCVR9E08WE tag=IKG2MO75CDCQ type=write size=305093 2024-09-13T00:00:49.510 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.510+0000 7fb8f2ce0640 1 > prepared operation key=8Z6U7OOA0GBH tag=A7KMFIEPGMMD type=write size=843276 2024-09-13T00:00:49.516 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.514+0000 7fb8f2ce0640 1 < completed delete operation key=70F7D20HI504 tag=B6C70ANEIHFC type=del 2024-09-13T00:00:49.523 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.522+0000 7fb8f2ce0640 1 > prepared operation key=QMZREZ1QEDNJ tag=K1BDN2INO11F type=del size=313413 2024-09-13T00:00:49.529 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.530+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~1YS0ZGT2HJ1U.0 size=1048576 2024-09-13T00:00:49.535 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.534+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~1YS0ZGT2HJ1U.1 size=1048576 2024-09-13T00:00:49.541 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.542+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~1YS0ZGT2HJ1U.2 size=158998 2024-09-13T00:00:49.552 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.550+0000 7fb8f2ce0640 1 < completed multipart upload key=NQJQC8CEAC15 upload=1YS0ZGT2HJ1U tag=0794UW85RAT0 type=write size=3304726 2024-09-13T00:00:49.557 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.558+0000 7fb8f2ce0640 1 > prepared operation key=8MPUVD37EDN9 tag=LXOYQFA9Q31D type=del size=1451429 2024-09-13T00:00:49.564 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.562+0000 7fb8f2ce0640 1 < completed delete operation key=8Z6U7OOA0GBH tag=872U1B8Q54CA type=del 2024-09-13T00:00:49.569 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.570+0000 7fb8f2ce0640 1 > prepared operation key=PVYZQLGYT3XU tag=7UQIKNVKD3QP type=del size=2314736 2024-09-13T00:00:49.575 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.574+0000 7fb8f2ce0640 1 < completed part key=_multipart_QMZREZ1QEDNJ.2~O6HUPAELPRO2.0 size=570930 2024-09-13T00:00:49.585 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.582+0000 7fb8f2ce0640 1 < completed multipart upload key=QMZREZ1QEDNJ upload=O6HUPAELPRO2 tag=YQ79MYPICP4H type=write size=1619506 2024-09-13T00:00:49.590 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.590+0000 7fb8f2ce0640 1 > prepared operation key=FX0NCGLK0KPJ tag=3AF9UBF8ZC23 type=write size=985727 2024-09-13T00:00:49.601 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.602+0000 7fb8f2ce0640 1 < suggested update operation key=0WC6P8YNL0H2 tag=E5A8CV2E3T7I type=write size=781724 2024-09-13T00:00:49.607 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.606+0000 7fb8f2ce0640 1 > prepared operation key=44EDYFT9VF8Y tag=TM3Q2C8EFX20 type=del size=308985 2024-09-13T00:00:49.613 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.614+0000 7fb8f2ce0640 1 < completed delete operation key=EJN6HR8I7XQF tag=H4WKNY8P0XHR type=del 2024-09-13T00:00:49.618 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.618+0000 7fb8f2ce0640 1 > prepared operation key=2AHYRZUDFHP4 tag=M8JRJ9Y03L1U type=del size=3547219 2024-09-13T00:00:49.625 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.622+0000 7fb8f2ce0640 1 < completed part key=_multipart_70F7D20HI504.2~DLRFZ98I4980.0 size=1048576 2024-09-13T00:00:49.629 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.630+0000 7fb8f2ce0640 1 < completed part key=_multipart_70F7D20HI504.2~DLRFZ98I4980.1 size=559203 2024-09-13T00:00:49.639 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.638+0000 7fb8f2ce0640 1 < completed multipart upload key=70F7D20HI504 upload=DLRFZ98I4980 tag=W6K2OKGL0OKE type=write size=2656355 2024-09-13T00:00:49.645 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.646+0000 7fb8f2ce0640 1 > prepared operation key=NQJQC8CEAC15 tag=3F5S58J2JK4I type=del size=1669100 2024-09-13T00:00:49.657 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.658+0000 7fb8f2ce0640 1 < suggested update operation key=H5EIP0E9CBWX tag=5L8H5CQF8P0X type=write size=958188 2024-09-13T00:00:49.663 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.662+0000 7fb8f2ce0640 1 > prepared operation key=ZTK9AYBSLCOZ tag=03ZE0WWQU4K9 type=write size=173022 2024-09-13T00:00:49.668 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.666+0000 7fb8f2ce0640 1 < completed delete operation key=P8OYS5ZE1NX0 tag=214WPD39S1Z1 type=del 2024-09-13T00:00:49.678 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.674+0000 7fb8f2ce0640 1 > prepared operation key=1PX7DBO9M3YI tag=4GCMDHUGXMM2 type=del size=1064960 2024-09-13T00:00:49.678 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.674+0000 7fb8f2ce0640 1 < canceled part key=_multipart_EUE3LT0PDIXV.2~ECQ7T61YZI9I.0 size=1048576 2024-09-13T00:00:49.680 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.678+0000 7fb8f2ce0640 1 < completed part key=_multipart_EUE3LT0PDIXV.2~ECQ7T61YZI9I.1 size=145697 2024-09-13T00:00:49.690 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.690+0000 7fb8f2ce0640 1 < canceled multipart upload key=EUE3LT0PDIXV upload=ECQ7T61YZI9I tag=9S5SAR0Q6EXQ type=write size=2242849 2024-09-13T00:00:49.700 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.698+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~EQGGGP314YB0.0 size=1048576 2024-09-13T00:00:49.705 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.706+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~EQGGGP314YB0.1 size=692525 2024-09-13T00:00:49.705 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.706+0000 7fb8f2ce0640 1 > prepared multipart upload key=W8XZ26311RJX upload=EQGGGP314YB0 tag=BWGOJFCNVV1O type=write size=2789677 2024-09-13T00:00:49.712 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.710+0000 7fb8f2ce0640 1 < completed part key=_multipart_63FJDWBO2T9S.2~E4GVLOB4N0U4.0 size=663217 2024-09-13T00:00:49.722 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.722+0000 7fb8f2ce0640 1 < completed multipart upload key=63FJDWBO2T9S upload=E4GVLOB4N0U4 tag=7J87MAG2Z2IF type=write size=1711793 2024-09-13T00:00:49.727 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.726+0000 7fb8f2ce0640 1 > prepared operation key=O4MZ4UWNXME9 tag=9HXFP7SF109W type=del size=980061 2024-09-13T00:00:49.733 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.734+0000 7fb8f2ce0640 1 < completed part key=_multipart_8Z6U7OOA0GBH.2~OBKKLL4H5USH.0 size=1048576 2024-09-13T00:00:49.738 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.738+0000 7fb8f2ce0640 1 < completed part key=_multipart_8Z6U7OOA0GBH.2~OBKKLL4H5USH.1 size=63318 2024-09-13T00:00:49.748 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.746+0000 7fb8f2ce0640 1 < completed multipart upload key=8Z6U7OOA0GBH upload=OBKKLL4H5USH tag=6O0O46MR8W1B type=write size=2160470 2024-09-13T00:00:49.753 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.754+0000 7fb8f2ce0640 1 > prepared operation key=7OX5OJZ8GXU5 tag=XQ0HAHTG5EYK type=write size=866791 2024-09-13T00:00:49.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:00:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:00:49] "GET /metrics HTTP/1.1" 200 36472 "" "Prometheus/2.43.0" 2024-09-13T00:00:49.759 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.758+0000 7fb8f2ce0640 1 < completed part key=_multipart_H4M1TCN5W6XU.2~OKBCCAY5G0ZP.0 size=808539 2024-09-13T00:00:49.769 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.770+0000 7fb8f2ce0640 1 < completed multipart upload key=H4M1TCN5W6XU upload=OKBCCAY5G0ZP tag=4AT0HY1Y1XN9 type=write size=1857115 2024-09-13T00:00:49.775 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.774+0000 7fb8f2ce0640 1 > prepared operation key=NQJQC8CEAC15 tag=FK75MB9GT445 type=del size=2304510 2024-09-13T00:00:49.781 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.778+0000 7fb8f2ce0640 1 < completed part key=_multipart_63FJDWBO2T9S.2~5GNMJK2L2XYJ.0 size=1048576 2024-09-13T00:00:49.785 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.786+0000 7fb8f2ce0640 1 < completed part key=_multipart_63FJDWBO2T9S.2~5GNMJK2L2XYJ.1 size=1048576 2024-09-13T00:00:49.785 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.786+0000 7fb8f2ce0640 1 < canceled part key=_multipart_63FJDWBO2T9S.2~5GNMJK2L2XYJ.2 size=337307 2024-09-13T00:00:49.795 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.794+0000 7fb8f2ce0640 1 < canceled multipart upload key=63FJDWBO2T9S upload=5GNMJK2L2XYJ tag=5IMQ4TYVPMNL type=write size=3483035 2024-09-13T00:00:49.806 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.806+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~8VOVC265BE26.0 size=1048576 2024-09-13T00:00:49.811 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.810+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~8VOVC265BE26.1 size=147210 2024-09-13T00:00:49.811 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.810+0000 7fb8f2ce0640 1 > prepared multipart upload key=8MPUVD37EDN9 upload=8VOVC265BE26 tag=JP9GSNO8G821 type=write size=2244362 2024-09-13T00:00:49.816 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.814+0000 7fb8f2ce0640 1 < completed part key=_multipart_0WC6P8YNL0H2.2~0SQ2WZEVVG3X.0 size=1048576 2024-09-13T00:00:49.822 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.822+0000 7fb8f2ce0640 1 < completed part key=_multipart_0WC6P8YNL0H2.2~0SQ2WZEVVG3X.1 size=1048576 2024-09-13T00:00:49.827 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.826+0000 7fb8f2ce0640 1 < completed part key=_multipart_0WC6P8YNL0H2.2~0SQ2WZEVVG3X.2 size=311504 2024-09-13T00:00:49.837 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.838+0000 7fb8f2ce0640 1 < completed multipart upload key=0WC6P8YNL0H2 upload=0SQ2WZEVVG3X tag=VT26PT7UHAL3 type=write size=3457232 2024-09-13T00:00:49.848 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.846+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H4M1TCN5W6XU.2~I1XJV1GH4U9D.0 size=766014 2024-09-13T00:00:49.848 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.846+0000 7fb8f2ce0640 1 > prepared multipart upload key=H4M1TCN5W6XU upload=I1XJV1GH4U9D tag=SQ1HP5ER3E2Z type=write size=1814590 2024-09-13T00:00:49.853 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.854+0000 7fb8f2ce0640 1 < canceled operation key=8Z6U7OOA0GBH tag=A7KMFIEPGMMD type=write size=843276 2024-09-13T00:00:49.859 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.858+0000 7fb8f2ce0640 1 > prepared operation key=8Z6U7OOA0GBH tag=YD56F9LO6AER type=del size=2854854 2024-09-13T00:00:49.866 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.866+0000 7fb8f2ce0640 1 < completed delete operation key=QMZREZ1QEDNJ tag=K1BDN2INO11F type=del 2024-09-13T00:00:49.871 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.870+0000 7fb8f2ce0640 1 > prepared operation key=1PX7DBO9M3YI tag=ZO6YDWDRDUGU type=del size=4168478 2024-09-13T00:00:49.877 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.878+0000 7fb8f2ce0640 1 < completed delete operation key=8MPUVD37EDN9 tag=LXOYQFA9Q31D type=del 2024-09-13T00:00:49.888 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.886+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~BBQATWYF54A3.0 size=518019 2024-09-13T00:00:49.888 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.886+0000 7fb8f2ce0640 1 > prepared multipart upload key=PVYZQLGYT3XU upload=BBQATWYF54A3 tag=315ZXDHAXBNK type=write size=1566595 2024-09-13T00:00:49.899 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.898+0000 7fb8f2ce0640 1 < suggested remove operation key=PVYZQLGYT3XU tag=7UQIKNVKD3QP type=del 2024-09-13T00:00:49.906 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.906+0000 7fb8f2ce0640 1 > prepared operation key=EUE3LT0PDIXV tag=7N0CQUEL69SR type=del size=1655197 2024-09-13T00:00:49.912 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.910+0000 7fb8f2ce0640 1 < completed write operation key=FX0NCGLK0KPJ tag=3AF9UBF8ZC23 type=write size=985727 2024-09-13T00:00:49.923 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.922+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~HOG0CLACGC8I.0 size=1048576 2024-09-13T00:00:49.928 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.926+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~HOG0CLACGC8I.1 size=454439 2024-09-13T00:00:49.928 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.926+0000 7fb8f2ce0640 1 > prepared multipart upload key=8MPUVD37EDN9 upload=HOG0CLACGC8I tag=AICRJHW4PKJI type=write size=2551591 2024-09-13T00:00:49.933 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.934+0000 7fb8f2ce0640 1 < completed delete operation key=44EDYFT9VF8Y tag=TM3Q2C8EFX20 type=del 2024-09-13T00:00:49.939 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.938+0000 7fb8f2ce0640 1 > prepared operation key=J8NGDSF9XRTP tag=44E3E7G9LWW2 type=del size=1598094 2024-09-13T00:00:49.951 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.950+0000 7fb8f2ce0640 1 < suggested remove operation key=2AHYRZUDFHP4 tag=M8JRJ9Y03L1U type=del 2024-09-13T00:00:49.957 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.958+0000 7fb8f2ce0640 1 > prepared operation key=85YEW8N43FTA tag=G184SJF7D9QK type=del size=1076551 2024-09-13T00:00:49.963 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.962+0000 7fb8f2ce0640 1 < completed delete operation key=NQJQC8CEAC15 tag=3F5S58J2JK4I type=del 2024-09-13T00:00:49.969 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.966+0000 7fb8f2ce0640 1 > prepared operation key=0WC6P8YNL0H2 tag=BIUX98Z9LPNG type=del size=1011603 2024-09-13T00:00:49.975 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.974+0000 7fb8f2ce0640 1 < completed write operation key=ZTK9AYBSLCOZ tag=03ZE0WWQU4K9 type=write size=173022 2024-09-13T00:00:49.987 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.986+0000 7fb8f2ce0640 1 > prepared part key=_multipart_N3Z5XIG3TZRQ.2~ZFW5T7NO35YA.0 size=1048576 2024-09-13T00:00:49.992 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.990+0000 7fb8f2ce0640 1 > prepared part key=_multipart_N3Z5XIG3TZRQ.2~ZFW5T7NO35YA.1 size=836691 2024-09-13T00:00:49.992 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:49.990+0000 7fb8f2ce0640 1 > prepared multipart upload key=N3Z5XIG3TZRQ upload=ZFW5T7NO35YA tag=JXALA4PYHPY3 type=write size=2933843 2024-09-13T00:00:50.005 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.006+0000 7fb8f2ce0640 1 < suggested remove operation key=1PX7DBO9M3YI tag=4GCMDHUGXMM2 type=del 2024-09-13T00:00:50.016 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.014+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H5EIP0E9CBWX.2~3HQH40V4Y44Q.0 size=1025028 2024-09-13T00:00:50.016 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.014+0000 7fb8f2ce0640 1 > prepared multipart upload key=H5EIP0E9CBWX upload=3HQH40V4Y44Q tag=KB0VV1JGIRZC type=write size=2073604 2024-09-13T00:00:50.022 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.022+0000 7fb8f2ce0640 1 < completed part key=_multipart_W8XZ26311RJX.2~EQGGGP314YB0.0 size=1048576 2024-09-13T00:00:50.027 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.026+0000 7fb8f2ce0640 1 < completed part key=_multipart_W8XZ26311RJX.2~EQGGGP314YB0.1 size=692525 2024-09-13T00:00:50.036 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.034+0000 7fb8f2ce0640 1 < completed multipart upload key=W8XZ26311RJX upload=EQGGGP314YB0 tag=BWGOJFCNVV1O type=write size=2789677 2024-09-13T00:00:50.041 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.042+0000 7fb8f2ce0640 1 > prepared operation key=PVYZQLGYT3XU tag=FKZJQZ926KAX type=del size=525832 2024-09-13T00:00:50.047 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.046+0000 7fb8f2ce0640 1 < completed delete operation key=O4MZ4UWNXME9 tag=9HXFP7SF109W type=del 2024-09-13T00:00:50.056 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.054+0000 7fb8f2ce0640 1 > prepared part key=_multipart_5GM88ERM89KT.2~85HN9ZHDBT37.0 size=1048576 2024-09-13T00:00:50.061 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.062+0000 7fb8f2ce0640 1 > prepared part key=_multipart_5GM88ERM89KT.2~85HN9ZHDBT37.1 size=382134 2024-09-13T00:00:50.061 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.062+0000 7fb8f2ce0640 1 > prepared multipart upload key=5GM88ERM89KT upload=85HN9ZHDBT37 tag=NUKS15WS0FSV type=write size=2479286 2024-09-13T00:00:50.067 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.066+0000 7fb8f2ce0640 1 < completed write operation key=7OX5OJZ8GXU5 tag=XQ0HAHTG5EYK type=write size=866791 2024-09-13T00:00:50.072 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.070+0000 7fb8f2ce0640 1 > prepared operation key=P8OYS5ZE1NX0 tag=NJNR5W1X3P6Q type=del size=2268481 2024-09-13T00:00:50.085 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.082+0000 7fb8f2ce0640 1 < suggested remove operation key=NQJQC8CEAC15 tag=FK75MB9GT445 type=del 2024-09-13T00:00:50.094 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.094+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H4M1TCN5W6XU.2~WU72J1E3G1I6.0 size=1048576 2024-09-13T00:00:50.099 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.098+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H4M1TCN5W6XU.2~WU72J1E3G1I6.1 size=494276 2024-09-13T00:00:50.099 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.098+0000 7fb8f2ce0640 1 > prepared multipart upload key=H4M1TCN5W6XU upload=WU72J1E3G1I6 tag=Q7D3MVZVRQ72 type=write size=2591428 2024-09-13T00:00:50.104 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.102+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~8VOVC265BE26.0 size=1048576 2024-09-13T00:00:50.108 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.106+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~8VOVC265BE26.1 size=147210 2024-09-13T00:00:50.118 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.118+0000 7fb8f2ce0640 1 < completed multipart upload key=8MPUVD37EDN9 upload=8VOVC265BE26 tag=JP9GSNO8G821 type=write size=2244362 2024-09-13T00:00:50.123 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.122+0000 7fb8f2ce0640 1 > prepared operation key=5GM88ERM89KT tag=G7E0L7XFJGDO type=write size=972183 2024-09-13T00:00:50.125 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.122+0000 7fb8f2ce0640 1 < canceled part key=_multipart_H4M1TCN5W6XU.2~I1XJV1GH4U9D.0 size=766014 2024-09-13T00:00:50.133 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.134+0000 7fb8f2ce0640 1 < canceled multipart upload key=H4M1TCN5W6XU upload=I1XJV1GH4U9D tag=SQ1HP5ER3E2Z type=write size=1814590 2024-09-13T00:00:50.139 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.138+0000 7fb8f2ce0640 1 > prepared operation key=H5EIP0E9CBWX tag=B0X6IH3K0ORJ type=write size=345632 2024-09-13T00:00:50.149 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.150+0000 7fb8f2ce0640 1 < suggested remove operation key=8Z6U7OOA0GBH tag=YD56F9LO6AER type=del 2024-09-13T00:00:50.155 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.154+0000 7fb8f2ce0640 1 > prepared operation key=0WC6P8YNL0H2 tag=GVFV08OMYXR7 type=del size=2650833 2024-09-13T00:00:50.163 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.162+0000 7fb8f2ce0640 -1 < failed to complete operation key=1PX7DBO9M3YI tag=ZO6YDWDRDUGU type=del size=4168478: (22) Invalid argument 2024-09-13T00:00:50.169 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.166+0000 7fb8f2ce0640 1 > prepared operation key=J8NGDSF9XRTP tag=YSBT33FZL63A type=del size=1016980 2024-09-13T00:00:50.174 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.174+0000 7fb8f2ce0640 1 < completed part key=_multipart_PVYZQLGYT3XU.2~BBQATWYF54A3.0 size=518019 2024-09-13T00:00:50.183 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.182+0000 7fb8f2ce0640 1 < completed multipart upload key=PVYZQLGYT3XU upload=BBQATWYF54A3 tag=315ZXDHAXBNK type=write size=1566595 2024-09-13T00:00:50.189 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.190+0000 7fb8f2ce0640 1 > prepared operation key=O4MZ4UWNXME9 tag=YO0MMRM0VV0J type=del size=2289894 2024-09-13T00:00:50.195 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.194+0000 7fb8f2ce0640 1 < completed delete operation key=EUE3LT0PDIXV tag=7N0CQUEL69SR type=del 2024-09-13T00:00:50.202 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.202+0000 7fb8f2ce0640 1 > prepared operation key=EUE3LT0PDIXV tag=AJ27LNFBCBDP type=del size=4076111 2024-09-13T00:00:50.207 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.206+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~HOG0CLACGC8I.0 size=1048576 2024-09-13T00:00:50.212 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.210+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~HOG0CLACGC8I.1 size=454439 2024-09-13T00:00:50.222 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.222+0000 7fb8f2ce0640 1 < completed multipart upload key=8MPUVD37EDN9 upload=HOG0CLACGC8I tag=AICRJHW4PKJI type=write size=2551591 2024-09-13T00:00:50.228 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.226+0000 7fb8f2ce0640 1 > prepared operation key=4K76U69GZ9AE tag=1OE6R4XESPHO type=del size=2322835 2024-09-13T00:00:50.234 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.234+0000 7fb8f2ce0640 1 < completed delete operation key=J8NGDSF9XRTP tag=44E3E7G9LWW2 type=del 2024-09-13T00:00:50.240 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.238+0000 7fb8f2ce0640 1 > prepared operation key=63FJDWBO2T9S tag=7DS4FM0JGTOC type=del size=2340610 2024-09-13T00:00:50.245 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.246+0000 7fb8f2ce0640 1 < completed delete operation key=85YEW8N43FTA tag=G184SJF7D9QK type=del 2024-09-13T00:00:50.258 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.258+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EUE3LT0PDIXV.2~3FF9VATD0GUD.0 size=1004201 2024-09-13T00:00:50.259 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.258+0000 7fb8f2ce0640 1 > prepared multipart upload key=EUE3LT0PDIXV upload=3FF9VATD0GUD tag=7OZ37FSE58Z1 type=write size=2052777 2024-09-13T00:00:50.265 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.262+0000 7fb8f2ce0640 1 < completed delete operation key=0WC6P8YNL0H2 tag=BIUX98Z9LPNG type=del 2024-09-13T00:00:50.270 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.270+0000 7fb8f2ce0640 1 > prepared operation key=63FJDWBO2T9S tag=6UA81C4MX6WK type=del size=2244121 2024-09-13T00:00:50.277 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.278+0000 7fb8f2ce0640 1 < completed part key=_multipart_N3Z5XIG3TZRQ.2~ZFW5T7NO35YA.0 size=1048576 2024-09-13T00:00:50.282 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.282+0000 7fb8f2ce0640 1 < completed part key=_multipart_N3Z5XIG3TZRQ.2~ZFW5T7NO35YA.1 size=836691 2024-09-13T00:00:50.291 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.290+0000 7fb8f2ce0640 1 < completed multipart upload key=N3Z5XIG3TZRQ upload=ZFW5T7NO35YA tag=JXALA4PYHPY3 type=write size=2933843 2024-09-13T00:00:50.298 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.298+0000 7fb8f2ce0640 1 > prepared operation key=8MPUVD37EDN9 tag=VWN515GAXY2G type=del size=3728465 2024-09-13T00:00:50.304 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.302+0000 7fb8f2ce0640 1 < completed part key=_multipart_H5EIP0E9CBWX.2~3HQH40V4Y44Q.0 size=1025028 2024-09-13T00:00:50.313 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.314+0000 7fb8f2ce0640 1 < completed multipart upload key=H5EIP0E9CBWX upload=3HQH40V4Y44Q tag=KB0VV1JGIRZC type=write size=2073604 2024-09-13T00:00:50.323 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.322+0000 7fb8f2ce0640 1 > prepared part key=_multipart_0WC6P8YNL0H2.2~RHDZQY31OCQB.0 size=1048576 2024-09-13T00:00:50.328 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.326+0000 7fb8f2ce0640 1 > prepared part key=_multipart_0WC6P8YNL0H2.2~RHDZQY31OCQB.1 size=952805 2024-09-13T00:00:50.329 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.326+0000 7fb8f2ce0640 1 > prepared multipart upload key=0WC6P8YNL0H2 upload=RHDZQY31OCQB tag=PP1KNFGDD3YB type=write size=3049957 2024-09-13T00:00:50.334 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.334+0000 7fb8f2ce0640 1 < completed delete operation key=PVYZQLGYT3XU tag=FKZJQZ926KAX type=del 2024-09-13T00:00:50.345 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.346+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NX26SI4F2ZMB.2~793G8ZRWNHNI.0 size=912253 2024-09-13T00:00:50.345 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.346+0000 7fb8f2ce0640 1 > prepared multipart upload key=NX26SI4F2ZMB upload=793G8ZRWNHNI tag=1XUUWPVO9YIU type=write size=1960829 2024-09-13T00:00:50.351 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.350+0000 7fb8f2ce0640 1 < completed part key=_multipart_5GM88ERM89KT.2~85HN9ZHDBT37.0 size=1048576 2024-09-13T00:00:50.355 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.354+0000 7fb8f2ce0640 1 < completed part key=_multipart_5GM88ERM89KT.2~85HN9ZHDBT37.1 size=382134 2024-09-13T00:00:50.365 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.366+0000 7fb8f2ce0640 1 < completed multipart upload key=5GM88ERM89KT upload=85HN9ZHDBT37 tag=NUKS15WS0FSV type=write size=2479286 2024-09-13T00:00:50.371 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.370+0000 7fb8f2ce0640 1 > prepared operation key=ZFHN2SNTZ5W7 tag=9NASZMFZ2T5K type=del size=3730634 2024-09-13T00:00:50.377 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.378+0000 7fb8f2ce0640 1 < completed delete operation key=P8OYS5ZE1NX0 tag=NJNR5W1X3P6Q type=del 2024-09-13T00:00:50.382 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.382+0000 7fb8f2ce0640 1 > prepared operation key=EJN6HR8I7XQF tag=SO2JX0LR5ME8 type=del size=400055 2024-09-13T00:00:50.388 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.386+0000 7fb8f2ce0640 1 < completed part key=_multipart_H4M1TCN5W6XU.2~WU72J1E3G1I6.0 size=1048576 2024-09-13T00:00:50.393 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.394+0000 7fb8f2ce0640 1 < completed part key=_multipart_H4M1TCN5W6XU.2~WU72J1E3G1I6.1 size=494276 2024-09-13T00:00:50.403 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.402+0000 7fb8f2ce0640 1 < completed multipart upload key=H4M1TCN5W6XU upload=WU72J1E3G1I6 tag=Q7D3MVZVRQ72 type=write size=2591428 2024-09-13T00:00:50.408 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.406+0000 7fb8f2ce0640 1 > prepared operation key=PVYZQLGYT3XU tag=YG7WBBWSZOAU type=del size=2372486 2024-09-13T00:00:50.419 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.418+0000 7fb8f2ce0640 1 < suggested remove operation key=5GM88ERM89KT tag=G7E0L7XFJGDO type=write 2024-09-13T00:00:50.425 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.426+0000 7fb8f2ce0640 1 > prepared operation key=70F7D20HI504 tag=LMQ4L8REGGT8 type=del size=3526343 2024-09-13T00:00:50.431 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.430+0000 7fb8f2ce0640 1 < completed write operation key=H5EIP0E9CBWX tag=B0X6IH3K0ORJ type=write size=345632 2024-09-13T00:00:50.437 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.434+0000 7fb8f2ce0640 1 > prepared operation key=ZFHN2SNTZ5W7 tag=0OFY203DOTJV type=write size=833676 2024-09-13T00:00:50.443 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.442+0000 7fb8f2ce0640 1 < canceled operation key=0WC6P8YNL0H2 tag=GVFV08OMYXR7 type=del size=2650833 2024-09-13T00:00:50.449 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.450+0000 7fb8f2ce0640 1 > prepared operation key=ZFHN2SNTZ5W7 tag=W6ACQJ7D6PIK type=del size=3296728 2024-09-13T00:00:50.456 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.454+0000 7fb8f2ce0640 1 < completed delete operation key=J8NGDSF9XRTP tag=YSBT33FZL63A type=del 2024-09-13T00:00:50.467 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.466+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~LLXD3D470B8W.0 size=1048576 2024-09-13T00:00:50.473 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.474+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~LLXD3D470B8W.1 size=1048576 2024-09-13T00:00:50.478 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.478+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~LLXD3D470B8W.2 size=728961 2024-09-13T00:00:50.478 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.478+0000 7fb8f2ce0640 1 > prepared multipart upload key=PVYZQLGYT3XU upload=LLXD3D470B8W tag=V8GMXLHGMYJA type=write size=3874689 2024-09-13T00:00:50.484 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.482+0000 7fb8f2ce0640 1 < completed delete operation key=O4MZ4UWNXME9 tag=YO0MMRM0VV0J type=del 2024-09-13T00:00:50.496 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.494+0000 7fb8f2ce0640 1 > prepared part key=_multipart_44EDYFT9VF8Y.2~698NX9V07CZB.0 size=1048576 2024-09-13T00:00:50.499 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.498+0000 7fb8f2ce0640 1 > prepared part key=_multipart_44EDYFT9VF8Y.2~698NX9V07CZB.1 size=1048576 2024-09-13T00:00:50.505 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.502+0000 7fb8f2ce0640 1 > prepared part key=_multipart_44EDYFT9VF8Y.2~698NX9V07CZB.2 size=798518 2024-09-13T00:00:50.505 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.502+0000 7fb8f2ce0640 1 > prepared multipart upload key=44EDYFT9VF8Y upload=698NX9V07CZB tag=I8PIQBEYSUG1 type=write size=3944246 2024-09-13T00:00:50.517 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.518+0000 7fb8f2ce0640 1 < suggested update operation key=EUE3LT0PDIXV tag=AJ27LNFBCBDP type=del size=4076111 2024-09-13T00:00:50.523 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.522+0000 7fb8f2ce0640 1 > prepared operation key=F0QCVR9E08WE tag=ESFYK11Z7H8C type=del size=3002094 2024-09-13T00:00:50.535 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.534+0000 7fb8f2ce0640 1 < suggested update operation key=4K76U69GZ9AE tag=1OE6R4XESPHO type=del size=2322835 2024-09-13T00:00:50.541 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.542+0000 7fb8f2ce0640 1 > prepared operation key=8MPUVD37EDN9 tag=Z331UDRHTABQ type=del size=2586921 2024-09-13T00:00:50.548 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.546+0000 7fb8f2ce0640 1 < completed delete operation key=63FJDWBO2T9S tag=7DS4FM0JGTOC type=del 2024-09-13T00:00:50.554 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.554+0000 7fb8f2ce0640 1 > prepared operation key=W8XZ26311RJX tag=03J795KYDXKO type=del size=486858 2024-09-13T00:00:50.555 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.554+0000 7fb8f2ce0640 1 < canceled part key=_multipart_EUE3LT0PDIXV.2~3FF9VATD0GUD.0 size=1004201 2024-09-13T00:00:50.566 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.566+0000 7fb8f2ce0640 1 < canceled multipart upload key=EUE3LT0PDIXV upload=3FF9VATD0GUD tag=7OZ37FSE58Z1 type=write size=2052777 2024-09-13T00:00:50.572 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.570+0000 7fb8f2ce0640 1 > prepared operation key=FX0NCGLK0KPJ tag=GKYQ9DPB1Y5E type=del size=3858093 2024-09-13T00:00:50.579 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.578+0000 7fb8f2ce0640 1 < completed delete operation key=63FJDWBO2T9S tag=6UA81C4MX6WK type=del 2024-09-13T00:00:50.592 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.590+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~B0KIBK82Q46O.0 size=808991 2024-09-13T00:00:50.592 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.590+0000 7fb8f2ce0640 1 > prepared multipart upload key=NQJQC8CEAC15 upload=B0KIBK82Q46O tag=81LYUOE9CQWA type=write size=1857567 2024-09-13T00:00:50.598 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.598+0000 7fb8f2ce0640 1 < completed delete operation key=8MPUVD37EDN9 tag=VWN515GAXY2G type=del 2024-09-13T00:00:50.609 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.610+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~2S4S7SG4DCN0.0 size=1048576 2024-09-13T00:00:50.614 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.614+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~2S4S7SG4DCN0.1 size=939962 2024-09-13T00:00:50.615 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.614+0000 7fb8f2ce0640 1 > prepared multipart upload key=8MPUVD37EDN9 upload=2S4S7SG4DCN0 tag=SI3F3810L42E type=write size=3037114 2024-09-13T00:00:50.620 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.618+0000 7fb8f2ce0640 1 < completed part key=_multipart_0WC6P8YNL0H2.2~RHDZQY31OCQB.0 size=1048576 2024-09-13T00:00:50.625 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.626+0000 7fb8f2ce0640 1 < completed part key=_multipart_0WC6P8YNL0H2.2~RHDZQY31OCQB.1 size=952805 2024-09-13T00:00:50.634 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.634+0000 7fb8f2ce0640 1 < completed multipart upload key=0WC6P8YNL0H2 upload=RHDZQY31OCQB tag=PP1KNFGDD3YB type=write size=3049957 2024-09-13T00:00:50.640 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.638+0000 7fb8f2ce0640 1 > prepared operation key=ZFHN2SNTZ5W7 tag=I1ZT27Z0EM2R type=del size=433213 2024-09-13T00:00:50.646 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.646+0000 7fb8f2ce0640 1 < completed part key=_multipart_NX26SI4F2ZMB.2~793G8ZRWNHNI.0 size=912253 2024-09-13T00:00:50.656 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.654+0000 7fb8f2ce0640 1 < completed multipart upload key=NX26SI4F2ZMB upload=793G8ZRWNHNI tag=1XUUWPVO9YIU type=write size=1960829 2024-09-13T00:00:50.661 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.662+0000 7fb8f2ce0640 1 > prepared operation key=QMZREZ1QEDNJ tag=3X62C02ZJIHG type=write size=994244 2024-09-13T00:00:50.673 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.674+0000 7fb8f2ce0640 1 < suggested remove operation key=ZFHN2SNTZ5W7 tag=9NASZMFZ2T5K type=del 2024-09-13T00:00:50.680 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.678+0000 7fb8f2ce0640 1 > prepared operation key=N3Z5XIG3TZRQ tag=FKR0WCGVXIHU type=del size=650850 2024-09-13T00:00:50.686 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.686+0000 7fb8f2ce0640 1 < completed delete operation key=EJN6HR8I7XQF tag=SO2JX0LR5ME8 type=del 2024-09-13T00:00:50.697 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.698+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~XMWF3ZIKJ61J.0 size=1048576 2024-09-13T00:00:50.702 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.702+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~XMWF3ZIKJ61J.1 size=1048576 2024-09-13T00:00:50.707 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.706+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~XMWF3ZIKJ61J.2 size=1000327 2024-09-13T00:00:50.707 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.706+0000 7fb8f2ce0640 1 > prepared multipart upload key=FX0NCGLK0KPJ upload=XMWF3ZIKJ61J tag=B0I8JCEVS32J type=write size=4146055 2024-09-13T00:00:50.721 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.722+0000 7fb8f2ce0640 1 < suggested remove operation key=PVYZQLGYT3XU tag=YG7WBBWSZOAU type=del 2024-09-13T00:00:50.727 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.726+0000 7fb8f2ce0640 1 > prepared operation key=H4M1TCN5W6XU tag=7U84F3XSQKYM type=write size=609598 2024-09-13T00:00:50.733 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.734+0000 7fb8f2ce0640 1 < completed delete operation key=70F7D20HI504 tag=LMQ4L8REGGT8 type=del 2024-09-13T00:00:50.738 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.738+0000 7fb8f2ce0640 1 > prepared operation key=4K76U69GZ9AE tag=4X14GJUEBPYD type=del size=1279499 2024-09-13T00:00:50.747 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.746+0000 7fb8f2ce0640 -1 < failed to complete operation key=ZFHN2SNTZ5W7 tag=0OFY203DOTJV type=write size=833676: (22) Invalid argument 2024-09-13T00:00:50.758 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.758+0000 7fb8f2ce0640 1 > prepared part key=_multipart_4K76U69GZ9AE.2~PGOK53PQRX83.0 size=1048576 2024-09-13T00:00:50.762 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.762+0000 7fb8f2ce0640 1 > prepared part key=_multipart_4K76U69GZ9AE.2~PGOK53PQRX83.1 size=925560 2024-09-13T00:00:50.763 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.762+0000 7fb8f2ce0640 1 > prepared multipart upload key=4K76U69GZ9AE upload=PGOK53PQRX83 tag=UM9R8071PPA1 type=write size=3022712 2024-09-13T00:00:50.770 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.770+0000 7fb8f2ce0640 -1 < failed to complete operation key=ZFHN2SNTZ5W7 tag=W6ACQJ7D6PIK type=del size=3296728: (22) Invalid argument 2024-09-13T00:00:50.777 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.778+0000 7fb8f2ce0640 1 > prepared operation key=0WC6P8YNL0H2 tag=CRFV5R1D71YG type=del size=1005141 2024-09-13T00:00:50.784 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.782+0000 7fb8f2ce0640 1 < completed part key=_multipart_PVYZQLGYT3XU.2~LLXD3D470B8W.0 size=1048576 2024-09-13T00:00:50.789 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.790+0000 7fb8f2ce0640 1 < completed part key=_multipart_PVYZQLGYT3XU.2~LLXD3D470B8W.1 size=1048576 2024-09-13T00:00:50.793 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.794+0000 7fb8f2ce0640 1 < completed part key=_multipart_PVYZQLGYT3XU.2~LLXD3D470B8W.2 size=728961 2024-09-13T00:00:50.803 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.802+0000 7fb8f2ce0640 1 < completed multipart upload key=PVYZQLGYT3XU upload=LLXD3D470B8W tag=V8GMXLHGMYJA type=write size=3874689 2024-09-13T00:00:50.813 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.814+0000 7fb8f2ce0640 1 > prepared part key=_multipart_J8NGDSF9XRTP.2~A2K5D208CBPY.0 size=853596 2024-09-13T00:00:50.813 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.814+0000 7fb8f2ce0640 1 > prepared multipart upload key=J8NGDSF9XRTP upload=A2K5D208CBPY tag=WCJVAMOGU3CQ type=write size=1902172 2024-09-13T00:00:50.820 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.818+0000 7fb8f2ce0640 1 < completed part key=_multipart_44EDYFT9VF8Y.2~698NX9V07CZB.0 size=1048576 2024-09-13T00:00:50.824 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.822+0000 7fb8f2ce0640 1 < completed part key=_multipart_44EDYFT9VF8Y.2~698NX9V07CZB.1 size=1048576 2024-09-13T00:00:50.829 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.830+0000 7fb8f2ce0640 1 < completed part key=_multipart_44EDYFT9VF8Y.2~698NX9V07CZB.2 size=798518 2024-09-13T00:00:50.840 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.838+0000 7fb8f2ce0640 1 < completed multipart upload key=44EDYFT9VF8Y upload=698NX9V07CZB tag=I8PIQBEYSUG1 type=write size=3944246 2024-09-13T00:00:50.846 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.846+0000 7fb8f2ce0640 1 > prepared operation key=QMZREZ1QEDNJ tag=W5HPOIN2TJK1 type=del size=3771931 2024-09-13T00:00:50.852 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.850+0000 7fb8f2ce0640 1 < completed delete operation key=F0QCVR9E08WE tag=ESFYK11Z7H8C type=del 2024-09-13T00:00:50.859 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.858+0000 7fb8f2ce0640 1 > prepared operation key=H5EIP0E9CBWX tag=FB17CV0A5HA6 type=write size=103043 2024-09-13T00:00:50.865 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.866+0000 7fb8f2ce0640 1 < canceled operation key=8MPUVD37EDN9 tag=Z331UDRHTABQ type=del size=2586921 2024-09-13T00:00:50.870 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.870+0000 7fb8f2ce0640 1 > prepared operation key=2TWD9YLMSYJI tag=3LZV2KB07OYF type=del size=112462 2024-09-13T00:00:50.876 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.874+0000 7fb8f2ce0640 1 < completed delete operation key=W8XZ26311RJX tag=03J795KYDXKO type=del 2024-09-13T00:00:50.883 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.882+0000 7fb8f2ce0640 1 > prepared operation key=PVYZQLGYT3XU tag=NC4BQVL2EZJM type=del size=3978839 2024-09-13T00:00:50.889 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.890+0000 7fb8f2ce0640 1 < completed delete operation key=FX0NCGLK0KPJ tag=GKYQ9DPB1Y5E type=del 2024-09-13T00:00:50.895 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.894+0000 7fb8f2ce0640 1 > prepared operation key=QMZREZ1QEDNJ tag=V49E31CWULWB type=del size=3502919 2024-09-13T00:00:50.902 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.902+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~B0KIBK82Q46O.0 size=808991 2024-09-13T00:00:50.914 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.914+0000 7fb8f2ce0640 1 < completed multipart upload key=NQJQC8CEAC15 upload=B0KIBK82Q46O tag=81LYUOE9CQWA type=write size=1857567 2024-09-13T00:00:50.920 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.918+0000 7fb8f2ce0640 1 > prepared operation key=P8OYS5ZE1NX0 tag=W1JCOGSDM1HQ type=del size=4170303 2024-09-13T00:00:50.925 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.926+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~2S4S7SG4DCN0.0 size=1048576 2024-09-13T00:00:50.930 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.930+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~2S4S7SG4DCN0.1 size=939962 2024-09-13T00:00:50.940 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.938+0000 7fb8f2ce0640 1 < completed multipart upload key=8MPUVD37EDN9 upload=2S4S7SG4DCN0 tag=SI3F3810L42E type=write size=3037114 2024-09-13T00:00:50.945 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.946+0000 7fb8f2ce0640 1 > prepared operation key=70F7D20HI504 tag=ALRI14L7EBYI type=del size=502425 2024-09-13T00:00:50.953 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.954+0000 7fb8f2ce0640 -1 < failed to complete operation key=ZFHN2SNTZ5W7 tag=I1ZT27Z0EM2R type=del size=433213: (22) Invalid argument 2024-09-13T00:00:50.964 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.962+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~UK0X3PAQG5NU.0 size=255033 2024-09-13T00:00:50.964 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.962+0000 7fb8f2ce0640 1 > prepared multipart upload key=PVYZQLGYT3XU upload=UK0X3PAQG5NU tag=9FETDBHD4LTR type=write size=1303609 2024-09-13T00:00:50.970 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.970+0000 7fb8f2ce0640 1 < completed write operation key=QMZREZ1QEDNJ tag=3X62C02ZJIHG type=write size=994244 2024-09-13T00:00:50.977 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.974+0000 7fb8f2ce0640 1 > prepared operation key=8Z6U7OOA0GBH tag=NL96HZME6285 type=del size=2013336 2024-09-13T00:00:50.983 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.982+0000 7fb8f2ce0640 1 < completed delete operation key=N3Z5XIG3TZRQ tag=FKR0WCGVXIHU type=del 2024-09-13T00:00:50.993 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.994+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~U8C4I73UTQS6.0 size=529530 2024-09-13T00:00:50.994 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.994+0000 7fb8f2ce0640 1 > prepared multipart upload key=FX0NCGLK0KPJ upload=U8C4I73UTQS6 tag=PVSFQFB6WIJ7 type=write size=1578106 2024-09-13T00:00:51.000 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:50.998+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~XMWF3ZIKJ61J.0 size=1048576 2024-09-13T00:00:51.004 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.002+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~XMWF3ZIKJ61J.1 size=1048576 2024-09-13T00:00:51.009 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.010+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~XMWF3ZIKJ61J.2 size=1000327 2024-09-13T00:00:51.020 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.018+0000 7fb8f2ce0640 1 < completed multipart upload key=FX0NCGLK0KPJ upload=XMWF3ZIKJ61J tag=B0I8JCEVS32J type=write size=4146055 2024-09-13T00:00:51.031 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.030+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~A908FRBLP5ZH.0 size=1048576 2024-09-13T00:00:51.036 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.034+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~A908FRBLP5ZH.1 size=1048576 2024-09-13T00:00:51.041 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.042+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~A908FRBLP5ZH.2 size=212180 2024-09-13T00:00:51.041 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.042+0000 7fb8f2ce0640 1 > prepared multipart upload key=2AHYRZUDFHP4 upload=A908FRBLP5ZH tag=1GIJ740OZ12R type=write size=3357908 2024-09-13T00:00:51.053 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.054+0000 7fb8f2ce0640 1 < suggested update operation key=H4M1TCN5W6XU tag=7U84F3XSQKYM type=write size=609598 2024-09-13T00:00:51.059 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.058+0000 7fb8f2ce0640 1 > prepared operation key=2TWD9YLMSYJI tag=G6WDI4B5H6GR type=del size=2133704 2024-09-13T00:00:51.070 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.070+0000 7fb8f2ce0640 1 < suggested remove operation key=4K76U69GZ9AE tag=4X14GJUEBPYD type=del 2024-09-13T00:00:51.076 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.074+0000 7fb8f2ce0640 1 > prepared operation key=2TWD9YLMSYJI tag=8ZG5CP0ZFGZ7 type=del size=88794 2024-09-13T00:00:51.082 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.082+0000 7fb8f2ce0640 1 < completed part key=_multipart_4K76U69GZ9AE.2~PGOK53PQRX83.0 size=1048576 2024-09-13T00:00:51.087 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.086+0000 7fb8f2ce0640 1 < completed part key=_multipart_4K76U69GZ9AE.2~PGOK53PQRX83.1 size=925560 2024-09-13T00:00:51.097 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.098+0000 7fb8f2ce0640 1 < completed multipart upload key=4K76U69GZ9AE upload=PGOK53PQRX83 tag=UM9R8071PPA1 type=write size=3022712 2024-09-13T00:00:51.109 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.106+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~35J5SXGV4YMK.0 size=1048576 2024-09-13T00:00:51.112 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.110+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~35J5SXGV4YMK.1 size=954472 2024-09-13T00:00:51.112 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.110+0000 7fb8f2ce0640 1 > prepared multipart upload key=70F7D20HI504 upload=35J5SXGV4YMK tag=WPKDVLNP2MV1 type=write size=3051624 2024-09-13T00:00:51.123 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.122+0000 7fb8f2ce0640 1 < suggested remove operation key=0WC6P8YNL0H2 tag=CRFV5R1D71YG type=del 2024-09-13T00:00:51.129 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.130+0000 7fb8f2ce0640 1 > prepared operation key=J8NGDSF9XRTP tag=S2K0LECHI4JO type=del size=673924 2024-09-13T00:00:51.135 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.134+0000 7fb8f2ce0640 1 < completed part key=_multipart_J8NGDSF9XRTP.2~A2K5D208CBPY.0 size=853596 2024-09-13T00:00:51.145 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.142+0000 7fb8f2ce0640 1 < completed multipart upload key=J8NGDSF9XRTP upload=A2K5D208CBPY tag=WCJVAMOGU3CQ type=write size=1902172 2024-09-13T00:00:51.151 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.150+0000 7fb8f2ce0640 1 > prepared operation key=F0QCVR9E08WE tag=V02ZRRXHZ1C7 type=write size=535603 2024-09-13T00:00:51.156 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.154+0000 7fb8f2ce0640 1 < canceled operation key=QMZREZ1QEDNJ tag=W5HPOIN2TJK1 type=del size=3771931 2024-09-13T00:00:51.162 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.162+0000 7fb8f2ce0640 1 > prepared operation key=O4MZ4UWNXME9 tag=7RALTYOC2U8C type=del size=2240980 2024-09-13T00:00:51.175 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.174+0000 7fb8f2ce0640 1 < suggested remove operation key=H5EIP0E9CBWX tag=FB17CV0A5HA6 type=write 2024-09-13T00:00:51.182 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.182+0000 7fb8f2ce0640 1 > prepared operation key=2TWD9YLMSYJI tag=03UX9FJUI8BM type=del size=3305587 2024-09-13T00:00:51.188 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.186+0000 7fb8f2ce0640 1 < completed delete operation key=2TWD9YLMSYJI tag=3LZV2KB07OYF type=del 2024-09-13T00:00:51.200 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.198+0000 7fb8f2ce0640 1 > prepared part key=_multipart_5GM88ERM89KT.2~V8F1693I78HN.0 size=1048576 2024-09-13T00:00:51.204 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.202+0000 7fb8f2ce0640 1 > prepared part key=_multipart_5GM88ERM89KT.2~V8F1693I78HN.1 size=592618 2024-09-13T00:00:51.204 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.202+0000 7fb8f2ce0640 1 > prepared multipart upload key=5GM88ERM89KT upload=V8F1693I78HN tag=P4VRELZMJRN5 type=write size=2689770 2024-09-13T00:00:51.218 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.218+0000 7fb8f2ce0640 1 < suggested update operation key=PVYZQLGYT3XU tag=NC4BQVL2EZJM type=del size=3978839 2024-09-13T00:00:51.229 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.230+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EUE3LT0PDIXV.2~TXV4G5KETRYA.0 size=741447 2024-09-13T00:00:51.229 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.230+0000 7fb8f2ce0640 1 > prepared multipart upload key=EUE3LT0PDIXV upload=TXV4G5KETRYA tag=ZHY76DK063SF type=write size=1790023 2024-09-13T00:00:51.235 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.234+0000 7fb8f2ce0640 1 < completed delete operation key=QMZREZ1QEDNJ tag=V49E31CWULWB type=del 2024-09-13T00:00:51.245 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.246+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZFHN2SNTZ5W7.2~M95NWKC0LHL7.0 size=1048576 2024-09-13T00:00:51.250 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.250+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZFHN2SNTZ5W7.2~M95NWKC0LHL7.1 size=1048576 2024-09-13T00:00:51.254 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.254+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZFHN2SNTZ5W7.2~M95NWKC0LHL7.2 size=260446 2024-09-13T00:00:51.255 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.254+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZFHN2SNTZ5W7 upload=M95NWKC0LHL7 tag=3W4B5JU3G0UC type=write size=3406174 2024-09-13T00:00:51.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:50 smithi136 bash[75164]: cluster 2024-09-13T00:00:49.134282+0000 mgr.y (mgr.24661) 711 : cluster [DBG] pgmap v1387: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 501 KiB/s rd, 174 KiB/s wr, 885 op/s 2024-09-13T00:00:51.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:50 smithi136 bash[76371]: cluster 2024-09-13T00:00:49.134282+0000 mgr.y (mgr.24661) 711 : cluster [DBG] pgmap v1387: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 501 KiB/s rd, 174 KiB/s wr, 885 op/s 2024-09-13T00:00:51.265 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.266+0000 7fb8f2ce0640 1 < suggested remove operation key=P8OYS5ZE1NX0 tag=W1JCOGSDM1HQ type=del 2024-09-13T00:00:51.271 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.270+0000 7fb8f2ce0640 1 > prepared operation key=5GM88ERM89KT tag=MRWGOJ61ULTU type=del size=3412694 2024-09-13T00:00:51.281 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.282+0000 7fb8f2ce0640 1 < suggested remove operation key=70F7D20HI504 tag=ALRI14L7EBYI type=del 2024-09-13T00:00:51.287 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.286+0000 7fb8f2ce0640 1 > prepared operation key=O4MZ4UWNXME9 tag=YWXXJY2E2NTY type=del size=2080551 2024-09-13T00:00:51.294 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.294+0000 7fb8f2ce0640 1 < completed part key=_multipart_PVYZQLGYT3XU.2~UK0X3PAQG5NU.0 size=255033 2024-09-13T00:00:51.304 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.302+0000 7fb8f2ce0640 1 < completed multipart upload key=PVYZQLGYT3XU upload=UK0X3PAQG5NU tag=9FETDBHD4LTR type=write size=1303609 2024-09-13T00:00:51.316 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.314+0000 7fb8f2ce0640 1 > prepared part key=_multipart_P8OYS5ZE1NX0.2~6LLRS3HPUDG3.0 size=1048576 2024-09-13T00:00:51.321 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.322+0000 7fb8f2ce0640 1 > prepared part key=_multipart_P8OYS5ZE1NX0.2~6LLRS3HPUDG3.1 size=1048576 2024-09-13T00:00:51.325 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.326+0000 7fb8f2ce0640 1 > prepared part key=_multipart_P8OYS5ZE1NX0.2~6LLRS3HPUDG3.2 size=321499 2024-09-13T00:00:51.325 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.326+0000 7fb8f2ce0640 1 > prepared multipart upload key=P8OYS5ZE1NX0 upload=6LLRS3HPUDG3 tag=J9U89PQAFDKS type=write size=3467227 2024-09-13T00:00:51.330 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.330+0000 7fb8f2ce0640 1 < completed delete operation key=8Z6U7OOA0GBH tag=NL96HZME6285 type=del 2024-09-13T00:00:51.340 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.338+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~VVTNNEXF5023.0 size=1048576 2024-09-13T00:00:51.347 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.346+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~VVTNNEXF5023.1 size=1048576 2024-09-13T00:00:51.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:50 smithi181 bash[67587]: cluster 2024-09-13T00:00:49.134282+0000 mgr.y (mgr.24661) 711 : cluster [DBG] pgmap v1387: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 501 KiB/s rd, 174 KiB/s wr, 885 op/s 2024-09-13T00:00:51.352 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.350+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~VVTNNEXF5023.2 size=65822 2024-09-13T00:00:51.352 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.350+0000 7fb8f2ce0640 1 > prepared multipart upload key=63FJDWBO2T9S upload=VVTNNEXF5023 tag=D64C5DPL7SY4 type=write size=3211550 2024-09-13T00:00:51.357 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.358+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~U8C4I73UTQS6.0 size=529530 2024-09-13T00:00:51.368 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.366+0000 7fb8f2ce0640 1 < completed multipart upload key=FX0NCGLK0KPJ upload=U8C4I73UTQS6 tag=PVSFQFB6WIJ7 type=write size=1578106 2024-09-13T00:00:51.374 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.374+0000 7fb8f2ce0640 1 > prepared operation key=PVYZQLGYT3XU tag=YBZA4V6S4P1F type=del size=3834714 2024-09-13T00:00:51.375 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.374+0000 7fb8f2ce0640 1 < canceled part key=_multipart_2AHYRZUDFHP4.2~A908FRBLP5ZH.0 size=1048576 2024-09-13T00:00:51.375 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.374+0000 7fb8f2ce0640 1 < canceled part key=_multipart_2AHYRZUDFHP4.2~A908FRBLP5ZH.1 size=1048576 2024-09-13T00:00:51.381 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.378+0000 7fb8f2ce0640 1 < completed part key=_multipart_2AHYRZUDFHP4.2~A908FRBLP5ZH.2 size=212180 2024-09-13T00:00:51.390 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.390+0000 7fb8f2ce0640 1 < canceled multipart upload key=2AHYRZUDFHP4 upload=A908FRBLP5ZH tag=1GIJ740OZ12R type=write size=3357908 2024-09-13T00:00:51.396 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.394+0000 7fb8f2ce0640 1 > prepared operation key=ZFHN2SNTZ5W7 tag=XHU5V4JF6HED type=del size=1938456 2024-09-13T00:00:51.402 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.402+0000 7fb8f2ce0640 1 < completed delete operation key=2TWD9YLMSYJI tag=G6WDI4B5H6GR type=del 2024-09-13T00:00:51.413 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.414+0000 7fb8f2ce0640 1 > prepared part key=_multipart_44EDYFT9VF8Y.2~HNB1VP57F7CT.0 size=558993 2024-09-13T00:00:51.413 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.414+0000 7fb8f2ce0640 1 > prepared multipart upload key=44EDYFT9VF8Y upload=HNB1VP57F7CT tag=K7IYMW26LKN2 type=write size=1607569 2024-09-13T00:00:51.426 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.426+0000 7fb8f2ce0640 1 < suggested update operation key=2TWD9YLMSYJI tag=8ZG5CP0ZFGZ7 type=del size=88794 2024-09-13T00:00:51.438 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.438+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~4T8W9KPCI88Q.0 size=999129 2024-09-13T00:00:51.438 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.438+0000 7fb8f2ce0640 1 > prepared multipart upload key=70F7D20HI504 upload=4T8W9KPCI88Q tag=L5ILJP6TTVWO type=write size=2047705 2024-09-13T00:00:51.444 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.442+0000 7fb8f2ce0640 1 < completed part key=_multipart_70F7D20HI504.2~35J5SXGV4YMK.0 size=1048576 2024-09-13T00:00:51.444 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.442+0000 7fb8f2ce0640 1 < canceled part key=_multipart_70F7D20HI504.2~35J5SXGV4YMK.1 size=954472 2024-09-13T00:00:51.453 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.454+0000 7fb8f2ce0640 1 < canceled multipart upload key=70F7D20HI504 upload=35J5SXGV4YMK tag=WPKDVLNP2MV1 type=write size=3051624 2024-09-13T00:00:51.464 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.462+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~EO2Q4TV20ZJF.0 size=1048576 2024-09-13T00:00:51.470 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.470+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~EO2Q4TV20ZJF.1 size=321558 2024-09-13T00:00:51.470 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.470+0000 7fb8f2ce0640 1 > prepared multipart upload key=8Z6U7OOA0GBH upload=EO2Q4TV20ZJF tag=0645H1IWS1EY type=write size=2418710 2024-09-13T00:00:51.476 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.474+0000 7fb8f2ce0640 1 < canceled operation key=J8NGDSF9XRTP tag=S2K0LECHI4JO type=del size=673924 2024-09-13T00:00:51.487 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.486+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~4H2879ICQ09D.0 size=1048576 2024-09-13T00:00:51.492 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.490+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~4H2879ICQ09D.1 size=278870 2024-09-13T00:00:51.492 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.490+0000 7fb8f2ce0640 1 > prepared multipart upload key=8Z6U7OOA0GBH upload=4H2879ICQ09D tag=DES0DQHJOKHA type=write size=2376022 2024-09-13T00:00:51.498 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.498+0000 7fb8f2ce0640 1 < completed write operation key=F0QCVR9E08WE tag=V02ZRRXHZ1C7 type=write size=535603 2024-09-13T00:00:51.504 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.502+0000 7fb8f2ce0640 1 > prepared operation key=NX26SI4F2ZMB tag=89ZFSTSF7IBA type=del size=234193 2024-09-13T00:00:51.509 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.510+0000 7fb8f2ce0640 1 < completed delete operation key=O4MZ4UWNXME9 tag=7RALTYOC2U8C type=del 2024-09-13T00:00:51.516 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.514+0000 7fb8f2ce0640 1 > prepared operation key=2AHYRZUDFHP4 tag=V9U43H9F1TWN type=del size=1048983 2024-09-13T00:00:51.526 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.526+0000 7fb8f2ce0640 1 < suggested update operation key=2TWD9YLMSYJI tag=03UX9FJUI8BM type=del size=3305587 2024-09-13T00:00:51.533 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.534+0000 7fb8f2ce0640 1 > prepared operation key=8Z6U7OOA0GBH tag=0RVEE26KIJ73 type=del size=3463868 2024-09-13T00:00:51.538 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.538+0000 7fb8f2ce0640 1 < completed part key=_multipart_5GM88ERM89KT.2~V8F1693I78HN.0 size=1048576 2024-09-13T00:00:51.543 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.542+0000 7fb8f2ce0640 1 < completed part key=_multipart_5GM88ERM89KT.2~V8F1693I78HN.1 size=592618 2024-09-13T00:00:51.555 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.554+0000 7fb8f2ce0640 1 < completed multipart upload key=5GM88ERM89KT upload=V8F1693I78HN tag=P4VRELZMJRN5 type=write size=2689770 2024-09-13T00:00:51.562 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.562+0000 7fb8f2ce0640 1 > prepared operation key=H9AU4HD247YU tag=KZELZUCUW0R7 type=del size=516508 2024-09-13T00:00:51.568 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.566+0000 7fb8f2ce0640 1 < completed part key=_multipart_EUE3LT0PDIXV.2~TXV4G5KETRYA.0 size=741447 2024-09-13T00:00:51.577 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.578+0000 7fb8f2ce0640 1 < completed multipart upload key=EUE3LT0PDIXV upload=TXV4G5KETRYA tag=ZHY76DK063SF type=write size=1790023 2024-09-13T00:00:51.583 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.582+0000 7fb8f2ce0640 1 > prepared operation key=N3Z5XIG3TZRQ tag=63H0VK01C1N1 type=del size=2587780 2024-09-13T00:00:51.589 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.590+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZFHN2SNTZ5W7.2~M95NWKC0LHL7.0 size=1048576 2024-09-13T00:00:51.594 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.594+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZFHN2SNTZ5W7.2~M95NWKC0LHL7.1 size=1048576 2024-09-13T00:00:51.594 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.594+0000 7fb8f2ce0640 1 < canceled part key=_multipart_ZFHN2SNTZ5W7.2~M95NWKC0LHL7.2 size=260446 2024-09-13T00:00:51.604 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.602+0000 7fb8f2ce0640 1 < canceled multipart upload key=ZFHN2SNTZ5W7 upload=M95NWKC0LHL7 tag=3W4B5JU3G0UC type=write size=3406174 2024-09-13T00:00:51.615 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.614+0000 7fb8f2ce0640 1 > prepared part key=_multipart_85YEW8N43FTA.2~3DVQXMS343HL.0 size=1048576 2024-09-13T00:00:51.620 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.618+0000 7fb8f2ce0640 1 > prepared part key=_multipart_85YEW8N43FTA.2~3DVQXMS343HL.1 size=904407 2024-09-13T00:00:51.621 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.618+0000 7fb8f2ce0640 1 > prepared multipart upload key=85YEW8N43FTA upload=3DVQXMS343HL tag=D3SUPYR3LDXN type=write size=3001559 2024-09-13T00:00:51.626 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.626+0000 7fb8f2ce0640 1 < canceled operation key=5GM88ERM89KT tag=MRWGOJ61ULTU type=del size=3412694 2024-09-13T00:00:51.632 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.630+0000 7fb8f2ce0640 1 > prepared operation key=ZFHN2SNTZ5W7 tag=35Y434118430 type=write size=700269 2024-09-13T00:00:51.638 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.638+0000 7fb8f2ce0640 1 < completed delete operation key=O4MZ4UWNXME9 tag=YWXXJY2E2NTY type=del 2024-09-13T00:00:51.644 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.642+0000 7fb8f2ce0640 1 > prepared operation key=2TWD9YLMSYJI tag=GZ2TDPK06C6S type=del size=3924708 2024-09-13T00:00:51.650 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.650+0000 7fb8f2ce0640 1 < completed part key=_multipart_P8OYS5ZE1NX0.2~6LLRS3HPUDG3.0 size=1048576 2024-09-13T00:00:51.655 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.654+0000 7fb8f2ce0640 1 < completed part key=_multipart_P8OYS5ZE1NX0.2~6LLRS3HPUDG3.1 size=1048576 2024-09-13T00:00:51.660 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.658+0000 7fb8f2ce0640 1 < completed part key=_multipart_P8OYS5ZE1NX0.2~6LLRS3HPUDG3.2 size=321499 2024-09-13T00:00:51.670 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.670+0000 7fb8f2ce0640 1 < completed multipart upload key=P8OYS5ZE1NX0 upload=6LLRS3HPUDG3 tag=J9U89PQAFDKS type=write size=3467227 2024-09-13T00:00:51.675 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.674+0000 7fb8f2ce0640 1 > prepared operation key=63FJDWBO2T9S tag=8Q8TETK7E6OZ type=del size=3004589 2024-09-13T00:00:51.681 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.682+0000 7fb8f2ce0640 1 < completed part key=_multipart_63FJDWBO2T9S.2~VVTNNEXF5023.0 size=1048576 2024-09-13T00:00:51.686 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.686+0000 7fb8f2ce0640 1 < completed part key=_multipart_63FJDWBO2T9S.2~VVTNNEXF5023.1 size=1048576 2024-09-13T00:00:51.691 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.690+0000 7fb8f2ce0640 1 < completed part key=_multipart_63FJDWBO2T9S.2~VVTNNEXF5023.2 size=65822 2024-09-13T00:00:51.701 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.702+0000 7fb8f2ce0640 1 < completed multipart upload key=63FJDWBO2T9S upload=VVTNNEXF5023 tag=D64C5DPL7SY4 type=write size=3211550 2024-09-13T00:00:51.707 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.706+0000 7fb8f2ce0640 1 > prepared operation key=63FJDWBO2T9S tag=PG2W3UDR82QB type=del size=138182 2024-09-13T00:00:51.713 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.714+0000 7fb8f2ce0640 1 < completed delete operation key=PVYZQLGYT3XU tag=YBZA4V6S4P1F type=del 2024-09-13T00:00:51.724 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.722+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZFHN2SNTZ5W7.2~98N11RATKQHH.0 size=1048576 2024-09-13T00:00:51.728 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.726+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZFHN2SNTZ5W7.2~98N11RATKQHH.1 size=1048576 2024-09-13T00:00:51.733 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.734+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZFHN2SNTZ5W7.2~98N11RATKQHH.2 size=577198 2024-09-13T00:00:51.733 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.734+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZFHN2SNTZ5W7 upload=98N11RATKQHH tag=WLT7XH4HF7B3 type=write size=3722926 2024-09-13T00:00:51.740 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.738+0000 7fb8f2ce0640 1 < canceled operation key=ZFHN2SNTZ5W7 tag=XHU5V4JF6HED type=del size=1938456 2024-09-13T00:00:51.747 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.746+0000 7fb8f2ce0640 1 > prepared operation key=N3Z5XIG3TZRQ tag=BU14O7AZCPYB type=del size=3100982 2024-09-13T00:00:51.754 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.754+0000 7fb8f2ce0640 1 < completed part key=_multipart_44EDYFT9VF8Y.2~HNB1VP57F7CT.0 size=558993 2024-09-13T00:00:51.765 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.766+0000 7fb8f2ce0640 1 < completed multipart upload key=44EDYFT9VF8Y upload=HNB1VP57F7CT tag=K7IYMW26LKN2 type=write size=1607569 2024-09-13T00:00:51.771 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.770+0000 7fb8f2ce0640 1 > prepared operation key=ZTK9AYBSLCOZ tag=VUQBB0Y5A6OI type=del size=2679279 2024-09-13T00:00:51.776 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.774+0000 7fb8f2ce0640 1 < completed part key=_multipart_70F7D20HI504.2~4T8W9KPCI88Q.0 size=999129 2024-09-13T00:00:51.786 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.786+0000 7fb8f2ce0640 1 < completed multipart upload key=70F7D20HI504 upload=4T8W9KPCI88Q tag=L5ILJP6TTVWO type=write size=2047705 2024-09-13T00:00:51.796 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.794+0000 7fb8f2ce0640 1 > prepared part key=_multipart_5GM88ERM89KT.2~8KKSYORLGUP9.0 size=284739 2024-09-13T00:00:51.796 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.794+0000 7fb8f2ce0640 1 > prepared multipart upload key=5GM88ERM89KT upload=8KKSYORLGUP9 tag=3Y60HJYEIIPF type=write size=1333315 2024-09-13T00:00:51.802 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.802+0000 7fb8f2ce0640 1 < completed part key=_multipart_8Z6U7OOA0GBH.2~EO2Q4TV20ZJF.0 size=1048576 2024-09-13T00:00:51.807 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.806+0000 7fb8f2ce0640 1 < completed part key=_multipart_8Z6U7OOA0GBH.2~EO2Q4TV20ZJF.1 size=321558 2024-09-13T00:00:51.818 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.818+0000 7fb8f2ce0640 1 < completed multipart upload key=8Z6U7OOA0GBH upload=EO2Q4TV20ZJF tag=0645H1IWS1EY type=write size=2418710 2024-09-13T00:00:51.829 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.826+0000 7fb8f2ce0640 1 > prepared part key=_multipart_O4MZ4UWNXME9.2~ZF8VABLFUDJE.0 size=891343 2024-09-13T00:00:51.829 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.826+0000 7fb8f2ce0640 1 > prepared multipart upload key=O4MZ4UWNXME9 upload=ZF8VABLFUDJE tag=E2VWLIG19153 type=write size=1939919 2024-09-13T00:00:51.835 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.834+0000 7fb8f2ce0640 1 < completed part key=_multipart_8Z6U7OOA0GBH.2~4H2879ICQ09D.0 size=1048576 2024-09-13T00:00:51.840 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.838+0000 7fb8f2ce0640 1 < completed part key=_multipart_8Z6U7OOA0GBH.2~4H2879ICQ09D.1 size=278870 2024-09-13T00:00:51.849 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.850+0000 7fb8f2ce0640 1 < completed multipart upload key=8Z6U7OOA0GBH upload=4H2879ICQ09D tag=DES0DQHJOKHA type=write size=2376022 2024-09-13T00:00:51.855 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.854+0000 7fb8f2ce0640 1 > prepared operation key=P8OYS5ZE1NX0 tag=8XINB4OCVC0J type=del size=880900 2024-09-13T00:00:51.861 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.862+0000 7fb8f2ce0640 1 < completed delete operation key=NX26SI4F2ZMB tag=89ZFSTSF7IBA type=del 2024-09-13T00:00:51.872 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.870+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~LXP0JD07R647.0 size=1048576 2024-09-13T00:00:51.877 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.878+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~LXP0JD07R647.1 size=519531 2024-09-13T00:00:51.877 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.878+0000 7fb8f2ce0640 1 > prepared multipart upload key=70F7D20HI504 upload=LXP0JD07R647 tag=SK77ZNCJITT8 type=write size=2616683 2024-09-13T00:00:51.884 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.882+0000 7fb8f2ce0640 1 < completed delete operation key=2AHYRZUDFHP4 tag=V9U43H9F1TWN type=del 2024-09-13T00:00:51.889 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.890+0000 7fb8f2ce0640 1 > prepared operation key=PVYZQLGYT3XU tag=XUWHRAY72SMJ type=write size=542689 2024-09-13T00:00:51.896 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.894+0000 7fb8f2ce0640 1 < completed delete operation key=8Z6U7OOA0GBH tag=0RVEE26KIJ73 type=del 2024-09-13T00:00:51.902 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.902+0000 7fb8f2ce0640 1 > prepared operation key=O4MZ4UWNXME9 tag=0DIM7JT4ENBE type=del size=3406842 2024-09-13T00:00:51.908 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.906+0000 7fb8f2ce0640 1 < completed delete operation key=H9AU4HD247YU tag=KZELZUCUW0R7 type=del 2024-09-13T00:00:51.920 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.918+0000 7fb8f2ce0640 1 > prepared part key=_multipart_85YEW8N43FTA.2~GVITVQ58S1OV.0 size=365071 2024-09-13T00:00:51.920 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.918+0000 7fb8f2ce0640 1 > prepared multipart upload key=85YEW8N43FTA upload=GVITVQ58S1OV tag=93S3UCSSZHU6 type=write size=1413647 2024-09-13T00:00:51.925 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.926+0000 7fb8f2ce0640 1 < completed delete operation key=N3Z5XIG3TZRQ tag=63H0VK01C1N1 type=del 2024-09-13T00:00:51.931 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.930+0000 7fb8f2ce0640 1 > prepared operation key=F0QCVR9E08WE tag=T1W34QYTJGKK type=del size=971906 2024-09-13T00:00:51.937 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.938+0000 7fb8f2ce0640 1 < completed part key=_multipart_85YEW8N43FTA.2~3DVQXMS343HL.0 size=1048576 2024-09-13T00:00:51.942 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.942+0000 7fb8f2ce0640 1 < completed part key=_multipart_85YEW8N43FTA.2~3DVQXMS343HL.1 size=904407 2024-09-13T00:00:51.951 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.950+0000 7fb8f2ce0640 1 < completed multipart upload key=85YEW8N43FTA upload=3DVQXMS343HL tag=D3SUPYR3LDXN type=write size=3001559 2024-09-13T00:00:51.957 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.954+0000 7fb8f2ce0640 1 > prepared operation key=70F7D20HI504 tag=ID1S1NZEF27E type=del size=2301870 2024-09-13T00:00:51.962 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.962+0000 7fb8f2ce0640 1 < completed write operation key=ZFHN2SNTZ5W7 tag=35Y434118430 type=write size=700269 2024-09-13T00:00:51.973 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.970+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H5EIP0E9CBWX.2~FSMYXHEV5RPE.0 size=1048576 2024-09-13T00:00:51.977 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.978+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H5EIP0E9CBWX.2~FSMYXHEV5RPE.1 size=849911 2024-09-13T00:00:51.978 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.978+0000 7fb8f2ce0640 1 > prepared multipart upload key=H5EIP0E9CBWX upload=FSMYXHEV5RPE tag=5PL84POD1VVW type=write size=2947063 2024-09-13T00:00:51.990 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.990+0000 7fb8f2ce0640 1 < suggested update operation key=2TWD9YLMSYJI tag=GZ2TDPK06C6S type=del size=3924708 2024-09-13T00:00:51.996 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:51.994+0000 7fb8f2ce0640 1 > prepared operation key=N3Z5XIG3TZRQ tag=W794IVY6KBA2 type=write size=291685 2024-09-13T00:00:52.001 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.002+0000 7fb8f2ce0640 1 < completed delete operation key=63FJDWBO2T9S tag=8Q8TETK7E6OZ type=del 2024-09-13T00:00:52.012 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.010+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~MVM5ZKRBHAFL.0 size=657745 2024-09-13T00:00:52.012 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.010+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZTK9AYBSLCOZ upload=MVM5ZKRBHAFL tag=H1R888XTEOBS type=write size=1706321 2024-09-13T00:00:52.018 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.018+0000 7fb8f2ce0640 1 < canceled operation key=63FJDWBO2T9S tag=PG2W3UDR82QB type=del size=138182 2024-09-13T00:00:52.024 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.022+0000 7fb8f2ce0640 1 > prepared operation key=44EDYFT9VF8Y tag=BQ987OKQ9C4A type=del size=539259 2024-09-13T00:00:52.030 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.030+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZFHN2SNTZ5W7.2~98N11RATKQHH.0 size=1048576 2024-09-13T00:00:52.034 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.034+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZFHN2SNTZ5W7.2~98N11RATKQHH.1 size=1048576 2024-09-13T00:00:52.040 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.038+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZFHN2SNTZ5W7.2~98N11RATKQHH.2 size=577198 2024-09-13T00:00:52.049 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.050+0000 7fb8f2ce0640 1 < completed multipart upload key=ZFHN2SNTZ5W7 upload=98N11RATKQHH tag=WLT7XH4HF7B3 type=write size=3722926 2024-09-13T00:00:52.055 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.054+0000 7fb8f2ce0640 1 > prepared operation key=PVYZQLGYT3XU tag=0E4K8H2FYL36 type=del size=115184 2024-09-13T00:00:52.061 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.062+0000 7fb8f2ce0640 1 < completed delete operation key=N3Z5XIG3TZRQ tag=BU14O7AZCPYB type=del 2024-09-13T00:00:52.067 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.066+0000 7fb8f2ce0640 1 > prepared operation key=5GM88ERM89KT tag=CRAT7A0R1AGR type=del size=2282468 2024-09-13T00:00:52.079 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.078+0000 7fb8f2ce0640 1 < suggested remove operation key=ZTK9AYBSLCOZ tag=VUQBB0Y5A6OI type=del 2024-09-13T00:00:52.084 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.082+0000 7fb8f2ce0640 1 > prepared operation key=NQJQC8CEAC15 tag=W1O48EGD70SN type=del size=3689824 2024-09-13T00:00:52.090 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.090+0000 7fb8f2ce0640 1 < completed part key=_multipart_5GM88ERM89KT.2~8KKSYORLGUP9.0 size=284739 2024-09-13T00:00:52.100 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.098+0000 7fb8f2ce0640 1 < completed multipart upload key=5GM88ERM89KT upload=8KKSYORLGUP9 tag=3Y60HJYEIIPF type=write size=1333315 2024-09-13T00:00:52.110 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.110+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~4J1G151F338D.0 size=1048576 2024-09-13T00:00:52.115 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.114+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~4J1G151F338D.1 size=1048576 2024-09-13T00:00:52.120 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.118+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~4J1G151F338D.2 size=632299 2024-09-13T00:00:52.120 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.118+0000 7fb8f2ce0640 1 > prepared multipart upload key=8Z6U7OOA0GBH upload=4J1G151F338D tag=1EJMBZQCE21G type=write size=3778027 2024-09-13T00:00:52.126 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.126+0000 7fb8f2ce0640 1 < completed part key=_multipart_O4MZ4UWNXME9.2~ZF8VABLFUDJE.0 size=891343 2024-09-13T00:00:52.136 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.134+0000 7fb8f2ce0640 1 < completed multipart upload key=O4MZ4UWNXME9 upload=ZF8VABLFUDJE tag=E2VWLIG19153 type=write size=1939919 2024-09-13T00:00:52.146 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.146+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~CWB6QCUM469O.0 size=455907 2024-09-13T00:00:52.146 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.146+0000 7fb8f2ce0640 1 > prepared multipart upload key=2AHYRZUDFHP4 upload=CWB6QCUM469O tag=NXIGX1RKFEAY type=write size=1504483 2024-09-13T00:00:52.152 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.150+0000 7fb8f2ce0640 1 < completed delete operation key=P8OYS5ZE1NX0 tag=8XINB4OCVC0J type=del 2024-09-13T00:00:52.163 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.162+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H4M1TCN5W6XU.2~439ZZ8MSH0HG.0 size=1048576 2024-09-13T00:00:52.168 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.166+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H4M1TCN5W6XU.2~439ZZ8MSH0HG.1 size=1048576 2024-09-13T00:00:52.174 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.174+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H4M1TCN5W6XU.2~439ZZ8MSH0HG.2 size=670542 2024-09-13T00:00:52.174 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.174+0000 7fb8f2ce0640 1 > prepared multipart upload key=H4M1TCN5W6XU upload=439ZZ8MSH0HG tag=0Z985F0VOA9Q type=write size=3816270 2024-09-13T00:00:52.180 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.178+0000 7fb8f2ce0640 1 < completed part key=_multipart_70F7D20HI504.2~LXP0JD07R647.0 size=1048576 2024-09-13T00:00:52.185 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.186+0000 7fb8f2ce0640 1 < completed part key=_multipart_70F7D20HI504.2~LXP0JD07R647.1 size=519531 2024-09-13T00:00:52.196 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.194+0000 7fb8f2ce0640 1 < completed multipart upload key=70F7D20HI504 upload=LXP0JD07R647 tag=SK77ZNCJITT8 type=write size=2616683 2024-09-13T00:00:52.207 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.206+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~2UZXIY0X4O2Y.0 size=1048576 2024-09-13T00:00:52.212 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.210+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~2UZXIY0X4O2Y.1 size=519288 2024-09-13T00:00:52.212 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.210+0000 7fb8f2ce0640 1 > prepared multipart upload key=1PX7DBO9M3YI upload=2UZXIY0X4O2Y tag=N4WEUYT1X6CN type=write size=2616440 2024-09-13T00:00:52.217 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.218+0000 7fb8f2ce0640 1 < completed write operation key=PVYZQLGYT3XU tag=XUWHRAY72SMJ type=write size=542689 2024-09-13T00:00:52.228 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.226+0000 7fb8f2ce0640 1 > prepared part key=_multipart_44EDYFT9VF8Y.2~W4EYNJGWDII4.0 size=1048576 2024-09-13T00:00:52.237 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.238+0000 7fb8f2ce0640 1 > prepared part key=_multipart_44EDYFT9VF8Y.2~W4EYNJGWDII4.1 size=529221 2024-09-13T00:00:52.237 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.238+0000 7fb8f2ce0640 1 > prepared multipart upload key=44EDYFT9VF8Y upload=W4EYNJGWDII4 tag=63NURS403HF1 type=write size=2626373 2024-09-13T00:00:52.244 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.242+0000 7fb8f2ce0640 1 < completed delete operation key=O4MZ4UWNXME9 tag=0DIM7JT4ENBE type=del 2024-09-13T00:00:52.256 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.254+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NX26SI4F2ZMB.2~RVYVTBSQ1VCF.0 size=1048576 2024-09-13T00:00:52.264 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.262+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NX26SI4F2ZMB.2~RVYVTBSQ1VCF.1 size=1048576 2024-09-13T00:00:52.269 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.266+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NX26SI4F2ZMB.2~RVYVTBSQ1VCF.2 size=304724 2024-09-13T00:00:52.269 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.266+0000 7fb8f2ce0640 1 > prepared multipart upload key=NX26SI4F2ZMB upload=RVYVTBSQ1VCF tag=RKSNWXZE5HD3 type=write size=3450452 2024-09-13T00:00:52.275 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.274+0000 7fb8f2ce0640 1 < completed part key=_multipart_85YEW8N43FTA.2~GVITVQ58S1OV.0 size=365071 2024-09-13T00:00:52.286 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.286+0000 7fb8f2ce0640 1 < completed multipart upload key=85YEW8N43FTA upload=GVITVQ58S1OV tag=93S3UCSSZHU6 type=write size=1413647 2024-09-13T00:00:52.300 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.298+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2TWD9YLMSYJI.2~KKBM44HLD8A3.0 size=492187 2024-09-13T00:00:52.300 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.298+0000 7fb8f2ce0640 1 > prepared multipart upload key=2TWD9YLMSYJI upload=KKBM44HLD8A3 tag=IX8IA4MGH3XU type=write size=1540763 2024-09-13T00:00:52.306 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.306+0000 7fb8f2ce0640 1 < completed delete operation key=F0QCVR9E08WE tag=T1W34QYTJGKK type=del 2024-09-13T00:00:52.317 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.318+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~MCQ4BXW09VJH.0 size=207391 2024-09-13T00:00:52.317 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.318+0000 7fb8f2ce0640 1 > prepared multipart upload key=1HX06CCZJ1DD upload=MCQ4BXW09VJH tag=LF3QGVEIBLZE type=write size=1255967 2024-09-13T00:00:52.324 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.322+0000 7fb8f2ce0640 1 < completed delete operation key=70F7D20HI504 tag=ID1S1NZEF27E type=del 2024-09-13T00:00:52.330 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.330+0000 7fb8f2ce0640 1 > prepared operation key=O4MZ4UWNXME9 tag=5QO2RQTVOVCZ type=del size=3770304 2024-09-13T00:00:52.337 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.334+0000 7fb8f2ce0640 1 < completed part key=_multipart_H5EIP0E9CBWX.2~FSMYXHEV5RPE.0 size=1048576 2024-09-13T00:00:52.341 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.338+0000 7fb8f2ce0640 1 < completed part key=_multipart_H5EIP0E9CBWX.2~FSMYXHEV5RPE.1 size=849911 2024-09-13T00:00:52.350 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.350+0000 7fb8f2ce0640 1 < completed multipart upload key=H5EIP0E9CBWX upload=FSMYXHEV5RPE tag=5PL84POD1VVW type=write size=2947063 2024-09-13T00:00:52.361 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.362+0000 7fb8f2ce0640 1 > prepared part key=_multipart_0WC6P8YNL0H2.2~U3YMPYGM3IK3.0 size=910358 2024-09-13T00:00:52.361 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.362+0000 7fb8f2ce0640 1 > prepared multipart upload key=0WC6P8YNL0H2 upload=U3YMPYGM3IK3 tag=7WSJPV04UWQS type=write size=1958934 2024-09-13T00:00:52.367 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.366+0000 7fb8f2ce0640 1 < completed write operation key=N3Z5XIG3TZRQ tag=W794IVY6KBA2 type=write size=291685 2024-09-13T00:00:52.373 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.370+0000 7fb8f2ce0640 1 > prepared operation key=2AHYRZUDFHP4 tag=U19RZFQ347OA type=del size=2510027 2024-09-13T00:00:52.379 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.378+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~MVM5ZKRBHAFL.0 size=657745 2024-09-13T00:00:52.389 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.390+0000 7fb8f2ce0640 1 < completed multipart upload key=ZTK9AYBSLCOZ upload=MVM5ZKRBHAFL tag=H1R888XTEOBS type=write size=1706321 2024-09-13T00:00:52.395 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.394+0000 7fb8f2ce0640 1 > prepared operation key=H9AU4HD247YU tag=E4C9VCTPIZQG type=write size=390391 2024-09-13T00:00:52.409 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.410+0000 7fb8f2ce0640 1 < suggested remove operation key=44EDYFT9VF8Y tag=BQ987OKQ9C4A type=del 2024-09-13T00:00:52.414 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.414+0000 7fb8f2ce0640 1 > prepared operation key=70F7D20HI504 tag=HOQ5S9ZP1U8B type=write size=555011 2024-09-13T00:00:52.422 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.422+0000 7fb8f2ce0640 1 < completed delete operation key=PVYZQLGYT3XU tag=0E4K8H2FYL36 type=del 2024-09-13T00:00:52.432 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.430+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~H8JGPH5LEZS4.0 size=1048576 2024-09-13T00:00:52.437 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.438+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~H8JGPH5LEZS4.1 size=1048576 2024-09-13T00:00:52.442 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.442+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~H8JGPH5LEZS4.2 size=105429 2024-09-13T00:00:52.442 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.442+0000 7fb8f2ce0640 1 > prepared multipart upload key=2AHYRZUDFHP4 upload=H8JGPH5LEZS4 tag=VIJLTHYA1CPV type=write size=3251157 2024-09-13T00:00:52.454 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.454+0000 7fb8f2ce0640 1 < suggested remove operation key=5GM88ERM89KT tag=CRAT7A0R1AGR type=del 2024-09-13T00:00:52.460 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.458+0000 7fb8f2ce0640 1 > prepared operation key=0WC6P8YNL0H2 tag=O47HEDDA4ALK type=del size=3411450 2024-09-13T00:00:52.465 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.466+0000 7fb8f2ce0640 1 < completed delete operation key=NQJQC8CEAC15 tag=W1O48EGD70SN type=del 2024-09-13T00:00:52.471 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.470+0000 7fb8f2ce0640 1 > prepared operation key=F0QCVR9E08WE tag=G8LDHP2VZ8Z8 type=write size=20884 2024-09-13T00:00:52.477 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.478+0000 7fb8f2ce0640 1 < completed part key=_multipart_8Z6U7OOA0GBH.2~4J1G151F338D.0 size=1048576 2024-09-13T00:00:52.482 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.482+0000 7fb8f2ce0640 1 < completed part key=_multipart_8Z6U7OOA0GBH.2~4J1G151F338D.1 size=1048576 2024-09-13T00:00:52.487 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.486+0000 7fb8f2ce0640 1 < completed part key=_multipart_8Z6U7OOA0GBH.2~4J1G151F338D.2 size=632299 2024-09-13T00:00:52.498 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.498+0000 7fb8f2ce0640 1 < completed multipart upload key=8Z6U7OOA0GBH upload=4J1G151F338D tag=1EJMBZQCE21G type=write size=3778027 2024-09-13T00:00:52.503 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.502+0000 7fb8f2ce0640 1 > prepared operation key=4K76U69GZ9AE tag=GPUIRJ0XFE43 type=del size=1042148 2024-09-13T00:00:52.512 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.510+0000 7fb8f2ce0640 1 < completed part key=_multipart_2AHYRZUDFHP4.2~CWB6QCUM469O.0 size=455907 2024-09-13T00:00:52.523 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.522+0000 7fb8f2ce0640 1 < completed multipart upload key=2AHYRZUDFHP4 upload=CWB6QCUM469O tag=NXIGX1RKFEAY type=write size=1504483 2024-09-13T00:00:52.529 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.530+0000 7fb8f2ce0640 1 > prepared operation key=7OX5OJZ8GXU5 tag=DDWMXAF9FQ3I type=del size=712741 2024-09-13T00:00:52.535 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.534+0000 7fb8f2ce0640 1 < completed part key=_multipart_H4M1TCN5W6XU.2~439ZZ8MSH0HG.0 size=1048576 2024-09-13T00:00:52.540 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.538+0000 7fb8f2ce0640 1 < completed part key=_multipart_H4M1TCN5W6XU.2~439ZZ8MSH0HG.1 size=1048576 2024-09-13T00:00:52.546 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.546+0000 7fb8f2ce0640 1 < completed part key=_multipart_H4M1TCN5W6XU.2~439ZZ8MSH0HG.2 size=670542 2024-09-13T00:00:52.557 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.558+0000 7fb8f2ce0640 1 < completed multipart upload key=H4M1TCN5W6XU upload=439ZZ8MSH0HG tag=0Z985F0VOA9Q type=write size=3816270 2024-09-13T00:00:52.568 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.566+0000 7fb8f2ce0640 1 > prepared part key=_multipart_O4MZ4UWNXME9.2~I4D3ZEGRYVL6.0 size=1048576 2024-09-13T00:00:52.575 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.574+0000 7fb8f2ce0640 1 > prepared part key=_multipart_O4MZ4UWNXME9.2~I4D3ZEGRYVL6.1 size=42285 2024-09-13T00:00:52.575 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.574+0000 7fb8f2ce0640 1 > prepared multipart upload key=O4MZ4UWNXME9 upload=I4D3ZEGRYVL6 tag=MEP8OYBS1SWB type=write size=2139437 2024-09-13T00:00:52.582 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.582+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~2UZXIY0X4O2Y.0 size=1048576 2024-09-13T00:00:52.587 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.586+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~2UZXIY0X4O2Y.1 size=519288 2024-09-13T00:00:52.597 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.598+0000 7fb8f2ce0640 1 < completed multipart upload key=1PX7DBO9M3YI upload=2UZXIY0X4O2Y tag=N4WEUYT1X6CN type=write size=2616440 2024-09-13T00:00:52.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:00:52 smithi181 bash[95505]: ts=2024-09-13T00:00:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:00:52.603 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.602+0000 7fb8f2ce0640 1 > prepared operation key=8MPUVD37EDN9 tag=2P8I2U12SCU7 type=write size=833221 2024-09-13T00:00:52.609 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.610+0000 7fb8f2ce0640 1 < completed part key=_multipart_44EDYFT9VF8Y.2~W4EYNJGWDII4.0 size=1048576 2024-09-13T00:00:52.614 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.614+0000 7fb8f2ce0640 1 < completed part key=_multipart_44EDYFT9VF8Y.2~W4EYNJGWDII4.1 size=529221 2024-09-13T00:00:52.624 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.622+0000 7fb8f2ce0640 1 < completed multipart upload key=44EDYFT9VF8Y upload=W4EYNJGWDII4 tag=63NURS403HF1 type=write size=2626373 2024-09-13T00:00:52.631 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.630+0000 7fb8f2ce0640 1 > prepared operation key=H4M1TCN5W6XU tag=57JM0RZN68JW type=del size=906328 2024-09-13T00:00:52.637 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.634+0000 7fb8f2ce0640 1 < completed part key=_multipart_NX26SI4F2ZMB.2~RVYVTBSQ1VCF.0 size=1048576 2024-09-13T00:00:52.642 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.642+0000 7fb8f2ce0640 1 < completed part key=_multipart_NX26SI4F2ZMB.2~RVYVTBSQ1VCF.1 size=1048576 2024-09-13T00:00:52.647 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.646+0000 7fb8f2ce0640 1 < completed part key=_multipart_NX26SI4F2ZMB.2~RVYVTBSQ1VCF.2 size=304724 2024-09-13T00:00:52.657 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.658+0000 7fb8f2ce0640 1 < completed multipart upload key=NX26SI4F2ZMB upload=RVYVTBSQ1VCF tag=RKSNWXZE5HD3 type=write size=3450452 2024-09-13T00:00:52.663 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.662+0000 7fb8f2ce0640 1 > prepared operation key=85YEW8N43FTA tag=6FNAOSDCH56Y type=del size=856520 2024-09-13T00:00:52.669 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.670+0000 7fb8f2ce0640 1 < completed part key=_multipart_2TWD9YLMSYJI.2~KKBM44HLD8A3.0 size=492187 2024-09-13T00:00:52.680 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.678+0000 7fb8f2ce0640 1 < completed multipart upload key=2TWD9YLMSYJI upload=KKBM44HLD8A3 tag=IX8IA4MGH3XU type=write size=1540763 2024-09-13T00:00:52.690 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.690+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H4M1TCN5W6XU.2~80E5LMLGOYC1.0 size=1048576 2024-09-13T00:00:52.695 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.694+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H4M1TCN5W6XU.2~80E5LMLGOYC1.1 size=1048576 2024-09-13T00:00:52.700 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.698+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H4M1TCN5W6XU.2~80E5LMLGOYC1.2 size=935565 2024-09-13T00:00:52.700 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.698+0000 7fb8f2ce0640 1 > prepared multipart upload key=H4M1TCN5W6XU upload=80E5LMLGOYC1 tag=MEXH3I8SOKQ2 type=write size=4081293 2024-09-13T00:00:52.707 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.706+0000 7fb8f2ce0640 1 < completed part key=_multipart_1HX06CCZJ1DD.2~MCQ4BXW09VJH.0 size=207391 2024-09-13T00:00:52.717 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.718+0000 7fb8f2ce0640 1 < completed multipart upload key=1HX06CCZJ1DD upload=MCQ4BXW09VJH tag=LF3QGVEIBLZE type=write size=1255967 2024-09-13T00:00:52.728 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.726+0000 7fb8f2ce0640 1 > prepared part key=_multipart_N3Z5XIG3TZRQ.2~UQPZ644BB0L7.0 size=1048576 2024-09-13T00:00:52.733 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.734+0000 7fb8f2ce0640 1 > prepared part key=_multipart_N3Z5XIG3TZRQ.2~UQPZ644BB0L7.1 size=1048576 2024-09-13T00:00:52.738 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.738+0000 7fb8f2ce0640 1 > prepared part key=_multipart_N3Z5XIG3TZRQ.2~UQPZ644BB0L7.2 size=197648 2024-09-13T00:00:52.738 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.738+0000 7fb8f2ce0640 1 > prepared multipart upload key=N3Z5XIG3TZRQ upload=UQPZ644BB0L7 tag=00JM8KNID5DP type=write size=3343376 2024-09-13T00:00:52.744 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.742+0000 7fb8f2ce0640 1 < canceled operation key=O4MZ4UWNXME9 tag=5QO2RQTVOVCZ type=del size=3770304 2024-09-13T00:00:52.751 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.750+0000 7fb8f2ce0640 1 > prepared operation key=EUE3LT0PDIXV tag=7P6HZHULVBHQ type=write size=843676 2024-09-13T00:00:52.752 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.750+0000 7fb8f2ce0640 1 < canceled part key=_multipart_0WC6P8YNL0H2.2~U3YMPYGM3IK3.0 size=910358 2024-09-13T00:00:52.762 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.762+0000 7fb8f2ce0640 1 < canceled multipart upload key=0WC6P8YNL0H2 upload=U3YMPYGM3IK3 tag=7WSJPV04UWQS type=write size=1958934 2024-09-13T00:00:52.768 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.766+0000 7fb8f2ce0640 1 > prepared operation key=0WC6P8YNL0H2 tag=VKI5O4DY3U53 type=del size=2957844 2024-09-13T00:00:52.780 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.778+0000 7fb8f2ce0640 1 < suggested remove operation key=2AHYRZUDFHP4 tag=U19RZFQ347OA type=del 2024-09-13T00:00:52.792 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.790+0000 7fb8f2ce0640 1 > prepared part key=_multipart_0WC6P8YNL0H2.2~W2CL7ZFT8F7V.0 size=1048576 2024-09-13T00:00:52.798 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.798+0000 7fb8f2ce0640 1 > prepared part key=_multipart_0WC6P8YNL0H2.2~W2CL7ZFT8F7V.1 size=1048576 2024-09-13T00:00:52.804 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.802+0000 7fb8f2ce0640 1 > prepared part key=_multipart_0WC6P8YNL0H2.2~W2CL7ZFT8F7V.2 size=988033 2024-09-13T00:00:52.804 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.802+0000 7fb8f2ce0640 1 > prepared multipart upload key=0WC6P8YNL0H2 upload=W2CL7ZFT8F7V tag=ARZ89L1ZVROG type=write size=4133761 2024-09-13T00:00:52.810 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.810+0000 7fb8f2ce0640 1 < canceled operation key=H9AU4HD247YU tag=E4C9VCTPIZQG type=write size=390391 2024-09-13T00:00:52.821 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.822+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~58VIYOY7SBN7.0 size=695771 2024-09-13T00:00:52.821 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.822+0000 7fb8f2ce0640 1 > prepared multipart upload key=1PX7DBO9M3YI upload=58VIYOY7SBN7 tag=G1X7OT066XS6 type=write size=1744347 2024-09-13T00:00:52.844 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.842+0000 7fb8f2ce0640 1 < suggested update operation key=70F7D20HI504 tag=HOQ5S9ZP1U8B type=write size=555011 2024-09-13T00:00:52.851 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.850+0000 7fb8f2ce0640 1 > prepared operation key=1PX7DBO9M3YI tag=C1BVISHATYZT type=del size=285551 2024-09-13T00:00:52.856 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.854+0000 7fb8f2ce0640 1 < completed part key=_multipart_2AHYRZUDFHP4.2~H8JGPH5LEZS4.0 size=1048576 2024-09-13T00:00:52.861 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.858+0000 7fb8f2ce0640 1 < completed part key=_multipart_2AHYRZUDFHP4.2~H8JGPH5LEZS4.1 size=1048576 2024-09-13T00:00:52.865 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.866+0000 7fb8f2ce0640 1 < completed part key=_multipart_2AHYRZUDFHP4.2~H8JGPH5LEZS4.2 size=105429 2024-09-13T00:00:52.874 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.874+0000 7fb8f2ce0640 1 < completed multipart upload key=2AHYRZUDFHP4 upload=H8JGPH5LEZS4 tag=VIJLTHYA1CPV type=write size=3251157 2024-09-13T00:00:52.880 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.878+0000 7fb8f2ce0640 1 > prepared operation key=70F7D20HI504 tag=FUCTWW19EUTK type=del size=1725968 2024-09-13T00:00:52.885 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.886+0000 7fb8f2ce0640 1 < completed delete operation key=0WC6P8YNL0H2 tag=O47HEDDA4ALK type=del 2024-09-13T00:00:52.891 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.890+0000 7fb8f2ce0640 1 > prepared operation key=8Z6U7OOA0GBH tag=1YJO0J057XBQ type=del size=3035625 2024-09-13T00:00:52.901 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.902+0000 7fb8f2ce0640 1 < suggested remove operation key=F0QCVR9E08WE tag=G8LDHP2VZ8Z8 type=write 2024-09-13T00:00:52.912 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.910+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~AZM5D2X4DV01.0 size=1048576 2024-09-13T00:00:52.918 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.918+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~AZM5D2X4DV01.1 size=1048576 2024-09-13T00:00:52.923 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.922+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~AZM5D2X4DV01.2 size=138671 2024-09-13T00:00:52.923 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.922+0000 7fb8f2ce0640 1 > prepared multipart upload key=FX0NCGLK0KPJ upload=AZM5D2X4DV01 tag=JSPEY6UGTZ47 type=write size=3284399 2024-09-13T00:00:52.933 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.934+0000 7fb8f2ce0640 1 < suggested remove operation key=4K76U69GZ9AE tag=GPUIRJ0XFE43 type=del 2024-09-13T00:00:52.939 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.938+0000 7fb8f2ce0640 1 > prepared operation key=4K76U69GZ9AE tag=NU1196HI770P type=del size=2463257 2024-09-13T00:00:52.953 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.950+0000 7fb8f2ce0640 1 < suggested remove operation key=7OX5OJZ8GXU5 tag=DDWMXAF9FQ3I type=del 2024-09-13T00:00:52.961 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.962+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~RT80JH8LBKYX.0 size=1043784 2024-09-13T00:00:52.961 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.962+0000 7fb8f2ce0640 1 > prepared multipart upload key=70F7D20HI504 upload=RT80JH8LBKYX tag=U70CE34ILJTB type=write size=2092360 2024-09-13T00:00:52.967 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.966+0000 7fb8f2ce0640 1 < completed part key=_multipart_O4MZ4UWNXME9.2~I4D3ZEGRYVL6.0 size=1048576 2024-09-13T00:00:52.972 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.970+0000 7fb8f2ce0640 1 < completed part key=_multipart_O4MZ4UWNXME9.2~I4D3ZEGRYVL6.1 size=42285 2024-09-13T00:00:52.982 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.982+0000 7fb8f2ce0640 1 < completed multipart upload key=O4MZ4UWNXME9 upload=I4D3ZEGRYVL6 tag=MEP8OYBS1SWB type=write size=2139437 2024-09-13T00:00:52.992 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.990+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~NIEWQ6XVYP5Y.0 size=1048576 2024-09-13T00:00:52.997 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.998+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~NIEWQ6XVYP5Y.1 size=33544 2024-09-13T00:00:52.997 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:52.998+0000 7fb8f2ce0640 1 > prepared multipart upload key=PVYZQLGYT3XU upload=NIEWQ6XVYP5Y tag=JJB03FU2J6ND type=write size=2130696 2024-09-13T00:00:53.011 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.010+0000 7fb8f2ce0640 1 < suggested remove operation key=8MPUVD37EDN9 tag=2P8I2U12SCU7 type=write 2024-09-13T00:00:53.017 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.018+0000 7fb8f2ce0640 1 > prepared operation key=7OX5OJZ8GXU5 tag=A0F108S2M1WD type=write size=442785 2024-09-13T00:00:53.023 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.022+0000 7fb8f2ce0640 1 < completed delete operation key=H4M1TCN5W6XU tag=57JM0RZN68JW type=del 2024-09-13T00:00:53.034 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.034+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2TWD9YLMSYJI.2~4QC0DLR4JEM4.0 size=541208 2024-09-13T00:00:53.034 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.034+0000 7fb8f2ce0640 1 > prepared multipart upload key=2TWD9YLMSYJI upload=4QC0DLR4JEM4 tag=PZXJCC71JIYX type=write size=1589784 2024-09-13T00:00:53.040 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.038+0000 7fb8f2ce0640 1 < completed delete operation key=85YEW8N43FTA tag=6FNAOSDCH56Y type=del 2024-09-13T00:00:53.051 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.050+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NX26SI4F2ZMB.2~VH5OF548JZF2.0 size=1048576 2024-09-13T00:00:53.056 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.054+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NX26SI4F2ZMB.2~VH5OF548JZF2.1 size=1048576 2024-09-13T00:00:53.061 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.062+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NX26SI4F2ZMB.2~VH5OF548JZF2.2 size=757021 2024-09-13T00:00:53.061 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.062+0000 7fb8f2ce0640 1 > prepared multipart upload key=NX26SI4F2ZMB upload=VH5OF548JZF2 tag=SFICB3CCUW13 type=write size=3902749 2024-09-13T00:00:53.067 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.066+0000 7fb8f2ce0640 1 < completed part key=_multipart_H4M1TCN5W6XU.2~80E5LMLGOYC1.0 size=1048576 2024-09-13T00:00:53.072 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.070+0000 7fb8f2ce0640 1 < completed part key=_multipart_H4M1TCN5W6XU.2~80E5LMLGOYC1.1 size=1048576 2024-09-13T00:00:53.077 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.078+0000 7fb8f2ce0640 1 < completed part key=_multipart_H4M1TCN5W6XU.2~80E5LMLGOYC1.2 size=935565 2024-09-13T00:00:53.087 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.086+0000 7fb8f2ce0640 1 < completed multipart upload key=H4M1TCN5W6XU upload=80E5LMLGOYC1 tag=MEXH3I8SOKQ2 type=write size=4081293 2024-09-13T00:00:53.098 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.098+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~78GM91GS2ROE.0 size=1048576 2024-09-13T00:00:53.102 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.102+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~78GM91GS2ROE.1 size=1048576 2024-09-13T00:00:53.107 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.106+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~78GM91GS2ROE.2 size=889800 2024-09-13T00:00:53.107 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.106+0000 7fb8f2ce0640 1 > prepared multipart upload key=1HX06CCZJ1DD upload=78GM91GS2ROE tag=B1RKKD7PTQ5T type=write size=4035528 2024-09-13T00:00:53.113 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.110+0000 7fb8f2ce0640 1 < completed part key=_multipart_N3Z5XIG3TZRQ.2~UQPZ644BB0L7.0 size=1048576 2024-09-13T00:00:53.117 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.118+0000 7fb8f2ce0640 1 < completed part key=_multipart_N3Z5XIG3TZRQ.2~UQPZ644BB0L7.1 size=1048576 2024-09-13T00:00:53.122 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.122+0000 7fb8f2ce0640 1 < completed part key=_multipart_N3Z5XIG3TZRQ.2~UQPZ644BB0L7.2 size=197648 2024-09-13T00:00:53.132 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.130+0000 7fb8f2ce0640 1 < completed multipart upload key=N3Z5XIG3TZRQ upload=UQPZ644BB0L7 tag=00JM8KNID5DP type=write size=3343376 2024-09-13T00:00:53.142 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.142+0000 7fb8f2ce0640 1 > prepared part key=_multipart_P8OYS5ZE1NX0.2~DK50DBBETJEV.0 size=9915 2024-09-13T00:00:53.143 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.142+0000 7fb8f2ce0640 1 > prepared multipart upload key=P8OYS5ZE1NX0 upload=DK50DBBETJEV tag=YAEL79FQO4N6 type=write size=1058491 2024-09-13T00:00:53.154 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.154+0000 7fb8f2ce0640 1 < suggested update operation key=EUE3LT0PDIXV tag=7P6HZHULVBHQ type=write size=843676 2024-09-13T00:00:53.160 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.158+0000 7fb8f2ce0640 1 > prepared operation key=P8OYS5ZE1NX0 tag=0L82HT9R1KLC type=del size=2439518 2024-09-13T00:00:53.165 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.166+0000 7fb8f2ce0640 1 < completed delete operation key=0WC6P8YNL0H2 tag=VKI5O4DY3U53 type=del 2024-09-13T00:00:53.171 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.170+0000 7fb8f2ce0640 1 > prepared operation key=QMZREZ1QEDNJ tag=065B10SSJOK9 type=del size=49423 2024-09-13T00:00:53.177 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.178+0000 7fb8f2ce0640 1 < completed part key=_multipart_0WC6P8YNL0H2.2~W2CL7ZFT8F7V.0 size=1048576 2024-09-13T00:00:53.183 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.182+0000 7fb8f2ce0640 1 < completed part key=_multipart_0WC6P8YNL0H2.2~W2CL7ZFT8F7V.1 size=1048576 2024-09-13T00:00:53.188 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.186+0000 7fb8f2ce0640 1 < completed part key=_multipart_0WC6P8YNL0H2.2~W2CL7ZFT8F7V.2 size=988033 2024-09-13T00:00:53.198 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.198+0000 7fb8f2ce0640 1 < completed multipart upload key=0WC6P8YNL0H2 upload=W2CL7ZFT8F7V tag=ARZ89L1ZVROG type=write size=4133761 2024-09-13T00:00:53.210 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.210+0000 7fb8f2ce0640 1 > prepared part key=_multipart_85YEW8N43FTA.2~J19PHUHOZIKN.0 size=1048576 2024-09-13T00:00:53.214 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.214+0000 7fb8f2ce0640 1 > prepared part key=_multipart_85YEW8N43FTA.2~J19PHUHOZIKN.1 size=426593 2024-09-13T00:00:53.214 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.214+0000 7fb8f2ce0640 1 > prepared multipart upload key=85YEW8N43FTA upload=J19PHUHOZIKN tag=7G1VHZB8F9LB type=write size=2523745 2024-09-13T00:00:53.220 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.218+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~58VIYOY7SBN7.0 size=695771 2024-09-13T00:00:53.231 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.230+0000 7fb8f2ce0640 1 < completed multipart upload key=1PX7DBO9M3YI upload=58VIYOY7SBN7 tag=G1X7OT066XS6 type=write size=1744347 2024-09-13T00:00:53.236 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.234+0000 7fb8f2ce0640 1 > prepared operation key=ZFHN2SNTZ5W7 tag=P3I3FODKZMPP type=write size=811420 2024-09-13T00:00:53.243 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.242+0000 7fb8f2ce0640 1 < canceled operation key=1PX7DBO9M3YI tag=C1BVISHATYZT type=del size=285551 2024-09-13T00:00:53.249 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.250+0000 7fb8f2ce0640 1 > prepared operation key=EUE3LT0PDIXV tag=2CDVZYC52C7P type=del size=2748285 2024-09-13T00:00:53.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:52 smithi136 bash[75164]: cluster 2024-09-13T00:00:51.135497+0000 mgr.y (mgr.24661) 712 : cluster [DBG] pgmap v1388: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 679 KiB/s rd, 236 KiB/s wr, 1.20k op/s 2024-09-13T00:00:53.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:52 smithi136 bash[76371]: cluster 2024-09-13T00:00:51.135497+0000 mgr.y (mgr.24661) 712 : cluster [DBG] pgmap v1388: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 679 KiB/s rd, 236 KiB/s wr, 1.20k op/s 2024-09-13T00:00:53.261 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.262+0000 7fb8f2ce0640 1 < suggested update operation key=70F7D20HI504 tag=FUCTWW19EUTK type=del size=1725968 2024-09-13T00:00:53.272 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.270+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~JU90U8Z1VRYK.0 size=1048576 2024-09-13T00:00:53.277 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.274+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~JU90U8Z1VRYK.1 size=1048576 2024-09-13T00:00:53.282 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.282+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~JU90U8Z1VRYK.2 size=703815 2024-09-13T00:00:53.282 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.282+0000 7fb8f2ce0640 1 > prepared multipart upload key=7OX5OJZ8GXU5 upload=JU90U8Z1VRYK tag=LATNRXIQCR4C type=write size=3849543 2024-09-13T00:00:53.287 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.286+0000 7fb8f2ce0640 1 < completed delete operation key=8Z6U7OOA0GBH tag=1YJO0J057XBQ type=del 2024-09-13T00:00:53.298 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.298+0000 7fb8f2ce0640 1 > prepared part key=_multipart_QMZREZ1QEDNJ.2~A13R946BIE5C.0 size=1048576 2024-09-13T00:00:53.303 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.302+0000 7fb8f2ce0640 1 > prepared part key=_multipart_QMZREZ1QEDNJ.2~A13R946BIE5C.1 size=1048576 2024-09-13T00:00:53.308 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.306+0000 7fb8f2ce0640 1 > prepared part key=_multipart_QMZREZ1QEDNJ.2~A13R946BIE5C.2 size=77906 2024-09-13T00:00:53.308 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.306+0000 7fb8f2ce0640 1 > prepared multipart upload key=QMZREZ1QEDNJ upload=A13R946BIE5C tag=QQPMS6KO04Y8 type=write size=3223634 2024-09-13T00:00:53.314 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.314+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~AZM5D2X4DV01.0 size=1048576 2024-09-13T00:00:53.319 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.318+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~AZM5D2X4DV01.1 size=1048576 2024-09-13T00:00:53.324 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.322+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~AZM5D2X4DV01.2 size=138671 2024-09-13T00:00:53.334 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.334+0000 7fb8f2ce0640 1 < completed multipart upload key=FX0NCGLK0KPJ upload=AZM5D2X4DV01 tag=JSPEY6UGTZ47 type=write size=3284399 2024-09-13T00:00:53.340 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.338+0000 7fb8f2ce0640 1 > prepared operation key=EUE3LT0PDIXV tag=NFXZJVYYTUPS type=del size=1498227 2024-09-13T00:00:53.346 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.346+0000 7fb8f2ce0640 1 < completed delete operation key=4K76U69GZ9AE tag=NU1196HI770P type=del 2024-09-13T00:00:53.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:52 smithi181 bash[67587]: cluster 2024-09-13T00:00:51.135497+0000 mgr.y (mgr.24661) 712 : cluster [DBG] pgmap v1388: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 679 KiB/s rd, 236 KiB/s wr, 1.20k op/s 2024-09-13T00:00:53.353 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.354+0000 7fb8f2ce0640 1 > prepared operation key=H4M1TCN5W6XU tag=8X53LP5RPRHL type=del size=2504696 2024-09-13T00:00:53.362 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.358+0000 7fb8f2ce0640 1 < completed part key=_multipart_70F7D20HI504.2~RT80JH8LBKYX.0 size=1043784 2024-09-13T00:00:53.369 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.370+0000 7fb8f2ce0640 1 < completed multipart upload key=70F7D20HI504 upload=RT80JH8LBKYX tag=U70CE34ILJTB type=write size=2092360 2024-09-13T00:00:53.380 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.378+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EJN6HR8I7XQF.2~X5X5RLMCROT8.0 size=1048576 2024-09-13T00:00:53.385 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.382+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EJN6HR8I7XQF.2~X5X5RLMCROT8.1 size=1048576 2024-09-13T00:00:53.390 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.390+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EJN6HR8I7XQF.2~X5X5RLMCROT8.2 size=635303 2024-09-13T00:00:53.390 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.390+0000 7fb8f2ce0640 1 > prepared multipart upload key=EJN6HR8I7XQF upload=X5X5RLMCROT8 tag=A1HBMLDIIECU type=write size=3781031 2024-09-13T00:00:53.396 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.394+0000 7fb8f2ce0640 1 < completed part key=_multipart_PVYZQLGYT3XU.2~NIEWQ6XVYP5Y.0 size=1048576 2024-09-13T00:00:53.401 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.402+0000 7fb8f2ce0640 1 < completed part key=_multipart_PVYZQLGYT3XU.2~NIEWQ6XVYP5Y.1 size=33544 2024-09-13T00:00:53.413 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.410+0000 7fb8f2ce0640 1 < completed multipart upload key=PVYZQLGYT3XU upload=NIEWQ6XVYP5Y tag=JJB03FU2J6ND type=write size=2130696 2024-09-13T00:00:53.418 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.418+0000 7fb8f2ce0640 1 > prepared operation key=J8NGDSF9XRTP tag=IOYFW1KVJBL8 type=del size=924411 2024-09-13T00:00:53.425 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.426+0000 7fb8f2ce0640 1 < completed write operation key=7OX5OJZ8GXU5 tag=A0F108S2M1WD type=write size=442785 2024-09-13T00:00:53.437 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.438+0000 7fb8f2ce0640 1 > prepared part key=_multipart_85YEW8N43FTA.2~6TJQJJZ617EO.0 size=1048576 2024-09-13T00:00:53.442 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.442+0000 7fb8f2ce0640 1 > prepared part key=_multipart_85YEW8N43FTA.2~6TJQJJZ617EO.1 size=1750 2024-09-13T00:00:53.442 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.442+0000 7fb8f2ce0640 1 > prepared multipart upload key=85YEW8N43FTA upload=6TJQJJZ617EO tag=9HGU1MQ9JCJO type=write size=2098902 2024-09-13T00:00:53.443 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.442+0000 7fb8f2ce0640 1 < canceled part key=_multipart_2TWD9YLMSYJI.2~4QC0DLR4JEM4.0 size=541208 2024-09-13T00:00:53.454 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.454+0000 7fb8f2ce0640 1 < canceled multipart upload key=2TWD9YLMSYJI upload=4QC0DLR4JEM4 tag=PZXJCC71JIYX type=write size=1589784 2024-09-13T00:00:53.461 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.458+0000 7fb8f2ce0640 1 > prepared operation key=ZFHN2SNTZ5W7 tag=7FNO1AZ5P043 type=write size=501018 2024-09-13T00:00:53.467 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.466+0000 7fb8f2ce0640 1 < completed part key=_multipart_NX26SI4F2ZMB.2~VH5OF548JZF2.0 size=1048576 2024-09-13T00:00:53.472 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.470+0000 7fb8f2ce0640 1 < completed part key=_multipart_NX26SI4F2ZMB.2~VH5OF548JZF2.1 size=1048576 2024-09-13T00:00:53.477 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.478+0000 7fb8f2ce0640 1 < completed part key=_multipart_NX26SI4F2ZMB.2~VH5OF548JZF2.2 size=757021 2024-09-13T00:00:53.487 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.486+0000 7fb8f2ce0640 1 < completed multipart upload key=NX26SI4F2ZMB upload=VH5OF548JZF2 tag=SFICB3CCUW13 type=write size=3902749 2024-09-13T00:00:53.498 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.498+0000 7fb8f2ce0640 1 > prepared part key=_multipart_F0QCVR9E08WE.2~OLJHF86ZWMSC.0 size=931077 2024-09-13T00:00:53.499 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.498+0000 7fb8f2ce0640 1 > prepared multipart upload key=F0QCVR9E08WE upload=OLJHF86ZWMSC tag=DXOJR5QBSFUY type=write size=1979653 2024-09-13T00:00:53.505 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.502+0000 7fb8f2ce0640 1 < completed part key=_multipart_1HX06CCZJ1DD.2~78GM91GS2ROE.0 size=1048576 2024-09-13T00:00:53.510 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.510+0000 7fb8f2ce0640 1 < completed part key=_multipart_1HX06CCZJ1DD.2~78GM91GS2ROE.1 size=1048576 2024-09-13T00:00:53.515 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.514+0000 7fb8f2ce0640 1 < completed part key=_multipart_1HX06CCZJ1DD.2~78GM91GS2ROE.2 size=889800 2024-09-13T00:00:53.526 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.526+0000 7fb8f2ce0640 1 < completed multipart upload key=1HX06CCZJ1DD upload=78GM91GS2ROE tag=B1RKKD7PTQ5T type=write size=4035528 2024-09-13T00:00:53.532 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.530+0000 7fb8f2ce0640 1 > prepared operation key=FX0NCGLK0KPJ tag=UXLMT9E7RDZL type=del size=344147 2024-09-13T00:00:53.538 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.538+0000 7fb8f2ce0640 1 < completed part key=_multipart_P8OYS5ZE1NX0.2~DK50DBBETJEV.0 size=9915 2024-09-13T00:00:53.548 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.546+0000 7fb8f2ce0640 1 < completed multipart upload key=P8OYS5ZE1NX0 upload=DK50DBBETJEV tag=YAEL79FQO4N6 type=write size=1058491 2024-09-13T00:00:53.554 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.554+0000 7fb8f2ce0640 1 > prepared operation key=W8XZ26311RJX tag=U4E1UV0CNOD1 type=write size=637242 2024-09-13T00:00:53.561 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.562+0000 7fb8f2ce0640 1 < completed delete operation key=P8OYS5ZE1NX0 tag=0L82HT9R1KLC type=del 2024-09-13T00:00:53.568 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.566+0000 7fb8f2ce0640 1 > prepared operation key=H5EIP0E9CBWX tag=0D3U02O25286 type=del size=2596730 2024-09-13T00:00:53.574 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.574+0000 7fb8f2ce0640 1 < canceled operation key=QMZREZ1QEDNJ tag=065B10SSJOK9 type=del size=49423 2024-09-13T00:00:53.585 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.586+0000 7fb8f2ce0640 1 > prepared part key=_multipart_F0QCVR9E08WE.2~SWIMRTXHPP51.0 size=1048576 2024-09-13T00:00:53.590 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.590+0000 7fb8f2ce0640 1 > prepared part key=_multipart_F0QCVR9E08WE.2~SWIMRTXHPP51.1 size=1048576 2024-09-13T00:00:53.596 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.594+0000 7fb8f2ce0640 1 > prepared part key=_multipart_F0QCVR9E08WE.2~SWIMRTXHPP51.2 size=643459 2024-09-13T00:00:53.596 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.594+0000 7fb8f2ce0640 1 > prepared multipart upload key=F0QCVR9E08WE upload=SWIMRTXHPP51 tag=B4TSAKTL0LQX type=write size=3789187 2024-09-13T00:00:53.602 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.602+0000 7fb8f2ce0640 1 < completed part key=_multipart_85YEW8N43FTA.2~J19PHUHOZIKN.0 size=1048576 2024-09-13T00:00:53.607 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.606+0000 7fb8f2ce0640 1 < completed part key=_multipart_85YEW8N43FTA.2~J19PHUHOZIKN.1 size=426593 2024-09-13T00:00:53.617 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.618+0000 7fb8f2ce0640 1 < completed multipart upload key=85YEW8N43FTA upload=J19PHUHOZIKN tag=7G1VHZB8F9LB type=write size=2523745 2024-09-13T00:00:53.627 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.626+0000 7fb8f2ce0640 1 > prepared part key=_multipart_44EDYFT9VF8Y.2~HPKDMQTGKK7Z.0 size=889220 2024-09-13T00:00:53.627 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.626+0000 7fb8f2ce0640 1 > prepared multipart upload key=44EDYFT9VF8Y upload=HPKDMQTGKK7Z tag=2M91WNA3EA09 type=write size=1937796 2024-09-13T00:00:53.634 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.634+0000 7fb8f2ce0640 1 < completed write operation key=ZFHN2SNTZ5W7 tag=P3I3FODKZMPP type=write size=811420 2024-09-13T00:00:53.640 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.638+0000 7fb8f2ce0640 1 > prepared operation key=5GM88ERM89KT tag=X8VG2OGV26FM type=del size=825214 2024-09-13T00:00:53.651 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.650+0000 7fb8f2ce0640 1 < suggested remove operation key=EUE3LT0PDIXV tag=2CDVZYC52C7P type=del 2024-09-13T00:00:53.662 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.662+0000 7fb8f2ce0640 1 > prepared part key=_multipart_P8OYS5ZE1NX0.2~MQENO2VGG162.0 size=109749 2024-09-13T00:00:53.662 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.662+0000 7fb8f2ce0640 1 > prepared multipart upload key=P8OYS5ZE1NX0 upload=MQENO2VGG162 tag=J1XVCJG18GUH type=write size=1158325 2024-09-13T00:00:53.668 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.666+0000 7fb8f2ce0640 1 < completed part key=_multipart_7OX5OJZ8GXU5.2~JU90U8Z1VRYK.0 size=1048576 2024-09-13T00:00:53.668 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.666+0000 7fb8f2ce0640 1 < canceled part key=_multipart_7OX5OJZ8GXU5.2~JU90U8Z1VRYK.1 size=1048576 2024-09-13T00:00:53.673 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.674+0000 7fb8f2ce0640 1 < completed part key=_multipart_7OX5OJZ8GXU5.2~JU90U8Z1VRYK.2 size=703815 2024-09-13T00:00:53.686 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.686+0000 7fb8f2ce0640 1 < canceled multipart upload key=7OX5OJZ8GXU5 upload=JU90U8Z1VRYK tag=LATNRXIQCR4C type=write size=3849543 2024-09-13T00:00:53.696 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.694+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~OW1FNRLEYSE1.0 size=1048576 2024-09-13T00:00:53.701 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.702+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~OW1FNRLEYSE1.1 size=960970 2024-09-13T00:00:53.701 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.702+0000 7fb8f2ce0640 1 > prepared multipart upload key=70F7D20HI504 upload=OW1FNRLEYSE1 tag=Q9IUSZ0P9JQQ type=write size=3058122 2024-09-13T00:00:53.707 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.706+0000 7fb8f2ce0640 1 < completed part key=_multipart_QMZREZ1QEDNJ.2~A13R946BIE5C.0 size=1048576 2024-09-13T00:00:53.712 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.710+0000 7fb8f2ce0640 1 < completed part key=_multipart_QMZREZ1QEDNJ.2~A13R946BIE5C.1 size=1048576 2024-09-13T00:00:53.717 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.718+0000 7fb8f2ce0640 1 < completed part key=_multipart_QMZREZ1QEDNJ.2~A13R946BIE5C.2 size=77906 2024-09-13T00:00:53.727 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.726+0000 7fb8f2ce0640 1 < completed multipart upload key=QMZREZ1QEDNJ upload=A13R946BIE5C tag=QQPMS6KO04Y8 type=write size=3223634 2024-09-13T00:00:53.732 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.730+0000 7fb8f2ce0640 1 > prepared operation key=0WC6P8YNL0H2 tag=YE8RG391UII4 type=del size=2534840 2024-09-13T00:00:53.740 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.738+0000 7fb8f2ce0640 -1 < failed to complete operation key=EUE3LT0PDIXV tag=NFXZJVYYTUPS type=del size=1498227: (22) Invalid argument 2024-09-13T00:00:53.751 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.750+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~90ZQWFHOFSXJ.0 size=1048576 2024-09-13T00:00:53.757 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.754+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~90ZQWFHOFSXJ.1 size=891083 2024-09-13T00:00:53.757 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.754+0000 7fb8f2ce0640 1 > prepared multipart upload key=PVYZQLGYT3XU upload=90ZQWFHOFSXJ tag=RTJ5Z6PKTUIZ type=write size=2988235 2024-09-13T00:00:53.762 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.762+0000 7fb8f2ce0640 1 < completed delete operation key=H4M1TCN5W6XU tag=8X53LP5RPRHL type=del 2024-09-13T00:00:53.774 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.774+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~0TOJJ6IW19ND.0 size=1048576 2024-09-13T00:00:53.780 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.778+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~0TOJJ6IW19ND.1 size=1048576 2024-09-13T00:00:53.786 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.786+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~0TOJJ6IW19ND.2 size=324128 2024-09-13T00:00:53.786 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.786+0000 7fb8f2ce0640 1 > prepared multipart upload key=1HX06CCZJ1DD upload=0TOJJ6IW19ND tag=YQO8P0WH5YFR type=write size=3469856 2024-09-13T00:00:53.792 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.790+0000 7fb8f2ce0640 1 < completed part key=_multipart_EJN6HR8I7XQF.2~X5X5RLMCROT8.0 size=1048576 2024-09-13T00:00:53.798 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.798+0000 7fb8f2ce0640 1 < completed part key=_multipart_EJN6HR8I7XQF.2~X5X5RLMCROT8.1 size=1048576 2024-09-13T00:00:53.803 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.802+0000 7fb8f2ce0640 1 < completed part key=_multipart_EJN6HR8I7XQF.2~X5X5RLMCROT8.2 size=635303 2024-09-13T00:00:53.814 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.814+0000 7fb8f2ce0640 1 < completed multipart upload key=EJN6HR8I7XQF upload=X5X5RLMCROT8 tag=A1HBMLDIIECU type=write size=3781031 2024-09-13T00:00:53.821 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.822+0000 7fb8f2ce0640 1 > prepared operation key=ZTK9AYBSLCOZ tag=XHPPU635APMI type=del size=2518250 2024-09-13T00:00:53.833 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.834+0000 7fb8f2ce0640 1 < suggested update operation key=J8NGDSF9XRTP tag=IOYFW1KVJBL8 type=del size=924411 2024-09-13T00:00:53.839 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.838+0000 7fb8f2ce0640 1 > prepared operation key=J8NGDSF9XRTP tag=DAFYVBWJ36R9 type=del size=3501489 2024-09-13T00:00:53.846 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.846+0000 7fb8f2ce0640 1 < completed part key=_multipart_85YEW8N43FTA.2~6TJQJJZ617EO.0 size=1048576 2024-09-13T00:00:53.851 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.850+0000 7fb8f2ce0640 1 < completed part key=_multipart_85YEW8N43FTA.2~6TJQJJZ617EO.1 size=1750 2024-09-13T00:00:53.863 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.862+0000 7fb8f2ce0640 1 < completed multipart upload key=85YEW8N43FTA upload=6TJQJJZ617EO tag=9HGU1MQ9JCJO type=write size=2098902 2024-09-13T00:00:53.874 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.874+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~2EQJXD2SLTTJ.0 size=1048576 2024-09-13T00:00:53.879 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.878+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~2EQJXD2SLTTJ.1 size=386208 2024-09-13T00:00:53.879 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.878+0000 7fb8f2ce0640 1 > prepared multipart upload key=NQJQC8CEAC15 upload=2EQJXD2SLTTJ tag=3ZM7RN12V5WO type=write size=2483360 2024-09-13T00:00:53.885 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.886+0000 7fb8f2ce0640 1 < completed write operation key=ZFHN2SNTZ5W7 tag=7FNO1AZ5P043 type=write size=501018 2024-09-13T00:00:53.898 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.898+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~DV4FDN2IPDG4.0 size=1048576 2024-09-13T00:00:53.904 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.902+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~DV4FDN2IPDG4.1 size=19833 2024-09-13T00:00:53.904 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.902+0000 7fb8f2ce0640 1 > prepared multipart upload key=70F7D20HI504 upload=DV4FDN2IPDG4 tag=20MS5FAJAPU3 type=write size=2116985 2024-09-13T00:00:53.910 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.910+0000 7fb8f2ce0640 1 < completed part key=_multipart_F0QCVR9E08WE.2~OLJHF86ZWMSC.0 size=931077 2024-09-13T00:00:53.924 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.922+0000 7fb8f2ce0640 1 < completed multipart upload key=F0QCVR9E08WE upload=OLJHF86ZWMSC tag=DXOJR5QBSFUY type=write size=1979653 2024-09-13T00:00:53.937 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.938+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H9AU4HD247YU.2~7WS0RBH2IMV6.0 size=730048 2024-09-13T00:00:53.937 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.938+0000 7fb8f2ce0640 1 > prepared multipart upload key=H9AU4HD247YU upload=7WS0RBH2IMV6 tag=ES1DJD8UQJ9D type=write size=1778624 2024-09-13T00:00:53.948 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.946+0000 7fb8f2ce0640 1 < suggested remove operation key=FX0NCGLK0KPJ tag=UXLMT9E7RDZL type=del 2024-09-13T00:00:53.959 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.958+0000 7fb8f2ce0640 1 > prepared part key=_multipart_O4MZ4UWNXME9.2~2KHHPNIBG54F.0 size=166742 2024-09-13T00:00:53.959 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.958+0000 7fb8f2ce0640 1 > prepared multipart upload key=O4MZ4UWNXME9 upload=2KHHPNIBG54F tag=3XHEHJYTRCKY type=write size=1215318 2024-09-13T00:00:53.965 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.966+0000 7fb8f2ce0640 1 < completed write operation key=W8XZ26311RJX tag=U4E1UV0CNOD1 type=write size=637242 2024-09-13T00:00:53.970 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.970+0000 7fb8f2ce0640 1 > prepared operation key=W8XZ26311RJX tag=V3DZWAZGG4AO type=del size=1126024 2024-09-13T00:00:53.981 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.982+0000 7fb8f2ce0640 1 < suggested remove operation key=H5EIP0E9CBWX tag=0D3U02O25286 type=del 2024-09-13T00:00:53.987 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.986+0000 7fb8f2ce0640 1 > prepared operation key=F0QCVR9E08WE tag=06FEC9861HO0 type=del size=250201 2024-09-13T00:00:53.993 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.994+0000 7fb8f2ce0640 1 < completed part key=_multipart_F0QCVR9E08WE.2~SWIMRTXHPP51.0 size=1048576 2024-09-13T00:00:53.999 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:53.998+0000 7fb8f2ce0640 1 < completed part key=_multipart_F0QCVR9E08WE.2~SWIMRTXHPP51.1 size=1048576 2024-09-13T00:00:54.003 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.002+0000 7fb8f2ce0640 1 < completed part key=_multipart_F0QCVR9E08WE.2~SWIMRTXHPP51.2 size=643459 2024-09-13T00:00:54.013 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.014+0000 7fb8f2ce0640 1 < completed multipart upload key=F0QCVR9E08WE upload=SWIMRTXHPP51 tag=B4TSAKTL0LQX type=write size=3789187 2024-09-13T00:00:54.019 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.018+0000 7fb8f2ce0640 1 > prepared operation key=7OX5OJZ8GXU5 tag=1SG5F7XEDUGW type=del size=1851727 2024-09-13T00:00:54.025 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.026+0000 7fb8f2ce0640 1 < completed part key=_multipart_44EDYFT9VF8Y.2~HPKDMQTGKK7Z.0 size=889220 2024-09-13T00:00:54.036 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.034+0000 7fb8f2ce0640 1 < completed multipart upload key=44EDYFT9VF8Y upload=HPKDMQTGKK7Z tag=2M91WNA3EA09 type=write size=1937796 2024-09-13T00:00:54.046 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.046+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2TWD9YLMSYJI.2~JSPFVC9WBZV9.0 size=876687 2024-09-13T00:00:54.046 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.046+0000 7fb8f2ce0640 1 > prepared multipart upload key=2TWD9YLMSYJI upload=JSPFVC9WBZV9 tag=Q7VTKZL7W4DV type=write size=1925263 2024-09-13T00:00:54.052 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.050+0000 7fb8f2ce0640 1 < completed delete operation key=5GM88ERM89KT tag=X8VG2OGV26FM type=del 2024-09-13T00:00:54.058 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.058+0000 7fb8f2ce0640 1 > prepared operation key=EJN6HR8I7XQF tag=MVE0GF2B3499 type=del size=3246288 2024-09-13T00:00:54.065 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.066+0000 7fb8f2ce0640 1 < completed part key=_multipart_P8OYS5ZE1NX0.2~MQENO2VGG162.0 size=109749 2024-09-13T00:00:54.075 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.074+0000 7fb8f2ce0640 1 < completed multipart upload key=P8OYS5ZE1NX0 upload=MQENO2VGG162 tag=J1XVCJG18GUH type=write size=1158325 2024-09-13T00:00:54.087 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.086+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H5EIP0E9CBWX.2~5UG5CGF7IJKK.0 size=426642 2024-09-13T00:00:54.087 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.086+0000 7fb8f2ce0640 1 > prepared multipart upload key=H5EIP0E9CBWX upload=5UG5CGF7IJKK tag=LWKDBITMXMOY type=write size=1475218 2024-09-13T00:00:54.094 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.094+0000 7fb8f2ce0640 1 < completed part key=_multipart_70F7D20HI504.2~OW1FNRLEYSE1.0 size=1048576 2024-09-13T00:00:54.099 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.098+0000 7fb8f2ce0640 1 < completed part key=_multipart_70F7D20HI504.2~OW1FNRLEYSE1.1 size=960970 2024-09-13T00:00:54.110 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.110+0000 7fb8f2ce0640 1 < completed multipart upload key=70F7D20HI504 upload=OW1FNRLEYSE1 tag=Q9IUSZ0P9JQQ type=write size=3058122 2024-09-13T00:00:54.115 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.114+0000 7fb8f2ce0640 1 > prepared operation key=70F7D20HI504 tag=0A6J8Z9LU7MQ type=del size=407332 2024-09-13T00:00:54.121 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.122+0000 7fb8f2ce0640 1 < canceled operation key=0WC6P8YNL0H2 tag=YE8RG391UII4 type=del size=2534840 2024-09-13T00:00:54.127 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.126+0000 7fb8f2ce0640 1 > prepared operation key=8Z6U7OOA0GBH tag=BIGWZPM31NV0 type=del size=1196495 2024-09-13T00:00:54.137 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.134+0000 7fb8f2ce0640 1 < completed part key=_multipart_PVYZQLGYT3XU.2~90ZQWFHOFSXJ.0 size=1048576 2024-09-13T00:00:54.141 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.142+0000 7fb8f2ce0640 1 < completed part key=_multipart_PVYZQLGYT3XU.2~90ZQWFHOFSXJ.1 size=891083 2024-09-13T00:00:54.152 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.150+0000 7fb8f2ce0640 1 < completed multipart upload key=PVYZQLGYT3XU upload=90ZQWFHOFSXJ tag=RTJ5Z6PKTUIZ type=write size=2988235 2024-09-13T00:00:54.163 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.162+0000 7fb8f2ce0640 1 > prepared part key=_multipart_5GM88ERM89KT.2~2D55ZNCCLB2X.0 size=1048576 2024-09-13T00:00:54.168 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.166+0000 7fb8f2ce0640 1 > prepared part key=_multipart_5GM88ERM89KT.2~2D55ZNCCLB2X.1 size=833664 2024-09-13T00:00:54.169 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.166+0000 7fb8f2ce0640 1 > prepared multipart upload key=5GM88ERM89KT upload=2D55ZNCCLB2X tag=V5P6IXQ3KG30 type=write size=2930816 2024-09-13T00:00:54.175 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.174+0000 7fb8f2ce0640 1 < completed part key=_multipart_1HX06CCZJ1DD.2~0TOJJ6IW19ND.0 size=1048576 2024-09-13T00:00:54.180 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.178+0000 7fb8f2ce0640 1 < completed part key=_multipart_1HX06CCZJ1DD.2~0TOJJ6IW19ND.1 size=1048576 2024-09-13T00:00:54.185 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.182+0000 7fb8f2ce0640 1 < completed part key=_multipart_1HX06CCZJ1DD.2~0TOJJ6IW19ND.2 size=324128 2024-09-13T00:00:54.196 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.194+0000 7fb8f2ce0640 1 < completed multipart upload key=1HX06CCZJ1DD upload=0TOJJ6IW19ND tag=YQO8P0WH5YFR type=write size=3469856 2024-09-13T00:00:54.207 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.206+0000 7fb8f2ce0640 1 > prepared part key=_multipart_N3Z5XIG3TZRQ.2~15Q0O3OURDEC.0 size=1048576 2024-09-13T00:00:54.212 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.210+0000 7fb8f2ce0640 1 > prepared part key=_multipart_N3Z5XIG3TZRQ.2~15Q0O3OURDEC.1 size=1048576 2024-09-13T00:00:54.217 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.218+0000 7fb8f2ce0640 1 > prepared part key=_multipart_N3Z5XIG3TZRQ.2~15Q0O3OURDEC.2 size=1007942 2024-09-13T00:00:54.218 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.218+0000 7fb8f2ce0640 1 > prepared multipart upload key=N3Z5XIG3TZRQ upload=15Q0O3OURDEC tag=RP8OLD1D7AWF type=write size=4153670 2024-09-13T00:00:54.224 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.222+0000 7fb8f2ce0640 1 < completed delete operation key=ZTK9AYBSLCOZ tag=XHPPU635APMI type=del 2024-09-13T00:00:54.230 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.230+0000 7fb8f2ce0640 1 > prepared operation key=63FJDWBO2T9S tag=MIOAMNMCVQ2E type=del size=21504 2024-09-13T00:00:54.236 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.234+0000 7fb8f2ce0640 1 < completed delete operation key=J8NGDSF9XRTP tag=DAFYVBWJ36R9 type=del 2024-09-13T00:00:54.247 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.246+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~JR6R95UKL319.0 size=1048576 2024-09-13T00:00:54.252 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.250+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~JR6R95UKL319.1 size=1048576 2024-09-13T00:00:54.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:53 smithi136 bash[75164]: audit 2024-09-13T00:00:53.770422+0000 mon.a (mon.0) 2079 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:00:54.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:53 smithi136 bash[76371]: audit 2024-09-13T00:00:53.770422+0000 mon.a (mon.0) 2079 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:00:54.258 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.258+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~JR6R95UKL319.2 size=852390 2024-09-13T00:00:54.258 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.258+0000 7fb8f2ce0640 1 > prepared multipart upload key=70F7D20HI504 upload=JR6R95UKL319 tag=JLUP7L76S1XO type=write size=3998118 2024-09-13T00:00:54.263 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.262+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~2EQJXD2SLTTJ.0 size=1048576 2024-09-13T00:00:54.268 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.266+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~2EQJXD2SLTTJ.1 size=386208 2024-09-13T00:00:54.279 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.278+0000 7fb8f2ce0640 1 < completed multipart upload key=NQJQC8CEAC15 upload=2EQJXD2SLTTJ tag=3ZM7RN12V5WO type=write size=2483360 2024-09-13T00:00:54.286 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.286+0000 7fb8f2ce0640 1 > prepared operation key=ZFHN2SNTZ5W7 tag=MCOPK5F2PFRE type=del size=1259798 2024-09-13T00:00:54.292 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.290+0000 7fb8f2ce0640 1 < completed part key=_multipart_70F7D20HI504.2~DV4FDN2IPDG4.0 size=1048576 2024-09-13T00:00:54.297 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.298+0000 7fb8f2ce0640 1 < completed part key=_multipart_70F7D20HI504.2~DV4FDN2IPDG4.1 size=19833 2024-09-13T00:00:54.307 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.306+0000 7fb8f2ce0640 1 < completed multipart upload key=70F7D20HI504 upload=DV4FDN2IPDG4 tag=20MS5FAJAPU3 type=write size=2116985 2024-09-13T00:00:54.318 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.318+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~BWH771LK0U8B.0 size=1048576 2024-09-13T00:00:54.323 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.322+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~BWH771LK0U8B.1 size=211071 2024-09-13T00:00:54.324 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.322+0000 7fb8f2ce0640 1 > prepared multipart upload key=1HX06CCZJ1DD upload=BWH771LK0U8B tag=UCDXL5LE1K3D type=write size=2308223 2024-09-13T00:00:54.330 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.330+0000 7fb8f2ce0640 1 < completed part key=_multipart_H9AU4HD247YU.2~7WS0RBH2IMV6.0 size=730048 2024-09-13T00:00:54.340 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.338+0000 7fb8f2ce0640 1 < completed multipart upload key=H9AU4HD247YU upload=7WS0RBH2IMV6 tag=ES1DJD8UQJ9D type=write size=1778624 2024-09-13T00:00:54.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:53 smithi181 bash[67587]: audit 2024-09-13T00:00:53.770422+0000 mon.a (mon.0) 2079 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:00:54.353 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.354+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~SZ6A8P7V87QD.0 size=1048576 2024-09-13T00:00:54.358 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.358+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~SZ6A8P7V87QD.1 size=1048576 2024-09-13T00:00:54.362 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.362+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~SZ6A8P7V87QD.2 size=809059 2024-09-13T00:00:54.363 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.362+0000 7fb8f2ce0640 1 > prepared multipart upload key=FX0NCGLK0KPJ upload=SZ6A8P7V87QD tag=CAMYI8XF2JDX type=write size=3954787 2024-09-13T00:00:54.369 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.366+0000 7fb8f2ce0640 1 < completed part key=_multipart_O4MZ4UWNXME9.2~2KHHPNIBG54F.0 size=166742 2024-09-13T00:00:54.381 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.378+0000 7fb8f2ce0640 1 < completed multipart upload key=O4MZ4UWNXME9 upload=2KHHPNIBG54F tag=3XHEHJYTRCKY type=write size=1215318 2024-09-13T00:00:54.387 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.386+0000 7fb8f2ce0640 1 > prepared operation key=H5EIP0E9CBWX tag=1UIIFWOWOM62 type=write size=678148 2024-09-13T00:00:54.394 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.394+0000 7fb8f2ce0640 1 < canceled operation key=W8XZ26311RJX tag=V3DZWAZGG4AO type=del size=1126024 2024-09-13T00:00:54.406 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.406+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EJN6HR8I7XQF.2~N0X5QCD8Z54Y.0 size=1048576 2024-09-13T00:00:54.411 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.410+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EJN6HR8I7XQF.2~N0X5QCD8Z54Y.1 size=1048576 2024-09-13T00:00:54.415 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.414+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EJN6HR8I7XQF.2~N0X5QCD8Z54Y.2 size=435259 2024-09-13T00:00:54.415 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.414+0000 7fb8f2ce0640 1 > prepared multipart upload key=EJN6HR8I7XQF upload=N0X5QCD8Z54Y tag=9WILL8GQOLV0 type=write size=3580987 2024-09-13T00:00:54.422 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.422+0000 7fb8f2ce0640 1 < completed delete operation key=F0QCVR9E08WE tag=06FEC9861HO0 type=del 2024-09-13T00:00:54.427 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.426+0000 7fb8f2ce0640 1 > prepared operation key=4K76U69GZ9AE tag=ZBRGNEA3963A type=del size=432786 2024-09-13T00:00:54.433 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.434+0000 7fb8f2ce0640 1 < completed delete operation key=7OX5OJZ8GXU5 tag=1SG5F7XEDUGW type=del 2024-09-13T00:00:54.439 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.438+0000 7fb8f2ce0640 1 > prepared operation key=J8NGDSF9XRTP tag=72BORAY1QS1H type=del size=330533 2024-09-13T00:00:54.445 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.442+0000 7fb8f2ce0640 1 < completed part key=_multipart_2TWD9YLMSYJI.2~JSPFVC9WBZV9.0 size=876687 2024-09-13T00:00:54.455 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.454+0000 7fb8f2ce0640 1 < completed multipart upload key=2TWD9YLMSYJI upload=JSPFVC9WBZV9 tag=Q7VTKZL7W4DV type=write size=1925263 2024-09-13T00:00:54.466 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.466+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~PCZWBUJNGZDK.0 size=1048576 2024-09-13T00:00:54.471 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.470+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~PCZWBUJNGZDK.1 size=259313 2024-09-13T00:00:54.471 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.470+0000 7fb8f2ce0640 1 > prepared multipart upload key=W8XZ26311RJX upload=PCZWBUJNGZDK tag=78G53TKNSFGC type=write size=2356465 2024-09-13T00:00:54.483 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.482+0000 7fb8f2ce0640 1 < suggested update operation key=EJN6HR8I7XQF tag=MVE0GF2B3499 type=del size=3246288 2024-09-13T00:00:54.494 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.494+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~5J2BYO7M7MQZ.0 size=620894 2024-09-13T00:00:54.494 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.494+0000 7fb8f2ce0640 1 > prepared multipart upload key=PVYZQLGYT3XU upload=5J2BYO7M7MQZ tag=Q1092E6G1YM8 type=write size=1669470 2024-09-13T00:00:54.500 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.498+0000 7fb8f2ce0640 1 < completed part key=_multipart_H5EIP0E9CBWX.2~5UG5CGF7IJKK.0 size=426642 2024-09-13T00:00:54.512 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.510+0000 7fb8f2ce0640 1 < completed multipart upload key=H5EIP0E9CBWX upload=5UG5CGF7IJKK tag=LWKDBITMXMOY type=write size=1475218 2024-09-13T00:00:54.523 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.522+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~09DXO0DGI6EL.0 size=1048576 2024-09-13T00:00:54.528 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.526+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~09DXO0DGI6EL.1 size=1048576 2024-09-13T00:00:54.534 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.534+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~09DXO0DGI6EL.2 size=738130 2024-09-13T00:00:54.534 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.534+0000 7fb8f2ce0640 1 > prepared multipart upload key=8MPUVD37EDN9 upload=09DXO0DGI6EL tag=Z08Z011Y8G4X type=write size=3883858 2024-09-13T00:00:54.540 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.538+0000 7fb8f2ce0640 1 < completed delete operation key=70F7D20HI504 tag=0A6J8Z9LU7MQ type=del 2024-09-13T00:00:54.546 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.546+0000 7fb8f2ce0640 1 > prepared operation key=J8NGDSF9XRTP tag=84KYBZQLT81N type=del size=1120638 2024-09-13T00:00:54.552 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.550+0000 7fb8f2ce0640 1 < completed delete operation key=8Z6U7OOA0GBH tag=BIGWZPM31NV0 type=del 2024-09-13T00:00:54.558 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.558+0000 7fb8f2ce0640 1 > prepared operation key=2AHYRZUDFHP4 tag=2AR0ZL6SL7PG type=del size=1927994 2024-09-13T00:00:54.566 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.566+0000 7fb8f2ce0640 1 < completed part key=_multipart_5GM88ERM89KT.2~2D55ZNCCLB2X.0 size=1048576 2024-09-13T00:00:54.571 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.570+0000 7fb8f2ce0640 1 < completed part key=_multipart_5GM88ERM89KT.2~2D55ZNCCLB2X.1 size=833664 2024-09-13T00:00:54.581 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.582+0000 7fb8f2ce0640 1 < completed multipart upload key=5GM88ERM89KT upload=2D55ZNCCLB2X tag=V5P6IXQ3KG30 type=write size=2930816 2024-09-13T00:00:54.593 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.590+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~O06XIYVVEJ6C.0 size=1048576 2024-09-13T00:00:54.597 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.598+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~O06XIYVVEJ6C.1 size=1048576 2024-09-13T00:00:54.604 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.602+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~O06XIYVVEJ6C.2 size=372835 2024-09-13T00:00:54.604 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.602+0000 7fb8f2ce0640 1 > prepared multipart upload key=W8XZ26311RJX upload=O06XIYVVEJ6C tag=QFH0YBMDS5LR type=write size=3518563 2024-09-13T00:00:54.611 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.610+0000 7fb8f2ce0640 1 < completed part key=_multipart_N3Z5XIG3TZRQ.2~15Q0O3OURDEC.0 size=1048576 2024-09-13T00:00:54.611 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.610+0000 7fb8f2ce0640 1 < canceled part key=_multipart_N3Z5XIG3TZRQ.2~15Q0O3OURDEC.1 size=1048576 2024-09-13T00:00:54.618 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.618+0000 7fb8f2ce0640 1 < completed part key=_multipart_N3Z5XIG3TZRQ.2~15Q0O3OURDEC.2 size=1007942 2024-09-13T00:00:54.630 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.630+0000 7fb8f2ce0640 1 < canceled multipart upload key=N3Z5XIG3TZRQ upload=15Q0O3OURDEC tag=RP8OLD1D7AWF type=write size=4153670 2024-09-13T00:00:54.642 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.642+0000 7fb8f2ce0640 1 > prepared part key=_multipart_85YEW8N43FTA.2~GHCE7G4X7A6P.0 size=1048576 2024-09-13T00:00:54.648 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.646+0000 7fb8f2ce0640 1 > prepared part key=_multipart_85YEW8N43FTA.2~GHCE7G4X7A6P.1 size=366680 2024-09-13T00:00:54.648 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.646+0000 7fb8f2ce0640 1 > prepared multipart upload key=85YEW8N43FTA upload=GHCE7G4X7A6P tag=7YHH3ZPL9PFV type=write size=2463832 2024-09-13T00:00:54.654 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.654+0000 7fb8f2ce0640 1 < completed delete operation key=63FJDWBO2T9S tag=MIOAMNMCVQ2E type=del 2024-09-13T00:00:54.660 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.658+0000 7fb8f2ce0640 1 > prepared operation key=8Z6U7OOA0GBH tag=8JBLPTNBV2Y1 type=del size=710278 2024-09-13T00:00:54.666 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.666+0000 7fb8f2ce0640 1 < completed part key=_multipart_70F7D20HI504.2~JR6R95UKL319.0 size=1048576 2024-09-13T00:00:54.671 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.670+0000 7fb8f2ce0640 1 < completed part key=_multipart_70F7D20HI504.2~JR6R95UKL319.1 size=1048576 2024-09-13T00:00:54.677 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.678+0000 7fb8f2ce0640 1 < completed part key=_multipart_70F7D20HI504.2~JR6R95UKL319.2 size=852390 2024-09-13T00:00:54.688 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.686+0000 7fb8f2ce0640 1 < completed multipart upload key=70F7D20HI504 upload=JR6R95UKL319 tag=JLUP7L76S1XO type=write size=3998118 2024-09-13T00:00:54.693 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.694+0000 7fb8f2ce0640 1 > prepared operation key=EUE3LT0PDIXV tag=OKYMW1K7XYZD type=del size=2678670 2024-09-13T00:00:54.701 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.698+0000 7fb8f2ce0640 1 < completed delete operation key=ZFHN2SNTZ5W7 tag=MCOPK5F2PFRE type=del 2024-09-13T00:00:54.710 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.710+0000 7fb8f2ce0640 1 > prepared operation key=1HX06CCZJ1DD tag=CETT48N0WKO1 type=del size=1511217 2024-09-13T00:00:54.717 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.714+0000 7fb8f2ce0640 1 < completed part key=_multipart_1HX06CCZJ1DD.2~BWH771LK0U8B.0 size=1048576 2024-09-13T00:00:54.722 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.722+0000 7fb8f2ce0640 1 < completed part key=_multipart_1HX06CCZJ1DD.2~BWH771LK0U8B.1 size=211071 2024-09-13T00:00:54.731 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.730+0000 7fb8f2ce0640 1 < completed multipart upload key=1HX06CCZJ1DD upload=BWH771LK0U8B tag=UCDXL5LE1K3D type=write size=2308223 2024-09-13T00:00:54.737 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.738+0000 7fb8f2ce0640 1 > prepared operation key=7OX5OJZ8GXU5 tag=O5KEYYHTLVTZ type=del size=3966374 2024-09-13T00:00:54.743 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.742+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~SZ6A8P7V87QD.0 size=1048576 2024-09-13T00:00:54.748 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.746+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~SZ6A8P7V87QD.1 size=1048576 2024-09-13T00:00:54.752 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.750+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~SZ6A8P7V87QD.2 size=809059 2024-09-13T00:00:54.762 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.762+0000 7fb8f2ce0640 1 < completed multipart upload key=FX0NCGLK0KPJ upload=SZ6A8P7V87QD tag=CAMYI8XF2JDX type=write size=3954787 2024-09-13T00:00:54.773 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.774+0000 7fb8f2ce0640 1 > prepared part key=_multipart_N3Z5XIG3TZRQ.2~DP2D03A9927U.0 size=1048576 2024-09-13T00:00:54.778 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.778+0000 7fb8f2ce0640 1 > prepared part key=_multipart_N3Z5XIG3TZRQ.2~DP2D03A9927U.1 size=476594 2024-09-13T00:00:54.778 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.778+0000 7fb8f2ce0640 1 > prepared multipart upload key=N3Z5XIG3TZRQ upload=DP2D03A9927U tag=Y5OR10IDE82C type=write size=2573746 2024-09-13T00:00:54.791 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.790+0000 7fb8f2ce0640 1 < suggested update operation key=H5EIP0E9CBWX tag=1UIIFWOWOM62 type=write size=678148 2024-09-13T00:00:54.802 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.802+0000 7fb8f2ce0640 1 > prepared part key=_multipart_QMZREZ1QEDNJ.2~03IRDZ768BDW.0 size=1048576 2024-09-13T00:00:54.807 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.806+0000 7fb8f2ce0640 1 > prepared part key=_multipart_QMZREZ1QEDNJ.2~03IRDZ768BDW.1 size=1048576 2024-09-13T00:00:54.812 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.810+0000 7fb8f2ce0640 1 > prepared part key=_multipart_QMZREZ1QEDNJ.2~03IRDZ768BDW.2 size=451531 2024-09-13T00:00:54.812 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.810+0000 7fb8f2ce0640 1 > prepared multipart upload key=QMZREZ1QEDNJ upload=03IRDZ768BDW tag=0I2CNZ34WEDO type=write size=3597259 2024-09-13T00:00:54.818 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.818+0000 7fb8f2ce0640 1 < completed part key=_multipart_EJN6HR8I7XQF.2~N0X5QCD8Z54Y.0 size=1048576 2024-09-13T00:00:54.823 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.822+0000 7fb8f2ce0640 1 < completed part key=_multipart_EJN6HR8I7XQF.2~N0X5QCD8Z54Y.1 size=1048576 2024-09-13T00:00:54.823 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.822+0000 7fb8f2ce0640 1 < canceled part key=_multipart_EJN6HR8I7XQF.2~N0X5QCD8Z54Y.2 size=435259 2024-09-13T00:00:54.832 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.830+0000 7fb8f2ce0640 1 < canceled multipart upload key=EJN6HR8I7XQF upload=N0X5QCD8Z54Y tag=9WILL8GQOLV0 type=write size=3580987 2024-09-13T00:00:54.838 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.838+0000 7fb8f2ce0640 1 > prepared operation key=8MPUVD37EDN9 tag=S8CBBYV7SEKM type=del size=2426338 2024-09-13T00:00:54.844 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.842+0000 7fb8f2ce0640 1 < completed delete operation key=4K76U69GZ9AE tag=ZBRGNEA3963A type=del 2024-09-13T00:00:54.854 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.854+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~ZVB27GR2LUTP.0 size=705857 2024-09-13T00:00:54.854 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.854+0000 7fb8f2ce0640 1 > prepared multipart upload key=PVYZQLGYT3XU upload=ZVB27GR2LUTP tag=1ZOS2JUO8B2M type=write size=1754433 2024-09-13T00:00:54.860 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.858+0000 7fb8f2ce0640 1 < completed delete operation key=J8NGDSF9XRTP tag=72BORAY1QS1H type=del 2024-09-13T00:00:54.865 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.866+0000 7fb8f2ce0640 1 > prepared operation key=2TWD9YLMSYJI tag=1Q072YO24BKY type=del size=1697688 2024-09-13T00:00:54.872 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.870+0000 7fb8f2ce0640 1 < completed part key=_multipart_W8XZ26311RJX.2~PCZWBUJNGZDK.0 size=1048576 2024-09-13T00:00:54.878 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.878+0000 7fb8f2ce0640 1 < completed part key=_multipart_W8XZ26311RJX.2~PCZWBUJNGZDK.1 size=259313 2024-09-13T00:00:54.888 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.886+0000 7fb8f2ce0640 1 < completed multipart upload key=W8XZ26311RJX upload=PCZWBUJNGZDK tag=78G53TKNSFGC type=write size=2356465 2024-09-13T00:00:54.901 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.898+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~9I48CZBO470J.0 size=1048576 2024-09-13T00:00:54.906 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.906+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~9I48CZBO470J.1 size=1048576 2024-09-13T00:00:54.911 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.910+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~9I48CZBO470J.2 size=749032 2024-09-13T00:00:54.911 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.910+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZTK9AYBSLCOZ upload=9I48CZBO470J tag=KDFKJNXJD2RR type=write size=3894760 2024-09-13T00:00:54.918 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.918+0000 7fb8f2ce0640 1 < completed part key=_multipart_PVYZQLGYT3XU.2~5J2BYO7M7MQZ.0 size=620894 2024-09-13T00:00:54.929 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.930+0000 7fb8f2ce0640 1 < completed multipart upload key=PVYZQLGYT3XU upload=5J2BYO7M7MQZ tag=Q1092E6G1YM8 type=write size=1669470 2024-09-13T00:00:54.936 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.934+0000 7fb8f2ce0640 1 > prepared operation key=8Z6U7OOA0GBH tag=BEXXGN1HWZRR type=del size=2812362 2024-09-13T00:00:54.942 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.942+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~09DXO0DGI6EL.0 size=1048576 2024-09-13T00:00:54.947 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.946+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~09DXO0DGI6EL.1 size=1048576 2024-09-13T00:00:54.951 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.950+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~09DXO0DGI6EL.2 size=738130 2024-09-13T00:00:54.963 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.962+0000 7fb8f2ce0640 1 < completed multipart upload key=8MPUVD37EDN9 upload=09DXO0DGI6EL tag=Z08Z011Y8G4X type=write size=3883858 2024-09-13T00:00:54.969 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.970+0000 7fb8f2ce0640 1 > prepared operation key=EJN6HR8I7XQF tag=HNMM4S703X7L type=del size=2102030 2024-09-13T00:00:54.975 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.974+0000 7fb8f2ce0640 1 < completed delete operation key=J8NGDSF9XRTP tag=84KYBZQLT81N type=del 2024-09-13T00:00:54.982 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.982+0000 7fb8f2ce0640 1 > prepared operation key=P8OYS5ZE1NX0 tag=ZWR8YXAAJG64 type=write size=863081 2024-09-13T00:00:54.989 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.986+0000 7fb8f2ce0640 1 < completed delete operation key=2AHYRZUDFHP4 tag=2AR0ZL6SL7PG type=del 2024-09-13T00:00:54.995 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.994+0000 7fb8f2ce0640 1 > prepared operation key=P8OYS5ZE1NX0 tag=TDEESE1OGST1 type=write size=243344 2024-09-13T00:00:55.000 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:54.998+0000 7fb8f2ce0640 1 < completed part key=_multipart_W8XZ26311RJX.2~O06XIYVVEJ6C.0 size=1048576 2024-09-13T00:00:55.005 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.006+0000 7fb8f2ce0640 1 < completed part key=_multipart_W8XZ26311RJX.2~O06XIYVVEJ6C.1 size=1048576 2024-09-13T00:00:55.010 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.010+0000 7fb8f2ce0640 1 < completed part key=_multipart_W8XZ26311RJX.2~O06XIYVVEJ6C.2 size=372835 2024-09-13T00:00:55.020 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.018+0000 7fb8f2ce0640 1 < completed multipart upload key=W8XZ26311RJX upload=O06XIYVVEJ6C tag=QFH0YBMDS5LR type=write size=3518563 2024-09-13T00:00:55.030 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.030+0000 7fb8f2ce0640 1 > prepared part key=_multipart_44EDYFT9VF8Y.2~7IVIF6LIUL5W.0 size=1048576 2024-09-13T00:00:55.035 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.034+0000 7fb8f2ce0640 1 > prepared part key=_multipart_44EDYFT9VF8Y.2~7IVIF6LIUL5W.1 size=1048576 2024-09-13T00:00:55.041 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.038+0000 7fb8f2ce0640 1 > prepared part key=_multipart_44EDYFT9VF8Y.2~7IVIF6LIUL5W.2 size=588490 2024-09-13T00:00:55.041 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.038+0000 7fb8f2ce0640 1 > prepared multipart upload key=44EDYFT9VF8Y upload=7IVIF6LIUL5W tag=7SPZFLABGDKA type=write size=3734218 2024-09-13T00:00:55.046 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.046+0000 7fb8f2ce0640 1 < completed part key=_multipart_85YEW8N43FTA.2~GHCE7G4X7A6P.0 size=1048576 2024-09-13T00:00:55.052 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.050+0000 7fb8f2ce0640 1 < completed part key=_multipart_85YEW8N43FTA.2~GHCE7G4X7A6P.1 size=366680 2024-09-13T00:00:55.062 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.062+0000 7fb8f2ce0640 1 < completed multipart upload key=85YEW8N43FTA upload=GHCE7G4X7A6P tag=7YHH3ZPL9PFV type=write size=2463832 2024-09-13T00:00:55.068 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.066+0000 7fb8f2ce0640 1 > prepared operation key=H4M1TCN5W6XU tag=ZNQNYC3Q6PIE type=write size=128732 2024-09-13T00:00:55.073 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.074+0000 7fb8f2ce0640 1 < completed delete operation key=8Z6U7OOA0GBH tag=8JBLPTNBV2Y1 type=del 2024-09-13T00:00:55.079 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.078+0000 7fb8f2ce0640 1 > prepared operation key=PVYZQLGYT3XU tag=MSFSY369A2ON type=write size=66134 2024-09-13T00:00:55.085 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.082+0000 7fb8f2ce0640 1 < completed delete operation key=EUE3LT0PDIXV tag=OKYMW1K7XYZD type=del 2024-09-13T00:00:55.095 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.094+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~VY29AHK2SJXI.0 size=1048576 2024-09-13T00:00:55.101 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.102+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~VY29AHK2SJXI.1 size=1048576 2024-09-13T00:00:55.106 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.106+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~VY29AHK2SJXI.2 size=971854 2024-09-13T00:00:55.106 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.106+0000 7fb8f2ce0640 1 > prepared multipart upload key=8Z6U7OOA0GBH upload=VY29AHK2SJXI tag=JJDJCPYVELMW type=write size=4117582 2024-09-13T00:00:55.111 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.110+0000 7fb8f2ce0640 1 < completed delete operation key=1HX06CCZJ1DD tag=CETT48N0WKO1 type=del 2024-09-13T00:00:55.117 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.118+0000 7fb8f2ce0640 1 > prepared operation key=J8NGDSF9XRTP tag=U3ACB62VGPG2 type=del size=146926 2024-09-13T00:00:55.123 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.122+0000 7fb8f2ce0640 1 < completed delete operation key=7OX5OJZ8GXU5 tag=O5KEYYHTLVTZ type=del 2024-09-13T00:00:55.128 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.126+0000 7fb8f2ce0640 1 > prepared operation key=8MPUVD37EDN9 tag=EH59B4LN3372 type=del size=974442 2024-09-13T00:00:55.130 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.130+0000 7fb8f2ce0640 1 < canceled part key=_multipart_N3Z5XIG3TZRQ.2~DP2D03A9927U.0 size=1048576 2024-09-13T00:00:55.135 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.134+0000 7fb8f2ce0640 1 < completed part key=_multipart_N3Z5XIG3TZRQ.2~DP2D03A9927U.1 size=476594 2024-09-13T00:00:55.145 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.146+0000 7fb8f2ce0640 1 < canceled multipart upload key=N3Z5XIG3TZRQ upload=DP2D03A9927U tag=Y5OR10IDE82C type=write size=2573746 2024-09-13T00:00:55.155 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.154+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~9RNEO5JWGYC4.0 size=1048576 2024-09-13T00:00:55.160 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.158+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~9RNEO5JWGYC4.1 size=1048576 2024-09-13T00:00:55.165 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.166+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~9RNEO5JWGYC4.2 size=823401 2024-09-13T00:00:55.165 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.166+0000 7fb8f2ce0640 1 > prepared multipart upload key=8Z6U7OOA0GBH upload=9RNEO5JWGYC4 tag=INUHDXHYHQFY type=write size=3969129 2024-09-13T00:00:55.171 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.170+0000 7fb8f2ce0640 1 < completed part key=_multipart_QMZREZ1QEDNJ.2~03IRDZ768BDW.0 size=1048576 2024-09-13T00:00:55.175 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.174+0000 7fb8f2ce0640 1 < completed part key=_multipart_QMZREZ1QEDNJ.2~03IRDZ768BDW.1 size=1048576 2024-09-13T00:00:55.180 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.178+0000 7fb8f2ce0640 1 < completed part key=_multipart_QMZREZ1QEDNJ.2~03IRDZ768BDW.2 size=451531 2024-09-13T00:00:55.192 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.190+0000 7fb8f2ce0640 1 < completed multipart upload key=QMZREZ1QEDNJ upload=03IRDZ768BDW tag=0I2CNZ34WEDO type=write size=3597259 2024-09-13T00:00:55.199 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.198+0000 7fb8f2ce0640 1 > prepared operation key=85YEW8N43FTA tag=D58U2JD7UTDH type=del size=1043359 2024-09-13T00:00:55.204 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.202+0000 7fb8f2ce0640 1 < completed delete operation key=8MPUVD37EDN9 tag=S8CBBYV7SEKM type=del 2024-09-13T00:00:55.211 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.210+0000 7fb8f2ce0640 1 > prepared operation key=NX26SI4F2ZMB tag=BXTMYPL70WG1 type=del size=155913 2024-09-13T00:00:55.217 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.218+0000 7fb8f2ce0640 1 < completed part key=_multipart_PVYZQLGYT3XU.2~ZVB27GR2LUTP.0 size=705857 2024-09-13T00:00:55.227 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.226+0000 7fb8f2ce0640 1 < completed multipart upload key=PVYZQLGYT3XU upload=ZVB27GR2LUTP tag=1ZOS2JUO8B2M type=write size=1754433 2024-09-13T00:00:55.237 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.238+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~NLA3NIN27M5P.0 size=1048576 2024-09-13T00:00:55.242 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.242+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~NLA3NIN27M5P.1 size=1048576 2024-09-13T00:00:55.247 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.246+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~NLA3NIN27M5P.2 size=900155 2024-09-13T00:00:55.247 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.246+0000 7fb8f2ce0640 1 > prepared multipart upload key=63FJDWBO2T9S upload=NLA3NIN27M5P tag=HR3IKZJYU36Q type=write size=4045883 2024-09-13T00:00:55.252 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.250+0000 7fb8f2ce0640 1 < completed delete operation key=2TWD9YLMSYJI tag=1Q072YO24BKY type=del 2024-09-13T00:00:55.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:54 smithi136 bash[75164]: cluster 2024-09-13T00:00:53.136744+0000 mgr.y (mgr.24661) 713 : cluster [DBG] pgmap v1389: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 376 KiB/s rd, 131 KiB/s wr, 666 op/s 2024-09-13T00:00:55.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:54 smithi136 bash[76371]: cluster 2024-09-13T00:00:53.136744+0000 mgr.y (mgr.24661) 713 : cluster [DBG] pgmap v1389: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 376 KiB/s rd, 131 KiB/s wr, 666 op/s 2024-09-13T00:00:55.259 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.258+0000 7fb8f2ce0640 1 > prepared operation key=N3Z5XIG3TZRQ tag=GWJN8DBL3Q1V type=del size=1730236 2024-09-13T00:00:55.264 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.262+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~9I48CZBO470J.0 size=1048576 2024-09-13T00:00:55.269 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.266+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~9I48CZBO470J.1 size=1048576 2024-09-13T00:00:55.274 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.274+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~9I48CZBO470J.2 size=749032 2024-09-13T00:00:55.283 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.282+0000 7fb8f2ce0640 1 < completed multipart upload key=ZTK9AYBSLCOZ upload=9I48CZBO470J tag=KDFKJNXJD2RR type=write size=3894760 2024-09-13T00:00:55.290 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.290+0000 7fb8f2ce0640 1 > prepared operation key=PVYZQLGYT3XU tag=UXV400ZD843M type=del size=1742768 2024-09-13T00:00:55.295 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.294+0000 7fb8f2ce0640 1 < completed delete operation key=8Z6U7OOA0GBH tag=BEXXGN1HWZRR type=del 2024-09-13T00:00:55.300 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.298+0000 7fb8f2ce0640 1 > prepared operation key=0WC6P8YNL0H2 tag=SUI44J1SOS9M type=del size=2052726 2024-09-13T00:00:55.305 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.306+0000 7fb8f2ce0640 1 < completed delete operation key=EJN6HR8I7XQF tag=HNMM4S703X7L type=del 2024-09-13T00:00:55.311 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.310+0000 7fb8f2ce0640 1 > prepared operation key=W8XZ26311RJX tag=6YEW746K6ON0 type=del size=2963587 2024-09-13T00:00:55.321 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.322+0000 7fb8f2ce0640 1 < suggested update operation key=P8OYS5ZE1NX0 tag=ZWR8YXAAJG64 type=write size=863081 2024-09-13T00:00:55.327 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.326+0000 7fb8f2ce0640 1 > prepared operation key=F0QCVR9E08WE tag=HLHLV4JA4G09 type=write size=691669 2024-09-13T00:00:55.337 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.338+0000 7fb8f2ce0640 1 < suggested update operation key=P8OYS5ZE1NX0 tag=TDEESE1OGST1 type=write size=243344 2024-09-13T00:00:55.343 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.342+0000 7fb8f2ce0640 1 > prepared operation key=W8XZ26311RJX tag=N2843Y893XKU type=del size=4032608 2024-09-13T00:00:55.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:54 smithi181 bash[67587]: cluster 2024-09-13T00:00:53.136744+0000 mgr.y (mgr.24661) 713 : cluster [DBG] pgmap v1389: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 376 KiB/s rd, 131 KiB/s wr, 666 op/s 2024-09-13T00:00:55.348 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.346+0000 7fb8f2ce0640 1 < completed part key=_multipart_44EDYFT9VF8Y.2~7IVIF6LIUL5W.0 size=1048576 2024-09-13T00:00:55.353 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.354+0000 7fb8f2ce0640 1 < completed part key=_multipart_44EDYFT9VF8Y.2~7IVIF6LIUL5W.1 size=1048576 2024-09-13T00:00:55.358 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.358+0000 7fb8f2ce0640 1 < completed part key=_multipart_44EDYFT9VF8Y.2~7IVIF6LIUL5W.2 size=588490 2024-09-13T00:00:55.367 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.366+0000 7fb8f2ce0640 1 < completed multipart upload key=44EDYFT9VF8Y upload=7IVIF6LIUL5W tag=7SPZFLABGDKA type=write size=3734218 2024-09-13T00:00:55.373 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.374+0000 7fb8f2ce0640 1 > prepared operation key=2AHYRZUDFHP4 tag=7UEF1JFQTHDP type=del size=2161170 2024-09-13T00:00:55.384 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.382+0000 7fb8f2ce0640 1 < suggested update operation key=H4M1TCN5W6XU tag=ZNQNYC3Q6PIE type=write size=128732 2024-09-13T00:00:55.389 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.390+0000 7fb8f2ce0640 1 > prepared operation key=2AHYRZUDFHP4 tag=JDPF7GBPT5OB type=del size=2522905 2024-09-13T00:00:55.394 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.394+0000 7fb8f2ce0640 1 < completed write operation key=PVYZQLGYT3XU tag=MSFSY369A2ON type=write size=66134 2024-09-13T00:00:55.399 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.398+0000 7fb8f2ce0640 1 > prepared operation key=85YEW8N43FTA tag=AG4U3S37F6JR type=write size=694681 2024-09-13T00:00:55.406 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.406+0000 7fb8f2ce0640 1 < completed part key=_multipart_8Z6U7OOA0GBH.2~VY29AHK2SJXI.0 size=1048576 2024-09-13T00:00:55.406 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.406+0000 7fb8f2ce0640 1 < canceled part key=_multipart_8Z6U7OOA0GBH.2~VY29AHK2SJXI.1 size=1048576 2024-09-13T00:00:55.411 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.410+0000 7fb8f2ce0640 1 < completed part key=_multipart_8Z6U7OOA0GBH.2~VY29AHK2SJXI.2 size=971854 2024-09-13T00:00:55.422 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.422+0000 7fb8f2ce0640 1 < canceled multipart upload key=8Z6U7OOA0GBH upload=VY29AHK2SJXI tag=JJDJCPYVELMW type=write size=4117582 2024-09-13T00:00:55.433 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.430+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NX26SI4F2ZMB.2~4WH5FVCGZP21.0 size=1048576 2024-09-13T00:00:55.438 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.438+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NX26SI4F2ZMB.2~4WH5FVCGZP21.1 size=1048576 2024-09-13T00:00:55.443 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.442+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NX26SI4F2ZMB.2~4WH5FVCGZP21.2 size=568073 2024-09-13T00:00:55.443 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.442+0000 7fb8f2ce0640 1 > prepared multipart upload key=NX26SI4F2ZMB upload=4WH5FVCGZP21 tag=J0SGB3INTDCN type=write size=3713801 2024-09-13T00:00:55.449 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.450+0000 7fb8f2ce0640 1 < completed delete operation key=J8NGDSF9XRTP tag=U3ACB62VGPG2 type=del 2024-09-13T00:00:55.455 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.454+0000 7fb8f2ce0640 1 > prepared operation key=J8NGDSF9XRTP tag=L68V6QMPRCMJ type=del size=2243890 2024-09-13T00:00:55.462 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.462+0000 7fb8f2ce0640 1 < completed delete operation key=8MPUVD37EDN9 tag=EH59B4LN3372 type=del 2024-09-13T00:00:55.468 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.466+0000 7fb8f2ce0640 1 > prepared operation key=63FJDWBO2T9S tag=AFAH9A4N2Z1R type=del size=852607 2024-09-13T00:00:55.474 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.474+0000 7fb8f2ce0640 1 < completed part key=_multipart_8Z6U7OOA0GBH.2~9RNEO5JWGYC4.0 size=1048576 2024-09-13T00:00:55.480 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.478+0000 7fb8f2ce0640 1 < completed part key=_multipart_8Z6U7OOA0GBH.2~9RNEO5JWGYC4.1 size=1048576 2024-09-13T00:00:55.485 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.486+0000 7fb8f2ce0640 1 < completed part key=_multipart_8Z6U7OOA0GBH.2~9RNEO5JWGYC4.2 size=823401 2024-09-13T00:00:55.497 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.494+0000 7fb8f2ce0640 1 < completed multipart upload key=8Z6U7OOA0GBH upload=9RNEO5JWGYC4 tag=INUHDXHYHQFY type=write size=3969129 2024-09-13T00:00:55.503 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.502+0000 7fb8f2ce0640 1 > prepared operation key=H9AU4HD247YU tag=YATLN8AS4EGS type=write size=5999 2024-09-13T00:00:55.509 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.510+0000 7fb8f2ce0640 1 < completed delete operation key=85YEW8N43FTA tag=D58U2JD7UTDH type=del 2024-09-13T00:00:55.516 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.514+0000 7fb8f2ce0640 1 > prepared operation key=O4MZ4UWNXME9 tag=6MNAVSE1JE16 type=del size=1267781 2024-09-13T00:00:55.528 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.526+0000 7fb8f2ce0640 1 < suggested remove operation key=NX26SI4F2ZMB tag=BXTMYPL70WG1 type=del 2024-09-13T00:00:55.538 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.538+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~QA110476GSUM.0 size=1001722 2024-09-13T00:00:55.539 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.538+0000 7fb8f2ce0640 1 > prepared multipart upload key=8MPUVD37EDN9 upload=QA110476GSUM tag=3ABPJDJ1VC2T type=write size=2050298 2024-09-13T00:00:55.544 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.542+0000 7fb8f2ce0640 1 < completed part key=_multipart_63FJDWBO2T9S.2~NLA3NIN27M5P.0 size=1048576 2024-09-13T00:00:55.548 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.546+0000 7fb8f2ce0640 1 < completed part key=_multipart_63FJDWBO2T9S.2~NLA3NIN27M5P.1 size=1048576 2024-09-13T00:00:55.553 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.554+0000 7fb8f2ce0640 1 < completed part key=_multipart_63FJDWBO2T9S.2~NLA3NIN27M5P.2 size=900155 2024-09-13T00:00:55.563 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.562+0000 7fb8f2ce0640 1 < completed multipart upload key=63FJDWBO2T9S upload=NLA3NIN27M5P tag=HR3IKZJYU36Q type=write size=4045883 2024-09-13T00:00:55.568 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.566+0000 7fb8f2ce0640 1 > prepared operation key=ZFHN2SNTZ5W7 tag=J9MFZX6FQ7D3 type=del size=2111061 2024-09-13T00:00:55.574 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.574+0000 7fb8f2ce0640 1 < completed delete operation key=N3Z5XIG3TZRQ tag=GWJN8DBL3Q1V type=del 2024-09-13T00:00:55.585 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.586+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~WB3YABB76V2A.0 size=1034907 2024-09-13T00:00:55.585 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.586+0000 7fb8f2ce0640 1 > prepared multipart upload key=PVYZQLGYT3XU upload=WB3YABB76V2A tag=BH01QCQAX2MR type=write size=2083483 2024-09-13T00:00:55.591 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.590+0000 7fb8f2ce0640 1 < completed delete operation key=PVYZQLGYT3XU tag=UXV400ZD843M type=del 2024-09-13T00:00:55.598 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.598+0000 7fb8f2ce0640 1 > prepared operation key=NQJQC8CEAC15 tag=WWO723L8RC34 type=del size=384416 2024-09-13T00:00:55.605 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.606+0000 7fb8f2ce0640 1 < completed delete operation key=0WC6P8YNL0H2 tag=SUI44J1SOS9M type=del 2024-09-13T00:00:55.611 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.610+0000 7fb8f2ce0640 1 > prepared operation key=H4M1TCN5W6XU tag=BH10XD6A2AT0 type=write size=600601 2024-09-13T00:00:55.618 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.618+0000 7fb8f2ce0640 1 < completed delete operation key=W8XZ26311RJX tag=6YEW746K6ON0 type=del 2024-09-13T00:00:55.630 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.630+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2TWD9YLMSYJI.2~ETWNMIO512OM.0 size=1048576 2024-09-13T00:00:55.635 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.634+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2TWD9YLMSYJI.2~ETWNMIO512OM.1 size=1048576 2024-09-13T00:00:55.641 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.638+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2TWD9YLMSYJI.2~ETWNMIO512OM.2 size=268699 2024-09-13T00:00:55.641 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.638+0000 7fb8f2ce0640 1 > prepared multipart upload key=2TWD9YLMSYJI upload=ETWNMIO512OM tag=JOPRKYJ8OQB2 type=write size=3414427 2024-09-13T00:00:55.653 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.654+0000 7fb8f2ce0640 1 < suggested remove operation key=F0QCVR9E08WE tag=HLHLV4JA4G09 type=write 2024-09-13T00:00:55.664 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.662+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~0JQJLCLU37SD.0 size=306086 2024-09-13T00:00:55.664 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.662+0000 7fb8f2ce0640 1 > prepared multipart upload key=NQJQC8CEAC15 upload=0JQJLCLU37SD tag=L0EUBW2805UV type=write size=1354662 2024-09-13T00:00:55.670 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.670+0000 7fb8f2ce0640 1 < completed delete operation key=W8XZ26311RJX tag=N2843Y893XKU type=del 2024-09-13T00:00:55.680 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.678+0000 7fb8f2ce0640 1 > prepared part key=_multipart_F0QCVR9E08WE.2~7FQYJ2IIS7O2.0 size=1048576 2024-09-13T00:00:55.685 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.686+0000 7fb8f2ce0640 1 > prepared part key=_multipart_F0QCVR9E08WE.2~7FQYJ2IIS7O2.1 size=1048576 2024-09-13T00:00:55.690 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.690+0000 7fb8f2ce0640 1 > prepared part key=_multipart_F0QCVR9E08WE.2~7FQYJ2IIS7O2.2 size=780929 2024-09-13T00:00:55.690 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.690+0000 7fb8f2ce0640 1 > prepared multipart upload key=F0QCVR9E08WE upload=7FQYJ2IIS7O2 tag=P7X5LF0SLZAP type=write size=3926657 2024-09-13T00:00:55.696 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.694+0000 7fb8f2ce0640 1 < completed delete operation key=2AHYRZUDFHP4 tag=7UEF1JFQTHDP type=del 2024-09-13T00:00:55.702 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.702+0000 7fb8f2ce0640 1 > prepared operation key=0WC6P8YNL0H2 tag=IY0BZFCJWZ9U type=write size=430355 2024-09-13T00:00:55.715 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.714+0000 7fb8f2ce0640 1 < suggested update operation key=2AHYRZUDFHP4 tag=JDPF7GBPT5OB type=del size=2522905 2024-09-13T00:00:55.727 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.726+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H4M1TCN5W6XU.2~BTGH57OR4V33.0 size=1048576 2024-09-13T00:00:55.732 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.730+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H4M1TCN5W6XU.2~BTGH57OR4V33.1 size=1048576 2024-09-13T00:00:55.736 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.734+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H4M1TCN5W6XU.2~BTGH57OR4V33.2 size=694469 2024-09-13T00:00:55.737 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.734+0000 7fb8f2ce0640 1 > prepared multipart upload key=H4M1TCN5W6XU upload=BTGH57OR4V33 tag=DOV31JS0K1V4 type=write size=3840197 2024-09-13T00:00:55.742 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.742+0000 7fb8f2ce0640 1 < canceled operation key=85YEW8N43FTA tag=AG4U3S37F6JR type=write size=694681 2024-09-13T00:00:55.748 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.746+0000 7fb8f2ce0640 1 > prepared operation key=N3Z5XIG3TZRQ tag=MR1FGGDL7O2O type=write size=476668 2024-09-13T00:00:55.754 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.754+0000 7fb8f2ce0640 1 < completed part key=_multipart_NX26SI4F2ZMB.2~4WH5FVCGZP21.0 size=1048576 2024-09-13T00:00:55.760 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.758+0000 7fb8f2ce0640 1 < completed part key=_multipart_NX26SI4F2ZMB.2~4WH5FVCGZP21.1 size=1048576 2024-09-13T00:00:55.766 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.766+0000 7fb8f2ce0640 1 < completed part key=_multipart_NX26SI4F2ZMB.2~4WH5FVCGZP21.2 size=568073 2024-09-13T00:00:55.778 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.778+0000 7fb8f2ce0640 1 < completed multipart upload key=NX26SI4F2ZMB upload=4WH5FVCGZP21 tag=J0SGB3INTDCN type=write size=3713801 2024-09-13T00:00:55.784 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.782+0000 7fb8f2ce0640 1 > prepared operation key=FX0NCGLK0KPJ tag=ZA047PGE0GE1 type=del size=856170 2024-09-13T00:00:55.791 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.790+0000 7fb8f2ce0640 1 < canceled operation key=J8NGDSF9XRTP tag=L68V6QMPRCMJ type=del size=2243890 2024-09-13T00:00:55.804 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.802+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~LSHI8JN3N2WC.0 size=1048576 2024-09-13T00:00:55.809 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.810+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~LSHI8JN3N2WC.1 size=1048576 2024-09-13T00:00:55.814 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.814+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~LSHI8JN3N2WC.2 size=627824 2024-09-13T00:00:55.814 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.814+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZTK9AYBSLCOZ upload=LSHI8JN3N2WC tag=EJKN6UGX0CN0 type=write size=3773552 2024-09-13T00:00:55.821 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.822+0000 7fb8f2ce0640 1 < completed delete operation key=63FJDWBO2T9S tag=AFAH9A4N2Z1R type=del 2024-09-13T00:00:55.834 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.834+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~5IWX3JATZ6AX.0 size=671580 2024-09-13T00:00:55.834 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.834+0000 7fb8f2ce0640 1 > prepared multipart upload key=2AHYRZUDFHP4 upload=5IWX3JATZ6AX tag=UUNM4MKD2L4P type=write size=1720156 2024-09-13T00:00:55.844 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.842+0000 7fb8f2ce0640 1 < completed write operation key=H9AU4HD247YU tag=YATLN8AS4EGS type=write size=5999 2024-09-13T00:00:55.854 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.854+0000 7fb8f2ce0640 1 > prepared operation key=1HX06CCZJ1DD tag=CMSI6II6GPBK type=del size=2735215 2024-09-13T00:00:55.864 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.862+0000 7fb8f2ce0640 1 < completed delete operation key=O4MZ4UWNXME9 tag=6MNAVSE1JE16 type=del 2024-09-13T00:00:55.875 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.874+0000 7fb8f2ce0640 1 > prepared part key=_multipart_0WC6P8YNL0H2.2~HQ9BKTWWMIGQ.0 size=129810 2024-09-13T00:00:55.875 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.874+0000 7fb8f2ce0640 1 > prepared multipart upload key=0WC6P8YNL0H2 upload=HQ9BKTWWMIGQ tag=6STQDMDFFD71 type=write size=1178386 2024-09-13T00:00:55.880 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.878+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~QA110476GSUM.0 size=1001722 2024-09-13T00:00:55.890 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.890+0000 7fb8f2ce0640 1 < completed multipart upload key=8MPUVD37EDN9 upload=QA110476GSUM tag=3ABPJDJ1VC2T type=write size=2050298 2024-09-13T00:00:55.895 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.894+0000 7fb8f2ce0640 1 > prepared operation key=N3Z5XIG3TZRQ tag=GDP15VAEQCWO type=del size=1141117 2024-09-13T00:00:55.901 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.902+0000 7fb8f2ce0640 1 < completed delete operation key=ZFHN2SNTZ5W7 tag=J9MFZX6FQ7D3 type=del 2024-09-13T00:00:55.913 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.910+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~21UWXB0TSK3P.0 size=1048576 2024-09-13T00:00:55.917 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.918+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~21UWXB0TSK3P.1 size=76054 2024-09-13T00:00:55.917 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.918+0000 7fb8f2ce0640 1 > prepared multipart upload key=W8XZ26311RJX upload=21UWXB0TSK3P tag=OF5ZBQ38RRZW type=write size=2173206 2024-09-13T00:00:55.923 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.922+0000 7fb8f2ce0640 1 < completed part key=_multipart_PVYZQLGYT3XU.2~WB3YABB76V2A.0 size=1034907 2024-09-13T00:00:55.934 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.934+0000 7fb8f2ce0640 1 < completed multipart upload key=PVYZQLGYT3XU upload=WB3YABB76V2A tag=BH01QCQAX2MR type=write size=2083483 2024-09-13T00:00:55.940 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.938+0000 7fb8f2ce0640 1 > prepared operation key=F0QCVR9E08WE tag=3TQFPVCM6Y43 type=del size=2379143 2024-09-13T00:00:55.946 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.946+0000 7fb8f2ce0640 1 < completed delete operation key=NQJQC8CEAC15 tag=WWO723L8RC34 type=del 2024-09-13T00:00:55.952 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.950+0000 7fb8f2ce0640 1 > prepared operation key=7OX5OJZ8GXU5 tag=ACFJ9NVUS3M7 type=del size=1296308 2024-09-13T00:00:55.964 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.962+0000 7fb8f2ce0640 1 < suggested remove operation key=H4M1TCN5W6XU tag=BH10XD6A2AT0 type=write 2024-09-13T00:00:55.975 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.974+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~GOX1WAYZ08U0.0 size=167305 2024-09-13T00:00:55.975 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.974+0000 7fb8f2ce0640 1 > prepared multipart upload key=1HX06CCZJ1DD upload=GOX1WAYZ08U0 tag=JSDZPH1LYY7A type=write size=1215881 2024-09-13T00:00:55.981 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.982+0000 7fb8f2ce0640 1 < completed part key=_multipart_2TWD9YLMSYJI.2~ETWNMIO512OM.0 size=1048576 2024-09-13T00:00:55.987 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.986+0000 7fb8f2ce0640 1 < completed part key=_multipart_2TWD9YLMSYJI.2~ETWNMIO512OM.1 size=1048576 2024-09-13T00:00:55.992 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:55.990+0000 7fb8f2ce0640 1 < completed part key=_multipart_2TWD9YLMSYJI.2~ETWNMIO512OM.2 size=268699 2024-09-13T00:00:56.006 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.006+0000 7fb8f2ce0640 1 < completed multipart upload key=2TWD9YLMSYJI upload=ETWNMIO512OM tag=JOPRKYJ8OQB2 type=write size=3414427 2024-09-13T00:00:56.012 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.010+0000 7fb8f2ce0640 1 > prepared operation key=8Z6U7OOA0GBH tag=VYQLXWX8MOUW type=del size=399533 2024-09-13T00:00:56.019 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.018+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~0JQJLCLU37SD.0 size=306086 2024-09-13T00:00:56.029 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.030+0000 7fb8f2ce0640 1 < completed multipart upload key=NQJQC8CEAC15 upload=0JQJLCLU37SD tag=L0EUBW2805UV type=write size=1354662 2024-09-13T00:00:56.041 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.038+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~HKQ6RTMHI39Z.0 size=17852 2024-09-13T00:00:56.041 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.038+0000 7fb8f2ce0640 1 > prepared multipart upload key=63FJDWBO2T9S upload=HKQ6RTMHI39Z tag=H6YVFHZKOWVP type=write size=1066428 2024-09-13T00:00:56.047 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.046+0000 7fb8f2ce0640 1 < completed part key=_multipart_F0QCVR9E08WE.2~7FQYJ2IIS7O2.0 size=1048576 2024-09-13T00:00:56.053 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.050+0000 7fb8f2ce0640 1 < completed part key=_multipart_F0QCVR9E08WE.2~7FQYJ2IIS7O2.1 size=1048576 2024-09-13T00:00:56.058 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.058+0000 7fb8f2ce0640 1 < completed part key=_multipart_F0QCVR9E08WE.2~7FQYJ2IIS7O2.2 size=780929 2024-09-13T00:00:56.068 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.066+0000 7fb8f2ce0640 1 < completed multipart upload key=F0QCVR9E08WE upload=7FQYJ2IIS7O2 tag=P7X5LF0SLZAP type=write size=3926657 2024-09-13T00:00:56.074 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.074+0000 7fb8f2ce0640 1 > prepared operation key=H9AU4HD247YU tag=9TB1NSV8YOCW type=del size=1824012 2024-09-13T00:00:56.080 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.078+0000 7fb8f2ce0640 1 < completed write operation key=0WC6P8YNL0H2 tag=IY0BZFCJWZ9U type=write size=430355 2024-09-13T00:00:56.085 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.086+0000 7fb8f2ce0640 1 > prepared operation key=1PX7DBO9M3YI tag=2AY197K4BZ16 type=del size=546980 2024-09-13T00:00:56.092 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.090+0000 7fb8f2ce0640 1 < completed part key=_multipart_H4M1TCN5W6XU.2~BTGH57OR4V33.0 size=1048576 2024-09-13T00:00:56.097 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.098+0000 7fb8f2ce0640 1 < completed part key=_multipart_H4M1TCN5W6XU.2~BTGH57OR4V33.1 size=1048576 2024-09-13T00:00:56.102 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.102+0000 7fb8f2ce0640 1 < completed part key=_multipart_H4M1TCN5W6XU.2~BTGH57OR4V33.2 size=694469 2024-09-13T00:00:56.115 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.114+0000 7fb8f2ce0640 1 < completed multipart upload key=H4M1TCN5W6XU upload=BTGH57OR4V33 tag=DOV31JS0K1V4 type=write size=3840197 2024-09-13T00:00:56.121 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.122+0000 7fb8f2ce0640 1 > prepared operation key=W8XZ26311RJX tag=EYNMS3L7DOIA type=del size=1549887 2024-09-13T00:00:56.127 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.126+0000 7fb8f2ce0640 1 < completed write operation key=N3Z5XIG3TZRQ tag=MR1FGGDL7O2O type=write size=476668 2024-09-13T00:00:56.133 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.134+0000 7fb8f2ce0640 1 > prepared operation key=O4MZ4UWNXME9 tag=TY322WIOQ35D type=del size=587203 2024-09-13T00:00:56.139 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.138+0000 7fb8f2ce0640 1 < completed delete operation key=FX0NCGLK0KPJ tag=ZA047PGE0GE1 type=del 2024-09-13T00:00:56.149 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.150+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EUE3LT0PDIXV.2~4FO6HCJJ0GXG.0 size=782426 2024-09-13T00:00:56.149 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.150+0000 7fb8f2ce0640 1 > prepared multipart upload key=EUE3LT0PDIXV upload=4FO6HCJJ0GXG tag=K5N8EAOAOMXF type=write size=1831002 2024-09-13T00:00:56.155 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.154+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~LSHI8JN3N2WC.0 size=1048576 2024-09-13T00:00:56.155 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.154+0000 7fb8f2ce0640 1 < canceled part key=_multipart_ZTK9AYBSLCOZ.2~LSHI8JN3N2WC.1 size=1048576 2024-09-13T00:00:56.160 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.158+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~LSHI8JN3N2WC.2 size=627824 2024-09-13T00:00:56.170 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.170+0000 7fb8f2ce0640 1 < canceled multipart upload key=ZTK9AYBSLCOZ upload=LSHI8JN3N2WC tag=EJKN6UGX0CN0 type=write size=3773552 2024-09-13T00:00:56.176 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.174+0000 7fb8f2ce0640 1 > prepared operation key=0WC6P8YNL0H2 tag=2J67CNP0828K type=del size=2472530 2024-09-13T00:00:56.182 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.182+0000 7fb8f2ce0640 1 < completed part key=_multipart_2AHYRZUDFHP4.2~5IWX3JATZ6AX.0 size=671580 2024-09-13T00:00:56.194 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.194+0000 7fb8f2ce0640 1 < completed multipart upload key=2AHYRZUDFHP4 upload=5IWX3JATZ6AX tag=UUNM4MKD2L4P type=write size=1720156 2024-09-13T00:00:56.208 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.206+0000 7fb8f2ce0640 1 > prepared part key=_multipart_44EDYFT9VF8Y.2~YOGCIW0WLEBP.0 size=963592 2024-09-13T00:00:56.209 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.206+0000 7fb8f2ce0640 1 > prepared multipart upload key=44EDYFT9VF8Y upload=YOGCIW0WLEBP tag=8C9R4EOU7EAS type=write size=2012168 2024-09-13T00:00:56.221 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.222+0000 7fb8f2ce0640 1 < suggested remove operation key=1HX06CCZJ1DD tag=CMSI6II6GPBK type=del 2024-09-13T00:00:56.227 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.226+0000 7fb8f2ce0640 1 > prepared operation key=4K76U69GZ9AE tag=S04O8OBWLLAK type=del size=851178 2024-09-13T00:00:56.228 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.226+0000 7fb8f2ce0640 1 < canceled part key=_multipart_0WC6P8YNL0H2.2~HQ9BKTWWMIGQ.0 size=129810 2024-09-13T00:00:56.237 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.238+0000 7fb8f2ce0640 1 < canceled multipart upload key=0WC6P8YNL0H2 upload=HQ9BKTWWMIGQ tag=6STQDMDFFD71 type=write size=1178386 2024-09-13T00:00:56.249 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.250+0000 7fb8f2ce0640 1 > prepared part key=_multipart_N3Z5XIG3TZRQ.2~FF5YUX96090E.0 size=903966 2024-09-13T00:00:56.249 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.250+0000 7fb8f2ce0640 1 > prepared multipart upload key=N3Z5XIG3TZRQ upload=FF5YUX96090E tag=FTUAEPB1EVWX type=write size=1952542 2024-09-13T00:00:56.255 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.254+0000 7fb8f2ce0640 1 < completed delete operation key=N3Z5XIG3TZRQ tag=GDP15VAEQCWO type=del 2024-09-13T00:00:56.269 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.270+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H9AU4HD247YU.2~ZPX5MTRVXMVI.0 size=1048576 2024-09-13T00:00:56.277 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.278+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H9AU4HD247YU.2~ZPX5MTRVXMVI.1 size=280871 2024-09-13T00:00:56.277 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.278+0000 7fb8f2ce0640 1 > prepared multipart upload key=H9AU4HD247YU upload=ZPX5MTRVXMVI tag=TXB9COA7YTMX type=write size=2378023 2024-09-13T00:00:56.284 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.282+0000 7fb8f2ce0640 1 < completed part key=_multipart_W8XZ26311RJX.2~21UWXB0TSK3P.0 size=1048576 2024-09-13T00:00:56.289 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.290+0000 7fb8f2ce0640 1 < completed part key=_multipart_W8XZ26311RJX.2~21UWXB0TSK3P.1 size=76054 2024-09-13T00:00:56.300 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.298+0000 7fb8f2ce0640 1 < completed multipart upload key=W8XZ26311RJX upload=21UWXB0TSK3P tag=OF5ZBQ38RRZW type=write size=2173206 2024-09-13T00:00:56.307 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.306+0000 7fb8f2ce0640 1 > prepared operation key=NX26SI4F2ZMB tag=K0TOEKEU2A2K type=del size=3692962 2024-09-13T00:00:56.318 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.318+0000 7fb8f2ce0640 1 < suggested update operation key=F0QCVR9E08WE tag=3TQFPVCM6Y43 type=del size=2379143 2024-09-13T00:00:56.329 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.330+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~KSTQ1COS69KA.0 size=475769 2024-09-13T00:00:56.329 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.330+0000 7fb8f2ce0640 1 > prepared multipart upload key=7OX5OJZ8GXU5 upload=KSTQ1COS69KA tag=OPG9EWN3IXSE type=write size=1524345 2024-09-13T00:00:56.341 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.338+0000 7fb8f2ce0640 1 < suggested remove operation key=7OX5OJZ8GXU5 tag=ACFJ9NVUS3M7 type=del 2024-09-13T00:00:56.346 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.346+0000 7fb8f2ce0640 1 > prepared operation key=ZTK9AYBSLCOZ tag=RAMGTD2GJCF4 type=del size=223720 2024-09-13T00:00:56.353 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.354+0000 7fb8f2ce0640 1 < completed part key=_multipart_1HX06CCZJ1DD.2~GOX1WAYZ08U0.0 size=167305 2024-09-13T00:00:56.366 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.366+0000 7fb8f2ce0640 1 < completed multipart upload key=1HX06CCZJ1DD upload=GOX1WAYZ08U0 tag=JSDZPH1LYY7A type=write size=1215881 2024-09-13T00:00:56.378 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.378+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~N699J2I5W654.0 size=1048576 2024-09-13T00:00:56.384 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.382+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~N699J2I5W654.1 size=687957 2024-09-13T00:00:56.384 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.382+0000 7fb8f2ce0640 1 > prepared multipart upload key=2AHYRZUDFHP4 upload=N699J2I5W654 tag=6PQD5B451Q5D type=write size=2785109 2024-09-13T00:00:56.390 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.390+0000 7fb8f2ce0640 1 < completed delete operation key=8Z6U7OOA0GBH tag=VYQLXWX8MOUW type=del 2024-09-13T00:00:56.401 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.398+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~POPFTQ47T9EN.0 size=520293 2024-09-13T00:00:56.401 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.398+0000 7fb8f2ce0640 1 > prepared multipart upload key=7OX5OJZ8GXU5 upload=POPFTQ47T9EN tag=IGQIX0XATY2C type=write size=1568869 2024-09-13T00:00:56.407 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.406+0000 7fb8f2ce0640 1 < completed part key=_multipart_63FJDWBO2T9S.2~HKQ6RTMHI39Z.0 size=17852 2024-09-13T00:00:56.419 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.418+0000 7fb8f2ce0640 1 < completed multipart upload key=63FJDWBO2T9S upload=HKQ6RTMHI39Z tag=H6YVFHZKOWVP type=write size=1066428 2024-09-13T00:00:56.430 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.430+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~K48EU68W4XKF.0 size=1048576 2024-09-13T00:00:56.435 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.434+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~K48EU68W4XKF.1 size=1048576 2024-09-13T00:00:56.440 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.438+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~K48EU68W4XKF.2 size=194710 2024-09-13T00:00:56.440 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.438+0000 7fb8f2ce0640 1 > prepared multipart upload key=8Z6U7OOA0GBH upload=K48EU68W4XKF tag=XC4VY83LIUDP type=write size=3340438 2024-09-13T00:00:56.446 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.446+0000 7fb8f2ce0640 1 < completed delete operation key=H9AU4HD247YU tag=9TB1NSV8YOCW type=del 2024-09-13T00:00:56.452 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.450+0000 7fb8f2ce0640 1 > prepared operation key=5GM88ERM89KT tag=FVOM3VMIJ46Y type=del size=3937341 2024-09-13T00:00:56.458 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.458+0000 7fb8f2ce0640 1 < completed delete operation key=1PX7DBO9M3YI tag=2AY197K4BZ16 type=del 2024-09-13T00:00:56.464 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.462+0000 7fb8f2ce0640 1 > prepared operation key=EJN6HR8I7XQF tag=NOZIG1EKTBCE type=del size=451428 2024-09-13T00:00:56.470 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.470+0000 7fb8f2ce0640 1 < completed delete operation key=W8XZ26311RJX tag=EYNMS3L7DOIA type=del 2024-09-13T00:00:56.481 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.482+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~CXA4BL6SV3RG.0 size=1048576 2024-09-13T00:00:56.486 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.486+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~CXA4BL6SV3RG.1 size=1048576 2024-09-13T00:00:56.491 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.490+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~CXA4BL6SV3RG.2 size=180320 2024-09-13T00:00:56.491 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.490+0000 7fb8f2ce0640 1 > prepared multipart upload key=2AHYRZUDFHP4 upload=CXA4BL6SV3RG tag=5ZDS07HTRYVR type=write size=3326048 2024-09-13T00:00:56.497 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.498+0000 7fb8f2ce0640 1 < completed delete operation key=O4MZ4UWNXME9 tag=TY322WIOQ35D type=del 2024-09-13T00:00:56.507 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.506+0000 7fb8f2ce0640 1 > prepared part key=_multipart_O4MZ4UWNXME9.2~H2WIHW33Z2RR.0 size=931856 2024-09-13T00:00:56.508 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.506+0000 7fb8f2ce0640 1 > prepared multipart upload key=O4MZ4UWNXME9 upload=H2WIHW33Z2RR tag=O3P2CWOM42YA type=write size=1980432 2024-09-13T00:00:56.514 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.514+0000 7fb8f2ce0640 1 < completed part key=_multipart_EUE3LT0PDIXV.2~4FO6HCJJ0GXG.0 size=782426 2024-09-13T00:00:56.526 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.526+0000 7fb8f2ce0640 1 < completed multipart upload key=EUE3LT0PDIXV upload=4FO6HCJJ0GXG tag=K5N8EAOAOMXF type=write size=1831002 2024-09-13T00:00:56.532 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.530+0000 7fb8f2ce0640 1 > prepared operation key=H5EIP0E9CBWX tag=BG69K6M1XMJY type=del size=368841 2024-09-13T00:00:56.538 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.538+0000 7fb8f2ce0640 1 < canceled operation key=0WC6P8YNL0H2 tag=2J67CNP0828K type=del size=2472530 2024-09-13T00:00:56.549 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.546+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZFHN2SNTZ5W7.2~ZSXYFOJ7LMOW.0 size=681470 2024-09-13T00:00:56.549 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.546+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZFHN2SNTZ5W7 upload=ZSXYFOJ7LMOW tag=2WXNSIAXH1BA type=write size=1730046 2024-09-13T00:00:56.555 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.554+0000 7fb8f2ce0640 1 < completed part key=_multipart_44EDYFT9VF8Y.2~YOGCIW0WLEBP.0 size=963592 2024-09-13T00:00:56.566 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.566+0000 7fb8f2ce0640 1 < completed multipart upload key=44EDYFT9VF8Y upload=YOGCIW0WLEBP tag=8C9R4EOU7EAS type=write size=2012168 2024-09-13T00:00:56.578 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.578+0000 7fb8f2ce0640 1 > prepared part key=_multipart_5GM88ERM89KT.2~6B5WZR5QFYU8.0 size=1048576 2024-09-13T00:00:56.583 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.582+0000 7fb8f2ce0640 1 > prepared part key=_multipart_5GM88ERM89KT.2~6B5WZR5QFYU8.1 size=730417 2024-09-13T00:00:56.583 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.582+0000 7fb8f2ce0640 1 > prepared multipart upload key=5GM88ERM89KT upload=6B5WZR5QFYU8 tag=22L787KST9ZU type=write size=2827569 2024-09-13T00:00:56.596 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.594+0000 7fb8f2ce0640 1 < suggested remove operation key=4K76U69GZ9AE tag=S04O8OBWLLAK type=del 2024-09-13T00:00:56.610 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.610+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~62LTSQLDGHM7.0 size=1010699 2024-09-13T00:00:56.610 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.610+0000 7fb8f2ce0640 1 > prepared multipart upload key=8Z6U7OOA0GBH upload=62LTSQLDGHM7 tag=TN2MR40EIWGE type=write size=2059275 2024-09-13T00:00:56.616 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.614+0000 7fb8f2ce0640 1 < completed part key=_multipart_N3Z5XIG3TZRQ.2~FF5YUX96090E.0 size=903966 2024-09-13T00:00:56.626 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.626+0000 7fb8f2ce0640 1 < completed multipart upload key=N3Z5XIG3TZRQ upload=FF5YUX96090E tag=FTUAEPB1EVWX type=write size=1952542 2024-09-13T00:00:56.637 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.638+0000 7fb8f2ce0640 1 > prepared part key=_multipart_4K76U69GZ9AE.2~BIXU7W1V36L6.0 size=1048576 2024-09-13T00:00:56.643 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.642+0000 7fb8f2ce0640 1 > prepared part key=_multipart_4K76U69GZ9AE.2~BIXU7W1V36L6.1 size=1048576 2024-09-13T00:00:56.647 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.646+0000 7fb8f2ce0640 1 > prepared part key=_multipart_4K76U69GZ9AE.2~BIXU7W1V36L6.2 size=565429 2024-09-13T00:00:56.647 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.646+0000 7fb8f2ce0640 1 > prepared multipart upload key=4K76U69GZ9AE upload=BIXU7W1V36L6 tag=FP802ODW7B6D type=write size=3711157 2024-09-13T00:00:56.654 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.654+0000 7fb8f2ce0640 1 < completed part key=_multipart_H9AU4HD247YU.2~ZPX5MTRVXMVI.0 size=1048576 2024-09-13T00:00:56.659 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.658+0000 7fb8f2ce0640 1 < completed part key=_multipart_H9AU4HD247YU.2~ZPX5MTRVXMVI.1 size=280871 2024-09-13T00:00:56.670 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.670+0000 7fb8f2ce0640 1 < completed multipart upload key=H9AU4HD247YU upload=ZPX5MTRVXMVI tag=TXB9COA7YTMX type=write size=2378023 2024-09-13T00:00:56.676 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.674+0000 7fb8f2ce0640 1 > prepared operation key=FX0NCGLK0KPJ tag=IDSYH4RJ8HS4 type=write size=207275 2024-09-13T00:00:56.682 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.682+0000 7fb8f2ce0640 1 < completed delete operation key=NX26SI4F2ZMB tag=K0TOEKEU2A2K type=del 2024-09-13T00:00:56.688 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.686+0000 7fb8f2ce0640 1 > prepared operation key=1HX06CCZJ1DD tag=1HX4N4S5UVAN type=del size=3987237 2024-09-13T00:00:56.689 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.690+0000 7fb8f2ce0640 1 < canceled part key=_multipart_7OX5OJZ8GXU5.2~KSTQ1COS69KA.0 size=475769 2024-09-13T00:00:56.699 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.698+0000 7fb8f2ce0640 1 < canceled multipart upload key=7OX5OJZ8GXU5 upload=KSTQ1COS69KA tag=OPG9EWN3IXSE type=write size=1524345 2024-09-13T00:00:56.711 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.710+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~ZB28AWNFU5I2.0 size=821707 2024-09-13T00:00:56.711 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.710+0000 7fb8f2ce0640 1 > prepared multipart upload key=PVYZQLGYT3XU upload=ZB28AWNFU5I2 tag=UIT2OYMTPYW7 type=write size=1870283 2024-09-13T00:00:56.716 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.714+0000 7fb8f2ce0640 1 < completed delete operation key=ZTK9AYBSLCOZ tag=RAMGTD2GJCF4 type=del 2024-09-13T00:00:56.722 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.722+0000 7fb8f2ce0640 1 > prepared operation key=85YEW8N43FTA tag=4OH1ZEAMWB3A type=del size=2310971 2024-09-13T00:00:56.728 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.726+0000 7fb8f2ce0640 1 < completed part key=_multipart_2AHYRZUDFHP4.2~N699J2I5W654.0 size=1048576 2024-09-13T00:00:56.733 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.730+0000 7fb8f2ce0640 1 < completed part key=_multipart_2AHYRZUDFHP4.2~N699J2I5W654.1 size=687957 2024-09-13T00:00:56.742 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.742+0000 7fb8f2ce0640 1 < completed multipart upload key=2AHYRZUDFHP4 upload=N699J2I5W654 tag=6PQD5B451Q5D type=write size=2785109 2024-09-13T00:00:56.753 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.754+0000 7fb8f2ce0640 1 > prepared part key=_multipart_P8OYS5ZE1NX0.2~JXWE207GR7LL.0 size=165654 2024-09-13T00:00:56.753 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.754+0000 7fb8f2ce0640 1 > prepared multipart upload key=P8OYS5ZE1NX0 upload=JXWE207GR7LL tag=7AXGV4N98DU6 type=write size=1214230 2024-09-13T00:00:56.754 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.754+0000 7fb8f2ce0640 1 < canceled part key=_multipart_7OX5OJZ8GXU5.2~POPFTQ47T9EN.0 size=520293 2024-09-13T00:00:56.764 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.762+0000 7fb8f2ce0640 1 < canceled multipart upload key=7OX5OJZ8GXU5 upload=POPFTQ47T9EN tag=IGQIX0XATY2C type=write size=1568869 2024-09-13T00:00:56.769 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.770+0000 7fb8f2ce0640 1 > prepared operation key=4K76U69GZ9AE tag=M818HYKJQ6FS type=del size=3624886 2024-09-13T00:00:56.775 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.774+0000 7fb8f2ce0640 1 < completed part key=_multipart_8Z6U7OOA0GBH.2~K48EU68W4XKF.0 size=1048576 2024-09-13T00:00:56.776 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.774+0000 7fb8f2ce0640 1 < canceled part key=_multipart_8Z6U7OOA0GBH.2~K48EU68W4XKF.1 size=1048576 2024-09-13T00:00:56.780 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.778+0000 7fb8f2ce0640 1 < completed part key=_multipart_8Z6U7OOA0GBH.2~K48EU68W4XKF.2 size=194710 2024-09-13T00:00:56.790 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.790+0000 7fb8f2ce0640 1 < canceled multipart upload key=8Z6U7OOA0GBH upload=K48EU68W4XKF tag=XC4VY83LIUDP type=write size=3340438 2024-09-13T00:00:56.796 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.794+0000 7fb8f2ce0640 1 > prepared operation key=P8OYS5ZE1NX0 tag=Y29LRUY2KU9W type=write size=790562 2024-09-13T00:00:56.801 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.802+0000 7fb8f2ce0640 1 < completed delete operation key=5GM88ERM89KT tag=FVOM3VMIJ46Y type=del 2024-09-13T00:00:56.807 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.806+0000 7fb8f2ce0640 1 > prepared operation key=8Z6U7OOA0GBH tag=8XI4BLCN0LNR type=del size=910498 2024-09-13T00:00:56.813 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.814+0000 7fb8f2ce0640 1 < completed delete operation key=EJN6HR8I7XQF tag=NOZIG1EKTBCE type=del 2024-09-13T00:00:56.819 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.818+0000 7fb8f2ce0640 1 > prepared operation key=8Z6U7OOA0GBH tag=SCAS3PNOVAZ7 type=del size=3805534 2024-09-13T00:00:56.826 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.826+0000 7fb8f2ce0640 1 < completed part key=_multipart_2AHYRZUDFHP4.2~CXA4BL6SV3RG.0 size=1048576 2024-09-13T00:00:56.831 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.830+0000 7fb8f2ce0640 1 < completed part key=_multipart_2AHYRZUDFHP4.2~CXA4BL6SV3RG.1 size=1048576 2024-09-13T00:00:56.837 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.838+0000 7fb8f2ce0640 1 < completed part key=_multipart_2AHYRZUDFHP4.2~CXA4BL6SV3RG.2 size=180320 2024-09-13T00:00:56.849 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.846+0000 7fb8f2ce0640 1 < completed multipart upload key=2AHYRZUDFHP4 upload=CXA4BL6SV3RG tag=5ZDS07HTRYVR type=write size=3326048 2024-09-13T00:00:56.854 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.854+0000 7fb8f2ce0640 1 > prepared operation key=8MPUVD37EDN9 tag=MMV77X84XZQG type=del size=3284238 2024-09-13T00:00:56.861 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.862+0000 7fb8f2ce0640 1 < completed part key=_multipart_O4MZ4UWNXME9.2~H2WIHW33Z2RR.0 size=931856 2024-09-13T00:00:56.872 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.870+0000 7fb8f2ce0640 1 < completed multipart upload key=O4MZ4UWNXME9 upload=H2WIHW33Z2RR tag=O3P2CWOM42YA type=write size=1980432 2024-09-13T00:00:56.877 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.878+0000 7fb8f2ce0640 1 > prepared operation key=EJN6HR8I7XQF tag=QYPNELVEYASB type=del size=527818 2024-09-13T00:00:56.886 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.886+0000 7fb8f2ce0640 1 < completed delete operation key=H5EIP0E9CBWX tag=BG69K6M1XMJY type=del 2024-09-13T00:00:56.893 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.894+0000 7fb8f2ce0640 1 > prepared operation key=ZTK9AYBSLCOZ tag=F05VCDL5OY22 type=write size=760960 2024-09-13T00:00:56.899 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.898+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZFHN2SNTZ5W7.2~ZSXYFOJ7LMOW.0 size=681470 2024-09-13T00:00:56.910 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.910+0000 7fb8f2ce0640 1 < completed multipart upload key=ZFHN2SNTZ5W7 upload=ZSXYFOJ7LMOW tag=2WXNSIAXH1BA type=write size=1730046 2024-09-13T00:00:56.916 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.914+0000 7fb8f2ce0640 1 > prepared operation key=F0QCVR9E08WE tag=EFAAZXI0VDJ8 type=del size=513113 2024-09-13T00:00:56.917 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.918+0000 7fb8f2ce0640 1 < canceled part key=_multipart_5GM88ERM89KT.2~6B5WZR5QFYU8.0 size=1048576 2024-09-13T00:00:56.922 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.922+0000 7fb8f2ce0640 1 < completed part key=_multipart_5GM88ERM89KT.2~6B5WZR5QFYU8.1 size=730417 2024-09-13T00:00:56.933 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.934+0000 7fb8f2ce0640 1 < canceled multipart upload key=5GM88ERM89KT upload=6B5WZR5QFYU8 tag=22L787KST9ZU type=write size=2827569 2024-09-13T00:00:56.940 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.938+0000 7fb8f2ce0640 1 > prepared operation key=44EDYFT9VF8Y tag=P8FW5L0SR15F type=del size=77762 2024-09-13T00:00:56.947 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.946+0000 7fb8f2ce0640 1 < completed part key=_multipart_8Z6U7OOA0GBH.2~62LTSQLDGHM7.0 size=1010699 2024-09-13T00:00:56.957 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.958+0000 7fb8f2ce0640 1 < completed multipart upload key=8Z6U7OOA0GBH upload=62LTSQLDGHM7 tag=TN2MR40EIWGE type=write size=2059275 2024-09-13T00:00:56.963 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.962+0000 7fb8f2ce0640 1 > prepared operation key=2AHYRZUDFHP4 tag=M248BT32UFIX type=del size=3207750 2024-09-13T00:00:56.969 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.970+0000 7fb8f2ce0640 1 < completed part key=_multipart_4K76U69GZ9AE.2~BIXU7W1V36L6.0 size=1048576 2024-09-13T00:00:56.975 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.974+0000 7fb8f2ce0640 1 < completed part key=_multipart_4K76U69GZ9AE.2~BIXU7W1V36L6.1 size=1048576 2024-09-13T00:00:56.981 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.978+0000 7fb8f2ce0640 1 < completed part key=_multipart_4K76U69GZ9AE.2~BIXU7W1V36L6.2 size=565429 2024-09-13T00:00:56.995 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:56.994+0000 7fb8f2ce0640 1 < completed multipart upload key=4K76U69GZ9AE upload=BIXU7W1V36L6 tag=FP802ODW7B6D type=write size=3711157 2024-09-13T00:00:57.008 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.006+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H4M1TCN5W6XU.2~WU0CEAZCXQKF.0 size=1048576 2024-09-13T00:00:57.013 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.014+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H4M1TCN5W6XU.2~WU0CEAZCXQKF.1 size=92448 2024-09-13T00:00:57.013 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.014+0000 7fb8f2ce0640 1 > prepared multipart upload key=H4M1TCN5W6XU upload=WU0CEAZCXQKF tag=Y066HKI03SLD type=write size=2189600 2024-09-13T00:00:57.019 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.018+0000 7fb8f2ce0640 1 < completed write operation key=FX0NCGLK0KPJ tag=IDSYH4RJ8HS4 type=write size=207275 2024-09-13T00:00:57.024 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.022+0000 7fb8f2ce0640 1 > prepared operation key=1HX06CCZJ1DD tag=TTX892GCQ98W type=del size=258535 2024-09-13T00:00:57.029 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.030+0000 7fb8f2ce0640 1 < completed delete operation key=1HX06CCZJ1DD tag=1HX4N4S5UVAN type=del 2024-09-13T00:00:57.035 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.034+0000 7fb8f2ce0640 1 > prepared operation key=0WC6P8YNL0H2 tag=VAX0ID1UD2RG type=write size=250180 2024-09-13T00:00:57.040 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.038+0000 7fb8f2ce0640 1 < completed part key=_multipart_PVYZQLGYT3XU.2~ZB28AWNFU5I2.0 size=821707 2024-09-13T00:00:57.049 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.050+0000 7fb8f2ce0640 1 < completed multipart upload key=PVYZQLGYT3XU upload=ZB28AWNFU5I2 tag=UIT2OYMTPYW7 type=write size=1870283 2024-09-13T00:00:57.060 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.058+0000 7fb8f2ce0640 1 > prepared part key=_multipart_J8NGDSF9XRTP.2~FYYKZRZL8RP0.0 size=1048576 2024-09-13T00:00:57.064 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.062+0000 7fb8f2ce0640 1 > prepared part key=_multipart_J8NGDSF9XRTP.2~FYYKZRZL8RP0.1 size=1048576 2024-09-13T00:00:57.069 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.070+0000 7fb8f2ce0640 1 > prepared part key=_multipart_J8NGDSF9XRTP.2~FYYKZRZL8RP0.2 size=545370 2024-09-13T00:00:57.069 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.070+0000 7fb8f2ce0640 1 > prepared multipart upload key=J8NGDSF9XRTP upload=FYYKZRZL8RP0 tag=CQDRSEG1H1Y8 type=write size=3691098 2024-09-13T00:00:57.074 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.074+0000 7fb8f2ce0640 1 < completed delete operation key=85YEW8N43FTA tag=4OH1ZEAMWB3A type=del 2024-09-13T00:00:57.080 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.078+0000 7fb8f2ce0640 1 > prepared operation key=ZFHN2SNTZ5W7 tag=PIABXUAYLP7Z type=del size=694622 2024-09-13T00:00:57.085 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.086+0000 7fb8f2ce0640 1 < completed part key=_multipart_P8OYS5ZE1NX0.2~JXWE207GR7LL.0 size=165654 2024-09-13T00:00:57.095 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.094+0000 7fb8f2ce0640 1 < completed multipart upload key=P8OYS5ZE1NX0 upload=JXWE207GR7LL tag=7AXGV4N98DU6 type=write size=1214230 2024-09-13T00:00:57.101 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.098+0000 7fb8f2ce0640 1 > prepared operation key=70F7D20HI504 tag=TWP6SON5AOW2 type=del size=1931854 2024-09-13T00:00:57.106 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.106+0000 7fb8f2ce0640 1 < canceled operation key=4K76U69GZ9AE tag=M818HYKJQ6FS type=del size=3624886 2024-09-13T00:00:57.117 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.118+0000 7fb8f2ce0640 1 > prepared part key=_multipart_P8OYS5ZE1NX0.2~MGL2DDTRABDE.0 size=1048576 2024-09-13T00:00:57.122 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.122+0000 7fb8f2ce0640 1 > prepared part key=_multipart_P8OYS5ZE1NX0.2~MGL2DDTRABDE.1 size=1048576 2024-09-13T00:00:57.127 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.126+0000 7fb8f2ce0640 1 > prepared part key=_multipart_P8OYS5ZE1NX0.2~MGL2DDTRABDE.2 size=343536 2024-09-13T00:00:57.127 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.126+0000 7fb8f2ce0640 1 > prepared multipart upload key=P8OYS5ZE1NX0 upload=MGL2DDTRABDE tag=XYENKIC247O5 type=write size=3489264 2024-09-13T00:00:57.133 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.134+0000 7fb8f2ce0640 1 < completed write operation key=P8OYS5ZE1NX0 tag=Y29LRUY2KU9W type=write size=790562 2024-09-13T00:00:57.140 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.138+0000 7fb8f2ce0640 1 > prepared operation key=P8OYS5ZE1NX0 tag=97FP1TBEBIGH type=del size=3903284 2024-09-13T00:00:57.147 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.146+0000 7fb8f2ce0640 1 < completed delete operation key=8Z6U7OOA0GBH tag=8XI4BLCN0LNR type=del 2024-09-13T00:00:57.158 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.158+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZFHN2SNTZ5W7.2~HIV4Z7I5WIQ2.0 size=1048576 2024-09-13T00:00:57.163 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.162+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZFHN2SNTZ5W7.2~HIV4Z7I5WIQ2.1 size=1048576 2024-09-13T00:00:57.168 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.166+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZFHN2SNTZ5W7.2~HIV4Z7I5WIQ2.2 size=187395 2024-09-13T00:00:57.168 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.166+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZFHN2SNTZ5W7 upload=HIV4Z7I5WIQ2 tag=WNQJ3021FFMD type=write size=3333123 2024-09-13T00:00:57.173 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.174+0000 7fb8f2ce0640 1 < canceled operation key=8Z6U7OOA0GBH tag=SCAS3PNOVAZ7 type=del size=3805534 2024-09-13T00:00:57.179 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.178+0000 7fb8f2ce0640 1 > prepared operation key=EUE3LT0PDIXV tag=OZSJ15R1JGKM type=del size=2055048 2024-09-13T00:00:57.188 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.186+0000 7fb8f2ce0640 1 < completed delete operation key=8MPUVD37EDN9 tag=MMV77X84XZQG type=del 2024-09-13T00:00:57.194 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.194+0000 7fb8f2ce0640 1 > prepared operation key=W8XZ26311RJX tag=KUB4LF2W6X3V type=del size=648089 2024-09-13T00:00:57.199 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.198+0000 7fb8f2ce0640 1 < completed delete operation key=EJN6HR8I7XQF tag=QYPNELVEYASB type=del 2024-09-13T00:00:57.205 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.206+0000 7fb8f2ce0640 1 > prepared operation key=1HX06CCZJ1DD tag=E04FYS85IWO6 type=del size=873757 2024-09-13T00:00:57.211 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.210+0000 7fb8f2ce0640 1 < completed write operation key=ZTK9AYBSLCOZ tag=F05VCDL5OY22 type=write size=760960 2024-09-13T00:00:57.217 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.218+0000 7fb8f2ce0640 1 > prepared operation key=J8NGDSF9XRTP tag=LU48VSEG57L7 type=del size=2160977 2024-09-13T00:00:57.223 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.222+0000 7fb8f2ce0640 1 < completed delete operation key=F0QCVR9E08WE tag=EFAAZXI0VDJ8 type=del 2024-09-13T00:00:57.233 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.234+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~EQXQ24I132F8.0 size=551729 2024-09-13T00:00:57.233 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.234+0000 7fb8f2ce0640 1 > prepared multipart upload key=1HX06CCZJ1DD upload=EQXQ24I132F8 tag=RU4PB1ESR7AG type=write size=1600305 2024-09-13T00:00:57.239 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.238+0000 7fb8f2ce0640 1 < canceled operation key=44EDYFT9VF8Y tag=P8FW5L0SR15F type=del size=77762 2024-09-13T00:00:57.245 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.246+0000 7fb8f2ce0640 1 > prepared operation key=70F7D20HI504 tag=KSN219QOLJVQ type=del size=2280399 2024-09-13T00:00:57.251 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.250+0000 7fb8f2ce0640 1 < completed delete operation key=2AHYRZUDFHP4 tag=M248BT32UFIX type=del 2024-09-13T00:00:57.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:56 smithi136 bash[75164]: cluster 2024-09-13T00:00:55.137610+0000 mgr.y (mgr.24661) 714 : cluster [DBG] pgmap v1390: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 376 KiB/s rd, 131 KiB/s wr, 666 op/s 2024-09-13T00:00:57.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:56 smithi136 bash[76371]: cluster 2024-09-13T00:00:55.137610+0000 mgr.y (mgr.24661) 714 : cluster [DBG] pgmap v1390: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 376 KiB/s rd, 131 KiB/s wr, 666 op/s 2024-09-13T00:00:57.263 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.262+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~JE8KWTWE2VHT.0 size=1048576 2024-09-13T00:00:57.268 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.266+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~JE8KWTWE2VHT.1 size=1048576 2024-09-13T00:00:57.273 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.274+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~JE8KWTWE2VHT.2 size=63553 2024-09-13T00:00:57.273 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.274+0000 7fb8f2ce0640 1 > prepared multipart upload key=PVYZQLGYT3XU upload=JE8KWTWE2VHT tag=77M3LPHVMIMR type=write size=3209281 2024-09-13T00:00:57.279 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.278+0000 7fb8f2ce0640 1 < completed part key=_multipart_H4M1TCN5W6XU.2~WU0CEAZCXQKF.0 size=1048576 2024-09-13T00:00:57.284 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.282+0000 7fb8f2ce0640 1 < completed part key=_multipart_H4M1TCN5W6XU.2~WU0CEAZCXQKF.1 size=92448 2024-09-13T00:00:57.296 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.294+0000 7fb8f2ce0640 1 < completed multipart upload key=H4M1TCN5W6XU upload=WU0CEAZCXQKF tag=Y066HKI03SLD type=write size=2189600 2024-09-13T00:00:57.308 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.306+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~A9LHVPXZ4VAU.0 size=642700 2024-09-13T00:00:57.308 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.306+0000 7fb8f2ce0640 1 > prepared multipart upload key=NQJQC8CEAC15 upload=A9LHVPXZ4VAU tag=KJYFRFETYTNG type=write size=1691276 2024-09-13T00:00:57.315 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.314+0000 7fb8f2ce0640 1 < completed delete operation key=1HX06CCZJ1DD tag=TTX892GCQ98W type=del 2024-09-13T00:00:57.321 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.322+0000 7fb8f2ce0640 1 > prepared operation key=QMZREZ1QEDNJ tag=A3937OEJCAP2 type=del size=3623705 2024-09-13T00:00:57.336 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.334+0000 7fb8f2ce0640 1 < suggested update operation key=0WC6P8YNL0H2 tag=VAX0ID1UD2RG type=write size=250180 2024-09-13T00:00:57.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:56 smithi181 bash[67587]: cluster 2024-09-13T00:00:55.137610+0000 mgr.y (mgr.24661) 714 : cluster [DBG] pgmap v1390: 137 pgs: 137 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 376 KiB/s rd, 131 KiB/s wr, 666 op/s 2024-09-13T00:00:57.348 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.346+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~E3767YV9A36Z.0 size=1048576 2024-09-13T00:00:57.353 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.354+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~E3767YV9A36Z.1 size=1048576 2024-09-13T00:00:57.358 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.358+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~E3767YV9A36Z.2 size=891167 2024-09-13T00:00:57.358 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.358+0000 7fb8f2ce0640 1 > prepared multipart upload key=70F7D20HI504 upload=E3767YV9A36Z tag=5R4686PHPAYT type=write size=4036895 2024-09-13T00:00:57.367 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.366+0000 7fb8f2ce0640 1 < completed part key=_multipart_J8NGDSF9XRTP.2~FYYKZRZL8RP0.0 size=1048576 2024-09-13T00:00:57.373 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.374+0000 7fb8f2ce0640 1 < completed part key=_multipart_J8NGDSF9XRTP.2~FYYKZRZL8RP0.1 size=1048576 2024-09-13T00:00:57.379 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.378+0000 7fb8f2ce0640 1 < completed part key=_multipart_J8NGDSF9XRTP.2~FYYKZRZL8RP0.2 size=545370 2024-09-13T00:00:57.390 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.390+0000 7fb8f2ce0640 1 < completed multipart upload key=J8NGDSF9XRTP upload=FYYKZRZL8RP0 tag=CQDRSEG1H1Y8 type=write size=3691098 2024-09-13T00:00:57.395 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.394+0000 7fb8f2ce0640 1 > prepared operation key=PVYZQLGYT3XU tag=DGF9PZ29CYZM type=write size=603238 2024-09-13T00:00:57.401 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.402+0000 7fb8f2ce0640 1 < completed delete operation key=ZFHN2SNTZ5W7 tag=PIABXUAYLP7Z type=del 2024-09-13T00:00:57.407 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.406+0000 7fb8f2ce0640 1 > prepared operation key=H5EIP0E9CBWX tag=62VU9RE5MKRT type=write size=2917 2024-09-13T00:00:57.413 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.414+0000 7fb8f2ce0640 1 < completed delete operation key=70F7D20HI504 tag=TWP6SON5AOW2 type=del 2024-09-13T00:00:57.418 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.418+0000 7fb8f2ce0640 1 > prepared operation key=EJN6HR8I7XQF tag=O5EIM7JEABWR type=del size=3911957 2024-09-13T00:00:57.424 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.422+0000 7fb8f2ce0640 1 < completed part key=_multipart_P8OYS5ZE1NX0.2~MGL2DDTRABDE.0 size=1048576 2024-09-13T00:00:57.429 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.430+0000 7fb8f2ce0640 1 < completed part key=_multipart_P8OYS5ZE1NX0.2~MGL2DDTRABDE.1 size=1048576 2024-09-13T00:00:57.435 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.434+0000 7fb8f2ce0640 1 < completed part key=_multipart_P8OYS5ZE1NX0.2~MGL2DDTRABDE.2 size=343536 2024-09-13T00:00:57.445 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.446+0000 7fb8f2ce0640 1 < completed multipart upload key=P8OYS5ZE1NX0 upload=MGL2DDTRABDE tag=XYENKIC247O5 type=write size=3489264 2024-09-13T00:00:57.456 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.454+0000 7fb8f2ce0640 1 > prepared part key=_multipart_QMZREZ1QEDNJ.2~7L9OSIOCS1QI.0 size=1048576 2024-09-13T00:00:57.462 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.462+0000 7fb8f2ce0640 1 > prepared part key=_multipart_QMZREZ1QEDNJ.2~7L9OSIOCS1QI.1 size=1048576 2024-09-13T00:00:57.467 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.466+0000 7fb8f2ce0640 1 > prepared part key=_multipart_QMZREZ1QEDNJ.2~7L9OSIOCS1QI.2 size=569297 2024-09-13T00:00:57.467 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.466+0000 7fb8f2ce0640 1 > prepared multipart upload key=QMZREZ1QEDNJ upload=7L9OSIOCS1QI tag=GFI4AHZXYT9Q type=write size=3715025 2024-09-13T00:00:57.473 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.474+0000 7fb8f2ce0640 1 < completed delete operation key=P8OYS5ZE1NX0 tag=97FP1TBEBIGH type=del 2024-09-13T00:00:57.485 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.486+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~9CZWVXA177TT.0 size=515028 2024-09-13T00:00:57.485 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.486+0000 7fb8f2ce0640 1 > prepared multipart upload key=2AHYRZUDFHP4 upload=9CZWVXA177TT tag=I39LTT5K1YBS type=write size=1563604 2024-09-13T00:00:57.492 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.490+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZFHN2SNTZ5W7.2~HIV4Z7I5WIQ2.0 size=1048576 2024-09-13T00:00:57.492 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.490+0000 7fb8f2ce0640 1 < canceled part key=_multipart_ZFHN2SNTZ5W7.2~HIV4Z7I5WIQ2.1 size=1048576 2024-09-13T00:00:57.498 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.498+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZFHN2SNTZ5W7.2~HIV4Z7I5WIQ2.2 size=187395 2024-09-13T00:00:57.509 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.510+0000 7fb8f2ce0640 1 < canceled multipart upload key=ZFHN2SNTZ5W7 upload=HIV4Z7I5WIQ2 tag=WNQJ3021FFMD type=write size=3333123 2024-09-13T00:00:57.515 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.514+0000 7fb8f2ce0640 1 > prepared operation key=NQJQC8CEAC15 tag=D1F1KA01ERU2 type=del size=4179292 2024-09-13T00:00:57.527 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.526+0000 7fb8f2ce0640 1 < suggested update operation key=EUE3LT0PDIXV tag=OZSJ15R1JGKM type=del size=2055048 2024-09-13T00:00:57.538 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.538+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~KCPJOAZV0C8B.0 size=538861 2024-09-13T00:00:57.538 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.538+0000 7fb8f2ce0640 1 > prepared multipart upload key=1PX7DBO9M3YI upload=KCPJOAZV0C8B tag=AQ3O3V9PSE2V type=write size=1587437 2024-09-13T00:00:57.544 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.542+0000 7fb8f2ce0640 1 < completed delete operation key=W8XZ26311RJX tag=KUB4LF2W6X3V type=del 2024-09-13T00:00:57.556 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.554+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EJN6HR8I7XQF.2~JHFUUQFLRYV2.0 size=1048576 2024-09-13T00:00:57.561 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.558+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EJN6HR8I7XQF.2~JHFUUQFLRYV2.1 size=1048576 2024-09-13T00:00:57.568 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.566+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EJN6HR8I7XQF.2~JHFUUQFLRYV2.2 size=451065 2024-09-13T00:00:57.568 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.566+0000 7fb8f2ce0640 1 > prepared multipart upload key=EJN6HR8I7XQF upload=JHFUUQFLRYV2 tag=310YCIQ0T24E type=write size=3596793 2024-09-13T00:00:57.573 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.574+0000 7fb8f2ce0640 1 < completed delete operation key=1HX06CCZJ1DD tag=E04FYS85IWO6 type=del 2024-09-13T00:00:57.579 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.578+0000 7fb8f2ce0640 1 > prepared operation key=O4MZ4UWNXME9 tag=2O7WIOH1NSR7 type=write size=157406 2024-09-13T00:00:57.585 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.586+0000 7fb8f2ce0640 1 < completed delete operation key=J8NGDSF9XRTP tag=LU48VSEG57L7 type=del 2024-09-13T00:00:57.591 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.590+0000 7fb8f2ce0640 1 > prepared operation key=W8XZ26311RJX tag=XTHRY0DITOMB type=del size=999113 2024-09-13T00:00:57.597 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.598+0000 7fb8f2ce0640 1 < completed part key=_multipart_1HX06CCZJ1DD.2~EQXQ24I132F8.0 size=551729 2024-09-13T00:00:57.607 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.606+0000 7fb8f2ce0640 1 < completed multipart upload key=1HX06CCZJ1DD upload=EQXQ24I132F8 tag=RU4PB1ESR7AG type=write size=1600305 2024-09-13T00:00:57.613 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.614+0000 7fb8f2ce0640 1 > prepared operation key=2AHYRZUDFHP4 tag=WC2RNRTTS5NA type=del size=2778796 2024-09-13T00:00:57.625 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.626+0000 7fb8f2ce0640 1 < suggested update operation key=70F7D20HI504 tag=KSN219QOLJVQ type=del size=2280399 2024-09-13T00:00:57.631 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.630+0000 7fb8f2ce0640 1 > prepared operation key=J8NGDSF9XRTP tag=HT4KTSBXLWOW type=write size=428191 2024-09-13T00:00:57.638 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.638+0000 7fb8f2ce0640 1 < completed part key=_multipart_PVYZQLGYT3XU.2~JE8KWTWE2VHT.0 size=1048576 2024-09-13T00:00:57.638 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.638+0000 7fb8f2ce0640 1 < canceled part key=_multipart_PVYZQLGYT3XU.2~JE8KWTWE2VHT.1 size=1048576 2024-09-13T00:00:57.643 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.642+0000 7fb8f2ce0640 1 < completed part key=_multipart_PVYZQLGYT3XU.2~JE8KWTWE2VHT.2 size=63553 2024-09-13T00:00:57.654 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.654+0000 7fb8f2ce0640 1 < canceled multipart upload key=PVYZQLGYT3XU upload=JE8KWTWE2VHT tag=77M3LPHVMIMR type=write size=3209281 2024-09-13T00:00:57.660 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.658+0000 7fb8f2ce0640 1 > prepared operation key=ZTK9AYBSLCOZ tag=2Y40X0TLC041 type=del size=2338089 2024-09-13T00:00:57.666 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.666+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~A9LHVPXZ4VAU.0 size=642700 2024-09-13T00:00:57.676 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.674+0000 7fb8f2ce0640 1 < completed multipart upload key=NQJQC8CEAC15 upload=A9LHVPXZ4VAU tag=KJYFRFETYTNG type=write size=1691276 2024-09-13T00:00:57.682 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.682+0000 7fb8f2ce0640 1 > prepared operation key=8MPUVD37EDN9 tag=NGBUBSNEYF18 type=del size=146718 2024-09-13T00:00:57.688 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.686+0000 7fb8f2ce0640 1 < completed delete operation key=QMZREZ1QEDNJ tag=A3937OEJCAP2 type=del 2024-09-13T00:00:57.694 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.694+0000 7fb8f2ce0640 1 > prepared operation key=5GM88ERM89KT tag=KGIZ0B5WVYHP type=del size=2346613 2024-09-13T00:00:57.700 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.698+0000 7fb8f2ce0640 1 < completed part key=_multipart_70F7D20HI504.2~E3767YV9A36Z.0 size=1048576 2024-09-13T00:00:57.704 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.702+0000 7fb8f2ce0640 1 < completed part key=_multipart_70F7D20HI504.2~E3767YV9A36Z.1 size=1048576 2024-09-13T00:00:57.709 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.710+0000 7fb8f2ce0640 1 < completed part key=_multipart_70F7D20HI504.2~E3767YV9A36Z.2 size=891167 2024-09-13T00:00:57.720 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.718+0000 7fb8f2ce0640 1 < completed multipart upload key=70F7D20HI504 upload=E3767YV9A36Z tag=5R4686PHPAYT type=write size=4036895 2024-09-13T00:00:57.726 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.726+0000 7fb8f2ce0640 1 > prepared operation key=5GM88ERM89KT tag=NEZALO080F20 type=write size=624898 2024-09-13T00:00:57.738 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.738+0000 7fb8f2ce0640 1 < suggested update operation key=PVYZQLGYT3XU tag=DGF9PZ29CYZM type=write size=603238 2024-09-13T00:00:57.751 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.750+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EJN6HR8I7XQF.2~V8Y2TGHJS70Q.0 size=280343 2024-09-13T00:00:57.751 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.750+0000 7fb8f2ce0640 1 > prepared multipart upload key=EJN6HR8I7XQF upload=V8Y2TGHJS70Q tag=M2YGCIEH0TBT type=write size=1328919 2024-09-13T00:00:57.756 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.754+0000 7fb8f2ce0640 1 < completed write operation key=H5EIP0E9CBWX tag=62VU9RE5MKRT type=write size=2917 2024-09-13T00:00:57.767 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.766+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~XU7PH8PLDE44.0 size=336545 2024-09-13T00:00:57.767 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.766+0000 7fb8f2ce0640 1 > prepared multipart upload key=2AHYRZUDFHP4 upload=XU7PH8PLDE44 tag=314MGCVFIDNW type=write size=1385121 2024-09-13T00:00:57.772 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.770+0000 7fb8f2ce0640 1 < completed delete operation key=EJN6HR8I7XQF tag=O5EIM7JEABWR type=del 2024-09-13T00:00:57.783 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.782+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~RXCAP7NLE8MY.0 size=1048576 2024-09-13T00:00:57.788 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.786+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~RXCAP7NLE8MY.1 size=58682 2024-09-13T00:00:57.788 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.786+0000 7fb8f2ce0640 1 > prepared multipart upload key=1PX7DBO9M3YI upload=RXCAP7NLE8MY tag=XYR6QJPKC6SM type=write size=2155834 2024-09-13T00:00:57.794 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.794+0000 7fb8f2ce0640 1 < completed part key=_multipart_QMZREZ1QEDNJ.2~7L9OSIOCS1QI.0 size=1048576 2024-09-13T00:00:57.799 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.798+0000 7fb8f2ce0640 1 < completed part key=_multipart_QMZREZ1QEDNJ.2~7L9OSIOCS1QI.1 size=1048576 2024-09-13T00:00:57.804 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.802+0000 7fb8f2ce0640 1 < completed part key=_multipart_QMZREZ1QEDNJ.2~7L9OSIOCS1QI.2 size=569297 2024-09-13T00:00:57.815 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.814+0000 7fb8f2ce0640 1 < completed multipart upload key=QMZREZ1QEDNJ upload=7L9OSIOCS1QI tag=GFI4AHZXYT9Q type=write size=3715025 2024-09-13T00:00:57.826 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.826+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H9AU4HD247YU.2~I6LGOJNVURSV.0 size=1048576 2024-09-13T00:00:57.832 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.830+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H9AU4HD247YU.2~I6LGOJNVURSV.1 size=679439 2024-09-13T00:00:57.833 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.830+0000 7fb8f2ce0640 1 > prepared multipart upload key=H9AU4HD247YU upload=I6LGOJNVURSV tag=3W1S3AQPAA07 type=write size=2776591 2024-09-13T00:00:57.838 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.838+0000 7fb8f2ce0640 1 < completed part key=_multipart_2AHYRZUDFHP4.2~9CZWVXA177TT.0 size=515028 2024-09-13T00:00:57.848 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.846+0000 7fb8f2ce0640 1 < completed multipart upload key=2AHYRZUDFHP4 upload=9CZWVXA177TT tag=I39LTT5K1YBS type=write size=1563604 2024-09-13T00:00:57.854 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.854+0000 7fb8f2ce0640 1 > prepared operation key=J8NGDSF9XRTP tag=UCKAZ7HVU4KO type=del size=3719806 2024-09-13T00:00:57.860 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.858+0000 7fb8f2ce0640 1 < completed delete operation key=NQJQC8CEAC15 tag=D1F1KA01ERU2 type=del 2024-09-13T00:00:57.866 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.866+0000 7fb8f2ce0640 1 > prepared operation key=8MPUVD37EDN9 tag=HDSW437NB6JG type=write size=654574 2024-09-13T00:00:57.871 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.870+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~KCPJOAZV0C8B.0 size=538861 2024-09-13T00:00:57.881 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.882+0000 7fb8f2ce0640 1 < completed multipart upload key=1PX7DBO9M3YI upload=KCPJOAZV0C8B tag=AQ3O3V9PSE2V type=write size=1587437 2024-09-13T00:00:57.887 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.886+0000 7fb8f2ce0640 1 > prepared operation key=F0QCVR9E08WE tag=KL5D7MRBW0SI type=del size=3764476 2024-09-13T00:00:57.893 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.894+0000 7fb8f2ce0640 1 < completed part key=_multipart_EJN6HR8I7XQF.2~JHFUUQFLRYV2.0 size=1048576 2024-09-13T00:00:57.899 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.898+0000 7fb8f2ce0640 1 < completed part key=_multipart_EJN6HR8I7XQF.2~JHFUUQFLRYV2.1 size=1048576 2024-09-13T00:00:57.905 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.906+0000 7fb8f2ce0640 1 < completed part key=_multipart_EJN6HR8I7XQF.2~JHFUUQFLRYV2.2 size=451065 2024-09-13T00:00:57.916 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.914+0000 7fb8f2ce0640 1 < completed multipart upload key=EJN6HR8I7XQF upload=JHFUUQFLRYV2 tag=310YCIQ0T24E type=write size=3596793 2024-09-13T00:00:57.923 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.922+0000 7fb8f2ce0640 1 > prepared operation key=5GM88ERM89KT tag=PREFCRDGP040 type=del size=1211794 2024-09-13T00:00:57.936 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.934+0000 7fb8f2ce0640 1 < suggested remove operation key=O4MZ4UWNXME9 tag=2O7WIOH1NSR7 type=write 2024-09-13T00:00:57.942 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.942+0000 7fb8f2ce0640 1 > prepared operation key=NX26SI4F2ZMB tag=TBKP824TGDFE type=del size=3370061 2024-09-13T00:00:57.949 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.946+0000 7fb8f2ce0640 1 < completed delete operation key=W8XZ26311RJX tag=XTHRY0DITOMB type=del 2024-09-13T00:00:57.959 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.958+0000 7fb8f2ce0640 1 > prepared part key=_multipart_O4MZ4UWNXME9.2~3T1TSQIZC159.0 size=1042466 2024-09-13T00:00:57.959 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.958+0000 7fb8f2ce0640 1 > prepared multipart upload key=O4MZ4UWNXME9 upload=3T1TSQIZC159 tag=8HL2U4M7QPQ9 type=write size=2091042 2024-09-13T00:00:57.965 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.966+0000 7fb8f2ce0640 1 < completed delete operation key=2AHYRZUDFHP4 tag=WC2RNRTTS5NA type=del 2024-09-13T00:00:57.971 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.970+0000 7fb8f2ce0640 1 > prepared operation key=8Z6U7OOA0GBH tag=LE1MD2BHNLAT type=write size=885977 2024-09-13T00:00:57.982 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.982+0000 7fb8f2ce0640 1 < suggested remove operation key=J8NGDSF9XRTP tag=HT4KTSBXLWOW type=write 2024-09-13T00:00:57.995 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.994+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~9DDXD2K5T71X.0 size=1048576 2024-09-13T00:00:58.000 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.998+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~9DDXD2K5T71X.1 size=114682 2024-09-13T00:00:58.000 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:57.998+0000 7fb8f2ce0640 1 > prepared multipart upload key=8MPUVD37EDN9 upload=9DDXD2K5T71X tag=99Q12YZAKYID type=write size=2211834 2024-09-13T00:00:58.006 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.006+0000 7fb8f2ce0640 1 < completed delete operation key=ZTK9AYBSLCOZ tag=2Y40X0TLC041 type=del 2024-09-13T00:00:58.020 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.018+0000 7fb8f2ce0640 1 > prepared part key=_multipart_5GM88ERM89KT.2~DCJ5ZZJJVCJD.0 size=1048576 2024-09-13T00:00:58.025 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.022+0000 7fb8f2ce0640 1 > prepared part key=_multipart_5GM88ERM89KT.2~DCJ5ZZJJVCJD.1 size=498317 2024-09-13T00:00:58.025 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.022+0000 7fb8f2ce0640 1 > prepared multipart upload key=5GM88ERM89KT upload=DCJ5ZZJJVCJD tag=V4EWTQ6GL2ZJ type=write size=2595469 2024-09-13T00:00:58.030 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.030+0000 7fb8f2ce0640 1 < completed delete operation key=8MPUVD37EDN9 tag=NGBUBSNEYF18 type=del 2024-09-13T00:00:58.042 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.042+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~UM23V8ZX2G2I.0 size=1048576 2024-09-13T00:00:58.047 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.046+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~UM23V8ZX2G2I.1 size=1048576 2024-09-13T00:00:58.054 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.054+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~UM23V8ZX2G2I.2 size=609190 2024-09-13T00:00:58.054 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.054+0000 7fb8f2ce0640 1 > prepared multipart upload key=PVYZQLGYT3XU upload=UM23V8ZX2G2I tag=FAMWZ9V2RRAV type=write size=3754918 2024-09-13T00:00:58.060 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.058+0000 7fb8f2ce0640 1 < completed delete operation key=5GM88ERM89KT tag=KGIZ0B5WVYHP type=del 2024-09-13T00:00:58.067 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.066+0000 7fb8f2ce0640 1 > prepared operation key=PVYZQLGYT3XU tag=027VD47QSURD type=del size=3649241 2024-09-13T00:00:58.073 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.074+0000 7fb8f2ce0640 1 < completed write operation key=5GM88ERM89KT tag=NEZALO080F20 type=write size=624898 2024-09-13T00:00:58.084 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.082+0000 7fb8f2ce0640 1 > prepared part key=_multipart_4K76U69GZ9AE.2~I0JUJS1UMBT3.0 size=642257 2024-09-13T00:00:58.084 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.082+0000 7fb8f2ce0640 1 > prepared multipart upload key=4K76U69GZ9AE upload=I0JUJS1UMBT3 tag=FNRVR0XQ884A type=write size=1690833 2024-09-13T00:00:58.090 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.090+0000 7fb8f2ce0640 1 < completed part key=_multipart_EJN6HR8I7XQF.2~V8Y2TGHJS70Q.0 size=280343 2024-09-13T00:00:58.100 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.098+0000 7fb8f2ce0640 1 < completed multipart upload key=EJN6HR8I7XQF upload=V8Y2TGHJS70Q tag=M2YGCIEH0TBT type=write size=1328919 2024-09-13T00:00:58.111 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.110+0000 7fb8f2ce0640 1 > prepared part key=_multipart_J8NGDSF9XRTP.2~MNTQC9D4P3QD.0 size=1048576 2024-09-13T00:00:58.116 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.114+0000 7fb8f2ce0640 1 > prepared part key=_multipart_J8NGDSF9XRTP.2~MNTQC9D4P3QD.1 size=77880 2024-09-13T00:00:58.116 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.114+0000 7fb8f2ce0640 1 > prepared multipart upload key=J8NGDSF9XRTP upload=MNTQC9D4P3QD tag=0RZK44GX9QGQ type=write size=2175032 2024-09-13T00:00:58.122 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.122+0000 7fb8f2ce0640 1 < completed part key=_multipart_2AHYRZUDFHP4.2~XU7PH8PLDE44.0 size=336545 2024-09-13T00:00:58.132 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.130+0000 7fb8f2ce0640 1 < completed multipart upload key=2AHYRZUDFHP4 upload=XU7PH8PLDE44 tag=314MGCVFIDNW type=write size=1385121 2024-09-13T00:00:58.140 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.138+0000 7fb8f2ce0640 1 > prepared operation key=PVYZQLGYT3XU tag=6U3A7DXSM1P8 type=del size=1917202 2024-09-13T00:00:58.146 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.146+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~RXCAP7NLE8MY.0 size=1048576 2024-09-13T00:00:58.151 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.150+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~RXCAP7NLE8MY.1 size=58682 2024-09-13T00:00:58.162 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.162+0000 7fb8f2ce0640 1 < completed multipart upload key=1PX7DBO9M3YI upload=RXCAP7NLE8MY tag=XYR6QJPKC6SM type=write size=2155834 2024-09-13T00:00:58.168 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.166+0000 7fb8f2ce0640 1 > prepared operation key=P8OYS5ZE1NX0 tag=OC3A959AC53D type=del size=3508736 2024-09-13T00:00:58.174 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.174+0000 7fb8f2ce0640 1 < completed part key=_multipart_H9AU4HD247YU.2~I6LGOJNVURSV.0 size=1048576 2024-09-13T00:00:58.179 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.178+0000 7fb8f2ce0640 1 < completed part key=_multipart_H9AU4HD247YU.2~I6LGOJNVURSV.1 size=679439 2024-09-13T00:00:58.189 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.190+0000 7fb8f2ce0640 1 < completed multipart upload key=H9AU4HD247YU upload=I6LGOJNVURSV tag=3W1S3AQPAA07 type=write size=2776591 2024-09-13T00:00:58.195 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.194+0000 7fb8f2ce0640 1 > prepared operation key=J8NGDSF9XRTP tag=2AUAAEGCE6BG type=write size=907810 2024-09-13T00:00:58.203 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.202+0000 7fb8f2ce0640 -1 < failed to complete operation key=J8NGDSF9XRTP tag=UCKAZ7HVU4KO type=del size=3719806: (22) Invalid argument 2024-09-13T00:00:58.210 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.210+0000 7fb8f2ce0640 1 > prepared operation key=1HX06CCZJ1DD tag=T2OMGXRUXA8J type=write size=602148 2024-09-13T00:00:58.221 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.222+0000 7fb8f2ce0640 1 < suggested update operation key=8MPUVD37EDN9 tag=HDSW437NB6JG type=write size=654574 2024-09-13T00:00:58.227 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.226+0000 7fb8f2ce0640 1 > prepared operation key=8MPUVD37EDN9 tag=C8OQTHOAIO4X type=del size=3183118 2024-09-13T00:00:58.239 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.238+0000 7fb8f2ce0640 1 < suggested update operation key=F0QCVR9E08WE tag=KL5D7MRBW0SI type=del size=3764476 2024-09-13T00:00:58.245 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.246+0000 7fb8f2ce0640 1 > prepared operation key=0WC6P8YNL0H2 tag=EC3F7NKQJ4NO type=del size=3175183 2024-09-13T00:00:58.252 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.250+0000 7fb8f2ce0640 1 < completed delete operation key=5GM88ERM89KT tag=PREFCRDGP040 type=del 2024-09-13T00:00:58.258 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.258+0000 7fb8f2ce0640 1 > prepared operation key=ZFHN2SNTZ5W7 tag=N43U4VXDQ4H6 type=del size=2483022 2024-09-13T00:00:58.271 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.270+0000 7fb8f2ce0640 1 < suggested remove operation key=NX26SI4F2ZMB tag=TBKP824TGDFE type=del 2024-09-13T00:00:58.282 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.282+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~JD7R30OYCW8W.0 size=1048576 2024-09-13T00:00:58.286 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.286+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~JD7R30OYCW8W.1 size=1048576 2024-09-13T00:00:58.292 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.290+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~JD7R30OYCW8W.2 size=976813 2024-09-13T00:00:58.292 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.290+0000 7fb8f2ce0640 1 > prepared multipart upload key=1HX06CCZJ1DD upload=JD7R30OYCW8W tag=QFYRCTYINQ9G type=write size=4122541 2024-09-13T00:00:58.298 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.298+0000 7fb8f2ce0640 1 < completed part key=_multipart_O4MZ4UWNXME9.2~3T1TSQIZC159.0 size=1042466 2024-09-13T00:00:58.307 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.306+0000 7fb8f2ce0640 1 < completed multipart upload key=O4MZ4UWNXME9 upload=3T1TSQIZC159 tag=8HL2U4M7QPQ9 type=write size=2091042 2024-09-13T00:00:58.313 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.314+0000 7fb8f2ce0640 1 > prepared operation key=P8OYS5ZE1NX0 tag=34C8LE39IC7D type=del size=3366168 2024-09-13T00:00:58.325 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.326+0000 7fb8f2ce0640 1 < suggested remove operation key=8Z6U7OOA0GBH tag=LE1MD2BHNLAT type=write 2024-09-13T00:00:58.331 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.330+0000 7fb8f2ce0640 1 > prepared operation key=F0QCVR9E08WE tag=YEMW1T83S7W8 type=del size=2401788 2024-09-13T00:00:58.336 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.334+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~9DDXD2K5T71X.0 size=1048576 2024-09-13T00:00:58.341 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.342+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~9DDXD2K5T71X.1 size=114682 2024-09-13T00:00:58.351 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.350+0000 7fb8f2ce0640 1 < completed multipart upload key=8MPUVD37EDN9 upload=9DDXD2K5T71X tag=99Q12YZAKYID type=write size=2211834 2024-09-13T00:00:58.362 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.362+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EUE3LT0PDIXV.2~EFTN6FYJX6N6.0 size=1048576 2024-09-13T00:00:58.367 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.366+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EUE3LT0PDIXV.2~EFTN6FYJX6N6.1 size=90566 2024-09-13T00:00:58.367 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.366+0000 7fb8f2ce0640 1 > prepared multipart upload key=EUE3LT0PDIXV upload=EFTN6FYJX6N6 tag=0TYOTXL05OSI type=write size=2187718 2024-09-13T00:00:58.373 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.374+0000 7fb8f2ce0640 1 < completed part key=_multipart_5GM88ERM89KT.2~DCJ5ZZJJVCJD.0 size=1048576 2024-09-13T00:00:58.378 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.378+0000 7fb8f2ce0640 1 < completed part key=_multipart_5GM88ERM89KT.2~DCJ5ZZJJVCJD.1 size=498317 2024-09-13T00:00:58.387 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.386+0000 7fb8f2ce0640 1 < completed multipart upload key=5GM88ERM89KT upload=DCJ5ZZJJVCJD tag=V4EWTQ6GL2ZJ type=write size=2595469 2024-09-13T00:00:58.394 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.394+0000 7fb8f2ce0640 1 > prepared operation key=8Z6U7OOA0GBH tag=XDQZ1Q2DITVC type=write size=786250 2024-09-13T00:00:58.399 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.398+0000 7fb8f2ce0640 1 < completed part key=_multipart_PVYZQLGYT3XU.2~UM23V8ZX2G2I.0 size=1048576 2024-09-13T00:00:58.404 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.402+0000 7fb8f2ce0640 1 < completed part key=_multipart_PVYZQLGYT3XU.2~UM23V8ZX2G2I.1 size=1048576 2024-09-13T00:00:58.409 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.406+0000 7fb8f2ce0640 1 < completed part key=_multipart_PVYZQLGYT3XU.2~UM23V8ZX2G2I.2 size=609190 2024-09-13T00:00:58.419 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.418+0000 7fb8f2ce0640 1 < completed multipart upload key=PVYZQLGYT3XU upload=UM23V8ZX2G2I tag=FAMWZ9V2RRAV type=write size=3754918 2024-09-13T00:00:58.425 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.426+0000 7fb8f2ce0640 1 > prepared operation key=H4M1TCN5W6XU tag=L1INBS1GFRSB type=del size=2808417 2024-09-13T00:00:58.430 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.430+0000 7fb8f2ce0640 1 < completed delete operation key=PVYZQLGYT3XU tag=027VD47QSURD type=del 2024-09-13T00:00:58.435 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.434+0000 7fb8f2ce0640 1 > prepared operation key=H9AU4HD247YU tag=NCK0VUW2WMQ0 type=del size=4025167 2024-09-13T00:00:58.441 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.442+0000 7fb8f2ce0640 1 < completed part key=_multipart_4K76U69GZ9AE.2~I0JUJS1UMBT3.0 size=642257 2024-09-13T00:00:58.451 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.450+0000 7fb8f2ce0640 1 < completed multipart upload key=4K76U69GZ9AE upload=I0JUJS1UMBT3 tag=FNRVR0XQ884A type=write size=1690833 2024-09-13T00:00:58.457 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.458+0000 7fb8f2ce0640 1 > prepared operation key=QMZREZ1QEDNJ tag=AA4DQVNBJF8P type=write size=705718 2024-09-13T00:00:58.462 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.462+0000 7fb8f2ce0640 1 < completed part key=_multipart_J8NGDSF9XRTP.2~MNTQC9D4P3QD.0 size=1048576 2024-09-13T00:00:58.467 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.466+0000 7fb8f2ce0640 1 < completed part key=_multipart_J8NGDSF9XRTP.2~MNTQC9D4P3QD.1 size=77880 2024-09-13T00:00:58.478 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.478+0000 7fb8f2ce0640 1 < completed multipart upload key=J8NGDSF9XRTP upload=MNTQC9D4P3QD tag=0RZK44GX9QGQ type=write size=2175032 2024-09-13T00:00:58.484 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.482+0000 7fb8f2ce0640 1 > prepared operation key=H9AU4HD247YU tag=E5O24NHE51DI type=del size=3221118 2024-09-13T00:00:58.495 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.494+0000 7fb8f2ce0640 1 < suggested update operation key=PVYZQLGYT3XU tag=6U3A7DXSM1P8 type=del size=1917202 2024-09-13T00:00:58.506 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.506+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H5EIP0E9CBWX.2~KMH01W929CWH.0 size=1048576 2024-09-13T00:00:58.512 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.510+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H5EIP0E9CBWX.2~KMH01W929CWH.1 size=1048576 2024-09-13T00:00:58.517 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.518+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H5EIP0E9CBWX.2~KMH01W929CWH.2 size=50686 2024-09-13T00:00:58.517 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.518+0000 7fb8f2ce0640 1 > prepared multipart upload key=H5EIP0E9CBWX upload=KMH01W929CWH tag=ECSGRFNF2IYF type=write size=3196414 2024-09-13T00:00:58.523 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.522+0000 7fb8f2ce0640 1 < completed delete operation key=P8OYS5ZE1NX0 tag=OC3A959AC53D type=del 2024-09-13T00:00:58.530 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.530+0000 7fb8f2ce0640 1 > prepared operation key=63FJDWBO2T9S tag=AYE493B07LYO type=del size=3636904 2024-09-13T00:00:58.536 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.534+0000 7fb8f2ce0640 1 < completed write operation key=J8NGDSF9XRTP tag=2AUAAEGCE6BG type=write size=907810 2024-09-13T00:00:58.542 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.542+0000 7fb8f2ce0640 1 > prepared operation key=EUE3LT0PDIXV tag=4JZIH6I7UCS3 type=del size=2435509 2024-09-13T00:00:58.555 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.554+0000 7fb8f2ce0640 1 < suggested remove operation key=1HX06CCZJ1DD tag=T2OMGXRUXA8J type=write 2024-09-13T00:00:58.566 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.566+0000 7fb8f2ce0640 1 > prepared part key=_multipart_N3Z5XIG3TZRQ.2~8QN6QESE1MUR.0 size=1048576 2024-09-13T00:00:58.571 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.570+0000 7fb8f2ce0640 1 > prepared part key=_multipart_N3Z5XIG3TZRQ.2~8QN6QESE1MUR.1 size=929988 2024-09-13T00:00:58.572 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.570+0000 7fb8f2ce0640 1 > prepared multipart upload key=N3Z5XIG3TZRQ upload=8QN6QESE1MUR tag=JYJ8LGY4FSNT type=write size=3027140 2024-09-13T00:00:58.578 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.578+0000 7fb8f2ce0640 1 < completed delete operation key=8MPUVD37EDN9 tag=C8OQTHOAIO4X type=del 2024-09-13T00:00:58.585 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.582+0000 7fb8f2ce0640 1 > prepared operation key=85YEW8N43FTA tag=J7WWVV49PPI3 type=del size=657494 2024-09-13T00:00:58.591 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.590+0000 7fb8f2ce0640 1 < completed delete operation key=0WC6P8YNL0H2 tag=EC3F7NKQJ4NO type=del 2024-09-13T00:00:58.598 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.598+0000 7fb8f2ce0640 1 > prepared operation key=63FJDWBO2T9S tag=ZG561PYXKUP4 type=del size=3809163 2024-09-13T00:00:58.605 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.602+0000 7fb8f2ce0640 1 < completed delete operation key=ZFHN2SNTZ5W7 tag=N43U4VXDQ4H6 type=del 2024-09-13T00:00:58.611 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.610+0000 7fb8f2ce0640 1 > prepared operation key=QMZREZ1QEDNJ tag=3W6J0UVTVYNB type=del size=210152 2024-09-13T00:00:58.617 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.618+0000 7fb8f2ce0640 1 < completed part key=_multipart_1HX06CCZJ1DD.2~JD7R30OYCW8W.0 size=1048576 2024-09-13T00:00:58.617 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.618+0000 7fb8f2ce0640 1 < canceled part key=_multipart_1HX06CCZJ1DD.2~JD7R30OYCW8W.1 size=1048576 2024-09-13T00:00:58.623 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.622+0000 7fb8f2ce0640 1 < completed part key=_multipart_1HX06CCZJ1DD.2~JD7R30OYCW8W.2 size=976813 2024-09-13T00:00:58.633 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.630+0000 7fb8f2ce0640 1 < canceled multipart upload key=1HX06CCZJ1DD upload=JD7R30OYCW8W tag=QFYRCTYINQ9G type=write size=4122541 2024-09-13T00:00:58.644 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.642+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H5EIP0E9CBWX.2~R9WGQR9OEN33.0 size=27318 2024-09-13T00:00:58.644 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.642+0000 7fb8f2ce0640 1 > prepared multipart upload key=H5EIP0E9CBWX upload=R9WGQR9OEN33 tag=0NW84327BYM3 type=write size=1075894 2024-09-13T00:00:58.649 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.650+0000 7fb8f2ce0640 1 < completed delete operation key=P8OYS5ZE1NX0 tag=34C8LE39IC7D type=del 2024-09-13T00:00:58.655 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.654+0000 7fb8f2ce0640 1 > prepared operation key=QMZREZ1QEDNJ tag=OHLH3XUI9HYJ type=del size=2056429 2024-09-13T00:00:58.669 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.670+0000 7fb8f2ce0640 1 < suggested remove operation key=F0QCVR9E08WE tag=YEMW1T83S7W8 type=del 2024-09-13T00:00:58.675 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.674+0000 7fb8f2ce0640 1 > prepared operation key=W8XZ26311RJX tag=C202PFQK6I7W type=del size=169953 2024-09-13T00:00:58.682 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.682+0000 7fb8f2ce0640 1 < completed part key=_multipart_EUE3LT0PDIXV.2~EFTN6FYJX6N6.0 size=1048576 2024-09-13T00:00:58.687 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.686+0000 7fb8f2ce0640 1 < completed part key=_multipart_EUE3LT0PDIXV.2~EFTN6FYJX6N6.1 size=90566 2024-09-13T00:00:58.698 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.698+0000 7fb8f2ce0640 1 < completed multipart upload key=EUE3LT0PDIXV upload=EFTN6FYJX6N6 tag=0TYOTXL05OSI type=write size=2187718 2024-09-13T00:00:58.705 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.702+0000 7fb8f2ce0640 1 > prepared operation key=70F7D20HI504 tag=2YG27ETKOS6D type=write size=276231 2024-09-13T00:00:58.712 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.710+0000 7fb8f2ce0640 1 < completed write operation key=8Z6U7OOA0GBH tag=XDQZ1Q2DITVC type=write size=786250 2024-09-13T00:00:58.718 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.718+0000 7fb8f2ce0640 1 > prepared operation key=4K76U69GZ9AE tag=EU2451Q8RFCY type=del size=629369 2024-09-13T00:00:58.725 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.726+0000 7fb8f2ce0640 1 < canceled operation key=H4M1TCN5W6XU tag=L1INBS1GFRSB type=del size=2808417 2024-09-13T00:00:58.737 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.738+0000 7fb8f2ce0640 1 > prepared part key=_multipart_5GM88ERM89KT.2~3RXUJETCAO7D.0 size=1048576 2024-09-13T00:00:58.743 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.742+0000 7fb8f2ce0640 1 > prepared part key=_multipart_5GM88ERM89KT.2~3RXUJETCAO7D.1 size=1048576 2024-09-13T00:00:58.747 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.746+0000 7fb8f2ce0640 1 > prepared part key=_multipart_5GM88ERM89KT.2~3RXUJETCAO7D.2 size=17430 2024-09-13T00:00:58.748 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.746+0000 7fb8f2ce0640 1 > prepared multipart upload key=5GM88ERM89KT upload=3RXUJETCAO7D tag=XZO0MOJ33SWN type=write size=3163158 2024-09-13T00:00:58.754 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.754+0000 7fb8f2ce0640 1 < completed delete operation key=H9AU4HD247YU tag=NCK0VUW2WMQ0 type=del 2024-09-13T00:00:58.760 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.758+0000 7fb8f2ce0640 1 > prepared operation key=H5EIP0E9CBWX tag=9TS29ZPN716U type=del size=1001563 2024-09-13T00:00:58.766 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.766+0000 7fb8f2ce0640 1 < completed write operation key=QMZREZ1QEDNJ tag=AA4DQVNBJF8P type=write size=705718 2024-09-13T00:00:58.771 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.770+0000 7fb8f2ce0640 1 > prepared operation key=85YEW8N43FTA tag=3BOXDRJSRHTP type=del size=3229686 2024-09-13T00:00:58.777 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.778+0000 7fb8f2ce0640 1 < completed delete operation key=H9AU4HD247YU tag=E5O24NHE51DI type=del 2024-09-13T00:00:58.788 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.786+0000 7fb8f2ce0640 1 > prepared part key=_multipart_N3Z5XIG3TZRQ.2~UMX9XOSB6NIC.0 size=1048576 2024-09-13T00:00:58.794 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.794+0000 7fb8f2ce0640 1 > prepared part key=_multipart_N3Z5XIG3TZRQ.2~UMX9XOSB6NIC.1 size=1048576 2024-09-13T00:00:58.798 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.798+0000 7fb8f2ce0640 1 > prepared part key=_multipart_N3Z5XIG3TZRQ.2~UMX9XOSB6NIC.2 size=404061 2024-09-13T00:00:58.799 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.798+0000 7fb8f2ce0640 1 > prepared multipart upload key=N3Z5XIG3TZRQ upload=UMX9XOSB6NIC tag=E2VWAPGLKDHZ type=write size=3549789 2024-09-13T00:00:58.804 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.802+0000 7fb8f2ce0640 1 < completed part key=_multipart_H5EIP0E9CBWX.2~KMH01W929CWH.0 size=1048576 2024-09-13T00:00:58.809 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.810+0000 7fb8f2ce0640 1 < completed part key=_multipart_H5EIP0E9CBWX.2~KMH01W929CWH.1 size=1048576 2024-09-13T00:00:58.809 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.810+0000 7fb8f2ce0640 1 < canceled part key=_multipart_H5EIP0E9CBWX.2~KMH01W929CWH.2 size=50686 2024-09-13T00:00:58.821 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.818+0000 7fb8f2ce0640 1 < canceled multipart upload key=H5EIP0E9CBWX upload=KMH01W929CWH tag=ECSGRFNF2IYF type=write size=3196414 2024-09-13T00:00:58.826 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.826+0000 7fb8f2ce0640 1 > prepared operation key=2AHYRZUDFHP4 tag=7ZORKU411AJH type=del size=2576831 2024-09-13T00:00:58.835 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.834+0000 7fb8f2ce0640 1 < completed delete operation key=63FJDWBO2T9S tag=AYE493B07LYO type=del 2024-09-13T00:00:58.846 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.846+0000 7fb8f2ce0640 1 > prepared operation key=PVYZQLGYT3XU tag=DAKLJ8615X6C type=del size=2332930 2024-09-13T00:00:58.854 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.854+0000 7fb8f2ce0640 1 < canceled operation key=EUE3LT0PDIXV tag=4JZIH6I7UCS3 type=del size=2435509 2024-09-13T00:00:58.858 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.858+0000 7fb8f2ce0640 1 > prepared operation key=EJN6HR8I7XQF tag=Q5PWO6M8YD0X type=del size=1927952 2024-09-13T00:00:58.865 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.866+0000 7fb8f2ce0640 1 < completed part key=_multipart_N3Z5XIG3TZRQ.2~8QN6QESE1MUR.0 size=1048576 2024-09-13T00:00:58.870 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.870+0000 7fb8f2ce0640 1 < completed part key=_multipart_N3Z5XIG3TZRQ.2~8QN6QESE1MUR.1 size=929988 2024-09-13T00:00:58.881 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.882+0000 7fb8f2ce0640 1 < completed multipart upload key=N3Z5XIG3TZRQ upload=8QN6QESE1MUR tag=JYJ8LGY4FSNT type=write size=3027140 2024-09-13T00:00:58.887 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.886+0000 7fb8f2ce0640 1 > prepared operation key=NX26SI4F2ZMB tag=0US5VI74V0IL type=del size=4072744 2024-09-13T00:00:58.893 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.894+0000 7fb8f2ce0640 1 < canceled operation key=85YEW8N43FTA tag=J7WWVV49PPI3 type=del size=657494 2024-09-13T00:00:58.899 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.898+0000 7fb8f2ce0640 1 > prepared operation key=85YEW8N43FTA tag=FWDZ5JHN4J3P type=del size=2346417 2024-09-13T00:00:58.905 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.906+0000 7fb8f2ce0640 1 < completed delete operation key=63FJDWBO2T9S tag=ZG561PYXKUP4 type=del 2024-09-13T00:00:58.915 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.914+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~DK6A2G2QYMI7.0 size=1048576 2024-09-13T00:00:58.920 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.918+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~DK6A2G2QYMI7.1 size=1048576 2024-09-13T00:00:58.925 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.926+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~DK6A2G2QYMI7.2 size=1029775 2024-09-13T00:00:58.925 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.926+0000 7fb8f2ce0640 1 > prepared multipart upload key=70F7D20HI504 upload=DK6A2G2QYMI7 tag=S5YL3GH1QCZM type=write size=4175503 2024-09-13T00:00:58.931 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.930+0000 7fb8f2ce0640 1 < completed delete operation key=QMZREZ1QEDNJ tag=3W6J0UVTVYNB type=del 2024-09-13T00:00:58.939 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.938+0000 7fb8f2ce0640 1 > prepared operation key=7OX5OJZ8GXU5 tag=G1GE1J3VMKZF type=write size=249339 2024-09-13T00:00:58.945 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.946+0000 7fb8f2ce0640 1 < completed part key=_multipart_H5EIP0E9CBWX.2~R9WGQR9OEN33.0 size=27318 2024-09-13T00:00:58.956 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.954+0000 7fb8f2ce0640 1 < completed multipart upload key=H5EIP0E9CBWX upload=R9WGQR9OEN33 tag=0NW84327BYM3 type=write size=1075894 2024-09-13T00:00:58.968 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.966+0000 7fb8f2ce0640 1 > prepared part key=_multipart_0WC6P8YNL0H2.2~KXLOBFOW6W9U.0 size=1048576 2024-09-13T00:00:58.973 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.974+0000 7fb8f2ce0640 1 > prepared part key=_multipart_0WC6P8YNL0H2.2~KXLOBFOW6W9U.1 size=1048576 2024-09-13T00:00:58.978 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.978+0000 7fb8f2ce0640 1 > prepared part key=_multipart_0WC6P8YNL0H2.2~KXLOBFOW6W9U.2 size=43238 2024-09-13T00:00:58.978 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.978+0000 7fb8f2ce0640 1 > prepared multipart upload key=0WC6P8YNL0H2 upload=KXLOBFOW6W9U tag=PAK1B4CSP5C7 type=write size=3188966 2024-09-13T00:00:58.984 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.982+0000 7fb8f2ce0640 1 < completed delete operation key=QMZREZ1QEDNJ tag=OHLH3XUI9HYJ type=del 2024-09-13T00:00:58.990 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.990+0000 7fb8f2ce0640 1 > prepared operation key=4K76U69GZ9AE tag=YR37VPOAPWJW type=write size=19588 2024-09-13T00:00:58.995 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.994+0000 7fb8f2ce0640 1 < completed delete operation key=W8XZ26311RJX tag=C202PFQK6I7W type=del 2024-09-13T00:00:59.001 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:58.998+0000 7fb8f2ce0640 1 > prepared operation key=QMZREZ1QEDNJ tag=PVWYE8BTY40T type=del size=3312718 2024-09-13T00:00:59.006 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.006+0000 7fb8f2ce0640 1 < completed write operation key=70F7D20HI504 tag=2YG27ETKOS6D type=write size=276231 2024-09-13T00:00:59.017 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.018+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~0PBE307D4ZYH.0 size=1005653 2024-09-13T00:00:59.017 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.018+0000 7fb8f2ce0640 1 > prepared multipart upload key=NQJQC8CEAC15 upload=0PBE307D4ZYH tag=3EPQAN3C5RF6 type=write size=2054229 2024-09-13T00:00:59.023 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.022+0000 7fb8f2ce0640 1 < completed delete operation key=4K76U69GZ9AE tag=EU2451Q8RFCY type=del 2024-09-13T00:00:59.029 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.030+0000 7fb8f2ce0640 1 > prepared operation key=O4MZ4UWNXME9 tag=NXGLQACYPVVV type=del size=2903594 2024-09-13T00:00:59.034 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.034+0000 7fb8f2ce0640 1 < completed part key=_multipart_5GM88ERM89KT.2~3RXUJETCAO7D.0 size=1048576 2024-09-13T00:00:59.039 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.038+0000 7fb8f2ce0640 1 < completed part key=_multipart_5GM88ERM89KT.2~3RXUJETCAO7D.1 size=1048576 2024-09-13T00:00:59.044 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.042+0000 7fb8f2ce0640 1 < completed part key=_multipart_5GM88ERM89KT.2~3RXUJETCAO7D.2 size=17430 2024-09-13T00:00:59.054 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.054+0000 7fb8f2ce0640 1 < completed multipart upload key=5GM88ERM89KT upload=3RXUJETCAO7D tag=XZO0MOJ33SWN type=write size=3163158 2024-09-13T00:00:59.060 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.058+0000 7fb8f2ce0640 1 > prepared operation key=NX26SI4F2ZMB tag=5WTPK08KACFD type=del size=39978 2024-09-13T00:00:59.066 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.066+0000 7fb8f2ce0640 1 < completed delete operation key=H5EIP0E9CBWX tag=9TS29ZPN716U type=del 2024-09-13T00:00:59.078 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.078+0000 7fb8f2ce0640 1 > prepared part key=_multipart_0WC6P8YNL0H2.2~H0HWNRMPYQ1G.0 size=1048576 2024-09-13T00:00:59.084 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.082+0000 7fb8f2ce0640 1 > prepared part key=_multipart_0WC6P8YNL0H2.2~H0HWNRMPYQ1G.1 size=245603 2024-09-13T00:00:59.084 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.082+0000 7fb8f2ce0640 1 > prepared multipart upload key=0WC6P8YNL0H2 upload=H0HWNRMPYQ1G tag=1CJAD65H95NB type=write size=2342755 2024-09-13T00:00:59.089 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.090+0000 7fb8f2ce0640 1 < completed delete operation key=85YEW8N43FTA tag=3BOXDRJSRHTP type=del 2024-09-13T00:00:59.095 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.094+0000 7fb8f2ce0640 1 > prepared operation key=FX0NCGLK0KPJ tag=SR98AOMG3BIL type=write size=640317 2024-09-13T00:00:59.100 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.098+0000 7fb8f2ce0640 1 < completed part key=_multipart_N3Z5XIG3TZRQ.2~UMX9XOSB6NIC.0 size=1048576 2024-09-13T00:00:59.105 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.106+0000 7fb8f2ce0640 1 < completed part key=_multipart_N3Z5XIG3TZRQ.2~UMX9XOSB6NIC.1 size=1048576 2024-09-13T00:00:59.110 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.110+0000 7fb8f2ce0640 1 < completed part key=_multipart_N3Z5XIG3TZRQ.2~UMX9XOSB6NIC.2 size=404061 2024-09-13T00:00:59.121 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.122+0000 7fb8f2ce0640 1 < completed multipart upload key=N3Z5XIG3TZRQ upload=UMX9XOSB6NIC tag=E2VWAPGLKDHZ type=write size=3549789 2024-09-13T00:00:59.126 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.126+0000 7fb8f2ce0640 1 > prepared operation key=W8XZ26311RJX tag=UQ0FJCT0SGOK type=del size=3012708 2024-09-13T00:00:59.132 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.130+0000 7fb8f2ce0640 1 < completed delete operation key=2AHYRZUDFHP4 tag=7ZORKU411AJH type=del 2024-09-13T00:00:59.144 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.142+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~FGS30UISJP2C.0 size=936976 2024-09-13T00:00:59.144 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.142+0000 7fb8f2ce0640 1 > prepared multipart upload key=FX0NCGLK0KPJ upload=FGS30UISJP2C tag=X0K7THX0C4NX type=write size=1985552 2024-09-13T00:00:59.150 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.150+0000 7fb8f2ce0640 1 < completed delete operation key=PVYZQLGYT3XU tag=DAKLJ8615X6C type=del 2024-09-13T00:00:59.155 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.154+0000 7fb8f2ce0640 1 > prepared operation key=NQJQC8CEAC15 tag=RYFBTW4ZBDQE type=del size=2779031 2024-09-13T00:00:59.167 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.166+0000 7fb8f2ce0640 1 < suggested update operation key=EJN6HR8I7XQF tag=Q5PWO6M8YD0X type=del size=1927952 2024-09-13T00:00:59.172 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.170+0000 7fb8f2ce0640 1 > prepared operation key=1HX06CCZJ1DD tag=I19F71WE7NWU type=del size=3365056 2024-09-13T00:00:59.178 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.178+0000 7fb8f2ce0640 1 < completed delete operation key=NX26SI4F2ZMB tag=0US5VI74V0IL type=del 2024-09-13T00:00:59.185 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.186+0000 7fb8f2ce0640 1 > prepared operation key=N3Z5XIG3TZRQ tag=FJJ1T8ITPZB5 type=del size=3197957 2024-09-13T00:00:59.191 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.190+0000 7fb8f2ce0640 1 < completed delete operation key=85YEW8N43FTA tag=FWDZ5JHN4J3P type=del 2024-09-13T00:00:59.202 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.202+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZFHN2SNTZ5W7.2~NOZRUID4XXXN.0 size=1048576 2024-09-13T00:00:59.207 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.206+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZFHN2SNTZ5W7.2~NOZRUID4XXXN.1 size=1048576 2024-09-13T00:00:59.212 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.210+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZFHN2SNTZ5W7.2~NOZRUID4XXXN.2 size=855688 2024-09-13T00:00:59.212 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.210+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZFHN2SNTZ5W7 upload=NOZRUID4XXXN tag=Y8YNXECNFXGY type=write size=4001416 2024-09-13T00:00:59.218 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.218+0000 7fb8f2ce0640 1 < completed part key=_multipart_70F7D20HI504.2~DK6A2G2QYMI7.0 size=1048576 2024-09-13T00:00:59.222 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.222+0000 7fb8f2ce0640 1 < completed part key=_multipart_70F7D20HI504.2~DK6A2G2QYMI7.1 size=1048576 2024-09-13T00:00:59.228 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.226+0000 7fb8f2ce0640 1 < completed part key=_multipart_70F7D20HI504.2~DK6A2G2QYMI7.2 size=1029775 2024-09-13T00:00:59.238 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.238+0000 7fb8f2ce0640 1 < completed multipart upload key=70F7D20HI504 upload=DK6A2G2QYMI7 tag=S5YL3GH1QCZM type=write size=4175503 2024-09-13T00:00:59.249 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.250+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~YMG04A1QLMHS.0 size=1048576 2024-09-13T00:00:59.254 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.254+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~YMG04A1QLMHS.1 size=845661 2024-09-13T00:00:59.254 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.254+0000 7fb8f2ce0640 1 > prepared multipart upload key=W8XZ26311RJX upload=YMG04A1QLMHS tag=DDB44YORU7RY type=write size=2942813 2024-09-13T00:00:59.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:00:58 smithi136 bash[75164]: cluster 2024-09-13T00:00:57.138952+0000 mgr.y (mgr.24661) 715 : cluster [DBG] pgmap v1391: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 546 KiB/s rd, 191 KiB/s wr, 970 op/s 2024-09-13T00:00:59.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:00:58 smithi136 bash[76371]: cluster 2024-09-13T00:00:57.138952+0000 mgr.y (mgr.24661) 715 : cluster [DBG] pgmap v1391: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 546 KiB/s rd, 191 KiB/s wr, 970 op/s 2024-09-13T00:00:59.266 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.266+0000 7fb8f2ce0640 1 < suggested update operation key=7OX5OJZ8GXU5 tag=G1GE1J3VMKZF type=write size=249339 2024-09-13T00:00:59.276 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.274+0000 7fb8f2ce0640 1 > prepared part key=_multipart_O4MZ4UWNXME9.2~3GSRLEDR2LMB.0 size=363512 2024-09-13T00:00:59.277 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.274+0000 7fb8f2ce0640 1 > prepared multipart upload key=O4MZ4UWNXME9 upload=3GSRLEDR2LMB tag=GCWI8S9FSDBL type=write size=1412088 2024-09-13T00:00:59.283 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.282+0000 7fb8f2ce0640 1 < completed part key=_multipart_0WC6P8YNL0H2.2~KXLOBFOW6W9U.0 size=1048576 2024-09-13T00:00:59.288 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.286+0000 7fb8f2ce0640 1 < completed part key=_multipart_0WC6P8YNL0H2.2~KXLOBFOW6W9U.1 size=1048576 2024-09-13T00:00:59.293 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.294+0000 7fb8f2ce0640 1 < completed part key=_multipart_0WC6P8YNL0H2.2~KXLOBFOW6W9U.2 size=43238 2024-09-13T00:00:59.303 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.302+0000 7fb8f2ce0640 1 < completed multipart upload key=0WC6P8YNL0H2 upload=KXLOBFOW6W9U tag=PAK1B4CSP5C7 type=write size=3188966 2024-09-13T00:00:59.315 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.314+0000 7fb8f2ce0640 1 > prepared part key=_multipart_85YEW8N43FTA.2~G3YH6UOSQHR3.0 size=201638 2024-09-13T00:00:59.315 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.314+0000 7fb8f2ce0640 1 > prepared multipart upload key=85YEW8N43FTA upload=G3YH6UOSQHR3 tag=QD2JNTOP2DS5 type=write size=1250214 2024-09-13T00:00:59.321 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.322+0000 7fb8f2ce0640 1 < completed write operation key=4K76U69GZ9AE tag=YR37VPOAPWJW type=write size=19588 2024-09-13T00:00:59.328 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.326+0000 7fb8f2ce0640 1 > prepared operation key=J8NGDSF9XRTP tag=JB3TYN4G8DNG type=del size=2775457 2024-09-13T00:00:59.336 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.334+0000 7fb8f2ce0640 1 < completed delete operation key=QMZREZ1QEDNJ tag=PVWYE8BTY40T type=del 2024-09-13T00:00:59.342 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.342+0000 7fb8f2ce0640 1 > prepared operation key=2TWD9YLMSYJI tag=JQYPIPEUGTYT type=del size=150378 2024-09-13T00:00:59.348 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.346+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~0PBE307D4ZYH.0 size=1005653 2024-09-13T00:00:59.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:00:58 smithi181 bash[67587]: cluster 2024-09-13T00:00:57.138952+0000 mgr.y (mgr.24661) 715 : cluster [DBG] pgmap v1391: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 546 KiB/s rd, 191 KiB/s wr, 970 op/s 2024-09-13T00:00:59.358 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.358+0000 7fb8f2ce0640 1 < completed multipart upload key=NQJQC8CEAC15 upload=0PBE307D4ZYH tag=3EPQAN3C5RF6 type=write size=2054229 2024-09-13T00:00:59.363 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.362+0000 7fb8f2ce0640 1 > prepared operation key=H5EIP0E9CBWX tag=2G56QADODKPS type=del size=3465205 2024-09-13T00:00:59.369 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.370+0000 7fb8f2ce0640 1 < completed delete operation key=O4MZ4UWNXME9 tag=NXGLQACYPVVV type=del 2024-09-13T00:00:59.376 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.374+0000 7fb8f2ce0640 1 > prepared operation key=EJN6HR8I7XQF tag=WOHPOPVRCMWH type=del size=903547 2024-09-13T00:00:59.382 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.382+0000 7fb8f2ce0640 1 < completed delete operation key=NX26SI4F2ZMB tag=5WTPK08KACFD type=del 2024-09-13T00:00:59.393 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.394+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~JL4PL6P0478X.0 size=1048576 2024-09-13T00:00:59.398 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.398+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~JL4PL6P0478X.1 size=196192 2024-09-13T00:00:59.398 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.398+0000 7fb8f2ce0640 1 > prepared multipart upload key=1PX7DBO9M3YI upload=JL4PL6P0478X tag=NYZJ26GEFOOI type=write size=2293344 2024-09-13T00:00:59.404 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.402+0000 7fb8f2ce0640 1 < completed part key=_multipart_0WC6P8YNL0H2.2~H0HWNRMPYQ1G.0 size=1048576 2024-09-13T00:00:59.409 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.410+0000 7fb8f2ce0640 1 < completed part key=_multipart_0WC6P8YNL0H2.2~H0HWNRMPYQ1G.1 size=245603 2024-09-13T00:00:59.420 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.418+0000 7fb8f2ce0640 1 < completed multipart upload key=0WC6P8YNL0H2 upload=H0HWNRMPYQ1G tag=1CJAD65H95NB type=write size=2342755 2024-09-13T00:00:59.426 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.426+0000 7fb8f2ce0640 1 > prepared operation key=NX26SI4F2ZMB tag=ASDXPMKGBJKY type=write size=820505 2024-09-13T00:00:59.432 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.430+0000 7fb8f2ce0640 1 < completed write operation key=FX0NCGLK0KPJ tag=SR98AOMG3BIL type=write size=640317 2024-09-13T00:00:59.443 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.442+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~OXB8N8Q6GRGE.0 size=1048576 2024-09-13T00:00:59.451 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.450+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~OXB8N8Q6GRGE.1 size=955910 2024-09-13T00:00:59.451 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.450+0000 7fb8f2ce0640 1 > prepared multipart upload key=7OX5OJZ8GXU5 upload=OXB8N8Q6GRGE tag=JYEH5MWV96S1 type=write size=3053062 2024-09-13T00:00:59.457 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.454+0000 7fb8f2ce0640 1 < canceled operation key=W8XZ26311RJX tag=UQ0FJCT0SGOK type=del size=3012708 2024-09-13T00:00:59.467 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.466+0000 7fb8f2ce0640 1 > prepared part key=_multipart_44EDYFT9VF8Y.2~0H3NFM73G9L3.0 size=500207 2024-09-13T00:00:59.467 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.466+0000 7fb8f2ce0640 1 > prepared multipart upload key=44EDYFT9VF8Y upload=0H3NFM73G9L3 tag=C5WQR1S0RANZ type=write size=1548783 2024-09-13T00:00:59.468 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.466+0000 7fb8f2ce0640 1 < canceled part key=_multipart_FX0NCGLK0KPJ.2~FGS30UISJP2C.0 size=936976 2024-09-13T00:00:59.478 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.478+0000 7fb8f2ce0640 1 < canceled multipart upload key=FX0NCGLK0KPJ upload=FGS30UISJP2C tag=X0K7THX0C4NX type=write size=1985552 2024-09-13T00:00:59.489 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.490+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H9AU4HD247YU.2~HS21GSSKK9DV.0 size=777170 2024-09-13T00:00:59.489 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.490+0000 7fb8f2ce0640 1 > prepared multipart upload key=H9AU4HD247YU upload=HS21GSSKK9DV tag=NIQS3SY8IUG2 type=write size=1825746 2024-09-13T00:00:59.495 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.494+0000 7fb8f2ce0640 1 < completed delete operation key=NQJQC8CEAC15 tag=RYFBTW4ZBDQE type=del 2024-09-13T00:00:59.506 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.506+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~MF9PV64DEM77.0 size=108621 2024-09-13T00:00:59.506 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.506+0000 7fb8f2ce0640 1 > prepared multipart upload key=63FJDWBO2T9S upload=MF9PV64DEM77 tag=7F757P0MWQVX type=write size=1157197 2024-09-13T00:00:59.513 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.514+0000 7fb8f2ce0640 1 < completed delete operation key=1HX06CCZJ1DD tag=I19F71WE7NWU type=del 2024-09-13T00:00:59.518 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.518+0000 7fb8f2ce0640 1 > prepared operation key=H5EIP0E9CBWX tag=WUUFZM6BQV8L type=del size=177734 2024-09-13T00:00:59.524 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.522+0000 7fb8f2ce0640 1 < completed delete operation key=N3Z5XIG3TZRQ tag=FJJ1T8ITPZB5 type=del 2024-09-13T00:00:59.530 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.530+0000 7fb8f2ce0640 1 > prepared operation key=70F7D20HI504 tag=GZNTG9TS8QEI type=del size=2381696 2024-09-13T00:00:59.531 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.530+0000 7fb8f2ce0640 1 < canceled part key=_multipart_ZFHN2SNTZ5W7.2~NOZRUID4XXXN.0 size=1048576 2024-09-13T00:00:59.536 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.534+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZFHN2SNTZ5W7.2~NOZRUID4XXXN.1 size=1048576 2024-09-13T00:00:59.541 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.542+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZFHN2SNTZ5W7.2~NOZRUID4XXXN.2 size=855688 2024-09-13T00:00:59.552 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.550+0000 7fb8f2ce0640 1 < canceled multipart upload key=ZFHN2SNTZ5W7 upload=NOZRUID4XXXN tag=Y8YNXECNFXGY type=write size=4001416 2024-09-13T00:00:59.558 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.558+0000 7fb8f2ce0640 1 > prepared operation key=8MPUVD37EDN9 tag=ZQLGT1L2RBA6 type=del size=511941 2024-09-13T00:00:59.564 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.562+0000 7fb8f2ce0640 1 < completed part key=_multipart_W8XZ26311RJX.2~YMG04A1QLMHS.0 size=1048576 2024-09-13T00:00:59.569 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.570+0000 7fb8f2ce0640 1 < completed part key=_multipart_W8XZ26311RJX.2~YMG04A1QLMHS.1 size=845661 2024-09-13T00:00:59.580 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.578+0000 7fb8f2ce0640 1 < completed multipart upload key=W8XZ26311RJX upload=YMG04A1QLMHS tag=DDB44YORU7RY type=write size=2942813 2024-09-13T00:00:59.585 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.586+0000 7fb8f2ce0640 1 > prepared operation key=8Z6U7OOA0GBH tag=URINXJ1ULQHI type=del size=2066986 2024-09-13T00:00:59.591 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.590+0000 7fb8f2ce0640 1 < completed part key=_multipart_O4MZ4UWNXME9.2~3GSRLEDR2LMB.0 size=363512 2024-09-13T00:00:59.601 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.602+0000 7fb8f2ce0640 1 < completed multipart upload key=O4MZ4UWNXME9 upload=3GSRLEDR2LMB tag=GCWI8S9FSDBL type=write size=1412088 2024-09-13T00:00:59.612 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.610+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~9AEH1FM4CL0L.0 size=1048576 2024-09-13T00:00:59.617 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.618+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~9AEH1FM4CL0L.1 size=1048576 2024-09-13T00:00:59.622 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.622+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~9AEH1FM4CL0L.2 size=862133 2024-09-13T00:00:59.622 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.622+0000 7fb8f2ce0640 1 > prepared multipart upload key=8Z6U7OOA0GBH upload=9AEH1FM4CL0L tag=ZWVSQG7GGG5I type=write size=4007861 2024-09-13T00:00:59.628 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.626+0000 7fb8f2ce0640 1 < completed part key=_multipart_85YEW8N43FTA.2~G3YH6UOSQHR3.0 size=201638 2024-09-13T00:00:59.639 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.638+0000 7fb8f2ce0640 1 < completed multipart upload key=85YEW8N43FTA upload=G3YH6UOSQHR3 tag=QD2JNTOP2DS5 type=write size=1250214 2024-09-13T00:00:59.645 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.646+0000 7fb8f2ce0640 1 > prepared operation key=2TWD9YLMSYJI tag=RJGV3DO5Y8LS type=del size=559326 2024-09-13T00:00:59.652 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.650+0000 7fb8f2ce0640 1 < completed delete operation key=J8NGDSF9XRTP tag=JB3TYN4G8DNG type=del 2024-09-13T00:00:59.664 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.662+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2TWD9YLMSYJI.2~525R788LJQDX.0 size=337075 2024-09-13T00:00:59.664 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.662+0000 7fb8f2ce0640 1 > prepared multipart upload key=2TWD9YLMSYJI upload=525R788LJQDX tag=92Z66FKZZ1K2 type=write size=1385651 2024-09-13T00:00:59.670 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.670+0000 7fb8f2ce0640 1 < completed delete operation key=2TWD9YLMSYJI tag=JQYPIPEUGTYT type=del 2024-09-13T00:00:59.676 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.674+0000 7fb8f2ce0640 1 > prepared operation key=8Z6U7OOA0GBH tag=SRIS9GO3VVGM type=del size=411702 2024-09-13T00:00:59.683 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.682+0000 7fb8f2ce0640 1 < completed delete operation key=H5EIP0E9CBWX tag=2G56QADODKPS type=del 2024-09-13T00:00:59.689 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.690+0000 7fb8f2ce0640 1 > prepared operation key=O4MZ4UWNXME9 tag=RGFBWKZJ20DW type=del size=157027 2024-09-13T00:00:59.698 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.698+0000 7fb8f2ce0640 1 < completed delete operation key=EJN6HR8I7XQF tag=WOHPOPVRCMWH type=del 2024-09-13T00:00:59.710 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.710+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~VSK222Q1V79P.0 size=1048576 2024-09-13T00:00:59.715 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.714+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~VSK222Q1V79P.1 size=1048576 2024-09-13T00:00:59.720 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.718+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~VSK222Q1V79P.2 size=179681 2024-09-13T00:00:59.720 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.718+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZTK9AYBSLCOZ upload=VSK222Q1V79P tag=D56AIB3Q8222 type=write size=3325409 2024-09-13T00:00:59.727 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.726+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~JL4PL6P0478X.0 size=1048576 2024-09-13T00:00:59.732 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.730+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~JL4PL6P0478X.1 size=196192 2024-09-13T00:00:59.742 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.742+0000 7fb8f2ce0640 1 < completed multipart upload key=1PX7DBO9M3YI upload=JL4PL6P0478X tag=NYZJ26GEFOOI type=write size=2293344 2024-09-13T00:00:59.748 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.746+0000 7fb8f2ce0640 1 > prepared operation key=7OX5OJZ8GXU5 tag=V8Z9C815PJ1M type=write size=566595 2024-09-13T00:00:59.757 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.754+0000 7fb8f2ce0640 1 < completed write operation key=NX26SI4F2ZMB tag=ASDXPMKGBJKY type=write size=820505 2024-09-13T00:00:59.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:00:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:00:59] "GET /metrics HTTP/1.1" 200 36472 "" "Prometheus/2.43.0" 2024-09-13T00:00:59.772 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.770+0000 7fb8f2ce0640 1 > prepared part key=_multipart_F0QCVR9E08WE.2~RWOEH59RBJHG.0 size=447483 2024-09-13T00:00:59.772 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.770+0000 7fb8f2ce0640 1 > prepared multipart upload key=F0QCVR9E08WE upload=RWOEH59RBJHG tag=0QQUT2K2VFVH type=write size=1496059 2024-09-13T00:00:59.778 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.778+0000 7fb8f2ce0640 1 < completed part key=_multipart_7OX5OJZ8GXU5.2~OXB8N8Q6GRGE.0 size=1048576 2024-09-13T00:00:59.783 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.782+0000 7fb8f2ce0640 1 < completed part key=_multipart_7OX5OJZ8GXU5.2~OXB8N8Q6GRGE.1 size=955910 2024-09-13T00:00:59.793 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.794+0000 7fb8f2ce0640 1 < completed multipart upload key=7OX5OJZ8GXU5 upload=OXB8N8Q6GRGE tag=JYEH5MWV96S1 type=write size=3053062 2024-09-13T00:00:59.800 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.798+0000 7fb8f2ce0640 1 > prepared operation key=F0QCVR9E08WE tag=JB735IATI1Z9 type=del size=1991428 2024-09-13T00:00:59.806 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.806+0000 7fb8f2ce0640 1 < completed part key=_multipart_44EDYFT9VF8Y.2~0H3NFM73G9L3.0 size=500207 2024-09-13T00:00:59.817 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.818+0000 7fb8f2ce0640 1 < completed multipart upload key=44EDYFT9VF8Y upload=0H3NFM73G9L3 tag=C5WQR1S0RANZ type=write size=1548783 2024-09-13T00:00:59.825 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.822+0000 7fb8f2ce0640 1 > prepared operation key=1HX06CCZJ1DD tag=N41H2BCL46KG type=del size=3401071 2024-09-13T00:00:59.831 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.830+0000 7fb8f2ce0640 1 < completed part key=_multipart_H9AU4HD247YU.2~HS21GSSKK9DV.0 size=777170 2024-09-13T00:00:59.841 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.842+0000 7fb8f2ce0640 1 < completed multipart upload key=H9AU4HD247YU upload=HS21GSSKK9DV tag=NIQS3SY8IUG2 type=write size=1825746 2024-09-13T00:00:59.852 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.849+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~FN238XZMRKKV.0 size=1048576 2024-09-13T00:00:59.856 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.853+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~FN238XZMRKKV.1 size=613005 2024-09-13T00:00:59.857 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.853+0000 7fb8f2ce0640 1 > prepared multipart upload key=70F7D20HI504 upload=FN238XZMRKKV tag=PLC2LCIHVF53 type=write size=2710157 2024-09-13T00:00:59.865 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.861+0000 7fb8f2ce0640 1 < completed part key=_multipart_63FJDWBO2T9S.2~MF9PV64DEM77.0 size=108621 2024-09-13T00:00:59.875 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.873+0000 7fb8f2ce0640 1 < completed multipart upload key=63FJDWBO2T9S upload=MF9PV64DEM77 tag=7F757P0MWQVX type=write size=1157197 2024-09-13T00:00:59.886 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.885+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H5EIP0E9CBWX.2~6LPHN2B3BS2C.0 size=1048576 2024-09-13T00:00:59.892 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.889+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H5EIP0E9CBWX.2~6LPHN2B3BS2C.1 size=1048576 2024-09-13T00:00:59.897 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.893+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H5EIP0E9CBWX.2~6LPHN2B3BS2C.2 size=754579 2024-09-13T00:00:59.897 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.893+0000 7fb8f2ce0640 1 > prepared multipart upload key=H5EIP0E9CBWX upload=6LPHN2B3BS2C tag=B3SGQK95PDJD type=write size=3900307 2024-09-13T00:00:59.903 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.901+0000 7fb8f2ce0640 1 < completed delete operation key=H5EIP0E9CBWX tag=WUUFZM6BQV8L type=del 2024-09-13T00:00:59.919 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.917+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~EK28J42FIBDF.0 size=1048576 2024-09-13T00:00:59.925 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.921+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~EK28J42FIBDF.1 size=633168 2024-09-13T00:00:59.925 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.921+0000 7fb8f2ce0640 1 > prepared multipart upload key=FX0NCGLK0KPJ upload=EK28J42FIBDF tag=Y1D60WDSE7O1 type=write size=2730320 2024-09-13T00:00:59.937 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.937+0000 7fb8f2ce0640 1 < suggested remove operation key=70F7D20HI504 tag=GZNTG9TS8QEI type=del 2024-09-13T00:00:59.945 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.945+0000 7fb8f2ce0640 1 > prepared operation key=QMZREZ1QEDNJ tag=3RE31KRSK6Q6 type=del size=627568 2024-09-13T00:00:59.951 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.949+0000 7fb8f2ce0640 1 < completed delete operation key=8MPUVD37EDN9 tag=ZQLGT1L2RBA6 type=del 2024-09-13T00:00:59.962 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.961+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~WZ70F0JFLZ70.0 size=370171 2024-09-13T00:00:59.962 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.961+0000 7fb8f2ce0640 1 > prepared multipart upload key=1PX7DBO9M3YI upload=WZ70F0JFLZ70 tag=0RQ6W126IHVC type=write size=1418747 2024-09-13T00:00:59.968 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.965+0000 7fb8f2ce0640 1 < completed delete operation key=8Z6U7OOA0GBH tag=URINXJ1ULQHI type=del 2024-09-13T00:00:59.974 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.973+0000 7fb8f2ce0640 1 > prepared operation key=2AHYRZUDFHP4 tag=WWJ61VRMH16M type=write size=65034 2024-09-13T00:00:59.974 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.973+0000 7fb8f2ce0640 1 < canceled part key=_multipart_8Z6U7OOA0GBH.2~9AEH1FM4CL0L.0 size=1048576 2024-09-13T00:00:59.980 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.977+0000 7fb8f2ce0640 1 < completed part key=_multipart_8Z6U7OOA0GBH.2~9AEH1FM4CL0L.1 size=1048576 2024-09-13T00:00:59.984 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.981+0000 7fb8f2ce0640 1 < completed part key=_multipart_8Z6U7OOA0GBH.2~9AEH1FM4CL0L.2 size=862133 2024-09-13T00:00:59.994 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.993+0000 7fb8f2ce0640 1 < canceled multipart upload key=8Z6U7OOA0GBH upload=9AEH1FM4CL0L tag=ZWVSQG7GGG5I type=write size=4007861 2024-09-13T00:01:00.000 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:00:59.997+0000 7fb8f2ce0640 1 > prepared operation key=70F7D20HI504 tag=9CIRP6IK31MC type=del size=3918776 2024-09-13T00:01:00.014 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.013+0000 7fb8f2ce0640 1 < suggested remove operation key=2TWD9YLMSYJI tag=RJGV3DO5Y8LS type=del 2024-09-13T00:01:00.020 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.017+0000 7fb8f2ce0640 1 > prepared operation key=2AHYRZUDFHP4 tag=HVKRV3WCU3MX type=del size=3376559 2024-09-13T00:01:00.027 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.025+0000 7fb8f2ce0640 1 < completed part key=_multipart_2TWD9YLMSYJI.2~525R788LJQDX.0 size=337075 2024-09-13T00:01:00.037 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.037+0000 7fb8f2ce0640 1 < completed multipart upload key=2TWD9YLMSYJI upload=525R788LJQDX tag=92Z66FKZZ1K2 type=write size=1385651 2024-09-13T00:01:00.050 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.049+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~P81IXF01PHV6.0 size=1048576 2024-09-13T00:01:00.055 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.053+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~P81IXF01PHV6.1 size=1048576 2024-09-13T00:01:00.060 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.057+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~P81IXF01PHV6.2 size=817907 2024-09-13T00:01:00.060 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.057+0000 7fb8f2ce0640 1 > prepared multipart upload key=W8XZ26311RJX upload=P81IXF01PHV6 tag=PJJH1CGUS3AA type=write size=3963635 2024-09-13T00:01:00.066 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.065+0000 7fb8f2ce0640 1 < completed delete operation key=8Z6U7OOA0GBH tag=SRIS9GO3VVGM type=del 2024-09-13T00:01:00.076 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.073+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZFHN2SNTZ5W7.2~2OPXNTAJ4XQ8.0 size=479361 2024-09-13T00:01:00.077 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.073+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZFHN2SNTZ5W7 upload=2OPXNTAJ4XQ8 tag=S23EQ4K6Z9X7 type=write size=1527937 2024-09-13T00:01:00.082 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.081+0000 7fb8f2ce0640 1 < completed delete operation key=O4MZ4UWNXME9 tag=RGFBWKZJ20DW type=del 2024-09-13T00:01:00.093 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.093+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~XTEOTTX1S33B.0 size=412095 2024-09-13T00:01:00.093 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.093+0000 7fb8f2ce0640 1 > prepared multipart upload key=FX0NCGLK0KPJ upload=XTEOTTX1S33B tag=U1I9V5G26FL3 type=write size=1460671 2024-09-13T00:01:00.099 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.097+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~VSK222Q1V79P.0 size=1048576 2024-09-13T00:01:00.104 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.101+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~VSK222Q1V79P.1 size=1048576 2024-09-13T00:01:00.109 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.105+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~VSK222Q1V79P.2 size=179681 2024-09-13T00:01:00.118 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.117+0000 7fb8f2ce0640 1 < completed multipart upload key=ZTK9AYBSLCOZ upload=VSK222Q1V79P tag=D56AIB3Q8222 type=write size=3325409 2024-09-13T00:01:00.124 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.121+0000 7fb8f2ce0640 1 > prepared operation key=FX0NCGLK0KPJ tag=MY1SNH2HGWQV type=write size=992081 2024-09-13T00:01:00.129 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.129+0000 7fb8f2ce0640 1 < completed write operation key=7OX5OJZ8GXU5 tag=V8Z9C815PJ1M type=write size=566595 2024-09-13T00:01:00.141 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.141+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~6DI5C0AR6UGY.0 size=1048576 2024-09-13T00:01:00.146 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.145+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~6DI5C0AR6UGY.1 size=1048576 2024-09-13T00:01:00.150 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.149+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~6DI5C0AR6UGY.2 size=925296 2024-09-13T00:01:00.150 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.149+0000 7fb8f2ce0640 1 > prepared multipart upload key=7OX5OJZ8GXU5 upload=6DI5C0AR6UGY tag=WBURAR6EWUSU type=write size=4071024 2024-09-13T00:01:00.156 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.153+0000 7fb8f2ce0640 1 < completed part key=_multipart_F0QCVR9E08WE.2~RWOEH59RBJHG.0 size=447483 2024-09-13T00:01:00.166 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.165+0000 7fb8f2ce0640 1 < completed multipart upload key=F0QCVR9E08WE upload=RWOEH59RBJHG tag=0QQUT2K2VFVH type=write size=1496059 2024-09-13T00:01:00.171 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.169+0000 7fb8f2ce0640 1 > prepared operation key=8Z6U7OOA0GBH tag=9UNUM7DVHRSX type=del size=2725180 2024-09-13T00:01:00.176 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.173+0000 7fb8f2ce0640 1 < completed delete operation key=F0QCVR9E08WE tag=JB735IATI1Z9 type=del 2024-09-13T00:01:00.182 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.181+0000 7fb8f2ce0640 1 > prepared operation key=N3Z5XIG3TZRQ tag=80X4WFTS2CR0 type=del size=2206656 2024-09-13T00:01:00.187 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.185+0000 7fb8f2ce0640 1 < canceled operation key=1HX06CCZJ1DD tag=N41H2BCL46KG type=del size=3401071 2024-09-13T00:01:00.193 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.193+0000 7fb8f2ce0640 1 > prepared operation key=NX26SI4F2ZMB tag=VDMQOZ2ZCTO2 type=del size=1674597 2024-09-13T00:01:00.198 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.197+0000 7fb8f2ce0640 1 < completed part key=_multipart_70F7D20HI504.2~FN238XZMRKKV.0 size=1048576 2024-09-13T00:01:00.203 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.201+0000 7fb8f2ce0640 1 < completed part key=_multipart_70F7D20HI504.2~FN238XZMRKKV.1 size=613005 2024-09-13T00:01:00.212 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.209+0000 7fb8f2ce0640 1 < completed multipart upload key=70F7D20HI504 upload=FN238XZMRKKV tag=PLC2LCIHVF53 type=write size=2710157 2024-09-13T00:01:00.218 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.217+0000 7fb8f2ce0640 1 > prepared operation key=QMZREZ1QEDNJ tag=YAGVWOIKVYZB type=del size=2192922 2024-09-13T00:01:00.219 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.217+0000 7fb8f2ce0640 1 < canceled part key=_multipart_H5EIP0E9CBWX.2~6LPHN2B3BS2C.0 size=1048576 2024-09-13T00:01:00.224 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.221+0000 7fb8f2ce0640 1 < completed part key=_multipart_H5EIP0E9CBWX.2~6LPHN2B3BS2C.1 size=1048576 2024-09-13T00:01:00.228 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.225+0000 7fb8f2ce0640 1 < completed part key=_multipart_H5EIP0E9CBWX.2~6LPHN2B3BS2C.2 size=754579 2024-09-13T00:01:00.238 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.237+0000 7fb8f2ce0640 1 < canceled multipart upload key=H5EIP0E9CBWX upload=6LPHN2B3BS2C tag=B3SGQK95PDJD type=write size=3900307 2024-09-13T00:01:00.248 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.245+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~PWA2BKL0JL0S.0 size=1048576 2024-09-13T00:01:00.253 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.253+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~PWA2BKL0JL0S.1 size=567105 2024-09-13T00:01:00.253 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.253+0000 7fb8f2ce0640 1 > prepared multipart upload key=1PX7DBO9M3YI upload=PWA2BKL0JL0S tag=HE0MED15LBGI type=write size=2664257 2024-09-13T00:01:00.253 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.253+0000 7fb8f2ce0640 1 < canceled part key=_multipart_FX0NCGLK0KPJ.2~EK28J42FIBDF.0 size=1048576 2024-09-13T00:01:00.253 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.253+0000 7fb8f2ce0640 1 < canceled part key=_multipart_FX0NCGLK0KPJ.2~EK28J42FIBDF.1 size=633168 2024-09-13T00:01:00.263 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.261+0000 7fb8f2ce0640 1 < canceled multipart upload key=FX0NCGLK0KPJ upload=EK28J42FIBDF tag=Y1D60WDSE7O1 type=write size=2730320 2024-09-13T00:01:00.276 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.273+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H5EIP0E9CBWX.2~FNNMKYEZ84BT.0 size=985097 2024-09-13T00:01:00.276 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.273+0000 7fb8f2ce0640 1 > prepared multipart upload key=H5EIP0E9CBWX upload=FNNMKYEZ84BT tag=AQKV2I6R5UZF type=write size=2033673 2024-09-13T00:01:00.288 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.285+0000 7fb8f2ce0640 1 < suggested remove operation key=QMZREZ1QEDNJ tag=3RE31KRSK6Q6 type=del 2024-09-13T00:01:00.292 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.289+0000 7fb8f2ce0640 1 > prepared operation key=PVYZQLGYT3XU tag=06BBREQZTS7J type=del size=273179 2024-09-13T00:01:00.298 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.297+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~WZ70F0JFLZ70.0 size=370171 2024-09-13T00:01:00.313 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.309+0000 7fb8f2ce0640 1 < completed multipart upload key=1PX7DBO9M3YI upload=WZ70F0JFLZ70 tag=0RQ6W126IHVC type=write size=1418747 2024-09-13T00:01:00.320 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.317+0000 7fb8f2ce0640 1 > prepared operation key=7OX5OJZ8GXU5 tag=J04C06ZPGEH6 type=write size=717167 2024-09-13T00:01:00.330 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.329+0000 7fb8f2ce0640 1 < suggested update operation key=2AHYRZUDFHP4 tag=WWJ61VRMH16M type=write size=65034 2024-09-13T00:01:00.337 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.333+0000 7fb8f2ce0640 1 > prepared operation key=8MPUVD37EDN9 tag=PEGTAM8G9PMY type=del size=2983541 2024-09-13T00:01:00.343 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.341+0000 7fb8f2ce0640 1 < completed delete operation key=70F7D20HI504 tag=9CIRP6IK31MC type=del 2024-09-13T00:01:00.349 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.345+0000 7fb8f2ce0640 1 > prepared operation key=8Z6U7OOA0GBH tag=619X1SCSI3TV type=del size=1809409 2024-09-13T00:01:00.357 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.353+0000 7fb8f2ce0640 -1 < failed to complete operation key=2AHYRZUDFHP4 tag=HVKRV3WCU3MX type=del size=3376559: (22) Invalid argument 2024-09-13T00:01:00.362 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.361+0000 7fb8f2ce0640 1 > prepared operation key=70F7D20HI504 tag=PTD78S2WQ4D0 type=del size=2104082 2024-09-13T00:01:00.369 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.369+0000 7fb8f2ce0640 1 < completed part key=_multipart_W8XZ26311RJX.2~P81IXF01PHV6.0 size=1048576 2024-09-13T00:01:00.374 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.373+0000 7fb8f2ce0640 1 < completed part key=_multipart_W8XZ26311RJX.2~P81IXF01PHV6.1 size=1048576 2024-09-13T00:01:00.374 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.373+0000 7fb8f2ce0640 1 < canceled part key=_multipart_W8XZ26311RJX.2~P81IXF01PHV6.2 size=817907 2024-09-13T00:01:00.383 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.381+0000 7fb8f2ce0640 1 < canceled multipart upload key=W8XZ26311RJX upload=P81IXF01PHV6 tag=PJJH1CGUS3AA type=write size=3963635 2024-09-13T00:01:00.391 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.389+0000 7fb8f2ce0640 1 > prepared operation key=63FJDWBO2T9S tag=8TOMZVK7JQ4Q type=write size=924066 2024-09-13T00:01:00.398 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.397+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZFHN2SNTZ5W7.2~2OPXNTAJ4XQ8.0 size=479361 2024-09-13T00:01:00.410 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.409+0000 7fb8f2ce0640 1 < completed multipart upload key=ZFHN2SNTZ5W7 upload=2OPXNTAJ4XQ8 tag=S23EQ4K6Z9X7 type=write size=1527937 2024-09-13T00:01:00.423 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.421+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~IXP8QBD95LBD.0 size=1048576 2024-09-13T00:01:00.429 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.429+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~IXP8QBD95LBD.1 size=1048576 2024-09-13T00:01:00.437 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.437+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~IXP8QBD95LBD.2 size=486367 2024-09-13T00:01:00.438 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.437+0000 7fb8f2ce0640 1 > prepared multipart upload key=1HX06CCZJ1DD upload=IXP8QBD95LBD tag=QAZN1FV9UJXT type=write size=3632095 2024-09-13T00:01:00.444 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.441+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~XTEOTTX1S33B.0 size=412095 2024-09-13T00:01:00.455 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.453+0000 7fb8f2ce0640 1 < completed multipart upload key=FX0NCGLK0KPJ upload=XTEOTTX1S33B tag=U1I9V5G26FL3 type=write size=1460671 2024-09-13T00:01:00.466 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.465+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~OF95WV9WOJUZ.0 size=254097 2024-09-13T00:01:00.466 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.465+0000 7fb8f2ce0640 1 > prepared multipart upload key=63FJDWBO2T9S upload=OF95WV9WOJUZ tag=97N9VQACRVDR type=write size=1302673 2024-09-13T00:01:00.472 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.469+0000 7fb8f2ce0640 1 < completed write operation key=FX0NCGLK0KPJ tag=MY1SNH2HGWQV type=write size=992081 2024-09-13T00:01:00.478 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.477+0000 7fb8f2ce0640 1 > prepared operation key=63FJDWBO2T9S tag=D584E5NOLWIH type=del size=224779 2024-09-13T00:01:00.484 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.481+0000 7fb8f2ce0640 1 < completed part key=_multipart_7OX5OJZ8GXU5.2~6DI5C0AR6UGY.0 size=1048576 2024-09-13T00:01:00.490 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.489+0000 7fb8f2ce0640 1 < completed part key=_multipart_7OX5OJZ8GXU5.2~6DI5C0AR6UGY.1 size=1048576 2024-09-13T00:01:00.496 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.493+0000 7fb8f2ce0640 1 < completed part key=_multipart_7OX5OJZ8GXU5.2~6DI5C0AR6UGY.2 size=925296 2024-09-13T00:01:00.506 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.505+0000 7fb8f2ce0640 1 < completed multipart upload key=7OX5OJZ8GXU5 upload=6DI5C0AR6UGY tag=WBURAR6EWUSU type=write size=4071024 2024-09-13T00:01:00.512 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.509+0000 7fb8f2ce0640 1 > prepared operation key=1HX06CCZJ1DD tag=ZEA14HORJYFZ type=del size=4067245 2024-09-13T00:01:00.517 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.517+0000 7fb8f2ce0640 1 < canceled operation key=8Z6U7OOA0GBH tag=9UNUM7DVHRSX type=del size=2725180 2024-09-13T00:01:00.528 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.525+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2TWD9YLMSYJI.2~DLUMTJVSFU8A.0 size=1048576 2024-09-13T00:01:00.533 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.533+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2TWD9YLMSYJI.2~DLUMTJVSFU8A.1 size=790664 2024-09-13T00:01:00.533 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.533+0000 7fb8f2ce0640 1 > prepared multipart upload key=2TWD9YLMSYJI upload=DLUMTJVSFU8A tag=N104PH8L51AK type=write size=2887816 2024-09-13T00:01:00.546 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.545+0000 7fb8f2ce0640 1 < suggested update operation key=N3Z5XIG3TZRQ tag=80X4WFTS2CR0 type=del size=2206656 2024-09-13T00:01:00.552 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.549+0000 7fb8f2ce0640 1 > prepared operation key=2AHYRZUDFHP4 tag=LCOEHRJCZWZA type=del size=3572832 2024-09-13T00:01:00.557 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.557+0000 7fb8f2ce0640 1 < completed delete operation key=NX26SI4F2ZMB tag=VDMQOZ2ZCTO2 type=del 2024-09-13T00:01:00.568 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.565+0000 7fb8f2ce0640 1 > prepared part key=_multipart_85YEW8N43FTA.2~XUN66PYUG86U.0 size=1048576 2024-09-13T00:01:00.573 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.573+0000 7fb8f2ce0640 1 > prepared part key=_multipart_85YEW8N43FTA.2~XUN66PYUG86U.1 size=1048576 2024-09-13T00:01:00.580 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.577+0000 7fb8f2ce0640 1 > prepared part key=_multipart_85YEW8N43FTA.2~XUN66PYUG86U.2 size=936771 2024-09-13T00:01:00.580 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.577+0000 7fb8f2ce0640 1 > prepared multipart upload key=85YEW8N43FTA upload=XUN66PYUG86U tag=BMF06VDOFKHY type=write size=4082499 2024-09-13T00:01:00.588 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.585+0000 7fb8f2ce0640 -1 < failed to complete operation key=QMZREZ1QEDNJ tag=YAGVWOIKVYZB type=del size=2192922: (22) Invalid argument 2024-09-13T00:01:00.595 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.593+0000 7fb8f2ce0640 1 > prepared operation key=P8OYS5ZE1NX0 tag=KHA1191XWWFQ type=del size=3216713 2024-09-13T00:01:00.601 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.601+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~PWA2BKL0JL0S.0 size=1048576 2024-09-13T00:01:00.606 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.605+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~PWA2BKL0JL0S.1 size=567105 2024-09-13T00:01:00.619 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.617+0000 7fb8f2ce0640 1 < completed multipart upload key=1PX7DBO9M3YI upload=PWA2BKL0JL0S tag=HE0MED15LBGI type=write size=2664257 2024-09-13T00:01:00.625 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.625+0000 7fb8f2ce0640 1 > prepared operation key=H5EIP0E9CBWX tag=U65MM675YM20 type=write size=937918 2024-09-13T00:01:00.631 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.629+0000 7fb8f2ce0640 1 < completed part key=_multipart_H5EIP0E9CBWX.2~FNNMKYEZ84BT.0 size=985097 2024-09-13T00:01:00.641 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.641+0000 7fb8f2ce0640 1 < completed multipart upload key=H5EIP0E9CBWX upload=FNNMKYEZ84BT tag=AQKV2I6R5UZF type=write size=2033673 2024-09-13T00:01:00.652 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.649+0000 7fb8f2ce0640 1 > prepared part key=_multipart_N3Z5XIG3TZRQ.2~TOIRHUZOHR00.0 size=1048576 2024-09-13T00:01:00.658 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.657+0000 7fb8f2ce0640 1 > prepared part key=_multipart_N3Z5XIG3TZRQ.2~TOIRHUZOHR00.1 size=1048576 2024-09-13T00:01:00.663 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.661+0000 7fb8f2ce0640 1 > prepared part key=_multipart_N3Z5XIG3TZRQ.2~TOIRHUZOHR00.2 size=914509 2024-09-13T00:01:00.663 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.661+0000 7fb8f2ce0640 1 > prepared multipart upload key=N3Z5XIG3TZRQ upload=TOIRHUZOHR00 tag=OXJEWZ8JYN1N type=write size=4060237 2024-09-13T00:01:00.670 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.669+0000 7fb8f2ce0640 1 < canceled operation key=PVYZQLGYT3XU tag=06BBREQZTS7J type=del size=273179 2024-09-13T00:01:00.682 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.681+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H9AU4HD247YU.2~HYXK5J9EPLE6.0 size=1048576 2024-09-13T00:01:00.687 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.685+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H9AU4HD247YU.2~HYXK5J9EPLE6.1 size=1048576 2024-09-13T00:01:00.691 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.689+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H9AU4HD247YU.2~HYXK5J9EPLE6.2 size=651729 2024-09-13T00:01:00.692 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.689+0000 7fb8f2ce0640 1 > prepared multipart upload key=H9AU4HD247YU upload=HYXK5J9EPLE6 tag=VCAVJ8MJ55F5 type=write size=3797457 2024-09-13T00:01:00.697 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.697+0000 7fb8f2ce0640 1 < canceled operation key=7OX5OJZ8GXU5 tag=J04C06ZPGEH6 type=write size=717167 2024-09-13T00:01:00.703 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.701+0000 7fb8f2ce0640 1 > prepared operation key=8Z6U7OOA0GBH tag=JMMSMAKPALNF type=del size=1890723 2024-09-13T00:01:00.709 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.709+0000 7fb8f2ce0640 1 < completed delete operation key=8MPUVD37EDN9 tag=PEGTAM8G9PMY type=del 2024-09-13T00:01:00.720 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.717+0000 7fb8f2ce0640 1 > prepared part key=_multipart_85YEW8N43FTA.2~78YZQ1ARE1DW.0 size=363916 2024-09-13T00:01:00.720 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.717+0000 7fb8f2ce0640 1 > prepared multipart upload key=85YEW8N43FTA upload=78YZQ1ARE1DW tag=0M8KJJIZ2BZA type=write size=1412492 2024-09-13T00:01:00.726 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.725+0000 7fb8f2ce0640 1 < completed delete operation key=8Z6U7OOA0GBH tag=619X1SCSI3TV type=del 2024-09-13T00:01:00.732 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.729+0000 7fb8f2ce0640 1 > prepared operation key=70F7D20HI504 tag=5ZR436V5ISJ1 type=del size=1254121 2024-09-13T00:01:00.738 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.737+0000 7fb8f2ce0640 1 < completed delete operation key=70F7D20HI504 tag=PTD78S2WQ4D0 type=del 2024-09-13T00:01:00.744 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.741+0000 7fb8f2ce0640 1 > prepared operation key=H9AU4HD247YU tag=RPHCDS8R3188 type=del size=1404294 2024-09-13T00:01:00.756 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.753+0000 7fb8f2ce0640 1 < suggested update operation key=63FJDWBO2T9S tag=8TOMZVK7JQ4Q type=write size=924066 2024-09-13T00:01:00.762 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.761+0000 7fb8f2ce0640 1 > prepared operation key=H5EIP0E9CBWX tag=FG2AH3ZK6GYQ type=write size=157382 2024-09-13T00:01:00.769 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.765+0000 7fb8f2ce0640 1 < completed part key=_multipart_1HX06CCZJ1DD.2~IXP8QBD95LBD.0 size=1048576 2024-09-13T00:01:00.774 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.773+0000 7fb8f2ce0640 1 < completed part key=_multipart_1HX06CCZJ1DD.2~IXP8QBD95LBD.1 size=1048576 2024-09-13T00:01:00.779 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.777+0000 7fb8f2ce0640 1 < completed part key=_multipart_1HX06CCZJ1DD.2~IXP8QBD95LBD.2 size=486367 2024-09-13T00:01:00.790 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.789+0000 7fb8f2ce0640 1 < completed multipart upload key=1HX06CCZJ1DD upload=IXP8QBD95LBD tag=QAZN1FV9UJXT type=write size=3632095 2024-09-13T00:01:00.797 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.793+0000 7fb8f2ce0640 1 > prepared operation key=ZTK9AYBSLCOZ tag=V2QIPG7XTSG1 type=write size=131048 2024-09-13T00:01:00.797 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.797+0000 7fb8f2ce0640 1 < canceled part key=_multipart_63FJDWBO2T9S.2~OF95WV9WOJUZ.0 size=254097 2024-09-13T00:01:00.807 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.805+0000 7fb8f2ce0640 1 < canceled multipart upload key=63FJDWBO2T9S upload=OF95WV9WOJUZ tag=97N9VQACRVDR type=write size=1302673 2024-09-13T00:01:00.813 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.813+0000 7fb8f2ce0640 1 > prepared operation key=F0QCVR9E08WE tag=SB99Q8IIBFMF type=write size=703590 2024-09-13T00:01:00.821 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.821+0000 7fb8f2ce0640 -1 < failed to complete operation key=63FJDWBO2T9S tag=D584E5NOLWIH type=del size=224779: (22) Invalid argument 2024-09-13T00:01:00.827 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.825+0000 7fb8f2ce0640 1 > prepared operation key=P8OYS5ZE1NX0 tag=Y06DN4QVS8WJ type=del size=2926482 2024-09-13T00:01:00.833 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.833+0000 7fb8f2ce0640 1 < completed delete operation key=1HX06CCZJ1DD tag=ZEA14HORJYFZ type=del 2024-09-13T00:01:00.839 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.837+0000 7fb8f2ce0640 1 > prepared operation key=H5EIP0E9CBWX tag=TNN8A3B0AD1M type=del size=1677119 2024-09-13T00:01:00.845 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.845+0000 7fb8f2ce0640 1 < completed part key=_multipart_2TWD9YLMSYJI.2~DLUMTJVSFU8A.0 size=1048576 2024-09-13T00:01:00.850 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.849+0000 7fb8f2ce0640 1 < completed part key=_multipart_2TWD9YLMSYJI.2~DLUMTJVSFU8A.1 size=790664 2024-09-13T00:01:00.861 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.857+0000 7fb8f2ce0640 1 < completed multipart upload key=2TWD9YLMSYJI upload=DLUMTJVSFU8A tag=N104PH8L51AK type=write size=2887816 2024-09-13T00:01:00.867 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.865+0000 7fb8f2ce0640 1 > prepared operation key=85YEW8N43FTA tag=TE2U8AL16O1G type=del size=1787337 2024-09-13T00:01:00.873 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.869+0000 7fb8f2ce0640 1 < canceled operation key=2AHYRZUDFHP4 tag=LCOEHRJCZWZA type=del size=3572832 2024-09-13T00:01:00.884 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.881+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~KL2FYN133A7X.0 size=1048576 2024-09-13T00:01:00.889 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.889+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~KL2FYN133A7X.1 size=1048576 2024-09-13T00:01:00.894 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.893+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~KL2FYN133A7X.2 size=931674 2024-09-13T00:01:00.894 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.893+0000 7fb8f2ce0640 1 > prepared multipart upload key=1PX7DBO9M3YI upload=KL2FYN133A7X tag=AYQ8WPAO063D type=write size=4077402 2024-09-13T00:01:00.900 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.897+0000 7fb8f2ce0640 1 < completed part key=_multipart_85YEW8N43FTA.2~XUN66PYUG86U.0 size=1048576 2024-09-13T00:01:00.900 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.897+0000 7fb8f2ce0640 1 < canceled part key=_multipart_85YEW8N43FTA.2~XUN66PYUG86U.1 size=1048576 2024-09-13T00:01:00.905 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.901+0000 7fb8f2ce0640 1 < completed part key=_multipart_85YEW8N43FTA.2~XUN66PYUG86U.2 size=936771 2024-09-13T00:01:00.915 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.913+0000 7fb8f2ce0640 1 < canceled multipart upload key=85YEW8N43FTA upload=XUN66PYUG86U tag=BMF06VDOFKHY type=write size=4082499 2024-09-13T00:01:00.924 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.921+0000 7fb8f2ce0640 1 > prepared operation key=EJN6HR8I7XQF tag=GAI70MIO1YYB type=del size=4021723 2024-09-13T00:01:00.932 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.929+0000 7fb8f2ce0640 1 < completed delete operation key=P8OYS5ZE1NX0 tag=KHA1191XWWFQ type=del 2024-09-13T00:01:00.945 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.945+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~I8FNW010TE5H.0 size=1048576 2024-09-13T00:01:00.950 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.949+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~I8FNW010TE5H.1 size=700750 2024-09-13T00:01:00.950 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.949+0000 7fb8f2ce0640 1 > prepared multipart upload key=63FJDWBO2T9S upload=I8FNW010TE5H tag=NO1PEAGP61CP type=write size=2797902 2024-09-13T00:01:00.957 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.953+0000 7fb8f2ce0640 1 < completed write operation key=H5EIP0E9CBWX tag=U65MM675YM20 type=write size=937918 2024-09-13T00:01:00.968 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.965+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZFHN2SNTZ5W7.2~1336GZ7L6TD0.0 size=1048576 2024-09-13T00:01:00.973 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.973+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZFHN2SNTZ5W7.2~1336GZ7L6TD0.1 size=1048576 2024-09-13T00:01:00.978 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.977+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZFHN2SNTZ5W7.2~1336GZ7L6TD0.2 size=333891 2024-09-13T00:01:00.978 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.977+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZFHN2SNTZ5W7 upload=1336GZ7L6TD0 tag=K8UG4ZPKSUC4 type=write size=3479619 2024-09-13T00:01:00.985 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.981+0000 7fb8f2ce0640 1 < completed part key=_multipart_N3Z5XIG3TZRQ.2~TOIRHUZOHR00.0 size=1048576 2024-09-13T00:01:00.990 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.989+0000 7fb8f2ce0640 1 < completed part key=_multipart_N3Z5XIG3TZRQ.2~TOIRHUZOHR00.1 size=1048576 2024-09-13T00:01:00.995 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:00.993+0000 7fb8f2ce0640 1 < completed part key=_multipart_N3Z5XIG3TZRQ.2~TOIRHUZOHR00.2 size=914509 2024-09-13T00:01:01.006 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.005+0000 7fb8f2ce0640 1 < completed multipart upload key=N3Z5XIG3TZRQ upload=TOIRHUZOHR00 tag=OXJEWZ8JYN1N type=write size=4060237 2024-09-13T00:01:01.012 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.009+0000 7fb8f2ce0640 1 > prepared operation key=8Z6U7OOA0GBH tag=M3NO4YKH8NE3 type=del size=1413414 2024-09-13T00:01:01.020 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.017+0000 7fb8f2ce0640 1 < completed part key=_multipart_H9AU4HD247YU.2~HYXK5J9EPLE6.0 size=1048576 2024-09-13T00:01:01.026 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.025+0000 7fb8f2ce0640 1 < completed part key=_multipart_H9AU4HD247YU.2~HYXK5J9EPLE6.1 size=1048576 2024-09-13T00:01:01.032 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.029+0000 7fb8f2ce0640 1 < completed part key=_multipart_H9AU4HD247YU.2~HYXK5J9EPLE6.2 size=651729 2024-09-13T00:01:01.045 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.045+0000 7fb8f2ce0640 1 < completed multipart upload key=H9AU4HD247YU upload=HYXK5J9EPLE6 tag=VCAVJ8MJ55F5 type=write size=3797457 2024-09-13T00:01:01.051 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.049+0000 7fb8f2ce0640 1 > prepared operation key=2TWD9YLMSYJI tag=6LA3C0Z1YA67 type=del size=2697180 2024-09-13T00:01:01.057 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.057+0000 7fb8f2ce0640 1 < completed delete operation key=8Z6U7OOA0GBH tag=JMMSMAKPALNF type=del 2024-09-13T00:01:01.063 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.061+0000 7fb8f2ce0640 1 > prepared operation key=QMZREZ1QEDNJ tag=7J5JM8TIYBJ4 type=del size=887968 2024-09-13T00:01:01.069 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.065+0000 7fb8f2ce0640 1 < completed part key=_multipart_85YEW8N43FTA.2~78YZQ1ARE1DW.0 size=363916 2024-09-13T00:01:01.079 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.077+0000 7fb8f2ce0640 1 < completed multipart upload key=85YEW8N43FTA upload=78YZQ1ARE1DW tag=0M8KJJIZ2BZA type=write size=1412492 2024-09-13T00:01:01.084 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.081+0000 7fb8f2ce0640 1 > prepared operation key=ZFHN2SNTZ5W7 tag=I4ZBWEQKIXMO type=del size=1693430 2024-09-13T00:01:01.090 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.089+0000 7fb8f2ce0640 1 < completed delete operation key=70F7D20HI504 tag=5ZR436V5ISJ1 type=del 2024-09-13T00:01:01.096 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.093+0000 7fb8f2ce0640 1 > prepared operation key=NX26SI4F2ZMB tag=81WBUTTIPGS5 type=del size=2659515 2024-09-13T00:01:01.107 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.105+0000 7fb8f2ce0640 1 < suggested update operation key=H9AU4HD247YU tag=RPHCDS8R3188 type=del size=1404294 2024-09-13T00:01:01.113 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.113+0000 7fb8f2ce0640 1 > prepared operation key=0WC6P8YNL0H2 tag=W7IL3FRV4ZIW type=del size=4116672 2024-09-13T00:01:01.119 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.117+0000 7fb8f2ce0640 1 < completed write operation key=H5EIP0E9CBWX tag=FG2AH3ZK6GYQ type=write size=157382 2024-09-13T00:01:01.130 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.129+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NX26SI4F2ZMB.2~EE4IDZHI1AAL.0 size=1048576 2024-09-13T00:01:01.135 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.133+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NX26SI4F2ZMB.2~EE4IDZHI1AAL.1 size=1048576 2024-09-13T00:01:01.140 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.137+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NX26SI4F2ZMB.2~EE4IDZHI1AAL.2 size=178923 2024-09-13T00:01:01.140 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.137+0000 7fb8f2ce0640 1 > prepared multipart upload key=NX26SI4F2ZMB upload=EE4IDZHI1AAL tag=17Q6G4YPOZH4 type=write size=3324651 2024-09-13T00:01:01.151 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.149+0000 7fb8f2ce0640 1 < suggested update operation key=ZTK9AYBSLCOZ tag=V2QIPG7XTSG1 type=write size=131048 2024-09-13T00:01:01.157 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.157+0000 7fb8f2ce0640 1 > prepared operation key=8MPUVD37EDN9 tag=2BYFDXOUA3A8 type=write size=820716 2024-09-13T00:01:01.169 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.169+0000 7fb8f2ce0640 1 < suggested remove operation key=F0QCVR9E08WE tag=SB99Q8IIBFMF type=write 2024-09-13T00:01:01.175 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.173+0000 7fb8f2ce0640 1 > prepared operation key=N3Z5XIG3TZRQ tag=E2TU1WKFTZJ8 type=del size=2587347 2024-09-13T00:01:01.181 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.181+0000 7fb8f2ce0640 1 < completed delete operation key=P8OYS5ZE1NX0 tag=Y06DN4QVS8WJ type=del 2024-09-13T00:01:01.187 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.185+0000 7fb8f2ce0640 1 > prepared operation key=O4MZ4UWNXME9 tag=5MZXSXFPVL4U type=del size=1368341 2024-09-13T00:01:01.192 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.189+0000 7fb8f2ce0640 1 < completed delete operation key=H5EIP0E9CBWX tag=TNN8A3B0AD1M type=del 2024-09-13T00:01:01.198 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.197+0000 7fb8f2ce0640 1 > prepared operation key=NQJQC8CEAC15 tag=U8BN5RJYBQNP type=del size=2918388 2024-09-13T00:01:01.204 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.201+0000 7fb8f2ce0640 1 < completed delete operation key=85YEW8N43FTA tag=TE2U8AL16O1G type=del 2024-09-13T00:01:01.211 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.209+0000 7fb8f2ce0640 1 > prepared operation key=J8NGDSF9XRTP tag=2SIPMYNFNRKB type=write size=193912 2024-09-13T00:01:01.217 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.217+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~KL2FYN133A7X.0 size=1048576 2024-09-13T00:01:01.224 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.221+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~KL2FYN133A7X.1 size=1048576 2024-09-13T00:01:01.229 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.229+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~KL2FYN133A7X.2 size=931674 2024-09-13T00:01:01.240 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.237+0000 7fb8f2ce0640 1 < completed multipart upload key=1PX7DBO9M3YI upload=KL2FYN133A7X tag=AYQ8WPAO063D type=write size=4077402 2024-09-13T00:01:01.246 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.245+0000 7fb8f2ce0640 1 > prepared operation key=2AHYRZUDFHP4 tag=CMXEX9W1KPE0 type=del size=283979 2024-09-13T00:01:01.252 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.249+0000 7fb8f2ce0640 1 < completed delete operation key=EJN6HR8I7XQF tag=GAI70MIO1YYB type=del 2024-09-13T00:01:01.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:01:00 smithi136 bash[75164]: cluster 2024-09-13T00:00:59.140167+0000 mgr.y (mgr.24661) 716 : cluster [DBG] pgmap v1392: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 348 KiB/s rd, 121 KiB/s wr, 619 op/s 2024-09-13T00:01:01.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:01:00 smithi136 bash[76371]: cluster 2024-09-13T00:00:59.140167+0000 mgr.y (mgr.24661) 716 : cluster [DBG] pgmap v1392: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 348 KiB/s rd, 121 KiB/s wr, 619 op/s 2024-09-13T00:01:01.257 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.257+0000 7fb8f2ce0640 1 > prepared operation key=H9AU4HD247YU tag=1UGOOV78AM35 type=del size=3288399 2024-09-13T00:01:01.263 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.261+0000 7fb8f2ce0640 1 < completed part key=_multipart_63FJDWBO2T9S.2~I8FNW010TE5H.0 size=1048576 2024-09-13T00:01:01.268 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.265+0000 7fb8f2ce0640 1 < completed part key=_multipart_63FJDWBO2T9S.2~I8FNW010TE5H.1 size=700750 2024-09-13T00:01:01.278 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.277+0000 7fb8f2ce0640 1 < completed multipart upload key=63FJDWBO2T9S upload=I8FNW010TE5H tag=NO1PEAGP61CP type=write size=2797902 2024-09-13T00:01:01.289 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.289+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~18IADIIJG4LW.0 size=1048576 2024-09-13T00:01:01.294 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.293+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~18IADIIJG4LW.1 size=1048576 2024-09-13T00:01:01.299 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.297+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~18IADIIJG4LW.2 size=547213 2024-09-13T00:01:01.299 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.297+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZTK9AYBSLCOZ upload=18IADIIJG4LW tag=4IL3K3DZRHI4 type=write size=3692941 2024-09-13T00:01:01.300 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.297+0000 7fb8f2ce0640 1 < canceled part key=_multipart_ZFHN2SNTZ5W7.2~1336GZ7L6TD0.0 size=1048576 2024-09-13T00:01:01.300 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.297+0000 7fb8f2ce0640 1 < canceled part key=_multipart_ZFHN2SNTZ5W7.2~1336GZ7L6TD0.1 size=1048576 2024-09-13T00:01:01.304 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.301+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZFHN2SNTZ5W7.2~1336GZ7L6TD0.2 size=333891 2024-09-13T00:01:01.314 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.313+0000 7fb8f2ce0640 1 < canceled multipart upload key=ZFHN2SNTZ5W7 upload=1336GZ7L6TD0 tag=K8UG4ZPKSUC4 type=write size=3479619 2024-09-13T00:01:01.320 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.317+0000 7fb8f2ce0640 1 > prepared operation key=NX26SI4F2ZMB tag=8K5EB536KDO7 type=del size=4168913 2024-09-13T00:01:01.326 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.325+0000 7fb8f2ce0640 1 < completed delete operation key=8Z6U7OOA0GBH tag=M3NO4YKH8NE3 type=del 2024-09-13T00:01:01.336 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.333+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H9AU4HD247YU.2~CTGDPRRU8U4I.0 size=609487 2024-09-13T00:01:01.336 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.333+0000 7fb8f2ce0640 1 > prepared multipart upload key=H9AU4HD247YU upload=CTGDPRRU8U4I tag=ZXIQTXEL1336 type=write size=1658063 2024-09-13T00:01:01.343 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.341+0000 7fb8f2ce0640 1 < canceled operation key=2TWD9YLMSYJI tag=6LA3C0Z1YA67 type=del size=2697180 2024-09-13T00:01:01.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:01:00 smithi181 bash[67587]: cluster 2024-09-13T00:00:59.140167+0000 mgr.y (mgr.24661) 716 : cluster [DBG] pgmap v1392: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 348 KiB/s rd, 121 KiB/s wr, 619 op/s 2024-09-13T00:01:01.349 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.349+0000 7fb8f2ce0640 1 > prepared operation key=8MPUVD37EDN9 tag=OKSEQZ1J4VWO type=del size=4031001 2024-09-13T00:01:01.355 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.353+0000 7fb8f2ce0640 1 < canceled operation key=QMZREZ1QEDNJ tag=7J5JM8TIYBJ4 type=del size=887968 2024-09-13T00:01:01.361 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.361+0000 7fb8f2ce0640 1 > prepared operation key=N3Z5XIG3TZRQ tag=15WPQ3DMXII5 type=del size=2846970 2024-09-13T00:01:01.367 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.365+0000 7fb8f2ce0640 1 < completed delete operation key=ZFHN2SNTZ5W7 tag=I4ZBWEQKIXMO type=del 2024-09-13T00:01:01.372 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.369+0000 7fb8f2ce0640 1 > prepared operation key=NX26SI4F2ZMB tag=QK9AO74H7QSG type=del size=2728829 2024-09-13T00:01:01.378 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.377+0000 7fb8f2ce0640 1 < completed delete operation key=NX26SI4F2ZMB tag=81WBUTTIPGS5 type=del 2024-09-13T00:01:01.384 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.381+0000 7fb8f2ce0640 1 > prepared operation key=70F7D20HI504 tag=SEMP66SR07NT type=del size=1976417 2024-09-13T00:01:01.395 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.393+0000 7fb8f2ce0640 1 < suggested remove operation key=0WC6P8YNL0H2 tag=W7IL3FRV4ZIW type=del 2024-09-13T00:01:01.401 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.401+0000 7fb8f2ce0640 1 > prepared operation key=EJN6HR8I7XQF tag=Q3X9Z3YAILNA type=del size=1367988 2024-09-13T00:01:01.407 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.405+0000 7fb8f2ce0640 1 < completed part key=_multipart_NX26SI4F2ZMB.2~EE4IDZHI1AAL.0 size=1048576 2024-09-13T00:01:01.412 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.409+0000 7fb8f2ce0640 1 < completed part key=_multipart_NX26SI4F2ZMB.2~EE4IDZHI1AAL.1 size=1048576 2024-09-13T00:01:01.417 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.417+0000 7fb8f2ce0640 1 < completed part key=_multipart_NX26SI4F2ZMB.2~EE4IDZHI1AAL.2 size=178923 2024-09-13T00:01:01.427 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.425+0000 7fb8f2ce0640 1 < completed multipart upload key=NX26SI4F2ZMB upload=EE4IDZHI1AAL tag=17Q6G4YPOZH4 type=write size=3324651 2024-09-13T00:01:01.433 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.429+0000 7fb8f2ce0640 1 > prepared operation key=FX0NCGLK0KPJ tag=I3JP89M5BON5 type=del size=3754717 2024-09-13T00:01:01.444 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.441+0000 7fb8f2ce0640 1 < suggested update operation key=8MPUVD37EDN9 tag=2BYFDXOUA3A8 type=write size=820716 2024-09-13T00:01:01.450 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.449+0000 7fb8f2ce0640 1 > prepared operation key=H4M1TCN5W6XU tag=109JQZW6J8CF type=write size=931926 2024-09-13T00:01:01.456 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.453+0000 7fb8f2ce0640 1 < canceled operation key=N3Z5XIG3TZRQ tag=E2TU1WKFTZJ8 type=del size=2587347 2024-09-13T00:01:01.462 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.461+0000 7fb8f2ce0640 1 > prepared operation key=44EDYFT9VF8Y tag=0ABP9Y3ZTR6R type=del size=786286 2024-09-13T00:01:01.468 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.465+0000 7fb8f2ce0640 1 < completed delete operation key=O4MZ4UWNXME9 tag=5MZXSXFPVL4U type=del 2024-09-13T00:01:01.474 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.473+0000 7fb8f2ce0640 1 > prepared operation key=8MPUVD37EDN9 tag=YXRY73RJ1E8Y type=del size=3324388 2024-09-13T00:01:01.485 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.485+0000 7fb8f2ce0640 1 < suggested update operation key=NQJQC8CEAC15 tag=U8BN5RJYBQNP type=del size=2918388 2024-09-13T00:01:01.492 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.489+0000 7fb8f2ce0640 1 > prepared operation key=70F7D20HI504 tag=PXYMT93UMBIN type=write size=98577 2024-09-13T00:01:01.503 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.501+0000 7fb8f2ce0640 1 < suggested remove operation key=J8NGDSF9XRTP tag=2SIPMYNFNRKB type=write 2024-09-13T00:01:01.509 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.509+0000 7fb8f2ce0640 1 > prepared operation key=H9AU4HD247YU tag=ADBB9O07UMFF type=write size=488833 2024-09-13T00:01:01.515 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.513+0000 7fb8f2ce0640 1 < completed delete operation key=2AHYRZUDFHP4 tag=CMXEX9W1KPE0 type=del 2024-09-13T00:01:01.521 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.517+0000 7fb8f2ce0640 1 > prepared operation key=8Z6U7OOA0GBH tag=NQ6WJV8D2CJ3 type=del size=441424 2024-09-13T00:01:01.527 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.525+0000 7fb8f2ce0640 1 < completed delete operation key=H9AU4HD247YU tag=1UGOOV78AM35 type=del 2024-09-13T00:01:01.533 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.529+0000 7fb8f2ce0640 1 > prepared operation key=NX26SI4F2ZMB tag=MFZORD8H2VMB type=del size=4112780 2024-09-13T00:01:01.539 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.537+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~18IADIIJG4LW.0 size=1048576 2024-09-13T00:01:01.539 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.537+0000 7fb8f2ce0640 1 < canceled part key=_multipart_ZTK9AYBSLCOZ.2~18IADIIJG4LW.1 size=1048576 2024-09-13T00:01:01.544 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.541+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~18IADIIJG4LW.2 size=547213 2024-09-13T00:01:01.557 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.553+0000 7fb8f2ce0640 1 < canceled multipart upload key=ZTK9AYBSLCOZ upload=18IADIIJG4LW tag=4IL3K3DZRHI4 type=write size=3692941 2024-09-13T00:01:01.563 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.561+0000 7fb8f2ce0640 1 > prepared operation key=5GM88ERM89KT tag=GF2KYICFRPW8 type=write size=810449 2024-09-13T00:01:01.575 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.573+0000 7fb8f2ce0640 1 < suggested remove operation key=NX26SI4F2ZMB tag=8K5EB536KDO7 type=del 2024-09-13T00:01:01.582 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.581+0000 7fb8f2ce0640 1 > prepared operation key=NX26SI4F2ZMB tag=89BQW7GAPSPJ type=del size=3884330 2024-09-13T00:01:01.588 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.585+0000 7fb8f2ce0640 1 < completed part key=_multipart_H9AU4HD247YU.2~CTGDPRRU8U4I.0 size=609487 2024-09-13T00:01:01.600 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.597+0000 7fb8f2ce0640 1 < completed multipart upload key=H9AU4HD247YU upload=CTGDPRRU8U4I tag=ZXIQTXEL1336 type=write size=1658063 2024-09-13T00:01:01.606 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.605+0000 7fb8f2ce0640 1 > prepared operation key=EUE3LT0PDIXV tag=GLCPVCPYMSZK type=del size=3891991 2024-09-13T00:01:01.617 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.613+0000 7fb8f2ce0640 1 < suggested update operation key=8MPUVD37EDN9 tag=OKSEQZ1J4VWO type=del size=4031001 2024-09-13T00:01:01.628 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.625+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~5HG96QM2VQNL.0 size=70489 2024-09-13T00:01:01.628 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.625+0000 7fb8f2ce0640 1 > prepared multipart upload key=PVYZQLGYT3XU upload=5HG96QM2VQNL tag=POJ4TUDT8LBB type=write size=1119065 2024-09-13T00:01:01.634 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.633+0000 7fb8f2ce0640 1 < completed delete operation key=N3Z5XIG3TZRQ tag=15WPQ3DMXII5 type=del 2024-09-13T00:01:01.640 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.637+0000 7fb8f2ce0640 1 > prepared operation key=H5EIP0E9CBWX tag=3U71GGSOZ1FH type=del size=484632 2024-09-13T00:01:01.651 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.649+0000 7fb8f2ce0640 1 < suggested update operation key=NX26SI4F2ZMB tag=QK9AO74H7QSG type=del size=2728829 2024-09-13T00:01:01.657 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.657+0000 7fb8f2ce0640 1 > prepared operation key=EJN6HR8I7XQF tag=2O3QQMGYN447 type=del size=1933187 2024-09-13T00:01:01.663 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.661+0000 7fb8f2ce0640 1 < completed delete operation key=70F7D20HI504 tag=SEMP66SR07NT type=del 2024-09-13T00:01:01.676 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.673+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H4M1TCN5W6XU.2~TIRT2IERXZYN.0 size=1048576 2024-09-13T00:01:01.681 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.677+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H4M1TCN5W6XU.2~TIRT2IERXZYN.1 size=400538 2024-09-13T00:01:01.681 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.677+0000 7fb8f2ce0640 1 > prepared multipart upload key=H4M1TCN5W6XU upload=TIRT2IERXZYN tag=N4ADF91L1XJ1 type=write size=2497690 2024-09-13T00:01:01.686 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.685+0000 7fb8f2ce0640 1 < completed delete operation key=EJN6HR8I7XQF tag=Q3X9Z3YAILNA type=del 2024-09-13T00:01:01.697 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.697+0000 7fb8f2ce0640 1 > prepared part key=_multipart_J8NGDSF9XRTP.2~QBNYKDHSNZQ2.0 size=1048576 2024-09-13T00:01:01.704 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.701+0000 7fb8f2ce0640 1 > prepared part key=_multipart_J8NGDSF9XRTP.2~QBNYKDHSNZQ2.1 size=198541 2024-09-13T00:01:01.704 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.701+0000 7fb8f2ce0640 1 > prepared multipart upload key=J8NGDSF9XRTP upload=QBNYKDHSNZQ2 tag=2VJT0X25NL6B type=write size=2295693 2024-09-13T00:01:01.715 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.713+0000 7fb8f2ce0640 1 < suggested update operation key=FX0NCGLK0KPJ tag=I3JP89M5BON5 type=del size=3754717 2024-09-13T00:01:01.721 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.721+0000 7fb8f2ce0640 1 > prepared operation key=H5EIP0E9CBWX tag=81W0CJW0HOGQ type=del size=4150468 2024-09-13T00:01:01.727 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.725+0000 7fb8f2ce0640 1 < completed write operation key=H4M1TCN5W6XU tag=109JQZW6J8CF type=write size=931926 2024-09-13T00:01:01.738 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.737+0000 7fb8f2ce0640 1 > prepared part key=_multipart_4K76U69GZ9AE.2~J3GZRLPI1X2W.0 size=1048576 2024-09-13T00:01:01.743 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.741+0000 7fb8f2ce0640 1 > prepared part key=_multipart_4K76U69GZ9AE.2~J3GZRLPI1X2W.1 size=1048576 2024-09-13T00:01:01.748 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.745+0000 7fb8f2ce0640 1 > prepared part key=_multipart_4K76U69GZ9AE.2~J3GZRLPI1X2W.2 size=256025 2024-09-13T00:01:01.748 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.745+0000 7fb8f2ce0640 1 > prepared multipart upload key=4K76U69GZ9AE upload=J3GZRLPI1X2W tag=SBEBEVMVC30M type=write size=3401753 2024-09-13T00:01:01.754 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.753+0000 7fb8f2ce0640 1 < completed delete operation key=44EDYFT9VF8Y tag=0ABP9Y3ZTR6R type=del 2024-09-13T00:01:01.762 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.761+0000 7fb8f2ce0640 1 > prepared operation key=P8OYS5ZE1NX0 tag=SKPZS0UK6S8C type=write size=843271 2024-09-13T00:01:01.770 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.769+0000 7fb8f2ce0640 -1 < failed to complete operation key=8MPUVD37EDN9 tag=YXRY73RJ1E8Y type=del size=3324388: (22) Invalid argument 2024-09-13T00:01:01.781 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.781+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~VVG61Y8DKUCM.0 size=1048576 2024-09-13T00:01:01.786 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.785+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~VVG61Y8DKUCM.1 size=829527 2024-09-13T00:01:01.786 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.785+0000 7fb8f2ce0640 1 > prepared multipart upload key=70F7D20HI504 upload=VVG61Y8DKUCM tag=76WLNIT4XI1N type=write size=2926679 2024-09-13T00:01:01.793 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.793+0000 7fb8f2ce0640 1 < completed write operation key=70F7D20HI504 tag=PXYMT93UMBIN type=write size=98577 2024-09-13T00:01:01.805 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.801+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~6NB0T81Z2QRL.0 size=310380 2024-09-13T00:01:01.805 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.801+0000 7fb8f2ce0640 1 > prepared multipart upload key=2AHYRZUDFHP4 upload=6NB0T81Z2QRL tag=E010XHO9D0OC type=write size=1358956 2024-09-13T00:01:01.811 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.809+0000 7fb8f2ce0640 1 < completed write operation key=H9AU4HD247YU tag=ADBB9O07UMFF type=write size=488833 2024-09-13T00:01:01.817 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.817+0000 7fb8f2ce0640 1 > prepared operation key=85YEW8N43FTA tag=5TKC1EZ13VW0 type=del size=1013224 2024-09-13T00:01:01.829 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.825+0000 7fb8f2ce0640 1 < suggested update operation key=8Z6U7OOA0GBH tag=NQ6WJV8D2CJ3 type=del size=441424 2024-09-13T00:01:01.840 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.837+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2TWD9YLMSYJI.2~96D7UXVYE0LW.0 size=1048576 2024-09-13T00:01:01.844 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.841+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2TWD9YLMSYJI.2~96D7UXVYE0LW.1 size=630038 2024-09-13T00:01:01.844 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.841+0000 7fb8f2ce0640 1 > prepared multipart upload key=2TWD9YLMSYJI upload=96D7UXVYE0LW tag=KBTN1NF41XJH type=write size=2727190 2024-09-13T00:01:01.852 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.849+0000 7fb8f2ce0640 -1 < failed to complete operation key=NX26SI4F2ZMB tag=MFZORD8H2VMB type=del size=4112780: (22) Invalid argument 2024-09-13T00:01:01.863 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.861+0000 7fb8f2ce0640 1 > prepared part key=_multipart_O4MZ4UWNXME9.2~BJQQ9MQKAKNM.0 size=938943 2024-09-13T00:01:01.863 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.861+0000 7fb8f2ce0640 1 > prepared multipart upload key=O4MZ4UWNXME9 upload=BJQQ9MQKAKNM tag=XZKR9MDQL0KV type=write size=1987519 2024-09-13T00:01:01.869 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.869+0000 7fb8f2ce0640 1 < completed write operation key=5GM88ERM89KT tag=GF2KYICFRPW8 type=write size=810449 2024-09-13T00:01:01.880 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.877+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~QKYC0VR4PJSL.0 size=619831 2024-09-13T00:01:01.880 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.877+0000 7fb8f2ce0640 1 > prepared multipart upload key=1HX06CCZJ1DD upload=QKYC0VR4PJSL tag=B930E51T7X5I type=write size=1668407 2024-09-13T00:01:01.889 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.889+0000 7fb8f2ce0640 -1 < failed to complete operation key=NX26SI4F2ZMB tag=89BQW7GAPSPJ type=del size=3884330: (22) Invalid argument 2024-09-13T00:01:01.900 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.897+0000 7fb8f2ce0640 1 > prepared part key=_multipart_F0QCVR9E08WE.2~CW4OOMVB4ZZ0.0 size=1048576 2024-09-13T00:01:01.905 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.905+0000 7fb8f2ce0640 1 > prepared part key=_multipart_F0QCVR9E08WE.2~CW4OOMVB4ZZ0.1 size=1048576 2024-09-13T00:01:01.910 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.909+0000 7fb8f2ce0640 1 > prepared part key=_multipart_F0QCVR9E08WE.2~CW4OOMVB4ZZ0.2 size=199606 2024-09-13T00:01:01.910 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.909+0000 7fb8f2ce0640 1 > prepared multipart upload key=F0QCVR9E08WE upload=CW4OOMVB4ZZ0 tag=82UVKCV0LCXX type=write size=3345334 2024-09-13T00:01:01.916 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.913+0000 7fb8f2ce0640 1 < completed delete operation key=EUE3LT0PDIXV tag=GLCPVCPYMSZK type=del 2024-09-13T00:01:01.928 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.925+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~I2XSEXWBP72U.0 size=387357 2024-09-13T00:01:01.928 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.925+0000 7fb8f2ce0640 1 > prepared multipart upload key=FX0NCGLK0KPJ upload=I2XSEXWBP72U tag=N1XMPSECB9L8 type=write size=1435933 2024-09-13T00:01:01.935 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.933+0000 7fb8f2ce0640 1 < completed part key=_multipart_PVYZQLGYT3XU.2~5HG96QM2VQNL.0 size=70489 2024-09-13T00:01:01.948 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.945+0000 7fb8f2ce0640 1 < completed multipart upload key=PVYZQLGYT3XU upload=5HG96QM2VQNL tag=POJ4TUDT8LBB type=write size=1119065 2024-09-13T00:01:01.960 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.957+0000 7fb8f2ce0640 1 > prepared part key=_multipart_P8OYS5ZE1NX0.2~D7INHP4QEA29.0 size=1048576 2024-09-13T00:01:01.965 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.965+0000 7fb8f2ce0640 1 > prepared part key=_multipart_P8OYS5ZE1NX0.2~D7INHP4QEA29.1 size=342285 2024-09-13T00:01:01.965 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.965+0000 7fb8f2ce0640 1 > prepared multipart upload key=P8OYS5ZE1NX0 upload=D7INHP4QEA29 tag=ZKG1P1B75TVC type=write size=2439437 2024-09-13T00:01:01.971 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.969+0000 7fb8f2ce0640 1 < completed delete operation key=H5EIP0E9CBWX tag=3U71GGSOZ1FH type=del 2024-09-13T00:01:01.977 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.977+0000 7fb8f2ce0640 1 > prepared operation key=ZFHN2SNTZ5W7 tag=PEUE71Z6RDQV type=write size=289275 2024-09-13T00:01:01.984 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.981+0000 7fb8f2ce0640 1 < completed delete operation key=EJN6HR8I7XQF tag=2O3QQMGYN447 type=del 2024-09-13T00:01:01.995 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.993+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~41JFCD6O7BSQ.0 size=1048576 2024-09-13T00:01:02.000 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.997+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~41JFCD6O7BSQ.1 size=725210 2024-09-13T00:01:02.000 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:01.997+0000 7fb8f2ce0640 1 > prepared multipart upload key=70F7D20HI504 upload=41JFCD6O7BSQ tag=UTK5GB4FWSDJ type=write size=2822362 2024-09-13T00:01:02.006 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.005+0000 7fb8f2ce0640 1 < completed part key=_multipart_H4M1TCN5W6XU.2~TIRT2IERXZYN.0 size=1048576 2024-09-13T00:01:02.011 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.009+0000 7fb8f2ce0640 1 < completed part key=_multipart_H4M1TCN5W6XU.2~TIRT2IERXZYN.1 size=400538 2024-09-13T00:01:02.021 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.021+0000 7fb8f2ce0640 1 < completed multipart upload key=H4M1TCN5W6XU upload=TIRT2IERXZYN tag=N4ADF91L1XJ1 type=write size=2497690 2024-09-13T00:01:02.028 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.025+0000 7fb8f2ce0640 1 > prepared operation key=0WC6P8YNL0H2 tag=D2X5HE4S2WMN type=del size=2372681 2024-09-13T00:01:02.028 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.025+0000 7fb8f2ce0640 1 < canceled part key=_multipart_J8NGDSF9XRTP.2~QBNYKDHSNZQ2.0 size=1048576 2024-09-13T00:01:02.034 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.033+0000 7fb8f2ce0640 1 < completed part key=_multipart_J8NGDSF9XRTP.2~QBNYKDHSNZQ2.1 size=198541 2024-09-13T00:01:02.047 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.045+0000 7fb8f2ce0640 1 < canceled multipart upload key=J8NGDSF9XRTP upload=QBNYKDHSNZQ2 tag=2VJT0X25NL6B type=write size=2295693 2024-09-13T00:01:02.053 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.053+0000 7fb8f2ce0640 1 > prepared operation key=ZFHN2SNTZ5W7 tag=2H3SRXGKPQZW type=write size=332663 2024-09-13T00:01:02.060 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.057+0000 7fb8f2ce0640 1 < completed delete operation key=H5EIP0E9CBWX tag=81W0CJW0HOGQ type=del 2024-09-13T00:01:02.071 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.069+0000 7fb8f2ce0640 1 > prepared part key=_multipart_J8NGDSF9XRTP.2~GWEFTQMRJMI4.0 size=1048576 2024-09-13T00:01:02.076 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.073+0000 7fb8f2ce0640 1 > prepared part key=_multipart_J8NGDSF9XRTP.2~GWEFTQMRJMI4.1 size=1048576 2024-09-13T00:01:02.081 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.081+0000 7fb8f2ce0640 1 > prepared part key=_multipart_J8NGDSF9XRTP.2~GWEFTQMRJMI4.2 size=722272 2024-09-13T00:01:02.081 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.081+0000 7fb8f2ce0640 1 > prepared multipart upload key=J8NGDSF9XRTP upload=GWEFTQMRJMI4 tag=JB8QGH7H3LHA type=write size=3868000 2024-09-13T00:01:02.087 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.085+0000 7fb8f2ce0640 1 < completed part key=_multipart_4K76U69GZ9AE.2~J3GZRLPI1X2W.0 size=1048576 2024-09-13T00:01:02.092 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.089+0000 7fb8f2ce0640 1 < completed part key=_multipart_4K76U69GZ9AE.2~J3GZRLPI1X2W.1 size=1048576 2024-09-13T00:01:02.097 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.097+0000 7fb8f2ce0640 1 < completed part key=_multipart_4K76U69GZ9AE.2~J3GZRLPI1X2W.2 size=256025 2024-09-13T00:01:02.108 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.105+0000 7fb8f2ce0640 1 < completed multipart upload key=4K76U69GZ9AE upload=J3GZRLPI1X2W tag=SBEBEVMVC30M type=write size=3401753 2024-09-13T00:01:02.119 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.117+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~RWTUE4YEFPY6.0 size=1048576 2024-09-13T00:01:02.125 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.121+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~RWTUE4YEFPY6.1 size=84488 2024-09-13T00:01:02.125 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.121+0000 7fb8f2ce0640 1 > prepared multipart upload key=8Z6U7OOA0GBH upload=RWTUE4YEFPY6 tag=GQJX3RLPZ01B type=write size=2181640 2024-09-13T00:01:02.131 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.129+0000 7fb8f2ce0640 1 < completed write operation key=P8OYS5ZE1NX0 tag=SKPZS0UK6S8C type=write size=843271 2024-09-13T00:01:02.137 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.137+0000 7fb8f2ce0640 1 > prepared operation key=N3Z5XIG3TZRQ tag=700ZB05PTD2F type=del size=963578 2024-09-13T00:01:02.143 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.141+0000 7fb8f2ce0640 1 < completed part key=_multipart_70F7D20HI504.2~VVG61Y8DKUCM.0 size=1048576 2024-09-13T00:01:02.148 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.145+0000 7fb8f2ce0640 1 < completed part key=_multipart_70F7D20HI504.2~VVG61Y8DKUCM.1 size=829527 2024-09-13T00:01:02.159 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.157+0000 7fb8f2ce0640 1 < completed multipart upload key=70F7D20HI504 upload=VVG61Y8DKUCM tag=76WLNIT4XI1N type=write size=2926679 2024-09-13T00:01:02.165 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.165+0000 7fb8f2ce0640 1 > prepared operation key=70F7D20HI504 tag=9NVEF21F5RXP type=del size=1013885 2024-09-13T00:01:02.170 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.169+0000 7fb8f2ce0640 1 < completed part key=_multipart_2AHYRZUDFHP4.2~6NB0T81Z2QRL.0 size=310380 2024-09-13T00:01:02.182 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.181+0000 7fb8f2ce0640 1 < completed multipart upload key=2AHYRZUDFHP4 upload=6NB0T81Z2QRL tag=E010XHO9D0OC type=write size=1358956 2024-09-13T00:01:02.189 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.185+0000 7fb8f2ce0640 1 > prepared operation key=63FJDWBO2T9S tag=M1KRBFLYEWDS type=del size=223828 2024-09-13T00:01:02.200 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.197+0000 7fb8f2ce0640 1 < suggested update operation key=85YEW8N43FTA tag=5TKC1EZ13VW0 type=del size=1013224 2024-09-13T00:01:02.206 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.205+0000 7fb8f2ce0640 1 > prepared operation key=ZFHN2SNTZ5W7 tag=N8UGILUD4C1J type=del size=3065197 2024-09-13T00:01:02.212 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.209+0000 7fb8f2ce0640 1 < completed part key=_multipart_2TWD9YLMSYJI.2~96D7UXVYE0LW.0 size=1048576 2024-09-13T00:01:02.217 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.217+0000 7fb8f2ce0640 1 < completed part key=_multipart_2TWD9YLMSYJI.2~96D7UXVYE0LW.1 size=630038 2024-09-13T00:01:02.228 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.225+0000 7fb8f2ce0640 1 < completed multipart upload key=2TWD9YLMSYJI upload=96D7UXVYE0LW tag=KBTN1NF41XJH type=write size=2727190 2024-09-13T00:01:02.238 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.237+0000 7fb8f2ce0640 1 > prepared part key=_multipart_P8OYS5ZE1NX0.2~1K94FV00LHIW.0 size=359586 2024-09-13T00:01:02.238 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.237+0000 7fb8f2ce0640 1 > prepared multipart upload key=P8OYS5ZE1NX0 upload=1K94FV00LHIW tag=O50129A698AG type=write size=1408162 2024-09-13T00:01:02.244 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.241+0000 7fb8f2ce0640 1 < completed part key=_multipart_O4MZ4UWNXME9.2~BJQQ9MQKAKNM.0 size=938943 2024-09-13T00:01:02.254 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.253+0000 7fb8f2ce0640 1 < completed multipart upload key=O4MZ4UWNXME9 upload=BJQQ9MQKAKNM tag=XZKR9MDQL0KV type=write size=1987519 2024-09-13T00:01:02.265 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.265+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZFHN2SNTZ5W7.2~L501FGCUAFYN.0 size=1044223 2024-09-13T00:01:02.265 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.265+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZFHN2SNTZ5W7 upload=L501FGCUAFYN tag=BQ8J17026DL8 type=write size=2092799 2024-09-13T00:01:02.271 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.269+0000 7fb8f2ce0640 1 < completed part key=_multipart_1HX06CCZJ1DD.2~QKYC0VR4PJSL.0 size=619831 2024-09-13T00:01:02.281 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.281+0000 7fb8f2ce0640 1 < completed multipart upload key=1HX06CCZJ1DD upload=QKYC0VR4PJSL tag=B930E51T7X5I type=write size=1668407 2024-09-13T00:01:02.292 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.289+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~0C0JTPRYIWZ4.0 size=1048576 2024-09-13T00:01:02.298 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.297+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~0C0JTPRYIWZ4.1 size=1048576 2024-09-13T00:01:02.302 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.301+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~0C0JTPRYIWZ4.2 size=295025 2024-09-13T00:01:02.303 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.301+0000 7fb8f2ce0640 1 > prepared multipart upload key=8Z6U7OOA0GBH upload=0C0JTPRYIWZ4 tag=FQEL55CJAV2I type=write size=3440753 2024-09-13T00:01:02.309 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.305+0000 7fb8f2ce0640 1 < completed part key=_multipart_F0QCVR9E08WE.2~CW4OOMVB4ZZ0.0 size=1048576 2024-09-13T00:01:02.314 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.313+0000 7fb8f2ce0640 1 < completed part key=_multipart_F0QCVR9E08WE.2~CW4OOMVB4ZZ0.1 size=1048576 2024-09-13T00:01:02.320 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.317+0000 7fb8f2ce0640 1 < completed part key=_multipart_F0QCVR9E08WE.2~CW4OOMVB4ZZ0.2 size=199606 2024-09-13T00:01:02.331 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.329+0000 7fb8f2ce0640 1 < completed multipart upload key=F0QCVR9E08WE upload=CW4OOMVB4ZZ0 tag=82UVKCV0LCXX type=write size=3345334 2024-09-13T00:01:02.342 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.341+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NX26SI4F2ZMB.2~VRJ8UOC3TOKJ.0 size=1048576 2024-09-13T00:01:02.348 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.345+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NX26SI4F2ZMB.2~VRJ8UOC3TOKJ.1 size=1048576 2024-09-13T00:01:02.353 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.353+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NX26SI4F2ZMB.2~VRJ8UOC3TOKJ.2 size=359560 2024-09-13T00:01:02.353 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.353+0000 7fb8f2ce0640 1 > prepared multipart upload key=NX26SI4F2ZMB upload=VRJ8UOC3TOKJ tag=3EUF0N5DQSBE type=write size=3505288 2024-09-13T00:01:02.359 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.357+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~I2XSEXWBP72U.0 size=387357 2024-09-13T00:01:02.369 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.369+0000 7fb8f2ce0640 1 < completed multipart upload key=FX0NCGLK0KPJ upload=I2XSEXWBP72U tag=N1XMPSECB9L8 type=write size=1435933 2024-09-13T00:01:02.380 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.377+0000 7fb8f2ce0640 1 > prepared part key=_multipart_O4MZ4UWNXME9.2~0KMUJE67C186.0 size=1048576 2024-09-13T00:01:02.386 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.385+0000 7fb8f2ce0640 1 > prepared part key=_multipart_O4MZ4UWNXME9.2~0KMUJE67C186.1 size=366949 2024-09-13T00:01:02.386 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.385+0000 7fb8f2ce0640 1 > prepared multipart upload key=O4MZ4UWNXME9 upload=0KMUJE67C186 tag=3V30838S3HDR type=write size=2464101 2024-09-13T00:01:02.393 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.393+0000 7fb8f2ce0640 1 < completed part key=_multipart_P8OYS5ZE1NX0.2~D7INHP4QEA29.0 size=1048576 2024-09-13T00:01:02.399 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.397+0000 7fb8f2ce0640 1 < completed part key=_multipart_P8OYS5ZE1NX0.2~D7INHP4QEA29.1 size=342285 2024-09-13T00:01:02.410 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.409+0000 7fb8f2ce0640 1 < completed multipart upload key=P8OYS5ZE1NX0 upload=D7INHP4QEA29 tag=ZKG1P1B75TVC type=write size=2439437 2024-09-13T00:01:02.417 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.417+0000 7fb8f2ce0640 1 > prepared operation key=85YEW8N43FTA tag=YL5255REXA6U type=del size=509025 2024-09-13T00:01:02.423 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.421+0000 7fb8f2ce0640 1 < completed write operation key=ZFHN2SNTZ5W7 tag=PEUE71Z6RDQV type=write size=289275 2024-09-13T00:01:02.430 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.429+0000 7fb8f2ce0640 1 > prepared operation key=H4M1TCN5W6XU tag=WBB6BA4KR6WQ type=del size=3467 2024-09-13T00:01:02.436 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.433+0000 7fb8f2ce0640 1 < completed part key=_multipart_70F7D20HI504.2~41JFCD6O7BSQ.0 size=1048576 2024-09-13T00:01:02.436 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.433+0000 7fb8f2ce0640 1 < canceled part key=_multipart_70F7D20HI504.2~41JFCD6O7BSQ.1 size=725210 2024-09-13T00:01:02.447 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.445+0000 7fb8f2ce0640 1 < canceled multipart upload key=70F7D20HI504 upload=41JFCD6O7BSQ tag=UTK5GB4FWSDJ type=write size=2822362 2024-09-13T00:01:02.452 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.449+0000 7fb8f2ce0640 1 > prepared operation key=1PX7DBO9M3YI tag=G5I8M35HEE3G type=del size=707144 2024-09-13T00:01:02.458 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.457+0000 7fb8f2ce0640 1 < canceled operation key=0WC6P8YNL0H2 tag=D2X5HE4S2WMN type=del size=2372681 2024-09-13T00:01:02.469 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.469+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~U4CR4ECI238L.0 size=1048576 2024-09-13T00:01:02.474 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.473+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~U4CR4ECI238L.1 size=1048576 2024-09-13T00:01:02.479 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.477+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~U4CR4ECI238L.2 size=890596 2024-09-13T00:01:02.479 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.477+0000 7fb8f2ce0640 1 > prepared multipart upload key=1HX06CCZJ1DD upload=U4CR4ECI238L tag=M7IOZQHRXQXR type=write size=4036324 2024-09-13T00:01:02.484 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.481+0000 7fb8f2ce0640 1 < completed write operation key=ZFHN2SNTZ5W7 tag=2H3SRXGKPQZW type=write size=332663 2024-09-13T00:01:02.490 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.489+0000 7fb8f2ce0640 1 > prepared operation key=2AHYRZUDFHP4 tag=WOQJZAROJU92 type=del size=4107205 2024-09-13T00:01:02.496 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.493+0000 7fb8f2ce0640 1 < completed part key=_multipart_J8NGDSF9XRTP.2~GWEFTQMRJMI4.0 size=1048576 2024-09-13T00:01:02.500 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.497+0000 7fb8f2ce0640 1 < completed part key=_multipart_J8NGDSF9XRTP.2~GWEFTQMRJMI4.1 size=1048576 2024-09-13T00:01:02.501 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.497+0000 7fb8f2ce0640 1 < canceled part key=_multipart_J8NGDSF9XRTP.2~GWEFTQMRJMI4.2 size=722272 2024-09-13T00:01:02.511 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.509+0000 7fb8f2ce0640 1 < canceled multipart upload key=J8NGDSF9XRTP upload=GWEFTQMRJMI4 tag=JB8QGH7H3LHA type=write size=3868000 2024-09-13T00:01:02.518 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.517+0000 7fb8f2ce0640 1 > prepared operation key=H9AU4HD247YU tag=S21UR9F7FT73 type=del size=2081018 2024-09-13T00:01:02.526 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.525+0000 7fb8f2ce0640 1 < completed part key=_multipart_8Z6U7OOA0GBH.2~RWTUE4YEFPY6.0 size=1048576 2024-09-13T00:01:02.531 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.529+0000 7fb8f2ce0640 1 < completed part key=_multipart_8Z6U7OOA0GBH.2~RWTUE4YEFPY6.1 size=84488 2024-09-13T00:01:02.543 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.541+0000 7fb8f2ce0640 1 < completed multipart upload key=8Z6U7OOA0GBH upload=RWTUE4YEFPY6 tag=GQJX3RLPZ01B type=write size=2181640 2024-09-13T00:01:02.549 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.549+0000 7fb8f2ce0640 1 > prepared operation key=W8XZ26311RJX tag=3D9BNO15LPCQ type=del size=2986529 2024-09-13T00:01:02.556 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.553+0000 7fb8f2ce0640 1 < completed delete operation key=N3Z5XIG3TZRQ tag=700ZB05PTD2F type=del 2024-09-13T00:01:02.561 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.561+0000 7fb8f2ce0640 1 > prepared operation key=H9AU4HD247YU tag=C4R7ELP7PKON type=write size=451234 2024-09-13T00:01:02.568 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.565+0000 7fb8f2ce0640 1 < completed delete operation key=70F7D20HI504 tag=9NVEF21F5RXP type=del 2024-09-13T00:01:02.578 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.577+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~1A682OCCWSMJ.0 size=1048576 2024-09-13T00:01:02.584 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.581+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~1A682OCCWSMJ.1 size=1048576 2024-09-13T00:01:02.589 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.585+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~1A682OCCWSMJ.2 size=480285 2024-09-13T00:01:02.589 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.589+0000 7fb8f2ce0640 1 > prepared multipart upload key=W8XZ26311RJX upload=1A682OCCWSMJ tag=7N2HSG1ZEJED type=write size=3626013 2024-09-13T00:01:02.594 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.593+0000 7fb8f2ce0640 1 < completed delete operation key=63FJDWBO2T9S tag=M1KRBFLYEWDS type=del 2024-09-13T00:01:02.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:01:02 smithi181 bash[95505]: ts=2024-09-13T00:01:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:01:02.601 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.597+0000 7fb8f2ce0640 1 > prepared operation key=70F7D20HI504 tag=JUDH3XXNVS48 type=del size=3353028 2024-09-13T00:01:02.606 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.605+0000 7fb8f2ce0640 1 < completed delete operation key=ZFHN2SNTZ5W7 tag=N8UGILUD4C1J type=del 2024-09-13T00:01:02.618 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.617+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~K9MAUM4ST38X.0 size=1048576 2024-09-13T00:01:02.623 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.621+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~K9MAUM4ST38X.1 size=1017906 2024-09-13T00:01:02.623 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.621+0000 7fb8f2ce0640 1 > prepared multipart upload key=1HX06CCZJ1DD upload=K9MAUM4ST38X tag=EX1MKPO76GE9 type=write size=3115058 2024-09-13T00:01:02.629 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.629+0000 7fb8f2ce0640 1 < completed part key=_multipart_P8OYS5ZE1NX0.2~1K94FV00LHIW.0 size=359586 2024-09-13T00:01:02.640 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.637+0000 7fb8f2ce0640 1 < completed multipart upload key=P8OYS5ZE1NX0 upload=1K94FV00LHIW tag=O50129A698AG type=write size=1408162 2024-09-13T00:01:02.646 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.645+0000 7fb8f2ce0640 1 > prepared operation key=F0QCVR9E08WE tag=0AZ4QBO8P5YR type=write size=560777 2024-09-13T00:01:02.652 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.649+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZFHN2SNTZ5W7.2~L501FGCUAFYN.0 size=1044223 2024-09-13T00:01:02.663 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.661+0000 7fb8f2ce0640 1 < completed multipart upload key=ZFHN2SNTZ5W7 upload=L501FGCUAFYN tag=BQ8J17026DL8 type=write size=2092799 2024-09-13T00:01:02.669 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.665+0000 7fb8f2ce0640 1 > prepared operation key=4K76U69GZ9AE tag=CLC63NXDDMCH type=del size=3679141 2024-09-13T00:01:02.676 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.673+0000 7fb8f2ce0640 1 < completed part key=_multipart_8Z6U7OOA0GBH.2~0C0JTPRYIWZ4.0 size=1048576 2024-09-13T00:01:02.676 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.673+0000 7fb8f2ce0640 1 < canceled part key=_multipart_8Z6U7OOA0GBH.2~0C0JTPRYIWZ4.1 size=1048576 2024-09-13T00:01:02.680 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.677+0000 7fb8f2ce0640 1 < completed part key=_multipart_8Z6U7OOA0GBH.2~0C0JTPRYIWZ4.2 size=295025 2024-09-13T00:01:02.689 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.689+0000 7fb8f2ce0640 1 < canceled multipart upload key=8Z6U7OOA0GBH upload=0C0JTPRYIWZ4 tag=FQEL55CJAV2I type=write size=3440753 2024-09-13T00:01:02.700 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.697+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~XL7ZJF8N0PCI.0 size=1048576 2024-09-13T00:01:02.705 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.705+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~XL7ZJF8N0PCI.1 size=1048576 2024-09-13T00:01:02.712 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.709+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~XL7ZJF8N0PCI.2 size=433526 2024-09-13T00:01:02.713 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.709+0000 7fb8f2ce0640 1 > prepared multipart upload key=FX0NCGLK0KPJ upload=XL7ZJF8N0PCI tag=MCGM1I5OMGMX type=write size=3579254 2024-09-13T00:01:02.719 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.717+0000 7fb8f2ce0640 1 < completed part key=_multipart_NX26SI4F2ZMB.2~VRJ8UOC3TOKJ.0 size=1048576 2024-09-13T00:01:02.724 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.721+0000 7fb8f2ce0640 1 < completed part key=_multipart_NX26SI4F2ZMB.2~VRJ8UOC3TOKJ.1 size=1048576 2024-09-13T00:01:02.728 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.725+0000 7fb8f2ce0640 1 < completed part key=_multipart_NX26SI4F2ZMB.2~VRJ8UOC3TOKJ.2 size=359560 2024-09-13T00:01:02.740 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.737+0000 7fb8f2ce0640 1 < completed multipart upload key=NX26SI4F2ZMB upload=VRJ8UOC3TOKJ tag=3EUF0N5DQSBE type=write size=3505288 2024-09-13T00:01:02.750 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.749+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~H21TKNMYPVB7.0 size=1048576 2024-09-13T00:01:02.755 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.753+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~H21TKNMYPVB7.1 size=170972 2024-09-13T00:01:02.755 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.753+0000 7fb8f2ce0640 1 > prepared multipart upload key=2AHYRZUDFHP4 upload=H21TKNMYPVB7 tag=S2SD2X9KVGXC type=write size=2268124 2024-09-13T00:01:02.761 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.761+0000 7fb8f2ce0640 1 < completed part key=_multipart_O4MZ4UWNXME9.2~0KMUJE67C186.0 size=1048576 2024-09-13T00:01:02.766 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.765+0000 7fb8f2ce0640 1 < completed part key=_multipart_O4MZ4UWNXME9.2~0KMUJE67C186.1 size=366949 2024-09-13T00:01:02.776 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.773+0000 7fb8f2ce0640 1 < completed multipart upload key=O4MZ4UWNXME9 upload=0KMUJE67C186 tag=3V30838S3HDR type=write size=2464101 2024-09-13T00:01:02.787 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.785+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~8DLEQ6ARHXY1.0 size=592198 2024-09-13T00:01:02.787 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.785+0000 7fb8f2ce0640 1 > prepared multipart upload key=1PX7DBO9M3YI upload=8DLEQ6ARHXY1 tag=SXQ8I4N6E345 type=write size=1640774 2024-09-13T00:01:02.792 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.789+0000 7fb8f2ce0640 1 < completed delete operation key=85YEW8N43FTA tag=YL5255REXA6U type=del 2024-09-13T00:01:02.802 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.801+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~70GKHYXAYV3O.0 size=998419 2024-09-13T00:01:02.802 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.801+0000 7fb8f2ce0640 1 > prepared multipart upload key=2AHYRZUDFHP4 upload=70GKHYXAYV3O tag=JM3Y8H8UBTHE type=write size=2046995 2024-09-13T00:01:02.808 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.805+0000 7fb8f2ce0640 1 < completed delete operation key=H4M1TCN5W6XU tag=WBB6BA4KR6WQ type=del 2024-09-13T00:01:02.818 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.817+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~DV5PCGS9RA7Q.0 size=1048576 2024-09-13T00:01:02.823 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.821+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~DV5PCGS9RA7Q.1 size=1048576 2024-09-13T00:01:02.829 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.829+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8Z6U7OOA0GBH.2~DV5PCGS9RA7Q.2 size=1045318 2024-09-13T00:01:02.829 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.829+0000 7fb8f2ce0640 1 > prepared multipart upload key=8Z6U7OOA0GBH upload=DV5PCGS9RA7Q tag=1TL35USURZ8U type=write size=4191046 2024-09-13T00:01:02.842 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.841+0000 7fb8f2ce0640 1 < suggested update operation key=1PX7DBO9M3YI tag=G5I8M35HEE3G type=del size=707144 2024-09-13T00:01:02.848 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.845+0000 7fb8f2ce0640 1 > prepared operation key=P8OYS5ZE1NX0 tag=144STW6SLCMC type=del size=2477561 2024-09-13T00:01:02.854 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.853+0000 7fb8f2ce0640 1 < completed part key=_multipart_1HX06CCZJ1DD.2~U4CR4ECI238L.0 size=1048576 2024-09-13T00:01:02.858 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.857+0000 7fb8f2ce0640 1 < completed part key=_multipart_1HX06CCZJ1DD.2~U4CR4ECI238L.1 size=1048576 2024-09-13T00:01:02.863 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.861+0000 7fb8f2ce0640 1 < completed part key=_multipart_1HX06CCZJ1DD.2~U4CR4ECI238L.2 size=890596 2024-09-13T00:01:02.874 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.873+0000 7fb8f2ce0640 1 < completed multipart upload key=1HX06CCZJ1DD upload=U4CR4ECI238L tag=M7IOZQHRXQXR type=write size=4036324 2024-09-13T00:01:02.879 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.877+0000 7fb8f2ce0640 1 > prepared operation key=8Z6U7OOA0GBH tag=XWVZEZZ62URE type=del size=3468027 2024-09-13T00:01:02.885 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.885+0000 7fb8f2ce0640 1 < completed delete operation key=2AHYRZUDFHP4 tag=WOQJZAROJU92 type=del 2024-09-13T00:01:02.890 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.889+0000 7fb8f2ce0640 1 > prepared operation key=70F7D20HI504 tag=ABD407IJDVAT type=del size=1130169 2024-09-13T00:01:02.896 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.893+0000 7fb8f2ce0640 1 < completed delete operation key=H9AU4HD247YU tag=S21UR9F7FT73 type=del 2024-09-13T00:01:02.906 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.905+0000 7fb8f2ce0640 1 > prepared part key=_multipart_0WC6P8YNL0H2.2~7REI9C7XN3MS.0 size=539215 2024-09-13T00:01:02.906 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.905+0000 7fb8f2ce0640 1 > prepared multipart upload key=0WC6P8YNL0H2 upload=7REI9C7XN3MS tag=6PLRQCFGT55R type=write size=1587791 2024-09-13T00:01:02.912 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.909+0000 7fb8f2ce0640 1 < canceled operation key=W8XZ26311RJX tag=3D9BNO15LPCQ type=del size=2986529 2024-09-13T00:01:02.918 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.917+0000 7fb8f2ce0640 1 > prepared operation key=N3Z5XIG3TZRQ tag=LG0BLWELDP4S type=del size=1529204 2024-09-13T00:01:02.924 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.921+0000 7fb8f2ce0640 1 < completed write operation key=H9AU4HD247YU tag=C4R7ELP7PKON type=write size=451234 2024-09-13T00:01:02.936 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.933+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~4ICH58UFQ81A.0 size=1048576 2024-09-13T00:01:02.941 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.941+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~4ICH58UFQ81A.1 size=1048576 2024-09-13T00:01:02.947 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.945+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~4ICH58UFQ81A.2 size=962899 2024-09-13T00:01:02.948 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.945+0000 7fb8f2ce0640 1 > prepared multipart upload key=1PX7DBO9M3YI upload=4ICH58UFQ81A tag=KDTA8M33R4XO type=write size=4108627 2024-09-13T00:01:02.954 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.953+0000 7fb8f2ce0640 1 < completed part key=_multipart_W8XZ26311RJX.2~1A682OCCWSMJ.0 size=1048576 2024-09-13T00:01:02.959 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.957+0000 7fb8f2ce0640 1 < completed part key=_multipart_W8XZ26311RJX.2~1A682OCCWSMJ.1 size=1048576 2024-09-13T00:01:02.963 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.961+0000 7fb8f2ce0640 1 < completed part key=_multipart_W8XZ26311RJX.2~1A682OCCWSMJ.2 size=480285 2024-09-13T00:01:02.972 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.969+0000 7fb8f2ce0640 1 < completed multipart upload key=W8XZ26311RJX upload=1A682OCCWSMJ tag=7N2HSG1ZEJED type=write size=3626013 2024-09-13T00:01:02.983 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.981+0000 7fb8f2ce0640 1 > prepared part key=_multipart_P8OYS5ZE1NX0.2~K0V67JTUKPB8.0 size=1048576 2024-09-13T00:01:02.987 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.985+0000 7fb8f2ce0640 1 > prepared part key=_multipart_P8OYS5ZE1NX0.2~K0V67JTUKPB8.1 size=751683 2024-09-13T00:01:02.987 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.985+0000 7fb8f2ce0640 1 > prepared multipart upload key=P8OYS5ZE1NX0 upload=K0V67JTUKPB8 tag=HR8R3GPKDIE4 type=write size=2848835 2024-09-13T00:01:02.998 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:02.997+0000 7fb8f2ce0640 1 < suggested remove operation key=70F7D20HI504 tag=JUDH3XXNVS48 type=del 2024-09-13T00:01:03.003 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.001+0000 7fb8f2ce0640 1 > prepared operation key=PVYZQLGYT3XU tag=W9IE30OOUOWJ type=del size=1197489 2024-09-13T00:01:03.005 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.001+0000 7fb8f2ce0640 1 < canceled part key=_multipart_1HX06CCZJ1DD.2~K9MAUM4ST38X.0 size=1048576 2024-09-13T00:01:03.009 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.009+0000 7fb8f2ce0640 1 < completed part key=_multipart_1HX06CCZJ1DD.2~K9MAUM4ST38X.1 size=1017906 2024-09-13T00:01:03.020 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.017+0000 7fb8f2ce0640 1 < canceled multipart upload key=1HX06CCZJ1DD upload=K9MAUM4ST38X tag=EX1MKPO76GE9 type=write size=3115058 2024-09-13T00:01:03.030 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.029+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EUE3LT0PDIXV.2~06HTH2M7AX5V.0 size=793465 2024-09-13T00:01:03.030 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.029+0000 7fb8f2ce0640 1 > prepared multipart upload key=EUE3LT0PDIXV upload=06HTH2M7AX5V tag=W0E7XMT7RAQZ type=write size=1842041 2024-09-13T00:01:03.036 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.033+0000 7fb8f2ce0640 1 < completed write operation key=F0QCVR9E08WE tag=0AZ4QBO8P5YR type=write size=560777 2024-09-13T00:01:03.041 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.041+0000 7fb8f2ce0640 1 > prepared operation key=FX0NCGLK0KPJ tag=3UB390WI6KQ0 type=del size=1855368 2024-09-13T00:01:03.053 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.053+0000 7fb8f2ce0640 1 < suggested update operation key=4K76U69GZ9AE tag=CLC63NXDDMCH type=del size=3679141 2024-09-13T00:01:03.063 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.061+0000 7fb8f2ce0640 1 > prepared part key=_multipart_5GM88ERM89KT.2~WQSI9WD716DA.0 size=651407 2024-09-13T00:01:03.063 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.061+0000 7fb8f2ce0640 1 > prepared multipart upload key=5GM88ERM89KT upload=WQSI9WD716DA tag=5RF4H7UYXED9 type=write size=1699983 2024-09-13T00:01:03.068 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.065+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~XL7ZJF8N0PCI.0 size=1048576 2024-09-13T00:01:03.073 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.073+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~XL7ZJF8N0PCI.1 size=1048576 2024-09-13T00:01:03.078 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.077+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~XL7ZJF8N0PCI.2 size=433526 2024-09-13T00:01:03.088 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.085+0000 7fb8f2ce0640 1 < completed multipart upload key=FX0NCGLK0KPJ upload=XL7ZJF8N0PCI tag=MCGM1I5OMGMX type=write size=3579254 2024-09-13T00:01:03.094 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.093+0000 7fb8f2ce0640 1 > prepared operation key=P8OYS5ZE1NX0 tag=GOGOZ8RVIET4 type=del size=770364 2024-09-13T00:01:03.100 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.097+0000 7fb8f2ce0640 1 < completed part key=_multipart_2AHYRZUDFHP4.2~H21TKNMYPVB7.0 size=1048576 2024-09-13T00:01:03.100 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.097+0000 7fb8f2ce0640 1 < canceled part key=_multipart_2AHYRZUDFHP4.2~H21TKNMYPVB7.1 size=170972 2024-09-13T00:01:03.110 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.109+0000 7fb8f2ce0640 1 < canceled multipart upload key=2AHYRZUDFHP4 upload=H21TKNMYPVB7 tag=S2SD2X9KVGXC type=write size=2268124 2024-09-13T00:01:03.122 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.121+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~23NTOC0U2L20.0 size=1048576 2024-09-13T00:01:03.127 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.125+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~23NTOC0U2L20.1 size=152868 2024-09-13T00:01:03.127 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.125+0000 7fb8f2ce0640 1 > prepared multipart upload key=1HX06CCZJ1DD upload=23NTOC0U2L20 tag=XEGJPW8JRYY8 type=write size=2250020 2024-09-13T00:01:03.133 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.133+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~8DLEQ6ARHXY1.0 size=592198 2024-09-13T00:01:03.143 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.141+0000 7fb8f2ce0640 1 < completed multipart upload key=1PX7DBO9M3YI upload=8DLEQ6ARHXY1 tag=SXQ8I4N6E345 type=write size=1640774 2024-09-13T00:01:03.149 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.149+0000 7fb8f2ce0640 1 > prepared operation key=EUE3LT0PDIXV tag=K63XXXNEJJ6W type=del size=1402934 2024-09-13T00:01:03.155 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.153+0000 7fb8f2ce0640 1 < completed part key=_multipart_2AHYRZUDFHP4.2~70GKHYXAYV3O.0 size=998419 2024-09-13T00:01:03.164 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.161+0000 7fb8f2ce0640 1 < completed multipart upload key=2AHYRZUDFHP4 upload=70GKHYXAYV3O tag=JM3Y8H8UBTHE type=write size=2046995 2024-09-13T00:01:03.170 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.169+0000 7fb8f2ce0640 1 > prepared operation key=F0QCVR9E08WE tag=W5FEAS9LI922 type=del size=1777337 2024-09-13T00:01:03.171 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.169+0000 7fb8f2ce0640 1 < canceled part key=_multipart_8Z6U7OOA0GBH.2~DV5PCGS9RA7Q.0 size=1048576 2024-09-13T00:01:03.177 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.177+0000 7fb8f2ce0640 1 < completed part key=_multipart_8Z6U7OOA0GBH.2~DV5PCGS9RA7Q.1 size=1048576 2024-09-13T00:01:03.183 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.181+0000 7fb8f2ce0640 1 < completed part key=_multipart_8Z6U7OOA0GBH.2~DV5PCGS9RA7Q.2 size=1045318 2024-09-13T00:01:03.198 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.197+0000 7fb8f2ce0640 1 < canceled multipart upload key=8Z6U7OOA0GBH upload=DV5PCGS9RA7Q tag=1TL35USURZ8U type=write size=4191046 2024-09-13T00:01:03.205 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.201+0000 7fb8f2ce0640 1 > prepared operation key=PVYZQLGYT3XU tag=XF0ET1JQ3COF type=del size=3442190 2024-09-13T00:01:03.216 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.213+0000 7fb8f2ce0640 1 < suggested remove operation key=P8OYS5ZE1NX0 tag=144STW6SLCMC type=del 2024-09-13T00:01:03.223 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.221+0000 7fb8f2ce0640 1 > prepared operation key=QMZREZ1QEDNJ tag=NMYV347IZ108 type=del size=1035258 2024-09-13T00:01:03.234 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.233+0000 7fb8f2ce0640 1 < suggested remove operation key=8Z6U7OOA0GBH tag=XWVZEZZ62URE type=del 2024-09-13T00:01:03.241 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.241+0000 7fb8f2ce0640 1 > prepared operation key=4K76U69GZ9AE tag=GETFGL6VCBQ0 type=del size=4000475 2024-09-13T00:01:03.249 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.245+0000 7fb8f2ce0640 -1 < failed to complete operation key=70F7D20HI504 tag=ABD407IJDVAT type=del size=1130169: (22) Invalid argument 2024-09-13T00:01:03.254 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.253+0000 7fb8f2ce0640 1 > prepared operation key=F0QCVR9E08WE tag=K21OM8R1456S type=del size=667343 2024-09-13T00:01:03.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:01:02 smithi136 bash[75164]: cluster 2024-09-13T00:01:01.141272+0000 mgr.y (mgr.24661) 717 : cluster [DBG] pgmap v1393: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 516 KiB/s rd, 178 KiB/s wr, 914 op/s 2024-09-13T00:01:03.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:01:02 smithi136 bash[76371]: cluster 2024-09-13T00:01:01.141272+0000 mgr.y (mgr.24661) 717 : cluster [DBG] pgmap v1393: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 516 KiB/s rd, 178 KiB/s wr, 914 op/s 2024-09-13T00:01:03.261 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.257+0000 7fb8f2ce0640 1 < completed part key=_multipart_0WC6P8YNL0H2.2~7REI9C7XN3MS.0 size=539215 2024-09-13T00:01:03.271 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.269+0000 7fb8f2ce0640 1 < completed multipart upload key=0WC6P8YNL0H2 upload=7REI9C7XN3MS tag=6PLRQCFGT55R type=write size=1587791 2024-09-13T00:01:03.277 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.273+0000 7fb8f2ce0640 1 > prepared operation key=NQJQC8CEAC15 tag=3UMBBFDU296L type=del size=2364574 2024-09-13T00:01:03.283 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.281+0000 7fb8f2ce0640 1 < completed delete operation key=N3Z5XIG3TZRQ tag=LG0BLWELDP4S type=del 2024-09-13T00:01:03.289 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.289+0000 7fb8f2ce0640 1 > prepared operation key=H5EIP0E9CBWX tag=9UJMIBN47N4S type=del size=290623 2024-09-13T00:01:03.297 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.297+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~4ICH58UFQ81A.0 size=1048576 2024-09-13T00:01:03.297 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.297+0000 7fb8f2ce0640 1 < canceled part key=_multipart_1PX7DBO9M3YI.2~4ICH58UFQ81A.1 size=1048576 2024-09-13T00:01:03.302 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.301+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~4ICH58UFQ81A.2 size=962899 2024-09-13T00:01:03.314 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.313+0000 7fb8f2ce0640 1 < canceled multipart upload key=1PX7DBO9M3YI upload=4ICH58UFQ81A tag=KDTA8M33R4XO type=write size=4108627 2024-09-13T00:01:03.320 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.317+0000 7fb8f2ce0640 1 > prepared operation key=4K76U69GZ9AE tag=JGN228DPJ7D1 type=del size=3571741 2024-09-13T00:01:03.326 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.325+0000 7fb8f2ce0640 1 < completed part key=_multipart_P8OYS5ZE1NX0.2~K0V67JTUKPB8.0 size=1048576 2024-09-13T00:01:03.326 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.325+0000 7fb8f2ce0640 1 < canceled part key=_multipart_P8OYS5ZE1NX0.2~K0V67JTUKPB8.1 size=751683 2024-09-13T00:01:03.337 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.333+0000 7fb8f2ce0640 1 < canceled multipart upload key=P8OYS5ZE1NX0 upload=K0V67JTUKPB8 tag=HR8R3GPKDIE4 type=write size=2848835 2024-09-13T00:01:03.342 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.341+0000 7fb8f2ce0640 1 > prepared operation key=85YEW8N43FTA tag=SQWLW6OB2GO3 type=del size=3344908 2024-09-13T00:01:03.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:01:02 smithi181 bash[67587]: cluster 2024-09-13T00:01:01.141272+0000 mgr.y (mgr.24661) 717 : cluster [DBG] pgmap v1393: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 516 KiB/s rd, 178 KiB/s wr, 914 op/s 2024-09-13T00:01:03.354 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.353+0000 7fb8f2ce0640 1 < suggested remove operation key=PVYZQLGYT3XU tag=W9IE30OOUOWJ type=del 2024-09-13T00:01:03.359 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.357+0000 7fb8f2ce0640 1 > prepared operation key=F0QCVR9E08WE tag=IFNIY6ZXJQGS type=write size=873975 2024-09-13T00:01:03.365 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.365+0000 7fb8f2ce0640 1 < completed part key=_multipart_EUE3LT0PDIXV.2~06HTH2M7AX5V.0 size=793465 2024-09-13T00:01:03.374 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.373+0000 7fb8f2ce0640 1 < completed multipart upload key=EUE3LT0PDIXV upload=06HTH2M7AX5V tag=W0E7XMT7RAQZ type=write size=1842041 2024-09-13T00:01:03.385 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.385+0000 7fb8f2ce0640 1 > prepared part key=_multipart_F0QCVR9E08WE.2~WTE544PUSHJH.0 size=1048576 2024-09-13T00:01:03.390 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.389+0000 7fb8f2ce0640 1 > prepared part key=_multipart_F0QCVR9E08WE.2~WTE544PUSHJH.1 size=977443 2024-09-13T00:01:03.390 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.389+0000 7fb8f2ce0640 1 > prepared multipart upload key=F0QCVR9E08WE upload=WTE544PUSHJH tag=2HH3UR6DAN5D type=write size=3074595 2024-09-13T00:01:03.401 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.401+0000 7fb8f2ce0640 1 < suggested update operation key=FX0NCGLK0KPJ tag=3UB390WI6KQ0 type=del size=1855368 2024-09-13T00:01:03.408 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.405+0000 7fb8f2ce0640 1 > prepared operation key=4K76U69GZ9AE tag=R0WNS4RIEG1X type=del size=523779 2024-09-13T00:01:03.414 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.413+0000 7fb8f2ce0640 1 < completed part key=_multipart_5GM88ERM89KT.2~WQSI9WD716DA.0 size=651407 2024-09-13T00:01:03.424 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.421+0000 7fb8f2ce0640 1 < completed multipart upload key=5GM88ERM89KT upload=WQSI9WD716DA tag=5RF4H7UYXED9 type=write size=1699983 2024-09-13T00:01:03.430 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.429+0000 7fb8f2ce0640 1 > prepared operation key=QMZREZ1QEDNJ tag=EF1OCG1WYOPQ type=del size=3948261 2024-09-13T00:01:03.433 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.429+0000 7fb8f2ce0640 -1 < no bi entry to suggest for operation key=P8OYS5ZE1NX0 tag=GOGOZ8RVIET4 type=del size=770364: (2) No such file or directory 2024-09-13T00:01:03.439 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.437+0000 7fb8f2ce0640 1 > prepared operation key=H4M1TCN5W6XU tag=6N8GLGHJ0BIE type=write size=103486 2024-09-13T00:01:03.445 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.445+0000 7fb8f2ce0640 1 < completed part key=_multipart_1HX06CCZJ1DD.2~23NTOC0U2L20.0 size=1048576 2024-09-13T00:01:03.450 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.449+0000 7fb8f2ce0640 1 < completed part key=_multipart_1HX06CCZJ1DD.2~23NTOC0U2L20.1 size=152868 2024-09-13T00:01:03.461 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.461+0000 7fb8f2ce0640 1 < completed multipart upload key=1HX06CCZJ1DD upload=23NTOC0U2L20 tag=XEGJPW8JRYY8 type=write size=2250020 2024-09-13T00:01:03.473 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.473+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~3DAVUOXXJ5Z0.0 size=1048576 2024-09-13T00:01:03.478 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.477+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~3DAVUOXXJ5Z0.1 size=1048576 2024-09-13T00:01:03.483 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.481+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~3DAVUOXXJ5Z0.2 size=878673 2024-09-13T00:01:03.483 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.481+0000 7fb8f2ce0640 1 > prepared multipart upload key=W8XZ26311RJX upload=3DAVUOXXJ5Z0 tag=WLZHM15N29UZ type=write size=4024401 2024-09-13T00:01:03.488 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.485+0000 7fb8f2ce0640 1 < completed delete operation key=EUE3LT0PDIXV tag=K63XXXNEJJ6W type=del 2024-09-13T00:01:03.494 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.493+0000 7fb8f2ce0640 1 > prepared operation key=QMZREZ1QEDNJ tag=LVKV26N72ASK type=del size=12435 2024-09-13T00:01:03.500 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.497+0000 7fb8f2ce0640 1 < completed delete operation key=F0QCVR9E08WE tag=W5FEAS9LI922 type=del 2024-09-13T00:01:03.507 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.505+0000 7fb8f2ce0640 1 > prepared operation key=8Z6U7OOA0GBH tag=UAB1124HV7UQ type=del size=1887680 2024-09-13T00:01:03.519 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.517+0000 7fb8f2ce0640 -1 < failed to complete operation key=PVYZQLGYT3XU tag=XF0ET1JQ3COF type=del size=3442190: (22) Invalid argument 2024-09-13T00:01:03.525 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.525+0000 7fb8f2ce0640 1 > prepared operation key=63FJDWBO2T9S tag=HKAIZTT36URF type=del size=2358046 2024-09-13T00:01:03.531 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.529+0000 7fb8f2ce0640 1 < completed delete operation key=QMZREZ1QEDNJ tag=NMYV347IZ108 type=del 2024-09-13T00:01:03.536 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.533+0000 7fb8f2ce0640 1 > prepared operation key=FX0NCGLK0KPJ tag=A73Y1D15KZ64 type=write size=1028488 2024-09-13T00:01:03.547 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.545+0000 7fb8f2ce0640 1 < suggested remove operation key=4K76U69GZ9AE tag=GETFGL6VCBQ0 type=del 2024-09-13T00:01:03.558 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.557+0000 7fb8f2ce0640 1 > prepared part key=_multipart_0WC6P8YNL0H2.2~19M7MJLQ9AP5.0 size=1048576 2024-09-13T00:01:03.563 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.561+0000 7fb8f2ce0640 1 > prepared part key=_multipart_0WC6P8YNL0H2.2~19M7MJLQ9AP5.1 size=624113 2024-09-13T00:01:03.563 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.561+0000 7fb8f2ce0640 1 > prepared multipart upload key=0WC6P8YNL0H2 upload=19M7MJLQ9AP5 tag=0MA3IN4A5W60 type=write size=2721265 2024-09-13T00:01:03.570 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.569+0000 7fb8f2ce0640 1 < canceled operation key=F0QCVR9E08WE tag=K21OM8R1456S type=del size=667343 2024-09-13T00:01:03.575 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.573+0000 7fb8f2ce0640 1 > prepared operation key=W8XZ26311RJX tag=8GXE1MN15VY5 type=del size=392918 2024-09-13T00:01:03.582 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.581+0000 7fb8f2ce0640 1 < completed delete operation key=NQJQC8CEAC15 tag=3UMBBFDU296L type=del 2024-09-13T00:01:03.588 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.585+0000 7fb8f2ce0640 1 > prepared operation key=F0QCVR9E08WE tag=ZDAVWHJW87FC type=write size=988266 2024-09-13T00:01:03.593 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.593+0000 7fb8f2ce0640 1 < completed delete operation key=H5EIP0E9CBWX tag=9UJMIBN47N4S type=del 2024-09-13T00:01:03.599 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.597+0000 7fb8f2ce0640 1 > prepared operation key=1HX06CCZJ1DD tag=1B0ENFY8X7OH type=del size=2758141 2024-09-13T00:01:03.607 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.605+0000 7fb8f2ce0640 -1 < failed to complete operation key=4K76U69GZ9AE tag=JGN228DPJ7D1 type=del size=3571741: (22) Invalid argument 2024-09-13T00:01:03.618 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.617+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~GYCS0SRPHCX5.0 size=678454 2024-09-13T00:01:03.619 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.617+0000 7fb8f2ce0640 1 > prepared multipart upload key=70F7D20HI504 upload=GYCS0SRPHCX5 tag=EQZR4J0WVMHH type=write size=1727030 2024-09-13T00:01:03.624 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.621+0000 7fb8f2ce0640 1 < completed delete operation key=85YEW8N43FTA tag=SQWLW6OB2GO3 type=del 2024-09-13T00:01:03.630 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.629+0000 7fb8f2ce0640 1 > prepared operation key=J8NGDSF9XRTP tag=XMP1XET6TW9M type=del size=1824355 2024-09-13T00:01:03.637 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.637+0000 7fb8f2ce0640 1 < canceled operation key=F0QCVR9E08WE tag=IFNIY6ZXJQGS type=write size=873975 2024-09-13T00:01:03.648 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.645+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~CBL3T7CRRPSE.0 size=951566 2024-09-13T00:01:03.648 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.645+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZTK9AYBSLCOZ upload=CBL3T7CRRPSE tag=2ZMGHD9GTGH1 type=write size=2000142 2024-09-13T00:01:03.654 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.653+0000 7fb8f2ce0640 1 < completed part key=_multipart_F0QCVR9E08WE.2~WTE544PUSHJH.0 size=1048576 2024-09-13T00:01:03.660 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.657+0000 7fb8f2ce0640 1 < completed part key=_multipart_F0QCVR9E08WE.2~WTE544PUSHJH.1 size=977443 2024-09-13T00:01:03.672 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.669+0000 7fb8f2ce0640 1 < completed multipart upload key=F0QCVR9E08WE upload=WTE544PUSHJH tag=2HH3UR6DAN5D type=write size=3074595 2024-09-13T00:01:03.679 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.677+0000 7fb8f2ce0640 1 > prepared operation key=PVYZQLGYT3XU tag=I6VBX1ZQ6OE2 type=del size=3474143 2024-09-13T00:01:03.687 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.685+0000 7fb8f2ce0640 -1 < failed to complete operation key=4K76U69GZ9AE tag=R0WNS4RIEG1X type=del size=523779: (22) Invalid argument 2024-09-13T00:01:03.699 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.697+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~JFIWOFBL3UPL.0 size=1048576 2024-09-13T00:01:03.704 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.701+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~JFIWOFBL3UPL.1 size=743548 2024-09-13T00:01:03.704 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.701+0000 7fb8f2ce0640 1 > prepared multipart upload key=7OX5OJZ8GXU5 upload=JFIWOFBL3UPL tag=SSE104WEE1AB type=write size=2840700 2024-09-13T00:01:03.710 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.709+0000 7fb8f2ce0640 1 < completed delete operation key=QMZREZ1QEDNJ tag=EF1OCG1WYOPQ type=del 2024-09-13T00:01:03.725 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.721+0000 7fb8f2ce0640 1 > prepared part key=_multipart_4K76U69GZ9AE.2~A5SBQ6OXR89A.0 size=1048576 2024-09-13T00:01:03.729 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.729+0000 7fb8f2ce0640 1 > prepared part key=_multipart_4K76U69GZ9AE.2~A5SBQ6OXR89A.1 size=791972 2024-09-13T00:01:03.729 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.729+0000 7fb8f2ce0640 1 > prepared multipart upload key=4K76U69GZ9AE upload=A5SBQ6OXR89A tag=OYY6LQ0IP58K type=write size=2889124 2024-09-13T00:01:03.735 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.733+0000 7fb8f2ce0640 1 < completed write operation key=H4M1TCN5W6XU tag=6N8GLGHJ0BIE type=write size=103486 2024-09-13T00:01:03.741 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.741+0000 7fb8f2ce0640 1 > prepared operation key=0WC6P8YNL0H2 tag=QZTVJ4PL82YX type=del size=1090441 2024-09-13T00:01:03.747 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.745+0000 7fb8f2ce0640 1 < completed part key=_multipart_W8XZ26311RJX.2~3DAVUOXXJ5Z0.0 size=1048576 2024-09-13T00:01:03.747 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.745+0000 7fb8f2ce0640 1 < canceled part key=_multipart_W8XZ26311RJX.2~3DAVUOXXJ5Z0.1 size=1048576 2024-09-13T00:01:03.752 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.749+0000 7fb8f2ce0640 1 < completed part key=_multipart_W8XZ26311RJX.2~3DAVUOXXJ5Z0.2 size=878673 2024-09-13T00:01:03.762 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.761+0000 7fb8f2ce0640 1 < canceled multipart upload key=W8XZ26311RJX upload=3DAVUOXXJ5Z0 tag=WLZHM15N29UZ type=write size=4024401 2024-09-13T00:01:03.768 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.765+0000 7fb8f2ce0640 1 > prepared operation key=FX0NCGLK0KPJ tag=D3YJJPLI11QY type=del size=238134 2024-09-13T00:01:03.774 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.773+0000 7fb8f2ce0640 1 < completed delete operation key=QMZREZ1QEDNJ tag=LVKV26N72ASK type=del 2024-09-13T00:01:03.780 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.777+0000 7fb8f2ce0640 1 > prepared operation key=1PX7DBO9M3YI tag=QZGR4JGCHVD7 type=del size=1609046 2024-09-13T00:01:03.787 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.785+0000 7fb8f2ce0640 1 < canceled operation key=8Z6U7OOA0GBH tag=UAB1124HV7UQ type=del size=1887680 2024-09-13T00:01:03.798 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.797+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZFHN2SNTZ5W7.2~L1Y4BVODXGF3.0 size=261399 2024-09-13T00:01:03.798 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.797+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZFHN2SNTZ5W7 upload=L1Y4BVODXGF3 tag=OOXJW5QH07KQ type=write size=1309975 2024-09-13T00:01:03.804 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.801+0000 7fb8f2ce0640 1 < completed delete operation key=63FJDWBO2T9S tag=HKAIZTT36URF type=del 2024-09-13T00:01:03.816 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.813+0000 7fb8f2ce0640 1 > prepared part key=_multipart_O4MZ4UWNXME9.2~AF4MNSGGIVU0.0 size=1048576 2024-09-13T00:01:03.862 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.817+0000 7fb8f2ce0640 1 > prepared part key=_multipart_O4MZ4UWNXME9.2~AF4MNSGGIVU0.1 size=231201 2024-09-13T00:01:03.862 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.817+0000 7fb8f2ce0640 1 > prepared multipart upload key=O4MZ4UWNXME9 upload=AF4MNSGGIVU0 tag=ZGQXIYZF1NE0 type=write size=2328353 2024-09-13T00:01:03.862 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.825+0000 7fb8f2ce0640 1 < completed write operation key=FX0NCGLK0KPJ tag=A73Y1D15KZ64 type=write size=1028488 2024-09-13T00:01:03.862 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.833+0000 7fb8f2ce0640 1 > prepared operation key=8Z6U7OOA0GBH tag=MFT9NMAZZ99L type=del size=3173071 2024-09-13T00:01:03.863 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.837+0000 7fb8f2ce0640 1 < completed part key=_multipart_0WC6P8YNL0H2.2~19M7MJLQ9AP5.0 size=1048576 2024-09-13T00:01:03.863 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.845+0000 7fb8f2ce0640 1 < completed part key=_multipart_0WC6P8YNL0H2.2~19M7MJLQ9AP5.1 size=624113 2024-09-13T00:01:03.863 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.857+0000 7fb8f2ce0640 1 < completed multipart upload key=0WC6P8YNL0H2 upload=19M7MJLQ9AP5 tag=0MA3IN4A5W60 type=write size=2721265 2024-09-13T00:01:03.863 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.861+0000 7fb8f2ce0640 1 > prepared operation key=63FJDWBO2T9S tag=I02Y9XRROISY type=del size=855669 2024-09-13T00:01:03.868 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.865+0000 7fb8f2ce0640 1 < canceled operation key=W8XZ26311RJX tag=8GXE1MN15VY5 type=del size=392918 2024-09-13T00:01:03.878 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.877+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H4M1TCN5W6XU.2~LUEHOF9ZR0VD.0 size=939889 2024-09-13T00:01:03.878 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.877+0000 7fb8f2ce0640 1 > prepared multipart upload key=H4M1TCN5W6XU upload=LUEHOF9ZR0VD tag=CUY8LGBRQ140 type=write size=1988465 2024-09-13T00:01:03.886 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.885+0000 7fb8f2ce0640 1 < completed write operation key=F0QCVR9E08WE tag=ZDAVWHJW87FC type=write size=988266 2024-09-13T00:01:03.891 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.889+0000 7fb8f2ce0640 1 > prepared operation key=QMZREZ1QEDNJ tag=AW25F9ARUPYE type=del size=847113 2024-09-13T00:01:03.897 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.893+0000 7fb8f2ce0640 1 < completed delete operation key=1HX06CCZJ1DD tag=1B0ENFY8X7OH type=del 2024-09-13T00:01:03.907 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.905+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~V2Y94HK4HVOD.0 size=1048576 2024-09-13T00:01:03.911 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.909+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~V2Y94HK4HVOD.1 size=1048576 2024-09-13T00:01:03.916 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.913+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~V2Y94HK4HVOD.2 size=101672 2024-09-13T00:01:03.916 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.913+0000 7fb8f2ce0640 1 > prepared multipart upload key=1HX06CCZJ1DD upload=V2Y94HK4HVOD tag=P1FGLHIX5GCN type=write size=3247400 2024-09-13T00:01:03.921 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.921+0000 7fb8f2ce0640 1 < completed part key=_multipart_70F7D20HI504.2~GYCS0SRPHCX5.0 size=678454 2024-09-13T00:01:03.931 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.929+0000 7fb8f2ce0640 1 < completed multipart upload key=70F7D20HI504 upload=GYCS0SRPHCX5 tag=EQZR4J0WVMHH type=write size=1727030 2024-09-13T00:01:03.936 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.933+0000 7fb8f2ce0640 1 > prepared operation key=FX0NCGLK0KPJ tag=DCJ9RIK3CXCH type=del size=395408 2024-09-13T00:01:03.941 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.941+0000 7fb8f2ce0640 1 < completed delete operation key=J8NGDSF9XRTP tag=XMP1XET6TW9M type=del 2024-09-13T00:01:03.948 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.945+0000 7fb8f2ce0640 1 > prepared operation key=2AHYRZUDFHP4 tag=OFF9AAH88LMQ type=del size=3920917 2024-09-13T00:01:03.954 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.953+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~CBL3T7CRRPSE.0 size=951566 2024-09-13T00:01:03.964 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.961+0000 7fb8f2ce0640 1 < completed multipart upload key=ZTK9AYBSLCOZ upload=CBL3T7CRRPSE tag=2ZMGHD9GTGH1 type=write size=2000142 2024-09-13T00:01:03.983 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.981+0000 7fb8f2ce0640 1 > prepared part key=_multipart_F0QCVR9E08WE.2~734LP6S2SP6V.0 size=418553 2024-09-13T00:01:03.983 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.981+0000 7fb8f2ce0640 1 > prepared multipart upload key=F0QCVR9E08WE upload=734LP6S2SP6V tag=O6PIMYMRE3LA type=write size=1467129 2024-09-13T00:01:03.989 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.989+0000 7fb8f2ce0640 1 < completed delete operation key=PVYZQLGYT3XU tag=I6VBX1ZQ6OE2 type=del 2024-09-13T00:01:03.995 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.993+0000 7fb8f2ce0640 1 > prepared operation key=1HX06CCZJ1DD tag=OJC1H9B8W9IG type=del size=3864195 2024-09-13T00:01:04.000 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:03.997+0000 7fb8f2ce0640 1 < completed part key=_multipart_7OX5OJZ8GXU5.2~JFIWOFBL3UPL.0 size=1048576 2024-09-13T00:01:04.005 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.005+0000 7fb8f2ce0640 1 < completed part key=_multipart_7OX5OJZ8GXU5.2~JFIWOFBL3UPL.1 size=743548 2024-09-13T00:01:04.015 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.013+0000 7fb8f2ce0640 1 < completed multipart upload key=7OX5OJZ8GXU5 upload=JFIWOFBL3UPL tag=SSE104WEE1AB type=write size=2840700 2024-09-13T00:01:04.026 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.025+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H4M1TCN5W6XU.2~6G8CCQD5Q82R.0 size=383806 2024-09-13T00:01:04.026 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.025+0000 7fb8f2ce0640 1 > prepared multipart upload key=H4M1TCN5W6XU upload=6G8CCQD5Q82R tag=RSMD0WDT65IW type=write size=1432382 2024-09-13T00:01:04.032 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.029+0000 7fb8f2ce0640 1 < completed part key=_multipart_4K76U69GZ9AE.2~A5SBQ6OXR89A.0 size=1048576 2024-09-13T00:01:04.037 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.033+0000 7fb8f2ce0640 1 < completed part key=_multipart_4K76U69GZ9AE.2~A5SBQ6OXR89A.1 size=791972 2024-09-13T00:01:04.046 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.045+0000 7fb8f2ce0640 1 < completed multipart upload key=4K76U69GZ9AE upload=A5SBQ6OXR89A tag=OYY6LQ0IP58K type=write size=2889124 2024-09-13T00:01:04.053 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.053+0000 7fb8f2ce0640 1 > prepared operation key=5GM88ERM89KT tag=9ULM5BVPNGUU type=del size=1915891 2024-09-13T00:01:04.065 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.065+0000 7fb8f2ce0640 1 < suggested update operation key=0WC6P8YNL0H2 tag=QZTVJ4PL82YX type=del size=1090441 2024-09-13T00:01:04.077 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.077+0000 7fb8f2ce0640 1 > prepared part key=_multipart_F0QCVR9E08WE.2~VL026VFJZHP0.0 size=1048576 2024-09-13T00:01:04.082 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.081+0000 7fb8f2ce0640 1 > prepared part key=_multipart_F0QCVR9E08WE.2~VL026VFJZHP0.1 size=1048576 2024-09-13T00:01:04.087 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.085+0000 7fb8f2ce0640 1 > prepared part key=_multipart_F0QCVR9E08WE.2~VL026VFJZHP0.2 size=386779 2024-09-13T00:01:04.087 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.085+0000 7fb8f2ce0640 1 > prepared multipart upload key=F0QCVR9E08WE upload=VL026VFJZHP0 tag=GOXLGMOAGIPY type=write size=3532507 2024-09-13T00:01:04.093 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.093+0000 7fb8f2ce0640 1 < completed delete operation key=FX0NCGLK0KPJ tag=D3YJJPLI11QY type=del 2024-09-13T00:01:04.105 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.105+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~6M2YBPPCDVNA.0 size=690647 2024-09-13T00:01:04.105 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.105+0000 7fb8f2ce0640 1 > prepared multipart upload key=FX0NCGLK0KPJ upload=6M2YBPPCDVNA tag=EXOTZ2NN0U7R type=write size=1739223 2024-09-13T00:01:04.111 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.109+0000 7fb8f2ce0640 1 < completed delete operation key=1PX7DBO9M3YI tag=QZGR4JGCHVD7 type=del 2024-09-13T00:01:04.122 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.121+0000 7fb8f2ce0640 1 > prepared part key=_multipart_QMZREZ1QEDNJ.2~E6AEMWUPLZ1I.0 size=1048576 2024-09-13T00:01:04.127 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.125+0000 7fb8f2ce0640 1 > prepared part key=_multipart_QMZREZ1QEDNJ.2~E6AEMWUPLZ1I.1 size=1048576 2024-09-13T00:01:04.132 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.129+0000 7fb8f2ce0640 1 > prepared part key=_multipart_QMZREZ1QEDNJ.2~E6AEMWUPLZ1I.2 size=13146 2024-09-13T00:01:04.132 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.129+0000 7fb8f2ce0640 1 > prepared multipart upload key=QMZREZ1QEDNJ upload=E6AEMWUPLZ1I tag=WZ2XOXVUN4RY type=write size=3158874 2024-09-13T00:01:04.138 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.137+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZFHN2SNTZ5W7.2~L1Y4BVODXGF3.0 size=261399 2024-09-13T00:01:04.150 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.149+0000 7fb8f2ce0640 1 < completed multipart upload key=ZFHN2SNTZ5W7 upload=L1Y4BVODXGF3 tag=OOXJW5QH07KQ type=write size=1309975 2024-09-13T00:01:04.161 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.161+0000 7fb8f2ce0640 1 > prepared part key=_multipart_5GM88ERM89KT.2~HZDVY73B23EP.0 size=600185 2024-09-13T00:01:04.161 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.161+0000 7fb8f2ce0640 1 > prepared multipart upload key=5GM88ERM89KT upload=HZDVY73B23EP tag=1LF6QN6NJ0JJ type=write size=1648761 2024-09-13T00:01:04.171 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.169+0000 7fb8f2ce0640 1 < completed part key=_multipart_O4MZ4UWNXME9.2~AF4MNSGGIVU0.0 size=1048576 2024-09-13T00:01:04.176 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.173+0000 7fb8f2ce0640 1 < completed part key=_multipart_O4MZ4UWNXME9.2~AF4MNSGGIVU0.1 size=231201 2024-09-13T00:01:04.186 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.185+0000 7fb8f2ce0640 1 < completed multipart upload key=O4MZ4UWNXME9 upload=AF4MNSGGIVU0 tag=ZGQXIYZF1NE0 type=write size=2328353 2024-09-13T00:01:04.197 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.193+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~T9QHP8O63IAA.0 size=1048576 2024-09-13T00:01:04.201 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.201+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~T9QHP8O63IAA.1 size=502234 2024-09-13T00:01:04.201 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.201+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZTK9AYBSLCOZ upload=T9QHP8O63IAA tag=L61340VV9CDC type=write size=2599386 2024-09-13T00:01:04.207 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.205+0000 7fb8f2ce0640 1 < completed delete operation key=8Z6U7OOA0GBH tag=MFT9NMAZZ99L type=del 2024-09-13T00:01:04.213 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.213+0000 7fb8f2ce0640 1 > prepared operation key=F0QCVR9E08WE tag=WUUWRAE8QXG8 type=del size=299410 2024-09-13T00:01:04.219 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.217+0000 7fb8f2ce0640 1 < completed delete operation key=63FJDWBO2T9S tag=I02Y9XRROISY type=del 2024-09-13T00:01:04.231 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.229+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~C1C6FZUFUDVN.0 size=1048576 2024-09-13T00:01:04.235 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.233+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~C1C6FZUFUDVN.1 size=1048576 2024-09-13T00:01:04.240 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.237+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~C1C6FZUFUDVN.2 size=922835 2024-09-13T00:01:04.241 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.237+0000 7fb8f2ce0640 1 > prepared multipart upload key=7OX5OJZ8GXU5 upload=C1C6FZUFUDVN tag=60DTDT7QD0PR type=write size=4068563 2024-09-13T00:01:04.246 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.245+0000 7fb8f2ce0640 1 < completed part key=_multipart_H4M1TCN5W6XU.2~LUEHOF9ZR0VD.0 size=939889 2024-09-13T00:01:04.256 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.253+0000 7fb8f2ce0640 1 < completed multipart upload key=H4M1TCN5W6XU upload=LUEHOF9ZR0VD tag=CUY8LGBRQ140 type=write size=1988465 2024-09-13T00:01:04.262 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.261+0000 7fb8f2ce0640 1 > prepared operation key=W8XZ26311RJX tag=V5D65YZFT5L5 type=del size=1936147 2024-09-13T00:01:04.274 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.273+0000 7fb8f2ce0640 1 < suggested update operation key=QMZREZ1QEDNJ tag=AW25F9ARUPYE type=del size=847113 2024-09-13T00:01:04.286 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.285+0000 7fb8f2ce0640 1 > prepared part key=_multipart_44EDYFT9VF8Y.2~68WF6XUSALC3.0 size=220384 2024-09-13T00:01:04.286 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.285+0000 7fb8f2ce0640 1 > prepared multipart upload key=44EDYFT9VF8Y upload=68WF6XUSALC3 tag=QQ52E4UV6CVP type=write size=1268960 2024-09-13T00:01:04.292 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.289+0000 7fb8f2ce0640 1 < completed part key=_multipart_1HX06CCZJ1DD.2~V2Y94HK4HVOD.0 size=1048576 2024-09-13T00:01:04.297 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.297+0000 7fb8f2ce0640 1 < completed part key=_multipart_1HX06CCZJ1DD.2~V2Y94HK4HVOD.1 size=1048576 2024-09-13T00:01:04.302 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.301+0000 7fb8f2ce0640 1 < completed part key=_multipart_1HX06CCZJ1DD.2~V2Y94HK4HVOD.2 size=101672 2024-09-13T00:01:04.313 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.313+0000 7fb8f2ce0640 1 < completed multipart upload key=1HX06CCZJ1DD upload=V2Y94HK4HVOD tag=P1FGLHIX5GCN type=write size=3247400 2024-09-13T00:01:04.320 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.317+0000 7fb8f2ce0640 1 > prepared operation key=H9AU4HD247YU tag=HHQEOV9DVHXP type=del size=630852 2024-09-13T00:01:04.327 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.325+0000 7fb8f2ce0640 1 < completed delete operation key=FX0NCGLK0KPJ tag=DCJ9RIK3CXCH type=del 2024-09-13T00:01:04.334 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.333+0000 7fb8f2ce0640 1 > prepared operation key=F0QCVR9E08WE tag=GLVH4WVAZ39X type=del size=397129 2024-09-13T00:01:04.346 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.345+0000 7fb8f2ce0640 1 < suggested remove operation key=2AHYRZUDFHP4 tag=OFF9AAH88LMQ type=del 2024-09-13T00:01:04.352 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.349+0000 7fb8f2ce0640 1 > prepared operation key=F0QCVR9E08WE tag=1LPEUF005E82 type=del size=1276949 2024-09-13T00:01:04.359 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.357+0000 7fb8f2ce0640 1 < completed part key=_multipart_F0QCVR9E08WE.2~734LP6S2SP6V.0 size=418553 2024-09-13T00:01:04.369 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.369+0000 7fb8f2ce0640 1 < completed multipart upload key=F0QCVR9E08WE upload=734LP6S2SP6V tag=O6PIMYMRE3LA type=write size=1467129 2024-09-13T00:01:04.375 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.373+0000 7fb8f2ce0640 1 > prepared operation key=FX0NCGLK0KPJ tag=2PPL8D60H3QG type=del size=3956673 2024-09-13T00:01:04.381 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.381+0000 7fb8f2ce0640 1 < completed delete operation key=1HX06CCZJ1DD tag=OJC1H9B8W9IG type=del 2024-09-13T00:01:04.392 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.389+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~E33EYKGGAXVQ.0 size=1048576 2024-09-13T00:01:04.396 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.393+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1PX7DBO9M3YI.2~E33EYKGGAXVQ.1 size=53089 2024-09-13T00:01:04.397 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.393+0000 7fb8f2ce0640 1 > prepared multipart upload key=1PX7DBO9M3YI upload=E33EYKGGAXVQ tag=OMPYFABFAWE6 type=write size=2150241 2024-09-13T00:01:04.402 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.401+0000 7fb8f2ce0640 1 < completed part key=_multipart_H4M1TCN5W6XU.2~6G8CCQD5Q82R.0 size=383806 2024-09-13T00:01:04.413 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.409+0000 7fb8f2ce0640 1 < completed multipart upload key=H4M1TCN5W6XU upload=6G8CCQD5Q82R tag=RSMD0WDT65IW type=write size=1432382 2024-09-13T00:01:04.418 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.417+0000 7fb8f2ce0640 1 > prepared operation key=4K76U69GZ9AE tag=EC5ZU6A4NJ5V type=del size=3465217 2024-09-13T00:01:04.424 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.421+0000 7fb8f2ce0640 1 < completed delete operation key=5GM88ERM89KT tag=9ULM5BVPNGUU type=del 2024-09-13T00:01:04.435 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.433+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~8SRCLODHVPNV.0 size=1048576 2024-09-13T00:01:04.440 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.437+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~8SRCLODHVPNV.1 size=1048576 2024-09-13T00:01:04.445 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.445+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~8SRCLODHVPNV.2 size=59327 2024-09-13T00:01:04.446 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.445+0000 7fb8f2ce0640 1 > prepared multipart upload key=FX0NCGLK0KPJ upload=8SRCLODHVPNV tag=FERZ5ND0TORW type=write size=3205055 2024-09-13T00:01:04.453 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.453+0000 7fb8f2ce0640 1 < completed part key=_multipart_F0QCVR9E08WE.2~VL026VFJZHP0.0 size=1048576 2024-09-13T00:01:04.459 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.457+0000 7fb8f2ce0640 1 < completed part key=_multipart_F0QCVR9E08WE.2~VL026VFJZHP0.1 size=1048576 2024-09-13T00:01:04.463 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.461+0000 7fb8f2ce0640 1 < completed part key=_multipart_F0QCVR9E08WE.2~VL026VFJZHP0.2 size=386779 2024-09-13T00:01:04.474 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.473+0000 7fb8f2ce0640 1 < completed multipart upload key=F0QCVR9E08WE upload=VL026VFJZHP0 tag=GOXLGMOAGIPY type=write size=3532507 2024-09-13T00:01:04.484 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.481+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~10CX1VIP4N7B.0 size=995985 2024-09-13T00:01:04.485 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.481+0000 7fb8f2ce0640 1 > prepared multipart upload key=2AHYRZUDFHP4 upload=10CX1VIP4N7B tag=2LLKBVCSX03Z type=write size=2044561 2024-09-13T00:01:04.491 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.489+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~6M2YBPPCDVNA.0 size=690647 2024-09-13T00:01:04.501 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.501+0000 7fb8f2ce0640 1 < completed multipart upload key=FX0NCGLK0KPJ upload=6M2YBPPCDVNA tag=EXOTZ2NN0U7R type=write size=1739223 2024-09-13T00:01:04.507 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.505+0000 7fb8f2ce0640 1 > prepared operation key=H9AU4HD247YU tag=CXKINFSO7HWU type=write size=1015735 2024-09-13T00:01:04.513 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.513+0000 7fb8f2ce0640 1 < completed part key=_multipart_QMZREZ1QEDNJ.2~E6AEMWUPLZ1I.0 size=1048576 2024-09-13T00:01:04.518 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.517+0000 7fb8f2ce0640 1 < completed part key=_multipart_QMZREZ1QEDNJ.2~E6AEMWUPLZ1I.1 size=1048576 2024-09-13T00:01:04.523 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.521+0000 7fb8f2ce0640 1 < completed part key=_multipart_QMZREZ1QEDNJ.2~E6AEMWUPLZ1I.2 size=13146 2024-09-13T00:01:04.534 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.533+0000 7fb8f2ce0640 1 < completed multipart upload key=QMZREZ1QEDNJ upload=E6AEMWUPLZ1I tag=WZ2XOXVUN4RY type=write size=3158874 2024-09-13T00:01:04.540 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.537+0000 7fb8f2ce0640 1 > prepared operation key=44EDYFT9VF8Y tag=SALZVNL3DUIJ type=del size=3780709 2024-09-13T00:01:04.541 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.537+0000 7fb8f2ce0640 1 < canceled part key=_multipart_5GM88ERM89KT.2~HZDVY73B23EP.0 size=600185 2024-09-13T00:01:04.550 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.549+0000 7fb8f2ce0640 1 < canceled multipart upload key=5GM88ERM89KT upload=HZDVY73B23EP tag=1LF6QN6NJ0JJ type=write size=1648761 2024-09-13T00:01:04.556 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.553+0000 7fb8f2ce0640 1 > prepared operation key=70F7D20HI504 tag=QNLWF1O1LAFZ type=del size=612067 2024-09-13T00:01:04.563 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.561+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~T9QHP8O63IAA.0 size=1048576 2024-09-13T00:01:04.569 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.569+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~T9QHP8O63IAA.1 size=502234 2024-09-13T00:01:04.579 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.577+0000 7fb8f2ce0640 1 < completed multipart upload key=ZTK9AYBSLCOZ upload=T9QHP8O63IAA tag=L61340VV9CDC type=write size=2599386 2024-09-13T00:01:04.585 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.585+0000 7fb8f2ce0640 1 > prepared operation key=EUE3LT0PDIXV tag=LACEMLK8YXXN type=del size=4099362 2024-09-13T00:01:04.592 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.589+0000 7fb8f2ce0640 1 < completed delete operation key=F0QCVR9E08WE tag=WUUWRAE8QXG8 type=del 2024-09-13T00:01:04.602 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.601+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~Y7VE03LFK6N5.0 size=1048576 2024-09-13T00:01:04.607 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.605+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~Y7VE03LFK6N5.1 size=1048576 2024-09-13T00:01:04.612 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.609+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~Y7VE03LFK6N5.2 size=508402 2024-09-13T00:01:04.612 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.609+0000 7fb8f2ce0640 1 > prepared multipart upload key=FX0NCGLK0KPJ upload=Y7VE03LFK6N5 tag=K06G3KRQZAWL type=write size=3654130 2024-09-13T00:01:04.618 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.617+0000 7fb8f2ce0640 1 < completed part key=_multipart_7OX5OJZ8GXU5.2~C1C6FZUFUDVN.0 size=1048576 2024-09-13T00:01:04.623 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.621+0000 7fb8f2ce0640 1 < completed part key=_multipart_7OX5OJZ8GXU5.2~C1C6FZUFUDVN.1 size=1048576 2024-09-13T00:01:04.627 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.625+0000 7fb8f2ce0640 1 < completed part key=_multipart_7OX5OJZ8GXU5.2~C1C6FZUFUDVN.2 size=922835 2024-09-13T00:01:04.637 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.637+0000 7fb8f2ce0640 1 < completed multipart upload key=7OX5OJZ8GXU5 upload=C1C6FZUFUDVN tag=60DTDT7QD0PR type=write size=4068563 2024-09-13T00:01:04.643 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.641+0000 7fb8f2ce0640 1 > prepared operation key=N3Z5XIG3TZRQ tag=48OM933H0Q6B type=del size=2289472 2024-09-13T00:01:04.654 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.653+0000 7fb8f2ce0640 1 < suggested remove operation key=W8XZ26311RJX tag=V5D65YZFT5L5 type=del 2024-09-13T00:01:04.666 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.665+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~VRT0BD3TW31K.0 size=1048576 2024-09-13T00:01:04.670 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.669+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~VRT0BD3TW31K.1 size=1048576 2024-09-13T00:01:04.675 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.673+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~VRT0BD3TW31K.2 size=837371 2024-09-13T00:01:04.675 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.673+0000 7fb8f2ce0640 1 > prepared multipart upload key=63FJDWBO2T9S upload=VRT0BD3TW31K tag=I2LC9XY75ERQ type=write size=3983099 2024-09-13T00:01:04.681 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.681+0000 7fb8f2ce0640 1 < completed part key=_multipart_44EDYFT9VF8Y.2~68WF6XUSALC3.0 size=220384 2024-09-13T00:01:04.692 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.689+0000 7fb8f2ce0640 1 < completed multipart upload key=44EDYFT9VF8Y upload=68WF6XUSALC3 tag=QQ52E4UV6CVP type=write size=1268960 2024-09-13T00:01:04.702 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.701+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~G3KQJDCAIN56.0 size=457893 2024-09-13T00:01:04.703 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.701+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZTK9AYBSLCOZ upload=G3KQJDCAIN56 tag=CL10NPTFUC5D type=write size=1506469 2024-09-13T00:01:04.709 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.709+0000 7fb8f2ce0640 1 < completed delete operation key=H9AU4HD247YU tag=HHQEOV9DVHXP type=del 2024-09-13T00:01:04.715 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.713+0000 7fb8f2ce0640 1 > prepared operation key=W8XZ26311RJX tag=ZQ5ZZ4VWY79Y type=del size=1490848 2024-09-13T00:01:04.721 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.717+0000 7fb8f2ce0640 1 < completed delete operation key=F0QCVR9E08WE tag=GLVH4WVAZ39X type=del 2024-09-13T00:01:04.732 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.729+0000 7fb8f2ce0640 1 > prepared part key=_multipart_2AHYRZUDFHP4.2~R6B1Z5H48D8L.0 size=93298 2024-09-13T00:01:04.732 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.729+0000 7fb8f2ce0640 1 > prepared multipart upload key=2AHYRZUDFHP4 upload=R6B1Z5H48D8L tag=VOWMBIY8153J type=write size=1141874 2024-09-13T00:01:04.738 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.737+0000 7fb8f2ce0640 1 < completed delete operation key=F0QCVR9E08WE tag=1LPEUF005E82 type=del 2024-09-13T00:01:04.744 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.741+0000 7fb8f2ce0640 1 > prepared operation key=2AHYRZUDFHP4 tag=ACWS0N30AQG5 type=del size=1381475 2024-09-13T00:01:04.750 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.749+0000 7fb8f2ce0640 1 < completed delete operation key=FX0NCGLK0KPJ tag=2PPL8D60H3QG type=del 2024-09-13T00:01:04.763 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.761+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~UWLA7UVC7RGY.0 size=1048576 2024-09-13T00:01:04.767 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.765+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~UWLA7UVC7RGY.1 size=1048576 2024-09-13T00:01:04.772 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.769+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~UWLA7UVC7RGY.2 size=903834 2024-09-13T00:01:04.772 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.769+0000 7fb8f2ce0640 1 > prepared multipart upload key=NQJQC8CEAC15 upload=UWLA7UVC7RGY tag=KUHZP0MW5Y1O type=write size=4049562 2024-09-13T00:01:04.778 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.777+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~E33EYKGGAXVQ.0 size=1048576 2024-09-13T00:01:04.784 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.781+0000 7fb8f2ce0640 1 < completed part key=_multipart_1PX7DBO9M3YI.2~E33EYKGGAXVQ.1 size=53089 2024-09-13T00:01:04.795 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.793+0000 7fb8f2ce0640 1 < completed multipart upload key=1PX7DBO9M3YI upload=E33EYKGGAXVQ tag=OMPYFABFAWE6 type=write size=2150241 2024-09-13T00:01:04.805 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.805+0000 7fb8f2ce0640 1 > prepared part key=_multipart_0WC6P8YNL0H2.2~EQ85DMDFTPZR.0 size=1048576 2024-09-13T00:01:04.810 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.809+0000 7fb8f2ce0640 1 > prepared part key=_multipart_0WC6P8YNL0H2.2~EQ85DMDFTPZR.1 size=824872 2024-09-13T00:01:04.810 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.809+0000 7fb8f2ce0640 1 > prepared multipart upload key=0WC6P8YNL0H2 upload=EQ85DMDFTPZR tag=RJ26AF2CTAR6 type=write size=2922024 2024-09-13T00:01:04.816 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.813+0000 7fb8f2ce0640 1 < completed delete operation key=4K76U69GZ9AE tag=EC5ZU6A4NJ5V type=del 2024-09-13T00:01:04.822 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.821+0000 7fb8f2ce0640 1 > prepared operation key=63FJDWBO2T9S tag=8T7PTT50BSY3 type=del size=630322 2024-09-13T00:01:04.828 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.825+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~8SRCLODHVPNV.0 size=1048576 2024-09-13T00:01:04.828 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.825+0000 7fb8f2ce0640 1 < canceled part key=_multipart_FX0NCGLK0KPJ.2~8SRCLODHVPNV.1 size=1048576 2024-09-13T00:01:04.833 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.833+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~8SRCLODHVPNV.2 size=59327 2024-09-13T00:01:04.858 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.857+0000 7fb8f2ce0640 1 < canceled multipart upload key=FX0NCGLK0KPJ upload=8SRCLODHVPNV tag=FERZ5ND0TORW type=write size=3205055 2024-09-13T00:01:04.863 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.861+0000 7fb8f2ce0640 1 > prepared operation key=H4M1TCN5W6XU tag=BM298ZOQKHS5 type=del size=3887916 2024-09-13T00:01:04.869 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.869+0000 7fb8f2ce0640 1 < completed part key=_multipart_2AHYRZUDFHP4.2~10CX1VIP4N7B.0 size=995985 2024-09-13T00:01:04.879 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.877+0000 7fb8f2ce0640 1 < completed multipart upload key=2AHYRZUDFHP4 upload=10CX1VIP4N7B tag=2LLKBVCSX03Z type=write size=2044561 2024-09-13T00:01:04.891 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.889+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~RSY7JTMD1WR3.0 size=1048576 2024-09-13T00:01:04.896 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.893+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~RSY7JTMD1WR3.1 size=1048576 2024-09-13T00:01:04.901 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.901+0000 7fb8f2ce0640 1 > prepared part key=_multipart_7OX5OJZ8GXU5.2~RSY7JTMD1WR3.2 size=1023191 2024-09-13T00:01:04.901 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.901+0000 7fb8f2ce0640 1 > prepared multipart upload key=7OX5OJZ8GXU5 upload=RSY7JTMD1WR3 tag=ICIU1W3ATLIS type=write size=4168919 2024-09-13T00:01:04.907 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.905+0000 7fb8f2ce0640 1 < completed write operation key=H9AU4HD247YU tag=CXKINFSO7HWU type=write size=1015735 2024-09-13T00:01:04.913 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.909+0000 7fb8f2ce0640 1 > prepared operation key=4K76U69GZ9AE tag=PTU7KK81RYZW type=write size=726485 2024-09-13T00:01:04.919 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.917+0000 7fb8f2ce0640 1 < completed delete operation key=44EDYFT9VF8Y tag=SALZVNL3DUIJ type=del 2024-09-13T00:01:04.930 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.929+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~33IIK0JOCPLK.0 size=1048576 2024-09-13T00:01:04.935 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.933+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~33IIK0JOCPLK.1 size=231289 2024-09-13T00:01:04.935 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.933+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZTK9AYBSLCOZ upload=33IIK0JOCPLK tag=WPYW0B8FOZKE type=write size=2328441 2024-09-13T00:01:04.941 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.941+0000 7fb8f2ce0640 1 < canceled operation key=70F7D20HI504 tag=QNLWF1O1LAFZ type=del size=612067 2024-09-13T00:01:04.954 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.953+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~B15NGZEOMHQO.0 size=1048576 2024-09-13T00:01:04.962 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.961+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~B15NGZEOMHQO.1 size=857166 2024-09-13T00:01:04.962 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.961+0000 7fb8f2ce0640 1 > prepared multipart upload key=PVYZQLGYT3XU upload=B15NGZEOMHQO tag=CQ03PB1H9A8N type=write size=2954318 2024-09-13T00:01:04.974 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.973+0000 7fb8f2ce0640 1 < suggested update operation key=EUE3LT0PDIXV tag=LACEMLK8YXXN type=del size=4099362 2024-09-13T00:01:04.980 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.977+0000 7fb8f2ce0640 1 > prepared operation key=W8XZ26311RJX tag=AGX2YQ19U3SN type=write size=236015 2024-09-13T00:01:04.986 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.985+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~Y7VE03LFK6N5.0 size=1048576 2024-09-13T00:01:04.986 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.985+0000 7fb8f2ce0640 1 < canceled part key=_multipart_FX0NCGLK0KPJ.2~Y7VE03LFK6N5.1 size=1048576 2024-09-13T00:01:04.986 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.985+0000 7fb8f2ce0640 1 < canceled part key=_multipart_FX0NCGLK0KPJ.2~Y7VE03LFK6N5.2 size=508402 2024-09-13T00:01:04.997 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:04.997+0000 7fb8f2ce0640 1 < canceled multipart upload key=FX0NCGLK0KPJ upload=Y7VE03LFK6N5 tag=K06G3KRQZAWL type=write size=3654130 2024-09-13T00:01:05.010 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.009+0000 7fb8f2ce0640 1 > prepared part key=_multipart_4K76U69GZ9AE.2~A80PBFSFID2F.0 size=1048576 2024-09-13T00:01:05.015 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.013+0000 7fb8f2ce0640 1 > prepared part key=_multipart_4K76U69GZ9AE.2~A80PBFSFID2F.1 size=1048576 2024-09-13T00:01:05.020 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.017+0000 7fb8f2ce0640 1 > prepared part key=_multipart_4K76U69GZ9AE.2~A80PBFSFID2F.2 size=747655 2024-09-13T00:01:05.020 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.017+0000 7fb8f2ce0640 1 > prepared multipart upload key=4K76U69GZ9AE upload=A80PBFSFID2F tag=VCY6CX34UQZB type=write size=3893383 2024-09-13T00:01:05.027 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.025+0000 7fb8f2ce0640 1 < completed delete operation key=N3Z5XIG3TZRQ tag=48OM933H0Q6B type=del 2024-09-13T00:01:05.032 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.029+0000 7fb8f2ce0640 1 > prepared operation key=W8XZ26311RJX tag=1XH7RVGLVZ0V type=del size=3663588 2024-09-13T00:01:05.039 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.037+0000 7fb8f2ce0640 1 < completed part key=_multipart_63FJDWBO2T9S.2~VRT0BD3TW31K.0 size=1048576 2024-09-13T00:01:05.043 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.041+0000 7fb8f2ce0640 1 < completed part key=_multipart_63FJDWBO2T9S.2~VRT0BD3TW31K.1 size=1048576 2024-09-13T00:01:05.044 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.041+0000 7fb8f2ce0640 1 < canceled part key=_multipart_63FJDWBO2T9S.2~VRT0BD3TW31K.2 size=837371 2024-09-13T00:01:05.053 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.053+0000 7fb8f2ce0640 1 < canceled multipart upload key=63FJDWBO2T9S upload=VRT0BD3TW31K tag=I2LC9XY75ERQ type=write size=3983099 2024-09-13T00:01:05.064 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.061+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~5TXIMK8RZ1J9.0 size=1048576 2024-09-13T00:01:05.069 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.069+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~5TXIMK8RZ1J9.1 size=1048576 2024-09-13T00:01:05.074 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.073+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~5TXIMK8RZ1J9.2 size=641784 2024-09-13T00:01:05.074 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.073+0000 7fb8f2ce0640 1 > prepared multipart upload key=NQJQC8CEAC15 upload=5TXIMK8RZ1J9 tag=8R5BR5WDXJ2J type=write size=3787512 2024-09-13T00:01:05.079 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.077+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~G3KQJDCAIN56.0 size=457893 2024-09-13T00:01:05.090 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.089+0000 7fb8f2ce0640 1 < completed multipart upload key=ZTK9AYBSLCOZ upload=G3KQJDCAIN56 tag=CL10NPTFUC5D type=write size=1506469 2024-09-13T00:01:05.096 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.093+0000 7fb8f2ce0640 1 > prepared operation key=NX26SI4F2ZMB tag=67781QLF8IVB type=write size=906461 2024-09-13T00:01:05.108 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.105+0000 7fb8f2ce0640 1 < suggested update operation key=W8XZ26311RJX tag=ZQ5ZZ4VWY79Y type=del size=1490848 2024-09-13T00:01:05.114 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.113+0000 7fb8f2ce0640 1 > prepared operation key=H9AU4HD247YU tag=GW7ZJFORT4Z7 type=write size=183466 2024-09-13T00:01:05.120 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.117+0000 7fb8f2ce0640 1 < completed part key=_multipart_2AHYRZUDFHP4.2~R6B1Z5H48D8L.0 size=93298 2024-09-13T00:01:05.130 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.129+0000 7fb8f2ce0640 1 < completed multipart upload key=2AHYRZUDFHP4 upload=R6B1Z5H48D8L tag=VOWMBIY8153J type=write size=1141874 2024-09-13T00:01:05.138 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.137+0000 7fb8f2ce0640 1 > prepared operation key=8MPUVD37EDN9 tag=VBYZ53XNNE89 type=del size=1671370 2024-09-13T00:01:05.151 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.149+0000 7fb8f2ce0640 1 < suggested remove operation key=2AHYRZUDFHP4 tag=ACWS0N30AQG5 type=del 2024-09-13T00:01:05.157 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.157+0000 7fb8f2ce0640 1 > prepared operation key=NQJQC8CEAC15 tag=ZVVFGUUFHPO6 type=del size=1067827 2024-09-13T00:01:05.163 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.161+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~UWLA7UVC7RGY.0 size=1048576 2024-09-13T00:01:05.169 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.169+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~UWLA7UVC7RGY.1 size=1048576 2024-09-13T00:01:05.169 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.169+0000 7fb8f2ce0640 1 < canceled part key=_multipart_NQJQC8CEAC15.2~UWLA7UVC7RGY.2 size=903834 2024-09-13T00:01:05.179 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.177+0000 7fb8f2ce0640 1 < canceled multipart upload key=NQJQC8CEAC15 upload=UWLA7UVC7RGY tag=KUHZP0MW5Y1O type=write size=4049562 2024-09-13T00:01:05.185 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.185+0000 7fb8f2ce0640 1 > prepared operation key=PVYZQLGYT3XU tag=28VJFQZVJST9 type=del size=1902233 2024-09-13T00:01:05.193 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.189+0000 7fb8f2ce0640 1 < completed part key=_multipart_0WC6P8YNL0H2.2~EQ85DMDFTPZR.0 size=1048576 2024-09-13T00:01:05.197 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.197+0000 7fb8f2ce0640 1 < completed part key=_multipart_0WC6P8YNL0H2.2~EQ85DMDFTPZR.1 size=824872 2024-09-13T00:01:05.208 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.205+0000 7fb8f2ce0640 1 < completed multipart upload key=0WC6P8YNL0H2 upload=EQ85DMDFTPZR tag=RJ26AF2CTAR6 type=write size=2922024 2024-09-13T00:01:05.214 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.213+0000 7fb8f2ce0640 1 > prepared operation key=FX0NCGLK0KPJ tag=9NF1M1IH6WLB type=del size=1786728 2024-09-13T00:01:05.220 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.217+0000 7fb8f2ce0640 1 < completed delete operation key=63FJDWBO2T9S tag=8T7PTT50BSY3 type=del 2024-09-13T00:01:05.226 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.225+0000 7fb8f2ce0640 1 > prepared operation key=ZTK9AYBSLCOZ tag=0QDO6055WZ3L type=del size=332504 2024-09-13T00:01:05.233 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.233+0000 7fb8f2ce0640 1 < completed delete operation key=H4M1TCN5W6XU tag=BM298ZOQKHS5 type=del 2024-09-13T00:01:05.239 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.237+0000 7fb8f2ce0640 1 > prepared operation key=ZTK9AYBSLCOZ tag=43EM147NYR9O type=del size=3584261 2024-09-13T00:01:05.245 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.245+0000 7fb8f2ce0640 1 < completed part key=_multipart_7OX5OJZ8GXU5.2~RSY7JTMD1WR3.0 size=1048576 2024-09-13T00:01:05.250 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.249+0000 7fb8f2ce0640 1 < completed part key=_multipart_7OX5OJZ8GXU5.2~RSY7JTMD1WR3.1 size=1048576 2024-09-13T00:01:05.255 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.253+0000 7fb8f2ce0640 1 < completed part key=_multipart_7OX5OJZ8GXU5.2~RSY7JTMD1WR3.2 size=1023191 2024-09-13T00:01:05.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:01:04 smithi136 bash[75164]: cluster 2024-09-13T00:01:03.142521+0000 mgr.y (mgr.24661) 718 : cluster [DBG] pgmap v1394: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 338 KiB/s rd, 117 KiB/s wr, 599 op/s 2024-09-13T00:01:05.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:01:04 smithi136 bash[76371]: cluster 2024-09-13T00:01:03.142521+0000 mgr.y (mgr.24661) 718 : cluster [DBG] pgmap v1394: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 338 KiB/s rd, 117 KiB/s wr, 599 op/s 2024-09-13T00:01:05.266 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.265+0000 7fb8f2ce0640 1 < completed multipart upload key=7OX5OJZ8GXU5 upload=RSY7JTMD1WR3 tag=ICIU1W3ATLIS type=write size=4168919 2024-09-13T00:01:05.272 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.269+0000 7fb8f2ce0640 1 > prepared operation key=0WC6P8YNL0H2 tag=KERY6LSE4NO5 type=del size=2293751 2024-09-13T00:01:05.277 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.277+0000 7fb8f2ce0640 1 < completed write operation key=4K76U69GZ9AE tag=PTU7KK81RYZW type=write size=726485 2024-09-13T00:01:05.284 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.281+0000 7fb8f2ce0640 1 > prepared operation key=H4M1TCN5W6XU tag=MCGSAF9JNHG1 type=del size=3646115 2024-09-13T00:01:05.290 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.289+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~33IIK0JOCPLK.0 size=1048576 2024-09-13T00:01:05.295 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.293+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~33IIK0JOCPLK.1 size=231289 2024-09-13T00:01:05.305 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.305+0000 7fb8f2ce0640 1 < completed multipart upload key=ZTK9AYBSLCOZ upload=33IIK0JOCPLK tag=WPYW0B8FOZKE type=write size=2328441 2024-09-13T00:01:05.316 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.313+0000 7fb8f2ce0640 1 > prepared part key=_multipart_F0QCVR9E08WE.2~296MFV64FDVH.0 size=1048576 2024-09-13T00:01:05.321 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.321+0000 7fb8f2ce0640 1 > prepared part key=_multipart_F0QCVR9E08WE.2~296MFV64FDVH.1 size=513090 2024-09-13T00:01:05.321 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.321+0000 7fb8f2ce0640 1 > prepared multipart upload key=F0QCVR9E08WE upload=296MFV64FDVH tag=L3QKHMO41MWO type=write size=2610242 2024-09-13T00:01:05.322 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.321+0000 7fb8f2ce0640 1 < canceled part key=_multipart_PVYZQLGYT3XU.2~B15NGZEOMHQO.0 size=1048576 2024-09-13T00:01:05.327 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.325+0000 7fb8f2ce0640 1 < completed part key=_multipart_PVYZQLGYT3XU.2~B15NGZEOMHQO.1 size=857166 2024-09-13T00:01:05.337 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.337+0000 7fb8f2ce0640 1 < canceled multipart upload key=PVYZQLGYT3XU upload=B15NGZEOMHQO tag=CQ03PB1H9A8N type=write size=2954318 2024-09-13T00:01:05.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:01:04 smithi181 bash[67587]: cluster 2024-09-13T00:01:03.142521+0000 mgr.y (mgr.24661) 718 : cluster [DBG] pgmap v1394: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 338 KiB/s rd, 117 KiB/s wr, 599 op/s 2024-09-13T00:01:05.348 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.345+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EJN6HR8I7XQF.2~A5TXLVAZAPTK.0 size=1048576 2024-09-13T00:01:05.353 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.353+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EJN6HR8I7XQF.2~A5TXLVAZAPTK.1 size=1048576 2024-09-13T00:01:05.358 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.357+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EJN6HR8I7XQF.2~A5TXLVAZAPTK.2 size=767803 2024-09-13T00:01:05.359 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.357+0000 7fb8f2ce0640 1 > prepared multipart upload key=EJN6HR8I7XQF upload=A5TXLVAZAPTK tag=D4VP7UPSRKHF type=write size=3913531 2024-09-13T00:01:05.370 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.369+0000 7fb8f2ce0640 1 < suggested remove operation key=W8XZ26311RJX tag=AGX2YQ19U3SN type=write 2024-09-13T00:01:05.382 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.381+0000 7fb8f2ce0640 1 > prepared part key=_multipart_5GM88ERM89KT.2~HW5KIK0AKNGM.0 size=708460 2024-09-13T00:01:05.382 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.381+0000 7fb8f2ce0640 1 > prepared multipart upload key=5GM88ERM89KT upload=HW5KIK0AKNGM tag=8LKJGK0R3N4S type=write size=1757036 2024-09-13T00:01:05.388 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.385+0000 7fb8f2ce0640 1 < completed part key=_multipart_4K76U69GZ9AE.2~A80PBFSFID2F.0 size=1048576 2024-09-13T00:01:05.393 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.393+0000 7fb8f2ce0640 1 < completed part key=_multipart_4K76U69GZ9AE.2~A80PBFSFID2F.1 size=1048576 2024-09-13T00:01:05.397 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.397+0000 7fb8f2ce0640 1 < completed part key=_multipart_4K76U69GZ9AE.2~A80PBFSFID2F.2 size=747655 2024-09-13T00:01:05.409 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.409+0000 7fb8f2ce0640 1 < completed multipart upload key=4K76U69GZ9AE upload=A80PBFSFID2F tag=VCY6CX34UQZB type=write size=3893383 2024-09-13T00:01:05.420 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.417+0000 7fb8f2ce0640 1 > prepared part key=_multipart_4K76U69GZ9AE.2~68RAICBPD2CG.0 size=937281 2024-09-13T00:01:05.420 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.417+0000 7fb8f2ce0640 1 > prepared multipart upload key=4K76U69GZ9AE upload=68RAICBPD2CG tag=RG4X1LLL2CHH type=write size=1985857 2024-09-13T00:01:05.429 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.425+0000 7fb8f2ce0640 -1 < failed to complete operation key=W8XZ26311RJX tag=1XH7RVGLVZ0V type=del size=3663588: (22) Invalid argument 2024-09-13T00:01:05.440 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.437+0000 7fb8f2ce0640 1 > prepared part key=_multipart_85YEW8N43FTA.2~68U369W69J8R.0 size=1048576 2024-09-13T00:01:05.445 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.445+0000 7fb8f2ce0640 1 > prepared part key=_multipart_85YEW8N43FTA.2~68U369W69J8R.1 size=547946 2024-09-13T00:01:05.446 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.445+0000 7fb8f2ce0640 1 > prepared multipart upload key=85YEW8N43FTA upload=68U369W69J8R tag=ER1FHAVT190H type=write size=2645098 2024-09-13T00:01:05.451 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.449+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~5TXIMK8RZ1J9.0 size=1048576 2024-09-13T00:01:05.457 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.453+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~5TXIMK8RZ1J9.1 size=1048576 2024-09-13T00:01:05.457 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.453+0000 7fb8f2ce0640 1 < canceled part key=_multipart_NQJQC8CEAC15.2~5TXIMK8RZ1J9.2 size=641784 2024-09-13T00:01:05.467 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.465+0000 7fb8f2ce0640 1 < canceled multipart upload key=NQJQC8CEAC15 upload=5TXIMK8RZ1J9 tag=8R5BR5WDXJ2J type=write size=3787512 2024-09-13T00:01:05.474 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.473+0000 7fb8f2ce0640 1 > prepared operation key=4K76U69GZ9AE tag=0XPK6LBWX42C type=del size=2894773 2024-09-13T00:01:05.481 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.477+0000 7fb8f2ce0640 1 < completed write operation key=NX26SI4F2ZMB tag=67781QLF8IVB type=write size=906461 2024-09-13T00:01:05.488 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.485+0000 7fb8f2ce0640 1 > prepared operation key=63FJDWBO2T9S tag=SRID0WP1FMWY type=del size=1489091 2024-09-13T00:01:05.500 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.497+0000 7fb8f2ce0640 1 < suggested remove operation key=H9AU4HD247YU tag=GW7ZJFORT4Z7 type=write 2024-09-13T00:01:05.508 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.505+0000 7fb8f2ce0640 1 > prepared operation key=70F7D20HI504 tag=D6O69F3N7XLC type=del size=3582526 2024-09-13T00:01:05.514 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.513+0000 7fb8f2ce0640 1 < completed delete operation key=8MPUVD37EDN9 tag=VBYZ53XNNE89 type=del 2024-09-13T00:01:05.521 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.521+0000 7fb8f2ce0640 1 > prepared operation key=44EDYFT9VF8Y tag=0IN7YLWW6CX0 type=write size=70767 2024-09-13T00:01:05.528 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.525+0000 7fb8f2ce0640 1 < canceled operation key=NQJQC8CEAC15 tag=ZVVFGUUFHPO6 type=del size=1067827 2024-09-13T00:01:05.535 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.533+0000 7fb8f2ce0640 1 > prepared operation key=8MPUVD37EDN9 tag=BD26ZYU5A3TJ type=write size=420644 2024-09-13T00:01:05.548 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.545+0000 7fb8f2ce0640 1 < suggested remove operation key=PVYZQLGYT3XU tag=28VJFQZVJST9 type=del 2024-09-13T00:01:05.559 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.557+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~C3LRJ5ODN7KJ.0 size=987080 2024-09-13T00:01:05.559 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.557+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZTK9AYBSLCOZ upload=C3LRJ5ODN7KJ tag=5BRGRC1OV83A type=write size=2035656 2024-09-13T00:01:05.571 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.569+0000 7fb8f2ce0640 1 < suggested update operation key=FX0NCGLK0KPJ tag=9NF1M1IH6WLB type=del size=1786728 2024-09-13T00:01:05.576 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.573+0000 7fb8f2ce0640 1 > prepared operation key=P8OYS5ZE1NX0 tag=52P09CY1FCLK type=del size=2294312 2024-09-13T00:01:05.583 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.581+0000 7fb8f2ce0640 1 < completed delete operation key=ZTK9AYBSLCOZ tag=0QDO6055WZ3L type=del 2024-09-13T00:01:05.588 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.585+0000 7fb8f2ce0640 1 > prepared operation key=H4M1TCN5W6XU tag=MAQUZ7T91XEX type=del size=1438744 2024-09-13T00:01:05.595 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.593+0000 7fb8f2ce0640 1 < completed delete operation key=ZTK9AYBSLCOZ tag=43EM147NYR9O type=del 2024-09-13T00:01:05.600 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.597+0000 7fb8f2ce0640 1 > prepared operation key=W8XZ26311RJX tag=S64G4SNKIRY1 type=del size=2647439 2024-09-13T00:01:05.608 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.605+0000 7fb8f2ce0640 1 < completed delete operation key=0WC6P8YNL0H2 tag=KERY6LSE4NO5 type=del 2024-09-13T00:01:05.620 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.617+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~E4NPMFY4MMNL.0 size=414501 2024-09-13T00:01:05.620 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.617+0000 7fb8f2ce0640 1 > prepared multipart upload key=PVYZQLGYT3XU upload=E4NPMFY4MMNL tag=35V2Y2X070BE type=write size=1463077 2024-09-13T00:01:05.632 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.629+0000 7fb8f2ce0640 1 < suggested remove operation key=H4M1TCN5W6XU tag=MCGSAF9JNHG1 type=del 2024-09-13T00:01:05.638 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.637+0000 7fb8f2ce0640 1 > prepared operation key=O4MZ4UWNXME9 tag=X65818RYD83W type=del size=1464272 2024-09-13T00:01:05.639 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.637+0000 7fb8f2ce0640 1 < canceled part key=_multipart_F0QCVR9E08WE.2~296MFV64FDVH.0 size=1048576 2024-09-13T00:01:05.639 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.637+0000 7fb8f2ce0640 1 < canceled part key=_multipart_F0QCVR9E08WE.2~296MFV64FDVH.1 size=513090 2024-09-13T00:01:05.649 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.649+0000 7fb8f2ce0640 1 < canceled multipart upload key=F0QCVR9E08WE upload=296MFV64FDVH tag=L3QKHMO41MWO type=write size=2610242 2024-09-13T00:01:05.655 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.653+0000 7fb8f2ce0640 1 > prepared operation key=ZFHN2SNTZ5W7 tag=KLJHGYX5461A type=del size=2268831 2024-09-13T00:01:05.660 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.657+0000 7fb8f2ce0640 1 < completed part key=_multipart_EJN6HR8I7XQF.2~A5TXLVAZAPTK.0 size=1048576 2024-09-13T00:01:05.665 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.665+0000 7fb8f2ce0640 1 < completed part key=_multipart_EJN6HR8I7XQF.2~A5TXLVAZAPTK.1 size=1048576 2024-09-13T00:01:05.670 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.669+0000 7fb8f2ce0640 1 < completed part key=_multipart_EJN6HR8I7XQF.2~A5TXLVAZAPTK.2 size=767803 2024-09-13T00:01:05.679 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.677+0000 7fb8f2ce0640 1 < completed multipart upload key=EJN6HR8I7XQF upload=A5TXLVAZAPTK tag=D4VP7UPSRKHF type=write size=3913531 2024-09-13T00:01:05.690 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.689+0000 7fb8f2ce0640 1 > prepared part key=_multipart_1HX06CCZJ1DD.2~DNFD7QTVZH3R.0 size=881750 2024-09-13T00:01:05.690 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.689+0000 7fb8f2ce0640 1 > prepared multipart upload key=1HX06CCZJ1DD upload=DNFD7QTVZH3R tag=NZZRVR0UDBJ0 type=write size=1930326 2024-09-13T00:01:05.696 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.693+0000 7fb8f2ce0640 1 < completed part key=_multipart_5GM88ERM89KT.2~HW5KIK0AKNGM.0 size=708460 2024-09-13T00:01:05.706 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.705+0000 7fb8f2ce0640 1 < completed multipart upload key=5GM88ERM89KT upload=HW5KIK0AKNGM tag=8LKJGK0R3N4S type=write size=1757036 2024-09-13T00:01:05.717 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.717+0000 7fb8f2ce0640 1 > prepared part key=_multipart_5GM88ERM89KT.2~RQ0O771LW3PM.0 size=1048576 2024-09-13T00:01:05.722 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.721+0000 7fb8f2ce0640 1 > prepared part key=_multipart_5GM88ERM89KT.2~RQ0O771LW3PM.1 size=258684 2024-09-13T00:01:05.722 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.721+0000 7fb8f2ce0640 1 > prepared multipart upload key=5GM88ERM89KT upload=RQ0O771LW3PM tag=7HG21KGRCPDO type=write size=2355836 2024-09-13T00:01:05.728 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.725+0000 7fb8f2ce0640 1 < completed part key=_multipart_4K76U69GZ9AE.2~68RAICBPD2CG.0 size=937281 2024-09-13T00:01:05.738 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.737+0000 7fb8f2ce0640 1 < completed multipart upload key=4K76U69GZ9AE upload=68RAICBPD2CG tag=RG4X1LLL2CHH type=write size=1985857 2024-09-13T00:01:05.744 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.741+0000 7fb8f2ce0640 1 > prepared operation key=NQJQC8CEAC15 tag=BIZUXO3538S3 type=del size=1252673 2024-09-13T00:01:05.750 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.749+0000 7fb8f2ce0640 1 < completed part key=_multipart_85YEW8N43FTA.2~68U369W69J8R.0 size=1048576 2024-09-13T00:01:05.755 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.753+0000 7fb8f2ce0640 1 < completed part key=_multipart_85YEW8N43FTA.2~68U369W69J8R.1 size=547946 2024-09-13T00:01:05.766 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.765+0000 7fb8f2ce0640 1 < completed multipart upload key=85YEW8N43FTA upload=68U369W69J8R tag=ER1FHAVT190H type=write size=2645098 2024-09-13T00:01:05.772 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.769+0000 7fb8f2ce0640 1 > prepared operation key=N3Z5XIG3TZRQ tag=XM6DYMO65OPF type=del size=3663327 2024-09-13T00:01:05.777 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.777+0000 7fb8f2ce0640 1 < completed delete operation key=4K76U69GZ9AE tag=0XPK6LBWX42C type=del 2024-09-13T00:01:05.783 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.781+0000 7fb8f2ce0640 1 > prepared operation key=63FJDWBO2T9S tag=LQDFT77FZGTF type=write size=316032 2024-09-13T00:01:05.791 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.789+0000 7fb8f2ce0640 1 < completed delete operation key=63FJDWBO2T9S tag=SRID0WP1FMWY type=del 2024-09-13T00:01:05.797 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.797+0000 7fb8f2ce0640 1 > prepared operation key=N3Z5XIG3TZRQ tag=171M97BEUZGX type=del size=3853033 2024-09-13T00:01:05.803 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.801+0000 7fb8f2ce0640 1 < completed delete operation key=70F7D20HI504 tag=D6O69F3N7XLC type=del 2024-09-13T00:01:05.814 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.813+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~NYCH3ISKP2NM.0 size=1048576 2024-09-13T00:01:05.818 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.817+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~NYCH3ISKP2NM.1 size=1048576 2024-09-13T00:01:05.824 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.821+0000 7fb8f2ce0640 1 > prepared part key=_multipart_8MPUVD37EDN9.2~NYCH3ISKP2NM.2 size=984103 2024-09-13T00:01:05.824 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.821+0000 7fb8f2ce0640 1 > prepared multipart upload key=8MPUVD37EDN9 upload=NYCH3ISKP2NM tag=WBKJW22NKW4U type=write size=4129831 2024-09-13T00:01:05.830 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.829+0000 7fb8f2ce0640 1 < completed write operation key=44EDYFT9VF8Y tag=0IN7YLWW6CX0 type=write size=70767 2024-09-13T00:01:05.841 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.841+0000 7fb8f2ce0640 1 > prepared part key=_multipart_J8NGDSF9XRTP.2~JP3EQJGXSQ0G.0 size=1048576 2024-09-13T00:01:05.847 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.845+0000 7fb8f2ce0640 1 > prepared part key=_multipart_J8NGDSF9XRTP.2~JP3EQJGXSQ0G.1 size=88417 2024-09-13T00:01:05.847 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.845+0000 7fb8f2ce0640 1 > prepared multipart upload key=J8NGDSF9XRTP upload=JP3EQJGXSQ0G tag=7DCTX0HMPTXF type=write size=2185569 2024-09-13T00:01:05.853 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.853+0000 7fb8f2ce0640 1 < completed write operation key=8MPUVD37EDN9 tag=BD26ZYU5A3TJ type=write size=420644 2024-09-13T00:01:05.860 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.857+0000 7fb8f2ce0640 1 > prepared operation key=W8XZ26311RJX tag=XTMCKYOBFWEP type=del size=221030 2024-09-13T00:01:05.866 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.865+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~C3LRJ5ODN7KJ.0 size=987080 2024-09-13T00:01:05.877 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.877+0000 7fb8f2ce0640 1 < completed multipart upload key=ZTK9AYBSLCOZ upload=C3LRJ5ODN7KJ tag=5BRGRC1OV83A type=write size=2035656 2024-09-13T00:01:05.889 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.889+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~JZRJHIGVMDQQ.0 size=1048576 2024-09-13T00:01:05.895 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.893+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~JZRJHIGVMDQQ.1 size=1048576 2024-09-13T00:01:05.900 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.897+0000 7fb8f2ce0640 1 > prepared part key=_multipart_FX0NCGLK0KPJ.2~JZRJHIGVMDQQ.2 size=914290 2024-09-13T00:01:05.900 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.897+0000 7fb8f2ce0640 1 > prepared multipart upload key=FX0NCGLK0KPJ upload=JZRJHIGVMDQQ tag=42GYJTJNIX45 type=write size=4060018 2024-09-13T00:01:05.913 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.913+0000 7fb8f2ce0640 1 < suggested update operation key=P8OYS5ZE1NX0 tag=52P09CY1FCLK type=del size=2294312 2024-09-13T00:01:05.926 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.925+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~SELQDREX52T3.0 size=1048576 2024-09-13T00:01:05.931 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.929+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~SELQDREX52T3.1 size=1048576 2024-09-13T00:01:05.936 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.933+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~SELQDREX52T3.2 size=573318 2024-09-13T00:01:05.936 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.933+0000 7fb8f2ce0640 1 > prepared multipart upload key=W8XZ26311RJX upload=SELQDREX52T3 tag=ER26B2SORHOA type=write size=3719046 2024-09-13T00:01:05.945 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.941+0000 7fb8f2ce0640 -1 < failed to complete operation key=H4M1TCN5W6XU tag=MAQUZ7T91XEX type=del size=1438744: (22) Invalid argument 2024-09-13T00:01:05.957 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.957+0000 7fb8f2ce0640 1 > prepared part key=_multipart_85YEW8N43FTA.2~PDKZMTHI3O9B.0 size=1048576 2024-09-13T00:01:05.962 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.961+0000 7fb8f2ce0640 1 > prepared part key=_multipart_85YEW8N43FTA.2~PDKZMTHI3O9B.1 size=274323 2024-09-13T00:01:05.963 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.961+0000 7fb8f2ce0640 1 > prepared multipart upload key=85YEW8N43FTA upload=PDKZMTHI3O9B tag=SFF8JW7X9IUI type=write size=2371475 2024-09-13T00:01:05.976 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.973+0000 7fb8f2ce0640 1 < suggested update operation key=W8XZ26311RJX tag=S64G4SNKIRY1 type=del size=2647439 2024-09-13T00:01:05.981 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.981+0000 7fb8f2ce0640 1 > prepared operation key=N3Z5XIG3TZRQ tag=H7AYEC7PRRLW type=del size=1545509 2024-09-13T00:01:05.987 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.985+0000 7fb8f2ce0640 1 < completed part key=_multipart_PVYZQLGYT3XU.2~E4NPMFY4MMNL.0 size=414501 2024-09-13T00:01:05.997 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:05.997+0000 7fb8f2ce0640 1 < completed multipart upload key=PVYZQLGYT3XU upload=E4NPMFY4MMNL tag=35V2Y2X070BE type=write size=1463077 2024-09-13T00:01:06.003 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.001+0000 7fb8f2ce0640 1 > prepared operation key=1HX06CCZJ1DD tag=IG7ACD5CXAKU type=del size=4124430 2024-09-13T00:01:06.011 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.009+0000 7fb8f2ce0640 1 < completed delete operation key=O4MZ4UWNXME9 tag=X65818RYD83W type=del 2024-09-13T00:01:06.017 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.017+0000 7fb8f2ce0640 1 > prepared operation key=8Z6U7OOA0GBH tag=CWOCCX9DPX1C type=del size=1512217 2024-09-13T00:01:06.023 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.021+0000 7fb8f2ce0640 1 < completed delete operation key=ZFHN2SNTZ5W7 tag=KLJHGYX5461A type=del 2024-09-13T00:01:06.035 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.033+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EJN6HR8I7XQF.2~UUG6CWB965F2.0 size=932289 2024-09-13T00:01:06.035 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.033+0000 7fb8f2ce0640 1 > prepared multipart upload key=EJN6HR8I7XQF upload=UUG6CWB965F2 tag=S4DDNEJ0DLW7 type=write size=1980865 2024-09-13T00:01:06.036 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.033+0000 7fb8f2ce0640 1 < canceled part key=_multipart_1HX06CCZJ1DD.2~DNFD7QTVZH3R.0 size=881750 2024-09-13T00:01:06.046 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.045+0000 7fb8f2ce0640 1 < canceled multipart upload key=1HX06CCZJ1DD upload=DNFD7QTVZH3R tag=NZZRVR0UDBJ0 type=write size=1930326 2024-09-13T00:01:06.052 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.049+0000 7fb8f2ce0640 1 > prepared operation key=1HX06CCZJ1DD tag=Q2WZUU8C1BF4 type=del size=478507 2024-09-13T00:01:06.058 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.057+0000 7fb8f2ce0640 1 < completed part key=_multipart_5GM88ERM89KT.2~RQ0O771LW3PM.0 size=1048576 2024-09-13T00:01:06.058 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.057+0000 7fb8f2ce0640 1 < canceled part key=_multipart_5GM88ERM89KT.2~RQ0O771LW3PM.1 size=258684 2024-09-13T00:01:06.068 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.065+0000 7fb8f2ce0640 1 < canceled multipart upload key=5GM88ERM89KT upload=RQ0O771LW3PM tag=7HG21KGRCPDO type=write size=2355836 2024-09-13T00:01:06.079 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.077+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H9AU4HD247YU.2~GDY7O88ZF0CD.0 size=207983 2024-09-13T00:01:06.079 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.077+0000 7fb8f2ce0640 1 > prepared multipart upload key=H9AU4HD247YU upload=GDY7O88ZF0CD tag=QBNMC4YNVQNN type=write size=1256559 2024-09-13T00:01:06.085 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.085+0000 7fb8f2ce0640 1 < completed delete operation key=NQJQC8CEAC15 tag=BIZUXO3538S3 type=del 2024-09-13T00:01:06.091 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.089+0000 7fb8f2ce0640 1 > prepared operation key=85YEW8N43FTA tag=QD9D7L6Y18TF type=del size=3534485 2024-09-13T00:01:06.097 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.097+0000 7fb8f2ce0640 1 < completed delete operation key=N3Z5XIG3TZRQ tag=XM6DYMO65OPF type=del 2024-09-13T00:01:06.111 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.109+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~SZB1WOQ65MB1.0 size=1048576 2024-09-13T00:01:06.116 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.113+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~SZB1WOQ65MB1.1 size=1048576 2024-09-13T00:01:06.122 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.121+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~SZB1WOQ65MB1.2 size=1014805 2024-09-13T00:01:06.122 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.121+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZTK9AYBSLCOZ upload=SZB1WOQ65MB1 tag=NB427WFEV356 type=write size=4160533 2024-09-13T00:01:06.133 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.133+0000 7fb8f2ce0640 1 < suggested update operation key=63FJDWBO2T9S tag=LQDFT77FZGTF type=write size=316032 2024-09-13T00:01:06.139 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.137+0000 7fb8f2ce0640 1 > prepared operation key=8Z6U7OOA0GBH tag=CODDYBCQZ4NU type=write size=360296 2024-09-13T00:01:06.145 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.145+0000 7fb8f2ce0640 1 < completed delete operation key=N3Z5XIG3TZRQ tag=171M97BEUZGX type=del 2024-09-13T00:01:06.151 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.149+0000 7fb8f2ce0640 1 > prepared operation key=N3Z5XIG3TZRQ tag=JLG7J5562AZC type=del size=3610767 2024-09-13T00:01:06.158 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.157+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~NYCH3ISKP2NM.0 size=1048576 2024-09-13T00:01:06.163 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.161+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~NYCH3ISKP2NM.1 size=1048576 2024-09-13T00:01:06.168 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.165+0000 7fb8f2ce0640 1 < completed part key=_multipart_8MPUVD37EDN9.2~NYCH3ISKP2NM.2 size=984103 2024-09-13T00:01:06.178 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.177+0000 7fb8f2ce0640 1 < completed multipart upload key=8MPUVD37EDN9 upload=NYCH3ISKP2NM tag=WBKJW22NKW4U type=write size=4129831 2024-09-13T00:01:06.186 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.185+0000 7fb8f2ce0640 1 > prepared operation key=ZFHN2SNTZ5W7 tag=LPTMZ4WGA4TT type=del size=2252997 2024-09-13T00:01:06.193 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.189+0000 7fb8f2ce0640 1 < completed part key=_multipart_J8NGDSF9XRTP.2~JP3EQJGXSQ0G.0 size=1048576 2024-09-13T00:01:06.198 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.197+0000 7fb8f2ce0640 1 < completed part key=_multipart_J8NGDSF9XRTP.2~JP3EQJGXSQ0G.1 size=88417 2024-09-13T00:01:06.209 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.209+0000 7fb8f2ce0640 1 < completed multipart upload key=J8NGDSF9XRTP upload=JP3EQJGXSQ0G tag=7DCTX0HMPTXF type=write size=2185569 2024-09-13T00:01:06.216 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.213+0000 7fb8f2ce0640 1 > prepared operation key=NQJQC8CEAC15 tag=0OD5WLFLB3NZ type=del size=3058370 2024-09-13T00:01:06.228 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.225+0000 7fb8f2ce0640 1 < suggested remove operation key=W8XZ26311RJX tag=XTMCKYOBFWEP type=del 2024-09-13T00:01:06.234 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.233+0000 7fb8f2ce0640 1 > prepared operation key=85YEW8N43FTA tag=KRZS7U9D8OHC type=del size=1049250 2024-09-13T00:01:06.240 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.237+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~JZRJHIGVMDQQ.0 size=1048576 2024-09-13T00:01:06.245 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.245+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~JZRJHIGVMDQQ.1 size=1048576 2024-09-13T00:01:06.250 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.249+0000 7fb8f2ce0640 1 < completed part key=_multipart_FX0NCGLK0KPJ.2~JZRJHIGVMDQQ.2 size=914290 2024-09-13T00:01:06.260 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.257+0000 7fb8f2ce0640 1 < completed multipart upload key=FX0NCGLK0KPJ upload=JZRJHIGVMDQQ tag=42GYJTJNIX45 type=write size=4060018 2024-09-13T00:01:06.268 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.265+0000 7fb8f2ce0640 1 > prepared operation key=P8OYS5ZE1NX0 tag=EEMK769Z14GQ type=del size=2876889 2024-09-13T00:01:06.274 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.273+0000 7fb8f2ce0640 1 < completed part key=_multipart_W8XZ26311RJX.2~SELQDREX52T3.0 size=1048576 2024-09-13T00:01:06.274 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.273+0000 7fb8f2ce0640 1 < canceled part key=_multipart_W8XZ26311RJX.2~SELQDREX52T3.1 size=1048576 2024-09-13T00:01:06.280 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.277+0000 7fb8f2ce0640 1 < completed part key=_multipart_W8XZ26311RJX.2~SELQDREX52T3.2 size=573318 2024-09-13T00:01:06.290 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.289+0000 7fb8f2ce0640 1 < canceled multipart upload key=W8XZ26311RJX upload=SELQDREX52T3 tag=ER26B2SORHOA type=write size=3719046 2024-09-13T00:01:06.296 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.293+0000 7fb8f2ce0640 1 > prepared operation key=PVYZQLGYT3XU tag=Q2DQGTAPDTM2 type=write size=395468 2024-09-13T00:01:06.302 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.301+0000 7fb8f2ce0640 1 < completed part key=_multipart_85YEW8N43FTA.2~PDKZMTHI3O9B.0 size=1048576 2024-09-13T00:01:06.307 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.305+0000 7fb8f2ce0640 1 < completed part key=_multipart_85YEW8N43FTA.2~PDKZMTHI3O9B.1 size=274323 2024-09-13T00:01:06.317 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.317+0000 7fb8f2ce0640 1 < completed multipart upload key=85YEW8N43FTA upload=PDKZMTHI3O9B tag=SFF8JW7X9IUI type=write size=2371475 2024-09-13T00:01:06.329 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.329+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~DWUS6SBGRIHS.0 size=1048576 2024-09-13T00:01:06.334 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.333+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~DWUS6SBGRIHS.1 size=202288 2024-09-13T00:01:06.334 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.333+0000 7fb8f2ce0640 1 > prepared multipart upload key=63FJDWBO2T9S upload=DWUS6SBGRIHS tag=I8PA2BLO4W09 type=write size=2299440 2024-09-13T00:01:06.339 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.337+0000 7fb8f2ce0640 1 < canceled operation key=N3Z5XIG3TZRQ tag=H7AYEC7PRRLW type=del size=1545509 2024-09-13T00:01:06.350 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.349+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~IXYODVIO4K5E.0 size=1048576 2024-09-13T00:01:06.355 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.353+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~IXYODVIO4K5E.1 size=132970 2024-09-13T00:01:06.356 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.353+0000 7fb8f2ce0640 1 > prepared multipart upload key=NQJQC8CEAC15 upload=IXYODVIO4K5E tag=LMMP918116IV type=write size=2230122 2024-09-13T00:01:06.361 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.361+0000 7fb8f2ce0640 1 < completed delete operation key=1HX06CCZJ1DD tag=IG7ACD5CXAKU type=del 2024-09-13T00:01:06.375 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.373+0000 7fb8f2ce0640 1 > prepared part key=_multipart_0WC6P8YNL0H2.2~OMNIBUD0PN10.0 size=991241 2024-09-13T00:01:06.375 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.373+0000 7fb8f2ce0640 1 > prepared multipart upload key=0WC6P8YNL0H2 upload=OMNIBUD0PN10 tag=NXSI9TLO4A2A type=write size=2039817 2024-09-13T00:01:06.380 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.377+0000 7fb8f2ce0640 1 < completed delete operation key=8Z6U7OOA0GBH tag=CWOCCX9DPX1C type=del 2024-09-13T00:01:06.392 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.389+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~BC61RL6YM6K2.0 size=1048576 2024-09-13T00:01:06.397 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.397+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~BC61RL6YM6K2.1 size=1048576 2024-09-13T00:01:06.402 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.401+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NQJQC8CEAC15.2~BC61RL6YM6K2.2 size=506244 2024-09-13T00:01:06.402 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.401+0000 7fb8f2ce0640 1 > prepared multipart upload key=NQJQC8CEAC15 upload=BC61RL6YM6K2 tag=BS33LPYPAJCC type=write size=3651972 2024-09-13T00:01:06.408 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.405+0000 7fb8f2ce0640 1 < completed part key=_multipart_EJN6HR8I7XQF.2~UUG6CWB965F2.0 size=932289 2024-09-13T00:01:06.418 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.417+0000 7fb8f2ce0640 1 < completed multipart upload key=EJN6HR8I7XQF upload=UUG6CWB965F2 tag=S4DDNEJ0DLW7 type=write size=1980865 2024-09-13T00:01:06.424 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.421+0000 7fb8f2ce0640 1 > prepared operation key=8Z6U7OOA0GBH tag=V6TV3623WCJ3 type=del size=574471 2024-09-13T00:01:06.430 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.429+0000 7fb8f2ce0640 1 < completed delete operation key=1HX06CCZJ1DD tag=Q2WZUU8C1BF4 type=del 2024-09-13T00:01:06.437 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.437+0000 7fb8f2ce0640 1 > prepared operation key=EUE3LT0PDIXV tag=ROH3HE763VVG type=del size=1518528 2024-09-13T00:01:06.438 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.437+0000 7fb8f2ce0640 1 < canceled part key=_multipart_H9AU4HD247YU.2~GDY7O88ZF0CD.0 size=207983 2024-09-13T00:01:06.450 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.449+0000 7fb8f2ce0640 1 < canceled multipart upload key=H9AU4HD247YU upload=GDY7O88ZF0CD tag=QBNMC4YNVQNN type=write size=1256559 2024-09-13T00:01:06.456 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.453+0000 7fb8f2ce0640 1 > prepared operation key=44EDYFT9VF8Y tag=C7T6UVSN2DUN type=del size=2949993 2024-09-13T00:01:06.463 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.461+0000 7fb8f2ce0640 1 < completed delete operation key=85YEW8N43FTA tag=QD9D7L6Y18TF type=del 2024-09-13T00:01:06.474 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.473+0000 7fb8f2ce0640 1 > prepared part key=_multipart_QMZREZ1QEDNJ.2~ENXADUERWHML.0 size=1048576 2024-09-13T00:01:06.480 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.477+0000 7fb8f2ce0640 1 > prepared part key=_multipart_QMZREZ1QEDNJ.2~ENXADUERWHML.1 size=1048576 2024-09-13T00:01:06.485 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.481+0000 7fb8f2ce0640 1 > prepared part key=_multipart_QMZREZ1QEDNJ.2~ENXADUERWHML.2 size=491051 2024-09-13T00:01:06.485 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.481+0000 7fb8f2ce0640 1 > prepared multipart upload key=QMZREZ1QEDNJ upload=ENXADUERWHML tag=21U6Z5VFQL9P type=write size=3636779 2024-09-13T00:01:06.491 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.489+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~SZB1WOQ65MB1.0 size=1048576 2024-09-13T00:01:06.496 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.493+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~SZB1WOQ65MB1.1 size=1048576 2024-09-13T00:01:06.501 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.497+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~SZB1WOQ65MB1.2 size=1014805 2024-09-13T00:01:06.511 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.509+0000 7fb8f2ce0640 1 < completed multipart upload key=ZTK9AYBSLCOZ upload=SZB1WOQ65MB1 tag=NB427WFEV356 type=write size=4160533 2024-09-13T00:01:06.517 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.517+0000 7fb8f2ce0640 1 > prepared operation key=F0QCVR9E08WE tag=36NIK8F8447A type=write size=883747 2024-09-13T00:01:06.523 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.521+0000 7fb8f2ce0640 1 < completed write operation key=8Z6U7OOA0GBH tag=CODDYBCQZ4NU type=write size=360296 2024-09-13T00:01:06.534 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.533+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NX26SI4F2ZMB.2~85G9HJZIIUFO.0 size=281506 2024-09-13T00:01:06.534 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.533+0000 7fb8f2ce0640 1 > prepared multipart upload key=NX26SI4F2ZMB upload=85G9HJZIIUFO tag=CGTY0GLLXA42 type=write size=1330082 2024-09-13T00:01:06.540 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.537+0000 7fb8f2ce0640 1 < completed delete operation key=N3Z5XIG3TZRQ tag=JLG7J5562AZC type=del 2024-09-13T00:01:06.551 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.549+0000 7fb8f2ce0640 1 > prepared part key=_multipart_70F7D20HI504.2~0C3ZMYILD93J.0 size=486879 2024-09-13T00:01:06.551 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.549+0000 7fb8f2ce0640 1 > prepared multipart upload key=70F7D20HI504 upload=0C3ZMYILD93J tag=N8OX84EMC8RC type=write size=1535455 2024-09-13T00:01:06.557 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.553+0000 7fb8f2ce0640 1 < completed delete operation key=ZFHN2SNTZ5W7 tag=LPTMZ4WGA4TT type=del 2024-09-13T00:01:06.569 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.569+0000 7fb8f2ce0640 1 > prepared part key=_multipart_QMZREZ1QEDNJ.2~K40DRH39XLMX.0 size=1048576 2024-09-13T00:01:06.574 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.573+0000 7fb8f2ce0640 1 > prepared part key=_multipart_QMZREZ1QEDNJ.2~K40DRH39XLMX.1 size=1048534 2024-09-13T00:01:06.575 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.573+0000 7fb8f2ce0640 1 > prepared multipart upload key=QMZREZ1QEDNJ upload=K40DRH39XLMX tag=EBGY5I6OTJR9 type=write size=3145686 2024-09-13T00:01:06.581 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.577+0000 7fb8f2ce0640 1 < completed delete operation key=NQJQC8CEAC15 tag=0OD5WLFLB3NZ type=del 2024-09-13T00:01:06.592 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.589+0000 7fb8f2ce0640 1 > prepared part key=_multipart_N3Z5XIG3TZRQ.2~MKRVB38SVKMD.0 size=194886 2024-09-13T00:01:06.592 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.589+0000 7fb8f2ce0640 1 > prepared multipart upload key=N3Z5XIG3TZRQ upload=MKRVB38SVKMD tag=CZFTHRUDMY1E type=write size=1243462 2024-09-13T00:01:06.598 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.597+0000 7fb8f2ce0640 1 < completed delete operation key=85YEW8N43FTA tag=KRZS7U9D8OHC type=del 2024-09-13T00:01:06.604 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.601+0000 7fb8f2ce0640 1 > prepared operation key=ZFHN2SNTZ5W7 tag=1PUM0A5V2PQG type=del size=369446 2024-09-13T00:01:06.610 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.609+0000 7fb8f2ce0640 1 < completed delete operation key=P8OYS5ZE1NX0 tag=EEMK769Z14GQ type=del 2024-09-13T00:01:06.617 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.617+0000 7fb8f2ce0640 1 > prepared operation key=F0QCVR9E08WE tag=X58XJ1HPSELD type=del size=2159755 2024-09-13T00:01:06.624 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.621+0000 7fb8f2ce0640 1 < completed write operation key=PVYZQLGYT3XU tag=Q2DQGTAPDTM2 type=write size=395468 2024-09-13T00:01:06.629 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.629+0000 7fb8f2ce0640 1 > prepared operation key=1HX06CCZJ1DD tag=K8U70YX4FLV1 type=del size=842219 2024-09-13T00:01:06.636 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.633+0000 7fb8f2ce0640 1 < completed part key=_multipart_63FJDWBO2T9S.2~DWUS6SBGRIHS.0 size=1048576 2024-09-13T00:01:06.642 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.641+0000 7fb8f2ce0640 1 < completed part key=_multipart_63FJDWBO2T9S.2~DWUS6SBGRIHS.1 size=202288 2024-09-13T00:01:06.654 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.653+0000 7fb8f2ce0640 1 < completed multipart upload key=63FJDWBO2T9S upload=DWUS6SBGRIHS tag=I8PA2BLO4W09 type=write size=2299440 2024-09-13T00:01:06.660 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.657+0000 7fb8f2ce0640 1 > prepared operation key=NX26SI4F2ZMB tag=82219DDWA3J3 type=del size=3151491 2024-09-13T00:01:06.666 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.665+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~IXYODVIO4K5E.0 size=1048576 2024-09-13T00:01:06.671 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.669+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~IXYODVIO4K5E.1 size=132970 2024-09-13T00:01:06.682 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.681+0000 7fb8f2ce0640 1 < completed multipart upload key=NQJQC8CEAC15 upload=IXYODVIO4K5E tag=LMMP918116IV type=write size=2230122 2024-09-13T00:01:06.694 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.693+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~BW4R9SXPQ93E.0 size=1048576 2024-09-13T00:01:06.699 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.697+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~BW4R9SXPQ93E.1 size=1048576 2024-09-13T00:01:06.704 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.701+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZTK9AYBSLCOZ.2~BW4R9SXPQ93E.2 size=380188 2024-09-13T00:01:06.704 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.701+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZTK9AYBSLCOZ upload=BW4R9SXPQ93E tag=9Q9R5TH3VQ5C type=write size=3525916 2024-09-13T00:01:06.710 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.709+0000 7fb8f2ce0640 1 < completed part key=_multipart_0WC6P8YNL0H2.2~OMNIBUD0PN10.0 size=991241 2024-09-13T00:01:06.720 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.717+0000 7fb8f2ce0640 1 < completed multipart upload key=0WC6P8YNL0H2 upload=OMNIBUD0PN10 tag=NXSI9TLO4A2A type=write size=2039817 2024-09-13T00:01:06.727 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.725+0000 7fb8f2ce0640 1 > prepared operation key=O4MZ4UWNXME9 tag=JQUEEJ3N1FI7 type=write size=861129 2024-09-13T00:01:06.734 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.733+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~BC61RL6YM6K2.0 size=1048576 2024-09-13T00:01:06.739 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.737+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~BC61RL6YM6K2.1 size=1048576 2024-09-13T00:01:06.745 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.745+0000 7fb8f2ce0640 1 < completed part key=_multipart_NQJQC8CEAC15.2~BC61RL6YM6K2.2 size=506244 2024-09-13T00:01:06.755 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.753+0000 7fb8f2ce0640 1 < completed multipart upload key=NQJQC8CEAC15 upload=BC61RL6YM6K2 tag=BS33LPYPAJCC type=write size=3651972 2024-09-13T00:01:06.768 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.765+0000 7fb8f2ce0640 1 > prepared part key=_multipart_O4MZ4UWNXME9.2~XCC9WRLTOQ35.0 size=1048576 2024-09-13T00:01:06.773 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.773+0000 7fb8f2ce0640 1 > prepared part key=_multipart_O4MZ4UWNXME9.2~XCC9WRLTOQ35.1 size=1048576 2024-09-13T00:01:06.778 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.777+0000 7fb8f2ce0640 1 > prepared part key=_multipart_O4MZ4UWNXME9.2~XCC9WRLTOQ35.2 size=350110 2024-09-13T00:01:06.778 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.777+0000 7fb8f2ce0640 1 > prepared multipart upload key=O4MZ4UWNXME9 upload=XCC9WRLTOQ35 tag=UWBOEVDON8PH type=write size=3495838 2024-09-13T00:01:06.784 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.781+0000 7fb8f2ce0640 1 < completed delete operation key=8Z6U7OOA0GBH tag=V6TV3623WCJ3 type=del 2024-09-13T00:01:06.796 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.793+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~9DG87WEZTJWX.0 size=1048576 2024-09-13T00:01:06.800 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.797+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~9DG87WEZTJWX.1 size=1048576 2024-09-13T00:01:06.806 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.805+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~9DG87WEZTJWX.2 size=209675 2024-09-13T00:01:06.806 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.805+0000 7fb8f2ce0640 1 > prepared multipart upload key=PVYZQLGYT3XU upload=9DG87WEZTJWX tag=N5K33QPAWJOY type=write size=3355403 2024-09-13T00:01:06.818 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.817+0000 7fb8f2ce0640 1 < suggested update operation key=EUE3LT0PDIXV tag=ROH3HE763VVG type=del size=1518528 2024-09-13T00:01:06.823 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.821+0000 7fb8f2ce0640 1 > prepared operation key=2TWD9YLMSYJI tag=8O48SE4VO5G9 type=del size=2494722 2024-09-13T00:01:06.830 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.829+0000 7fb8f2ce0640 1 < canceled operation key=44EDYFT9VF8Y tag=C7T6UVSN2DUN type=del size=2949993 2024-09-13T00:01:06.842 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.841+0000 7fb8f2ce0640 1 > prepared part key=_multipart_QMZREZ1QEDNJ.2~LC1XJN69EY3O.0 size=1048576 2024-09-13T00:01:06.847 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.845+0000 7fb8f2ce0640 1 > prepared part key=_multipart_QMZREZ1QEDNJ.2~LC1XJN69EY3O.1 size=1048576 2024-09-13T00:01:06.852 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.849+0000 7fb8f2ce0640 1 > prepared part key=_multipart_QMZREZ1QEDNJ.2~LC1XJN69EY3O.2 size=321713 2024-09-13T00:01:06.852 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.849+0000 7fb8f2ce0640 1 > prepared multipart upload key=QMZREZ1QEDNJ upload=LC1XJN69EY3O tag=HUF5RYRNKCHR type=write size=3467441 2024-09-13T00:01:06.858 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.857+0000 7fb8f2ce0640 1 < completed part key=_multipart_QMZREZ1QEDNJ.2~ENXADUERWHML.0 size=1048576 2024-09-13T00:01:06.863 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.861+0000 7fb8f2ce0640 1 < completed part key=_multipart_QMZREZ1QEDNJ.2~ENXADUERWHML.1 size=1048576 2024-09-13T00:01:06.868 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.865+0000 7fb8f2ce0640 1 < completed part key=_multipart_QMZREZ1QEDNJ.2~ENXADUERWHML.2 size=491051 2024-09-13T00:01:06.879 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.877+0000 7fb8f2ce0640 1 < completed multipart upload key=QMZREZ1QEDNJ upload=ENXADUERWHML tag=21U6Z5VFQL9P type=write size=3636779 2024-09-13T00:01:06.890 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.889+0000 7fb8f2ce0640 1 > prepared part key=_multipart_EJN6HR8I7XQF.2~MBELAYTTTBPG.0 size=648423 2024-09-13T00:01:06.890 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.889+0000 7fb8f2ce0640 1 > prepared multipart upload key=EJN6HR8I7XQF upload=MBELAYTTTBPG tag=P58AI66JR23G type=write size=1696999 2024-09-13T00:01:06.896 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.893+0000 7fb8f2ce0640 1 < completed write operation key=F0QCVR9E08WE tag=36NIK8F8447A type=write size=883747 2024-09-13T00:01:06.902 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.901+0000 7fb8f2ce0640 1 > prepared operation key=F0QCVR9E08WE tag=89IS3V9KV9Y4 type=del size=2588990 2024-09-13T00:01:06.903 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.901+0000 7fb8f2ce0640 1 < canceled part key=_multipart_NX26SI4F2ZMB.2~85G9HJZIIUFO.0 size=281506 2024-09-13T00:01:06.914 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.913+0000 7fb8f2ce0640 1 < canceled multipart upload key=NX26SI4F2ZMB upload=85G9HJZIIUFO tag=CGTY0GLLXA42 type=write size=1330082 2024-09-13T00:01:06.920 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.917+0000 7fb8f2ce0640 1 > prepared operation key=44EDYFT9VF8Y tag=1CCNGYNPK9AK type=del size=2645159 2024-09-13T00:01:06.927 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.925+0000 7fb8f2ce0640 1 < completed part key=_multipart_70F7D20HI504.2~0C3ZMYILD93J.0 size=486879 2024-09-13T00:01:06.941 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.941+0000 7fb8f2ce0640 1 < completed multipart upload key=70F7D20HI504 upload=0C3ZMYILD93J tag=N8OX84EMC8RC type=write size=1535455 2024-09-13T00:01:06.955 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.953+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~NCPFD2AI4I4C.0 size=1048576 2024-09-13T00:01:06.960 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.957+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~NCPFD2AI4I4C.1 size=1048576 2024-09-13T00:01:06.967 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.965+0000 7fb8f2ce0640 1 > prepared part key=_multipart_PVYZQLGYT3XU.2~NCPFD2AI4I4C.2 size=124323 2024-09-13T00:01:06.967 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.965+0000 7fb8f2ce0640 1 > prepared multipart upload key=PVYZQLGYT3XU upload=NCPFD2AI4I4C tag=3E9CE8ZTYX8I type=write size=3270051 2024-09-13T00:01:06.973 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.973+0000 7fb8f2ce0640 1 < completed part key=_multipart_QMZREZ1QEDNJ.2~K40DRH39XLMX.0 size=1048576 2024-09-13T00:01:06.973 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.973+0000 7fb8f2ce0640 1 < canceled part key=_multipart_QMZREZ1QEDNJ.2~K40DRH39XLMX.1 size=1048534 2024-09-13T00:01:06.984 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.981+0000 7fb8f2ce0640 1 < canceled multipart upload key=QMZREZ1QEDNJ upload=K40DRH39XLMX tag=EBGY5I6OTJR9 type=write size=3145686 2024-09-13T00:01:06.996 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:06.993+0000 7fb8f2ce0640 1 > prepared part key=_multipart_4K76U69GZ9AE.2~MHXYBJZYJK58.0 size=1048576 2024-09-13T00:01:07.001 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.001+0000 7fb8f2ce0640 1 > prepared part key=_multipart_4K76U69GZ9AE.2~MHXYBJZYJK58.1 size=987898 2024-09-13T00:01:07.001 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.001+0000 7fb8f2ce0640 1 > prepared multipart upload key=4K76U69GZ9AE upload=MHXYBJZYJK58 tag=J1BFY1181ZVH type=write size=3085050 2024-09-13T00:01:07.008 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.005+0000 7fb8f2ce0640 1 < completed part key=_multipart_N3Z5XIG3TZRQ.2~MKRVB38SVKMD.0 size=194886 2024-09-13T00:01:07.018 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.017+0000 7fb8f2ce0640 1 < completed multipart upload key=N3Z5XIG3TZRQ upload=MKRVB38SVKMD tag=CZFTHRUDMY1E type=write size=1243462 2024-09-13T00:01:07.025 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.025+0000 7fb8f2ce0640 1 > prepared operation key=7OX5OJZ8GXU5 tag=M3W1JE640TRA type=del size=835179 2024-09-13T00:01:07.040 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.037+0000 7fb8f2ce0640 1 < suggested update operation key=ZFHN2SNTZ5W7 tag=1PUM0A5V2PQG type=del size=369446 2024-09-13T00:01:07.046 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.045+0000 7fb8f2ce0640 1 > prepared operation key=5GM88ERM89KT tag=XBZ2O6HR0E83 type=del size=1974358 2024-09-13T00:01:07.052 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.049+0000 7fb8f2ce0640 1 < completed delete operation key=F0QCVR9E08WE tag=X58XJ1HPSELD type=del 2024-09-13T00:01:07.062 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.061+0000 7fb8f2ce0640 1 > prepared part key=_multipart_44EDYFT9VF8Y.2~KQYPC6FNWRBC.0 size=1048576 2024-09-13T00:01:07.068 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.065+0000 7fb8f2ce0640 1 > prepared part key=_multipart_44EDYFT9VF8Y.2~KQYPC6FNWRBC.1 size=604656 2024-09-13T00:01:07.068 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.065+0000 7fb8f2ce0640 1 > prepared multipart upload key=44EDYFT9VF8Y upload=KQYPC6FNWRBC tag=NK14UDQ2J5P7 type=write size=2701808 2024-09-13T00:01:07.080 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.077+0000 7fb8f2ce0640 1 < suggested remove operation key=1HX06CCZJ1DD tag=K8U70YX4FLV1 type=del 2024-09-13T00:01:07.091 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.089+0000 7fb8f2ce0640 1 > prepared part key=_multipart_4K76U69GZ9AE.2~8G0WXW57R3PB.0 size=1048576 2024-09-13T00:01:07.096 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.093+0000 7fb8f2ce0640 1 > prepared part key=_multipart_4K76U69GZ9AE.2~8G0WXW57R3PB.1 size=1048576 2024-09-13T00:01:07.101 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.101+0000 7fb8f2ce0640 1 > prepared part key=_multipart_4K76U69GZ9AE.2~8G0WXW57R3PB.2 size=295762 2024-09-13T00:01:07.102 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.101+0000 7fb8f2ce0640 1 > prepared multipart upload key=4K76U69GZ9AE upload=8G0WXW57R3PB tag=WUYT73QPHCST type=write size=3441490 2024-09-13T00:01:07.107 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.105+0000 7fb8f2ce0640 1 < completed delete operation key=NX26SI4F2ZMB tag=82219DDWA3J3 type=del 2024-09-13T00:01:07.114 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.113+0000 7fb8f2ce0640 1 > prepared operation key=ZFHN2SNTZ5W7 tag=5M9WMA8T2V1L type=del size=902991 2024-09-13T00:01:07.121 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.121+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~BW4R9SXPQ93E.0 size=1048576 2024-09-13T00:01:07.128 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.125+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~BW4R9SXPQ93E.1 size=1048576 2024-09-13T00:01:07.133 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.133+0000 7fb8f2ce0640 1 < completed part key=_multipart_ZTK9AYBSLCOZ.2~BW4R9SXPQ93E.2 size=380188 2024-09-13T00:01:07.143 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.141+0000 7fb8f2ce0640 1 < completed multipart upload key=ZTK9AYBSLCOZ upload=BW4R9SXPQ93E tag=9Q9R5TH3VQ5C type=write size=3525916 2024-09-13T00:01:07.149 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.149+0000 7fb8f2ce0640 1 > prepared operation key=8MPUVD37EDN9 tag=2XFQNBVKHZE2 type=del size=3604205 2024-09-13T00:01:07.161 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.157+0000 7fb8f2ce0640 1 < suggested remove operation key=O4MZ4UWNXME9 tag=JQUEEJ3N1FI7 type=write 2024-09-13T00:01:07.167 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.165+0000 7fb8f2ce0640 1 > prepared operation key=F0QCVR9E08WE tag=2SQKTK5TCJGE type=del size=2522870 2024-09-13T00:01:07.173 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.173+0000 7fb8f2ce0640 1 < completed part key=_multipart_O4MZ4UWNXME9.2~XCC9WRLTOQ35.0 size=1048576 2024-09-13T00:01:07.179 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.177+0000 7fb8f2ce0640 1 < completed part key=_multipart_O4MZ4UWNXME9.2~XCC9WRLTOQ35.1 size=1048576 2024-09-13T00:01:07.184 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.181+0000 7fb8f2ce0640 1 < completed part key=_multipart_O4MZ4UWNXME9.2~XCC9WRLTOQ35.2 size=350110 2024-09-13T00:01:07.196 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.193+0000 7fb8f2ce0640 1 < completed multipart upload key=O4MZ4UWNXME9 upload=XCC9WRLTOQ35 tag=UWBOEVDON8PH type=write size=3495838 2024-09-13T00:01:07.206 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.205+0000 7fb8f2ce0640 1 > prepared part key=_multipart_W8XZ26311RJX.2~QXJ9V19N8KCS.0 size=1002999 2024-09-13T00:01:07.206 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.205+0000 7fb8f2ce0640 1 > prepared multipart upload key=W8XZ26311RJX upload=QXJ9V19N8KCS tag=EF0B4O8IWQFZ type=write size=2051575 2024-09-13T00:01:07.213 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.213+0000 7fb8f2ce0640 1 < completed part key=_multipart_PVYZQLGYT3XU.2~9DG87WEZTJWX.0 size=1048576 2024-09-13T00:01:07.218 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.217+0000 7fb8f2ce0640 1 < completed part key=_multipart_PVYZQLGYT3XU.2~9DG87WEZTJWX.1 size=1048576 2024-09-13T00:01:07.224 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.221+0000 7fb8f2ce0640 1 < completed part key=_multipart_PVYZQLGYT3XU.2~9DG87WEZTJWX.2 size=209675 2024-09-13T00:01:07.235 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.233+0000 7fb8f2ce0640 1 < completed multipart upload key=PVYZQLGYT3XU upload=9DG87WEZTJWX tag=N5K33QPAWJOY type=write size=3355403 2024-09-13T00:01:07.244 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.241+0000 7fb8f2ce0640 1 > prepared operation key=8MPUVD37EDN9 tag=3MY6A4VMW48E type=del size=2537306 2024-09-13T00:01:07.251 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.249+0000 7fb8f2ce0640 1 < completed delete operation key=2TWD9YLMSYJI tag=8O48SE4VO5G9 type=del 2024-09-13T00:01:07.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:01:06 smithi136 bash[75164]: cluster 2024-09-13T00:01:05.143587+0000 mgr.y (mgr.24661) 719 : cluster [DBG] pgmap v1395: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 338 KiB/s rd, 117 KiB/s wr, 599 op/s 2024-09-13T00:01:07.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:01:06 smithi136 bash[76371]: cluster 2024-09-13T00:01:05.143587+0000 mgr.y (mgr.24661) 719 : cluster [DBG] pgmap v1395: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 338 KiB/s rd, 117 KiB/s wr, 599 op/s 2024-09-13T00:01:07.264 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.261+0000 7fb8f2ce0640 1 > prepared part key=_multipart_4K76U69GZ9AE.2~4YACBBOG3HD0.0 size=487066 2024-09-13T00:01:07.264 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.261+0000 7fb8f2ce0640 1 > prepared multipart upload key=4K76U69GZ9AE upload=4YACBBOG3HD0 tag=PWSCKP4QX1UP type=write size=1535642 2024-09-13T00:01:07.269 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.269+0000 7fb8f2ce0640 1 < completed part key=_multipart_QMZREZ1QEDNJ.2~LC1XJN69EY3O.0 size=1048576 2024-09-13T00:01:07.274 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.273+0000 7fb8f2ce0640 1 < completed part key=_multipart_QMZREZ1QEDNJ.2~LC1XJN69EY3O.1 size=1048576 2024-09-13T00:01:07.280 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.277+0000 7fb8f2ce0640 1 < completed part key=_multipart_QMZREZ1QEDNJ.2~LC1XJN69EY3O.2 size=321713 2024-09-13T00:01:07.293 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.293+0000 7fb8f2ce0640 1 < completed multipart upload key=QMZREZ1QEDNJ upload=LC1XJN69EY3O tag=HUF5RYRNKCHR type=write size=3467441 2024-09-13T00:01:07.299 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.297+0000 7fb8f2ce0640 1 > prepared operation key=QMZREZ1QEDNJ tag=UE3EYTYAPX5O type=del size=2568477 2024-09-13T00:01:07.300 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.297+0000 7fb8f2ce0640 1 < canceled part key=_multipart_EJN6HR8I7XQF.2~MBELAYTTTBPG.0 size=648423 2024-09-13T00:01:07.311 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.309+0000 7fb8f2ce0640 1 < canceled multipart upload key=EJN6HR8I7XQF upload=MBELAYTTTBPG tag=P58AI66JR23G type=write size=1696999 2024-09-13T00:01:07.317 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.313+0000 7fb8f2ce0640 1 > prepared operation key=2TWD9YLMSYJI tag=302KUG1YRR11 type=del size=2587919 2024-09-13T00:01:07.324 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.321+0000 7fb8f2ce0640 1 < completed delete operation key=F0QCVR9E08WE tag=89IS3V9KV9Y4 type=del 2024-09-13T00:01:07.330 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.329+0000 7fb8f2ce0640 1 > prepared operation key=QMZREZ1QEDNJ tag=M99OZ7K8J3LX type=del size=1037273 2024-09-13T00:01:07.337 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.337+0000 7fb8f2ce0640 1 < completed delete operation key=44EDYFT9VF8Y tag=1CCNGYNPK9AK type=del 2024-09-13T00:01:07.344 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.341+0000 7fb8f2ce0640 1 > prepared operation key=P8OYS5ZE1NX0 tag=EOW7TL7K05D7 type=del size=3505042 2024-09-13T00:01:07.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:01:06 smithi181 bash[67587]: cluster 2024-09-13T00:01:05.143587+0000 mgr.y (mgr.24661) 719 : cluster [DBG] pgmap v1395: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 338 KiB/s rd, 117 KiB/s wr, 599 op/s 2024-09-13T00:01:07.350 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.349+0000 7fb8f2ce0640 1 < completed part key=_multipart_PVYZQLGYT3XU.2~NCPFD2AI4I4C.0 size=1048576 2024-09-13T00:01:07.356 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.353+0000 7fb8f2ce0640 1 < completed part key=_multipart_PVYZQLGYT3XU.2~NCPFD2AI4I4C.1 size=1048576 2024-09-13T00:01:07.362 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.361+0000 7fb8f2ce0640 1 < completed part key=_multipart_PVYZQLGYT3XU.2~NCPFD2AI4I4C.2 size=124323 2024-09-13T00:01:07.374 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.373+0000 7fb8f2ce0640 1 < completed multipart upload key=PVYZQLGYT3XU upload=NCPFD2AI4I4C tag=3E9CE8ZTYX8I type=write size=3270051 2024-09-13T00:01:07.381 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.377+0000 7fb8f2ce0640 1 > prepared operation key=2AHYRZUDFHP4 tag=VSCWJZKYJQ0W type=del size=3289692 2024-09-13T00:01:07.387 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.385+0000 7fb8f2ce0640 1 < completed part key=_multipart_4K76U69GZ9AE.2~MHXYBJZYJK58.0 size=1048576 2024-09-13T00:01:07.392 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.389+0000 7fb8f2ce0640 1 < completed part key=_multipart_4K76U69GZ9AE.2~MHXYBJZYJK58.1 size=987898 2024-09-13T00:01:07.402 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.401+0000 7fb8f2ce0640 1 < completed multipart upload key=4K76U69GZ9AE upload=MHXYBJZYJK58 tag=J1BFY1181ZVH type=write size=3085050 2024-09-13T00:01:07.408 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.405+0000 7fb8f2ce0640 1 > prepared operation key=0WC6P8YNL0H2 tag=RECY34A452BF type=del size=1314061 2024-09-13T00:01:07.419 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.417+0000 7fb8f2ce0640 1 < suggested remove operation key=7OX5OJZ8GXU5 tag=M3W1JE640TRA type=del 2024-09-13T00:01:07.425 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.425+0000 7fb8f2ce0640 1 > prepared operation key=PVYZQLGYT3XU tag=PFMB57NIVST2 type=write size=884914 2024-09-13T00:01:07.431 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.429+0000 7fb8f2ce0640 1 < canceled operation key=5GM88ERM89KT tag=XBZ2O6HR0E83 type=del size=1974358 2024-09-13T00:01:07.444 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.441+0000 7fb8f2ce0640 1 > prepared part key=_multipart_O4MZ4UWNXME9.2~W7TNUDCR97Y9.0 size=812790 2024-09-13T00:01:07.444 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.441+0000 7fb8f2ce0640 1 > prepared multipart upload key=O4MZ4UWNXME9 upload=W7TNUDCR97Y9 tag=EEERW1HPGOGS type=write size=1861366 2024-09-13T00:01:07.450 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.449+0000 7fb8f2ce0640 1 < completed part key=_multipart_44EDYFT9VF8Y.2~KQYPC6FNWRBC.0 size=1048576 2024-09-13T00:01:07.450 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.449+0000 7fb8f2ce0640 1 < canceled part key=_multipart_44EDYFT9VF8Y.2~KQYPC6FNWRBC.1 size=604656 2024-09-13T00:01:07.460 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.457+0000 7fb8f2ce0640 1 < canceled multipart upload key=44EDYFT9VF8Y upload=KQYPC6FNWRBC tag=NK14UDQ2J5P7 type=write size=2701808 2024-09-13T00:01:07.471 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.469+0000 7fb8f2ce0640 1 > prepared part key=_multipart_QMZREZ1QEDNJ.2~PTBPJOIXJ1YI.0 size=1048576 2024-09-13T00:01:07.476 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.473+0000 7fb8f2ce0640 1 > prepared part key=_multipart_QMZREZ1QEDNJ.2~PTBPJOIXJ1YI.1 size=1048576 2024-09-13T00:01:07.481 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.481+0000 7fb8f2ce0640 1 > prepared part key=_multipart_QMZREZ1QEDNJ.2~PTBPJOIXJ1YI.2 size=750664 2024-09-13T00:01:07.481 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.481+0000 7fb8f2ce0640 1 > prepared multipart upload key=QMZREZ1QEDNJ upload=PTBPJOIXJ1YI tag=ZULYXZW2Q58C type=write size=3896392 2024-09-13T00:01:07.488 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.485+0000 7fb8f2ce0640 1 < completed part key=_multipart_4K76U69GZ9AE.2~8G0WXW57R3PB.0 size=1048576 2024-09-13T00:01:07.494 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.493+0000 7fb8f2ce0640 1 < completed part key=_multipart_4K76U69GZ9AE.2~8G0WXW57R3PB.1 size=1048576 2024-09-13T00:01:07.499 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.497+0000 7fb8f2ce0640 1 < completed part key=_multipart_4K76U69GZ9AE.2~8G0WXW57R3PB.2 size=295762 2024-09-13T00:01:07.511 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.509+0000 7fb8f2ce0640 1 < completed multipart upload key=4K76U69GZ9AE upload=8G0WXW57R3PB tag=WUYT73QPHCST type=write size=3441490 2024-09-13T00:01:07.518 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.517+0000 7fb8f2ce0640 1 > prepared operation key=5GM88ERM89KT tag=MSNC84S4T8N0 type=del size=1670068 2024-09-13T00:01:07.525 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.521+0000 7fb8f2ce0640 1 < canceled operation key=ZFHN2SNTZ5W7 tag=5M9WMA8T2V1L type=del size=902991 2024-09-13T00:01:07.530 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.529+0000 7fb8f2ce0640 1 > prepared operation key=H4M1TCN5W6XU tag=RNXHDNFLJSRN type=del size=3343720 2024-09-13T00:01:07.538 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.537+0000 7fb8f2ce0640 1 < completed delete operation key=8MPUVD37EDN9 tag=2XFQNBVKHZE2 type=del 2024-09-13T00:01:07.545 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.545+0000 7fb8f2ce0640 1 > prepared operation key=70F7D20HI504 tag=0XRGY5UD9KFP type=del size=3976562 2024-09-13T00:01:07.552 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.549+0000 7fb8f2ce0640 1 < completed delete operation key=F0QCVR9E08WE tag=2SQKTK5TCJGE type=del 2024-09-13T00:01:07.564 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.561+0000 7fb8f2ce0640 1 > prepared part key=_multipart_85YEW8N43FTA.2~XUD43VGBC0YB.0 size=1048576 2024-09-13T00:01:07.569 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.569+0000 7fb8f2ce0640 1 > prepared part key=_multipart_85YEW8N43FTA.2~XUD43VGBC0YB.1 size=392041 2024-09-13T00:01:07.569 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.569+0000 7fb8f2ce0640 1 > prepared multipart upload key=85YEW8N43FTA upload=XUD43VGBC0YB tag=8WJYTGVHS3BQ type=write size=2489193 2024-09-13T00:01:07.575 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.573+0000 7fb8f2ce0640 1 < completed part key=_multipart_W8XZ26311RJX.2~QXJ9V19N8KCS.0 size=1002999 2024-09-13T00:01:07.586 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.585+0000 7fb8f2ce0640 1 < completed multipart upload key=W8XZ26311RJX upload=QXJ9V19N8KCS tag=EF0B4O8IWQFZ type=write size=2051575 2024-09-13T00:01:07.597 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.597+0000 7fb8f2ce0640 1 > prepared part key=_multipart_H5EIP0E9CBWX.2~BWIPA7IXE2LP.0 size=964619 2024-09-13T00:01:07.597 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.597+0000 7fb8f2ce0640 1 > prepared multipart upload key=H5EIP0E9CBWX upload=BWIPA7IXE2LP tag=XDOP1RLSKD8I type=write size=2013195 2024-09-13T00:01:07.603 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.601+0000 7fb8f2ce0640 1 < completed delete operation key=8MPUVD37EDN9 tag=3MY6A4VMW48E type=del 2024-09-13T00:01:07.614 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.613+0000 7fb8f2ce0640 1 > prepared part key=_multipart_QMZREZ1QEDNJ.2~T8QD1P5WUFXK.0 size=1048576 2024-09-13T00:01:07.619 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.617+0000 7fb8f2ce0640 1 > prepared part key=_multipart_QMZREZ1QEDNJ.2~T8QD1P5WUFXK.1 size=1048576 2024-09-13T00:01:07.625 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.625+0000 7fb8f2ce0640 1 > prepared part key=_multipart_QMZREZ1QEDNJ.2~T8QD1P5WUFXK.2 size=209733 2024-09-13T00:01:07.626 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.625+0000 7fb8f2ce0640 1 > prepared multipart upload key=QMZREZ1QEDNJ upload=T8QD1P5WUFXK tag=OTP3J7G273C1 type=write size=3355461 2024-09-13T00:01:07.632 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.629+0000 7fb8f2ce0640 1 < completed part key=_multipart_4K76U69GZ9AE.2~4YACBBOG3HD0.0 size=487066 2024-09-13T00:01:07.642 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.641+0000 7fb8f2ce0640 1 < completed multipart upload key=4K76U69GZ9AE upload=4YACBBOG3HD0 tag=PWSCKP4QX1UP type=write size=1535642 2024-09-13T00:01:07.648 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.645+0000 7fb8f2ce0640 1 > prepared operation key=0WC6P8YNL0H2 tag=9008H7QV3XKD type=del size=1608571 2024-09-13T00:01:07.659 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.657+0000 7fb8f2ce0640 1 < suggested update operation key=QMZREZ1QEDNJ tag=UE3EYTYAPX5O type=del size=2568477 2024-09-13T00:01:07.671 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.669+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZFHN2SNTZ5W7.2~BW2I1Z2R0BVW.0 size=1048576 2024-09-13T00:01:07.676 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.673+0000 7fb8f2ce0640 1 > prepared part key=_multipart_ZFHN2SNTZ5W7.2~BW2I1Z2R0BVW.1 size=336761 2024-09-13T00:01:07.676 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.673+0000 7fb8f2ce0640 1 > prepared multipart upload key=ZFHN2SNTZ5W7 upload=BW2I1Z2R0BVW tag=ML7S08NUCJZA type=write size=2433913 2024-09-13T00:01:07.682 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.681+0000 7fb8f2ce0640 1 < completed delete operation key=2TWD9YLMSYJI tag=302KUG1YRR11 type=del 2024-09-13T00:01:07.689 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.685+0000 7fb8f2ce0640 1 > prepared operation key=ZTK9AYBSLCOZ tag=HF1HM0ZYSA68 type=write size=931584 2024-09-13T00:01:07.696 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.693+0000 7fb8f2ce0640 -1 < failed to complete operation key=QMZREZ1QEDNJ tag=M99OZ7K8J3LX type=del size=1037273: (22) Invalid argument 2024-09-13T00:01:07.706 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.705+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NX26SI4F2ZMB.2~C5OKYGJG6PS5.0 size=1048576 2024-09-13T00:01:07.711 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.709+0000 7fb8f2ce0640 1 > prepared part key=_multipart_NX26SI4F2ZMB.2~C5OKYGJG6PS5.1 size=875130 2024-09-13T00:01:07.711 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.709+0000 7fb8f2ce0640 1 > prepared multipart upload key=NX26SI4F2ZMB upload=C5OKYGJG6PS5 tag=V6MZKTVO7XUP type=write size=2972282 2024-09-13T00:01:07.717 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.717+0000 7fb8f2ce0640 1 < completed delete operation key=P8OYS5ZE1NX0 tag=EOW7TL7K05D7 type=del 2024-09-13T00:01:07.730 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.729+0000 7fb8f2ce0640 1 > prepared part key=_multipart_85YEW8N43FTA.2~ZY236UBPZNW3.0 size=1048576 2024-09-13T00:01:07.735 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.733+0000 7fb8f2ce0640 1 > prepared part key=_multipart_85YEW8N43FTA.2~ZY236UBPZNW3.1 size=1048576 2024-09-13T00:01:07.740 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.737+0000 7fb8f2ce0640 1 > prepared part key=_multipart_85YEW8N43FTA.2~ZY236UBPZNW3.2 size=468094 2024-09-13T00:01:07.740 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.737+0000 7fb8f2ce0640 1 > prepared multipart upload key=85YEW8N43FTA upload=ZY236UBPZNW3 tag=R5ZE1MNNES8X type=write size=3613822 2024-09-13T00:01:07.746 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.745+0000 7fb8f2ce0640 1 < completed delete operation key=2AHYRZUDFHP4 tag=VSCWJZKYJQ0W type=del 2024-09-13T00:01:07.757 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.757+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~4UBKDEXH769T.0 size=1048576 2024-09-13T00:01:07.761 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.761+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~4UBKDEXH769T.1 size=1048576 2024-09-13T00:01:07.766 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.765+0000 7fb8f2ce0640 1 > prepared part key=_multipart_63FJDWBO2T9S.2~4UBKDEXH769T.2 size=28961 2024-09-13T00:01:07.766 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.765+0000 7fb8f2ce0640 1 > prepared multipart upload key=63FJDWBO2T9S upload=4UBKDEXH769T tag=CXOOP97CUGKU type=write size=3174689 2024-09-13T00:01:07.772 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.769+0000 7fb8f2ce0640 1 < completed delete operation key=0WC6P8YNL0H2 tag=RECY34A452BF type=del 2024-09-13T00:01:07.778 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.777+0000 7fb8f2ce0640 1 > prepared operation key=EUE3LT0PDIXV tag=7T3PLMRN976C type=del size=970724 2024-09-13T00:01:07.790 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.789+0000 7fb8f2ce0640 1 < suggested update operation key=PVYZQLGYT3XU tag=PFMB57NIVST2 type=write size=884914 2024-09-13T00:01:07.796 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.793+0000 7fb8f2ce0640 1 > prepared operation key=H5EIP0E9CBWX tag=HNDMRI6UNVGR type=del size=200391 2024-09-13T00:01:07.802 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.801+0000 7fb8f2ce0640 1 < completed part key=_multipart_O4MZ4UWNXME9.2~W7TNUDCR97Y9.0 size=812790 2024-09-13T00:01:07.813 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.809+0000 7fb8f2ce0640 1 < completed multipart upload key=O4MZ4UWNXME9 upload=W7TNUDCR97Y9 tag=EEERW1HPGOGS type=write size=1861366 2024-09-13T00:01:07.819 INFO:tasks.workunit.client.0.smithi136.stderr:2024-09-13T00:01:07.817+0000 7fb8f2ce0640 1 > prepared operation key=H9AU4HD247YU tag=ENNNKEKKKS8I type=del size=1026477 2024-09-13T00:01:07.819 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] cls_rgw_stats.simulate (37136 ms) 2024-09-13T00:01:07.820 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 1 test from cls_rgw_stats (37136 ms total) 2024-09-13T00:01:07.820 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:01:07.820 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] Global test environment tear-down 2024-09-13T00:01:07.975 INFO:tasks.workunit.client.0.smithi136.stdout:[==========] 1 test from 1 test suite ran. (39183 ms total) 2024-09-13T00:01:07.976 INFO:tasks.workunit.client.0.smithi136.stdout:[ PASSED ] 1 test. 2024-09-13T00:01:07.985 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-13T00:01:07.985 DEBUG:teuthology.orchestra.run.smithi136:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-13T00:01:08.041 INFO:tasks.workunit:Running workunit cls/test_cls_sdk.sh... 2024-09-13T00:01:08.041 DEBUG:teuthology.orchestra.run.smithi136:workunit test cls/test_cls_sdk.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_sdk.sh 2024-09-13T00:01:08.103 INFO:tasks.workunit.client.0.smithi136.stdout:Running main() from gmock_main.cc 2024-09-13T00:01:08.103 INFO:tasks.workunit.client.0.smithi136.stdout:[==========] Running 2 tests from 1 test suite. 2024-09-13T00:01:08.103 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] Global test environment set-up. 2024-09-13T00:01:08.103 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 2 tests from ClsSDK 2024-09-13T00:01:08.104 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] ClsSDK.TestSDKCoverageWrite 2024-09-13T00:01:09.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:01:08 smithi136 bash[75164]: cluster 2024-09-13T00:01:07.144930+0000 mgr.y (mgr.24661) 720 : cluster [DBG] pgmap v1396: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 505 KiB/s rd, 174 KiB/s wr, 894 op/s 2024-09-13T00:01:09.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:01:08 smithi136 bash[75164]: cluster 2024-09-13T00:01:07.969101+0000 mon.a (mon.0) 2080 : cluster [DBG] osdmap e1116: 8 total, 8 up, 8 in 2024-09-13T00:01:09.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:01:08 smithi136 bash[75164]: audit 2024-09-13T00:01:08.770626+0000 mon.a (mon.0) 2081 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:01:09.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:01:08 smithi136 bash[76371]: cluster 2024-09-13T00:01:07.144930+0000 mgr.y (mgr.24661) 720 : cluster [DBG] pgmap v1396: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 505 KiB/s rd, 174 KiB/s wr, 894 op/s 2024-09-13T00:01:09.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:01:08 smithi136 bash[76371]: cluster 2024-09-13T00:01:07.969101+0000 mon.a (mon.0) 2080 : cluster [DBG] osdmap e1116: 8 total, 8 up, 8 in 2024-09-13T00:01:09.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:01:08 smithi136 bash[76371]: audit 2024-09-13T00:01:08.770626+0000 mon.a (mon.0) 2081 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:01:09.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:01:08 smithi181 bash[67587]: cluster 2024-09-13T00:01:07.144930+0000 mgr.y (mgr.24661) 720 : cluster [DBG] pgmap v1396: 137 pgs: 137 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 505 KiB/s rd, 174 KiB/s wr, 894 op/s 2024-09-13T00:01:09.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:01:08 smithi181 bash[67587]: cluster 2024-09-13T00:01:07.969101+0000 mon.a (mon.0) 2080 : cluster [DBG] osdmap e1116: 8 total, 8 up, 8 in 2024-09-13T00:01:09.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:01:08 smithi181 bash[67587]: audit 2024-09-13T00:01:08.770626+0000 mon.a (mon.0) 2081 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:01:09.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:01:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:01:09] "GET /metrics HTTP/1.1" 200 34085 "" "Prometheus/2.43.0" 2024-09-13T00:01:10.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:01:09 smithi136 bash[75164]: cluster 2024-09-13T00:01:08.989712+0000 mon.a (mon.0) 2082 : cluster [DBG] osdmap e1117: 8 total, 8 up, 8 in 2024-09-13T00:01:10.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:01:09 smithi136 bash[75164]: audit 2024-09-13T00:01:08.992311+0000 mon.c (mon.2) 92 : audit [INF] from='client.? 172.21.15.136:0/1228299569' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-111034-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:01:10.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:01:09 smithi136 bash[75164]: audit 2024-09-13T00:01:08.993462+0000 mon.a (mon.0) 2083 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-111034-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:01:10.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:01:10 smithi136 bash[76371]: cluster 2024-09-13T00:01:08.989712+0000 mon.a (mon.0) 2082 : cluster [DBG] osdmap e1117: 8 total, 8 up, 8 in 2024-09-13T00:01:10.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:01:10 smithi136 bash[76371]: audit 2024-09-13T00:01:08.992311+0000 mon.c (mon.2) 92 : audit [INF] from='client.? 172.21.15.136:0/1228299569' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-111034-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:01:10.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:01:10 smithi136 bash[76371]: audit 2024-09-13T00:01:08.993462+0000 mon.a (mon.0) 2083 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-111034-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:01:10.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:01:09 smithi181 bash[67587]: cluster 2024-09-13T00:01:08.989712+0000 mon.a (mon.0) 2082 : cluster [DBG] osdmap e1117: 8 total, 8 up, 8 in 2024-09-13T00:01:10.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:01:09 smithi181 bash[67587]: audit 2024-09-13T00:01:08.992311+0000 mon.c (mon.2) 92 : audit [INF] from='client.? 172.21.15.136:0/1228299569' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-111034-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:01:10.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:01:09 smithi181 bash[67587]: audit 2024-09-13T00:01:08.993462+0000 mon.a (mon.0) 2083 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-111034-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:01:11.023 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] ClsSDK.TestSDKCoverageWrite (2919 ms) 2024-09-13T00:01:11.023 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] ClsSDK.TestSDKCoverageReplay 2024-09-13T00:01:11.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:01:11 smithi136 bash[75164]: cluster 2024-09-13T00:01:09.145659+0000 mgr.y (mgr.24661) 721 : cluster [DBG] pgmap v1399: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:01:11.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:01:11 smithi136 bash[75164]: cluster 2024-09-13T00:01:09.983785+0000 mon.a (mon.0) 2084 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:01:11.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:01:11 smithi136 bash[75164]: audit 2024-09-13T00:01:09.993991+0000 mon.a (mon.0) 2085 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-111034-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:01:11.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:01:11 smithi136 bash[75164]: cluster 2024-09-13T00:01:09.995004+0000 mon.a (mon.0) 2086 : cluster [DBG] osdmap e1118: 8 total, 8 up, 8 in 2024-09-13T00:01:11.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:01:11 smithi136 bash[76371]: cluster 2024-09-13T00:01:09.145659+0000 mgr.y (mgr.24661) 721 : cluster [DBG] pgmap v1399: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:01:11.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:01:11 smithi136 bash[76371]: cluster 2024-09-13T00:01:09.983785+0000 mon.a (mon.0) 2084 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:01:11.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:01:11 smithi136 bash[76371]: audit 2024-09-13T00:01:09.993991+0000 mon.a (mon.0) 2085 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-111034-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:01:11.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:01:11 smithi136 bash[76371]: cluster 2024-09-13T00:01:09.995004+0000 mon.a (mon.0) 2086 : cluster [DBG] osdmap e1118: 8 total, 8 up, 8 in 2024-09-13T00:01:11.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:01:11 smithi181 bash[67587]: cluster 2024-09-13T00:01:09.145659+0000 mgr.y (mgr.24661) 721 : cluster [DBG] pgmap v1399: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:01:11.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:01:11 smithi181 bash[67587]: cluster 2024-09-13T00:01:09.983785+0000 mon.a (mon.0) 2084 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:01:11.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:01:11 smithi181 bash[67587]: audit 2024-09-13T00:01:09.993991+0000 mon.a (mon.0) 2085 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-111034-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:01:11.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:01:11 smithi181 bash[67587]: cluster 2024-09-13T00:01:09.995004+0000 mon.a (mon.0) 2086 : cluster [DBG] osdmap e1118: 8 total, 8 up, 8 in 2024-09-13T00:01:12.314 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:01:12 smithi181 bash[67587]: cluster 2024-09-13T00:01:11.015041+0000 mon.a (mon.0) 2087 : cluster [DBG] osdmap e1119: 8 total, 8 up, 8 in 2024-09-13T00:01:12.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:01:12 smithi136 bash[75164]: cluster 2024-09-13T00:01:11.015041+0000 mon.a (mon.0) 2087 : cluster [DBG] osdmap e1119: 8 total, 8 up, 8 in 2024-09-13T00:01:12.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:01:12 smithi136 bash[76371]: cluster 2024-09-13T00:01:11.015041+0000 mon.a (mon.0) 2087 : cluster [DBG] osdmap e1119: 8 total, 8 up, 8 in 2024-09-13T00:01:12.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:01:12 smithi181 bash[95505]: ts=2024-09-13T00:01:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:01:13.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:01:13 smithi181 bash[67587]: cluster 2024-09-13T00:01:11.146229+0000 mgr.y (mgr.24661) 722 : cluster [DBG] pgmap v1402: 105 pgs: 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:01:13.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:01:13 smithi181 bash[67587]: cluster 2024-09-13T00:01:12.008707+0000 mon.a (mon.0) 2088 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:01:13.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:01:13 smithi181 bash[67587]: cluster 2024-09-13T00:01:12.008789+0000 mon.a (mon.0) 2089 : cluster [INF] Cluster is now healthy 2024-09-13T00:01:13.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:01:13 smithi181 bash[67587]: cluster 2024-09-13T00:01:12.031983+0000 mon.a (mon.0) 2090 : cluster [DBG] osdmap e1120: 8 total, 8 up, 8 in 2024-09-13T00:01:13.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:01:13 smithi181 bash[67587]: audit 2024-09-13T00:01:12.034096+0000 mon.a (mon.0) 2091 : audit [INF] from='client.? 172.21.15.136:0/3837194697' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-111034-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:01:13.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:01:13 smithi136 bash[75164]: cluster 2024-09-13T00:01:11.146229+0000 mgr.y (mgr.24661) 722 : cluster [DBG] pgmap v1402: 105 pgs: 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:01:13.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:01:13 smithi136 bash[75164]: cluster 2024-09-13T00:01:12.008707+0000 mon.a (mon.0) 2088 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:01:13.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:01:13 smithi136 bash[75164]: cluster 2024-09-13T00:01:12.008789+0000 mon.a (mon.0) 2089 : cluster [INF] Cluster is now healthy 2024-09-13T00:01:13.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:01:13 smithi136 bash[75164]: cluster 2024-09-13T00:01:12.031983+0000 mon.a (mon.0) 2090 : cluster [DBG] osdmap e1120: 8 total, 8 up, 8 in 2024-09-13T00:01:13.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:01:13 smithi136 bash[75164]: audit 2024-09-13T00:01:12.034096+0000 mon.a (mon.0) 2091 : audit [INF] from='client.? 172.21.15.136:0/3837194697' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-111034-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:01:13.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:01:13 smithi136 bash[76371]: cluster 2024-09-13T00:01:11.146229+0000 mgr.y (mgr.24661) 722 : cluster [DBG] pgmap v1402: 105 pgs: 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:01:13.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:01:13 smithi136 bash[76371]: cluster 2024-09-13T00:01:12.008707+0000 mon.a (mon.0) 2088 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:01:13.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:01:13 smithi136 bash[76371]: cluster 2024-09-13T00:01:12.008789+0000 mon.a (mon.0) 2089 : cluster [INF] Cluster is now healthy 2024-09-13T00:01:13.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:01:13 smithi136 bash[76371]: cluster 2024-09-13T00:01:12.031983+0000 mon.a (mon.0) 2090 : cluster [DBG] osdmap e1120: 8 total, 8 up, 8 in 2024-09-13T00:01:13.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:01:13 smithi136 bash[76371]: audit 2024-09-13T00:01:12.034096+0000 mon.a (mon.0) 2091 : audit [INF] from='client.? 172.21.15.136:0/3837194697' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-111034-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:01:14.053 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] ClsSDK.TestSDKCoverageReplay (3030 ms) 2024-09-13T00:01:14.053 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 2 tests from ClsSDK (5949 ms total) 2024-09-13T00:01:14.053 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:01:14.053 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] Global test environment tear-down 2024-09-13T00:01:14.053 INFO:tasks.workunit.client.0.smithi136.stdout:[==========] 2 tests from 1 test suite ran. (5949 ms total) 2024-09-13T00:01:14.053 INFO:tasks.workunit.client.0.smithi136.stdout:[ PASSED ] 2 tests. 2024-09-13T00:01:14.057 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-13T00:01:14.057 DEBUG:teuthology.orchestra.run.smithi136:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-13T00:01:14.116 INFO:tasks.workunit:Stopping ['cls'] on client.0... 2024-09-13T00:01:14.116 DEBUG:teuthology.orchestra.run.smithi136:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2024-09-13T00:01:14.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:01:14 smithi181 bash[67587]: audit 2024-09-13T00:01:13.039516+0000 mon.a (mon.0) 2092 : audit [INF] from='client.? 172.21.15.136:0/3837194697' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-111034-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:01:14.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:01:14 smithi181 bash[67587]: cluster 2024-09-13T00:01:13.039982+0000 mon.a (mon.0) 2093 : cluster [DBG] osdmap e1121: 8 total, 8 up, 8 in 2024-09-13T00:01:14.382 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:01:14 smithi136 bash[76371]: audit 2024-09-13T00:01:13.039516+0000 mon.a (mon.0) 2092 : audit [INF] from='client.? 172.21.15.136:0/3837194697' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-111034-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:01:14.382 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:01:14 smithi136 bash[76371]: cluster 2024-09-13T00:01:13.039982+0000 mon.a (mon.0) 2093 : cluster [DBG] osdmap e1121: 8 total, 8 up, 8 in 2024-09-13T00:01:14.382 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:01:14 smithi136 bash[75164]: audit 2024-09-13T00:01:13.039516+0000 mon.a (mon.0) 2092 : audit [INF] from='client.? 172.21.15.136:0/3837194697' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi136-111034-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:01:14.382 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:01:14 smithi136 bash[75164]: cluster 2024-09-13T00:01:13.039982+0000 mon.a (mon.0) 2093 : cluster [DBG] osdmap e1121: 8 total, 8 up, 8 in 2024-09-13T00:01:14.383 DEBUG:teuthology.parallel:result is None 2024-09-13T00:01:14.384 DEBUG:teuthology.orchestra.run.smithi136:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2024-09-13T00:01:14.396 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2024-09-13T00:01:14.396 DEBUG:teuthology.orchestra.run.smithi136:> rmdir -- /home/ubuntu/cephtest/mnt.0 2024-09-13T00:01:14.441 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2024-09-13T00:01:14.441 INFO:teuthology.task.full_sequential:In full_sequential, running task print... 2024-09-13T00:01:14.441 INFO:teuthology.task.print:**** done end rados_api.yaml 2024-09-13T00:01:14.441 INFO:teuthology.task.full_sequential:In full_sequential, running task print... 2024-09-13T00:01:14.442 INFO:teuthology.task.print:**** done start rados_loadgenbig.yaml 2024-09-13T00:01:14.442 INFO:teuthology.task.full_sequential:In full_sequential, running task workunit... 2024-09-13T00:01:14.443 INFO:tasks.workunit:Pulling workunits from ref quincy 2024-09-13T00:01:14.443 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-09-13T00:01:14.443 DEBUG:teuthology.orchestra.run.smithi136:> stat -- /home/ubuntu/cephtest/mnt.0 2024-09-13T00:01:14.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T00:01:14.491 INFO:teuthology.orchestra.run.smithi136.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2024-09-13T00:01:14.491 DEBUG:teuthology.orchestra.run.smithi136:> mkdir -- /home/ubuntu/cephtest/mnt.0 2024-09-13T00:01:14.538 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2024-09-13T00:01:14.538 DEBUG:teuthology.orchestra.run.smithi136:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2024-09-13T00:01:14.586 INFO:tasks.workunit:timeout=3h 2024-09-13T00:01:14.586 INFO:tasks.workunit:cleanup=True 2024-09-13T00:01:14.586 DEBUG:teuthology.orchestra.run.smithi136:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://git.ceph.com/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2024-09-13T00:01:14.636 INFO:tasks.workunit.client.0.smithi136.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-09-13T00:01:15.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:01:15 smithi181 bash[67587]: cluster 2024-09-13T00:01:13.146936+0000 mgr.y (mgr.24661) 723 : cluster [DBG] pgmap v1405: 137 pgs: 1 creating+peering, 31 unknown, 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:01:15.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:01:15 smithi181 bash[67587]: cluster 2024-09-13T00:01:14.049190+0000 mon.a (mon.0) 2094 : cluster [DBG] osdmap e1122: 8 total, 8 up, 8 in 2024-09-13T00:01:15.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:01:15 smithi136 bash[75164]: cluster 2024-09-13T00:01:13.146936+0000 mgr.y (mgr.24661) 723 : cluster [DBG] pgmap v1405: 137 pgs: 1 creating+peering, 31 unknown, 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:01:15.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:01:15 smithi136 bash[75164]: cluster 2024-09-13T00:01:14.049190+0000 mon.a (mon.0) 2094 : cluster [DBG] osdmap e1122: 8 total, 8 up, 8 in 2024-09-13T00:01:15.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:01:15 smithi136 bash[76371]: cluster 2024-09-13T00:01:13.146936+0000 mgr.y (mgr.24661) 723 : cluster [DBG] pgmap v1405: 137 pgs: 1 creating+peering, 31 unknown, 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:01:15.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:01:15 smithi136 bash[76371]: cluster 2024-09-13T00:01:14.049190+0000 mon.a (mon.0) 2094 : cluster [DBG] osdmap e1122: 8 total, 8 up, 8 in 2024-09-13T00:01:16.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:01:16 smithi181 bash[67587]: cluster 2024-09-13T00:01:15.147688+0000 mgr.y (mgr.24661) 724 : cluster [DBG] pgmap v1407: 105 pgs: 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:01:16.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:01:16 smithi136 bash[75164]: cluster 2024-09-13T00:01:15.147688+0000 mgr.y (mgr.24661) 724 : cluster [DBG] pgmap v1407: 105 pgs: 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:01:16.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:01:16 smithi136 bash[76371]: cluster 2024-09-13T00:01:15.147688+0000 mgr.y (mgr.24661) 724 : cluster [DBG] pgmap v1407: 105 pgs: 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:01:18.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:01:18 smithi136 bash[75164]: cluster 2024-09-13T00:01:17.148839+0000 mgr.y (mgr.24661) 725 : cluster [DBG] pgmap v1408: 105 pgs: 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:01:18.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:01:18 smithi136 bash[76371]: cluster 2024-09-13T00:01:17.148839+0000 mgr.y (mgr.24661) 725 : cluster [DBG] pgmap v1408: 105 pgs: 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:01:18.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:01:18 smithi181 bash[67587]: cluster 2024-09-13T00:01:17.148839+0000 mgr.y (mgr.24661) 725 : cluster [DBG] pgmap v1408: 105 pgs: 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:01:19.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:01:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:01:19] "GET /metrics HTTP/1.1" 200 34085 "" "Prometheus/2.43.0" 2024-09-13T00:01:20.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:01:20 smithi136 bash[76371]: cluster 2024-09-13T00:01:19.149725+0000 mgr.y (mgr.24661) 726 : cluster [DBG] pgmap v1409: 105 pgs: 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:01:20.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:01:20 smithi136 bash[75164]: cluster 2024-09-13T00:01:19.149725+0000 mgr.y (mgr.24661) 726 : cluster [DBG] pgmap v1409: 105 pgs: 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:01:20.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:01:20 smithi181 bash[67587]: cluster 2024-09-13T00:01:19.149725+0000 mgr.y (mgr.24661) 726 : cluster [DBG] pgmap v1409: 105 pgs: 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:01:20.931 DEBUG:teuthology.orchestra.run.smithi136:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-09-13T00:01:20.979 INFO:tasks.workunit.client.0.smithi136.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-09-13T00:01:20.982 INFO:tasks.workunit.client.0.smithi136.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-13T00:01:20.982 INFO:tasks.workunit.client.0.smithi136.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-09-13T00:01:21.038 INFO:tasks.workunit.client.0.smithi136.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-09-13T00:01:21.094 INFO:tasks.workunit.client.0.smithi136.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-09-13T00:01:21.140 INFO:tasks.workunit.client.0.smithi136.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-13T00:01:21.141 INFO:tasks.workunit.client.0.smithi136.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-13T00:01:21.141 INFO:tasks.workunit.client.0.smithi136.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-09-13T00:01:21.186 INFO:tasks.workunit.client.0.smithi136.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-13T00:01:21.189 DEBUG:teuthology.orchestra.run.smithi136:> set -ex 2024-09-13T00:01:21.189 DEBUG:teuthology.orchestra.run.smithi136:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-09-13T00:01:21.238 INFO:tasks.workunit:Running workunits matching rados/load-gen-big.sh on client.0... 2024-09-13T00:01:21.239 INFO:tasks.workunit:Running workunit rados/load-gen-big.sh... 2024-09-13T00:01:21.239 DEBUG:teuthology.orchestra.run.smithi136:workunit test rados/load-gen-big.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rados/load-gen-big.sh 2024-09-13T00:01:21.335 INFO:tasks.workunit.client.0.smithi136.stdout:run length 1200 seconds 2024-09-13T00:01:21.336 INFO:tasks.workunit.client.0.smithi136.stdout:preparing 10240 objects 2024-09-13T00:01:22.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:01:22 smithi136 bash[75164]: cluster 2024-09-13T00:01:21.150280+0000 mgr.y (mgr.24661) 727 : cluster [DBG] pgmap v1410: 105 pgs: 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:01:22.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:01:22 smithi136 bash[76371]: cluster 2024-09-13T00:01:21.150280+0000 mgr.y (mgr.24661) 727 : cluster [DBG] pgmap v1410: 105 pgs: 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:01:22.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:01:22 smithi181 bash[67587]: cluster 2024-09-13T00:01:21.150280+0000 mgr.y (mgr.24661) 727 : cluster [DBG] pgmap v1410: 105 pgs: 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:01:22.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:01:22 smithi181 bash[95505]: ts=2024-09-13T00:01:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:01:23.389 INFO:tasks.workunit.client.0.smithi136.stdout:load-gen will run 1200 seconds 2024-09-13T00:01:24.389 INFO:tasks.workunit.client.0.smithi136.stdout: 1: throughput=0MB/sec pending data=0 2024-09-13T00:01:24.389 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-_0pCNiFIGD0UGDm off=9686703 len=1289673 2024-09-13T00:01:24.393 INFO:tasks.workunit.client.0.smithi136.stdout:op 0 completed, throughput=1.22MB/sec 2024-09-13T00:01:24.394 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-YLqu8emunEYo_pY off=5187673 len=1619894 2024-09-13T00:01:24.406 INFO:tasks.workunit.client.0.smithi136.stdout:op 1 completed, throughput=2.73MB/sec 2024-09-13T00:01:24.406 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Lc5CUTvXz7YCB22 off=10574431 len=1575671 2024-09-13T00:01:24.411 INFO:tasks.workunit.client.0.smithi136.stdout:op 2 completed, throughput=4.18MB/sec 2024-09-13T00:01:24.411 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj--GJUDLGKJ7v5OZr off=9591294 len=621098 2024-09-13T00:01:24.419 INFO:tasks.workunit.client.0.smithi136.stdout:op 3 completed, throughput=4.73MB/sec 2024-09-13T00:01:24.419 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-3wCOxqNCIpPHao_ off=728677 len=854600 2024-09-13T00:01:24.427 INFO:tasks.workunit.client.0.smithi136.stdout:op 4 completed, throughput=5.47MB/sec 2024-09-13T00:01:24.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:01:24 smithi136 bash[75164]: cluster 2024-09-13T00:01:23.150755+0000 mgr.y (mgr.24661) 728 : cluster [DBG] pgmap v1411: 105 pgs: 105 active+clean; 9.8 GiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 79 KiB/s wr, 78 op/s 2024-09-13T00:01:24.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:01:24 smithi136 bash[75164]: audit 2024-09-13T00:01:23.771104+0000 mon.a (mon.0) 2095 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:01:24.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:01:24 smithi136 bash[76371]: cluster 2024-09-13T00:01:23.150755+0000 mgr.y (mgr.24661) 728 : cluster [DBG] pgmap v1411: 105 pgs: 105 active+clean; 9.8 GiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 79 KiB/s wr, 78 op/s 2024-09-13T00:01:24.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:01:24 smithi136 bash[76371]: audit 2024-09-13T00:01:23.771104+0000 mon.a (mon.0) 2095 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:01:24.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:01:24 smithi181 bash[67587]: cluster 2024-09-13T00:01:23.150755+0000 mgr.y (mgr.24661) 728 : cluster [DBG] pgmap v1411: 105 pgs: 105 active+clean; 9.8 GiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 79 KiB/s wr, 78 op/s 2024-09-13T00:01:24.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:01:24 smithi181 bash[67587]: audit 2024-09-13T00:01:23.771104+0000 mon.a (mon.0) 2095 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:01:25.428 INFO:tasks.workunit.client.0.smithi136.stdout: 2: throughput=2.79MB/sec pending data=0 2024-09-13T00:01:25.428 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Cro4zs88Sh2u4Dr off=18002279 len=789371 2024-09-13T00:01:25.433 INFO:tasks.workunit.client.0.smithi136.stdout:op 5 completed, throughput=3.15MB/sec 2024-09-13T00:01:25.433 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-0Hb_Lid62p9vuWF off=3881833 len=1102121 2024-09-13T00:01:25.437 INFO:tasks.workunit.client.0.smithi136.stdout:op 6 completed, throughput=3.66MB/sec 2024-09-13T00:01:25.437 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-haPAeGysRvCyXrw off=7446758 len=2032256 2024-09-13T00:01:25.442 INFO:tasks.workunit.client.0.smithi136.stdout:op 7 completed, throughput=4.59MB/sec 2024-09-13T00:01:25.442 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-30GHqkf67fKAwz- off=9672053 len=1134751 2024-09-13T00:01:25.446 INFO:tasks.workunit.client.0.smithi136.stdout:op 8 completed, throughput=5.11MB/sec 2024-09-13T00:01:26.446 INFO:tasks.workunit.client.0.smithi136.stdout: 3: throughput=3.44MB/sec pending data=0 2024-09-13T00:01:26.447 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-cpnBl4C5a0_GDVv off=19910011 len=1111281 2024-09-13T00:01:26.449 INFO:tasks.workunit.client.0.smithi136.stdout:op 9 completed, throughput=3.78MB/sec 2024-09-13T00:01:26.449 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-2W7QmQd9B8G_s5w off=4218926 len=1993961 2024-09-13T00:01:26.464 INFO:tasks.workunit.client.0.smithi136.stdout:op 10 completed, throughput=4.38MB/sec 2024-09-13T00:01:26.464 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-rKBHBPSFELrb_nN off=8462066 len=767037 2024-09-13T00:01:26.473 INFO:tasks.workunit.client.0.smithi136.stdout:op 11 completed, throughput=4.61MB/sec 2024-09-13T00:01:26.473 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ENMfrXmrLTx5g4W off=870297 len=1573708 2024-09-13T00:01:26.477 INFO:tasks.workunit.client.0.smithi136.stdout:op 12 completed, throughput=5.09MB/sec 2024-09-13T00:01:26.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:01:26 smithi136 bash[75164]: cluster 2024-09-13T00:01:25.151603+0000 mgr.y (mgr.24661) 729 : cluster [DBG] pgmap v1412: 105 pgs: 105 active+clean; 57 GiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 414 KiB/s wr, 413 op/s 2024-09-13T00:01:26.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:01:26 smithi136 bash[76371]: cluster 2024-09-13T00:01:25.151603+0000 mgr.y (mgr.24661) 729 : cluster [DBG] pgmap v1412: 105 pgs: 105 active+clean; 57 GiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 414 KiB/s wr, 413 op/s 2024-09-13T00:01:26.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:01:26 smithi181 bash[67587]: cluster 2024-09-13T00:01:25.151603+0000 mgr.y (mgr.24661) 729 : cluster [DBG] pgmap v1412: 105 pgs: 105 active+clean; 57 GiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 414 KiB/s wr, 413 op/s 2024-09-13T00:01:27.477 INFO:tasks.workunit.client.0.smithi136.stdout: 4: throughput=3.84MB/sec pending data=0 2024-09-13T00:01:27.477 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-cfkWvbxgQ-CApty off=9453678 len=481664 2024-09-13T00:01:27.479 INFO:tasks.workunit.client.0.smithi136.stdout:op 13 completed, throughput=3.95MB/sec 2024-09-13T00:01:27.479 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-lHDxzzgZlnUddTL off=23254180 len=735084 2024-09-13T00:01:27.486 INFO:tasks.workunit.client.0.smithi136.stdout:op 14 completed, throughput=4.12MB/sec 2024-09-13T00:01:27.486 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-1D7DymcpfUxqz1X off=151927 len=677574 2024-09-13T00:01:27.488 INFO:tasks.workunit.client.0.smithi136.stdout:op 15 completed, throughput=4.27MB/sec 2024-09-13T00:01:27.488 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-rPDfD77M7RgaVIe off=7433250 len=682617 2024-09-13T00:01:27.495 INFO:tasks.workunit.client.0.smithi136.stdout:op 16 completed, throughput=4.42MB/sec 2024-09-13T00:01:27.495 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-kzt_Nj7z3O10Nwh off=2303488 len=510028 2024-09-13T00:01:27.502 INFO:tasks.workunit.client.0.smithi136.stdout:op 17 completed, throughput=4.53MB/sec 2024-09-13T00:01:27.502 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-jkl1-ERuRYpwJLl off=5276176 len=956252 2024-09-13T00:01:27.507 INFO:tasks.workunit.client.0.smithi136.stdout:op 18 completed, throughput=4.75MB/sec 2024-09-13T00:01:27.508 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Lhe2yOHLdGQ-RB5 off=8248852 len=1008326 2024-09-13T00:01:27.517 INFO:tasks.workunit.client.0.smithi136.stdout:op 19 completed, throughput=4.97MB/sec 2024-09-13T00:01:27.517 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ccF7ntrdvY0yWYE off=14376376 len=870202 2024-09-13T00:01:27.520 INFO:tasks.workunit.client.0.smithi136.stdout:op 20 completed, throughput=5.17MB/sec 2024-09-13T00:01:28.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:01:28 smithi136 bash[75164]: cluster 2024-09-13T00:01:27.152749+0000 mgr.y (mgr.24661) 730 : cluster [DBG] pgmap v1413: 105 pgs: 105 active+clean; 126 GiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 645 KiB/s rd, 1.1 MiB/s wr, 853 op/s 2024-09-13T00:01:28.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:01:28 smithi136 bash[76371]: cluster 2024-09-13T00:01:27.152749+0000 mgr.y (mgr.24661) 730 : cluster [DBG] pgmap v1413: 105 pgs: 105 active+clean; 126 GiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 645 KiB/s rd, 1.1 MiB/s wr, 853 op/s 2024-09-13T00:01:28.520 INFO:tasks.workunit.client.0.smithi136.stdout: 5: throughput=4.16MB/sec pending data=0 2024-09-13T00:01:28.520 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-eodJoVEe9LeG4u6 off=19201177 len=833168 2024-09-13T00:01:28.523 INFO:tasks.workunit.client.0.smithi136.stdout:op 21 completed, throughput=4.31MB/sec 2024-09-13T00:01:28.523 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-EsIASGR-a6t9OcN off=14960387 len=1756112 2024-09-13T00:01:28.539 INFO:tasks.workunit.client.0.smithi136.stdout:op 22 completed, throughput=4.63MB/sec 2024-09-13T00:01:28.539 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-CH4UW-OioDeqzj2 off=897701 len=2008468 2024-09-13T00:01:28.544 INFO:tasks.workunit.client.0.smithi136.stdout:op 23 completed, throughput=4.99MB/sec 2024-09-13T00:01:28.544 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-VFRr8zoHxUiHrX3 off=9600207 len=1467008 2024-09-13T00:01:28.548 INFO:tasks.workunit.client.0.smithi136.stdout:op 24 completed, throughput=5.26MB/sec 2024-09-13T00:01:28.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:01:28 smithi181 bash[67587]: cluster 2024-09-13T00:01:27.152749+0000 mgr.y (mgr.24661) 730 : cluster [DBG] pgmap v1413: 105 pgs: 105 active+clean; 126 GiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 645 KiB/s rd, 1.1 MiB/s wr, 853 op/s 2024-09-13T00:01:29.548 INFO:tasks.workunit.client.0.smithi136.stdout: 6: throughput=4.41MB/sec pending data=0 2024-09-13T00:01:29.549 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-B_kqt_9sudj_4VR off=14509760 len=173827 2024-09-13T00:01:29.550 INFO:tasks.workunit.client.0.smithi136.stdout:op 25 completed, throughput=4.43MB/sec 2024-09-13T00:01:29.550 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-opo7fIkBkH0sI-L off=5946211 len=1842338 2024-09-13T00:01:29.554 INFO:tasks.workunit.client.0.smithi136.stdout:op 26 completed, throughput=4.71MB/sec 2024-09-13T00:01:29.554 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-l5JVPLDt15k8LRC off=25246268 len=279778 2024-09-13T00:01:29.556 INFO:tasks.workunit.client.0.smithi136.stdout:op 27 completed, throughput=4.76MB/sec 2024-09-13T00:01:29.556 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-v6cLDT4FxTchIqH off=4673180 len=1176301 2024-09-13T00:01:29.559 INFO:tasks.workunit.client.0.smithi136.stdout:op 28 completed, throughput=4.93MB/sec 2024-09-13T00:01:29.559 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-vsg2dEbbmDKb1_g off=314919 len=1391186 2024-09-13T00:01:29.570 INFO:tasks.workunit.client.0.smithi136.stdout:op 29 completed, throughput=5.14MB/sec 2024-09-13T00:01:29.761 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:01:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:01:29] "GET /metrics HTTP/1.1" 200 34095 "" "Prometheus/2.43.0" 2024-09-13T00:01:30.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:01:30 smithi136 bash[75164]: cluster 2024-09-13T00:01:29.153642+0000 mgr.y (mgr.24661) 731 : cluster [DBG] pgmap v1414: 105 pgs: 105 active+clean; 126 GiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 918 KiB/s rd, 1.3 MiB/s wr, 854 op/s 2024-09-13T00:01:30.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:01:30 smithi136 bash[76371]: cluster 2024-09-13T00:01:29.153642+0000 mgr.y (mgr.24661) 731 : cluster [DBG] pgmap v1414: 105 pgs: 105 active+clean; 126 GiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 918 KiB/s rd, 1.3 MiB/s wr, 854 op/s 2024-09-13T00:01:30.571 INFO:tasks.workunit.client.0.smithi136.stdout: 7: throughput=4.42MB/sec pending data=0 2024-09-13T00:01:30.571 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj--oWhqIUwN8XU52A off=4656620 len=978097 2024-09-13T00:01:30.574 INFO:tasks.workunit.client.0.smithi136.stdout:op 30 completed, throughput=4.55MB/sec 2024-09-13T00:01:30.574 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-QrWU-mXOPiS9UuL off=14358438 len=102572 2024-09-13T00:01:30.574 INFO:tasks.workunit.client.0.smithi136.stdout:op 31 completed, throughput=4.56MB/sec 2024-09-13T00:01:30.574 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-UXHi-b7cQSdNOV3 off=13607378 len=1940425 2024-09-13T00:01:30.579 INFO:tasks.workunit.client.0.smithi136.stdout:op 32 completed, throughput=4.82MB/sec 2024-09-13T00:01:30.579 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-p0JqCzTBAesGpN8 off=4959419 len=571595 2024-09-13T00:01:30.585 INFO:tasks.workunit.client.0.smithi136.stdout:op 33 completed, throughput=4.89MB/sec 2024-09-13T00:01:30.586 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-WBFjuw3R3RTobgK off=6173184 len=138211 2024-09-13T00:01:30.590 INFO:tasks.workunit.client.0.smithi136.stdout:op 34 completed, throughput=4.91MB/sec 2024-09-13T00:01:30.591 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-S3QBKtiKaGyFf1U off=3429532 len=1212669 2024-09-13T00:01:30.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:01:30 smithi181 bash[67587]: cluster 2024-09-13T00:01:29.153642+0000 mgr.y (mgr.24661) 731 : cluster [DBG] pgmap v1414: 105 pgs: 105 active+clean; 126 GiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 918 KiB/s rd, 1.3 MiB/s wr, 854 op/s 2024-09-13T00:01:30.599 INFO:tasks.workunit.client.0.smithi136.stdout:op 35 completed, throughput=5.06MB/sec 2024-09-13T00:01:31.600 INFO:tasks.workunit.client.0.smithi136.stdout: 8: throughput=4.44MB/sec pending data=0 2024-09-13T00:01:31.600 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Ss9IMZ9bi5jYDMZ off=9109653 len=1874144 2024-09-13T00:01:31.606 INFO:tasks.workunit.client.0.smithi136.stdout:op 36 completed, throughput=4.66MB/sec 2024-09-13T00:01:31.606 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-YaBsstYDRP0NHLl off=11830014 len=1388680 2024-09-13T00:01:31.609 INFO:tasks.workunit.client.0.smithi136.stdout:op 37 completed, throughput=4.82MB/sec 2024-09-13T00:01:31.610 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-BNT7bfJoxDryJXe off=7527900 len=2069708 2024-09-13T00:01:31.619 INFO:tasks.workunit.client.0.smithi136.stdout:op 38 completed, throughput=5.05MB/sec 2024-09-13T00:01:32.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:01:32 smithi136 bash[75164]: cluster 2024-09-13T00:01:31.154543+0000 mgr.y (mgr.24661) 732 : cluster [DBG] pgmap v1415: 105 pgs: 105 active+clean; 126 GiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 1.2 MiB/s rd, 1.5 MiB/s wr, 854 op/s 2024-09-13T00:01:32.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:01:32 smithi136 bash[76371]: cluster 2024-09-13T00:01:31.154543+0000 mgr.y (mgr.24661) 732 : cluster [DBG] pgmap v1415: 105 pgs: 105 active+clean; 126 GiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 1.2 MiB/s rd, 1.5 MiB/s wr, 854 op/s 2024-09-13T00:01:32.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:01:32 smithi181 bash[67587]: cluster 2024-09-13T00:01:31.154543+0000 mgr.y (mgr.24661) 732 : cluster [DBG] pgmap v1415: 105 pgs: 105 active+clean; 126 GiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 1.2 MiB/s rd, 1.5 MiB/s wr, 854 op/s 2024-09-13T00:01:32.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:01:32 smithi181 bash[95505]: ts=2024-09-13T00:01:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:01:32.620 INFO:tasks.workunit.client.0.smithi136.stdout: 9: throughput=4.5MB/sec pending data=0 2024-09-13T00:01:32.620 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-MWYH2JzuacwEROe off=4588485 len=550209 2024-09-13T00:01:32.621 INFO:tasks.workunit.client.0.smithi136.stdout:op 39 completed, throughput=4.56MB/sec 2024-09-13T00:01:32.621 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj--959XdFghub_eqK off=233017 len=30732 2024-09-13T00:01:32.623 INFO:tasks.workunit.client.0.smithi136.stdout:op 40 completed, throughput=4.56MB/sec 2024-09-13T00:01:32.623 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-SAQVi45GMBxIWam off=685797 len=36364 2024-09-13T00:01:32.624 INFO:tasks.workunit.client.0.smithi136.stdout:op 41 completed, throughput=4.57MB/sec 2024-09-13T00:01:32.624 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2Te3flRop9rv-ot off=298195 len=2096483 2024-09-13T00:01:32.630 INFO:tasks.workunit.client.0.smithi136.stdout:op 42 completed, throughput=4.78MB/sec 2024-09-13T00:01:32.630 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-8_5fD51_2R-AV8D off=9512929 len=1463424 2024-09-13T00:01:32.636 INFO:tasks.workunit.client.0.smithi136.stdout:op 43 completed, throughput=4.93MB/sec 2024-09-13T00:01:32.636 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-gGvdkuEIS_MlLqT off=4576816 len=1012399 2024-09-13T00:01:32.639 INFO:tasks.workunit.client.0.smithi136.stdout:op 44 completed, throughput=5.03MB/sec 2024-09-13T00:01:33.641 INFO:tasks.workunit.client.0.smithi136.stdout: 10: throughput=4.54MB/sec pending data=0 2024-09-13T00:01:33.641 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-727YlzuqMoSTpDq off=6502762 len=2067673 2024-09-13T00:01:33.656 INFO:tasks.workunit.client.0.smithi136.stdout:op 45 completed, throughput=4.72MB/sec 2024-09-13T00:01:33.656 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-kTj8cc7VEq0FMLl off=867227 len=324091 2024-09-13T00:01:33.658 INFO:tasks.workunit.client.0.smithi136.stdout:op 46 completed, throughput=4.75MB/sec 2024-09-13T00:01:33.658 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_T6y2CDwkgMd2RP off=1146415 len=1265411 2024-09-13T00:01:33.670 INFO:tasks.workunit.client.0.smithi136.stdout:op 47 completed, throughput=4.86MB/sec 2024-09-13T00:01:33.670 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-IcHB19FlrK_KT_m off=5822971 len=726525 2024-09-13T00:01:33.672 INFO:tasks.workunit.client.0.smithi136.stdout:op 48 completed, throughput=4.93MB/sec 2024-09-13T00:01:33.672 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-n8_S36UudTWEaKA off=521866 len=840079 2024-09-13T00:01:33.680 INFO:tasks.workunit.client.0.smithi136.stdout:op 49 completed, throughput=5MB/sec 2024-09-13T00:01:34.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:01:34 smithi136 bash[75164]: cluster 2024-09-13T00:01:33.155548+0000 mgr.y (mgr.24661) 733 : cluster [DBG] pgmap v1416: 105 pgs: 105 active+clean; 126 GiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.7 MiB/s wr, 855 op/s 2024-09-13T00:01:34.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:01:34 smithi136 bash[76371]: cluster 2024-09-13T00:01:33.155548+0000 mgr.y (mgr.24661) 733 : cluster [DBG] pgmap v1416: 105 pgs: 105 active+clean; 126 GiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.7 MiB/s wr, 855 op/s 2024-09-13T00:01:34.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:01:34 smithi181 bash[67587]: cluster 2024-09-13T00:01:33.155548+0000 mgr.y (mgr.24661) 733 : cluster [DBG] pgmap v1416: 105 pgs: 105 active+clean; 126 GiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.7 MiB/s wr, 855 op/s 2024-09-13T00:01:34.680 INFO:tasks.workunit.client.0.smithi136.stdout: 11: throughput=4.56MB/sec pending data=0 2024-09-13T00:01:34.680 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-QOWKVWqJlLdueh2 off=14115729 len=176805 2024-09-13T00:01:34.682 INFO:tasks.workunit.client.0.smithi136.stdout:op 50 completed, throughput=4.58MB/sec 2024-09-13T00:01:34.682 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-1piooayvlRLhm2_ off=3422358 len=149542 2024-09-13T00:01:34.686 INFO:tasks.workunit.client.0.smithi136.stdout:op 51 completed, throughput=4.59MB/sec 2024-09-13T00:01:34.686 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-8zF4wSFw7mvVfCl off=2813052 len=377774 2024-09-13T00:01:34.688 INFO:tasks.workunit.client.0.smithi136.stdout:op 52 completed, throughput=4.62MB/sec 2024-09-13T00:01:34.688 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-3J6SKifbW3y2APz off=6762216 len=1255205 2024-09-13T00:01:34.700 INFO:tasks.workunit.client.0.smithi136.stdout:op 53 completed, throughput=4.72MB/sec 2024-09-13T00:01:34.700 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-BEBm3oOF3eSHnYc off=10292394 len=421083 2024-09-13T00:01:34.702 INFO:tasks.workunit.client.0.smithi136.stdout:op 54 completed, throughput=4.75MB/sec 2024-09-13T00:01:34.702 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-cthGtpp4k3tl1HK off=14466520 len=597542 2024-09-13T00:01:34.705 INFO:tasks.workunit.client.0.smithi136.stdout:op 55 completed, throughput=4.8MB/sec 2024-09-13T00:01:34.705 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj--ivS5BngKl5-4qH off=18481464 len=1094062 2024-09-13T00:01:34.708 INFO:tasks.workunit.client.0.smithi136.stdout:op 56 completed, throughput=4.89MB/sec 2024-09-13T00:01:34.709 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-v9r8G6DQbPgm-nJ off=13589826 len=424759 2024-09-13T00:01:34.710 INFO:tasks.workunit.client.0.smithi136.stdout:op 57 completed, throughput=4.93MB/sec 2024-09-13T00:01:34.710 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-UxlndsrM57S8an5 off=3900162 len=1825682 2024-09-13T00:01:34.725 INFO:tasks.workunit.client.0.smithi136.stdout:op 58 completed, throughput=5.08MB/sec 2024-09-13T00:01:35.725 INFO:tasks.workunit.client.0.smithi136.stdout: 12: throughput=4.66MB/sec pending data=0 2024-09-13T00:01:35.725 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-WGOPkVyhTo6FAjj off=5310184 len=347833 2024-09-13T00:01:35.727 INFO:tasks.workunit.client.0.smithi136.stdout:op 59 completed, throughput=4.69MB/sec 2024-09-13T00:01:35.727 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-cSnp1rljh8iCs8F off=539488 len=1699517 2024-09-13T00:01:35.731 INFO:tasks.workunit.client.0.smithi136.stdout:op 60 completed, throughput=4.82MB/sec 2024-09-13T00:01:35.731 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-OQtUHGrSzBEyrJ6 off=2198071 len=1634070 2024-09-13T00:01:35.745 INFO:tasks.workunit.client.0.smithi136.stdout:op 61 completed, throughput=4.94MB/sec 2024-09-13T00:01:35.745 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-4xC5T41AGFlno8O off=978936 len=1743942 2024-09-13T00:01:35.750 INFO:tasks.workunit.client.0.smithi136.stdout:op 62 completed, throughput=5.07MB/sec 2024-09-13T00:01:36.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:01:36 smithi136 bash[75164]: cluster 2024-09-13T00:01:35.156373+0000 mgr.y (mgr.24661) 734 : cluster [DBG] pgmap v1417: 105 pgs: 105 active+clean; 126 GiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.9 MiB/s wr, 790 op/s 2024-09-13T00:01:36.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:01:36 smithi136 bash[76371]: cluster 2024-09-13T00:01:35.156373+0000 mgr.y (mgr.24661) 734 : cluster [DBG] pgmap v1417: 105 pgs: 105 active+clean; 126 GiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.9 MiB/s wr, 790 op/s 2024-09-13T00:01:36.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:01:36 smithi181 bash[67587]: cluster 2024-09-13T00:01:35.156373+0000 mgr.y (mgr.24661) 734 : cluster [DBG] pgmap v1417: 105 pgs: 105 active+clean; 126 GiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.9 MiB/s wr, 790 op/s 2024-09-13T00:01:36.750 INFO:tasks.workunit.client.0.smithi136.stdout: 13: throughput=4.69MB/sec pending data=0 2024-09-13T00:01:36.750 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-IMntVfPgjJcRykP off=3818634 len=60583 2024-09-13T00:01:36.750 INFO:tasks.workunit.client.0.smithi136.stdout:op 63 completed, throughput=4.7MB/sec 2024-09-13T00:01:36.751 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj--O7Pqyvbg2BOK0t off=876160 len=384654 2024-09-13T00:01:36.753 INFO:tasks.workunit.client.0.smithi136.stdout:op 64 completed, throughput=4.72MB/sec 2024-09-13T00:01:36.753 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-24z3jjHGJ3cpoLN off=8592716 len=390411 2024-09-13T00:01:36.754 INFO:tasks.workunit.client.0.smithi136.stdout:op 65 completed, throughput=4.75MB/sec 2024-09-13T00:01:36.755 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-9M9pADxIMTDsYx9 off=3734883 len=1037591 2024-09-13T00:01:36.764 INFO:tasks.workunit.client.0.smithi136.stdout:op 66 completed, throughput=4.82MB/sec 2024-09-13T00:01:36.764 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj--YlmXOxLkd6xYhj off=11616938 len=792471 2024-09-13T00:01:36.772 INFO:tasks.workunit.client.0.smithi136.stdout:op 67 completed, throughput=4.88MB/sec 2024-09-13T00:01:36.772 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Aw1I2QVqxHWEK-P off=10897412 len=1309793 2024-09-13T00:01:36.780 INFO:tasks.workunit.client.0.smithi136.stdout:op 68 completed, throughput=4.97MB/sec 2024-09-13T00:01:36.780 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-PJkAiTjdPf9etM4 off=396879 len=1978267 2024-09-13T00:01:36.784 INFO:tasks.workunit.client.0.smithi136.stdout:op 69 completed, throughput=5.11MB/sec 2024-09-13T00:01:37.784 INFO:tasks.workunit.client.0.smithi136.stdout: 14: throughput=4.75MB/sec pending data=0 2024-09-13T00:01:37.785 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-gq_q1ssrAfVvibQ off=16101231 len=170579 2024-09-13T00:01:37.789 INFO:tasks.workunit.client.0.smithi136.stdout:op 70 completed, throughput=4.76MB/sec 2024-09-13T00:01:37.789 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-aBpupqJ0htq2VGJ off=1458686 len=1841551 2024-09-13T00:01:37.794 INFO:tasks.workunit.client.0.smithi136.stdout:op 71 completed, throughput=4.88MB/sec 2024-09-13T00:01:37.794 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-P6yS3HWt60m32vU off=140406 len=1985877 2024-09-13T00:01:37.809 INFO:tasks.workunit.client.0.smithi136.stdout:op 72 completed, throughput=5.01MB/sec 2024-09-13T00:01:38.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:01:38 smithi136 bash[75164]: cluster 2024-09-13T00:01:37.157497+0000 mgr.y (mgr.24661) 735 : cluster [DBG] pgmap v1418: 105 pgs: 105 active+clean; 126 GiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.9 MiB/s wr, 474 op/s 2024-09-13T00:01:38.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:01:38 smithi136 bash[76371]: cluster 2024-09-13T00:01:37.157497+0000 mgr.y (mgr.24661) 735 : cluster [DBG] pgmap v1418: 105 pgs: 105 active+clean; 126 GiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.9 MiB/s wr, 474 op/s 2024-09-13T00:01:38.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:01:38 smithi181 bash[67587]: cluster 2024-09-13T00:01:37.157497+0000 mgr.y (mgr.24661) 735 : cluster [DBG] pgmap v1418: 105 pgs: 105 active+clean; 126 GiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.9 MiB/s wr, 474 op/s 2024-09-13T00:01:38.810 INFO:tasks.workunit.client.0.smithi136.stdout: 15: throughput=4.68MB/sec pending data=0 2024-09-13T00:01:38.810 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-pMUhUH86P6TrTAq off=10939570 len=37892 2024-09-13T00:01:38.815 INFO:tasks.workunit.client.0.smithi136.stdout:op 73 completed, throughput=4.68MB/sec 2024-09-13T00:01:38.815 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-yEfAL3DSccGy0dS off=14727676 len=619792 2024-09-13T00:01:38.820 INFO:tasks.workunit.client.0.smithi136.stdout:op 74 completed, throughput=4.72MB/sec 2024-09-13T00:01:38.820 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Q7X_sR0vBFLGNkF off=12074161 len=532429 2024-09-13T00:01:38.822 INFO:tasks.workunit.client.0.smithi136.stdout:op 75 completed, throughput=4.75MB/sec 2024-09-13T00:01:38.822 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj--lw5HeAAbQDDYBe off=5413968 len=323862 2024-09-13T00:01:38.827 INFO:tasks.workunit.client.0.smithi136.stdout:op 76 completed, throughput=4.77MB/sec 2024-09-13T00:01:38.827 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-IBUztgY9wThsXmB off=10720598 len=1928046 2024-09-13T00:01:38.832 INFO:tasks.workunit.client.0.smithi136.stdout:op 77 completed, throughput=4.89MB/sec 2024-09-13T00:01:38.832 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-m4imqlNFOvMbdde off=2908074 len=1651995 2024-09-13T00:01:38.841 INFO:tasks.workunit.client.0.smithi136.stdout:op 78 completed, throughput=4.99MB/sec 2024-09-13T00:01:38.841 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-hzVY9ZvJaxGKNMf off=14191037 len=590439 2024-09-13T00:01:38.849 INFO:tasks.workunit.client.0.smithi136.stdout:op 79 completed, throughput=5.02MB/sec 2024-09-13T00:01:39.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:01:39 smithi136 bash[75164]: audit 2024-09-13T00:01:38.771597+0000 mon.a (mon.0) 2096 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:01:39.507 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:01:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:01:39] "GET /metrics HTTP/1.1" 200 34122 "" "Prometheus/2.43.0" 2024-09-13T00:01:39.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:01:39 smithi136 bash[76371]: audit 2024-09-13T00:01:38.771597+0000 mon.a (mon.0) 2096 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:01:39.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:01:39 smithi181 bash[67587]: audit 2024-09-13T00:01:38.771597+0000 mon.a (mon.0) 2096 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:01:39.850 INFO:tasks.workunit.client.0.smithi136.stdout: 16: throughput=4.72MB/sec pending data=0 2024-09-13T00:01:39.850 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-_CP5fivn3u4OJaX off=7002042 len=10131 2024-09-13T00:01:39.851 INFO:tasks.workunit.client.0.smithi136.stdout:op 80 completed, throughput=4.72MB/sec 2024-09-13T00:01:39.851 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-GmerYEDH-kmhwrN off=3093185 len=923163 2024-09-13T00:01:39.854 INFO:tasks.workunit.client.0.smithi136.stdout:op 81 completed, throughput=4.77MB/sec 2024-09-13T00:01:39.854 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Qd1yMQLlSvcbqf3 off=22823460 len=1903892 2024-09-13T00:01:39.858 INFO:tasks.workunit.client.0.smithi136.stdout:op 82 completed, throughput=4.88MB/sec 2024-09-13T00:01:39.858 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-8i5Q5X_8E1sULjv off=4301289 len=1636233 2024-09-13T00:01:39.873 INFO:tasks.workunit.client.0.smithi136.stdout:op 83 completed, throughput=4.97MB/sec 2024-09-13T00:01:39.873 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-nLW7XkI_LEjuGiX off=851434 len=2050791 2024-09-13T00:01:39.887 INFO:tasks.workunit.client.0.smithi136.stdout:op 84 completed, throughput=5.08MB/sec 2024-09-13T00:01:40.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:01:40 smithi181 bash[67587]: cluster 2024-09-13T00:01:39.158392+0000 mgr.y (mgr.24661) 736 : cluster [DBG] pgmap v1419: 105 pgs: 105 active+clean; 126 GiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.5 MiB/s wr, 4 op/s 2024-09-13T00:01:40.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:01:40 smithi136 bash[75164]: cluster 2024-09-13T00:01:39.158392+0000 mgr.y (mgr.24661) 736 : cluster [DBG] pgmap v1419: 105 pgs: 105 active+clean; 126 GiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.5 MiB/s wr, 4 op/s 2024-09-13T00:01:40.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:01:40 smithi136 bash[76371]: cluster 2024-09-13T00:01:39.158392+0000 mgr.y (mgr.24661) 736 : cluster [DBG] pgmap v1419: 105 pgs: 105 active+clean; 126 GiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.5 MiB/s wr, 4 op/s 2024-09-13T00:01:40.888 INFO:tasks.workunit.client.0.smithi136.stdout: 17: throughput=4.79MB/sec pending data=0 2024-09-13T00:01:40.888 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-EqR3GOFpG0BMhnc off=3843171 len=63460 2024-09-13T00:01:40.889 INFO:tasks.workunit.client.0.smithi136.stdout:op 85 completed, throughput=4.79MB/sec 2024-09-13T00:01:40.889 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-QVocSE0_I6hm4eN off=5537660 len=1173486 2024-09-13T00:01:40.897 INFO:tasks.workunit.client.0.smithi136.stdout:op 86 completed, throughput=4.86MB/sec 2024-09-13T00:01:40.897 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-YQyAgwrtA5K_0ku off=7853192 len=1767975 2024-09-13T00:01:40.901 INFO:tasks.workunit.client.0.smithi136.stdout:op 87 completed, throughput=4.95MB/sec 2024-09-13T00:01:40.901 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-SReLVd0DXKirgJn off=2590768 len=125345 2024-09-13T00:01:40.902 INFO:tasks.workunit.client.0.smithi136.stdout:op 88 completed, throughput=4.96MB/sec 2024-09-13T00:01:40.902 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-hkD2Hv1fgwAOIeB off=562232 len=209915 2024-09-13T00:01:40.903 INFO:tasks.workunit.client.0.smithi136.stdout:op 89 completed, throughput=4.97MB/sec 2024-09-13T00:01:40.903 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ZuJkVbq1O3YnESi off=10678751 len=2069283 2024-09-13T00:01:40.907 INFO:tasks.workunit.client.0.smithi136.stdout:op 90 completed, throughput=5.08MB/sec 2024-09-13T00:01:41.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:01:41 smithi181 bash[67587]: audit 2024-09-13T00:01:40.715067+0000 mon.a (mon.0) 2097 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:01:41.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:01:41 smithi181 bash[67587]: audit 2024-09-13T00:01:41.255920+0000 mon.a (mon.0) 2098 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:01:41.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:01:41 smithi181 bash[67587]: audit 2024-09-13T00:01:41.257493+0000 mon.a (mon.0) 2099 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:01:41.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:01:41 smithi136 bash[75164]: audit 2024-09-13T00:01:40.715067+0000 mon.a (mon.0) 2097 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:01:41.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:01:41 smithi136 bash[75164]: audit 2024-09-13T00:01:41.255920+0000 mon.a (mon.0) 2098 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:01:41.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:01:41 smithi136 bash[75164]: audit 2024-09-13T00:01:41.257493+0000 mon.a (mon.0) 2099 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:01:41.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:01:41 smithi136 bash[76371]: audit 2024-09-13T00:01:40.715067+0000 mon.a (mon.0) 2097 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:01:41.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:01:41 smithi136 bash[76371]: audit 2024-09-13T00:01:41.255920+0000 mon.a (mon.0) 2098 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:01:41.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:01:41 smithi136 bash[76371]: audit 2024-09-13T00:01:41.257493+0000 mon.a (mon.0) 2099 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:01:41.908 INFO:tasks.workunit.client.0.smithi136.stdout: 18: throughput=4.81MB/sec pending data=0 2024-09-13T00:01:41.908 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-l1_xp5sRHm8vHya off=809536 len=1757538 2024-09-13T00:01:41.919 INFO:tasks.workunit.client.0.smithi136.stdout:op 91 completed, throughput=4.89MB/sec 2024-09-13T00:01:41.919 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-idSDAQmkd277Pdp off=614572 len=1894056 2024-09-13T00:01:41.932 INFO:tasks.workunit.client.0.smithi136.stdout:op 92 completed, throughput=4.99MB/sec 2024-09-13T00:01:41.932 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-9wbqjtcge7m2zlL off=4939963 len=1756060 2024-09-13T00:01:41.940 INFO:tasks.workunit.client.0.smithi136.stdout:op 93 completed, throughput=5.08MB/sec 2024-09-13T00:01:42.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:01:42 smithi181 bash[67587]: cluster 2024-09-13T00:01:41.158935+0000 mgr.y (mgr.24661) 737 : cluster [DBG] pgmap v1420: 105 pgs: 105 active+clean; 126 GiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 3.0 MiB/s rd, 1.6 MiB/s wr, 4 op/s 2024-09-13T00:01:42.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:01:42 smithi181 bash[67587]: audit 2024-09-13T00:01:41.275615+0000 mon.a (mon.0) 2100 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:01:42.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:01:42 smithi181 bash[95505]: ts=2024-09-13T00:01:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:01:42.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:01:42 smithi136 bash[75164]: cluster 2024-09-13T00:01:41.158935+0000 mgr.y (mgr.24661) 737 : cluster [DBG] pgmap v1420: 105 pgs: 105 active+clean; 126 GiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 3.0 MiB/s rd, 1.6 MiB/s wr, 4 op/s 2024-09-13T00:01:42.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:01:42 smithi136 bash[75164]: audit 2024-09-13T00:01:41.275615+0000 mon.a (mon.0) 2100 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:01:42.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:01:42 smithi136 bash[76371]: cluster 2024-09-13T00:01:41.158935+0000 mgr.y (mgr.24661) 737 : cluster [DBG] pgmap v1420: 105 pgs: 105 active+clean; 126 GiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 3.0 MiB/s rd, 1.6 MiB/s wr, 4 op/s 2024-09-13T00:01:42.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:01:42 smithi136 bash[76371]: audit 2024-09-13T00:01:41.275615+0000 mon.a (mon.0) 2100 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:01:42.941 INFO:tasks.workunit.client.0.smithi136.stdout: 19: throughput=4.82MB/sec pending data=0 2024-09-13T00:01:42.941 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-AGtgYsQDz7Wdhts off=6526207 len=560694 2024-09-13T00:01:42.947 INFO:tasks.workunit.client.0.smithi136.stdout:op 94 completed, throughput=4.84MB/sec 2024-09-13T00:01:42.947 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-LspD4eq8oUbwdGX off=3372007 len=501675 2024-09-13T00:01:42.948 INFO:tasks.workunit.client.0.smithi136.stdout:op 95 completed, throughput=4.87MB/sec 2024-09-13T00:01:42.949 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-_BgM43XOIr5aO-e off=609633 len=2057881 2024-09-13T00:01:42.955 INFO:tasks.workunit.client.0.smithi136.stdout:op 96 completed, throughput=4.96MB/sec 2024-09-13T00:01:42.955 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-VbJn7QlzvjetSbr off=8360579 len=1933434 2024-09-13T00:01:42.961 INFO:tasks.workunit.client.0.smithi136.stdout:op 97 completed, throughput=5.06MB/sec 2024-09-13T00:01:43.962 INFO:tasks.workunit.client.0.smithi136.stdout: 20: throughput=4.81MB/sec pending data=0 2024-09-13T00:01:43.962 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-myhtFFof4VIHHsm off=3910553 len=449385 2024-09-13T00:01:43.967 INFO:tasks.workunit.client.0.smithi136.stdout:op 98 completed, throughput=4.83MB/sec 2024-09-13T00:01:43.968 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-i1dGOXUN8PqhzIe off=892787 len=1933995 2024-09-13T00:01:43.972 INFO:tasks.workunit.client.0.smithi136.stdout:op 99 completed, throughput=4.92MB/sec 2024-09-13T00:01:43.972 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj--fNj-OZdf123_RG off=1571603 len=626565 2024-09-13T00:01:43.974 INFO:tasks.workunit.client.0.smithi136.stdout:op 100 completed, throughput=4.95MB/sec 2024-09-13T00:01:43.974 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-INeYeCBESM3kJLY off=11478516 len=1586799 2024-09-13T00:01:43.985 INFO:tasks.workunit.client.0.smithi136.stdout:op 101 completed, throughput=5.02MB/sec 2024-09-13T00:01:44.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:01:44 smithi181 bash[67587]: cluster 2024-09-13T00:01:43.159963+0000 mgr.y (mgr.24661) 738 : cluster [DBG] pgmap v1421: 105 pgs: 105 active+clean; 126 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.8 MiB/s wr, 5 op/s 2024-09-13T00:01:44.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:01:44 smithi136 bash[75164]: cluster 2024-09-13T00:01:43.159963+0000 mgr.y (mgr.24661) 738 : cluster [DBG] pgmap v1421: 105 pgs: 105 active+clean; 126 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.8 MiB/s wr, 5 op/s 2024-09-13T00:01:44.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:01:44 smithi136 bash[76371]: cluster 2024-09-13T00:01:43.159963+0000 mgr.y (mgr.24661) 738 : cluster [DBG] pgmap v1421: 105 pgs: 105 active+clean; 126 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.8 MiB/s wr, 5 op/s 2024-09-13T00:01:44.985 INFO:tasks.workunit.client.0.smithi136.stdout: 21: throughput=4.79MB/sec pending data=0 2024-09-13T00:01:44.985 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-rOLUge0VeNXwffJ off=105599 len=1477279 2024-09-13T00:01:44.999 INFO:tasks.workunit.client.0.smithi136.stdout:op 102 completed, throughput=4.85MB/sec 2024-09-13T00:01:44.999 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-LyRunD-y_a66cCT off=892397 len=238998 2024-09-13T00:01:45.001 INFO:tasks.workunit.client.0.smithi136.stdout:op 103 completed, throughput=4.86MB/sec 2024-09-13T00:01:45.001 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj--GS-xS5hh9SkIak off=9543287 len=1883646 2024-09-13T00:01:45.017 INFO:tasks.workunit.client.0.smithi136.stdout:op 104 completed, throughput=4.94MB/sec 2024-09-13T00:01:45.017 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-VfJZza754EG84yc off=6815515 len=1460093 2024-09-13T00:01:45.028 INFO:tasks.workunit.client.0.smithi136.stdout:op 105 completed, throughput=5MB/sec 2024-09-13T00:01:45.028 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-f5g39Uk4WvKf53i off=11897407 len=113018 2024-09-13T00:01:45.029 INFO:tasks.workunit.client.0.smithi136.stdout:op 106 completed, throughput=5MB/sec 2024-09-13T00:01:46.030 INFO:tasks.workunit.client.0.smithi136.stdout: 22: throughput=4.78MB/sec pending data=0 2024-09-13T00:01:46.030 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-CCensgeXqcYlp6w off=11179781 len=1799295 2024-09-13T00:01:46.042 INFO:tasks.workunit.client.0.smithi136.stdout:op 107 completed, throughput=4.86MB/sec 2024-09-13T00:01:46.042 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-FrvDwBAoRLnzZT0 off=21746253 len=2030531 2024-09-13T00:01:46.060 INFO:tasks.workunit.client.0.smithi136.stdout:op 108 completed, throughput=4.94MB/sec 2024-09-13T00:01:46.060 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-xQKVgDrB4Ez1CfL off=17684636 len=78496 2024-09-13T00:01:46.061 INFO:tasks.workunit.client.0.smithi136.stdout:op 109 completed, throughput=4.94MB/sec 2024-09-13T00:01:46.061 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-aHNg9s17XldeMVz off=21137653 len=526454 2024-09-13T00:01:46.067 INFO:tasks.workunit.client.0.smithi136.stdout:op 110 completed, throughput=4.96MB/sec 2024-09-13T00:01:46.067 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Sw04MR8PtecHuOS off=6904020 len=280136 2024-09-13T00:01:46.073 INFO:tasks.workunit.client.0.smithi136.stdout:op 111 completed, throughput=4.97MB/sec 2024-09-13T00:01:46.073 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-kU_waJCn9i8zZFG off=964066 len=1462548 2024-09-13T00:01:46.075 INFO:tasks.workunit.client.0.smithi136.stdout:op 112 completed, throughput=5.03MB/sec 2024-09-13T00:01:46.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:01:46 smithi181 bash[67587]: cluster 2024-09-13T00:01:45.160866+0000 mgr.y (mgr.24661) 739 : cluster [DBG] pgmap v1422: 105 pgs: 105 active+clean; 126 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:01:46.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:01:46 smithi136 bash[75164]: cluster 2024-09-13T00:01:45.160866+0000 mgr.y (mgr.24661) 739 : cluster [DBG] pgmap v1422: 105 pgs: 105 active+clean; 126 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:01:46.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:01:46 smithi136 bash[76371]: cluster 2024-09-13T00:01:45.160866+0000 mgr.y (mgr.24661) 739 : cluster [DBG] pgmap v1422: 105 pgs: 105 active+clean; 126 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:01:47.075 INFO:tasks.workunit.client.0.smithi136.stdout: 23: throughput=4.82MB/sec pending data=0 2024-09-13T00:01:47.076 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-SX3JOhng0ZnVZR2 off=16160828 len=655452 2024-09-13T00:01:47.078 INFO:tasks.workunit.client.0.smithi136.stdout:op 113 completed, throughput=4.85MB/sec 2024-09-13T00:01:47.078 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-w4O-BhPyrGZfZPb off=1514663 len=375145 2024-09-13T00:01:47.083 INFO:tasks.workunit.client.0.smithi136.stdout:op 114 completed, throughput=4.86MB/sec 2024-09-13T00:01:47.083 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-osePGyX2sMFF6mi off=2928328 len=711530 2024-09-13T00:01:47.086 INFO:tasks.workunit.client.0.smithi136.stdout:op 115 completed, throughput=4.89MB/sec 2024-09-13T00:01:47.086 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-lhQB6cjTMiWizdx off=209013 len=1529811 2024-09-13T00:01:47.097 INFO:tasks.workunit.client.0.smithi136.stdout:op 116 completed, throughput=4.95MB/sec 2024-09-13T00:01:47.097 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Nv0DfnJHCP89mg2 off=1322591 len=627698 2024-09-13T00:01:47.104 INFO:tasks.workunit.client.0.smithi136.stdout:op 117 completed, throughput=4.97MB/sec 2024-09-13T00:01:47.104 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ksTf8TKZqPvKuL8 off=10652441 len=1829964 2024-09-13T00:01:47.108 INFO:tasks.workunit.client.0.smithi136.stdout:op 118 completed, throughput=5.04MB/sec 2024-09-13T00:01:48.109 INFO:tasks.workunit.client.0.smithi136.stdout: 24: throughput=4.84MB/sec pending data=0 2024-09-13T00:01:48.109 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Eb-c24sfwvvXaOC off=154299 len=1866963 2024-09-13T00:01:48.113 INFO:tasks.workunit.client.0.smithi136.stdout:op 119 completed, throughput=4.91MB/sec 2024-09-13T00:01:48.113 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ruRFX901PaWJ0G0 off=5186124 len=1972062 2024-09-13T00:01:48.127 INFO:tasks.workunit.client.0.smithi136.stdout:op 120 completed, throughput=4.98MB/sec 2024-09-13T00:01:48.127 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-JEpehMgFaIt3ag6 off=554311 len=961942 2024-09-13T00:01:48.135 INFO:tasks.workunit.client.0.smithi136.stdout:op 121 completed, throughput=5.02MB/sec 2024-09-13T00:01:48.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:01:48 smithi181 bash[67587]: cluster 2024-09-13T00:01:47.161998+0000 mgr.y (mgr.24661) 740 : cluster [DBG] pgmap v1423: 105 pgs: 105 active+clean; 126 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:01:48.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:01:48 smithi136 bash[76371]: cluster 2024-09-13T00:01:47.161998+0000 mgr.y (mgr.24661) 740 : cluster [DBG] pgmap v1423: 105 pgs: 105 active+clean; 126 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:01:48.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:01:48 smithi136 bash[75164]: cluster 2024-09-13T00:01:47.161998+0000 mgr.y (mgr.24661) 740 : cluster [DBG] pgmap v1423: 105 pgs: 105 active+clean; 126 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:01:49.135 INFO:tasks.workunit.client.0.smithi136.stdout: 25: throughput=4.83MB/sec pending data=0 2024-09-13T00:01:49.135 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-FHQyGjpjUqQisl8 off=10334692 len=945217 2024-09-13T00:01:49.138 INFO:tasks.workunit.client.0.smithi136.stdout:op 122 completed, throughput=4.86MB/sec 2024-09-13T00:01:49.138 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-JMnb7QxA1C6aU8I off=11712304 len=485498 2024-09-13T00:01:49.140 INFO:tasks.workunit.client.0.smithi136.stdout:op 123 completed, throughput=4.88MB/sec 2024-09-13T00:01:49.140 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-d_y8uHDq2Sda9lk off=17631297 len=684522 2024-09-13T00:01:49.142 INFO:tasks.workunit.client.0.smithi136.stdout:op 124 completed, throughput=4.9MB/sec 2024-09-13T00:01:49.142 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-algcuK0YUtsz6vl off=5170765 len=2045104 2024-09-13T00:01:49.155 INFO:tasks.workunit.client.0.smithi136.stdout:op 125 completed, throughput=4.98MB/sec 2024-09-13T00:01:49.155 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-huN5X7-FYtuT1gL off=6878862 len=1388184 2024-09-13T00:01:49.158 INFO:tasks.workunit.client.0.smithi136.stdout:op 126 completed, throughput=5.03MB/sec 2024-09-13T00:01:49.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:01:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:01:49] "GET /metrics HTTP/1.1" 200 34122 "" "Prometheus/2.43.0" 2024-09-13T00:01:50.158 INFO:tasks.workunit.client.0.smithi136.stdout: 26: throughput=4.84MB/sec pending data=0 2024-09-13T00:01:50.158 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-t9oRPA3m75XiKcO off=275372 len=940616 2024-09-13T00:01:50.162 INFO:tasks.workunit.client.0.smithi136.stdout:op 127 completed, throughput=4.87MB/sec 2024-09-13T00:01:50.162 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-OxaFeTNllECE-so off=5897553 len=1870895 2024-09-13T00:01:50.164 INFO:tasks.workunit.client.0.smithi136.stdout:op 128 completed, throughput=4.94MB/sec 2024-09-13T00:01:50.164 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj--wS15E33xtQh7m0 off=3655742 len=280568 2024-09-13T00:01:50.169 INFO:tasks.workunit.client.0.smithi136.stdout:op 129 completed, throughput=4.95MB/sec 2024-09-13T00:01:50.169 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ZqRK_qyYPi4DMD9 off=7072924 len=1905864 2024-09-13T00:01:50.173 INFO:tasks.workunit.client.0.smithi136.stdout:op 130 completed, throughput=5.01MB/sec 2024-09-13T00:01:50.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:01:50 smithi181 bash[67587]: cluster 2024-09-13T00:01:49.162875+0000 mgr.y (mgr.24661) 741 : cluster [DBG] pgmap v1424: 105 pgs: 105 active+clean; 126 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 1.8 MiB/s rd, 2.8 MiB/s wr, 4 op/s 2024-09-13T00:01:50.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:01:50 smithi136 bash[75164]: cluster 2024-09-13T00:01:49.162875+0000 mgr.y (mgr.24661) 741 : cluster [DBG] pgmap v1424: 105 pgs: 105 active+clean; 126 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 1.8 MiB/s rd, 2.8 MiB/s wr, 4 op/s 2024-09-13T00:01:50.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:01:50 smithi136 bash[76371]: cluster 2024-09-13T00:01:49.162875+0000 mgr.y (mgr.24661) 741 : cluster [DBG] pgmap v1424: 105 pgs: 105 active+clean; 126 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 1.8 MiB/s rd, 2.8 MiB/s wr, 4 op/s 2024-09-13T00:01:51.173 INFO:tasks.workunit.client.0.smithi136.stdout: 27: throughput=4.83MB/sec pending data=0 2024-09-13T00:01:51.174 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-X6YIpn57nQqc22x off=10354995 len=1764578 2024-09-13T00:01:51.177 INFO:tasks.workunit.client.0.smithi136.stdout:op 131 completed, throughput=4.89MB/sec 2024-09-13T00:01:51.178 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-o_yk357yCEHrLT3 off=6204404 len=1681142 2024-09-13T00:01:51.189 INFO:tasks.workunit.client.0.smithi136.stdout:op 132 completed, throughput=4.95MB/sec 2024-09-13T00:01:51.189 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-MRVrGScboMvQ3H8 off=22976105 len=1090578 2024-09-13T00:01:51.198 INFO:tasks.workunit.client.0.smithi136.stdout:op 133 completed, throughput=4.98MB/sec 2024-09-13T00:01:51.198 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-CazU11TOFdlAlQW off=2851230 len=1477178 2024-09-13T00:01:51.203 INFO:tasks.workunit.client.0.smithi136.stdout:op 134 completed, throughput=5.03MB/sec 2024-09-13T00:01:52.203 INFO:tasks.workunit.client.0.smithi136.stdout: 28: throughput=4.86MB/sec pending data=0 2024-09-13T00:01:52.204 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-E5Dv2N2UucinPoh off=5345401 len=1780595 2024-09-13T00:01:52.208 INFO:tasks.workunit.client.0.smithi136.stdout:op 135 completed, throughput=4.92MB/sec 2024-09-13T00:01:52.208 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-mn40BKe1tUqQ6Gw off=1339614 len=1795730 2024-09-13T00:01:52.221 INFO:tasks.workunit.client.0.smithi136.stdout:op 136 completed, throughput=4.97MB/sec 2024-09-13T00:01:52.221 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-F4P3SeAKE1pmonm off=2256612 len=1393455 2024-09-13T00:01:52.226 INFO:tasks.workunit.client.0.smithi136.stdout:op 137 completed, throughput=5.02MB/sec 2024-09-13T00:01:52.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:01:52 smithi181 bash[67587]: cluster 2024-09-13T00:01:51.163796+0000 mgr.y (mgr.24661) 742 : cluster [DBG] pgmap v1425: 105 pgs: 105 active+clean; 126 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 2.0 MiB/s rd, 3.2 MiB/s wr, 4 op/s 2024-09-13T00:01:52.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:01:52 smithi181 bash[95505]: ts=2024-09-13T00:01:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:01:52.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:01:52 smithi136 bash[75164]: cluster 2024-09-13T00:01:51.163796+0000 mgr.y (mgr.24661) 742 : cluster [DBG] pgmap v1425: 105 pgs: 105 active+clean; 126 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 2.0 MiB/s rd, 3.2 MiB/s wr, 4 op/s 2024-09-13T00:01:52.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:01:52 smithi136 bash[76371]: cluster 2024-09-13T00:01:51.163796+0000 mgr.y (mgr.24661) 742 : cluster [DBG] pgmap v1425: 105 pgs: 105 active+clean; 126 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 2.0 MiB/s rd, 3.2 MiB/s wr, 4 op/s 2024-09-13T00:01:53.227 INFO:tasks.workunit.client.0.smithi136.stdout: 29: throughput=4.85MB/sec pending data=0 2024-09-13T00:01:53.227 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ptkJ_L7qmRbjVyX off=2685451 len=256998 2024-09-13T00:01:53.232 INFO:tasks.workunit.client.0.smithi136.stdout:op 138 completed, throughput=4.86MB/sec 2024-09-13T00:01:53.232 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-EIuMH_ZOhNmvx80 off=259231 len=585239 2024-09-13T00:01:53.238 INFO:tasks.workunit.client.0.smithi136.stdout:op 139 completed, throughput=4.88MB/sec 2024-09-13T00:01:53.238 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-RsrEzmH6aPzRlJQ off=8115859 len=584190 2024-09-13T00:01:53.244 INFO:tasks.workunit.client.0.smithi136.stdout:op 140 completed, throughput=4.89MB/sec 2024-09-13T00:01:53.244 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-V6VOjGhCCVHD71S off=9106297 len=942546 2024-09-13T00:01:53.247 INFO:tasks.workunit.client.0.smithi136.stdout:op 141 completed, throughput=4.92MB/sec 2024-09-13T00:01:53.247 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-u633xRoBpZ-Rg4R off=6557664 len=790338 2024-09-13T00:01:53.249 INFO:tasks.workunit.client.0.smithi136.stdout:op 142 completed, throughput=4.95MB/sec 2024-09-13T00:01:53.249 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-iaHr3FjEpzn2sYc off=11434157 len=1749744 2024-09-13T00:01:53.253 INFO:tasks.workunit.client.0.smithi136.stdout:op 143 completed, throughput=5MB/sec 2024-09-13T00:01:54.253 INFO:tasks.workunit.client.0.smithi136.stdout: 30: throughput=4.84MB/sec pending data=0 2024-09-13T00:01:54.254 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-f-LAVid6Mh4TKd0 off=7909813 len=1320048 2024-09-13T00:01:54.257 INFO:tasks.workunit.client.0.smithi136.stdout:op 144 completed, throughput=4.88MB/sec 2024-09-13T00:01:54.257 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-BneYOUyRI9O9uQ6 off=3951393 len=1295820 2024-09-13T00:01:54.269 INFO:tasks.workunit.client.0.smithi136.stdout:op 145 completed, throughput=4.92MB/sec 2024-09-13T00:01:54.269 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-gNhLRrTsAAWsXf2 off=18638801 len=46316 2024-09-13T00:01:54.273 INFO:tasks.workunit.client.0.smithi136.stdout:op 146 completed, throughput=4.92MB/sec 2024-09-13T00:01:54.273 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-aKjVf4k1rqMDqYk off=10746060 len=1205742 2024-09-13T00:01:54.283 INFO:tasks.workunit.client.0.smithi136.stdout:op 147 completed, throughput=4.96MB/sec 2024-09-13T00:01:54.283 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-gvBFlf_Om9-ua_o off=18303582 len=1077226 2024-09-13T00:01:54.291 INFO:tasks.workunit.client.0.smithi136.stdout:op 148 completed, throughput=4.99MB/sec 2024-09-13T00:01:54.292 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-jJ8--YpHX5bdvYn off=3630096 len=759486 2024-09-13T00:01:54.298 INFO:tasks.workunit.client.0.smithi136.stdout:op 149 completed, throughput=5.01MB/sec 2024-09-13T00:01:54.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:01:54 smithi181 bash[67587]: cluster 2024-09-13T00:01:53.164848+0000 mgr.y (mgr.24661) 743 : cluster [DBG] pgmap v1426: 105 pgs: 105 active+clean; 126 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 2.1 MiB/s rd, 3.0 MiB/s wr, 4 op/s 2024-09-13T00:01:54.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:01:54 smithi181 bash[67587]: audit 2024-09-13T00:01:53.771897+0000 mon.a (mon.0) 2101 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:01:54.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:01:54 smithi136 bash[75164]: cluster 2024-09-13T00:01:53.164848+0000 mgr.y (mgr.24661) 743 : cluster [DBG] pgmap v1426: 105 pgs: 105 active+clean; 126 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 2.1 MiB/s rd, 3.0 MiB/s wr, 4 op/s 2024-09-13T00:01:54.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:01:54 smithi136 bash[75164]: audit 2024-09-13T00:01:53.771897+0000 mon.a (mon.0) 2101 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:01:54.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:01:54 smithi136 bash[76371]: cluster 2024-09-13T00:01:53.164848+0000 mgr.y (mgr.24661) 743 : cluster [DBG] pgmap v1426: 105 pgs: 105 active+clean; 126 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 2.1 MiB/s rd, 3.0 MiB/s wr, 4 op/s 2024-09-13T00:01:54.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:01:54 smithi136 bash[76371]: audit 2024-09-13T00:01:53.771897+0000 mon.a (mon.0) 2101 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:01:55.299 INFO:tasks.workunit.client.0.smithi136.stdout: 31: throughput=4.85MB/sec pending data=0 2024-09-13T00:01:55.299 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-m4mRNZdt-_IG0TW off=11104667 len=1670002 2024-09-13T00:01:55.310 INFO:tasks.workunit.client.0.smithi136.stdout:op 150 completed, throughput=4.9MB/sec 2024-09-13T00:01:55.311 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-O4zHCuu6wIxG6Ts off=12442806 len=1475002 2024-09-13T00:01:55.313 INFO:tasks.workunit.client.0.smithi136.stdout:op 151 completed, throughput=4.95MB/sec 2024-09-13T00:01:55.313 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Yd1rfxa6tNBjARB off=7615454 len=142770 2024-09-13T00:01:55.317 INFO:tasks.workunit.client.0.smithi136.stdout:op 152 completed, throughput=4.95MB/sec 2024-09-13T00:01:55.317 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-et5YJ3G4s_qM4Hn off=1927323 len=625991 2024-09-13T00:01:55.324 INFO:tasks.workunit.client.0.smithi136.stdout:op 153 completed, throughput=4.97MB/sec 2024-09-13T00:01:55.324 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Nc-b1C3HI24aE8b off=16013266 len=1277716 2024-09-13T00:01:55.328 INFO:tasks.workunit.client.0.smithi136.stdout:op 154 completed, throughput=5MB/sec 2024-09-13T00:01:56.329 INFO:tasks.workunit.client.0.smithi136.stdout: 32: throughput=4.85MB/sec pending data=0 2024-09-13T00:01:56.329 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-k7aaWRW0yY021Th off=12027730 len=1689721 2024-09-13T00:01:56.333 INFO:tasks.workunit.client.0.smithi136.stdout:op 155 completed, throughput=4.9MB/sec 2024-09-13T00:01:56.333 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-s62BkN_gw53VlpL off=9180573 len=783962 2024-09-13T00:01:56.335 INFO:tasks.workunit.client.0.smithi136.stdout:op 156 completed, throughput=4.92MB/sec 2024-09-13T00:01:56.336 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-PwXedJPU3Wwh4Cb off=3911156 len=424551 2024-09-13T00:01:56.338 INFO:tasks.workunit.client.0.smithi136.stdout:op 157 completed, throughput=4.93MB/sec 2024-09-13T00:01:56.338 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-1M6o7P5EhF7sZ45 off=20982517 len=1074232 2024-09-13T00:01:56.341 INFO:tasks.workunit.client.0.smithi136.stdout:op 158 completed, throughput=4.97MB/sec 2024-09-13T00:01:56.341 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Rndouy9YwqiR82T off=2680297 len=1067846 2024-09-13T00:01:56.343 INFO:tasks.workunit.client.0.smithi136.stdout:op 159 completed, throughput=5MB/sec 2024-09-13T00:01:56.343 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-8viwAKWesmqZgbn off=9427914 len=523280 2024-09-13T00:01:56.345 INFO:tasks.workunit.client.0.smithi136.stdout:op 160 completed, throughput=5.01MB/sec 2024-09-13T00:01:56.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:01:56 smithi181 bash[67587]: cluster 2024-09-13T00:01:55.165651+0000 mgr.y (mgr.24661) 744 : cluster [DBG] pgmap v1427: 105 pgs: 105 active+clean; 126 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.8 MiB/s wr, 4 op/s 2024-09-13T00:01:56.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:01:56 smithi136 bash[75164]: cluster 2024-09-13T00:01:55.165651+0000 mgr.y (mgr.24661) 744 : cluster [DBG] pgmap v1427: 105 pgs: 105 active+clean; 126 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.8 MiB/s wr, 4 op/s 2024-09-13T00:01:56.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:01:56 smithi136 bash[76371]: cluster 2024-09-13T00:01:55.165651+0000 mgr.y (mgr.24661) 744 : cluster [DBG] pgmap v1427: 105 pgs: 105 active+clean; 126 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.8 MiB/s wr, 4 op/s 2024-09-13T00:01:57.346 INFO:tasks.workunit.client.0.smithi136.stdout: 33: throughput=4.86MB/sec pending data=0 2024-09-13T00:01:57.346 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Hxu55CHaEmvwK7A off=3486831 len=508300 2024-09-13T00:01:57.352 INFO:tasks.workunit.client.0.smithi136.stdout:op 161 completed, throughput=4.88MB/sec 2024-09-13T00:01:57.353 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-mbBBdnGEB3j2889 off=7969821 len=1137356 2024-09-13T00:01:57.363 INFO:tasks.workunit.client.0.smithi136.stdout:op 162 completed, throughput=4.91MB/sec 2024-09-13T00:01:57.363 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-dBVvkNcniut0_7Z off=10544544 len=735912 2024-09-13T00:01:57.366 INFO:tasks.workunit.client.0.smithi136.stdout:op 163 completed, throughput=4.93MB/sec 2024-09-13T00:01:57.366 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-1iuX0fJrcJ8T_8m off=587229 len=1393434 2024-09-13T00:01:57.376 INFO:tasks.workunit.client.0.smithi136.stdout:op 164 completed, throughput=4.96MB/sec 2024-09-13T00:01:57.376 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-w6zg3oCx9ud0auD off=2438540 len=416074 2024-09-13T00:01:57.378 INFO:tasks.workunit.client.0.smithi136.stdout:op 165 completed, throughput=4.98MB/sec 2024-09-13T00:01:57.378 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-eaVqHKlDGeDLBjP off=1034913 len=54381 2024-09-13T00:01:57.379 INFO:tasks.workunit.client.0.smithi136.stdout:op 166 completed, throughput=4.98MB/sec 2024-09-13T00:01:57.379 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Kt1XIBeTrZHHVGa off=21874467 len=358386 2024-09-13T00:01:57.385 INFO:tasks.workunit.client.0.smithi136.stdout:op 167 completed, throughput=4.99MB/sec 2024-09-13T00:01:57.385 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-lPonLj6OKajAt4r off=19640472 len=702635 2024-09-13T00:01:57.388 INFO:tasks.workunit.client.0.smithi136.stdout:op 168 completed, throughput=5.01MB/sec 2024-09-13T00:01:58.388 INFO:tasks.workunit.client.0.smithi136.stdout: 34: throughput=4.86MB/sec pending data=0 2024-09-13T00:01:58.388 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-UuEgoep9XccJ2xb off=3616963 len=814260 2024-09-13T00:01:58.390 INFO:tasks.workunit.client.0.smithi136.stdout:op 169 completed, throughput=4.88MB/sec 2024-09-13T00:01:58.390 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-06AXCyEJFLvtqZ_ off=7056376 len=1071836 2024-09-13T00:01:58.399 INFO:tasks.workunit.client.0.smithi136.stdout:op 170 completed, throughput=4.91MB/sec 2024-09-13T00:01:58.399 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-RvnQeNnhGiCkZLV off=3631286 len=422295 2024-09-13T00:01:58.404 INFO:tasks.workunit.client.0.smithi136.stdout:op 171 completed, throughput=4.92MB/sec 2024-09-13T00:01:58.405 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-kq3dcLPYUPTCxc1 off=302060 len=1639243 2024-09-13T00:01:58.417 INFO:tasks.workunit.client.0.smithi136.stdout:op 172 completed, throughput=4.97MB/sec 2024-09-13T00:01:58.417 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-A6sr1K1jAPZUkHy off=12957149 len=1645688 2024-09-13T00:01:58.428 INFO:tasks.workunit.client.0.smithi136.stdout:op 173 completed, throughput=5.01MB/sec 2024-09-13T00:01:58.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:01:58 smithi181 bash[67587]: cluster 2024-09-13T00:01:57.166741+0000 mgr.y (mgr.24661) 745 : cluster [DBG] pgmap v1428: 105 pgs: 105 active+clean; 126 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.9 MiB/s wr, 4 op/s 2024-09-13T00:01:58.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:01:58 smithi136 bash[75164]: cluster 2024-09-13T00:01:57.166741+0000 mgr.y (mgr.24661) 745 : cluster [DBG] pgmap v1428: 105 pgs: 105 active+clean; 126 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.9 MiB/s wr, 4 op/s 2024-09-13T00:01:58.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:01:58 smithi136 bash[76371]: cluster 2024-09-13T00:01:57.166741+0000 mgr.y (mgr.24661) 745 : cluster [DBG] pgmap v1428: 105 pgs: 105 active+clean; 126 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.9 MiB/s wr, 4 op/s 2024-09-13T00:01:59.429 INFO:tasks.workunit.client.0.smithi136.stdout: 35: throughput=4.87MB/sec pending data=0 2024-09-13T00:01:59.429 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-nA5obpmVmVRpVAG off=11746201 len=1594804 2024-09-13T00:01:59.433 INFO:tasks.workunit.client.0.smithi136.stdout:op 174 completed, throughput=4.91MB/sec 2024-09-13T00:01:59.433 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ojbcEVm4rpWJSM1 off=3624635 len=258611 2024-09-13T00:01:59.438 INFO:tasks.workunit.client.0.smithi136.stdout:op 175 completed, throughput=4.92MB/sec 2024-09-13T00:01:59.438 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-xumGWUCbqC2nm85 off=5860348 len=955740 2024-09-13T00:01:59.441 INFO:tasks.workunit.client.0.smithi136.stdout:op 176 completed, throughput=4.94MB/sec 2024-09-13T00:01:59.441 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-dyyWKBtqKpb7_v_ off=19010726 len=756301 2024-09-13T00:01:59.448 INFO:tasks.workunit.client.0.smithi136.stdout:op 177 completed, throughput=4.96MB/sec 2024-09-13T00:01:59.448 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-xRQWp9m_o7oiAor off=1476950 len=1923495 2024-09-13T00:01:59.461 INFO:tasks.workunit.client.0.smithi136.stdout:op 178 completed, throughput=5.01MB/sec 2024-09-13T00:01:59.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:01:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:01:59] "GET /metrics HTTP/1.1" 200 34118 "" "Prometheus/2.43.0" 2024-09-13T00:02:00.461 INFO:tasks.workunit.client.0.smithi136.stdout: 36: throughput=4.88MB/sec pending data=0 2024-09-13T00:02:00.461 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Lw2kR0xVusvC9Kw off=1462617 len=345817 2024-09-13T00:02:00.463 INFO:tasks.workunit.client.0.smithi136.stdout:op 179 completed, throughput=4.88MB/sec 2024-09-13T00:02:00.463 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-gJwTpCpEFfhzJTf off=4692094 len=249931 2024-09-13T00:02:00.465 INFO:tasks.workunit.client.0.smithi136.stdout:op 180 completed, throughput=4.89MB/sec 2024-09-13T00:02:00.465 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj--aMh9Lv9TzUFOJF off=170713 len=341047 2024-09-13T00:02:00.471 INFO:tasks.workunit.client.0.smithi136.stdout:op 181 completed, throughput=4.9MB/sec 2024-09-13T00:02:00.471 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-kLQGCxcco91JEZ0 off=9057796 len=789213 2024-09-13T00:02:00.473 INFO:tasks.workunit.client.0.smithi136.stdout:op 182 completed, throughput=4.92MB/sec 2024-09-13T00:02:00.473 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-VR9XOdGhn1E_xLw off=19502961 len=1230565 2024-09-13T00:02:00.478 INFO:tasks.workunit.client.0.smithi136.stdout:op 183 completed, throughput=4.95MB/sec 2024-09-13T00:02:00.478 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-xFcb8rMe9U_T0vv off=3556165 len=1334324 2024-09-13T00:02:00.488 INFO:tasks.workunit.client.0.smithi136.stdout:op 184 completed, throughput=4.98MB/sec 2024-09-13T00:02:00.488 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-IoqHS4cLmRsmOWD off=20351594 len=1988737 2024-09-13T00:02:00.492 INFO:tasks.workunit.client.0.smithi136.stdout:op 185 completed, throughput=5.03MB/sec 2024-09-13T00:02:00.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:00 smithi181 bash[67587]: cluster 2024-09-13T00:01:59.167638+0000 mgr.y (mgr.24661) 746 : cluster [DBG] pgmap v1429: 105 pgs: 105 active+clean; 126 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:02:00.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:00 smithi136 bash[75164]: cluster 2024-09-13T00:01:59.167638+0000 mgr.y (mgr.24661) 746 : cluster [DBG] pgmap v1429: 105 pgs: 105 active+clean; 126 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:02:00.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:00 smithi136 bash[76371]: cluster 2024-09-13T00:01:59.167638+0000 mgr.y (mgr.24661) 746 : cluster [DBG] pgmap v1429: 105 pgs: 105 active+clean; 126 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:02:01.493 INFO:tasks.workunit.client.0.smithi136.stdout: 37: throughput=4.9MB/sec pending data=0 2024-09-13T00:02:01.493 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-veU1XH3rJTKqmmC off=5630096 len=1049065 2024-09-13T00:02:01.496 INFO:tasks.workunit.client.0.smithi136.stdout:op 186 completed, throughput=4.93MB/sec 2024-09-13T00:02:01.496 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-BleoWRvKYLJuAy5 off=437295 len=1117828 2024-09-13T00:02:01.499 INFO:tasks.workunit.client.0.smithi136.stdout:op 187 completed, throughput=4.95MB/sec 2024-09-13T00:02:01.499 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Y_Ppbkoqt-oyEs_ off=11701066 len=585700 2024-09-13T00:02:01.506 INFO:tasks.workunit.client.0.smithi136.stdout:op 188 completed, throughput=4.97MB/sec 2024-09-13T00:02:01.506 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-IAsgZ2wMUkiQ4Ho off=8364182 len=1241259 2024-09-13T00:02:01.517 INFO:tasks.workunit.client.0.smithi136.stdout:op 189 completed, throughput=5MB/sec 2024-09-13T00:02:01.518 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-eVlrf1qLfR6zC7e off=19710513 len=2059224 2024-09-13T00:02:01.530 INFO:tasks.workunit.client.0.smithi136.stdout:op 190 completed, throughput=5.05MB/sec 2024-09-13T00:02:02.530 INFO:tasks.workunit.client.0.smithi136.stdout: 38: throughput=4.92MB/sec pending data=0 2024-09-13T00:02:02.531 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-lNg_ZF0qp3moeTG off=1592215 len=2008867 2024-09-13T00:02:02.535 INFO:tasks.workunit.client.0.smithi136.stdout:op 191 completed, throughput=4.97MB/sec 2024-09-13T00:02:02.535 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-C1OdJEkPuSvsJTP off=3353011 len=298829 2024-09-13T00:02:02.541 INFO:tasks.workunit.client.0.smithi136.stdout:op 192 completed, throughput=4.97MB/sec 2024-09-13T00:02:02.541 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-9XvJqkHIgS3avVo off=7518328 len=1317400 2024-09-13T00:02:02.544 INFO:tasks.workunit.client.0.smithi136.stdout:op 193 completed, throughput=5.01MB/sec 2024-09-13T00:02:02.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:02 smithi181 bash[67587]: cluster 2024-09-13T00:02:01.168518+0000 mgr.y (mgr.24661) 747 : cluster [DBG] pgmap v1430: 105 pgs: 105 active+clean; 126 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:02:02.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:02:02 smithi181 bash[95505]: ts=2024-09-13T00:02:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:02:02.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:02 smithi136 bash[75164]: cluster 2024-09-13T00:02:01.168518+0000 mgr.y (mgr.24661) 747 : cluster [DBG] pgmap v1430: 105 pgs: 105 active+clean; 126 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:02:02.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:02 smithi136 bash[76371]: cluster 2024-09-13T00:02:01.168518+0000 mgr.y (mgr.24661) 747 : cluster [DBG] pgmap v1430: 105 pgs: 105 active+clean; 126 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:02:03.545 INFO:tasks.workunit.client.0.smithi136.stdout: 39: throughput=4.88MB/sec pending data=0 2024-09-13T00:02:03.545 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_kt9Se9E8Qo3vbj off=1798961 len=510378 2024-09-13T00:02:03.551 INFO:tasks.workunit.client.0.smithi136.stdout:op 194 completed, throughput=4.89MB/sec 2024-09-13T00:02:03.551 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-I7mhV0ccdDv3OwL off=1790396 len=1535438 2024-09-13T00:02:03.565 INFO:tasks.workunit.client.0.smithi136.stdout:op 195 completed, throughput=4.93MB/sec 2024-09-13T00:02:03.566 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-sHRUnmHgNG63b8q off=3792760 len=1088672 2024-09-13T00:02:03.574 INFO:tasks.workunit.client.0.smithi136.stdout:op 196 completed, throughput=4.95MB/sec 2024-09-13T00:02:03.574 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-sVyH_XckjTNOky6 off=13891144 len=1088438 2024-09-13T00:02:03.583 INFO:tasks.workunit.client.0.smithi136.stdout:op 197 completed, throughput=4.98MB/sec 2024-09-13T00:02:03.583 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-lgJzMSCUYjWpNy7 off=2348018 len=1702771 2024-09-13T00:02:03.587 INFO:tasks.workunit.client.0.smithi136.stdout:op 198 completed, throughput=5.02MB/sec 2024-09-13T00:02:04.588 INFO:tasks.workunit.client.0.smithi136.stdout: 40: throughput=4.89MB/sec pending data=0 2024-09-13T00:02:04.588 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-NiGJX1z8DX2wN9a off=1646757 len=887613 2024-09-13T00:02:04.596 INFO:tasks.workunit.client.0.smithi136.stdout:op 199 completed, throughput=4.91MB/sec 2024-09-13T00:02:04.596 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-lVFX4Nl4zcvaCUI off=3050949 len=251153 2024-09-13T00:02:04.597 INFO:tasks.workunit.client.0.smithi136.stdout:op 200 completed, throughput=4.92MB/sec 2024-09-13T00:02:04.597 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-SsL-mb69IX1vZye off=381727 len=992644 2024-09-13T00:02:04.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:04 smithi181 bash[67587]: cluster 2024-09-13T00:02:03.169565+0000 mgr.y (mgr.24661) 748 : cluster [DBG] pgmap v1431: 105 pgs: 105 active+clean; 126 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:02:04.600 INFO:tasks.workunit.client.0.smithi136.stdout:op 201 completed, throughput=4.94MB/sec 2024-09-13T00:02:04.600 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-OvwuVHi9XphRprU off=15131243 len=1819125 2024-09-13T00:02:04.604 INFO:tasks.workunit.client.0.smithi136.stdout:op 202 completed, throughput=4.98MB/sec 2024-09-13T00:02:04.604 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-7KA80XLuocBv0Bs off=1670307 len=1214011 2024-09-13T00:02:04.614 INFO:tasks.workunit.client.0.smithi136.stdout:op 203 completed, throughput=5.01MB/sec 2024-09-13T00:02:04.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:04 smithi136 bash[75164]: cluster 2024-09-13T00:02:03.169565+0000 mgr.y (mgr.24661) 748 : cluster [DBG] pgmap v1431: 105 pgs: 105 active+clean; 126 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:02:04.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:04 smithi136 bash[76371]: cluster 2024-09-13T00:02:03.169565+0000 mgr.y (mgr.24661) 748 : cluster [DBG] pgmap v1431: 105 pgs: 105 active+clean; 126 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:02:05.614 INFO:tasks.workunit.client.0.smithi136.stdout: 41: throughput=4.89MB/sec pending data=0 2024-09-13T00:02:05.614 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ySveQ3HUtQwmsL_ off=9486779 len=33959 2024-09-13T00:02:05.615 INFO:tasks.workunit.client.0.smithi136.stdout:op 204 completed, throughput=4.89MB/sec 2024-09-13T00:02:05.616 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Rc2CTFbB-EN7lN1 off=7178567 len=447425 2024-09-13T00:02:05.618 INFO:tasks.workunit.client.0.smithi136.stdout:op 205 completed, throughput=4.9MB/sec 2024-09-13T00:02:05.618 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-kLQGCxcco91JEZ0 off=6054117 len=966868 2024-09-13T00:02:05.621 INFO:tasks.workunit.client.0.smithi136.stdout:op 206 completed, throughput=4.92MB/sec 2024-09-13T00:02:05.621 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-oK58nb1hDj_yaWF off=12890465 len=543910 2024-09-13T00:02:05.627 INFO:tasks.workunit.client.0.smithi136.stdout:op 207 completed, throughput=4.94MB/sec 2024-09-13T00:02:05.627 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-NS-6lxOiwgZVX-M off=11199577 len=714189 2024-09-13T00:02:05.635 INFO:tasks.workunit.client.0.smithi136.stdout:op 208 completed, throughput=4.95MB/sec 2024-09-13T00:02:05.635 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj--ABJHGKJNdS-2TR off=1319688 len=665238 2024-09-13T00:02:05.637 INFO:tasks.workunit.client.0.smithi136.stdout:op 209 completed, throughput=4.97MB/sec 2024-09-13T00:02:05.637 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ptgFVU5A6wi5kXi off=11830506 len=44390 2024-09-13T00:02:05.638 INFO:tasks.workunit.client.0.smithi136.stdout:op 210 completed, throughput=4.97MB/sec 2024-09-13T00:02:05.638 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-CViGNPhEeioiJYT off=14719783 len=841397 2024-09-13T00:02:05.641 INFO:tasks.workunit.client.0.smithi136.stdout:op 211 completed, throughput=4.98MB/sec 2024-09-13T00:02:05.642 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-PAwVe_18eC_rahJ off=4407483 len=397843 2024-09-13T00:02:05.647 INFO:tasks.workunit.client.0.smithi136.stdout:op 212 completed, throughput=4.99MB/sec 2024-09-13T00:02:05.647 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-AbFsDbdVvVjXY5Y off=1488841 len=773675 2024-09-13T00:02:05.654 INFO:tasks.workunit.client.0.smithi136.stdout:op 213 completed, throughput=5.01MB/sec 2024-09-13T00:02:06.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:06 smithi181 bash[67587]: cluster 2024-09-13T00:02:05.170383+0000 mgr.y (mgr.24661) 749 : cluster [DBG] pgmap v1432: 105 pgs: 105 active+clean; 126 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.2 MiB/s wr, 5 op/s 2024-09-13T00:02:06.654 INFO:tasks.workunit.client.0.smithi136.stdout: 42: throughput=4.89MB/sec pending data=0 2024-09-13T00:02:06.654 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-SSySA-0m2OUPHj_ off=4162125 len=1621346 2024-09-13T00:02:06.658 INFO:tasks.workunit.client.0.smithi136.stdout:op 214 completed, throughput=4.93MB/sec 2024-09-13T00:02:06.658 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-FmdKzks6eCSC2FT off=5352887 len=1337482 2024-09-13T00:02:06.662 INFO:tasks.workunit.client.0.smithi136.stdout:op 215 completed, throughput=4.96MB/sec 2024-09-13T00:02:06.662 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-8UkYCP7KF1AU0vi off=12080485 len=191660 2024-09-13T00:02:06.667 INFO:tasks.workunit.client.0.smithi136.stdout:op 216 completed, throughput=4.96MB/sec 2024-09-13T00:02:06.667 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Y6enpbJTLF5avAp off=10410657 len=1395106 2024-09-13T00:02:06.679 INFO:tasks.workunit.client.0.smithi136.stdout:op 217 completed, throughput=4.99MB/sec 2024-09-13T00:02:06.679 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-dX2u_QZL-KrK-7G off=19746185 len=117540 2024-09-13T00:02:06.684 INFO:tasks.workunit.client.0.smithi136.stdout:op 218 completed, throughput=4.99MB/sec 2024-09-13T00:02:06.684 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-edcXD6FUxD21FyU off=6193736 len=844364 2024-09-13T00:02:06.686 INFO:tasks.workunit.client.0.smithi136.stdout:op 219 completed, throughput=5.01MB/sec 2024-09-13T00:02:06.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:06 smithi136 bash[75164]: cluster 2024-09-13T00:02:05.170383+0000 mgr.y (mgr.24661) 749 : cluster [DBG] pgmap v1432: 105 pgs: 105 active+clean; 126 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.2 MiB/s wr, 5 op/s 2024-09-13T00:02:06.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:06 smithi136 bash[76371]: cluster 2024-09-13T00:02:05.170383+0000 mgr.y (mgr.24661) 749 : cluster [DBG] pgmap v1432: 105 pgs: 105 active+clean; 126 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.2 MiB/s wr, 5 op/s 2024-09-13T00:02:07.686 INFO:tasks.workunit.client.0.smithi136.stdout: 43: throughput=4.9MB/sec pending data=0 2024-09-13T00:02:07.687 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-crKtsMmdVN2cxKN off=3154087 len=1223212 2024-09-13T00:02:07.698 INFO:tasks.workunit.client.0.smithi136.stdout:op 220 completed, throughput=4.92MB/sec 2024-09-13T00:02:07.699 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-G69-TJiUfi65J_M off=3175046 len=1015187 2024-09-13T00:02:07.701 INFO:tasks.workunit.client.0.smithi136.stdout:op 221 completed, throughput=4.95MB/sec 2024-09-13T00:02:07.701 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-YeVupRoyKM3KCg6 off=3002782 len=474302 2024-09-13T00:02:07.708 INFO:tasks.workunit.client.0.smithi136.stdout:op 222 completed, throughput=4.95MB/sec 2024-09-13T00:02:07.708 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-CALRqHvoDj_RZUD off=2885883 len=49259 2024-09-13T00:02:07.709 INFO:tasks.workunit.client.0.smithi136.stdout:op 223 completed, throughput=4.96MB/sec 2024-09-13T00:02:07.709 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-9DSnnvlPCxbQbd_ off=16976579 len=530927 2024-09-13T00:02:07.710 INFO:tasks.workunit.client.0.smithi136.stdout:op 224 completed, throughput=4.97MB/sec 2024-09-13T00:02:07.711 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-FwIFW_A8nBuABLx off=931938 len=1342911 2024-09-13T00:02:07.714 INFO:tasks.workunit.client.0.smithi136.stdout:op 225 completed, throughput=5MB/sec 2024-09-13T00:02:07.714 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-8BPydAoSXJRGT5i off=14880004 len=339496 2024-09-13T00:02:07.719 INFO:tasks.workunit.client.0.smithi136.stdout:op 226 completed, throughput=5MB/sec 2024-09-13T00:02:08.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:08 smithi181 bash[67587]: cluster 2024-09-13T00:02:07.171500+0000 mgr.y (mgr.24661) 750 : cluster [DBG] pgmap v1433: 105 pgs: 105 active+clean; 126 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:02:08.720 INFO:tasks.workunit.client.0.smithi136.stdout: 44: throughput=4.89MB/sec pending data=0 2024-09-13T00:02:08.720 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-LDpg2u1q1mdul_K off=6350342 len=1018879 2024-09-13T00:02:08.722 INFO:tasks.workunit.client.0.smithi136.stdout:op 227 completed, throughput=4.91MB/sec 2024-09-13T00:02:08.723 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-IGbyra7w00fmkp1 off=9771394 len=310955 2024-09-13T00:02:08.724 INFO:tasks.workunit.client.0.smithi136.stdout:op 228 completed, throughput=4.92MB/sec 2024-09-13T00:02:08.724 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-gmSt3zHn2du42Vl off=5517859 len=1263311 2024-09-13T00:02:08.728 INFO:tasks.workunit.client.0.smithi136.stdout:op 229 completed, throughput=4.95MB/sec 2024-09-13T00:02:08.728 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-MAtw_oJ18hB_8st off=18672869 len=1726583 2024-09-13T00:02:08.733 INFO:tasks.workunit.client.0.smithi136.stdout:op 230 completed, throughput=4.98MB/sec 2024-09-13T00:02:08.733 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Jot25jeGaQd_TZC off=12789180 len=78828 2024-09-13T00:02:08.734 INFO:tasks.workunit.client.0.smithi136.stdout:op 231 completed, throughput=4.98MB/sec 2024-09-13T00:02:08.734 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ohEVnJe3vqs6fqT off=112571 len=431141 2024-09-13T00:02:08.741 INFO:tasks.workunit.client.0.smithi136.stdout:op 232 completed, throughput=4.99MB/sec 2024-09-13T00:02:08.741 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Nl3H27IglkAhL-J off=15388074 len=424234 2024-09-13T00:02:08.747 INFO:tasks.workunit.client.0.smithi136.stdout:op 233 completed, throughput=5MB/sec 2024-09-13T00:02:08.747 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-yqhWY3yC7tpQqwV off=20284357 len=1754771 2024-09-13T00:02:08.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:08 smithi136 bash[75164]: cluster 2024-09-13T00:02:07.171500+0000 mgr.y (mgr.24661) 750 : cluster [DBG] pgmap v1433: 105 pgs: 105 active+clean; 126 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:02:08.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:08 smithi136 bash[76371]: cluster 2024-09-13T00:02:07.171500+0000 mgr.y (mgr.24661) 750 : cluster [DBG] pgmap v1433: 105 pgs: 105 active+clean; 126 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:02:08.760 INFO:tasks.workunit.client.0.smithi136.stdout:op 234 completed, throughput=5.03MB/sec 2024-09-13T00:02:09.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:09 smithi181 bash[67587]: audit 2024-09-13T00:02:08.772455+0000 mon.a (mon.0) 2102 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:02:09.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:02:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:02:09] "GET /metrics HTTP/1.1" 200 34122 "" "Prometheus/2.43.0" 2024-09-13T00:02:09.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:09 smithi136 bash[76371]: audit 2024-09-13T00:02:08.772455+0000 mon.a (mon.0) 2102 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:02:09.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:09 smithi136 bash[75164]: audit 2024-09-13T00:02:08.772455+0000 mon.a (mon.0) 2102 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:02:09.761 INFO:tasks.workunit.client.0.smithi136.stdout: 45: throughput=4.93MB/sec pending data=0 2024-09-13T00:02:09.761 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-iNfg9a7E3DmIlJX off=14085706 len=1520170 2024-09-13T00:02:09.764 INFO:tasks.workunit.client.0.smithi136.stdout:op 235 completed, throughput=4.96MB/sec 2024-09-13T00:02:09.764 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-g8nO0ywRD3twOc_ off=1901803 len=144935 2024-09-13T00:02:09.766 INFO:tasks.workunit.client.0.smithi136.stdout:op 236 completed, throughput=4.96MB/sec 2024-09-13T00:02:09.766 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-2gayoh-UNBjLqm3 off=5202650 len=582774 2024-09-13T00:02:09.773 INFO:tasks.workunit.client.0.smithi136.stdout:op 237 completed, throughput=4.97MB/sec 2024-09-13T00:02:09.773 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-qkK3HCPmrQwc7QG off=2220384 len=1657662 2024-09-13T00:02:09.785 INFO:tasks.workunit.client.0.smithi136.stdout:op 238 completed, throughput=5MB/sec 2024-09-13T00:02:10.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:10 smithi181 bash[67587]: cluster 2024-09-13T00:02:09.172388+0000 mgr.y (mgr.24661) 751 : cluster [DBG] pgmap v1434: 105 pgs: 105 active+clean; 126 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.0 MiB/s wr, 5 op/s 2024-09-13T00:02:10.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:10 smithi136 bash[75164]: cluster 2024-09-13T00:02:09.172388+0000 mgr.y (mgr.24661) 751 : cluster [DBG] pgmap v1434: 105 pgs: 105 active+clean; 126 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.0 MiB/s wr, 5 op/s 2024-09-13T00:02:10.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:10 smithi136 bash[76371]: cluster 2024-09-13T00:02:09.172388+0000 mgr.y (mgr.24661) 751 : cluster [DBG] pgmap v1434: 105 pgs: 105 active+clean; 126 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.0 MiB/s wr, 5 op/s 2024-09-13T00:02:10.785 INFO:tasks.workunit.client.0.smithi136.stdout: 46: throughput=4.9MB/sec pending data=0 2024-09-13T00:02:10.785 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-LQtLEALphM9QB9U off=7107804 len=1812874 2024-09-13T00:02:10.791 INFO:tasks.workunit.client.0.smithi136.stdout:op 239 completed, throughput=4.93MB/sec 2024-09-13T00:02:10.791 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-7jAyxwQWPtlZQZr off=15901354 len=604664 2024-09-13T00:02:10.793 INFO:tasks.workunit.client.0.smithi136.stdout:op 240 completed, throughput=4.95MB/sec 2024-09-13T00:02:10.793 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-mCc_KdWGY3blAJn off=18492246 len=1747 2024-09-13T00:02:10.798 INFO:tasks.workunit.client.0.smithi136.stdout:op 241 completed, throughput=4.95MB/sec 2024-09-13T00:02:10.798 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_nfArrBMeoWhwRh off=6642208 len=734881 2024-09-13T00:02:10.805 INFO:tasks.workunit.client.0.smithi136.stdout:op 242 completed, throughput=4.96MB/sec 2024-09-13T00:02:10.805 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-QT3Ong7_7vZXdIR off=8828661 len=878389 2024-09-13T00:02:10.813 INFO:tasks.workunit.client.0.smithi136.stdout:op 243 completed, throughput=4.98MB/sec 2024-09-13T00:02:10.813 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-dQRsoHNoEcjcZgJ off=292730 len=1553397 2024-09-13T00:02:10.824 INFO:tasks.workunit.client.0.smithi136.stdout:op 244 completed, throughput=5.01MB/sec 2024-09-13T00:02:11.824 INFO:tasks.workunit.client.0.smithi136.stdout: 47: throughput=4.9MB/sec pending data=0 2024-09-13T00:02:11.824 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-37CHdfCObV23f9n off=9031023 len=34574 2024-09-13T00:02:11.828 INFO:tasks.workunit.client.0.smithi136.stdout:op 245 completed, throughput=4.9MB/sec 2024-09-13T00:02:11.828 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-21lusSh1IZSSjOB off=18186917 len=1814784 2024-09-13T00:02:11.839 INFO:tasks.workunit.client.0.smithi136.stdout:op 246 completed, throughput=4.94MB/sec 2024-09-13T00:02:11.840 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-zPD-MbuHAFcnZA0 off=5858974 len=328103 2024-09-13T00:02:11.844 INFO:tasks.workunit.client.0.smithi136.stdout:op 247 completed, throughput=4.94MB/sec 2024-09-13T00:02:11.844 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-99Q-oWjqX0RWGiB off=16637205 len=1782733 2024-09-13T00:02:11.845 INFO:tasks.workunit.client.0.smithi136.stdout:op 248 completed, throughput=4.98MB/sec 2024-09-13T00:02:11.845 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-NQJJ6ZS9y5E5W1M off=4740090 len=687280 2024-09-13T00:02:11.847 INFO:tasks.workunit.client.0.smithi136.stdout:op 249 completed, throughput=4.99MB/sec 2024-09-13T00:02:11.847 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_biujfqXMfWMDJ1 off=17524 len=534328 2024-09-13T00:02:11.852 INFO:tasks.workunit.client.0.smithi136.stdout:op 250 completed, throughput=5MB/sec 2024-09-13T00:02:12.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:12 smithi181 bash[67587]: cluster 2024-09-13T00:02:11.173271+0000 mgr.y (mgr.24661) 752 : cluster [DBG] pgmap v1435: 105 pgs: 105 active+clean; 126 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:02:12.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:02:12 smithi181 bash[95505]: ts=2024-09-13T00:02:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:02:12.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:12 smithi136 bash[75164]: cluster 2024-09-13T00:02:11.173271+0000 mgr.y (mgr.24661) 752 : cluster [DBG] pgmap v1435: 105 pgs: 105 active+clean; 126 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:02:12.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:12 smithi136 bash[76371]: cluster 2024-09-13T00:02:11.173271+0000 mgr.y (mgr.24661) 752 : cluster [DBG] pgmap v1435: 105 pgs: 105 active+clean; 126 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:02:12.853 INFO:tasks.workunit.client.0.smithi136.stdout: 48: throughput=4.9MB/sec pending data=0 2024-09-13T00:02:12.853 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-VF7tt3FnjmYBabL off=8378882 len=1541788 2024-09-13T00:02:12.856 INFO:tasks.workunit.client.0.smithi136.stdout:op 251 completed, throughput=4.93MB/sec 2024-09-13T00:02:12.856 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-R2RA3IRMN-KQDpp off=1562915 len=1477293 2024-09-13T00:02:12.868 INFO:tasks.workunit.client.0.smithi136.stdout:op 252 completed, throughput=4.96MB/sec 2024-09-13T00:02:12.868 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-CoemjvNh0kmYpUk off=2874921 len=363780 2024-09-13T00:02:12.869 INFO:tasks.workunit.client.0.smithi136.stdout:op 253 completed, throughput=4.96MB/sec 2024-09-13T00:02:12.869 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-teFQoIHZ2R7kzbL off=11882859 len=567827 2024-09-13T00:02:12.875 INFO:tasks.workunit.client.0.smithi136.stdout:op 254 completed, throughput=4.97MB/sec 2024-09-13T00:02:12.876 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Uyq-CPTIbRmqTPy off=12565864 len=641981 2024-09-13T00:02:12.883 INFO:tasks.workunit.client.0.smithi136.stdout:op 255 completed, throughput=4.99MB/sec 2024-09-13T00:02:12.883 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-YgLypEadHwge-zY off=13906269 len=1532284 2024-09-13T00:02:12.893 INFO:tasks.workunit.client.0.smithi136.stdout:op 256 completed, throughput=5.01MB/sec 2024-09-13T00:02:13.893 INFO:tasks.workunit.client.0.smithi136.stdout: 49: throughput=4.92MB/sec pending data=0 2024-09-13T00:02:13.894 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-j6akZpPKtbXJcqV off=1524487 len=532963 2024-09-13T00:02:13.900 INFO:tasks.workunit.client.0.smithi136.stdout:op 257 completed, throughput=4.92MB/sec 2024-09-13T00:02:13.900 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-nAyym8IgiRcaKeJ off=369695 len=697681 2024-09-13T00:02:13.907 INFO:tasks.workunit.client.0.smithi136.stdout:op 258 completed, throughput=4.94MB/sec 2024-09-13T00:02:13.907 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-77C5KCiXt0yAwyU off=1101484 len=1126434 2024-09-13T00:02:13.911 INFO:tasks.workunit.client.0.smithi136.stdout:op 259 completed, throughput=4.96MB/sec 2024-09-13T00:02:13.912 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-mLn3DmRR7CSaE2d off=14400269 len=1831748 2024-09-13T00:02:13.923 INFO:tasks.workunit.client.0.smithi136.stdout:op 260 completed, throughput=4.99MB/sec 2024-09-13T00:02:13.923 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-z0JVrVFKt7FSoOC off=12629883 len=1537382 2024-09-13T00:02:13.932 INFO:tasks.workunit.client.0.smithi136.stdout:op 261 completed, throughput=5.02MB/sec 2024-09-13T00:02:14.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:14 smithi181 bash[67587]: cluster 2024-09-13T00:02:13.174264+0000 mgr.y (mgr.24661) 753 : cluster [DBG] pgmap v1436: 105 pgs: 105 active+clean; 126 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:02:14.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:14 smithi136 bash[75164]: cluster 2024-09-13T00:02:13.174264+0000 mgr.y (mgr.24661) 753 : cluster [DBG] pgmap v1436: 105 pgs: 105 active+clean; 126 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:02:14.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:14 smithi136 bash[76371]: cluster 2024-09-13T00:02:13.174264+0000 mgr.y (mgr.24661) 753 : cluster [DBG] pgmap v1436: 105 pgs: 105 active+clean; 126 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:02:14.932 INFO:tasks.workunit.client.0.smithi136.stdout: 50: throughput=4.92MB/sec pending data=0 2024-09-13T00:02:14.932 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-HR6BruNuZR0HDGH off=1639888 len=2013939 2024-09-13T00:02:14.947 INFO:tasks.workunit.client.0.smithi136.stdout:op 262 completed, throughput=4.96MB/sec 2024-09-13T00:02:14.947 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-VmVQi9yB3glCjbg off=14335147 len=388551 2024-09-13T00:02:14.953 INFO:tasks.workunit.client.0.smithi136.stdout:op 263 completed, throughput=4.96MB/sec 2024-09-13T00:02:14.953 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-FMv0OzYDavrE3Hy off=12925087 len=719729 2024-09-13T00:02:14.955 INFO:tasks.workunit.client.0.smithi136.stdout:op 264 completed, throughput=4.98MB/sec 2024-09-13T00:02:14.955 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ZHVFotzwVD0lFrv off=2173914 len=1172084 2024-09-13T00:02:14.967 INFO:tasks.workunit.client.0.smithi136.stdout:op 265 completed, throughput=5MB/sec 2024-09-13T00:02:14.967 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-d-IBfRw1geNCkFp off=603822 len=1827549 2024-09-13T00:02:14.970 INFO:tasks.workunit.client.0.smithi136.stdout:op 266 completed, throughput=5.03MB/sec 2024-09-13T00:02:15.970 INFO:tasks.workunit.client.0.smithi136.stdout: 51: throughput=4.94MB/sec pending data=0 2024-09-13T00:02:15.970 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-hYZ6gHbc1BlRN0L off=9895669 len=1811984 2024-09-13T00:02:15.985 INFO:tasks.workunit.client.0.smithi136.stdout:op 267 completed, throughput=4.97MB/sec 2024-09-13T00:02:15.985 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-UUi0fiNlxM4IcUc off=16945649 len=865486 2024-09-13T00:02:15.993 INFO:tasks.workunit.client.0.smithi136.stdout:op 268 completed, throughput=4.98MB/sec 2024-09-13T00:02:15.994 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-WCsOTRRwhf2yMnS off=11191906 len=1169275 2024-09-13T00:02:16.004 INFO:tasks.workunit.client.0.smithi136.stdout:op 269 completed, throughput=5MB/sec 2024-09-13T00:02:16.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:16 smithi181 bash[67587]: cluster 2024-09-13T00:02:15.175150+0000 mgr.y (mgr.24661) 754 : cluster [DBG] pgmap v1437: 105 pgs: 105 active+clean; 126 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.0 MiB/s wr, 5 op/s 2024-09-13T00:02:16.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:16 smithi136 bash[75164]: cluster 2024-09-13T00:02:15.175150+0000 mgr.y (mgr.24661) 754 : cluster [DBG] pgmap v1437: 105 pgs: 105 active+clean; 126 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.0 MiB/s wr, 5 op/s 2024-09-13T00:02:16.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:16 smithi136 bash[76371]: cluster 2024-09-13T00:02:15.175150+0000 mgr.y (mgr.24661) 754 : cluster [DBG] pgmap v1437: 105 pgs: 105 active+clean; 126 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.0 MiB/s wr, 5 op/s 2024-09-13T00:02:17.004 INFO:tasks.workunit.client.0.smithi136.stdout: 52: throughput=4.91MB/sec pending data=0 2024-09-13T00:02:17.004 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-sCZT7XZ_6HRji6e off=5374401 len=524907 2024-09-13T00:02:17.012 INFO:tasks.workunit.client.0.smithi136.stdout:op 270 completed, throughput=4.92MB/sec 2024-09-13T00:02:17.012 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-StcjnFFkOMjOR5D off=2009154 len=459493 2024-09-13T00:02:17.014 INFO:tasks.workunit.client.0.smithi136.stdout:op 271 completed, throughput=4.93MB/sec 2024-09-13T00:02:17.014 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-KQtqdpz8Mvn05SF off=14996414 len=975905 2024-09-13T00:02:17.017 INFO:tasks.workunit.client.0.smithi136.stdout:op 272 completed, throughput=4.94MB/sec 2024-09-13T00:02:17.017 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-hPHr-lvFUMQ2LGz off=7992739 len=880892 2024-09-13T00:02:17.020 INFO:tasks.workunit.client.0.smithi136.stdout:op 273 completed, throughput=4.96MB/sec 2024-09-13T00:02:17.020 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-DFrHJq3H4drUnkj off=14783466 len=2083786 2024-09-13T00:02:17.028 INFO:tasks.workunit.client.0.smithi136.stdout:op 274 completed, throughput=5MB/sec 2024-09-13T00:02:17.028 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-cSNh0q2BvqLj98a off=3034868 len=1166222 2024-09-13T00:02:17.029 INFO:tasks.workunit.client.0.smithi136.stdout:op 275 completed, throughput=5.02MB/sec 2024-09-13T00:02:18.029 INFO:tasks.workunit.client.0.smithi136.stdout: 53: throughput=4.92MB/sec pending data=0 2024-09-13T00:02:18.029 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-zZuQp7tlUXjPHw3 off=10699142 len=401116 2024-09-13T00:02:18.035 INFO:tasks.workunit.client.0.smithi136.stdout:op 276 completed, throughput=4.93MB/sec 2024-09-13T00:02:18.035 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-QHOQsx3zFDaOpWi off=498214 len=1863741 2024-09-13T00:02:18.052 INFO:tasks.workunit.client.0.smithi136.stdout:op 277 completed, throughput=4.96MB/sec 2024-09-13T00:02:18.052 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-xQKVgDrB4Ez1CfL off=11787673 len=1727781 2024-09-13T00:02:18.057 INFO:tasks.workunit.client.0.smithi136.stdout:op 278 completed, throughput=4.99MB/sec 2024-09-13T00:02:18.057 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-k_zN_ASmwaBmcqY off=5687304 len=1886706 2024-09-13T00:02:18.071 INFO:tasks.workunit.client.0.smithi136.stdout:op 279 completed, throughput=5.02MB/sec 2024-09-13T00:02:18.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:18 smithi136 bash[75164]: cluster 2024-09-13T00:02:17.176313+0000 mgr.y (mgr.24661) 755 : cluster [DBG] pgmap v1438: 105 pgs: 105 active+clean; 126 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.8 MiB/s wr, 6 op/s 2024-09-13T00:02:18.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:18 smithi136 bash[76371]: cluster 2024-09-13T00:02:17.176313+0000 mgr.y (mgr.24661) 755 : cluster [DBG] pgmap v1438: 105 pgs: 105 active+clean; 126 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.8 MiB/s wr, 6 op/s 2024-09-13T00:02:18.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:18 smithi181 bash[67587]: cluster 2024-09-13T00:02:17.176313+0000 mgr.y (mgr.24661) 755 : cluster [DBG] pgmap v1438: 105 pgs: 105 active+clean; 126 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.8 MiB/s wr, 6 op/s 2024-09-13T00:02:19.072 INFO:tasks.workunit.client.0.smithi136.stdout: 54: throughput=4.93MB/sec pending data=0 2024-09-13T00:02:19.072 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_7R4gbxqRhR7Btt off=7692550 len=1913056 2024-09-13T00:02:19.084 INFO:tasks.workunit.client.0.smithi136.stdout:op 280 completed, throughput=4.96MB/sec 2024-09-13T00:02:19.084 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-dp4pv6rBPcFaUY9 off=9165270 len=170219 2024-09-13T00:02:19.086 INFO:tasks.workunit.client.0.smithi136.stdout:op 281 completed, throughput=4.97MB/sec 2024-09-13T00:02:19.086 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-cOLFGf0ll2i8NIU off=1575109 len=900836 2024-09-13T00:02:19.088 INFO:tasks.workunit.client.0.smithi136.stdout:op 282 completed, throughput=4.98MB/sec 2024-09-13T00:02:19.088 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-b9qaj-VUZ4FW3I_ off=5048645 len=1414555 2024-09-13T00:02:19.099 INFO:tasks.workunit.client.0.smithi136.stdout:op 283 completed, throughput=5.01MB/sec 2024-09-13T00:02:19.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:02:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:02:19] "GET /metrics HTTP/1.1" 200 34122 "" "Prometheus/2.43.0" 2024-09-13T00:02:20.099 INFO:tasks.workunit.client.0.smithi136.stdout: 55: throughput=4.92MB/sec pending data=0 2024-09-13T00:02:20.099 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-5DYPVqR-5126pkx off=2734459 len=556675 2024-09-13T00:02:20.106 INFO:tasks.workunit.client.0.smithi136.stdout:op 284 completed, throughput=4.93MB/sec 2024-09-13T00:02:20.106 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-xLtVTq7rZ35n2dh off=15172084 len=1763376 2024-09-13T00:02:20.110 INFO:tasks.workunit.client.0.smithi136.stdout:op 285 completed, throughput=4.95MB/sec 2024-09-13T00:02:20.110 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-s52IrXD58yRrwKz off=7351492 len=316316 2024-09-13T00:02:20.113 INFO:tasks.workunit.client.0.smithi136.stdout:op 286 completed, throughput=4.96MB/sec 2024-09-13T00:02:20.113 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-lyQcGNEF_q9bO0r off=1284558 len=402082 2024-09-13T00:02:20.118 INFO:tasks.workunit.client.0.smithi136.stdout:op 287 completed, throughput=4.97MB/sec 2024-09-13T00:02:20.118 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-cogpNKWU8P6_btn off=6164662 len=1760969 2024-09-13T00:02:20.122 INFO:tasks.workunit.client.0.smithi136.stdout:op 288 completed, throughput=5MB/sec 2024-09-13T00:02:20.122 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-QceALxebN49DPsB off=2299546 len=914054 2024-09-13T00:02:20.124 INFO:tasks.workunit.client.0.smithi136.stdout:op 289 completed, throughput=5.01MB/sec 2024-09-13T00:02:20.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:20 smithi136 bash[75164]: cluster 2024-09-13T00:02:19.177229+0000 mgr.y (mgr.24661) 756 : cluster [DBG] pgmap v1439: 105 pgs: 105 active+clean; 126 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:02:20.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:20 smithi136 bash[76371]: cluster 2024-09-13T00:02:19.177229+0000 mgr.y (mgr.24661) 756 : cluster [DBG] pgmap v1439: 105 pgs: 105 active+clean; 126 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:02:20.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:20 smithi181 bash[67587]: cluster 2024-09-13T00:02:19.177229+0000 mgr.y (mgr.24661) 756 : cluster [DBG] pgmap v1439: 105 pgs: 105 active+clean; 126 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:02:21.125 INFO:tasks.workunit.client.0.smithi136.stdout: 56: throughput=4.92MB/sec pending data=0 2024-09-13T00:02:21.125 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-8JzcXITXuryJXU0 off=1719230 len=2081234 2024-09-13T00:02:21.139 INFO:tasks.workunit.client.0.smithi136.stdout:op 290 completed, throughput=4.96MB/sec 2024-09-13T00:02:21.139 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Fid34LFAeff0vZY off=3045645 len=948872 2024-09-13T00:02:21.142 INFO:tasks.workunit.client.0.smithi136.stdout:op 291 completed, throughput=4.97MB/sec 2024-09-13T00:02:21.142 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-cCwv4i0swTReL32 off=13067379 len=875043 2024-09-13T00:02:21.146 INFO:tasks.workunit.client.0.smithi136.stdout:op 292 completed, throughput=4.99MB/sec 2024-09-13T00:02:21.146 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-j6akZpPKtbXJcqV off=3866916 len=622943 2024-09-13T00:02:21.148 INFO:tasks.workunit.client.0.smithi136.stdout:op 293 completed, throughput=5MB/sec 2024-09-13T00:02:21.148 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-JZkntZTO45w2WHL off=9237546 len=1456366 2024-09-13T00:02:21.152 INFO:tasks.workunit.client.0.smithi136.stdout:op 294 completed, throughput=5.02MB/sec 2024-09-13T00:02:22.153 INFO:tasks.workunit.client.0.smithi136.stdout: 57: throughput=4.93MB/sec pending data=0 2024-09-13T00:02:22.153 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-cGj_BkJ5qMRJ4by off=3622265 len=1468760 2024-09-13T00:02:22.158 INFO:tasks.workunit.client.0.smithi136.stdout:op 295 completed, throughput=4.96MB/sec 2024-09-13T00:02:22.158 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-tHDPsaELzi-jmQ4 off=19043796 len=608812 2024-09-13T00:02:22.165 INFO:tasks.workunit.client.0.smithi136.stdout:op 296 completed, throughput=4.97MB/sec 2024-09-13T00:02:22.165 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-HJSJ4UVvcne9UvB off=5559010 len=427300 2024-09-13T00:02:22.170 INFO:tasks.workunit.client.0.smithi136.stdout:op 297 completed, throughput=4.97MB/sec 2024-09-13T00:02:22.171 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-gq_q1ssrAfVvibQ off=14951076 len=383818 2024-09-13T00:02:22.172 INFO:tasks.workunit.client.0.smithi136.stdout:op 298 completed, throughput=4.98MB/sec 2024-09-13T00:02:22.172 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-oyxcC0Ban7YeIni off=1941673 len=1237464 2024-09-13T00:02:22.175 INFO:tasks.workunit.client.0.smithi136.stdout:op 299 completed, throughput=5MB/sec 2024-09-13T00:02:22.175 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-kN3i8R-vRRwPF3t off=5199874 len=767232 2024-09-13T00:02:22.178 INFO:tasks.workunit.client.0.smithi136.stdout:op 300 completed, throughput=5.01MB/sec 2024-09-13T00:02:22.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:22 smithi181 bash[67587]: cluster 2024-09-13T00:02:21.178172+0000 mgr.y (mgr.24661) 757 : cluster [DBG] pgmap v1440: 105 pgs: 105 active+clean; 126 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:02:22.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:02:22 smithi181 bash[95505]: ts=2024-09-13T00:02:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:02:22.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:22 smithi136 bash[75164]: cluster 2024-09-13T00:02:21.178172+0000 mgr.y (mgr.24661) 757 : cluster [DBG] pgmap v1440: 105 pgs: 105 active+clean; 126 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:02:22.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:22 smithi136 bash[76371]: cluster 2024-09-13T00:02:21.178172+0000 mgr.y (mgr.24661) 757 : cluster [DBG] pgmap v1440: 105 pgs: 105 active+clean; 126 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:02:23.178 INFO:tasks.workunit.client.0.smithi136.stdout: 58: throughput=4.93MB/sec pending data=0 2024-09-13T00:02:23.178 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-otIgQx9JH-c7-Z6 off=15719589 len=1929054 2024-09-13T00:02:23.185 INFO:tasks.workunit.client.0.smithi136.stdout:op 301 completed, throughput=4.96MB/sec 2024-09-13T00:02:23.185 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-_-yHP28zv5nszYC off=4651373 len=1821036 2024-09-13T00:02:23.189 INFO:tasks.workunit.client.0.smithi136.stdout:op 302 completed, throughput=4.99MB/sec 2024-09-13T00:02:23.189 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-yf020yXf3hCWNM1 off=7691731 len=485831 2024-09-13T00:02:23.192 INFO:tasks.workunit.client.0.smithi136.stdout:op 303 completed, throughput=4.99MB/sec 2024-09-13T00:02:23.192 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-pPiQlgtQ5qnGPb9 off=3799054 len=1163229 2024-09-13T00:02:23.204 INFO:tasks.workunit.client.0.smithi136.stdout:op 304 completed, throughput=5.01MB/sec 2024-09-13T00:02:24.204 INFO:tasks.workunit.client.0.smithi136.stdout: 59: throughput=4.93MB/sec pending data=0 2024-09-13T00:02:24.204 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-tcwCrAU3VJPGvlg off=7056960 len=1222657 2024-09-13T00:02:24.208 INFO:tasks.workunit.client.0.smithi136.stdout:op 305 completed, throughput=4.95MB/sec 2024-09-13T00:02:24.208 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Op4eQqTNKL6nO-G off=19793707 len=1210112 2024-09-13T00:02:24.217 INFO:tasks.workunit.client.0.smithi136.stdout:op 306 completed, throughput=4.97MB/sec 2024-09-13T00:02:24.217 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-PvkiYSDsWvT4-Cx off=5443168 len=1094330 2024-09-13T00:02:24.219 INFO:tasks.workunit.client.0.smithi136.stdout:op 307 completed, throughput=4.98MB/sec 2024-09-13T00:02:24.219 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-BlSPTO7TABw3Gfp off=1566753 len=1119948 2024-09-13T00:02:24.222 INFO:tasks.workunit.client.0.smithi136.stdout:op 308 completed, throughput=5MB/sec 2024-09-13T00:02:24.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:24 smithi136 bash[75164]: cluster 2024-09-13T00:02:23.179175+0000 mgr.y (mgr.24661) 758 : cluster [DBG] pgmap v1441: 105 pgs: 105 active+clean; 126 GiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:02:24.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:24 smithi136 bash[75164]: audit 2024-09-13T00:02:23.772815+0000 mon.a (mon.0) 2103 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:02:24.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:24 smithi136 bash[76371]: cluster 2024-09-13T00:02:23.179175+0000 mgr.y (mgr.24661) 758 : cluster [DBG] pgmap v1441: 105 pgs: 105 active+clean; 126 GiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:02:24.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:24 smithi136 bash[76371]: audit 2024-09-13T00:02:23.772815+0000 mon.a (mon.0) 2103 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:02:24.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:24 smithi181 bash[67587]: cluster 2024-09-13T00:02:23.179175+0000 mgr.y (mgr.24661) 758 : cluster [DBG] pgmap v1441: 105 pgs: 105 active+clean; 126 GiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:02:24.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:24 smithi181 bash[67587]: audit 2024-09-13T00:02:23.772815+0000 mon.a (mon.0) 2103 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:02:25.223 INFO:tasks.workunit.client.0.smithi136.stdout: 60: throughput=4.92MB/sec pending data=0 2024-09-13T00:02:25.223 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-S_WId5VNdRZn7_D off=4171072 len=2033234 2024-09-13T00:02:25.229 INFO:tasks.workunit.client.0.smithi136.stdout:op 309 completed, throughput=4.95MB/sec 2024-09-13T00:02:25.229 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-LYNb40N9nx-0NHO off=20336863 len=2004321 2024-09-13T00:02:25.233 INFO:tasks.workunit.client.0.smithi136.stdout:op 310 completed, throughput=4.98MB/sec 2024-09-13T00:02:25.233 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-vZXJ65J8bNmOwOO off=3373832 len=1983190 2024-09-13T00:02:25.238 INFO:tasks.workunit.client.0.smithi136.stdout:op 311 completed, throughput=5.01MB/sec 2024-09-13T00:02:26.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:25 smithi136 bash[76371]: audit 2024-09-13T00:02:24.645989+0000 mon.a (mon.0) 2104 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:02:26.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:25 smithi136 bash[75164]: audit 2024-09-13T00:02:24.645989+0000 mon.a (mon.0) 2104 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:02:26.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:25 smithi181 bash[67587]: audit 2024-09-13T00:02:24.645989+0000 mon.a (mon.0) 2104 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:02:26.238 INFO:tasks.workunit.client.0.smithi136.stdout: 61: throughput=4.93MB/sec pending data=0 2024-09-13T00:02:26.238 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-rvVGdhEyPyZI1OI off=14550595 len=1702329 2024-09-13T00:02:26.243 INFO:tasks.workunit.client.0.smithi136.stdout:op 312 completed, throughput=4.96MB/sec 2024-09-13T00:02:26.243 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-1qwIpjmeRq4oyRd off=932849 len=593988 2024-09-13T00:02:26.245 INFO:tasks.workunit.client.0.smithi136.stdout:op 313 completed, throughput=4.97MB/sec 2024-09-13T00:02:26.246 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-V8N5vU2f-KlMGJl off=614024 len=29110 2024-09-13T00:02:26.249 INFO:tasks.workunit.client.0.smithi136.stdout:op 314 completed, throughput=4.97MB/sec 2024-09-13T00:02:26.250 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ctJm9xb5cGD3K8H off=18334619 len=411402 2024-09-13T00:02:26.251 INFO:tasks.workunit.client.0.smithi136.stdout:op 315 completed, throughput=4.97MB/sec 2024-09-13T00:02:26.251 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-PkuD6Qxzmzw0bnd off=1689114 len=1901108 2024-09-13T00:02:26.264 INFO:tasks.workunit.client.0.smithi136.stdout:op 316 completed, throughput=5MB/sec 2024-09-13T00:02:27.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:26 smithi136 bash[75164]: cluster 2024-09-13T00:02:25.179969+0000 mgr.y (mgr.24661) 759 : cluster [DBG] pgmap v1442: 105 pgs: 105 active+clean; 126 GiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:02:27.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:26 smithi136 bash[76371]: cluster 2024-09-13T00:02:25.179969+0000 mgr.y (mgr.24661) 759 : cluster [DBG] pgmap v1442: 105 pgs: 105 active+clean; 126 GiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:02:27.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:26 smithi181 bash[67587]: cluster 2024-09-13T00:02:25.179969+0000 mgr.y (mgr.24661) 759 : cluster [DBG] pgmap v1442: 105 pgs: 105 active+clean; 126 GiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:02:27.264 INFO:tasks.workunit.client.0.smithi136.stdout: 62: throughput=4.92MB/sec pending data=0 2024-09-13T00:02:27.264 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ApHemU6XYq0jWiW off=19650443 len=653588 2024-09-13T00:02:27.266 INFO:tasks.workunit.client.0.smithi136.stdout:op 317 completed, throughput=4.93MB/sec 2024-09-13T00:02:27.266 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-QKml_14aIj8a0hx off=2694130 len=2074243 2024-09-13T00:02:27.283 INFO:tasks.workunit.client.0.smithi136.stdout:op 318 completed, throughput=4.96MB/sec 2024-09-13T00:02:27.283 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-O9vDV8_kxnX1VGe off=1595865 len=219771 2024-09-13T00:02:27.289 INFO:tasks.workunit.client.0.smithi136.stdout:op 319 completed, throughput=4.96MB/sec 2024-09-13T00:02:27.289 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-xJdpNQaTD4cuV2b off=3761836 len=1803383 2024-09-13T00:02:27.300 INFO:tasks.workunit.client.0.smithi136.stdout:op 320 completed, throughput=4.99MB/sec 2024-09-13T00:02:27.300 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-F8gohYMOWckeLua off=11611526 len=477012 2024-09-13T00:02:27.306 INFO:tasks.workunit.client.0.smithi136.stdout:op 321 completed, throughput=5MB/sec 2024-09-13T00:02:27.306 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-P8voCH6eTDMmt5C off=336303 len=43476 2024-09-13T00:02:27.310 INFO:tasks.workunit.client.0.smithi136.stdout:op 322 completed, throughput=5MB/sec 2024-09-13T00:02:27.310 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Jx5OHQdbvGXSp39 off=10974195 len=1871641 2024-09-13T00:02:27.320 INFO:tasks.workunit.client.0.smithi136.stdout:op 323 completed, throughput=5.02MB/sec 2024-09-13T00:02:28.320 INFO:tasks.workunit.client.0.smithi136.stdout: 63: throughput=4.95MB/sec pending data=0 2024-09-13T00:02:28.320 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-pw3XJjxgXCkzmwv off=22017 len=233238 2024-09-13T00:02:28.325 INFO:tasks.workunit.client.0.smithi136.stdout:op 324 completed, throughput=4.95MB/sec 2024-09-13T00:02:28.325 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-l2OWAlylhfzD5hp off=17134482 len=1036558 2024-09-13T00:02:28.329 INFO:tasks.workunit.client.0.smithi136.stdout:op 325 completed, throughput=4.96MB/sec 2024-09-13T00:02:28.329 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-0b1V3Yx1yayZUVb off=2944870 len=770997 2024-09-13T00:02:28.331 INFO:tasks.workunit.client.0.smithi136.stdout:op 326 completed, throughput=4.98MB/sec 2024-09-13T00:02:28.331 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-6TD23bEup3-zDab off=1175977 len=1304121 2024-09-13T00:02:28.340 INFO:tasks.workunit.client.0.smithi136.stdout:op 327 completed, throughput=4.99MB/sec 2024-09-13T00:02:28.340 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-6rBZVVLcA9qAnqj off=13733717 len=933701 2024-09-13T00:02:28.341 INFO:tasks.workunit.client.0.smithi136.stdout:op 328 completed, throughput=5.01MB/sec 2024-09-13T00:02:29.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:28 smithi136 bash[75164]: cluster 2024-09-13T00:02:27.181049+0000 mgr.y (mgr.24661) 760 : cluster [DBG] pgmap v1443: 105 pgs: 105 active+clean; 126 GiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.6 MiB/s wr, 8 op/s 2024-09-13T00:02:29.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:28 smithi136 bash[75164]: audit 2024-09-13T00:02:28.381086+0000 mon.a (mon.0) 2105 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:02:29.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:28 smithi136 bash[76371]: cluster 2024-09-13T00:02:27.181049+0000 mgr.y (mgr.24661) 760 : cluster [DBG] pgmap v1443: 105 pgs: 105 active+clean; 126 GiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.6 MiB/s wr, 8 op/s 2024-09-13T00:02:29.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:28 smithi136 bash[76371]: audit 2024-09-13T00:02:28.381086+0000 mon.a (mon.0) 2105 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:02:29.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:28 smithi181 bash[67587]: cluster 2024-09-13T00:02:27.181049+0000 mgr.y (mgr.24661) 760 : cluster [DBG] pgmap v1443: 105 pgs: 105 active+clean; 126 GiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.6 MiB/s wr, 8 op/s 2024-09-13T00:02:29.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:28 smithi181 bash[67587]: audit 2024-09-13T00:02:28.381086+0000 mon.a (mon.0) 2105 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:02:29.342 INFO:tasks.workunit.client.0.smithi136.stdout: 64: throughput=4.93MB/sec pending data=0 2024-09-13T00:02:29.342 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-M3-DLZVOAF6qGO0 off=8866173 len=1957512 2024-09-13T00:02:29.358 INFO:tasks.workunit.client.0.smithi136.stdout:op 329 completed, throughput=4.96MB/sec 2024-09-13T00:02:29.358 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-hyiIUgmeJXVwSuG off=9207313 len=1895969 2024-09-13T00:02:29.376 INFO:tasks.workunit.client.0.smithi136.stdout:op 330 completed, throughput=4.99MB/sec 2024-09-13T00:02:29.377 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-bsOIO2tJz7c0SDD off=433382 len=560059 2024-09-13T00:02:29.384 INFO:tasks.workunit.client.0.smithi136.stdout:op 331 completed, throughput=4.99MB/sec 2024-09-13T00:02:29.384 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-uMszFhRvrvcVcH0 off=11703716 len=1119222 2024-09-13T00:02:29.387 INFO:tasks.workunit.client.0.smithi136.stdout:op 332 completed, throughput=5.01MB/sec 2024-09-13T00:02:29.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:02:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:02:29] "GET /metrics HTTP/1.1" 200 34122 "" "Prometheus/2.43.0" 2024-09-13T00:02:30.388 INFO:tasks.workunit.client.0.smithi136.stdout: 65: throughput=4.93MB/sec pending data=0 2024-09-13T00:02:30.388 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-AdsfvPHVzNw13zd off=19566611 len=812202 2024-09-13T00:02:30.390 INFO:tasks.workunit.client.0.smithi136.stdout:op 333 completed, throughput=4.95MB/sec 2024-09-13T00:02:30.390 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ppArA47C0y5yAJx off=5659433 len=322358 2024-09-13T00:02:30.398 INFO:tasks.workunit.client.0.smithi136.stdout:op 334 completed, throughput=4.95MB/sec 2024-09-13T00:02:30.398 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-P75hTKngVfYBrB9 off=6070547 len=1015534 2024-09-13T00:02:30.401 INFO:tasks.workunit.client.0.smithi136.stdout:op 335 completed, throughput=4.96MB/sec 2024-09-13T00:02:30.401 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-HJBeoOhPMfeDJGg off=1325467 len=755949 2024-09-13T00:02:30.408 INFO:tasks.workunit.client.0.smithi136.stdout:op 336 completed, throughput=4.97MB/sec 2024-09-13T00:02:30.408 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Ew7SLJI_Kv61OqF off=20177581 len=590972 2024-09-13T00:02:30.415 INFO:tasks.workunit.client.0.smithi136.stdout:op 337 completed, throughput=4.98MB/sec 2024-09-13T00:02:30.415 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-gU5oHt5EcOHqZ6C off=4901390 len=442828 2024-09-13T00:02:30.421 INFO:tasks.workunit.client.0.smithi136.stdout:op 338 completed, throughput=4.99MB/sec 2024-09-13T00:02:30.421 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Z2qaGU6uP7XV2w7 off=3612259 len=1644032 2024-09-13T00:02:30.424 INFO:tasks.workunit.client.0.smithi136.stdout:op 339 completed, throughput=5.01MB/sec 2024-09-13T00:02:31.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:30 smithi136 bash[75164]: cluster 2024-09-13T00:02:29.182070+0000 mgr.y (mgr.24661) 761 : cluster [DBG] pgmap v1444: 105 pgs: 105 active+clean; 126 GiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.8 MiB/s wr, 7 op/s 2024-09-13T00:02:31.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:30 smithi136 bash[76371]: cluster 2024-09-13T00:02:29.182070+0000 mgr.y (mgr.24661) 761 : cluster [DBG] pgmap v1444: 105 pgs: 105 active+clean; 126 GiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.8 MiB/s wr, 7 op/s 2024-09-13T00:02:31.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:30 smithi181 bash[67587]: cluster 2024-09-13T00:02:29.182070+0000 mgr.y (mgr.24661) 761 : cluster [DBG] pgmap v1444: 105 pgs: 105 active+clean; 126 GiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.8 MiB/s wr, 7 op/s 2024-09-13T00:02:31.425 INFO:tasks.workunit.client.0.smithi136.stdout: 66: throughput=4.94MB/sec pending data=0 2024-09-13T00:02:31.425 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-zAWl2qD7l-Cqra0 off=5412006 len=934583 2024-09-13T00:02:31.427 INFO:tasks.workunit.client.0.smithi136.stdout:op 340 completed, throughput=4.95MB/sec 2024-09-13T00:02:31.427 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-fKJ6rbz-REPrhbe off=154235 len=1745552 2024-09-13T00:02:31.431 INFO:tasks.workunit.client.0.smithi136.stdout:op 341 completed, throughput=4.97MB/sec 2024-09-13T00:02:31.431 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-C-v0SrweKpjdGWI off=11612573 len=1887752 2024-09-13T00:02:31.448 INFO:tasks.workunit.client.0.smithi136.stdout:op 342 completed, throughput=5MB/sec 2024-09-13T00:02:31.448 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-NiGJX1z8DX2wN9a off=2262375 len=748625 2024-09-13T00:02:31.450 INFO:tasks.workunit.client.0.smithi136.stdout:op 343 completed, throughput=5.01MB/sec 2024-09-13T00:02:32.451 INFO:tasks.workunit.client.0.smithi136.stdout: 67: throughput=4.94MB/sec pending data=0 2024-09-13T00:02:32.451 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-URkvkhcFIFDjP-4 off=3662343 len=1918933 2024-09-13T00:02:32.462 INFO:tasks.workunit.client.0.smithi136.stdout:op 344 completed, throughput=4.96MB/sec 2024-09-13T00:02:32.463 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-NCQb79jqu7oHpkE off=18789 len=1952507 2024-09-13T00:02:32.467 INFO:tasks.workunit.client.0.smithi136.stdout:op 345 completed, throughput=4.99MB/sec 2024-09-13T00:02:32.467 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-X6X_Oq0RwilG5Ox off=2734773 len=394358 2024-09-13T00:02:32.469 INFO:tasks.workunit.client.0.smithi136.stdout:op 346 completed, throughput=4.99MB/sec 2024-09-13T00:02:32.469 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-WM5dvQTI4mLomLe off=3410055 len=117761 2024-09-13T00:02:32.473 INFO:tasks.workunit.client.0.smithi136.stdout:op 347 completed, throughput=5MB/sec 2024-09-13T00:02:32.473 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-2mjsDX9hhA2ZSrN off=4014134 len=378275 2024-09-13T00:02:32.477 INFO:tasks.workunit.client.0.smithi136.stdout:op 348 completed, throughput=5MB/sec 2024-09-13T00:02:32.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:02:32 smithi181 bash[95505]: ts=2024-09-13T00:02:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:02:33.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:32 smithi136 bash[75164]: audit 2024-09-13T00:02:31.112551+0000 mon.a (mon.0) 2106 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T00:02:33.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:32 smithi136 bash[75164]: cluster 2024-09-13T00:02:31.182993+0000 mgr.y (mgr.24661) 762 : cluster [DBG] pgmap v1445: 105 pgs: 105 active+clean; 126 GiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 2.5 MiB/s rd, 1.9 MiB/s wr, 9 op/s 2024-09-13T00:02:33.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:32 smithi136 bash[75164]: audit 2024-09-13T00:02:32.033384+0000 mon.a (mon.0) 2107 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T00:02:33.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:32 smithi136 bash[75164]: audit 2024-09-13T00:02:32.034525+0000 mon.a (mon.0) 2108 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:02:33.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:32 smithi136 bash[75164]: audit 2024-09-13T00:02:32.034890+0000 mon.a (mon.0) 2109 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:02:33.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:32 smithi136 bash[75164]: audit 2024-09-13T00:02:32.035119+0000 mon.a (mon.0) 2110 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:02:33.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:32 smithi136 bash[75164]: audit 2024-09-13T00:02:32.038446+0000 mon.a (mon.0) 2111 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:02:33.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:32 smithi136 bash[75164]: audit 2024-09-13T00:02:32.038652+0000 mon.a (mon.0) 2112 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:02:33.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:32 smithi136 bash[75164]: audit 2024-09-13T00:02:32.038805+0000 mon.a (mon.0) 2113 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:02:33.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:32 smithi136 bash[76371]: audit 2024-09-13T00:02:31.112551+0000 mon.a (mon.0) 2106 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T00:02:33.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:32 smithi136 bash[76371]: cluster 2024-09-13T00:02:31.182993+0000 mgr.y (mgr.24661) 762 : cluster [DBG] pgmap v1445: 105 pgs: 105 active+clean; 126 GiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 2.5 MiB/s rd, 1.9 MiB/s wr, 9 op/s 2024-09-13T00:02:33.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:32 smithi136 bash[76371]: audit 2024-09-13T00:02:32.033384+0000 mon.a (mon.0) 2107 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T00:02:33.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:32 smithi136 bash[76371]: audit 2024-09-13T00:02:32.034525+0000 mon.a (mon.0) 2108 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:02:33.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:32 smithi136 bash[76371]: audit 2024-09-13T00:02:32.034890+0000 mon.a (mon.0) 2109 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:02:33.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:32 smithi136 bash[76371]: audit 2024-09-13T00:02:32.035119+0000 mon.a (mon.0) 2110 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:02:33.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:32 smithi136 bash[76371]: audit 2024-09-13T00:02:32.038446+0000 mon.a (mon.0) 2111 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:02:33.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:32 smithi136 bash[76371]: audit 2024-09-13T00:02:32.038652+0000 mon.a (mon.0) 2112 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:02:33.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:32 smithi136 bash[76371]: audit 2024-09-13T00:02:32.038805+0000 mon.a (mon.0) 2113 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:02:33.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:32 smithi181 bash[67587]: audit 2024-09-13T00:02:31.112551+0000 mon.a (mon.0) 2106 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T00:02:33.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:32 smithi181 bash[67587]: cluster 2024-09-13T00:02:31.182993+0000 mgr.y (mgr.24661) 762 : cluster [DBG] pgmap v1445: 105 pgs: 105 active+clean; 126 GiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 2.5 MiB/s rd, 1.9 MiB/s wr, 9 op/s 2024-09-13T00:02:33.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:32 smithi181 bash[67587]: audit 2024-09-13T00:02:32.033384+0000 mon.a (mon.0) 2107 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T00:02:33.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:32 smithi181 bash[67587]: audit 2024-09-13T00:02:32.034525+0000 mon.a (mon.0) 2108 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:02:33.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:32 smithi181 bash[67587]: audit 2024-09-13T00:02:32.034890+0000 mon.a (mon.0) 2109 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:02:33.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:32 smithi181 bash[67587]: audit 2024-09-13T00:02:32.035119+0000 mon.a (mon.0) 2110 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:02:33.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:32 smithi181 bash[67587]: audit 2024-09-13T00:02:32.038446+0000 mon.a (mon.0) 2111 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:02:33.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:32 smithi181 bash[67587]: audit 2024-09-13T00:02:32.038652+0000 mon.a (mon.0) 2112 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:02:33.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:32 smithi181 bash[67587]: audit 2024-09-13T00:02:32.038805+0000 mon.a (mon.0) 2113 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:02:33.478 INFO:tasks.workunit.client.0.smithi136.stdout: 68: throughput=4.93MB/sec pending data=0 2024-09-13T00:02:33.478 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-WAaFd61YV8miWdl off=11937721 len=323425 2024-09-13T00:02:33.482 INFO:tasks.workunit.client.0.smithi136.stdout:op 349 completed, throughput=4.93MB/sec 2024-09-13T00:02:33.482 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-pZKL_rMGgt0nNXE off=295751 len=554293 2024-09-13T00:02:33.488 INFO:tasks.workunit.client.0.smithi136.stdout:op 350 completed, throughput=4.94MB/sec 2024-09-13T00:02:33.488 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-p_RYyWLYA5RenkZ off=7444899 len=448299 2024-09-13T00:02:33.494 INFO:tasks.workunit.client.0.smithi136.stdout:op 351 completed, throughput=4.95MB/sec 2024-09-13T00:02:33.494 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-OBsO45Qlf9vvGyr off=170527 len=597466 2024-09-13T00:02:33.496 INFO:tasks.workunit.client.0.smithi136.stdout:op 352 completed, throughput=4.95MB/sec 2024-09-13T00:02:33.496 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-t950K-LeYpv_JOf off=7435466 len=611229 2024-09-13T00:02:33.498 INFO:tasks.workunit.client.0.smithi136.stdout:op 353 completed, throughput=4.96MB/sec 2024-09-13T00:02:33.498 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-inbzKp5y-9uqs0h off=5245770 len=932661 2024-09-13T00:02:33.507 INFO:tasks.workunit.client.0.smithi136.stdout:op 354 completed, throughput=4.97MB/sec 2024-09-13T00:02:33.507 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-xljblRhZ9p85wrU off=7128611 len=133786 2024-09-13T00:02:33.511 INFO:tasks.workunit.client.0.smithi136.stdout:op 355 completed, throughput=4.98MB/sec 2024-09-13T00:02:33.511 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-XBVcVJA8mkzvqrl off=4668148 len=1821411 2024-09-13T00:02:33.515 INFO:tasks.workunit.client.0.smithi136.stdout:op 356 completed, throughput=5MB/sec 2024-09-13T00:02:34.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:33 smithi136 bash[75164]: audit 2024-09-13T00:02:32.035746+0000 mon.b (mon.1) 12 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T00:02:34.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:33 smithi136 bash[75164]: audit 2024-09-13T00:02:32.920684+0000 mon.b (mon.1) 13 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T00:02:34.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:33 smithi136 bash[75164]: audit 2024-09-13T00:02:32.933376+0000 mon.c (mon.2) 93 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T00:02:34.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:33 smithi136 bash[75164]: audit 2024-09-13T00:02:32.935955+0000 mon.a (mon.0) 2114 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:02:34.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:33 smithi136 bash[75164]: audit 2024-09-13T00:02:32.936555+0000 mon.a (mon.0) 2115 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:02:34.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:33 smithi136 bash[75164]: audit 2024-09-13T00:02:32.936766+0000 mon.a (mon.0) 2116 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:02:34.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:33 smithi136 bash[75164]: audit 2024-09-13T00:02:33.857440+0000 mon.c (mon.2) 94 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T00:02:34.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:33 smithi136 bash[76371]: audit 2024-09-13T00:02:32.035746+0000 mon.b (mon.1) 12 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T00:02:34.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:33 smithi136 bash[76371]: audit 2024-09-13T00:02:32.920684+0000 mon.b (mon.1) 13 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T00:02:34.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:33 smithi136 bash[76371]: audit 2024-09-13T00:02:32.933376+0000 mon.c (mon.2) 93 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T00:02:34.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:33 smithi136 bash[76371]: audit 2024-09-13T00:02:32.935955+0000 mon.a (mon.0) 2114 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:02:34.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:33 smithi136 bash[76371]: audit 2024-09-13T00:02:32.936555+0000 mon.a (mon.0) 2115 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:02:34.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:33 smithi136 bash[76371]: audit 2024-09-13T00:02:32.936766+0000 mon.a (mon.0) 2116 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:02:34.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:33 smithi136 bash[76371]: audit 2024-09-13T00:02:33.857440+0000 mon.c (mon.2) 94 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T00:02:34.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:33 smithi181 bash[67587]: audit 2024-09-13T00:02:32.035746+0000 mon.b (mon.1) 12 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T00:02:34.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:33 smithi181 bash[67587]: audit 2024-09-13T00:02:32.920684+0000 mon.b (mon.1) 13 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T00:02:34.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:33 smithi181 bash[67587]: audit 2024-09-13T00:02:32.933376+0000 mon.c (mon.2) 93 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T00:02:34.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:33 smithi181 bash[67587]: audit 2024-09-13T00:02:32.935955+0000 mon.a (mon.0) 2114 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:02:34.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:33 smithi181 bash[67587]: audit 2024-09-13T00:02:32.936555+0000 mon.a (mon.0) 2115 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:02:34.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:33 smithi181 bash[67587]: audit 2024-09-13T00:02:32.936766+0000 mon.a (mon.0) 2116 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:02:34.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:33 smithi181 bash[67587]: audit 2024-09-13T00:02:33.857440+0000 mon.c (mon.2) 94 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T00:02:34.515 INFO:tasks.workunit.client.0.smithi136.stdout: 69: throughput=4.93MB/sec pending data=0 2024-09-13T00:02:34.515 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-qwam0757UqiJXDz off=1422395 len=682197 2024-09-13T00:02:34.522 INFO:tasks.workunit.client.0.smithi136.stdout:op 357 completed, throughput=4.94MB/sec 2024-09-13T00:02:34.522 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-1BQ7nZTUoMYe6JV off=5899980 len=860460 2024-09-13T00:02:34.524 INFO:tasks.workunit.client.0.smithi136.stdout:op 358 completed, throughput=4.95MB/sec 2024-09-13T00:02:34.525 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-eP1QoJSjLoJWZ6n off=0 len=1297890 2024-09-13T00:02:34.534 INFO:tasks.workunit.client.0.smithi136.stdout:op 359 completed, throughput=4.97MB/sec 2024-09-13T00:02:34.534 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-SzbaixESONCRByI off=10089135 len=220688 2024-09-13T00:02:34.535 INFO:tasks.workunit.client.0.smithi136.stdout:op 360 completed, throughput=4.97MB/sec 2024-09-13T00:02:34.535 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-7_Mz7Ifx3V8v0-e off=2579368 len=1907195 2024-09-13T00:02:34.539 INFO:tasks.workunit.client.0.smithi136.stdout:op 361 completed, throughput=5MB/sec 2024-09-13T00:02:34.539 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-KRQkFV9ddiy2HQ_ off=176619 len=1403916 2024-09-13T00:02:34.550 INFO:tasks.workunit.client.0.smithi136.stdout:op 362 completed, throughput=5.01MB/sec 2024-09-13T00:02:35.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:34 smithi136 bash[75164]: cluster 2024-09-13T00:02:33.183866+0000 mgr.y (mgr.24661) 763 : cluster [DBG] pgmap v1446: 105 pgs: 105 active+clean; 126 GiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.0 MiB/s wr, 10 op/s 2024-09-13T00:02:35.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:34 smithi136 bash[76371]: cluster 2024-09-13T00:02:33.183866+0000 mgr.y (mgr.24661) 763 : cluster [DBG] pgmap v1446: 105 pgs: 105 active+clean; 126 GiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.0 MiB/s wr, 10 op/s 2024-09-13T00:02:35.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:34 smithi181 bash[67587]: cluster 2024-09-13T00:02:33.183866+0000 mgr.y (mgr.24661) 763 : cluster [DBG] pgmap v1446: 105 pgs: 105 active+clean; 126 GiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.0 MiB/s wr, 10 op/s 2024-09-13T00:02:35.551 INFO:tasks.workunit.client.0.smithi136.stdout: 70: throughput=4.94MB/sec pending data=0 2024-09-13T00:02:35.551 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-rqBq44tohi8KS7_ off=3843280 len=305410 2024-09-13T00:02:35.552 INFO:tasks.workunit.client.0.smithi136.stdout:op 363 completed, throughput=4.95MB/sec 2024-09-13T00:02:35.552 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-R4synxWDJQ4RD7W off=6643653 len=1693087 2024-09-13T00:02:35.556 INFO:tasks.workunit.client.0.smithi136.stdout:op 364 completed, throughput=4.97MB/sec 2024-09-13T00:02:35.557 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-nCQNaCnrslBaLpl off=13451856 len=1691001 2024-09-13T00:02:35.568 INFO:tasks.workunit.client.0.smithi136.stdout:op 365 completed, throughput=4.99MB/sec 2024-09-13T00:02:35.568 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-2ZCCXZi8bc7Uu1X off=1700605 len=210410 2024-09-13T00:02:35.574 INFO:tasks.workunit.client.0.smithi136.stdout:op 366 completed, throughput=4.99MB/sec 2024-09-13T00:02:35.575 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Opr3lUIdbGMabW- off=1941302 len=670709 2024-09-13T00:02:35.577 INFO:tasks.workunit.client.0.smithi136.stdout:op 367 completed, throughput=5MB/sec 2024-09-13T00:02:36.577 INFO:tasks.workunit.client.0.smithi136.stdout: 71: throughput=4.93MB/sec pending data=0 2024-09-13T00:02:36.577 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-0pFftVIysRw_ZDN off=12939217 len=637473 2024-09-13T00:02:36.579 INFO:tasks.workunit.client.0.smithi136.stdout:op 368 completed, throughput=4.94MB/sec 2024-09-13T00:02:36.580 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-U5moPMlBiOj6ZRu off=8377235 len=424077 2024-09-13T00:02:36.582 INFO:tasks.workunit.client.0.smithi136.stdout:op 369 completed, throughput=4.95MB/sec 2024-09-13T00:02:36.582 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2pajBL_aULc8Mfm off=16991315 len=1468388 2024-09-13T00:02:36.587 INFO:tasks.workunit.client.0.smithi136.stdout:op 370 completed, throughput=4.97MB/sec 2024-09-13T00:02:36.587 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-MhANPQRgPDrhU8X off=5904292 len=234576 2024-09-13T00:02:36.588 INFO:tasks.workunit.client.0.smithi136.stdout:op 371 completed, throughput=4.97MB/sec 2024-09-13T00:02:36.588 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-1_YagsXMne2HQMj off=2824158 len=445756 2024-09-13T00:02:36.595 INFO:tasks.workunit.client.0.smithi136.stdout:op 372 completed, throughput=4.97MB/sec 2024-09-13T00:02:36.595 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-CTWlVSETM9mwCGL off=12058213 len=428885 2024-09-13T00:02:36.597 INFO:tasks.workunit.client.0.smithi136.stdout:op 373 completed, throughput=4.98MB/sec 2024-09-13T00:02:36.597 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-VqB8BJTuiKntwP4 off=86423 len=627413 2024-09-13T00:02:36.599 INFO:tasks.workunit.client.0.smithi136.stdout:op 374 completed, throughput=4.99MB/sec 2024-09-13T00:02:36.599 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-tncM78rrZSIYDTC off=10583916 len=1728032 2024-09-13T00:02:36.611 INFO:tasks.workunit.client.0.smithi136.stdout:op 375 completed, throughput=5.01MB/sec 2024-09-13T00:02:37.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:36 smithi136 bash[75164]: cluster 2024-09-13T00:02:35.184684+0000 mgr.y (mgr.24661) 764 : cluster [DBG] pgmap v1447: 105 pgs: 105 active+clean; 126 GiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 3.0 MiB/s rd, 1.8 MiB/s wr, 10 op/s 2024-09-13T00:02:37.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:36 smithi136 bash[76371]: cluster 2024-09-13T00:02:35.184684+0000 mgr.y (mgr.24661) 764 : cluster [DBG] pgmap v1447: 105 pgs: 105 active+clean; 126 GiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 3.0 MiB/s rd, 1.8 MiB/s wr, 10 op/s 2024-09-13T00:02:37.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:36 smithi181 bash[67587]: cluster 2024-09-13T00:02:35.184684+0000 mgr.y (mgr.24661) 764 : cluster [DBG] pgmap v1447: 105 pgs: 105 active+clean; 126 GiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 3.0 MiB/s rd, 1.8 MiB/s wr, 10 op/s 2024-09-13T00:02:37.612 INFO:tasks.workunit.client.0.smithi136.stdout: 72: throughput=4.94MB/sec pending data=0 2024-09-13T00:02:37.612 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-RsrEzmH6aPzRlJQ off=734766 len=1950070 2024-09-13T00:02:37.624 INFO:tasks.workunit.client.0.smithi136.stdout:op 376 completed, throughput=4.97MB/sec 2024-09-13T00:02:37.625 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2oekCu2B-TJuB1c off=1408491 len=1690664 2024-09-13T00:02:37.628 INFO:tasks.workunit.client.0.smithi136.stdout:op 377 completed, throughput=4.99MB/sec 2024-09-13T00:02:37.628 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-sfAjtnSMkz-IC7i off=12660581 len=1561246 2024-09-13T00:02:37.638 INFO:tasks.workunit.client.0.smithi136.stdout:op 378 completed, throughput=5.01MB/sec 2024-09-13T00:02:38.638 INFO:tasks.workunit.client.0.smithi136.stdout: 73: throughput=4.94MB/sec pending data=0 2024-09-13T00:02:38.638 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-AN0kAwOdRGL4CTi off=6512993 len=1970235 2024-09-13T00:02:38.643 INFO:tasks.workunit.client.0.smithi136.stdout:op 379 completed, throughput=4.97MB/sec 2024-09-13T00:02:38.643 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Drhd9SZUqdLwvLC off=12823026 len=21058 2024-09-13T00:02:38.647 INFO:tasks.workunit.client.0.smithi136.stdout:op 380 completed, throughput=4.97MB/sec 2024-09-13T00:02:38.647 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-2BlZ8E4Bpo2ng6D off=722826 len=803636 2024-09-13T00:02:38.655 INFO:tasks.workunit.client.0.smithi136.stdout:op 381 completed, throughput=4.98MB/sec 2024-09-13T00:02:38.656 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-eC-zcktzcXQlds2 off=10440138 len=387022 2024-09-13T00:02:38.657 INFO:tasks.workunit.client.0.smithi136.stdout:op 382 completed, throughput=4.98MB/sec 2024-09-13T00:02:38.658 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-YvzTJXMHXIvzs74 off=5887283 len=1311601 2024-09-13T00:02:38.668 INFO:tasks.workunit.client.0.smithi136.stdout:op 383 completed, throughput=5MB/sec 2024-09-13T00:02:38.668 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-womwHtNx0aFOxtV off=8362760 len=1235938 2024-09-13T00:02:38.677 INFO:tasks.workunit.client.0.smithi136.stdout:op 384 completed, throughput=5.01MB/sec 2024-09-13T00:02:39.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:38 smithi136 bash[75164]: cluster 2024-09-13T00:02:37.185725+0000 mgr.y (mgr.24661) 765 : cluster [DBG] pgmap v1448: 105 pgs: 105 active+clean; 126 GiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.5 MiB/s wr, 13 op/s 2024-09-13T00:02:39.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:38 smithi136 bash[75164]: audit 2024-09-13T00:02:38.773364+0000 mon.a (mon.0) 2117 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:02:39.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:38 smithi136 bash[76371]: cluster 2024-09-13T00:02:37.185725+0000 mgr.y (mgr.24661) 765 : cluster [DBG] pgmap v1448: 105 pgs: 105 active+clean; 126 GiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.5 MiB/s wr, 13 op/s 2024-09-13T00:02:39.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:38 smithi136 bash[76371]: audit 2024-09-13T00:02:38.773364+0000 mon.a (mon.0) 2117 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:02:39.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:38 smithi181 bash[67587]: cluster 2024-09-13T00:02:37.185725+0000 mgr.y (mgr.24661) 765 : cluster [DBG] pgmap v1448: 105 pgs: 105 active+clean; 126 GiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.5 MiB/s wr, 13 op/s 2024-09-13T00:02:39.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:38 smithi181 bash[67587]: audit 2024-09-13T00:02:38.773364+0000 mon.a (mon.0) 2117 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:02:39.677 INFO:tasks.workunit.client.0.smithi136.stdout: 74: throughput=4.95MB/sec pending data=0 2024-09-13T00:02:39.678 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-K-Lmb2t6lMh_oMW off=588371 len=1910783 2024-09-13T00:02:39.684 INFO:tasks.workunit.client.0.smithi136.stdout:op 385 completed, throughput=4.97MB/sec 2024-09-13T00:02:39.684 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-qIwVqoKY5jmRlLq off=4139213 len=262856 2024-09-13T00:02:39.690 INFO:tasks.workunit.client.0.smithi136.stdout:op 386 completed, throughput=4.97MB/sec 2024-09-13T00:02:39.690 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-IIV1KRepJXQXXdM off=12503404 len=896759 2024-09-13T00:02:39.692 INFO:tasks.workunit.client.0.smithi136.stdout:op 387 completed, throughput=4.98MB/sec 2024-09-13T00:02:39.692 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-KarAhlfijaCiZYt off=13328390 len=1507438 2024-09-13T00:02:39.705 INFO:tasks.workunit.client.0.smithi136.stdout:op 388 completed, throughput=5MB/sec 2024-09-13T00:02:39.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:02:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:02:39] "GET /metrics HTTP/1.1" 200 34122 "" "Prometheus/2.43.0" 2024-09-13T00:02:40.706 INFO:tasks.workunit.client.0.smithi136.stdout: 75: throughput=4.94MB/sec pending data=0 2024-09-13T00:02:40.706 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-akBhBk15d_c0HGz off=1788352 len=1361245 2024-09-13T00:02:40.717 INFO:tasks.workunit.client.0.smithi136.stdout:op 389 completed, throughput=4.95MB/sec 2024-09-13T00:02:40.717 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-mVTC-1S7kbgWk8v off=15988211 len=208372 2024-09-13T00:02:40.722 INFO:tasks.workunit.client.0.smithi136.stdout:op 390 completed, throughput=4.95MB/sec 2024-09-13T00:02:40.722 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-5WXh9uabVsBpKFu off=9516967 len=275593 2024-09-13T00:02:40.727 INFO:tasks.workunit.client.0.smithi136.stdout:op 391 completed, throughput=4.96MB/sec 2024-09-13T00:02:40.727 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-T4oViNLSuTtOaSs off=21228440 len=1559740 2024-09-13T00:02:40.733 INFO:tasks.workunit.client.0.smithi136.stdout:op 392 completed, throughput=4.98MB/sec 2024-09-13T00:02:40.733 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-dOz8Tw93YBpNJhK off=10210661 len=410117 2024-09-13T00:02:40.738 INFO:tasks.workunit.client.0.smithi136.stdout:op 393 completed, throughput=4.98MB/sec 2024-09-13T00:02:40.739 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-CzD30NY7MmO2GRx off=17226404 len=2060602 2024-09-13T00:02:40.743 INFO:tasks.workunit.client.0.smithi136.stdout:op 394 completed, throughput=5.01MB/sec 2024-09-13T00:02:41.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:40 smithi136 bash[75164]: cluster 2024-09-13T00:02:39.186590+0000 mgr.y (mgr.24661) 766 : cluster [DBG] pgmap v1449: 105 pgs: 105 active+clean; 126 GiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.4 MiB/s wr, 9 op/s 2024-09-13T00:02:41.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:40 smithi136 bash[76371]: cluster 2024-09-13T00:02:39.186590+0000 mgr.y (mgr.24661) 766 : cluster [DBG] pgmap v1449: 105 pgs: 105 active+clean; 126 GiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.4 MiB/s wr, 9 op/s 2024-09-13T00:02:41.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:40 smithi181 bash[67587]: cluster 2024-09-13T00:02:39.186590+0000 mgr.y (mgr.24661) 766 : cluster [DBG] pgmap v1449: 105 pgs: 105 active+clean; 126 GiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.4 MiB/s wr, 9 op/s 2024-09-13T00:02:41.743 INFO:tasks.workunit.client.0.smithi136.stdout: 76: throughput=4.94MB/sec pending data=0 2024-09-13T00:02:41.744 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-XvRdSRZFFnl4JG1 off=12672845 len=1929309 2024-09-13T00:02:41.753 INFO:tasks.workunit.client.0.smithi136.stdout:op 395 completed, throughput=4.96MB/sec 2024-09-13T00:02:41.753 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-mkSo8GcVM68YXCQ off=5794626 len=791527 2024-09-13T00:02:41.757 INFO:tasks.workunit.client.0.smithi136.stdout:op 396 completed, throughput=4.97MB/sec 2024-09-13T00:02:41.757 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-bLNcjfnNGZyKCjb off=2643331 len=1888346 2024-09-13T00:02:41.768 INFO:tasks.workunit.client.0.smithi136.stdout:op 397 completed, throughput=5MB/sec 2024-09-13T00:02:41.768 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-LW4q0y_4BX8iUec off=1535682 len=896670 2024-09-13T00:02:41.773 INFO:tasks.workunit.client.0.smithi136.stdout:op 398 completed, throughput=5.01MB/sec 2024-09-13T00:02:42.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:41 smithi136 bash[75164]: audit 2024-09-13T00:02:41.352738+0000 mon.a (mon.0) 2118 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:02:42.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:41 smithi136 bash[75164]: audit 2024-09-13T00:02:41.901411+0000 mon.a (mon.0) 2119 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:02:42.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:41 smithi136 bash[75164]: audit 2024-09-13T00:02:41.903087+0000 mon.a (mon.0) 2120 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:02:42.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:41 smithi136 bash[75164]: audit 2024-09-13T00:02:41.912954+0000 mon.a (mon.0) 2121 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:02:42.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:41 smithi136 bash[76371]: audit 2024-09-13T00:02:41.352738+0000 mon.a (mon.0) 2118 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:02:42.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:41 smithi136 bash[76371]: audit 2024-09-13T00:02:41.901411+0000 mon.a (mon.0) 2119 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:02:42.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:41 smithi136 bash[76371]: audit 2024-09-13T00:02:41.903087+0000 mon.a (mon.0) 2120 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:02:42.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:41 smithi136 bash[76371]: audit 2024-09-13T00:02:41.912954+0000 mon.a (mon.0) 2121 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:02:42.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:41 smithi181 bash[67587]: audit 2024-09-13T00:02:41.352738+0000 mon.a (mon.0) 2118 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:02:42.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:41 smithi181 bash[67587]: audit 2024-09-13T00:02:41.901411+0000 mon.a (mon.0) 2119 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:02:42.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:41 smithi181 bash[67587]: audit 2024-09-13T00:02:41.903087+0000 mon.a (mon.0) 2120 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:02:42.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:41 smithi181 bash[67587]: audit 2024-09-13T00:02:41.912954+0000 mon.a (mon.0) 2121 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:02:42.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:02:42 smithi181 bash[95505]: ts=2024-09-13T00:02:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:02:42.770 INFO:tasks.workunit.client.0.smithi136.stdout: 77: throughput=4.94MB/sec pending data=0 2024-09-13T00:02:42.770 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-eTUDQdH7gnbp0RB off=11055535 len=1025164 2024-09-13T00:02:42.774 INFO:tasks.workunit.client.0.smithi136.stdout:op 399 completed, throughput=4.96MB/sec 2024-09-13T00:02:42.775 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-8zivxD1yY7Mnmi3 off=1715583 len=1287831 2024-09-13T00:02:42.783 INFO:tasks.workunit.client.0.smithi136.stdout:op 400 completed, throughput=4.97MB/sec 2024-09-13T00:02:42.784 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ahzzhxPhOJQkQHk off=3985061 len=205400 2024-09-13T00:02:42.789 INFO:tasks.workunit.client.0.smithi136.stdout:op 401 completed, throughput=4.97MB/sec 2024-09-13T00:02:42.789 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-X2bhwOV7L8CQVb6 off=1466950 len=940999 2024-09-13T00:02:42.797 INFO:tasks.workunit.client.0.smithi136.stdout:op 402 completed, throughput=4.98MB/sec 2024-09-13T00:02:42.797 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-kqKIOatVPIetEG9 off=10689366 len=1158094 2024-09-13T00:02:42.801 INFO:tasks.workunit.client.0.smithi136.stdout:op 403 completed, throughput=5MB/sec 2024-09-13T00:02:42.801 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-h7xpIfN57Vk-UeV off=577131 len=520744 2024-09-13T00:02:42.803 INFO:tasks.workunit.client.0.smithi136.stdout:op 404 completed, throughput=5MB/sec 2024-09-13T00:02:43.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:42 smithi136 bash[75164]: cluster 2024-09-13T00:02:41.187529+0000 mgr.y (mgr.24661) 767 : cluster [DBG] pgmap v1450: 105 pgs: 105 active+clean; 126 GiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.4 MiB/s wr, 8 op/s 2024-09-13T00:02:43.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:42 smithi136 bash[76371]: cluster 2024-09-13T00:02:41.187529+0000 mgr.y (mgr.24661) 767 : cluster [DBG] pgmap v1450: 105 pgs: 105 active+clean; 126 GiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.4 MiB/s wr, 8 op/s 2024-09-13T00:02:43.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:42 smithi181 bash[67587]: cluster 2024-09-13T00:02:41.187529+0000 mgr.y (mgr.24661) 767 : cluster [DBG] pgmap v1450: 105 pgs: 105 active+clean; 126 GiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.4 MiB/s wr, 8 op/s 2024-09-13T00:02:43.804 INFO:tasks.workunit.client.0.smithi136.stdout: 78: throughput=4.94MB/sec pending data=0 2024-09-13T00:02:43.804 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-E3Vtkz7YVSxNU2y off=2403340 len=171398 2024-09-13T00:02:43.809 INFO:tasks.workunit.client.0.smithi136.stdout:op 405 completed, throughput=4.94MB/sec 2024-09-13T00:02:43.809 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-r83nCDyE9V3AZnD off=6064528 len=1333615 2024-09-13T00:02:43.812 INFO:tasks.workunit.client.0.smithi136.stdout:op 406 completed, throughput=4.96MB/sec 2024-09-13T00:02:43.812 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-1ExgHTKni3EFTKq off=6374559 len=1718429 2024-09-13T00:02:43.818 INFO:tasks.workunit.client.0.smithi136.stdout:op 407 completed, throughput=4.98MB/sec 2024-09-13T00:02:43.819 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Chls-1BKs1JTNd9 off=2029315 len=1590245 2024-09-13T00:02:43.830 INFO:tasks.workunit.client.0.smithi136.stdout:op 408 completed, throughput=5MB/sec 2024-09-13T00:02:43.830 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-BneYOUyRI9O9uQ6 off=3288595 len=1086425 2024-09-13T00:02:43.842 INFO:tasks.workunit.client.0.smithi136.stdout:op 409 completed, throughput=5.01MB/sec 2024-09-13T00:02:44.845 INFO:tasks.workunit.client.0.smithi136.stdout: 79: throughput=4.95MB/sec pending data=0 2024-09-13T00:02:44.845 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-j0hkqGFyDuL2GwM off=4881581 len=1487812 2024-09-13T00:02:44.853 INFO:tasks.workunit.client.0.smithi136.stdout:op 410 completed, throughput=4.96MB/sec 2024-09-13T00:02:44.853 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-yt2mDkjhSbnYw7G off=2282851 len=779582 2024-09-13T00:02:44.860 INFO:tasks.workunit.client.0.smithi136.stdout:op 411 completed, throughput=4.97MB/sec 2024-09-13T00:02:44.861 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-NB2b0wJdRnthspn off=4581998 len=257125 2024-09-13T00:02:44.865 INFO:tasks.workunit.client.0.smithi136.stdout:op 412 completed, throughput=4.98MB/sec 2024-09-13T00:02:44.865 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-0XU9s6N0kLRvrjA off=14766933 len=142727 2024-09-13T00:02:44.866 INFO:tasks.workunit.client.0.smithi136.stdout:op 413 completed, throughput=4.98MB/sec 2024-09-13T00:02:44.866 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-DDqnUdJMzfAcXUg off=9242475 len=156956 2024-09-13T00:02:44.868 INFO:tasks.workunit.client.0.smithi136.stdout:op 414 completed, throughput=4.98MB/sec 2024-09-13T00:02:44.868 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-mSL677_zGx20HAC off=1066853 len=50427 2024-09-13T00:02:44.869 INFO:tasks.workunit.client.0.smithi136.stdout:op 415 completed, throughput=4.98MB/sec 2024-09-13T00:02:44.869 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-LJ3v0hvHylsV6wY off=811301 len=550420 2024-09-13T00:02:44.871 INFO:tasks.workunit.client.0.smithi136.stdout:op 416 completed, throughput=4.99MB/sec 2024-09-13T00:02:44.871 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-GtAUT8FAHltSJog off=3711521 len=1083899 2024-09-13T00:02:44.879 INFO:tasks.workunit.client.0.smithi136.stdout:op 417 completed, throughput=5MB/sec 2024-09-13T00:02:44.879 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-i3DTIaUFrCKaZXZ off=10078618 len=584431 2024-09-13T00:02:44.887 INFO:tasks.workunit.client.0.smithi136.stdout:op 418 completed, throughput=5MB/sec 2024-09-13T00:02:45.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:44 smithi136 bash[75164]: cluster 2024-09-13T00:02:43.188588+0000 mgr.y (mgr.24661) 768 : cluster [DBG] pgmap v1451: 105 pgs: 105 active+clean; 126 GiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.6 MiB/s wr, 7 op/s 2024-09-13T00:02:45.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:44 smithi136 bash[76371]: cluster 2024-09-13T00:02:43.188588+0000 mgr.y (mgr.24661) 768 : cluster [DBG] pgmap v1451: 105 pgs: 105 active+clean; 126 GiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.6 MiB/s wr, 7 op/s 2024-09-13T00:02:45.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:44 smithi181 bash[67587]: cluster 2024-09-13T00:02:43.188588+0000 mgr.y (mgr.24661) 768 : cluster [DBG] pgmap v1451: 105 pgs: 105 active+clean; 126 GiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.6 MiB/s wr, 7 op/s 2024-09-13T00:02:45.887 INFO:tasks.workunit.client.0.smithi136.stdout: 80: throughput=4.94MB/sec pending data=0 2024-09-13T00:02:45.887 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-C7mbgyp2qhFkWAc off=18473348 len=386064 2024-09-13T00:02:45.889 INFO:tasks.workunit.client.0.smithi136.stdout:op 419 completed, throughput=4.95MB/sec 2024-09-13T00:02:45.889 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Sw04MR8PtecHuOS off=34767 len=1713359 2024-09-13T00:02:45.900 INFO:tasks.workunit.client.0.smithi136.stdout:op 420 completed, throughput=4.97MB/sec 2024-09-13T00:02:45.900 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Sy8p5XwsMS-kT5D off=1604237 len=2093173 2024-09-13T00:02:45.907 INFO:tasks.workunit.client.0.smithi136.stdout:op 421 completed, throughput=4.99MB/sec 2024-09-13T00:02:45.908 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-XXSwhJbLMjl3Lll off=1841603 len=1747545 2024-09-13T00:02:45.912 INFO:tasks.workunit.client.0.smithi136.stdout:op 422 completed, throughput=5.01MB/sec 2024-09-13T00:02:46.913 INFO:tasks.workunit.client.0.smithi136.stdout: 81: throughput=4.95MB/sec pending data=0 2024-09-13T00:02:46.913 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-53AM8mhQ9uBJ2Ku off=2172297 len=1541064 2024-09-13T00:02:46.916 INFO:tasks.workunit.client.0.smithi136.stdout:op 423 completed, throughput=4.97MB/sec 2024-09-13T00:02:46.916 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-p9Fg0GBNSMai2wr off=4751067 len=934911 2024-09-13T00:02:46.919 INFO:tasks.workunit.client.0.smithi136.stdout:op 424 completed, throughput=4.98MB/sec 2024-09-13T00:02:46.919 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-6AwCaCZbrutzXxs off=1893616 len=922185 2024-09-13T00:02:46.921 INFO:tasks.workunit.client.0.smithi136.stdout:op 425 completed, throughput=4.99MB/sec 2024-09-13T00:02:46.921 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-laEGWJlloZcxnps off=16880506 len=596687 2024-09-13T00:02:46.923 INFO:tasks.workunit.client.0.smithi136.stdout:op 426 completed, throughput=5MB/sec 2024-09-13T00:02:46.923 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-fa70JbYfWzxF8NJ off=13006414 len=1611523 2024-09-13T00:02:46.927 INFO:tasks.workunit.client.0.smithi136.stdout:op 427 completed, throughput=5.01MB/sec 2024-09-13T00:02:47.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:46 smithi136 bash[75164]: cluster 2024-09-13T00:02:45.189392+0000 mgr.y (mgr.24661) 769 : cluster [DBG] pgmap v1452: 105 pgs: 105 active+clean; 126 GiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.4 MiB/s wr, 7 op/s 2024-09-13T00:02:47.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:46 smithi136 bash[76371]: cluster 2024-09-13T00:02:45.189392+0000 mgr.y (mgr.24661) 769 : cluster [DBG] pgmap v1452: 105 pgs: 105 active+clean; 126 GiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.4 MiB/s wr, 7 op/s 2024-09-13T00:02:47.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:46 smithi181 bash[67587]: cluster 2024-09-13T00:02:45.189392+0000 mgr.y (mgr.24661) 769 : cluster [DBG] pgmap v1452: 105 pgs: 105 active+clean; 126 GiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.4 MiB/s wr, 7 op/s 2024-09-13T00:02:47.927 INFO:tasks.workunit.client.0.smithi136.stdout: 82: throughput=4.95MB/sec pending data=0 2024-09-13T00:02:47.927 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj--1NnifipLOMq8FR off=10982320 len=1699846 2024-09-13T00:02:47.933 INFO:tasks.workunit.client.0.smithi136.stdout:op 428 completed, throughput=4.97MB/sec 2024-09-13T00:02:47.933 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-NOTqtPNaWNeUvim off=3702875 len=430708 2024-09-13T00:02:47.935 INFO:tasks.workunit.client.0.smithi136.stdout:op 429 completed, throughput=4.98MB/sec 2024-09-13T00:02:47.935 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-3x0o8X9lS7rcRzT off=4867968 len=814236 2024-09-13T00:02:47.943 INFO:tasks.workunit.client.0.smithi136.stdout:op 430 completed, throughput=4.99MB/sec 2024-09-13T00:02:47.943 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-J49tUN5a7GdtYmz off=195811 len=745956 2024-09-13T00:02:47.950 INFO:tasks.workunit.client.0.smithi136.stdout:op 431 completed, throughput=5MB/sec 2024-09-13T00:02:47.951 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-255opUWhxJWBxXJ off=380734 len=1701254 2024-09-13T00:02:47.955 INFO:tasks.workunit.client.0.smithi136.stdout:op 432 completed, throughput=5.01MB/sec 2024-09-13T00:02:48.955 INFO:tasks.workunit.client.0.smithi136.stdout: 83: throughput=4.96MB/sec pending data=0 2024-09-13T00:02:48.956 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-qk0XHCQuCFGWfX6 off=1525889 len=556801 2024-09-13T00:02:48.963 INFO:tasks.workunit.client.0.smithi136.stdout:op 433 completed, throughput=4.96MB/sec 2024-09-13T00:02:48.963 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-MzSkisYdriMaQzJ off=6243331 len=1354972 2024-09-13T00:02:48.972 INFO:tasks.workunit.client.0.smithi136.stdout:op 434 completed, throughput=4.98MB/sec 2024-09-13T00:02:48.972 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-BaA8-aDTuNa-7gd off=19163177 len=1634241 2024-09-13T00:02:48.987 INFO:tasks.workunit.client.0.smithi136.stdout:op 435 completed, throughput=4.99MB/sec 2024-09-13T00:02:48.987 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-02_JPoICKAF87GQ off=780123 len=23123 2024-09-13T00:02:48.992 INFO:tasks.workunit.client.0.smithi136.stdout:op 436 completed, throughput=4.99MB/sec 2024-09-13T00:02:48.992 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-O-KPFSuW60ziwP8 off=3837762 len=1178489 2024-09-13T00:02:49.000 INFO:tasks.workunit.client.0.smithi136.stdout:op 437 completed, throughput=5.01MB/sec 2024-09-13T00:02:49.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:48 smithi136 bash[75164]: cluster 2024-09-13T00:02:47.190493+0000 mgr.y (mgr.24661) 770 : cluster [DBG] pgmap v1453: 105 pgs: 105 active+clean; 126 GiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 2.7 MiB/s rd, 3.1 MiB/s wr, 8 op/s 2024-09-13T00:02:49.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:48 smithi136 bash[76371]: cluster 2024-09-13T00:02:47.190493+0000 mgr.y (mgr.24661) 770 : cluster [DBG] pgmap v1453: 105 pgs: 105 active+clean; 126 GiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 2.7 MiB/s rd, 3.1 MiB/s wr, 8 op/s 2024-09-13T00:02:49.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:48 smithi181 bash[67587]: cluster 2024-09-13T00:02:47.190493+0000 mgr.y (mgr.24661) 770 : cluster [DBG] pgmap v1453: 105 pgs: 105 active+clean; 126 GiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 2.7 MiB/s rd, 3.1 MiB/s wr, 8 op/s 2024-09-13T00:02:49.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:02:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:02:49] "GET /metrics HTTP/1.1" 200 34122 "" "Prometheus/2.43.0" 2024-09-13T00:02:50.001 INFO:tasks.workunit.client.0.smithi136.stdout: 84: throughput=4.95MB/sec pending data=0 2024-09-13T00:02:50.001 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-AO-gaRCSiqgp1Pn off=9822193 len=987235 2024-09-13T00:02:50.011 INFO:tasks.workunit.client.0.smithi136.stdout:op 438 completed, throughput=4.96MB/sec 2024-09-13T00:02:50.011 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-6JQQC5bCrfDAfe- off=4144177 len=336205 2024-09-13T00:02:50.013 INFO:tasks.workunit.client.0.smithi136.stdout:op 439 completed, throughput=4.96MB/sec 2024-09-13T00:02:50.013 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-YGqPIVN_33H_0XR off=6139301 len=1056355 2024-09-13T00:02:50.023 INFO:tasks.workunit.client.0.smithi136.stdout:op 440 completed, throughput=4.97MB/sec 2024-09-13T00:02:50.023 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-c1_3EQgqElanFBL off=18052743 len=1840608 2024-09-13T00:02:50.027 INFO:tasks.workunit.client.0.smithi136.stdout:op 441 completed, throughput=4.99MB/sec 2024-09-13T00:02:50.027 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-osZ_PxfuOQs-rsQ off=1127506 len=379453 2024-09-13T00:02:50.033 INFO:tasks.workunit.client.0.smithi136.stdout:op 442 completed, throughput=5MB/sec 2024-09-13T00:02:50.034 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-DIqU4hBdqn2aa_N off=686289 len=121346 2024-09-13T00:02:50.034 INFO:tasks.workunit.client.0.smithi136.stdout:op 443 completed, throughput=5MB/sec 2024-09-13T00:02:50.035 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-sBbG2oIJQWS74JW off=1175316 len=1187197 2024-09-13T00:02:50.038 INFO:tasks.workunit.client.0.smithi136.stdout:op 444 completed, throughput=5.01MB/sec 2024-09-13T00:02:51.038 INFO:tasks.workunit.client.0.smithi136.stdout: 85: throughput=4.95MB/sec pending data=0 2024-09-13T00:02:51.038 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-EQAFrZQgIBOqJWM off=6672706 len=658559 2024-09-13T00:02:51.046 INFO:tasks.workunit.client.0.smithi136.stdout:op 445 completed, throughput=4.96MB/sec 2024-09-13T00:02:51.046 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-X0DEa9Occe3mjGJ off=4760900 len=1678330 2024-09-13T00:02:51.058 INFO:tasks.workunit.client.0.smithi136.stdout:op 446 completed, throughput=4.98MB/sec 2024-09-13T00:02:51.058 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-l1-pxYviwykYsBK off=5266006 len=737305 2024-09-13T00:02:51.065 INFO:tasks.workunit.client.0.smithi136.stdout:op 447 completed, throughput=4.99MB/sec 2024-09-13T00:02:51.065 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-jveSN2zTX1k07o2 off=5105616 len=1362856 2024-09-13T00:02:51.074 INFO:tasks.workunit.client.0.smithi136.stdout:op 448 completed, throughput=5MB/sec 2024-09-13T00:02:51.074 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj--iGeLr323OUEbCf off=6131087 len=1049634 2024-09-13T00:02:51.083 INFO:tasks.workunit.client.0.smithi136.stdout:op 449 completed, throughput=5.01MB/sec 2024-09-13T00:02:51.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:50 smithi136 bash[75164]: cluster 2024-09-13T00:02:49.191433+0000 mgr.y (mgr.24661) 771 : cluster [DBG] pgmap v1454: 105 pgs: 105 active+clean; 126 GiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:02:51.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:50 smithi136 bash[76371]: cluster 2024-09-13T00:02:49.191433+0000 mgr.y (mgr.24661) 771 : cluster [DBG] pgmap v1454: 105 pgs: 105 active+clean; 126 GiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:02:51.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:50 smithi181 bash[67587]: cluster 2024-09-13T00:02:49.191433+0000 mgr.y (mgr.24661) 771 : cluster [DBG] pgmap v1454: 105 pgs: 105 active+clean; 126 GiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:02:52.084 INFO:tasks.workunit.client.0.smithi136.stdout: 86: throughput=4.95MB/sec pending data=0 2024-09-13T00:02:52.084 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Hro9FUmsRKXIshL off=849791 len=298271 2024-09-13T00:02:52.089 INFO:tasks.workunit.client.0.smithi136.stdout:op 450 completed, throughput=4.96MB/sec 2024-09-13T00:02:52.089 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-jiXEet0Gd0cjDC9 off=722873 len=1251578 2024-09-13T00:02:52.092 INFO:tasks.workunit.client.0.smithi136.stdout:op 451 completed, throughput=4.97MB/sec 2024-09-13T00:02:52.093 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-9KfX1CiYMRqM1ru off=21126250 len=1655828 2024-09-13T00:02:52.098 INFO:tasks.workunit.client.0.smithi136.stdout:op 452 completed, throughput=4.99MB/sec 2024-09-13T00:02:52.098 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-SsL-mb69IX1vZye off=2048282 len=923794 2024-09-13T00:02:52.107 INFO:tasks.workunit.client.0.smithi136.stdout:op 453 completed, throughput=5MB/sec 2024-09-13T00:02:52.107 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-W3O72uofoKYM6Uv off=144746 len=1343519 2024-09-13T00:02:52.118 INFO:tasks.workunit.client.0.smithi136.stdout:op 454 completed, throughput=5.01MB/sec 2024-09-13T00:02:52.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:02:52 smithi181 bash[95505]: ts=2024-09-13T00:02:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:02:53.119 INFO:tasks.workunit.client.0.smithi136.stdout: 87: throughput=4.96MB/sec pending data=0 2024-09-13T00:02:53.120 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-s6r9hlh6fFjmt_a off=1999112 len=400136 2024-09-13T00:02:53.124 INFO:tasks.workunit.client.0.smithi136.stdout:op 455 completed, throughput=4.96MB/sec 2024-09-13T00:02:53.124 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-64L9UL8fg2FvDPe off=10214942 len=380417 2024-09-13T00:02:53.129 INFO:tasks.workunit.client.0.smithi136.stdout:op 456 completed, throughput=4.96MB/sec 2024-09-13T00:02:53.130 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-FvuoML4EoEh58Po off=4655225 len=2037238 2024-09-13T00:02:53.136 INFO:tasks.workunit.client.0.smithi136.stdout:op 457 completed, throughput=4.98MB/sec 2024-09-13T00:02:53.136 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-sK6VAF4YEjKxSSV off=10013814 len=1409205 2024-09-13T00:02:53.140 INFO:tasks.workunit.client.0.smithi136.stdout:op 458 completed, throughput=5MB/sec 2024-09-13T00:02:53.140 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-lgJzMSCUYjWpNy7 off=1088704 len=644140 2024-09-13T00:02:53.142 INFO:tasks.workunit.client.0.smithi136.stdout:op 459 completed, throughput=5.01MB/sec 2024-09-13T00:02:53.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:52 smithi136 bash[75164]: cluster 2024-09-13T00:02:51.192422+0000 mgr.y (mgr.24661) 772 : cluster [DBG] pgmap v1455: 105 pgs: 105 active+clean; 126 GiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:02:53.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:52 smithi136 bash[76371]: cluster 2024-09-13T00:02:51.192422+0000 mgr.y (mgr.24661) 772 : cluster [DBG] pgmap v1455: 105 pgs: 105 active+clean; 126 GiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:02:53.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:52 smithi181 bash[67587]: cluster 2024-09-13T00:02:51.192422+0000 mgr.y (mgr.24661) 772 : cluster [DBG] pgmap v1455: 105 pgs: 105 active+clean; 126 GiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:02:54.143 INFO:tasks.workunit.client.0.smithi136.stdout: 88: throughput=4.95MB/sec pending data=0 2024-09-13T00:02:54.143 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-r4mLXYtXJvoAOkY off=14582121 len=1546497 2024-09-13T00:02:54.157 INFO:tasks.workunit.client.0.smithi136.stdout:op 460 completed, throughput=4.97MB/sec 2024-09-13T00:02:54.157 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-3Up6DYjn8sMBcOD off=2149407 len=1930517 2024-09-13T00:02:54.169 INFO:tasks.workunit.client.0.smithi136.stdout:op 461 completed, throughput=4.99MB/sec 2024-09-13T00:02:54.169 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-zUbhgEYaV6LSfMJ off=12803112 len=2043968 2024-09-13T00:02:54.174 INFO:tasks.workunit.client.0.smithi136.stdout:op 462 completed, throughput=5.01MB/sec 2024-09-13T00:02:54.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:53 smithi136 bash[75164]: audit 2024-09-13T00:02:53.773575+0000 mon.a (mon.0) 2122 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:02:54.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:53 smithi136 bash[76371]: audit 2024-09-13T00:02:53.773575+0000 mon.a (mon.0) 2122 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:02:54.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:53 smithi181 bash[67587]: audit 2024-09-13T00:02:53.773575+0000 mon.a (mon.0) 2122 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:02:55.174 INFO:tasks.workunit.client.0.smithi136.stdout: 89: throughput=4.95MB/sec pending data=0 2024-09-13T00:02:55.174 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-crKtsMmdVN2cxKN off=3195707 len=363970 2024-09-13T00:02:55.177 INFO:tasks.workunit.client.0.smithi136.stdout:op 463 completed, throughput=4.96MB/sec 2024-09-13T00:02:55.177 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-xOmyz-U8JBZk002 off=8010878 len=1229840 2024-09-13T00:02:55.180 INFO:tasks.workunit.client.0.smithi136.stdout:op 464 completed, throughput=4.97MB/sec 2024-09-13T00:02:55.180 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-sJZNr4gkKcBKvlM off=2380950 len=796966 2024-09-13T00:02:55.183 INFO:tasks.workunit.client.0.smithi136.stdout:op 465 completed, throughput=4.98MB/sec 2024-09-13T00:02:55.183 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-W8NQ4_AOlO1rpiV off=3802387 len=328528 2024-09-13T00:02:55.189 INFO:tasks.workunit.client.0.smithi136.stdout:op 466 completed, throughput=4.98MB/sec 2024-09-13T00:02:55.189 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-sV3GUtpUABrBjue off=107788 len=1912099 2024-09-13T00:02:55.208 INFO:tasks.workunit.client.0.smithi136.stdout:op 467 completed, throughput=5MB/sec 2024-09-13T00:02:55.209 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-UPlHblrwTVo41KF off=16143045 len=1745043 2024-09-13T00:02:55.220 INFO:tasks.workunit.client.0.smithi136.stdout:op 468 completed, throughput=5.02MB/sec 2024-09-13T00:02:55.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:54 smithi136 bash[75164]: cluster 2024-09-13T00:02:53.193507+0000 mgr.y (mgr.24661) 773 : cluster [DBG] pgmap v1456: 105 pgs: 105 active+clean; 126 GiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:02:55.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:54 smithi136 bash[76371]: cluster 2024-09-13T00:02:53.193507+0000 mgr.y (mgr.24661) 773 : cluster [DBG] pgmap v1456: 105 pgs: 105 active+clean; 126 GiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:02:55.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:54 smithi181 bash[67587]: cluster 2024-09-13T00:02:53.193507+0000 mgr.y (mgr.24661) 773 : cluster [DBG] pgmap v1456: 105 pgs: 105 active+clean; 126 GiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:02:56.221 INFO:tasks.workunit.client.0.smithi136.stdout: 90: throughput=4.96MB/sec pending data=0 2024-09-13T00:02:56.221 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-3I12s_qs1pKBxfO off=4276545 len=1004535 2024-09-13T00:02:56.224 INFO:tasks.workunit.client.0.smithi136.stdout:op 469 completed, throughput=4.97MB/sec 2024-09-13T00:02:56.224 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-rx2sD7QgI0XNHrs off=3051387 len=1055935 2024-09-13T00:02:56.227 INFO:tasks.workunit.client.0.smithi136.stdout:op 470 completed, throughput=4.98MB/sec 2024-09-13T00:02:56.227 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-HJSJ4UVvcne9UvB off=839852 len=989025 2024-09-13T00:02:56.235 INFO:tasks.workunit.client.0.smithi136.stdout:op 471 completed, throughput=4.99MB/sec 2024-09-13T00:02:56.236 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-0OtStCDE0QjfevJ off=1447220 len=714554 2024-09-13T00:02:56.237 INFO:tasks.workunit.client.0.smithi136.stdout:op 472 completed, throughput=5MB/sec 2024-09-13T00:02:56.237 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-KUSEdjWWCOpctm9 off=6682159 len=190000 2024-09-13T00:02:56.238 INFO:tasks.workunit.client.0.smithi136.stdout:op 473 completed, throughput=5MB/sec 2024-09-13T00:02:57.239 INFO:tasks.workunit.client.0.smithi136.stdout: 91: throughput=4.95MB/sec pending data=0 2024-09-13T00:02:57.239 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Uxmf0BWSihy-ejf off=5455592 len=815124 2024-09-13T00:02:57.247 INFO:tasks.workunit.client.0.smithi136.stdout:op 474 completed, throughput=4.96MB/sec 2024-09-13T00:02:57.247 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-BMNiiOEn7-4pr6a off=11386452 len=1187227 2024-09-13T00:02:57.249 INFO:tasks.workunit.client.0.smithi136.stdout:op 475 completed, throughput=4.97MB/sec 2024-09-13T00:02:57.249 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-xKcGfAQWjQ3nfsr off=678394 len=991771 2024-09-13T00:02:57.252 INFO:tasks.workunit.client.0.smithi136.stdout:op 476 completed, throughput=4.98MB/sec 2024-09-13T00:02:57.252 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-bnr22mvPwBq5oZA off=5779882 len=346496 2024-09-13T00:02:57.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:56 smithi136 bash[75164]: cluster 2024-09-13T00:02:55.194432+0000 mgr.y (mgr.24661) 774 : cluster [DBG] pgmap v1457: 105 pgs: 105 active+clean; 126 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:02:57.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:56 smithi136 bash[76371]: cluster 2024-09-13T00:02:55.194432+0000 mgr.y (mgr.24661) 774 : cluster [DBG] pgmap v1457: 105 pgs: 105 active+clean; 126 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:02:57.257 INFO:tasks.workunit.client.0.smithi136.stdout:op 477 completed, throughput=4.98MB/sec 2024-09-13T00:02:57.257 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-tAAF3iRWMAemGf6 off=9515427 len=484100 2024-09-13T00:02:57.263 INFO:tasks.workunit.client.0.smithi136.stdout:op 478 completed, throughput=4.99MB/sec 2024-09-13T00:02:57.264 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-hrhq_7JyFY1dYHd off=1924876 len=771551 2024-09-13T00:02:57.271 INFO:tasks.workunit.client.0.smithi136.stdout:op 479 completed, throughput=4.99MB/sec 2024-09-13T00:02:57.271 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-4PkeFsoTA_8WgpH off=9643101 len=1827959 2024-09-13T00:02:57.275 INFO:tasks.workunit.client.0.smithi136.stdout:op 480 completed, throughput=5.01MB/sec 2024-09-13T00:02:57.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:56 smithi181 bash[67587]: cluster 2024-09-13T00:02:55.194432+0000 mgr.y (mgr.24661) 774 : cluster [DBG] pgmap v1457: 105 pgs: 105 active+clean; 126 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:02:58.275 INFO:tasks.workunit.client.0.smithi136.stdout: 92: throughput=4.96MB/sec pending data=0 2024-09-13T00:02:58.275 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-4ss57xsbqzYWdyy off=3522345 len=589527 2024-09-13T00:02:58.277 INFO:tasks.workunit.client.0.smithi136.stdout:op 481 completed, throughput=4.96MB/sec 2024-09-13T00:02:58.277 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Fid34LFAeff0vZY off=7275316 len=1306280 2024-09-13T00:02:58.281 INFO:tasks.workunit.client.0.smithi136.stdout:op 482 completed, throughput=4.98MB/sec 2024-09-13T00:02:58.281 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Kx4LUdSgJxdNytU off=3430326 len=471525 2024-09-13T00:02:58.287 INFO:tasks.workunit.client.0.smithi136.stdout:op 483 completed, throughput=4.98MB/sec 2024-09-13T00:02:58.287 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-9EBHsI-A88fgyKK off=13098836 len=828876 2024-09-13T00:02:58.294 INFO:tasks.workunit.client.0.smithi136.stdout:op 484 completed, throughput=4.99MB/sec 2024-09-13T00:02:58.294 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-CZ_E6uvUBKRVNWw off=14260807 len=1533009 2024-09-13T00:02:58.305 INFO:tasks.workunit.client.0.smithi136.stdout:op 485 completed, throughput=5MB/sec 2024-09-13T00:02:59.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:02:58 smithi136 bash[75164]: cluster 2024-09-13T00:02:57.195512+0000 mgr.y (mgr.24661) 775 : cluster [DBG] pgmap v1458: 105 pgs: 105 active+clean; 126 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:02:59.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:02:58 smithi136 bash[76371]: cluster 2024-09-13T00:02:57.195512+0000 mgr.y (mgr.24661) 775 : cluster [DBG] pgmap v1458: 105 pgs: 105 active+clean; 126 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:02:59.305 INFO:tasks.workunit.client.0.smithi136.stdout: 93: throughput=4.95MB/sec pending data=0 2024-09-13T00:02:59.305 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Vm749obMu6VzHvF off=24651831 len=879962 2024-09-13T00:02:59.314 INFO:tasks.workunit.client.0.smithi136.stdout:op 486 completed, throughput=4.96MB/sec 2024-09-13T00:02:59.314 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-NEliPv7JV8FvnDT off=2462252 len=1888063 2024-09-13T00:02:59.327 INFO:tasks.workunit.client.0.smithi136.stdout:op 487 completed, throughput=4.98MB/sec 2024-09-13T00:02:59.327 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-H0VyenghcM72052 off=20104960 len=1291974 2024-09-13T00:02:59.338 INFO:tasks.workunit.client.0.smithi136.stdout:op 488 completed, throughput=4.99MB/sec 2024-09-13T00:02:59.338 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-3V3U3O0ciXTvpzm off=3071580 len=987416 2024-09-13T00:02:59.341 INFO:tasks.workunit.client.0.smithi136.stdout:op 489 completed, throughput=5MB/sec 2024-09-13T00:02:59.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:02:58 smithi181 bash[67587]: cluster 2024-09-13T00:02:57.195512+0000 mgr.y (mgr.24661) 775 : cluster [DBG] pgmap v1458: 105 pgs: 105 active+clean; 126 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:02:59.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:02:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:02:59] "GET /metrics HTTP/1.1" 200 34123 "" "Prometheus/2.43.0" 2024-09-13T00:03:00.341 INFO:tasks.workunit.client.0.smithi136.stdout: 94: throughput=4.95MB/sec pending data=0 2024-09-13T00:03:00.342 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-jSGmAb_ja4iw9dk off=5843667 len=362426 2024-09-13T00:03:00.347 INFO:tasks.workunit.client.0.smithi136.stdout:op 490 completed, throughput=4.95MB/sec 2024-09-13T00:03:00.347 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-5XRjF0bQpVWn07Z off=954220 len=1742804 2024-09-13T00:03:00.358 INFO:tasks.workunit.client.0.smithi136.stdout:op 491 completed, throughput=4.97MB/sec 2024-09-13T00:03:00.358 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-VF7tt3FnjmYBabL off=825285 len=709147 2024-09-13T00:03:00.361 INFO:tasks.workunit.client.0.smithi136.stdout:op 492 completed, throughput=4.98MB/sec 2024-09-13T00:03:00.361 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-A_5LjSoWqTunyAs off=701133 len=1718980 2024-09-13T00:03:00.372 INFO:tasks.workunit.client.0.smithi136.stdout:op 493 completed, throughput=4.99MB/sec 2024-09-13T00:03:00.372 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_L-irlFXMFlFLm9 off=5326466 len=1540320 2024-09-13T00:03:00.381 INFO:tasks.workunit.client.0.smithi136.stdout:op 494 completed, throughput=5.01MB/sec 2024-09-13T00:03:01.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:03:00 smithi136 bash[75164]: cluster 2024-09-13T00:02:59.196420+0000 mgr.y (mgr.24661) 776 : cluster [DBG] pgmap v1459: 105 pgs: 105 active+clean; 126 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:03:01.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:03:00 smithi136 bash[76371]: cluster 2024-09-13T00:02:59.196420+0000 mgr.y (mgr.24661) 776 : cluster [DBG] pgmap v1459: 105 pgs: 105 active+clean; 126 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:03:01.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:03:00 smithi181 bash[67587]: cluster 2024-09-13T00:02:59.196420+0000 mgr.y (mgr.24661) 776 : cluster [DBG] pgmap v1459: 105 pgs: 105 active+clean; 126 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:03:01.381 INFO:tasks.workunit.client.0.smithi136.stdout: 95: throughput=4.96MB/sec pending data=0 2024-09-13T00:03:01.381 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-LiVgF0G0Vxa4Ajw off=290262 len=1013874 2024-09-13T00:03:01.384 INFO:tasks.workunit.client.0.smithi136.stdout:op 495 completed, throughput=4.97MB/sec 2024-09-13T00:03:01.384 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-u7xE4SJ2CItSh82 off=246594 len=1617157 2024-09-13T00:03:01.388 INFO:tasks.workunit.client.0.smithi136.stdout:op 496 completed, throughput=4.98MB/sec 2024-09-13T00:03:01.388 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-oIX7QIL7p44BBKm off=7448639 len=1137352 2024-09-13T00:03:01.397 INFO:tasks.workunit.client.0.smithi136.stdout:op 497 completed, throughput=4.99MB/sec 2024-09-13T00:03:01.398 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-fOCuAEhqghqVdNb off=17638641 len=1809900 2024-09-13T00:03:01.409 INFO:tasks.workunit.client.0.smithi136.stdout:op 498 completed, throughput=5.01MB/sec 2024-09-13T00:03:02.409 INFO:tasks.workunit.client.0.smithi136.stdout: 96: throughput=4.96MB/sec pending data=0 2024-09-13T00:03:02.410 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj--LYCQXJNs0jnhev off=4980399 len=1530606 2024-09-13T00:03:02.414 INFO:tasks.workunit.client.0.smithi136.stdout:op 499 completed, throughput=4.97MB/sec 2024-09-13T00:03:02.414 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-GrTDRtN3Iq6moh7 off=337865 len=578128 2024-09-13T00:03:02.421 INFO:tasks.workunit.client.0.smithi136.stdout:op 500 completed, throughput=4.98MB/sec 2024-09-13T00:03:02.421 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-RpMkQruYyt51tT7 off=19467013 len=1188770 2024-09-13T00:03:02.424 INFO:tasks.workunit.client.0.smithi136.stdout:op 501 completed, throughput=4.99MB/sec 2024-09-13T00:03:02.424 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-PqEecel-9hgd0T- off=9123943 len=409143 2024-09-13T00:03:02.426 INFO:tasks.workunit.client.0.smithi136.stdout:op 502 completed, throughput=4.99MB/sec 2024-09-13T00:03:02.426 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-GuyMoTIUMsCSUIm off=3458263 len=633425 2024-09-13T00:03:02.433 INFO:tasks.workunit.client.0.smithi136.stdout:op 503 completed, throughput=5MB/sec 2024-09-13T00:03:02.433 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-QaZckdYUxJuuJxg off=3261364 len=439523 2024-09-13T00:03:02.435 INFO:tasks.workunit.client.0.smithi136.stdout:op 504 completed, throughput=5MB/sec 2024-09-13T00:03:02.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:03:02 smithi181 bash[95505]: ts=2024-09-13T00:03:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:03:03.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:03:02 smithi136 bash[75164]: cluster 2024-09-13T00:03:01.197321+0000 mgr.y (mgr.24661) 777 : cluster [DBG] pgmap v1460: 105 pgs: 105 active+clean; 126 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:03:03.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:03:03 smithi136 bash[76371]: cluster 2024-09-13T00:03:01.197321+0000 mgr.y (mgr.24661) 777 : cluster [DBG] pgmap v1460: 105 pgs: 105 active+clean; 126 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:03:03.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:03:02 smithi181 bash[67587]: cluster 2024-09-13T00:03:01.197321+0000 mgr.y (mgr.24661) 777 : cluster [DBG] pgmap v1460: 105 pgs: 105 active+clean; 126 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:03:03.435 INFO:tasks.workunit.client.0.smithi136.stdout: 97: throughput=4.95MB/sec pending data=0 2024-09-13T00:03:03.436 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-VuScuHax2XtkjzQ off=12049729 len=2082910 2024-09-13T00:03:03.442 INFO:tasks.workunit.client.0.smithi136.stdout:op 505 completed, throughput=4.97MB/sec 2024-09-13T00:03:03.442 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-41UkL5BumETfD4y off=2878811 len=971666 2024-09-13T00:03:03.445 INFO:tasks.workunit.client.0.smithi136.stdout:op 506 completed, throughput=4.98MB/sec 2024-09-13T00:03:03.445 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-kfHa6mesj5PbJrH off=3864887 len=374889 2024-09-13T00:03:03.451 INFO:tasks.workunit.client.0.smithi136.stdout:op 507 completed, throughput=4.98MB/sec 2024-09-13T00:03:03.451 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ZloIdEPV6XDXiJI off=3822611 len=2056274 2024-09-13T00:03:03.456 INFO:tasks.workunit.client.0.smithi136.stdout:op 508 completed, throughput=5MB/sec 2024-09-13T00:03:04.456 INFO:tasks.workunit.client.0.smithi136.stdout: 98: throughput=4.95MB/sec pending data=0 2024-09-13T00:03:04.457 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-34bIIm75U_aUk9q off=7330500 len=898356 2024-09-13T00:03:04.465 INFO:tasks.workunit.client.0.smithi136.stdout:op 509 completed, throughput=4.96MB/sec 2024-09-13T00:03:04.465 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-k6XmQEeVV-O8UB7 off=12929341 len=1184297 2024-09-13T00:03:04.469 INFO:tasks.workunit.client.0.smithi136.stdout:op 510 completed, throughput=4.97MB/sec 2024-09-13T00:03:04.469 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-2i1DhNsm3jtbAdW off=1225116 len=1956958 2024-09-13T00:03:04.483 INFO:tasks.workunit.client.0.smithi136.stdout:op 511 completed, throughput=4.99MB/sec 2024-09-13T00:03:04.483 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-KRm3Pkp6WTwjVyq off=836795 len=1554647 2024-09-13T00:03:04.487 INFO:tasks.workunit.client.0.smithi136.stdout:op 512 completed, throughput=5.01MB/sec 2024-09-13T00:03:05.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:03:05 smithi136 bash[76371]: cluster 2024-09-13T00:03:03.198532+0000 mgr.y (mgr.24661) 778 : cluster [DBG] pgmap v1461: 105 pgs: 105 active+clean; 126 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 2.5 MiB/s rd, 3.0 MiB/s wr, 5 op/s 2024-09-13T00:03:05.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:03:05 smithi136 bash[75164]: cluster 2024-09-13T00:03:03.198532+0000 mgr.y (mgr.24661) 778 : cluster [DBG] pgmap v1461: 105 pgs: 105 active+clean; 126 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 2.5 MiB/s rd, 3.0 MiB/s wr, 5 op/s 2024-09-13T00:03:05.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:03:05 smithi181 bash[67587]: cluster 2024-09-13T00:03:03.198532+0000 mgr.y (mgr.24661) 778 : cluster [DBG] pgmap v1461: 105 pgs: 105 active+clean; 126 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 2.5 MiB/s rd, 3.0 MiB/s wr, 5 op/s 2024-09-13T00:03:05.487 INFO:tasks.workunit.client.0.smithi136.stdout: 99: throughput=4.96MB/sec pending data=0 2024-09-13T00:03:05.487 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-U1wFfAsf8FUFY37 off=273420 len=719626 2024-09-13T00:03:05.489 INFO:tasks.workunit.client.0.smithi136.stdout:op 513 completed, throughput=4.96MB/sec 2024-09-13T00:03:05.489 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-zQbecdWyKEjVYZq off=1415372 len=1456967 2024-09-13T00:03:05.494 INFO:tasks.workunit.client.0.smithi136.stdout:op 514 completed, throughput=4.98MB/sec 2024-09-13T00:03:05.494 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-YvzTJXMHXIvzs74 off=16474123 len=1193364 2024-09-13T00:03:05.500 INFO:tasks.workunit.client.0.smithi136.stdout:op 515 completed, throughput=4.99MB/sec 2024-09-13T00:03:05.500 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-cPVTb2IlAHx-NQ2 off=3643775 len=557710 2024-09-13T00:03:05.504 INFO:tasks.workunit.client.0.smithi136.stdout:op 516 completed, throughput=4.99MB/sec 2024-09-13T00:03:05.504 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2g9dPXApDWVBJXh off=1095185 len=1090858 2024-09-13T00:03:05.506 INFO:tasks.workunit.client.0.smithi136.stdout:op 517 completed, throughput=5MB/sec 2024-09-13T00:03:06.507 INFO:tasks.workunit.client.0.smithi136.stdout: 100: throughput=4.95MB/sec pending data=0 2024-09-13T00:03:06.507 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-WJfeU9J7r7SfhCe off=7627752 len=1920608 2024-09-13T00:03:06.524 INFO:tasks.workunit.client.0.smithi136.stdout:op 518 completed, throughput=4.97MB/sec 2024-09-13T00:03:06.524 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-PEEew3i0zm8Fwew off=7920825 len=1462121 2024-09-13T00:03:06.535 INFO:tasks.workunit.client.0.smithi136.stdout:op 519 completed, throughput=4.98MB/sec 2024-09-13T00:03:06.535 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Upg73xrXbFS8Ai0 off=1878009 len=1498264 2024-09-13T00:03:06.539 INFO:tasks.workunit.client.0.smithi136.stdout:op 520 completed, throughput=5MB/sec 2024-09-13T00:03:06.539 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Rc2CTFbB-EN7lN1 off=10750330 len=1217842 2024-09-13T00:03:06.543 INFO:tasks.workunit.client.0.smithi136.stdout:op 521 completed, throughput=5.01MB/sec 2024-09-13T00:03:07.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:03:07 smithi136 bash[75164]: cluster 2024-09-13T00:03:05.199447+0000 mgr.y (mgr.24661) 779 : cluster [DBG] pgmap v1462: 105 pgs: 105 active+clean; 126 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:03:07.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:03:07 smithi136 bash[76371]: cluster 2024-09-13T00:03:05.199447+0000 mgr.y (mgr.24661) 779 : cluster [DBG] pgmap v1462: 105 pgs: 105 active+clean; 126 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:03:07.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:03:07 smithi181 bash[67587]: cluster 2024-09-13T00:03:05.199447+0000 mgr.y (mgr.24661) 779 : cluster [DBG] pgmap v1462: 105 pgs: 105 active+clean; 126 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:03:07.544 INFO:tasks.workunit.client.0.smithi136.stdout: 101: throughput=4.96MB/sec pending data=0 2024-09-13T00:03:07.545 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-5p-egg7_xDJo-S6 off=2567916 len=1581608 2024-09-13T00:03:07.548 INFO:tasks.workunit.client.0.smithi136.stdout:op 522 completed, throughput=4.97MB/sec 2024-09-13T00:03:07.548 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-iaLLFm6rUJ4SVZ- off=18961075 len=28704 2024-09-13T00:03:07.553 INFO:tasks.workunit.client.0.smithi136.stdout:op 523 completed, throughput=4.98MB/sec 2024-09-13T00:03:07.553 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-gYqTkc_LX5OnQMb off=15905213 len=184055 2024-09-13T00:03:07.554 INFO:tasks.workunit.client.0.smithi136.stdout:op 524 completed, throughput=4.98MB/sec 2024-09-13T00:03:07.554 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-5DdRcVs9SyPbWwF off=2785360 len=1316797 2024-09-13T00:03:07.558 INFO:tasks.workunit.client.0.smithi136.stdout:op 525 completed, throughput=4.99MB/sec 2024-09-13T00:03:07.558 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-rfyurwYVW9gn5pI off=17345212 len=2013072 2024-09-13T00:03:07.562 INFO:tasks.workunit.client.0.smithi136.stdout:op 526 completed, throughput=5.01MB/sec 2024-09-13T00:03:08.563 INFO:tasks.workunit.client.0.smithi136.stdout: 102: throughput=4.96MB/sec pending data=0 2024-09-13T00:03:08.563 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-6NaOELq6igKBZMu off=2872758 len=727255 2024-09-13T00:03:08.569 INFO:tasks.workunit.client.0.smithi136.stdout:op 527 completed, throughput=4.97MB/sec 2024-09-13T00:03:08.569 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-bJ9Oxh_9HgDNIpa off=487762 len=624311 2024-09-13T00:03:08.569 INFO:tasks.workunit.client.0.smithi136.stdout:op 528 completed, throughput=4.97MB/sec 2024-09-13T00:03:08.569 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Qvbc1Yl4txAUioY off=5804107 len=510923 2024-09-13T00:03:08.569 INFO:tasks.workunit.client.0.smithi136.stdout:op 529 completed, throughput=4.98MB/sec 2024-09-13T00:03:08.570 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-1m8-gc0JUbTmqVX off=13895929 len=140764 2024-09-13T00:03:08.575 INFO:tasks.workunit.client.0.smithi136.stdout:op 530 completed, throughput=4.98MB/sec 2024-09-13T00:03:08.575 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-y9QmGKDJKOfm9d_ off=4399838 len=1041199 2024-09-13T00:03:08.584 INFO:tasks.workunit.client.0.smithi136.stdout:op 531 completed, throughput=4.99MB/sec 2024-09-13T00:03:08.584 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-4GpNEurykf4eDIt off=4241921 len=501049 2024-09-13T00:03:08.586 INFO:tasks.workunit.client.0.smithi136.stdout:op 532 completed, throughput=4.99MB/sec 2024-09-13T00:03:08.586 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2Te3flRop9rv-ot off=1380842 len=897130 2024-09-13T00:03:08.589 INFO:tasks.workunit.client.0.smithi136.stdout:op 533 completed, throughput=5MB/sec 2024-09-13T00:03:08.589 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-IX--FzbDizjwwJu off=20002345 len=1672428 2024-09-13T00:03:08.602 INFO:tasks.workunit.client.0.smithi136.stdout:op 534 completed, throughput=5.01MB/sec 2024-09-13T00:03:09.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:03:09 smithi181 bash[67587]: cluster 2024-09-13T00:03:07.200550+0000 mgr.y (mgr.24661) 780 : cluster [DBG] pgmap v1463: 105 pgs: 105 active+clean; 126 GiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:03:09.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:03:09 smithi181 bash[67587]: audit 2024-09-13T00:03:08.774065+0000 mon.a (mon.0) 2123 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:03:09.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:03:09 smithi136 bash[75164]: cluster 2024-09-13T00:03:07.200550+0000 mgr.y (mgr.24661) 780 : cluster [DBG] pgmap v1463: 105 pgs: 105 active+clean; 126 GiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:03:09.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:03:09 smithi136 bash[75164]: audit 2024-09-13T00:03:08.774065+0000 mon.a (mon.0) 2123 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:03:09.414 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:03:09 smithi136 bash[76371]: cluster 2024-09-13T00:03:07.200550+0000 mgr.y (mgr.24661) 780 : cluster [DBG] pgmap v1463: 105 pgs: 105 active+clean; 126 GiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:03:09.414 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:03:09 smithi136 bash[76371]: audit 2024-09-13T00:03:08.774065+0000 mon.a (mon.0) 2123 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:03:09.602 INFO:tasks.workunit.client.0.smithi136.stdout: 103: throughput=4.97MB/sec pending data=0 2024-09-13T00:03:09.602 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-_lOqfCU0opl54EB off=8790195 len=537889 2024-09-13T00:03:09.604 INFO:tasks.workunit.client.0.smithi136.stdout:op 535 completed, throughput=4.97MB/sec 2024-09-13T00:03:09.604 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj--1hW2XKB4rZDap4 off=213163 len=1793499 2024-09-13T00:03:09.609 INFO:tasks.workunit.client.0.smithi136.stdout:op 536 completed, throughput=4.99MB/sec 2024-09-13T00:03:09.609 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-IKblgefD_uaEB2F off=4318534 len=536024 2024-09-13T00:03:09.615 INFO:tasks.workunit.client.0.smithi136.stdout:op 537 completed, throughput=4.99MB/sec 2024-09-13T00:03:09.615 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-y2Nqn1mtZv1t0sw off=3659313 len=1364792 2024-09-13T00:03:09.624 INFO:tasks.workunit.client.0.smithi136.stdout:op 538 completed, throughput=5MB/sec 2024-09-13T00:03:09.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:03:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:03:09] "GET /metrics HTTP/1.1" 200 34125 "" "Prometheus/2.43.0" 2024-09-13T00:03:10.625 INFO:tasks.workunit.client.0.smithi136.stdout: 104: throughput=4.96MB/sec pending data=0 2024-09-13T00:03:10.625 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-UNiodbQNvQVFUms off=644051 len=824693 2024-09-13T00:03:10.632 INFO:tasks.workunit.client.0.smithi136.stdout:op 539 completed, throughput=4.96MB/sec 2024-09-13T00:03:10.632 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-0hFWgFAdK5jMbYw off=5167111 len=166887 2024-09-13T00:03:10.638 INFO:tasks.workunit.client.0.smithi136.stdout:op 540 completed, throughput=4.96MB/sec 2024-09-13T00:03:10.638 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-xKcGfAQWjQ3nfsr off=5403714 len=1796633 2024-09-13T00:03:10.642 INFO:tasks.workunit.client.0.smithi136.stdout:op 541 completed, throughput=4.98MB/sec 2024-09-13T00:03:10.642 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-TXAEMAJYn1N0PGX off=952966 len=573018 2024-09-13T00:03:10.648 INFO:tasks.workunit.client.0.smithi136.stdout:op 542 completed, throughput=4.98MB/sec 2024-09-13T00:03:10.648 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-BHhC7YAaaM934uU off=9401572 len=645187 2024-09-13T00:03:10.650 INFO:tasks.workunit.client.0.smithi136.stdout:op 543 completed, throughput=4.99MB/sec 2024-09-13T00:03:10.650 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-k2QDakv2YKmgCdv off=18181682 len=381085 2024-09-13T00:03:10.652 INFO:tasks.workunit.client.0.smithi136.stdout:op 544 completed, throughput=4.99MB/sec 2024-09-13T00:03:10.652 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-oBXn94nKgIPU3AX off=8829855 len=975987 2024-09-13T00:03:10.655 INFO:tasks.workunit.client.0.smithi136.stdout:op 545 completed, throughput=5MB/sec 2024-09-13T00:03:11.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:03:11 smithi181 bash[67587]: cluster 2024-09-13T00:03:09.201386+0000 mgr.y (mgr.24661) 781 : cluster [DBG] pgmap v1464: 105 pgs: 105 active+clean; 126 GiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:03:11.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:03:11 smithi136 bash[75164]: cluster 2024-09-13T00:03:09.201386+0000 mgr.y (mgr.24661) 781 : cluster [DBG] pgmap v1464: 105 pgs: 105 active+clean; 126 GiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:03:11.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:03:11 smithi136 bash[76371]: cluster 2024-09-13T00:03:09.201386+0000 mgr.y (mgr.24661) 781 : cluster [DBG] pgmap v1464: 105 pgs: 105 active+clean; 126 GiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:03:11.655 INFO:tasks.workunit.client.0.smithi136.stdout: 105: throughput=4.96MB/sec pending data=0 2024-09-13T00:03:11.656 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-s4Z8Mi6lS4sxzjW off=1429827 len=825728 2024-09-13T00:03:11.663 INFO:tasks.workunit.client.0.smithi136.stdout:op 546 completed, throughput=4.96MB/sec 2024-09-13T00:03:11.663 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-a-x9Y87DJ2Ah8Ge off=8357598 len=496943 2024-09-13T00:03:11.669 INFO:tasks.workunit.client.0.smithi136.stdout:op 547 completed, throughput=4.97MB/sec 2024-09-13T00:03:11.669 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-tWzJm7cKq_M7CR7 off=12934336 len=1021575 2024-09-13T00:03:11.673 INFO:tasks.workunit.client.0.smithi136.stdout:op 548 completed, throughput=4.98MB/sec 2024-09-13T00:03:11.673 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-C50qpI5oCzouL5z off=5742987 len=2037386 2024-09-13T00:03:11.688 INFO:tasks.workunit.client.0.smithi136.stdout:op 549 completed, throughput=4.99MB/sec 2024-09-13T00:03:11.688 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-TXAEMAJYn1N0PGX off=4144385 len=395989 2024-09-13T00:03:11.693 INFO:tasks.workunit.client.0.smithi136.stdout:op 550 completed, throughput=5MB/sec 2024-09-13T00:03:11.693 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-05UWFjT7FhycV9m off=6117738 len=1283458 2024-09-13T00:03:11.703 INFO:tasks.workunit.client.0.smithi136.stdout:op 551 completed, throughput=5.01MB/sec 2024-09-13T00:03:12.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:03:12 smithi181 bash[95505]: ts=2024-09-13T00:03:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:03:12.703 INFO:tasks.workunit.client.0.smithi136.stdout: 106: throughput=4.96MB/sec pending data=0 2024-09-13T00:03:12.704 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-BptRRU6DDiyqD-2 off=2142475 len=1155571 2024-09-13T00:03:12.708 INFO:tasks.workunit.client.0.smithi136.stdout:op 552 completed, throughput=4.97MB/sec 2024-09-13T00:03:12.709 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-SaMKMmSDDXEp-Y_ off=6942361 len=1547688 2024-09-13T00:03:12.723 INFO:tasks.workunit.client.0.smithi136.stdout:op 553 completed, throughput=4.98MB/sec 2024-09-13T00:03:12.723 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-PGc6DOk50KG3n0_ off=15322364 len=1271356 2024-09-13T00:03:12.726 INFO:tasks.workunit.client.0.smithi136.stdout:op 554 completed, throughput=4.99MB/sec 2024-09-13T00:03:12.727 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-QumPDJEqCRqG54n off=16725240 len=1673162 2024-09-13T00:03:12.742 INFO:tasks.workunit.client.0.smithi136.stdout:op 555 completed, throughput=5.01MB/sec 2024-09-13T00:03:13.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:03:13 smithi181 bash[67587]: cluster 2024-09-13T00:03:11.202315+0000 mgr.y (mgr.24661) 782 : cluster [DBG] pgmap v1465: 105 pgs: 105 active+clean; 126 GiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:03:13.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:03:13 smithi136 bash[75164]: cluster 2024-09-13T00:03:11.202315+0000 mgr.y (mgr.24661) 782 : cluster [DBG] pgmap v1465: 105 pgs: 105 active+clean; 126 GiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:03:13.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:03:13 smithi136 bash[76371]: cluster 2024-09-13T00:03:11.202315+0000 mgr.y (mgr.24661) 782 : cluster [DBG] pgmap v1465: 105 pgs: 105 active+clean; 126 GiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:03:13.742 INFO:tasks.workunit.client.0.smithi136.stdout: 107: throughput=4.96MB/sec pending data=0 2024-09-13T00:03:13.742 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Gsz0jZeRUAxVNPq off=4088430 len=941501 2024-09-13T00:03:13.752 INFO:tasks.workunit.client.0.smithi136.stdout:op 556 completed, throughput=4.97MB/sec 2024-09-13T00:03:13.752 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-rvVGdhEyPyZI1OI off=13365597 len=1083421 2024-09-13T00:03:13.760 INFO:tasks.workunit.client.0.smithi136.stdout:op 557 completed, throughput=4.98MB/sec 2024-09-13T00:03:13.761 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-GIHwOF46tLr8-Mw off=16803082 len=2022647 2024-09-13T00:03:13.763 INFO:tasks.workunit.client.0.smithi136.stdout:op 558 completed, throughput=5MB/sec 2024-09-13T00:03:13.763 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-EzucmjsGVmIAV4y off=1088780 len=1771407 2024-09-13T00:03:13.779 INFO:tasks.workunit.client.0.smithi136.stdout:op 559 completed, throughput=5.01MB/sec 2024-09-13T00:03:14.779 INFO:tasks.workunit.client.0.smithi136.stdout: 108: throughput=4.97MB/sec pending data=0 2024-09-13T00:03:14.780 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-NAjWkksS1eWFk4e off=12689629 len=188889 2024-09-13T00:03:14.781 INFO:tasks.workunit.client.0.smithi136.stdout:op 560 completed, throughput=4.97MB/sec 2024-09-13T00:03:14.781 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-GB8j7aBpzWL-rSH off=0 len=1674604 2024-09-13T00:03:14.785 INFO:tasks.workunit.client.0.smithi136.stdout:op 561 completed, throughput=4.98MB/sec 2024-09-13T00:03:14.785 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-uESlNqRCcBW_abY off=21068713 len=389954 2024-09-13T00:03:14.787 INFO:tasks.workunit.client.0.smithi136.stdout:op 562 completed, throughput=4.99MB/sec 2024-09-13T00:03:14.788 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-tcwCrAU3VJPGvlg off=673448 len=21118 2024-09-13T00:03:14.788 INFO:tasks.workunit.client.0.smithi136.stdout:op 563 completed, throughput=4.99MB/sec 2024-09-13T00:03:14.788 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-EWDQxh4KSMBQBxw off=15209198 len=182180 2024-09-13T00:03:14.793 INFO:tasks.workunit.client.0.smithi136.stdout:op 564 completed, throughput=4.99MB/sec 2024-09-13T00:03:14.793 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-0xXOCHpgTQQCNQA off=3083525 len=1444507 2024-09-13T00:03:14.804 INFO:tasks.workunit.client.0.smithi136.stdout:op 565 completed, throughput=5MB/sec 2024-09-13T00:03:14.804 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Pi25Fnsgr_YTSF9 off=10681200 len=1719820 2024-09-13T00:03:14.808 INFO:tasks.workunit.client.0.smithi136.stdout:op 566 completed, throughput=5.01MB/sec 2024-09-13T00:03:15.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:03:15 smithi181 bash[67587]: cluster 2024-09-13T00:03:13.203344+0000 mgr.y (mgr.24661) 783 : cluster [DBG] pgmap v1466: 105 pgs: 105 active+clean; 126 GiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:03:15.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:03:15 smithi136 bash[75164]: cluster 2024-09-13T00:03:13.203344+0000 mgr.y (mgr.24661) 783 : cluster [DBG] pgmap v1466: 105 pgs: 105 active+clean; 126 GiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:03:15.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:03:15 smithi136 bash[76371]: cluster 2024-09-13T00:03:13.203344+0000 mgr.y (mgr.24661) 783 : cluster [DBG] pgmap v1466: 105 pgs: 105 active+clean; 126 GiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:03:15.809 INFO:tasks.workunit.client.0.smithi136.stdout: 109: throughput=4.97MB/sec pending data=0 2024-09-13T00:03:15.809 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-C4lJB-hVMW30Zu_ off=8249212 len=1212075 2024-09-13T00:03:15.812 INFO:tasks.workunit.client.0.smithi136.stdout:op 567 completed, throughput=4.98MB/sec 2024-09-13T00:03:15.812 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-XVlmapOzisRZekK off=17354420 len=1914497 2024-09-13T00:03:15.817 INFO:tasks.workunit.client.0.smithi136.stdout:op 568 completed, throughput=5MB/sec 2024-09-13T00:03:15.817 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-pRvpwnOH7xn77zQ off=6638735 len=1540092 2024-09-13T00:03:15.820 INFO:tasks.workunit.client.0.smithi136.stdout:op 569 completed, throughput=5.01MB/sec 2024-09-13T00:03:16.821 INFO:tasks.workunit.client.0.smithi136.stdout: 110: throughput=4.96MB/sec pending data=0 2024-09-13T00:03:16.821 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-DdCMhYdfrTXqT0A off=4155335 len=1360294 2024-09-13T00:03:16.834 INFO:tasks.workunit.client.0.smithi136.stdout:op 570 completed, throughput=4.97MB/sec 2024-09-13T00:03:16.834 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ti3A_Llpommr26u off=2677076 len=1165473 2024-09-13T00:03:16.844 INFO:tasks.workunit.client.0.smithi136.stdout:op 571 completed, throughput=4.98MB/sec 2024-09-13T00:03:16.845 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Gq-pdG-XTnIMYHT off=1290272 len=424914 2024-09-13T00:03:16.850 INFO:tasks.workunit.client.0.smithi136.stdout:op 572 completed, throughput=4.99MB/sec 2024-09-13T00:03:16.851 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Ux8UoPPb44cHLNP off=8479924 len=769743 2024-09-13T00:03:16.858 INFO:tasks.workunit.client.0.smithi136.stdout:op 573 completed, throughput=4.99MB/sec 2024-09-13T00:03:16.858 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Jx5OHQdbvGXSp39 off=19700410 len=672320 2024-09-13T00:03:16.860 INFO:tasks.workunit.client.0.smithi136.stdout:op 574 completed, throughput=5MB/sec 2024-09-13T00:03:16.861 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-qoSWKlSkNbBZIxP off=3074475 len=579677 2024-09-13T00:03:16.862 INFO:tasks.workunit.client.0.smithi136.stdout:op 575 completed, throughput=5MB/sec 2024-09-13T00:03:17.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:03:17 smithi181 bash[67587]: cluster 2024-09-13T00:03:15.204205+0000 mgr.y (mgr.24661) 784 : cluster [DBG] pgmap v1467: 105 pgs: 105 active+clean; 126 GiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 2.8 MiB/s rd, 1.8 MiB/s wr, 4 op/s 2024-09-13T00:03:17.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:03:17 smithi136 bash[75164]: cluster 2024-09-13T00:03:15.204205+0000 mgr.y (mgr.24661) 784 : cluster [DBG] pgmap v1467: 105 pgs: 105 active+clean; 126 GiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 2.8 MiB/s rd, 1.8 MiB/s wr, 4 op/s 2024-09-13T00:03:17.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:03:17 smithi136 bash[76371]: cluster 2024-09-13T00:03:15.204205+0000 mgr.y (mgr.24661) 784 : cluster [DBG] pgmap v1467: 105 pgs: 105 active+clean; 126 GiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 2.8 MiB/s rd, 1.8 MiB/s wr, 4 op/s 2024-09-13T00:03:17.862 INFO:tasks.workunit.client.0.smithi136.stdout: 111: throughput=4.96MB/sec pending data=0 2024-09-13T00:03:17.862 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-tWO2w13xpEFzTZ- off=746680 len=1314073 2024-09-13T00:03:17.872 INFO:tasks.workunit.client.0.smithi136.stdout:op 576 completed, throughput=4.97MB/sec 2024-09-13T00:03:17.873 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-6uLfNN-p5tSumCY off=6258299 len=1884992 2024-09-13T00:03:17.884 INFO:tasks.workunit.client.0.smithi136.stdout:op 577 completed, throughput=4.99MB/sec 2024-09-13T00:03:17.884 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-W8NQ4_AOlO1rpiV off=4303398 len=136754 2024-09-13T00:03:17.885 INFO:tasks.workunit.client.0.smithi136.stdout:op 578 completed, throughput=4.99MB/sec 2024-09-13T00:03:17.885 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-8sdye1fUGCBWC_Y off=7835823 len=686993 2024-09-13T00:03:17.887 INFO:tasks.workunit.client.0.smithi136.stdout:op 579 completed, throughput=4.99MB/sec 2024-09-13T00:03:17.887 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-TY3rFohDxD5Z9Wf off=4084700 len=1410457 2024-09-13T00:03:17.897 INFO:tasks.workunit.client.0.smithi136.stdout:op 580 completed, throughput=5MB/sec 2024-09-13T00:03:18.897 INFO:tasks.workunit.client.0.smithi136.stdout: 112: throughput=4.96MB/sec pending data=0 2024-09-13T00:03:18.898 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-KOjoQOK2I9OnCbF off=12922436 len=870453 2024-09-13T00:03:18.907 INFO:tasks.workunit.client.0.smithi136.stdout:op 581 completed, throughput=4.97MB/sec 2024-09-13T00:03:18.908 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-aouaWsECixrsOEW off=17211075 len=1183543 2024-09-13T00:03:18.917 INFO:tasks.workunit.client.0.smithi136.stdout:op 582 completed, throughput=4.98MB/sec 2024-09-13T00:03:18.917 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-9GBZ88ZdY526CE3 off=12983787 len=1872 2024-09-13T00:03:18.918 INFO:tasks.workunit.client.0.smithi136.stdout:op 583 completed, throughput=4.98MB/sec 2024-09-13T00:03:18.918 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-LW4q0y_4BX8iUec off=38038 len=18619 2024-09-13T00:03:18.922 INFO:tasks.workunit.client.0.smithi136.stdout:op 584 completed, throughput=4.98MB/sec 2024-09-13T00:03:18.922 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-PObhZtKfAplnmrp off=71581 len=83753 2024-09-13T00:03:18.924 INFO:tasks.workunit.client.0.smithi136.stdout:op 585 completed, throughput=4.98MB/sec 2024-09-13T00:03:18.924 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-u8o6131kO8tVMb7 off=7332837 len=617114 2024-09-13T00:03:18.930 INFO:tasks.workunit.client.0.smithi136.stdout:op 586 completed, throughput=4.98MB/sec 2024-09-13T00:03:18.931 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj--98h-BtfbZ9owHt off=7980871 len=1981434 2024-09-13T00:03:18.945 INFO:tasks.workunit.client.0.smithi136.stdout:op 587 completed, throughput=5MB/sec 2024-09-13T00:03:18.945 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-XczSvAGbl9gss1v off=2319323 len=2034455 2024-09-13T00:03:18.949 INFO:tasks.workunit.client.0.smithi136.stdout:op 588 completed, throughput=5.01MB/sec 2024-09-13T00:03:19.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:03:19 smithi181 bash[67587]: cluster 2024-09-13T00:03:17.205348+0000 mgr.y (mgr.24661) 785 : cluster [DBG] pgmap v1468: 105 pgs: 105 active+clean; 126 GiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 3.4 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:03:19.414 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:03:19 smithi136 bash[76371]: cluster 2024-09-13T00:03:17.205348+0000 mgr.y (mgr.24661) 785 : cluster [DBG] pgmap v1468: 105 pgs: 105 active+clean; 126 GiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 3.4 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:03:19.414 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:03:19 smithi136 bash[75164]: cluster 2024-09-13T00:03:17.205348+0000 mgr.y (mgr.24661) 785 : cluster [DBG] pgmap v1468: 105 pgs: 105 active+clean; 126 GiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 3.4 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:03:19.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:03:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:03:19] "GET /metrics HTTP/1.1" 200 34125 "" "Prometheus/2.43.0" 2024-09-13T00:03:19.950 INFO:tasks.workunit.client.0.smithi136.stdout: 113: throughput=4.97MB/sec pending data=0 2024-09-13T00:03:19.950 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-R63_ooH3F2YiK0v off=2115237 len=639729 2024-09-13T00:03:19.957 INFO:tasks.workunit.client.0.smithi136.stdout:op 589 completed, throughput=4.98MB/sec 2024-09-13T00:03:19.957 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-y9Df4CKKLysPh71 off=328555 len=1799710 2024-09-13T00:03:19.970 INFO:tasks.workunit.client.0.smithi136.stdout:op 590 completed, throughput=4.99MB/sec 2024-09-13T00:03:19.970 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-hyiIUgmeJXVwSuG off=15717521 len=763647 2024-09-13T00:03:19.973 INFO:tasks.workunit.client.0.smithi136.stdout:op 591 completed, throughput=5MB/sec 2024-09-13T00:03:19.973 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-fS1fZzkEWEH7jEK off=4330726 len=394506 2024-09-13T00:03:19.975 INFO:tasks.workunit.client.0.smithi136.stdout:op 592 completed, throughput=5MB/sec 2024-09-13T00:03:19.975 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-BgIhChe__9pzM1f off=12261975 len=1161706 2024-09-13T00:03:19.985 INFO:tasks.workunit.client.0.smithi136.stdout:op 593 completed, throughput=5.01MB/sec 2024-09-13T00:03:20.985 INFO:tasks.workunit.client.0.smithi136.stdout: 114: throughput=4.97MB/sec pending data=0 2024-09-13T00:03:20.986 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-FATTyWi4HgAlZav off=16488298 len=1927335 2024-09-13T00:03:21.002 INFO:tasks.workunit.client.0.smithi136.stdout:op 594 completed, throughput=4.98MB/sec 2024-09-13T00:03:21.002 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-TZQHUQS-hY_Lfgm off=1460786 len=479942 2024-09-13T00:03:21.004 INFO:tasks.workunit.client.0.smithi136.stdout:op 595 completed, throughput=4.99MB/sec 2024-09-13T00:03:21.004 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-WsjuP9lhhYd4W4P off=9688841 len=1141589 2024-09-13T00:03:21.007 INFO:tasks.workunit.client.0.smithi136.stdout:op 596 completed, throughput=4.99MB/sec 2024-09-13T00:03:21.007 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-OgBC5cbQ4nMHMBn off=1828732 len=320561 2024-09-13T00:03:21.009 INFO:tasks.workunit.client.0.smithi136.stdout:op 597 completed, throughput=5MB/sec 2024-09-13T00:03:21.009 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-U7X1B6MlNlj2zUq off=4661707 len=1891983 2024-09-13T00:03:21.024 INFO:tasks.workunit.client.0.smithi136.stdout:op 598 completed, throughput=5.01MB/sec 2024-09-13T00:03:21.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:03:21 smithi181 bash[67587]: cluster 2024-09-13T00:03:19.206255+0000 mgr.y (mgr.24661) 786 : cluster [DBG] pgmap v1469: 105 pgs: 105 active+clean; 126 GiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.0 MiB/s wr, 5 op/s 2024-09-13T00:03:21.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:03:21 smithi136 bash[76371]: cluster 2024-09-13T00:03:19.206255+0000 mgr.y (mgr.24661) 786 : cluster [DBG] pgmap v1469: 105 pgs: 105 active+clean; 126 GiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.0 MiB/s wr, 5 op/s 2024-09-13T00:03:21.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:03:21 smithi136 bash[75164]: cluster 2024-09-13T00:03:19.206255+0000 mgr.y (mgr.24661) 786 : cluster [DBG] pgmap v1469: 105 pgs: 105 active+clean; 126 GiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.0 MiB/s wr, 5 op/s 2024-09-13T00:03:22.024 INFO:tasks.workunit.client.0.smithi136.stdout: 115: throughput=4.97MB/sec pending data=0 2024-09-13T00:03:22.024 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-GWLCCkmdW18kyHS off=2603335 len=1764914 2024-09-13T00:03:22.030 INFO:tasks.workunit.client.0.smithi136.stdout:op 599 completed, throughput=4.98MB/sec 2024-09-13T00:03:22.030 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-FWtUDzrbmGStBaZ off=3585278 len=1429787 2024-09-13T00:03:22.036 INFO:tasks.workunit.client.0.smithi136.stdout:op 600 completed, throughput=4.99MB/sec 2024-09-13T00:03:22.037 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-z92NZK1nJ9G_OgE off=2060697 len=1006560 2024-09-13T00:03:22.044 INFO:tasks.workunit.client.0.smithi136.stdout:op 601 completed, throughput=5MB/sec 2024-09-13T00:03:22.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:03:22 smithi181 bash[95505]: ts=2024-09-13T00:03:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:03:23.044 INFO:tasks.workunit.client.0.smithi136.stdout: 116: throughput=4.96MB/sec pending data=0 2024-09-13T00:03:23.044 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-2zBhUwbI-zsmTGw off=10510711 len=455901 2024-09-13T00:03:23.050 INFO:tasks.workunit.client.0.smithi136.stdout:op 602 completed, throughput=4.96MB/sec 2024-09-13T00:03:23.050 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-l1_xp5sRHm8vHya off=527926 len=1913134 2024-09-13T00:03:23.064 INFO:tasks.workunit.client.0.smithi136.stdout:op 603 completed, throughput=4.98MB/sec 2024-09-13T00:03:23.064 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-rY52X4ELULgHQ7j off=13960370 len=1125262 2024-09-13T00:03:23.073 INFO:tasks.workunit.client.0.smithi136.stdout:op 604 completed, throughput=4.99MB/sec 2024-09-13T00:03:23.073 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-0q6M3r68jbqnGWu off=1565862 len=1189677 2024-09-13T00:03:23.077 INFO:tasks.workunit.client.0.smithi136.stdout:op 605 completed, throughput=5MB/sec 2024-09-13T00:03:23.077 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-3DJidXVOgKmAh-8 off=420765 len=1717234 2024-09-13T00:03:23.091 INFO:tasks.workunit.client.0.smithi136.stdout:op 606 completed, throughput=5.01MB/sec 2024-09-13T00:03:23.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:03:23 smithi181 bash[67587]: cluster 2024-09-13T00:03:21.207385+0000 mgr.y (mgr.24661) 787 : cluster [DBG] pgmap v1470: 105 pgs: 105 active+clean; 126 GiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:03:23.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:03:23 smithi136 bash[75164]: cluster 2024-09-13T00:03:21.207385+0000 mgr.y (mgr.24661) 787 : cluster [DBG] pgmap v1470: 105 pgs: 105 active+clean; 126 GiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:03:23.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:03:23 smithi136 bash[76371]: cluster 2024-09-13T00:03:21.207385+0000 mgr.y (mgr.24661) 787 : cluster [DBG] pgmap v1470: 105 pgs: 105 active+clean; 126 GiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:03:24.091 INFO:tasks.workunit.client.0.smithi136.stdout: 117: throughput=4.97MB/sec pending data=0 2024-09-13T00:03:24.091 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-yhpzJnXFlVoXIKq off=6066526 len=1542200 2024-09-13T00:03:24.095 INFO:tasks.workunit.client.0.smithi136.stdout:op 607 completed, throughput=4.98MB/sec 2024-09-13T00:03:24.095 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-FdSwmJ2xjMeToS_ off=9137890 len=1569324 2024-09-13T00:03:24.107 INFO:tasks.workunit.client.0.smithi136.stdout:op 608 completed, throughput=4.99MB/sec 2024-09-13T00:03:24.108 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-NiGJX1z8DX2wN9a off=88944 len=1224853 2024-09-13T00:03:24.110 INFO:tasks.workunit.client.0.smithi136.stdout:op 609 completed, throughput=5MB/sec 2024-09-13T00:03:24.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:03:24 smithi181 bash[67587]: audit 2024-09-13T00:03:23.774436+0000 mon.a (mon.0) 2124 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:03:24.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:03:24 smithi136 bash[75164]: audit 2024-09-13T00:03:23.774436+0000 mon.a (mon.0) 2124 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:03:24.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:03:24 smithi136 bash[76371]: audit 2024-09-13T00:03:23.774436+0000 mon.a (mon.0) 2124 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:03:25.111 INFO:tasks.workunit.client.0.smithi136.stdout: 118: throughput=4.96MB/sec pending data=0 2024-09-13T00:03:25.111 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-nOdLy-ykhYdYMqU off=9445748 len=1862662 2024-09-13T00:03:25.123 INFO:tasks.workunit.client.0.smithi136.stdout:op 610 completed, throughput=4.97MB/sec 2024-09-13T00:03:25.123 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-VeWZJK6ciuIfnud off=4744742 len=2048038 2024-09-13T00:03:25.126 INFO:tasks.workunit.client.0.smithi136.stdout:op 611 completed, throughput=4.99MB/sec 2024-09-13T00:03:25.126 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Ut5tatRHQ6bxlyX off=10499197 len=1493996 2024-09-13T00:03:25.137 INFO:tasks.workunit.client.0.smithi136.stdout:op 612 completed, throughput=5MB/sec 2024-09-13T00:03:25.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:03:25 smithi181 bash[67587]: cluster 2024-09-13T00:03:23.208494+0000 mgr.y (mgr.24661) 788 : cluster [DBG] pgmap v1471: 105 pgs: 105 active+clean; 126 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:03:25.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:03:25 smithi136 bash[75164]: cluster 2024-09-13T00:03:23.208494+0000 mgr.y (mgr.24661) 788 : cluster [DBG] pgmap v1471: 105 pgs: 105 active+clean; 126 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:03:25.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:03:25 smithi136 bash[76371]: cluster 2024-09-13T00:03:23.208494+0000 mgr.y (mgr.24661) 788 : cluster [DBG] pgmap v1471: 105 pgs: 105 active+clean; 126 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:03:26.137 INFO:tasks.workunit.client.0.smithi136.stdout: 119: throughput=4.96MB/sec pending data=0 2024-09-13T00:03:26.137 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-2P8VWUZGiOWSnJX off=2562393 len=1437136 2024-09-13T00:03:26.148 INFO:tasks.workunit.client.0.smithi136.stdout:op 613 completed, throughput=4.97MB/sec 2024-09-13T00:03:26.148 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-dHvnDxZuuPbJIXT off=1117837 len=210785 2024-09-13T00:03:26.153 INFO:tasks.workunit.client.0.smithi136.stdout:op 614 completed, throughput=4.97MB/sec 2024-09-13T00:03:26.153 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-zu8KTTjyAmdOpqd off=5147502 len=848189 2024-09-13T00:03:26.160 INFO:tasks.workunit.client.0.smithi136.stdout:op 615 completed, throughput=4.98MB/sec 2024-09-13T00:03:26.160 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-kXthstME66Dk3ZZ off=11668426 len=2096783 2024-09-13T00:03:26.166 INFO:tasks.workunit.client.0.smithi136.stdout:op 616 completed, throughput=5MB/sec 2024-09-13T00:03:26.166 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-jtN8TBdnerEDCXP off=1215018 len=1847628 2024-09-13T00:03:26.177 INFO:tasks.workunit.client.0.smithi136.stdout:op 617 completed, throughput=5.01MB/sec 2024-09-13T00:03:27.177 INFO:tasks.workunit.client.0.smithi136.stdout: 120: throughput=4.97MB/sec pending data=0 2024-09-13T00:03:27.177 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Gh0PYZWvtj6J09n off=10366621 len=346463 2024-09-13T00:03:27.183 INFO:tasks.workunit.client.0.smithi136.stdout:op 618 completed, throughput=4.97MB/sec 2024-09-13T00:03:27.183 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-87QJJAmWEapcHBJ off=9655539 len=472263 2024-09-13T00:03:27.189 INFO:tasks.workunit.client.0.smithi136.stdout:op 619 completed, throughput=4.97MB/sec 2024-09-13T00:03:27.189 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-hSzM7AxNTxkTmtG off=472633 len=1621774 2024-09-13T00:03:27.204 INFO:tasks.workunit.client.0.smithi136.stdout:op 620 completed, throughput=4.99MB/sec 2024-09-13T00:03:27.204 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-s6r9hlh6fFjmt_a off=1870978 len=1620882 2024-09-13T00:03:27.215 INFO:tasks.workunit.client.0.smithi136.stdout:op 621 completed, throughput=5MB/sec 2024-09-13T00:03:27.215 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-rWC82f-CPDOcLVd off=12453992 len=1553979 2024-09-13T00:03:27.226 INFO:tasks.workunit.client.0.smithi136.stdout:op 622 completed, throughput=5.01MB/sec 2024-09-13T00:03:27.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:03:27 smithi181 bash[67587]: cluster 2024-09-13T00:03:25.209354+0000 mgr.y (mgr.24661) 789 : cluster [DBG] pgmap v1472: 105 pgs: 105 active+clean; 126 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 2.0 MiB/s rd, 3.1 MiB/s wr, 5 op/s 2024-09-13T00:03:27.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:03:27 smithi136 bash[75164]: cluster 2024-09-13T00:03:25.209354+0000 mgr.y (mgr.24661) 789 : cluster [DBG] pgmap v1472: 105 pgs: 105 active+clean; 126 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 2.0 MiB/s rd, 3.1 MiB/s wr, 5 op/s 2024-09-13T00:03:27.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:03:27 smithi136 bash[76371]: cluster 2024-09-13T00:03:25.209354+0000 mgr.y (mgr.24661) 789 : cluster [DBG] pgmap v1472: 105 pgs: 105 active+clean; 126 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 2.0 MiB/s rd, 3.1 MiB/s wr, 5 op/s 2024-09-13T00:03:28.227 INFO:tasks.workunit.client.0.smithi136.stdout: 121: throughput=4.97MB/sec pending data=0 2024-09-13T00:03:28.227 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Y6EsK-E8x_Aanhk off=4086037 len=1764933 2024-09-13T00:03:28.232 INFO:tasks.workunit.client.0.smithi136.stdout:op 623 completed, throughput=4.98MB/sec 2024-09-13T00:03:28.233 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-TNoHWXuv_mFeSKo off=419508 len=1426496 2024-09-13T00:03:28.236 INFO:tasks.workunit.client.0.smithi136.stdout:op 624 completed, throughput=4.99MB/sec 2024-09-13T00:03:28.236 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-vEdpQl_DSnWl1o0 off=1717236 len=363304 2024-09-13T00:03:28.242 INFO:tasks.workunit.client.0.smithi136.stdout:op 625 completed, throughput=5MB/sec 2024-09-13T00:03:28.242 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Nj_v8s_2ikD-TSw off=464229 len=1684924 2024-09-13T00:03:28.253 INFO:tasks.workunit.client.0.smithi136.stdout:op 626 completed, throughput=5.01MB/sec 2024-09-13T00:03:28.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:03:28 smithi136 bash[75164]: cluster 2024-09-13T00:03:27.210459+0000 mgr.y (mgr.24661) 790 : cluster [DBG] pgmap v1473: 105 pgs: 105 active+clean; 126 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 2.1 MiB/s rd, 3.3 MiB/s wr, 5 op/s 2024-09-13T00:03:28.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:03:28 smithi136 bash[76371]: cluster 2024-09-13T00:03:27.210459+0000 mgr.y (mgr.24661) 790 : cluster [DBG] pgmap v1473: 105 pgs: 105 active+clean; 126 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 2.1 MiB/s rd, 3.3 MiB/s wr, 5 op/s 2024-09-13T00:03:28.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:03:28 smithi181 bash[67587]: cluster 2024-09-13T00:03:27.210459+0000 mgr.y (mgr.24661) 790 : cluster [DBG] pgmap v1473: 105 pgs: 105 active+clean; 126 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 2.1 MiB/s rd, 3.3 MiB/s wr, 5 op/s 2024-09-13T00:03:29.254 INFO:tasks.workunit.client.0.smithi136.stdout: 122: throughput=4.97MB/sec pending data=0 2024-09-13T00:03:29.254 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-t4XZ3aHacPD__kM off=10620065 len=1814013 2024-09-13T00:03:29.269 INFO:tasks.workunit.client.0.smithi136.stdout:op 627 completed, throughput=4.98MB/sec 2024-09-13T00:03:29.269 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-bDq9vo3aC_iGPOG off=6765637 len=778377 2024-09-13T00:03:29.272 INFO:tasks.workunit.client.0.smithi136.stdout:op 628 completed, throughput=4.99MB/sec 2024-09-13T00:03:29.272 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-j3NywYFwRXJKGX_ off=12651259 len=1184246 2024-09-13T00:03:29.275 INFO:tasks.workunit.client.0.smithi136.stdout:op 629 completed, throughput=5MB/sec 2024-09-13T00:03:29.275 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-9NHPepxNDEnercg off=275604 len=958585 2024-09-13T00:03:29.278 INFO:tasks.workunit.client.0.smithi136.stdout:op 630 completed, throughput=5MB/sec 2024-09-13T00:03:29.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:03:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:03:29] "GET /metrics HTTP/1.1" 200 34122 "" "Prometheus/2.43.0" 2024-09-13T00:03:30.279 INFO:tasks.workunit.client.0.smithi136.stdout: 123: throughput=4.96MB/sec pending data=0 2024-09-13T00:03:30.279 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-qm7uUUmhJaglt7R off=4421864 len=723766 2024-09-13T00:03:30.286 INFO:tasks.workunit.client.0.smithi136.stdout:op 631 completed, throughput=4.97MB/sec 2024-09-13T00:03:30.286 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-dziPHcz6OJ__BZe off=4755975 len=1125051 2024-09-13T00:03:30.290 INFO:tasks.workunit.client.0.smithi136.stdout:op 632 completed, throughput=4.98MB/sec 2024-09-13T00:03:30.290 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-iHA_Sd0mplbIr5v off=2052781 len=98736 2024-09-13T00:03:30.291 INFO:tasks.workunit.client.0.smithi136.stdout:op 633 completed, throughput=4.98MB/sec 2024-09-13T00:03:30.291 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-UMV41NQo0WgLk9E off=260760 len=499021 2024-09-13T00:03:30.297 INFO:tasks.workunit.client.0.smithi136.stdout:op 634 completed, throughput=4.98MB/sec 2024-09-13T00:03:30.297 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-METphTVlXOjynb3 off=13262779 len=261555 2024-09-13T00:03:30.298 INFO:tasks.workunit.client.0.smithi136.stdout:op 635 completed, throughput=4.98MB/sec 2024-09-13T00:03:30.299 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-T9CzvW3-EKqyH8i off=21474200 len=1054386 2024-09-13T00:03:30.309 INFO:tasks.workunit.client.0.smithi136.stdout:op 636 completed, throughput=4.99MB/sec 2024-09-13T00:03:30.309 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-TyBaQESJjbK9D60 off=5821821 len=1090874 2024-09-13T00:03:30.311 INFO:tasks.workunit.client.0.smithi136.stdout:op 637 completed, throughput=5MB/sec 2024-09-13T00:03:30.311 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-RatCT3z0B8Nct5U off=5045728 len=1869739 2024-09-13T00:03:30.315 INFO:tasks.workunit.client.0.smithi136.stdout:op 638 completed, throughput=5.01MB/sec 2024-09-13T00:03:30.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:03:30 smithi181 bash[67587]: cluster 2024-09-13T00:03:29.211418+0000 mgr.y (mgr.24661) 791 : cluster [DBG] pgmap v1474: 105 pgs: 105 active+clean; 126 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.0 MiB/s wr, 4 op/s 2024-09-13T00:03:30.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:03:30 smithi136 bash[75164]: cluster 2024-09-13T00:03:29.211418+0000 mgr.y (mgr.24661) 791 : cluster [DBG] pgmap v1474: 105 pgs: 105 active+clean; 126 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.0 MiB/s wr, 4 op/s 2024-09-13T00:03:30.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:03:30 smithi136 bash[76371]: cluster 2024-09-13T00:03:29.211418+0000 mgr.y (mgr.24661) 791 : cluster [DBG] pgmap v1474: 105 pgs: 105 active+clean; 126 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.0 MiB/s wr, 4 op/s 2024-09-13T00:03:31.315 INFO:tasks.workunit.client.0.smithi136.stdout: 124: throughput=4.97MB/sec pending data=0 2024-09-13T00:03:31.316 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_JVzH1awT2pfIAU off=2191410 len=379251 2024-09-13T00:03:31.322 INFO:tasks.workunit.client.0.smithi136.stdout:op 639 completed, throughput=4.98MB/sec 2024-09-13T00:03:31.322 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-w4uOjbErabIL4m2 off=15634556 len=105793 2024-09-13T00:03:31.323 INFO:tasks.workunit.client.0.smithi136.stdout:op 640 completed, throughput=4.98MB/sec 2024-09-13T00:03:31.323 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-xU9lbX6SoQv8VyW off=212379 len=1899656 2024-09-13T00:03:31.329 INFO:tasks.workunit.client.0.smithi136.stdout:op 641 completed, throughput=4.99MB/sec 2024-09-13T00:03:31.329 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-zzRoimrVdLyVfi4 off=1120949 len=293712 2024-09-13T00:03:31.331 INFO:tasks.workunit.client.0.smithi136.stdout:op 642 completed, throughput=4.99MB/sec 2024-09-13T00:03:31.332 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-E2r2MkHoNbPk4QH off=99321 len=799613 2024-09-13T00:03:31.338 INFO:tasks.workunit.client.0.smithi136.stdout:op 643 completed, throughput=5MB/sec 2024-09-13T00:03:31.338 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-y_gHdtPGHEKFaXh off=10250765 len=1674262 2024-09-13T00:03:31.351 INFO:tasks.workunit.client.0.smithi136.stdout:op 644 completed, throughput=5.01MB/sec 2024-09-13T00:03:32.351 INFO:tasks.workunit.client.0.smithi136.stdout: 125: throughput=4.97MB/sec pending data=0 2024-09-13T00:03:32.351 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-wuGkuEhBSvmXM1G off=12865880 len=1456899 2024-09-13T00:03:32.355 INFO:tasks.workunit.client.0.smithi136.stdout:op 645 completed, throughput=4.98MB/sec 2024-09-13T00:03:32.355 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-pKOEGForNqgRotR off=19222930 len=1070355 2024-09-13T00:03:32.363 INFO:tasks.workunit.client.0.smithi136.stdout:op 646 completed, throughput=4.99MB/sec 2024-09-13T00:03:32.363 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-c1epx15L0Dh2sPT off=6010232 len=1746635 2024-09-13T00:03:32.368 INFO:tasks.workunit.client.0.smithi136.stdout:op 647 completed, throughput=5MB/sec 2024-09-13T00:03:32.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:03:32 smithi181 bash[67587]: cluster 2024-09-13T00:03:31.212501+0000 mgr.y (mgr.24661) 792 : cluster [DBG] pgmap v1475: 105 pgs: 105 active+clean; 126 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.4 MiB/s wr, 4 op/s 2024-09-13T00:03:32.600 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:03:32 smithi181 bash[95505]: ts=2024-09-13T00:03:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:03:32.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:03:32 smithi136 bash[75164]: cluster 2024-09-13T00:03:31.212501+0000 mgr.y (mgr.24661) 792 : cluster [DBG] pgmap v1475: 105 pgs: 105 active+clean; 126 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.4 MiB/s wr, 4 op/s 2024-09-13T00:03:32.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:03:32 smithi136 bash[76371]: cluster 2024-09-13T00:03:31.212501+0000 mgr.y (mgr.24661) 792 : cluster [DBG] pgmap v1475: 105 pgs: 105 active+clean; 126 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.4 MiB/s wr, 4 op/s 2024-09-13T00:03:33.368 INFO:tasks.workunit.client.0.smithi136.stdout: 126: throughput=4.96MB/sec pending data=0 2024-09-13T00:03:33.369 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-z4PEqq_FsrcecRd off=3934086 len=723859 2024-09-13T00:03:33.376 INFO:tasks.workunit.client.0.smithi136.stdout:op 648 completed, throughput=4.97MB/sec 2024-09-13T00:03:33.377 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-JhVfwbDlgq7RtoV off=8291189 len=564247 2024-09-13T00:03:33.382 INFO:tasks.workunit.client.0.smithi136.stdout:op 649 completed, throughput=4.97MB/sec 2024-09-13T00:03:33.382 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-S4miOTEombnIwzB off=5151533 len=656782 2024-09-13T00:03:33.389 INFO:tasks.workunit.client.0.smithi136.stdout:op 650 completed, throughput=4.98MB/sec 2024-09-13T00:03:33.390 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-mtbdGCLq_PEvRPF off=4234129 len=24545 2024-09-13T00:03:33.390 INFO:tasks.workunit.client.0.smithi136.stdout:op 651 completed, throughput=4.98MB/sec 2024-09-13T00:03:33.390 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-jtN8TBdnerEDCXP off=20402922 len=507496 2024-09-13T00:03:33.397 INFO:tasks.workunit.client.0.smithi136.stdout:op 652 completed, throughput=4.98MB/sec 2024-09-13T00:03:33.397 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-YiJ5si0HsFL_GkB off=3352116 len=2073029 2024-09-13T00:03:33.403 INFO:tasks.workunit.client.0.smithi136.stdout:op 653 completed, throughput=5MB/sec 2024-09-13T00:03:33.403 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-C4orQ0DTweYS61_ off=3202814 len=866792 2024-09-13T00:03:33.411 INFO:tasks.workunit.client.0.smithi136.stdout:op 654 completed, throughput=5MB/sec 2024-09-13T00:03:34.412 INFO:tasks.workunit.client.0.smithi136.stdout: 127: throughput=4.96MB/sec pending data=0 2024-09-13T00:03:34.412 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-KmFeyEWNROlNzOg off=9075920 len=813655 2024-09-13T00:03:34.414 INFO:tasks.workunit.client.0.smithi136.stdout:op 655 completed, throughput=4.97MB/sec 2024-09-13T00:03:34.414 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-m49uKpcNHqzXGcC off=3923558 len=410913 2024-09-13T00:03:34.419 INFO:tasks.workunit.client.0.smithi136.stdout:op 656 completed, throughput=4.97MB/sec 2024-09-13T00:03:34.419 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-CmYnvDQq3OR0YpH off=8006137 len=936578 2024-09-13T00:03:34.422 INFO:tasks.workunit.client.0.smithi136.stdout:op 657 completed, throughput=4.98MB/sec 2024-09-13T00:03:34.422 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-DdNctNdLGdyAARQ off=21020251 len=1252160 2024-09-13T00:03:34.426 INFO:tasks.workunit.client.0.smithi136.stdout:op 658 completed, throughput=4.99MB/sec 2024-09-13T00:03:34.426 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-qAGyjMQm7LwtmzX off=297408 len=1716607 2024-09-13T00:03:34.432 INFO:tasks.workunit.client.0.smithi136.stdout:op 659 completed, throughput=5MB/sec 2024-09-13T00:03:35.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:03:34 smithi136 bash[75164]: cluster 2024-09-13T00:03:33.213617+0000 mgr.y (mgr.24661) 793 : cluster [DBG] pgmap v1476: 105 pgs: 105 active+clean; 126 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 1.9 MiB/s rd, 3.2 MiB/s wr, 4 op/s 2024-09-13T00:03:35.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:03:34 smithi136 bash[76371]: cluster 2024-09-13T00:03:33.213617+0000 mgr.y (mgr.24661) 793 : cluster [DBG] pgmap v1476: 105 pgs: 105 active+clean; 126 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 1.9 MiB/s rd, 3.2 MiB/s wr, 4 op/s 2024-09-13T00:03:35.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:03:34 smithi181 bash[67587]: cluster 2024-09-13T00:03:33.213617+0000 mgr.y (mgr.24661) 793 : cluster [DBG] pgmap v1476: 105 pgs: 105 active+clean; 126 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 1.9 MiB/s rd, 3.2 MiB/s wr, 4 op/s 2024-09-13T00:03:35.433 INFO:tasks.workunit.client.0.smithi136.stdout: 128: throughput=4.96MB/sec pending data=0 2024-09-13T00:03:35.433 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-uwiyCscWw19jP3H off=10349762 len=613760 2024-09-13T00:03:35.436 INFO:tasks.workunit.client.0.smithi136.stdout:op 660 completed, throughput=4.97MB/sec 2024-09-13T00:03:35.436 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-MIzSBBVrQos1voV off=918175 len=2080587 2024-09-13T00:03:35.441 INFO:tasks.workunit.client.0.smithi136.stdout:op 661 completed, throughput=4.98MB/sec 2024-09-13T00:03:35.441 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-dEtkatSmZTDA9Ut off=5481226 len=82857 2024-09-13T00:03:35.445 INFO:tasks.workunit.client.0.smithi136.stdout:op 662 completed, throughput=4.98MB/sec 2024-09-13T00:03:35.445 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj--ybhl_sz9tpVK9y off=4888131 len=759971 2024-09-13T00:03:35.447 INFO:tasks.workunit.client.0.smithi136.stdout:op 663 completed, throughput=4.99MB/sec 2024-09-13T00:03:35.448 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj--gJKkYzh5DJauZv off=2096796 len=1607445 2024-09-13T00:03:35.452 INFO:tasks.workunit.client.0.smithi136.stdout:op 664 completed, throughput=5MB/sec 2024-09-13T00:03:35.452 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-wQW_ar52BwLiRCO off=1371400 len=686384 2024-09-13T00:03:35.459 INFO:tasks.workunit.client.0.smithi136.stdout:op 665 completed, throughput=5MB/sec 2024-09-13T00:03:36.459 INFO:tasks.workunit.client.0.smithi136.stdout: 129: throughput=4.97MB/sec pending data=0 2024-09-13T00:03:36.459 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-J5y2K5WIzOMX-Ua off=18630317 len=2068579 2024-09-13T00:03:36.464 INFO:tasks.workunit.client.0.smithi136.stdout:op 666 completed, throughput=4.98MB/sec 2024-09-13T00:03:36.464 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj--pinnbQMkGjJIxt off=6921320 len=1784769 2024-09-13T00:03:36.468 INFO:tasks.workunit.client.0.smithi136.stdout:op 667 completed, throughput=4.99MB/sec 2024-09-13T00:03:36.468 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-vnwtfpa4Cp3xL5V off=0 len=1699682 2024-09-13T00:03:36.480 INFO:tasks.workunit.client.0.smithi136.stdout:op 668 completed, throughput=5.01MB/sec 2024-09-13T00:03:37.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:03:36 smithi136 bash[75164]: cluster 2024-09-13T00:03:35.214533+0000 mgr.y (mgr.24661) 794 : cluster [DBG] pgmap v1477: 105 pgs: 105 active+clean; 126 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 2.2 MiB/s rd, 3.1 MiB/s wr, 4 op/s 2024-09-13T00:03:37.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:03:36 smithi136 bash[76371]: cluster 2024-09-13T00:03:35.214533+0000 mgr.y (mgr.24661) 794 : cluster [DBG] pgmap v1477: 105 pgs: 105 active+clean; 126 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 2.2 MiB/s rd, 3.1 MiB/s wr, 4 op/s 2024-09-13T00:03:37.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:03:36 smithi181 bash[67587]: cluster 2024-09-13T00:03:35.214533+0000 mgr.y (mgr.24661) 794 : cluster [DBG] pgmap v1477: 105 pgs: 105 active+clean; 126 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 2.2 MiB/s rd, 3.1 MiB/s wr, 4 op/s 2024-09-13T00:03:37.480 INFO:tasks.workunit.client.0.smithi136.stdout: 130: throughput=4.97MB/sec pending data=0 2024-09-13T00:03:37.485 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-3fDP4hu4S1sXmfk off=14770942 len=2030992 2024-09-13T00:03:37.498 INFO:tasks.workunit.client.0.smithi136.stdout:op 669 completed, throughput=4.98MB/sec 2024-09-13T00:03:37.498 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-dDcsgBHq-ck0KEM off=19726302 len=2031288 2024-09-13T00:03:37.510 INFO:tasks.workunit.client.0.smithi136.stdout:op 670 completed, throughput=5MB/sec 2024-09-13T00:03:37.510 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-6_RI1dWMf4kOSBf off=5680900 len=673565 2024-09-13T00:03:37.517 INFO:tasks.workunit.client.0.smithi136.stdout:op 671 completed, throughput=5MB/sec 2024-09-13T00:03:38.517 INFO:tasks.workunit.client.0.smithi136.stdout: 131: throughput=4.96MB/sec pending data=0 2024-09-13T00:03:38.518 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-JlP65l45jEFebyE off=4821227 len=1966099 2024-09-13T00:03:38.522 INFO:tasks.workunit.client.0.smithi136.stdout:op 672 completed, throughput=4.98MB/sec 2024-09-13T00:03:38.522 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ww-vtytHgyidSV- off=4919793 len=113286 2024-09-13T00:03:38.523 INFO:tasks.workunit.client.0.smithi136.stdout:op 673 completed, throughput=4.98MB/sec 2024-09-13T00:03:38.523 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_nfArrBMeoWhwRh off=18996841 len=370343 2024-09-13T00:03:38.529 INFO:tasks.workunit.client.0.smithi136.stdout:op 674 completed, throughput=4.98MB/sec 2024-09-13T00:03:38.529 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-siV131WWOdPcqvl off=5139064 len=1943496 2024-09-13T00:03:38.533 INFO:tasks.workunit.client.0.smithi136.stdout:op 675 completed, throughput=4.99MB/sec 2024-09-13T00:03:38.533 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-zC65U9GSCNgtE0a off=273395 len=644259 2024-09-13T00:03:38.535 INFO:tasks.workunit.client.0.smithi136.stdout:op 676 completed, throughput=5MB/sec 2024-09-13T00:03:38.536 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-IoqHS4cLmRsmOWD off=9369229 len=443166 2024-09-13T00:03:38.537 INFO:tasks.workunit.client.0.smithi136.stdout:op 677 completed, throughput=5MB/sec 2024-09-13T00:03:39.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:03:38 smithi136 bash[75164]: cluster 2024-09-13T00:03:37.215687+0000 mgr.y (mgr.24661) 795 : cluster [DBG] pgmap v1478: 105 pgs: 105 active+clean; 126 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:03:39.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:03:38 smithi136 bash[76371]: cluster 2024-09-13T00:03:37.215687+0000 mgr.y (mgr.24661) 795 : cluster [DBG] pgmap v1478: 105 pgs: 105 active+clean; 126 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:03:39.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:03:38 smithi181 bash[67587]: cluster 2024-09-13T00:03:37.215687+0000 mgr.y (mgr.24661) 795 : cluster [DBG] pgmap v1478: 105 pgs: 105 active+clean; 126 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:03:39.537 INFO:tasks.workunit.client.0.smithi136.stdout: 132: throughput=4.97MB/sec pending data=0 2024-09-13T00:03:39.538 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-63FBoIJHze1WZy8 off=11206331 len=855193 2024-09-13T00:03:39.540 INFO:tasks.workunit.client.0.smithi136.stdout:op 678 completed, throughput=4.97MB/sec 2024-09-13T00:03:39.540 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-UOgGES4Vql8kd3B off=1336628 len=478934 2024-09-13T00:03:39.542 INFO:tasks.workunit.client.0.smithi136.stdout:op 679 completed, throughput=4.97MB/sec 2024-09-13T00:03:39.543 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-IJ2XvGNi7yRrbFH off=2317164 len=1605445 2024-09-13T00:03:39.546 INFO:tasks.workunit.client.0.smithi136.stdout:op 680 completed, throughput=4.99MB/sec 2024-09-13T00:03:39.546 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-UOgGES4Vql8kd3B off=10822352 len=981106 2024-09-13T00:03:39.550 INFO:tasks.workunit.client.0.smithi136.stdout:op 681 completed, throughput=4.99MB/sec 2024-09-13T00:03:39.550 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-R63_ooH3F2YiK0v off=1506290 len=1216679 2024-09-13T00:03:39.559 INFO:tasks.workunit.client.0.smithi136.stdout:op 682 completed, throughput=5MB/sec 2024-09-13T00:03:39.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:03:39 smithi136 bash[75164]: audit 2024-09-13T00:03:38.774843+0000 mon.a (mon.0) 2125 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:03:39.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:03:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:03:39] "GET /metrics HTTP/1.1" 200 34121 "" "Prometheus/2.43.0" 2024-09-13T00:03:39.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:03:39 smithi136 bash[76371]: audit 2024-09-13T00:03:38.774843+0000 mon.a (mon.0) 2125 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:03:40.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:03:39 smithi181 bash[67587]: audit 2024-09-13T00:03:38.774843+0000 mon.a (mon.0) 2125 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:03:40.559 INFO:tasks.workunit.client.0.smithi136.stdout: 133: throughput=4.96MB/sec pending data=0 2024-09-13T00:03:40.559 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2bh14RyU2xxcJrN off=9193375 len=2013945 2024-09-13T00:03:40.563 INFO:tasks.workunit.client.0.smithi136.stdout:op 683 completed, throughput=4.98MB/sec 2024-09-13T00:03:40.563 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-dnbt96cQHyTgi4L off=6214551 len=1533136 2024-09-13T00:03:40.567 INFO:tasks.workunit.client.0.smithi136.stdout:op 684 completed, throughput=4.99MB/sec 2024-09-13T00:03:40.568 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-tWzJm7cKq_M7CR7 off=1615446 len=455356 2024-09-13T00:03:40.570 INFO:tasks.workunit.client.0.smithi136.stdout:op 685 completed, throughput=4.99MB/sec 2024-09-13T00:03:40.570 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-wg_kxB3Pm1cS7lH off=1987165 len=1189809 2024-09-13T00:03:40.580 INFO:tasks.workunit.client.0.smithi136.stdout:op 686 completed, throughput=5MB/sec 2024-09-13T00:03:40.580 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-jDjkLxqRnk0MQt7 off=3151221 len=1722629 2024-09-13T00:03:40.591 INFO:tasks.workunit.client.0.smithi136.stdout:op 687 completed, throughput=5.01MB/sec 2024-09-13T00:03:41.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:03:40 smithi136 bash[75164]: cluster 2024-09-13T00:03:39.216562+0000 mgr.y (mgr.24661) 796 : cluster [DBG] pgmap v1479: 105 pgs: 105 active+clean; 126 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:03:41.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:03:40 smithi136 bash[76371]: cluster 2024-09-13T00:03:39.216562+0000 mgr.y (mgr.24661) 796 : cluster [DBG] pgmap v1479: 105 pgs: 105 active+clean; 126 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:03:41.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:03:40 smithi181 bash[67587]: cluster 2024-09-13T00:03:39.216562+0000 mgr.y (mgr.24661) 796 : cluster [DBG] pgmap v1479: 105 pgs: 105 active+clean; 126 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:03:41.592 INFO:tasks.workunit.client.0.smithi136.stdout: 134: throughput=4.97MB/sec pending data=0 2024-09-13T00:03:41.592 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-rbyS-pKvKxZ8ZSR off=387279 len=2033168 2024-09-13T00:03:41.605 INFO:tasks.workunit.client.0.smithi136.stdout:op 688 completed, throughput=4.99MB/sec 2024-09-13T00:03:41.605 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-KryyLW33Q3E1ofD off=2179391 len=502409 2024-09-13T00:03:41.606 INFO:tasks.workunit.client.0.smithi136.stdout:op 689 completed, throughput=4.99MB/sec 2024-09-13T00:03:41.607 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Z3yG_oxe4oM9EJR off=8251741 len=643507 2024-09-13T00:03:41.609 INFO:tasks.workunit.client.0.smithi136.stdout:op 690 completed, throughput=5MB/sec 2024-09-13T00:03:41.609 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-C6m44DVZXySY-0S off=3514470 len=1134226 2024-09-13T00:03:41.613 INFO:tasks.workunit.client.0.smithi136.stdout:op 691 completed, throughput=5MB/sec 2024-09-13T00:03:42.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:03:42 smithi181 bash[95505]: ts=2024-09-13T00:03:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:03:42.613 INFO:tasks.workunit.client.0.smithi136.stdout: 135: throughput=4.97MB/sec pending data=0 2024-09-13T00:03:42.613 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-bJ4A-DceRidEAJP off=442974 len=1367371 2024-09-13T00:03:42.622 INFO:tasks.workunit.client.0.smithi136.stdout:op 692 completed, throughput=4.98MB/sec 2024-09-13T00:03:42.622 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-wMEgMpaN16S0xC3 off=14345296 len=724707 2024-09-13T00:03:42.623 INFO:tasks.workunit.client.0.smithi136.stdout:op 693 completed, throughput=4.98MB/sec 2024-09-13T00:03:42.623 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_G3cJwDsF3UJ5br off=23136005 len=1575591 2024-09-13T00:03:42.633 INFO:tasks.workunit.client.0.smithi136.stdout:op 694 completed, throughput=4.99MB/sec 2024-09-13T00:03:42.633 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-q8oE3G1-Vyb8XZM off=3078411 len=1293938 2024-09-13T00:03:42.639 INFO:tasks.workunit.client.0.smithi136.stdout:op 695 completed, throughput=5MB/sec 2024-09-13T00:03:43.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:03:42 smithi136 bash[75164]: cluster 2024-09-13T00:03:41.217573+0000 mgr.y (mgr.24661) 797 : cluster [DBG] pgmap v1480: 105 pgs: 105 active+clean; 126 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:03:43.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:03:42 smithi136 bash[75164]: audit 2024-09-13T00:03:42.005975+0000 mon.a (mon.0) 2126 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:03:43.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:03:42 smithi136 bash[75164]: audit 2024-09-13T00:03:42.548474+0000 mon.a (mon.0) 2127 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:03:43.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:03:42 smithi136 bash[75164]: audit 2024-09-13T00:03:42.549984+0000 mon.a (mon.0) 2128 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:03:43.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:03:42 smithi136 bash[75164]: audit 2024-09-13T00:03:42.561955+0000 mon.a (mon.0) 2129 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:03:43.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:03:42 smithi136 bash[76371]: cluster 2024-09-13T00:03:41.217573+0000 mgr.y (mgr.24661) 797 : cluster [DBG] pgmap v1480: 105 pgs: 105 active+clean; 126 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:03:43.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:03:42 smithi136 bash[76371]: audit 2024-09-13T00:03:42.005975+0000 mon.a (mon.0) 2126 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:03:43.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:03:42 smithi136 bash[76371]: audit 2024-09-13T00:03:42.548474+0000 mon.a (mon.0) 2127 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:03:43.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:03:42 smithi136 bash[76371]: audit 2024-09-13T00:03:42.549984+0000 mon.a (mon.0) 2128 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:03:43.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:03:42 smithi136 bash[76371]: audit 2024-09-13T00:03:42.561955+0000 mon.a (mon.0) 2129 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:03:43.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:03:42 smithi181 bash[67587]: cluster 2024-09-13T00:03:41.217573+0000 mgr.y (mgr.24661) 797 : cluster [DBG] pgmap v1480: 105 pgs: 105 active+clean; 126 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:03:43.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:03:42 smithi181 bash[67587]: audit 2024-09-13T00:03:42.005975+0000 mon.a (mon.0) 2126 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:03:43.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:03:42 smithi181 bash[67587]: audit 2024-09-13T00:03:42.548474+0000 mon.a (mon.0) 2127 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:03:43.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:03:42 smithi181 bash[67587]: audit 2024-09-13T00:03:42.549984+0000 mon.a (mon.0) 2128 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:03:43.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:03:42 smithi181 bash[67587]: audit 2024-09-13T00:03:42.561955+0000 mon.a (mon.0) 2129 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:03:43.639 INFO:tasks.workunit.client.0.smithi136.stdout: 136: throughput=4.96MB/sec pending data=0 2024-09-13T00:03:43.640 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-O0WEOEyLw7roJzv off=16494947 len=1587294 2024-09-13T00:03:43.652 INFO:tasks.workunit.client.0.smithi136.stdout:op 696 completed, throughput=4.98MB/sec 2024-09-13T00:03:43.652 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-cOLFGf0ll2i8NIU off=1507336 len=297032 2024-09-13T00:03:43.657 INFO:tasks.workunit.client.0.smithi136.stdout:op 697 completed, throughput=4.98MB/sec 2024-09-13T00:03:43.657 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-nL-GwDGCW4RXAXk off=5339544 len=489439 2024-09-13T00:03:43.659 INFO:tasks.workunit.client.0.smithi136.stdout:op 698 completed, throughput=4.98MB/sec 2024-09-13T00:03:43.659 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-3LMBFSJYWpEqKML off=12776587 len=648577 2024-09-13T00:03:43.661 INFO:tasks.workunit.client.0.smithi136.stdout:op 699 completed, throughput=4.98MB/sec 2024-09-13T00:03:43.661 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-czBHIa0OnpdzkvT off=71089 len=603330 2024-09-13T00:03:43.663 INFO:tasks.workunit.client.0.smithi136.stdout:op 700 completed, throughput=4.99MB/sec 2024-09-13T00:03:43.663 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-T9A1rn9KXQyPLNi off=10380163 len=1902724 2024-09-13T00:03:43.668 INFO:tasks.workunit.client.0.smithi136.stdout:op 701 completed, throughput=5MB/sec 2024-09-13T00:03:44.668 INFO:tasks.workunit.client.0.smithi136.stdout: 137: throughput=4.97MB/sec pending data=0 2024-09-13T00:03:44.668 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-IAM9OKldQBwUSZD off=4115263 len=1505125 2024-09-13T00:03:44.672 INFO:tasks.workunit.client.0.smithi136.stdout:op 702 completed, throughput=4.98MB/sec 2024-09-13T00:03:44.672 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-RVSpobPtxPJn_x2 off=22382616 len=109320 2024-09-13T00:03:44.673 INFO:tasks.workunit.client.0.smithi136.stdout:op 703 completed, throughput=4.98MB/sec 2024-09-13T00:03:44.673 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Iq_-6MHuwQKObkV off=18389303 len=1366326 2024-09-13T00:03:44.682 INFO:tasks.workunit.client.0.smithi136.stdout:op 704 completed, throughput=4.99MB/sec 2024-09-13T00:03:44.682 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-HKmmY_xsIiNa6CY off=1793589 len=1828043 2024-09-13T00:03:44.698 INFO:tasks.workunit.client.0.smithi136.stdout:op 705 completed, throughput=5MB/sec 2024-09-13T00:03:44.698 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-9Nlv0al6Es0b-Ss off=7694868 len=1981435 2024-09-13T00:03:44.700 INFO:tasks.workunit.client.0.smithi136.stdout:op 706 completed, throughput=5.01MB/sec 2024-09-13T00:03:45.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:03:44 smithi136 bash[75164]: cluster 2024-09-13T00:03:43.218473+0000 mgr.y (mgr.24661) 798 : cluster [DBG] pgmap v1481: 105 pgs: 105 active+clean; 126 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.9 MiB/s wr, 4 op/s 2024-09-13T00:03:45.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:03:44 smithi136 bash[76371]: cluster 2024-09-13T00:03:43.218473+0000 mgr.y (mgr.24661) 798 : cluster [DBG] pgmap v1481: 105 pgs: 105 active+clean; 126 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.9 MiB/s wr, 4 op/s 2024-09-13T00:03:45.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:03:44 smithi181 bash[67587]: cluster 2024-09-13T00:03:43.218473+0000 mgr.y (mgr.24661) 798 : cluster [DBG] pgmap v1481: 105 pgs: 105 active+clean; 126 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.9 MiB/s wr, 4 op/s 2024-09-13T00:03:45.701 INFO:tasks.workunit.client.0.smithi136.stdout: 138: throughput=4.98MB/sec pending data=0 2024-09-13T00:03:45.701 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_FEDz88YYMI9DgT off=11201744 len=735533 2024-09-13T00:03:45.708 INFO:tasks.workunit.client.0.smithi136.stdout:op 707 completed, throughput=4.98MB/sec 2024-09-13T00:03:45.708 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-cyMzC0Noeclwpn2 off=1050561 len=485940 2024-09-13T00:03:45.715 INFO:tasks.workunit.client.0.smithi136.stdout:op 708 completed, throughput=4.98MB/sec 2024-09-13T00:03:45.715 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj--R9osdA1vO89gsa off=12500456 len=503720 2024-09-13T00:03:45.721 INFO:tasks.workunit.client.0.smithi136.stdout:op 709 completed, throughput=4.99MB/sec 2024-09-13T00:03:45.721 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Ce6ubgl4URioDUT off=9099451 len=953141 2024-09-13T00:03:45.728 INFO:tasks.workunit.client.0.smithi136.stdout:op 710 completed, throughput=4.99MB/sec 2024-09-13T00:03:45.728 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-2v-YG_BzTLu2fws off=5304277 len=605695 2024-09-13T00:03:45.735 INFO:tasks.workunit.client.0.smithi136.stdout:op 711 completed, throughput=5MB/sec 2024-09-13T00:03:45.735 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-M5eDAUWQvIgsN7m off=8601795 len=1731811 2024-09-13T00:03:45.739 INFO:tasks.workunit.client.0.smithi136.stdout:op 712 completed, throughput=5.01MB/sec 2024-09-13T00:03:46.739 INFO:tasks.workunit.client.0.smithi136.stdout: 139: throughput=4.97MB/sec pending data=0 2024-09-13T00:03:46.739 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-h1dm3XGd8LJmGBT off=1606997 len=1243615 2024-09-13T00:03:46.750 INFO:tasks.workunit.client.0.smithi136.stdout:op 713 completed, throughput=4.98MB/sec 2024-09-13T00:03:46.750 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-HtClZe920waRtDg off=4599688 len=500886 2024-09-13T00:03:46.756 INFO:tasks.workunit.client.0.smithi136.stdout:op 714 completed, throughput=4.98MB/sec 2024-09-13T00:03:46.757 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-rKBHBPSFELrb_nN off=6253133 len=71483 2024-09-13T00:03:46.761 INFO:tasks.workunit.client.0.smithi136.stdout:op 715 completed, throughput=4.98MB/sec 2024-09-13T00:03:46.761 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ks6GekkzpTKFFbY off=6221228 len=294473 2024-09-13T00:03:46.762 INFO:tasks.workunit.client.0.smithi136.stdout:op 716 completed, throughput=4.99MB/sec 2024-09-13T00:03:46.762 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-BjpWSwhqN4Q46GK off=3891353 len=13817 2024-09-13T00:03:46.763 INFO:tasks.workunit.client.0.smithi136.stdout:op 717 completed, throughput=4.99MB/sec 2024-09-13T00:03:46.763 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-GYFytrp1QMxPNRJ off=12290564 len=1470267 2024-09-13T00:03:46.768 INFO:tasks.workunit.client.0.smithi136.stdout:op 718 completed, throughput=5MB/sec 2024-09-13T00:03:46.768 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj--iGeLr323OUEbCf off=6201049 len=743428 2024-09-13T00:03:46.770 INFO:tasks.workunit.client.0.smithi136.stdout:op 719 completed, throughput=5MB/sec 2024-09-13T00:03:47.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:03:46 smithi136 bash[75164]: cluster 2024-09-13T00:03:45.219314+0000 mgr.y (mgr.24661) 799 : cluster [DBG] pgmap v1482: 105 pgs: 105 active+clean; 126 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 3.0 MiB/s rd, 1.9 MiB/s wr, 4 op/s 2024-09-13T00:03:47.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:03:46 smithi136 bash[76371]: cluster 2024-09-13T00:03:45.219314+0000 mgr.y (mgr.24661) 799 : cluster [DBG] pgmap v1482: 105 pgs: 105 active+clean; 126 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 3.0 MiB/s rd, 1.9 MiB/s wr, 4 op/s 2024-09-13T00:03:47.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:03:46 smithi181 bash[67587]: cluster 2024-09-13T00:03:45.219314+0000 mgr.y (mgr.24661) 799 : cluster [DBG] pgmap v1482: 105 pgs: 105 active+clean; 126 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 3.0 MiB/s rd, 1.9 MiB/s wr, 4 op/s 2024-09-13T00:03:47.770 INFO:tasks.workunit.client.0.smithi136.stdout: 140: throughput=4.97MB/sec pending data=0 2024-09-13T00:03:47.771 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-UMd6sbzD1ptvP2s off=7531285 len=219931 2024-09-13T00:03:47.775 INFO:tasks.workunit.client.0.smithi136.stdout:op 720 completed, throughput=4.97MB/sec 2024-09-13T00:03:47.776 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-zbb2zLdwIIiEJ11 off=340291 len=802610 2024-09-13T00:03:47.783 INFO:tasks.workunit.client.0.smithi136.stdout:op 721 completed, throughput=4.97MB/sec 2024-09-13T00:03:47.783 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-LDax_HCILzsbvA7 off=5945759 len=493317 2024-09-13T00:03:47.789 INFO:tasks.workunit.client.0.smithi136.stdout:op 722 completed, throughput=4.98MB/sec 2024-09-13T00:03:47.789 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-DJU4JRnOKl-JFqp off=4470986 len=519762 2024-09-13T00:03:47.797 INFO:tasks.workunit.client.0.smithi136.stdout:op 723 completed, throughput=4.98MB/sec 2024-09-13T00:03:47.797 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-l31DvrQB7QxfMeO off=3807051 len=959897 2024-09-13T00:03:47.800 INFO:tasks.workunit.client.0.smithi136.stdout:op 724 completed, throughput=4.98MB/sec 2024-09-13T00:03:47.800 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-KqmPbZbN4kvfLG7 off=9569899 len=804030 2024-09-13T00:03:47.802 INFO:tasks.workunit.client.0.smithi136.stdout:op 725 completed, throughput=4.99MB/sec 2024-09-13T00:03:47.802 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-_QOQuDW6PWFfct- off=4218203 len=1642528 2024-09-13T00:03:47.805 INFO:tasks.workunit.client.0.smithi136.stdout:op 726 completed, throughput=5MB/sec 2024-09-13T00:03:48.806 INFO:tasks.workunit.client.0.smithi136.stdout: 141: throughput=4.97MB/sec pending data=0 2024-09-13T00:03:48.806 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-kKDLSlDDS5dkuQW off=3688216 len=114799 2024-09-13T00:03:48.807 INFO:tasks.workunit.client.0.smithi136.stdout:op 727 completed, throughput=4.97MB/sec 2024-09-13T00:03:48.807 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-zEQeWH5-amY9L67 off=965510 len=617766 2024-09-13T00:03:48.809 INFO:tasks.workunit.client.0.smithi136.stdout:op 728 completed, throughput=4.97MB/sec 2024-09-13T00:03:48.809 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ybUZyofNH6Ib_vC off=139337 len=942104 2024-09-13T00:03:48.819 INFO:tasks.workunit.client.0.smithi136.stdout:op 729 completed, throughput=4.98MB/sec 2024-09-13T00:03:48.819 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-EkBUj9K3exLWEdE off=2680505 len=1027077 2024-09-13T00:03:48.828 INFO:tasks.workunit.client.0.smithi136.stdout:op 730 completed, throughput=4.98MB/sec 2024-09-13T00:03:48.828 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-iLz_0wvCXrhx8Z2 off=3903524 len=448760 2024-09-13T00:03:48.834 INFO:tasks.workunit.client.0.smithi136.stdout:op 731 completed, throughput=4.99MB/sec 2024-09-13T00:03:48.834 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-LJ3v0hvHylsV6wY off=784471 len=435200 2024-09-13T00:03:48.836 INFO:tasks.workunit.client.0.smithi136.stdout:op 732 completed, throughput=4.99MB/sec 2024-09-13T00:03:48.836 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-qfL_hyUMjYGaP8M off=5193281 len=1029359 2024-09-13T00:03:48.839 INFO:tasks.workunit.client.0.smithi136.stdout:op 733 completed, throughput=5MB/sec 2024-09-13T00:03:48.839 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-FP5-2zZJvgT-bt6 off=42249 len=1295850 2024-09-13T00:03:48.849 INFO:tasks.workunit.client.0.smithi136.stdout:op 734 completed, throughput=5MB/sec 2024-09-13T00:03:49.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:03:48 smithi136 bash[75164]: cluster 2024-09-13T00:03:47.220417+0000 mgr.y (mgr.24661) 800 : cluster [DBG] pgmap v1483: 105 pgs: 105 active+clean; 126 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:03:49.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:03:48 smithi136 bash[76371]: cluster 2024-09-13T00:03:47.220417+0000 mgr.y (mgr.24661) 800 : cluster [DBG] pgmap v1483: 105 pgs: 105 active+clean; 126 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:03:49.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:03:48 smithi181 bash[67587]: cluster 2024-09-13T00:03:47.220417+0000 mgr.y (mgr.24661) 800 : cluster [DBG] pgmap v1483: 105 pgs: 105 active+clean; 126 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:03:49.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:03:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:03:49] "GET /metrics HTTP/1.1" 200 34121 "" "Prometheus/2.43.0" 2024-09-13T00:03:49.849 INFO:tasks.workunit.client.0.smithi136.stdout: 142: throughput=4.97MB/sec pending data=0 2024-09-13T00:03:49.849 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-KQtqdpz8Mvn05SF off=1597154 len=1693156 2024-09-13T00:03:49.863 INFO:tasks.workunit.client.0.smithi136.stdout:op 735 completed, throughput=4.98MB/sec 2024-09-13T00:03:49.864 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-_j97ouhKUT93ZwN off=7017125 len=730474 2024-09-13T00:03:49.867 INFO:tasks.workunit.client.0.smithi136.stdout:op 736 completed, throughput=4.98MB/sec 2024-09-13T00:03:49.867 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-iWGlQHL7biT8uP5 off=15867972 len=517078 2024-09-13T00:03:49.869 INFO:tasks.workunit.client.0.smithi136.stdout:op 737 completed, throughput=4.99MB/sec 2024-09-13T00:03:49.869 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-G3DRCOIg_RCCb60 off=9645759 len=1555506 2024-09-13T00:03:49.880 INFO:tasks.workunit.client.0.smithi136.stdout:op 738 completed, throughput=5MB/sec 2024-09-13T00:03:49.880 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-YLGORZiOzuFZVfq off=14805611 len=879443 2024-09-13T00:03:49.887 INFO:tasks.workunit.client.0.smithi136.stdout:op 739 completed, throughput=5MB/sec 2024-09-13T00:03:50.888 INFO:tasks.workunit.client.0.smithi136.stdout: 143: throughput=4.97MB/sec pending data=0 2024-09-13T00:03:50.888 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-RpNEH9Gaw7A4Pdp off=3077657 len=54075 2024-09-13T00:03:50.892 INFO:tasks.workunit.client.0.smithi136.stdout:op 740 completed, throughput=4.97MB/sec 2024-09-13T00:03:50.892 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-q7vawnmlR_1aMbi off=1101386 len=664767 2024-09-13T00:03:50.894 INFO:tasks.workunit.client.0.smithi136.stdout:op 741 completed, throughput=4.97MB/sec 2024-09-13T00:03:50.894 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-DQUpl3WFX8V8RjH off=6834351 len=150042 2024-09-13T00:03:50.895 INFO:tasks.workunit.client.0.smithi136.stdout:op 742 completed, throughput=4.97MB/sec 2024-09-13T00:03:50.895 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-BIn2nGkc7Yvxk66 off=13672159 len=226574 2024-09-13T00:03:50.897 INFO:tasks.workunit.client.0.smithi136.stdout:op 743 completed, throughput=4.98MB/sec 2024-09-13T00:03:50.897 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-kPaqi0H6vC-_I5h off=782720 len=810724 2024-09-13T00:03:50.899 INFO:tasks.workunit.client.0.smithi136.stdout:op 744 completed, throughput=4.98MB/sec 2024-09-13T00:03:50.899 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-XvRdSRZFFnl4JG1 off=13871081 len=60509 2024-09-13T00:03:50.900 INFO:tasks.workunit.client.0.smithi136.stdout:op 745 completed, throughput=4.98MB/sec 2024-09-13T00:03:50.901 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-u7_DNYy4k7BB48d off=16943864 len=75978 2024-09-13T00:03:50.901 INFO:tasks.workunit.client.0.smithi136.stdout:op 746 completed, throughput=4.98MB/sec 2024-09-13T00:03:50.901 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-4atbjGKIoTAkEXk off=1741089 len=1449113 2024-09-13T00:03:50.905 INFO:tasks.workunit.client.0.smithi136.stdout:op 747 completed, throughput=4.99MB/sec 2024-09-13T00:03:50.905 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-w9GhpNu37jJwjXM off=18389 len=1223887 2024-09-13T00:03:50.915 INFO:tasks.workunit.client.0.smithi136.stdout:op 748 completed, throughput=5MB/sec 2024-09-13T00:03:50.916 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-6W2iMvUyvjLlduU off=3080421 len=2014249 2024-09-13T00:03:50.929 INFO:tasks.workunit.client.0.smithi136.stdout:op 749 completed, throughput=5.01MB/sec 2024-09-13T00:03:51.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:03:50 smithi136 bash[75164]: cluster 2024-09-13T00:03:49.221319+0000 mgr.y (mgr.24661) 801 : cluster [DBG] pgmap v1484: 105 pgs: 105 active+clean; 126 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:03:51.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:03:50 smithi136 bash[76371]: cluster 2024-09-13T00:03:49.221319+0000 mgr.y (mgr.24661) 801 : cluster [DBG] pgmap v1484: 105 pgs: 105 active+clean; 126 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:03:51.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:03:50 smithi181 bash[67587]: cluster 2024-09-13T00:03:49.221319+0000 mgr.y (mgr.24661) 801 : cluster [DBG] pgmap v1484: 105 pgs: 105 active+clean; 126 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:03:51.929 INFO:tasks.workunit.client.0.smithi136.stdout: 144: throughput=4.98MB/sec pending data=0 2024-09-13T00:03:51.929 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-1lPjdUdX8Ez726P off=11834303 len=1115397 2024-09-13T00:03:51.938 INFO:tasks.workunit.client.0.smithi136.stdout:op 750 completed, throughput=4.98MB/sec 2024-09-13T00:03:51.938 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-IcHRIiHmjInDuQI off=1318325 len=209696 2024-09-13T00:03:51.943 INFO:tasks.workunit.client.0.smithi136.stdout:op 751 completed, throughput=4.99MB/sec 2024-09-13T00:03:51.943 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-juIi78EIIkXHiKG off=736601 len=1538425 2024-09-13T00:03:51.948 INFO:tasks.workunit.client.0.smithi136.stdout:op 752 completed, throughput=5MB/sec 2024-09-13T00:03:51.948 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-R6BxEDXrb6FgHZI off=1162790 len=979543 2024-09-13T00:03:51.956 INFO:tasks.workunit.client.0.smithi136.stdout:op 753 completed, throughput=5MB/sec 2024-09-13T00:03:52.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:03:52 smithi181 bash[95505]: ts=2024-09-13T00:03:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:03:52.957 INFO:tasks.workunit.client.0.smithi136.stdout: 145: throughput=4.97MB/sec pending data=0 2024-09-13T00:03:52.957 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-sUVk6_apeIkb5QD off=8725625 len=579736 2024-09-13T00:03:52.963 INFO:tasks.workunit.client.0.smithi136.stdout:op 754 completed, throughput=4.97MB/sec 2024-09-13T00:03:52.963 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-7URYUeUJKE2wH9G off=363258 len=69781 2024-09-13T00:03:52.968 INFO:tasks.workunit.client.0.smithi136.stdout:op 755 completed, throughput=4.97MB/sec 2024-09-13T00:03:52.968 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-wp0oR6hQ4G7pnAM off=4629641 len=1461086 2024-09-13T00:03:52.972 INFO:tasks.workunit.client.0.smithi136.stdout:op 756 completed, throughput=4.98MB/sec 2024-09-13T00:03:52.972 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-PFbvXjB5sEYFdD- off=422425 len=7889 2024-09-13T00:03:52.973 INFO:tasks.workunit.client.0.smithi136.stdout:op 757 completed, throughput=4.98MB/sec 2024-09-13T00:03:52.973 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-CoLwQogdUr7v0TX off=6327048 len=139876 2024-09-13T00:03:52.978 INFO:tasks.workunit.client.0.smithi136.stdout:op 758 completed, throughput=4.98MB/sec 2024-09-13T00:03:52.978 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-fxvRJ5nBy2r0XdH off=17527645 len=461326 2024-09-13T00:03:52.983 INFO:tasks.workunit.client.0.smithi136.stdout:op 759 completed, throughput=4.98MB/sec 2024-09-13T00:03:52.983 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-DVhsfcx3jfMvByJ off=2628066 len=812615 2024-09-13T00:03:52.991 INFO:tasks.workunit.client.0.smithi136.stdout:op 760 completed, throughput=4.99MB/sec 2024-09-13T00:03:52.991 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-sNpO1ZSLeE0kLcJ off=2515031 len=1564512 2024-09-13T00:03:53.002 INFO:tasks.workunit.client.0.smithi136.stdout:op 761 completed, throughput=5MB/sec 2024-09-13T00:03:53.002 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-3mmOmC48i4qyUlj off=11841092 len=100929 2024-09-13T00:03:53.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:03:52 smithi136 bash[75164]: cluster 2024-09-13T00:03:51.222322+0000 mgr.y (mgr.24661) 802 : cluster [DBG] pgmap v1485: 105 pgs: 105 active+clean; 126 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.2 MiB/s wr, 5 op/s 2024-09-13T00:03:53.007 INFO:tasks.workunit.client.0.smithi136.stdout:op 762 completed, throughput=5MB/sec 2024-09-13T00:03:53.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:03:52 smithi136 bash[76371]: cluster 2024-09-13T00:03:51.222322+0000 mgr.y (mgr.24661) 802 : cluster [DBG] pgmap v1485: 105 pgs: 105 active+clean; 126 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.2 MiB/s wr, 5 op/s 2024-09-13T00:03:53.008 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-I2WECEE6b8b67dM off=5318023 len=1426091 2024-09-13T00:03:53.020 INFO:tasks.workunit.client.0.smithi136.stdout:op 763 completed, throughput=5.01MB/sec 2024-09-13T00:03:53.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:03:52 smithi181 bash[67587]: cluster 2024-09-13T00:03:51.222322+0000 mgr.y (mgr.24661) 802 : cluster [DBG] pgmap v1485: 105 pgs: 105 active+clean; 126 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.2 MiB/s wr, 5 op/s 2024-09-13T00:03:54.020 INFO:tasks.workunit.client.0.smithi136.stdout: 146: throughput=4.97MB/sec pending data=0 2024-09-13T00:03:54.021 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-WcOVJ1g6LthDWFw off=16547387 len=1595053 2024-09-13T00:03:54.025 INFO:tasks.workunit.client.0.smithi136.stdout:op 764 completed, throughput=4.98MB/sec 2024-09-13T00:03:54.025 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-qWcO5wgvZySY3LJ off=4641823 len=306288 2024-09-13T00:03:54.026 INFO:tasks.workunit.client.0.smithi136.stdout:op 765 completed, throughput=4.99MB/sec 2024-09-13T00:03:54.027 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-RMo4Xyl8vXK4MkY off=4939807 len=1521632 2024-09-13T00:03:54.030 INFO:tasks.workunit.client.0.smithi136.stdout:op 766 completed, throughput=5MB/sec 2024-09-13T00:03:54.030 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-1xBatDkTyFh9qzx off=9340793 len=1480351 2024-09-13T00:03:54.033 INFO:tasks.workunit.client.0.smithi136.stdout:op 767 completed, throughput=5.01MB/sec 2024-09-13T00:03:55.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:03:54 smithi136 bash[75164]: cluster 2024-09-13T00:03:53.223297+0000 mgr.y (mgr.24661) 803 : cluster [DBG] pgmap v1486: 105 pgs: 105 active+clean; 126 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:03:55.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:03:54 smithi136 bash[75164]: audit 2024-09-13T00:03:53.775306+0000 mon.a (mon.0) 2130 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:03:55.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:03:54 smithi136 bash[76371]: cluster 2024-09-13T00:03:53.223297+0000 mgr.y (mgr.24661) 803 : cluster [DBG] pgmap v1486: 105 pgs: 105 active+clean; 126 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:03:55.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:03:54 smithi136 bash[76371]: audit 2024-09-13T00:03:53.775306+0000 mon.a (mon.0) 2130 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:03:55.033 INFO:tasks.workunit.client.0.smithi136.stdout: 147: throughput=4.97MB/sec pending data=0 2024-09-13T00:03:55.033 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-kap9Z9felSTRbGo off=2929154 len=472093 2024-09-13T00:03:55.035 INFO:tasks.workunit.client.0.smithi136.stdout:op 768 completed, throughput=4.98MB/sec 2024-09-13T00:03:55.035 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj--yGxquIM9yBQzSm off=991569 len=545565 2024-09-13T00:03:55.041 INFO:tasks.workunit.client.0.smithi136.stdout:op 769 completed, throughput=4.98MB/sec 2024-09-13T00:03:55.042 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj--MKxcSDt5Q3pwo6 off=7629369 len=562195 2024-09-13T00:03:55.048 INFO:tasks.workunit.client.0.smithi136.stdout:op 770 completed, throughput=4.98MB/sec 2024-09-13T00:03:55.048 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-MH15o01KIxsRE84 off=7264960 len=2027475 2024-09-13T00:03:55.061 INFO:tasks.workunit.client.0.smithi136.stdout:op 771 completed, throughput=4.99MB/sec 2024-09-13T00:03:55.061 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-salyjcwYGgBSIuy off=7596439 len=1873778 2024-09-13T00:03:55.073 INFO:tasks.workunit.client.0.smithi136.stdout:op 772 completed, throughput=5.01MB/sec 2024-09-13T00:03:55.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:03:54 smithi181 bash[67587]: cluster 2024-09-13T00:03:53.223297+0000 mgr.y (mgr.24661) 803 : cluster [DBG] pgmap v1486: 105 pgs: 105 active+clean; 126 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:03:55.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:03:54 smithi181 bash[67587]: audit 2024-09-13T00:03:53.775306+0000 mon.a (mon.0) 2130 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:03:56.074 INFO:tasks.workunit.client.0.smithi136.stdout: 148: throughput=4.97MB/sec pending data=0 2024-09-13T00:03:56.074 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-sBPlANlO8eHguWA off=3655312 len=1052684 2024-09-13T00:03:56.083 INFO:tasks.workunit.client.0.smithi136.stdout:op 773 completed, throughput=4.98MB/sec 2024-09-13T00:03:56.083 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-D-Oq9mH2Az-m9i7 off=5206483 len=559941 2024-09-13T00:03:56.090 INFO:tasks.workunit.client.0.smithi136.stdout:op 774 completed, throughput=4.98MB/sec 2024-09-13T00:03:56.091 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-QDTqeZrxRXHqhfX off=14028006 len=1041386 2024-09-13T00:03:56.093 INFO:tasks.workunit.client.0.smithi136.stdout:op 775 completed, throughput=4.99MB/sec 2024-09-13T00:03:56.094 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-IswiMNc_ZKPJJEH off=3341303 len=584397 2024-09-13T00:03:56.101 INFO:tasks.workunit.client.0.smithi136.stdout:op 776 completed, throughput=4.99MB/sec 2024-09-13T00:03:56.101 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-s1_uCtgo7uUJ2wk off=5336710 len=390429 2024-09-13T00:03:56.106 INFO:tasks.workunit.client.0.smithi136.stdout:op 777 completed, throughput=4.99MB/sec 2024-09-13T00:03:56.106 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Vi0-9ROetc4zzVQ off=21719810 len=385457 2024-09-13T00:03:56.112 INFO:tasks.workunit.client.0.smithi136.stdout:op 778 completed, throughput=5MB/sec 2024-09-13T00:03:56.112 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-PkanUapLnSmmf3k off=18293014 len=1076481 2024-09-13T00:03:56.116 INFO:tasks.workunit.client.0.smithi136.stdout:op 779 completed, throughput=5MB/sec 2024-09-13T00:03:57.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:03:56 smithi136 bash[75164]: cluster 2024-09-13T00:03:55.224259+0000 mgr.y (mgr.24661) 804 : cluster [DBG] pgmap v1487: 105 pgs: 105 active+clean; 126 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:03:57.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:03:56 smithi136 bash[76371]: cluster 2024-09-13T00:03:55.224259+0000 mgr.y (mgr.24661) 804 : cluster [DBG] pgmap v1487: 105 pgs: 105 active+clean; 126 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:03:57.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:03:56 smithi181 bash[67587]: cluster 2024-09-13T00:03:55.224259+0000 mgr.y (mgr.24661) 804 : cluster [DBG] pgmap v1487: 105 pgs: 105 active+clean; 126 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:03:57.117 INFO:tasks.workunit.client.0.smithi136.stdout: 149: throughput=4.97MB/sec pending data=0 2024-09-13T00:03:57.117 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-v6cLDT4FxTchIqH off=1780802 len=1083764 2024-09-13T00:03:57.126 INFO:tasks.workunit.client.0.smithi136.stdout:op 780 completed, throughput=4.98MB/sec 2024-09-13T00:03:57.126 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-2RLOhkZeVkGOnnT off=13671350 len=1932913 2024-09-13T00:03:57.140 INFO:tasks.workunit.client.0.smithi136.stdout:op 781 completed, throughput=4.99MB/sec 2024-09-13T00:03:57.140 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-yggiKNOt7spz4Bn off=8260885 len=1244150 2024-09-13T00:03:57.150 INFO:tasks.workunit.client.0.smithi136.stdout:op 782 completed, throughput=5MB/sec 2024-09-13T00:03:57.151 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-mXeRLJPhSSaknQw off=16044973 len=139017 2024-09-13T00:03:57.152 INFO:tasks.workunit.client.0.smithi136.stdout:op 783 completed, throughput=5MB/sec 2024-09-13T00:03:57.152 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-IvYmRlO-DTQrtq5 off=1293128 len=2065777 2024-09-13T00:03:57.155 INFO:tasks.workunit.client.0.smithi136.stdout:op 784 completed, throughput=5.01MB/sec 2024-09-13T00:03:58.155 INFO:tasks.workunit.client.0.smithi136.stdout: 150: throughput=4.98MB/sec pending data=0 2024-09-13T00:03:58.156 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-sBPlANlO8eHguWA off=10652097 len=1020545 2024-09-13T00:03:58.159 INFO:tasks.workunit.client.0.smithi136.stdout:op 785 completed, throughput=4.98MB/sec 2024-09-13T00:03:58.159 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-__CRy3Aty8FRp15 off=7836326 len=790575 2024-09-13T00:03:58.161 INFO:tasks.workunit.client.0.smithi136.stdout:op 786 completed, throughput=4.99MB/sec 2024-09-13T00:03:58.162 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-6VMjD0nPftCUFtA off=1265732 len=364635 2024-09-13T00:03:58.168 INFO:tasks.workunit.client.0.smithi136.stdout:op 787 completed, throughput=4.99MB/sec 2024-09-13T00:03:58.168 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-sp6UZqEyrWR5dhy off=21684147 len=385211 2024-09-13T00:03:58.169 INFO:tasks.workunit.client.0.smithi136.stdout:op 788 completed, throughput=4.99MB/sec 2024-09-13T00:03:58.169 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-E54Vrw-YS8ZPCDS off=2307205 len=1774041 2024-09-13T00:03:58.184 INFO:tasks.workunit.client.0.smithi136.stdout:op 789 completed, throughput=5MB/sec 2024-09-13T00:03:59.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:03:58 smithi136 bash[75164]: cluster 2024-09-13T00:03:57.225474+0000 mgr.y (mgr.24661) 805 : cluster [DBG] pgmap v1488: 105 pgs: 105 active+clean; 126 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 2.2 MiB/s rd, 3.2 MiB/s wr, 6 op/s 2024-09-13T00:03:59.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:03:58 smithi136 bash[76371]: cluster 2024-09-13T00:03:57.225474+0000 mgr.y (mgr.24661) 805 : cluster [DBG] pgmap v1488: 105 pgs: 105 active+clean; 126 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 2.2 MiB/s rd, 3.2 MiB/s wr, 6 op/s 2024-09-13T00:03:59.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:03:58 smithi181 bash[67587]: cluster 2024-09-13T00:03:57.225474+0000 mgr.y (mgr.24661) 805 : cluster [DBG] pgmap v1488: 105 pgs: 105 active+clean; 126 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 2.2 MiB/s rd, 3.2 MiB/s wr, 6 op/s 2024-09-13T00:03:59.184 INFO:tasks.workunit.client.0.smithi136.stdout: 151: throughput=4.97MB/sec pending data=0 2024-09-13T00:03:59.185 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-O4zHCuu6wIxG6Ts off=3922855 len=381004 2024-09-13T00:03:59.190 INFO:tasks.workunit.client.0.smithi136.stdout:op 790 completed, throughput=4.97MB/sec 2024-09-13T00:03:59.190 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-pL0ATTO-xhxK0Ij off=2442802 len=1333 2024-09-13T00:03:59.191 INFO:tasks.workunit.client.0.smithi136.stdout:op 791 completed, throughput=4.97MB/sec 2024-09-13T00:03:59.191 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-4XjX6qOJybbLu9h off=8407118 len=997954 2024-09-13T00:03:59.202 INFO:tasks.workunit.client.0.smithi136.stdout:op 792 completed, throughput=4.98MB/sec 2024-09-13T00:03:59.202 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-LrdRvAzMYcL4DMn off=6965540 len=933406 2024-09-13T00:03:59.210 INFO:tasks.workunit.client.0.smithi136.stdout:op 793 completed, throughput=4.98MB/sec 2024-09-13T00:03:59.210 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-H8HJ-sZHoy0PZgy off=2494966 len=30777 2024-09-13T00:03:59.211 INFO:tasks.workunit.client.0.smithi136.stdout:op 794 completed, throughput=4.98MB/sec 2024-09-13T00:03:59.211 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-9H_6OJ6swdJ6c8R off=2064125 len=552978 2024-09-13T00:03:59.217 INFO:tasks.workunit.client.0.smithi136.stdout:op 795 completed, throughput=4.99MB/sec 2024-09-13T00:03:59.217 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-zK_Axdk4YHNZl0R off=1776985 len=1666233 2024-09-13T00:03:59.221 INFO:tasks.workunit.client.0.smithi136.stdout:op 796 completed, throughput=5MB/sec 2024-09-13T00:03:59.221 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-fC1O6JFT1ae1njq off=6662499 len=1029916 2024-09-13T00:03:59.229 INFO:tasks.workunit.client.0.smithi136.stdout:op 797 completed, throughput=5MB/sec 2024-09-13T00:03:59.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:03:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:03:59] "GET /metrics HTTP/1.1" 200 34124 "" "Prometheus/2.43.0" 2024-09-13T00:04:00.230 INFO:tasks.workunit.client.0.smithi136.stdout: 152: throughput=4.97MB/sec pending data=0 2024-09-13T00:04:00.230 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ar2WLyaWf7VrVvd off=482829 len=2028677 2024-09-13T00:04:00.236 INFO:tasks.workunit.client.0.smithi136.stdout:op 798 completed, throughput=4.98MB/sec 2024-09-13T00:04:00.237 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-nQkpjs3Qm2hpbmU off=16350042 len=751652 2024-09-13T00:04:00.244 INFO:tasks.workunit.client.0.smithi136.stdout:op 799 completed, throughput=4.99MB/sec 2024-09-13T00:04:00.244 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-EDpaEgDYPZU4uW5 off=4996954 len=235920 2024-09-13T00:04:00.249 INFO:tasks.workunit.client.0.smithi136.stdout:op 800 completed, throughput=4.99MB/sec 2024-09-13T00:04:00.249 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-p0TlEZ8wCEbkc_7 off=12451290 len=468194 2024-09-13T00:04:00.254 INFO:tasks.workunit.client.0.smithi136.stdout:op 801 completed, throughput=4.99MB/sec 2024-09-13T00:04:00.254 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-sr_uQswyKs1V33a off=24738971 len=295249 2024-09-13T00:04:00.256 INFO:tasks.workunit.client.0.smithi136.stdout:op 802 completed, throughput=4.99MB/sec 2024-09-13T00:04:00.256 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-A5B5Psi_O0Buj5h off=5216081 len=99277 2024-09-13T00:04:00.257 INFO:tasks.workunit.client.0.smithi136.stdout:op 803 completed, throughput=4.99MB/sec 2024-09-13T00:04:00.257 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-OI6rrd4lJ4gyGJm off=10582883 len=1821905 2024-09-13T00:04:00.269 INFO:tasks.workunit.client.0.smithi136.stdout:op 804 completed, throughput=5MB/sec 2024-09-13T00:04:01.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:04:00 smithi136 bash[75164]: cluster 2024-09-13T00:03:59.226354+0000 mgr.y (mgr.24661) 806 : cluster [DBG] pgmap v1489: 105 pgs: 105 active+clean; 126 GiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.8 MiB/s wr, 6 op/s 2024-09-13T00:04:01.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:04:00 smithi136 bash[76371]: cluster 2024-09-13T00:03:59.226354+0000 mgr.y (mgr.24661) 806 : cluster [DBG] pgmap v1489: 105 pgs: 105 active+clean; 126 GiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.8 MiB/s wr, 6 op/s 2024-09-13T00:04:01.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:04:00 smithi181 bash[67587]: cluster 2024-09-13T00:03:59.226354+0000 mgr.y (mgr.24661) 806 : cluster [DBG] pgmap v1489: 105 pgs: 105 active+clean; 126 GiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.8 MiB/s wr, 6 op/s 2024-09-13T00:04:01.269 INFO:tasks.workunit.client.0.smithi136.stdout: 153: throughput=4.97MB/sec pending data=0 2024-09-13T00:04:01.270 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-hF9yfSHiF7BELjT off=1518636 len=584506 2024-09-13T00:04:01.272 INFO:tasks.workunit.client.0.smithi136.stdout:op 805 completed, throughput=4.98MB/sec 2024-09-13T00:04:01.272 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-9Uh-EbTHEobO_RA off=850502 len=1106368 2024-09-13T00:04:01.276 INFO:tasks.workunit.client.0.smithi136.stdout:op 806 completed, throughput=4.98MB/sec 2024-09-13T00:04:01.276 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-RqmUN-OSKCy6-tV off=2017637 len=118963 2024-09-13T00:04:01.281 INFO:tasks.workunit.client.0.smithi136.stdout:op 807 completed, throughput=4.98MB/sec 2024-09-13T00:04:01.281 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-gkinSTc0kvUc9vJ off=18218996 len=1631486 2024-09-13T00:04:01.284 INFO:tasks.workunit.client.0.smithi136.stdout:op 808 completed, throughput=4.99MB/sec 2024-09-13T00:04:01.284 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ho65qP8bgcBA_5J off=4020674 len=908589 2024-09-13T00:04:01.293 INFO:tasks.workunit.client.0.smithi136.stdout:op 809 completed, throughput=5MB/sec 2024-09-13T00:04:01.293 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-1Axb4oTOt-uVqhb off=1159242 len=1818304 2024-09-13T00:04:01.307 INFO:tasks.workunit.client.0.smithi136.stdout:op 810 completed, throughput=5.01MB/sec 2024-09-13T00:04:02.308 INFO:tasks.workunit.client.0.smithi136.stdout: 154: throughput=4.98MB/sec pending data=0 2024-09-13T00:04:02.308 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Sht23eEynpAT0X7 off=9736327 len=1336582 2024-09-13T00:04:02.311 INFO:tasks.workunit.client.0.smithi136.stdout:op 811 completed, throughput=4.99MB/sec 2024-09-13T00:04:02.311 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-w0rS0S3CxlYer6p off=6675404 len=1927826 2024-09-13T00:04:02.316 INFO:tasks.workunit.client.0.smithi136.stdout:op 812 completed, throughput=5MB/sec 2024-09-13T00:04:02.316 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-NmQ-SPR-rROLpt8 off=1263814 len=398709 2024-09-13T00:04:02.318 INFO:tasks.workunit.client.0.smithi136.stdout:op 813 completed, throughput=5MB/sec 2024-09-13T00:04:02.318 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-wkoiBzpz5tSwoKZ off=4200942 len=611603 2024-09-13T00:04:02.325 INFO:tasks.workunit.client.0.smithi136.stdout:op 814 completed, throughput=5MB/sec 2024-09-13T00:04:02.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:04:02 smithi181 bash[95505]: ts=2024-09-13T00:04:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:04:03.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:04:02 smithi136 bash[75164]: cluster 2024-09-13T00:04:01.227392+0000 mgr.y (mgr.24661) 807 : cluster [DBG] pgmap v1490: 105 pgs: 105 active+clean; 126 GiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:04:03.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:04:02 smithi136 bash[76371]: cluster 2024-09-13T00:04:01.227392+0000 mgr.y (mgr.24661) 807 : cluster [DBG] pgmap v1490: 105 pgs: 105 active+clean; 126 GiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:04:03.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:04:02 smithi181 bash[67587]: cluster 2024-09-13T00:04:01.227392+0000 mgr.y (mgr.24661) 807 : cluster [DBG] pgmap v1490: 105 pgs: 105 active+clean; 126 GiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:04:03.325 INFO:tasks.workunit.client.0.smithi136.stdout: 155: throughput=4.97MB/sec pending data=0 2024-09-13T00:04:03.326 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-jgQ0Qoyquc28QEQ off=3513762 len=2089906 2024-09-13T00:04:03.340 INFO:tasks.workunit.client.0.smithi136.stdout:op 815 completed, throughput=4.98MB/sec 2024-09-13T00:04:03.340 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-9mbLgjGuoJsh0nC off=16340887 len=648607 2024-09-13T00:04:03.342 INFO:tasks.workunit.client.0.smithi136.stdout:op 816 completed, throughput=4.99MB/sec 2024-09-13T00:04:03.342 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-zMI6yZZgdLBypb4 off=2628947 len=1462726 2024-09-13T00:04:03.345 INFO:tasks.workunit.client.0.smithi136.stdout:op 817 completed, throughput=5MB/sec 2024-09-13T00:04:03.345 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj--AJlCrJaMy8v6Jh off=410067 len=1956805 2024-09-13T00:04:03.356 INFO:tasks.workunit.client.0.smithi136.stdout:op 818 completed, throughput=5.01MB/sec 2024-09-13T00:04:04.356 INFO:tasks.workunit.client.0.smithi136.stdout: 156: throughput=4.98MB/sec pending data=0 2024-09-13T00:04:04.357 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-bNAajd0hXtMJSDm off=1057154 len=2044153 2024-09-13T00:04:04.375 INFO:tasks.workunit.client.0.smithi136.stdout:op 819 completed, throughput=4.99MB/sec 2024-09-13T00:04:04.375 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-nJ3rwBPZ46CXcpd off=7720600 len=613888 2024-09-13T00:04:04.382 INFO:tasks.workunit.client.0.smithi136.stdout:op 820 completed, throughput=4.99MB/sec 2024-09-13T00:04:04.382 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-YjVTJNGS7d3nwld off=8801517 len=2021735 2024-09-13T00:04:04.397 INFO:tasks.workunit.client.0.smithi136.stdout:op 821 completed, throughput=5MB/sec 2024-09-13T00:04:05.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:04:04 smithi136 bash[75164]: cluster 2024-09-13T00:04:03.228355+0000 mgr.y (mgr.24661) 808 : cluster [DBG] pgmap v1491: 105 pgs: 105 active+clean; 126 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.0 MiB/s wr, 5 op/s 2024-09-13T00:04:05.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:04:04 smithi136 bash[76371]: cluster 2024-09-13T00:04:03.228355+0000 mgr.y (mgr.24661) 808 : cluster [DBG] pgmap v1491: 105 pgs: 105 active+clean; 126 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.0 MiB/s wr, 5 op/s 2024-09-13T00:04:05.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:04:04 smithi181 bash[67587]: cluster 2024-09-13T00:04:03.228355+0000 mgr.y (mgr.24661) 808 : cluster [DBG] pgmap v1491: 105 pgs: 105 active+clean; 126 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.0 MiB/s wr, 5 op/s 2024-09-13T00:04:05.397 INFO:tasks.workunit.client.0.smithi136.stdout: 157: throughput=4.97MB/sec pending data=0 2024-09-13T00:04:05.397 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-TxbMUs_-s4ymgVR off=3128243 len=316402 2024-09-13T00:04:05.403 INFO:tasks.workunit.client.0.smithi136.stdout:op 822 completed, throughput=4.97MB/sec 2024-09-13T00:04:05.403 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-izl_EYPsU5JGN0o off=18321451 len=426130 2024-09-13T00:04:05.409 INFO:tasks.workunit.client.0.smithi136.stdout:op 823 completed, throughput=4.98MB/sec 2024-09-13T00:04:05.409 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-V_ZbVLortSJYty- off=1792398 len=439404 2024-09-13T00:04:05.411 INFO:tasks.workunit.client.0.smithi136.stdout:op 824 completed, throughput=4.98MB/sec 2024-09-13T00:04:05.411 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-veU1XH3rJTKqmmC off=10233939 len=840833 2024-09-13T00:04:05.419 INFO:tasks.workunit.client.0.smithi136.stdout:op 825 completed, throughput=4.98MB/sec 2024-09-13T00:04:05.420 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-BHtJ6UXt88ns-2q off=6483843 len=660773 2024-09-13T00:04:05.421 INFO:tasks.workunit.client.0.smithi136.stdout:op 826 completed, throughput=4.99MB/sec 2024-09-13T00:04:05.421 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-7MTe0WAiqoDh_n- off=3898025 len=96013 2024-09-13T00:04:05.422 INFO:tasks.workunit.client.0.smithi136.stdout:op 827 completed, throughput=4.99MB/sec 2024-09-13T00:04:05.423 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Y6enpbJTLF5avAp off=20064875 len=282134 2024-09-13T00:04:05.424 INFO:tasks.workunit.client.0.smithi136.stdout:op 828 completed, throughput=4.99MB/sec 2024-09-13T00:04:05.424 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-GP8NxXYGxcY9oN7 off=1057089 len=230013 2024-09-13T00:04:05.426 INFO:tasks.workunit.client.0.smithi136.stdout:op 829 completed, throughput=4.99MB/sec 2024-09-13T00:04:05.426 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2Jt9eAeNN4r2SG_ off=1341854 len=657131 2024-09-13T00:04:05.429 INFO:tasks.workunit.client.0.smithi136.stdout:op 830 completed, throughput=4.99MB/sec 2024-09-13T00:04:05.429 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-X1gfp5_f5eMX3wG off=1845354 len=1726277 2024-09-13T00:04:05.440 INFO:tasks.workunit.client.0.smithi136.stdout:op 831 completed, throughput=5MB/sec 2024-09-13T00:04:06.441 INFO:tasks.workunit.client.0.smithi136.stdout: 158: throughput=4.97MB/sec pending data=0 2024-09-13T00:04:06.441 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-vk33gKYF4jIqjo- off=194827 len=1770719 2024-09-13T00:04:06.452 INFO:tasks.workunit.client.0.smithi136.stdout:op 832 completed, throughput=4.98MB/sec 2024-09-13T00:04:06.452 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ssR9cP25MzoUmBo off=8292443 len=1053267 2024-09-13T00:04:06.456 INFO:tasks.workunit.client.0.smithi136.stdout:op 833 completed, throughput=4.99MB/sec 2024-09-13T00:04:06.456 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-IgFVF1GVmm79tC2 off=2713219 len=936679 2024-09-13T00:04:06.458 INFO:tasks.workunit.client.0.smithi136.stdout:op 834 completed, throughput=4.99MB/sec 2024-09-13T00:04:06.458 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-dioiS1MhmLyQmAM off=255834 len=706274 2024-09-13T00:04:06.461 INFO:tasks.workunit.client.0.smithi136.stdout:op 835 completed, throughput=5MB/sec 2024-09-13T00:04:06.462 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2ANs4JEXxJt4ogN off=8350115 len=1473982 2024-09-13T00:04:06.465 INFO:tasks.workunit.client.0.smithi136.stdout:op 836 completed, throughput=5.01MB/sec 2024-09-13T00:04:07.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:04:06 smithi136 bash[75164]: cluster 2024-09-13T00:04:05.229205+0000 mgr.y (mgr.24661) 809 : cluster [DBG] pgmap v1492: 105 pgs: 105 active+clean; 126 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:04:07.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:04:06 smithi136 bash[76371]: cluster 2024-09-13T00:04:05.229205+0000 mgr.y (mgr.24661) 809 : cluster [DBG] pgmap v1492: 105 pgs: 105 active+clean; 126 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:04:07.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:04:06 smithi181 bash[67587]: cluster 2024-09-13T00:04:05.229205+0000 mgr.y (mgr.24661) 809 : cluster [DBG] pgmap v1492: 105 pgs: 105 active+clean; 126 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:04:07.465 INFO:tasks.workunit.client.0.smithi136.stdout: 159: throughput=4.98MB/sec pending data=0 2024-09-13T00:04:07.466 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-WAr6La3ep_eh_U3 off=2043305 len=1972573 2024-09-13T00:04:07.482 INFO:tasks.workunit.client.0.smithi136.stdout:op 837 completed, throughput=4.99MB/sec 2024-09-13T00:04:07.482 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-q63xdgcKpq6z7fk off=7244138 len=1633696 2024-09-13T00:04:07.493 INFO:tasks.workunit.client.0.smithi136.stdout:op 838 completed, throughput=5MB/sec 2024-09-13T00:04:07.493 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-rtn_sRqhzdV3UCn off=284568 len=1412895 2024-09-13T00:04:07.497 INFO:tasks.workunit.client.0.smithi136.stdout:op 839 completed, throughput=5MB/sec 2024-09-13T00:04:08.497 INFO:tasks.workunit.client.0.smithi136.stdout: 160: throughput=4.97MB/sec pending data=0 2024-09-13T00:04:08.497 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-CwHlPM6_RiO15uW off=4261989 len=292652 2024-09-13T00:04:08.498 INFO:tasks.workunit.client.0.smithi136.stdout:op 840 completed, throughput=4.98MB/sec 2024-09-13T00:04:08.498 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-IMoGEQuVOMjVr3U off=5315173 len=710832 2024-09-13T00:04:08.501 INFO:tasks.workunit.client.0.smithi136.stdout:op 841 completed, throughput=4.98MB/sec 2024-09-13T00:04:08.501 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-uIli1KaHggZRPra off=8059980 len=616240 2024-09-13T00:04:08.504 INFO:tasks.workunit.client.0.smithi136.stdout:op 842 completed, throughput=4.98MB/sec 2024-09-13T00:04:08.504 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-tWvc2zdMHBV49bv off=7864381 len=1424789 2024-09-13T00:04:08.508 INFO:tasks.workunit.client.0.smithi136.stdout:op 843 completed, throughput=4.99MB/sec 2024-09-13T00:04:08.508 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-MYGsTRvbdqWB0w1 off=3437425 len=1415 2024-09-13T00:04:08.512 INFO:tasks.workunit.client.0.smithi136.stdout:op 844 completed, throughput=4.99MB/sec 2024-09-13T00:04:08.512 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-3FGdSHPRtGzIZf_ off=14159093 len=1977510 2024-09-13T00:04:08.527 INFO:tasks.workunit.client.0.smithi136.stdout:op 845 completed, throughput=5MB/sec 2024-09-13T00:04:09.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:04:08 smithi136 bash[75164]: cluster 2024-09-13T00:04:07.230582+0000 mgr.y (mgr.24661) 810 : cluster [DBG] pgmap v1493: 105 pgs: 105 active+clean; 126 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 2.6 MiB/s rd, 3.2 MiB/s wr, 6 op/s 2024-09-13T00:04:09.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:04:08 smithi136 bash[76371]: cluster 2024-09-13T00:04:07.230582+0000 mgr.y (mgr.24661) 810 : cluster [DBG] pgmap v1493: 105 pgs: 105 active+clean; 126 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 2.6 MiB/s rd, 3.2 MiB/s wr, 6 op/s 2024-09-13T00:04:09.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:04:08 smithi181 bash[67587]: cluster 2024-09-13T00:04:07.230582+0000 mgr.y (mgr.24661) 810 : cluster [DBG] pgmap v1493: 105 pgs: 105 active+clean; 126 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 2.6 MiB/s rd, 3.2 MiB/s wr, 6 op/s 2024-09-13T00:04:09.528 INFO:tasks.workunit.client.0.smithi136.stdout: 161: throughput=4.97MB/sec pending data=0 2024-09-13T00:04:09.528 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-iaJP_tyrAdmC0pq off=16250759 len=1694491 2024-09-13T00:04:09.531 INFO:tasks.workunit.client.0.smithi136.stdout:op 846 completed, throughput=4.98MB/sec 2024-09-13T00:04:09.532 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-R4MQ8BWyal05USL off=1619984 len=458104 2024-09-13T00:04:09.533 INFO:tasks.workunit.client.0.smithi136.stdout:op 847 completed, throughput=4.98MB/sec 2024-09-13T00:04:09.533 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-juVT17xSvmZlxuw off=914742 len=2069382 2024-09-13T00:04:09.548 INFO:tasks.workunit.client.0.smithi136.stdout:op 848 completed, throughput=5MB/sec 2024-09-13T00:04:09.548 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-s1DwEZjXzoT5NXu off=2927499 len=1997392 2024-09-13T00:04:09.561 INFO:tasks.workunit.client.0.smithi136.stdout:op 849 completed, throughput=5.01MB/sec 2024-09-13T00:04:09.721 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:04:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:04:09] "GET /metrics HTTP/1.1" 200 34124 "" "Prometheus/2.43.0" 2024-09-13T00:04:10.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:04:09 smithi136 bash[75164]: audit 2024-09-13T00:04:08.775861+0000 mon.a (mon.0) 2131 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:04:10.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:04:09 smithi136 bash[76371]: audit 2024-09-13T00:04:08.775861+0000 mon.a (mon.0) 2131 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:04:10.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:04:09 smithi181 bash[67587]: audit 2024-09-13T00:04:08.775861+0000 mon.a (mon.0) 2131 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:04:10.561 INFO:tasks.workunit.client.0.smithi136.stdout: 162: throughput=4.98MB/sec pending data=0 2024-09-13T00:04:10.561 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-_v5MngtVT75kzkY off=6030006 len=1312128 2024-09-13T00:04:10.565 INFO:tasks.workunit.client.0.smithi136.stdout:op 850 completed, throughput=4.98MB/sec 2024-09-13T00:04:10.565 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-KnbHSgwK4q_fe1u off=13019927 len=1980460 2024-09-13T00:04:10.579 INFO:tasks.workunit.client.0.smithi136.stdout:op 851 completed, throughput=5MB/sec 2024-09-13T00:04:10.579 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-bJ4A-DceRidEAJP off=724787 len=1685653 2024-09-13T00:04:10.583 INFO:tasks.workunit.client.0.smithi136.stdout:op 852 completed, throughput=5MB/sec 2024-09-13T00:04:11.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:04:10 smithi136 bash[75164]: cluster 2024-09-13T00:04:09.231496+0000 mgr.y (mgr.24661) 811 : cluster [DBG] pgmap v1494: 105 pgs: 105 active+clean; 126 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:04:11.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:04:10 smithi136 bash[76371]: cluster 2024-09-13T00:04:09.231496+0000 mgr.y (mgr.24661) 811 : cluster [DBG] pgmap v1494: 105 pgs: 105 active+clean; 126 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:04:11.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:04:10 smithi181 bash[67587]: cluster 2024-09-13T00:04:09.231496+0000 mgr.y (mgr.24661) 811 : cluster [DBG] pgmap v1494: 105 pgs: 105 active+clean; 126 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:04:11.583 INFO:tasks.workunit.client.0.smithi136.stdout: 163: throughput=4.98MB/sec pending data=0 2024-09-13T00:04:11.583 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-WoOHYdvI7-dr1gT off=6128051 len=1610267 2024-09-13T00:04:11.596 INFO:tasks.workunit.client.0.smithi136.stdout:op 853 completed, throughput=4.98MB/sec 2024-09-13T00:04:11.596 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-rOklyJUTuyuK00E off=3028504 len=1399225 2024-09-13T00:04:11.605 INFO:tasks.workunit.client.0.smithi136.stdout:op 854 completed, throughput=4.99MB/sec 2024-09-13T00:04:11.606 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-5EXimb0b39TIHXr off=452241 len=1461693 2024-09-13T00:04:11.615 INFO:tasks.workunit.client.0.smithi136.stdout:op 855 completed, throughput=5MB/sec 2024-09-13T00:04:11.615 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-BxGuBn4TaNMjgjB off=11545243 len=1438094 2024-09-13T00:04:11.623 INFO:tasks.workunit.client.0.smithi136.stdout:op 856 completed, throughput=5.01MB/sec 2024-09-13T00:04:12.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:04:12 smithi181 bash[95505]: ts=2024-09-13T00:04:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:04:12.624 INFO:tasks.workunit.client.0.smithi136.stdout: 164: throughput=4.98MB/sec pending data=0 2024-09-13T00:04:12.624 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-rtd_ufN_57vStTi off=1025579 len=62186 2024-09-13T00:04:12.625 INFO:tasks.workunit.client.0.smithi136.stdout:op 857 completed, throughput=4.98MB/sec 2024-09-13T00:04:12.625 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-imWzFDWzXa15Qkj off=3176090 len=213329 2024-09-13T00:04:12.626 INFO:tasks.workunit.client.0.smithi136.stdout:op 858 completed, throughput=4.98MB/sec 2024-09-13T00:04:12.626 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-DTpJWomBHpQYGRe off=695534 len=773416 2024-09-13T00:04:12.628 INFO:tasks.workunit.client.0.smithi136.stdout:op 859 completed, throughput=4.98MB/sec 2024-09-13T00:04:12.628 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-3A9DmIt2gwg40Zl off=6030107 len=1627691 2024-09-13T00:04:12.634 INFO:tasks.workunit.client.0.smithi136.stdout:op 860 completed, throughput=4.99MB/sec 2024-09-13T00:04:12.634 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-owy7tQtIduv51Pb off=15535968 len=72138 2024-09-13T00:04:12.639 INFO:tasks.workunit.client.0.smithi136.stdout:op 861 completed, throughput=4.99MB/sec 2024-09-13T00:04:12.639 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-gkjch8FjVwyiekX off=7921519 len=620272 2024-09-13T00:04:12.642 INFO:tasks.workunit.client.0.smithi136.stdout:op 862 completed, throughput=5MB/sec 2024-09-13T00:04:12.642 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-CBYLCT50pDQI1cY off=5090463 len=629836 2024-09-13T00:04:12.649 INFO:tasks.workunit.client.0.smithi136.stdout:op 863 completed, throughput=5MB/sec 2024-09-13T00:04:12.649 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-eVHsRn4AdtjVFAx off=10207528 len=90577 2024-09-13T00:04:12.650 INFO:tasks.workunit.client.0.smithi136.stdout:op 864 completed, throughput=5MB/sec 2024-09-13T00:04:13.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:04:12 smithi136 bash[76371]: cluster 2024-09-13T00:04:11.232543+0000 mgr.y (mgr.24661) 812 : cluster [DBG] pgmap v1495: 105 pgs: 105 active+clean; 126 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:04:13.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:04:12 smithi136 bash[75164]: cluster 2024-09-13T00:04:11.232543+0000 mgr.y (mgr.24661) 812 : cluster [DBG] pgmap v1495: 105 pgs: 105 active+clean; 126 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:04:13.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:04:12 smithi181 bash[67587]: cluster 2024-09-13T00:04:11.232543+0000 mgr.y (mgr.24661) 812 : cluster [DBG] pgmap v1495: 105 pgs: 105 active+clean; 126 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:04:13.650 INFO:tasks.workunit.client.0.smithi136.stdout: 165: throughput=4.97MB/sec pending data=0 2024-09-13T00:04:13.650 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-YinxB5qwrkRhswW off=677061 len=1664083 2024-09-13T00:04:13.664 INFO:tasks.workunit.client.0.smithi136.stdout:op 865 completed, throughput=4.98MB/sec 2024-09-13T00:04:13.664 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-TuTN7kign620rJh off=4339740 len=1027423 2024-09-13T00:04:13.668 INFO:tasks.workunit.client.0.smithi136.stdout:op 866 completed, throughput=4.99MB/sec 2024-09-13T00:04:13.668 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-sHRUnmHgNG63b8q off=1283379 len=804249 2024-09-13T00:04:13.675 INFO:tasks.workunit.client.0.smithi136.stdout:op 867 completed, throughput=4.99MB/sec 2024-09-13T00:04:13.675 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-txeSzz6LVShnnX1 off=7850337 len=410099 2024-09-13T00:04:13.681 INFO:tasks.workunit.client.0.smithi136.stdout:op 868 completed, throughput=4.99MB/sec 2024-09-13T00:04:13.681 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ptR-4KN33tstAd6 off=12282827 len=596698 2024-09-13T00:04:13.682 INFO:tasks.workunit.client.0.smithi136.stdout:op 869 completed, throughput=5MB/sec 2024-09-13T00:04:13.683 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-JHknwCQaopresgq off=2196682 len=1310226 2024-09-13T00:04:13.686 INFO:tasks.workunit.client.0.smithi136.stdout:op 870 completed, throughput=5MB/sec 2024-09-13T00:04:14.686 INFO:tasks.workunit.client.0.smithi136.stdout: 166: throughput=4.97MB/sec pending data=0 2024-09-13T00:04:14.687 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-AxfBwZgccW8xZxp off=4231125 len=1619174 2024-09-13T00:04:14.698 INFO:tasks.workunit.client.0.smithi136.stdout:op 871 completed, throughput=4.98MB/sec 2024-09-13T00:04:14.698 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-JGNStWKlhywFbxQ off=15831571 len=1801199 2024-09-13T00:04:14.703 INFO:tasks.workunit.client.0.smithi136.stdout:op 872 completed, throughput=4.99MB/sec 2024-09-13T00:04:14.703 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-99GR4HqUSN29L7W off=6857201 len=1345073 2024-09-13T00:04:14.707 INFO:tasks.workunit.client.0.smithi136.stdout:op 873 completed, throughput=5MB/sec 2024-09-13T00:04:14.707 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-JsHNhnp33_5zbwn off=1466239 len=1522596 2024-09-13T00:04:14.717 INFO:tasks.workunit.client.0.smithi136.stdout:op 874 completed, throughput=5.01MB/sec 2024-09-13T00:04:15.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:04:14 smithi136 bash[76371]: cluster 2024-09-13T00:04:13.233549+0000 mgr.y (mgr.24661) 813 : cluster [DBG] pgmap v1496: 105 pgs: 105 active+clean; 126 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 2.1 MiB/s rd, 3.4 MiB/s wr, 5 op/s 2024-09-13T00:04:15.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:04:14 smithi136 bash[75164]: cluster 2024-09-13T00:04:13.233549+0000 mgr.y (mgr.24661) 813 : cluster [DBG] pgmap v1496: 105 pgs: 105 active+clean; 126 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 2.1 MiB/s rd, 3.4 MiB/s wr, 5 op/s 2024-09-13T00:04:15.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:04:14 smithi181 bash[67587]: cluster 2024-09-13T00:04:13.233549+0000 mgr.y (mgr.24661) 813 : cluster [DBG] pgmap v1496: 105 pgs: 105 active+clean; 126 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 2.1 MiB/s rd, 3.4 MiB/s wr, 5 op/s 2024-09-13T00:04:15.718 INFO:tasks.workunit.client.0.smithi136.stdout: 167: throughput=4.98MB/sec pending data=0 2024-09-13T00:04:15.718 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-xAgl_C0nrnACkRK off=5980557 len=2030600 2024-09-13T00:04:15.734 INFO:tasks.workunit.client.0.smithi136.stdout:op 875 completed, throughput=4.99MB/sec 2024-09-13T00:04:15.735 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-uEzkzP08HI45unF off=2514484 len=460547 2024-09-13T00:04:15.736 INFO:tasks.workunit.client.0.smithi136.stdout:op 876 completed, throughput=4.99MB/sec 2024-09-13T00:04:15.736 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-IZrJ49A7f0c5TFm off=18279347 len=1099306 2024-09-13T00:04:15.740 INFO:tasks.workunit.client.0.smithi136.stdout:op 877 completed, throughput=5MB/sec 2024-09-13T00:04:15.740 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-KTpTPZVgD2mV3fT off=1162875 len=197079 2024-09-13T00:04:15.741 INFO:tasks.workunit.client.0.smithi136.stdout:op 878 completed, throughput=5MB/sec 2024-09-13T00:04:15.741 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-tfTbHIgBxHKm3MO off=2503198 len=1152464 2024-09-13T00:04:15.744 INFO:tasks.workunit.client.0.smithi136.stdout:op 879 completed, throughput=5MB/sec 2024-09-13T00:04:16.744 INFO:tasks.workunit.client.0.smithi136.stdout: 168: throughput=4.98MB/sec pending data=0 2024-09-13T00:04:16.745 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ppArA47C0y5yAJx off=6740314 len=1937871 2024-09-13T00:04:16.751 INFO:tasks.workunit.client.0.smithi136.stdout:op 880 completed, throughput=4.99MB/sec 2024-09-13T00:04:16.751 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-1J0BVuavJ_dXqRF off=108286 len=338557 2024-09-13T00:04:16.756 INFO:tasks.workunit.client.0.smithi136.stdout:op 881 completed, throughput=4.99MB/sec 2024-09-13T00:04:16.756 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-5Jki-q1SrQhLipC off=974582 len=1400420 2024-09-13T00:04:16.767 INFO:tasks.workunit.client.0.smithi136.stdout:op 882 completed, throughput=5MB/sec 2024-09-13T00:04:16.767 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-UNoFpVxjVrTmSmf off=1369987 len=1402045 2024-09-13T00:04:16.777 INFO:tasks.workunit.client.0.smithi136.stdout:op 883 completed, throughput=5MB/sec 2024-09-13T00:04:17.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:04:16 smithi136 bash[75164]: cluster 2024-09-13T00:04:15.234361+0000 mgr.y (mgr.24661) 814 : cluster [DBG] pgmap v1497: 105 pgs: 105 active+clean; 126 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 2.1 MiB/s rd, 3.1 MiB/s wr, 5 op/s 2024-09-13T00:04:17.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:04:16 smithi136 bash[76371]: cluster 2024-09-13T00:04:15.234361+0000 mgr.y (mgr.24661) 814 : cluster [DBG] pgmap v1497: 105 pgs: 105 active+clean; 126 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 2.1 MiB/s rd, 3.1 MiB/s wr, 5 op/s 2024-09-13T00:04:17.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:04:16 smithi181 bash[67587]: cluster 2024-09-13T00:04:15.234361+0000 mgr.y (mgr.24661) 814 : cluster [DBG] pgmap v1497: 105 pgs: 105 active+clean; 126 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 2.1 MiB/s rd, 3.1 MiB/s wr, 5 op/s 2024-09-13T00:04:17.778 INFO:tasks.workunit.client.0.smithi136.stdout: 169: throughput=4.97MB/sec pending data=0 2024-09-13T00:04:17.778 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj--ZdFTgHw0MgcuDK off=3495251 len=1179080 2024-09-13T00:04:17.781 INFO:tasks.workunit.client.0.smithi136.stdout:op 884 completed, throughput=4.98MB/sec 2024-09-13T00:04:17.781 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-FMa2n0E7RJmhCZ7 off=381537 len=466828 2024-09-13T00:04:17.784 INFO:tasks.workunit.client.0.smithi136.stdout:op 885 completed, throughput=4.98MB/sec 2024-09-13T00:04:17.784 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-HFNV_jrLep6H4QE off=17921370 len=830768 2024-09-13T00:04:17.787 INFO:tasks.workunit.client.0.smithi136.stdout:op 886 completed, throughput=4.99MB/sec 2024-09-13T00:04:17.787 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj--f_67xZWoF1ZGVp off=3223251 len=1772921 2024-09-13T00:04:17.805 INFO:tasks.workunit.client.0.smithi136.stdout:op 887 completed, throughput=5MB/sec 2024-09-13T00:04:17.805 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-oLFU3F0Mb_-kU-B off=766575 len=1700144 2024-09-13T00:04:17.816 INFO:tasks.workunit.client.0.smithi136.stdout:op 888 completed, throughput=5.01MB/sec 2024-09-13T00:04:18.816 INFO:tasks.workunit.client.0.smithi136.stdout: 170: throughput=4.98MB/sec pending data=0 2024-09-13T00:04:18.817 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-aHKfh2GwiASpMSd off=9131637 len=944689 2024-09-13T00:04:18.819 INFO:tasks.workunit.client.0.smithi136.stdout:op 889 completed, throughput=4.98MB/sec 2024-09-13T00:04:18.819 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-uQ0eswn60LXPHX8 off=4268309 len=1382843 2024-09-13T00:04:18.830 INFO:tasks.workunit.client.0.smithi136.stdout:op 890 completed, throughput=4.99MB/sec 2024-09-13T00:04:18.830 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-NZxCjfqT_Hz_SzZ off=6196410 len=1332950 2024-09-13T00:04:18.839 INFO:tasks.workunit.client.0.smithi136.stdout:op 891 completed, throughput=5MB/sec 2024-09-13T00:04:18.839 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-YDB5E-VMIx9frAS off=1764581 len=1139773 2024-09-13T00:04:18.842 INFO:tasks.workunit.client.0.smithi136.stdout:op 892 completed, throughput=5MB/sec 2024-09-13T00:04:19.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:04:18 smithi136 bash[75164]: cluster 2024-09-13T00:04:17.235495+0000 mgr.y (mgr.24661) 815 : cluster [DBG] pgmap v1498: 105 pgs: 105 active+clean; 126 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 2.5 MiB/s rd, 3.0 MiB/s wr, 5 op/s 2024-09-13T00:04:19.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:04:18 smithi136 bash[76371]: cluster 2024-09-13T00:04:17.235495+0000 mgr.y (mgr.24661) 815 : cluster [DBG] pgmap v1498: 105 pgs: 105 active+clean; 126 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 2.5 MiB/s rd, 3.0 MiB/s wr, 5 op/s 2024-09-13T00:04:19.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:04:18 smithi181 bash[67587]: cluster 2024-09-13T00:04:17.235495+0000 mgr.y (mgr.24661) 815 : cluster [DBG] pgmap v1498: 105 pgs: 105 active+clean; 126 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 2.5 MiB/s rd, 3.0 MiB/s wr, 5 op/s 2024-09-13T00:04:19.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:04:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:04:19] "GET /metrics HTTP/1.1" 200 34124 "" "Prometheus/2.43.0" 2024-09-13T00:04:19.842 INFO:tasks.workunit.client.0.smithi136.stdout: 171: throughput=4.97MB/sec pending data=0 2024-09-13T00:04:19.842 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-E6DksNud2ceUUml off=14371520 len=865457 2024-09-13T00:04:19.851 INFO:tasks.workunit.client.0.smithi136.stdout:op 893 completed, throughput=4.98MB/sec 2024-09-13T00:04:19.852 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-RHqLlztOo4ZMSIF off=1595189 len=1137058 2024-09-13T00:04:19.854 INFO:tasks.workunit.client.0.smithi136.stdout:op 894 completed, throughput=4.98MB/sec 2024-09-13T00:04:19.855 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-1XMdLy3krAMvvCd off=699308 len=1823808 2024-09-13T00:04:19.859 INFO:tasks.workunit.client.0.smithi136.stdout:op 895 completed, throughput=4.99MB/sec 2024-09-13T00:04:19.859 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-9qaSo5FO7p_I14p off=6323677 len=60716 2024-09-13T00:04:19.863 INFO:tasks.workunit.client.0.smithi136.stdout:op 896 completed, throughput=4.99MB/sec 2024-09-13T00:04:19.863 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-VxhcIM9I26gk7c- off=5335876 len=818603 2024-09-13T00:04:19.870 INFO:tasks.workunit.client.0.smithi136.stdout:op 897 completed, throughput=5MB/sec 2024-09-13T00:04:19.870 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-yRCAE5i9LhlnBsa off=596121 len=1565725 2024-09-13T00:04:19.881 INFO:tasks.workunit.client.0.smithi136.stdout:op 898 completed, throughput=5.01MB/sec 2024-09-13T00:04:20.882 INFO:tasks.workunit.client.0.smithi136.stdout: 172: throughput=4.98MB/sec pending data=0 2024-09-13T00:04:20.882 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-uLcoMfpL8IEey6h off=661342 len=845301 2024-09-13T00:04:20.889 INFO:tasks.workunit.client.0.smithi136.stdout:op 899 completed, throughput=4.98MB/sec 2024-09-13T00:04:20.890 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-GmerYEDH-kmhwrN off=2629906 len=935637 2024-09-13T00:04:20.893 INFO:tasks.workunit.client.0.smithi136.stdout:op 900 completed, throughput=4.99MB/sec 2024-09-13T00:04:20.893 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-2eZwZHsVMb-Hnoi off=4728971 len=1700679 2024-09-13T00:04:20.905 INFO:tasks.workunit.client.0.smithi136.stdout:op 901 completed, throughput=5MB/sec 2024-09-13T00:04:20.905 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-TOrDIvaoK9rN_zW off=2515530 len=95389 2024-09-13T00:04:20.909 INFO:tasks.workunit.client.0.smithi136.stdout:op 902 completed, throughput=5MB/sec 2024-09-13T00:04:20.910 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-wUvpD6T2ZBrGNV_ off=7864590 len=1072812 2024-09-13T00:04:20.912 INFO:tasks.workunit.client.0.smithi136.stdout:op 903 completed, throughput=5MB/sec 2024-09-13T00:04:21.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:04:20 smithi136 bash[75164]: cluster 2024-09-13T00:04:19.236413+0000 mgr.y (mgr.24661) 816 : cluster [DBG] pgmap v1499: 105 pgs: 105 active+clean; 126 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.8 MiB/s wr, 4 op/s 2024-09-13T00:04:21.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:04:20 smithi136 bash[76371]: cluster 2024-09-13T00:04:19.236413+0000 mgr.y (mgr.24661) 816 : cluster [DBG] pgmap v1499: 105 pgs: 105 active+clean; 126 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.8 MiB/s wr, 4 op/s 2024-09-13T00:04:21.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:04:20 smithi181 bash[67587]: cluster 2024-09-13T00:04:19.236413+0000 mgr.y (mgr.24661) 816 : cluster [DBG] pgmap v1499: 105 pgs: 105 active+clean; 126 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.8 MiB/s wr, 4 op/s 2024-09-13T00:04:21.913 INFO:tasks.workunit.client.0.smithi136.stdout: 173: throughput=4.97MB/sec pending data=0 2024-09-13T00:04:21.913 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj--ZdFTgHw0MgcuDK off=4303345 len=217667 2024-09-13T00:04:21.917 INFO:tasks.workunit.client.0.smithi136.stdout:op 904 completed, throughput=4.98MB/sec 2024-09-13T00:04:21.918 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-_JXh9Gdau60c3gh off=4805522 len=1780277 2024-09-13T00:04:21.921 INFO:tasks.workunit.client.0.smithi136.stdout:op 905 completed, throughput=4.99MB/sec 2024-09-13T00:04:21.922 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-UrJBvI7j-uBZKNB off=3686328 len=1950085 2024-09-13T00:04:21.927 INFO:tasks.workunit.client.0.smithi136.stdout:op 906 completed, throughput=5MB/sec 2024-09-13T00:04:21.927 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-bcCMr7fv4v9kSTd off=6295931 len=459865 2024-09-13T00:04:21.928 INFO:tasks.workunit.client.0.smithi136.stdout:op 907 completed, throughput=5MB/sec 2024-09-13T00:04:21.928 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-FUGECpMZVuCajmE off=259006 len=1183570 2024-09-13T00:04:21.940 INFO:tasks.workunit.client.0.smithi136.stdout:op 908 completed, throughput=5MB/sec 2024-09-13T00:04:22.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:04:22 smithi181 bash[95505]: ts=2024-09-13T00:04:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:04:22.940 INFO:tasks.workunit.client.0.smithi136.stdout: 174: throughput=4.98MB/sec pending data=0 2024-09-13T00:04:22.940 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-L8HQUxxVhTHw1_8 off=5401725 len=1660992 2024-09-13T00:04:22.953 INFO:tasks.workunit.client.0.smithi136.stdout:op 909 completed, throughput=4.98MB/sec 2024-09-13T00:04:22.953 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-PRAOOEz6oVfbKUH off=12644802 len=1232313 2024-09-13T00:04:22.962 INFO:tasks.workunit.client.0.smithi136.stdout:op 910 completed, throughput=4.99MB/sec 2024-09-13T00:04:22.962 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-7wqkO1HfE1AMYR6 off=8043540 len=120110 2024-09-13T00:04:22.964 INFO:tasks.workunit.client.0.smithi136.stdout:op 911 completed, throughput=4.99MB/sec 2024-09-13T00:04:22.964 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-LxUltW5oyBlxzHC off=3442159 len=203276 2024-09-13T00:04:22.969 INFO:tasks.workunit.client.0.smithi136.stdout:op 912 completed, throughput=4.99MB/sec 2024-09-13T00:04:22.969 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-FKxLmmdho2AqZ6O off=4644237 len=2046672 2024-09-13T00:04:22.981 INFO:tasks.workunit.client.0.smithi136.stdout:op 913 completed, throughput=5MB/sec 2024-09-13T00:04:23.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:04:22 smithi136 bash[75164]: cluster 2024-09-13T00:04:21.237428+0000 mgr.y (mgr.24661) 817 : cluster [DBG] pgmap v1500: 105 pgs: 105 active+clean; 126 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.7 MiB/s wr, 4 op/s 2024-09-13T00:04:23.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:04:22 smithi136 bash[76371]: cluster 2024-09-13T00:04:21.237428+0000 mgr.y (mgr.24661) 817 : cluster [DBG] pgmap v1500: 105 pgs: 105 active+clean; 126 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.7 MiB/s wr, 4 op/s 2024-09-13T00:04:23.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:04:22 smithi181 bash[67587]: cluster 2024-09-13T00:04:21.237428+0000 mgr.y (mgr.24661) 817 : cluster [DBG] pgmap v1500: 105 pgs: 105 active+clean; 126 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.7 MiB/s wr, 4 op/s 2024-09-13T00:04:23.982 INFO:tasks.workunit.client.0.smithi136.stdout: 175: throughput=4.98MB/sec pending data=0 2024-09-13T00:04:23.982 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-t_Fbff-_Tlrzn_I off=7512614 len=2081273 2024-09-13T00:04:23.989 INFO:tasks.workunit.client.0.smithi136.stdout:op 914 completed, throughput=4.99MB/sec 2024-09-13T00:04:23.989 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-jNNIPjA53mbv6Rg off=3120316 len=1085537 2024-09-13T00:04:23.998 INFO:tasks.workunit.client.0.smithi136.stdout:op 915 completed, throughput=4.99MB/sec 2024-09-13T00:04:23.998 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-WVjnUAdosOCnzPW off=2136426 len=1614678 2024-09-13T00:04:24.010 INFO:tasks.workunit.client.0.smithi136.stdout:op 916 completed, throughput=5MB/sec 2024-09-13T00:04:24.010 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-V3yAb83dLgAcZri off=1440443 len=1080271 2024-09-13T00:04:24.018 INFO:tasks.workunit.client.0.smithi136.stdout:op 917 completed, throughput=5MB/sec 2024-09-13T00:04:25.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:04:24 smithi136 bash[75164]: cluster 2024-09-13T00:04:23.238362+0000 mgr.y (mgr.24661) 818 : cluster [DBG] pgmap v1501: 105 pgs: 105 active+clean; 126 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.8 MiB/s wr, 4 op/s 2024-09-13T00:04:25.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:04:24 smithi136 bash[75164]: audit 2024-09-13T00:04:23.776170+0000 mon.a (mon.0) 2132 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:04:25.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:04:24 smithi136 bash[76371]: cluster 2024-09-13T00:04:23.238362+0000 mgr.y (mgr.24661) 818 : cluster [DBG] pgmap v1501: 105 pgs: 105 active+clean; 126 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.8 MiB/s wr, 4 op/s 2024-09-13T00:04:25.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:04:24 smithi136 bash[76371]: audit 2024-09-13T00:04:23.776170+0000 mon.a (mon.0) 2132 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:04:25.018 INFO:tasks.workunit.client.0.smithi136.stdout: 176: throughput=4.98MB/sec pending data=0 2024-09-13T00:04:25.019 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-q9P2SKeO8_LP-ZO off=12229820 len=1057738 2024-09-13T00:04:25.027 INFO:tasks.workunit.client.0.smithi136.stdout:op 918 completed, throughput=4.98MB/sec 2024-09-13T00:04:25.027 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-jhcfgiOqxA3fDDX off=3575545 len=930159 2024-09-13T00:04:25.030 INFO:tasks.workunit.client.0.smithi136.stdout:op 919 completed, throughput=4.99MB/sec 2024-09-13T00:04:25.030 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-n6h-se6NccG4_tZ off=18856935 len=1882808 2024-09-13T00:04:25.046 INFO:tasks.workunit.client.0.smithi136.stdout:op 920 completed, throughput=5MB/sec 2024-09-13T00:04:25.046 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-0FOxdf5yS4ecGwc off=7153535 len=964744 2024-09-13T00:04:25.048 INFO:tasks.workunit.client.0.smithi136.stdout:op 921 completed, throughput=5MB/sec 2024-09-13T00:04:25.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:04:24 smithi181 bash[67587]: cluster 2024-09-13T00:04:23.238362+0000 mgr.y (mgr.24661) 818 : cluster [DBG] pgmap v1501: 105 pgs: 105 active+clean; 126 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.8 MiB/s wr, 4 op/s 2024-09-13T00:04:25.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:04:24 smithi181 bash[67587]: audit 2024-09-13T00:04:23.776170+0000 mon.a (mon.0) 2132 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:04:26.048 INFO:tasks.workunit.client.0.smithi136.stdout: 177: throughput=4.97MB/sec pending data=0 2024-09-13T00:04:26.048 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-JeXqMKawSeZRl3B off=10942034 len=1032631 2024-09-13T00:04:26.057 INFO:tasks.workunit.client.0.smithi136.stdout:op 922 completed, throughput=4.98MB/sec 2024-09-13T00:04:26.057 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-CYsSEEwC-kqpqRc off=19001745 len=1695380 2024-09-13T00:04:26.069 INFO:tasks.workunit.client.0.smithi136.stdout:op 923 completed, throughput=4.99MB/sec 2024-09-13T00:04:26.069 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-AMrRVstkUToq8PC off=3818460 len=1729496 2024-09-13T00:04:26.080 INFO:tasks.workunit.client.0.smithi136.stdout:op 924 completed, throughput=5MB/sec 2024-09-13T00:04:26.080 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-hSX-iC4pTG_mJu1 off=9560952 len=336853 2024-09-13T00:04:26.081 INFO:tasks.workunit.client.0.smithi136.stdout:op 925 completed, throughput=5MB/sec 2024-09-13T00:04:26.081 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-bq9b-052aOWXO00 off=9313805 len=348461 2024-09-13T00:04:26.082 INFO:tasks.workunit.client.0.smithi136.stdout:op 926 completed, throughput=5MB/sec 2024-09-13T00:04:27.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:04:26 smithi136 bash[75164]: cluster 2024-09-13T00:04:25.239194+0000 mgr.y (mgr.24661) 819 : cluster [DBG] pgmap v1502: 105 pgs: 105 active+clean; 126 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:04:27.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:04:26 smithi136 bash[76371]: cluster 2024-09-13T00:04:25.239194+0000 mgr.y (mgr.24661) 819 : cluster [DBG] pgmap v1502: 105 pgs: 105 active+clean; 126 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:04:27.083 INFO:tasks.workunit.client.0.smithi136.stdout: 178: throughput=4.97MB/sec pending data=0 2024-09-13T00:04:27.083 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-jdDt72bctfuJtIm off=18616359 len=1523317 2024-09-13T00:04:27.096 INFO:tasks.workunit.client.0.smithi136.stdout:op 927 completed, throughput=4.98MB/sec 2024-09-13T00:04:27.096 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-sJZNr4gkKcBKvlM off=1210930 len=1766063 2024-09-13T00:04:27.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:04:26 smithi181 bash[67587]: cluster 2024-09-13T00:04:25.239194+0000 mgr.y (mgr.24661) 819 : cluster [DBG] pgmap v1502: 105 pgs: 105 active+clean; 126 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:04:27.102 INFO:tasks.workunit.client.0.smithi136.stdout:op 928 completed, throughput=4.99MB/sec 2024-09-13T00:04:27.102 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-eU1BDkmU6DC5BxF off=8267955 len=1252407 2024-09-13T00:04:27.105 INFO:tasks.workunit.client.0.smithi136.stdout:op 929 completed, throughput=5MB/sec 2024-09-13T00:04:27.105 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-dnQIDWnbiBwRFC2 off=1426896 len=1151510 2024-09-13T00:04:27.108 INFO:tasks.workunit.client.0.smithi136.stdout:op 930 completed, throughput=5MB/sec 2024-09-13T00:04:28.109 INFO:tasks.workunit.client.0.smithi136.stdout: 179: throughput=4.98MB/sec pending data=0 2024-09-13T00:04:28.109 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-NsViMDioT8klqlw off=3232167 len=869958 2024-09-13T00:04:28.111 INFO:tasks.workunit.client.0.smithi136.stdout:op 931 completed, throughput=4.98MB/sec 2024-09-13T00:04:28.112 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-8y4nmzz27iLGmgD off=2563678 len=111110 2024-09-13T00:04:28.116 INFO:tasks.workunit.client.0.smithi136.stdout:op 932 completed, throughput=4.98MB/sec 2024-09-13T00:04:28.116 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-RexV4FxS1w9Y8ga off=169951 len=230625 2024-09-13T00:04:28.118 INFO:tasks.workunit.client.0.smithi136.stdout:op 933 completed, throughput=4.98MB/sec 2024-09-13T00:04:28.118 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-pHE3C6aZowWhYCJ off=16327780 len=560749 2024-09-13T00:04:28.124 INFO:tasks.workunit.client.0.smithi136.stdout:op 934 completed, throughput=4.98MB/sec 2024-09-13T00:04:28.124 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-A9UeBJSNmhvpliP off=2867814 len=253607 2024-09-13T00:04:28.125 INFO:tasks.workunit.client.0.smithi136.stdout:op 935 completed, throughput=4.99MB/sec 2024-09-13T00:04:28.125 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-WL4ZjghOO1_5aq3 off=1180481 len=2010744 2024-09-13T00:04:28.130 INFO:tasks.workunit.client.0.smithi136.stdout:op 936 completed, throughput=5MB/sec 2024-09-13T00:04:28.130 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-18aYpgq0rqXMtIq off=49089 len=1620152 2024-09-13T00:04:28.141 INFO:tasks.workunit.client.0.smithi136.stdout:op 937 completed, throughput=5MB/sec 2024-09-13T00:04:29.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:04:28 smithi181 bash[67587]: cluster 2024-09-13T00:04:27.240313+0000 mgr.y (mgr.24661) 820 : cluster [DBG] pgmap v1503: 105 pgs: 105 active+clean; 126 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 2.5 MiB/s rd, 3.0 MiB/s wr, 5 op/s 2024-09-13T00:04:29.142 INFO:tasks.workunit.client.0.smithi136.stdout: 180: throughput=4.98MB/sec pending data=0 2024-09-13T00:04:29.142 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-HEjb3x8zh0lMyT5 off=6777960 len=1741780 2024-09-13T00:04:29.146 INFO:tasks.workunit.client.0.smithi136.stdout:op 938 completed, throughput=4.99MB/sec 2024-09-13T00:04:29.146 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-U7PsL4DhkPIDHC1 off=4264640 len=530137 2024-09-13T00:04:29.148 INFO:tasks.workunit.client.0.smithi136.stdout:op 939 completed, throughput=4.99MB/sec 2024-09-13T00:04:29.148 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-AqRn11gWE0KMg32 off=13766683 len=171791 2024-09-13T00:04:29.149 INFO:tasks.workunit.client.0.smithi136.stdout:op 940 completed, throughput=4.99MB/sec 2024-09-13T00:04:29.150 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-uIQUlBeX-GyP8Px off=10161054 len=486087 2024-09-13T00:04:29.151 INFO:tasks.workunit.client.0.smithi136.stdout:op 941 completed, throughput=4.99MB/sec 2024-09-13T00:04:29.152 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-nmwEKbT-2-LCije off=2416007 len=1079377 2024-09-13T00:04:29.160 INFO:tasks.workunit.client.0.smithi136.stdout:op 942 completed, throughput=5MB/sec 2024-09-13T00:04:29.160 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-cJkGw7V_iQwbJOR off=220905 len=2032676 2024-09-13T00:04:29.174 INFO:tasks.workunit.client.0.smithi136.stdout:op 943 completed, throughput=5.01MB/sec 2024-09-13T00:04:29.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:04:28 smithi136 bash[75164]: cluster 2024-09-13T00:04:27.240313+0000 mgr.y (mgr.24661) 820 : cluster [DBG] pgmap v1503: 105 pgs: 105 active+clean; 126 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 2.5 MiB/s rd, 3.0 MiB/s wr, 5 op/s 2024-09-13T00:04:29.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:04:28 smithi136 bash[76371]: cluster 2024-09-13T00:04:27.240313+0000 mgr.y (mgr.24661) 820 : cluster [DBG] pgmap v1503: 105 pgs: 105 active+clean; 126 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 2.5 MiB/s rd, 3.0 MiB/s wr, 5 op/s 2024-09-13T00:04:29.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:04:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:04:29] "GET /metrics HTTP/1.1" 200 34122 "" "Prometheus/2.43.0" 2024-09-13T00:04:30.174 INFO:tasks.workunit.client.0.smithi136.stdout: 181: throughput=4.98MB/sec pending data=0 2024-09-13T00:04:30.174 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-XEFWl29ZvoukAGQ off=12893747 len=745310 2024-09-13T00:04:30.176 INFO:tasks.workunit.client.0.smithi136.stdout:op 944 completed, throughput=4.98MB/sec 2024-09-13T00:04:30.176 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-O4zHCuu6wIxG6Ts off=14633717 len=1538780 2024-09-13T00:04:30.180 INFO:tasks.workunit.client.0.smithi136.stdout:op 945 completed, throughput=4.99MB/sec 2024-09-13T00:04:30.180 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-vq1qDrGTb9sv-ZB off=14566503 len=1404648 2024-09-13T00:04:30.192 INFO:tasks.workunit.client.0.smithi136.stdout:op 946 completed, throughput=5MB/sec 2024-09-13T00:04:30.192 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-iGd4VUdq4TYJDVW off=4385458 len=1644150 2024-09-13T00:04:30.197 INFO:tasks.workunit.client.0.smithi136.stdout:op 947 completed, throughput=5.01MB/sec 2024-09-13T00:04:31.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:04:30 smithi181 bash[67587]: cluster 2024-09-13T00:04:29.241244+0000 mgr.y (mgr.24661) 821 : cluster [DBG] pgmap v1504: 105 pgs: 105 active+clean; 126 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:04:31.198 INFO:tasks.workunit.client.0.smithi136.stdout: 182: throughput=4.98MB/sec pending data=0 2024-09-13T00:04:31.198 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-TiLY_z1NoavS74L off=3050707 len=1417185 2024-09-13T00:04:31.205 INFO:tasks.workunit.client.0.smithi136.stdout:op 948 completed, throughput=4.99MB/sec 2024-09-13T00:04:31.206 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-PmX6h2KoCm-AwAZ off=869164 len=1389322 2024-09-13T00:04:31.216 INFO:tasks.workunit.client.0.smithi136.stdout:op 949 completed, throughput=4.99MB/sec 2024-09-13T00:04:31.216 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-LD-9a6W_8j0-8WB off=6733329 len=334985 2024-09-13T00:04:31.218 INFO:tasks.workunit.client.0.smithi136.stdout:op 950 completed, throughput=5MB/sec 2024-09-13T00:04:31.218 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-TafkDJMMnmULuwl off=815690 len=1445104 2024-09-13T00:04:31.221 INFO:tasks.workunit.client.0.smithi136.stdout:op 951 completed, throughput=5MB/sec 2024-09-13T00:04:31.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:04:30 smithi136 bash[75164]: cluster 2024-09-13T00:04:29.241244+0000 mgr.y (mgr.24661) 821 : cluster [DBG] pgmap v1504: 105 pgs: 105 active+clean; 126 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:04:31.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:04:30 smithi136 bash[76371]: cluster 2024-09-13T00:04:29.241244+0000 mgr.y (mgr.24661) 821 : cluster [DBG] pgmap v1504: 105 pgs: 105 active+clean; 126 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:04:32.221 INFO:tasks.workunit.client.0.smithi136.stdout: 183: throughput=4.98MB/sec pending data=0 2024-09-13T00:04:32.221 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-wCnVGu8iefdLV_a off=10752910 len=1198643 2024-09-13T00:04:32.225 INFO:tasks.workunit.client.0.smithi136.stdout:op 952 completed, throughput=4.98MB/sec 2024-09-13T00:04:32.225 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-dFKRh3s0h5JuzmB off=10276872 len=1694250 2024-09-13T00:04:32.238 INFO:tasks.workunit.client.0.smithi136.stdout:op 953 completed, throughput=4.99MB/sec 2024-09-13T00:04:32.238 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-yC914n9ZAOqbplz off=0 len=1097886 2024-09-13T00:04:32.241 INFO:tasks.workunit.client.0.smithi136.stdout:op 954 completed, throughput=5MB/sec 2024-09-13T00:04:32.241 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-UQzqAAJRc4hQsm5 off=9126 len=283148 2024-09-13T00:04:32.242 INFO:tasks.workunit.client.0.smithi136.stdout:op 955 completed, throughput=5MB/sec 2024-09-13T00:04:32.242 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-XvJO_m7_eRNfZ0G off=1910617 len=1733018 2024-09-13T00:04:32.246 INFO:tasks.workunit.client.0.smithi136.stdout:op 956 completed, throughput=5.01MB/sec 2024-09-13T00:04:32.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:04:32 smithi181 bash[95505]: ts=2024-09-13T00:04:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:04:33.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:04:32 smithi181 bash[67587]: cluster 2024-09-13T00:04:31.242370+0000 mgr.y (mgr.24661) 822 : cluster [DBG] pgmap v1505: 105 pgs: 105 active+clean; 126 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:04:33.247 INFO:tasks.workunit.client.0.smithi136.stdout: 184: throughput=4.98MB/sec pending data=0 2024-09-13T00:04:33.247 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-gjcU6SpgNUrtxX9 off=766993 len=151885 2024-09-13T00:04:33.248 INFO:tasks.workunit.client.0.smithi136.stdout:op 957 completed, throughput=4.98MB/sec 2024-09-13T00:04:33.249 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-8Y81WfFy-gw72pn off=316642 len=1695768 2024-09-13T00:04:33.254 INFO:tasks.workunit.client.0.smithi136.stdout:op 958 completed, throughput=4.99MB/sec 2024-09-13T00:04:33.254 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-H2ku3USbP2pp50q off=18489412 len=1238430 2024-09-13T00:04:33.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:04:32 smithi136 bash[75164]: cluster 2024-09-13T00:04:31.242370+0000 mgr.y (mgr.24661) 822 : cluster [DBG] pgmap v1505: 105 pgs: 105 active+clean; 126 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:04:33.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:04:32 smithi136 bash[76371]: cluster 2024-09-13T00:04:31.242370+0000 mgr.y (mgr.24661) 822 : cluster [DBG] pgmap v1505: 105 pgs: 105 active+clean; 126 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:04:33.266 INFO:tasks.workunit.client.0.smithi136.stdout:op 959 completed, throughput=4.99MB/sec 2024-09-13T00:04:33.266 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-9qDqhr-N6yk4t7q off=10308087 len=1567442 2024-09-13T00:04:33.278 INFO:tasks.workunit.client.0.smithi136.stdout:op 960 completed, throughput=5MB/sec 2024-09-13T00:04:34.278 INFO:tasks.workunit.client.0.smithi136.stdout: 185: throughput=4.98MB/sec pending data=0 2024-09-13T00:04:34.278 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-pbzmfyBG656m4HX off=11612596 len=1016527 2024-09-13T00:04:34.280 INFO:tasks.workunit.client.0.smithi136.stdout:op 961 completed, throughput=4.98MB/sec 2024-09-13T00:04:34.281 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-YqomyBOQ08Q-De7 off=10286958 len=59618 2024-09-13T00:04:34.285 INFO:tasks.workunit.client.0.smithi136.stdout:op 962 completed, throughput=4.98MB/sec 2024-09-13T00:04:34.285 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-FM3eK7SFmv2eYt1 off=9806981 len=1470132 2024-09-13T00:04:34.288 INFO:tasks.workunit.client.0.smithi136.stdout:op 963 completed, throughput=4.99MB/sec 2024-09-13T00:04:34.288 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-KdBKgAVYSkhiQTk off=6524000 len=2077926 2024-09-13T00:04:34.306 INFO:tasks.workunit.client.0.smithi136.stdout:op 964 completed, throughput=5MB/sec 2024-09-13T00:04:34.306 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-D_LnGSwmxe0TPDm off=12078235 len=1676254 2024-09-13T00:04:34.319 INFO:tasks.workunit.client.0.smithi136.stdout:op 965 completed, throughput=5.01MB/sec 2024-09-13T00:04:35.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:04:34 smithi181 bash[67587]: cluster 2024-09-13T00:04:33.243477+0000 mgr.y (mgr.24661) 823 : cluster [DBG] pgmap v1506: 105 pgs: 105 active+clean; 126 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 1.8 MiB/s rd, 2.7 MiB/s wr, 4 op/s 2024-09-13T00:04:35.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:04:34 smithi136 bash[75164]: cluster 2024-09-13T00:04:33.243477+0000 mgr.y (mgr.24661) 823 : cluster [DBG] pgmap v1506: 105 pgs: 105 active+clean; 126 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 1.8 MiB/s rd, 2.7 MiB/s wr, 4 op/s 2024-09-13T00:04:35.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:04:34 smithi136 bash[76371]: cluster 2024-09-13T00:04:33.243477+0000 mgr.y (mgr.24661) 823 : cluster [DBG] pgmap v1506: 105 pgs: 105 active+clean; 126 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 1.8 MiB/s rd, 2.7 MiB/s wr, 4 op/s 2024-09-13T00:04:35.320 INFO:tasks.workunit.client.0.smithi136.stdout: 186: throughput=4.98MB/sec pending data=0 2024-09-13T00:04:35.320 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-C1OdJEkPuSvsJTP off=393109 len=693679 2024-09-13T00:04:35.323 INFO:tasks.workunit.client.0.smithi136.stdout:op 966 completed, throughput=4.98MB/sec 2024-09-13T00:04:35.323 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-0LNNQbEYf7td_Mv off=1035064 len=859026 2024-09-13T00:04:35.331 INFO:tasks.workunit.client.0.smithi136.stdout:op 967 completed, throughput=4.99MB/sec 2024-09-13T00:04:35.331 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-7Mop19PBN7xc5ne off=15485817 len=161486 2024-09-13T00:04:35.336 INFO:tasks.workunit.client.0.smithi136.stdout:op 968 completed, throughput=4.99MB/sec 2024-09-13T00:04:35.336 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-TZ7_thyjkte5ODB off=2548305 len=556763 2024-09-13T00:04:35.342 INFO:tasks.workunit.client.0.smithi136.stdout:op 969 completed, throughput=4.99MB/sec 2024-09-13T00:04:35.342 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj--9TeIZlSZnGN0xG off=5106719 len=754664 2024-09-13T00:04:35.348 INFO:tasks.workunit.client.0.smithi136.stdout:op 970 completed, throughput=4.99MB/sec 2024-09-13T00:04:35.349 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-XgS2w5whA9G6rOf off=0 len=2031298 2024-09-13T00:04:35.365 INFO:tasks.workunit.client.0.smithi136.stdout:op 971 completed, throughput=5MB/sec 2024-09-13T00:04:36.366 INFO:tasks.workunit.client.0.smithi136.stdout: 187: throughput=4.98MB/sec pending data=0 2024-09-13T00:04:36.366 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-_D9Sp6KF2Bl0RiN off=4515679 len=102402 2024-09-13T00:04:36.367 INFO:tasks.workunit.client.0.smithi136.stdout:op 972 completed, throughput=4.98MB/sec 2024-09-13T00:04:36.367 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-RvnQeNnhGiCkZLV off=19885941 len=251186 2024-09-13T00:04:36.369 INFO:tasks.workunit.client.0.smithi136.stdout:op 973 completed, throughput=4.98MB/sec 2024-09-13T00:04:36.369 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-5mZy4JBMT0pB7WX off=2826172 len=1995289 2024-09-13T00:04:36.373 INFO:tasks.workunit.client.0.smithi136.stdout:op 974 completed, throughput=4.99MB/sec 2024-09-13T00:04:36.373 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-gdNGt5Omci4YS20 off=376784 len=1802203 2024-09-13T00:04:36.384 INFO:tasks.workunit.client.0.smithi136.stdout:op 975 completed, throughput=5MB/sec 2024-09-13T00:04:36.385 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-YxtYLhCyjU0t0iI off=0 len=1545143 2024-09-13T00:04:36.388 INFO:tasks.workunit.client.0.smithi136.stdout:op 976 completed, throughput=5.01MB/sec 2024-09-13T00:04:37.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:04:36 smithi181 bash[67587]: cluster 2024-09-13T00:04:35.244399+0000 mgr.y (mgr.24661) 824 : cluster [DBG] pgmap v1507: 105 pgs: 105 active+clean; 126 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:04:37.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:04:36 smithi136 bash[75164]: cluster 2024-09-13T00:04:35.244399+0000 mgr.y (mgr.24661) 824 : cluster [DBG] pgmap v1507: 105 pgs: 105 active+clean; 126 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:04:37.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:04:36 smithi136 bash[76371]: cluster 2024-09-13T00:04:35.244399+0000 mgr.y (mgr.24661) 824 : cluster [DBG] pgmap v1507: 105 pgs: 105 active+clean; 126 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:04:37.389 INFO:tasks.workunit.client.0.smithi136.stdout: 188: throughput=4.98MB/sec pending data=0 2024-09-13T00:04:37.389 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-pLzUXN9KYO1UYId off=15031475 len=338141 2024-09-13T00:04:37.394 INFO:tasks.workunit.client.0.smithi136.stdout:op 977 completed, throughput=4.98MB/sec 2024-09-13T00:04:37.394 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-0J4Ry2YIn7m8HSV off=3591000 len=1258004 2024-09-13T00:04:37.404 INFO:tasks.workunit.client.0.smithi136.stdout:op 978 completed, throughput=4.99MB/sec 2024-09-13T00:04:37.404 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-fhFcFLbA85VmyWC off=17646276 len=740085 2024-09-13T00:04:37.411 INFO:tasks.workunit.client.0.smithi136.stdout:op 979 completed, throughput=4.99MB/sec 2024-09-13T00:04:37.411 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-mDh3TdCcTnO90WS off=2567167 len=1278679 2024-09-13T00:04:37.420 INFO:tasks.workunit.client.0.smithi136.stdout:op 980 completed, throughput=5MB/sec 2024-09-13T00:04:37.420 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ZXf6ir1MOWIvTXn off=722033 len=1046057 2024-09-13T00:04:37.421 INFO:tasks.workunit.client.0.smithi136.stdout:op 981 completed, throughput=5MB/sec 2024-09-13T00:04:38.422 INFO:tasks.workunit.client.0.smithi136.stdout: 189: throughput=4.98MB/sec pending data=0 2024-09-13T00:04:38.422 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-mTtC_e0FsVGY50J off=554136 len=2075846 2024-09-13T00:04:38.428 INFO:tasks.workunit.client.0.smithi136.stdout:op 982 completed, throughput=4.99MB/sec 2024-09-13T00:04:38.428 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-1bMTGJB5lju03f9 off=857319 len=1429778 2024-09-13T00:04:38.439 INFO:tasks.workunit.client.0.smithi136.stdout:op 983 completed, throughput=4.99MB/sec 2024-09-13T00:04:38.439 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-IcHRIiHmjInDuQI off=964809 len=1631660 2024-09-13T00:04:38.451 INFO:tasks.workunit.client.0.smithi136.stdout:op 984 completed, throughput=5MB/sec 2024-09-13T00:04:39.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:04:38 smithi181 bash[67587]: cluster 2024-09-13T00:04:37.245643+0000 mgr.y (mgr.24661) 825 : cluster [DBG] pgmap v1508: 105 pgs: 105 active+clean; 126 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.7 MiB/s wr, 4 op/s 2024-09-13T00:04:39.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:04:38 smithi181 bash[67587]: audit 2024-09-13T00:04:38.776704+0000 mon.a (mon.0) 2133 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:04:39.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:04:38 smithi136 bash[75164]: cluster 2024-09-13T00:04:37.245643+0000 mgr.y (mgr.24661) 825 : cluster [DBG] pgmap v1508: 105 pgs: 105 active+clean; 126 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.7 MiB/s wr, 4 op/s 2024-09-13T00:04:39.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:04:38 smithi136 bash[75164]: audit 2024-09-13T00:04:38.776704+0000 mon.a (mon.0) 2133 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:04:39.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:04:38 smithi136 bash[76371]: cluster 2024-09-13T00:04:37.245643+0000 mgr.y (mgr.24661) 825 : cluster [DBG] pgmap v1508: 105 pgs: 105 active+clean; 126 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.7 MiB/s wr, 4 op/s 2024-09-13T00:04:39.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:04:38 smithi136 bash[76371]: audit 2024-09-13T00:04:38.776704+0000 mon.a (mon.0) 2133 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:04:39.451 INFO:tasks.workunit.client.0.smithi136.stdout: 190: throughput=4.97MB/sec pending data=0 2024-09-13T00:04:39.452 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Oj9BEcHaUGqcQXl off=5584472 len=1165930 2024-09-13T00:04:39.455 INFO:tasks.workunit.client.0.smithi136.stdout:op 985 completed, throughput=4.98MB/sec 2024-09-13T00:04:39.455 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-C4hPj47rr4ShLQL off=1304701 len=2077226 2024-09-13T00:04:39.472 INFO:tasks.workunit.client.0.smithi136.stdout:op 986 completed, throughput=4.99MB/sec 2024-09-13T00:04:39.472 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-lMOF-XbKwhtxNNX off=1894850 len=1657097 2024-09-13T00:04:39.477 INFO:tasks.workunit.client.0.smithi136.stdout:op 987 completed, throughput=5MB/sec 2024-09-13T00:04:39.477 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-mE1zSOQrbnxLL-U off=20648883 len=1533674 2024-09-13T00:04:39.481 INFO:tasks.workunit.client.0.smithi136.stdout:op 988 completed, throughput=5.01MB/sec 2024-09-13T00:04:39.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:04:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:04:39] "GET /metrics HTTP/1.1" 200 34128 "" "Prometheus/2.43.0" 2024-09-13T00:04:40.482 INFO:tasks.workunit.client.0.smithi136.stdout: 191: throughput=4.98MB/sec pending data=0 2024-09-13T00:04:40.482 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ku73_865_b39IWz off=4003613 len=24291 2024-09-13T00:04:40.486 INFO:tasks.workunit.client.0.smithi136.stdout:op 989 completed, throughput=4.98MB/sec 2024-09-13T00:04:40.486 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-CTWlVSETM9mwCGL off=254459 len=1087348 2024-09-13T00:04:40.495 INFO:tasks.workunit.client.0.smithi136.stdout:op 990 completed, throughput=4.99MB/sec 2024-09-13T00:04:40.495 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-xaJOSJyayRVNuzM off=2101187 len=1230983 2024-09-13T00:04:40.500 INFO:tasks.workunit.client.0.smithi136.stdout:op 991 completed, throughput=4.99MB/sec 2024-09-13T00:04:40.500 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-rlPUitirJBgemwO off=2160780 len=204549 2024-09-13T00:04:40.505 INFO:tasks.workunit.client.0.smithi136.stdout:op 992 completed, throughput=4.99MB/sec 2024-09-13T00:04:40.505 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-W2Rc-70RN9IGPgv off=3507999 len=1389264 2024-09-13T00:04:40.508 INFO:tasks.workunit.client.0.smithi136.stdout:op 993 completed, throughput=5MB/sec 2024-09-13T00:04:40.508 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-3aEBloDkk9md7mJ off=5857805 len=433922 2024-09-13T00:04:40.514 INFO:tasks.workunit.client.0.smithi136.stdout:op 994 completed, throughput=5MB/sec 2024-09-13T00:04:41.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:04:40 smithi181 bash[67587]: cluster 2024-09-13T00:04:39.246589+0000 mgr.y (mgr.24661) 826 : cluster [DBG] pgmap v1509: 105 pgs: 105 active+clean; 126 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:04:41.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:04:40 smithi136 bash[75164]: cluster 2024-09-13T00:04:39.246589+0000 mgr.y (mgr.24661) 826 : cluster [DBG] pgmap v1509: 105 pgs: 105 active+clean; 126 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:04:41.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:04:40 smithi136 bash[76371]: cluster 2024-09-13T00:04:39.246589+0000 mgr.y (mgr.24661) 826 : cluster [DBG] pgmap v1509: 105 pgs: 105 active+clean; 126 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:04:41.514 INFO:tasks.workunit.client.0.smithi136.stdout: 192: throughput=4.98MB/sec pending data=0 2024-09-13T00:04:41.515 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ArE9LTTGCFJPFL_ off=18622037 len=23842 2024-09-13T00:04:41.515 INFO:tasks.workunit.client.0.smithi136.stdout:op 995 completed, throughput=4.98MB/sec 2024-09-13T00:04:41.515 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Mf-uZ_uu3drZiL1 off=12303000 len=378942 2024-09-13T00:04:41.521 INFO:tasks.workunit.client.0.smithi136.stdout:op 996 completed, throughput=4.98MB/sec 2024-09-13T00:04:41.521 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-AGVQFdN3LWrhh2z off=13416701 len=1229805 2024-09-13T00:04:41.524 INFO:tasks.workunit.client.0.smithi136.stdout:op 997 completed, throughput=4.98MB/sec 2024-09-13T00:04:41.524 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Aka11Yhib8Ff5MZ off=11007746 len=138504 2024-09-13T00:04:41.529 INFO:tasks.workunit.client.0.smithi136.stdout:op 998 completed, throughput=4.98MB/sec 2024-09-13T00:04:41.529 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-mB9AlOJoNiwCzWR off=6190516 len=526129 2024-09-13T00:04:41.535 INFO:tasks.workunit.client.0.smithi136.stdout:op 999 completed, throughput=4.99MB/sec 2024-09-13T00:04:41.535 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-YYL5I9GMlHCh7s6 off=13800274 len=905766 2024-09-13T00:04:41.544 INFO:tasks.workunit.client.0.smithi136.stdout:op 1000 completed, throughput=4.99MB/sec 2024-09-13T00:04:41.544 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-clx0UtHlX6eRgb- off=11368997 len=1528134 2024-09-13T00:04:41.556 INFO:tasks.workunit.client.0.smithi136.stdout:op 1001 completed, throughput=5MB/sec 2024-09-13T00:04:41.556 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-QOWCIAUGmBXh5ry off=8775924 len=1088413 2024-09-13T00:04:41.565 INFO:tasks.workunit.client.0.smithi136.stdout:op 1002 completed, throughput=5MB/sec 2024-09-13T00:04:42.565 INFO:tasks.workunit.client.0.smithi136.stdout: 193: throughput=4.98MB/sec pending data=0 2024-09-13T00:04:42.565 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-4P7JyGAkYaEMxHU off=5575835 len=15426 2024-09-13T00:04:42.566 INFO:tasks.workunit.client.0.smithi136.stdout:op 1003 completed, throughput=4.98MB/sec 2024-09-13T00:04:42.566 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-C30Jkr48ltKrJgi off=3495940 len=1565314 2024-09-13T00:04:42.570 INFO:tasks.workunit.client.0.smithi136.stdout:op 1004 completed, throughput=4.98MB/sec 2024-09-13T00:04:42.570 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-L-W_x9Q-2T52IEt off=14976788 len=738184 2024-09-13T00:04:42.573 INFO:tasks.workunit.client.0.smithi136.stdout:op 1005 completed, throughput=4.99MB/sec 2024-09-13T00:04:42.573 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-5c2hrPzJcquPveO off=12514785 len=1121480 2024-09-13T00:04:42.583 INFO:tasks.workunit.client.0.smithi136.stdout:op 1006 completed, throughput=4.99MB/sec 2024-09-13T00:04:42.583 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-zZ18CnSoRRAAejc off=2428385 len=1728114 2024-09-13T00:04:42.587 INFO:tasks.workunit.client.0.smithi136.stdout:op 1007 completed, throughput=5MB/sec 2024-09-13T00:04:42.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:04:42 smithi181 bash[95505]: ts=2024-09-13T00:04:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:04:43.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:04:42 smithi181 bash[67587]: cluster 2024-09-13T00:04:41.247568+0000 mgr.y (mgr.24661) 827 : cluster [DBG] pgmap v1510: 105 pgs: 105 active+clean; 126 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:04:43.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:04:42 smithi181 bash[67587]: audit 2024-09-13T00:04:42.692431+0000 mon.a (mon.0) 2134 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:04:43.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:04:42 smithi136 bash[75164]: cluster 2024-09-13T00:04:41.247568+0000 mgr.y (mgr.24661) 827 : cluster [DBG] pgmap v1510: 105 pgs: 105 active+clean; 126 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:04:43.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:04:42 smithi136 bash[75164]: audit 2024-09-13T00:04:42.692431+0000 mon.a (mon.0) 2134 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:04:43.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:04:42 smithi136 bash[76371]: cluster 2024-09-13T00:04:41.247568+0000 mgr.y (mgr.24661) 827 : cluster [DBG] pgmap v1510: 105 pgs: 105 active+clean; 126 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:04:43.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:04:42 smithi136 bash[76371]: audit 2024-09-13T00:04:42.692431+0000 mon.a (mon.0) 2134 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:04:43.588 INFO:tasks.workunit.client.0.smithi136.stdout: 194: throughput=4.98MB/sec pending data=0 2024-09-13T00:04:43.588 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-hFc7egzQ3uiVRMB off=12927301 len=1103540 2024-09-13T00:04:43.596 INFO:tasks.workunit.client.0.smithi136.stdout:op 1008 completed, throughput=4.98MB/sec 2024-09-13T00:04:43.596 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Gq-pdG-XTnIMYHT off=6785249 len=305952 2024-09-13T00:04:43.602 INFO:tasks.workunit.client.0.smithi136.stdout:op 1009 completed, throughput=4.98MB/sec 2024-09-13T00:04:43.602 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-qvp4F4u86otPOLn off=13479194 len=1824743 2024-09-13T00:04:43.616 INFO:tasks.workunit.client.0.smithi136.stdout:op 1010 completed, throughput=4.99MB/sec 2024-09-13T00:04:43.616 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-lV-QahN0wcd9B4N off=3135922 len=1636277 2024-09-13T00:04:43.620 INFO:tasks.workunit.client.0.smithi136.stdout:op 1011 completed, throughput=5MB/sec 2024-09-13T00:04:43.620 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-tIsaO8nJlF7HPX5 off=5782283 len=921872 2024-09-13T00:04:43.628 INFO:tasks.workunit.client.0.smithi136.stdout:op 1012 completed, throughput=5MB/sec 2024-09-13T00:04:44.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:04:43 smithi181 bash[67587]: audit 2024-09-13T00:04:43.267377+0000 mon.a (mon.0) 2135 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:04:44.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:04:43 smithi181 bash[67587]: audit 2024-09-13T00:04:43.269250+0000 mon.a (mon.0) 2136 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:04:44.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:04:43 smithi181 bash[67587]: audit 2024-09-13T00:04:43.282913+0000 mon.a (mon.0) 2137 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:04:44.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:04:43 smithi136 bash[75164]: audit 2024-09-13T00:04:43.267377+0000 mon.a (mon.0) 2135 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:04:44.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:04:43 smithi136 bash[75164]: audit 2024-09-13T00:04:43.269250+0000 mon.a (mon.0) 2136 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:04:44.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:04:43 smithi136 bash[75164]: audit 2024-09-13T00:04:43.282913+0000 mon.a (mon.0) 2137 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:04:44.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:04:43 smithi136 bash[76371]: audit 2024-09-13T00:04:43.267377+0000 mon.a (mon.0) 2135 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:04:44.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:04:43 smithi136 bash[76371]: audit 2024-09-13T00:04:43.269250+0000 mon.a (mon.0) 2136 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:04:44.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:04:43 smithi136 bash[76371]: audit 2024-09-13T00:04:43.282913+0000 mon.a (mon.0) 2137 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:04:44.629 INFO:tasks.workunit.client.0.smithi136.stdout: 195: throughput=4.98MB/sec pending data=0 2024-09-13T00:04:44.629 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-pUsneX9tqohZljh off=3524971 len=284925 2024-09-13T00:04:44.634 INFO:tasks.workunit.client.0.smithi136.stdout:op 1013 completed, throughput=4.98MB/sec 2024-09-13T00:04:44.635 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-uMszFhRvrvcVcH0 off=19293177 len=2014296 2024-09-13T00:04:44.647 INFO:tasks.workunit.client.0.smithi136.stdout:op 1014 completed, throughput=4.99MB/sec 2024-09-13T00:04:44.647 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-e-ZSQSh9Bu-rjKl off=13905879 len=99798 2024-09-13T00:04:44.652 INFO:tasks.workunit.client.0.smithi136.stdout:op 1015 completed, throughput=4.99MB/sec 2024-09-13T00:04:44.652 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-zdYJOt6vpC3ekFT off=2640451 len=344447 2024-09-13T00:04:44.653 INFO:tasks.workunit.client.0.smithi136.stdout:op 1016 completed, throughput=4.99MB/sec 2024-09-13T00:04:44.653 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-hfqvqR7QTEW2u_A off=4084962 len=866670 2024-09-13T00:04:44.661 INFO:tasks.workunit.client.0.smithi136.stdout:op 1017 completed, throughput=4.99MB/sec 2024-09-13T00:04:44.661 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-NgsZD6ndSMPAF50 off=8061148 len=106376 2024-09-13T00:04:44.666 INFO:tasks.workunit.client.0.smithi136.stdout:op 1018 completed, throughput=4.99MB/sec 2024-09-13T00:04:44.666 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-e_ohZcmJJN9lgY_ off=3899387 len=55604 2024-09-13T00:04:44.671 INFO:tasks.workunit.client.0.smithi136.stdout:op 1019 completed, throughput=4.99MB/sec 2024-09-13T00:04:44.671 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-7Yarka7paDerRbC off=4317881 len=1665690 2024-09-13T00:04:44.675 INFO:tasks.workunit.client.0.smithi136.stdout:op 1020 completed, throughput=5MB/sec 2024-09-13T00:04:45.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:04:44 smithi181 bash[67587]: cluster 2024-09-13T00:04:43.248497+0000 mgr.y (mgr.24661) 828 : cluster [DBG] pgmap v1511: 105 pgs: 105 active+clean; 126 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:04:45.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:04:44 smithi136 bash[76371]: cluster 2024-09-13T00:04:43.248497+0000 mgr.y (mgr.24661) 828 : cluster [DBG] pgmap v1511: 105 pgs: 105 active+clean; 126 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:04:45.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:04:44 smithi136 bash[75164]: cluster 2024-09-13T00:04:43.248497+0000 mgr.y (mgr.24661) 828 : cluster [DBG] pgmap v1511: 105 pgs: 105 active+clean; 126 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:04:45.675 INFO:tasks.workunit.client.0.smithi136.stdout: 196: throughput=4.98MB/sec pending data=0 2024-09-13T00:04:45.675 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-I_Kn5u7lXdQ54eF off=1627602 len=853623 2024-09-13T00:04:45.678 INFO:tasks.workunit.client.0.smithi136.stdout:op 1021 completed, throughput=4.98MB/sec 2024-09-13T00:04:45.678 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-LjxYa6bOH5UnHH2 off=685432 len=1364913 2024-09-13T00:04:45.689 INFO:tasks.workunit.client.0.smithi136.stdout:op 1022 completed, throughput=4.99MB/sec 2024-09-13T00:04:45.689 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Sy3Onl4BwDFPWLU off=6166914 len=34122 2024-09-13T00:04:45.693 INFO:tasks.workunit.client.0.smithi136.stdout:op 1023 completed, throughput=4.99MB/sec 2024-09-13T00:04:45.693 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-yl2YqN6PLuab3Ph off=13197781 len=40282 2024-09-13T00:04:45.697 INFO:tasks.workunit.client.0.smithi136.stdout:op 1024 completed, throughput=4.99MB/sec 2024-09-13T00:04:45.697 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-FiUQ5PQ478m8Vcu off=7509019 len=1652714 2024-09-13T00:04:45.701 INFO:tasks.workunit.client.0.smithi136.stdout:op 1025 completed, throughput=5MB/sec 2024-09-13T00:04:45.701 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-4T-3ip7PYqgpDb9 off=927352 len=1907911 2024-09-13T00:04:45.713 INFO:tasks.workunit.client.0.smithi136.stdout:op 1026 completed, throughput=5MB/sec 2024-09-13T00:04:46.714 INFO:tasks.workunit.client.0.smithi136.stdout: 197: throughput=4.98MB/sec pending data=0 2024-09-13T00:04:46.714 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-u3hcli0MRAjRVjC off=10730106 len=1193866 2024-09-13T00:04:46.723 INFO:tasks.workunit.client.0.smithi136.stdout:op 1027 completed, throughput=4.98MB/sec 2024-09-13T00:04:46.723 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-QhYkc1BjJXMXBcA off=10953769 len=296600 2024-09-13T00:04:46.728 INFO:tasks.workunit.client.0.smithi136.stdout:op 1028 completed, throughput=4.99MB/sec 2024-09-13T00:04:46.729 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Hxu55CHaEmvwK7A off=6766574 len=352999 2024-09-13T00:04:46.730 INFO:tasks.workunit.client.0.smithi136.stdout:op 1029 completed, throughput=4.99MB/sec 2024-09-13T00:04:46.730 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-dXFyMH7pXcjtG4S off=77422 len=1045920 2024-09-13T00:04:46.738 INFO:tasks.workunit.client.0.smithi136.stdout:op 1030 completed, throughput=4.99MB/sec 2024-09-13T00:04:46.738 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-y3J6msZo_glZeua off=487384 len=1779052 2024-09-13T00:04:46.749 INFO:tasks.workunit.client.0.smithi136.stdout:op 1031 completed, throughput=5MB/sec 2024-09-13T00:04:47.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:04:46 smithi181 bash[67587]: cluster 2024-09-13T00:04:45.249325+0000 mgr.y (mgr.24661) 829 : cluster [DBG] pgmap v1512: 105 pgs: 105 active+clean; 126 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:04:47.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:04:46 smithi136 bash[75164]: cluster 2024-09-13T00:04:45.249325+0000 mgr.y (mgr.24661) 829 : cluster [DBG] pgmap v1512: 105 pgs: 105 active+clean; 126 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:04:47.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:04:46 smithi136 bash[76371]: cluster 2024-09-13T00:04:45.249325+0000 mgr.y (mgr.24661) 829 : cluster [DBG] pgmap v1512: 105 pgs: 105 active+clean; 126 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:04:47.749 INFO:tasks.workunit.client.0.smithi136.stdout: 198: throughput=4.98MB/sec pending data=0 2024-09-13T00:04:47.750 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-qdXfvxbz6FDveRP off=2205929 len=147506 2024-09-13T00:04:47.754 INFO:tasks.workunit.client.0.smithi136.stdout:op 1032 completed, throughput=4.98MB/sec 2024-09-13T00:04:47.754 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-iIbbDvaDxJaRLDc off=9352227 len=1641918 2024-09-13T00:04:47.758 INFO:tasks.workunit.client.0.smithi136.stdout:op 1033 completed, throughput=4.98MB/sec 2024-09-13T00:04:47.758 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-X5eQEq8WrcEOb7I off=8683446 len=1622442 2024-09-13T00:04:47.762 INFO:tasks.workunit.client.0.smithi136.stdout:op 1034 completed, throughput=4.99MB/sec 2024-09-13T00:04:47.762 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-tfogj_WAH7JiPes off=1373241 len=1551418 2024-09-13T00:04:47.772 INFO:tasks.workunit.client.0.smithi136.stdout:op 1035 completed, throughput=5MB/sec 2024-09-13T00:04:47.772 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-UzCvqwI6NscnLZl off=2398039 len=312744 2024-09-13T00:04:47.777 INFO:tasks.workunit.client.0.smithi136.stdout:op 1036 completed, throughput=5MB/sec 2024-09-13T00:04:48.777 INFO:tasks.workunit.client.0.smithi136.stdout: 199: throughput=4.98MB/sec pending data=0 2024-09-13T00:04:48.778 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-iR-TuX1snNKnf92 off=4802076 len=121726 2024-09-13T00:04:48.778 INFO:tasks.workunit.client.0.smithi136.stdout:op 1037 completed, throughput=4.98MB/sec 2024-09-13T00:04:48.779 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-3E2AjZvaQbUO9SV off=554870 len=1017039 2024-09-13T00:04:48.788 INFO:tasks.workunit.client.0.smithi136.stdout:op 1038 completed, throughput=4.98MB/sec 2024-09-13T00:04:48.788 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-G8qwQAG3Yopj_2c off=3921145 len=1508755 2024-09-13T00:04:48.798 INFO:tasks.workunit.client.0.smithi136.stdout:op 1039 completed, throughput=4.99MB/sec 2024-09-13T00:04:48.798 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-6_RI1dWMf4kOSBf off=1725699 len=236764 2024-09-13T00:04:48.803 INFO:tasks.workunit.client.0.smithi136.stdout:op 1040 completed, throughput=4.99MB/sec 2024-09-13T00:04:48.803 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-pgd9maktiD7V3FQ off=4434524 len=845194 2024-09-13T00:04:48.807 INFO:tasks.workunit.client.0.smithi136.stdout:op 1041 completed, throughput=4.99MB/sec 2024-09-13T00:04:48.807 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-f2cNdNfbZWyXQhr off=3037395 len=1968744 2024-09-13T00:04:48.810 INFO:tasks.workunit.client.0.smithi136.stdout:op 1042 completed, throughput=5MB/sec 2024-09-13T00:04:49.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:04:48 smithi181 bash[67587]: cluster 2024-09-13T00:04:47.250464+0000 mgr.y (mgr.24661) 830 : cluster [DBG] pgmap v1513: 105 pgs: 105 active+clean; 126 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:04:49.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:04:48 smithi136 bash[75164]: cluster 2024-09-13T00:04:47.250464+0000 mgr.y (mgr.24661) 830 : cluster [DBG] pgmap v1513: 105 pgs: 105 active+clean; 126 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:04:49.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:04:48 smithi136 bash[76371]: cluster 2024-09-13T00:04:47.250464+0000 mgr.y (mgr.24661) 830 : cluster [DBG] pgmap v1513: 105 pgs: 105 active+clean; 126 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:04:49.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:04:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:04:49] "GET /metrics HTTP/1.1" 200 34128 "" "Prometheus/2.43.0" 2024-09-13T00:04:49.810 INFO:tasks.workunit.client.0.smithi136.stdout: 200: throughput=4.98MB/sec pending data=0 2024-09-13T00:04:49.810 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Hn3iFXGkqzKnMO- off=13420263 len=1459277 2024-09-13T00:04:49.822 INFO:tasks.workunit.client.0.smithi136.stdout:op 1043 completed, throughput=4.98MB/sec 2024-09-13T00:04:49.822 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-P2pbde1akvmHc3L off=12319525 len=1317374 2024-09-13T00:04:49.827 INFO:tasks.workunit.client.0.smithi136.stdout:op 1044 completed, throughput=4.99MB/sec 2024-09-13T00:04:49.827 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-R8AEGSKd4SS5zt1 off=179108 len=1579089 2024-09-13T00:04:49.839 INFO:tasks.workunit.client.0.smithi136.stdout:op 1045 completed, throughput=5MB/sec 2024-09-13T00:04:49.839 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-0vRs9ZQBW-vCHs8 off=6145521 len=360418 2024-09-13T00:04:49.845 INFO:tasks.workunit.client.0.smithi136.stdout:op 1046 completed, throughput=5MB/sec 2024-09-13T00:04:49.846 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-vhNfpbaUuATRgOS off=11218381 len=874311 2024-09-13T00:04:49.853 INFO:tasks.workunit.client.0.smithi136.stdout:op 1047 completed, throughput=5MB/sec 2024-09-13T00:04:50.853 INFO:tasks.workunit.client.0.smithi136.stdout: 201: throughput=4.98MB/sec pending data=0 2024-09-13T00:04:50.853 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ejDfJ96ZEFwcLyf off=1854473 len=1188777 2024-09-13T00:04:50.862 INFO:tasks.workunit.client.0.smithi136.stdout:op 1048 completed, throughput=4.98MB/sec 2024-09-13T00:04:50.862 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-c5fanqD5Ey5Bbgm off=5913537 len=745179 2024-09-13T00:04:50.869 INFO:tasks.workunit.client.0.smithi136.stdout:op 1049 completed, throughput=4.99MB/sec 2024-09-13T00:04:50.869 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-FATTyWi4HgAlZav off=18742720 len=46065 2024-09-13T00:04:50.870 INFO:tasks.workunit.client.0.smithi136.stdout:op 1050 completed, throughput=4.99MB/sec 2024-09-13T00:04:50.870 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Zv_SwpBwwbhI276 off=211272 len=1502248 2024-09-13T00:04:50.880 INFO:tasks.workunit.client.0.smithi136.stdout:op 1051 completed, throughput=4.99MB/sec 2024-09-13T00:04:50.881 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-gcijFIsfoT8HyyZ off=17705930 len=692015 2024-09-13T00:04:50.887 INFO:tasks.workunit.client.0.smithi136.stdout:op 1052 completed, throughput=5MB/sec 2024-09-13T00:04:50.887 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-7vbmEy_aH9vQf0K off=8530391 len=1761718 2024-09-13T00:04:50.891 INFO:tasks.workunit.client.0.smithi136.stdout:op 1053 completed, throughput=5MB/sec 2024-09-13T00:04:51.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:04:50 smithi181 bash[67587]: cluster 2024-09-13T00:04:49.251501+0000 mgr.y (mgr.24661) 831 : cluster [DBG] pgmap v1514: 105 pgs: 105 active+clean; 126 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:04:51.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:04:50 smithi136 bash[75164]: cluster 2024-09-13T00:04:49.251501+0000 mgr.y (mgr.24661) 831 : cluster [DBG] pgmap v1514: 105 pgs: 105 active+clean; 126 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:04:51.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:04:50 smithi136 bash[76371]: cluster 2024-09-13T00:04:49.251501+0000 mgr.y (mgr.24661) 831 : cluster [DBG] pgmap v1514: 105 pgs: 105 active+clean; 126 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:04:51.892 INFO:tasks.workunit.client.0.smithi136.stdout: 202: throughput=4.98MB/sec pending data=0 2024-09-13T00:04:51.892 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-5UD5CHj1MhVvR77 off=10293302 len=1111525 2024-09-13T00:04:51.901 INFO:tasks.workunit.client.0.smithi136.stdout:op 1054 completed, throughput=4.98MB/sec 2024-09-13T00:04:51.901 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-c0wbxLE_hmXc6Sh off=6774514 len=72559 2024-09-13T00:04:51.905 INFO:tasks.workunit.client.0.smithi136.stdout:op 1055 completed, throughput=4.99MB/sec 2024-09-13T00:04:51.906 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-nUdQXnVVbXaXp82 off=107883 len=660365 2024-09-13T00:04:51.913 INFO:tasks.workunit.client.0.smithi136.stdout:op 1056 completed, throughput=4.99MB/sec 2024-09-13T00:04:51.913 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-5G2sFuqZonMvCfU off=6453966 len=250622 2024-09-13T00:04:51.914 INFO:tasks.workunit.client.0.smithi136.stdout:op 1057 completed, throughput=4.99MB/sec 2024-09-13T00:04:51.914 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-NVmsutJY_UCgkFD off=2289484 len=548649 2024-09-13T00:04:51.917 INFO:tasks.workunit.client.0.smithi136.stdout:op 1058 completed, throughput=4.99MB/sec 2024-09-13T00:04:51.917 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-yYLyYIp5CFjfE44 off=2294836 len=15386 2024-09-13T00:04:51.921 INFO:tasks.workunit.client.0.smithi136.stdout:op 1059 completed, throughput=4.99MB/sec 2024-09-13T00:04:51.921 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-KqDft1CUEs7c-sX off=5317568 len=1759572 2024-09-13T00:04:51.924 INFO:tasks.workunit.client.0.smithi136.stdout:op 1060 completed, throughput=5MB/sec 2024-09-13T00:04:51.924 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ARZxPdrMzLDFaCl off=0 len=1548950 2024-09-13T00:04:51.928 INFO:tasks.workunit.client.0.smithi136.stdout:op 1061 completed, throughput=5.01MB/sec 2024-09-13T00:04:52.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:04:52 smithi181 bash[95505]: ts=2024-09-13T00:04:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:04:52.929 INFO:tasks.workunit.client.0.smithi136.stdout: 203: throughput=4.98MB/sec pending data=0 2024-09-13T00:04:52.929 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-HqdMkU5fNvQMApG off=3933554 len=1621169 2024-09-13T00:04:52.934 INFO:tasks.workunit.client.0.smithi136.stdout:op 1062 completed, throughput=4.99MB/sec 2024-09-13T00:04:52.934 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-c1y-gq90mQOoTfY off=1539095 len=382740 2024-09-13T00:04:52.941 INFO:tasks.workunit.client.0.smithi136.stdout:op 1063 completed, throughput=4.99MB/sec 2024-09-13T00:04:52.941 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-AhQhFZliegbdLmD off=9441479 len=1911451 2024-09-13T00:04:52.945 INFO:tasks.workunit.client.0.smithi136.stdout:op 1064 completed, throughput=5MB/sec 2024-09-13T00:04:53.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:04:52 smithi181 bash[67587]: cluster 2024-09-13T00:04:51.252620+0000 mgr.y (mgr.24661) 832 : cluster [DBG] pgmap v1515: 105 pgs: 105 active+clean; 126 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:04:53.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:04:52 smithi136 bash[75164]: cluster 2024-09-13T00:04:51.252620+0000 mgr.y (mgr.24661) 832 : cluster [DBG] pgmap v1515: 105 pgs: 105 active+clean; 126 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:04:53.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:04:52 smithi136 bash[76371]: cluster 2024-09-13T00:04:51.252620+0000 mgr.y (mgr.24661) 832 : cluster [DBG] pgmap v1515: 105 pgs: 105 active+clean; 126 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:04:53.946 INFO:tasks.workunit.client.0.smithi136.stdout: 204: throughput=4.98MB/sec pending data=0 2024-09-13T00:04:53.946 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-LGp6Ljqd4YOr2I3 off=135384 len=1019591 2024-09-13T00:04:53.949 INFO:tasks.workunit.client.0.smithi136.stdout:op 1065 completed, throughput=4.98MB/sec 2024-09-13T00:04:53.949 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-v856p8Uvo4482L5 off=14687964 len=402414 2024-09-13T00:04:53.950 INFO:tasks.workunit.client.0.smithi136.stdout:op 1066 completed, throughput=4.98MB/sec 2024-09-13T00:04:53.951 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-nEebHdRadbalqHg off=3707621 len=1928877 2024-09-13T00:04:53.955 INFO:tasks.workunit.client.0.smithi136.stdout:op 1067 completed, throughput=4.99MB/sec 2024-09-13T00:04:53.955 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Qy9y9YxD2osps0W off=7925556 len=355073 2024-09-13T00:04:53.957 INFO:tasks.workunit.client.0.smithi136.stdout:op 1068 completed, throughput=4.99MB/sec 2024-09-13T00:04:53.957 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-bWcAPAoTbHCpyDz off=100430 len=827563 2024-09-13T00:04:53.965 INFO:tasks.workunit.client.0.smithi136.stdout:op 1069 completed, throughput=5MB/sec 2024-09-13T00:04:53.966 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-NQwLRhFsdgx3-0s off=5571480 len=799980 2024-09-13T00:04:53.968 INFO:tasks.workunit.client.0.smithi136.stdout:op 1070 completed, throughput=5MB/sec 2024-09-13T00:04:53.968 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-rr5KCDbWLJE1CBZ off=4865000 len=1556539 2024-09-13T00:04:53.973 INFO:tasks.workunit.client.0.smithi136.stdout:op 1071 completed, throughput=5.01MB/sec 2024-09-13T00:04:54.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:04:53 smithi181 bash[67587]: audit 2024-09-13T00:04:53.777062+0000 mon.a (mon.0) 2138 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:04:54.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:04:53 smithi136 bash[75164]: audit 2024-09-13T00:04:53.777062+0000 mon.a (mon.0) 2138 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:04:54.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:04:53 smithi136 bash[76371]: audit 2024-09-13T00:04:53.777062+0000 mon.a (mon.0) 2138 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:04:54.974 INFO:tasks.workunit.client.0.smithi136.stdout: 205: throughput=4.98MB/sec pending data=0 2024-09-13T00:04:54.974 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Xw9Gn0vlq66N9sm off=3791027 len=477267 2024-09-13T00:04:54.975 INFO:tasks.workunit.client.0.smithi136.stdout:op 1072 completed, throughput=4.99MB/sec 2024-09-13T00:04:54.976 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-f2Pmw-0K_1LPEq1 off=17466643 len=308709 2024-09-13T00:04:54.981 INFO:tasks.workunit.client.0.smithi136.stdout:op 1073 completed, throughput=4.99MB/sec 2024-09-13T00:04:54.981 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ZLL00eWKRgx-lWk off=22002968 len=1698498 2024-09-13T00:04:54.993 INFO:tasks.workunit.client.0.smithi136.stdout:op 1074 completed, throughput=4.99MB/sec 2024-09-13T00:04:54.993 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_44FmzQvpSgAQxU off=3608636 len=261356 2024-09-13T00:04:54.998 INFO:tasks.workunit.client.0.smithi136.stdout:op 1075 completed, throughput=4.99MB/sec 2024-09-13T00:04:54.998 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-yqrQkVKWTKtTy0a off=0 len=1658425 2024-09-13T00:04:55.005 INFO:tasks.workunit.client.0.smithi136.stdout:op 1076 completed, throughput=5MB/sec 2024-09-13T00:04:55.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:04:54 smithi181 bash[67587]: cluster 2024-09-13T00:04:53.253738+0000 mgr.y (mgr.24661) 833 : cluster [DBG] pgmap v1516: 105 pgs: 105 active+clean; 126 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 1.6 MiB/s rd, 2.9 MiB/s wr, 4 op/s 2024-09-13T00:04:55.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:04:54 smithi136 bash[75164]: cluster 2024-09-13T00:04:53.253738+0000 mgr.y (mgr.24661) 833 : cluster [DBG] pgmap v1516: 105 pgs: 105 active+clean; 126 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 1.6 MiB/s rd, 2.9 MiB/s wr, 4 op/s 2024-09-13T00:04:55.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:04:54 smithi136 bash[76371]: cluster 2024-09-13T00:04:53.253738+0000 mgr.y (mgr.24661) 833 : cluster [DBG] pgmap v1516: 105 pgs: 105 active+clean; 126 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 1.6 MiB/s rd, 2.9 MiB/s wr, 4 op/s 2024-09-13T00:04:56.005 INFO:tasks.workunit.client.0.smithi136.stdout: 206: throughput=4.98MB/sec pending data=0 2024-09-13T00:04:56.005 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-VH2W1AU6_UHFT3z off=3395735 len=796906 2024-09-13T00:04:56.008 INFO:tasks.workunit.client.0.smithi136.stdout:op 1077 completed, throughput=4.98MB/sec 2024-09-13T00:04:56.008 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj--TsRZH_cQEleUA5 off=0 len=1189382 2024-09-13T00:04:56.011 INFO:tasks.workunit.client.0.smithi136.stdout:op 1078 completed, throughput=4.99MB/sec 2024-09-13T00:04:56.011 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ErCpZ9vDl7d3h7W off=22698016 len=1355863 2024-09-13T00:04:56.015 INFO:tasks.workunit.client.0.smithi136.stdout:op 1079 completed, throughput=4.99MB/sec 2024-09-13T00:04:56.015 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-jfzdEp7J4yB4bb- off=2407537 len=1429098 2024-09-13T00:04:56.026 INFO:tasks.workunit.client.0.smithi136.stdout:op 1080 completed, throughput=5MB/sec 2024-09-13T00:04:56.026 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-RVbs-JVotDLUHZT off=9523704 len=1351874 2024-09-13T00:04:56.028 INFO:tasks.workunit.client.0.smithi136.stdout:op 1081 completed, throughput=5.01MB/sec 2024-09-13T00:04:57.029 INFO:tasks.workunit.client.0.smithi136.stdout: 207: throughput=4.98MB/sec pending data=0 2024-09-13T00:04:57.029 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-qrxq6kn4gBnxFeb off=4803390 len=1507998 2024-09-13T00:04:57.033 INFO:tasks.workunit.client.0.smithi136.stdout:op 1082 completed, throughput=4.99MB/sec 2024-09-13T00:04:57.033 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-XIxYQIEmNHIveD6 off=3595511 len=859654 2024-09-13T00:04:57.041 INFO:tasks.workunit.client.0.smithi136.stdout:op 1083 completed, throughput=4.99MB/sec 2024-09-13T00:04:57.041 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-UnRVDKISJW6ZlBo off=1394778 len=772731 2024-09-13T00:04:57.048 INFO:tasks.workunit.client.0.smithi136.stdout:op 1084 completed, throughput=5MB/sec 2024-09-13T00:04:57.048 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-dieFuaweozlfAas off=1728871 len=159407 2024-09-13T00:04:57.054 INFO:tasks.workunit.client.0.smithi136.stdout:op 1085 completed, throughput=5MB/sec 2024-09-13T00:04:57.054 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-8h8i_ETSdlb-a9j off=13310771 len=1261104 2024-09-13T00:04:57.057 INFO:tasks.workunit.client.0.smithi136.stdout:op 1086 completed, throughput=5MB/sec 2024-09-13T00:04:57.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:04:56 smithi181 bash[67587]: cluster 2024-09-13T00:04:55.254702+0000 mgr.y (mgr.24661) 834 : cluster [DBG] pgmap v1517: 105 pgs: 105 active+clean; 126 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 1.9 MiB/s rd, 3.0 MiB/s wr, 5 op/s 2024-09-13T00:04:57.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:04:56 smithi136 bash[75164]: cluster 2024-09-13T00:04:55.254702+0000 mgr.y (mgr.24661) 834 : cluster [DBG] pgmap v1517: 105 pgs: 105 active+clean; 126 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 1.9 MiB/s rd, 3.0 MiB/s wr, 5 op/s 2024-09-13T00:04:57.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:04:56 smithi136 bash[76371]: cluster 2024-09-13T00:04:55.254702+0000 mgr.y (mgr.24661) 834 : cluster [DBG] pgmap v1517: 105 pgs: 105 active+clean; 126 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 1.9 MiB/s rd, 3.0 MiB/s wr, 5 op/s 2024-09-13T00:04:58.058 INFO:tasks.workunit.client.0.smithi136.stdout: 208: throughput=4.98MB/sec pending data=0 2024-09-13T00:04:58.058 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-A-uLZAXXDpnjObv off=2705591 len=1338538 2024-09-13T00:04:58.063 INFO:tasks.workunit.client.0.smithi136.stdout:op 1087 completed, throughput=4.98MB/sec 2024-09-13T00:04:58.063 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-J9cUYr9KpRmdIz7 off=2240880 len=1238891 2024-09-13T00:04:58.066 INFO:tasks.workunit.client.0.smithi136.stdout:op 1088 completed, throughput=4.99MB/sec 2024-09-13T00:04:58.066 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-1XRZwmJsUw4RCnh off=778687 len=62598 2024-09-13T00:04:58.067 INFO:tasks.workunit.client.0.smithi136.stdout:op 1089 completed, throughput=4.99MB/sec 2024-09-13T00:04:58.067 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-5TfW7TXDU55pviY off=3958008 len=1168473 2024-09-13T00:04:58.078 INFO:tasks.workunit.client.0.smithi136.stdout:op 1090 completed, throughput=5MB/sec 2024-09-13T00:04:58.078 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-7ZW0KP_nSxlNP29 off=12999284 len=1390660 2024-09-13T00:04:58.090 INFO:tasks.workunit.client.0.smithi136.stdout:op 1091 completed, throughput=5MB/sec 2024-09-13T00:04:59.091 INFO:tasks.workunit.client.0.smithi136.stdout: 209: throughput=4.98MB/sec pending data=0 2024-09-13T00:04:59.091 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-d4CZ5t6lNCIS6-p off=2741226 len=1484404 2024-09-13T00:04:59.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:04:58 smithi181 bash[67587]: cluster 2024-09-13T00:04:57.255870+0000 mgr.y (mgr.24661) 835 : cluster [DBG] pgmap v1518: 105 pgs: 105 active+clean; 126 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:04:59.104 INFO:tasks.workunit.client.0.smithi136.stdout:op 1092 completed, throughput=4.98MB/sec 2024-09-13T00:04:59.104 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-KDf1xMLEUnGtQV0 off=8948475 len=1431720 2024-09-13T00:04:59.113 INFO:tasks.workunit.client.0.smithi136.stdout:op 1093 completed, throughput=4.99MB/sec 2024-09-13T00:04:59.113 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-lF6zC6LPUPKvEqQ off=1014318 len=744101 2024-09-13T00:04:59.115 INFO:tasks.workunit.client.0.smithi136.stdout:op 1094 completed, throughput=4.99MB/sec 2024-09-13T00:04:59.116 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-wbLJmaAklZ0dt3J off=6568830 len=615301 2024-09-13T00:04:59.117 INFO:tasks.workunit.client.0.smithi136.stdout:op 1095 completed, throughput=5MB/sec 2024-09-13T00:04:59.117 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-RGf873Umrcg50Vg off=17822098 len=1699618 2024-09-13T00:04:59.121 INFO:tasks.workunit.client.0.smithi136.stdout:op 1096 completed, throughput=5MB/sec 2024-09-13T00:04:59.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:04:58 smithi136 bash[75164]: cluster 2024-09-13T00:04:57.255870+0000 mgr.y (mgr.24661) 835 : cluster [DBG] pgmap v1518: 105 pgs: 105 active+clean; 126 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:04:59.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:04:58 smithi136 bash[76371]: cluster 2024-09-13T00:04:57.255870+0000 mgr.y (mgr.24661) 835 : cluster [DBG] pgmap v1518: 105 pgs: 105 active+clean; 126 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:04:59.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:04:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:04:59] "GET /metrics HTTP/1.1" 200 34121 "" "Prometheus/2.43.0" 2024-09-13T00:05:00.122 INFO:tasks.workunit.client.0.smithi136.stdout: 210: throughput=4.98MB/sec pending data=0 2024-09-13T00:05:00.122 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-MIUYEtpXMvPUA9_ off=17048189 len=1922482 2024-09-13T00:05:00.126 INFO:tasks.workunit.client.0.smithi136.stdout:op 1097 completed, throughput=4.99MB/sec 2024-09-13T00:05:00.126 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-dB7BXuaAWt45GD8 off=20805102 len=110568 2024-09-13T00:05:00.131 INFO:tasks.workunit.client.0.smithi136.stdout:op 1098 completed, throughput=4.99MB/sec 2024-09-13T00:05:00.132 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-RKD3H__15FPmaBC off=8677251 len=982083 2024-09-13T00:05:00.139 INFO:tasks.workunit.client.0.smithi136.stdout:op 1099 completed, throughput=4.99MB/sec 2024-09-13T00:05:00.139 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-_VB4zUgnMUZS221 off=20011768 len=1672660 2024-09-13T00:05:00.145 INFO:tasks.workunit.client.0.smithi136.stdout:op 1100 completed, throughput=5MB/sec 2024-09-13T00:05:01.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:05:00 smithi181 bash[67587]: cluster 2024-09-13T00:04:59.256883+0000 mgr.y (mgr.24661) 836 : cluster [DBG] pgmap v1519: 105 pgs: 105 active+clean; 126 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:05:01.145 INFO:tasks.workunit.client.0.smithi136.stdout: 211: throughput=4.98MB/sec pending data=0 2024-09-13T00:05:01.145 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-MIX5FWbwEAxY5jP off=1852998 len=112212 2024-09-13T00:05:01.146 INFO:tasks.workunit.client.0.smithi136.stdout:op 1101 completed, throughput=4.98MB/sec 2024-09-13T00:05:01.146 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Z7Tohs2-YESRDL6 off=2045429 len=1171378 2024-09-13T00:05:01.155 INFO:tasks.workunit.client.0.smithi136.stdout:op 1102 completed, throughput=4.98MB/sec 2024-09-13T00:05:01.155 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-2_SUalRXWK7yZOv off=2572146 len=108674 2024-09-13T00:05:01.160 INFO:tasks.workunit.client.0.smithi136.stdout:op 1103 completed, throughput=4.98MB/sec 2024-09-13T00:05:01.160 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-K5d-sqeZfIyjyg4 off=7017871 len=23666 2024-09-13T00:05:01.161 INFO:tasks.workunit.client.0.smithi136.stdout:op 1104 completed, throughput=4.98MB/sec 2024-09-13T00:05:01.161 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-YjJYdE_JjSzAiH7 off=2229902 len=1978854 2024-09-13T00:05:01.168 INFO:tasks.workunit.client.0.smithi136.stdout:op 1105 completed, throughput=4.99MB/sec 2024-09-13T00:05:01.168 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Bga1tsLL7Y1Yaek off=9259051 len=946409 2024-09-13T00:05:01.171 INFO:tasks.workunit.client.0.smithi136.stdout:op 1106 completed, throughput=5MB/sec 2024-09-13T00:05:01.171 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-EHG_0qDCxCqWRUu off=129565 len=2073857 2024-09-13T00:05:01.173 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:05:00 smithi136 bash[75164]: cluster 2024-09-13T00:04:59.256883+0000 mgr.y (mgr.24661) 836 : cluster [DBG] pgmap v1519: 105 pgs: 105 active+clean; 126 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:05:01.173 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:05:00 smithi136 bash[76371]: cluster 2024-09-13T00:04:59.256883+0000 mgr.y (mgr.24661) 836 : cluster [DBG] pgmap v1519: 105 pgs: 105 active+clean; 126 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:05:01.186 INFO:tasks.workunit.client.0.smithi136.stdout:op 1107 completed, throughput=5MB/sec 2024-09-13T00:05:02.187 INFO:tasks.workunit.client.0.smithi136.stdout: 212: throughput=4.98MB/sec pending data=0 2024-09-13T00:05:02.187 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-oMhMtyqVeP61chl off=1595652 len=1693287 2024-09-13T00:05:02.199 INFO:tasks.workunit.client.0.smithi136.stdout:op 1108 completed, throughput=4.99MB/sec 2024-09-13T00:05:02.199 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-2O-zaTzDqzJJ50R off=2685111 len=44351 2024-09-13T00:05:02.204 INFO:tasks.workunit.client.0.smithi136.stdout:op 1109 completed, throughput=4.99MB/sec 2024-09-13T00:05:02.204 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Rg2rC9OvRo4pYJf off=9167372 len=331857 2024-09-13T00:05:02.205 INFO:tasks.workunit.client.0.smithi136.stdout:op 1110 completed, throughput=4.99MB/sec 2024-09-13T00:05:02.205 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ABNMegkrnT9o6vD off=9947034 len=1470139 2024-09-13T00:05:02.209 INFO:tasks.workunit.client.0.smithi136.stdout:op 1111 completed, throughput=5MB/sec 2024-09-13T00:05:02.209 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-hcgo4WieZvWDVG7 off=0 len=1154614 2024-09-13T00:05:02.218 INFO:tasks.workunit.client.0.smithi136.stdout:op 1112 completed, throughput=5MB/sec 2024-09-13T00:05:02.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:05:02 smithi181 bash[95505]: ts=2024-09-13T00:05:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:05:03.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:05:02 smithi181 bash[67587]: cluster 2024-09-13T00:05:01.257909+0000 mgr.y (mgr.24661) 837 : cluster [DBG] pgmap v1520: 105 pgs: 105 active+clean; 126 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.2 MiB/s wr, 5 op/s 2024-09-13T00:05:03.218 INFO:tasks.workunit.client.0.smithi136.stdout: 213: throughput=4.98MB/sec pending data=0 2024-09-13T00:05:03.218 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-UXg5bBi_cq-rqpl off=3240620 len=748987 2024-09-13T00:05:03.226 INFO:tasks.workunit.client.0.smithi136.stdout:op 1113 completed, throughput=4.98MB/sec 2024-09-13T00:05:03.226 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-6rBZVVLcA9qAnqj off=18325737 len=615793 2024-09-13T00:05:03.229 INFO:tasks.workunit.client.0.smithi136.stdout:op 1114 completed, throughput=4.98MB/sec 2024-09-13T00:05:03.229 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-YhJSJtoAq1_WUX2 off=9466282 len=69246 2024-09-13T00:05:03.233 INFO:tasks.workunit.client.0.smithi136.stdout:op 1115 completed, throughput=4.98MB/sec 2024-09-13T00:05:03.233 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-W0tvbyeIILJICCT off=1695670 len=737457 2024-09-13T00:05:03.240 INFO:tasks.workunit.client.0.smithi136.stdout:op 1116 completed, throughput=4.99MB/sec 2024-09-13T00:05:03.240 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-6YWQZ1Nqw7EviOA off=21125481 len=491650 2024-09-13T00:05:03.242 INFO:tasks.workunit.client.0.smithi136.stdout:op 1117 completed, throughput=4.99MB/sec 2024-09-13T00:05:03.242 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-NhGREjYXjQCigtF off=5450225 len=1701877 2024-09-13T00:05:03.246 INFO:tasks.workunit.client.0.smithi136.stdout:op 1118 completed, throughput=5MB/sec 2024-09-13T00:05:03.247 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Jvy5Zults8_vEMB off=19571748 len=1125884 2024-09-13T00:05:03.254 INFO:tasks.workunit.client.0.smithi136.stdout:op 1119 completed, throughput=5MB/sec 2024-09-13T00:05:03.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:05:02 smithi136 bash[75164]: cluster 2024-09-13T00:05:01.257909+0000 mgr.y (mgr.24661) 837 : cluster [DBG] pgmap v1520: 105 pgs: 105 active+clean; 126 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.2 MiB/s wr, 5 op/s 2024-09-13T00:05:03.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:05:02 smithi136 bash[76371]: cluster 2024-09-13T00:05:01.257909+0000 mgr.y (mgr.24661) 837 : cluster [DBG] pgmap v1520: 105 pgs: 105 active+clean; 126 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.2 MiB/s wr, 5 op/s 2024-09-13T00:05:04.255 INFO:tasks.workunit.client.0.smithi136.stdout: 214: throughput=4.98MB/sec pending data=0 2024-09-13T00:05:04.255 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-OUn-lGn1GlwWqh1 off=674286 len=61470 2024-09-13T00:05:04.255 INFO:tasks.workunit.client.0.smithi136.stdout:op 1120 completed, throughput=4.98MB/sec 2024-09-13T00:05:04.256 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-oZ-JgNkNHzrtjVh off=8547464 len=1003538 2024-09-13T00:05:04.268 INFO:tasks.workunit.client.0.smithi136.stdout:op 1121 completed, throughput=4.98MB/sec 2024-09-13T00:05:04.268 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-6InMVJTtijU1oHA off=6635637 len=660162 2024-09-13T00:05:04.273 INFO:tasks.workunit.client.0.smithi136.stdout:op 1122 completed, throughput=4.99MB/sec 2024-09-13T00:05:04.274 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-R1uO5R8cJRSB0LP off=11395398 len=1747160 2024-09-13T00:05:04.278 INFO:tasks.workunit.client.0.smithi136.stdout:op 1123 completed, throughput=4.99MB/sec 2024-09-13T00:05:04.278 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-zu8KTTjyAmdOpqd off=1279367 len=449863 2024-09-13T00:05:04.284 INFO:tasks.workunit.client.0.smithi136.stdout:op 1124 completed, throughput=5MB/sec 2024-09-13T00:05:04.284 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj--WBb82hI1Ybd5sF off=12005380 len=7368 2024-09-13T00:05:04.288 INFO:tasks.workunit.client.0.smithi136.stdout:op 1125 completed, throughput=5MB/sec 2024-09-13T00:05:04.288 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-XCXzdOJbOwsEvUM off=8387855 len=622605 2024-09-13T00:05:04.294 INFO:tasks.workunit.client.0.smithi136.stdout:op 1126 completed, throughput=5MB/sec 2024-09-13T00:05:04.294 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-039NnjwWKRKrrV4 off=172504 len=1115795 2024-09-13T00:05:04.297 INFO:tasks.workunit.client.0.smithi136.stdout:op 1127 completed, throughput=5MB/sec 2024-09-13T00:05:05.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:05:04 smithi136 bash[75164]: cluster 2024-09-13T00:05:03.258852+0000 mgr.y (mgr.24661) 838 : cluster [DBG] pgmap v1521: 105 pgs: 105 active+clean; 126 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.2 MiB/s wr, 5 op/s 2024-09-13T00:05:05.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:05:04 smithi136 bash[76371]: cluster 2024-09-13T00:05:03.258852+0000 mgr.y (mgr.24661) 838 : cluster [DBG] pgmap v1521: 105 pgs: 105 active+clean; 126 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.2 MiB/s wr, 5 op/s 2024-09-13T00:05:05.298 INFO:tasks.workunit.client.0.smithi136.stdout: 215: throughput=4.98MB/sec pending data=0 2024-09-13T00:05:05.298 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-T2gOTGz3eM43NHo off=2900205 len=693190 2024-09-13T00:05:05.300 INFO:tasks.workunit.client.0.smithi136.stdout:op 1128 completed, throughput=4.98MB/sec 2024-09-13T00:05:05.300 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-OP54VMU7J0Oy8va off=7454502 len=777611 2024-09-13T00:05:05.303 INFO:tasks.workunit.client.0.smithi136.stdout:op 1129 completed, throughput=4.99MB/sec 2024-09-13T00:05:05.303 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-1hkuGQhLb1Ay432 off=3890928 len=1817245 2024-09-13T00:05:05.315 INFO:tasks.workunit.client.0.smithi136.stdout:op 1130 completed, throughput=4.99MB/sec 2024-09-13T00:05:05.315 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-TVN4n1W_nPX89Xl off=470939 len=769195 2024-09-13T00:05:05.323 INFO:tasks.workunit.client.0.smithi136.stdout:op 1131 completed, throughput=5MB/sec 2024-09-13T00:05:05.323 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-kLQGCxcco91JEZ0 off=9277259 len=2020654 2024-09-13T00:05:05.327 INFO:tasks.workunit.client.0.smithi136.stdout:op 1132 completed, throughput=5.01MB/sec 2024-09-13T00:05:05.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:05:04 smithi181 bash[67587]: cluster 2024-09-13T00:05:03.258852+0000 mgr.y (mgr.24661) 838 : cluster [DBG] pgmap v1521: 105 pgs: 105 active+clean; 126 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.2 MiB/s wr, 5 op/s 2024-09-13T00:05:06.328 INFO:tasks.workunit.client.0.smithi136.stdout: 216: throughput=4.98MB/sec pending data=0 2024-09-13T00:05:06.328 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-axm0sEEgCl7G-cA off=1446139 len=98708 2024-09-13T00:05:06.329 INFO:tasks.workunit.client.0.smithi136.stdout:op 1133 completed, throughput=4.98MB/sec 2024-09-13T00:05:06.329 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-T_9zaW7zsdXY_Vf off=678787 len=1860158 2024-09-13T00:05:06.335 INFO:tasks.workunit.client.0.smithi136.stdout:op 1134 completed, throughput=4.99MB/sec 2024-09-13T00:05:06.335 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-B6QZZoHSkEWguTT off=3717071 len=1292458 2024-09-13T00:05:06.345 INFO:tasks.workunit.client.0.smithi136.stdout:op 1135 completed, throughput=5MB/sec 2024-09-13T00:05:06.345 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-F8NwrOnDTEStxkS off=17347749 len=1040653 2024-09-13T00:05:06.349 INFO:tasks.workunit.client.0.smithi136.stdout:op 1136 completed, throughput=5MB/sec 2024-09-13T00:05:07.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:05:06 smithi136 bash[75164]: cluster 2024-09-13T00:05:05.259750+0000 mgr.y (mgr.24661) 839 : cluster [DBG] pgmap v1522: 105 pgs: 105 active+clean; 126 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.8 MiB/s wr, 5 op/s 2024-09-13T00:05:07.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:05:06 smithi136 bash[76371]: cluster 2024-09-13T00:05:05.259750+0000 mgr.y (mgr.24661) 839 : cluster [DBG] pgmap v1522: 105 pgs: 105 active+clean; 126 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.8 MiB/s wr, 5 op/s 2024-09-13T00:05:07.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:05:06 smithi181 bash[67587]: cluster 2024-09-13T00:05:05.259750+0000 mgr.y (mgr.24661) 839 : cluster [DBG] pgmap v1522: 105 pgs: 105 active+clean; 126 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.8 MiB/s wr, 5 op/s 2024-09-13T00:05:07.349 INFO:tasks.workunit.client.0.smithi136.stdout: 217: throughput=4.98MB/sec pending data=0 2024-09-13T00:05:07.349 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-kxK9TSuRACFNZ-n off=299728 len=1406588 2024-09-13T00:05:07.360 INFO:tasks.workunit.client.0.smithi136.stdout:op 1137 completed, throughput=4.98MB/sec 2024-09-13T00:05:07.361 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-g8nO0ywRD3twOc_ off=720257 len=1024971 2024-09-13T00:05:07.369 INFO:tasks.workunit.client.0.smithi136.stdout:op 1138 completed, throughput=4.99MB/sec 2024-09-13T00:05:07.369 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-4iPQmbxxGg0iiEL off=8030624 len=401865 2024-09-13T00:05:07.370 INFO:tasks.workunit.client.0.smithi136.stdout:op 1139 completed, throughput=4.99MB/sec 2024-09-13T00:05:07.371 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-TlaXfofWlMbB9Ro off=44078 len=639890 2024-09-13T00:05:07.378 INFO:tasks.workunit.client.0.smithi136.stdout:op 1140 completed, throughput=4.99MB/sec 2024-09-13T00:05:07.378 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-8yel6Gje62halO4 off=4165099 len=1183578 2024-09-13T00:05:07.387 INFO:tasks.workunit.client.0.smithi136.stdout:op 1141 completed, throughput=5MB/sec 2024-09-13T00:05:07.387 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Aka11Yhib8Ff5MZ off=6296324 len=2046824 2024-09-13T00:05:07.391 INFO:tasks.workunit.client.0.smithi136.stdout:op 1142 completed, throughput=5.01MB/sec 2024-09-13T00:05:08.392 INFO:tasks.workunit.client.0.smithi136.stdout: 218: throughput=4.98MB/sec pending data=0 2024-09-13T00:05:08.392 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Ilq-oT4ygwxsVi4 off=1004720 len=1601829 2024-09-13T00:05:08.396 INFO:tasks.workunit.client.0.smithi136.stdout:op 1143 completed, throughput=4.99MB/sec 2024-09-13T00:05:08.396 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-d5X7vIZTvJVWRBL off=3160870 len=265822 2024-09-13T00:05:08.401 INFO:tasks.workunit.client.0.smithi136.stdout:op 1144 completed, throughput=4.99MB/sec 2024-09-13T00:05:08.401 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-s8kK8Qq-O6QHFb6 off=6592303 len=990619 2024-09-13T00:05:08.409 INFO:tasks.workunit.client.0.smithi136.stdout:op 1145 completed, throughput=5MB/sec 2024-09-13T00:05:08.409 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-8MTsUJGhKanO01y off=4537517 len=1982504 2024-09-13T00:05:08.421 INFO:tasks.workunit.client.0.smithi136.stdout:op 1146 completed, throughput=5MB/sec 2024-09-13T00:05:09.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:05:08 smithi136 bash[75164]: cluster 2024-09-13T00:05:07.261112+0000 mgr.y (mgr.24661) 840 : cluster [DBG] pgmap v1523: 105 pgs: 105 active+clean; 126 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.9 MiB/s wr, 5 op/s 2024-09-13T00:05:09.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:05:08 smithi136 bash[75164]: audit 2024-09-13T00:05:08.777508+0000 mon.a (mon.0) 2139 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:05:09.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:05:08 smithi136 bash[76371]: cluster 2024-09-13T00:05:07.261112+0000 mgr.y (mgr.24661) 840 : cluster [DBG] pgmap v1523: 105 pgs: 105 active+clean; 126 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.9 MiB/s wr, 5 op/s 2024-09-13T00:05:09.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:05:08 smithi136 bash[76371]: audit 2024-09-13T00:05:08.777508+0000 mon.a (mon.0) 2139 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:05:09.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:05:08 smithi181 bash[67587]: cluster 2024-09-13T00:05:07.261112+0000 mgr.y (mgr.24661) 840 : cluster [DBG] pgmap v1523: 105 pgs: 105 active+clean; 126 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.9 MiB/s wr, 5 op/s 2024-09-13T00:05:09.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:05:08 smithi181 bash[67587]: audit 2024-09-13T00:05:08.777508+0000 mon.a (mon.0) 2139 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:05:09.421 INFO:tasks.workunit.client.0.smithi136.stdout: 219: throughput=4.98MB/sec pending data=0 2024-09-13T00:05:09.421 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-XzZQIbNYnrCgmey off=293838 len=1743193 2024-09-13T00:05:09.427 INFO:tasks.workunit.client.0.smithi136.stdout:op 1147 completed, throughput=4.99MB/sec 2024-09-13T00:05:09.427 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-rAXwpsxzfgppPOP off=2767955 len=1250584 2024-09-13T00:05:09.430 INFO:tasks.workunit.client.0.smithi136.stdout:op 1148 completed, throughput=4.99MB/sec 2024-09-13T00:05:09.431 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-jhPdheEDH9qCYCW off=9318315 len=585930 2024-09-13T00:05:09.437 INFO:tasks.workunit.client.0.smithi136.stdout:op 1149 completed, throughput=5MB/sec 2024-09-13T00:05:09.437 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-d1dDQTye0mwJ-8q off=5192978 len=27412 2024-09-13T00:05:09.438 INFO:tasks.workunit.client.0.smithi136.stdout:op 1150 completed, throughput=5MB/sec 2024-09-13T00:05:09.438 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Jk2bWvh10ixpd1y off=7019380 len=665438 2024-09-13T00:05:09.440 INFO:tasks.workunit.client.0.smithi136.stdout:op 1151 completed, throughput=5MB/sec 2024-09-13T00:05:09.440 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-77_NcyM7CTReRtw off=5572705 len=721064 2024-09-13T00:05:09.448 INFO:tasks.workunit.client.0.smithi136.stdout:op 1152 completed, throughput=5MB/sec 2024-09-13T00:05:09.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:05:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:05:09] "GET /metrics HTTP/1.1" 200 34124 "" "Prometheus/2.43.0" 2024-09-13T00:05:10.449 INFO:tasks.workunit.client.0.smithi136.stdout: 220: throughput=4.98MB/sec pending data=0 2024-09-13T00:05:10.449 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-eF05k4Bnc_1t0b8 off=6807891 len=890633 2024-09-13T00:05:10.458 INFO:tasks.workunit.client.0.smithi136.stdout:op 1153 completed, throughput=4.98MB/sec 2024-09-13T00:05:10.458 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-sLWtnMqSZNfMJFv off=3897006 len=717096 2024-09-13T00:05:10.461 INFO:tasks.workunit.client.0.smithi136.stdout:op 1154 completed, throughput=4.99MB/sec 2024-09-13T00:05:10.461 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Xjnw-zZB6dENIGN off=1692797 len=388767 2024-09-13T00:05:10.462 INFO:tasks.workunit.client.0.smithi136.stdout:op 1155 completed, throughput=4.99MB/sec 2024-09-13T00:05:10.462 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-W8k7i0ZxH_HOSlk off=3793627 len=119729 2024-09-13T00:05:10.463 INFO:tasks.workunit.client.0.smithi136.stdout:op 1156 completed, throughput=4.99MB/sec 2024-09-13T00:05:10.463 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-CIyy62jcDsFIR_l off=4697096 len=213984 2024-09-13T00:05:10.469 INFO:tasks.workunit.client.0.smithi136.stdout:op 1157 completed, throughput=4.99MB/sec 2024-09-13T00:05:10.469 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-qdIBXcHAsGH0Fip off=12904583 len=1348453 2024-09-13T00:05:10.479 INFO:tasks.workunit.client.0.smithi136.stdout:op 1158 completed, throughput=4.99MB/sec 2024-09-13T00:05:10.479 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ToqGPxeYv9yLT9- off=2015578 len=41715 2024-09-13T00:05:10.483 INFO:tasks.workunit.client.0.smithi136.stdout:op 1159 completed, throughput=4.99MB/sec 2024-09-13T00:05:10.483 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-cfkWvbxgQ-CApty off=7962107 len=709116 2024-09-13T00:05:10.489 INFO:tasks.workunit.client.0.smithi136.stdout:op 1160 completed, throughput=5MB/sec 2024-09-13T00:05:10.489 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-YcySZaIsPwJdixH off=5742281 len=920422 2024-09-13T00:05:10.497 INFO:tasks.workunit.client.0.smithi136.stdout:op 1161 completed, throughput=5MB/sec 2024-09-13T00:05:11.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:05:10 smithi136 bash[75164]: cluster 2024-09-13T00:05:09.261969+0000 mgr.y (mgr.24661) 841 : cluster [DBG] pgmap v1524: 105 pgs: 105 active+clean; 126 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:05:11.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:05:10 smithi136 bash[76371]: cluster 2024-09-13T00:05:09.261969+0000 mgr.y (mgr.24661) 841 : cluster [DBG] pgmap v1524: 105 pgs: 105 active+clean; 126 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:05:11.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:05:10 smithi181 bash[67587]: cluster 2024-09-13T00:05:09.261969+0000 mgr.y (mgr.24661) 841 : cluster [DBG] pgmap v1524: 105 pgs: 105 active+clean; 126 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:05:11.497 INFO:tasks.workunit.client.0.smithi136.stdout: 221: throughput=4.98MB/sec pending data=0 2024-09-13T00:05:11.498 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-g7BzP6D5XLFingG off=23372733 len=26602 2024-09-13T00:05:11.498 INFO:tasks.workunit.client.0.smithi136.stdout:op 1162 completed, throughput=4.98MB/sec 2024-09-13T00:05:11.498 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-BxZcqi2oOr92uuA off=13136034 len=1768452 2024-09-13T00:05:11.503 INFO:tasks.workunit.client.0.smithi136.stdout:op 1163 completed, throughput=4.99MB/sec 2024-09-13T00:05:11.503 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-cogMxpN3PF4f8Uj off=22607766 len=65777 2024-09-13T00:05:11.504 INFO:tasks.workunit.client.0.smithi136.stdout:op 1164 completed, throughput=4.99MB/sec 2024-09-13T00:05:11.504 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-nukFMjoFddBWZ3b off=4985409 len=2048101 2024-09-13T00:05:11.509 INFO:tasks.workunit.client.0.smithi136.stdout:op 1165 completed, throughput=5MB/sec 2024-09-13T00:05:11.509 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Z6r2_FZNHzdFgX8 off=13566705 len=1387425 2024-09-13T00:05:11.513 INFO:tasks.workunit.client.0.smithi136.stdout:op 1166 completed, throughput=5MB/sec 2024-09-13T00:05:12.513 INFO:tasks.workunit.client.0.smithi136.stdout: 222: throughput=4.98MB/sec pending data=0 2024-09-13T00:05:12.513 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-FHOfsD8-WkKbEwE off=7842175 len=700515 2024-09-13T00:05:12.516 INFO:tasks.workunit.client.0.smithi136.stdout:op 1167 completed, throughput=4.98MB/sec 2024-09-13T00:05:12.516 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ohyuORRuvkWvrKq off=17813553 len=987490 2024-09-13T00:05:12.524 INFO:tasks.workunit.client.0.smithi136.stdout:op 1168 completed, throughput=4.99MB/sec 2024-09-13T00:05:12.524 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-LcvBEIjLs9BhP5a off=1426303 len=1976887 2024-09-13T00:05:12.536 INFO:tasks.workunit.client.0.smithi136.stdout:op 1169 completed, throughput=4.99MB/sec 2024-09-13T00:05:12.536 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-WvqBjllK7vD6Vhv off=3844901 len=886233 2024-09-13T00:05:12.544 INFO:tasks.workunit.client.0.smithi136.stdout:op 1170 completed, throughput=5MB/sec 2024-09-13T00:05:12.544 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-WJfeU9J7r7SfhCe off=8617365 len=930995 2024-09-13T00:05:12.548 INFO:tasks.workunit.client.0.smithi136.stdout:op 1171 completed, throughput=5MB/sec 2024-09-13T00:05:12.599 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 13 00:05:12 smithi181 bash[98820]: logger=sqlstore.transactions t=2024-09-13T00:05:12.362291888Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-13T00:05:12.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:05:12 smithi181 bash[95505]: ts=2024-09-13T00:05:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:05:13.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:05:12 smithi136 bash[75164]: cluster 2024-09-13T00:05:11.262949+0000 mgr.y (mgr.24661) 842 : cluster [DBG] pgmap v1525: 105 pgs: 105 active+clean; 126 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:05:13.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:05:12 smithi136 bash[76371]: cluster 2024-09-13T00:05:11.262949+0000 mgr.y (mgr.24661) 842 : cluster [DBG] pgmap v1525: 105 pgs: 105 active+clean; 126 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:05:13.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:05:12 smithi181 bash[67587]: cluster 2024-09-13T00:05:11.262949+0000 mgr.y (mgr.24661) 842 : cluster [DBG] pgmap v1525: 105 pgs: 105 active+clean; 126 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:05:13.549 INFO:tasks.workunit.client.0.smithi136.stdout: 223: throughput=4.98MB/sec pending data=0 2024-09-13T00:05:13.549 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-__3KYqvimJ9Euov off=15914471 len=624127 2024-09-13T00:05:13.551 INFO:tasks.workunit.client.0.smithi136.stdout:op 1172 completed, throughput=4.98MB/sec 2024-09-13T00:05:13.551 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_LwenDuoweD2xpj off=7765670 len=2006130 2024-09-13T00:05:13.565 INFO:tasks.workunit.client.0.smithi136.stdout:op 1173 completed, throughput=4.99MB/sec 2024-09-13T00:05:13.565 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-maaAqpQodVU1inO off=8328038 len=610905 2024-09-13T00:05:13.573 INFO:tasks.workunit.client.0.smithi136.stdout:op 1174 completed, throughput=4.99MB/sec 2024-09-13T00:05:13.573 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ncj7wEyrdPHKI9x off=9090910 len=510568 2024-09-13T00:05:13.578 INFO:tasks.workunit.client.0.smithi136.stdout:op 1175 completed, throughput=4.99MB/sec 2024-09-13T00:05:13.578 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj--abD8qCnRY0iPBm off=4284931 len=1152925 2024-09-13T00:05:13.581 INFO:tasks.workunit.client.0.smithi136.stdout:op 1176 completed, throughput=5MB/sec 2024-09-13T00:05:13.581 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-OV8F4WjajZ2JI_7 off=8224408 len=1223413 2024-09-13T00:05:13.592 INFO:tasks.workunit.client.0.smithi136.stdout:op 1177 completed, throughput=5MB/sec 2024-09-13T00:05:14.592 INFO:tasks.workunit.client.0.smithi136.stdout: 224: throughput=4.98MB/sec pending data=0 2024-09-13T00:05:14.592 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-VorJmXGu8klhJYF off=7318566 len=1779430 2024-09-13T00:05:14.605 INFO:tasks.workunit.client.0.smithi136.stdout:op 1178 completed, throughput=4.99MB/sec 2024-09-13T00:05:14.605 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-BbQJUw8N7jXMoPV off=1274591 len=53408 2024-09-13T00:05:14.606 INFO:tasks.workunit.client.0.smithi136.stdout:op 1179 completed, throughput=4.99MB/sec 2024-09-13T00:05:14.606 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-adHVRbgxVcilRsL off=4234150 len=345411 2024-09-13T00:05:14.611 INFO:tasks.workunit.client.0.smithi136.stdout:op 1180 completed, throughput=4.99MB/sec 2024-09-13T00:05:14.611 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-GrTDRtN3Iq6moh7 off=7843948 len=492109 2024-09-13T00:05:14.617 INFO:tasks.workunit.client.0.smithi136.stdout:op 1181 completed, throughput=4.99MB/sec 2024-09-13T00:05:14.617 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-kXthstME66Dk3ZZ off=2870348 len=137432 2024-09-13T00:05:14.620 INFO:tasks.workunit.client.0.smithi136.stdout:op 1182 completed, throughput=4.99MB/sec 2024-09-13T00:05:14.621 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-QlGyDsA1d0ZfYSR off=7730236 len=551485 2024-09-13T00:05:14.626 INFO:tasks.workunit.client.0.smithi136.stdout:op 1183 completed, throughput=5MB/sec 2024-09-13T00:05:14.626 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-pG9KylrK4OUaQQO off=7658019 len=581944 2024-09-13T00:05:14.631 INFO:tasks.workunit.client.0.smithi136.stdout:op 1184 completed, throughput=5MB/sec 2024-09-13T00:05:14.631 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-DEtn7o17M_S4mQX off=1244410 len=1982663 2024-09-13T00:05:14.636 INFO:tasks.workunit.client.0.smithi136.stdout:op 1185 completed, throughput=5.01MB/sec 2024-09-13T00:05:15.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:05:14 smithi136 bash[75164]: cluster 2024-09-13T00:05:13.263902+0000 mgr.y (mgr.24661) 843 : cluster [DBG] pgmap v1526: 105 pgs: 105 active+clean; 126 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:05:15.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:05:14 smithi136 bash[76371]: cluster 2024-09-13T00:05:13.263902+0000 mgr.y (mgr.24661) 843 : cluster [DBG] pgmap v1526: 105 pgs: 105 active+clean; 126 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:05:15.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:05:14 smithi181 bash[67587]: cluster 2024-09-13T00:05:13.263902+0000 mgr.y (mgr.24661) 843 : cluster [DBG] pgmap v1526: 105 pgs: 105 active+clean; 126 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:05:15.636 INFO:tasks.workunit.client.0.smithi136.stdout: 225: throughput=4.98MB/sec pending data=0 2024-09-13T00:05:15.636 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-6TgMB3WknW_booM off=486388 len=1077522 2024-09-13T00:05:15.645 INFO:tasks.workunit.client.0.smithi136.stdout:op 1186 completed, throughput=4.99MB/sec 2024-09-13T00:05:15.645 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-t950K-LeYpv_JOf off=578368 len=881963 2024-09-13T00:05:15.652 INFO:tasks.workunit.client.0.smithi136.stdout:op 1187 completed, throughput=4.99MB/sec 2024-09-13T00:05:15.652 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-y9PIw_M0wa7V9aS off=1534077 len=923394 2024-09-13T00:05:15.659 INFO:tasks.workunit.client.0.smithi136.stdout:op 1188 completed, throughput=5MB/sec 2024-09-13T00:05:15.659 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-JDjB8yCi-l-pevJ off=8065670 len=121653 2024-09-13T00:05:15.664 INFO:tasks.workunit.client.0.smithi136.stdout:op 1189 completed, throughput=5MB/sec 2024-09-13T00:05:15.664 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-HknceQN6SW0kcCk off=470460 len=1346163 2024-09-13T00:05:15.673 INFO:tasks.workunit.client.0.smithi136.stdout:op 1190 completed, throughput=5MB/sec 2024-09-13T00:05:16.673 INFO:tasks.workunit.client.0.smithi136.stdout: 226: throughput=4.98MB/sec pending data=0 2024-09-13T00:05:16.674 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-YSgL1LfNAzabq0Y off=1147603 len=1959552 2024-09-13T00:05:16.678 INFO:tasks.workunit.client.0.smithi136.stdout:op 1191 completed, throughput=4.99MB/sec 2024-09-13T00:05:16.678 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-mCF4-AljgQIxFIs off=87661 len=1764644 2024-09-13T00:05:16.690 INFO:tasks.workunit.client.0.smithi136.stdout:op 1192 completed, throughput=5MB/sec 2024-09-13T00:05:16.690 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ecSxXOJtzrVEoBJ off=846566 len=1043950 2024-09-13T00:05:16.698 INFO:tasks.workunit.client.0.smithi136.stdout:op 1193 completed, throughput=5MB/sec 2024-09-13T00:05:16.699 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-xkFujCzrw1wQDod off=7523865 len=30667 2024-09-13T00:05:16.699 INFO:tasks.workunit.client.0.smithi136.stdout:op 1194 completed, throughput=5MB/sec 2024-09-13T00:05:16.699 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-l8VcuIjWzgahuDW off=1200164 len=150056 2024-09-13T00:05:16.700 INFO:tasks.workunit.client.0.smithi136.stdout:op 1195 completed, throughput=5MB/sec 2024-09-13T00:05:16.700 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-rUimuS95r67nT4x off=736094 len=89373 2024-09-13T00:05:16.701 INFO:tasks.workunit.client.0.smithi136.stdout:op 1196 completed, throughput=5MB/sec 2024-09-13T00:05:17.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:05:16 smithi136 bash[75164]: cluster 2024-09-13T00:05:15.264849+0000 mgr.y (mgr.24661) 844 : cluster [DBG] pgmap v1527: 105 pgs: 105 active+clean; 126 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:05:17.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:05:16 smithi136 bash[76371]: cluster 2024-09-13T00:05:15.264849+0000 mgr.y (mgr.24661) 844 : cluster [DBG] pgmap v1527: 105 pgs: 105 active+clean; 126 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:05:17.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:05:16 smithi181 bash[67587]: cluster 2024-09-13T00:05:15.264849+0000 mgr.y (mgr.24661) 844 : cluster [DBG] pgmap v1527: 105 pgs: 105 active+clean; 126 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:05:17.702 INFO:tasks.workunit.client.0.smithi136.stdout: 227: throughput=4.98MB/sec pending data=0 2024-09-13T00:05:17.702 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-U6rOk2x4Bs_9trp off=8143195 len=109155 2024-09-13T00:05:17.707 INFO:tasks.workunit.client.0.smithi136.stdout:op 1197 completed, throughput=4.98MB/sec 2024-09-13T00:05:17.707 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-CI5gzV8wgtWLtSU off=2761247 len=1271330 2024-09-13T00:05:17.710 INFO:tasks.workunit.client.0.smithi136.stdout:op 1198 completed, throughput=4.98MB/sec 2024-09-13T00:05:17.710 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-3V3U3O0ciXTvpzm off=8984089 len=329743 2024-09-13T00:05:17.716 INFO:tasks.workunit.client.0.smithi136.stdout:op 1199 completed, throughput=4.99MB/sec 2024-09-13T00:05:17.716 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-QahPMu9wo7SF6PK off=13052619 len=397651 2024-09-13T00:05:17.721 INFO:tasks.workunit.client.0.smithi136.stdout:op 1200 completed, throughput=4.99MB/sec 2024-09-13T00:05:17.721 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-mDqU_hmJB9quYBh off=7277765 len=889301 2024-09-13T00:05:17.729 INFO:tasks.workunit.client.0.smithi136.stdout:op 1201 completed, throughput=4.99MB/sec 2024-09-13T00:05:17.729 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-_AxXLdPC_IQnC-M off=8610995 len=1167545 2024-09-13T00:05:17.732 INFO:tasks.workunit.client.0.smithi136.stdout:op 1202 completed, throughput=5MB/sec 2024-09-13T00:05:17.732 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-0pKZ7qMfwIzm0YF off=9402517 len=1811632 2024-09-13T00:05:17.736 INFO:tasks.workunit.client.0.smithi136.stdout:op 1203 completed, throughput=5MB/sec 2024-09-13T00:05:18.737 INFO:tasks.workunit.client.0.smithi136.stdout: 228: throughput=4.98MB/sec pending data=0 2024-09-13T00:05:18.737 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-KsfoAlO61kJJTL7 off=3427310 len=149915 2024-09-13T00:05:18.738 INFO:tasks.workunit.client.0.smithi136.stdout:op 1204 completed, throughput=4.98MB/sec 2024-09-13T00:05:18.738 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-rIg93gj4UDoP0GO off=1241746 len=1208275 2024-09-13T00:05:18.741 INFO:tasks.workunit.client.0.smithi136.stdout:op 1205 completed, throughput=4.99MB/sec 2024-09-13T00:05:18.741 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-5pFuz2_huYFvYm9 off=7657379 len=1419544 2024-09-13T00:05:18.751 INFO:tasks.workunit.client.0.smithi136.stdout:op 1206 completed, throughput=4.99MB/sec 2024-09-13T00:05:18.752 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-n2WQ8uOUm6w0UHx off=10009494 len=106694 2024-09-13T00:05:18.755 INFO:tasks.workunit.client.0.smithi136.stdout:op 1207 completed, throughput=4.99MB/sec 2024-09-13T00:05:18.756 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-5_ZLdb3sOXee8Vz off=1687240 len=74684 2024-09-13T00:05:18.756 INFO:tasks.workunit.client.0.smithi136.stdout:op 1208 completed, throughput=4.99MB/sec 2024-09-13T00:05:18.756 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-rb0_o3VnjzCxU3f off=6552745 len=1849067 2024-09-13T00:05:18.762 INFO:tasks.workunit.client.0.smithi136.stdout:op 1209 completed, throughput=5MB/sec 2024-09-13T00:05:19.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:05:18 smithi136 bash[75164]: cluster 2024-09-13T00:05:17.266029+0000 mgr.y (mgr.24661) 845 : cluster [DBG] pgmap v1528: 105 pgs: 105 active+clean; 126 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 2.4 MiB/s rd, 3.1 MiB/s wr, 6 op/s 2024-09-13T00:05:19.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:05:18 smithi136 bash[76371]: cluster 2024-09-13T00:05:17.266029+0000 mgr.y (mgr.24661) 845 : cluster [DBG] pgmap v1528: 105 pgs: 105 active+clean; 126 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 2.4 MiB/s rd, 3.1 MiB/s wr, 6 op/s 2024-09-13T00:05:19.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:05:18 smithi181 bash[67587]: cluster 2024-09-13T00:05:17.266029+0000 mgr.y (mgr.24661) 845 : cluster [DBG] pgmap v1528: 105 pgs: 105 active+clean; 126 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 2.4 MiB/s rd, 3.1 MiB/s wr, 6 op/s 2024-09-13T00:05:19.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:05:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:05:19] "GET /metrics HTTP/1.1" 200 34124 "" "Prometheus/2.43.0" 2024-09-13T00:05:19.762 INFO:tasks.workunit.client.0.smithi136.stdout: 229: throughput=4.98MB/sec pending data=0 2024-09-13T00:05:19.762 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj--TMXAPVcy5UIaPA off=3510664 len=1942765 2024-09-13T00:05:19.767 INFO:tasks.workunit.client.0.smithi136.stdout:op 1210 completed, throughput=4.99MB/sec 2024-09-13T00:05:19.767 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-QVPcOTq-yMkRE1I off=14631238 len=164588 2024-09-13T00:05:19.773 INFO:tasks.workunit.client.0.smithi136.stdout:op 1211 completed, throughput=4.99MB/sec 2024-09-13T00:05:19.773 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-IF9d_LAR20NyzZm off=1048843 len=1202794 2024-09-13T00:05:19.776 INFO:tasks.workunit.client.0.smithi136.stdout:op 1212 completed, throughput=4.99MB/sec 2024-09-13T00:05:19.776 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-zRVLeWOYZsRsiA5 off=1776496 len=1154489 2024-09-13T00:05:19.785 INFO:tasks.workunit.client.0.smithi136.stdout:op 1213 completed, throughput=5MB/sec 2024-09-13T00:05:19.785 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-CWwG_6mEM5k9LMv off=22924122 len=1594194 2024-09-13T00:05:19.789 INFO:tasks.workunit.client.0.smithi136.stdout:op 1214 completed, throughput=5MB/sec 2024-09-13T00:05:20.790 INFO:tasks.workunit.client.0.smithi136.stdout: 230: throughput=4.98MB/sec pending data=0 2024-09-13T00:05:20.790 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-qJoPSFWDjLU1g13 off=70046 len=511245 2024-09-13T00:05:20.796 INFO:tasks.workunit.client.0.smithi136.stdout:op 1215 completed, throughput=4.98MB/sec 2024-09-13T00:05:20.796 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-pEMNOiRHmFxXomN off=17417348 len=1153079 2024-09-13T00:05:20.801 INFO:tasks.workunit.client.0.smithi136.stdout:op 1216 completed, throughput=4.99MB/sec 2024-09-13T00:05:20.801 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-MzyNUrca7ust1fV off=4495172 len=1071694 2024-09-13T00:05:20.804 INFO:tasks.workunit.client.0.smithi136.stdout:op 1217 completed, throughput=4.99MB/sec 2024-09-13T00:05:20.804 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-uYltVOCcJzVOBJ4 off=2317703 len=1916289 2024-09-13T00:05:20.808 INFO:tasks.workunit.client.0.smithi136.stdout:op 1218 completed, throughput=5MB/sec 2024-09-13T00:05:21.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:05:20 smithi136 bash[76371]: cluster 2024-09-13T00:05:19.267013+0000 mgr.y (mgr.24661) 846 : cluster [DBG] pgmap v1529: 105 pgs: 105 active+clean; 126 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 2.4 MiB/s rd, 3.0 MiB/s wr, 6 op/s 2024-09-13T00:05:21.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:05:20 smithi136 bash[75164]: cluster 2024-09-13T00:05:19.267013+0000 mgr.y (mgr.24661) 846 : cluster [DBG] pgmap v1529: 105 pgs: 105 active+clean; 126 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 2.4 MiB/s rd, 3.0 MiB/s wr, 6 op/s 2024-09-13T00:05:21.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:05:20 smithi181 bash[67587]: cluster 2024-09-13T00:05:19.267013+0000 mgr.y (mgr.24661) 846 : cluster [DBG] pgmap v1529: 105 pgs: 105 active+clean; 126 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 2.4 MiB/s rd, 3.0 MiB/s wr, 6 op/s 2024-09-13T00:05:21.809 INFO:tasks.workunit.client.0.smithi136.stdout: 231: throughput=4.98MB/sec pending data=0 2024-09-13T00:05:21.809 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-R_DcndibmrVhd6- off=240287 len=1654964 2024-09-13T00:05:21.820 INFO:tasks.workunit.client.0.smithi136.stdout:op 1219 completed, throughput=4.99MB/sec 2024-09-13T00:05:21.820 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-YiFV8fbTxzj1Ccj off=1062333 len=1891940 2024-09-13T00:05:21.832 INFO:tasks.workunit.client.0.smithi136.stdout:op 1220 completed, throughput=4.99MB/sec 2024-09-13T00:05:21.832 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-hWfaiq2NeWD_vWs off=1351867 len=1375604 2024-09-13T00:05:21.841 INFO:tasks.workunit.client.0.smithi136.stdout:op 1221 completed, throughput=5MB/sec 2024-09-13T00:05:21.841 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-RmoxbNWwHLX95XA off=1053613 len=1175086 2024-09-13T00:05:21.843 INFO:tasks.workunit.client.0.smithi136.stdout:op 1222 completed, throughput=5MB/sec 2024-09-13T00:05:22.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:05:22 smithi181 bash[95505]: ts=2024-09-13T00:05:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:05:22.844 INFO:tasks.workunit.client.0.smithi136.stdout: 232: throughput=4.98MB/sec pending data=0 2024-09-13T00:05:22.844 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-C7SD_UOa6Q2jFqm off=12925920 len=559245 2024-09-13T00:05:22.851 INFO:tasks.workunit.client.0.smithi136.stdout:op 1223 completed, throughput=4.98MB/sec 2024-09-13T00:05:22.851 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-dkMYszPOb513JdX off=12092979 len=1938681 2024-09-13T00:05:22.868 INFO:tasks.workunit.client.0.smithi136.stdout:op 1224 completed, throughput=4.99MB/sec 2024-09-13T00:05:22.868 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ctXDl9ppaCSnUE2 off=9505075 len=20046 2024-09-13T00:05:22.872 INFO:tasks.workunit.client.0.smithi136.stdout:op 1225 completed, throughput=4.99MB/sec 2024-09-13T00:05:22.872 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-0J4Ry2YIn7m8HSV off=1543156 len=1805233 2024-09-13T00:05:22.876 INFO:tasks.workunit.client.0.smithi136.stdout:op 1226 completed, throughput=5MB/sec 2024-09-13T00:05:22.876 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-kpV3HkTYIC_aXpA off=4179985 len=724277 2024-09-13T00:05:22.880 INFO:tasks.workunit.client.0.smithi136.stdout:op 1227 completed, throughput=5MB/sec 2024-09-13T00:05:23.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:05:22 smithi136 bash[76371]: cluster 2024-09-13T00:05:21.268129+0000 mgr.y (mgr.24661) 847 : cluster [DBG] pgmap v1530: 105 pgs: 105 active+clean; 126 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:05:23.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:05:22 smithi136 bash[75164]: cluster 2024-09-13T00:05:21.268129+0000 mgr.y (mgr.24661) 847 : cluster [DBG] pgmap v1530: 105 pgs: 105 active+clean; 126 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:05:23.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:05:22 smithi181 bash[67587]: cluster 2024-09-13T00:05:21.268129+0000 mgr.y (mgr.24661) 847 : cluster [DBG] pgmap v1530: 105 pgs: 105 active+clean; 126 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:05:23.878 INFO:tasks.workunit.client.0.smithi136.stdout: 233: throughput=4.98MB/sec pending data=0 2024-09-13T00:05:23.879 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-qRCRWMd2hKSjRDz off=3566069 len=65408 2024-09-13T00:05:23.880 INFO:tasks.workunit.client.0.smithi136.stdout:op 1228 completed, throughput=4.98MB/sec 2024-09-13T00:05:23.880 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-o4WYg2lWwdltvfb off=14830239 len=1498213 2024-09-13T00:05:23.894 INFO:tasks.workunit.client.0.smithi136.stdout:op 1229 completed, throughput=4.99MB/sec 2024-09-13T00:05:23.894 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-T-SuihNveykEwhg off=9671244 len=1638830 2024-09-13T00:05:23.907 INFO:tasks.workunit.client.0.smithi136.stdout:op 1230 completed, throughput=4.99MB/sec 2024-09-13T00:05:23.907 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-WN9nttuhzPiH8E3 off=65331 len=171223 2024-09-13T00:05:23.912 INFO:tasks.workunit.client.0.smithi136.stdout:op 1231 completed, throughput=4.99MB/sec 2024-09-13T00:05:23.912 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-G47gPo_oLHaypum off=7987642 len=1737139 2024-09-13T00:05:23.925 INFO:tasks.workunit.client.0.smithi136.stdout:op 1232 completed, throughput=5MB/sec 2024-09-13T00:05:23.925 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-NVroBdLPkmPmskp off=2795440 len=818970 2024-09-13T00:05:23.927 INFO:tasks.workunit.client.0.smithi136.stdout:op 1233 completed, throughput=5MB/sec 2024-09-13T00:05:24.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:05:23 smithi136 bash[76371]: audit 2024-09-13T00:05:23.777958+0000 mon.a (mon.0) 2140 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:05:24.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:05:23 smithi136 bash[75164]: audit 2024-09-13T00:05:23.777958+0000 mon.a (mon.0) 2140 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:05:24.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:05:23 smithi181 bash[67587]: audit 2024-09-13T00:05:23.777958+0000 mon.a (mon.0) 2140 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:05:24.928 INFO:tasks.workunit.client.0.smithi136.stdout: 234: throughput=4.98MB/sec pending data=0 2024-09-13T00:05:24.928 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Re2YcNt4oP_bsmn off=14318059 len=913855 2024-09-13T00:05:24.931 INFO:tasks.workunit.client.0.smithi136.stdout:op 1234 completed, throughput=4.99MB/sec 2024-09-13T00:05:24.931 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ylznHO-RHB_qefS off=6957609 len=1242893 2024-09-13T00:05:24.934 INFO:tasks.workunit.client.0.smithi136.stdout:op 1235 completed, throughput=4.99MB/sec 2024-09-13T00:05:24.935 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-OcYI1D0LsY6sdPW off=1569382 len=534349 2024-09-13T00:05:24.937 INFO:tasks.workunit.client.0.smithi136.stdout:op 1236 completed, throughput=4.99MB/sec 2024-09-13T00:05:24.937 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-CTWlVSETM9mwCGL off=11221703 len=613173 2024-09-13T00:05:24.939 INFO:tasks.workunit.client.0.smithi136.stdout:op 1237 completed, throughput=5MB/sec 2024-09-13T00:05:24.939 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Gfbfg_aCQBZZbvQ off=5612515 len=1974447 2024-09-13T00:05:24.944 INFO:tasks.workunit.client.0.smithi136.stdout:op 1238 completed, throughput=5MB/sec 2024-09-13T00:05:25.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:05:24 smithi136 bash[75164]: cluster 2024-09-13T00:05:23.269158+0000 mgr.y (mgr.24661) 848 : cluster [DBG] pgmap v1531: 105 pgs: 105 active+clean; 126 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:05:25.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:05:24 smithi136 bash[76371]: cluster 2024-09-13T00:05:23.269158+0000 mgr.y (mgr.24661) 848 : cluster [DBG] pgmap v1531: 105 pgs: 105 active+clean; 126 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:05:25.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:05:24 smithi181 bash[67587]: cluster 2024-09-13T00:05:23.269158+0000 mgr.y (mgr.24661) 848 : cluster [DBG] pgmap v1531: 105 pgs: 105 active+clean; 126 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:05:25.944 INFO:tasks.workunit.client.0.smithi136.stdout: 235: throughput=4.98MB/sec pending data=0 2024-09-13T00:05:25.944 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-hRHcj8a6QNs3762 off=16783276 len=470542 2024-09-13T00:05:25.950 INFO:tasks.workunit.client.0.smithi136.stdout:op 1239 completed, throughput=4.98MB/sec 2024-09-13T00:05:25.950 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-xQKVgDrB4Ez1CfL off=17035510 len=154357 2024-09-13T00:05:25.954 INFO:tasks.workunit.client.0.smithi136.stdout:op 1240 completed, throughput=4.98MB/sec 2024-09-13T00:05:25.954 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-jqi3D8r8Mj6Su2n off=18895035 len=1470532 2024-09-13T00:05:25.958 INFO:tasks.workunit.client.0.smithi136.stdout:op 1241 completed, throughput=4.99MB/sec 2024-09-13T00:05:25.959 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-vlFtZPNMfKV8bgo off=76852 len=1764894 2024-09-13T00:05:25.973 INFO:tasks.workunit.client.0.smithi136.stdout:op 1242 completed, throughput=5MB/sec 2024-09-13T00:05:25.973 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-d6s6sme-dVetau8 off=8727754 len=879183 2024-09-13T00:05:25.975 INFO:tasks.workunit.client.0.smithi136.stdout:op 1243 completed, throughput=5MB/sec 2024-09-13T00:05:26.976 INFO:tasks.workunit.client.0.smithi136.stdout: 236: throughput=4.98MB/sec pending data=0 2024-09-13T00:05:26.976 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-hOyysKexeYDsBVU off=4589293 len=765454 2024-09-13T00:05:26.978 INFO:tasks.workunit.client.0.smithi136.stdout:op 1244 completed, throughput=4.98MB/sec 2024-09-13T00:05:26.978 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-V-5q5TH5XU7dC-a off=10092007 len=1675744 2024-09-13T00:05:26.982 INFO:tasks.workunit.client.0.smithi136.stdout:op 1245 completed, throughput=4.99MB/sec 2024-09-13T00:05:26.982 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-FTvSlTGkjnv6GzF off=12898198 len=676650 2024-09-13T00:05:26.990 INFO:tasks.workunit.client.0.smithi136.stdout:op 1246 completed, throughput=4.99MB/sec 2024-09-13T00:05:26.991 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-SHI27vaFr7YBdmv off=17656967 len=1003483 2024-09-13T00:05:26.994 INFO:tasks.workunit.client.0.smithi136.stdout:op 1247 completed, throughput=5MB/sec 2024-09-13T00:05:26.994 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-mSbHWY2ffEh9u6K off=14529973 len=748781 2024-09-13T00:05:27.001 INFO:tasks.workunit.client.0.smithi136.stdout:op 1248 completed, throughput=5MB/sec 2024-09-13T00:05:27.002 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-oJt9DNBuNNk4i5Q off=12193633 len=1946274 2024-09-13T00:05:27.019 INFO:tasks.workunit.client.0.smithi136.stdout:op 1249 completed, throughput=5.01MB/sec 2024-09-13T00:05:27.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:05:26 smithi136 bash[75164]: cluster 2024-09-13T00:05:25.270008+0000 mgr.y (mgr.24661) 849 : cluster [DBG] pgmap v1532: 105 pgs: 105 active+clean; 126 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.2 MiB/s wr, 5 op/s 2024-09-13T00:05:27.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:05:26 smithi136 bash[76371]: cluster 2024-09-13T00:05:25.270008+0000 mgr.y (mgr.24661) 849 : cluster [DBG] pgmap v1532: 105 pgs: 105 active+clean; 126 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.2 MiB/s wr, 5 op/s 2024-09-13T00:05:27.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:05:26 smithi181 bash[67587]: cluster 2024-09-13T00:05:25.270008+0000 mgr.y (mgr.24661) 849 : cluster [DBG] pgmap v1532: 105 pgs: 105 active+clean; 126 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.2 MiB/s wr, 5 op/s 2024-09-13T00:05:28.019 INFO:tasks.workunit.client.0.smithi136.stdout: 237: throughput=4.98MB/sec pending data=0 2024-09-13T00:05:28.020 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-_4--reQ2SM8HGYd off=1297842 len=1476325 2024-09-13T00:05:28.025 INFO:tasks.workunit.client.0.smithi136.stdout:op 1250 completed, throughput=4.99MB/sec 2024-09-13T00:05:28.025 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-XtCMMLx20nnxRxD off=2327433 len=543592 2024-09-13T00:05:28.027 INFO:tasks.workunit.client.0.smithi136.stdout:op 1251 completed, throughput=4.99MB/sec 2024-09-13T00:05:28.027 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-aPVhwdAVXULEzTV off=985965 len=1938096 2024-09-13T00:05:28.031 INFO:tasks.workunit.client.0.smithi136.stdout:op 1252 completed, throughput=5MB/sec 2024-09-13T00:05:29.033 INFO:tasks.workunit.client.0.smithi136.stdout: 238: throughput=4.98MB/sec pending data=0 2024-09-13T00:05:29.033 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-E5UZfSGQzfft0gt off=9636258 len=931444 2024-09-13T00:05:29.041 INFO:tasks.workunit.client.0.smithi136.stdout:op 1253 completed, throughput=4.98MB/sec 2024-09-13T00:05:29.041 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-HV43JAzIMinU6mI off=5807664 len=1918575 2024-09-13T00:05:29.057 INFO:tasks.workunit.client.0.smithi136.stdout:op 1254 completed, throughput=4.99MB/sec 2024-09-13T00:05:29.057 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-I5kZ_l35prdu6er off=3945919 len=1327494 2024-09-13T00:05:29.060 INFO:tasks.workunit.client.0.smithi136.stdout:op 1255 completed, throughput=5MB/sec 2024-09-13T00:05:29.060 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Aka11Yhib8Ff5MZ off=11453539 len=492879 2024-09-13T00:05:29.067 INFO:tasks.workunit.client.0.smithi136.stdout:op 1256 completed, throughput=5MB/sec 2024-09-13T00:05:29.067 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-3bKjNuwGEiTgTmt off=13989077 len=478782 2024-09-13T00:05:29.068 INFO:tasks.workunit.client.0.smithi136.stdout:op 1257 completed, throughput=5MB/sec 2024-09-13T00:05:29.068 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-LGp6Ljqd4YOr2I3 off=15834937 len=252919 2024-09-13T00:05:29.073 INFO:tasks.workunit.client.0.smithi136.stdout:op 1258 completed, throughput=5MB/sec 2024-09-13T00:05:29.073 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Mth5IeiGtrSevdE off=293888 len=212775 2024-09-13T00:05:29.078 INFO:tasks.workunit.client.0.smithi136.stdout:op 1259 completed, throughput=5MB/sec 2024-09-13T00:05:29.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:05:28 smithi136 bash[76371]: cluster 2024-09-13T00:05:27.271120+0000 mgr.y (mgr.24661) 850 : cluster [DBG] pgmap v1533: 105 pgs: 105 active+clean; 126 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.8 MiB/s wr, 6 op/s 2024-09-13T00:05:29.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:05:28 smithi136 bash[75164]: cluster 2024-09-13T00:05:27.271120+0000 mgr.y (mgr.24661) 850 : cluster [DBG] pgmap v1533: 105 pgs: 105 active+clean; 126 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.8 MiB/s wr, 6 op/s 2024-09-13T00:05:29.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:05:28 smithi181 bash[67587]: cluster 2024-09-13T00:05:27.271120+0000 mgr.y (mgr.24661) 850 : cluster [DBG] pgmap v1533: 105 pgs: 105 active+clean; 126 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.8 MiB/s wr, 6 op/s 2024-09-13T00:05:29.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:05:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:05:29] "GET /metrics HTTP/1.1" 200 34123 "" "Prometheus/2.43.0" 2024-09-13T00:05:30.078 INFO:tasks.workunit.client.0.smithi136.stdout: 239: throughput=4.98MB/sec pending data=0 2024-09-13T00:05:30.079 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-q6Z2Cb4hvp_gt8E off=4038728 len=640359 2024-09-13T00:05:30.081 INFO:tasks.workunit.client.0.smithi136.stdout:op 1260 completed, throughput=4.98MB/sec 2024-09-13T00:05:30.081 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-s8njUQvancXoZ86 off=8896246 len=1986066 2024-09-13T00:05:30.087 INFO:tasks.workunit.client.0.smithi136.stdout:op 1261 completed, throughput=4.99MB/sec 2024-09-13T00:05:30.087 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-I_WPqpv-ABWIeAi off=7101476 len=704083 2024-09-13T00:05:30.090 INFO:tasks.workunit.client.0.smithi136.stdout:op 1262 completed, throughput=4.99MB/sec 2024-09-13T00:05:30.090 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-vGtJBAX-CA3nODJ off=3027925 len=407073 2024-09-13T00:05:30.096 INFO:tasks.workunit.client.0.smithi136.stdout:op 1263 completed, throughput=4.99MB/sec 2024-09-13T00:05:30.096 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-yk1wNFfA9kGI81e off=553397 len=329986 2024-09-13T00:05:30.098 INFO:tasks.workunit.client.0.smithi136.stdout:op 1264 completed, throughput=5MB/sec 2024-09-13T00:05:30.098 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-HuNaxHSKJTzySVW off=10005886 len=258680 2024-09-13T00:05:30.103 INFO:tasks.workunit.client.0.smithi136.stdout:op 1265 completed, throughput=5MB/sec 2024-09-13T00:05:30.103 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-0hFWgFAdK5jMbYw off=626830 len=718878 2024-09-13T00:05:30.110 INFO:tasks.workunit.client.0.smithi136.stdout:op 1266 completed, throughput=5MB/sec 2024-09-13T00:05:30.110 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-S9yTLhp2-E7Xbxq off=1765520 len=174980 2024-09-13T00:05:30.115 INFO:tasks.workunit.client.0.smithi136.stdout:op 1267 completed, throughput=5MB/sec 2024-09-13T00:05:30.115 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-X24MZI87KtLUWAU off=1143138 len=1994387 2024-09-13T00:05:30.120 INFO:tasks.workunit.client.0.smithi136.stdout:op 1268 completed, throughput=5.01MB/sec 2024-09-13T00:05:31.120 INFO:tasks.workunit.client.0.smithi136.stdout: 240: throughput=4.99MB/sec pending data=0 2024-09-13T00:05:31.120 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-CDrAzOM15FtjXkt off=11528493 len=1191256 2024-09-13T00:05:31.124 INFO:tasks.workunit.client.0.smithi136.stdout:op 1269 completed, throughput=4.99MB/sec 2024-09-13T00:05:31.124 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-sNC0LN6oUuyicIC off=3027166 len=1754057 2024-09-13T00:05:31.137 INFO:tasks.workunit.client.0.smithi136.stdout:op 1270 completed, throughput=5MB/sec 2024-09-13T00:05:31.137 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-mSPPGiCH5d_eDl2 off=9506084 len=1050211 2024-09-13T00:05:31.147 INFO:tasks.workunit.client.0.smithi136.stdout:op 1271 completed, throughput=5MB/sec 2024-09-13T00:05:31.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:05:30 smithi136 bash[75164]: cluster 2024-09-13T00:05:29.272020+0000 mgr.y (mgr.24661) 851 : cluster [DBG] pgmap v1534: 105 pgs: 105 active+clean; 126 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:05:31.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:05:30 smithi136 bash[76371]: cluster 2024-09-13T00:05:29.272020+0000 mgr.y (mgr.24661) 851 : cluster [DBG] pgmap v1534: 105 pgs: 105 active+clean; 126 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:05:31.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:05:30 smithi181 bash[67587]: cluster 2024-09-13T00:05:29.272020+0000 mgr.y (mgr.24661) 851 : cluster [DBG] pgmap v1534: 105 pgs: 105 active+clean; 126 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:05:32.147 INFO:tasks.workunit.client.0.smithi136.stdout: 241: throughput=4.98MB/sec pending data=0 2024-09-13T00:05:32.147 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-RgMmcW0-n3ALhfh off=3687673 len=1154129 2024-09-13T00:05:32.156 INFO:tasks.workunit.client.0.smithi136.stdout:op 1272 completed, throughput=4.99MB/sec 2024-09-13T00:05:32.157 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-E_wWAHEXg0vmgNr off=11727863 len=472925 2024-09-13T00:05:32.158 INFO:tasks.workunit.client.0.smithi136.stdout:op 1273 completed, throughput=4.99MB/sec 2024-09-13T00:05:32.158 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-IV5I-fC0ZFiZ1R0 off=8508185 len=1336715 2024-09-13T00:05:32.162 INFO:tasks.workunit.client.0.smithi136.stdout:op 1274 completed, throughput=4.99MB/sec 2024-09-13T00:05:32.162 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-k6XmQEeVV-O8UB7 off=11663225 len=233714 2024-09-13T00:05:32.167 INFO:tasks.workunit.client.0.smithi136.stdout:op 1275 completed, throughput=4.99MB/sec 2024-09-13T00:05:32.167 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-AkC9I_fhQdG9jG6 off=983506 len=952220 2024-09-13T00:05:32.171 INFO:tasks.workunit.client.0.smithi136.stdout:op 1276 completed, throughput=5MB/sec 2024-09-13T00:05:32.171 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-nLW7XkI_LEjuGiX off=2632085 len=734708 2024-09-13T00:05:32.178 INFO:tasks.workunit.client.0.smithi136.stdout:op 1277 completed, throughput=5MB/sec 2024-09-13T00:05:32.178 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-y9Df4CKKLysPh71 off=1314729 len=187334 2024-09-13T00:05:32.180 INFO:tasks.workunit.client.0.smithi136.stdout:op 1278 completed, throughput=5MB/sec 2024-09-13T00:05:32.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:05:32 smithi181 bash[95505]: ts=2024-09-13T00:05:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:05:33.180 INFO:tasks.workunit.client.0.smithi136.stdout: 242: throughput=4.98MB/sec pending data=0 2024-09-13T00:05:33.180 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-b0SilrXC1hdgIpf off=19099091 len=371740 2024-09-13T00:05:33.186 INFO:tasks.workunit.client.0.smithi136.stdout:op 1279 completed, throughput=4.98MB/sec 2024-09-13T00:05:33.186 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-JTSQ8zFc8bSK0F1 off=4805646 len=1135802 2024-09-13T00:05:33.189 INFO:tasks.workunit.client.0.smithi136.stdout:op 1280 completed, throughput=4.99MB/sec 2024-09-13T00:05:33.189 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-GrTDRtN3Iq6moh7 off=143106 len=131793 2024-09-13T00:05:33.195 INFO:tasks.workunit.client.0.smithi136.stdout:op 1281 completed, throughput=4.99MB/sec 2024-09-13T00:05:33.195 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-jINyMOVXJZ8xfIB off=1538785 len=474732 2024-09-13T00:05:33.200 INFO:tasks.workunit.client.0.smithi136.stdout:op 1282 completed, throughput=4.99MB/sec 2024-09-13T00:05:33.201 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ew85MWJj07gC6Km off=577745 len=1402934 2024-09-13T00:05:33.212 INFO:tasks.workunit.client.0.smithi136.stdout:op 1283 completed, throughput=4.99MB/sec 2024-09-13T00:05:33.212 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ow7TaPfk0N6FxPN off=13031706 len=199753 2024-09-13T00:05:33.213 INFO:tasks.workunit.client.0.smithi136.stdout:op 1284 completed, throughput=4.99MB/sec 2024-09-13T00:05:33.214 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-JZz29_EnTnMLoGp off=666222 len=425677 2024-09-13T00:05:33.215 INFO:tasks.workunit.client.0.smithi136.stdout:op 1285 completed, throughput=5MB/sec 2024-09-13T00:05:33.215 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-SFRm2REGF4vGumB off=1771570 len=931566 2024-09-13T00:05:33.218 INFO:tasks.workunit.client.0.smithi136.stdout:op 1286 completed, throughput=5MB/sec 2024-09-13T00:05:33.218 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_yPQ2dZmqzTAmby off=6657488 len=1088013 2024-09-13T00:05:33.226 INFO:tasks.workunit.client.0.smithi136.stdout:op 1287 completed, throughput=5MB/sec 2024-09-13T00:05:33.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:05:32 smithi136 bash[75164]: cluster 2024-09-13T00:05:31.273033+0000 mgr.y (mgr.24661) 852 : cluster [DBG] pgmap v1535: 105 pgs: 105 active+clean; 126 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.8 MiB/s wr, 4 op/s 2024-09-13T00:05:33.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:05:32 smithi136 bash[76371]: cluster 2024-09-13T00:05:31.273033+0000 mgr.y (mgr.24661) 852 : cluster [DBG] pgmap v1535: 105 pgs: 105 active+clean; 126 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.8 MiB/s wr, 4 op/s 2024-09-13T00:05:33.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:05:32 smithi181 bash[67587]: cluster 2024-09-13T00:05:31.273033+0000 mgr.y (mgr.24661) 852 : cluster [DBG] pgmap v1535: 105 pgs: 105 active+clean; 126 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.8 MiB/s wr, 4 op/s 2024-09-13T00:05:34.226 INFO:tasks.workunit.client.0.smithi136.stdout: 243: throughput=4.98MB/sec pending data=0 2024-09-13T00:05:34.226 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-oIX7QIL7p44BBKm off=8022260 len=13480 2024-09-13T00:05:34.227 INFO:tasks.workunit.client.0.smithi136.stdout:op 1288 completed, throughput=4.98MB/sec 2024-09-13T00:05:34.227 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-x7JAqLL_NjJKmIn off=7408319 len=896404 2024-09-13T00:05:34.236 INFO:tasks.workunit.client.0.smithi136.stdout:op 1289 completed, throughput=4.99MB/sec 2024-09-13T00:05:34.237 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-p60I84EkDqjkLhh off=16194221 len=686649 2024-09-13T00:05:34.243 INFO:tasks.workunit.client.0.smithi136.stdout:op 1290 completed, throughput=4.99MB/sec 2024-09-13T00:05:34.243 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-kpV3HkTYIC_aXpA off=1652723 len=1446472 2024-09-13T00:05:34.258 INFO:tasks.workunit.client.0.smithi136.stdout:op 1291 completed, throughput=4.99MB/sec 2024-09-13T00:05:34.258 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-NeXjBARYNnoKow2 off=2822581 len=113616 2024-09-13T00:05:34.259 INFO:tasks.workunit.client.0.smithi136.stdout:op 1292 completed, throughput=4.99MB/sec 2024-09-13T00:05:34.259 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-TnEGrVoYH5G0me0 off=43343 len=1459395 2024-09-13T00:05:34.269 INFO:tasks.workunit.client.0.smithi136.stdout:op 1293 completed, throughput=5MB/sec 2024-09-13T00:05:34.269 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-_OQoyUYqdS8b5pU off=8015552 len=319775 2024-09-13T00:05:34.270 INFO:tasks.workunit.client.0.smithi136.stdout:op 1294 completed, throughput=5MB/sec 2024-09-13T00:05:35.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:05:34 smithi136 bash[75164]: cluster 2024-09-13T00:05:33.273938+0000 mgr.y (mgr.24661) 853 : cluster [DBG] pgmap v1536: 105 pgs: 105 active+clean; 126 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:05:35.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:05:34 smithi136 bash[76371]: cluster 2024-09-13T00:05:33.273938+0000 mgr.y (mgr.24661) 853 : cluster [DBG] pgmap v1536: 105 pgs: 105 active+clean; 126 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:05:35.271 INFO:tasks.workunit.client.0.smithi136.stdout: 244: throughput=4.98MB/sec pending data=0 2024-09-13T00:05:35.271 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-hRHcj8a6QNs3762 off=2566811 len=1164848 2024-09-13T00:05:35.274 INFO:tasks.workunit.client.0.smithi136.stdout:op 1295 completed, throughput=4.99MB/sec 2024-09-13T00:05:35.274 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-wLwsbzDWif5FwbL off=4508109 len=405338 2024-09-13T00:05:35.280 INFO:tasks.workunit.client.0.smithi136.stdout:op 1296 completed, throughput=4.99MB/sec 2024-09-13T00:05:35.280 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-qlYWDdbkR6bxnEN off=5144093 len=624456 2024-09-13T00:05:35.282 INFO:tasks.workunit.client.0.smithi136.stdout:op 1297 completed, throughput=4.99MB/sec 2024-09-13T00:05:35.282 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-UviiMQ4vgrVAolp off=10772923 len=708078 2024-09-13T00:05:35.284 INFO:tasks.workunit.client.0.smithi136.stdout:op 1298 completed, throughput=4.99MB/sec 2024-09-13T00:05:35.284 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ZWUraeha5HqY6l9 off=1344035 len=797908 2024-09-13T00:05:35.287 INFO:tasks.workunit.client.0.smithi136.stdout:op 1299 completed, throughput=4.99MB/sec 2024-09-13T00:05:35.287 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-NYXp2liAhdGYUqA off=2144062 len=1796606 2024-09-13T00:05:35.291 INFO:tasks.workunit.client.0.smithi136.stdout:op 1300 completed, throughput=5MB/sec 2024-09-13T00:05:35.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:05:34 smithi181 bash[67587]: cluster 2024-09-13T00:05:33.273938+0000 mgr.y (mgr.24661) 853 : cluster [DBG] pgmap v1536: 105 pgs: 105 active+clean; 126 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:05:36.291 INFO:tasks.workunit.client.0.smithi136.stdout: 245: throughput=4.98MB/sec pending data=0 2024-09-13T00:05:36.291 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-HGAHgXqB3fakzZr off=2128954 len=735805 2024-09-13T00:05:36.293 INFO:tasks.workunit.client.0.smithi136.stdout:op 1301 completed, throughput=4.98MB/sec 2024-09-13T00:05:36.293 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-UV_NGd7HQoqivuZ off=6533130 len=1944994 2024-09-13T00:05:36.313 INFO:tasks.workunit.client.0.smithi136.stdout:op 1302 completed, throughput=4.99MB/sec 2024-09-13T00:05:36.313 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-5UD5CHj1MhVvR77 off=9906175 len=1604397 2024-09-13T00:05:36.325 INFO:tasks.workunit.client.0.smithi136.stdout:op 1303 completed, throughput=5MB/sec 2024-09-13T00:05:36.325 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-hFc7egzQ3uiVRMB off=5786424 len=490199 2024-09-13T00:05:36.331 INFO:tasks.workunit.client.0.smithi136.stdout:op 1304 completed, throughput=5MB/sec 2024-09-13T00:05:36.331 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Ewn48mCdr6LHhgx off=9482582 len=350213 2024-09-13T00:05:36.336 INFO:tasks.workunit.client.0.smithi136.stdout:op 1305 completed, throughput=5MB/sec 2024-09-13T00:05:37.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:05:36 smithi136 bash[75164]: cluster 2024-09-13T00:05:35.274805+0000 mgr.y (mgr.24661) 854 : cluster [DBG] pgmap v1537: 105 pgs: 105 active+clean; 126 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:05:37.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:05:36 smithi136 bash[76371]: cluster 2024-09-13T00:05:35.274805+0000 mgr.y (mgr.24661) 854 : cluster [DBG] pgmap v1537: 105 pgs: 105 active+clean; 126 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:05:37.336 INFO:tasks.workunit.client.0.smithi136.stdout: 246: throughput=4.98MB/sec pending data=0 2024-09-13T00:05:37.337 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-BiN3AqMWRBNiGHJ off=2837254 len=509012 2024-09-13T00:05:37.343 INFO:tasks.workunit.client.0.smithi136.stdout:op 1306 completed, throughput=4.98MB/sec 2024-09-13T00:05:37.343 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-zCjRJ2Qswki4ZRA off=3945731 len=914817 2024-09-13T00:05:37.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:05:36 smithi181 bash[67587]: cluster 2024-09-13T00:05:35.274805+0000 mgr.y (mgr.24661) 854 : cluster [DBG] pgmap v1537: 105 pgs: 105 active+clean; 126 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:05:37.353 INFO:tasks.workunit.client.0.smithi136.stdout:op 1307 completed, throughput=4.99MB/sec 2024-09-13T00:05:37.353 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-LclkUrFI8KSfnOq off=4836363 len=541588 2024-09-13T00:05:37.355 INFO:tasks.workunit.client.0.smithi136.stdout:op 1308 completed, throughput=4.99MB/sec 2024-09-13T00:05:37.355 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-6x7S30ON6TqooKD off=1666155 len=214923 2024-09-13T00:05:37.357 INFO:tasks.workunit.client.0.smithi136.stdout:op 1309 completed, throughput=4.99MB/sec 2024-09-13T00:05:37.357 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-mPCa48JDHRlUXW- off=10967376 len=1153349 2024-09-13T00:05:37.366 INFO:tasks.workunit.client.0.smithi136.stdout:op 1310 completed, throughput=4.99MB/sec 2024-09-13T00:05:37.366 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-CiC5mivP8DLGdvo off=18574942 len=1190388 2024-09-13T00:05:37.376 INFO:tasks.workunit.client.0.smithi136.stdout:op 1311 completed, throughput=5MB/sec 2024-09-13T00:05:37.376 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Mppy4pprx0eTGQj off=7636410 len=1170892 2024-09-13T00:05:37.384 INFO:tasks.workunit.client.0.smithi136.stdout:op 1312 completed, throughput=5MB/sec 2024-09-13T00:05:38.384 INFO:tasks.workunit.client.0.smithi136.stdout: 247: throughput=4.98MB/sec pending data=0 2024-09-13T00:05:38.385 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-s1bXq6loPTqnA-h off=2749943 len=718520 2024-09-13T00:05:38.392 INFO:tasks.workunit.client.0.smithi136.stdout:op 1313 completed, throughput=4.98MB/sec 2024-09-13T00:05:38.392 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-C9-KMvCFrFSHzKo off=4876928 len=953382 2024-09-13T00:05:38.400 INFO:tasks.workunit.client.0.smithi136.stdout:op 1314 completed, throughput=4.99MB/sec 2024-09-13T00:05:38.400 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-MYiVUtmAOgNLnjx off=4502089 len=899457 2024-09-13T00:05:38.403 INFO:tasks.workunit.client.0.smithi136.stdout:op 1315 completed, throughput=4.99MB/sec 2024-09-13T00:05:38.404 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-iBvkG860dfZS5sd off=7432595 len=1878030 2024-09-13T00:05:38.409 INFO:tasks.workunit.client.0.smithi136.stdout:op 1316 completed, throughput=5MB/sec 2024-09-13T00:05:38.409 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-3_Hcj0hC1v1SKbD off=13792888 len=29836 2024-09-13T00:05:38.413 INFO:tasks.workunit.client.0.smithi136.stdout:op 1317 completed, throughput=5MB/sec 2024-09-13T00:05:38.413 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-b9qaj-VUZ4FW3I_ off=13274137 len=2060987 2024-09-13T00:05:38.418 INFO:tasks.workunit.client.0.smithi136.stdout:op 1318 completed, throughput=5.01MB/sec 2024-09-13T00:05:39.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:05:38 smithi136 bash[75164]: cluster 2024-09-13T00:05:37.275903+0000 mgr.y (mgr.24661) 855 : cluster [DBG] pgmap v1538: 105 pgs: 105 active+clean; 126 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.7 MiB/s wr, 6 op/s 2024-09-13T00:05:39.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:05:38 smithi136 bash[75164]: audit 2024-09-13T00:05:38.778407+0000 mon.a (mon.0) 2141 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:05:39.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:05:38 smithi136 bash[76371]: cluster 2024-09-13T00:05:37.275903+0000 mgr.y (mgr.24661) 855 : cluster [DBG] pgmap v1538: 105 pgs: 105 active+clean; 126 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.7 MiB/s wr, 6 op/s 2024-09-13T00:05:39.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:05:38 smithi136 bash[76371]: audit 2024-09-13T00:05:38.778407+0000 mon.a (mon.0) 2141 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:05:39.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:05:38 smithi181 bash[67587]: cluster 2024-09-13T00:05:37.275903+0000 mgr.y (mgr.24661) 855 : cluster [DBG] pgmap v1538: 105 pgs: 105 active+clean; 126 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.7 MiB/s wr, 6 op/s 2024-09-13T00:05:39.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:05:38 smithi181 bash[67587]: audit 2024-09-13T00:05:38.778407+0000 mon.a (mon.0) 2141 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:05:39.419 INFO:tasks.workunit.client.0.smithi136.stdout: 248: throughput=4.99MB/sec pending data=0 2024-09-13T00:05:39.419 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-gGYJom2oqJ_mx-R off=2103283 len=73113 2024-09-13T00:05:39.420 INFO:tasks.workunit.client.0.smithi136.stdout:op 1319 completed, throughput=4.99MB/sec 2024-09-13T00:05:39.420 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-YbJJRoBYdcSIkWc off=856248 len=398719 2024-09-13T00:05:39.421 INFO:tasks.workunit.client.0.smithi136.stdout:op 1320 completed, throughput=4.99MB/sec 2024-09-13T00:05:39.421 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-CQdCAGoGEEt-8q4 off=552338 len=1647982 2024-09-13T00:05:39.425 INFO:tasks.workunit.client.0.smithi136.stdout:op 1321 completed, throughput=4.99MB/sec 2024-09-13T00:05:39.425 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-J5kF33hpnmUNLVB off=3139558 len=1342110 2024-09-13T00:05:39.429 INFO:tasks.workunit.client.0.smithi136.stdout:op 1322 completed, throughput=5MB/sec 2024-09-13T00:05:39.429 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-lBDN19razU8IA3Q off=1911781 len=187455 2024-09-13T00:05:39.434 INFO:tasks.workunit.client.0.smithi136.stdout:op 1323 completed, throughput=5MB/sec 2024-09-13T00:05:39.434 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-JsRVX57M5WPVXAy off=260748 len=915903 2024-09-13T00:05:39.442 INFO:tasks.workunit.client.0.smithi136.stdout:op 1324 completed, throughput=5MB/sec 2024-09-13T00:05:39.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:05:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:05:39] "GET /metrics HTTP/1.1" 200 34125 "" "Prometheus/2.43.0" 2024-09-13T00:05:40.443 INFO:tasks.workunit.client.0.smithi136.stdout: 249: throughput=4.98MB/sec pending data=0 2024-09-13T00:05:40.443 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ecFWsuMTOF0RAsO off=915236 len=1344812 2024-09-13T00:05:40.446 INFO:tasks.workunit.client.0.smithi136.stdout:op 1325 completed, throughput=4.99MB/sec 2024-09-13T00:05:40.446 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-sHRUnmHgNG63b8q off=4395504 len=1588075 2024-09-13T00:05:40.461 INFO:tasks.workunit.client.0.smithi136.stdout:op 1326 completed, throughput=4.99MB/sec 2024-09-13T00:05:40.461 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-_M9Bnj3SKESZU56 off=19074924 len=1592817 2024-09-13T00:05:40.465 INFO:tasks.workunit.client.0.smithi136.stdout:op 1327 completed, throughput=5MB/sec 2024-09-13T00:05:40.465 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj--hKA5EbeBxFs6wQ off=6908018 len=50377 2024-09-13T00:05:40.469 INFO:tasks.workunit.client.0.smithi136.stdout:op 1328 completed, throughput=5MB/sec 2024-09-13T00:05:40.469 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-77hDQQqCCOygvhG off=21810468 len=567624 2024-09-13T00:05:40.476 INFO:tasks.workunit.client.0.smithi136.stdout:op 1329 completed, throughput=5MB/sec 2024-09-13T00:05:41.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:05:40 smithi136 bash[76371]: cluster 2024-09-13T00:05:39.276816+0000 mgr.y (mgr.24661) 856 : cluster [DBG] pgmap v1539: 105 pgs: 105 active+clean; 126 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:05:41.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:05:40 smithi136 bash[75164]: cluster 2024-09-13T00:05:39.276816+0000 mgr.y (mgr.24661) 856 : cluster [DBG] pgmap v1539: 105 pgs: 105 active+clean; 126 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:05:41.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:05:40 smithi181 bash[67587]: cluster 2024-09-13T00:05:39.276816+0000 mgr.y (mgr.24661) 856 : cluster [DBG] pgmap v1539: 105 pgs: 105 active+clean; 126 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:05:41.476 INFO:tasks.workunit.client.0.smithi136.stdout: 250: throughput=4.98MB/sec pending data=0 2024-09-13T00:05:41.476 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-womwHtNx0aFOxtV off=1994315 len=863643 2024-09-13T00:05:41.479 INFO:tasks.workunit.client.0.smithi136.stdout:op 1330 completed, throughput=4.98MB/sec 2024-09-13T00:05:41.479 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Z_DzpAkOwxrPFNe off=3417258 len=1853170 2024-09-13T00:05:41.495 INFO:tasks.workunit.client.0.smithi136.stdout:op 1331 completed, throughput=4.99MB/sec 2024-09-13T00:05:41.495 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj--oCtKRY9qk0ebkq off=3142579 len=1898558 2024-09-13T00:05:41.510 INFO:tasks.workunit.client.0.smithi136.stdout:op 1332 completed, throughput=5MB/sec 2024-09-13T00:05:41.510 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-moN_DeyeShb_aQp off=449347 len=1479527 2024-09-13T00:05:41.514 INFO:tasks.workunit.client.0.smithi136.stdout:op 1333 completed, throughput=5MB/sec 2024-09-13T00:05:42.514 INFO:tasks.workunit.client.0.smithi136.stdout: 251: throughput=4.98MB/sec pending data=0 2024-09-13T00:05:42.515 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-V-3fSFqFJzuXz82 off=1705044 len=560090 2024-09-13T00:05:42.517 INFO:tasks.workunit.client.0.smithi136.stdout:op 1334 completed, throughput=4.99MB/sec 2024-09-13T00:05:42.517 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-45pgDsvVVEOhfvj off=332118 len=795490 2024-09-13T00:05:42.524 INFO:tasks.workunit.client.0.smithi136.stdout:op 1335 completed, throughput=4.99MB/sec 2024-09-13T00:05:42.525 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2ZV03rt34E5SPFL off=1214405 len=448121 2024-09-13T00:05:42.526 INFO:tasks.workunit.client.0.smithi136.stdout:op 1336 completed, throughput=4.99MB/sec 2024-09-13T00:05:42.527 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-twFWT8_JKH0v61j off=2406451 len=1116060 2024-09-13T00:05:42.535 INFO:tasks.workunit.client.0.smithi136.stdout:op 1337 completed, throughput=4.99MB/sec 2024-09-13T00:05:42.535 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-egFbl11_1dqM0do off=8932047 len=243537 2024-09-13T00:05:42.540 INFO:tasks.workunit.client.0.smithi136.stdout:op 1338 completed, throughput=5MB/sec 2024-09-13T00:05:42.540 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ODEwtwF4fSC9VfL off=1162888 len=1944457 2024-09-13T00:05:42.543 INFO:tasks.workunit.client.0.smithi136.stdout:op 1339 completed, throughput=5MB/sec 2024-09-13T00:05:42.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:05:42 smithi181 bash[95505]: ts=2024-09-13T00:05:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:05:43.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:05:42 smithi136 bash[75164]: cluster 2024-09-13T00:05:41.277734+0000 mgr.y (mgr.24661) 857 : cluster [DBG] pgmap v1540: 105 pgs: 105 active+clean; 126 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.6 MiB/s wr, 6 op/s 2024-09-13T00:05:43.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:05:42 smithi136 bash[76371]: cluster 2024-09-13T00:05:41.277734+0000 mgr.y (mgr.24661) 857 : cluster [DBG] pgmap v1540: 105 pgs: 105 active+clean; 126 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.6 MiB/s wr, 6 op/s 2024-09-13T00:05:43.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:05:42 smithi181 bash[67587]: cluster 2024-09-13T00:05:41.277734+0000 mgr.y (mgr.24661) 857 : cluster [DBG] pgmap v1540: 105 pgs: 105 active+clean; 126 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.6 MiB/s wr, 6 op/s 2024-09-13T00:05:43.544 INFO:tasks.workunit.client.0.smithi136.stdout: 252: throughput=4.98MB/sec pending data=0 2024-09-13T00:05:43.544 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-7ZW0KP_nSxlNP29 off=4647986 len=1048576 2024-09-13T00:05:43.552 INFO:tasks.workunit.client.0.smithi136.stdout:op 1340 completed, throughput=4.99MB/sec 2024-09-13T00:05:43.552 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-DQmHWy15N2BsZ6w off=11077495 len=86577 2024-09-13T00:05:43.553 INFO:tasks.workunit.client.0.smithi136.stdout:op 1341 completed, throughput=4.99MB/sec 2024-09-13T00:05:43.553 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-1334USv1Sv76NIy off=13242725 len=1104507 2024-09-13T00:05:43.563 INFO:tasks.workunit.client.0.smithi136.stdout:op 1342 completed, throughput=4.99MB/sec 2024-09-13T00:05:43.563 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-QN5KebFIqTJD_fp off=29539 len=650621 2024-09-13T00:05:43.565 INFO:tasks.workunit.client.0.smithi136.stdout:op 1343 completed, throughput=4.99MB/sec 2024-09-13T00:05:43.565 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-RhUCgMh159kQwdn off=1706827 len=2030436 2024-09-13T00:05:43.569 INFO:tasks.workunit.client.0.smithi136.stdout:op 1344 completed, throughput=5MB/sec 2024-09-13T00:05:44.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:05:43 smithi136 bash[75164]: audit 2024-09-13T00:05:43.410935+0000 mon.a (mon.0) 2142 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:05:44.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:05:43 smithi136 bash[75164]: audit 2024-09-13T00:05:43.780714+0000 mon.a (mon.0) 2143 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:05:44.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:05:43 smithi136 bash[75164]: audit 2024-09-13T00:05:43.788222+0000 mon.a (mon.0) 2144 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:05:44.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:05:43 smithi136 bash[76371]: audit 2024-09-13T00:05:43.410935+0000 mon.a (mon.0) 2142 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:05:44.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:05:43 smithi136 bash[76371]: audit 2024-09-13T00:05:43.780714+0000 mon.a (mon.0) 2143 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:05:44.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:05:43 smithi136 bash[76371]: audit 2024-09-13T00:05:43.788222+0000 mon.a (mon.0) 2144 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:05:44.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:05:43 smithi181 bash[67587]: audit 2024-09-13T00:05:43.410935+0000 mon.a (mon.0) 2142 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:05:44.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:05:43 smithi181 bash[67587]: audit 2024-09-13T00:05:43.780714+0000 mon.a (mon.0) 2143 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:05:44.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:05:43 smithi181 bash[67587]: audit 2024-09-13T00:05:43.788222+0000 mon.a (mon.0) 2144 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:05:44.570 INFO:tasks.workunit.client.0.smithi136.stdout: 253: throughput=4.98MB/sec pending data=0 2024-09-13T00:05:44.570 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-eSVlgCfprtCaFar off=1447923 len=6660 2024-09-13T00:05:44.574 INFO:tasks.workunit.client.0.smithi136.stdout:op 1345 completed, throughput=4.98MB/sec 2024-09-13T00:05:44.574 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-jkxbYNvJSOgj8o0 off=14875924 len=814702 2024-09-13T00:05:44.582 INFO:tasks.workunit.client.0.smithi136.stdout:op 1346 completed, throughput=4.98MB/sec 2024-09-13T00:05:44.582 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-nI1Sqbbr5VLP33L off=3782203 len=587098 2024-09-13T00:05:44.584 INFO:tasks.workunit.client.0.smithi136.stdout:op 1347 completed, throughput=4.99MB/sec 2024-09-13T00:05:44.584 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-a_2V8o6ecxKa84g off=178800 len=1739293 2024-09-13T00:05:44.596 INFO:tasks.workunit.client.0.smithi136.stdout:op 1348 completed, throughput=4.99MB/sec 2024-09-13T00:05:44.596 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-cJkGw7V_iQwbJOR off=1602837 len=990462 2024-09-13T00:05:44.605 INFO:tasks.workunit.client.0.smithi136.stdout:op 1349 completed, throughput=5MB/sec 2024-09-13T00:05:44.605 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-smch0ZBAejx6E8m off=3364716 len=353544 2024-09-13T00:05:44.606 INFO:tasks.workunit.client.0.smithi136.stdout:op 1350 completed, throughput=5MB/sec 2024-09-13T00:05:44.606 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-E5Dv2N2UucinPoh off=5910797 len=191414 2024-09-13T00:05:44.607 INFO:tasks.workunit.client.0.smithi136.stdout:op 1351 completed, throughput=5MB/sec 2024-09-13T00:05:44.607 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-wbWPj42SZkW8LVz off=7633615 len=422106 2024-09-13T00:05:44.613 INFO:tasks.workunit.client.0.smithi136.stdout:op 1352 completed, throughput=5MB/sec 2024-09-13T00:05:44.613 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-KS8IZOk_1TAUqxX off=2772272 len=984094 2024-09-13T00:05:44.620 INFO:tasks.workunit.client.0.smithi136.stdout:op 1353 completed, throughput=5MB/sec 2024-09-13T00:05:45.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:05:44 smithi136 bash[75164]: cluster 2024-09-13T00:05:43.278688+0000 mgr.y (mgr.24661) 858 : cluster [DBG] pgmap v1541: 105 pgs: 105 active+clean; 126 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.4 MiB/s wr, 6 op/s 2024-09-13T00:05:45.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:05:44 smithi136 bash[75164]: audit 2024-09-13T00:05:44.204002+0000 mon.a (mon.0) 2145 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:05:45.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:05:44 smithi136 bash[75164]: audit 2024-09-13T00:05:44.205213+0000 mon.a (mon.0) 2146 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:05:45.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:05:44 smithi136 bash[75164]: audit 2024-09-13T00:05:44.214309+0000 mon.a (mon.0) 2147 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:05:45.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:05:44 smithi136 bash[76371]: cluster 2024-09-13T00:05:43.278688+0000 mgr.y (mgr.24661) 858 : cluster [DBG] pgmap v1541: 105 pgs: 105 active+clean; 126 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.4 MiB/s wr, 6 op/s 2024-09-13T00:05:45.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:05:44 smithi136 bash[76371]: audit 2024-09-13T00:05:44.204002+0000 mon.a (mon.0) 2145 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:05:45.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:05:44 smithi136 bash[76371]: audit 2024-09-13T00:05:44.205213+0000 mon.a (mon.0) 2146 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:05:45.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:05:44 smithi136 bash[76371]: audit 2024-09-13T00:05:44.214309+0000 mon.a (mon.0) 2147 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:05:45.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:05:44 smithi181 bash[67587]: cluster 2024-09-13T00:05:43.278688+0000 mgr.y (mgr.24661) 858 : cluster [DBG] pgmap v1541: 105 pgs: 105 active+clean; 126 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.4 MiB/s wr, 6 op/s 2024-09-13T00:05:45.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:05:44 smithi181 bash[67587]: audit 2024-09-13T00:05:44.204002+0000 mon.a (mon.0) 2145 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:05:45.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:05:44 smithi181 bash[67587]: audit 2024-09-13T00:05:44.205213+0000 mon.a (mon.0) 2146 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:05:45.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:05:44 smithi181 bash[67587]: audit 2024-09-13T00:05:44.214309+0000 mon.a (mon.0) 2147 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:05:45.621 INFO:tasks.workunit.client.0.smithi136.stdout: 254: throughput=4.98MB/sec pending data=0 2024-09-13T00:05:45.621 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-hQuivZ3ks7qFGV3 off=157895 len=1539046 2024-09-13T00:05:45.631 INFO:tasks.workunit.client.0.smithi136.stdout:op 1354 completed, throughput=4.99MB/sec 2024-09-13T00:05:45.631 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-YIx6oPdZlw9L3NF off=7433218 len=1910715 2024-09-13T00:05:45.644 INFO:tasks.workunit.client.0.smithi136.stdout:op 1355 completed, throughput=5MB/sec 2024-09-13T00:05:45.644 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-7bwaoI79y2UH0Yu off=0 len=1130166 2024-09-13T00:05:45.652 INFO:tasks.workunit.client.0.smithi136.stdout:op 1356 completed, throughput=5MB/sec 2024-09-13T00:05:45.652 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-I_LfDC4E2nYuasw off=288892 len=390982 2024-09-13T00:05:45.657 INFO:tasks.workunit.client.0.smithi136.stdout:op 1357 completed, throughput=5MB/sec 2024-09-13T00:05:46.658 INFO:tasks.workunit.client.0.smithi136.stdout: 255: throughput=4.98MB/sec pending data=0 2024-09-13T00:05:46.658 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-xVhiefuxfZVZ4ct off=11196222 len=1373918 2024-09-13T00:05:46.661 INFO:tasks.workunit.client.0.smithi136.stdout:op 1358 completed, throughput=4.99MB/sec 2024-09-13T00:05:46.662 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Ug_k_NoWs3viRvn off=2902941 len=179694 2024-09-13T00:05:46.666 INFO:tasks.workunit.client.0.smithi136.stdout:op 1359 completed, throughput=4.99MB/sec 2024-09-13T00:05:46.666 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-7vUQDRfu3odHdBf off=1291986 len=68683 2024-09-13T00:05:46.670 INFO:tasks.workunit.client.0.smithi136.stdout:op 1360 completed, throughput=4.99MB/sec 2024-09-13T00:05:46.670 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-YMRbXiGSjWBto3z off=2689267 len=681230 2024-09-13T00:05:46.677 INFO:tasks.workunit.client.0.smithi136.stdout:op 1361 completed, throughput=4.99MB/sec 2024-09-13T00:05:46.677 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-cn8v1klRmhMfyW- off=513783 len=812863 2024-09-13T00:05:46.680 INFO:tasks.workunit.client.0.smithi136.stdout:op 1362 completed, throughput=4.99MB/sec 2024-09-13T00:05:46.680 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-J-Famk1FlwwTxXz off=332046 len=1478745 2024-09-13T00:05:46.685 INFO:tasks.workunit.client.0.smithi136.stdout:op 1363 completed, throughput=5MB/sec 2024-09-13T00:05:46.686 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj--eB3Eq8BxofMcPw off=7720468 len=507290 2024-09-13T00:05:46.688 INFO:tasks.workunit.client.0.smithi136.stdout:op 1364 completed, throughput=5MB/sec 2024-09-13T00:05:47.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:05:46 smithi136 bash[75164]: cluster 2024-09-13T00:05:45.279545+0000 mgr.y (mgr.24661) 859 : cluster [DBG] pgmap v1542: 105 pgs: 105 active+clean; 126 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:05:47.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:05:46 smithi136 bash[76371]: cluster 2024-09-13T00:05:45.279545+0000 mgr.y (mgr.24661) 859 : cluster [DBG] pgmap v1542: 105 pgs: 105 active+clean; 126 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:05:47.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:05:46 smithi181 bash[67587]: cluster 2024-09-13T00:05:45.279545+0000 mgr.y (mgr.24661) 859 : cluster [DBG] pgmap v1542: 105 pgs: 105 active+clean; 126 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:05:47.688 INFO:tasks.workunit.client.0.smithi136.stdout: 256: throughput=4.98MB/sec pending data=0 2024-09-13T00:05:47.688 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-vr8cNCAR1XaeC7q off=1910143 len=1684136 2024-09-13T00:05:47.702 INFO:tasks.workunit.client.0.smithi136.stdout:op 1365 completed, throughput=4.99MB/sec 2024-09-13T00:05:47.702 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-kJwxDIqoZJjzSqZ off=8864734 len=1395249 2024-09-13T00:05:47.705 INFO:tasks.workunit.client.0.smithi136.stdout:op 1366 completed, throughput=4.99MB/sec 2024-09-13T00:05:47.705 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-aPVhwdAVXULEzTV off=10913360 len=141519 2024-09-13T00:05:47.709 INFO:tasks.workunit.client.0.smithi136.stdout:op 1367 completed, throughput=4.99MB/sec 2024-09-13T00:05:47.709 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-iqRRXWYWYQQIL3y off=3240452 len=1556800 2024-09-13T00:05:47.713 INFO:tasks.workunit.client.0.smithi136.stdout:op 1368 completed, throughput=5MB/sec 2024-09-13T00:05:47.713 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-uXfx_P7uKJj-H8S off=636881 len=273185 2024-09-13T00:05:47.718 INFO:tasks.workunit.client.0.smithi136.stdout:op 1369 completed, throughput=5MB/sec 2024-09-13T00:05:47.718 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-LVmKtwaZg1J2VH6 off=7755368 len=348574 2024-09-13T00:05:47.719 INFO:tasks.workunit.client.0.smithi136.stdout:op 1370 completed, throughput=5MB/sec 2024-09-13T00:05:48.720 INFO:tasks.workunit.client.0.smithi136.stdout: 257: throughput=4.98MB/sec pending data=0 2024-09-13T00:05:48.720 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Bd5OqLfTUTzC71m off=3784360 len=468592 2024-09-13T00:05:48.721 INFO:tasks.workunit.client.0.smithi136.stdout:op 1371 completed, throughput=4.98MB/sec 2024-09-13T00:05:48.722 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-FW2bM8xYFxHjuY4 off=11414701 len=447819 2024-09-13T00:05:48.728 INFO:tasks.workunit.client.0.smithi136.stdout:op 1372 completed, throughput=4.98MB/sec 2024-09-13T00:05:48.728 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-X7NGaBf1FbdVCgO off=1813360 len=1452836 2024-09-13T00:05:48.732 INFO:tasks.workunit.client.0.smithi136.stdout:op 1373 completed, throughput=4.99MB/sec 2024-09-13T00:05:48.732 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-kBhwvGRH0tAWtiW off=12769882 len=1702590 2024-09-13T00:05:48.736 INFO:tasks.workunit.client.0.smithi136.stdout:op 1374 completed, throughput=5MB/sec 2024-09-13T00:05:48.736 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-pZ80vndzgt0GxET off=5266515 len=420816 2024-09-13T00:05:48.741 INFO:tasks.workunit.client.0.smithi136.stdout:op 1375 completed, throughput=5MB/sec 2024-09-13T00:05:48.741 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-FTvSlTGkjnv6GzF off=17827630 len=690779 2024-09-13T00:05:48.743 INFO:tasks.workunit.client.0.smithi136.stdout:op 1376 completed, throughput=5MB/sec 2024-09-13T00:05:48.743 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-lD5Ol2bCoJVBpK0 off=4169024 len=963254 2024-09-13T00:05:48.752 INFO:tasks.workunit.client.0.smithi136.stdout:op 1377 completed, throughput=5MB/sec 2024-09-13T00:05:49.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:05:48 smithi136 bash[76371]: cluster 2024-09-13T00:05:47.280717+0000 mgr.y (mgr.24661) 860 : cluster [DBG] pgmap v1543: 105 pgs: 105 active+clean; 126 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.9 MiB/s wr, 6 op/s 2024-09-13T00:05:49.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:05:48 smithi136 bash[75164]: cluster 2024-09-13T00:05:47.280717+0000 mgr.y (mgr.24661) 860 : cluster [DBG] pgmap v1543: 105 pgs: 105 active+clean; 126 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.9 MiB/s wr, 6 op/s 2024-09-13T00:05:49.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:05:48 smithi181 bash[67587]: cluster 2024-09-13T00:05:47.280717+0000 mgr.y (mgr.24661) 860 : cluster [DBG] pgmap v1543: 105 pgs: 105 active+clean; 126 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.9 MiB/s wr, 6 op/s 2024-09-13T00:05:49.753 INFO:tasks.workunit.client.0.smithi136.stdout: 258: throughput=4.98MB/sec pending data=0 2024-09-13T00:05:49.753 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-oPVsdBOUp3VlWQJ off=0 len=1472665 2024-09-13T00:05:49.757 INFO:tasks.workunit.client.0.smithi136.stdout:op 1378 completed, throughput=4.99MB/sec 2024-09-13T00:05:49.757 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-5mwRCIFxEoJktbG off=6435306 len=680290 2024-09-13T00:05:49.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:05:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:05:49] "GET /metrics HTTP/1.1" 200 34125 "" "Prometheus/2.43.0" 2024-09-13T00:05:49.764 INFO:tasks.workunit.client.0.smithi136.stdout:op 1379 completed, throughput=4.99MB/sec 2024-09-13T00:05:49.765 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-anhCLWfm0BMY36T off=8858279 len=700109 2024-09-13T00:05:49.771 INFO:tasks.workunit.client.0.smithi136.stdout:op 1380 completed, throughput=4.99MB/sec 2024-09-13T00:05:49.771 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-4r0JE8ngrfls3RL off=12341373 len=314664 2024-09-13T00:05:49.776 INFO:tasks.workunit.client.0.smithi136.stdout:op 1381 completed, throughput=4.99MB/sec 2024-09-13T00:05:49.777 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-C4lJB-hVMW30Zu_ off=582803 len=1585341 2024-09-13T00:05:49.781 INFO:tasks.workunit.client.0.smithi136.stdout:op 1382 completed, throughput=5MB/sec 2024-09-13T00:05:50.781 INFO:tasks.workunit.client.0.smithi136.stdout: 259: throughput=4.98MB/sec pending data=0 2024-09-13T00:05:50.781 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ybUZyofNH6Ib_vC off=10534178 len=221074 2024-09-13T00:05:50.786 INFO:tasks.workunit.client.0.smithi136.stdout:op 1383 completed, throughput=4.98MB/sec 2024-09-13T00:05:50.786 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-BZO3D6iDyYy_AAy off=1455968 len=614958 2024-09-13T00:05:50.793 INFO:tasks.workunit.client.0.smithi136.stdout:op 1384 completed, throughput=4.98MB/sec 2024-09-13T00:05:50.793 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-_ZqJOLl14vUDw8Q off=13844191 len=970017 2024-09-13T00:05:50.797 INFO:tasks.workunit.client.0.smithi136.stdout:op 1385 completed, throughput=4.99MB/sec 2024-09-13T00:05:50.797 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-_OSt0a0N8XciSBj off=3343862 len=1751192 2024-09-13T00:05:50.801 INFO:tasks.workunit.client.0.smithi136.stdout:op 1386 completed, throughput=4.99MB/sec 2024-09-13T00:05:50.801 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-z6ZLZIYf5XZcIvw off=18423482 len=1402252 2024-09-13T00:05:50.805 INFO:tasks.workunit.client.0.smithi136.stdout:op 1387 completed, throughput=5MB/sec 2024-09-13T00:05:50.805 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-cZcM9NjEVZHWTzO off=0 len=1681977 2024-09-13T00:05:50.811 INFO:tasks.workunit.client.0.smithi136.stdout:op 1388 completed, throughput=5MB/sec 2024-09-13T00:05:51.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:05:50 smithi136 bash[75164]: cluster 2024-09-13T00:05:49.281589+0000 mgr.y (mgr.24661) 861 : cluster [DBG] pgmap v1544: 105 pgs: 105 active+clean; 126 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 1.9 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:05:51.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:05:50 smithi136 bash[76371]: cluster 2024-09-13T00:05:49.281589+0000 mgr.y (mgr.24661) 861 : cluster [DBG] pgmap v1544: 105 pgs: 105 active+clean; 126 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 1.9 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:05:51.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:05:50 smithi181 bash[67587]: cluster 2024-09-13T00:05:49.281589+0000 mgr.y (mgr.24661) 861 : cluster [DBG] pgmap v1544: 105 pgs: 105 active+clean; 126 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 1.9 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:05:51.811 INFO:tasks.workunit.client.0.smithi136.stdout: 260: throughput=4.99MB/sec pending data=0 2024-09-13T00:05:51.812 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-vQItEjQNJCWN5oH off=975313 len=2050932 2024-09-13T00:05:51.827 INFO:tasks.workunit.client.0.smithi136.stdout:op 1389 completed, throughput=4.99MB/sec 2024-09-13T00:05:51.827 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-8ygBJjnIBoXHGTR off=3815508 len=2060229 2024-09-13T00:05:51.831 INFO:tasks.workunit.client.0.smithi136.stdout:op 1390 completed, throughput=5MB/sec 2024-09-13T00:05:52.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:05:52 smithi181 bash[95505]: ts=2024-09-13T00:05:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:05:52.831 INFO:tasks.workunit.client.0.smithi136.stdout: 261: throughput=4.98MB/sec pending data=0 2024-09-13T00:05:52.831 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Am2jrlGqoJbujJg off=4298897 len=194845 2024-09-13T00:05:52.837 INFO:tasks.workunit.client.0.smithi136.stdout:op 1391 completed, throughput=4.98MB/sec 2024-09-13T00:05:52.837 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-xHm2KNNM97tlft7 off=2361729 len=119895 2024-09-13T00:05:52.838 INFO:tasks.workunit.client.0.smithi136.stdout:op 1392 completed, throughput=4.98MB/sec 2024-09-13T00:05:52.838 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-vjMfhYuIvpP6VXP off=6196298 len=1888772 2024-09-13T00:05:52.852 INFO:tasks.workunit.client.0.smithi136.stdout:op 1393 completed, throughput=4.99MB/sec 2024-09-13T00:05:52.852 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-oIKLWUIC6ihi3wM off=9207134 len=624213 2024-09-13T00:05:52.860 INFO:tasks.workunit.client.0.smithi136.stdout:op 1394 completed, throughput=4.99MB/sec 2024-09-13T00:05:52.860 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-KN_XEAKhU2nNBzO off=3263503 len=290936 2024-09-13T00:05:52.862 INFO:tasks.workunit.client.0.smithi136.stdout:op 1395 completed, throughput=4.99MB/sec 2024-09-13T00:05:52.862 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-sTZyyIX8gq0XyrI off=5758139 len=1663857 2024-09-13T00:05:52.875 INFO:tasks.workunit.client.0.smithi136.stdout:op 1396 completed, throughput=5MB/sec 2024-09-13T00:05:52.875 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-F0XqJ8Mgx6XwDwk off=18441553 len=387938 2024-09-13T00:05:52.876 INFO:tasks.workunit.client.0.smithi136.stdout:op 1397 completed, throughput=5MB/sec 2024-09-13T00:05:52.877 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-FrvDwBAoRLnzZT0 off=20138167 len=452574 2024-09-13T00:05:52.883 INFO:tasks.workunit.client.0.smithi136.stdout:op 1398 completed, throughput=5MB/sec 2024-09-13T00:05:53.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:05:52 smithi136 bash[75164]: cluster 2024-09-13T00:05:51.282578+0000 mgr.y (mgr.24661) 862 : cluster [DBG] pgmap v1545: 105 pgs: 105 active+clean; 126 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:05:53.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:05:52 smithi136 bash[76371]: cluster 2024-09-13T00:05:51.282578+0000 mgr.y (mgr.24661) 862 : cluster [DBG] pgmap v1545: 105 pgs: 105 active+clean; 126 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:05:53.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:05:52 smithi181 bash[67587]: cluster 2024-09-13T00:05:51.282578+0000 mgr.y (mgr.24661) 862 : cluster [DBG] pgmap v1545: 105 pgs: 105 active+clean; 126 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:05:53.883 INFO:tasks.workunit.client.0.smithi136.stdout: 262: throughput=4.98MB/sec pending data=0 2024-09-13T00:05:53.883 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-MN5Xs8CkLYIZ8nR off=4398118 len=133066 2024-09-13T00:05:53.889 INFO:tasks.workunit.client.0.smithi136.stdout:op 1399 completed, throughput=4.98MB/sec 2024-09-13T00:05:53.889 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-EKag3sMgH794DMk off=257541 len=910664 2024-09-13T00:05:53.898 INFO:tasks.workunit.client.0.smithi136.stdout:op 1400 completed, throughput=4.99MB/sec 2024-09-13T00:05:53.898 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2Jb5vZeTrBDy8E7 off=9935724 len=126611 2024-09-13T00:05:53.899 INFO:tasks.workunit.client.0.smithi136.stdout:op 1401 completed, throughput=4.99MB/sec 2024-09-13T00:05:53.899 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-G40o5VRHdGXx4V0 off=919009 len=347841 2024-09-13T00:05:53.905 INFO:tasks.workunit.client.0.smithi136.stdout:op 1402 completed, throughput=4.99MB/sec 2024-09-13T00:05:53.905 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-vJPwHbCCLOUiM0i off=3426258 len=1355544 2024-09-13T00:05:53.908 INFO:tasks.workunit.client.0.smithi136.stdout:op 1403 completed, throughput=4.99MB/sec 2024-09-13T00:05:53.908 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-US6DHjEGgb_0iWQ off=13017948 len=2047094 2024-09-13T00:05:53.922 INFO:tasks.workunit.client.0.smithi136.stdout:op 1404 completed, throughput=5MB/sec 2024-09-13T00:05:53.923 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-nxbhowT96yloAvI off=7981980 len=1356730 2024-09-13T00:05:53.932 INFO:tasks.workunit.client.0.smithi136.stdout:op 1405 completed, throughput=5MB/sec 2024-09-13T00:05:54.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:05:53 smithi136 bash[75164]: audit 2024-09-13T00:05:53.778786+0000 mon.a (mon.0) 2148 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:05:54.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:05:53 smithi136 bash[76371]: audit 2024-09-13T00:05:53.778786+0000 mon.a (mon.0) 2148 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:05:54.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:05:53 smithi181 bash[67587]: audit 2024-09-13T00:05:53.778786+0000 mon.a (mon.0) 2148 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:05:54.933 INFO:tasks.workunit.client.0.smithi136.stdout: 263: throughput=4.99MB/sec pending data=0 2024-09-13T00:05:54.933 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-s0DWPZCl821W-4Q off=8345727 len=1897963 2024-09-13T00:05:54.937 INFO:tasks.workunit.client.0.smithi136.stdout:op 1406 completed, throughput=4.99MB/sec 2024-09-13T00:05:54.937 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-5zD8W8cYI5idJe7 off=20451063 len=1164187 2024-09-13T00:05:54.940 INFO:tasks.workunit.client.0.smithi136.stdout:op 1407 completed, throughput=5MB/sec 2024-09-13T00:05:54.940 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-4qK0HNfY4DYFdVC off=8626965 len=951401 2024-09-13T00:05:54.942 INFO:tasks.workunit.client.0.smithi136.stdout:op 1408 completed, throughput=5MB/sec 2024-09-13T00:05:54.942 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-rUimuS95r67nT4x off=4110585 len=791281 2024-09-13T00:05:54.949 INFO:tasks.workunit.client.0.smithi136.stdout:op 1409 completed, throughput=5MB/sec 2024-09-13T00:05:55.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:05:54 smithi136 bash[75164]: cluster 2024-09-13T00:05:53.283523+0000 mgr.y (mgr.24661) 863 : cluster [DBG] pgmap v1546: 105 pgs: 105 active+clean; 126 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.2 MiB/s wr, 5 op/s 2024-09-13T00:05:55.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:05:54 smithi136 bash[76371]: cluster 2024-09-13T00:05:53.283523+0000 mgr.y (mgr.24661) 863 : cluster [DBG] pgmap v1546: 105 pgs: 105 active+clean; 126 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.2 MiB/s wr, 5 op/s 2024-09-13T00:05:55.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:05:54 smithi181 bash[67587]: cluster 2024-09-13T00:05:53.283523+0000 mgr.y (mgr.24661) 863 : cluster [DBG] pgmap v1546: 105 pgs: 105 active+clean; 126 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.2 MiB/s wr, 5 op/s 2024-09-13T00:05:55.949 INFO:tasks.workunit.client.0.smithi136.stdout: 264: throughput=4.98MB/sec pending data=0 2024-09-13T00:05:55.949 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-iDhWs6_eveYNVck off=4464576 len=1205298 2024-09-13T00:05:55.959 INFO:tasks.workunit.client.0.smithi136.stdout:op 1410 completed, throughput=4.99MB/sec 2024-09-13T00:05:55.959 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-dQTjS9VkrwXJChB off=871470 len=2049652 2024-09-13T00:05:55.963 INFO:tasks.workunit.client.0.smithi136.stdout:op 1411 completed, throughput=4.99MB/sec 2024-09-13T00:05:55.964 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-CE794b5vMKl9OwW off=9640528 len=954396 2024-09-13T00:05:55.966 INFO:tasks.workunit.client.0.smithi136.stdout:op 1412 completed, throughput=5MB/sec 2024-09-13T00:05:55.966 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-wGMi8rZfe0GMEKv off=1741919 len=257474 2024-09-13T00:05:55.967 INFO:tasks.workunit.client.0.smithi136.stdout:op 1413 completed, throughput=5MB/sec 2024-09-13T00:05:55.967 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-P6q8Uerl1NA63Ec off=1042255 len=1628093 2024-09-13T00:05:55.971 INFO:tasks.workunit.client.0.smithi136.stdout:op 1414 completed, throughput=5MB/sec 2024-09-13T00:05:56.972 INFO:tasks.workunit.client.0.smithi136.stdout: 265: throughput=4.99MB/sec pending data=0 2024-09-13T00:05:56.972 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-oO4yX0rHQIAuBOT off=8839648 len=1261857 2024-09-13T00:05:56.982 INFO:tasks.workunit.client.0.smithi136.stdout:op 1415 completed, throughput=4.99MB/sec 2024-09-13T00:05:56.982 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-6hT0v0Tjxarbzxq off=11597509 len=1238423 2024-09-13T00:05:56.993 INFO:tasks.workunit.client.0.smithi136.stdout:op 1416 completed, throughput=4.99MB/sec 2024-09-13T00:05:56.993 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-AHt3d86kXFKrgVH off=8845471 len=980209 2024-09-13T00:05:56.996 INFO:tasks.workunit.client.0.smithi136.stdout:op 1417 completed, throughput=5MB/sec 2024-09-13T00:05:56.996 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-TeuHo2b6A1NsLJ4 off=1246096 len=496545 2024-09-13T00:05:56.998 INFO:tasks.workunit.client.0.smithi136.stdout:op 1418 completed, throughput=5MB/sec 2024-09-13T00:05:56.998 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-n4hnQrbPatO6AM4 off=12613933 len=829013 2024-09-13T00:05:57.006 INFO:tasks.workunit.client.0.smithi136.stdout:op 1419 completed, throughput=5MB/sec 2024-09-13T00:05:57.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:05:56 smithi136 bash[75164]: cluster 2024-09-13T00:05:55.284369+0000 mgr.y (mgr.24661) 864 : cluster [DBG] pgmap v1547: 105 pgs: 105 active+clean; 126 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:05:57.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:05:56 smithi136 bash[76371]: cluster 2024-09-13T00:05:55.284369+0000 mgr.y (mgr.24661) 864 : cluster [DBG] pgmap v1547: 105 pgs: 105 active+clean; 126 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:05:57.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:05:56 smithi181 bash[67587]: cluster 2024-09-13T00:05:55.284369+0000 mgr.y (mgr.24661) 864 : cluster [DBG] pgmap v1547: 105 pgs: 105 active+clean; 126 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:05:58.006 INFO:tasks.workunit.client.0.smithi136.stdout: 266: throughput=4.98MB/sec pending data=0 2024-09-13T00:05:58.007 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-nyNCqm7o5q2PZeI off=21489000 len=1593569 2024-09-13T00:05:58.018 INFO:tasks.workunit.client.0.smithi136.stdout:op 1420 completed, throughput=4.99MB/sec 2024-09-13T00:05:58.018 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-S9mJmnitye-67r6 off=7407242 len=1062747 2024-09-13T00:05:58.027 INFO:tasks.workunit.client.0.smithi136.stdout:op 1421 completed, throughput=4.99MB/sec 2024-09-13T00:05:58.027 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-_FPLLwuuCOgq2m7 off=2547277 len=320121 2024-09-13T00:05:58.029 INFO:tasks.workunit.client.0.smithi136.stdout:op 1422 completed, throughput=4.99MB/sec 2024-09-13T00:05:58.029 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-n82mv00FKBXVTXi off=5822376 len=118882 2024-09-13T00:05:58.030 INFO:tasks.workunit.client.0.smithi136.stdout:op 1423 completed, throughput=4.99MB/sec 2024-09-13T00:05:58.030 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-OrYxPA3mLMUZwQF off=5659611 len=688722 2024-09-13T00:05:58.038 INFO:tasks.workunit.client.0.smithi136.stdout:op 1424 completed, throughput=5MB/sec 2024-09-13T00:05:58.038 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-UHqyy9sRa9fWFVi off=5745844 len=211558 2024-09-13T00:05:58.042 INFO:tasks.workunit.client.0.smithi136.stdout:op 1425 completed, throughput=5MB/sec 2024-09-13T00:05:58.043 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-gGvdUJ1uEXdpcCC off=15603455 len=14979 2024-09-13T00:05:58.047 INFO:tasks.workunit.client.0.smithi136.stdout:op 1426 completed, throughput=5MB/sec 2024-09-13T00:05:58.047 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-4tytiE-CiCocAFp off=3968773 len=1238352 2024-09-13T00:05:58.056 INFO:tasks.workunit.client.0.smithi136.stdout:op 1427 completed, throughput=5MB/sec 2024-09-13T00:05:59.056 INFO:tasks.workunit.client.0.smithi136.stdout: 267: throughput=4.98MB/sec pending data=0 2024-09-13T00:05:59.056 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-cuN4jb_Q0Fwx9qc off=4147635 len=880522 2024-09-13T00:05:59.063 INFO:tasks.workunit.client.0.smithi136.stdout:op 1428 completed, throughput=4.99MB/sec 2024-09-13T00:05:59.063 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-4tytiE-CiCocAFp off=14913951 len=957918 2024-09-13T00:05:59.067 INFO:tasks.workunit.client.0.smithi136.stdout:op 1429 completed, throughput=4.99MB/sec 2024-09-13T00:05:59.067 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-8ViXIzZDRX7u4Og off=2779452 len=1171987 2024-09-13T00:05:59.070 INFO:tasks.workunit.client.0.smithi136.stdout:op 1430 completed, throughput=4.99MB/sec 2024-09-13T00:05:59.070 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-GXlkHTimMF-IsqB off=3279314 len=2093756 2024-09-13T00:05:59.087 INFO:tasks.workunit.client.0.smithi136.stdout:op 1431 completed, throughput=5MB/sec 2024-09-13T00:05:59.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:05:58 smithi136 bash[75164]: cluster 2024-09-13T00:05:57.285448+0000 mgr.y (mgr.24661) 865 : cluster [DBG] pgmap v1548: 105 pgs: 105 active+clean; 126 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:05:59.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:05:58 smithi136 bash[76371]: cluster 2024-09-13T00:05:57.285448+0000 mgr.y (mgr.24661) 865 : cluster [DBG] pgmap v1548: 105 pgs: 105 active+clean; 126 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:05:59.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:05:58 smithi181 bash[67587]: cluster 2024-09-13T00:05:57.285448+0000 mgr.y (mgr.24661) 865 : cluster [DBG] pgmap v1548: 105 pgs: 105 active+clean; 126 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:05:59.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:05:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:05:59] "GET /metrics HTTP/1.1" 200 34123 "" "Prometheus/2.43.0" 2024-09-13T00:06:00.087 INFO:tasks.workunit.client.0.smithi136.stdout: 268: throughput=4.98MB/sec pending data=0 2024-09-13T00:06:00.088 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-99Q-oWjqX0RWGiB off=11872550 len=1436430 2024-09-13T00:06:00.099 INFO:tasks.workunit.client.0.smithi136.stdout:op 1432 completed, throughput=4.99MB/sec 2024-09-13T00:06:00.099 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-MsuHurMplamOdk5 off=419822 len=1551901 2024-09-13T00:06:00.110 INFO:tasks.workunit.client.0.smithi136.stdout:op 1433 completed, throughput=4.99MB/sec 2024-09-13T00:06:00.110 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-nDUeadIwRUUAOlp off=4463869 len=1089646 2024-09-13T00:06:00.113 INFO:tasks.workunit.client.0.smithi136.stdout:op 1434 completed, throughput=5MB/sec 2024-09-13T00:06:00.113 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-fttc9phMlM_nKBM off=17782209 len=277900 2024-09-13T00:06:00.114 INFO:tasks.workunit.client.0.smithi136.stdout:op 1435 completed, throughput=5MB/sec 2024-09-13T00:06:00.114 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-CtWHZVv0tl8-vAA off=8564005 len=1582454 2024-09-13T00:06:00.125 INFO:tasks.workunit.client.0.smithi136.stdout:op 1436 completed, throughput=5MB/sec 2024-09-13T00:06:01.125 INFO:tasks.workunit.client.0.smithi136.stdout: 269: throughput=4.98MB/sec pending data=0 2024-09-13T00:06:01.125 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-OpOrLwbEyFcLWUu off=1474718 len=1804942 2024-09-13T00:06:01.139 INFO:tasks.workunit.client.0.smithi136.stdout:op 1437 completed, throughput=4.99MB/sec 2024-09-13T00:06:01.139 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-LrdRvAzMYcL4DMn off=20105367 len=1031802 2024-09-13T00:06:01.148 INFO:tasks.workunit.client.0.smithi136.stdout:op 1438 completed, throughput=4.99MB/sec 2024-09-13T00:06:01.148 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-NPyhqcHZBOU3E4w off=2362317 len=2001236 2024-09-13T00:06:01.152 INFO:tasks.workunit.client.0.smithi136.stdout:op 1439 completed, throughput=5MB/sec 2024-09-13T00:06:01.152 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-p6JeW8bz7LcAy32 off=5730889 len=1353992 2024-09-13T00:06:01.163 INFO:tasks.workunit.client.0.smithi136.stdout:op 1440 completed, throughput=5MB/sec 2024-09-13T00:06:01.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:06:00 smithi136 bash[75164]: cluster 2024-09-13T00:05:59.286371+0000 mgr.y (mgr.24661) 866 : cluster [DBG] pgmap v1549: 105 pgs: 105 active+clean; 126 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 2.5 MiB/s rd, 1.8 MiB/s wr, 4 op/s 2024-09-13T00:06:01.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:06:00 smithi136 bash[76371]: cluster 2024-09-13T00:05:59.286371+0000 mgr.y (mgr.24661) 866 : cluster [DBG] pgmap v1549: 105 pgs: 105 active+clean; 126 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 2.5 MiB/s rd, 1.8 MiB/s wr, 4 op/s 2024-09-13T00:06:01.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:06:00 smithi181 bash[67587]: cluster 2024-09-13T00:05:59.286371+0000 mgr.y (mgr.24661) 866 : cluster [DBG] pgmap v1549: 105 pgs: 105 active+clean; 126 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 2.5 MiB/s rd, 1.8 MiB/s wr, 4 op/s 2024-09-13T00:06:02.163 INFO:tasks.workunit.client.0.smithi136.stdout: 270: throughput=4.99MB/sec pending data=0 2024-09-13T00:06:02.163 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-OwhYSs9m1W4aRXi off=11436854 len=905491 2024-09-13T00:06:02.171 INFO:tasks.workunit.client.0.smithi136.stdout:op 1441 completed, throughput=4.99MB/sec 2024-09-13T00:06:02.171 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-YHpH6W_tjIIzXTF off=21041222 len=66395 2024-09-13T00:06:02.172 INFO:tasks.workunit.client.0.smithi136.stdout:op 1442 completed, throughput=4.99MB/sec 2024-09-13T00:06:02.173 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-xkK2Rmj-XEIaWMz off=1918710 len=1668841 2024-09-13T00:06:02.178 INFO:tasks.workunit.client.0.smithi136.stdout:op 1443 completed, throughput=5MB/sec 2024-09-13T00:06:02.178 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-GWJdWnWKB47a4y4 off=14391465 len=270373 2024-09-13T00:06:02.183 INFO:tasks.workunit.client.0.smithi136.stdout:op 1444 completed, throughput=5MB/sec 2024-09-13T00:06:02.183 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Hqh3K7iaFzZy_5I off=519394 len=60062 2024-09-13T00:06:02.187 INFO:tasks.workunit.client.0.smithi136.stdout:op 1445 completed, throughput=5MB/sec 2024-09-13T00:06:02.187 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Ii6DwtE8n7jehJk off=13187598 len=1319002 2024-09-13T00:06:02.197 INFO:tasks.workunit.client.0.smithi136.stdout:op 1446 completed, throughput=5MB/sec 2024-09-13T00:06:02.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:06:02 smithi181 bash[95505]: ts=2024-09-13T00:06:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:06:03.198 INFO:tasks.workunit.client.0.smithi136.stdout: 271: throughput=4.98MB/sec pending data=0 2024-09-13T00:06:03.198 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-WP12STUW5XP8aor off=8008314 len=1696596 2024-09-13T00:06:03.209 INFO:tasks.workunit.client.0.smithi136.stdout:op 1447 completed, throughput=4.99MB/sec 2024-09-13T00:06:03.209 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-D-Oq9mH2Az-m9i7 off=15423858 len=95333 2024-09-13T00:06:03.214 INFO:tasks.workunit.client.0.smithi136.stdout:op 1448 completed, throughput=4.99MB/sec 2024-09-13T00:06:03.215 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-sWFggHgjf5Lzl0Z off=5154379 len=1515418 2024-09-13T00:06:03.226 INFO:tasks.workunit.client.0.smithi136.stdout:op 1449 completed, throughput=4.99MB/sec 2024-09-13T00:06:03.226 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-xGqMVntHF5dRru6 off=8011615 len=366928 2024-09-13T00:06:03.228 INFO:tasks.workunit.client.0.smithi136.stdout:op 1450 completed, throughput=4.99MB/sec 2024-09-13T00:06:03.228 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-b7PkGhhEOtFDwNJ off=6374264 len=2066297 2024-09-13T00:06:03.233 INFO:tasks.workunit.client.0.smithi136.stdout:op 1451 completed, throughput=5MB/sec 2024-09-13T00:06:03.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:06:02 smithi136 bash[75164]: cluster 2024-09-13T00:06:01.287333+0000 mgr.y (mgr.24661) 867 : cluster [DBG] pgmap v1550: 105 pgs: 105 active+clean; 126 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.1 MiB/s wr, 5 op/s 2024-09-13T00:06:03.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:06:02 smithi136 bash[76371]: cluster 2024-09-13T00:06:01.287333+0000 mgr.y (mgr.24661) 867 : cluster [DBG] pgmap v1550: 105 pgs: 105 active+clean; 126 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.1 MiB/s wr, 5 op/s 2024-09-13T00:06:03.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:06:02 smithi181 bash[67587]: cluster 2024-09-13T00:06:01.287333+0000 mgr.y (mgr.24661) 867 : cluster [DBG] pgmap v1550: 105 pgs: 105 active+clean; 126 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.1 MiB/s wr, 5 op/s 2024-09-13T00:06:04.233 INFO:tasks.workunit.client.0.smithi136.stdout: 272: throughput=4.98MB/sec pending data=0 2024-09-13T00:06:04.233 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-8KREKTzxyl4U4X6 off=13986793 len=1527643 2024-09-13T00:06:04.245 INFO:tasks.workunit.client.0.smithi136.stdout:op 1452 completed, throughput=4.99MB/sec 2024-09-13T00:06:04.246 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-asEGkXsStFEM-qk off=11008795 len=363314 2024-09-13T00:06:04.251 INFO:tasks.workunit.client.0.smithi136.stdout:op 1453 completed, throughput=4.99MB/sec 2024-09-13T00:06:04.252 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Q5zB89-CTnwu4Gj off=4053066 len=739687 2024-09-13T00:06:04.254 INFO:tasks.workunit.client.0.smithi136.stdout:op 1454 completed, throughput=4.99MB/sec 2024-09-13T00:06:04.254 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj--pinnbQMkGjJIxt off=4189435 len=1496765 2024-09-13T00:06:04.264 INFO:tasks.workunit.client.0.smithi136.stdout:op 1455 completed, throughput=5MB/sec 2024-09-13T00:06:04.264 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-IKXnFn8zYd6BAKu off=3217147 len=1557607 2024-09-13T00:06:04.270 INFO:tasks.workunit.client.0.smithi136.stdout:op 1456 completed, throughput=5MB/sec 2024-09-13T00:06:05.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:06:05 smithi136 bash[75164]: cluster 2024-09-13T00:06:03.288291+0000 mgr.y (mgr.24661) 868 : cluster [DBG] pgmap v1551: 105 pgs: 105 active+clean; 126 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:06:05.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:06:05 smithi136 bash[76371]: cluster 2024-09-13T00:06:03.288291+0000 mgr.y (mgr.24661) 868 : cluster [DBG] pgmap v1551: 105 pgs: 105 active+clean; 126 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:06:05.270 INFO:tasks.workunit.client.0.smithi136.stdout: 273: throughput=4.98MB/sec pending data=0 2024-09-13T00:06:05.270 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-S6lvp6-wJsg4mbo off=139838 len=197305 2024-09-13T00:06:05.275 INFO:tasks.workunit.client.0.smithi136.stdout:op 1457 completed, throughput=4.99MB/sec 2024-09-13T00:06:05.276 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-pg6RBA-AiScuOzV off=16665184 len=2002433 2024-09-13T00:06:05.289 INFO:tasks.workunit.client.0.smithi136.stdout:op 1458 completed, throughput=4.99MB/sec 2024-09-13T00:06:05.289 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-WjBM8cQ9ER0kscW off=5872032 len=908466 2024-09-13T00:06:05.292 INFO:tasks.workunit.client.0.smithi136.stdout:op 1459 completed, throughput=4.99MB/sec 2024-09-13T00:06:05.292 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-5DdRcVs9SyPbWwF off=2536571 len=1565586 2024-09-13T00:06:05.302 INFO:tasks.workunit.client.0.smithi136.stdout:op 1460 completed, throughput=5MB/sec 2024-09-13T00:06:05.303 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-N1ghkhw-z119b_Z off=20690644 len=1354114 2024-09-13T00:06:05.306 INFO:tasks.workunit.client.0.smithi136.stdout:op 1461 completed, throughput=5MB/sec 2024-09-13T00:06:05.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:06:04 smithi181 bash[67587]: cluster 2024-09-13T00:06:03.288291+0000 mgr.y (mgr.24661) 868 : cluster [DBG] pgmap v1551: 105 pgs: 105 active+clean; 126 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:06:06.307 INFO:tasks.workunit.client.0.smithi136.stdout: 274: throughput=4.99MB/sec pending data=0 2024-09-13T00:06:06.307 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ksTf8TKZqPvKuL8 off=5091402 len=255227 2024-09-13T00:06:06.312 INFO:tasks.workunit.client.0.smithi136.stdout:op 1462 completed, throughput=4.99MB/sec 2024-09-13T00:06:06.312 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-B5MykD0FJwLxs1t off=4260185 len=147662 2024-09-13T00:06:06.313 INFO:tasks.workunit.client.0.smithi136.stdout:op 1463 completed, throughput=4.99MB/sec 2024-09-13T00:06:06.313 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-nEebHdRadbalqHg off=2223523 len=1103083 2024-09-13T00:06:06.316 INFO:tasks.workunit.client.0.smithi136.stdout:op 1464 completed, throughput=4.99MB/sec 2024-09-13T00:06:06.316 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-D_Hp3ytuam8Wjqm off=17255 len=670087 2024-09-13T00:06:06.319 INFO:tasks.workunit.client.0.smithi136.stdout:op 1465 completed, throughput=4.99MB/sec 2024-09-13T00:06:06.319 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-MJr0j1i1l4sWlKd off=7602910 len=1455861 2024-09-13T00:06:06.329 INFO:tasks.workunit.client.0.smithi136.stdout:op 1466 completed, throughput=5MB/sec 2024-09-13T00:06:06.329 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-iJqHuProgoYFz7U off=23178515 len=2007331 2024-09-13T00:06:06.343 INFO:tasks.workunit.client.0.smithi136.stdout:op 1467 completed, throughput=5.01MB/sec 2024-09-13T00:06:07.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:06:07 smithi136 bash[75164]: cluster 2024-09-13T00:06:05.289246+0000 mgr.y (mgr.24661) 869 : cluster [DBG] pgmap v1552: 105 pgs: 105 active+clean; 126 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:06:07.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:06:07 smithi136 bash[76371]: cluster 2024-09-13T00:06:05.289246+0000 mgr.y (mgr.24661) 869 : cluster [DBG] pgmap v1552: 105 pgs: 105 active+clean; 126 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:06:07.343 INFO:tasks.workunit.client.0.smithi136.stdout: 275: throughput=4.99MB/sec pending data=0 2024-09-13T00:06:07.343 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Et4KTQ14MHXxCpK off=1079518 len=1447586 2024-09-13T00:06:07.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:06:07 smithi181 bash[67587]: cluster 2024-09-13T00:06:05.289246+0000 mgr.y (mgr.24661) 869 : cluster [DBG] pgmap v1552: 105 pgs: 105 active+clean; 126 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:06:07.353 INFO:tasks.workunit.client.0.smithi136.stdout:op 1468 completed, throughput=4.99MB/sec 2024-09-13T00:06:07.354 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-pG9KylrK4OUaQQO off=7961701 len=698116 2024-09-13T00:06:07.361 INFO:tasks.workunit.client.0.smithi136.stdout:op 1469 completed, throughput=4.99MB/sec 2024-09-13T00:06:07.361 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-samu0pi7X8mYbP_ off=856033 len=1828535 2024-09-13T00:06:07.365 INFO:tasks.workunit.client.0.smithi136.stdout:op 1470 completed, throughput=5MB/sec 2024-09-13T00:06:08.365 INFO:tasks.workunit.client.0.smithi136.stdout: 276: throughput=4.98MB/sec pending data=0 2024-09-13T00:06:08.365 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-8YMaFNtl6v8kMj9 off=103181 len=1203268 2024-09-13T00:06:08.368 INFO:tasks.workunit.client.0.smithi136.stdout:op 1471 completed, throughput=4.99MB/sec 2024-09-13T00:06:08.368 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-b0MZ4n9nrTqpsrA off=17307958 len=2083043 2024-09-13T00:06:08.374 INFO:tasks.workunit.client.0.smithi136.stdout:op 1472 completed, throughput=4.99MB/sec 2024-09-13T00:06:08.374 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-tPAtDXRFEnnuPPO off=5242864 len=1752483 2024-09-13T00:06:08.380 INFO:tasks.workunit.client.0.smithi136.stdout:op 1473 completed, throughput=5MB/sec 2024-09-13T00:06:08.380 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-drmvQPXH-YCoufc off=10444500 len=49778 2024-09-13T00:06:08.384 INFO:tasks.workunit.client.0.smithi136.stdout:op 1474 completed, throughput=5MB/sec 2024-09-13T00:06:08.384 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-vjJLNrXl7vNtuoq off=4765289 len=1701108 2024-09-13T00:06:08.396 INFO:tasks.workunit.client.0.smithi136.stdout:op 1475 completed, throughput=5.01MB/sec 2024-09-13T00:06:09.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:06:09 smithi136 bash[75164]: cluster 2024-09-13T00:06:07.290382+0000 mgr.y (mgr.24661) 870 : cluster [DBG] pgmap v1553: 105 pgs: 105 active+clean; 126 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 2.3 MiB/s rd, 3.6 MiB/s wr, 5 op/s 2024-09-13T00:06:09.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:06:09 smithi136 bash[75164]: audit 2024-09-13T00:06:08.779267+0000 mon.a (mon.0) 2149 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:06:09.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:06:09 smithi136 bash[76371]: cluster 2024-09-13T00:06:07.290382+0000 mgr.y (mgr.24661) 870 : cluster [DBG] pgmap v1553: 105 pgs: 105 active+clean; 126 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 2.3 MiB/s rd, 3.6 MiB/s wr, 5 op/s 2024-09-13T00:06:09.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:06:09 smithi136 bash[76371]: audit 2024-09-13T00:06:08.779267+0000 mon.a (mon.0) 2149 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:06:09.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:06:09 smithi181 bash[67587]: cluster 2024-09-13T00:06:07.290382+0000 mgr.y (mgr.24661) 870 : cluster [DBG] pgmap v1553: 105 pgs: 105 active+clean; 126 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 2.3 MiB/s rd, 3.6 MiB/s wr, 5 op/s 2024-09-13T00:06:09.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:06:09 smithi181 bash[67587]: audit 2024-09-13T00:06:08.779267+0000 mon.a (mon.0) 2149 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:06:09.396 INFO:tasks.workunit.client.0.smithi136.stdout: 277: throughput=4.99MB/sec pending data=0 2024-09-13T00:06:09.396 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-kDGrLV0v-uHFOvM off=269424 len=792378 2024-09-13T00:06:09.404 INFO:tasks.workunit.client.0.smithi136.stdout:op 1476 completed, throughput=4.99MB/sec 2024-09-13T00:06:09.404 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-nW4wVaLLtlmPOV0 off=1166233 len=1014470 2024-09-13T00:06:09.413 INFO:tasks.workunit.client.0.smithi136.stdout:op 1477 completed, throughput=4.99MB/sec 2024-09-13T00:06:09.413 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-VXToBgctMlnY5Od off=1351863 len=657620 2024-09-13T00:06:09.415 INFO:tasks.workunit.client.0.smithi136.stdout:op 1478 completed, throughput=5MB/sec 2024-09-13T00:06:09.415 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Dn-fKcCrxcAzP9z off=1305506 len=1165714 2024-09-13T00:06:09.418 INFO:tasks.workunit.client.0.smithi136.stdout:op 1479 completed, throughput=5MB/sec 2024-09-13T00:06:09.419 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-rvePLAqWeQxguub off=5479581 len=1090570 2024-09-13T00:06:09.428 INFO:tasks.workunit.client.0.smithi136.stdout:op 1480 completed, throughput=5MB/sec 2024-09-13T00:06:09.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:06:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:06:09] "GET /metrics HTTP/1.1" 200 34121 "" "Prometheus/2.43.0" 2024-09-13T00:06:10.428 INFO:tasks.workunit.client.0.smithi136.stdout: 278: throughput=4.99MB/sec pending data=0 2024-09-13T00:06:10.428 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-9lOTuEzqHaw_ipO off=7759115 len=595090 2024-09-13T00:06:10.431 INFO:tasks.workunit.client.0.smithi136.stdout:op 1481 completed, throughput=4.99MB/sec 2024-09-13T00:06:10.431 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-DJU4JRnOKl-JFqp off=6093960 len=40875 2024-09-13T00:06:10.435 INFO:tasks.workunit.client.0.smithi136.stdout:op 1482 completed, throughput=4.99MB/sec 2024-09-13T00:06:10.435 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Y6enpbJTLF5avAp off=2426920 len=369632 2024-09-13T00:06:10.438 INFO:tasks.workunit.client.0.smithi136.stdout:op 1483 completed, throughput=4.99MB/sec 2024-09-13T00:06:10.438 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-zjk6eCcH00jqkTf off=1299036 len=611994 2024-09-13T00:06:10.440 INFO:tasks.workunit.client.0.smithi136.stdout:op 1484 completed, throughput=4.99MB/sec 2024-09-13T00:06:10.440 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-vRag2RK_ctloyEJ off=140825 len=1347386 2024-09-13T00:06:10.443 INFO:tasks.workunit.client.0.smithi136.stdout:op 1485 completed, throughput=4.99MB/sec 2024-09-13T00:06:10.443 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-qYUk0zY2k0Rg9bF off=7110005 len=1466776 2024-09-13T00:06:10.455 INFO:tasks.workunit.client.0.smithi136.stdout:op 1486 completed, throughput=5MB/sec 2024-09-13T00:06:10.455 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-mSYtlP3ZyFKeJkA off=1430181 len=1555481 2024-09-13T00:06:10.467 INFO:tasks.workunit.client.0.smithi136.stdout:op 1487 completed, throughput=5MB/sec 2024-09-13T00:06:11.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:06:11 smithi181 bash[67587]: cluster 2024-09-13T00:06:09.291263+0000 mgr.y (mgr.24661) 871 : cluster [DBG] pgmap v1554: 105 pgs: 105 active+clean; 126 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 1.6 MiB/s rd, 3.2 MiB/s wr, 4 op/s 2024-09-13T00:06:11.468 INFO:tasks.workunit.client.0.smithi136.stdout: 279: throughput=4.99MB/sec pending data=0 2024-09-13T00:06:11.468 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-S1GNNhwtVhyuMIY off=343429 len=251588 2024-09-13T00:06:11.469 INFO:tasks.workunit.client.0.smithi136.stdout:op 1488 completed, throughput=4.99MB/sec 2024-09-13T00:06:11.469 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-wHM5PeVelnS2Lwa off=10412547 len=759875 2024-09-13T00:06:11.477 INFO:tasks.workunit.client.0.smithi136.stdout:op 1489 completed, throughput=4.99MB/sec 2024-09-13T00:06:11.477 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-QCS9rc8b-E0LvVz off=1031060 len=61549 2024-09-13T00:06:11.478 INFO:tasks.workunit.client.0.smithi136.stdout:op 1490 completed, throughput=4.99MB/sec 2024-09-13T00:06:11.478 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-CM9QbXZv9CYvJF4 off=7618052 len=1761102 2024-09-13T00:06:11.482 INFO:tasks.workunit.client.0.smithi136.stdout:op 1491 completed, throughput=5MB/sec 2024-09-13T00:06:11.482 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-TY7zU2nN51oOIAt off=18114507 len=1125754 2024-09-13T00:06:11.491 INFO:tasks.workunit.client.0.smithi136.stdout:op 1492 completed, throughput=5MB/sec 2024-09-13T00:06:11.492 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-srW7hbdRLQ4HqCe off=5545546 len=1558826 2024-09-13T00:06:11.495 INFO:tasks.workunit.client.0.smithi136.stdout:op 1493 completed, throughput=5MB/sec 2024-09-13T00:06:11.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:06:11 smithi136 bash[75164]: cluster 2024-09-13T00:06:09.291263+0000 mgr.y (mgr.24661) 871 : cluster [DBG] pgmap v1554: 105 pgs: 105 active+clean; 126 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 1.6 MiB/s rd, 3.2 MiB/s wr, 4 op/s 2024-09-13T00:06:11.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:06:11 smithi136 bash[76371]: cluster 2024-09-13T00:06:09.291263+0000 mgr.y (mgr.24661) 871 : cluster [DBG] pgmap v1554: 105 pgs: 105 active+clean; 126 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 1.6 MiB/s rd, 3.2 MiB/s wr, 4 op/s 2024-09-13T00:06:12.496 INFO:tasks.workunit.client.0.smithi136.stdout: 280: throughput=4.99MB/sec pending data=0 2024-09-13T00:06:12.496 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-9XJyAsD9cvjBEq5 off=2262556 len=1927792 2024-09-13T00:06:12.500 INFO:tasks.workunit.client.0.smithi136.stdout:op 1494 completed, throughput=4.99MB/sec 2024-09-13T00:06:12.500 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-VHSfT8lziofmYmt off=12442524 len=2005674 2024-09-13T00:06:12.505 INFO:tasks.workunit.client.0.smithi136.stdout:op 1495 completed, throughput=5MB/sec 2024-09-13T00:06:12.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:06:12 smithi181 bash[95505]: ts=2024-09-13T00:06:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:06:13.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:06:13 smithi181 bash[67587]: cluster 2024-09-13T00:06:11.292230+0000 mgr.y (mgr.24661) 872 : cluster [DBG] pgmap v1555: 105 pgs: 105 active+clean; 126 GiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 1.6 MiB/s rd, 3.3 MiB/s wr, 5 op/s 2024-09-13T00:06:13.506 INFO:tasks.workunit.client.0.smithi136.stdout: 281: throughput=4.98MB/sec pending data=0 2024-09-13T00:06:13.506 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ueg900GShJSQMmi off=218576 len=1917756 2024-09-13T00:06:13.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:06:13 smithi136 bash[75164]: cluster 2024-09-13T00:06:11.292230+0000 mgr.y (mgr.24661) 872 : cluster [DBG] pgmap v1555: 105 pgs: 105 active+clean; 126 GiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 1.6 MiB/s rd, 3.3 MiB/s wr, 5 op/s 2024-09-13T00:06:13.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:06:13 smithi136 bash[76371]: cluster 2024-09-13T00:06:11.292230+0000 mgr.y (mgr.24661) 872 : cluster [DBG] pgmap v1555: 105 pgs: 105 active+clean; 126 GiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 1.6 MiB/s rd, 3.3 MiB/s wr, 5 op/s 2024-09-13T00:06:13.519 INFO:tasks.workunit.client.0.smithi136.stdout:op 1496 completed, throughput=4.99MB/sec 2024-09-13T00:06:13.519 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-LJV3VWCQNCI74-u off=14676980 len=1925217 2024-09-13T00:06:13.523 INFO:tasks.workunit.client.0.smithi136.stdout:op 1497 completed, throughput=5MB/sec 2024-09-13T00:06:13.523 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-qkK3HCPmrQwc7QG off=3070584 len=807462 2024-09-13T00:06:13.531 INFO:tasks.workunit.client.0.smithi136.stdout:op 1498 completed, throughput=5MB/sec 2024-09-13T00:06:13.531 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-or7XYpc686hSXmg off=10080999 len=1656276 2024-09-13T00:06:13.542 INFO:tasks.workunit.client.0.smithi136.stdout:op 1499 completed, throughput=5MB/sec 2024-09-13T00:06:14.543 INFO:tasks.workunit.client.0.smithi136.stdout: 282: throughput=4.99MB/sec pending data=0 2024-09-13T00:06:14.543 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-NUiArYPXTa5Zs9V off=1155191 len=994400 2024-09-13T00:06:14.546 INFO:tasks.workunit.client.0.smithi136.stdout:op 1500 completed, throughput=4.99MB/sec 2024-09-13T00:06:14.546 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-iqEnWI5O5as1j3i off=2940425 len=1952602 2024-09-13T00:06:14.551 INFO:tasks.workunit.client.0.smithi136.stdout:op 1501 completed, throughput=5MB/sec 2024-09-13T00:06:14.551 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ORtZKScCNOqc-_Q off=1704998 len=1221748 2024-09-13T00:06:14.560 INFO:tasks.workunit.client.0.smithi136.stdout:op 1502 completed, throughput=5MB/sec 2024-09-13T00:06:14.560 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-YinxB5qwrkRhswW off=3880664 len=169420 2024-09-13T00:06:14.562 INFO:tasks.workunit.client.0.smithi136.stdout:op 1503 completed, throughput=5MB/sec 2024-09-13T00:06:14.562 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-h7wcmVKhFFEeC5B off=8174271 len=1211923 2024-09-13T00:06:14.564 INFO:tasks.workunit.client.0.smithi136.stdout:op 1504 completed, throughput=5MB/sec 2024-09-13T00:06:15.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:06:15 smithi181 bash[67587]: cluster 2024-09-13T00:06:13.293169+0000 mgr.y (mgr.24661) 873 : cluster [DBG] pgmap v1556: 105 pgs: 105 active+clean; 126 GiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 2.3 MiB/s rd, 3.0 MiB/s wr, 5 op/s 2024-09-13T00:06:15.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:06:15 smithi136 bash[75164]: cluster 2024-09-13T00:06:13.293169+0000 mgr.y (mgr.24661) 873 : cluster [DBG] pgmap v1556: 105 pgs: 105 active+clean; 126 GiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 2.3 MiB/s rd, 3.0 MiB/s wr, 5 op/s 2024-09-13T00:06:15.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:06:15 smithi136 bash[76371]: cluster 2024-09-13T00:06:13.293169+0000 mgr.y (mgr.24661) 873 : cluster [DBG] pgmap v1556: 105 pgs: 105 active+clean; 126 GiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 2.3 MiB/s rd, 3.0 MiB/s wr, 5 op/s 2024-09-13T00:06:15.564 INFO:tasks.workunit.client.0.smithi136.stdout: 283: throughput=4.99MB/sec pending data=0 2024-09-13T00:06:15.564 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-W2GnRohvu_fAFLF off=3469103 len=1542716 2024-09-13T00:06:15.575 INFO:tasks.workunit.client.0.smithi136.stdout:op 1505 completed, throughput=4.99MB/sec 2024-09-13T00:06:15.575 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-uQgAfRKbl_4pz7x off=13712224 len=433365 2024-09-13T00:06:15.577 INFO:tasks.workunit.client.0.smithi136.stdout:op 1506 completed, throughput=4.99MB/sec 2024-09-13T00:06:15.577 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-V8jCN-ypWSjRhEs off=1275032 len=1271650 2024-09-13T00:06:15.587 INFO:tasks.workunit.client.0.smithi136.stdout:op 1507 completed, throughput=5MB/sec 2024-09-13T00:06:15.587 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Uxcw9G4gXz0NcUF off=4496471 len=283806 2024-09-13T00:06:15.593 INFO:tasks.workunit.client.0.smithi136.stdout:op 1508 completed, throughput=5MB/sec 2024-09-13T00:06:15.593 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-zDhnuLSKLLdRLkD off=7864792 len=311218 2024-09-13T00:06:15.594 INFO:tasks.workunit.client.0.smithi136.stdout:op 1509 completed, throughput=5MB/sec 2024-09-13T00:06:15.594 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-a7y-AM6Mz515dTc off=3754260 len=1780676 2024-09-13T00:06:15.605 INFO:tasks.workunit.client.0.smithi136.stdout:op 1510 completed, throughput=5MB/sec 2024-09-13T00:06:16.606 INFO:tasks.workunit.client.0.smithi136.stdout: 284: throughput=4.99MB/sec pending data=0 2024-09-13T00:06:16.606 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-kUPNwr1w15674n- off=1372716 len=361196 2024-09-13T00:06:16.612 INFO:tasks.workunit.client.0.smithi136.stdout:op 1511 completed, throughput=4.99MB/sec 2024-09-13T00:06:16.612 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-U9EQHcFvzDEucFb off=4689104 len=1573525 2024-09-13T00:06:16.617 INFO:tasks.workunit.client.0.smithi136.stdout:op 1512 completed, throughput=4.99MB/sec 2024-09-13T00:06:16.617 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-QGV7OKZWFkbOguT off=3394638 len=359049 2024-09-13T00:06:16.619 INFO:tasks.workunit.client.0.smithi136.stdout:op 1513 completed, throughput=4.99MB/sec 2024-09-13T00:06:16.620 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-S53JDWQJxsYCJlU off=1622822 len=1355389 2024-09-13T00:06:16.623 INFO:tasks.workunit.client.0.smithi136.stdout:op 1514 completed, throughput=5MB/sec 2024-09-13T00:06:16.623 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-u_u8yAyNYPmXWbB off=2393839 len=277314 2024-09-13T00:06:16.628 INFO:tasks.workunit.client.0.smithi136.stdout:op 1515 completed, throughput=5MB/sec 2024-09-13T00:06:16.628 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-nL_RNJLuCMaTKaa off=575227 len=986921 2024-09-13T00:06:16.631 INFO:tasks.workunit.client.0.smithi136.stdout:op 1516 completed, throughput=5MB/sec 2024-09-13T00:06:17.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:06:17 smithi181 bash[67587]: cluster 2024-09-13T00:06:15.294026+0000 mgr.y (mgr.24661) 874 : cluster [DBG] pgmap v1557: 105 pgs: 105 active+clean; 126 GiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:06:17.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:06:17 smithi136 bash[75164]: cluster 2024-09-13T00:06:15.294026+0000 mgr.y (mgr.24661) 874 : cluster [DBG] pgmap v1557: 105 pgs: 105 active+clean; 126 GiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:06:17.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:06:17 smithi136 bash[76371]: cluster 2024-09-13T00:06:15.294026+0000 mgr.y (mgr.24661) 874 : cluster [DBG] pgmap v1557: 105 pgs: 105 active+clean; 126 GiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:06:17.631 INFO:tasks.workunit.client.0.smithi136.stdout: 285: throughput=4.99MB/sec pending data=0 2024-09-13T00:06:17.631 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-UviiMQ4vgrVAolp off=6356177 len=870765 2024-09-13T00:06:17.640 INFO:tasks.workunit.client.0.smithi136.stdout:op 1517 completed, throughput=4.99MB/sec 2024-09-13T00:06:17.640 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-GVI92sXfBSZ2xgR off=3608742 len=355109 2024-09-13T00:06:17.647 INFO:tasks.workunit.client.0.smithi136.stdout:op 1518 completed, throughput=4.99MB/sec 2024-09-13T00:06:17.647 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Lv4sf6kyZh6xmgI off=15135696 len=89471 2024-09-13T00:06:17.648 INFO:tasks.workunit.client.0.smithi136.stdout:op 1519 completed, throughput=4.99MB/sec 2024-09-13T00:06:17.648 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-41NrddnlJ7rKMDk off=7218385 len=1844173 2024-09-13T00:06:17.655 INFO:tasks.workunit.client.0.smithi136.stdout:op 1520 completed, throughput=5MB/sec 2024-09-13T00:06:17.655 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-BRaUJr8aa7XjV9T off=6226534 len=755269 2024-09-13T00:06:17.657 INFO:tasks.workunit.client.0.smithi136.stdout:op 1521 completed, throughput=5MB/sec 2024-09-13T00:06:17.657 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-SKcWztMtJPeUIGf off=584658 len=1858108 2024-09-13T00:06:17.672 INFO:tasks.workunit.client.0.smithi136.stdout:op 1522 completed, throughput=5MB/sec 2024-09-13T00:06:18.672 INFO:tasks.workunit.client.0.smithi136.stdout: 286: throughput=4.99MB/sec pending data=0 2024-09-13T00:06:18.672 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-I4xMnfLHI4xMHtA off=1171068 len=922671 2024-09-13T00:06:18.676 INFO:tasks.workunit.client.0.smithi136.stdout:op 1523 completed, throughput=4.99MB/sec 2024-09-13T00:06:18.676 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-HeMMGKPbef_JxEe off=4029666 len=1158927 2024-09-13T00:06:18.679 INFO:tasks.workunit.client.0.smithi136.stdout:op 1524 completed, throughput=4.99MB/sec 2024-09-13T00:06:18.679 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-nukFMjoFddBWZ3b off=4240840 len=1419329 2024-09-13T00:06:18.690 INFO:tasks.workunit.client.0.smithi136.stdout:op 1525 completed, throughput=5MB/sec 2024-09-13T00:06:18.690 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-UxlndsrM57S8an5 off=18863730 len=313952 2024-09-13T00:06:18.695 INFO:tasks.workunit.client.0.smithi136.stdout:op 1526 completed, throughput=5MB/sec 2024-09-13T00:06:18.695 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-h-MsHamN9HjTIDu off=925195 len=1903550 2024-09-13T00:06:18.699 INFO:tasks.workunit.client.0.smithi136.stdout:op 1527 completed, throughput=5MB/sec 2024-09-13T00:06:19.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:06:19 smithi181 bash[67587]: cluster 2024-09-13T00:06:17.295108+0000 mgr.y (mgr.24661) 875 : cluster [DBG] pgmap v1558: 105 pgs: 105 active+clean; 126 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:06:19.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:06:19 smithi136 bash[75164]: cluster 2024-09-13T00:06:17.295108+0000 mgr.y (mgr.24661) 875 : cluster [DBG] pgmap v1558: 105 pgs: 105 active+clean; 126 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:06:19.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:06:19 smithi136 bash[76371]: cluster 2024-09-13T00:06:17.295108+0000 mgr.y (mgr.24661) 875 : cluster [DBG] pgmap v1558: 105 pgs: 105 active+clean; 126 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:06:19.700 INFO:tasks.workunit.client.0.smithi136.stdout: 287: throughput=4.99MB/sec pending data=0 2024-09-13T00:06:19.700 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-3Tim9P1Plj2SxQv off=17022677 len=2001934 2024-09-13T00:06:19.714 INFO:tasks.workunit.client.0.smithi136.stdout:op 1528 completed, throughput=4.99MB/sec 2024-09-13T00:06:19.714 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-TOZVEpS7mUOSttj off=2835739 len=1365561 2024-09-13T00:06:19.718 INFO:tasks.workunit.client.0.smithi136.stdout:op 1529 completed, throughput=5MB/sec 2024-09-13T00:06:19.718 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-eByCGo62xz_91_H off=2807048 len=813292 2024-09-13T00:06:19.725 INFO:tasks.workunit.client.0.smithi136.stdout:op 1530 completed, throughput=5MB/sec 2024-09-13T00:06:19.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:06:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:06:19] "GET /metrics HTTP/1.1" 200 34121 "" "Prometheus/2.43.0" 2024-09-13T00:06:20.725 INFO:tasks.workunit.client.0.smithi136.stdout: 288: throughput=4.98MB/sec pending data=0 2024-09-13T00:06:20.726 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-mxVTf50LP4tXwJS off=776518 len=1926497 2024-09-13T00:06:20.743 INFO:tasks.workunit.client.0.smithi136.stdout:op 1531 completed, throughput=4.99MB/sec 2024-09-13T00:06:20.744 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-44rJuIImLTNfGuY off=10966718 len=553791 2024-09-13T00:06:20.746 INFO:tasks.workunit.client.0.smithi136.stdout:op 1532 completed, throughput=4.99MB/sec 2024-09-13T00:06:20.746 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-V6_7rA7DJffBNE_ off=1662267 len=107421 2024-09-13T00:06:20.750 INFO:tasks.workunit.client.0.smithi136.stdout:op 1533 completed, throughput=4.99MB/sec 2024-09-13T00:06:20.750 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-wUY8UvKQFbPF9hA off=1033148 len=1401191 2024-09-13T00:06:20.753 INFO:tasks.workunit.client.0.smithi136.stdout:op 1534 completed, throughput=5MB/sec 2024-09-13T00:06:20.754 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-UMd6sbzD1ptvP2s off=9551557 len=692729 2024-09-13T00:06:20.761 INFO:tasks.workunit.client.0.smithi136.stdout:op 1535 completed, throughput=5MB/sec 2024-09-13T00:06:20.761 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ZiDVLtqMm8Lbcw0 off=6979041 len=15790 2024-09-13T00:06:20.765 INFO:tasks.workunit.client.0.smithi136.stdout:op 1536 completed, throughput=5MB/sec 2024-09-13T00:06:20.765 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj--uht5o2V_fi6YDO off=5245908 len=645578 2024-09-13T00:06:20.770 INFO:tasks.workunit.client.0.smithi136.stdout:op 1537 completed, throughput=5MB/sec 2024-09-13T00:06:21.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:06:21 smithi181 bash[67587]: cluster 2024-09-13T00:06:19.295973+0000 mgr.y (mgr.24661) 876 : cluster [DBG] pgmap v1559: 105 pgs: 105 active+clean; 126 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:06:21.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:06:21 smithi136 bash[75164]: cluster 2024-09-13T00:06:19.295973+0000 mgr.y (mgr.24661) 876 : cluster [DBG] pgmap v1559: 105 pgs: 105 active+clean; 126 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:06:21.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:06:21 smithi136 bash[76371]: cluster 2024-09-13T00:06:19.295973+0000 mgr.y (mgr.24661) 876 : cluster [DBG] pgmap v1559: 105 pgs: 105 active+clean; 126 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:06:21.771 INFO:tasks.workunit.client.0.smithi136.stdout: 289: throughput=4.98MB/sec pending data=0 2024-09-13T00:06:21.771 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-wvkdOBjpF7sDQXY off=3220217 len=210623 2024-09-13T00:06:21.776 INFO:tasks.workunit.client.0.smithi136.stdout:op 1538 completed, throughput=4.98MB/sec 2024-09-13T00:06:21.776 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_V_6qV525LDY6J6 off=17085232 len=1712578 2024-09-13T00:06:21.792 INFO:tasks.workunit.client.0.smithi136.stdout:op 1539 completed, throughput=4.99MB/sec 2024-09-13T00:06:21.792 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-0eqPMDQMCsDnZuv off=9999869 len=102671 2024-09-13T00:06:21.793 INFO:tasks.workunit.client.0.smithi136.stdout:op 1540 completed, throughput=4.99MB/sec 2024-09-13T00:06:21.793 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-iS0eXrwzuluowzv off=18672962 len=387062 2024-09-13T00:06:21.799 INFO:tasks.workunit.client.0.smithi136.stdout:op 1541 completed, throughput=4.99MB/sec 2024-09-13T00:06:21.799 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Xdn5X0Rghtv9xXv off=2272007 len=1775011 2024-09-13T00:06:21.811 INFO:tasks.workunit.client.0.smithi136.stdout:op 1542 completed, throughput=5MB/sec 2024-09-13T00:06:21.811 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-5lqvCfjxGfZzVic off=1705467 len=228903 2024-09-13T00:06:21.812 INFO:tasks.workunit.client.0.smithi136.stdout:op 1543 completed, throughput=5MB/sec 2024-09-13T00:06:21.812 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-w48wk0Z7Q9A_5Nc off=8251439 len=226664 2024-09-13T00:06:21.817 INFO:tasks.workunit.client.0.smithi136.stdout:op 1544 completed, throughput=5MB/sec 2024-09-13T00:06:21.817 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-f5g39Uk4WvKf53i off=20106495 len=1661693 2024-09-13T00:06:21.823 INFO:tasks.workunit.client.0.smithi136.stdout:op 1545 completed, throughput=5MB/sec 2024-09-13T00:06:22.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:06:22 smithi181 bash[95505]: ts=2024-09-13T00:06:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:06:22.823 INFO:tasks.workunit.client.0.smithi136.stdout: 290: throughput=4.99MB/sec pending data=0 2024-09-13T00:06:22.823 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-pSJ2j83gsV7ulnJ off=4010521 len=786847 2024-09-13T00:06:22.831 INFO:tasks.workunit.client.0.smithi136.stdout:op 1546 completed, throughput=4.99MB/sec 2024-09-13T00:06:22.831 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-f5RyCJD9HfC6v7w off=5676944 len=1929880 2024-09-13T00:06:22.835 INFO:tasks.workunit.client.0.smithi136.stdout:op 1547 completed, throughput=4.99MB/sec 2024-09-13T00:06:22.835 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-S5ngWrrbNdXG2b9 off=0 len=1362005 2024-09-13T00:06:22.848 INFO:tasks.workunit.client.0.smithi136.stdout:op 1548 completed, throughput=5MB/sec 2024-09-13T00:06:22.848 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-4JgXutV3Q1rtdhP off=342502 len=948049 2024-09-13T00:06:22.851 INFO:tasks.workunit.client.0.smithi136.stdout:op 1549 completed, throughput=5MB/sec 2024-09-13T00:06:23.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:06:23 smithi181 bash[67587]: cluster 2024-09-13T00:06:21.296956+0000 mgr.y (mgr.24661) 877 : cluster [DBG] pgmap v1560: 105 pgs: 105 active+clean; 126 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 2.8 MiB/s rd, 1.9 MiB/s wr, 4 op/s 2024-09-13T00:06:23.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:06:23 smithi136 bash[75164]: cluster 2024-09-13T00:06:21.296956+0000 mgr.y (mgr.24661) 877 : cluster [DBG] pgmap v1560: 105 pgs: 105 active+clean; 126 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 2.8 MiB/s rd, 1.9 MiB/s wr, 4 op/s 2024-09-13T00:06:23.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:06:23 smithi136 bash[76371]: cluster 2024-09-13T00:06:21.296956+0000 mgr.y (mgr.24661) 877 : cluster [DBG] pgmap v1560: 105 pgs: 105 active+clean; 126 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 2.8 MiB/s rd, 1.9 MiB/s wr, 4 op/s 2024-09-13T00:06:23.851 INFO:tasks.workunit.client.0.smithi136.stdout: 291: throughput=4.99MB/sec pending data=0 2024-09-13T00:06:23.852 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-__XHRn9S_uasj3m off=1869962 len=10005 2024-09-13T00:06:23.852 INFO:tasks.workunit.client.0.smithi136.stdout:op 1550 completed, throughput=4.99MB/sec 2024-09-13T00:06:23.852 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-tx-l8f_SHTk1HUe off=10331182 len=709712 2024-09-13T00:06:23.861 INFO:tasks.workunit.client.0.smithi136.stdout:op 1551 completed, throughput=4.99MB/sec 2024-09-13T00:06:23.861 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-PK0mcBj5j3POWJS off=443538 len=1807820 2024-09-13T00:06:23.874 INFO:tasks.workunit.client.0.smithi136.stdout:op 1552 completed, throughput=4.99MB/sec 2024-09-13T00:06:23.874 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-7eFqvR1V5YAi900 off=1249830 len=1650865 2024-09-13T00:06:23.876 INFO:tasks.workunit.client.0.smithi136.stdout:op 1553 completed, throughput=5MB/sec 2024-09-13T00:06:23.876 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-TZlXHzsjUBP3zzH off=2592092 len=956848 2024-09-13T00:06:23.885 INFO:tasks.workunit.client.0.smithi136.stdout:op 1554 completed, throughput=5MB/sec 2024-09-13T00:06:24.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:06:24 smithi181 bash[67587]: audit 2024-09-13T00:06:23.779523+0000 mon.a (mon.0) 2150 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:06:24.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:06:24 smithi136 bash[75164]: audit 2024-09-13T00:06:23.779523+0000 mon.a (mon.0) 2150 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:06:24.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:06:24 smithi136 bash[76371]: audit 2024-09-13T00:06:23.779523+0000 mon.a (mon.0) 2150 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:06:24.886 INFO:tasks.workunit.client.0.smithi136.stdout: 292: throughput=4.98MB/sec pending data=0 2024-09-13T00:06:24.886 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-0fku96VuoxDSZ4Z off=440680 len=903936 2024-09-13T00:06:24.888 INFO:tasks.workunit.client.0.smithi136.stdout:op 1555 completed, throughput=4.99MB/sec 2024-09-13T00:06:24.888 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-EQtrpKyGSI1-uWH off=2266606 len=1055071 2024-09-13T00:06:24.891 INFO:tasks.workunit.client.0.smithi136.stdout:op 1556 completed, throughput=4.99MB/sec 2024-09-13T00:06:24.891 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-5GzOdO0qFz6pZtO off=0 len=1182385 2024-09-13T00:06:24.894 INFO:tasks.workunit.client.0.smithi136.stdout:op 1557 completed, throughput=4.99MB/sec 2024-09-13T00:06:24.895 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-6vmSxdzzLIuCWLy off=7421168 len=381654 2024-09-13T00:06:24.900 INFO:tasks.workunit.client.0.smithi136.stdout:op 1558 completed, throughput=5MB/sec 2024-09-13T00:06:24.900 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-WuytO3OiuGCPFFF off=0 len=1787301 2024-09-13T00:06:24.912 INFO:tasks.workunit.client.0.smithi136.stdout:op 1559 completed, throughput=5MB/sec 2024-09-13T00:06:25.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:06:25 smithi181 bash[67587]: cluster 2024-09-13T00:06:23.297840+0000 mgr.y (mgr.24661) 878 : cluster [DBG] pgmap v1561: 105 pgs: 105 active+clean; 126 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:06:25.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:06:25 smithi136 bash[75164]: cluster 2024-09-13T00:06:23.297840+0000 mgr.y (mgr.24661) 878 : cluster [DBG] pgmap v1561: 105 pgs: 105 active+clean; 126 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:06:25.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:06:25 smithi136 bash[76371]: cluster 2024-09-13T00:06:23.297840+0000 mgr.y (mgr.24661) 878 : cluster [DBG] pgmap v1561: 105 pgs: 105 active+clean; 126 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:06:25.913 INFO:tasks.workunit.client.0.smithi136.stdout: 293: throughput=4.98MB/sec pending data=0 2024-09-13T00:06:25.913 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-A5B5Psi_O0Buj5h off=4340437 len=1397339 2024-09-13T00:06:25.924 INFO:tasks.workunit.client.0.smithi136.stdout:op 1560 completed, throughput=4.99MB/sec 2024-09-13T00:06:25.924 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ZSS81VPaD4KgHuM off=1263279 len=1743273 2024-09-13T00:06:25.938 INFO:tasks.workunit.client.0.smithi136.stdout:op 1561 completed, throughput=4.99MB/sec 2024-09-13T00:06:25.938 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-GeRf4vDLj0tyuYe off=1055304 len=1664221 2024-09-13T00:06:25.942 INFO:tasks.workunit.client.0.smithi136.stdout:op 1562 completed, throughput=5MB/sec 2024-09-13T00:06:25.942 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-MelwMvYxjlyidoo off=3417393 len=175921 2024-09-13T00:06:25.946 INFO:tasks.workunit.client.0.smithi136.stdout:op 1563 completed, throughput=5MB/sec 2024-09-13T00:06:25.946 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-I8_HzlEEhL6qP-F off=382041 len=520229 2024-09-13T00:06:25.948 INFO:tasks.workunit.client.0.smithi136.stdout:op 1564 completed, throughput=5MB/sec 2024-09-13T00:06:26.949 INFO:tasks.workunit.client.0.smithi136.stdout: 294: throughput=4.98MB/sec pending data=0 2024-09-13T00:06:26.949 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-aSSHjKW3h2j8bwV off=874421 len=1030153 2024-09-13T00:06:26.958 INFO:tasks.workunit.client.0.smithi136.stdout:op 1565 completed, throughput=4.99MB/sec 2024-09-13T00:06:26.958 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-RoUiRB8GpjnbDr9 off=2196913 len=490982 2024-09-13T00:06:26.960 INFO:tasks.workunit.client.0.smithi136.stdout:op 1566 completed, throughput=4.99MB/sec 2024-09-13T00:06:26.960 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-sQrSKBMovEk0AHA off=2487649 len=17134 2024-09-13T00:06:26.964 INFO:tasks.workunit.client.0.smithi136.stdout:op 1567 completed, throughput=4.99MB/sec 2024-09-13T00:06:26.964 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ccF7ntrdvY0yWYE off=4878398 len=539475 2024-09-13T00:06:26.971 INFO:tasks.workunit.client.0.smithi136.stdout:op 1568 completed, throughput=4.99MB/sec 2024-09-13T00:06:26.971 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-6gCMeZsa5zMX5NB off=252802 len=850767 2024-09-13T00:06:26.974 INFO:tasks.workunit.client.0.smithi136.stdout:op 1569 completed, throughput=4.99MB/sec 2024-09-13T00:06:26.974 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-1lNc94HXwjqK3Bi off=1131813 len=1582227 2024-09-13T00:06:26.977 INFO:tasks.workunit.client.0.smithi136.stdout:op 1570 completed, throughput=5MB/sec 2024-09-13T00:06:26.977 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-XGMDp2UQR-tdvFV off=12752055 len=1973749 2024-09-13T00:06:26.993 INFO:tasks.workunit.client.0.smithi136.stdout:op 1571 completed, throughput=5MB/sec 2024-09-13T00:06:27.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:06:27 smithi181 bash[67587]: cluster 2024-09-13T00:06:25.298650+0000 mgr.y (mgr.24661) 879 : cluster [DBG] pgmap v1562: 105 pgs: 105 active+clean; 126 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:06:27.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:06:27 smithi136 bash[75164]: cluster 2024-09-13T00:06:25.298650+0000 mgr.y (mgr.24661) 879 : cluster [DBG] pgmap v1562: 105 pgs: 105 active+clean; 126 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:06:27.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:06:27 smithi136 bash[76371]: cluster 2024-09-13T00:06:25.298650+0000 mgr.y (mgr.24661) 879 : cluster [DBG] pgmap v1562: 105 pgs: 105 active+clean; 126 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:06:27.993 INFO:tasks.workunit.client.0.smithi136.stdout: 295: throughput=4.99MB/sec pending data=0 2024-09-13T00:06:27.993 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ZuJkVbq1O3YnESi off=4702847 len=1698036 2024-09-13T00:06:27.997 INFO:tasks.workunit.client.0.smithi136.stdout:op 1572 completed, throughput=4.99MB/sec 2024-09-13T00:06:27.997 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-9lezh3IRY3moocX off=8292047 len=368923 2024-09-13T00:06:28.004 INFO:tasks.workunit.client.0.smithi136.stdout:op 1573 completed, throughput=4.99MB/sec 2024-09-13T00:06:28.004 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-vltilreAgk_HETp off=2108084 len=1882724 2024-09-13T00:06:28.016 INFO:tasks.workunit.client.0.smithi136.stdout:op 1574 completed, throughput=5MB/sec 2024-09-13T00:06:28.016 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-EsTdwznTeDXZ58v off=3985131 len=486940 2024-09-13T00:06:28.022 INFO:tasks.workunit.client.0.smithi136.stdout:op 1575 completed, throughput=5MB/sec 2024-09-13T00:06:29.022 INFO:tasks.workunit.client.0.smithi136.stdout: 296: throughput=4.98MB/sec pending data=0 2024-09-13T00:06:29.023 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ZgAI2PfQycP5SwB off=4382344 len=1131290 2024-09-13T00:06:29.025 INFO:tasks.workunit.client.0.smithi136.stdout:op 1576 completed, throughput=4.99MB/sec 2024-09-13T00:06:29.026 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-vhmbiM_iq1vF0xU off=1146859 len=259965 2024-09-13T00:06:29.027 INFO:tasks.workunit.client.0.smithi136.stdout:op 1577 completed, throughput=4.99MB/sec 2024-09-13T00:06:29.027 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-tx-l8f_SHTk1HUe off=8544684 len=1116477 2024-09-13T00:06:29.037 INFO:tasks.workunit.client.0.smithi136.stdout:op 1578 completed, throughput=4.99MB/sec 2024-09-13T00:06:29.037 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-n3uspmDhaYSQGdv off=378045 len=906978 2024-09-13T00:06:29.046 INFO:tasks.workunit.client.0.smithi136.stdout:op 1579 completed, throughput=5MB/sec 2024-09-13T00:06:29.046 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-FZov6wTa9nVLLmC off=13869070 len=1656791 2024-09-13T00:06:29.058 INFO:tasks.workunit.client.0.smithi136.stdout:op 1580 completed, throughput=5MB/sec 2024-09-13T00:06:29.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:06:29 smithi181 bash[67587]: cluster 2024-09-13T00:06:27.299736+0000 mgr.y (mgr.24661) 880 : cluster [DBG] pgmap v1563: 105 pgs: 105 active+clean; 126 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:06:29.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:06:29 smithi136 bash[75164]: cluster 2024-09-13T00:06:27.299736+0000 mgr.y (mgr.24661) 880 : cluster [DBG] pgmap v1563: 105 pgs: 105 active+clean; 126 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:06:29.414 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:06:29 smithi136 bash[76371]: cluster 2024-09-13T00:06:27.299736+0000 mgr.y (mgr.24661) 880 : cluster [DBG] pgmap v1563: 105 pgs: 105 active+clean; 126 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:06:29.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:06:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:06:29] "GET /metrics HTTP/1.1" 200 34125 "" "Prometheus/2.43.0" 2024-09-13T00:06:30.058 INFO:tasks.workunit.client.0.smithi136.stdout: 297: throughput=4.98MB/sec pending data=0 2024-09-13T00:06:30.058 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-yXV7K2EccwrHe_Q off=21276951 len=567140 2024-09-13T00:06:30.065 INFO:tasks.workunit.client.0.smithi136.stdout:op 1581 completed, throughput=4.99MB/sec 2024-09-13T00:06:30.065 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-2tG6YD5ltp7fDYy off=8477901 len=1103555 2024-09-13T00:06:30.074 INFO:tasks.workunit.client.0.smithi136.stdout:op 1582 completed, throughput=4.99MB/sec 2024-09-13T00:06:30.074 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-FW2bM8xYFxHjuY4 off=10250262 len=121639 2024-09-13T00:06:30.079 INFO:tasks.workunit.client.0.smithi136.stdout:op 1583 completed, throughput=4.99MB/sec 2024-09-13T00:06:30.079 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-XUSsfJqqeW-RNkX off=17199989 len=582344 2024-09-13T00:06:30.086 INFO:tasks.workunit.client.0.smithi136.stdout:op 1584 completed, throughput=4.99MB/sec 2024-09-13T00:06:30.086 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-b8KR1R5Ee4lv70z off=1177248 len=1874954 2024-09-13T00:06:30.092 INFO:tasks.workunit.client.0.smithi136.stdout:op 1585 completed, throughput=5MB/sec 2024-09-13T00:06:30.092 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-7uq5w_L5MqlAgSw off=11706982 len=758133 2024-09-13T00:06:30.095 INFO:tasks.workunit.client.0.smithi136.stdout:op 1586 completed, throughput=5MB/sec 2024-09-13T00:06:30.095 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-YFFND9PTUIYpTly off=3678402 len=729664 2024-09-13T00:06:30.102 INFO:tasks.workunit.client.0.smithi136.stdout:op 1587 completed, throughput=5MB/sec 2024-09-13T00:06:31.102 INFO:tasks.workunit.client.0.smithi136.stdout: 298: throughput=4.98MB/sec pending data=0 2024-09-13T00:06:31.102 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-wSJxpR7HIb2EikH off=465339 len=410956 2024-09-13T00:06:31.104 INFO:tasks.workunit.client.0.smithi136.stdout:op 1588 completed, throughput=4.99MB/sec 2024-09-13T00:06:31.104 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-kZVkIm9gmnYv7gb off=4903414 len=793564 2024-09-13T00:06:31.107 INFO:tasks.workunit.client.0.smithi136.stdout:op 1589 completed, throughput=4.99MB/sec 2024-09-13T00:06:31.107 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-8JAa-r0rsVT7bBO off=5521872 len=1373617 2024-09-13T00:06:31.117 INFO:tasks.workunit.client.0.smithi136.stdout:op 1590 completed, throughput=4.99MB/sec 2024-09-13T00:06:31.117 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-gdNGt5Omci4YS20 off=670244 len=24860 2024-09-13T00:06:31.121 INFO:tasks.workunit.client.0.smithi136.stdout:op 1591 completed, throughput=4.99MB/sec 2024-09-13T00:06:31.122 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-NqdKiUCOweVFCIN off=3210658 len=1785763 2024-09-13T00:06:31.133 INFO:tasks.workunit.client.0.smithi136.stdout:op 1592 completed, throughput=5MB/sec 2024-09-13T00:06:31.133 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-lTofQ89ZFCbXxj4 off=1562692 len=527866 2024-09-13T00:06:31.135 INFO:tasks.workunit.client.0.smithi136.stdout:op 1593 completed, throughput=5MB/sec 2024-09-13T00:06:31.135 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-xGDffZ5L1e2pf9i off=7855797 len=2012495 2024-09-13T00:06:31.139 INFO:tasks.workunit.client.0.smithi136.stdout:op 1594 completed, throughput=5.01MB/sec 2024-09-13T00:06:31.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:06:31 smithi181 bash[67587]: cluster 2024-09-13T00:06:29.300605+0000 mgr.y (mgr.24661) 881 : cluster [DBG] pgmap v1564: 105 pgs: 105 active+clean; 126 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:06:31.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:06:31 smithi136 bash[75164]: cluster 2024-09-13T00:06:29.300605+0000 mgr.y (mgr.24661) 881 : cluster [DBG] pgmap v1564: 105 pgs: 105 active+clean; 126 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:06:31.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:06:31 smithi136 bash[76371]: cluster 2024-09-13T00:06:29.300605+0000 mgr.y (mgr.24661) 881 : cluster [DBG] pgmap v1564: 105 pgs: 105 active+clean; 126 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:06:32.140 INFO:tasks.workunit.client.0.smithi136.stdout: 299: throughput=4.99MB/sec pending data=0 2024-09-13T00:06:32.140 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-7g-DxTRoxxFNYA4 off=13044455 len=1465172 2024-09-13T00:06:32.151 INFO:tasks.workunit.client.0.smithi136.stdout:op 1595 completed, throughput=4.99MB/sec 2024-09-13T00:06:32.151 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-HhiLLE2sxhn5brK off=2136694 len=679339 2024-09-13T00:06:32.154 INFO:tasks.workunit.client.0.smithi136.stdout:op 1596 completed, throughput=5MB/sec 2024-09-13T00:06:32.154 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-gvq246D4ps-fncU off=1626462 len=127214 2024-09-13T00:06:32.155 INFO:tasks.workunit.client.0.smithi136.stdout:op 1597 completed, throughput=5MB/sec 2024-09-13T00:06:32.155 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-RsO7_dGEGLVCC_S off=14904463 len=1546249 2024-09-13T00:06:32.166 INFO:tasks.workunit.client.0.smithi136.stdout:op 1598 completed, throughput=5MB/sec 2024-09-13T00:06:32.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:06:32 smithi181 bash[95505]: ts=2024-09-13T00:06:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:06:33.166 INFO:tasks.workunit.client.0.smithi136.stdout: 300: throughput=4.98MB/sec pending data=0 2024-09-13T00:06:33.166 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-WM5dvQTI4mLomLe off=1066437 len=1212894 2024-09-13T00:06:33.169 INFO:tasks.workunit.client.0.smithi136.stdout:op 1599 completed, throughput=4.99MB/sec 2024-09-13T00:06:33.169 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-2CdPL1REissFLlB off=19540219 len=1553007 2024-09-13T00:06:33.183 INFO:tasks.workunit.client.0.smithi136.stdout:op 1600 completed, throughput=4.99MB/sec 2024-09-13T00:06:33.184 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-n4fKYCZpgVdRGT0 off=19452537 len=227844 2024-09-13T00:06:33.185 INFO:tasks.workunit.client.0.smithi136.stdout:op 1601 completed, throughput=4.99MB/sec 2024-09-13T00:06:33.185 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-e1yJY-TC4VNI0sE off=1523318 len=1740484 2024-09-13T00:06:33.189 INFO:tasks.workunit.client.0.smithi136.stdout:op 1602 completed, throughput=5MB/sec 2024-09-13T00:06:33.189 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-b5b9-CWCiubySmZ off=5597711 len=1140573 2024-09-13T00:06:33.192 INFO:tasks.workunit.client.0.smithi136.stdout:op 1603 completed, throughput=5MB/sec 2024-09-13T00:06:33.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:06:33 smithi181 bash[67587]: cluster 2024-09-13T00:06:31.301553+0000 mgr.y (mgr.24661) 882 : cluster [DBG] pgmap v1565: 105 pgs: 105 active+clean; 126 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:06:33.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:06:33 smithi136 bash[75164]: cluster 2024-09-13T00:06:31.301553+0000 mgr.y (mgr.24661) 882 : cluster [DBG] pgmap v1565: 105 pgs: 105 active+clean; 126 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:06:33.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:06:33 smithi136 bash[76371]: cluster 2024-09-13T00:06:31.301553+0000 mgr.y (mgr.24661) 882 : cluster [DBG] pgmap v1565: 105 pgs: 105 active+clean; 126 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:06:34.192 INFO:tasks.workunit.client.0.smithi136.stdout: 301: throughput=4.99MB/sec pending data=0 2024-09-13T00:06:34.193 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-jWBcN2xf8Owsqdq off=12168923 len=1606740 2024-09-13T00:06:34.196 INFO:tasks.workunit.client.0.smithi136.stdout:op 1604 completed, throughput=4.99MB/sec 2024-09-13T00:06:34.196 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-DJa2iBeRDaFk-OB off=4298936 len=1683575 2024-09-13T00:06:34.208 INFO:tasks.workunit.client.0.smithi136.stdout:op 1605 completed, throughput=5MB/sec 2024-09-13T00:06:34.208 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-PObhZtKfAplnmrp off=4804929 len=1434435 2024-09-13T00:06:34.211 INFO:tasks.workunit.client.0.smithi136.stdout:op 1606 completed, throughput=5MB/sec 2024-09-13T00:06:35.212 INFO:tasks.workunit.client.0.smithi136.stdout: 302: throughput=4.98MB/sec pending data=0 2024-09-13T00:06:35.212 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-CZ5FQNwjdrFVXoK off=4083693 len=647037 2024-09-13T00:06:35.214 INFO:tasks.workunit.client.0.smithi136.stdout:op 1607 completed, throughput=4.99MB/sec 2024-09-13T00:06:35.214 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-sRIt_JyGHKGhM0R off=1675546 len=595480 2024-09-13T00:06:35.217 INFO:tasks.workunit.client.0.smithi136.stdout:op 1608 completed, throughput=4.99MB/sec 2024-09-13T00:06:35.217 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-qBX-_pf_-LWFNXD off=7712956 len=1261083 2024-09-13T00:06:35.227 INFO:tasks.workunit.client.0.smithi136.stdout:op 1609 completed, throughput=4.99MB/sec 2024-09-13T00:06:35.227 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-5_i5J8YTylhwgu1 off=3504445 len=422179 2024-09-13T00:06:35.230 INFO:tasks.workunit.client.0.smithi136.stdout:op 1610 completed, throughput=4.99MB/sec 2024-09-13T00:06:35.230 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-XYtk4lTVYCG2w-4 off=14019029 len=65986 2024-09-13T00:06:35.231 INFO:tasks.workunit.client.0.smithi136.stdout:op 1611 completed, throughput=4.99MB/sec 2024-09-13T00:06:35.231 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-hcgo4WieZvWDVG7 off=0 len=1154614 2024-09-13T00:06:35.234 INFO:tasks.workunit.client.0.smithi136.stdout:op 1612 completed, throughput=5MB/sec 2024-09-13T00:06:35.234 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-pvLBiDarUJxNvdT off=1691522 len=48630 2024-09-13T00:06:35.235 INFO:tasks.workunit.client.0.smithi136.stdout:op 1613 completed, throughput=5MB/sec 2024-09-13T00:06:35.235 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ZuwCHv5miUXlwER off=9749532 len=1254416 2024-09-13T00:06:35.237 INFO:tasks.workunit.client.0.smithi136.stdout:op 1614 completed, throughput=5MB/sec 2024-09-13T00:06:35.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:06:35 smithi181 bash[67587]: cluster 2024-09-13T00:06:33.302490+0000 mgr.y (mgr.24661) 883 : cluster [DBG] pgmap v1566: 105 pgs: 105 active+clean; 126 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 2.1 MiB/s rd, 3.1 MiB/s wr, 5 op/s 2024-09-13T00:06:35.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:06:35 smithi136 bash[75164]: cluster 2024-09-13T00:06:33.302490+0000 mgr.y (mgr.24661) 883 : cluster [DBG] pgmap v1566: 105 pgs: 105 active+clean; 126 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 2.1 MiB/s rd, 3.1 MiB/s wr, 5 op/s 2024-09-13T00:06:35.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:06:35 smithi136 bash[76371]: cluster 2024-09-13T00:06:33.302490+0000 mgr.y (mgr.24661) 883 : cluster [DBG] pgmap v1566: 105 pgs: 105 active+clean; 126 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 2.1 MiB/s rd, 3.1 MiB/s wr, 5 op/s 2024-09-13T00:06:36.238 INFO:tasks.workunit.client.0.smithi136.stdout: 303: throughput=4.98MB/sec pending data=0 2024-09-13T00:06:36.238 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-DFg6lEMjs2-fvhu off=2512094 len=1352018 2024-09-13T00:06:36.241 INFO:tasks.workunit.client.0.smithi136.stdout:op 1615 completed, throughput=4.99MB/sec 2024-09-13T00:06:36.241 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-U78k802nE40z-q_ off=874006 len=1677442 2024-09-13T00:06:36.247 INFO:tasks.workunit.client.0.smithi136.stdout:op 1616 completed, throughput=4.99MB/sec 2024-09-13T00:06:36.247 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-XSZSLiVZUV9O9eU off=2525214 len=2018350 2024-09-13T00:06:36.252 INFO:tasks.workunit.client.0.smithi136.stdout:op 1617 completed, throughput=5MB/sec 2024-09-13T00:06:36.252 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-9P7CXhNNHqnnOSs off=1827841 len=574935 2024-09-13T00:06:36.254 INFO:tasks.workunit.client.0.smithi136.stdout:op 1618 completed, throughput=5MB/sec 2024-09-13T00:06:37.255 INFO:tasks.workunit.client.0.smithi136.stdout: 304: throughput=4.99MB/sec pending data=0 2024-09-13T00:06:37.255 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-TIk043-KeYRizbz off=7882149 len=154431 2024-09-13T00:06:37.259 INFO:tasks.workunit.client.0.smithi136.stdout:op 1619 completed, throughput=4.99MB/sec 2024-09-13T00:06:37.259 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-gz3yqiv8PWOkbsI off=76234 len=1712486 2024-09-13T00:06:37.271 INFO:tasks.workunit.client.0.smithi136.stdout:op 1620 completed, throughput=4.99MB/sec 2024-09-13T00:06:37.271 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-3XnDA1q41-6eS9u off=2639955 len=626426 2024-09-13T00:06:37.279 INFO:tasks.workunit.client.0.smithi136.stdout:op 1621 completed, throughput=4.99MB/sec 2024-09-13T00:06:37.279 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-h7xpIfN57Vk-UeV off=1624005 len=459884 2024-09-13T00:06:37.284 INFO:tasks.workunit.client.0.smithi136.stdout:op 1622 completed, throughput=4.99MB/sec 2024-09-13T00:06:37.284 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-jKbfh7c4efX6URp off=3323738 len=1218052 2024-09-13T00:06:37.289 INFO:tasks.workunit.client.0.smithi136.stdout:op 1623 completed, throughput=5MB/sec 2024-09-13T00:06:37.289 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Y-vk2KXFXpTzzpl off=19555501 len=1687881 2024-09-13T00:06:37.294 INFO:tasks.workunit.client.0.smithi136.stdout:op 1624 completed, throughput=5MB/sec 2024-09-13T00:06:37.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:06:37 smithi181 bash[67587]: cluster 2024-09-13T00:06:35.303336+0000 mgr.y (mgr.24661) 884 : cluster [DBG] pgmap v1567: 105 pgs: 105 active+clean; 126 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 2.3 MiB/s rd, 3.0 MiB/s wr, 5 op/s 2024-09-13T00:06:37.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:06:37 smithi136 bash[75164]: cluster 2024-09-13T00:06:35.303336+0000 mgr.y (mgr.24661) 884 : cluster [DBG] pgmap v1567: 105 pgs: 105 active+clean; 126 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 2.3 MiB/s rd, 3.0 MiB/s wr, 5 op/s 2024-09-13T00:06:37.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:06:37 smithi136 bash[76371]: cluster 2024-09-13T00:06:35.303336+0000 mgr.y (mgr.24661) 884 : cluster [DBG] pgmap v1567: 105 pgs: 105 active+clean; 126 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 2.3 MiB/s rd, 3.0 MiB/s wr, 5 op/s 2024-09-13T00:06:38.295 INFO:tasks.workunit.client.0.smithi136.stdout: 305: throughput=4.99MB/sec pending data=0 2024-09-13T00:06:38.295 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-eGDtNOOCPd0W6Sx off=3131189 len=1258097 2024-09-13T00:06:38.300 INFO:tasks.workunit.client.0.smithi136.stdout:op 1625 completed, throughput=4.99MB/sec 2024-09-13T00:06:38.300 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-xvW5rOlJnLp80yx off=20212645 len=285993 2024-09-13T00:06:38.301 INFO:tasks.workunit.client.0.smithi136.stdout:op 1626 completed, throughput=4.99MB/sec 2024-09-13T00:06:38.301 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-qZkjZZA9DRzn8Do off=242299 len=1006747 2024-09-13T00:06:38.310 INFO:tasks.workunit.client.0.smithi136.stdout:op 1627 completed, throughput=4.99MB/sec 2024-09-13T00:06:38.310 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-btH7d1DkN2JcOrr off=9222119 len=257141 2024-09-13T00:06:38.315 INFO:tasks.workunit.client.0.smithi136.stdout:op 1628 completed, throughput=4.99MB/sec 2024-09-13T00:06:38.315 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-YjVTJNGS7d3nwld off=11489138 len=1774439 2024-09-13T00:06:38.319 INFO:tasks.workunit.client.0.smithi136.stdout:op 1629 completed, throughput=5MB/sec 2024-09-13T00:06:39.320 INFO:tasks.workunit.client.0.smithi136.stdout: 306: throughput=4.98MB/sec pending data=0 2024-09-13T00:06:39.320 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-grtyFl20GsrRjug off=5988897 len=1985806 2024-09-13T00:06:39.326 INFO:tasks.workunit.client.0.smithi136.stdout:op 1630 completed, throughput=4.99MB/sec 2024-09-13T00:06:39.326 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-BSI8wxSF_6Qoh1x off=16522745 len=512225 2024-09-13T00:06:39.328 INFO:tasks.workunit.client.0.smithi136.stdout:op 1631 completed, throughput=4.99MB/sec 2024-09-13T00:06:39.328 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-bLnsf1Tyc0DdMc1 off=1536603 len=114619 2024-09-13T00:06:39.333 INFO:tasks.workunit.client.0.smithi136.stdout:op 1632 completed, throughput=4.99MB/sec 2024-09-13T00:06:39.333 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-GzjrF2_SRnERDJq off=3048941 len=405108 2024-09-13T00:06:39.334 INFO:tasks.workunit.client.0.smithi136.stdout:op 1633 completed, throughput=4.99MB/sec 2024-09-13T00:06:39.334 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-H4OO1HxPuJhClm_ off=12423534 len=113044 2024-09-13T00:06:39.336 INFO:tasks.workunit.client.0.smithi136.stdout:op 1634 completed, throughput=4.99MB/sec 2024-09-13T00:06:39.336 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ZScCoNxN2WEbAku off=361489 len=1780992 2024-09-13T00:06:39.340 INFO:tasks.workunit.client.0.smithi136.stdout:op 1635 completed, throughput=5MB/sec 2024-09-13T00:06:39.340 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ymVhMGBOk1Y9F5G off=122349 len=570892 2024-09-13T00:06:39.342 INFO:tasks.workunit.client.0.smithi136.stdout:op 1636 completed, throughput=5MB/sec 2024-09-13T00:06:39.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:06:39 smithi181 bash[67587]: cluster 2024-09-13T00:06:37.304454+0000 mgr.y (mgr.24661) 885 : cluster [DBG] pgmap v1568: 105 pgs: 105 active+clean; 126 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:06:39.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:06:39 smithi181 bash[67587]: audit 2024-09-13T00:06:38.780020+0000 mon.a (mon.0) 2151 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:06:39.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:06:39 smithi136 bash[76371]: cluster 2024-09-13T00:06:37.304454+0000 mgr.y (mgr.24661) 885 : cluster [DBG] pgmap v1568: 105 pgs: 105 active+clean; 126 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:06:39.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:06:39 smithi136 bash[76371]: audit 2024-09-13T00:06:38.780020+0000 mon.a (mon.0) 2151 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:06:39.414 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:06:39 smithi136 bash[75164]: cluster 2024-09-13T00:06:37.304454+0000 mgr.y (mgr.24661) 885 : cluster [DBG] pgmap v1568: 105 pgs: 105 active+clean; 126 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:06:39.414 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:06:39 smithi136 bash[75164]: audit 2024-09-13T00:06:38.780020+0000 mon.a (mon.0) 2151 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:06:39.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:06:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:06:39] "GET /metrics HTTP/1.1" 200 34123 "" "Prometheus/2.43.0" 2024-09-13T00:06:40.343 INFO:tasks.workunit.client.0.smithi136.stdout: 307: throughput=4.98MB/sec pending data=0 2024-09-13T00:06:40.343 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-lb9mlN_eO-KPbv4 off=506510 len=1531809 2024-09-13T00:06:40.353 INFO:tasks.workunit.client.0.smithi136.stdout:op 1637 completed, throughput=4.99MB/sec 2024-09-13T00:06:40.353 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ZT49rjlM-HK_3Pz off=6438478 len=729897 2024-09-13T00:06:40.355 INFO:tasks.workunit.client.0.smithi136.stdout:op 1638 completed, throughput=4.99MB/sec 2024-09-13T00:06:40.355 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ESEw0AccS17iZb- off=4199865 len=1443393 2024-09-13T00:06:40.360 INFO:tasks.workunit.client.0.smithi136.stdout:op 1639 completed, throughput=5MB/sec 2024-09-13T00:06:40.360 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-xr8BUEStW3cQQXU off=12216465 len=868177 2024-09-13T00:06:40.364 INFO:tasks.workunit.client.0.smithi136.stdout:op 1640 completed, throughput=5MB/sec 2024-09-13T00:06:40.364 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-KdBKgAVYSkhiQTk off=4374895 len=1218797 2024-09-13T00:06:40.375 INFO:tasks.workunit.client.0.smithi136.stdout:op 1641 completed, throughput=5MB/sec 2024-09-13T00:06:41.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:06:41 smithi181 bash[67587]: cluster 2024-09-13T00:06:39.305336+0000 mgr.y (mgr.24661) 886 : cluster [DBG] pgmap v1569: 105 pgs: 105 active+clean; 126 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:06:41.376 INFO:tasks.workunit.client.0.smithi136.stdout: 308: throughput=4.99MB/sec pending data=0 2024-09-13T00:06:41.376 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj--9aA3YrrSLlXgE4 off=4642393 len=1885866 2024-09-13T00:06:41.380 INFO:tasks.workunit.client.0.smithi136.stdout:op 1642 completed, throughput=4.99MB/sec 2024-09-13T00:06:41.380 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-zIQt-vHUqMmti4t off=301957 len=2053883 2024-09-13T00:06:41.385 INFO:tasks.workunit.client.0.smithi136.stdout:op 1643 completed, throughput=5MB/sec 2024-09-13T00:06:41.385 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-6gpP8itaidCLXJg off=17045794 len=618046 2024-09-13T00:06:41.387 INFO:tasks.workunit.client.0.smithi136.stdout:op 1644 completed, throughput=5MB/sec 2024-09-13T00:06:41.387 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-W7o-3NC71bLm15u off=1344157 len=427597 2024-09-13T00:06:41.393 INFO:tasks.workunit.client.0.smithi136.stdout:op 1645 completed, throughput=5MB/sec 2024-09-13T00:06:41.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:06:41 smithi136 bash[75164]: cluster 2024-09-13T00:06:39.305336+0000 mgr.y (mgr.24661) 886 : cluster [DBG] pgmap v1569: 105 pgs: 105 active+clean; 126 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:06:41.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:06:41 smithi136 bash[76371]: cluster 2024-09-13T00:06:39.305336+0000 mgr.y (mgr.24661) 886 : cluster [DBG] pgmap v1569: 105 pgs: 105 active+clean; 126 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:06:42.394 INFO:tasks.workunit.client.0.smithi136.stdout: 309: throughput=4.98MB/sec pending data=0 2024-09-13T00:06:42.394 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-M3m_z_pL3etfNiG off=805117 len=1709290 2024-09-13T00:06:42.405 INFO:tasks.workunit.client.0.smithi136.stdout:op 1646 completed, throughput=4.99MB/sec 2024-09-13T00:06:42.406 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-fCfrgMalS0EY1xV off=1592392 len=1850183 2024-09-13T00:06:42.410 INFO:tasks.workunit.client.0.smithi136.stdout:op 1647 completed, throughput=5MB/sec 2024-09-13T00:06:42.410 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-pgd9maktiD7V3FQ off=6989580 len=827500 2024-09-13T00:06:42.419 INFO:tasks.workunit.client.0.smithi136.stdout:op 1648 completed, throughput=5MB/sec 2024-09-13T00:06:42.419 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-H4OO1HxPuJhClm_ off=11013255 len=1291192 2024-09-13T00:06:42.424 INFO:tasks.workunit.client.0.smithi136.stdout:op 1649 completed, throughput=5MB/sec 2024-09-13T00:06:42.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:06:42 smithi181 bash[95505]: ts=2024-09-13T00:06:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:06:43.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:06:43 smithi181 bash[67587]: cluster 2024-09-13T00:06:41.306293+0000 mgr.y (mgr.24661) 887 : cluster [DBG] pgmap v1570: 105 pgs: 105 active+clean; 126 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.1 MiB/s wr, 5 op/s 2024-09-13T00:06:43.424 INFO:tasks.workunit.client.0.smithi136.stdout: 310: throughput=4.99MB/sec pending data=0 2024-09-13T00:06:43.424 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-CpUCcZJPQshYZux off=212713 len=868875 2024-09-13T00:06:43.427 INFO:tasks.workunit.client.0.smithi136.stdout:op 1650 completed, throughput=4.99MB/sec 2024-09-13T00:06:43.427 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-MyUBeku8joM3Fy6 off=1206571 len=1777399 2024-09-13T00:06:43.440 INFO:tasks.workunit.client.0.smithi136.stdout:op 1651 completed, throughput=4.99MB/sec 2024-09-13T00:06:43.440 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-GWLCCkmdW18kyHS off=9366911 len=532535 2024-09-13T00:06:43.442 INFO:tasks.workunit.client.0.smithi136.stdout:op 1652 completed, throughput=5MB/sec 2024-09-13T00:06:43.442 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-7gBkrqaGYzxZr_M off=13292713 len=255811 2024-09-13T00:06:43.444 INFO:tasks.workunit.client.0.smithi136.stdout:op 1653 completed, throughput=5MB/sec 2024-09-13T00:06:43.444 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-sRZYpf3NqEfEMKK off=8539571 len=522183 2024-09-13T00:06:43.450 INFO:tasks.workunit.client.0.smithi136.stdout:op 1654 completed, throughput=5MB/sec 2024-09-13T00:06:43.450 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-_3ZAUz0wvYkXPNc off=208011 len=250707 2024-09-13T00:06:43.451 INFO:tasks.workunit.client.0.smithi136.stdout:op 1655 completed, throughput=5MB/sec 2024-09-13T00:06:43.451 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-LiUZ705F5Y8O4cK off=15005394 len=903876 2024-09-13T00:06:43.455 INFO:tasks.workunit.client.0.smithi136.stdout:op 1656 completed, throughput=5MB/sec 2024-09-13T00:06:43.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:06:43 smithi136 bash[75164]: cluster 2024-09-13T00:06:41.306293+0000 mgr.y (mgr.24661) 887 : cluster [DBG] pgmap v1570: 105 pgs: 105 active+clean; 126 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.1 MiB/s wr, 5 op/s 2024-09-13T00:06:43.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:06:43 smithi136 bash[76371]: cluster 2024-09-13T00:06:41.306293+0000 mgr.y (mgr.24661) 887 : cluster [DBG] pgmap v1570: 105 pgs: 105 active+clean; 126 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.1 MiB/s wr, 5 op/s 2024-09-13T00:06:44.455 INFO:tasks.workunit.client.0.smithi136.stdout: 311: throughput=4.99MB/sec pending data=0 2024-09-13T00:06:44.456 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-lw0FHBGgFyU7bo8 off=4105793 len=78830 2024-09-13T00:06:44.459 INFO:tasks.workunit.client.0.smithi136.stdout:op 1657 completed, throughput=4.99MB/sec 2024-09-13T00:06:44.459 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-7zjja-QLoOhygmz off=14073885 len=856496 2024-09-13T00:06:44.466 INFO:tasks.workunit.client.0.smithi136.stdout:op 1658 completed, throughput=4.99MB/sec 2024-09-13T00:06:44.467 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-q_Wrx26jUOiGw6c off=7158250 len=2082377 2024-09-13T00:06:44.481 INFO:tasks.workunit.client.0.smithi136.stdout:op 1659 completed, throughput=4.99MB/sec 2024-09-13T00:06:44.481 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-dXFyMH7pXcjtG4S off=888738 len=1800842 2024-09-13T00:06:44.492 INFO:tasks.workunit.client.0.smithi136.stdout:op 1660 completed, throughput=5MB/sec 2024-09-13T00:06:44.492 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-HFNV_jrLep6H4QE off=10532556 len=438676 2024-09-13T00:06:44.494 INFO:tasks.workunit.client.0.smithi136.stdout:op 1661 completed, throughput=5MB/sec 2024-09-13T00:06:45.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:06:45 smithi181 bash[67587]: cluster 2024-09-13T00:06:43.307219+0000 mgr.y (mgr.24661) 888 : cluster [DBG] pgmap v1571: 105 pgs: 105 active+clean; 126 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 3.0 MiB/s rd, 1.9 MiB/s wr, 5 op/s 2024-09-13T00:06:45.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:06:45 smithi181 bash[67587]: audit 2024-09-13T00:06:44.339244+0000 mon.a (mon.0) 2152 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:06:45.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:06:45 smithi181 bash[67587]: audit 2024-09-13T00:06:44.717118+0000 mon.a (mon.0) 2153 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:06:45.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:06:45 smithi181 bash[67587]: audit 2024-09-13T00:06:44.726953+0000 mon.a (mon.0) 2154 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:06:45.494 INFO:tasks.workunit.client.0.smithi136.stdout: 312: throughput=4.98MB/sec pending data=0 2024-09-13T00:06:45.494 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-bWm0CCZny0Q7Pb5 off=13447819 len=1192986 2024-09-13T00:06:45.503 INFO:tasks.workunit.client.0.smithi136.stdout:op 1662 completed, throughput=4.99MB/sec 2024-09-13T00:06:45.503 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-_eecqh3TjlEp8O2 off=3037195 len=207310 2024-09-13T00:06:45.505 INFO:tasks.workunit.client.0.smithi136.stdout:op 1663 completed, throughput=4.99MB/sec 2024-09-13T00:06:45.505 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Pbz9aO3Ot8t1v5Y off=15630180 len=1089071 2024-09-13T00:06:45.507 INFO:tasks.workunit.client.0.smithi136.stdout:op 1664 completed, throughput=4.99MB/sec 2024-09-13T00:06:45.507 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-mDh3TdCcTnO90WS off=1831599 len=1199057 2024-09-13T00:06:45.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:06:45 smithi136 bash[76371]: cluster 2024-09-13T00:06:43.307219+0000 mgr.y (mgr.24661) 888 : cluster [DBG] pgmap v1571: 105 pgs: 105 active+clean; 126 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 3.0 MiB/s rd, 1.9 MiB/s wr, 5 op/s 2024-09-13T00:06:45.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:06:45 smithi136 bash[76371]: audit 2024-09-13T00:06:44.339244+0000 mon.a (mon.0) 2152 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:06:45.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:06:45 smithi136 bash[76371]: audit 2024-09-13T00:06:44.717118+0000 mon.a (mon.0) 2153 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:06:45.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:06:45 smithi136 bash[76371]: audit 2024-09-13T00:06:44.726953+0000 mon.a (mon.0) 2154 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:06:45.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:06:45 smithi136 bash[75164]: cluster 2024-09-13T00:06:43.307219+0000 mgr.y (mgr.24661) 888 : cluster [DBG] pgmap v1571: 105 pgs: 105 active+clean; 126 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 3.0 MiB/s rd, 1.9 MiB/s wr, 5 op/s 2024-09-13T00:06:45.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:06:45 smithi136 bash[75164]: audit 2024-09-13T00:06:44.339244+0000 mon.a (mon.0) 2152 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:06:45.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:06:45 smithi136 bash[75164]: audit 2024-09-13T00:06:44.717118+0000 mon.a (mon.0) 2153 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:06:45.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:06:45 smithi136 bash[75164]: audit 2024-09-13T00:06:44.726953+0000 mon.a (mon.0) 2154 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:06:45.510 INFO:tasks.workunit.client.0.smithi136.stdout:op 1665 completed, throughput=5MB/sec 2024-09-13T00:06:45.510 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-E0xtRi8t_cLLGLq off=91414 len=656736 2024-09-13T00:06:45.517 INFO:tasks.workunit.client.0.smithi136.stdout:op 1666 completed, throughput=5MB/sec 2024-09-13T00:06:45.517 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-30lfS6yEdgsPGOn off=6694892 len=970224 2024-09-13T00:06:45.520 INFO:tasks.workunit.client.0.smithi136.stdout:op 1667 completed, throughput=5MB/sec 2024-09-13T00:06:45.520 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-QV93M3VE7DDas_8 off=7725 len=1266777 2024-09-13T00:06:45.523 INFO:tasks.workunit.client.0.smithi136.stdout:op 1668 completed, throughput=5MB/sec 2024-09-13T00:06:46.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:06:46 smithi181 bash[67587]: audit 2024-09-13T00:06:45.182869+0000 mon.a (mon.0) 2155 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:06:46.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:06:46 smithi181 bash[67587]: audit 2024-09-13T00:06:45.184456+0000 mon.a (mon.0) 2156 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:06:46.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:06:46 smithi181 bash[67587]: audit 2024-09-13T00:06:45.195470+0000 mon.a (mon.0) 2157 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:06:46.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:06:46 smithi136 bash[75164]: audit 2024-09-13T00:06:45.182869+0000 mon.a (mon.0) 2155 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:06:46.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:06:46 smithi136 bash[75164]: audit 2024-09-13T00:06:45.184456+0000 mon.a (mon.0) 2156 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:06:46.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:06:46 smithi136 bash[75164]: audit 2024-09-13T00:06:45.195470+0000 mon.a (mon.0) 2157 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:06:46.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:06:46 smithi136 bash[76371]: audit 2024-09-13T00:06:45.182869+0000 mon.a (mon.0) 2155 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:06:46.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:06:46 smithi136 bash[76371]: audit 2024-09-13T00:06:45.184456+0000 mon.a (mon.0) 2156 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:06:46.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:06:46 smithi136 bash[76371]: audit 2024-09-13T00:06:45.195470+0000 mon.a (mon.0) 2157 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:06:46.523 INFO:tasks.workunit.client.0.smithi136.stdout: 313: throughput=4.99MB/sec pending data=0 2024-09-13T00:06:46.524 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-jiXEet0Gd0cjDC9 off=1972096 len=1683228 2024-09-13T00:06:46.535 INFO:tasks.workunit.client.0.smithi136.stdout:op 1669 completed, throughput=4.99MB/sec 2024-09-13T00:06:46.535 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-DeuSSkZ49TNFzOK off=837652 len=1608329 2024-09-13T00:06:46.539 INFO:tasks.workunit.client.0.smithi136.stdout:op 1670 completed, throughput=5MB/sec 2024-09-13T00:06:46.539 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ExZfvmS6OTgpMfd off=3553678 len=1084256 2024-09-13T00:06:46.544 INFO:tasks.workunit.client.0.smithi136.stdout:op 1671 completed, throughput=5MB/sec 2024-09-13T00:06:47.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:06:47 smithi181 bash[67587]: cluster 2024-09-13T00:06:45.308020+0000 mgr.y (mgr.24661) 889 : cluster [DBG] pgmap v1572: 105 pgs: 105 active+clean; 126 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.4 MiB/s wr, 4 op/s 2024-09-13T00:06:47.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:06:47 smithi136 bash[75164]: cluster 2024-09-13T00:06:45.308020+0000 mgr.y (mgr.24661) 889 : cluster [DBG] pgmap v1572: 105 pgs: 105 active+clean; 126 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.4 MiB/s wr, 4 op/s 2024-09-13T00:06:47.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:06:47 smithi136 bash[76371]: cluster 2024-09-13T00:06:45.308020+0000 mgr.y (mgr.24661) 889 : cluster [DBG] pgmap v1572: 105 pgs: 105 active+clean; 126 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.4 MiB/s wr, 4 op/s 2024-09-13T00:06:47.544 INFO:tasks.workunit.client.0.smithi136.stdout: 314: throughput=4.99MB/sec pending data=0 2024-09-13T00:06:47.544 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-FGIESCWEmYrEAKI off=6978492 len=551778 2024-09-13T00:06:47.550 INFO:tasks.workunit.client.0.smithi136.stdout:op 1672 completed, throughput=4.99MB/sec 2024-09-13T00:06:47.551 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-IAM9OKldQBwUSZD off=9792301 len=2051434 2024-09-13T00:06:47.564 INFO:tasks.workunit.client.0.smithi136.stdout:op 1673 completed, throughput=4.99MB/sec 2024-09-13T00:06:47.564 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-tneUHlEOa7bX5Qk off=16818348 len=439615 2024-09-13T00:06:47.565 INFO:tasks.workunit.client.0.smithi136.stdout:op 1674 completed, throughput=4.99MB/sec 2024-09-13T00:06:47.566 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-vdxwubwMT6XGQp0 off=446570 len=67913 2024-09-13T00:06:47.566 INFO:tasks.workunit.client.0.smithi136.stdout:op 1675 completed, throughput=4.99MB/sec 2024-09-13T00:06:47.566 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-z4S_VVGd9vjXKDG off=8292401 len=1160195 2024-09-13T00:06:47.570 INFO:tasks.workunit.client.0.smithi136.stdout:op 1676 completed, throughput=5MB/sec 2024-09-13T00:06:47.570 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-OnOT3hcL1ar2s-G off=1516849 len=318916 2024-09-13T00:06:47.572 INFO:tasks.workunit.client.0.smithi136.stdout:op 1677 completed, throughput=5MB/sec 2024-09-13T00:06:47.572 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-9MWduyuuNRJzjEv off=3872334 len=778860 2024-09-13T00:06:47.580 INFO:tasks.workunit.client.0.smithi136.stdout:op 1678 completed, throughput=5MB/sec 2024-09-13T00:06:48.580 INFO:tasks.workunit.client.0.smithi136.stdout: 315: throughput=4.99MB/sec pending data=0 2024-09-13T00:06:48.580 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-cM1Dfil8JEk56xF off=646880 len=1880442 2024-09-13T00:06:48.594 INFO:tasks.workunit.client.0.smithi136.stdout:op 1679 completed, throughput=4.99MB/sec 2024-09-13T00:06:48.594 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-7d1x6keyUFndRe5 off=16088784 len=1633231 2024-09-13T00:06:48.603 INFO:tasks.workunit.client.0.smithi136.stdout:op 1680 completed, throughput=5MB/sec 2024-09-13T00:06:48.603 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-6MF2SJxY-E2sVFk off=1125749 len=424437 2024-09-13T00:06:48.610 INFO:tasks.workunit.client.0.smithi136.stdout:op 1681 completed, throughput=5MB/sec 2024-09-13T00:06:48.610 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-30U-z2c9BF9FjF6 off=2111071 len=1391565 2024-09-13T00:06:48.615 INFO:tasks.workunit.client.0.smithi136.stdout:op 1682 completed, throughput=5MB/sec 2024-09-13T00:06:49.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:06:49 smithi181 bash[67587]: cluster 2024-09-13T00:06:47.309186+0000 mgr.y (mgr.24661) 890 : cluster [DBG] pgmap v1573: 105 pgs: 105 active+clean; 126 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.4 MiB/s wr, 5 op/s 2024-09-13T00:06:49.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:06:49 smithi136 bash[75164]: cluster 2024-09-13T00:06:47.309186+0000 mgr.y (mgr.24661) 890 : cluster [DBG] pgmap v1573: 105 pgs: 105 active+clean; 126 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.4 MiB/s wr, 5 op/s 2024-09-13T00:06:49.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:06:49 smithi136 bash[76371]: cluster 2024-09-13T00:06:47.309186+0000 mgr.y (mgr.24661) 890 : cluster [DBG] pgmap v1573: 105 pgs: 105 active+clean; 126 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.4 MiB/s wr, 5 op/s 2024-09-13T00:06:49.615 INFO:tasks.workunit.client.0.smithi136.stdout: 316: throughput=4.98MB/sec pending data=0 2024-09-13T00:06:49.615 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-YSY8Dq93oNAdzPz off=501384 len=1080241 2024-09-13T00:06:49.618 INFO:tasks.workunit.client.0.smithi136.stdout:op 1683 completed, throughput=4.99MB/sec 2024-09-13T00:06:49.618 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-u9DH3vxbjJZk-OD off=9846151 len=815294 2024-09-13T00:06:49.621 INFO:tasks.workunit.client.0.smithi136.stdout:op 1684 completed, throughput=4.99MB/sec 2024-09-13T00:06:49.621 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Klye7XJS67r6M5x off=17029583 len=818067 2024-09-13T00:06:49.623 INFO:tasks.workunit.client.0.smithi136.stdout:op 1685 completed, throughput=4.99MB/sec 2024-09-13T00:06:49.624 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-wymyqb9pmotpRbq off=723928 len=844315 2024-09-13T00:06:49.631 INFO:tasks.workunit.client.0.smithi136.stdout:op 1686 completed, throughput=4.99MB/sec 2024-09-13T00:06:49.631 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-GfLM6prvMGwfvmA off=1934789 len=1846326 2024-09-13T00:06:49.644 INFO:tasks.workunit.client.0.smithi136.stdout:op 1687 completed, throughput=5MB/sec 2024-09-13T00:06:49.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:06:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:06:49] "GET /metrics HTTP/1.1" 200 34123 "" "Prometheus/2.43.0" 2024-09-13T00:06:50.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:06:50 smithi136 bash[75164]: cluster 2024-09-13T00:06:49.310040+0000 mgr.y (mgr.24661) 891 : cluster [DBG] pgmap v1574: 105 pgs: 105 active+clean; 126 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 3.0 MiB/s rd, 1.5 MiB/s wr, 4 op/s 2024-09-13T00:06:50.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:06:50 smithi136 bash[76371]: cluster 2024-09-13T00:06:49.310040+0000 mgr.y (mgr.24661) 891 : cluster [DBG] pgmap v1574: 105 pgs: 105 active+clean; 126 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 3.0 MiB/s rd, 1.5 MiB/s wr, 4 op/s 2024-09-13T00:06:50.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:06:50 smithi181 bash[67587]: cluster 2024-09-13T00:06:49.310040+0000 mgr.y (mgr.24661) 891 : cluster [DBG] pgmap v1574: 105 pgs: 105 active+clean; 126 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 3.0 MiB/s rd, 1.5 MiB/s wr, 4 op/s 2024-09-13T00:06:50.644 INFO:tasks.workunit.client.0.smithi136.stdout: 317: throughput=4.98MB/sec pending data=0 2024-09-13T00:06:50.644 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-rU02zLzs6qODMub off=58324 len=1056281 2024-09-13T00:06:50.647 INFO:tasks.workunit.client.0.smithi136.stdout:op 1688 completed, throughput=4.99MB/sec 2024-09-13T00:06:50.647 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Up6zA52YECPvLZc off=6263196 len=1130108 2024-09-13T00:06:50.650 INFO:tasks.workunit.client.0.smithi136.stdout:op 1689 completed, throughput=4.99MB/sec 2024-09-13T00:06:50.650 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-aEs28_Hid2HXFpG off=1605805 len=1743317 2024-09-13T00:06:50.665 INFO:tasks.workunit.client.0.smithi136.stdout:op 1690 completed, throughput=5MB/sec 2024-09-13T00:06:50.665 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-nLW7XkI_LEjuGiX off=1746102 len=213461 2024-09-13T00:06:50.667 INFO:tasks.workunit.client.0.smithi136.stdout:op 1691 completed, throughput=5MB/sec 2024-09-13T00:06:50.667 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-KPRKQnvPNkMvSOr off=8745867 len=980214 2024-09-13T00:06:50.670 INFO:tasks.workunit.client.0.smithi136.stdout:op 1692 completed, throughput=5MB/sec 2024-09-13T00:06:50.671 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-h5gSn4L3r-GVqTE off=374414 len=171973 2024-09-13T00:06:50.671 INFO:tasks.workunit.client.0.smithi136.stdout:op 1693 completed, throughput=5MB/sec 2024-09-13T00:06:50.672 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-MmXonT9PqHsZt0O off=14642380 len=187268 2024-09-13T00:06:50.676 INFO:tasks.workunit.client.0.smithi136.stdout:op 1694 completed, throughput=5MB/sec 2024-09-13T00:06:51.677 INFO:tasks.workunit.client.0.smithi136.stdout: 318: throughput=4.99MB/sec pending data=0 2024-09-13T00:06:51.677 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-7wqxkcfLlguKs4Q off=14566880 len=1225298 2024-09-13T00:06:51.681 INFO:tasks.workunit.client.0.smithi136.stdout:op 1695 completed, throughput=4.99MB/sec 2024-09-13T00:06:51.681 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-u9wWD4QZgX032w1 off=2910914 len=980746 2024-09-13T00:06:51.689 INFO:tasks.workunit.client.0.smithi136.stdout:op 1696 completed, throughput=4.99MB/sec 2024-09-13T00:06:51.689 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-YNbttY32xlJ7KRl off=11731947 len=1104123 2024-09-13T00:06:51.697 INFO:tasks.workunit.client.0.smithi136.stdout:op 1697 completed, throughput=4.99MB/sec 2024-09-13T00:06:51.697 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-1Nx0DJdxEj0RAVr off=4504861 len=1304545 2024-09-13T00:06:51.710 INFO:tasks.workunit.client.0.smithi136.stdout:op 1698 completed, throughput=5MB/sec 2024-09-13T00:06:51.710 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-HdjOmKP_eq7dDzw off=7408047 len=267778 2024-09-13T00:06:51.715 INFO:tasks.workunit.client.0.smithi136.stdout:op 1699 completed, throughput=5MB/sec 2024-09-13T00:06:51.715 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-qy1tAMtaEoZ6MlP off=7283299 len=793037 2024-09-13T00:06:51.718 INFO:tasks.workunit.client.0.smithi136.stdout:op 1700 completed, throughput=5MB/sec 2024-09-13T00:06:52.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:06:52 smithi181 bash[67587]: cluster 2024-09-13T00:06:51.311009+0000 mgr.y (mgr.24661) 892 : cluster [DBG] pgmap v1575: 105 pgs: 105 active+clean; 126 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.8 MiB/s wr, 4 op/s 2024-09-13T00:06:52.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:06:52 smithi181 bash[95505]: ts=2024-09-13T00:06:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:06:52.718 INFO:tasks.workunit.client.0.smithi136.stdout: 319: throughput=4.99MB/sec pending data=0 2024-09-13T00:06:52.718 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-bWEgusSq5d5Uwnz off=2541634 len=2083288 2024-09-13T00:06:52.726 INFO:tasks.workunit.client.0.smithi136.stdout:op 1701 completed, throughput=4.99MB/sec 2024-09-13T00:06:52.726 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-MmXonT9PqHsZt0O off=15805212 len=619506 2024-09-13T00:06:52.728 INFO:tasks.workunit.client.0.smithi136.stdout:op 1702 completed, throughput=4.99MB/sec 2024-09-13T00:06:52.728 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-9urESG4F9-GOpVk off=11438361 len=19849 2024-09-13T00:06:52.729 INFO:tasks.workunit.client.0.smithi136.stdout:op 1703 completed, throughput=4.99MB/sec 2024-09-13T00:06:52.729 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-GQ0QzMPUnKt6MK8 off=14706693 len=1548755 2024-09-13T00:06:52.732 INFO:tasks.workunit.client.0.smithi136.stdout:op 1704 completed, throughput=5MB/sec 2024-09-13T00:06:52.733 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-xDNoBNv3xjy8-ft off=20714080 len=2094797 2024-09-13T00:06:52.738 INFO:tasks.workunit.client.0.smithi136.stdout:op 1705 completed, throughput=5MB/sec 2024-09-13T00:06:52.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:06:52 smithi136 bash[75164]: cluster 2024-09-13T00:06:51.311009+0000 mgr.y (mgr.24661) 892 : cluster [DBG] pgmap v1575: 105 pgs: 105 active+clean; 126 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.8 MiB/s wr, 4 op/s 2024-09-13T00:06:52.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:06:52 smithi136 bash[76371]: cluster 2024-09-13T00:06:51.311009+0000 mgr.y (mgr.24661) 892 : cluster [DBG] pgmap v1575: 105 pgs: 105 active+clean; 126 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.8 MiB/s wr, 4 op/s 2024-09-13T00:06:53.739 INFO:tasks.workunit.client.0.smithi136.stdout: 320: throughput=4.99MB/sec pending data=0 2024-09-13T00:06:53.739 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-RGsTiUCNhwLUCdf off=4112846 len=715307 2024-09-13T00:06:53.746 INFO:tasks.workunit.client.0.smithi136.stdout:op 1706 completed, throughput=4.99MB/sec 2024-09-13T00:06:53.746 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-uUOBlvLr7hijlwZ off=20840454 len=1301962 2024-09-13T00:06:53.749 INFO:tasks.workunit.client.0.smithi136.stdout:op 1707 completed, throughput=4.99MB/sec 2024-09-13T00:06:53.749 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-7ePsls1HPEb_bbe off=54464 len=384222 2024-09-13T00:06:53.751 INFO:tasks.workunit.client.0.smithi136.stdout:op 1708 completed, throughput=5MB/sec 2024-09-13T00:06:53.751 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-un0r4cpWizgAPkk off=5863631 len=705590 2024-09-13T00:06:53.753 INFO:tasks.workunit.client.0.smithi136.stdout:op 1709 completed, throughput=5MB/sec 2024-09-13T00:06:53.753 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-tQgHOTOHxblMCLB off=11785494 len=2086649 2024-09-13T00:06:53.758 INFO:tasks.workunit.client.0.smithi136.stdout:op 1710 completed, throughput=5MB/sec 2024-09-13T00:06:54.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:06:54 smithi136 bash[75164]: cluster 2024-09-13T00:06:53.311993+0000 mgr.y (mgr.24661) 893 : cluster [DBG] pgmap v1576: 105 pgs: 105 active+clean; 126 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:06:54.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:06:54 smithi136 bash[75164]: audit 2024-09-13T00:06:53.780715+0000 mon.a (mon.0) 2158 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:06:54.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:06:54 smithi136 bash[76371]: cluster 2024-09-13T00:06:53.311993+0000 mgr.y (mgr.24661) 893 : cluster [DBG] pgmap v1576: 105 pgs: 105 active+clean; 126 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:06:54.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:06:54 smithi136 bash[76371]: audit 2024-09-13T00:06:53.780715+0000 mon.a (mon.0) 2158 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:06:54.758 INFO:tasks.workunit.client.0.smithi136.stdout: 321: throughput=4.99MB/sec pending data=0 2024-09-13T00:06:54.759 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-yk6RLcL0d2ad5Qb off=17567649 len=1145360 2024-09-13T00:06:54.762 INFO:tasks.workunit.client.0.smithi136.stdout:op 1711 completed, throughput=4.99MB/sec 2024-09-13T00:06:54.762 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-tu74R9FQzlkW_Jw off=459290 len=1064423 2024-09-13T00:06:54.765 INFO:tasks.workunit.client.0.smithi136.stdout:op 1712 completed, throughput=4.99MB/sec 2024-09-13T00:06:54.765 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-TvEF5ZWebUASGAc off=1308990 len=1553031 2024-09-13T00:06:54.778 INFO:tasks.workunit.client.0.smithi136.stdout:op 1713 completed, throughput=5MB/sec 2024-09-13T00:06:54.778 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-mFEAuFuIW2aTF2t off=2808625 len=331476 2024-09-13T00:06:54.782 INFO:tasks.workunit.client.0.smithi136.stdout:op 1714 completed, throughput=5MB/sec 2024-09-13T00:06:54.783 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Mfl0u6ETqIwYjZU off=10350296 len=860954 2024-09-13T00:06:54.791 INFO:tasks.workunit.client.0.smithi136.stdout:op 1715 completed, throughput=5MB/sec 2024-09-13T00:06:54.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:06:54 smithi181 bash[67587]: cluster 2024-09-13T00:06:53.311993+0000 mgr.y (mgr.24661) 893 : cluster [DBG] pgmap v1576: 105 pgs: 105 active+clean; 126 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:06:54.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:06:54 smithi181 bash[67587]: audit 2024-09-13T00:06:53.780715+0000 mon.a (mon.0) 2158 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:06:55.792 INFO:tasks.workunit.client.0.smithi136.stdout: 322: throughput=4.99MB/sec pending data=0 2024-09-13T00:06:55.792 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-BOvI_1uqaSE8Rfk off=8293689 len=1297074 2024-09-13T00:06:55.796 INFO:tasks.workunit.client.0.smithi136.stdout:op 1716 completed, throughput=4.99MB/sec 2024-09-13T00:06:55.796 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-n8IJnF0wAiCTjlu off=1638436 len=787771 2024-09-13T00:06:55.804 INFO:tasks.workunit.client.0.smithi136.stdout:op 1717 completed, throughput=4.99MB/sec 2024-09-13T00:06:55.804 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-DGzG629Gj8gT7Z7 off=4143293 len=450913 2024-09-13T00:06:55.805 INFO:tasks.workunit.client.0.smithi136.stdout:op 1718 completed, throughput=4.99MB/sec 2024-09-13T00:06:55.806 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Z5c5gGQp1eec-d8 off=2429426 len=1924500 2024-09-13T00:06:55.822 INFO:tasks.workunit.client.0.smithi136.stdout:op 1719 completed, throughput=5MB/sec 2024-09-13T00:06:55.823 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-34BKQ3vvXrAXoYF off=6486538 len=957910 2024-09-13T00:06:55.825 INFO:tasks.workunit.client.0.smithi136.stdout:op 1720 completed, throughput=5MB/sec 2024-09-13T00:06:56.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:06:56 smithi136 bash[75164]: cluster 2024-09-13T00:06:55.312880+0000 mgr.y (mgr.24661) 894 : cluster [DBG] pgmap v1577: 105 pgs: 105 active+clean; 126 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:06:56.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:06:56 smithi136 bash[76371]: cluster 2024-09-13T00:06:55.312880+0000 mgr.y (mgr.24661) 894 : cluster [DBG] pgmap v1577: 105 pgs: 105 active+clean; 126 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:06:56.825 INFO:tasks.workunit.client.0.smithi136.stdout: 323: throughput=4.99MB/sec pending data=0 2024-09-13T00:06:56.826 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ja777X5WBPbv9Pf off=882412 len=938742 2024-09-13T00:06:56.828 INFO:tasks.workunit.client.0.smithi136.stdout:op 1721 completed, throughput=4.99MB/sec 2024-09-13T00:06:56.828 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-AkC9I_fhQdG9jG6 off=9576248 len=1523921 2024-09-13T00:06:56.833 INFO:tasks.workunit.client.0.smithi136.stdout:op 1722 completed, throughput=4.99MB/sec 2024-09-13T00:06:56.833 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-0UVUZFfBgZe8-bc off=2960261 len=1465384 2024-09-13T00:06:56.837 INFO:tasks.workunit.client.0.smithi136.stdout:op 1723 completed, throughput=5MB/sec 2024-09-13T00:06:56.837 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-QOWCIAUGmBXh5ry off=5627908 len=716673 2024-09-13T00:06:56.839 INFO:tasks.workunit.client.0.smithi136.stdout:op 1724 completed, throughput=5MB/sec 2024-09-13T00:06:56.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:06:56 smithi181 bash[67587]: cluster 2024-09-13T00:06:55.312880+0000 mgr.y (mgr.24661) 894 : cluster [DBG] pgmap v1577: 105 pgs: 105 active+clean; 126 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:06:57.839 INFO:tasks.workunit.client.0.smithi136.stdout: 324: throughput=4.99MB/sec pending data=0 2024-09-13T00:06:57.839 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-wn3NJMq-mxfdfo0 off=2042628 len=759579 2024-09-13T00:06:57.846 INFO:tasks.workunit.client.0.smithi136.stdout:op 1725 completed, throughput=4.99MB/sec 2024-09-13T00:06:57.846 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-TWAkvcXJjDSswj9 off=15801936 len=759388 2024-09-13T00:06:57.849 INFO:tasks.workunit.client.0.smithi136.stdout:op 1726 completed, throughput=4.99MB/sec 2024-09-13T00:06:57.849 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ZvVcNtE3UctYO99 off=16318026 len=1715197 2024-09-13T00:06:57.853 INFO:tasks.workunit.client.0.smithi136.stdout:op 1727 completed, throughput=4.99MB/sec 2024-09-13T00:06:57.853 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-4FcZxoDLfbgerIM off=4316909 len=1410048 2024-09-13T00:06:57.863 INFO:tasks.workunit.client.0.smithi136.stdout:op 1728 completed, throughput=5MB/sec 2024-09-13T00:06:57.863 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-sHyujmtXB2DYWcU off=2567785 len=386282 2024-09-13T00:06:57.865 INFO:tasks.workunit.client.0.smithi136.stdout:op 1729 completed, throughput=5MB/sec 2024-09-13T00:06:57.865 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-R4MQ8BWyal05USL off=4444034 len=1451870 2024-09-13T00:06:57.876 INFO:tasks.workunit.client.0.smithi136.stdout:op 1730 completed, throughput=5MB/sec 2024-09-13T00:06:58.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:06:58 smithi136 bash[75164]: cluster 2024-09-13T00:06:57.313974+0000 mgr.y (mgr.24661) 895 : cluster [DBG] pgmap v1578: 105 pgs: 105 active+clean; 126 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 3.3 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:06:58.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:06:58 smithi136 bash[76371]: cluster 2024-09-13T00:06:57.313974+0000 mgr.y (mgr.24661) 895 : cluster [DBG] pgmap v1578: 105 pgs: 105 active+clean; 126 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 3.3 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:06:58.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:06:58 smithi181 bash[67587]: cluster 2024-09-13T00:06:57.313974+0000 mgr.y (mgr.24661) 895 : cluster [DBG] pgmap v1578: 105 pgs: 105 active+clean; 126 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 3.3 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:06:58.876 INFO:tasks.workunit.client.0.smithi136.stdout: 325: throughput=4.99MB/sec pending data=0 2024-09-13T00:06:58.877 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-EGosW2Jxa5sxppc off=9885192 len=370055 2024-09-13T00:06:58.878 INFO:tasks.workunit.client.0.smithi136.stdout:op 1731 completed, throughput=4.99MB/sec 2024-09-13T00:06:58.878 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-LyRunD-y_a66cCT off=3202506 len=1200099 2024-09-13T00:06:58.882 INFO:tasks.workunit.client.0.smithi136.stdout:op 1732 completed, throughput=4.99MB/sec 2024-09-13T00:06:58.883 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-hzq11RHsI-oN8t- off=514659 len=313134 2024-09-13T00:06:58.885 INFO:tasks.workunit.client.0.smithi136.stdout:op 1733 completed, throughput=4.99MB/sec 2024-09-13T00:06:58.885 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-abeLfmf4wkOFv6t off=1643691 len=536584 2024-09-13T00:06:58.887 INFO:tasks.workunit.client.0.smithi136.stdout:op 1734 completed, throughput=5MB/sec 2024-09-13T00:06:58.887 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-IjG9iVxSvL-t3g8 off=0 len=1738888 2024-09-13T00:06:58.898 INFO:tasks.workunit.client.0.smithi136.stdout:op 1735 completed, throughput=5MB/sec 2024-09-13T00:06:58.899 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-xlPTVYY3TgLqLt7 off=5322123 len=739149 2024-09-13T00:06:58.900 INFO:tasks.workunit.client.0.smithi136.stdout:op 1736 completed, throughput=5MB/sec 2024-09-13T00:06:59.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:06:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:06:59] "GET /metrics HTTP/1.1" 200 34135 "" "Prometheus/2.43.0" 2024-09-13T00:06:59.901 INFO:tasks.workunit.client.0.smithi136.stdout: 326: throughput=4.99MB/sec pending data=0 2024-09-13T00:06:59.901 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-dQRsoHNoEcjcZgJ off=1883853 len=1456344 2024-09-13T00:06:59.913 INFO:tasks.workunit.client.0.smithi136.stdout:op 1737 completed, throughput=4.99MB/sec 2024-09-13T00:06:59.913 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Am2jrlGqoJbujJg off=3268097 len=67187 2024-09-13T00:06:59.917 INFO:tasks.workunit.client.0.smithi136.stdout:op 1738 completed, throughput=4.99MB/sec 2024-09-13T00:06:59.917 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-TGms_zIPLEtfX7I off=13218234 len=1940646 2024-09-13T00:06:59.921 INFO:tasks.workunit.client.0.smithi136.stdout:op 1739 completed, throughput=5MB/sec 2024-09-13T00:06:59.922 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-bZ4PKpFFCPSFi5r off=3236416 len=1244721 2024-09-13T00:06:59.934 INFO:tasks.workunit.client.0.smithi136.stdout:op 1740 completed, throughput=5MB/sec 2024-09-13T00:06:59.934 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-s6NFAXfvWQn8YCY off=1611123 len=239111 2024-09-13T00:06:59.935 INFO:tasks.workunit.client.0.smithi136.stdout:op 1741 completed, throughput=5MB/sec 2024-09-13T00:07:00.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:07:00 smithi136 bash[75164]: cluster 2024-09-13T00:06:59.314870+0000 mgr.y (mgr.24661) 896 : cluster [DBG] pgmap v1579: 105 pgs: 105 active+clean; 126 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:07:00.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:07:00 smithi136 bash[76371]: cluster 2024-09-13T00:06:59.314870+0000 mgr.y (mgr.24661) 896 : cluster [DBG] pgmap v1579: 105 pgs: 105 active+clean; 126 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:07:00.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:07:00 smithi181 bash[67587]: cluster 2024-09-13T00:06:59.314870+0000 mgr.y (mgr.24661) 896 : cluster [DBG] pgmap v1579: 105 pgs: 105 active+clean; 126 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:07:00.935 INFO:tasks.workunit.client.0.smithi136.stdout: 327: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:00.935 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-w4O-BhPyrGZfZPb off=4863866 len=353513 2024-09-13T00:07:00.941 INFO:tasks.workunit.client.0.smithi136.stdout:op 1742 completed, throughput=4.99MB/sec 2024-09-13T00:07:00.941 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-_2V-NjnrwSCblZP off=13665099 len=318903 2024-09-13T00:07:00.943 INFO:tasks.workunit.client.0.smithi136.stdout:op 1743 completed, throughput=4.99MB/sec 2024-09-13T00:07:00.943 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-3rRRl7uHtAuaHzC off=1890145 len=874109 2024-09-13T00:07:00.946 INFO:tasks.workunit.client.0.smithi136.stdout:op 1744 completed, throughput=4.99MB/sec 2024-09-13T00:07:00.946 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-F1gHhPc3kFBC_ef off=10658247 len=182418 2024-09-13T00:07:00.947 INFO:tasks.workunit.client.0.smithi136.stdout:op 1745 completed, throughput=4.99MB/sec 2024-09-13T00:07:00.947 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-POLsLV8H5JHL6EB off=4822241 len=624336 2024-09-13T00:07:00.949 INFO:tasks.workunit.client.0.smithi136.stdout:op 1746 completed, throughput=4.99MB/sec 2024-09-13T00:07:00.949 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-krvFyMAR3_u-g4s off=1675414 len=1349985 2024-09-13T00:07:00.960 INFO:tasks.workunit.client.0.smithi136.stdout:op 1747 completed, throughput=5MB/sec 2024-09-13T00:07:00.961 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-SaMKMmSDDXEp-Y_ off=20159586 len=1452048 2024-09-13T00:07:00.973 INFO:tasks.workunit.client.0.smithi136.stdout:op 1748 completed, throughput=5MB/sec 2024-09-13T00:07:00.973 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-9jroEgGbBIXbTs5 off=6411751 len=359740 2024-09-13T00:07:00.980 INFO:tasks.workunit.client.0.smithi136.stdout:op 1749 completed, throughput=5MB/sec 2024-09-13T00:07:01.980 INFO:tasks.workunit.client.0.smithi136.stdout: 328: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:01.980 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Ut00C2ephc7KxJj off=8882347 len=1127574 2024-09-13T00:07:01.983 INFO:tasks.workunit.client.0.smithi136.stdout:op 1750 completed, throughput=4.99MB/sec 2024-09-13T00:07:01.983 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-XOlKV_rFPE_Qz16 off=7298146 len=867950 2024-09-13T00:07:01.986 INFO:tasks.workunit.client.0.smithi136.stdout:op 1751 completed, throughput=4.99MB/sec 2024-09-13T00:07:01.986 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-1VG3jaerfJ77-ME off=4412735 len=1529148 2024-09-13T00:07:01.997 INFO:tasks.workunit.client.0.smithi136.stdout:op 1752 completed, throughput=5MB/sec 2024-09-13T00:07:01.997 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-SRRk94Xxhze-0t- off=1673573 len=573990 2024-09-13T00:07:02.000 INFO:tasks.workunit.client.0.smithi136.stdout:op 1753 completed, throughput=5MB/sec 2024-09-13T00:07:02.000 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-nL4KmnrCtzToFH9 off=17242737 len=482768 2024-09-13T00:07:02.006 INFO:tasks.workunit.client.0.smithi136.stdout:op 1754 completed, throughput=5MB/sec 2024-09-13T00:07:02.006 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-mPTTrTMNxnLeQH7 off=6445129 len=129639 2024-09-13T00:07:02.011 INFO:tasks.workunit.client.0.smithi136.stdout:op 1755 completed, throughput=5MB/sec 2024-09-13T00:07:02.011 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-IACk12bUSp5Bec9 off=3878827 len=1388737 2024-09-13T00:07:02.022 INFO:tasks.workunit.client.0.smithi136.stdout:op 1756 completed, throughput=5MB/sec 2024-09-13T00:07:02.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:07:02 smithi181 bash[67587]: cluster 2024-09-13T00:07:01.315849+0000 mgr.y (mgr.24661) 897 : cluster [DBG] pgmap v1580: 105 pgs: 105 active+clean; 126 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:07:02.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:07:02 smithi181 bash[95505]: ts=2024-09-13T00:07:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:07:02.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:07:02 smithi136 bash[75164]: cluster 2024-09-13T00:07:01.315849+0000 mgr.y (mgr.24661) 897 : cluster [DBG] pgmap v1580: 105 pgs: 105 active+clean; 126 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:07:02.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:07:02 smithi136 bash[76371]: cluster 2024-09-13T00:07:01.315849+0000 mgr.y (mgr.24661) 897 : cluster [DBG] pgmap v1580: 105 pgs: 105 active+clean; 126 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:07:03.023 INFO:tasks.workunit.client.0.smithi136.stdout: 329: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:03.023 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-68zKyfnfwMft5qL off=7659550 len=1679839 2024-09-13T00:07:03.026 INFO:tasks.workunit.client.0.smithi136.stdout:op 1757 completed, throughput=4.99MB/sec 2024-09-13T00:07:03.027 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-G9aq3aGbkE05nPI off=10955354 len=1078950 2024-09-13T00:07:03.030 INFO:tasks.workunit.client.0.smithi136.stdout:op 1758 completed, throughput=5MB/sec 2024-09-13T00:07:03.030 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-BFKTyyd-JnHgNRW off=3700930 len=1443124 2024-09-13T00:07:03.034 INFO:tasks.workunit.client.0.smithi136.stdout:op 1759 completed, throughput=5MB/sec 2024-09-13T00:07:03.034 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-qvuvxF54xl-hChR off=3076935 len=1274687 2024-09-13T00:07:03.043 INFO:tasks.workunit.client.0.smithi136.stdout:op 1760 completed, throughput=5MB/sec 2024-09-13T00:07:04.043 INFO:tasks.workunit.client.0.smithi136.stdout: 330: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:04.043 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-NfgfEcsu1AitXgQ off=4899927 len=646864 2024-09-13T00:07:04.050 INFO:tasks.workunit.client.0.smithi136.stdout:op 1761 completed, throughput=4.99MB/sec 2024-09-13T00:07:04.050 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ni8BW4acphOfm8m off=3947799 len=1344634 2024-09-13T00:07:04.055 INFO:tasks.workunit.client.0.smithi136.stdout:op 1762 completed, throughput=4.99MB/sec 2024-09-13T00:07:04.055 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-tWO2w13xpEFzTZ- off=1911908 len=1047179 2024-09-13T00:07:04.063 INFO:tasks.workunit.client.0.smithi136.stdout:op 1763 completed, throughput=5MB/sec 2024-09-13T00:07:04.063 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_DvGXxkuTlw0GlJ off=113542 len=1197026 2024-09-13T00:07:04.073 INFO:tasks.workunit.client.0.smithi136.stdout:op 1764 completed, throughput=5MB/sec 2024-09-13T00:07:04.073 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-LcKsP_Lqxmak8_U off=5734473 len=55269 2024-09-13T00:07:04.076 INFO:tasks.workunit.client.0.smithi136.stdout:op 1765 completed, throughput=5MB/sec 2024-09-13T00:07:04.076 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj--1cHsOsuNIfILth off=53998 len=670336 2024-09-13T00:07:04.077 INFO:tasks.workunit.client.0.smithi136.stdout:op 1766 completed, throughput=5MB/sec 2024-09-13T00:07:04.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:07:04 smithi136 bash[75164]: cluster 2024-09-13T00:07:03.316850+0000 mgr.y (mgr.24661) 898 : cluster [DBG] pgmap v1581: 105 pgs: 105 active+clean; 126 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.2 MiB/s wr, 5 op/s 2024-09-13T00:07:04.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:07:04 smithi136 bash[76371]: cluster 2024-09-13T00:07:03.316850+0000 mgr.y (mgr.24661) 898 : cluster [DBG] pgmap v1581: 105 pgs: 105 active+clean; 126 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.2 MiB/s wr, 5 op/s 2024-09-13T00:07:04.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:07:04 smithi181 bash[67587]: cluster 2024-09-13T00:07:03.316850+0000 mgr.y (mgr.24661) 898 : cluster [DBG] pgmap v1581: 105 pgs: 105 active+clean; 126 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.2 MiB/s wr, 5 op/s 2024-09-13T00:07:05.077 INFO:tasks.workunit.client.0.smithi136.stdout: 331: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:05.077 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-52Q4hqN2yjiyeiB off=2525929 len=640290 2024-09-13T00:07:05.080 INFO:tasks.workunit.client.0.smithi136.stdout:op 1767 completed, throughput=4.99MB/sec 2024-09-13T00:07:05.080 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-KDYAla7sp2zaoqu off=3417232 len=360049 2024-09-13T00:07:05.085 INFO:tasks.workunit.client.0.smithi136.stdout:op 1768 completed, throughput=4.99MB/sec 2024-09-13T00:07:05.085 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-u7xE4SJ2CItSh82 off=1501448 len=1172707 2024-09-13T00:07:05.088 INFO:tasks.workunit.client.0.smithi136.stdout:op 1769 completed, throughput=4.99MB/sec 2024-09-13T00:07:05.088 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-5lx5XcNTpNaTPGJ off=12844860 len=1548348 2024-09-13T00:07:05.099 INFO:tasks.workunit.client.0.smithi136.stdout:op 1770 completed, throughput=5MB/sec 2024-09-13T00:07:05.099 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-S9MyWz7kB4WkIu8 off=2028278 len=410316 2024-09-13T00:07:05.101 INFO:tasks.workunit.client.0.smithi136.stdout:op 1771 completed, throughput=5MB/sec 2024-09-13T00:07:05.101 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ZcgF6WqR7SSf7vu off=2018661 len=40355 2024-09-13T00:07:05.105 INFO:tasks.workunit.client.0.smithi136.stdout:op 1772 completed, throughput=5MB/sec 2024-09-13T00:07:05.105 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-3ebWuNEUBhZeLuu off=3868817 len=464522 2024-09-13T00:07:05.107 INFO:tasks.workunit.client.0.smithi136.stdout:op 1773 completed, throughput=5MB/sec 2024-09-13T00:07:05.107 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-DOB1FQlzaNErSQb off=8367914 len=973388 2024-09-13T00:07:05.109 INFO:tasks.workunit.client.0.smithi136.stdout:op 1774 completed, throughput=5MB/sec 2024-09-13T00:07:06.110 INFO:tasks.workunit.client.0.smithi136.stdout: 332: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:06.110 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-NJabSpL2rlp9pAO off=660120 len=1765522 2024-09-13T00:07:06.121 INFO:tasks.workunit.client.0.smithi136.stdout:op 1775 completed, throughput=4.99MB/sec 2024-09-13T00:07:06.121 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-wSvcaBi7mSI7_aG off=11958513 len=955161 2024-09-13T00:07:06.123 INFO:tasks.workunit.client.0.smithi136.stdout:op 1776 completed, throughput=4.99MB/sec 2024-09-13T00:07:06.123 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-t4O6V3smWB6RSdH off=11769232 len=513730 2024-09-13T00:07:06.130 INFO:tasks.workunit.client.0.smithi136.stdout:op 1777 completed, throughput=5MB/sec 2024-09-13T00:07:06.130 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-oG1-1PHaDC4Ndm9 off=6909150 len=1621999 2024-09-13T00:07:06.143 INFO:tasks.workunit.client.0.smithi136.stdout:op 1778 completed, throughput=5MB/sec 2024-09-13T00:07:06.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:07:06 smithi136 bash[75164]: cluster 2024-09-13T00:07:05.317708+0000 mgr.y (mgr.24661) 899 : cluster [DBG] pgmap v1582: 105 pgs: 105 active+clean; 126 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.0 MiB/s wr, 5 op/s 2024-09-13T00:07:06.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:07:06 smithi136 bash[76371]: cluster 2024-09-13T00:07:05.317708+0000 mgr.y (mgr.24661) 899 : cluster [DBG] pgmap v1582: 105 pgs: 105 active+clean; 126 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.0 MiB/s wr, 5 op/s 2024-09-13T00:07:06.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:07:06 smithi181 bash[67587]: cluster 2024-09-13T00:07:05.317708+0000 mgr.y (mgr.24661) 899 : cluster [DBG] pgmap v1582: 105 pgs: 105 active+clean; 126 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.0 MiB/s wr, 5 op/s 2024-09-13T00:07:07.143 INFO:tasks.workunit.client.0.smithi136.stdout: 333: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:07.143 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-MvzV1hNfOAjAYOr off=2115418 len=1881866 2024-09-13T00:07:07.159 INFO:tasks.workunit.client.0.smithi136.stdout:op 1779 completed, throughput=4.99MB/sec 2024-09-13T00:07:07.159 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-bgzndf_dYOrCRK7 off=0 len=1333854 2024-09-13T00:07:07.169 INFO:tasks.workunit.client.0.smithi136.stdout:op 1780 completed, throughput=4.99MB/sec 2024-09-13T00:07:07.169 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-yKHN6go1y-W7Wqy off=4516335 len=560606 2024-09-13T00:07:07.171 INFO:tasks.workunit.client.0.smithi136.stdout:op 1781 completed, throughput=5MB/sec 2024-09-13T00:07:07.171 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-_GX8JnL9XNoK-8R off=698866 len=1540316 2024-09-13T00:07:07.175 INFO:tasks.workunit.client.0.smithi136.stdout:op 1782 completed, throughput=5MB/sec 2024-09-13T00:07:08.175 INFO:tasks.workunit.client.0.smithi136.stdout: 334: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:08.175 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-0pkyY-tUYojv-A- off=4865065 len=336597 2024-09-13T00:07:08.181 INFO:tasks.workunit.client.0.smithi136.stdout:op 1783 completed, throughput=4.99MB/sec 2024-09-13T00:07:08.181 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-wYG23C6Go7mfa5X off=5801331 len=1744962 2024-09-13T00:07:08.186 INFO:tasks.workunit.client.0.smithi136.stdout:op 1784 completed, throughput=4.99MB/sec 2024-09-13T00:07:08.186 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-bo_6Hq27blLJpAA off=7524545 len=963772 2024-09-13T00:07:08.189 INFO:tasks.workunit.client.0.smithi136.stdout:op 1785 completed, throughput=4.99MB/sec 2024-09-13T00:07:08.189 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-D1qGT_072CwGuJj off=14978770 len=1751164 2024-09-13T00:07:08.204 INFO:tasks.workunit.client.0.smithi136.stdout:op 1786 completed, throughput=5MB/sec 2024-09-13T00:07:08.204 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj--GS-xS5hh9SkIak off=11970371 len=1251299 2024-09-13T00:07:08.209 INFO:tasks.workunit.client.0.smithi136.stdout:op 1787 completed, throughput=5MB/sec 2024-09-13T00:07:08.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:07:08 smithi136 bash[75164]: cluster 2024-09-13T00:07:07.318825+0000 mgr.y (mgr.24661) 900 : cluster [DBG] pgmap v1583: 105 pgs: 105 active+clean; 126 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:07:08.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:07:08 smithi136 bash[76371]: cluster 2024-09-13T00:07:07.318825+0000 mgr.y (mgr.24661) 900 : cluster [DBG] pgmap v1583: 105 pgs: 105 active+clean; 126 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:07:08.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:07:08 smithi181 bash[67587]: cluster 2024-09-13T00:07:07.318825+0000 mgr.y (mgr.24661) 900 : cluster [DBG] pgmap v1583: 105 pgs: 105 active+clean; 126 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:07:09.209 INFO:tasks.workunit.client.0.smithi136.stdout: 335: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:09.209 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ccqMCc8KS72Lpw1 off=8426302 len=1620827 2024-09-13T00:07:09.213 INFO:tasks.workunit.client.0.smithi136.stdout:op 1788 completed, throughput=4.99MB/sec 2024-09-13T00:07:09.213 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-kh9tt5UgefcVGi8 off=331764 len=1680371 2024-09-13T00:07:09.228 INFO:tasks.workunit.client.0.smithi136.stdout:op 1789 completed, throughput=5MB/sec 2024-09-13T00:07:09.228 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-SOp0o8wH9fGICJn off=7994969 len=537040 2024-09-13T00:07:09.235 INFO:tasks.workunit.client.0.smithi136.stdout:op 1790 completed, throughput=5MB/sec 2024-09-13T00:07:09.235 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-btlL_rEmAhWNem3 off=5470960 len=1249718 2024-09-13T00:07:09.238 INFO:tasks.workunit.client.0.smithi136.stdout:op 1791 completed, throughput=5MB/sec 2024-09-13T00:07:09.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:07:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:07:09] "GET /metrics HTTP/1.1" 200 34135 "" "Prometheus/2.43.0" 2024-09-13T00:07:09.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:07:09 smithi136 bash[76371]: audit 2024-09-13T00:07:08.781023+0000 mon.a (mon.0) 2159 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:07:09.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:07:09 smithi136 bash[75164]: audit 2024-09-13T00:07:08.781023+0000 mon.a (mon.0) 2159 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:07:09.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:07:09 smithi181 bash[67587]: audit 2024-09-13T00:07:08.781023+0000 mon.a (mon.0) 2159 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:07:10.239 INFO:tasks.workunit.client.0.smithi136.stdout: 336: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:10.239 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-yrlmVeLlIW0FSu3 off=9802868 len=1164532 2024-09-13T00:07:10.248 INFO:tasks.workunit.client.0.smithi136.stdout:op 1792 completed, throughput=4.99MB/sec 2024-09-13T00:07:10.248 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-gp_fF9Pykb48LXM off=22366238 len=2053620 2024-09-13T00:07:10.261 INFO:tasks.workunit.client.0.smithi136.stdout:op 1793 completed, throughput=5MB/sec 2024-09-13T00:07:10.261 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-KnbHSgwK4q_fe1u off=1966460 len=1942713 2024-09-13T00:07:10.265 INFO:tasks.workunit.client.0.smithi136.stdout:op 1794 completed, throughput=5MB/sec 2024-09-13T00:07:10.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:07:10 smithi136 bash[75164]: cluster 2024-09-13T00:07:09.319728+0000 mgr.y (mgr.24661) 901 : cluster [DBG] pgmap v1584: 105 pgs: 105 active+clean; 126 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:07:10.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:07:10 smithi136 bash[76371]: cluster 2024-09-13T00:07:09.319728+0000 mgr.y (mgr.24661) 901 : cluster [DBG] pgmap v1584: 105 pgs: 105 active+clean; 126 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:07:10.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:07:10 smithi181 bash[67587]: cluster 2024-09-13T00:07:09.319728+0000 mgr.y (mgr.24661) 901 : cluster [DBG] pgmap v1584: 105 pgs: 105 active+clean; 126 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:07:11.265 INFO:tasks.workunit.client.0.smithi136.stdout: 337: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:11.266 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-uEzkzP08HI45unF off=1007310 len=358425 2024-09-13T00:07:11.272 INFO:tasks.workunit.client.0.smithi136.stdout:op 1795 completed, throughput=4.99MB/sec 2024-09-13T00:07:11.272 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-i288335ELi1uV2Y off=10861295 len=1355681 2024-09-13T00:07:11.275 INFO:tasks.workunit.client.0.smithi136.stdout:op 1796 completed, throughput=4.99MB/sec 2024-09-13T00:07:11.275 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-dVdyENO1K0cjZSF off=7622673 len=1073192 2024-09-13T00:07:11.284 INFO:tasks.workunit.client.0.smithi136.stdout:op 1797 completed, throughput=4.99MB/sec 2024-09-13T00:07:11.284 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-OCfnIrZr16cvPKx off=12202807 len=593818 2024-09-13T00:07:11.291 INFO:tasks.workunit.client.0.smithi136.stdout:op 1798 completed, throughput=4.99MB/sec 2024-09-13T00:07:11.291 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-LxUltW5oyBlxzHC off=1496701 len=68327 2024-09-13T00:07:11.295 INFO:tasks.workunit.client.0.smithi136.stdout:op 1799 completed, throughput=5MB/sec 2024-09-13T00:07:11.296 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-qn9LNplQCguKRfN off=20379048 len=1182172 2024-09-13T00:07:11.299 INFO:tasks.workunit.client.0.smithi136.stdout:op 1800 completed, throughput=5MB/sec 2024-09-13T00:07:11.299 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-vdgPv980Iosgufm off=8408182 len=1714169 2024-09-13T00:07:11.303 INFO:tasks.workunit.client.0.smithi136.stdout:op 1801 completed, throughput=5MB/sec 2024-09-13T00:07:12.303 INFO:tasks.workunit.client.0.smithi136.stdout: 338: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:12.303 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-sH8nBszGccsTYSD off=466724 len=672240 2024-09-13T00:07:12.311 INFO:tasks.workunit.client.0.smithi136.stdout:op 1802 completed, throughput=4.99MB/sec 2024-09-13T00:07:12.311 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-CdrDuE8gS5Y3AHm off=4889941 len=1215725 2024-09-13T00:07:12.314 INFO:tasks.workunit.client.0.smithi136.stdout:op 1803 completed, throughput=4.99MB/sec 2024-09-13T00:07:12.314 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-eBWHzZhv_1UN0vh off=10862614 len=1982713 2024-09-13T00:07:12.329 INFO:tasks.workunit.client.0.smithi136.stdout:op 1804 completed, throughput=5MB/sec 2024-09-13T00:07:12.329 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Jnkh15zk684wD58 off=2683075 len=671498 2024-09-13T00:07:12.338 INFO:tasks.workunit.client.0.smithi136.stdout:op 1805 completed, throughput=5MB/sec 2024-09-13T00:07:12.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:07:12 smithi181 bash[67587]: cluster 2024-09-13T00:07:11.320816+0000 mgr.y (mgr.24661) 902 : cluster [DBG] pgmap v1585: 105 pgs: 105 active+clean; 126 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:07:12.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:07:12 smithi181 bash[95505]: ts=2024-09-13T00:07:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:07:12.757 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:07:12 smithi136 bash[76371]: cluster 2024-09-13T00:07:11.320816+0000 mgr.y (mgr.24661) 902 : cluster [DBG] pgmap v1585: 105 pgs: 105 active+clean; 126 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:07:12.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:07:12 smithi136 bash[75164]: cluster 2024-09-13T00:07:11.320816+0000 mgr.y (mgr.24661) 902 : cluster [DBG] pgmap v1585: 105 pgs: 105 active+clean; 126 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:07:13.338 INFO:tasks.workunit.client.0.smithi136.stdout: 339: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:13.338 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-jtWlyPZ5pVkKHVt off=1463701 len=203051 2024-09-13T00:07:13.343 INFO:tasks.workunit.client.0.smithi136.stdout:op 1806 completed, throughput=4.99MB/sec 2024-09-13T00:07:13.344 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-8RBYzGrOXmejV48 off=14304587 len=19459 2024-09-13T00:07:13.347 INFO:tasks.workunit.client.0.smithi136.stdout:op 1807 completed, throughput=4.99MB/sec 2024-09-13T00:07:13.347 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-O5lSJcEHqB7OouO off=5533543 len=37615 2024-09-13T00:07:13.348 INFO:tasks.workunit.client.0.smithi136.stdout:op 1808 completed, throughput=4.99MB/sec 2024-09-13T00:07:13.348 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-leWwEhOI4vY5Ipi off=24205118 len=1149730 2024-09-13T00:07:13.356 INFO:tasks.workunit.client.0.smithi136.stdout:op 1809 completed, throughput=4.99MB/sec 2024-09-13T00:07:13.357 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-XgS2w5whA9G6rOf off=177291 len=1854007 2024-09-13T00:07:13.371 INFO:tasks.workunit.client.0.smithi136.stdout:op 1810 completed, throughput=4.99MB/sec 2024-09-13T00:07:13.371 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-bLNcjfnNGZyKCjb off=4428741 len=1376288 2024-09-13T00:07:13.375 INFO:tasks.workunit.client.0.smithi136.stdout:op 1811 completed, throughput=5MB/sec 2024-09-13T00:07:13.375 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-c7m-fq6LvcGE8v7 off=9850260 len=888183 2024-09-13T00:07:13.384 INFO:tasks.workunit.client.0.smithi136.stdout:op 1812 completed, throughput=5MB/sec 2024-09-13T00:07:14.384 INFO:tasks.workunit.client.0.smithi136.stdout: 340: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:14.384 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-V2cK7Jr6EkSEw9Z off=12410171 len=1130847 2024-09-13T00:07:14.395 INFO:tasks.workunit.client.0.smithi136.stdout:op 1813 completed, throughput=4.99MB/sec 2024-09-13T00:07:14.395 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-32_ShXdKVik5Coy off=10710488 len=193749 2024-09-13T00:07:14.397 INFO:tasks.workunit.client.0.smithi136.stdout:op 1814 completed, throughput=4.99MB/sec 2024-09-13T00:07:14.397 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-cVUUbmcuuYyzoqQ off=5898836 len=733394 2024-09-13T00:07:14.399 INFO:tasks.workunit.client.0.smithi136.stdout:op 1815 completed, throughput=4.99MB/sec 2024-09-13T00:07:14.399 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-dLoVxWI13uJLam9 off=4652406 len=328596 2024-09-13T00:07:14.400 INFO:tasks.workunit.client.0.smithi136.stdout:op 1816 completed, throughput=4.99MB/sec 2024-09-13T00:07:14.401 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-tLVrUh8HLHslAg4 off=1727589 len=250972 2024-09-13T00:07:14.405 INFO:tasks.workunit.client.0.smithi136.stdout:op 1817 completed, throughput=4.99MB/sec 2024-09-13T00:07:14.406 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Ut5tatRHQ6bxlyX off=7484113 len=809915 2024-09-13T00:07:14.414 INFO:tasks.workunit.client.0.smithi136.stdout:op 1818 completed, throughput=5MB/sec 2024-09-13T00:07:14.414 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-fU5gDyGwYGVnNPr off=3051215 len=1005777 2024-09-13T00:07:14.423 INFO:tasks.workunit.client.0.smithi136.stdout:op 1819 completed, throughput=5MB/sec 2024-09-13T00:07:14.423 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ZFkBekF74Ampn87 off=67118 len=1479774 2024-09-13T00:07:14.433 INFO:tasks.workunit.client.0.smithi136.stdout:op 1820 completed, throughput=5MB/sec 2024-09-13T00:07:14.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:07:14 smithi136 bash[75164]: cluster 2024-09-13T00:07:13.321828+0000 mgr.y (mgr.24661) 903 : cluster [DBG] pgmap v1586: 105 pgs: 105 active+clean; 126 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:07:14.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:07:14 smithi136 bash[76371]: cluster 2024-09-13T00:07:13.321828+0000 mgr.y (mgr.24661) 903 : cluster [DBG] pgmap v1586: 105 pgs: 105 active+clean; 126 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:07:14.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:07:14 smithi181 bash[67587]: cluster 2024-09-13T00:07:13.321828+0000 mgr.y (mgr.24661) 903 : cluster [DBG] pgmap v1586: 105 pgs: 105 active+clean; 126 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:07:15.433 INFO:tasks.workunit.client.0.smithi136.stdout: 341: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:15.434 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ew85MWJj07gC6Km off=1243431 len=1257846 2024-09-13T00:07:15.437 INFO:tasks.workunit.client.0.smithi136.stdout:op 1821 completed, throughput=4.99MB/sec 2024-09-13T00:07:15.437 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-f82d6Wn1Gh_dRJh off=8023637 len=345285 2024-09-13T00:07:15.439 INFO:tasks.workunit.client.0.smithi136.stdout:op 1822 completed, throughput=4.99MB/sec 2024-09-13T00:07:15.439 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-V1U-EP-d6zgqn8r off=132684 len=1013109 2024-09-13T00:07:15.448 INFO:tasks.workunit.client.0.smithi136.stdout:op 1823 completed, throughput=4.99MB/sec 2024-09-13T00:07:15.448 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_f2qoumnPV6cVx7 off=12965597 len=1626083 2024-09-13T00:07:15.459 INFO:tasks.workunit.client.0.smithi136.stdout:op 1824 completed, throughput=5MB/sec 2024-09-13T00:07:15.459 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-KMfnmDd1Cu0guk9 off=15305353 len=1652552 2024-09-13T00:07:15.465 INFO:tasks.workunit.client.0.smithi136.stdout:op 1825 completed, throughput=5MB/sec 2024-09-13T00:07:16.465 INFO:tasks.workunit.client.0.smithi136.stdout: 342: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:16.466 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-_SP5SwKjKDmo0rx off=2221170 len=2086665 2024-09-13T00:07:16.470 INFO:tasks.workunit.client.0.smithi136.stdout:op 1826 completed, throughput=4.99MB/sec 2024-09-13T00:07:16.470 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-sMujDMsbne2jbep off=6440339 len=62968 2024-09-13T00:07:16.471 INFO:tasks.workunit.client.0.smithi136.stdout:op 1827 completed, throughput=4.99MB/sec 2024-09-13T00:07:16.471 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-zxR30OLf_E2PbrV off=1611334 len=854133 2024-09-13T00:07:16.474 INFO:tasks.workunit.client.0.smithi136.stdout:op 1828 completed, throughput=5MB/sec 2024-09-13T00:07:16.474 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Pog99LB38QoUizC off=14728411 len=417141 2024-09-13T00:07:16.475 INFO:tasks.workunit.client.0.smithi136.stdout:op 1829 completed, throughput=5MB/sec 2024-09-13T00:07:16.476 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-pgvBhwxqDMagL9W off=18809132 len=842010 2024-09-13T00:07:16.482 INFO:tasks.workunit.client.0.smithi136.stdout:op 1830 completed, throughput=5MB/sec 2024-09-13T00:07:16.757 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:07:16 smithi136 bash[75164]: cluster 2024-09-13T00:07:15.322708+0000 mgr.y (mgr.24661) 904 : cluster [DBG] pgmap v1587: 105 pgs: 105 active+clean; 126 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:07:16.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:07:16 smithi136 bash[76371]: cluster 2024-09-13T00:07:15.322708+0000 mgr.y (mgr.24661) 904 : cluster [DBG] pgmap v1587: 105 pgs: 105 active+clean; 126 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:07:16.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:07:16 smithi181 bash[67587]: cluster 2024-09-13T00:07:15.322708+0000 mgr.y (mgr.24661) 904 : cluster [DBG] pgmap v1587: 105 pgs: 105 active+clean; 126 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:07:17.483 INFO:tasks.workunit.client.0.smithi136.stdout: 343: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:17.483 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-kuYCRRQw9M7ZHHa off=2024093 len=2039722 2024-09-13T00:07:17.487 INFO:tasks.workunit.client.0.smithi136.stdout:op 1831 completed, throughput=4.99MB/sec 2024-09-13T00:07:17.487 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-oFYxhgAC1YYLcbs off=15441940 len=514700 2024-09-13T00:07:17.494 INFO:tasks.workunit.client.0.smithi136.stdout:op 1832 completed, throughput=4.99MB/sec 2024-09-13T00:07:17.494 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-b3fOTgFHMUE05SG off=10310247 len=1738021 2024-09-13T00:07:17.498 INFO:tasks.workunit.client.0.smithi136.stdout:op 1833 completed, throughput=5MB/sec 2024-09-13T00:07:17.498 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-7bwaoI79y2UH0Yu off=465815 len=240275 2024-09-13T00:07:17.504 INFO:tasks.workunit.client.0.smithi136.stdout:op 1834 completed, throughput=5MB/sec 2024-09-13T00:07:17.504 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-tD5oSS_Ayc8zjOz off=8096554 len=1464899 2024-09-13T00:07:17.507 INFO:tasks.workunit.client.0.smithi136.stdout:op 1835 completed, throughput=5MB/sec 2024-09-13T00:07:18.508 INFO:tasks.workunit.client.0.smithi136.stdout: 344: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:18.508 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-jHKp-zbgy7_S0Oa off=2089798 len=2023196 2024-09-13T00:07:18.522 INFO:tasks.workunit.client.0.smithi136.stdout:op 1836 completed, throughput=4.99MB/sec 2024-09-13T00:07:18.522 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-7EjLGfgweN6yUjx off=9715781 len=506720 2024-09-13T00:07:18.527 INFO:tasks.workunit.client.0.smithi136.stdout:op 1837 completed, throughput=4.99MB/sec 2024-09-13T00:07:18.528 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ijzAGhM4i-HJesb off=889194 len=1969664 2024-09-13T00:07:18.530 INFO:tasks.workunit.client.0.smithi136.stdout:op 1838 completed, throughput=5MB/sec 2024-09-13T00:07:18.530 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-uIEIbADJuiAOFOr off=7133417 len=786173 2024-09-13T00:07:18.532 INFO:tasks.workunit.client.0.smithi136.stdout:op 1839 completed, throughput=5MB/sec 2024-09-13T00:07:18.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:07:18 smithi136 bash[75164]: cluster 2024-09-13T00:07:17.323810+0000 mgr.y (mgr.24661) 905 : cluster [DBG] pgmap v1588: 105 pgs: 105 active+clean; 126 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:07:18.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:07:18 smithi136 bash[76371]: cluster 2024-09-13T00:07:17.323810+0000 mgr.y (mgr.24661) 905 : cluster [DBG] pgmap v1588: 105 pgs: 105 active+clean; 126 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:07:18.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:07:18 smithi181 bash[67587]: cluster 2024-09-13T00:07:17.323810+0000 mgr.y (mgr.24661) 905 : cluster [DBG] pgmap v1588: 105 pgs: 105 active+clean; 126 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:07:19.533 INFO:tasks.workunit.client.0.smithi136.stdout: 345: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:19.533 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-EnaS6YA-YAtOKef off=6262683 len=1746955 2024-09-13T00:07:19.539 INFO:tasks.workunit.client.0.smithi136.stdout:op 1840 completed, throughput=4.99MB/sec 2024-09-13T00:07:19.539 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-XkF-UKGWr5PmXUm off=3495110 len=335956 2024-09-13T00:07:19.541 INFO:tasks.workunit.client.0.smithi136.stdout:op 1841 completed, throughput=4.99MB/sec 2024-09-13T00:07:19.541 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Wj4AigmPoBo0iLu off=6178698 len=814397 2024-09-13T00:07:19.549 INFO:tasks.workunit.client.0.smithi136.stdout:op 1842 completed, throughput=5MB/sec 2024-09-13T00:07:19.549 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-jTcJuj1Xqx59s6J off=10377344 len=477672 2024-09-13T00:07:19.552 INFO:tasks.workunit.client.0.smithi136.stdout:op 1843 completed, throughput=5MB/sec 2024-09-13T00:07:19.552 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-nyt9pZNUhFFotDm off=6416404 len=2090097 2024-09-13T00:07:19.566 INFO:tasks.workunit.client.0.smithi136.stdout:op 1844 completed, throughput=5MB/sec 2024-09-13T00:07:19.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:07:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:07:19] "GET /metrics HTTP/1.1" 200 34135 "" "Prometheus/2.43.0" 2024-09-13T00:07:20.566 INFO:tasks.workunit.client.0.smithi136.stdout: 346: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:20.567 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-gHUqbkK4sWXRCs0 off=67989 len=1774005 2024-09-13T00:07:20.573 INFO:tasks.workunit.client.0.smithi136.stdout:op 1845 completed, throughput=4.99MB/sec 2024-09-13T00:07:20.573 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-7JdmpAK1MgoMDuE off=13025258 len=1153927 2024-09-13T00:07:20.576 INFO:tasks.workunit.client.0.smithi136.stdout:op 1846 completed, throughput=5MB/sec 2024-09-13T00:07:20.576 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-bFPCwTn2nq-H5M_ off=8041518 len=2003313 2024-09-13T00:07:20.582 INFO:tasks.workunit.client.0.smithi136.stdout:op 1847 completed, throughput=5MB/sec 2024-09-13T00:07:21.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:07:20 smithi136 bash[76371]: cluster 2024-09-13T00:07:19.324722+0000 mgr.y (mgr.24661) 906 : cluster [DBG] pgmap v1589: 105 pgs: 105 active+clean; 126 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:07:21.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:07:20 smithi136 bash[75164]: cluster 2024-09-13T00:07:19.324722+0000 mgr.y (mgr.24661) 906 : cluster [DBG] pgmap v1589: 105 pgs: 105 active+clean; 126 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:07:21.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:07:20 smithi181 bash[67587]: cluster 2024-09-13T00:07:19.324722+0000 mgr.y (mgr.24661) 906 : cluster [DBG] pgmap v1589: 105 pgs: 105 active+clean; 126 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:07:21.583 INFO:tasks.workunit.client.0.smithi136.stdout: 347: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:21.583 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-kLgkg4G7BVhra_F off=2245150 len=1580857 2024-09-13T00:07:21.587 INFO:tasks.workunit.client.0.smithi136.stdout:op 1848 completed, throughput=4.99MB/sec 2024-09-13T00:07:21.587 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Ay-miABl3XeHuK1 off=7384816 len=428575 2024-09-13T00:07:21.588 INFO:tasks.workunit.client.0.smithi136.stdout:op 1849 completed, throughput=4.99MB/sec 2024-09-13T00:07:21.588 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-7Oap0aQjtkbuntj off=3243609 len=498633 2024-09-13T00:07:21.591 INFO:tasks.workunit.client.0.smithi136.stdout:op 1850 completed, throughput=4.99MB/sec 2024-09-13T00:07:21.592 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-eXTKICNwu0TVBwH off=13694740 len=1844981 2024-09-13T00:07:21.607 INFO:tasks.workunit.client.0.smithi136.stdout:op 1851 completed, throughput=5MB/sec 2024-09-13T00:07:21.607 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-j2R9giGL80aVqNZ off=5065939 len=1685884 2024-09-13T00:07:21.619 INFO:tasks.workunit.client.0.smithi136.stdout:op 1852 completed, throughput=5MB/sec 2024-09-13T00:07:22.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:07:22 smithi181 bash[95505]: ts=2024-09-13T00:07:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:07:22.619 INFO:tasks.workunit.client.0.smithi136.stdout: 348: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:22.619 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Q45oiLcBBeoyBK2 off=1511074 len=1459833 2024-09-13T00:07:22.632 INFO:tasks.workunit.client.0.smithi136.stdout:op 1853 completed, throughput=4.99MB/sec 2024-09-13T00:07:22.632 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-bYQwYOzOjLmAQUH off=13968512 len=360755 2024-09-13T00:07:22.637 INFO:tasks.workunit.client.0.smithi136.stdout:op 1854 completed, throughput=4.99MB/sec 2024-09-13T00:07:22.637 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-PVZVGxXySIcACiX off=7325350 len=1871826 2024-09-13T00:07:22.642 INFO:tasks.workunit.client.0.smithi136.stdout:op 1855 completed, throughput=5MB/sec 2024-09-13T00:07:22.642 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-OIxy7Voi4zA1_fk off=10528797 len=996594 2024-09-13T00:07:22.650 INFO:tasks.workunit.client.0.smithi136.stdout:op 1856 completed, throughput=5MB/sec 2024-09-13T00:07:23.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:07:22 smithi136 bash[75164]: cluster 2024-09-13T00:07:21.325650+0000 mgr.y (mgr.24661) 907 : cluster [DBG] pgmap v1590: 105 pgs: 105 active+clean; 126 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:07:23.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:07:22 smithi136 bash[76371]: cluster 2024-09-13T00:07:21.325650+0000 mgr.y (mgr.24661) 907 : cluster [DBG] pgmap v1590: 105 pgs: 105 active+clean; 126 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:07:23.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:07:22 smithi181 bash[67587]: cluster 2024-09-13T00:07:21.325650+0000 mgr.y (mgr.24661) 907 : cluster [DBG] pgmap v1590: 105 pgs: 105 active+clean; 126 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:07:23.650 INFO:tasks.workunit.client.0.smithi136.stdout: 349: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:23.650 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-laEGWJlloZcxnps off=19679768 len=1626625 2024-09-13T00:07:23.654 INFO:tasks.workunit.client.0.smithi136.stdout:op 1857 completed, throughput=4.99MB/sec 2024-09-13T00:07:23.655 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-w0DsHIYWz8uv_uk off=5067490 len=933964 2024-09-13T00:07:23.657 INFO:tasks.workunit.client.0.smithi136.stdout:op 1858 completed, throughput=4.99MB/sec 2024-09-13T00:07:23.657 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-80B2EnfthRCc25s off=6810520 len=711590 2024-09-13T00:07:23.660 INFO:tasks.workunit.client.0.smithi136.stdout:op 1859 completed, throughput=4.99MB/sec 2024-09-13T00:07:23.660 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-C-c8ob20wKKdVC3 off=152210 len=234485 2024-09-13T00:07:23.661 INFO:tasks.workunit.client.0.smithi136.stdout:op 1860 completed, throughput=5MB/sec 2024-09-13T00:07:23.661 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-9ENVV8bZ7w9t_yV off=5025010 len=1432896 2024-09-13T00:07:23.673 INFO:tasks.workunit.client.0.smithi136.stdout:op 1861 completed, throughput=5MB/sec 2024-09-13T00:07:23.673 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-wGYtcyYoFjpasi5 off=17743740 len=37922 2024-09-13T00:07:23.678 INFO:tasks.workunit.client.0.smithi136.stdout:op 1862 completed, throughput=5MB/sec 2024-09-13T00:07:23.678 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-bkCconnovOrCYi3 off=2736935 len=1320718 2024-09-13T00:07:23.680 INFO:tasks.workunit.client.0.smithi136.stdout:op 1863 completed, throughput=5MB/sec 2024-09-13T00:07:24.680 INFO:tasks.workunit.client.0.smithi136.stdout: 350: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:24.681 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-3QfIhweXydrkv0L off=9617221 len=1344579 2024-09-13T00:07:24.684 INFO:tasks.workunit.client.0.smithi136.stdout:op 1864 completed, throughput=4.99MB/sec 2024-09-13T00:07:24.684 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-7d1x6keyUFndRe5 off=4980194 len=1748828 2024-09-13T00:07:24.688 INFO:tasks.workunit.client.0.smithi136.stdout:op 1865 completed, throughput=5MB/sec 2024-09-13T00:07:24.688 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-BHE2wkm00CgmGux off=10566319 len=1010912 2024-09-13T00:07:24.691 INFO:tasks.workunit.client.0.smithi136.stdout:op 1866 completed, throughput=5MB/sec 2024-09-13T00:07:24.691 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-1BQ7nZTUoMYe6JV off=1066313 len=1988386 2024-09-13T00:07:24.695 INFO:tasks.workunit.client.0.smithi136.stdout:op 1867 completed, throughput=5MB/sec 2024-09-13T00:07:25.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:07:24 smithi136 bash[75164]: cluster 2024-09-13T00:07:23.326592+0000 mgr.y (mgr.24661) 908 : cluster [DBG] pgmap v1591: 105 pgs: 105 active+clean; 126 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:07:25.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:07:24 smithi136 bash[75164]: audit 2024-09-13T00:07:23.781277+0000 mon.a (mon.0) 2160 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:07:25.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:07:24 smithi136 bash[76371]: cluster 2024-09-13T00:07:23.326592+0000 mgr.y (mgr.24661) 908 : cluster [DBG] pgmap v1591: 105 pgs: 105 active+clean; 126 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:07:25.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:07:24 smithi136 bash[76371]: audit 2024-09-13T00:07:23.781277+0000 mon.a (mon.0) 2160 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:07:25.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:07:24 smithi181 bash[67587]: cluster 2024-09-13T00:07:23.326592+0000 mgr.y (mgr.24661) 908 : cluster [DBG] pgmap v1591: 105 pgs: 105 active+clean; 126 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:07:25.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:07:24 smithi181 bash[67587]: audit 2024-09-13T00:07:23.781277+0000 mon.a (mon.0) 2160 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:07:25.696 INFO:tasks.workunit.client.0.smithi136.stdout: 351: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:25.696 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-n9W-2-lPcBd4KDk off=1256143 len=8455 2024-09-13T00:07:25.700 INFO:tasks.workunit.client.0.smithi136.stdout:op 1868 completed, throughput=4.99MB/sec 2024-09-13T00:07:25.700 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-NYRYziEXlaKp5Ql off=17240580 len=616300 2024-09-13T00:07:25.707 INFO:tasks.workunit.client.0.smithi136.stdout:op 1869 completed, throughput=4.99MB/sec 2024-09-13T00:07:25.707 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ik6q6vNIif-4Rgm off=17440335 len=1559868 2024-09-13T00:07:25.711 INFO:tasks.workunit.client.0.smithi136.stdout:op 1870 completed, throughput=5MB/sec 2024-09-13T00:07:25.711 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-PMtOpfMbpRexzCz off=6871325 len=2004899 2024-09-13T00:07:25.726 INFO:tasks.workunit.client.0.smithi136.stdout:op 1871 completed, throughput=5MB/sec 2024-09-13T00:07:26.726 INFO:tasks.workunit.client.0.smithi136.stdout: 352: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:26.726 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-EqR3GOFpG0BMhnc off=1173774 len=1601946 2024-09-13T00:07:26.731 INFO:tasks.workunit.client.0.smithi136.stdout:op 1872 completed, throughput=4.99MB/sec 2024-09-13T00:07:26.732 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-egFbl11_1dqM0do off=5583816 len=2013494 2024-09-13T00:07:26.748 INFO:tasks.workunit.client.0.smithi136.stdout:op 1873 completed, throughput=5MB/sec 2024-09-13T00:07:26.748 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-wCnVGu8iefdLV_a off=1362896 len=445249 2024-09-13T00:07:26.750 INFO:tasks.workunit.client.0.smithi136.stdout:op 1874 completed, throughput=5MB/sec 2024-09-13T00:07:26.750 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-iCjo4lTGKH3xrG8 off=2582855 len=1522846 2024-09-13T00:07:26.760 INFO:tasks.workunit.client.0.smithi136.stdout:op 1875 completed, throughput=5MB/sec 2024-09-13T00:07:27.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:07:26 smithi136 bash[75164]: cluster 2024-09-13T00:07:25.327419+0000 mgr.y (mgr.24661) 909 : cluster [DBG] pgmap v1592: 105 pgs: 105 active+clean; 126 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:07:27.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:07:26 smithi136 bash[76371]: cluster 2024-09-13T00:07:25.327419+0000 mgr.y (mgr.24661) 909 : cluster [DBG] pgmap v1592: 105 pgs: 105 active+clean; 126 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:07:27.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:07:26 smithi181 bash[67587]: cluster 2024-09-13T00:07:25.327419+0000 mgr.y (mgr.24661) 909 : cluster [DBG] pgmap v1592: 105 pgs: 105 active+clean; 126 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:07:27.760 INFO:tasks.workunit.client.0.smithi136.stdout: 353: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:27.761 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-HJcZciw4qmbPSSs off=2630302 len=1566799 2024-09-13T00:07:27.772 INFO:tasks.workunit.client.0.smithi136.stdout:op 1876 completed, throughput=4.99MB/sec 2024-09-13T00:07:27.772 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-3sN4SN9myYLrKPH off=1390243 len=343300 2024-09-13T00:07:27.778 INFO:tasks.workunit.client.0.smithi136.stdout:op 1877 completed, throughput=4.99MB/sec 2024-09-13T00:07:27.778 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-6ko6L7C2uTqw1HJ off=2051947 len=1525341 2024-09-13T00:07:27.795 INFO:tasks.workunit.client.0.smithi136.stdout:op 1878 completed, throughput=5MB/sec 2024-09-13T00:07:27.795 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-6IJV7CF-D3h5mfZ off=18719556 len=1323774 2024-09-13T00:07:27.796 INFO:tasks.workunit.client.0.smithi136.stdout:op 1879 completed, throughput=5MB/sec 2024-09-13T00:07:28.797 INFO:tasks.workunit.client.0.smithi136.stdout: 354: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:28.797 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-21caWpEQCukFhvs off=3128209 len=117434 2024-09-13T00:07:28.801 INFO:tasks.workunit.client.0.smithi136.stdout:op 1880 completed, throughput=4.99MB/sec 2024-09-13T00:07:28.801 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ZGvAOB7E-_eVsgd off=2244137 len=813803 2024-09-13T00:07:28.804 INFO:tasks.workunit.client.0.smithi136.stdout:op 1881 completed, throughput=4.99MB/sec 2024-09-13T00:07:28.804 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj--TMXAPVcy5UIaPA off=5622503 len=1321414 2024-09-13T00:07:28.808 INFO:tasks.workunit.client.0.smithi136.stdout:op 1882 completed, throughput=4.99MB/sec 2024-09-13T00:07:28.808 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ESfoWm9s5PlnLFc off=2836732 len=228956 2024-09-13T00:07:28.813 INFO:tasks.workunit.client.0.smithi136.stdout:op 1883 completed, throughput=4.99MB/sec 2024-09-13T00:07:28.814 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ecFWsuMTOF0RAsO off=1309690 len=942673 2024-09-13T00:07:28.816 INFO:tasks.workunit.client.0.smithi136.stdout:op 1884 completed, throughput=5MB/sec 2024-09-13T00:07:28.816 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj--MKxcSDt5Q3pwo6 off=6815376 len=1458350 2024-09-13T00:07:28.827 INFO:tasks.workunit.client.0.smithi136.stdout:op 1885 completed, throughput=5MB/sec 2024-09-13T00:07:28.827 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-C-v0SrweKpjdGWI off=10378636 len=121505 2024-09-13T00:07:28.829 INFO:tasks.workunit.client.0.smithi136.stdout:op 1886 completed, throughput=5MB/sec 2024-09-13T00:07:28.829 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-L-W_x9Q-2T52IEt off=17135524 len=1000856 2024-09-13T00:07:28.838 INFO:tasks.workunit.client.0.smithi136.stdout:op 1887 completed, throughput=5MB/sec 2024-09-13T00:07:29.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:07:28 smithi136 bash[75164]: cluster 2024-09-13T00:07:27.328548+0000 mgr.y (mgr.24661) 910 : cluster [DBG] pgmap v1593: 105 pgs: 105 active+clean; 126 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 3.4 MiB/s rd, 2.1 MiB/s wr, 5 op/s 2024-09-13T00:07:29.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:07:28 smithi136 bash[76371]: cluster 2024-09-13T00:07:27.328548+0000 mgr.y (mgr.24661) 910 : cluster [DBG] pgmap v1593: 105 pgs: 105 active+clean; 126 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 3.4 MiB/s rd, 2.1 MiB/s wr, 5 op/s 2024-09-13T00:07:29.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:07:28 smithi181 bash[67587]: cluster 2024-09-13T00:07:27.328548+0000 mgr.y (mgr.24661) 910 : cluster [DBG] pgmap v1593: 105 pgs: 105 active+clean; 126 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 3.4 MiB/s rd, 2.1 MiB/s wr, 5 op/s 2024-09-13T00:07:29.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:07:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:07:29] "GET /metrics HTTP/1.1" 200 34135 "" "Prometheus/2.43.0" 2024-09-13T00:07:29.838 INFO:tasks.workunit.client.0.smithi136.stdout: 355: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:29.839 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-zQtLCvITZSgT4wt off=18659568 len=440102 2024-09-13T00:07:29.844 INFO:tasks.workunit.client.0.smithi136.stdout:op 1888 completed, throughput=4.99MB/sec 2024-09-13T00:07:29.844 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-peldLUSMv1Qzlyn off=7790270 len=49013 2024-09-13T00:07:29.846 INFO:tasks.workunit.client.0.smithi136.stdout:op 1889 completed, throughput=4.99MB/sec 2024-09-13T00:07:29.846 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-FdF6zKe4KzmKoHA off=9941168 len=483974 2024-09-13T00:07:29.847 INFO:tasks.workunit.client.0.smithi136.stdout:op 1890 completed, throughput=4.99MB/sec 2024-09-13T00:07:29.847 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-cvK0Bpa0WyNPE7T off=6811274 len=2078034 2024-09-13T00:07:29.854 INFO:tasks.workunit.client.0.smithi136.stdout:op 1891 completed, throughput=5MB/sec 2024-09-13T00:07:29.854 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-UWWSU-TsA6YKxXO off=7244946 len=1575077 2024-09-13T00:07:29.858 INFO:tasks.workunit.client.0.smithi136.stdout:op 1892 completed, throughput=5MB/sec 2024-09-13T00:07:29.858 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-gXecj42inlKMnT7 off=5040894 len=245900 2024-09-13T00:07:29.864 INFO:tasks.workunit.client.0.smithi136.stdout:op 1893 completed, throughput=5MB/sec 2024-09-13T00:07:30.864 INFO:tasks.workunit.client.0.smithi136.stdout: 356: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:30.864 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-qOQsL3qkcMOFIQO off=13933116 len=1214488 2024-09-13T00:07:30.867 INFO:tasks.workunit.client.0.smithi136.stdout:op 1894 completed, throughput=4.99MB/sec 2024-09-13T00:07:30.867 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2gglCUwf_RIkrMh off=12699301 len=1761467 2024-09-13T00:07:30.871 INFO:tasks.workunit.client.0.smithi136.stdout:op 1895 completed, throughput=4.99MB/sec 2024-09-13T00:07:30.871 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-bWEgusSq5d5Uwnz off=939848 len=1752939 2024-09-13T00:07:30.887 INFO:tasks.workunit.client.0.smithi136.stdout:op 1896 completed, throughput=5MB/sec 2024-09-13T00:07:30.887 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-3TL3jKSkpIKdT0z off=630973 len=1169041 2024-09-13T00:07:30.898 INFO:tasks.workunit.client.0.smithi136.stdout:op 1897 completed, throughput=5MB/sec 2024-09-13T00:07:31.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:07:30 smithi136 bash[75164]: cluster 2024-09-13T00:07:29.329407+0000 mgr.y (mgr.24661) 911 : cluster [DBG] pgmap v1594: 105 pgs: 105 active+clean; 126 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.7 MiB/s wr, 4 op/s 2024-09-13T00:07:31.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:07:30 smithi136 bash[76371]: cluster 2024-09-13T00:07:29.329407+0000 mgr.y (mgr.24661) 911 : cluster [DBG] pgmap v1594: 105 pgs: 105 active+clean; 126 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.7 MiB/s wr, 4 op/s 2024-09-13T00:07:31.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:07:30 smithi181 bash[67587]: cluster 2024-09-13T00:07:29.329407+0000 mgr.y (mgr.24661) 911 : cluster [DBG] pgmap v1594: 105 pgs: 105 active+clean; 126 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.7 MiB/s wr, 4 op/s 2024-09-13T00:07:31.898 INFO:tasks.workunit.client.0.smithi136.stdout: 357: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:31.898 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-YiFV8fbTxzj1Ccj off=4558726 len=1154204 2024-09-13T00:07:31.908 INFO:tasks.workunit.client.0.smithi136.stdout:op 1898 completed, throughput=4.99MB/sec 2024-09-13T00:07:31.908 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-VURvZ-0Tyafmuyu off=4669952 len=1749343 2024-09-13T00:07:31.912 INFO:tasks.workunit.client.0.smithi136.stdout:op 1899 completed, throughput=5MB/sec 2024-09-13T00:07:31.912 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-kSggKJhiUd-nUnp off=2566064 len=1143944 2024-09-13T00:07:31.923 INFO:tasks.workunit.client.0.smithi136.stdout:op 1900 completed, throughput=5MB/sec 2024-09-13T00:07:31.924 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-19xMnYg-RatOfZa off=8356178 len=76109 2024-09-13T00:07:31.928 INFO:tasks.workunit.client.0.smithi136.stdout:op 1901 completed, throughput=5MB/sec 2024-09-13T00:07:31.928 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-LRIWvcwW69Tm9nW off=3279714 len=213741 2024-09-13T00:07:31.929 INFO:tasks.workunit.client.0.smithi136.stdout:op 1902 completed, throughput=5MB/sec 2024-09-13T00:07:31.929 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-X5eQEq8WrcEOb7I off=9140616 len=1782535 2024-09-13T00:07:31.934 INFO:tasks.workunit.client.0.smithi136.stdout:op 1903 completed, throughput=5MB/sec 2024-09-13T00:07:32.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:07:32 smithi181 bash[95505]: ts=2024-09-13T00:07:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:07:32.934 INFO:tasks.workunit.client.0.smithi136.stdout: 358: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:32.934 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-LXVxRyr0NCutw1h off=14093637 len=736955 2024-09-13T00:07:32.941 INFO:tasks.workunit.client.0.smithi136.stdout:op 1904 completed, throughput=4.99MB/sec 2024-09-13T00:07:32.942 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-7qNJvwWOVr_O6fY off=569470 len=13151 2024-09-13T00:07:32.942 INFO:tasks.workunit.client.0.smithi136.stdout:op 1905 completed, throughput=4.99MB/sec 2024-09-13T00:07:32.943 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ziq9J1eO46nhCiz off=4044212 len=616625 2024-09-13T00:07:32.949 INFO:tasks.workunit.client.0.smithi136.stdout:op 1906 completed, throughput=4.99MB/sec 2024-09-13T00:07:32.949 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-gjcU6SpgNUrtxX9 off=14357730 len=1837219 2024-09-13T00:07:32.953 INFO:tasks.workunit.client.0.smithi136.stdout:op 1907 completed, throughput=5MB/sec 2024-09-13T00:07:32.954 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-3nC8A_W0_GMmusH off=840557 len=1840514 2024-09-13T00:07:32.969 INFO:tasks.workunit.client.0.smithi136.stdout:op 1908 completed, throughput=5MB/sec 2024-09-13T00:07:33.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:07:32 smithi136 bash[75164]: cluster 2024-09-13T00:07:31.330336+0000 mgr.y (mgr.24661) 912 : cluster [DBG] pgmap v1595: 105 pgs: 105 active+clean; 126 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:07:33.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:07:32 smithi136 bash[76371]: cluster 2024-09-13T00:07:31.330336+0000 mgr.y (mgr.24661) 912 : cluster [DBG] pgmap v1595: 105 pgs: 105 active+clean; 126 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:07:33.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:07:32 smithi181 bash[67587]: cluster 2024-09-13T00:07:31.330336+0000 mgr.y (mgr.24661) 912 : cluster [DBG] pgmap v1595: 105 pgs: 105 active+clean; 126 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:07:33.969 INFO:tasks.workunit.client.0.smithi136.stdout: 359: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:33.970 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-74LCRNxzNu-zFR5 off=6677529 len=1310191 2024-09-13T00:07:33.974 INFO:tasks.workunit.client.0.smithi136.stdout:op 1909 completed, throughput=4.99MB/sec 2024-09-13T00:07:33.975 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-gkLYlN2wDVE8PTG off=15627666 len=127395 2024-09-13T00:07:33.976 INFO:tasks.workunit.client.0.smithi136.stdout:op 1910 completed, throughput=4.99MB/sec 2024-09-13T00:07:33.976 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-rShEgtiCXnw6yKz off=10485530 len=1898490 2024-09-13T00:07:33.992 INFO:tasks.workunit.client.0.smithi136.stdout:op 1911 completed, throughput=5MB/sec 2024-09-13T00:07:33.992 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Cndv86wJRR_HGkZ off=19565033 len=948812 2024-09-13T00:07:33.995 INFO:tasks.workunit.client.0.smithi136.stdout:op 1912 completed, throughput=5MB/sec 2024-09-13T00:07:33.995 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-bUN2oLS9cNoKpyM off=5361190 len=514773 2024-09-13T00:07:33.997 INFO:tasks.workunit.client.0.smithi136.stdout:op 1913 completed, throughput=5MB/sec 2024-09-13T00:07:34.997 INFO:tasks.workunit.client.0.smithi136.stdout: 360: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:34.998 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-QfH-XSLyIl4EFVo off=22148827 len=1394914 2024-09-13T00:07:35.007 INFO:tasks.workunit.client.0.smithi136.stdout:op 1914 completed, throughput=4.99MB/sec 2024-09-13T00:07:35.007 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-G8Bw20GkwrtG0s6 off=204876 len=387715 2024-09-13T00:07:35.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:07:34 smithi136 bash[75164]: cluster 2024-09-13T00:07:33.331284+0000 mgr.y (mgr.24661) 913 : cluster [DBG] pgmap v1596: 105 pgs: 105 active+clean; 126 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:07:35.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:07:34 smithi136 bash[76371]: cluster 2024-09-13T00:07:33.331284+0000 mgr.y (mgr.24661) 913 : cluster [DBG] pgmap v1596: 105 pgs: 105 active+clean; 126 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:07:35.010 INFO:tasks.workunit.client.0.smithi136.stdout:op 1915 completed, throughput=4.99MB/sec 2024-09-13T00:07:35.010 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-s5-VAqY_CPxaJ_n off=631979 len=1402429 2024-09-13T00:07:35.013 INFO:tasks.workunit.client.0.smithi136.stdout:op 1916 completed, throughput=5MB/sec 2024-09-13T00:07:35.013 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-3cNPSP6f1_BzgJm off=9715362 len=120391 2024-09-13T00:07:35.014 INFO:tasks.workunit.client.0.smithi136.stdout:op 1917 completed, throughput=5MB/sec 2024-09-13T00:07:35.014 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-3u76JqPs7gOeHYa off=18936812 len=716369 2024-09-13T00:07:35.023 INFO:tasks.workunit.client.0.smithi136.stdout:op 1918 completed, throughput=5MB/sec 2024-09-13T00:07:35.023 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-KHlU4eIeeShayZh off=281046 len=1529088 2024-09-13T00:07:35.027 INFO:tasks.workunit.client.0.smithi136.stdout:op 1919 completed, throughput=5MB/sec 2024-09-13T00:07:35.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:07:34 smithi181 bash[67587]: cluster 2024-09-13T00:07:33.331284+0000 mgr.y (mgr.24661) 913 : cluster [DBG] pgmap v1596: 105 pgs: 105 active+clean; 126 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:07:36.027 INFO:tasks.workunit.client.0.smithi136.stdout: 361: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:36.027 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-XOMP6ORjQuxhkUJ off=9670043 len=437931 2024-09-13T00:07:36.033 INFO:tasks.workunit.client.0.smithi136.stdout:op 1920 completed, throughput=4.99MB/sec 2024-09-13T00:07:36.034 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-vtE2n00RgS83wn2 off=2495674 len=2067495 2024-09-13T00:07:36.050 INFO:tasks.workunit.client.0.smithi136.stdout:op 1921 completed, throughput=4.99MB/sec 2024-09-13T00:07:36.050 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-rICIEtpjYjmZTab off=812528 len=332446 2024-09-13T00:07:36.051 INFO:tasks.workunit.client.0.smithi136.stdout:op 1922 completed, throughput=4.99MB/sec 2024-09-13T00:07:36.051 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-OkJYY3QJnZq9cFG off=1297480 len=125752 2024-09-13T00:07:36.052 INFO:tasks.workunit.client.0.smithi136.stdout:op 1923 completed, throughput=4.99MB/sec 2024-09-13T00:07:36.052 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-4iPQmbxxGg0iiEL off=16461277 len=1923344 2024-09-13T00:07:36.055 INFO:tasks.workunit.client.0.smithi136.stdout:op 1924 completed, throughput=5MB/sec 2024-09-13T00:07:36.055 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-DmtwNW6_T5U3ujr off=2785258 len=56703 2024-09-13T00:07:36.059 INFO:tasks.workunit.client.0.smithi136.stdout:op 1925 completed, throughput=5MB/sec 2024-09-13T00:07:36.059 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-727YlzuqMoSTpDq off=9590024 len=1934068 2024-09-13T00:07:36.070 INFO:tasks.workunit.client.0.smithi136.stdout:op 1926 completed, throughput=5MB/sec 2024-09-13T00:07:37.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:07:36 smithi136 bash[75164]: cluster 2024-09-13T00:07:35.332109+0000 mgr.y (mgr.24661) 914 : cluster [DBG] pgmap v1597: 105 pgs: 105 active+clean; 126 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:07:37.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:07:36 smithi136 bash[76371]: cluster 2024-09-13T00:07:35.332109+0000 mgr.y (mgr.24661) 914 : cluster [DBG] pgmap v1597: 105 pgs: 105 active+clean; 126 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:07:37.071 INFO:tasks.workunit.client.0.smithi136.stdout: 362: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:37.071 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-BRzwY14e5uhVdwL off=6233172 len=1462663 2024-09-13T00:07:37.076 INFO:tasks.workunit.client.0.smithi136.stdout:op 1927 completed, throughput=5MB/sec 2024-09-13T00:07:37.076 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-tBMtJlJPb3QOkoZ off=2030103 len=1825182 2024-09-13T00:07:37.080 INFO:tasks.workunit.client.0.smithi136.stdout:op 1928 completed, throughput=5MB/sec 2024-09-13T00:07:37.081 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-gw3615AkXlnJqDo off=1105222 len=469103 2024-09-13T00:07:37.086 INFO:tasks.workunit.client.0.smithi136.stdout:op 1929 completed, throughput=5MB/sec 2024-09-13T00:07:37.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:07:36 smithi181 bash[67587]: cluster 2024-09-13T00:07:35.332109+0000 mgr.y (mgr.24661) 914 : cluster [DBG] pgmap v1597: 105 pgs: 105 active+clean; 126 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:07:38.086 INFO:tasks.workunit.client.0.smithi136.stdout: 363: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:38.087 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-SLu3A03tPSs6-iJ off=3814006 len=31135 2024-09-13T00:07:38.091 INFO:tasks.workunit.client.0.smithi136.stdout:op 1930 completed, throughput=4.99MB/sec 2024-09-13T00:07:38.091 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-vy9do2YUzaUNjKK off=1729416 len=385644 2024-09-13T00:07:38.092 INFO:tasks.workunit.client.0.smithi136.stdout:op 1931 completed, throughput=4.99MB/sec 2024-09-13T00:07:38.092 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-dFXsW4XAwFGizP4 off=2569320 len=1163751 2024-09-13T00:07:38.102 INFO:tasks.workunit.client.0.smithi136.stdout:op 1932 completed, throughput=4.99MB/sec 2024-09-13T00:07:38.102 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-L89lCGjANm-twk0 off=20698855 len=904281 2024-09-13T00:07:38.109 INFO:tasks.workunit.client.0.smithi136.stdout:op 1933 completed, throughput=4.99MB/sec 2024-09-13T00:07:38.109 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2XC6Wo_6-pmVScF off=4999514 len=1336975 2024-09-13T00:07:38.113 INFO:tasks.workunit.client.0.smithi136.stdout:op 1934 completed, throughput=5MB/sec 2024-09-13T00:07:38.113 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-CD__KpD2PhuyZgQ off=1344078 len=1364366 2024-09-13T00:07:38.116 INFO:tasks.workunit.client.0.smithi136.stdout:op 1935 completed, throughput=5MB/sec 2024-09-13T00:07:39.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:07:38 smithi136 bash[76371]: cluster 2024-09-13T00:07:37.333289+0000 mgr.y (mgr.24661) 915 : cluster [DBG] pgmap v1598: 105 pgs: 105 active+clean; 126 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.2 MiB/s wr, 5 op/s 2024-09-13T00:07:39.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:07:38 smithi136 bash[75164]: cluster 2024-09-13T00:07:37.333289+0000 mgr.y (mgr.24661) 915 : cluster [DBG] pgmap v1598: 105 pgs: 105 active+clean; 126 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.2 MiB/s wr, 5 op/s 2024-09-13T00:07:39.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:07:38 smithi181 bash[67587]: cluster 2024-09-13T00:07:37.333289+0000 mgr.y (mgr.24661) 915 : cluster [DBG] pgmap v1598: 105 pgs: 105 active+clean; 126 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.2 MiB/s wr, 5 op/s 2024-09-13T00:07:39.117 INFO:tasks.workunit.client.0.smithi136.stdout: 364: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:39.117 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-BZcjPpN4ZVeiIuY off=1328436 len=603836 2024-09-13T00:07:39.119 INFO:tasks.workunit.client.0.smithi136.stdout:op 1936 completed, throughput=4.99MB/sec 2024-09-13T00:07:39.119 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-vrPcKAeC1X8HKAI off=6747549 len=1349188 2024-09-13T00:07:39.129 INFO:tasks.workunit.client.0.smithi136.stdout:op 1937 completed, throughput=4.99MB/sec 2024-09-13T00:07:39.129 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-1qOkF6TCT9dLq_c off=275617 len=658495 2024-09-13T00:07:39.131 INFO:tasks.workunit.client.0.smithi136.stdout:op 1938 completed, throughput=4.99MB/sec 2024-09-13T00:07:39.131 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-JdWAsujHlMMQp_z off=2680091 len=1131156 2024-09-13T00:07:39.139 INFO:tasks.workunit.client.0.smithi136.stdout:op 1939 completed, throughput=5MB/sec 2024-09-13T00:07:39.140 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-bYQwYOzOjLmAQUH off=10340446 len=1467884 2024-09-13T00:07:39.143 INFO:tasks.workunit.client.0.smithi136.stdout:op 1940 completed, throughput=5MB/sec 2024-09-13T00:07:39.143 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-R0WoQFcPY1tiCEN off=8787517 len=364461 2024-09-13T00:07:39.148 INFO:tasks.workunit.client.0.smithi136.stdout:op 1941 completed, throughput=5MB/sec 2024-09-13T00:07:39.710 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:07:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:07:39] "GET /metrics HTTP/1.1" 200 34134 "" "Prometheus/2.43.0" 2024-09-13T00:07:40.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:07:39 smithi136 bash[76371]: audit 2024-09-13T00:07:38.781773+0000 mon.a (mon.0) 2161 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:07:40.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:07:39 smithi136 bash[75164]: audit 2024-09-13T00:07:38.781773+0000 mon.a (mon.0) 2161 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:07:40.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:07:39 smithi181 bash[67587]: audit 2024-09-13T00:07:38.781773+0000 mon.a (mon.0) 2161 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:07:40.148 INFO:tasks.workunit.client.0.smithi136.stdout: 365: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:40.148 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-N6Ebzcln6J7rZ-K off=10984744 len=1541348 2024-09-13T00:07:40.152 INFO:tasks.workunit.client.0.smithi136.stdout:op 1942 completed, throughput=4.99MB/sec 2024-09-13T00:07:40.152 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-tj_jFGQZiFINZWV off=15455138 len=1493365 2024-09-13T00:07:40.157 INFO:tasks.workunit.client.0.smithi136.stdout:op 1943 completed, throughput=4.99MB/sec 2024-09-13T00:07:40.158 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-0qG8QXyZdgtTT56 off=489847 len=326951 2024-09-13T00:07:40.159 INFO:tasks.workunit.client.0.smithi136.stdout:op 1944 completed, throughput=5MB/sec 2024-09-13T00:07:40.159 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-KijbpnY3kxG8qbv off=2196916 len=919043 2024-09-13T00:07:40.162 INFO:tasks.workunit.client.0.smithi136.stdout:op 1945 completed, throughput=5MB/sec 2024-09-13T00:07:40.162 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Kiygriefjvoz7n4 off=660214 len=950521 2024-09-13T00:07:40.171 INFO:tasks.workunit.client.0.smithi136.stdout:op 1946 completed, throughput=5MB/sec 2024-09-13T00:07:41.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:07:40 smithi136 bash[76371]: cluster 2024-09-13T00:07:39.334195+0000 mgr.y (mgr.24661) 916 : cluster [DBG] pgmap v1599: 105 pgs: 105 active+clean; 126 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:07:41.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:07:40 smithi136 bash[75164]: cluster 2024-09-13T00:07:39.334195+0000 mgr.y (mgr.24661) 916 : cluster [DBG] pgmap v1599: 105 pgs: 105 active+clean; 126 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:07:41.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:07:40 smithi181 bash[67587]: cluster 2024-09-13T00:07:39.334195+0000 mgr.y (mgr.24661) 916 : cluster [DBG] pgmap v1599: 105 pgs: 105 active+clean; 126 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:07:41.171 INFO:tasks.workunit.client.0.smithi136.stdout: 366: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:41.171 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-5Tw6saESqwk7gSv off=2353945 len=1056703 2024-09-13T00:07:41.174 INFO:tasks.workunit.client.0.smithi136.stdout:op 1947 completed, throughput=4.99MB/sec 2024-09-13T00:07:41.174 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-dUmqiPGmsOthrbX off=5578912 len=1095378 2024-09-13T00:07:41.177 INFO:tasks.workunit.client.0.smithi136.stdout:op 1948 completed, throughput=4.99MB/sec 2024-09-13T00:07:41.177 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-s5-VAqY_CPxaJ_n off=921511 len=1324825 2024-09-13T00:07:41.187 INFO:tasks.workunit.client.0.smithi136.stdout:op 1949 completed, throughput=5MB/sec 2024-09-13T00:07:41.188 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-DUPvYziwiqhRmLe off=5213462 len=843337 2024-09-13T00:07:41.195 INFO:tasks.workunit.client.0.smithi136.stdout:op 1950 completed, throughput=5MB/sec 2024-09-13T00:07:41.195 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-fCVNxpWXM6IyBzZ off=5033272 len=771432 2024-09-13T00:07:41.201 INFO:tasks.workunit.client.0.smithi136.stdout:op 1951 completed, throughput=5MB/sec 2024-09-13T00:07:41.202 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-V8N5vU2f-KlMGJl off=940060 len=1803841 2024-09-13T00:07:41.206 INFO:tasks.workunit.client.0.smithi136.stdout:op 1952 completed, throughput=5MB/sec 2024-09-13T00:07:42.206 INFO:tasks.workunit.client.0.smithi136.stdout: 367: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:42.207 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-LVmKtwaZg1J2VH6 off=15800698 len=1645815 2024-09-13T00:07:42.210 INFO:tasks.workunit.client.0.smithi136.stdout:op 1953 completed, throughput=4.99MB/sec 2024-09-13T00:07:42.210 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-KkpMrkX7EaB4WCR off=935342 len=1148370 2024-09-13T00:07:42.213 INFO:tasks.workunit.client.0.smithi136.stdout:op 1954 completed, throughput=5MB/sec 2024-09-13T00:07:42.214 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-b7PkGhhEOtFDwNJ off=423127 len=1601312 2024-09-13T00:07:42.226 INFO:tasks.workunit.client.0.smithi136.stdout:op 1955 completed, throughput=5MB/sec 2024-09-13T00:07:42.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:07:42 smithi181 bash[95505]: ts=2024-09-13T00:07:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:07:43.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:07:42 smithi136 bash[76371]: cluster 2024-09-13T00:07:41.335206+0000 mgr.y (mgr.24661) 917 : cluster [DBG] pgmap v1600: 105 pgs: 105 active+clean; 126 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:07:43.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:07:42 smithi136 bash[75164]: cluster 2024-09-13T00:07:41.335206+0000 mgr.y (mgr.24661) 917 : cluster [DBG] pgmap v1600: 105 pgs: 105 active+clean; 126 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:07:43.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:07:42 smithi181 bash[67587]: cluster 2024-09-13T00:07:41.335206+0000 mgr.y (mgr.24661) 917 : cluster [DBG] pgmap v1600: 105 pgs: 105 active+clean; 126 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:07:43.226 INFO:tasks.workunit.client.0.smithi136.stdout: 368: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:43.233 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-uXq0PB_EkrdDAcF off=8485413 len=1469741 2024-09-13T00:07:43.239 INFO:tasks.workunit.client.0.smithi136.stdout:op 1956 completed, throughput=4.99MB/sec 2024-09-13T00:07:43.239 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-fueO_9KlQ4hRevb off=3070835 len=934758 2024-09-13T00:07:43.241 INFO:tasks.workunit.client.0.smithi136.stdout:op 1957 completed, throughput=4.99MB/sec 2024-09-13T00:07:43.241 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ffUNhcHALEcja3F off=10153753 len=83097 2024-09-13T00:07:43.246 INFO:tasks.workunit.client.0.smithi136.stdout:op 1958 completed, throughput=4.99MB/sec 2024-09-13T00:07:43.246 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-C-c6rPSPR_WQIex off=12167804 len=1723679 2024-09-13T00:07:43.257 INFO:tasks.workunit.client.0.smithi136.stdout:op 1959 completed, throughput=5MB/sec 2024-09-13T00:07:43.257 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-HH86vkaYS-XTCIt off=4779301 len=1678734 2024-09-13T00:07:43.267 INFO:tasks.workunit.client.0.smithi136.stdout:op 1960 completed, throughput=5MB/sec 2024-09-13T00:07:44.268 INFO:tasks.workunit.client.0.smithi136.stdout: 369: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:44.268 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-W8NQ4_AOlO1rpiV off=2304804 len=981834 2024-09-13T00:07:44.276 INFO:tasks.workunit.client.0.smithi136.stdout:op 1961 completed, throughput=4.99MB/sec 2024-09-13T00:07:44.276 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-d1GwKnj18L9FrlX off=1446862 len=375134 2024-09-13T00:07:44.281 INFO:tasks.workunit.client.0.smithi136.stdout:op 1962 completed, throughput=4.99MB/sec 2024-09-13T00:07:44.282 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-3V-GrwUGTaTnjQa off=4867614 len=1145546 2024-09-13T00:07:44.284 INFO:tasks.workunit.client.0.smithi136.stdout:op 1963 completed, throughput=5MB/sec 2024-09-13T00:07:44.285 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-0pkyY-tUYojv-A- off=6139075 len=468999 2024-09-13T00:07:44.287 INFO:tasks.workunit.client.0.smithi136.stdout:op 1964 completed, throughput=5MB/sec 2024-09-13T00:07:44.287 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-x8LsW8XpzQSyjex off=162725 len=1957886 2024-09-13T00:07:44.301 INFO:tasks.workunit.client.0.smithi136.stdout:op 1965 completed, throughput=5MB/sec 2024-09-13T00:07:45.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:07:44 smithi136 bash[75164]: cluster 2024-09-13T00:07:43.336151+0000 mgr.y (mgr.24661) 918 : cluster [DBG] pgmap v1601: 105 pgs: 105 active+clean; 126 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:07:45.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:07:44 smithi136 bash[76371]: cluster 2024-09-13T00:07:43.336151+0000 mgr.y (mgr.24661) 918 : cluster [DBG] pgmap v1601: 105 pgs: 105 active+clean; 126 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:07:45.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:07:44 smithi181 bash[67587]: cluster 2024-09-13T00:07:43.336151+0000 mgr.y (mgr.24661) 918 : cluster [DBG] pgmap v1601: 105 pgs: 105 active+clean; 126 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:07:45.301 INFO:tasks.workunit.client.0.smithi136.stdout: 370: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:45.301 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-NgVZyQp0mQjWSOG off=5733040 len=174758 2024-09-13T00:07:45.302 INFO:tasks.workunit.client.0.smithi136.stdout:op 1966 completed, throughput=4.99MB/sec 2024-09-13T00:07:45.302 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-1arUr0QMyiDlx07 off=906963 len=351558 2024-09-13T00:07:45.304 INFO:tasks.workunit.client.0.smithi136.stdout:op 1967 completed, throughput=4.99MB/sec 2024-09-13T00:07:45.305 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Ycl84r327NP6UnU off=10078013 len=1109391 2024-09-13T00:07:45.313 INFO:tasks.workunit.client.0.smithi136.stdout:op 1968 completed, throughput=4.99MB/sec 2024-09-13T00:07:45.313 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-h4w0INPd320sCv4 off=4071255 len=141854 2024-09-13T00:07:45.317 INFO:tasks.workunit.client.0.smithi136.stdout:op 1969 completed, throughput=4.99MB/sec 2024-09-13T00:07:45.317 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-aMr4ZK0WWqITesN off=4514560 len=234189 2024-09-13T00:07:45.318 INFO:tasks.workunit.client.0.smithi136.stdout:op 1970 completed, throughput=4.99MB/sec 2024-09-13T00:07:45.319 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-pK9Z8XqL-XTPKHq off=1156134 len=1276569 2024-09-13T00:07:45.328 INFO:tasks.workunit.client.0.smithi136.stdout:op 1971 completed, throughput=5MB/sec 2024-09-13T00:07:45.328 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-g1rx-LrcFuOdH2a off=7151593 len=441600 2024-09-13T00:07:45.330 INFO:tasks.workunit.client.0.smithi136.stdout:op 1972 completed, throughput=5MB/sec 2024-09-13T00:07:45.330 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-QcYF5UDEf63CdhG off=7144934 len=374035 2024-09-13T00:07:45.335 INFO:tasks.workunit.client.0.smithi136.stdout:op 1973 completed, throughput=5MB/sec 2024-09-13T00:07:45.335 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-pZRbgSPiZ3QuVML off=11724411 len=1554959 2024-09-13T00:07:45.346 INFO:tasks.workunit.client.0.smithi136.stdout:op 1974 completed, throughput=5MB/sec 2024-09-13T00:07:46.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:07:45 smithi136 bash[75164]: audit 2024-09-13T00:07:45.321777+0000 mon.a (mon.0) 2162 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:07:46.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:07:45 smithi136 bash[76371]: audit 2024-09-13T00:07:45.321777+0000 mon.a (mon.0) 2162 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:07:46.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:07:45 smithi181 bash[67587]: audit 2024-09-13T00:07:45.321777+0000 mon.a (mon.0) 2162 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:07:46.346 INFO:tasks.workunit.client.0.smithi136.stdout: 371: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:46.346 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-R-B2gPjmKdbLTYk off=6638500 len=439463 2024-09-13T00:07:46.348 INFO:tasks.workunit.client.0.smithi136.stdout:op 1975 completed, throughput=4.99MB/sec 2024-09-13T00:07:46.348 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-BUOFZxChCTLXBnI off=9710250 len=194606 2024-09-13T00:07:46.353 INFO:tasks.workunit.client.0.smithi136.stdout:op 1976 completed, throughput=4.99MB/sec 2024-09-13T00:07:46.353 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-pzOrTKRQrGOwBBu off=12716126 len=1940935 2024-09-13T00:07:46.370 INFO:tasks.workunit.client.0.smithi136.stdout:op 1977 completed, throughput=4.99MB/sec 2024-09-13T00:07:46.370 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-oOLgelI0zZqn25V off=1730428 len=69302 2024-09-13T00:07:46.375 INFO:tasks.workunit.client.0.smithi136.stdout:op 1978 completed, throughput=5MB/sec 2024-09-13T00:07:46.375 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-gNV6DExtPCR21Dh off=3428079 len=753333 2024-09-13T00:07:46.382 INFO:tasks.workunit.client.0.smithi136.stdout:op 1979 completed, throughput=5MB/sec 2024-09-13T00:07:46.382 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-4F3TI8t4OGzf6TY off=3724964 len=1042447 2024-09-13T00:07:46.391 INFO:tasks.workunit.client.0.smithi136.stdout:op 1980 completed, throughput=5MB/sec 2024-09-13T00:07:46.391 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-pGmVgFWy-jlvEGB off=199358 len=1802330 2024-09-13T00:07:46.396 INFO:tasks.workunit.client.0.smithi136.stdout:op 1981 completed, throughput=5MB/sec 2024-09-13T00:07:47.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:07:46 smithi136 bash[75164]: cluster 2024-09-13T00:07:45.337121+0000 mgr.y (mgr.24661) 919 : cluster [DBG] pgmap v1602: 105 pgs: 105 active+clean; 126 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:07:47.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:07:46 smithi136 bash[75164]: audit 2024-09-13T00:07:45.863095+0000 mon.a (mon.0) 2163 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:07:47.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:07:46 smithi136 bash[75164]: audit 2024-09-13T00:07:45.864758+0000 mon.a (mon.0) 2164 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:07:47.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:07:46 smithi136 bash[75164]: audit 2024-09-13T00:07:45.873581+0000 mon.a (mon.0) 2165 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:07:47.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:07:46 smithi136 bash[76371]: cluster 2024-09-13T00:07:45.337121+0000 mgr.y (mgr.24661) 919 : cluster [DBG] pgmap v1602: 105 pgs: 105 active+clean; 126 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:07:47.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:07:46 smithi136 bash[76371]: audit 2024-09-13T00:07:45.863095+0000 mon.a (mon.0) 2163 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:07:47.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:07:46 smithi136 bash[76371]: audit 2024-09-13T00:07:45.864758+0000 mon.a (mon.0) 2164 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:07:47.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:07:46 smithi136 bash[76371]: audit 2024-09-13T00:07:45.873581+0000 mon.a (mon.0) 2165 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:07:47.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:07:46 smithi181 bash[67587]: cluster 2024-09-13T00:07:45.337121+0000 mgr.y (mgr.24661) 919 : cluster [DBG] pgmap v1602: 105 pgs: 105 active+clean; 126 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:07:47.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:07:46 smithi181 bash[67587]: audit 2024-09-13T00:07:45.863095+0000 mon.a (mon.0) 2163 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:07:47.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:07:46 smithi181 bash[67587]: audit 2024-09-13T00:07:45.864758+0000 mon.a (mon.0) 2164 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:07:47.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:07:46 smithi181 bash[67587]: audit 2024-09-13T00:07:45.873581+0000 mon.a (mon.0) 2165 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:07:47.396 INFO:tasks.workunit.client.0.smithi136.stdout: 372: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:47.397 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-5uoc1gSOeXhvgVR off=3607223 len=1277440 2024-09-13T00:07:47.400 INFO:tasks.workunit.client.0.smithi136.stdout:op 1982 completed, throughput=4.99MB/sec 2024-09-13T00:07:47.400 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-gTxu3VB4ah3azYl off=11408379 len=78594 2024-09-13T00:07:47.402 INFO:tasks.workunit.client.0.smithi136.stdout:op 1983 completed, throughput=4.99MB/sec 2024-09-13T00:07:47.402 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-awccGGwXjeWwV2K off=1240943 len=590913 2024-09-13T00:07:47.408 INFO:tasks.workunit.client.0.smithi136.stdout:op 1984 completed, throughput=5MB/sec 2024-09-13T00:07:47.408 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-kVmwrblQWidnVs9 off=18418477 len=1333388 2024-09-13T00:07:47.418 INFO:tasks.workunit.client.0.smithi136.stdout:op 1985 completed, throughput=5MB/sec 2024-09-13T00:07:47.418 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-d4P_TS-nRdx5wbj off=7626590 len=1422870 2024-09-13T00:07:47.429 INFO:tasks.workunit.client.0.smithi136.stdout:op 1986 completed, throughput=5MB/sec 2024-09-13T00:07:48.429 INFO:tasks.workunit.client.0.smithi136.stdout: 373: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:48.430 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-kTj8cc7VEq0FMLl off=1261738 len=17019 2024-09-13T00:07:48.430 INFO:tasks.workunit.client.0.smithi136.stdout:op 1987 completed, throughput=4.99MB/sec 2024-09-13T00:07:48.431 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-bLnsf1Tyc0DdMc1 off=2259388 len=128242 2024-09-13T00:07:48.435 INFO:tasks.workunit.client.0.smithi136.stdout:op 1988 completed, throughput=4.99MB/sec 2024-09-13T00:07:48.435 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-MPEbqUE9PtPmtqg off=3082768 len=817842 2024-09-13T00:07:48.438 INFO:tasks.workunit.client.0.smithi136.stdout:op 1989 completed, throughput=4.99MB/sec 2024-09-13T00:07:48.438 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-NsfRV9bePsXPQJh off=4567197 len=1824356 2024-09-13T00:07:48.453 INFO:tasks.workunit.client.0.smithi136.stdout:op 1990 completed, throughput=5MB/sec 2024-09-13T00:07:48.453 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-XoVhRtC9LALQv19 off=17667278 len=1154441 2024-09-13T00:07:48.463 INFO:tasks.workunit.client.0.smithi136.stdout:op 1991 completed, throughput=5MB/sec 2024-09-13T00:07:48.463 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-wLWk4r6WVlkI6Jl off=2870610 len=684114 2024-09-13T00:07:48.469 INFO:tasks.workunit.client.0.smithi136.stdout:op 1992 completed, throughput=5MB/sec 2024-09-13T00:07:48.469 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-CVYhOHdfexlW6vS off=637715 len=511248 2024-09-13T00:07:48.476 INFO:tasks.workunit.client.0.smithi136.stdout:op 1993 completed, throughput=5MB/sec 2024-09-13T00:07:49.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:07:48 smithi136 bash[75164]: cluster 2024-09-13T00:07:47.338295+0000 mgr.y (mgr.24661) 920 : cluster [DBG] pgmap v1603: 105 pgs: 105 active+clean; 126 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:07:49.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:07:48 smithi136 bash[76371]: cluster 2024-09-13T00:07:47.338295+0000 mgr.y (mgr.24661) 920 : cluster [DBG] pgmap v1603: 105 pgs: 105 active+clean; 126 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:07:49.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:07:48 smithi181 bash[67587]: cluster 2024-09-13T00:07:47.338295+0000 mgr.y (mgr.24661) 920 : cluster [DBG] pgmap v1603: 105 pgs: 105 active+clean; 126 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:07:49.476 INFO:tasks.workunit.client.0.smithi136.stdout: 374: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:49.477 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-P0ysQmP6O4iqa-H off=7713899 len=1397880 2024-09-13T00:07:49.480 INFO:tasks.workunit.client.0.smithi136.stdout:op 1994 completed, throughput=4.99MB/sec 2024-09-13T00:07:49.480 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Ijst6yhFWHEwQDW off=7250116 len=2029476 2024-09-13T00:07:49.493 INFO:tasks.workunit.client.0.smithi136.stdout:op 1995 completed, throughput=5MB/sec 2024-09-13T00:07:49.493 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-QYzS_0LKRJYiXhg off=22087841 len=524212 2024-09-13T00:07:49.499 INFO:tasks.workunit.client.0.smithi136.stdout:op 1996 completed, throughput=5MB/sec 2024-09-13T00:07:49.500 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-RhiMcaqypc_TJkD off=6285813 len=1407564 2024-09-13T00:07:49.503 INFO:tasks.workunit.client.0.smithi136.stdout:op 1997 completed, throughput=5MB/sec 2024-09-13T00:07:49.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:07:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:07:49] "GET /metrics HTTP/1.1" 200 34134 "" "Prometheus/2.43.0" 2024-09-13T00:07:50.503 INFO:tasks.workunit.client.0.smithi136.stdout: 375: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:50.504 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ToqGPxeYv9yLT9- off=443035 len=1974523 2024-09-13T00:07:50.508 INFO:tasks.workunit.client.0.smithi136.stdout:op 1998 completed, throughput=4.99MB/sec 2024-09-13T00:07:50.509 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-W1n6fvHEk69dJ4E off=1499106 len=1551441 2024-09-13T00:07:50.520 INFO:tasks.workunit.client.0.smithi136.stdout:op 1999 completed, throughput=5MB/sec 2024-09-13T00:07:50.520 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-KNdEHtDFlgB8X3O off=363097 len=1350524 2024-09-13T00:07:50.532 INFO:tasks.workunit.client.0.smithi136.stdout:op 2000 completed, throughput=5MB/sec 2024-09-13T00:07:50.532 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-gedR78EFP_QRvLA off=13755320 len=1167430 2024-09-13T00:07:50.541 INFO:tasks.workunit.client.0.smithi136.stdout:op 2001 completed, throughput=5MB/sec 2024-09-13T00:07:51.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:07:50 smithi136 bash[75164]: cluster 2024-09-13T00:07:49.339192+0000 mgr.y (mgr.24661) 921 : cluster [DBG] pgmap v1604: 105 pgs: 105 active+clean; 126 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:07:51.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:07:50 smithi136 bash[76371]: cluster 2024-09-13T00:07:49.339192+0000 mgr.y (mgr.24661) 921 : cluster [DBG] pgmap v1604: 105 pgs: 105 active+clean; 126 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:07:51.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:07:50 smithi181 bash[67587]: cluster 2024-09-13T00:07:49.339192+0000 mgr.y (mgr.24661) 921 : cluster [DBG] pgmap v1604: 105 pgs: 105 active+clean; 126 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:07:51.541 INFO:tasks.workunit.client.0.smithi136.stdout: 376: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:51.542 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-laNUucGT-XqZrQk off=7691429 len=1309631 2024-09-13T00:07:51.545 INFO:tasks.workunit.client.0.smithi136.stdout:op 2002 completed, throughput=4.99MB/sec 2024-09-13T00:07:51.545 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Fid34LFAeff0vZY off=3587054 len=1560916 2024-09-13T00:07:51.556 INFO:tasks.workunit.client.0.smithi136.stdout:op 2003 completed, throughput=5MB/sec 2024-09-13T00:07:51.556 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-kBUZd73oGRnMIHm off=7930809 len=644950 2024-09-13T00:07:51.558 INFO:tasks.workunit.client.0.smithi136.stdout:op 2004 completed, throughput=5MB/sec 2024-09-13T00:07:51.558 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-hqUlKtJ4vv0mhnu off=16357141 len=545293 2024-09-13T00:07:51.565 INFO:tasks.workunit.client.0.smithi136.stdout:op 2005 completed, throughput=5MB/sec 2024-09-13T00:07:51.565 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Hc65hjG-yZmnk3- off=12778624 len=822516 2024-09-13T00:07:51.567 INFO:tasks.workunit.client.0.smithi136.stdout:op 2006 completed, throughput=5MB/sec 2024-09-13T00:07:52.569 INFO:tasks.workunit.client.0.smithi136.stdout: 377: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:52.569 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-IVESCAS-NGoJWtf off=7311190 len=1805420 2024-09-13T00:07:52.584 INFO:tasks.workunit.client.0.smithi136.stdout:op 2007 completed, throughput=4.99MB/sec 2024-09-13T00:07:52.584 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-q5lH8cqic_ljnBV off=5107242 len=717838 2024-09-13T00:07:52.592 INFO:tasks.workunit.client.0.smithi136.stdout:op 2008 completed, throughput=4.99MB/sec 2024-09-13T00:07:52.592 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-d1GwKnj18L9FrlX off=3294286 len=1608725 2024-09-13T00:07:52.595 INFO:tasks.workunit.client.0.smithi136.stdout:op 2009 completed, throughput=5MB/sec 2024-09-13T00:07:52.596 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-a2BOEECtcZ88gPA off=2908248 len=1675699 2024-09-13T00:07:52.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:07:52 smithi181 bash[95505]: ts=2024-09-13T00:07:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:07:52.599 INFO:tasks.workunit.client.0.smithi136.stdout:op 2010 completed, throughput=5MB/sec 2024-09-13T00:07:53.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:07:52 smithi136 bash[76371]: cluster 2024-09-13T00:07:51.340170+0000 mgr.y (mgr.24661) 922 : cluster [DBG] pgmap v1605: 105 pgs: 105 active+clean; 126 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:07:53.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:07:52 smithi136 bash[75164]: cluster 2024-09-13T00:07:51.340170+0000 mgr.y (mgr.24661) 922 : cluster [DBG] pgmap v1605: 105 pgs: 105 active+clean; 126 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:07:53.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:07:52 smithi181 bash[67587]: cluster 2024-09-13T00:07:51.340170+0000 mgr.y (mgr.24661) 922 : cluster [DBG] pgmap v1605: 105 pgs: 105 active+clean; 126 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:07:53.600 INFO:tasks.workunit.client.0.smithi136.stdout: 378: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:53.600 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ik6q6vNIif-4Rgm off=628365 len=969309 2024-09-13T00:07:53.608 INFO:tasks.workunit.client.0.smithi136.stdout:op 2011 completed, throughput=4.99MB/sec 2024-09-13T00:07:53.608 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-5lx5XcNTpNaTPGJ off=2162569 len=960582 2024-09-13T00:07:53.616 INFO:tasks.workunit.client.0.smithi136.stdout:op 2012 completed, throughput=4.99MB/sec 2024-09-13T00:07:53.616 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-k8K_nY5iJrLGvd7 off=17674103 len=2096919 2024-09-13T00:07:53.621 INFO:tasks.workunit.client.0.smithi136.stdout:op 2013 completed, throughput=5MB/sec 2024-09-13T00:07:53.621 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-tJYfhJR6so6Cken off=6288597 len=2088743 2024-09-13T00:07:53.634 INFO:tasks.workunit.client.0.smithi136.stdout:op 2014 completed, throughput=5MB/sec 2024-09-13T00:07:54.635 INFO:tasks.workunit.client.0.smithi136.stdout: 379: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:54.635 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-c7m-fq6LvcGE8v7 off=1407264 len=1229608 2024-09-13T00:07:54.645 INFO:tasks.workunit.client.0.smithi136.stdout:op 2015 completed, throughput=4.99MB/sec 2024-09-13T00:07:54.645 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-lZ7dCJ3oSXrWPbE off=5921993 len=342470 2024-09-13T00:07:54.651 INFO:tasks.workunit.client.0.smithi136.stdout:op 2016 completed, throughput=4.99MB/sec 2024-09-13T00:07:54.651 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-GergtG53LLJH0pG off=332237 len=1711739 2024-09-13T00:07:54.663 INFO:tasks.workunit.client.0.smithi136.stdout:op 2017 completed, throughput=5MB/sec 2024-09-13T00:07:54.663 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ao6YzqPXIu7aV0o off=12473507 len=190504 2024-09-13T00:07:54.665 INFO:tasks.workunit.client.0.smithi136.stdout:op 2018 completed, throughput=5MB/sec 2024-09-13T00:07:54.665 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-7yRgPXVnvOVTO6M off=5708278 len=851809 2024-09-13T00:07:54.672 INFO:tasks.workunit.client.0.smithi136.stdout:op 2019 completed, throughput=5MB/sec 2024-09-13T00:07:55.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:07:54 smithi136 bash[76371]: cluster 2024-09-13T00:07:53.341270+0000 mgr.y (mgr.24661) 923 : cluster [DBG] pgmap v1606: 105 pgs: 105 active+clean; 126 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 2.2 MiB/s rd, 3.0 MiB/s wr, 5 op/s 2024-09-13T00:07:55.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:07:54 smithi136 bash[76371]: audit 2024-09-13T00:07:53.782414+0000 mon.a (mon.0) 2166 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:07:55.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:07:54 smithi136 bash[75164]: cluster 2024-09-13T00:07:53.341270+0000 mgr.y (mgr.24661) 923 : cluster [DBG] pgmap v1606: 105 pgs: 105 active+clean; 126 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 2.2 MiB/s rd, 3.0 MiB/s wr, 5 op/s 2024-09-13T00:07:55.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:07:54 smithi136 bash[75164]: audit 2024-09-13T00:07:53.782414+0000 mon.a (mon.0) 2166 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:07:55.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:07:54 smithi181 bash[67587]: cluster 2024-09-13T00:07:53.341270+0000 mgr.y (mgr.24661) 923 : cluster [DBG] pgmap v1606: 105 pgs: 105 active+clean; 126 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 2.2 MiB/s rd, 3.0 MiB/s wr, 5 op/s 2024-09-13T00:07:55.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:07:54 smithi181 bash[67587]: audit 2024-09-13T00:07:53.782414+0000 mon.a (mon.0) 2166 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:07:55.672 INFO:tasks.workunit.client.0.smithi136.stdout: 380: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:55.672 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-inFa4fMhO9Bxrft off=2128532 len=1390231 2024-09-13T00:07:55.684 INFO:tasks.workunit.client.0.smithi136.stdout:op 2020 completed, throughput=4.99MB/sec 2024-09-13T00:07:55.684 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-NKg3izkQRbSM90b off=5014515 len=733697 2024-09-13T00:07:55.692 INFO:tasks.workunit.client.0.smithi136.stdout:op 2021 completed, throughput=4.99MB/sec 2024-09-13T00:07:55.692 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ACsOPEHhq13FNPA off=3592945 len=205968 2024-09-13T00:07:55.694 INFO:tasks.workunit.client.0.smithi136.stdout:op 2022 completed, throughput=4.99MB/sec 2024-09-13T00:07:55.694 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-oD_YzkSxgDmJGUL off=1343350 len=530129 2024-09-13T00:07:55.700 INFO:tasks.workunit.client.0.smithi136.stdout:op 2023 completed, throughput=4.99MB/sec 2024-09-13T00:07:55.700 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-thZpwV_7h-jUk6r off=3838952 len=1567153 2024-09-13T00:07:55.710 INFO:tasks.workunit.client.0.smithi136.stdout:op 2024 completed, throughput=5MB/sec 2024-09-13T00:07:55.711 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-z5BUxtex-ynZFcw off=24197994 len=141648 2024-09-13T00:07:55.715 INFO:tasks.workunit.client.0.smithi136.stdout:op 2025 completed, throughput=5MB/sec 2024-09-13T00:07:55.716 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-4LYAOSk1hqZ9OCw off=222982 len=781147 2024-09-13T00:07:55.718 INFO:tasks.workunit.client.0.smithi136.stdout:op 2026 completed, throughput=5MB/sec 2024-09-13T00:07:56.719 INFO:tasks.workunit.client.0.smithi136.stdout: 381: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:56.719 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Vs_GJZISHrz18oA off=1838392 len=1443062 2024-09-13T00:07:56.722 INFO:tasks.workunit.client.0.smithi136.stdout:op 2027 completed, throughput=4.99MB/sec 2024-09-13T00:07:56.723 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-8ViXIzZDRX7u4Og off=2419345 len=2037944 2024-09-13T00:07:56.739 INFO:tasks.workunit.client.0.smithi136.stdout:op 2028 completed, throughput=5MB/sec 2024-09-13T00:07:56.739 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-msCTyVftx6wfCU9 off=8166961 len=1110653 2024-09-13T00:07:56.748 INFO:tasks.workunit.client.0.smithi136.stdout:op 2029 completed, throughput=5MB/sec 2024-09-13T00:07:56.748 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-3i-5g79CH5G-_bK off=13062516 len=1307894 2024-09-13T00:07:56.756 INFO:tasks.workunit.client.0.smithi136.stdout:op 2030 completed, throughput=5MB/sec 2024-09-13T00:07:57.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:07:56 smithi181 bash[67587]: cluster 2024-09-13T00:07:55.342171+0000 mgr.y (mgr.24661) 924 : cluster [DBG] pgmap v1607: 105 pgs: 105 active+clean; 126 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 1.9 MiB/s rd, 2.8 MiB/s wr, 4 op/s 2024-09-13T00:07:57.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:07:56 smithi136 bash[75164]: cluster 2024-09-13T00:07:55.342171+0000 mgr.y (mgr.24661) 924 : cluster [DBG] pgmap v1607: 105 pgs: 105 active+clean; 126 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 1.9 MiB/s rd, 2.8 MiB/s wr, 4 op/s 2024-09-13T00:07:57.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:07:56 smithi136 bash[76371]: cluster 2024-09-13T00:07:55.342171+0000 mgr.y (mgr.24661) 924 : cluster [DBG] pgmap v1607: 105 pgs: 105 active+clean; 126 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 1.9 MiB/s rd, 2.8 MiB/s wr, 4 op/s 2024-09-13T00:07:57.757 INFO:tasks.workunit.client.0.smithi136.stdout: 382: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:57.757 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-zh11NPNZ11pqSiO off=10455255 len=1125114 2024-09-13T00:07:57.760 INFO:tasks.workunit.client.0.smithi136.stdout:op 2031 completed, throughput=4.99MB/sec 2024-09-13T00:07:57.760 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-QPPD3QsLL1pOyaH off=17540761 len=1239073 2024-09-13T00:07:57.764 INFO:tasks.workunit.client.0.smithi136.stdout:op 2032 completed, throughput=4.99MB/sec 2024-09-13T00:07:57.765 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-x7f2IfEqOeFlqVH off=1082455 len=539201 2024-09-13T00:07:57.767 INFO:tasks.workunit.client.0.smithi136.stdout:op 2033 completed, throughput=5MB/sec 2024-09-13T00:07:57.767 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ZUCOTuXgrJKobt3 off=6142476 len=498315 2024-09-13T00:07:57.769 INFO:tasks.workunit.client.0.smithi136.stdout:op 2034 completed, throughput=5MB/sec 2024-09-13T00:07:57.769 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ESNYBTl4ukPIyzg off=3538142 len=1050480 2024-09-13T00:07:57.778 INFO:tasks.workunit.client.0.smithi136.stdout:op 2035 completed, throughput=5MB/sec 2024-09-13T00:07:57.778 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-TP5lvKm1vix5Gim off=4327558 len=221434 2024-09-13T00:07:57.780 INFO:tasks.workunit.client.0.smithi136.stdout:op 2036 completed, throughput=5MB/sec 2024-09-13T00:07:58.780 INFO:tasks.workunit.client.0.smithi136.stdout: 383: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:58.780 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-aAt-qsr4JjXshLc off=786508 len=1756094 2024-09-13T00:07:58.792 INFO:tasks.workunit.client.0.smithi136.stdout:op 2037 completed, throughput=4.99MB/sec 2024-09-13T00:07:58.792 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-5G4RHm6KSA6nlR- off=23118981 len=1085712 2024-09-13T00:07:58.802 INFO:tasks.workunit.client.0.smithi136.stdout:op 2038 completed, throughput=4.99MB/sec 2024-09-13T00:07:58.802 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-nRWrgBMxhpzvg7- off=4661193 len=144544 2024-09-13T00:07:58.803 INFO:tasks.workunit.client.0.smithi136.stdout:op 2039 completed, throughput=4.99MB/sec 2024-09-13T00:07:58.803 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Gl4v1l_mlgOykX6 off=1434163 len=501057 2024-09-13T00:07:58.805 INFO:tasks.workunit.client.0.smithi136.stdout:op 2040 completed, throughput=5MB/sec 2024-09-13T00:07:58.805 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-V5oijc1KrV1era5 off=1963522 len=1304789 2024-09-13T00:07:58.814 INFO:tasks.workunit.client.0.smithi136.stdout:op 2041 completed, throughput=5MB/sec 2024-09-13T00:07:58.814 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-u7xE4SJ2CItSh82 off=6180467 len=1536602 2024-09-13T00:07:58.824 INFO:tasks.workunit.client.0.smithi136.stdout:op 2042 completed, throughput=5MB/sec 2024-09-13T00:07:59.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:07:58 smithi181 bash[67587]: cluster 2024-09-13T00:07:57.343310+0000 mgr.y (mgr.24661) 925 : cluster [DBG] pgmap v1608: 105 pgs: 105 active+clean; 126 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:07:59.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:07:58 smithi136 bash[75164]: cluster 2024-09-13T00:07:57.343310+0000 mgr.y (mgr.24661) 925 : cluster [DBG] pgmap v1608: 105 pgs: 105 active+clean; 126 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:07:59.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:07:58 smithi136 bash[76371]: cluster 2024-09-13T00:07:57.343310+0000 mgr.y (mgr.24661) 925 : cluster [DBG] pgmap v1608: 105 pgs: 105 active+clean; 126 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:07:59.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:07:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:07:59] "GET /metrics HTTP/1.1" 200 34136 "" "Prometheus/2.43.0" 2024-09-13T00:07:59.824 INFO:tasks.workunit.client.0.smithi136.stdout: 384: throughput=4.99MB/sec pending data=0 2024-09-13T00:07:59.825 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-vJPwHbCCLOUiM0i off=2643525 len=774814 2024-09-13T00:07:59.827 INFO:tasks.workunit.client.0.smithi136.stdout:op 2043 completed, throughput=4.99MB/sec 2024-09-13T00:07:59.827 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-K9-2nwh1i729Ix_ off=5789392 len=1832832 2024-09-13T00:07:59.832 INFO:tasks.workunit.client.0.smithi136.stdout:op 2044 completed, throughput=5MB/sec 2024-09-13T00:07:59.832 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-NVZN48m_p_fgQe_ off=1889102 len=1092974 2024-09-13T00:07:59.840 INFO:tasks.workunit.client.0.smithi136.stdout:op 2045 completed, throughput=5MB/sec 2024-09-13T00:07:59.840 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-KQsOJzI-ssTmwV1 off=3131190 len=1309277 2024-09-13T00:07:59.842 INFO:tasks.workunit.client.0.smithi136.stdout:op 2046 completed, throughput=5MB/sec 2024-09-13T00:08:00.843 INFO:tasks.workunit.client.0.smithi136.stdout: 385: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:00.843 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-1iuX0fJrcJ8T_8m off=666060 len=1505487 2024-09-13T00:08:00.847 INFO:tasks.workunit.client.0.smithi136.stdout:op 2047 completed, throughput=4.99MB/sec 2024-09-13T00:08:00.847 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Hxu55CHaEmvwK7A off=1110584 len=839551 2024-09-13T00:08:00.849 INFO:tasks.workunit.client.0.smithi136.stdout:op 2048 completed, throughput=4.99MB/sec 2024-09-13T00:08:00.850 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-OHW5mevbbCf0kXx off=15329642 len=1221577 2024-09-13T00:08:00.853 INFO:tasks.workunit.client.0.smithi136.stdout:op 2049 completed, throughput=5MB/sec 2024-09-13T00:08:00.853 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-7RO6-v8HFP8WrAG off=42527 len=359263 2024-09-13T00:08:00.859 INFO:tasks.workunit.client.0.smithi136.stdout:op 2050 completed, throughput=5MB/sec 2024-09-13T00:08:00.860 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-MEmLaWJlLvvBHvx off=9692227 len=570190 2024-09-13T00:08:00.866 INFO:tasks.workunit.client.0.smithi136.stdout:op 2051 completed, throughput=5MB/sec 2024-09-13T00:08:00.866 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-inkMeAq38uTU_hh off=783498 len=1184927 2024-09-13T00:08:00.869 INFO:tasks.workunit.client.0.smithi136.stdout:op 2052 completed, throughput=5MB/sec 2024-09-13T00:08:01.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:08:00 smithi181 bash[67587]: cluster 2024-09-13T00:07:59.344190+0000 mgr.y (mgr.24661) 926 : cluster [DBG] pgmap v1609: 105 pgs: 105 active+clean; 126 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 1.5 MiB/s rd, 3.3 MiB/s wr, 4 op/s 2024-09-13T00:08:01.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:08:00 smithi136 bash[75164]: cluster 2024-09-13T00:07:59.344190+0000 mgr.y (mgr.24661) 926 : cluster [DBG] pgmap v1609: 105 pgs: 105 active+clean; 126 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 1.5 MiB/s rd, 3.3 MiB/s wr, 4 op/s 2024-09-13T00:08:01.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:08:00 smithi136 bash[76371]: cluster 2024-09-13T00:07:59.344190+0000 mgr.y (mgr.24661) 926 : cluster [DBG] pgmap v1609: 105 pgs: 105 active+clean; 126 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 1.5 MiB/s rd, 3.3 MiB/s wr, 4 op/s 2024-09-13T00:08:01.870 INFO:tasks.workunit.client.0.smithi136.stdout: 386: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:01.870 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-OpV5QAl7LZrki52 off=867499 len=613318 2024-09-13T00:08:01.877 INFO:tasks.workunit.client.0.smithi136.stdout:op 2053 completed, throughput=4.99MB/sec 2024-09-13T00:08:01.877 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-fdNJBs3IdrqIDVo off=3758211 len=480196 2024-09-13T00:08:01.879 INFO:tasks.workunit.client.0.smithi136.stdout:op 2054 completed, throughput=4.99MB/sec 2024-09-13T00:08:01.879 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-qYKiolwFAPz0Rmj off=109315 len=1149742 2024-09-13T00:08:01.888 INFO:tasks.workunit.client.0.smithi136.stdout:op 2055 completed, throughput=4.99MB/sec 2024-09-13T00:08:01.888 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-SXWBho90dK7ZpRT off=2818724 len=1694978 2024-09-13T00:08:01.891 INFO:tasks.workunit.client.0.smithi136.stdout:op 2056 completed, throughput=5MB/sec 2024-09-13T00:08:01.891 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-KENefUO8qIoNso6 off=4596574 len=618521 2024-09-13T00:08:01.894 INFO:tasks.workunit.client.0.smithi136.stdout:op 2057 completed, throughput=5MB/sec 2024-09-13T00:08:02.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:08:02 smithi181 bash[95505]: ts=2024-09-13T00:08:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:08:02.894 INFO:tasks.workunit.client.0.smithi136.stdout: 387: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:02.895 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-B6NOcSZuE7bJZ98 off=7743105 len=85198 2024-09-13T00:08:02.899 INFO:tasks.workunit.client.0.smithi136.stdout:op 2058 completed, throughput=4.99MB/sec 2024-09-13T00:08:02.899 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-g80HB3amvmMgntu off=16671177 len=517316 2024-09-13T00:08:02.901 INFO:tasks.workunit.client.0.smithi136.stdout:op 2059 completed, throughput=4.99MB/sec 2024-09-13T00:08:02.901 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-FjnVfTETpbDvO7t off=3932900 len=1356210 2024-09-13T00:08:02.912 INFO:tasks.workunit.client.0.smithi136.stdout:op 2060 completed, throughput=4.99MB/sec 2024-09-13T00:08:02.912 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-mBYhuUOzqxbQ1Cl off=1221331 len=778815 2024-09-13T00:08:02.919 INFO:tasks.workunit.client.0.smithi136.stdout:op 2061 completed, throughput=4.99MB/sec 2024-09-13T00:08:02.919 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-j4dcYQCbDPRa8SZ off=1746192 len=485616 2024-09-13T00:08:02.921 INFO:tasks.workunit.client.0.smithi136.stdout:op 2062 completed, throughput=5MB/sec 2024-09-13T00:08:02.921 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-3Tm9yj_IkfrrdBl off=4244533 len=1939604 2024-09-13T00:08:02.925 INFO:tasks.workunit.client.0.smithi136.stdout:op 2063 completed, throughput=5MB/sec 2024-09-13T00:08:02.925 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-iC4bf4s8l_pT9DG off=12826353 len=1151669 2024-09-13T00:08:02.928 INFO:tasks.workunit.client.0.smithi136.stdout:op 2064 completed, throughput=5MB/sec 2024-09-13T00:08:03.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:08:02 smithi181 bash[67587]: cluster 2024-09-13T00:08:01.345208+0000 mgr.y (mgr.24661) 927 : cluster [DBG] pgmap v1610: 105 pgs: 105 active+clean; 126 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.3 MiB/s wr, 4 op/s 2024-09-13T00:08:03.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:08:02 smithi136 bash[75164]: cluster 2024-09-13T00:08:01.345208+0000 mgr.y (mgr.24661) 927 : cluster [DBG] pgmap v1610: 105 pgs: 105 active+clean; 126 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.3 MiB/s wr, 4 op/s 2024-09-13T00:08:03.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:08:02 smithi136 bash[76371]: cluster 2024-09-13T00:08:01.345208+0000 mgr.y (mgr.24661) 927 : cluster [DBG] pgmap v1610: 105 pgs: 105 active+clean; 126 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.3 MiB/s wr, 4 op/s 2024-09-13T00:08:03.929 INFO:tasks.workunit.client.0.smithi136.stdout: 388: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:03.929 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-HaIduPJgnceVcRi off=4807506 len=28892 2024-09-13T00:08:03.930 INFO:tasks.workunit.client.0.smithi136.stdout:op 2065 completed, throughput=4.99MB/sec 2024-09-13T00:08:03.930 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-OQLw0OZVVNInFMC off=11785904 len=1377669 2024-09-13T00:08:03.933 INFO:tasks.workunit.client.0.smithi136.stdout:op 2066 completed, throughput=4.99MB/sec 2024-09-13T00:08:03.934 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-T69IC5DcQ3UNFKR off=1468771 len=304012 2024-09-13T00:08:03.935 INFO:tasks.workunit.client.0.smithi136.stdout:op 2067 completed, throughput=4.99MB/sec 2024-09-13T00:08:03.935 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-cGYw-uVIicUl9AL off=15926797 len=988749 2024-09-13T00:08:03.944 INFO:tasks.workunit.client.0.smithi136.stdout:op 2068 completed, throughput=5MB/sec 2024-09-13T00:08:03.944 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-XQmSomUtsVU8qZZ off=10795978 len=1675988 2024-09-13T00:08:03.949 INFO:tasks.workunit.client.0.smithi136.stdout:op 2069 completed, throughput=5MB/sec 2024-09-13T00:08:04.949 INFO:tasks.workunit.client.0.smithi136.stdout: 389: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:04.949 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-JUZMyCtEg7OYyUx off=6837114 len=1917049 2024-09-13T00:08:04.962 INFO:tasks.workunit.client.0.smithi136.stdout:op 2070 completed, throughput=4.99MB/sec 2024-09-13T00:08:04.962 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-QTHKNOKVLTQvqxA off=17057285 len=1163299 2024-09-13T00:08:04.965 INFO:tasks.workunit.client.0.smithi136.stdout:op 2071 completed, throughput=4.99MB/sec 2024-09-13T00:08:04.965 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-kOuZpsp9pV38EIo off=5649391 len=1606669 2024-09-13T00:08:04.970 INFO:tasks.workunit.client.0.smithi136.stdout:op 2072 completed, throughput=5MB/sec 2024-09-13T00:08:04.971 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_BgNvfha8Ufw_90 off=4172266 len=897996 2024-09-13T00:08:04.979 INFO:tasks.workunit.client.0.smithi136.stdout:op 2073 completed, throughput=5MB/sec 2024-09-13T00:08:05.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:08:04 smithi181 bash[67587]: cluster 2024-09-13T00:08:03.346288+0000 mgr.y (mgr.24661) 928 : cluster [DBG] pgmap v1611: 105 pgs: 105 active+clean; 126 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 2.2 MiB/s rd, 3.1 MiB/s wr, 4 op/s 2024-09-13T00:08:05.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:08:04 smithi136 bash[75164]: cluster 2024-09-13T00:08:03.346288+0000 mgr.y (mgr.24661) 928 : cluster [DBG] pgmap v1611: 105 pgs: 105 active+clean; 126 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 2.2 MiB/s rd, 3.1 MiB/s wr, 4 op/s 2024-09-13T00:08:05.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:08:04 smithi136 bash[76371]: cluster 2024-09-13T00:08:03.346288+0000 mgr.y (mgr.24661) 928 : cluster [DBG] pgmap v1611: 105 pgs: 105 active+clean; 126 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 2.2 MiB/s rd, 3.1 MiB/s wr, 4 op/s 2024-09-13T00:08:05.980 INFO:tasks.workunit.client.0.smithi136.stdout: 390: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:05.980 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-34bIIm75U_aUk9q off=7002201 len=1356790 2024-09-13T00:08:05.990 INFO:tasks.workunit.client.0.smithi136.stdout:op 2074 completed, throughput=4.99MB/sec 2024-09-13T00:08:05.990 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-KqDYQrgOZPQPFse off=10874152 len=600676 2024-09-13T00:08:05.993 INFO:tasks.workunit.client.0.smithi136.stdout:op 2075 completed, throughput=4.99MB/sec 2024-09-13T00:08:05.993 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-B3eDzD5-3h-etKk off=2221758 len=757201 2024-09-13T00:08:05.995 INFO:tasks.workunit.client.0.smithi136.stdout:op 2076 completed, throughput=4.99MB/sec 2024-09-13T00:08:05.995 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-C6m44DVZXySY-0S off=3399372 len=910994 2024-09-13T00:08:05.998 INFO:tasks.workunit.client.0.smithi136.stdout:op 2077 completed, throughput=5MB/sec 2024-09-13T00:08:05.998 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-kq2Hf4D9yCKN91E off=523959 len=454225 2024-09-13T00:08:06.004 INFO:tasks.workunit.client.0.smithi136.stdout:op 2078 completed, throughput=5MB/sec 2024-09-13T00:08:06.004 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-m8KCOh4ZQQj8qsF off=1863618 len=1039115 2024-09-13T00:08:06.007 INFO:tasks.workunit.client.0.smithi136.stdout:op 2079 completed, throughput=5MB/sec 2024-09-13T00:08:06.007 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-oxNA410hJSmsek7 off=3860746 len=1796125 2024-09-13T00:08:06.019 INFO:tasks.workunit.client.0.smithi136.stdout:op 2080 completed, throughput=5MB/sec 2024-09-13T00:08:07.019 INFO:tasks.workunit.client.0.smithi136.stdout: 391: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:07.020 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-bHn9RXUUE5wogq0 off=1970292 len=1140794 2024-09-13T00:08:07.028 INFO:tasks.workunit.client.0.smithi136.stdout:op 2081 completed, throughput=4.99MB/sec 2024-09-13T00:08:07.028 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-tfHWeIsFQTjKnNs off=22380771 len=198658 2024-09-13T00:08:07.033 INFO:tasks.workunit.client.0.smithi136.stdout:op 2082 completed, throughput=4.99MB/sec 2024-09-13T00:08:07.034 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-jSGmAb_ja4iw9dk off=4015832 len=301271 2024-09-13T00:08:07.035 INFO:tasks.workunit.client.0.smithi136.stdout:op 2083 completed, throughput=5MB/sec 2024-09-13T00:08:07.035 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-yNqOeqFsjD3upoQ off=286924 len=1552537 2024-09-13T00:08:07.039 INFO:tasks.workunit.client.0.smithi136.stdout:op 2084 completed, throughput=5MB/sec 2024-09-13T00:08:07.039 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-RJBdOL-F-Ru7EfX off=7701144 len=586656 2024-09-13T00:08:07.045 INFO:tasks.workunit.client.0.smithi136.stdout:op 2085 completed, throughput=5MB/sec 2024-09-13T00:08:07.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:08:06 smithi181 bash[67587]: cluster 2024-09-13T00:08:05.347276+0000 mgr.y (mgr.24661) 929 : cluster [DBG] pgmap v1612: 105 pgs: 105 active+clean; 126 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 1.9 MiB/s rd, 2.8 MiB/s wr, 4 op/s 2024-09-13T00:08:07.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:08:06 smithi136 bash[75164]: cluster 2024-09-13T00:08:05.347276+0000 mgr.y (mgr.24661) 929 : cluster [DBG] pgmap v1612: 105 pgs: 105 active+clean; 126 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 1.9 MiB/s rd, 2.8 MiB/s wr, 4 op/s 2024-09-13T00:08:07.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:08:06 smithi136 bash[76371]: cluster 2024-09-13T00:08:05.347276+0000 mgr.y (mgr.24661) 929 : cluster [DBG] pgmap v1612: 105 pgs: 105 active+clean; 126 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 1.9 MiB/s rd, 2.8 MiB/s wr, 4 op/s 2024-09-13T00:08:08.046 INFO:tasks.workunit.client.0.smithi136.stdout: 392: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:08.046 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-4aP6d38nIMuf7ed off=4275939 len=1348036 2024-09-13T00:08:08.049 INFO:tasks.workunit.client.0.smithi136.stdout:op 2086 completed, throughput=4.99MB/sec 2024-09-13T00:08:08.049 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-OtokW20moVI27yd off=7817823 len=66850 2024-09-13T00:08:08.054 INFO:tasks.workunit.client.0.smithi136.stdout:op 2087 completed, throughput=4.99MB/sec 2024-09-13T00:08:08.054 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-D5qYU_2KO8TyJyL off=2775034 len=1951845 2024-09-13T00:08:08.067 INFO:tasks.workunit.client.0.smithi136.stdout:op 2088 completed, throughput=5MB/sec 2024-09-13T00:08:08.068 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-FTvSlTGkjnv6GzF off=14473485 len=1999186 2024-09-13T00:08:08.083 INFO:tasks.workunit.client.0.smithi136.stdout:op 2089 completed, throughput=5MB/sec 2024-09-13T00:08:08.083 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-KUSEdjWWCOpctm9 off=1807961 len=1108824 2024-09-13T00:08:08.091 INFO:tasks.workunit.client.0.smithi136.stdout:op 2090 completed, throughput=5MB/sec 2024-09-13T00:08:09.092 INFO:tasks.workunit.client.0.smithi136.stdout: 393: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:09.092 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-PaDOeApMt5SDXhH off=341836 len=2019920 2024-09-13T00:08:09.097 INFO:tasks.workunit.client.0.smithi136.stdout:op 2091 completed, throughput=4.99MB/sec 2024-09-13T00:08:09.097 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-XO6fHAhzSidXOx3 off=7726445 len=1087470 2024-09-13T00:08:09.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:08:08 smithi181 bash[67587]: cluster 2024-09-13T00:08:07.348533+0000 mgr.y (mgr.24661) 930 : cluster [DBG] pgmap v1613: 105 pgs: 105 active+clean; 126 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 2.2 MiB/s rd, 3.0 MiB/s wr, 5 op/s 2024-09-13T00:08:09.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:08:08 smithi181 bash[67587]: audit 2024-09-13T00:08:08.782684+0000 mon.a (mon.0) 2167 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:08:09.106 INFO:tasks.workunit.client.0.smithi136.stdout:op 2092 completed, throughput=5MB/sec 2024-09-13T00:08:09.106 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-w966Rd-Kqo3XKIZ off=24427879 len=97724 2024-09-13T00:08:09.110 INFO:tasks.workunit.client.0.smithi136.stdout:op 2093 completed, throughput=5MB/sec 2024-09-13T00:08:09.110 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-cJaKgBtIEaa10oK off=5588416 len=33421 2024-09-13T00:08:09.111 INFO:tasks.workunit.client.0.smithi136.stdout:op 2094 completed, throughput=5MB/sec 2024-09-13T00:08:09.112 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-KD1x89qo0xMRn8e off=1849047 len=1433623 2024-09-13T00:08:09.115 INFO:tasks.workunit.client.0.smithi136.stdout:op 2095 completed, throughput=5MB/sec 2024-09-13T00:08:09.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:08:08 smithi136 bash[75164]: cluster 2024-09-13T00:08:07.348533+0000 mgr.y (mgr.24661) 930 : cluster [DBG] pgmap v1613: 105 pgs: 105 active+clean; 126 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 2.2 MiB/s rd, 3.0 MiB/s wr, 5 op/s 2024-09-13T00:08:09.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:08:08 smithi136 bash[75164]: audit 2024-09-13T00:08:08.782684+0000 mon.a (mon.0) 2167 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:08:09.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:08:08 smithi136 bash[76371]: cluster 2024-09-13T00:08:07.348533+0000 mgr.y (mgr.24661) 930 : cluster [DBG] pgmap v1613: 105 pgs: 105 active+clean; 126 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 2.2 MiB/s rd, 3.0 MiB/s wr, 5 op/s 2024-09-13T00:08:09.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:08:08 smithi136 bash[76371]: audit 2024-09-13T00:08:08.782684+0000 mon.a (mon.0) 2167 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:08:09.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:08:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:08:09] "GET /metrics HTTP/1.1" 200 34141 "" "Prometheus/2.43.0" 2024-09-13T00:08:10.115 INFO:tasks.workunit.client.0.smithi136.stdout: 394: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:10.115 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-vatqpSaZAQqcQWw off=741351 len=1864864 2024-09-13T00:08:10.131 INFO:tasks.workunit.client.0.smithi136.stdout:op 2096 completed, throughput=4.99MB/sec 2024-09-13T00:08:10.131 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-bckcq3vLsXh9B4M off=5470104 len=1166880 2024-09-13T00:08:10.134 INFO:tasks.workunit.client.0.smithi136.stdout:op 2097 completed, throughput=5MB/sec 2024-09-13T00:08:10.134 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-7G8FOUyWH4E3SFL off=5614324 len=1699304 2024-09-13T00:08:10.146 INFO:tasks.workunit.client.0.smithi136.stdout:op 2098 completed, throughput=5MB/sec 2024-09-13T00:08:10.146 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-AqX_8pfH00biHpm off=808507 len=779619 2024-09-13T00:08:10.148 INFO:tasks.workunit.client.0.smithi136.stdout:op 2099 completed, throughput=5MB/sec 2024-09-13T00:08:11.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:08:10 smithi181 bash[67587]: cluster 2024-09-13T00:08:09.349491+0000 mgr.y (mgr.24661) 931 : cluster [DBG] pgmap v1614: 105 pgs: 105 active+clean; 126 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:08:11.149 INFO:tasks.workunit.client.0.smithi136.stdout: 395: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:11.149 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-IqkF4cLxGoBKUtF off=6949904 len=930294 2024-09-13T00:08:11.159 INFO:tasks.workunit.client.0.smithi136.stdout:op 2100 completed, throughput=4.99MB/sec 2024-09-13T00:08:11.159 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-qMFOPqGtQ4CmQqv off=4923245 len=647902 2024-09-13T00:08:11.166 INFO:tasks.workunit.client.0.smithi136.stdout:op 2101 completed, throughput=4.99MB/sec 2024-09-13T00:08:11.166 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ISTXWs3TCBVAGy9 off=9238787 len=1657508 2024-09-13T00:08:11.170 INFO:tasks.workunit.client.0.smithi136.stdout:op 2102 completed, throughput=5MB/sec 2024-09-13T00:08:11.170 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-xGpjMfCl7eQn68y off=1520105 len=1844154 2024-09-13T00:08:11.176 INFO:tasks.workunit.client.0.smithi136.stdout:op 2103 completed, throughput=5MB/sec 2024-09-13T00:08:11.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:08:10 smithi136 bash[76371]: cluster 2024-09-13T00:08:09.349491+0000 mgr.y (mgr.24661) 931 : cluster [DBG] pgmap v1614: 105 pgs: 105 active+clean; 126 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:08:11.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:08:10 smithi136 bash[75164]: cluster 2024-09-13T00:08:09.349491+0000 mgr.y (mgr.24661) 931 : cluster [DBG] pgmap v1614: 105 pgs: 105 active+clean; 126 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:08:12.177 INFO:tasks.workunit.client.0.smithi136.stdout: 396: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:12.177 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-9Ck2Sx5Juq8d5yu off=643854 len=1201098 2024-09-13T00:08:12.187 INFO:tasks.workunit.client.0.smithi136.stdout:op 2104 completed, throughput=4.99MB/sec 2024-09-13T00:08:12.187 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-XHdOsWbnaAUDu25 off=6675366 len=490322 2024-09-13T00:08:12.193 INFO:tasks.workunit.client.0.smithi136.stdout:op 2105 completed, throughput=4.99MB/sec 2024-09-13T00:08:12.193 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-KIki3xDFqIVsXCd off=9171193 len=1200016 2024-09-13T00:08:12.201 INFO:tasks.workunit.client.0.smithi136.stdout:op 2106 completed, throughput=4.99MB/sec 2024-09-13T00:08:12.202 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Yd1rfxa6tNBjARB off=98768 len=1589455 2024-09-13T00:08:12.205 INFO:tasks.workunit.client.0.smithi136.stdout:op 2107 completed, throughput=5MB/sec 2024-09-13T00:08:12.206 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-g80HB3amvmMgntu off=1178591 len=419661 2024-09-13T00:08:12.208 INFO:tasks.workunit.client.0.smithi136.stdout:op 2108 completed, throughput=5MB/sec 2024-09-13T00:08:12.208 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-hC7dmzbpEIGZVpM off=18402844 len=866968 2024-09-13T00:08:12.216 INFO:tasks.workunit.client.0.smithi136.stdout:op 2109 completed, throughput=5MB/sec 2024-09-13T00:08:12.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:08:12 smithi181 bash[95505]: ts=2024-09-13T00:08:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:08:13.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:08:12 smithi181 bash[67587]: cluster 2024-09-13T00:08:11.350491+0000 mgr.y (mgr.24661) 932 : cluster [DBG] pgmap v1615: 105 pgs: 105 active+clean; 126 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:08:13.216 INFO:tasks.workunit.client.0.smithi136.stdout: 397: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:13.217 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-iqRRXWYWYQQIL3y off=4637436 len=1715659 2024-09-13T00:08:13.228 INFO:tasks.workunit.client.0.smithi136.stdout:op 2110 completed, throughput=4.99MB/sec 2024-09-13T00:08:13.229 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-V6VOjGhCCVHD71S off=8808581 len=110272 2024-09-13T00:08:13.230 INFO:tasks.workunit.client.0.smithi136.stdout:op 2111 completed, throughput=4.99MB/sec 2024-09-13T00:08:13.230 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-vZXJ65J8bNmOwOO off=5194284 len=1710832 2024-09-13T00:08:13.240 INFO:tasks.workunit.client.0.smithi136.stdout:op 2112 completed, throughput=5MB/sec 2024-09-13T00:08:13.240 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-s9iXg1KICHdrMVW off=1464017 len=151406 2024-09-13T00:08:13.245 INFO:tasks.workunit.client.0.smithi136.stdout:op 2113 completed, throughput=5MB/sec 2024-09-13T00:08:13.246 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-QId5V3X3ko8cO1w off=13720348 len=81112 2024-09-13T00:08:13.247 INFO:tasks.workunit.client.0.smithi136.stdout:op 2114 completed, throughput=5MB/sec 2024-09-13T00:08:13.247 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-hj76XBIPySb5bqd off=0 len=1516301 2024-09-13T00:08:13.249 INFO:tasks.workunit.client.0.smithi136.stdout:op 2115 completed, throughput=5MB/sec 2024-09-13T00:08:13.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:08:12 smithi136 bash[75164]: cluster 2024-09-13T00:08:11.350491+0000 mgr.y (mgr.24661) 932 : cluster [DBG] pgmap v1615: 105 pgs: 105 active+clean; 126 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:08:13.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:08:12 smithi136 bash[76371]: cluster 2024-09-13T00:08:11.350491+0000 mgr.y (mgr.24661) 932 : cluster [DBG] pgmap v1615: 105 pgs: 105 active+clean; 126 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:08:14.249 INFO:tasks.workunit.client.0.smithi136.stdout: 398: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:14.250 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Bd5OqLfTUTzC71m off=2209103 len=300506 2024-09-13T00:08:14.256 INFO:tasks.workunit.client.0.smithi136.stdout:op 2116 completed, throughput=4.99MB/sec 2024-09-13T00:08:14.256 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-_gYKgghTl1lemZn off=854212 len=302532 2024-09-13T00:08:14.257 INFO:tasks.workunit.client.0.smithi136.stdout:op 2117 completed, throughput=4.99MB/sec 2024-09-13T00:08:14.257 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Hzsd4Uxr-OCBbGg off=2612922 len=38324 2024-09-13T00:08:14.258 INFO:tasks.workunit.client.0.smithi136.stdout:op 2118 completed, throughput=4.99MB/sec 2024-09-13T00:08:14.258 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ClLAB47l5peSoaC off=4555426 len=1878521 2024-09-13T00:08:14.270 INFO:tasks.workunit.client.0.smithi136.stdout:op 2119 completed, throughput=4.99MB/sec 2024-09-13T00:08:14.270 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-8WPoo_25VG3EIZN off=8599593 len=2012885 2024-09-13T00:08:14.276 INFO:tasks.workunit.client.0.smithi136.stdout:op 2120 completed, throughput=5MB/sec 2024-09-13T00:08:14.276 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-LJootBZ6gpA_8iI off=4747587 len=1794165 2024-09-13T00:08:14.278 INFO:tasks.workunit.client.0.smithi136.stdout:op 2121 completed, throughput=5MB/sec 2024-09-13T00:08:15.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:08:14 smithi181 bash[67587]: cluster 2024-09-13T00:08:13.351474+0000 mgr.y (mgr.24661) 933 : cluster [DBG] pgmap v1616: 105 pgs: 105 active+clean; 126 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:08:15.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:08:14 smithi136 bash[76371]: cluster 2024-09-13T00:08:13.351474+0000 mgr.y (mgr.24661) 933 : cluster [DBG] pgmap v1616: 105 pgs: 105 active+clean; 126 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:08:15.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:08:14 smithi136 bash[75164]: cluster 2024-09-13T00:08:13.351474+0000 mgr.y (mgr.24661) 933 : cluster [DBG] pgmap v1616: 105 pgs: 105 active+clean; 126 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:08:15.279 INFO:tasks.workunit.client.0.smithi136.stdout: 399: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:15.279 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-iXnXhl576Jwujaz off=5512431 len=1812337 2024-09-13T00:08:15.295 INFO:tasks.workunit.client.0.smithi136.stdout:op 2122 completed, throughput=4.99MB/sec 2024-09-13T00:08:15.295 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-iQQm9-qzZj3ejj5 off=1084687 len=1513098 2024-09-13T00:08:15.306 INFO:tasks.workunit.client.0.smithi136.stdout:op 2123 completed, throughput=5MB/sec 2024-09-13T00:08:15.306 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-P6yS3HWt60m32vU off=4992002 len=1139340 2024-09-13T00:08:15.317 INFO:tasks.workunit.client.0.smithi136.stdout:op 2124 completed, throughput=5MB/sec 2024-09-13T00:08:16.317 INFO:tasks.workunit.client.0.smithi136.stdout: 400: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:16.317 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-oolG5_DI_WAqUIi off=5027668 len=1011381 2024-09-13T00:08:16.321 INFO:tasks.workunit.client.0.smithi136.stdout:op 2125 completed, throughput=4.99MB/sec 2024-09-13T00:08:16.321 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-NoOYnLsSa9TTedu off=6688780 len=471473 2024-09-13T00:08:16.323 INFO:tasks.workunit.client.0.smithi136.stdout:op 2126 completed, throughput=4.99MB/sec 2024-09-13T00:08:16.323 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-oPnVTv381LVG16H off=6919913 len=1162295 2024-09-13T00:08:16.333 INFO:tasks.workunit.client.0.smithi136.stdout:op 2127 completed, throughput=4.99MB/sec 2024-09-13T00:08:16.333 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-4QsevQF7tyXNWJD off=5804805 len=592753 2024-09-13T00:08:16.335 INFO:tasks.workunit.client.0.smithi136.stdout:op 2128 completed, throughput=5MB/sec 2024-09-13T00:08:16.336 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-UzCvqwI6NscnLZl off=10773864 len=514346 2024-09-13T00:08:16.341 INFO:tasks.workunit.client.0.smithi136.stdout:op 2129 completed, throughput=5MB/sec 2024-09-13T00:08:16.341 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-8TXWbq4H0uJwsYT off=20021134 len=79735 2024-09-13T00:08:16.343 INFO:tasks.workunit.client.0.smithi136.stdout:op 2130 completed, throughput=5MB/sec 2024-09-13T00:08:16.343 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-LMZ6PxjGh6BWSZM off=948753 len=592896 2024-09-13T00:08:16.345 INFO:tasks.workunit.client.0.smithi136.stdout:op 2131 completed, throughput=5MB/sec 2024-09-13T00:08:16.345 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-qtxZD7vyFlM7NXt off=21105211 len=978902 2024-09-13T00:08:16.347 INFO:tasks.workunit.client.0.smithi136.stdout:op 2132 completed, throughput=5MB/sec 2024-09-13T00:08:17.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:08:16 smithi181 bash[67587]: cluster 2024-09-13T00:08:15.352343+0000 mgr.y (mgr.24661) 934 : cluster [DBG] pgmap v1617: 105 pgs: 105 active+clean; 126 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:08:17.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:08:16 smithi136 bash[75164]: cluster 2024-09-13T00:08:15.352343+0000 mgr.y (mgr.24661) 934 : cluster [DBG] pgmap v1617: 105 pgs: 105 active+clean; 126 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:08:17.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:08:16 smithi136 bash[76371]: cluster 2024-09-13T00:08:15.352343+0000 mgr.y (mgr.24661) 934 : cluster [DBG] pgmap v1617: 105 pgs: 105 active+clean; 126 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:08:17.348 INFO:tasks.workunit.client.0.smithi136.stdout: 401: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:17.348 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-r58ATQbtW66rRTg off=8423117 len=1763248 2024-09-13T00:08:17.353 INFO:tasks.workunit.client.0.smithi136.stdout:op 2133 completed, throughput=4.99MB/sec 2024-09-13T00:08:17.353 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-9EL0lB8mduMFMP6 off=6529000 len=803721 2024-09-13T00:08:17.356 INFO:tasks.workunit.client.0.smithi136.stdout:op 2134 completed, throughput=4.99MB/sec 2024-09-13T00:08:17.356 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-JGNStWKlhywFbxQ off=16779887 len=852883 2024-09-13T00:08:17.365 INFO:tasks.workunit.client.0.smithi136.stdout:op 2135 completed, throughput=5MB/sec 2024-09-13T00:08:17.365 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Y2HM5144tNHoRnF off=4858035 len=872284 2024-09-13T00:08:17.367 INFO:tasks.workunit.client.0.smithi136.stdout:op 2136 completed, throughput=5MB/sec 2024-09-13T00:08:17.368 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-YTXy7g2p8Wy6_k- off=11804554 len=709454 2024-09-13T00:08:17.375 INFO:tasks.workunit.client.0.smithi136.stdout:op 2137 completed, throughput=5MB/sec 2024-09-13T00:08:17.375 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-yOregddklDmTc-i off=14469518 len=948957 2024-09-13T00:08:17.382 INFO:tasks.workunit.client.0.smithi136.stdout:op 2138 completed, throughput=5MB/sec 2024-09-13T00:08:18.383 INFO:tasks.workunit.client.0.smithi136.stdout: 402: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:18.383 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Qil_B54oeMZt1vq off=14865702 len=1154467 2024-09-13T00:08:18.387 INFO:tasks.workunit.client.0.smithi136.stdout:op 2139 completed, throughput=4.99MB/sec 2024-09-13T00:08:18.387 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-C6Zkz-un-Y-hi4T off=3438254 len=41016 2024-09-13T00:08:18.388 INFO:tasks.workunit.client.0.smithi136.stdout:op 2140 completed, throughput=4.99MB/sec 2024-09-13T00:08:18.388 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Dag052dlY-rAVjn off=4122321 len=552337 2024-09-13T00:08:18.394 INFO:tasks.workunit.client.0.smithi136.stdout:op 2141 completed, throughput=4.99MB/sec 2024-09-13T00:08:18.394 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-a7uiVYbQ-knUgsW off=5753717 len=1968592 2024-09-13T00:08:18.411 INFO:tasks.workunit.client.0.smithi136.stdout:op 2142 completed, throughput=5MB/sec 2024-09-13T00:08:18.411 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-azeqk6IGI2kFEFF off=21997948 len=1918854 2024-09-13T00:08:18.415 INFO:tasks.workunit.client.0.smithi136.stdout:op 2143 completed, throughput=5MB/sec 2024-09-13T00:08:19.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:08:18 smithi181 bash[67587]: cluster 2024-09-13T00:08:17.353481+0000 mgr.y (mgr.24661) 935 : cluster [DBG] pgmap v1618: 105 pgs: 105 active+clean; 126 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:08:19.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:08:18 smithi136 bash[75164]: cluster 2024-09-13T00:08:17.353481+0000 mgr.y (mgr.24661) 935 : cluster [DBG] pgmap v1618: 105 pgs: 105 active+clean; 126 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:08:19.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:08:18 smithi136 bash[76371]: cluster 2024-09-13T00:08:17.353481+0000 mgr.y (mgr.24661) 935 : cluster [DBG] pgmap v1618: 105 pgs: 105 active+clean; 126 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:08:19.415 INFO:tasks.workunit.client.0.smithi136.stdout: 403: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:19.416 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-vGfQyG7YvRhF-Dq off=934743 len=1195924 2024-09-13T00:08:19.424 INFO:tasks.workunit.client.0.smithi136.stdout:op 2144 completed, throughput=4.99MB/sec 2024-09-13T00:08:19.424 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj--FR6ZnrIyq3vebz off=1756632 len=534858 2024-09-13T00:08:19.431 INFO:tasks.workunit.client.0.smithi136.stdout:op 2145 completed, throughput=4.99MB/sec 2024-09-13T00:08:19.431 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-rzVsdJyCKPuCXS_ off=4064435 len=478853 2024-09-13T00:08:19.433 INFO:tasks.workunit.client.0.smithi136.stdout:op 2146 completed, throughput=5MB/sec 2024-09-13T00:08:19.433 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-RoLx9Ljmzgo39K9 off=3478643 len=1941479 2024-09-13T00:08:19.446 INFO:tasks.workunit.client.0.smithi136.stdout:op 2147 completed, throughput=5MB/sec 2024-09-13T00:08:19.446 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-lVFX4Nl4zcvaCUI off=2484379 len=191620 2024-09-13T00:08:19.450 INFO:tasks.workunit.client.0.smithi136.stdout:op 2148 completed, throughput=5MB/sec 2024-09-13T00:08:19.450 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-2Jt9eAeNN4r2SG_ off=6794376 len=827925 2024-09-13T00:08:19.458 INFO:tasks.workunit.client.0.smithi136.stdout:op 2149 completed, throughput=5MB/sec 2024-09-13T00:08:19.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:08:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:08:19] "GET /metrics HTTP/1.1" 200 34141 "" "Prometheus/2.43.0" 2024-09-13T00:08:20.459 INFO:tasks.workunit.client.0.smithi136.stdout: 404: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:20.459 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-v856p8Uvo4482L5 off=329326 len=1675678 2024-09-13T00:08:20.463 INFO:tasks.workunit.client.0.smithi136.stdout:op 2150 completed, throughput=4.99MB/sec 2024-09-13T00:08:20.463 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-wg_kxB3Pm1cS7lH off=1339547 len=1264045 2024-09-13T00:08:20.474 INFO:tasks.workunit.client.0.smithi136.stdout:op 2151 completed, throughput=5MB/sec 2024-09-13T00:08:20.474 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-vAIiuSWLL-47e1H off=9288796 len=1106517 2024-09-13T00:08:20.484 INFO:tasks.workunit.client.0.smithi136.stdout:op 2152 completed, throughput=5MB/sec 2024-09-13T00:08:20.484 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-y6Ts2YZB0pEjFat off=9176637 len=895765 2024-09-13T00:08:20.487 INFO:tasks.workunit.client.0.smithi136.stdout:op 2153 completed, throughput=5MB/sec 2024-09-13T00:08:21.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:08:20 smithi181 bash[67587]: cluster 2024-09-13T00:08:19.354406+0000 mgr.y (mgr.24661) 936 : cluster [DBG] pgmap v1619: 105 pgs: 105 active+clean; 126 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:08:21.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:08:20 smithi136 bash[75164]: cluster 2024-09-13T00:08:19.354406+0000 mgr.y (mgr.24661) 936 : cluster [DBG] pgmap v1619: 105 pgs: 105 active+clean; 126 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:08:21.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:08:20 smithi136 bash[76371]: cluster 2024-09-13T00:08:19.354406+0000 mgr.y (mgr.24661) 936 : cluster [DBG] pgmap v1619: 105 pgs: 105 active+clean; 126 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:08:21.487 INFO:tasks.workunit.client.0.smithi136.stdout: 405: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:21.488 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Dyu73DI5uebT0B6 off=319838 len=1459256 2024-09-13T00:08:21.491 INFO:tasks.workunit.client.0.smithi136.stdout:op 2154 completed, throughput=4.99MB/sec 2024-09-13T00:08:21.491 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Aw1I2QVqxHWEK-P off=11941848 len=1919968 2024-09-13T00:08:21.496 INFO:tasks.workunit.client.0.smithi136.stdout:op 2155 completed, throughput=5MB/sec 2024-09-13T00:08:21.497 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-NYaxhc2hRSe73pw off=22639818 len=1740433 2024-09-13T00:08:21.503 INFO:tasks.workunit.client.0.smithi136.stdout:op 2156 completed, throughput=5MB/sec 2024-09-13T00:08:22.503 INFO:tasks.workunit.client.0.smithi136.stdout: 406: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:22.503 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-IioKpXoIgYFNkp5 off=4524218 len=1813741 2024-09-13T00:08:22.507 INFO:tasks.workunit.client.0.smithi136.stdout:op 2157 completed, throughput=4.99MB/sec 2024-09-13T00:08:22.508 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-QTmPdY_DBOmgTPS off=0 len=1083800 2024-09-13T00:08:22.517 INFO:tasks.workunit.client.0.smithi136.stdout:op 2158 completed, throughput=4.99MB/sec 2024-09-13T00:08:22.517 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ivz8ntzAp8kBC8H off=2486222 len=1732648 2024-09-13T00:08:22.520 INFO:tasks.workunit.client.0.smithi136.stdout:op 2159 completed, throughput=5MB/sec 2024-09-13T00:08:22.520 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-3JSF8GTWWI8zAGI off=13550307 len=86487 2024-09-13T00:08:22.522 INFO:tasks.workunit.client.0.smithi136.stdout:op 2160 completed, throughput=5MB/sec 2024-09-13T00:08:22.522 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-O0jEKYVzCmJHaWy off=17140494 len=238573 2024-09-13T00:08:22.523 INFO:tasks.workunit.client.0.smithi136.stdout:op 2161 completed, throughput=5MB/sec 2024-09-13T00:08:22.523 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-5FTZsD4jzLne8ak off=728184 len=2013579 2024-09-13T00:08:22.535 INFO:tasks.workunit.client.0.smithi136.stdout:op 2162 completed, throughput=5MB/sec 2024-09-13T00:08:22.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:08:22 smithi181 bash[95505]: ts=2024-09-13T00:08:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:08:23.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:08:22 smithi181 bash[67587]: cluster 2024-09-13T00:08:21.355431+0000 mgr.y (mgr.24661) 937 : cluster [DBG] pgmap v1620: 105 pgs: 105 active+clean; 126 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:08:23.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:08:22 smithi136 bash[75164]: cluster 2024-09-13T00:08:21.355431+0000 mgr.y (mgr.24661) 937 : cluster [DBG] pgmap v1620: 105 pgs: 105 active+clean; 126 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:08:23.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:08:22 smithi136 bash[76371]: cluster 2024-09-13T00:08:21.355431+0000 mgr.y (mgr.24661) 937 : cluster [DBG] pgmap v1620: 105 pgs: 105 active+clean; 126 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:08:23.536 INFO:tasks.workunit.client.0.smithi136.stdout: 407: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:23.536 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-lR9hcTcT4rVyM_- off=0 len=1203816 2024-09-13T00:08:23.544 INFO:tasks.workunit.client.0.smithi136.stdout:op 2163 completed, throughput=4.99MB/sec 2024-09-13T00:08:23.545 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-L4jza3YBJGDfJON off=242492 len=1600188 2024-09-13T00:08:23.548 INFO:tasks.workunit.client.0.smithi136.stdout:op 2164 completed, throughput=5MB/sec 2024-09-13T00:08:23.548 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-mRlTKLDGeLg7Trs off=705339 len=1689914 2024-09-13T00:08:23.564 INFO:tasks.workunit.client.0.smithi136.stdout:op 2165 completed, throughput=5MB/sec 2024-09-13T00:08:24.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:08:23 smithi181 bash[67587]: audit 2024-09-13T00:08:23.783155+0000 mon.a (mon.0) 2168 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:08:24.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:08:23 smithi136 bash[75164]: audit 2024-09-13T00:08:23.783155+0000 mon.a (mon.0) 2168 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:08:24.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:08:23 smithi136 bash[76371]: audit 2024-09-13T00:08:23.783155+0000 mon.a (mon.0) 2168 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:08:24.564 INFO:tasks.workunit.client.0.smithi136.stdout: 408: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:24.565 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-5aVakWad-zMxunN off=9431174 len=1369915 2024-09-13T00:08:24.577 INFO:tasks.workunit.client.0.smithi136.stdout:op 2166 completed, throughput=4.99MB/sec 2024-09-13T00:08:24.577 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-GjcB4hRlRdTto4P off=10422581 len=629087 2024-09-13T00:08:24.579 INFO:tasks.workunit.client.0.smithi136.stdout:op 2167 completed, throughput=4.99MB/sec 2024-09-13T00:08:24.579 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-v9jRkyaSjXsLSY9 off=8926065 len=1656394 2024-09-13T00:08:24.593 INFO:tasks.workunit.client.0.smithi136.stdout:op 2168 completed, throughput=5MB/sec 2024-09-13T00:08:24.593 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-oIeETv0tD01bMxP off=121560 len=1871704 2024-09-13T00:08:24.604 INFO:tasks.workunit.client.0.smithi136.stdout:op 2169 completed, throughput=5MB/sec 2024-09-13T00:08:25.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:08:24 smithi181 bash[67587]: cluster 2024-09-13T00:08:23.356383+0000 mgr.y (mgr.24661) 938 : cluster [DBG] pgmap v1621: 105 pgs: 105 active+clean; 126 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:08:25.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:08:24 smithi136 bash[75164]: cluster 2024-09-13T00:08:23.356383+0000 mgr.y (mgr.24661) 938 : cluster [DBG] pgmap v1621: 105 pgs: 105 active+clean; 126 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:08:25.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:08:24 smithi136 bash[76371]: cluster 2024-09-13T00:08:23.356383+0000 mgr.y (mgr.24661) 938 : cluster [DBG] pgmap v1621: 105 pgs: 105 active+clean; 126 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:08:25.605 INFO:tasks.workunit.client.0.smithi136.stdout: 409: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:25.605 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-fLBVyeFzdzi9x6_ off=9955141 len=1937790 2024-09-13T00:08:25.609 INFO:tasks.workunit.client.0.smithi136.stdout:op 2170 completed, throughput=4.99MB/sec 2024-09-13T00:08:25.609 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-rljAsnxGVtZvc-u off=4245826 len=340726 2024-09-13T00:08:25.615 INFO:tasks.workunit.client.0.smithi136.stdout:op 2171 completed, throughput=4.99MB/sec 2024-09-13T00:08:25.615 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-XZwiazcqkHUyL3E off=19597942 len=907031 2024-09-13T00:08:25.623 INFO:tasks.workunit.client.0.smithi136.stdout:op 2172 completed, throughput=5MB/sec 2024-09-13T00:08:25.623 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-IttH8EKQhtBKZbh off=1969927 len=1635151 2024-09-13T00:08:25.627 INFO:tasks.workunit.client.0.smithi136.stdout:op 2173 completed, throughput=5MB/sec 2024-09-13T00:08:26.627 INFO:tasks.workunit.client.0.smithi136.stdout: 410: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:26.627 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2tiq5zZUOifRzhY off=6170635 len=1850022 2024-09-13T00:08:26.631 INFO:tasks.workunit.client.0.smithi136.stdout:op 2174 completed, throughput=4.99MB/sec 2024-09-13T00:08:26.631 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-hNL9h0JHoZ4JyfX off=8247462 len=620376 2024-09-13T00:08:26.639 INFO:tasks.workunit.client.0.smithi136.stdout:op 2175 completed, throughput=4.99MB/sec 2024-09-13T00:08:26.639 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ebYkVuEVcfdZd5T off=3823955 len=1079403 2024-09-13T00:08:26.647 INFO:tasks.workunit.client.0.smithi136.stdout:op 2176 completed, throughput=5MB/sec 2024-09-13T00:08:26.647 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-CVYhOHdfexlW6vS off=7244690 len=212333 2024-09-13T00:08:26.652 INFO:tasks.workunit.client.0.smithi136.stdout:op 2177 completed, throughput=5MB/sec 2024-09-13T00:08:26.652 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-w9bO-JnN6LibDse off=122800 len=1907639 2024-09-13T00:08:26.668 INFO:tasks.workunit.client.0.smithi136.stdout:op 2178 completed, throughput=5MB/sec 2024-09-13T00:08:27.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:08:26 smithi181 bash[67587]: cluster 2024-09-13T00:08:25.357185+0000 mgr.y (mgr.24661) 939 : cluster [DBG] pgmap v1622: 105 pgs: 105 active+clean; 126 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:08:27.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:08:26 smithi136 bash[75164]: cluster 2024-09-13T00:08:25.357185+0000 mgr.y (mgr.24661) 939 : cluster [DBG] pgmap v1622: 105 pgs: 105 active+clean; 126 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:08:27.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:08:26 smithi136 bash[76371]: cluster 2024-09-13T00:08:25.357185+0000 mgr.y (mgr.24661) 939 : cluster [DBG] pgmap v1622: 105 pgs: 105 active+clean; 126 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:08:27.668 INFO:tasks.workunit.client.0.smithi136.stdout: 411: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:27.668 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-S5ngWrrbNdXG2b9 off=58443 len=1303562 2024-09-13T00:08:27.679 INFO:tasks.workunit.client.0.smithi136.stdout:op 2179 completed, throughput=4.99MB/sec 2024-09-13T00:08:27.680 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-YE4P7M0Sdvh7wgN off=2882289 len=1487049 2024-09-13T00:08:27.683 INFO:tasks.workunit.client.0.smithi136.stdout:op 2180 completed, throughput=5MB/sec 2024-09-13T00:08:27.683 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-GQ0QzMPUnKt6MK8 off=1797547 len=1465062 2024-09-13T00:08:27.686 INFO:tasks.workunit.client.0.smithi136.stdout:op 2181 completed, throughput=5MB/sec 2024-09-13T00:08:27.687 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-1WyegfiEeAj2VAv off=13296989 len=1440658 2024-09-13T00:08:27.696 INFO:tasks.workunit.client.0.smithi136.stdout:op 2182 completed, throughput=5MB/sec 2024-09-13T00:08:28.697 INFO:tasks.workunit.client.0.smithi136.stdout: 412: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:28.697 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ebYkVuEVcfdZd5T off=10146767 len=730595 2024-09-13T00:08:28.705 INFO:tasks.workunit.client.0.smithi136.stdout:op 2183 completed, throughput=4.99MB/sec 2024-09-13T00:08:28.705 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Y0IkuQgnVqAnqvJ off=1696124 len=972434 2024-09-13T00:08:28.713 INFO:tasks.workunit.client.0.smithi136.stdout:op 2184 completed, throughput=4.99MB/sec 2024-09-13T00:08:28.714 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-BneYOUyRI9O9uQ6 off=2080052 len=1599610 2024-09-13T00:08:28.720 INFO:tasks.workunit.client.0.smithi136.stdout:op 2185 completed, throughput=5MB/sec 2024-09-13T00:08:28.720 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-qAVnh1bbMxHXVv3 off=13719236 len=22822 2024-09-13T00:08:28.724 INFO:tasks.workunit.client.0.smithi136.stdout:op 2186 completed, throughput=5MB/sec 2024-09-13T00:08:28.725 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-kXthstME66Dk3ZZ off=1206204 len=1409534 2024-09-13T00:08:28.735 INFO:tasks.workunit.client.0.smithi136.stdout:op 2187 completed, throughput=5MB/sec 2024-09-13T00:08:29.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:08:28 smithi181 bash[67587]: cluster 2024-09-13T00:08:27.358394+0000 mgr.y (mgr.24661) 940 : cluster [DBG] pgmap v1623: 105 pgs: 105 active+clean; 126 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:08:29.265 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:08:28 smithi136 bash[75164]: cluster 2024-09-13T00:08:27.358394+0000 mgr.y (mgr.24661) 940 : cluster [DBG] pgmap v1623: 105 pgs: 105 active+clean; 126 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:08:29.266 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:08:28 smithi136 bash[76371]: cluster 2024-09-13T00:08:27.358394+0000 mgr.y (mgr.24661) 940 : cluster [DBG] pgmap v1623: 105 pgs: 105 active+clean; 126 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:08:29.736 INFO:tasks.workunit.client.0.smithi136.stdout: 413: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:29.736 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-j2TzLaPMmIlbD5C off=15734954 len=1785206 2024-09-13T00:08:29.740 INFO:tasks.workunit.client.0.smithi136.stdout:op 2188 completed, throughput=4.99MB/sec 2024-09-13T00:08:29.740 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-gedR78EFP_QRvLA off=15253014 len=1396915 2024-09-13T00:08:29.751 INFO:tasks.workunit.client.0.smithi136.stdout:op 2189 completed, throughput=5MB/sec 2024-09-13T00:08:29.751 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-amh0asVBqwcS5mh off=68575 len=92736 2024-09-13T00:08:29.752 INFO:tasks.workunit.client.0.smithi136.stdout:op 2190 completed, throughput=5MB/sec 2024-09-13T00:08:29.752 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-OK3sM3F6Gyj6gQz off=16498846 len=852098 2024-09-13T00:08:29.755 INFO:tasks.workunit.client.0.smithi136.stdout:op 2191 completed, throughput=5MB/sec 2024-09-13T00:08:29.755 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-kizzQAL7tWwHOPl off=6492394 len=1678087 2024-09-13T00:08:29.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:08:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:08:29] "GET /metrics HTTP/1.1" 200 34142 "" "Prometheus/2.43.0" 2024-09-13T00:08:29.765 INFO:tasks.workunit.client.0.smithi136.stdout:op 2192 completed, throughput=5MB/sec 2024-09-13T00:08:30.766 INFO:tasks.workunit.client.0.smithi136.stdout: 414: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:30.766 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-hcRxtb3iiZPNOgs off=3535650 len=85890 2024-09-13T00:08:30.767 INFO:tasks.workunit.client.0.smithi136.stdout:op 2193 completed, throughput=4.99MB/sec 2024-09-13T00:08:30.767 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-cmLE10rkdu6yxFh off=751544 len=244351 2024-09-13T00:08:30.768 INFO:tasks.workunit.client.0.smithi136.stdout:op 2194 completed, throughput=4.99MB/sec 2024-09-13T00:08:30.768 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Ss9IMZ9bi5jYDMZ off=9509206 len=521049 2024-09-13T00:08:30.771 INFO:tasks.workunit.client.0.smithi136.stdout:op 2195 completed, throughput=4.99MB/sec 2024-09-13T00:08:30.771 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-wjMfrDZQNRqFVlI off=8013428 len=694413 2024-09-13T00:08:30.774 INFO:tasks.workunit.client.0.smithi136.stdout:op 2196 completed, throughput=4.99MB/sec 2024-09-13T00:08:30.774 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-64yitRApfSsxR35 off=18503503 len=1659302 2024-09-13T00:08:30.778 INFO:tasks.workunit.client.0.smithi136.stdout:op 2197 completed, throughput=5MB/sec 2024-09-13T00:08:30.778 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-8ZkuMnlRgiHFTwN off=3002456 len=1005011 2024-09-13T00:08:30.785 INFO:tasks.workunit.client.0.smithi136.stdout:op 2198 completed, throughput=5MB/sec 2024-09-13T00:08:30.785 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-21caWpEQCukFhvs off=1493931 len=963496 2024-09-13T00:08:30.794 INFO:tasks.workunit.client.0.smithi136.stdout:op 2199 completed, throughput=5MB/sec 2024-09-13T00:08:31.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:08:30 smithi181 bash[67587]: cluster 2024-09-13T00:08:29.359419+0000 mgr.y (mgr.24661) 941 : cluster [DBG] pgmap v1624: 105 pgs: 105 active+clean; 126 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:08:31.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:08:30 smithi136 bash[75164]: cluster 2024-09-13T00:08:29.359419+0000 mgr.y (mgr.24661) 941 : cluster [DBG] pgmap v1624: 105 pgs: 105 active+clean; 126 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:08:31.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:08:30 smithi136 bash[76371]: cluster 2024-09-13T00:08:29.359419+0000 mgr.y (mgr.24661) 941 : cluster [DBG] pgmap v1624: 105 pgs: 105 active+clean; 126 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:08:31.794 INFO:tasks.workunit.client.0.smithi136.stdout: 415: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:31.794 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-VH2W1AU6_UHFT3z off=6442562 len=993223 2024-09-13T00:08:31.797 INFO:tasks.workunit.client.0.smithi136.stdout:op 2200 completed, throughput=4.99MB/sec 2024-09-13T00:08:31.797 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj--ggbs-Ks3Z9PFwS off=1413517 len=2574 2024-09-13T00:08:31.801 INFO:tasks.workunit.client.0.smithi136.stdout:op 2201 completed, throughput=4.99MB/sec 2024-09-13T00:08:31.801 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-qIn544TvMnzulM9 off=16072166 len=1490732 2024-09-13T00:08:31.815 INFO:tasks.workunit.client.0.smithi136.stdout:op 2202 completed, throughput=4.99MB/sec 2024-09-13T00:08:31.815 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ZNcsFrzudfzbRGV off=2614756 len=1799255 2024-09-13T00:08:31.819 INFO:tasks.workunit.client.0.smithi136.stdout:op 2203 completed, throughput=5MB/sec 2024-09-13T00:08:31.819 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-02q5RmWiK53jl34 off=1044719 len=550307 2024-09-13T00:08:31.826 INFO:tasks.workunit.client.0.smithi136.stdout:op 2204 completed, throughput=5MB/sec 2024-09-13T00:08:31.826 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-kWY2WnGhUhPMyCz off=14715562 len=563735 2024-09-13T00:08:31.832 INFO:tasks.workunit.client.0.smithi136.stdout:op 2205 completed, throughput=5MB/sec 2024-09-13T00:08:32.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:08:32 smithi181 bash[95505]: ts=2024-09-13T00:08:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:08:32.832 INFO:tasks.workunit.client.0.smithi136.stdout: 416: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:32.832 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-J6NK2PxMPJK7N7j off=7965086 len=864220 2024-09-13T00:08:32.840 INFO:tasks.workunit.client.0.smithi136.stdout:op 2206 completed, throughput=4.99MB/sec 2024-09-13T00:08:32.841 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-TcZR5ZD0Yq4XAOl off=5104527 len=286836 2024-09-13T00:08:32.843 INFO:tasks.workunit.client.0.smithi136.stdout:op 2207 completed, throughput=4.99MB/sec 2024-09-13T00:08:32.843 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-0MNpvgEn1w92Aum off=721363 len=579377 2024-09-13T00:08:32.850 INFO:tasks.workunit.client.0.smithi136.stdout:op 2208 completed, throughput=4.99MB/sec 2024-09-13T00:08:32.850 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-FP5-2zZJvgT-bt6 off=7228197 len=1239945 2024-09-13T00:08:32.853 INFO:tasks.workunit.client.0.smithi136.stdout:op 2209 completed, throughput=5MB/sec 2024-09-13T00:08:32.853 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-mAA-FYuC3-hCRnU off=8259893 len=986448 2024-09-13T00:08:32.861 INFO:tasks.workunit.client.0.smithi136.stdout:op 2210 completed, throughput=5MB/sec 2024-09-13T00:08:32.861 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-GXlkHTimMF-IsqB off=4797893 len=848134 2024-09-13T00:08:32.864 INFO:tasks.workunit.client.0.smithi136.stdout:op 2211 completed, throughput=5MB/sec 2024-09-13T00:08:32.864 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-08pV3eii7ViF71l off=8926845 len=1916297 2024-09-13T00:08:32.868 INFO:tasks.workunit.client.0.smithi136.stdout:op 2212 completed, throughput=5MB/sec 2024-09-13T00:08:33.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:08:32 smithi181 bash[67587]: cluster 2024-09-13T00:08:31.360540+0000 mgr.y (mgr.24661) 942 : cluster [DBG] pgmap v1625: 105 pgs: 105 active+clean; 126 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:08:33.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:08:32 smithi136 bash[75164]: cluster 2024-09-13T00:08:31.360540+0000 mgr.y (mgr.24661) 942 : cluster [DBG] pgmap v1625: 105 pgs: 105 active+clean; 126 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:08:33.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:08:32 smithi136 bash[76371]: cluster 2024-09-13T00:08:31.360540+0000 mgr.y (mgr.24661) 942 : cluster [DBG] pgmap v1625: 105 pgs: 105 active+clean; 126 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:08:33.868 INFO:tasks.workunit.client.0.smithi136.stdout: 417: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:33.869 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-axpfWbGhZ3wFZKV off=7838561 len=1210418 2024-09-13T00:08:33.879 INFO:tasks.workunit.client.0.smithi136.stdout:op 2213 completed, throughput=4.99MB/sec 2024-09-13T00:08:33.880 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-OWtao2YiOQ1yei2 off=7627298 len=1882271 2024-09-13T00:08:33.884 INFO:tasks.workunit.client.0.smithi136.stdout:op 2214 completed, throughput=5MB/sec 2024-09-13T00:08:33.884 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Y_Dmsc9KK_5uWXT off=11603367 len=552509 2024-09-13T00:08:33.886 INFO:tasks.workunit.client.0.smithi136.stdout:op 2215 completed, throughput=5MB/sec 2024-09-13T00:08:33.886 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-0m031rOmmqmx0V5 off=6133099 len=1906309 2024-09-13T00:08:33.900 INFO:tasks.workunit.client.0.smithi136.stdout:op 2216 completed, throughput=5MB/sec 2024-09-13T00:08:34.901 INFO:tasks.workunit.client.0.smithi136.stdout: 418: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:34.901 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-WL4ZjghOO1_5aq3 off=1104607 len=1180715 2024-09-13T00:08:34.903 INFO:tasks.workunit.client.0.smithi136.stdout:op 2217 completed, throughput=4.99MB/sec 2024-09-13T00:08:34.904 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-VAe1q2Xldr13EW_ off=2025014 len=983169 2024-09-13T00:08:34.912 INFO:tasks.workunit.client.0.smithi136.stdout:op 2218 completed, throughput=5MB/sec 2024-09-13T00:08:34.912 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-yhMgaCVGjikMjVK off=1114891 len=1828022 2024-09-13T00:08:34.929 INFO:tasks.workunit.client.0.smithi136.stdout:op 2219 completed, throughput=5MB/sec 2024-09-13T00:08:35.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:08:34 smithi181 bash[67587]: cluster 2024-09-13T00:08:33.361672+0000 mgr.y (mgr.24661) 943 : cluster [DBG] pgmap v1626: 105 pgs: 105 active+clean; 126 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:08:35.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:08:34 smithi136 bash[75164]: cluster 2024-09-13T00:08:33.361672+0000 mgr.y (mgr.24661) 943 : cluster [DBG] pgmap v1626: 105 pgs: 105 active+clean; 126 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:08:35.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:08:34 smithi136 bash[76371]: cluster 2024-09-13T00:08:33.361672+0000 mgr.y (mgr.24661) 943 : cluster [DBG] pgmap v1626: 105 pgs: 105 active+clean; 126 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:08:35.929 INFO:tasks.workunit.client.0.smithi136.stdout: 419: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:35.929 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-KCCy3kuD7NsXpTR off=7133677 len=921571 2024-09-13T00:08:35.936 INFO:tasks.workunit.client.0.smithi136.stdout:op 2220 completed, throughput=4.99MB/sec 2024-09-13T00:08:35.936 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-8zivxD1yY7Mnmi3 off=2641572 len=328552 2024-09-13T00:08:35.942 INFO:tasks.workunit.client.0.smithi136.stdout:op 2221 completed, throughput=4.99MB/sec 2024-09-13T00:08:35.942 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-dimfPzMHbB51cco off=7981415 len=2063565 2024-09-13T00:08:35.949 INFO:tasks.workunit.client.0.smithi136.stdout:op 2222 completed, throughput=5MB/sec 2024-09-13T00:08:35.949 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj--8KSwtKx7iF0BFR off=1651283 len=610204 2024-09-13T00:08:35.951 INFO:tasks.workunit.client.0.smithi136.stdout:op 2223 completed, throughput=5MB/sec 2024-09-13T00:08:35.951 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-21lusSh1IZSSjOB off=8446476 len=1995657 2024-09-13T00:08:35.964 INFO:tasks.workunit.client.0.smithi136.stdout:op 2224 completed, throughput=5MB/sec 2024-09-13T00:08:36.968 INFO:tasks.workunit.client.0.smithi136.stdout: 420: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:36.968 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-HKiiTlu8v8lHf7Y off=945494 len=527241 2024-09-13T00:08:36.968 INFO:tasks.workunit.client.0.smithi136.stdout:op 2225 completed, throughput=4.99MB/sec 2024-09-13T00:08:36.969 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-fXU1Gq_SeQD1MGB off=727375 len=1460594 2024-09-13T00:08:36.971 INFO:tasks.workunit.client.0.smithi136.stdout:op 2226 completed, throughput=4.99MB/sec 2024-09-13T00:08:36.971 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-KpQsVIeq6C791-s off=19289955 len=1186641 2024-09-13T00:08:36.974 INFO:tasks.workunit.client.0.smithi136.stdout:op 2227 completed, throughput=5MB/sec 2024-09-13T00:08:36.975 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ypk4taL0L63L02Z off=8379967 len=673186 2024-09-13T00:08:36.983 INFO:tasks.workunit.client.0.smithi136.stdout:op 2228 completed, throughput=5MB/sec 2024-09-13T00:08:36.983 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-hvM5pEtkkTx4Gor off=2263053 len=903076 2024-09-13T00:08:36.990 INFO:tasks.workunit.client.0.smithi136.stdout:op 2229 completed, throughput=5MB/sec 2024-09-13T00:08:37.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:08:36 smithi136 bash[75164]: cluster 2024-09-13T00:08:35.362653+0000 mgr.y (mgr.24661) 944 : cluster [DBG] pgmap v1627: 105 pgs: 105 active+clean; 126 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:08:37.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:08:36 smithi136 bash[76371]: cluster 2024-09-13T00:08:35.362653+0000 mgr.y (mgr.24661) 944 : cluster [DBG] pgmap v1627: 105 pgs: 105 active+clean; 126 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:08:37.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:08:36 smithi181 bash[67587]: cluster 2024-09-13T00:08:35.362653+0000 mgr.y (mgr.24661) 944 : cluster [DBG] pgmap v1627: 105 pgs: 105 active+clean; 126 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:08:37.991 INFO:tasks.workunit.client.0.smithi136.stdout: 421: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:37.991 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ZK_kcOZNcpgsj0m off=1849612 len=680125 2024-09-13T00:08:37.998 INFO:tasks.workunit.client.0.smithi136.stdout:op 2230 completed, throughput=4.99MB/sec 2024-09-13T00:08:37.998 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-EhpcZm9ciaXk3Vp off=5726493 len=576164 2024-09-13T00:08:38.004 INFO:tasks.workunit.client.0.smithi136.stdout:op 2231 completed, throughput=4.99MB/sec 2024-09-13T00:08:38.004 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-6YZnNtuH2VfyPuv off=2816659 len=1503897 2024-09-13T00:08:38.008 INFO:tasks.workunit.client.0.smithi136.stdout:op 2232 completed, throughput=4.99MB/sec 2024-09-13T00:08:38.008 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-pj5RRntcYmnrOIk off=19229853 len=662468 2024-09-13T00:08:38.014 INFO:tasks.workunit.client.0.smithi136.stdout:op 2233 completed, throughput=5MB/sec 2024-09-13T00:08:38.014 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ZGvAOB7E-_eVsgd off=1328612 len=217069 2024-09-13T00:08:38.019 INFO:tasks.workunit.client.0.smithi136.stdout:op 2234 completed, throughput=5MB/sec 2024-09-13T00:08:38.019 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-6v25Ihd7xMXnqTQ off=5155571 len=167509 2024-09-13T00:08:38.023 INFO:tasks.workunit.client.0.smithi136.stdout:op 2235 completed, throughput=5MB/sec 2024-09-13T00:08:38.023 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-msCTyVftx6wfCU9 off=6019964 len=1403856 2024-09-13T00:08:38.033 INFO:tasks.workunit.client.0.smithi136.stdout:op 2236 completed, throughput=5MB/sec 2024-09-13T00:08:38.033 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-c1_3EQgqElanFBL off=226445 len=2050137 2024-09-13T00:08:38.038 INFO:tasks.workunit.client.0.smithi136.stdout:op 2237 completed, throughput=5MB/sec 2024-09-13T00:08:39.038 INFO:tasks.workunit.client.0.smithi136.stdout: 422: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:39.038 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-s0DWPZCl821W-4Q off=830275 len=1022341 2024-09-13T00:08:39.041 INFO:tasks.workunit.client.0.smithi136.stdout:op 2238 completed, throughput=4.99MB/sec 2024-09-13T00:08:39.041 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-q_Uh_v-NVMqFW-g off=10462484 len=2004414 2024-09-13T00:08:39.045 INFO:tasks.workunit.client.0.smithi136.stdout:op 2239 completed, throughput=5MB/sec 2024-09-13T00:08:39.045 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Fv4fVDP2q55cyvj off=6623130 len=1404480 2024-09-13T00:08:39.056 INFO:tasks.workunit.client.0.smithi136.stdout:op 2240 completed, throughput=5MB/sec 2024-09-13T00:08:39.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:08:38 smithi136 bash[76371]: cluster 2024-09-13T00:08:37.363916+0000 mgr.y (mgr.24661) 945 : cluster [DBG] pgmap v1628: 105 pgs: 105 active+clean; 126 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.9 MiB/s wr, 4 op/s 2024-09-13T00:08:39.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:08:38 smithi136 bash[76371]: audit 2024-09-13T00:08:38.783478+0000 mon.a (mon.0) 2169 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:08:39.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:08:38 smithi136 bash[75164]: cluster 2024-09-13T00:08:37.363916+0000 mgr.y (mgr.24661) 945 : cluster [DBG] pgmap v1628: 105 pgs: 105 active+clean; 126 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.9 MiB/s wr, 4 op/s 2024-09-13T00:08:39.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:08:38 smithi136 bash[75164]: audit 2024-09-13T00:08:38.783478+0000 mon.a (mon.0) 2169 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:08:39.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:08:38 smithi181 bash[67587]: cluster 2024-09-13T00:08:37.363916+0000 mgr.y (mgr.24661) 945 : cluster [DBG] pgmap v1628: 105 pgs: 105 active+clean; 126 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.9 MiB/s wr, 4 op/s 2024-09-13T00:08:39.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:08:38 smithi181 bash[67587]: audit 2024-09-13T00:08:38.783478+0000 mon.a (mon.0) 2169 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:08:39.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:08:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:08:39] "GET /metrics HTTP/1.1" 200 34138 "" "Prometheus/2.43.0" 2024-09-13T00:08:40.056 INFO:tasks.workunit.client.0.smithi136.stdout: 423: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:40.056 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-aoUDGhsIrQzcpi8 off=6593409 len=434242 2024-09-13T00:08:40.063 INFO:tasks.workunit.client.0.smithi136.stdout:op 2241 completed, throughput=4.99MB/sec 2024-09-13T00:08:40.063 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Ii59CziXclZ-F1h off=4432202 len=893890 2024-09-13T00:08:40.071 INFO:tasks.workunit.client.0.smithi136.stdout:op 2242 completed, throughput=4.99MB/sec 2024-09-13T00:08:40.071 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-S9MyWz7kB4WkIu8 off=6577083 len=1086532 2024-09-13T00:08:40.081 INFO:tasks.workunit.client.0.smithi136.stdout:op 2243 completed, throughput=5MB/sec 2024-09-13T00:08:40.081 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj--xYBYHHfY9pcT3L off=3257242 len=624217 2024-09-13T00:08:40.083 INFO:tasks.workunit.client.0.smithi136.stdout:op 2244 completed, throughput=5MB/sec 2024-09-13T00:08:40.083 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-WLVt43Qjn1B7xZB off=9466420 len=2040707 2024-09-13T00:08:40.097 INFO:tasks.workunit.client.0.smithi136.stdout:op 2245 completed, throughput=5MB/sec 2024-09-13T00:08:41.097 INFO:tasks.workunit.client.0.smithi136.stdout: 424: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:41.097 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-tLVrUh8HLHslAg4 off=5832399 len=359821 2024-09-13T00:08:41.102 INFO:tasks.workunit.client.0.smithi136.stdout:op 2246 completed, throughput=4.99MB/sec 2024-09-13T00:08:41.102 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-MYKkoT8t-ULOGiY off=10651320 len=2027405 2024-09-13T00:08:41.121 INFO:tasks.workunit.client.0.smithi136.stdout:op 2247 completed, throughput=4.99MB/sec 2024-09-13T00:08:41.122 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-CxFY_ob8PZgPjtA off=8805363 len=656447 2024-09-13T00:08:41.123 INFO:tasks.workunit.client.0.smithi136.stdout:op 2248 completed, throughput=5MB/sec 2024-09-13T00:08:41.123 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-3Zys62Qc8QlfrDV off=7630602 len=1264489 2024-09-13T00:08:41.129 INFO:tasks.workunit.client.0.smithi136.stdout:op 2249 completed, throughput=5MB/sec 2024-09-13T00:08:41.129 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-KSCchZdWbyYOFxa off=937780 len=907552 2024-09-13T00:08:41.132 INFO:tasks.workunit.client.0.smithi136.stdout:op 2250 completed, throughput=5MB/sec 2024-09-13T00:08:41.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:08:40 smithi136 bash[76371]: cluster 2024-09-13T00:08:39.364883+0000 mgr.y (mgr.24661) 946 : cluster [DBG] pgmap v1629: 105 pgs: 105 active+clean; 126 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:08:41.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:08:40 smithi136 bash[75164]: cluster 2024-09-13T00:08:39.364883+0000 mgr.y (mgr.24661) 946 : cluster [DBG] pgmap v1629: 105 pgs: 105 active+clean; 126 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:08:41.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:08:40 smithi181 bash[67587]: cluster 2024-09-13T00:08:39.364883+0000 mgr.y (mgr.24661) 946 : cluster [DBG] pgmap v1629: 105 pgs: 105 active+clean; 126 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:08:42.132 INFO:tasks.workunit.client.0.smithi136.stdout: 425: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:42.133 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-HuZcf1rDQ5nzZxB off=3809033 len=769743 2024-09-13T00:08:42.141 INFO:tasks.workunit.client.0.smithi136.stdout:op 2251 completed, throughput=4.99MB/sec 2024-09-13T00:08:42.141 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-apJTiTnoTTf6k9g off=8568398 len=1140865 2024-09-13T00:08:42.150 INFO:tasks.workunit.client.0.smithi136.stdout:op 2252 completed, throughput=4.99MB/sec 2024-09-13T00:08:42.150 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-s2vylAlPBweaWo- off=982294 len=354281 2024-09-13T00:08:42.151 INFO:tasks.workunit.client.0.smithi136.stdout:op 2253 completed, throughput=4.99MB/sec 2024-09-13T00:08:42.151 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-0eqPMDQMCsDnZuv off=2438635 len=1641256 2024-09-13T00:08:42.155 INFO:tasks.workunit.client.0.smithi136.stdout:op 2254 completed, throughput=5MB/sec 2024-09-13T00:08:42.155 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-FYRdGBXzzDzIfEl off=2150195 len=1639378 2024-09-13T00:08:42.170 INFO:tasks.workunit.client.0.smithi136.stdout:op 2255 completed, throughput=5MB/sec 2024-09-13T00:08:42.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:08:42 smithi181 bash[95505]: ts=2024-09-13T00:08:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:08:43.170 INFO:tasks.workunit.client.0.smithi136.stdout: 426: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:43.170 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-hSzM7AxNTxkTmtG off=2319177 len=611204 2024-09-13T00:08:43.172 INFO:tasks.workunit.client.0.smithi136.stdout:op 2256 completed, throughput=4.99MB/sec 2024-09-13T00:08:43.172 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_S3kD8X-reoTUia off=1291730 len=1596608 2024-09-13T00:08:43.185 INFO:tasks.workunit.client.0.smithi136.stdout:op 2257 completed, throughput=4.99MB/sec 2024-09-13T00:08:43.185 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-70AesVwYmnDwaXK off=2588180 len=1639955 2024-09-13T00:08:43.189 INFO:tasks.workunit.client.0.smithi136.stdout:op 2258 completed, throughput=5MB/sec 2024-09-13T00:08:43.189 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-mbBBdnGEB3j2889 off=6819712 len=994595 2024-09-13T00:08:43.199 INFO:tasks.workunit.client.0.smithi136.stdout:op 2259 completed, throughput=5MB/sec 2024-09-13T00:08:43.199 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-mbBBdnGEB3j2889 off=378076 len=716134 2024-09-13T00:08:43.202 INFO:tasks.workunit.client.0.smithi136.stdout:op 2260 completed, throughput=5MB/sec 2024-09-13T00:08:43.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:08:42 smithi136 bash[75164]: cluster 2024-09-13T00:08:41.365918+0000 mgr.y (mgr.24661) 947 : cluster [DBG] pgmap v1630: 105 pgs: 105 active+clean; 126 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:08:43.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:08:42 smithi136 bash[76371]: cluster 2024-09-13T00:08:41.365918+0000 mgr.y (mgr.24661) 947 : cluster [DBG] pgmap v1630: 105 pgs: 105 active+clean; 126 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:08:43.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:08:42 smithi181 bash[67587]: cluster 2024-09-13T00:08:41.365918+0000 mgr.y (mgr.24661) 947 : cluster [DBG] pgmap v1630: 105 pgs: 105 active+clean; 126 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:08:44.202 INFO:tasks.workunit.client.0.smithi136.stdout: 427: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:44.202 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-DaVr5yd9qX30Cp- off=1103976 len=302626 2024-09-13T00:08:44.208 INFO:tasks.workunit.client.0.smithi136.stdout:op 2261 completed, throughput=4.99MB/sec 2024-09-13T00:08:44.208 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-8qw0Noyg6EdRI9O off=3711131 len=968505 2024-09-13T00:08:44.212 INFO:tasks.workunit.client.0.smithi136.stdout:op 2262 completed, throughput=4.99MB/sec 2024-09-13T00:08:44.212 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-NsMR5_qEEPAG9ym off=7833385 len=772985 2024-09-13T00:08:44.215 INFO:tasks.workunit.client.0.smithi136.stdout:op 2263 completed, throughput=4.99MB/sec 2024-09-13T00:08:44.215 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj--6h8LIzZRPda4mU off=6018748 len=1841061 2024-09-13T00:08:44.219 INFO:tasks.workunit.client.0.smithi136.stdout:op 2264 completed, throughput=5MB/sec 2024-09-13T00:08:44.219 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-jQMZ0R17nhX5S4p off=11927924 len=787332 2024-09-13T00:08:44.222 INFO:tasks.workunit.client.0.smithi136.stdout:op 2265 completed, throughput=5MB/sec 2024-09-13T00:08:44.222 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-LxzD93Ql9nfok3h off=10822861 len=54240 2024-09-13T00:08:44.223 INFO:tasks.workunit.client.0.smithi136.stdout:op 2266 completed, throughput=5MB/sec 2024-09-13T00:08:44.223 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-x_2wMfgT_z5eSLc off=11735184 len=1457421 2024-09-13T00:08:44.227 INFO:tasks.workunit.client.0.smithi136.stdout:op 2267 completed, throughput=5MB/sec 2024-09-13T00:08:45.228 INFO:tasks.workunit.client.0.smithi136.stdout: 428: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:45.228 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-H0QdDHHeicM-9X4 off=1100990 len=1071860 2024-09-13T00:08:45.236 INFO:tasks.workunit.client.0.smithi136.stdout:op 2268 completed, throughput=4.99MB/sec 2024-09-13T00:08:45.236 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-2bhs6Cg1AZMEqbQ off=1550208 len=78486 2024-09-13T00:08:45.241 INFO:tasks.workunit.client.0.smithi136.stdout:op 2269 completed, throughput=4.99MB/sec 2024-09-13T00:08:45.241 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-LF-WFBSAOB0o4MS off=8862432 len=1837445 2024-09-13T00:08:45.253 INFO:tasks.workunit.client.0.smithi136.stdout:op 2270 completed, throughput=5MB/sec 2024-09-13T00:08:45.253 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-1z53HacjSpF2Cn1 off=7696628 len=1529182 2024-09-13T00:08:45.256 INFO:tasks.workunit.client.0.smithi136.stdout:op 2271 completed, throughput=5MB/sec 2024-09-13T00:08:45.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:08:44 smithi136 bash[75164]: cluster 2024-09-13T00:08:43.367013+0000 mgr.y (mgr.24661) 948 : cluster [DBG] pgmap v1631: 105 pgs: 105 active+clean; 126 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:08:45.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:08:44 smithi136 bash[76371]: cluster 2024-09-13T00:08:43.367013+0000 mgr.y (mgr.24661) 948 : cluster [DBG] pgmap v1631: 105 pgs: 105 active+clean; 126 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:08:45.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:08:44 smithi181 bash[67587]: cluster 2024-09-13T00:08:43.367013+0000 mgr.y (mgr.24661) 948 : cluster [DBG] pgmap v1631: 105 pgs: 105 active+clean; 126 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:08:46.256 INFO:tasks.workunit.client.0.smithi136.stdout: 429: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:46.257 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-nC4OLO5ma6CXvc5 off=2181995 len=2015974 2024-09-13T00:08:46.269 INFO:tasks.workunit.client.0.smithi136.stdout:op 2272 completed, throughput=4.99MB/sec 2024-09-13T00:08:46.269 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Re2YcNt4oP_bsmn off=11091748 len=1525752 2024-09-13T00:08:46.271 INFO:tasks.workunit.client.0.smithi136.stdout:op 2273 completed, throughput=5MB/sec 2024-09-13T00:08:46.271 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-fjCQVl0nOE_VGPT off=2353259 len=1442915 2024-09-13T00:08:46.275 INFO:tasks.workunit.client.0.smithi136.stdout:op 2274 completed, throughput=5MB/sec 2024-09-13T00:08:47.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:08:46 smithi136 bash[76371]: cluster 2024-09-13T00:08:45.367899+0000 mgr.y (mgr.24661) 949 : cluster [DBG] pgmap v1632: 105 pgs: 105 active+clean; 126 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:08:47.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:08:46 smithi136 bash[76371]: audit 2024-09-13T00:08:46.002085+0000 mon.a (mon.0) 2170 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:08:47.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:08:46 smithi136 bash[76371]: audit 2024-09-13T00:08:46.564882+0000 mon.a (mon.0) 2171 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:08:47.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:08:46 smithi136 bash[76371]: audit 2024-09-13T00:08:46.566428+0000 mon.a (mon.0) 2172 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:08:47.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:08:46 smithi136 bash[76371]: audit 2024-09-13T00:08:46.576153+0000 mon.a (mon.0) 2173 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:08:47.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:08:46 smithi136 bash[75164]: cluster 2024-09-13T00:08:45.367899+0000 mgr.y (mgr.24661) 949 : cluster [DBG] pgmap v1632: 105 pgs: 105 active+clean; 126 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:08:47.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:08:46 smithi136 bash[75164]: audit 2024-09-13T00:08:46.002085+0000 mon.a (mon.0) 2170 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:08:47.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:08:46 smithi136 bash[75164]: audit 2024-09-13T00:08:46.564882+0000 mon.a (mon.0) 2171 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:08:47.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:08:46 smithi136 bash[75164]: audit 2024-09-13T00:08:46.566428+0000 mon.a (mon.0) 2172 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:08:47.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:08:46 smithi136 bash[75164]: audit 2024-09-13T00:08:46.576153+0000 mon.a (mon.0) 2173 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:08:47.275 INFO:tasks.workunit.client.0.smithi136.stdout: 430: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:47.275 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-A2AGM7q6fRHQQj6 off=7610925 len=218321 2024-09-13T00:08:47.280 INFO:tasks.workunit.client.0.smithi136.stdout:op 2275 completed, throughput=4.99MB/sec 2024-09-13T00:08:47.281 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-CuDRu4K1aZglJ4v off=10229073 len=1072593 2024-09-13T00:08:47.283 INFO:tasks.workunit.client.0.smithi136.stdout:op 2276 completed, throughput=4.99MB/sec 2024-09-13T00:08:47.283 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-wMfv6Z_MYR3h72o off=6862184 len=129347 2024-09-13T00:08:47.285 INFO:tasks.workunit.client.0.smithi136.stdout:op 2277 completed, throughput=4.99MB/sec 2024-09-13T00:08:47.285 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-oHD8nIB0ESDYrF4 off=949126 len=1088391 2024-09-13T00:08:47.288 INFO:tasks.workunit.client.0.smithi136.stdout:op 2278 completed, throughput=4.99MB/sec 2024-09-13T00:08:47.288 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-90nqCPvk6uvh026 off=6694527 len=44981 2024-09-13T00:08:47.289 INFO:tasks.workunit.client.0.smithi136.stdout:op 2279 completed, throughput=4.99MB/sec 2024-09-13T00:08:47.289 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-dUy0Swvtf38N5h9 off=80029 len=1202144 2024-09-13T00:08:47.300 INFO:tasks.workunit.client.0.smithi136.stdout:op 2280 completed, throughput=5MB/sec 2024-09-13T00:08:47.300 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-6YZnNtuH2VfyPuv off=4591877 len=737408 2024-09-13T00:08:47.302 INFO:tasks.workunit.client.0.smithi136.stdout:op 2281 completed, throughput=5MB/sec 2024-09-13T00:08:47.302 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-hWALfWq6LHp3mlL off=6524449 len=1714791 2024-09-13T00:08:47.314 INFO:tasks.workunit.client.0.smithi136.stdout:op 2282 completed, throughput=5MB/sec 2024-09-13T00:08:47.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:08:46 smithi181 bash[67587]: cluster 2024-09-13T00:08:45.367899+0000 mgr.y (mgr.24661) 949 : cluster [DBG] pgmap v1632: 105 pgs: 105 active+clean; 126 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:08:47.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:08:46 smithi181 bash[67587]: audit 2024-09-13T00:08:46.002085+0000 mon.a (mon.0) 2170 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:08:47.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:08:46 smithi181 bash[67587]: audit 2024-09-13T00:08:46.564882+0000 mon.a (mon.0) 2171 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:08:47.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:08:46 smithi181 bash[67587]: audit 2024-09-13T00:08:46.566428+0000 mon.a (mon.0) 2172 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:08:47.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:08:46 smithi181 bash[67587]: audit 2024-09-13T00:08:46.576153+0000 mon.a (mon.0) 2173 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:08:48.315 INFO:tasks.workunit.client.0.smithi136.stdout: 431: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:48.315 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-oD_YzkSxgDmJGUL off=2836029 len=464920 2024-09-13T00:08:48.321 INFO:tasks.workunit.client.0.smithi136.stdout:op 2283 completed, throughput=4.99MB/sec 2024-09-13T00:08:48.321 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-oNykxhsuLsg6uuk off=6753811 len=132210 2024-09-13T00:08:48.322 INFO:tasks.workunit.client.0.smithi136.stdout:op 2284 completed, throughput=4.99MB/sec 2024-09-13T00:08:48.322 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-IhXarTCu38aO2tA off=10514432 len=467503 2024-09-13T00:08:48.323 INFO:tasks.workunit.client.0.smithi136.stdout:op 2285 completed, throughput=4.99MB/sec 2024-09-13T00:08:48.324 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-eCn7GnEEmWl1HvL off=13701210 len=1986338 2024-09-13T00:08:48.338 INFO:tasks.workunit.client.0.smithi136.stdout:op 2286 completed, throughput=5MB/sec 2024-09-13T00:08:48.338 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-jCPM3NZSrpy0Wx0 off=4840852 len=443763 2024-09-13T00:08:48.345 INFO:tasks.workunit.client.0.smithi136.stdout:op 2287 completed, throughput=5MB/sec 2024-09-13T00:08:48.345 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-pfPJxwl0j3Bd5MY off=4354959 len=629127 2024-09-13T00:08:48.352 INFO:tasks.workunit.client.0.smithi136.stdout:op 2288 completed, throughput=5MB/sec 2024-09-13T00:08:48.352 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-lcCY3mNv7xO6SDU off=4991297 len=1485926 2024-09-13T00:08:48.356 INFO:tasks.workunit.client.0.smithi136.stdout:op 2289 completed, throughput=5MB/sec 2024-09-13T00:08:49.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:08:48 smithi136 bash[76371]: cluster 2024-09-13T00:08:47.369167+0000 mgr.y (mgr.24661) 950 : cluster [DBG] pgmap v1633: 105 pgs: 105 active+clean; 126 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:08:49.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:08:48 smithi136 bash[75164]: cluster 2024-09-13T00:08:47.369167+0000 mgr.y (mgr.24661) 950 : cluster [DBG] pgmap v1633: 105 pgs: 105 active+clean; 126 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:08:49.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:08:48 smithi181 bash[67587]: cluster 2024-09-13T00:08:47.369167+0000 mgr.y (mgr.24661) 950 : cluster [DBG] pgmap v1633: 105 pgs: 105 active+clean; 126 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:08:49.357 INFO:tasks.workunit.client.0.smithi136.stdout: 432: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:49.357 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-bV9yL-0y7eGQdnh off=6358764 len=1799792 2024-09-13T00:08:49.361 INFO:tasks.workunit.client.0.smithi136.stdout:op 2290 completed, throughput=4.99MB/sec 2024-09-13T00:08:49.361 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-G8mZWocV2qhP9yT off=1700649 len=1700271 2024-09-13T00:08:49.373 INFO:tasks.workunit.client.0.smithi136.stdout:op 2291 completed, throughput=5MB/sec 2024-09-13T00:08:49.373 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Mr9YnCU8cWP2lwv off=19197235 len=247057 2024-09-13T00:08:49.378 INFO:tasks.workunit.client.0.smithi136.stdout:op 2292 completed, throughput=5MB/sec 2024-09-13T00:08:49.379 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-o3dFrB_4YtTdDk7 off=1790561 len=1618201 2024-09-13T00:08:49.382 INFO:tasks.workunit.client.0.smithi136.stdout:op 2293 completed, throughput=5MB/sec 2024-09-13T00:08:49.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:08:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:08:49] "GET /metrics HTTP/1.1" 200 34138 "" "Prometheus/2.43.0" 2024-09-13T00:08:50.383 INFO:tasks.workunit.client.0.smithi136.stdout: 433: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:50.383 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-jFptE5tTZF6cAeH off=12860690 len=1581918 2024-09-13T00:08:50.394 INFO:tasks.workunit.client.0.smithi136.stdout:op 2294 completed, throughput=4.99MB/sec 2024-09-13T00:08:50.394 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Nj_v8s_2ikD-TSw off=2656431 len=50772 2024-09-13T00:08:50.398 INFO:tasks.workunit.client.0.smithi136.stdout:op 2295 completed, throughput=4.99MB/sec 2024-09-13T00:08:50.398 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-sK6VAF4YEjKxSSV off=4654721 len=897084 2024-09-13T00:08:50.408 INFO:tasks.workunit.client.0.smithi136.stdout:op 2296 completed, throughput=5MB/sec 2024-09-13T00:08:50.408 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-0ZVDvBJYr9N3WoN off=228505 len=1895423 2024-09-13T00:08:50.412 INFO:tasks.workunit.client.0.smithi136.stdout:op 2297 completed, throughput=5MB/sec 2024-09-13T00:08:51.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:08:50 smithi136 bash[75164]: cluster 2024-09-13T00:08:49.370348+0000 mgr.y (mgr.24661) 951 : cluster [DBG] pgmap v1634: 105 pgs: 105 active+clean; 126 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:08:51.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:08:50 smithi136 bash[76371]: cluster 2024-09-13T00:08:49.370348+0000 mgr.y (mgr.24661) 951 : cluster [DBG] pgmap v1634: 105 pgs: 105 active+clean; 126 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:08:51.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:08:50 smithi181 bash[67587]: cluster 2024-09-13T00:08:49.370348+0000 mgr.y (mgr.24661) 951 : cluster [DBG] pgmap v1634: 105 pgs: 105 active+clean; 126 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:08:51.412 INFO:tasks.workunit.client.0.smithi136.stdout: 434: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:51.412 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-7NLCitYvGxgkcHG off=10632822 len=694099 2024-09-13T00:08:51.415 INFO:tasks.workunit.client.0.smithi136.stdout:op 2298 completed, throughput=4.99MB/sec 2024-09-13T00:08:51.415 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-yafIHiOmdG9sMS5 off=2358191 len=293209 2024-09-13T00:08:51.420 INFO:tasks.workunit.client.0.smithi136.stdout:op 2299 completed, throughput=4.99MB/sec 2024-09-13T00:08:51.420 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-lZDJlyYWUuZtA6t off=2923565 len=1618074 2024-09-13T00:08:51.434 INFO:tasks.workunit.client.0.smithi136.stdout:op 2300 completed, throughput=4.99MB/sec 2024-09-13T00:08:51.434 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-8mkG2PeuOu9Dqjq off=848279 len=1627766 2024-09-13T00:08:51.438 INFO:tasks.workunit.client.0.smithi136.stdout:op 2301 completed, throughput=5MB/sec 2024-09-13T00:08:51.438 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-GNeO5oFoGi3COsa off=4592875 len=1021555 2024-09-13T00:08:51.447 INFO:tasks.workunit.client.0.smithi136.stdout:op 2302 completed, throughput=5MB/sec 2024-09-13T00:08:51.447 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2oekCu2B-TJuB1c off=3430513 len=771941 2024-09-13T00:08:51.449 INFO:tasks.workunit.client.0.smithi136.stdout:op 2303 completed, throughput=5MB/sec 2024-09-13T00:08:52.450 INFO:tasks.workunit.client.0.smithi136.stdout: 435: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:52.450 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-BEgwXhvUBgruqBx off=17831145 len=206246 2024-09-13T00:08:52.451 INFO:tasks.workunit.client.0.smithi136.stdout:op 2304 completed, throughput=4.99MB/sec 2024-09-13T00:08:52.451 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-vN00p9NcL3tW-s2 off=1213887 len=677725 2024-09-13T00:08:52.459 INFO:tasks.workunit.client.0.smithi136.stdout:op 2305 completed, throughput=4.99MB/sec 2024-09-13T00:08:52.459 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Ec_n4607PqQ_Vuo off=1792985 len=1520000 2024-09-13T00:08:52.463 INFO:tasks.workunit.client.0.smithi136.stdout:op 2306 completed, throughput=5MB/sec 2024-09-13T00:08:52.463 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-I5knHmm6KyArpgC off=8411616 len=1670235 2024-09-13T00:08:52.467 INFO:tasks.workunit.client.0.smithi136.stdout:op 2307 completed, throughput=5MB/sec 2024-09-13T00:08:52.467 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-b1Lklq-tR9F1UJd off=3522416 len=853805 2024-09-13T00:08:52.469 INFO:tasks.workunit.client.0.smithi136.stdout:op 2308 completed, throughput=5MB/sec 2024-09-13T00:08:52.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:08:52 smithi181 bash[95505]: ts=2024-09-13T00:08:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:08:53.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:08:52 smithi136 bash[75164]: cluster 2024-09-13T00:08:51.371397+0000 mgr.y (mgr.24661) 952 : cluster [DBG] pgmap v1635: 105 pgs: 105 active+clean; 126 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:08:53.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:08:52 smithi136 bash[76371]: cluster 2024-09-13T00:08:51.371397+0000 mgr.y (mgr.24661) 952 : cluster [DBG] pgmap v1635: 105 pgs: 105 active+clean; 126 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:08:53.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:08:52 smithi181 bash[67587]: cluster 2024-09-13T00:08:51.371397+0000 mgr.y (mgr.24661) 952 : cluster [DBG] pgmap v1635: 105 pgs: 105 active+clean; 126 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:08:53.470 INFO:tasks.workunit.client.0.smithi136.stdout: 436: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:53.470 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-J5y2K5WIzOMX-Ua off=9177418 len=909585 2024-09-13T00:08:53.478 INFO:tasks.workunit.client.0.smithi136.stdout:op 2309 completed, throughput=4.99MB/sec 2024-09-13T00:08:53.478 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-727YlzuqMoSTpDq off=8415630 len=53145 2024-09-13T00:08:53.479 INFO:tasks.workunit.client.0.smithi136.stdout:op 2310 completed, throughput=4.99MB/sec 2024-09-13T00:08:53.479 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-m8KCOh4ZQQj8qsF off=1458036 len=219807 2024-09-13T00:08:53.485 INFO:tasks.workunit.client.0.smithi136.stdout:op 2311 completed, throughput=4.99MB/sec 2024-09-13T00:08:53.485 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-MpdHzs4aBv2BJEK off=1747778 len=380637 2024-09-13T00:08:53.487 INFO:tasks.workunit.client.0.smithi136.stdout:op 2312 completed, throughput=4.99MB/sec 2024-09-13T00:08:53.487 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-rJlOsMskWtfiVTb off=2244398 len=893359 2024-09-13T00:08:53.490 INFO:tasks.workunit.client.0.smithi136.stdout:op 2313 completed, throughput=4.99MB/sec 2024-09-13T00:08:53.490 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-FzkR6nITozEvwOr off=1617556 len=2027066 2024-09-13T00:08:53.505 INFO:tasks.workunit.client.0.smithi136.stdout:op 2314 completed, throughput=5MB/sec 2024-09-13T00:08:53.505 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-4HMoV5aXxv1Gk-t off=11317663 len=1486257 2024-09-13T00:08:53.511 INFO:tasks.workunit.client.0.smithi136.stdout:op 2315 completed, throughput=5MB/sec 2024-09-13T00:08:54.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:08:53 smithi136 bash[75164]: audit 2024-09-13T00:08:53.783949+0000 mon.a (mon.0) 2174 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:08:54.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:08:53 smithi136 bash[76371]: audit 2024-09-13T00:08:53.783949+0000 mon.a (mon.0) 2174 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:08:54.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:08:53 smithi181 bash[67587]: audit 2024-09-13T00:08:53.783949+0000 mon.a (mon.0) 2174 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:08:54.511 INFO:tasks.workunit.client.0.smithi136.stdout: 437: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:54.511 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-DlLFIHARxKuixiS off=18917090 len=1416488 2024-09-13T00:08:54.522 INFO:tasks.workunit.client.0.smithi136.stdout:op 2316 completed, throughput=4.99MB/sec 2024-09-13T00:08:54.522 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-y0u7mnRDLbr7fnV off=2746962 len=672974 2024-09-13T00:08:54.529 INFO:tasks.workunit.client.0.smithi136.stdout:op 2317 completed, throughput=4.99MB/sec 2024-09-13T00:08:54.529 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-UFy2H7E8GLb7Gzv off=1609569 len=1650246 2024-09-13T00:08:54.535 INFO:tasks.workunit.client.0.smithi136.stdout:op 2318 completed, throughput=5MB/sec 2024-09-13T00:08:54.535 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-YCgzsMdlLeeJ6Z0 off=3625788 len=1894439 2024-09-13T00:08:54.544 INFO:tasks.workunit.client.0.smithi136.stdout:op 2319 completed, throughput=5MB/sec 2024-09-13T00:08:55.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:08:54 smithi136 bash[76371]: cluster 2024-09-13T00:08:53.372377+0000 mgr.y (mgr.24661) 953 : cluster [DBG] pgmap v1636: 105 pgs: 105 active+clean; 126 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:08:55.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:08:54 smithi136 bash[75164]: cluster 2024-09-13T00:08:53.372377+0000 mgr.y (mgr.24661) 953 : cluster [DBG] pgmap v1636: 105 pgs: 105 active+clean; 126 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:08:55.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:08:54 smithi181 bash[67587]: cluster 2024-09-13T00:08:53.372377+0000 mgr.y (mgr.24661) 953 : cluster [DBG] pgmap v1636: 105 pgs: 105 active+clean; 126 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:08:55.541 INFO:tasks.workunit.client.0.smithi136.stdout: 438: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:55.541 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-xQERektCKLgunmW off=11063077 len=1842726 2024-09-13T00:08:55.545 INFO:tasks.workunit.client.0.smithi136.stdout:op 2320 completed, throughput=4.99MB/sec 2024-09-13T00:08:55.545 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-BkqgF5TAo72-0Z8 off=5621108 len=558251 2024-09-13T00:08:55.552 INFO:tasks.workunit.client.0.smithi136.stdout:op 2321 completed, throughput=5MB/sec 2024-09-13T00:08:55.552 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_SP5SwKjKDmo0rx off=3449586 len=51409 2024-09-13T00:08:55.556 INFO:tasks.workunit.client.0.smithi136.stdout:op 2322 completed, throughput=5MB/sec 2024-09-13T00:08:55.557 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-d_y8uHDq2Sda9lk off=1536925 len=1734454 2024-09-13T00:08:55.561 INFO:tasks.workunit.client.0.smithi136.stdout:op 2323 completed, throughput=5MB/sec 2024-09-13T00:08:55.561 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-R2RA3IRMN-KQDpp off=2554723 len=617329 2024-09-13T00:08:55.563 INFO:tasks.workunit.client.0.smithi136.stdout:op 2324 completed, throughput=5MB/sec 2024-09-13T00:08:56.564 INFO:tasks.workunit.client.0.smithi136.stdout: 439: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:56.564 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2RMUAoDGF8xNlnc off=11484226 len=1749150 2024-09-13T00:08:56.568 INFO:tasks.workunit.client.0.smithi136.stdout:op 2325 completed, throughput=4.99MB/sec 2024-09-13T00:08:56.568 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-lnXBqSBkmdGjWx0 off=5240478 len=1123187 2024-09-13T00:08:56.571 INFO:tasks.workunit.client.0.smithi136.stdout:op 2326 completed, throughput=5MB/sec 2024-09-13T00:08:56.571 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-mw0PYDOnreruUSa off=8662062 len=1838805 2024-09-13T00:08:56.588 INFO:tasks.workunit.client.0.smithi136.stdout:op 2327 completed, throughput=5MB/sec 2024-09-13T00:08:56.589 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-S744G7CBLD6SPYS off=3939054 len=1800764 2024-09-13T00:08:56.599 INFO:tasks.workunit.client.0.smithi136.stdout:op 2328 completed, throughput=5MB/sec 2024-09-13T00:08:57.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:08:56 smithi136 bash[75164]: cluster 2024-09-13T00:08:55.373234+0000 mgr.y (mgr.24661) 954 : cluster [DBG] pgmap v1637: 105 pgs: 105 active+clean; 126 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:08:57.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:08:56 smithi136 bash[76371]: cluster 2024-09-13T00:08:55.373234+0000 mgr.y (mgr.24661) 954 : cluster [DBG] pgmap v1637: 105 pgs: 105 active+clean; 126 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:08:57.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:08:56 smithi181 bash[67587]: cluster 2024-09-13T00:08:55.373234+0000 mgr.y (mgr.24661) 954 : cluster [DBG] pgmap v1637: 105 pgs: 105 active+clean; 126 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:08:57.599 INFO:tasks.workunit.client.0.smithi136.stdout: 440: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:57.599 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-P0Uh6lOO1Wz9iPK off=12042600 len=160443 2024-09-13T00:08:57.600 INFO:tasks.workunit.client.0.smithi136.stdout:op 2329 completed, throughput=4.99MB/sec 2024-09-13T00:08:57.601 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-s1DwEZjXzoT5NXu off=7003765 len=877557 2024-09-13T00:08:57.603 INFO:tasks.workunit.client.0.smithi136.stdout:op 2330 completed, throughput=4.99MB/sec 2024-09-13T00:08:57.603 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-P2pbde1akvmHc3L off=9440146 len=526014 2024-09-13T00:08:57.613 INFO:tasks.workunit.client.0.smithi136.stdout:op 2331 completed, throughput=5MB/sec 2024-09-13T00:08:57.613 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-sWUjiVQyEMm9ocJ off=11123631 len=1360722 2024-09-13T00:08:57.622 INFO:tasks.workunit.client.0.smithi136.stdout:op 2332 completed, throughput=5MB/sec 2024-09-13T00:08:57.623 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj--30ydjBKxtOOoN7 off=4137740 len=1332355 2024-09-13T00:08:57.633 INFO:tasks.workunit.client.0.smithi136.stdout:op 2333 completed, throughput=5MB/sec 2024-09-13T00:08:58.636 INFO:tasks.workunit.client.0.smithi136.stdout: 441: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:58.636 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-HtClZe920waRtDg off=1957094 len=750592 2024-09-13T00:08:58.637 INFO:tasks.workunit.client.0.smithi136.stdout:op 2334 completed, throughput=4.99MB/sec 2024-09-13T00:08:58.637 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-6mcsi-W8_Fk2sLm off=7039995 len=1823582 2024-09-13T00:08:58.640 INFO:tasks.workunit.client.0.smithi136.stdout:op 2335 completed, throughput=5MB/sec 2024-09-13T00:08:58.641 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-FR_shSLzBPaTgIe off=0 len=1118029 2024-09-13T00:08:58.650 INFO:tasks.workunit.client.0.smithi136.stdout:op 2336 completed, throughput=5MB/sec 2024-09-13T00:08:58.650 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-72Fpwu03HMXib5X off=487842 len=770578 2024-09-13T00:08:58.658 INFO:tasks.workunit.client.0.smithi136.stdout:op 2337 completed, throughput=5MB/sec 2024-09-13T00:08:58.658 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-fpsSYQL68lUORZl off=0 len=1157378 2024-09-13T00:08:58.662 INFO:tasks.workunit.client.0.smithi136.stdout:op 2338 completed, throughput=5MB/sec 2024-09-13T00:08:59.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:08:58 smithi136 bash[75164]: cluster 2024-09-13T00:08:57.374352+0000 mgr.y (mgr.24661) 955 : cluster [DBG] pgmap v1638: 105 pgs: 105 active+clean; 126 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 3.5 MiB/s rd, 2.1 MiB/s wr, 5 op/s 2024-09-13T00:08:59.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:08:58 smithi136 bash[76371]: cluster 2024-09-13T00:08:57.374352+0000 mgr.y (mgr.24661) 955 : cluster [DBG] pgmap v1638: 105 pgs: 105 active+clean; 126 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 3.5 MiB/s rd, 2.1 MiB/s wr, 5 op/s 2024-09-13T00:08:59.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:08:58 smithi181 bash[67587]: cluster 2024-09-13T00:08:57.374352+0000 mgr.y (mgr.24661) 955 : cluster [DBG] pgmap v1638: 105 pgs: 105 active+clean; 126 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 3.5 MiB/s rd, 2.1 MiB/s wr, 5 op/s 2024-09-13T00:08:59.662 INFO:tasks.workunit.client.0.smithi136.stdout: 442: throughput=4.99MB/sec pending data=0 2024-09-13T00:08:59.662 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-9KGiVw7XuI2umbb off=20883227 len=1185411 2024-09-13T00:08:59.671 INFO:tasks.workunit.client.0.smithi136.stdout:op 2339 completed, throughput=4.99MB/sec 2024-09-13T00:08:59.671 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-LtRTGLjYqUq5JLH off=824635 len=627545 2024-09-13T00:08:59.674 INFO:tasks.workunit.client.0.smithi136.stdout:op 2340 completed, throughput=4.99MB/sec 2024-09-13T00:08:59.674 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-umyn6udE1eQ3YN- off=10004094 len=943041 2024-09-13T00:08:59.677 INFO:tasks.workunit.client.0.smithi136.stdout:op 2341 completed, throughput=5MB/sec 2024-09-13T00:08:59.677 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-4SpoR6HcYH7HALf off=439322 len=1152110 2024-09-13T00:08:59.681 INFO:tasks.workunit.client.0.smithi136.stdout:op 2342 completed, throughput=5MB/sec 2024-09-13T00:08:59.681 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-NcFxodbQRlOl8of off=1934584 len=1204651 2024-09-13T00:08:59.690 INFO:tasks.workunit.client.0.smithi136.stdout:op 2343 completed, throughput=5MB/sec 2024-09-13T00:08:59.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:08:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:08:59] "GET /metrics HTTP/1.1" 200 34141 "" "Prometheus/2.43.0" 2024-09-13T00:09:00.690 INFO:tasks.workunit.client.0.smithi136.stdout: 443: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:00.691 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-XoVhRtC9LALQv19 off=9151243 len=2050806 2024-09-13T00:09:00.707 INFO:tasks.workunit.client.0.smithi136.stdout:op 2344 completed, throughput=4.99MB/sec 2024-09-13T00:09:00.707 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-fMsoJieeP1qz49d off=2173641 len=1282563 2024-09-13T00:09:00.711 INFO:tasks.workunit.client.0.smithi136.stdout:op 2345 completed, throughput=5MB/sec 2024-09-13T00:09:00.711 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-26GeHpAOmdnPV4B off=8697790 len=834786 2024-09-13T00:09:00.714 INFO:tasks.workunit.client.0.smithi136.stdout:op 2346 completed, throughput=5MB/sec 2024-09-13T00:09:00.714 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-sfrzkCVWKF35pND off=3548426 len=235253 2024-09-13T00:09:00.715 INFO:tasks.workunit.client.0.smithi136.stdout:op 2347 completed, throughput=5MB/sec 2024-09-13T00:09:00.715 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Bd5OqLfTUTzC71m off=9503737 len=459893 2024-09-13T00:09:00.721 INFO:tasks.workunit.client.0.smithi136.stdout:op 2348 completed, throughput=5MB/sec 2024-09-13T00:09:00.721 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-i059RDrESy9aeXS off=13324288 len=1776080 2024-09-13T00:09:00.725 INFO:tasks.workunit.client.0.smithi136.stdout:op 2349 completed, throughput=5MB/sec 2024-09-13T00:09:01.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:09:00 smithi136 bash[75164]: cluster 2024-09-13T00:08:59.375256+0000 mgr.y (mgr.24661) 956 : cluster [DBG] pgmap v1639: 105 pgs: 105 active+clean; 126 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:09:01.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:09:00 smithi136 bash[76371]: cluster 2024-09-13T00:08:59.375256+0000 mgr.y (mgr.24661) 956 : cluster [DBG] pgmap v1639: 105 pgs: 105 active+clean; 126 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:09:01.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:09:00 smithi181 bash[67587]: cluster 2024-09-13T00:08:59.375256+0000 mgr.y (mgr.24661) 956 : cluster [DBG] pgmap v1639: 105 pgs: 105 active+clean; 126 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:09:01.725 INFO:tasks.workunit.client.0.smithi136.stdout: 444: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:01.726 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-GVynPhzk-l95hSX off=337280 len=1396014 2024-09-13T00:09:01.730 INFO:tasks.workunit.client.0.smithi136.stdout:op 2350 completed, throughput=5MB/sec 2024-09-13T00:09:01.730 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-IJ2XvGNi7yRrbFH off=5333131 len=839332 2024-09-13T00:09:01.738 INFO:tasks.workunit.client.0.smithi136.stdout:op 2351 completed, throughput=5MB/sec 2024-09-13T00:09:01.738 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-3q10ko2thww8ahx off=2849698 len=1030723 2024-09-13T00:09:01.748 INFO:tasks.workunit.client.0.smithi136.stdout:op 2352 completed, throughput=5MB/sec 2024-09-13T00:09:01.748 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-PbJjPSjTmjcvXBy off=20716575 len=129651 2024-09-13T00:09:01.752 INFO:tasks.workunit.client.0.smithi136.stdout:op 2353 completed, throughput=5MB/sec 2024-09-13T00:09:01.752 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-5uoc1gSOeXhvgVR off=1859441 len=2077991 2024-09-13T00:09:01.767 INFO:tasks.workunit.client.0.smithi136.stdout:op 2354 completed, throughput=5MB/sec 2024-09-13T00:09:02.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:09:02 smithi181 bash[95505]: ts=2024-09-13T00:09:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:09:02.768 INFO:tasks.workunit.client.0.smithi136.stdout: 445: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:02.768 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-jJXvt2NqqdE6wxP off=4395262 len=1030238 2024-09-13T00:09:02.776 INFO:tasks.workunit.client.0.smithi136.stdout:op 2355 completed, throughput=4.99MB/sec 2024-09-13T00:09:02.777 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-XrsFIgs-7uWqiSr off=12386661 len=470393 2024-09-13T00:09:02.782 INFO:tasks.workunit.client.0.smithi136.stdout:op 2356 completed, throughput=5MB/sec 2024-09-13T00:09:02.783 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-6KoZ9xsHZJK5OZs off=8553177 len=74233 2024-09-13T00:09:02.784 INFO:tasks.workunit.client.0.smithi136.stdout:op 2357 completed, throughput=5MB/sec 2024-09-13T00:09:02.784 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-zH4yPFxMOXqLW04 off=745043 len=460186 2024-09-13T00:09:02.789 INFO:tasks.workunit.client.0.smithi136.stdout:op 2358 completed, throughput=5MB/sec 2024-09-13T00:09:02.789 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ZP4lw1Zrq0f8Wk8 off=2249804 len=946036 2024-09-13T00:09:02.791 INFO:tasks.workunit.client.0.smithi136.stdout:op 2359 completed, throughput=5MB/sec 2024-09-13T00:09:02.792 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-lNmCSy13-gQ8uEV off=1569856 len=1193560 2024-09-13T00:09:02.801 INFO:tasks.workunit.client.0.smithi136.stdout:op 2360 completed, throughput=5MB/sec 2024-09-13T00:09:03.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:09:02 smithi136 bash[75164]: cluster 2024-09-13T00:09:01.376353+0000 mgr.y (mgr.24661) 957 : cluster [DBG] pgmap v1640: 105 pgs: 105 active+clean; 126 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:09:03.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:09:02 smithi136 bash[76371]: cluster 2024-09-13T00:09:01.376353+0000 mgr.y (mgr.24661) 957 : cluster [DBG] pgmap v1640: 105 pgs: 105 active+clean; 126 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:09:03.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:09:02 smithi181 bash[67587]: cluster 2024-09-13T00:09:01.376353+0000 mgr.y (mgr.24661) 957 : cluster [DBG] pgmap v1640: 105 pgs: 105 active+clean; 126 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:09:03.801 INFO:tasks.workunit.client.0.smithi136.stdout: 446: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:03.801 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-TiSgjdNkuSxpDkr off=2486640 len=572071 2024-09-13T00:09:03.809 INFO:tasks.workunit.client.0.smithi136.stdout:op 2361 completed, throughput=4.99MB/sec 2024-09-13T00:09:03.809 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-sXrinWuhfHo4IeA off=2264305 len=1275754 2024-09-13T00:09:03.819 INFO:tasks.workunit.client.0.smithi136.stdout:op 2362 completed, throughput=4.99MB/sec 2024-09-13T00:09:03.820 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-GcKD9RSFVFYdhn9 off=334469 len=773628 2024-09-13T00:09:03.827 INFO:tasks.workunit.client.0.smithi136.stdout:op 2363 completed, throughput=4.99MB/sec 2024-09-13T00:09:03.828 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-8UunpRXm2IRWGf6 off=21511733 len=1569354 2024-09-13T00:09:03.831 INFO:tasks.workunit.client.0.smithi136.stdout:op 2364 completed, throughput=5MB/sec 2024-09-13T00:09:03.831 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-FRz_1ouellqDvpE off=34950 len=1215118 2024-09-13T00:09:03.843 INFO:tasks.workunit.client.0.smithi136.stdout:op 2365 completed, throughput=5MB/sec 2024-09-13T00:09:04.843 INFO:tasks.workunit.client.0.smithi136.stdout: 447: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:04.843 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-g1rx-LrcFuOdH2a off=503697 len=261319 2024-09-13T00:09:04.849 INFO:tasks.workunit.client.0.smithi136.stdout:op 2366 completed, throughput=4.99MB/sec 2024-09-13T00:09:04.850 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-rq3fTw94ec-bocV off=667440 len=551589 2024-09-13T00:09:04.856 INFO:tasks.workunit.client.0.smithi136.stdout:op 2367 completed, throughput=4.99MB/sec 2024-09-13T00:09:04.856 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-fxb1BO1Z0B6HKbG off=2337983 len=733459 2024-09-13T00:09:04.858 INFO:tasks.workunit.client.0.smithi136.stdout:op 2368 completed, throughput=4.99MB/sec 2024-09-13T00:09:04.859 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-R7n8SAhBEIkC2tc off=3531 len=1858476 2024-09-13T00:09:04.863 INFO:tasks.workunit.client.0.smithi136.stdout:op 2369 completed, throughput=5MB/sec 2024-09-13T00:09:04.863 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-dYBC54xZKF6dapa off=9659113 len=394182 2024-09-13T00:09:04.864 INFO:tasks.workunit.client.0.smithi136.stdout:op 2370 completed, throughput=5MB/sec 2024-09-13T00:09:04.865 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-M1rSRT1oFY7QHK2 off=7217459 len=544350 2024-09-13T00:09:04.871 INFO:tasks.workunit.client.0.smithi136.stdout:op 2371 completed, throughput=5MB/sec 2024-09-13T00:09:04.871 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-thZpwV_7h-jUk6r off=5760179 len=1543337 2024-09-13T00:09:04.882 INFO:tasks.workunit.client.0.smithi136.stdout:op 2372 completed, throughput=5MB/sec 2024-09-13T00:09:05.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:09:04 smithi136 bash[75164]: cluster 2024-09-13T00:09:03.377354+0000 mgr.y (mgr.24661) 958 : cluster [DBG] pgmap v1641: 105 pgs: 105 active+clean; 126 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:09:05.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:09:04 smithi136 bash[76371]: cluster 2024-09-13T00:09:03.377354+0000 mgr.y (mgr.24661) 958 : cluster [DBG] pgmap v1641: 105 pgs: 105 active+clean; 126 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:09:05.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:09:04 smithi181 bash[67587]: cluster 2024-09-13T00:09:03.377354+0000 mgr.y (mgr.24661) 958 : cluster [DBG] pgmap v1641: 105 pgs: 105 active+clean; 126 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:09:05.882 INFO:tasks.workunit.client.0.smithi136.stdout: 448: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:05.882 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-0GvrQHwLiVq3U3G off=2002462 len=46728 2024-09-13T00:09:05.886 INFO:tasks.workunit.client.0.smithi136.stdout:op 2373 completed, throughput=4.99MB/sec 2024-09-13T00:09:05.886 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-VV_qd_BJj48Q91L off=17478346 len=1496098 2024-09-13T00:09:05.899 INFO:tasks.workunit.client.0.smithi136.stdout:op 2374 completed, throughput=4.99MB/sec 2024-09-13T00:09:05.899 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-s254A9D3cPmJChW off=10675313 len=2007789 2024-09-13T00:09:05.915 INFO:tasks.workunit.client.0.smithi136.stdout:op 2375 completed, throughput=5MB/sec 2024-09-13T00:09:05.915 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-eFMzMS1H1Y4GkZp off=224209 len=281042 2024-09-13T00:09:05.918 INFO:tasks.workunit.client.0.smithi136.stdout:op 2376 completed, throughput=5MB/sec 2024-09-13T00:09:05.918 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ACsOPEHhq13FNPA off=2221867 len=1588491 2024-09-13T00:09:05.922 INFO:tasks.workunit.client.0.smithi136.stdout:op 2377 completed, throughput=5MB/sec 2024-09-13T00:09:06.922 INFO:tasks.workunit.client.0.smithi136.stdout: 449: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:06.922 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-8RBYzGrOXmejV48 off=17277342 len=1752158 2024-09-13T00:09:06.935 INFO:tasks.workunit.client.0.smithi136.stdout:op 2378 completed, throughput=4.99MB/sec 2024-09-13T00:09:06.935 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-3X0h57Tw5HcgHC1 off=1573567 len=1932378 2024-09-13T00:09:06.947 INFO:tasks.workunit.client.0.smithi136.stdout:op 2379 completed, throughput=5MB/sec 2024-09-13T00:09:06.947 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-lcNImTHwV4rbu1l off=1436560 len=1618041 2024-09-13T00:09:06.953 INFO:tasks.workunit.client.0.smithi136.stdout:op 2380 completed, throughput=5MB/sec 2024-09-13T00:09:07.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:09:06 smithi136 bash[75164]: cluster 2024-09-13T00:09:05.378181+0000 mgr.y (mgr.24661) 959 : cluster [DBG] pgmap v1642: 105 pgs: 105 active+clean; 126 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:09:07.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:09:06 smithi136 bash[76371]: cluster 2024-09-13T00:09:05.378181+0000 mgr.y (mgr.24661) 959 : cluster [DBG] pgmap v1642: 105 pgs: 105 active+clean; 126 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:09:07.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:09:06 smithi181 bash[67587]: cluster 2024-09-13T00:09:05.378181+0000 mgr.y (mgr.24661) 959 : cluster [DBG] pgmap v1642: 105 pgs: 105 active+clean; 126 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:09:07.953 INFO:tasks.workunit.client.0.smithi136.stdout: 450: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:07.953 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-wNSjj06QzHS-G4l off=3706499 len=626103 2024-09-13T00:09:07.955 INFO:tasks.workunit.client.0.smithi136.stdout:op 2381 completed, throughput=4.99MB/sec 2024-09-13T00:09:07.955 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-RB_m2NbdtktIR7Q off=5025377 len=1644570 2024-09-13T00:09:07.959 INFO:tasks.workunit.client.0.smithi136.stdout:op 2382 completed, throughput=4.99MB/sec 2024-09-13T00:09:07.959 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-VsEiRw085mf7OtV off=238655 len=309237 2024-09-13T00:09:07.961 INFO:tasks.workunit.client.0.smithi136.stdout:op 2383 completed, throughput=5MB/sec 2024-09-13T00:09:07.961 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-LGrMD2AZKQeTlG8 off=907706 len=846046 2024-09-13T00:09:07.968 INFO:tasks.workunit.client.0.smithi136.stdout:op 2384 completed, throughput=5MB/sec 2024-09-13T00:09:07.968 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-6n1c5UdHwfzK9eW off=14194537 len=1733648 2024-09-13T00:09:07.974 INFO:tasks.workunit.client.0.smithi136.stdout:op 2385 completed, throughput=5MB/sec 2024-09-13T00:09:08.974 INFO:tasks.workunit.client.0.smithi136.stdout: 451: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:08.974 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ENMfrXmrLTx5g4W off=1017010 len=1218414 2024-09-13T00:09:08.978 INFO:tasks.workunit.client.0.smithi136.stdout:op 2386 completed, throughput=4.99MB/sec 2024-09-13T00:09:08.978 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-LFwAHLfSKN1tzbA off=1109958 len=875114 2024-09-13T00:09:08.980 INFO:tasks.workunit.client.0.smithi136.stdout:op 2387 completed, throughput=4.99MB/sec 2024-09-13T00:09:08.980 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-vWn979gaeBdyOUO off=1218373 len=1302789 2024-09-13T00:09:08.993 INFO:tasks.workunit.client.0.smithi136.stdout:op 2388 completed, throughput=5MB/sec 2024-09-13T00:09:08.993 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-IU_QKG6AdQQxBLf off=5444259 len=872930 2024-09-13T00:09:08.995 INFO:tasks.workunit.client.0.smithi136.stdout:op 2389 completed, throughput=5MB/sec 2024-09-13T00:09:08.995 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-JCLSek_KW4siu9p off=11137226 len=2002319 2024-09-13T00:09:09.008 INFO:tasks.workunit.client.0.smithi136.stdout:op 2390 completed, throughput=5MB/sec 2024-09-13T00:09:09.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:09:08 smithi136 bash[75164]: cluster 2024-09-13T00:09:07.379297+0000 mgr.y (mgr.24661) 960 : cluster [DBG] pgmap v1643: 105 pgs: 105 active+clean; 126 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:09:09.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:09:08 smithi136 bash[75164]: audit 2024-09-13T00:09:08.784323+0000 mon.a (mon.0) 2175 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:09:09.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:09:08 smithi136 bash[76371]: cluster 2024-09-13T00:09:07.379297+0000 mgr.y (mgr.24661) 960 : cluster [DBG] pgmap v1643: 105 pgs: 105 active+clean; 126 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:09:09.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:09:08 smithi136 bash[76371]: audit 2024-09-13T00:09:08.784323+0000 mon.a (mon.0) 2175 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:09:09.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:09:08 smithi181 bash[67587]: cluster 2024-09-13T00:09:07.379297+0000 mgr.y (mgr.24661) 960 : cluster [DBG] pgmap v1643: 105 pgs: 105 active+clean; 126 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:09:09.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:09:08 smithi181 bash[67587]: audit 2024-09-13T00:09:08.784323+0000 mon.a (mon.0) 2175 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:09:09.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:09:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:09:09] "GET /metrics HTTP/1.1" 200 34142 "" "Prometheus/2.43.0" 2024-09-13T00:09:10.008 INFO:tasks.workunit.client.0.smithi136.stdout: 452: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:10.008 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-lpCRzvuw48OLPEG off=4289259 len=1938421 2024-09-13T00:09:10.014 INFO:tasks.workunit.client.0.smithi136.stdout:op 2391 completed, throughput=5MB/sec 2024-09-13T00:09:10.015 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-rPaITeF_s0olF0D off=13884206 len=1009354 2024-09-13T00:09:10.018 INFO:tasks.workunit.client.0.smithi136.stdout:op 2392 completed, throughput=5MB/sec 2024-09-13T00:09:10.019 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-EI_b8HRaypqJD0B off=439271 len=718756 2024-09-13T00:09:10.026 INFO:tasks.workunit.client.0.smithi136.stdout:op 2393 completed, throughput=5MB/sec 2024-09-13T00:09:10.027 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-rE2H3ILm3HNMeg4 off=465682 len=199210 2024-09-13T00:09:10.028 INFO:tasks.workunit.client.0.smithi136.stdout:op 2394 completed, throughput=5MB/sec 2024-09-13T00:09:10.028 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-jkl1-ERuRYpwJLl off=11880115 len=1405525 2024-09-13T00:09:10.032 INFO:tasks.workunit.client.0.smithi136.stdout:op 2395 completed, throughput=5MB/sec 2024-09-13T00:09:11.032 INFO:tasks.workunit.client.0.smithi136.stdout: 453: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:11.032 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-wl9BXxOGuAu_Df9 off=2276848 len=391377 2024-09-13T00:09:11.034 INFO:tasks.workunit.client.0.smithi136.stdout:op 2396 completed, throughput=4.99MB/sec 2024-09-13T00:09:11.034 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-AUWpxuLNQztZgMB off=3158463 len=1122237 2024-09-13T00:09:11.037 INFO:tasks.workunit.client.0.smithi136.stdout:op 2397 completed, throughput=4.99MB/sec 2024-09-13T00:09:11.037 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-oldX9u8JHmvEnG- off=15651828 len=727446 2024-09-13T00:09:11.044 INFO:tasks.workunit.client.0.smithi136.stdout:op 2398 completed, throughput=5MB/sec 2024-09-13T00:09:11.044 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-W7fcbQsmBaHYYdg off=2036675 len=2054625 2024-09-13T00:09:11.049 INFO:tasks.workunit.client.0.smithi136.stdout:op 2399 completed, throughput=5MB/sec 2024-09-13T00:09:11.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:09:10 smithi136 bash[75164]: cluster 2024-09-13T00:09:09.380165+0000 mgr.y (mgr.24661) 961 : cluster [DBG] pgmap v1644: 105 pgs: 105 active+clean; 126 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 1.9 MiB/s rd, 2.7 MiB/s wr, 4 op/s 2024-09-13T00:09:11.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:09:10 smithi136 bash[76371]: cluster 2024-09-13T00:09:09.380165+0000 mgr.y (mgr.24661) 961 : cluster [DBG] pgmap v1644: 105 pgs: 105 active+clean; 126 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 1.9 MiB/s rd, 2.7 MiB/s wr, 4 op/s 2024-09-13T00:09:11.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:09:10 smithi181 bash[67587]: cluster 2024-09-13T00:09:09.380165+0000 mgr.y (mgr.24661) 961 : cluster [DBG] pgmap v1644: 105 pgs: 105 active+clean; 126 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 1.9 MiB/s rd, 2.7 MiB/s wr, 4 op/s 2024-09-13T00:09:12.049 INFO:tasks.workunit.client.0.smithi136.stdout: 454: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:12.049 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-xzfvtu_tPmMWo-R off=2548763 len=496407 2024-09-13T00:09:12.052 INFO:tasks.workunit.client.0.smithi136.stdout:op 2400 completed, throughput=4.99MB/sec 2024-09-13T00:09:12.052 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-GxKgFgDI-6HEJz6 off=657650 len=226322 2024-09-13T00:09:12.054 INFO:tasks.workunit.client.0.smithi136.stdout:op 2401 completed, throughput=4.99MB/sec 2024-09-13T00:09:12.054 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-r1DxT2ElR_dFiHD off=7320495 len=1508073 2024-09-13T00:09:12.064 INFO:tasks.workunit.client.0.smithi136.stdout:op 2402 completed, throughput=4.99MB/sec 2024-09-13T00:09:12.064 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-x_2wMfgT_z5eSLc off=8148510 len=861196 2024-09-13T00:09:12.066 INFO:tasks.workunit.client.0.smithi136.stdout:op 2403 completed, throughput=5MB/sec 2024-09-13T00:09:12.067 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-FuWvsYpEA9Smwn9 off=275566 len=999633 2024-09-13T00:09:12.075 INFO:tasks.workunit.client.0.smithi136.stdout:op 2404 completed, throughput=5MB/sec 2024-09-13T00:09:12.076 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-NEliPv7JV8FvnDT off=14486520 len=1375416 2024-09-13T00:09:12.086 INFO:tasks.workunit.client.0.smithi136.stdout:op 2405 completed, throughput=5MB/sec 2024-09-13T00:09:12.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:09:12 smithi181 bash[95505]: ts=2024-09-13T00:09:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:09:13.086 INFO:tasks.workunit.client.0.smithi136.stdout: 455: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:13.087 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-2HGWzD4iveZ3g49 off=13264466 len=344285 2024-09-13T00:09:13.092 INFO:tasks.workunit.client.0.smithi136.stdout:op 2406 completed, throughput=4.99MB/sec 2024-09-13T00:09:13.092 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-wBABaPGixm9RBRr off=3446357 len=685552 2024-09-13T00:09:13.100 INFO:tasks.workunit.client.0.smithi136.stdout:op 2407 completed, throughput=4.99MB/sec 2024-09-13T00:09:13.100 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-t_Fbff-_Tlrzn_I off=7283858 len=1680233 2024-09-13T00:09:13.114 INFO:tasks.workunit.client.0.smithi136.stdout:op 2408 completed, throughput=4.99MB/sec 2024-09-13T00:09:13.114 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-MJYutaNeurI8Dei off=3146525 len=317030 2024-09-13T00:09:13.116 INFO:tasks.workunit.client.0.smithi136.stdout:op 2409 completed, throughput=5MB/sec 2024-09-13T00:09:13.116 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-p9Fg0GBNSMai2wr off=5020077 len=1218909 2024-09-13T00:09:13.119 INFO:tasks.workunit.client.0.smithi136.stdout:op 2410 completed, throughput=5MB/sec 2024-09-13T00:09:13.119 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-vK37dGAyKtiuFhS off=17610890 len=1348501 2024-09-13T00:09:13.123 INFO:tasks.workunit.client.0.smithi136.stdout:op 2411 completed, throughput=5MB/sec 2024-09-13T00:09:13.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:09:12 smithi136 bash[75164]: cluster 2024-09-13T00:09:11.381271+0000 mgr.y (mgr.24661) 962 : cluster [DBG] pgmap v1645: 105 pgs: 105 active+clean; 126 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:09:13.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:09:12 smithi136 bash[76371]: cluster 2024-09-13T00:09:11.381271+0000 mgr.y (mgr.24661) 962 : cluster [DBG] pgmap v1645: 105 pgs: 105 active+clean; 126 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:09:13.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:09:12 smithi181 bash[67587]: cluster 2024-09-13T00:09:11.381271+0000 mgr.y (mgr.24661) 962 : cluster [DBG] pgmap v1645: 105 pgs: 105 active+clean; 126 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:09:14.123 INFO:tasks.workunit.client.0.smithi136.stdout: 456: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:14.123 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-xD-OGeEMy54Gen8 off=989789 len=861028 2024-09-13T00:09:14.132 INFO:tasks.workunit.client.0.smithi136.stdout:op 2412 completed, throughput=4.99MB/sec 2024-09-13T00:09:14.132 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-jBSy7AWeHc2HLOS off=15818445 len=55887 2024-09-13T00:09:14.134 INFO:tasks.workunit.client.0.smithi136.stdout:op 2413 completed, throughput=4.99MB/sec 2024-09-13T00:09:14.134 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-l4yae0xGbk1LQTr off=3873703 len=1967319 2024-09-13T00:09:14.139 INFO:tasks.workunit.client.0.smithi136.stdout:op 2414 completed, throughput=5MB/sec 2024-09-13T00:09:14.139 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-o5JuPYQZt_vxv2l off=7197939 len=1111027 2024-09-13T00:09:14.144 INFO:tasks.workunit.client.0.smithi136.stdout:op 2415 completed, throughput=5MB/sec 2024-09-13T00:09:14.144 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-qZkjZZA9DRzn8Do off=1416049 len=695968 2024-09-13T00:09:14.146 INFO:tasks.workunit.client.0.smithi136.stdout:op 2416 completed, throughput=5MB/sec 2024-09-13T00:09:14.146 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Hk5dZ_Ns7c4BTK1 off=8852129 len=1901588 2024-09-13T00:09:14.158 INFO:tasks.workunit.client.0.smithi136.stdout:op 2417 completed, throughput=5MB/sec 2024-09-13T00:09:15.159 INFO:tasks.workunit.client.0.smithi136.stdout: 457: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:15.159 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-4ifnufJtcG1oyNK off=14354630 len=1319132 2024-09-13T00:09:15.162 INFO:tasks.workunit.client.0.smithi136.stdout:op 2418 completed, throughput=4.99MB/sec 2024-09-13T00:09:15.162 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-039riKmm2A-UBsb off=9990138 len=1432164 2024-09-13T00:09:15.172 INFO:tasks.workunit.client.0.smithi136.stdout:op 2419 completed, throughput=5MB/sec 2024-09-13T00:09:15.172 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-tDOaufq8uOTkp7p off=10825099 len=546116 2024-09-13T00:09:15.175 INFO:tasks.workunit.client.0.smithi136.stdout:op 2420 completed, throughput=5MB/sec 2024-09-13T00:09:15.175 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-vJC3pFLKTJo5bQq off=4545596 len=1095371 2024-09-13T00:09:15.177 INFO:tasks.workunit.client.0.smithi136.stdout:op 2421 completed, throughput=5MB/sec 2024-09-13T00:09:15.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:09:14 smithi136 bash[75164]: cluster 2024-09-13T00:09:13.382289+0000 mgr.y (mgr.24661) 963 : cluster [DBG] pgmap v1646: 105 pgs: 105 active+clean; 126 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:09:15.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:09:14 smithi136 bash[76371]: cluster 2024-09-13T00:09:13.382289+0000 mgr.y (mgr.24661) 963 : cluster [DBG] pgmap v1646: 105 pgs: 105 active+clean; 126 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:09:15.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:09:14 smithi181 bash[67587]: cluster 2024-09-13T00:09:13.382289+0000 mgr.y (mgr.24661) 963 : cluster [DBG] pgmap v1646: 105 pgs: 105 active+clean; 126 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:09:16.177 INFO:tasks.workunit.client.0.smithi136.stdout: 458: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:16.177 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-JBQtw-oI3t2bdmU off=12792687 len=1553985 2024-09-13T00:09:16.181 INFO:tasks.workunit.client.0.smithi136.stdout:op 2422 completed, throughput=4.99MB/sec 2024-09-13T00:09:16.181 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-clx0UtHlX6eRgb- off=9705951 len=1300777 2024-09-13T00:09:16.193 INFO:tasks.workunit.client.0.smithi136.stdout:op 2423 completed, throughput=5MB/sec 2024-09-13T00:09:16.193 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-oPnVTv381LVG16H off=6278997 len=1877892 2024-09-13T00:09:16.198 INFO:tasks.workunit.client.0.smithi136.stdout:op 2424 completed, throughput=5MB/sec 2024-09-13T00:09:16.198 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Dbv0qls2_-77iil off=943078 len=279351 2024-09-13T00:09:16.200 INFO:tasks.workunit.client.0.smithi136.stdout:op 2425 completed, throughput=5MB/sec 2024-09-13T00:09:17.201 INFO:tasks.workunit.client.0.smithi136.stdout: 459: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:17.201 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-vW4WhMxOhyNDa2q off=189219 len=313723 2024-09-13T00:09:17.206 INFO:tasks.workunit.client.0.smithi136.stdout:op 2426 completed, throughput=4.99MB/sec 2024-09-13T00:09:17.206 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2Jb5vZeTrBDy8E7 off=15244559 len=757024 2024-09-13T00:09:17.208 INFO:tasks.workunit.client.0.smithi136.stdout:op 2427 completed, throughput=4.99MB/sec 2024-09-13T00:09:17.208 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ngKFDjZM9ZDMeMf off=250314 len=2001647 2024-09-13T00:09:17.226 INFO:tasks.workunit.client.0.smithi136.stdout:op 2428 completed, throughput=5MB/sec 2024-09-13T00:09:17.226 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-K5d-sqeZfIyjyg4 off=7732503 len=4955 2024-09-13T00:09:17.231 INFO:tasks.workunit.client.0.smithi136.stdout:op 2429 completed, throughput=5MB/sec 2024-09-13T00:09:17.231 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-1KjVJQR21MBlkT1 off=2918100 len=1235098 2024-09-13T00:09:17.234 INFO:tasks.workunit.client.0.smithi136.stdout:op 2430 completed, throughput=5MB/sec 2024-09-13T00:09:17.234 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-G8cwn2otG4huPFy off=14669550 len=111947 2024-09-13T00:09:17.235 INFO:tasks.workunit.client.0.smithi136.stdout:op 2431 completed, throughput=5MB/sec 2024-09-13T00:09:17.235 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-OMhQyipKkOmDhUP off=8191273 len=1443675 2024-09-13T00:09:17.245 INFO:tasks.workunit.client.0.smithi136.stdout:op 2432 completed, throughput=5MB/sec 2024-09-13T00:09:17.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:09:16 smithi136 bash[75164]: cluster 2024-09-13T00:09:15.383125+0000 mgr.y (mgr.24661) 964 : cluster [DBG] pgmap v1647: 105 pgs: 105 active+clean; 126 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:09:17.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:09:16 smithi136 bash[76371]: cluster 2024-09-13T00:09:15.383125+0000 mgr.y (mgr.24661) 964 : cluster [DBG] pgmap v1647: 105 pgs: 105 active+clean; 126 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:09:17.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:09:16 smithi181 bash[67587]: cluster 2024-09-13T00:09:15.383125+0000 mgr.y (mgr.24661) 964 : cluster [DBG] pgmap v1647: 105 pgs: 105 active+clean; 126 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:09:18.245 INFO:tasks.workunit.client.0.smithi136.stdout: 460: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:18.246 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-wBABaPGixm9RBRr off=1511282 len=1461976 2024-09-13T00:09:18.257 INFO:tasks.workunit.client.0.smithi136.stdout:op 2433 completed, throughput=4.99MB/sec 2024-09-13T00:09:18.257 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-FP8dLsdxNp7skMd off=5737304 len=182741 2024-09-13T00:09:18.259 INFO:tasks.workunit.client.0.smithi136.stdout:op 2434 completed, throughput=4.99MB/sec 2024-09-13T00:09:18.259 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-61WqVQDs_iVR1mm off=4848656 len=1107268 2024-09-13T00:09:18.262 INFO:tasks.workunit.client.0.smithi136.stdout:op 2435 completed, throughput=5MB/sec 2024-09-13T00:09:18.262 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-kOa3zKkubUWtKcB off=8892332 len=968361 2024-09-13T00:09:18.265 INFO:tasks.workunit.client.0.smithi136.stdout:op 2436 completed, throughput=5MB/sec 2024-09-13T00:09:18.265 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-GRa0p_PvUSpeOYM off=8758212 len=1246500 2024-09-13T00:09:18.268 INFO:tasks.workunit.client.0.smithi136.stdout:op 2437 completed, throughput=5MB/sec 2024-09-13T00:09:19.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:09:18 smithi136 bash[75164]: cluster 2024-09-13T00:09:17.384291+0000 mgr.y (mgr.24661) 965 : cluster [DBG] pgmap v1648: 105 pgs: 105 active+clean; 126 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:09:19.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:09:18 smithi136 bash[76371]: cluster 2024-09-13T00:09:17.384291+0000 mgr.y (mgr.24661) 965 : cluster [DBG] pgmap v1648: 105 pgs: 105 active+clean; 126 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:09:19.268 INFO:tasks.workunit.client.0.smithi136.stdout: 461: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:19.268 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-pR8E71IMuNKAzW0 off=134257 len=312847 2024-09-13T00:09:19.274 INFO:tasks.workunit.client.0.smithi136.stdout:op 2438 completed, throughput=4.99MB/sec 2024-09-13T00:09:19.274 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-wgUyWV1CzWbMPMg off=11798557 len=1403631 2024-09-13T00:09:19.278 INFO:tasks.workunit.client.0.smithi136.stdout:op 2439 completed, throughput=4.99MB/sec 2024-09-13T00:09:19.278 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-m3EtT7x9Y8_0h8g off=17016262 len=633012 2024-09-13T00:09:19.284 INFO:tasks.workunit.client.0.smithi136.stdout:op 2440 completed, throughput=4.99MB/sec 2024-09-13T00:09:19.284 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj--C0BOycTdn_vEvr off=5531917 len=969482 2024-09-13T00:09:19.287 INFO:tasks.workunit.client.0.smithi136.stdout:op 2441 completed, throughput=5MB/sec 2024-09-13T00:09:19.287 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ryrfKBGeSjy7CDz off=14521610 len=359835 2024-09-13T00:09:19.293 INFO:tasks.workunit.client.0.smithi136.stdout:op 2442 completed, throughput=5MB/sec 2024-09-13T00:09:19.293 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-a9VWcuGWwjkapRm off=1840282 len=381056 2024-09-13T00:09:19.298 INFO:tasks.workunit.client.0.smithi136.stdout:op 2443 completed, throughput=5MB/sec 2024-09-13T00:09:19.298 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-rYlP8XJKjV3KJGm off=1209422 len=487248 2024-09-13T00:09:19.304 INFO:tasks.workunit.client.0.smithi136.stdout:op 2444 completed, throughput=5MB/sec 2024-09-13T00:09:19.304 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-1B8ommSi5cWXLEi off=3502832 len=539604 2024-09-13T00:09:19.310 INFO:tasks.workunit.client.0.smithi136.stdout:op 2445 completed, throughput=5MB/sec 2024-09-13T00:09:19.310 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ptmFfQtGfpGkM5o off=6918701 len=57209 2024-09-13T00:09:19.314 INFO:tasks.workunit.client.0.smithi136.stdout:op 2446 completed, throughput=5MB/sec 2024-09-13T00:09:19.314 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-BPYp0H1sCUzZ0n1 off=5225228 len=925971 2024-09-13T00:09:19.324 INFO:tasks.workunit.client.0.smithi136.stdout:op 2447 completed, throughput=5MB/sec 2024-09-13T00:09:19.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:09:18 smithi181 bash[67587]: cluster 2024-09-13T00:09:17.384291+0000 mgr.y (mgr.24661) 965 : cluster [DBG] pgmap v1648: 105 pgs: 105 active+clean; 126 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:09:19.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:09:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:09:19] "GET /metrics HTTP/1.1" 200 34142 "" "Prometheus/2.43.0" 2024-09-13T00:09:20.322 INFO:tasks.workunit.client.0.smithi136.stdout: 462: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:20.322 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-dPSw1wYSN7z2Qa_ off=883308 len=243193 2024-09-13T00:09:20.323 INFO:tasks.workunit.client.0.smithi136.stdout:op 2448 completed, throughput=4.99MB/sec 2024-09-13T00:09:20.324 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-BUOFZxChCTLXBnI off=4064616 len=1054425 2024-09-13T00:09:20.333 INFO:tasks.workunit.client.0.smithi136.stdout:op 2449 completed, throughput=4.99MB/sec 2024-09-13T00:09:20.333 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-46hGXZU3-CaaYE8 off=3244083 len=423170 2024-09-13T00:09:20.335 INFO:tasks.workunit.client.0.smithi136.stdout:op 2450 completed, throughput=4.99MB/sec 2024-09-13T00:09:20.335 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-5v2CigT4BZGliyN off=8204067 len=1918683 2024-09-13T00:09:20.349 INFO:tasks.workunit.client.0.smithi136.stdout:op 2451 completed, throughput=5MB/sec 2024-09-13T00:09:20.350 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-1NHPhzkxh_TFryj off=15404299 len=785740 2024-09-13T00:09:20.352 INFO:tasks.workunit.client.0.smithi136.stdout:op 2452 completed, throughput=5MB/sec 2024-09-13T00:09:20.352 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-KmS8lbAQnH27D0D off=21152248 len=1171176 2024-09-13T00:09:20.355 INFO:tasks.workunit.client.0.smithi136.stdout:op 2453 completed, throughput=5MB/sec 2024-09-13T00:09:21.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:09:20 smithi136 bash[75164]: cluster 2024-09-13T00:09:19.385226+0000 mgr.y (mgr.24661) 966 : cluster [DBG] pgmap v1649: 105 pgs: 105 active+clean; 126 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 2.8 MiB/s rd, 1.9 MiB/s wr, 4 op/s 2024-09-13T00:09:21.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:09:20 smithi136 bash[76371]: cluster 2024-09-13T00:09:19.385226+0000 mgr.y (mgr.24661) 966 : cluster [DBG] pgmap v1649: 105 pgs: 105 active+clean; 126 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 2.8 MiB/s rd, 1.9 MiB/s wr, 4 op/s 2024-09-13T00:09:21.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:09:20 smithi181 bash[67587]: cluster 2024-09-13T00:09:19.385226+0000 mgr.y (mgr.24661) 966 : cluster [DBG] pgmap v1649: 105 pgs: 105 active+clean; 126 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 2.8 MiB/s rd, 1.9 MiB/s wr, 4 op/s 2024-09-13T00:09:21.355 INFO:tasks.workunit.client.0.smithi136.stdout: 463: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:21.356 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-8a0EyhLi_lXHACg off=5408338 len=1770545 2024-09-13T00:09:21.370 INFO:tasks.workunit.client.0.smithi136.stdout:op 2454 completed, throughput=4.99MB/sec 2024-09-13T00:09:21.370 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-AW6lvXSJKDKWrK9 off=21445901 len=1312305 2024-09-13T00:09:21.374 INFO:tasks.workunit.client.0.smithi136.stdout:op 2455 completed, throughput=5MB/sec 2024-09-13T00:09:21.374 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Pbz9aO3Ot8t1v5Y off=264181 len=197052 2024-09-13T00:09:21.380 INFO:tasks.workunit.client.0.smithi136.stdout:op 2456 completed, throughput=5MB/sec 2024-09-13T00:09:21.381 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-5pS0LoezVgpu4Hk off=2661399 len=1547564 2024-09-13T00:09:21.389 INFO:tasks.workunit.client.0.smithi136.stdout:op 2457 completed, throughput=5MB/sec 2024-09-13T00:09:22.389 INFO:tasks.workunit.client.0.smithi136.stdout: 464: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:22.390 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2Jnk2r9_hZajV2J off=2580327 len=1912795 2024-09-13T00:09:22.395 INFO:tasks.workunit.client.0.smithi136.stdout:op 2458 completed, throughput=4.99MB/sec 2024-09-13T00:09:22.396 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-shrECNZAS_6w97v off=9297264 len=331368 2024-09-13T00:09:22.404 INFO:tasks.workunit.client.0.smithi136.stdout:op 2459 completed, throughput=4.99MB/sec 2024-09-13T00:09:22.404 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-GGuOSh1GZDxCE3p off=13047143 len=2049589 2024-09-13T00:09:22.406 INFO:tasks.workunit.client.0.smithi136.stdout:op 2460 completed, throughput=5MB/sec 2024-09-13T00:09:22.406 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-sMAckbyFf4vd5vZ off=18000112 len=576477 2024-09-13T00:09:22.408 INFO:tasks.workunit.client.0.smithi136.stdout:op 2461 completed, throughput=5MB/sec 2024-09-13T00:09:22.408 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-NdFkjHPI8QufATR off=9826901 len=605544 2024-09-13T00:09:22.410 INFO:tasks.workunit.client.0.smithi136.stdout:op 2462 completed, throughput=5MB/sec 2024-09-13T00:09:22.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:09:22 smithi181 bash[95505]: ts=2024-09-13T00:09:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:09:23.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:09:22 smithi136 bash[75164]: cluster 2024-09-13T00:09:21.386534+0000 mgr.y (mgr.24661) 967 : cluster [DBG] pgmap v1650: 105 pgs: 105 active+clean; 126 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:09:23.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:09:22 smithi136 bash[76371]: cluster 2024-09-13T00:09:21.386534+0000 mgr.y (mgr.24661) 967 : cluster [DBG] pgmap v1650: 105 pgs: 105 active+clean; 126 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:09:23.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:09:22 smithi181 bash[67587]: cluster 2024-09-13T00:09:21.386534+0000 mgr.y (mgr.24661) 967 : cluster [DBG] pgmap v1650: 105 pgs: 105 active+clean; 126 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:09:23.411 INFO:tasks.workunit.client.0.smithi136.stdout: 465: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:23.411 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-JSEo0706dtw3yQp off=23423191 len=1644099 2024-09-13T00:09:23.416 INFO:tasks.workunit.client.0.smithi136.stdout:op 2463 completed, throughput=4.99MB/sec 2024-09-13T00:09:23.416 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-nq-QP9gc0kI1CS8 off=6557675 len=981066 2024-09-13T00:09:23.420 INFO:tasks.workunit.client.0.smithi136.stdout:op 2464 completed, throughput=5MB/sec 2024-09-13T00:09:23.420 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-S4lENDNxiCylZOj off=15563 len=707396 2024-09-13T00:09:23.427 INFO:tasks.workunit.client.0.smithi136.stdout:op 2465 completed, throughput=5MB/sec 2024-09-13T00:09:23.427 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-d06ZDyty0H0h009 off=1338367 len=294168 2024-09-13T00:09:23.433 INFO:tasks.workunit.client.0.smithi136.stdout:op 2466 completed, throughput=5MB/sec 2024-09-13T00:09:23.433 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-uEzkzP08HI45unF off=2584646 len=630997 2024-09-13T00:09:23.435 INFO:tasks.workunit.client.0.smithi136.stdout:op 2467 completed, throughput=5MB/sec 2024-09-13T00:09:23.435 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ZxrE5CZHVeFkv3r off=14021395 len=654208 2024-09-13T00:09:23.442 INFO:tasks.workunit.client.0.smithi136.stdout:op 2468 completed, throughput=5MB/sec 2024-09-13T00:09:23.442 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-w3tyqqm7X9plA9g off=210966 len=1231341 2024-09-13T00:09:23.451 INFO:tasks.workunit.client.0.smithi136.stdout:op 2469 completed, throughput=5MB/sec 2024-09-13T00:09:24.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:09:23 smithi136 bash[75164]: audit 2024-09-13T00:09:23.784774+0000 mon.a (mon.0) 2176 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:09:24.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:09:23 smithi136 bash[76371]: audit 2024-09-13T00:09:23.784774+0000 mon.a (mon.0) 2176 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:09:24.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:09:23 smithi181 bash[67587]: audit 2024-09-13T00:09:23.784774+0000 mon.a (mon.0) 2176 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:09:24.452 INFO:tasks.workunit.client.0.smithi136.stdout: 466: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:24.452 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-NJwVYmhmH7gEguF off=12393461 len=261425 2024-09-13T00:09:24.454 INFO:tasks.workunit.client.0.smithi136.stdout:op 2470 completed, throughput=4.99MB/sec 2024-09-13T00:09:24.454 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-r-WjkElaVJgvjj2 off=1509673 len=1699537 2024-09-13T00:09:24.460 INFO:tasks.workunit.client.0.smithi136.stdout:op 2471 completed, throughput=5MB/sec 2024-09-13T00:09:24.461 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-pPg5rNnhv_WDSaV off=883653 len=1579745 2024-09-13T00:09:24.462 INFO:tasks.workunit.client.0.smithi136.stdout:op 2472 completed, throughput=5MB/sec 2024-09-13T00:09:24.462 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_2qpBodyVpN2Qi6 off=2855742 len=232638 2024-09-13T00:09:24.467 INFO:tasks.workunit.client.0.smithi136.stdout:op 2473 completed, throughput=5MB/sec 2024-09-13T00:09:24.468 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-iIRfadjM3dEZAdD off=19369325 len=1723582 2024-09-13T00:09:24.479 INFO:tasks.workunit.client.0.smithi136.stdout:op 2474 completed, throughput=5MB/sec 2024-09-13T00:09:25.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:09:24 smithi136 bash[75164]: cluster 2024-09-13T00:09:23.387532+0000 mgr.y (mgr.24661) 968 : cluster [DBG] pgmap v1651: 105 pgs: 105 active+clean; 126 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:09:25.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:09:24 smithi136 bash[76371]: cluster 2024-09-13T00:09:23.387532+0000 mgr.y (mgr.24661) 968 : cluster [DBG] pgmap v1651: 105 pgs: 105 active+clean; 126 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:09:25.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:09:24 smithi181 bash[67587]: cluster 2024-09-13T00:09:23.387532+0000 mgr.y (mgr.24661) 968 : cluster [DBG] pgmap v1651: 105 pgs: 105 active+clean; 126 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:09:25.479 INFO:tasks.workunit.client.0.smithi136.stdout: 467: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:25.479 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-lhSpxJw3vDIoeLm off=1284669 len=286405 2024-09-13T00:09:25.481 INFO:tasks.workunit.client.0.smithi136.stdout:op 2475 completed, throughput=4.99MB/sec 2024-09-13T00:09:25.481 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-SReLVd0DXKirgJn off=2819633 len=365173 2024-09-13T00:09:25.485 INFO:tasks.workunit.client.0.smithi136.stdout:op 2476 completed, throughput=4.99MB/sec 2024-09-13T00:09:25.485 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-RDnNwMFapHqDMIy off=11923123 len=1876884 2024-09-13T00:09:25.495 INFO:tasks.workunit.client.0.smithi136.stdout:op 2477 completed, throughput=5MB/sec 2024-09-13T00:09:25.495 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-HXrW9ltpATeFyRr off=3975195 len=1815700 2024-09-13T00:09:25.501 INFO:tasks.workunit.client.0.smithi136.stdout:op 2478 completed, throughput=5MB/sec 2024-09-13T00:09:26.502 INFO:tasks.workunit.client.0.smithi136.stdout: 468: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:26.502 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-2CKCOGa0r2UTav0 off=5555853 len=1895190 2024-09-13T00:09:26.518 INFO:tasks.workunit.client.0.smithi136.stdout:op 2479 completed, throughput=4.99MB/sec 2024-09-13T00:09:26.519 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-iWjdjRpnGSgK_9h off=501837 len=1547607 2024-09-13T00:09:26.522 INFO:tasks.workunit.client.0.smithi136.stdout:op 2480 completed, throughput=5MB/sec 2024-09-13T00:09:26.522 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-VlpVU_BIh8LYDw7 off=8427176 len=298663 2024-09-13T00:09:26.524 INFO:tasks.workunit.client.0.smithi136.stdout:op 2481 completed, throughput=5MB/sec 2024-09-13T00:09:26.524 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-nM3LiNqveYX772Q off=8143210 len=960420 2024-09-13T00:09:26.532 INFO:tasks.workunit.client.0.smithi136.stdout:op 2482 completed, throughput=5MB/sec 2024-09-13T00:09:26.533 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-h5eKEwk5FcLb7GV off=1959253 len=1723481 2024-09-13T00:09:26.535 INFO:tasks.workunit.client.0.smithi136.stdout:op 2483 completed, throughput=5MB/sec 2024-09-13T00:09:27.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:09:26 smithi136 bash[76371]: cluster 2024-09-13T00:09:25.388365+0000 mgr.y (mgr.24661) 969 : cluster [DBG] pgmap v1652: 105 pgs: 105 active+clean; 126 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:09:27.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:09:26 smithi136 bash[75164]: cluster 2024-09-13T00:09:25.388365+0000 mgr.y (mgr.24661) 969 : cluster [DBG] pgmap v1652: 105 pgs: 105 active+clean; 126 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:09:27.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:09:26 smithi181 bash[67587]: cluster 2024-09-13T00:09:25.388365+0000 mgr.y (mgr.24661) 969 : cluster [DBG] pgmap v1652: 105 pgs: 105 active+clean; 126 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:09:27.535 INFO:tasks.workunit.client.0.smithi136.stdout: 469: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:27.535 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-mK-hxYwSNu8PXj- off=14173099 len=1980588 2024-09-13T00:09:27.555 INFO:tasks.workunit.client.0.smithi136.stdout:op 2484 completed, throughput=5MB/sec 2024-09-13T00:09:27.555 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-xj_n2N5wwwIGb3G off=7706820 len=76939 2024-09-13T00:09:27.556 INFO:tasks.workunit.client.0.smithi136.stdout:op 2485 completed, throughput=5MB/sec 2024-09-13T00:09:27.556 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-0-W66Al6oAEYyjp off=2504188 len=1984912 2024-09-13T00:09:27.560 INFO:tasks.workunit.client.0.smithi136.stdout:op 2486 completed, throughput=5MB/sec 2024-09-13T00:09:27.560 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-7XMShwu0tcco6FZ off=1946156 len=1911795 2024-09-13T00:09:27.573 INFO:tasks.workunit.client.0.smithi136.stdout:op 2487 completed, throughput=5MB/sec 2024-09-13T00:09:28.573 INFO:tasks.workunit.client.0.smithi136.stdout: 470: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:28.573 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-8wNCIz2RDlQ5wqt off=849696 len=286898 2024-09-13T00:09:28.579 INFO:tasks.workunit.client.0.smithi136.stdout:op 2488 completed, throughput=4.99MB/sec 2024-09-13T00:09:28.579 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2VjgRYXSylgjfwU off=3885116 len=244673 2024-09-13T00:09:28.580 INFO:tasks.workunit.client.0.smithi136.stdout:op 2489 completed, throughput=4.99MB/sec 2024-09-13T00:09:28.580 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-8Y81WfFy-gw72pn off=7761410 len=1460503 2024-09-13T00:09:28.585 INFO:tasks.workunit.client.0.smithi136.stdout:op 2490 completed, throughput=5MB/sec 2024-09-13T00:09:28.585 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-yPBXUUC4S-uyedQ off=4224407 len=1897814 2024-09-13T00:09:28.590 INFO:tasks.workunit.client.0.smithi136.stdout:op 2491 completed, throughput=5MB/sec 2024-09-13T00:09:29.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:09:28 smithi136 bash[75164]: cluster 2024-09-13T00:09:27.389504+0000 mgr.y (mgr.24661) 970 : cluster [DBG] pgmap v1653: 105 pgs: 105 active+clean; 126 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.9 MiB/s wr, 6 op/s 2024-09-13T00:09:29.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:09:28 smithi136 bash[76371]: cluster 2024-09-13T00:09:27.389504+0000 mgr.y (mgr.24661) 970 : cluster [DBG] pgmap v1653: 105 pgs: 105 active+clean; 126 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.9 MiB/s wr, 6 op/s 2024-09-13T00:09:29.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:09:28 smithi181 bash[67587]: cluster 2024-09-13T00:09:27.389504+0000 mgr.y (mgr.24661) 970 : cluster [DBG] pgmap v1653: 105 pgs: 105 active+clean; 126 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.9 MiB/s wr, 6 op/s 2024-09-13T00:09:29.590 INFO:tasks.workunit.client.0.smithi136.stdout: 471: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:29.590 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Npf5sD3wRwlMuBm off=14194248 len=1494710 2024-09-13T00:09:29.603 INFO:tasks.workunit.client.0.smithi136.stdout:op 2492 completed, throughput=4.99MB/sec 2024-09-13T00:09:29.603 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-QAVBqDiegF9TkM4 off=13202623 len=896421 2024-09-13T00:09:29.611 INFO:tasks.workunit.client.0.smithi136.stdout:op 2493 completed, throughput=4.99MB/sec 2024-09-13T00:09:29.611 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-JVQna1WD40vR4gx off=5388593 len=1621427 2024-09-13T00:09:29.623 INFO:tasks.workunit.client.0.smithi136.stdout:op 2494 completed, throughput=5MB/sec 2024-09-13T00:09:29.623 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-A5B5Psi_O0Buj5h off=6605176 len=958516 2024-09-13T00:09:29.632 INFO:tasks.workunit.client.0.smithi136.stdout:op 2495 completed, throughput=5MB/sec 2024-09-13T00:09:29.632 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-yE-aiPYUA5TrCCO off=2126404 len=72611 2024-09-13T00:09:29.636 INFO:tasks.workunit.client.0.smithi136.stdout:op 2496 completed, throughput=5MB/sec 2024-09-13T00:09:29.636 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-S_Fm-24P04ZcEP7 off=14496883 len=1813028 2024-09-13T00:09:29.642 INFO:tasks.workunit.client.0.smithi136.stdout:op 2497 completed, throughput=5MB/sec 2024-09-13T00:09:29.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:09:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:09:29] "GET /metrics HTTP/1.1" 200 34137 "" "Prometheus/2.43.0" 2024-09-13T00:09:30.643 INFO:tasks.workunit.client.0.smithi136.stdout: 472: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:30.643 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-NZaYuK41P4kvozT off=1460051 len=691139 2024-09-13T00:09:30.645 INFO:tasks.workunit.client.0.smithi136.stdout:op 2498 completed, throughput=4.99MB/sec 2024-09-13T00:09:30.646 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-t9oRPA3m75XiKcO off=2793574 len=353126 2024-09-13T00:09:30.647 INFO:tasks.workunit.client.0.smithi136.stdout:op 2499 completed, throughput=4.99MB/sec 2024-09-13T00:09:30.647 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-keTVvxjyrJcjveU off=1706814 len=45333 2024-09-13T00:09:30.648 INFO:tasks.workunit.client.0.smithi136.stdout:op 2500 completed, throughput=4.99MB/sec 2024-09-13T00:09:30.648 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-pG0ayiV7C_0KP80 off=8090105 len=835159 2024-09-13T00:09:30.657 INFO:tasks.workunit.client.0.smithi136.stdout:op 2501 completed, throughput=5MB/sec 2024-09-13T00:09:30.657 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-hkrD8I5Q75BkkM- off=3482058 len=702941 2024-09-13T00:09:30.664 INFO:tasks.workunit.client.0.smithi136.stdout:op 2502 completed, throughput=5MB/sec 2024-09-13T00:09:30.664 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-FRfhXUqDMouaF8a off=12075373 len=1510710 2024-09-13T00:09:30.669 INFO:tasks.workunit.client.0.smithi136.stdout:op 2503 completed, throughput=5MB/sec 2024-09-13T00:09:31.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:09:30 smithi136 bash[75164]: cluster 2024-09-13T00:09:29.390431+0000 mgr.y (mgr.24661) 971 : cluster [DBG] pgmap v1654: 105 pgs: 105 active+clean; 126 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:09:31.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:09:30 smithi136 bash[76371]: cluster 2024-09-13T00:09:29.390431+0000 mgr.y (mgr.24661) 971 : cluster [DBG] pgmap v1654: 105 pgs: 105 active+clean; 126 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:09:31.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:09:30 smithi181 bash[67587]: cluster 2024-09-13T00:09:29.390431+0000 mgr.y (mgr.24661) 971 : cluster [DBG] pgmap v1654: 105 pgs: 105 active+clean; 126 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:09:31.669 INFO:tasks.workunit.client.0.smithi136.stdout: 473: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:31.669 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-5PHsUYNiObnB1Wy off=1783102 len=647309 2024-09-13T00:09:31.676 INFO:tasks.workunit.client.0.smithi136.stdout:op 2504 completed, throughput=4.99MB/sec 2024-09-13T00:09:31.676 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-d4KuJPnifgWSkY_ off=116082 len=1724591 2024-09-13T00:09:31.687 INFO:tasks.workunit.client.0.smithi136.stdout:op 2505 completed, throughput=4.99MB/sec 2024-09-13T00:09:31.687 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-yqeZ29AKBm4vbzF off=5254749 len=887922 2024-09-13T00:09:31.690 INFO:tasks.workunit.client.0.smithi136.stdout:op 2506 completed, throughput=5MB/sec 2024-09-13T00:09:31.690 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-4vVFNlgj5MUE4xA off=13036919 len=1272129 2024-09-13T00:09:31.700 INFO:tasks.workunit.client.0.smithi136.stdout:op 2507 completed, throughput=5MB/sec 2024-09-13T00:09:31.700 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Zy80fopSAcEQ2Av off=1762868 len=401336 2024-09-13T00:09:31.705 INFO:tasks.workunit.client.0.smithi136.stdout:op 2508 completed, throughput=5MB/sec 2024-09-13T00:09:31.705 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-MzSkisYdriMaQzJ off=6320760 len=324178 2024-09-13T00:09:31.706 INFO:tasks.workunit.client.0.smithi136.stdout:op 2509 completed, throughput=5MB/sec 2024-09-13T00:09:32.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:09:32 smithi181 bash[95505]: ts=2024-09-13T00:09:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:09:32.707 INFO:tasks.workunit.client.0.smithi136.stdout: 474: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:32.707 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ZT3XhQKZCaCM77_ off=16981381 len=1398132 2024-09-13T00:09:32.712 INFO:tasks.workunit.client.0.smithi136.stdout:op 2510 completed, throughput=4.99MB/sec 2024-09-13T00:09:32.712 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_usTu9eIwgHVOaQ off=524080 len=1884319 2024-09-13T00:09:32.724 INFO:tasks.workunit.client.0.smithi136.stdout:op 2511 completed, throughput=5MB/sec 2024-09-13T00:09:32.724 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-F_xzSQ5DOH35VGd off=8601397 len=797933 2024-09-13T00:09:32.726 INFO:tasks.workunit.client.0.smithi136.stdout:op 2512 completed, throughput=5MB/sec 2024-09-13T00:09:32.726 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj--aeVyYxjVTIGWPR off=19756337 len=96959 2024-09-13T00:09:32.727 INFO:tasks.workunit.client.0.smithi136.stdout:op 2513 completed, throughput=5MB/sec 2024-09-13T00:09:32.728 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-CemCOekThIbpRy5 off=9081556 len=469921 2024-09-13T00:09:32.734 INFO:tasks.workunit.client.0.smithi136.stdout:op 2514 completed, throughput=5MB/sec 2024-09-13T00:09:32.734 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-7N7tzH1_jHrTOPl off=4999140 len=2027077 2024-09-13T00:09:32.750 INFO:tasks.workunit.client.0.smithi136.stdout:op 2515 completed, throughput=5MB/sec 2024-09-13T00:09:33.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:09:32 smithi136 bash[75164]: cluster 2024-09-13T00:09:31.391428+0000 mgr.y (mgr.24661) 972 : cluster [DBG] pgmap v1655: 105 pgs: 105 active+clean; 126 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:09:33.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:09:32 smithi136 bash[76371]: cluster 2024-09-13T00:09:31.391428+0000 mgr.y (mgr.24661) 972 : cluster [DBG] pgmap v1655: 105 pgs: 105 active+clean; 126 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:09:33.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:09:32 smithi181 bash[67587]: cluster 2024-09-13T00:09:31.391428+0000 mgr.y (mgr.24661) 972 : cluster [DBG] pgmap v1655: 105 pgs: 105 active+clean; 126 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:09:33.750 INFO:tasks.workunit.client.0.smithi136.stdout: 475: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:33.750 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-jAlxxOi0JfMF4MX off=3874037 len=746344 2024-09-13T00:09:33.758 INFO:tasks.workunit.client.0.smithi136.stdout:op 2516 completed, throughput=4.99MB/sec 2024-09-13T00:09:33.759 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-dtqImhXIKaueO1u off=5756822 len=1576924 2024-09-13T00:09:33.769 INFO:tasks.workunit.client.0.smithi136.stdout:op 2517 completed, throughput=5MB/sec 2024-09-13T00:09:33.770 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-aIUgzoy8VxfaYHG off=3040301 len=1228129 2024-09-13T00:09:33.773 INFO:tasks.workunit.client.0.smithi136.stdout:op 2518 completed, throughput=5MB/sec 2024-09-13T00:09:33.773 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-mB9AlOJoNiwCzWR off=10685432 len=1431300 2024-09-13T00:09:33.782 INFO:tasks.workunit.client.0.smithi136.stdout:op 2519 completed, throughput=5MB/sec 2024-09-13T00:09:34.783 INFO:tasks.workunit.client.0.smithi136.stdout: 476: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:34.783 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-dDcsgBHq-ck0KEM off=577773 len=997882 2024-09-13T00:09:34.786 INFO:tasks.workunit.client.0.smithi136.stdout:op 2520 completed, throughput=4.99MB/sec 2024-09-13T00:09:34.787 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Js0tfa6s-yWp-4Y off=8167330 len=1608160 2024-09-13T00:09:34.798 INFO:tasks.workunit.client.0.smithi136.stdout:op 2521 completed, throughput=5MB/sec 2024-09-13T00:09:34.798 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-18Ja4tEokhSkZH1 off=6178481 len=1167129 2024-09-13T00:09:34.801 INFO:tasks.workunit.client.0.smithi136.stdout:op 2522 completed, throughput=5MB/sec 2024-09-13T00:09:34.801 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-keLy2BNH-2Wq3qd off=8487262 len=1434944 2024-09-13T00:09:34.805 INFO:tasks.workunit.client.0.smithi136.stdout:op 2523 completed, throughput=5MB/sec 2024-09-13T00:09:35.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:09:34 smithi136 bash[75164]: cluster 2024-09-13T00:09:33.392368+0000 mgr.y (mgr.24661) 973 : cluster [DBG] pgmap v1656: 105 pgs: 105 active+clean; 126 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:09:35.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:09:34 smithi136 bash[76371]: cluster 2024-09-13T00:09:33.392368+0000 mgr.y (mgr.24661) 973 : cluster [DBG] pgmap v1656: 105 pgs: 105 active+clean; 126 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:09:35.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:09:34 smithi181 bash[67587]: cluster 2024-09-13T00:09:33.392368+0000 mgr.y (mgr.24661) 973 : cluster [DBG] pgmap v1656: 105 pgs: 105 active+clean; 126 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:09:35.805 INFO:tasks.workunit.client.0.smithi136.stdout: 477: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:35.805 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Fhe7B9hdW_h8J8e off=5184669 len=1697805 2024-09-13T00:09:35.809 INFO:tasks.workunit.client.0.smithi136.stdout:op 2524 completed, throughput=4.99MB/sec 2024-09-13T00:09:35.809 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-xyheI9CXS5TeF3T off=3742846 len=95706 2024-09-13T00:09:35.814 INFO:tasks.workunit.client.0.smithi136.stdout:op 2525 completed, throughput=4.99MB/sec 2024-09-13T00:09:35.814 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-T5oIKv6DY3LxEbx off=1376765 len=1073005 2024-09-13T00:09:35.818 INFO:tasks.workunit.client.0.smithi136.stdout:op 2526 completed, throughput=5MB/sec 2024-09-13T00:09:35.818 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ZoHpzEAa4D5O4aU off=8662842 len=65972 2024-09-13T00:09:35.822 INFO:tasks.workunit.client.0.smithi136.stdout:op 2527 completed, throughput=5MB/sec 2024-09-13T00:09:35.822 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-dB7BXuaAWt45GD8 off=9056743 len=1373886 2024-09-13T00:09:35.835 INFO:tasks.workunit.client.0.smithi136.stdout:op 2528 completed, throughput=5MB/sec 2024-09-13T00:09:35.835 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-5NlHUqHA-ho71be off=15881382 len=617258 2024-09-13T00:09:35.841 INFO:tasks.workunit.client.0.smithi136.stdout:op 2529 completed, throughput=5MB/sec 2024-09-13T00:09:36.842 INFO:tasks.workunit.client.0.smithi136.stdout: 478: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:36.842 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Eo20Msjb4CyMRSU off=10588148 len=1053359 2024-09-13T00:09:36.846 INFO:tasks.workunit.client.0.smithi136.stdout:op 2530 completed, throughput=4.99MB/sec 2024-09-13T00:09:36.846 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-u9DH3vxbjJZk-OD off=9692445 len=1092675 2024-09-13T00:09:36.849 INFO:tasks.workunit.client.0.smithi136.stdout:op 2531 completed, throughput=4.99MB/sec 2024-09-13T00:09:36.849 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-oTiblziXvHbOJMf off=3981364 len=327988 2024-09-13T00:09:36.850 INFO:tasks.workunit.client.0.smithi136.stdout:op 2532 completed, throughput=5MB/sec 2024-09-13T00:09:36.850 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_Bez8ScVhKM63eJ off=1348603 len=1073447 2024-09-13T00:09:36.858 INFO:tasks.workunit.client.0.smithi136.stdout:op 2533 completed, throughput=5MB/sec 2024-09-13T00:09:36.858 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-C9X1YL67JKpb-mt off=8069626 len=408413 2024-09-13T00:09:36.860 INFO:tasks.workunit.client.0.smithi136.stdout:op 2534 completed, throughput=5MB/sec 2024-09-13T00:09:36.860 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-fJ8pQZjoViJy7CX off=3158043 len=1946713 2024-09-13T00:09:36.865 INFO:tasks.workunit.client.0.smithi136.stdout:op 2535 completed, throughput=5MB/sec 2024-09-13T00:09:37.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:09:37 smithi136 bash[75164]: cluster 2024-09-13T00:09:35.393229+0000 mgr.y (mgr.24661) 974 : cluster [DBG] pgmap v1657: 105 pgs: 105 active+clean; 126 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:09:37.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:09:37 smithi136 bash[76371]: cluster 2024-09-13T00:09:35.393229+0000 mgr.y (mgr.24661) 974 : cluster [DBG] pgmap v1657: 105 pgs: 105 active+clean; 126 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:09:37.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:09:37 smithi181 bash[67587]: cluster 2024-09-13T00:09:35.393229+0000 mgr.y (mgr.24661) 974 : cluster [DBG] pgmap v1657: 105 pgs: 105 active+clean; 126 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:09:37.865 INFO:tasks.workunit.client.0.smithi136.stdout: 479: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:37.865 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-TfyTFAFaEdN6bQP off=2955147 len=108729 2024-09-13T00:09:37.866 INFO:tasks.workunit.client.0.smithi136.stdout:op 2536 completed, throughput=4.99MB/sec 2024-09-13T00:09:37.866 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-wbLJmaAklZ0dt3J off=1357064 len=108481 2024-09-13T00:09:37.867 INFO:tasks.workunit.client.0.smithi136.stdout:op 2537 completed, throughput=4.99MB/sec 2024-09-13T00:09:37.867 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-KDXkRskzkjYlHP- off=14915344 len=1029528 2024-09-13T00:09:37.871 INFO:tasks.workunit.client.0.smithi136.stdout:op 2538 completed, throughput=4.99MB/sec 2024-09-13T00:09:37.871 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-cXOmC8HY4AKfEnC off=8006731 len=1935661 2024-09-13T00:09:37.884 INFO:tasks.workunit.client.0.smithi136.stdout:op 2539 completed, throughput=5MB/sec 2024-09-13T00:09:37.884 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-QDTpMJQLfsPH_0g off=3030854 len=1413062 2024-09-13T00:09:37.889 INFO:tasks.workunit.client.0.smithi136.stdout:op 2540 completed, throughput=5MB/sec 2024-09-13T00:09:38.890 INFO:tasks.workunit.client.0.smithi136.stdout: 480: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:38.890 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-3sJHBOC7h_CE60s off=5714554 len=97440 2024-09-13T00:09:38.891 INFO:tasks.workunit.client.0.smithi136.stdout:op 2541 completed, throughput=4.99MB/sec 2024-09-13T00:09:38.891 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-WWnK7Tkwe8FRLPG off=1081769 len=520473 2024-09-13T00:09:38.893 INFO:tasks.workunit.client.0.smithi136.stdout:op 2542 completed, throughput=4.99MB/sec 2024-09-13T00:09:38.893 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-lW16hTMCNWvwGEU off=282834 len=558041 2024-09-13T00:09:38.895 INFO:tasks.workunit.client.0.smithi136.stdout:op 2543 completed, throughput=4.99MB/sec 2024-09-13T00:09:38.895 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-5c7SFTiCvmgEFQN off=6978709 len=1917670 2024-09-13T00:09:38.901 INFO:tasks.workunit.client.0.smithi136.stdout:op 2544 completed, throughput=5MB/sec 2024-09-13T00:09:38.902 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-VRW6C220mHBxVzu off=1402508 len=1606059 2024-09-13T00:09:38.906 INFO:tasks.workunit.client.0.smithi136.stdout:op 2545 completed, throughput=5MB/sec 2024-09-13T00:09:38.906 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-eqbcVw7Pkgye6rF off=2517133 len=1604210 2024-09-13T00:09:38.909 INFO:tasks.workunit.client.0.smithi136.stdout:op 2546 completed, throughput=5MB/sec 2024-09-13T00:09:39.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:09:39 smithi136 bash[75164]: cluster 2024-09-13T00:09:37.394336+0000 mgr.y (mgr.24661) 975 : cluster [DBG] pgmap v1658: 105 pgs: 105 active+clean; 126 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:09:39.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:09:39 smithi136 bash[75164]: audit 2024-09-13T00:09:38.785170+0000 mon.a (mon.0) 2177 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:09:39.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:09:39 smithi136 bash[76371]: cluster 2024-09-13T00:09:37.394336+0000 mgr.y (mgr.24661) 975 : cluster [DBG] pgmap v1658: 105 pgs: 105 active+clean; 126 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:09:39.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:09:39 smithi136 bash[76371]: audit 2024-09-13T00:09:38.785170+0000 mon.a (mon.0) 2177 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:09:39.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:09:39 smithi181 bash[67587]: cluster 2024-09-13T00:09:37.394336+0000 mgr.y (mgr.24661) 975 : cluster [DBG] pgmap v1658: 105 pgs: 105 active+clean; 126 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:09:39.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:09:39 smithi181 bash[67587]: audit 2024-09-13T00:09:38.785170+0000 mon.a (mon.0) 2177 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:09:39.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:09:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:09:39] "GET /metrics HTTP/1.1" 200 34134 "" "Prometheus/2.43.0" 2024-09-13T00:09:39.909 INFO:tasks.workunit.client.0.smithi136.stdout: 481: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:39.909 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-GbJmJnrLRk_Xb5s off=11140920 len=1320876 2024-09-13T00:09:39.914 INFO:tasks.workunit.client.0.smithi136.stdout:op 2547 completed, throughput=4.99MB/sec 2024-09-13T00:09:39.914 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-kfHa6mesj5PbJrH off=760752 len=438752 2024-09-13T00:09:39.916 INFO:tasks.workunit.client.0.smithi136.stdout:op 2548 completed, throughput=5MB/sec 2024-09-13T00:09:39.916 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-cj5v-18axKZvf81 off=3964024 len=2050655 2024-09-13T00:09:39.921 INFO:tasks.workunit.client.0.smithi136.stdout:op 2549 completed, throughput=5MB/sec 2024-09-13T00:09:39.921 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-OJIiAhzsVcDXMdY off=3052915 len=1196812 2024-09-13T00:09:39.924 INFO:tasks.workunit.client.0.smithi136.stdout:op 2550 completed, throughput=5MB/sec 2024-09-13T00:09:40.924 INFO:tasks.workunit.client.0.smithi136.stdout: 482: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:40.925 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-YE4P7M0Sdvh7wgN off=4765031 len=1784421 2024-09-13T00:09:40.929 INFO:tasks.workunit.client.0.smithi136.stdout:op 2551 completed, throughput=4.99MB/sec 2024-09-13T00:09:40.929 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-PuduaMmADHgHoAr off=3756403 len=787278 2024-09-13T00:09:40.938 INFO:tasks.workunit.client.0.smithi136.stdout:op 2552 completed, throughput=5MB/sec 2024-09-13T00:09:40.938 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-lWP2aitEosv_6Px off=2266408 len=375894 2024-09-13T00:09:40.944 INFO:tasks.workunit.client.0.smithi136.stdout:op 2553 completed, throughput=5MB/sec 2024-09-13T00:09:40.944 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-iifxO-uzhPMOC5S off=1620627 len=1632666 2024-09-13T00:09:40.949 INFO:tasks.workunit.client.0.smithi136.stdout:op 2554 completed, throughput=5MB/sec 2024-09-13T00:09:40.949 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-DNRAZD__9Y1310_ off=12763108 len=730299 2024-09-13T00:09:40.957 INFO:tasks.workunit.client.0.smithi136.stdout:op 2555 completed, throughput=5MB/sec 2024-09-13T00:09:41.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:09:41 smithi181 bash[67587]: cluster 2024-09-13T00:09:39.395450+0000 mgr.y (mgr.24661) 976 : cluster [DBG] pgmap v1659: 105 pgs: 105 active+clean; 126 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 1.8 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:09:41.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:09:41 smithi136 bash[75164]: cluster 2024-09-13T00:09:39.395450+0000 mgr.y (mgr.24661) 976 : cluster [DBG] pgmap v1659: 105 pgs: 105 active+clean; 126 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 1.8 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:09:41.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:09:41 smithi136 bash[76371]: cluster 2024-09-13T00:09:39.395450+0000 mgr.y (mgr.24661) 976 : cluster [DBG] pgmap v1659: 105 pgs: 105 active+clean; 126 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 1.8 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:09:41.957 INFO:tasks.workunit.client.0.smithi136.stdout: 483: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:41.957 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-IYJFFX7oGnKlRgu off=3172007 len=1622438 2024-09-13T00:09:41.962 INFO:tasks.workunit.client.0.smithi136.stdout:op 2556 completed, throughput=4.99MB/sec 2024-09-13T00:09:41.962 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-3sJHBOC7h_CE60s off=4833395 len=1809013 2024-09-13T00:09:41.979 INFO:tasks.workunit.client.0.smithi136.stdout:op 2557 completed, throughput=5MB/sec 2024-09-13T00:09:41.979 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ZGr3BfUGCbt4wCf off=2561960 len=1569646 2024-09-13T00:09:41.989 INFO:tasks.workunit.client.0.smithi136.stdout:op 2558 completed, throughput=5MB/sec 2024-09-13T00:09:42.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:09:42 smithi181 bash[95505]: ts=2024-09-13T00:09:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:09:42.990 INFO:tasks.workunit.client.0.smithi136.stdout: 484: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:42.990 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-5G2sFuqZonMvCfU off=14586010 len=2043867 2024-09-13T00:09:42.995 INFO:tasks.workunit.client.0.smithi136.stdout:op 2559 completed, throughput=4.99MB/sec 2024-09-13T00:09:42.995 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-NZaYuK41P4kvozT off=448986 len=1702204 2024-09-13T00:09:42.999 INFO:tasks.workunit.client.0.smithi136.stdout:op 2560 completed, throughput=5MB/sec 2024-09-13T00:09:43.000 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-mbivlDeUr7Jxcai off=14035657 len=1876425 2024-09-13T00:09:43.004 INFO:tasks.workunit.client.0.smithi136.stdout:op 2561 completed, throughput=5MB/sec 2024-09-13T00:09:43.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:09:43 smithi181 bash[67587]: cluster 2024-09-13T00:09:41.396469+0000 mgr.y (mgr.24661) 977 : cluster [DBG] pgmap v1660: 105 pgs: 105 active+clean; 126 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:09:43.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:09:43 smithi136 bash[75164]: cluster 2024-09-13T00:09:41.396469+0000 mgr.y (mgr.24661) 977 : cluster [DBG] pgmap v1660: 105 pgs: 105 active+clean; 126 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:09:43.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:09:43 smithi136 bash[76371]: cluster 2024-09-13T00:09:41.396469+0000 mgr.y (mgr.24661) 977 : cluster [DBG] pgmap v1660: 105 pgs: 105 active+clean; 126 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:09:44.004 INFO:tasks.workunit.client.0.smithi136.stdout: 485: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:44.004 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-dZ1z9iX1NaDmHff off=5738634 len=1299918 2024-09-13T00:09:44.007 INFO:tasks.workunit.client.0.smithi136.stdout:op 2562 completed, throughput=4.99MB/sec 2024-09-13T00:09:44.008 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-XwGv9XPjx55mdUx off=1344181 len=1592476 2024-09-13T00:09:44.011 INFO:tasks.workunit.client.0.smithi136.stdout:op 2563 completed, throughput=5MB/sec 2024-09-13T00:09:44.012 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-0XU9s6N0kLRvrjA off=20830081 len=79996 2024-09-13T00:09:44.013 INFO:tasks.workunit.client.0.smithi136.stdout:op 2564 completed, throughput=5MB/sec 2024-09-13T00:09:44.013 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-xELHivF3Yj7Mnar off=14984113 len=1758380 2024-09-13T00:09:44.018 INFO:tasks.workunit.client.0.smithi136.stdout:op 2565 completed, throughput=5MB/sec 2024-09-13T00:09:44.018 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-wQW_ar52BwLiRCO off=628009 len=1579538 2024-09-13T00:09:44.022 INFO:tasks.workunit.client.0.smithi136.stdout:op 2566 completed, throughput=5MB/sec 2024-09-13T00:09:45.022 INFO:tasks.workunit.client.0.smithi136.stdout: 486: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:45.022 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-KEFX51rMAu4CO-e off=8242609 len=656215 2024-09-13T00:09:45.025 INFO:tasks.workunit.client.0.smithi136.stdout:op 2567 completed, throughput=4.99MB/sec 2024-09-13T00:09:45.025 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-lmZCaGw0lvDBTMF off=21924339 len=805091 2024-09-13T00:09:45.028 INFO:tasks.workunit.client.0.smithi136.stdout:op 2568 completed, throughput=5MB/sec 2024-09-13T00:09:45.028 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-hIpTlQpwJjbK7Vf off=12490907 len=1357453 2024-09-13T00:09:45.039 INFO:tasks.workunit.client.0.smithi136.stdout:op 2569 completed, throughput=5MB/sec 2024-09-13T00:09:45.039 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-5c7SFTiCvmgEFQN off=15562559 len=281509 2024-09-13T00:09:45.045 INFO:tasks.workunit.client.0.smithi136.stdout:op 2570 completed, throughput=5MB/sec 2024-09-13T00:09:45.045 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-wgUyWV1CzWbMPMg off=814938 len=1819178 2024-09-13T00:09:45.057 INFO:tasks.workunit.client.0.smithi136.stdout:op 2571 completed, throughput=5MB/sec 2024-09-13T00:09:45.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:09:45 smithi181 bash[67587]: cluster 2024-09-13T00:09:43.397416+0000 mgr.y (mgr.24661) 978 : cluster [DBG] pgmap v1661: 105 pgs: 105 active+clean; 126 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:09:45.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:09:45 smithi136 bash[75164]: cluster 2024-09-13T00:09:43.397416+0000 mgr.y (mgr.24661) 978 : cluster [DBG] pgmap v1661: 105 pgs: 105 active+clean; 126 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:09:45.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:09:45 smithi136 bash[76371]: cluster 2024-09-13T00:09:43.397416+0000 mgr.y (mgr.24661) 978 : cluster [DBG] pgmap v1661: 105 pgs: 105 active+clean; 126 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:09:46.058 INFO:tasks.workunit.client.0.smithi136.stdout: 487: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:46.058 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-UW7LK2o6J60m_ge off=6158768 len=1893226 2024-09-13T00:09:46.075 INFO:tasks.workunit.client.0.smithi136.stdout:op 2572 completed, throughput=5MB/sec 2024-09-13T00:09:46.075 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-nxbhowT96yloAvI off=7474996 len=1863714 2024-09-13T00:09:46.085 INFO:tasks.workunit.client.0.smithi136.stdout:op 2573 completed, throughput=5MB/sec 2024-09-13T00:09:46.085 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-JLcFXXv_G4s2bh1 off=15083542 len=1158080 2024-09-13T00:09:46.088 INFO:tasks.workunit.client.0.smithi136.stdout:op 2574 completed, throughput=5MB/sec 2024-09-13T00:09:47.089 INFO:tasks.workunit.client.0.smithi136.stdout: 488: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:47.089 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-CazU11TOFdlAlQW off=7979864 len=716822 2024-09-13T00:09:47.091 INFO:tasks.workunit.client.0.smithi136.stdout:op 2575 completed, throughput=4.99MB/sec 2024-09-13T00:09:47.091 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-iiAqBWs-_1TsdLR off=8850408 len=298297 2024-09-13T00:09:47.094 INFO:tasks.workunit.client.0.smithi136.stdout:op 2576 completed, throughput=4.99MB/sec 2024-09-13T00:09:47.094 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-fMsoJieeP1qz49d off=319799 len=1070884 2024-09-13T00:09:47.101 INFO:tasks.workunit.client.0.smithi136.stdout:op 2577 completed, throughput=4.99MB/sec 2024-09-13T00:09:47.101 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-utJn-9szqqWeXzB off=468196 len=627040 2024-09-13T00:09:47.102 INFO:tasks.workunit.client.0.smithi136.stdout:op 2578 completed, throughput=5MB/sec 2024-09-13T00:09:47.102 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-E54Vrw-YS8ZPCDS off=3057988 len=474860 2024-09-13T00:09:47.103 INFO:tasks.workunit.client.0.smithi136.stdout:op 2579 completed, throughput=5MB/sec 2024-09-13T00:09:47.103 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-F4P3SeAKE1pmonm off=5881164 len=543635 2024-09-13T00:09:47.104 INFO:tasks.workunit.client.0.smithi136.stdout:op 2580 completed, throughput=5MB/sec 2024-09-13T00:09:47.104 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-kp-6jz1hGdnBXHI off=2927853 len=69824 2024-09-13T00:09:47.105 INFO:tasks.workunit.client.0.smithi136.stdout:op 2581 completed, throughput=5MB/sec 2024-09-13T00:09:47.105 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-jttbsG5ha0ZMW-L off=9626258 len=732029 2024-09-13T00:09:47.106 INFO:tasks.workunit.client.0.smithi136.stdout:op 2582 completed, throughput=5MB/sec 2024-09-13T00:09:47.106 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ylznHO-RHB_qefS off=6750064 len=809160 2024-09-13T00:09:47.111 INFO:tasks.workunit.client.0.smithi136.stdout:op 2583 completed, throughput=5MB/sec 2024-09-13T00:09:47.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:09:47 smithi181 bash[67587]: cluster 2024-09-13T00:09:45.398248+0000 mgr.y (mgr.24661) 979 : cluster [DBG] pgmap v1662: 105 pgs: 105 active+clean; 126 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.8 MiB/s wr, 4 op/s 2024-09-13T00:09:47.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:09:47 smithi181 bash[67587]: audit 2024-09-13T00:09:46.705525+0000 mon.a (mon.0) 2178 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:09:47.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:09:47 smithi136 bash[75164]: cluster 2024-09-13T00:09:45.398248+0000 mgr.y (mgr.24661) 979 : cluster [DBG] pgmap v1662: 105 pgs: 105 active+clean; 126 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.8 MiB/s wr, 4 op/s 2024-09-13T00:09:47.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:09:47 smithi136 bash[75164]: audit 2024-09-13T00:09:46.705525+0000 mon.a (mon.0) 2178 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:09:47.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:09:47 smithi136 bash[76371]: cluster 2024-09-13T00:09:45.398248+0000 mgr.y (mgr.24661) 979 : cluster [DBG] pgmap v1662: 105 pgs: 105 active+clean; 126 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.8 MiB/s wr, 4 op/s 2024-09-13T00:09:47.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:09:47 smithi136 bash[76371]: audit 2024-09-13T00:09:46.705525+0000 mon.a (mon.0) 2178 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:09:48.112 INFO:tasks.workunit.client.0.smithi136.stdout: 489: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:48.113 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-s7jWdzffJ5UyQ1U off=4116269 len=729351 2024-09-13T00:09:48.114 INFO:tasks.workunit.client.0.smithi136.stdout:op 2584 completed, throughput=4.99MB/sec 2024-09-13T00:09:48.115 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-HrnjWwTMqUcgc5t off=17256253 len=431097 2024-09-13T00:09:48.120 INFO:tasks.workunit.client.0.smithi136.stdout:op 2585 completed, throughput=4.99MB/sec 2024-09-13T00:09:48.121 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-wiv8bWk29XGByCO off=9406502 len=1785207 2024-09-13T00:09:48.124 INFO:tasks.workunit.client.0.smithi136.stdout:op 2586 completed, throughput=5MB/sec 2024-09-13T00:09:48.124 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ZYeHbCIs0O-uOXv off=6008607 len=903344 2024-09-13T00:09:48.126 INFO:tasks.workunit.client.0.smithi136.stdout:op 2587 completed, throughput=5MB/sec 2024-09-13T00:09:48.127 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-NA6EZMQDf_yisAk off=4319806 len=1235731 2024-09-13T00:09:48.136 INFO:tasks.workunit.client.0.smithi136.stdout:op 2588 completed, throughput=5MB/sec 2024-09-13T00:09:48.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:09:48 smithi181 bash[67587]: audit 2024-09-13T00:09:47.249694+0000 mon.a (mon.0) 2179 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:09:48.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:09:48 smithi181 bash[67587]: audit 2024-09-13T00:09:47.250924+0000 mon.a (mon.0) 2180 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:09:48.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:09:48 smithi181 bash[67587]: audit 2024-09-13T00:09:47.257674+0000 mon.a (mon.0) 2181 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:09:48.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:09:48 smithi136 bash[75164]: audit 2024-09-13T00:09:47.249694+0000 mon.a (mon.0) 2179 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:09:48.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:09:48 smithi136 bash[75164]: audit 2024-09-13T00:09:47.250924+0000 mon.a (mon.0) 2180 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:09:48.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:09:48 smithi136 bash[75164]: audit 2024-09-13T00:09:47.257674+0000 mon.a (mon.0) 2181 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:09:48.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:09:48 smithi136 bash[76371]: audit 2024-09-13T00:09:47.249694+0000 mon.a (mon.0) 2179 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:09:48.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:09:48 smithi136 bash[76371]: audit 2024-09-13T00:09:47.250924+0000 mon.a (mon.0) 2180 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:09:48.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:09:48 smithi136 bash[76371]: audit 2024-09-13T00:09:47.257674+0000 mon.a (mon.0) 2181 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:09:49.136 INFO:tasks.workunit.client.0.smithi136.stdout: 490: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:49.137 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-P0K_0gYh2eaSbYm off=0 len=1336063 2024-09-13T00:09:49.147 INFO:tasks.workunit.client.0.smithi136.stdout:op 2589 completed, throughput=4.99MB/sec 2024-09-13T00:09:49.147 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-s6r9hlh6fFjmt_a off=1222452 len=1347552 2024-09-13T00:09:49.151 INFO:tasks.workunit.client.0.smithi136.stdout:op 2590 completed, throughput=5MB/sec 2024-09-13T00:09:49.151 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2repnMgwm4I5ekW off=4097703 len=1754092 2024-09-13T00:09:49.157 INFO:tasks.workunit.client.0.smithi136.stdout:op 2591 completed, throughput=5MB/sec 2024-09-13T00:09:49.157 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-oIX7QIL7p44BBKm off=3868955 len=541489 2024-09-13T00:09:49.159 INFO:tasks.workunit.client.0.smithi136.stdout:op 2592 completed, throughput=5MB/sec 2024-09-13T00:09:49.159 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-pPiQlgtQ5qnGPb9 off=17281439 len=1152314 2024-09-13T00:09:49.161 INFO:tasks.workunit.client.0.smithi136.stdout:op 2593 completed, throughput=5MB/sec 2024-09-13T00:09:49.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:09:49 smithi181 bash[67587]: cluster 2024-09-13T00:09:47.399029+0000 mgr.y (mgr.24661) 980 : cluster [DBG] pgmap v1663: 105 pgs: 105 active+clean; 126 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.9 MiB/s wr, 4 op/s 2024-09-13T00:09:49.412 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:09:49 smithi136 bash[75164]: cluster 2024-09-13T00:09:47.399029+0000 mgr.y (mgr.24661) 980 : cluster [DBG] pgmap v1663: 105 pgs: 105 active+clean; 126 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.9 MiB/s wr, 4 op/s 2024-09-13T00:09:49.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:09:49 smithi136 bash[76371]: cluster 2024-09-13T00:09:47.399029+0000 mgr.y (mgr.24661) 980 : cluster [DBG] pgmap v1663: 105 pgs: 105 active+clean; 126 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.9 MiB/s wr, 4 op/s 2024-09-13T00:09:49.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:09:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:09:49] "GET /metrics HTTP/1.1" 200 34134 "" "Prometheus/2.43.0" 2024-09-13T00:09:50.161 INFO:tasks.workunit.client.0.smithi136.stdout: 491: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:50.161 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-KUP-E2dIZdRiurd off=14366533 len=614036 2024-09-13T00:09:50.168 INFO:tasks.workunit.client.0.smithi136.stdout:op 2594 completed, throughput=4.99MB/sec 2024-09-13T00:09:50.168 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-oMSZGP8Jsm9NyR2 off=3030882 len=1771918 2024-09-13T00:09:50.183 INFO:tasks.workunit.client.0.smithi136.stdout:op 2595 completed, throughput=5MB/sec 2024-09-13T00:09:50.183 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-vWG5I9EG9nfVzKd off=4390592 len=1950062 2024-09-13T00:09:50.197 INFO:tasks.workunit.client.0.smithi136.stdout:op 2596 completed, throughput=5MB/sec 2024-09-13T00:09:50.197 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-stgo8AYZHdD6-nV off=2957102 len=460728 2024-09-13T00:09:50.198 INFO:tasks.workunit.client.0.smithi136.stdout:op 2597 completed, throughput=5MB/sec 2024-09-13T00:09:51.199 INFO:tasks.workunit.client.0.smithi136.stdout: 492: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:51.199 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-fueO_9KlQ4hRevb off=18790306 len=715859 2024-09-13T00:09:51.207 INFO:tasks.workunit.client.0.smithi136.stdout:op 2598 completed, throughput=4.99MB/sec 2024-09-13T00:09:51.207 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-8AuxloQxTuwF4Zj off=962368 len=534255 2024-09-13T00:09:51.208 INFO:tasks.workunit.client.0.smithi136.stdout:op 2599 completed, throughput=4.99MB/sec 2024-09-13T00:09:51.208 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-b_fJNczPOoxIbZ- off=3999961 len=641910 2024-09-13T00:09:51.217 INFO:tasks.workunit.client.0.smithi136.stdout:op 2600 completed, throughput=4.99MB/sec 2024-09-13T00:09:51.217 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-s2vylAlPBweaWo- off=497239 len=588236 2024-09-13T00:09:51.225 INFO:tasks.workunit.client.0.smithi136.stdout:op 2601 completed, throughput=4.99MB/sec 2024-09-13T00:09:51.226 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-BFKTyyd-JnHgNRW off=1637551 len=1213387 2024-09-13T00:09:51.228 INFO:tasks.workunit.client.0.smithi136.stdout:op 2602 completed, throughput=5MB/sec 2024-09-13T00:09:51.229 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-4DamFa4n-dAVRag off=12375042 len=1537258 2024-09-13T00:09:51.232 INFO:tasks.workunit.client.0.smithi136.stdout:op 2603 completed, throughput=5MB/sec 2024-09-13T00:09:51.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:09:51 smithi181 bash[67587]: cluster 2024-09-13T00:09:49.399836+0000 mgr.y (mgr.24661) 981 : cluster [DBG] pgmap v1664: 105 pgs: 105 active+clean; 126 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.4 MiB/s wr, 4 op/s 2024-09-13T00:09:51.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:09:51 smithi136 bash[76371]: cluster 2024-09-13T00:09:49.399836+0000 mgr.y (mgr.24661) 981 : cluster [DBG] pgmap v1664: 105 pgs: 105 active+clean; 126 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.4 MiB/s wr, 4 op/s 2024-09-13T00:09:51.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:09:51 smithi136 bash[75164]: cluster 2024-09-13T00:09:49.399836+0000 mgr.y (mgr.24661) 981 : cluster [DBG] pgmap v1664: 105 pgs: 105 active+clean; 126 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.4 MiB/s wr, 4 op/s 2024-09-13T00:09:52.233 INFO:tasks.workunit.client.0.smithi136.stdout: 493: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:52.233 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-gTw9gdRmhB-JEh7 off=0 len=1080875 2024-09-13T00:09:52.236 INFO:tasks.workunit.client.0.smithi136.stdout:op 2604 completed, throughput=4.99MB/sec 2024-09-13T00:09:52.236 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-2_ILQJ9_osNZb01 off=7223980 len=1162642 2024-09-13T00:09:52.244 INFO:tasks.workunit.client.0.smithi136.stdout:op 2605 completed, throughput=4.99MB/sec 2024-09-13T00:09:52.244 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-m_RwkvBeK_9OLV0 off=19545914 len=401934 2024-09-13T00:09:52.246 INFO:tasks.workunit.client.0.smithi136.stdout:op 2606 completed, throughput=5MB/sec 2024-09-13T00:09:52.246 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-LdTfpplAdfAKdG9 off=3148543 len=954239 2024-09-13T00:09:52.255 INFO:tasks.workunit.client.0.smithi136.stdout:op 2607 completed, throughput=5MB/sec 2024-09-13T00:09:52.255 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-GKCIkWcI0ffMFHd off=6623535 len=1070573 2024-09-13T00:09:52.261 INFO:tasks.workunit.client.0.smithi136.stdout:op 2608 completed, throughput=5MB/sec 2024-09-13T00:09:52.261 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-tMhFTHfcH5-kZM2 off=0 len=1459597 2024-09-13T00:09:52.268 INFO:tasks.workunit.client.0.smithi136.stdout:op 2609 completed, throughput=5MB/sec 2024-09-13T00:09:52.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:09:52 smithi181 bash[95505]: ts=2024-09-13T00:09:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:09:53.269 INFO:tasks.workunit.client.0.smithi136.stdout: 494: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:53.269 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-lID-SY11_8PeC16 off=4009516 len=2022139 2024-09-13T00:09:53.281 INFO:tasks.workunit.client.0.smithi136.stdout:op 2610 completed, throughput=5MB/sec 2024-09-13T00:09:53.281 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-R0aPzGcVj_E3wZo off=6511735 len=1596632 2024-09-13T00:09:53.293 INFO:tasks.workunit.client.0.smithi136.stdout:op 2611 completed, throughput=5MB/sec 2024-09-13T00:09:53.293 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Pn-G63rziYqymrz off=4721310 len=295276 2024-09-13T00:09:53.298 INFO:tasks.workunit.client.0.smithi136.stdout:op 2612 completed, throughput=5MB/sec 2024-09-13T00:09:53.298 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-zcjHhlEIJu2pDen off=8171053 len=1284950 2024-09-13T00:09:53.303 INFO:tasks.workunit.client.0.smithi136.stdout:op 2613 completed, throughput=5MB/sec 2024-09-13T00:09:53.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:09:53 smithi181 bash[67587]: cluster 2024-09-13T00:09:51.400885+0000 mgr.y (mgr.24661) 982 : cluster [DBG] pgmap v1665: 105 pgs: 105 active+clean; 126 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.7 MiB/s wr, 4 op/s 2024-09-13T00:09:53.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:09:53 smithi136 bash[76371]: cluster 2024-09-13T00:09:51.400885+0000 mgr.y (mgr.24661) 982 : cluster [DBG] pgmap v1665: 105 pgs: 105 active+clean; 126 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.7 MiB/s wr, 4 op/s 2024-09-13T00:09:53.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:09:53 smithi136 bash[75164]: cluster 2024-09-13T00:09:51.400885+0000 mgr.y (mgr.24661) 982 : cluster [DBG] pgmap v1665: 105 pgs: 105 active+clean; 126 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.7 MiB/s wr, 4 op/s 2024-09-13T00:09:54.304 INFO:tasks.workunit.client.0.smithi136.stdout: 495: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:54.304 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-KeKQeve0M8InYLA off=2214421 len=230533 2024-09-13T00:09:54.309 INFO:tasks.workunit.client.0.smithi136.stdout:op 2614 completed, throughput=4.99MB/sec 2024-09-13T00:09:54.309 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Oj9fSXGJ2KBelqS off=12569173 len=1527042 2024-09-13T00:09:54.320 INFO:tasks.workunit.client.0.smithi136.stdout:op 2615 completed, throughput=4.99MB/sec 2024-09-13T00:09:54.320 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_cwrFhH7xU75RkE off=20468540 len=412538 2024-09-13T00:09:54.326 INFO:tasks.workunit.client.0.smithi136.stdout:op 2616 completed, throughput=5MB/sec 2024-09-13T00:09:54.326 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-NjS_3jaZO8XDUFb off=93586 len=863001 2024-09-13T00:09:54.329 INFO:tasks.workunit.client.0.smithi136.stdout:op 2617 completed, throughput=5MB/sec 2024-09-13T00:09:54.329 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-6NaOELq6igKBZMu off=9300333 len=1313385 2024-09-13T00:09:54.339 INFO:tasks.workunit.client.0.smithi136.stdout:op 2618 completed, throughput=5MB/sec 2024-09-13T00:09:54.339 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-l3Ou5RWQzi9ua95 off=3766659 len=1426659 2024-09-13T00:09:54.344 INFO:tasks.workunit.client.0.smithi136.stdout:op 2619 completed, throughput=5MB/sec 2024-09-13T00:09:54.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:09:54 smithi181 bash[67587]: audit 2024-09-13T00:09:53.785861+0000 mon.a (mon.0) 2182 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:09:54.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:09:54 smithi136 bash[76371]: audit 2024-09-13T00:09:53.785861+0000 mon.a (mon.0) 2182 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:09:54.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:09:54 smithi136 bash[75164]: audit 2024-09-13T00:09:53.785861+0000 mon.a (mon.0) 2182 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:09:55.344 INFO:tasks.workunit.client.0.smithi136.stdout: 496: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:55.345 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-VH2W1AU6_UHFT3z off=6948770 len=1799174 2024-09-13T00:09:55.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:09:55 smithi181 bash[67587]: cluster 2024-09-13T00:09:53.401852+0000 mgr.y (mgr.24661) 983 : cluster [DBG] pgmap v1666: 105 pgs: 105 active+clean; 126 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.9 MiB/s wr, 4 op/s 2024-09-13T00:09:55.348 INFO:tasks.workunit.client.0.smithi136.stdout:op 2620 completed, throughput=5MB/sec 2024-09-13T00:09:55.348 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-IsTHHp1XZPOY6Ep off=1912932 len=1542303 2024-09-13T00:09:55.361 INFO:tasks.workunit.client.0.smithi136.stdout:op 2621 completed, throughput=5MB/sec 2024-09-13T00:09:55.361 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-bHn9RXUUE5wogq0 off=2609497 len=297556 2024-09-13T00:09:55.366 INFO:tasks.workunit.client.0.smithi136.stdout:op 2622 completed, throughput=5MB/sec 2024-09-13T00:09:55.366 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-8Ia2VbwB5I0ihJp off=9023687 len=1352953 2024-09-13T00:09:55.378 INFO:tasks.workunit.client.0.smithi136.stdout:op 2623 completed, throughput=5MB/sec 2024-09-13T00:09:55.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:09:55 smithi136 bash[76371]: cluster 2024-09-13T00:09:53.401852+0000 mgr.y (mgr.24661) 983 : cluster [DBG] pgmap v1666: 105 pgs: 105 active+clean; 126 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.9 MiB/s wr, 4 op/s 2024-09-13T00:09:55.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:09:55 smithi136 bash[75164]: cluster 2024-09-13T00:09:53.401852+0000 mgr.y (mgr.24661) 983 : cluster [DBG] pgmap v1666: 105 pgs: 105 active+clean; 126 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.9 MiB/s wr, 4 op/s 2024-09-13T00:09:56.379 INFO:tasks.workunit.client.0.smithi136.stdout: 497: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:56.379 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-dRJWdYnqW0vroza off=6845259 len=1282431 2024-09-13T00:09:56.392 INFO:tasks.workunit.client.0.smithi136.stdout:op 2624 completed, throughput=4.99MB/sec 2024-09-13T00:09:56.392 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-qGa413l_IxGREEW off=18967562 len=135655 2024-09-13T00:09:56.397 INFO:tasks.workunit.client.0.smithi136.stdout:op 2625 completed, throughput=4.99MB/sec 2024-09-13T00:09:56.397 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-dc-KxBvkJMi3Amd off=12038687 len=647984 2024-09-13T00:09:56.399 INFO:tasks.workunit.client.0.smithi136.stdout:op 2626 completed, throughput=4.99MB/sec 2024-09-13T00:09:56.399 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-47cU06WIj1mHsgX off=2046595 len=1852561 2024-09-13T00:09:56.404 INFO:tasks.workunit.client.0.smithi136.stdout:op 2627 completed, throughput=5MB/sec 2024-09-13T00:09:56.404 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-5qNE1-IM2gsuA1v off=1452921 len=758561 2024-09-13T00:09:56.406 INFO:tasks.workunit.client.0.smithi136.stdout:op 2628 completed, throughput=5MB/sec 2024-09-13T00:09:56.406 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-zrKtur2niK_sgOm off=21678995 len=1148725 2024-09-13T00:09:56.409 INFO:tasks.workunit.client.0.smithi136.stdout:op 2629 completed, throughput=5MB/sec 2024-09-13T00:09:57.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:09:57 smithi181 bash[67587]: cluster 2024-09-13T00:09:55.402730+0000 mgr.y (mgr.24661) 984 : cluster [DBG] pgmap v1667: 105 pgs: 105 active+clean; 126 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:09:57.409 INFO:tasks.workunit.client.0.smithi136.stdout: 498: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:57.409 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-y_pyQJTlecXrgpi off=3394858 len=352917 2024-09-13T00:09:57.414 INFO:tasks.workunit.client.0.smithi136.stdout:op 2630 completed, throughput=4.99MB/sec 2024-09-13T00:09:57.415 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-bOoZqcDsbepqhxz off=20164243 len=188040 2024-09-13T00:09:57.416 INFO:tasks.workunit.client.0.smithi136.stdout:op 2631 completed, throughput=4.99MB/sec 2024-09-13T00:09:57.416 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-3Eiyn6L3fxBC326 off=7491565 len=195845 2024-09-13T00:09:57.422 INFO:tasks.workunit.client.0.smithi136.stdout:op 2632 completed, throughput=4.99MB/sec 2024-09-13T00:09:57.422 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-qLGP2IXMqTz8D9l off=9313741 len=210238 2024-09-13T00:09:57.426 INFO:tasks.workunit.client.0.smithi136.stdout:op 2633 completed, throughput=4.99MB/sec 2024-09-13T00:09:57.427 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-1wZ9MfdOJRnl8LQ off=12373238 len=129043 2024-09-13T00:09:57.431 INFO:tasks.workunit.client.0.smithi136.stdout:op 2634 completed, throughput=4.99MB/sec 2024-09-13T00:09:57.431 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-GpfVCnKlAR7-lBT off=4872736 len=229293 2024-09-13T00:09:57.432 INFO:tasks.workunit.client.0.smithi136.stdout:op 2635 completed, throughput=4.99MB/sec 2024-09-13T00:09:57.432 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-CW0siSl9CU022fT off=828747 len=1595280 2024-09-13T00:09:57.444 INFO:tasks.workunit.client.0.smithi136.stdout:op 2636 completed, throughput=5MB/sec 2024-09-13T00:09:57.444 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Db6awmFK_tyW6cG off=452908 len=1881625 2024-09-13T00:09:57.448 INFO:tasks.workunit.client.0.smithi136.stdout:op 2637 completed, throughput=5MB/sec 2024-09-13T00:09:57.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:09:57 smithi136 bash[76371]: cluster 2024-09-13T00:09:55.402730+0000 mgr.y (mgr.24661) 984 : cluster [DBG] pgmap v1667: 105 pgs: 105 active+clean; 126 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:09:57.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:09:57 smithi136 bash[75164]: cluster 2024-09-13T00:09:55.402730+0000 mgr.y (mgr.24661) 984 : cluster [DBG] pgmap v1667: 105 pgs: 105 active+clean; 126 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:09:58.448 INFO:tasks.workunit.client.0.smithi136.stdout: 499: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:58.449 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-x_3xScAT3aBMmoe off=187064 len=1511344 2024-09-13T00:09:58.459 INFO:tasks.workunit.client.0.smithi136.stdout:op 2638 completed, throughput=4.99MB/sec 2024-09-13T00:09:58.459 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-DzOyApZL8iDXHdp off=6920091 len=1928711 2024-09-13T00:09:58.464 INFO:tasks.workunit.client.0.smithi136.stdout:op 2639 completed, throughput=5MB/sec 2024-09-13T00:09:58.464 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-uEldETpBx4ulGrA off=1854656 len=1846009 2024-09-13T00:09:58.475 INFO:tasks.workunit.client.0.smithi136.stdout:op 2640 completed, throughput=5MB/sec 2024-09-13T00:09:59.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:09:59 smithi181 bash[67587]: cluster 2024-09-13T00:09:57.403900+0000 mgr.y (mgr.24661) 985 : cluster [DBG] pgmap v1668: 105 pgs: 105 active+clean; 126 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 3.3 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:09:59.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:09:59 smithi136 bash[75164]: cluster 2024-09-13T00:09:57.403900+0000 mgr.y (mgr.24661) 985 : cluster [DBG] pgmap v1668: 105 pgs: 105 active+clean; 126 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 3.3 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:09:59.414 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:09:59 smithi136 bash[76371]: cluster 2024-09-13T00:09:57.403900+0000 mgr.y (mgr.24661) 985 : cluster [DBG] pgmap v1668: 105 pgs: 105 active+clean; 126 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 3.3 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:09:59.475 INFO:tasks.workunit.client.0.smithi136.stdout: 500: throughput=4.99MB/sec pending data=0 2024-09-13T00:09:59.476 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-H_e2C-ViKuEAZsg off=3406547 len=48356 2024-09-13T00:09:59.479 INFO:tasks.workunit.client.0.smithi136.stdout:op 2641 completed, throughput=4.99MB/sec 2024-09-13T00:09:59.479 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-OQIWR8fWb7dYfSB off=2412185 len=1325614 2024-09-13T00:09:59.493 INFO:tasks.workunit.client.0.smithi136.stdout:op 2642 completed, throughput=4.99MB/sec 2024-09-13T00:09:59.493 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Ye1iVX0XroAi2ZV off=2221500 len=571307 2024-09-13T00:09:59.496 INFO:tasks.workunit.client.0.smithi136.stdout:op 2643 completed, throughput=4.99MB/sec 2024-09-13T00:09:59.496 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-IZrJ49A7f0c5TFm off=12604764 len=2033115 2024-09-13T00:09:59.512 INFO:tasks.workunit.client.0.smithi136.stdout:op 2644 completed, throughput=5MB/sec 2024-09-13T00:09:59.512 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-R9ZrNogvg88An83 off=11329137 len=178162 2024-09-13T00:09:59.516 INFO:tasks.workunit.client.0.smithi136.stdout:op 2645 completed, throughput=5MB/sec 2024-09-13T00:09:59.516 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-dvMXeRPEx-6CTCM off=16161947 len=1467144 2024-09-13T00:09:59.527 INFO:tasks.workunit.client.0.smithi136.stdout:op 2646 completed, throughput=5MB/sec 2024-09-13T00:09:59.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:09:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:09:59] "GET /metrics HTTP/1.1" 200 34141 "" "Prometheus/2.43.0" 2024-09-13T00:10:00.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:10:00 smithi181 bash[67587]: cluster 2024-09-13T00:10:00.000165+0000 mon.a (mon.0) 2183 : cluster [INF] overall HEALTH_OK 2024-09-13T00:10:00.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:10:00 smithi136 bash[76371]: cluster 2024-09-13T00:10:00.000165+0000 mon.a (mon.0) 2183 : cluster [INF] overall HEALTH_OK 2024-09-13T00:10:00.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:10:00 smithi136 bash[75164]: cluster 2024-09-13T00:10:00.000165+0000 mon.a (mon.0) 2183 : cluster [INF] overall HEALTH_OK 2024-09-13T00:10:00.527 INFO:tasks.workunit.client.0.smithi136.stdout: 501: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:00.527 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-SYfZI3LiE7tSeBu off=14243855 len=1953039 2024-09-13T00:10:00.532 INFO:tasks.workunit.client.0.smithi136.stdout:op 2647 completed, throughput=4.99MB/sec 2024-09-13T00:10:00.532 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-oxNA410hJSmsek7 off=5272784 len=1602733 2024-09-13T00:10:00.543 INFO:tasks.workunit.client.0.smithi136.stdout:op 2648 completed, throughput=5MB/sec 2024-09-13T00:10:00.543 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-IVESCAS-NGoJWtf off=2604633 len=1424782 2024-09-13T00:10:00.546 INFO:tasks.workunit.client.0.smithi136.stdout:op 2649 completed, throughput=5MB/sec 2024-09-13T00:10:00.546 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-LYNb40N9nx-0NHO off=21574208 len=430996 2024-09-13T00:10:00.552 INFO:tasks.workunit.client.0.smithi136.stdout:op 2650 completed, throughput=5MB/sec 2024-09-13T00:10:01.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:10:01 smithi181 bash[67587]: cluster 2024-09-13T00:09:59.404956+0000 mgr.y (mgr.24661) 986 : cluster [DBG] pgmap v1669: 105 pgs: 105 active+clean; 126 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:10:01.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:10:01 smithi136 bash[75164]: cluster 2024-09-13T00:09:59.404956+0000 mgr.y (mgr.24661) 986 : cluster [DBG] pgmap v1669: 105 pgs: 105 active+clean; 126 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:10:01.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:10:01 smithi136 bash[76371]: cluster 2024-09-13T00:09:59.404956+0000 mgr.y (mgr.24661) 986 : cluster [DBG] pgmap v1669: 105 pgs: 105 active+clean; 126 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:10:01.552 INFO:tasks.workunit.client.0.smithi136.stdout: 502: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:01.553 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-YS-nSOr9OV9Yxun off=7945297 len=1979323 2024-09-13T00:10:01.557 INFO:tasks.workunit.client.0.smithi136.stdout:op 2651 completed, throughput=4.99MB/sec 2024-09-13T00:10:01.557 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2z8mdAeQ-e1zYCS off=2024295 len=1713086 2024-09-13T00:10:01.561 INFO:tasks.workunit.client.0.smithi136.stdout:op 2652 completed, throughput=5MB/sec 2024-09-13T00:10:01.562 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Js0tfa6s-yWp-4Y off=1214916 len=1273156 2024-09-13T00:10:01.564 INFO:tasks.workunit.client.0.smithi136.stdout:op 2653 completed, throughput=5MB/sec 2024-09-13T00:10:01.564 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-0yWOJ7lfUIZXmVc off=3446249 len=2047286 2024-09-13T00:10:01.569 INFO:tasks.workunit.client.0.smithi136.stdout:op 2654 completed, throughput=5MB/sec 2024-09-13T00:10:02.569 INFO:tasks.workunit.client.0.smithi136.stdout: 503: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:02.570 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-N2uYhZ2cMHOXdXN off=4028819 len=1731476 2024-09-13T00:10:02.573 INFO:tasks.workunit.client.0.smithi136.stdout:op 2655 completed, throughput=5MB/sec 2024-09-13T00:10:02.574 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-UZVu4nVWO9jUA09 off=6779699 len=85414 2024-09-13T00:10:02.575 INFO:tasks.workunit.client.0.smithi136.stdout:op 2656 completed, throughput=5MB/sec 2024-09-13T00:10:02.575 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-fRuTwRjouOqfaH3 off=193524 len=933875 2024-09-13T00:10:02.583 INFO:tasks.workunit.client.0.smithi136.stdout:op 2657 completed, throughput=5MB/sec 2024-09-13T00:10:02.583 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-mlf2PmFU6lgJHIq off=2397691 len=1614827 2024-09-13T00:10:02.587 INFO:tasks.workunit.client.0.smithi136.stdout:op 2658 completed, throughput=5MB/sec 2024-09-13T00:10:02.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:10:02 smithi181 bash[95505]: ts=2024-09-13T00:10:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:10:03.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:10:03 smithi181 bash[67587]: cluster 2024-09-13T00:10:01.405985+0000 mgr.y (mgr.24661) 987 : cluster [DBG] pgmap v1670: 105 pgs: 105 active+clean; 126 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:10:03.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:10:03 smithi136 bash[75164]: cluster 2024-09-13T00:10:01.405985+0000 mgr.y (mgr.24661) 987 : cluster [DBG] pgmap v1670: 105 pgs: 105 active+clean; 126 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:10:03.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:10:03 smithi136 bash[76371]: cluster 2024-09-13T00:10:01.405985+0000 mgr.y (mgr.24661) 987 : cluster [DBG] pgmap v1670: 105 pgs: 105 active+clean; 126 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:10:03.588 INFO:tasks.workunit.client.0.smithi136.stdout: 504: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:03.588 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-kG5AMpvhpF6nHwF off=554815 len=1335791 2024-09-13T00:10:03.597 INFO:tasks.workunit.client.0.smithi136.stdout:op 2659 completed, throughput=4.99MB/sec 2024-09-13T00:10:03.597 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-9rcJcDKvH47cryD off=3497632 len=432903 2024-09-13T00:10:03.603 INFO:tasks.workunit.client.0.smithi136.stdout:op 2660 completed, throughput=5MB/sec 2024-09-13T00:10:03.603 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-cQVfwrznoKPPHCU off=1289411 len=2013095 2024-09-13T00:10:03.608 INFO:tasks.workunit.client.0.smithi136.stdout:op 2661 completed, throughput=5MB/sec 2024-09-13T00:10:03.608 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Jluy9aobOjxkchq off=14484508 len=1234670 2024-09-13T00:10:03.612 INFO:tasks.workunit.client.0.smithi136.stdout:op 2662 completed, throughput=5MB/sec 2024-09-13T00:10:04.612 INFO:tasks.workunit.client.0.smithi136.stdout: 505: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:04.612 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-lV-QahN0wcd9B4N off=5998362 len=651804 2024-09-13T00:10:04.614 INFO:tasks.workunit.client.0.smithi136.stdout:op 2663 completed, throughput=4.99MB/sec 2024-09-13T00:10:04.614 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-cyvrNATdBnSGYCc off=5351917 len=1107120 2024-09-13T00:10:04.617 INFO:tasks.workunit.client.0.smithi136.stdout:op 2664 completed, throughput=4.99MB/sec 2024-09-13T00:10:04.617 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-UMd6sbzD1ptvP2s off=8711156 len=541368 2024-09-13T00:10:04.624 INFO:tasks.workunit.client.0.smithi136.stdout:op 2665 completed, throughput=5MB/sec 2024-09-13T00:10:04.625 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-OUNmXeyxoQ3VsQv off=490914 len=1736580 2024-09-13T00:10:04.640 INFO:tasks.workunit.client.0.smithi136.stdout:op 2666 completed, throughput=5MB/sec 2024-09-13T00:10:04.640 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-khkgkXKsMhkbKKR off=2958807 len=1754018 2024-09-13T00:10:04.651 INFO:tasks.workunit.client.0.smithi136.stdout:op 2667 completed, throughput=5MB/sec 2024-09-13T00:10:05.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:10:05 smithi181 bash[67587]: cluster 2024-09-13T00:10:03.406941+0000 mgr.y (mgr.24661) 988 : cluster [DBG] pgmap v1671: 105 pgs: 105 active+clean; 126 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:10:05.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:10:05 smithi136 bash[75164]: cluster 2024-09-13T00:10:03.406941+0000 mgr.y (mgr.24661) 988 : cluster [DBG] pgmap v1671: 105 pgs: 105 active+clean; 126 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:10:05.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:10:05 smithi136 bash[76371]: cluster 2024-09-13T00:10:03.406941+0000 mgr.y (mgr.24661) 988 : cluster [DBG] pgmap v1671: 105 pgs: 105 active+clean; 126 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:10:05.651 INFO:tasks.workunit.client.0.smithi136.stdout: 506: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:05.651 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-UXg5bBi_cq-rqpl off=7898326 len=1967152 2024-09-13T00:10:05.671 INFO:tasks.workunit.client.0.smithi136.stdout:op 2668 completed, throughput=5MB/sec 2024-09-13T00:10:05.671 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-0Pr6aXuVOwSKruP off=1455800 len=1000343 2024-09-13T00:10:05.680 INFO:tasks.workunit.client.0.smithi136.stdout:op 2669 completed, throughput=5MB/sec 2024-09-13T00:10:05.680 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-0IEuV-Vpn-gvKvI off=2949563 len=2083103 2024-09-13T00:10:05.695 INFO:tasks.workunit.client.0.smithi136.stdout:op 2670 completed, throughput=5MB/sec 2024-09-13T00:10:06.695 INFO:tasks.workunit.client.0.smithi136.stdout: 507: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:06.695 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-NrVRyrRI-n0c22S off=4131933 len=1747979 2024-09-13T00:10:06.708 INFO:tasks.workunit.client.0.smithi136.stdout:op 2671 completed, throughput=4.99MB/sec 2024-09-13T00:10:06.708 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-OMhQyipKkOmDhUP off=8372144 len=1049252 2024-09-13T00:10:06.716 INFO:tasks.workunit.client.0.smithi136.stdout:op 2672 completed, throughput=5MB/sec 2024-09-13T00:10:06.716 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-KmS8lbAQnH27D0D off=14073281 len=1609862 2024-09-13T00:10:06.718 INFO:tasks.workunit.client.0.smithi136.stdout:op 2673 completed, throughput=5MB/sec 2024-09-13T00:10:06.718 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-sGeauTLdtjmYukj off=2486767 len=1919332 2024-09-13T00:10:06.725 INFO:tasks.workunit.client.0.smithi136.stdout:op 2674 completed, throughput=5MB/sec 2024-09-13T00:10:07.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:10:07 smithi181 bash[67587]: cluster 2024-09-13T00:10:05.407781+0000 mgr.y (mgr.24661) 989 : cluster [DBG] pgmap v1672: 105 pgs: 105 active+clean; 126 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:10:07.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:10:07 smithi136 bash[75164]: cluster 2024-09-13T00:10:05.407781+0000 mgr.y (mgr.24661) 989 : cluster [DBG] pgmap v1672: 105 pgs: 105 active+clean; 126 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:10:07.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:10:07 smithi136 bash[76371]: cluster 2024-09-13T00:10:05.407781+0000 mgr.y (mgr.24661) 989 : cluster [DBG] pgmap v1672: 105 pgs: 105 active+clean; 126 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:10:07.725 INFO:tasks.workunit.client.0.smithi136.stdout: 508: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:07.725 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-u7xE4SJ2CItSh82 off=12315037 len=1257776 2024-09-13T00:10:07.735 INFO:tasks.workunit.client.0.smithi136.stdout:op 2675 completed, throughput=5MB/sec 2024-09-13T00:10:07.735 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-n9W-2-lPcBd4KDk off=988680 len=148691 2024-09-13T00:10:07.739 INFO:tasks.workunit.client.0.smithi136.stdout:op 2676 completed, throughput=5MB/sec 2024-09-13T00:10:07.740 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-YlgDqQ8gCEiaX-D off=4172222 len=2073926 2024-09-13T00:10:07.754 INFO:tasks.workunit.client.0.smithi136.stdout:op 2677 completed, throughput=5MB/sec 2024-09-13T00:10:07.755 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-vbwd3fp9KRR4pLO off=8634892 len=1913927 2024-09-13T00:10:07.770 INFO:tasks.workunit.client.0.smithi136.stdout:op 2678 completed, throughput=5MB/sec 2024-09-13T00:10:08.770 INFO:tasks.workunit.client.0.smithi136.stdout: 509: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:08.770 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-O_aaP4TSQ5Fjt82 off=10383008 len=1654147 2024-09-13T00:10:08.776 INFO:tasks.workunit.client.0.smithi136.stdout:op 2679 completed, throughput=5MB/sec 2024-09-13T00:10:08.776 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-u7xE4SJ2CItSh82 off=3807706 len=616464 2024-09-13T00:10:08.783 INFO:tasks.workunit.client.0.smithi136.stdout:op 2680 completed, throughput=5MB/sec 2024-09-13T00:10:08.783 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-EEmoYfjKbUdKQBy off=340802 len=1100711 2024-09-13T00:10:08.786 INFO:tasks.workunit.client.0.smithi136.stdout:op 2681 completed, throughput=5MB/sec 2024-09-13T00:10:08.787 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-KRXPREsJ3cId9dw off=11262236 len=1216606 2024-09-13T00:10:08.790 INFO:tasks.workunit.client.0.smithi136.stdout:op 2682 completed, throughput=5MB/sec 2024-09-13T00:10:09.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:10:09 smithi181 bash[67587]: cluster 2024-09-13T00:10:07.408937+0000 mgr.y (mgr.24661) 990 : cluster [DBG] pgmap v1673: 105 pgs: 105 active+clean; 126 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:10:09.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:10:09 smithi181 bash[67587]: audit 2024-09-13T00:10:08.786251+0000 mon.a (mon.0) 2184 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:10:09.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:10:09 smithi136 bash[75164]: cluster 2024-09-13T00:10:07.408937+0000 mgr.y (mgr.24661) 990 : cluster [DBG] pgmap v1673: 105 pgs: 105 active+clean; 126 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:10:09.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:10:09 smithi136 bash[75164]: audit 2024-09-13T00:10:08.786251+0000 mon.a (mon.0) 2184 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:10:09.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:10:09 smithi136 bash[76371]: cluster 2024-09-13T00:10:07.408937+0000 mgr.y (mgr.24661) 990 : cluster [DBG] pgmap v1673: 105 pgs: 105 active+clean; 126 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:10:09.414 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:10:09 smithi136 bash[76371]: audit 2024-09-13T00:10:08.786251+0000 mon.a (mon.0) 2184 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:10:09.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:10:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:10:09] "GET /metrics HTTP/1.1" 200 34138 "" "Prometheus/2.43.0" 2024-09-13T00:10:09.790 INFO:tasks.workunit.client.0.smithi136.stdout: 510: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:09.790 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-GVgDnG85eWObM2C off=7489275 len=1130012 2024-09-13T00:10:09.801 INFO:tasks.workunit.client.0.smithi136.stdout:op 2683 completed, throughput=4.99MB/sec 2024-09-13T00:10:09.801 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-I3Bqj3S1EjiP8zs off=6466496 len=1550061 2024-09-13T00:10:09.805 INFO:tasks.workunit.client.0.smithi136.stdout:op 2684 completed, throughput=5MB/sec 2024-09-13T00:10:09.805 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-gYK9dXRGPg0L_RP off=100831 len=628910 2024-09-13T00:10:09.807 INFO:tasks.workunit.client.0.smithi136.stdout:op 2685 completed, throughput=5MB/sec 2024-09-13T00:10:09.807 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-zYXqzBQPzx9jo6u off=17737506 len=1107397 2024-09-13T00:10:09.815 INFO:tasks.workunit.client.0.smithi136.stdout:op 2686 completed, throughput=5MB/sec 2024-09-13T00:10:09.816 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-hqcsuZyQbzrvFc4 off=9530251 len=1516399 2024-09-13T00:10:09.826 INFO:tasks.workunit.client.0.smithi136.stdout:op 2687 completed, throughput=5MB/sec 2024-09-13T00:10:10.826 INFO:tasks.workunit.client.0.smithi136.stdout: 511: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:10.827 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-wEL5UOt1PbXG2PB off=575651 len=250336 2024-09-13T00:10:10.827 INFO:tasks.workunit.client.0.smithi136.stdout:op 2688 completed, throughput=4.99MB/sec 2024-09-13T00:10:10.828 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-LrdRvAzMYcL4DMn off=20971734 len=1635985 2024-09-13T00:10:10.832 INFO:tasks.workunit.client.0.smithi136.stdout:op 2689 completed, throughput=5MB/sec 2024-09-13T00:10:10.832 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-PRWWfEAQgR93Bfb off=3387918 len=1361136 2024-09-13T00:10:10.836 INFO:tasks.workunit.client.0.smithi136.stdout:op 2690 completed, throughput=5MB/sec 2024-09-13T00:10:10.836 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-KbS2phJ59Tkdtn2 off=5948028 len=473178 2024-09-13T00:10:10.843 INFO:tasks.workunit.client.0.smithi136.stdout:op 2691 completed, throughput=5MB/sec 2024-09-13T00:10:10.844 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-c8moNPDj-QDqeVh off=1140073 len=203691 2024-09-13T00:10:10.849 INFO:tasks.workunit.client.0.smithi136.stdout:op 2692 completed, throughput=5MB/sec 2024-09-13T00:10:10.849 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-K7sWiFA-inmWCnC off=4799303 len=1492411 2024-09-13T00:10:10.853 INFO:tasks.workunit.client.0.smithi136.stdout:op 2693 completed, throughput=5MB/sec 2024-09-13T00:10:11.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:10:11 smithi181 bash[67587]: cluster 2024-09-13T00:10:09.409842+0000 mgr.y (mgr.24661) 991 : cluster [DBG] pgmap v1674: 105 pgs: 105 active+clean; 126 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.2 MiB/s wr, 3 op/s 2024-09-13T00:10:11.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:10:11 smithi136 bash[75164]: cluster 2024-09-13T00:10:09.409842+0000 mgr.y (mgr.24661) 991 : cluster [DBG] pgmap v1674: 105 pgs: 105 active+clean; 126 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.2 MiB/s wr, 3 op/s 2024-09-13T00:10:11.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:10:11 smithi136 bash[76371]: cluster 2024-09-13T00:10:09.409842+0000 mgr.y (mgr.24661) 991 : cluster [DBG] pgmap v1674: 105 pgs: 105 active+clean; 126 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.2 MiB/s wr, 3 op/s 2024-09-13T00:10:11.853 INFO:tasks.workunit.client.0.smithi136.stdout: 512: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:11.854 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ihrU7EAIqBbQjrc off=234196 len=1828657 2024-09-13T00:10:11.859 INFO:tasks.workunit.client.0.smithi136.stdout:op 2694 completed, throughput=5MB/sec 2024-09-13T00:10:11.859 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-4y5s4UtBMb9GI3T off=4456875 len=793417 2024-09-13T00:10:11.862 INFO:tasks.workunit.client.0.smithi136.stdout:op 2695 completed, throughput=5MB/sec 2024-09-13T00:10:11.862 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-aJIXVmcpHrNOgiD off=6218885 len=798941 2024-09-13T00:10:11.869 INFO:tasks.workunit.client.0.smithi136.stdout:op 2696 completed, throughput=5MB/sec 2024-09-13T00:10:11.870 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Na7aYvjQcfBTKqG off=10367167 len=1648569 2024-09-13T00:10:11.882 INFO:tasks.workunit.client.0.smithi136.stdout:op 2697 completed, throughput=5MB/sec 2024-09-13T00:10:12.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:10:12 smithi181 bash[95505]: ts=2024-09-13T00:10:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:10:12.882 INFO:tasks.workunit.client.0.smithi136.stdout: 513: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:12.882 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_wa6A7gpvBVY2YS off=22395981 len=1818907 2024-09-13T00:10:12.897 INFO:tasks.workunit.client.0.smithi136.stdout:op 2698 completed, throughput=5MB/sec 2024-09-13T00:10:12.897 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-3-2zkYd59I1h5d7 off=1825885 len=1664878 2024-09-13T00:10:12.900 INFO:tasks.workunit.client.0.smithi136.stdout:op 2699 completed, throughput=5MB/sec 2024-09-13T00:10:12.901 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-v_FFWmtTzbADzXC off=2435067 len=1620811 2024-09-13T00:10:12.915 INFO:tasks.workunit.client.0.smithi136.stdout:op 2700 completed, throughput=5MB/sec 2024-09-13T00:10:13.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:10:13 smithi181 bash[67587]: cluster 2024-09-13T00:10:11.410821+0000 mgr.y (mgr.24661) 992 : cluster [DBG] pgmap v1675: 105 pgs: 105 active+clean; 126 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.4 MiB/s wr, 3 op/s 2024-09-13T00:10:13.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:10:13 smithi136 bash[75164]: cluster 2024-09-13T00:10:11.410821+0000 mgr.y (mgr.24661) 992 : cluster [DBG] pgmap v1675: 105 pgs: 105 active+clean; 126 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.4 MiB/s wr, 3 op/s 2024-09-13T00:10:13.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:10:13 smithi136 bash[76371]: cluster 2024-09-13T00:10:11.410821+0000 mgr.y (mgr.24661) 992 : cluster [DBG] pgmap v1675: 105 pgs: 105 active+clean; 126 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.4 MiB/s wr, 3 op/s 2024-09-13T00:10:13.915 INFO:tasks.workunit.client.0.smithi136.stdout: 514: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:13.915 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_2qpBodyVpN2Qi6 off=2428404 len=1890690 2024-09-13T00:10:13.931 INFO:tasks.workunit.client.0.smithi136.stdout:op 2701 completed, throughput=4.99MB/sec 2024-09-13T00:10:13.931 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-WRo3tPcordaW7Ed off=1951073 len=1467934 2024-09-13T00:10:13.936 INFO:tasks.workunit.client.0.smithi136.stdout:op 2702 completed, throughput=5MB/sec 2024-09-13T00:10:13.936 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-rUCXwz-51VEH6Qb off=6226496 len=1929009 2024-09-13T00:10:13.950 INFO:tasks.workunit.client.0.smithi136.stdout:op 2703 completed, throughput=5MB/sec 2024-09-13T00:10:14.098 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 13 00:10:13 smithi181 bash[98820]: logger=cleanup t=2024-09-13T00:10:13.741032779Z level=info msg="Completed cleanup jobs" duration=46.941891ms 2024-09-13T00:10:14.950 INFO:tasks.workunit.client.0.smithi136.stdout: 515: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:14.951 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-5pFuz2_huYFvYm9 off=11450887 len=751250 2024-09-13T00:10:14.957 INFO:tasks.workunit.client.0.smithi136.stdout:op 2704 completed, throughput=4.99MB/sec 2024-09-13T00:10:14.957 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-DZPoYcqxxkNzp3B off=8739310 len=1332732 2024-09-13T00:10:14.962 INFO:tasks.workunit.client.0.smithi136.stdout:op 2705 completed, throughput=4.99MB/sec 2024-09-13T00:10:14.962 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-uLAPfuuXBCwcuYX off=22286138 len=1810930 2024-09-13T00:10:14.966 INFO:tasks.workunit.client.0.smithi136.stdout:op 2706 completed, throughput=5MB/sec 2024-09-13T00:10:14.967 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-9sZRBMwK0lvTWzE off=3826153 len=840696 2024-09-13T00:10:14.975 INFO:tasks.workunit.client.0.smithi136.stdout:op 2707 completed, throughput=5MB/sec 2024-09-13T00:10:14.975 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-zEQeWH5-amY9L67 off=519679 len=1063597 2024-09-13T00:10:14.984 INFO:tasks.workunit.client.0.smithi136.stdout:op 2708 completed, throughput=5MB/sec 2024-09-13T00:10:15.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:10:15 smithi181 bash[67587]: cluster 2024-09-13T00:10:13.411743+0000 mgr.y (mgr.24661) 993 : cluster [DBG] pgmap v1676: 105 pgs: 105 active+clean; 126 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.5 MiB/s wr, 3 op/s 2024-09-13T00:10:15.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:10:15 smithi136 bash[75164]: cluster 2024-09-13T00:10:13.411743+0000 mgr.y (mgr.24661) 993 : cluster [DBG] pgmap v1676: 105 pgs: 105 active+clean; 126 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.5 MiB/s wr, 3 op/s 2024-09-13T00:10:15.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:10:15 smithi136 bash[76371]: cluster 2024-09-13T00:10:13.411743+0000 mgr.y (mgr.24661) 993 : cluster [DBG] pgmap v1676: 105 pgs: 105 active+clean; 126 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.5 MiB/s wr, 3 op/s 2024-09-13T00:10:15.984 INFO:tasks.workunit.client.0.smithi136.stdout: 516: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:15.985 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-jHKp-zbgy7_S0Oa off=11466489 len=1546595 2024-09-13T00:10:15.996 INFO:tasks.workunit.client.0.smithi136.stdout:op 2709 completed, throughput=4.99MB/sec 2024-09-13T00:10:15.996 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-UAMeuyrzbY4vf5J off=12487038 len=1003188 2024-09-13T00:10:15.999 INFO:tasks.workunit.client.0.smithi136.stdout:op 2710 completed, throughput=5MB/sec 2024-09-13T00:10:15.999 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-khHemI2tUBBv6mp off=3685239 len=1657826 2024-09-13T00:10:16.009 INFO:tasks.workunit.client.0.smithi136.stdout:op 2711 completed, throughput=5MB/sec 2024-09-13T00:10:16.010 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-kwM5MbvHhZyYSfU off=0 len=1753096 2024-09-13T00:10:16.023 INFO:tasks.workunit.client.0.smithi136.stdout:op 2712 completed, throughput=5MB/sec 2024-09-13T00:10:17.024 INFO:tasks.workunit.client.0.smithi136.stdout: 517: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:17.024 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-OSMEQkWdtl6XGf9 off=1826542 len=475941 2024-09-13T00:10:17.025 INFO:tasks.workunit.client.0.smithi136.stdout:op 2713 completed, throughput=4.99MB/sec 2024-09-13T00:10:17.026 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Sy3Onl4BwDFPWLU off=16718507 len=1608632 2024-09-13T00:10:17.029 INFO:tasks.workunit.client.0.smithi136.stdout:op 2714 completed, throughput=5MB/sec 2024-09-13T00:10:17.030 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-o9odtEtnMfmge0B off=750142 len=235022 2024-09-13T00:10:17.037 INFO:tasks.workunit.client.0.smithi136.stdout:op 2715 completed, throughput=5MB/sec 2024-09-13T00:10:17.037 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-09r9VxXy0OXMkDT off=0 len=1507505 2024-09-13T00:10:17.042 INFO:tasks.workunit.client.0.smithi136.stdout:op 2716 completed, throughput=5MB/sec 2024-09-13T00:10:17.042 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-NUiArYPXTa5Zs9V off=2922181 len=1007150 2024-09-13T00:10:17.045 INFO:tasks.workunit.client.0.smithi136.stdout:op 2717 completed, throughput=5MB/sec 2024-09-13T00:10:17.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:10:17 smithi181 bash[67587]: cluster 2024-09-13T00:10:15.412562+0000 mgr.y (mgr.24661) 994 : cluster [DBG] pgmap v1677: 105 pgs: 105 active+clean; 126 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.5 MiB/s wr, 3 op/s 2024-09-13T00:10:17.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:10:17 smithi136 bash[75164]: cluster 2024-09-13T00:10:15.412562+0000 mgr.y (mgr.24661) 994 : cluster [DBG] pgmap v1677: 105 pgs: 105 active+clean; 126 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.5 MiB/s wr, 3 op/s 2024-09-13T00:10:17.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:10:17 smithi136 bash[76371]: cluster 2024-09-13T00:10:15.412562+0000 mgr.y (mgr.24661) 994 : cluster [DBG] pgmap v1677: 105 pgs: 105 active+clean; 126 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.5 MiB/s wr, 3 op/s 2024-09-13T00:10:18.045 INFO:tasks.workunit.client.0.smithi136.stdout: 518: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:18.046 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-11wC9PLHGJKXQTl off=1715466 len=334376 2024-09-13T00:10:18.051 INFO:tasks.workunit.client.0.smithi136.stdout:op 2718 completed, throughput=4.99MB/sec 2024-09-13T00:10:18.052 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Rg2rC9OvRo4pYJf off=22661919 len=284259 2024-09-13T00:10:18.053 INFO:tasks.workunit.client.0.smithi136.stdout:op 2719 completed, throughput=4.99MB/sec 2024-09-13T00:10:18.053 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Ta47-u9bvcDuPGk off=17941958 len=1417735 2024-09-13T00:10:18.063 INFO:tasks.workunit.client.0.smithi136.stdout:op 2720 completed, throughput=5MB/sec 2024-09-13T00:10:18.063 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-E3Vtkz7YVSxNU2y off=4075611 len=1743180 2024-09-13T00:10:18.067 INFO:tasks.workunit.client.0.smithi136.stdout:op 2721 completed, throughput=5MB/sec 2024-09-13T00:10:18.067 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-I3Bqj3S1EjiP8zs off=1482398 len=1169458 2024-09-13T00:10:18.076 INFO:tasks.workunit.client.0.smithi136.stdout:op 2722 completed, throughput=5MB/sec 2024-09-13T00:10:19.076 INFO:tasks.workunit.client.0.smithi136.stdout: 519: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:19.076 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-lJGe4yfpnm9wDNU off=463571 len=2014145 2024-09-13T00:10:19.081 INFO:tasks.workunit.client.0.smithi136.stdout:op 2723 completed, throughput=4.99MB/sec 2024-09-13T00:10:19.081 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-N0YE2ZHTYUe1ULp off=876011 len=678834 2024-09-13T00:10:19.089 INFO:tasks.workunit.client.0.smithi136.stdout:op 2724 completed, throughput=5MB/sec 2024-09-13T00:10:19.089 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-D1qGT_072CwGuJj off=14016280 len=980764 2024-09-13T00:10:19.092 INFO:tasks.workunit.client.0.smithi136.stdout:op 2725 completed, throughput=5MB/sec 2024-09-13T00:10:19.092 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-xD-OGeEMy54Gen8 off=2128909 len=1753853 2024-09-13T00:10:19.096 INFO:tasks.workunit.client.0.smithi136.stdout:op 2726 completed, throughput=5MB/sec 2024-09-13T00:10:19.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:10:19 smithi136 bash[75164]: cluster 2024-09-13T00:10:17.413602+0000 mgr.y (mgr.24661) 995 : cluster [DBG] pgmap v1678: 105 pgs: 105 active+clean; 126 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.9 MiB/s wr, 4 op/s 2024-09-13T00:10:19.414 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:10:19 smithi136 bash[76371]: cluster 2024-09-13T00:10:17.413602+0000 mgr.y (mgr.24661) 995 : cluster [DBG] pgmap v1678: 105 pgs: 105 active+clean; 126 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.9 MiB/s wr, 4 op/s 2024-09-13T00:10:19.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:10:19 smithi181 bash[67587]: cluster 2024-09-13T00:10:17.413602+0000 mgr.y (mgr.24661) 995 : cluster [DBG] pgmap v1678: 105 pgs: 105 active+clean; 126 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.9 MiB/s wr, 4 op/s 2024-09-13T00:10:19.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:10:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:10:19] "GET /metrics HTTP/1.1" 200 34138 "" "Prometheus/2.43.0" 2024-09-13T00:10:20.097 INFO:tasks.workunit.client.0.smithi136.stdout: 520: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:20.097 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-OrriI7dJzeV_vya off=6867560 len=1463469 2024-09-13T00:10:20.110 INFO:tasks.workunit.client.0.smithi136.stdout:op 2727 completed, throughput=4.99MB/sec 2024-09-13T00:10:20.110 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-dVdyENO1K0cjZSF off=3551864 len=1856014 2024-09-13T00:10:20.123 INFO:tasks.workunit.client.0.smithi136.stdout:op 2728 completed, throughput=5MB/sec 2024-09-13T00:10:20.123 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-knosrobamwjf5xN off=267962 len=1292984 2024-09-13T00:10:20.126 INFO:tasks.workunit.client.0.smithi136.stdout:op 2729 completed, throughput=5MB/sec 2024-09-13T00:10:20.126 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-BrMS9ZJdHubncCk off=7267744 len=156568 2024-09-13T00:10:20.131 INFO:tasks.workunit.client.0.smithi136.stdout:op 2730 completed, throughput=5MB/sec 2024-09-13T00:10:20.131 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-PqMtgnikc41wSPw off=5424783 len=956063 2024-09-13T00:10:20.139 INFO:tasks.workunit.client.0.smithi136.stdout:op 2731 completed, throughput=5MB/sec 2024-09-13T00:10:21.139 INFO:tasks.workunit.client.0.smithi136.stdout: 521: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:21.140 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-3OTd7g_VjZr613h off=2232582 len=602414 2024-09-13T00:10:21.142 INFO:tasks.workunit.client.0.smithi136.stdout:op 2732 completed, throughput=4.99MB/sec 2024-09-13T00:10:21.142 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-fbzZVYJQqs5Bl7d off=9904552 len=658717 2024-09-13T00:10:21.144 INFO:tasks.workunit.client.0.smithi136.stdout:op 2733 completed, throughput=4.99MB/sec 2024-09-13T00:10:21.144 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-9LD75ISBPsxZiQ_ off=5621786 len=1068594 2024-09-13T00:10:21.148 INFO:tasks.workunit.client.0.smithi136.stdout:op 2734 completed, throughput=5MB/sec 2024-09-13T00:10:21.148 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-qOQsL3qkcMOFIQO off=14438997 len=72889 2024-09-13T00:10:21.151 INFO:tasks.workunit.client.0.smithi136.stdout:op 2735 completed, throughput=5MB/sec 2024-09-13T00:10:21.151 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ikmqTAUUtQSDHOC off=2392956 len=32984 2024-09-13T00:10:21.152 INFO:tasks.workunit.client.0.smithi136.stdout:op 2736 completed, throughput=5MB/sec 2024-09-13T00:10:21.152 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2fWO-SRY5EePA8z off=182441 len=1663708 2024-09-13T00:10:21.156 INFO:tasks.workunit.client.0.smithi136.stdout:op 2737 completed, throughput=5MB/sec 2024-09-13T00:10:21.156 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-zGpu2CKsNPAotoq off=6428356 len=1109241 2024-09-13T00:10:21.159 INFO:tasks.workunit.client.0.smithi136.stdout:op 2738 completed, throughput=5MB/sec 2024-09-13T00:10:21.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:10:21 smithi136 bash[75164]: cluster 2024-09-13T00:10:19.414469+0000 mgr.y (mgr.24661) 996 : cluster [DBG] pgmap v1679: 105 pgs: 105 active+clean; 126 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.5 MiB/s wr, 3 op/s 2024-09-13T00:10:21.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:10:21 smithi136 bash[76371]: cluster 2024-09-13T00:10:19.414469+0000 mgr.y (mgr.24661) 996 : cluster [DBG] pgmap v1679: 105 pgs: 105 active+clean; 126 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.5 MiB/s wr, 3 op/s 2024-09-13T00:10:21.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:10:21 smithi181 bash[67587]: cluster 2024-09-13T00:10:19.414469+0000 mgr.y (mgr.24661) 996 : cluster [DBG] pgmap v1679: 105 pgs: 105 active+clean; 126 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.5 MiB/s wr, 3 op/s 2024-09-13T00:10:22.160 INFO:tasks.workunit.client.0.smithi136.stdout: 522: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:22.160 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-SnSSXKsPY-KCojX off=90821 len=411795 2024-09-13T00:10:22.165 INFO:tasks.workunit.client.0.smithi136.stdout:op 2739 completed, throughput=4.99MB/sec 2024-09-13T00:10:22.165 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-RGsTiUCNhwLUCdf off=5991657 len=943925 2024-09-13T00:10:22.173 INFO:tasks.workunit.client.0.smithi136.stdout:op 2740 completed, throughput=4.99MB/sec 2024-09-13T00:10:22.173 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-KOSXC9iWG3gbEmh off=7624892 len=1123389 2024-09-13T00:10:22.183 INFO:tasks.workunit.client.0.smithi136.stdout:op 2741 completed, throughput=5MB/sec 2024-09-13T00:10:22.183 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-DctBvLBV2s6akZw off=7149356 len=215688 2024-09-13T00:10:22.188 INFO:tasks.workunit.client.0.smithi136.stdout:op 2742 completed, throughput=5MB/sec 2024-09-13T00:10:22.188 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ihrU7EAIqBbQjrc off=1074414 len=1788362 2024-09-13T00:10:22.194 INFO:tasks.workunit.client.0.smithi136.stdout:op 2743 completed, throughput=5MB/sec 2024-09-13T00:10:22.194 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-sJ644YTPjvHBdmP off=6432454 len=911763 2024-09-13T00:10:22.201 INFO:tasks.workunit.client.0.smithi136.stdout:op 2744 completed, throughput=5MB/sec 2024-09-13T00:10:22.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:10:22 smithi181 bash[95505]: ts=2024-09-13T00:10:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:10:23.201 INFO:tasks.workunit.client.0.smithi136.stdout: 523: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:23.201 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-eNPZ09XqAcFQf9N off=15667 len=1485388 2024-09-13T00:10:23.205 INFO:tasks.workunit.client.0.smithi136.stdout:op 2745 completed, throughput=4.99MB/sec 2024-09-13T00:10:23.205 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-u3udWV_4WNEDkJO off=1571473 len=1606784 2024-09-13T00:10:23.211 INFO:tasks.workunit.client.0.smithi136.stdout:op 2746 completed, throughput=5MB/sec 2024-09-13T00:10:23.211 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-6vBD1xvy7eqa4Zy off=9981457 len=410171 2024-09-13T00:10:23.213 INFO:tasks.workunit.client.0.smithi136.stdout:op 2747 completed, throughput=5MB/sec 2024-09-13T00:10:23.213 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-RystXcIaXHMjQYD off=6179958 len=1815732 2024-09-13T00:10:23.219 INFO:tasks.workunit.client.0.smithi136.stdout:op 2748 completed, throughput=5MB/sec 2024-09-13T00:10:23.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:10:23 smithi136 bash[75164]: cluster 2024-09-13T00:10:21.415382+0000 mgr.y (mgr.24661) 997 : cluster [DBG] pgmap v1680: 105 pgs: 105 active+clean; 126 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:10:23.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:10:23 smithi136 bash[76371]: cluster 2024-09-13T00:10:21.415382+0000 mgr.y (mgr.24661) 997 : cluster [DBG] pgmap v1680: 105 pgs: 105 active+clean; 126 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:10:23.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:10:23 smithi181 bash[67587]: cluster 2024-09-13T00:10:21.415382+0000 mgr.y (mgr.24661) 997 : cluster [DBG] pgmap v1680: 105 pgs: 105 active+clean; 126 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:10:24.220 INFO:tasks.workunit.client.0.smithi136.stdout: 524: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:24.220 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-8H2TPKvXw-dclZ3 off=14534970 len=908375 2024-09-13T00:10:24.228 INFO:tasks.workunit.client.0.smithi136.stdout:op 2749 completed, throughput=4.99MB/sec 2024-09-13T00:10:24.229 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-6TgMB3WknW_booM off=451536 len=1306799 2024-09-13T00:10:24.232 INFO:tasks.workunit.client.0.smithi136.stdout:op 2750 completed, throughput=5MB/sec 2024-09-13T00:10:24.232 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-nNO79DvuvouQOQQ off=9027487 len=792443 2024-09-13T00:10:24.240 INFO:tasks.workunit.client.0.smithi136.stdout:op 2751 completed, throughput=5MB/sec 2024-09-13T00:10:24.240 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-zcYLjGdUeSaDZdK off=1110361 len=1661236 2024-09-13T00:10:24.250 INFO:tasks.workunit.client.0.smithi136.stdout:op 2752 completed, throughput=5MB/sec 2024-09-13T00:10:24.250 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-MjoacfZNLvjm3PK off=4117620 len=1840795 2024-09-13T00:10:24.265 INFO:tasks.workunit.client.0.smithi136.stdout:op 2753 completed, throughput=5MB/sec 2024-09-13T00:10:24.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:10:24 smithi136 bash[75164]: audit 2024-09-13T00:10:23.786472+0000 mon.a (mon.0) 2185 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:10:24.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:10:24 smithi136 bash[76371]: audit 2024-09-13T00:10:23.786472+0000 mon.a (mon.0) 2185 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:10:24.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:10:24 smithi181 bash[67587]: audit 2024-09-13T00:10:23.786472+0000 mon.a (mon.0) 2185 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:10:25.266 INFO:tasks.workunit.client.0.smithi136.stdout: 525: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:25.266 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ZcgF6WqR7SSf7vu off=1308351 len=330081 2024-09-13T00:10:25.271 INFO:tasks.workunit.client.0.smithi136.stdout:op 2754 completed, throughput=4.99MB/sec 2024-09-13T00:10:25.271 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-l0JPg7jui65oMC6 off=7926528 len=756129 2024-09-13T00:10:25.274 INFO:tasks.workunit.client.0.smithi136.stdout:op 2755 completed, throughput=5MB/sec 2024-09-13T00:10:25.274 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_aSrh-ZqxONvT9_ off=5846542 len=1393852 2024-09-13T00:10:25.284 INFO:tasks.workunit.client.0.smithi136.stdout:op 2756 completed, throughput=5MB/sec 2024-09-13T00:10:25.284 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-OkloqHRzxyzcGwv off=21527078 len=224369 2024-09-13T00:10:25.289 INFO:tasks.workunit.client.0.smithi136.stdout:op 2757 completed, throughput=5MB/sec 2024-09-13T00:10:25.289 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-pKvkV0v1I87MS07 off=12310639 len=1707359 2024-09-13T00:10:25.294 INFO:tasks.workunit.client.0.smithi136.stdout:op 2758 completed, throughput=5MB/sec 2024-09-13T00:10:25.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:10:25 smithi136 bash[75164]: cluster 2024-09-13T00:10:23.416308+0000 mgr.y (mgr.24661) 998 : cluster [DBG] pgmap v1681: 105 pgs: 105 active+clean; 126 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:10:25.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:10:25 smithi136 bash[76371]: cluster 2024-09-13T00:10:23.416308+0000 mgr.y (mgr.24661) 998 : cluster [DBG] pgmap v1681: 105 pgs: 105 active+clean; 126 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:10:25.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:10:25 smithi181 bash[67587]: cluster 2024-09-13T00:10:23.416308+0000 mgr.y (mgr.24661) 998 : cluster [DBG] pgmap v1681: 105 pgs: 105 active+clean; 126 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:10:26.295 INFO:tasks.workunit.client.0.smithi136.stdout: 526: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:26.295 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-k2bpnkphmTsRFIL off=12699442 len=570367 2024-09-13T00:10:26.297 INFO:tasks.workunit.client.0.smithi136.stdout:op 2759 completed, throughput=4.99MB/sec 2024-09-13T00:10:26.297 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-C71iXLppXY83J_a off=4356256 len=61404 2024-09-13T00:10:26.298 INFO:tasks.workunit.client.0.smithi136.stdout:op 2760 completed, throughput=4.99MB/sec 2024-09-13T00:10:26.298 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-YmKh8zs29kWECie off=13338176 len=1325752 2024-09-13T00:10:26.303 INFO:tasks.workunit.client.0.smithi136.stdout:op 2761 completed, throughput=4.99MB/sec 2024-09-13T00:10:26.303 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-zQP6TTgm-B81WUx off=2794421 len=254091 2024-09-13T00:10:26.308 INFO:tasks.workunit.client.0.smithi136.stdout:op 2762 completed, throughput=5MB/sec 2024-09-13T00:10:26.308 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-d_r8nyNoH3EUCtH off=3305702 len=997371 2024-09-13T00:10:26.311 INFO:tasks.workunit.client.0.smithi136.stdout:op 2763 completed, throughput=5MB/sec 2024-09-13T00:10:26.311 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-u3O2Tvtk3aq-YHh off=5964042 len=1185643 2024-09-13T00:10:26.314 INFO:tasks.workunit.client.0.smithi136.stdout:op 2764 completed, throughput=5MB/sec 2024-09-13T00:10:26.314 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-stB3mlkj4I5kYkf off=2158183 len=594609 2024-09-13T00:10:26.320 INFO:tasks.workunit.client.0.smithi136.stdout:op 2765 completed, throughput=5MB/sec 2024-09-13T00:10:26.320 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Yj6Ed91LfIE3KSU off=487881 len=1547305 2024-09-13T00:10:26.333 INFO:tasks.workunit.client.0.smithi136.stdout:op 2766 completed, throughput=5MB/sec 2024-09-13T00:10:27.333 INFO:tasks.workunit.client.0.smithi136.stdout: 527: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:27.333 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ZH-G-IpBCJuT-ws off=13268355 len=1050990 2024-09-13T00:10:27.342 INFO:tasks.workunit.client.0.smithi136.stdout:op 2767 completed, throughput=5MB/sec 2024-09-13T00:10:27.342 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-3Qknh1IZUdzPHkx off=1057600 len=113845 2024-09-13T00:10:27.346 INFO:tasks.workunit.client.0.smithi136.stdout:op 2768 completed, throughput=5MB/sec 2024-09-13T00:10:27.347 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-cVUUbmcuuYyzoqQ off=6424081 len=615482 2024-09-13T00:10:27.354 INFO:tasks.workunit.client.0.smithi136.stdout:op 2769 completed, throughput=5MB/sec 2024-09-13T00:10:27.354 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ptgFVU5A6wi5kXi off=4967739 len=1587236 2024-09-13T00:10:27.357 INFO:tasks.workunit.client.0.smithi136.stdout:op 2770 completed, throughput=5MB/sec 2024-09-13T00:10:27.357 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-2IdiYpgnNhKcPmF off=5602066 len=113862 2024-09-13T00:10:27.362 INFO:tasks.workunit.client.0.smithi136.stdout:op 2771 completed, throughput=5MB/sec 2024-09-13T00:10:27.362 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-8YMaFNtl6v8kMj9 off=809337 len=1533335 2024-09-13T00:10:27.373 INFO:tasks.workunit.client.0.smithi136.stdout:op 2772 completed, throughput=5MB/sec 2024-09-13T00:10:27.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:10:27 smithi136 bash[75164]: cluster 2024-09-13T00:10:25.417176+0000 mgr.y (mgr.24661) 999 : cluster [DBG] pgmap v1682: 105 pgs: 105 active+clean; 126 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:10:27.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:10:27 smithi136 bash[76371]: cluster 2024-09-13T00:10:25.417176+0000 mgr.y (mgr.24661) 999 : cluster [DBG] pgmap v1682: 105 pgs: 105 active+clean; 126 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:10:27.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:10:27 smithi181 bash[67587]: cluster 2024-09-13T00:10:25.417176+0000 mgr.y (mgr.24661) 999 : cluster [DBG] pgmap v1682: 105 pgs: 105 active+clean; 126 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:10:28.373 INFO:tasks.workunit.client.0.smithi136.stdout: 528: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:28.374 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ou88DcS7bhoyUF6 off=19657487 len=1976391 2024-09-13T00:10:28.378 INFO:tasks.workunit.client.0.smithi136.stdout:op 2773 completed, throughput=5MB/sec 2024-09-13T00:10:28.378 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-faIJA1GSfqZVsj0 off=3381693 len=1019993 2024-09-13T00:10:28.387 INFO:tasks.workunit.client.0.smithi136.stdout:op 2774 completed, throughput=5MB/sec 2024-09-13T00:10:28.387 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-F1gHhPc3kFBC_ef off=12979513 len=1545278 2024-09-13T00:10:28.397 INFO:tasks.workunit.client.0.smithi136.stdout:op 2775 completed, throughput=5MB/sec 2024-09-13T00:10:29.398 INFO:tasks.workunit.client.0.smithi136.stdout: 529: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:29.398 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-MjoacfZNLvjm3PK off=8574041 len=1311276 2024-09-13T00:10:29.402 INFO:tasks.workunit.client.0.smithi136.stdout:op 2776 completed, throughput=4.99MB/sec 2024-09-13T00:10:29.402 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-kXthstME66Dk3ZZ off=1014690 len=43582 2024-09-13T00:10:29.406 INFO:tasks.workunit.client.0.smithi136.stdout:op 2777 completed, throughput=4.99MB/sec 2024-09-13T00:10:29.406 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Gr7N75-c5LujV-1 off=12847218 len=1887238 2024-09-13T00:10:29.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:10:29 smithi136 bash[75164]: cluster 2024-09-13T00:10:27.418311+0000 mgr.y (mgr.24661) 1000 : cluster [DBG] pgmap v1683: 105 pgs: 105 active+clean; 126 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:10:29.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:10:29 smithi136 bash[76371]: cluster 2024-09-13T00:10:27.418311+0000 mgr.y (mgr.24661) 1000 : cluster [DBG] pgmap v1683: 105 pgs: 105 active+clean; 126 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:10:29.423 INFO:tasks.workunit.client.0.smithi136.stdout:op 2778 completed, throughput=5MB/sec 2024-09-13T00:10:29.423 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-qZGDQO06BOxrPzb off=90685 len=127180 2024-09-13T00:10:29.428 INFO:tasks.workunit.client.0.smithi136.stdout:op 2779 completed, throughput=5MB/sec 2024-09-13T00:10:29.428 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_D2AVE20Kr2pcuQ off=3908958 len=1018289 2024-09-13T00:10:29.435 INFO:tasks.workunit.client.0.smithi136.stdout:op 2780 completed, throughput=5MB/sec 2024-09-13T00:10:29.435 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-eCiUjo2T1q4IVRs off=1308335 len=1848900 2024-09-13T00:10:29.449 INFO:tasks.workunit.client.0.smithi136.stdout:op 2781 completed, throughput=5MB/sec 2024-09-13T00:10:29.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:10:29 smithi181 bash[67587]: cluster 2024-09-13T00:10:27.418311+0000 mgr.y (mgr.24661) 1000 : cluster [DBG] pgmap v1683: 105 pgs: 105 active+clean; 126 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:10:29.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:10:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:10:29] "GET /metrics HTTP/1.1" 200 34142 "" "Prometheus/2.43.0" 2024-09-13T00:10:30.449 INFO:tasks.workunit.client.0.smithi136.stdout: 530: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:30.450 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-RNdohji3pX7NyPQ off=930046 len=494405 2024-09-13T00:10:30.452 INFO:tasks.workunit.client.0.smithi136.stdout:op 2782 completed, throughput=4.99MB/sec 2024-09-13T00:10:30.452 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-cM1Dfil8JEk56xF off=1364512 len=121695 2024-09-13T00:10:30.457 INFO:tasks.workunit.client.0.smithi136.stdout:op 2783 completed, throughput=4.99MB/sec 2024-09-13T00:10:30.457 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-TZlXHzsjUBP3zzH off=5099886 len=783677 2024-09-13T00:10:30.466 INFO:tasks.workunit.client.0.smithi136.stdout:op 2784 completed, throughput=4.99MB/sec 2024-09-13T00:10:30.466 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Mf-uZ_uu3drZiL1 off=14024423 len=1962269 2024-09-13T00:10:30.471 INFO:tasks.workunit.client.0.smithi136.stdout:op 2785 completed, throughput=5MB/sec 2024-09-13T00:10:30.471 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-XS5v7FTo-G_Vg01 off=13770679 len=1063374 2024-09-13T00:10:30.479 INFO:tasks.workunit.client.0.smithi136.stdout:op 2786 completed, throughput=5MB/sec 2024-09-13T00:10:30.480 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2RHrdLYLsdDvZaR off=10923683 len=572732 2024-09-13T00:10:30.481 INFO:tasks.workunit.client.0.smithi136.stdout:op 2787 completed, throughput=5MB/sec 2024-09-13T00:10:31.482 INFO:tasks.workunit.client.0.smithi136.stdout: 531: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:31.482 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-LNngOxA01-bF3Le off=9979961 len=240300 2024-09-13T00:10:31.487 INFO:tasks.workunit.client.0.smithi136.stdout:op 2788 completed, throughput=4.99MB/sec 2024-09-13T00:10:31.487 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-4ZyF4aYPGFxGU_4 off=2863281 len=1382093 2024-09-13T00:10:31.491 INFO:tasks.workunit.client.0.smithi136.stdout:op 2789 completed, throughput=4.99MB/sec 2024-09-13T00:10:31.491 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-729apXf9VU0ojSP off=7353975 len=1001055 2024-09-13T00:10:31.499 INFO:tasks.workunit.client.0.smithi136.stdout:op 2790 completed, throughput=5MB/sec 2024-09-13T00:10:31.499 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-fLBd2_xK_EThuZK off=3991304 len=285675 2024-09-13T00:10:31.501 INFO:tasks.workunit.client.0.smithi136.stdout:op 2791 completed, throughput=5MB/sec 2024-09-13T00:10:31.501 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ZUCOTuXgrJKobt3 off=11318201 len=515201 2024-09-13T00:10:31.507 INFO:tasks.workunit.client.0.smithi136.stdout:op 2792 completed, throughput=5MB/sec 2024-09-13T00:10:31.507 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-WyPPj4O35vDQys2 off=16857476 len=2069094 2024-09-13T00:10:31.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:10:31 smithi136 bash[76371]: cluster 2024-09-13T00:10:29.419222+0000 mgr.y (mgr.24661) 1001 : cluster [DBG] pgmap v1684: 105 pgs: 105 active+clean; 126 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:10:31.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:10:31 smithi136 bash[75164]: cluster 2024-09-13T00:10:29.419222+0000 mgr.y (mgr.24661) 1001 : cluster [DBG] pgmap v1684: 105 pgs: 105 active+clean; 126 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:10:31.512 INFO:tasks.workunit.client.0.smithi136.stdout:op 2793 completed, throughput=5MB/sec 2024-09-13T00:10:31.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:10:31 smithi181 bash[67587]: cluster 2024-09-13T00:10:29.419222+0000 mgr.y (mgr.24661) 1001 : cluster [DBG] pgmap v1684: 105 pgs: 105 active+clean; 126 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:10:32.512 INFO:tasks.workunit.client.0.smithi136.stdout: 532: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:32.512 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-6c0Fo3gaL9IotuY off=0 len=1183067 2024-09-13T00:10:32.522 INFO:tasks.workunit.client.0.smithi136.stdout:op 2794 completed, throughput=4.99MB/sec 2024-09-13T00:10:32.522 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-CZNAezQxf919IrL off=5629942 len=1752969 2024-09-13T00:10:32.536 INFO:tasks.workunit.client.0.smithi136.stdout:op 2795 completed, throughput=5MB/sec 2024-09-13T00:10:32.536 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-3i-5g79CH5G-_bK off=11408072 len=1888854 2024-09-13T00:10:32.539 INFO:tasks.workunit.client.0.smithi136.stdout:op 2796 completed, throughput=5MB/sec 2024-09-13T00:10:32.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:10:32 smithi181 bash[95505]: ts=2024-09-13T00:10:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:10:33.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:10:33 smithi136 bash[75164]: cluster 2024-09-13T00:10:31.420230+0000 mgr.y (mgr.24661) 1002 : cluster [DBG] pgmap v1685: 105 pgs: 105 active+clean; 126 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.2 MiB/s wr, 5 op/s 2024-09-13T00:10:33.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:10:33 smithi136 bash[76371]: cluster 2024-09-13T00:10:31.420230+0000 mgr.y (mgr.24661) 1002 : cluster [DBG] pgmap v1685: 105 pgs: 105 active+clean; 126 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.2 MiB/s wr, 5 op/s 2024-09-13T00:10:33.539 INFO:tasks.workunit.client.0.smithi136.stdout: 533: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:33.539 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-2Odl6s4fNF1u1bp off=4649881 len=204275 2024-09-13T00:10:33.544 INFO:tasks.workunit.client.0.smithi136.stdout:op 2797 completed, throughput=4.99MB/sec 2024-09-13T00:10:33.545 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-lzrs6jRWdtn4GTI off=20354855 len=475538 2024-09-13T00:10:33.546 INFO:tasks.workunit.client.0.smithi136.stdout:op 2798 completed, throughput=4.99MB/sec 2024-09-13T00:10:33.546 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-RHSeYuXGramGx4t off=4391203 len=359312 2024-09-13T00:10:33.551 INFO:tasks.workunit.client.0.smithi136.stdout:op 2799 completed, throughput=4.99MB/sec 2024-09-13T00:10:33.552 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-rOLUge0VeNXwffJ off=4141823 len=987679 2024-09-13T00:10:33.555 INFO:tasks.workunit.client.0.smithi136.stdout:op 2800 completed, throughput=4.99MB/sec 2024-09-13T00:10:33.556 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-dUmqiPGmsOthrbX off=8069622 len=836859 2024-09-13T00:10:33.563 INFO:tasks.workunit.client.0.smithi136.stdout:op 2801 completed, throughput=5MB/sec 2024-09-13T00:10:33.563 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-NflYgB7WZrXNA9V off=0 len=1281055 2024-09-13T00:10:33.574 INFO:tasks.workunit.client.0.smithi136.stdout:op 2802 completed, throughput=5MB/sec 2024-09-13T00:10:33.574 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ybUZyofNH6Ib_vC off=12085129 len=135860 2024-09-13T00:10:33.578 INFO:tasks.workunit.client.0.smithi136.stdout:op 2803 completed, throughput=5MB/sec 2024-09-13T00:10:33.578 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-cOop6P5LXvv9tyn off=6526853 len=1882220 2024-09-13T00:10:33.589 INFO:tasks.workunit.client.0.smithi136.stdout:op 2804 completed, throughput=5MB/sec 2024-09-13T00:10:33.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:10:33 smithi181 bash[67587]: cluster 2024-09-13T00:10:31.420230+0000 mgr.y (mgr.24661) 1002 : cluster [DBG] pgmap v1685: 105 pgs: 105 active+clean; 126 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.2 MiB/s wr, 5 op/s 2024-09-13T00:10:34.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:10:34 smithi136 bash[75164]: cluster 2024-09-13T00:10:33.421251+0000 mgr.y (mgr.24661) 1003 : cluster [DBG] pgmap v1686: 105 pgs: 105 active+clean; 126 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.2 MiB/s wr, 5 op/s 2024-09-13T00:10:34.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:10:34 smithi136 bash[76371]: cluster 2024-09-13T00:10:33.421251+0000 mgr.y (mgr.24661) 1003 : cluster [DBG] pgmap v1686: 105 pgs: 105 active+clean; 126 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.2 MiB/s wr, 5 op/s 2024-09-13T00:10:34.590 INFO:tasks.workunit.client.0.smithi136.stdout: 534: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:34.590 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-cpnBl4C5a0_GDVv off=6472097 len=1711146 2024-09-13T00:10:34.594 INFO:tasks.workunit.client.0.smithi136.stdout:op 2805 completed, throughput=5MB/sec 2024-09-13T00:10:34.594 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-1ndWVo6NQC6l1UV off=4443538 len=1106851 2024-09-13T00:10:34.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:10:34 smithi181 bash[67587]: cluster 2024-09-13T00:10:33.421251+0000 mgr.y (mgr.24661) 1003 : cluster [DBG] pgmap v1686: 105 pgs: 105 active+clean; 126 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.2 MiB/s wr, 5 op/s 2024-09-13T00:10:34.604 INFO:tasks.workunit.client.0.smithi136.stdout:op 2806 completed, throughput=5MB/sec 2024-09-13T00:10:34.604 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-njja-kpMEBq7Fmj off=3493260 len=609835 2024-09-13T00:10:34.606 INFO:tasks.workunit.client.0.smithi136.stdout:op 2807 completed, throughput=5MB/sec 2024-09-13T00:10:34.606 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-UNVkEqJKjFZ3X_k off=2530055 len=1828449 2024-09-13T00:10:34.624 INFO:tasks.workunit.client.0.smithi136.stdout:op 2808 completed, throughput=5MB/sec 2024-09-13T00:10:35.623 INFO:tasks.workunit.client.0.smithi136.stdout: 535: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:35.623 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-j78ZrepuwxVJyFW off=913168 len=833333 2024-09-13T00:10:35.625 INFO:tasks.workunit.client.0.smithi136.stdout:op 2809 completed, throughput=4.99MB/sec 2024-09-13T00:10:35.625 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-VEt9N9c3_kGztpz off=999931 len=1316822 2024-09-13T00:10:35.637 INFO:tasks.workunit.client.0.smithi136.stdout:op 2810 completed, throughput=5MB/sec 2024-09-13T00:10:35.637 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ShcddaHCyDssTdP off=2415745 len=64188 2024-09-13T00:10:35.641 INFO:tasks.workunit.client.0.smithi136.stdout:op 2811 completed, throughput=5MB/sec 2024-09-13T00:10:35.641 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-MV_B7NwmOI9d6Ct off=11824201 len=566213 2024-09-13T00:10:35.648 INFO:tasks.workunit.client.0.smithi136.stdout:op 2812 completed, throughput=5MB/sec 2024-09-13T00:10:35.648 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-RnG_6jwEXifkeJZ off=3225713 len=786878 2024-09-13T00:10:35.650 INFO:tasks.workunit.client.0.smithi136.stdout:op 2813 completed, throughput=5MB/sec 2024-09-13T00:10:35.651 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj--Q0Hj6IZu6X1UGK off=15501706 len=1439677 2024-09-13T00:10:35.654 INFO:tasks.workunit.client.0.smithi136.stdout:op 2814 completed, throughput=5MB/sec 2024-09-13T00:10:36.654 INFO:tasks.workunit.client.0.smithi136.stdout: 536: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:36.655 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-waWPa4iVE_UTdyq off=1844858 len=2043318 2024-09-13T00:10:36.659 INFO:tasks.workunit.client.0.smithi136.stdout:op 2815 completed, throughput=4.99MB/sec 2024-09-13T00:10:36.659 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-0vaozphrU-zHwsa off=4883555 len=616580 2024-09-13T00:10:36.666 INFO:tasks.workunit.client.0.smithi136.stdout:op 2816 completed, throughput=5MB/sec 2024-09-13T00:10:36.666 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-PUMhHX0aYq5eMEK off=3031304 len=1970428 2024-09-13T00:10:36.671 INFO:tasks.workunit.client.0.smithi136.stdout:op 2817 completed, throughput=5MB/sec 2024-09-13T00:10:36.671 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-QxW5pPo4hnRd-iG off=2770423 len=1949589 2024-09-13T00:10:36.675 INFO:tasks.workunit.client.0.smithi136.stdout:op 2818 completed, throughput=5MB/sec 2024-09-13T00:10:36.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:10:36 smithi136 bash[75164]: cluster 2024-09-13T00:10:35.422104+0000 mgr.y (mgr.24661) 1004 : cluster [DBG] pgmap v1687: 105 pgs: 105 active+clean; 126 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:10:36.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:10:36 smithi136 bash[76371]: cluster 2024-09-13T00:10:35.422104+0000 mgr.y (mgr.24661) 1004 : cluster [DBG] pgmap v1687: 105 pgs: 105 active+clean; 126 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:10:36.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:10:36 smithi181 bash[67587]: cluster 2024-09-13T00:10:35.422104+0000 mgr.y (mgr.24661) 1004 : cluster [DBG] pgmap v1687: 105 pgs: 105 active+clean; 126 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:10:37.675 INFO:tasks.workunit.client.0.smithi136.stdout: 537: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:37.676 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-LRIWvcwW69Tm9nW off=3042759 len=1274357 2024-09-13T00:10:37.679 INFO:tasks.workunit.client.0.smithi136.stdout:op 2819 completed, throughput=5MB/sec 2024-09-13T00:10:37.679 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-rICIEtpjYjmZTab off=4510667 len=1331 2024-09-13T00:10:37.680 INFO:tasks.workunit.client.0.smithi136.stdout:op 2820 completed, throughput=5MB/sec 2024-09-13T00:10:37.680 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-w9GhpNu37jJwjXM off=0 len=1242276 2024-09-13T00:10:37.683 INFO:tasks.workunit.client.0.smithi136.stdout:op 2821 completed, throughput=5MB/sec 2024-09-13T00:10:37.683 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-LfbfRLFsgSEhu94 off=3550031 len=108565 2024-09-13T00:10:37.684 INFO:tasks.workunit.client.0.smithi136.stdout:op 2822 completed, throughput=5MB/sec 2024-09-13T00:10:37.684 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-eea0xvAphBe7ZsU off=2247237 len=413091 2024-09-13T00:10:37.686 INFO:tasks.workunit.client.0.smithi136.stdout:op 2823 completed, throughput=5MB/sec 2024-09-13T00:10:37.686 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-TZQHUQS-hY_Lfgm off=8636172 len=805955 2024-09-13T00:10:37.693 INFO:tasks.workunit.client.0.smithi136.stdout:op 2824 completed, throughput=5MB/sec 2024-09-13T00:10:37.693 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-VCeFZJTtVLLNI1U off=15786963 len=1887414 2024-09-13T00:10:37.706 INFO:tasks.workunit.client.0.smithi136.stdout:op 2825 completed, throughput=5MB/sec 2024-09-13T00:10:38.706 INFO:tasks.workunit.client.0.smithi136.stdout: 538: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:38.706 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-KJv-HFtDOjW49P5 off=13258338 len=36416 2024-09-13T00:10:38.711 INFO:tasks.workunit.client.0.smithi136.stdout:op 2826 completed, throughput=4.99MB/sec 2024-09-13T00:10:38.711 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-dYNKZ8bWs3eZ5fC off=3671915 len=769314 2024-09-13T00:10:38.718 INFO:tasks.workunit.client.0.smithi136.stdout:op 2827 completed, throughput=5MB/sec 2024-09-13T00:10:38.719 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-CW0siSl9CU022fT off=211780 len=1849253 2024-09-13T00:10:38.731 INFO:tasks.workunit.client.0.smithi136.stdout:op 2828 completed, throughput=5MB/sec 2024-09-13T00:10:38.731 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ySPC5CozZvdON51 off=8491103 len=431355 2024-09-13T00:10:38.733 INFO:tasks.workunit.client.0.smithi136.stdout:op 2829 completed, throughput=5MB/sec 2024-09-13T00:10:38.733 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-nl_mqBLey-PKPkt off=10647657 len=1872571 2024-09-13T00:10:38.736 INFO:tasks.workunit.client.0.smithi136.stdout:op 2830 completed, throughput=5MB/sec 2024-09-13T00:10:38.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:10:38 smithi136 bash[75164]: cluster 2024-09-13T00:10:37.423169+0000 mgr.y (mgr.24661) 1005 : cluster [DBG] pgmap v1688: 105 pgs: 105 active+clean; 126 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:10:38.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:10:38 smithi136 bash[76371]: cluster 2024-09-13T00:10:37.423169+0000 mgr.y (mgr.24661) 1005 : cluster [DBG] pgmap v1688: 105 pgs: 105 active+clean; 126 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:10:38.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:10:38 smithi181 bash[67587]: cluster 2024-09-13T00:10:37.423169+0000 mgr.y (mgr.24661) 1005 : cluster [DBG] pgmap v1688: 105 pgs: 105 active+clean; 126 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:10:39.736 INFO:tasks.workunit.client.0.smithi136.stdout: 539: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:39.736 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-hj76XBIPySb5bqd off=346436 len=445887 2024-09-13T00:10:39.738 INFO:tasks.workunit.client.0.smithi136.stdout:op 2831 completed, throughput=4.99MB/sec 2024-09-13T00:10:39.738 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-BlSPTO7TABw3Gfp off=4561719 len=33694 2024-09-13T00:10:39.742 INFO:tasks.workunit.client.0.smithi136.stdout:op 2832 completed, throughput=4.99MB/sec 2024-09-13T00:10:39.742 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-vtE2n00RgS83wn2 off=9761291 len=1334713 2024-09-13T00:10:39.753 INFO:tasks.workunit.client.0.smithi136.stdout:op 2833 completed, throughput=5MB/sec 2024-09-13T00:10:39.754 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-4ZyF4aYPGFxGU_4 off=8447694 len=1049974 2024-09-13T00:10:39.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:10:39 smithi136 bash[75164]: audit 2024-09-13T00:10:38.787069+0000 mon.a (mon.0) 2186 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:10:39.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:10:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:10:39] "GET /metrics HTTP/1.1" 200 34139 "" "Prometheus/2.43.0" 2024-09-13T00:10:39.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:10:39 smithi136 bash[76371]: audit 2024-09-13T00:10:38.787069+0000 mon.a (mon.0) 2186 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:10:39.762 INFO:tasks.workunit.client.0.smithi136.stdout:op 2834 completed, throughput=5MB/sec 2024-09-13T00:10:39.762 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-KEsT41_etaWwII_ off=11844795 len=1955156 2024-09-13T00:10:39.776 INFO:tasks.workunit.client.0.smithi136.stdout:op 2835 completed, throughput=5MB/sec 2024-09-13T00:10:39.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:10:39 smithi181 bash[67587]: audit 2024-09-13T00:10:38.787069+0000 mon.a (mon.0) 2186 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:10:40.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:10:40 smithi136 bash[75164]: cluster 2024-09-13T00:10:39.424016+0000 mgr.y (mgr.24661) 1006 : cluster [DBG] pgmap v1689: 105 pgs: 105 active+clean; 126 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:10:40.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:10:40 smithi136 bash[76371]: cluster 2024-09-13T00:10:39.424016+0000 mgr.y (mgr.24661) 1006 : cluster [DBG] pgmap v1689: 105 pgs: 105 active+clean; 126 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:10:40.776 INFO:tasks.workunit.client.0.smithi136.stdout: 540: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:40.777 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-d9yAJaGTPTZagGw off=12257501 len=1202608 2024-09-13T00:10:40.780 INFO:tasks.workunit.client.0.smithi136.stdout:op 2836 completed, throughput=4.99MB/sec 2024-09-13T00:10:40.780 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-TnEGrVoYH5G0me0 off=114657 len=1388081 2024-09-13T00:10:40.791 INFO:tasks.workunit.client.0.smithi136.stdout:op 2837 completed, throughput=5MB/sec 2024-09-13T00:10:40.791 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-YFOEF43ChCKeZKu off=411754 len=584758 2024-09-13T00:10:40.798 INFO:tasks.workunit.client.0.smithi136.stdout:op 2838 completed, throughput=5MB/sec 2024-09-13T00:10:40.798 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-boJt5u6mYRjgnwa off=4865838 len=26339 2024-09-13T00:10:40.802 INFO:tasks.workunit.client.0.smithi136.stdout:op 2839 completed, throughput=5MB/sec 2024-09-13T00:10:40.802 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-HaZZ8XMN8gfYNab off=302326 len=1360740 2024-09-13T00:10:40.811 INFO:tasks.workunit.client.0.smithi136.stdout:op 2840 completed, throughput=5MB/sec 2024-09-13T00:10:40.811 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-K8e2em5kwqsfzmS off=1368856 len=322311 2024-09-13T00:10:40.812 INFO:tasks.workunit.client.0.smithi136.stdout:op 2841 completed, throughput=5MB/sec 2024-09-13T00:10:40.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:10:40 smithi181 bash[67587]: cluster 2024-09-13T00:10:39.424016+0000 mgr.y (mgr.24661) 1006 : cluster [DBG] pgmap v1689: 105 pgs: 105 active+clean; 126 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:10:41.812 INFO:tasks.workunit.client.0.smithi136.stdout: 541: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:41.813 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-f5hz1BaOxD_3Omw off=8480730 len=757238 2024-09-13T00:10:41.820 INFO:tasks.workunit.client.0.smithi136.stdout:op 2842 completed, throughput=4.99MB/sec 2024-09-13T00:10:41.820 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-UKeaekLPztLaBm1 off=7405538 len=1646962 2024-09-13T00:10:41.835 INFO:tasks.workunit.client.0.smithi136.stdout:op 2843 completed, throughput=5MB/sec 2024-09-13T00:10:41.835 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-adHVRbgxVcilRsL off=2246632 len=711948 2024-09-13T00:10:41.843 INFO:tasks.workunit.client.0.smithi136.stdout:op 2844 completed, throughput=5MB/sec 2024-09-13T00:10:41.843 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-sJZNr4gkKcBKvlM off=1096279 len=2081637 2024-09-13T00:10:41.849 INFO:tasks.workunit.client.0.smithi136.stdout:op 2845 completed, throughput=5MB/sec 2024-09-13T00:10:41.849 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-1AgZHgJbRGx57hq off=13082438 len=236701 2024-09-13T00:10:41.854 INFO:tasks.workunit.client.0.smithi136.stdout:op 2846 completed, throughput=5MB/sec 2024-09-13T00:10:42.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:10:42 smithi181 bash[95505]: ts=2024-09-13T00:10:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:10:42.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:10:42 smithi181 bash[67587]: cluster 2024-09-13T00:10:41.424994+0000 mgr.y (mgr.24661) 1007 : cluster [DBG] pgmap v1690: 105 pgs: 105 active+clean; 126 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:10:42.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:10:42 smithi136 bash[76371]: cluster 2024-09-13T00:10:41.424994+0000 mgr.y (mgr.24661) 1007 : cluster [DBG] pgmap v1690: 105 pgs: 105 active+clean; 126 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:10:42.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:10:42 smithi136 bash[75164]: cluster 2024-09-13T00:10:41.424994+0000 mgr.y (mgr.24661) 1007 : cluster [DBG] pgmap v1690: 105 pgs: 105 active+clean; 126 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:10:42.855 INFO:tasks.workunit.client.0.smithi136.stdout: 542: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:42.855 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-nz9K97G6hHgYarC off=13581670 len=799179 2024-09-13T00:10:42.863 INFO:tasks.workunit.client.0.smithi136.stdout:op 2847 completed, throughput=4.99MB/sec 2024-09-13T00:10:42.863 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-r8Fl5YNDBTztRH- off=11796103 len=808532 2024-09-13T00:10:42.865 INFO:tasks.workunit.client.0.smithi136.stdout:op 2848 completed, throughput=4.99MB/sec 2024-09-13T00:10:42.865 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Kt1XIBeTrZHHVGa off=5926144 len=507907 2024-09-13T00:10:42.868 INFO:tasks.workunit.client.0.smithi136.stdout:op 2849 completed, throughput=4.99MB/sec 2024-09-13T00:10:42.868 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-AMEFOz7hBTrLScd off=804079 len=236562 2024-09-13T00:10:42.873 INFO:tasks.workunit.client.0.smithi136.stdout:op 2850 completed, throughput=4.99MB/sec 2024-09-13T00:10:42.873 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-6S9PUWg0GawNPCd off=14844 len=1086723 2024-09-13T00:10:42.876 INFO:tasks.workunit.client.0.smithi136.stdout:op 2851 completed, throughput=5MB/sec 2024-09-13T00:10:42.876 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-rq3fTw94ec-bocV off=85890 len=1133643 2024-09-13T00:10:42.886 INFO:tasks.workunit.client.0.smithi136.stdout:op 2852 completed, throughput=5MB/sec 2024-09-13T00:10:42.886 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-w72icWw0VJ8lNry off=9781200 len=298858 2024-09-13T00:10:42.891 INFO:tasks.workunit.client.0.smithi136.stdout:op 2853 completed, throughput=5MB/sec 2024-09-13T00:10:42.891 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-u-vmKUkTB1Fi9Yw off=5163774 len=1078552 2024-09-13T00:10:42.895 INFO:tasks.workunit.client.0.smithi136.stdout:op 2854 completed, throughput=5MB/sec 2024-09-13T00:10:43.895 INFO:tasks.workunit.client.0.smithi136.stdout: 543: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:43.895 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-gQ4Vt18twhWpsyw off=2888099 len=1357274 2024-09-13T00:10:43.898 INFO:tasks.workunit.client.0.smithi136.stdout:op 2855 completed, throughput=4.99MB/sec 2024-09-13T00:10:43.898 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-pGmVgFWy-jlvEGB off=3833553 len=1994349 2024-09-13T00:10:43.903 INFO:tasks.workunit.client.0.smithi136.stdout:op 2856 completed, throughput=5MB/sec 2024-09-13T00:10:43.903 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-PEEew3i0zm8Fwew off=11894845 len=936836 2024-09-13T00:10:43.906 INFO:tasks.workunit.client.0.smithi136.stdout:op 2857 completed, throughput=5MB/sec 2024-09-13T00:10:43.906 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-szFlO_kfBcPmLD4 off=6729660 len=66609 2024-09-13T00:10:43.911 INFO:tasks.workunit.client.0.smithi136.stdout:op 2858 completed, throughput=5MB/sec 2024-09-13T00:10:43.911 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-5-pIJCBP-PmTXYC off=7044373 len=793855 2024-09-13T00:10:43.919 INFO:tasks.workunit.client.0.smithi136.stdout:op 2859 completed, throughput=5MB/sec 2024-09-13T00:10:44.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:10:44 smithi136 bash[75164]: cluster 2024-09-13T00:10:43.425886+0000 mgr.y (mgr.24661) 1008 : cluster [DBG] pgmap v1691: 105 pgs: 105 active+clean; 126 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:10:44.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:10:44 smithi136 bash[76371]: cluster 2024-09-13T00:10:43.425886+0000 mgr.y (mgr.24661) 1008 : cluster [DBG] pgmap v1691: 105 pgs: 105 active+clean; 126 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:10:44.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:10:44 smithi181 bash[67587]: cluster 2024-09-13T00:10:43.425886+0000 mgr.y (mgr.24661) 1008 : cluster [DBG] pgmap v1691: 105 pgs: 105 active+clean; 126 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:10:44.919 INFO:tasks.workunit.client.0.smithi136.stdout: 544: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:44.919 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-z38DmqTDvoy_eR_ off=12392204 len=1130853 2024-09-13T00:10:44.922 INFO:tasks.workunit.client.0.smithi136.stdout:op 2860 completed, throughput=4.99MB/sec 2024-09-13T00:10:44.922 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-HJhJJY7I75iaNge off=6584370 len=1150915 2024-09-13T00:10:44.932 INFO:tasks.workunit.client.0.smithi136.stdout:op 2861 completed, throughput=5MB/sec 2024-09-13T00:10:44.933 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-HO21fj9kSSColVa off=1179413 len=1817471 2024-09-13T00:10:44.946 INFO:tasks.workunit.client.0.smithi136.stdout:op 2862 completed, throughput=5MB/sec 2024-09-13T00:10:44.946 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-EucnKFTka9E3wKL off=773013 len=1893170 2024-09-13T00:10:44.961 INFO:tasks.workunit.client.0.smithi136.stdout:op 2863 completed, throughput=5MB/sec 2024-09-13T00:10:45.961 INFO:tasks.workunit.client.0.smithi136.stdout: 545: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:45.961 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-6rXusronOxuA9s- off=815119 len=663729 2024-09-13T00:10:45.963 INFO:tasks.workunit.client.0.smithi136.stdout:op 2864 completed, throughput=4.99MB/sec 2024-09-13T00:10:45.963 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-NS-6lxOiwgZVX-M off=2721241 len=1680181 2024-09-13T00:10:45.978 INFO:tasks.workunit.client.0.smithi136.stdout:op 2865 completed, throughput=5MB/sec 2024-09-13T00:10:45.978 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-N7slgrr1-XBYttq off=13451611 len=698284 2024-09-13T00:10:45.985 INFO:tasks.workunit.client.0.smithi136.stdout:op 2866 completed, throughput=5MB/sec 2024-09-13T00:10:45.985 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-m8j4m_TpF_iTqDH off=10511902 len=23346 2024-09-13T00:10:45.986 INFO:tasks.workunit.client.0.smithi136.stdout:op 2867 completed, throughput=5MB/sec 2024-09-13T00:10:45.987 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_-yHP28zv5nszYC off=4326846 len=1817507 2024-09-13T00:10:46.000 INFO:tasks.workunit.client.0.smithi136.stdout:op 2868 completed, throughput=5MB/sec 2024-09-13T00:10:46.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:10:46 smithi136 bash[75164]: cluster 2024-09-13T00:10:45.426719+0000 mgr.y (mgr.24661) 1009 : cluster [DBG] pgmap v1692: 105 pgs: 105 active+clean; 126 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:10:46.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:10:46 smithi136 bash[76371]: cluster 2024-09-13T00:10:45.426719+0000 mgr.y (mgr.24661) 1009 : cluster [DBG] pgmap v1692: 105 pgs: 105 active+clean; 126 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:10:46.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:10:46 smithi181 bash[67587]: cluster 2024-09-13T00:10:45.426719+0000 mgr.y (mgr.24661) 1009 : cluster [DBG] pgmap v1692: 105 pgs: 105 active+clean; 126 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:10:47.000 INFO:tasks.workunit.client.0.smithi136.stdout: 546: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:47.001 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-6dSHhH5zDI_4bjX off=2932215 len=1440127 2024-09-13T00:10:47.013 INFO:tasks.workunit.client.0.smithi136.stdout:op 2869 completed, throughput=4.99MB/sec 2024-09-13T00:10:47.013 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-8rypcKuxHLxGS-a off=2262367 len=1061040 2024-09-13T00:10:47.017 INFO:tasks.workunit.client.0.smithi136.stdout:op 2870 completed, throughput=5MB/sec 2024-09-13T00:10:47.017 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-jWBcN2xf8Owsqdq off=20737611 len=1940828 2024-09-13T00:10:47.029 INFO:tasks.workunit.client.0.smithi136.stdout:op 2871 completed, throughput=5MB/sec 2024-09-13T00:10:47.029 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-qJXi-MzQv6oQwNi off=3157094 len=587011 2024-09-13T00:10:47.031 INFO:tasks.workunit.client.0.smithi136.stdout:op 2872 completed, throughput=5MB/sec 2024-09-13T00:10:47.031 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-hm7BSrP6icqA6tw off=1963536 len=1598727 2024-09-13T00:10:47.043 INFO:tasks.workunit.client.0.smithi136.stdout:op 2873 completed, throughput=5MB/sec 2024-09-13T00:10:47.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:10:47 smithi136 bash[75164]: audit 2024-09-13T00:10:47.385312+0000 mon.a (mon.0) 2187 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:10:47.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:10:47 smithi136 bash[76371]: audit 2024-09-13T00:10:47.385312+0000 mon.a (mon.0) 2187 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:10:47.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:10:47 smithi181 bash[67587]: audit 2024-09-13T00:10:47.385312+0000 mon.a (mon.0) 2187 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:10:48.043 INFO:tasks.workunit.client.0.smithi136.stdout: 547: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:48.043 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-jiODoYafePNpxuf off=9166535 len=1275386 2024-09-13T00:10:48.047 INFO:tasks.workunit.client.0.smithi136.stdout:op 2874 completed, throughput=5MB/sec 2024-09-13T00:10:48.047 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-kExqDF65ZVOCZwj off=12440689 len=1110777 2024-09-13T00:10:48.055 INFO:tasks.workunit.client.0.smithi136.stdout:op 2875 completed, throughput=5MB/sec 2024-09-13T00:10:48.056 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-R0aPzGcVj_E3wZo off=7712503 len=1108012 2024-09-13T00:10:48.058 INFO:tasks.workunit.client.0.smithi136.stdout:op 2876 completed, throughput=5MB/sec 2024-09-13T00:10:48.059 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-vBDikl0bXfRPJY5 off=13243393 len=1946926 2024-09-13T00:10:48.063 INFO:tasks.workunit.client.0.smithi136.stdout:op 2877 completed, throughput=5MB/sec 2024-09-13T00:10:48.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:10:48 smithi181 bash[67587]: cluster 2024-09-13T00:10:47.427789+0000 mgr.y (mgr.24661) 1010 : cluster [DBG] pgmap v1693: 105 pgs: 105 active+clean; 126 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:10:49.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:10:48 smithi136 bash[75164]: cluster 2024-09-13T00:10:47.427789+0000 mgr.y (mgr.24661) 1010 : cluster [DBG] pgmap v1693: 105 pgs: 105 active+clean; 126 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:10:49.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:10:48 smithi136 bash[76371]: cluster 2024-09-13T00:10:47.427789+0000 mgr.y (mgr.24661) 1010 : cluster [DBG] pgmap v1693: 105 pgs: 105 active+clean; 126 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:10:49.063 INFO:tasks.workunit.client.0.smithi136.stdout: 548: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:49.063 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-zrsOHRjLE8iPYv0 off=1918786 len=1464595 2024-09-13T00:10:49.076 INFO:tasks.workunit.client.0.smithi136.stdout:op 2878 completed, throughput=5MB/sec 2024-09-13T00:10:49.076 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-XgS2w5whA9G6rOf off=1035755 len=433569 2024-09-13T00:10:49.078 INFO:tasks.workunit.client.0.smithi136.stdout:op 2879 completed, throughput=5MB/sec 2024-09-13T00:10:49.078 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-K8MtHhvbkbR29U3 off=1933904 len=799514 2024-09-13T00:10:49.087 INFO:tasks.workunit.client.0.smithi136.stdout:op 2880 completed, throughput=5MB/sec 2024-09-13T00:10:49.087 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-xDDxo5mFgqKDmz1 off=15612030 len=156591 2024-09-13T00:10:49.088 INFO:tasks.workunit.client.0.smithi136.stdout:op 2881 completed, throughput=5MB/sec 2024-09-13T00:10:49.088 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-IoqHS4cLmRsmOWD off=14797988 len=640221 2024-09-13T00:10:49.090 INFO:tasks.workunit.client.0.smithi136.stdout:op 2882 completed, throughput=5MB/sec 2024-09-13T00:10:49.090 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-m-VYOSS8_f9iZvg off=4563640 len=1701294 2024-09-13T00:10:49.094 INFO:tasks.workunit.client.0.smithi136.stdout:op 2883 completed, throughput=5MB/sec 2024-09-13T00:10:49.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:10:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:10:49] "GET /metrics HTTP/1.1" 200 34139 "" "Prometheus/2.43.0" 2024-09-13T00:10:50.095 INFO:tasks.workunit.client.0.smithi136.stdout: 549: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:50.095 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-vCJG0NvmkFmpIQM off=20728495 len=1805694 2024-09-13T00:10:50.099 INFO:tasks.workunit.client.0.smithi136.stdout:op 2884 completed, throughput=5MB/sec 2024-09-13T00:10:50.099 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-_8T7Js0VQ7iq7Dj off=3427370 len=703556 2024-09-13T00:10:50.101 INFO:tasks.workunit.client.0.smithi136.stdout:op 2885 completed, throughput=5MB/sec 2024-09-13T00:10:50.101 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-oJt9DNBuNNk4i5Q off=76844 len=602581 2024-09-13T00:10:50.108 INFO:tasks.workunit.client.0.smithi136.stdout:op 2886 completed, throughput=5MB/sec 2024-09-13T00:10:50.108 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-05UWFjT7FhycV9m off=6949243 len=1922600 2024-09-13T00:10:50.120 INFO:tasks.workunit.client.0.smithi136.stdout:op 2887 completed, throughput=5MB/sec 2024-09-13T00:10:50.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:10:50 smithi181 bash[67587]: cluster 2024-09-13T00:10:49.428678+0000 mgr.y (mgr.24661) 1011 : cluster [DBG] pgmap v1694: 105 pgs: 105 active+clean; 126 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:10:51.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:10:50 smithi136 bash[75164]: cluster 2024-09-13T00:10:49.428678+0000 mgr.y (mgr.24661) 1011 : cluster [DBG] pgmap v1694: 105 pgs: 105 active+clean; 126 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:10:51.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:10:50 smithi136 bash[76371]: cluster 2024-09-13T00:10:49.428678+0000 mgr.y (mgr.24661) 1011 : cluster [DBG] pgmap v1694: 105 pgs: 105 active+clean; 126 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:10:51.120 INFO:tasks.workunit.client.0.smithi136.stdout: 550: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:51.120 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-KEVeOyttHN872zi off=5234525 len=83910 2024-09-13T00:10:51.121 INFO:tasks.workunit.client.0.smithi136.stdout:op 2888 completed, throughput=4.99MB/sec 2024-09-13T00:10:51.121 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-HMKd7Esgq9eNWU3 off=3892773 len=459978 2024-09-13T00:10:51.128 INFO:tasks.workunit.client.0.smithi136.stdout:op 2889 completed, throughput=4.99MB/sec 2024-09-13T00:10:51.128 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-AkPWx636-GfB1vq off=11570535 len=477674 2024-09-13T00:10:51.135 INFO:tasks.workunit.client.0.smithi136.stdout:op 2890 completed, throughput=4.99MB/sec 2024-09-13T00:10:51.136 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-jdDt72bctfuJtIm off=12799446 len=54508 2024-09-13T00:10:51.137 INFO:tasks.workunit.client.0.smithi136.stdout:op 2891 completed, throughput=4.99MB/sec 2024-09-13T00:10:51.137 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-sfjugzBffXc7zcS off=18361018 len=1615382 2024-09-13T00:10:51.142 INFO:tasks.workunit.client.0.smithi136.stdout:op 2892 completed, throughput=5MB/sec 2024-09-13T00:10:51.142 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-PTrTDXyNSfnuzf0 off=4840286 len=212858 2024-09-13T00:10:51.147 INFO:tasks.workunit.client.0.smithi136.stdout:op 2893 completed, throughput=5MB/sec 2024-09-13T00:10:51.147 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-QprNxGFW_me8NI4 off=12260400 len=829273 2024-09-13T00:10:51.154 INFO:tasks.workunit.client.0.smithi136.stdout:op 2894 completed, throughput=5MB/sec 2024-09-13T00:10:51.154 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ahzzhxPhOJQkQHk off=2289398 len=55991 2024-09-13T00:10:51.155 INFO:tasks.workunit.client.0.smithi136.stdout:op 2895 completed, throughput=5MB/sec 2024-09-13T00:10:51.155 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-7y6R6erJPE4ZprR off=22730991 len=1402033 2024-09-13T00:10:51.159 INFO:tasks.workunit.client.0.smithi136.stdout:op 2896 completed, throughput=5MB/sec 2024-09-13T00:10:52.159 INFO:tasks.workunit.client.0.smithi136.stdout: 551: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:52.159 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-HcNHMSpDDKaiHAP off=4203697 len=1923172 2024-09-13T00:10:52.166 INFO:tasks.workunit.client.0.smithi136.stdout:op 2897 completed, throughput=5MB/sec 2024-09-13T00:10:52.166 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-5ylZtj-YyKi1Z-D off=4859895 len=2035840 2024-09-13T00:10:52.183 INFO:tasks.workunit.client.0.smithi136.stdout:op 2898 completed, throughput=5MB/sec 2024-09-13T00:10:52.183 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-oLuTdyqcH6urrPy off=7767299 len=1276925 2024-09-13T00:10:52.186 INFO:tasks.workunit.client.0.smithi136.stdout:op 2899 completed, throughput=5MB/sec 2024-09-13T00:10:52.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:10:52 smithi181 bash[67587]: cluster 2024-09-13T00:10:51.429679+0000 mgr.y (mgr.24661) 1012 : cluster [DBG] pgmap v1695: 105 pgs: 105 active+clean; 126 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:10:52.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:10:52 smithi181 bash[95505]: ts=2024-09-13T00:10:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:10:53.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:10:52 smithi136 bash[75164]: cluster 2024-09-13T00:10:51.429679+0000 mgr.y (mgr.24661) 1012 : cluster [DBG] pgmap v1695: 105 pgs: 105 active+clean; 126 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:10:53.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:10:52 smithi136 bash[76371]: cluster 2024-09-13T00:10:51.429679+0000 mgr.y (mgr.24661) 1012 : cluster [DBG] pgmap v1695: 105 pgs: 105 active+clean; 126 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:10:53.187 INFO:tasks.workunit.client.0.smithi136.stdout: 552: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:53.187 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-c6Rk9r3BxAbHCmm off=11583977 len=547549 2024-09-13T00:10:53.191 INFO:tasks.workunit.client.0.smithi136.stdout:op 2900 completed, throughput=4.99MB/sec 2024-09-13T00:10:53.192 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-mhAhLuueShX4ase off=8792325 len=340171 2024-09-13T00:10:53.193 INFO:tasks.workunit.client.0.smithi136.stdout:op 2901 completed, throughput=4.99MB/sec 2024-09-13T00:10:53.193 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-cBBrB-XjOm4Kiqj off=6822240 len=854426 2024-09-13T00:10:53.195 INFO:tasks.workunit.client.0.smithi136.stdout:op 2902 completed, throughput=5MB/sec 2024-09-13T00:10:53.195 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-L8JCXdcfki4YaYS off=2438079 len=1168849 2024-09-13T00:10:53.202 INFO:tasks.workunit.client.0.smithi136.stdout:op 2903 completed, throughput=5MB/sec 2024-09-13T00:10:53.202 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-aV4MwiEF43s3sOs off=1056020 len=1726353 2024-09-13T00:10:53.204 INFO:tasks.workunit.client.0.smithi136.stdout:op 2904 completed, throughput=5MB/sec 2024-09-13T00:10:53.204 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-kh9tt5UgefcVGi8 off=489765 len=783415 2024-09-13T00:10:53.210 INFO:tasks.workunit.client.0.smithi136.stdout:op 2905 completed, throughput=5MB/sec 2024-09-13T00:10:54.211 INFO:tasks.workunit.client.0.smithi136.stdout: 553: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:54.211 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-SJFV68lZxpOZ20X off=19367910 len=1659052 2024-09-13T00:10:54.222 INFO:tasks.workunit.client.0.smithi136.stdout:op 2906 completed, throughput=4.99MB/sec 2024-09-13T00:10:54.222 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-NBvzzlMtAF9b8Jp off=4491483 len=2069819 2024-09-13T00:10:54.237 INFO:tasks.workunit.client.0.smithi136.stdout:op 2907 completed, throughput=5MB/sec 2024-09-13T00:10:54.237 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-dIHYsl2IvAXEh3f off=9169385 len=408784 2024-09-13T00:10:54.238 INFO:tasks.workunit.client.0.smithi136.stdout:op 2908 completed, throughput=5MB/sec 2024-09-13T00:10:54.238 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-H_LE3WeigPQ3dHF off=9234421 len=1187617 2024-09-13T00:10:54.239 INFO:tasks.workunit.client.0.smithi136.stdout:op 2909 completed, throughput=5MB/sec 2024-09-13T00:10:54.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:10:53 smithi136 bash[75164]: audit 2024-09-13T00:10:52.865498+0000 mon.a (mon.0) 2188 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:10:54.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:10:53 smithi136 bash[75164]: audit 2024-09-13T00:10:52.870305+0000 mon.a (mon.0) 2189 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:10:54.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:10:53 smithi136 bash[75164]: audit 2024-09-13T00:10:53.085383+0000 mon.a (mon.0) 2190 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:10:54.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:10:53 smithi136 bash[75164]: audit 2024-09-13T00:10:53.094288+0000 mon.a (mon.0) 2191 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:10:54.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:10:53 smithi136 bash[75164]: audit 2024-09-13T00:10:53.517643+0000 mon.a (mon.0) 2192 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:10:54.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:10:53 smithi136 bash[75164]: audit 2024-09-13T00:10:53.519130+0000 mon.a (mon.0) 2193 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:10:54.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:10:53 smithi136 bash[75164]: audit 2024-09-13T00:10:53.528740+0000 mon.a (mon.0) 2194 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:10:54.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:10:53 smithi136 bash[75164]: audit 2024-09-13T00:10:53.787248+0000 mon.a (mon.0) 2195 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:10:54.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:10:53 smithi136 bash[76371]: audit 2024-09-13T00:10:52.865498+0000 mon.a (mon.0) 2188 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:10:54.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:10:53 smithi136 bash[76371]: audit 2024-09-13T00:10:52.870305+0000 mon.a (mon.0) 2189 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:10:54.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:10:53 smithi136 bash[76371]: audit 2024-09-13T00:10:53.085383+0000 mon.a (mon.0) 2190 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:10:54.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:10:53 smithi136 bash[76371]: audit 2024-09-13T00:10:53.094288+0000 mon.a (mon.0) 2191 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:10:54.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:10:53 smithi136 bash[76371]: audit 2024-09-13T00:10:53.517643+0000 mon.a (mon.0) 2192 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:10:54.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:10:53 smithi136 bash[76371]: audit 2024-09-13T00:10:53.519130+0000 mon.a (mon.0) 2193 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:10:54.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:10:53 smithi136 bash[76371]: audit 2024-09-13T00:10:53.528740+0000 mon.a (mon.0) 2194 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:10:54.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:10:53 smithi136 bash[76371]: audit 2024-09-13T00:10:53.787248+0000 mon.a (mon.0) 2195 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:10:54.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:10:53 smithi181 bash[67587]: audit 2024-09-13T00:10:52.865498+0000 mon.a (mon.0) 2188 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:10:54.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:10:53 smithi181 bash[67587]: audit 2024-09-13T00:10:52.870305+0000 mon.a (mon.0) 2189 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:10:54.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:10:53 smithi181 bash[67587]: audit 2024-09-13T00:10:53.085383+0000 mon.a (mon.0) 2190 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:10:54.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:10:53 smithi181 bash[67587]: audit 2024-09-13T00:10:53.094288+0000 mon.a (mon.0) 2191 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:10:54.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:10:53 smithi181 bash[67587]: audit 2024-09-13T00:10:53.517643+0000 mon.a (mon.0) 2192 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:10:54.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:10:53 smithi181 bash[67587]: audit 2024-09-13T00:10:53.519130+0000 mon.a (mon.0) 2193 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:10:54.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:10:53 smithi181 bash[67587]: audit 2024-09-13T00:10:53.528740+0000 mon.a (mon.0) 2194 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:10:54.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:10:53 smithi181 bash[67587]: audit 2024-09-13T00:10:53.787248+0000 mon.a (mon.0) 2195 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:10:55.240 INFO:tasks.workunit.client.0.smithi136.stdout: 554: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:55.240 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-B6QZZoHSkEWguTT off=3210368 len=628989 2024-09-13T00:10:55.247 INFO:tasks.workunit.client.0.smithi136.stdout:op 2910 completed, throughput=4.99MB/sec 2024-09-13T00:10:55.248 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-MxrXQG9nbh1GPK_ off=14613876 len=1498226 2024-09-13T00:10:55.259 INFO:tasks.workunit.client.0.smithi136.stdout:op 2911 completed, throughput=5MB/sec 2024-09-13T00:10:55.259 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-H0RWMW2CFcvA3M2 off=3540635 len=444100 2024-09-13T00:10:55.265 INFO:tasks.workunit.client.0.smithi136.stdout:op 2912 completed, throughput=5MB/sec 2024-09-13T00:10:55.265 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-YQyAgwrtA5K_0ku off=4284754 len=653740 2024-09-13T00:10:55.267 INFO:tasks.workunit.client.0.smithi136.stdout:op 2913 completed, throughput=5MB/sec 2024-09-13T00:10:55.267 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-y5ZH9TLHUgvKYn4 off=9969255 len=1802760 2024-09-13T00:10:55.272 INFO:tasks.workunit.client.0.smithi136.stdout:op 2914 completed, throughput=5MB/sec 2024-09-13T00:10:55.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:10:55 smithi181 bash[67587]: cluster 2024-09-13T00:10:53.430613+0000 mgr.y (mgr.24661) 1013 : cluster [DBG] pgmap v1696: 105 pgs: 105 active+clean; 126 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 2.3 MiB/s rd, 3.1 MiB/s wr, 5 op/s 2024-09-13T00:10:55.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:10:55 smithi136 bash[75164]: cluster 2024-09-13T00:10:53.430613+0000 mgr.y (mgr.24661) 1013 : cluster [DBG] pgmap v1696: 105 pgs: 105 active+clean; 126 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 2.3 MiB/s rd, 3.1 MiB/s wr, 5 op/s 2024-09-13T00:10:55.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:10:55 smithi136 bash[76371]: cluster 2024-09-13T00:10:53.430613+0000 mgr.y (mgr.24661) 1013 : cluster [DBG] pgmap v1696: 105 pgs: 105 active+clean; 126 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 2.3 MiB/s rd, 3.1 MiB/s wr, 5 op/s 2024-09-13T00:10:56.272 INFO:tasks.workunit.client.0.smithi136.stdout: 555: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:56.272 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-1M2loYJA79HhBF2 off=1264945 len=1257628 2024-09-13T00:10:56.276 INFO:tasks.workunit.client.0.smithi136.stdout:op 2915 completed, throughput=4.99MB/sec 2024-09-13T00:10:56.276 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-5_ze_3wZZy87X4X off=851823 len=956584 2024-09-13T00:10:56.285 INFO:tasks.workunit.client.0.smithi136.stdout:op 2916 completed, throughput=5MB/sec 2024-09-13T00:10:56.285 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-v6XdtKklpnLVocT off=20270534 len=956926 2024-09-13T00:10:56.288 INFO:tasks.workunit.client.0.smithi136.stdout:op 2917 completed, throughput=5MB/sec 2024-09-13T00:10:56.288 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-jqi3D8r8Mj6Su2n off=2597163 len=431775 2024-09-13T00:10:56.290 INFO:tasks.workunit.client.0.smithi136.stdout:op 2918 completed, throughput=5MB/sec 2024-09-13T00:10:56.290 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-mp4d0ntf2pEqA7V off=14625081 len=1268981 2024-09-13T00:10:56.293 INFO:tasks.workunit.client.0.smithi136.stdout:op 2919 completed, throughput=5MB/sec 2024-09-13T00:10:56.293 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-KJLsotRb8Z7gFek off=3749264 len=1232329 2024-09-13T00:10:56.297 INFO:tasks.workunit.client.0.smithi136.stdout:op 2920 completed, throughput=5MB/sec 2024-09-13T00:10:57.297 INFO:tasks.workunit.client.0.smithi136.stdout: 556: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:57.298 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-sh__BE-f47m8g1Q off=409628 len=210785 2024-09-13T00:10:57.303 INFO:tasks.workunit.client.0.smithi136.stdout:op 2921 completed, throughput=4.99MB/sec 2024-09-13T00:10:57.303 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-94QfvvcYqt3Mhlf off=3836235 len=1336601 2024-09-13T00:10:57.316 INFO:tasks.workunit.client.0.smithi136.stdout:op 2922 completed, throughput=5MB/sec 2024-09-13T00:10:57.316 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-_mS3VqyffNx8Ld3 off=1334703 len=715400 2024-09-13T00:10:57.318 INFO:tasks.workunit.client.0.smithi136.stdout:op 2923 completed, throughput=5MB/sec 2024-09-13T00:10:57.318 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-7rEjTG2meLCpmVR off=2201842 len=1745949 2024-09-13T00:10:57.322 INFO:tasks.workunit.client.0.smithi136.stdout:op 2924 completed, throughput=5MB/sec 2024-09-13T00:10:57.322 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-kp-6jz1hGdnBXHI off=1369396 len=1177332 2024-09-13T00:10:57.325 INFO:tasks.workunit.client.0.smithi136.stdout:op 2925 completed, throughput=5MB/sec 2024-09-13T00:10:57.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:10:57 smithi181 bash[67587]: cluster 2024-09-13T00:10:55.431451+0000 mgr.y (mgr.24661) 1014 : cluster [DBG] pgmap v1697: 105 pgs: 105 active+clean; 126 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 2.5 MiB/s rd, 3.0 MiB/s wr, 5 op/s 2024-09-13T00:10:57.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:10:57 smithi136 bash[75164]: cluster 2024-09-13T00:10:55.431451+0000 mgr.y (mgr.24661) 1014 : cluster [DBG] pgmap v1697: 105 pgs: 105 active+clean; 126 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 2.5 MiB/s rd, 3.0 MiB/s wr, 5 op/s 2024-09-13T00:10:57.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:10:57 smithi136 bash[76371]: cluster 2024-09-13T00:10:55.431451+0000 mgr.y (mgr.24661) 1014 : cluster [DBG] pgmap v1697: 105 pgs: 105 active+clean; 126 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 2.5 MiB/s rd, 3.0 MiB/s wr, 5 op/s 2024-09-13T00:10:58.326 INFO:tasks.workunit.client.0.smithi136.stdout: 557: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:58.326 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-JWGrijIzOpp9vG7 off=458318 len=1291241 2024-09-13T00:10:58.336 INFO:tasks.workunit.client.0.smithi136.stdout:op 2926 completed, throughput=4.99MB/sec 2024-09-13T00:10:58.336 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Fpt-wqvdNlKw6Yd off=7141288 len=1110969 2024-09-13T00:10:58.339 INFO:tasks.workunit.client.0.smithi136.stdout:op 2927 completed, throughput=5MB/sec 2024-09-13T00:10:58.339 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-tRUa1xQATbtWKdh off=5323338 len=967359 2024-09-13T00:10:58.347 INFO:tasks.workunit.client.0.smithi136.stdout:op 2928 completed, throughput=5MB/sec 2024-09-13T00:10:58.347 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-lNZSF3P7ajYiupk off=5438155 len=1221174 2024-09-13T00:10:58.351 INFO:tasks.workunit.client.0.smithi136.stdout:op 2929 completed, throughput=5MB/sec 2024-09-13T00:10:58.351 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-2o0QSmHgQHpJwAV off=2843768 len=245235 2024-09-13T00:10:58.357 INFO:tasks.workunit.client.0.smithi136.stdout:op 2930 completed, throughput=5MB/sec 2024-09-13T00:10:59.357 INFO:tasks.workunit.client.0.smithi136.stdout: 558: throughput=4.99MB/sec pending data=0 2024-09-13T00:10:59.357 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-7xVggD2qlmt4qn- off=12978998 len=1996317 2024-09-13T00:10:59.362 INFO:tasks.workunit.client.0.smithi136.stdout:op 2931 completed, throughput=4.99MB/sec 2024-09-13T00:10:59.362 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-J-V6cc1il2qS6ES off=4193273 len=1031844 2024-09-13T00:10:59.372 INFO:tasks.workunit.client.0.smithi136.stdout:op 2932 completed, throughput=5MB/sec 2024-09-13T00:10:59.372 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Gcfr3vEaccFaMG3 off=6421610 len=61341 2024-09-13T00:10:59.373 INFO:tasks.workunit.client.0.smithi136.stdout:op 2933 completed, throughput=5MB/sec 2024-09-13T00:10:59.373 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-TNx4TJetQIL24AA off=15500649 len=141502 2024-09-13T00:10:59.377 INFO:tasks.workunit.client.0.smithi136.stdout:op 2934 completed, throughput=5MB/sec 2024-09-13T00:10:59.377 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-hCWotdwq7j1bUjX off=14343444 len=796436 2024-09-13T00:10:59.383 INFO:tasks.workunit.client.0.smithi136.stdout:op 2935 completed, throughput=5MB/sec 2024-09-13T00:10:59.384 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-4GpNEurykf4eDIt off=10076130 len=617021 2024-09-13T00:10:59.387 INFO:tasks.workunit.client.0.smithi136.stdout:op 2936 completed, throughput=5MB/sec 2024-09-13T00:10:59.387 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Y_Dmsc9KK_5uWXT off=13240503 len=480898 2024-09-13T00:10:59.393 INFO:tasks.workunit.client.0.smithi136.stdout:op 2937 completed, throughput=5MB/sec 2024-09-13T00:10:59.393 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-M3-DLZVOAF6qGO0 off=338509 len=934370 2024-09-13T00:10:59.402 INFO:tasks.workunit.client.0.smithi136.stdout:op 2938 completed, throughput=5MB/sec 2024-09-13T00:10:59.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:10:59 smithi136 bash[75164]: cluster 2024-09-13T00:10:57.432581+0000 mgr.y (mgr.24661) 1015 : cluster [DBG] pgmap v1698: 105 pgs: 105 active+clean; 126 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:10:59.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:10:59 smithi136 bash[76371]: cluster 2024-09-13T00:10:57.432581+0000 mgr.y (mgr.24661) 1015 : cluster [DBG] pgmap v1698: 105 pgs: 105 active+clean; 126 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:10:59.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:10:59 smithi181 bash[67587]: cluster 2024-09-13T00:10:57.432581+0000 mgr.y (mgr.24661) 1015 : cluster [DBG] pgmap v1698: 105 pgs: 105 active+clean; 126 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:10:59.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:10:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:10:59] "GET /metrics HTTP/1.1" 200 34137 "" "Prometheus/2.43.0" 2024-09-13T00:11:00.403 INFO:tasks.workunit.client.0.smithi136.stdout: 559: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:00.403 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-yggiKNOt7spz4Bn off=2748291 len=1662601 2024-09-13T00:11:00.406 INFO:tasks.workunit.client.0.smithi136.stdout:op 2939 completed, throughput=5MB/sec 2024-09-13T00:11:00.407 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-iJqHuProgoYFz7U off=6972532 len=2027960 2024-09-13T00:11:00.411 INFO:tasks.workunit.client.0.smithi136.stdout:op 2940 completed, throughput=5MB/sec 2024-09-13T00:11:00.411 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-9NDvHNEyVwMbIft off=3532084 len=1356660 2024-09-13T00:11:00.421 INFO:tasks.workunit.client.0.smithi136.stdout:op 2941 completed, throughput=5MB/sec 2024-09-13T00:11:01.421 INFO:tasks.workunit.client.0.smithi136.stdout: 560: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:01.421 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-POLsLV8H5JHL6EB off=893695 len=1988379 2024-09-13T00:11:01.427 INFO:tasks.workunit.client.0.smithi136.stdout:op 2942 completed, throughput=5MB/sec 2024-09-13T00:11:01.427 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-KCLuAOuXHphZdhF off=3272905 len=693156 2024-09-13T00:11:01.434 INFO:tasks.workunit.client.0.smithi136.stdout:op 2943 completed, throughput=5MB/sec 2024-09-13T00:11:01.434 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-IMntVfPgjJcRykP off=3870531 len=2052673 2024-09-13T00:11:01.439 INFO:tasks.workunit.client.0.smithi136.stdout:op 2944 completed, throughput=5MB/sec 2024-09-13T00:11:01.439 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-PPuggrfRhuoVAyo off=125057 len=437375 2024-09-13T00:11:01.444 INFO:tasks.workunit.client.0.smithi136.stdout:op 2945 completed, throughput=5MB/sec 2024-09-13T00:11:01.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:11:01 smithi136 bash[75164]: cluster 2024-09-13T00:10:59.433461+0000 mgr.y (mgr.24661) 1016 : cluster [DBG] pgmap v1699: 105 pgs: 105 active+clean; 126 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:11:01.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:11:01 smithi136 bash[76371]: cluster 2024-09-13T00:10:59.433461+0000 mgr.y (mgr.24661) 1016 : cluster [DBG] pgmap v1699: 105 pgs: 105 active+clean; 126 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:11:01.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:11:01 smithi181 bash[67587]: cluster 2024-09-13T00:10:59.433461+0000 mgr.y (mgr.24661) 1016 : cluster [DBG] pgmap v1699: 105 pgs: 105 active+clean; 126 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:11:02.445 INFO:tasks.workunit.client.0.smithi136.stdout: 561: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:02.445 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-cXOmC8HY4AKfEnC off=2415842 len=1571882 2024-09-13T00:11:02.455 INFO:tasks.workunit.client.0.smithi136.stdout:op 2946 completed, throughput=4.99MB/sec 2024-09-13T00:11:02.456 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Ppu1b2wQR1RzF0O off=2911497 len=882377 2024-09-13T00:11:02.458 INFO:tasks.workunit.client.0.smithi136.stdout:op 2947 completed, throughput=5MB/sec 2024-09-13T00:11:02.458 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-EwY6u70WqkHL0JP off=3686359 len=253596 2024-09-13T00:11:02.463 INFO:tasks.workunit.client.0.smithi136.stdout:op 2948 completed, throughput=5MB/sec 2024-09-13T00:11:02.463 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-FDGPny9nNPJWoax off=16771896 len=1535639 2024-09-13T00:11:02.465 INFO:tasks.workunit.client.0.smithi136.stdout:op 2949 completed, throughput=5MB/sec 2024-09-13T00:11:02.465 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-XljjUqB_fmnQMgP off=6342318 len=1910812 2024-09-13T00:11:02.469 INFO:tasks.workunit.client.0.smithi136.stdout:op 2950 completed, throughput=5MB/sec 2024-09-13T00:11:02.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:11:02 smithi181 bash[95505]: ts=2024-09-13T00:11:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:11:03.470 INFO:tasks.workunit.client.0.smithi136.stdout: 562: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:03.470 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-LMU5_GDDBhdFJyr off=11568525 len=1583933 2024-09-13T00:11:03.473 INFO:tasks.workunit.client.0.smithi136.stdout:op 2951 completed, throughput=5MB/sec 2024-09-13T00:11:03.474 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-3EESrycVt1FJIj9 off=2884778 len=677101 2024-09-13T00:11:03.481 INFO:tasks.workunit.client.0.smithi136.stdout:op 2952 completed, throughput=5MB/sec 2024-09-13T00:11:03.481 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj--LFvj99BZf53C4e off=922787 len=1164342 2024-09-13T00:11:03.493 INFO:tasks.workunit.client.0.smithi136.stdout:op 2953 completed, throughput=5MB/sec 2024-09-13T00:11:03.493 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-cML8wClb6_Phvd6 off=10971664 len=1303781 2024-09-13T00:11:03.505 INFO:tasks.workunit.client.0.smithi136.stdout:op 2954 completed, throughput=5MB/sec 2024-09-13T00:11:03.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:11:03 smithi136 bash[75164]: cluster 2024-09-13T00:11:01.434444+0000 mgr.y (mgr.24661) 1017 : cluster [DBG] pgmap v1700: 105 pgs: 105 active+clean; 126 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:11:03.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:11:03 smithi136 bash[76371]: cluster 2024-09-13T00:11:01.434444+0000 mgr.y (mgr.24661) 1017 : cluster [DBG] pgmap v1700: 105 pgs: 105 active+clean; 126 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:11:03.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:11:03 smithi181 bash[67587]: cluster 2024-09-13T00:11:01.434444+0000 mgr.y (mgr.24661) 1017 : cluster [DBG] pgmap v1700: 105 pgs: 105 active+clean; 126 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:11:04.505 INFO:tasks.workunit.client.0.smithi136.stdout: 563: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:04.505 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-m3YB2k7q6fI990C off=6810157 len=855627 2024-09-13T00:11:04.514 INFO:tasks.workunit.client.0.smithi136.stdout:op 2955 completed, throughput=4.99MB/sec 2024-09-13T00:11:04.514 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-lHtsxPsV8DXy2mA off=13847635 len=1275002 2024-09-13T00:11:04.519 INFO:tasks.workunit.client.0.smithi136.stdout:op 2956 completed, throughput=5MB/sec 2024-09-13T00:11:04.519 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-kDGrLV0v-uHFOvM off=1036893 len=656395 2024-09-13T00:11:04.521 INFO:tasks.workunit.client.0.smithi136.stdout:op 2957 completed, throughput=5MB/sec 2024-09-13T00:11:04.521 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-TfXkiyItuMCmwZO off=4608270 len=1063623 2024-09-13T00:11:04.530 INFO:tasks.workunit.client.0.smithi136.stdout:op 2958 completed, throughput=5MB/sec 2024-09-13T00:11:04.531 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-N6Ebzcln6J7rZ-K off=3912791 len=719020 2024-09-13T00:11:04.539 INFO:tasks.workunit.client.0.smithi136.stdout:op 2959 completed, throughput=5MB/sec 2024-09-13T00:11:04.539 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-uUOBlvLr7hijlwZ off=14139222 len=16700 2024-09-13T00:11:04.540 INFO:tasks.workunit.client.0.smithi136.stdout:op 2960 completed, throughput=5MB/sec 2024-09-13T00:11:04.540 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-r-WjkElaVJgvjj2 off=417571 len=99357 2024-09-13T00:11:04.541 INFO:tasks.workunit.client.0.smithi136.stdout:op 2961 completed, throughput=5MB/sec 2024-09-13T00:11:04.541 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-3_pOJiYVjEX7ykQ off=1215330 len=864997 2024-09-13T00:11:04.550 INFO:tasks.workunit.client.0.smithi136.stdout:op 2962 completed, throughput=5MB/sec 2024-09-13T00:11:05.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:11:05 smithi136 bash[75164]: cluster 2024-09-13T00:11:03.435375+0000 mgr.y (mgr.24661) 1018 : cluster [DBG] pgmap v1701: 105 pgs: 105 active+clean; 126 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.1 MiB/s wr, 5 op/s 2024-09-13T00:11:05.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:11:05 smithi136 bash[76371]: cluster 2024-09-13T00:11:03.435375+0000 mgr.y (mgr.24661) 1018 : cluster [DBG] pgmap v1701: 105 pgs: 105 active+clean; 126 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.1 MiB/s wr, 5 op/s 2024-09-13T00:11:05.550 INFO:tasks.workunit.client.0.smithi136.stdout: 564: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:05.550 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-nC4OLO5ma6CXvc5 off=7581768 len=1104785 2024-09-13T00:11:05.558 INFO:tasks.workunit.client.0.smithi136.stdout:op 2963 completed, throughput=4.99MB/sec 2024-09-13T00:11:05.558 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-fbzZVYJQqs5Bl7d off=8300881 len=1303554 2024-09-13T00:11:05.560 INFO:tasks.workunit.client.0.smithi136.stdout:op 2964 completed, throughput=5MB/sec 2024-09-13T00:11:05.560 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-H0QdDHHeicM-9X4 off=350910 len=1821940 2024-09-13T00:11:05.573 INFO:tasks.workunit.client.0.smithi136.stdout:op 2965 completed, throughput=5MB/sec 2024-09-13T00:11:05.573 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-0FcSZ-i4MdwdCXP off=420531 len=486432 2024-09-13T00:11:05.579 INFO:tasks.workunit.client.0.smithi136.stdout:op 2966 completed, throughput=5MB/sec 2024-09-13T00:11:05.579 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-vlFtZPNMfKV8bgo off=2551144 len=2059919 2024-09-13T00:11:05.594 INFO:tasks.workunit.client.0.smithi136.stdout:op 2967 completed, throughput=5MB/sec 2024-09-13T00:11:05.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:11:05 smithi181 bash[67587]: cluster 2024-09-13T00:11:03.435375+0000 mgr.y (mgr.24661) 1018 : cluster [DBG] pgmap v1701: 105 pgs: 105 active+clean; 126 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.1 MiB/s wr, 5 op/s 2024-09-13T00:11:06.594 INFO:tasks.workunit.client.0.smithi136.stdout: 565: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:06.594 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-q4xBcCP6EjSYP4U off=12225091 len=1909078 2024-09-13T00:11:06.606 INFO:tasks.workunit.client.0.smithi136.stdout:op 2968 completed, throughput=5MB/sec 2024-09-13T00:11:06.606 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-BdxiR1zH0robcvA off=1792936 len=1505529 2024-09-13T00:11:06.618 INFO:tasks.workunit.client.0.smithi136.stdout:op 2969 completed, throughput=5MB/sec 2024-09-13T00:11:06.618 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-vSjD7_pETLmcmle off=11566422 len=841280 2024-09-13T00:11:06.621 INFO:tasks.workunit.client.0.smithi136.stdout:op 2970 completed, throughput=5MB/sec 2024-09-13T00:11:07.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:11:07 smithi136 bash[75164]: cluster 2024-09-13T00:11:05.436227+0000 mgr.y (mgr.24661) 1019 : cluster [DBG] pgmap v1702: 105 pgs: 105 active+clean; 126 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.9 MiB/s wr, 5 op/s 2024-09-13T00:11:07.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:11:07 smithi136 bash[76371]: cluster 2024-09-13T00:11:05.436227+0000 mgr.y (mgr.24661) 1019 : cluster [DBG] pgmap v1702: 105 pgs: 105 active+clean; 126 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.9 MiB/s wr, 5 op/s 2024-09-13T00:11:07.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:11:07 smithi181 bash[67587]: cluster 2024-09-13T00:11:05.436227+0000 mgr.y (mgr.24661) 1019 : cluster [DBG] pgmap v1702: 105 pgs: 105 active+clean; 126 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.9 MiB/s wr, 5 op/s 2024-09-13T00:11:07.621 INFO:tasks.workunit.client.0.smithi136.stdout: 566: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:07.621 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-QC8SMVc_LGKEIRI off=22435142 len=488281 2024-09-13T00:11:07.623 INFO:tasks.workunit.client.0.smithi136.stdout:op 2971 completed, throughput=4.99MB/sec 2024-09-13T00:11:07.623 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_kt9Se9E8Qo3vbj off=15914931 len=1654420 2024-09-13T00:11:07.635 INFO:tasks.workunit.client.0.smithi136.stdout:op 2972 completed, throughput=5MB/sec 2024-09-13T00:11:07.635 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-qYoMvyDSzeOs0cF off=4170294 len=433908 2024-09-13T00:11:07.640 INFO:tasks.workunit.client.0.smithi136.stdout:op 2973 completed, throughput=5MB/sec 2024-09-13T00:11:07.640 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-G5x55CflBIEAZcr off=12817451 len=1083169 2024-09-13T00:11:07.649 INFO:tasks.workunit.client.0.smithi136.stdout:op 2974 completed, throughput=5MB/sec 2024-09-13T00:11:07.649 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-IpqFd_eGDhxeJzQ off=8600232 len=621863 2024-09-13T00:11:07.651 INFO:tasks.workunit.client.0.smithi136.stdout:op 2975 completed, throughput=5MB/sec 2024-09-13T00:11:07.651 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-tOF4BhZ1jOKUVcH off=4286748 len=1966002 2024-09-13T00:11:07.657 INFO:tasks.workunit.client.0.smithi136.stdout:op 2976 completed, throughput=5MB/sec 2024-09-13T00:11:08.658 INFO:tasks.workunit.client.0.smithi136.stdout: 567: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:08.658 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-B0FVdxf-AXCg2DN off=9382520 len=939785 2024-09-13T00:11:08.661 INFO:tasks.workunit.client.0.smithi136.stdout:op 2977 completed, throughput=5MB/sec 2024-09-13T00:11:08.661 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-7NsN2fXwBRAdEl6 off=2718436 len=1028713 2024-09-13T00:11:08.664 INFO:tasks.workunit.client.0.smithi136.stdout:op 2978 completed, throughput=5MB/sec 2024-09-13T00:11:08.664 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-GkllNmJpDrzt_VW off=2053659 len=1838154 2024-09-13T00:11:08.677 INFO:tasks.workunit.client.0.smithi136.stdout:op 2979 completed, throughput=5MB/sec 2024-09-13T00:11:08.677 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-O3X68-m1TdAoe_a off=776337 len=1317942 2024-09-13T00:11:08.679 INFO:tasks.workunit.client.0.smithi136.stdout:op 2980 completed, throughput=5MB/sec 2024-09-13T00:11:09.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:11:09 smithi136 bash[76371]: cluster 2024-09-13T00:11:07.437385+0000 mgr.y (mgr.24661) 1020 : cluster [DBG] pgmap v1703: 105 pgs: 105 active+clean; 126 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:11:09.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:11:09 smithi136 bash[76371]: audit 2024-09-13T00:11:08.787695+0000 mon.a (mon.0) 2196 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:11:09.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:11:09 smithi136 bash[75164]: cluster 2024-09-13T00:11:07.437385+0000 mgr.y (mgr.24661) 1020 : cluster [DBG] pgmap v1703: 105 pgs: 105 active+clean; 126 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:11:09.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:11:09 smithi136 bash[75164]: audit 2024-09-13T00:11:08.787695+0000 mon.a (mon.0) 2196 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:11:09.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:11:09 smithi181 bash[67587]: cluster 2024-09-13T00:11:07.437385+0000 mgr.y (mgr.24661) 1020 : cluster [DBG] pgmap v1703: 105 pgs: 105 active+clean; 126 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:11:09.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:11:09 smithi181 bash[67587]: audit 2024-09-13T00:11:08.787695+0000 mon.a (mon.0) 2196 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:11:09.679 INFO:tasks.workunit.client.0.smithi136.stdout: 568: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:09.679 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-s6NFAXfvWQn8YCY off=4089693 len=807367 2024-09-13T00:11:09.686 INFO:tasks.workunit.client.0.smithi136.stdout:op 2981 completed, throughput=4.99MB/sec 2024-09-13T00:11:09.686 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-xRdiLimoxt0osKJ off=10007113 len=770988 2024-09-13T00:11:09.693 INFO:tasks.workunit.client.0.smithi136.stdout:op 2982 completed, throughput=5MB/sec 2024-09-13T00:11:09.693 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-S0qpLtVdffxsDJR off=15635892 len=26350 2024-09-13T00:11:09.697 INFO:tasks.workunit.client.0.smithi136.stdout:op 2983 completed, throughput=5MB/sec 2024-09-13T00:11:09.697 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-E6SNZ6benec9zRx off=6791709 len=343623 2024-09-13T00:11:09.699 INFO:tasks.workunit.client.0.smithi136.stdout:op 2984 completed, throughput=5MB/sec 2024-09-13T00:11:09.699 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-hj76XBIPySb5bqd off=0 len=1516301 2024-09-13T00:11:09.713 INFO:tasks.workunit.client.0.smithi136.stdout:op 2985 completed, throughput=5MB/sec 2024-09-13T00:11:09.713 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-4ZyF4aYPGFxGU_4 off=13251733 len=129394 2024-09-13T00:11:09.714 INFO:tasks.workunit.client.0.smithi136.stdout:op 2986 completed, throughput=5MB/sec 2024-09-13T00:11:09.714 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-rq3fTw94ec-bocV off=1216130 len=737579 2024-09-13T00:11:09.715 INFO:tasks.workunit.client.0.smithi136.stdout:op 2987 completed, throughput=5MB/sec 2024-09-13T00:11:09.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:11:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:11:09] "GET /metrics HTTP/1.1" 200 34142 "" "Prometheus/2.43.0" 2024-09-13T00:11:10.716 INFO:tasks.workunit.client.0.smithi136.stdout: 569: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:10.716 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-B2qOu8rxS07AQ59 off=4343612 len=1963565 2024-09-13T00:11:10.721 INFO:tasks.workunit.client.0.smithi136.stdout:op 2988 completed, throughput=4.99MB/sec 2024-09-13T00:11:10.721 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-NRjqEjFYUlxP6Ad off=3834751 len=865546 2024-09-13T00:11:10.723 INFO:tasks.workunit.client.0.smithi136.stdout:op 2989 completed, throughput=5MB/sec 2024-09-13T00:11:10.723 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-gAmMKZpJWQLWbjA off=102052 len=1305334 2024-09-13T00:11:10.736 INFO:tasks.workunit.client.0.smithi136.stdout:op 2990 completed, throughput=5MB/sec 2024-09-13T00:11:10.736 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-RlrzvAG4Yed1r4U off=3377013 len=1037332 2024-09-13T00:11:10.742 INFO:tasks.workunit.client.0.smithi136.stdout:op 2991 completed, throughput=5MB/sec 2024-09-13T00:11:10.742 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-HJSJ4UVvcne9UvB off=19284161 len=256686 2024-09-13T00:11:10.743 INFO:tasks.workunit.client.0.smithi136.stdout:op 2992 completed, throughput=5MB/sec 2024-09-13T00:11:11.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:11:11 smithi136 bash[75164]: cluster 2024-09-13T00:11:09.438291+0000 mgr.y (mgr.24661) 1021 : cluster [DBG] pgmap v1704: 105 pgs: 105 active+clean; 126 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:11:11.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:11:11 smithi136 bash[76371]: cluster 2024-09-13T00:11:09.438291+0000 mgr.y (mgr.24661) 1021 : cluster [DBG] pgmap v1704: 105 pgs: 105 active+clean; 126 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:11:11.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:11:11 smithi181 bash[67587]: cluster 2024-09-13T00:11:09.438291+0000 mgr.y (mgr.24661) 1021 : cluster [DBG] pgmap v1704: 105 pgs: 105 active+clean; 126 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:11:11.744 INFO:tasks.workunit.client.0.smithi136.stdout: 570: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:11.744 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-KXYdHX5JE9eEkNd off=5911315 len=141738 2024-09-13T00:11:11.745 INFO:tasks.workunit.client.0.smithi136.stdout:op 2993 completed, throughput=4.99MB/sec 2024-09-13T00:11:11.745 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-pJLHmu23q1nYqjS off=11082826 len=637387 2024-09-13T00:11:11.752 INFO:tasks.workunit.client.0.smithi136.stdout:op 2994 completed, throughput=4.99MB/sec 2024-09-13T00:11:11.752 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-rWf00dSoLif6gZ2 off=1756299 len=1008881 2024-09-13T00:11:11.755 INFO:tasks.workunit.client.0.smithi136.stdout:op 2995 completed, throughput=4.99MB/sec 2024-09-13T00:11:11.755 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-mNeWlMIWeIp3iRm off=14480594 len=675277 2024-09-13T00:11:11.762 INFO:tasks.workunit.client.0.smithi136.stdout:op 2996 completed, throughput=5MB/sec 2024-09-13T00:11:11.762 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-qsqByV3oRKQk5DA off=2648373 len=213140 2024-09-13T00:11:11.767 INFO:tasks.workunit.client.0.smithi136.stdout:op 2997 completed, throughput=5MB/sec 2024-09-13T00:11:11.767 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-lmZCaGw0lvDBTMF off=20999744 len=685423 2024-09-13T00:11:11.771 INFO:tasks.workunit.client.0.smithi136.stdout:op 2998 completed, throughput=5MB/sec 2024-09-13T00:11:11.771 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-xQlKfnUJ8_1zRAt off=6881367 len=506161 2024-09-13T00:11:11.777 INFO:tasks.workunit.client.0.smithi136.stdout:op 2999 completed, throughput=5MB/sec 2024-09-13T00:11:11.777 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-JDDYO9doZeY6SnM off=154837 len=805598 2024-09-13T00:11:11.785 INFO:tasks.workunit.client.0.smithi136.stdout:op 3000 completed, throughput=5MB/sec 2024-09-13T00:11:11.785 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-2soRjjMzw9lQrPP off=4642069 len=627285 2024-09-13T00:11:11.792 INFO:tasks.workunit.client.0.smithi136.stdout:op 3001 completed, throughput=5MB/sec 2024-09-13T00:11:11.792 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-cWoF3jfNSBVb05z off=4081005 len=1158889 2024-09-13T00:11:11.795 INFO:tasks.workunit.client.0.smithi136.stdout:op 3002 completed, throughput=5MB/sec 2024-09-13T00:11:12.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:11:12 smithi181 bash[95505]: ts=2024-09-13T00:11:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:11:12.796 INFO:tasks.workunit.client.0.smithi136.stdout: 571: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:12.796 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-t2WRcWchEOCUtVc off=14759439 len=2093897 2024-09-13T00:11:12.801 INFO:tasks.workunit.client.0.smithi136.stdout:op 3003 completed, throughput=5MB/sec 2024-09-13T00:11:12.802 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-k6thAq0hb9bM6zA off=705152 len=313350 2024-09-13T00:11:12.808 INFO:tasks.workunit.client.0.smithi136.stdout:op 3004 completed, throughput=5MB/sec 2024-09-13T00:11:12.808 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-HUMyz5h5LlHFGAi off=3995743 len=609216 2024-09-13T00:11:12.814 INFO:tasks.workunit.client.0.smithi136.stdout:op 3005 completed, throughput=5MB/sec 2024-09-13T00:11:12.815 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-eFaRHZamktcV74H off=6926824 len=1252970 2024-09-13T00:11:12.826 INFO:tasks.workunit.client.0.smithi136.stdout:op 3006 completed, throughput=5MB/sec 2024-09-13T00:11:12.826 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-otIgQx9JH-c7-Z6 off=2185986 len=1977276 2024-09-13T00:11:12.832 INFO:tasks.workunit.client.0.smithi136.stdout:op 3007 completed, throughput=5MB/sec 2024-09-13T00:11:13.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:11:13 smithi136 bash[75164]: cluster 2024-09-13T00:11:11.439310+0000 mgr.y (mgr.24661) 1022 : cluster [DBG] pgmap v1705: 105 pgs: 105 active+clean; 126 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:11:13.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:11:13 smithi136 bash[76371]: cluster 2024-09-13T00:11:11.439310+0000 mgr.y (mgr.24661) 1022 : cluster [DBG] pgmap v1705: 105 pgs: 105 active+clean; 126 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:11:13.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:11:13 smithi181 bash[67587]: cluster 2024-09-13T00:11:11.439310+0000 mgr.y (mgr.24661) 1022 : cluster [DBG] pgmap v1705: 105 pgs: 105 active+clean; 126 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:11:13.832 INFO:tasks.workunit.client.0.smithi136.stdout: 572: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:13.832 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-IQwXGkXUYdVPLda off=5826857 len=290416 2024-09-13T00:11:13.834 INFO:tasks.workunit.client.0.smithi136.stdout:op 3008 completed, throughput=4.99MB/sec 2024-09-13T00:11:13.834 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-vTKp6dCjingMNjr off=4441669 len=9443 2024-09-13T00:11:13.837 INFO:tasks.workunit.client.0.smithi136.stdout:op 3009 completed, throughput=4.99MB/sec 2024-09-13T00:11:13.837 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-sVNFbrFsySB3-39 off=16138850 len=1952150 2024-09-13T00:11:13.850 INFO:tasks.workunit.client.0.smithi136.stdout:op 3010 completed, throughput=5MB/sec 2024-09-13T00:11:13.850 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-BzuVz7sshIMq_ch off=479547 len=880591 2024-09-13T00:11:13.858 INFO:tasks.workunit.client.0.smithi136.stdout:op 3011 completed, throughput=5MB/sec 2024-09-13T00:11:13.858 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-d-KsQ3Lx30rnkqQ off=10379945 len=316291 2024-09-13T00:11:13.859 INFO:tasks.workunit.client.0.smithi136.stdout:op 3012 completed, throughput=5MB/sec 2024-09-13T00:11:13.859 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-LT2OXgyU7fJ8poQ off=2232561 len=1174702 2024-09-13T00:11:13.870 INFO:tasks.workunit.client.0.smithi136.stdout:op 3013 completed, throughput=5MB/sec 2024-09-13T00:11:14.870 INFO:tasks.workunit.client.0.smithi136.stdout: 573: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:14.871 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-hx11B6udNP66s57 off=6519168 len=1552193 2024-09-13T00:11:14.881 INFO:tasks.workunit.client.0.smithi136.stdout:op 3014 completed, throughput=5MB/sec 2024-09-13T00:11:14.881 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-87QJJAmWEapcHBJ off=15723803 len=118315 2024-09-13T00:11:14.882 INFO:tasks.workunit.client.0.smithi136.stdout:op 3015 completed, throughput=5MB/sec 2024-09-13T00:11:14.882 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-AJoNgepcatDaLd6 off=4038612 len=1358634 2024-09-13T00:11:14.885 INFO:tasks.workunit.client.0.smithi136.stdout:op 3016 completed, throughput=5MB/sec 2024-09-13T00:11:14.885 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-aNVisX-x_9Mewxc off=2492445 len=1564276 2024-09-13T00:11:14.889 INFO:tasks.workunit.client.0.smithi136.stdout:op 3017 completed, throughput=5MB/sec 2024-09-13T00:11:15.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:11:15 smithi136 bash[75164]: cluster 2024-09-13T00:11:13.440260+0000 mgr.y (mgr.24661) 1023 : cluster [DBG] pgmap v1706: 105 pgs: 105 active+clean; 126 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:11:15.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:11:15 smithi136 bash[76371]: cluster 2024-09-13T00:11:13.440260+0000 mgr.y (mgr.24661) 1023 : cluster [DBG] pgmap v1706: 105 pgs: 105 active+clean; 126 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:11:15.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:11:15 smithi181 bash[67587]: cluster 2024-09-13T00:11:13.440260+0000 mgr.y (mgr.24661) 1023 : cluster [DBG] pgmap v1706: 105 pgs: 105 active+clean; 126 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:11:15.889 INFO:tasks.workunit.client.0.smithi136.stdout: 574: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:15.889 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-X0DEa9Occe3mjGJ off=2713519 len=2041171 2024-09-13T00:11:15.902 INFO:tasks.workunit.client.0.smithi136.stdout:op 3018 completed, throughput=5MB/sec 2024-09-13T00:11:15.903 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-jSGmAb_ja4iw9dk off=5226836 len=1059309 2024-09-13T00:11:15.906 INFO:tasks.workunit.client.0.smithi136.stdout:op 3019 completed, throughput=5MB/sec 2024-09-13T00:11:15.906 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-hA9sxtvCVda5a23 off=1048948 len=1248156 2024-09-13T00:11:15.910 INFO:tasks.workunit.client.0.smithi136.stdout:op 3020 completed, throughput=5MB/sec 2024-09-13T00:11:15.910 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-YQyAgwrtA5K_0ku off=1868668 len=1225246 2024-09-13T00:11:15.920 INFO:tasks.workunit.client.0.smithi136.stdout:op 3021 completed, throughput=5MB/sec 2024-09-13T00:11:16.920 INFO:tasks.workunit.client.0.smithi136.stdout: 575: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:16.920 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-15ozKtTqG15PyVl off=20664456 len=1630125 2024-09-13T00:11:16.935 INFO:tasks.workunit.client.0.smithi136.stdout:op 3022 completed, throughput=4.99MB/sec 2024-09-13T00:11:16.935 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-LIOuRyRucjdjtgM off=809236 len=138790 2024-09-13T00:11:16.936 INFO:tasks.workunit.client.0.smithi136.stdout:op 3023 completed, throughput=4.99MB/sec 2024-09-13T00:11:16.936 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Uy-1JkgsU45v5Z4 off=9881326 len=1091447 2024-09-13T00:11:16.946 INFO:tasks.workunit.client.0.smithi136.stdout:op 3024 completed, throughput=5MB/sec 2024-09-13T00:11:16.946 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-oo5WFPHp3nAN7or off=15601059 len=358874 2024-09-13T00:11:16.947 INFO:tasks.workunit.client.0.smithi136.stdout:op 3025 completed, throughput=5MB/sec 2024-09-13T00:11:16.947 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-YLGORZiOzuFZVfq off=11029655 len=962644 2024-09-13T00:11:16.955 INFO:tasks.workunit.client.0.smithi136.stdout:op 3026 completed, throughput=5MB/sec 2024-09-13T00:11:16.956 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-IpqFd_eGDhxeJzQ off=4085381 len=743829 2024-09-13T00:11:16.957 INFO:tasks.workunit.client.0.smithi136.stdout:op 3027 completed, throughput=5MB/sec 2024-09-13T00:11:16.958 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-11ArVi8gDbijhoc off=16841941 len=1716089 2024-09-13T00:11:16.971 INFO:tasks.workunit.client.0.smithi136.stdout:op 3028 completed, throughput=5MB/sec 2024-09-13T00:11:17.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:11:17 smithi136 bash[75164]: cluster 2024-09-13T00:11:15.441213+0000 mgr.y (mgr.24661) 1024 : cluster [DBG] pgmap v1707: 105 pgs: 105 active+clean; 126 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:11:17.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:11:17 smithi136 bash[76371]: cluster 2024-09-13T00:11:15.441213+0000 mgr.y (mgr.24661) 1024 : cluster [DBG] pgmap v1707: 105 pgs: 105 active+clean; 126 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:11:17.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:11:17 smithi181 bash[67587]: cluster 2024-09-13T00:11:15.441213+0000 mgr.y (mgr.24661) 1024 : cluster [DBG] pgmap v1707: 105 pgs: 105 active+clean; 126 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:11:17.971 INFO:tasks.workunit.client.0.smithi136.stdout: 576: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:17.972 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-lhQB6cjTMiWizdx off=1049474 len=275917 2024-09-13T00:11:17.973 INFO:tasks.workunit.client.0.smithi136.stdout:op 3029 completed, throughput=4.99MB/sec 2024-09-13T00:11:17.973 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-3Orf8lsVhVEG5Vn off=14104335 len=28186 2024-09-13T00:11:17.978 INFO:tasks.workunit.client.0.smithi136.stdout:op 3030 completed, throughput=4.99MB/sec 2024-09-13T00:11:17.978 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-SsL-mb69IX1vZye off=5293547 len=722206 2024-09-13T00:11:17.980 INFO:tasks.workunit.client.0.smithi136.stdout:op 3031 completed, throughput=5MB/sec 2024-09-13T00:11:17.980 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-BNyYB6vG2b9Bt96 off=4382362 len=968162 2024-09-13T00:11:17.989 INFO:tasks.workunit.client.0.smithi136.stdout:op 3032 completed, throughput=5MB/sec 2024-09-13T00:11:17.989 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-iYeuoc9oitCXhDg off=1126375 len=1702901 2024-09-13T00:11:18.000 INFO:tasks.workunit.client.0.smithi136.stdout:op 3033 completed, throughput=5MB/sec 2024-09-13T00:11:18.000 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Hc65hjG-yZmnk3- off=6424016 len=419818 2024-09-13T00:11:18.002 INFO:tasks.workunit.client.0.smithi136.stdout:op 3034 completed, throughput=5MB/sec 2024-09-13T00:11:19.003 INFO:tasks.workunit.client.0.smithi136.stdout: 577: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:19.003 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Hpk_3pvfqal0lHc off=11615175 len=1436439 2024-09-13T00:11:19.016 INFO:tasks.workunit.client.0.smithi136.stdout:op 3035 completed, throughput=4.99MB/sec 2024-09-13T00:11:19.016 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-GkllNmJpDrzt_VW off=7695969 len=1142188 2024-09-13T00:11:19.024 INFO:tasks.workunit.client.0.smithi136.stdout:op 3036 completed, throughput=5MB/sec 2024-09-13T00:11:19.024 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-NJF0vpBEGpb8VAt off=2678719 len=2028998 2024-09-13T00:11:19.031 INFO:tasks.workunit.client.0.smithi136.stdout:op 3037 completed, throughput=5MB/sec 2024-09-13T00:11:19.031 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-TZQHUQS-hY_Lfgm off=7803383 len=1927595 2024-09-13T00:11:19.035 INFO:tasks.workunit.client.0.smithi136.stdout:op 3038 completed, throughput=5MB/sec 2024-09-13T00:11:19.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:11:19 smithi136 bash[76371]: cluster 2024-09-13T00:11:17.442372+0000 mgr.y (mgr.24661) 1025 : cluster [DBG] pgmap v1708: 105 pgs: 105 active+clean; 126 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 2.4 MiB/s rd, 3.0 MiB/s wr, 5 op/s 2024-09-13T00:11:19.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:11:19 smithi136 bash[75164]: cluster 2024-09-13T00:11:17.442372+0000 mgr.y (mgr.24661) 1025 : cluster [DBG] pgmap v1708: 105 pgs: 105 active+clean; 126 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 2.4 MiB/s rd, 3.0 MiB/s wr, 5 op/s 2024-09-13T00:11:19.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:11:19 smithi181 bash[67587]: cluster 2024-09-13T00:11:17.442372+0000 mgr.y (mgr.24661) 1025 : cluster [DBG] pgmap v1708: 105 pgs: 105 active+clean; 126 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 2.4 MiB/s rd, 3.0 MiB/s wr, 5 op/s 2024-09-13T00:11:19.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:11:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:11:19] "GET /metrics HTTP/1.1" 200 34142 "" "Prometheus/2.43.0" 2024-09-13T00:11:20.036 INFO:tasks.workunit.client.0.smithi136.stdout: 578: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:20.036 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-XpZjAsDI52ZBJcs off=1371873 len=1843886 2024-09-13T00:11:20.050 INFO:tasks.workunit.client.0.smithi136.stdout:op 3039 completed, throughput=5MB/sec 2024-09-13T00:11:20.050 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-p7ztppHodw88Tw7 off=2279216 len=74414 2024-09-13T00:11:20.051 INFO:tasks.workunit.client.0.smithi136.stdout:op 3040 completed, throughput=5MB/sec 2024-09-13T00:11:20.051 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-dNK2W-NHypJOU1- off=5579470 len=1974451 2024-09-13T00:11:20.056 INFO:tasks.workunit.client.0.smithi136.stdout:op 3041 completed, throughput=5MB/sec 2024-09-13T00:11:20.056 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-5XVYFAMbCrowSd7 off=1751304 len=247779 2024-09-13T00:11:20.061 INFO:tasks.workunit.client.0.smithi136.stdout:op 3042 completed, throughput=5MB/sec 2024-09-13T00:11:21.061 INFO:tasks.workunit.client.0.smithi136.stdout: 579: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:21.062 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-f2ndTmn4VkkDPQP off=3427325 len=1943629 2024-09-13T00:11:21.067 INFO:tasks.workunit.client.0.smithi136.stdout:op 3043 completed, throughput=4.99MB/sec 2024-09-13T00:11:21.067 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-otIgQx9JH-c7-Z6 off=22843063 len=2021181 2024-09-13T00:11:21.084 INFO:tasks.workunit.client.0.smithi136.stdout:op 3044 completed, throughput=5MB/sec 2024-09-13T00:11:21.085 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-rbCizqawtIddOyD off=20707877 len=609938 2024-09-13T00:11:21.087 INFO:tasks.workunit.client.0.smithi136.stdout:op 3045 completed, throughput=5MB/sec 2024-09-13T00:11:21.087 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-kFml7mpJqZPtQNy off=3817556 len=256667 2024-09-13T00:11:21.092 INFO:tasks.workunit.client.0.smithi136.stdout:op 3046 completed, throughput=5MB/sec 2024-09-13T00:11:21.092 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Ni0nvZaMXnn2TAt off=4563029 len=1600459 2024-09-13T00:11:21.105 INFO:tasks.workunit.client.0.smithi136.stdout:op 3047 completed, throughput=5MB/sec 2024-09-13T00:11:21.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:11:21 smithi136 bash[75164]: cluster 2024-09-13T00:11:19.443267+0000 mgr.y (mgr.24661) 1026 : cluster [DBG] pgmap v1709: 105 pgs: 105 active+clean; 126 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:11:21.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:11:21 smithi136 bash[76371]: cluster 2024-09-13T00:11:19.443267+0000 mgr.y (mgr.24661) 1026 : cluster [DBG] pgmap v1709: 105 pgs: 105 active+clean; 126 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:11:21.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:11:21 smithi181 bash[67587]: cluster 2024-09-13T00:11:19.443267+0000 mgr.y (mgr.24661) 1026 : cluster [DBG] pgmap v1709: 105 pgs: 105 active+clean; 126 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:11:22.105 INFO:tasks.workunit.client.0.smithi136.stdout: 580: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:22.105 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-bZ4PKpFFCPSFi5r off=8418950 len=1561332 2024-09-13T00:11:22.110 INFO:tasks.workunit.client.0.smithi136.stdout:op 3048 completed, throughput=5MB/sec 2024-09-13T00:11:22.110 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-xkK2Rmj-XEIaWMz off=6083413 len=1941545 2024-09-13T00:11:22.127 INFO:tasks.workunit.client.0.smithi136.stdout:op 3049 completed, throughput=5MB/sec 2024-09-13T00:11:22.127 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_cv_v2e2NAb1vRE off=2078621 len=118643 2024-09-13T00:11:22.131 INFO:tasks.workunit.client.0.smithi136.stdout:op 3050 completed, throughput=5MB/sec 2024-09-13T00:11:22.132 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-t0bY31V9FH_O2JE off=427853 len=1129154 2024-09-13T00:11:22.141 INFO:tasks.workunit.client.0.smithi136.stdout:op 3051 completed, throughput=5MB/sec 2024-09-13T00:11:22.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:11:22 smithi136 bash[75164]: cluster 2024-09-13T00:11:21.444353+0000 mgr.y (mgr.24661) 1027 : cluster [DBG] pgmap v1710: 105 pgs: 105 active+clean; 126 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:11:22.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:11:22 smithi136 bash[76371]: cluster 2024-09-13T00:11:21.444353+0000 mgr.y (mgr.24661) 1027 : cluster [DBG] pgmap v1710: 105 pgs: 105 active+clean; 126 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:11:22.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:11:22 smithi181 bash[67587]: cluster 2024-09-13T00:11:21.444353+0000 mgr.y (mgr.24661) 1027 : cluster [DBG] pgmap v1710: 105 pgs: 105 active+clean; 126 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:11:22.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:11:22 smithi181 bash[95505]: ts=2024-09-13T00:11:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:11:23.142 INFO:tasks.workunit.client.0.smithi136.stdout: 581: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:23.142 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-xHaLOOjCpT5jXTw off=18783870 len=247831 2024-09-13T00:11:23.143 INFO:tasks.workunit.client.0.smithi136.stdout:op 3052 completed, throughput=4.99MB/sec 2024-09-13T00:11:23.143 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Uo17ikEmn9hexnl off=3020502 len=2004912 2024-09-13T00:11:23.148 INFO:tasks.workunit.client.0.smithi136.stdout:op 3053 completed, throughput=5MB/sec 2024-09-13T00:11:23.148 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-EWDQxh4KSMBQBxw off=10000888 len=1205702 2024-09-13T00:11:23.151 INFO:tasks.workunit.client.0.smithi136.stdout:op 3054 completed, throughput=5MB/sec 2024-09-13T00:11:23.151 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-H269m1cM8MOoi2A off=14370906 len=1647300 2024-09-13T00:11:23.163 INFO:tasks.workunit.client.0.smithi136.stdout:op 3055 completed, throughput=5MB/sec 2024-09-13T00:11:24.163 INFO:tasks.workunit.client.0.smithi136.stdout: 582: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:24.163 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-mDzgiu-dw-xyuZ_ off=3026333 len=8974 2024-09-13T00:11:24.164 INFO:tasks.workunit.client.0.smithi136.stdout:op 3056 completed, throughput=4.99MB/sec 2024-09-13T00:11:24.164 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-QId5V3X3ko8cO1w off=2363971 len=1345904 2024-09-13T00:11:24.167 INFO:tasks.workunit.client.0.smithi136.stdout:op 3057 completed, throughput=4.99MB/sec 2024-09-13T00:11:24.167 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-6_LfXTHiHEAuCpM off=3753468 len=38257 2024-09-13T00:11:24.172 INFO:tasks.workunit.client.0.smithi136.stdout:op 3058 completed, throughput=4.99MB/sec 2024-09-13T00:11:24.172 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-hx11B6udNP66s57 off=9282437 len=1519022 2024-09-13T00:11:24.182 INFO:tasks.workunit.client.0.smithi136.stdout:op 3059 completed, throughput=5MB/sec 2024-09-13T00:11:24.182 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-QsZIcypIM5Au9KM off=3552425 len=700092 2024-09-13T00:11:24.190 INFO:tasks.workunit.client.0.smithi136.stdout:op 3060 completed, throughput=5MB/sec 2024-09-13T00:11:24.190 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-m9A0f5kj87NGwOk off=3611382 len=1656354 2024-09-13T00:11:24.193 INFO:tasks.workunit.client.0.smithi136.stdout:op 3061 completed, throughput=5MB/sec 2024-09-13T00:11:24.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:11:24 smithi136 bash[75164]: cluster 2024-09-13T00:11:23.445341+0000 mgr.y (mgr.24661) 1028 : cluster [DBG] pgmap v1711: 105 pgs: 105 active+clean; 126 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:11:24.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:11:24 smithi136 bash[75164]: audit 2024-09-13T00:11:23.788166+0000 mon.a (mon.0) 2197 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:11:24.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:11:24 smithi136 bash[76371]: cluster 2024-09-13T00:11:23.445341+0000 mgr.y (mgr.24661) 1028 : cluster [DBG] pgmap v1711: 105 pgs: 105 active+clean; 126 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:11:24.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:11:24 smithi136 bash[76371]: audit 2024-09-13T00:11:23.788166+0000 mon.a (mon.0) 2197 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:11:24.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:11:24 smithi181 bash[67587]: cluster 2024-09-13T00:11:23.445341+0000 mgr.y (mgr.24661) 1028 : cluster [DBG] pgmap v1711: 105 pgs: 105 active+clean; 126 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:11:24.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:11:24 smithi181 bash[67587]: audit 2024-09-13T00:11:23.788166+0000 mon.a (mon.0) 2197 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:11:25.194 INFO:tasks.workunit.client.0.smithi136.stdout: 583: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:25.194 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-pggy0XQ9VilMo_7 off=8034915 len=765335 2024-09-13T00:11:25.196 INFO:tasks.workunit.client.0.smithi136.stdout:op 3062 completed, throughput=4.99MB/sec 2024-09-13T00:11:25.196 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-TeO75DmmH6u5ZY_ off=195965 len=1217189 2024-09-13T00:11:25.200 INFO:tasks.workunit.client.0.smithi136.stdout:op 3063 completed, throughput=4.99MB/sec 2024-09-13T00:11:25.200 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-1XhAOR6Hq_ciPjD off=8853921 len=1606342 2024-09-13T00:11:25.214 INFO:tasks.workunit.client.0.smithi136.stdout:op 3064 completed, throughput=5MB/sec 2024-09-13T00:11:25.214 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-jQKPU-QQkTxw44n off=13268151 len=1031550 2024-09-13T00:11:25.217 INFO:tasks.workunit.client.0.smithi136.stdout:op 3065 completed, throughput=5MB/sec 2024-09-13T00:11:25.217 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-JGlQBdKuFw39lri off=1287463 len=1781620 2024-09-13T00:11:25.230 INFO:tasks.workunit.client.0.smithi136.stdout:op 3066 completed, throughput=5MB/sec 2024-09-13T00:11:26.230 INFO:tasks.workunit.client.0.smithi136.stdout: 584: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:26.231 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-QEjiWFpEbt4-q8y off=109032 len=754563 2024-09-13T00:11:26.239 INFO:tasks.workunit.client.0.smithi136.stdout:op 3067 completed, throughput=4.99MB/sec 2024-09-13T00:11:26.239 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Qi0fTj6fsl-1zKx off=2590761 len=1056198 2024-09-13T00:11:26.243 INFO:tasks.workunit.client.0.smithi136.stdout:op 3068 completed, throughput=5MB/sec 2024-09-13T00:11:26.243 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Nnq3eY5w-Sf-f9r off=1780611 len=534469 2024-09-13T00:11:26.245 INFO:tasks.workunit.client.0.smithi136.stdout:op 3069 completed, throughput=5MB/sec 2024-09-13T00:11:26.245 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-qiDiYV8h_jizih1 off=23186283 len=948091 2024-09-13T00:11:26.254 INFO:tasks.workunit.client.0.smithi136.stdout:op 3070 completed, throughput=5MB/sec 2024-09-13T00:11:26.254 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-j9iosAgoT8riSwC off=2296016 len=654692 2024-09-13T00:11:26.261 INFO:tasks.workunit.client.0.smithi136.stdout:op 3071 completed, throughput=5MB/sec 2024-09-13T00:11:26.261 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-euSM8KF5mKrDQVR off=7738961 len=1704051 2024-09-13T00:11:26.265 INFO:tasks.workunit.client.0.smithi136.stdout:op 3072 completed, throughput=5MB/sec 2024-09-13T00:11:26.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:11:26 smithi136 bash[75164]: cluster 2024-09-13T00:11:25.446175+0000 mgr.y (mgr.24661) 1029 : cluster [DBG] pgmap v1712: 105 pgs: 105 active+clean; 126 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:11:26.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:11:26 smithi136 bash[76371]: cluster 2024-09-13T00:11:25.446175+0000 mgr.y (mgr.24661) 1029 : cluster [DBG] pgmap v1712: 105 pgs: 105 active+clean; 126 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:11:26.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:11:26 smithi181 bash[67587]: cluster 2024-09-13T00:11:25.446175+0000 mgr.y (mgr.24661) 1029 : cluster [DBG] pgmap v1712: 105 pgs: 105 active+clean; 126 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:11:27.265 INFO:tasks.workunit.client.0.smithi136.stdout: 585: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:27.266 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-IpzkxTg9ElenfYa off=4607948 len=1424329 2024-09-13T00:11:27.271 INFO:tasks.workunit.client.0.smithi136.stdout:op 3073 completed, throughput=5MB/sec 2024-09-13T00:11:27.271 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-v8SGjgcNIhohi8B off=8923139 len=2088379 2024-09-13T00:11:27.290 INFO:tasks.workunit.client.0.smithi136.stdout:op 3074 completed, throughput=5MB/sec 2024-09-13T00:11:27.290 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-8uabFZEykR3SkOv off=6242916 len=818533 2024-09-13T00:11:27.298 INFO:tasks.workunit.client.0.smithi136.stdout:op 3075 completed, throughput=5MB/sec 2024-09-13T00:11:28.298 INFO:tasks.workunit.client.0.smithi136.stdout: 586: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:28.298 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj--jdaL8fjVb9WWNq off=4168723 len=1585498 2024-09-13T00:11:28.308 INFO:tasks.workunit.client.0.smithi136.stdout:op 3076 completed, throughput=4.99MB/sec 2024-09-13T00:11:28.309 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Z3yG_oxe4oM9EJR off=5668263 len=2064440 2024-09-13T00:11:28.313 INFO:tasks.workunit.client.0.smithi136.stdout:op 3077 completed, throughput=5MB/sec 2024-09-13T00:11:28.313 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-8_5fD51_2R-AV8D off=6517197 len=1112874 2024-09-13T00:11:28.318 INFO:tasks.workunit.client.0.smithi136.stdout:op 3078 completed, throughput=5MB/sec 2024-09-13T00:11:28.318 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-LJpHB8O2YQSVB4o off=1002925 len=1342372 2024-09-13T00:11:28.321 INFO:tasks.workunit.client.0.smithi136.stdout:op 3079 completed, throughput=5MB/sec 2024-09-13T00:11:28.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:11:28 smithi136 bash[75164]: cluster 2024-09-13T00:11:27.447306+0000 mgr.y (mgr.24661) 1030 : cluster [DBG] pgmap v1713: 105 pgs: 105 active+clean; 126 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 2.3 MiB/s rd, 3.0 MiB/s wr, 5 op/s 2024-09-13T00:11:28.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:11:28 smithi136 bash[76371]: cluster 2024-09-13T00:11:27.447306+0000 mgr.y (mgr.24661) 1030 : cluster [DBG] pgmap v1713: 105 pgs: 105 active+clean; 126 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 2.3 MiB/s rd, 3.0 MiB/s wr, 5 op/s 2024-09-13T00:11:28.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:11:28 smithi181 bash[67587]: cluster 2024-09-13T00:11:27.447306+0000 mgr.y (mgr.24661) 1030 : cluster [DBG] pgmap v1713: 105 pgs: 105 active+clean; 126 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 2.3 MiB/s rd, 3.0 MiB/s wr, 5 op/s 2024-09-13T00:11:29.322 INFO:tasks.workunit.client.0.smithi136.stdout: 587: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:29.322 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-WfTCQSyrW7oBbR3 off=2014 len=1202313 2024-09-13T00:11:29.333 INFO:tasks.workunit.client.0.smithi136.stdout:op 3080 completed, throughput=4.99MB/sec 2024-09-13T00:11:29.333 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-HAtLUK-pLhVJ6pR off=3401511 len=831891 2024-09-13T00:11:29.336 INFO:tasks.workunit.client.0.smithi136.stdout:op 3081 completed, throughput=5MB/sec 2024-09-13T00:11:29.336 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-MJbzuak7sLIoj-u off=2861204 len=408630 2024-09-13T00:11:29.342 INFO:tasks.workunit.client.0.smithi136.stdout:op 3082 completed, throughput=5MB/sec 2024-09-13T00:11:29.342 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-i6OIhjaM7H1b-pg off=1599334 len=1428143 2024-09-13T00:11:29.353 INFO:tasks.workunit.client.0.smithi136.stdout:op 3083 completed, throughput=5MB/sec 2024-09-13T00:11:29.353 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-WfTCQSyrW7oBbR3 off=0 len=1204327 2024-09-13T00:11:29.362 INFO:tasks.workunit.client.0.smithi136.stdout:op 3084 completed, throughput=5MB/sec 2024-09-13T00:11:29.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:11:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:11:29] "GET /metrics HTTP/1.1" 200 34138 "" "Prometheus/2.43.0" 2024-09-13T00:11:30.363 INFO:tasks.workunit.client.0.smithi136.stdout: 588: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:30.363 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-oD_YzkSxgDmJGUL off=2686676 len=356607 2024-09-13T00:11:30.369 INFO:tasks.workunit.client.0.smithi136.stdout:op 3085 completed, throughput=4.99MB/sec 2024-09-13T00:11:30.369 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-keHpHBb5hvY2BWJ off=279843 len=1258855 2024-09-13T00:11:30.379 INFO:tasks.workunit.client.0.smithi136.stdout:op 3086 completed, throughput=4.99MB/sec 2024-09-13T00:11:30.379 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-kpy0Ifmdk2ls95- off=8325260 len=598954 2024-09-13T00:11:30.387 INFO:tasks.workunit.client.0.smithi136.stdout:op 3087 completed, throughput=5MB/sec 2024-09-13T00:11:30.387 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-bIpbwSlBrqH-CFF off=340104 len=717246 2024-09-13T00:11:30.389 INFO:tasks.workunit.client.0.smithi136.stdout:op 3088 completed, throughput=5MB/sec 2024-09-13T00:11:30.389 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-cpPrWHeSNZODl9D off=221615 len=140653 2024-09-13T00:11:30.394 INFO:tasks.workunit.client.0.smithi136.stdout:op 3089 completed, throughput=5MB/sec 2024-09-13T00:11:30.394 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-JXLbZoKagPMwHZr off=3819784 len=1563343 2024-09-13T00:11:30.399 INFO:tasks.workunit.client.0.smithi136.stdout:op 3090 completed, throughput=5MB/sec 2024-09-13T00:11:30.399 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-smch0ZBAejx6E8m off=7446533 len=283818 2024-09-13T00:11:30.401 INFO:tasks.workunit.client.0.smithi136.stdout:op 3091 completed, throughput=5MB/sec 2024-09-13T00:11:30.401 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-KYcufsAcMMDTanC off=16363232 len=344509 2024-09-13T00:11:30.402 INFO:tasks.workunit.client.0.smithi136.stdout:op 3092 completed, throughput=5MB/sec 2024-09-13T00:11:30.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:11:30 smithi181 bash[67587]: cluster 2024-09-13T00:11:29.448182+0000 mgr.y (mgr.24661) 1031 : cluster [DBG] pgmap v1714: 105 pgs: 105 active+clean; 126 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:11:31.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:11:30 smithi136 bash[75164]: cluster 2024-09-13T00:11:29.448182+0000 mgr.y (mgr.24661) 1031 : cluster [DBG] pgmap v1714: 105 pgs: 105 active+clean; 126 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:11:31.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:11:30 smithi136 bash[76371]: cluster 2024-09-13T00:11:29.448182+0000 mgr.y (mgr.24661) 1031 : cluster [DBG] pgmap v1714: 105 pgs: 105 active+clean; 126 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:11:31.403 INFO:tasks.workunit.client.0.smithi136.stdout: 589: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:31.403 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-aBqrq8zFIOP2DSH off=12577168 len=607769 2024-09-13T00:11:31.409 INFO:tasks.workunit.client.0.smithi136.stdout:op 3093 completed, throughput=4.99MB/sec 2024-09-13T00:11:31.409 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-pZKL_rMGgt0nNXE off=745269 len=972305 2024-09-13T00:11:31.417 INFO:tasks.workunit.client.0.smithi136.stdout:op 3094 completed, throughput=4.99MB/sec 2024-09-13T00:11:31.417 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-MOqj4kDRADNnzSQ off=11297708 len=967924 2024-09-13T00:11:31.421 INFO:tasks.workunit.client.0.smithi136.stdout:op 3095 completed, throughput=5MB/sec 2024-09-13T00:11:31.421 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-dB3zF1F2K5KPeL8 off=386078 len=1816332 2024-09-13T00:11:31.425 INFO:tasks.workunit.client.0.smithi136.stdout:op 3096 completed, throughput=5MB/sec 2024-09-13T00:11:31.425 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Qai7ums3zFkH-wM off=7308368 len=800530 2024-09-13T00:11:31.431 INFO:tasks.workunit.client.0.smithi136.stdout:op 3097 completed, throughput=5MB/sec 2024-09-13T00:11:32.431 INFO:tasks.workunit.client.0.smithi136.stdout: 590: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:32.432 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-f2N7K2HDmRmElCO off=1348556 len=1063234 2024-09-13T00:11:32.437 INFO:tasks.workunit.client.0.smithi136.stdout:op 3098 completed, throughput=4.99MB/sec 2024-09-13T00:11:32.437 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-DgfCl5mWdNIgmwQ off=10171890 len=338013 2024-09-13T00:11:32.440 INFO:tasks.workunit.client.0.smithi136.stdout:op 3099 completed, throughput=4.99MB/sec 2024-09-13T00:11:32.441 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-6FjIlfdWE9QipHj off=11866950 len=1603866 2024-09-13T00:11:32.452 INFO:tasks.workunit.client.0.smithi136.stdout:op 3100 completed, throughput=5MB/sec 2024-09-13T00:11:32.452 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-H8HJ-sZHoy0PZgy off=4212754 len=1071843 2024-09-13T00:11:32.460 INFO:tasks.workunit.client.0.smithi136.stdout:op 3101 completed, throughput=5MB/sec 2024-09-13T00:11:32.460 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-9MWduyuuNRJzjEv off=4410844 len=328796 2024-09-13T00:11:32.466 INFO:tasks.workunit.client.0.smithi136.stdout:op 3102 completed, throughput=5MB/sec 2024-09-13T00:11:32.466 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-AYOfOPhEZYjvPP0 off=2145948 len=1858596 2024-09-13T00:11:32.478 INFO:tasks.workunit.client.0.smithi136.stdout:op 3103 completed, throughput=5MB/sec 2024-09-13T00:11:32.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:11:32 smithi181 bash[67587]: cluster 2024-09-13T00:11:31.449229+0000 mgr.y (mgr.24661) 1032 : cluster [DBG] pgmap v1715: 105 pgs: 105 active+clean; 126 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:11:32.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:11:32 smithi181 bash[95505]: ts=2024-09-13T00:11:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:11:33.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:11:32 smithi136 bash[75164]: cluster 2024-09-13T00:11:31.449229+0000 mgr.y (mgr.24661) 1032 : cluster [DBG] pgmap v1715: 105 pgs: 105 active+clean; 126 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:11:33.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:11:32 smithi136 bash[76371]: cluster 2024-09-13T00:11:31.449229+0000 mgr.y (mgr.24661) 1032 : cluster [DBG] pgmap v1715: 105 pgs: 105 active+clean; 126 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:11:33.479 INFO:tasks.workunit.client.0.smithi136.stdout: 591: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:33.479 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-uPBYtGAFJmAAYx5 off=9799985 len=1661030 2024-09-13T00:11:33.494 INFO:tasks.workunit.client.0.smithi136.stdout:op 3104 completed, throughput=5MB/sec 2024-09-13T00:11:33.494 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ItKC2QbsvSh8wBu off=5907589 len=632567 2024-09-13T00:11:33.496 INFO:tasks.workunit.client.0.smithi136.stdout:op 3105 completed, throughput=5MB/sec 2024-09-13T00:11:33.497 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-rNmsFFduPofaWpu off=18837231 len=159990 2024-09-13T00:11:33.502 INFO:tasks.workunit.client.0.smithi136.stdout:op 3106 completed, throughput=5MB/sec 2024-09-13T00:11:33.502 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-4nLDET4xZallq31 off=312223 len=1325806 2024-09-13T00:11:33.513 INFO:tasks.workunit.client.0.smithi136.stdout:op 3107 completed, throughput=5MB/sec 2024-09-13T00:11:33.513 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ksyL157zq4RBCB5 off=16511318 len=1355720 2024-09-13T00:11:33.522 INFO:tasks.workunit.client.0.smithi136.stdout:op 3108 completed, throughput=5MB/sec 2024-09-13T00:11:34.523 INFO:tasks.workunit.client.0.smithi136.stdout: 592: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:34.523 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-3fcpMoOXVDv_zaW off=4965755 len=569839 2024-09-13T00:11:34.525 INFO:tasks.workunit.client.0.smithi136.stdout:op 3109 completed, throughput=4.99MB/sec 2024-09-13T00:11:34.525 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-MxpbwzpqBBLERxW off=18901397 len=1424095 2024-09-13T00:11:34.535 INFO:tasks.workunit.client.0.smithi136.stdout:op 3110 completed, throughput=5MB/sec 2024-09-13T00:11:34.535 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-5pS0LoezVgpu4Hk off=910254 len=625427 2024-09-13T00:11:34.542 INFO:tasks.workunit.client.0.smithi136.stdout:op 3111 completed, throughput=5MB/sec 2024-09-13T00:11:34.542 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-G6KRESiVhMOmQMv off=11465729 len=944541 2024-09-13T00:11:34.544 INFO:tasks.workunit.client.0.smithi136.stdout:op 3112 completed, throughput=5MB/sec 2024-09-13T00:11:34.544 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-jFSIMjCPRF3a8QJ off=12168115 len=520176 2024-09-13T00:11:34.545 INFO:tasks.workunit.client.0.smithi136.stdout:op 3113 completed, throughput=5MB/sec 2024-09-13T00:11:34.546 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-XZwiazcqkHUyL3E off=5364925 len=1785263 2024-09-13T00:11:34.556 INFO:tasks.workunit.client.0.smithi136.stdout:op 3114 completed, throughput=5MB/sec 2024-09-13T00:11:35.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:11:34 smithi136 bash[75164]: cluster 2024-09-13T00:11:33.450181+0000 mgr.y (mgr.24661) 1033 : cluster [DBG] pgmap v1716: 105 pgs: 105 active+clean; 126 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.7 MiB/s wr, 4 op/s 2024-09-13T00:11:35.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:11:34 smithi136 bash[76371]: cluster 2024-09-13T00:11:33.450181+0000 mgr.y (mgr.24661) 1033 : cluster [DBG] pgmap v1716: 105 pgs: 105 active+clean; 126 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.7 MiB/s wr, 4 op/s 2024-09-13T00:11:35.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:11:34 smithi181 bash[67587]: cluster 2024-09-13T00:11:33.450181+0000 mgr.y (mgr.24661) 1033 : cluster [DBG] pgmap v1716: 105 pgs: 105 active+clean; 126 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.7 MiB/s wr, 4 op/s 2024-09-13T00:11:35.557 INFO:tasks.workunit.client.0.smithi136.stdout: 593: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:35.557 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-f9cN0oRh86h4OG9 off=944993 len=186731 2024-09-13T00:11:35.562 INFO:tasks.workunit.client.0.smithi136.stdout:op 3115 completed, throughput=4.99MB/sec 2024-09-13T00:11:35.562 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-HBkyF-EVhjlT_KE off=11768071 len=1413989 2024-09-13T00:11:35.572 INFO:tasks.workunit.client.0.smithi136.stdout:op 3116 completed, throughput=5MB/sec 2024-09-13T00:11:35.572 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-jwcv1GkRAnou8W0 off=9080833 len=1236393 2024-09-13T00:11:35.576 INFO:tasks.workunit.client.0.smithi136.stdout:op 3117 completed, throughput=5MB/sec 2024-09-13T00:11:35.576 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-18yGrfu-mjbuKiH off=3606405 len=788409 2024-09-13T00:11:35.584 INFO:tasks.workunit.client.0.smithi136.stdout:op 3118 completed, throughput=5MB/sec 2024-09-13T00:11:35.584 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-DE68BAJIqsVOeyM off=2295338 len=1972937 2024-09-13T00:11:35.601 INFO:tasks.workunit.client.0.smithi136.stdout:op 3119 completed, throughput=5MB/sec 2024-09-13T00:11:36.601 INFO:tasks.workunit.client.0.smithi136.stdout: 594: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:36.601 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-SOdHT_lq994Od_m off=20570769 len=1274252 2024-09-13T00:11:36.604 INFO:tasks.workunit.client.0.smithi136.stdout:op 3120 completed, throughput=5MB/sec 2024-09-13T00:11:36.604 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-1gexyEcxAfjUR39 off=60944 len=1919257 2024-09-13T00:11:36.609 INFO:tasks.workunit.client.0.smithi136.stdout:op 3121 completed, throughput=5MB/sec 2024-09-13T00:11:36.609 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-9RiqpNo-T0mbxTQ off=171021 len=91992 2024-09-13T00:11:36.614 INFO:tasks.workunit.client.0.smithi136.stdout:op 3122 completed, throughput=5MB/sec 2024-09-13T00:11:36.614 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-f-LAVid6Mh4TKd0 off=1058951 len=1935853 2024-09-13T00:11:36.626 INFO:tasks.workunit.client.0.smithi136.stdout:op 3123 completed, throughput=5MB/sec 2024-09-13T00:11:37.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:11:36 smithi136 bash[75164]: cluster 2024-09-13T00:11:35.451045+0000 mgr.y (mgr.24661) 1034 : cluster [DBG] pgmap v1717: 105 pgs: 105 active+clean; 126 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:11:37.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:11:36 smithi136 bash[76371]: cluster 2024-09-13T00:11:35.451045+0000 mgr.y (mgr.24661) 1034 : cluster [DBG] pgmap v1717: 105 pgs: 105 active+clean; 126 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:11:37.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:11:36 smithi181 bash[67587]: cluster 2024-09-13T00:11:35.451045+0000 mgr.y (mgr.24661) 1034 : cluster [DBG] pgmap v1717: 105 pgs: 105 active+clean; 126 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:11:37.627 INFO:tasks.workunit.client.0.smithi136.stdout: 595: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:37.627 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-C3-jtLCZcmCKNMn off=17627270 len=510971 2024-09-13T00:11:37.634 INFO:tasks.workunit.client.0.smithi136.stdout:op 3124 completed, throughput=4.99MB/sec 2024-09-13T00:11:37.634 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-QOWEcuwL5w38ff9 off=1346306 len=1686682 2024-09-13T00:11:37.639 INFO:tasks.workunit.client.0.smithi136.stdout:op 3125 completed, throughput=5MB/sec 2024-09-13T00:11:37.639 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-AH7065nmKvmkHWH off=11729367 len=1318258 2024-09-13T00:11:37.643 INFO:tasks.workunit.client.0.smithi136.stdout:op 3126 completed, throughput=5MB/sec 2024-09-13T00:11:37.643 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-emj3ugT6HKMeFRR off=1898108 len=1252316 2024-09-13T00:11:37.651 INFO:tasks.workunit.client.0.smithi136.stdout:op 3127 completed, throughput=5MB/sec 2024-09-13T00:11:38.652 INFO:tasks.workunit.client.0.smithi136.stdout: 596: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:38.652 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-9g9CrjM2Lg_z_VY off=12499614 len=1222666 2024-09-13T00:11:38.655 INFO:tasks.workunit.client.0.smithi136.stdout:op 3128 completed, throughput=4.99MB/sec 2024-09-13T00:11:38.655 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-8Qv44hLTejff4F8 off=0 len=1593061 2024-09-13T00:11:38.669 INFO:tasks.workunit.client.0.smithi136.stdout:op 3129 completed, throughput=5MB/sec 2024-09-13T00:11:38.669 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj--iGeLr323OUEbCf off=3986874 len=204226 2024-09-13T00:11:38.674 INFO:tasks.workunit.client.0.smithi136.stdout:op 3130 completed, throughput=5MB/sec 2024-09-13T00:11:38.674 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-uY8595GdHVUQiqZ off=2622366 len=1332925 2024-09-13T00:11:38.677 INFO:tasks.workunit.client.0.smithi136.stdout:op 3131 completed, throughput=5MB/sec 2024-09-13T00:11:38.677 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-krtYspnusyx80Mi off=3997402 len=287708 2024-09-13T00:11:38.683 INFO:tasks.workunit.client.0.smithi136.stdout:op 3132 completed, throughput=5MB/sec 2024-09-13T00:11:38.683 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-AktHC1FTm6hHFOn off=13481909 len=758501 2024-09-13T00:11:38.685 INFO:tasks.workunit.client.0.smithi136.stdout:op 3133 completed, throughput=5MB/sec 2024-09-13T00:11:39.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:11:38 smithi136 bash[75164]: cluster 2024-09-13T00:11:37.452133+0000 mgr.y (mgr.24661) 1035 : cluster [DBG] pgmap v1718: 105 pgs: 105 active+clean; 126 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 2.6 MiB/s rd, 3.2 MiB/s wr, 5 op/s 2024-09-13T00:11:39.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:11:38 smithi136 bash[76371]: cluster 2024-09-13T00:11:37.452133+0000 mgr.y (mgr.24661) 1035 : cluster [DBG] pgmap v1718: 105 pgs: 105 active+clean; 126 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 2.6 MiB/s rd, 3.2 MiB/s wr, 5 op/s 2024-09-13T00:11:39.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:11:38 smithi181 bash[67587]: cluster 2024-09-13T00:11:37.452133+0000 mgr.y (mgr.24661) 1035 : cluster [DBG] pgmap v1718: 105 pgs: 105 active+clean; 126 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 2.6 MiB/s rd, 3.2 MiB/s wr, 5 op/s 2024-09-13T00:11:39.686 INFO:tasks.workunit.client.0.smithi136.stdout: 597: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:39.686 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-zVjpqaXvFzUPR67 off=4927434 len=1359534 2024-09-13T00:11:39.698 INFO:tasks.workunit.client.0.smithi136.stdout:op 3134 completed, throughput=4.99MB/sec 2024-09-13T00:11:39.698 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-rhjtDhWGhTIsz0n off=8125187 len=437218 2024-09-13T00:11:39.704 INFO:tasks.workunit.client.0.smithi136.stdout:op 3135 completed, throughput=4.99MB/sec 2024-09-13T00:11:39.704 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-JALPfv9li71aZnl off=831152 len=227970 2024-09-13T00:11:39.705 INFO:tasks.workunit.client.0.smithi136.stdout:op 3136 completed, throughput=5MB/sec 2024-09-13T00:11:39.706 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-bDciU_yT2JzGLdu off=5642649 len=780869 2024-09-13T00:11:39.714 INFO:tasks.workunit.client.0.smithi136.stdout:op 3137 completed, throughput=5MB/sec 2024-09-13T00:11:39.714 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-nVsXnQPwnsXOB_T off=9766507 len=1990182 2024-09-13T00:11:39.726 INFO:tasks.workunit.client.0.smithi136.stdout:op 3138 completed, throughput=5MB/sec 2024-09-13T00:11:39.726 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-9JbfZQ4UK3EIi7- off=447894 len=1787489 2024-09-13T00:11:39.738 INFO:tasks.workunit.client.0.smithi136.stdout:op 3139 completed, throughput=5MB/sec 2024-09-13T00:11:39.748 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:11:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:11:39] "GET /metrics HTTP/1.1" 200 34142 "" "Prometheus/2.43.0" 2024-09-13T00:11:40.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:11:39 smithi136 bash[75164]: audit 2024-09-13T00:11:38.788480+0000 mon.a (mon.0) 2198 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:11:40.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:11:39 smithi136 bash[76371]: audit 2024-09-13T00:11:38.788480+0000 mon.a (mon.0) 2198 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:11:40.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:11:39 smithi181 bash[67587]: audit 2024-09-13T00:11:38.788480+0000 mon.a (mon.0) 2198 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:11:40.738 INFO:tasks.workunit.client.0.smithi136.stdout: 598: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:40.739 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-taYHowutALIqpwg off=3671570 len=1526224 2024-09-13T00:11:40.749 INFO:tasks.workunit.client.0.smithi136.stdout:op 3140 completed, throughput=5MB/sec 2024-09-13T00:11:40.749 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-NrVRyrRI-n0c22S off=1112129 len=1680931 2024-09-13T00:11:40.754 INFO:tasks.workunit.client.0.smithi136.stdout:op 3141 completed, throughput=5MB/sec 2024-09-13T00:11:40.754 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Q-EKEHr3BZQmZ5q off=4096667 len=306514 2024-09-13T00:11:40.760 INFO:tasks.workunit.client.0.smithi136.stdout:op 3142 completed, throughput=5MB/sec 2024-09-13T00:11:40.760 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-khHemI2tUBBv6mp off=1802373 len=1765433 2024-09-13T00:11:40.772 INFO:tasks.workunit.client.0.smithi136.stdout:op 3143 completed, throughput=5MB/sec 2024-09-13T00:11:41.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:11:40 smithi136 bash[75164]: cluster 2024-09-13T00:11:39.453264+0000 mgr.y (mgr.24661) 1036 : cluster [DBG] pgmap v1719: 105 pgs: 105 active+clean; 126 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.8 MiB/s wr, 4 op/s 2024-09-13T00:11:41.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:11:40 smithi136 bash[76371]: cluster 2024-09-13T00:11:39.453264+0000 mgr.y (mgr.24661) 1036 : cluster [DBG] pgmap v1719: 105 pgs: 105 active+clean; 126 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.8 MiB/s wr, 4 op/s 2024-09-13T00:11:41.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:11:40 smithi181 bash[67587]: cluster 2024-09-13T00:11:39.453264+0000 mgr.y (mgr.24661) 1036 : cluster [DBG] pgmap v1719: 105 pgs: 105 active+clean; 126 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.8 MiB/s wr, 4 op/s 2024-09-13T00:11:41.773 INFO:tasks.workunit.client.0.smithi136.stdout: 599: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:41.773 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Mzy8AV4Gt7aUlRu off=7347893 len=884516 2024-09-13T00:11:41.775 INFO:tasks.workunit.client.0.smithi136.stdout:op 3144 completed, throughput=5MB/sec 2024-09-13T00:11:41.775 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-qN8VzALVF6tZb49 off=5552164 len=2001774 2024-09-13T00:11:41.781 INFO:tasks.workunit.client.0.smithi136.stdout:op 3145 completed, throughput=5MB/sec 2024-09-13T00:11:41.782 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-U-nHgshNaXlE021 off=14620648 len=1343637 2024-09-13T00:11:41.792 INFO:tasks.workunit.client.0.smithi136.stdout:op 3146 completed, throughput=5MB/sec 2024-09-13T00:11:42.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:11:42 smithi181 bash[95505]: ts=2024-09-13T00:11:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:11:42.792 INFO:tasks.workunit.client.0.smithi136.stdout: 600: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:42.793 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-aSoV6IrMxoFME4l off=4050890 len=832941 2024-09-13T00:11:42.795 INFO:tasks.workunit.client.0.smithi136.stdout:op 3147 completed, throughput=4.99MB/sec 2024-09-13T00:11:42.795 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-p8Ttje7xWwwMkur off=320789 len=1164462 2024-09-13T00:11:42.798 INFO:tasks.workunit.client.0.smithi136.stdout:op 3148 completed, throughput=5MB/sec 2024-09-13T00:11:42.798 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-IQqU5BBfdACkWHQ off=8905925 len=1401484 2024-09-13T00:11:42.801 INFO:tasks.workunit.client.0.smithi136.stdout:op 3149 completed, throughput=5MB/sec 2024-09-13T00:11:42.802 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-6hrVDpLltF-Q8qr off=6519351 len=2024597 2024-09-13T00:11:42.806 INFO:tasks.workunit.client.0.smithi136.stdout:op 3150 completed, throughput=5MB/sec 2024-09-13T00:11:43.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:11:42 smithi136 bash[75164]: cluster 2024-09-13T00:11:41.454254+0000 mgr.y (mgr.24661) 1037 : cluster [DBG] pgmap v1720: 105 pgs: 105 active+clean; 126 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 1.9 MiB/s rd, 3.2 MiB/s wr, 4 op/s 2024-09-13T00:11:43.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:11:42 smithi136 bash[76371]: cluster 2024-09-13T00:11:41.454254+0000 mgr.y (mgr.24661) 1037 : cluster [DBG] pgmap v1720: 105 pgs: 105 active+clean; 126 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 1.9 MiB/s rd, 3.2 MiB/s wr, 4 op/s 2024-09-13T00:11:43.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:11:42 smithi181 bash[67587]: cluster 2024-09-13T00:11:41.454254+0000 mgr.y (mgr.24661) 1037 : cluster [DBG] pgmap v1720: 105 pgs: 105 active+clean; 126 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 1.9 MiB/s rd, 3.2 MiB/s wr, 4 op/s 2024-09-13T00:11:43.807 INFO:tasks.workunit.client.0.smithi136.stdout: 601: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:43.807 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-kidq2nU-uasepRI off=7186569 len=778069 2024-09-13T00:11:43.810 INFO:tasks.workunit.client.0.smithi136.stdout:op 3151 completed, throughput=4.99MB/sec 2024-09-13T00:11:43.810 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-eSVlgCfprtCaFar off=4591 len=265804 2024-09-13T00:11:43.812 INFO:tasks.workunit.client.0.smithi136.stdout:op 3152 completed, throughput=4.99MB/sec 2024-09-13T00:11:43.812 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-FfqgbptFS19KHi0 off=2977595 len=1087619 2024-09-13T00:11:43.835 INFO:tasks.workunit.client.0.smithi136.stdout:op 3153 completed, throughput=5MB/sec 2024-09-13T00:11:43.836 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-EOF1yYXaEwaIneY off=6094556 len=902960 2024-09-13T00:11:43.842 INFO:tasks.workunit.client.0.smithi136.stdout:op 3154 completed, throughput=5MB/sec 2024-09-13T00:11:43.842 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-vWG5I9EG9nfVzKd off=517137 len=1837898 2024-09-13T00:11:43.853 INFO:tasks.workunit.client.0.smithi136.stdout:op 3155 completed, throughput=5MB/sec 2024-09-13T00:11:43.853 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-gkVzwKAsbRkVYIf off=9931043 len=42291 2024-09-13T00:11:43.858 INFO:tasks.workunit.client.0.smithi136.stdout:op 3156 completed, throughput=5MB/sec 2024-09-13T00:11:43.858 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-yrGJ3KolSdWQM0k off=562968 len=608275 2024-09-13T00:11:43.865 INFO:tasks.workunit.client.0.smithi136.stdout:op 3157 completed, throughput=5MB/sec 2024-09-13T00:11:44.865 INFO:tasks.workunit.client.0.smithi136.stdout: 602: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:44.866 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ROE3SoJfLLNaQ-s off=8652790 len=1053627 2024-09-13T00:11:44.874 INFO:tasks.workunit.client.0.smithi136.stdout:op 3158 completed, throughput=4.99MB/sec 2024-09-13T00:11:44.874 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-j6nIK5fIW-tArQs off=9353338 len=1889417 2024-09-13T00:11:44.879 INFO:tasks.workunit.client.0.smithi136.stdout:op 3159 completed, throughput=5MB/sec 2024-09-13T00:11:44.879 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-oBeNnossotLFL23 off=14423665 len=1564326 2024-09-13T00:11:44.882 INFO:tasks.workunit.client.0.smithi136.stdout:op 3160 completed, throughput=5MB/sec 2024-09-13T00:11:44.882 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-2IdiYpgnNhKcPmF off=14965598 len=1241424 2024-09-13T00:11:44.893 INFO:tasks.workunit.client.0.smithi136.stdout:op 3161 completed, throughput=5MB/sec 2024-09-13T00:11:45.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:11:44 smithi181 bash[67587]: cluster 2024-09-13T00:11:43.455214+0000 mgr.y (mgr.24661) 1038 : cluster [DBG] pgmap v1721: 105 pgs: 105 active+clean; 126 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.3 MiB/s wr, 4 op/s 2024-09-13T00:11:45.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:11:44 smithi136 bash[75164]: cluster 2024-09-13T00:11:43.455214+0000 mgr.y (mgr.24661) 1038 : cluster [DBG] pgmap v1721: 105 pgs: 105 active+clean; 126 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.3 MiB/s wr, 4 op/s 2024-09-13T00:11:45.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:11:44 smithi136 bash[76371]: cluster 2024-09-13T00:11:43.455214+0000 mgr.y (mgr.24661) 1038 : cluster [DBG] pgmap v1721: 105 pgs: 105 active+clean; 126 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.3 MiB/s wr, 4 op/s 2024-09-13T00:11:45.894 INFO:tasks.workunit.client.0.smithi136.stdout: 603: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:45.894 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-5g-ScAiMw9OzIfs off=11262195 len=645072 2024-09-13T00:11:45.897 INFO:tasks.workunit.client.0.smithi136.stdout:op 3162 completed, throughput=4.99MB/sec 2024-09-13T00:11:45.897 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Rcd51LgslGifQM3 off=5868900 len=1942385 2024-09-13T00:11:45.910 INFO:tasks.workunit.client.0.smithi136.stdout:op 3163 completed, throughput=5MB/sec 2024-09-13T00:11:45.910 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-tPdUnDYSOVogpcL off=2906482 len=2044059 2024-09-13T00:11:45.916 INFO:tasks.workunit.client.0.smithi136.stdout:op 3164 completed, throughput=5MB/sec 2024-09-13T00:11:45.917 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-8i5Q5X_8E1sULjv off=4922636 len=59523 2024-09-13T00:11:45.918 INFO:tasks.workunit.client.0.smithi136.stdout:op 3165 completed, throughput=5MB/sec 2024-09-13T00:11:45.918 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-50XTs9I-5Z3E54y off=2312801 len=1036464 2024-09-13T00:11:45.921 INFO:tasks.workunit.client.0.smithi136.stdout:op 3166 completed, throughput=5MB/sec 2024-09-13T00:11:46.921 INFO:tasks.workunit.client.0.smithi136.stdout: 604: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:46.922 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-jSGmAb_ja4iw9dk off=9203494 len=1761592 2024-09-13T00:11:46.937 INFO:tasks.workunit.client.0.smithi136.stdout:op 3167 completed, throughput=5MB/sec 2024-09-13T00:11:46.937 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-fjylOOpKzaI4gIZ off=1314820 len=1341593 2024-09-13T00:11:46.940 INFO:tasks.workunit.client.0.smithi136.stdout:op 3168 completed, throughput=5MB/sec 2024-09-13T00:11:46.940 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-1BQ7nZTUoMYe6JV off=6854747 len=668689 2024-09-13T00:11:46.947 INFO:tasks.workunit.client.0.smithi136.stdout:op 3169 completed, throughput=5MB/sec 2024-09-13T00:11:46.947 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-VqrV2Pod-vB_dR2 off=2003272 len=256971 2024-09-13T00:11:46.948 INFO:tasks.workunit.client.0.smithi136.stdout:op 3170 completed, throughput=5MB/sec 2024-09-13T00:11:46.948 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-RVbs-JVotDLUHZT off=5237144 len=432068 2024-09-13T00:11:46.954 INFO:tasks.workunit.client.0.smithi136.stdout:op 3171 completed, throughput=5MB/sec 2024-09-13T00:11:46.954 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-2oH3WIsJiXjAaba off=5678725 len=1909074 2024-09-13T00:11:46.966 INFO:tasks.workunit.client.0.smithi136.stdout:op 3172 completed, throughput=5MB/sec 2024-09-13T00:11:47.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:11:46 smithi181 bash[67587]: cluster 2024-09-13T00:11:45.456147+0000 mgr.y (mgr.24661) 1039 : cluster [DBG] pgmap v1722: 105 pgs: 105 active+clean; 126 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.8 MiB/s wr, 4 op/s 2024-09-13T00:11:47.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:11:46 smithi136 bash[75164]: cluster 2024-09-13T00:11:45.456147+0000 mgr.y (mgr.24661) 1039 : cluster [DBG] pgmap v1722: 105 pgs: 105 active+clean; 126 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.8 MiB/s wr, 4 op/s 2024-09-13T00:11:47.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:11:46 smithi136 bash[76371]: cluster 2024-09-13T00:11:45.456147+0000 mgr.y (mgr.24661) 1039 : cluster [DBG] pgmap v1722: 105 pgs: 105 active+clean; 126 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.8 MiB/s wr, 4 op/s 2024-09-13T00:11:47.967 INFO:tasks.workunit.client.0.smithi136.stdout: 605: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:47.967 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-WJrf6QW0gMtlP7A off=1749089 len=2013340 2024-09-13T00:11:47.971 INFO:tasks.workunit.client.0.smithi136.stdout:op 3173 completed, throughput=5MB/sec 2024-09-13T00:11:47.971 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-s1l2JiZncT9tIWE off=4017288 len=502279 2024-09-13T00:11:47.978 INFO:tasks.workunit.client.0.smithi136.stdout:op 3174 completed, throughput=5MB/sec 2024-09-13T00:11:47.978 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Vj1yhhg9WTOYgpd off=13504671 len=405544 2024-09-13T00:11:47.984 INFO:tasks.workunit.client.0.smithi136.stdout:op 3175 completed, throughput=5MB/sec 2024-09-13T00:11:47.984 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-d-2MOKPaQHPSZ2i off=2560166 len=565909 2024-09-13T00:11:47.985 INFO:tasks.workunit.client.0.smithi136.stdout:op 3176 completed, throughput=5MB/sec 2024-09-13T00:11:47.985 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-3-UUHaL0iCprCOC off=6224238 len=1257355 2024-09-13T00:11:47.989 INFO:tasks.workunit.client.0.smithi136.stdout:op 3177 completed, throughput=5MB/sec 2024-09-13T00:11:48.989 INFO:tasks.workunit.client.0.smithi136.stdout: 606: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:48.989 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-CznTfUIhmnZvkgn off=1606649 len=1197037 2024-09-13T00:11:48.993 INFO:tasks.workunit.client.0.smithi136.stdout:op 3178 completed, throughput=5MB/sec 2024-09-13T00:11:48.993 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-XUOVn5hP-eZB8ba off=3662893 len=1890858 2024-09-13T00:11:48.997 INFO:tasks.workunit.client.0.smithi136.stdout:op 3179 completed, throughput=5MB/sec 2024-09-13T00:11:48.997 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-sMAckbyFf4vd5vZ off=3915813 len=1801848 2024-09-13T00:11:49.003 INFO:tasks.workunit.client.0.smithi136.stdout:op 3180 completed, throughput=5MB/sec 2024-09-13T00:11:49.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:11:48 smithi181 bash[67587]: cluster 2024-09-13T00:11:47.457254+0000 mgr.y (mgr.24661) 1040 : cluster [DBG] pgmap v1723: 105 pgs: 105 active+clean; 126 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:11:49.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:11:48 smithi136 bash[75164]: cluster 2024-09-13T00:11:47.457254+0000 mgr.y (mgr.24661) 1040 : cluster [DBG] pgmap v1723: 105 pgs: 105 active+clean; 126 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:11:49.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:11:48 smithi136 bash[76371]: cluster 2024-09-13T00:11:47.457254+0000 mgr.y (mgr.24661) 1040 : cluster [DBG] pgmap v1723: 105 pgs: 105 active+clean; 126 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:11:49.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:11:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:11:49] "GET /metrics HTTP/1.1" 200 34142 "" "Prometheus/2.43.0" 2024-09-13T00:11:50.003 INFO:tasks.workunit.client.0.smithi136.stdout: 607: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:50.004 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-r-zOB-czJdQmDaV off=5774491 len=748538 2024-09-13T00:11:50.006 INFO:tasks.workunit.client.0.smithi136.stdout:op 3181 completed, throughput=4.99MB/sec 2024-09-13T00:11:50.006 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-j5CcBJwEWj92Zqw off=910341 len=2003559 2024-09-13T00:11:50.011 INFO:tasks.workunit.client.0.smithi136.stdout:op 3182 completed, throughput=5MB/sec 2024-09-13T00:11:50.011 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-wO8EnBic4NHUADI off=13332637 len=1559127 2024-09-13T00:11:50.014 INFO:tasks.workunit.client.0.smithi136.stdout:op 3183 completed, throughput=5MB/sec 2024-09-13T00:11:50.015 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-GNBibFaJVsNFApi off=3021121 len=498655 2024-09-13T00:11:50.021 INFO:tasks.workunit.client.0.smithi136.stdout:op 3184 completed, throughput=5MB/sec 2024-09-13T00:11:51.021 INFO:tasks.workunit.client.0.smithi136.stdout: 608: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:51.021 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-xRQWp9m_o7oiAor off=7434151 len=781686 2024-09-13T00:11:51.024 INFO:tasks.workunit.client.0.smithi136.stdout:op 3185 completed, throughput=4.99MB/sec 2024-09-13T00:11:51.024 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-hPHwe9hQHaiWD8- off=14298105 len=307219 2024-09-13T00:11:51.029 INFO:tasks.workunit.client.0.smithi136.stdout:op 3186 completed, throughput=4.99MB/sec 2024-09-13T00:11:51.029 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-wGYtcyYoFjpasi5 off=310549 len=1061393 2024-09-13T00:11:51.033 INFO:tasks.workunit.client.0.smithi136.stdout:op 3187 completed, throughput=5MB/sec 2024-09-13T00:11:51.033 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-lbm8_aNcbNdoemY off=13808030 len=831292 2024-09-13T00:11:51.041 INFO:tasks.workunit.client.0.smithi136.stdout:op 3188 completed, throughput=5MB/sec 2024-09-13T00:11:51.041 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-7n0svx16qZj0OV0 off=4266701 len=1105459 2024-09-13T00:11:51.044 INFO:tasks.workunit.client.0.smithi136.stdout:op 3189 completed, throughput=5MB/sec 2024-09-13T00:11:51.044 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-gbeHr_iEiM96eqo off=1818707 len=713019 2024-09-13T00:11:51.046 INFO:tasks.workunit.client.0.smithi136.stdout:op 3190 completed, throughput=5MB/sec 2024-09-13T00:11:51.047 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-FbkXT2Mdw5PgsgD off=3360549 len=721804 2024-09-13T00:11:51.054 INFO:tasks.workunit.client.0.smithi136.stdout:op 3191 completed, throughput=5MB/sec 2024-09-13T00:11:51.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:11:50 smithi181 bash[67587]: cluster 2024-09-13T00:11:49.458243+0000 mgr.y (mgr.24661) 1041 : cluster [DBG] pgmap v1724: 105 pgs: 105 active+clean; 126 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:11:51.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:11:50 smithi136 bash[75164]: cluster 2024-09-13T00:11:49.458243+0000 mgr.y (mgr.24661) 1041 : cluster [DBG] pgmap v1724: 105 pgs: 105 active+clean; 126 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:11:51.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:11:50 smithi136 bash[76371]: cluster 2024-09-13T00:11:49.458243+0000 mgr.y (mgr.24661) 1041 : cluster [DBG] pgmap v1724: 105 pgs: 105 active+clean; 126 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:11:52.054 INFO:tasks.workunit.client.0.smithi136.stdout: 609: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:52.054 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-JuHXgS__YorAOvM off=465690 len=253851 2024-09-13T00:11:52.055 INFO:tasks.workunit.client.0.smithi136.stdout:op 3192 completed, throughput=4.99MB/sec 2024-09-13T00:11:52.055 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-7OHRIoLkAi64kn1 off=5294357 len=907696 2024-09-13T00:11:52.058 INFO:tasks.workunit.client.0.smithi136.stdout:op 3193 completed, throughput=4.99MB/sec 2024-09-13T00:11:52.058 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj--mg85q10jr-3YW5 off=2017136 len=1605936 2024-09-13T00:11:52.069 INFO:tasks.workunit.client.0.smithi136.stdout:op 3194 completed, throughput=5MB/sec 2024-09-13T00:11:52.069 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-LMZ6PxjGh6BWSZM off=1352541 len=1101275 2024-09-13T00:11:52.073 INFO:tasks.workunit.client.0.smithi136.stdout:op 3195 completed, throughput=5MB/sec 2024-09-13T00:11:52.073 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-r3bOVm5Ta7hbENA off=11389808 len=837159 2024-09-13T00:11:52.082 INFO:tasks.workunit.client.0.smithi136.stdout:op 3196 completed, throughput=5MB/sec 2024-09-13T00:11:52.082 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-RYeCKvlkkdn6zUx off=35746 len=771032 2024-09-13T00:11:52.084 INFO:tasks.workunit.client.0.smithi136.stdout:op 3197 completed, throughput=5MB/sec 2024-09-13T00:11:52.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:11:52 smithi181 bash[95505]: ts=2024-09-13T00:11:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:11:53.085 INFO:tasks.workunit.client.0.smithi136.stdout: 610: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:53.085 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-6PTiYoABZgWtXMo off=6720760 len=1476132 2024-09-13T00:11:53.090 INFO:tasks.workunit.client.0.smithi136.stdout:op 3198 completed, throughput=4.99MB/sec 2024-09-13T00:11:53.090 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Fe_3o_UW09EJiPR off=1405828 len=1232515 2024-09-13T00:11:53.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:11:52 smithi181 bash[67587]: cluster 2024-09-13T00:11:51.459271+0000 mgr.y (mgr.24661) 1042 : cluster [DBG] pgmap v1725: 105 pgs: 105 active+clean; 126 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:11:53.102 INFO:tasks.workunit.client.0.smithi136.stdout:op 3199 completed, throughput=5MB/sec 2024-09-13T00:11:53.102 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-jQKPU-QQkTxw44n off=8769904 len=1313888 2024-09-13T00:11:53.105 INFO:tasks.workunit.client.0.smithi136.stdout:op 3200 completed, throughput=5MB/sec 2024-09-13T00:11:53.105 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-YSsVis8rj68POH5 off=8021844 len=1072820 2024-09-13T00:11:53.108 INFO:tasks.workunit.client.0.smithi136.stdout:op 3201 completed, throughput=5MB/sec 2024-09-13T00:11:53.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:11:52 smithi136 bash[75164]: cluster 2024-09-13T00:11:51.459271+0000 mgr.y (mgr.24661) 1042 : cluster [DBG] pgmap v1725: 105 pgs: 105 active+clean; 126 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:11:53.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:11:52 smithi136 bash[76371]: cluster 2024-09-13T00:11:51.459271+0000 mgr.y (mgr.24661) 1042 : cluster [DBG] pgmap v1725: 105 pgs: 105 active+clean; 126 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:11:54.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:11:53 smithi181 bash[67587]: audit 2024-09-13T00:11:53.653834+0000 mon.a (mon.0) 2199 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:11:54.108 INFO:tasks.workunit.client.0.smithi136.stdout: 611: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:54.108 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-chtrm2Qu8q5iMdB off=7856449 len=1293056 2024-09-13T00:11:54.114 INFO:tasks.workunit.client.0.smithi136.stdout:op 3202 completed, throughput=4.99MB/sec 2024-09-13T00:11:54.114 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-i6EwMpGRkJSiErM off=1377493 len=1613797 2024-09-13T00:11:54.122 INFO:tasks.workunit.client.0.smithi136.stdout:op 3203 completed, throughput=5MB/sec 2024-09-13T00:11:54.123 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-bJsx9Wh4xaRG8iC off=9467844 len=768731 2024-09-13T00:11:54.129 INFO:tasks.workunit.client.0.smithi136.stdout:op 3204 completed, throughput=5MB/sec 2024-09-13T00:11:54.130 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Dag052dlY-rAVjn off=10341108 len=96086 2024-09-13T00:11:54.130 INFO:tasks.workunit.client.0.smithi136.stdout:op 3205 completed, throughput=5MB/sec 2024-09-13T00:11:54.130 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-fbUuXMsbXXUjaXF off=10772968 len=157354 2024-09-13T00:11:54.134 INFO:tasks.workunit.client.0.smithi136.stdout:op 3206 completed, throughput=5MB/sec 2024-09-13T00:11:54.134 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-FWTsTpHV6mchaGi off=2280830 len=1747958 2024-09-13T00:11:54.144 INFO:tasks.workunit.client.0.smithi136.stdout:op 3207 completed, throughput=5MB/sec 2024-09-13T00:11:54.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:11:53 smithi136 bash[76371]: audit 2024-09-13T00:11:53.653834+0000 mon.a (mon.0) 2199 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:11:54.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:11:53 smithi136 bash[75164]: audit 2024-09-13T00:11:53.653834+0000 mon.a (mon.0) 2199 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:11:55.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:11:54 smithi181 bash[67587]: cluster 2024-09-13T00:11:53.460224+0000 mgr.y (mgr.24661) 1043 : cluster [DBG] pgmap v1726: 105 pgs: 105 active+clean; 126 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:11:55.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:11:54 smithi181 bash[67587]: audit 2024-09-13T00:11:53.792211+0000 mon.a (mon.0) 2200 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:11:55.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:11:54 smithi181 bash[67587]: audit 2024-09-13T00:11:54.187016+0000 mon.a (mon.0) 2201 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:11:55.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:11:54 smithi181 bash[67587]: audit 2024-09-13T00:11:54.188204+0000 mon.a (mon.0) 2202 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:11:55.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:11:54 smithi181 bash[67587]: audit 2024-09-13T00:11:54.196461+0000 mon.a (mon.0) 2203 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:11:55.144 INFO:tasks.workunit.client.0.smithi136.stdout: 612: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:55.145 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-8JAa-r0rsVT7bBO off=3142479 len=829570 2024-09-13T00:11:55.147 INFO:tasks.workunit.client.0.smithi136.stdout:op 3208 completed, throughput=4.99MB/sec 2024-09-13T00:11:55.147 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-xWAKN8c9Gk3JeyT off=3025865 len=1351534 2024-09-13T00:11:55.150 INFO:tasks.workunit.client.0.smithi136.stdout:op 3209 completed, throughput=5MB/sec 2024-09-13T00:11:55.150 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-wWTvNBzMTeqAxn2 off=442661 len=1999041 2024-09-13T00:11:55.157 INFO:tasks.workunit.client.0.smithi136.stdout:op 3210 completed, throughput=5MB/sec 2024-09-13T00:11:55.157 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-P4tQOwFIVbZiVVF off=5997613 len=374441 2024-09-13T00:11:55.158 INFO:tasks.workunit.client.0.smithi136.stdout:op 3211 completed, throughput=5MB/sec 2024-09-13T00:11:55.159 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-z_VLFu6-2q0rDrB off=16693333 len=296223 2024-09-13T00:11:55.160 INFO:tasks.workunit.client.0.smithi136.stdout:op 3212 completed, throughput=5MB/sec 2024-09-13T00:11:55.160 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Iu4mF-7E3HITZot off=3772115 len=1523513 2024-09-13T00:11:55.164 INFO:tasks.workunit.client.0.smithi136.stdout:op 3213 completed, throughput=5MB/sec 2024-09-13T00:11:55.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:11:54 smithi136 bash[75164]: cluster 2024-09-13T00:11:53.460224+0000 mgr.y (mgr.24661) 1043 : cluster [DBG] pgmap v1726: 105 pgs: 105 active+clean; 126 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:11:55.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:11:54 smithi136 bash[75164]: audit 2024-09-13T00:11:53.792211+0000 mon.a (mon.0) 2200 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:11:55.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:11:54 smithi136 bash[75164]: audit 2024-09-13T00:11:54.187016+0000 mon.a (mon.0) 2201 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:11:55.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:11:54 smithi136 bash[75164]: audit 2024-09-13T00:11:54.188204+0000 mon.a (mon.0) 2202 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:11:55.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:11:54 smithi136 bash[75164]: audit 2024-09-13T00:11:54.196461+0000 mon.a (mon.0) 2203 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:11:55.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:11:54 smithi136 bash[76371]: cluster 2024-09-13T00:11:53.460224+0000 mgr.y (mgr.24661) 1043 : cluster [DBG] pgmap v1726: 105 pgs: 105 active+clean; 126 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:11:55.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:11:54 smithi136 bash[76371]: audit 2024-09-13T00:11:53.792211+0000 mon.a (mon.0) 2200 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:11:55.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:11:54 smithi136 bash[76371]: audit 2024-09-13T00:11:54.187016+0000 mon.a (mon.0) 2201 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:11:55.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:11:54 smithi136 bash[76371]: audit 2024-09-13T00:11:54.188204+0000 mon.a (mon.0) 2202 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:11:55.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:11:54 smithi136 bash[76371]: audit 2024-09-13T00:11:54.196461+0000 mon.a (mon.0) 2203 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:11:56.164 INFO:tasks.workunit.client.0.smithi136.stdout: 613: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:56.165 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-fjylOOpKzaI4gIZ off=849728 len=1493414 2024-09-13T00:11:56.168 INFO:tasks.workunit.client.0.smithi136.stdout:op 3214 completed, throughput=5MB/sec 2024-09-13T00:11:56.168 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-1GfwCntriAvQmxF off=1923623 len=872000 2024-09-13T00:11:56.171 INFO:tasks.workunit.client.0.smithi136.stdout:op 3215 completed, throughput=5MB/sec 2024-09-13T00:11:56.171 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Y0Ud_RoaW1vyC3n off=1743326 len=1288254 2024-09-13T00:11:56.174 INFO:tasks.workunit.client.0.smithi136.stdout:op 3216 completed, throughput=5MB/sec 2024-09-13T00:11:56.174 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-NW-JIygNfcZSg6_ off=3703094 len=1332693 2024-09-13T00:11:56.178 INFO:tasks.workunit.client.0.smithi136.stdout:op 3217 completed, throughput=5MB/sec 2024-09-13T00:11:57.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:11:56 smithi181 bash[67587]: cluster 2024-09-13T00:11:55.461084+0000 mgr.y (mgr.24661) 1044 : cluster [DBG] pgmap v1727: 105 pgs: 105 active+clean; 126 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.7 MiB/s wr, 4 op/s 2024-09-13T00:11:57.179 INFO:tasks.workunit.client.0.smithi136.stdout: 614: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:57.179 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-k7RxnUQxnDmqa02 off=833854 len=88333 2024-09-13T00:11:57.180 INFO:tasks.workunit.client.0.smithi136.stdout:op 3218 completed, throughput=4.99MB/sec 2024-09-13T00:11:57.180 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-u28PTEOHqdRgbuF off=11951161 len=1479120 2024-09-13T00:11:57.183 INFO:tasks.workunit.client.0.smithi136.stdout:op 3219 completed, throughput=5MB/sec 2024-09-13T00:11:57.184 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-BbQJUw8N7jXMoPV off=398661 len=594003 2024-09-13T00:11:57.185 INFO:tasks.workunit.client.0.smithi136.stdout:op 3220 completed, throughput=5MB/sec 2024-09-13T00:11:57.185 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-MmrMGgzNmPaDATx off=5123602 len=393067 2024-09-13T00:11:57.187 INFO:tasks.workunit.client.0.smithi136.stdout:op 3221 completed, throughput=5MB/sec 2024-09-13T00:11:57.187 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-XnjYKsCJqyH1O-z off=657405 len=2013346 2024-09-13T00:11:57.191 INFO:tasks.workunit.client.0.smithi136.stdout:op 3222 completed, throughput=5MB/sec 2024-09-13T00:11:57.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:11:56 smithi136 bash[75164]: cluster 2024-09-13T00:11:55.461084+0000 mgr.y (mgr.24661) 1044 : cluster [DBG] pgmap v1727: 105 pgs: 105 active+clean; 126 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.7 MiB/s wr, 4 op/s 2024-09-13T00:11:57.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:11:56 smithi136 bash[76371]: cluster 2024-09-13T00:11:55.461084+0000 mgr.y (mgr.24661) 1044 : cluster [DBG] pgmap v1727: 105 pgs: 105 active+clean; 126 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.7 MiB/s wr, 4 op/s 2024-09-13T00:11:58.192 INFO:tasks.workunit.client.0.smithi136.stdout: 615: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:58.192 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-09ZMqLvWoN9vHVF off=2976893 len=1061802 2024-09-13T00:11:58.195 INFO:tasks.workunit.client.0.smithi136.stdout:op 3223 completed, throughput=4.99MB/sec 2024-09-13T00:11:58.195 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-RatCT3z0B8Nct5U off=13284593 len=1427078 2024-09-13T00:11:58.205 INFO:tasks.workunit.client.0.smithi136.stdout:op 3224 completed, throughput=5MB/sec 2024-09-13T00:11:58.205 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-0eJEUwqDJKfT1T6 off=1419127 len=940463 2024-09-13T00:11:58.208 INFO:tasks.workunit.client.0.smithi136.stdout:op 3225 completed, throughput=5MB/sec 2024-09-13T00:11:58.208 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Om7UZfm08yXW2dq off=17929500 len=894185 2024-09-13T00:11:58.210 INFO:tasks.workunit.client.0.smithi136.stdout:op 3226 completed, throughput=5MB/sec 2024-09-13T00:11:58.210 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-0GcUlQ6CjO5OUwz off=3966930 len=1615734 2024-09-13T00:11:58.221 INFO:tasks.workunit.client.0.smithi136.stdout:op 3227 completed, throughput=5MB/sec 2024-09-13T00:11:59.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:11:58 smithi181 bash[67587]: cluster 2024-09-13T00:11:57.462238+0000 mgr.y (mgr.24661) 1045 : cluster [DBG] pgmap v1728: 105 pgs: 105 active+clean; 126 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.7 MiB/s wr, 5 op/s 2024-09-13T00:11:59.221 INFO:tasks.workunit.client.0.smithi136.stdout: 616: throughput=4.99MB/sec pending data=0 2024-09-13T00:11:59.231 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-fu6AkNGsxTlATKA off=530726 len=505758 2024-09-13T00:11:59.232 INFO:tasks.workunit.client.0.smithi136.stdout:op 3228 completed, throughput=4.99MB/sec 2024-09-13T00:11:59.232 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-eqlnOCIKRO_GFlr off=6466390 len=1645329 2024-09-13T00:11:59.234 INFO:tasks.workunit.client.0.smithi136.stdout:op 3229 completed, throughput=5MB/sec 2024-09-13T00:11:59.235 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-h3lI8sI7yR-cyAe off=2070172 len=1420429 2024-09-13T00:11:59.247 INFO:tasks.workunit.client.0.smithi136.stdout:op 3230 completed, throughput=5MB/sec 2024-09-13T00:11:59.247 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-sJZNr4gkKcBKvlM off=1802826 len=402756 2024-09-13T00:11:59.253 INFO:tasks.workunit.client.0.smithi136.stdout:op 3231 completed, throughput=5MB/sec 2024-09-13T00:11:59.253 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-f9cN0oRh86h4OG9 off=2298361 len=321171 2024-09-13T00:11:59.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:11:58 smithi136 bash[75164]: cluster 2024-09-13T00:11:57.462238+0000 mgr.y (mgr.24661) 1045 : cluster [DBG] pgmap v1728: 105 pgs: 105 active+clean; 126 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.7 MiB/s wr, 5 op/s 2024-09-13T00:11:59.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:11:58 smithi136 bash[76371]: cluster 2024-09-13T00:11:57.462238+0000 mgr.y (mgr.24661) 1045 : cluster [DBG] pgmap v1728: 105 pgs: 105 active+clean; 126 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.7 MiB/s wr, 5 op/s 2024-09-13T00:11:59.258 INFO:tasks.workunit.client.0.smithi136.stdout:op 3232 completed, throughput=5MB/sec 2024-09-13T00:11:59.258 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-UNzyI7PTVtdYgmS off=7308666 len=1196812 2024-09-13T00:11:59.260 INFO:tasks.workunit.client.0.smithi136.stdout:op 3233 completed, throughput=5MB/sec 2024-09-13T00:11:59.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:11:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:11:59] "GET /metrics HTTP/1.1" 200 34141 "" "Prometheus/2.43.0" 2024-09-13T00:12:00.260 INFO:tasks.workunit.client.0.smithi136.stdout: 617: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:00.261 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-KtArSGXvrpU7ZpR off=13872856 len=1481466 2024-09-13T00:12:00.271 INFO:tasks.workunit.client.0.smithi136.stdout:op 3234 completed, throughput=5MB/sec 2024-09-13T00:12:00.271 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-bWQbdaYiHSzOmfT off=3453779 len=1705179 2024-09-13T00:12:00.275 INFO:tasks.workunit.client.0.smithi136.stdout:op 3235 completed, throughput=5MB/sec 2024-09-13T00:12:00.275 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-M1XA5ZCUu7bF777 off=4567271 len=682360 2024-09-13T00:12:00.278 INFO:tasks.workunit.client.0.smithi136.stdout:op 3236 completed, throughput=5MB/sec 2024-09-13T00:12:00.278 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-f9wH9jZ8JncAgTT off=1924703 len=493308 2024-09-13T00:12:00.283 INFO:tasks.workunit.client.0.smithi136.stdout:op 3237 completed, throughput=5MB/sec 2024-09-13T00:12:00.283 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-HMKd7Esgq9eNWU3 off=4675564 len=1820691 2024-09-13T00:12:00.289 INFO:tasks.workunit.client.0.smithi136.stdout:op 3238 completed, throughput=5MB/sec 2024-09-13T00:12:01.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:12:00 smithi181 bash[67587]: cluster 2024-09-13T00:11:59.463227+0000 mgr.y (mgr.24661) 1046 : cluster [DBG] pgmap v1729: 105 pgs: 105 active+clean; 126 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.6 MiB/s wr, 4 op/s 2024-09-13T00:12:01.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:12:00 smithi136 bash[75164]: cluster 2024-09-13T00:11:59.463227+0000 mgr.y (mgr.24661) 1046 : cluster [DBG] pgmap v1729: 105 pgs: 105 active+clean; 126 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.6 MiB/s wr, 4 op/s 2024-09-13T00:12:01.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:12:00 smithi136 bash[76371]: cluster 2024-09-13T00:11:59.463227+0000 mgr.y (mgr.24661) 1046 : cluster [DBG] pgmap v1729: 105 pgs: 105 active+clean; 126 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.6 MiB/s wr, 4 op/s 2024-09-13T00:12:01.289 INFO:tasks.workunit.client.0.smithi136.stdout: 618: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:01.289 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2YOYCAitDqevWg5 off=1886939 len=1037245 2024-09-13T00:12:01.292 INFO:tasks.workunit.client.0.smithi136.stdout:op 3239 completed, throughput=5MB/sec 2024-09-13T00:12:01.292 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-n-nUxsNw85NO8ie off=8032850 len=1416747 2024-09-13T00:12:01.296 INFO:tasks.workunit.client.0.smithi136.stdout:op 3240 completed, throughput=5MB/sec 2024-09-13T00:12:01.296 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-oD_YzkSxgDmJGUL off=2159131 len=390405 2024-09-13T00:12:01.298 INFO:tasks.workunit.client.0.smithi136.stdout:op 3241 completed, throughput=5MB/sec 2024-09-13T00:12:01.298 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-PvLsQaVxjHj_Hyp off=860024 len=628694 2024-09-13T00:12:01.300 INFO:tasks.workunit.client.0.smithi136.stdout:op 3242 completed, throughput=5MB/sec 2024-09-13T00:12:01.300 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-bJTzZIIjlIe60to off=10731705 len=455586 2024-09-13T00:12:01.301 INFO:tasks.workunit.client.0.smithi136.stdout:op 3243 completed, throughput=5MB/sec 2024-09-13T00:12:02.302 INFO:tasks.workunit.client.0.smithi136.stdout: 619: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:02.302 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-aDFidXtGfpHPrFz off=4180096 len=764388 2024-09-13T00:12:02.304 INFO:tasks.workunit.client.0.smithi136.stdout:op 3244 completed, throughput=4.99MB/sec 2024-09-13T00:12:02.304 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-FYRdGBXzzDzIfEl off=5387138 len=405587 2024-09-13T00:12:02.306 INFO:tasks.workunit.client.0.smithi136.stdout:op 3245 completed, throughput=4.99MB/sec 2024-09-13T00:12:02.307 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-FHT4T4CYtDAWWNG off=17013902 len=1916099 2024-09-13T00:12:02.320 INFO:tasks.workunit.client.0.smithi136.stdout:op 3246 completed, throughput=5MB/sec 2024-09-13T00:12:02.320 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-5ZD0xOTsgQxCUCF off=91347 len=232494 2024-09-13T00:12:02.325 INFO:tasks.workunit.client.0.smithi136.stdout:op 3247 completed, throughput=5MB/sec 2024-09-13T00:12:02.326 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-mkSo8GcVM68YXCQ off=6640629 len=463738 2024-09-13T00:12:02.327 INFO:tasks.workunit.client.0.smithi136.stdout:op 3248 completed, throughput=5MB/sec 2024-09-13T00:12:02.327 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2wtRPuRif54nAu7 off=10339920 len=574958 2024-09-13T00:12:02.329 INFO:tasks.workunit.client.0.smithi136.stdout:op 3249 completed, throughput=5MB/sec 2024-09-13T00:12:02.329 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-WEaVbobjNhky33s off=1234638 len=1968613 2024-09-13T00:12:02.334 INFO:tasks.workunit.client.0.smithi136.stdout:op 3250 completed, throughput=5MB/sec 2024-09-13T00:12:02.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:12:02 smithi181 bash[95505]: ts=2024-09-13T00:12:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:12:03.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:12:02 smithi181 bash[67587]: cluster 2024-09-13T00:12:01.464237+0000 mgr.y (mgr.24661) 1047 : cluster [DBG] pgmap v1730: 105 pgs: 105 active+clean; 126 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.4 MiB/s wr, 4 op/s 2024-09-13T00:12:03.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:12:02 smithi136 bash[75164]: cluster 2024-09-13T00:12:01.464237+0000 mgr.y (mgr.24661) 1047 : cluster [DBG] pgmap v1730: 105 pgs: 105 active+clean; 126 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.4 MiB/s wr, 4 op/s 2024-09-13T00:12:03.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:12:02 smithi136 bash[76371]: cluster 2024-09-13T00:12:01.464237+0000 mgr.y (mgr.24661) 1047 : cluster [DBG] pgmap v1730: 105 pgs: 105 active+clean; 126 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.4 MiB/s wr, 4 op/s 2024-09-13T00:12:03.334 INFO:tasks.workunit.client.0.smithi136.stdout: 620: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:03.334 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-O0RRXbIYd8yjruT off=11426790 len=451051 2024-09-13T00:12:03.336 INFO:tasks.workunit.client.0.smithi136.stdout:op 3251 completed, throughput=4.99MB/sec 2024-09-13T00:12:03.336 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-DkRQeVIRe_FlXCX off=14245931 len=1803526 2024-09-13T00:12:03.351 INFO:tasks.workunit.client.0.smithi136.stdout:op 3252 completed, throughput=5MB/sec 2024-09-13T00:12:03.351 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-WjBM8cQ9ER0kscW off=5333197 len=1931395 2024-09-13T00:12:03.365 INFO:tasks.workunit.client.0.smithi136.stdout:op 3253 completed, throughput=5MB/sec 2024-09-13T00:12:03.365 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-pJLQoW7vSs2nIil off=1672711 len=1954198 2024-09-13T00:12:03.384 INFO:tasks.workunit.client.0.smithi136.stdout:op 3254 completed, throughput=5MB/sec 2024-09-13T00:12:04.382 INFO:tasks.workunit.client.0.smithi136.stdout: 621: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:04.383 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-eVHsRn4AdtjVFAx off=10786083 len=240941 2024-09-13T00:12:04.388 INFO:tasks.workunit.client.0.smithi136.stdout:op 3255 completed, throughput=5MB/sec 2024-09-13T00:12:04.388 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-KHsR2iDF2-5DZXy off=13409627 len=431873 2024-09-13T00:12:04.393 INFO:tasks.workunit.client.0.smithi136.stdout:op 3256 completed, throughput=5MB/sec 2024-09-13T00:12:04.393 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-bHAJHslzZTx8s13 off=10990809 len=1645476 2024-09-13T00:12:04.397 INFO:tasks.workunit.client.0.smithi136.stdout:op 3257 completed, throughput=5MB/sec 2024-09-13T00:12:04.397 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-LYbToT667oCJQ0H off=4533546 len=1933287 2024-09-13T00:12:04.409 INFO:tasks.workunit.client.0.smithi136.stdout:op 3258 completed, throughput=5MB/sec 2024-09-13T00:12:05.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:12:04 smithi181 bash[67587]: cluster 2024-09-13T00:12:03.465271+0000 mgr.y (mgr.24661) 1048 : cluster [DBG] pgmap v1731: 105 pgs: 105 active+clean; 126 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.6 MiB/s wr, 5 op/s 2024-09-13T00:12:05.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:12:04 smithi136 bash[75164]: cluster 2024-09-13T00:12:03.465271+0000 mgr.y (mgr.24661) 1048 : cluster [DBG] pgmap v1731: 105 pgs: 105 active+clean; 126 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.6 MiB/s wr, 5 op/s 2024-09-13T00:12:05.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:12:04 smithi136 bash[76371]: cluster 2024-09-13T00:12:03.465271+0000 mgr.y (mgr.24661) 1048 : cluster [DBG] pgmap v1731: 105 pgs: 105 active+clean; 126 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.6 MiB/s wr, 5 op/s 2024-09-13T00:12:05.409 INFO:tasks.workunit.client.0.smithi136.stdout: 622: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:05.409 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-19E1X0fsMf24Xcc off=5382680 len=1729744 2024-09-13T00:12:05.421 INFO:tasks.workunit.client.0.smithi136.stdout:op 3259 completed, throughput=5MB/sec 2024-09-13T00:12:05.421 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-VcUB9y-2ztcMh9I off=7371035 len=918815 2024-09-13T00:12:05.431 INFO:tasks.workunit.client.0.smithi136.stdout:op 3260 completed, throughput=5MB/sec 2024-09-13T00:12:05.431 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ESEw0AccS17iZb- off=1263226 len=162628 2024-09-13T00:12:05.432 INFO:tasks.workunit.client.0.smithi136.stdout:op 3261 completed, throughput=5MB/sec 2024-09-13T00:12:05.432 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-WWsNNxmFUKTCK1t off=11891592 len=537642 2024-09-13T00:12:05.434 INFO:tasks.workunit.client.0.smithi136.stdout:op 3262 completed, throughput=5MB/sec 2024-09-13T00:12:05.436 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-u7xE4SJ2CItSh82 off=1499369 len=1644993 2024-09-13T00:12:05.445 INFO:tasks.workunit.client.0.smithi136.stdout:op 3263 completed, throughput=5MB/sec 2024-09-13T00:12:06.445 INFO:tasks.workunit.client.0.smithi136.stdout: 623: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:06.445 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-e1yJY-TC4VNI0sE off=487270 len=242361 2024-09-13T00:12:06.447 INFO:tasks.workunit.client.0.smithi136.stdout:op 3264 completed, throughput=4.99MB/sec 2024-09-13T00:12:06.447 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-FQXXAWE8_vqrr-W off=7480057 len=1991732 2024-09-13T00:12:06.452 INFO:tasks.workunit.client.0.smithi136.stdout:op 3265 completed, throughput=5MB/sec 2024-09-13T00:12:06.452 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-s0DWPZCl821W-4Q off=353934 len=98727 2024-09-13T00:12:06.453 INFO:tasks.workunit.client.0.smithi136.stdout:op 3266 completed, throughput=5MB/sec 2024-09-13T00:12:06.453 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-vyp3nYMgpnp7hgw off=1326121 len=1143007 2024-09-13T00:12:06.464 INFO:tasks.workunit.client.0.smithi136.stdout:op 3267 completed, throughput=5MB/sec 2024-09-13T00:12:06.464 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ZMohR7nktdpaq3f off=2877006 len=1306657 2024-09-13T00:12:06.467 INFO:tasks.workunit.client.0.smithi136.stdout:op 3268 completed, throughput=5MB/sec 2024-09-13T00:12:06.467 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-_zV0tX2DhJ5shPv off=7972455 len=616858 2024-09-13T00:12:06.469 INFO:tasks.workunit.client.0.smithi136.stdout:op 3269 completed, throughput=5MB/sec 2024-09-13T00:12:07.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:12:06 smithi181 bash[67587]: cluster 2024-09-13T00:12:05.466172+0000 mgr.y (mgr.24661) 1049 : cluster [DBG] pgmap v1732: 105 pgs: 105 active+clean; 126 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 3.5 MiB/s rd, 2.1 MiB/s wr, 5 op/s 2024-09-13T00:12:07.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:12:06 smithi136 bash[75164]: cluster 2024-09-13T00:12:05.466172+0000 mgr.y (mgr.24661) 1049 : cluster [DBG] pgmap v1732: 105 pgs: 105 active+clean; 126 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 3.5 MiB/s rd, 2.1 MiB/s wr, 5 op/s 2024-09-13T00:12:07.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:12:06 smithi136 bash[76371]: cluster 2024-09-13T00:12:05.466172+0000 mgr.y (mgr.24661) 1049 : cluster [DBG] pgmap v1732: 105 pgs: 105 active+clean; 126 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 3.5 MiB/s rd, 2.1 MiB/s wr, 5 op/s 2024-09-13T00:12:07.470 INFO:tasks.workunit.client.0.smithi136.stdout: 624: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:07.470 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-hWmWJEm5o5htOQv off=3063768 len=1755211 2024-09-13T00:12:07.485 INFO:tasks.workunit.client.0.smithi136.stdout:op 3270 completed, throughput=5MB/sec 2024-09-13T00:12:07.485 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Ye1iVX0XroAi2ZV off=4373987 len=691763 2024-09-13T00:12:07.492 INFO:tasks.workunit.client.0.smithi136.stdout:op 3271 completed, throughput=5MB/sec 2024-09-13T00:12:07.492 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-z7ThNzHeTDKPPRf off=6996254 len=1988562 2024-09-13T00:12:07.505 INFO:tasks.workunit.client.0.smithi136.stdout:op 3272 completed, throughput=5MB/sec 2024-09-13T00:12:07.505 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-pFkphxlANWLuYJv off=8530548 len=925424 2024-09-13T00:12:07.507 INFO:tasks.workunit.client.0.smithi136.stdout:op 3273 completed, throughput=5MB/sec 2024-09-13T00:12:08.508 INFO:tasks.workunit.client.0.smithi136.stdout: 625: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:08.508 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Y9Hc5GnyyoyK_n5 off=5827173 len=323514 2024-09-13T00:12:08.510 INFO:tasks.workunit.client.0.smithi136.stdout:op 3274 completed, throughput=4.99MB/sec 2024-09-13T00:12:08.510 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-pEMNOiRHmFxXomN off=7505098 len=1041470 2024-09-13T00:12:08.513 INFO:tasks.workunit.client.0.smithi136.stdout:op 3275 completed, throughput=4.99MB/sec 2024-09-13T00:12:08.513 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-4FcZxoDLfbgerIM off=13314351 len=1455023 2024-09-13T00:12:08.517 INFO:tasks.workunit.client.0.smithi136.stdout:op 3276 completed, throughput=5MB/sec 2024-09-13T00:12:08.517 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-3PRkUxqRu5faMOe off=5754214 len=1464228 2024-09-13T00:12:08.521 INFO:tasks.workunit.client.0.smithi136.stdout:op 3277 completed, throughput=5MB/sec 2024-09-13T00:12:08.521 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-NraADZEUPMt87Fc off=23019145 len=471429 2024-09-13T00:12:08.523 INFO:tasks.workunit.client.0.smithi136.stdout:op 3278 completed, throughput=5MB/sec 2024-09-13T00:12:08.523 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-DkXT1gyYZ9VH1OU off=19771990 len=181059 2024-09-13T00:12:08.527 INFO:tasks.workunit.client.0.smithi136.stdout:op 3279 completed, throughput=5MB/sec 2024-09-13T00:12:08.527 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-gGvdkuEIS_MlLqT off=1348046 len=1575617 2024-09-13T00:12:08.538 INFO:tasks.workunit.client.0.smithi136.stdout:op 3280 completed, throughput=5MB/sec 2024-09-13T00:12:09.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:12:08 smithi181 bash[67587]: cluster 2024-09-13T00:12:07.467295+0000 mgr.y (mgr.24661) 1050 : cluster [DBG] pgmap v1733: 105 pgs: 105 active+clean; 126 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 3.3 MiB/s rd, 2.0 MiB/s wr, 5 op/s 2024-09-13T00:12:09.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:12:08 smithi181 bash[67587]: audit 2024-09-13T00:12:08.789449+0000 mon.a (mon.0) 2204 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:12:09.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:12:08 smithi136 bash[75164]: cluster 2024-09-13T00:12:07.467295+0000 mgr.y (mgr.24661) 1050 : cluster [DBG] pgmap v1733: 105 pgs: 105 active+clean; 126 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 3.3 MiB/s rd, 2.0 MiB/s wr, 5 op/s 2024-09-13T00:12:09.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:12:08 smithi136 bash[75164]: audit 2024-09-13T00:12:08.789449+0000 mon.a (mon.0) 2204 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:12:09.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:12:08 smithi136 bash[76371]: cluster 2024-09-13T00:12:07.467295+0000 mgr.y (mgr.24661) 1050 : cluster [DBG] pgmap v1733: 105 pgs: 105 active+clean; 126 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 3.3 MiB/s rd, 2.0 MiB/s wr, 5 op/s 2024-09-13T00:12:09.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:12:08 smithi136 bash[76371]: audit 2024-09-13T00:12:08.789449+0000 mon.a (mon.0) 2204 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:12:09.538 INFO:tasks.workunit.client.0.smithi136.stdout: 626: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:09.539 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_DvGXxkuTlw0GlJ off=1159017 len=1311704 2024-09-13T00:12:09.548 INFO:tasks.workunit.client.0.smithi136.stdout:op 3281 completed, throughput=5MB/sec 2024-09-13T00:12:09.549 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ciKdNdBSkjRoZ-8 off=5152022 len=987847 2024-09-13T00:12:09.556 INFO:tasks.workunit.client.0.smithi136.stdout:op 3282 completed, throughput=5MB/sec 2024-09-13T00:12:09.556 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-B773FBKQ0fyOdCD off=128827 len=511869 2024-09-13T00:12:09.563 INFO:tasks.workunit.client.0.smithi136.stdout:op 3283 completed, throughput=5MB/sec 2024-09-13T00:12:09.563 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-xSX3chqHXJ-_UUc off=6198606 len=141132 2024-09-13T00:12:09.568 INFO:tasks.workunit.client.0.smithi136.stdout:op 3284 completed, throughput=5MB/sec 2024-09-13T00:12:09.568 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-065j-kLxxIteAsB off=19042766 len=2089561 2024-09-13T00:12:09.572 INFO:tasks.workunit.client.0.smithi136.stdout:op 3285 completed, throughput=5MB/sec 2024-09-13T00:12:09.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:12:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:12:09] "GET /metrics HTTP/1.1" 200 34139 "" "Prometheus/2.43.0" 2024-09-13T00:12:10.573 INFO:tasks.workunit.client.0.smithi136.stdout: 627: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:10.573 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-aXbd09ivSQifB-K off=612470 len=39959 2024-09-13T00:12:10.574 INFO:tasks.workunit.client.0.smithi136.stdout:op 3286 completed, throughput=4.99MB/sec 2024-09-13T00:12:10.574 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-nUdQXnVVbXaXp82 off=0 len=1136700 2024-09-13T00:12:10.577 INFO:tasks.workunit.client.0.smithi136.stdout:op 3287 completed, throughput=5MB/sec 2024-09-13T00:12:10.577 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-7OkRiwARne1kyqG off=5386266 len=1778919 2024-09-13T00:12:10.592 INFO:tasks.workunit.client.0.smithi136.stdout:op 3288 completed, throughput=5MB/sec 2024-09-13T00:12:10.592 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-gMRq7iUZmlyLvSZ off=10021259 len=971507 2024-09-13T00:12:10.595 INFO:tasks.workunit.client.0.smithi136.stdout:op 3289 completed, throughput=5MB/sec 2024-09-13T00:12:10.595 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-pk0Vwu3yERLj6ZP off=5780020 len=924043 2024-09-13T00:12:10.602 INFO:tasks.workunit.client.0.smithi136.stdout:op 3290 completed, throughput=5MB/sec 2024-09-13T00:12:11.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:12:10 smithi181 bash[67587]: cluster 2024-09-13T00:12:09.468202+0000 mgr.y (mgr.24661) 1051 : cluster [DBG] pgmap v1734: 105 pgs: 105 active+clean; 126 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:12:11.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:12:10 smithi136 bash[75164]: cluster 2024-09-13T00:12:09.468202+0000 mgr.y (mgr.24661) 1051 : cluster [DBG] pgmap v1734: 105 pgs: 105 active+clean; 126 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:12:11.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:12:10 smithi136 bash[76371]: cluster 2024-09-13T00:12:09.468202+0000 mgr.y (mgr.24661) 1051 : cluster [DBG] pgmap v1734: 105 pgs: 105 active+clean; 126 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:12:11.602 INFO:tasks.workunit.client.0.smithi136.stdout: 628: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:11.602 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-KUSEdjWWCOpctm9 off=7327231 len=1388254 2024-09-13T00:12:11.613 INFO:tasks.workunit.client.0.smithi136.stdout:op 3291 completed, throughput=4.99MB/sec 2024-09-13T00:12:11.613 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-s0DWPZCl821W-4Q off=12804923 len=1244902 2024-09-13T00:12:11.617 INFO:tasks.workunit.client.0.smithi136.stdout:op 3292 completed, throughput=5MB/sec 2024-09-13T00:12:11.617 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-4x973sDQNA2omJq off=8714561 len=586192 2024-09-13T00:12:11.618 INFO:tasks.workunit.client.0.smithi136.stdout:op 3293 completed, throughput=5MB/sec 2024-09-13T00:12:11.619 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-WeL0_rXdRkdOEzG off=264106 len=1948194 2024-09-13T00:12:11.621 INFO:tasks.workunit.client.0.smithi136.stdout:op 3294 completed, throughput=5MB/sec 2024-09-13T00:12:12.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:12:12 smithi181 bash[95505]: ts=2024-09-13T00:12:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:12:12.621 INFO:tasks.workunit.client.0.smithi136.stdout: 629: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:12.621 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-6dlsgV2ZmvTIatM off=1581507 len=203751 2024-09-13T00:12:12.623 INFO:tasks.workunit.client.0.smithi136.stdout:op 3295 completed, throughput=4.99MB/sec 2024-09-13T00:12:12.623 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-owhz9e5CM1jYtxZ off=8607200 len=444361 2024-09-13T00:12:12.629 INFO:tasks.workunit.client.0.smithi136.stdout:op 3296 completed, throughput=4.99MB/sec 2024-09-13T00:12:12.629 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-eLAz410ggJeB8ag off=915196 len=1316828 2024-09-13T00:12:12.640 INFO:tasks.workunit.client.0.smithi136.stdout:op 3297 completed, throughput=5MB/sec 2024-09-13T00:12:12.640 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Q1U0oIRw5dd-6qc off=12469381 len=672525 2024-09-13T00:12:12.642 INFO:tasks.workunit.client.0.smithi136.stdout:op 3298 completed, throughput=5MB/sec 2024-09-13T00:12:12.642 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-HJpIXvifJ3XH16j off=9449894 len=1938531 2024-09-13T00:12:12.656 INFO:tasks.workunit.client.0.smithi136.stdout:op 3299 completed, throughput=5MB/sec 2024-09-13T00:12:12.656 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-iBU6t0TccCgPeGf off=587271 len=71660 2024-09-13T00:12:12.657 INFO:tasks.workunit.client.0.smithi136.stdout:op 3300 completed, throughput=5MB/sec 2024-09-13T00:12:12.657 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ODEwtwF4fSC9VfL off=13590958 len=1144464 2024-09-13T00:12:12.660 INFO:tasks.workunit.client.0.smithi136.stdout:op 3301 completed, throughput=5MB/sec 2024-09-13T00:12:13.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:12:12 smithi181 bash[67587]: cluster 2024-09-13T00:12:11.469230+0000 mgr.y (mgr.24661) 1052 : cluster [DBG] pgmap v1735: 105 pgs: 105 active+clean; 126 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:12:13.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:12:12 smithi136 bash[75164]: cluster 2024-09-13T00:12:11.469230+0000 mgr.y (mgr.24661) 1052 : cluster [DBG] pgmap v1735: 105 pgs: 105 active+clean; 126 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:12:13.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:12:12 smithi136 bash[76371]: cluster 2024-09-13T00:12:11.469230+0000 mgr.y (mgr.24661) 1052 : cluster [DBG] pgmap v1735: 105 pgs: 105 active+clean; 126 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:12:13.661 INFO:tasks.workunit.client.0.smithi136.stdout: 630: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:13.661 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-C6QSWd9IYtTS_Sr off=248550 len=336188 2024-09-13T00:12:13.666 INFO:tasks.workunit.client.0.smithi136.stdout:op 3302 completed, throughput=4.99MB/sec 2024-09-13T00:12:13.666 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-lCbMhaUVy2mBawX off=3052904 len=408586 2024-09-13T00:12:13.668 INFO:tasks.workunit.client.0.smithi136.stdout:op 3303 completed, throughput=4.99MB/sec 2024-09-13T00:12:13.668 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-EGosW2Jxa5sxppc off=16849428 len=796716 2024-09-13T00:12:13.671 INFO:tasks.workunit.client.0.smithi136.stdout:op 3304 completed, throughput=5MB/sec 2024-09-13T00:12:13.671 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-WBFjuw3R3RTobgK off=5605765 len=1233886 2024-09-13T00:12:13.675 INFO:tasks.workunit.client.0.smithi136.stdout:op 3305 completed, throughput=5MB/sec 2024-09-13T00:12:13.675 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-1brZAYUF6h3aS_o off=3481644 len=1509363 2024-09-13T00:12:13.679 INFO:tasks.workunit.client.0.smithi136.stdout:op 3306 completed, throughput=5MB/sec 2024-09-13T00:12:13.679 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-VZS8SFDnhOK_ouJ off=2494728 len=555465 2024-09-13T00:12:13.681 INFO:tasks.workunit.client.0.smithi136.stdout:op 3307 completed, throughput=5MB/sec 2024-09-13T00:12:14.681 INFO:tasks.workunit.client.0.smithi136.stdout: 631: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:14.681 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-4oztVgkiLBHG1CV off=2980280 len=1362416 2024-09-13T00:12:14.684 INFO:tasks.workunit.client.0.smithi136.stdout:op 3308 completed, throughput=4.99MB/sec 2024-09-13T00:12:14.685 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-5v2CigT4BZGliyN off=2221534 len=975062 2024-09-13T00:12:14.687 INFO:tasks.workunit.client.0.smithi136.stdout:op 3309 completed, throughput=5MB/sec 2024-09-13T00:12:14.687 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-tj_naRVQtKVSDrR off=8692505 len=1091896 2024-09-13T00:12:14.697 INFO:tasks.workunit.client.0.smithi136.stdout:op 3310 completed, throughput=5MB/sec 2024-09-13T00:12:14.697 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-AYOfOPhEZYjvPP0 off=1896108 len=942661 2024-09-13T00:12:14.699 INFO:tasks.workunit.client.0.smithi136.stdout:op 3311 completed, throughput=5MB/sec 2024-09-13T00:12:14.700 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-zgC8WIhXcClE0Xo off=15052699 len=2064820 2024-09-13T00:12:14.706 INFO:tasks.workunit.client.0.smithi136.stdout:op 3312 completed, throughput=5MB/sec 2024-09-13T00:12:15.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:12:14 smithi181 bash[67587]: cluster 2024-09-13T00:12:13.470214+0000 mgr.y (mgr.24661) 1053 : cluster [DBG] pgmap v1736: 105 pgs: 105 active+clean; 126 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:12:15.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:12:14 smithi136 bash[75164]: cluster 2024-09-13T00:12:13.470214+0000 mgr.y (mgr.24661) 1053 : cluster [DBG] pgmap v1736: 105 pgs: 105 active+clean; 126 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:12:15.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:12:14 smithi136 bash[76371]: cluster 2024-09-13T00:12:13.470214+0000 mgr.y (mgr.24661) 1053 : cluster [DBG] pgmap v1736: 105 pgs: 105 active+clean; 126 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:12:15.706 INFO:tasks.workunit.client.0.smithi136.stdout: 632: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:15.706 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-YGzEEJZBZTep4Wk off=5970260 len=1898691 2024-09-13T00:12:15.722 INFO:tasks.workunit.client.0.smithi136.stdout:op 3313 completed, throughput=5MB/sec 2024-09-13T00:12:15.722 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Up6zA52YECPvLZc off=677072 len=2076848 2024-09-13T00:12:15.727 INFO:tasks.workunit.client.0.smithi136.stdout:op 3314 completed, throughput=5MB/sec 2024-09-13T00:12:15.727 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-U7xQjByZE956jl7 off=312054 len=1910645 2024-09-13T00:12:15.731 INFO:tasks.workunit.client.0.smithi136.stdout:op 3315 completed, throughput=5MB/sec 2024-09-13T00:12:16.732 INFO:tasks.workunit.client.0.smithi136.stdout: 633: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:16.732 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-scoVShsVCfEWnl9 off=9416386 len=581053 2024-09-13T00:12:16.733 INFO:tasks.workunit.client.0.smithi136.stdout:op 3316 completed, throughput=5MB/sec 2024-09-13T00:12:16.733 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-MeYlnad-uFalwIg off=2392536 len=441444 2024-09-13T00:12:16.740 INFO:tasks.workunit.client.0.smithi136.stdout:op 3317 completed, throughput=5MB/sec 2024-09-13T00:12:16.740 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-WqwetPXjI6y67PF off=816376 len=672715 2024-09-13T00:12:16.747 INFO:tasks.workunit.client.0.smithi136.stdout:op 3318 completed, throughput=5MB/sec 2024-09-13T00:12:16.747 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-e20ibhOOivqiRc9 off=333604 len=1080610 2024-09-13T00:12:16.754 INFO:tasks.workunit.client.0.smithi136.stdout:op 3319 completed, throughput=5MB/sec 2024-09-13T00:12:16.755 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-pWyued40TQwPBuN off=10102339 len=383456 2024-09-13T00:12:16.756 INFO:tasks.workunit.client.0.smithi136.stdout:op 3320 completed, throughput=5MB/sec 2024-09-13T00:12:16.756 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-HjAwxHsH49WuyHY off=1779494 len=298966 2024-09-13T00:12:16.761 INFO:tasks.workunit.client.0.smithi136.stdout:op 3321 completed, throughput=5MB/sec 2024-09-13T00:12:16.761 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Q-KSbcTJPXVfIJw off=3686643 len=713173 2024-09-13T00:12:16.764 INFO:tasks.workunit.client.0.smithi136.stdout:op 3322 completed, throughput=5MB/sec 2024-09-13T00:12:17.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:12:16 smithi181 bash[67587]: cluster 2024-09-13T00:12:15.471085+0000 mgr.y (mgr.24661) 1054 : cluster [DBG] pgmap v1737: 105 pgs: 105 active+clean; 126 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:12:17.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:12:16 smithi136 bash[75164]: cluster 2024-09-13T00:12:15.471085+0000 mgr.y (mgr.24661) 1054 : cluster [DBG] pgmap v1737: 105 pgs: 105 active+clean; 126 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:12:17.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:12:16 smithi136 bash[76371]: cluster 2024-09-13T00:12:15.471085+0000 mgr.y (mgr.24661) 1054 : cluster [DBG] pgmap v1737: 105 pgs: 105 active+clean; 126 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:12:17.764 INFO:tasks.workunit.client.0.smithi136.stdout: 634: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:17.765 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ZRK4GFXKBhE8JNY off=16493384 len=1006483 2024-09-13T00:12:17.773 INFO:tasks.workunit.client.0.smithi136.stdout:op 3323 completed, throughput=4.99MB/sec 2024-09-13T00:12:17.773 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-X0xcNBey45j0gJ6 off=2883927 len=1729491 2024-09-13T00:12:17.777 INFO:tasks.workunit.client.0.smithi136.stdout:op 3324 completed, throughput=5MB/sec 2024-09-13T00:12:17.777 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-0DkP86A8FbM3Lbr off=76894 len=571639 2024-09-13T00:12:17.779 INFO:tasks.workunit.client.0.smithi136.stdout:op 3325 completed, throughput=5MB/sec 2024-09-13T00:12:17.779 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-oZ-JgNkNHzrtjVh off=18555907 len=943072 2024-09-13T00:12:17.781 INFO:tasks.workunit.client.0.smithi136.stdout:op 3326 completed, throughput=5MB/sec 2024-09-13T00:12:17.781 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Kx4LUdSgJxdNytU off=3683719 len=1258893 2024-09-13T00:12:17.784 INFO:tasks.workunit.client.0.smithi136.stdout:op 3327 completed, throughput=5MB/sec 2024-09-13T00:12:18.785 INFO:tasks.workunit.client.0.smithi136.stdout: 635: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:18.785 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-8Ia2VbwB5I0ihJp off=6830282 len=639950 2024-09-13T00:12:18.788 INFO:tasks.workunit.client.0.smithi136.stdout:op 3328 completed, throughput=4.99MB/sec 2024-09-13T00:12:18.788 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-GQ3S7SZYw1AGX0W off=13903266 len=1967498 2024-09-13T00:12:18.800 INFO:tasks.workunit.client.0.smithi136.stdout:op 3329 completed, throughput=5MB/sec 2024-09-13T00:12:18.801 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Q4Ex2saQOrZNhDh off=3660087 len=1025890 2024-09-13T00:12:18.804 INFO:tasks.workunit.client.0.smithi136.stdout:op 3330 completed, throughput=5MB/sec 2024-09-13T00:12:18.804 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-EKVrGQ5fVOj4cjj off=1868234 len=584164 2024-09-13T00:12:18.811 INFO:tasks.workunit.client.0.smithi136.stdout:op 3331 completed, throughput=5MB/sec 2024-09-13T00:12:18.811 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Gl4v1l_mlgOykX6 off=3084728 len=531058 2024-09-13T00:12:18.812 INFO:tasks.workunit.client.0.smithi136.stdout:op 3332 completed, throughput=5MB/sec 2024-09-13T00:12:18.812 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-J2855SUuba979zv off=10166022 len=1654460 2024-09-13T00:12:18.825 INFO:tasks.workunit.client.0.smithi136.stdout:op 3333 completed, throughput=5MB/sec 2024-09-13T00:12:19.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:12:18 smithi181 bash[67587]: cluster 2024-09-13T00:12:17.472219+0000 mgr.y (mgr.24661) 1055 : cluster [DBG] pgmap v1738: 105 pgs: 105 active+clean; 126 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.2 MiB/s wr, 5 op/s 2024-09-13T00:12:19.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:12:18 smithi136 bash[75164]: cluster 2024-09-13T00:12:17.472219+0000 mgr.y (mgr.24661) 1055 : cluster [DBG] pgmap v1738: 105 pgs: 105 active+clean; 126 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.2 MiB/s wr, 5 op/s 2024-09-13T00:12:19.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:12:18 smithi136 bash[76371]: cluster 2024-09-13T00:12:17.472219+0000 mgr.y (mgr.24661) 1055 : cluster [DBG] pgmap v1738: 105 pgs: 105 active+clean; 126 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.2 MiB/s wr, 5 op/s 2024-09-13T00:12:19.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:12:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:12:19] "GET /metrics HTTP/1.1" 200 34139 "" "Prometheus/2.43.0" 2024-09-13T00:12:19.825 INFO:tasks.workunit.client.0.smithi136.stdout: 636: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:19.825 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-mFPBiq7SGvprmf9 off=10220931 len=1035920 2024-09-13T00:12:19.836 INFO:tasks.workunit.client.0.smithi136.stdout:op 3334 completed, throughput=5MB/sec 2024-09-13T00:12:19.836 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-3Up6DYjn8sMBcOD off=5670233 len=474052 2024-09-13T00:12:19.842 INFO:tasks.workunit.client.0.smithi136.stdout:op 3335 completed, throughput=5MB/sec 2024-09-13T00:12:19.842 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-MSCkM-RjSOA4sja off=13507802 len=577820 2024-09-13T00:12:19.849 INFO:tasks.workunit.client.0.smithi136.stdout:op 3336 completed, throughput=5MB/sec 2024-09-13T00:12:19.849 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-OpOrLwbEyFcLWUu off=1666754 len=1612906 2024-09-13T00:12:19.853 INFO:tasks.workunit.client.0.smithi136.stdout:op 3337 completed, throughput=5MB/sec 2024-09-13T00:12:19.853 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-fE7AhANJrz2ORfJ off=1029489 len=804362 2024-09-13T00:12:19.856 INFO:tasks.workunit.client.0.smithi136.stdout:op 3338 completed, throughput=5MB/sec 2024-09-13T00:12:20.857 INFO:tasks.workunit.client.0.smithi136.stdout: 637: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:20.857 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-tk0rUWqhHxiBFPN off=16440004 len=1941918 2024-09-13T00:12:20.871 INFO:tasks.workunit.client.0.smithi136.stdout:op 3339 completed, throughput=5MB/sec 2024-09-13T00:12:20.871 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Dn-fKcCrxcAzP9z off=17859532 len=600316 2024-09-13T00:12:20.873 INFO:tasks.workunit.client.0.smithi136.stdout:op 3340 completed, throughput=5MB/sec 2024-09-13T00:12:20.873 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-oPnVTv381LVG16H off=1425380 len=1581734 2024-09-13T00:12:20.877 INFO:tasks.workunit.client.0.smithi136.stdout:op 3341 completed, throughput=5MB/sec 2024-09-13T00:12:20.877 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-7eFqvR1V5YAi900 off=5087680 len=1847954 2024-09-13T00:12:20.889 INFO:tasks.workunit.client.0.smithi136.stdout:op 3342 completed, throughput=5MB/sec 2024-09-13T00:12:21.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:12:20 smithi136 bash[75164]: cluster 2024-09-13T00:12:19.473140+0000 mgr.y (mgr.24661) 1056 : cluster [DBG] pgmap v1739: 105 pgs: 105 active+clean; 126 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:12:21.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:12:20 smithi136 bash[76371]: cluster 2024-09-13T00:12:19.473140+0000 mgr.y (mgr.24661) 1056 : cluster [DBG] pgmap v1739: 105 pgs: 105 active+clean; 126 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:12:21.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:12:20 smithi181 bash[67587]: cluster 2024-09-13T00:12:19.473140+0000 mgr.y (mgr.24661) 1056 : cluster [DBG] pgmap v1739: 105 pgs: 105 active+clean; 126 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:12:21.889 INFO:tasks.workunit.client.0.smithi136.stdout: 638: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:21.890 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-EQsYq2FzX2m7zPR off=10033499 len=1940385 2024-09-13T00:12:21.902 INFO:tasks.workunit.client.0.smithi136.stdout:op 3343 completed, throughput=5MB/sec 2024-09-13T00:12:21.902 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-dNbsvMQ7F_a7XR3 off=8156086 len=885568 2024-09-13T00:12:21.911 INFO:tasks.workunit.client.0.smithi136.stdout:op 3344 completed, throughput=5MB/sec 2024-09-13T00:12:21.911 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-9TiLHtw_Ug5LYGQ off=5489713 len=1541351 2024-09-13T00:12:21.914 INFO:tasks.workunit.client.0.smithi136.stdout:op 3345 completed, throughput=5MB/sec 2024-09-13T00:12:22.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:12:22 smithi181 bash[95505]: ts=2024-09-13T00:12:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:12:22.915 INFO:tasks.workunit.client.0.smithi136.stdout: 639: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:22.915 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-hVwi8UJbDD6Iy55 off=3717887 len=1856901 2024-09-13T00:12:22.919 INFO:tasks.workunit.client.0.smithi136.stdout:op 3346 completed, throughput=5MB/sec 2024-09-13T00:12:22.919 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-d9Qd0lkyoY4n7pH off=2945185 len=741486 2024-09-13T00:12:22.921 INFO:tasks.workunit.client.0.smithi136.stdout:op 3347 completed, throughput=5MB/sec 2024-09-13T00:12:22.922 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-IAS5hp9xo-N1rNA off=124534 len=1595902 2024-09-13T00:12:22.926 INFO:tasks.workunit.client.0.smithi136.stdout:op 3348 completed, throughput=5MB/sec 2024-09-13T00:12:22.926 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ivM3RB98_aqv07v off=3193479 len=2069008 2024-09-13T00:12:22.939 INFO:tasks.workunit.client.0.smithi136.stdout:op 3349 completed, throughput=5MB/sec 2024-09-13T00:12:23.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:12:22 smithi136 bash[75164]: cluster 2024-09-13T00:12:21.474160+0000 mgr.y (mgr.24661) 1057 : cluster [DBG] pgmap v1740: 105 pgs: 105 active+clean; 126 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.1 MiB/s wr, 5 op/s 2024-09-13T00:12:23.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:12:22 smithi136 bash[76371]: cluster 2024-09-13T00:12:21.474160+0000 mgr.y (mgr.24661) 1057 : cluster [DBG] pgmap v1740: 105 pgs: 105 active+clean; 126 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.1 MiB/s wr, 5 op/s 2024-09-13T00:12:23.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:12:22 smithi181 bash[67587]: cluster 2024-09-13T00:12:21.474160+0000 mgr.y (mgr.24661) 1057 : cluster [DBG] pgmap v1740: 105 pgs: 105 active+clean; 126 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.1 MiB/s wr, 5 op/s 2024-09-13T00:12:23.939 INFO:tasks.workunit.client.0.smithi136.stdout: 640: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:23.939 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-PDdztB9MhbmkDn5 off=2485481 len=769247 2024-09-13T00:12:23.942 INFO:tasks.workunit.client.0.smithi136.stdout:op 3350 completed, throughput=5MB/sec 2024-09-13T00:12:23.942 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-CecYVQLJAXecOqO off=4308348 len=1085246 2024-09-13T00:12:23.952 INFO:tasks.workunit.client.0.smithi136.stdout:op 3351 completed, throughput=5MB/sec 2024-09-13T00:12:23.952 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-vtR4CbPA0vX7iy_ off=2997940 len=1104829 2024-09-13T00:12:23.955 INFO:tasks.workunit.client.0.smithi136.stdout:op 3352 completed, throughput=5MB/sec 2024-09-13T00:12:23.955 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-j2tEz34OQHfgL-N off=9981962 len=1432586 2024-09-13T00:12:23.964 INFO:tasks.workunit.client.0.smithi136.stdout:op 3353 completed, throughput=5MB/sec 2024-09-13T00:12:24.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:12:23 smithi136 bash[75164]: audit 2024-09-13T00:12:23.789869+0000 mon.a (mon.0) 2205 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:12:24.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:12:23 smithi136 bash[76371]: audit 2024-09-13T00:12:23.789869+0000 mon.a (mon.0) 2205 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:12:24.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:12:23 smithi181 bash[67587]: audit 2024-09-13T00:12:23.789869+0000 mon.a (mon.0) 2205 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:12:24.965 INFO:tasks.workunit.client.0.smithi136.stdout: 641: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:24.965 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-BUQXSVpwEiiz8ZW off=992055 len=403693 2024-09-13T00:12:24.971 INFO:tasks.workunit.client.0.smithi136.stdout:op 3354 completed, throughput=4.99MB/sec 2024-09-13T00:12:24.971 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Ud1-aswa1Z8oZE3 off=5706284 len=900765 2024-09-13T00:12:24.973 INFO:tasks.workunit.client.0.smithi136.stdout:op 3355 completed, throughput=4.99MB/sec 2024-09-13T00:12:24.974 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-wQahq6PfOyGh-im off=2649931 len=228391 2024-09-13T00:12:24.975 INFO:tasks.workunit.client.0.smithi136.stdout:op 3356 completed, throughput=4.99MB/sec 2024-09-13T00:12:24.975 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-0NaNbUVFoGCmK0W off=1591595 len=695362 2024-09-13T00:12:24.978 INFO:tasks.workunit.client.0.smithi136.stdout:op 3357 completed, throughput=5MB/sec 2024-09-13T00:12:24.978 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-maVUnNqgLVWwY6J off=5086115 len=8721 2024-09-13T00:12:24.979 INFO:tasks.workunit.client.0.smithi136.stdout:op 3358 completed, throughput=5MB/sec 2024-09-13T00:12:24.980 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-uwDDvzcN__SUDpg off=6750230 len=1064469 2024-09-13T00:12:24.987 INFO:tasks.workunit.client.0.smithi136.stdout:op 3359 completed, throughput=5MB/sec 2024-09-13T00:12:24.987 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-6NaOELq6igKBZMu off=16399554 len=292374 2024-09-13T00:12:24.992 INFO:tasks.workunit.client.0.smithi136.stdout:op 3360 completed, throughput=5MB/sec 2024-09-13T00:12:24.993 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-itFNE7K6O4K5Vhu off=14342368 len=203365 2024-09-13T00:12:24.998 INFO:tasks.workunit.client.0.smithi136.stdout:op 3361 completed, throughput=5MB/sec 2024-09-13T00:12:24.998 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-oyxcC0Ban7YeIni off=19596913 len=1270995 2024-09-13T00:12:25.001 INFO:tasks.workunit.client.0.smithi136.stdout:op 3362 completed, throughput=5MB/sec 2024-09-13T00:12:25.001 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-1dy6CFhkoeWPoh9 off=10869738 len=1697248 2024-09-13T00:12:25.005 INFO:tasks.workunit.client.0.smithi136.stdout:op 3363 completed, throughput=5MB/sec 2024-09-13T00:12:25.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:12:24 smithi136 bash[75164]: cluster 2024-09-13T00:12:23.475106+0000 mgr.y (mgr.24661) 1058 : cluster [DBG] pgmap v1741: 105 pgs: 105 active+clean; 126 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.9 MiB/s wr, 4 op/s 2024-09-13T00:12:25.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:12:24 smithi136 bash[76371]: cluster 2024-09-13T00:12:23.475106+0000 mgr.y (mgr.24661) 1058 : cluster [DBG] pgmap v1741: 105 pgs: 105 active+clean; 126 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.9 MiB/s wr, 4 op/s 2024-09-13T00:12:25.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:12:24 smithi181 bash[67587]: cluster 2024-09-13T00:12:23.475106+0000 mgr.y (mgr.24661) 1058 : cluster [DBG] pgmap v1741: 105 pgs: 105 active+clean; 126 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.9 MiB/s wr, 4 op/s 2024-09-13T00:12:26.005 INFO:tasks.workunit.client.0.smithi136.stdout: 642: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:26.006 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-FKxLmmdho2AqZ6O off=3971056 len=1514987 2024-09-13T00:12:26.020 INFO:tasks.workunit.client.0.smithi136.stdout:op 3364 completed, throughput=5MB/sec 2024-09-13T00:12:26.020 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-oPUbrXg3zHQ1KRy off=876281 len=1112622 2024-09-13T00:12:26.023 INFO:tasks.workunit.client.0.smithi136.stdout:op 3365 completed, throughput=5MB/sec 2024-09-13T00:12:26.023 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-fnl1QS3QR3VSY5J off=757377 len=817583 2024-09-13T00:12:26.026 INFO:tasks.workunit.client.0.smithi136.stdout:op 3366 completed, throughput=5MB/sec 2024-09-13T00:12:26.026 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Ko49jtRMkDsHGi8 off=3246497 len=1003522 2024-09-13T00:12:26.036 INFO:tasks.workunit.client.0.smithi136.stdout:op 3367 completed, throughput=5MB/sec 2024-09-13T00:12:27.036 INFO:tasks.workunit.client.0.smithi136.stdout: 643: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:27.036 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-SOoP4KohqOBHTQq off=19834118 len=633268 2024-09-13T00:12:27.039 INFO:tasks.workunit.client.0.smithi136.stdout:op 3368 completed, throughput=4.99MB/sec 2024-09-13T00:12:27.039 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-A2A-OLyL83m3raB off=2190513 len=972475 2024-09-13T00:12:27.047 INFO:tasks.workunit.client.0.smithi136.stdout:op 3369 completed, throughput=5MB/sec 2024-09-13T00:12:27.047 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj--FR6ZnrIyq3vebz off=5303041 len=1495863 2024-09-13T00:12:27.051 INFO:tasks.workunit.client.0.smithi136.stdout:op 3370 completed, throughput=5MB/sec 2024-09-13T00:12:27.052 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-WWsNNxmFUKTCK1t off=13159229 len=30675 2024-09-13T00:12:27.056 INFO:tasks.workunit.client.0.smithi136.stdout:op 3371 completed, throughput=5MB/sec 2024-09-13T00:12:27.056 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-aFMZHX4csyELGXr off=17143067 len=1405018 2024-09-13T00:12:27.070 INFO:tasks.workunit.client.0.smithi136.stdout:op 3372 completed, throughput=5MB/sec 2024-09-13T00:12:27.070 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-GQ3S7SZYw1AGX0W off=6339425 len=17161 2024-09-13T00:12:27.074 INFO:tasks.workunit.client.0.smithi136.stdout:op 3373 completed, throughput=5MB/sec 2024-09-13T00:12:27.074 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ciCdyavLLalzlLD off=414695 len=193192 2024-09-13T00:12:27.075 INFO:tasks.workunit.client.0.smithi136.stdout:op 3374 completed, throughput=5MB/sec 2024-09-13T00:12:27.075 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-BXGqdU5ZdaggRZU off=2385387 len=1226627 2024-09-13T00:12:27.078 INFO:tasks.workunit.client.0.smithi136.stdout:op 3375 completed, throughput=5MB/sec 2024-09-13T00:12:27.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:12:26 smithi136 bash[75164]: cluster 2024-09-13T00:12:25.475945+0000 mgr.y (mgr.24661) 1059 : cluster [DBG] pgmap v1742: 105 pgs: 105 active+clean; 126 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 3.0 MiB/s rd, 1.8 MiB/s wr, 4 op/s 2024-09-13T00:12:27.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:12:26 smithi136 bash[76371]: cluster 2024-09-13T00:12:25.475945+0000 mgr.y (mgr.24661) 1059 : cluster [DBG] pgmap v1742: 105 pgs: 105 active+clean; 126 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 3.0 MiB/s rd, 1.8 MiB/s wr, 4 op/s 2024-09-13T00:12:27.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:12:26 smithi181 bash[67587]: cluster 2024-09-13T00:12:25.475945+0000 mgr.y (mgr.24661) 1059 : cluster [DBG] pgmap v1742: 105 pgs: 105 active+clean; 126 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 3.0 MiB/s rd, 1.8 MiB/s wr, 4 op/s 2024-09-13T00:12:28.079 INFO:tasks.workunit.client.0.smithi136.stdout: 644: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:28.079 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-juH5w0y2xGlinWt off=1183801 len=499644 2024-09-13T00:12:28.085 INFO:tasks.workunit.client.0.smithi136.stdout:op 3376 completed, throughput=4.99MB/sec 2024-09-13T00:12:28.085 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-KeHB2S89HmvcNga off=1894680 len=461685 2024-09-13T00:12:28.087 INFO:tasks.workunit.client.0.smithi136.stdout:op 3377 completed, throughput=5MB/sec 2024-09-13T00:12:28.087 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-vZUKZycyYOHRTFT off=0 len=1064565 2024-09-13T00:12:28.090 INFO:tasks.workunit.client.0.smithi136.stdout:op 3378 completed, throughput=5MB/sec 2024-09-13T00:12:28.090 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Eo20Msjb4CyMRSU off=5529075 len=396113 2024-09-13T00:12:28.096 INFO:tasks.workunit.client.0.smithi136.stdout:op 3379 completed, throughput=5MB/sec 2024-09-13T00:12:28.096 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-UBUaJxk64n7xO6F off=4212437 len=817414 2024-09-13T00:12:28.099 INFO:tasks.workunit.client.0.smithi136.stdout:op 3380 completed, throughput=5MB/sec 2024-09-13T00:12:28.099 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-CI5gzV8wgtWLtSU off=2906368 len=862185 2024-09-13T00:12:28.101 INFO:tasks.workunit.client.0.smithi136.stdout:op 3381 completed, throughput=5MB/sec 2024-09-13T00:12:28.101 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-imLlCfSip9PVKHv off=3277223 len=703678 2024-09-13T00:12:28.103 INFO:tasks.workunit.client.0.smithi136.stdout:op 3382 completed, throughput=5MB/sec 2024-09-13T00:12:29.104 INFO:tasks.workunit.client.0.smithi136.stdout: 645: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:29.104 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-hVxhF-vvrusx23W off=2758227 len=1444540 2024-09-13T00:12:29.114 INFO:tasks.workunit.client.0.smithi136.stdout:op 3383 completed, throughput=4.99MB/sec 2024-09-13T00:12:29.114 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-X6B7FyOR0jPtuaK off=471748 len=251457 2024-09-13T00:12:29.115 INFO:tasks.workunit.client.0.smithi136.stdout:op 3384 completed, throughput=5MB/sec 2024-09-13T00:12:29.115 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2CpK8DywGhBXsPU off=10046608 len=303841 2024-09-13T00:12:29.116 INFO:tasks.workunit.client.0.smithi136.stdout:op 3385 completed, throughput=5MB/sec 2024-09-13T00:12:29.116 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-SOoP4KohqOBHTQq off=15141926 len=1622110 2024-09-13T00:12:29.120 INFO:tasks.workunit.client.0.smithi136.stdout:op 3386 completed, throughput=5MB/sec 2024-09-13T00:12:29.120 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-XIE9PdvTVzP9uxH off=15513222 len=1726457 2024-09-13T00:12:29.132 INFO:tasks.workunit.client.0.smithi136.stdout:op 3387 completed, throughput=5MB/sec 2024-09-13T00:12:29.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:12:28 smithi136 bash[76371]: cluster 2024-09-13T00:12:27.477097+0000 mgr.y (mgr.24661) 1060 : cluster [DBG] pgmap v1743: 105 pgs: 105 active+clean; 126 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:12:29.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:12:28 smithi136 bash[75164]: cluster 2024-09-13T00:12:27.477097+0000 mgr.y (mgr.24661) 1060 : cluster [DBG] pgmap v1743: 105 pgs: 105 active+clean; 126 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:12:29.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:12:28 smithi181 bash[67587]: cluster 2024-09-13T00:12:27.477097+0000 mgr.y (mgr.24661) 1060 : cluster [DBG] pgmap v1743: 105 pgs: 105 active+clean; 126 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:12:29.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:12:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:12:29] "GET /metrics HTTP/1.1" 200 34140 "" "Prometheus/2.43.0" 2024-09-13T00:12:30.132 INFO:tasks.workunit.client.0.smithi136.stdout: 646: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:30.132 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-WBFjuw3R3RTobgK off=12064541 len=1141234 2024-09-13T00:12:30.145 INFO:tasks.workunit.client.0.smithi136.stdout:op 3388 completed, throughput=4.99MB/sec 2024-09-13T00:12:30.145 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-QUPIS8DJn1tfzXj off=1560624 len=521489 2024-09-13T00:12:30.151 INFO:tasks.workunit.client.0.smithi136.stdout:op 3389 completed, throughput=5MB/sec 2024-09-13T00:12:30.151 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-5bgfeM0UoZQRU3O off=823721 len=1335163 2024-09-13T00:12:30.155 INFO:tasks.workunit.client.0.smithi136.stdout:op 3390 completed, throughput=5MB/sec 2024-09-13T00:12:30.155 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-5GRFEM1q85HoVJ0 off=1513530 len=1783149 2024-09-13T00:12:30.170 INFO:tasks.workunit.client.0.smithi136.stdout:op 3391 completed, throughput=5MB/sec 2024-09-13T00:12:30.170 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-B6QZZoHSkEWguTT off=3275871 len=1733658 2024-09-13T00:12:30.175 INFO:tasks.workunit.client.0.smithi136.stdout:op 3392 completed, throughput=5MB/sec 2024-09-13T00:12:31.175 INFO:tasks.workunit.client.0.smithi136.stdout: 647: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:31.175 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-5JBLDmLY0p3Llvr off=11387467 len=1703581 2024-09-13T00:12:31.188 INFO:tasks.workunit.client.0.smithi136.stdout:op 3393 completed, throughput=5MB/sec 2024-09-13T00:12:31.189 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-2tiq5zZUOifRzhY off=23123585 len=1756153 2024-09-13T00:12:31.203 INFO:tasks.workunit.client.0.smithi136.stdout:op 3394 completed, throughput=5MB/sec 2024-09-13T00:12:31.203 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_bka397JcRppLwg off=1665068 len=1401345 2024-09-13T00:12:31.213 INFO:tasks.workunit.client.0.smithi136.stdout:op 3395 completed, throughput=5MB/sec 2024-09-13T00:12:31.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:12:30 smithi136 bash[75164]: cluster 2024-09-13T00:12:29.478022+0000 mgr.y (mgr.24661) 1061 : cluster [DBG] pgmap v1744: 105 pgs: 105 active+clean; 126 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:12:31.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:12:30 smithi136 bash[76371]: cluster 2024-09-13T00:12:29.478022+0000 mgr.y (mgr.24661) 1061 : cluster [DBG] pgmap v1744: 105 pgs: 105 active+clean; 126 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:12:31.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:12:30 smithi181 bash[67587]: cluster 2024-09-13T00:12:29.478022+0000 mgr.y (mgr.24661) 1061 : cluster [DBG] pgmap v1744: 105 pgs: 105 active+clean; 126 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:12:32.214 INFO:tasks.workunit.client.0.smithi136.stdout: 648: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:32.214 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-0VY8v5CHQZxM749 off=2091969 len=751236 2024-09-13T00:12:32.222 INFO:tasks.workunit.client.0.smithi136.stdout:op 3396 completed, throughput=4.99MB/sec 2024-09-13T00:12:32.222 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-2ZCCXZi8bc7Uu1X off=3497200 len=1563642 2024-09-13T00:12:32.236 INFO:tasks.workunit.client.0.smithi136.stdout:op 3397 completed, throughput=5MB/sec 2024-09-13T00:12:32.237 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-VYj7sJO5IEJa07E off=9978220 len=789698 2024-09-13T00:12:32.239 INFO:tasks.workunit.client.0.smithi136.stdout:op 3398 completed, throughput=5MB/sec 2024-09-13T00:12:32.239 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Q7X_sR0vBFLGNkF off=2463577 len=953603 2024-09-13T00:12:32.247 INFO:tasks.workunit.client.0.smithi136.stdout:op 3399 completed, throughput=5MB/sec 2024-09-13T00:12:32.247 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-rQZ2hT1gVSiJzM0 off=732916 len=1599570 2024-09-13T00:12:32.256 INFO:tasks.workunit.client.0.smithi136.stdout:op 3400 completed, throughput=5MB/sec 2024-09-13T00:12:32.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:12:32 smithi181 bash[95505]: ts=2024-09-13T00:12:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:12:33.257 INFO:tasks.workunit.client.0.smithi136.stdout: 649: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:33.257 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-hFc7egzQ3uiVRMB off=3588990 len=176365 2024-09-13T00:12:33.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:12:32 smithi136 bash[76371]: cluster 2024-09-13T00:12:31.479058+0000 mgr.y (mgr.24661) 1062 : cluster [DBG] pgmap v1745: 105 pgs: 105 active+clean; 126 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:12:33.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:12:32 smithi136 bash[75164]: cluster 2024-09-13T00:12:31.479058+0000 mgr.y (mgr.24661) 1062 : cluster [DBG] pgmap v1745: 105 pgs: 105 active+clean; 126 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:12:33.262 INFO:tasks.workunit.client.0.smithi136.stdout:op 3401 completed, throughput=4.99MB/sec 2024-09-13T00:12:33.262 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-1cmc_wCWjBZmVX4 off=13162068 len=261405 2024-09-13T00:12:33.263 INFO:tasks.workunit.client.0.smithi136.stdout:op 3402 completed, throughput=4.99MB/sec 2024-09-13T00:12:33.263 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-YU-mm3ECKrsJn_T off=10402142 len=872708 2024-09-13T00:12:33.270 INFO:tasks.workunit.client.0.smithi136.stdout:op 3403 completed, throughput=5MB/sec 2024-09-13T00:12:33.271 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Fk-UyVPO7-gEwE2 off=1006412 len=1148895 2024-09-13T00:12:33.275 INFO:tasks.workunit.client.0.smithi136.stdout:op 3404 completed, throughput=5MB/sec 2024-09-13T00:12:33.275 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-33ugmjuzqilophS off=14083136 len=442240 2024-09-13T00:12:33.280 INFO:tasks.workunit.client.0.smithi136.stdout:op 3405 completed, throughput=5MB/sec 2024-09-13T00:12:33.281 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-QQnrG5vL6qzuIgi off=5483566 len=953243 2024-09-13T00:12:33.292 INFO:tasks.workunit.client.0.smithi136.stdout:op 3406 completed, throughput=5MB/sec 2024-09-13T00:12:33.292 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-BYchLNfZfcrbfsu off=11376444 len=1187981 2024-09-13T00:12:33.299 INFO:tasks.workunit.client.0.smithi136.stdout:op 3407 completed, throughput=5MB/sec 2024-09-13T00:12:33.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:12:32 smithi181 bash[67587]: cluster 2024-09-13T00:12:31.479058+0000 mgr.y (mgr.24661) 1062 : cluster [DBG] pgmap v1745: 105 pgs: 105 active+clean; 126 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:12:34.300 INFO:tasks.workunit.client.0.smithi136.stdout: 650: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:34.300 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-8_5fD51_2R-AV8D off=6283191 len=712082 2024-09-13T00:12:34.303 INFO:tasks.workunit.client.0.smithi136.stdout:op 3408 completed, throughput=4.99MB/sec 2024-09-13T00:12:34.304 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Gx1NURvsQRUck1B off=640697 len=2090698 2024-09-13T00:12:34.308 INFO:tasks.workunit.client.0.smithi136.stdout:op 3409 completed, throughput=5MB/sec 2024-09-13T00:12:34.309 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-kLQGCxcco91JEZ0 off=1423804 len=1749808 2024-09-13T00:12:34.313 INFO:tasks.workunit.client.0.smithi136.stdout:op 3410 completed, throughput=5MB/sec 2024-09-13T00:12:34.313 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-CFzy6Xuf-9OrBaA off=6141318 len=113015 2024-09-13T00:12:34.317 INFO:tasks.workunit.client.0.smithi136.stdout:op 3411 completed, throughput=5MB/sec 2024-09-13T00:12:34.317 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-tMcUWjJb27JUHd3 off=11693179 len=834751 2024-09-13T00:12:34.320 INFO:tasks.workunit.client.0.smithi136.stdout:op 3412 completed, throughput=5MB/sec 2024-09-13T00:12:35.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:12:34 smithi136 bash[76371]: cluster 2024-09-13T00:12:33.480019+0000 mgr.y (mgr.24661) 1063 : cluster [DBG] pgmap v1746: 105 pgs: 105 active+clean; 126 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:12:35.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:12:34 smithi136 bash[75164]: cluster 2024-09-13T00:12:33.480019+0000 mgr.y (mgr.24661) 1063 : cluster [DBG] pgmap v1746: 105 pgs: 105 active+clean; 126 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:12:35.320 INFO:tasks.workunit.client.0.smithi136.stdout: 651: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:35.320 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-hFQR9m0Q1KDe1nZ off=17083520 len=190802 2024-09-13T00:12:35.321 INFO:tasks.workunit.client.0.smithi136.stdout:op 3413 completed, throughput=4.99MB/sec 2024-09-13T00:12:35.322 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ZuwCHv5miUXlwER off=1584994 len=878359 2024-09-13T00:12:35.330 INFO:tasks.workunit.client.0.smithi136.stdout:op 3414 completed, throughput=4.99MB/sec 2024-09-13T00:12:35.330 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-lyzki7s7jUcuMWa off=8677259 len=1849161 2024-09-13T00:12:35.336 INFO:tasks.workunit.client.0.smithi136.stdout:op 3415 completed, throughput=5MB/sec 2024-09-13T00:12:35.336 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-pGocdpz05m7vw78 off=6604864 len=1595128 2024-09-13T00:12:35.340 INFO:tasks.workunit.client.0.smithi136.stdout:op 3416 completed, throughput=5MB/sec 2024-09-13T00:12:35.340 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-F1Kd-RcGZSLN6Eu off=3568538 len=1854751 2024-09-13T00:12:35.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:12:34 smithi181 bash[67587]: cluster 2024-09-13T00:12:33.480019+0000 mgr.y (mgr.24661) 1063 : cluster [DBG] pgmap v1746: 105 pgs: 105 active+clean; 126 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:12:35.356 INFO:tasks.workunit.client.0.smithi136.stdout:op 3417 completed, throughput=5MB/sec 2024-09-13T00:12:36.356 INFO:tasks.workunit.client.0.smithi136.stdout: 652: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:36.356 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-NDA6QKeM8HLmUDo off=984494 len=1886309 2024-09-13T00:12:36.361 INFO:tasks.workunit.client.0.smithi136.stdout:op 3418 completed, throughput=5MB/sec 2024-09-13T00:12:36.361 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-SgRAQAEiHDM1Mwr off=2629121 len=1331674 2024-09-13T00:12:36.364 INFO:tasks.workunit.client.0.smithi136.stdout:op 3419 completed, throughput=5MB/sec 2024-09-13T00:12:36.364 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-gQCn_DCpYbvDdng off=168632 len=924386 2024-09-13T00:12:36.368 INFO:tasks.workunit.client.0.smithi136.stdout:op 3420 completed, throughput=5MB/sec 2024-09-13T00:12:37.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:12:36 smithi136 bash[76371]: cluster 2024-09-13T00:12:35.480891+0000 mgr.y (mgr.24661) 1064 : cluster [DBG] pgmap v1747: 105 pgs: 105 active+clean; 126 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:12:37.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:12:36 smithi136 bash[75164]: cluster 2024-09-13T00:12:35.480891+0000 mgr.y (mgr.24661) 1064 : cluster [DBG] pgmap v1747: 105 pgs: 105 active+clean; 126 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:12:37.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:12:36 smithi181 bash[67587]: cluster 2024-09-13T00:12:35.480891+0000 mgr.y (mgr.24661) 1064 : cluster [DBG] pgmap v1747: 105 pgs: 105 active+clean; 126 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:12:37.368 INFO:tasks.workunit.client.0.smithi136.stdout: 653: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:37.368 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-FRfhXUqDMouaF8a off=5767806 len=1932398 2024-09-13T00:12:37.374 INFO:tasks.workunit.client.0.smithi136.stdout:op 3421 completed, throughput=5MB/sec 2024-09-13T00:12:37.374 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-RFXgO-jKCyOuhYc off=4080218 len=1879675 2024-09-13T00:12:37.392 INFO:tasks.workunit.client.0.smithi136.stdout:op 3422 completed, throughput=5MB/sec 2024-09-13T00:12:37.392 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-SwgtR06gSS1EcFd off=11090908 len=1186873 2024-09-13T00:12:37.403 INFO:tasks.workunit.client.0.smithi136.stdout:op 3423 completed, throughput=5MB/sec 2024-09-13T00:12:38.403 INFO:tasks.workunit.client.0.smithi136.stdout: 654: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:38.403 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-S15ChRN07Edf1Ac off=2423272 len=1362934 2024-09-13T00:12:38.416 INFO:tasks.workunit.client.0.smithi136.stdout:op 3424 completed, throughput=4.99MB/sec 2024-09-13T00:12:38.417 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-KSL0_tcOOio6G2w off=1054909 len=13259 2024-09-13T00:12:38.417 INFO:tasks.workunit.client.0.smithi136.stdout:op 3425 completed, throughput=4.99MB/sec 2024-09-13T00:12:38.417 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-XOlKV_rFPE_Qz16 off=7445091 len=1689494 2024-09-13T00:12:38.420 INFO:tasks.workunit.client.0.smithi136.stdout:op 3426 completed, throughput=5MB/sec 2024-09-13T00:12:38.420 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-q_wKA0WCmRHIyUr off=9725443 len=1241996 2024-09-13T00:12:38.423 INFO:tasks.workunit.client.0.smithi136.stdout:op 3427 completed, throughput=5MB/sec 2024-09-13T00:12:38.423 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-A3LZfh7JWze_cw2 off=7177585 len=76199 2024-09-13T00:12:38.428 INFO:tasks.workunit.client.0.smithi136.stdout:op 3428 completed, throughput=5MB/sec 2024-09-13T00:12:38.428 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-dxR2_1MT0tciwd3 off=12029172 len=911143 2024-09-13T00:12:38.437 INFO:tasks.workunit.client.0.smithi136.stdout:op 3429 completed, throughput=5MB/sec 2024-09-13T00:12:38.437 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj--hKA5EbeBxFs6wQ off=5960619 len=5947 2024-09-13T00:12:38.438 INFO:tasks.workunit.client.0.smithi136.stdout:op 3430 completed, throughput=5MB/sec 2024-09-13T00:12:38.438 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-W-LedAjSPTo6TLv off=2621355 len=146282 2024-09-13T00:12:38.439 INFO:tasks.workunit.client.0.smithi136.stdout:op 3431 completed, throughput=5MB/sec 2024-09-13T00:12:39.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:12:38 smithi136 bash[75164]: cluster 2024-09-13T00:12:37.482048+0000 mgr.y (mgr.24661) 1065 : cluster [DBG] pgmap v1748: 105 pgs: 105 active+clean; 126 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:12:39.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:12:38 smithi136 bash[75164]: audit 2024-09-13T00:12:38.790474+0000 mon.a (mon.0) 2206 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:12:39.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:12:38 smithi136 bash[76371]: cluster 2024-09-13T00:12:37.482048+0000 mgr.y (mgr.24661) 1065 : cluster [DBG] pgmap v1748: 105 pgs: 105 active+clean; 126 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:12:39.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:12:38 smithi136 bash[76371]: audit 2024-09-13T00:12:38.790474+0000 mon.a (mon.0) 2206 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:12:39.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:12:38 smithi181 bash[67587]: cluster 2024-09-13T00:12:37.482048+0000 mgr.y (mgr.24661) 1065 : cluster [DBG] pgmap v1748: 105 pgs: 105 active+clean; 126 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:12:39.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:12:38 smithi181 bash[67587]: audit 2024-09-13T00:12:38.790474+0000 mon.a (mon.0) 2206 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:12:39.440 INFO:tasks.workunit.client.0.smithi136.stdout: 655: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:39.440 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Mth5IeiGtrSevdE off=14943937 len=1166952 2024-09-13T00:12:39.444 INFO:tasks.workunit.client.0.smithi136.stdout:op 3432 completed, throughput=4.99MB/sec 2024-09-13T00:12:39.444 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Qf3GpngbT0IaraW off=19058910 len=965672 2024-09-13T00:12:39.448 INFO:tasks.workunit.client.0.smithi136.stdout:op 3433 completed, throughput=5MB/sec 2024-09-13T00:12:39.448 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-aNFI5m5JQQncgKg off=686289 len=217937 2024-09-13T00:12:39.453 INFO:tasks.workunit.client.0.smithi136.stdout:op 3434 completed, throughput=5MB/sec 2024-09-13T00:12:39.453 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-guRnfy81GdBMQ21 off=4096315 len=1197391 2024-09-13T00:12:39.456 INFO:tasks.workunit.client.0.smithi136.stdout:op 3435 completed, throughput=5MB/sec 2024-09-13T00:12:39.456 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-M48wZ2wF3IQrzWl off=886137 len=1453953 2024-09-13T00:12:39.470 INFO:tasks.workunit.client.0.smithi136.stdout:op 3436 completed, throughput=5MB/sec 2024-09-13T00:12:39.470 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-PqEecel-9hgd0T- off=10839990 len=564260 2024-09-13T00:12:39.472 INFO:tasks.workunit.client.0.smithi136.stdout:op 3437 completed, throughput=5MB/sec 2024-09-13T00:12:39.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:12:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:12:39] "GET /metrics HTTP/1.1" 200 34139 "" "Prometheus/2.43.0" 2024-09-13T00:12:40.473 INFO:tasks.workunit.client.0.smithi136.stdout: 656: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:40.473 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-cZcM9NjEVZHWTzO off=101926 len=1580051 2024-09-13T00:12:40.478 INFO:tasks.workunit.client.0.smithi136.stdout:op 3438 completed, throughput=5MB/sec 2024-09-13T00:12:40.479 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-7TbZTbMDtJSbiwg off=11305745 len=2054319 2024-09-13T00:12:40.491 INFO:tasks.workunit.client.0.smithi136.stdout:op 3439 completed, throughput=5MB/sec 2024-09-13T00:12:40.491 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-oS0BD9xqxYDXbRz off=9921424 len=800827 2024-09-13T00:12:40.494 INFO:tasks.workunit.client.0.smithi136.stdout:op 3440 completed, throughput=5MB/sec 2024-09-13T00:12:40.494 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-U8yVgAQgU5_YJAm off=3590501 len=558697 2024-09-13T00:12:40.496 INFO:tasks.workunit.client.0.smithi136.stdout:op 3441 completed, throughput=5MB/sec 2024-09-13T00:12:40.496 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-RvnQeNnhGiCkZLV off=21728005 len=574871 2024-09-13T00:12:40.502 INFO:tasks.workunit.client.0.smithi136.stdout:op 3442 completed, throughput=5MB/sec 2024-09-13T00:12:41.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:12:40 smithi136 bash[76371]: cluster 2024-09-13T00:12:39.483075+0000 mgr.y (mgr.24661) 1066 : cluster [DBG] pgmap v1749: 105 pgs: 105 active+clean; 126 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:12:41.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:12:40 smithi136 bash[75164]: cluster 2024-09-13T00:12:39.483075+0000 mgr.y (mgr.24661) 1066 : cluster [DBG] pgmap v1749: 105 pgs: 105 active+clean; 126 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:12:41.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:12:40 smithi181 bash[67587]: cluster 2024-09-13T00:12:39.483075+0000 mgr.y (mgr.24661) 1066 : cluster [DBG] pgmap v1749: 105 pgs: 105 active+clean; 126 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:12:41.502 INFO:tasks.workunit.client.0.smithi136.stdout: 657: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:41.502 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-otIN0BH9b5elABv off=10334932 len=809272 2024-09-13T00:12:41.509 INFO:tasks.workunit.client.0.smithi136.stdout:op 3443 completed, throughput=4.99MB/sec 2024-09-13T00:12:41.509 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_T6y2CDwkgMd2RP off=11341225 len=1099147 2024-09-13T00:12:41.519 INFO:tasks.workunit.client.0.smithi136.stdout:op 3444 completed, throughput=5MB/sec 2024-09-13T00:12:41.519 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2cxT6af1XZ3O6Pm off=3026115 len=868890 2024-09-13T00:12:41.522 INFO:tasks.workunit.client.0.smithi136.stdout:op 3445 completed, throughput=5MB/sec 2024-09-13T00:12:41.522 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-LNCoc7SFb-klV88 off=8263691 len=268414 2024-09-13T00:12:41.528 INFO:tasks.workunit.client.0.smithi136.stdout:op 3446 completed, throughput=5MB/sec 2024-09-13T00:12:41.528 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-OYCCFG8nsf2MaNw off=1524540 len=850083 2024-09-13T00:12:41.535 INFO:tasks.workunit.client.0.smithi136.stdout:op 3447 completed, throughput=5MB/sec 2024-09-13T00:12:41.536 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj--tJZQA9AR-aCsOo off=6840852 len=687964 2024-09-13T00:12:41.542 INFO:tasks.workunit.client.0.smithi136.stdout:op 3448 completed, throughput=5MB/sec 2024-09-13T00:12:41.542 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-7RWESC_WGX9NSRA off=6438621 len=99018 2024-09-13T00:12:41.546 INFO:tasks.workunit.client.0.smithi136.stdout:op 3449 completed, throughput=5MB/sec 2024-09-13T00:12:41.546 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-tKZU9MjDwMjFVEs off=8243408 len=1946005 2024-09-13T00:12:41.562 INFO:tasks.workunit.client.0.smithi136.stdout:op 3450 completed, throughput=5MB/sec 2024-09-13T00:12:42.562 INFO:tasks.workunit.client.0.smithi136.stdout: 658: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:42.562 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-hPHr-lvFUMQ2LGz off=6889947 len=266974 2024-09-13T00:12:42.568 INFO:tasks.workunit.client.0.smithi136.stdout:op 3451 completed, throughput=4.99MB/sec 2024-09-13T00:12:42.568 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-NhGREjYXjQCigtF off=5756931 len=1334008 2024-09-13T00:12:42.579 INFO:tasks.workunit.client.0.smithi136.stdout:op 3452 completed, throughput=5MB/sec 2024-09-13T00:12:42.579 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-xYzKXphuufZiibd off=7038231 len=1872750 2024-09-13T00:12:42.583 INFO:tasks.workunit.client.0.smithi136.stdout:op 3453 completed, throughput=5MB/sec 2024-09-13T00:12:42.583 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-NDDc6C9btoUaEu3 off=15651917 len=1465133 2024-09-13T00:12:42.596 INFO:tasks.workunit.client.0.smithi136.stdout:op 3454 completed, throughput=5MB/sec 2024-09-13T00:12:42.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:12:42 smithi181 bash[95505]: ts=2024-09-13T00:12:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:12:43.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:12:42 smithi136 bash[75164]: cluster 2024-09-13T00:12:41.484175+0000 mgr.y (mgr.24661) 1067 : cluster [DBG] pgmap v1750: 105 pgs: 105 active+clean; 126 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:12:43.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:12:42 smithi136 bash[76371]: cluster 2024-09-13T00:12:41.484175+0000 mgr.y (mgr.24661) 1067 : cluster [DBG] pgmap v1750: 105 pgs: 105 active+clean; 126 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:12:43.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:12:42 smithi181 bash[67587]: cluster 2024-09-13T00:12:41.484175+0000 mgr.y (mgr.24661) 1067 : cluster [DBG] pgmap v1750: 105 pgs: 105 active+clean; 126 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:12:43.597 INFO:tasks.workunit.client.0.smithi136.stdout: 659: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:43.597 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-5FMaIDeV-mHYT1f off=20802078 len=2070708 2024-09-13T00:12:43.601 INFO:tasks.workunit.client.0.smithi136.stdout:op 3455 completed, throughput=5MB/sec 2024-09-13T00:12:43.601 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-3mmOmC48i4qyUlj off=9852889 len=1287434 2024-09-13T00:12:43.606 INFO:tasks.workunit.client.0.smithi136.stdout:op 3456 completed, throughput=5MB/sec 2024-09-13T00:12:43.606 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-TGms_zIPLEtfX7I off=6696456 len=325855 2024-09-13T00:12:43.611 INFO:tasks.workunit.client.0.smithi136.stdout:op 3457 completed, throughput=5MB/sec 2024-09-13T00:12:43.611 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-raYHMYzIJWHoaZR off=1977329 len=750980 2024-09-13T00:12:43.614 INFO:tasks.workunit.client.0.smithi136.stdout:op 3458 completed, throughput=5MB/sec 2024-09-13T00:12:44.615 INFO:tasks.workunit.client.0.smithi136.stdout: 660: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:44.615 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-1CH1-pCWKImA6ws off=6881292 len=1459096 2024-09-13T00:12:44.619 INFO:tasks.workunit.client.0.smithi136.stdout:op 3459 completed, throughput=4.99MB/sec 2024-09-13T00:12:44.619 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-cGYw-uVIicUl9AL off=6549631 len=722036 2024-09-13T00:12:44.622 INFO:tasks.workunit.client.0.smithi136.stdout:op 3460 completed, throughput=5MB/sec 2024-09-13T00:12:44.622 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Fy5v-aZiZotRosQ off=6160302 len=695343 2024-09-13T00:12:44.629 INFO:tasks.workunit.client.0.smithi136.stdout:op 3461 completed, throughput=5MB/sec 2024-09-13T00:12:44.629 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-GaisF9MQyEHjR7h off=5229648 len=728814 2024-09-13T00:12:44.636 INFO:tasks.workunit.client.0.smithi136.stdout:op 3462 completed, throughput=5MB/sec 2024-09-13T00:12:44.636 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Y_Ppbkoqt-oyEs_ off=14919660 len=771395 2024-09-13T00:12:44.639 INFO:tasks.workunit.client.0.smithi136.stdout:op 3463 completed, throughput=5MB/sec 2024-09-13T00:12:44.639 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-LT2OXgyU7fJ8poQ off=20331022 len=1234443 2024-09-13T00:12:44.651 INFO:tasks.workunit.client.0.smithi136.stdout:op 3464 completed, throughput=5MB/sec 2024-09-13T00:12:45.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:12:44 smithi136 bash[75164]: cluster 2024-09-13T00:12:43.485181+0000 mgr.y (mgr.24661) 1068 : cluster [DBG] pgmap v1751: 105 pgs: 105 active+clean; 126 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:12:45.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:12:44 smithi136 bash[76371]: cluster 2024-09-13T00:12:43.485181+0000 mgr.y (mgr.24661) 1068 : cluster [DBG] pgmap v1751: 105 pgs: 105 active+clean; 126 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:12:45.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:12:44 smithi181 bash[67587]: cluster 2024-09-13T00:12:43.485181+0000 mgr.y (mgr.24661) 1068 : cluster [DBG] pgmap v1751: 105 pgs: 105 active+clean; 126 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:12:45.651 INFO:tasks.workunit.client.0.smithi136.stdout: 661: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:45.651 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-cMAzxAiw2Xz1CMT off=1437994 len=589537 2024-09-13T00:12:45.658 INFO:tasks.workunit.client.0.smithi136.stdout:op 3465 completed, throughput=4.99MB/sec 2024-09-13T00:12:45.658 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-YHgbY7nQP98Z2IF off=6253925 len=912255 2024-09-13T00:12:45.661 INFO:tasks.workunit.client.0.smithi136.stdout:op 3466 completed, throughput=5MB/sec 2024-09-13T00:12:45.661 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-xqzTqFMRGdIpc_w off=8421064 len=622594 2024-09-13T00:12:45.667 INFO:tasks.workunit.client.0.smithi136.stdout:op 3467 completed, throughput=5MB/sec 2024-09-13T00:12:45.667 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-T-e5BOeLVni4_rN off=4205516 len=1048397 2024-09-13T00:12:45.671 INFO:tasks.workunit.client.0.smithi136.stdout:op 3468 completed, throughput=5MB/sec 2024-09-13T00:12:45.672 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-vioMbLYMMGcu8Az off=339636 len=120938 2024-09-13T00:12:45.673 INFO:tasks.workunit.client.0.smithi136.stdout:op 3469 completed, throughput=5MB/sec 2024-09-13T00:12:45.673 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ovwXaWzUHHyKAcA off=5532526 len=1750493 2024-09-13T00:12:45.678 INFO:tasks.workunit.client.0.smithi136.stdout:op 3470 completed, throughput=5MB/sec 2024-09-13T00:12:45.678 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ENxkgYFKQFSYKZw off=2627249 len=668255 2024-09-13T00:12:45.681 INFO:tasks.workunit.client.0.smithi136.stdout:op 3471 completed, throughput=5MB/sec 2024-09-13T00:12:46.681 INFO:tasks.workunit.client.0.smithi136.stdout: 662: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:46.681 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-aXSpw0e1NbIrTmP off=9150074 len=670039 2024-09-13T00:12:46.684 INFO:tasks.workunit.client.0.smithi136.stdout:op 3472 completed, throughput=4.99MB/sec 2024-09-13T00:12:46.684 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-v8EEe1ATJUPefdy off=256293 len=1487774 2024-09-13T00:12:46.696 INFO:tasks.workunit.client.0.smithi136.stdout:op 3473 completed, throughput=5MB/sec 2024-09-13T00:12:46.696 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-TptSkTfg24YQFu6 off=19134306 len=392700 2024-09-13T00:12:46.701 INFO:tasks.workunit.client.0.smithi136.stdout:op 3474 completed, throughput=5MB/sec 2024-09-13T00:12:46.701 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ZNqhb3zAMpN1A77 off=15981416 len=1113392 2024-09-13T00:12:46.710 INFO:tasks.workunit.client.0.smithi136.stdout:op 3475 completed, throughput=5MB/sec 2024-09-13T00:12:46.710 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-D-Oq9mH2Az-m9i7 off=348872 len=1690277 2024-09-13T00:12:46.725 INFO:tasks.workunit.client.0.smithi136.stdout:op 3476 completed, throughput=5MB/sec 2024-09-13T00:12:47.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:12:46 smithi136 bash[75164]: cluster 2024-09-13T00:12:45.486066+0000 mgr.y (mgr.24661) 1069 : cluster [DBG] pgmap v1752: 105 pgs: 105 active+clean; 126 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:12:47.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:12:46 smithi136 bash[76371]: cluster 2024-09-13T00:12:45.486066+0000 mgr.y (mgr.24661) 1069 : cluster [DBG] pgmap v1752: 105 pgs: 105 active+clean; 126 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:12:47.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:12:46 smithi181 bash[67587]: cluster 2024-09-13T00:12:45.486066+0000 mgr.y (mgr.24661) 1069 : cluster [DBG] pgmap v1752: 105 pgs: 105 active+clean; 126 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:12:47.725 INFO:tasks.workunit.client.0.smithi136.stdout: 663: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:47.725 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-EsTdwznTeDXZ58v off=16390813 len=232108 2024-09-13T00:12:47.726 INFO:tasks.workunit.client.0.smithi136.stdout:op 3477 completed, throughput=4.99MB/sec 2024-09-13T00:12:47.726 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-0jEsdkgY832KzAA off=7052730 len=133251 2024-09-13T00:12:47.727 INFO:tasks.workunit.client.0.smithi136.stdout:op 3478 completed, throughput=4.99MB/sec 2024-09-13T00:12:47.727 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-wkjIfxDgFnsGtqJ off=13017433 len=225710 2024-09-13T00:12:47.729 INFO:tasks.workunit.client.0.smithi136.stdout:op 3479 completed, throughput=4.99MB/sec 2024-09-13T00:12:47.729 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-dW1ES7KY3_7IofG off=1725394 len=1886562 2024-09-13T00:12:47.741 INFO:tasks.workunit.client.0.smithi136.stdout:op 3480 completed, throughput=5MB/sec 2024-09-13T00:12:47.741 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-mYy20JdzGWUzCw4 off=608628 len=1965860 2024-09-13T00:12:47.746 INFO:tasks.workunit.client.0.smithi136.stdout:op 3481 completed, throughput=5MB/sec 2024-09-13T00:12:47.746 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-qeeHiFdWkPyYOsC off=6453186 len=709246 2024-09-13T00:12:47.748 INFO:tasks.workunit.client.0.smithi136.stdout:op 3482 completed, throughput=5MB/sec 2024-09-13T00:12:48.748 INFO:tasks.workunit.client.0.smithi136.stdout: 664: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:48.749 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-haPAeGysRvCyXrw off=1555022 len=2024575 2024-09-13T00:12:48.764 INFO:tasks.workunit.client.0.smithi136.stdout:op 3483 completed, throughput=5MB/sec 2024-09-13T00:12:48.764 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Wb2yMGkyajRfONs off=11147711 len=432065 2024-09-13T00:12:48.766 INFO:tasks.workunit.client.0.smithi136.stdout:op 3484 completed, throughput=5MB/sec 2024-09-13T00:12:48.766 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_oyvfPIdv0eeDaf off=2319713 len=1298519 2024-09-13T00:12:48.776 INFO:tasks.workunit.client.0.smithi136.stdout:op 3485 completed, throughput=5MB/sec 2024-09-13T00:12:48.776 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-gC6Poj4cgo288II off=15075148 len=123759 2024-09-13T00:12:48.780 INFO:tasks.workunit.client.0.smithi136.stdout:op 3486 completed, throughput=5MB/sec 2024-09-13T00:12:48.780 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-iTXfsQ3SS3TBIVN off=4285772 len=561701 2024-09-13T00:12:48.786 INFO:tasks.workunit.client.0.smithi136.stdout:op 3487 completed, throughput=5MB/sec 2024-09-13T00:12:48.786 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Uq1JBIRFzT1EQ4J off=5082076 len=769585 2024-09-13T00:12:48.790 INFO:tasks.workunit.client.0.smithi136.stdout:op 3488 completed, throughput=5MB/sec 2024-09-13T00:12:48.790 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-tr4pDWnRpbha-RH off=845728 len=468861 2024-09-13T00:12:48.791 INFO:tasks.workunit.client.0.smithi136.stdout:op 3489 completed, throughput=5MB/sec 2024-09-13T00:12:49.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:12:48 smithi136 bash[75164]: cluster 2024-09-13T00:12:47.487242+0000 mgr.y (mgr.24661) 1070 : cluster [DBG] pgmap v1753: 105 pgs: 105 active+clean; 126 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:12:49.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:12:48 smithi136 bash[76371]: cluster 2024-09-13T00:12:47.487242+0000 mgr.y (mgr.24661) 1070 : cluster [DBG] pgmap v1753: 105 pgs: 105 active+clean; 126 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:12:49.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:12:48 smithi181 bash[67587]: cluster 2024-09-13T00:12:47.487242+0000 mgr.y (mgr.24661) 1070 : cluster [DBG] pgmap v1753: 105 pgs: 105 active+clean; 126 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:12:49.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:12:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:12:49] "GET /metrics HTTP/1.1" 200 34139 "" "Prometheus/2.43.0" 2024-09-13T00:12:49.792 INFO:tasks.workunit.client.0.smithi136.stdout: 665: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:49.792 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-zQP6TTgm-B81WUx off=15997728 len=1390027 2024-09-13T00:12:49.802 INFO:tasks.workunit.client.0.smithi136.stdout:op 3490 completed, throughput=5MB/sec 2024-09-13T00:12:49.802 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-dC2Iz_49RYDs-ID off=8094696 len=1038282 2024-09-13T00:12:49.812 INFO:tasks.workunit.client.0.smithi136.stdout:op 3491 completed, throughput=5MB/sec 2024-09-13T00:12:49.812 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-8XSevUubnGim2Fp off=9423289 len=525645 2024-09-13T00:12:49.818 INFO:tasks.workunit.client.0.smithi136.stdout:op 3492 completed, throughput=5MB/sec 2024-09-13T00:12:49.818 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-FER3o75qKLci6WL off=7089681 len=1319113 2024-09-13T00:12:49.828 INFO:tasks.workunit.client.0.smithi136.stdout:op 3493 completed, throughput=5MB/sec 2024-09-13T00:12:49.828 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-B989J-3chD6aEEr off=960959 len=1143543 2024-09-13T00:12:49.839 INFO:tasks.workunit.client.0.smithi136.stdout:op 3494 completed, throughput=5MB/sec 2024-09-13T00:12:50.839 INFO:tasks.workunit.client.0.smithi136.stdout: 666: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:50.839 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-arTGzcy3MSMJzq7 off=10747236 len=1787912 2024-09-13T00:12:50.855 INFO:tasks.workunit.client.0.smithi136.stdout:op 3495 completed, throughput=5MB/sec 2024-09-13T00:12:50.855 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-5z6t7L6KR4s8Aut off=135977 len=286281 2024-09-13T00:12:50.856 INFO:tasks.workunit.client.0.smithi136.stdout:op 3496 completed, throughput=5MB/sec 2024-09-13T00:12:50.856 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_Crr2tvMRj_-0uU off=3078801 len=1483002 2024-09-13T00:12:50.867 INFO:tasks.workunit.client.0.smithi136.stdout:op 3497 completed, throughput=5MB/sec 2024-09-13T00:12:50.867 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-OPprjfVOWQOZxiB off=8289196 len=964128 2024-09-13T00:12:50.871 INFO:tasks.workunit.client.0.smithi136.stdout:op 3498 completed, throughput=5MB/sec 2024-09-13T00:12:50.871 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-YYCOldh0WySUyHO off=4334795 len=438732 2024-09-13T00:12:50.873 INFO:tasks.workunit.client.0.smithi136.stdout:op 3499 completed, throughput=5MB/sec 2024-09-13T00:12:50.873 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-iENR_qRfobDJOE8 off=459727 len=305880 2024-09-13T00:12:50.879 INFO:tasks.workunit.client.0.smithi136.stdout:op 3500 completed, throughput=5MB/sec 2024-09-13T00:12:51.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:12:50 smithi136 bash[76371]: cluster 2024-09-13T00:12:49.488243+0000 mgr.y (mgr.24661) 1071 : cluster [DBG] pgmap v1754: 105 pgs: 105 active+clean; 126 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:12:51.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:12:50 smithi136 bash[75164]: cluster 2024-09-13T00:12:49.488243+0000 mgr.y (mgr.24661) 1071 : cluster [DBG] pgmap v1754: 105 pgs: 105 active+clean; 126 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:12:51.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:12:50 smithi181 bash[67587]: cluster 2024-09-13T00:12:49.488243+0000 mgr.y (mgr.24661) 1071 : cluster [DBG] pgmap v1754: 105 pgs: 105 active+clean; 126 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:12:51.879 INFO:tasks.workunit.client.0.smithi136.stdout: 667: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:51.879 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-eNN9obTibn-ajQ6 off=4601429 len=856906 2024-09-13T00:12:51.882 INFO:tasks.workunit.client.0.smithi136.stdout:op 3501 completed, throughput=4.99MB/sec 2024-09-13T00:12:51.882 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-2g9dPXApDWVBJXh off=1635591 len=1048318 2024-09-13T00:12:51.891 INFO:tasks.workunit.client.0.smithi136.stdout:op 3502 completed, throughput=5MB/sec 2024-09-13T00:12:51.892 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Mw6KB8nS95-4RF6 off=2044652 len=1499279 2024-09-13T00:12:51.903 INFO:tasks.workunit.client.0.smithi136.stdout:op 3503 completed, throughput=5MB/sec 2024-09-13T00:12:51.903 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-q0KGBYdpqRt67Ur off=723526 len=463988 2024-09-13T00:12:51.909 INFO:tasks.workunit.client.0.smithi136.stdout:op 3504 completed, throughput=5MB/sec 2024-09-13T00:12:51.909 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-kOa3zKkubUWtKcB off=3966863 len=1923260 2024-09-13T00:12:51.922 INFO:tasks.workunit.client.0.smithi136.stdout:op 3505 completed, throughput=5MB/sec 2024-09-13T00:12:52.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:12:52 smithi181 bash[95505]: ts=2024-09-13T00:12:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:12:52.923 INFO:tasks.workunit.client.0.smithi136.stdout: 668: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:52.923 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Zg9lsKtFXPVDNw4 off=3220543 len=401095 2024-09-13T00:12:52.925 INFO:tasks.workunit.client.0.smithi136.stdout:op 3506 completed, throughput=4.99MB/sec 2024-09-13T00:12:52.925 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-HMFp8kixomwxh0q off=28987 len=409174 2024-09-13T00:12:52.928 INFO:tasks.workunit.client.0.smithi136.stdout:op 3507 completed, throughput=4.99MB/sec 2024-09-13T00:12:52.928 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-VFflZExul8Nn6by off=10317406 len=979383 2024-09-13T00:12:52.936 INFO:tasks.workunit.client.0.smithi136.stdout:op 3508 completed, throughput=5MB/sec 2024-09-13T00:12:52.936 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-k0a0FOgkDXXec0P off=1736558 len=695433 2024-09-13T00:12:52.938 INFO:tasks.workunit.client.0.smithi136.stdout:op 3509 completed, throughput=5MB/sec 2024-09-13T00:12:52.938 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Wd2eO0XOLKgjPP5 off=2618683 len=1978408 2024-09-13T00:12:52.952 INFO:tasks.workunit.client.0.smithi136.stdout:op 3510 completed, throughput=5MB/sec 2024-09-13T00:12:52.952 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-eg_ChRWjbLLVj2u off=3557803 len=69216 2024-09-13T00:12:52.952 INFO:tasks.workunit.client.0.smithi136.stdout:op 3511 completed, throughput=5MB/sec 2024-09-13T00:12:52.953 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-HPjuFIF6-vhGiza off=14956905 len=256819 2024-09-13T00:12:52.957 INFO:tasks.workunit.client.0.smithi136.stdout:op 3512 completed, throughput=5MB/sec 2024-09-13T00:12:52.957 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-FYRdGBXzzDzIfEl off=9776006 len=165039 2024-09-13T00:12:52.959 INFO:tasks.workunit.client.0.smithi136.stdout:op 3513 completed, throughput=5MB/sec 2024-09-13T00:12:53.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:12:52 smithi136 bash[75164]: cluster 2024-09-13T00:12:51.489364+0000 mgr.y (mgr.24661) 1072 : cluster [DBG] pgmap v1755: 105 pgs: 105 active+clean; 126 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:12:53.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:12:52 smithi136 bash[76371]: cluster 2024-09-13T00:12:51.489364+0000 mgr.y (mgr.24661) 1072 : cluster [DBG] pgmap v1755: 105 pgs: 105 active+clean; 126 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:12:53.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:12:52 smithi181 bash[67587]: cluster 2024-09-13T00:12:51.489364+0000 mgr.y (mgr.24661) 1072 : cluster [DBG] pgmap v1755: 105 pgs: 105 active+clean; 126 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:12:53.959 INFO:tasks.workunit.client.0.smithi136.stdout: 669: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:53.959 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-MiuSYy3q8nD9EWV off=9807998 len=857336 2024-09-13T00:12:53.962 INFO:tasks.workunit.client.0.smithi136.stdout:op 3514 completed, throughput=4.99MB/sec 2024-09-13T00:12:53.962 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-5mZy4JBMT0pB7WX off=4292904 len=1264740 2024-09-13T00:12:53.965 INFO:tasks.workunit.client.0.smithi136.stdout:op 3515 completed, throughput=5MB/sec 2024-09-13T00:12:53.965 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-N2Hnq5ALcEw4pEI off=882025 len=1379603 2024-09-13T00:12:53.978 INFO:tasks.workunit.client.0.smithi136.stdout:op 3516 completed, throughput=5MB/sec 2024-09-13T00:12:53.978 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-L54NR1S6uc74a3r off=4006567 len=290532 2024-09-13T00:12:53.984 INFO:tasks.workunit.client.0.smithi136.stdout:op 3517 completed, throughput=5MB/sec 2024-09-13T00:12:53.984 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-8yel6Gje62halO4 off=5609057 len=1738228 2024-09-13T00:12:53.988 INFO:tasks.workunit.client.0.smithi136.stdout:op 3518 completed, throughput=5MB/sec 2024-09-13T00:12:54.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:12:53 smithi136 bash[75164]: audit 2024-09-13T00:12:53.790699+0000 mon.a (mon.0) 2207 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:12:54.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:12:53 smithi136 bash[76371]: audit 2024-09-13T00:12:53.790699+0000 mon.a (mon.0) 2207 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:12:54.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:12:53 smithi181 bash[67587]: audit 2024-09-13T00:12:53.790699+0000 mon.a (mon.0) 2207 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:12:54.989 INFO:tasks.workunit.client.0.smithi136.stdout: 670: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:54.989 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-rMYZma07cU5UXpm off=11945223 len=1071707 2024-09-13T00:12:54.992 INFO:tasks.workunit.client.0.smithi136.stdout:op 3519 completed, throughput=4.99MB/sec 2024-09-13T00:12:54.992 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-xSpQIiSnkTd3dth off=10343693 len=1022553 2024-09-13T00:12:55.002 INFO:tasks.workunit.client.0.smithi136.stdout:op 3520 completed, throughput=5MB/sec 2024-09-13T00:12:55.002 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-_cwrFhH7xU75RkE off=1312176 len=7395 2024-09-13T00:12:55.003 INFO:tasks.workunit.client.0.smithi136.stdout:op 3521 completed, throughput=5MB/sec 2024-09-13T00:12:55.003 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-u2j1szggk9op3xF off=1483795 len=1789392 2024-09-13T00:12:55.007 INFO:tasks.workunit.client.0.smithi136.stdout:op 3522 completed, throughput=5MB/sec 2024-09-13T00:12:55.007 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-0YMFzo1aKp-eaSo off=2266073 len=1286503 2024-09-13T00:12:55.017 INFO:tasks.workunit.client.0.smithi136.stdout:op 3523 completed, throughput=5MB/sec 2024-09-13T00:12:55.017 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-gpS6cnXySwaEWOp off=1868153 len=140193 2024-09-13T00:12:55.018 INFO:tasks.workunit.client.0.smithi136.stdout:op 3524 completed, throughput=5MB/sec 2024-09-13T00:12:55.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:12:54 smithi136 bash[75164]: cluster 2024-09-13T00:12:53.490310+0000 mgr.y (mgr.24661) 1073 : cluster [DBG] pgmap v1756: 105 pgs: 105 active+clean; 126 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:12:55.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:12:54 smithi136 bash[75164]: audit 2024-09-13T00:12:54.287679+0000 mon.a (mon.0) 2208 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:12:55.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:12:54 smithi136 bash[75164]: audit 2024-09-13T00:12:54.820448+0000 mon.a (mon.0) 2209 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:12:55.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:12:54 smithi136 bash[75164]: audit 2024-09-13T00:12:54.822269+0000 mon.a (mon.0) 2210 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:12:55.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:12:54 smithi136 bash[75164]: audit 2024-09-13T00:12:54.831896+0000 mon.a (mon.0) 2211 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:12:55.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:12:54 smithi136 bash[76371]: cluster 2024-09-13T00:12:53.490310+0000 mgr.y (mgr.24661) 1073 : cluster [DBG] pgmap v1756: 105 pgs: 105 active+clean; 126 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:12:55.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:12:54 smithi136 bash[76371]: audit 2024-09-13T00:12:54.287679+0000 mon.a (mon.0) 2208 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:12:55.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:12:54 smithi136 bash[76371]: audit 2024-09-13T00:12:54.820448+0000 mon.a (mon.0) 2209 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:12:55.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:12:54 smithi136 bash[76371]: audit 2024-09-13T00:12:54.822269+0000 mon.a (mon.0) 2210 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:12:55.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:12:54 smithi136 bash[76371]: audit 2024-09-13T00:12:54.831896+0000 mon.a (mon.0) 2211 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:12:55.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:12:54 smithi181 bash[67587]: cluster 2024-09-13T00:12:53.490310+0000 mgr.y (mgr.24661) 1073 : cluster [DBG] pgmap v1756: 105 pgs: 105 active+clean; 126 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:12:55.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:12:54 smithi181 bash[67587]: audit 2024-09-13T00:12:54.287679+0000 mon.a (mon.0) 2208 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:12:55.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:12:54 smithi181 bash[67587]: audit 2024-09-13T00:12:54.820448+0000 mon.a (mon.0) 2209 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:12:55.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:12:54 smithi181 bash[67587]: audit 2024-09-13T00:12:54.822269+0000 mon.a (mon.0) 2210 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:12:55.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:12:54 smithi181 bash[67587]: audit 2024-09-13T00:12:54.831896+0000 mon.a (mon.0) 2211 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:12:56.020 INFO:tasks.workunit.client.0.smithi136.stdout: 671: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:56.021 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-N6B7CuJUHvSJgku off=4717090 len=2048431 2024-09-13T00:12:56.024 INFO:tasks.workunit.client.0.smithi136.stdout:op 3525 completed, throughput=5MB/sec 2024-09-13T00:12:56.024 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-8Ia2VbwB5I0ihJp off=11873559 len=1986105 2024-09-13T00:12:56.042 INFO:tasks.workunit.client.0.smithi136.stdout:op 3526 completed, throughput=5MB/sec 2024-09-13T00:12:56.043 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-lXWq_jrO6y6dPQZ off=293683 len=1343144 2024-09-13T00:12:56.046 INFO:tasks.workunit.client.0.smithi136.stdout:op 3527 completed, throughput=5MB/sec 2024-09-13T00:12:57.046 INFO:tasks.workunit.client.0.smithi136.stdout: 672: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:57.046 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-5TfW7TXDU55pviY off=15278598 len=1264240 2024-09-13T00:12:57.057 INFO:tasks.workunit.client.0.smithi136.stdout:op 3528 completed, throughput=4.99MB/sec 2024-09-13T00:12:57.057 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-oEkIt6pua5Lskwl off=9533095 len=1317378 2024-09-13T00:12:57.066 INFO:tasks.workunit.client.0.smithi136.stdout:op 3529 completed, throughput=5MB/sec 2024-09-13T00:12:57.066 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Ril7QNIhCK5T3Gp off=10997544 len=297817 2024-09-13T00:12:57.071 INFO:tasks.workunit.client.0.smithi136.stdout:op 3530 completed, throughput=5MB/sec 2024-09-13T00:12:57.071 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-NhHFolXfGvozn3K off=3191 len=630531 2024-09-13T00:12:57.078 INFO:tasks.workunit.client.0.smithi136.stdout:op 3531 completed, throughput=5MB/sec 2024-09-13T00:12:57.078 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-knosrobamwjf5xN off=6125313 len=590816 2024-09-13T00:12:57.085 INFO:tasks.workunit.client.0.smithi136.stdout:op 3532 completed, throughput=5MB/sec 2024-09-13T00:12:57.085 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-V2pUNUrmb95ihk9 off=5219708 len=1592732 2024-09-13T00:12:57.090 INFO:tasks.workunit.client.0.smithi136.stdout:op 3533 completed, throughput=5MB/sec 2024-09-13T00:12:57.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:12:56 smithi136 bash[76371]: cluster 2024-09-13T00:12:55.491169+0000 mgr.y (mgr.24661) 1074 : cluster [DBG] pgmap v1757: 105 pgs: 105 active+clean; 126 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 1.9 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:12:57.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:12:56 smithi136 bash[75164]: cluster 2024-09-13T00:12:55.491169+0000 mgr.y (mgr.24661) 1074 : cluster [DBG] pgmap v1757: 105 pgs: 105 active+clean; 126 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 1.9 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:12:57.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:12:56 smithi181 bash[67587]: cluster 2024-09-13T00:12:55.491169+0000 mgr.y (mgr.24661) 1074 : cluster [DBG] pgmap v1757: 105 pgs: 105 active+clean; 126 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 1.9 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:12:58.090 INFO:tasks.workunit.client.0.smithi136.stdout: 673: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:58.090 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-0b1V3Yx1yayZUVb off=3087798 len=998974 2024-09-13T00:12:58.094 INFO:tasks.workunit.client.0.smithi136.stdout:op 3534 completed, throughput=4.99MB/sec 2024-09-13T00:12:58.094 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-O0WEOEyLw7roJzv off=15098771 len=1591279 2024-09-13T00:12:58.099 INFO:tasks.workunit.client.0.smithi136.stdout:op 3535 completed, throughput=5MB/sec 2024-09-13T00:12:58.099 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ESNYBTl4ukPIyzg off=3501174 len=1829346 2024-09-13T00:12:58.103 INFO:tasks.workunit.client.0.smithi136.stdout:op 3536 completed, throughput=5MB/sec 2024-09-13T00:12:58.103 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-p_RYyWLYA5RenkZ off=12062646 len=88232 2024-09-13T00:12:58.105 INFO:tasks.workunit.client.0.smithi136.stdout:op 3537 completed, throughput=5MB/sec 2024-09-13T00:12:58.105 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-lcNImTHwV4rbu1l off=5442141 len=354447 2024-09-13T00:12:58.106 INFO:tasks.workunit.client.0.smithi136.stdout:op 3538 completed, throughput=5MB/sec 2024-09-13T00:12:58.107 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-46hGXZU3-CaaYE8 off=9851043 len=308790 2024-09-13T00:12:58.112 INFO:tasks.workunit.client.0.smithi136.stdout:op 3539 completed, throughput=5MB/sec 2024-09-13T00:12:59.113 INFO:tasks.workunit.client.0.smithi136.stdout: 674: throughput=4.99MB/sec pending data=0 2024-09-13T00:12:59.113 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-bTvYNQLYzR8DMZx off=9512752 len=1660516 2024-09-13T00:12:59.125 INFO:tasks.workunit.client.0.smithi136.stdout:op 3540 completed, throughput=5MB/sec 2024-09-13T00:12:59.125 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-oS0BD9xqxYDXbRz off=6125935 len=1296784 2024-09-13T00:12:59.129 INFO:tasks.workunit.client.0.smithi136.stdout:op 3541 completed, throughput=5MB/sec 2024-09-13T00:12:59.129 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-2-QLcbeYKfGuhGY off=3417084 len=1312386 2024-09-13T00:12:59.138 INFO:tasks.workunit.client.0.smithi136.stdout:op 3542 completed, throughput=5MB/sec 2024-09-13T00:12:59.138 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-lxUTGoKsJ-N7Ls2 off=724234 len=1748924 2024-09-13T00:12:59.149 INFO:tasks.workunit.client.0.smithi136.stdout:op 3543 completed, throughput=5MB/sec 2024-09-13T00:12:59.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:12:58 smithi136 bash[76371]: cluster 2024-09-13T00:12:57.492306+0000 mgr.y (mgr.24661) 1075 : cluster [DBG] pgmap v1758: 105 pgs: 105 active+clean; 126 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 2.2 MiB/s rd, 3.3 MiB/s wr, 6 op/s 2024-09-13T00:12:59.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:12:58 smithi136 bash[75164]: cluster 2024-09-13T00:12:57.492306+0000 mgr.y (mgr.24661) 1075 : cluster [DBG] pgmap v1758: 105 pgs: 105 active+clean; 126 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 2.2 MiB/s rd, 3.3 MiB/s wr, 6 op/s 2024-09-13T00:12:59.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:12:58 smithi181 bash[67587]: cluster 2024-09-13T00:12:57.492306+0000 mgr.y (mgr.24661) 1075 : cluster [DBG] pgmap v1758: 105 pgs: 105 active+clean; 126 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 2.2 MiB/s rd, 3.3 MiB/s wr, 6 op/s 2024-09-13T00:12:59.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:12:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:12:59] "GET /metrics HTTP/1.1" 200 34140 "" "Prometheus/2.43.0" 2024-09-13T00:13:00.149 INFO:tasks.workunit.client.0.smithi136.stdout: 675: throughput=4.99MB/sec pending data=0 2024-09-13T00:13:00.150 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-vZS9K3T6eIyNQwt off=5280735 len=1532149 2024-09-13T00:13:00.153 INFO:tasks.workunit.client.0.smithi136.stdout:op 3544 completed, throughput=5MB/sec 2024-09-13T00:13:00.153 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-JAp1C48yUMxFWMh off=4708742 len=1834591 2024-09-13T00:13:00.157 INFO:tasks.workunit.client.0.smithi136.stdout:op 3545 completed, throughput=5MB/sec 2024-09-13T00:13:00.157 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-_cwrFhH7xU75RkE off=9822393 len=320015 2024-09-13T00:13:00.159 INFO:tasks.workunit.client.0.smithi136.stdout:op 3546 completed, throughput=5MB/sec 2024-09-13T00:13:00.159 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-fS1fZzkEWEH7jEK off=8242054 len=987380 2024-09-13T00:13:00.162 INFO:tasks.workunit.client.0.smithi136.stdout:op 3547 completed, throughput=5MB/sec 2024-09-13T00:13:01.162 INFO:tasks.workunit.client.0.smithi136.stdout: 676: throughput=4.99MB/sec pending data=0 2024-09-13T00:13:01.162 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-MTVPiTIvxTZY4yb off=3564197 len=360908 2024-09-13T00:13:01.163 INFO:tasks.workunit.client.0.smithi136.stdout:op 3548 completed, throughput=4.99MB/sec 2024-09-13T00:13:01.163 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-En6cQM979gkoZlV off=9370273 len=972652 2024-09-13T00:13:01.171 INFO:tasks.workunit.client.0.smithi136.stdout:op 3549 completed, throughput=4.99MB/sec 2024-09-13T00:13:01.171 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-_Y-LXHbKbq_WU1b off=9945574 len=753059 2024-09-13T00:13:01.175 INFO:tasks.workunit.client.0.smithi136.stdout:op 3550 completed, throughput=5MB/sec 2024-09-13T00:13:01.175 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-w72icWw0VJ8lNry off=8636002 len=1682763 2024-09-13T00:13:01.180 INFO:tasks.workunit.client.0.smithi136.stdout:op 3551 completed, throughput=5MB/sec 2024-09-13T00:13:01.180 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-IJkcSpzAKx680aU off=10762038 len=1218917 2024-09-13T00:13:01.183 INFO:tasks.workunit.client.0.smithi136.stdout:op 3552 completed, throughput=5MB/sec 2024-09-13T00:13:01.183 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-f-b6fXufavGZ7f- off=1842790 len=299026 2024-09-13T00:13:01.188 INFO:tasks.workunit.client.0.smithi136.stdout:op 3553 completed, throughput=5MB/sec 2024-09-13T00:13:01.189 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-r-AM-OQ_jPSuUdI off=12948144 len=1196100 2024-09-13T00:13:01.196 INFO:tasks.workunit.client.0.smithi136.stdout:op 3554 completed, throughput=5MB/sec 2024-09-13T00:13:01.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:13:00 smithi136 bash[76371]: cluster 2024-09-13T00:12:59.493174+0000 mgr.y (mgr.24661) 1076 : cluster [DBG] pgmap v1759: 105 pgs: 105 active+clean; 126 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:13:01.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:13:00 smithi136 bash[75164]: cluster 2024-09-13T00:12:59.493174+0000 mgr.y (mgr.24661) 1076 : cluster [DBG] pgmap v1759: 105 pgs: 105 active+clean; 126 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:13:01.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:13:00 smithi181 bash[67587]: cluster 2024-09-13T00:12:59.493174+0000 mgr.y (mgr.24661) 1076 : cluster [DBG] pgmap v1759: 105 pgs: 105 active+clean; 126 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:13:02.196 INFO:tasks.workunit.client.0.smithi136.stdout: 677: throughput=4.99MB/sec pending data=0 2024-09-13T00:13:02.196 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Zdy09BubR4rtdP- off=5219357 len=18482 2024-09-13T00:13:02.197 INFO:tasks.workunit.client.0.smithi136.stdout:op 3555 completed, throughput=4.99MB/sec 2024-09-13T00:13:02.197 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-A3LZfh7JWze_cw2 off=9622622 len=1363809 2024-09-13T00:13:02.202 INFO:tasks.workunit.client.0.smithi136.stdout:op 3556 completed, throughput=5MB/sec 2024-09-13T00:13:02.202 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-MrCIKj8wGPHcyUL off=1281275 len=1585481 2024-09-13T00:13:02.205 INFO:tasks.workunit.client.0.smithi136.stdout:op 3557 completed, throughput=5MB/sec 2024-09-13T00:13:02.206 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-d6s6sme-dVetau8 off=7044127 len=138854 2024-09-13T00:13:02.210 INFO:tasks.workunit.client.0.smithi136.stdout:op 3558 completed, throughput=5MB/sec 2024-09-13T00:13:02.210 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-iifxO-uzhPMOC5S off=4462992 len=767529 2024-09-13T00:13:02.213 INFO:tasks.workunit.client.0.smithi136.stdout:op 3559 completed, throughput=5MB/sec 2024-09-13T00:13:02.214 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-TiSgjdNkuSxpDkr off=1472347 len=954991 2024-09-13T00:13:02.221 INFO:tasks.workunit.client.0.smithi136.stdout:op 3560 completed, throughput=5MB/sec 2024-09-13T00:13:02.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:13:02 smithi181 bash[95505]: ts=2024-09-13T00:13:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:13:03.222 INFO:tasks.workunit.client.0.smithi136.stdout: 678: throughput=4.99MB/sec pending data=0 2024-09-13T00:13:03.222 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-tXrS6QRS8gZUkSH off=2014625 len=1677197 2024-09-13T00:13:03.227 INFO:tasks.workunit.client.0.smithi136.stdout:op 3561 completed, throughput=5MB/sec 2024-09-13T00:13:03.227 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-HS1Vln4XXtO-h1E off=10951734 len=1288448 2024-09-13T00:13:03.240 INFO:tasks.workunit.client.0.smithi136.stdout:op 3562 completed, throughput=5MB/sec 2024-09-13T00:13:03.240 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-wbCdFcNFTltKU8e off=2267421 len=492165 2024-09-13T00:13:03.243 INFO:tasks.workunit.client.0.smithi136.stdout:op 3563 completed, throughput=5MB/sec 2024-09-13T00:13:03.243 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-EQdyvK2UdJqgB6K off=22953908 len=1518951 2024-09-13T00:13:03.256 INFO:tasks.workunit.client.0.smithi136.stdout:op 3564 completed, throughput=5MB/sec 2024-09-13T00:13:03.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:13:02 smithi136 bash[76371]: cluster 2024-09-13T00:13:01.494280+0000 mgr.y (mgr.24661) 1077 : cluster [DBG] pgmap v1760: 105 pgs: 105 active+clean; 126 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 2.4 MiB/s rd, 3.1 MiB/s wr, 5 op/s 2024-09-13T00:13:03.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:13:02 smithi136 bash[75164]: cluster 2024-09-13T00:13:01.494280+0000 mgr.y (mgr.24661) 1077 : cluster [DBG] pgmap v1760: 105 pgs: 105 active+clean; 126 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 2.4 MiB/s rd, 3.1 MiB/s wr, 5 op/s 2024-09-13T00:13:03.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:13:02 smithi181 bash[67587]: cluster 2024-09-13T00:13:01.494280+0000 mgr.y (mgr.24661) 1077 : cluster [DBG] pgmap v1760: 105 pgs: 105 active+clean; 126 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 2.4 MiB/s rd, 3.1 MiB/s wr, 5 op/s 2024-09-13T00:13:04.256 INFO:tasks.workunit.client.0.smithi136.stdout: 679: throughput=4.99MB/sec pending data=0 2024-09-13T00:13:04.256 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-s3jk07QQLM0-dQz off=155249 len=1335836 2024-09-13T00:13:04.260 INFO:tasks.workunit.client.0.smithi136.stdout:op 3565 completed, throughput=4.99MB/sec 2024-09-13T00:13:04.260 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-NKg3izkQRbSM90b off=2010528 len=114237 2024-09-13T00:13:04.265 INFO:tasks.workunit.client.0.smithi136.stdout:op 3566 completed, throughput=4.99MB/sec 2024-09-13T00:13:04.265 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-7F5IIjC07wcHLkP off=6824851 len=270350 2024-09-13T00:13:04.270 INFO:tasks.workunit.client.0.smithi136.stdout:op 3567 completed, throughput=5MB/sec 2024-09-13T00:13:04.270 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-r5dcMaRWhuAlcXJ off=8769677 len=1065717 2024-09-13T00:13:04.274 INFO:tasks.workunit.client.0.smithi136.stdout:op 3568 completed, throughput=5MB/sec 2024-09-13T00:13:04.274 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-APRx7l6sUclpm1x off=1410346 len=1086356 2024-09-13T00:13:04.277 INFO:tasks.workunit.client.0.smithi136.stdout:op 3569 completed, throughput=5MB/sec 2024-09-13T00:13:04.277 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Roqy7tg37i8Oo1U off=14488081 len=1629985 2024-09-13T00:13:04.289 INFO:tasks.workunit.client.0.smithi136.stdout:op 3570 completed, throughput=5MB/sec 2024-09-13T00:13:05.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:13:04 smithi136 bash[76371]: cluster 2024-09-13T00:13:03.495407+0000 mgr.y (mgr.24661) 1078 : cluster [DBG] pgmap v1761: 105 pgs: 105 active+clean; 126 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:13:05.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:13:04 smithi136 bash[75164]: cluster 2024-09-13T00:13:03.495407+0000 mgr.y (mgr.24661) 1078 : cluster [DBG] pgmap v1761: 105 pgs: 105 active+clean; 126 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:13:05.289 INFO:tasks.workunit.client.0.smithi136.stdout: 680: throughput=4.99MB/sec pending data=0 2024-09-13T00:13:05.289 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-X-xT8igPyqplJM5 off=958411 len=380460 2024-09-13T00:13:05.295 INFO:tasks.workunit.client.0.smithi136.stdout:op 3571 completed, throughput=4.99MB/sec 2024-09-13T00:13:05.296 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-QRuo6b7ro8ZER6M off=2727065 len=786899 2024-09-13T00:13:05.303 INFO:tasks.workunit.client.0.smithi136.stdout:op 3572 completed, throughput=4.99MB/sec 2024-09-13T00:13:05.303 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-7M4yDR329QwmHPK off=4613712 len=1757872 2024-09-13T00:13:05.308 INFO:tasks.workunit.client.0.smithi136.stdout:op 3573 completed, throughput=5MB/sec 2024-09-13T00:13:05.308 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-uw-7-EO3tr-nXg7 off=13414143 len=1305172 2024-09-13T00:13:05.311 INFO:tasks.workunit.client.0.smithi136.stdout:op 3574 completed, throughput=5MB/sec 2024-09-13T00:13:05.312 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-sCZT7XZ_6HRji6e off=2555089 len=946001 2024-09-13T00:13:05.316 INFO:tasks.workunit.client.0.smithi136.stdout:op 3575 completed, throughput=5MB/sec 2024-09-13T00:13:05.316 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-HxJUfDhVF6SBDdO off=475258 len=1877677 2024-09-13T00:13:05.322 INFO:tasks.workunit.client.0.smithi136.stdout:op 3576 completed, throughput=5MB/sec 2024-09-13T00:13:05.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:13:04 smithi181 bash[67587]: cluster 2024-09-13T00:13:03.495407+0000 mgr.y (mgr.24661) 1078 : cluster [DBG] pgmap v1761: 105 pgs: 105 active+clean; 126 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:13:06.322 INFO:tasks.workunit.client.0.smithi136.stdout: 681: throughput=5MB/sec pending data=0 2024-09-13T00:13:06.323 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-h2TW4Qcn64ptG8v off=2181204 len=1035858 2024-09-13T00:13:06.325 INFO:tasks.workunit.client.0.smithi136.stdout:op 3577 completed, throughput=5MB/sec 2024-09-13T00:13:06.326 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-rY52X4ELULgHQ7j off=6819403 len=1398721 2024-09-13T00:13:06.329 INFO:tasks.workunit.client.0.smithi136.stdout:op 3578 completed, throughput=5MB/sec 2024-09-13T00:13:06.329 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-jeW1uMN61ObqYw8 off=2940002 len=4665 2024-09-13T00:13:06.330 INFO:tasks.workunit.client.0.smithi136.stdout:op 3579 completed, throughput=5MB/sec 2024-09-13T00:13:06.330 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-5lx5XcNTpNaTPGJ off=21057926 len=1208502 2024-09-13T00:13:06.333 INFO:tasks.workunit.client.0.smithi136.stdout:op 3580 completed, throughput=5MB/sec 2024-09-13T00:13:07.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:13:06 smithi136 bash[75164]: cluster 2024-09-13T00:13:05.496293+0000 mgr.y (mgr.24661) 1079 : cluster [DBG] pgmap v1762: 105 pgs: 105 active+clean; 126 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:13:07.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:13:06 smithi136 bash[76371]: cluster 2024-09-13T00:13:05.496293+0000 mgr.y (mgr.24661) 1079 : cluster [DBG] pgmap v1762: 105 pgs: 105 active+clean; 126 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:13:07.334 INFO:tasks.workunit.client.0.smithi136.stdout: 682: throughput=4.99MB/sec pending data=0 2024-09-13T00:13:07.334 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-K1fgm-eHOmTuYBA off=5608986 len=941642 2024-09-13T00:13:07.342 INFO:tasks.workunit.client.0.smithi136.stdout:op 3581 completed, throughput=4.99MB/sec 2024-09-13T00:13:07.343 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-99Q-oWjqX0RWGiB off=16808905 len=1611033 2024-09-13T00:13:07.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:13:06 smithi181 bash[67587]: cluster 2024-09-13T00:13:05.496293+0000 mgr.y (mgr.24661) 1079 : cluster [DBG] pgmap v1762: 105 pgs: 105 active+clean; 126 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:13:07.355 INFO:tasks.workunit.client.0.smithi136.stdout:op 3582 completed, throughput=5MB/sec 2024-09-13T00:13:07.355 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-vEdpQl_DSnWl1o0 off=1494779 len=666634 2024-09-13T00:13:07.360 INFO:tasks.workunit.client.0.smithi136.stdout:op 3583 completed, throughput=5MB/sec 2024-09-13T00:13:07.360 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-qwi6U5Y41OAfirG off=0 len=1735680 2024-09-13T00:13:07.364 INFO:tasks.workunit.client.0.smithi136.stdout:op 3584 completed, throughput=5MB/sec 2024-09-13T00:13:07.364 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-sSi0q6LcIb6B2Zc off=0 len=1492377 2024-09-13T00:13:07.367 INFO:tasks.workunit.client.0.smithi136.stdout:op 3585 completed, throughput=5MB/sec 2024-09-13T00:13:08.368 INFO:tasks.workunit.client.0.smithi136.stdout: 683: throughput=4.99MB/sec pending data=0 2024-09-13T00:13:08.368 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-sfdJbhxy7V6-et1 off=14948764 len=926355 2024-09-13T00:13:08.370 INFO:tasks.workunit.client.0.smithi136.stdout:op 3586 completed, throughput=5MB/sec 2024-09-13T00:13:08.371 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-MPEbqUE9PtPmtqg off=17793014 len=160347 2024-09-13T00:13:08.372 INFO:tasks.workunit.client.0.smithi136.stdout:op 3587 completed, throughput=5MB/sec 2024-09-13T00:13:08.372 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-L4ognc_jUkMeWQd off=1011193 len=32260 2024-09-13T00:13:08.373 INFO:tasks.workunit.client.0.smithi136.stdout:op 3588 completed, throughput=5MB/sec 2024-09-13T00:13:08.374 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-BPKTE0VchIL631y off=1805457 len=745374 2024-09-13T00:13:08.376 INFO:tasks.workunit.client.0.smithi136.stdout:op 3589 completed, throughput=5MB/sec 2024-09-13T00:13:08.376 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2EZu6LcfAdci_Ol off=12213040 len=1301451 2024-09-13T00:13:08.379 INFO:tasks.workunit.client.0.smithi136.stdout:op 3590 completed, throughput=5MB/sec 2024-09-13T00:13:08.379 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-X1gfp5_f5eMX3wG off=1002551 len=924544 2024-09-13T00:13:08.387 INFO:tasks.workunit.client.0.smithi136.stdout:op 3591 completed, throughput=5MB/sec 2024-09-13T00:13:08.387 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-P0ysQmP6O4iqa-H off=3035203 len=65693 2024-09-13T00:13:08.392 INFO:tasks.workunit.client.0.smithi136.stdout:op 3592 completed, throughput=5MB/sec 2024-09-13T00:13:08.392 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-huN5X7-FYtuT1gL off=3838422 len=1363912 2024-09-13T00:13:08.402 INFO:tasks.workunit.client.0.smithi136.stdout:op 3593 completed, throughput=5MB/sec 2024-09-13T00:13:09.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:13:08 smithi136 bash[75164]: cluster 2024-09-13T00:13:07.497489+0000 mgr.y (mgr.24661) 1080 : cluster [DBG] pgmap v1763: 105 pgs: 105 active+clean; 126 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 3.3 MiB/s rd, 2.0 MiB/s wr, 5 op/s 2024-09-13T00:13:09.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:13:08 smithi136 bash[75164]: audit 2024-09-13T00:13:08.791101+0000 mon.a (mon.0) 2212 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:13:09.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:13:08 smithi136 bash[76371]: cluster 2024-09-13T00:13:07.497489+0000 mgr.y (mgr.24661) 1080 : cluster [DBG] pgmap v1763: 105 pgs: 105 active+clean; 126 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 3.3 MiB/s rd, 2.0 MiB/s wr, 5 op/s 2024-09-13T00:13:09.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:13:08 smithi136 bash[76371]: audit 2024-09-13T00:13:08.791101+0000 mon.a (mon.0) 2212 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:13:09.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:13:08 smithi181 bash[67587]: cluster 2024-09-13T00:13:07.497489+0000 mgr.y (mgr.24661) 1080 : cluster [DBG] pgmap v1763: 105 pgs: 105 active+clean; 126 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 3.3 MiB/s rd, 2.0 MiB/s wr, 5 op/s 2024-09-13T00:13:09.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:13:08 smithi181 bash[67587]: audit 2024-09-13T00:13:08.791101+0000 mon.a (mon.0) 2212 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:13:09.403 INFO:tasks.workunit.client.0.smithi136.stdout: 684: throughput=4.99MB/sec pending data=0 2024-09-13T00:13:09.403 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-NRjqEjFYUlxP6Ad off=1407418 len=456236 2024-09-13T00:13:09.408 INFO:tasks.workunit.client.0.smithi136.stdout:op 3594 completed, throughput=5MB/sec 2024-09-13T00:13:09.409 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-KEFX51rMAu4CO-e off=5948357 len=992090 2024-09-13T00:13:09.418 INFO:tasks.workunit.client.0.smithi136.stdout:op 3595 completed, throughput=5MB/sec 2024-09-13T00:13:09.418 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Abzj5ULjdFCMVSA off=4037672 len=421254 2024-09-13T00:13:09.425 INFO:tasks.workunit.client.0.smithi136.stdout:op 3596 completed, throughput=5MB/sec 2024-09-13T00:13:09.425 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-rMToENcmDOQqMMV off=3288838 len=1066604 2024-09-13T00:13:09.433 INFO:tasks.workunit.client.0.smithi136.stdout:op 3597 completed, throughput=5MB/sec 2024-09-13T00:13:09.433 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ohEVnJe3vqs6fqT off=498830 len=1980227 2024-09-13T00:13:09.449 INFO:tasks.workunit.client.0.smithi136.stdout:op 3598 completed, throughput=5MB/sec 2024-09-13T00:13:09.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:13:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:13:09] "GET /metrics HTTP/1.1" 200 34138 "" "Prometheus/2.43.0" 2024-09-13T00:13:10.449 INFO:tasks.workunit.client.0.smithi136.stdout: 685: throughput=4.99MB/sec pending data=0 2024-09-13T00:13:10.449 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Ns7B9U8kTyOSz2A off=1839966 len=81157 2024-09-13T00:13:10.451 INFO:tasks.workunit.client.0.smithi136.stdout:op 3599 completed, throughput=4.99MB/sec 2024-09-13T00:13:10.451 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-F3usj_7slB2ExDL off=13645481 len=1617232 2024-09-13T00:13:10.462 INFO:tasks.workunit.client.0.smithi136.stdout:op 3600 completed, throughput=5MB/sec 2024-09-13T00:13:10.462 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-nL_RNJLuCMaTKaa off=994487 len=272706 2024-09-13T00:13:10.464 INFO:tasks.workunit.client.0.smithi136.stdout:op 3601 completed, throughput=5MB/sec 2024-09-13T00:13:10.464 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2S2Ae6KPoTLle0k off=6639107 len=102503 2024-09-13T00:13:10.465 INFO:tasks.workunit.client.0.smithi136.stdout:op 3602 completed, throughput=5MB/sec 2024-09-13T00:13:10.465 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-SAQVi45GMBxIWam off=1679536 len=1229412 2024-09-13T00:13:10.473 INFO:tasks.workunit.client.0.smithi136.stdout:op 3603 completed, throughput=5MB/sec 2024-09-13T00:13:10.474 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-n4NnHYZAzVcsKeE off=2963631 len=1228149 2024-09-13T00:13:10.478 INFO:tasks.workunit.client.0.smithi136.stdout:op 3604 completed, throughput=5MB/sec 2024-09-13T00:13:10.478 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-VqB8BJTuiKntwP4 off=2928315 len=1706918 2024-09-13T00:13:10.489 INFO:tasks.workunit.client.0.smithi136.stdout:op 3605 completed, throughput=5MB/sec 2024-09-13T00:13:11.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:13:10 smithi136 bash[75164]: cluster 2024-09-13T00:13:09.498499+0000 mgr.y (mgr.24661) 1081 : cluster [DBG] pgmap v1764: 105 pgs: 105 active+clean; 126 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 2.8 MiB/s rd, 1.7 MiB/s wr, 4 op/s 2024-09-13T00:13:11.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:13:10 smithi136 bash[76371]: cluster 2024-09-13T00:13:09.498499+0000 mgr.y (mgr.24661) 1081 : cluster [DBG] pgmap v1764: 105 pgs: 105 active+clean; 126 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 2.8 MiB/s rd, 1.7 MiB/s wr, 4 op/s 2024-09-13T00:13:11.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:13:10 smithi181 bash[67587]: cluster 2024-09-13T00:13:09.498499+0000 mgr.y (mgr.24661) 1081 : cluster [DBG] pgmap v1764: 105 pgs: 105 active+clean; 126 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 2.8 MiB/s rd, 1.7 MiB/s wr, 4 op/s 2024-09-13T00:13:11.489 INFO:tasks.workunit.client.0.smithi136.stdout: 686: throughput=4.99MB/sec pending data=0 2024-09-13T00:13:11.490 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-pSJ2j83gsV7ulnJ off=4075371 len=960415 2024-09-13T00:13:11.492 INFO:tasks.workunit.client.0.smithi136.stdout:op 3606 completed, throughput=5MB/sec 2024-09-13T00:13:11.492 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-iEnrqP4KFvIY5Z2 off=13734408 len=1958267 2024-09-13T00:13:11.508 INFO:tasks.workunit.client.0.smithi136.stdout:op 3607 completed, throughput=5MB/sec 2024-09-13T00:13:11.508 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-HtulShxFV18bPbC off=4295598 len=478047 2024-09-13T00:13:11.513 INFO:tasks.workunit.client.0.smithi136.stdout:op 3608 completed, throughput=5MB/sec 2024-09-13T00:13:11.513 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ptkJ_L7qmRbjVyX off=1593976 len=1909667 2024-09-13T00:13:11.518 INFO:tasks.workunit.client.0.smithi136.stdout:op 3609 completed, throughput=5MB/sec 2024-09-13T00:13:12.519 INFO:tasks.workunit.client.0.smithi136.stdout: 687: throughput=4.99MB/sec pending data=0 2024-09-13T00:13:12.519 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-kzcP0tyy_lbMu1F off=7679100 len=858799 2024-09-13T00:13:12.527 INFO:tasks.workunit.client.0.smithi136.stdout:op 3610 completed, throughput=5MB/sec 2024-09-13T00:13:12.527 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-h0Gq9FeDg2QNRGs off=529827 len=529734 2024-09-13T00:13:12.528 INFO:tasks.workunit.client.0.smithi136.stdout:op 3611 completed, throughput=5MB/sec 2024-09-13T00:13:12.528 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-twFWT8_JKH0v61j off=5962145 len=161278 2024-09-13T00:13:12.529 INFO:tasks.workunit.client.0.smithi136.stdout:op 3612 completed, throughput=5MB/sec 2024-09-13T00:13:12.529 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-edgnPmDsblEa2O6 off=0 len=1872453 2024-09-13T00:13:12.542 INFO:tasks.workunit.client.0.smithi136.stdout:op 3613 completed, throughput=5MB/sec 2024-09-13T00:13:12.543 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-u18_RvxCI9FWIAN off=1638034 len=1880451 2024-09-13T00:13:12.546 INFO:tasks.workunit.client.0.smithi136.stdout:op 3614 completed, throughput=5MB/sec 2024-09-13T00:13:12.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:13:12 smithi181 bash[95505]: ts=2024-09-13T00:13:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:13:13.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:13:12 smithi136 bash[75164]: cluster 2024-09-13T00:13:11.499626+0000 mgr.y (mgr.24661) 1082 : cluster [DBG] pgmap v1765: 105 pgs: 105 active+clean; 126 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.8 MiB/s wr, 5 op/s 2024-09-13T00:13:13.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:13:13 smithi136 bash[76371]: cluster 2024-09-13T00:13:11.499626+0000 mgr.y (mgr.24661) 1082 : cluster [DBG] pgmap v1765: 105 pgs: 105 active+clean; 126 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.8 MiB/s wr, 5 op/s 2024-09-13T00:13:13.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:13:12 smithi181 bash[67587]: cluster 2024-09-13T00:13:11.499626+0000 mgr.y (mgr.24661) 1082 : cluster [DBG] pgmap v1765: 105 pgs: 105 active+clean; 126 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.8 MiB/s wr, 5 op/s 2024-09-13T00:13:13.547 INFO:tasks.workunit.client.0.smithi136.stdout: 688: throughput=4.99MB/sec pending data=0 2024-09-13T00:13:13.547 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-XGKeCTdPy9gJTQF off=1875776 len=1224497 2024-09-13T00:13:13.556 INFO:tasks.workunit.client.0.smithi136.stdout:op 3615 completed, throughput=5MB/sec 2024-09-13T00:13:13.556 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-0ikJwjQ0y0eTKp7 off=16786132 len=1969881 2024-09-13T00:13:13.561 INFO:tasks.workunit.client.0.smithi136.stdout:op 3616 completed, throughput=5MB/sec 2024-09-13T00:13:13.561 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-LS4xSoGYQpmZx8F off=7926654 len=1116867 2024-09-13T00:13:13.570 INFO:tasks.workunit.client.0.smithi136.stdout:op 3617 completed, throughput=5MB/sec 2024-09-13T00:13:14.571 INFO:tasks.workunit.client.0.smithi136.stdout: 689: throughput=4.99MB/sec pending data=0 2024-09-13T00:13:14.571 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-T3XmZGy65yJqzqR off=3315544 len=188504 2024-09-13T00:13:14.572 INFO:tasks.workunit.client.0.smithi136.stdout:op 3618 completed, throughput=4.99MB/sec 2024-09-13T00:13:14.572 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-dioiS1MhmLyQmAM off=6993267 len=951269 2024-09-13T00:13:14.576 INFO:tasks.workunit.client.0.smithi136.stdout:op 3619 completed, throughput=4.99MB/sec 2024-09-13T00:13:14.576 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-tgJmHdPzJYSkFLR off=1080435 len=1232346 2024-09-13T00:13:14.586 INFO:tasks.workunit.client.0.smithi136.stdout:op 3620 completed, throughput=5MB/sec 2024-09-13T00:13:14.587 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Q5zB89-CTnwu4Gj off=354546 len=1127657 2024-09-13T00:13:14.590 INFO:tasks.workunit.client.0.smithi136.stdout:op 3621 completed, throughput=5MB/sec 2024-09-13T00:13:14.590 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-F5yG1dTpArpPzLf off=99962 len=1119663 2024-09-13T00:13:14.599 INFO:tasks.workunit.client.0.smithi136.stdout:op 3622 completed, throughput=5MB/sec 2024-09-13T00:13:14.599 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-IVdlvOR-APPvvON off=7336749 len=1448773 2024-09-13T00:13:14.603 INFO:tasks.workunit.client.0.smithi136.stdout:op 3623 completed, throughput=5MB/sec 2024-09-13T00:13:15.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:13:14 smithi136 bash[75164]: cluster 2024-09-13T00:13:13.500576+0000 mgr.y (mgr.24661) 1083 : cluster [DBG] pgmap v1766: 105 pgs: 105 active+clean; 126 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 2.8 MiB/s rd, 1.8 MiB/s wr, 5 op/s 2024-09-13T00:13:15.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:13:14 smithi136 bash[76371]: cluster 2024-09-13T00:13:13.500576+0000 mgr.y (mgr.24661) 1083 : cluster [DBG] pgmap v1766: 105 pgs: 105 active+clean; 126 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 2.8 MiB/s rd, 1.8 MiB/s wr, 5 op/s 2024-09-13T00:13:15.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:13:14 smithi181 bash[67587]: cluster 2024-09-13T00:13:13.500576+0000 mgr.y (mgr.24661) 1083 : cluster [DBG] pgmap v1766: 105 pgs: 105 active+clean; 126 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 2.8 MiB/s rd, 1.8 MiB/s wr, 5 op/s 2024-09-13T00:13:15.603 INFO:tasks.workunit.client.0.smithi136.stdout: 690: throughput=4.99MB/sec pending data=0 2024-09-13T00:13:15.603 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-HludWJHlQ5WHQ2R off=1822342 len=813910 2024-09-13T00:13:15.606 INFO:tasks.workunit.client.0.smithi136.stdout:op 3624 completed, throughput=5MB/sec 2024-09-13T00:13:15.606 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-B-ma7OvqaxyfOAA off=3602057 len=742868 2024-09-13T00:13:15.613 INFO:tasks.workunit.client.0.smithi136.stdout:op 3625 completed, throughput=5MB/sec 2024-09-13T00:13:15.613 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-0Iev_ddvQdbzOz5 off=6974187 len=843199 2024-09-13T00:13:15.621 INFO:tasks.workunit.client.0.smithi136.stdout:op 3626 completed, throughput=5MB/sec 2024-09-13T00:13:15.621 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-nJbHtVDZnrRMA8Z off=8889736 len=1513281 2024-09-13T00:13:15.625 INFO:tasks.workunit.client.0.smithi136.stdout:op 3627 completed, throughput=5MB/sec 2024-09-13T00:13:15.625 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-xGRjv-3yE-EGcv_ off=10647260 len=1099562 2024-09-13T00:13:15.634 INFO:tasks.workunit.client.0.smithi136.stdout:op 3628 completed, throughput=5MB/sec 2024-09-13T00:13:16.634 INFO:tasks.workunit.client.0.smithi136.stdout: 691: throughput=4.99MB/sec pending data=0 2024-09-13T00:13:16.635 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-rgGvcdt23V8DuVu off=7486235 len=821927 2024-09-13T00:13:16.637 INFO:tasks.workunit.client.0.smithi136.stdout:op 3629 completed, throughput=4.99MB/sec 2024-09-13T00:13:16.637 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Q7X_sR0vBFLGNkF off=951915 len=1066640 2024-09-13T00:13:16.640 INFO:tasks.workunit.client.0.smithi136.stdout:op 3630 completed, throughput=5MB/sec 2024-09-13T00:13:16.640 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-GXZrfGbz20flsuy off=710148 len=1951816 2024-09-13T00:13:16.656 INFO:tasks.workunit.client.0.smithi136.stdout:op 3631 completed, throughput=5MB/sec 2024-09-13T00:13:16.656 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-NMfyXLPPlJdlhBw off=7248299 len=835598 2024-09-13T00:13:16.664 INFO:tasks.workunit.client.0.smithi136.stdout:op 3632 completed, throughput=5MB/sec 2024-09-13T00:13:16.664 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-me6wqGHhqLReb4c off=98080 len=2061472 2024-09-13T00:13:16.680 INFO:tasks.workunit.client.0.smithi136.stdout:op 3633 completed, throughput=5MB/sec 2024-09-13T00:13:17.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:13:16 smithi136 bash[76371]: cluster 2024-09-13T00:13:15.501361+0000 mgr.y (mgr.24661) 1084 : cluster [DBG] pgmap v1767: 105 pgs: 105 active+clean; 126 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.1 MiB/s wr, 5 op/s 2024-09-13T00:13:17.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:13:16 smithi136 bash[75164]: cluster 2024-09-13T00:13:15.501361+0000 mgr.y (mgr.24661) 1084 : cluster [DBG] pgmap v1767: 105 pgs: 105 active+clean; 126 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.1 MiB/s wr, 5 op/s 2024-09-13T00:13:17.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:13:16 smithi181 bash[67587]: cluster 2024-09-13T00:13:15.501361+0000 mgr.y (mgr.24661) 1084 : cluster [DBG] pgmap v1767: 105 pgs: 105 active+clean; 126 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.1 MiB/s wr, 5 op/s 2024-09-13T00:13:17.680 INFO:tasks.workunit.client.0.smithi136.stdout: 692: throughput=5MB/sec pending data=0 2024-09-13T00:13:17.680 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-wWDuwhgU0OBcdJE off=9992359 len=1304319 2024-09-13T00:13:17.683 INFO:tasks.workunit.client.0.smithi136.stdout:op 3634 completed, throughput=5MB/sec 2024-09-13T00:13:17.683 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-NgVZyQp0mQjWSOG off=1329337 len=1622655 2024-09-13T00:13:17.696 INFO:tasks.workunit.client.0.smithi136.stdout:op 3635 completed, throughput=5MB/sec 2024-09-13T00:13:17.696 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-cHFk8rTxZdCbpTI off=11950677 len=1598557 2024-09-13T00:13:17.701 INFO:tasks.workunit.client.0.smithi136.stdout:op 3636 completed, throughput=5MB/sec 2024-09-13T00:13:18.701 INFO:tasks.workunit.client.0.smithi136.stdout: 693: throughput=4.99MB/sec pending data=0 2024-09-13T00:13:18.701 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-QXpHUxYLqlmejJ2 off=6872911 len=714576 2024-09-13T00:13:18.708 INFO:tasks.workunit.client.0.smithi136.stdout:op 3637 completed, throughput=5MB/sec 2024-09-13T00:13:18.708 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-bEoAJERm_xQwwHv off=861443 len=1062446 2024-09-13T00:13:18.717 INFO:tasks.workunit.client.0.smithi136.stdout:op 3638 completed, throughput=5MB/sec 2024-09-13T00:13:18.717 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-YMq6XVBFYseNU4O off=15693132 len=473402 2024-09-13T00:13:18.724 INFO:tasks.workunit.client.0.smithi136.stdout:op 3639 completed, throughput=5MB/sec 2024-09-13T00:13:18.724 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-q63xdgcKpq6z7fk off=10485031 len=1156450 2024-09-13T00:13:18.726 INFO:tasks.workunit.client.0.smithi136.stdout:op 3640 completed, throughput=5MB/sec 2024-09-13T00:13:18.726 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-3G9kWNz_9TV8gr3 off=3068799 len=1402962 2024-09-13T00:13:18.729 INFO:tasks.workunit.client.0.smithi136.stdout:op 3641 completed, throughput=5MB/sec 2024-09-13T00:13:19.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:13:18 smithi136 bash[75164]: cluster 2024-09-13T00:13:17.502473+0000 mgr.y (mgr.24661) 1085 : cluster [DBG] pgmap v1768: 105 pgs: 105 active+clean; 126 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:13:19.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:13:18 smithi136 bash[76371]: cluster 2024-09-13T00:13:17.502473+0000 mgr.y (mgr.24661) 1085 : cluster [DBG] pgmap v1768: 105 pgs: 105 active+clean; 126 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:13:19.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:13:18 smithi181 bash[67587]: cluster 2024-09-13T00:13:17.502473+0000 mgr.y (mgr.24661) 1085 : cluster [DBG] pgmap v1768: 105 pgs: 105 active+clean; 126 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:13:19.729 INFO:tasks.workunit.client.0.smithi136.stdout: 694: throughput=4.99MB/sec pending data=0 2024-09-13T00:13:19.729 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-3QfIhweXydrkv0L off=11356930 len=2050213 2024-09-13T00:13:19.734 INFO:tasks.workunit.client.0.smithi136.stdout:op 3642 completed, throughput=5MB/sec 2024-09-13T00:13:19.734 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-s7CnKZVk-YsL1S2 off=9314352 len=618138 2024-09-13T00:13:19.741 INFO:tasks.workunit.client.0.smithi136.stdout:op 3643 completed, throughput=5MB/sec 2024-09-13T00:13:19.741 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-QN5KebFIqTJD_fp off=1767165 len=112107 2024-09-13T00:13:19.745 INFO:tasks.workunit.client.0.smithi136.stdout:op 3644 completed, throughput=5MB/sec 2024-09-13T00:13:19.745 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-VpivjKXKxk5Ni7P off=1006964 len=1835276 2024-09-13T00:13:19.757 INFO:tasks.workunit.client.0.smithi136.stdout:op 3645 completed, throughput=5MB/sec 2024-09-13T00:13:19.757 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-I4O9VZtiQ9njAGC off=11556421 len=604190 2024-09-13T00:13:19.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:13:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:13:19] "GET /metrics HTTP/1.1" 200 34138 "" "Prometheus/2.43.0" 2024-09-13T00:13:19.759 INFO:tasks.workunit.client.0.smithi136.stdout:op 3646 completed, throughput=5MB/sec 2024-09-13T00:13:20.759 INFO:tasks.workunit.client.0.smithi136.stdout: 695: throughput=4.99MB/sec pending data=0 2024-09-13T00:13:20.760 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-fShB1OOzUbk9Yox off=13703181 len=871312 2024-09-13T00:13:20.766 INFO:tasks.workunit.client.0.smithi136.stdout:op 3647 completed, throughput=4.99MB/sec 2024-09-13T00:13:20.766 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-h0Gq9FeDg2QNRGs off=909072 len=1821709 2024-09-13T00:13:20.779 INFO:tasks.workunit.client.0.smithi136.stdout:op 3648 completed, throughput=5MB/sec 2024-09-13T00:13:20.779 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-BPK0IRgfgBPKeF3 off=2802466 len=240202 2024-09-13T00:13:20.784 INFO:tasks.workunit.client.0.smithi136.stdout:op 3649 completed, throughput=5MB/sec 2024-09-13T00:13:20.784 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-E8ZT-e3kuhbnJRq off=474382 len=1544786 2024-09-13T00:13:20.788 INFO:tasks.workunit.client.0.smithi136.stdout:op 3650 completed, throughput=5MB/sec 2024-09-13T00:13:20.788 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-w72icWw0VJ8lNry off=1141863 len=481683 2024-09-13T00:13:20.789 INFO:tasks.workunit.client.0.smithi136.stdout:op 3651 completed, throughput=5MB/sec 2024-09-13T00:13:20.789 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-td93MxoPlYePS91 off=5587628 len=17933 2024-09-13T00:13:20.790 INFO:tasks.workunit.client.0.smithi136.stdout:op 3652 completed, throughput=5MB/sec 2024-09-13T00:13:20.790 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-g5ZAhds_ihkS2St off=20271201 len=691774 2024-09-13T00:13:20.797 INFO:tasks.workunit.client.0.smithi136.stdout:op 3653 completed, throughput=5MB/sec 2024-09-13T00:13:21.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:13:20 smithi136 bash[75164]: cluster 2024-09-13T00:13:19.503372+0000 mgr.y (mgr.24661) 1086 : cluster [DBG] pgmap v1769: 105 pgs: 105 active+clean; 126 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:13:21.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:13:20 smithi136 bash[76371]: cluster 2024-09-13T00:13:19.503372+0000 mgr.y (mgr.24661) 1086 : cluster [DBG] pgmap v1769: 105 pgs: 105 active+clean; 126 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:13:21.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:13:20 smithi181 bash[67587]: cluster 2024-09-13T00:13:19.503372+0000 mgr.y (mgr.24661) 1086 : cluster [DBG] pgmap v1769: 105 pgs: 105 active+clean; 126 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:13:21.797 INFO:tasks.workunit.client.0.smithi136.stdout: 696: throughput=4.99MB/sec pending data=0 2024-09-13T00:13:21.797 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-gC6Poj4cgo288II off=1187893 len=258822 2024-09-13T00:13:21.799 INFO:tasks.workunit.client.0.smithi136.stdout:op 3654 completed, throughput=4.99MB/sec 2024-09-13T00:13:21.799 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-TTMre3IhF3NyP0x off=2792105 len=858651 2024-09-13T00:13:21.808 INFO:tasks.workunit.client.0.smithi136.stdout:op 3655 completed, throughput=4.99MB/sec 2024-09-13T00:13:21.808 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ZtdbjrIgH6w2_2- off=16493484 len=623043 2024-09-13T00:13:21.810 INFO:tasks.workunit.client.0.smithi136.stdout:op 3656 completed, throughput=5MB/sec 2024-09-13T00:13:21.810 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-DayLEWmwCrK0SgV off=21164214 len=497871 2024-09-13T00:13:21.812 INFO:tasks.workunit.client.0.smithi136.stdout:op 3657 completed, throughput=5MB/sec 2024-09-13T00:13:21.812 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-4EOyPdZ62VcnhWl off=517739 len=1155736 2024-09-13T00:13:21.820 INFO:tasks.workunit.client.0.smithi136.stdout:op 3658 completed, throughput=5MB/sec 2024-09-13T00:13:21.821 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-EZLRXyXU-XPn_M7 off=3100255 len=2070257 2024-09-13T00:13:21.833 INFO:tasks.workunit.client.0.smithi136.stdout:op 3659 completed, throughput=5MB/sec 2024-09-13T00:13:22.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:13:22 smithi181 bash[95505]: ts=2024-09-13T00:13:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:13:22.833 INFO:tasks.workunit.client.0.smithi136.stdout: 697: throughput=4.99MB/sec pending data=0 2024-09-13T00:13:22.833 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-9XJyAsD9cvjBEq5 off=2299535 len=750758 2024-09-13T00:13:22.842 INFO:tasks.workunit.client.0.smithi136.stdout:op 3660 completed, throughput=4.99MB/sec 2024-09-13T00:13:22.842 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-3b5yAQtXfvp8qub off=4821681 len=1861662 2024-09-13T00:13:22.854 INFO:tasks.workunit.client.0.smithi136.stdout:op 3661 completed, throughput=5MB/sec 2024-09-13T00:13:22.854 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-cGYw-uVIicUl9AL off=18483441 len=568262 2024-09-13T00:13:22.862 INFO:tasks.workunit.client.0.smithi136.stdout:op 3662 completed, throughput=5MB/sec 2024-09-13T00:13:22.862 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-oy-ixB23heFryVe off=5176843 len=1751789 2024-09-13T00:13:22.864 INFO:tasks.workunit.client.0.smithi136.stdout:op 3663 completed, throughput=5MB/sec 2024-09-13T00:13:22.864 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-uQ0eswn60LXPHX8 off=12266002 len=40893 2024-09-13T00:13:22.868 INFO:tasks.workunit.client.0.smithi136.stdout:op 3664 completed, throughput=5MB/sec 2024-09-13T00:13:22.868 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-v8B93muZxI619t- off=17370 len=112202 2024-09-13T00:13:22.872 INFO:tasks.workunit.client.0.smithi136.stdout:op 3665 completed, throughput=5MB/sec 2024-09-13T00:13:23.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:13:22 smithi136 bash[75164]: cluster 2024-09-13T00:13:21.504361+0000 mgr.y (mgr.24661) 1087 : cluster [DBG] pgmap v1770: 105 pgs: 105 active+clean; 126 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:13:23.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:13:22 smithi136 bash[76371]: cluster 2024-09-13T00:13:21.504361+0000 mgr.y (mgr.24661) 1087 : cluster [DBG] pgmap v1770: 105 pgs: 105 active+clean; 126 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:13:23.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:13:22 smithi181 bash[67587]: cluster 2024-09-13T00:13:21.504361+0000 mgr.y (mgr.24661) 1087 : cluster [DBG] pgmap v1770: 105 pgs: 105 active+clean; 126 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:13:23.873 INFO:tasks.workunit.client.0.smithi136.stdout: 698: throughput=4.99MB/sec pending data=0 2024-09-13T00:13:23.873 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-gBV9v4VUoC4hl1N off=13589048 len=1237422 2024-09-13T00:13:23.882 INFO:tasks.workunit.client.0.smithi136.stdout:op 3666 completed, throughput=4.99MB/sec 2024-09-13T00:13:23.882 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-hCdOP4fXtQNfkim off=542809 len=1793104 2024-09-13T00:13:23.894 INFO:tasks.workunit.client.0.smithi136.stdout:op 3667 completed, throughput=5MB/sec 2024-09-13T00:13:23.894 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-YlhaM6grJF6wDhO off=7410892 len=1462553 2024-09-13T00:13:23.898 INFO:tasks.workunit.client.0.smithi136.stdout:op 3668 completed, throughput=5MB/sec 2024-09-13T00:13:23.898 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ZuJkVbq1O3YnESi off=19836048 len=1967013 2024-09-13T00:13:23.902 INFO:tasks.workunit.client.0.smithi136.stdout:op 3669 completed, throughput=5MB/sec 2024-09-13T00:13:24.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:13:24 smithi136 bash[75164]: audit 2024-09-13T00:13:23.791560+0000 mon.a (mon.0) 2213 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:13:24.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:13:24 smithi136 bash[76371]: audit 2024-09-13T00:13:23.791560+0000 mon.a (mon.0) 2213 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:13:24.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:13:24 smithi181 bash[67587]: audit 2024-09-13T00:13:23.791560+0000 mon.a (mon.0) 2213 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:13:24.902 INFO:tasks.workunit.client.0.smithi136.stdout: 699: throughput=4.99MB/sec pending data=0 2024-09-13T00:13:24.902 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-pt6Eu2QIA5rYmG2 off=9035406 len=346876 2024-09-13T00:13:24.908 INFO:tasks.workunit.client.0.smithi136.stdout:op 3670 completed, throughput=4.99MB/sec 2024-09-13T00:13:24.908 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-r83nCDyE9V3AZnD off=670745 len=434638 2024-09-13T00:13:24.910 INFO:tasks.workunit.client.0.smithi136.stdout:op 3671 completed, throughput=5MB/sec 2024-09-13T00:13:24.910 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-a81YEqL0aYVo9FQ off=10666891 len=193532 2024-09-13T00:13:24.915 INFO:tasks.workunit.client.0.smithi136.stdout:op 3672 completed, throughput=5MB/sec 2024-09-13T00:13:24.915 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj--ZdFTgHw0MgcuDK off=6568239 len=703694 2024-09-13T00:13:24.922 INFO:tasks.workunit.client.0.smithi136.stdout:op 3673 completed, throughput=5MB/sec 2024-09-13T00:13:24.922 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Eb-c24sfwvvXaOC off=451640 len=1854114 2024-09-13T00:13:24.934 INFO:tasks.workunit.client.0.smithi136.stdout:op 3674 completed, throughput=5MB/sec 2024-09-13T00:13:24.934 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Cgl1wjfM1FNLv4D off=4552779 len=1080829 2024-09-13T00:13:24.937 INFO:tasks.workunit.client.0.smithi136.stdout:op 3675 completed, throughput=5MB/sec 2024-09-13T00:13:25.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:13:25 smithi136 bash[75164]: cluster 2024-09-13T00:13:23.505424+0000 mgr.y (mgr.24661) 1088 : cluster [DBG] pgmap v1771: 105 pgs: 105 active+clean; 126 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:13:25.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:13:25 smithi136 bash[76371]: cluster 2024-09-13T00:13:23.505424+0000 mgr.y (mgr.24661) 1088 : cluster [DBG] pgmap v1771: 105 pgs: 105 active+clean; 126 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:13:25.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:13:25 smithi181 bash[67587]: cluster 2024-09-13T00:13:23.505424+0000 mgr.y (mgr.24661) 1088 : cluster [DBG] pgmap v1771: 105 pgs: 105 active+clean; 126 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:13:25.938 INFO:tasks.workunit.client.0.smithi136.stdout: 700: throughput=4.99MB/sec pending data=0 2024-09-13T00:13:25.938 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-QWKfTHmmup_y3vJ off=6719950 len=1740512 2024-09-13T00:13:25.942 INFO:tasks.workunit.client.0.smithi136.stdout:op 3676 completed, throughput=5MB/sec 2024-09-13T00:13:25.942 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-MWYH2JzuacwEROe off=8189439 len=1823851 2024-09-13T00:13:25.947 INFO:tasks.workunit.client.0.smithi136.stdout:op 3677 completed, throughput=5MB/sec 2024-09-13T00:13:25.947 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-WoOHYdvI7-dr1gT off=219130 len=1994987 2024-09-13T00:13:25.952 INFO:tasks.workunit.client.0.smithi136.stdout:op 3678 completed, throughput=5MB/sec 2024-09-13T00:13:26.952 INFO:tasks.workunit.client.0.smithi136.stdout: 701: throughput=4.99MB/sec pending data=0 2024-09-13T00:13:26.953 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-pPiQlgtQ5qnGPb9 off=23264812 len=2096047 2024-09-13T00:13:26.969 INFO:tasks.workunit.client.0.smithi136.stdout:op 3679 completed, throughput=5MB/sec 2024-09-13T00:13:26.969 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-PRAOOEz6oVfbKUH off=12882564 len=302840 2024-09-13T00:13:26.971 INFO:tasks.workunit.client.0.smithi136.stdout:op 3680 completed, throughput=5MB/sec 2024-09-13T00:13:26.971 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-pSJ2j83gsV7ulnJ off=3893700 len=940821 2024-09-13T00:13:26.980 INFO:tasks.workunit.client.0.smithi136.stdout:op 3681 completed, throughput=5MB/sec 2024-09-13T00:13:26.980 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ySoUrJDD4t0uZUb off=1486010 len=1135538 2024-09-13T00:13:26.991 INFO:tasks.workunit.client.0.smithi136.stdout:op 3682 completed, throughput=5MB/sec 2024-09-13T00:13:26.991 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-bGqCjIkbHtLO08g off=868002 len=856433 2024-09-13T00:13:26.994 INFO:tasks.workunit.client.0.smithi136.stdout:op 3683 completed, throughput=5MB/sec 2024-09-13T00:13:27.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:13:27 smithi181 bash[67587]: cluster 2024-09-13T00:13:25.506310+0000 mgr.y (mgr.24661) 1089 : cluster [DBG] pgmap v1772: 105 pgs: 105 active+clean; 126 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:13:27.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:13:27 smithi136 bash[75164]: cluster 2024-09-13T00:13:25.506310+0000 mgr.y (mgr.24661) 1089 : cluster [DBG] pgmap v1772: 105 pgs: 105 active+clean; 126 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:13:27.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:13:27 smithi136 bash[76371]: cluster 2024-09-13T00:13:25.506310+0000 mgr.y (mgr.24661) 1089 : cluster [DBG] pgmap v1772: 105 pgs: 105 active+clean; 126 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:13:27.994 INFO:tasks.workunit.client.0.smithi136.stdout: 702: throughput=4.99MB/sec pending data=0 2024-09-13T00:13:27.995 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ce2IEaSQvpDbdbY off=7683808 len=1683611 2024-09-13T00:13:28.009 INFO:tasks.workunit.client.0.smithi136.stdout:op 3684 completed, throughput=5MB/sec 2024-09-13T00:13:28.009 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-iSlh6bEYReF4DuB off=5053731 len=601654 2024-09-13T00:13:28.017 INFO:tasks.workunit.client.0.smithi136.stdout:op 3685 completed, throughput=5MB/sec 2024-09-13T00:13:28.017 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-vwfNchABakh3yIq off=2191015 len=710786 2024-09-13T00:13:28.023 INFO:tasks.workunit.client.0.smithi136.stdout:op 3686 completed, throughput=5MB/sec 2024-09-13T00:13:28.024 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-DEtn7o17M_S4mQX off=3854919 len=988880 2024-09-13T00:13:28.027 INFO:tasks.workunit.client.0.smithi136.stdout:op 3687 completed, throughput=5MB/sec 2024-09-13T00:13:28.027 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-j1astrUR2FyGQHh off=9377514 len=1659555 2024-09-13T00:13:28.031 INFO:tasks.workunit.client.0.smithi136.stdout:op 3688 completed, throughput=5MB/sec 2024-09-13T00:13:29.032 INFO:tasks.workunit.client.0.smithi136.stdout: 703: throughput=4.99MB/sec pending data=0 2024-09-13T00:13:29.032 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-7n0svx16qZj0OV0 off=19046902 len=254667 2024-09-13T00:13:29.037 INFO:tasks.workunit.client.0.smithi136.stdout:op 3689 completed, throughput=4.99MB/sec 2024-09-13T00:13:29.037 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-tWzJm7cKq_M7CR7 off=2527813 len=179728 2024-09-13T00:13:29.044 INFO:tasks.workunit.client.0.smithi136.stdout:op 3690 completed, throughput=4.99MB/sec 2024-09-13T00:13:29.044 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-L0us9uvC5JwaxaB off=6224958 len=158027 2024-09-13T00:13:29.048 INFO:tasks.workunit.client.0.smithi136.stdout:op 3691 completed, throughput=4.99MB/sec 2024-09-13T00:13:29.049 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Ub6a3n1HaptvF50 off=1645038 len=286331 2024-09-13T00:13:29.054 INFO:tasks.workunit.client.0.smithi136.stdout:op 3692 completed, throughput=4.99MB/sec 2024-09-13T00:13:29.054 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-L6Ip4sK8QyVYJCJ off=10982014 len=1056130 2024-09-13T00:13:29.063 INFO:tasks.workunit.client.0.smithi136.stdout:op 3693 completed, throughput=5MB/sec 2024-09-13T00:13:29.063 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_EVBhWvKs2jtayx off=9502480 len=835111 2024-09-13T00:13:29.070 INFO:tasks.workunit.client.0.smithi136.stdout:op 3694 completed, throughput=5MB/sec 2024-09-13T00:13:29.070 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-uLAPfuuXBCwcuYX off=690249 len=1888052 2024-09-13T00:13:29.083 INFO:tasks.workunit.client.0.smithi136.stdout:op 3695 completed, throughput=5MB/sec 2024-09-13T00:13:29.083 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-mrTMiNoPFK_xLiA off=6565881 len=298529 2024-09-13T00:13:29.087 INFO:tasks.workunit.client.0.smithi136.stdout:op 3696 completed, throughput=5MB/sec 2024-09-13T00:13:29.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:13:29 smithi181 bash[67587]: cluster 2024-09-13T00:13:27.507411+0000 mgr.y (mgr.24661) 1090 : cluster [DBG] pgmap v1773: 105 pgs: 105 active+clean; 126 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:13:29.412 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:13:29 smithi136 bash[75164]: cluster 2024-09-13T00:13:27.507411+0000 mgr.y (mgr.24661) 1090 : cluster [DBG] pgmap v1773: 105 pgs: 105 active+clean; 126 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:13:29.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:13:29 smithi136 bash[76371]: cluster 2024-09-13T00:13:27.507411+0000 mgr.y (mgr.24661) 1090 : cluster [DBG] pgmap v1773: 105 pgs: 105 active+clean; 126 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:13:29.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:13:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:13:29] "GET /metrics HTTP/1.1" 200 34140 "" "Prometheus/2.43.0" 2024-09-13T00:13:30.088 INFO:tasks.workunit.client.0.smithi136.stdout: 704: throughput=4.99MB/sec pending data=0 2024-09-13T00:13:30.088 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-lYRqq3Xyl0RE_vS off=635504 len=546594 2024-09-13T00:13:30.095 INFO:tasks.workunit.client.0.smithi136.stdout:op 3697 completed, throughput=4.99MB/sec 2024-09-13T00:13:30.095 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Fpt-wqvdNlKw6Yd off=7806793 len=794410 2024-09-13T00:13:30.102 INFO:tasks.workunit.client.0.smithi136.stdout:op 3698 completed, throughput=4.99MB/sec 2024-09-13T00:13:30.102 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-rWNP8BByb1K0j-3 off=4064809 len=2054296 2024-09-13T00:13:30.107 INFO:tasks.workunit.client.0.smithi136.stdout:op 3699 completed, throughput=5MB/sec 2024-09-13T00:13:30.107 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-RVbs-JVotDLUHZT off=1170484 len=2081469 2024-09-13T00:13:30.120 INFO:tasks.workunit.client.0.smithi136.stdout:op 3700 completed, throughput=5MB/sec 2024-09-13T00:13:31.121 INFO:tasks.workunit.client.0.smithi136.stdout: 705: throughput=4.99MB/sec pending data=0 2024-09-13T00:13:31.121 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-xDLqpy-2dZwmqnF off=1294024 len=604623 2024-09-13T00:13:31.123 INFO:tasks.workunit.client.0.smithi136.stdout:op 3701 completed, throughput=4.99MB/sec 2024-09-13T00:13:31.123 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ljhMuLnotd8AakC off=1197719 len=1035463 2024-09-13T00:13:31.127 INFO:tasks.workunit.client.0.smithi136.stdout:op 3702 completed, throughput=5MB/sec 2024-09-13T00:13:31.128 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-eXTKICNwu0TVBwH off=6777078 len=922815 2024-09-13T00:13:31.130 INFO:tasks.workunit.client.0.smithi136.stdout:op 3703 completed, throughput=5MB/sec 2024-09-13T00:13:31.130 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-TZQHUQS-hY_Lfgm off=5069874 len=17684 2024-09-13T00:13:31.131 INFO:tasks.workunit.client.0.smithi136.stdout:op 3704 completed, throughput=5MB/sec 2024-09-13T00:13:31.131 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-kYDjg1wwi4cQ1XY off=19669309 len=1442584 2024-09-13T00:13:31.136 INFO:tasks.workunit.client.0.smithi136.stdout:op 3705 completed, throughput=5MB/sec 2024-09-13T00:13:31.136 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-IU_QKG6AdQQxBLf off=1257323 len=941372 2024-09-13T00:13:31.139 INFO:tasks.workunit.client.0.smithi136.stdout:op 3706 completed, throughput=5MB/sec 2024-09-13T00:13:31.139 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-fIDbl87Bd75YPEi off=19464040 len=1246044 2024-09-13T00:13:31.152 INFO:tasks.workunit.client.0.smithi136.stdout:op 3707 completed, throughput=5MB/sec 2024-09-13T00:13:31.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:13:31 smithi181 bash[67587]: cluster 2024-09-13T00:13:29.508299+0000 mgr.y (mgr.24661) 1091 : cluster [DBG] pgmap v1774: 105 pgs: 105 active+clean; 126 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 1.8 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:13:31.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:13:31 smithi136 bash[75164]: cluster 2024-09-13T00:13:29.508299+0000 mgr.y (mgr.24661) 1091 : cluster [DBG] pgmap v1774: 105 pgs: 105 active+clean; 126 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 1.8 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:13:31.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:13:31 smithi136 bash[76371]: cluster 2024-09-13T00:13:29.508299+0000 mgr.y (mgr.24661) 1091 : cluster [DBG] pgmap v1774: 105 pgs: 105 active+clean; 126 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 1.8 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:13:32.152 INFO:tasks.workunit.client.0.smithi136.stdout: 706: throughput=4.99MB/sec pending data=0 2024-09-13T00:13:32.152 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-lCY77TKABgAqMse off=903589 len=1237998 2024-09-13T00:13:32.164 INFO:tasks.workunit.client.0.smithi136.stdout:op 3708 completed, throughput=5MB/sec 2024-09-13T00:13:32.164 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-aNVisX-x_9Mewxc off=1530826 len=1272533 2024-09-13T00:13:32.168 INFO:tasks.workunit.client.0.smithi136.stdout:op 3709 completed, throughput=5MB/sec 2024-09-13T00:13:32.168 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ylznHO-RHB_qefS off=1824416 len=74787 2024-09-13T00:13:32.173 INFO:tasks.workunit.client.0.smithi136.stdout:op 3710 completed, throughput=5MB/sec 2024-09-13T00:13:32.173 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-dfFiSYwJcqL_CDD off=2101296 len=24126 2024-09-13T00:13:32.174 INFO:tasks.workunit.client.0.smithi136.stdout:op 3711 completed, throughput=5MB/sec 2024-09-13T00:13:32.174 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Ev5ObJoipm2OUVD off=4755716 len=390364 2024-09-13T00:13:32.179 INFO:tasks.workunit.client.0.smithi136.stdout:op 3712 completed, throughput=5MB/sec 2024-09-13T00:13:32.179 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-CQdCAGoGEEt-8q4 off=3338456 len=1895702 2024-09-13T00:13:32.183 INFO:tasks.workunit.client.0.smithi136.stdout:op 3713 completed, throughput=5MB/sec 2024-09-13T00:13:32.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:13:32 smithi181 bash[95505]: ts=2024-09-13T00:13:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:13:33.184 INFO:tasks.workunit.client.0.smithi136.stdout: 707: throughput=4.99MB/sec pending data=0 2024-09-13T00:13:33.184 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-M6tehv4Eykv32-5 off=14340226 len=2021881 2024-09-13T00:13:33.189 INFO:tasks.workunit.client.0.smithi136.stdout:op 3714 completed, throughput=5MB/sec 2024-09-13T00:13:33.189 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-OQtUHGrSzBEyrJ6 off=2959547 len=943093 2024-09-13T00:13:33.193 INFO:tasks.workunit.client.0.smithi136.stdout:op 3715 completed, throughput=5MB/sec 2024-09-13T00:13:33.193 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-gDNMxLLQjAm38Qs off=2855754 len=276561 2024-09-13T00:13:33.199 INFO:tasks.workunit.client.0.smithi136.stdout:op 3716 completed, throughput=5MB/sec 2024-09-13T00:13:33.199 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-VPzjDi277nLfnlV off=4184026 len=145043 2024-09-13T00:13:33.200 INFO:tasks.workunit.client.0.smithi136.stdout:op 3717 completed, throughput=5MB/sec 2024-09-13T00:13:33.200 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-TC6oHSZLvR2Mjb2 off=2692769 len=646930 2024-09-13T00:13:33.202 INFO:tasks.workunit.client.0.smithi136.stdout:op 3718 completed, throughput=5MB/sec 2024-09-13T00:13:33.202 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-2tiq5zZUOifRzhY off=4060462 len=500158 2024-09-13T00:13:33.208 INFO:tasks.workunit.client.0.smithi136.stdout:op 3719 completed, throughput=5MB/sec 2024-09-13T00:13:33.209 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj--DLgcL5KCWTIVUl off=1840019 len=394553 2024-09-13T00:13:33.214 INFO:tasks.workunit.client.0.smithi136.stdout:op 3720 completed, throughput=5MB/sec 2024-09-13T00:13:33.214 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-HUMyz5h5LlHFGAi off=20706356 len=1845614 2024-09-13T00:13:33.229 INFO:tasks.workunit.client.0.smithi136.stdout:op 3721 completed, throughput=5MB/sec 2024-09-13T00:13:33.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:13:33 smithi181 bash[67587]: cluster 2024-09-13T00:13:31.509445+0000 mgr.y (mgr.24661) 1092 : cluster [DBG] pgmap v1775: 105 pgs: 105 active+clean; 126 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 2.3 MiB/s rd, 3.0 MiB/s wr, 5 op/s 2024-09-13T00:13:33.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:13:33 smithi136 bash[75164]: cluster 2024-09-13T00:13:31.509445+0000 mgr.y (mgr.24661) 1092 : cluster [DBG] pgmap v1775: 105 pgs: 105 active+clean; 126 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 2.3 MiB/s rd, 3.0 MiB/s wr, 5 op/s 2024-09-13T00:13:33.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:13:33 smithi136 bash[76371]: cluster 2024-09-13T00:13:31.509445+0000 mgr.y (mgr.24661) 1092 : cluster [DBG] pgmap v1775: 105 pgs: 105 active+clean; 126 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 2.3 MiB/s rd, 3.0 MiB/s wr, 5 op/s 2024-09-13T00:13:34.230 INFO:tasks.workunit.client.0.smithi136.stdout: 708: throughput=5MB/sec pending data=0 2024-09-13T00:13:34.230 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-1UdSDKYe6MHfxzc off=5432293 len=1999544 2024-09-13T00:13:34.244 INFO:tasks.workunit.client.0.smithi136.stdout:op 3722 completed, throughput=5MB/sec 2024-09-13T00:13:34.244 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Z7VaAbVAb46WlV7 off=4064017 len=715318 2024-09-13T00:13:34.251 INFO:tasks.workunit.client.0.smithi136.stdout:op 3723 completed, throughput=5MB/sec 2024-09-13T00:13:34.251 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ciKdNdBSkjRoZ-8 off=4848637 len=1589649 2024-09-13T00:13:34.261 INFO:tasks.workunit.client.0.smithi136.stdout:op 3724 completed, throughput=5MB/sec 2024-09-13T00:13:35.261 INFO:tasks.workunit.client.0.smithi136.stdout: 709: throughput=4.99MB/sec pending data=0 2024-09-13T00:13:35.261 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-asEGkXsStFEM-qk off=8775336 len=1442012 2024-09-13T00:13:35.265 INFO:tasks.workunit.client.0.smithi136.stdout:op 3725 completed, throughput=5MB/sec 2024-09-13T00:13:35.265 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-YCptYrfmW7mOSpj off=6161785 len=313133 2024-09-13T00:13:35.267 INFO:tasks.workunit.client.0.smithi136.stdout:op 3726 completed, throughput=5MB/sec 2024-09-13T00:13:35.267 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-5XVYFAMbCrowSd7 off=1859958 len=360208 2024-09-13T00:13:35.269 INFO:tasks.workunit.client.0.smithi136.stdout:op 3727 completed, throughput=5MB/sec 2024-09-13T00:13:35.269 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-C516hJ6NM5Mt3Pg off=5304266 len=801889 2024-09-13T00:13:35.277 INFO:tasks.workunit.client.0.smithi136.stdout:op 3728 completed, throughput=5MB/sec 2024-09-13T00:13:35.277 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-tjJp8XsiVAVaXNY off=1870321 len=302313 2024-09-13T00:13:35.283 INFO:tasks.workunit.client.0.smithi136.stdout:op 3729 completed, throughput=5MB/sec 2024-09-13T00:13:35.283 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Y-0-NOpnfxYUNkj off=15313667 len=846121 2024-09-13T00:13:35.290 INFO:tasks.workunit.client.0.smithi136.stdout:op 3730 completed, throughput=5MB/sec 2024-09-13T00:13:35.290 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-jpy4nkJOJR_pmSt off=661538 len=1039493 2024-09-13T00:13:35.293 INFO:tasks.workunit.client.0.smithi136.stdout:op 3731 completed, throughput=5MB/sec 2024-09-13T00:13:35.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:13:35 smithi181 bash[67587]: cluster 2024-09-13T00:13:33.510459+0000 mgr.y (mgr.24661) 1093 : cluster [DBG] pgmap v1776: 105 pgs: 105 active+clean; 126 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:13:35.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:13:35 smithi136 bash[75164]: cluster 2024-09-13T00:13:33.510459+0000 mgr.y (mgr.24661) 1093 : cluster [DBG] pgmap v1776: 105 pgs: 105 active+clean; 126 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:13:35.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:13:35 smithi136 bash[76371]: cluster 2024-09-13T00:13:33.510459+0000 mgr.y (mgr.24661) 1093 : cluster [DBG] pgmap v1776: 105 pgs: 105 active+clean; 126 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:13:36.293 INFO:tasks.workunit.client.0.smithi136.stdout: 710: throughput=4.99MB/sec pending data=0 2024-09-13T00:13:36.294 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2umlpcazWvk7U_Q off=9191510 len=1136832 2024-09-13T00:13:36.298 INFO:tasks.workunit.client.0.smithi136.stdout:op 3732 completed, throughput=5MB/sec 2024-09-13T00:13:36.298 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-eN-LRvdUC7fSXTF off=2042100 len=2071359 2024-09-13T00:13:36.304 INFO:tasks.workunit.client.0.smithi136.stdout:op 3733 completed, throughput=5MB/sec 2024-09-13T00:13:36.304 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-_QOQuDW6PWFfct- off=10634686 len=1093445 2024-09-13T00:13:36.307 INFO:tasks.workunit.client.0.smithi136.stdout:op 3734 completed, throughput=5MB/sec 2024-09-13T00:13:36.308 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ORtZKScCNOqc-_Q off=9570037 len=1802670 2024-09-13T00:13:36.319 INFO:tasks.workunit.client.0.smithi136.stdout:op 3735 completed, throughput=5MB/sec 2024-09-13T00:13:37.320 INFO:tasks.workunit.client.0.smithi136.stdout: 711: throughput=4.99MB/sec pending data=0 2024-09-13T00:13:37.320 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-mw0PYDOnreruUSa off=11693297 len=236166 2024-09-13T00:13:37.321 INFO:tasks.workunit.client.0.smithi136.stdout:op 3736 completed, throughput=4.99MB/sec 2024-09-13T00:13:37.321 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-JsG67K5kcGUBbZ3 off=7356356 len=701584 2024-09-13T00:13:37.323 INFO:tasks.workunit.client.0.smithi136.stdout:op 3737 completed, throughput=5MB/sec 2024-09-13T00:13:37.323 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_LnDh9BmGi-FWeX off=11446427 len=1799888 2024-09-13T00:13:37.336 INFO:tasks.workunit.client.0.smithi136.stdout:op 3738 completed, throughput=5MB/sec 2024-09-13T00:13:37.336 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Njww3tIXiYbTmwn off=8867627 len=1338394 2024-09-13T00:13:37.347 INFO:tasks.workunit.client.0.smithi136.stdout:op 3739 completed, throughput=5MB/sec 2024-09-13T00:13:37.347 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-fCVNxpWXM6IyBzZ off=16536800 len=1133226 2024-09-13T00:13:37.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:13:37 smithi181 bash[67587]: cluster 2024-09-13T00:13:35.511467+0000 mgr.y (mgr.24661) 1094 : cluster [DBG] pgmap v1777: 105 pgs: 105 active+clean; 126 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:13:37.355 INFO:tasks.workunit.client.0.smithi136.stdout:op 3740 completed, throughput=5MB/sec 2024-09-13T00:13:37.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:13:37 smithi136 bash[75164]: cluster 2024-09-13T00:13:35.511467+0000 mgr.y (mgr.24661) 1094 : cluster [DBG] pgmap v1777: 105 pgs: 105 active+clean; 126 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:13:37.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:13:37 smithi136 bash[76371]: cluster 2024-09-13T00:13:35.511467+0000 mgr.y (mgr.24661) 1094 : cluster [DBG] pgmap v1777: 105 pgs: 105 active+clean; 126 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:13:38.355 INFO:tasks.workunit.client.0.smithi136.stdout: 712: throughput=4.99MB/sec pending data=0 2024-09-13T00:13:38.355 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-KsfoAlO61kJJTL7 off=9028331 len=1359694 2024-09-13T00:13:38.359 INFO:tasks.workunit.client.0.smithi136.stdout:op 3741 completed, throughput=5MB/sec 2024-09-13T00:13:38.359 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-zIQt-vHUqMmti4t off=1300106 len=1080997 2024-09-13T00:13:38.368 INFO:tasks.workunit.client.0.smithi136.stdout:op 3742 completed, throughput=5MB/sec 2024-09-13T00:13:38.368 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-rWysJL7i_gxmBeT off=3548481 len=1083619 2024-09-13T00:13:38.377 INFO:tasks.workunit.client.0.smithi136.stdout:op 3743 completed, throughput=5MB/sec 2024-09-13T00:13:38.377 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-YDB5E-VMIx9frAS off=7422078 len=1907230 2024-09-13T00:13:38.382 INFO:tasks.workunit.client.0.smithi136.stdout:op 3744 completed, throughput=5MB/sec 2024-09-13T00:13:39.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:13:39 smithi181 bash[67587]: cluster 2024-09-13T00:13:37.512539+0000 mgr.y (mgr.24661) 1095 : cluster [DBG] pgmap v1778: 105 pgs: 105 active+clean; 126 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 2.7 MiB/s rd, 3.2 MiB/s wr, 5 op/s 2024-09-13T00:13:39.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:13:39 smithi181 bash[67587]: audit 2024-09-13T00:13:38.792020+0000 mon.a (mon.0) 2214 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:13:39.382 INFO:tasks.workunit.client.0.smithi136.stdout: 713: throughput=4.99MB/sec pending data=0 2024-09-13T00:13:39.382 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-S0yeW6N2ipxnNcq off=1250729 len=721760 2024-09-13T00:13:39.389 INFO:tasks.workunit.client.0.smithi136.stdout:op 3745 completed, throughput=5MB/sec 2024-09-13T00:13:39.389 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Ng4WJl3jkbVRTq8 off=4155224 len=1053808 2024-09-13T00:13:39.392 INFO:tasks.workunit.client.0.smithi136.stdout:op 3746 completed, throughput=5MB/sec 2024-09-13T00:13:39.392 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-HbZ501onx8KcT-l off=1093122 len=1915637 2024-09-13T00:13:39.405 INFO:tasks.workunit.client.0.smithi136.stdout:op 3747 completed, throughput=5MB/sec 2024-09-13T00:13:39.405 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-oWA3Ggw0Jf9hIj- off=7703382 len=153624 2024-09-13T00:13:39.407 INFO:tasks.workunit.client.0.smithi136.stdout:op 3748 completed, throughput=5MB/sec 2024-09-13T00:13:39.407 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-59Imt6O2PaKO0VU off=932312 len=109963 2024-09-13T00:13:39.408 INFO:tasks.workunit.client.0.smithi136.stdout:op 3749 completed, throughput=5MB/sec 2024-09-13T00:13:39.408 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ZHnAt5BX013JcJF off=11831254 len=1984599 2024-09-13T00:13:39.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:13:39 smithi136 bash[75164]: cluster 2024-09-13T00:13:37.512539+0000 mgr.y (mgr.24661) 1095 : cluster [DBG] pgmap v1778: 105 pgs: 105 active+clean; 126 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 2.7 MiB/s rd, 3.2 MiB/s wr, 5 op/s 2024-09-13T00:13:39.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:13:39 smithi136 bash[75164]: audit 2024-09-13T00:13:38.792020+0000 mon.a (mon.0) 2214 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:13:39.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:13:39 smithi136 bash[76371]: cluster 2024-09-13T00:13:37.512539+0000 mgr.y (mgr.24661) 1095 : cluster [DBG] pgmap v1778: 105 pgs: 105 active+clean; 126 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 2.7 MiB/s rd, 3.2 MiB/s wr, 5 op/s 2024-09-13T00:13:39.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:13:39 smithi136 bash[76371]: audit 2024-09-13T00:13:38.792020+0000 mon.a (mon.0) 2214 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:13:39.413 INFO:tasks.workunit.client.0.smithi136.stdout:op 3750 completed, throughput=5MB/sec 2024-09-13T00:13:39.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:13:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:13:39] "GET /metrics HTTP/1.1" 200 34138 "" "Prometheus/2.43.0" 2024-09-13T00:13:40.413 INFO:tasks.workunit.client.0.smithi136.stdout: 714: throughput=4.99MB/sec pending data=0 2024-09-13T00:13:40.414 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-o5fRrZYFwFXsTaV off=9774176 len=1264207 2024-09-13T00:13:40.423 INFO:tasks.workunit.client.0.smithi136.stdout:op 3751 completed, throughput=5MB/sec 2024-09-13T00:13:40.423 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-twFWT8_JKH0v61j off=4915610 len=1870736 2024-09-13T00:13:40.427 INFO:tasks.workunit.client.0.smithi136.stdout:op 3752 completed, throughput=5MB/sec 2024-09-13T00:13:40.427 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-d4P_TS-nRdx5wbj off=1996175 len=1018120 2024-09-13T00:13:40.437 INFO:tasks.workunit.client.0.smithi136.stdout:op 3753 completed, throughput=5MB/sec 2024-09-13T00:13:41.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:13:41 smithi181 bash[67587]: cluster 2024-09-13T00:13:39.513501+0000 mgr.y (mgr.24661) 1096 : cluster [DBG] pgmap v1779: 105 pgs: 105 active+clean; 126 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:13:41.438 INFO:tasks.workunit.client.0.smithi136.stdout: 715: throughput=4.99MB/sec pending data=0 2024-09-13T00:13:41.438 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-G5VYSq8MAlUR3P7 off=5288496 len=495487 2024-09-13T00:13:41.440 INFO:tasks.workunit.client.0.smithi136.stdout:op 3754 completed, throughput=4.99MB/sec 2024-09-13T00:13:41.440 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-mpYAlDdRDm3xMDu off=4674973 len=737034 2024-09-13T00:13:41.448 INFO:tasks.workunit.client.0.smithi136.stdout:op 3755 completed, throughput=4.99MB/sec 2024-09-13T00:13:41.448 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-cPVTb2IlAHx-NQ2 off=7621560 len=1509344 2024-09-13T00:13:41.459 INFO:tasks.workunit.client.0.smithi136.stdout:op 3756 completed, throughput=5MB/sec 2024-09-13T00:13:41.459 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj--pyXQvLjpohZOIw off=2471328 len=202084 2024-09-13T00:13:41.464 INFO:tasks.workunit.client.0.smithi136.stdout:op 3757 completed, throughput=5MB/sec 2024-09-13T00:13:41.464 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-VFRr8zoHxUiHrX3 off=12080467 len=1639480 2024-09-13T00:13:41.476 INFO:tasks.workunit.client.0.smithi136.stdout:op 3758 completed, throughput=5MB/sec 2024-09-13T00:13:41.476 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-97qCZb4NxE9qU-W off=2048107 len=278485 2024-09-13T00:13:41.481 INFO:tasks.workunit.client.0.smithi136.stdout:op 3759 completed, throughput=5MB/sec 2024-09-13T00:13:41.481 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-vv73vL6IUlACBuP off=4885047 len=388948 2024-09-13T00:13:41.487 INFO:tasks.workunit.client.0.smithi136.stdout:op 3760 completed, throughput=5MB/sec 2024-09-13T00:13:41.487 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-OUNmXeyxoQ3VsQv off=2711258 len=2059606 2024-09-13T00:13:41.493 INFO:tasks.workunit.client.0.smithi136.stdout:op 3761 completed, throughput=5MB/sec 2024-09-13T00:13:41.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:13:41 smithi136 bash[75164]: cluster 2024-09-13T00:13:39.513501+0000 mgr.y (mgr.24661) 1096 : cluster [DBG] pgmap v1779: 105 pgs: 105 active+clean; 126 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:13:41.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:13:41 smithi136 bash[76371]: cluster 2024-09-13T00:13:39.513501+0000 mgr.y (mgr.24661) 1096 : cluster [DBG] pgmap v1779: 105 pgs: 105 active+clean; 126 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:13:42.493 INFO:tasks.workunit.client.0.smithi136.stdout: 716: throughput=5MB/sec pending data=0 2024-09-13T00:13:42.494 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-pdLiaNbgmdM8wEH off=2131246 len=478948 2024-09-13T00:13:42.499 INFO:tasks.workunit.client.0.smithi136.stdout:op 3762 completed, throughput=5MB/sec 2024-09-13T00:13:42.500 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-kfHa6mesj5PbJrH off=10714754 len=357721 2024-09-13T00:13:42.506 INFO:tasks.workunit.client.0.smithi136.stdout:op 3763 completed, throughput=5MB/sec 2024-09-13T00:13:42.506 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-fHrxff6WVYcY0rP off=1997726 len=1480542 2024-09-13T00:13:42.517 INFO:tasks.workunit.client.0.smithi136.stdout:op 3764 completed, throughput=5MB/sec 2024-09-13T00:13:42.517 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-NsfRV9bePsXPQJh off=5897547 len=1854572 2024-09-13T00:13:42.532 INFO:tasks.workunit.client.0.smithi136.stdout:op 3765 completed, throughput=5MB/sec 2024-09-13T00:13:42.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:13:42 smithi181 bash[95505]: ts=2024-09-13T00:13:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:13:43.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:13:43 smithi181 bash[67587]: cluster 2024-09-13T00:13:41.514612+0000 mgr.y (mgr.24661) 1097 : cluster [DBG] pgmap v1780: 105 pgs: 105 active+clean; 126 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:13:43.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:13:43 smithi136 bash[75164]: cluster 2024-09-13T00:13:41.514612+0000 mgr.y (mgr.24661) 1097 : cluster [DBG] pgmap v1780: 105 pgs: 105 active+clean; 126 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:13:43.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:13:43 smithi136 bash[76371]: cluster 2024-09-13T00:13:41.514612+0000 mgr.y (mgr.24661) 1097 : cluster [DBG] pgmap v1780: 105 pgs: 105 active+clean; 126 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:13:43.533 INFO:tasks.workunit.client.0.smithi136.stdout: 717: throughput=4.99MB/sec pending data=0 2024-09-13T00:13:43.533 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-RRwfOo1orGsWah3 off=9307657 len=786358 2024-09-13T00:13:43.535 INFO:tasks.workunit.client.0.smithi136.stdout:op 3766 completed, throughput=5MB/sec 2024-09-13T00:13:43.535 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-18Ja4tEokhSkZH1 off=3005153 len=1910284 2024-09-13T00:13:43.552 INFO:tasks.workunit.client.0.smithi136.stdout:op 3767 completed, throughput=5MB/sec 2024-09-13T00:13:43.552 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-kN_FzH05TJ9Nkd3 off=17208005 len=106922 2024-09-13T00:13:43.554 INFO:tasks.workunit.client.0.smithi136.stdout:op 3768 completed, throughput=5MB/sec 2024-09-13T00:13:43.554 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-1XhAOR6Hq_ciPjD off=22038222 len=529693 2024-09-13T00:13:43.556 INFO:tasks.workunit.client.0.smithi136.stdout:op 3769 completed, throughput=5MB/sec 2024-09-13T00:13:43.557 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-7bBKSZFAe6ov_40 off=19494758 len=1173165 2024-09-13T00:13:43.560 INFO:tasks.workunit.client.0.smithi136.stdout:op 3770 completed, throughput=5MB/sec 2024-09-13T00:13:43.560 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-qvp4F4u86otPOLn off=242476 len=90616 2024-09-13T00:13:43.561 INFO:tasks.workunit.client.0.smithi136.stdout:op 3771 completed, throughput=5MB/sec 2024-09-13T00:13:43.561 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-YeZ4XC_sR8dQMY8 off=1567027 len=1103091 2024-09-13T00:13:43.571 INFO:tasks.workunit.client.0.smithi136.stdout:op 3772 completed, throughput=5MB/sec 2024-09-13T00:13:44.571 INFO:tasks.workunit.client.0.smithi136.stdout: 718: throughput=4.99MB/sec pending data=0 2024-09-13T00:13:44.571 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-bP-X5kqkeZXxWH8 off=1310146 len=246389 2024-09-13T00:13:44.573 INFO:tasks.workunit.client.0.smithi136.stdout:op 3773 completed, throughput=4.99MB/sec 2024-09-13T00:13:44.573 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-T4BbRgFfc6JLsge off=21253449 len=896298 2024-09-13T00:13:44.577 INFO:tasks.workunit.client.0.smithi136.stdout:op 3774 completed, throughput=5MB/sec 2024-09-13T00:13:44.577 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-J0glx6ltKYdyf_z off=810015 len=928133 2024-09-13T00:13:44.580 INFO:tasks.workunit.client.0.smithi136.stdout:op 3775 completed, throughput=5MB/sec 2024-09-13T00:13:44.580 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-iWjdjRpnGSgK_9h off=1301599 len=4971 2024-09-13T00:13:44.584 INFO:tasks.workunit.client.0.smithi136.stdout:op 3776 completed, throughput=5MB/sec 2024-09-13T00:13:44.584 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-d3Ti85MjRnz6TSp off=9167783 len=1433465 2024-09-13T00:13:44.587 INFO:tasks.workunit.client.0.smithi136.stdout:op 3777 completed, throughput=5MB/sec 2024-09-13T00:13:44.587 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-dUy0Swvtf38N5h9 off=1475666 len=1021387 2024-09-13T00:13:44.590 INFO:tasks.workunit.client.0.smithi136.stdout:op 3778 completed, throughput=5MB/sec 2024-09-13T00:13:45.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:13:45 smithi181 bash[67587]: cluster 2024-09-13T00:13:43.515684+0000 mgr.y (mgr.24661) 1098 : cluster [DBG] pgmap v1781: 105 pgs: 105 active+clean; 126 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:13:45.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:13:45 smithi136 bash[75164]: cluster 2024-09-13T00:13:43.515684+0000 mgr.y (mgr.24661) 1098 : cluster [DBG] pgmap v1781: 105 pgs: 105 active+clean; 126 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:13:45.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:13:45 smithi136 bash[76371]: cluster 2024-09-13T00:13:43.515684+0000 mgr.y (mgr.24661) 1098 : cluster [DBG] pgmap v1781: 105 pgs: 105 active+clean; 126 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:13:45.590 INFO:tasks.workunit.client.0.smithi136.stdout: 719: throughput=4.99MB/sec pending data=0 2024-09-13T00:13:45.591 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-_aSrh-ZqxONvT9_ off=1992638 len=2067872 2024-09-13T00:13:45.595 INFO:tasks.workunit.client.0.smithi136.stdout:op 3779 completed, throughput=5MB/sec 2024-09-13T00:13:45.595 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-zINzekcscjGDxSX off=15562861 len=1523285 2024-09-13T00:13:45.600 INFO:tasks.workunit.client.0.smithi136.stdout:op 3780 completed, throughput=5MB/sec 2024-09-13T00:13:45.601 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-OcZehOHHe_uDgHd off=175191 len=1678423 2024-09-13T00:13:45.605 INFO:tasks.workunit.client.0.smithi136.stdout:op 3781 completed, throughput=5MB/sec 2024-09-13T00:13:46.605 INFO:tasks.workunit.client.0.smithi136.stdout: 720: throughput=4.99MB/sec pending data=0 2024-09-13T00:13:46.605 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-uwz9ytO5-W0VmB6 off=10324414 len=1457260 2024-09-13T00:13:46.615 INFO:tasks.workunit.client.0.smithi136.stdout:op 3782 completed, throughput=5MB/sec 2024-09-13T00:13:46.616 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-CmT0atkr-jNGSz7 off=551498 len=943479 2024-09-13T00:13:46.623 INFO:tasks.workunit.client.0.smithi136.stdout:op 3783 completed, throughput=5MB/sec 2024-09-13T00:13:46.623 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-r1e3JfLfnHnKi_T off=9730447 len=233192 2024-09-13T00:13:46.624 INFO:tasks.workunit.client.0.smithi136.stdout:op 3784 completed, throughput=5MB/sec 2024-09-13T00:13:46.625 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-32da10P6fnZZrkx off=22779459 len=1700131 2024-09-13T00:13:46.637 INFO:tasks.workunit.client.0.smithi136.stdout:op 3785 completed, throughput=5MB/sec 2024-09-13T00:13:46.637 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2Jnk2r9_hZajV2J off=172588 len=1898283 2024-09-13T00:13:46.643 INFO:tasks.workunit.client.0.smithi136.stdout:op 3786 completed, throughput=5MB/sec 2024-09-13T00:13:47.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:13:47 smithi181 bash[67587]: cluster 2024-09-13T00:13:45.516535+0000 mgr.y (mgr.24661) 1099 : cluster [DBG] pgmap v1782: 105 pgs: 105 active+clean; 126 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:13:47.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:13:47 smithi136 bash[76371]: cluster 2024-09-13T00:13:45.516535+0000 mgr.y (mgr.24661) 1099 : cluster [DBG] pgmap v1782: 105 pgs: 105 active+clean; 126 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:13:47.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:13:47 smithi136 bash[75164]: cluster 2024-09-13T00:13:45.516535+0000 mgr.y (mgr.24661) 1099 : cluster [DBG] pgmap v1782: 105 pgs: 105 active+clean; 126 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:13:47.644 INFO:tasks.workunit.client.0.smithi136.stdout: 721: throughput=4.99MB/sec pending data=0 2024-09-13T00:13:47.644 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-LjxYa6bOH5UnHH2 off=1085662 len=964683 2024-09-13T00:13:47.647 INFO:tasks.workunit.client.0.smithi136.stdout:op 3787 completed, throughput=5MB/sec 2024-09-13T00:13:47.647 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-cn8v1klRmhMfyW- off=1654636 len=459193 2024-09-13T00:13:47.653 INFO:tasks.workunit.client.0.smithi136.stdout:op 3788 completed, throughput=5MB/sec 2024-09-13T00:13:47.653 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-srW7hbdRLQ4HqCe off=6393370 len=1335233 2024-09-13T00:13:47.663 INFO:tasks.workunit.client.0.smithi136.stdout:op 3789 completed, throughput=5MB/sec 2024-09-13T00:13:47.663 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-gTw9gdRmhB-JEh7 off=0 len=1080875 2024-09-13T00:13:47.666 INFO:tasks.workunit.client.0.smithi136.stdout:op 3790 completed, throughput=5MB/sec 2024-09-13T00:13:47.666 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-rE_RPyQ7QPOTHqM off=1136669 len=339278 2024-09-13T00:13:47.672 INFO:tasks.workunit.client.0.smithi136.stdout:op 3791 completed, throughput=5MB/sec 2024-09-13T00:13:47.672 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-S51Il27sahOk2PM off=0 len=1377227 2024-09-13T00:13:47.675 INFO:tasks.workunit.client.0.smithi136.stdout:op 3792 completed, throughput=5MB/sec 2024-09-13T00:13:48.676 INFO:tasks.workunit.client.0.smithi136.stdout: 722: throughput=4.99MB/sec pending data=0 2024-09-13T00:13:48.676 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-jBmcTf3t2lbGPrr off=5697970 len=167151 2024-09-13T00:13:48.680 INFO:tasks.workunit.client.0.smithi136.stdout:op 3793 completed, throughput=4.99MB/sec 2024-09-13T00:13:48.680 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj--bS23nabhl-BIlN off=449653 len=1128652 2024-09-13T00:13:48.689 INFO:tasks.workunit.client.0.smithi136.stdout:op 3794 completed, throughput=5MB/sec 2024-09-13T00:13:48.689 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-AdsfvPHVzNw13zd off=18673393 len=369358 2024-09-13T00:13:48.690 INFO:tasks.workunit.client.0.smithi136.stdout:op 3795 completed, throughput=5MB/sec 2024-09-13T00:13:48.690 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-NS7ldX6Q5EXh2uS off=5734559 len=1738760 2024-09-13T00:13:48.696 INFO:tasks.workunit.client.0.smithi136.stdout:op 3796 completed, throughput=5MB/sec 2024-09-13T00:13:48.697 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-wK1VHtvJg7Wu3s_ off=767654 len=267557 2024-09-13T00:13:48.698 INFO:tasks.workunit.client.0.smithi136.stdout:op 3797 completed, throughput=5MB/sec 2024-09-13T00:13:48.698 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ehJ0zpKHP369oYJ off=8921294 len=41167 2024-09-13T00:13:48.700 INFO:tasks.workunit.client.0.smithi136.stdout:op 3798 completed, throughput=5MB/sec 2024-09-13T00:13:48.700 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-4R-fvwSdTSi2F30 off=6535262 len=1615892 2024-09-13T00:13:48.713 INFO:tasks.workunit.client.0.smithi136.stdout:op 3799 completed, throughput=5MB/sec 2024-09-13T00:13:49.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:13:49 smithi181 bash[67587]: cluster 2024-09-13T00:13:47.517607+0000 mgr.y (mgr.24661) 1100 : cluster [DBG] pgmap v1783: 105 pgs: 105 active+clean; 126 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 2.7 MiB/s rd, 3.0 MiB/s wr, 5 op/s 2024-09-13T00:13:49.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:13:49 smithi136 bash[76371]: cluster 2024-09-13T00:13:47.517607+0000 mgr.y (mgr.24661) 1100 : cluster [DBG] pgmap v1783: 105 pgs: 105 active+clean; 126 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 2.7 MiB/s rd, 3.0 MiB/s wr, 5 op/s 2024-09-13T00:13:49.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:13:49 smithi136 bash[75164]: cluster 2024-09-13T00:13:47.517607+0000 mgr.y (mgr.24661) 1100 : cluster [DBG] pgmap v1783: 105 pgs: 105 active+clean; 126 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 2.7 MiB/s rd, 3.0 MiB/s wr, 5 op/s 2024-09-13T00:13:49.713 INFO:tasks.workunit.client.0.smithi136.stdout: 723: throughput=4.99MB/sec pending data=0 2024-09-13T00:13:49.713 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-2EZu6LcfAdci_Ol off=12427384 len=959435 2024-09-13T00:13:49.722 INFO:tasks.workunit.client.0.smithi136.stdout:op 3800 completed, throughput=5MB/sec 2024-09-13T00:13:49.722 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-qAmp5J2MCbzg5r- off=5449704 len=1336229 2024-09-13T00:13:49.725 INFO:tasks.workunit.client.0.smithi136.stdout:op 3801 completed, throughput=5MB/sec 2024-09-13T00:13:49.725 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-nM3LiNqveYX772Q off=1439864 len=1788493 2024-09-13T00:13:49.729 INFO:tasks.workunit.client.0.smithi136.stdout:op 3802 completed, throughput=5MB/sec 2024-09-13T00:13:49.729 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-3aKLAaZUt0eeT45 off=725123 len=2085579 2024-09-13T00:13:49.746 INFO:tasks.workunit.client.0.smithi136.stdout:op 3803 completed, throughput=5MB/sec 2024-09-13T00:13:49.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:13:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:13:49] "GET /metrics HTTP/1.1" 200 34138 "" "Prometheus/2.43.0" 2024-09-13T00:13:50.746 INFO:tasks.workunit.client.0.smithi136.stdout: 724: throughput=5MB/sec pending data=0 2024-09-13T00:13:50.747 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Nh7r9ggCtAUL8pL off=5284304 len=1691741 2024-09-13T00:13:50.751 INFO:tasks.workunit.client.0.smithi136.stdout:op 3804 completed, throughput=5MB/sec 2024-09-13T00:13:50.751 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-gkLYlN2wDVE8PTG off=12967986 len=1888128 2024-09-13T00:13:50.766 INFO:tasks.workunit.client.0.smithi136.stdout:op 3805 completed, throughput=5MB/sec 2024-09-13T00:13:50.766 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-m_RwkvBeK_9OLV0 off=13492096 len=1625723 2024-09-13T00:13:50.777 INFO:tasks.workunit.client.0.smithi136.stdout:op 3806 completed, throughput=5MB/sec 2024-09-13T00:13:51.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:13:51 smithi181 bash[67587]: cluster 2024-09-13T00:13:49.518472+0000 mgr.y (mgr.24661) 1101 : cluster [DBG] pgmap v1784: 105 pgs: 105 active+clean; 126 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:13:51.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:13:51 smithi136 bash[76371]: cluster 2024-09-13T00:13:49.518472+0000 mgr.y (mgr.24661) 1101 : cluster [DBG] pgmap v1784: 105 pgs: 105 active+clean; 126 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:13:51.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:13:51 smithi136 bash[75164]: cluster 2024-09-13T00:13:49.518472+0000 mgr.y (mgr.24661) 1101 : cluster [DBG] pgmap v1784: 105 pgs: 105 active+clean; 126 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:13:51.777 INFO:tasks.workunit.client.0.smithi136.stdout: 725: throughput=5MB/sec pending data=0 2024-09-13T00:13:51.778 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-dbmDbyFpOKkBYEl off=1984432 len=1916793 2024-09-13T00:13:51.789 INFO:tasks.workunit.client.0.smithi136.stdout:op 3807 completed, throughput=5MB/sec 2024-09-13T00:13:51.789 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-8jRNTiYgBEgzS6Q off=10277817 len=1989702 2024-09-13T00:13:51.794 INFO:tasks.workunit.client.0.smithi136.stdout:op 3808 completed, throughput=5MB/sec 2024-09-13T00:13:51.794 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-f9cN0oRh86h4OG9 off=407291 len=243734 2024-09-13T00:13:51.799 INFO:tasks.workunit.client.0.smithi136.stdout:op 3809 completed, throughput=5MB/sec 2024-09-13T00:13:52.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:13:52 smithi181 bash[95505]: ts=2024-09-13T00:13:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:13:52.799 INFO:tasks.workunit.client.0.smithi136.stdout: 726: throughput=4.99MB/sec pending data=0 2024-09-13T00:13:52.800 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-8OyXEO3lx4ikwp6 off=5000608 len=2044426 2024-09-13T00:13:52.804 INFO:tasks.workunit.client.0.smithi136.stdout:op 3810 completed, throughput=5MB/sec 2024-09-13T00:13:52.804 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-5c2hrPzJcquPveO off=15218387 len=1648310 2024-09-13T00:13:52.808 INFO:tasks.workunit.client.0.smithi136.stdout:op 3811 completed, throughput=5MB/sec 2024-09-13T00:13:52.808 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-bgmXX7LBtlYOYeI off=877578 len=668131 2024-09-13T00:13:52.811 INFO:tasks.workunit.client.0.smithi136.stdout:op 3812 completed, throughput=5MB/sec 2024-09-13T00:13:52.811 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-nElm29kUFhReOvz off=927212 len=1767019 2024-09-13T00:13:52.817 INFO:tasks.workunit.client.0.smithi136.stdout:op 3813 completed, throughput=5MB/sec 2024-09-13T00:13:53.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:13:53 smithi181 bash[67587]: cluster 2024-09-13T00:13:51.519477+0000 mgr.y (mgr.24661) 1102 : cluster [DBG] pgmap v1785: 105 pgs: 105 active+clean; 126 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:13:53.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:13:53 smithi136 bash[76371]: cluster 2024-09-13T00:13:51.519477+0000 mgr.y (mgr.24661) 1102 : cluster [DBG] pgmap v1785: 105 pgs: 105 active+clean; 126 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:13:53.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:13:53 smithi136 bash[75164]: cluster 2024-09-13T00:13:51.519477+0000 mgr.y (mgr.24661) 1102 : cluster [DBG] pgmap v1785: 105 pgs: 105 active+clean; 126 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:13:53.818 INFO:tasks.workunit.client.0.smithi136.stdout: 727: throughput=4.99MB/sec pending data=0 2024-09-13T00:13:53.818 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-MrNjka26glYYblm off=1728508 len=1124912 2024-09-13T00:13:53.827 INFO:tasks.workunit.client.0.smithi136.stdout:op 3814 completed, throughput=5MB/sec 2024-09-13T00:13:53.827 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-YlfxA7k4iLAXP29 off=1003832 len=1368987 2024-09-13T00:13:53.831 INFO:tasks.workunit.client.0.smithi136.stdout:op 3815 completed, throughput=5MB/sec 2024-09-13T00:13:53.831 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-iPhU4UYIwQhUURI off=637458 len=1697920 2024-09-13T00:13:53.837 INFO:tasks.workunit.client.0.smithi136.stdout:op 3816 completed, throughput=5MB/sec 2024-09-13T00:13:53.837 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-pD-98H4cIYvL-gI off=21257869 len=885698 2024-09-13T00:13:53.841 INFO:tasks.workunit.client.0.smithi136.stdout:op 3817 completed, throughput=5MB/sec 2024-09-13T00:13:54.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:13:54 smithi181 bash[67587]: audit 2024-09-13T00:13:53.792367+0000 mon.a (mon.0) 2215 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:13:54.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:13:54 smithi136 bash[75164]: audit 2024-09-13T00:13:53.792367+0000 mon.a (mon.0) 2215 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:13:54.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:13:54 smithi136 bash[76371]: audit 2024-09-13T00:13:53.792367+0000 mon.a (mon.0) 2215 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:13:54.841 INFO:tasks.workunit.client.0.smithi136.stdout: 728: throughput=4.99MB/sec pending data=0 2024-09-13T00:13:54.841 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-pJLQoW7vSs2nIil off=10479361 len=833400 2024-09-13T00:13:54.850 INFO:tasks.workunit.client.0.smithi136.stdout:op 3818 completed, throughput=5MB/sec 2024-09-13T00:13:54.850 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-jqBWv7yqKbsZEoP off=5935315 len=1615483 2024-09-13T00:13:54.868 INFO:tasks.workunit.client.0.smithi136.stdout:op 3819 completed, throughput=5MB/sec 2024-09-13T00:13:54.868 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-CIyy62jcDsFIR_l off=149105 len=739465 2024-09-13T00:13:54.870 INFO:tasks.workunit.client.0.smithi136.stdout:op 3820 completed, throughput=5MB/sec 2024-09-13T00:13:54.870 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-t2WRcWchEOCUtVc off=15862060 len=1507570 2024-09-13T00:13:54.882 INFO:tasks.workunit.client.0.smithi136.stdout:op 3821 completed, throughput=5MB/sec 2024-09-13T00:13:54.882 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-AM2MK2ruNkNXcwF off=363794 len=783539 2024-09-13T00:13:54.883 INFO:tasks.workunit.client.0.smithi136.stdout:op 3822 completed, throughput=5MB/sec 2024-09-13T00:13:55.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:13:55 smithi181 bash[67587]: cluster 2024-09-13T00:13:53.520392+0000 mgr.y (mgr.24661) 1103 : cluster [DBG] pgmap v1786: 105 pgs: 105 active+clean; 126 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:13:55.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:13:55 smithi181 bash[67587]: audit 2024-09-13T00:13:54.961717+0000 mon.a (mon.0) 2216 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:13:55.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:13:55 smithi136 bash[76371]: cluster 2024-09-13T00:13:53.520392+0000 mgr.y (mgr.24661) 1103 : cluster [DBG] pgmap v1786: 105 pgs: 105 active+clean; 126 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:13:55.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:13:55 smithi136 bash[76371]: audit 2024-09-13T00:13:54.961717+0000 mon.a (mon.0) 2216 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:13:55.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:13:55 smithi136 bash[75164]: cluster 2024-09-13T00:13:53.520392+0000 mgr.y (mgr.24661) 1103 : cluster [DBG] pgmap v1786: 105 pgs: 105 active+clean; 126 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:13:55.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:13:55 smithi136 bash[75164]: audit 2024-09-13T00:13:54.961717+0000 mon.a (mon.0) 2216 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:13:55.884 INFO:tasks.workunit.client.0.smithi136.stdout: 729: throughput=4.99MB/sec pending data=0 2024-09-13T00:13:55.884 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-wUo64QuRaSEFWWv off=8309932 len=1100279 2024-09-13T00:13:55.887 INFO:tasks.workunit.client.0.smithi136.stdout:op 3823 completed, throughput=5MB/sec 2024-09-13T00:13:55.887 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-p7ztppHodw88Tw7 off=9544677 len=2008174 2024-09-13T00:13:55.892 INFO:tasks.workunit.client.0.smithi136.stdout:op 3824 completed, throughput=5MB/sec 2024-09-13T00:13:55.892 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-BoeZcRpor9UHO2u off=3467339 len=510956 2024-09-13T00:13:55.898 INFO:tasks.workunit.client.0.smithi136.stdout:op 3825 completed, throughput=5MB/sec 2024-09-13T00:13:55.898 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-3J2djkrN-lnlwJE off=4665272 len=1515045 2024-09-13T00:13:55.908 INFO:tasks.workunit.client.0.smithi136.stdout:op 3826 completed, throughput=5MB/sec 2024-09-13T00:13:56.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:13:56 smithi181 bash[67587]: audit 2024-09-13T00:13:55.491262+0000 mon.a (mon.0) 2217 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:13:56.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:13:56 smithi181 bash[67587]: audit 2024-09-13T00:13:55.492393+0000 mon.a (mon.0) 2218 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:13:56.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:13:56 smithi181 bash[67587]: audit 2024-09-13T00:13:55.501502+0000 mon.a (mon.0) 2219 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:13:56.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:13:56 smithi136 bash[75164]: audit 2024-09-13T00:13:55.491262+0000 mon.a (mon.0) 2217 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:13:56.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:13:56 smithi136 bash[75164]: audit 2024-09-13T00:13:55.492393+0000 mon.a (mon.0) 2218 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:13:56.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:13:56 smithi136 bash[75164]: audit 2024-09-13T00:13:55.501502+0000 mon.a (mon.0) 2219 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:13:56.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:13:56 smithi136 bash[76371]: audit 2024-09-13T00:13:55.491262+0000 mon.a (mon.0) 2217 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:13:56.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:13:56 smithi136 bash[76371]: audit 2024-09-13T00:13:55.492393+0000 mon.a (mon.0) 2218 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:13:56.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:13:56 smithi136 bash[76371]: audit 2024-09-13T00:13:55.501502+0000 mon.a (mon.0) 2219 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:13:56.909 INFO:tasks.workunit.client.0.smithi136.stdout: 730: throughput=4.99MB/sec pending data=0 2024-09-13T00:13:56.909 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-A9JNvjVj_rSQO7I off=6788919 len=1398326 2024-09-13T00:13:56.912 INFO:tasks.workunit.client.0.smithi136.stdout:op 3827 completed, throughput=5MB/sec 2024-09-13T00:13:56.912 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-AfOSf7aUOsAFKWs off=4184696 len=1408564 2024-09-13T00:13:56.925 INFO:tasks.workunit.client.0.smithi136.stdout:op 3828 completed, throughput=5MB/sec 2024-09-13T00:13:56.925 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-BTNbFlAKBTweJe6 off=198901 len=457107 2024-09-13T00:13:56.927 INFO:tasks.workunit.client.0.smithi136.stdout:op 3829 completed, throughput=5MB/sec 2024-09-13T00:13:56.927 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-f8EHmM6O3RqKJEw off=1644404 len=1113573 2024-09-13T00:13:56.932 INFO:tasks.workunit.client.0.smithi136.stdout:op 3830 completed, throughput=5MB/sec 2024-09-13T00:13:56.932 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-xaFxrSo29AVyt7T off=13219330 len=1026684 2024-09-13T00:13:56.934 INFO:tasks.workunit.client.0.smithi136.stdout:op 3831 completed, throughput=5MB/sec 2024-09-13T00:13:57.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:13:57 smithi181 bash[67587]: cluster 2024-09-13T00:13:55.521110+0000 mgr.y (mgr.24661) 1104 : cluster [DBG] pgmap v1787: 105 pgs: 105 active+clean; 126 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:13:57.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:13:57 smithi136 bash[75164]: cluster 2024-09-13T00:13:55.521110+0000 mgr.y (mgr.24661) 1104 : cluster [DBG] pgmap v1787: 105 pgs: 105 active+clean; 126 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:13:57.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:13:57 smithi136 bash[76371]: cluster 2024-09-13T00:13:55.521110+0000 mgr.y (mgr.24661) 1104 : cluster [DBG] pgmap v1787: 105 pgs: 105 active+clean; 126 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:13:57.935 INFO:tasks.workunit.client.0.smithi136.stdout: 731: throughput=4.99MB/sec pending data=0 2024-09-13T00:13:57.935 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-dNbsvMQ7F_a7XR3 off=1069453 len=52322 2024-09-13T00:13:57.939 INFO:tasks.workunit.client.0.smithi136.stdout:op 3832 completed, throughput=4.99MB/sec 2024-09-13T00:13:57.940 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-zEB5NeBB2DZ93Ar off=6636996 len=18575 2024-09-13T00:13:57.943 INFO:tasks.workunit.client.0.smithi136.stdout:op 3833 completed, throughput=4.99MB/sec 2024-09-13T00:13:57.943 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-WvaGkRSZ9d8gGMP off=8252745 len=1645147 2024-09-13T00:13:57.947 INFO:tasks.workunit.client.0.smithi136.stdout:op 3834 completed, throughput=5MB/sec 2024-09-13T00:13:57.948 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-7vbmEy_aH9vQf0K off=7249673 len=1407212 2024-09-13T00:13:57.959 INFO:tasks.workunit.client.0.smithi136.stdout:op 3835 completed, throughput=5MB/sec 2024-09-13T00:13:57.959 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-CW0siSl9CU022fT off=1180795 len=609130 2024-09-13T00:13:57.965 INFO:tasks.workunit.client.0.smithi136.stdout:op 3836 completed, throughput=5MB/sec 2024-09-13T00:13:57.965 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ctN319tYjkTFM5t off=18636514 len=1911615 2024-09-13T00:13:57.978 INFO:tasks.workunit.client.0.smithi136.stdout:op 3837 completed, throughput=5MB/sec 2024-09-13T00:13:58.978 INFO:tasks.workunit.client.0.smithi136.stdout: 732: throughput=4.99MB/sec pending data=0 2024-09-13T00:13:58.979 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Z_rIsGcxp0L6ePR off=1835888 len=617351 2024-09-13T00:13:58.980 INFO:tasks.workunit.client.0.smithi136.stdout:op 3838 completed, throughput=5MB/sec 2024-09-13T00:13:58.981 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_mS3VqyffNx8Ld3 off=547449 len=2021569 2024-09-13T00:13:58.995 INFO:tasks.workunit.client.0.smithi136.stdout:op 3839 completed, throughput=5MB/sec 2024-09-13T00:13:58.995 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-biONdZwH8zM3xpa off=2006463 len=2059516 2024-09-13T00:13:59.011 INFO:tasks.workunit.client.0.smithi136.stdout:op 3840 completed, throughput=5MB/sec 2024-09-13T00:13:59.011 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-0C_gxkG_LjKAEZw off=21083877 len=1360357 2024-09-13T00:13:59.021 INFO:tasks.workunit.client.0.smithi136.stdout:op 3841 completed, throughput=5MB/sec 2024-09-13T00:13:59.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:13:59 smithi181 bash[67587]: cluster 2024-09-13T00:13:57.522190+0000 mgr.y (mgr.24661) 1105 : cluster [DBG] pgmap v1788: 105 pgs: 105 active+clean; 126 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 3.3 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:13:59.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:13:59 smithi136 bash[76371]: cluster 2024-09-13T00:13:57.522190+0000 mgr.y (mgr.24661) 1105 : cluster [DBG] pgmap v1788: 105 pgs: 105 active+clean; 126 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 3.3 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:13:59.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:13:59 smithi136 bash[75164]: cluster 2024-09-13T00:13:57.522190+0000 mgr.y (mgr.24661) 1105 : cluster [DBG] pgmap v1788: 105 pgs: 105 active+clean; 126 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 3.3 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:13:59.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:13:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:13:59] "GET /metrics HTTP/1.1" 200 34141 "" "Prometheus/2.43.0" 2024-09-13T00:14:00.022 INFO:tasks.workunit.client.0.smithi136.stdout: 733: throughput=4.99MB/sec pending data=0 2024-09-13T00:14:00.022 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-88NVCq6d0IR3JqX off=5343584 len=1402828 2024-09-13T00:14:00.025 INFO:tasks.workunit.client.0.smithi136.stdout:op 3842 completed, throughput=5MB/sec 2024-09-13T00:14:00.026 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_vE3VRH5EN3OJUc off=6434618 len=563894 2024-09-13T00:14:00.032 INFO:tasks.workunit.client.0.smithi136.stdout:op 3843 completed, throughput=5MB/sec 2024-09-13T00:14:00.032 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-I8_HzlEEhL6qP-F off=11630309 len=865297 2024-09-13T00:14:00.040 INFO:tasks.workunit.client.0.smithi136.stdout:op 3844 completed, throughput=5MB/sec 2024-09-13T00:14:00.040 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-OpBWT6oXdXb7wJ2 off=1058574 len=817081 2024-09-13T00:14:00.048 INFO:tasks.workunit.client.0.smithi136.stdout:op 3845 completed, throughput=5MB/sec 2024-09-13T00:14:00.048 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-AbFsDbdVvVjXY5Y off=5012793 len=1597516 2024-09-13T00:14:00.062 INFO:tasks.workunit.client.0.smithi136.stdout:op 3846 completed, throughput=5MB/sec 2024-09-13T00:14:01.062 INFO:tasks.workunit.client.0.smithi136.stdout: 734: throughput=4.99MB/sec pending data=0 2024-09-13T00:14:01.062 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-6DkuqFleiqMtsOb off=943559 len=468937 2024-09-13T00:14:01.065 INFO:tasks.workunit.client.0.smithi136.stdout:op 3847 completed, throughput=5MB/sec 2024-09-13T00:14:01.065 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Mw6KB8nS95-4RF6 off=677589 len=1052545 2024-09-13T00:14:01.074 INFO:tasks.workunit.client.0.smithi136.stdout:op 3848 completed, throughput=5MB/sec 2024-09-13T00:14:01.074 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-NqXcFoGoWthqCMl off=13978703 len=1322893 2024-09-13T00:14:01.078 INFO:tasks.workunit.client.0.smithi136.stdout:op 3849 completed, throughput=5MB/sec 2024-09-13T00:14:01.078 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-X-hWvvd_JEM0Pyf off=3626489 len=1189444 2024-09-13T00:14:01.086 INFO:tasks.workunit.client.0.smithi136.stdout:op 3850 completed, throughput=5MB/sec 2024-09-13T00:14:01.087 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-rAESgV70WL0r9qR off=14401883 len=1911838 2024-09-13T00:14:01.092 INFO:tasks.workunit.client.0.smithi136.stdout:op 3851 completed, throughput=5MB/sec 2024-09-13T00:14:01.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:14:01 smithi181 bash[67587]: cluster 2024-09-13T00:13:59.523077+0000 mgr.y (mgr.24661) 1106 : cluster [DBG] pgmap v1789: 105 pgs: 105 active+clean; 126 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 2.5 MiB/s rd, 1.5 MiB/s wr, 3 op/s 2024-09-13T00:14:01.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:14:01 smithi136 bash[75164]: cluster 2024-09-13T00:13:59.523077+0000 mgr.y (mgr.24661) 1106 : cluster [DBG] pgmap v1789: 105 pgs: 105 active+clean; 126 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 2.5 MiB/s rd, 1.5 MiB/s wr, 3 op/s 2024-09-13T00:14:01.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:14:01 smithi136 bash[76371]: cluster 2024-09-13T00:13:59.523077+0000 mgr.y (mgr.24661) 1106 : cluster [DBG] pgmap v1789: 105 pgs: 105 active+clean; 126 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 2.5 MiB/s rd, 1.5 MiB/s wr, 3 op/s 2024-09-13T00:14:02.092 INFO:tasks.workunit.client.0.smithi136.stdout: 735: throughput=5MB/sec pending data=0 2024-09-13T00:14:02.095 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-s1_uCtgo7uUJ2wk off=643693 len=1303223 2024-09-13T00:14:02.103 INFO:tasks.workunit.client.0.smithi136.stdout:op 3852 completed, throughput=5MB/sec 2024-09-13T00:14:02.103 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-uIQUlBeX-GyP8Px off=5082328 len=1613062 2024-09-13T00:14:02.107 INFO:tasks.workunit.client.0.smithi136.stdout:op 3853 completed, throughput=5MB/sec 2024-09-13T00:14:02.107 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-pyPZhBQqxhYJ781 off=2461772 len=1349063 2024-09-13T00:14:02.116 INFO:tasks.workunit.client.0.smithi136.stdout:op 3854 completed, throughput=5MB/sec 2024-09-13T00:14:02.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:14:02 smithi181 bash[95505]: ts=2024-09-13T00:14:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:14:03.116 INFO:tasks.workunit.client.0.smithi136.stdout: 736: throughput=4.99MB/sec pending data=0 2024-09-13T00:14:03.117 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-_f2qoumnPV6cVx7 off=8869301 len=1591123 2024-09-13T00:14:03.121 INFO:tasks.workunit.client.0.smithi136.stdout:op 3855 completed, throughput=5MB/sec 2024-09-13T00:14:03.121 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-_oXBHqfCaEOBBaR off=3553209 len=2084113 2024-09-13T00:14:03.128 INFO:tasks.workunit.client.0.smithi136.stdout:op 3856 completed, throughput=5MB/sec 2024-09-13T00:14:03.128 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-W7RlJOkF8CWHU68 off=16066448 len=196408 2024-09-13T00:14:03.134 INFO:tasks.workunit.client.0.smithi136.stdout:op 3857 completed, throughput=5MB/sec 2024-09-13T00:14:03.134 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-N6B7CuJUHvSJgku off=11218300 len=154850 2024-09-13T00:14:03.135 INFO:tasks.workunit.client.0.smithi136.stdout:op 3858 completed, throughput=5MB/sec 2024-09-13T00:14:03.135 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-keTVvxjyrJcjveU off=9618199 len=1801651 2024-09-13T00:14:03.139 INFO:tasks.workunit.client.0.smithi136.stdout:op 3859 completed, throughput=5MB/sec 2024-09-13T00:14:03.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:14:03 smithi181 bash[67587]: cluster 2024-09-13T00:14:01.524127+0000 mgr.y (mgr.24661) 1107 : cluster [DBG] pgmap v1790: 105 pgs: 105 active+clean; 126 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:14:03.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:14:03 smithi136 bash[75164]: cluster 2024-09-13T00:14:01.524127+0000 mgr.y (mgr.24661) 1107 : cluster [DBG] pgmap v1790: 105 pgs: 105 active+clean; 126 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:14:03.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:14:03 smithi136 bash[76371]: cluster 2024-09-13T00:14:01.524127+0000 mgr.y (mgr.24661) 1107 : cluster [DBG] pgmap v1790: 105 pgs: 105 active+clean; 126 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:14:04.139 INFO:tasks.workunit.client.0.smithi136.stdout: 737: throughput=4.99MB/sec pending data=0 2024-09-13T00:14:04.139 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Efv8pmjcLwhuxC2 off=527649 len=2000227 2024-09-13T00:14:04.156 INFO:tasks.workunit.client.0.smithi136.stdout:op 3860 completed, throughput=5MB/sec 2024-09-13T00:14:04.156 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-zXJUV2dYq_cMGB1 off=3472464 len=138464 2024-09-13T00:14:04.161 INFO:tasks.workunit.client.0.smithi136.stdout:op 3861 completed, throughput=5MB/sec 2024-09-13T00:14:04.161 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-0TAHqiC4OFv40Cy off=29029 len=1874823 2024-09-13T00:14:04.167 INFO:tasks.workunit.client.0.smithi136.stdout:op 3862 completed, throughput=5MB/sec 2024-09-13T00:14:04.167 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-hyiIUgmeJXVwSuG off=11605533 len=462530 2024-09-13T00:14:04.174 INFO:tasks.workunit.client.0.smithi136.stdout:op 3863 completed, throughput=5MB/sec 2024-09-13T00:14:05.175 INFO:tasks.workunit.client.0.smithi136.stdout: 738: throughput=4.99MB/sec pending data=0 2024-09-13T00:14:05.175 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-bZ4PKpFFCPSFi5r off=6692260 len=647189 2024-09-13T00:14:05.177 INFO:tasks.workunit.client.0.smithi136.stdout:op 3864 completed, throughput=4.99MB/sec 2024-09-13T00:14:05.178 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-gmhOIabl_NmKXgY off=10126751 len=1933041 2024-09-13T00:14:05.182 INFO:tasks.workunit.client.0.smithi136.stdout:op 3865 completed, throughput=5MB/sec 2024-09-13T00:14:05.183 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_LwenDuoweD2xpj off=21452525 len=2021558 2024-09-13T00:14:05.196 INFO:tasks.workunit.client.0.smithi136.stdout:op 3866 completed, throughput=5MB/sec 2024-09-13T00:14:05.196 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-tWeyKgATRPxYiuB off=10524104 len=193998 2024-09-13T00:14:05.197 INFO:tasks.workunit.client.0.smithi136.stdout:op 3867 completed, throughput=5MB/sec 2024-09-13T00:14:05.197 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-wf110viXnGvMVFd off=1381855 len=1640139 2024-09-13T00:14:05.201 INFO:tasks.workunit.client.0.smithi136.stdout:op 3868 completed, throughput=5MB/sec 2024-09-13T00:14:05.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:14:05 smithi181 bash[67587]: cluster 2024-09-13T00:14:03.525242+0000 mgr.y (mgr.24661) 1108 : cluster [DBG] pgmap v1791: 105 pgs: 105 active+clean; 126 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:14:05.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:14:05 smithi136 bash[75164]: cluster 2024-09-13T00:14:03.525242+0000 mgr.y (mgr.24661) 1108 : cluster [DBG] pgmap v1791: 105 pgs: 105 active+clean; 126 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:14:05.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:14:05 smithi136 bash[76371]: cluster 2024-09-13T00:14:03.525242+0000 mgr.y (mgr.24661) 1108 : cluster [DBG] pgmap v1791: 105 pgs: 105 active+clean; 126 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:14:06.201 INFO:tasks.workunit.client.0.smithi136.stdout: 739: throughput=4.99MB/sec pending data=0 2024-09-13T00:14:06.202 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-5sBMwu3htVEp0e9 off=398869 len=1378213 2024-09-13T00:14:06.205 INFO:tasks.workunit.client.0.smithi136.stdout:op 3869 completed, throughput=5MB/sec 2024-09-13T00:14:06.205 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-x--ntcWmBK5TJt2 off=2521331 len=972638 2024-09-13T00:14:06.209 INFO:tasks.workunit.client.0.smithi136.stdout:op 3870 completed, throughput=5MB/sec 2024-09-13T00:14:06.209 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-CSqT2nDSePExdtR off=1125188 len=598348 2024-09-13T00:14:06.212 INFO:tasks.workunit.client.0.smithi136.stdout:op 3871 completed, throughput=5MB/sec 2024-09-13T00:14:06.212 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-DqpfrgGuKq3OHCJ off=2179651 len=885719 2024-09-13T00:14:06.220 INFO:tasks.workunit.client.0.smithi136.stdout:op 3872 completed, throughput=5MB/sec 2024-09-13T00:14:06.221 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-C44yiYTaFfxVk7s off=2732452 len=428029 2024-09-13T00:14:06.222 INFO:tasks.workunit.client.0.smithi136.stdout:op 3873 completed, throughput=5MB/sec 2024-09-13T00:14:06.222 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-LF-WFBSAOB0o4MS off=5422741 len=1972309 2024-09-13T00:14:06.227 INFO:tasks.workunit.client.0.smithi136.stdout:op 3874 completed, throughput=5MB/sec 2024-09-13T00:14:07.228 INFO:tasks.workunit.client.0.smithi136.stdout: 740: throughput=5MB/sec pending data=0 2024-09-13T00:14:07.228 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-FVgpsZFWxAwjmyo off=5077149 len=1564028 2024-09-13T00:14:07.232 INFO:tasks.workunit.client.0.smithi136.stdout:op 3875 completed, throughput=5MB/sec 2024-09-13T00:14:07.232 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-HyML9oPG-dLpIAU off=657727 len=1005629 2024-09-13T00:14:07.235 INFO:tasks.workunit.client.0.smithi136.stdout:op 3876 completed, throughput=5MB/sec 2024-09-13T00:14:07.235 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-JkZOIDYr9mxzHOz off=9274210 len=1478393 2024-09-13T00:14:07.239 INFO:tasks.workunit.client.0.smithi136.stdout:op 3877 completed, throughput=5MB/sec 2024-09-13T00:14:07.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:14:07 smithi136 bash[75164]: cluster 2024-09-13T00:14:05.526033+0000 mgr.y (mgr.24661) 1109 : cluster [DBG] pgmap v1792: 105 pgs: 105 active+clean; 126 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:14:07.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:14:07 smithi136 bash[76371]: cluster 2024-09-13T00:14:05.526033+0000 mgr.y (mgr.24661) 1109 : cluster [DBG] pgmap v1792: 105 pgs: 105 active+clean; 126 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:14:07.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:14:07 smithi181 bash[67587]: cluster 2024-09-13T00:14:05.526033+0000 mgr.y (mgr.24661) 1109 : cluster [DBG] pgmap v1792: 105 pgs: 105 active+clean; 126 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:14:08.239 INFO:tasks.workunit.client.0.smithi136.stdout: 741: throughput=4.99MB/sec pending data=0 2024-09-13T00:14:08.240 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-XJ5agJUte1d_8sU off=3830827 len=613996 2024-09-13T00:14:08.248 INFO:tasks.workunit.client.0.smithi136.stdout:op 3878 completed, throughput=5MB/sec 2024-09-13T00:14:08.248 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-NVZN48m_p_fgQe_ off=11294401 len=45655 2024-09-13T00:14:08.251 INFO:tasks.workunit.client.0.smithi136.stdout:op 3879 completed, throughput=5MB/sec 2024-09-13T00:14:08.251 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-BXkbZDdwjdFTJuZ off=17988006 len=1566347 2024-09-13T00:14:08.261 INFO:tasks.workunit.client.0.smithi136.stdout:op 3880 completed, throughput=5MB/sec 2024-09-13T00:14:08.261 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_7R4gbxqRhR7Btt off=5937448 len=1707534 2024-09-13T00:14:08.272 INFO:tasks.workunit.client.0.smithi136.stdout:op 3881 completed, throughput=5MB/sec 2024-09-13T00:14:08.272 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-vnmk_XlQe2yG-Jb off=600047 len=1051919 2024-09-13T00:14:08.275 INFO:tasks.workunit.client.0.smithi136.stdout:op 3882 completed, throughput=5MB/sec 2024-09-13T00:14:09.275 INFO:tasks.workunit.client.0.smithi136.stdout: 742: throughput=4.99MB/sec pending data=0 2024-09-13T00:14:09.275 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-PjvuaJaOvb9Y5ol off=6197420 len=1133082 2024-09-13T00:14:09.285 INFO:tasks.workunit.client.0.smithi136.stdout:op 3883 completed, throughput=5MB/sec 2024-09-13T00:14:09.285 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-4tytiE-CiCocAFp off=700195 len=820573 2024-09-13T00:14:09.289 INFO:tasks.workunit.client.0.smithi136.stdout:op 3884 completed, throughput=5MB/sec 2024-09-13T00:14:09.289 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-60yPmz44P_TI1PJ off=2145703 len=760437 2024-09-13T00:14:09.291 INFO:tasks.workunit.client.0.smithi136.stdout:op 3885 completed, throughput=5MB/sec 2024-09-13T00:14:09.291 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-32_ShXdKVik5Coy off=1394386 len=1354879 2024-09-13T00:14:09.303 INFO:tasks.workunit.client.0.smithi136.stdout:op 3886 completed, throughput=5MB/sec 2024-09-13T00:14:09.303 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-1rMrJZingH6Vd6m off=8740647 len=350072 2024-09-13T00:14:09.304 INFO:tasks.workunit.client.0.smithi136.stdout:op 3887 completed, throughput=5MB/sec 2024-09-13T00:14:09.304 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-DlLFIHARxKuixiS off=14562411 len=1178512 2024-09-13T00:14:09.307 INFO:tasks.workunit.client.0.smithi136.stdout:op 3888 completed, throughput=5MB/sec 2024-09-13T00:14:09.412 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:14:09 smithi136 bash[76371]: cluster 2024-09-13T00:14:07.527224+0000 mgr.y (mgr.24661) 1110 : cluster [DBG] pgmap v1793: 105 pgs: 105 active+clean; 126 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:14:09.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:14:09 smithi136 bash[76371]: audit 2024-09-13T00:14:08.792773+0000 mon.a (mon.0) 2220 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:14:09.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:14:09 smithi136 bash[75164]: cluster 2024-09-13T00:14:07.527224+0000 mgr.y (mgr.24661) 1110 : cluster [DBG] pgmap v1793: 105 pgs: 105 active+clean; 126 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:14:09.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:14:09 smithi136 bash[75164]: audit 2024-09-13T00:14:08.792773+0000 mon.a (mon.0) 2220 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:14:09.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:14:09 smithi181 bash[67587]: cluster 2024-09-13T00:14:07.527224+0000 mgr.y (mgr.24661) 1110 : cluster [DBG] pgmap v1793: 105 pgs: 105 active+clean; 126 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:14:09.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:14:09 smithi181 bash[67587]: audit 2024-09-13T00:14:08.792773+0000 mon.a (mon.0) 2220 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:14:09.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:14:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:14:09] "GET /metrics HTTP/1.1" 200 34139 "" "Prometheus/2.43.0" 2024-09-13T00:14:10.308 INFO:tasks.workunit.client.0.smithi136.stdout: 743: throughput=4.99MB/sec pending data=0 2024-09-13T00:14:10.308 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-inGDp-ebjt_JRFj off=6881002 len=2090755 2024-09-13T00:14:10.321 INFO:tasks.workunit.client.0.smithi136.stdout:op 3889 completed, throughput=5MB/sec 2024-09-13T00:14:10.321 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-RxrG0dHKtloERHA off=3918441 len=855582 2024-09-13T00:14:10.329 INFO:tasks.workunit.client.0.smithi136.stdout:op 3890 completed, throughput=5MB/sec 2024-09-13T00:14:10.329 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Jot25jeGaQd_TZC off=7961305 len=1409191 2024-09-13T00:14:10.331 INFO:tasks.workunit.client.0.smithi136.stdout:op 3891 completed, throughput=5MB/sec 2024-09-13T00:14:10.331 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-91qsNaZHOPh3eg2 off=8703913 len=1371563 2024-09-13T00:14:10.336 INFO:tasks.workunit.client.0.smithi136.stdout:op 3892 completed, throughput=5MB/sec 2024-09-13T00:14:11.337 INFO:tasks.workunit.client.0.smithi136.stdout: 744: throughput=4.99MB/sec pending data=0 2024-09-13T00:14:11.337 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-I-WhaGkUhWHq6zg off=3327089 len=1200547 2024-09-13T00:14:11.346 INFO:tasks.workunit.client.0.smithi136.stdout:op 3893 completed, throughput=5MB/sec 2024-09-13T00:14:11.346 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-5_ZTLnR--ymQodu off=1277791 len=1536938 2024-09-13T00:14:11.350 INFO:tasks.workunit.client.0.smithi136.stdout:op 3894 completed, throughput=5MB/sec 2024-09-13T00:14:11.350 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-vZUKZycyYOHRTFT off=0 len=1064565 2024-09-13T00:14:11.351 INFO:tasks.workunit.client.0.smithi136.stdout:op 3895 completed, throughput=5MB/sec 2024-09-13T00:14:11.351 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ku73_865_b39IWz off=5877549 len=827892 2024-09-13T00:14:11.360 INFO:tasks.workunit.client.0.smithi136.stdout:op 3896 completed, throughput=5MB/sec 2024-09-13T00:14:11.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:14:11 smithi136 bash[76371]: cluster 2024-09-13T00:14:09.528235+0000 mgr.y (mgr.24661) 1111 : cluster [DBG] pgmap v1794: 105 pgs: 105 active+clean; 126 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:14:11.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:14:11 smithi136 bash[75164]: cluster 2024-09-13T00:14:09.528235+0000 mgr.y (mgr.24661) 1111 : cluster [DBG] pgmap v1794: 105 pgs: 105 active+clean; 126 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:14:11.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:14:11 smithi181 bash[67587]: cluster 2024-09-13T00:14:09.528235+0000 mgr.y (mgr.24661) 1111 : cluster [DBG] pgmap v1794: 105 pgs: 105 active+clean; 126 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:14:12.361 INFO:tasks.workunit.client.0.smithi136.stdout: 745: throughput=4.99MB/sec pending data=0 2024-09-13T00:14:12.361 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Fbp2_gp0O-wRu45 off=4321778 len=702962 2024-09-13T00:14:12.368 INFO:tasks.workunit.client.0.smithi136.stdout:op 3897 completed, throughput=4.99MB/sec 2024-09-13T00:14:12.368 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Fv4fVDP2q55cyvj off=7302684 len=1296679 2024-09-13T00:14:12.372 INFO:tasks.workunit.client.0.smithi136.stdout:op 3898 completed, throughput=5MB/sec 2024-09-13T00:14:12.372 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj--hl_ozik4qU5a6m off=10034450 len=2080055 2024-09-13T00:14:12.388 INFO:tasks.workunit.client.0.smithi136.stdout:op 3899 completed, throughput=5MB/sec 2024-09-13T00:14:12.388 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-CViGNPhEeioiJYT off=8716802 len=1693338 2024-09-13T00:14:12.403 INFO:tasks.workunit.client.0.smithi136.stdout:op 3900 completed, throughput=5MB/sec 2024-09-13T00:14:12.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:14:12 smithi181 bash[95505]: ts=2024-09-13T00:14:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:14:13.403 INFO:tasks.workunit.client.0.smithi136.stdout: 746: throughput=4.99MB/sec pending data=0 2024-09-13T00:14:13.403 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-kJ7-El9wgJ7qczS off=3223431 len=1547081 2024-09-13T00:14:13.414 INFO:tasks.workunit.client.0.smithi136.stdout:op 3901 completed, throughput=5MB/sec 2024-09-13T00:14:13.414 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-hlEoUqX-kSMdK8V off=13805134 len=395303 2024-09-13T00:14:13.421 INFO:tasks.workunit.client.0.smithi136.stdout:op 3902 completed, throughput=5MB/sec 2024-09-13T00:14:13.421 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-kdwMigcL618xYxK off=10497840 len=1940294 2024-09-13T00:14:13.425 INFO:tasks.workunit.client.0.smithi136.stdout:op 3903 completed, throughput=5MB/sec 2024-09-13T00:14:13.425 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-sPILdS_tXMsxixb off=13500170 len=1148063 2024-09-13T00:14:13.435 INFO:tasks.workunit.client.0.smithi136.stdout:op 3904 completed, throughput=5MB/sec 2024-09-13T00:14:13.435 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-6ktRbqRfoSQnjpO off=3454692 len=1432284 2024-09-13T00:14:13.446 INFO:tasks.workunit.client.0.smithi136.stdout:op 3905 completed, throughput=5MB/sec 2024-09-13T00:14:13.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:14:13 smithi136 bash[76371]: cluster 2024-09-13T00:14:11.529377+0000 mgr.y (mgr.24661) 1112 : cluster [DBG] pgmap v1795: 105 pgs: 105 active+clean; 126 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:14:13.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:14:13 smithi136 bash[75164]: cluster 2024-09-13T00:14:11.529377+0000 mgr.y (mgr.24661) 1112 : cluster [DBG] pgmap v1795: 105 pgs: 105 active+clean; 126 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:14:13.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:14:13 smithi181 bash[67587]: cluster 2024-09-13T00:14:11.529377+0000 mgr.y (mgr.24661) 1112 : cluster [DBG] pgmap v1795: 105 pgs: 105 active+clean; 126 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:14:14.446 INFO:tasks.workunit.client.0.smithi136.stdout: 747: throughput=5MB/sec pending data=0 2024-09-13T00:14:14.446 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-HGAHgXqB3fakzZr off=1982792 len=1094202 2024-09-13T00:14:14.455 INFO:tasks.workunit.client.0.smithi136.stdout:op 3906 completed, throughput=5MB/sec 2024-09-13T00:14:14.455 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2ZV03rt34E5SPFL off=5131614 len=907734 2024-09-13T00:14:14.459 INFO:tasks.workunit.client.0.smithi136.stdout:op 3907 completed, throughput=5MB/sec 2024-09-13T00:14:14.459 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-vmx68gqJbFw-KRr off=16955458 len=1848333 2024-09-13T00:14:14.471 INFO:tasks.workunit.client.0.smithi136.stdout:op 3908 completed, throughput=5MB/sec 2024-09-13T00:14:14.471 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-5XuID_RwwAGiN4p off=1242346 len=404840 2024-09-13T00:14:14.476 INFO:tasks.workunit.client.0.smithi136.stdout:op 3909 completed, throughput=5MB/sec 2024-09-13T00:14:15.476 INFO:tasks.workunit.client.0.smithi136.stdout: 748: throughput=4.99MB/sec pending data=0 2024-09-13T00:14:15.476 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-mB9AlOJoNiwCzWR off=11549227 len=807288 2024-09-13T00:14:15.479 INFO:tasks.workunit.client.0.smithi136.stdout:op 3910 completed, throughput=4.99MB/sec 2024-09-13T00:14:15.479 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-szFlO_kfBcPmLD4 off=5332477 len=978225 2024-09-13T00:14:15.488 INFO:tasks.workunit.client.0.smithi136.stdout:op 3911 completed, throughput=5MB/sec 2024-09-13T00:14:15.488 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-q63xdgcKpq6z7fk off=4418670 len=705566 2024-09-13T00:14:15.496 INFO:tasks.workunit.client.0.smithi136.stdout:op 3912 completed, throughput=5MB/sec 2024-09-13T00:14:15.496 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-KCLuAOuXHphZdhF off=1562993 len=42374 2024-09-13T00:14:15.500 INFO:tasks.workunit.client.0.smithi136.stdout:op 3913 completed, throughput=5MB/sec 2024-09-13T00:14:15.500 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-3-2zkYd59I1h5d7 off=2602641 len=54637 2024-09-13T00:14:15.501 INFO:tasks.workunit.client.0.smithi136.stdout:op 3914 completed, throughput=5MB/sec 2024-09-13T00:14:15.501 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-laEGWJlloZcxnps off=19955430 len=1773620 2024-09-13T00:14:15.505 INFO:tasks.workunit.client.0.smithi136.stdout:op 3915 completed, throughput=5MB/sec 2024-09-13T00:14:15.506 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-yqhWY3yC7tpQqwV off=17647415 len=1905739 2024-09-13T00:14:15.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:14:15 smithi136 bash[75164]: cluster 2024-09-13T00:14:13.530475+0000 mgr.y (mgr.24661) 1113 : cluster [DBG] pgmap v1796: 105 pgs: 105 active+clean; 126 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:14:15.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:14:15 smithi136 bash[76371]: cluster 2024-09-13T00:14:13.530475+0000 mgr.y (mgr.24661) 1113 : cluster [DBG] pgmap v1796: 105 pgs: 105 active+clean; 126 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:14:15.524 INFO:tasks.workunit.client.0.smithi136.stdout:op 3916 completed, throughput=5MB/sec 2024-09-13T00:14:15.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:14:15 smithi181 bash[67587]: cluster 2024-09-13T00:14:13.530475+0000 mgr.y (mgr.24661) 1113 : cluster [DBG] pgmap v1796: 105 pgs: 105 active+clean; 126 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:14:16.524 INFO:tasks.workunit.client.0.smithi136.stdout: 749: throughput=4.99MB/sec pending data=0 2024-09-13T00:14:16.524 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-QzcDu0OZuGr6pYe off=15929144 len=247303 2024-09-13T00:14:16.526 INFO:tasks.workunit.client.0.smithi136.stdout:op 3917 completed, throughput=4.99MB/sec 2024-09-13T00:14:16.526 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-owaRXF70RDb2kkI off=6557953 len=497971 2024-09-13T00:14:16.532 INFO:tasks.workunit.client.0.smithi136.stdout:op 3918 completed, throughput=5MB/sec 2024-09-13T00:14:16.532 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-sE26qC0AkAv6S6C off=523813 len=1042582 2024-09-13T00:14:16.535 INFO:tasks.workunit.client.0.smithi136.stdout:op 3919 completed, throughput=5MB/sec 2024-09-13T00:14:16.535 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-R6LOXr-Hq-g2TkG off=10908074 len=47123 2024-09-13T00:14:16.539 INFO:tasks.workunit.client.0.smithi136.stdout:op 3920 completed, throughput=5MB/sec 2024-09-13T00:14:16.540 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-C74WU3gDmPTGiPg off=3541189 len=1888357 2024-09-13T00:14:16.553 INFO:tasks.workunit.client.0.smithi136.stdout:op 3921 completed, throughput=5MB/sec 2024-09-13T00:14:16.553 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-jP9S4InZ9QZZoTM off=7288877 len=694965 2024-09-13T00:14:16.555 INFO:tasks.workunit.client.0.smithi136.stdout:op 3922 completed, throughput=5MB/sec 2024-09-13T00:14:16.555 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-dRb0pOWR0P6go5S off=4129765 len=761093 2024-09-13T00:14:16.561 INFO:tasks.workunit.client.0.smithi136.stdout:op 3923 completed, throughput=5MB/sec 2024-09-13T00:14:17.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:14:17 smithi136 bash[75164]: cluster 2024-09-13T00:14:15.531254+0000 mgr.y (mgr.24661) 1114 : cluster [DBG] pgmap v1797: 105 pgs: 105 active+clean; 126 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 2.8 MiB/s rd, 1.7 MiB/s wr, 3 op/s 2024-09-13T00:14:17.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:14:17 smithi136 bash[76371]: cluster 2024-09-13T00:14:15.531254+0000 mgr.y (mgr.24661) 1114 : cluster [DBG] pgmap v1797: 105 pgs: 105 active+clean; 126 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 2.8 MiB/s rd, 1.7 MiB/s wr, 3 op/s 2024-09-13T00:14:17.562 INFO:tasks.workunit.client.0.smithi136.stdout: 750: throughput=4.99MB/sec pending data=0 2024-09-13T00:14:17.562 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-WsjuP9lhhYd4W4P off=11471855 len=824294 2024-09-13T00:14:17.564 INFO:tasks.workunit.client.0.smithi136.stdout:op 3924 completed, throughput=5MB/sec 2024-09-13T00:14:17.564 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-sDT1irEpm6w5ovT off=10149418 len=898411 2024-09-13T00:14:17.572 INFO:tasks.workunit.client.0.smithi136.stdout:op 3925 completed, throughput=5MB/sec 2024-09-13T00:14:17.573 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-4xc5Kgi3iRRubwy off=11688073 len=9201 2024-09-13T00:14:17.576 INFO:tasks.workunit.client.0.smithi136.stdout:op 3926 completed, throughput=5MB/sec 2024-09-13T00:14:17.576 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Gr7N75-c5LujV-1 off=11450305 len=1435159 2024-09-13T00:14:17.590 INFO:tasks.workunit.client.0.smithi136.stdout:op 3927 completed, throughput=5MB/sec 2024-09-13T00:14:17.590 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-LfbfRLFsgSEhu94 off=3614480 len=527708 2024-09-13T00:14:17.596 INFO:tasks.workunit.client.0.smithi136.stdout:op 3928 completed, throughput=5MB/sec 2024-09-13T00:14:17.596 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-U7Bj9XQAw6pLJg4 off=3649307 len=431369 2024-09-13T00:14:17.597 INFO:tasks.workunit.client.0.smithi136.stdout:op 3929 completed, throughput=5MB/sec 2024-09-13T00:14:17.597 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-mWBdp4KM8QZuYVY off=7502046 len=1755237 2024-09-13T00:14:17.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:14:17 smithi181 bash[67587]: cluster 2024-09-13T00:14:15.531254+0000 mgr.y (mgr.24661) 1114 : cluster [DBG] pgmap v1797: 105 pgs: 105 active+clean; 126 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 2.8 MiB/s rd, 1.7 MiB/s wr, 3 op/s 2024-09-13T00:14:17.602 INFO:tasks.workunit.client.0.smithi136.stdout:op 3930 completed, throughput=5MB/sec 2024-09-13T00:14:18.602 INFO:tasks.workunit.client.0.smithi136.stdout: 751: throughput=4.99MB/sec pending data=0 2024-09-13T00:14:18.602 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-npo4yWjiE5b8MhG off=4526904 len=685483 2024-09-13T00:14:18.604 INFO:tasks.workunit.client.0.smithi136.stdout:op 3931 completed, throughput=5MB/sec 2024-09-13T00:14:18.604 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-3TL3jKSkpIKdT0z off=984360 len=815654 2024-09-13T00:14:18.613 INFO:tasks.workunit.client.0.smithi136.stdout:op 3932 completed, throughput=5MB/sec 2024-09-13T00:14:18.613 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-q_Wrx26jUOiGw6c off=6343423 len=567810 2024-09-13T00:14:18.620 INFO:tasks.workunit.client.0.smithi136.stdout:op 3933 completed, throughput=5MB/sec 2024-09-13T00:14:18.620 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-mNeWlMIWeIp3iRm off=17628354 len=343041 2024-09-13T00:14:18.625 INFO:tasks.workunit.client.0.smithi136.stdout:op 3934 completed, throughput=5MB/sec 2024-09-13T00:14:18.625 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Dgt_dcrqb4HmTEK off=12446278 len=1774078 2024-09-13T00:14:18.629 INFO:tasks.workunit.client.0.smithi136.stdout:op 3935 completed, throughput=5MB/sec 2024-09-13T00:14:18.629 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-EPeiZeUdbhOLCZc off=2910949 len=800033 2024-09-13T00:14:18.631 INFO:tasks.workunit.client.0.smithi136.stdout:op 3936 completed, throughput=5MB/sec 2024-09-13T00:14:19.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:14:19 smithi136 bash[75164]: cluster 2024-09-13T00:14:17.532354+0000 mgr.y (mgr.24661) 1115 : cluster [DBG] pgmap v1798: 105 pgs: 105 active+clean; 126 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.7 MiB/s wr, 4 op/s 2024-09-13T00:14:19.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:14:19 smithi136 bash[76371]: cluster 2024-09-13T00:14:17.532354+0000 mgr.y (mgr.24661) 1115 : cluster [DBG] pgmap v1798: 105 pgs: 105 active+clean; 126 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.7 MiB/s wr, 4 op/s 2024-09-13T00:14:19.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:14:19 smithi181 bash[67587]: cluster 2024-09-13T00:14:17.532354+0000 mgr.y (mgr.24661) 1115 : cluster [DBG] pgmap v1798: 105 pgs: 105 active+clean; 126 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.7 MiB/s wr, 4 op/s 2024-09-13T00:14:19.632 INFO:tasks.workunit.client.0.smithi136.stdout: 752: throughput=4.99MB/sec pending data=0 2024-09-13T00:14:19.632 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-N5lYVX4q_pQgUlX off=11774362 len=1775193 2024-09-13T00:14:19.636 INFO:tasks.workunit.client.0.smithi136.stdout:op 3937 completed, throughput=5MB/sec 2024-09-13T00:14:19.636 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-EiqDNhRu6yMFdbh off=4091936 len=1715200 2024-09-13T00:14:19.648 INFO:tasks.workunit.client.0.smithi136.stdout:op 3938 completed, throughput=5MB/sec 2024-09-13T00:14:19.648 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-5aVakWad-zMxunN off=2244140 len=724880 2024-09-13T00:14:19.651 INFO:tasks.workunit.client.0.smithi136.stdout:op 3939 completed, throughput=5MB/sec 2024-09-13T00:14:19.651 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-yggiKNOt7spz4Bn off=4109710 len=1148278 2024-09-13T00:14:19.654 INFO:tasks.workunit.client.0.smithi136.stdout:op 3940 completed, throughput=5MB/sec 2024-09-13T00:14:19.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:14:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:14:19] "GET /metrics HTTP/1.1" 200 34139 "" "Prometheus/2.43.0" 2024-09-13T00:14:20.655 INFO:tasks.workunit.client.0.smithi136.stdout: 753: throughput=4.99MB/sec pending data=0 2024-09-13T00:14:20.655 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-4R-fvwSdTSi2F30 off=351380 len=925014 2024-09-13T00:14:20.657 INFO:tasks.workunit.client.0.smithi136.stdout:op 3941 completed, throughput=5MB/sec 2024-09-13T00:14:20.657 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-JcZiPKGXpvm0Jgm off=1612340 len=2001513 2024-09-13T00:14:20.671 INFO:tasks.workunit.client.0.smithi136.stdout:op 3942 completed, throughput=5MB/sec 2024-09-13T00:14:20.671 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_bXtQnDCOTqycXI off=434802 len=1579608 2024-09-13T00:14:20.682 INFO:tasks.workunit.client.0.smithi136.stdout:op 3943 completed, throughput=5MB/sec 2024-09-13T00:14:20.682 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-wBABaPGixm9RBRr off=1155390 len=2020972 2024-09-13T00:14:20.686 INFO:tasks.workunit.client.0.smithi136.stdout:op 3944 completed, throughput=5MB/sec 2024-09-13T00:14:21.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:14:21 smithi136 bash[75164]: cluster 2024-09-13T00:14:19.533224+0000 mgr.y (mgr.24661) 1116 : cluster [DBG] pgmap v1799: 105 pgs: 105 active+clean; 126 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:14:21.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:14:21 smithi136 bash[76371]: cluster 2024-09-13T00:14:19.533224+0000 mgr.y (mgr.24661) 1116 : cluster [DBG] pgmap v1799: 105 pgs: 105 active+clean; 126 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:14:21.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:14:21 smithi181 bash[67587]: cluster 2024-09-13T00:14:19.533224+0000 mgr.y (mgr.24661) 1116 : cluster [DBG] pgmap v1799: 105 pgs: 105 active+clean; 126 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:14:21.687 INFO:tasks.workunit.client.0.smithi136.stdout: 754: throughput=5MB/sec pending data=0 2024-09-13T00:14:21.687 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-xKqZvvaBPhAUhAf off=12524864 len=154580 2024-09-13T00:14:21.692 INFO:tasks.workunit.client.0.smithi136.stdout:op 3945 completed, throughput=5MB/sec 2024-09-13T00:14:21.692 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-clA_gEe2_I4xuj6 off=8648411 len=10437 2024-09-13T00:14:21.696 INFO:tasks.workunit.client.0.smithi136.stdout:op 3946 completed, throughput=5MB/sec 2024-09-13T00:14:21.696 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-rz1ESBAloXZKwzq off=10185994 len=1005602 2024-09-13T00:14:21.698 INFO:tasks.workunit.client.0.smithi136.stdout:op 3947 completed, throughput=5MB/sec 2024-09-13T00:14:21.699 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-uwiyCscWw19jP3H off=11913924 len=1258501 2024-09-13T00:14:21.711 INFO:tasks.workunit.client.0.smithi136.stdout:op 3948 completed, throughput=5MB/sec 2024-09-13T00:14:21.711 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-G6_fencUoMSKGd_ off=2571681 len=1859115 2024-09-13T00:14:21.715 INFO:tasks.workunit.client.0.smithi136.stdout:op 3949 completed, throughput=5MB/sec 2024-09-13T00:14:22.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:14:22 smithi181 bash[95505]: ts=2024-09-13T00:14:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:14:22.715 INFO:tasks.workunit.client.0.smithi136.stdout: 755: throughput=4.99MB/sec pending data=0 2024-09-13T00:14:22.716 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-7IRHYWemcpNi8p8 off=13823092 len=883447 2024-09-13T00:14:22.718 INFO:tasks.workunit.client.0.smithi136.stdout:op 3950 completed, throughput=5MB/sec 2024-09-13T00:14:22.718 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-hyiIUgmeJXVwSuG off=704314 len=855511 2024-09-13T00:14:22.721 INFO:tasks.workunit.client.0.smithi136.stdout:op 3951 completed, throughput=5MB/sec 2024-09-13T00:14:22.721 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-5CNuT-7x5q3ZcDc off=12064560 len=181075 2024-09-13T00:14:22.726 INFO:tasks.workunit.client.0.smithi136.stdout:op 3952 completed, throughput=5MB/sec 2024-09-13T00:14:22.726 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-axm0sEEgCl7G-cA off=473065 len=1207808 2024-09-13T00:14:22.730 INFO:tasks.workunit.client.0.smithi136.stdout:op 3953 completed, throughput=5MB/sec 2024-09-13T00:14:22.731 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-EkBUj9K3exLWEdE off=1306873 len=1744972 2024-09-13T00:14:22.735 INFO:tasks.workunit.client.0.smithi136.stdout:op 3954 completed, throughput=5MB/sec 2024-09-13T00:14:23.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:14:23 smithi136 bash[75164]: cluster 2024-09-13T00:14:21.534205+0000 mgr.y (mgr.24661) 1117 : cluster [DBG] pgmap v1800: 105 pgs: 105 active+clean; 126 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.7 MiB/s wr, 4 op/s 2024-09-13T00:14:23.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:14:23 smithi136 bash[76371]: cluster 2024-09-13T00:14:21.534205+0000 mgr.y (mgr.24661) 1117 : cluster [DBG] pgmap v1800: 105 pgs: 105 active+clean; 126 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.7 MiB/s wr, 4 op/s 2024-09-13T00:14:23.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:14:23 smithi181 bash[67587]: cluster 2024-09-13T00:14:21.534205+0000 mgr.y (mgr.24661) 1117 : cluster [DBG] pgmap v1800: 105 pgs: 105 active+clean; 126 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.7 MiB/s wr, 4 op/s 2024-09-13T00:14:23.735 INFO:tasks.workunit.client.0.smithi136.stdout: 756: throughput=4.99MB/sec pending data=0 2024-09-13T00:14:23.735 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-njja-kpMEBq7Fmj off=5903062 len=1367253 2024-09-13T00:14:23.739 INFO:tasks.workunit.client.0.smithi136.stdout:op 3955 completed, throughput=5MB/sec 2024-09-13T00:14:23.739 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-gZ-yl37A99pSbpo off=3258340 len=447430 2024-09-13T00:14:23.745 INFO:tasks.workunit.client.0.smithi136.stdout:op 3956 completed, throughput=5MB/sec 2024-09-13T00:14:23.746 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-lssQw1mtQw8EF6H off=534185 len=312353 2024-09-13T00:14:23.751 INFO:tasks.workunit.client.0.smithi136.stdout:op 3957 completed, throughput=5MB/sec 2024-09-13T00:14:23.751 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-gG0PRxh7yQGgLZx off=715309 len=525479 2024-09-13T00:14:23.758 INFO:tasks.workunit.client.0.smithi136.stdout:op 3958 completed, throughput=5MB/sec 2024-09-13T00:14:23.758 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_QOQuDW6PWFfct- off=10042633 len=1658810 2024-09-13T00:14:23.769 INFO:tasks.workunit.client.0.smithi136.stdout:op 3959 completed, throughput=5MB/sec 2024-09-13T00:14:23.769 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ETIEC2_udbiYl9- off=95018 len=1174070 2024-09-13T00:14:23.773 INFO:tasks.workunit.client.0.smithi136.stdout:op 3960 completed, throughput=5MB/sec 2024-09-13T00:14:24.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:14:24 smithi136 bash[75164]: audit 2024-09-13T00:14:23.793390+0000 mon.a (mon.0) 2221 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:14:24.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:14:24 smithi136 bash[76371]: audit 2024-09-13T00:14:23.793390+0000 mon.a (mon.0) 2221 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:14:24.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:14:24 smithi181 bash[67587]: audit 2024-09-13T00:14:23.793390+0000 mon.a (mon.0) 2221 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:14:24.773 INFO:tasks.workunit.client.0.smithi136.stdout: 757: throughput=4.99MB/sec pending data=0 2024-09-13T00:14:24.773 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-06qbGQ1MHYeiEcd off=20371203 len=328431 2024-09-13T00:14:24.778 INFO:tasks.workunit.client.0.smithi136.stdout:op 3961 completed, throughput=4.99MB/sec 2024-09-13T00:14:24.778 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-X0YKsdOmm7b3sX- off=6562647 len=1608208 2024-09-13T00:14:24.793 INFO:tasks.workunit.client.0.smithi136.stdout:op 3962 completed, throughput=5MB/sec 2024-09-13T00:14:24.793 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-d1p_N_2qzaZoaqI off=14995713 len=566100 2024-09-13T00:14:24.799 INFO:tasks.workunit.client.0.smithi136.stdout:op 3963 completed, throughput=5MB/sec 2024-09-13T00:14:24.799 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Fv4fVDP2q55cyvj off=6035607 len=755126 2024-09-13T00:14:24.806 INFO:tasks.workunit.client.0.smithi136.stdout:op 3964 completed, throughput=5MB/sec 2024-09-13T00:14:24.807 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-J0X8x_DmNAZsgLz off=5404611 len=1268934 2024-09-13T00:14:24.810 INFO:tasks.workunit.client.0.smithi136.stdout:op 3965 completed, throughput=5MB/sec 2024-09-13T00:14:24.810 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-I_sA0joJuMCvkby off=8622264 len=340155 2024-09-13T00:14:24.812 INFO:tasks.workunit.client.0.smithi136.stdout:op 3966 completed, throughput=5MB/sec 2024-09-13T00:14:24.812 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-HCCNSKJ-7XbAHoK off=2215983 len=2036920 2024-09-13T00:14:24.825 INFO:tasks.workunit.client.0.smithi136.stdout:op 3967 completed, throughput=5MB/sec 2024-09-13T00:14:25.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:14:25 smithi136 bash[75164]: cluster 2024-09-13T00:14:23.535210+0000 mgr.y (mgr.24661) 1118 : cluster [DBG] pgmap v1801: 105 pgs: 105 active+clean; 126 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:14:25.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:14:25 smithi136 bash[76371]: cluster 2024-09-13T00:14:23.535210+0000 mgr.y (mgr.24661) 1118 : cluster [DBG] pgmap v1801: 105 pgs: 105 active+clean; 126 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:14:25.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:14:25 smithi181 bash[67587]: cluster 2024-09-13T00:14:23.535210+0000 mgr.y (mgr.24661) 1118 : cluster [DBG] pgmap v1801: 105 pgs: 105 active+clean; 126 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:14:25.825 INFO:tasks.workunit.client.0.smithi136.stdout: 758: throughput=5MB/sec pending data=0 2024-09-13T00:14:25.825 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-q5QPKEdy4pUfvkK off=11088821 len=807024 2024-09-13T00:14:25.828 INFO:tasks.workunit.client.0.smithi136.stdout:op 3968 completed, throughput=5MB/sec 2024-09-13T00:14:25.828 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-SpXulRAwF10WQdX off=8709387 len=861702 2024-09-13T00:14:25.830 INFO:tasks.workunit.client.0.smithi136.stdout:op 3969 completed, throughput=5MB/sec 2024-09-13T00:14:25.830 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-uv-ncc23pbAr6Aw off=20883029 len=1159700 2024-09-13T00:14:25.834 INFO:tasks.workunit.client.0.smithi136.stdout:op 3970 completed, throughput=5MB/sec 2024-09-13T00:14:25.835 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-oDjDk5U98VHKgIf off=11068602 len=566252 2024-09-13T00:14:25.837 INFO:tasks.workunit.client.0.smithi136.stdout:op 3971 completed, throughput=5MB/sec 2024-09-13T00:14:25.837 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-9NvkFq-mrFy_Yqv off=7915128 len=611507 2024-09-13T00:14:25.839 INFO:tasks.workunit.client.0.smithi136.stdout:op 3972 completed, throughput=5MB/sec 2024-09-13T00:14:26.839 INFO:tasks.workunit.client.0.smithi136.stdout: 759: throughput=4.99MB/sec pending data=0 2024-09-13T00:14:26.839 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ku73_865_b39IWz off=5385463 len=1365306 2024-09-13T00:14:26.851 INFO:tasks.workunit.client.0.smithi136.stdout:op 3973 completed, throughput=5MB/sec 2024-09-13T00:14:26.851 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-6ju_Xt5Mz5JJepc off=0 len=1206678 2024-09-13T00:14:26.855 INFO:tasks.workunit.client.0.smithi136.stdout:op 3974 completed, throughput=5MB/sec 2024-09-13T00:14:26.855 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-7SvwsEe3MBrOvO1 off=3897494 len=1720757 2024-09-13T00:14:26.861 INFO:tasks.workunit.client.0.smithi136.stdout:op 3975 completed, throughput=5MB/sec 2024-09-13T00:14:26.861 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Y6enpbJTLF5avAp off=17597389 len=608541 2024-09-13T00:14:26.864 INFO:tasks.workunit.client.0.smithi136.stdout:op 3976 completed, throughput=5MB/sec 2024-09-13T00:14:26.864 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-aF5S3txfYT_G6Gx off=12011852 len=527914 2024-09-13T00:14:26.865 INFO:tasks.workunit.client.0.smithi136.stdout:op 3977 completed, throughput=5MB/sec 2024-09-13T00:14:27.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:14:27 smithi136 bash[75164]: cluster 2024-09-13T00:14:25.535955+0000 mgr.y (mgr.24661) 1119 : cluster [DBG] pgmap v1802: 105 pgs: 105 active+clean; 126 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:14:27.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:14:27 smithi136 bash[76371]: cluster 2024-09-13T00:14:25.535955+0000 mgr.y (mgr.24661) 1119 : cluster [DBG] pgmap v1802: 105 pgs: 105 active+clean; 126 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:14:27.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:14:27 smithi181 bash[67587]: cluster 2024-09-13T00:14:25.535955+0000 mgr.y (mgr.24661) 1119 : cluster [DBG] pgmap v1802: 105 pgs: 105 active+clean; 126 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:14:27.866 INFO:tasks.workunit.client.0.smithi136.stdout: 760: throughput=4.99MB/sec pending data=0 2024-09-13T00:14:27.866 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-silOq8EuA6fgwbc off=5345479 len=1008661 2024-09-13T00:14:27.874 INFO:tasks.workunit.client.0.smithi136.stdout:op 3978 completed, throughput=5MB/sec 2024-09-13T00:14:27.874 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-SLu5VE92F-x5mft off=658205 len=484423 2024-09-13T00:14:27.880 INFO:tasks.workunit.client.0.smithi136.stdout:op 3979 completed, throughput=5MB/sec 2024-09-13T00:14:27.880 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-eiS4gO2MsK45y1Y off=2117655 len=2080681 2024-09-13T00:14:27.896 INFO:tasks.workunit.client.0.smithi136.stdout:op 3980 completed, throughput=5MB/sec 2024-09-13T00:14:27.896 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-CQdCAGoGEEt-8q4 off=7429525 len=96072 2024-09-13T00:14:27.896 INFO:tasks.workunit.client.0.smithi136.stdout:op 3981 completed, throughput=5MB/sec 2024-09-13T00:14:27.896 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-dF3k-kvHGEbVnq9 off=7590229 len=351472 2024-09-13T00:14:27.898 INFO:tasks.workunit.client.0.smithi136.stdout:op 3982 completed, throughput=5MB/sec 2024-09-13T00:14:27.898 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-NznuPYeHrlbvPj1 off=7780036 len=662673 2024-09-13T00:14:27.904 INFO:tasks.workunit.client.0.smithi136.stdout:op 3983 completed, throughput=5MB/sec 2024-09-13T00:14:27.905 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Wz6JzvAWHP1SSGg off=9040550 len=221322 2024-09-13T00:14:27.909 INFO:tasks.workunit.client.0.smithi136.stdout:op 3984 completed, throughput=5MB/sec 2024-09-13T00:14:27.909 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-TXqQujalorFPrww off=3265446 len=469333 2024-09-13T00:14:27.911 INFO:tasks.workunit.client.0.smithi136.stdout:op 3985 completed, throughput=5MB/sec 2024-09-13T00:14:28.912 INFO:tasks.workunit.client.0.smithi136.stdout: 761: throughput=4.99MB/sec pending data=0 2024-09-13T00:14:28.912 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-x8LsW8XpzQSyjex off=775655 len=1344956 2024-09-13T00:14:28.915 INFO:tasks.workunit.client.0.smithi136.stdout:op 3986 completed, throughput=5MB/sec 2024-09-13T00:14:28.915 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-TXqQujalorFPrww off=13386695 len=888389 2024-09-13T00:14:28.917 INFO:tasks.workunit.client.0.smithi136.stdout:op 3987 completed, throughput=5MB/sec 2024-09-13T00:14:28.918 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ha1hkwMI6Uwowzg off=1621820 len=1096090 2024-09-13T00:14:28.928 INFO:tasks.workunit.client.0.smithi136.stdout:op 3988 completed, throughput=5MB/sec 2024-09-13T00:14:28.928 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-JhVfwbDlgq7RtoV off=6604028 len=2011408 2024-09-13T00:14:28.933 INFO:tasks.workunit.client.0.smithi136.stdout:op 3989 completed, throughput=5MB/sec 2024-09-13T00:14:29.412 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:14:29 smithi136 bash[76371]: cluster 2024-09-13T00:14:27.537066+0000 mgr.y (mgr.24661) 1120 : cluster [DBG] pgmap v1803: 105 pgs: 105 active+clean; 126 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:14:29.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:14:29 smithi136 bash[75164]: cluster 2024-09-13T00:14:27.537066+0000 mgr.y (mgr.24661) 1120 : cluster [DBG] pgmap v1803: 105 pgs: 105 active+clean; 126 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:14:29.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:14:29 smithi181 bash[67587]: cluster 2024-09-13T00:14:27.537066+0000 mgr.y (mgr.24661) 1120 : cluster [DBG] pgmap v1803: 105 pgs: 105 active+clean; 126 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:14:29.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:14:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:14:29] "GET /metrics HTTP/1.1" 200 34140 "" "Prometheus/2.43.0" 2024-09-13T00:14:29.933 INFO:tasks.workunit.client.0.smithi136.stdout: 762: throughput=4.99MB/sec pending data=0 2024-09-13T00:14:29.934 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-WhH78pk6CUblt-l off=1948881 len=1394694 2024-09-13T00:14:29.937 INFO:tasks.workunit.client.0.smithi136.stdout:op 3990 completed, throughput=5MB/sec 2024-09-13T00:14:29.937 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-oKhTj18jIV8ZCzZ off=8935228 len=1080956 2024-09-13T00:14:29.946 INFO:tasks.workunit.client.0.smithi136.stdout:op 3991 completed, throughput=5MB/sec 2024-09-13T00:14:29.946 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-mDAdkoWiugqFFNL off=17559637 len=738945 2024-09-13T00:14:29.953 INFO:tasks.workunit.client.0.smithi136.stdout:op 3992 completed, throughput=5MB/sec 2024-09-13T00:14:29.953 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-mGD7OKL8xs71Fn- off=11408903 len=273866 2024-09-13T00:14:29.956 INFO:tasks.workunit.client.0.smithi136.stdout:op 3993 completed, throughput=5MB/sec 2024-09-13T00:14:29.956 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-C40NA9qTM94EC5u off=13510191 len=280512 2024-09-13T00:14:29.957 INFO:tasks.workunit.client.0.smithi136.stdout:op 3994 completed, throughput=5MB/sec 2024-09-13T00:14:29.958 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-7hpr3z--2cnnj3W off=1343091 len=1020858 2024-09-13T00:14:29.961 INFO:tasks.workunit.client.0.smithi136.stdout:op 3995 completed, throughput=5MB/sec 2024-09-13T00:14:29.961 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-iBU6t0TccCgPeGf off=2329606 len=1655509 2024-09-13T00:14:29.965 INFO:tasks.workunit.client.0.smithi136.stdout:op 3996 completed, throughput=5MB/sec 2024-09-13T00:14:30.965 INFO:tasks.workunit.client.0.smithi136.stdout: 763: throughput=5MB/sec pending data=0 2024-09-13T00:14:30.965 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-J2855SUuba979zv off=354284 len=1642363 2024-09-13T00:14:30.977 INFO:tasks.workunit.client.0.smithi136.stdout:op 3997 completed, throughput=5MB/sec 2024-09-13T00:14:30.977 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-obOqydtACeEPjqY off=7965182 len=1771068 2024-09-13T00:14:30.983 INFO:tasks.workunit.client.0.smithi136.stdout:op 3998 completed, throughput=5MB/sec 2024-09-13T00:14:30.983 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2DB6gBw2Edl54eq off=2007749 len=1225984 2024-09-13T00:14:30.988 INFO:tasks.workunit.client.0.smithi136.stdout:op 3999 completed, throughput=5MB/sec 2024-09-13T00:14:31.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:14:31 smithi136 bash[75164]: cluster 2024-09-13T00:14:29.537934+0000 mgr.y (mgr.24661) 1121 : cluster [DBG] pgmap v1804: 105 pgs: 105 active+clean; 126 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 2.5 MiB/s rd, 1.9 MiB/s wr, 4 op/s 2024-09-13T00:14:31.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:14:31 smithi136 bash[76371]: cluster 2024-09-13T00:14:29.537934+0000 mgr.y (mgr.24661) 1121 : cluster [DBG] pgmap v1804: 105 pgs: 105 active+clean; 126 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 2.5 MiB/s rd, 1.9 MiB/s wr, 4 op/s 2024-09-13T00:14:31.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:14:31 smithi181 bash[67587]: cluster 2024-09-13T00:14:29.537934+0000 mgr.y (mgr.24661) 1121 : cluster [DBG] pgmap v1804: 105 pgs: 105 active+clean; 126 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 2.5 MiB/s rd, 1.9 MiB/s wr, 4 op/s 2024-09-13T00:14:31.988 INFO:tasks.workunit.client.0.smithi136.stdout: 764: throughput=4.99MB/sec pending data=0 2024-09-13T00:14:31.989 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Ko5cMdBgPhMw7eo off=1785302 len=1791106 2024-09-13T00:14:31.993 INFO:tasks.workunit.client.0.smithi136.stdout:op 4000 completed, throughput=5MB/sec 2024-09-13T00:14:31.993 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-sWFggHgjf5Lzl0Z off=2142958 len=276819 2024-09-13T00:14:31.995 INFO:tasks.workunit.client.0.smithi136.stdout:op 4001 completed, throughput=5MB/sec 2024-09-13T00:14:31.995 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2rE50P7Wcdu8Btf off=10472801 len=1460429 2024-09-13T00:14:31.999 INFO:tasks.workunit.client.0.smithi136.stdout:op 4002 completed, throughput=5MB/sec 2024-09-13T00:14:31.999 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-lkt79S6EqeE9gwa off=1405680 len=631201 2024-09-13T00:14:32.006 INFO:tasks.workunit.client.0.smithi136.stdout:op 4003 completed, throughput=5MB/sec 2024-09-13T00:14:32.006 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-qAciF1JiKGrfWzU off=1708677 len=1456948 2024-09-13T00:14:32.010 INFO:tasks.workunit.client.0.smithi136.stdout:op 4004 completed, throughput=5MB/sec 2024-09-13T00:14:32.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:14:32 smithi181 bash[95505]: ts=2024-09-13T00:14:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:14:33.011 INFO:tasks.workunit.client.0.smithi136.stdout: 765: throughput=4.99MB/sec pending data=0 2024-09-13T00:14:33.011 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-jRXV3m1IGzjeC4x off=3682545 len=684455 2024-09-13T00:14:33.018 INFO:tasks.workunit.client.0.smithi136.stdout:op 4005 completed, throughput=5MB/sec 2024-09-13T00:14:33.018 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Q5TiZOe38UJZFCi off=4524431 len=1313967 2024-09-13T00:14:33.021 INFO:tasks.workunit.client.0.smithi136.stdout:op 4006 completed, throughput=5MB/sec 2024-09-13T00:14:33.021 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-8B29QrDF5G2byA3 off=3059954 len=1798494 2024-09-13T00:14:33.026 INFO:tasks.workunit.client.0.smithi136.stdout:op 4007 completed, throughput=5MB/sec 2024-09-13T00:14:33.026 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-L0us9uvC5JwaxaB off=8435924 len=530752 2024-09-13T00:14:33.028 INFO:tasks.workunit.client.0.smithi136.stdout:op 4008 completed, throughput=5MB/sec 2024-09-13T00:14:33.029 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-G8TEewTbWZIMsJI off=18805141 len=1568099 2024-09-13T00:14:33.032 INFO:tasks.workunit.client.0.smithi136.stdout:op 4009 completed, throughput=5MB/sec 2024-09-13T00:14:33.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:14:33 smithi136 bash[75164]: cluster 2024-09-13T00:14:31.538895+0000 mgr.y (mgr.24661) 1122 : cluster [DBG] pgmap v1805: 105 pgs: 105 active+clean; 126 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.2 MiB/s wr, 5 op/s 2024-09-13T00:14:33.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:14:33 smithi136 bash[76371]: cluster 2024-09-13T00:14:31.538895+0000 mgr.y (mgr.24661) 1122 : cluster [DBG] pgmap v1805: 105 pgs: 105 active+clean; 126 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.2 MiB/s wr, 5 op/s 2024-09-13T00:14:33.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:14:33 smithi181 bash[67587]: cluster 2024-09-13T00:14:31.538895+0000 mgr.y (mgr.24661) 1122 : cluster [DBG] pgmap v1805: 105 pgs: 105 active+clean; 126 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.2 MiB/s wr, 5 op/s 2024-09-13T00:14:34.033 INFO:tasks.workunit.client.0.smithi136.stdout: 766: throughput=5MB/sec pending data=0 2024-09-13T00:14:34.033 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ROiZtMsey9FCI0s off=4807256 len=1354093 2024-09-13T00:14:34.046 INFO:tasks.workunit.client.0.smithi136.stdout:op 4010 completed, throughput=5MB/sec 2024-09-13T00:14:34.046 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-3KOEt40zYyAb7o7 off=3723553 len=1657289 2024-09-13T00:14:34.059 INFO:tasks.workunit.client.0.smithi136.stdout:op 4011 completed, throughput=5MB/sec 2024-09-13T00:14:34.059 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-RZfswUxtGadHoBx off=13775087 len=66388 2024-09-13T00:14:34.062 INFO:tasks.workunit.client.0.smithi136.stdout:op 4012 completed, throughput=5MB/sec 2024-09-13T00:14:34.063 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-1yNGhc2HZN8Tc58 off=4682650 len=1743454 2024-09-13T00:14:34.076 INFO:tasks.workunit.client.0.smithi136.stdout:op 4013 completed, throughput=5MB/sec 2024-09-13T00:14:35.077 INFO:tasks.workunit.client.0.smithi136.stdout: 767: throughput=4.99MB/sec pending data=0 2024-09-13T00:14:35.077 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ijzAGhM4i-HJesb off=3778765 len=1813099 2024-09-13T00:14:35.081 INFO:tasks.workunit.client.0.smithi136.stdout:op 4014 completed, throughput=5MB/sec 2024-09-13T00:14:35.081 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Xi388F8tpDJKSZr off=15275753 len=646340 2024-09-13T00:14:35.084 INFO:tasks.workunit.client.0.smithi136.stdout:op 4015 completed, throughput=5MB/sec 2024-09-13T00:14:35.084 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-HuZcf1rDQ5nzZxB off=1206398 len=51585 2024-09-13T00:14:35.089 INFO:tasks.workunit.client.0.smithi136.stdout:op 4016 completed, throughput=5MB/sec 2024-09-13T00:14:35.089 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-4T-3ip7PYqgpDb9 off=2209205 len=1220039 2024-09-13T00:14:35.093 INFO:tasks.workunit.client.0.smithi136.stdout:op 4017 completed, throughput=5MB/sec 2024-09-13T00:14:35.093 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-3_XzYCkWlhO_0AH off=1400993 len=617318 2024-09-13T00:14:35.095 INFO:tasks.workunit.client.0.smithi136.stdout:op 4018 completed, throughput=5MB/sec 2024-09-13T00:14:35.095 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-jo4X4n3Q1zurbeb off=3899109 len=1738668 2024-09-13T00:14:35.099 INFO:tasks.workunit.client.0.smithi136.stdout:op 4019 completed, throughput=5MB/sec 2024-09-13T00:14:35.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:14:35 smithi136 bash[75164]: cluster 2024-09-13T00:14:33.539888+0000 mgr.y (mgr.24661) 1123 : cluster [DBG] pgmap v1806: 105 pgs: 105 active+clean; 126 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 3.5 MiB/s rd, 2.2 MiB/s wr, 5 op/s 2024-09-13T00:14:35.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:14:35 smithi136 bash[76371]: cluster 2024-09-13T00:14:33.539888+0000 mgr.y (mgr.24661) 1123 : cluster [DBG] pgmap v1806: 105 pgs: 105 active+clean; 126 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 3.5 MiB/s rd, 2.2 MiB/s wr, 5 op/s 2024-09-13T00:14:35.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:14:35 smithi181 bash[67587]: cluster 2024-09-13T00:14:33.539888+0000 mgr.y (mgr.24661) 1123 : cluster [DBG] pgmap v1806: 105 pgs: 105 active+clean; 126 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 3.5 MiB/s rd, 2.2 MiB/s wr, 5 op/s 2024-09-13T00:14:36.100 INFO:tasks.workunit.client.0.smithi136.stdout: 768: throughput=5MB/sec pending data=0 2024-09-13T00:14:36.100 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-hu_ijIhZrfHaO8y off=15601563 len=861232 2024-09-13T00:14:36.107 INFO:tasks.workunit.client.0.smithi136.stdout:op 4020 completed, throughput=5MB/sec 2024-09-13T00:14:36.107 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-87puGLOTD7QJsw9 off=4237054 len=1317109 2024-09-13T00:14:36.119 INFO:tasks.workunit.client.0.smithi136.stdout:op 4021 completed, throughput=5MB/sec 2024-09-13T00:14:36.119 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-IL9Xjo29HlkQs5L off=4276939 len=23338 2024-09-13T00:14:36.122 INFO:tasks.workunit.client.0.smithi136.stdout:op 4022 completed, throughput=5MB/sec 2024-09-13T00:14:36.123 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-PmPtm-BgC-1Dz04 off=1303056 len=452891 2024-09-13T00:14:36.128 INFO:tasks.workunit.client.0.smithi136.stdout:op 4023 completed, throughput=5MB/sec 2024-09-13T00:14:36.128 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-AviAzYkxt6H4yeU off=6904761 len=524040 2024-09-13T00:14:36.134 INFO:tasks.workunit.client.0.smithi136.stdout:op 4024 completed, throughput=5MB/sec 2024-09-13T00:14:36.134 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-VSmLXzGP3VLkyF_ off=3313459 len=773170 2024-09-13T00:14:36.137 INFO:tasks.workunit.client.0.smithi136.stdout:op 4025 completed, throughput=5MB/sec 2024-09-13T00:14:36.137 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-BFOYvl3WliVqJD6 off=234504 len=1798085 2024-09-13T00:14:36.140 INFO:tasks.workunit.client.0.smithi136.stdout:op 4026 completed, throughput=5MB/sec 2024-09-13T00:14:37.141 INFO:tasks.workunit.client.0.smithi136.stdout: 769: throughput=5MB/sec pending data=0 2024-09-13T00:14:37.141 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_LegQYl6U29CESO off=4011295 len=1571674 2024-09-13T00:14:37.156 INFO:tasks.workunit.client.0.smithi136.stdout:op 4027 completed, throughput=5MB/sec 2024-09-13T00:14:37.156 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-meXvFwIWBlRKPkd off=1166413 len=1918004 2024-09-13T00:14:37.171 INFO:tasks.workunit.client.0.smithi136.stdout:op 4028 completed, throughput=5MB/sec 2024-09-13T00:14:37.172 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-hSGiiUTuq5i8fbT off=2008157 len=440937 2024-09-13T00:14:37.178 INFO:tasks.workunit.client.0.smithi136.stdout:op 4029 completed, throughput=5MB/sec 2024-09-13T00:14:37.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:14:37 smithi136 bash[75164]: cluster 2024-09-13T00:14:35.540540+0000 mgr.y (mgr.24661) 1124 : cluster [DBG] pgmap v1807: 105 pgs: 105 active+clean; 126 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 2.7 MiB/s rd, 1.7 MiB/s wr, 4 op/s 2024-09-13T00:14:37.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:14:37 smithi136 bash[76371]: cluster 2024-09-13T00:14:35.540540+0000 mgr.y (mgr.24661) 1124 : cluster [DBG] pgmap v1807: 105 pgs: 105 active+clean; 126 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 2.7 MiB/s rd, 1.7 MiB/s wr, 4 op/s 2024-09-13T00:14:37.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:14:37 smithi181 bash[67587]: cluster 2024-09-13T00:14:35.540540+0000 mgr.y (mgr.24661) 1124 : cluster [DBG] pgmap v1807: 105 pgs: 105 active+clean; 126 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 2.7 MiB/s rd, 1.7 MiB/s wr, 4 op/s 2024-09-13T00:14:38.178 INFO:tasks.workunit.client.0.smithi136.stdout: 770: throughput=4.99MB/sec pending data=0 2024-09-13T00:14:38.179 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Uz1Em9LIEsidggu off=13476939 len=1243433 2024-09-13T00:14:38.183 INFO:tasks.workunit.client.0.smithi136.stdout:op 4030 completed, throughput=5MB/sec 2024-09-13T00:14:38.183 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-co1qgi2UqLh3eDs off=9056691 len=355206 2024-09-13T00:14:38.184 INFO:tasks.workunit.client.0.smithi136.stdout:op 4031 completed, throughput=5MB/sec 2024-09-13T00:14:38.185 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-tgfPgVmV3nwFj3o off=17634360 len=821916 2024-09-13T00:14:38.187 INFO:tasks.workunit.client.0.smithi136.stdout:op 4032 completed, throughput=5MB/sec 2024-09-13T00:14:38.187 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-aKVx45xbFsvEe3o off=674354 len=1610310 2024-09-13T00:14:38.199 INFO:tasks.workunit.client.0.smithi136.stdout:op 4033 completed, throughput=5MB/sec 2024-09-13T00:14:38.200 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Lm2Z0slN6O2Wpcd off=16703296 len=1879303 2024-09-13T00:14:38.202 INFO:tasks.workunit.client.0.smithi136.stdout:op 4034 completed, throughput=5MB/sec 2024-09-13T00:14:39.203 INFO:tasks.workunit.client.0.smithi136.stdout: 771: throughput=4.99MB/sec pending data=0 2024-09-13T00:14:39.203 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-5z2krfSFdugeGLJ off=9270385 len=272468 2024-09-13T00:14:39.208 INFO:tasks.workunit.client.0.smithi136.stdout:op 4035 completed, throughput=4.99MB/sec 2024-09-13T00:14:39.208 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-v_Kc8h87DgWSIt3 off=1572900 len=593711 2024-09-13T00:14:39.215 INFO:tasks.workunit.client.0.smithi136.stdout:op 4036 completed, throughput=5MB/sec 2024-09-13T00:14:39.216 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-B7-1i_Jrk2BOdds off=360108 len=731502 2024-09-13T00:14:39.222 INFO:tasks.workunit.client.0.smithi136.stdout:op 4037 completed, throughput=5MB/sec 2024-09-13T00:14:39.222 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Y-0-NOpnfxYUNkj off=10553740 len=376674 2024-09-13T00:14:39.229 INFO:tasks.workunit.client.0.smithi136.stdout:op 4038 completed, throughput=5MB/sec 2024-09-13T00:14:39.229 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-91ikvFB32hJ9ZCD off=18341270 len=602186 2024-09-13T00:14:39.232 INFO:tasks.workunit.client.0.smithi136.stdout:op 4039 completed, throughput=5MB/sec 2024-09-13T00:14:39.232 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-3-UUHaL0iCprCOC off=6426388 len=642324 2024-09-13T00:14:39.234 INFO:tasks.workunit.client.0.smithi136.stdout:op 4040 completed, throughput=5MB/sec 2024-09-13T00:14:39.235 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-GsIzTJNGpV_Qpkv off=5429382 len=1503389 2024-09-13T00:14:39.239 INFO:tasks.workunit.client.0.smithi136.stdout:op 4041 completed, throughput=5MB/sec 2024-09-13T00:14:39.508 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:14:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:14:39] "GET /metrics HTTP/1.1" 200 34140 "" "Prometheus/2.43.0" 2024-09-13T00:14:39.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:14:39 smithi136 bash[75164]: cluster 2024-09-13T00:14:37.541650+0000 mgr.y (mgr.24661) 1125 : cluster [DBG] pgmap v1808: 105 pgs: 105 active+clean; 126 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 3.6 MiB/s rd, 2.0 MiB/s wr, 5 op/s 2024-09-13T00:14:39.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:14:39 smithi136 bash[75164]: audit 2024-09-13T00:14:38.793725+0000 mon.a (mon.0) 2222 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:14:39.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:14:39 smithi136 bash[76371]: cluster 2024-09-13T00:14:37.541650+0000 mgr.y (mgr.24661) 1125 : cluster [DBG] pgmap v1808: 105 pgs: 105 active+clean; 126 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 3.6 MiB/s rd, 2.0 MiB/s wr, 5 op/s 2024-09-13T00:14:39.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:14:39 smithi136 bash[76371]: audit 2024-09-13T00:14:38.793725+0000 mon.a (mon.0) 2222 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:14:39.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:14:39 smithi181 bash[67587]: cluster 2024-09-13T00:14:37.541650+0000 mgr.y (mgr.24661) 1125 : cluster [DBG] pgmap v1808: 105 pgs: 105 active+clean; 126 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 3.6 MiB/s rd, 2.0 MiB/s wr, 5 op/s 2024-09-13T00:14:39.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:14:39 smithi181 bash[67587]: audit 2024-09-13T00:14:38.793725+0000 mon.a (mon.0) 2222 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:14:40.239 INFO:tasks.workunit.client.0.smithi136.stdout: 772: throughput=4.99MB/sec pending data=0 2024-09-13T00:14:40.240 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-rf9nf8o41NPZ6hf off=5919601 len=643221 2024-09-13T00:14:40.246 INFO:tasks.workunit.client.0.smithi136.stdout:op 4042 completed, throughput=4.99MB/sec 2024-09-13T00:14:40.246 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-WoOHYdvI7-dr1gT off=5340991 len=1941415 2024-09-13T00:14:40.260 INFO:tasks.workunit.client.0.smithi136.stdout:op 4043 completed, throughput=5MB/sec 2024-09-13T00:14:40.260 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-IoO6XqZ7culrQP8 off=1173392 len=99007 2024-09-13T00:14:40.261 INFO:tasks.workunit.client.0.smithi136.stdout:op 4044 completed, throughput=5MB/sec 2024-09-13T00:14:40.261 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-qJy6fjXbHK1ewbd off=2028532 len=1931344 2024-09-13T00:14:40.265 INFO:tasks.workunit.client.0.smithi136.stdout:op 4045 completed, throughput=5MB/sec 2024-09-13T00:14:40.265 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-vFybpi81dvAcJRO off=15683777 len=372724 2024-09-13T00:14:40.266 INFO:tasks.workunit.client.0.smithi136.stdout:op 4046 completed, throughput=5MB/sec 2024-09-13T00:14:40.266 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-HdjOmKP_eq7dDzw off=949067 len=1908151 2024-09-13T00:14:40.271 INFO:tasks.workunit.client.0.smithi136.stdout:op 4047 completed, throughput=5MB/sec 2024-09-13T00:14:41.271 INFO:tasks.workunit.client.0.smithi136.stdout: 773: throughput=5MB/sec pending data=0 2024-09-13T00:14:41.271 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ArkgBTRpqDdUEGE off=3679540 len=1539642 2024-09-13T00:14:41.282 INFO:tasks.workunit.client.0.smithi136.stdout:op 4048 completed, throughput=5MB/sec 2024-09-13T00:14:41.282 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-FfWU7gSInXiXHvG off=9553246 len=1916960 2024-09-13T00:14:41.286 INFO:tasks.workunit.client.0.smithi136.stdout:op 4049 completed, throughput=5MB/sec 2024-09-13T00:14:41.286 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Lhe2yOHLdGQ-RB5 off=10091031 len=975155 2024-09-13T00:14:41.290 INFO:tasks.workunit.client.0.smithi136.stdout:op 4050 completed, throughput=5MB/sec 2024-09-13T00:14:41.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:14:41 smithi136 bash[75164]: cluster 2024-09-13T00:14:39.542570+0000 mgr.y (mgr.24661) 1126 : cluster [DBG] pgmap v1809: 105 pgs: 105 active+clean; 126 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:14:41.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:14:41 smithi136 bash[76371]: cluster 2024-09-13T00:14:39.542570+0000 mgr.y (mgr.24661) 1126 : cluster [DBG] pgmap v1809: 105 pgs: 105 active+clean; 126 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:14:41.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:14:41 smithi181 bash[67587]: cluster 2024-09-13T00:14:39.542570+0000 mgr.y (mgr.24661) 1126 : cluster [DBG] pgmap v1809: 105 pgs: 105 active+clean; 126 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:14:42.290 INFO:tasks.workunit.client.0.smithi136.stdout: 774: throughput=4.99MB/sec pending data=0 2024-09-13T00:14:42.291 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-sqZ64XBVPZ9lLmY off=5334064 len=1451979 2024-09-13T00:14:42.295 INFO:tasks.workunit.client.0.smithi136.stdout:op 4051 completed, throughput=5MB/sec 2024-09-13T00:14:42.295 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ZT49rjlM-HK_3Pz off=17787982 len=1130888 2024-09-13T00:14:42.299 INFO:tasks.workunit.client.0.smithi136.stdout:op 4052 completed, throughput=5MB/sec 2024-09-13T00:14:42.299 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-PaW9_z6-CLLom_F off=13443136 len=957172 2024-09-13T00:14:42.307 INFO:tasks.workunit.client.0.smithi136.stdout:op 4053 completed, throughput=5MB/sec 2024-09-13T00:14:42.307 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-sHyujmtXB2DYWcU off=2414978 len=1184486 2024-09-13T00:14:42.310 INFO:tasks.workunit.client.0.smithi136.stdout:op 4054 completed, throughput=5MB/sec 2024-09-13T00:14:42.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:14:42 smithi136 bash[75164]: cluster 2024-09-13T00:14:41.543563+0000 mgr.y (mgr.24661) 1127 : cluster [DBG] pgmap v1810: 105 pgs: 105 active+clean; 126 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 3.6 MiB/s rd, 2.0 MiB/s wr, 5 op/s 2024-09-13T00:14:42.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:14:42 smithi136 bash[76371]: cluster 2024-09-13T00:14:41.543563+0000 mgr.y (mgr.24661) 1127 : cluster [DBG] pgmap v1810: 105 pgs: 105 active+clean; 126 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 3.6 MiB/s rd, 2.0 MiB/s wr, 5 op/s 2024-09-13T00:14:42.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:14:42 smithi181 bash[67587]: cluster 2024-09-13T00:14:41.543563+0000 mgr.y (mgr.24661) 1127 : cluster [DBG] pgmap v1810: 105 pgs: 105 active+clean; 126 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 3.6 MiB/s rd, 2.0 MiB/s wr, 5 op/s 2024-09-13T00:14:42.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:14:42 smithi181 bash[95505]: ts=2024-09-13T00:14:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:14:43.310 INFO:tasks.workunit.client.0.smithi136.stdout: 775: throughput=4.99MB/sec pending data=0 2024-09-13T00:14:43.311 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-emqur-MdX9KF3ac off=13746489 len=2014041 2024-09-13T00:14:43.323 INFO:tasks.workunit.client.0.smithi136.stdout:op 4055 completed, throughput=5MB/sec 2024-09-13T00:14:43.323 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-xog0S7XmMnHNtPB off=20880508 len=2054374 2024-09-13T00:14:43.330 INFO:tasks.workunit.client.0.smithi136.stdout:op 4056 completed, throughput=5MB/sec 2024-09-13T00:14:43.331 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-JZgyrW02OkPaaO3 off=9765636 len=301560 2024-09-13T00:14:43.332 INFO:tasks.workunit.client.0.smithi136.stdout:op 4057 completed, throughput=5MB/sec 2024-09-13T00:14:43.332 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-O22SJ4kAsemMbSP off=329267 len=1501369 2024-09-13T00:14:43.343 INFO:tasks.workunit.client.0.smithi136.stdout:op 4058 completed, throughput=5MB/sec 2024-09-13T00:14:44.343 INFO:tasks.workunit.client.0.smithi136.stdout: 776: throughput=4.99MB/sec pending data=0 2024-09-13T00:14:44.344 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-74_lB4B5xTJhs4h off=834969 len=2093710 2024-09-13T00:14:44.350 INFO:tasks.workunit.client.0.smithi136.stdout:op 4059 completed, throughput=5MB/sec 2024-09-13T00:14:44.351 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-EO8IkQ4jZx9N0Io off=5147767 len=834208 2024-09-13T00:14:44.354 INFO:tasks.workunit.client.0.smithi136.stdout:op 4060 completed, throughput=5MB/sec 2024-09-13T00:14:44.354 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-uwiyCscWw19jP3H off=9879644 len=1567314 2024-09-13T00:14:44.368 INFO:tasks.workunit.client.0.smithi136.stdout:op 4061 completed, throughput=5MB/sec 2024-09-13T00:14:44.368 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-pKvGb4YyocG8Lpp off=9656932 len=1891772 2024-09-13T00:14:44.372 INFO:tasks.workunit.client.0.smithi136.stdout:op 4062 completed, throughput=5MB/sec 2024-09-13T00:14:45.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:14:44 smithi136 bash[75164]: cluster 2024-09-13T00:14:43.544566+0000 mgr.y (mgr.24661) 1128 : cluster [DBG] pgmap v1811: 105 pgs: 105 active+clean; 126 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 3.5 MiB/s rd, 2.1 MiB/s wr, 5 op/s 2024-09-13T00:14:45.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:14:44 smithi136 bash[76371]: cluster 2024-09-13T00:14:43.544566+0000 mgr.y (mgr.24661) 1128 : cluster [DBG] pgmap v1811: 105 pgs: 105 active+clean; 126 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 3.5 MiB/s rd, 2.1 MiB/s wr, 5 op/s 2024-09-13T00:14:45.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:14:44 smithi181 bash[67587]: cluster 2024-09-13T00:14:43.544566+0000 mgr.y (mgr.24661) 1128 : cluster [DBG] pgmap v1811: 105 pgs: 105 active+clean; 126 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 3.5 MiB/s rd, 2.1 MiB/s wr, 5 op/s 2024-09-13T00:14:45.372 INFO:tasks.workunit.client.0.smithi136.stdout: 777: throughput=5MB/sec pending data=0 2024-09-13T00:14:45.372 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-wiuQPsmsi96tM7P off=3822311 len=803495 2024-09-13T00:14:45.380 INFO:tasks.workunit.client.0.smithi136.stdout:op 4063 completed, throughput=5MB/sec 2024-09-13T00:14:45.380 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-_3Qh6O__5GvdzYM off=10067411 len=43886 2024-09-13T00:14:45.381 INFO:tasks.workunit.client.0.smithi136.stdout:op 4064 completed, throughput=5MB/sec 2024-09-13T00:14:45.381 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-m_Ba4wVsGcRTn_Z off=20648294 len=1598039 2024-09-13T00:14:45.385 INFO:tasks.workunit.client.0.smithi136.stdout:op 4065 completed, throughput=5MB/sec 2024-09-13T00:14:45.385 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-o2mSIQEASFZ6NTu off=415003 len=1204892 2024-09-13T00:14:45.388 INFO:tasks.workunit.client.0.smithi136.stdout:op 4066 completed, throughput=5MB/sec 2024-09-13T00:14:45.388 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-t2WRcWchEOCUtVc off=10681805 len=969120 2024-09-13T00:14:45.391 INFO:tasks.workunit.client.0.smithi136.stdout:op 4067 completed, throughput=5MB/sec 2024-09-13T00:14:46.391 INFO:tasks.workunit.client.0.smithi136.stdout: 778: throughput=4.99MB/sec pending data=0 2024-09-13T00:14:46.391 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-vdgPv980Iosgufm off=2323909 len=1425761 2024-09-13T00:14:46.402 INFO:tasks.workunit.client.0.smithi136.stdout:op 4068 completed, throughput=5MB/sec 2024-09-13T00:14:46.402 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Vr66Kb0JDjhbgeL off=4841954 len=558738 2024-09-13T00:14:46.404 INFO:tasks.workunit.client.0.smithi136.stdout:op 4069 completed, throughput=5MB/sec 2024-09-13T00:14:46.404 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-JyOGgzEPblNosdP off=6297149 len=510859 2024-09-13T00:14:46.406 INFO:tasks.workunit.client.0.smithi136.stdout:op 4070 completed, throughput=5MB/sec 2024-09-13T00:14:46.407 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-L0fmvE6TvX4B94e off=16568209 len=602303 2024-09-13T00:14:46.414 INFO:tasks.workunit.client.0.smithi136.stdout:op 4071 completed, throughput=5MB/sec 2024-09-13T00:14:46.415 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-aIDP6BfYFz8D_-6 off=7249475 len=236907 2024-09-13T00:14:46.416 INFO:tasks.workunit.client.0.smithi136.stdout:op 4072 completed, throughput=5MB/sec 2024-09-13T00:14:46.416 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-IWWGGhCYREpApjD off=2202628 len=1253813 2024-09-13T00:14:46.425 INFO:tasks.workunit.client.0.smithi136.stdout:op 4073 completed, throughput=5MB/sec 2024-09-13T00:14:46.425 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-3n-wkCPtoNnWNIk off=4218375 len=1850900 2024-09-13T00:14:46.429 INFO:tasks.workunit.client.0.smithi136.stdout:op 4074 completed, throughput=5MB/sec 2024-09-13T00:14:47.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:14:46 smithi136 bash[75164]: cluster 2024-09-13T00:14:45.545319+0000 mgr.y (mgr.24661) 1129 : cluster [DBG] pgmap v1812: 105 pgs: 105 active+clean; 126 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.7 MiB/s wr, 4 op/s 2024-09-13T00:14:47.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:14:46 smithi136 bash[76371]: cluster 2024-09-13T00:14:45.545319+0000 mgr.y (mgr.24661) 1129 : cluster [DBG] pgmap v1812: 105 pgs: 105 active+clean; 126 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.7 MiB/s wr, 4 op/s 2024-09-13T00:14:47.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:14:46 smithi181 bash[67587]: cluster 2024-09-13T00:14:45.545319+0000 mgr.y (mgr.24661) 1129 : cluster [DBG] pgmap v1812: 105 pgs: 105 active+clean; 126 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.7 MiB/s wr, 4 op/s 2024-09-13T00:14:47.430 INFO:tasks.workunit.client.0.smithi136.stdout: 779: throughput=5MB/sec pending data=0 2024-09-13T00:14:47.430 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-bXj95Q_QoqGPpDc off=14294393 len=1240287 2024-09-13T00:14:47.439 INFO:tasks.workunit.client.0.smithi136.stdout:op 4075 completed, throughput=5MB/sec 2024-09-13T00:14:47.439 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-AzOCeHg1b6bDqyP off=3559501 len=666621 2024-09-13T00:14:47.446 INFO:tasks.workunit.client.0.smithi136.stdout:op 4076 completed, throughput=5MB/sec 2024-09-13T00:14:47.446 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-YIesOiyha8m2HN1 off=2510239 len=65871 2024-09-13T00:14:47.447 INFO:tasks.workunit.client.0.smithi136.stdout:op 4077 completed, throughput=5MB/sec 2024-09-13T00:14:47.447 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-vNo66wpsRBvlErj off=21679037 len=1655571 2024-09-13T00:14:47.453 INFO:tasks.workunit.client.0.smithi136.stdout:op 4078 completed, throughput=5MB/sec 2024-09-13T00:14:47.453 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-lRNUadvGE_yohFI off=2376899 len=956870 2024-09-13T00:14:47.462 INFO:tasks.workunit.client.0.smithi136.stdout:op 4079 completed, throughput=5MB/sec 2024-09-13T00:14:48.463 INFO:tasks.workunit.client.0.smithi136.stdout: 780: throughput=4.99MB/sec pending data=0 2024-09-13T00:14:48.463 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-E6DksNud2ceUUml off=8124233 len=1067682 2024-09-13T00:14:48.472 INFO:tasks.workunit.client.0.smithi136.stdout:op 4080 completed, throughput=5MB/sec 2024-09-13T00:14:48.472 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-NA6EZMQDf_yisAk off=4562100 len=1609194 2024-09-13T00:14:48.484 INFO:tasks.workunit.client.0.smithi136.stdout:op 4081 completed, throughput=5MB/sec 2024-09-13T00:14:48.484 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Exxz9pCIKiVCSKY off=9246148 len=1008050 2024-09-13T00:14:48.486 INFO:tasks.workunit.client.0.smithi136.stdout:op 4082 completed, throughput=5MB/sec 2024-09-13T00:14:48.487 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-hZL4kMq8oGq91GO off=5172686 len=740255 2024-09-13T00:14:48.494 INFO:tasks.workunit.client.0.smithi136.stdout:op 4083 completed, throughput=5MB/sec 2024-09-13T00:14:48.494 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-hFmaTase-cQzxL3 off=1082462 len=1304223 2024-09-13T00:14:48.507 INFO:tasks.workunit.client.0.smithi136.stdout:op 4084 completed, throughput=5MB/sec 2024-09-13T00:14:49.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:14:48 smithi136 bash[75164]: cluster 2024-09-13T00:14:47.546412+0000 mgr.y (mgr.24661) 1130 : cluster [DBG] pgmap v1813: 105 pgs: 105 active+clean; 126 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 3.6 MiB/s rd, 2.1 MiB/s wr, 5 op/s 2024-09-13T00:14:49.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:14:48 smithi136 bash[76371]: cluster 2024-09-13T00:14:47.546412+0000 mgr.y (mgr.24661) 1130 : cluster [DBG] pgmap v1813: 105 pgs: 105 active+clean; 126 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 3.6 MiB/s rd, 2.1 MiB/s wr, 5 op/s 2024-09-13T00:14:49.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:14:48 smithi181 bash[67587]: cluster 2024-09-13T00:14:47.546412+0000 mgr.y (mgr.24661) 1130 : cluster [DBG] pgmap v1813: 105 pgs: 105 active+clean; 126 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 3.6 MiB/s rd, 2.1 MiB/s wr, 5 op/s 2024-09-13T00:14:49.507 INFO:tasks.workunit.client.0.smithi136.stdout: 781: throughput=5MB/sec pending data=0 2024-09-13T00:14:49.507 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-tbbqJHU0iTQRbO_ off=2515003 len=1240020 2024-09-13T00:14:49.510 INFO:tasks.workunit.client.0.smithi136.stdout:op 4085 completed, throughput=5MB/sec 2024-09-13T00:14:49.511 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-e-ZSQSh9Bu-rjKl off=11724094 len=53119 2024-09-13T00:14:49.516 INFO:tasks.workunit.client.0.smithi136.stdout:op 4086 completed, throughput=5MB/sec 2024-09-13T00:14:49.516 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-GfIW-2170MxoUFA off=2615662 len=2043092 2024-09-13T00:14:49.529 INFO:tasks.workunit.client.0.smithi136.stdout:op 4087 completed, throughput=5MB/sec 2024-09-13T00:14:49.529 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-NCW0VtcOb2vZBO7 off=2400772 len=1036585 2024-09-13T00:14:49.531 INFO:tasks.workunit.client.0.smithi136.stdout:op 4088 completed, throughput=5MB/sec 2024-09-13T00:14:49.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:14:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:14:49] "GET /metrics HTTP/1.1" 200 34140 "" "Prometheus/2.43.0" 2024-09-13T00:14:50.531 INFO:tasks.workunit.client.0.smithi136.stdout: 782: throughput=4.99MB/sec pending data=0 2024-09-13T00:14:50.532 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-edgnPmDsblEa2O6 off=225602 len=1646851 2024-09-13T00:14:50.543 INFO:tasks.workunit.client.0.smithi136.stdout:op 4089 completed, throughput=5MB/sec 2024-09-13T00:14:50.543 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-YjVTJNGS7d3nwld off=11843334 len=582308 2024-09-13T00:14:50.545 INFO:tasks.workunit.client.0.smithi136.stdout:op 4090 completed, throughput=5MB/sec 2024-09-13T00:14:50.545 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-4ZJvdB-81pSRmdx off=6752207 len=973396 2024-09-13T00:14:50.548 INFO:tasks.workunit.client.0.smithi136.stdout:op 4091 completed, throughput=5MB/sec 2024-09-13T00:14:50.548 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-m2c4BiHGW6DAX4Y off=2292878 len=1905058 2024-09-13T00:14:50.553 INFO:tasks.workunit.client.0.smithi136.stdout:op 4092 completed, throughput=5MB/sec 2024-09-13T00:14:50.553 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-0fCvSt_PKWIKN3f off=6111890 len=1150645 2024-09-13T00:14:50.562 INFO:tasks.workunit.client.0.smithi136.stdout:op 4093 completed, throughput=5MB/sec 2024-09-13T00:14:51.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:14:50 smithi136 bash[75164]: cluster 2024-09-13T00:14:49.547356+0000 mgr.y (mgr.24661) 1131 : cluster [DBG] pgmap v1814: 105 pgs: 105 active+clean; 126 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:14:51.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:14:50 smithi136 bash[76371]: cluster 2024-09-13T00:14:49.547356+0000 mgr.y (mgr.24661) 1131 : cluster [DBG] pgmap v1814: 105 pgs: 105 active+clean; 126 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:14:51.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:14:50 smithi181 bash[67587]: cluster 2024-09-13T00:14:49.547356+0000 mgr.y (mgr.24661) 1131 : cluster [DBG] pgmap v1814: 105 pgs: 105 active+clean; 126 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:14:51.562 INFO:tasks.workunit.client.0.smithi136.stdout: 783: throughput=5MB/sec pending data=0 2024-09-13T00:14:51.562 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-63FBoIJHze1WZy8 off=6154035 len=1258200 2024-09-13T00:14:51.565 INFO:tasks.workunit.client.0.smithi136.stdout:op 4094 completed, throughput=5MB/sec 2024-09-13T00:14:51.566 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-rhknVCZIaivCahP off=0 len=1489926 2024-09-13T00:14:51.569 INFO:tasks.workunit.client.0.smithi136.stdout:op 4095 completed, throughput=5MB/sec 2024-09-13T00:14:51.569 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-k00lTb0gTDqnJ5F off=2495404 len=439322 2024-09-13T00:14:51.576 INFO:tasks.workunit.client.0.smithi136.stdout:op 4096 completed, throughput=5MB/sec 2024-09-13T00:14:51.576 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-i059RDrESy9aeXS off=9295396 len=2078144 2024-09-13T00:14:51.580 INFO:tasks.workunit.client.0.smithi136.stdout:op 4097 completed, throughput=5MB/sec 2024-09-13T00:14:52.581 INFO:tasks.workunit.client.0.smithi136.stdout: 784: throughput=4.99MB/sec pending data=0 2024-09-13T00:14:52.581 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-UWoQ_depcW9Tj6J off=4348210 len=2059685 2024-09-13T00:14:52.587 INFO:tasks.workunit.client.0.smithi136.stdout:op 4098 completed, throughput=5MB/sec 2024-09-13T00:14:52.587 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-oqjIB5SQwJfxdYh off=9057526 len=1910381 2024-09-13T00:14:52.591 INFO:tasks.workunit.client.0.smithi136.stdout:op 4099 completed, throughput=5MB/sec 2024-09-13T00:14:52.591 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-h79abMC7bf43QvP off=13926730 len=1075599 2024-09-13T00:14:52.595 INFO:tasks.workunit.client.0.smithi136.stdout:op 4100 completed, throughput=5MB/sec 2024-09-13T00:14:52.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:14:52 smithi181 bash[95505]: ts=2024-09-13T00:14:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:14:53.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:14:52 smithi136 bash[75164]: cluster 2024-09-13T00:14:51.548343+0000 mgr.y (mgr.24661) 1132 : cluster [DBG] pgmap v1815: 105 pgs: 105 active+clean; 126 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:14:53.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:14:52 smithi136 bash[76371]: cluster 2024-09-13T00:14:51.548343+0000 mgr.y (mgr.24661) 1132 : cluster [DBG] pgmap v1815: 105 pgs: 105 active+clean; 126 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:14:53.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:14:52 smithi181 bash[67587]: cluster 2024-09-13T00:14:51.548343+0000 mgr.y (mgr.24661) 1132 : cluster [DBG] pgmap v1815: 105 pgs: 105 active+clean; 126 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:14:53.595 INFO:tasks.workunit.client.0.smithi136.stdout: 785: throughput=4.99MB/sec pending data=0 2024-09-13T00:14:53.595 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-gpGfBJtswpN4BR8 off=1932651 len=1395317 2024-09-13T00:14:53.599 INFO:tasks.workunit.client.0.smithi136.stdout:op 4101 completed, throughput=5MB/sec 2024-09-13T00:14:53.599 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-C6QSWd9IYtTS_Sr off=1388298 len=1757731 2024-09-13T00:14:53.611 INFO:tasks.workunit.client.0.smithi136.stdout:op 4102 completed, throughput=5MB/sec 2024-09-13T00:14:53.611 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-1xR1YJSTfpqu5kU off=4079957 len=1318709 2024-09-13T00:14:53.622 INFO:tasks.workunit.client.0.smithi136.stdout:op 4103 completed, throughput=5MB/sec 2024-09-13T00:14:53.622 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj--AHvPA1t8ES7DPY off=12408180 len=1598632 2024-09-13T00:14:53.633 INFO:tasks.workunit.client.0.smithi136.stdout:op 4104 completed, throughput=5MB/sec 2024-09-13T00:14:54.634 INFO:tasks.workunit.client.0.smithi136.stdout: 786: throughput=5MB/sec pending data=0 2024-09-13T00:14:54.634 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-v9jRkyaSjXsLSY9 off=17501898 len=1314184 2024-09-13T00:14:54.644 INFO:tasks.workunit.client.0.smithi136.stdout:op 4105 completed, throughput=5MB/sec 2024-09-13T00:14:54.644 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-hwgNhiBiJ3-NiMU off=5214736 len=896264 2024-09-13T00:14:54.653 INFO:tasks.workunit.client.0.smithi136.stdout:op 4106 completed, throughput=5MB/sec 2024-09-13T00:14:54.653 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-99_FOO0JXRtUb3a off=16270784 len=1791566 2024-09-13T00:14:54.665 INFO:tasks.workunit.client.0.smithi136.stdout:op 4107 completed, throughput=5MB/sec 2024-09-13T00:14:54.665 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-YSUw9Yqn37GmT3S off=837857 len=1257947 2024-09-13T00:14:54.668 INFO:tasks.workunit.client.0.smithi136.stdout:op 4108 completed, throughput=5MB/sec 2024-09-13T00:14:55.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:14:54 smithi136 bash[75164]: cluster 2024-09-13T00:14:53.549485+0000 mgr.y (mgr.24661) 1133 : cluster [DBG] pgmap v1816: 105 pgs: 105 active+clean; 126 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:14:55.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:14:54 smithi136 bash[75164]: audit 2024-09-13T00:14:53.794103+0000 mon.a (mon.0) 2223 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:14:55.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:14:54 smithi136 bash[76371]: cluster 2024-09-13T00:14:53.549485+0000 mgr.y (mgr.24661) 1133 : cluster [DBG] pgmap v1816: 105 pgs: 105 active+clean; 126 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:14:55.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:14:54 smithi136 bash[76371]: audit 2024-09-13T00:14:53.794103+0000 mon.a (mon.0) 2223 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:14:55.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:14:54 smithi181 bash[67587]: cluster 2024-09-13T00:14:53.549485+0000 mgr.y (mgr.24661) 1133 : cluster [DBG] pgmap v1816: 105 pgs: 105 active+clean; 126 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:14:55.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:14:54 smithi181 bash[67587]: audit 2024-09-13T00:14:53.794103+0000 mon.a (mon.0) 2223 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:14:55.668 INFO:tasks.workunit.client.0.smithi136.stdout: 787: throughput=5MB/sec pending data=0 2024-09-13T00:14:55.668 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-euSM8KF5mKrDQVR off=15266576 len=414390 2024-09-13T00:14:55.669 INFO:tasks.workunit.client.0.smithi136.stdout:op 4109 completed, throughput=5MB/sec 2024-09-13T00:14:55.669 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-VTbKQ8WjHycOogt off=207442 len=1597440 2024-09-13T00:14:55.680 INFO:tasks.workunit.client.0.smithi136.stdout:op 4110 completed, throughput=5MB/sec 2024-09-13T00:14:55.680 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj--mSPvJ0U2cUtFcS off=12770122 len=42763 2024-09-13T00:14:55.681 INFO:tasks.workunit.client.0.smithi136.stdout:op 4111 completed, throughput=5MB/sec 2024-09-13T00:14:55.681 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-F1gHhPc3kFBC_ef off=8250510 len=468514 2024-09-13T00:14:55.682 INFO:tasks.workunit.client.0.smithi136.stdout:op 4112 completed, throughput=5MB/sec 2024-09-13T00:14:55.682 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-xsKofZlutnaxpbF off=197488 len=2020443 2024-09-13T00:14:55.685 INFO:tasks.workunit.client.0.smithi136.stdout:op 4113 completed, throughput=5MB/sec 2024-09-13T00:14:56.686 INFO:tasks.workunit.client.0.smithi136.stdout: 788: throughput=4.99MB/sec pending data=0 2024-09-13T00:14:56.686 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-rmzFrSxh8gAJ2NM off=5368933 len=963375 2024-09-13T00:14:56.694 INFO:tasks.workunit.client.0.smithi136.stdout:op 4114 completed, throughput=5MB/sec 2024-09-13T00:14:56.694 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-veI7rdJoE7fUYfN off=1277846 len=522901 2024-09-13T00:14:56.700 INFO:tasks.workunit.client.0.smithi136.stdout:op 4115 completed, throughput=5MB/sec 2024-09-13T00:14:56.700 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Aj6BhpGBi2b1gzo off=201001 len=2047766 2024-09-13T00:14:56.703 INFO:tasks.workunit.client.0.smithi136.stdout:op 4116 completed, throughput=5MB/sec 2024-09-13T00:14:56.703 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-kidq2nU-uasepRI off=9598485 len=1542853 2024-09-13T00:14:56.716 INFO:tasks.workunit.client.0.smithi136.stdout:op 4117 completed, throughput=5MB/sec 2024-09-13T00:14:57.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:14:56 smithi136 bash[75164]: cluster 2024-09-13T00:14:55.550199+0000 mgr.y (mgr.24661) 1134 : cluster [DBG] pgmap v1817: 105 pgs: 105 active+clean; 126 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 2.3 MiB/s rd, 1.8 MiB/s wr, 3 op/s 2024-09-13T00:14:57.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:14:56 smithi136 bash[75164]: audit 2024-09-13T00:14:55.629507+0000 mon.a (mon.0) 2224 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:14:57.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:14:56 smithi136 bash[75164]: audit 2024-09-13T00:14:56.164611+0000 mon.a (mon.0) 2225 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:14:57.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:14:56 smithi136 bash[75164]: audit 2024-09-13T00:14:56.166041+0000 mon.a (mon.0) 2226 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:14:57.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:14:56 smithi136 bash[75164]: audit 2024-09-13T00:14:56.175601+0000 mon.a (mon.0) 2227 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:14:57.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:14:56 smithi136 bash[76371]: cluster 2024-09-13T00:14:55.550199+0000 mgr.y (mgr.24661) 1134 : cluster [DBG] pgmap v1817: 105 pgs: 105 active+clean; 126 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 2.3 MiB/s rd, 1.8 MiB/s wr, 3 op/s 2024-09-13T00:14:57.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:14:56 smithi136 bash[76371]: audit 2024-09-13T00:14:55.629507+0000 mon.a (mon.0) 2224 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:14:57.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:14:56 smithi136 bash[76371]: audit 2024-09-13T00:14:56.164611+0000 mon.a (mon.0) 2225 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:14:57.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:14:56 smithi136 bash[76371]: audit 2024-09-13T00:14:56.166041+0000 mon.a (mon.0) 2226 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:14:57.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:14:56 smithi136 bash[76371]: audit 2024-09-13T00:14:56.175601+0000 mon.a (mon.0) 2227 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:14:57.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:14:56 smithi181 bash[67587]: cluster 2024-09-13T00:14:55.550199+0000 mgr.y (mgr.24661) 1134 : cluster [DBG] pgmap v1817: 105 pgs: 105 active+clean; 126 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 2.3 MiB/s rd, 1.8 MiB/s wr, 3 op/s 2024-09-13T00:14:57.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:14:56 smithi181 bash[67587]: audit 2024-09-13T00:14:55.629507+0000 mon.a (mon.0) 2224 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:14:57.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:14:56 smithi181 bash[67587]: audit 2024-09-13T00:14:56.164611+0000 mon.a (mon.0) 2225 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:14:57.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:14:56 smithi181 bash[67587]: audit 2024-09-13T00:14:56.166041+0000 mon.a (mon.0) 2226 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:14:57.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:14:56 smithi181 bash[67587]: audit 2024-09-13T00:14:56.175601+0000 mon.a (mon.0) 2227 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:14:57.717 INFO:tasks.workunit.client.0.smithi136.stdout: 789: throughput=4.99MB/sec pending data=0 2024-09-13T00:14:57.717 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-eNZEsOoFzqhLRIw off=6399228 len=1990070 2024-09-13T00:14:57.721 INFO:tasks.workunit.client.0.smithi136.stdout:op 4118 completed, throughput=5MB/sec 2024-09-13T00:14:57.721 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-VMZ1uIGzSstotkf off=2204003 len=29242 2024-09-13T00:14:57.723 INFO:tasks.workunit.client.0.smithi136.stdout:op 4119 completed, throughput=5MB/sec 2024-09-13T00:14:57.723 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-7wqxkcfLlguKs4Q off=5636595 len=192587 2024-09-13T00:14:57.728 INFO:tasks.workunit.client.0.smithi136.stdout:op 4120 completed, throughput=5MB/sec 2024-09-13T00:14:57.728 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-7RjoGFxsHwT9HfM off=3929672 len=1579320 2024-09-13T00:14:57.738 INFO:tasks.workunit.client.0.smithi136.stdout:op 4121 completed, throughput=5MB/sec 2024-09-13T00:14:57.738 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-YCptYrfmW7mOSpj off=11678971 len=412367 2024-09-13T00:14:57.744 INFO:tasks.workunit.client.0.smithi136.stdout:op 4122 completed, throughput=5MB/sec 2024-09-13T00:14:57.744 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-nVYpwUDJxEsFLNq off=4061927 len=592399 2024-09-13T00:14:57.751 INFO:tasks.workunit.client.0.smithi136.stdout:op 4123 completed, throughput=5MB/sec 2024-09-13T00:14:57.751 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-E6SNZ6benec9zRx off=4693364 len=856562 2024-09-13T00:14:57.753 INFO:tasks.workunit.client.0.smithi136.stdout:op 4124 completed, throughput=5MB/sec 2024-09-13T00:14:58.754 INFO:tasks.workunit.client.0.smithi136.stdout: 790: throughput=4.99MB/sec pending data=0 2024-09-13T00:14:58.754 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-q7s9w-FDeCyOFXL off=1268943 len=336880 2024-09-13T00:14:58.759 INFO:tasks.workunit.client.0.smithi136.stdout:op 4125 completed, throughput=4.99MB/sec 2024-09-13T00:14:58.760 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-vltilreAgk_HETp off=2119293 len=362049 2024-09-13T00:14:58.766 INFO:tasks.workunit.client.0.smithi136.stdout:op 4126 completed, throughput=4.99MB/sec 2024-09-13T00:14:58.766 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-E3Vtkz7YVSxNU2y off=4280223 len=1291574 2024-09-13T00:14:58.774 INFO:tasks.workunit.client.0.smithi136.stdout:op 4127 completed, throughput=5MB/sec 2024-09-13T00:14:58.774 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-kExqDF65ZVOCZwj off=6416040 len=979587 2024-09-13T00:14:58.777 INFO:tasks.workunit.client.0.smithi136.stdout:op 4128 completed, throughput=5MB/sec 2024-09-13T00:14:58.777 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-5TWEOqtG9o67cMH off=8446558 len=251265 2024-09-13T00:14:58.779 INFO:tasks.workunit.client.0.smithi136.stdout:op 4129 completed, throughput=5MB/sec 2024-09-13T00:14:58.779 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-9ONFWcw4SyPKaOo off=18219311 len=654720 2024-09-13T00:14:58.786 INFO:tasks.workunit.client.0.smithi136.stdout:op 4130 completed, throughput=5MB/sec 2024-09-13T00:14:58.786 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-jtN8TBdnerEDCXP off=417744 len=1661703 2024-09-13T00:14:58.790 INFO:tasks.workunit.client.0.smithi136.stdout:op 4131 completed, throughput=5MB/sec 2024-09-13T00:14:59.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:14:58 smithi136 bash[75164]: cluster 2024-09-13T00:14:57.551255+0000 mgr.y (mgr.24661) 1135 : cluster [DBG] pgmap v1818: 105 pgs: 105 active+clean; 126 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:14:59.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:14:58 smithi136 bash[76371]: cluster 2024-09-13T00:14:57.551255+0000 mgr.y (mgr.24661) 1135 : cluster [DBG] pgmap v1818: 105 pgs: 105 active+clean; 126 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:14:59.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:14:58 smithi181 bash[67587]: cluster 2024-09-13T00:14:57.551255+0000 mgr.y (mgr.24661) 1135 : cluster [DBG] pgmap v1818: 105 pgs: 105 active+clean; 126 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:14:59.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:14:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:14:59] "GET /metrics HTTP/1.1" 200 34143 "" "Prometheus/2.43.0" 2024-09-13T00:14:59.791 INFO:tasks.workunit.client.0.smithi136.stdout: 791: throughput=4.99MB/sec pending data=0 2024-09-13T00:14:59.791 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-O29fTbDDvH7Ufkt off=16890527 len=1488380 2024-09-13T00:14:59.802 INFO:tasks.workunit.client.0.smithi136.stdout:op 4132 completed, throughput=5MB/sec 2024-09-13T00:14:59.802 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-PLbuimiT8xqOckM off=14709770 len=872088 2024-09-13T00:14:59.805 INFO:tasks.workunit.client.0.smithi136.stdout:op 4133 completed, throughput=5MB/sec 2024-09-13T00:14:59.805 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-pzkUO4DwzToFmne off=107107 len=1389809 2024-09-13T00:14:59.809 INFO:tasks.workunit.client.0.smithi136.stdout:op 4134 completed, throughput=5MB/sec 2024-09-13T00:14:59.809 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ZXpG1lQpXjXTUn_ off=768667 len=466453 2024-09-13T00:14:59.810 INFO:tasks.workunit.client.0.smithi136.stdout:op 4135 completed, throughput=5MB/sec 2024-09-13T00:14:59.811 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-WAr6La3ep_eh_U3 off=4827472 len=51301 2024-09-13T00:14:59.815 INFO:tasks.workunit.client.0.smithi136.stdout:op 4136 completed, throughput=5MB/sec 2024-09-13T00:14:59.815 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ojbcEVm4rpWJSM1 off=54914 len=379477 2024-09-13T00:14:59.821 INFO:tasks.workunit.client.0.smithi136.stdout:op 4137 completed, throughput=5MB/sec 2024-09-13T00:14:59.821 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-IVESCAS-NGoJWtf off=6489726 len=1580558 2024-09-13T00:14:59.833 INFO:tasks.workunit.client.0.smithi136.stdout:op 4138 completed, throughput=5MB/sec 2024-09-13T00:15:00.834 INFO:tasks.workunit.client.0.smithi136.stdout: 792: throughput=5MB/sec pending data=0 2024-09-13T00:15:00.834 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-x2Qylz4Y8tmMs8C off=260428 len=1466675 2024-09-13T00:15:00.838 INFO:tasks.workunit.client.0.smithi136.stdout:op 4139 completed, throughput=5MB/sec 2024-09-13T00:15:00.838 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-oIKLWUIC6ihi3wM off=16827451 len=306190 2024-09-13T00:15:00.839 INFO:tasks.workunit.client.0.smithi136.stdout:op 4140 completed, throughput=5MB/sec 2024-09-13T00:15:00.839 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-HcTwsPTIl0Yf3uM off=15632375 len=1067372 2024-09-13T00:15:00.848 INFO:tasks.workunit.client.0.smithi136.stdout:op 4141 completed, throughput=5MB/sec 2024-09-13T00:15:00.848 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-aexuQZrCHDf6f1_ off=3219202 len=1018990 2024-09-13T00:15:00.852 INFO:tasks.workunit.client.0.smithi136.stdout:op 4142 completed, throughput=5MB/sec 2024-09-13T00:15:00.852 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-RatCT3z0B8Nct5U off=6196583 len=703816 2024-09-13T00:15:00.854 INFO:tasks.workunit.client.0.smithi136.stdout:op 4143 completed, throughput=5MB/sec 2024-09-13T00:15:01.007 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:15:00 smithi136 bash[76371]: cluster 2024-09-13T00:14:59.552142+0000 mgr.y (mgr.24661) 1136 : cluster [DBG] pgmap v1819: 105 pgs: 105 active+clean; 126 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.1 MiB/s wr, 3 op/s 2024-09-13T00:15:01.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:15:00 smithi136 bash[75164]: cluster 2024-09-13T00:14:59.552142+0000 mgr.y (mgr.24661) 1136 : cluster [DBG] pgmap v1819: 105 pgs: 105 active+clean; 126 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.1 MiB/s wr, 3 op/s 2024-09-13T00:15:01.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:15:00 smithi181 bash[67587]: cluster 2024-09-13T00:14:59.552142+0000 mgr.y (mgr.24661) 1136 : cluster [DBG] pgmap v1819: 105 pgs: 105 active+clean; 126 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.1 MiB/s wr, 3 op/s 2024-09-13T00:15:01.855 INFO:tasks.workunit.client.0.smithi136.stdout: 793: throughput=4.99MB/sec pending data=0 2024-09-13T00:15:01.855 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-0LNNQbEYf7td_Mv off=1395302 len=1668473 2024-09-13T00:15:01.859 INFO:tasks.workunit.client.0.smithi136.stdout:op 4144 completed, throughput=5MB/sec 2024-09-13T00:15:01.859 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-zuP7Vy6d9Il5FaG off=2817297 len=609483 2024-09-13T00:15:01.866 INFO:tasks.workunit.client.0.smithi136.stdout:op 4145 completed, throughput=5MB/sec 2024-09-13T00:15:01.866 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-KemgNXLQyOXs2UR off=2081133 len=810904 2024-09-13T00:15:01.874 INFO:tasks.workunit.client.0.smithi136.stdout:op 4146 completed, throughput=5MB/sec 2024-09-13T00:15:01.874 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ohyuORRuvkWvrKq off=10569503 len=299174 2024-09-13T00:15:01.876 INFO:tasks.workunit.client.0.smithi136.stdout:op 4147 completed, throughput=5MB/sec 2024-09-13T00:15:01.876 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-XLXxO0Jwv5vSs0s off=3796262 len=593532 2024-09-13T00:15:01.882 INFO:tasks.workunit.client.0.smithi136.stdout:op 4148 completed, throughput=5MB/sec 2024-09-13T00:15:01.883 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-kiDfJmHRIsZFF2s off=287884 len=1613165 2024-09-13T00:15:01.897 INFO:tasks.workunit.client.0.smithi136.stdout:op 4149 completed, throughput=5MB/sec 2024-09-13T00:15:02.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:15:02 smithi181 bash[95505]: ts=2024-09-13T00:15:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:15:02.897 INFO:tasks.workunit.client.0.smithi136.stdout: 794: throughput=4.99MB/sec pending data=0 2024-09-13T00:15:02.897 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-0vaozphrU-zHwsa off=3266120 len=524735 2024-09-13T00:15:02.900 INFO:tasks.workunit.client.0.smithi136.stdout:op 4150 completed, throughput=4.99MB/sec 2024-09-13T00:15:02.900 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-64Ltx-fOiGhhCnv off=5572409 len=641491 2024-09-13T00:15:02.907 INFO:tasks.workunit.client.0.smithi136.stdout:op 4151 completed, throughput=5MB/sec 2024-09-13T00:15:02.907 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-xltYhJSOJ-OJwKr off=1586379 len=1776889 2024-09-13T00:15:02.923 INFO:tasks.workunit.client.0.smithi136.stdout:op 4152 completed, throughput=5MB/sec 2024-09-13T00:15:02.923 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-l_vJeSBTZCpKmLA off=15678236 len=1238190 2024-09-13T00:15:02.927 INFO:tasks.workunit.client.0.smithi136.stdout:op 4153 completed, throughput=5MB/sec 2024-09-13T00:15:02.927 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-m581X4nNsQkD_PH off=8148693 len=1003339 2024-09-13T00:15:02.936 INFO:tasks.workunit.client.0.smithi136.stdout:op 4154 completed, throughput=5MB/sec 2024-09-13T00:15:03.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:15:02 smithi136 bash[75164]: cluster 2024-09-13T00:15:01.553151+0000 mgr.y (mgr.24661) 1137 : cluster [DBG] pgmap v1820: 105 pgs: 105 active+clean; 126 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:15:03.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:15:02 smithi136 bash[76371]: cluster 2024-09-13T00:15:01.553151+0000 mgr.y (mgr.24661) 1137 : cluster [DBG] pgmap v1820: 105 pgs: 105 active+clean; 126 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:15:03.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:15:02 smithi181 bash[67587]: cluster 2024-09-13T00:15:01.553151+0000 mgr.y (mgr.24661) 1137 : cluster [DBG] pgmap v1820: 105 pgs: 105 active+clean; 126 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:15:03.936 INFO:tasks.workunit.client.0.smithi136.stdout: 795: throughput=4.99MB/sec pending data=0 2024-09-13T00:15:03.936 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-modWPc2kkFC4taa off=747468 len=122814 2024-09-13T00:15:03.938 INFO:tasks.workunit.client.0.smithi136.stdout:op 4155 completed, throughput=4.99MB/sec 2024-09-13T00:15:03.938 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-aLP3SuA26-pUIat off=0 len=1279779 2024-09-13T00:15:03.952 INFO:tasks.workunit.client.0.smithi136.stdout:op 4156 completed, throughput=5MB/sec 2024-09-13T00:15:03.953 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-1yqLw5zqmuq0o3Y off=3095711 len=1633163 2024-09-13T00:15:03.953 INFO:tasks.workunit.client.0.smithi136.stdout:op 4157 completed, throughput=5MB/sec 2024-09-13T00:15:03.953 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-HboWJr0oQGixdCB off=3191518 len=610266 2024-09-13T00:15:03.955 INFO:tasks.workunit.client.0.smithi136.stdout:op 4158 completed, throughput=5MB/sec 2024-09-13T00:15:03.956 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-L1ntSVFvFPalC2k off=401003 len=1104839 2024-09-13T00:15:03.958 INFO:tasks.workunit.client.0.smithi136.stdout:op 4159 completed, throughput=5MB/sec 2024-09-13T00:15:03.958 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-KryyLW33Q3E1ofD off=9218785 len=1443325 2024-09-13T00:15:03.961 INFO:tasks.workunit.client.0.smithi136.stdout:op 4160 completed, throughput=5MB/sec 2024-09-13T00:15:04.961 INFO:tasks.workunit.client.0.smithi136.stdout: 796: throughput=5MB/sec pending data=0 2024-09-13T00:15:04.961 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Envv6D5Lg7ADl4W off=1380567 len=2067879 2024-09-13T00:15:04.968 INFO:tasks.workunit.client.0.smithi136.stdout:op 4161 completed, throughput=5MB/sec 2024-09-13T00:15:04.968 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-hRlZnyUdlczeApt off=5058935 len=1732741 2024-09-13T00:15:04.972 INFO:tasks.workunit.client.0.smithi136.stdout:op 4162 completed, throughput=5MB/sec 2024-09-13T00:15:04.973 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-oPVsdBOUp3VlWQJ off=492790 len=979875 2024-09-13T00:15:04.975 INFO:tasks.workunit.client.0.smithi136.stdout:op 4163 completed, throughput=5MB/sec 2024-09-13T00:15:05.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:15:04 smithi136 bash[75164]: cluster 2024-09-13T00:15:03.554193+0000 mgr.y (mgr.24661) 1138 : cluster [DBG] pgmap v1821: 105 pgs: 105 active+clean; 126 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:15:05.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:15:04 smithi136 bash[76371]: cluster 2024-09-13T00:15:03.554193+0000 mgr.y (mgr.24661) 1138 : cluster [DBG] pgmap v1821: 105 pgs: 105 active+clean; 126 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:15:05.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:15:04 smithi181 bash[67587]: cluster 2024-09-13T00:15:03.554193+0000 mgr.y (mgr.24661) 1138 : cluster [DBG] pgmap v1821: 105 pgs: 105 active+clean; 126 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:15:05.976 INFO:tasks.workunit.client.0.smithi136.stdout: 797: throughput=4.99MB/sec pending data=0 2024-09-13T00:15:05.976 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-lv25mK7j40z1ESm off=1586335 len=1166135 2024-09-13T00:15:05.978 INFO:tasks.workunit.client.0.smithi136.stdout:op 4164 completed, throughput=5MB/sec 2024-09-13T00:15:05.979 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-6sEZMuU0gKdfxoP off=7316840 len=1120600 2024-09-13T00:15:05.988 INFO:tasks.workunit.client.0.smithi136.stdout:op 4165 completed, throughput=5MB/sec 2024-09-13T00:15:05.988 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-rbyS-pKvKxZ8ZSR off=12174268 len=1141331 2024-09-13T00:15:05.991 INFO:tasks.workunit.client.0.smithi136.stdout:op 4166 completed, throughput=5MB/sec 2024-09-13T00:15:05.992 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-vmPbGgQvbZvcvnE off=1476225 len=558237 2024-09-13T00:15:05.993 INFO:tasks.workunit.client.0.smithi136.stdout:op 4167 completed, throughput=5MB/sec 2024-09-13T00:15:05.993 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-wtM9AL6Nm8jamyr off=11634233 len=792816 2024-09-13T00:15:06.001 INFO:tasks.workunit.client.0.smithi136.stdout:op 4168 completed, throughput=5MB/sec 2024-09-13T00:15:06.001 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-sm9em3bhhtNQo55 off=311477 len=790268 2024-09-13T00:15:06.003 INFO:tasks.workunit.client.0.smithi136.stdout:op 4169 completed, throughput=5MB/sec 2024-09-13T00:15:07.004 INFO:tasks.workunit.client.0.smithi136.stdout: 798: throughput=4.99MB/sec pending data=0 2024-09-13T00:15:07.004 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-FKxLmmdho2AqZ6O off=17011931 len=855492 2024-09-13T00:15:07.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:15:06 smithi136 bash[75164]: cluster 2024-09-13T00:15:05.554910+0000 mgr.y (mgr.24661) 1139 : cluster [DBG] pgmap v1822: 105 pgs: 105 active+clean; 126 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:15:07.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:15:06 smithi136 bash[76371]: cluster 2024-09-13T00:15:05.554910+0000 mgr.y (mgr.24661) 1139 : cluster [DBG] pgmap v1822: 105 pgs: 105 active+clean; 126 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:15:07.012 INFO:tasks.workunit.client.0.smithi136.stdout:op 4170 completed, throughput=5MB/sec 2024-09-13T00:15:07.012 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-oNykxhsuLsg6uuk off=11107360 len=860196 2024-09-13T00:15:07.015 INFO:tasks.workunit.client.0.smithi136.stdout:op 4171 completed, throughput=5MB/sec 2024-09-13T00:15:07.015 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-iIxFdYtAzvaU98E off=2332934 len=1218384 2024-09-13T00:15:07.025 INFO:tasks.workunit.client.0.smithi136.stdout:op 4172 completed, throughput=5MB/sec 2024-09-13T00:15:07.025 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-GtdSZ_B48AvaZa1 off=1862950 len=964573 2024-09-13T00:15:07.034 INFO:tasks.workunit.client.0.smithi136.stdout:op 4173 completed, throughput=5MB/sec 2024-09-13T00:15:07.034 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-EhAWswJv4l1ha2a off=12283083 len=1779568 2024-09-13T00:15:07.038 INFO:tasks.workunit.client.0.smithi136.stdout:op 4174 completed, throughput=5MB/sec 2024-09-13T00:15:07.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:15:06 smithi181 bash[67587]: cluster 2024-09-13T00:15:05.554910+0000 mgr.y (mgr.24661) 1139 : cluster [DBG] pgmap v1822: 105 pgs: 105 active+clean; 126 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:15:08.040 INFO:tasks.workunit.client.0.smithi136.stdout: 799: throughput=4.99MB/sec pending data=0 2024-09-13T00:15:08.040 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_eecqh3TjlEp8O2 off=2683102 len=964213 2024-09-13T00:15:08.050 INFO:tasks.workunit.client.0.smithi136.stdout:op 4175 completed, throughput=5MB/sec 2024-09-13T00:15:08.050 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-NL-1nR9z7ddvKWP off=6945479 len=1848547 2024-09-13T00:15:08.062 INFO:tasks.workunit.client.0.smithi136.stdout:op 4176 completed, throughput=5MB/sec 2024-09-13T00:15:08.062 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-oXx9OLq9Jdqrq70 off=1810234 len=1561265 2024-09-13T00:15:08.066 INFO:tasks.workunit.client.0.smithi136.stdout:op 4177 completed, throughput=5MB/sec 2024-09-13T00:15:08.066 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-5hn0zziq6-0XWSg off=11492303 len=96944 2024-09-13T00:15:08.067 INFO:tasks.workunit.client.0.smithi136.stdout:op 4178 completed, throughput=5MB/sec 2024-09-13T00:15:08.067 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-5mZWtIH2IqfjN97 off=14382 len=2008834 2024-09-13T00:15:08.080 INFO:tasks.workunit.client.0.smithi136.stdout:op 4179 completed, throughput=5MB/sec 2024-09-13T00:15:09.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:15:08 smithi136 bash[75164]: cluster 2024-09-13T00:15:07.556047+0000 mgr.y (mgr.24661) 1140 : cluster [DBG] pgmap v1823: 105 pgs: 105 active+clean; 126 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:15:09.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:15:08 smithi136 bash[76371]: cluster 2024-09-13T00:15:07.556047+0000 mgr.y (mgr.24661) 1140 : cluster [DBG] pgmap v1823: 105 pgs: 105 active+clean; 126 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:15:09.080 INFO:tasks.workunit.client.0.smithi136.stdout: 800: throughput=5MB/sec pending data=0 2024-09-13T00:15:09.080 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-5vFIuEszB4kTd0I off=5213191 len=1725240 2024-09-13T00:15:09.093 INFO:tasks.workunit.client.0.smithi136.stdout:op 4180 completed, throughput=5MB/sec 2024-09-13T00:15:09.093 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-GjovfeqBU3V0Ib4 off=6251351 len=1923192 2024-09-13T00:15:09.097 INFO:tasks.workunit.client.0.smithi136.stdout:op 4181 completed, throughput=5MB/sec 2024-09-13T00:15:09.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:15:08 smithi181 bash[67587]: cluster 2024-09-13T00:15:07.556047+0000 mgr.y (mgr.24661) 1140 : cluster [DBG] pgmap v1823: 105 pgs: 105 active+clean; 126 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:15:09.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:15:09 smithi136 bash[75164]: audit 2024-09-13T00:15:08.794521+0000 mon.a (mon.0) 2228 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:15:09.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:15:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:15:09] "GET /metrics HTTP/1.1" 200 34142 "" "Prometheus/2.43.0" 2024-09-13T00:15:09.758 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:15:09 smithi136 bash[76371]: audit 2024-09-13T00:15:08.794521+0000 mon.a (mon.0) 2228 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:15:10.098 INFO:tasks.workunit.client.0.smithi136.stdout: 801: throughput=4.99MB/sec pending data=0 2024-09-13T00:15:10.098 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-h4oKpzBpdNi15Ru off=14699789 len=1004692 2024-09-13T00:15:10.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:15:09 smithi181 bash[67587]: audit 2024-09-13T00:15:08.794521+0000 mon.a (mon.0) 2228 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:15:10.101 INFO:tasks.workunit.client.0.smithi136.stdout:op 4182 completed, throughput=5MB/sec 2024-09-13T00:15:10.101 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-gedR78EFP_QRvLA off=13450898 len=431235 2024-09-13T00:15:10.107 INFO:tasks.workunit.client.0.smithi136.stdout:op 4183 completed, throughput=5MB/sec 2024-09-13T00:15:10.108 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-4xC5T41AGFlno8O off=3076363 len=316753 2024-09-13T00:15:10.109 INFO:tasks.workunit.client.0.smithi136.stdout:op 4184 completed, throughput=5MB/sec 2024-09-13T00:15:10.109 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-YaBsstYDRP0NHLl off=1226852 len=677334 2024-09-13T00:15:10.112 INFO:tasks.workunit.client.0.smithi136.stdout:op 4185 completed, throughput=5MB/sec 2024-09-13T00:15:10.112 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-vJPwHbCCLOUiM0i off=2026897 len=258097 2024-09-13T00:15:10.114 INFO:tasks.workunit.client.0.smithi136.stdout:op 4186 completed, throughput=5MB/sec 2024-09-13T00:15:10.114 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-0-W66Al6oAEYyjp off=1851090 len=620325 2024-09-13T00:15:10.116 INFO:tasks.workunit.client.0.smithi136.stdout:op 4187 completed, throughput=5MB/sec 2024-09-13T00:15:10.116 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-k7aaWRW0yY021Th off=2413012 len=1912020 2024-09-13T00:15:10.120 INFO:tasks.workunit.client.0.smithi136.stdout:op 4188 completed, throughput=5MB/sec 2024-09-13T00:15:11.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:15:10 smithi136 bash[75164]: cluster 2024-09-13T00:15:09.556965+0000 mgr.y (mgr.24661) 1141 : cluster [DBG] pgmap v1824: 105 pgs: 105 active+clean; 126 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:15:11.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:15:10 smithi136 bash[76371]: cluster 2024-09-13T00:15:09.556965+0000 mgr.y (mgr.24661) 1141 : cluster [DBG] pgmap v1824: 105 pgs: 105 active+clean; 126 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:15:11.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:15:10 smithi181 bash[67587]: cluster 2024-09-13T00:15:09.556965+0000 mgr.y (mgr.24661) 1141 : cluster [DBG] pgmap v1824: 105 pgs: 105 active+clean; 126 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:15:11.120 INFO:tasks.workunit.client.0.smithi136.stdout: 802: throughput=4.99MB/sec pending data=0 2024-09-13T00:15:11.120 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj--OXKrbtJ1373tbU off=4788545 len=363406 2024-09-13T00:15:11.122 INFO:tasks.workunit.client.0.smithi136.stdout:op 4189 completed, throughput=4.99MB/sec 2024-09-13T00:15:11.122 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-gBFm4nVEOQJTUnH off=7266479 len=1243532 2024-09-13T00:15:11.127 INFO:tasks.workunit.client.0.smithi136.stdout:op 4190 completed, throughput=5MB/sec 2024-09-13T00:15:11.127 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-UPT_854PKayGhjP off=2152316 len=224147 2024-09-13T00:15:11.129 INFO:tasks.workunit.client.0.smithi136.stdout:op 4191 completed, throughput=5MB/sec 2024-09-13T00:15:11.129 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ztqgouKVWUUYDJc off=23423995 len=1166953 2024-09-13T00:15:11.132 INFO:tasks.workunit.client.0.smithi136.stdout:op 4192 completed, throughput=5MB/sec 2024-09-13T00:15:11.132 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-MvCMVYX9RVEsZuT off=2172401 len=1425135 2024-09-13T00:15:11.136 INFO:tasks.workunit.client.0.smithi136.stdout:op 4193 completed, throughput=5MB/sec 2024-09-13T00:15:11.136 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2Odl6s4fNF1u1bp off=711011 len=1524087 2024-09-13T00:15:11.139 INFO:tasks.workunit.client.0.smithi136.stdout:op 4194 completed, throughput=5MB/sec 2024-09-13T00:15:12.141 INFO:tasks.workunit.client.0.smithi136.stdout: 803: throughput=4.99MB/sec pending data=0 2024-09-13T00:15:12.141 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-VNfrTo8_qFx7dad off=16788206 len=1756623 2024-09-13T00:15:12.144 INFO:tasks.workunit.client.0.smithi136.stdout:op 4195 completed, throughput=5MB/sec 2024-09-13T00:15:12.144 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-GVynPhzk-l95hSX off=701495 len=29898 2024-09-13T00:15:12.148 INFO:tasks.workunit.client.0.smithi136.stdout:op 4196 completed, throughput=5MB/sec 2024-09-13T00:15:12.148 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-hyiIUgmeJXVwSuG off=17003251 len=254185 2024-09-13T00:15:12.150 INFO:tasks.workunit.client.0.smithi136.stdout:op 4197 completed, throughput=5MB/sec 2024-09-13T00:15:12.150 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-tTNRTZ42Ejgor1e off=17900421 len=195174 2024-09-13T00:15:12.155 INFO:tasks.workunit.client.0.smithi136.stdout:op 4198 completed, throughput=5MB/sec 2024-09-13T00:15:12.155 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-N7VUR2iAXPqvx3S off=1024744 len=405814 2024-09-13T00:15:12.156 INFO:tasks.workunit.client.0.smithi136.stdout:op 4199 completed, throughput=5MB/sec 2024-09-13T00:15:12.157 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-CzUT7aX9Nr2xhnr off=4910064 len=1582682 2024-09-13T00:15:12.160 INFO:tasks.workunit.client.0.smithi136.stdout:op 4200 completed, throughput=5MB/sec 2024-09-13T00:15:12.161 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-5DYPVqR-5126pkx off=5060672 len=1577000 2024-09-13T00:15:12.175 INFO:tasks.workunit.client.0.smithi136.stdout:op 4201 completed, throughput=5MB/sec 2024-09-13T00:15:12.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:15:12 smithi181 bash[95505]: ts=2024-09-13T00:15:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:15:13.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:15:12 smithi136 bash[75164]: cluster 2024-09-13T00:15:11.557974+0000 mgr.y (mgr.24661) 1142 : cluster [DBG] pgmap v1825: 105 pgs: 105 active+clean; 126 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:15:13.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:15:12 smithi136 bash[76371]: cluster 2024-09-13T00:15:11.557974+0000 mgr.y (mgr.24661) 1142 : cluster [DBG] pgmap v1825: 105 pgs: 105 active+clean; 126 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:15:13.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:15:12 smithi181 bash[67587]: cluster 2024-09-13T00:15:11.557974+0000 mgr.y (mgr.24661) 1142 : cluster [DBG] pgmap v1825: 105 pgs: 105 active+clean; 126 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:15:13.175 INFO:tasks.workunit.client.0.smithi136.stdout: 804: throughput=5MB/sec pending data=0 2024-09-13T00:15:13.175 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-qYUk0zY2k0Rg9bF off=2588963 len=1635160 2024-09-13T00:15:13.187 INFO:tasks.workunit.client.0.smithi136.stdout:op 4202 completed, throughput=5MB/sec 2024-09-13T00:15:13.188 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-a0Rn9Vdwgcnx8XZ off=1677296 len=278028 2024-09-13T00:15:13.189 INFO:tasks.workunit.client.0.smithi136.stdout:op 4203 completed, throughput=5MB/sec 2024-09-13T00:15:13.189 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-yBcdmyLZn4OfxCW off=9585248 len=591672 2024-09-13T00:15:13.191 INFO:tasks.workunit.client.0.smithi136.stdout:op 4204 completed, throughput=5MB/sec 2024-09-13T00:15:13.191 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-QEjiWFpEbt4-q8y off=4770551 len=630516 2024-09-13T00:15:13.193 INFO:tasks.workunit.client.0.smithi136.stdout:op 4205 completed, throughput=5MB/sec 2024-09-13T00:15:13.193 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-zBqRQ90Xvm-_BEm off=1648772 len=1217666 2024-09-13T00:15:13.203 INFO:tasks.workunit.client.0.smithi136.stdout:op 4206 completed, throughput=5MB/sec 2024-09-13T00:15:13.203 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2wtRPuRif54nAu7 off=3565281 len=824089 2024-09-13T00:15:13.206 INFO:tasks.workunit.client.0.smithi136.stdout:op 4207 completed, throughput=5MB/sec 2024-09-13T00:15:14.206 INFO:tasks.workunit.client.0.smithi136.stdout: 805: throughput=4.99MB/sec pending data=0 2024-09-13T00:15:14.206 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-avcTvphN4nDV3Lw off=5118404 len=169513 2024-09-13T00:15:14.211 INFO:tasks.workunit.client.0.smithi136.stdout:op 4208 completed, throughput=5MB/sec 2024-09-13T00:15:14.211 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-x_3xScAT3aBMmoe off=15929288 len=2086425 2024-09-13T00:15:14.225 INFO:tasks.workunit.client.0.smithi136.stdout:op 4209 completed, throughput=5MB/sec 2024-09-13T00:15:14.225 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Ut5KorIHaaF27NM off=11716511 len=38054 2024-09-13T00:15:14.226 INFO:tasks.workunit.client.0.smithi136.stdout:op 4210 completed, throughput=5MB/sec 2024-09-13T00:15:14.226 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-YaBsstYDRP0NHLl off=11347888 len=142005 2024-09-13T00:15:14.231 INFO:tasks.workunit.client.0.smithi136.stdout:op 4211 completed, throughput=5MB/sec 2024-09-13T00:15:14.231 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_3ZAUz0wvYkXPNc off=9226866 len=811293 2024-09-13T00:15:14.238 INFO:tasks.workunit.client.0.smithi136.stdout:op 4212 completed, throughput=5MB/sec 2024-09-13T00:15:14.238 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-5qJMz69uW7QVcaz off=6141512 len=763828 2024-09-13T00:15:14.239 INFO:tasks.workunit.client.0.smithi136.stdout:op 4213 completed, throughput=5MB/sec 2024-09-13T00:15:14.240 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-yYFAlaclEh60mqL off=7760356 len=306170 2024-09-13T00:15:14.245 INFO:tasks.workunit.client.0.smithi136.stdout:op 4214 completed, throughput=5MB/sec 2024-09-13T00:15:14.245 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-H-8YB7IL_9K4EXX off=907044 len=2023698 2024-09-13T00:15:14.257 INFO:tasks.workunit.client.0.smithi136.stdout:op 4215 completed, throughput=5MB/sec 2024-09-13T00:15:15.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:15:14 smithi136 bash[75164]: cluster 2024-09-13T00:15:13.559040+0000 mgr.y (mgr.24661) 1143 : cluster [DBG] pgmap v1826: 105 pgs: 105 active+clean; 126 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 3.6 MiB/s rd, 2.1 MiB/s wr, 5 op/s 2024-09-13T00:15:15.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:15:14 smithi136 bash[76371]: cluster 2024-09-13T00:15:13.559040+0000 mgr.y (mgr.24661) 1143 : cluster [DBG] pgmap v1826: 105 pgs: 105 active+clean; 126 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 3.6 MiB/s rd, 2.1 MiB/s wr, 5 op/s 2024-09-13T00:15:15.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:15:14 smithi181 bash[67587]: cluster 2024-09-13T00:15:13.559040+0000 mgr.y (mgr.24661) 1143 : cluster [DBG] pgmap v1826: 105 pgs: 105 active+clean; 126 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 3.6 MiB/s rd, 2.1 MiB/s wr, 5 op/s 2024-09-13T00:15:15.257 INFO:tasks.workunit.client.0.smithi136.stdout: 806: throughput=5MB/sec pending data=0 2024-09-13T00:15:15.257 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-xaJOSJyayRVNuzM off=12707889 len=1825402 2024-09-13T00:15:15.264 INFO:tasks.workunit.client.0.smithi136.stdout:op 4216 completed, throughput=5MB/sec 2024-09-13T00:15:15.264 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_LwenDuoweD2xpj off=19925217 len=1106417 2024-09-13T00:15:15.274 INFO:tasks.workunit.client.0.smithi136.stdout:op 4217 completed, throughput=5MB/sec 2024-09-13T00:15:15.274 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-g1aTZ0YZa8AWQ43 off=18076664 len=674074 2024-09-13T00:15:15.276 INFO:tasks.workunit.client.0.smithi136.stdout:op 4218 completed, throughput=5MB/sec 2024-09-13T00:15:15.276 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-33ugmjuzqilophS off=310831 len=1075244 2024-09-13T00:15:15.279 INFO:tasks.workunit.client.0.smithi136.stdout:op 4219 completed, throughput=5MB/sec 2024-09-13T00:15:16.279 INFO:tasks.workunit.client.0.smithi136.stdout: 807: throughput=5MB/sec pending data=0 2024-09-13T00:15:16.279 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-T-4gGN-p1-kWWlQ off=964417 len=1231471 2024-09-13T00:15:16.289 INFO:tasks.workunit.client.0.smithi136.stdout:op 4220 completed, throughput=5MB/sec 2024-09-13T00:15:16.289 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Oj9BEcHaUGqcQXl off=6083858 len=875205 2024-09-13T00:15:16.291 INFO:tasks.workunit.client.0.smithi136.stdout:op 4221 completed, throughput=5MB/sec 2024-09-13T00:15:16.291 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-AEN8PcCQ8EwBl_k off=14786098 len=177046 2024-09-13T00:15:16.293 INFO:tasks.workunit.client.0.smithi136.stdout:op 4222 completed, throughput=5MB/sec 2024-09-13T00:15:16.293 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-3HQyUFbPEYGrUjL off=12052676 len=865975 2024-09-13T00:15:16.295 INFO:tasks.workunit.client.0.smithi136.stdout:op 4223 completed, throughput=5MB/sec 2024-09-13T00:15:16.295 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-P4tQOwFIVbZiVVF off=5599239 len=1766090 2024-09-13T00:15:16.301 INFO:tasks.workunit.client.0.smithi136.stdout:op 4224 completed, throughput=5MB/sec 2024-09-13T00:15:17.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:15:16 smithi136 bash[75164]: cluster 2024-09-13T00:15:15.559761+0000 mgr.y (mgr.24661) 1144 : cluster [DBG] pgmap v1827: 105 pgs: 105 active+clean; 126 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 3.0 MiB/s rd, 1.6 MiB/s wr, 4 op/s 2024-09-13T00:15:17.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:15:16 smithi136 bash[76371]: cluster 2024-09-13T00:15:15.559761+0000 mgr.y (mgr.24661) 1144 : cluster [DBG] pgmap v1827: 105 pgs: 105 active+clean; 126 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 3.0 MiB/s rd, 1.6 MiB/s wr, 4 op/s 2024-09-13T00:15:17.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:15:16 smithi181 bash[67587]: cluster 2024-09-13T00:15:15.559761+0000 mgr.y (mgr.24661) 1144 : cluster [DBG] pgmap v1827: 105 pgs: 105 active+clean; 126 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 3.0 MiB/s rd, 1.6 MiB/s wr, 4 op/s 2024-09-13T00:15:17.302 INFO:tasks.workunit.client.0.smithi136.stdout: 808: throughput=4.99MB/sec pending data=0 2024-09-13T00:15:17.302 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-UkEs5xAEF9gnxuZ off=299731 len=1976656 2024-09-13T00:15:17.306 INFO:tasks.workunit.client.0.smithi136.stdout:op 4225 completed, throughput=5MB/sec 2024-09-13T00:15:17.306 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-3KJbdm_Sa4-8G0b off=3757364 len=1236108 2024-09-13T00:15:17.309 INFO:tasks.workunit.client.0.smithi136.stdout:op 4226 completed, throughput=5MB/sec 2024-09-13T00:15:17.310 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-TWAkvcXJjDSswj9 off=15110747 len=907746 2024-09-13T00:15:17.312 INFO:tasks.workunit.client.0.smithi136.stdout:op 4227 completed, throughput=5MB/sec 2024-09-13T00:15:17.313 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-5lx5XcNTpNaTPGJ off=20167507 len=1050523 2024-09-13T00:15:17.315 INFO:tasks.workunit.client.0.smithi136.stdout:op 4228 completed, throughput=5MB/sec 2024-09-13T00:15:18.317 INFO:tasks.workunit.client.0.smithi136.stdout: 809: throughput=4.99MB/sec pending data=0 2024-09-13T00:15:18.317 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2zBhUwbI-zsmTGw off=19681228 len=1746038 2024-09-13T00:15:18.322 INFO:tasks.workunit.client.0.smithi136.stdout:op 4229 completed, throughput=5MB/sec 2024-09-13T00:15:18.322 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-qzXxsr9S0rje62s off=537988 len=587113 2024-09-13T00:15:18.330 INFO:tasks.workunit.client.0.smithi136.stdout:op 4230 completed, throughput=5MB/sec 2024-09-13T00:15:18.330 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-BBo44F03hFkUZ2y off=2390573 len=443733 2024-09-13T00:15:18.336 INFO:tasks.workunit.client.0.smithi136.stdout:op 4231 completed, throughput=5MB/sec 2024-09-13T00:15:18.336 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-dz3o5acegaGCDu9 off=653820 len=1745206 2024-09-13T00:15:18.350 INFO:tasks.workunit.client.0.smithi136.stdout:op 4232 completed, throughput=5MB/sec 2024-09-13T00:15:18.350 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-rQqTNI9xWgyj7c7 off=11243464 len=156815 2024-09-13T00:15:18.355 INFO:tasks.workunit.client.0.smithi136.stdout:op 4233 completed, throughput=5MB/sec 2024-09-13T00:15:18.355 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-lMo07EDCqtoUty8 off=676891 len=770704 2024-09-13T00:15:18.362 INFO:tasks.workunit.client.0.smithi136.stdout:op 4234 completed, throughput=5MB/sec 2024-09-13T00:15:19.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:15:18 smithi136 bash[75164]: cluster 2024-09-13T00:15:17.560906+0000 mgr.y (mgr.24661) 1145 : cluster [DBG] pgmap v1828: 105 pgs: 105 active+clean; 126 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 3.4 MiB/s rd, 2.1 MiB/s wr, 5 op/s 2024-09-13T00:15:19.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:15:18 smithi136 bash[76371]: cluster 2024-09-13T00:15:17.560906+0000 mgr.y (mgr.24661) 1145 : cluster [DBG] pgmap v1828: 105 pgs: 105 active+clean; 126 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 3.4 MiB/s rd, 2.1 MiB/s wr, 5 op/s 2024-09-13T00:15:19.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:15:18 smithi181 bash[67587]: cluster 2024-09-13T00:15:17.560906+0000 mgr.y (mgr.24661) 1145 : cluster [DBG] pgmap v1828: 105 pgs: 105 active+clean; 126 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 3.4 MiB/s rd, 2.1 MiB/s wr, 5 op/s 2024-09-13T00:15:19.363 INFO:tasks.workunit.client.0.smithi136.stdout: 810: throughput=4.99MB/sec pending data=0 2024-09-13T00:15:19.363 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-5uoc1gSOeXhvgVR off=942067 len=1192174 2024-09-13T00:15:19.374 INFO:tasks.workunit.client.0.smithi136.stdout:op 4235 completed, throughput=5MB/sec 2024-09-13T00:15:19.374 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-eeb6e-p5imy_Hl6 off=3987415 len=653637 2024-09-13T00:15:19.381 INFO:tasks.workunit.client.0.smithi136.stdout:op 4236 completed, throughput=5MB/sec 2024-09-13T00:15:19.381 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-q1CJ1hoGW0LtT64 off=9927125 len=706616 2024-09-13T00:15:19.385 INFO:tasks.workunit.client.0.smithi136.stdout:op 4237 completed, throughput=5MB/sec 2024-09-13T00:15:19.385 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-tz1bym3x-silhh_ off=5505165 len=1616560 2024-09-13T00:15:19.397 INFO:tasks.workunit.client.0.smithi136.stdout:op 4238 completed, throughput=5MB/sec 2024-09-13T00:15:19.397 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Kp6R3uxSbyV1oHi off=24278431 len=120433 2024-09-13T00:15:19.398 INFO:tasks.workunit.client.0.smithi136.stdout:op 4239 completed, throughput=5MB/sec 2024-09-13T00:15:19.398 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-fBNITX6L8m4Imo6 off=7718079 len=353324 2024-09-13T00:15:19.403 INFO:tasks.workunit.client.0.smithi136.stdout:op 4240 completed, throughput=5MB/sec 2024-09-13T00:15:19.403 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-gb4y3gv1JaWmdDU off=405391 len=470085 2024-09-13T00:15:19.404 INFO:tasks.workunit.client.0.smithi136.stdout:op 4241 completed, throughput=5MB/sec 2024-09-13T00:15:19.405 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-r-AM-OQ_jPSuUdI off=10244183 len=573191 2024-09-13T00:15:19.411 INFO:tasks.workunit.client.0.smithi136.stdout:op 4242 completed, throughput=5MB/sec 2024-09-13T00:15:19.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:15:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:15:19] "GET /metrics HTTP/1.1" 200 34142 "" "Prometheus/2.43.0" 2024-09-13T00:15:20.412 INFO:tasks.workunit.client.0.smithi136.stdout: 811: throughput=4.99MB/sec pending data=0 2024-09-13T00:15:20.412 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-jnUofj_KLyrnntY off=24309807 len=894684 2024-09-13T00:15:20.419 INFO:tasks.workunit.client.0.smithi136.stdout:op 4243 completed, throughput=5MB/sec 2024-09-13T00:15:20.419 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-s1RmE2vfRjKA-as off=565730 len=2082724 2024-09-13T00:15:20.425 INFO:tasks.workunit.client.0.smithi136.stdout:op 4244 completed, throughput=5MB/sec 2024-09-13T00:15:20.425 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-R4MQ8BWyal05USL off=942946 len=2011589 2024-09-13T00:15:20.443 INFO:tasks.workunit.client.0.smithi136.stdout:op 4245 completed, throughput=5MB/sec 2024-09-13T00:15:21.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:15:20 smithi136 bash[75164]: cluster 2024-09-13T00:15:19.561837+0000 mgr.y (mgr.24661) 1146 : cluster [DBG] pgmap v1829: 105 pgs: 105 active+clean; 126 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:15:21.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:15:20 smithi136 bash[76371]: cluster 2024-09-13T00:15:19.561837+0000 mgr.y (mgr.24661) 1146 : cluster [DBG] pgmap v1829: 105 pgs: 105 active+clean; 126 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:15:21.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:15:20 smithi181 bash[67587]: cluster 2024-09-13T00:15:19.561837+0000 mgr.y (mgr.24661) 1146 : cluster [DBG] pgmap v1829: 105 pgs: 105 active+clean; 126 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:15:21.443 INFO:tasks.workunit.client.0.smithi136.stdout: 812: throughput=4.99MB/sec pending data=0 2024-09-13T00:15:21.443 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-0UVUZFfBgZe8-bc off=1942517 len=1268329 2024-09-13T00:15:21.446 INFO:tasks.workunit.client.0.smithi136.stdout:op 4246 completed, throughput=5MB/sec 2024-09-13T00:15:21.446 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-LJLJZdaGoPN5YoQ off=11433718 len=1967724 2024-09-13T00:15:21.450 INFO:tasks.workunit.client.0.smithi136.stdout:op 4247 completed, throughput=5MB/sec 2024-09-13T00:15:21.451 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ks6GekkzpTKFFbY off=2376972 len=1529971 2024-09-13T00:15:21.462 INFO:tasks.workunit.client.0.smithi136.stdout:op 4248 completed, throughput=5MB/sec 2024-09-13T00:15:21.462 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Ic-335yqalYSQYZ off=4016303 len=1661278 2024-09-13T00:15:21.472 INFO:tasks.workunit.client.0.smithi136.stdout:op 4249 completed, throughput=5MB/sec 2024-09-13T00:15:22.473 INFO:tasks.workunit.client.0.smithi136.stdout: 813: throughput=5MB/sec pending data=0 2024-09-13T00:15:22.473 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Ik7PhC4zVodUEfF off=2290516 len=2093722 2024-09-13T00:15:22.490 INFO:tasks.workunit.client.0.smithi136.stdout:op 4250 completed, throughput=5MB/sec 2024-09-13T00:15:22.491 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-tzmdm9mnnX1Kaze off=8303719 len=704127 2024-09-13T00:15:22.493 INFO:tasks.workunit.client.0.smithi136.stdout:op 4251 completed, throughput=5MB/sec 2024-09-13T00:15:22.493 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-uRCJ2Z4qdMElZp1 off=1235149 len=219165 2024-09-13T00:15:22.494 INFO:tasks.workunit.client.0.smithi136.stdout:op 4252 completed, throughput=5MB/sec 2024-09-13T00:15:22.495 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-yYFAlaclEh60mqL off=13470144 len=427876 2024-09-13T00:15:22.500 INFO:tasks.workunit.client.0.smithi136.stdout:op 4253 completed, throughput=5MB/sec 2024-09-13T00:15:22.501 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ojbcEVm4rpWJSM1 off=6358012 len=347737 2024-09-13T00:15:22.505 INFO:tasks.workunit.client.0.smithi136.stdout:op 4254 completed, throughput=5MB/sec 2024-09-13T00:15:22.506 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-LA3P6mjskRDYz_4 off=2464450 len=238249 2024-09-13T00:15:22.510 INFO:tasks.workunit.client.0.smithi136.stdout:op 4255 completed, throughput=5MB/sec 2024-09-13T00:15:22.510 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-VZz62E_g-Zv1UJb off=16496234 len=420381 2024-09-13T00:15:22.511 INFO:tasks.workunit.client.0.smithi136.stdout:op 4256 completed, throughput=5MB/sec 2024-09-13T00:15:22.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:15:22 smithi181 bash[95505]: ts=2024-09-13T00:15:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:15:23.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:15:22 smithi136 bash[75164]: cluster 2024-09-13T00:15:21.562911+0000 mgr.y (mgr.24661) 1147 : cluster [DBG] pgmap v1830: 105 pgs: 105 active+clean; 126 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.0 MiB/s wr, 5 op/s 2024-09-13T00:15:23.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:15:22 smithi136 bash[76371]: cluster 2024-09-13T00:15:21.562911+0000 mgr.y (mgr.24661) 1147 : cluster [DBG] pgmap v1830: 105 pgs: 105 active+clean; 126 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.0 MiB/s wr, 5 op/s 2024-09-13T00:15:23.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:15:22 smithi181 bash[67587]: cluster 2024-09-13T00:15:21.562911+0000 mgr.y (mgr.24661) 1147 : cluster [DBG] pgmap v1830: 105 pgs: 105 active+clean; 126 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.0 MiB/s wr, 5 op/s 2024-09-13T00:15:23.511 INFO:tasks.workunit.client.0.smithi136.stdout: 814: throughput=4.99MB/sec pending data=0 2024-09-13T00:15:23.511 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-3mmOmC48i4qyUlj off=7539648 len=259742 2024-09-13T00:15:23.517 INFO:tasks.workunit.client.0.smithi136.stdout:op 4257 completed, throughput=4.99MB/sec 2024-09-13T00:15:23.517 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-icO5stxVkPBncDr off=4604418 len=1543089 2024-09-13T00:15:23.521 INFO:tasks.workunit.client.0.smithi136.stdout:op 4258 completed, throughput=5MB/sec 2024-09-13T00:15:23.521 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-f2Pmw-0K_1LPEq1 off=19109716 len=1326665 2024-09-13T00:15:23.531 INFO:tasks.workunit.client.0.smithi136.stdout:op 4259 completed, throughput=5MB/sec 2024-09-13T00:15:23.531 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-HBkyF-EVhjlT_KE off=13553041 len=1188537 2024-09-13T00:15:23.541 INFO:tasks.workunit.client.0.smithi136.stdout:op 4260 completed, throughput=5MB/sec 2024-09-13T00:15:23.541 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-WPbCsrbNH58PtBC off=0 len=1436145 2024-09-13T00:15:23.545 INFO:tasks.workunit.client.0.smithi136.stdout:op 4261 completed, throughput=5MB/sec 2024-09-13T00:15:24.545 INFO:tasks.workunit.client.0.smithi136.stdout: 815: throughput=4.99MB/sec pending data=0 2024-09-13T00:15:24.546 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-bb_kzO7dtgQOn8E off=14650269 len=629513 2024-09-13T00:15:24.552 INFO:tasks.workunit.client.0.smithi136.stdout:op 4262 completed, throughput=5MB/sec 2024-09-13T00:15:24.553 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Oi4LJoasqigKMCW off=223640 len=1543387 2024-09-13T00:15:24.556 INFO:tasks.workunit.client.0.smithi136.stdout:op 4263 completed, throughput=5MB/sec 2024-09-13T00:15:24.557 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-KWe0VY0kXtYdjMb off=7945356 len=985499 2024-09-13T00:15:24.559 INFO:tasks.workunit.client.0.smithi136.stdout:op 4264 completed, throughput=5MB/sec 2024-09-13T00:15:24.559 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-SX3JOhng0ZnVZR2 off=13673357 len=1600635 2024-09-13T00:15:24.563 INFO:tasks.workunit.client.0.smithi136.stdout:op 4265 completed, throughput=5MB/sec 2024-09-13T00:15:24.563 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Epp9TWFf6gcgatt off=9626923 len=791797 2024-09-13T00:15:24.572 INFO:tasks.workunit.client.0.smithi136.stdout:op 4266 completed, throughput=5MB/sec 2024-09-13T00:15:25.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:15:24 smithi136 bash[75164]: cluster 2024-09-13T00:15:23.563983+0000 mgr.y (mgr.24661) 1148 : cluster [DBG] pgmap v1831: 105 pgs: 105 active+clean; 126 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:15:25.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:15:24 smithi136 bash[75164]: audit 2024-09-13T00:15:23.795014+0000 mon.a (mon.0) 2229 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:15:25.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:15:24 smithi136 bash[76371]: cluster 2024-09-13T00:15:23.563983+0000 mgr.y (mgr.24661) 1148 : cluster [DBG] pgmap v1831: 105 pgs: 105 active+clean; 126 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:15:25.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:15:24 smithi136 bash[76371]: audit 2024-09-13T00:15:23.795014+0000 mon.a (mon.0) 2229 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:15:25.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:15:24 smithi181 bash[67587]: cluster 2024-09-13T00:15:23.563983+0000 mgr.y (mgr.24661) 1148 : cluster [DBG] pgmap v1831: 105 pgs: 105 active+clean; 126 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:15:25.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:15:24 smithi181 bash[67587]: audit 2024-09-13T00:15:23.795014+0000 mon.a (mon.0) 2229 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:15:25.572 INFO:tasks.workunit.client.0.smithi136.stdout: 816: throughput=4.99MB/sec pending data=0 2024-09-13T00:15:25.572 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-vJdoiuG-9U6k3Xj off=14069970 len=506974 2024-09-13T00:15:25.574 INFO:tasks.workunit.client.0.smithi136.stdout:op 4267 completed, throughput=5MB/sec 2024-09-13T00:15:25.574 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-5mUbQM2nZv8R_Xg off=1004861 len=1249249 2024-09-13T00:15:25.578 INFO:tasks.workunit.client.0.smithi136.stdout:op 4268 completed, throughput=5MB/sec 2024-09-13T00:15:25.578 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-uFnUsXOqsZFenpl off=21490559 len=18117 2024-09-13T00:15:25.579 INFO:tasks.workunit.client.0.smithi136.stdout:op 4269 completed, throughput=5MB/sec 2024-09-13T00:15:25.579 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-aSSHjKW3h2j8bwV off=1279369 len=423607 2024-09-13T00:15:25.582 INFO:tasks.workunit.client.0.smithi136.stdout:op 4270 completed, throughput=5MB/sec 2024-09-13T00:15:25.582 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Ok3XgwNRVak8Br2 off=8418105 len=1721468 2024-09-13T00:15:25.587 INFO:tasks.workunit.client.0.smithi136.stdout:op 4271 completed, throughput=5MB/sec 2024-09-13T00:15:25.587 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-lWBe4C67-TM73H6 off=9070083 len=1278198 2024-09-13T00:15:25.590 INFO:tasks.workunit.client.0.smithi136.stdout:op 4272 completed, throughput=5MB/sec 2024-09-13T00:15:26.591 INFO:tasks.workunit.client.0.smithi136.stdout: 817: throughput=4.99MB/sec pending data=0 2024-09-13T00:15:26.591 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-e-WIXmmVUsw2N6F off=3050593 len=1956383 2024-09-13T00:15:26.597 INFO:tasks.workunit.client.0.smithi136.stdout:op 4273 completed, throughput=5MB/sec 2024-09-13T00:15:26.597 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-vyvco_KWQDJfLld off=10599623 len=168168 2024-09-13T00:15:26.599 INFO:tasks.workunit.client.0.smithi136.stdout:op 4274 completed, throughput=5MB/sec 2024-09-13T00:15:26.599 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-GYkMJrAeNTACdF_ off=6346187 len=1174990 2024-09-13T00:15:26.609 INFO:tasks.workunit.client.0.smithi136.stdout:op 4275 completed, throughput=5MB/sec 2024-09-13T00:15:26.609 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-YFOEF43ChCKeZKu off=419143 len=1769405 2024-09-13T00:15:26.614 INFO:tasks.workunit.client.0.smithi136.stdout:op 4276 completed, throughput=5MB/sec 2024-09-13T00:15:27.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:15:26 smithi136 bash[75164]: cluster 2024-09-13T00:15:25.564717+0000 mgr.y (mgr.24661) 1149 : cluster [DBG] pgmap v1832: 105 pgs: 105 active+clean; 126 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:15:27.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:15:26 smithi136 bash[76371]: cluster 2024-09-13T00:15:25.564717+0000 mgr.y (mgr.24661) 1149 : cluster [DBG] pgmap v1832: 105 pgs: 105 active+clean; 126 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:15:27.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:15:26 smithi181 bash[67587]: cluster 2024-09-13T00:15:25.564717+0000 mgr.y (mgr.24661) 1149 : cluster [DBG] pgmap v1832: 105 pgs: 105 active+clean; 126 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:15:27.614 INFO:tasks.workunit.client.0.smithi136.stdout: 818: throughput=4.99MB/sec pending data=0 2024-09-13T00:15:27.615 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-FKCCxxhvfls553g off=6350628 len=1160677 2024-09-13T00:15:27.619 INFO:tasks.workunit.client.0.smithi136.stdout:op 4277 completed, throughput=5MB/sec 2024-09-13T00:15:27.619 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-fyDQBefeZku83pu off=1914988 len=472982 2024-09-13T00:15:27.626 INFO:tasks.workunit.client.0.smithi136.stdout:op 4278 completed, throughput=5MB/sec 2024-09-13T00:15:27.626 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-eCzSgy83MLl1-k4 off=442027 len=1550794 2024-09-13T00:15:27.640 INFO:tasks.workunit.client.0.smithi136.stdout:op 4279 completed, throughput=5MB/sec 2024-09-13T00:15:27.640 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-NfHkJR07HyXgJ-- off=1651609 len=1908773 2024-09-13T00:15:27.652 INFO:tasks.workunit.client.0.smithi136.stdout:op 4280 completed, throughput=5MB/sec 2024-09-13T00:15:27.652 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-l0czEnnLmaErmxw off=971034 len=1406164 2024-09-13T00:15:27.656 INFO:tasks.workunit.client.0.smithi136.stdout:op 4281 completed, throughput=5MB/sec 2024-09-13T00:15:28.657 INFO:tasks.workunit.client.0.smithi136.stdout: 819: throughput=5MB/sec pending data=0 2024-09-13T00:15:28.657 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-8CXU9a2-rz7kQNo off=1164587 len=266630 2024-09-13T00:15:28.662 INFO:tasks.workunit.client.0.smithi136.stdout:op 4282 completed, throughput=5MB/sec 2024-09-13T00:15:28.662 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-aopf2-pw4_FruaW off=14313 len=1212534 2024-09-13T00:15:28.666 INFO:tasks.workunit.client.0.smithi136.stdout:op 4283 completed, throughput=5MB/sec 2024-09-13T00:15:28.666 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-KQsOJzI-ssTmwV1 off=3168550 len=382922 2024-09-13T00:15:28.667 INFO:tasks.workunit.client.0.smithi136.stdout:op 4284 completed, throughput=5MB/sec 2024-09-13T00:15:28.668 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-9POz9ngJJI_0oQG off=22560 len=702530 2024-09-13T00:15:28.675 INFO:tasks.workunit.client.0.smithi136.stdout:op 4285 completed, throughput=5MB/sec 2024-09-13T00:15:28.675 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj--7h3xypmYnMzV8V off=1364872 len=1910891 2024-09-13T00:15:28.691 INFO:tasks.workunit.client.0.smithi136.stdout:op 4286 completed, throughput=5MB/sec 2024-09-13T00:15:29.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:15:28 smithi136 bash[75164]: cluster 2024-09-13T00:15:27.565775+0000 mgr.y (mgr.24661) 1150 : cluster [DBG] pgmap v1833: 105 pgs: 105 active+clean; 126 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:15:29.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:15:28 smithi136 bash[76371]: cluster 2024-09-13T00:15:27.565775+0000 mgr.y (mgr.24661) 1150 : cluster [DBG] pgmap v1833: 105 pgs: 105 active+clean; 126 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:15:29.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:15:28 smithi181 bash[67587]: cluster 2024-09-13T00:15:27.565775+0000 mgr.y (mgr.24661) 1150 : cluster [DBG] pgmap v1833: 105 pgs: 105 active+clean; 126 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:15:29.692 INFO:tasks.workunit.client.0.smithi136.stdout: 820: throughput=4.99MB/sec pending data=0 2024-09-13T00:15:29.692 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-a0EKWetZCed2Ecy off=10032514 len=1883995 2024-09-13T00:15:29.705 INFO:tasks.workunit.client.0.smithi136.stdout:op 4287 completed, throughput=5MB/sec 2024-09-13T00:15:29.705 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-1vq2hYD1vGMsKZT off=3441534 len=1048728 2024-09-13T00:15:29.713 INFO:tasks.workunit.client.0.smithi136.stdout:op 4288 completed, throughput=5MB/sec 2024-09-13T00:15:29.714 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Ie4dv8hjDO1uxB1 off=24627995 len=762292 2024-09-13T00:15:29.721 INFO:tasks.workunit.client.0.smithi136.stdout:op 4289 completed, throughput=5MB/sec 2024-09-13T00:15:29.721 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-dyyWKBtqKpb7_v_ off=3647716 len=1899426 2024-09-13T00:15:29.736 INFO:tasks.workunit.client.0.smithi136.stdout:op 4290 completed, throughput=5MB/sec 2024-09-13T00:15:29.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:15:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:15:29] "GET /metrics HTTP/1.1" 200 34145 "" "Prometheus/2.43.0" 2024-09-13T00:15:30.736 INFO:tasks.workunit.client.0.smithi136.stdout: 821: throughput=4.99MB/sec pending data=0 2024-09-13T00:15:30.737 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-y_8shLfd6oFMQiB off=6728502 len=605557 2024-09-13T00:15:30.738 INFO:tasks.workunit.client.0.smithi136.stdout:op 4291 completed, throughput=5MB/sec 2024-09-13T00:15:30.739 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-1EQhA8cuv8yq61O off=6093792 len=801139 2024-09-13T00:15:30.747 INFO:tasks.workunit.client.0.smithi136.stdout:op 4292 completed, throughput=5MB/sec 2024-09-13T00:15:30.747 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-SmMYm9xDadn2JbF off=765851 len=896466 2024-09-13T00:15:30.754 INFO:tasks.workunit.client.0.smithi136.stdout:op 4293 completed, throughput=5MB/sec 2024-09-13T00:15:30.754 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-etZblQfoBPo5N9U off=5176482 len=1957744 2024-09-13T00:15:30.759 INFO:tasks.workunit.client.0.smithi136.stdout:op 4294 completed, throughput=5MB/sec 2024-09-13T00:15:30.759 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ob-y0AhswHz5VmJ off=2922559 len=1230136 2024-09-13T00:15:30.769 INFO:tasks.workunit.client.0.smithi136.stdout:op 4295 completed, throughput=5MB/sec 2024-09-13T00:15:31.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:15:30 smithi136 bash[75164]: cluster 2024-09-13T00:15:29.566659+0000 mgr.y (mgr.24661) 1151 : cluster [DBG] pgmap v1834: 105 pgs: 105 active+clean; 126 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:15:31.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:15:30 smithi136 bash[76371]: cluster 2024-09-13T00:15:29.566659+0000 mgr.y (mgr.24661) 1151 : cluster [DBG] pgmap v1834: 105 pgs: 105 active+clean; 126 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:15:31.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:15:30 smithi181 bash[67587]: cluster 2024-09-13T00:15:29.566659+0000 mgr.y (mgr.24661) 1151 : cluster [DBG] pgmap v1834: 105 pgs: 105 active+clean; 126 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:15:31.769 INFO:tasks.workunit.client.0.smithi136.stdout: 822: throughput=4.99MB/sec pending data=0 2024-09-13T00:15:31.769 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-iWGlQHL7biT8uP5 off=11008217 len=608386 2024-09-13T00:15:31.775 INFO:tasks.workunit.client.0.smithi136.stdout:op 4296 completed, throughput=5MB/sec 2024-09-13T00:15:31.776 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ROE3SoJfLLNaQ-s off=10384973 len=540308 2024-09-13T00:15:31.778 INFO:tasks.workunit.client.0.smithi136.stdout:op 4297 completed, throughput=5MB/sec 2024-09-13T00:15:31.778 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-spz7UaGJH10p9vp off=269038 len=1270023 2024-09-13T00:15:31.788 INFO:tasks.workunit.client.0.smithi136.stdout:op 4298 completed, throughput=5MB/sec 2024-09-13T00:15:31.788 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-hj76XBIPySb5bqd off=0 len=1516301 2024-09-13T00:15:31.791 INFO:tasks.workunit.client.0.smithi136.stdout:op 4299 completed, throughput=5MB/sec 2024-09-13T00:15:31.791 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-PkuD6Qxzmzw0bnd off=2081311 len=316161 2024-09-13T00:15:31.797 INFO:tasks.workunit.client.0.smithi136.stdout:op 4300 completed, throughput=5MB/sec 2024-09-13T00:15:31.797 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-scoVShsVCfEWnl9 off=15393318 len=231960 2024-09-13T00:15:31.798 INFO:tasks.workunit.client.0.smithi136.stdout:op 4301 completed, throughput=5MB/sec 2024-09-13T00:15:31.798 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-kkzRXQJG2z6aolH off=8770491 len=109786 2024-09-13T00:15:31.799 INFO:tasks.workunit.client.0.smithi136.stdout:op 4302 completed, throughput=5MB/sec 2024-09-13T00:15:31.799 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-xZGQPy8PnlAvrSx off=9691723 len=220584 2024-09-13T00:15:31.801 INFO:tasks.workunit.client.0.smithi136.stdout:op 4303 completed, throughput=5MB/sec 2024-09-13T00:15:31.801 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-GtdSZ_B48AvaZa1 off=2050674 len=1359997 2024-09-13T00:15:31.813 INFO:tasks.workunit.client.0.smithi136.stdout:op 4304 completed, throughput=5MB/sec 2024-09-13T00:15:32.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:15:32 smithi181 bash[95505]: ts=2024-09-13T00:15:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:15:32.814 INFO:tasks.workunit.client.0.smithi136.stdout: 823: throughput=5MB/sec pending data=0 2024-09-13T00:15:32.814 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj--JdqiMh1Mi9LMk4 off=16038625 len=580592 2024-09-13T00:15:32.821 INFO:tasks.workunit.client.0.smithi136.stdout:op 4305 completed, throughput=5MB/sec 2024-09-13T00:15:32.821 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-BPmRpPYwoUsMuPi off=5087217 len=144246 2024-09-13T00:15:32.822 INFO:tasks.workunit.client.0.smithi136.stdout:op 4306 completed, throughput=5MB/sec 2024-09-13T00:15:32.822 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-shrECNZAS_6w97v off=6983958 len=1045586 2024-09-13T00:15:32.830 INFO:tasks.workunit.client.0.smithi136.stdout:op 4307 completed, throughput=5MB/sec 2024-09-13T00:15:32.830 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-iBN_Bmjo5m6TcFR off=8130993 len=158189 2024-09-13T00:15:32.835 INFO:tasks.workunit.client.0.smithi136.stdout:op 4308 completed, throughput=5MB/sec 2024-09-13T00:15:32.835 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-xqwgV0oKxmbpCud off=9395585 len=248141 2024-09-13T00:15:32.839 INFO:tasks.workunit.client.0.smithi136.stdout:op 4309 completed, throughput=5MB/sec 2024-09-13T00:15:32.839 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-aGvEU-BO22s62SM off=9436661 len=1699979 2024-09-13T00:15:32.850 INFO:tasks.workunit.client.0.smithi136.stdout:op 4310 completed, throughput=5MB/sec 2024-09-13T00:15:32.851 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-tvw2_yFJEzaMjCz off=775311 len=166019 2024-09-13T00:15:32.852 INFO:tasks.workunit.client.0.smithi136.stdout:op 4311 completed, throughput=5MB/sec 2024-09-13T00:15:32.852 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-LiVgF0G0Vxa4Ajw off=0 len=1514067 2024-09-13T00:15:32.855 INFO:tasks.workunit.client.0.smithi136.stdout:op 4312 completed, throughput=5MB/sec 2024-09-13T00:15:33.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:15:32 smithi136 bash[75164]: cluster 2024-09-13T00:15:31.567650+0000 mgr.y (mgr.24661) 1152 : cluster [DBG] pgmap v1835: 105 pgs: 105 active+clean; 126 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:15:33.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:15:32 smithi136 bash[76371]: cluster 2024-09-13T00:15:31.567650+0000 mgr.y (mgr.24661) 1152 : cluster [DBG] pgmap v1835: 105 pgs: 105 active+clean; 126 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:15:33.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:15:32 smithi181 bash[67587]: cluster 2024-09-13T00:15:31.567650+0000 mgr.y (mgr.24661) 1152 : cluster [DBG] pgmap v1835: 105 pgs: 105 active+clean; 126 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:15:33.856 INFO:tasks.workunit.client.0.smithi136.stdout: 824: throughput=5MB/sec pending data=0 2024-09-13T00:15:33.856 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-R1_1FnOLMPeTmk2 off=959758 len=15628 2024-09-13T00:15:33.857 INFO:tasks.workunit.client.0.smithi136.stdout:op 4313 completed, throughput=5MB/sec 2024-09-13T00:15:33.857 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-SM-f8w-a72uRkt8 off=4103948 len=1817100 2024-09-13T00:15:33.862 INFO:tasks.workunit.client.0.smithi136.stdout:op 4314 completed, throughput=5MB/sec 2024-09-13T00:15:33.862 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-zwaS_Y6OXitYzqN off=7248632 len=1125211 2024-09-13T00:15:33.872 INFO:tasks.workunit.client.0.smithi136.stdout:op 4315 completed, throughput=5MB/sec 2024-09-13T00:15:33.873 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-QTIZFDhpdtIvR9d off=600959 len=946859 2024-09-13T00:15:33.881 INFO:tasks.workunit.client.0.smithi136.stdout:op 4316 completed, throughput=5MB/sec 2024-09-13T00:15:33.881 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Yw8SgvkEcZTf1tl off=5853683 len=1226260 2024-09-13T00:15:33.892 INFO:tasks.workunit.client.0.smithi136.stdout:op 4317 completed, throughput=5MB/sec 2024-09-13T00:15:34.892 INFO:tasks.workunit.client.0.smithi136.stdout: 825: throughput=5MB/sec pending data=0 2024-09-13T00:15:34.892 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-VUwLmNWFgK8AFEg off=11468342 len=991084 2024-09-13T00:15:34.895 INFO:tasks.workunit.client.0.smithi136.stdout:op 4318 completed, throughput=5MB/sec 2024-09-13T00:15:34.896 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-3DGcOUgQSjgTChg off=1316265 len=1707130 2024-09-13T00:15:34.907 INFO:tasks.workunit.client.0.smithi136.stdout:op 4319 completed, throughput=5MB/sec 2024-09-13T00:15:34.907 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-l0JPg7jui65oMC6 off=1314733 len=744590 2024-09-13T00:15:34.910 INFO:tasks.workunit.client.0.smithi136.stdout:op 4320 completed, throughput=5MB/sec 2024-09-13T00:15:34.910 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-wNSjj06QzHS-G4l off=3482901 len=271104 2024-09-13T00:15:34.916 INFO:tasks.workunit.client.0.smithi136.stdout:op 4321 completed, throughput=5MB/sec 2024-09-13T00:15:34.916 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-tDgKi40CU9dzyhP off=12234222 len=1177790 2024-09-13T00:15:34.926 INFO:tasks.workunit.client.0.smithi136.stdout:op 4322 completed, throughput=5MB/sec 2024-09-13T00:15:35.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:15:34 smithi181 bash[67587]: cluster 2024-09-13T00:15:33.568714+0000 mgr.y (mgr.24661) 1153 : cluster [DBG] pgmap v1836: 105 pgs: 105 active+clean; 126 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:15:35.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:15:34 smithi136 bash[75164]: cluster 2024-09-13T00:15:33.568714+0000 mgr.y (mgr.24661) 1153 : cluster [DBG] pgmap v1836: 105 pgs: 105 active+clean; 126 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:15:35.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:15:34 smithi136 bash[76371]: cluster 2024-09-13T00:15:33.568714+0000 mgr.y (mgr.24661) 1153 : cluster [DBG] pgmap v1836: 105 pgs: 105 active+clean; 126 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:15:35.927 INFO:tasks.workunit.client.0.smithi136.stdout: 826: throughput=4.99MB/sec pending data=0 2024-09-13T00:15:35.927 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-yiE2uIjts5owuIE off=6061919 len=1353699 2024-09-13T00:15:35.936 INFO:tasks.workunit.client.0.smithi136.stdout:op 4323 completed, throughput=5MB/sec 2024-09-13T00:15:35.936 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-dWmAUIRb9IKxOh4 off=219092 len=952013 2024-09-13T00:15:35.945 INFO:tasks.workunit.client.0.smithi136.stdout:op 4324 completed, throughput=5MB/sec 2024-09-13T00:15:35.945 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-goIurOu0rX0J10E off=474425 len=1125415 2024-09-13T00:15:35.947 INFO:tasks.workunit.client.0.smithi136.stdout:op 4325 completed, throughput=5MB/sec 2024-09-13T00:15:35.947 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-zD3lTl_QH0mwYhg off=4770379 len=1629159 2024-09-13T00:15:35.952 INFO:tasks.workunit.client.0.smithi136.stdout:op 4326 completed, throughput=5MB/sec 2024-09-13T00:15:36.952 INFO:tasks.workunit.client.0.smithi136.stdout: 827: throughput=4.99MB/sec pending data=0 2024-09-13T00:15:36.953 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-MYThLg0SeSsh16v off=1988352 len=1412667 2024-09-13T00:15:36.963 INFO:tasks.workunit.client.0.smithi136.stdout:op 4327 completed, throughput=5MB/sec 2024-09-13T00:15:36.963 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-tk0rUWqhHxiBFPN off=12818977 len=2036605 2024-09-13T00:15:36.979 INFO:tasks.workunit.client.0.smithi136.stdout:op 4328 completed, throughput=5MB/sec 2024-09-13T00:15:36.979 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-HhiLLE2sxhn5brK off=323865 len=1720599 2024-09-13T00:15:36.982 INFO:tasks.workunit.client.0.smithi136.stdout:op 4329 completed, throughput=5MB/sec 2024-09-13T00:15:36.983 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ZloIdEPV6XDXiJI off=7570114 len=1290158 2024-09-13T00:15:36.992 INFO:tasks.workunit.client.0.smithi136.stdout:op 4330 completed, throughput=5MB/sec 2024-09-13T00:15:37.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:15:36 smithi181 bash[67587]: cluster 2024-09-13T00:15:35.569453+0000 mgr.y (mgr.24661) 1154 : cluster [DBG] pgmap v1837: 105 pgs: 105 active+clean; 126 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:15:37.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:15:36 smithi136 bash[75164]: cluster 2024-09-13T00:15:35.569453+0000 mgr.y (mgr.24661) 1154 : cluster [DBG] pgmap v1837: 105 pgs: 105 active+clean; 126 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:15:37.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:15:36 smithi136 bash[76371]: cluster 2024-09-13T00:15:35.569453+0000 mgr.y (mgr.24661) 1154 : cluster [DBG] pgmap v1837: 105 pgs: 105 active+clean; 126 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:15:37.992 INFO:tasks.workunit.client.0.smithi136.stdout: 828: throughput=5MB/sec pending data=0 2024-09-13T00:15:37.992 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-JsHNhnp33_5zbwn off=893926 len=1510483 2024-09-13T00:15:38.003 INFO:tasks.workunit.client.0.smithi136.stdout:op 4331 completed, throughput=5MB/sec 2024-09-13T00:15:38.003 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-DU6DxfhoBFsUJTa off=3936535 len=1648455 2024-09-13T00:15:38.018 INFO:tasks.workunit.client.0.smithi136.stdout:op 4332 completed, throughput=5MB/sec 2024-09-13T00:15:38.018 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-f5g39Uk4WvKf53i off=19883145 len=1885043 2024-09-13T00:15:38.024 INFO:tasks.workunit.client.0.smithi136.stdout:op 4333 completed, throughput=5MB/sec 2024-09-13T00:15:39.025 INFO:tasks.workunit.client.0.smithi136.stdout: 829: throughput=4.99MB/sec pending data=0 2024-09-13T00:15:39.025 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-VPzjDi277nLfnlV off=7066097 len=569932 2024-09-13T00:15:39.032 INFO:tasks.workunit.client.0.smithi136.stdout:op 4334 completed, throughput=5MB/sec 2024-09-13T00:15:39.032 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-kh9tt5UgefcVGi8 off=364341 len=1647794 2024-09-13T00:15:39.037 INFO:tasks.workunit.client.0.smithi136.stdout:op 4335 completed, throughput=5MB/sec 2024-09-13T00:15:39.037 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-vq1qDrGTb9sv-ZB off=676505 len=932999 2024-09-13T00:15:39.040 INFO:tasks.workunit.client.0.smithi136.stdout:op 4336 completed, throughput=5MB/sec 2024-09-13T00:15:39.040 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-VCRqUCOEL4x2kmM off=8195914 len=949766 2024-09-13T00:15:39.049 INFO:tasks.workunit.client.0.smithi136.stdout:op 4337 completed, throughput=5MB/sec 2024-09-13T00:15:39.049 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-S83rOVmVAWFB8pJ off=8101082 len=322775 2024-09-13T00:15:39.055 INFO:tasks.workunit.client.0.smithi136.stdout:op 4338 completed, throughput=5MB/sec 2024-09-13T00:15:39.055 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-_37NFoLj_-bCwCQ off=200165 len=2049889 2024-09-13T00:15:39.060 INFO:tasks.workunit.client.0.smithi136.stdout:op 4339 completed, throughput=5MB/sec 2024-09-13T00:15:39.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:15:38 smithi181 bash[67587]: cluster 2024-09-13T00:15:37.570565+0000 mgr.y (mgr.24661) 1155 : cluster [DBG] pgmap v1838: 105 pgs: 105 active+clean; 126 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:15:39.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:15:38 smithi136 bash[75164]: cluster 2024-09-13T00:15:37.570565+0000 mgr.y (mgr.24661) 1155 : cluster [DBG] pgmap v1838: 105 pgs: 105 active+clean; 126 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:15:39.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:15:38 smithi136 bash[76371]: cluster 2024-09-13T00:15:37.570565+0000 mgr.y (mgr.24661) 1155 : cluster [DBG] pgmap v1838: 105 pgs: 105 active+clean; 126 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:15:39.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:15:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:15:39] "GET /metrics HTTP/1.1" 200 34142 "" "Prometheus/2.43.0" 2024-09-13T00:15:40.060 INFO:tasks.workunit.client.0.smithi136.stdout: 830: throughput=5MB/sec pending data=0 2024-09-13T00:15:40.060 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-CD9f1QDDTEOWqJp off=15647872 len=1638471 2024-09-13T00:15:40.075 INFO:tasks.workunit.client.0.smithi136.stdout:op 4340 completed, throughput=5MB/sec 2024-09-13T00:15:40.075 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-K2WuZgsa_lDJXI8 off=1195779 len=519111 2024-09-13T00:15:40.082 INFO:tasks.workunit.client.0.smithi136.stdout:op 4341 completed, throughput=5MB/sec 2024-09-13T00:15:40.082 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-fiUtAhCHy56u-S- off=2068602 len=842830 2024-09-13T00:15:40.084 INFO:tasks.workunit.client.0.smithi136.stdout:op 4342 completed, throughput=5MB/sec 2024-09-13T00:15:40.084 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-eV8R4ccDOmQyr7n off=12754025 len=285602 2024-09-13T00:15:40.085 INFO:tasks.workunit.client.0.smithi136.stdout:op 4343 completed, throughput=5MB/sec 2024-09-13T00:15:40.086 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Ss9IMZ9bi5jYDMZ off=7442999 len=771463 2024-09-13T00:15:40.089 INFO:tasks.workunit.client.0.smithi136.stdout:op 4344 completed, throughput=5MB/sec 2024-09-13T00:15:40.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:15:39 smithi181 bash[67587]: audit 2024-09-13T00:15:38.795145+0000 mon.a (mon.0) 2230 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:15:40.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:15:39 smithi136 bash[75164]: audit 2024-09-13T00:15:38.795145+0000 mon.a (mon.0) 2230 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:15:40.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:15:39 smithi136 bash[76371]: audit 2024-09-13T00:15:38.795145+0000 mon.a (mon.0) 2230 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:15:41.089 INFO:tasks.workunit.client.0.smithi136.stdout: 831: throughput=4.99MB/sec pending data=0 2024-09-13T00:15:41.089 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-G0xfEjxJc51w6uD off=12956529 len=1595646 2024-09-13T00:15:41.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:15:40 smithi181 bash[67587]: cluster 2024-09-13T00:15:39.571472+0000 mgr.y (mgr.24661) 1156 : cluster [DBG] pgmap v1839: 105 pgs: 105 active+clean; 126 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.7 MiB/s wr, 4 op/s 2024-09-13T00:15:41.103 INFO:tasks.workunit.client.0.smithi136.stdout:op 4345 completed, throughput=5MB/sec 2024-09-13T00:15:41.104 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-w6zg3oCx9ud0auD off=3155540 len=1712284 2024-09-13T00:15:41.109 INFO:tasks.workunit.client.0.smithi136.stdout:op 4346 completed, throughput=5MB/sec 2024-09-13T00:15:41.109 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-krtYspnusyx80Mi off=4573094 len=1085302 2024-09-13T00:15:41.113 INFO:tasks.workunit.client.0.smithi136.stdout:op 4347 completed, throughput=5MB/sec 2024-09-13T00:15:41.113 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-M40NwOT47aeSrSX off=8113842 len=389197 2024-09-13T00:15:41.115 INFO:tasks.workunit.client.0.smithi136.stdout:op 4348 completed, throughput=5MB/sec 2024-09-13T00:15:41.115 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-gc2zEvRFOiT7Fpq off=19359422 len=119785 2024-09-13T00:15:41.116 INFO:tasks.workunit.client.0.smithi136.stdout:op 4349 completed, throughput=5MB/sec 2024-09-13T00:15:41.116 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-0WPR0-kgiIZVEkl off=19962930 len=2033139 2024-09-13T00:15:41.129 INFO:tasks.workunit.client.0.smithi136.stdout:op 4350 completed, throughput=5MB/sec 2024-09-13T00:15:41.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:15:40 smithi136 bash[75164]: cluster 2024-09-13T00:15:39.571472+0000 mgr.y (mgr.24661) 1156 : cluster [DBG] pgmap v1839: 105 pgs: 105 active+clean; 126 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.7 MiB/s wr, 4 op/s 2024-09-13T00:15:41.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:15:40 smithi136 bash[76371]: cluster 2024-09-13T00:15:39.571472+0000 mgr.y (mgr.24661) 1156 : cluster [DBG] pgmap v1839: 105 pgs: 105 active+clean; 126 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.7 MiB/s wr, 4 op/s 2024-09-13T00:15:42.130 INFO:tasks.workunit.client.0.smithi136.stdout: 832: throughput=5MB/sec pending data=0 2024-09-13T00:15:42.130 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-TqNIMGA65qI1zQH off=21673963 len=1285604 2024-09-13T00:15:42.134 INFO:tasks.workunit.client.0.smithi136.stdout:op 4351 completed, throughput=5MB/sec 2024-09-13T00:15:42.134 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-CfgAJfGzGs2XTgH off=6053625 len=216586 2024-09-13T00:15:42.140 INFO:tasks.workunit.client.0.smithi136.stdout:op 4352 completed, throughput=5MB/sec 2024-09-13T00:15:42.140 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-fTLD6-AchcFN7VU off=14372106 len=899756 2024-09-13T00:15:42.148 INFO:tasks.workunit.client.0.smithi136.stdout:op 4353 completed, throughput=5MB/sec 2024-09-13T00:15:42.149 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-YFN5MDO3b528144 off=4020686 len=715312 2024-09-13T00:15:42.156 INFO:tasks.workunit.client.0.smithi136.stdout:op 4354 completed, throughput=5MB/sec 2024-09-13T00:15:42.156 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-wX50VM-wQCRRNoH off=17778693 len=1323277 2024-09-13T00:15:42.165 INFO:tasks.workunit.client.0.smithi136.stdout:op 4355 completed, throughput=5MB/sec 2024-09-13T00:15:42.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:15:42 smithi181 bash[95505]: ts=2024-09-13T00:15:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:15:43.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:15:42 smithi181 bash[67587]: cluster 2024-09-13T00:15:41.572468+0000 mgr.y (mgr.24661) 1157 : cluster [DBG] pgmap v1840: 105 pgs: 105 active+clean; 126 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 2.3 MiB/s rd, 3.0 MiB/s wr, 5 op/s 2024-09-13T00:15:43.165 INFO:tasks.workunit.client.0.smithi136.stdout: 833: throughput=5MB/sec pending data=0 2024-09-13T00:15:43.165 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-iaJP_tyrAdmC0pq off=18373032 len=1830262 2024-09-13T00:15:43.178 INFO:tasks.workunit.client.0.smithi136.stdout:op 4356 completed, throughput=5MB/sec 2024-09-13T00:15:43.178 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-GGsZVrRB-swCI2r off=724771 len=1684 2024-09-13T00:15:43.179 INFO:tasks.workunit.client.0.smithi136.stdout:op 4357 completed, throughput=5MB/sec 2024-09-13T00:15:43.179 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-aQRpb5A-vsfZAxP off=8125813 len=811523 2024-09-13T00:15:43.187 INFO:tasks.workunit.client.0.smithi136.stdout:op 4358 completed, throughput=5MB/sec 2024-09-13T00:15:43.187 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-8Qv44hLTejff4F8 off=0 len=1593061 2024-09-13T00:15:43.200 INFO:tasks.workunit.client.0.smithi136.stdout:op 4359 completed, throughput=5MB/sec 2024-09-13T00:15:43.200 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-hdZBDdP6HJjgSIM off=1819178 len=1264844 2024-09-13T00:15:43.210 INFO:tasks.workunit.client.0.smithi136.stdout:op 4360 completed, throughput=5MB/sec 2024-09-13T00:15:43.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:15:42 smithi136 bash[75164]: cluster 2024-09-13T00:15:41.572468+0000 mgr.y (mgr.24661) 1157 : cluster [DBG] pgmap v1840: 105 pgs: 105 active+clean; 126 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 2.3 MiB/s rd, 3.0 MiB/s wr, 5 op/s 2024-09-13T00:15:43.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:15:42 smithi136 bash[76371]: cluster 2024-09-13T00:15:41.572468+0000 mgr.y (mgr.24661) 1157 : cluster [DBG] pgmap v1840: 105 pgs: 105 active+clean; 126 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 2.3 MiB/s rd, 3.0 MiB/s wr, 5 op/s 2024-09-13T00:15:44.210 INFO:tasks.workunit.client.0.smithi136.stdout: 834: throughput=5MB/sec pending data=0 2024-09-13T00:15:44.210 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-U8yVgAQgU5_YJAm off=6848205 len=1998425 2024-09-13T00:15:44.222 INFO:tasks.workunit.client.0.smithi136.stdout:op 4361 completed, throughput=5MB/sec 2024-09-13T00:15:44.222 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-I4O9VZtiQ9njAGC off=5767638 len=29779 2024-09-13T00:15:44.223 INFO:tasks.workunit.client.0.smithi136.stdout:op 4362 completed, throughput=5MB/sec 2024-09-13T00:15:44.224 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-qHrP5xM139v0fLJ off=20905881 len=899033 2024-09-13T00:15:44.226 INFO:tasks.workunit.client.0.smithi136.stdout:op 4363 completed, throughput=5MB/sec 2024-09-13T00:15:44.226 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-IBMBQBZ4Wyxl4iq off=2244055 len=570439 2024-09-13T00:15:44.228 INFO:tasks.workunit.client.0.smithi136.stdout:op 4364 completed, throughput=5MB/sec 2024-09-13T00:15:44.228 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj--aMh9Lv9TzUFOJF off=2946392 len=1584374 2024-09-13T00:15:44.242 INFO:tasks.workunit.client.0.smithi136.stdout:op 4365 completed, throughput=5MB/sec 2024-09-13T00:15:45.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:15:44 smithi181 bash[67587]: cluster 2024-09-13T00:15:43.573484+0000 mgr.y (mgr.24661) 1158 : cluster [DBG] pgmap v1841: 105 pgs: 105 active+clean; 126 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 2.0 MiB/s rd, 3.2 MiB/s wr, 5 op/s 2024-09-13T00:15:45.242 INFO:tasks.workunit.client.0.smithi136.stdout: 835: throughput=4.99MB/sec pending data=0 2024-09-13T00:15:45.243 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-BPK0IRgfgBPKeF3 off=788436 len=612093 2024-09-13T00:15:45.245 INFO:tasks.workunit.client.0.smithi136.stdout:op 4366 completed, throughput=5MB/sec 2024-09-13T00:15:45.245 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-et2ig7ks75UtDKh off=1480346 len=1905160 2024-09-13T00:15:45.249 INFO:tasks.workunit.client.0.smithi136.stdout:op 4367 completed, throughput=5MB/sec 2024-09-13T00:15:45.250 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-fHrxff6WVYcY0rP off=10308890 len=2071955 2024-09-13T00:15:45.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:15:44 smithi136 bash[75164]: cluster 2024-09-13T00:15:43.573484+0000 mgr.y (mgr.24661) 1158 : cluster [DBG] pgmap v1841: 105 pgs: 105 active+clean; 126 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 2.0 MiB/s rd, 3.2 MiB/s wr, 5 op/s 2024-09-13T00:15:45.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:15:44 smithi136 bash[76371]: cluster 2024-09-13T00:15:43.573484+0000 mgr.y (mgr.24661) 1158 : cluster [DBG] pgmap v1841: 105 pgs: 105 active+clean; 126 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 2.0 MiB/s rd, 3.2 MiB/s wr, 5 op/s 2024-09-13T00:15:45.266 INFO:tasks.workunit.client.0.smithi136.stdout:op 4368 completed, throughput=5MB/sec 2024-09-13T00:15:45.266 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-keEtIa80fR8dHNx off=1149205 len=180763 2024-09-13T00:15:45.271 INFO:tasks.workunit.client.0.smithi136.stdout:op 4369 completed, throughput=5MB/sec 2024-09-13T00:15:46.271 INFO:tasks.workunit.client.0.smithi136.stdout: 836: throughput=4.99MB/sec pending data=0 2024-09-13T00:15:46.272 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Jvy5Zults8_vEMB off=14094680 len=969631 2024-09-13T00:15:46.274 INFO:tasks.workunit.client.0.smithi136.stdout:op 4370 completed, throughput=5MB/sec 2024-09-13T00:15:46.275 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Y4ErdxymgIlX1HV off=6857077 len=178700 2024-09-13T00:15:46.275 INFO:tasks.workunit.client.0.smithi136.stdout:op 4371 completed, throughput=5MB/sec 2024-09-13T00:15:46.276 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-2wj-F7Ik58CyxI3 off=708583 len=1209513 2024-09-13T00:15:46.286 INFO:tasks.workunit.client.0.smithi136.stdout:op 4372 completed, throughput=5MB/sec 2024-09-13T00:15:46.286 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-NpCcH4QeMNAUEhB off=3136072 len=1100365 2024-09-13T00:15:46.295 INFO:tasks.workunit.client.0.smithi136.stdout:op 4373 completed, throughput=5MB/sec 2024-09-13T00:15:46.295 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-fQfEr4nfgntVv-B off=12621091 len=1029648 2024-09-13T00:15:46.298 INFO:tasks.workunit.client.0.smithi136.stdout:op 4374 completed, throughput=5MB/sec 2024-09-13T00:15:46.298 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-bjfBwHAlcTrifIc off=148845 len=1568742 2024-09-13T00:15:46.312 INFO:tasks.workunit.client.0.smithi136.stdout:op 4375 completed, throughput=5MB/sec 2024-09-13T00:15:47.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:15:46 smithi181 bash[67587]: cluster 2024-09-13T00:15:45.574283+0000 mgr.y (mgr.24661) 1159 : cluster [DBG] pgmap v1842: 105 pgs: 105 active+clean; 126 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 1.9 MiB/s rd, 3.0 MiB/s wr, 4 op/s 2024-09-13T00:15:47.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:15:46 smithi136 bash[75164]: cluster 2024-09-13T00:15:45.574283+0000 mgr.y (mgr.24661) 1159 : cluster [DBG] pgmap v1842: 105 pgs: 105 active+clean; 126 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 1.9 MiB/s rd, 3.0 MiB/s wr, 4 op/s 2024-09-13T00:15:47.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:15:46 smithi136 bash[76371]: cluster 2024-09-13T00:15:45.574283+0000 mgr.y (mgr.24661) 1159 : cluster [DBG] pgmap v1842: 105 pgs: 105 active+clean; 126 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 1.9 MiB/s rd, 3.0 MiB/s wr, 4 op/s 2024-09-13T00:15:47.311 INFO:tasks.workunit.client.0.smithi136.stdout: 837: throughput=4.99MB/sec pending data=0 2024-09-13T00:15:47.311 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-KKz6UHGIsFgMlIJ off=665764 len=1037494 2024-09-13T00:15:47.314 INFO:tasks.workunit.client.0.smithi136.stdout:op 4376 completed, throughput=5MB/sec 2024-09-13T00:15:47.314 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Q-AIp7kAFZ5u9OX off=2798757 len=2061509 2024-09-13T00:15:47.319 INFO:tasks.workunit.client.0.smithi136.stdout:op 4377 completed, throughput=5MB/sec 2024-09-13T00:15:47.319 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ikmqTAUUtQSDHOC off=2573053 len=2054272 2024-09-13T00:15:47.336 INFO:tasks.workunit.client.0.smithi136.stdout:op 4378 completed, throughput=5MB/sec 2024-09-13T00:15:48.337 INFO:tasks.workunit.client.0.smithi136.stdout: 838: throughput=4.99MB/sec pending data=0 2024-09-13T00:15:48.337 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-dkttGpjqFo2jdcg off=1053984 len=1282141 2024-09-13T00:15:48.346 INFO:tasks.workunit.client.0.smithi136.stdout:op 4379 completed, throughput=5MB/sec 2024-09-13T00:15:48.346 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-g0Q3-p3wJSpVRSz off=7048169 len=1175669 2024-09-13T00:15:48.356 INFO:tasks.workunit.client.0.smithi136.stdout:op 4380 completed, throughput=5MB/sec 2024-09-13T00:15:48.356 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-NiGJX1z8DX2wN9a off=3166437 len=859199 2024-09-13T00:15:48.364 INFO:tasks.workunit.client.0.smithi136.stdout:op 4381 completed, throughput=5MB/sec 2024-09-13T00:15:48.364 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Jluy9aobOjxkchq off=13443207 len=352106 2024-09-13T00:15:48.369 INFO:tasks.workunit.client.0.smithi136.stdout:op 4382 completed, throughput=5MB/sec 2024-09-13T00:15:48.369 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ZEG_zqRnTwa8z7N off=6265032 len=735920 2024-09-13T00:15:48.377 INFO:tasks.workunit.client.0.smithi136.stdout:op 4383 completed, throughput=5MB/sec 2024-09-13T00:15:48.377 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-5W5LU5YRXJCHd1B off=2639842 len=842177 2024-09-13T00:15:48.380 INFO:tasks.workunit.client.0.smithi136.stdout:op 4384 completed, throughput=5MB/sec 2024-09-13T00:15:49.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:15:48 smithi181 bash[67587]: cluster 2024-09-13T00:15:47.575349+0000 mgr.y (mgr.24661) 1160 : cluster [DBG] pgmap v1843: 105 pgs: 105 active+clean; 126 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 2.0 MiB/s rd, 3.8 MiB/s wr, 5 op/s 2024-09-13T00:15:49.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:15:48 smithi136 bash[75164]: cluster 2024-09-13T00:15:47.575349+0000 mgr.y (mgr.24661) 1160 : cluster [DBG] pgmap v1843: 105 pgs: 105 active+clean; 126 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 2.0 MiB/s rd, 3.8 MiB/s wr, 5 op/s 2024-09-13T00:15:49.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:15:48 smithi136 bash[76371]: cluster 2024-09-13T00:15:47.575349+0000 mgr.y (mgr.24661) 1160 : cluster [DBG] pgmap v1843: 105 pgs: 105 active+clean; 126 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 2.0 MiB/s rd, 3.8 MiB/s wr, 5 op/s 2024-09-13T00:15:49.381 INFO:tasks.workunit.client.0.smithi136.stdout: 839: throughput=4.99MB/sec pending data=0 2024-09-13T00:15:49.381 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-4YQ5GAtTE1RkQCz off=8810078 len=1236383 2024-09-13T00:15:49.391 INFO:tasks.workunit.client.0.smithi136.stdout:op 4385 completed, throughput=5MB/sec 2024-09-13T00:15:49.391 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-sMujDMsbne2jbep off=2294584 len=1642878 2024-09-13T00:15:49.401 INFO:tasks.workunit.client.0.smithi136.stdout:op 4386 completed, throughput=5MB/sec 2024-09-13T00:15:49.401 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-HMFp8kixomwxh0q off=9753751 len=1636958 2024-09-13T00:15:49.405 INFO:tasks.workunit.client.0.smithi136.stdout:op 4387 completed, throughput=5MB/sec 2024-09-13T00:15:49.405 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-TyBaQESJjbK9D60 off=18643262 len=1786721 2024-09-13T00:15:49.417 INFO:tasks.workunit.client.0.smithi136.stdout:op 4388 completed, throughput=5MB/sec 2024-09-13T00:15:49.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:15:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:15:49] "GET /metrics HTTP/1.1" 200 34142 "" "Prometheus/2.43.0" 2024-09-13T00:15:50.418 INFO:tasks.workunit.client.0.smithi136.stdout: 840: throughput=5MB/sec pending data=0 2024-09-13T00:15:50.418 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-U7Bj9XQAw6pLJg4 off=1359173 len=754102 2024-09-13T00:15:50.425 INFO:tasks.workunit.client.0.smithi136.stdout:op 4389 completed, throughput=5MB/sec 2024-09-13T00:15:50.425 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ce2IEaSQvpDbdbY off=12105620 len=777887 2024-09-13T00:15:50.428 INFO:tasks.workunit.client.0.smithi136.stdout:op 4390 completed, throughput=5MB/sec 2024-09-13T00:15:50.428 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-7FE5AD2kXG48ZGG off=1887871 len=1242413 2024-09-13T00:15:50.438 INFO:tasks.workunit.client.0.smithi136.stdout:op 4391 completed, throughput=5MB/sec 2024-09-13T00:15:50.438 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Dyu73DI5uebT0B6 off=34606 len=1521202 2024-09-13T00:15:50.448 INFO:tasks.workunit.client.0.smithi136.stdout:op 4392 completed, throughput=5MB/sec 2024-09-13T00:15:50.448 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-yUszB_IMjojGH1x off=281040 len=651880 2024-09-13T00:15:50.455 INFO:tasks.workunit.client.0.smithi136.stdout:op 4393 completed, throughput=5MB/sec 2024-09-13T00:15:51.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:15:50 smithi181 bash[67587]: cluster 2024-09-13T00:15:49.576355+0000 mgr.y (mgr.24661) 1161 : cluster [DBG] pgmap v1844: 105 pgs: 105 active+clean; 126 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.1 MiB/s wr, 4 op/s 2024-09-13T00:15:51.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:15:50 smithi136 bash[76371]: cluster 2024-09-13T00:15:49.576355+0000 mgr.y (mgr.24661) 1161 : cluster [DBG] pgmap v1844: 105 pgs: 105 active+clean; 126 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.1 MiB/s wr, 4 op/s 2024-09-13T00:15:51.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:15:50 smithi136 bash[75164]: cluster 2024-09-13T00:15:49.576355+0000 mgr.y (mgr.24661) 1161 : cluster [DBG] pgmap v1844: 105 pgs: 105 active+clean; 126 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.1 MiB/s wr, 4 op/s 2024-09-13T00:15:51.455 INFO:tasks.workunit.client.0.smithi136.stdout: 841: throughput=4.99MB/sec pending data=0 2024-09-13T00:15:51.455 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-XwGgYSIxkuLij2L off=2736443 len=1583090 2024-09-13T00:15:51.459 INFO:tasks.workunit.client.0.smithi136.stdout:op 4394 completed, throughput=5MB/sec 2024-09-13T00:15:51.459 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-mqxxFF_5rHL4ZbU off=12869539 len=2082276 2024-09-13T00:15:51.466 INFO:tasks.workunit.client.0.smithi136.stdout:op 4395 completed, throughput=5MB/sec 2024-09-13T00:15:51.466 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-LpFLjYnjC6b0kJr off=4863482 len=122017 2024-09-13T00:15:51.470 INFO:tasks.workunit.client.0.smithi136.stdout:op 4396 completed, throughput=5MB/sec 2024-09-13T00:15:51.470 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-f7SnEd-W1-KsnSP off=2525159 len=2061173 2024-09-13T00:15:51.484 INFO:tasks.workunit.client.0.smithi136.stdout:op 4397 completed, throughput=5MB/sec 2024-09-13T00:15:52.485 INFO:tasks.workunit.client.0.smithi136.stdout: 842: throughput=5MB/sec pending data=0 2024-09-13T00:15:52.485 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-MhANPQRgPDrhU8X off=2241610 len=404717 2024-09-13T00:15:52.491 INFO:tasks.workunit.client.0.smithi136.stdout:op 4398 completed, throughput=5MB/sec 2024-09-13T00:15:52.491 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-4c1lV0MxCCmdzTc off=9473008 len=1278235 2024-09-13T00:15:52.494 INFO:tasks.workunit.client.0.smithi136.stdout:op 4399 completed, throughput=5MB/sec 2024-09-13T00:15:52.494 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-GRA_8XQXo7yynVD off=4090939 len=107769 2024-09-13T00:15:52.495 INFO:tasks.workunit.client.0.smithi136.stdout:op 4400 completed, throughput=5MB/sec 2024-09-13T00:15:52.495 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj--eSSeHtXNH35Z6E off=56118 len=316214 2024-09-13T00:15:52.497 INFO:tasks.workunit.client.0.smithi136.stdout:op 4401 completed, throughput=5MB/sec 2024-09-13T00:15:52.497 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-MqbUTs76LysuMEA off=7603488 len=394796 2024-09-13T00:15:52.499 INFO:tasks.workunit.client.0.smithi136.stdout:op 4402 completed, throughput=5MB/sec 2024-09-13T00:15:52.499 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-32USNrgVQ915eGZ off=908630 len=323093 2024-09-13T00:15:52.504 INFO:tasks.workunit.client.0.smithi136.stdout:op 4403 completed, throughput=5MB/sec 2024-09-13T00:15:52.504 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-0pKZ7qMfwIzm0YF off=14193149 len=1866586 2024-09-13T00:15:52.516 INFO:tasks.workunit.client.0.smithi136.stdout:op 4404 completed, throughput=5MB/sec 2024-09-13T00:15:52.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:15:52 smithi181 bash[95505]: ts=2024-09-13T00:15:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:15:53.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:15:52 smithi181 bash[67587]: cluster 2024-09-13T00:15:51.577572+0000 mgr.y (mgr.24661) 1162 : cluster [DBG] pgmap v1845: 105 pgs: 105 active+clean; 126 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 2.0 MiB/s rd, 3.2 MiB/s wr, 4 op/s 2024-09-13T00:15:53.257 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:15:52 smithi136 bash[76371]: cluster 2024-09-13T00:15:51.577572+0000 mgr.y (mgr.24661) 1162 : cluster [DBG] pgmap v1845: 105 pgs: 105 active+clean; 126 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 2.0 MiB/s rd, 3.2 MiB/s wr, 4 op/s 2024-09-13T00:15:53.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:15:52 smithi136 bash[75164]: cluster 2024-09-13T00:15:51.577572+0000 mgr.y (mgr.24661) 1162 : cluster [DBG] pgmap v1845: 105 pgs: 105 active+clean; 126 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 2.0 MiB/s rd, 3.2 MiB/s wr, 4 op/s 2024-09-13T00:15:53.516 INFO:tasks.workunit.client.0.smithi136.stdout: 843: throughput=4.99MB/sec pending data=0 2024-09-13T00:15:53.517 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-t0bY31V9FH_O2JE off=1612568 len=1065291 2024-09-13T00:15:53.528 INFO:tasks.workunit.client.0.smithi136.stdout:op 4405 completed, throughput=5MB/sec 2024-09-13T00:15:53.528 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-kJry5wPjc5GsPyv off=15295496 len=253449 2024-09-13T00:15:53.533 INFO:tasks.workunit.client.0.smithi136.stdout:op 4406 completed, throughput=5MB/sec 2024-09-13T00:15:53.533 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-rV8nUGSFLSV2Gxc off=2690227 len=1706455 2024-09-13T00:15:53.545 INFO:tasks.workunit.client.0.smithi136.stdout:op 4407 completed, throughput=5MB/sec 2024-09-13T00:15:53.545 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-eB7PecfZpdU3Vu7 off=292130 len=605578 2024-09-13T00:15:53.547 INFO:tasks.workunit.client.0.smithi136.stdout:op 4408 completed, throughput=5MB/sec 2024-09-13T00:15:53.547 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-sfOQV8Z-YTSX2d9 off=2368388 len=719827 2024-09-13T00:15:53.554 INFO:tasks.workunit.client.0.smithi136.stdout:op 4409 completed, throughput=5MB/sec 2024-09-13T00:15:53.554 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-XEFWl29ZvoukAGQ off=7236329 len=1187727 2024-09-13T00:15:53.563 INFO:tasks.workunit.client.0.smithi136.stdout:op 4410 completed, throughput=5MB/sec 2024-09-13T00:15:54.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:15:53 smithi181 bash[67587]: audit 2024-09-13T00:15:53.795705+0000 mon.a (mon.0) 2231 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:15:54.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:15:53 smithi136 bash[76371]: audit 2024-09-13T00:15:53.795705+0000 mon.a (mon.0) 2231 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:15:54.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:15:53 smithi136 bash[75164]: audit 2024-09-13T00:15:53.795705+0000 mon.a (mon.0) 2231 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:15:54.563 INFO:tasks.workunit.client.0.smithi136.stdout: 844: throughput=4.99MB/sec pending data=0 2024-09-13T00:15:54.563 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-iD-hD-uaNcIDwre off=815821 len=432865 2024-09-13T00:15:54.565 INFO:tasks.workunit.client.0.smithi136.stdout:op 4411 completed, throughput=5MB/sec 2024-09-13T00:15:54.565 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-S1npyQZsdYRvB0S off=1429609 len=764926 2024-09-13T00:15:54.573 INFO:tasks.workunit.client.0.smithi136.stdout:op 4412 completed, throughput=5MB/sec 2024-09-13T00:15:54.574 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-n1sc93HhtYvAJ_A off=19863268 len=1089948 2024-09-13T00:15:54.582 INFO:tasks.workunit.client.0.smithi136.stdout:op 4413 completed, throughput=5MB/sec 2024-09-13T00:15:54.582 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-dB7BXuaAWt45GD8 off=6508035 len=1545941 2024-09-13T00:15:54.596 INFO:tasks.workunit.client.0.smithi136.stdout:op 4414 completed, throughput=5MB/sec 2024-09-13T00:15:54.596 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-otgWj3xchPSFbHv off=284067 len=1177566 2024-09-13T00:15:54.600 INFO:tasks.workunit.client.0.smithi136.stdout:op 4415 completed, throughput=5MB/sec 2024-09-13T00:15:54.600 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-z_VLFu6-2q0rDrB off=11209301 len=642076 2024-09-13T00:15:54.606 INFO:tasks.workunit.client.0.smithi136.stdout:op 4416 completed, throughput=5MB/sec 2024-09-13T00:15:55.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:15:54 smithi181 bash[67587]: cluster 2024-09-13T00:15:53.578726+0000 mgr.y (mgr.24661) 1163 : cluster [DBG] pgmap v1846: 105 pgs: 105 active+clean; 126 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.5 MiB/s wr, 4 op/s 2024-09-13T00:15:55.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:15:54 smithi136 bash[75164]: cluster 2024-09-13T00:15:53.578726+0000 mgr.y (mgr.24661) 1163 : cluster [DBG] pgmap v1846: 105 pgs: 105 active+clean; 126 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.5 MiB/s wr, 4 op/s 2024-09-13T00:15:55.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:15:54 smithi136 bash[76371]: cluster 2024-09-13T00:15:53.578726+0000 mgr.y (mgr.24661) 1163 : cluster [DBG] pgmap v1846: 105 pgs: 105 active+clean; 126 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.5 MiB/s wr, 4 op/s 2024-09-13T00:15:55.607 INFO:tasks.workunit.client.0.smithi136.stdout: 845: throughput=4.99MB/sec pending data=0 2024-09-13T00:15:55.607 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-b2ZcbDQ7Vzc7FPt off=6094431 len=977106 2024-09-13T00:15:55.615 INFO:tasks.workunit.client.0.smithi136.stdout:op 4417 completed, throughput=5MB/sec 2024-09-13T00:15:55.616 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-zziII2FMztKycAW off=14539322 len=1228180 2024-09-13T00:15:55.625 INFO:tasks.workunit.client.0.smithi136.stdout:op 4418 completed, throughput=5MB/sec 2024-09-13T00:15:55.625 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-6YkTR05rO8nxQTs off=535558 len=949473 2024-09-13T00:15:55.631 INFO:tasks.workunit.client.0.smithi136.stdout:op 4419 completed, throughput=5MB/sec 2024-09-13T00:15:55.631 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj--C0BOycTdn_vEvr off=11378250 len=1770752 2024-09-13T00:15:55.645 INFO:tasks.workunit.client.0.smithi136.stdout:op 4420 completed, throughput=5MB/sec 2024-09-13T00:15:55.645 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-l5JVPLDt15k8LRC off=24461488 len=703271 2024-09-13T00:15:55.647 INFO:tasks.workunit.client.0.smithi136.stdout:op 4421 completed, throughput=5MB/sec 2024-09-13T00:15:56.648 INFO:tasks.workunit.client.0.smithi136.stdout: 846: throughput=5MB/sec pending data=0 2024-09-13T00:15:56.648 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-JloXCs6ngJPy5k1 off=10424947 len=1579026 2024-09-13T00:15:56.658 INFO:tasks.workunit.client.0.smithi136.stdout:op 4422 completed, throughput=5MB/sec 2024-09-13T00:15:56.658 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-vioMbLYMMGcu8Az off=9744699 len=271727 2024-09-13T00:15:56.663 INFO:tasks.workunit.client.0.smithi136.stdout:op 4423 completed, throughput=5MB/sec 2024-09-13T00:15:56.663 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-k_rHrCz-EB4iQXb off=2748697 len=207795 2024-09-13T00:15:56.667 INFO:tasks.workunit.client.0.smithi136.stdout:op 4424 completed, throughput=5MB/sec 2024-09-13T00:15:56.667 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-K_8pF2BCAAR8ysY off=16199742 len=576856 2024-09-13T00:15:56.668 INFO:tasks.workunit.client.0.smithi136.stdout:op 4425 completed, throughput=5MB/sec 2024-09-13T00:15:56.668 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-V2pUNUrmb95ihk9 off=16370401 len=970908 2024-09-13T00:15:56.675 INFO:tasks.workunit.client.0.smithi136.stdout:op 4426 completed, throughput=5MB/sec 2024-09-13T00:15:56.675 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-PHuU9Mc4i3j7wCF off=1743093 len=1653742 2024-09-13T00:15:56.676 INFO:tasks.workunit.client.0.smithi136.stdout:op 4427 completed, throughput=5MB/sec 2024-09-13T00:15:57.257 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:15:56 smithi136 bash[75164]: cluster 2024-09-13T00:15:55.579688+0000 mgr.y (mgr.24661) 1164 : cluster [DBG] pgmap v1847: 105 pgs: 105 active+clean; 126 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 1.5 MiB/s rd, 3.3 MiB/s wr, 4 op/s 2024-09-13T00:15:57.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:15:56 smithi136 bash[75164]: audit 2024-09-13T00:15:56.261143+0000 mon.a (mon.0) 2232 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:15:57.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:15:56 smithi136 bash[75164]: audit 2024-09-13T00:15:56.607252+0000 mon.a (mon.0) 2233 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:15:57.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:15:56 smithi136 bash[75164]: audit 2024-09-13T00:15:56.614512+0000 mon.a (mon.0) 2234 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:15:57.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:15:56 smithi136 bash[76371]: cluster 2024-09-13T00:15:55.579688+0000 mgr.y (mgr.24661) 1164 : cluster [DBG] pgmap v1847: 105 pgs: 105 active+clean; 126 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 1.5 MiB/s rd, 3.3 MiB/s wr, 4 op/s 2024-09-13T00:15:57.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:15:56 smithi136 bash[76371]: audit 2024-09-13T00:15:56.261143+0000 mon.a (mon.0) 2232 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:15:57.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:15:56 smithi136 bash[76371]: audit 2024-09-13T00:15:56.607252+0000 mon.a (mon.0) 2233 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:15:57.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:15:56 smithi136 bash[76371]: audit 2024-09-13T00:15:56.614512+0000 mon.a (mon.0) 2234 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:15:57.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:15:56 smithi181 bash[67587]: cluster 2024-09-13T00:15:55.579688+0000 mgr.y (mgr.24661) 1164 : cluster [DBG] pgmap v1847: 105 pgs: 105 active+clean; 126 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 1.5 MiB/s rd, 3.3 MiB/s wr, 4 op/s 2024-09-13T00:15:57.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:15:56 smithi181 bash[67587]: audit 2024-09-13T00:15:56.261143+0000 mon.a (mon.0) 2232 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:15:57.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:15:56 smithi181 bash[67587]: audit 2024-09-13T00:15:56.607252+0000 mon.a (mon.0) 2233 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:15:57.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:15:56 smithi181 bash[67587]: audit 2024-09-13T00:15:56.614512+0000 mon.a (mon.0) 2234 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:15:57.677 INFO:tasks.workunit.client.0.smithi136.stdout: 847: throughput=4.99MB/sec pending data=0 2024-09-13T00:15:57.677 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-2Ij_PCwNOaAp9XE off=4107015 len=409965 2024-09-13T00:15:57.683 INFO:tasks.workunit.client.0.smithi136.stdout:op 4428 completed, throughput=5MB/sec 2024-09-13T00:15:57.683 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-3GNMmTarIdPvigC off=721939 len=2063658 2024-09-13T00:15:57.695 INFO:tasks.workunit.client.0.smithi136.stdout:op 4429 completed, throughput=5MB/sec 2024-09-13T00:15:57.695 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-y5Am3C1zJc-8uDs off=12773978 len=222834 2024-09-13T00:15:57.697 INFO:tasks.workunit.client.0.smithi136.stdout:op 4430 completed, throughput=5MB/sec 2024-09-13T00:15:57.697 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-X6X_Oq0RwilG5Ox off=3444904 len=1803490 2024-09-13T00:15:57.700 INFO:tasks.workunit.client.0.smithi136.stdout:op 4431 completed, throughput=5MB/sec 2024-09-13T00:15:57.701 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-fty-PcefAXjWZEU off=17414 len=2068479 2024-09-13T00:15:57.714 INFO:tasks.workunit.client.0.smithi136.stdout:op 4432 completed, throughput=5MB/sec 2024-09-13T00:15:58.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:15:57 smithi136 bash[75164]: audit 2024-09-13T00:15:57.027243+0000 mon.a (mon.0) 2235 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:15:58.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:15:57 smithi136 bash[75164]: audit 2024-09-13T00:15:57.028689+0000 mon.a (mon.0) 2236 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:15:58.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:15:57 smithi136 bash[75164]: audit 2024-09-13T00:15:57.037143+0000 mon.a (mon.0) 2237 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:15:58.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:15:57 smithi136 bash[76371]: audit 2024-09-13T00:15:57.027243+0000 mon.a (mon.0) 2235 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:15:58.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:15:57 smithi136 bash[76371]: audit 2024-09-13T00:15:57.028689+0000 mon.a (mon.0) 2236 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:15:58.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:15:57 smithi136 bash[76371]: audit 2024-09-13T00:15:57.037143+0000 mon.a (mon.0) 2237 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:15:58.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:15:57 smithi181 bash[67587]: audit 2024-09-13T00:15:57.027243+0000 mon.a (mon.0) 2235 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:15:58.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:15:57 smithi181 bash[67587]: audit 2024-09-13T00:15:57.028689+0000 mon.a (mon.0) 2236 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:15:58.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:15:57 smithi181 bash[67587]: audit 2024-09-13T00:15:57.037143+0000 mon.a (mon.0) 2237 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:15:58.714 INFO:tasks.workunit.client.0.smithi136.stdout: 848: throughput=5MB/sec pending data=0 2024-09-13T00:15:58.715 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-RHPTp88XutXerwQ off=23612414 len=1581964 2024-09-13T00:15:58.726 INFO:tasks.workunit.client.0.smithi136.stdout:op 4433 completed, throughput=5MB/sec 2024-09-13T00:15:58.726 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-A2TjON7cOmBwPjI off=14343849 len=1924928 2024-09-13T00:15:58.740 INFO:tasks.workunit.client.0.smithi136.stdout:op 4434 completed, throughput=5MB/sec 2024-09-13T00:15:58.741 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-6pfxh-BaV8-WuWA off=7569227 len=740875 2024-09-13T00:15:58.747 INFO:tasks.workunit.client.0.smithi136.stdout:op 4435 completed, throughput=5MB/sec 2024-09-13T00:15:59.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:15:59 smithi181 bash[67587]: cluster 2024-09-13T00:15:57.580791+0000 mgr.y (mgr.24661) 1165 : cluster [DBG] pgmap v1848: 105 pgs: 105 active+clean; 126 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 1.6 MiB/s rd, 3.4 MiB/s wr, 4 op/s 2024-09-13T00:15:59.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:15:59 smithi136 bash[76371]: cluster 2024-09-13T00:15:57.580791+0000 mgr.y (mgr.24661) 1165 : cluster [DBG] pgmap v1848: 105 pgs: 105 active+clean; 126 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 1.6 MiB/s rd, 3.4 MiB/s wr, 4 op/s 2024-09-13T00:15:59.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:15:59 smithi136 bash[75164]: cluster 2024-09-13T00:15:57.580791+0000 mgr.y (mgr.24661) 1165 : cluster [DBG] pgmap v1848: 105 pgs: 105 active+clean; 126 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 1.6 MiB/s rd, 3.4 MiB/s wr, 4 op/s 2024-09-13T00:15:59.747 INFO:tasks.workunit.client.0.smithi136.stdout: 849: throughput=4.99MB/sec pending data=0 2024-09-13T00:15:59.747 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-jfzdEp7J4yB4bb- off=2538073 len=1298562 2024-09-13T00:15:59.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:15:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:15:59] "GET /metrics HTTP/1.1" 200 34140 "" "Prometheus/2.43.0" 2024-09-13T00:15:59.759 INFO:tasks.workunit.client.0.smithi136.stdout:op 4436 completed, throughput=5MB/sec 2024-09-13T00:15:59.759 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-7xN81kC9AoncF0i off=2123992 len=1709201 2024-09-13T00:15:59.764 INFO:tasks.workunit.client.0.smithi136.stdout:op 4437 completed, throughput=5MB/sec 2024-09-13T00:15:59.764 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-AELgp3P7vLrgkHw off=2183111 len=30234 2024-09-13T00:15:59.765 INFO:tasks.workunit.client.0.smithi136.stdout:op 4438 completed, throughput=5MB/sec 2024-09-13T00:15:59.765 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-64Ltx-fOiGhhCnv off=4466412 len=1421211 2024-09-13T00:15:59.775 INFO:tasks.workunit.client.0.smithi136.stdout:op 4439 completed, throughput=5MB/sec 2024-09-13T00:15:59.775 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-W13OyQhwBC8_QdY off=114752 len=1022712 2024-09-13T00:15:59.784 INFO:tasks.workunit.client.0.smithi136.stdout:op 4440 completed, throughput=5MB/sec 2024-09-13T00:16:00.784 INFO:tasks.workunit.client.0.smithi136.stdout: 850: throughput=4.99MB/sec pending data=0 2024-09-13T00:16:00.785 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-aHEP5iTdNwyhgud off=10637717 len=514790 2024-09-13T00:16:00.790 INFO:tasks.workunit.client.0.smithi136.stdout:op 4441 completed, throughput=5MB/sec 2024-09-13T00:16:00.791 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-_M9Bnj3SKESZU56 off=3031150 len=667234 2024-09-13T00:16:00.793 INFO:tasks.workunit.client.0.smithi136.stdout:op 4442 completed, throughput=5MB/sec 2024-09-13T00:16:00.793 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-G_WO8YZnDyXGp9m off=18467800 len=1428585 2024-09-13T00:16:00.804 INFO:tasks.workunit.client.0.smithi136.stdout:op 4443 completed, throughput=5MB/sec 2024-09-13T00:16:00.804 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Bz5r3QlTcJV2Lbm off=2688351 len=1485727 2024-09-13T00:16:00.815 INFO:tasks.workunit.client.0.smithi136.stdout:op 4444 completed, throughput=5MB/sec 2024-09-13T00:16:00.816 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-tj_jFGQZiFINZWV off=9857137 len=1642182 2024-09-13T00:16:00.821 INFO:tasks.workunit.client.0.smithi136.stdout:op 4445 completed, throughput=5MB/sec 2024-09-13T00:16:01.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:16:01 smithi181 bash[67587]: cluster 2024-09-13T00:15:59.581731+0000 mgr.y (mgr.24661) 1166 : cluster [DBG] pgmap v1849: 105 pgs: 105 active+clean; 126 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 1.6 MiB/s rd, 2.8 MiB/s wr, 4 op/s 2024-09-13T00:16:01.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:16:01 smithi136 bash[75164]: cluster 2024-09-13T00:15:59.581731+0000 mgr.y (mgr.24661) 1166 : cluster [DBG] pgmap v1849: 105 pgs: 105 active+clean; 126 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 1.6 MiB/s rd, 2.8 MiB/s wr, 4 op/s 2024-09-13T00:16:01.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:16:01 smithi136 bash[76371]: cluster 2024-09-13T00:15:59.581731+0000 mgr.y (mgr.24661) 1166 : cluster [DBG] pgmap v1849: 105 pgs: 105 active+clean; 126 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 1.6 MiB/s rd, 2.8 MiB/s wr, 4 op/s 2024-09-13T00:16:01.821 INFO:tasks.workunit.client.0.smithi136.stdout: 851: throughput=5MB/sec pending data=0 2024-09-13T00:16:01.821 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj--LYCQXJNs0jnhev off=4532844 len=1586378 2024-09-13T00:16:01.833 INFO:tasks.workunit.client.0.smithi136.stdout:op 4446 completed, throughput=5MB/sec 2024-09-13T00:16:01.833 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-oLuTdyqcH6urrPy off=1195941 len=1297603 2024-09-13T00:16:01.835 INFO:tasks.workunit.client.0.smithi136.stdout:op 4447 completed, throughput=5MB/sec 2024-09-13T00:16:01.835 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-u4ShhHaivassJ7x off=5468153 len=1711353 2024-09-13T00:16:01.839 INFO:tasks.workunit.client.0.smithi136.stdout:op 4448 completed, throughput=5MB/sec 2024-09-13T00:16:02.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:16:02 smithi181 bash[95505]: ts=2024-09-13T00:16:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:16:02.840 INFO:tasks.workunit.client.0.smithi136.stdout: 852: throughput=4.99MB/sec pending data=0 2024-09-13T00:16:02.840 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-KnbHSgwK4q_fe1u off=4897243 len=1709893 2024-09-13T00:16:02.851 INFO:tasks.workunit.client.0.smithi136.stdout:op 4449 completed, throughput=5MB/sec 2024-09-13T00:16:02.852 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-rWf00dSoLif6gZ2 off=3562908 len=1819589 2024-09-13T00:16:02.865 INFO:tasks.workunit.client.0.smithi136.stdout:op 4450 completed, throughput=5MB/sec 2024-09-13T00:16:02.865 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-lKWSwey26r0GiJG off=12638794 len=1734181 2024-09-13T00:16:02.867 INFO:tasks.workunit.client.0.smithi136.stdout:op 4451 completed, throughput=5MB/sec 2024-09-13T00:16:02.867 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-nJkqK-X2bayXHEg off=3462977 len=562672 2024-09-13T00:16:02.874 INFO:tasks.workunit.client.0.smithi136.stdout:op 4452 completed, throughput=5MB/sec 2024-09-13T00:16:03.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:16:03 smithi181 bash[67587]: cluster 2024-09-13T00:16:01.582694+0000 mgr.y (mgr.24661) 1167 : cluster [DBG] pgmap v1850: 105 pgs: 105 active+clean; 126 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.1 MiB/s wr, 4 op/s 2024-09-13T00:16:03.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:16:03 smithi136 bash[75164]: cluster 2024-09-13T00:16:01.582694+0000 mgr.y (mgr.24661) 1167 : cluster [DBG] pgmap v1850: 105 pgs: 105 active+clean; 126 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.1 MiB/s wr, 4 op/s 2024-09-13T00:16:03.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:16:03 smithi136 bash[76371]: cluster 2024-09-13T00:16:01.582694+0000 mgr.y (mgr.24661) 1167 : cluster [DBG] pgmap v1850: 105 pgs: 105 active+clean; 126 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.1 MiB/s wr, 4 op/s 2024-09-13T00:16:03.874 INFO:tasks.workunit.client.0.smithi136.stdout: 853: throughput=4.99MB/sec pending data=0 2024-09-13T00:16:03.874 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-VMZ1uIGzSstotkf off=8443311 len=61114 2024-09-13T00:16:03.879 INFO:tasks.workunit.client.0.smithi136.stdout:op 4453 completed, throughput=4.99MB/sec 2024-09-13T00:16:03.879 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-eCn7GnEEmWl1HvL off=9904934 len=2084639 2024-09-13T00:16:03.884 INFO:tasks.workunit.client.0.smithi136.stdout:op 4454 completed, throughput=5MB/sec 2024-09-13T00:16:03.884 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-7j2bk6pssUBTXgx off=2525290 len=1439404 2024-09-13T00:16:03.897 INFO:tasks.workunit.client.0.smithi136.stdout:op 4455 completed, throughput=5MB/sec 2024-09-13T00:16:03.897 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-e-HT8UNgZ3FyL4u off=1928638 len=1817188 2024-09-13T00:16:03.900 INFO:tasks.workunit.client.0.smithi136.stdout:op 4456 completed, throughput=5MB/sec 2024-09-13T00:16:04.900 INFO:tasks.workunit.client.0.smithi136.stdout: 854: throughput=4.99MB/sec pending data=0 2024-09-13T00:16:04.900 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-r1JN863ICJMmTkS off=3130236 len=890941 2024-09-13T00:16:04.909 INFO:tasks.workunit.client.0.smithi136.stdout:op 4457 completed, throughput=5MB/sec 2024-09-13T00:16:04.909 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-uaDdGNBimyBNeU_ off=3518923 len=1261683 2024-09-13T00:16:04.912 INFO:tasks.workunit.client.0.smithi136.stdout:op 4458 completed, throughput=5MB/sec 2024-09-13T00:16:04.912 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-lf2TSQMdCe7JwIU off=1152897 len=1465572 2024-09-13T00:16:04.916 INFO:tasks.workunit.client.0.smithi136.stdout:op 4459 completed, throughput=5MB/sec 2024-09-13T00:16:04.917 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-adizo1aTh-WUHwP off=0 len=1073745 2024-09-13T00:16:04.924 INFO:tasks.workunit.client.0.smithi136.stdout:op 4460 completed, throughput=5MB/sec 2024-09-13T00:16:04.925 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ZIZFVdp20vqMmzp off=12166255 len=1480471 2024-09-13T00:16:04.936 INFO:tasks.workunit.client.0.smithi136.stdout:op 4461 completed, throughput=5MB/sec 2024-09-13T00:16:05.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:16:05 smithi181 bash[67587]: cluster 2024-09-13T00:16:03.583736+0000 mgr.y (mgr.24661) 1168 : cluster [DBG] pgmap v1851: 105 pgs: 105 active+clean; 126 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.8 MiB/s wr, 5 op/s 2024-09-13T00:16:05.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:16:05 smithi136 bash[75164]: cluster 2024-09-13T00:16:03.583736+0000 mgr.y (mgr.24661) 1168 : cluster [DBG] pgmap v1851: 105 pgs: 105 active+clean; 126 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.8 MiB/s wr, 5 op/s 2024-09-13T00:16:05.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:16:05 smithi136 bash[76371]: cluster 2024-09-13T00:16:03.583736+0000 mgr.y (mgr.24661) 1168 : cluster [DBG] pgmap v1851: 105 pgs: 105 active+clean; 126 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.8 MiB/s wr, 5 op/s 2024-09-13T00:16:05.937 INFO:tasks.workunit.client.0.smithi136.stdout: 855: throughput=5MB/sec pending data=0 2024-09-13T00:16:05.937 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-3SBoeSozEXdz3BW off=3062353 len=375258 2024-09-13T00:16:05.943 INFO:tasks.workunit.client.0.smithi136.stdout:op 4462 completed, throughput=5MB/sec 2024-09-13T00:16:05.943 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-o4WYg2lWwdltvfb off=444626 len=1612413 2024-09-13T00:16:05.949 INFO:tasks.workunit.client.0.smithi136.stdout:op 4463 completed, throughput=5MB/sec 2024-09-13T00:16:05.949 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-4zPna_yzPFWSS9f off=1641968 len=643158 2024-09-13T00:16:05.951 INFO:tasks.workunit.client.0.smithi136.stdout:op 4464 completed, throughput=5MB/sec 2024-09-13T00:16:05.951 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Z4wbb5ousMn_Z3T off=6920418 len=263851 2024-09-13T00:16:05.956 INFO:tasks.workunit.client.0.smithi136.stdout:op 4465 completed, throughput=5MB/sec 2024-09-13T00:16:05.957 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-0G5Lah0N_l4P-1l off=3258339 len=1089426 2024-09-13T00:16:05.965 INFO:tasks.workunit.client.0.smithi136.stdout:op 4466 completed, throughput=5MB/sec 2024-09-13T00:16:05.965 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-60sBX6pxbhzjKN8 off=345995 len=788883 2024-09-13T00:16:05.967 INFO:tasks.workunit.client.0.smithi136.stdout:op 4467 completed, throughput=5MB/sec 2024-09-13T00:16:06.969 INFO:tasks.workunit.client.0.smithi136.stdout: 856: throughput=5MB/sec pending data=0 2024-09-13T00:16:06.969 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-zZwnhM2X0DcydId off=9549374 len=408655 2024-09-13T00:16:06.970 INFO:tasks.workunit.client.0.smithi136.stdout:op 4468 completed, throughput=5MB/sec 2024-09-13T00:16:06.970 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-EOF1yYXaEwaIneY off=18760794 len=485031 2024-09-13T00:16:06.977 INFO:tasks.workunit.client.0.smithi136.stdout:op 4469 completed, throughput=5MB/sec 2024-09-13T00:16:06.977 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-zuiWox-fSLku3EB off=11392363 len=613286 2024-09-13T00:16:06.979 INFO:tasks.workunit.client.0.smithi136.stdout:op 4470 completed, throughput=5MB/sec 2024-09-13T00:16:06.979 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-lQ-u06ir2ZPFw6w off=3106506 len=1752059 2024-09-13T00:16:06.994 INFO:tasks.workunit.client.0.smithi136.stdout:op 4471 completed, throughput=5MB/sec 2024-09-13T00:16:06.994 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Dbv0_mRfsh1NWSB off=12325793 len=743668 2024-09-13T00:16:06.996 INFO:tasks.workunit.client.0.smithi136.stdout:op 4472 completed, throughput=5MB/sec 2024-09-13T00:16:06.996 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-fUo2AiMYsXAxRBj off=3493857 len=1669110 2024-09-13T00:16:07.000 INFO:tasks.workunit.client.0.smithi136.stdout:op 4473 completed, throughput=5MB/sec 2024-09-13T00:16:07.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:16:07 smithi181 bash[67587]: cluster 2024-09-13T00:16:05.584549+0000 mgr.y (mgr.24661) 1169 : cluster [DBG] pgmap v1852: 105 pgs: 105 active+clean; 126 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 1.3 MiB/s rd, 3.0 MiB/s wr, 4 op/s 2024-09-13T00:16:07.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:16:07 smithi136 bash[75164]: cluster 2024-09-13T00:16:05.584549+0000 mgr.y (mgr.24661) 1169 : cluster [DBG] pgmap v1852: 105 pgs: 105 active+clean; 126 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 1.3 MiB/s rd, 3.0 MiB/s wr, 4 op/s 2024-09-13T00:16:07.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:16:07 smithi136 bash[76371]: cluster 2024-09-13T00:16:05.584549+0000 mgr.y (mgr.24661) 1169 : cluster [DBG] pgmap v1852: 105 pgs: 105 active+clean; 126 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 1.3 MiB/s rd, 3.0 MiB/s wr, 4 op/s 2024-09-13T00:16:08.000 INFO:tasks.workunit.client.0.smithi136.stdout: 857: throughput=5MB/sec pending data=0 2024-09-13T00:16:08.000 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-xkXmxDPDQAbQ7ES off=12274286 len=1868543 2024-09-13T00:16:08.016 INFO:tasks.workunit.client.0.smithi136.stdout:op 4474 completed, throughput=5MB/sec 2024-09-13T00:16:08.017 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-adoiGVxc5AIR_yD off=14758702 len=168814 2024-09-13T00:16:08.018 INFO:tasks.workunit.client.0.smithi136.stdout:op 4475 completed, throughput=5MB/sec 2024-09-13T00:16:08.018 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-3cd3E3Eo9o7k5c3 off=5220431 len=1019544 2024-09-13T00:16:08.027 INFO:tasks.workunit.client.0.smithi136.stdout:op 4476 completed, throughput=5MB/sec 2024-09-13T00:16:08.027 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-9O2bfYUXeM1HqTW off=16088820 len=1831602 2024-09-13T00:16:08.031 INFO:tasks.workunit.client.0.smithi136.stdout:op 4477 completed, throughput=5MB/sec 2024-09-13T00:16:09.031 INFO:tasks.workunit.client.0.smithi136.stdout: 858: throughput=4.99MB/sec pending data=0 2024-09-13T00:16:09.031 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-NTt_klBM-decGaH off=5634949 len=373465 2024-09-13T00:16:09.038 INFO:tasks.workunit.client.0.smithi136.stdout:op 4478 completed, throughput=5MB/sec 2024-09-13T00:16:09.038 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-HKmmY_xsIiNa6CY off=298919 len=1096955 2024-09-13T00:16:09.048 INFO:tasks.workunit.client.0.smithi136.stdout:op 4479 completed, throughput=5MB/sec 2024-09-13T00:16:09.048 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-FHOfsD8-WkKbEwE off=2889159 len=1874363 2024-09-13T00:16:09.063 INFO:tasks.workunit.client.0.smithi136.stdout:op 4480 completed, throughput=5MB/sec 2024-09-13T00:16:09.063 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-mBYhuUOzqxbQ1Cl off=1558091 len=945657 2024-09-13T00:16:09.071 INFO:tasks.workunit.client.0.smithi136.stdout:op 4481 completed, throughput=5MB/sec 2024-09-13T00:16:09.071 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-E6DksNud2ceUUml off=5256269 len=1137290 2024-09-13T00:16:09.075 INFO:tasks.workunit.client.0.smithi136.stdout:op 4482 completed, throughput=5MB/sec 2024-09-13T00:16:09.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:16:09 smithi181 bash[67587]: cluster 2024-09-13T00:16:07.585565+0000 mgr.y (mgr.24661) 1170 : cluster [DBG] pgmap v1853: 105 pgs: 105 active+clean; 126 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 1.9 MiB/s rd, 2.9 MiB/s wr, 4 op/s 2024-09-13T00:16:09.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:16:09 smithi181 bash[67587]: audit 2024-09-13T00:16:08.796612+0000 mon.a (mon.0) 2238 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:16:09.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:16:09 smithi136 bash[76371]: cluster 2024-09-13T00:16:07.585565+0000 mgr.y (mgr.24661) 1170 : cluster [DBG] pgmap v1853: 105 pgs: 105 active+clean; 126 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 1.9 MiB/s rd, 2.9 MiB/s wr, 4 op/s 2024-09-13T00:16:09.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:16:09 smithi136 bash[76371]: audit 2024-09-13T00:16:08.796612+0000 mon.a (mon.0) 2238 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:16:09.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:16:09 smithi136 bash[75164]: cluster 2024-09-13T00:16:07.585565+0000 mgr.y (mgr.24661) 1170 : cluster [DBG] pgmap v1853: 105 pgs: 105 active+clean; 126 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 1.9 MiB/s rd, 2.9 MiB/s wr, 4 op/s 2024-09-13T00:16:09.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:16:09 smithi136 bash[75164]: audit 2024-09-13T00:16:08.796612+0000 mon.a (mon.0) 2238 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:16:09.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:16:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:16:09] "GET /metrics HTTP/1.1" 200 34141 "" "Prometheus/2.43.0" 2024-09-13T00:16:10.075 INFO:tasks.workunit.client.0.smithi136.stdout: 859: throughput=4.99MB/sec pending data=0 2024-09-13T00:16:10.076 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2wj-F7Ik58CyxI3 off=7369244 len=489008 2024-09-13T00:16:10.078 INFO:tasks.workunit.client.0.smithi136.stdout:op 4483 completed, throughput=5MB/sec 2024-09-13T00:16:10.078 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-wQW_ar52BwLiRCO off=1203949 len=790066 2024-09-13T00:16:10.086 INFO:tasks.workunit.client.0.smithi136.stdout:op 4484 completed, throughput=5MB/sec 2024-09-13T00:16:10.086 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-QooXyygxDL2DM-l off=18302769 len=1000371 2024-09-13T00:16:10.089 INFO:tasks.workunit.client.0.smithi136.stdout:op 4485 completed, throughput=5MB/sec 2024-09-13T00:16:10.089 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-fxi0dEtS6zkpDcd off=10713307 len=2026771 2024-09-13T00:16:10.104 INFO:tasks.workunit.client.0.smithi136.stdout:op 4486 completed, throughput=5MB/sec 2024-09-13T00:16:10.104 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-u3sXfRk_owy6Ink off=3869221 len=313115 2024-09-13T00:16:10.106 INFO:tasks.workunit.client.0.smithi136.stdout:op 4487 completed, throughput=5MB/sec 2024-09-13T00:16:10.106 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_3Qh6O__5GvdzYM off=4082500 len=1298371 2024-09-13T00:16:10.116 INFO:tasks.workunit.client.0.smithi136.stdout:op 4488 completed, throughput=5MB/sec 2024-09-13T00:16:11.116 INFO:tasks.workunit.client.0.smithi136.stdout: 860: throughput=5MB/sec pending data=0 2024-09-13T00:16:11.117 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-c79gEoJnQLO8R3O off=874940 len=477090 2024-09-13T00:16:11.119 INFO:tasks.workunit.client.0.smithi136.stdout:op 4489 completed, throughput=5MB/sec 2024-09-13T00:16:11.119 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-nPuAUm2D5OWIW2y off=10795342 len=1169012 2024-09-13T00:16:11.128 INFO:tasks.workunit.client.0.smithi136.stdout:op 4490 completed, throughput=5MB/sec 2024-09-13T00:16:11.128 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-9y4jcSB3yJxaZgg off=3866729 len=1689143 2024-09-13T00:16:11.140 INFO:tasks.workunit.client.0.smithi136.stdout:op 4491 completed, throughput=5MB/sec 2024-09-13T00:16:11.140 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-dxunP-WV6LVrmdY off=1137822 len=2033356 2024-09-13T00:16:11.145 INFO:tasks.workunit.client.0.smithi136.stdout:op 4492 completed, throughput=5MB/sec 2024-09-13T00:16:11.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:16:11 smithi181 bash[67587]: cluster 2024-09-13T00:16:09.586463+0000 mgr.y (mgr.24661) 1171 : cluster [DBG] pgmap v1854: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 1.8 MiB/s rd, 2.9 MiB/s wr, 4 op/s 2024-09-13T00:16:11.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:16:11 smithi136 bash[75164]: cluster 2024-09-13T00:16:09.586463+0000 mgr.y (mgr.24661) 1171 : cluster [DBG] pgmap v1854: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 1.8 MiB/s rd, 2.9 MiB/s wr, 4 op/s 2024-09-13T00:16:11.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:16:11 smithi136 bash[76371]: cluster 2024-09-13T00:16:09.586463+0000 mgr.y (mgr.24661) 1171 : cluster [DBG] pgmap v1854: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 1.8 MiB/s rd, 2.9 MiB/s wr, 4 op/s 2024-09-13T00:16:12.145 INFO:tasks.workunit.client.0.smithi136.stdout: 861: throughput=5MB/sec pending data=0 2024-09-13T00:16:12.145 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-lNmCSy13-gQ8uEV off=338952 len=1162717 2024-09-13T00:16:12.154 INFO:tasks.workunit.client.0.smithi136.stdout:op 4493 completed, throughput=5MB/sec 2024-09-13T00:16:12.154 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-T3af_Not45GEzby off=1207063 len=988538 2024-09-13T00:16:12.157 INFO:tasks.workunit.client.0.smithi136.stdout:op 4494 completed, throughput=5MB/sec 2024-09-13T00:16:12.157 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ebGmVRCpZJr0T2P off=3337883 len=1992386 2024-09-13T00:16:12.163 INFO:tasks.workunit.client.0.smithi136.stdout:op 4495 completed, throughput=5MB/sec 2024-09-13T00:16:12.163 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-5ymzdokwgCPaNHK off=4144944 len=1929753 2024-09-13T00:16:12.168 INFO:tasks.workunit.client.0.smithi136.stdout:op 4496 completed, throughput=5MB/sec 2024-09-13T00:16:12.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:16:12 smithi181 bash[95505]: ts=2024-09-13T00:16:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:16:13.168 INFO:tasks.workunit.client.0.smithi136.stdout: 862: throughput=5MB/sec pending data=0 2024-09-13T00:16:13.168 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-OOCBhBL7PBVN9hS off=512175 len=899438 2024-09-13T00:16:13.176 INFO:tasks.workunit.client.0.smithi136.stdout:op 4497 completed, throughput=5MB/sec 2024-09-13T00:16:13.176 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-x9CxXMVLzqJMIyY off=8877415 len=1171641 2024-09-13T00:16:13.179 INFO:tasks.workunit.client.0.smithi136.stdout:op 4498 completed, throughput=5MB/sec 2024-09-13T00:16:13.180 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-RXo6d6x5HBiN7Nt off=4467114 len=962845 2024-09-13T00:16:13.196 INFO:tasks.workunit.client.0.smithi136.stdout:op 4499 completed, throughput=5MB/sec 2024-09-13T00:16:13.196 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-f5RyCJD9HfC6v7w off=691474 len=180459 2024-09-13T00:16:13.201 INFO:tasks.workunit.client.0.smithi136.stdout:op 4500 completed, throughput=5MB/sec 2024-09-13T00:16:13.201 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-OKZfq50nXUT9R9U off=4855851 len=1272814 2024-09-13T00:16:13.210 INFO:tasks.workunit.client.0.smithi136.stdout:op 4501 completed, throughput=5MB/sec 2024-09-13T00:16:13.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:16:13 smithi181 bash[67587]: cluster 2024-09-13T00:16:11.587486+0000 mgr.y (mgr.24661) 1172 : cluster [DBG] pgmap v1855: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.1 MiB/s rd, 3.1 MiB/s wr, 4 op/s 2024-09-13T00:16:13.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:16:13 smithi136 bash[75164]: cluster 2024-09-13T00:16:11.587486+0000 mgr.y (mgr.24661) 1172 : cluster [DBG] pgmap v1855: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.1 MiB/s rd, 3.1 MiB/s wr, 4 op/s 2024-09-13T00:16:13.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:16:13 smithi136 bash[76371]: cluster 2024-09-13T00:16:11.587486+0000 mgr.y (mgr.24661) 1172 : cluster [DBG] pgmap v1855: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.1 MiB/s rd, 3.1 MiB/s wr, 4 op/s 2024-09-13T00:16:14.211 INFO:tasks.workunit.client.0.smithi136.stdout: 863: throughput=4.99MB/sec pending data=0 2024-09-13T00:16:14.211 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-m6hGEZNV_LVWwzg off=7482694 len=599903 2024-09-13T00:16:14.217 INFO:tasks.workunit.client.0.smithi136.stdout:op 4502 completed, throughput=5MB/sec 2024-09-13T00:16:14.217 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ZsZJjHyT75Eav3D off=420415 len=1890424 2024-09-13T00:16:14.232 INFO:tasks.workunit.client.0.smithi136.stdout:op 4503 completed, throughput=5MB/sec 2024-09-13T00:16:14.232 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-2mVAORFkhr1PLHn off=912984 len=1130399 2024-09-13T00:16:14.240 INFO:tasks.workunit.client.0.smithi136.stdout:op 4504 completed, throughput=5MB/sec 2024-09-13T00:16:14.240 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-O4CxvOPia8N5-og off=17338811 len=596590 2024-09-13T00:16:14.241 INFO:tasks.workunit.client.0.smithi136.stdout:op 4505 completed, throughput=5MB/sec 2024-09-13T00:16:14.241 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-GtdSZ_B48AvaZa1 off=21066463 len=854751 2024-09-13T00:16:14.250 INFO:tasks.workunit.client.0.smithi136.stdout:op 4506 completed, throughput=5MB/sec 2024-09-13T00:16:15.250 INFO:tasks.workunit.client.0.smithi136.stdout: 864: throughput=4.99MB/sec pending data=0 2024-09-13T00:16:15.251 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-WBFjuw3R3RTobgK off=8647929 len=778472 2024-09-13T00:16:15.253 INFO:tasks.workunit.client.0.smithi136.stdout:op 4507 completed, throughput=5MB/sec 2024-09-13T00:16:15.254 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-BiN3AqMWRBNiGHJ off=16020839 len=57009 2024-09-13T00:16:15.255 INFO:tasks.workunit.client.0.smithi136.stdout:op 4508 completed, throughput=5MB/sec 2024-09-13T00:16:15.255 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-IHNrB_FWRIjOtYU off=3341874 len=1735380 2024-09-13T00:16:15.259 INFO:tasks.workunit.client.0.smithi136.stdout:op 4509 completed, throughput=5MB/sec 2024-09-13T00:16:15.259 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-SudKhljMtw87KL9 off=806950 len=1791371 2024-09-13T00:16:15.273 INFO:tasks.workunit.client.0.smithi136.stdout:op 4510 completed, throughput=5MB/sec 2024-09-13T00:16:15.273 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Rbkx2dOdoDzMtlB off=2302159 len=174033 2024-09-13T00:16:15.278 INFO:tasks.workunit.client.0.smithi136.stdout:op 4511 completed, throughput=5MB/sec 2024-09-13T00:16:15.278 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-vJC3pFLKTJo5bQq off=8297125 len=1126849 2024-09-13T00:16:15.280 INFO:tasks.workunit.client.0.smithi136.stdout:op 4512 completed, throughput=5MB/sec 2024-09-13T00:16:15.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:16:15 smithi181 bash[67587]: cluster 2024-09-13T00:16:13.588538+0000 mgr.y (mgr.24661) 1173 : cluster [DBG] pgmap v1856: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.4 MiB/s rd, 3.5 MiB/s wr, 5 op/s 2024-09-13T00:16:15.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:16:15 smithi136 bash[75164]: cluster 2024-09-13T00:16:13.588538+0000 mgr.y (mgr.24661) 1173 : cluster [DBG] pgmap v1856: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.4 MiB/s rd, 3.5 MiB/s wr, 5 op/s 2024-09-13T00:16:15.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:16:15 smithi136 bash[76371]: cluster 2024-09-13T00:16:13.588538+0000 mgr.y (mgr.24661) 1173 : cluster [DBG] pgmap v1856: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.4 MiB/s rd, 3.5 MiB/s wr, 5 op/s 2024-09-13T00:16:16.280 INFO:tasks.workunit.client.0.smithi136.stdout: 865: throughput=4.99MB/sec pending data=0 2024-09-13T00:16:16.281 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-WLVt43Qjn1B7xZB off=17102438 len=1796808 2024-09-13T00:16:16.285 INFO:tasks.workunit.client.0.smithi136.stdout:op 4513 completed, throughput=5MB/sec 2024-09-13T00:16:16.285 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_1derCP0tUDcfFW off=1923932 len=525186 2024-09-13T00:16:16.291 INFO:tasks.workunit.client.0.smithi136.stdout:op 4514 completed, throughput=5MB/sec 2024-09-13T00:16:16.291 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-2KsbQgyVybJbF2q off=6928688 len=1304746 2024-09-13T00:16:16.301 INFO:tasks.workunit.client.0.smithi136.stdout:op 4515 completed, throughput=5MB/sec 2024-09-13T00:16:16.301 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-bW_gmj5_ebuHocX off=12213703 len=352269 2024-09-13T00:16:16.307 INFO:tasks.workunit.client.0.smithi136.stdout:op 4516 completed, throughput=5MB/sec 2024-09-13T00:16:16.307 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-O473V7Zh0bd5tfZ off=4579657 len=677607 2024-09-13T00:16:16.314 INFO:tasks.workunit.client.0.smithi136.stdout:op 4517 completed, throughput=5MB/sec 2024-09-13T00:16:16.314 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-IQwXGkXUYdVPLda off=15367795 len=1239782 2024-09-13T00:16:16.315 INFO:tasks.workunit.client.0.smithi136.stdout:op 4518 completed, throughput=5MB/sec 2024-09-13T00:16:17.316 INFO:tasks.workunit.client.0.smithi136.stdout: 866: throughput=5MB/sec pending data=0 2024-09-13T00:16:17.316 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-gAmMKZpJWQLWbjA off=6465718 len=685871 2024-09-13T00:16:17.323 INFO:tasks.workunit.client.0.smithi136.stdout:op 4519 completed, throughput=5MB/sec 2024-09-13T00:16:17.324 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-NA6EZMQDf_yisAk off=5099151 len=1664224 2024-09-13T00:16:17.327 INFO:tasks.workunit.client.0.smithi136.stdout:op 4520 completed, throughput=5MB/sec 2024-09-13T00:16:17.328 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-tshLvmxWiYHXJFs off=13504624 len=246307 2024-09-13T00:16:17.329 INFO:tasks.workunit.client.0.smithi136.stdout:op 4521 completed, throughput=5MB/sec 2024-09-13T00:16:17.329 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-tncM78rrZSIYDTC off=15837181 len=1622132 2024-09-13T00:16:17.340 INFO:tasks.workunit.client.0.smithi136.stdout:op 4522 completed, throughput=5MB/sec 2024-09-13T00:16:17.340 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-y1LzOmJDein_7by off=9012119 len=1643907 2024-09-13T00:16:17.343 INFO:tasks.workunit.client.0.smithi136.stdout:op 4523 completed, throughput=5MB/sec 2024-09-13T00:16:17.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:16:17 smithi181 bash[67587]: cluster 2024-09-13T00:16:15.589364+0000 mgr.y (mgr.24661) 1174 : cluster [DBG] pgmap v1857: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:16:17.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:16:17 smithi136 bash[75164]: cluster 2024-09-13T00:16:15.589364+0000 mgr.y (mgr.24661) 1174 : cluster [DBG] pgmap v1857: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:16:17.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:16:17 smithi136 bash[76371]: cluster 2024-09-13T00:16:15.589364+0000 mgr.y (mgr.24661) 1174 : cluster [DBG] pgmap v1857: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:16:18.344 INFO:tasks.workunit.client.0.smithi136.stdout: 867: throughput=5MB/sec pending data=0 2024-09-13T00:16:18.344 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-QTmPdY_DBOmgTPS off=0 len=1083800 2024-09-13T00:16:18.347 INFO:tasks.workunit.client.0.smithi136.stdout:op 4524 completed, throughput=5MB/sec 2024-09-13T00:16:18.347 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-fyDQBefeZku83pu off=1337441 len=610724 2024-09-13T00:16:18.355 INFO:tasks.workunit.client.0.smithi136.stdout:op 4525 completed, throughput=5MB/sec 2024-09-13T00:16:18.355 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Nq7PadkmezlPnjn off=5075532 len=951912 2024-09-13T00:16:18.363 INFO:tasks.workunit.client.0.smithi136.stdout:op 4526 completed, throughput=5MB/sec 2024-09-13T00:16:18.364 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-4Hah-fVdwHa2rEr off=13128484 len=1732140 2024-09-13T00:16:18.376 INFO:tasks.workunit.client.0.smithi136.stdout:op 4527 completed, throughput=5MB/sec 2024-09-13T00:16:19.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:16:19 smithi181 bash[67587]: cluster 2024-09-13T00:16:17.590407+0000 mgr.y (mgr.24661) 1175 : cluster [DBG] pgmap v1858: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:16:19.376 INFO:tasks.workunit.client.0.smithi136.stdout: 868: throughput=4.99MB/sec pending data=0 2024-09-13T00:16:19.376 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-LT2OXgyU7fJ8poQ off=19322966 len=412147 2024-09-13T00:16:19.378 INFO:tasks.workunit.client.0.smithi136.stdout:op 4528 completed, throughput=5MB/sec 2024-09-13T00:16:19.378 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-w5bLcRqbYiUZB55 off=8249837 len=1347460 2024-09-13T00:16:19.382 INFO:tasks.workunit.client.0.smithi136.stdout:op 4529 completed, throughput=5MB/sec 2024-09-13T00:16:19.382 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-cueNb95zauvqFdN off=14857843 len=1436685 2024-09-13T00:16:19.395 INFO:tasks.workunit.client.0.smithi136.stdout:op 4530 completed, throughput=5MB/sec 2024-09-13T00:16:19.395 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj--ivS5BngKl5-4qH off=7292566 len=1956718 2024-09-13T00:16:19.399 INFO:tasks.workunit.client.0.smithi136.stdout:op 4531 completed, throughput=5MB/sec 2024-09-13T00:16:19.412 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:16:19 smithi136 bash[75164]: cluster 2024-09-13T00:16:17.590407+0000 mgr.y (mgr.24661) 1175 : cluster [DBG] pgmap v1858: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:16:19.412 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:16:19 smithi136 bash[76371]: cluster 2024-09-13T00:16:17.590407+0000 mgr.y (mgr.24661) 1175 : cluster [DBG] pgmap v1858: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:16:19.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:16:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:16:19] "GET /metrics HTTP/1.1" 200 34141 "" "Prometheus/2.43.0" 2024-09-13T00:16:20.400 INFO:tasks.workunit.client.0.smithi136.stdout: 869: throughput=4.99MB/sec pending data=0 2024-09-13T00:16:20.400 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_OQoyUYqdS8b5pU off=2677909 len=1920513 2024-09-13T00:16:20.413 INFO:tasks.workunit.client.0.smithi136.stdout:op 4532 completed, throughput=5MB/sec 2024-09-13T00:16:20.413 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-NPzeYPe2sNJDJe8 off=7407945 len=21700 2024-09-13T00:16:20.413 INFO:tasks.workunit.client.0.smithi136.stdout:op 4533 completed, throughput=5MB/sec 2024-09-13T00:16:20.414 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Odx8eqyAgZR3vW5 off=4520878 len=1831833 2024-09-13T00:16:20.426 INFO:tasks.workunit.client.0.smithi136.stdout:op 4534 completed, throughput=5MB/sec 2024-09-13T00:16:20.426 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-oldX9u8JHmvEnG- off=15218393 len=1574690 2024-09-13T00:16:20.429 INFO:tasks.workunit.client.0.smithi136.stdout:op 4535 completed, throughput=5MB/sec 2024-09-13T00:16:21.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:16:21 smithi181 bash[67587]: cluster 2024-09-13T00:16:19.591381+0000 mgr.y (mgr.24661) 1176 : cluster [DBG] pgmap v1859: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:16:21.430 INFO:tasks.workunit.client.0.smithi136.stdout: 870: throughput=4.99MB/sec pending data=0 2024-09-13T00:16:21.430 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-8nwTnttgMZaQSAx off=14692071 len=890276 2024-09-13T00:16:21.438 INFO:tasks.workunit.client.0.smithi136.stdout:op 4536 completed, throughput=5MB/sec 2024-09-13T00:16:21.438 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-D-7XCpB7yj3qWm_ off=1194612 len=1084594 2024-09-13T00:16:21.446 INFO:tasks.workunit.client.0.smithi136.stdout:op 4537 completed, throughput=5MB/sec 2024-09-13T00:16:21.447 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-6dSHhH5zDI_4bjX off=2211713 len=1130544 2024-09-13T00:16:21.456 INFO:tasks.workunit.client.0.smithi136.stdout:op 4538 completed, throughput=5MB/sec 2024-09-13T00:16:21.456 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ER770Ktu3WpejyY off=1672428 len=944371 2024-09-13T00:16:21.464 INFO:tasks.workunit.client.0.smithi136.stdout:op 4539 completed, throughput=5MB/sec 2024-09-13T00:16:21.464 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Ki7zQqlUg84ZhaR off=10193403 len=76865 2024-09-13T00:16:21.465 INFO:tasks.workunit.client.0.smithi136.stdout:op 4540 completed, throughput=5MB/sec 2024-09-13T00:16:21.465 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-n8DVEJBKUzTBeq5 off=2068601 len=1282514 2024-09-13T00:16:21.474 INFO:tasks.workunit.client.0.smithi136.stdout:op 4541 completed, throughput=5MB/sec 2024-09-13T00:16:21.475 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-UP_FQyBNdLvaF1N off=2252498 len=1138020 2024-09-13T00:16:21.483 INFO:tasks.workunit.client.0.smithi136.stdout:op 4542 completed, throughput=5MB/sec 2024-09-13T00:16:21.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:16:21 smithi136 bash[75164]: cluster 2024-09-13T00:16:19.591381+0000 mgr.y (mgr.24661) 1176 : cluster [DBG] pgmap v1859: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:16:21.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:16:21 smithi136 bash[76371]: cluster 2024-09-13T00:16:19.591381+0000 mgr.y (mgr.24661) 1176 : cluster [DBG] pgmap v1859: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:16:22.483 INFO:tasks.workunit.client.0.smithi136.stdout: 871: throughput=5MB/sec pending data=0 2024-09-13T00:16:22.484 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-G47gPo_oLHaypum off=1202961 len=2071359 2024-09-13T00:16:22.488 INFO:tasks.workunit.client.0.smithi136.stdout:op 4543 completed, throughput=5MB/sec 2024-09-13T00:16:22.488 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-xKqZvvaBPhAUhAf off=17045954 len=272149 2024-09-13T00:16:22.490 INFO:tasks.workunit.client.0.smithi136.stdout:op 4544 completed, throughput=5MB/sec 2024-09-13T00:16:22.490 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-EucnKFTka9E3wKL off=14040631 len=1989091 2024-09-13T00:16:22.505 INFO:tasks.workunit.client.0.smithi136.stdout:op 4545 completed, throughput=5MB/sec 2024-09-13T00:16:22.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:16:22 smithi181 bash[95505]: ts=2024-09-13T00:16:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:16:23.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:16:23 smithi181 bash[67587]: cluster 2024-09-13T00:16:21.592399+0000 mgr.y (mgr.24661) 1177 : cluster [DBG] pgmap v1860: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:16:23.506 INFO:tasks.workunit.client.0.smithi136.stdout: 872: throughput=4.99MB/sec pending data=0 2024-09-13T00:16:23.506 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-_wa6A7gpvBVY2YS off=14692489 len=1132463 2024-09-13T00:16:23.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:16:23 smithi136 bash[75164]: cluster 2024-09-13T00:16:21.592399+0000 mgr.y (mgr.24661) 1177 : cluster [DBG] pgmap v1860: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:16:23.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:16:23 smithi136 bash[76371]: cluster 2024-09-13T00:16:21.592399+0000 mgr.y (mgr.24661) 1177 : cluster [DBG] pgmap v1860: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:16:23.509 INFO:tasks.workunit.client.0.smithi136.stdout:op 4546 completed, throughput=5MB/sec 2024-09-13T00:16:23.509 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-aPnkUxLtXbPZ48W off=17998504 len=1823905 2024-09-13T00:16:23.525 INFO:tasks.workunit.client.0.smithi136.stdout:op 4547 completed, throughput=5MB/sec 2024-09-13T00:16:23.525 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-qdJeFfbinPY28kX off=10759133 len=1172899 2024-09-13T00:16:23.529 INFO:tasks.workunit.client.0.smithi136.stdout:op 4548 completed, throughput=5MB/sec 2024-09-13T00:16:23.529 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-PCqU6LJ2vjyowGu off=6361279 len=918303 2024-09-13T00:16:23.536 INFO:tasks.workunit.client.0.smithi136.stdout:op 4549 completed, throughput=5MB/sec 2024-09-13T00:16:23.536 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-un0r4cpWizgAPkk off=1701373 len=459854 2024-09-13T00:16:23.538 INFO:tasks.workunit.client.0.smithi136.stdout:op 4550 completed, throughput=5MB/sec 2024-09-13T00:16:24.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:16:24 smithi136 bash[75164]: audit 2024-09-13T00:16:23.797006+0000 mon.a (mon.0) 2239 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:16:24.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:16:24 smithi136 bash[76371]: audit 2024-09-13T00:16:23.797006+0000 mon.a (mon.0) 2239 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:16:24.538 INFO:tasks.workunit.client.0.smithi136.stdout: 873: throughput=4.99MB/sec pending data=0 2024-09-13T00:16:24.538 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-qiFOu1LP5VMwRxg off=4256865 len=847068 2024-09-13T00:16:24.546 INFO:tasks.workunit.client.0.smithi136.stdout:op 4551 completed, throughput=5MB/sec 2024-09-13T00:16:24.547 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-3zuaaz18Wl_vlPL off=14088262 len=252417 2024-09-13T00:16:24.552 INFO:tasks.workunit.client.0.smithi136.stdout:op 4552 completed, throughput=5MB/sec 2024-09-13T00:16:24.552 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-OKZfq50nXUT9R9U off=8687090 len=617390 2024-09-13T00:16:24.554 INFO:tasks.workunit.client.0.smithi136.stdout:op 4553 completed, throughput=5MB/sec 2024-09-13T00:16:24.554 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-So-V2ADAftzY5Oq off=556763 len=612294 2024-09-13T00:16:24.556 INFO:tasks.workunit.client.0.smithi136.stdout:op 4554 completed, throughput=5MB/sec 2024-09-13T00:16:24.556 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-BjYn4t1QphbSro5 off=9589027 len=1170316 2024-09-13T00:16:24.559 INFO:tasks.workunit.client.0.smithi136.stdout:op 4555 completed, throughput=5MB/sec 2024-09-13T00:16:24.559 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-fMe_WcabfVo9JeJ off=2245344 len=1257977 2024-09-13T00:16:24.569 INFO:tasks.workunit.client.0.smithi136.stdout:op 4556 completed, throughput=5MB/sec 2024-09-13T00:16:24.569 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-BrWOgb2bmrymMKZ off=5056484 len=579070 2024-09-13T00:16:24.575 INFO:tasks.workunit.client.0.smithi136.stdout:op 4557 completed, throughput=5MB/sec 2024-09-13T00:16:24.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:16:24 smithi181 bash[67587]: audit 2024-09-13T00:16:23.797006+0000 mon.a (mon.0) 2239 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:16:25.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:16:25 smithi136 bash[75164]: cluster 2024-09-13T00:16:23.593470+0000 mgr.y (mgr.24661) 1178 : cluster [DBG] pgmap v1861: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.9 MiB/s wr, 4 op/s 2024-09-13T00:16:25.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:16:25 smithi136 bash[76371]: cluster 2024-09-13T00:16:23.593470+0000 mgr.y (mgr.24661) 1178 : cluster [DBG] pgmap v1861: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.9 MiB/s wr, 4 op/s 2024-09-13T00:16:25.575 INFO:tasks.workunit.client.0.smithi136.stdout: 874: throughput=4.99MB/sec pending data=0 2024-09-13T00:16:25.575 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-xsktg_KeeE6jfFY off=2418237 len=1949129 2024-09-13T00:16:25.579 INFO:tasks.workunit.client.0.smithi136.stdout:op 4558 completed, throughput=5MB/sec 2024-09-13T00:16:25.579 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-d1dDQTye0mwJ-8q off=5596715 len=734786 2024-09-13T00:16:25.582 INFO:tasks.workunit.client.0.smithi136.stdout:op 4559 completed, throughput=5MB/sec 2024-09-13T00:16:25.582 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-BZAkNgQLGB2-YEg off=743052 len=527387 2024-09-13T00:16:25.589 INFO:tasks.workunit.client.0.smithi136.stdout:op 4560 completed, throughput=5MB/sec 2024-09-13T00:16:25.589 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-0zoGU_DNe9wbirp off=9347684 len=1930776 2024-09-13T00:16:25.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:16:25 smithi181 bash[67587]: cluster 2024-09-13T00:16:23.593470+0000 mgr.y (mgr.24661) 1178 : cluster [DBG] pgmap v1861: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.9 MiB/s wr, 4 op/s 2024-09-13T00:16:25.607 INFO:tasks.workunit.client.0.smithi136.stdout:op 4561 completed, throughput=5MB/sec 2024-09-13T00:16:25.607 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-VRW6C220mHBxVzu off=2124487 len=910174 2024-09-13T00:16:25.609 INFO:tasks.workunit.client.0.smithi136.stdout:op 4562 completed, throughput=5MB/sec 2024-09-13T00:16:26.610 INFO:tasks.workunit.client.0.smithi136.stdout: 875: throughput=5MB/sec pending data=0 2024-09-13T00:16:26.610 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-GkaJeT-BucSWFcw off=5335008 len=1610279 2024-09-13T00:16:26.620 INFO:tasks.workunit.client.0.smithi136.stdout:op 4563 completed, throughput=5MB/sec 2024-09-13T00:16:26.621 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-rNmsFFduPofaWpu off=17821786 len=1175435 2024-09-13T00:16:26.631 INFO:tasks.workunit.client.0.smithi136.stdout:op 4564 completed, throughput=5MB/sec 2024-09-13T00:16:26.631 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-GOXdRlRfzpvNLyR off=9882054 len=1836345 2024-09-13T00:16:26.636 INFO:tasks.workunit.client.0.smithi136.stdout:op 4565 completed, throughput=5MB/sec 2024-09-13T00:16:26.636 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-hIpTlQpwJjbK7Vf off=7011912 len=747622 2024-09-13T00:16:26.638 INFO:tasks.workunit.client.0.smithi136.stdout:op 4566 completed, throughput=5MB/sec 2024-09-13T00:16:27.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:16:27 smithi136 bash[75164]: cluster 2024-09-13T00:16:25.594356+0000 mgr.y (mgr.24661) 1179 : cluster [DBG] pgmap v1862: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:16:27.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:16:27 smithi136 bash[76371]: cluster 2024-09-13T00:16:25.594356+0000 mgr.y (mgr.24661) 1179 : cluster [DBG] pgmap v1862: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:16:27.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:16:27 smithi181 bash[67587]: cluster 2024-09-13T00:16:25.594356+0000 mgr.y (mgr.24661) 1179 : cluster [DBG] pgmap v1862: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:16:27.638 INFO:tasks.workunit.client.0.smithi136.stdout: 876: throughput=5MB/sec pending data=0 2024-09-13T00:16:27.638 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-1MGScVDZo3l-fII off=7446145 len=913842 2024-09-13T00:16:27.646 INFO:tasks.workunit.client.0.smithi136.stdout:op 4567 completed, throughput=5MB/sec 2024-09-13T00:16:27.646 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-WNiLpBTeOPLmMbo off=551805 len=732232 2024-09-13T00:16:27.653 INFO:tasks.workunit.client.0.smithi136.stdout:op 4568 completed, throughput=5MB/sec 2024-09-13T00:16:27.653 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-w0DsHIYWz8uv_uk off=2302380 len=672126 2024-09-13T00:16:27.655 INFO:tasks.workunit.client.0.smithi136.stdout:op 4569 completed, throughput=5MB/sec 2024-09-13T00:16:27.655 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-8rypcKuxHLxGS-a off=6786829 len=217211 2024-09-13T00:16:27.660 INFO:tasks.workunit.client.0.smithi136.stdout:op 4570 completed, throughput=5MB/sec 2024-09-13T00:16:27.661 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-zuiWox-fSLku3EB off=12235649 len=621279 2024-09-13T00:16:27.662 INFO:tasks.workunit.client.0.smithi136.stdout:op 4571 completed, throughput=5MB/sec 2024-09-13T00:16:27.662 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-fKJ6rbz-REPrhbe off=20180066 len=614658 2024-09-13T00:16:27.669 INFO:tasks.workunit.client.0.smithi136.stdout:op 4572 completed, throughput=5MB/sec 2024-09-13T00:16:27.669 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-KpQsVIeq6C791-s off=22250316 len=360623 2024-09-13T00:16:27.670 INFO:tasks.workunit.client.0.smithi136.stdout:op 4573 completed, throughput=5MB/sec 2024-09-13T00:16:27.670 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-jEKcx5OguoFRhUg off=4497167 len=998280 2024-09-13T00:16:27.672 INFO:tasks.workunit.client.0.smithi136.stdout:op 4574 completed, throughput=5MB/sec 2024-09-13T00:16:28.672 INFO:tasks.workunit.client.0.smithi136.stdout: 877: throughput=4.99MB/sec pending data=0 2024-09-13T00:16:28.672 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-WM5dvQTI4mLomLe off=2633307 len=1553446 2024-09-13T00:16:28.677 INFO:tasks.workunit.client.0.smithi136.stdout:op 4575 completed, throughput=5MB/sec 2024-09-13T00:16:28.677 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-4s7rPYnwuhIeccl off=2062877 len=830821 2024-09-13T00:16:28.679 INFO:tasks.workunit.client.0.smithi136.stdout:op 4576 completed, throughput=5MB/sec 2024-09-13T00:16:28.679 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Rk4kPl2HeByLnMY off=883237 len=1194186 2024-09-13T00:16:28.688 INFO:tasks.workunit.client.0.smithi136.stdout:op 4577 completed, throughput=5MB/sec 2024-09-13T00:16:28.688 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ntxjUnfUO6v7RQd off=102357 len=975308 2024-09-13T00:16:28.692 INFO:tasks.workunit.client.0.smithi136.stdout:op 4578 completed, throughput=5MB/sec 2024-09-13T00:16:28.692 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-gpkqoGAa5BMng2u off=6326447 len=2081821 2024-09-13T00:16:28.698 INFO:tasks.workunit.client.0.smithi136.stdout:op 4579 completed, throughput=5MB/sec 2024-09-13T00:16:29.412 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:16:29 smithi136 bash[75164]: cluster 2024-09-13T00:16:27.595507+0000 mgr.y (mgr.24661) 1180 : cluster [DBG] pgmap v1863: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.8 MiB/s wr, 4 op/s 2024-09-13T00:16:29.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:16:29 smithi136 bash[76371]: cluster 2024-09-13T00:16:27.595507+0000 mgr.y (mgr.24661) 1180 : cluster [DBG] pgmap v1863: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.8 MiB/s wr, 4 op/s 2024-09-13T00:16:29.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:16:29 smithi181 bash[67587]: cluster 2024-09-13T00:16:27.595507+0000 mgr.y (mgr.24661) 1180 : cluster [DBG] pgmap v1863: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.8 MiB/s wr, 4 op/s 2024-09-13T00:16:29.698 INFO:tasks.workunit.client.0.smithi136.stdout: 878: throughput=5MB/sec pending data=0 2024-09-13T00:16:29.699 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Mx4NOFATJac51yc off=8447750 len=1296148 2024-09-13T00:16:29.703 INFO:tasks.workunit.client.0.smithi136.stdout:op 4580 completed, throughput=5MB/sec 2024-09-13T00:16:29.703 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-n1sc93HhtYvAJ_A off=18985312 len=423002 2024-09-13T00:16:29.709 INFO:tasks.workunit.client.0.smithi136.stdout:op 4581 completed, throughput=5MB/sec 2024-09-13T00:16:29.709 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-3i-5g79CH5G-_bK off=21932657 len=1552878 2024-09-13T00:16:29.713 INFO:tasks.workunit.client.0.smithi136.stdout:op 4582 completed, throughput=5MB/sec 2024-09-13T00:16:29.714 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-mAA-FYuC3-hCRnU off=16928123 len=738315 2024-09-13T00:16:29.721 INFO:tasks.workunit.client.0.smithi136.stdout:op 4583 completed, throughput=5MB/sec 2024-09-13T00:16:29.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:16:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:16:29] "GET /metrics HTTP/1.1" 200 34143 "" "Prometheus/2.43.0" 2024-09-13T00:16:30.721 INFO:tasks.workunit.client.0.smithi136.stdout: 879: throughput=4.99MB/sec pending data=0 2024-09-13T00:16:30.722 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-nkYKrqj-EXzsgEL off=9741130 len=61502 2024-09-13T00:16:30.722 INFO:tasks.workunit.client.0.smithi136.stdout:op 4584 completed, throughput=4.99MB/sec 2024-09-13T00:16:30.723 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-kGhx2y2XjaGVdXG off=17168801 len=651582 2024-09-13T00:16:30.730 INFO:tasks.workunit.client.0.smithi136.stdout:op 4585 completed, throughput=5MB/sec 2024-09-13T00:16:30.730 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-vdgPv980Iosgufm off=11407764 len=90324 2024-09-13T00:16:30.734 INFO:tasks.workunit.client.0.smithi136.stdout:op 4586 completed, throughput=5MB/sec 2024-09-13T00:16:30.734 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-jVNo5IjjwbTqiu2 off=572228 len=967098 2024-09-13T00:16:30.743 INFO:tasks.workunit.client.0.smithi136.stdout:op 4587 completed, throughput=5MB/sec 2024-09-13T00:16:30.743 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-4FcZxoDLfbgerIM off=1721333 len=1490016 2024-09-13T00:16:30.747 INFO:tasks.workunit.client.0.smithi136.stdout:op 4588 completed, throughput=5MB/sec 2024-09-13T00:16:30.747 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-2ZCCXZi8bc7Uu1X off=5900836 len=1170103 2024-09-13T00:16:30.757 INFO:tasks.workunit.client.0.smithi136.stdout:op 4589 completed, throughput=5MB/sec 2024-09-13T00:16:30.758 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-dc-KxBvkJMi3Amd off=9458548 len=1016557 2024-09-13T00:16:30.761 INFO:tasks.workunit.client.0.smithi136.stdout:op 4590 completed, throughput=5MB/sec 2024-09-13T00:16:31.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:16:31 smithi136 bash[75164]: cluster 2024-09-13T00:16:29.596568+0000 mgr.y (mgr.24661) 1181 : cluster [DBG] pgmap v1864: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.7 MiB/s wr, 4 op/s 2024-09-13T00:16:31.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:16:31 smithi136 bash[76371]: cluster 2024-09-13T00:16:29.596568+0000 mgr.y (mgr.24661) 1181 : cluster [DBG] pgmap v1864: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.7 MiB/s wr, 4 op/s 2024-09-13T00:16:31.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:16:31 smithi181 bash[67587]: cluster 2024-09-13T00:16:29.596568+0000 mgr.y (mgr.24661) 1181 : cluster [DBG] pgmap v1864: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.7 MiB/s wr, 4 op/s 2024-09-13T00:16:31.761 INFO:tasks.workunit.client.0.smithi136.stdout: 880: throughput=4.99MB/sec pending data=0 2024-09-13T00:16:31.761 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-vJPwHbCCLOUiM0i off=4630936 len=1477346 2024-09-13T00:16:31.773 INFO:tasks.workunit.client.0.smithi136.stdout:op 4591 completed, throughput=5MB/sec 2024-09-13T00:16:31.773 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Mt9DWHvMbNLEaxu off=2766762 len=1345951 2024-09-13T00:16:31.776 INFO:tasks.workunit.client.0.smithi136.stdout:op 4592 completed, throughput=5MB/sec 2024-09-13T00:16:31.777 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-pLARYZ-yjFdMfoS off=5580670 len=1479681 2024-09-13T00:16:31.786 INFO:tasks.workunit.client.0.smithi136.stdout:op 4593 completed, throughput=5MB/sec 2024-09-13T00:16:31.786 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-MG_9jFu-IjEWID9 off=126096 len=1046170 2024-09-13T00:16:31.795 INFO:tasks.workunit.client.0.smithi136.stdout:op 4594 completed, throughput=5MB/sec 2024-09-13T00:16:32.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:16:32 smithi181 bash[95505]: ts=2024-09-13T00:16:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:16:32.795 INFO:tasks.workunit.client.0.smithi136.stdout: 881: throughput=4.99MB/sec pending data=0 2024-09-13T00:16:32.795 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-WGOPkVyhTo6FAjj off=12761711 len=324444 2024-09-13T00:16:32.797 INFO:tasks.workunit.client.0.smithi136.stdout:op 4595 completed, throughput=4.99MB/sec 2024-09-13T00:16:32.797 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-xJ6KuBy_-bU_fnd off=959425 len=1294110 2024-09-13T00:16:32.800 INFO:tasks.workunit.client.0.smithi136.stdout:op 4596 completed, throughput=5MB/sec 2024-09-13T00:16:32.800 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-TxbtbjEXtNJJLl3 off=3905296 len=1639568 2024-09-13T00:16:32.805 INFO:tasks.workunit.client.0.smithi136.stdout:op 4597 completed, throughput=5MB/sec 2024-09-13T00:16:32.805 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-qZkQZYlNv0b_ca8 off=8058875 len=1480502 2024-09-13T00:16:32.816 INFO:tasks.workunit.client.0.smithi136.stdout:op 4598 completed, throughput=5MB/sec 2024-09-13T00:16:32.816 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Q5u0dknY1V47KA_ off=3935135 len=1344308 2024-09-13T00:16:32.827 INFO:tasks.workunit.client.0.smithi136.stdout:op 4599 completed, throughput=5MB/sec 2024-09-13T00:16:33.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:16:33 smithi136 bash[75164]: cluster 2024-09-13T00:16:31.597612+0000 mgr.y (mgr.24661) 1182 : cluster [DBG] pgmap v1865: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.7 MiB/s wr, 4 op/s 2024-09-13T00:16:33.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:16:33 smithi136 bash[76371]: cluster 2024-09-13T00:16:31.597612+0000 mgr.y (mgr.24661) 1182 : cluster [DBG] pgmap v1865: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.7 MiB/s wr, 4 op/s 2024-09-13T00:16:33.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:16:33 smithi181 bash[67587]: cluster 2024-09-13T00:16:31.597612+0000 mgr.y (mgr.24661) 1182 : cluster [DBG] pgmap v1865: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.7 MiB/s wr, 4 op/s 2024-09-13T00:16:33.827 INFO:tasks.workunit.client.0.smithi136.stdout: 882: throughput=5MB/sec pending data=0 2024-09-13T00:16:33.827 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-6Fse1YZAQ2N_hm8 off=6034184 len=557641 2024-09-13T00:16:33.834 INFO:tasks.workunit.client.0.smithi136.stdout:op 4600 completed, throughput=5MB/sec 2024-09-13T00:16:33.834 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-vk33gKYF4jIqjo- off=1130159 len=835387 2024-09-13T00:16:33.837 INFO:tasks.workunit.client.0.smithi136.stdout:op 4601 completed, throughput=5MB/sec 2024-09-13T00:16:33.837 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-H-taMNGhiZ12wf0 off=1380434 len=1458943 2024-09-13T00:16:33.841 INFO:tasks.workunit.client.0.smithi136.stdout:op 4602 completed, throughput=5MB/sec 2024-09-13T00:16:33.841 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-BXkbZDdwjdFTJuZ off=4060953 len=801875 2024-09-13T00:16:33.850 INFO:tasks.workunit.client.0.smithi136.stdout:op 4603 completed, throughput=5MB/sec 2024-09-13T00:16:33.850 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-VJzOOKZwkvFw8ri off=13223787 len=240331 2024-09-13T00:16:33.856 INFO:tasks.workunit.client.0.smithi136.stdout:op 4604 completed, throughput=5MB/sec 2024-09-13T00:16:33.856 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-yhA1tz-uSwDHu9r off=18832334 len=1914488 2024-09-13T00:16:33.873 INFO:tasks.workunit.client.0.smithi136.stdout:op 4605 completed, throughput=5MB/sec 2024-09-13T00:16:34.873 INFO:tasks.workunit.client.0.smithi136.stdout: 883: throughput=5MB/sec pending data=0 2024-09-13T00:16:34.874 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-yBc_e2Fu87BCIKH off=5464398 len=1399575 2024-09-13T00:16:34.883 INFO:tasks.workunit.client.0.smithi136.stdout:op 4606 completed, throughput=5MB/sec 2024-09-13T00:16:34.883 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-6lK5JTDaEfrrzp7 off=3861164 len=558718 2024-09-13T00:16:34.884 INFO:tasks.workunit.client.0.smithi136.stdout:op 4607 completed, throughput=5MB/sec 2024-09-13T00:16:34.884 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-WLVt43Qjn1B7xZB off=6076541 len=248244 2024-09-13T00:16:34.890 INFO:tasks.workunit.client.0.smithi136.stdout:op 4608 completed, throughput=5MB/sec 2024-09-13T00:16:34.890 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-rp0qQL8YfJh3ws0 off=7527224 len=1532593 2024-09-13T00:16:34.893 INFO:tasks.workunit.client.0.smithi136.stdout:op 4609 completed, throughput=5MB/sec 2024-09-13T00:16:34.893 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-w0rS0S3CxlYer6p off=8257445 len=345785 2024-09-13T00:16:34.895 INFO:tasks.workunit.client.0.smithi136.stdout:op 4610 completed, throughput=5MB/sec 2024-09-13T00:16:34.895 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-XIxYQIEmNHIveD6 off=13155096 len=605809 2024-09-13T00:16:34.902 INFO:tasks.workunit.client.0.smithi136.stdout:op 4611 completed, throughput=5MB/sec 2024-09-13T00:16:35.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:16:35 smithi136 bash[75164]: cluster 2024-09-13T00:16:33.598647+0000 mgr.y (mgr.24661) 1183 : cluster [DBG] pgmap v1866: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:16:35.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:16:35 smithi136 bash[76371]: cluster 2024-09-13T00:16:33.598647+0000 mgr.y (mgr.24661) 1183 : cluster [DBG] pgmap v1866: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:16:35.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:16:35 smithi181 bash[67587]: cluster 2024-09-13T00:16:33.598647+0000 mgr.y (mgr.24661) 1183 : cluster [DBG] pgmap v1866: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:16:35.902 INFO:tasks.workunit.client.0.smithi136.stdout: 884: throughput=4.99MB/sec pending data=0 2024-09-13T00:16:35.902 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-j3KaPgeXzfa-HV_ off=2986689 len=1238102 2024-09-13T00:16:35.905 INFO:tasks.workunit.client.0.smithi136.stdout:op 4612 completed, throughput=5MB/sec 2024-09-13T00:16:35.905 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-QceALxebN49DPsB off=3877164 len=163156 2024-09-13T00:16:35.907 INFO:tasks.workunit.client.0.smithi136.stdout:op 4613 completed, throughput=5MB/sec 2024-09-13T00:16:35.907 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-PRavblzxG0xNWQm off=18751553 len=1869188 2024-09-13T00:16:35.911 INFO:tasks.workunit.client.0.smithi136.stdout:op 4614 completed, throughput=5MB/sec 2024-09-13T00:16:35.911 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2hwfTHyN2o30CW_ off=1078272 len=1346142 2024-09-13T00:16:35.914 INFO:tasks.workunit.client.0.smithi136.stdout:op 4615 completed, throughput=5MB/sec 2024-09-13T00:16:35.915 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-AVq52FNlYz2E0Cf off=10227935 len=1578460 2024-09-13T00:16:35.918 INFO:tasks.workunit.client.0.smithi136.stdout:op 4616 completed, throughput=5MB/sec 2024-09-13T00:16:36.919 INFO:tasks.workunit.client.0.smithi136.stdout: 885: throughput=5MB/sec pending data=0 2024-09-13T00:16:36.919 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-vOsVwkCVp-jt42D off=6722905 len=446108 2024-09-13T00:16:36.926 INFO:tasks.workunit.client.0.smithi136.stdout:op 4617 completed, throughput=5MB/sec 2024-09-13T00:16:36.926 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-SudKhljMtw87KL9 off=2224190 len=1491323 2024-09-13T00:16:36.938 INFO:tasks.workunit.client.0.smithi136.stdout:op 4618 completed, throughput=5MB/sec 2024-09-13T00:16:36.938 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-RWBR-FLGx3smpE6 off=2735708 len=1185139 2024-09-13T00:16:36.950 INFO:tasks.workunit.client.0.smithi136.stdout:op 4619 completed, throughput=5MB/sec 2024-09-13T00:16:36.950 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-qrtVQ1lwHl3M3zf off=6234433 len=670519 2024-09-13T00:16:36.952 INFO:tasks.workunit.client.0.smithi136.stdout:op 4620 completed, throughput=5MB/sec 2024-09-13T00:16:36.952 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-1CH1-pCWKImA6ws off=18524191 len=2053725 2024-09-13T00:16:36.965 INFO:tasks.workunit.client.0.smithi136.stdout:op 4621 completed, throughput=5MB/sec 2024-09-13T00:16:37.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:16:37 smithi136 bash[75164]: cluster 2024-09-13T00:16:35.599473+0000 mgr.y (mgr.24661) 1184 : cluster [DBG] pgmap v1867: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:16:37.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:16:37 smithi136 bash[76371]: cluster 2024-09-13T00:16:35.599473+0000 mgr.y (mgr.24661) 1184 : cluster [DBG] pgmap v1867: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:16:37.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:16:37 smithi181 bash[67587]: cluster 2024-09-13T00:16:35.599473+0000 mgr.y (mgr.24661) 1184 : cluster [DBG] pgmap v1867: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:16:37.966 INFO:tasks.workunit.client.0.smithi136.stdout: 886: throughput=5MB/sec pending data=0 2024-09-13T00:16:37.966 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-lbT0beu-rDJgJBm off=247606 len=2031171 2024-09-13T00:16:37.972 INFO:tasks.workunit.client.0.smithi136.stdout:op 4622 completed, throughput=5MB/sec 2024-09-13T00:16:37.972 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-jdKKMydG3IqT8LD off=2075095 len=1176612 2024-09-13T00:16:37.981 INFO:tasks.workunit.client.0.smithi136.stdout:op 4623 completed, throughput=5MB/sec 2024-09-13T00:16:37.981 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-AbFsDbdVvVjXY5Y off=1397572 len=820091 2024-09-13T00:16:37.993 INFO:tasks.workunit.client.0.smithi136.stdout:op 4624 completed, throughput=5MB/sec 2024-09-13T00:16:38.993 INFO:tasks.workunit.client.0.smithi136.stdout: 887: throughput=4.99MB/sec pending data=0 2024-09-13T00:16:38.993 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Lv4sf6kyZh6xmgI off=14576032 len=920367 2024-09-13T00:16:38.997 INFO:tasks.workunit.client.0.smithi136.stdout:op 4625 completed, throughput=5MB/sec 2024-09-13T00:16:38.997 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-RGsTiUCNhwLUCdf off=8424489 len=1442834 2024-09-13T00:16:39.007 INFO:tasks.workunit.client.0.smithi136.stdout:op 4626 completed, throughput=5MB/sec 2024-09-13T00:16:39.007 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-hoIRQfIpqBRdeeT off=9323315 len=128743 2024-09-13T00:16:39.012 INFO:tasks.workunit.client.0.smithi136.stdout:op 4627 completed, throughput=5MB/sec 2024-09-13T00:16:39.012 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-9TiLHtw_Ug5LYGQ off=2516534 len=1417038 2024-09-13T00:16:39.015 INFO:tasks.workunit.client.0.smithi136.stdout:op 4628 completed, throughput=5MB/sec 2024-09-13T00:16:39.015 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-QtbhmyQQ97itgrz off=5586800 len=1481393 2024-09-13T00:16:39.020 INFO:tasks.workunit.client.0.smithi136.stdout:op 4629 completed, throughput=5MB/sec 2024-09-13T00:16:39.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:16:39 smithi136 bash[76371]: cluster 2024-09-13T00:16:37.600509+0000 mgr.y (mgr.24661) 1185 : cluster [DBG] pgmap v1868: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:16:39.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:16:39 smithi136 bash[76371]: audit 2024-09-13T00:16:38.797227+0000 mon.a (mon.0) 2240 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:16:39.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:16:39 smithi136 bash[75164]: cluster 2024-09-13T00:16:37.600509+0000 mgr.y (mgr.24661) 1185 : cluster [DBG] pgmap v1868: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:16:39.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:16:39 smithi136 bash[75164]: audit 2024-09-13T00:16:38.797227+0000 mon.a (mon.0) 2240 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:16:39.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:16:39 smithi181 bash[67587]: cluster 2024-09-13T00:16:37.600509+0000 mgr.y (mgr.24661) 1185 : cluster [DBG] pgmap v1868: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:16:39.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:16:39 smithi181 bash[67587]: audit 2024-09-13T00:16:38.797227+0000 mon.a (mon.0) 2240 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:16:39.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:16:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:16:39] "GET /metrics HTTP/1.1" 200 34142 "" "Prometheus/2.43.0" 2024-09-13T00:16:40.021 INFO:tasks.workunit.client.0.smithi136.stdout: 888: throughput=4.99MB/sec pending data=0 2024-09-13T00:16:40.021 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-h5eKEwk5FcLb7GV off=13916988 len=760407 2024-09-13T00:16:40.023 INFO:tasks.workunit.client.0.smithi136.stdout:op 4630 completed, throughput=5MB/sec 2024-09-13T00:16:40.023 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-DyGMYhz7qVXsdfq off=11587006 len=61770 2024-09-13T00:16:40.025 INFO:tasks.workunit.client.0.smithi136.stdout:op 4631 completed, throughput=5MB/sec 2024-09-13T00:16:40.025 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-qLGP2IXMqTz8D9l off=11978158 len=1470408 2024-09-13T00:16:40.035 INFO:tasks.workunit.client.0.smithi136.stdout:op 4632 completed, throughput=5MB/sec 2024-09-13T00:16:40.035 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Hf_A0qaF3CC2JBB off=674658 len=687939 2024-09-13T00:16:40.037 INFO:tasks.workunit.client.0.smithi136.stdout:op 4633 completed, throughput=5MB/sec 2024-09-13T00:16:40.037 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-R-g3JGa9FtPYhwR off=1759188 len=1362010 2024-09-13T00:16:40.040 INFO:tasks.workunit.client.0.smithi136.stdout:op 4634 completed, throughput=5MB/sec 2024-09-13T00:16:40.040 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-7NLCitYvGxgkcHG off=7317762 len=1568030 2024-09-13T00:16:40.044 INFO:tasks.workunit.client.0.smithi136.stdout:op 4635 completed, throughput=5MB/sec 2024-09-13T00:16:41.044 INFO:tasks.workunit.client.0.smithi136.stdout: 889: throughput=5MB/sec pending data=0 2024-09-13T00:16:41.044 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-2repnMgwm4I5ekW off=3039104 len=1812225 2024-09-13T00:16:41.061 INFO:tasks.workunit.client.0.smithi136.stdout:op 4636 completed, throughput=5MB/sec 2024-09-13T00:16:41.062 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-itFNE7K6O4K5Vhu off=602072 len=1233089 2024-09-13T00:16:41.073 INFO:tasks.workunit.client.0.smithi136.stdout:op 4637 completed, throughput=5MB/sec 2024-09-13T00:16:41.073 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-KSL0_tcOOio6G2w off=15377452 len=748680 2024-09-13T00:16:41.080 INFO:tasks.workunit.client.0.smithi136.stdout:op 4638 completed, throughput=5MB/sec 2024-09-13T00:16:41.080 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-YquiF-bbcbog1oI off=14606878 len=1063718 2024-09-13T00:16:41.083 INFO:tasks.workunit.client.0.smithi136.stdout:op 4639 completed, throughput=5MB/sec 2024-09-13T00:16:41.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:16:41 smithi136 bash[75164]: cluster 2024-09-13T00:16:39.601434+0000 mgr.y (mgr.24661) 1186 : cluster [DBG] pgmap v1869: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:16:41.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:16:41 smithi136 bash[76371]: cluster 2024-09-13T00:16:39.601434+0000 mgr.y (mgr.24661) 1186 : cluster [DBG] pgmap v1869: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:16:41.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:16:41 smithi181 bash[67587]: cluster 2024-09-13T00:16:39.601434+0000 mgr.y (mgr.24661) 1186 : cluster [DBG] pgmap v1869: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:16:42.083 INFO:tasks.workunit.client.0.smithi136.stdout: 890: throughput=4.99MB/sec pending data=0 2024-09-13T00:16:42.084 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-EogyGeohUnE-VC7 off=1071940 len=1732848 2024-09-13T00:16:42.098 INFO:tasks.workunit.client.0.smithi136.stdout:op 4640 completed, throughput=5MB/sec 2024-09-13T00:16:42.098 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-xWddmTu_Xn1mlMw off=10585863 len=210969 2024-09-13T00:16:42.099 INFO:tasks.workunit.client.0.smithi136.stdout:op 4641 completed, throughput=5MB/sec 2024-09-13T00:16:42.099 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-myRYu1v3vCKKQEk off=20903956 len=1310551 2024-09-13T00:16:42.111 INFO:tasks.workunit.client.0.smithi136.stdout:op 4642 completed, throughput=5MB/sec 2024-09-13T00:16:42.111 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-w-v83117crn4GRh off=1364204 len=622630 2024-09-13T00:16:42.117 INFO:tasks.workunit.client.0.smithi136.stdout:op 4643 completed, throughput=5MB/sec 2024-09-13T00:16:42.118 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-i3DTIaUFrCKaZXZ off=4191233 len=565425 2024-09-13T00:16:42.124 INFO:tasks.workunit.client.0.smithi136.stdout:op 4644 completed, throughput=5MB/sec 2024-09-13T00:16:42.124 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Nq7PadkmezlPnjn off=1969991 len=502538 2024-09-13T00:16:42.130 INFO:tasks.workunit.client.0.smithi136.stdout:op 4645 completed, throughput=5MB/sec 2024-09-13T00:16:42.130 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-rTRpIl22qIA8Qv3 off=12762644 len=402140 2024-09-13T00:16:42.136 INFO:tasks.workunit.client.0.smithi136.stdout:op 4646 completed, throughput=5MB/sec 2024-09-13T00:16:42.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:16:42 smithi181 bash[95505]: ts=2024-09-13T00:16:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:16:43.136 INFO:tasks.workunit.client.0.smithi136.stdout: 891: throughput=4.99MB/sec pending data=0 2024-09-13T00:16:43.136 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-iUJtDUEa7_d61MJ off=9528593 len=1552378 2024-09-13T00:16:43.141 INFO:tasks.workunit.client.0.smithi136.stdout:op 4647 completed, throughput=5MB/sec 2024-09-13T00:16:43.141 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-OpOrLwbEyFcLWUu off=200317 len=1347017 2024-09-13T00:16:43.145 INFO:tasks.workunit.client.0.smithi136.stdout:op 4648 completed, throughput=5MB/sec 2024-09-13T00:16:43.145 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ziq9J1eO46nhCiz off=6665221 len=599949 2024-09-13T00:16:43.147 INFO:tasks.workunit.client.0.smithi136.stdout:op 4649 completed, throughput=5MB/sec 2024-09-13T00:16:43.147 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-S7ZZNefr-KG_UI6 off=3156224 len=1225871 2024-09-13T00:16:43.155 INFO:tasks.workunit.client.0.smithi136.stdout:op 4650 completed, throughput=5MB/sec 2024-09-13T00:16:43.155 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-XpZjAsDI52ZBJcs off=7428674 len=1494172 2024-09-13T00:16:43.159 INFO:tasks.workunit.client.0.smithi136.stdout:op 4651 completed, throughput=5MB/sec 2024-09-13T00:16:43.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:16:43 smithi136 bash[75164]: cluster 2024-09-13T00:16:41.602380+0000 mgr.y (mgr.24661) 1187 : cluster [DBG] pgmap v1870: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:16:43.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:16:43 smithi136 bash[76371]: cluster 2024-09-13T00:16:41.602380+0000 mgr.y (mgr.24661) 1187 : cluster [DBG] pgmap v1870: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:16:43.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:16:43 smithi181 bash[67587]: cluster 2024-09-13T00:16:41.602380+0000 mgr.y (mgr.24661) 1187 : cluster [DBG] pgmap v1870: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:16:44.159 INFO:tasks.workunit.client.0.smithi136.stdout: 892: throughput=5MB/sec pending data=0 2024-09-13T00:16:44.159 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-TZPPlNM-YfY9SwG off=12282262 len=763271 2024-09-13T00:16:44.167 INFO:tasks.workunit.client.0.smithi136.stdout:op 4652 completed, throughput=5MB/sec 2024-09-13T00:16:44.167 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-l_vJeSBTZCpKmLA off=16326915 len=362749 2024-09-13T00:16:44.173 INFO:tasks.workunit.client.0.smithi136.stdout:op 4653 completed, throughput=5MB/sec 2024-09-13T00:16:44.173 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-vWeQGPghgBNcStb off=14635311 len=1550887 2024-09-13T00:16:44.177 INFO:tasks.workunit.client.0.smithi136.stdout:op 4654 completed, throughput=5MB/sec 2024-09-13T00:16:44.177 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-SChBzLJD53jmLUy off=4444622 len=1566295 2024-09-13T00:16:44.189 INFO:tasks.workunit.client.0.smithi136.stdout:op 4655 completed, throughput=5MB/sec 2024-09-13T00:16:44.189 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-d-IBfRw1geNCkFp off=2998789 len=1237564 2024-09-13T00:16:44.192 INFO:tasks.workunit.client.0.smithi136.stdout:op 4656 completed, throughput=5MB/sec 2024-09-13T00:16:45.193 INFO:tasks.workunit.client.0.smithi136.stdout: 893: throughput=5MB/sec pending data=0 2024-09-13T00:16:45.193 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-rNk3QUWOASJ2XkF off=6931746 len=27064 2024-09-13T00:16:45.197 INFO:tasks.workunit.client.0.smithi136.stdout:op 4657 completed, throughput=5MB/sec 2024-09-13T00:16:45.197 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-GjcB4hRlRdTto4P off=12374296 len=889385 2024-09-13T00:16:45.199 INFO:tasks.workunit.client.0.smithi136.stdout:op 4658 completed, throughput=5MB/sec 2024-09-13T00:16:45.199 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-VXN9inaqjmFPRJa off=5448995 len=1420238 2024-09-13T00:16:45.213 INFO:tasks.workunit.client.0.smithi136.stdout:op 4659 completed, throughput=5MB/sec 2024-09-13T00:16:45.213 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-pjvBU13-plfsv6_ off=9012994 len=240198 2024-09-13T00:16:45.214 INFO:tasks.workunit.client.0.smithi136.stdout:op 4660 completed, throughput=5MB/sec 2024-09-13T00:16:45.215 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-rPaITeF_s0olF0D off=7932515 len=562593 2024-09-13T00:16:45.221 INFO:tasks.workunit.client.0.smithi136.stdout:op 4661 completed, throughput=5MB/sec 2024-09-13T00:16:45.221 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Dbv0_mRfsh1NWSB off=15923099 len=315205 2024-09-13T00:16:45.223 INFO:tasks.workunit.client.0.smithi136.stdout:op 4662 completed, throughput=5MB/sec 2024-09-13T00:16:45.223 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-RH9vDs2xfuYWJAz off=2487988 len=1902454 2024-09-13T00:16:45.228 INFO:tasks.workunit.client.0.smithi136.stdout:op 4663 completed, throughput=5MB/sec 2024-09-13T00:16:45.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:16:45 smithi136 bash[75164]: cluster 2024-09-13T00:16:43.603427+0000 mgr.y (mgr.24661) 1188 : cluster [DBG] pgmap v1871: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:16:45.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:16:45 smithi136 bash[76371]: cluster 2024-09-13T00:16:43.603427+0000 mgr.y (mgr.24661) 1188 : cluster [DBG] pgmap v1871: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:16:45.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:16:45 smithi181 bash[67587]: cluster 2024-09-13T00:16:43.603427+0000 mgr.y (mgr.24661) 1188 : cluster [DBG] pgmap v1871: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:16:46.228 INFO:tasks.workunit.client.0.smithi136.stdout: 894: throughput=5MB/sec pending data=0 2024-09-13T00:16:46.229 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-tcwCrAU3VJPGvlg off=8828609 len=1800020 2024-09-13T00:16:46.233 INFO:tasks.workunit.client.0.smithi136.stdout:op 4664 completed, throughput=5MB/sec 2024-09-13T00:16:46.233 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-edgnPmDsblEa2O6 off=341246 len=1531207 2024-09-13T00:16:46.244 INFO:tasks.workunit.client.0.smithi136.stdout:op 4665 completed, throughput=5MB/sec 2024-09-13T00:16:46.245 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-RnmqK-ofci7mMXn off=16008220 len=275745 2024-09-13T00:16:46.250 INFO:tasks.workunit.client.0.smithi136.stdout:op 4666 completed, throughput=5MB/sec 2024-09-13T00:16:46.250 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-jfzdEp7J4yB4bb- off=1642947 len=843082 2024-09-13T00:16:46.258 INFO:tasks.workunit.client.0.smithi136.stdout:op 4667 completed, throughput=5MB/sec 2024-09-13T00:16:46.258 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-g2ykF6tb_Bz2i39 off=5455280 len=2079127 2024-09-13T00:16:46.273 INFO:tasks.workunit.client.0.smithi136.stdout:op 4668 completed, throughput=5MB/sec 2024-09-13T00:16:47.274 INFO:tasks.workunit.client.0.smithi136.stdout: 895: throughput=5MB/sec pending data=0 2024-09-13T00:16:47.274 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_f06hJzM9o4r6DY off=7508816 len=1803254 2024-09-13T00:16:47.285 INFO:tasks.workunit.client.0.smithi136.stdout:op 4669 completed, throughput=5MB/sec 2024-09-13T00:16:47.286 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-qlYWDdbkR6bxnEN off=4246898 len=1170979 2024-09-13T00:16:47.289 INFO:tasks.workunit.client.0.smithi136.stdout:op 4670 completed, throughput=5MB/sec 2024-09-13T00:16:47.289 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-cVUUbmcuuYyzoqQ off=6209514 len=883131 2024-09-13T00:16:47.296 INFO:tasks.workunit.client.0.smithi136.stdout:op 4671 completed, throughput=5MB/sec 2024-09-13T00:16:47.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:16:47 smithi136 bash[75164]: cluster 2024-09-13T00:16:45.604314+0000 mgr.y (mgr.24661) 1189 : cluster [DBG] pgmap v1872: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:16:47.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:16:47 smithi136 bash[76371]: cluster 2024-09-13T00:16:45.604314+0000 mgr.y (mgr.24661) 1189 : cluster [DBG] pgmap v1872: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:16:47.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:16:47 smithi181 bash[67587]: cluster 2024-09-13T00:16:45.604314+0000 mgr.y (mgr.24661) 1189 : cluster [DBG] pgmap v1872: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:16:48.297 INFO:tasks.workunit.client.0.smithi136.stdout: 896: throughput=5MB/sec pending data=0 2024-09-13T00:16:48.297 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-alezcT-YNfh94r0 off=20122340 len=1241920 2024-09-13T00:16:48.301 INFO:tasks.workunit.client.0.smithi136.stdout:op 4672 completed, throughput=5MB/sec 2024-09-13T00:16:48.301 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-4cxp6MKCk7RHl0N off=5610734 len=714675 2024-09-13T00:16:48.303 INFO:tasks.workunit.client.0.smithi136.stdout:op 4673 completed, throughput=5MB/sec 2024-09-13T00:16:48.304 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-sJHGYpynpA_C7j5 off=277415 len=593600 2024-09-13T00:16:48.306 INFO:tasks.workunit.client.0.smithi136.stdout:op 4674 completed, throughput=5MB/sec 2024-09-13T00:16:48.306 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-hI3eRLTyHtfQbxd off=13055904 len=787319 2024-09-13T00:16:48.314 INFO:tasks.workunit.client.0.smithi136.stdout:op 4675 completed, throughput=5MB/sec 2024-09-13T00:16:48.314 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-YFFND9PTUIYpTly off=3353150 len=902103 2024-09-13T00:16:48.317 INFO:tasks.workunit.client.0.smithi136.stdout:op 4676 completed, throughput=5MB/sec 2024-09-13T00:16:48.317 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-iXYtBOdZLXjSI7D off=12335541 len=378179 2024-09-13T00:16:48.318 INFO:tasks.workunit.client.0.smithi136.stdout:op 4677 completed, throughput=5MB/sec 2024-09-13T00:16:48.319 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-1x0jhDMH3RY_Bsk off=18465515 len=685302 2024-09-13T00:16:48.325 INFO:tasks.workunit.client.0.smithi136.stdout:op 4678 completed, throughput=5MB/sec 2024-09-13T00:16:49.326 INFO:tasks.workunit.client.0.smithi136.stdout: 897: throughput=5MB/sec pending data=0 2024-09-13T00:16:49.326 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-pbzmfyBG656m4HX off=1958054 len=1765276 2024-09-13T00:16:49.330 INFO:tasks.workunit.client.0.smithi136.stdout:op 4679 completed, throughput=5MB/sec 2024-09-13T00:16:49.330 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-jRXV3m1IGzjeC4x off=6803992 len=706807 2024-09-13T00:16:49.338 INFO:tasks.workunit.client.0.smithi136.stdout:op 4680 completed, throughput=5MB/sec 2024-09-13T00:16:49.338 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-M8sX4zSspiT5Wpp off=1032874 len=1098387 2024-09-13T00:16:49.342 INFO:tasks.workunit.client.0.smithi136.stdout:op 4681 completed, throughput=5MB/sec 2024-09-13T00:16:49.343 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-QfcqC0hzn393dTa off=6606444 len=1675534 2024-09-13T00:16:49.356 INFO:tasks.workunit.client.0.smithi136.stdout:op 4682 completed, throughput=5MB/sec 2024-09-13T00:16:49.412 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:16:49 smithi136 bash[75164]: cluster 2024-09-13T00:16:47.605464+0000 mgr.y (mgr.24661) 1190 : cluster [DBG] pgmap v1873: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:16:49.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:16:49 smithi136 bash[76371]: cluster 2024-09-13T00:16:47.605464+0000 mgr.y (mgr.24661) 1190 : cluster [DBG] pgmap v1873: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:16:49.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:16:49 smithi181 bash[67587]: cluster 2024-09-13T00:16:47.605464+0000 mgr.y (mgr.24661) 1190 : cluster [DBG] pgmap v1873: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:16:49.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:16:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:16:49] "GET /metrics HTTP/1.1" 200 34142 "" "Prometheus/2.43.0" 2024-09-13T00:16:50.357 INFO:tasks.workunit.client.0.smithi136.stdout: 898: throughput=4.99MB/sec pending data=0 2024-09-13T00:16:50.357 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-hrhq_7JyFY1dYHd off=9882167 len=2019667 2024-09-13T00:16:50.371 INFO:tasks.workunit.client.0.smithi136.stdout:op 4683 completed, throughput=5MB/sec 2024-09-13T00:16:50.371 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-4qK0HNfY4DYFdVC off=15465725 len=1307450 2024-09-13T00:16:50.380 INFO:tasks.workunit.client.0.smithi136.stdout:op 4684 completed, throughput=5MB/sec 2024-09-13T00:16:50.380 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-s3vx3YvAjt5tSLx off=3324661 len=1461702 2024-09-13T00:16:50.385 INFO:tasks.workunit.client.0.smithi136.stdout:op 4685 completed, throughput=5MB/sec 2024-09-13T00:16:50.385 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-irFg_1sGu-01RQr off=10294825 len=1156921 2024-09-13T00:16:50.394 INFO:tasks.workunit.client.0.smithi136.stdout:op 4686 completed, throughput=5MB/sec 2024-09-13T00:16:51.395 INFO:tasks.workunit.client.0.smithi136.stdout: 899: throughput=5MB/sec pending data=0 2024-09-13T00:16:51.395 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-owhz9e5CM1jYtxZ off=852930 len=227758 2024-09-13T00:16:51.396 INFO:tasks.workunit.client.0.smithi136.stdout:op 4687 completed, throughput=5MB/sec 2024-09-13T00:16:51.396 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-rljAsnxGVtZvc-u off=4917387 len=1465239 2024-09-13T00:16:51.409 INFO:tasks.workunit.client.0.smithi136.stdout:op 4688 completed, throughput=5MB/sec 2024-09-13T00:16:51.409 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-SrGqBHQejqMSv2W off=2243192 len=1354796 2024-09-13T00:16:51.421 INFO:tasks.workunit.client.0.smithi136.stdout:op 4689 completed, throughput=5MB/sec 2024-09-13T00:16:51.421 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-xgPlsbWgt7QWpiy off=7344311 len=1414929 2024-09-13T00:16:51.424 INFO:tasks.workunit.client.0.smithi136.stdout:op 4690 completed, throughput=5MB/sec 2024-09-13T00:16:51.424 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-wxiEXR1xudJVc_6 off=11270817 len=1836578 2024-09-13T00:16:51.428 INFO:tasks.workunit.client.0.smithi136.stdout:op 4691 completed, throughput=5MB/sec 2024-09-13T00:16:51.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:16:51 smithi136 bash[76371]: cluster 2024-09-13T00:16:49.606312+0000 mgr.y (mgr.24661) 1191 : cluster [DBG] pgmap v1874: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:16:51.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:16:51 smithi136 bash[75164]: cluster 2024-09-13T00:16:49.606312+0000 mgr.y (mgr.24661) 1191 : cluster [DBG] pgmap v1874: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:16:51.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:16:51 smithi181 bash[67587]: cluster 2024-09-13T00:16:49.606312+0000 mgr.y (mgr.24661) 1191 : cluster [DBG] pgmap v1874: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:16:52.429 INFO:tasks.workunit.client.0.smithi136.stdout: 900: throughput=5MB/sec pending data=0 2024-09-13T00:16:52.429 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-lPonLj6OKajAt4r off=9594430 len=2062167 2024-09-13T00:16:52.435 INFO:tasks.workunit.client.0.smithi136.stdout:op 4692 completed, throughput=5MB/sec 2024-09-13T00:16:52.436 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-L54NR1S6uc74a3r off=4424638 len=620449 2024-09-13T00:16:52.443 INFO:tasks.workunit.client.0.smithi136.stdout:op 4693 completed, throughput=5MB/sec 2024-09-13T00:16:52.443 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-zos237ZzlC85WRy off=10758318 len=527049 2024-09-13T00:16:52.449 INFO:tasks.workunit.client.0.smithi136.stdout:op 4694 completed, throughput=5MB/sec 2024-09-13T00:16:52.449 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2NqdkRdwGUvg4Pv off=0 len=1285492 2024-09-13T00:16:52.453 INFO:tasks.workunit.client.0.smithi136.stdout:op 4695 completed, throughput=5MB/sec 2024-09-13T00:16:52.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:16:52 smithi181 bash[95505]: ts=2024-09-13T00:16:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:16:53.453 INFO:tasks.workunit.client.0.smithi136.stdout: 901: throughput=5MB/sec pending data=0 2024-09-13T00:16:53.453 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-jKbfh7c4efX6URp off=376539 len=1378211 2024-09-13T00:16:53.458 INFO:tasks.workunit.client.0.smithi136.stdout:op 4696 completed, throughput=5MB/sec 2024-09-13T00:16:53.458 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-9DaNCX-6qTwbpV7 off=1567299 len=1148290 2024-09-13T00:16:53.461 INFO:tasks.workunit.client.0.smithi136.stdout:op 4697 completed, throughput=5MB/sec 2024-09-13T00:16:53.461 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ygzYDYYlMGrTU6R off=2400703 len=365398 2024-09-13T00:16:53.462 INFO:tasks.workunit.client.0.smithi136.stdout:op 4698 completed, throughput=5MB/sec 2024-09-13T00:16:53.462 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-bDilLhjfFCNgMCc off=3650615 len=1460078 2024-09-13T00:16:53.473 INFO:tasks.workunit.client.0.smithi136.stdout:op 4699 completed, throughput=5MB/sec 2024-09-13T00:16:53.473 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-UqZVYLM3E1ROBx8 off=1592078 len=1165480 2024-09-13T00:16:53.476 INFO:tasks.workunit.client.0.smithi136.stdout:op 4700 completed, throughput=5MB/sec 2024-09-13T00:16:53.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:16:53 smithi136 bash[76371]: cluster 2024-09-13T00:16:51.607318+0000 mgr.y (mgr.24661) 1192 : cluster [DBG] pgmap v1875: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:16:53.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:16:53 smithi136 bash[75164]: cluster 2024-09-13T00:16:51.607318+0000 mgr.y (mgr.24661) 1192 : cluster [DBG] pgmap v1875: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:16:53.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:16:53 smithi181 bash[67587]: cluster 2024-09-13T00:16:51.607318+0000 mgr.y (mgr.24661) 1192 : cluster [DBG] pgmap v1875: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:16:54.476 INFO:tasks.workunit.client.0.smithi136.stdout: 902: throughput=5MB/sec pending data=0 2024-09-13T00:16:54.476 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-p0JqCzTBAesGpN8 off=761729 len=1993874 2024-09-13T00:16:54.491 INFO:tasks.workunit.client.0.smithi136.stdout:op 4701 completed, throughput=5MB/sec 2024-09-13T00:16:54.491 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-s4Z8Mi6lS4sxzjW off=1155403 len=1557421 2024-09-13T00:16:54.502 INFO:tasks.workunit.client.0.smithi136.stdout:op 4702 completed, throughput=5MB/sec 2024-09-13T00:16:54.503 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ndUuOekJOeekj-M off=10345090 len=563359 2024-09-13T00:16:54.504 INFO:tasks.workunit.client.0.smithi136.stdout:op 4703 completed, throughput=5MB/sec 2024-09-13T00:16:54.504 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-mEW3Tx9cfqHEuHU off=70122 len=546434 2024-09-13T00:16:54.507 INFO:tasks.workunit.client.0.smithi136.stdout:op 4704 completed, throughput=5MB/sec 2024-09-13T00:16:54.507 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:16:54 smithi136 bash[76371]: audit 2024-09-13T00:16:53.797627+0000 mon.a (mon.0) 2241 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:16:54.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:16:54 smithi136 bash[75164]: audit 2024-09-13T00:16:53.797627+0000 mon.a (mon.0) 2241 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:16:54.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:16:54 smithi181 bash[67587]: audit 2024-09-13T00:16:53.797627+0000 mon.a (mon.0) 2241 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:16:55.507 INFO:tasks.workunit.client.0.smithi136.stdout: 903: throughput=4.99MB/sec pending data=0 2024-09-13T00:16:55.507 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-iWjdjRpnGSgK_9h off=878813 len=716806 2024-09-13T00:16:55.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:16:55 smithi136 bash[76371]: cluster 2024-09-13T00:16:53.608563+0000 mgr.y (mgr.24661) 1193 : cluster [DBG] pgmap v1876: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:16:55.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:16:55 smithi136 bash[75164]: cluster 2024-09-13T00:16:53.608563+0000 mgr.y (mgr.24661) 1193 : cluster [DBG] pgmap v1876: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:16:55.510 INFO:tasks.workunit.client.0.smithi136.stdout:op 4705 completed, throughput=5MB/sec 2024-09-13T00:16:55.510 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ies0w-ttFwg2NX- off=3513369 len=1674819 2024-09-13T00:16:55.522 INFO:tasks.workunit.client.0.smithi136.stdout:op 4706 completed, throughput=5MB/sec 2024-09-13T00:16:55.522 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-HJcZciw4qmbPSSs off=157965 len=1199899 2024-09-13T00:16:55.532 INFO:tasks.workunit.client.0.smithi136.stdout:op 4707 completed, throughput=5MB/sec 2024-09-13T00:16:55.532 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-KxYJJebyxbjk8U5 off=5051506 len=1998793 2024-09-13T00:16:55.545 INFO:tasks.workunit.client.0.smithi136.stdout:op 4708 completed, throughput=5MB/sec 2024-09-13T00:16:55.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:16:55 smithi181 bash[67587]: cluster 2024-09-13T00:16:53.608563+0000 mgr.y (mgr.24661) 1193 : cluster [DBG] pgmap v1876: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:16:56.545 INFO:tasks.workunit.client.0.smithi136.stdout: 904: throughput=4.99MB/sec pending data=0 2024-09-13T00:16:56.545 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Roqy7tg37i8Oo1U off=10135846 len=996454 2024-09-13T00:16:56.548 INFO:tasks.workunit.client.0.smithi136.stdout:op 4709 completed, throughput=5MB/sec 2024-09-13T00:16:56.548 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-YNbttY32xlJ7KRl off=8856834 len=350322 2024-09-13T00:16:56.555 INFO:tasks.workunit.client.0.smithi136.stdout:op 4710 completed, throughput=5MB/sec 2024-09-13T00:16:56.555 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-W0tvbyeIILJICCT off=434715 len=1998412 2024-09-13T00:16:56.559 INFO:tasks.workunit.client.0.smithi136.stdout:op 4711 completed, throughput=5MB/sec 2024-09-13T00:16:56.559 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-GdidoVnCJv444X1 off=15602852 len=717211 2024-09-13T00:16:56.562 INFO:tasks.workunit.client.0.smithi136.stdout:op 4712 completed, throughput=5MB/sec 2024-09-13T00:16:56.562 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-s2vylAlPBweaWo- off=2336891 len=1462826 2024-09-13T00:16:56.572 INFO:tasks.workunit.client.0.smithi136.stdout:op 4713 completed, throughput=5MB/sec 2024-09-13T00:16:57.507 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:16:57 smithi136 bash[75164]: cluster 2024-09-13T00:16:55.609489+0000 mgr.y (mgr.24661) 1194 : cluster [DBG] pgmap v1877: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:16:57.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:16:57 smithi136 bash[75164]: audit 2024-09-13T00:16:57.128175+0000 mon.a (mon.0) 2242 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:16:57.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:16:57 smithi136 bash[76371]: cluster 2024-09-13T00:16:55.609489+0000 mgr.y (mgr.24661) 1194 : cluster [DBG] pgmap v1877: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:16:57.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:16:57 smithi136 bash[76371]: audit 2024-09-13T00:16:57.128175+0000 mon.a (mon.0) 2242 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:16:57.572 INFO:tasks.workunit.client.0.smithi136.stdout: 905: throughput=5MB/sec pending data=0 2024-09-13T00:16:57.573 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-NUiArYPXTa5Zs9V off=4946783 len=724303 2024-09-13T00:16:57.578 INFO:tasks.workunit.client.0.smithi136.stdout:op 4714 completed, throughput=5MB/sec 2024-09-13T00:16:57.578 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-CD9f1QDDTEOWqJp off=4248881 len=2058649 2024-09-13T00:16:57.581 INFO:tasks.workunit.client.0.smithi136.stdout:op 4715 completed, throughput=5MB/sec 2024-09-13T00:16:57.582 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-dEkiBC2TeczJ58D off=8138331 len=1720888 2024-09-13T00:16:57.585 INFO:tasks.workunit.client.0.smithi136.stdout:op 4716 completed, throughput=5MB/sec 2024-09-13T00:16:57.585 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-VcImkT6gfpz6VCs off=216541 len=1363608 2024-09-13T00:16:57.587 INFO:tasks.workunit.client.0.smithi136.stdout:op 4717 completed, throughput=5MB/sec 2024-09-13T00:16:57.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:16:57 smithi181 bash[67587]: cluster 2024-09-13T00:16:55.609489+0000 mgr.y (mgr.24661) 1194 : cluster [DBG] pgmap v1877: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:16:57.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:16:57 smithi181 bash[67587]: audit 2024-09-13T00:16:57.128175+0000 mon.a (mon.0) 2242 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:16:58.587 INFO:tasks.workunit.client.0.smithi136.stdout: 906: throughput=5MB/sec pending data=0 2024-09-13T00:16:58.587 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-NOrbJXD4DwMo35a off=1970180 len=1916540 2024-09-13T00:16:58.591 INFO:tasks.workunit.client.0.smithi136.stdout:op 4718 completed, throughput=5MB/sec 2024-09-13T00:16:58.592 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-lJCupTtf6bB5zcu off=19632019 len=45808 2024-09-13T00:16:58.597 INFO:tasks.workunit.client.0.smithi136.stdout:op 4719 completed, throughput=5MB/sec 2024-09-13T00:16:58.597 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-hFmaTase-cQzxL3 off=3865570 len=2026677 2024-09-13T00:16:58.604 INFO:tasks.workunit.client.0.smithi136.stdout:op 4720 completed, throughput=5MB/sec 2024-09-13T00:16:58.604 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-W7o-3NC71bLm15u off=279892 len=1030712 2024-09-13T00:16:58.607 INFO:tasks.workunit.client.0.smithi136.stdout:op 4721 completed, throughput=5MB/sec 2024-09-13T00:16:58.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:16:58 smithi181 bash[67587]: audit 2024-09-13T00:16:57.511345+0000 mon.a (mon.0) 2243 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:16:58.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:16:58 smithi181 bash[67587]: audit 2024-09-13T00:16:57.518199+0000 mon.a (mon.0) 2244 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:16:58.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:16:58 smithi181 bash[67587]: cluster 2024-09-13T00:16:57.610574+0000 mgr.y (mgr.24661) 1195 : cluster [DBG] pgmap v1878: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.8 MiB/s wr, 4 op/s 2024-09-13T00:16:58.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:16:58 smithi181 bash[67587]: audit 2024-09-13T00:16:57.943067+0000 mon.a (mon.0) 2245 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:16:58.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:16:58 smithi181 bash[67587]: audit 2024-09-13T00:16:57.944487+0000 mon.a (mon.0) 2246 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:16:58.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:16:58 smithi181 bash[67587]: audit 2024-09-13T00:16:57.953165+0000 mon.a (mon.0) 2247 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:16:59.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:16:58 smithi136 bash[75164]: audit 2024-09-13T00:16:57.511345+0000 mon.a (mon.0) 2243 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:16:59.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:16:58 smithi136 bash[75164]: audit 2024-09-13T00:16:57.518199+0000 mon.a (mon.0) 2244 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:16:59.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:16:58 smithi136 bash[75164]: cluster 2024-09-13T00:16:57.610574+0000 mgr.y (mgr.24661) 1195 : cluster [DBG] pgmap v1878: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.8 MiB/s wr, 4 op/s 2024-09-13T00:16:59.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:16:58 smithi136 bash[75164]: audit 2024-09-13T00:16:57.943067+0000 mon.a (mon.0) 2245 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:16:59.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:16:58 smithi136 bash[75164]: audit 2024-09-13T00:16:57.944487+0000 mon.a (mon.0) 2246 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:16:59.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:16:58 smithi136 bash[75164]: audit 2024-09-13T00:16:57.953165+0000 mon.a (mon.0) 2247 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:16:59.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:16:58 smithi136 bash[76371]: audit 2024-09-13T00:16:57.511345+0000 mon.a (mon.0) 2243 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:16:59.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:16:58 smithi136 bash[76371]: audit 2024-09-13T00:16:57.518199+0000 mon.a (mon.0) 2244 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:16:59.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:16:58 smithi136 bash[76371]: cluster 2024-09-13T00:16:57.610574+0000 mgr.y (mgr.24661) 1195 : cluster [DBG] pgmap v1878: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.8 MiB/s wr, 4 op/s 2024-09-13T00:16:59.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:16:58 smithi136 bash[76371]: audit 2024-09-13T00:16:57.943067+0000 mon.a (mon.0) 2245 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:16:59.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:16:58 smithi136 bash[76371]: audit 2024-09-13T00:16:57.944487+0000 mon.a (mon.0) 2246 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:16:59.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:16:58 smithi136 bash[76371]: audit 2024-09-13T00:16:57.953165+0000 mon.a (mon.0) 2247 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:16:59.607 INFO:tasks.workunit.client.0.smithi136.stdout: 907: throughput=5MB/sec pending data=0 2024-09-13T00:16:59.607 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-QTHKNOKVLTQvqxA off=6361896 len=1845407 2024-09-13T00:16:59.612 INFO:tasks.workunit.client.0.smithi136.stdout:op 4722 completed, throughput=5MB/sec 2024-09-13T00:16:59.612 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-3bgVxUmtT0u5tnD off=15614523 len=645849 2024-09-13T00:16:59.619 INFO:tasks.workunit.client.0.smithi136.stdout:op 4723 completed, throughput=5MB/sec 2024-09-13T00:16:59.619 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-7EjLGfgweN6yUjx off=12088732 len=98967 2024-09-13T00:16:59.620 INFO:tasks.workunit.client.0.smithi136.stdout:op 4724 completed, throughput=5MB/sec 2024-09-13T00:16:59.620 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-KbS2phJ59Tkdtn2 off=3532270 len=1407096 2024-09-13T00:16:59.625 INFO:tasks.workunit.client.0.smithi136.stdout:op 4725 completed, throughput=5MB/sec 2024-09-13T00:16:59.625 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-gID-r1ObK13jQFX off=6016365 len=742244 2024-09-13T00:16:59.633 INFO:tasks.workunit.client.0.smithi136.stdout:op 4726 completed, throughput=5MB/sec 2024-09-13T00:16:59.757 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:16:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:16:59] "GET /metrics HTTP/1.1" 200 34142 "" "Prometheus/2.43.0" 2024-09-13T00:17:00.633 INFO:tasks.workunit.client.0.smithi136.stdout: 908: throughput=4.99MB/sec pending data=0 2024-09-13T00:17:00.633 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-osoCOgwdbAoRfBU off=2835358 len=201512 2024-09-13T00:17:00.635 INFO:tasks.workunit.client.0.smithi136.stdout:op 4727 completed, throughput=4.99MB/sec 2024-09-13T00:17:00.635 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-UzaiDgo6lSMTebb off=8514545 len=1187616 2024-09-13T00:17:00.638 INFO:tasks.workunit.client.0.smithi136.stdout:op 4728 completed, throughput=5MB/sec 2024-09-13T00:17:00.638 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2KwY2d1dqbO6be3 off=259039 len=460714 2024-09-13T00:17:00.640 INFO:tasks.workunit.client.0.smithi136.stdout:op 4729 completed, throughput=5MB/sec 2024-09-13T00:17:00.640 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-NdNiNP_1og2wjtK off=16288330 len=389676 2024-09-13T00:17:00.642 INFO:tasks.workunit.client.0.smithi136.stdout:op 4730 completed, throughput=5MB/sec 2024-09-13T00:17:00.642 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-0BipSrWJJfGzqzW off=11565358 len=1699609 2024-09-13T00:17:00.646 INFO:tasks.workunit.client.0.smithi136.stdout:op 4731 completed, throughput=5MB/sec 2024-09-13T00:17:00.646 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-PRAOOEz6oVfbKUH off=8815712 len=277977 2024-09-13T00:17:00.647 INFO:tasks.workunit.client.0.smithi136.stdout:op 4732 completed, throughput=5MB/sec 2024-09-13T00:17:00.647 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-RmoQy3nMzzTf5xb off=2977682 len=79750 2024-09-13T00:17:00.652 INFO:tasks.workunit.client.0.smithi136.stdout:op 4733 completed, throughput=5MB/sec 2024-09-13T00:17:00.652 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-hQuivZ3ks7qFGV3 off=5152189 len=1400371 2024-09-13T00:17:00.655 INFO:tasks.workunit.client.0.smithi136.stdout:op 4734 completed, throughput=5MB/sec 2024-09-13T00:17:01.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:17:00 smithi136 bash[75164]: cluster 2024-09-13T00:16:59.611359+0000 mgr.y (mgr.24661) 1196 : cluster [DBG] pgmap v1879: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:17:01.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:17:00 smithi136 bash[76371]: cluster 2024-09-13T00:16:59.611359+0000 mgr.y (mgr.24661) 1196 : cluster [DBG] pgmap v1879: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:17:01.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:17:00 smithi181 bash[67587]: cluster 2024-09-13T00:16:59.611359+0000 mgr.y (mgr.24661) 1196 : cluster [DBG] pgmap v1879: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:17:01.656 INFO:tasks.workunit.client.0.smithi136.stdout: 909: throughput=5MB/sec pending data=0 2024-09-13T00:17:01.656 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ITuA-NTvnkH4c93 off=10417189 len=1175896 2024-09-13T00:17:01.659 INFO:tasks.workunit.client.0.smithi136.stdout:op 4735 completed, throughput=5MB/sec 2024-09-13T00:17:01.659 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-3bGJobrf5u8YD5n off=1286246 len=505916 2024-09-13T00:17:01.666 INFO:tasks.workunit.client.0.smithi136.stdout:op 4736 completed, throughput=5MB/sec 2024-09-13T00:17:01.666 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-QEgV_8p9prs9BG2 off=1580549 len=755447 2024-09-13T00:17:01.669 INFO:tasks.workunit.client.0.smithi136.stdout:op 4737 completed, throughput=5MB/sec 2024-09-13T00:17:01.669 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-zxR30OLf_E2PbrV off=1503314 len=239052 2024-09-13T00:17:01.673 INFO:tasks.workunit.client.0.smithi136.stdout:op 4738 completed, throughput=5MB/sec 2024-09-13T00:17:01.674 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-J-Famk1FlwwTxXz off=244876 len=1565915 2024-09-13T00:17:01.679 INFO:tasks.workunit.client.0.smithi136.stdout:op 4739 completed, throughput=5MB/sec 2024-09-13T00:17:01.679 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-d4CZ5t6lNCIS6-p off=5149180 len=1612065 2024-09-13T00:17:01.692 INFO:tasks.workunit.client.0.smithi136.stdout:op 4740 completed, throughput=5MB/sec 2024-09-13T00:17:02.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:17:02 smithi181 bash[95505]: ts=2024-09-13T00:17:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:17:02.692 INFO:tasks.workunit.client.0.smithi136.stdout: 910: throughput=5MB/sec pending data=0 2024-09-13T00:17:02.692 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-nukFMjoFddBWZ3b off=6051188 len=1145611 2024-09-13T00:17:02.701 INFO:tasks.workunit.client.0.smithi136.stdout:op 4741 completed, throughput=5MB/sec 2024-09-13T00:17:02.701 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-heHPohs0QlKs9yD off=1269385 len=1934119 2024-09-13T00:17:02.708 INFO:tasks.workunit.client.0.smithi136.stdout:op 4742 completed, throughput=5MB/sec 2024-09-13T00:17:02.708 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-g7p4GL5QGgOoLw6 off=5162487 len=1136364 2024-09-13T00:17:02.713 INFO:tasks.workunit.client.0.smithi136.stdout:op 4743 completed, throughput=5MB/sec 2024-09-13T00:17:02.713 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-n0BcZp0iDnjH49e off=8414050 len=21706 2024-09-13T00:17:02.714 INFO:tasks.workunit.client.0.smithi136.stdout:op 4744 completed, throughput=5MB/sec 2024-09-13T00:17:02.714 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-kUPNwr1w15674n- off=1876270 len=257154 2024-09-13T00:17:02.715 INFO:tasks.workunit.client.0.smithi136.stdout:op 4745 completed, throughput=5MB/sec 2024-09-13T00:17:02.715 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-rBGCoxQ0qVW2L9h off=2781883 len=1129940 2024-09-13T00:17:02.718 INFO:tasks.workunit.client.0.smithi136.stdout:op 4746 completed, throughput=5MB/sec 2024-09-13T00:17:03.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:17:02 smithi136 bash[75164]: cluster 2024-09-13T00:17:01.612406+0000 mgr.y (mgr.24661) 1197 : cluster [DBG] pgmap v1880: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.1 MiB/s wr, 3 op/s 2024-09-13T00:17:03.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:17:02 smithi136 bash[76371]: cluster 2024-09-13T00:17:01.612406+0000 mgr.y (mgr.24661) 1197 : cluster [DBG] pgmap v1880: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.1 MiB/s wr, 3 op/s 2024-09-13T00:17:03.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:17:02 smithi181 bash[67587]: cluster 2024-09-13T00:17:01.612406+0000 mgr.y (mgr.24661) 1197 : cluster [DBG] pgmap v1880: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.1 MiB/s wr, 3 op/s 2024-09-13T00:17:03.719 INFO:tasks.workunit.client.0.smithi136.stdout: 911: throughput=5MB/sec pending data=0 2024-09-13T00:17:03.719 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-On6b4LXuYRLB7nO off=1731907 len=233395 2024-09-13T00:17:03.721 INFO:tasks.workunit.client.0.smithi136.stdout:op 4747 completed, throughput=5MB/sec 2024-09-13T00:17:03.721 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-H-8YB7IL_9K4EXX off=61427 len=329192 2024-09-13T00:17:03.722 INFO:tasks.workunit.client.0.smithi136.stdout:op 4748 completed, throughput=5MB/sec 2024-09-13T00:17:03.723 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-3RajO5mAP3_l1fv off=10992177 len=1502164 2024-09-13T00:17:03.735 INFO:tasks.workunit.client.0.smithi136.stdout:op 4749 completed, throughput=5MB/sec 2024-09-13T00:17:03.735 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-kN3i8R-vRRwPF3t off=6910335 len=56641 2024-09-13T00:17:03.739 INFO:tasks.workunit.client.0.smithi136.stdout:op 4750 completed, throughput=5MB/sec 2024-09-13T00:17:03.740 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-cv_DVK_yUlN64GO off=861309 len=1181313 2024-09-13T00:17:03.750 INFO:tasks.workunit.client.0.smithi136.stdout:op 4751 completed, throughput=5MB/sec 2024-09-13T00:17:03.750 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-9qQWjqekj9TOlYo off=10660038 len=313328 2024-09-13T00:17:03.752 INFO:tasks.workunit.client.0.smithi136.stdout:op 4752 completed, throughput=5MB/sec 2024-09-13T00:17:03.752 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Ng4WJl3jkbVRTq8 off=17308169 len=740581 2024-09-13T00:17:03.754 INFO:tasks.workunit.client.0.smithi136.stdout:op 4753 completed, throughput=5MB/sec 2024-09-13T00:17:03.754 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-M6tehv4Eykv32-5 off=12365288 len=1223663 2024-09-13T00:17:03.757 INFO:tasks.workunit.client.0.smithi136.stdout:op 4754 completed, throughput=5MB/sec 2024-09-13T00:17:04.758 INFO:tasks.workunit.client.0.smithi136.stdout: 912: throughput=5MB/sec pending data=0 2024-09-13T00:17:04.758 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-IL2WBZhGKTCbw4x off=4779125 len=798269 2024-09-13T00:17:04.761 INFO:tasks.workunit.client.0.smithi136.stdout:op 4755 completed, throughput=5MB/sec 2024-09-13T00:17:04.761 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj--riK5zaVeiahOfe off=133011 len=1907925 2024-09-13T00:17:04.777 INFO:tasks.workunit.client.0.smithi136.stdout:op 4756 completed, throughput=5MB/sec 2024-09-13T00:17:04.777 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-aXPMdtad5mrAakZ off=19627185 len=1534034 2024-09-13T00:17:04.781 INFO:tasks.workunit.client.0.smithi136.stdout:op 4757 completed, throughput=5MB/sec 2024-09-13T00:17:05.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:17:04 smithi136 bash[75164]: cluster 2024-09-13T00:17:03.613474+0000 mgr.y (mgr.24661) 1198 : cluster [DBG] pgmap v1881: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:17:05.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:17:04 smithi136 bash[76371]: cluster 2024-09-13T00:17:03.613474+0000 mgr.y (mgr.24661) 1198 : cluster [DBG] pgmap v1881: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:17:05.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:17:04 smithi181 bash[67587]: cluster 2024-09-13T00:17:03.613474+0000 mgr.y (mgr.24661) 1198 : cluster [DBG] pgmap v1881: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:17:05.782 INFO:tasks.workunit.client.0.smithi136.stdout: 913: throughput=4.99MB/sec pending data=0 2024-09-13T00:17:05.782 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-q-pU86PgDUmeV5a off=713188 len=630162 2024-09-13T00:17:05.788 INFO:tasks.workunit.client.0.smithi136.stdout:op 4758 completed, throughput=5MB/sec 2024-09-13T00:17:05.788 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-NHTaATXUikBz8U5 off=8723548 len=1348674 2024-09-13T00:17:05.791 INFO:tasks.workunit.client.0.smithi136.stdout:op 4759 completed, throughput=5MB/sec 2024-09-13T00:17:05.791 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-UHqyy9sRa9fWFVi off=15162471 len=1161301 2024-09-13T00:17:05.795 INFO:tasks.workunit.client.0.smithi136.stdout:op 4760 completed, throughput=5MB/sec 2024-09-13T00:17:05.795 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-RGf873Umrcg50Vg off=18560826 len=1223343 2024-09-13T00:17:05.803 INFO:tasks.workunit.client.0.smithi136.stdout:op 4761 completed, throughput=5MB/sec 2024-09-13T00:17:05.803 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-3PlC-8K52vmkp25 off=2647307 len=161514 2024-09-13T00:17:05.808 INFO:tasks.workunit.client.0.smithi136.stdout:op 4762 completed, throughput=5MB/sec 2024-09-13T00:17:05.808 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-iXgKdAVXDbi-rIe off=8161761 len=481544 2024-09-13T00:17:05.809 INFO:tasks.workunit.client.0.smithi136.stdout:op 4763 completed, throughput=5MB/sec 2024-09-13T00:17:05.809 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-f2N7K2HDmRmElCO off=479308 len=1081880 2024-09-13T00:17:05.817 INFO:tasks.workunit.client.0.smithi136.stdout:op 4764 completed, throughput=5MB/sec 2024-09-13T00:17:06.818 INFO:tasks.workunit.client.0.smithi136.stdout: 914: throughput=5MB/sec pending data=0 2024-09-13T00:17:06.818 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-5jPAJ49FuEK9Mvv off=10698747 len=806898 2024-09-13T00:17:06.820 INFO:tasks.workunit.client.0.smithi136.stdout:op 4765 completed, throughput=5MB/sec 2024-09-13T00:17:06.820 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-JycIb_fR013f031 off=375860 len=934281 2024-09-13T00:17:06.823 INFO:tasks.workunit.client.0.smithi136.stdout:op 4766 completed, throughput=5MB/sec 2024-09-13T00:17:06.823 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-yQ4MD06AmLZRFH4 off=376493 len=1022839 2024-09-13T00:17:06.826 INFO:tasks.workunit.client.0.smithi136.stdout:op 4767 completed, throughput=5MB/sec 2024-09-13T00:17:06.826 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-NjxIcMTt4MdEzLC off=11282124 len=1221555 2024-09-13T00:17:06.829 INFO:tasks.workunit.client.0.smithi136.stdout:op 4768 completed, throughput=5MB/sec 2024-09-13T00:17:06.829 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-k_rHrCz-EB4iQXb off=3345194 len=315233 2024-09-13T00:17:06.835 INFO:tasks.workunit.client.0.smithi136.stdout:op 4769 completed, throughput=5MB/sec 2024-09-13T00:17:06.835 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-GA1YrQPOmVt0ZVG off=5529444 len=78912 2024-09-13T00:17:06.836 INFO:tasks.workunit.client.0.smithi136.stdout:op 4770 completed, throughput=5MB/sec 2024-09-13T00:17:06.836 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ffQwrCGGwt8vwQJ off=8145603 len=1838994 2024-09-13T00:17:06.841 INFO:tasks.workunit.client.0.smithi136.stdout:op 4771 completed, throughput=5MB/sec 2024-09-13T00:17:07.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:17:06 smithi136 bash[75164]: cluster 2024-09-13T00:17:05.614252+0000 mgr.y (mgr.24661) 1199 : cluster [DBG] pgmap v1882: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 3.0 MiB/s rd, 1.6 MiB/s wr, 4 op/s 2024-09-13T00:17:07.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:17:06 smithi136 bash[76371]: cluster 2024-09-13T00:17:05.614252+0000 mgr.y (mgr.24661) 1199 : cluster [DBG] pgmap v1882: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 3.0 MiB/s rd, 1.6 MiB/s wr, 4 op/s 2024-09-13T00:17:07.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:17:06 smithi181 bash[67587]: cluster 2024-09-13T00:17:05.614252+0000 mgr.y (mgr.24661) 1199 : cluster [DBG] pgmap v1882: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 3.0 MiB/s rd, 1.6 MiB/s wr, 4 op/s 2024-09-13T00:17:07.841 INFO:tasks.workunit.client.0.smithi136.stdout: 915: throughput=5MB/sec pending data=0 2024-09-13T00:17:07.841 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-6Fse1YZAQ2N_hm8 off=6523212 len=467434 2024-09-13T00:17:07.850 INFO:tasks.workunit.client.0.smithi136.stdout:op 4772 completed, throughput=5MB/sec 2024-09-13T00:17:07.850 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Vs_GJZISHrz18oA off=1165706 len=649292 2024-09-13T00:17:07.856 INFO:tasks.workunit.client.0.smithi136.stdout:op 4773 completed, throughput=5MB/sec 2024-09-13T00:17:07.856 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-T-SlO6W0QlN3gj_ off=8963628 len=592722 2024-09-13T00:17:07.865 INFO:tasks.workunit.client.0.smithi136.stdout:op 4774 completed, throughput=5MB/sec 2024-09-13T00:17:07.865 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-w6_kYqPqNmweZbw off=417700 len=832195 2024-09-13T00:17:07.874 INFO:tasks.workunit.client.0.smithi136.stdout:op 4775 completed, throughput=5MB/sec 2024-09-13T00:17:07.874 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Kiygriefjvoz7n4 off=609169 len=1001566 2024-09-13T00:17:07.877 INFO:tasks.workunit.client.0.smithi136.stdout:op 4776 completed, throughput=5MB/sec 2024-09-13T00:17:07.877 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-V2DGPwXM5ZjHqKR off=1381092 len=859798 2024-09-13T00:17:07.886 INFO:tasks.workunit.client.0.smithi136.stdout:op 4777 completed, throughput=5MB/sec 2024-09-13T00:17:08.886 INFO:tasks.workunit.client.0.smithi136.stdout: 916: throughput=5MB/sec pending data=0 2024-09-13T00:17:08.886 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-WFS5gOgrEafDYb5 off=1887026 len=281311 2024-09-13T00:17:08.888 INFO:tasks.workunit.client.0.smithi136.stdout:op 4778 completed, throughput=5MB/sec 2024-09-13T00:17:08.888 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-hKZGaUIOUcL6B6g off=15584385 len=1958640 2024-09-13T00:17:08.906 INFO:tasks.workunit.client.0.smithi136.stdout:op 4779 completed, throughput=5MB/sec 2024-09-13T00:17:08.906 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-0f7piFDXmxghXIF off=10417429 len=1419182 2024-09-13T00:17:08.910 INFO:tasks.workunit.client.0.smithi136.stdout:op 4780 completed, throughput=5MB/sec 2024-09-13T00:17:08.910 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-32USNrgVQ915eGZ off=527695 len=1571245 2024-09-13T00:17:08.916 INFO:tasks.workunit.client.0.smithi136.stdout:op 4781 completed, throughput=5MB/sec 2024-09-13T00:17:09.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:17:08 smithi136 bash[75164]: cluster 2024-09-13T00:17:07.615347+0000 mgr.y (mgr.24661) 1200 : cluster [DBG] pgmap v1883: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.7 MiB/s wr, 5 op/s 2024-09-13T00:17:09.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:17:08 smithi136 bash[76371]: cluster 2024-09-13T00:17:07.615347+0000 mgr.y (mgr.24661) 1200 : cluster [DBG] pgmap v1883: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.7 MiB/s wr, 5 op/s 2024-09-13T00:17:09.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:17:08 smithi181 bash[67587]: cluster 2024-09-13T00:17:07.615347+0000 mgr.y (mgr.24661) 1200 : cluster [DBG] pgmap v1883: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.7 MiB/s wr, 5 op/s 2024-09-13T00:17:09.689 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:17:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:17:09] "GET /metrics HTTP/1.1" 200 34145 "" "Prometheus/2.43.0" 2024-09-13T00:17:09.915 INFO:tasks.workunit.client.0.smithi136.stdout: 917: throughput=5MB/sec pending data=0 2024-09-13T00:17:09.915 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj--qzkUsxoSZrqnfX off=13290608 len=921002 2024-09-13T00:17:09.918 INFO:tasks.workunit.client.0.smithi136.stdout:op 4782 completed, throughput=5MB/sec 2024-09-13T00:17:09.918 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-YCptYrfmW7mOSpj off=7841883 len=1138304 2024-09-13T00:17:09.922 INFO:tasks.workunit.client.0.smithi136.stdout:op 4783 completed, throughput=5MB/sec 2024-09-13T00:17:09.922 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-dnbt96cQHyTgi4L off=6408600 len=1628220 2024-09-13T00:17:09.934 INFO:tasks.workunit.client.0.smithi136.stdout:op 4784 completed, throughput=5MB/sec 2024-09-13T00:17:09.934 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-dDSsdoudkQwcVSI off=10433877 len=565206 2024-09-13T00:17:09.941 INFO:tasks.workunit.client.0.smithi136.stdout:op 4785 completed, throughput=5MB/sec 2024-09-13T00:17:09.941 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-u3Fx7DbFtGRyWXv off=3338683 len=2079874 2024-09-13T00:17:09.945 INFO:tasks.workunit.client.0.smithi136.stdout:op 4786 completed, throughput=5MB/sec 2024-09-13T00:17:10.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:17:09 smithi136 bash[75164]: audit 2024-09-13T00:17:08.798103+0000 mon.a (mon.0) 2248 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:17:10.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:17:09 smithi136 bash[76371]: audit 2024-09-13T00:17:08.798103+0000 mon.a (mon.0) 2248 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:17:10.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:17:09 smithi181 bash[67587]: audit 2024-09-13T00:17:08.798103+0000 mon.a (mon.0) 2248 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:17:10.945 INFO:tasks.workunit.client.0.smithi136.stdout: 918: throughput=5MB/sec pending data=0 2024-09-13T00:17:10.946 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-P4tQOwFIVbZiVVF off=1785253 len=1401935 2024-09-13T00:17:10.958 INFO:tasks.workunit.client.0.smithi136.stdout:op 4787 completed, throughput=5MB/sec 2024-09-13T00:17:10.958 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-hQGh_r_ZSluGdqn off=12420533 len=351526 2024-09-13T00:17:10.964 INFO:tasks.workunit.client.0.smithi136.stdout:op 4788 completed, throughput=5MB/sec 2024-09-13T00:17:10.964 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-q7vawnmlR_1aMbi off=1963971 len=629990 2024-09-13T00:17:10.966 INFO:tasks.workunit.client.0.smithi136.stdout:op 4789 completed, throughput=5MB/sec 2024-09-13T00:17:10.967 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-fdsxVOVicVtzx7A off=2115835 len=1823566 2024-09-13T00:17:10.972 INFO:tasks.workunit.client.0.smithi136.stdout:op 4790 completed, throughput=5MB/sec 2024-09-13T00:17:11.007 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:17:10 smithi136 bash[75164]: cluster 2024-09-13T00:17:09.616176+0000 mgr.y (mgr.24661) 1201 : cluster [DBG] pgmap v1884: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.7 MiB/s wr, 5 op/s 2024-09-13T00:17:11.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:17:10 smithi136 bash[76371]: cluster 2024-09-13T00:17:09.616176+0000 mgr.y (mgr.24661) 1201 : cluster [DBG] pgmap v1884: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.7 MiB/s wr, 5 op/s 2024-09-13T00:17:11.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:17:10 smithi181 bash[67587]: cluster 2024-09-13T00:17:09.616176+0000 mgr.y (mgr.24661) 1201 : cluster [DBG] pgmap v1884: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.7 MiB/s wr, 5 op/s 2024-09-13T00:17:11.973 INFO:tasks.workunit.client.0.smithi136.stdout: 919: throughput=4.99MB/sec pending data=0 2024-09-13T00:17:11.973 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-hdZBDdP6HJjgSIM off=1549148 len=1969839 2024-09-13T00:17:11.977 INFO:tasks.workunit.client.0.smithi136.stdout:op 4791 completed, throughput=5MB/sec 2024-09-13T00:17:11.978 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-qI1VF3gOHuKaCmp off=8942111 len=1333466 2024-09-13T00:17:11.981 INFO:tasks.workunit.client.0.smithi136.stdout:op 4792 completed, throughput=5MB/sec 2024-09-13T00:17:11.981 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-D_LnGSwmxe0TPDm off=15148106 len=195154 2024-09-13T00:17:11.986 INFO:tasks.workunit.client.0.smithi136.stdout:op 4793 completed, throughput=5MB/sec 2024-09-13T00:17:11.986 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ISOIb7FB4YwC27C off=9936599 len=1988173 2024-09-13T00:17:12.003 INFO:tasks.workunit.client.0.smithi136.stdout:op 4794 completed, throughput=5MB/sec 2024-09-13T00:17:12.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:17:12 smithi181 bash[95505]: ts=2024-09-13T00:17:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:17:13.003 INFO:tasks.workunit.client.0.smithi136.stdout: 920: throughput=4.99MB/sec pending data=0 2024-09-13T00:17:13.004 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_n2YkPTxuyjaLty off=5926643 len=799538 2024-09-13T00:17:13.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:17:12 smithi136 bash[75164]: cluster 2024-09-13T00:17:11.617175+0000 mgr.y (mgr.24661) 1202 : cluster [DBG] pgmap v1885: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.5 MiB/s wr, 5 op/s 2024-09-13T00:17:13.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:17:12 smithi136 bash[76371]: cluster 2024-09-13T00:17:11.617175+0000 mgr.y (mgr.24661) 1202 : cluster [DBG] pgmap v1885: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.5 MiB/s wr, 5 op/s 2024-09-13T00:17:13.012 INFO:tasks.workunit.client.0.smithi136.stdout:op 4795 completed, throughput=5MB/sec 2024-09-13T00:17:13.012 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-5Z_qOD439FVMWaH off=2092731 len=1248088 2024-09-13T00:17:13.022 INFO:tasks.workunit.client.0.smithi136.stdout:op 4796 completed, throughput=5MB/sec 2024-09-13T00:17:13.022 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-RS16mqP6Lxe2zVu off=1596946 len=872867 2024-09-13T00:17:13.030 INFO:tasks.workunit.client.0.smithi136.stdout:op 4797 completed, throughput=5MB/sec 2024-09-13T00:17:13.030 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-mTlriB-q8gfEMET off=347714 len=1767905 2024-09-13T00:17:13.043 INFO:tasks.workunit.client.0.smithi136.stdout:op 4798 completed, throughput=5MB/sec 2024-09-13T00:17:13.043 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-DkXT1gyYZ9VH1OU off=21758095 len=1520566 2024-09-13T00:17:13.047 INFO:tasks.workunit.client.0.smithi136.stdout:op 4799 completed, throughput=5MB/sec 2024-09-13T00:17:13.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:17:12 smithi181 bash[67587]: cluster 2024-09-13T00:17:11.617175+0000 mgr.y (mgr.24661) 1202 : cluster [DBG] pgmap v1885: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.5 MiB/s wr, 5 op/s 2024-09-13T00:17:14.047 INFO:tasks.workunit.client.0.smithi136.stdout: 921: throughput=5MB/sec pending data=0 2024-09-13T00:17:14.047 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-GeRf4vDLj0tyuYe off=3751119 len=1432075 2024-09-13T00:17:14.051 INFO:tasks.workunit.client.0.smithi136.stdout:op 4800 completed, throughput=5MB/sec 2024-09-13T00:17:14.051 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-HAtLUK-pLhVJ6pR off=3226642 len=1308790 2024-09-13T00:17:14.053 INFO:tasks.workunit.client.0.smithi136.stdout:op 4801 completed, throughput=5MB/sec 2024-09-13T00:17:14.053 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-XfzgPt2ByZXX9Cg off=9834372 len=586574 2024-09-13T00:17:14.060 INFO:tasks.workunit.client.0.smithi136.stdout:op 4802 completed, throughput=5MB/sec 2024-09-13T00:17:14.060 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ZiDVLtqMm8Lbcw0 off=13420621 len=665552 2024-09-13T00:17:14.067 INFO:tasks.workunit.client.0.smithi136.stdout:op 4803 completed, throughput=5MB/sec 2024-09-13T00:17:14.067 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-M8lKNxJeS2R3-CN off=7766250 len=1042767 2024-09-13T00:17:14.076 INFO:tasks.workunit.client.0.smithi136.stdout:op 4804 completed, throughput=5MB/sec 2024-09-13T00:17:15.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:17:14 smithi136 bash[76371]: cluster 2024-09-13T00:17:13.618205+0000 mgr.y (mgr.24661) 1203 : cluster [DBG] pgmap v1886: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.9 MiB/s wr, 5 op/s 2024-09-13T00:17:15.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:17:14 smithi136 bash[75164]: cluster 2024-09-13T00:17:13.618205+0000 mgr.y (mgr.24661) 1203 : cluster [DBG] pgmap v1886: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.9 MiB/s wr, 5 op/s 2024-09-13T00:17:15.077 INFO:tasks.workunit.client.0.smithi136.stdout: 922: throughput=5MB/sec pending data=0 2024-09-13T00:17:15.077 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Eb-c24sfwvvXaOC off=4222200 len=181503 2024-09-13T00:17:15.078 INFO:tasks.workunit.client.0.smithi136.stdout:op 4805 completed, throughput=5MB/sec 2024-09-13T00:17:15.078 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ja5Bmt8feMWGk9G off=7430550 len=152552 2024-09-13T00:17:15.083 INFO:tasks.workunit.client.0.smithi136.stdout:op 4806 completed, throughput=5MB/sec 2024-09-13T00:17:15.083 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-JsRVX57M5WPVXAy off=3066007 len=1071171 2024-09-13T00:17:15.091 INFO:tasks.workunit.client.0.smithi136.stdout:op 4807 completed, throughput=5MB/sec 2024-09-13T00:17:15.091 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-o-iBYKHNeodPFZA off=8470207 len=1298813 2024-09-13T00:17:15.094 INFO:tasks.workunit.client.0.smithi136.stdout:op 4808 completed, throughput=5MB/sec 2024-09-13T00:17:15.094 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-YIzKL0xYhblZfmL off=1327090 len=1890064 2024-09-13T00:17:15.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:17:14 smithi181 bash[67587]: cluster 2024-09-13T00:17:13.618205+0000 mgr.y (mgr.24661) 1203 : cluster [DBG] pgmap v1886: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.9 MiB/s wr, 5 op/s 2024-09-13T00:17:15.099 INFO:tasks.workunit.client.0.smithi136.stdout:op 4809 completed, throughput=5MB/sec 2024-09-13T00:17:15.099 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-_5d3ObeHW6vs1sz off=110619 len=1450576 2024-09-13T00:17:15.102 INFO:tasks.workunit.client.0.smithi136.stdout:op 4810 completed, throughput=5MB/sec 2024-09-13T00:17:16.103 INFO:tasks.workunit.client.0.smithi136.stdout: 923: throughput=5MB/sec pending data=0 2024-09-13T00:17:16.103 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-vhqF9k8M5ypbsz3 off=4926923 len=203833 2024-09-13T00:17:16.108 INFO:tasks.workunit.client.0.smithi136.stdout:op 4811 completed, throughput=5MB/sec 2024-09-13T00:17:16.108 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-15qKFJsdsQDrX5t off=12420828 len=1442804 2024-09-13T00:17:16.112 INFO:tasks.workunit.client.0.smithi136.stdout:op 4812 completed, throughput=5MB/sec 2024-09-13T00:17:16.112 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-zD3lTl_QH0mwYhg off=21694518 len=315833 2024-09-13T00:17:16.114 INFO:tasks.workunit.client.0.smithi136.stdout:op 4813 completed, throughput=5MB/sec 2024-09-13T00:17:16.114 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-aGE-uyGfWeFeTto off=10843708 len=498233 2024-09-13T00:17:16.120 INFO:tasks.workunit.client.0.smithi136.stdout:op 4814 completed, throughput=5MB/sec 2024-09-13T00:17:16.121 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-okNTkErqNqyrXfp off=4475352 len=1490285 2024-09-13T00:17:16.131 INFO:tasks.workunit.client.0.smithi136.stdout:op 4815 completed, throughput=5MB/sec 2024-09-13T00:17:16.131 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj--6iflnH8oKwniSZ off=5135950 len=977998 2024-09-13T00:17:16.139 INFO:tasks.workunit.client.0.smithi136.stdout:op 4816 completed, throughput=5MB/sec 2024-09-13T00:17:17.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:17:16 smithi136 bash[75164]: cluster 2024-09-13T00:17:15.619090+0000 mgr.y (mgr.24661) 1204 : cluster [DBG] pgmap v1887: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.2 MiB/s wr, 5 op/s 2024-09-13T00:17:17.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:17:16 smithi136 bash[76371]: cluster 2024-09-13T00:17:15.619090+0000 mgr.y (mgr.24661) 1204 : cluster [DBG] pgmap v1887: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.2 MiB/s wr, 5 op/s 2024-09-13T00:17:17.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:17:16 smithi181 bash[67587]: cluster 2024-09-13T00:17:15.619090+0000 mgr.y (mgr.24661) 1204 : cluster [DBG] pgmap v1887: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.2 MiB/s wr, 5 op/s 2024-09-13T00:17:17.139 INFO:tasks.workunit.client.0.smithi136.stdout: 924: throughput=5MB/sec pending data=0 2024-09-13T00:17:17.140 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-1qOkF6TCT9dLq_c off=983920 len=534626 2024-09-13T00:17:17.146 INFO:tasks.workunit.client.0.smithi136.stdout:op 4817 completed, throughput=5MB/sec 2024-09-13T00:17:17.146 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-y8LlQk3iSLWRgAd off=4066202 len=1562455 2024-09-13T00:17:17.157 INFO:tasks.workunit.client.0.smithi136.stdout:op 4818 completed, throughput=5MB/sec 2024-09-13T00:17:17.157 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-rlEhLh44hMdx291 off=2213102 len=104787 2024-09-13T00:17:17.158 INFO:tasks.workunit.client.0.smithi136.stdout:op 4819 completed, throughput=5MB/sec 2024-09-13T00:17:17.158 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-lrXFWujYEJPUf6N off=3793604 len=1841973 2024-09-13T00:17:17.163 INFO:tasks.workunit.client.0.smithi136.stdout:op 4820 completed, throughput=5MB/sec 2024-09-13T00:17:17.163 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-FjnVfTETpbDvO7t off=2016602 len=1196573 2024-09-13T00:17:17.173 INFO:tasks.workunit.client.0.smithi136.stdout:op 4821 completed, throughput=5MB/sec 2024-09-13T00:17:18.173 INFO:tasks.workunit.client.0.smithi136.stdout: 925: throughput=5MB/sec pending data=0 2024-09-13T00:17:18.174 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Cr1lqupoEPwEjXg off=10901965 len=1982737 2024-09-13T00:17:18.178 INFO:tasks.workunit.client.0.smithi136.stdout:op 4822 completed, throughput=5MB/sec 2024-09-13T00:17:18.178 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-yaXyUyrAo5CPd3X off=1261191 len=281815 2024-09-13T00:17:18.180 INFO:tasks.workunit.client.0.smithi136.stdout:op 4823 completed, throughput=5MB/sec 2024-09-13T00:17:18.180 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-94QfvvcYqt3Mhlf off=1996626 len=1868746 2024-09-13T00:17:18.186 INFO:tasks.workunit.client.0.smithi136.stdout:op 4824 completed, throughput=5MB/sec 2024-09-13T00:17:18.187 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-sSqA9Qk3Og7KnN2 off=2134628 len=1510999 2024-09-13T00:17:18.190 INFO:tasks.workunit.client.0.smithi136.stdout:op 4825 completed, throughput=5MB/sec 2024-09-13T00:17:19.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:17:18 smithi136 bash[75164]: cluster 2024-09-13T00:17:17.620268+0000 mgr.y (mgr.24661) 1205 : cluster [DBG] pgmap v1888: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:17:19.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:17:18 smithi136 bash[76371]: cluster 2024-09-13T00:17:17.620268+0000 mgr.y (mgr.24661) 1205 : cluster [DBG] pgmap v1888: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:17:19.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:17:18 smithi181 bash[67587]: cluster 2024-09-13T00:17:17.620268+0000 mgr.y (mgr.24661) 1205 : cluster [DBG] pgmap v1888: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:17:19.191 INFO:tasks.workunit.client.0.smithi136.stdout: 926: throughput=5MB/sec pending data=0 2024-09-13T00:17:19.191 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-bU0JzXAzi-h1SeH off=7123849 len=2056474 2024-09-13T00:17:19.210 INFO:tasks.workunit.client.0.smithi136.stdout:op 4826 completed, throughput=5MB/sec 2024-09-13T00:17:19.210 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-GpRKt8Mdd1UGJoZ off=1827285 len=251338 2024-09-13T00:17:19.211 INFO:tasks.workunit.client.0.smithi136.stdout:op 4827 completed, throughput=5MB/sec 2024-09-13T00:17:19.211 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-YFOEF43ChCKeZKu off=1213646 len=1302420 2024-09-13T00:17:19.221 INFO:tasks.workunit.client.0.smithi136.stdout:op 4828 completed, throughput=5MB/sec 2024-09-13T00:17:19.222 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-zcCRYEREDzxt1D5 off=18585235 len=545292 2024-09-13T00:17:19.228 INFO:tasks.workunit.client.0.smithi136.stdout:op 4829 completed, throughput=5MB/sec 2024-09-13T00:17:19.228 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj--jdaL8fjVb9WWNq off=6896679 len=1854295 2024-09-13T00:17:19.232 INFO:tasks.workunit.client.0.smithi136.stdout:op 4830 completed, throughput=5MB/sec 2024-09-13T00:17:19.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:17:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:17:19] "GET /metrics HTTP/1.1" 200 34145 "" "Prometheus/2.43.0" 2024-09-13T00:17:20.232 INFO:tasks.workunit.client.0.smithi136.stdout: 927: throughput=5MB/sec pending data=0 2024-09-13T00:17:20.233 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-2e6rCfldMmGgSZD off=1871420 len=237919 2024-09-13T00:17:20.237 INFO:tasks.workunit.client.0.smithi136.stdout:op 4831 completed, throughput=5MB/sec 2024-09-13T00:17:20.238 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-IF9d_LAR20NyzZm off=256019 len=771538 2024-09-13T00:17:20.245 INFO:tasks.workunit.client.0.smithi136.stdout:op 4832 completed, throughput=5MB/sec 2024-09-13T00:17:20.245 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-UNVkEqJKjFZ3X_k off=168718 len=754065 2024-09-13T00:17:20.252 INFO:tasks.workunit.client.0.smithi136.stdout:op 4833 completed, throughput=5MB/sec 2024-09-13T00:17:20.252 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-NL0axwmZq7tO4pZ off=2049242 len=2069408 2024-09-13T00:17:20.266 INFO:tasks.workunit.client.0.smithi136.stdout:op 4834 completed, throughput=5MB/sec 2024-09-13T00:17:20.266 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-jZqM2A5Er7klxEX off=185542 len=1183801 2024-09-13T00:17:20.276 INFO:tasks.workunit.client.0.smithi136.stdout:op 4835 completed, throughput=5MB/sec 2024-09-13T00:17:21.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:17:20 smithi136 bash[75164]: cluster 2024-09-13T00:17:19.621263+0000 mgr.y (mgr.24661) 1206 : cluster [DBG] pgmap v1889: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:17:21.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:17:20 smithi136 bash[76371]: cluster 2024-09-13T00:17:19.621263+0000 mgr.y (mgr.24661) 1206 : cluster [DBG] pgmap v1889: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:17:21.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:17:20 smithi181 bash[67587]: cluster 2024-09-13T00:17:19.621263+0000 mgr.y (mgr.24661) 1206 : cluster [DBG] pgmap v1889: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:17:21.276 INFO:tasks.workunit.client.0.smithi136.stdout: 928: throughput=5MB/sec pending data=0 2024-09-13T00:17:21.276 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-v7el_1kBHr8wdYw off=3744722 len=1824555 2024-09-13T00:17:21.280 INFO:tasks.workunit.client.0.smithi136.stdout:op 4836 completed, throughput=5MB/sec 2024-09-13T00:17:21.280 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-RmM1UZsX3i23TZF off=8272106 len=1053733 2024-09-13T00:17:21.290 INFO:tasks.workunit.client.0.smithi136.stdout:op 4837 completed, throughput=5MB/sec 2024-09-13T00:17:21.290 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Tg4Lz23SdMJjdpC off=2906499 len=337245 2024-09-13T00:17:21.292 INFO:tasks.workunit.client.0.smithi136.stdout:op 4838 completed, throughput=5MB/sec 2024-09-13T00:17:21.292 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-aPnkUxLtXbPZ48W off=9050276 len=1870387 2024-09-13T00:17:21.306 INFO:tasks.workunit.client.0.smithi136.stdout:op 4839 completed, throughput=5MB/sec 2024-09-13T00:17:22.306 INFO:tasks.workunit.client.0.smithi136.stdout: 929: throughput=5MB/sec pending data=0 2024-09-13T00:17:22.306 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj--wHSKeRqh5Pi1TB off=2083650 len=746792 2024-09-13T00:17:22.314 INFO:tasks.workunit.client.0.smithi136.stdout:op 4840 completed, throughput=5MB/sec 2024-09-13T00:17:22.314 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-9wG57vjeh1Cn_Xh off=7248732 len=430550 2024-09-13T00:17:22.316 INFO:tasks.workunit.client.0.smithi136.stdout:op 4841 completed, throughput=5MB/sec 2024-09-13T00:17:22.316 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-CuOh6TczEnUidHi off=175810 len=1076123 2024-09-13T00:17:22.330 INFO:tasks.workunit.client.0.smithi136.stdout:op 4842 completed, throughput=5MB/sec 2024-09-13T00:17:22.330 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-S4miOTEombnIwzB off=2295554 len=207553 2024-09-13T00:17:22.335 INFO:tasks.workunit.client.0.smithi136.stdout:op 4843 completed, throughput=5MB/sec 2024-09-13T00:17:22.335 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Ryf-_mr3aHbtAKs off=11557317 len=924093 2024-09-13T00:17:22.342 INFO:tasks.workunit.client.0.smithi136.stdout:op 4844 completed, throughput=5MB/sec 2024-09-13T00:17:22.342 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-E2dd5rUOKJFEgmd off=12985844 len=1340139 2024-09-13T00:17:22.346 INFO:tasks.workunit.client.0.smithi136.stdout:op 4845 completed, throughput=5MB/sec 2024-09-13T00:17:22.346 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Dyu73DI5uebT0B6 off=935739 len=843355 2024-09-13T00:17:22.349 INFO:tasks.workunit.client.0.smithi136.stdout:op 4846 completed, throughput=5MB/sec 2024-09-13T00:17:22.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:17:22 smithi181 bash[95505]: ts=2024-09-13T00:17:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:17:23.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:17:22 smithi136 bash[75164]: cluster 2024-09-13T00:17:21.622371+0000 mgr.y (mgr.24661) 1207 : cluster [DBG] pgmap v1890: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.9 MiB/s wr, 4 op/s 2024-09-13T00:17:23.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:17:22 smithi136 bash[76371]: cluster 2024-09-13T00:17:21.622371+0000 mgr.y (mgr.24661) 1207 : cluster [DBG] pgmap v1890: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.9 MiB/s wr, 4 op/s 2024-09-13T00:17:23.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:17:22 smithi181 bash[67587]: cluster 2024-09-13T00:17:21.622371+0000 mgr.y (mgr.24661) 1207 : cluster [DBG] pgmap v1890: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.9 MiB/s wr, 4 op/s 2024-09-13T00:17:23.349 INFO:tasks.workunit.client.0.smithi136.stdout: 930: throughput=5MB/sec pending data=0 2024-09-13T00:17:23.350 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-j3NywYFwRXJKGX_ off=7991635 len=1730000 2024-09-13T00:17:23.353 INFO:tasks.workunit.client.0.smithi136.stdout:op 4847 completed, throughput=5MB/sec 2024-09-13T00:17:23.353 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-EcRa5NmaHPTG9IX off=5802994 len=487115 2024-09-13T00:17:23.355 INFO:tasks.workunit.client.0.smithi136.stdout:op 4848 completed, throughput=5MB/sec 2024-09-13T00:17:23.355 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-wVK01-A2j89IC0G off=0 len=1224394 2024-09-13T00:17:23.358 INFO:tasks.workunit.client.0.smithi136.stdout:op 4849 completed, throughput=5MB/sec 2024-09-13T00:17:23.358 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-khkgkXKsMhkbKKR off=1693381 len=62958 2024-09-13T00:17:23.359 INFO:tasks.workunit.client.0.smithi136.stdout:op 4850 completed, throughput=5MB/sec 2024-09-13T00:17:23.359 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-dM74AMk05HYpJJv off=7636412 len=1394605 2024-09-13T00:17:23.373 INFO:tasks.workunit.client.0.smithi136.stdout:op 4851 completed, throughput=5MB/sec 2024-09-13T00:17:24.374 INFO:tasks.workunit.client.0.smithi136.stdout: 931: throughput=4.99MB/sec pending data=0 2024-09-13T00:17:24.374 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-MPEbqUE9PtPmtqg off=19157633 len=1318425 2024-09-13T00:17:24.377 INFO:tasks.workunit.client.0.smithi136.stdout:op 4852 completed, throughput=5MB/sec 2024-09-13T00:17:24.377 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-adizo1aTh-WUHwP off=0 len=1073745 2024-09-13T00:17:24.380 INFO:tasks.workunit.client.0.smithi136.stdout:op 4853 completed, throughput=5MB/sec 2024-09-13T00:17:24.380 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-mmLCGGkmNPK92sF off=7305461 len=863108 2024-09-13T00:17:24.383 INFO:tasks.workunit.client.0.smithi136.stdout:op 4854 completed, throughput=5MB/sec 2024-09-13T00:17:24.383 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-AEFlxa03ko3Ua4C off=5919361 len=1967763 2024-09-13T00:17:24.388 INFO:tasks.workunit.client.0.smithi136.stdout:op 4855 completed, throughput=5MB/sec 2024-09-13T00:17:25.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:17:24 smithi136 bash[75164]: cluster 2024-09-13T00:17:23.623487+0000 mgr.y (mgr.24661) 1208 : cluster [DBG] pgmap v1891: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 3.5 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:17:25.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:17:24 smithi136 bash[75164]: audit 2024-09-13T00:17:23.798476+0000 mon.a (mon.0) 2249 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:17:25.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:17:24 smithi136 bash[76371]: cluster 2024-09-13T00:17:23.623487+0000 mgr.y (mgr.24661) 1208 : cluster [DBG] pgmap v1891: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 3.5 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:17:25.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:17:24 smithi136 bash[76371]: audit 2024-09-13T00:17:23.798476+0000 mon.a (mon.0) 2249 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:17:25.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:17:24 smithi181 bash[67587]: cluster 2024-09-13T00:17:23.623487+0000 mgr.y (mgr.24661) 1208 : cluster [DBG] pgmap v1891: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 3.5 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:17:25.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:17:24 smithi181 bash[67587]: audit 2024-09-13T00:17:23.798476+0000 mon.a (mon.0) 2249 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:17:25.388 INFO:tasks.workunit.client.0.smithi136.stdout: 932: throughput=4.99MB/sec pending data=0 2024-09-13T00:17:25.388 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-TxbtbjEXtNJJLl3 off=4920830 len=1318788 2024-09-13T00:17:25.399 INFO:tasks.workunit.client.0.smithi136.stdout:op 4856 completed, throughput=5MB/sec 2024-09-13T00:17:25.400 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ud8Z5Y79Gb9sk_t off=7124534 len=1198095 2024-09-13T00:17:25.412 INFO:tasks.workunit.client.0.smithi136.stdout:op 4857 completed, throughput=5MB/sec 2024-09-13T00:17:25.412 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-oMjPvomW0LPPCst off=1292531 len=444016 2024-09-13T00:17:25.416 INFO:tasks.workunit.client.0.smithi136.stdout:op 4858 completed, throughput=5MB/sec 2024-09-13T00:17:25.416 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-CF0AUH05xQLlyvo off=438666 len=1707812 2024-09-13T00:17:25.430 INFO:tasks.workunit.client.0.smithi136.stdout:op 4859 completed, throughput=5MB/sec 2024-09-13T00:17:25.430 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-9EL0lB8mduMFMP6 off=6778478 len=471928 2024-09-13T00:17:25.436 INFO:tasks.workunit.client.0.smithi136.stdout:op 4860 completed, throughput=5MB/sec 2024-09-13T00:17:25.436 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ZoObUTXkyyUuz8- off=17271193 len=1646701 2024-09-13T00:17:25.449 INFO:tasks.workunit.client.0.smithi136.stdout:op 4861 completed, throughput=5MB/sec 2024-09-13T00:17:26.449 INFO:tasks.workunit.client.0.smithi136.stdout: 933: throughput=5MB/sec pending data=0 2024-09-13T00:17:26.449 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-VuScuHax2XtkjzQ off=2911664 len=374880 2024-09-13T00:17:26.456 INFO:tasks.workunit.client.0.smithi136.stdout:op 4862 completed, throughput=5MB/sec 2024-09-13T00:17:26.456 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ZD0gAflpEgmRozK off=219578 len=648002 2024-09-13T00:17:26.462 INFO:tasks.workunit.client.0.smithi136.stdout:op 4863 completed, throughput=5MB/sec 2024-09-13T00:17:26.462 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-sqZ64XBVPZ9lLmY off=18981695 len=1967025 2024-09-13T00:17:26.469 INFO:tasks.workunit.client.0.smithi136.stdout:op 4864 completed, throughput=5MB/sec 2024-09-13T00:17:26.469 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-R0WoQFcPY1tiCEN off=11042080 len=1063116 2024-09-13T00:17:26.477 INFO:tasks.workunit.client.0.smithi136.stdout:op 4865 completed, throughput=5MB/sec 2024-09-13T00:17:26.477 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-gb2F0A6Hrq4C4xA off=5436684 len=79623 2024-09-13T00:17:26.481 INFO:tasks.workunit.client.0.smithi136.stdout:op 4866 completed, throughput=5MB/sec 2024-09-13T00:17:27.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:17:26 smithi136 bash[75164]: cluster 2024-09-13T00:17:25.624440+0000 mgr.y (mgr.24661) 1209 : cluster [DBG] pgmap v1892: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:17:27.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:17:26 smithi136 bash[76371]: cluster 2024-09-13T00:17:25.624440+0000 mgr.y (mgr.24661) 1209 : cluster [DBG] pgmap v1892: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:17:27.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:17:26 smithi181 bash[67587]: cluster 2024-09-13T00:17:25.624440+0000 mgr.y (mgr.24661) 1209 : cluster [DBG] pgmap v1892: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:17:27.482 INFO:tasks.workunit.client.0.smithi136.stdout: 934: throughput=4.99MB/sec pending data=0 2024-09-13T00:17:27.482 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-FoE5DadFYLt-NDf off=6700363 len=675700 2024-09-13T00:17:27.484 INFO:tasks.workunit.client.0.smithi136.stdout:op 4867 completed, throughput=5MB/sec 2024-09-13T00:17:27.484 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-qn9LNplQCguKRfN off=2533420 len=1423887 2024-09-13T00:17:27.494 INFO:tasks.workunit.client.0.smithi136.stdout:op 4868 completed, throughput=5MB/sec 2024-09-13T00:17:27.495 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-D_Hp3ytuam8Wjqm off=281827 len=1846511 2024-09-13T00:17:27.499 INFO:tasks.workunit.client.0.smithi136.stdout:op 4869 completed, throughput=5MB/sec 2024-09-13T00:17:27.499 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ombYel1ZU3gvPvr off=11661524 len=668399 2024-09-13T00:17:27.506 INFO:tasks.workunit.client.0.smithi136.stdout:op 4870 completed, throughput=5MB/sec 2024-09-13T00:17:27.506 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ALBa-lc7jhdakqz off=32127 len=432757 2024-09-13T00:17:27.512 INFO:tasks.workunit.client.0.smithi136.stdout:op 4871 completed, throughput=5MB/sec 2024-09-13T00:17:27.512 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-tQgHOTOHxblMCLB off=637190 len=2018321 2024-09-13T00:17:27.527 INFO:tasks.workunit.client.0.smithi136.stdout:op 4872 completed, throughput=5MB/sec 2024-09-13T00:17:28.527 INFO:tasks.workunit.client.0.smithi136.stdout: 935: throughput=5MB/sec pending data=0 2024-09-13T00:17:28.527 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-oqQHZz6tdJZ08pM off=8283416 len=1334084 2024-09-13T00:17:28.540 INFO:tasks.workunit.client.0.smithi136.stdout:op 4873 completed, throughput=5MB/sec 2024-09-13T00:17:28.540 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-HJZ3OPPrSJZiz8p off=2083533 len=1251200 2024-09-13T00:17:28.549 INFO:tasks.workunit.client.0.smithi136.stdout:op 4874 completed, throughput=5MB/sec 2024-09-13T00:17:28.549 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-IhTlzlGy0BJYodC off=3359910 len=905565 2024-09-13T00:17:28.551 INFO:tasks.workunit.client.0.smithi136.stdout:op 4875 completed, throughput=5MB/sec 2024-09-13T00:17:28.551 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-YHpH6W_tjIIzXTF off=6125359 len=342201 2024-09-13T00:17:28.553 INFO:tasks.workunit.client.0.smithi136.stdout:op 4876 completed, throughput=5MB/sec 2024-09-13T00:17:29.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:17:28 smithi136 bash[75164]: cluster 2024-09-13T00:17:27.625712+0000 mgr.y (mgr.24661) 1210 : cluster [DBG] pgmap v1893: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:17:29.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:17:28 smithi136 bash[76371]: cluster 2024-09-13T00:17:27.625712+0000 mgr.y (mgr.24661) 1210 : cluster [DBG] pgmap v1893: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:17:29.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:17:28 smithi181 bash[67587]: cluster 2024-09-13T00:17:27.625712+0000 mgr.y (mgr.24661) 1210 : cluster [DBG] pgmap v1893: 105 pgs: 105 active+clean; 126 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:17:29.554 INFO:tasks.workunit.client.0.smithi136.stdout: 936: throughput=4.99MB/sec pending data=0 2024-09-13T00:17:29.554 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-oBeNnossotLFL23 off=14332307 len=873303 2024-09-13T00:17:29.563 INFO:tasks.workunit.client.0.smithi136.stdout:op 4877 completed, throughput=5MB/sec 2024-09-13T00:17:29.563 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-hEo_1IAfZFAQcPY off=3784564 len=53955 2024-09-13T00:17:29.564 INFO:tasks.workunit.client.0.smithi136.stdout:op 4878 completed, throughput=5MB/sec 2024-09-13T00:17:29.564 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-SJFV68lZxpOZ20X off=2362748 len=1196039 2024-09-13T00:17:29.568 INFO:tasks.workunit.client.0.smithi136.stdout:op 4879 completed, throughput=5MB/sec 2024-09-13T00:17:29.568 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-k_zN_ASmwaBmcqY off=21638665 len=1306944 2024-09-13T00:17:29.579 INFO:tasks.workunit.client.0.smithi136.stdout:op 4880 completed, throughput=5MB/sec 2024-09-13T00:17:29.579 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-8WJbOaEiP9OWyvh off=3859747 len=510268 2024-09-13T00:17:29.586 INFO:tasks.workunit.client.0.smithi136.stdout:op 4881 completed, throughput=5MB/sec 2024-09-13T00:17:29.586 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-K2kaF89EZOTqatF off=5096764 len=71065 2024-09-13T00:17:29.591 INFO:tasks.workunit.client.0.smithi136.stdout:op 4882 completed, throughput=5MB/sec 2024-09-13T00:17:29.591 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-5v2CigT4BZGliyN off=7653490 len=368680 2024-09-13T00:17:29.596 INFO:tasks.workunit.client.0.smithi136.stdout:op 4883 completed, throughput=5MB/sec 2024-09-13T00:17:29.596 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Mfl0u6ETqIwYjZU off=417645 len=912409 2024-09-13T00:17:29.605 INFO:tasks.workunit.client.0.smithi136.stdout:op 4884 completed, throughput=5MB/sec 2024-09-13T00:17:29.605 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ZrtiP4oKPY-JyXu off=3450350 len=1769539 2024-09-13T00:17:29.618 INFO:tasks.workunit.client.0.smithi136.stdout:op 4885 completed, throughput=5MB/sec 2024-09-13T00:17:29.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:17:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:17:29] "GET /metrics HTTP/1.1" 200 34147 "" "Prometheus/2.43.0" 2024-09-13T00:17:30.619 INFO:tasks.workunit.client.0.smithi136.stdout: 937: throughput=5MB/sec pending data=0 2024-09-13T00:17:30.619 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-qAciF1JiKGrfWzU off=9972386 len=2033966 2024-09-13T00:17:30.624 INFO:tasks.workunit.client.0.smithi136.stdout:op 4886 completed, throughput=5MB/sec 2024-09-13T00:17:30.624 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Gvot-_IzWvfThyK off=20889660 len=611051 2024-09-13T00:17:30.631 INFO:tasks.workunit.client.0.smithi136.stdout:op 4887 completed, throughput=5MB/sec 2024-09-13T00:17:30.631 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-hYZ6gHbc1BlRN0L off=6402635 len=2036318 2024-09-13T00:17:30.650 INFO:tasks.workunit.client.0.smithi136.stdout:op 4888 completed, throughput=5MB/sec 2024-09-13T00:17:31.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:17:30 smithi136 bash[75164]: cluster 2024-09-13T00:17:29.626575+0000 mgr.y (mgr.24661) 1211 : cluster [DBG] pgmap v1894: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:17:31.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:17:30 smithi136 bash[76371]: cluster 2024-09-13T00:17:29.626575+0000 mgr.y (mgr.24661) 1211 : cluster [DBG] pgmap v1894: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:17:31.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:17:30 smithi181 bash[67587]: cluster 2024-09-13T00:17:29.626575+0000 mgr.y (mgr.24661) 1211 : cluster [DBG] pgmap v1894: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:17:31.652 INFO:tasks.workunit.client.0.smithi136.stdout: 938: throughput=5MB/sec pending data=0 2024-09-13T00:17:31.653 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-24z3jjHGJ3cpoLN off=9739898 len=577339 2024-09-13T00:17:31.657 INFO:tasks.workunit.client.0.smithi136.stdout:op 4889 completed, throughput=5MB/sec 2024-09-13T00:17:31.657 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Gwcs0aAYFyY1C-P off=4514776 len=379225 2024-09-13T00:17:31.659 INFO:tasks.workunit.client.0.smithi136.stdout:op 4890 completed, throughput=5MB/sec 2024-09-13T00:17:31.659 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-4P7JyGAkYaEMxHU off=525533 len=1855237 2024-09-13T00:17:31.663 INFO:tasks.workunit.client.0.smithi136.stdout:op 4891 completed, throughput=5MB/sec 2024-09-13T00:17:31.664 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-d0BhuMD_oRHMi_B off=10576231 len=1025266 2024-09-13T00:17:31.673 INFO:tasks.workunit.client.0.smithi136.stdout:op 4892 completed, throughput=5MB/sec 2024-09-13T00:17:31.673 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-6w31CucywvYw_rT off=15732297 len=1391767 2024-09-13T00:17:31.676 INFO:tasks.workunit.client.0.smithi136.stdout:op 4893 completed, throughput=5MB/sec 2024-09-13T00:17:32.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:17:32 smithi181 bash[95505]: ts=2024-09-13T00:17:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:17:32.676 INFO:tasks.workunit.client.0.smithi136.stdout: 939: throughput=5MB/sec pending data=0 2024-09-13T00:17:32.677 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-nBfzwPoRZMUf5_b off=11509634 len=1738752 2024-09-13T00:17:32.680 INFO:tasks.workunit.client.0.smithi136.stdout:op 4894 completed, throughput=5MB/sec 2024-09-13T00:17:32.681 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-yEfAL3DSccGy0dS off=8756831 len=1830470 2024-09-13T00:17:32.693 INFO:tasks.workunit.client.0.smithi136.stdout:op 4895 completed, throughput=5MB/sec 2024-09-13T00:17:32.693 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-_wa6A7gpvBVY2YS off=22213301 len=1601417 2024-09-13T00:17:32.697 INFO:tasks.workunit.client.0.smithi136.stdout:op 4896 completed, throughput=5MB/sec 2024-09-13T00:17:33.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:17:32 smithi136 bash[75164]: cluster 2024-09-13T00:17:31.627546+0000 mgr.y (mgr.24661) 1212 : cluster [DBG] pgmap v1895: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:17:33.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:17:32 smithi136 bash[76371]: cluster 2024-09-13T00:17:31.627546+0000 mgr.y (mgr.24661) 1212 : cluster [DBG] pgmap v1895: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:17:33.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:17:32 smithi181 bash[67587]: cluster 2024-09-13T00:17:31.627546+0000 mgr.y (mgr.24661) 1212 : cluster [DBG] pgmap v1895: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:17:33.698 INFO:tasks.workunit.client.0.smithi136.stdout: 940: throughput=5MB/sec pending data=0 2024-09-13T00:17:33.698 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Oa2l1kBHBRcpHJc off=1048945 len=199676 2024-09-13T00:17:33.700 INFO:tasks.workunit.client.0.smithi136.stdout:op 4897 completed, throughput=5MB/sec 2024-09-13T00:17:33.700 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-rBGCoxQ0qVW2L9h off=5305260 len=872812 2024-09-13T00:17:33.707 INFO:tasks.workunit.client.0.smithi136.stdout:op 4898 completed, throughput=5MB/sec 2024-09-13T00:17:33.707 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-o3dFrB_4YtTdDk7 off=266039 len=1635255 2024-09-13T00:17:33.711 INFO:tasks.workunit.client.0.smithi136.stdout:op 4899 completed, throughput=5MB/sec 2024-09-13T00:17:33.712 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-90e6gq24HBQTdFa off=2500324 len=628049 2024-09-13T00:17:33.713 INFO:tasks.workunit.client.0.smithi136.stdout:op 4900 completed, throughput=5MB/sec 2024-09-13T00:17:33.713 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-56QEox930qjz8Kr off=15525518 len=1705353 2024-09-13T00:17:33.728 INFO:tasks.workunit.client.0.smithi136.stdout:op 4901 completed, throughput=5MB/sec 2024-09-13T00:17:34.729 INFO:tasks.workunit.client.0.smithi136.stdout: 941: throughput=4.99MB/sec pending data=0 2024-09-13T00:17:34.729 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-I-AE1CZdL-nlrmx off=775186 len=1679310 2024-09-13T00:17:34.733 INFO:tasks.workunit.client.0.smithi136.stdout:op 4902 completed, throughput=5MB/sec 2024-09-13T00:17:34.733 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-jb5v9X8knBuScek off=1326599 len=478189 2024-09-13T00:17:34.739 INFO:tasks.workunit.client.0.smithi136.stdout:op 4903 completed, throughput=5MB/sec 2024-09-13T00:17:34.739 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-vyQpdqYcf9ECg-1 off=35520 len=207407 2024-09-13T00:17:34.741 INFO:tasks.workunit.client.0.smithi136.stdout:op 4904 completed, throughput=5MB/sec 2024-09-13T00:17:34.741 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-pEMNOiRHmFxXomN off=2371340 len=644270 2024-09-13T00:17:34.744 INFO:tasks.workunit.client.0.smithi136.stdout:op 4905 completed, throughput=5MB/sec 2024-09-13T00:17:34.744 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-O4ie9C22DUpkiGd off=0 len=2033992 2024-09-13T00:17:34.763 INFO:tasks.workunit.client.0.smithi136.stdout:op 4906 completed, throughput=5MB/sec 2024-09-13T00:17:34.763 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-gUezmJikjy7NVs8 off=4375028 len=974840 2024-09-13T00:17:34.771 INFO:tasks.workunit.client.0.smithi136.stdout:op 4907 completed, throughput=5MB/sec 2024-09-13T00:17:35.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:17:34 smithi136 bash[75164]: cluster 2024-09-13T00:17:33.628566+0000 mgr.y (mgr.24661) 1213 : cluster [DBG] pgmap v1896: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:17:35.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:17:34 smithi136 bash[76371]: cluster 2024-09-13T00:17:33.628566+0000 mgr.y (mgr.24661) 1213 : cluster [DBG] pgmap v1896: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:17:35.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:17:34 smithi181 bash[67587]: cluster 2024-09-13T00:17:33.628566+0000 mgr.y (mgr.24661) 1213 : cluster [DBG] pgmap v1896: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:17:35.772 INFO:tasks.workunit.client.0.smithi136.stdout: 942: throughput=5MB/sec pending data=0 2024-09-13T00:17:35.772 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-pYpHNkarNiURwT0 off=4749314 len=1674760 2024-09-13T00:17:35.783 INFO:tasks.workunit.client.0.smithi136.stdout:op 4908 completed, throughput=5MB/sec 2024-09-13T00:17:35.783 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-vNo66wpsRBvlErj off=4000788 len=87016 2024-09-13T00:17:35.784 INFO:tasks.workunit.client.0.smithi136.stdout:op 4909 completed, throughput=5MB/sec 2024-09-13T00:17:35.784 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-OgBC5cbQ4nMHMBn off=476482 len=54830 2024-09-13T00:17:35.788 INFO:tasks.workunit.client.0.smithi136.stdout:op 4910 completed, throughput=5MB/sec 2024-09-13T00:17:35.789 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-jKbfh7c4efX6URp off=4247790 len=1402617 2024-09-13T00:17:35.793 INFO:tasks.workunit.client.0.smithi136.stdout:op 4911 completed, throughput=5MB/sec 2024-09-13T00:17:35.793 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-pWi2lcMFEnQcdr9 off=349231 len=1619902 2024-09-13T00:17:35.797 INFO:tasks.workunit.client.0.smithi136.stdout:op 4912 completed, throughput=5MB/sec 2024-09-13T00:17:36.797 INFO:tasks.workunit.client.0.smithi136.stdout: 943: throughput=5MB/sec pending data=0 2024-09-13T00:17:36.798 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Ut00C2ephc7KxJj off=1339301 len=603961 2024-09-13T00:17:36.804 INFO:tasks.workunit.client.0.smithi136.stdout:op 4913 completed, throughput=5MB/sec 2024-09-13T00:17:36.805 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-3PODk5iqsziZ1Ty off=6019619 len=716574 2024-09-13T00:17:36.807 INFO:tasks.workunit.client.0.smithi136.stdout:op 4914 completed, throughput=5MB/sec 2024-09-13T00:17:36.807 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-TnEGrVoYH5G0me0 off=339074 len=1163664 2024-09-13T00:17:36.810 INFO:tasks.workunit.client.0.smithi136.stdout:op 4915 completed, throughput=5MB/sec 2024-09-13T00:17:36.810 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-lEyMb109TNrXae8 off=7644081 len=1577891 2024-09-13T00:17:36.821 INFO:tasks.workunit.client.0.smithi136.stdout:op 4916 completed, throughput=5MB/sec 2024-09-13T00:17:36.822 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-0vaozphrU-zHwsa off=21160600 len=2082058 2024-09-13T00:17:36.828 INFO:tasks.workunit.client.0.smithi136.stdout:op 4917 completed, throughput=5MB/sec 2024-09-13T00:17:37.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:17:36 smithi136 bash[75164]: cluster 2024-09-13T00:17:35.629457+0000 mgr.y (mgr.24661) 1214 : cluster [DBG] pgmap v1897: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 1.8 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:17:37.008 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:17:36 smithi136 bash[76371]: cluster 2024-09-13T00:17:35.629457+0000 mgr.y (mgr.24661) 1214 : cluster [DBG] pgmap v1897: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 1.8 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:17:37.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:17:36 smithi181 bash[67587]: cluster 2024-09-13T00:17:35.629457+0000 mgr.y (mgr.24661) 1214 : cluster [DBG] pgmap v1897: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 1.8 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:17:37.828 INFO:tasks.workunit.client.0.smithi136.stdout: 944: throughput=5MB/sec pending data=0 2024-09-13T00:17:37.828 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-0xXOCHpgTQQCNQA off=2863478 len=1802867 2024-09-13T00:17:37.844 INFO:tasks.workunit.client.0.smithi136.stdout:op 4918 completed, throughput=5MB/sec 2024-09-13T00:17:37.845 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-oBeNnossotLFL23 off=14036667 len=1107635 2024-09-13T00:17:37.854 INFO:tasks.workunit.client.0.smithi136.stdout:op 4919 completed, throughput=5MB/sec 2024-09-13T00:17:37.855 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-7Dp_fNcAJoNe8Zo off=3537574 len=158021 2024-09-13T00:17:37.860 INFO:tasks.workunit.client.0.smithi136.stdout:op 4920 completed, throughput=5MB/sec 2024-09-13T00:17:37.860 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-eea0xvAphBe7ZsU off=1098070 len=305008 2024-09-13T00:17:37.862 INFO:tasks.workunit.client.0.smithi136.stdout:op 4921 completed, throughput=5MB/sec 2024-09-13T00:17:37.862 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-CxFY_ob8PZgPjtA off=1397248 len=797498 2024-09-13T00:17:37.864 INFO:tasks.workunit.client.0.smithi136.stdout:op 4922 completed, throughput=5MB/sec 2024-09-13T00:17:37.864 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-rJlOsMskWtfiVTb off=3020314 len=991574 2024-09-13T00:17:37.873 INFO:tasks.workunit.client.0.smithi136.stdout:op 4923 completed, throughput=5MB/sec 2024-09-13T00:17:38.873 INFO:tasks.workunit.client.0.smithi136.stdout: 945: throughput=5MB/sec pending data=0 2024-09-13T00:17:38.873 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-5oxxu3YslHOPiXU off=5550423 len=95901 2024-09-13T00:17:38.874 INFO:tasks.workunit.client.0.smithi136.stdout:op 4924 completed, throughput=5MB/sec 2024-09-13T00:17:38.874 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-hTwbaJO_DV6ttHh off=2842620 len=1296869 2024-09-13T00:17:38.884 INFO:tasks.workunit.client.0.smithi136.stdout:op 4925 completed, throughput=5MB/sec 2024-09-13T00:17:38.884 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-a2BOEECtcZ88gPA off=10642111 len=360847 2024-09-13T00:17:38.890 INFO:tasks.workunit.client.0.smithi136.stdout:op 4926 completed, throughput=5MB/sec 2024-09-13T00:17:38.890 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-CzD30NY7MmO2GRx off=4134293 len=1004144 2024-09-13T00:17:38.892 INFO:tasks.workunit.client.0.smithi136.stdout:op 4927 completed, throughput=5MB/sec 2024-09-13T00:17:38.892 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-HMKd7Esgq9eNWU3 off=21444028 len=48171 2024-09-13T00:17:38.894 INFO:tasks.workunit.client.0.smithi136.stdout:op 4928 completed, throughput=5MB/sec 2024-09-13T00:17:38.894 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-06qbGQ1MHYeiEcd off=8006510 len=767582 2024-09-13T00:17:38.902 INFO:tasks.workunit.client.0.smithi136.stdout:op 4929 completed, throughput=5MB/sec 2024-09-13T00:17:38.902 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-uIQUlBeX-GyP8Px off=17957458 len=458923 2024-09-13T00:17:38.908 INFO:tasks.workunit.client.0.smithi136.stdout:op 4930 completed, throughput=5MB/sec 2024-09-13T00:17:38.908 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-3A9DmIt2gwg40Zl off=10438914 len=1646073 2024-09-13T00:17:38.922 INFO:tasks.workunit.client.0.smithi136.stdout:op 4931 completed, throughput=5MB/sec 2024-09-13T00:17:39.008 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:17:38 smithi136 bash[75164]: cluster 2024-09-13T00:17:37.630592+0000 mgr.y (mgr.24661) 1215 : cluster [DBG] pgmap v1898: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:17:39.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:17:38 smithi136 bash[76371]: cluster 2024-09-13T00:17:37.630592+0000 mgr.y (mgr.24661) 1215 : cluster [DBG] pgmap v1898: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:17:39.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:17:38 smithi181 bash[67587]: cluster 2024-09-13T00:17:37.630592+0000 mgr.y (mgr.24661) 1215 : cluster [DBG] pgmap v1898: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:17:39.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:17:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:17:39] "GET /metrics HTTP/1.1" 200 34143 "" "Prometheus/2.43.0" 2024-09-13T00:17:39.923 INFO:tasks.workunit.client.0.smithi136.stdout: 946: throughput=5MB/sec pending data=0 2024-09-13T00:17:39.923 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Tzo41MF6buF3-0o off=898828 len=1538689 2024-09-13T00:17:39.928 INFO:tasks.workunit.client.0.smithi136.stdout:op 4932 completed, throughput=5MB/sec 2024-09-13T00:17:39.928 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-g0F79IT_032PBpV off=9375096 len=225954 2024-09-13T00:17:39.934 INFO:tasks.workunit.client.0.smithi136.stdout:op 4933 completed, throughput=5MB/sec 2024-09-13T00:17:39.934 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-PyKqntTvDNI8E1u off=0 len=1513129 2024-09-13T00:17:39.944 INFO:tasks.workunit.client.0.smithi136.stdout:op 4934 completed, throughput=5MB/sec 2024-09-13T00:17:39.945 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-WDSKkE1x7B9-Abi off=3409258 len=1531971 2024-09-13T00:17:39.955 INFO:tasks.workunit.client.0.smithi136.stdout:op 4935 completed, throughput=5MB/sec 2024-09-13T00:17:40.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:17:39 smithi181 bash[67587]: audit 2024-09-13T00:17:38.798928+0000 mon.a (mon.0) 2250 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:17:40.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:17:39 smithi136 bash[75164]: audit 2024-09-13T00:17:38.798928+0000 mon.a (mon.0) 2250 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:17:40.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:17:39 smithi136 bash[76371]: audit 2024-09-13T00:17:38.798928+0000 mon.a (mon.0) 2250 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:17:40.955 INFO:tasks.workunit.client.0.smithi136.stdout: 947: throughput=5MB/sec pending data=0 2024-09-13T00:17:40.956 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-X6X_Oq0RwilG5Ox off=2990906 len=792279 2024-09-13T00:17:40.958 INFO:tasks.workunit.client.0.smithi136.stdout:op 4936 completed, throughput=5MB/sec 2024-09-13T00:17:40.958 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-jj2cbFs4rX3DNNC off=1511988 len=412784 2024-09-13T00:17:40.960 INFO:tasks.workunit.client.0.smithi136.stdout:op 4937 completed, throughput=5MB/sec 2024-09-13T00:17:40.960 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-4I-zNQLfYtTsYn_ off=9480341 len=634967 2024-09-13T00:17:40.963 INFO:tasks.workunit.client.0.smithi136.stdout:op 4938 completed, throughput=5MB/sec 2024-09-13T00:17:40.963 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-W8NQ4_AOlO1rpiV off=3181482 len=299690 2024-09-13T00:17:40.964 INFO:tasks.workunit.client.0.smithi136.stdout:op 4939 completed, throughput=5MB/sec 2024-09-13T00:17:40.964 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-UfF4yFzeO9-JkyO off=553673 len=1761028 2024-09-13T00:17:40.969 INFO:tasks.workunit.client.0.smithi136.stdout:op 4940 completed, throughput=5MB/sec 2024-09-13T00:17:40.969 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-htOyAuBaRq87BpM off=11534361 len=646586 2024-09-13T00:17:40.976 INFO:tasks.workunit.client.0.smithi136.stdout:op 4941 completed, throughput=5MB/sec 2024-09-13T00:17:40.976 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-GbJmJnrLRk_Xb5s off=11768504 len=991714 2024-09-13T00:17:40.979 INFO:tasks.workunit.client.0.smithi136.stdout:op 4942 completed, throughput=5MB/sec 2024-09-13T00:17:41.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:17:40 smithi181 bash[67587]: cluster 2024-09-13T00:17:39.631418+0000 mgr.y (mgr.24661) 1216 : cluster [DBG] pgmap v1899: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:17:41.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:17:40 smithi136 bash[75164]: cluster 2024-09-13T00:17:39.631418+0000 mgr.y (mgr.24661) 1216 : cluster [DBG] pgmap v1899: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:17:41.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:17:40 smithi136 bash[76371]: cluster 2024-09-13T00:17:39.631418+0000 mgr.y (mgr.24661) 1216 : cluster [DBG] pgmap v1899: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:17:41.980 INFO:tasks.workunit.client.0.smithi136.stdout: 948: throughput=5MB/sec pending data=0 2024-09-13T00:17:41.980 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-srW7hbdRLQ4HqCe off=7639950 len=616649 2024-09-13T00:17:41.987 INFO:tasks.workunit.client.0.smithi136.stdout:op 4943 completed, throughput=5MB/sec 2024-09-13T00:17:41.987 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-dMi87yvuSm8wqWj off=8612933 len=1989666 2024-09-13T00:17:42.004 INFO:tasks.workunit.client.0.smithi136.stdout:op 4944 completed, throughput=5MB/sec 2024-09-13T00:17:42.004 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-FzkR6nITozEvwOr off=7956913 len=1311767 2024-09-13T00:17:42.015 INFO:tasks.workunit.client.0.smithi136.stdout:op 4945 completed, throughput=5MB/sec 2024-09-13T00:17:42.015 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-f2Pmw-0K_1LPEq1 off=10284884 len=376919 2024-09-13T00:17:42.020 INFO:tasks.workunit.client.0.smithi136.stdout:op 4946 completed, throughput=5MB/sec 2024-09-13T00:17:42.020 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-jjxA-1uqopLFw7w off=9628778 len=542683 2024-09-13T00:17:42.021 INFO:tasks.workunit.client.0.smithi136.stdout:op 4947 completed, throughput=5MB/sec 2024-09-13T00:17:42.021 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-DQUpl3WFX8V8RjH off=11726147 len=42258 2024-09-13T00:17:42.025 INFO:tasks.workunit.client.0.smithi136.stdout:op 4948 completed, throughput=5MB/sec 2024-09-13T00:17:42.025 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-SNpoymIibbx4Sq2 off=6453568 len=892529 2024-09-13T00:17:42.027 INFO:tasks.workunit.client.0.smithi136.stdout:op 4949 completed, throughput=5MB/sec 2024-09-13T00:17:42.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:17:42 smithi181 bash[95505]: ts=2024-09-13T00:17:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:17:43.027 INFO:tasks.workunit.client.0.smithi136.stdout: 949: throughput=5MB/sec pending data=0 2024-09-13T00:17:43.028 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-JBQtw-oI3t2bdmU off=16184888 len=937608 2024-09-13T00:17:43.031 INFO:tasks.workunit.client.0.smithi136.stdout:op 4950 completed, throughput=5MB/sec 2024-09-13T00:17:43.031 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-MS9YHy7i3kTbwah off=1261338 len=166566 2024-09-13T00:17:43.033 INFO:tasks.workunit.client.0.smithi136.stdout:op 4951 completed, throughput=5MB/sec 2024-09-13T00:17:43.033 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ciw3M-fQPNOkt2O off=1278401 len=1254567 2024-09-13T00:17:43.036 INFO:tasks.workunit.client.0.smithi136.stdout:op 4952 completed, throughput=5MB/sec 2024-09-13T00:17:43.036 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-13g7TzdzQ0EjvO2 off=4277525 len=127081 2024-09-13T00:17:43.041 INFO:tasks.workunit.client.0.smithi136.stdout:op 4953 completed, throughput=5MB/sec 2024-09-13T00:17:43.041 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-s3vx3YvAjt5tSLx off=17846743 len=1689518 2024-09-13T00:17:43.046 INFO:tasks.workunit.client.0.smithi136.stdout:op 4954 completed, throughput=5MB/sec 2024-09-13T00:17:43.046 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_j97ouhKUT93ZwN off=5239513 len=194390 2024-09-13T00:17:43.052 INFO:tasks.workunit.client.0.smithi136.stdout:op 4955 completed, throughput=5MB/sec 2024-09-13T00:17:43.052 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-iGd4VUdq4TYJDVW off=1449969 len=6403 2024-09-13T00:17:43.056 INFO:tasks.workunit.client.0.smithi136.stdout:op 4956 completed, throughput=5MB/sec 2024-09-13T00:17:43.056 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-RrktD84LuLECkZl off=15879243 len=1179681 2024-09-13T00:17:43.065 INFO:tasks.workunit.client.0.smithi136.stdout:op 4957 completed, throughput=5MB/sec 2024-09-13T00:17:43.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:17:42 smithi181 bash[67587]: cluster 2024-09-13T00:17:41.632384+0000 mgr.y (mgr.24661) 1217 : cluster [DBG] pgmap v1900: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:17:43.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:17:42 smithi136 bash[75164]: cluster 2024-09-13T00:17:41.632384+0000 mgr.y (mgr.24661) 1217 : cluster [DBG] pgmap v1900: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:17:43.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:17:42 smithi136 bash[76371]: cluster 2024-09-13T00:17:41.632384+0000 mgr.y (mgr.24661) 1217 : cluster [DBG] pgmap v1900: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:17:44.065 INFO:tasks.workunit.client.0.smithi136.stdout: 950: throughput=5MB/sec pending data=0 2024-09-13T00:17:44.065 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-uAy5guxHoeUq4Lz off=13156218 len=1343093 2024-09-13T00:17:44.076 INFO:tasks.workunit.client.0.smithi136.stdout:op 4958 completed, throughput=5MB/sec 2024-09-13T00:17:44.076 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-dXLvA_I8xADnLPF off=5722716 len=1518916 2024-09-13T00:17:44.086 INFO:tasks.workunit.client.0.smithi136.stdout:op 4959 completed, throughput=5MB/sec 2024-09-13T00:17:44.086 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-fXfQfHLuHruXcI- off=16663486 len=10627 2024-09-13T00:17:44.087 INFO:tasks.workunit.client.0.smithi136.stdout:op 4960 completed, throughput=5MB/sec 2024-09-13T00:17:44.087 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-a28YRL-FYyw8Hjl off=2296275 len=1510125 2024-09-13T00:17:44.091 INFO:tasks.workunit.client.0.smithi136.stdout:op 4961 completed, throughput=5MB/sec 2024-09-13T00:17:44.091 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Ityujx0MHnDxAuR off=1522484 len=680495 2024-09-13T00:17:44.093 INFO:tasks.workunit.client.0.smithi136.stdout:op 4962 completed, throughput=5MB/sec 2024-09-13T00:17:45.093 INFO:tasks.workunit.client.0.smithi136.stdout: 951: throughput=5MB/sec pending data=0 2024-09-13T00:17:45.093 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ID8_2L0-dgYQtnh off=10231028 len=1714056 2024-09-13T00:17:45.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:17:44 smithi181 bash[67587]: cluster 2024-09-13T00:17:43.633441+0000 mgr.y (mgr.24661) 1218 : cluster [DBG] pgmap v1901: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 2.5 MiB/s rd, 3.0 MiB/s wr, 5 op/s 2024-09-13T00:17:45.107 INFO:tasks.workunit.client.0.smithi136.stdout:op 4963 completed, throughput=5MB/sec 2024-09-13T00:17:45.107 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-5m1GCv5eTCsGpd8 off=79571 len=1784862 2024-09-13T00:17:45.120 INFO:tasks.workunit.client.0.smithi136.stdout:op 4964 completed, throughput=5MB/sec 2024-09-13T00:17:45.120 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-rNk3QUWOASJ2XkF off=6612228 len=1353206 2024-09-13T00:17:45.124 INFO:tasks.workunit.client.0.smithi136.stdout:op 4965 completed, throughput=5MB/sec 2024-09-13T00:17:45.124 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ZUCOTuXgrJKobt3 off=682303 len=460463 2024-09-13T00:17:45.130 INFO:tasks.workunit.client.0.smithi136.stdout:op 4966 completed, throughput=5MB/sec 2024-09-13T00:17:45.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:17:44 smithi136 bash[75164]: cluster 2024-09-13T00:17:43.633441+0000 mgr.y (mgr.24661) 1218 : cluster [DBG] pgmap v1901: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 2.5 MiB/s rd, 3.0 MiB/s wr, 5 op/s 2024-09-13T00:17:45.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:17:44 smithi136 bash[76371]: cluster 2024-09-13T00:17:43.633441+0000 mgr.y (mgr.24661) 1218 : cluster [DBG] pgmap v1901: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 2.5 MiB/s rd, 3.0 MiB/s wr, 5 op/s 2024-09-13T00:17:46.131 INFO:tasks.workunit.client.0.smithi136.stdout: 952: throughput=5MB/sec pending data=0 2024-09-13T00:17:46.131 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-qWcO5wgvZySY3LJ off=7304172 len=435965 2024-09-13T00:17:46.133 INFO:tasks.workunit.client.0.smithi136.stdout:op 4967 completed, throughput=5MB/sec 2024-09-13T00:17:46.133 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-bERU4u2RpHjOY6t off=10917091 len=813148 2024-09-13T00:17:46.142 INFO:tasks.workunit.client.0.smithi136.stdout:op 4968 completed, throughput=5MB/sec 2024-09-13T00:17:46.142 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-NZFKHpLBGHe_62A off=2356163 len=1187935 2024-09-13T00:17:46.145 INFO:tasks.workunit.client.0.smithi136.stdout:op 4969 completed, throughput=5MB/sec 2024-09-13T00:17:46.145 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-VCRqUCOEL4x2kmM off=15577341 len=1294949 2024-09-13T00:17:46.157 INFO:tasks.workunit.client.0.smithi136.stdout:op 4970 completed, throughput=5MB/sec 2024-09-13T00:17:46.157 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-iC4bf4s8l_pT9DG off=13078971 len=1639304 2024-09-13T00:17:46.161 INFO:tasks.workunit.client.0.smithi136.stdout:op 4971 completed, throughput=5MB/sec 2024-09-13T00:17:47.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:17:46 smithi181 bash[67587]: cluster 2024-09-13T00:17:45.634349+0000 mgr.y (mgr.24661) 1219 : cluster [DBG] pgmap v1902: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 1.8 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:17:47.161 INFO:tasks.workunit.client.0.smithi136.stdout: 953: throughput=5MB/sec pending data=0 2024-09-13T00:17:47.161 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-CAOOWVfEhm3Z1J1 off=888955 len=545625 2024-09-13T00:17:47.168 INFO:tasks.workunit.client.0.smithi136.stdout:op 4972 completed, throughput=5MB/sec 2024-09-13T00:17:47.168 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-nEebHdRadbalqHg off=16152987 len=1705987 2024-09-13T00:17:47.179 INFO:tasks.workunit.client.0.smithi136.stdout:op 4973 completed, throughput=5MB/sec 2024-09-13T00:17:47.180 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-fdNJBs3IdrqIDVo off=19682447 len=398937 2024-09-13T00:17:47.181 INFO:tasks.workunit.client.0.smithi136.stdout:op 4974 completed, throughput=5MB/sec 2024-09-13T00:17:47.181 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-RgpgXjddd9vf--9 off=14798611 len=1152220 2024-09-13T00:17:47.193 INFO:tasks.workunit.client.0.smithi136.stdout:op 4975 completed, throughput=5MB/sec 2024-09-13T00:17:47.193 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-a_2V8o6ecxKa84g off=89251 len=1828842 2024-09-13T00:17:47.204 INFO:tasks.workunit.client.0.smithi136.stdout:op 4976 completed, throughput=5MB/sec 2024-09-13T00:17:47.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:17:46 smithi136 bash[75164]: cluster 2024-09-13T00:17:45.634349+0000 mgr.y (mgr.24661) 1219 : cluster [DBG] pgmap v1902: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 1.8 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:17:47.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:17:46 smithi136 bash[76371]: cluster 2024-09-13T00:17:45.634349+0000 mgr.y (mgr.24661) 1219 : cluster [DBG] pgmap v1902: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 1.8 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:17:48.205 INFO:tasks.workunit.client.0.smithi136.stdout: 954: throughput=5MB/sec pending data=0 2024-09-13T00:17:48.205 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-05u3-7vrrZM66O- off=2812202 len=524993 2024-09-13T00:17:48.207 INFO:tasks.workunit.client.0.smithi136.stdout:op 4977 completed, throughput=5MB/sec 2024-09-13T00:17:48.207 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-09p2k2EtW5Y28MY off=11529645 len=859356 2024-09-13T00:17:48.210 INFO:tasks.workunit.client.0.smithi136.stdout:op 4978 completed, throughput=5MB/sec 2024-09-13T00:17:48.210 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Eyk667sjfpwfjZ7 off=4313175 len=1757836 2024-09-13T00:17:48.223 INFO:tasks.workunit.client.0.smithi136.stdout:op 4979 completed, throughput=5MB/sec 2024-09-13T00:17:48.223 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-lUnNEXV3Qfxytc3 off=12631661 len=2081036 2024-09-13T00:17:48.241 INFO:tasks.workunit.client.0.smithi136.stdout:op 4980 completed, throughput=5MB/sec 2024-09-13T00:17:49.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:17:48 smithi181 bash[67587]: cluster 2024-09-13T00:17:47.635372+0000 mgr.y (mgr.24661) 1220 : cluster [DBG] pgmap v1903: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:17:49.242 INFO:tasks.workunit.client.0.smithi136.stdout: 955: throughput=5MB/sec pending data=0 2024-09-13T00:17:49.242 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-qwi6U5Y41OAfirG off=965308 len=571954 2024-09-13T00:17:49.244 INFO:tasks.workunit.client.0.smithi136.stdout:op 4981 completed, throughput=5MB/sec 2024-09-13T00:17:49.245 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-yrGJ3KolSdWQM0k off=289540 len=881703 2024-09-13T00:17:49.248 INFO:tasks.workunit.client.0.smithi136.stdout:op 4982 completed, throughput=5MB/sec 2024-09-13T00:17:49.248 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_CmXJ6zij71rfBA off=7089138 len=1265821 2024-09-13T00:17:49.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:17:48 smithi136 bash[75164]: cluster 2024-09-13T00:17:47.635372+0000 mgr.y (mgr.24661) 1220 : cluster [DBG] pgmap v1903: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:17:49.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:17:48 smithi136 bash[76371]: cluster 2024-09-13T00:17:47.635372+0000 mgr.y (mgr.24661) 1220 : cluster [DBG] pgmap v1903: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:17:49.258 INFO:tasks.workunit.client.0.smithi136.stdout:op 4983 completed, throughput=5MB/sec 2024-09-13T00:17:49.258 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ZIZFVdp20vqMmzp off=10175439 len=1174627 2024-09-13T00:17:49.267 INFO:tasks.workunit.client.0.smithi136.stdout:op 4984 completed, throughput=5MB/sec 2024-09-13T00:17:49.267 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-3JimBC8932otyi7 off=3966187 len=1808712 2024-09-13T00:17:49.278 INFO:tasks.workunit.client.0.smithi136.stdout:op 4985 completed, throughput=5MB/sec 2024-09-13T00:17:49.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:17:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:17:49] "GET /metrics HTTP/1.1" 200 34143 "" "Prometheus/2.43.0" 2024-09-13T00:17:50.278 INFO:tasks.workunit.client.0.smithi136.stdout: 956: throughput=5MB/sec pending data=0 2024-09-13T00:17:50.278 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-RMzFp08ZszETveD off=2514232 len=1047097 2024-09-13T00:17:50.281 INFO:tasks.workunit.client.0.smithi136.stdout:op 4986 completed, throughput=5MB/sec 2024-09-13T00:17:50.281 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-hmBPRdljmV2G1_V off=10777726 len=1616964 2024-09-13T00:17:50.292 INFO:tasks.workunit.client.0.smithi136.stdout:op 4987 completed, throughput=5MB/sec 2024-09-13T00:17:50.292 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj--R9osdA1vO89gsa off=5239696 len=1967819 2024-09-13T00:17:50.306 INFO:tasks.workunit.client.0.smithi136.stdout:op 4988 completed, throughput=5MB/sec 2024-09-13T00:17:50.306 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-c6c8_TYTuw47VA1 off=575394 len=1304735 2024-09-13T00:17:50.315 INFO:tasks.workunit.client.0.smithi136.stdout:op 4989 completed, throughput=5MB/sec 2024-09-13T00:17:51.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:17:50 smithi181 bash[67587]: cluster 2024-09-13T00:17:49.636202+0000 mgr.y (mgr.24661) 1221 : cluster [DBG] pgmap v1904: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:17:51.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:17:50 smithi136 bash[75164]: cluster 2024-09-13T00:17:49.636202+0000 mgr.y (mgr.24661) 1221 : cluster [DBG] pgmap v1904: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:17:51.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:17:50 smithi136 bash[76371]: cluster 2024-09-13T00:17:49.636202+0000 mgr.y (mgr.24661) 1221 : cluster [DBG] pgmap v1904: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:17:51.317 INFO:tasks.workunit.client.0.smithi136.stdout: 957: throughput=5MB/sec pending data=0 2024-09-13T00:17:51.317 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-B0d1j-3FrAsmrTW off=7179083 len=1733180 2024-09-13T00:17:51.321 INFO:tasks.workunit.client.0.smithi136.stdout:op 4990 completed, throughput=5MB/sec 2024-09-13T00:17:51.321 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj--TsRZH_cQEleUA5 off=156761 len=1032621 2024-09-13T00:17:51.323 INFO:tasks.workunit.client.0.smithi136.stdout:op 4991 completed, throughput=5MB/sec 2024-09-13T00:17:51.323 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-XgS2w5whA9G6rOf off=935291 len=757613 2024-09-13T00:17:51.326 INFO:tasks.workunit.client.0.smithi136.stdout:op 4992 completed, throughput=5MB/sec 2024-09-13T00:17:51.326 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-I_sA0joJuMCvkby off=1741498 len=1687801 2024-09-13T00:17:51.342 INFO:tasks.workunit.client.0.smithi136.stdout:op 4993 completed, throughput=5MB/sec 2024-09-13T00:17:52.342 INFO:tasks.workunit.client.0.smithi136.stdout: 958: throughput=5MB/sec pending data=0 2024-09-13T00:17:52.342 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-CFzy6Xuf-9OrBaA off=4584186 len=888372 2024-09-13T00:17:52.350 INFO:tasks.workunit.client.0.smithi136.stdout:op 4994 completed, throughput=5MB/sec 2024-09-13T00:17:52.350 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Q5zB89-CTnwu4Gj off=1105068 len=113476 2024-09-13T00:17:52.355 INFO:tasks.workunit.client.0.smithi136.stdout:op 4995 completed, throughput=5MB/sec 2024-09-13T00:17:52.355 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-inFa4fMhO9Bxrft off=1646624 len=657521 2024-09-13T00:17:52.362 INFO:tasks.workunit.client.0.smithi136.stdout:op 4996 completed, throughput=5MB/sec 2024-09-13T00:17:52.362 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-HqPTbBYkbqIe_sm off=3005530 len=601077 2024-09-13T00:17:52.369 INFO:tasks.workunit.client.0.smithi136.stdout:op 4997 completed, throughput=5MB/sec 2024-09-13T00:17:52.369 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-jBmcTf3t2lbGPrr off=5808151 len=437048 2024-09-13T00:17:52.378 INFO:tasks.workunit.client.0.smithi136.stdout:op 4998 completed, throughput=5MB/sec 2024-09-13T00:17:52.378 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-kKeN5MBeWFJtdYQ off=7196171 len=760501 2024-09-13T00:17:52.380 INFO:tasks.workunit.client.0.smithi136.stdout:op 4999 completed, throughput=5MB/sec 2024-09-13T00:17:52.380 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-QRIwCS0FiEWYSdb off=6761809 len=1103565 2024-09-13T00:17:52.390 INFO:tasks.workunit.client.0.smithi136.stdout:op 5000 completed, throughput=5MB/sec 2024-09-13T00:17:52.390 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-j9unxF2sxDA9coG off=794649 len=164420 2024-09-13T00:17:52.391 INFO:tasks.workunit.client.0.smithi136.stdout:op 5001 completed, throughput=5MB/sec 2024-09-13T00:17:52.391 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj--g2M_faM7FONOZT off=619832 len=1352389 2024-09-13T00:17:52.401 INFO:tasks.workunit.client.0.smithi136.stdout:op 5002 completed, throughput=5MB/sec 2024-09-13T00:17:52.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:17:52 smithi181 bash[95505]: ts=2024-09-13T00:17:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:17:53.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:17:52 smithi181 bash[67587]: cluster 2024-09-13T00:17:51.637231+0000 mgr.y (mgr.24661) 1222 : cluster [DBG] pgmap v1905: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:17:53.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:17:52 smithi136 bash[75164]: cluster 2024-09-13T00:17:51.637231+0000 mgr.y (mgr.24661) 1222 : cluster [DBG] pgmap v1905: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:17:53.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:17:52 smithi136 bash[76371]: cluster 2024-09-13T00:17:51.637231+0000 mgr.y (mgr.24661) 1222 : cluster [DBG] pgmap v1905: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:17:53.402 INFO:tasks.workunit.client.0.smithi136.stdout: 959: throughput=5MB/sec pending data=0 2024-09-13T00:17:53.402 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-HaMBLc-ytxlOouL off=2281789 len=20267 2024-09-13T00:17:53.407 INFO:tasks.workunit.client.0.smithi136.stdout:op 5003 completed, throughput=5MB/sec 2024-09-13T00:17:53.407 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-2CKCOGa0r2UTav0 off=6402845 len=1473568 2024-09-13T00:17:53.419 INFO:tasks.workunit.client.0.smithi136.stdout:op 5004 completed, throughput=5MB/sec 2024-09-13T00:17:53.419 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-KH-X2h3iyPjFPOm off=700642 len=590113 2024-09-13T00:17:53.421 INFO:tasks.workunit.client.0.smithi136.stdout:op 5005 completed, throughput=5MB/sec 2024-09-13T00:17:53.421 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ZVVFKg-wlzyFzPk off=3759617 len=243277 2024-09-13T00:17:53.426 INFO:tasks.workunit.client.0.smithi136.stdout:op 5006 completed, throughput=5MB/sec 2024-09-13T00:17:53.427 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Cv0pPqfT62sGxbH off=1714538 len=1164594 2024-09-13T00:17:53.430 INFO:tasks.workunit.client.0.smithi136.stdout:op 5007 completed, throughput=5MB/sec 2024-09-13T00:17:53.430 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-rlPUitirJBgemwO off=23440113 len=223752 2024-09-13T00:17:53.431 INFO:tasks.workunit.client.0.smithi136.stdout:op 5008 completed, throughput=5MB/sec 2024-09-13T00:17:53.431 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-T7GmQ9Vhzv6dehD off=10780282 len=1719280 2024-09-13T00:17:53.435 INFO:tasks.workunit.client.0.smithi136.stdout:op 5009 completed, throughput=5MB/sec 2024-09-13T00:17:54.436 INFO:tasks.workunit.client.0.smithi136.stdout: 960: throughput=5MB/sec pending data=0 2024-09-13T00:17:54.436 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-pJ2Xd4Qetz9QOD7 off=12838079 len=1943410 2024-09-13T00:17:54.451 INFO:tasks.workunit.client.0.smithi136.stdout:op 5010 completed, throughput=5MB/sec 2024-09-13T00:17:54.451 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-NjxIcMTt4MdEzLC off=11365141 len=683011 2024-09-13T00:17:54.453 INFO:tasks.workunit.client.0.smithi136.stdout:op 5011 completed, throughput=5MB/sec 2024-09-13T00:17:54.453 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-jHKp-zbgy7_S0Oa off=12590955 len=1517279 2024-09-13T00:17:54.457 INFO:tasks.workunit.client.0.smithi136.stdout:op 5012 completed, throughput=5MB/sec 2024-09-13T00:17:55.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:17:54 smithi181 bash[67587]: cluster 2024-09-13T00:17:53.638310+0000 mgr.y (mgr.24661) 1223 : cluster [DBG] pgmap v1906: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 2.0 MiB/s rd, 3.4 MiB/s wr, 5 op/s 2024-09-13T00:17:55.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:17:54 smithi181 bash[67587]: audit 2024-09-13T00:17:53.799330+0000 mon.a (mon.0) 2251 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:17:55.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:17:54 smithi136 bash[75164]: cluster 2024-09-13T00:17:53.638310+0000 mgr.y (mgr.24661) 1223 : cluster [DBG] pgmap v1906: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 2.0 MiB/s rd, 3.4 MiB/s wr, 5 op/s 2024-09-13T00:17:55.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:17:54 smithi136 bash[75164]: audit 2024-09-13T00:17:53.799330+0000 mon.a (mon.0) 2251 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:17:55.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:17:54 smithi136 bash[76371]: cluster 2024-09-13T00:17:53.638310+0000 mgr.y (mgr.24661) 1223 : cluster [DBG] pgmap v1906: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 2.0 MiB/s rd, 3.4 MiB/s wr, 5 op/s 2024-09-13T00:17:55.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:17:54 smithi136 bash[76371]: audit 2024-09-13T00:17:53.799330+0000 mon.a (mon.0) 2251 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:17:55.458 INFO:tasks.workunit.client.0.smithi136.stdout: 961: throughput=5MB/sec pending data=0 2024-09-13T00:17:55.458 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-rSdJgp_UbEswp2X off=5206178 len=470385 2024-09-13T00:17:55.459 INFO:tasks.workunit.client.0.smithi136.stdout:op 5013 completed, throughput=5MB/sec 2024-09-13T00:17:55.459 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-JwqKmuLD8WRpUR- off=1030248 len=643396 2024-09-13T00:17:55.462 INFO:tasks.workunit.client.0.smithi136.stdout:op 5014 completed, throughput=5MB/sec 2024-09-13T00:17:55.462 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2yKdOhpNI1wFm4k off=8328055 len=1466010 2024-09-13T00:17:55.466 INFO:tasks.workunit.client.0.smithi136.stdout:op 5015 completed, throughput=5MB/sec 2024-09-13T00:17:55.466 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-r10gHR34ETwgT3G off=9293533 len=680312 2024-09-13T00:17:55.472 INFO:tasks.workunit.client.0.smithi136.stdout:op 5016 completed, throughput=5MB/sec 2024-09-13T00:17:55.473 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-QYzS_0LKRJYiXhg off=3171119 len=1200661 2024-09-13T00:17:55.475 INFO:tasks.workunit.client.0.smithi136.stdout:op 5017 completed, throughput=5MB/sec 2024-09-13T00:17:55.475 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ualpAa-HeZo3Exw off=2105802 len=293814 2024-09-13T00:17:55.481 INFO:tasks.workunit.client.0.smithi136.stdout:op 5018 completed, throughput=5MB/sec 2024-09-13T00:17:55.481 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-AhQhFZliegbdLmD off=7802009 len=972232 2024-09-13T00:17:55.488 INFO:tasks.workunit.client.0.smithi136.stdout:op 5019 completed, throughput=5MB/sec 2024-09-13T00:17:56.489 INFO:tasks.workunit.client.0.smithi136.stdout: 962: throughput=5MB/sec pending data=0 2024-09-13T00:17:56.489 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-iBvkG860dfZS5sd off=2228527 len=305260 2024-09-13T00:17:56.494 INFO:tasks.workunit.client.0.smithi136.stdout:op 5020 completed, throughput=5MB/sec 2024-09-13T00:17:56.495 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Xrviw6gmU88okR8 off=736660 len=827736 2024-09-13T00:17:56.503 INFO:tasks.workunit.client.0.smithi136.stdout:op 5021 completed, throughput=5MB/sec 2024-09-13T00:17:56.503 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-VXWyvXT2yck8dNv off=12027141 len=1727268 2024-09-13T00:17:56.517 INFO:tasks.workunit.client.0.smithi136.stdout:op 5022 completed, throughput=5MB/sec 2024-09-13T00:17:56.517 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-8ygBJjnIBoXHGTR off=1687145 len=1146526 2024-09-13T00:17:56.526 INFO:tasks.workunit.client.0.smithi136.stdout:op 5023 completed, throughput=5MB/sec 2024-09-13T00:17:56.526 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-msxCikkWnyQWe-w off=4032053 len=1555659 2024-09-13T00:17:56.537 INFO:tasks.workunit.client.0.smithi136.stdout:op 5024 completed, throughput=5MB/sec 2024-09-13T00:17:57.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:17:56 smithi181 bash[67587]: cluster 2024-09-13T00:17:55.639219+0000 mgr.y (mgr.24661) 1224 : cluster [DBG] pgmap v1907: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 1.8 MiB/s rd, 2.9 MiB/s wr, 4 op/s 2024-09-13T00:17:57.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:17:56 smithi136 bash[75164]: cluster 2024-09-13T00:17:55.639219+0000 mgr.y (mgr.24661) 1224 : cluster [DBG] pgmap v1907: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 1.8 MiB/s rd, 2.9 MiB/s wr, 4 op/s 2024-09-13T00:17:57.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:17:56 smithi136 bash[76371]: cluster 2024-09-13T00:17:55.639219+0000 mgr.y (mgr.24661) 1224 : cluster [DBG] pgmap v1907: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 1.8 MiB/s rd, 2.9 MiB/s wr, 4 op/s 2024-09-13T00:17:57.538 INFO:tasks.workunit.client.0.smithi136.stdout: 963: throughput=5MB/sec pending data=0 2024-09-13T00:17:57.538 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-DFrHJq3H4drUnkj off=16408262 len=94827 2024-09-13T00:17:57.539 INFO:tasks.workunit.client.0.smithi136.stdout:op 5025 completed, throughput=5MB/sec 2024-09-13T00:17:57.539 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-yxWliLQKvBjT_6U off=6721818 len=1732148 2024-09-13T00:17:57.554 INFO:tasks.workunit.client.0.smithi136.stdout:op 5026 completed, throughput=5MB/sec 2024-09-13T00:17:57.554 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-060r9ZJGTRQv02P off=6860074 len=1788530 2024-09-13T00:17:57.568 INFO:tasks.workunit.client.0.smithi136.stdout:op 5027 completed, throughput=5MB/sec 2024-09-13T00:17:57.568 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-nUJV2F7VdL10HCv off=9378914 len=1148047 2024-09-13T00:17:57.576 INFO:tasks.workunit.client.0.smithi136.stdout:op 5028 completed, throughput=5MB/sec 2024-09-13T00:17:57.576 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-RiqooyqvwRn28rT off=23591460 len=1700501 2024-09-13T00:17:57.588 INFO:tasks.workunit.client.0.smithi136.stdout:op 5029 completed, throughput=5MB/sec 2024-09-13T00:17:58.589 INFO:tasks.workunit.client.0.smithi136.stdout: 964: throughput=5MB/sec pending data=0 2024-09-13T00:17:58.589 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-QGIsrNwNwEYYuTs off=811517 len=844333 2024-09-13T00:17:58.595 INFO:tasks.workunit.client.0.smithi136.stdout:op 5030 completed, throughput=5MB/sec 2024-09-13T00:17:58.595 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-EGdpXMlxzcPJBqF off=1348294 len=265204 2024-09-13T00:17:58.597 INFO:tasks.workunit.client.0.smithi136.stdout:op 5031 completed, throughput=5MB/sec 2024-09-13T00:17:58.597 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-XEFWl29ZvoukAGQ off=6589179 len=1750597 2024-09-13T00:17:58.600 INFO:tasks.workunit.client.0.smithi136.stdout:op 5032 completed, throughput=5MB/sec 2024-09-13T00:17:58.600 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-kKeHeiRMnJxBjYo off=20785916 len=1942059 2024-09-13T00:17:58.612 INFO:tasks.workunit.client.0.smithi136.stdout:op 5033 completed, throughput=5MB/sec 2024-09-13T00:17:59.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:17:58 smithi181 bash[67587]: cluster 2024-09-13T00:17:57.640314+0000 mgr.y (mgr.24661) 1225 : cluster [DBG] pgmap v1908: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.2 MiB/s rd, 3.3 MiB/s wr, 5 op/s 2024-09-13T00:17:59.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:17:58 smithi181 bash[67587]: audit 2024-09-13T00:17:58.082214+0000 mon.a (mon.0) 2252 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:17:59.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:17:58 smithi181 bash[67587]: audit 2024-09-13T00:17:58.631566+0000 mon.a (mon.0) 2253 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:17:59.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:17:58 smithi181 bash[67587]: audit 2024-09-13T00:17:58.633108+0000 mon.a (mon.0) 2254 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:17:59.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:17:58 smithi181 bash[67587]: audit 2024-09-13T00:17:58.642847+0000 mon.a (mon.0) 2255 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:17:59.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:17:58 smithi136 bash[75164]: cluster 2024-09-13T00:17:57.640314+0000 mgr.y (mgr.24661) 1225 : cluster [DBG] pgmap v1908: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.2 MiB/s rd, 3.3 MiB/s wr, 5 op/s 2024-09-13T00:17:59.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:17:58 smithi136 bash[75164]: audit 2024-09-13T00:17:58.082214+0000 mon.a (mon.0) 2252 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:17:59.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:17:58 smithi136 bash[75164]: audit 2024-09-13T00:17:58.631566+0000 mon.a (mon.0) 2253 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:17:59.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:17:58 smithi136 bash[75164]: audit 2024-09-13T00:17:58.633108+0000 mon.a (mon.0) 2254 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:17:59.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:17:58 smithi136 bash[75164]: audit 2024-09-13T00:17:58.642847+0000 mon.a (mon.0) 2255 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:17:59.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:17:58 smithi136 bash[76371]: cluster 2024-09-13T00:17:57.640314+0000 mgr.y (mgr.24661) 1225 : cluster [DBG] pgmap v1908: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.2 MiB/s rd, 3.3 MiB/s wr, 5 op/s 2024-09-13T00:17:59.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:17:58 smithi136 bash[76371]: audit 2024-09-13T00:17:58.082214+0000 mon.a (mon.0) 2252 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:17:59.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:17:58 smithi136 bash[76371]: audit 2024-09-13T00:17:58.631566+0000 mon.a (mon.0) 2253 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:17:59.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:17:58 smithi136 bash[76371]: audit 2024-09-13T00:17:58.633108+0000 mon.a (mon.0) 2254 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:17:59.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:17:58 smithi136 bash[76371]: audit 2024-09-13T00:17:58.642847+0000 mon.a (mon.0) 2255 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:17:59.613 INFO:tasks.workunit.client.0.smithi136.stdout: 965: throughput=5MB/sec pending data=0 2024-09-13T00:17:59.613 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-QjPFLhqVCgoHoX5 off=10714151 len=326499 2024-09-13T00:17:59.614 INFO:tasks.workunit.client.0.smithi136.stdout:op 5034 completed, throughput=5MB/sec 2024-09-13T00:17:59.614 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-WBFjuw3R3RTobgK off=6554877 len=1353307 2024-09-13T00:17:59.628 INFO:tasks.workunit.client.0.smithi136.stdout:op 5035 completed, throughput=5MB/sec 2024-09-13T00:17:59.629 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-2V7Cis_f8Oq7dfr off=6614912 len=281312 2024-09-13T00:17:59.634 INFO:tasks.workunit.client.0.smithi136.stdout:op 5036 completed, throughput=5MB/sec 2024-09-13T00:17:59.634 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-l9WqaIbDANJOCtZ off=5444967 len=1731231 2024-09-13T00:17:59.648 INFO:tasks.workunit.client.0.smithi136.stdout:op 5037 completed, throughput=5MB/sec 2024-09-13T00:17:59.648 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-PULVqqggzS_a0mv off=541453 len=1580223 2024-09-13T00:17:59.652 INFO:tasks.workunit.client.0.smithi136.stdout:op 5038 completed, throughput=5MB/sec 2024-09-13T00:17:59.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:17:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:17:59] "GET /metrics HTTP/1.1" 200 34141 "" "Prometheus/2.43.0" 2024-09-13T00:18:00.652 INFO:tasks.workunit.client.0.smithi136.stdout: 966: throughput=5MB/sec pending data=0 2024-09-13T00:18:00.653 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-D-zvPlSFOlNnQl- off=2141600 len=1568385 2024-09-13T00:18:00.656 INFO:tasks.workunit.client.0.smithi136.stdout:op 5039 completed, throughput=5MB/sec 2024-09-13T00:18:00.656 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-BvZUWa7yqli5MaP off=6101163 len=311651 2024-09-13T00:18:00.662 INFO:tasks.workunit.client.0.smithi136.stdout:op 5040 completed, throughput=5MB/sec 2024-09-13T00:18:00.662 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-cyDij3xfd38nGKx off=6028414 len=335106 2024-09-13T00:18:00.664 INFO:tasks.workunit.client.0.smithi136.stdout:op 5041 completed, throughput=5MB/sec 2024-09-13T00:18:00.664 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-G8qwQAG3Yopj_2c off=8037257 len=1669130 2024-09-13T00:18:00.676 INFO:tasks.workunit.client.0.smithi136.stdout:op 5042 completed, throughput=5MB/sec 2024-09-13T00:18:00.676 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-FcE5zNWJyQ5MpPH off=10517697 len=1076570 2024-09-13T00:18:00.686 INFO:tasks.workunit.client.0.smithi136.stdout:op 5043 completed, throughput=5MB/sec 2024-09-13T00:18:01.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:18:00 smithi181 bash[67587]: cluster 2024-09-13T00:17:59.641188+0000 mgr.y (mgr.24661) 1226 : cluster [DBG] pgmap v1909: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 1.6 MiB/s rd, 3.4 MiB/s wr, 4 op/s 2024-09-13T00:18:01.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:18:00 smithi136 bash[75164]: cluster 2024-09-13T00:17:59.641188+0000 mgr.y (mgr.24661) 1226 : cluster [DBG] pgmap v1909: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 1.6 MiB/s rd, 3.4 MiB/s wr, 4 op/s 2024-09-13T00:18:01.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:18:00 smithi136 bash[76371]: cluster 2024-09-13T00:17:59.641188+0000 mgr.y (mgr.24661) 1226 : cluster [DBG] pgmap v1909: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 1.6 MiB/s rd, 3.4 MiB/s wr, 4 op/s 2024-09-13T00:18:01.686 INFO:tasks.workunit.client.0.smithi136.stdout: 967: throughput=5MB/sec pending data=0 2024-09-13T00:18:01.687 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Nv9Uz4kE9q7ZC31 off=3168158 len=143735 2024-09-13T00:18:01.688 INFO:tasks.workunit.client.0.smithi136.stdout:op 5044 completed, throughput=5MB/sec 2024-09-13T00:18:01.688 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-GYFytrp1QMxPNRJ off=1990852 len=767554 2024-09-13T00:18:01.696 INFO:tasks.workunit.client.0.smithi136.stdout:op 5045 completed, throughput=5MB/sec 2024-09-13T00:18:01.696 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-dRb0pOWR0P6go5S off=3943481 len=1522989 2024-09-13T00:18:01.706 INFO:tasks.workunit.client.0.smithi136.stdout:op 5046 completed, throughput=5MB/sec 2024-09-13T00:18:01.707 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-RZfswUxtGadHoBx off=15271030 len=939975 2024-09-13T00:18:01.715 INFO:tasks.workunit.client.0.smithi136.stdout:op 5047 completed, throughput=5MB/sec 2024-09-13T00:18:01.715 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-xog0S7XmMnHNtPB off=6701483 len=770818 2024-09-13T00:18:01.724 INFO:tasks.workunit.client.0.smithi136.stdout:op 5048 completed, throughput=5MB/sec 2024-09-13T00:18:01.724 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-RSl3mr-_8M0LF5z off=9257993 len=871899 2024-09-13T00:18:01.728 INFO:tasks.workunit.client.0.smithi136.stdout:op 5049 completed, throughput=5MB/sec 2024-09-13T00:18:01.728 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_arWAiWoQama57y off=10136235 len=920117 2024-09-13T00:18:01.735 INFO:tasks.workunit.client.0.smithi136.stdout:op 5050 completed, throughput=5MB/sec 2024-09-13T00:18:02.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:18:02 smithi181 bash[95505]: ts=2024-09-13T00:18:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:18:02.736 INFO:tasks.workunit.client.0.smithi136.stdout: 968: throughput=5MB/sec pending data=0 2024-09-13T00:18:02.736 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-5uoc1gSOeXhvgVR off=5857602 len=1993480 2024-09-13T00:18:02.740 INFO:tasks.workunit.client.0.smithi136.stdout:op 5051 completed, throughput=5MB/sec 2024-09-13T00:18:02.740 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-vhNfpbaUuATRgOS off=14487013 len=571999 2024-09-13T00:18:02.742 INFO:tasks.workunit.client.0.smithi136.stdout:op 5052 completed, throughput=5MB/sec 2024-09-13T00:18:02.742 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-3J2djkrN-lnlwJE off=5763671 len=1105424 2024-09-13T00:18:02.745 INFO:tasks.workunit.client.0.smithi136.stdout:op 5053 completed, throughput=5MB/sec 2024-09-13T00:18:02.745 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Ic-335yqalYSQYZ off=4604540 len=828018 2024-09-13T00:18:02.747 INFO:tasks.workunit.client.0.smithi136.stdout:op 5054 completed, throughput=5MB/sec 2024-09-13T00:18:02.747 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-755w9s4e46dn4y2 off=562165 len=988294 2024-09-13T00:18:02.757 INFO:tasks.workunit.client.0.smithi136.stdout:op 5055 completed, throughput=5MB/sec 2024-09-13T00:18:03.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:18:02 smithi181 bash[67587]: cluster 2024-09-13T00:18:01.642162+0000 mgr.y (mgr.24661) 1227 : cluster [DBG] pgmap v1910: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 1.5 MiB/s rd, 3.3 MiB/s wr, 4 op/s 2024-09-13T00:18:03.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:18:02 smithi136 bash[75164]: cluster 2024-09-13T00:18:01.642162+0000 mgr.y (mgr.24661) 1227 : cluster [DBG] pgmap v1910: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 1.5 MiB/s rd, 3.3 MiB/s wr, 4 op/s 2024-09-13T00:18:03.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:18:02 smithi136 bash[76371]: cluster 2024-09-13T00:18:01.642162+0000 mgr.y (mgr.24661) 1227 : cluster [DBG] pgmap v1910: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 1.5 MiB/s rd, 3.3 MiB/s wr, 4 op/s 2024-09-13T00:18:03.758 INFO:tasks.workunit.client.0.smithi136.stdout: 969: throughput=5MB/sec pending data=0 2024-09-13T00:18:03.758 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-fCVNxpWXM6IyBzZ off=7715293 len=1710904 2024-09-13T00:18:03.769 INFO:tasks.workunit.client.0.smithi136.stdout:op 5056 completed, throughput=5MB/sec 2024-09-13T00:18:03.769 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-s85A9vLU3YgBf9L off=988078 len=1375343 2024-09-13T00:18:03.772 INFO:tasks.workunit.client.0.smithi136.stdout:op 5057 completed, throughput=5MB/sec 2024-09-13T00:18:03.772 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-hWALfWq6LHp3mlL off=849652 len=612571 2024-09-13T00:18:03.774 INFO:tasks.workunit.client.0.smithi136.stdout:op 5058 completed, throughput=5MB/sec 2024-09-13T00:18:03.774 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-apJTiTnoTTf6k9g off=5407815 len=498834 2024-09-13T00:18:03.781 INFO:tasks.workunit.client.0.smithi136.stdout:op 5059 completed, throughput=5MB/sec 2024-09-13T00:18:03.781 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-gqKntz1iedEs1Va off=460411 len=1914508 2024-09-13T00:18:03.792 INFO:tasks.workunit.client.0.smithi136.stdout:op 5060 completed, throughput=5MB/sec 2024-09-13T00:18:04.792 INFO:tasks.workunit.client.0.smithi136.stdout: 970: throughput=5MB/sec pending data=0 2024-09-13T00:18:04.793 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-e_omJ6RztAikJ90 off=6460544 len=1562598 2024-09-13T00:18:04.796 INFO:tasks.workunit.client.0.smithi136.stdout:op 5061 completed, throughput=5MB/sec 2024-09-13T00:18:04.796 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-KVABFOVtKG0Vp5E off=1975078 len=1908979 2024-09-13T00:18:04.809 INFO:tasks.workunit.client.0.smithi136.stdout:op 5062 completed, throughput=5MB/sec 2024-09-13T00:18:04.809 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-NjzLDlN80rFbaMY off=13548527 len=1973394 2024-09-13T00:18:04.817 INFO:tasks.workunit.client.0.smithi136.stdout:op 5063 completed, throughput=5MB/sec 2024-09-13T00:18:05.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:18:04 smithi181 bash[67587]: cluster 2024-09-13T00:18:03.643198+0000 mgr.y (mgr.24661) 1228 : cluster [DBG] pgmap v1911: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 1.6 MiB/s rd, 3.2 MiB/s wr, 4 op/s 2024-09-13T00:18:05.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:18:04 smithi136 bash[75164]: cluster 2024-09-13T00:18:03.643198+0000 mgr.y (mgr.24661) 1228 : cluster [DBG] pgmap v1911: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 1.6 MiB/s rd, 3.2 MiB/s wr, 4 op/s 2024-09-13T00:18:05.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:18:04 smithi136 bash[76371]: cluster 2024-09-13T00:18:03.643198+0000 mgr.y (mgr.24661) 1228 : cluster [DBG] pgmap v1911: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 1.6 MiB/s rd, 3.2 MiB/s wr, 4 op/s 2024-09-13T00:18:05.818 INFO:tasks.workunit.client.0.smithi136.stdout: 971: throughput=5MB/sec pending data=0 2024-09-13T00:18:05.818 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-SLDkGETMS4FVkli off=5887322 len=2018423 2024-09-13T00:18:05.838 INFO:tasks.workunit.client.0.smithi136.stdout:op 5064 completed, throughput=5MB/sec 2024-09-13T00:18:05.838 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-tmFRbqZSHGbD39N off=10284173 len=569870 2024-09-13T00:18:05.845 INFO:tasks.workunit.client.0.smithi136.stdout:op 5065 completed, throughput=5MB/sec 2024-09-13T00:18:05.845 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-hdf0vM742U90El9 off=3297903 len=1735222 2024-09-13T00:18:05.858 INFO:tasks.workunit.client.0.smithi136.stdout:op 5066 completed, throughput=5MB/sec 2024-09-13T00:18:06.859 INFO:tasks.workunit.client.0.smithi136.stdout: 972: throughput=5MB/sec pending data=0 2024-09-13T00:18:06.859 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-FuoaTep-Krxs5BL off=3861754 len=2029469 2024-09-13T00:18:06.872 INFO:tasks.workunit.client.0.smithi136.stdout:op 5067 completed, throughput=5MB/sec 2024-09-13T00:18:06.872 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-jRJ6PW1-rZTsMPV off=15221033 len=464400 2024-09-13T00:18:06.878 INFO:tasks.workunit.client.0.smithi136.stdout:op 5068 completed, throughput=5MB/sec 2024-09-13T00:18:06.879 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-5Ndad4rKfBJHICR off=811866 len=87516 2024-09-13T00:18:06.883 INFO:tasks.workunit.client.0.smithi136.stdout:op 5069 completed, throughput=5MB/sec 2024-09-13T00:18:06.883 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-OvUTCqSEkTYXAfA off=5374091 len=1922039 2024-09-13T00:18:06.895 INFO:tasks.workunit.client.0.smithi136.stdout:op 5070 completed, throughput=5MB/sec 2024-09-13T00:18:06.895 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj--JdqiMh1Mi9LMk4 off=6687304 len=591640 2024-09-13T00:18:06.896 INFO:tasks.workunit.client.0.smithi136.stdout:op 5071 completed, throughput=5MB/sec 2024-09-13T00:18:07.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:18:06 smithi181 bash[67587]: cluster 2024-09-13T00:18:05.644138+0000 mgr.y (mgr.24661) 1229 : cluster [DBG] pgmap v1912: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 1.6 MiB/s rd, 2.7 MiB/s wr, 4 op/s 2024-09-13T00:18:07.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:18:06 smithi136 bash[75164]: cluster 2024-09-13T00:18:05.644138+0000 mgr.y (mgr.24661) 1229 : cluster [DBG] pgmap v1912: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 1.6 MiB/s rd, 2.7 MiB/s wr, 4 op/s 2024-09-13T00:18:07.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:18:06 smithi136 bash[76371]: cluster 2024-09-13T00:18:05.644138+0000 mgr.y (mgr.24661) 1229 : cluster [DBG] pgmap v1912: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 1.6 MiB/s rd, 2.7 MiB/s wr, 4 op/s 2024-09-13T00:18:07.897 INFO:tasks.workunit.client.0.smithi136.stdout: 973: throughput=5MB/sec pending data=0 2024-09-13T00:18:07.897 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-j9Q_ADAbwaBuqAu off=2555163 len=740099 2024-09-13T00:18:07.905 INFO:tasks.workunit.client.0.smithi136.stdout:op 5072 completed, throughput=5MB/sec 2024-09-13T00:18:07.905 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-R-B2gPjmKdbLTYk off=6982375 len=1940689 2024-09-13T00:18:07.920 INFO:tasks.workunit.client.0.smithi136.stdout:op 5073 completed, throughput=5MB/sec 2024-09-13T00:18:07.920 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-kTj8cc7VEq0FMLl off=2925193 len=796791 2024-09-13T00:18:07.928 INFO:tasks.workunit.client.0.smithi136.stdout:op 5074 completed, throughput=5MB/sec 2024-09-13T00:18:07.928 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-dB3zF1F2K5KPeL8 off=24611 len=66284 2024-09-13T00:18:07.932 INFO:tasks.workunit.client.0.smithi136.stdout:op 5075 completed, throughput=5MB/sec 2024-09-13T00:18:07.932 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ZD1WjXvp_uzsj5X off=23903474 len=719867 2024-09-13T00:18:07.939 INFO:tasks.workunit.client.0.smithi136.stdout:op 5076 completed, throughput=5MB/sec 2024-09-13T00:18:07.939 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-hxGeHbfBs0fiVr_ off=12487333 len=560737 2024-09-13T00:18:07.945 INFO:tasks.workunit.client.0.smithi136.stdout:op 5077 completed, throughput=5MB/sec 2024-09-13T00:18:07.945 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-wPoqRW3Y-XGwPsU off=16335617 len=873080 2024-09-13T00:18:07.947 INFO:tasks.workunit.client.0.smithi136.stdout:op 5078 completed, throughput=5MB/sec 2024-09-13T00:18:08.948 INFO:tasks.workunit.client.0.smithi136.stdout: 974: throughput=5MB/sec pending data=0 2024-09-13T00:18:08.948 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-kFml7mpJqZPtQNy off=1497850 len=486450 2024-09-13T00:18:08.950 INFO:tasks.workunit.client.0.smithi136.stdout:op 5079 completed, throughput=5MB/sec 2024-09-13T00:18:08.951 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-S744G7CBLD6SPYS off=5147976 len=1057976 2024-09-13T00:18:08.954 INFO:tasks.workunit.client.0.smithi136.stdout:op 5080 completed, throughput=5MB/sec 2024-09-13T00:18:08.954 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-cz0ZxUixkjj2t5R off=4372381 len=1875237 2024-09-13T00:18:08.959 INFO:tasks.workunit.client.0.smithi136.stdout:op 5081 completed, throughput=5MB/sec 2024-09-13T00:18:08.959 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-h5gSn4L3r-GVqTE off=13032979 len=924102 2024-09-13T00:18:08.963 INFO:tasks.workunit.client.0.smithi136.stdout:op 5082 completed, throughput=5MB/sec 2024-09-13T00:18:08.963 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ZWUraeha5HqY6l9 off=1386211 len=310275 2024-09-13T00:18:08.964 INFO:tasks.workunit.client.0.smithi136.stdout:op 5083 completed, throughput=5MB/sec 2024-09-13T00:18:08.965 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ybAcWup3E35iWqc off=7228891 len=717362 2024-09-13T00:18:08.966 INFO:tasks.workunit.client.0.smithi136.stdout:op 5084 completed, throughput=5MB/sec 2024-09-13T00:18:09.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:18:08 smithi181 bash[67587]: cluster 2024-09-13T00:18:07.645154+0000 mgr.y (mgr.24661) 1230 : cluster [DBG] pgmap v1913: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 1.9 MiB/s rd, 3.6 MiB/s wr, 5 op/s 2024-09-13T00:18:09.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:18:08 smithi181 bash[67587]: audit 2024-09-13T00:18:08.799976+0000 mon.a (mon.0) 2256 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:18:09.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:18:08 smithi136 bash[75164]: cluster 2024-09-13T00:18:07.645154+0000 mgr.y (mgr.24661) 1230 : cluster [DBG] pgmap v1913: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 1.9 MiB/s rd, 3.6 MiB/s wr, 5 op/s 2024-09-13T00:18:09.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:18:08 smithi136 bash[75164]: audit 2024-09-13T00:18:08.799976+0000 mon.a (mon.0) 2256 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:18:09.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:18:08 smithi136 bash[76371]: cluster 2024-09-13T00:18:07.645154+0000 mgr.y (mgr.24661) 1230 : cluster [DBG] pgmap v1913: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 1.9 MiB/s rd, 3.6 MiB/s wr, 5 op/s 2024-09-13T00:18:09.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:18:08 smithi136 bash[76371]: audit 2024-09-13T00:18:08.799976+0000 mon.a (mon.0) 2256 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:18:09.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:18:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:18:09] "GET /metrics HTTP/1.1" 200 34141 "" "Prometheus/2.43.0" 2024-09-13T00:18:09.967 INFO:tasks.workunit.client.0.smithi136.stdout: 975: throughput=5MB/sec pending data=0 2024-09-13T00:18:09.967 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ZYnNgPrPGewEIRi off=4282313 len=953453 2024-09-13T00:18:09.976 INFO:tasks.workunit.client.0.smithi136.stdout:op 5085 completed, throughput=5MB/sec 2024-09-13T00:18:09.976 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-GL1bPp1RJRLwdcJ off=3302431 len=1986029 2024-09-13T00:18:09.990 INFO:tasks.workunit.client.0.smithi136.stdout:op 5086 completed, throughput=5MB/sec 2024-09-13T00:18:09.990 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-IxWM8Fqx4tQKZaS off=745726 len=1922742 2024-09-13T00:18:10.007 INFO:tasks.workunit.client.0.smithi136.stdout:op 5087 completed, throughput=5MB/sec 2024-09-13T00:18:10.007 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-RoOG6TkaVtnXyhM off=6007533 len=1521366 2024-09-13T00:18:10.010 INFO:tasks.workunit.client.0.smithi136.stdout:op 5088 completed, throughput=5MB/sec 2024-09-13T00:18:11.011 INFO:tasks.workunit.client.0.smithi136.stdout: 976: throughput=5MB/sec pending data=0 2024-09-13T00:18:11.011 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-a7y-AM6Mz515dTc off=3164474 len=1150660 2024-09-13T00:18:11.021 INFO:tasks.workunit.client.0.smithi136.stdout:op 5089 completed, throughput=5MB/sec 2024-09-13T00:18:11.022 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-xzKA-v7N1cUdo_0 off=10775874 len=1891261 2024-09-13T00:18:11.024 INFO:tasks.workunit.client.0.smithi136.stdout:op 5090 completed, throughput=5MB/sec 2024-09-13T00:18:11.024 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-FdSwmJ2xjMeToS_ off=2123160 len=2008604 2024-09-13T00:18:11.041 INFO:tasks.workunit.client.0.smithi136.stdout:op 5091 completed, throughput=5MB/sec 2024-09-13T00:18:11.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:18:10 smithi181 bash[67587]: cluster 2024-09-13T00:18:09.646045+0000 mgr.y (mgr.24661) 1231 : cluster [DBG] pgmap v1914: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.2 MiB/s wr, 4 op/s 2024-09-13T00:18:11.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:18:10 smithi136 bash[76371]: cluster 2024-09-13T00:18:09.646045+0000 mgr.y (mgr.24661) 1231 : cluster [DBG] pgmap v1914: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.2 MiB/s wr, 4 op/s 2024-09-13T00:18:11.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:18:10 smithi136 bash[75164]: cluster 2024-09-13T00:18:09.646045+0000 mgr.y (mgr.24661) 1231 : cluster [DBG] pgmap v1914: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.2 MiB/s wr, 4 op/s 2024-09-13T00:18:12.041 INFO:tasks.workunit.client.0.smithi136.stdout: 977: throughput=5MB/sec pending data=0 2024-09-13T00:18:12.041 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-zwajWCSqDKoEWFx off=4609337 len=1187822 2024-09-13T00:18:12.044 INFO:tasks.workunit.client.0.smithi136.stdout:op 5092 completed, throughput=5MB/sec 2024-09-13T00:18:12.045 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-7lRgcMRnuOBpxQO off=12331061 len=1210460 2024-09-13T00:18:12.053 INFO:tasks.workunit.client.0.smithi136.stdout:op 5093 completed, throughput=5MB/sec 2024-09-13T00:18:12.053 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-tTCNna-OLNAZjPz off=4185024 len=1956861 2024-09-13T00:18:12.060 INFO:tasks.workunit.client.0.smithi136.stdout:op 5094 completed, throughput=5MB/sec 2024-09-13T00:18:12.060 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-lfYndaDCwM8kK1l off=449383 len=743458 2024-09-13T00:18:12.062 INFO:tasks.workunit.client.0.smithi136.stdout:op 5095 completed, throughput=5MB/sec 2024-09-13T00:18:12.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:18:12 smithi181 bash[95505]: ts=2024-09-13T00:18:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:18:13.063 INFO:tasks.workunit.client.0.smithi136.stdout: 978: throughput=5MB/sec pending data=0 2024-09-13T00:18:13.063 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-jhPdheEDH9qCYCW off=1395453 len=586994 2024-09-13T00:18:13.069 INFO:tasks.workunit.client.0.smithi136.stdout:op 5096 completed, throughput=5MB/sec 2024-09-13T00:18:13.069 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-L5IkbHyHyOg66UT off=7716433 len=1910857 2024-09-13T00:18:13.074 INFO:tasks.workunit.client.0.smithi136.stdout:op 5097 completed, throughput=5MB/sec 2024-09-13T00:18:13.074 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-6j2LlohRchH6qEe off=152461 len=442992 2024-09-13T00:18:13.075 INFO:tasks.workunit.client.0.smithi136.stdout:op 5098 completed, throughput=5MB/sec 2024-09-13T00:18:13.075 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-krvFyMAR3_u-g4s off=536141 len=611836 2024-09-13T00:18:13.084 INFO:tasks.workunit.client.0.smithi136.stdout:op 5099 completed, throughput=5MB/sec 2024-09-13T00:18:13.084 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-RMdX2p6Mv2vMRAp off=13902328 len=1718927 2024-09-13T00:18:13.086 INFO:tasks.workunit.client.0.smithi136.stdout:op 5100 completed, throughput=5MB/sec 2024-09-13T00:18:13.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:18:12 smithi181 bash[67587]: cluster 2024-09-13T00:18:11.647132+0000 mgr.y (mgr.24661) 1232 : cluster [DBG] pgmap v1915: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 1.8 MiB/s rd, 2.9 MiB/s wr, 4 op/s 2024-09-13T00:18:13.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:18:12 smithi136 bash[76371]: cluster 2024-09-13T00:18:11.647132+0000 mgr.y (mgr.24661) 1232 : cluster [DBG] pgmap v1915: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 1.8 MiB/s rd, 2.9 MiB/s wr, 4 op/s 2024-09-13T00:18:13.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:18:12 smithi136 bash[75164]: cluster 2024-09-13T00:18:11.647132+0000 mgr.y (mgr.24661) 1232 : cluster [DBG] pgmap v1915: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 1.8 MiB/s rd, 2.9 MiB/s wr, 4 op/s 2024-09-13T00:18:14.086 INFO:tasks.workunit.client.0.smithi136.stdout: 979: throughput=5MB/sec pending data=0 2024-09-13T00:18:14.086 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-rlSr5S2mRpMY44k off=6589583 len=1916672 2024-09-13T00:18:14.103 INFO:tasks.workunit.client.0.smithi136.stdout:op 5101 completed, throughput=5MB/sec 2024-09-13T00:18:14.103 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-0-mgyLkkws3VZvH off=19009096 len=982153 2024-09-13T00:18:14.111 INFO:tasks.workunit.client.0.smithi136.stdout:op 5102 completed, throughput=5MB/sec 2024-09-13T00:18:14.111 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2V7Cis_f8Oq7dfr off=3165567 len=1222372 2024-09-13T00:18:14.114 INFO:tasks.workunit.client.0.smithi136.stdout:op 5103 completed, throughput=5MB/sec 2024-09-13T00:18:14.115 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Q5u0dknY1V47KA_ off=5282082 len=1190846 2024-09-13T00:18:14.125 INFO:tasks.workunit.client.0.smithi136.stdout:op 5104 completed, throughput=5MB/sec 2024-09-13T00:18:15.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:18:14 smithi181 bash[67587]: cluster 2024-09-13T00:18:13.648188+0000 mgr.y (mgr.24661) 1233 : cluster [DBG] pgmap v1916: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.2 MiB/s rd, 3.6 MiB/s wr, 5 op/s 2024-09-13T00:18:15.125 INFO:tasks.workunit.client.0.smithi136.stdout: 980: throughput=5MB/sec pending data=0 2024-09-13T00:18:15.125 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-0WlvmCe_D9Sn1fP off=1947879 len=1871150 2024-09-13T00:18:15.141 INFO:tasks.workunit.client.0.smithi136.stdout:op 5105 completed, throughput=5MB/sec 2024-09-13T00:18:15.141 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-r2rDhbH5VvgEuDs off=2301161 len=2048810 2024-09-13T00:18:15.146 INFO:tasks.workunit.client.0.smithi136.stdout:op 5106 completed, throughput=5MB/sec 2024-09-13T00:18:15.146 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-NAtsRHa1XF8m8IT off=7744700 len=355870 2024-09-13T00:18:15.152 INFO:tasks.workunit.client.0.smithi136.stdout:op 5107 completed, throughput=5MB/sec 2024-09-13T00:18:15.152 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-JHknwCQaopresgq off=2171365 len=284421 2024-09-13T00:18:15.157 INFO:tasks.workunit.client.0.smithi136.stdout:op 5108 completed, throughput=5MB/sec 2024-09-13T00:18:15.157 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-T_lg2nto650TYgs off=4117233 len=1301781 2024-09-13T00:18:15.167 INFO:tasks.workunit.client.0.smithi136.stdout:op 5109 completed, throughput=5MB/sec 2024-09-13T00:18:15.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:18:14 smithi136 bash[75164]: cluster 2024-09-13T00:18:13.648188+0000 mgr.y (mgr.24661) 1233 : cluster [DBG] pgmap v1916: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.2 MiB/s rd, 3.6 MiB/s wr, 5 op/s 2024-09-13T00:18:15.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:18:14 smithi136 bash[76371]: cluster 2024-09-13T00:18:13.648188+0000 mgr.y (mgr.24661) 1233 : cluster [DBG] pgmap v1916: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.2 MiB/s rd, 3.6 MiB/s wr, 5 op/s 2024-09-13T00:18:16.167 INFO:tasks.workunit.client.0.smithi136.stdout: 981: throughput=5MB/sec pending data=0 2024-09-13T00:18:16.167 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ZpP50SWhc_tM32N off=1347271 len=529474 2024-09-13T00:18:16.173 INFO:tasks.workunit.client.0.smithi136.stdout:op 5110 completed, throughput=5MB/sec 2024-09-13T00:18:16.174 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-gkinSTc0kvUc9vJ off=3089375 len=153895 2024-09-13T00:18:16.175 INFO:tasks.workunit.client.0.smithi136.stdout:op 5111 completed, throughput=5MB/sec 2024-09-13T00:18:16.175 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-LspD4eq8oUbwdGX off=7122822 len=694004 2024-09-13T00:18:16.181 INFO:tasks.workunit.client.0.smithi136.stdout:op 5112 completed, throughput=5MB/sec 2024-09-13T00:18:16.181 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-bMfIzzU9IY4BBBp off=13971264 len=1987602 2024-09-13T00:18:16.187 INFO:tasks.workunit.client.0.smithi136.stdout:op 5113 completed, throughput=5MB/sec 2024-09-13T00:18:16.188 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-H6m4e5_qMLMWH59 off=4197795 len=1923543 2024-09-13T00:18:16.192 INFO:tasks.workunit.client.0.smithi136.stdout:op 5114 completed, throughput=5MB/sec 2024-09-13T00:18:17.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:18:16 smithi181 bash[67587]: cluster 2024-09-13T00:18:15.649106+0000 mgr.y (mgr.24661) 1234 : cluster [DBG] pgmap v1917: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.1 MiB/s rd, 3.1 MiB/s wr, 4 op/s 2024-09-13T00:18:17.192 INFO:tasks.workunit.client.0.smithi136.stdout: 982: throughput=5MB/sec pending data=0 2024-09-13T00:18:17.192 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-C7CUc88Kbr0mp9X off=3379780 len=977711 2024-09-13T00:18:17.195 INFO:tasks.workunit.client.0.smithi136.stdout:op 5115 completed, throughput=5MB/sec 2024-09-13T00:18:17.195 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-hqUlKtJ4vv0mhnu off=7948453 len=945151 2024-09-13T00:18:17.198 INFO:tasks.workunit.client.0.smithi136.stdout:op 5116 completed, throughput=5MB/sec 2024-09-13T00:18:17.198 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-yBFPNsy5Kfpvxdh off=14175724 len=837641 2024-09-13T00:18:17.200 INFO:tasks.workunit.client.0.smithi136.stdout:op 5117 completed, throughput=5MB/sec 2024-09-13T00:18:17.200 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-RrarZD-PsnUIk_G off=2778542 len=294162 2024-09-13T00:18:17.201 INFO:tasks.workunit.client.0.smithi136.stdout:op 5118 completed, throughput=5MB/sec 2024-09-13T00:18:17.202 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-8TmpyMVaUWYlRLy off=10602977 len=807162 2024-09-13T00:18:17.203 INFO:tasks.workunit.client.0.smithi136.stdout:op 5119 completed, throughput=5MB/sec 2024-09-13T00:18:17.203 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ZvpPzOZWN-W-Gcr off=4737374 len=1128568 2024-09-13T00:18:17.213 INFO:tasks.workunit.client.0.smithi136.stdout:op 5120 completed, throughput=5MB/sec 2024-09-13T00:18:17.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:18:16 smithi136 bash[75164]: cluster 2024-09-13T00:18:15.649106+0000 mgr.y (mgr.24661) 1234 : cluster [DBG] pgmap v1917: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.1 MiB/s rd, 3.1 MiB/s wr, 4 op/s 2024-09-13T00:18:17.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:18:16 smithi136 bash[76371]: cluster 2024-09-13T00:18:15.649106+0000 mgr.y (mgr.24661) 1234 : cluster [DBG] pgmap v1917: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.1 MiB/s rd, 3.1 MiB/s wr, 4 op/s 2024-09-13T00:18:18.213 INFO:tasks.workunit.client.0.smithi136.stdout: 983: throughput=5MB/sec pending data=0 2024-09-13T00:18:18.213 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-FiUQ5PQ478m8Vcu off=3101852 len=329108 2024-09-13T00:18:18.219 INFO:tasks.workunit.client.0.smithi136.stdout:op 5121 completed, throughput=5MB/sec 2024-09-13T00:18:18.219 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-QL_-T1lkb5kvWCG off=13427940 len=1044147 2024-09-13T00:18:18.227 INFO:tasks.workunit.client.0.smithi136.stdout:op 5122 completed, throughput=5MB/sec 2024-09-13T00:18:18.227 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-04Sm3pYuBS4pAxf off=3979507 len=1150393 2024-09-13T00:18:18.236 INFO:tasks.workunit.client.0.smithi136.stdout:op 5123 completed, throughput=5MB/sec 2024-09-13T00:18:18.236 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-_jZbDlQHptNH-IF off=8724020 len=992710 2024-09-13T00:18:18.238 INFO:tasks.workunit.client.0.smithi136.stdout:op 5124 completed, throughput=5MB/sec 2024-09-13T00:18:18.238 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-tMhFTHfcH5-kZM2 off=349423 len=438402 2024-09-13T00:18:18.244 INFO:tasks.workunit.client.0.smithi136.stdout:op 5125 completed, throughput=5MB/sec 2024-09-13T00:18:18.244 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-rhknVCZIaivCahP off=920267 len=100923 2024-09-13T00:18:18.249 INFO:tasks.workunit.client.0.smithi136.stdout:op 5126 completed, throughput=5MB/sec 2024-09-13T00:18:18.249 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-85vliTRExkRgmoV off=1610931 len=2046755 2024-09-13T00:18:18.266 INFO:tasks.workunit.client.0.smithi136.stdout:op 5127 completed, throughput=5MB/sec 2024-09-13T00:18:19.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:18:18 smithi181 bash[67587]: cluster 2024-09-13T00:18:17.650226+0000 mgr.y (mgr.24661) 1235 : cluster [DBG] pgmap v1918: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.5 MiB/s rd, 3.1 MiB/s wr, 5 op/s 2024-09-13T00:18:19.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:18:18 smithi136 bash[75164]: cluster 2024-09-13T00:18:17.650226+0000 mgr.y (mgr.24661) 1235 : cluster [DBG] pgmap v1918: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.5 MiB/s rd, 3.1 MiB/s wr, 5 op/s 2024-09-13T00:18:19.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:18:18 smithi136 bash[76371]: cluster 2024-09-13T00:18:17.650226+0000 mgr.y (mgr.24661) 1235 : cluster [DBG] pgmap v1918: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.5 MiB/s rd, 3.1 MiB/s wr, 5 op/s 2024-09-13T00:18:19.266 INFO:tasks.workunit.client.0.smithi136.stdout: 984: throughput=5MB/sec pending data=0 2024-09-13T00:18:19.266 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-bG6rVSYoyUqiOWK off=747082 len=1270555 2024-09-13T00:18:19.276 INFO:tasks.workunit.client.0.smithi136.stdout:op 5128 completed, throughput=5MB/sec 2024-09-13T00:18:19.276 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-MS9YHy7i3kTbwah off=259790 len=1255939 2024-09-13T00:18:19.281 INFO:tasks.workunit.client.0.smithi136.stdout:op 5129 completed, throughput=5MB/sec 2024-09-13T00:18:19.281 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-YGzEEJZBZTep4Wk off=16921168 len=329420 2024-09-13T00:18:19.287 INFO:tasks.workunit.client.0.smithi136.stdout:op 5130 completed, throughput=5MB/sec 2024-09-13T00:18:19.287 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-cJkGw7V_iQwbJOR off=488792 len=1220505 2024-09-13T00:18:19.290 INFO:tasks.workunit.client.0.smithi136.stdout:op 5131 completed, throughput=5MB/sec 2024-09-13T00:18:19.291 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-XoPIEpxVyTSmnUe off=1506600 len=1473057 2024-09-13T00:18:19.293 INFO:tasks.workunit.client.0.smithi136.stdout:op 5132 completed, throughput=5MB/sec 2024-09-13T00:18:19.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:18:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:18:19] "GET /metrics HTTP/1.1" 200 34141 "" "Prometheus/2.43.0" 2024-09-13T00:18:20.293 INFO:tasks.workunit.client.0.smithi136.stdout: 985: throughput=5MB/sec pending data=0 2024-09-13T00:18:20.293 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-n9W-2-lPcBd4KDk off=465120 len=1781238 2024-09-13T00:18:20.309 INFO:tasks.workunit.client.0.smithi136.stdout:op 5133 completed, throughput=5MB/sec 2024-09-13T00:18:20.309 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-SEVeFYNKCEx-2fk off=1201930 len=1227484 2024-09-13T00:18:20.312 INFO:tasks.workunit.client.0.smithi136.stdout:op 5134 completed, throughput=5MB/sec 2024-09-13T00:18:20.312 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ExZfvmS6OTgpMfd off=475274 len=89620 2024-09-13T00:18:20.317 INFO:tasks.workunit.client.0.smithi136.stdout:op 5135 completed, throughput=5MB/sec 2024-09-13T00:18:20.318 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-EKgx0MvxN7iuMKo off=8708 len=1754786 2024-09-13T00:18:20.327 INFO:tasks.workunit.client.0.smithi136.stdout:op 5136 completed, throughput=5MB/sec 2024-09-13T00:18:21.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:18:20 smithi136 bash[75164]: cluster 2024-09-13T00:18:19.651079+0000 mgr.y (mgr.24661) 1236 : cluster [DBG] pgmap v1919: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:18:21.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:18:20 smithi136 bash[76371]: cluster 2024-09-13T00:18:19.651079+0000 mgr.y (mgr.24661) 1236 : cluster [DBG] pgmap v1919: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:18:21.328 INFO:tasks.workunit.client.0.smithi136.stdout: 986: throughput=5MB/sec pending data=0 2024-09-13T00:18:21.328 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-26GeHpAOmdnPV4B off=5075026 len=745893 2024-09-13T00:18:21.335 INFO:tasks.workunit.client.0.smithi136.stdout:op 5137 completed, throughput=5MB/sec 2024-09-13T00:18:21.335 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-0eqPMDQMCsDnZuv off=4417948 len=1352284 2024-09-13T00:18:21.340 INFO:tasks.workunit.client.0.smithi136.stdout:op 5138 completed, throughput=5MB/sec 2024-09-13T00:18:21.341 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-nRXfmlEaoIofEaX off=1123340 len=314489 2024-09-13T00:18:21.344 INFO:tasks.workunit.client.0.smithi136.stdout:op 5139 completed, throughput=5MB/sec 2024-09-13T00:18:21.344 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-VmVQi9yB3glCjbg off=8301150 len=710067 2024-09-13T00:18:21.346 INFO:tasks.workunit.client.0.smithi136.stdout:op 5140 completed, throughput=5MB/sec 2024-09-13T00:18:21.346 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-kJry5wPjc5GsPyv off=624629 len=1511223 2024-09-13T00:18:21.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:18:20 smithi181 bash[67587]: cluster 2024-09-13T00:18:19.651079+0000 mgr.y (mgr.24661) 1236 : cluster [DBG] pgmap v1919: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:18:21.357 INFO:tasks.workunit.client.0.smithi136.stdout:op 5141 completed, throughput=5MB/sec 2024-09-13T00:18:21.357 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-uEzkzP08HI45unF off=2856874 len=621149 2024-09-13T00:18:21.363 INFO:tasks.workunit.client.0.smithi136.stdout:op 5142 completed, throughput=5MB/sec 2024-09-13T00:18:22.364 INFO:tasks.workunit.client.0.smithi136.stdout: 987: throughput=5MB/sec pending data=0 2024-09-13T00:18:22.364 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj--s6IgLLwd7TM7tL off=3697500 len=1366521 2024-09-13T00:18:22.367 INFO:tasks.workunit.client.0.smithi136.stdout:op 5143 completed, throughput=5MB/sec 2024-09-13T00:18:22.368 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-KEFX51rMAu4CO-e off=1976751 len=19516 2024-09-13T00:18:22.371 INFO:tasks.workunit.client.0.smithi136.stdout:op 5144 completed, throughput=5MB/sec 2024-09-13T00:18:22.371 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-xGqMVntHF5dRru6 off=3120816 len=1143891 2024-09-13T00:18:22.374 INFO:tasks.workunit.client.0.smithi136.stdout:op 5145 completed, throughput=5MB/sec 2024-09-13T00:18:22.375 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-RrarZD-PsnUIk_G off=6232527 len=253277 2024-09-13T00:18:22.376 INFO:tasks.workunit.client.0.smithi136.stdout:op 5146 completed, throughput=5MB/sec 2024-09-13T00:18:22.376 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-LcvBEIjLs9BhP5a off=444446 len=548817 2024-09-13T00:18:22.382 INFO:tasks.workunit.client.0.smithi136.stdout:op 5147 completed, throughput=5MB/sec 2024-09-13T00:18:22.382 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-t9oRPA3m75XiKcO off=157963 len=861502 2024-09-13T00:18:22.386 INFO:tasks.workunit.client.0.smithi136.stdout:op 5148 completed, throughput=5MB/sec 2024-09-13T00:18:22.386 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-CMceovYAbWqdLh7 off=2937559 len=466100 2024-09-13T00:18:22.388 INFO:tasks.workunit.client.0.smithi136.stdout:op 5149 completed, throughput=5MB/sec 2024-09-13T00:18:22.388 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-SJFV68lZxpOZ20X off=15586380 len=1394128 2024-09-13T00:18:22.398 INFO:tasks.workunit.client.0.smithi136.stdout:op 5150 completed, throughput=5MB/sec 2024-09-13T00:18:22.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:18:22 smithi181 bash[95505]: ts=2024-09-13T00:18:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:18:23.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:18:22 smithi136 bash[75164]: cluster 2024-09-13T00:18:21.652050+0000 mgr.y (mgr.24661) 1237 : cluster [DBG] pgmap v1920: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:18:23.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:18:22 smithi136 bash[76371]: cluster 2024-09-13T00:18:21.652050+0000 mgr.y (mgr.24661) 1237 : cluster [DBG] pgmap v1920: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:18:23.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:18:22 smithi181 bash[67587]: cluster 2024-09-13T00:18:21.652050+0000 mgr.y (mgr.24661) 1237 : cluster [DBG] pgmap v1920: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:18:23.399 INFO:tasks.workunit.client.0.smithi136.stdout: 988: throughput=5MB/sec pending data=0 2024-09-13T00:18:23.399 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ctXDl9ppaCSnUE2 off=11059241 len=1239391 2024-09-13T00:18:23.402 INFO:tasks.workunit.client.0.smithi136.stdout:op 5151 completed, throughput=5MB/sec 2024-09-13T00:18:23.402 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-mkSo8GcVM68YXCQ off=7080221 len=1408524 2024-09-13T00:18:23.406 INFO:tasks.workunit.client.0.smithi136.stdout:op 5152 completed, throughput=5MB/sec 2024-09-13T00:18:23.406 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-piqkPk-H0RgXnAK off=731470 len=1203907 2024-09-13T00:18:23.410 INFO:tasks.workunit.client.0.smithi136.stdout:op 5153 completed, throughput=5MB/sec 2024-09-13T00:18:23.410 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-q0KGBYdpqRt67Ur off=39484 len=805098 2024-09-13T00:18:23.413 INFO:tasks.workunit.client.0.smithi136.stdout:op 5154 completed, throughput=5MB/sec 2024-09-13T00:18:24.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:18:23 smithi136 bash[76371]: audit 2024-09-13T00:18:23.800148+0000 mon.a (mon.0) 2257 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:18:24.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:18:23 smithi136 bash[75164]: audit 2024-09-13T00:18:23.800148+0000 mon.a (mon.0) 2257 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:18:24.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:18:23 smithi181 bash[67587]: audit 2024-09-13T00:18:23.800148+0000 mon.a (mon.0) 2257 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:18:24.413 INFO:tasks.workunit.client.0.smithi136.stdout: 989: throughput=5MB/sec pending data=0 2024-09-13T00:18:24.414 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-NjzLDlN80rFbaMY off=13925057 len=1440330 2024-09-13T00:18:24.426 INFO:tasks.workunit.client.0.smithi136.stdout:op 5155 completed, throughput=5MB/sec 2024-09-13T00:18:24.427 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-qDly4HO6u6WI0Uo off=9067491 len=1568926 2024-09-13T00:18:24.430 INFO:tasks.workunit.client.0.smithi136.stdout:op 5156 completed, throughput=5MB/sec 2024-09-13T00:18:24.430 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-dnkiAeNAa9EctH2 off=9823173 len=174203 2024-09-13T00:18:24.432 INFO:tasks.workunit.client.0.smithi136.stdout:op 5157 completed, throughput=5MB/sec 2024-09-13T00:18:24.432 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-lKPqEndmX-fW9vs off=7781201 len=1541212 2024-09-13T00:18:24.436 INFO:tasks.workunit.client.0.smithi136.stdout:op 5158 completed, throughput=5MB/sec 2024-09-13T00:18:24.436 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-_qd2jiZ_Zdk_NTq off=5960615 len=792786 2024-09-13T00:18:24.438 INFO:tasks.workunit.client.0.smithi136.stdout:op 5159 completed, throughput=5MB/sec 2024-09-13T00:18:25.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:18:24 smithi136 bash[75164]: cluster 2024-09-13T00:18:23.653122+0000 mgr.y (mgr.24661) 1238 : cluster [DBG] pgmap v1921: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.8 MiB/s rd, 3.1 MiB/s wr, 5 op/s 2024-09-13T00:18:25.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:18:24 smithi136 bash[76371]: cluster 2024-09-13T00:18:23.653122+0000 mgr.y (mgr.24661) 1238 : cluster [DBG] pgmap v1921: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.8 MiB/s rd, 3.1 MiB/s wr, 5 op/s 2024-09-13T00:18:25.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:18:24 smithi181 bash[67587]: cluster 2024-09-13T00:18:23.653122+0000 mgr.y (mgr.24661) 1238 : cluster [DBG] pgmap v1921: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.8 MiB/s rd, 3.1 MiB/s wr, 5 op/s 2024-09-13T00:18:25.438 INFO:tasks.workunit.client.0.smithi136.stdout: 990: throughput=5MB/sec pending data=0 2024-09-13T00:18:25.438 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Nj119F4vWA5hggS off=13952202 len=234807 2024-09-13T00:18:25.444 INFO:tasks.workunit.client.0.smithi136.stdout:op 5160 completed, throughput=5MB/sec 2024-09-13T00:18:25.444 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-MhhK1gesdlYKMKz off=1560328 len=489603 2024-09-13T00:18:25.446 INFO:tasks.workunit.client.0.smithi136.stdout:op 5161 completed, throughput=5MB/sec 2024-09-13T00:18:25.446 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-AH7065nmKvmkHWH off=2704969 len=187459 2024-09-13T00:18:25.447 INFO:tasks.workunit.client.0.smithi136.stdout:op 5162 completed, throughput=5MB/sec 2024-09-13T00:18:25.447 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-wGeirLmFmyHTB3m off=4614288 len=700004 2024-09-13T00:18:25.454 INFO:tasks.workunit.client.0.smithi136.stdout:op 5163 completed, throughput=5MB/sec 2024-09-13T00:18:25.454 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-7go0K5hB9nM3OUO off=4058679 len=1085274 2024-09-13T00:18:25.464 INFO:tasks.workunit.client.0.smithi136.stdout:op 5164 completed, throughput=5MB/sec 2024-09-13T00:18:25.464 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-xFcb8rMe9U_T0vv off=6312998 len=1250880 2024-09-13T00:18:25.467 INFO:tasks.workunit.client.0.smithi136.stdout:op 5165 completed, throughput=5MB/sec 2024-09-13T00:18:25.467 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-qGcXdZy6IhL9JdN off=13520665 len=754280 2024-09-13T00:18:25.469 INFO:tasks.workunit.client.0.smithi136.stdout:op 5166 completed, throughput=5MB/sec 2024-09-13T00:18:25.469 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-0IEuV-Vpn-gvKvI off=3458040 len=2007616 2024-09-13T00:18:25.486 INFO:tasks.workunit.client.0.smithi136.stdout:op 5167 completed, throughput=5MB/sec 2024-09-13T00:18:26.487 INFO:tasks.workunit.client.0.smithi136.stdout: 991: throughput=5MB/sec pending data=0 2024-09-13T00:18:26.487 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-jYlIXLhsIIU7DEN off=1490454 len=1898347 2024-09-13T00:18:26.491 INFO:tasks.workunit.client.0.smithi136.stdout:op 5168 completed, throughput=5MB/sec 2024-09-13T00:18:26.491 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-OBsO45Qlf9vvGyr off=14557549 len=326767 2024-09-13T00:18:26.493 INFO:tasks.workunit.client.0.smithi136.stdout:op 5169 completed, throughput=5MB/sec 2024-09-13T00:18:26.494 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-IZrJ49A7f0c5TFm off=20032824 len=195421 2024-09-13T00:18:26.494 INFO:tasks.workunit.client.0.smithi136.stdout:op 5170 completed, throughput=5MB/sec 2024-09-13T00:18:26.494 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-9H_wn10TgC_JQ8q off=9282435 len=145633 2024-09-13T00:18:26.496 INFO:tasks.workunit.client.0.smithi136.stdout:op 5171 completed, throughput=5MB/sec 2024-09-13T00:18:26.496 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-QR-QU_jOZDCCPzA off=3409254 len=36012 2024-09-13T00:18:26.497 INFO:tasks.workunit.client.0.smithi136.stdout:op 5172 completed, throughput=5MB/sec 2024-09-13T00:18:26.497 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Ss9IMZ9bi5jYDMZ off=5892229 len=1743725 2024-09-13T00:18:26.503 INFO:tasks.workunit.client.0.smithi136.stdout:op 5173 completed, throughput=5MB/sec 2024-09-13T00:18:27.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:18:26 smithi136 bash[75164]: cluster 2024-09-13T00:18:25.654031+0000 mgr.y (mgr.24661) 1239 : cluster [DBG] pgmap v1922: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:18:27.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:18:26 smithi136 bash[76371]: cluster 2024-09-13T00:18:25.654031+0000 mgr.y (mgr.24661) 1239 : cluster [DBG] pgmap v1922: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:18:27.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:18:26 smithi181 bash[67587]: cluster 2024-09-13T00:18:25.654031+0000 mgr.y (mgr.24661) 1239 : cluster [DBG] pgmap v1922: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.3 MiB/s wr, 4 op/s 2024-09-13T00:18:27.504 INFO:tasks.workunit.client.0.smithi136.stdout: 992: throughput=5MB/sec pending data=0 2024-09-13T00:18:27.504 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-C30Jkr48ltKrJgi off=725189 len=615946 2024-09-13T00:18:27.510 INFO:tasks.workunit.client.0.smithi136.stdout:op 5174 completed, throughput=5MB/sec 2024-09-13T00:18:27.510 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-bS2YsMU_qKz5oEQ off=6459360 len=624266 2024-09-13T00:18:27.517 INFO:tasks.workunit.client.0.smithi136.stdout:op 5175 completed, throughput=5MB/sec 2024-09-13T00:18:27.517 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ToqGPxeYv9yLT9- off=10275497 len=863439 2024-09-13T00:18:27.519 INFO:tasks.workunit.client.0.smithi136.stdout:op 5176 completed, throughput=5MB/sec 2024-09-13T00:18:27.519 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Pbz9aO3Ot8t1v5Y off=17268264 len=129285 2024-09-13T00:18:27.524 INFO:tasks.workunit.client.0.smithi136.stdout:op 5177 completed, throughput=5MB/sec 2024-09-13T00:18:27.524 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-H9uofNuK07B5g2Q off=800375 len=1413033 2024-09-13T00:18:27.527 INFO:tasks.workunit.client.0.smithi136.stdout:op 5178 completed, throughput=5MB/sec 2024-09-13T00:18:27.528 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-0xZ8fdfCJXAYph3 off=10641794 len=749877 2024-09-13T00:18:27.535 INFO:tasks.workunit.client.0.smithi136.stdout:op 5179 completed, throughput=5MB/sec 2024-09-13T00:18:27.535 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-VHItPTGFIzrbAd- off=1680701 len=350068 2024-09-13T00:18:27.542 INFO:tasks.workunit.client.0.smithi136.stdout:op 5180 completed, throughput=5MB/sec 2024-09-13T00:18:27.542 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-LJ3v0hvHylsV6wY off=290800 len=65561 2024-09-13T00:18:27.543 INFO:tasks.workunit.client.0.smithi136.stdout:op 5181 completed, throughput=5MB/sec 2024-09-13T00:18:27.543 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-A8_cHF7E00ycYCp off=840427 len=83641 2024-09-13T00:18:27.548 INFO:tasks.workunit.client.0.smithi136.stdout:op 5182 completed, throughput=5MB/sec 2024-09-13T00:18:28.548 INFO:tasks.workunit.client.0.smithi136.stdout: 993: throughput=5MB/sec pending data=0 2024-09-13T00:18:28.548 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-QRuo6b7ro8ZER6M off=12695471 len=384940 2024-09-13T00:18:28.554 INFO:tasks.workunit.client.0.smithi136.stdout:op 5183 completed, throughput=5MB/sec 2024-09-13T00:18:28.554 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-BR8Kev8Qq-6H0wH off=1414652 len=1335467 2024-09-13T00:18:28.564 INFO:tasks.workunit.client.0.smithi136.stdout:op 5184 completed, throughput=5MB/sec 2024-09-13T00:18:28.564 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-QXnXOvP171KQJjr off=8205050 len=824261 2024-09-13T00:18:28.568 INFO:tasks.workunit.client.0.smithi136.stdout:op 5185 completed, throughput=5MB/sec 2024-09-13T00:18:28.568 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ZbwSuCA-hLk9Vn_ off=1372071 len=94227 2024-09-13T00:18:28.570 INFO:tasks.workunit.client.0.smithi136.stdout:op 5186 completed, throughput=5MB/sec 2024-09-13T00:18:28.570 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-HFW8eCyEYIcjkgx off=9076748 len=455463 2024-09-13T00:18:28.576 INFO:tasks.workunit.client.0.smithi136.stdout:op 5187 completed, throughput=5MB/sec 2024-09-13T00:18:28.576 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-JlP65l45jEFebyE off=3021970 len=1463467 2024-09-13T00:18:28.579 INFO:tasks.workunit.client.0.smithi136.stdout:op 5188 completed, throughput=5MB/sec 2024-09-13T00:18:28.579 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-0ucvlNE_sXkkl96 off=4681344 len=675104 2024-09-13T00:18:28.587 INFO:tasks.workunit.client.0.smithi136.stdout:op 5189 completed, throughput=5MB/sec 2024-09-13T00:18:28.587 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-l61HeY4DyH0sMkR off=910376 len=1392086 2024-09-13T00:18:28.598 INFO:tasks.workunit.client.0.smithi136.stdout:op 5190 completed, throughput=5MB/sec 2024-09-13T00:18:29.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:18:28 smithi136 bash[75164]: cluster 2024-09-13T00:18:27.655071+0000 mgr.y (mgr.24661) 1240 : cluster [DBG] pgmap v1923: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:18:29.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:18:28 smithi136 bash[76371]: cluster 2024-09-13T00:18:27.655071+0000 mgr.y (mgr.24661) 1240 : cluster [DBG] pgmap v1923: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:18:29.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:18:28 smithi181 bash[67587]: cluster 2024-09-13T00:18:27.655071+0000 mgr.y (mgr.24661) 1240 : cluster [DBG] pgmap v1923: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:18:29.598 INFO:tasks.workunit.client.0.smithi136.stdout: 994: throughput=5MB/sec pending data=0 2024-09-13T00:18:29.598 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ywX8VmD_vrppt54 off=3408064 len=1067038 2024-09-13T00:18:29.607 INFO:tasks.workunit.client.0.smithi136.stdout:op 5191 completed, throughput=5MB/sec 2024-09-13T00:18:29.607 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-AxA4BRPMkp5C4uM off=11873460 len=541521 2024-09-13T00:18:29.609 INFO:tasks.workunit.client.0.smithi136.stdout:op 5192 completed, throughput=5MB/sec 2024-09-13T00:18:29.609 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-RVSpobPtxPJn_x2 off=16822385 len=1437434 2024-09-13T00:18:29.613 INFO:tasks.workunit.client.0.smithi136.stdout:op 5193 completed, throughput=5MB/sec 2024-09-13T00:18:29.613 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-_vE3VRH5EN3OJUc off=3880503 len=232432 2024-09-13T00:18:29.614 INFO:tasks.workunit.client.0.smithi136.stdout:op 5194 completed, throughput=5MB/sec 2024-09-13T00:18:29.614 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_JJ6yyKZYK18fcw off=4740587 len=373810 2024-09-13T00:18:29.620 INFO:tasks.workunit.client.0.smithi136.stdout:op 5195 completed, throughput=5MB/sec 2024-09-13T00:18:29.620 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-mp4qxo5gdz7T4C1 off=15384522 len=2094423 2024-09-13T00:18:29.624 INFO:tasks.workunit.client.0.smithi136.stdout:op 5196 completed, throughput=5MB/sec 2024-09-13T00:18:29.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:18:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:18:29] "GET /metrics HTTP/1.1" 200 34140 "" "Prometheus/2.43.0" 2024-09-13T00:18:30.625 INFO:tasks.workunit.client.0.smithi136.stdout: 995: throughput=5MB/sec pending data=0 2024-09-13T00:18:30.625 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-inkMeAq38uTU_hh off=7988733 len=873653 2024-09-13T00:18:30.627 INFO:tasks.workunit.client.0.smithi136.stdout:op 5197 completed, throughput=5MB/sec 2024-09-13T00:18:30.627 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-C30Jkr48ltKrJgi off=3160441 len=758740 2024-09-13T00:18:30.630 INFO:tasks.workunit.client.0.smithi136.stdout:op 5198 completed, throughput=5MB/sec 2024-09-13T00:18:30.630 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj--xl59SjXrZmRISD off=11243367 len=781480 2024-09-13T00:18:30.632 INFO:tasks.workunit.client.0.smithi136.stdout:op 5199 completed, throughput=5MB/sec 2024-09-13T00:18:30.632 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-24ojWzqw81L_x7M off=10287031 len=609119 2024-09-13T00:18:30.640 INFO:tasks.workunit.client.0.smithi136.stdout:op 5200 completed, throughput=5MB/sec 2024-09-13T00:18:30.640 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-9uwJv07hicNOzbh off=15208875 len=965024 2024-09-13T00:18:30.649 INFO:tasks.workunit.client.0.smithi136.stdout:op 5201 completed, throughput=5MB/sec 2024-09-13T00:18:31.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:18:30 smithi136 bash[76371]: cluster 2024-09-13T00:18:29.656006+0000 mgr.y (mgr.24661) 1241 : cluster [DBG] pgmap v1924: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.0 MiB/s wr, 5 op/s 2024-09-13T00:18:31.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:18:30 smithi136 bash[75164]: cluster 2024-09-13T00:18:29.656006+0000 mgr.y (mgr.24661) 1241 : cluster [DBG] pgmap v1924: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.0 MiB/s wr, 5 op/s 2024-09-13T00:18:31.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:18:30 smithi181 bash[67587]: cluster 2024-09-13T00:18:29.656006+0000 mgr.y (mgr.24661) 1241 : cluster [DBG] pgmap v1924: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.0 MiB/s wr, 5 op/s 2024-09-13T00:18:31.649 INFO:tasks.workunit.client.0.smithi136.stdout: 996: throughput=5MB/sec pending data=0 2024-09-13T00:18:31.649 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-NgpsuChBafhEw5y off=340559 len=1412281 2024-09-13T00:18:31.652 INFO:tasks.workunit.client.0.smithi136.stdout:op 5202 completed, throughput=5MB/sec 2024-09-13T00:18:31.653 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Mj6IGAgnYrwZSs7 off=2994422 len=1018266 2024-09-13T00:18:31.662 INFO:tasks.workunit.client.0.smithi136.stdout:op 5203 completed, throughput=5MB/sec 2024-09-13T00:18:31.663 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-KXYdHX5JE9eEkNd off=4939775 len=1717301 2024-09-13T00:18:31.666 INFO:tasks.workunit.client.0.smithi136.stdout:op 5204 completed, throughput=5MB/sec 2024-09-13T00:18:31.667 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ftDNtJCV0VHFe9o off=3487953 len=473257 2024-09-13T00:18:31.673 INFO:tasks.workunit.client.0.smithi136.stdout:op 5205 completed, throughput=5MB/sec 2024-09-13T00:18:31.673 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-oXwm-UWivR6z4DY off=1304437 len=798864 2024-09-13T00:18:31.681 INFO:tasks.workunit.client.0.smithi136.stdout:op 5206 completed, throughput=5MB/sec 2024-09-13T00:18:32.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:18:32 smithi181 bash[95505]: ts=2024-09-13T00:18:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:18:32.682 INFO:tasks.workunit.client.0.smithi136.stdout: 997: throughput=5MB/sec pending data=0 2024-09-13T00:18:32.682 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-BBQjGWDJ9S1alKj off=3175631 len=291702 2024-09-13T00:18:32.687 INFO:tasks.workunit.client.0.smithi136.stdout:op 5207 completed, throughput=5MB/sec 2024-09-13T00:18:32.687 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-YnuJ2rhBPqOc17y off=13993140 len=1835228 2024-09-13T00:18:32.692 INFO:tasks.workunit.client.0.smithi136.stdout:op 5208 completed, throughput=5MB/sec 2024-09-13T00:18:32.692 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-zEQeWH5-amY9L67 off=315257 len=1268019 2024-09-13T00:18:32.702 INFO:tasks.workunit.client.0.smithi136.stdout:op 5209 completed, throughput=5MB/sec 2024-09-13T00:18:32.702 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-viTdNcGpTHNzcgq off=9764400 len=1303677 2024-09-13T00:18:32.715 INFO:tasks.workunit.client.0.smithi136.stdout:op 5210 completed, throughput=5MB/sec 2024-09-13T00:18:32.715 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-m_Ba4wVsGcRTn_Z off=16924130 len=1792551 2024-09-13T00:18:32.718 INFO:tasks.workunit.client.0.smithi136.stdout:op 5211 completed, throughput=5MB/sec 2024-09-13T00:18:33.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:18:32 smithi136 bash[76371]: cluster 2024-09-13T00:18:31.656965+0000 mgr.y (mgr.24661) 1242 : cluster [DBG] pgmap v1925: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.6 MiB/s wr, 5 op/s 2024-09-13T00:18:33.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:18:32 smithi136 bash[75164]: cluster 2024-09-13T00:18:31.656965+0000 mgr.y (mgr.24661) 1242 : cluster [DBG] pgmap v1925: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.6 MiB/s wr, 5 op/s 2024-09-13T00:18:33.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:18:32 smithi181 bash[67587]: cluster 2024-09-13T00:18:31.656965+0000 mgr.y (mgr.24661) 1242 : cluster [DBG] pgmap v1925: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.6 MiB/s wr, 5 op/s 2024-09-13T00:18:33.718 INFO:tasks.workunit.client.0.smithi136.stdout: 998: throughput=5MB/sec pending data=0 2024-09-13T00:18:33.718 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-GpfVCnKlAR7-lBT off=11252939 len=836906 2024-09-13T00:18:33.726 INFO:tasks.workunit.client.0.smithi136.stdout:op 5212 completed, throughput=5MB/sec 2024-09-13T00:18:33.726 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-tTNRTZ42Ejgor1e off=10509906 len=1685728 2024-09-13T00:18:33.731 INFO:tasks.workunit.client.0.smithi136.stdout:op 5213 completed, throughput=5MB/sec 2024-09-13T00:18:33.731 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-GglVwhBJGNQfGjv off=12611142 len=944081 2024-09-13T00:18:33.734 INFO:tasks.workunit.client.0.smithi136.stdout:op 5214 completed, throughput=5MB/sec 2024-09-13T00:18:33.734 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2i1KhaA9PghDbD_ off=12786175 len=121169 2024-09-13T00:18:33.735 INFO:tasks.workunit.client.0.smithi136.stdout:op 5215 completed, throughput=5MB/sec 2024-09-13T00:18:33.735 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-HsVA7d4Vu02px8_ off=2468417 len=1824270 2024-09-13T00:18:33.739 INFO:tasks.workunit.client.0.smithi136.stdout:op 5216 completed, throughput=5MB/sec 2024-09-13T00:18:34.739 INFO:tasks.workunit.client.0.smithi136.stdout: 999: throughput=5MB/sec pending data=0 2024-09-13T00:18:34.740 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-8FOo3Y5P4LYwHBV off=3598132 len=442541 2024-09-13T00:18:34.741 INFO:tasks.workunit.client.0.smithi136.stdout:op 5217 completed, throughput=5MB/sec 2024-09-13T00:18:34.741 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-DmkQfJwYtfshctz off=4181453 len=254478 2024-09-13T00:18:34.743 INFO:tasks.workunit.client.0.smithi136.stdout:op 5218 completed, throughput=5MB/sec 2024-09-13T00:18:34.743 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj--r9o0BpbEZ2fuOl off=4258309 len=1946771 2024-09-13T00:18:34.747 INFO:tasks.workunit.client.0.smithi136.stdout:op 5219 completed, throughput=5MB/sec 2024-09-13T00:18:34.747 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-vPuVVMtS1zWXEB- off=11627194 len=591912 2024-09-13T00:18:34.753 INFO:tasks.workunit.client.0.smithi136.stdout:op 5220 completed, throughput=5MB/sec 2024-09-13T00:18:34.753 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-v7Dj2ua_jIu8vL2 off=2259661 len=522711 2024-09-13T00:18:34.759 INFO:tasks.workunit.client.0.smithi136.stdout:op 5221 completed, throughput=5MB/sec 2024-09-13T00:18:34.760 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-dMi87yvuSm8wqWj off=14334101 len=974125 2024-09-13T00:18:34.763 INFO:tasks.workunit.client.0.smithi136.stdout:op 5222 completed, throughput=5MB/sec 2024-09-13T00:18:35.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:18:34 smithi136 bash[75164]: cluster 2024-09-13T00:18:33.657992+0000 mgr.y (mgr.24661) 1243 : cluster [DBG] pgmap v1926: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 3.3 MiB/s rd, 2.3 MiB/s wr, 6 op/s 2024-09-13T00:18:35.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:18:34 smithi136 bash[76371]: cluster 2024-09-13T00:18:33.657992+0000 mgr.y (mgr.24661) 1243 : cluster [DBG] pgmap v1926: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 3.3 MiB/s rd, 2.3 MiB/s wr, 6 op/s 2024-09-13T00:18:35.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:18:34 smithi181 bash[67587]: cluster 2024-09-13T00:18:33.657992+0000 mgr.y (mgr.24661) 1243 : cluster [DBG] pgmap v1926: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 3.3 MiB/s rd, 2.3 MiB/s wr, 6 op/s 2024-09-13T00:18:35.764 INFO:tasks.workunit.client.0.smithi136.stdout: 1000: throughput=5MB/sec pending data=0 2024-09-13T00:18:35.764 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-TfyTFAFaEdN6bQP off=10466257 len=1923562 2024-09-13T00:18:35.769 INFO:tasks.workunit.client.0.smithi136.stdout:op 5223 completed, throughput=5MB/sec 2024-09-13T00:18:35.769 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-sxQVbKOXQ16w8hC off=14068431 len=1372064 2024-09-13T00:18:35.780 INFO:tasks.workunit.client.0.smithi136.stdout:op 5224 completed, throughput=5MB/sec 2024-09-13T00:18:35.780 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-VXWyvXT2yck8dNv off=12043542 len=364437 2024-09-13T00:18:35.787 INFO:tasks.workunit.client.0.smithi136.stdout:op 5225 completed, throughput=5MB/sec 2024-09-13T00:18:35.787 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-JXLbZoKagPMwHZr off=2809827 len=1159532 2024-09-13T00:18:35.790 INFO:tasks.workunit.client.0.smithi136.stdout:op 5226 completed, throughput=5MB/sec 2024-09-13T00:18:36.791 INFO:tasks.workunit.client.0.smithi136.stdout: 1001: throughput=5MB/sec pending data=0 2024-09-13T00:18:36.791 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-1TZ7unCgcizFGTH off=695576 len=970967 2024-09-13T00:18:36.799 INFO:tasks.workunit.client.0.smithi136.stdout:op 5227 completed, throughput=5MB/sec 2024-09-13T00:18:36.799 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Un2g0Bx0EUo8NZN off=9098472 len=911846 2024-09-13T00:18:36.802 INFO:tasks.workunit.client.0.smithi136.stdout:op 5228 completed, throughput=5MB/sec 2024-09-13T00:18:36.802 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-KHsR2iDF2-5DZXy off=32036 len=103592 2024-09-13T00:18:36.807 INFO:tasks.workunit.client.0.smithi136.stdout:op 5229 completed, throughput=5MB/sec 2024-09-13T00:18:36.807 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-W0tvbyeIILJICCT off=1571897 len=367703 2024-09-13T00:18:36.812 INFO:tasks.workunit.client.0.smithi136.stdout:op 5230 completed, throughput=5MB/sec 2024-09-13T00:18:36.812 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-7XWgX3GylVqqV8Z off=1673023 len=113483 2024-09-13T00:18:36.813 INFO:tasks.workunit.client.0.smithi136.stdout:op 5231 completed, throughput=5MB/sec 2024-09-13T00:18:36.813 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-j02Z7QasR6VPbM5 off=2541558 len=1304612 2024-09-13T00:18:36.825 INFO:tasks.workunit.client.0.smithi136.stdout:op 5232 completed, throughput=5MB/sec 2024-09-13T00:18:36.825 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-MhJ9WR-vw316cJ4 off=136895 len=1152481 2024-09-13T00:18:36.828 INFO:tasks.workunit.client.0.smithi136.stdout:op 5233 completed, throughput=5MB/sec 2024-09-13T00:18:36.828 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-44rJuIImLTNfGuY off=26393 len=788501 2024-09-13T00:18:36.830 INFO:tasks.workunit.client.0.smithi136.stdout:op 5234 completed, throughput=5MB/sec 2024-09-13T00:18:37.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:18:36 smithi136 bash[75164]: cluster 2024-09-13T00:18:35.658875+0000 mgr.y (mgr.24661) 1244 : cluster [DBG] pgmap v1927: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.8 MiB/s rd, 1.7 MiB/s wr, 5 op/s 2024-09-13T00:18:37.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:18:36 smithi136 bash[76371]: cluster 2024-09-13T00:18:35.658875+0000 mgr.y (mgr.24661) 1244 : cluster [DBG] pgmap v1927: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.8 MiB/s rd, 1.7 MiB/s wr, 5 op/s 2024-09-13T00:18:37.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:18:36 smithi181 bash[67587]: cluster 2024-09-13T00:18:35.658875+0000 mgr.y (mgr.24661) 1244 : cluster [DBG] pgmap v1927: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.8 MiB/s rd, 1.7 MiB/s wr, 5 op/s 2024-09-13T00:18:37.831 INFO:tasks.workunit.client.0.smithi136.stdout: 1002: throughput=5MB/sec pending data=0 2024-09-13T00:18:37.831 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-FUCrynt_MIPtfX6 off=1223722 len=743423 2024-09-13T00:18:37.835 INFO:tasks.workunit.client.0.smithi136.stdout:op 5235 completed, throughput=5MB/sec 2024-09-13T00:18:37.835 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-M1-qll-b9SrZMfm off=17570118 len=1818399 2024-09-13T00:18:37.839 INFO:tasks.workunit.client.0.smithi136.stdout:op 5236 completed, throughput=5MB/sec 2024-09-13T00:18:37.839 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-KnbHSgwK4q_fe1u off=10101073 len=418591 2024-09-13T00:18:37.841 INFO:tasks.workunit.client.0.smithi136.stdout:op 5237 completed, throughput=5MB/sec 2024-09-13T00:18:37.841 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-gc7urGk_D-eQa_q off=17892794 len=1412369 2024-09-13T00:18:37.845 INFO:tasks.workunit.client.0.smithi136.stdout:op 5238 completed, throughput=5MB/sec 2024-09-13T00:18:37.845 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-V4JRb8uTOW-AOMJ off=6017573 len=1933272 2024-09-13T00:18:37.863 INFO:tasks.workunit.client.0.smithi136.stdout:op 5239 completed, throughput=5MB/sec 2024-09-13T00:18:38.863 INFO:tasks.workunit.client.0.smithi136.stdout: 1003: throughput=5MB/sec pending data=0 2024-09-13T00:18:38.863 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ZRK4GFXKBhE8JNY off=10342736 len=10206 2024-09-13T00:18:38.864 INFO:tasks.workunit.client.0.smithi136.stdout:op 5240 completed, throughput=5MB/sec 2024-09-13T00:18:38.864 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-AJtfS-pAik71zFB off=5339794 len=255015 2024-09-13T00:18:38.869 INFO:tasks.workunit.client.0.smithi136.stdout:op 5241 completed, throughput=5MB/sec 2024-09-13T00:18:38.869 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-taPXvRw32f2QEA2 off=5354711 len=1510804 2024-09-13T00:18:38.881 INFO:tasks.workunit.client.0.smithi136.stdout:op 5242 completed, throughput=5MB/sec 2024-09-13T00:18:38.881 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-3ohWpNwbOCk4FXA off=5679891 len=1653293 2024-09-13T00:18:38.894 INFO:tasks.workunit.client.0.smithi136.stdout:op 5243 completed, throughput=5MB/sec 2024-09-13T00:18:38.895 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ns_SrmaRKd_Yu4H off=3018869 len=1264426 2024-09-13T00:18:38.897 INFO:tasks.workunit.client.0.smithi136.stdout:op 5244 completed, throughput=5MB/sec 2024-09-13T00:18:39.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:18:38 smithi136 bash[75164]: cluster 2024-09-13T00:18:37.659934+0000 mgr.y (mgr.24661) 1245 : cluster [DBG] pgmap v1928: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.7 MiB/s wr, 5 op/s 2024-09-13T00:18:39.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:18:38 smithi136 bash[75164]: audit 2024-09-13T00:18:38.800572+0000 mon.a (mon.0) 2258 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:18:39.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:18:38 smithi136 bash[76371]: cluster 2024-09-13T00:18:37.659934+0000 mgr.y (mgr.24661) 1245 : cluster [DBG] pgmap v1928: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.7 MiB/s wr, 5 op/s 2024-09-13T00:18:39.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:18:38 smithi136 bash[76371]: audit 2024-09-13T00:18:38.800572+0000 mon.a (mon.0) 2258 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:18:39.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:18:38 smithi181 bash[67587]: cluster 2024-09-13T00:18:37.659934+0000 mgr.y (mgr.24661) 1245 : cluster [DBG] pgmap v1928: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.7 MiB/s wr, 5 op/s 2024-09-13T00:18:39.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:18:38 smithi181 bash[67587]: audit 2024-09-13T00:18:38.800572+0000 mon.a (mon.0) 2258 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:18:39.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:18:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:18:39] "GET /metrics HTTP/1.1" 200 34141 "" "Prometheus/2.43.0" 2024-09-13T00:18:39.897 INFO:tasks.workunit.client.0.smithi136.stdout: 1004: throughput=5MB/sec pending data=0 2024-09-13T00:18:39.897 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-NswvUz6wV9_tPRw off=7090100 len=717051 2024-09-13T00:18:39.905 INFO:tasks.workunit.client.0.smithi136.stdout:op 5245 completed, throughput=5MB/sec 2024-09-13T00:18:39.905 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-7Qf1UuKs4IXjBwy off=0 len=1622403 2024-09-13T00:18:39.909 INFO:tasks.workunit.client.0.smithi136.stdout:op 5246 completed, throughput=5MB/sec 2024-09-13T00:18:39.910 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-tDgKi40CU9dzyhP off=6395725 len=1707175 2024-09-13T00:18:39.915 INFO:tasks.workunit.client.0.smithi136.stdout:op 5247 completed, throughput=5MB/sec 2024-09-13T00:18:39.915 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-qGegGX94yajuobs off=14040499 len=1652965 2024-09-13T00:18:39.919 INFO:tasks.workunit.client.0.smithi136.stdout:op 5248 completed, throughput=5MB/sec 2024-09-13T00:18:40.919 INFO:tasks.workunit.client.0.smithi136.stdout: 1005: throughput=5MB/sec pending data=0 2024-09-13T00:18:40.920 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-HFNV_jrLep6H4QE off=5056165 len=1095856 2024-09-13T00:18:40.930 INFO:tasks.workunit.client.0.smithi136.stdout:op 5249 completed, throughput=5MB/sec 2024-09-13T00:18:40.930 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-PwXedJPU3Wwh4Cb off=12422919 len=1193376 2024-09-13T00:18:40.933 INFO:tasks.workunit.client.0.smithi136.stdout:op 5250 completed, throughput=5MB/sec 2024-09-13T00:18:40.933 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-HNgGwhrG98yx1rh off=7145821 len=266218 2024-09-13T00:18:40.938 INFO:tasks.workunit.client.0.smithi136.stdout:op 5251 completed, throughput=5MB/sec 2024-09-13T00:18:40.938 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-9h9iLGZvFKSeTCk off=246908 len=779653 2024-09-13T00:18:40.946 INFO:tasks.workunit.client.0.smithi136.stdout:op 5252 completed, throughput=5MB/sec 2024-09-13T00:18:40.946 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-3kssQfeVEi1Pwzw off=12271100 len=824376 2024-09-13T00:18:40.949 INFO:tasks.workunit.client.0.smithi136.stdout:op 5253 completed, throughput=5MB/sec 2024-09-13T00:18:40.949 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-azts5Q9duQAliPk off=2551801 len=947018 2024-09-13T00:18:40.957 INFO:tasks.workunit.client.0.smithi136.stdout:op 5254 completed, throughput=5MB/sec 2024-09-13T00:18:41.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:18:40 smithi136 bash[75164]: cluster 2024-09-13T00:18:39.660845+0000 mgr.y (mgr.24661) 1246 : cluster [DBG] pgmap v1929: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.8 MiB/s rd, 1.6 MiB/s wr, 5 op/s 2024-09-13T00:18:41.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:18:40 smithi136 bash[76371]: cluster 2024-09-13T00:18:39.660845+0000 mgr.y (mgr.24661) 1246 : cluster [DBG] pgmap v1929: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.8 MiB/s rd, 1.6 MiB/s wr, 5 op/s 2024-09-13T00:18:41.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:18:40 smithi181 bash[67587]: cluster 2024-09-13T00:18:39.660845+0000 mgr.y (mgr.24661) 1246 : cluster [DBG] pgmap v1929: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.8 MiB/s rd, 1.6 MiB/s wr, 5 op/s 2024-09-13T00:18:41.957 INFO:tasks.workunit.client.0.smithi136.stdout: 1006: throughput=5MB/sec pending data=0 2024-09-13T00:18:41.957 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-sFaGNWqAv0mmMyA off=11388186 len=2065558 2024-09-13T00:18:41.963 INFO:tasks.workunit.client.0.smithi136.stdout:op 5255 completed, throughput=5MB/sec 2024-09-13T00:18:41.964 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-73ChvK1OIUBk1og off=13746591 len=457797 2024-09-13T00:18:41.970 INFO:tasks.workunit.client.0.smithi136.stdout:op 5256 completed, throughput=5MB/sec 2024-09-13T00:18:41.970 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-DyGMYhz7qVXsdfq off=99954 len=881836 2024-09-13T00:18:41.973 INFO:tasks.workunit.client.0.smithi136.stdout:op 5257 completed, throughput=5MB/sec 2024-09-13T00:18:41.973 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-g9eRpRZePKkr9f9 off=4190745 len=1290103 2024-09-13T00:18:41.976 INFO:tasks.workunit.client.0.smithi136.stdout:op 5258 completed, throughput=5MB/sec 2024-09-13T00:18:41.977 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-bT3IONbZG3xUc6o off=3723896 len=2000668 2024-09-13T00:18:41.989 INFO:tasks.workunit.client.0.smithi136.stdout:op 5259 completed, throughput=5MB/sec 2024-09-13T00:18:42.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:18:42 smithi181 bash[95505]: ts=2024-09-13T00:18:42.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:18:42.990 INFO:tasks.workunit.client.0.smithi136.stdout: 1007: throughput=5MB/sec pending data=0 2024-09-13T00:18:42.990 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-lxUTGoKsJ-N7Ls2 off=4735114 len=140177 2024-09-13T00:18:42.991 INFO:tasks.workunit.client.0.smithi136.stdout:op 5260 completed, throughput=5MB/sec 2024-09-13T00:18:42.991 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-sxh-S90JvvxTOx5 off=6218194 len=598253 2024-09-13T00:18:42.993 INFO:tasks.workunit.client.0.smithi136.stdout:op 5261 completed, throughput=5MB/sec 2024-09-13T00:18:42.993 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-TN_7VJ0q2CCmjRu off=2215700 len=1463706 2024-09-13T00:18:42.997 INFO:tasks.workunit.client.0.smithi136.stdout:op 5262 completed, throughput=5MB/sec 2024-09-13T00:18:42.997 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-rmCSIr5CbtP1yM- off=1247642 len=395102 2024-09-13T00:18:42.999 INFO:tasks.workunit.client.0.smithi136.stdout:op 5263 completed, throughput=5MB/sec 2024-09-13T00:18:42.999 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-rLpq4Bp2aNN99OX off=2786372 len=1809929 2024-09-13T00:18:43.003 INFO:tasks.workunit.client.0.smithi136.stdout:op 5264 completed, throughput=5MB/sec 2024-09-13T00:18:43.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:18:42 smithi136 bash[75164]: cluster 2024-09-13T00:18:41.661837+0000 mgr.y (mgr.24661) 1247 : cluster [DBG] pgmap v1930: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.5 MiB/s rd, 1.9 MiB/s wr, 4 op/s 2024-09-13T00:18:43.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:18:42 smithi136 bash[76371]: cluster 2024-09-13T00:18:41.661837+0000 mgr.y (mgr.24661) 1247 : cluster [DBG] pgmap v1930: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.5 MiB/s rd, 1.9 MiB/s wr, 4 op/s 2024-09-13T00:18:43.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:18:42 smithi181 bash[67587]: cluster 2024-09-13T00:18:41.661837+0000 mgr.y (mgr.24661) 1247 : cluster [DBG] pgmap v1930: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.5 MiB/s rd, 1.9 MiB/s wr, 4 op/s 2024-09-13T00:18:44.003 INFO:tasks.workunit.client.0.smithi136.stdout: 1008: throughput=5MB/sec pending data=0 2024-09-13T00:18:44.003 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Mds6Rom8_o6kIVl off=4930266 len=633835 2024-09-13T00:18:44.005 INFO:tasks.workunit.client.0.smithi136.stdout:op 5265 completed, throughput=5MB/sec 2024-09-13T00:18:44.006 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-XfpNb7Ss5_a0jM4 off=14214493 len=1786685 2024-09-13T00:18:44.010 INFO:tasks.workunit.client.0.smithi136.stdout:op 5266 completed, throughput=5MB/sec 2024-09-13T00:18:44.010 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-MS8-mKvcuD98d10 off=7463690 len=528374 2024-09-13T00:18:44.016 INFO:tasks.workunit.client.0.smithi136.stdout:op 5267 completed, throughput=5MB/sec 2024-09-13T00:18:44.016 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-7kIyBPOd7-X9G3K off=2795931 len=1292132 2024-09-13T00:18:44.026 INFO:tasks.workunit.client.0.smithi136.stdout:op 5268 completed, throughput=5MB/sec 2024-09-13T00:18:44.026 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-rbu6y7_hVi6p7rN off=3117104 len=14867 2024-09-13T00:18:44.027 INFO:tasks.workunit.client.0.smithi136.stdout:op 5269 completed, throughput=5MB/sec 2024-09-13T00:18:44.027 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-nJkqK-X2bayXHEg off=41931 len=584488 2024-09-13T00:18:44.035 INFO:tasks.workunit.client.0.smithi136.stdout:op 5270 completed, throughput=5MB/sec 2024-09-13T00:18:45.035 INFO:tasks.workunit.client.0.smithi136.stdout: 1009: throughput=5MB/sec pending data=0 2024-09-13T00:18:45.035 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-NjS_3jaZO8XDUFb off=0 len=1545481 2024-09-13T00:18:45.039 INFO:tasks.workunit.client.0.smithi136.stdout:op 5271 completed, throughput=5MB/sec 2024-09-13T00:18:45.039 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-UA0W1xaOirzz9Dw off=4136853 len=82405 2024-09-13T00:18:45.044 INFO:tasks.workunit.client.0.smithi136.stdout:op 5272 completed, throughput=5MB/sec 2024-09-13T00:18:45.044 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-SzbaixESONCRByI off=913434 len=218393 2024-09-13T00:18:45.049 INFO:tasks.workunit.client.0.smithi136.stdout:op 5273 completed, throughput=5MB/sec 2024-09-13T00:18:45.049 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-g9eRpRZePKkr9f9 off=3950116 len=112790 2024-09-13T00:18:45.053 INFO:tasks.workunit.client.0.smithi136.stdout:op 5274 completed, throughput=5MB/sec 2024-09-13T00:18:45.053 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-HjJD_s-fXkikXAs off=7002744 len=1737028 2024-09-13T00:18:45.057 INFO:tasks.workunit.client.0.smithi136.stdout:op 5275 completed, throughput=5MB/sec 2024-09-13T00:18:45.057 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-l61HeY4DyH0sMkR off=696566 len=1835666 2024-09-13T00:18:45.069 INFO:tasks.workunit.client.0.smithi136.stdout:op 5276 completed, throughput=5MB/sec 2024-09-13T00:18:45.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:18:44 smithi136 bash[75164]: cluster 2024-09-13T00:18:43.662935+0000 mgr.y (mgr.24661) 1248 : cluster [DBG] pgmap v1931: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 3.3 MiB/s rd, 2.1 MiB/s wr, 5 op/s 2024-09-13T00:18:45.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:18:44 smithi136 bash[76371]: cluster 2024-09-13T00:18:43.662935+0000 mgr.y (mgr.24661) 1248 : cluster [DBG] pgmap v1931: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 3.3 MiB/s rd, 2.1 MiB/s wr, 5 op/s 2024-09-13T00:18:45.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:18:44 smithi181 bash[67587]: cluster 2024-09-13T00:18:43.662935+0000 mgr.y (mgr.24661) 1248 : cluster [DBG] pgmap v1931: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 3.3 MiB/s rd, 2.1 MiB/s wr, 5 op/s 2024-09-13T00:18:46.069 INFO:tasks.workunit.client.0.smithi136.stdout: 1010: throughput=5MB/sec pending data=0 2024-09-13T00:18:46.069 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-PsXgKd8RdLHjPq8 off=4907804 len=159722 2024-09-13T00:18:46.071 INFO:tasks.workunit.client.0.smithi136.stdout:op 5277 completed, throughput=5MB/sec 2024-09-13T00:18:46.071 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-alezcT-YNfh94r0 off=16740998 len=1935422 2024-09-13T00:18:46.089 INFO:tasks.workunit.client.0.smithi136.stdout:op 5278 completed, throughput=5MB/sec 2024-09-13T00:18:46.089 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-yQ4MD06AmLZRFH4 off=293953 len=744208 2024-09-13T00:18:46.091 INFO:tasks.workunit.client.0.smithi136.stdout:op 5279 completed, throughput=5MB/sec 2024-09-13T00:18:46.091 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-6zMoJ5QzOMmnmfv off=7089308 len=1245889 2024-09-13T00:18:46.095 INFO:tasks.workunit.client.0.smithi136.stdout:op 5280 completed, throughput=5MB/sec 2024-09-13T00:18:46.095 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Ll3frxAHnsIfG-I off=7181418 len=323364 2024-09-13T00:18:46.100 INFO:tasks.workunit.client.0.smithi136.stdout:op 5281 completed, throughput=5MB/sec 2024-09-13T00:18:46.101 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Ckqvo14s0AB987V off=16219675 len=1059233 2024-09-13T00:18:46.109 INFO:tasks.workunit.client.0.smithi136.stdout:op 5282 completed, throughput=5MB/sec 2024-09-13T00:18:47.110 INFO:tasks.workunit.client.0.smithi136.stdout: 1011: throughput=5MB/sec pending data=0 2024-09-13T00:18:47.110 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-tgfPgVmV3nwFj3o off=17817705 len=451325 2024-09-13T00:18:47.112 INFO:tasks.workunit.client.0.smithi136.stdout:op 5283 completed, throughput=5MB/sec 2024-09-13T00:18:47.112 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-OMtOU6Nw_GqnGjr off=980270 len=873258 2024-09-13T00:18:47.120 INFO:tasks.workunit.client.0.smithi136.stdout:op 5284 completed, throughput=5MB/sec 2024-09-13T00:18:47.120 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-S9MIROddUqVlDe1 off=1017425 len=131228 2024-09-13T00:18:47.121 INFO:tasks.workunit.client.0.smithi136.stdout:op 5285 completed, throughput=5MB/sec 2024-09-13T00:18:47.121 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-hu-K95JlG_EIQ2I off=14552686 len=1600909 2024-09-13T00:18:47.125 INFO:tasks.workunit.client.0.smithi136.stdout:op 5286 completed, throughput=5MB/sec 2024-09-13T00:18:47.125 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_37NFoLj_-bCwCQ off=1065718 len=841295 2024-09-13T00:18:47.133 INFO:tasks.workunit.client.0.smithi136.stdout:op 5287 completed, throughput=5MB/sec 2024-09-13T00:18:47.133 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-pclwX2BuHjBM3y2 off=6066267 len=1956622 2024-09-13T00:18:47.138 INFO:tasks.workunit.client.0.smithi136.stdout:op 5288 completed, throughput=5MB/sec 2024-09-13T00:18:47.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:18:46 smithi136 bash[75164]: cluster 2024-09-13T00:18:45.663863+0000 mgr.y (mgr.24661) 1249 : cluster [DBG] pgmap v1932: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.8 MiB/s wr, 4 op/s 2024-09-13T00:18:47.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:18:46 smithi136 bash[76371]: cluster 2024-09-13T00:18:45.663863+0000 mgr.y (mgr.24661) 1249 : cluster [DBG] pgmap v1932: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.8 MiB/s wr, 4 op/s 2024-09-13T00:18:47.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:18:46 smithi181 bash[67587]: cluster 2024-09-13T00:18:45.663863+0000 mgr.y (mgr.24661) 1249 : cluster [DBG] pgmap v1932: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.8 MiB/s wr, 4 op/s 2024-09-13T00:18:48.138 INFO:tasks.workunit.client.0.smithi136.stdout: 1012: throughput=5MB/sec pending data=0 2024-09-13T00:18:48.138 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-pMFJzKy2JgVKRAL off=237772 len=1210878 2024-09-13T00:18:48.142 INFO:tasks.workunit.client.0.smithi136.stdout:op 5289 completed, throughput=5MB/sec 2024-09-13T00:18:48.142 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-m4mRNZdt-_IG0TW off=13781125 len=917853 2024-09-13T00:18:48.143 INFO:tasks.workunit.client.0.smithi136.stdout:op 5290 completed, throughput=5MB/sec 2024-09-13T00:18:48.143 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-11wC9PLHGJKXQTl off=2311174 len=797211 2024-09-13T00:18:48.146 INFO:tasks.workunit.client.0.smithi136.stdout:op 5291 completed, throughput=5MB/sec 2024-09-13T00:18:48.146 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-yyZm9oQnzrHvmvN off=5535911 len=777304 2024-09-13T00:18:48.154 INFO:tasks.workunit.client.0.smithi136.stdout:op 5292 completed, throughput=5MB/sec 2024-09-13T00:18:48.154 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-vmkZdv0Nh297sD7 off=9761324 len=871815 2024-09-13T00:18:48.160 INFO:tasks.workunit.client.0.smithi136.stdout:op 5293 completed, throughput=5MB/sec 2024-09-13T00:18:49.161 INFO:tasks.workunit.client.0.smithi136.stdout: 1013: throughput=5MB/sec pending data=0 2024-09-13T00:18:49.161 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-FZov6wTa9nVLLmC off=10829501 len=603643 2024-09-13T00:18:49.163 INFO:tasks.workunit.client.0.smithi136.stdout:op 5294 completed, throughput=5MB/sec 2024-09-13T00:18:49.163 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-PuduaMmADHgHoAr off=9712233 len=1109122 2024-09-13T00:18:49.166 INFO:tasks.workunit.client.0.smithi136.stdout:op 5295 completed, throughput=5MB/sec 2024-09-13T00:18:49.166 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-lrXFWujYEJPUf6N off=4828695 len=444593 2024-09-13T00:18:49.167 INFO:tasks.workunit.client.0.smithi136.stdout:op 5296 completed, throughput=5MB/sec 2024-09-13T00:18:49.167 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-G8gYfHPBTPcfK4J off=1432489 len=213359 2024-09-13T00:18:49.168 INFO:tasks.workunit.client.0.smithi136.stdout:op 5297 completed, throughput=5MB/sec 2024-09-13T00:18:49.169 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-__Gj0JaWLdMXuAi off=3786971 len=1568052 2024-09-13T00:18:49.183 INFO:tasks.workunit.client.0.smithi136.stdout:op 5298 completed, throughput=5MB/sec 2024-09-13T00:18:49.183 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-div-9XioE4oGAHI off=837331 len=717040 2024-09-13T00:18:49.191 INFO:tasks.workunit.client.0.smithi136.stdout:op 5299 completed, throughput=5MB/sec 2024-09-13T00:18:49.191 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-QDTqeZrxRXHqhfX off=10801162 len=723680 2024-09-13T00:18:49.199 INFO:tasks.workunit.client.0.smithi136.stdout:op 5300 completed, throughput=5MB/sec 2024-09-13T00:18:49.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:18:48 smithi136 bash[75164]: cluster 2024-09-13T00:18:47.664904+0000 mgr.y (mgr.24661) 1250 : cluster [DBG] pgmap v1933: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.9 MiB/s wr, 5 op/s 2024-09-13T00:18:49.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:18:48 smithi136 bash[76371]: cluster 2024-09-13T00:18:47.664904+0000 mgr.y (mgr.24661) 1250 : cluster [DBG] pgmap v1933: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.9 MiB/s wr, 5 op/s 2024-09-13T00:18:49.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:18:48 smithi181 bash[67587]: cluster 2024-09-13T00:18:47.664904+0000 mgr.y (mgr.24661) 1250 : cluster [DBG] pgmap v1933: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.9 MiB/s wr, 5 op/s 2024-09-13T00:18:49.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:18:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:18:49] "GET /metrics HTTP/1.1" 200 34141 "" "Prometheus/2.43.0" 2024-09-13T00:18:50.199 INFO:tasks.workunit.client.0.smithi136.stdout: 1014: throughput=5MB/sec pending data=0 2024-09-13T00:18:50.199 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-8leH6Kt7mldGFNR off=12512811 len=492876 2024-09-13T00:18:50.202 INFO:tasks.workunit.client.0.smithi136.stdout:op 5301 completed, throughput=5MB/sec 2024-09-13T00:18:50.202 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-AEN8PcCQ8EwBl_k off=8830413 len=641687 2024-09-13T00:18:50.209 INFO:tasks.workunit.client.0.smithi136.stdout:op 5302 completed, throughput=5MB/sec 2024-09-13T00:18:50.209 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-fAtAxEUiY7-3JIm off=7755718 len=529378 2024-09-13T00:18:50.211 INFO:tasks.workunit.client.0.smithi136.stdout:op 5303 completed, throughput=5MB/sec 2024-09-13T00:18:50.211 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-rcrXNtJ3KUrd3TR off=4142880 len=638430 2024-09-13T00:18:50.217 INFO:tasks.workunit.client.0.smithi136.stdout:op 5304 completed, throughput=5MB/sec 2024-09-13T00:18:50.218 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-h42Q9vIkPfW1Txq off=5629582 len=1688956 2024-09-13T00:18:50.229 INFO:tasks.workunit.client.0.smithi136.stdout:op 5305 completed, throughput=5MB/sec 2024-09-13T00:18:50.229 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-5WDJQe3k8pd8D1O off=5423846 len=811775 2024-09-13T00:18:50.236 INFO:tasks.workunit.client.0.smithi136.stdout:op 5306 completed, throughput=5MB/sec 2024-09-13T00:18:50.237 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-wFn5QlybMBbT_RO off=9631100 len=1274824 2024-09-13T00:18:50.240 INFO:tasks.workunit.client.0.smithi136.stdout:op 5307 completed, throughput=5MB/sec 2024-09-13T00:18:51.241 INFO:tasks.workunit.client.0.smithi136.stdout: 1015: throughput=5MB/sec pending data=0 2024-09-13T00:18:51.241 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-US6DHjEGgb_0iWQ off=9541165 len=749377 2024-09-13T00:18:51.243 INFO:tasks.workunit.client.0.smithi136.stdout:op 5308 completed, throughput=5MB/sec 2024-09-13T00:18:51.243 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-3b-rik60xrux4rF off=16901775 len=1829861 2024-09-13T00:18:51.247 INFO:tasks.workunit.client.0.smithi136.stdout:op 5309 completed, throughput=5MB/sec 2024-09-13T00:18:51.248 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-zH4yPFxMOXqLW04 off=0 len=1205229 2024-09-13T00:18:51.251 INFO:tasks.workunit.client.0.smithi136.stdout:op 5310 completed, throughput=5MB/sec 2024-09-13T00:18:51.251 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-i6uIyG3atJ-m4jZ off=2959413 len=941772 2024-09-13T00:18:51.253 INFO:tasks.workunit.client.0.smithi136.stdout:op 5311 completed, throughput=5MB/sec 2024-09-13T00:18:51.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:18:50 smithi136 bash[75164]: cluster 2024-09-13T00:18:49.665779+0000 mgr.y (mgr.24661) 1251 : cluster [DBG] pgmap v1934: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.0 MiB/s wr, 5 op/s 2024-09-13T00:18:51.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:18:50 smithi136 bash[76371]: cluster 2024-09-13T00:18:49.665779+0000 mgr.y (mgr.24661) 1251 : cluster [DBG] pgmap v1934: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.0 MiB/s wr, 5 op/s 2024-09-13T00:18:51.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:18:50 smithi181 bash[67587]: cluster 2024-09-13T00:18:49.665779+0000 mgr.y (mgr.24661) 1251 : cluster [DBG] pgmap v1934: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.0 MiB/s wr, 5 op/s 2024-09-13T00:18:52.254 INFO:tasks.workunit.client.0.smithi136.stdout: 1016: throughput=5MB/sec pending data=0 2024-09-13T00:18:52.254 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-xph9bzj-6YYkHFP off=13029338 len=747554 2024-09-13T00:18:52.257 INFO:tasks.workunit.client.0.smithi136.stdout:op 5312 completed, throughput=5MB/sec 2024-09-13T00:18:52.257 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ckrbKrK5chPMNMZ off=10573997 len=904409 2024-09-13T00:18:52.267 INFO:tasks.workunit.client.0.smithi136.stdout:op 5313 completed, throughput=5MB/sec 2024-09-13T00:18:52.267 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Oif9h-zJ_VhwjG6 off=854027 len=1139521 2024-09-13T00:18:52.270 INFO:tasks.workunit.client.0.smithi136.stdout:op 5314 completed, throughput=5MB/sec 2024-09-13T00:18:52.270 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-cYuNduodHomwy05 off=10416579 len=1199993 2024-09-13T00:18:52.279 INFO:tasks.workunit.client.0.smithi136.stdout:op 5315 completed, throughput=5MB/sec 2024-09-13T00:18:52.279 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-UXHi-b7cQSdNOV3 off=12479024 len=727071 2024-09-13T00:18:52.281 INFO:tasks.workunit.client.0.smithi136.stdout:op 5316 completed, throughput=5MB/sec 2024-09-13T00:18:52.281 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-NL0axwmZq7tO4pZ off=2609455 len=1933081 2024-09-13T00:18:52.294 INFO:tasks.workunit.client.0.smithi136.stdout:op 5317 completed, throughput=5MB/sec 2024-09-13T00:18:52.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:18:52 smithi181 bash[95505]: ts=2024-09-13T00:18:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:18:53.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:18:52 smithi136 bash[75164]: cluster 2024-09-13T00:18:51.666893+0000 mgr.y (mgr.24661) 1252 : cluster [DBG] pgmap v1935: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.1 MiB/s wr, 5 op/s 2024-09-13T00:18:53.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:18:52 smithi136 bash[76371]: cluster 2024-09-13T00:18:51.666893+0000 mgr.y (mgr.24661) 1252 : cluster [DBG] pgmap v1935: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.1 MiB/s wr, 5 op/s 2024-09-13T00:18:53.294 INFO:tasks.workunit.client.0.smithi136.stdout: 1017: throughput=5MB/sec pending data=0 2024-09-13T00:18:53.295 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-YwBDYgtKQg7bbjB off=7331816 len=838455 2024-09-13T00:18:53.297 INFO:tasks.workunit.client.0.smithi136.stdout:op 5318 completed, throughput=5MB/sec 2024-09-13T00:18:53.298 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-chtrm2Qu8q5iMdB off=7687710 len=1329739 2024-09-13T00:18:53.307 INFO:tasks.workunit.client.0.smithi136.stdout:op 5319 completed, throughput=5MB/sec 2024-09-13T00:18:53.307 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-3jSTGikJ9RFfH-7 off=825585 len=1057581 2024-09-13T00:18:53.317 INFO:tasks.workunit.client.0.smithi136.stdout:op 5320 completed, throughput=5MB/sec 2024-09-13T00:18:53.317 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-xOq0Lc_qPzbiaq5 off=14883047 len=1245764 2024-09-13T00:18:53.326 INFO:tasks.workunit.client.0.smithi136.stdout:op 5321 completed, throughput=5MB/sec 2024-09-13T00:18:53.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:18:52 smithi181 bash[67587]: cluster 2024-09-13T00:18:51.666893+0000 mgr.y (mgr.24661) 1252 : cluster [DBG] pgmap v1935: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.1 MiB/s wr, 5 op/s 2024-09-13T00:18:54.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:18:53 smithi136 bash[76371]: audit 2024-09-13T00:18:53.800979+0000 mon.a (mon.0) 2259 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:18:54.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:18:53 smithi136 bash[75164]: audit 2024-09-13T00:18:53.800979+0000 mon.a (mon.0) 2259 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:18:54.326 INFO:tasks.workunit.client.0.smithi136.stdout: 1018: throughput=5MB/sec pending data=0 2024-09-13T00:18:54.327 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-cNfwecRozfqWxNI off=1225710 len=531347 2024-09-13T00:18:54.333 INFO:tasks.workunit.client.0.smithi136.stdout:op 5322 completed, throughput=5MB/sec 2024-09-13T00:18:54.333 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-84hg7N9A7_4R1GL off=780102 len=196934 2024-09-13T00:18:54.338 INFO:tasks.workunit.client.0.smithi136.stdout:op 5323 completed, throughput=5MB/sec 2024-09-13T00:18:54.338 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-kvREdaMfmz_X9PH off=1294476 len=1918148 2024-09-13T00:18:54.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:18:53 smithi181 bash[67587]: audit 2024-09-13T00:18:53.800979+0000 mon.a (mon.0) 2259 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:18:54.350 INFO:tasks.workunit.client.0.smithi136.stdout:op 5324 completed, throughput=5MB/sec 2024-09-13T00:18:54.350 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Aw1I2QVqxHWEK-P off=12132316 len=1213349 2024-09-13T00:18:54.360 INFO:tasks.workunit.client.0.smithi136.stdout:op 5325 completed, throughput=5MB/sec 2024-09-13T00:18:54.360 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Rv96aM3fD4_HZnP off=6653545 len=845203 2024-09-13T00:18:54.363 INFO:tasks.workunit.client.0.smithi136.stdout:op 5326 completed, throughput=5MB/sec 2024-09-13T00:18:54.363 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ies0w-ttFwg2NX- off=15320852 len=1123356 2024-09-13T00:18:54.366 INFO:tasks.workunit.client.0.smithi136.stdout:op 5327 completed, throughput=5MB/sec 2024-09-13T00:18:55.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:18:54 smithi136 bash[75164]: cluster 2024-09-13T00:18:53.668059+0000 mgr.y (mgr.24661) 1253 : cluster [DBG] pgmap v1936: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 3.6 MiB/s rd, 2.5 MiB/s wr, 6 op/s 2024-09-13T00:18:55.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:18:54 smithi136 bash[76371]: cluster 2024-09-13T00:18:53.668059+0000 mgr.y (mgr.24661) 1253 : cluster [DBG] pgmap v1936: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 3.6 MiB/s rd, 2.5 MiB/s wr, 6 op/s 2024-09-13T00:18:55.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:18:54 smithi181 bash[67587]: cluster 2024-09-13T00:18:53.668059+0000 mgr.y (mgr.24661) 1253 : cluster [DBG] pgmap v1936: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 3.6 MiB/s rd, 2.5 MiB/s wr, 6 op/s 2024-09-13T00:18:55.367 INFO:tasks.workunit.client.0.smithi136.stdout: 1019: throughput=5MB/sec pending data=0 2024-09-13T00:18:55.367 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-lHNIvbLeN128_pn off=1150628 len=263709 2024-09-13T00:18:55.369 INFO:tasks.workunit.client.0.smithi136.stdout:op 5328 completed, throughput=5MB/sec 2024-09-13T00:18:55.369 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-8hARRf08HWdCFui off=5403172 len=572856 2024-09-13T00:18:55.376 INFO:tasks.workunit.client.0.smithi136.stdout:op 5329 completed, throughput=5MB/sec 2024-09-13T00:18:55.376 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-OWtao2YiOQ1yei2 off=9120663 len=1825534 2024-09-13T00:18:55.390 INFO:tasks.workunit.client.0.smithi136.stdout:op 5330 completed, throughput=5MB/sec 2024-09-13T00:18:55.390 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-0kA049cZ3RLRyQw off=14577988 len=459048 2024-09-13T00:18:55.397 INFO:tasks.workunit.client.0.smithi136.stdout:op 5331 completed, throughput=5MB/sec 2024-09-13T00:18:55.397 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-iT74bW0Ze_ZDNt0 off=5385268 len=1765091 2024-09-13T00:18:55.399 INFO:tasks.workunit.client.0.smithi136.stdout:op 5332 completed, throughput=5MB/sec 2024-09-13T00:18:56.400 INFO:tasks.workunit.client.0.smithi136.stdout: 1020: throughput=5MB/sec pending data=0 2024-09-13T00:18:56.400 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-67IO_L8ypTVG8j1 off=2896427 len=2056548 2024-09-13T00:18:56.417 INFO:tasks.workunit.client.0.smithi136.stdout:op 5333 completed, throughput=5MB/sec 2024-09-13T00:18:56.417 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-UFAixOzfeuHZXcq off=16172625 len=1122268 2024-09-13T00:18:56.420 INFO:tasks.workunit.client.0.smithi136.stdout:op 5334 completed, throughput=5MB/sec 2024-09-13T00:18:56.420 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-GB8j7aBpzWL-rSH off=1258315 len=392672 2024-09-13T00:18:56.422 INFO:tasks.workunit.client.0.smithi136.stdout:op 5335 completed, throughput=5MB/sec 2024-09-13T00:18:56.422 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-WbTv-NH8fl9IRpR off=10952436 len=2009906 2024-09-13T00:18:56.426 INFO:tasks.workunit.client.0.smithi136.stdout:op 5336 completed, throughput=5MB/sec 2024-09-13T00:18:57.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:18:56 smithi136 bash[75164]: cluster 2024-09-13T00:18:55.669156+0000 mgr.y (mgr.24661) 1254 : cluster [DBG] pgmap v1937: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:18:57.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:18:56 smithi136 bash[76371]: cluster 2024-09-13T00:18:55.669156+0000 mgr.y (mgr.24661) 1254 : cluster [DBG] pgmap v1937: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:18:57.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:18:56 smithi181 bash[67587]: cluster 2024-09-13T00:18:55.669156+0000 mgr.y (mgr.24661) 1254 : cluster [DBG] pgmap v1937: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:18:57.427 INFO:tasks.workunit.client.0.smithi136.stdout: 1021: throughput=5MB/sec pending data=0 2024-09-13T00:18:57.427 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-hmRzGrIn7Ko5Yqw off=6814219 len=1176641 2024-09-13T00:18:57.430 INFO:tasks.workunit.client.0.smithi136.stdout:op 5337 completed, throughput=5MB/sec 2024-09-13T00:18:57.430 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-EucnKFTka9E3wKL off=15583105 len=582224 2024-09-13T00:18:57.432 INFO:tasks.workunit.client.0.smithi136.stdout:op 5338 completed, throughput=5MB/sec 2024-09-13T00:18:57.432 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-4snPdCiUfD4k-AC off=7549726 len=276203 2024-09-13T00:18:57.439 INFO:tasks.workunit.client.0.smithi136.stdout:op 5339 completed, throughput=5MB/sec 2024-09-13T00:18:57.439 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj--GS-xS5hh9SkIak off=4232131 len=81046 2024-09-13T00:18:57.443 INFO:tasks.workunit.client.0.smithi136.stdout:op 5340 completed, throughput=5MB/sec 2024-09-13T00:18:57.443 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ozImzjoWMNlbYb4 off=4621076 len=983236 2024-09-13T00:18:57.453 INFO:tasks.workunit.client.0.smithi136.stdout:op 5341 completed, throughput=5MB/sec 2024-09-13T00:18:57.453 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2BzaOiTWDnHxNvB off=399849 len=2029051 2024-09-13T00:18:57.458 INFO:tasks.workunit.client.0.smithi136.stdout:op 5342 completed, throughput=5MB/sec 2024-09-13T00:18:58.458 INFO:tasks.workunit.client.0.smithi136.stdout: 1022: throughput=5MB/sec pending data=0 2024-09-13T00:18:58.458 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-MWYH2JzuacwEROe off=12220182 len=895281 2024-09-13T00:18:58.461 INFO:tasks.workunit.client.0.smithi136.stdout:op 5343 completed, throughput=5MB/sec 2024-09-13T00:18:58.461 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-WEaVbobjNhky33s off=1321499 len=1352301 2024-09-13T00:18:58.473 INFO:tasks.workunit.client.0.smithi136.stdout:op 5344 completed, throughput=5MB/sec 2024-09-13T00:18:58.473 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-eCzSgy83MLl1-k4 off=816362 len=1688864 2024-09-13T00:18:58.487 INFO:tasks.workunit.client.0.smithi136.stdout:op 5345 completed, throughput=5MB/sec 2024-09-13T00:18:58.487 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-8QApCvMphI3JmQA off=978970 len=587095 2024-09-13T00:18:58.489 INFO:tasks.workunit.client.0.smithi136.stdout:op 5346 completed, throughput=5MB/sec 2024-09-13T00:18:58.489 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-0VZo0JFS-0ORS1f off=2615616 len=361159 2024-09-13T00:18:58.491 INFO:tasks.workunit.client.0.smithi136.stdout:op 5347 completed, throughput=5MB/sec 2024-09-13T00:18:58.492 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-wU27c8k1nI3ggo6 off=1062565 len=571394 2024-09-13T00:18:58.494 INFO:tasks.workunit.client.0.smithi136.stdout:op 5348 completed, throughput=5MB/sec 2024-09-13T00:18:59.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:18:58 smithi136 bash[75164]: cluster 2024-09-13T00:18:57.670341+0000 mgr.y (mgr.24661) 1255 : cluster [DBG] pgmap v1938: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:18:59.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:18:58 smithi136 bash[75164]: audit 2024-09-13T00:18:58.739218+0000 mon.a (mon.0) 2260 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:18:59.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:18:58 smithi136 bash[76371]: cluster 2024-09-13T00:18:57.670341+0000 mgr.y (mgr.24661) 1255 : cluster [DBG] pgmap v1938: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:18:59.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:18:58 smithi136 bash[76371]: audit 2024-09-13T00:18:58.739218+0000 mon.a (mon.0) 2260 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:18:59.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:18:58 smithi181 bash[67587]: cluster 2024-09-13T00:18:57.670341+0000 mgr.y (mgr.24661) 1255 : cluster [DBG] pgmap v1938: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:18:59.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:18:58 smithi181 bash[67587]: audit 2024-09-13T00:18:58.739218+0000 mon.a (mon.0) 2260 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:18:59.494 INFO:tasks.workunit.client.0.smithi136.stdout: 1023: throughput=5MB/sec pending data=0 2024-09-13T00:18:59.495 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-RZJECEqpoK_ZEwX off=10100365 len=1083281 2024-09-13T00:18:59.498 INFO:tasks.workunit.client.0.smithi136.stdout:op 5349 completed, throughput=5MB/sec 2024-09-13T00:18:59.498 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-U4KKHMY33ZYcogw off=4378700 len=1340488 2024-09-13T00:18:59.501 INFO:tasks.workunit.client.0.smithi136.stdout:op 5350 completed, throughput=5MB/sec 2024-09-13T00:18:59.501 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-3h5f-fABWNNA6WX off=502975 len=18401 2024-09-13T00:18:59.506 INFO:tasks.workunit.client.0.smithi136.stdout:op 5351 completed, throughput=5MB/sec 2024-09-13T00:18:59.507 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-qLGP2IXMqTz8D9l off=3434413 len=1702784 2024-09-13T00:18:59.518 INFO:tasks.workunit.client.0.smithi136.stdout:op 5352 completed, throughput=5MB/sec 2024-09-13T00:18:59.518 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-5uoc1gSOeXhvgVR off=802924 len=517120 2024-09-13T00:18:59.525 INFO:tasks.workunit.client.0.smithi136.stdout:op 5353 completed, throughput=5MB/sec 2024-09-13T00:18:59.525 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-1J0BVuavJ_dXqRF off=1097397 len=268374 2024-09-13T00:18:59.530 INFO:tasks.workunit.client.0.smithi136.stdout:op 5354 completed, throughput=5MB/sec 2024-09-13T00:18:59.530 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-kud354bJdgfLq55 off=12961890 len=1164056 2024-09-13T00:18:59.533 INFO:tasks.workunit.client.0.smithi136.stdout:op 5355 completed, throughput=5MB/sec 2024-09-13T00:18:59.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:18:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:18:59] "GET /metrics HTTP/1.1" 200 34146 "" "Prometheus/2.43.0" 2024-09-13T00:19:00.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:18:59 smithi136 bash[75164]: audit 2024-09-13T00:18:59.290209+0000 mon.a (mon.0) 2261 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:19:00.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:18:59 smithi136 bash[75164]: audit 2024-09-13T00:18:59.291714+0000 mon.a (mon.0) 2262 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:19:00.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:18:59 smithi136 bash[75164]: audit 2024-09-13T00:18:59.299476+0000 mon.a (mon.0) 2263 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:19:00.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:18:59 smithi136 bash[76371]: audit 2024-09-13T00:18:59.290209+0000 mon.a (mon.0) 2261 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:19:00.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:18:59 smithi136 bash[76371]: audit 2024-09-13T00:18:59.291714+0000 mon.a (mon.0) 2262 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:19:00.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:18:59 smithi136 bash[76371]: audit 2024-09-13T00:18:59.299476+0000 mon.a (mon.0) 2263 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:19:00.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:18:59 smithi181 bash[67587]: audit 2024-09-13T00:18:59.290209+0000 mon.a (mon.0) 2261 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:19:00.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:18:59 smithi181 bash[67587]: audit 2024-09-13T00:18:59.291714+0000 mon.a (mon.0) 2262 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:19:00.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:18:59 smithi181 bash[67587]: audit 2024-09-13T00:18:59.299476+0000 mon.a (mon.0) 2263 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:19:00.533 INFO:tasks.workunit.client.0.smithi136.stdout: 1024: throughput=5MB/sec pending data=0 2024-09-13T00:19:00.534 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj--Ufpv_WFNtSPR7m off=4405427 len=2091103 2024-09-13T00:19:00.539 INFO:tasks.workunit.client.0.smithi136.stdout:op 5356 completed, throughput=5MB/sec 2024-09-13T00:19:00.539 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-zlEaRTxXkzHcgXS off=6779424 len=743250 2024-09-13T00:19:00.547 INFO:tasks.workunit.client.0.smithi136.stdout:op 5357 completed, throughput=5MB/sec 2024-09-13T00:19:00.547 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-EwY6u70WqkHL0JP off=2905446 len=813348 2024-09-13T00:19:00.555 INFO:tasks.workunit.client.0.smithi136.stdout:op 5358 completed, throughput=5MB/sec 2024-09-13T00:19:00.555 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-TvEF5ZWebUASGAc off=1296229 len=474196 2024-09-13T00:19:00.562 INFO:tasks.workunit.client.0.smithi136.stdout:op 5359 completed, throughput=5MB/sec 2024-09-13T00:19:00.562 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-SOp0o8wH9fGICJn off=2080837 len=1333308 2024-09-13T00:19:00.567 INFO:tasks.workunit.client.0.smithi136.stdout:op 5360 completed, throughput=5MB/sec 2024-09-13T00:19:01.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:19:00 smithi136 bash[75164]: cluster 2024-09-13T00:18:59.671229+0000 mgr.y (mgr.24661) 1256 : cluster [DBG] pgmap v1939: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:19:01.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:19:00 smithi136 bash[76371]: cluster 2024-09-13T00:18:59.671229+0000 mgr.y (mgr.24661) 1256 : cluster [DBG] pgmap v1939: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:19:01.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:19:00 smithi181 bash[67587]: cluster 2024-09-13T00:18:59.671229+0000 mgr.y (mgr.24661) 1256 : cluster [DBG] pgmap v1939: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:19:01.567 INFO:tasks.workunit.client.0.smithi136.stdout: 1025: throughput=5MB/sec pending data=0 2024-09-13T00:19:01.567 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-I-AE1CZdL-nlrmx off=3829333 len=2091788 2024-09-13T00:19:01.572 INFO:tasks.workunit.client.0.smithi136.stdout:op 5361 completed, throughput=5MB/sec 2024-09-13T00:19:01.572 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-KSCchZdWbyYOFxa off=7108281 len=422496 2024-09-13T00:19:01.575 INFO:tasks.workunit.client.0.smithi136.stdout:op 5362 completed, throughput=5MB/sec 2024-09-13T00:19:01.575 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-5HUYQyKvhTWceLQ off=306613 len=1042022 2024-09-13T00:19:01.578 INFO:tasks.workunit.client.0.smithi136.stdout:op 5363 completed, throughput=5MB/sec 2024-09-13T00:19:01.578 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-9uwJv07hicNOzbh off=20620873 len=623536 2024-09-13T00:19:01.585 INFO:tasks.workunit.client.0.smithi136.stdout:op 5364 completed, throughput=5MB/sec 2024-09-13T00:19:01.585 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-mqTGnrPwuTgF0TZ off=8491126 len=1363276 2024-09-13T00:19:01.588 INFO:tasks.workunit.client.0.smithi136.stdout:op 5365 completed, throughput=5MB/sec 2024-09-13T00:19:02.588 INFO:tasks.workunit.client.0.smithi136.stdout: 1026: throughput=5MB/sec pending data=0 2024-09-13T00:19:02.589 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-SRUR8_KDovn_S3O off=12878348 len=639346 2024-09-13T00:19:02.591 INFO:tasks.workunit.client.0.smithi136.stdout:op 5366 completed, throughput=5MB/sec 2024-09-13T00:19:02.591 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-TVgNMg7koRYKZXx off=8694951 len=1716966 2024-09-13T00:19:02.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:19:02 smithi181 bash[95505]: ts=2024-09-13T00:19:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:19:02.602 INFO:tasks.workunit.client.0.smithi136.stdout:op 5367 completed, throughput=5MB/sec 2024-09-13T00:19:02.602 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-MUC8yOpcr7-mg3Z off=12095601 len=1093307 2024-09-13T00:19:02.606 INFO:tasks.workunit.client.0.smithi136.stdout:op 5368 completed, throughput=5MB/sec 2024-09-13T00:19:02.606 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-10pJMw-Yh0wvWsJ off=294382 len=1535805 2024-09-13T00:19:02.611 INFO:tasks.workunit.client.0.smithi136.stdout:op 5369 completed, throughput=5MB/sec 2024-09-13T00:19:03.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:19:02 smithi136 bash[75164]: cluster 2024-09-13T00:19:01.672299+0000 mgr.y (mgr.24661) 1257 : cluster [DBG] pgmap v1940: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:19:03.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:19:02 smithi136 bash[76371]: cluster 2024-09-13T00:19:01.672299+0000 mgr.y (mgr.24661) 1257 : cluster [DBG] pgmap v1940: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:19:03.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:19:02 smithi181 bash[67587]: cluster 2024-09-13T00:19:01.672299+0000 mgr.y (mgr.24661) 1257 : cluster [DBG] pgmap v1940: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:19:03.612 INFO:tasks.workunit.client.0.smithi136.stdout: 1027: throughput=5MB/sec pending data=0 2024-09-13T00:19:03.612 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-bM4UiN8W418Dji4 off=162301 len=1066757 2024-09-13T00:19:03.621 INFO:tasks.workunit.client.0.smithi136.stdout:op 5370 completed, throughput=5MB/sec 2024-09-13T00:19:03.621 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-nDUeadIwRUUAOlp off=4908509 len=223689 2024-09-13T00:19:03.627 INFO:tasks.workunit.client.0.smithi136.stdout:op 5371 completed, throughput=5MB/sec 2024-09-13T00:19:03.627 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-KpFrri0fFFY4LQT off=11866791 len=1261003 2024-09-13T00:19:03.630 INFO:tasks.workunit.client.0.smithi136.stdout:op 5372 completed, throughput=5MB/sec 2024-09-13T00:19:03.630 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-LEJPi51X-5DmKeM off=159842 len=1710990 2024-09-13T00:19:03.634 INFO:tasks.workunit.client.0.smithi136.stdout:op 5373 completed, throughput=5MB/sec 2024-09-13T00:19:03.634 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-0VZo0JFS-0ORS1f off=4178812 len=134577 2024-09-13T00:19:03.635 INFO:tasks.workunit.client.0.smithi136.stdout:op 5374 completed, throughput=5MB/sec 2024-09-13T00:19:03.636 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-QDTqeZrxRXHqhfX off=6979016 len=882376 2024-09-13T00:19:03.644 INFO:tasks.workunit.client.0.smithi136.stdout:op 5375 completed, throughput=5MB/sec 2024-09-13T00:19:04.645 INFO:tasks.workunit.client.0.smithi136.stdout: 1028: throughput=5MB/sec pending data=0 2024-09-13T00:19:04.645 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-S83rOVmVAWFB8pJ off=12117705 len=1739936 2024-09-13T00:19:04.651 INFO:tasks.workunit.client.0.smithi136.stdout:op 5376 completed, throughput=5MB/sec 2024-09-13T00:19:04.651 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-znrZBPXADDrwPb3 off=6613261 len=340979 2024-09-13T00:19:04.652 INFO:tasks.workunit.client.0.smithi136.stdout:op 5377 completed, throughput=5MB/sec 2024-09-13T00:19:04.652 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-4tyuoa1Ci1bVtGk off=8152297 len=1659953 2024-09-13T00:19:04.664 INFO:tasks.workunit.client.0.smithi136.stdout:op 5378 completed, throughput=5MB/sec 2024-09-13T00:19:04.664 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-0oNKbM21jQNC2Rs off=619249 len=876763 2024-09-13T00:19:04.671 INFO:tasks.workunit.client.0.smithi136.stdout:op 5379 completed, throughput=5MB/sec 2024-09-13T00:19:04.671 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-AqS1K0OqX5bt1vM off=20288892 len=1913984 2024-09-13T00:19:04.687 INFO:tasks.workunit.client.0.smithi136.stdout:op 5380 completed, throughput=5MB/sec 2024-09-13T00:19:05.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:19:04 smithi136 bash[75164]: cluster 2024-09-13T00:19:03.673393+0000 mgr.y (mgr.24661) 1258 : cluster [DBG] pgmap v1941: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:19:05.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:19:04 smithi136 bash[76371]: cluster 2024-09-13T00:19:03.673393+0000 mgr.y (mgr.24661) 1258 : cluster [DBG] pgmap v1941: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:19:05.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:19:04 smithi181 bash[67587]: cluster 2024-09-13T00:19:03.673393+0000 mgr.y (mgr.24661) 1258 : cluster [DBG] pgmap v1941: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:19:05.687 INFO:tasks.workunit.client.0.smithi136.stdout: 1029: throughput=5MB/sec pending data=0 2024-09-13T00:19:05.688 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-T7-CN-aR22eHXF1 off=2193006 len=538653 2024-09-13T00:19:05.694 INFO:tasks.workunit.client.0.smithi136.stdout:op 5381 completed, throughput=5MB/sec 2024-09-13T00:19:05.695 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-qZFOIxz030TjY-i off=10273349 len=821009 2024-09-13T00:19:05.702 INFO:tasks.workunit.client.0.smithi136.stdout:op 5382 completed, throughput=5MB/sec 2024-09-13T00:19:05.702 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-niepa1QFgQhe3XW off=869581 len=884340 2024-09-13T00:19:05.709 INFO:tasks.workunit.client.0.smithi136.stdout:op 5383 completed, throughput=5MB/sec 2024-09-13T00:19:05.709 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-sQrSKBMovEk0AHA off=5465530 len=2060648 2024-09-13T00:19:05.716 INFO:tasks.workunit.client.0.smithi136.stdout:op 5384 completed, throughput=5MB/sec 2024-09-13T00:19:06.716 INFO:tasks.workunit.client.0.smithi136.stdout: 1030: throughput=5MB/sec pending data=0 2024-09-13T00:19:06.716 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-oEkIt6pua5Lskwl off=12679581 len=364716 2024-09-13T00:19:06.722 INFO:tasks.workunit.client.0.smithi136.stdout:op 5385 completed, throughput=5MB/sec 2024-09-13T00:19:06.722 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-tbbqJHU0iTQRbO_ off=1974426 len=975224 2024-09-13T00:19:06.724 INFO:tasks.workunit.client.0.smithi136.stdout:op 5386 completed, throughput=5MB/sec 2024-09-13T00:19:06.724 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-DVnWP9AREF0V-If off=2185306 len=334944 2024-09-13T00:19:06.729 INFO:tasks.workunit.client.0.smithi136.stdout:op 5387 completed, throughput=5MB/sec 2024-09-13T00:19:06.729 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-t4XZ3aHacPD__kM off=10605854 len=1310656 2024-09-13T00:19:06.740 INFO:tasks.workunit.client.0.smithi136.stdout:op 5388 completed, throughput=5MB/sec 2024-09-13T00:19:06.740 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-V-AynF5bVaqdo_m off=10883130 len=976270 2024-09-13T00:19:06.748 INFO:tasks.workunit.client.0.smithi136.stdout:op 5389 completed, throughput=5MB/sec 2024-09-13T00:19:06.748 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-i7PN3UmAyd36q4M off=4190538 len=1892718 2024-09-13T00:19:06.752 INFO:tasks.workunit.client.0.smithi136.stdout:op 5390 completed, throughput=5MB/sec 2024-09-13T00:19:07.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:19:06 smithi136 bash[75164]: cluster 2024-09-13T00:19:05.674300+0000 mgr.y (mgr.24661) 1259 : cluster [DBG] pgmap v1942: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.7 MiB/s wr, 4 op/s 2024-09-13T00:19:07.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:19:06 smithi136 bash[76371]: cluster 2024-09-13T00:19:05.674300+0000 mgr.y (mgr.24661) 1259 : cluster [DBG] pgmap v1942: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.7 MiB/s wr, 4 op/s 2024-09-13T00:19:07.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:19:06 smithi181 bash[67587]: cluster 2024-09-13T00:19:05.674300+0000 mgr.y (mgr.24661) 1259 : cluster [DBG] pgmap v1942: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.7 MiB/s wr, 4 op/s 2024-09-13T00:19:07.753 INFO:tasks.workunit.client.0.smithi136.stdout: 1031: throughput=5MB/sec pending data=0 2024-09-13T00:19:07.753 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Nq7PadkmezlPnjn off=6950574 len=863721 2024-09-13T00:19:07.760 INFO:tasks.workunit.client.0.smithi136.stdout:op 5391 completed, throughput=5MB/sec 2024-09-13T00:19:07.760 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-F_xzSQ5DOH35VGd off=1699746 len=1123670 2024-09-13T00:19:07.763 INFO:tasks.workunit.client.0.smithi136.stdout:op 5392 completed, throughput=5MB/sec 2024-09-13T00:19:07.763 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-GIHwOF46tLr8-Mw off=5749350 len=160645 2024-09-13T00:19:07.768 INFO:tasks.workunit.client.0.smithi136.stdout:op 5393 completed, throughput=5MB/sec 2024-09-13T00:19:07.768 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-wlYs-kDUGI9nO_G off=6135794 len=1679573 2024-09-13T00:19:07.783 INFO:tasks.workunit.client.0.smithi136.stdout:op 5394 completed, throughput=5MB/sec 2024-09-13T00:19:07.783 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-6e79ymI0bRIPx5r off=22585424 len=775266 2024-09-13T00:19:07.791 INFO:tasks.workunit.client.0.smithi136.stdout:op 5395 completed, throughput=5MB/sec 2024-09-13T00:19:08.791 INFO:tasks.workunit.client.0.smithi136.stdout: 1032: throughput=5MB/sec pending data=0 2024-09-13T00:19:08.792 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-P4tQOwFIVbZiVVF off=2901371 len=453634 2024-09-13T00:19:08.798 INFO:tasks.workunit.client.0.smithi136.stdout:op 5396 completed, throughput=5MB/sec 2024-09-13T00:19:08.798 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-CP-Ll5dglk0PuGh off=0 len=1278439 2024-09-13T00:19:08.802 INFO:tasks.workunit.client.0.smithi136.stdout:op 5397 completed, throughput=5MB/sec 2024-09-13T00:19:08.802 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-nMH-q4JPIxxWDvA off=8930167 len=695470 2024-09-13T00:19:08.810 INFO:tasks.workunit.client.0.smithi136.stdout:op 5398 completed, throughput=5MB/sec 2024-09-13T00:19:08.810 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-uWjsg-eTCuGszIH off=461203 len=251470 2024-09-13T00:19:08.811 INFO:tasks.workunit.client.0.smithi136.stdout:op 5399 completed, throughput=5MB/sec 2024-09-13T00:19:08.811 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-lfYndaDCwM8kK1l off=6750930 len=1492292 2024-09-13T00:19:08.815 INFO:tasks.workunit.client.0.smithi136.stdout:op 5400 completed, throughput=5MB/sec 2024-09-13T00:19:08.815 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-RxD3VrDJ_pyRRj7 off=0 len=1151561 2024-09-13T00:19:08.818 INFO:tasks.workunit.client.0.smithi136.stdout:op 5401 completed, throughput=5MB/sec 2024-09-13T00:19:08.818 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-RNuvMAm4Ia8QT9j off=13616305 len=819409 2024-09-13T00:19:08.826 INFO:tasks.workunit.client.0.smithi136.stdout:op 5402 completed, throughput=5MB/sec 2024-09-13T00:19:09.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:19:08 smithi136 bash[75164]: cluster 2024-09-13T00:19:07.675323+0000 mgr.y (mgr.24661) 1260 : cluster [DBG] pgmap v1943: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.1 MiB/s wr, 5 op/s 2024-09-13T00:19:09.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:19:08 smithi136 bash[75164]: audit 2024-09-13T00:19:08.801625+0000 mon.a (mon.0) 2264 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:19:09.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:19:08 smithi136 bash[76371]: cluster 2024-09-13T00:19:07.675323+0000 mgr.y (mgr.24661) 1260 : cluster [DBG] pgmap v1943: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.1 MiB/s wr, 5 op/s 2024-09-13T00:19:09.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:19:08 smithi136 bash[76371]: audit 2024-09-13T00:19:08.801625+0000 mon.a (mon.0) 2264 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:19:09.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:19:08 smithi181 bash[67587]: cluster 2024-09-13T00:19:07.675323+0000 mgr.y (mgr.24661) 1260 : cluster [DBG] pgmap v1943: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.1 MiB/s wr, 5 op/s 2024-09-13T00:19:09.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:19:08 smithi181 bash[67587]: audit 2024-09-13T00:19:08.801625+0000 mon.a (mon.0) 2264 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:19:09.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:19:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:19:09] "GET /metrics HTTP/1.1" 200 34137 "" "Prometheus/2.43.0" 2024-09-13T00:19:09.826 INFO:tasks.workunit.client.0.smithi136.stdout: 1033: throughput=5MB/sec pending data=0 2024-09-13T00:19:09.826 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-K-9RmwlmTp_0OLU off=21653730 len=501255 2024-09-13T00:19:09.832 INFO:tasks.workunit.client.0.smithi136.stdout:op 5403 completed, throughput=5MB/sec 2024-09-13T00:19:09.832 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-N8AtkaGQS_aY-Dd off=4531889 len=85970 2024-09-13T00:19:09.836 INFO:tasks.workunit.client.0.smithi136.stdout:op 5404 completed, throughput=5MB/sec 2024-09-13T00:19:09.836 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-xz29u9C8Zqsv31Q off=14690056 len=1602922 2024-09-13T00:19:09.840 INFO:tasks.workunit.client.0.smithi136.stdout:op 5405 completed, throughput=5MB/sec 2024-09-13T00:19:09.840 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-t0xQc2BUZFpNaT5 off=2428295 len=314682 2024-09-13T00:19:09.842 INFO:tasks.workunit.client.0.smithi136.stdout:op 5406 completed, throughput=5MB/sec 2024-09-13T00:19:09.842 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-pr2OcPLXkck4mX0 off=11406915 len=1126509 2024-09-13T00:19:09.853 INFO:tasks.workunit.client.0.smithi136.stdout:op 5407 completed, throughput=5MB/sec 2024-09-13T00:19:09.853 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-LW4q0y_4BX8iUec off=1736315 len=696037 2024-09-13T00:19:09.860 INFO:tasks.workunit.client.0.smithi136.stdout:op 5408 completed, throughput=5MB/sec 2024-09-13T00:19:09.861 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ndj6DmOhb9b2tDD off=2338544 len=1975782 2024-09-13T00:19:09.867 INFO:tasks.workunit.client.0.smithi136.stdout:op 5409 completed, throughput=5MB/sec 2024-09-13T00:19:10.867 INFO:tasks.workunit.client.0.smithi136.stdout: 1034: throughput=5MB/sec pending data=0 2024-09-13T00:19:10.868 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-T3cvv2hKTpcufCS off=417342 len=1876575 2024-09-13T00:19:10.872 INFO:tasks.workunit.client.0.smithi136.stdout:op 5410 completed, throughput=5MB/sec 2024-09-13T00:19:10.872 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-37pTbSLDQycChCy off=3359814 len=791783 2024-09-13T00:19:10.875 INFO:tasks.workunit.client.0.smithi136.stdout:op 5411 completed, throughput=5MB/sec 2024-09-13T00:19:10.875 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-xzWGnQbRPOJ8Tdu off=841770 len=668221 2024-09-13T00:19:10.882 INFO:tasks.workunit.client.0.smithi136.stdout:op 5412 completed, throughput=5MB/sec 2024-09-13T00:19:10.882 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-7hbv_kAVvAzGHH_ off=1960745 len=427128 2024-09-13T00:19:10.884 INFO:tasks.workunit.client.0.smithi136.stdout:op 5413 completed, throughput=5MB/sec 2024-09-13T00:19:11.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:19:10 smithi136 bash[76371]: cluster 2024-09-13T00:19:09.676147+0000 mgr.y (mgr.24661) 1261 : cluster [DBG] pgmap v1944: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:19:11.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:19:10 smithi136 bash[75164]: cluster 2024-09-13T00:19:09.676147+0000 mgr.y (mgr.24661) 1261 : cluster [DBG] pgmap v1944: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:19:11.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:19:10 smithi181 bash[67587]: cluster 2024-09-13T00:19:09.676147+0000 mgr.y (mgr.24661) 1261 : cluster [DBG] pgmap v1944: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:19:11.884 INFO:tasks.workunit.client.0.smithi136.stdout: 1035: throughput=5MB/sec pending data=0 2024-09-13T00:19:11.884 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-DQmHWy15N2BsZ6w off=15710458 len=1476899 2024-09-13T00:19:11.889 INFO:tasks.workunit.client.0.smithi136.stdout:op 5414 completed, throughput=5MB/sec 2024-09-13T00:19:11.889 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-OMtOU6Nw_GqnGjr off=216716 len=1899161 2024-09-13T00:19:11.902 INFO:tasks.workunit.client.0.smithi136.stdout:op 5415 completed, throughput=5MB/sec 2024-09-13T00:19:11.902 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-NL0axwmZq7tO4pZ off=2068337 len=860929 2024-09-13T00:19:11.910 INFO:tasks.workunit.client.0.smithi136.stdout:op 5416 completed, throughput=5MB/sec 2024-09-13T00:19:11.910 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-rE31NFdOnxc0wfh off=1065805 len=934400 2024-09-13T00:19:11.912 INFO:tasks.workunit.client.0.smithi136.stdout:op 5417 completed, throughput=5MB/sec 2024-09-13T00:19:11.912 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2cxT6af1XZ3O6Pm off=7547728 len=2002676 2024-09-13T00:19:11.919 INFO:tasks.workunit.client.0.smithi136.stdout:op 5418 completed, throughput=5MB/sec 2024-09-13T00:19:12.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:19:12 smithi181 bash[95505]: ts=2024-09-13T00:19:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:19:12.919 INFO:tasks.workunit.client.0.smithi136.stdout: 1036: throughput=5MB/sec pending data=0 2024-09-13T00:19:12.920 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-R5bTjeyU_zWS5CV off=1432885 len=744468 2024-09-13T00:19:12.928 INFO:tasks.workunit.client.0.smithi136.stdout:op 5419 completed, throughput=5MB/sec 2024-09-13T00:19:12.928 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-s2tRTbowonTv7dD off=11856888 len=215609 2024-09-13T00:19:12.933 INFO:tasks.workunit.client.0.smithi136.stdout:op 5420 completed, throughput=5MB/sec 2024-09-13T00:19:12.933 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-oeDCh92odslPmLR off=219700 len=1713492 2024-09-13T00:19:12.945 INFO:tasks.workunit.client.0.smithi136.stdout:op 5421 completed, throughput=5MB/sec 2024-09-13T00:19:12.945 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-tD5oSS_Ayc8zjOz off=5086332 len=1958985 2024-09-13T00:19:12.949 INFO:tasks.workunit.client.0.smithi136.stdout:op 5422 completed, throughput=5MB/sec 2024-09-13T00:19:13.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:19:12 smithi136 bash[76371]: cluster 2024-09-13T00:19:11.677183+0000 mgr.y (mgr.24661) 1262 : cluster [DBG] pgmap v1945: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:19:13.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:19:12 smithi136 bash[75164]: cluster 2024-09-13T00:19:11.677183+0000 mgr.y (mgr.24661) 1262 : cluster [DBG] pgmap v1945: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:19:13.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:19:12 smithi181 bash[67587]: cluster 2024-09-13T00:19:11.677183+0000 mgr.y (mgr.24661) 1262 : cluster [DBG] pgmap v1945: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:19:13.950 INFO:tasks.workunit.client.0.smithi136.stdout: 1037: throughput=5MB/sec pending data=0 2024-09-13T00:19:13.950 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-QN5KebFIqTJD_fp off=2492207 len=1234503 2024-09-13T00:19:13.959 INFO:tasks.workunit.client.0.smithi136.stdout:op 5423 completed, throughput=5MB/sec 2024-09-13T00:19:13.959 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-HrE4ww0c2bLEwtv off=4887240 len=758466 2024-09-13T00:19:13.966 INFO:tasks.workunit.client.0.smithi136.stdout:op 5424 completed, throughput=5MB/sec 2024-09-13T00:19:13.966 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-qmaxDiy139oOD1h off=8719879 len=1161516 2024-09-13T00:19:13.969 INFO:tasks.workunit.client.0.smithi136.stdout:op 5425 completed, throughput=5MB/sec 2024-09-13T00:19:13.969 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj--5Cf0-j17A-QSK1 off=7319270 len=449910 2024-09-13T00:19:13.976 INFO:tasks.workunit.client.0.smithi136.stdout:op 5426 completed, throughput=5MB/sec 2024-09-13T00:19:13.976 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-jAQRTMgA7YB3WJz off=2409297 len=2050779 2024-09-13T00:19:13.993 INFO:tasks.workunit.client.0.smithi136.stdout:op 5427 completed, throughput=5MB/sec 2024-09-13T00:19:14.993 INFO:tasks.workunit.client.0.smithi136.stdout: 1038: throughput=5MB/sec pending data=0 2024-09-13T00:19:14.993 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ZVIeG8gWN7DwyhG off=1146558 len=83887 2024-09-13T00:19:14.998 INFO:tasks.workunit.client.0.smithi136.stdout:op 5428 completed, throughput=5MB/sec 2024-09-13T00:19:14.998 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-06CQXeQ-5YApFcD off=4019348 len=1533335 2024-09-13T00:19:15.011 INFO:tasks.workunit.client.0.smithi136.stdout:op 5429 completed, throughput=5MB/sec 2024-09-13T00:19:15.011 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-XoVLzBxKsDMxt9v off=2229708 len=930885 2024-09-13T00:19:15.014 INFO:tasks.workunit.client.0.smithi136.stdout:op 5430 completed, throughput=5MB/sec 2024-09-13T00:19:15.014 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ywl3KqikKBimwDA off=15286824 len=393657 2024-09-13T00:19:15.015 INFO:tasks.workunit.client.0.smithi136.stdout:op 5431 completed, throughput=5MB/sec 2024-09-13T00:19:15.015 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-iYy4D7HEnYlYOhY off=7146989 len=1170117 2024-09-13T00:19:15.018 INFO:tasks.workunit.client.0.smithi136.stdout:op 5432 completed, throughput=5MB/sec 2024-09-13T00:19:15.018 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-JZgyrW02OkPaaO3 off=6412713 len=1533100 2024-09-13T00:19:15.022 INFO:tasks.workunit.client.0.smithi136.stdout:op 5433 completed, throughput=5MB/sec 2024-09-13T00:19:15.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:19:14 smithi136 bash[76371]: cluster 2024-09-13T00:19:13.678250+0000 mgr.y (mgr.24661) 1263 : cluster [DBG] pgmap v1946: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:19:15.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:19:14 smithi136 bash[75164]: cluster 2024-09-13T00:19:13.678250+0000 mgr.y (mgr.24661) 1263 : cluster [DBG] pgmap v1946: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:19:15.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:19:14 smithi181 bash[67587]: cluster 2024-09-13T00:19:13.678250+0000 mgr.y (mgr.24661) 1263 : cluster [DBG] pgmap v1946: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:19:16.022 INFO:tasks.workunit.client.0.smithi136.stdout: 1039: throughput=5MB/sec pending data=0 2024-09-13T00:19:16.023 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Z_DzpAkOwxrPFNe off=3274725 len=781819 2024-09-13T00:19:16.032 INFO:tasks.workunit.client.0.smithi136.stdout:op 5434 completed, throughput=5MB/sec 2024-09-13T00:19:16.032 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-O0RRXbIYd8yjruT off=17843656 len=681310 2024-09-13T00:19:16.039 INFO:tasks.workunit.client.0.smithi136.stdout:op 5435 completed, throughput=5MB/sec 2024-09-13T00:19:16.039 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-DFg6lEMjs2-fvhu off=7075357 len=108133 2024-09-13T00:19:16.044 INFO:tasks.workunit.client.0.smithi136.stdout:op 5436 completed, throughput=5MB/sec 2024-09-13T00:19:16.044 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-4gxO1S-6qVPI3ct off=738603 len=1615307 2024-09-13T00:19:16.055 INFO:tasks.workunit.client.0.smithi136.stdout:op 5437 completed, throughput=5MB/sec 2024-09-13T00:19:16.055 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-dIHYsl2IvAXEh3f off=1522444 len=1142266 2024-09-13T00:19:16.063 INFO:tasks.workunit.client.0.smithi136.stdout:op 5438 completed, throughput=5MB/sec 2024-09-13T00:19:17.064 INFO:tasks.workunit.client.0.smithi136.stdout: 1040: throughput=5MB/sec pending data=0 2024-09-13T00:19:17.064 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-XZ2bRyO83VaURmB off=17884636 len=1404250 2024-09-13T00:19:17.067 INFO:tasks.workunit.client.0.smithi136.stdout:op 5439 completed, throughput=5MB/sec 2024-09-13T00:19:17.067 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Srl4d28vj5z-xX1 off=11372087 len=244672 2024-09-13T00:19:17.069 INFO:tasks.workunit.client.0.smithi136.stdout:op 5440 completed, throughput=5MB/sec 2024-09-13T00:19:17.069 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-KnbHSgwK4q_fe1u off=5583262 len=1383976 2024-09-13T00:19:17.080 INFO:tasks.workunit.client.0.smithi136.stdout:op 5441 completed, throughput=5MB/sec 2024-09-13T00:19:17.080 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-mSYtlP3ZyFKeJkA off=937080 len=2048582 2024-09-13T00:19:17.098 INFO:tasks.workunit.client.0.smithi136.stdout:op 5442 completed, throughput=5MB/sec 2024-09-13T00:19:17.098 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-PHxSxrIuumQz03U off=749634 len=1631279 2024-09-13T00:19:17.102 INFO:tasks.workunit.client.0.smithi136.stdout:op 5443 completed, throughput=5MB/sec 2024-09-13T00:19:17.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:19:16 smithi136 bash[76371]: cluster 2024-09-13T00:19:15.679191+0000 mgr.y (mgr.24661) 1264 : cluster [DBG] pgmap v1947: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:19:17.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:19:16 smithi136 bash[75164]: cluster 2024-09-13T00:19:15.679191+0000 mgr.y (mgr.24661) 1264 : cluster [DBG] pgmap v1947: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:19:17.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:19:16 smithi181 bash[67587]: cluster 2024-09-13T00:19:15.679191+0000 mgr.y (mgr.24661) 1264 : cluster [DBG] pgmap v1947: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:19:18.103 INFO:tasks.workunit.client.0.smithi136.stdout: 1041: throughput=5MB/sec pending data=0 2024-09-13T00:19:18.103 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-OwhYSs9m1W4aRXi off=975637 len=1519584 2024-09-13T00:19:18.114 INFO:tasks.workunit.client.0.smithi136.stdout:op 5444 completed, throughput=5MB/sec 2024-09-13T00:19:18.115 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-XgS2w5whA9G6rOf off=382783 len=1648515 2024-09-13T00:19:18.118 INFO:tasks.workunit.client.0.smithi136.stdout:op 5445 completed, throughput=5MB/sec 2024-09-13T00:19:18.118 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-BdsZNt2GU6kxlYE off=632323 len=860782 2024-09-13T00:19:18.122 INFO:tasks.workunit.client.0.smithi136.stdout:op 5446 completed, throughput=5MB/sec 2024-09-13T00:19:19.122 INFO:tasks.workunit.client.0.smithi136.stdout: 1042: throughput=5MB/sec pending data=0 2024-09-13T00:19:19.123 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-NdFkjHPI8QufATR off=6864540 len=1326486 2024-09-13T00:19:19.126 INFO:tasks.workunit.client.0.smithi136.stdout:op 5447 completed, throughput=5MB/sec 2024-09-13T00:19:19.126 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-HUMyz5h5LlHFGAi off=17922654 len=1789073 2024-09-13T00:19:19.141 INFO:tasks.workunit.client.0.smithi136.stdout:op 5448 completed, throughput=5MB/sec 2024-09-13T00:19:19.141 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-2BzaOiTWDnHxNvB off=2396566 len=633545 2024-09-13T00:19:19.148 INFO:tasks.workunit.client.0.smithi136.stdout:op 5449 completed, throughput=5MB/sec 2024-09-13T00:19:19.149 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ffDS0duCdlkKFUs off=15973289 len=891938 2024-09-13T00:19:19.151 INFO:tasks.workunit.client.0.smithi136.stdout:op 5450 completed, throughput=5MB/sec 2024-09-13T00:19:19.152 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-eCn7GnEEmWl1HvL off=1601300 len=308738 2024-09-13T00:19:19.157 INFO:tasks.workunit.client.0.smithi136.stdout:op 5451 completed, throughput=5MB/sec 2024-09-13T00:19:19.157 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-JGNStWKlhywFbxQ off=4176847 len=1693923 2024-09-13T00:19:19.162 INFO:tasks.workunit.client.0.smithi136.stdout:op 5452 completed, throughput=5MB/sec 2024-09-13T00:19:19.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:19:18 smithi136 bash[75164]: cluster 2024-09-13T00:19:17.680246+0000 mgr.y (mgr.24661) 1265 : cluster [DBG] pgmap v1948: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:19:19.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:19:18 smithi136 bash[76371]: cluster 2024-09-13T00:19:17.680246+0000 mgr.y (mgr.24661) 1265 : cluster [DBG] pgmap v1948: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:19:19.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:19:18 smithi181 bash[67587]: cluster 2024-09-13T00:19:17.680246+0000 mgr.y (mgr.24661) 1265 : cluster [DBG] pgmap v1948: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:19:19.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:19:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:19:19] "GET /metrics HTTP/1.1" 200 34137 "" "Prometheus/2.43.0" 2024-09-13T00:19:20.162 INFO:tasks.workunit.client.0.smithi136.stdout: 1043: throughput=5MB/sec pending data=0 2024-09-13T00:19:20.163 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-HR6BruNuZR0HDGH off=1346711 len=995554 2024-09-13T00:19:20.171 INFO:tasks.workunit.client.0.smithi136.stdout:op 5453 completed, throughput=5MB/sec 2024-09-13T00:19:20.171 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Px2dR68TKpdKXMt off=608311 len=1771510 2024-09-13T00:19:20.183 INFO:tasks.workunit.client.0.smithi136.stdout:op 5454 completed, throughput=5MB/sec 2024-09-13T00:19:20.183 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-S6gSiCl-dymv628 off=3160649 len=1633185 2024-09-13T00:19:20.189 INFO:tasks.workunit.client.0.smithi136.stdout:op 5455 completed, throughput=5MB/sec 2024-09-13T00:19:21.190 INFO:tasks.workunit.client.0.smithi136.stdout: 1044: throughput=5MB/sec pending data=0 2024-09-13T00:19:21.190 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-p8qgDRqYAMyVFCa off=298245 len=1231514 2024-09-13T00:19:21.193 INFO:tasks.workunit.client.0.smithi136.stdout:op 5456 completed, throughput=5MB/sec 2024-09-13T00:19:21.193 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-LyRunD-y_a66cCT off=570960 len=1752928 2024-09-13T00:19:21.209 INFO:tasks.workunit.client.0.smithi136.stdout:op 5457 completed, throughput=5MB/sec 2024-09-13T00:19:21.209 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-KJv-HFtDOjW49P5 off=4629298 len=1147516 2024-09-13T00:19:21.220 INFO:tasks.workunit.client.0.smithi136.stdout:op 5458 completed, throughput=5MB/sec 2024-09-13T00:19:21.220 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-z7LbxtdXV_q7oy5 off=5371655 len=1211612 2024-09-13T00:19:21.223 INFO:tasks.workunit.client.0.smithi136.stdout:op 5459 completed, throughput=5MB/sec 2024-09-13T00:19:21.258 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:19:20 smithi136 bash[75164]: cluster 2024-09-13T00:19:19.681091+0000 mgr.y (mgr.24661) 1266 : cluster [DBG] pgmap v1949: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:19:21.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:19:20 smithi136 bash[76371]: cluster 2024-09-13T00:19:19.681091+0000 mgr.y (mgr.24661) 1266 : cluster [DBG] pgmap v1949: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:19:21.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:19:20 smithi181 bash[67587]: cluster 2024-09-13T00:19:19.681091+0000 mgr.y (mgr.24661) 1266 : cluster [DBG] pgmap v1949: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:19:22.224 INFO:tasks.workunit.client.0.smithi136.stdout: 1045: throughput=5MB/sec pending data=0 2024-09-13T00:19:22.224 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-r-AM-OQ_jPSuUdI off=1274330 len=1114180 2024-09-13T00:19:22.227 INFO:tasks.workunit.client.0.smithi136.stdout:op 5460 completed, throughput=5MB/sec 2024-09-13T00:19:22.227 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-tWOfI2Tf14DAknH off=507393 len=494418 2024-09-13T00:19:22.229 INFO:tasks.workunit.client.0.smithi136.stdout:op 5461 completed, throughput=5MB/sec 2024-09-13T00:19:22.229 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-i1Hgv-uYOPwEtiZ off=2224318 len=45332 2024-09-13T00:19:22.230 INFO:tasks.workunit.client.0.smithi136.stdout:op 5462 completed, throughput=5MB/sec 2024-09-13T00:19:22.230 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-iR-TuX1snNKnf92 off=13512832 len=719622 2024-09-13T00:19:22.237 INFO:tasks.workunit.client.0.smithi136.stdout:op 5463 completed, throughput=5MB/sec 2024-09-13T00:19:22.237 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-S5btLKR2RInwVIY off=6808646 len=272392 2024-09-13T00:19:22.238 INFO:tasks.workunit.client.0.smithi136.stdout:op 5464 completed, throughput=5MB/sec 2024-09-13T00:19:22.238 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-HiO2ebjHSlqdnH- off=2672139 len=270328 2024-09-13T00:19:22.239 INFO:tasks.workunit.client.0.smithi136.stdout:op 5465 completed, throughput=5MB/sec 2024-09-13T00:19:22.239 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-WyPPj4O35vDQys2 off=12566423 len=1087427 2024-09-13T00:19:22.243 INFO:tasks.workunit.client.0.smithi136.stdout:op 5466 completed, throughput=5MB/sec 2024-09-13T00:19:22.243 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-sW33cqgcoJKVs47 off=7689321 len=68751 2024-09-13T00:19:22.244 INFO:tasks.workunit.client.0.smithi136.stdout:op 5467 completed, throughput=5MB/sec 2024-09-13T00:19:22.244 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Kp6R3uxSbyV1oHi off=11641240 len=247145 2024-09-13T00:19:22.245 INFO:tasks.workunit.client.0.smithi136.stdout:op 5468 completed, throughput=5MB/sec 2024-09-13T00:19:22.245 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj--IhbIMKftSlR1ni off=13770038 len=1810524 2024-09-13T00:19:22.260 INFO:tasks.workunit.client.0.smithi136.stdout:op 5469 completed, throughput=5MB/sec 2024-09-13T00:19:22.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:19:22 smithi181 bash[95505]: ts=2024-09-13T00:19:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:19:23.258 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:19:23 smithi136 bash[76371]: cluster 2024-09-13T00:19:21.682090+0000 mgr.y (mgr.24661) 1267 : cluster [DBG] pgmap v1950: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:19:23.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:19:23 smithi136 bash[75164]: cluster 2024-09-13T00:19:21.682090+0000 mgr.y (mgr.24661) 1267 : cluster [DBG] pgmap v1950: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:19:23.261 INFO:tasks.workunit.client.0.smithi136.stdout: 1046: throughput=5MB/sec pending data=0 2024-09-13T00:19:23.261 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-pCzyiQsOW2TvcXF off=3505092 len=1368904 2024-09-13T00:19:23.265 INFO:tasks.workunit.client.0.smithi136.stdout:op 5470 completed, throughput=5MB/sec 2024-09-13T00:19:23.265 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-dziPHcz6OJ__BZe off=24753879 len=819193 2024-09-13T00:19:23.273 INFO:tasks.workunit.client.0.smithi136.stdout:op 5471 completed, throughput=5MB/sec 2024-09-13T00:19:23.273 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-TQvWuK6a2T5pHt8 off=6456146 len=1949537 2024-09-13T00:19:23.278 INFO:tasks.workunit.client.0.smithi136.stdout:op 5472 completed, throughput=5MB/sec 2024-09-13T00:19:23.278 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-YlfxA7k4iLAXP29 off=349702 len=1152236 2024-09-13T00:19:23.286 INFO:tasks.workunit.client.0.smithi136.stdout:op 5473 completed, throughput=5MB/sec 2024-09-13T00:19:23.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:19:23 smithi181 bash[67587]: cluster 2024-09-13T00:19:21.682090+0000 mgr.y (mgr.24661) 1267 : cluster [DBG] pgmap v1950: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2024-09-13T00:19:24.286 INFO:tasks.workunit.client.0.smithi136.stdout: 1047: throughput=5MB/sec pending data=0 2024-09-13T00:19:24.286 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-1a3b1C89phwsk07 off=10223462 len=179294 2024-09-13T00:19:24.291 INFO:tasks.workunit.client.0.smithi136.stdout:op 5474 completed, throughput=5MB/sec 2024-09-13T00:19:24.291 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Mppy4pprx0eTGQj off=5460206 len=1329590 2024-09-13T00:19:24.302 INFO:tasks.workunit.client.0.smithi136.stdout:op 5475 completed, throughput=5MB/sec 2024-09-13T00:19:24.302 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-bXj95Q_QoqGPpDc off=5574540 len=1403069 2024-09-13T00:19:24.311 INFO:tasks.workunit.client.0.smithi136.stdout:op 5476 completed, throughput=5MB/sec 2024-09-13T00:19:24.311 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-pP_1PWS98O17e8N off=9003342 len=1198640 2024-09-13T00:19:24.320 INFO:tasks.workunit.client.0.smithi136.stdout:op 5477 completed, throughput=5MB/sec 2024-09-13T00:19:24.320 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2OOm0lf5MzpAW-S off=12835220 len=68841 2024-09-13T00:19:24.321 INFO:tasks.workunit.client.0.smithi136.stdout:op 5478 completed, throughput=5MB/sec 2024-09-13T00:19:24.321 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-kAbXmGCV--fH3kF off=1137649 len=501029 2024-09-13T00:19:24.323 INFO:tasks.workunit.client.0.smithi136.stdout:op 5479 completed, throughput=5MB/sec 2024-09-13T00:19:24.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:19:24 smithi181 bash[67587]: audit 2024-09-13T00:19:23.801790+0000 mon.a (mon.0) 2265 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:19:24.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:19:24 smithi136 bash[75164]: audit 2024-09-13T00:19:23.801790+0000 mon.a (mon.0) 2265 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:19:24.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:19:24 smithi136 bash[76371]: audit 2024-09-13T00:19:23.801790+0000 mon.a (mon.0) 2265 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:19:25.323 INFO:tasks.workunit.client.0.smithi136.stdout: 1048: throughput=5MB/sec pending data=0 2024-09-13T00:19:25.323 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-vrSZ6diYSHEDi7o off=4668503 len=1438886 2024-09-13T00:19:25.327 INFO:tasks.workunit.client.0.smithi136.stdout:op 5480 completed, throughput=5MB/sec 2024-09-13T00:19:25.327 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-5DLfinzz6O5umZv off=6244729 len=1196257 2024-09-13T00:19:25.336 INFO:tasks.workunit.client.0.smithi136.stdout:op 5481 completed, throughput=5MB/sec 2024-09-13T00:19:25.336 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-xLbfYSweoQ9mPhZ off=12662303 len=697606 2024-09-13T00:19:25.338 INFO:tasks.workunit.client.0.smithi136.stdout:op 5482 completed, throughput=5MB/sec 2024-09-13T00:19:25.339 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-gupQi8xQKozMK4N off=2477264 len=1778636 2024-09-13T00:19:25.345 INFO:tasks.workunit.client.0.smithi136.stdout:op 5483 completed, throughput=5MB/sec 2024-09-13T00:19:25.345 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-qjlBQBTD-9BSo2U off=750393 len=1610889 2024-09-13T00:19:25.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:19:25 smithi181 bash[67587]: cluster 2024-09-13T00:19:23.683225+0000 mgr.y (mgr.24661) 1268 : cluster [DBG] pgmap v1951: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:19:25.349 INFO:tasks.workunit.client.0.smithi136.stdout:op 5484 completed, throughput=5MB/sec 2024-09-13T00:19:25.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:19:25 smithi136 bash[75164]: cluster 2024-09-13T00:19:23.683225+0000 mgr.y (mgr.24661) 1268 : cluster [DBG] pgmap v1951: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:19:25.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:19:25 smithi136 bash[76371]: cluster 2024-09-13T00:19:23.683225+0000 mgr.y (mgr.24661) 1268 : cluster [DBG] pgmap v1951: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:19:26.349 INFO:tasks.workunit.client.0.smithi136.stdout: 1049: throughput=5MB/sec pending data=0 2024-09-13T00:19:26.350 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-DqpfrgGuKq3OHCJ off=275165 len=543063 2024-09-13T00:19:26.357 INFO:tasks.workunit.client.0.smithi136.stdout:op 5485 completed, throughput=5MB/sec 2024-09-13T00:19:26.357 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-NO0CJB8czzgJGBy off=20042026 len=1305445 2024-09-13T00:19:26.360 INFO:tasks.workunit.client.0.smithi136.stdout:op 5486 completed, throughput=5MB/sec 2024-09-13T00:19:26.360 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-W7vSu8S-RNivl8q off=8980351 len=564984 2024-09-13T00:19:26.362 INFO:tasks.workunit.client.0.smithi136.stdout:op 5487 completed, throughput=5MB/sec 2024-09-13T00:19:26.362 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-iWGlQHL7biT8uP5 off=9793844 len=1430794 2024-09-13T00:19:26.373 INFO:tasks.workunit.client.0.smithi136.stdout:op 5488 completed, throughput=5MB/sec 2024-09-13T00:19:27.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:19:27 smithi181 bash[67587]: cluster 2024-09-13T00:19:25.684305+0000 mgr.y (mgr.24661) 1269 : cluster [DBG] pgmap v1952: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.0 MiB/s wr, 3 op/s 2024-09-13T00:19:27.373 INFO:tasks.workunit.client.0.smithi136.stdout: 1050: throughput=5MB/sec pending data=0 2024-09-13T00:19:27.373 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-2wtRPuRif54nAu7 off=6498432 len=1874497 2024-09-13T00:19:27.389 INFO:tasks.workunit.client.0.smithi136.stdout:op 5489 completed, throughput=5MB/sec 2024-09-13T00:19:27.389 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-755w9s4e46dn4y2 off=570250 len=1783133 2024-09-13T00:19:27.395 INFO:tasks.workunit.client.0.smithi136.stdout:op 5490 completed, throughput=5MB/sec 2024-09-13T00:19:27.395 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Nnq3eY5w-Sf-f9r off=10055541 len=1817398 2024-09-13T00:19:27.399 INFO:tasks.workunit.client.0.smithi136.stdout:op 5491 completed, throughput=5MB/sec 2024-09-13T00:19:27.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:19:27 smithi136 bash[75164]: cluster 2024-09-13T00:19:25.684305+0000 mgr.y (mgr.24661) 1269 : cluster [DBG] pgmap v1952: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.0 MiB/s wr, 3 op/s 2024-09-13T00:19:27.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:19:27 smithi136 bash[76371]: cluster 2024-09-13T00:19:25.684305+0000 mgr.y (mgr.24661) 1269 : cluster [DBG] pgmap v1952: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.0 MiB/s wr, 3 op/s 2024-09-13T00:19:28.400 INFO:tasks.workunit.client.0.smithi136.stdout: 1051: throughput=5MB/sec pending data=0 2024-09-13T00:19:28.400 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-LEJPi51X-5DmKeM off=8078492 len=426135 2024-09-13T00:19:28.407 INFO:tasks.workunit.client.0.smithi136.stdout:op 5492 completed, throughput=5MB/sec 2024-09-13T00:19:28.407 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-oxB8Rn58n04SJbr off=4827308 len=1081593 2024-09-13T00:19:28.410 INFO:tasks.workunit.client.0.smithi136.stdout:op 5493 completed, throughput=5MB/sec 2024-09-13T00:19:28.410 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-pa4FRXR6ht93JA1 off=17966663 len=458928 2024-09-13T00:19:28.411 INFO:tasks.workunit.client.0.smithi136.stdout:op 5494 completed, throughput=5MB/sec 2024-09-13T00:19:28.412 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-LXAeTAn_IRDGnYw off=1579397 len=148435 2024-09-13T00:19:28.412 INFO:tasks.workunit.client.0.smithi136.stdout:op 5495 completed, throughput=5MB/sec 2024-09-13T00:19:28.412 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-qjlBQBTD-9BSo2U off=187510 len=1650480 2024-09-13T00:19:28.424 INFO:tasks.workunit.client.0.smithi136.stdout:op 5496 completed, throughput=5MB/sec 2024-09-13T00:19:28.425 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-HHYxvvha2ghuiN3 off=980578 len=53065 2024-09-13T00:19:28.426 INFO:tasks.workunit.client.0.smithi136.stdout:op 5497 completed, throughput=5MB/sec 2024-09-13T00:19:28.426 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-K8hle4kJ-qX1NNQ off=16486162 len=725096 2024-09-13T00:19:28.427 INFO:tasks.workunit.client.0.smithi136.stdout:op 5498 completed, throughput=5MB/sec 2024-09-13T00:19:28.427 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-zcjHhlEIJu2pDen off=8840448 len=1305795 2024-09-13T00:19:28.433 INFO:tasks.workunit.client.0.smithi136.stdout:op 5499 completed, throughput=5MB/sec 2024-09-13T00:19:29.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:19:29 smithi181 bash[67587]: cluster 2024-09-13T00:19:27.685401+0000 mgr.y (mgr.24661) 1270 : cluster [DBG] pgmap v1953: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:19:29.412 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:19:29 smithi136 bash[76371]: cluster 2024-09-13T00:19:27.685401+0000 mgr.y (mgr.24661) 1270 : cluster [DBG] pgmap v1953: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:19:29.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:19:29 smithi136 bash[75164]: cluster 2024-09-13T00:19:27.685401+0000 mgr.y (mgr.24661) 1270 : cluster [DBG] pgmap v1953: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:19:29.433 INFO:tasks.workunit.client.0.smithi136.stdout: 1052: throughput=5MB/sec pending data=0 2024-09-13T00:19:29.433 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-R3GHeqGHSCvHQ25 off=4437654 len=1519006 2024-09-13T00:19:29.444 INFO:tasks.workunit.client.0.smithi136.stdout:op 5500 completed, throughput=5MB/sec 2024-09-13T00:19:29.444 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-f50cmAbu40nUIau off=21706732 len=911392 2024-09-13T00:19:29.453 INFO:tasks.workunit.client.0.smithi136.stdout:op 5501 completed, throughput=5MB/sec 2024-09-13T00:19:29.453 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Go3xAkQSuqsfxIy off=13625587 len=1096188 2024-09-13T00:19:29.464 INFO:tasks.workunit.client.0.smithi136.stdout:op 5502 completed, throughput=5MB/sec 2024-09-13T00:19:29.465 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-elB1ZnNr7nF6gbb off=12314617 len=1617384 2024-09-13T00:19:29.468 INFO:tasks.workunit.client.0.smithi136.stdout:op 5503 completed, throughput=5MB/sec 2024-09-13T00:19:29.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:19:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:19:29] "GET /metrics HTTP/1.1" 200 34143 "" "Prometheus/2.43.0" 2024-09-13T00:19:30.469 INFO:tasks.workunit.client.0.smithi136.stdout: 1053: throughput=5MB/sec pending data=0 2024-09-13T00:19:30.469 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-us-D_SGXStTDi4s off=2975854 len=1747889 2024-09-13T00:19:30.484 INFO:tasks.workunit.client.0.smithi136.stdout:op 5504 completed, throughput=5MB/sec 2024-09-13T00:19:30.484 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-YTBpqxE8YkAe5En off=770092 len=654804 2024-09-13T00:19:30.490 INFO:tasks.workunit.client.0.smithi136.stdout:op 5505 completed, throughput=5MB/sec 2024-09-13T00:19:30.491 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-C3DdKcNJ__VogVl off=5831148 len=371397 2024-09-13T00:19:30.496 INFO:tasks.workunit.client.0.smithi136.stdout:op 5506 completed, throughput=5MB/sec 2024-09-13T00:19:30.496 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-gaViLu7sz2lKlmD off=8945025 len=106387 2024-09-13T00:19:30.501 INFO:tasks.workunit.client.0.smithi136.stdout:op 5507 completed, throughput=5MB/sec 2024-09-13T00:19:30.501 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-3fXuWXlXcwrJd3h off=1875127 len=1243332 2024-09-13T00:19:30.504 INFO:tasks.workunit.client.0.smithi136.stdout:op 5508 completed, throughput=5MB/sec 2024-09-13T00:19:30.504 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Pm6Jm8XhiiQAwZE off=7596758 len=806403 2024-09-13T00:19:30.506 INFO:tasks.workunit.client.0.smithi136.stdout:op 5509 completed, throughput=5MB/sec 2024-09-13T00:19:30.506 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-y2VkzrOFqb8fPEj off=7415865 len=84097 2024-09-13T00:19:30.511 INFO:tasks.workunit.client.0.smithi136.stdout:op 5510 completed, throughput=5MB/sec 2024-09-13T00:19:30.511 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-8y4nmzz27iLGmgD off=2088690 len=852163 2024-09-13T00:19:30.515 INFO:tasks.workunit.client.0.smithi136.stdout:op 5511 completed, throughput=5MB/sec 2024-09-13T00:19:31.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:19:31 smithi181 bash[67587]: cluster 2024-09-13T00:19:29.686236+0000 mgr.y (mgr.24661) 1271 : cluster [DBG] pgmap v1954: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:19:31.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:19:31 smithi136 bash[75164]: cluster 2024-09-13T00:19:29.686236+0000 mgr.y (mgr.24661) 1271 : cluster [DBG] pgmap v1954: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:19:31.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:19:31 smithi136 bash[76371]: cluster 2024-09-13T00:19:29.686236+0000 mgr.y (mgr.24661) 1271 : cluster [DBG] pgmap v1954: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:19:31.515 INFO:tasks.workunit.client.0.smithi136.stdout: 1054: throughput=5MB/sec pending data=0 2024-09-13T00:19:31.515 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-E7yQ1-8cToF9Io1 off=93340 len=592474 2024-09-13T00:19:31.517 INFO:tasks.workunit.client.0.smithi136.stdout:op 5512 completed, throughput=5MB/sec 2024-09-13T00:19:31.517 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Qf3GpngbT0IaraW off=13348264 len=1594772 2024-09-13T00:19:31.532 INFO:tasks.workunit.client.0.smithi136.stdout:op 5513 completed, throughput=5MB/sec 2024-09-13T00:19:31.532 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-KfFD1UKN-kGTSxw off=4914106 len=568616 2024-09-13T00:19:31.539 INFO:tasks.workunit.client.0.smithi136.stdout:op 5514 completed, throughput=5MB/sec 2024-09-13T00:19:31.539 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Ii59CziXclZ-F1h off=1878657 len=1223136 2024-09-13T00:19:31.543 INFO:tasks.workunit.client.0.smithi136.stdout:op 5515 completed, throughput=5MB/sec 2024-09-13T00:19:31.543 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-UVkiljyD0d_tnPC off=9880673 len=1738290 2024-09-13T00:19:31.556 INFO:tasks.workunit.client.0.smithi136.stdout:op 5516 completed, throughput=5MB/sec 2024-09-13T00:19:32.556 INFO:tasks.workunit.client.0.smithi136.stdout: 1055: throughput=5MB/sec pending data=0 2024-09-13T00:19:32.557 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-xph9bzj-6YYkHFP off=5167563 len=2062656 2024-09-13T00:19:32.563 INFO:tasks.workunit.client.0.smithi136.stdout:op 5517 completed, throughput=5MB/sec 2024-09-13T00:19:32.563 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-SYIlJ2CC30GB1MJ off=3394728 len=289255 2024-09-13T00:19:32.565 INFO:tasks.workunit.client.0.smithi136.stdout:op 5518 completed, throughput=5MB/sec 2024-09-13T00:19:32.565 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-hPHr-lvFUMQ2LGz off=16170869 len=898044 2024-09-13T00:19:32.568 INFO:tasks.workunit.client.0.smithi136.stdout:op 5519 completed, throughput=5MB/sec 2024-09-13T00:19:32.568 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-j02Z7QasR6VPbM5 off=5671589 len=1542224 2024-09-13T00:19:32.581 INFO:tasks.workunit.client.0.smithi136.stdout:op 5520 completed, throughput=5MB/sec 2024-09-13T00:19:32.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:19:32 smithi181 bash[95505]: ts=2024-09-13T00:19:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:19:33.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:19:33 smithi181 bash[67587]: cluster 2024-09-13T00:19:31.687210+0000 mgr.y (mgr.24661) 1272 : cluster [DBG] pgmap v1955: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:19:33.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:19:33 smithi136 bash[75164]: cluster 2024-09-13T00:19:31.687210+0000 mgr.y (mgr.24661) 1272 : cluster [DBG] pgmap v1955: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:19:33.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:19:33 smithi136 bash[76371]: cluster 2024-09-13T00:19:31.687210+0000 mgr.y (mgr.24661) 1272 : cluster [DBG] pgmap v1955: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:19:33.581 INFO:tasks.workunit.client.0.smithi136.stdout: 1056: throughput=5MB/sec pending data=0 2024-09-13T00:19:33.581 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-kJwxDIqoZJjzSqZ off=13066294 len=882570 2024-09-13T00:19:33.584 INFO:tasks.workunit.client.0.smithi136.stdout:op 5521 completed, throughput=5MB/sec 2024-09-13T00:19:33.584 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-YHgbY7nQP98Z2IF off=4352921 len=1153546 2024-09-13T00:19:33.587 INFO:tasks.workunit.client.0.smithi136.stdout:op 5522 completed, throughput=5MB/sec 2024-09-13T00:19:33.587 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-PRZb9vMyzF6cizK off=4313372 len=502912 2024-09-13T00:19:33.589 INFO:tasks.workunit.client.0.smithi136.stdout:op 5523 completed, throughput=5MB/sec 2024-09-13T00:19:33.589 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-pqglykTD4LacG-V off=2451224 len=202823 2024-09-13T00:19:33.595 INFO:tasks.workunit.client.0.smithi136.stdout:op 5524 completed, throughput=5MB/sec 2024-09-13T00:19:33.595 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-1KjVJQR21MBlkT1 off=20971961 len=302099 2024-09-13T00:19:33.597 INFO:tasks.workunit.client.0.smithi136.stdout:op 5525 completed, throughput=5MB/sec 2024-09-13T00:19:33.597 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-4DamFa4n-dAVRag off=2770703 len=1324444 2024-09-13T00:19:33.606 INFO:tasks.workunit.client.0.smithi136.stdout:op 5526 completed, throughput=5MB/sec 2024-09-13T00:19:33.606 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-j9iosAgoT8riSwC off=2045957 len=722563 2024-09-13T00:19:33.609 INFO:tasks.workunit.client.0.smithi136.stdout:op 5527 completed, throughput=5MB/sec 2024-09-13T00:19:34.609 INFO:tasks.workunit.client.0.smithi136.stdout: 1057: throughput=5MB/sec pending data=0 2024-09-13T00:19:34.610 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-lXWq_jrO6y6dPQZ off=7514577 len=1125868 2024-09-13T00:19:34.619 INFO:tasks.workunit.client.0.smithi136.stdout:op 5528 completed, throughput=5MB/sec 2024-09-13T00:19:34.619 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-w0rS0S3CxlYer6p off=812277 len=1740946 2024-09-13T00:19:34.631 INFO:tasks.workunit.client.0.smithi136.stdout:op 5529 completed, throughput=5MB/sec 2024-09-13T00:19:34.631 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-YHgbY7nQP98Z2IF off=502981 len=214652 2024-09-13T00:19:34.636 INFO:tasks.workunit.client.0.smithi136.stdout:op 5530 completed, throughput=5MB/sec 2024-09-13T00:19:34.636 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-8tPUKcf5hHeRS6v off=14745671 len=1725449 2024-09-13T00:19:34.638 INFO:tasks.workunit.client.0.smithi136.stdout:op 5531 completed, throughput=5MB/sec 2024-09-13T00:19:34.638 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-zJQXhQ32rdivqC9 off=207983 len=1310960 2024-09-13T00:19:34.649 INFO:tasks.workunit.client.0.smithi136.stdout:op 5532 completed, throughput=5MB/sec 2024-09-13T00:19:35.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:19:35 smithi181 bash[67587]: cluster 2024-09-13T00:19:33.688284+0000 mgr.y (mgr.24661) 1273 : cluster [DBG] pgmap v1956: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:19:35.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:19:35 smithi136 bash[75164]: cluster 2024-09-13T00:19:33.688284+0000 mgr.y (mgr.24661) 1273 : cluster [DBG] pgmap v1956: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:19:35.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:19:35 smithi136 bash[76371]: cluster 2024-09-13T00:19:33.688284+0000 mgr.y (mgr.24661) 1273 : cluster [DBG] pgmap v1956: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:19:35.650 INFO:tasks.workunit.client.0.smithi136.stdout: 1058: throughput=5MB/sec pending data=0 2024-09-13T00:19:35.650 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-lyQcGNEF_q9bO0r off=1679971 len=1263310 2024-09-13T00:19:35.659 INFO:tasks.workunit.client.0.smithi136.stdout:op 5533 completed, throughput=5MB/sec 2024-09-13T00:19:35.659 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-k_QqjxLLEB_Et7h off=6860395 len=423996 2024-09-13T00:19:35.665 INFO:tasks.workunit.client.0.smithi136.stdout:op 5534 completed, throughput=5MB/sec 2024-09-13T00:19:35.665 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-oD_YzkSxgDmJGUL off=1561127 len=1000052 2024-09-13T00:19:35.673 INFO:tasks.workunit.client.0.smithi136.stdout:op 5535 completed, throughput=5MB/sec 2024-09-13T00:19:35.673 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-xKcGfAQWjQ3nfsr off=4288477 len=851551 2024-09-13T00:19:35.682 INFO:tasks.workunit.client.0.smithi136.stdout:op 5536 completed, throughput=5MB/sec 2024-09-13T00:19:35.682 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-LQ4Ou0lXFpWtoRj off=3049080 len=6163 2024-09-13T00:19:35.686 INFO:tasks.workunit.client.0.smithi136.stdout:op 5537 completed, throughput=5MB/sec 2024-09-13T00:19:35.686 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-9AWsbZkXySAVDrh off=9802677 len=1554628 2024-09-13T00:19:35.698 INFO:tasks.workunit.client.0.smithi136.stdout:op 5538 completed, throughput=5MB/sec 2024-09-13T00:19:36.698 INFO:tasks.workunit.client.0.smithi136.stdout: 1059: throughput=5MB/sec pending data=0 2024-09-13T00:19:36.699 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-5AASLKsRs8WxsIB off=3128278 len=767811 2024-09-13T00:19:36.705 INFO:tasks.workunit.client.0.smithi136.stdout:op 5539 completed, throughput=5MB/sec 2024-09-13T00:19:36.706 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-iPhU4UYIwQhUURI off=667898 len=26639 2024-09-13T00:19:36.710 INFO:tasks.workunit.client.0.smithi136.stdout:op 5540 completed, throughput=5MB/sec 2024-09-13T00:19:36.710 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-SFRm2REGF4vGumB off=933871 len=280689 2024-09-13T00:19:36.715 INFO:tasks.workunit.client.0.smithi136.stdout:op 5541 completed, throughput=5MB/sec 2024-09-13T00:19:36.715 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-S_8BD31bt6b-tc3 off=2074272 len=212312 2024-09-13T00:19:36.717 INFO:tasks.workunit.client.0.smithi136.stdout:op 5542 completed, throughput=5MB/sec 2024-09-13T00:19:36.717 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-72Fpwu03HMXib5X off=0 len=1323545 2024-09-13T00:19:36.721 INFO:tasks.workunit.client.0.smithi136.stdout:op 5543 completed, throughput=5MB/sec 2024-09-13T00:19:36.721 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-E8ZT-e3kuhbnJRq off=14084074 len=1396018 2024-09-13T00:19:36.731 INFO:tasks.workunit.client.0.smithi136.stdout:op 5544 completed, throughput=5MB/sec 2024-09-13T00:19:36.731 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-545pebVsAsPwlih off=5546633 len=476986 2024-09-13T00:19:36.737 INFO:tasks.workunit.client.0.smithi136.stdout:op 5545 completed, throughput=5MB/sec 2024-09-13T00:19:36.737 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-5ylZtj-YyKi1Z-D off=5187439 len=1815984 2024-09-13T00:19:36.743 INFO:tasks.workunit.client.0.smithi136.stdout:op 5546 completed, throughput=5MB/sec 2024-09-13T00:19:37.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:19:37 smithi181 bash[67587]: cluster 2024-09-13T00:19:35.689506+0000 mgr.y (mgr.24661) 1274 : cluster [DBG] pgmap v1957: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:19:37.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:19:37 smithi136 bash[75164]: cluster 2024-09-13T00:19:35.689506+0000 mgr.y (mgr.24661) 1274 : cluster [DBG] pgmap v1957: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:19:37.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:19:37 smithi136 bash[76371]: cluster 2024-09-13T00:19:35.689506+0000 mgr.y (mgr.24661) 1274 : cluster [DBG] pgmap v1957: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:19:37.743 INFO:tasks.workunit.client.0.smithi136.stdout: 1060: throughput=5MB/sec pending data=0 2024-09-13T00:19:37.743 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-yJCgtSxV3C1qLOS off=4277753 len=964619 2024-09-13T00:19:37.747 INFO:tasks.workunit.client.0.smithi136.stdout:op 5547 completed, throughput=5MB/sec 2024-09-13T00:19:37.747 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-W1n6fvHEk69dJ4E off=1140050 len=1730986 2024-09-13T00:19:37.754 INFO:tasks.workunit.client.0.smithi136.stdout:op 5548 completed, throughput=5MB/sec 2024-09-13T00:19:37.754 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Jnu-VlY8jaM34yL off=4129777 len=663883 2024-09-13T00:19:37.756 INFO:tasks.workunit.client.0.smithi136.stdout:op 5549 completed, throughput=5MB/sec 2024-09-13T00:19:37.756 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-RvqmAqI9ZijmbFa off=22464077 len=1343005 2024-09-13T00:19:37.767 INFO:tasks.workunit.client.0.smithi136.stdout:op 5550 completed, throughput=5MB/sec 2024-09-13T00:19:38.768 INFO:tasks.workunit.client.0.smithi136.stdout: 1061: throughput=5MB/sec pending data=0 2024-09-13T00:19:38.768 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-6mcsi-W8_Fk2sLm off=4940359 len=1298626 2024-09-13T00:19:38.771 INFO:tasks.workunit.client.0.smithi136.stdout:op 5551 completed, throughput=5MB/sec 2024-09-13T00:19:38.771 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-4XjX6qOJybbLu9h off=12385561 len=1420115 2024-09-13T00:19:38.782 INFO:tasks.workunit.client.0.smithi136.stdout:op 5552 completed, throughput=5MB/sec 2024-09-13T00:19:38.782 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj--RSy7krhktjmzoR off=3682073 len=1575372 2024-09-13T00:19:38.792 INFO:tasks.workunit.client.0.smithi136.stdout:op 5553 completed, throughput=5MB/sec 2024-09-13T00:19:38.792 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-FQXXAWE8_vqrr-W off=6525675 len=414251 2024-09-13T00:19:38.794 INFO:tasks.workunit.client.0.smithi136.stdout:op 5554 completed, throughput=5MB/sec 2024-09-13T00:19:38.794 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-EfXEz3NqAbnAwcG off=15449626 len=1097540 2024-09-13T00:19:38.803 INFO:tasks.workunit.client.0.smithi136.stdout:op 5555 completed, throughput=5MB/sec 2024-09-13T00:19:39.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:19:39 smithi181 bash[67587]: cluster 2024-09-13T00:19:37.690809+0000 mgr.y (mgr.24661) 1275 : cluster [DBG] pgmap v1958: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:19:39.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:19:39 smithi181 bash[67587]: audit 2024-09-13T00:19:38.802405+0000 mon.a (mon.0) 2266 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:19:39.412 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:19:39 smithi136 bash[75164]: cluster 2024-09-13T00:19:37.690809+0000 mgr.y (mgr.24661) 1275 : cluster [DBG] pgmap v1958: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:19:39.412 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:19:39 smithi136 bash[75164]: audit 2024-09-13T00:19:38.802405+0000 mon.a (mon.0) 2266 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:19:39.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:19:39 smithi136 bash[76371]: cluster 2024-09-13T00:19:37.690809+0000 mgr.y (mgr.24661) 1275 : cluster [DBG] pgmap v1958: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:19:39.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:19:39 smithi136 bash[76371]: audit 2024-09-13T00:19:38.802405+0000 mon.a (mon.0) 2266 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:19:39.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:19:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:19:39] "GET /metrics HTTP/1.1" 200 34141 "" "Prometheus/2.43.0" 2024-09-13T00:19:39.803 INFO:tasks.workunit.client.0.smithi136.stdout: 1062: throughput=5MB/sec pending data=0 2024-09-13T00:19:39.804 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-IAsgZ2wMUkiQ4Ho off=5598201 len=897224 2024-09-13T00:19:39.807 INFO:tasks.workunit.client.0.smithi136.stdout:op 5556 completed, throughput=5MB/sec 2024-09-13T00:19:39.807 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-J-V6cc1il2qS6ES off=4739432 len=1358642 2024-09-13T00:19:39.818 INFO:tasks.workunit.client.0.smithi136.stdout:op 5557 completed, throughput=5MB/sec 2024-09-13T00:19:39.818 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-H269m1cM8MOoi2A off=3982427 len=473830 2024-09-13T00:19:39.825 INFO:tasks.workunit.client.0.smithi136.stdout:op 5558 completed, throughput=5MB/sec 2024-09-13T00:19:39.825 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-h7e5-lLShBufynq off=158765 len=582742 2024-09-13T00:19:39.832 INFO:tasks.workunit.client.0.smithi136.stdout:op 5559 completed, throughput=5MB/sec 2024-09-13T00:19:39.832 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj--QB9gmEiZ-YI_5n off=6864296 len=1396589 2024-09-13T00:19:39.836 INFO:tasks.workunit.client.0.smithi136.stdout:op 5560 completed, throughput=5MB/sec 2024-09-13T00:19:40.837 INFO:tasks.workunit.client.0.smithi136.stdout: 1063: throughput=5MB/sec pending data=0 2024-09-13T00:19:40.837 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-TRNp1Heb6Gy3-qZ off=5112586 len=670788 2024-09-13T00:19:40.839 INFO:tasks.workunit.client.0.smithi136.stdout:op 5561 completed, throughput=5MB/sec 2024-09-13T00:19:40.839 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-D9zDgVsgKhF9y4X off=2485794 len=478071 2024-09-13T00:19:40.846 INFO:tasks.workunit.client.0.smithi136.stdout:op 5562 completed, throughput=5MB/sec 2024-09-13T00:19:40.846 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-1uAxQ6rIwbtGU6F off=8142618 len=1586706 2024-09-13T00:19:40.851 INFO:tasks.workunit.client.0.smithi136.stdout:op 5563 completed, throughput=5MB/sec 2024-09-13T00:19:40.851 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-X5eQEq8WrcEOb7I off=19914424 len=507107 2024-09-13T00:19:40.869 INFO:tasks.workunit.client.0.smithi136.stdout:op 5564 completed, throughput=5MB/sec 2024-09-13T00:19:40.869 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-NbFzC7EWrVNDoEe off=161693 len=157125 2024-09-13T00:19:40.874 INFO:tasks.workunit.client.0.smithi136.stdout:op 5565 completed, throughput=5MB/sec 2024-09-13T00:19:40.874 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-KiDB9GBITB2eh7u off=715019 len=505762 2024-09-13T00:19:40.876 INFO:tasks.workunit.client.0.smithi136.stdout:op 5566 completed, throughput=5MB/sec 2024-09-13T00:19:40.876 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Od6NYuqQNrkgzP0 off=513142 len=1339899 2024-09-13T00:19:40.881 INFO:tasks.workunit.client.0.smithi136.stdout:op 5567 completed, throughput=5MB/sec 2024-09-13T00:19:40.881 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-tHckBeku4WbPGWK off=5935678 len=418836 2024-09-13T00:19:40.887 INFO:tasks.workunit.client.0.smithi136.stdout:op 5568 completed, throughput=5MB/sec 2024-09-13T00:19:41.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:19:41 smithi181 bash[67587]: cluster 2024-09-13T00:19:39.691616+0000 mgr.y (mgr.24661) 1276 : cluster [DBG] pgmap v1959: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:19:41.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:19:41 smithi136 bash[75164]: cluster 2024-09-13T00:19:39.691616+0000 mgr.y (mgr.24661) 1276 : cluster [DBG] pgmap v1959: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:19:41.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:19:41 smithi136 bash[76371]: cluster 2024-09-13T00:19:39.691616+0000 mgr.y (mgr.24661) 1276 : cluster [DBG] pgmap v1959: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:19:41.887 INFO:tasks.workunit.client.0.smithi136.stdout: 1064: throughput=5MB/sec pending data=0 2024-09-13T00:19:41.887 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-OrriI7dJzeV_vya off=6378301 len=1840297 2024-09-13T00:19:41.891 INFO:tasks.workunit.client.0.smithi136.stdout:op 5569 completed, throughput=5MB/sec 2024-09-13T00:19:41.892 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-5FTZsD4jzLne8ak off=8954173 len=1442883 2024-09-13T00:19:41.902 INFO:tasks.workunit.client.0.smithi136.stdout:op 5570 completed, throughput=5MB/sec 2024-09-13T00:19:41.903 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-CxIpL66puVyFFT3 off=10743957 len=1222062 2024-09-13T00:19:41.906 INFO:tasks.workunit.client.0.smithi136.stdout:op 5571 completed, throughput=5MB/sec 2024-09-13T00:19:41.906 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-sW33cqgcoJKVs47 off=3016967 len=436289 2024-09-13T00:19:41.907 INFO:tasks.workunit.client.0.smithi136.stdout:op 5572 completed, throughput=5MB/sec 2024-09-13T00:19:41.907 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-nq-QP9gc0kI1CS8 off=15831420 len=785583 2024-09-13T00:19:41.916 INFO:tasks.workunit.client.0.smithi136.stdout:op 5573 completed, throughput=5MB/sec 2024-09-13T00:19:42.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:19:42 smithi181 bash[95505]: ts=2024-09-13T00:19:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:19:42.916 INFO:tasks.workunit.client.0.smithi136.stdout: 1065: throughput=5MB/sec pending data=0 2024-09-13T00:19:42.917 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-h6zZAzhedulLmMM off=6836051 len=729265 2024-09-13T00:19:42.925 INFO:tasks.workunit.client.0.smithi136.stdout:op 5574 completed, throughput=5MB/sec 2024-09-13T00:19:42.925 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-eKDWvndMFB0eBff off=5135923 len=964144 2024-09-13T00:19:42.932 INFO:tasks.workunit.client.0.smithi136.stdout:op 5575 completed, throughput=5MB/sec 2024-09-13T00:19:42.933 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-a-o0Ae6Hl8ib3Wn off=1678275 len=920187 2024-09-13T00:19:42.939 INFO:tasks.workunit.client.0.smithi136.stdout:op 5576 completed, throughput=5MB/sec 2024-09-13T00:19:42.939 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ZshOYo8Ym9xTOHm off=6403324 len=1453406 2024-09-13T00:19:42.943 INFO:tasks.workunit.client.0.smithi136.stdout:op 5577 completed, throughput=5MB/sec 2024-09-13T00:19:42.943 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-JB2t5raq8dOJ8A3 off=12340977 len=1603499 2024-09-13T00:19:42.947 INFO:tasks.workunit.client.0.smithi136.stdout:op 5578 completed, throughput=5MB/sec 2024-09-13T00:19:43.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:19:43 smithi181 bash[67587]: cluster 2024-09-13T00:19:41.692714+0000 mgr.y (mgr.24661) 1277 : cluster [DBG] pgmap v1960: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:19:43.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:19:43 smithi136 bash[75164]: cluster 2024-09-13T00:19:41.692714+0000 mgr.y (mgr.24661) 1277 : cluster [DBG] pgmap v1960: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:19:43.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:19:43 smithi136 bash[76371]: cluster 2024-09-13T00:19:41.692714+0000 mgr.y (mgr.24661) 1277 : cluster [DBG] pgmap v1960: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:19:43.947 INFO:tasks.workunit.client.0.smithi136.stdout: 1066: throughput=5MB/sec pending data=0 2024-09-13T00:19:43.947 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Ii4lQwUPKrUK0k5 off=2057298 len=1172501 2024-09-13T00:19:43.956 INFO:tasks.workunit.client.0.smithi136.stdout:op 5579 completed, throughput=5MB/sec 2024-09-13T00:19:43.956 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-dY5icUJ-5a231b6 off=1185521 len=1560040 2024-09-13T00:19:43.967 INFO:tasks.workunit.client.0.smithi136.stdout:op 5580 completed, throughput=5MB/sec 2024-09-13T00:19:43.967 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-G8mZWocV2qhP9yT off=8518427 len=722440 2024-09-13T00:19:43.973 INFO:tasks.workunit.client.0.smithi136.stdout:op 5581 completed, throughput=5MB/sec 2024-09-13T00:19:43.973 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-9uKnqBkvG0gOyZ0 off=1224541 len=1076513 2024-09-13T00:19:43.975 INFO:tasks.workunit.client.0.smithi136.stdout:op 5582 completed, throughput=5MB/sec 2024-09-13T00:19:44.976 INFO:tasks.workunit.client.0.smithi136.stdout: 1067: throughput=5MB/sec pending data=0 2024-09-13T00:19:44.976 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-cccQHXF9XuOLMy5 off=1842770 len=1023977 2024-09-13T00:19:44.979 INFO:tasks.workunit.client.0.smithi136.stdout:op 5583 completed, throughput=5MB/sec 2024-09-13T00:19:44.979 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-71Q9-134C-pvw4V off=9493892 len=69615 2024-09-13T00:19:44.980 INFO:tasks.workunit.client.0.smithi136.stdout:op 5584 completed, throughput=5MB/sec 2024-09-13T00:19:44.980 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-hThBPlyZX7S6YUH off=1717259 len=354434 2024-09-13T00:19:44.981 INFO:tasks.workunit.client.0.smithi136.stdout:op 5585 completed, throughput=5MB/sec 2024-09-13T00:19:44.982 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-akBhBk15d_c0HGz off=1145801 len=323334 2024-09-13T00:19:44.982 INFO:tasks.workunit.client.0.smithi136.stdout:op 5586 completed, throughput=5MB/sec 2024-09-13T00:19:44.983 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-8hvsBFQmvs0K0VF off=6489626 len=480963 2024-09-13T00:19:44.988 INFO:tasks.workunit.client.0.smithi136.stdout:op 5587 completed, throughput=5MB/sec 2024-09-13T00:19:44.988 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2i9jvDIRHiD8PJp off=8993728 len=1520297 2024-09-13T00:19:44.992 INFO:tasks.workunit.client.0.smithi136.stdout:op 5588 completed, throughput=5MB/sec 2024-09-13T00:19:44.992 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-KOSXC9iWG3gbEmh off=2209719 len=111109 2024-09-13T00:19:44.997 INFO:tasks.workunit.client.0.smithi136.stdout:op 5589 completed, throughput=5MB/sec 2024-09-13T00:19:44.997 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-RYkIbkkfjkLstRJ off=1629435 len=1995873 2024-09-13T00:19:45.001 INFO:tasks.workunit.client.0.smithi136.stdout:op 5590 completed, throughput=5MB/sec 2024-09-13T00:19:45.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:19:45 smithi181 bash[67587]: cluster 2024-09-13T00:19:43.693733+0000 mgr.y (mgr.24661) 1278 : cluster [DBG] pgmap v1961: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:19:45.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:19:45 smithi136 bash[75164]: cluster 2024-09-13T00:19:43.693733+0000 mgr.y (mgr.24661) 1278 : cluster [DBG] pgmap v1961: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:19:45.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:19:45 smithi136 bash[76371]: cluster 2024-09-13T00:19:43.693733+0000 mgr.y (mgr.24661) 1278 : cluster [DBG] pgmap v1961: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T00:19:46.001 INFO:tasks.workunit.client.0.smithi136.stdout: 1068: throughput=5MB/sec pending data=0 2024-09-13T00:19:46.001 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-BZ90O70Heu0oJfu off=6274707 len=2069031 2024-09-13T00:19:46.016 INFO:tasks.workunit.client.0.smithi136.stdout:op 5591 completed, throughput=5MB/sec 2024-09-13T00:19:46.016 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-yWM8BcC2xADiA8v off=0 len=1131142 2024-09-13T00:19:46.019 INFO:tasks.workunit.client.0.smithi136.stdout:op 5592 completed, throughput=5MB/sec 2024-09-13T00:19:46.019 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-hB6lnaq4Qx71XgS off=16311116 len=713613 2024-09-13T00:19:46.022 INFO:tasks.workunit.client.0.smithi136.stdout:op 5593 completed, throughput=5MB/sec 2024-09-13T00:19:46.022 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-nLG5fFQM9YQ_3El off=3229160 len=1581492 2024-09-13T00:19:46.033 INFO:tasks.workunit.client.0.smithi136.stdout:op 5594 completed, throughput=5MB/sec 2024-09-13T00:19:47.034 INFO:tasks.workunit.client.0.smithi136.stdout: 1069: throughput=5MB/sec pending data=0 2024-09-13T00:19:47.034 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-p5uKhPs_fZkP5wb off=9333526 len=282514 2024-09-13T00:19:47.035 INFO:tasks.workunit.client.0.smithi136.stdout:op 5595 completed, throughput=5MB/sec 2024-09-13T00:19:47.035 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-edgWAmwgMHc6yQb off=1960808 len=1240041 2024-09-13T00:19:47.048 INFO:tasks.workunit.client.0.smithi136.stdout:op 5596 completed, throughput=5MB/sec 2024-09-13T00:19:47.048 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-f6GL5G4z19BWGOt off=28386 len=107211 2024-09-13T00:19:47.049 INFO:tasks.workunit.client.0.smithi136.stdout:op 5597 completed, throughput=5MB/sec 2024-09-13T00:19:47.049 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-0vRs9ZQBW-vCHs8 off=21704216 len=1608915 2024-09-13T00:19:47.061 INFO:tasks.workunit.client.0.smithi136.stdout:op 5598 completed, throughput=5MB/sec 2024-09-13T00:19:47.061 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ZvpPzOZWN-W-Gcr off=2607344 len=323227 2024-09-13T00:19:47.063 INFO:tasks.workunit.client.0.smithi136.stdout:op 5599 completed, throughput=5MB/sec 2024-09-13T00:19:47.063 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Q5u0dknY1V47KA_ off=7389967 len=1818006 2024-09-13T00:19:47.069 INFO:tasks.workunit.client.0.smithi136.stdout:op 5600 completed, throughput=5MB/sec 2024-09-13T00:19:47.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:19:47 smithi181 bash[67587]: cluster 2024-09-13T00:19:45.694637+0000 mgr.y (mgr.24661) 1279 : cluster [DBG] pgmap v1962: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:19:47.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:19:47 smithi136 bash[75164]: cluster 2024-09-13T00:19:45.694637+0000 mgr.y (mgr.24661) 1279 : cluster [DBG] pgmap v1962: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:19:47.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:19:47 smithi136 bash[76371]: cluster 2024-09-13T00:19:45.694637+0000 mgr.y (mgr.24661) 1279 : cluster [DBG] pgmap v1962: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:19:48.069 INFO:tasks.workunit.client.0.smithi136.stdout: 1070: throughput=5MB/sec pending data=0 2024-09-13T00:19:48.070 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-9JPJy_UBCxx0rYX off=3818473 len=967086 2024-09-13T00:19:48.074 INFO:tasks.workunit.client.0.smithi136.stdout:op 5601 completed, throughput=5MB/sec 2024-09-13T00:19:48.074 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-OqXRAbUyo6gSJQ8 off=5853078 len=1199387 2024-09-13T00:19:48.078 INFO:tasks.workunit.client.0.smithi136.stdout:op 5602 completed, throughput=5MB/sec 2024-09-13T00:19:48.078 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-l4yae0xGbk1LQTr off=11267176 len=391187 2024-09-13T00:19:48.080 INFO:tasks.workunit.client.0.smithi136.stdout:op 5603 completed, throughput=5MB/sec 2024-09-13T00:19:48.080 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-1qwIpjmeRq4oyRd off=2464666 len=890739 2024-09-13T00:19:48.082 INFO:tasks.workunit.client.0.smithi136.stdout:op 5604 completed, throughput=5MB/sec 2024-09-13T00:19:48.083 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-3wMkKpzjzRPl0mI off=18156102 len=371841 2024-09-13T00:19:48.084 INFO:tasks.workunit.client.0.smithi136.stdout:op 5605 completed, throughput=5MB/sec 2024-09-13T00:19:48.084 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-uEldETpBx4ulGrA off=12582918 len=284219 2024-09-13T00:19:48.090 INFO:tasks.workunit.client.0.smithi136.stdout:op 5606 completed, throughput=5MB/sec 2024-09-13T00:19:48.090 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-sQGE8z7j8adU3mG off=6018334 len=1729747 2024-09-13T00:19:48.094 INFO:tasks.workunit.client.0.smithi136.stdout:op 5607 completed, throughput=5MB/sec 2024-09-13T00:19:49.094 INFO:tasks.workunit.client.0.smithi136.stdout: 1071: throughput=5MB/sec pending data=0 2024-09-13T00:19:49.094 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-lf2TSQMdCe7JwIU off=2158583 len=1934135 2024-09-13T00:19:49.099 INFO:tasks.workunit.client.0.smithi136.stdout:op 5608 completed, throughput=5MB/sec 2024-09-13T00:19:49.099 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Rc2CTFbB-EN7lN1 off=524985 len=1468174 2024-09-13T00:19:49.112 INFO:tasks.workunit.client.0.smithi136.stdout:op 5609 completed, throughput=5MB/sec 2024-09-13T00:19:49.112 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-2RLOhkZeVkGOnnT off=17087612 len=1090676 2024-09-13T00:19:49.122 INFO:tasks.workunit.client.0.smithi136.stdout:op 5610 completed, throughput=5MB/sec 2024-09-13T00:19:49.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:19:49 smithi181 bash[67587]: cluster 2024-09-13T00:19:47.695684+0000 mgr.y (mgr.24661) 1280 : cluster [DBG] pgmap v1963: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:19:49.412 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:19:49 smithi136 bash[75164]: cluster 2024-09-13T00:19:47.695684+0000 mgr.y (mgr.24661) 1280 : cluster [DBG] pgmap v1963: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:19:49.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:19:49 smithi136 bash[76371]: cluster 2024-09-13T00:19:47.695684+0000 mgr.y (mgr.24661) 1280 : cluster [DBG] pgmap v1963: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:19:49.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:19:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:19:49] "GET /metrics HTTP/1.1" 200 34141 "" "Prometheus/2.43.0" 2024-09-13T00:19:50.122 INFO:tasks.workunit.client.0.smithi136.stdout: 1072: throughput=5MB/sec pending data=0 2024-09-13T00:19:50.123 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-e1yJY-TC4VNI0sE off=2551748 len=1347355 2024-09-13T00:19:50.135 INFO:tasks.workunit.client.0.smithi136.stdout:op 5611 completed, throughput=5MB/sec 2024-09-13T00:19:50.135 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-TToG6Kn_Yl1UBIy off=3361403 len=2036803 2024-09-13T00:19:50.152 INFO:tasks.workunit.client.0.smithi136.stdout:op 5612 completed, throughput=5MB/sec 2024-09-13T00:19:50.152 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-sfrzkCVWKF35pND off=6348711 len=2093594 2024-09-13T00:19:50.167 INFO:tasks.workunit.client.0.smithi136.stdout:op 5613 completed, throughput=5MB/sec 2024-09-13T00:19:51.168 INFO:tasks.workunit.client.0.smithi136.stdout: 1073: throughput=5MB/sec pending data=0 2024-09-13T00:19:51.168 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-JWGrijIzOpp9vG7 off=4509495 len=956280 2024-09-13T00:19:51.171 INFO:tasks.workunit.client.0.smithi136.stdout:op 5614 completed, throughput=5MB/sec 2024-09-13T00:19:51.171 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2BzaOiTWDnHxNvB off=13437107 len=176572 2024-09-13T00:19:51.172 INFO:tasks.workunit.client.0.smithi136.stdout:op 5615 completed, throughput=5MB/sec 2024-09-13T00:19:51.172 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2yPeIAsUepDasNs off=11900210 len=251418 2024-09-13T00:19:51.173 INFO:tasks.workunit.client.0.smithi136.stdout:op 5616 completed, throughput=5MB/sec 2024-09-13T00:19:51.173 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-_LnDh9BmGi-FWeX off=8982542 len=370538 2024-09-13T00:19:51.175 INFO:tasks.workunit.client.0.smithi136.stdout:op 5617 completed, throughput=5MB/sec 2024-09-13T00:19:51.175 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-LEQsz-UbHFeWm88 off=9651491 len=147972 2024-09-13T00:19:51.180 INFO:tasks.workunit.client.0.smithi136.stdout:op 5618 completed, throughput=5MB/sec 2024-09-13T00:19:51.180 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-PUDAD5shIidKn4J off=651908 len=699538 2024-09-13T00:19:51.187 INFO:tasks.workunit.client.0.smithi136.stdout:op 5619 completed, throughput=5MB/sec 2024-09-13T00:19:51.188 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-4fhm4NUG9rrsfWt off=17828668 len=419894 2024-09-13T00:19:51.194 INFO:tasks.workunit.client.0.smithi136.stdout:op 5620 completed, throughput=5MB/sec 2024-09-13T00:19:51.194 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-aMfkWxrxJfz2Q8e off=3090119 len=1678137 2024-09-13T00:19:51.205 INFO:tasks.workunit.client.0.smithi136.stdout:op 5621 completed, throughput=5MB/sec 2024-09-13T00:19:51.206 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-3ED0zGlojCBXXkh off=2462723 len=698641 2024-09-13T00:19:51.207 INFO:tasks.workunit.client.0.smithi136.stdout:op 5622 completed, throughput=5MB/sec 2024-09-13T00:19:51.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:19:51 smithi181 bash[67587]: cluster 2024-09-13T00:19:49.696522+0000 mgr.y (mgr.24661) 1281 : cluster [DBG] pgmap v1964: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:19:51.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:19:51 smithi136 bash[75164]: cluster 2024-09-13T00:19:49.696522+0000 mgr.y (mgr.24661) 1281 : cluster [DBG] pgmap v1964: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:19:51.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:19:51 smithi136 bash[76371]: cluster 2024-09-13T00:19:49.696522+0000 mgr.y (mgr.24661) 1281 : cluster [DBG] pgmap v1964: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.4 MiB/s wr, 5 op/s 2024-09-13T00:19:52.207 INFO:tasks.workunit.client.0.smithi136.stdout: 1074: throughput=5MB/sec pending data=0 2024-09-13T00:19:52.207 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-nMH-q4JPIxxWDvA off=12518834 len=1299419 2024-09-13T00:19:52.218 INFO:tasks.workunit.client.0.smithi136.stdout:op 5623 completed, throughput=5MB/sec 2024-09-13T00:19:52.218 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-D7NSc_y7VjKYixw off=1107598 len=1308982 2024-09-13T00:19:52.228 INFO:tasks.workunit.client.0.smithi136.stdout:op 5624 completed, throughput=5MB/sec 2024-09-13T00:19:52.228 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-3A9DmIt2gwg40Zl off=10035459 len=1092802 2024-09-13T00:19:52.236 INFO:tasks.workunit.client.0.smithi136.stdout:op 5625 completed, throughput=5MB/sec 2024-09-13T00:19:52.237 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-tj_naRVQtKVSDrR off=6189293 len=1994651 2024-09-13T00:19:52.241 INFO:tasks.workunit.client.0.smithi136.stdout:op 5626 completed, throughput=5MB/sec 2024-09-13T00:19:52.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:19:52 smithi181 bash[95505]: ts=2024-09-13T00:19:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:19:53.241 INFO:tasks.workunit.client.0.smithi136.stdout: 1075: throughput=5MB/sec pending data=0 2024-09-13T00:19:53.241 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-2w2MMDAarHS2gbE off=3636098 len=932943 2024-09-13T00:19:53.250 INFO:tasks.workunit.client.0.smithi136.stdout:op 5627 completed, throughput=5MB/sec 2024-09-13T00:19:53.250 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-rfyurwYVW9gn5pI off=2795380 len=1899671 2024-09-13T00:19:53.263 INFO:tasks.workunit.client.0.smithi136.stdout:op 5628 completed, throughput=5MB/sec 2024-09-13T00:19:53.264 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-vw34GTDXfZ_0-kJ off=5756252 len=1522878 2024-09-13T00:19:53.266 INFO:tasks.workunit.client.0.smithi136.stdout:op 5629 completed, throughput=5MB/sec 2024-09-13T00:19:53.266 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-SKKm3TMlBay_sdY off=20819528 len=1632611 2024-09-13T00:19:53.276 INFO:tasks.workunit.client.0.smithi136.stdout:op 5630 completed, throughput=5MB/sec 2024-09-13T00:19:53.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:19:53 smithi181 bash[67587]: cluster 2024-09-13T00:19:51.697538+0000 mgr.y (mgr.24661) 1282 : cluster [DBG] pgmap v1965: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.7 MiB/s wr, 6 op/s 2024-09-13T00:19:53.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:19:53 smithi136 bash[75164]: cluster 2024-09-13T00:19:51.697538+0000 mgr.y (mgr.24661) 1282 : cluster [DBG] pgmap v1965: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.7 MiB/s wr, 6 op/s 2024-09-13T00:19:53.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:19:53 smithi136 bash[76371]: cluster 2024-09-13T00:19:51.697538+0000 mgr.y (mgr.24661) 1282 : cluster [DBG] pgmap v1965: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.7 MiB/s wr, 6 op/s 2024-09-13T00:19:54.276 INFO:tasks.workunit.client.0.smithi136.stdout: 1076: throughput=5MB/sec pending data=0 2024-09-13T00:19:54.276 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-PuTkPlYV4ncEcDz off=14297308 len=82249 2024-09-13T00:19:54.278 INFO:tasks.workunit.client.0.smithi136.stdout:op 5631 completed, throughput=5MB/sec 2024-09-13T00:19:54.278 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-dkttGpjqFo2jdcg off=2720330 len=1816718 2024-09-13T00:19:54.290 INFO:tasks.workunit.client.0.smithi136.stdout:op 5632 completed, throughput=5MB/sec 2024-09-13T00:19:54.290 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-F901rfheJ2njxQV off=3650130 len=1093244 2024-09-13T00:19:54.299 INFO:tasks.workunit.client.0.smithi136.stdout:op 5633 completed, throughput=5MB/sec 2024-09-13T00:19:54.299 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-HP6hSToKxkdzu23 off=67981 len=742336 2024-09-13T00:19:54.302 INFO:tasks.workunit.client.0.smithi136.stdout:op 5634 completed, throughput=5MB/sec 2024-09-13T00:19:54.302 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-eim9JGbzNbPAs0x off=770136 len=1623303 2024-09-13T00:19:54.306 INFO:tasks.workunit.client.0.smithi136.stdout:op 5635 completed, throughput=5MB/sec 2024-09-13T00:19:54.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:19:54 smithi181 bash[67587]: audit 2024-09-13T00:19:53.802707+0000 mon.a (mon.0) 2267 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:19:54.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:19:54 smithi136 bash[75164]: audit 2024-09-13T00:19:53.802707+0000 mon.a (mon.0) 2267 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:19:54.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:19:54 smithi136 bash[76371]: audit 2024-09-13T00:19:53.802707+0000 mon.a (mon.0) 2267 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:19:55.307 INFO:tasks.workunit.client.0.smithi136.stdout: 1077: throughput=5MB/sec pending data=0 2024-09-13T00:19:55.307 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-w8mbQzu-8MtZZhF off=12817968 len=1921205 2024-09-13T00:19:55.311 INFO:tasks.workunit.client.0.smithi136.stdout:op 5636 completed, throughput=5MB/sec 2024-09-13T00:19:55.311 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-wp0oR6hQ4G7pnAM off=6950754 len=1015766 2024-09-13T00:19:55.314 INFO:tasks.workunit.client.0.smithi136.stdout:op 5637 completed, throughput=5MB/sec 2024-09-13T00:19:55.314 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-NMfKFLi1i5HrrP_ off=4590640 len=2019717 2024-09-13T00:19:55.330 INFO:tasks.workunit.client.0.smithi136.stdout:op 5638 completed, throughput=5MB/sec 2024-09-13T00:19:55.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:19:55 smithi181 bash[67587]: cluster 2024-09-13T00:19:53.698425+0000 mgr.y (mgr.24661) 1283 : cluster [DBG] pgmap v1966: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:19:55.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:19:55 smithi136 bash[75164]: cluster 2024-09-13T00:19:53.698425+0000 mgr.y (mgr.24661) 1283 : cluster [DBG] pgmap v1966: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:19:55.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:19:55 smithi136 bash[76371]: cluster 2024-09-13T00:19:53.698425+0000 mgr.y (mgr.24661) 1283 : cluster [DBG] pgmap v1966: 105 pgs: 105 active+clean; 126 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:19:56.331 INFO:tasks.workunit.client.0.smithi136.stdout: 1078: throughput=5MB/sec pending data=0 2024-09-13T00:19:56.331 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-xQERektCKLgunmW off=10066692 len=831762 2024-09-13T00:19:56.339 INFO:tasks.workunit.client.0.smithi136.stdout:op 5639 completed, throughput=5MB/sec 2024-09-13T00:19:56.340 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-dxR2_1MT0tciwd3 off=12600439 len=981900 2024-09-13T00:19:56.342 INFO:tasks.workunit.client.0.smithi136.stdout:op 5640 completed, throughput=5MB/sec 2024-09-13T00:19:56.343 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-1eptpA7faMLW2XZ off=1566666 len=1598861 2024-09-13T00:19:56.346 INFO:tasks.workunit.client.0.smithi136.stdout:op 5641 completed, throughput=5MB/sec 2024-09-13T00:19:56.346 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-bD-6eGIxjLN0RtB off=8093215 len=1083301 2024-09-13T00:19:56.350 INFO:tasks.workunit.client.0.smithi136.stdout:op 5642 completed, throughput=5MB/sec 2024-09-13T00:19:56.350 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-NpFBzKgONolv1k5 off=215315 len=1778497 2024-09-13T00:19:56.363 INFO:tasks.workunit.client.0.smithi136.stdout:op 5643 completed, throughput=5MB/sec 2024-09-13T00:19:57.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:19:57 smithi181 bash[67587]: cluster 2024-09-13T00:19:55.699359+0000 mgr.y (mgr.24661) 1284 : cluster [DBG] pgmap v1967: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:19:57.364 INFO:tasks.workunit.client.0.smithi136.stdout: 1079: throughput=5MB/sec pending data=0 2024-09-13T00:19:57.364 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-hPG434jXa-vYs_r off=1478988 len=1647235 2024-09-13T00:19:57.368 INFO:tasks.workunit.client.0.smithi136.stdout:op 5644 completed, throughput=5MB/sec 2024-09-13T00:19:57.368 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ROt2XD_0bw8mYDv off=171834 len=1857985 2024-09-13T00:19:57.380 INFO:tasks.workunit.client.0.smithi136.stdout:op 5645 completed, throughput=5MB/sec 2024-09-13T00:19:57.380 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-QahPMu9wo7SF6PK off=10345519 len=831364 2024-09-13T00:19:57.388 INFO:tasks.workunit.client.0.smithi136.stdout:op 5646 completed, throughput=5MB/sec 2024-09-13T00:19:57.508 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:19:57 smithi136 bash[76371]: cluster 2024-09-13T00:19:55.699359+0000 mgr.y (mgr.24661) 1284 : cluster [DBG] pgmap v1967: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:19:57.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:19:57 smithi136 bash[75164]: cluster 2024-09-13T00:19:55.699359+0000 mgr.y (mgr.24661) 1284 : cluster [DBG] pgmap v1967: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:19:58.389 INFO:tasks.workunit.client.0.smithi136.stdout: 1080: throughput=5MB/sec pending data=0 2024-09-13T00:19:58.389 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-c_UnBJSxRIanhiW off=15436144 len=139516 2024-09-13T00:19:58.394 INFO:tasks.workunit.client.0.smithi136.stdout:op 5647 completed, throughput=5MB/sec 2024-09-13T00:19:58.394 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-A9U2HDq-6QM78un off=10906938 len=1476219 2024-09-13T00:19:58.405 INFO:tasks.workunit.client.0.smithi136.stdout:op 5648 completed, throughput=5MB/sec 2024-09-13T00:19:58.405 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-nkTgTyXIw7M3pC6 off=9776093 len=122250 2024-09-13T00:19:58.409 INFO:tasks.workunit.client.0.smithi136.stdout:op 5649 completed, throughput=5MB/sec 2024-09-13T00:19:58.410 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-qANNAZNe_FIZ-2- off=677022 len=2022097 2024-09-13T00:19:58.417 INFO:tasks.workunit.client.0.smithi136.stdout:op 5650 completed, throughput=5MB/sec 2024-09-13T00:19:58.417 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Q8jwmm4s55_RuUV off=1015104 len=164500 2024-09-13T00:19:58.418 INFO:tasks.workunit.client.0.smithi136.stdout:op 5651 completed, throughput=5MB/sec 2024-09-13T00:19:58.419 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-5mblezIVlmZF1E8 off=11959639 len=1927438 2024-09-13T00:19:58.431 INFO:tasks.workunit.client.0.smithi136.stdout:op 5652 completed, throughput=5MB/sec 2024-09-13T00:19:59.412 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:19:59 smithi136 bash[75164]: cluster 2024-09-13T00:19:57.700390+0000 mgr.y (mgr.24661) 1285 : cluster [DBG] pgmap v1968: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 2.2 MiB/s rd, 3.2 MiB/s wr, 5 op/s 2024-09-13T00:19:59.412 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:19:59 smithi136 bash[76371]: cluster 2024-09-13T00:19:57.700390+0000 mgr.y (mgr.24661) 1285 : cluster [DBG] pgmap v1968: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 2.2 MiB/s rd, 3.2 MiB/s wr, 5 op/s 2024-09-13T00:19:59.431 INFO:tasks.workunit.client.0.smithi136.stdout: 1081: throughput=5MB/sec pending data=0 2024-09-13T00:19:59.431 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-4tv3NjadQiPLXHm off=99800 len=670467 2024-09-13T00:19:59.439 INFO:tasks.workunit.client.0.smithi136.stdout:op 5653 completed, throughput=5MB/sec 2024-09-13T00:19:59.439 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-G6KRESiVhMOmQMv off=5593479 len=1598720 2024-09-13T00:19:59.443 INFO:tasks.workunit.client.0.smithi136.stdout:op 5654 completed, throughput=5MB/sec 2024-09-13T00:19:59.443 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-DdCMhYdfrTXqT0A off=5117974 len=1014172 2024-09-13T00:19:59.446 INFO:tasks.workunit.client.0.smithi136.stdout:op 5655 completed, throughput=5MB/sec 2024-09-13T00:19:59.447 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-LJLJZdaGoPN5YoQ off=17054902 len=1737415 2024-09-13T00:19:59.456 INFO:tasks.workunit.client.0.smithi136.stdout:op 5656 completed, throughput=5MB/sec 2024-09-13T00:19:59.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:19:59 smithi181 bash[67587]: cluster 2024-09-13T00:19:57.700390+0000 mgr.y (mgr.24661) 1285 : cluster [DBG] pgmap v1968: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 2.2 MiB/s rd, 3.2 MiB/s wr, 5 op/s 2024-09-13T00:19:59.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:19:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:19:59] "GET /metrics HTTP/1.1" 200 34142 "" "Prometheus/2.43.0" 2024-09-13T00:20:00.457 INFO:tasks.workunit.client.0.smithi136.stdout: 1082: throughput=5MB/sec pending data=0 2024-09-13T00:20:00.457 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-v4W7HIzfV3UWg6f off=4098708 len=1970634 2024-09-13T00:20:00.461 INFO:tasks.workunit.client.0.smithi136.stdout:op 5657 completed, throughput=5MB/sec 2024-09-13T00:20:00.461 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-mFcKx5dPSQRHJjw off=7558 len=1897906 2024-09-13T00:20:00.467 INFO:tasks.workunit.client.0.smithi136.stdout:op 5658 completed, throughput=5MB/sec 2024-09-13T00:20:00.467 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_6JQz6FJxkmMSYO off=2638254 len=1871976 2024-09-13T00:20:00.481 INFO:tasks.workunit.client.0.smithi136.stdout:op 5659 completed, throughput=5MB/sec 2024-09-13T00:20:00.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:20:00 smithi136 bash[75164]: audit 2024-09-13T00:19:59.428631+0000 mon.a (mon.0) 2268 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:20:00.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:20:00 smithi136 bash[75164]: cluster 2024-09-13T00:20:00.000130+0000 mon.a (mon.0) 2269 : cluster [INF] overall HEALTH_OK 2024-09-13T00:20:00.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:20:00 smithi136 bash[76371]: audit 2024-09-13T00:19:59.428631+0000 mon.a (mon.0) 2268 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:20:00.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:20:00 smithi136 bash[76371]: cluster 2024-09-13T00:20:00.000130+0000 mon.a (mon.0) 2269 : cluster [INF] overall HEALTH_OK 2024-09-13T00:20:00.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:20:00 smithi181 bash[67587]: audit 2024-09-13T00:19:59.428631+0000 mon.a (mon.0) 2268 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:20:00.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:20:00 smithi181 bash[67587]: cluster 2024-09-13T00:20:00.000130+0000 mon.a (mon.0) 2269 : cluster [INF] overall HEALTH_OK 2024-09-13T00:20:01.481 INFO:tasks.workunit.client.0.smithi136.stdout: 1083: throughput=5MB/sec pending data=0 2024-09-13T00:20:01.481 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-dC2Iz_49RYDs-ID off=8014541 len=1609767 2024-09-13T00:20:01.487 INFO:tasks.workunit.client.0.smithi136.stdout:op 5660 completed, throughput=5MB/sec 2024-09-13T00:20:01.487 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-LEJPi51X-5DmKeM off=3789835 len=1974076 2024-09-13T00:20:01.504 INFO:tasks.workunit.client.0.smithi136.stdout:op 5661 completed, throughput=5MB/sec 2024-09-13T00:20:01.504 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-tzmp0LrZqv3LARQ off=83781 len=1657211 2024-09-13T00:20:01.507 INFO:tasks.workunit.client.0.smithi136.stdout:op 5662 completed, throughput=5MB/sec 2024-09-13T00:20:01.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:20:01 smithi136 bash[75164]: cluster 2024-09-13T00:19:59.700977+0000 mgr.y (mgr.24661) 1286 : cluster [DBG] pgmap v1969: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:20:01.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:20:01 smithi136 bash[75164]: audit 2024-09-13T00:20:00.252364+0000 mon.a (mon.0) 2270 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:20:01.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:20:01 smithi136 bash[75164]: audit 2024-09-13T00:20:00.258963+0000 mon.a (mon.0) 2271 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:20:01.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:20:01 smithi136 bash[75164]: audit 2024-09-13T00:20:00.260076+0000 mon.a (mon.0) 2272 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:20:01.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:20:01 smithi136 bash[75164]: audit 2024-09-13T00:20:00.260928+0000 mon.a (mon.0) 2273 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:20:01.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:20:01 smithi136 bash[75164]: audit 2024-09-13T00:20:00.265751+0000 mon.a (mon.0) 2274 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:20:01.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:20:01 smithi136 bash[76371]: cluster 2024-09-13T00:19:59.700977+0000 mgr.y (mgr.24661) 1286 : cluster [DBG] pgmap v1969: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:20:01.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:20:01 smithi136 bash[76371]: audit 2024-09-13T00:20:00.252364+0000 mon.a (mon.0) 2270 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:20:01.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:20:01 smithi136 bash[76371]: audit 2024-09-13T00:20:00.258963+0000 mon.a (mon.0) 2271 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:20:01.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:20:01 smithi136 bash[76371]: audit 2024-09-13T00:20:00.260076+0000 mon.a (mon.0) 2272 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:20:01.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:20:01 smithi136 bash[76371]: audit 2024-09-13T00:20:00.260928+0000 mon.a (mon.0) 2273 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:20:01.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:20:01 smithi136 bash[76371]: audit 2024-09-13T00:20:00.265751+0000 mon.a (mon.0) 2274 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:20:01.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:20:01 smithi181 bash[67587]: cluster 2024-09-13T00:19:59.700977+0000 mgr.y (mgr.24661) 1286 : cluster [DBG] pgmap v1969: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:20:01.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:20:01 smithi181 bash[67587]: audit 2024-09-13T00:20:00.252364+0000 mon.a (mon.0) 2270 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:20:01.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:20:01 smithi181 bash[67587]: audit 2024-09-13T00:20:00.258963+0000 mon.a (mon.0) 2271 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:20:01.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:20:01 smithi181 bash[67587]: audit 2024-09-13T00:20:00.260076+0000 mon.a (mon.0) 2272 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:20:01.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:20:01 smithi181 bash[67587]: audit 2024-09-13T00:20:00.260928+0000 mon.a (mon.0) 2273 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:20:01.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:20:01 smithi181 bash[67587]: audit 2024-09-13T00:20:00.265751+0000 mon.a (mon.0) 2274 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:20:02.508 INFO:tasks.workunit.client.0.smithi136.stdout: 1084: throughput=5MB/sec pending data=0 2024-09-13T00:20:02.508 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-vsg2dEbbmDKb1_g off=3628451 len=1809504 2024-09-13T00:20:02.512 INFO:tasks.workunit.client.0.smithi136.stdout:op 5663 completed, throughput=5MB/sec 2024-09-13T00:20:02.512 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-V3hEhUGhb9Gu2xc off=3807589 len=782655 2024-09-13T00:20:02.514 INFO:tasks.workunit.client.0.smithi136.stdout:op 5664 completed, throughput=5MB/sec 2024-09-13T00:20:02.514 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-G9w1bxXVbw_-Ho7 off=8304086 len=33698 2024-09-13T00:20:02.519 INFO:tasks.workunit.client.0.smithi136.stdout:op 5665 completed, throughput=5MB/sec 2024-09-13T00:20:02.519 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-viAOssnSJ-PWPk8 off=3442148 len=1413925 2024-09-13T00:20:02.532 INFO:tasks.workunit.client.0.smithi136.stdout:op 5666 completed, throughput=5MB/sec 2024-09-13T00:20:02.532 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-qYUk0zY2k0Rg9bF off=4280933 len=1315253 2024-09-13T00:20:02.535 INFO:tasks.workunit.client.0.smithi136.stdout:op 5667 completed, throughput=5MB/sec 2024-09-13T00:20:02.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:20:02 smithi181 bash[95505]: ts=2024-09-13T00:20:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:20:03.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:20:03 smithi136 bash[76371]: cluster 2024-09-13T00:20:01.702077+0000 mgr.y (mgr.24661) 1287 : cluster [DBG] pgmap v1970: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 2.5 MiB/s rd, 3.0 MiB/s wr, 4 op/s 2024-09-13T00:20:03.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:20:03 smithi136 bash[75164]: cluster 2024-09-13T00:20:01.702077+0000 mgr.y (mgr.24661) 1287 : cluster [DBG] pgmap v1970: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 2.5 MiB/s rd, 3.0 MiB/s wr, 4 op/s 2024-09-13T00:20:03.535 INFO:tasks.workunit.client.0.smithi136.stdout: 1085: throughput=5MB/sec pending data=0 2024-09-13T00:20:03.535 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-0wXHE_KWOvG9aaf off=783851 len=2026582 2024-09-13T00:20:03.540 INFO:tasks.workunit.client.0.smithi136.stdout:op 5668 completed, throughput=5MB/sec 2024-09-13T00:20:03.540 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-GpfVCnKlAR7-lBT off=5781875 len=243028 2024-09-13T00:20:03.542 INFO:tasks.workunit.client.0.smithi136.stdout:op 5669 completed, throughput=5MB/sec 2024-09-13T00:20:03.542 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-VSVr889GLqF0Ex1 off=886918 len=2002974 2024-09-13T00:20:03.547 INFO:tasks.workunit.client.0.smithi136.stdout:op 5670 completed, throughput=5MB/sec 2024-09-13T00:20:03.547 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-V6VOjGhCCVHD71S off=6128602 len=436173 2024-09-13T00:20:03.548 INFO:tasks.workunit.client.0.smithi136.stdout:op 5671 completed, throughput=5MB/sec 2024-09-13T00:20:03.548 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Ui-yomzYxZr-eBO off=6600825 len=364544 2024-09-13T00:20:03.554 INFO:tasks.workunit.client.0.smithi136.stdout:op 5672 completed, throughput=5MB/sec 2024-09-13T00:20:03.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:20:03 smithi181 bash[67587]: cluster 2024-09-13T00:20:01.702077+0000 mgr.y (mgr.24661) 1287 : cluster [DBG] pgmap v1970: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 2.5 MiB/s rd, 3.0 MiB/s wr, 4 op/s 2024-09-13T00:20:04.555 INFO:tasks.workunit.client.0.smithi136.stdout: 1086: throughput=5MB/sec pending data=0 2024-09-13T00:20:04.555 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-tZNRfhOcPto__Cm off=295030 len=926873 2024-09-13T00:20:04.558 INFO:tasks.workunit.client.0.smithi136.stdout:op 5673 completed, throughput=5MB/sec 2024-09-13T00:20:04.558 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ym6S43gvEXgpEYI off=19250337 len=1575843 2024-09-13T00:20:04.572 INFO:tasks.workunit.client.0.smithi136.stdout:op 5674 completed, throughput=5MB/sec 2024-09-13T00:20:04.572 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-aoAgmu8H6AT8SKV off=4142170 len=1146819 2024-09-13T00:20:04.575 INFO:tasks.workunit.client.0.smithi136.stdout:op 5675 completed, throughput=5MB/sec 2024-09-13T00:20:04.575 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-sE4DsAfO7ysZHYn off=3277251 len=648831 2024-09-13T00:20:04.579 INFO:tasks.workunit.client.0.smithi136.stdout:op 5676 completed, throughput=5MB/sec 2024-09-13T00:20:04.579 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-yBc_e2Fu87BCIKH off=15334679 len=738062 2024-09-13T00:20:04.586 INFO:tasks.workunit.client.0.smithi136.stdout:op 5677 completed, throughput=5MB/sec 2024-09-13T00:20:04.586 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-cojcm1TuGCb1UfE off=12871281 len=1291176 2024-09-13T00:20:04.596 INFO:tasks.workunit.client.0.smithi136.stdout:op 5678 completed, throughput=5MB/sec 2024-09-13T00:20:05.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:20:05 smithi136 bash[75164]: cluster 2024-09-13T00:20:03.703128+0000 mgr.y (mgr.24661) 1288 : cluster [DBG] pgmap v1971: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.6 MiB/s wr, 3 op/s 2024-09-13T00:20:05.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:20:05 smithi136 bash[76371]: cluster 2024-09-13T00:20:03.703128+0000 mgr.y (mgr.24661) 1288 : cluster [DBG] pgmap v1971: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.6 MiB/s wr, 3 op/s 2024-09-13T00:20:05.596 INFO:tasks.workunit.client.0.smithi136.stdout: 1087: throughput=5MB/sec pending data=0 2024-09-13T00:20:05.597 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-IWdOobmaFzj4g0h off=2635583 len=1585512 2024-09-13T00:20:05.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:20:05 smithi181 bash[67587]: cluster 2024-09-13T00:20:03.703128+0000 mgr.y (mgr.24661) 1288 : cluster [DBG] pgmap v1971: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.6 MiB/s wr, 3 op/s 2024-09-13T00:20:05.607 INFO:tasks.workunit.client.0.smithi136.stdout:op 5679 completed, throughput=5MB/sec 2024-09-13T00:20:05.607 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-3i-5g79CH5G-_bK off=22825986 len=1055890 2024-09-13T00:20:05.610 INFO:tasks.workunit.client.0.smithi136.stdout:op 5680 completed, throughput=5MB/sec 2024-09-13T00:20:05.610 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-sE26qC0AkAv6S6C off=1432745 len=335769 2024-09-13T00:20:05.612 INFO:tasks.workunit.client.0.smithi136.stdout:op 5681 completed, throughput=5MB/sec 2024-09-13T00:20:05.612 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-WDQmij1tLC-irt8 off=1157144 len=1959071 2024-09-13T00:20:05.630 INFO:tasks.workunit.client.0.smithi136.stdout:op 5682 completed, throughput=5MB/sec 2024-09-13T00:20:06.630 INFO:tasks.workunit.client.0.smithi136.stdout: 1088: throughput=5MB/sec pending data=0 2024-09-13T00:20:06.630 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-gq_q1ssrAfVvibQ off=16201063 len=1426101 2024-09-13T00:20:06.642 INFO:tasks.workunit.client.0.smithi136.stdout:op 5683 completed, throughput=5MB/sec 2024-09-13T00:20:06.642 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-VH2W1AU6_UHFT3z off=5402288 len=1341590 2024-09-13T00:20:06.652 INFO:tasks.workunit.client.0.smithi136.stdout:op 5684 completed, throughput=5MB/sec 2024-09-13T00:20:06.653 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-u9wWD4QZgX032w1 off=2101189 len=2067460 2024-09-13T00:20:06.658 INFO:tasks.workunit.client.0.smithi136.stdout:op 5685 completed, throughput=5MB/sec 2024-09-13T00:20:07.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:20:07 smithi136 bash[75164]: cluster 2024-09-13T00:20:05.704124+0000 mgr.y (mgr.24661) 1289 : cluster [DBG] pgmap v1972: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.2 MiB/s wr, 3 op/s 2024-09-13T00:20:07.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:20:07 smithi136 bash[76371]: cluster 2024-09-13T00:20:05.704124+0000 mgr.y (mgr.24661) 1289 : cluster [DBG] pgmap v1972: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.2 MiB/s wr, 3 op/s 2024-09-13T00:20:07.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:20:07 smithi181 bash[67587]: cluster 2024-09-13T00:20:05.704124+0000 mgr.y (mgr.24661) 1289 : cluster [DBG] pgmap v1972: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.2 MiB/s wr, 3 op/s 2024-09-13T00:20:07.659 INFO:tasks.workunit.client.0.smithi136.stdout: 1089: throughput=5MB/sec pending data=0 2024-09-13T00:20:07.659 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-26GeHpAOmdnPV4B off=6489950 len=1629592 2024-09-13T00:20:07.662 INFO:tasks.workunit.client.0.smithi136.stdout:op 5686 completed, throughput=5MB/sec 2024-09-13T00:20:07.663 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-3RQG0o36Q0VQrPC off=1032522 len=204311 2024-09-13T00:20:07.664 INFO:tasks.workunit.client.0.smithi136.stdout:op 5687 completed, throughput=5MB/sec 2024-09-13T00:20:07.664 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-a2BOEECtcZ88gPA off=7030706 len=855814 2024-09-13T00:20:07.672 INFO:tasks.workunit.client.0.smithi136.stdout:op 5688 completed, throughput=5MB/sec 2024-09-13T00:20:07.672 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-FmjiUep3GRuUkzQ off=10290615 len=856934 2024-09-13T00:20:07.679 INFO:tasks.workunit.client.0.smithi136.stdout:op 5689 completed, throughput=5MB/sec 2024-09-13T00:20:07.680 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-RIm6J7aO52PchJK off=9314440 len=638254 2024-09-13T00:20:07.686 INFO:tasks.workunit.client.0.smithi136.stdout:op 5690 completed, throughput=5MB/sec 2024-09-13T00:20:07.687 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-M40NwOT47aeSrSX off=2989644 len=1516500 2024-09-13T00:20:07.697 INFO:tasks.workunit.client.0.smithi136.stdout:op 5691 completed, throughput=5MB/sec 2024-09-13T00:20:08.697 INFO:tasks.workunit.client.0.smithi136.stdout: 1090: throughput=5MB/sec pending data=0 2024-09-13T00:20:08.698 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-W6bPJf140K4U9Az off=1071598 len=1763829 2024-09-13T00:20:08.704 INFO:tasks.workunit.client.0.smithi136.stdout:op 5692 completed, throughput=5MB/sec 2024-09-13T00:20:08.704 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-px97j-1GNZP6bhs off=11059396 len=723214 2024-09-13T00:20:08.712 INFO:tasks.workunit.client.0.smithi136.stdout:op 5693 completed, throughput=5MB/sec 2024-09-13T00:20:08.712 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-pzdPd6GFldH4078 off=363085 len=1826036 2024-09-13T00:20:08.718 INFO:tasks.workunit.client.0.smithi136.stdout:op 5694 completed, throughput=5MB/sec 2024-09-13T00:20:08.718 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-1OiPNPTRZtexzo8 off=3062993 len=1858435 2024-09-13T00:20:08.731 INFO:tasks.workunit.client.0.smithi136.stdout:op 5695 completed, throughput=5MB/sec 2024-09-13T00:20:09.412 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:20:09 smithi136 bash[76371]: cluster 2024-09-13T00:20:07.705212+0000 mgr.y (mgr.24661) 1290 : cluster [DBG] pgmap v1973: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.1 MiB/s wr, 3 op/s 2024-09-13T00:20:09.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:20:09 smithi136 bash[76371]: audit 2024-09-13T00:20:08.803394+0000 mon.a (mon.0) 2275 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:20:09.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:20:09 smithi136 bash[75164]: cluster 2024-09-13T00:20:07.705212+0000 mgr.y (mgr.24661) 1290 : cluster [DBG] pgmap v1973: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.1 MiB/s wr, 3 op/s 2024-09-13T00:20:09.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:20:09 smithi136 bash[75164]: audit 2024-09-13T00:20:08.803394+0000 mon.a (mon.0) 2275 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:20:09.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:20:09 smithi181 bash[67587]: cluster 2024-09-13T00:20:07.705212+0000 mgr.y (mgr.24661) 1290 : cluster [DBG] pgmap v1973: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.1 MiB/s wr, 3 op/s 2024-09-13T00:20:09.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:20:09 smithi181 bash[67587]: audit 2024-09-13T00:20:08.803394+0000 mon.a (mon.0) 2275 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:20:09.731 INFO:tasks.workunit.client.0.smithi136.stdout: 1091: throughput=5MB/sec pending data=0 2024-09-13T00:20:09.731 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-vdxwubwMT6XGQp0 off=1495401 len=383373 2024-09-13T00:20:09.733 INFO:tasks.workunit.client.0.smithi136.stdout:op 5696 completed, throughput=5MB/sec 2024-09-13T00:20:09.733 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-I4O9VZtiQ9njAGC off=3002567 len=712040 2024-09-13T00:20:09.735 INFO:tasks.workunit.client.0.smithi136.stdout:op 5697 completed, throughput=5MB/sec 2024-09-13T00:20:09.735 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-7FeDZRI-M9Ud-mH off=6205989 len=480229 2024-09-13T00:20:09.742 INFO:tasks.workunit.client.0.smithi136.stdout:op 5698 completed, throughput=5MB/sec 2024-09-13T00:20:09.742 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-AUWpxuLNQztZgMB off=1054651 len=1101721 2024-09-13T00:20:09.751 INFO:tasks.workunit.client.0.smithi136.stdout:op 5699 completed, throughput=5MB/sec 2024-09-13T00:20:09.751 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-IgFVF1GVmm79tC2 off=3630 len=813332 2024-09-13T00:20:09.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:20:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:20:09] "GET /metrics HTTP/1.1" 200 34142 "" "Prometheus/2.43.0" 2024-09-13T00:20:09.759 INFO:tasks.workunit.client.0.smithi136.stdout:op 5700 completed, throughput=5MB/sec 2024-09-13T00:20:09.759 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-3y7uM9q4ADFEWlP off=11813618 len=1801903 2024-09-13T00:20:09.771 INFO:tasks.workunit.client.0.smithi136.stdout:op 5701 completed, throughput=5MB/sec 2024-09-13T00:20:10.771 INFO:tasks.workunit.client.0.smithi136.stdout: 1092: throughput=5MB/sec pending data=0 2024-09-13T00:20:10.772 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-5c7SFTiCvmgEFQN off=15914742 len=1111538 2024-09-13T00:20:10.781 INFO:tasks.workunit.client.0.smithi136.stdout:op 5702 completed, throughput=5MB/sec 2024-09-13T00:20:10.781 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-z--DXdBfMQa1c5W off=13841018 len=359009 2024-09-13T00:20:10.787 INFO:tasks.workunit.client.0.smithi136.stdout:op 5703 completed, throughput=5MB/sec 2024-09-13T00:20:10.787 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-SWBWgxKp02BesFy off=2646412 len=1806984 2024-09-13T00:20:10.801 INFO:tasks.workunit.client.0.smithi136.stdout:op 5704 completed, throughput=5MB/sec 2024-09-13T00:20:10.802 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-nPuAUm2D5OWIW2y off=20407319 len=1472279 2024-09-13T00:20:10.806 INFO:tasks.workunit.client.0.smithi136.stdout:op 5705 completed, throughput=5MB/sec 2024-09-13T00:20:11.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:20:11 smithi136 bash[75164]: cluster 2024-09-13T00:20:09.706029+0000 mgr.y (mgr.24661) 1291 : cluster [DBG] pgmap v1974: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 2.8 MiB/s rd, 1.9 MiB/s wr, 3 op/s 2024-09-13T00:20:11.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:20:11 smithi136 bash[76371]: cluster 2024-09-13T00:20:09.706029+0000 mgr.y (mgr.24661) 1291 : cluster [DBG] pgmap v1974: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 2.8 MiB/s rd, 1.9 MiB/s wr, 3 op/s 2024-09-13T00:20:11.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:20:11 smithi181 bash[67587]: cluster 2024-09-13T00:20:09.706029+0000 mgr.y (mgr.24661) 1291 : cluster [DBG] pgmap v1974: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 2.8 MiB/s rd, 1.9 MiB/s wr, 3 op/s 2024-09-13T00:20:11.806 INFO:tasks.workunit.client.0.smithi136.stdout: 1093: throughput=5MB/sec pending data=0 2024-09-13T00:20:11.806 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-UQzqAAJRc4hQsm5 off=2393875 len=1001319 2024-09-13T00:20:11.815 INFO:tasks.workunit.client.0.smithi136.stdout:op 5706 completed, throughput=5MB/sec 2024-09-13T00:20:11.815 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-tITvC0QF8ylb2vk off=4668621 len=1236497 2024-09-13T00:20:11.825 INFO:tasks.workunit.client.0.smithi136.stdout:op 5707 completed, throughput=5MB/sec 2024-09-13T00:20:11.826 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-tncM78rrZSIYDTC off=8104391 len=767470 2024-09-13T00:20:11.836 INFO:tasks.workunit.client.0.smithi136.stdout:op 5708 completed, throughput=5MB/sec 2024-09-13T00:20:11.836 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ZvpPzOZWN-W-Gcr off=9833439 len=1107904 2024-09-13T00:20:11.836 INFO:tasks.workunit.client.0.smithi136.stdout:op 5709 completed, throughput=5MB/sec 2024-09-13T00:20:11.836 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-UuRQxXoYMYwnKSd off=5955720 len=845109 2024-09-13T00:20:11.839 INFO:tasks.workunit.client.0.smithi136.stdout:op 5710 completed, throughput=5MB/sec 2024-09-13T00:20:11.839 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Z9c9yZ-BJe9iRoQ off=256900 len=553046 2024-09-13T00:20:11.846 INFO:tasks.workunit.client.0.smithi136.stdout:op 5711 completed, throughput=5MB/sec 2024-09-13T00:20:12.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:20:12 smithi181 bash[95505]: ts=2024-09-13T00:20:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:20:12.846 INFO:tasks.workunit.client.0.smithi136.stdout: 1094: throughput=5MB/sec pending data=0 2024-09-13T00:20:12.847 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-l7wHCILOxo-effa off=16949865 len=761642 2024-09-13T00:20:12.849 INFO:tasks.workunit.client.0.smithi136.stdout:op 5712 completed, throughput=5MB/sec 2024-09-13T00:20:12.849 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-kSggKJhiUd-nUnp off=3230772 len=1040788 2024-09-13T00:20:12.860 INFO:tasks.workunit.client.0.smithi136.stdout:op 5713 completed, throughput=5MB/sec 2024-09-13T00:20:12.860 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj--bS23nabhl-BIlN off=5918121 len=1246137 2024-09-13T00:20:12.863 INFO:tasks.workunit.client.0.smithi136.stdout:op 5714 completed, throughput=5MB/sec 2024-09-13T00:20:12.863 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_OFjUJaDq5Yh80v off=3128975 len=1518357 2024-09-13T00:20:12.874 INFO:tasks.workunit.client.0.smithi136.stdout:op 5715 completed, throughput=5MB/sec 2024-09-13T00:20:12.874 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-tITvC0QF8ylb2vk off=1495849 len=488172 2024-09-13T00:20:12.877 INFO:tasks.workunit.client.0.smithi136.stdout:op 5716 completed, throughput=5MB/sec 2024-09-13T00:20:12.877 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-4GydhGg6XIjj-0Z off=5844896 len=372305 2024-09-13T00:20:12.883 INFO:tasks.workunit.client.0.smithi136.stdout:op 5717 completed, throughput=5MB/sec 2024-09-13T00:20:13.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:20:13 smithi136 bash[75164]: cluster 2024-09-13T00:20:11.707144+0000 mgr.y (mgr.24661) 1292 : cluster [DBG] pgmap v1975: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.8 MiB/s wr, 4 op/s 2024-09-13T00:20:13.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:20:13 smithi136 bash[76371]: cluster 2024-09-13T00:20:11.707144+0000 mgr.y (mgr.24661) 1292 : cluster [DBG] pgmap v1975: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.8 MiB/s wr, 4 op/s 2024-09-13T00:20:13.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:20:13 smithi181 bash[67587]: cluster 2024-09-13T00:20:11.707144+0000 mgr.y (mgr.24661) 1292 : cluster [DBG] pgmap v1975: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.8 MiB/s wr, 4 op/s 2024-09-13T00:20:13.884 INFO:tasks.workunit.client.0.smithi136.stdout: 1095: throughput=5MB/sec pending data=0 2024-09-13T00:20:13.885 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-z0gjw-Cv4b4-7Fi off=9702745 len=224230 2024-09-13T00:20:13.889 INFO:tasks.workunit.client.0.smithi136.stdout:op 5718 completed, throughput=5MB/sec 2024-09-13T00:20:13.889 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-zqXO_OailN-ltMl off=5364841 len=283373 2024-09-13T00:20:13.894 INFO:tasks.workunit.client.0.smithi136.stdout:op 5719 completed, throughput=5MB/sec 2024-09-13T00:20:13.894 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2UY9yVah5vJGxGp off=2554321 len=671291 2024-09-13T00:20:13.896 INFO:tasks.workunit.client.0.smithi136.stdout:op 5720 completed, throughput=5MB/sec 2024-09-13T00:20:13.896 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-46Gs9k15MYbwq5R off=10067388 len=756149 2024-09-13T00:20:13.899 INFO:tasks.workunit.client.0.smithi136.stdout:op 5721 completed, throughput=5MB/sec 2024-09-13T00:20:13.899 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-MdH40jZyUmmr61W off=19151084 len=1030432 2024-09-13T00:20:13.901 INFO:tasks.workunit.client.0.smithi136.stdout:op 5722 completed, throughput=5MB/sec 2024-09-13T00:20:13.901 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-KGX6j1KmjTLt7K4 off=6681083 len=883636 2024-09-13T00:20:13.905 INFO:tasks.workunit.client.0.smithi136.stdout:op 5723 completed, throughput=5MB/sec 2024-09-13T00:20:13.905 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-cZZ09Ou1taHj4UY off=16404487 len=232843 2024-09-13T00:20:13.906 INFO:tasks.workunit.client.0.smithi136.stdout:op 5724 completed, throughput=5MB/sec 2024-09-13T00:20:13.906 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-qvp4F4u86otPOLn off=18808122 len=986264 2024-09-13T00:20:13.916 INFO:tasks.workunit.client.0.smithi136.stdout:op 5725 completed, throughput=5MB/sec 2024-09-13T00:20:13.916 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-H67ORsXsmdU-Xz_ off=7917405 len=1747237 2024-09-13T00:20:13.920 INFO:tasks.workunit.client.0.smithi136.stdout:op 5726 completed, throughput=5MB/sec 2024-09-13T00:20:14.098 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 13 00:20:13 smithi181 bash[98820]: logger=cleanup t=2024-09-13T00:20:13.841455488Z level=info msg="Completed cleanup jobs" duration=147.633338ms 2024-09-13T00:20:14.921 INFO:tasks.workunit.client.0.smithi136.stdout: 1096: throughput=5MB/sec pending data=0 2024-09-13T00:20:14.921 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-C-c6rPSPR_WQIex off=2545572 len=1894555 2024-09-13T00:20:14.925 INFO:tasks.workunit.client.0.smithi136.stdout:op 5727 completed, throughput=5MB/sec 2024-09-13T00:20:14.925 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-5cIhqG8zr4u-1oO off=1829979 len=923148 2024-09-13T00:20:14.927 INFO:tasks.workunit.client.0.smithi136.stdout:op 5728 completed, throughput=5MB/sec 2024-09-13T00:20:14.928 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-bXEizNFGggny7x_ off=8783567 len=1306644 2024-09-13T00:20:14.932 INFO:tasks.workunit.client.0.smithi136.stdout:op 5729 completed, throughput=5MB/sec 2024-09-13T00:20:15.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:20:15 smithi136 bash[76371]: cluster 2024-09-13T00:20:13.708027+0000 mgr.y (mgr.24661) 1293 : cluster [DBG] pgmap v1976: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:20:15.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:20:15 smithi136 bash[75164]: cluster 2024-09-13T00:20:13.708027+0000 mgr.y (mgr.24661) 1293 : cluster [DBG] pgmap v1976: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:20:15.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:20:15 smithi181 bash[67587]: cluster 2024-09-13T00:20:13.708027+0000 mgr.y (mgr.24661) 1293 : cluster [DBG] pgmap v1976: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:20:15.933 INFO:tasks.workunit.client.0.smithi136.stdout: 1097: throughput=5MB/sec pending data=0 2024-09-13T00:20:15.933 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-e72ji0deJOEXos0 off=1390366 len=753079 2024-09-13T00:20:15.935 INFO:tasks.workunit.client.0.smithi136.stdout:op 5730 completed, throughput=5MB/sec 2024-09-13T00:20:15.935 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-LMv2xlFTjH4Jp6r off=11929413 len=1850688 2024-09-13T00:20:15.939 INFO:tasks.workunit.client.0.smithi136.stdout:op 5731 completed, throughput=5MB/sec 2024-09-13T00:20:15.939 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-UzaiDgo6lSMTebb off=6334773 len=1405335 2024-09-13T00:20:15.943 INFO:tasks.workunit.client.0.smithi136.stdout:op 5732 completed, throughput=5MB/sec 2024-09-13T00:20:15.943 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-s1RmE2vfRjKA-as off=1804159 len=790815 2024-09-13T00:20:15.945 INFO:tasks.workunit.client.0.smithi136.stdout:op 5733 completed, throughput=5MB/sec 2024-09-13T00:20:15.945 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-a13j71WO3IrILvH off=8378917 len=110384 2024-09-13T00:20:15.950 INFO:tasks.workunit.client.0.smithi136.stdout:op 5734 completed, throughput=5MB/sec 2024-09-13T00:20:16.951 INFO:tasks.workunit.client.0.smithi136.stdout: 1098: throughput=5MB/sec pending data=0 2024-09-13T00:20:16.951 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-9kbXGeEQKGZ2HKB off=3044466 len=1186470 2024-09-13T00:20:16.955 INFO:tasks.workunit.client.0.smithi136.stdout:op 5735 completed, throughput=5MB/sec 2024-09-13T00:20:16.955 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Aco17u9PeFgXhfI off=1941690 len=1948526 2024-09-13T00:20:16.961 INFO:tasks.workunit.client.0.smithi136.stdout:op 5736 completed, throughput=5MB/sec 2024-09-13T00:20:16.961 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-n8DVEJBKUzTBeq5 off=14895557 len=2075088 2024-09-13T00:20:16.966 INFO:tasks.workunit.client.0.smithi136.stdout:op 5737 completed, throughput=5MB/sec 2024-09-13T00:20:16.966 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-9KHHCG5YyAdS6-S off=5638811 len=150055 2024-09-13T00:20:16.967 INFO:tasks.workunit.client.0.smithi136.stdout:op 5738 completed, throughput=5MB/sec 2024-09-13T00:20:17.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:20:17 smithi136 bash[75164]: cluster 2024-09-13T00:20:15.708945+0000 mgr.y (mgr.24661) 1294 : cluster [DBG] pgmap v1977: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:20:17.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:20:17 smithi136 bash[76371]: cluster 2024-09-13T00:20:15.708945+0000 mgr.y (mgr.24661) 1294 : cluster [DBG] pgmap v1977: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:20:17.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:20:17 smithi181 bash[67587]: cluster 2024-09-13T00:20:15.708945+0000 mgr.y (mgr.24661) 1294 : cluster [DBG] pgmap v1977: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:20:17.968 INFO:tasks.workunit.client.0.smithi136.stdout: 1099: throughput=5MB/sec pending data=0 2024-09-13T00:20:17.968 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-2XCRV2xJDFwdYtJ off=14268884 len=1661321 2024-09-13T00:20:17.981 INFO:tasks.workunit.client.0.smithi136.stdout:op 5739 completed, throughput=5MB/sec 2024-09-13T00:20:17.981 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-JrWJ6GPXyQfuAyf off=21420158 len=1426484 2024-09-13T00:20:17.994 INFO:tasks.workunit.client.0.smithi136.stdout:op 5740 completed, throughput=5MB/sec 2024-09-13T00:20:17.994 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-AEFlxa03ko3Ua4C off=7017789 len=651854 2024-09-13T00:20:17.997 INFO:tasks.workunit.client.0.smithi136.stdout:op 5741 completed, throughput=5MB/sec 2024-09-13T00:20:17.997 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-YcqxO9Ag9w9R3Uh off=3115579 len=177852 2024-09-13T00:20:18.002 INFO:tasks.workunit.client.0.smithi136.stdout:op 5742 completed, throughput=5MB/sec 2024-09-13T00:20:18.002 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-QvSoyP4sRqgw16A off=2638226 len=294819 2024-09-13T00:20:18.004 INFO:tasks.workunit.client.0.smithi136.stdout:op 5743 completed, throughput=5MB/sec 2024-09-13T00:20:18.004 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-aGE-uyGfWeFeTto off=3134379 len=558685 2024-09-13T00:20:18.006 INFO:tasks.workunit.client.0.smithi136.stdout:op 5744 completed, throughput=5MB/sec 2024-09-13T00:20:18.006 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-cz0ZxUixkjj2t5R off=7926060 len=1308901 2024-09-13T00:20:18.011 INFO:tasks.workunit.client.0.smithi136.stdout:op 5745 completed, throughput=5MB/sec 2024-09-13T00:20:19.011 INFO:tasks.workunit.client.0.smithi136.stdout: 1100: throughput=5MB/sec pending data=0 2024-09-13T00:20:19.011 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-JVQna1WD40vR4gx off=13854652 len=1206587 2024-09-13T00:20:19.020 INFO:tasks.workunit.client.0.smithi136.stdout:op 5746 completed, throughput=5MB/sec 2024-09-13T00:20:19.021 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-X5N6DVROc8fqVJD off=3862649 len=1604594 2024-09-13T00:20:19.032 INFO:tasks.workunit.client.0.smithi136.stdout:op 5747 completed, throughput=5MB/sec 2024-09-13T00:20:19.032 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-UfF4yFzeO9-JkyO off=6181274 len=1715400 2024-09-13T00:20:19.044 INFO:tasks.workunit.client.0.smithi136.stdout:op 5748 completed, throughput=5MB/sec 2024-09-13T00:20:19.044 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj--BtqvmkxC5_HLMc off=10833450 len=451718 2024-09-13T00:20:19.050 INFO:tasks.workunit.client.0.smithi136.stdout:op 5749 completed, throughput=5MB/sec 2024-09-13T00:20:19.412 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:20:19 smithi136 bash[75164]: cluster 2024-09-13T00:20:17.709966+0000 mgr.y (mgr.24661) 1295 : cluster [DBG] pgmap v1978: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:20:19.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:20:19 smithi136 bash[76371]: cluster 2024-09-13T00:20:17.709966+0000 mgr.y (mgr.24661) 1295 : cluster [DBG] pgmap v1978: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:20:19.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:20:19 smithi181 bash[67587]: cluster 2024-09-13T00:20:17.709966+0000 mgr.y (mgr.24661) 1295 : cluster [DBG] pgmap v1978: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.5 MiB/s wr, 5 op/s 2024-09-13T00:20:19.758 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:20:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:20:19] "GET /metrics HTTP/1.1" 200 34142 "" "Prometheus/2.43.0" 2024-09-13T00:20:20.050 INFO:tasks.workunit.client.0.smithi136.stdout: 1101: throughput=5MB/sec pending data=0 2024-09-13T00:20:20.050 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-vq1qDrGTb9sv-ZB off=2421275 len=60562 2024-09-13T00:20:20.051 INFO:tasks.workunit.client.0.smithi136.stdout:op 5750 completed, throughput=5MB/sec 2024-09-13T00:20:20.051 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-662kQYWXPSk2hNS off=15361382 len=441197 2024-09-13T00:20:20.057 INFO:tasks.workunit.client.0.smithi136.stdout:op 5751 completed, throughput=5MB/sec 2024-09-13T00:20:20.057 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-l6z8CJk_tl30-f2 off=12270819 len=321826 2024-09-13T00:20:20.063 INFO:tasks.workunit.client.0.smithi136.stdout:op 5752 completed, throughput=5MB/sec 2024-09-13T00:20:20.063 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-hm7BSrP6icqA6tw off=2060849 len=30389 2024-09-13T00:20:20.068 INFO:tasks.workunit.client.0.smithi136.stdout:op 5753 completed, throughput=5MB/sec 2024-09-13T00:20:20.068 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj--pinnbQMkGjJIxt off=15123758 len=1752744 2024-09-13T00:20:20.072 INFO:tasks.workunit.client.0.smithi136.stdout:op 5754 completed, throughput=5MB/sec 2024-09-13T00:20:20.072 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_vE3VRH5EN3OJUc off=4619773 len=1636952 2024-09-13T00:20:20.084 INFO:tasks.workunit.client.0.smithi136.stdout:op 5755 completed, throughput=5MB/sec 2024-09-13T00:20:20.084 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-wtmbmxMcBMDFlto off=12817358 len=840596 2024-09-13T00:20:20.091 INFO:tasks.workunit.client.0.smithi136.stdout:op 5756 completed, throughput=5MB/sec 2024-09-13T00:20:20.091 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-039NnjwWKRKrrV4 off=21355774 len=1447816 2024-09-13T00:20:20.095 INFO:tasks.workunit.client.0.smithi136.stdout:op 5757 completed, throughput=5MB/sec 2024-09-13T00:20:21.096 INFO:tasks.workunit.client.0.smithi136.stdout: 1102: throughput=5MB/sec pending data=0 2024-09-13T00:20:21.096 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-OR_iXrzC4pGE3jS off=2090307 len=1423933 2024-09-13T00:20:21.100 INFO:tasks.workunit.client.0.smithi136.stdout:op 5758 completed, throughput=5MB/sec 2024-09-13T00:20:21.100 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-5Z6-0JIJ9KYuRtR off=5838999 len=1798910 2024-09-13T00:20:21.105 INFO:tasks.workunit.client.0.smithi136.stdout:op 5759 completed, throughput=5MB/sec 2024-09-13T00:20:21.105 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-15qKFJsdsQDrX5t off=15883175 len=974161 2024-09-13T00:20:21.107 INFO:tasks.workunit.client.0.smithi136.stdout:op 5760 completed, throughput=5MB/sec 2024-09-13T00:20:21.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:20:21 smithi136 bash[75164]: cluster 2024-09-13T00:20:19.710789+0000 mgr.y (mgr.24661) 1296 : cluster [DBG] pgmap v1979: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:20:21.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:20:21 smithi136 bash[76371]: cluster 2024-09-13T00:20:19.710789+0000 mgr.y (mgr.24661) 1296 : cluster [DBG] pgmap v1979: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:20:21.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:20:21 smithi181 bash[67587]: cluster 2024-09-13T00:20:19.710789+0000 mgr.y (mgr.24661) 1296 : cluster [DBG] pgmap v1979: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.2 MiB/s wr, 4 op/s 2024-09-13T00:20:22.108 INFO:tasks.workunit.client.0.smithi136.stdout: 1103: throughput=5MB/sec pending data=0 2024-09-13T00:20:22.108 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-3yNrwryM3RwXmaK off=134745 len=99974 2024-09-13T00:20:22.113 INFO:tasks.workunit.client.0.smithi136.stdout:op 5761 completed, throughput=5MB/sec 2024-09-13T00:20:22.113 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Fe_3o_UW09EJiPR off=1510838 len=602512 2024-09-13T00:20:22.116 INFO:tasks.workunit.client.0.smithi136.stdout:op 5762 completed, throughput=5MB/sec 2024-09-13T00:20:22.116 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-xr6P8ggWmpQ9Dp3 off=692261 len=329453 2024-09-13T00:20:22.118 INFO:tasks.workunit.client.0.smithi136.stdout:op 5763 completed, throughput=5MB/sec 2024-09-13T00:20:22.118 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-uOzMnPW8gcmiXy_ off=8762437 len=289369 2024-09-13T00:20:22.124 INFO:tasks.workunit.client.0.smithi136.stdout:op 5764 completed, throughput=5MB/sec 2024-09-13T00:20:22.124 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-S9mJmnitye-67r6 off=6874313 len=1631376 2024-09-13T00:20:22.140 INFO:tasks.workunit.client.0.smithi136.stdout:op 5765 completed, throughput=5MB/sec 2024-09-13T00:20:22.140 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-XsdCJo-0vb7whiU off=4984346 len=297401 2024-09-13T00:20:22.147 INFO:tasks.workunit.client.0.smithi136.stdout:op 5766 completed, throughput=5MB/sec 2024-09-13T00:20:22.147 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-E7yQ1-8cToF9Io1 off=7363206 len=42636 2024-09-13T00:20:22.151 INFO:tasks.workunit.client.0.smithi136.stdout:op 5767 completed, throughput=5MB/sec 2024-09-13T00:20:22.151 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-gcFCVMlEqtRqKO1 off=4604233 len=121621 2024-09-13T00:20:22.156 INFO:tasks.workunit.client.0.smithi136.stdout:op 5768 completed, throughput=5MB/sec 2024-09-13T00:20:22.156 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-c_UnBJSxRIanhiW off=10036829 len=1764841 2024-09-13T00:20:22.170 INFO:tasks.workunit.client.0.smithi136.stdout:op 5769 completed, throughput=5MB/sec 2024-09-13T00:20:22.170 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-dkttGpjqFo2jdcg off=747159 len=512003 2024-09-13T00:20:22.172 INFO:tasks.workunit.client.0.smithi136.stdout:op 5770 completed, throughput=5MB/sec 2024-09-13T00:20:22.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:20:22 smithi181 bash[95505]: ts=2024-09-13T00:20:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:20:23.172 INFO:tasks.workunit.client.0.smithi136.stdout: 1104: throughput=5MB/sec pending data=0 2024-09-13T00:20:23.173 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-1bkIpAP7RpfVtgp off=18733116 len=680557 2024-09-13T00:20:23.175 INFO:tasks.workunit.client.0.smithi136.stdout:op 5771 completed, throughput=5MB/sec 2024-09-13T00:20:23.175 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-i059RDrESy9aeXS off=9051998 len=426574 2024-09-13T00:20:23.177 INFO:tasks.workunit.client.0.smithi136.stdout:op 5772 completed, throughput=5MB/sec 2024-09-13T00:20:23.178 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-O-vEWFwkB3wyZIn off=694631 len=1147343 2024-09-13T00:20:23.181 INFO:tasks.workunit.client.0.smithi136.stdout:op 5773 completed, throughput=5MB/sec 2024-09-13T00:20:23.181 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-psBvz_dZsGPREPW off=422802 len=943228 2024-09-13T00:20:23.184 INFO:tasks.workunit.client.0.smithi136.stdout:op 5774 completed, throughput=5MB/sec 2024-09-13T00:20:23.184 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-NdZTmYsKyG0cJnm off=9020791 len=1174619 2024-09-13T00:20:23.194 INFO:tasks.workunit.client.0.smithi136.stdout:op 5775 completed, throughput=5MB/sec 2024-09-13T00:20:23.194 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-PPGHDkknmSBoWra off=3270208 len=60435 2024-09-13T00:20:23.195 INFO:tasks.workunit.client.0.smithi136.stdout:op 5776 completed, throughput=5MB/sec 2024-09-13T00:20:23.195 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj--r9o0BpbEZ2fuOl off=811403 len=1567382 2024-09-13T00:20:23.199 INFO:tasks.workunit.client.0.smithi136.stdout:op 5777 completed, throughput=5MB/sec 2024-09-13T00:20:23.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:20:23 smithi136 bash[75164]: cluster 2024-09-13T00:20:21.711891+0000 mgr.y (mgr.24661) 1297 : cluster [DBG] pgmap v1980: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:20:23.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:20:23 smithi136 bash[76371]: cluster 2024-09-13T00:20:21.711891+0000 mgr.y (mgr.24661) 1297 : cluster [DBG] pgmap v1980: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:20:23.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:20:23 smithi181 bash[67587]: cluster 2024-09-13T00:20:21.711891+0000 mgr.y (mgr.24661) 1297 : cluster [DBG] pgmap v1980: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:20:24.200 INFO:tasks.workunit.client.0.smithi136.stdout: 1105: throughput=5MB/sec pending data=0 2024-09-13T00:20:24.200 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-DDDUVGRqcIKUCyj off=275337 len=770815 2024-09-13T00:20:24.203 INFO:tasks.workunit.client.0.smithi136.stdout:op 5778 completed, throughput=5MB/sec 2024-09-13T00:20:24.203 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-0vqzbk5CqR9Qphg off=1790403 len=656046 2024-09-13T00:20:24.205 INFO:tasks.workunit.client.0.smithi136.stdout:op 5779 completed, throughput=5MB/sec 2024-09-13T00:20:24.205 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-rwMieGPWe72VZTa off=7721278 len=1469105 2024-09-13T00:20:24.216 INFO:tasks.workunit.client.0.smithi136.stdout:op 5780 completed, throughput=5MB/sec 2024-09-13T00:20:24.216 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-6J8iC2lNXVAjB-7 off=17141925 len=627495 2024-09-13T00:20:24.218 INFO:tasks.workunit.client.0.smithi136.stdout:op 5781 completed, throughput=5MB/sec 2024-09-13T00:20:24.218 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-dioiS1MhmLyQmAM off=730920 len=977172 2024-09-13T00:20:24.227 INFO:tasks.workunit.client.0.smithi136.stdout:op 5782 completed, throughput=5MB/sec 2024-09-13T00:20:24.228 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-S5UE04dVkGnnBuc off=7749026 len=54742 2024-09-13T00:20:24.232 INFO:tasks.workunit.client.0.smithi136.stdout:op 5783 completed, throughput=5MB/sec 2024-09-13T00:20:24.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:20:24 smithi136 bash[75164]: audit 2024-09-13T00:20:23.803542+0000 mon.a (mon.0) 2276 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:20:24.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:20:24 smithi136 bash[76371]: audit 2024-09-13T00:20:23.803542+0000 mon.a (mon.0) 2276 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:20:24.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:20:24 smithi181 bash[67587]: audit 2024-09-13T00:20:23.803542+0000 mon.a (mon.0) 2276 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:20:25.233 INFO:tasks.workunit.client.0.smithi136.stdout: 1106: throughput=5MB/sec pending data=0 2024-09-13T00:20:25.233 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-jBmcTf3t2lbGPrr off=7458981 len=1526253 2024-09-13T00:20:25.237 INFO:tasks.workunit.client.0.smithi136.stdout:op 5784 completed, throughput=5MB/sec 2024-09-13T00:20:25.237 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ANELxY6qlPkF_GL off=3106944 len=1807838 2024-09-13T00:20:25.241 INFO:tasks.workunit.client.0.smithi136.stdout:op 5785 completed, throughput=5MB/sec 2024-09-13T00:20:25.241 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-xAF2RzYVwPGMhG7 off=1691310 len=2085209 2024-09-13T00:20:25.246 INFO:tasks.workunit.client.0.smithi136.stdout:op 5786 completed, throughput=5MB/sec 2024-09-13T00:20:25.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:20:25 smithi136 bash[75164]: cluster 2024-09-13T00:20:23.712814+0000 mgr.y (mgr.24661) 1298 : cluster [DBG] pgmap v1981: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 2.5 MiB/s rd, 1.7 MiB/s wr, 4 op/s 2024-09-13T00:20:25.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:20:25 smithi136 bash[76371]: cluster 2024-09-13T00:20:23.712814+0000 mgr.y (mgr.24661) 1298 : cluster [DBG] pgmap v1981: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 2.5 MiB/s rd, 1.7 MiB/s wr, 4 op/s 2024-09-13T00:20:25.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:20:25 smithi181 bash[67587]: cluster 2024-09-13T00:20:23.712814+0000 mgr.y (mgr.24661) 1298 : cluster [DBG] pgmap v1981: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 2.5 MiB/s rd, 1.7 MiB/s wr, 4 op/s 2024-09-13T00:20:26.246 INFO:tasks.workunit.client.0.smithi136.stdout: 1107: throughput=5MB/sec pending data=0 2024-09-13T00:20:26.246 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-YRodQNlG2uZeoQ0 off=3075547 len=1563544 2024-09-13T00:20:26.250 INFO:tasks.workunit.client.0.smithi136.stdout:op 5787 completed, throughput=5MB/sec 2024-09-13T00:20:26.250 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-RURuyLzQ_HNAcbe off=7901113 len=678385 2024-09-13T00:20:26.252 INFO:tasks.workunit.client.0.smithi136.stdout:op 5788 completed, throughput=5MB/sec 2024-09-13T00:20:26.253 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-zINzekcscjGDxSX off=13584234 len=761750 2024-09-13T00:20:26.261 INFO:tasks.workunit.client.0.smithi136.stdout:op 5789 completed, throughput=5MB/sec 2024-09-13T00:20:26.261 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-TYUXwCBz6U9YEMV off=1284070 len=977384 2024-09-13T00:20:26.264 INFO:tasks.workunit.client.0.smithi136.stdout:op 5790 completed, throughput=5MB/sec 2024-09-13T00:20:26.265 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_OaoZGBhG7-DiQQ off=11416817 len=1468505 2024-09-13T00:20:26.275 INFO:tasks.workunit.client.0.smithi136.stdout:op 5791 completed, throughput=5MB/sec 2024-09-13T00:20:27.276 INFO:tasks.workunit.client.0.smithi136.stdout: 1108: throughput=5MB/sec pending data=0 2024-09-13T00:20:27.276 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-NQwLRhFsdgx3-0s off=9918474 len=670579 2024-09-13T00:20:27.278 INFO:tasks.workunit.client.0.smithi136.stdout:op 5792 completed, throughput=5MB/sec 2024-09-13T00:20:27.278 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-GjB3-RY6_bv8SzN off=636622 len=256004 2024-09-13T00:20:27.279 INFO:tasks.workunit.client.0.smithi136.stdout:op 5793 completed, throughput=5MB/sec 2024-09-13T00:20:27.279 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-wf110viXnGvMVFd off=733659 len=621685 2024-09-13T00:20:27.282 INFO:tasks.workunit.client.0.smithi136.stdout:op 5794 completed, throughput=5MB/sec 2024-09-13T00:20:27.282 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-32da10P6fnZZrkx off=14321407 len=951658 2024-09-13T00:20:27.284 INFO:tasks.workunit.client.0.smithi136.stdout:op 5795 completed, throughput=5MB/sec 2024-09-13T00:20:27.285 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-No4cG3d9fxOere4 off=1004324 len=1933233 2024-09-13T00:20:27.289 INFO:tasks.workunit.client.0.smithi136.stdout:op 5796 completed, throughput=5MB/sec 2024-09-13T00:20:27.289 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-w6zg3oCx9ud0auD off=4984064 len=1939783 2024-09-13T00:20:27.303 INFO:tasks.workunit.client.0.smithi136.stdout:op 5797 completed, throughput=5MB/sec 2024-09-13T00:20:27.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:20:27 smithi136 bash[75164]: cluster 2024-09-13T00:20:25.713668+0000 mgr.y (mgr.24661) 1299 : cluster [DBG] pgmap v1982: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.8 MiB/s wr, 5 op/s 2024-09-13T00:20:27.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:20:27 smithi136 bash[76371]: cluster 2024-09-13T00:20:25.713668+0000 mgr.y (mgr.24661) 1299 : cluster [DBG] pgmap v1982: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.8 MiB/s wr, 5 op/s 2024-09-13T00:20:27.600 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:20:27 smithi181 bash[67587]: cluster 2024-09-13T00:20:25.713668+0000 mgr.y (mgr.24661) 1299 : cluster [DBG] pgmap v1982: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.8 MiB/s wr, 5 op/s 2024-09-13T00:20:28.304 INFO:tasks.workunit.client.0.smithi136.stdout: 1109: throughput=5MB/sec pending data=0 2024-09-13T00:20:28.304 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj--qgXJLFZXy9bnYj off=6971490 len=990728 2024-09-13T00:20:28.313 INFO:tasks.workunit.client.0.smithi136.stdout:op 5798 completed, throughput=5MB/sec 2024-09-13T00:20:28.313 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-WZCndp825N43pF4 off=2030209 len=1368869 2024-09-13T00:20:28.316 INFO:tasks.workunit.client.0.smithi136.stdout:op 5799 completed, throughput=5MB/sec 2024-09-13T00:20:28.316 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-BPK0IRgfgBPKeF3 off=1224765 len=6185 2024-09-13T00:20:28.317 INFO:tasks.workunit.client.0.smithi136.stdout:op 5800 completed, throughput=5MB/sec 2024-09-13T00:20:28.317 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-MbFk4Z-pMQPoYRC off=860350 len=1135273 2024-09-13T00:20:28.327 INFO:tasks.workunit.client.0.smithi136.stdout:op 5801 completed, throughput=5MB/sec 2024-09-13T00:20:28.327 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-uLAPfuuXBCwcuYX off=6140893 len=1722874 2024-09-13T00:20:28.330 INFO:tasks.workunit.client.0.smithi136.stdout:op 5802 completed, throughput=5MB/sec 2024-09-13T00:20:29.330 INFO:tasks.workunit.client.0.smithi136.stdout: 1110: throughput=5MB/sec pending data=0 2024-09-13T00:20:29.330 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Dbv0qls2_-77iil off=973869 len=219459 2024-09-13T00:20:29.332 INFO:tasks.workunit.client.0.smithi136.stdout:op 5803 completed, throughput=5MB/sec 2024-09-13T00:20:29.332 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-KSCchZdWbyYOFxa off=474670 len=171852 2024-09-13T00:20:29.333 INFO:tasks.workunit.client.0.smithi136.stdout:op 5804 completed, throughput=5MB/sec 2024-09-13T00:20:29.333 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-7umIPbYhWXnsh4o off=268576 len=1275891 2024-09-13T00:20:29.344 INFO:tasks.workunit.client.0.smithi136.stdout:op 5805 completed, throughput=5MB/sec 2024-09-13T00:20:29.344 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2anT3D23ieyBKGI off=1137683 len=1597844 2024-09-13T00:20:29.348 INFO:tasks.workunit.client.0.smithi136.stdout:op 5806 completed, throughput=5MB/sec 2024-09-13T00:20:29.348 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-w0eXjafyffiWy5x off=2870339 len=2051560 2024-09-13T00:20:29.364 INFO:tasks.workunit.client.0.smithi136.stdout:op 5807 completed, throughput=5MB/sec 2024-09-13T00:20:29.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:20:29 smithi136 bash[75164]: cluster 2024-09-13T00:20:27.714705+0000 mgr.y (mgr.24661) 1300 : cluster [DBG] pgmap v1983: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 3.6 MiB/s rd, 1.8 MiB/s wr, 5 op/s 2024-09-13T00:20:29.509 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:20:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:20:29] "GET /metrics HTTP/1.1" 200 34140 "" "Prometheus/2.43.0" 2024-09-13T00:20:29.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:20:29 smithi136 bash[76371]: cluster 2024-09-13T00:20:27.714705+0000 mgr.y (mgr.24661) 1300 : cluster [DBG] pgmap v1983: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 3.6 MiB/s rd, 1.8 MiB/s wr, 5 op/s 2024-09-13T00:20:29.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:20:29 smithi181 bash[67587]: cluster 2024-09-13T00:20:27.714705+0000 mgr.y (mgr.24661) 1300 : cluster [DBG] pgmap v1983: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 3.6 MiB/s rd, 1.8 MiB/s wr, 5 op/s 2024-09-13T00:20:30.364 INFO:tasks.workunit.client.0.smithi136.stdout: 1111: throughput=5MB/sec pending data=0 2024-09-13T00:20:30.364 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-_vJz6OtFJtcJFcA off=263367 len=382982 2024-09-13T00:20:30.366 INFO:tasks.workunit.client.0.smithi136.stdout:op 5808 completed, throughput=5MB/sec 2024-09-13T00:20:30.366 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ZoLaz-63ATAYZNO off=4253876 len=818722 2024-09-13T00:20:30.374 INFO:tasks.workunit.client.0.smithi136.stdout:op 5809 completed, throughput=5MB/sec 2024-09-13T00:20:30.374 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-0bvfJ7kgZOQMlTi off=511087 len=1482151 2024-09-13T00:20:30.384 INFO:tasks.workunit.client.0.smithi136.stdout:op 5810 completed, throughput=5MB/sec 2024-09-13T00:20:30.384 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-yhA1tz-uSwDHu9r off=3520896 len=12898 2024-09-13T00:20:30.389 INFO:tasks.workunit.client.0.smithi136.stdout:op 5811 completed, throughput=5MB/sec 2024-09-13T00:20:30.389 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-_GX8JnL9XNoK-8R off=613782 len=1235229 2024-09-13T00:20:30.393 INFO:tasks.workunit.client.0.smithi136.stdout:op 5812 completed, throughput=5MB/sec 2024-09-13T00:20:30.393 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2wjTY5B-2zk_RTN off=5177004 len=174259 2024-09-13T00:20:30.394 INFO:tasks.workunit.client.0.smithi136.stdout:op 5813 completed, throughput=5MB/sec 2024-09-13T00:20:30.394 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-2_6w15OilP2QrYs off=455388 len=147788 2024-09-13T00:20:30.398 INFO:tasks.workunit.client.0.smithi136.stdout:op 5814 completed, throughput=5MB/sec 2024-09-13T00:20:30.398 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-dCr_7QPJ6pzNuFr off=296800 len=997633 2024-09-13T00:20:30.407 INFO:tasks.workunit.client.0.smithi136.stdout:op 5815 completed, throughput=5MB/sec 2024-09-13T00:20:31.408 INFO:tasks.workunit.client.0.smithi136.stdout: 1112: throughput=5MB/sec pending data=0 2024-09-13T00:20:31.408 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-z2tlyWofM9anqpp off=18805270 len=1070111 2024-09-13T00:20:31.417 INFO:tasks.workunit.client.0.smithi136.stdout:op 5816 completed, throughput=5MB/sec 2024-09-13T00:20:31.417 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-wJCqFdl_l7P2whi off=1620214 len=770437 2024-09-13T00:20:31.425 INFO:tasks.workunit.client.0.smithi136.stdout:op 5817 completed, throughput=5MB/sec 2024-09-13T00:20:31.425 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-8C3DKbB3zVLbCPr off=807552 len=467057 2024-09-13T00:20:31.431 INFO:tasks.workunit.client.0.smithi136.stdout:op 5818 completed, throughput=5MB/sec 2024-09-13T00:20:31.431 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-rmCSIr5CbtP1yM- off=5001325 len=430374 2024-09-13T00:20:31.437 INFO:tasks.workunit.client.0.smithi136.stdout:op 5819 completed, throughput=5MB/sec 2024-09-13T00:20:31.437 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-nYQd1SMj3kmilH5 off=2604950 len=1635432 2024-09-13T00:20:31.448 INFO:tasks.workunit.client.0.smithi136.stdout:op 5820 completed, throughput=5MB/sec 2024-09-13T00:20:31.448 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-G9aq3aGbkE05nPI off=2054011 len=231132 2024-09-13T00:20:31.453 INFO:tasks.workunit.client.0.smithi136.stdout:op 5821 completed, throughput=5MB/sec 2024-09-13T00:20:31.453 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-4LYAOSk1hqZ9OCw off=1952678 len=1250923 2024-09-13T00:20:31.458 INFO:tasks.workunit.client.0.smithi136.stdout:op 5822 completed, throughput=5MB/sec 2024-09-13T00:20:31.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:20:31 smithi136 bash[76371]: cluster 2024-09-13T00:20:29.715555+0000 mgr.y (mgr.24661) 1301 : cluster [DBG] pgmap v1984: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.0 MiB/s wr, 5 op/s 2024-09-13T00:20:31.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:20:31 smithi136 bash[75164]: cluster 2024-09-13T00:20:29.715555+0000 mgr.y (mgr.24661) 1301 : cluster [DBG] pgmap v1984: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.0 MiB/s wr, 5 op/s 2024-09-13T00:20:31.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:20:31 smithi181 bash[67587]: cluster 2024-09-13T00:20:29.715555+0000 mgr.y (mgr.24661) 1301 : cluster [DBG] pgmap v1984: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.0 MiB/s wr, 5 op/s 2024-09-13T00:20:32.458 INFO:tasks.workunit.client.0.smithi136.stdout: 1113: throughput=5MB/sec pending data=0 2024-09-13T00:20:32.458 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-zvyl3mzM48TXTwq off=9392132 len=1157575 2024-09-13T00:20:32.469 INFO:tasks.workunit.client.0.smithi136.stdout:op 5823 completed, throughput=5MB/sec 2024-09-13T00:20:32.469 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-XVzTqlVoeuPAlfA off=1205784 len=1664081 2024-09-13T00:20:32.475 INFO:tasks.workunit.client.0.smithi136.stdout:op 5824 completed, throughput=5MB/sec 2024-09-13T00:20:32.475 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-j1HBsVb1BGJ5X3J off=3610753 len=880252 2024-09-13T00:20:32.483 INFO:tasks.workunit.client.0.smithi136.stdout:op 5825 completed, throughput=5MB/sec 2024-09-13T00:20:32.483 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-rhknVCZIaivCahP off=941891 len=548035 2024-09-13T00:20:32.486 INFO:tasks.workunit.client.0.smithi136.stdout:op 5826 completed, throughput=5MB/sec 2024-09-13T00:20:32.486 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Z-0gCoP91L-6CGG off=4857739 len=1719785 2024-09-13T00:20:32.491 INFO:tasks.workunit.client.0.smithi136.stdout:op 5827 completed, throughput=5MB/sec 2024-09-13T00:20:32.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:20:32 smithi181 bash[95505]: ts=2024-09-13T00:20:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:20:33.491 INFO:tasks.workunit.client.0.smithi136.stdout: 1114: throughput=5MB/sec pending data=0 2024-09-13T00:20:33.491 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-m_nvv386nVNo-Dj off=8522405 len=1215459 2024-09-13T00:20:33.495 INFO:tasks.workunit.client.0.smithi136.stdout:op 5828 completed, throughput=5MB/sec 2024-09-13T00:20:33.495 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-dUpQe1CcivETkLu off=12877680 len=2077654 2024-09-13T00:20:33.508 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:20:33 smithi136 bash[75164]: cluster 2024-09-13T00:20:31.716633+0000 mgr.y (mgr.24661) 1302 : cluster [DBG] pgmap v1985: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.5 MiB/s wr, 6 op/s 2024-09-13T00:20:33.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:20:33 smithi136 bash[76371]: cluster 2024-09-13T00:20:31.716633+0000 mgr.y (mgr.24661) 1302 : cluster [DBG] pgmap v1985: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.5 MiB/s wr, 6 op/s 2024-09-13T00:20:33.515 INFO:tasks.workunit.client.0.smithi136.stdout:op 5829 completed, throughput=5MB/sec 2024-09-13T00:20:33.515 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-WcF-J1VdYNJeRHm off=23273020 len=350643 2024-09-13T00:20:33.517 INFO:tasks.workunit.client.0.smithi136.stdout:op 5830 completed, throughput=5MB/sec 2024-09-13T00:20:33.518 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Mqa-SMxcGAyX1vo off=6291799 len=1849684 2024-09-13T00:20:33.522 INFO:tasks.workunit.client.0.smithi136.stdout:op 5831 completed, throughput=5MB/sec 2024-09-13T00:20:33.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:20:33 smithi181 bash[67587]: cluster 2024-09-13T00:20:31.716633+0000 mgr.y (mgr.24661) 1302 : cluster [DBG] pgmap v1985: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.5 MiB/s wr, 6 op/s 2024-09-13T00:20:34.522 INFO:tasks.workunit.client.0.smithi136.stdout: 1115: throughput=5MB/sec pending data=0 2024-09-13T00:20:34.522 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-lGBZNdzdOWvSeSC off=10505298 len=1787972 2024-09-13T00:20:34.526 INFO:tasks.workunit.client.0.smithi136.stdout:op 5832 completed, throughput=5MB/sec 2024-09-13T00:20:34.527 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-VXN9inaqjmFPRJa off=5606711 len=1262522 2024-09-13T00:20:34.531 INFO:tasks.workunit.client.0.smithi136.stdout:op 5833 completed, throughput=5MB/sec 2024-09-13T00:20:34.532 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-FUVdo6Ohx9dSr-c off=12716703 len=46927 2024-09-13T00:20:34.532 INFO:tasks.workunit.client.0.smithi136.stdout:op 5834 completed, throughput=5MB/sec 2024-09-13T00:20:34.533 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-n8ckdVHaXeYcqjb off=3147156 len=1360558 2024-09-13T00:20:34.536 INFO:tasks.workunit.client.0.smithi136.stdout:op 5835 completed, throughput=5MB/sec 2024-09-13T00:20:35.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:20:35 smithi136 bash[75164]: cluster 2024-09-13T00:20:33.717541+0000 mgr.y (mgr.24661) 1303 : cluster [DBG] pgmap v1986: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.1 MiB/s wr, 5 op/s 2024-09-13T00:20:35.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:20:35 smithi136 bash[76371]: cluster 2024-09-13T00:20:33.717541+0000 mgr.y (mgr.24661) 1303 : cluster [DBG] pgmap v1986: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.1 MiB/s wr, 5 op/s 2024-09-13T00:20:35.537 INFO:tasks.workunit.client.0.smithi136.stdout: 1116: throughput=5MB/sec pending data=0 2024-09-13T00:20:35.537 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-pUGreGv21S1HfOa off=1951745 len=711898 2024-09-13T00:20:35.544 INFO:tasks.workunit.client.0.smithi136.stdout:op 5836 completed, throughput=5MB/sec 2024-09-13T00:20:35.545 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-u1S3-6hyXPEWBQJ off=1644111 len=1264543 2024-09-13T00:20:35.549 INFO:tasks.workunit.client.0.smithi136.stdout:op 5837 completed, throughput=5MB/sec 2024-09-13T00:20:35.549 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-0XU9s6N0kLRvrjA off=7687029 len=1407194 2024-09-13T00:20:35.553 INFO:tasks.workunit.client.0.smithi136.stdout:op 5838 completed, throughput=5MB/sec 2024-09-13T00:20:35.553 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-sW33cqgcoJKVs47 off=2117957 len=408078 2024-09-13T00:20:35.558 INFO:tasks.workunit.client.0.smithi136.stdout:op 5839 completed, throughput=5MB/sec 2024-09-13T00:20:35.559 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-wRqoyqA_eFXQWH5 off=3609128 len=586129 2024-09-13T00:20:35.565 INFO:tasks.workunit.client.0.smithi136.stdout:op 5840 completed, throughput=5MB/sec 2024-09-13T00:20:35.565 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-dxgvtuhRg9NeVxE off=420298 len=1882312 2024-09-13T00:20:35.571 INFO:tasks.workunit.client.0.smithi136.stdout:op 5841 completed, throughput=5MB/sec 2024-09-13T00:20:35.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:20:35 smithi181 bash[67587]: cluster 2024-09-13T00:20:33.717541+0000 mgr.y (mgr.24661) 1303 : cluster [DBG] pgmap v1986: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.1 MiB/s wr, 5 op/s 2024-09-13T00:20:36.572 INFO:tasks.workunit.client.0.smithi136.stdout: 1117: throughput=5MB/sec pending data=0 2024-09-13T00:20:36.572 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-RnmqK-ofci7mMXn off=19842849 len=199525 2024-09-13T00:20:36.577 INFO:tasks.workunit.client.0.smithi136.stdout:op 5842 completed, throughput=5MB/sec 2024-09-13T00:20:36.577 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-E6SNZ6benec9zRx off=5256078 len=2019005 2024-09-13T00:20:36.582 INFO:tasks.workunit.client.0.smithi136.stdout:op 5843 completed, throughput=5MB/sec 2024-09-13T00:20:36.582 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-rMToENcmDOQqMMV off=495447 len=1256684 2024-09-13T00:20:36.592 INFO:tasks.workunit.client.0.smithi136.stdout:op 5844 completed, throughput=5MB/sec 2024-09-13T00:20:36.593 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-cZZ09Ou1taHj4UY off=9710319 len=1932782 2024-09-13T00:20:36.607 INFO:tasks.workunit.client.0.smithi136.stdout:op 5845 completed, throughput=5MB/sec 2024-09-13T00:20:37.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:20:37 smithi136 bash[75164]: cluster 2024-09-13T00:20:35.718455+0000 mgr.y (mgr.24661) 1304 : cluster [DBG] pgmap v1987: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:20:37.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:20:37 smithi136 bash[76371]: cluster 2024-09-13T00:20:35.718455+0000 mgr.y (mgr.24661) 1304 : cluster [DBG] pgmap v1987: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:20:37.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:20:37 smithi181 bash[67587]: cluster 2024-09-13T00:20:35.718455+0000 mgr.y (mgr.24661) 1304 : cluster [DBG] pgmap v1987: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:20:37.607 INFO:tasks.workunit.client.0.smithi136.stdout: 1118: throughput=5MB/sec pending data=0 2024-09-13T00:20:37.607 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-M3m_z_pL3etfNiG off=479383 len=2035024 2024-09-13T00:20:37.612 INFO:tasks.workunit.client.0.smithi136.stdout:op 5846 completed, throughput=5MB/sec 2024-09-13T00:20:37.612 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-7jAyxwQWPtlZQZr off=53022 len=379227 2024-09-13T00:20:37.614 INFO:tasks.workunit.client.0.smithi136.stdout:op 5847 completed, throughput=5MB/sec 2024-09-13T00:20:37.614 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-kprhVH2cVjvWqOE off=7085560 len=1491187 2024-09-13T00:20:37.621 INFO:tasks.workunit.client.0.smithi136.stdout:op 5848 completed, throughput=5MB/sec 2024-09-13T00:20:38.618 INFO:tasks.workunit.client.0.smithi136.stdout: 1119: throughput=5MB/sec pending data=0 2024-09-13T00:20:38.618 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-UTvDllhPdpXbxd6 off=5557968 len=2000573 2024-09-13T00:20:38.623 INFO:tasks.workunit.client.0.smithi136.stdout:op 5849 completed, throughput=5MB/sec 2024-09-13T00:20:38.623 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-rqrH_G9pNrgIZ3z off=1922414 len=94347 2024-09-13T00:20:38.628 INFO:tasks.workunit.client.0.smithi136.stdout:op 5850 completed, throughput=5MB/sec 2024-09-13T00:20:38.628 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-UTvDllhPdpXbxd6 off=6968807 len=574995 2024-09-13T00:20:38.629 INFO:tasks.workunit.client.0.smithi136.stdout:op 5851 completed, throughput=5MB/sec 2024-09-13T00:20:38.629 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-5IyGVV07Zw7PwNp off=840966 len=591484 2024-09-13T00:20:38.636 INFO:tasks.workunit.client.0.smithi136.stdout:op 5852 completed, throughput=5MB/sec 2024-09-13T00:20:38.636 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-CWwG_6mEM5k9LMv off=18533344 len=94826 2024-09-13T00:20:38.637 INFO:tasks.workunit.client.0.smithi136.stdout:op 5853 completed, throughput=5MB/sec 2024-09-13T00:20:38.637 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-5AHcM9arxim0r0J off=3906827 len=1868790 2024-09-13T00:20:38.642 INFO:tasks.workunit.client.0.smithi136.stdout:op 5854 completed, throughput=5MB/sec 2024-09-13T00:20:39.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:20:39 smithi136 bash[76371]: cluster 2024-09-13T00:20:37.719600+0000 mgr.y (mgr.24661) 1305 : cluster [DBG] pgmap v1988: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.2 MiB/s wr, 5 op/s 2024-09-13T00:20:39.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:20:39 smithi136 bash[76371]: audit 2024-09-13T00:20:38.803959+0000 mon.a (mon.0) 2277 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:20:39.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:20:39 smithi136 bash[75164]: cluster 2024-09-13T00:20:37.719600+0000 mgr.y (mgr.24661) 1305 : cluster [DBG] pgmap v1988: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.2 MiB/s wr, 5 op/s 2024-09-13T00:20:39.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:20:39 smithi136 bash[75164]: audit 2024-09-13T00:20:38.803959+0000 mon.a (mon.0) 2277 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:20:39.510 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:20:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:20:39] "GET /metrics HTTP/1.1" 200 34142 "" "Prometheus/2.43.0" 2024-09-13T00:20:39.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:20:39 smithi181 bash[67587]: cluster 2024-09-13T00:20:37.719600+0000 mgr.y (mgr.24661) 1305 : cluster [DBG] pgmap v1988: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.2 MiB/s wr, 5 op/s 2024-09-13T00:20:39.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:20:39 smithi181 bash[67587]: audit 2024-09-13T00:20:38.803959+0000 mon.a (mon.0) 2277 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:20:39.642 INFO:tasks.workunit.client.0.smithi136.stdout: 1120: throughput=5MB/sec pending data=0 2024-09-13T00:20:39.642 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-EjvNrJwzSJJayUP off=8750318 len=974263 2024-09-13T00:20:39.646 INFO:tasks.workunit.client.0.smithi136.stdout:op 5855 completed, throughput=5MB/sec 2024-09-13T00:20:39.646 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-qzcl-f3Zv1-SPC3 off=2349108 len=345454 2024-09-13T00:20:39.647 INFO:tasks.workunit.client.0.smithi136.stdout:op 5856 completed, throughput=5MB/sec 2024-09-13T00:20:39.648 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-QGYLaWS5Oahg2B- off=1202450 len=157756 2024-09-13T00:20:39.649 INFO:tasks.workunit.client.0.smithi136.stdout:op 5857 completed, throughput=5MB/sec 2024-09-13T00:20:39.649 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Epp9TWFf6gcgatt off=21568710 len=1280122 2024-09-13T00:20:39.659 INFO:tasks.workunit.client.0.smithi136.stdout:op 5858 completed, throughput=5MB/sec 2024-09-13T00:20:39.659 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-V4huDbqZ408zKNo off=1081328 len=1812706 2024-09-13T00:20:39.671 INFO:tasks.workunit.client.0.smithi136.stdout:op 5859 completed, throughput=5MB/sec 2024-09-13T00:20:39.671 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-VHR9Wkya4xRr7GC off=707338 len=592848 2024-09-13T00:20:39.673 INFO:tasks.workunit.client.0.smithi136.stdout:op 5860 completed, throughput=5MB/sec 2024-09-13T00:20:39.674 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-C4hPj47rr4ShLQL off=9248577 len=416005 2024-09-13T00:20:39.679 INFO:tasks.workunit.client.0.smithi136.stdout:op 5861 completed, throughput=5MB/sec 2024-09-13T00:20:40.680 INFO:tasks.workunit.client.0.smithi136.stdout: 1121: throughput=5MB/sec pending data=0 2024-09-13T00:20:40.680 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-rysmtN8xHDOZ3Q4 off=6786422 len=184531 2024-09-13T00:20:40.685 INFO:tasks.workunit.client.0.smithi136.stdout:op 5862 completed, throughput=5MB/sec 2024-09-13T00:20:40.685 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-uUOBlvLr7hijlwZ off=9310078 len=1769686 2024-09-13T00:20:40.698 INFO:tasks.workunit.client.0.smithi136.stdout:op 5863 completed, throughput=5MB/sec 2024-09-13T00:20:40.698 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-TFnsPN5Jl8BYC6v off=4640626 len=946184 2024-09-13T00:20:40.707 INFO:tasks.workunit.client.0.smithi136.stdout:op 5864 completed, throughput=5MB/sec 2024-09-13T00:20:40.707 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-T-e5BOeLVni4_rN off=2547967 len=516169 2024-09-13T00:20:40.709 INFO:tasks.workunit.client.0.smithi136.stdout:op 5865 completed, throughput=5MB/sec 2024-09-13T00:20:40.709 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-S6iCpTnjWX8PaXC off=17585366 len=30325 2024-09-13T00:20:40.713 INFO:tasks.workunit.client.0.smithi136.stdout:op 5866 completed, throughput=5MB/sec 2024-09-13T00:20:40.714 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-o3zxnXdi4fLI-Aq off=2329288 len=608485 2024-09-13T00:20:40.720 INFO:tasks.workunit.client.0.smithi136.stdout:op 5867 completed, throughput=5MB/sec 2024-09-13T00:20:40.720 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-VmVQi9yB3glCjbg off=5108032 len=1269554 2024-09-13T00:20:40.731 INFO:tasks.workunit.client.0.smithi136.stdout:op 5868 completed, throughput=5MB/sec 2024-09-13T00:20:41.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:20:41 smithi136 bash[75164]: cluster 2024-09-13T00:20:39.720438+0000 mgr.y (mgr.24661) 1306 : cluster [DBG] pgmap v1989: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.7 MiB/s rd, 1.9 MiB/s wr, 4 op/s 2024-09-13T00:20:41.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:20:41 smithi136 bash[76371]: cluster 2024-09-13T00:20:39.720438+0000 mgr.y (mgr.24661) 1306 : cluster [DBG] pgmap v1989: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.7 MiB/s rd, 1.9 MiB/s wr, 4 op/s 2024-09-13T00:20:41.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:20:41 smithi181 bash[67587]: cluster 2024-09-13T00:20:39.720438+0000 mgr.y (mgr.24661) 1306 : cluster [DBG] pgmap v1989: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.7 MiB/s rd, 1.9 MiB/s wr, 4 op/s 2024-09-13T00:20:41.731 INFO:tasks.workunit.client.0.smithi136.stdout: 1122: throughput=5MB/sec pending data=0 2024-09-13T00:20:41.731 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-G8cwn2otG4huPFy off=13188224 len=2009956 2024-09-13T00:20:41.736 INFO:tasks.workunit.client.0.smithi136.stdout:op 5869 completed, throughput=5MB/sec 2024-09-13T00:20:41.736 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-13cQiUoYOZF1AMG off=358349 len=1870098 2024-09-13T00:20:41.751 INFO:tasks.workunit.client.0.smithi136.stdout:op 5870 completed, throughput=5MB/sec 2024-09-13T00:20:41.751 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-M5_OZ-CbmmlY6aN off=1870545 len=1118821 2024-09-13T00:20:41.762 INFO:tasks.workunit.client.0.smithi136.stdout:op 5871 completed, throughput=5MB/sec 2024-09-13T00:20:41.762 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-0YMFzo1aKp-eaSo off=6053517 len=1928881 2024-09-13T00:20:41.772 INFO:tasks.workunit.client.0.smithi136.stdout:op 5872 completed, throughput=5MB/sec 2024-09-13T00:20:42.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:20:42 smithi181 bash[95505]: ts=2024-09-13T00:20:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:20:42.772 INFO:tasks.workunit.client.0.smithi136.stdout: 1123: throughput=5MB/sec pending data=0 2024-09-13T00:20:42.772 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-CYsSEEwC-kqpqRc off=18484446 len=509107 2024-09-13T00:20:42.779 INFO:tasks.workunit.client.0.smithi136.stdout:op 5873 completed, throughput=5MB/sec 2024-09-13T00:20:42.779 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-tTCNna-OLNAZjPz off=7346011 len=1132119 2024-09-13T00:20:42.790 INFO:tasks.workunit.client.0.smithi136.stdout:op 5874 completed, throughput=5MB/sec 2024-09-13T00:20:42.790 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-WCsOTRRwhf2yMnS off=10022825 len=699601 2024-09-13T00:20:42.792 INFO:tasks.workunit.client.0.smithi136.stdout:op 5875 completed, throughput=5MB/sec 2024-09-13T00:20:42.792 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-YYL5I9GMlHCh7s6 off=12218787 len=660191 2024-09-13T00:20:42.795 INFO:tasks.workunit.client.0.smithi136.stdout:op 5876 completed, throughput=5MB/sec 2024-09-13T00:20:42.795 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-mctI32l-H70movf off=8480087 len=1538242 2024-09-13T00:20:42.806 INFO:tasks.workunit.client.0.smithi136.stdout:op 5877 completed, throughput=5MB/sec 2024-09-13T00:20:43.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:20:43 smithi136 bash[75164]: cluster 2024-09-13T00:20:41.721551+0000 mgr.y (mgr.24661) 1307 : cluster [DBG] pgmap v1990: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:20:43.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:20:43 smithi136 bash[76371]: cluster 2024-09-13T00:20:41.721551+0000 mgr.y (mgr.24661) 1307 : cluster [DBG] pgmap v1990: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:20:43.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:20:43 smithi181 bash[67587]: cluster 2024-09-13T00:20:41.721551+0000 mgr.y (mgr.24661) 1307 : cluster [DBG] pgmap v1990: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:20:43.807 INFO:tasks.workunit.client.0.smithi136.stdout: 1124: throughput=5MB/sec pending data=0 2024-09-13T00:20:43.807 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-GwPHgkCD7lTjHnU off=2204405 len=1312470 2024-09-13T00:20:43.811 INFO:tasks.workunit.client.0.smithi136.stdout:op 5878 completed, throughput=5MB/sec 2024-09-13T00:20:43.811 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-3KJbdm_Sa4-8G0b off=9340432 len=81822 2024-09-13T00:20:43.816 INFO:tasks.workunit.client.0.smithi136.stdout:op 5879 completed, throughput=5MB/sec 2024-09-13T00:20:43.816 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-I11iZXm0YiLTZdf off=7248138 len=1634871 2024-09-13T00:20:43.828 INFO:tasks.workunit.client.0.smithi136.stdout:op 5880 completed, throughput=5MB/sec 2024-09-13T00:20:43.828 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-_VwKMohdarWXhyt off=5909043 len=759710 2024-09-13T00:20:43.830 INFO:tasks.workunit.client.0.smithi136.stdout:op 5881 completed, throughput=5MB/sec 2024-09-13T00:20:43.830 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-GdidoVnCJv444X1 off=21652480 len=981149 2024-09-13T00:20:43.833 INFO:tasks.workunit.client.0.smithi136.stdout:op 5882 completed, throughput=5MB/sec 2024-09-13T00:20:44.833 INFO:tasks.workunit.client.0.smithi136.stdout: 1125: throughput=5MB/sec pending data=0 2024-09-13T00:20:44.833 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-JWZT_mxlkA77ckK off=5157464 len=267334 2024-09-13T00:20:44.835 INFO:tasks.workunit.client.0.smithi136.stdout:op 5883 completed, throughput=5MB/sec 2024-09-13T00:20:44.835 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-mJEk5SafN3syHO2 off=7108455 len=232526 2024-09-13T00:20:44.840 INFO:tasks.workunit.client.0.smithi136.stdout:op 5884 completed, throughput=5MB/sec 2024-09-13T00:20:44.840 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-RxK0o5A0Dbiy5r8 off=4699662 len=1542185 2024-09-13T00:20:44.850 INFO:tasks.workunit.client.0.smithi136.stdout:op 5885 completed, throughput=5MB/sec 2024-09-13T00:20:44.850 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-0wEnOep_rN6Yubc off=2878365 len=1203406 2024-09-13T00:20:44.862 INFO:tasks.workunit.client.0.smithi136.stdout:op 5886 completed, throughput=5MB/sec 2024-09-13T00:20:44.862 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-pGmVgFWy-jlvEGB off=123886 len=507876 2024-09-13T00:20:44.869 INFO:tasks.workunit.client.0.smithi136.stdout:op 5887 completed, throughput=5MB/sec 2024-09-13T00:20:44.869 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-gl0GbIeulIU7ujk off=2992604 len=259300 2024-09-13T00:20:44.874 INFO:tasks.workunit.client.0.smithi136.stdout:op 5888 completed, throughput=5MB/sec 2024-09-13T00:20:44.874 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Gbl8XZoZ-WG_CVT off=5165761 len=1483736 2024-09-13T00:20:44.884 INFO:tasks.workunit.client.0.smithi136.stdout:op 5889 completed, throughput=5MB/sec 2024-09-13T00:20:44.884 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-wO8EnBic4NHUADI off=4115619 len=991924 2024-09-13T00:20:44.886 INFO:tasks.workunit.client.0.smithi136.stdout:op 5890 completed, throughput=5MB/sec 2024-09-13T00:20:45.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:20:45 smithi136 bash[75164]: cluster 2024-09-13T00:20:43.722382+0000 mgr.y (mgr.24661) 1308 : cluster [DBG] pgmap v1991: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:20:45.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:20:45 smithi136 bash[76371]: cluster 2024-09-13T00:20:43.722382+0000 mgr.y (mgr.24661) 1308 : cluster [DBG] pgmap v1991: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:20:45.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:20:45 smithi181 bash[67587]: cluster 2024-09-13T00:20:43.722382+0000 mgr.y (mgr.24661) 1308 : cluster [DBG] pgmap v1991: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.0 MiB/s wr, 4 op/s 2024-09-13T00:20:45.887 INFO:tasks.workunit.client.0.smithi136.stdout: 1126: throughput=5MB/sec pending data=0 2024-09-13T00:20:45.887 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-eXLyV8keYXCoT2R off=2658167 len=1819461 2024-09-13T00:20:45.891 INFO:tasks.workunit.client.0.smithi136.stdout:op 5891 completed, throughput=5MB/sec 2024-09-13T00:20:45.891 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-tfoUYr3r99o1If0 off=3479994 len=519138 2024-09-13T00:20:45.893 INFO:tasks.workunit.client.0.smithi136.stdout:op 5892 completed, throughput=5MB/sec 2024-09-13T00:20:45.893 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-wISbf0BPjawu6LQ off=3363606 len=1354891 2024-09-13T00:20:45.899 INFO:tasks.workunit.client.0.smithi136.stdout:op 5893 completed, throughput=5MB/sec 2024-09-13T00:20:45.899 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-nrCFslPJa7jF_N_ off=2250098 len=1941326 2024-09-13T00:20:45.912 INFO:tasks.workunit.client.0.smithi136.stdout:op 5894 completed, throughput=5MB/sec 2024-09-13T00:20:46.912 INFO:tasks.workunit.client.0.smithi136.stdout: 1127: throughput=5MB/sec pending data=0 2024-09-13T00:20:46.912 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-YdbN7n0Vt4eyBMi off=6119186 len=1921171 2024-09-13T00:20:46.927 INFO:tasks.workunit.client.0.smithi136.stdout:op 5895 completed, throughput=5MB/sec 2024-09-13T00:20:46.927 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ppArA47C0y5yAJx off=13314054 len=1450646 2024-09-13T00:20:46.932 INFO:tasks.workunit.client.0.smithi136.stdout:op 5896 completed, throughput=5MB/sec 2024-09-13T00:20:46.932 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-BrWOgb2bmrymMKZ off=2854073 len=491194 2024-09-13T00:20:46.935 INFO:tasks.workunit.client.0.smithi136.stdout:op 5897 completed, throughput=5MB/sec 2024-09-13T00:20:46.935 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-MJr0j1i1l4sWlKd off=4271668 len=1035213 2024-09-13T00:20:46.937 INFO:tasks.workunit.client.0.smithi136.stdout:op 5898 completed, throughput=5MB/sec 2024-09-13T00:20:47.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:20:47 smithi136 bash[75164]: cluster 2024-09-13T00:20:45.723291+0000 mgr.y (mgr.24661) 1309 : cluster [DBG] pgmap v1992: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.7 MiB/s rd, 1.8 MiB/s wr, 4 op/s 2024-09-13T00:20:47.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:20:47 smithi136 bash[76371]: cluster 2024-09-13T00:20:45.723291+0000 mgr.y (mgr.24661) 1309 : cluster [DBG] pgmap v1992: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.7 MiB/s rd, 1.8 MiB/s wr, 4 op/s 2024-09-13T00:20:47.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:20:47 smithi181 bash[67587]: cluster 2024-09-13T00:20:45.723291+0000 mgr.y (mgr.24661) 1309 : cluster [DBG] pgmap v1992: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.7 MiB/s rd, 1.8 MiB/s wr, 4 op/s 2024-09-13T00:20:47.938 INFO:tasks.workunit.client.0.smithi136.stdout: 1128: throughput=5MB/sec pending data=0 2024-09-13T00:20:47.938 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-wQahq6PfOyGh-im off=2791353 len=833443 2024-09-13T00:20:47.945 INFO:tasks.workunit.client.0.smithi136.stdout:op 5899 completed, throughput=5MB/sec 2024-09-13T00:20:47.945 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-7Oap0aQjtkbuntj off=1062904 len=551582 2024-09-13T00:20:47.952 INFO:tasks.workunit.client.0.smithi136.stdout:op 5900 completed, throughput=5MB/sec 2024-09-13T00:20:47.952 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-QrWU-mXOPiS9UuL off=3645367 len=1230742 2024-09-13T00:20:47.962 INFO:tasks.workunit.client.0.smithi136.stdout:op 5901 completed, throughput=5MB/sec 2024-09-13T00:20:47.962 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-UgydT1TJKG4qWtA off=545784 len=448995 2024-09-13T00:20:47.968 INFO:tasks.workunit.client.0.smithi136.stdout:op 5902 completed, throughput=5MB/sec 2024-09-13T00:20:47.968 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Odx8eqyAgZR3vW5 off=6921988 len=818538 2024-09-13T00:20:47.970 INFO:tasks.workunit.client.0.smithi136.stdout:op 5903 completed, throughput=5MB/sec 2024-09-13T00:20:47.971 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-7Dp_fNcAJoNe8Zo off=13371925 len=629008 2024-09-13T00:20:47.978 INFO:tasks.workunit.client.0.smithi136.stdout:op 5904 completed, throughput=5MB/sec 2024-09-13T00:20:47.978 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-XfpNb7Ss5_a0jM4 off=11277106 len=1472850 2024-09-13T00:20:47.981 INFO:tasks.workunit.client.0.smithi136.stdout:op 5905 completed, throughput=5MB/sec 2024-09-13T00:20:48.982 INFO:tasks.workunit.client.0.smithi136.stdout: 1129: throughput=5MB/sec pending data=0 2024-09-13T00:20:48.982 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-_LwenDuoweD2xpj off=13530198 len=439567 2024-09-13T00:20:48.984 INFO:tasks.workunit.client.0.smithi136.stdout:op 5906 completed, throughput=5MB/sec 2024-09-13T00:20:48.984 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-sSqA9Qk3Og7KnN2 off=6875689 len=1655828 2024-09-13T00:20:48.995 INFO:tasks.workunit.client.0.smithi136.stdout:op 5907 completed, throughput=5MB/sec 2024-09-13T00:20:48.995 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-Sc5tGmOZqfjK64n off=11299730 len=248945 2024-09-13T00:20:48.997 INFO:tasks.workunit.client.0.smithi136.stdout:op 5908 completed, throughput=5MB/sec 2024-09-13T00:20:48.997 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-JQimYveLIFCTVEY off=3670294 len=485140 2024-09-13T00:20:49.004 INFO:tasks.workunit.client.0.smithi136.stdout:op 5909 completed, throughput=5MB/sec 2024-09-13T00:20:49.004 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-PuTkPlYV4ncEcDz off=10702260 len=124269 2024-09-13T00:20:49.005 INFO:tasks.workunit.client.0.smithi136.stdout:op 5910 completed, throughput=5MB/sec 2024-09-13T00:20:49.006 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-f2Pmw-0K_1LPEq1 off=16420928 len=1011206 2024-09-13T00:20:49.008 INFO:tasks.workunit.client.0.smithi136.stdout:op 5911 completed, throughput=5MB/sec 2024-09-13T00:20:49.008 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-wjMfrDZQNRqFVlI off=5008838 len=1636691 2024-09-13T00:20:49.014 INFO:tasks.workunit.client.0.smithi136.stdout:op 5912 completed, throughput=5MB/sec 2024-09-13T00:20:49.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:20:49 smithi136 bash[75164]: cluster 2024-09-13T00:20:47.724541+0000 mgr.y (mgr.24661) 1310 : cluster [DBG] pgmap v1993: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:20:49.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:20:49 smithi136 bash[76371]: cluster 2024-09-13T00:20:47.724541+0000 mgr.y (mgr.24661) 1310 : cluster [DBG] pgmap v1993: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:20:49.509 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:20:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:20:49] "GET /metrics HTTP/1.1" 200 34142 "" "Prometheus/2.43.0" 2024-09-13T00:20:49.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:20:49 smithi181 bash[67587]: cluster 2024-09-13T00:20:47.724541+0000 mgr.y (mgr.24661) 1310 : cluster [DBG] pgmap v1993: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-13T00:20:50.014 INFO:tasks.workunit.client.0.smithi136.stdout: 1130: throughput=5MB/sec pending data=0 2024-09-13T00:20:50.014 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-KfFD1UKN-kGTSxw off=921854 len=1447225 2024-09-13T00:20:50.018 INFO:tasks.workunit.client.0.smithi136.stdout:op 5913 completed, throughput=5MB/sec 2024-09-13T00:20:50.018 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-RRi1u-pQjNzpE1p off=5953110 len=306651 2024-09-13T00:20:50.020 INFO:tasks.workunit.client.0.smithi136.stdout:op 5914 completed, throughput=5MB/sec 2024-09-13T00:20:50.020 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-WcOVJ1g6LthDWFw off=16164961 len=513345 2024-09-13T00:20:50.027 INFO:tasks.workunit.client.0.smithi136.stdout:op 5915 completed, throughput=5MB/sec 2024-09-13T00:20:50.027 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-dxrYe9VZxy24Ho5 off=328792 len=816827 2024-09-13T00:20:50.035 INFO:tasks.workunit.client.0.smithi136.stdout:op 5916 completed, throughput=5MB/sec 2024-09-13T00:20:50.035 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-rY52X4ELULgHQ7j off=638542 len=913534 2024-09-13T00:20:50.042 INFO:tasks.workunit.client.0.smithi136.stdout:op 5917 completed, throughput=5MB/sec 2024-09-13T00:20:51.043 INFO:tasks.workunit.client.0.smithi136.stdout: 1131: throughput=5MB/sec pending data=0 2024-09-13T00:20:51.043 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-pKOEGForNqgRotR off=12192295 len=1824353 2024-09-13T00:20:51.056 INFO:tasks.workunit.client.0.smithi136.stdout:op 5918 completed, throughput=5MB/sec 2024-09-13T00:20:51.056 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-WkGcqwLlelocavb off=4247980 len=1877382 2024-09-13T00:20:51.068 INFO:tasks.workunit.client.0.smithi136.stdout:op 5919 completed, throughput=5MB/sec 2024-09-13T00:20:51.068 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-D-7XCpB7yj3qWm_ off=734164 len=592264 2024-09-13T00:20:51.075 INFO:tasks.workunit.client.0.smithi136.stdout:op 5920 completed, throughput=5MB/sec 2024-09-13T00:20:51.075 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-A9YdR2WZcnhIb-t off=7956826 len=1932017 2024-09-13T00:20:51.086 INFO:tasks.workunit.client.0.smithi136.stdout:op 5921 completed, throughput=5MB/sec 2024-09-13T00:20:51.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:20:51 smithi136 bash[75164]: cluster 2024-09-13T00:20:49.725351+0000 mgr.y (mgr.24661) 1311 : cluster [DBG] pgmap v1994: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:20:51.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:20:51 smithi136 bash[76371]: cluster 2024-09-13T00:20:49.725351+0000 mgr.y (mgr.24661) 1311 : cluster [DBG] pgmap v1994: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:20:51.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:20:51 smithi181 bash[67587]: cluster 2024-09-13T00:20:49.725351+0000 mgr.y (mgr.24661) 1311 : cluster [DBG] pgmap v1994: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.5 MiB/s wr, 4 op/s 2024-09-13T00:20:52.087 INFO:tasks.workunit.client.0.smithi136.stdout: 1132: throughput=5MB/sec pending data=0 2024-09-13T00:20:52.087 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-m49uKpcNHqzXGcC off=2814476 len=31426 2024-09-13T00:20:52.088 INFO:tasks.workunit.client.0.smithi136.stdout:op 5922 completed, throughput=5MB/sec 2024-09-13T00:20:52.088 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-EJjP95xfbQYW_c4 off=1836383 len=703173 2024-09-13T00:20:52.090 INFO:tasks.workunit.client.0.smithi136.stdout:op 5923 completed, throughput=5MB/sec 2024-09-13T00:20:52.090 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-mmLCGGkmNPK92sF off=6777891 len=1226576 2024-09-13T00:20:52.093 INFO:tasks.workunit.client.0.smithi136.stdout:op 5924 completed, throughput=5MB/sec 2024-09-13T00:20:52.093 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-fOCuAEhqghqVdNb off=17560636 len=144231 2024-09-13T00:20:52.095 INFO:tasks.workunit.client.0.smithi136.stdout:op 5925 completed, throughput=5MB/sec 2024-09-13T00:20:52.095 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-iLz_0wvCXrhx8Z2 off=9316574 len=1358439 2024-09-13T00:20:52.100 INFO:tasks.workunit.client.0.smithi136.stdout:op 5926 completed, throughput=5MB/sec 2024-09-13T00:20:52.100 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-aMqwNBfo3s9nD4m off=3803244 len=626809 2024-09-13T00:20:52.101 INFO:tasks.workunit.client.0.smithi136.stdout:op 5927 completed, throughput=5MB/sec 2024-09-13T00:20:52.102 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-M5L1YxpFmkFqZvp off=334296 len=1708830 2024-09-13T00:20:52.117 INFO:tasks.workunit.client.0.smithi136.stdout:op 5928 completed, throughput=5MB/sec 2024-09-13T00:20:52.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:20:52 smithi181 bash[95505]: ts=2024-09-13T00:20:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:20:53.118 INFO:tasks.workunit.client.0.smithi136.stdout: 1133: throughput=5MB/sec pending data=0 2024-09-13T00:20:53.118 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-k4wtDavRX29BYyC off=218853 len=86513 2024-09-13T00:20:53.122 INFO:tasks.workunit.client.0.smithi136.stdout:op 5929 completed, throughput=5MB/sec 2024-09-13T00:20:53.123 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-C3s2fQ5Bnv9gDEC off=11904746 len=1239046 2024-09-13T00:20:53.126 INFO:tasks.workunit.client.0.smithi136.stdout:op 5930 completed, throughput=5MB/sec 2024-09-13T00:20:53.126 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ACBGIRmPDYkktbh off=791098 len=190768 2024-09-13T00:20:53.132 INFO:tasks.workunit.client.0.smithi136.stdout:op 5931 completed, throughput=5MB/sec 2024-09-13T00:20:53.132 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-VdqWYNekZ3dy7gc off=3601801 len=420062 2024-09-13T00:20:53.134 INFO:tasks.workunit.client.0.smithi136.stdout:op 5932 completed, throughput=5MB/sec 2024-09-13T00:20:53.134 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-X6YIpn57nQqc22x off=15526146 len=188905 2024-09-13T00:20:53.135 INFO:tasks.workunit.client.0.smithi136.stdout:op 5933 completed, throughput=5MB/sec 2024-09-13T00:20:53.135 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-uWEGoE0DrjeTN9U off=9393948 len=1287803 2024-09-13T00:20:53.139 INFO:tasks.workunit.client.0.smithi136.stdout:op 5934 completed, throughput=5MB/sec 2024-09-13T00:20:53.140 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-3u76JqPs7gOeHYa off=9340916 len=809021 2024-09-13T00:20:53.142 INFO:tasks.workunit.client.0.smithi136.stdout:op 5935 completed, throughput=5MB/sec 2024-09-13T00:20:53.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:20:53 smithi136 bash[75164]: cluster 2024-09-13T00:20:51.726373+0000 mgr.y (mgr.24661) 1312 : cluster [DBG] pgmap v1995: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:20:53.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:20:53 smithi136 bash[76371]: cluster 2024-09-13T00:20:51.726373+0000 mgr.y (mgr.24661) 1312 : cluster [DBG] pgmap v1995: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:20:53.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:20:53 smithi181 bash[67587]: cluster 2024-09-13T00:20:51.726373+0000 mgr.y (mgr.24661) 1312 : cluster [DBG] pgmap v1995: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:20:54.143 INFO:tasks.workunit.client.0.smithi136.stdout: 1134: throughput=5MB/sec pending data=0 2024-09-13T00:20:54.143 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-aNClBUepx3-ebFQ off=3227540 len=419810 2024-09-13T00:20:54.149 INFO:tasks.workunit.client.0.smithi136.stdout:op 5936 completed, throughput=5MB/sec 2024-09-13T00:20:54.150 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-nmwEKbT-2-LCije off=13910642 len=1755385 2024-09-13T00:20:54.162 INFO:tasks.workunit.client.0.smithi136.stdout:op 5937 completed, throughput=5MB/sec 2024-09-13T00:20:54.162 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-QtJ07MqoR4-dsIz off=1548760 len=1646761 2024-09-13T00:20:54.172 INFO:tasks.workunit.client.0.smithi136.stdout:op 5938 completed, throughput=5MB/sec 2024-09-13T00:20:54.172 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-tHDPsaELzi-jmQ4 off=10929903 len=1671001 2024-09-13T00:20:54.175 INFO:tasks.workunit.client.0.smithi136.stdout:op 5939 completed, throughput=5MB/sec 2024-09-13T00:20:54.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:20:54 smithi136 bash[75164]: audit 2024-09-13T00:20:53.804502+0000 mon.a (mon.0) 2278 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:20:54.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:20:54 smithi136 bash[76371]: audit 2024-09-13T00:20:53.804502+0000 mon.a (mon.0) 2278 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:20:54.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:20:54 smithi181 bash[67587]: audit 2024-09-13T00:20:53.804502+0000 mon.a (mon.0) 2278 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:20:55.176 INFO:tasks.workunit.client.0.smithi136.stdout: 1135: throughput=5MB/sec pending data=0 2024-09-13T00:20:55.176 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-fMsoJieeP1qz49d off=1808065 len=1611219 2024-09-13T00:20:55.181 INFO:tasks.workunit.client.0.smithi136.stdout:op 5940 completed, throughput=5MB/sec 2024-09-13T00:20:55.181 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-IeIBlBoMZgI7IS8 off=6619441 len=1109852 2024-09-13T00:20:55.184 INFO:tasks.workunit.client.0.smithi136.stdout:op 5941 completed, throughput=5MB/sec 2024-09-13T00:20:55.184 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ndj6DmOhb9b2tDD off=9365016 len=720178 2024-09-13T00:20:55.187 INFO:tasks.workunit.client.0.smithi136.stdout:op 5942 completed, throughput=5MB/sec 2024-09-13T00:20:55.187 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-FOZQmO2azxVxhCt off=14687056 len=984371 2024-09-13T00:20:55.190 INFO:tasks.workunit.client.0.smithi136.stdout:op 5943 completed, throughput=5MB/sec 2024-09-13T00:20:55.190 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-lMOF-XbKwhtxNNX off=14299661 len=1975869 2024-09-13T00:20:55.207 INFO:tasks.workunit.client.0.smithi136.stdout:op 5944 completed, throughput=5MB/sec 2024-09-13T00:20:55.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:20:55 smithi136 bash[75164]: cluster 2024-09-13T00:20:53.727254+0000 mgr.y (mgr.24661) 1313 : cluster [DBG] pgmap v1996: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:20:55.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:20:55 smithi136 bash[76371]: cluster 2024-09-13T00:20:53.727254+0000 mgr.y (mgr.24661) 1313 : cluster [DBG] pgmap v1996: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:20:55.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:20:55 smithi181 bash[67587]: cluster 2024-09-13T00:20:53.727254+0000 mgr.y (mgr.24661) 1313 : cluster [DBG] pgmap v1996: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:20:56.208 INFO:tasks.workunit.client.0.smithi136.stdout: 1136: throughput=5MB/sec pending data=0 2024-09-13T00:20:56.208 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-7MTNK8JrroJAeYn off=5712206 len=848977 2024-09-13T00:20:56.216 INFO:tasks.workunit.client.0.smithi136.stdout:op 5945 completed, throughput=5MB/sec 2024-09-13T00:20:56.216 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-h4oKpzBpdNi15Ru off=14504233 len=96553 2024-09-13T00:20:56.217 INFO:tasks.workunit.client.0.smithi136.stdout:op 5946 completed, throughput=5MB/sec 2024-09-13T00:20:56.217 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_PJKMRj85t0rqMz off=4514136 len=970161 2024-09-13T00:20:56.228 INFO:tasks.workunit.client.0.smithi136.stdout:op 5947 completed, throughput=5MB/sec 2024-09-13T00:20:56.228 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-lTofQ89ZFCbXxj4 off=1985889 len=881994 2024-09-13T00:20:56.229 INFO:tasks.workunit.client.0.smithi136.stdout:op 5948 completed, throughput=5MB/sec 2024-09-13T00:20:56.229 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-sDT1irEpm6w5ovT off=7197707 len=1982955 2024-09-13T00:20:56.233 INFO:tasks.workunit.client.0.smithi136.stdout:op 5949 completed, throughput=5MB/sec 2024-09-13T00:20:57.234 INFO:tasks.workunit.client.0.smithi136.stdout: 1137: throughput=5MB/sec pending data=0 2024-09-13T00:20:57.234 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-HEXZxYzGKovSDEL off=4197061 len=415028 2024-09-13T00:20:57.236 INFO:tasks.workunit.client.0.smithi136.stdout:op 5950 completed, throughput=5MB/sec 2024-09-13T00:20:57.236 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-qwEADQmIP_nFObk off=317137 len=487046 2024-09-13T00:20:57.242 INFO:tasks.workunit.client.0.smithi136.stdout:op 5951 completed, throughput=5MB/sec 2024-09-13T00:20:57.243 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-XHdOsWbnaAUDu25 off=8174681 len=1635299 2024-09-13T00:20:57.256 INFO:tasks.workunit.client.0.smithi136.stdout:op 5952 completed, throughput=5MB/sec 2024-09-13T00:20:57.256 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-KXnb7T4c-LqKeml off=12344898 len=869676 2024-09-13T00:20:57.263 INFO:tasks.workunit.client.0.smithi136.stdout:op 5953 completed, throughput=5MB/sec 2024-09-13T00:20:57.263 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-gZ-fvXIFGHFQNDu off=3981578 len=996530 2024-09-13T00:20:57.266 INFO:tasks.workunit.client.0.smithi136.stdout:op 5954 completed, throughput=5MB/sec 2024-09-13T00:20:57.266 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-1hkuGQhLb1Ay432 off=5493828 len=718599 2024-09-13T00:20:57.274 INFO:tasks.workunit.client.0.smithi136.stdout:op 5955 completed, throughput=5MB/sec 2024-09-13T00:20:57.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:20:57 smithi136 bash[75164]: cluster 2024-09-13T00:20:55.728152+0000 mgr.y (mgr.24661) 1314 : cluster [DBG] pgmap v1997: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 1.8 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:20:57.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:20:57 smithi136 bash[76371]: cluster 2024-09-13T00:20:55.728152+0000 mgr.y (mgr.24661) 1314 : cluster [DBG] pgmap v1997: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 1.8 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:20:57.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:20:57 smithi181 bash[67587]: cluster 2024-09-13T00:20:55.728152+0000 mgr.y (mgr.24661) 1314 : cluster [DBG] pgmap v1997: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 1.8 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:20:58.276 INFO:tasks.workunit.client.0.smithi136.stdout: 1138: throughput=5MB/sec pending data=0 2024-09-13T00:20:58.277 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Tgsod-g5VfBaKUq off=4260026 len=1666053 2024-09-13T00:20:58.287 INFO:tasks.workunit.client.0.smithi136.stdout:op 5956 completed, throughput=5MB/sec 2024-09-13T00:20:58.287 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-wISbf0BPjawu6LQ off=13974305 len=1341482 2024-09-13T00:20:58.300 INFO:tasks.workunit.client.0.smithi136.stdout:op 5957 completed, throughput=5MB/sec 2024-09-13T00:20:58.300 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-zvNM95iKqHlxqx6 off=1617204 len=1373348 2024-09-13T00:20:58.312 INFO:tasks.workunit.client.0.smithi136.stdout:op 5958 completed, throughput=5MB/sec 2024-09-13T00:20:58.312 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-CF0AUH05xQLlyvo off=194193 len=1952285 2024-09-13T00:20:58.329 INFO:tasks.workunit.client.0.smithi136.stdout:op 5959 completed, throughput=5MB/sec 2024-09-13T00:20:59.329 INFO:tasks.workunit.client.0.smithi136.stdout: 1139: throughput=5MB/sec pending data=0 2024-09-13T00:20:59.329 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-mn40BKe1tUqQ6Gw off=2963444 len=1663113 2024-09-13T00:20:59.342 INFO:tasks.workunit.client.0.smithi136.stdout:op 5960 completed, throughput=5MB/sec 2024-09-13T00:20:59.342 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-tcwCrAU3VJPGvlg off=656971 len=1898285 2024-09-13T00:20:59.354 INFO:tasks.workunit.client.0.smithi136.stdout:op 5961 completed, throughput=5MB/sec 2024-09-13T00:20:59.354 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-o4IiTX5Stnryh4m off=2372364 len=854772 2024-09-13T00:20:59.357 INFO:tasks.workunit.client.0.smithi136.stdout:op 5962 completed, throughput=5MB/sec 2024-09-13T00:20:59.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:20:59 smithi136 bash[75164]: cluster 2024-09-13T00:20:57.729183+0000 mgr.y (mgr.24661) 1315 : cluster [DBG] pgmap v1998: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:20:59.509 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:20:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:20:59] "GET /metrics HTTP/1.1" 200 34138 "" "Prometheus/2.43.0" 2024-09-13T00:20:59.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:20:59 smithi136 bash[76371]: cluster 2024-09-13T00:20:57.729183+0000 mgr.y (mgr.24661) 1315 : cluster [DBG] pgmap v1998: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:20:59.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:20:59 smithi181 bash[67587]: cluster 2024-09-13T00:20:57.729183+0000 mgr.y (mgr.24661) 1315 : cluster [DBG] pgmap v1998: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-13T00:21:00.357 INFO:tasks.workunit.client.0.smithi136.stdout: 1140: throughput=5MB/sec pending data=0 2024-09-13T00:21:00.357 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-azxbmwkYnmP2YDT off=2765047 len=165816 2024-09-13T00:21:00.363 INFO:tasks.workunit.client.0.smithi136.stdout:op 5963 completed, throughput=5MB/sec 2024-09-13T00:21:00.363 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-YMRbXiGSjWBto3z off=3515478 len=821119 2024-09-13T00:21:00.371 INFO:tasks.workunit.client.0.smithi136.stdout:op 5964 completed, throughput=5MB/sec 2024-09-13T00:21:00.371 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-HqYRfJfSSCbzKqQ off=3609636 len=341348 2024-09-13T00:21:00.373 INFO:tasks.workunit.client.0.smithi136.stdout:op 5965 completed, throughput=5MB/sec 2024-09-13T00:21:00.373 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-UXjBdu0s3TNC4d8 off=2066413 len=730621 2024-09-13T00:21:00.376 INFO:tasks.workunit.client.0.smithi136.stdout:op 5966 completed, throughput=5MB/sec 2024-09-13T00:21:00.376 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-nElm29kUFhReOvz off=19014390 len=1173440 2024-09-13T00:21:00.386 INFO:tasks.workunit.client.0.smithi136.stdout:op 5967 completed, throughput=5MB/sec 2024-09-13T00:21:00.387 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ZmKgTapdRuivUqm off=6977842 len=19417 2024-09-13T00:21:00.390 INFO:tasks.workunit.client.0.smithi136.stdout:op 5968 completed, throughput=5MB/sec 2024-09-13T00:21:00.391 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-BrMS9ZJdHubncCk off=10921555 len=1883802 2024-09-13T00:21:00.395 INFO:tasks.workunit.client.0.smithi136.stdout:op 5969 completed, throughput=5MB/sec 2024-09-13T00:21:00.395 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-z0gjw-Cv4b4-7Fi off=9755667 len=1987868 2024-09-13T00:21:00.409 INFO:tasks.workunit.client.0.smithi136.stdout:op 5970 completed, throughput=5MB/sec 2024-09-13T00:21:01.410 INFO:tasks.workunit.client.0.smithi136.stdout: 1141: throughput=5MB/sec pending data=0 2024-09-13T00:21:01.410 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-or7XYpc686hSXmg off=10617657 len=639335 2024-09-13T00:21:01.417 INFO:tasks.workunit.client.0.smithi136.stdout:op 5971 completed, throughput=5MB/sec 2024-09-13T00:21:01.417 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-tl0GgKwP3baTJmk off=8389690 len=35924 2024-09-13T00:21:01.418 INFO:tasks.workunit.client.0.smithi136.stdout:op 5972 completed, throughput=5MB/sec 2024-09-13T00:21:01.418 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-1dM4-Fy7--rpaEM off=2186573 len=303515 2024-09-13T00:21:01.420 INFO:tasks.workunit.client.0.smithi136.stdout:op 5973 completed, throughput=5MB/sec 2024-09-13T00:21:01.420 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2Jnk2r9_hZajV2J off=3709383 len=1543219 2024-09-13T00:21:01.425 INFO:tasks.workunit.client.0.smithi136.stdout:op 5974 completed, throughput=5MB/sec 2024-09-13T00:21:01.425 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-eemGviXlmt-xm-0 off=1436410 len=223031 2024-09-13T00:21:01.427 INFO:tasks.workunit.client.0.smithi136.stdout:op 5975 completed, throughput=5MB/sec 2024-09-13T00:21:01.427 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj--abD8qCnRY0iPBm off=4885236 len=2096760 2024-09-13T00:21:01.432 INFO:tasks.workunit.client.0.smithi136.stdout:op 5976 completed, throughput=5MB/sec 2024-09-13T00:21:01.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:21:01 smithi136 bash[75164]: cluster 2024-09-13T00:20:59.730024+0000 mgr.y (mgr.24661) 1316 : cluster [DBG] pgmap v1999: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:21:01.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:21:01 smithi136 bash[75164]: audit 2024-09-13T00:21:00.390380+0000 mon.a (mon.0) 2279 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:21:01.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:21:01 smithi136 bash[76371]: cluster 2024-09-13T00:20:59.730024+0000 mgr.y (mgr.24661) 1316 : cluster [DBG] pgmap v1999: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:21:01.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:21:01 smithi136 bash[76371]: audit 2024-09-13T00:21:00.390380+0000 mon.a (mon.0) 2279 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:21:01.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:21:01 smithi181 bash[67587]: cluster 2024-09-13T00:20:59.730024+0000 mgr.y (mgr.24661) 1316 : cluster [DBG] pgmap v1999: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.8 MiB/s wr, 5 op/s 2024-09-13T00:21:01.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:21:01 smithi181 bash[67587]: audit 2024-09-13T00:21:00.390380+0000 mon.a (mon.0) 2279 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:21:02.433 INFO:tasks.workunit.client.0.smithi136.stdout: 1142: throughput=5MB/sec pending data=0 2024-09-13T00:21:02.433 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-WgJSj2sea3TrUuq off=12004985 len=762013 2024-09-13T00:21:02.435 INFO:tasks.workunit.client.0.smithi136.stdout:op 5977 completed, throughput=5MB/sec 2024-09-13T00:21:02.435 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-KarAhlfijaCiZYt off=493112 len=1653219 2024-09-13T00:21:02.450 INFO:tasks.workunit.client.0.smithi136.stdout:op 5978 completed, throughput=5MB/sec 2024-09-13T00:21:02.450 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-PwXedJPU3Wwh4Cb off=14654053 len=1442201 2024-09-13T00:21:02.454 INFO:tasks.workunit.client.0.smithi136.stdout:op 5979 completed, throughput=5MB/sec 2024-09-13T00:21:02.454 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-EWQMIz1roitZJTh off=3150521 len=1279338 2024-09-13T00:21:02.465 INFO:tasks.workunit.client.0.smithi136.stdout:op 5980 completed, throughput=5MB/sec 2024-09-13T00:21:02.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:21:02 smithi181 bash[95505]: ts=2024-09-13T00:21:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:21:03.465 INFO:tasks.workunit.client.0.smithi136.stdout: 1143: throughput=5MB/sec pending data=0 2024-09-13T00:21:03.466 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-zINzekcscjGDxSX off=11525174 len=1605616 2024-09-13T00:21:03.480 INFO:tasks.workunit.client.0.smithi136.stdout:op 5981 completed, throughput=5MB/sec 2024-09-13T00:21:03.480 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-R5ZBV8VXWqyamx_ off=1306570 len=1871858 2024-09-13T00:21:03.493 INFO:tasks.workunit.client.0.smithi136.stdout:op 5982 completed, throughput=5MB/sec 2024-09-13T00:21:03.493 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-xCjmvmK8LH3ph_K off=3420797 len=641696 2024-09-13T00:21:03.500 INFO:tasks.workunit.client.0.smithi136.stdout:op 5983 completed, throughput=5MB/sec 2024-09-13T00:21:03.500 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-tFV2qOTVw1EQaNm off=971488 len=1335662 2024-09-13T00:21:03.503 INFO:tasks.workunit.client.0.smithi136.stdout:op 5984 completed, throughput=5MB/sec 2024-09-13T00:21:03.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:21:03 smithi136 bash[75164]: cluster 2024-09-13T00:21:01.731232+0000 mgr.y (mgr.24661) 1317 : cluster [DBG] pgmap v2000: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:21:03.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:21:03 smithi136 bash[76371]: cluster 2024-09-13T00:21:01.731232+0000 mgr.y (mgr.24661) 1317 : cluster [DBG] pgmap v2000: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:21:03.600 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:21:03 smithi181 bash[67587]: cluster 2024-09-13T00:21:01.731232+0000 mgr.y (mgr.24661) 1317 : cluster [DBG] pgmap v2000: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.9 MiB/s wr, 5 op/s 2024-09-13T00:21:04.504 INFO:tasks.workunit.client.0.smithi136.stdout: 1144: throughput=5MB/sec pending data=0 2024-09-13T00:21:04.504 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-zsK9NRO3ccw6u7i off=12303960 len=1732779 2024-09-13T00:21:04.508 INFO:tasks.workunit.client.0.smithi136.stdout:op 5985 completed, throughput=5MB/sec 2024-09-13T00:21:04.509 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-UD7f4pERxpaC5_m off=20273117 len=1016949 2024-09-13T00:21:04.518 INFO:tasks.workunit.client.0.smithi136.stdout:op 5986 completed, throughput=5MB/sec 2024-09-13T00:21:04.518 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-rBGCoxQ0qVW2L9h off=5568638 len=2089170 2024-09-13T00:21:04.523 INFO:tasks.workunit.client.0.smithi136.stdout:op 5987 completed, throughput=5MB/sec 2024-09-13T00:21:04.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:21:04 smithi181 bash[67587]: cluster 2024-09-13T00:21:03.732190+0000 mgr.y (mgr.24661) 1318 : cluster [DBG] pgmap v2001: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 1.8 MiB/s rd, 2.8 MiB/s wr, 4 op/s 2024-09-13T00:21:04.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:21:04 smithi136 bash[75164]: cluster 2024-09-13T00:21:03.732190+0000 mgr.y (mgr.24661) 1318 : cluster [DBG] pgmap v2001: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 1.8 MiB/s rd, 2.8 MiB/s wr, 4 op/s 2024-09-13T00:21:04.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:21:04 smithi136 bash[76371]: cluster 2024-09-13T00:21:03.732190+0000 mgr.y (mgr.24661) 1318 : cluster [DBG] pgmap v2001: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 1.8 MiB/s rd, 2.8 MiB/s wr, 4 op/s 2024-09-13T00:21:05.524 INFO:tasks.workunit.client.0.smithi136.stdout: 1145: throughput=5MB/sec pending data=0 2024-09-13T00:21:05.524 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-sqgOuercJZpYvlM off=4812411 len=1314804 2024-09-13T00:21:05.530 INFO:tasks.workunit.client.0.smithi136.stdout:op 5988 completed, throughput=5MB/sec 2024-09-13T00:21:05.530 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-1xR1YJSTfpqu5kU off=5425359 len=580528 2024-09-13T00:21:05.531 INFO:tasks.workunit.client.0.smithi136.stdout:op 5989 completed, throughput=5MB/sec 2024-09-13T00:21:05.531 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-kud354bJdgfLq55 off=14792750 len=1728075 2024-09-13T00:21:05.537 INFO:tasks.workunit.client.0.smithi136.stdout:op 5990 completed, throughput=5MB/sec 2024-09-13T00:21:05.537 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-zu8KTTjyAmdOpqd off=9160020 len=21941 2024-09-13T00:21:05.539 INFO:tasks.workunit.client.0.smithi136.stdout:op 5991 completed, throughput=5MB/sec 2024-09-13T00:21:05.539 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-GcKD9RSFVFYdhn9 off=0 len=1108097 2024-09-13T00:21:05.542 INFO:tasks.workunit.client.0.smithi136.stdout:op 5992 completed, throughput=5MB/sec 2024-09-13T00:21:05.542 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ciEUGyen2plOAFQ off=1813893 len=1428136 2024-09-13T00:21:05.549 INFO:tasks.workunit.client.0.smithi136.stdout:op 5993 completed, throughput=5MB/sec 2024-09-13T00:21:06.549 INFO:tasks.workunit.client.0.smithi136.stdout: 1146: throughput=5MB/sec pending data=0 2024-09-13T00:21:06.549 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2HzNZkiBSnpfdkm off=4572885 len=1265734 2024-09-13T00:21:06.553 INFO:tasks.workunit.client.0.smithi136.stdout:op 5994 completed, throughput=5MB/sec 2024-09-13T00:21:06.553 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-yBFPNsy5Kfpvxdh off=7832248 len=259621 2024-09-13T00:21:06.556 INFO:tasks.workunit.client.0.smithi136.stdout:op 5995 completed, throughput=5MB/sec 2024-09-13T00:21:06.556 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-hO4LEulQ5CZnlTS off=473861 len=1418655 2024-09-13T00:21:06.564 INFO:tasks.workunit.client.0.smithi136.stdout:op 5996 completed, throughput=5MB/sec 2024-09-13T00:21:06.565 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-KUSEdjWWCOpctm9 off=6458785 len=1265339 2024-09-13T00:21:06.566 INFO:tasks.workunit.client.0.smithi136.stdout:op 5997 completed, throughput=5MB/sec 2024-09-13T00:21:07.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:21:06 smithi136 bash[76371]: cluster 2024-09-13T00:21:05.732745+0000 mgr.y (mgr.24661) 1319 : cluster [DBG] pgmap v2002: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.8 MiB/s wr, 4 op/s 2024-09-13T00:21:07.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:21:06 smithi136 bash[76371]: audit 2024-09-13T00:21:05.858311+0000 mon.a (mon.0) 2280 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:21:07.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:21:06 smithi136 bash[76371]: audit 2024-09-13T00:21:05.864569+0000 mon.a (mon.0) 2281 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:21:07.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:21:06 smithi136 bash[76371]: audit 2024-09-13T00:21:06.125927+0000 mon.a (mon.0) 2282 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:21:07.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:21:06 smithi136 bash[76371]: audit 2024-09-13T00:21:06.133279+0000 mon.a (mon.0) 2283 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:21:07.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:21:06 smithi136 bash[76371]: audit 2024-09-13T00:21:06.836381+0000 mon.a (mon.0) 2284 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:21:07.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:21:06 smithi136 bash[76371]: audit 2024-09-13T00:21:06.846307+0000 mon.a (mon.0) 2285 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:21:07.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:21:06 smithi136 bash[75164]: cluster 2024-09-13T00:21:05.732745+0000 mgr.y (mgr.24661) 1319 : cluster [DBG] pgmap v2002: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.8 MiB/s wr, 4 op/s 2024-09-13T00:21:07.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:21:06 smithi136 bash[75164]: audit 2024-09-13T00:21:05.858311+0000 mon.a (mon.0) 2280 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:21:07.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:21:06 smithi136 bash[75164]: audit 2024-09-13T00:21:05.864569+0000 mon.a (mon.0) 2281 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:21:07.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:21:06 smithi136 bash[75164]: audit 2024-09-13T00:21:06.125927+0000 mon.a (mon.0) 2282 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:21:07.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:21:06 smithi136 bash[75164]: audit 2024-09-13T00:21:06.133279+0000 mon.a (mon.0) 2283 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:21:07.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:21:06 smithi136 bash[75164]: audit 2024-09-13T00:21:06.836381+0000 mon.a (mon.0) 2284 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:21:07.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:21:06 smithi136 bash[75164]: audit 2024-09-13T00:21:06.846307+0000 mon.a (mon.0) 2285 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:21:07.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:21:06 smithi181 bash[67587]: cluster 2024-09-13T00:21:05.732745+0000 mgr.y (mgr.24661) 1319 : cluster [DBG] pgmap v2002: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.8 MiB/s wr, 4 op/s 2024-09-13T00:21:07.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:21:06 smithi181 bash[67587]: audit 2024-09-13T00:21:05.858311+0000 mon.a (mon.0) 2280 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:21:07.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:21:06 smithi181 bash[67587]: audit 2024-09-13T00:21:05.864569+0000 mon.a (mon.0) 2281 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:21:07.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:21:06 smithi181 bash[67587]: audit 2024-09-13T00:21:06.125927+0000 mon.a (mon.0) 2282 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:21:07.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:21:06 smithi181 bash[67587]: audit 2024-09-13T00:21:06.133279+0000 mon.a (mon.0) 2283 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:21:07.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:21:06 smithi181 bash[67587]: audit 2024-09-13T00:21:06.836381+0000 mon.a (mon.0) 2284 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:21:07.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:21:06 smithi181 bash[67587]: audit 2024-09-13T00:21:06.846307+0000 mon.a (mon.0) 2285 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:21:07.567 INFO:tasks.workunit.client.0.smithi136.stdout: 1147: throughput=5MB/sec pending data=0 2024-09-13T00:21:07.567 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-jEKcx5OguoFRhUg off=17517243 len=905024 2024-09-13T00:21:07.576 INFO:tasks.workunit.client.0.smithi136.stdout:op 5998 completed, throughput=5MB/sec 2024-09-13T00:21:07.576 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-vmGt60Ngl9kwxvF off=5221070 len=606228 2024-09-13T00:21:07.582 INFO:tasks.workunit.client.0.smithi136.stdout:op 5999 completed, throughput=5MB/sec 2024-09-13T00:21:07.583 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ftuj4kYR8cqQoz9 off=0 len=1092114 2024-09-13T00:21:07.591 INFO:tasks.workunit.client.0.smithi136.stdout:op 6000 completed, throughput=5MB/sec 2024-09-13T00:21:07.592 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Vl8DALFrtm2HI3k off=10746312 len=186997 2024-09-13T00:21:07.597 INFO:tasks.workunit.client.0.smithi136.stdout:op 6001 completed, throughput=5MB/sec 2024-09-13T00:21:07.598 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-P0-xQkFNmHCbsQ2 off=6059159 len=469785 2024-09-13T00:21:07.603 INFO:tasks.workunit.client.0.smithi136.stdout:op 6002 completed, throughput=5MB/sec 2024-09-13T00:21:07.603 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-DcRkDtTMtbAH4zm off=692421 len=842989 2024-09-13T00:21:07.605 INFO:tasks.workunit.client.0.smithi136.stdout:op 6003 completed, throughput=5MB/sec 2024-09-13T00:21:07.605 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-vp4gahfPCvc6IPF off=2611161 len=871741 2024-09-13T00:21:07.607 INFO:tasks.workunit.client.0.smithi136.stdout:op 6004 completed, throughput=5MB/sec 2024-09-13T00:21:07.607 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-lL7yxzTOOcFl149 off=8198557 len=15801 2024-09-13T00:21:07.610 INFO:tasks.workunit.client.0.smithi136.stdout:op 6005 completed, throughput=5MB/sec 2024-09-13T00:21:07.610 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-L0eYFvlfudLLe7l off=6121029 len=159804 2024-09-13T00:21:07.612 INFO:tasks.workunit.client.0.smithi136.stdout:op 6006 completed, throughput=5MB/sec 2024-09-13T00:21:07.612 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-lnZsvaATKfiOBt3 off=1077281 len=1199526 2024-09-13T00:21:07.620 INFO:tasks.workunit.client.0.smithi136.stdout:op 6007 completed, throughput=5MB/sec 2024-09-13T00:21:08.620 INFO:tasks.workunit.client.0.smithi136.stdout: 1148: throughput=5MB/sec pending data=0 2024-09-13T00:21:08.620 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ER8miHtzEGpDsJV off=5775101 len=953887 2024-09-13T00:21:08.624 INFO:tasks.workunit.client.0.smithi136.stdout:op 6008 completed, throughput=5MB/sec 2024-09-13T00:21:08.624 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ixQPjl3QRNzRZLC off=6862511 len=1885978 2024-09-13T00:21:08.636 INFO:tasks.workunit.client.0.smithi136.stdout:op 6009 completed, throughput=5MB/sec 2024-09-13T00:21:08.637 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-MFU7NTiao27XT8q off=3523519 len=1120834 2024-09-13T00:21:08.640 INFO:tasks.workunit.client.0.smithi136.stdout:op 6010 completed, throughput=5MB/sec 2024-09-13T00:21:08.640 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Sw04MR8PtecHuOS off=4988802 len=574833 2024-09-13T00:21:08.646 INFO:tasks.workunit.client.0.smithi136.stdout:op 6011 completed, throughput=5MB/sec 2024-09-13T00:21:08.646 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-LD-9a6W_8j0-8WB off=5658776 len=1118086 2024-09-13T00:21:08.654 INFO:tasks.workunit.client.0.smithi136.stdout:op 6012 completed, throughput=5MB/sec 2024-09-13T00:21:09.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:21:08 smithi136 bash[75164]: cluster 2024-09-13T00:21:07.733737+0000 mgr.y (mgr.24661) 1320 : cluster [DBG] pgmap v2003: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.5 MiB/s rd, 3.3 MiB/s wr, 5 op/s 2024-09-13T00:21:09.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:21:08 smithi136 bash[75164]: audit 2024-09-13T00:21:08.804947+0000 mon.a (mon.0) 2286 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:21:09.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:21:08 smithi136 bash[76371]: cluster 2024-09-13T00:21:07.733737+0000 mgr.y (mgr.24661) 1320 : cluster [DBG] pgmap v2003: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.5 MiB/s rd, 3.3 MiB/s wr, 5 op/s 2024-09-13T00:21:09.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:21:08 smithi136 bash[76371]: audit 2024-09-13T00:21:08.804947+0000 mon.a (mon.0) 2286 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:21:09.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:21:08 smithi181 bash[67587]: cluster 2024-09-13T00:21:07.733737+0000 mgr.y (mgr.24661) 1320 : cluster [DBG] pgmap v2003: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.5 MiB/s rd, 3.3 MiB/s wr, 5 op/s 2024-09-13T00:21:09.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:21:08 smithi181 bash[67587]: audit 2024-09-13T00:21:08.804947+0000 mon.a (mon.0) 2286 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:21:09.655 INFO:tasks.workunit.client.0.smithi136.stdout: 1149: throughput=5MB/sec pending data=0 2024-09-13T00:21:09.655 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-jt8_4VQRAiu83yw off=1824567 len=166746 2024-09-13T00:21:09.660 INFO:tasks.workunit.client.0.smithi136.stdout:op 6013 completed, throughput=5MB/sec 2024-09-13T00:21:09.660 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-SwgtR06gSS1EcFd off=499411 len=339052 2024-09-13T00:21:09.665 INFO:tasks.workunit.client.0.smithi136.stdout:op 6014 completed, throughput=5MB/sec 2024-09-13T00:21:09.665 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-LBZ4FnNkOvkpSVx off=3128502 len=1733244 2024-09-13T00:21:09.677 INFO:tasks.workunit.client.0.smithi136.stdout:op 6015 completed, throughput=5MB/sec 2024-09-13T00:21:09.677 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-mqxxFF_5rHL4ZbU off=308328 len=1447333 2024-09-13T00:21:09.687 INFO:tasks.workunit.client.0.smithi136.stdout:op 6016 completed, throughput=5MB/sec 2024-09-13T00:21:09.687 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-_3Qh6O__5GvdzYM off=5946437 len=473597 2024-09-13T00:21:09.693 INFO:tasks.workunit.client.0.smithi136.stdout:op 6017 completed, throughput=5MB/sec 2024-09-13T00:21:09.693 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-z4S_VVGd9vjXKDG off=3742031 len=295797 2024-09-13T00:21:09.694 INFO:tasks.workunit.client.0.smithi136.stdout:op 6018 completed, throughput=5MB/sec 2024-09-13T00:21:09.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:21:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:21:09] "GET /metrics HTTP/1.1" 200 34142 "" "Prometheus/2.43.0" 2024-09-13T00:21:10.695 INFO:tasks.workunit.client.0.smithi136.stdout: 1150: throughput=5MB/sec pending data=0 2024-09-13T00:21:10.695 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-AJoNgepcatDaLd6 off=5789264 len=490185 2024-09-13T00:21:10.696 INFO:tasks.workunit.client.0.smithi136.stdout:op 6019 completed, throughput=5MB/sec 2024-09-13T00:21:10.696 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2eZwZHsVMb-Hnoi off=7973084 len=2025511 2024-09-13T00:21:10.700 INFO:tasks.workunit.client.0.smithi136.stdout:op 6020 completed, throughput=5MB/sec 2024-09-13T00:21:10.700 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-lNg_ZF0qp3moeTG off=4170796 len=87830 2024-09-13T00:21:10.701 INFO:tasks.workunit.client.0.smithi136.stdout:op 6021 completed, throughput=5MB/sec 2024-09-13T00:21:10.701 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-BjYn4t1QphbSro5 off=371929 len=978792 2024-09-13T00:21:10.703 INFO:tasks.workunit.client.0.smithi136.stdout:op 6022 completed, throughput=5MB/sec 2024-09-13T00:21:10.704 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-JDDYO9doZeY6SnM off=2892120 len=1549705 2024-09-13T00:21:10.715 INFO:tasks.workunit.client.0.smithi136.stdout:op 6023 completed, throughput=5MB/sec 2024-09-13T00:21:10.716 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-gFI_EOy680od_jR off=11635526 len=1187748 2024-09-13T00:21:10.724 INFO:tasks.workunit.client.0.smithi136.stdout:op 6024 completed, throughput=5MB/sec 2024-09-13T00:21:11.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:21:10 smithi136 bash[75164]: cluster 2024-09-13T00:21:09.734554+0000 mgr.y (mgr.24661) 1321 : cluster [DBG] pgmap v2004: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.2 MiB/s wr, 4 op/s 2024-09-13T00:21:11.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:21:10 smithi136 bash[76371]: cluster 2024-09-13T00:21:09.734554+0000 mgr.y (mgr.24661) 1321 : cluster [DBG] pgmap v2004: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.2 MiB/s wr, 4 op/s 2024-09-13T00:21:11.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:21:10 smithi181 bash[67587]: cluster 2024-09-13T00:21:09.734554+0000 mgr.y (mgr.24661) 1321 : cluster [DBG] pgmap v2004: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.2 MiB/s wr, 4 op/s 2024-09-13T00:21:11.725 INFO:tasks.workunit.client.0.smithi136.stdout: 1151: throughput=5MB/sec pending data=0 2024-09-13T00:21:11.725 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-CazU11TOFdlAlQW off=1294336 len=827942 2024-09-13T00:21:11.728 INFO:tasks.workunit.client.0.smithi136.stdout:op 6025 completed, throughput=5MB/sec 2024-09-13T00:21:11.728 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-1M2loYJA79HhBF2 off=1037348 len=1485225 2024-09-13T00:21:11.731 INFO:tasks.workunit.client.0.smithi136.stdout:op 6026 completed, throughput=5MB/sec 2024-09-13T00:21:11.731 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj--hl_ozik4qU5a6m off=5587435 len=1274317 2024-09-13T00:21:11.735 INFO:tasks.workunit.client.0.smithi136.stdout:op 6027 completed, throughput=5MB/sec 2024-09-13T00:21:11.735 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-WRvWj9j3oR-If7D off=4292828 len=2095517 2024-09-13T00:21:11.741 INFO:tasks.workunit.client.0.smithi136.stdout:op 6028 completed, throughput=5MB/sec 2024-09-13T00:21:12.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:21:12 smithi181 bash[95505]: ts=2024-09-13T00:21:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:21:12.741 INFO:tasks.workunit.client.0.smithi136.stdout: 1152: throughput=5MB/sec pending data=0 2024-09-13T00:21:12.742 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-iDhWs6_eveYNVck off=13636105 len=288378 2024-09-13T00:21:12.747 INFO:tasks.workunit.client.0.smithi136.stdout:op 6029 completed, throughput=5MB/sec 2024-09-13T00:21:12.747 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2AnqZi-d9oeEUvh off=100724 len=427521 2024-09-13T00:21:12.748 INFO:tasks.workunit.client.0.smithi136.stdout:op 6030 completed, throughput=5MB/sec 2024-09-13T00:21:12.748 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-yXV7K2EccwrHe_Q off=9148958 len=631332 2024-09-13T00:21:12.751 INFO:tasks.workunit.client.0.smithi136.stdout:op 6031 completed, throughput=5MB/sec 2024-09-13T00:21:12.751 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-WJb5_FMhH_Nv99e off=989534 len=375142 2024-09-13T00:21:12.756 INFO:tasks.workunit.client.0.smithi136.stdout:op 6032 completed, throughput=5MB/sec 2024-09-13T00:21:12.757 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-5ymzdokwgCPaNHK off=4033627 len=1648876 2024-09-13T00:21:12.768 INFO:tasks.workunit.client.0.smithi136.stdout:op 6033 completed, throughput=5MB/sec 2024-09-13T00:21:12.768 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-PuU9J8TMStARPKz off=53321 len=384097 2024-09-13T00:21:12.775 INFO:tasks.workunit.client.0.smithi136.stdout:op 6034 completed, throughput=5MB/sec 2024-09-13T00:21:12.775 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-ho65qP8bgcBA_5J off=1628297 len=1592344 2024-09-13T00:21:12.789 INFO:tasks.workunit.client.0.smithi136.stdout:op 6035 completed, throughput=5MB/sec 2024-09-13T00:21:13.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:21:12 smithi136 bash[75164]: cluster 2024-09-13T00:21:11.735612+0000 mgr.y (mgr.24661) 1322 : cluster [DBG] pgmap v2005: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.2 MiB/s rd, 3.2 MiB/s wr, 5 op/s 2024-09-13T00:21:13.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:21:12 smithi136 bash[76371]: cluster 2024-09-13T00:21:11.735612+0000 mgr.y (mgr.24661) 1322 : cluster [DBG] pgmap v2005: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.2 MiB/s rd, 3.2 MiB/s wr, 5 op/s 2024-09-13T00:21:13.086 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:21:12 smithi181 bash[67587]: cluster 2024-09-13T00:21:11.735612+0000 mgr.y (mgr.24661) 1322 : cluster [DBG] pgmap v2005: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.2 MiB/s rd, 3.2 MiB/s wr, 5 op/s 2024-09-13T00:21:13.790 INFO:tasks.workunit.client.0.smithi136.stdout: 1153: throughput=5MB/sec pending data=0 2024-09-13T00:21:13.790 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-DFrHJq3H4drUnkj off=13024829 len=312795 2024-09-13T00:21:13.796 INFO:tasks.workunit.client.0.smithi136.stdout:op 6036 completed, throughput=5MB/sec 2024-09-13T00:21:13.796 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-2UFCoxD3y-d3Reu off=4666877 len=1481931 2024-09-13T00:21:13.801 INFO:tasks.workunit.client.0.smithi136.stdout:op 6037 completed, throughput=5MB/sec 2024-09-13T00:21:13.801 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-DU6DxfhoBFsUJTa off=3583127 len=355350 2024-09-13T00:21:13.803 INFO:tasks.workunit.client.0.smithi136.stdout:op 6038 completed, throughput=5MB/sec 2024-09-13T00:21:13.804 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-P6q8Uerl1NA63Ec off=564637 len=1744954 2024-09-13T00:21:13.816 INFO:tasks.workunit.client.0.smithi136.stdout:op 6039 completed, throughput=5MB/sec 2024-09-13T00:21:13.816 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-kJjNYrOouSxagp- off=5925862 len=1896729 2024-09-13T00:21:13.821 INFO:tasks.workunit.client.0.smithi136.stdout:op 6040 completed, throughput=5MB/sec 2024-09-13T00:21:14.821 INFO:tasks.workunit.client.0.smithi136.stdout: 1154: throughput=5MB/sec pending data=0 2024-09-13T00:21:14.821 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-1bMTGJB5lju03f9 off=2055129 len=379450 2024-09-13T00:21:14.823 INFO:tasks.workunit.client.0.smithi136.stdout:op 6041 completed, throughput=5MB/sec 2024-09-13T00:21:14.823 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-_nRdKUtWEzd_BeJ off=7951754 len=1972716 2024-09-13T00:21:14.826 INFO:tasks.workunit.client.0.smithi136.stdout:op 6042 completed, throughput=5MB/sec 2024-09-13T00:21:14.826 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-0NoZaTPXTwxzjRk off=92027 len=1720474 2024-09-13T00:21:14.830 INFO:tasks.workunit.client.0.smithi136.stdout:op 6043 completed, throughput=5MB/sec 2024-09-13T00:21:15.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:21:14 smithi136 bash[75164]: cluster 2024-09-13T00:21:13.736490+0000 mgr.y (mgr.24661) 1323 : cluster [DBG] pgmap v2006: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 1.9 MiB/s rd, 2.7 MiB/s wr, 4 op/s 2024-09-13T00:21:15.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:21:14 smithi136 bash[76371]: cluster 2024-09-13T00:21:13.736490+0000 mgr.y (mgr.24661) 1323 : cluster [DBG] pgmap v2006: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 1.9 MiB/s rd, 2.7 MiB/s wr, 4 op/s 2024-09-13T00:21:15.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:21:14 smithi181 bash[67587]: cluster 2024-09-13T00:21:13.736490+0000 mgr.y (mgr.24661) 1323 : cluster [DBG] pgmap v2006: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 1.9 MiB/s rd, 2.7 MiB/s wr, 4 op/s 2024-09-13T00:21:15.831 INFO:tasks.workunit.client.0.smithi136.stdout: 1155: throughput=5MB/sec pending data=0 2024-09-13T00:21:15.831 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-8EZeyM0JrGV-5VW off=11733316 len=1877879 2024-09-13T00:21:15.835 INFO:tasks.workunit.client.0.smithi136.stdout:op 6044 completed, throughput=5MB/sec 2024-09-13T00:21:15.835 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-mDzgiu-dw-xyuZ_ off=3042527 len=981747 2024-09-13T00:21:15.844 INFO:tasks.workunit.client.0.smithi136.stdout:op 6045 completed, throughput=5MB/sec 2024-09-13T00:21:15.845 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-bgmXX7LBtlYOYeI off=908998 len=885372 2024-09-13T00:21:15.846 INFO:tasks.workunit.client.0.smithi136.stdout:op 6046 completed, throughput=5MB/sec 2024-09-13T00:21:15.846 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-zeZVfPjnO7CyBSF off=2926754 len=1715388 2024-09-13T00:21:15.857 INFO:tasks.workunit.client.0.smithi136.stdout:op 6047 completed, throughput=5MB/sec 2024-09-13T00:21:16.857 INFO:tasks.workunit.client.0.smithi136.stdout: 1156: throughput=5MB/sec pending data=0 2024-09-13T00:21:16.858 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-YQiH2V-WZiud1vB off=704992 len=929648 2024-09-13T00:21:16.860 INFO:tasks.workunit.client.0.smithi136.stdout:op 6048 completed, throughput=5MB/sec 2024-09-13T00:21:16.860 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-_8eUh7ySXW4RyBC off=2777701 len=514376 2024-09-13T00:21:16.863 INFO:tasks.workunit.client.0.smithi136.stdout:op 6049 completed, throughput=5MB/sec 2024-09-13T00:21:16.863 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-sVNFbrFsySB3-39 off=17635598 len=1914974 2024-09-13T00:21:16.868 INFO:tasks.workunit.client.0.smithi136.stdout:op 6050 completed, throughput=5MB/sec 2024-09-13T00:21:16.868 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-f1RODT3Ai4YgB1- off=1797161 len=1472423 2024-09-13T00:21:16.873 INFO:tasks.workunit.client.0.smithi136.stdout:op 6051 completed, throughput=5MB/sec 2024-09-13T00:21:16.873 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-eLAz410ggJeB8ag off=4013017 len=1141815 2024-09-13T00:21:16.883 INFO:tasks.workunit.client.0.smithi136.stdout:op 6052 completed, throughput=5MB/sec 2024-09-13T00:21:17.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:21:17 smithi136 bash[75164]: cluster 2024-09-13T00:21:15.737632+0000 mgr.y (mgr.24661) 1324 : cluster [DBG] pgmap v2007: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:21:17.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:21:17 smithi136 bash[75164]: audit 2024-09-13T00:21:16.235884+0000 mon.a (mon.0) 2287 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:21:17.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:21:17 smithi136 bash[75164]: audit 2024-09-13T00:21:16.244489+0000 mon.a (mon.0) 2288 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:21:17.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:21:17 smithi136 bash[75164]: audit 2024-09-13T00:21:16.303138+0000 mon.a (mon.0) 2289 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:21:17.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:21:17 smithi136 bash[75164]: audit 2024-09-13T00:21:16.313126+0000 mon.a (mon.0) 2290 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:21:17.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:21:17 smithi136 bash[75164]: audit 2024-09-13T00:21:16.315257+0000 mon.a (mon.0) 2291 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:21:17.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:21:17 smithi136 bash[75164]: audit 2024-09-13T00:21:16.316968+0000 mon.a (mon.0) 2292 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:21:17.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:21:17 smithi136 bash[75164]: audit 2024-09-13T00:21:16.329661+0000 mon.a (mon.0) 2293 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:21:17.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:21:17 smithi136 bash[76371]: cluster 2024-09-13T00:21:15.737632+0000 mgr.y (mgr.24661) 1324 : cluster [DBG] pgmap v2007: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:21:17.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:21:17 smithi136 bash[76371]: audit 2024-09-13T00:21:16.235884+0000 mon.a (mon.0) 2287 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:21:17.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:21:17 smithi136 bash[76371]: audit 2024-09-13T00:21:16.244489+0000 mon.a (mon.0) 2288 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:21:17.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:21:17 smithi136 bash[76371]: audit 2024-09-13T00:21:16.303138+0000 mon.a (mon.0) 2289 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:21:17.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:21:17 smithi136 bash[76371]: audit 2024-09-13T00:21:16.313126+0000 mon.a (mon.0) 2290 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:21:17.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:21:17 smithi136 bash[76371]: audit 2024-09-13T00:21:16.315257+0000 mon.a (mon.0) 2291 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:21:17.512 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:21:17 smithi136 bash[76371]: audit 2024-09-13T00:21:16.316968+0000 mon.a (mon.0) 2292 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:21:17.512 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:21:17 smithi136 bash[76371]: audit 2024-09-13T00:21:16.329661+0000 mon.a (mon.0) 2293 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:21:17.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:21:17 smithi181 bash[67587]: cluster 2024-09-13T00:21:15.737632+0000 mgr.y (mgr.24661) 1324 : cluster [DBG] pgmap v2007: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.6 MiB/s wr, 4 op/s 2024-09-13T00:21:17.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:21:17 smithi181 bash[67587]: audit 2024-09-13T00:21:16.235884+0000 mon.a (mon.0) 2287 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:21:17.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:21:17 smithi181 bash[67587]: audit 2024-09-13T00:21:16.244489+0000 mon.a (mon.0) 2288 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:21:17.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:21:17 smithi181 bash[67587]: audit 2024-09-13T00:21:16.303138+0000 mon.a (mon.0) 2289 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:21:17.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:21:17 smithi181 bash[67587]: audit 2024-09-13T00:21:16.313126+0000 mon.a (mon.0) 2290 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:21:17.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:21:17 smithi181 bash[67587]: audit 2024-09-13T00:21:16.315257+0000 mon.a (mon.0) 2291 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:21:17.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:21:17 smithi181 bash[67587]: audit 2024-09-13T00:21:16.316968+0000 mon.a (mon.0) 2292 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:21:17.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:21:17 smithi181 bash[67587]: audit 2024-09-13T00:21:16.329661+0000 mon.a (mon.0) 2293 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:21:17.883 INFO:tasks.workunit.client.0.smithi136.stdout: 1157: throughput=5MB/sec pending data=0 2024-09-13T00:21:17.883 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-vKdgXZ9oC4dszh4 off=5710820 len=888422 2024-09-13T00:21:17.887 INFO:tasks.workunit.client.0.smithi136.stdout:op 6053 completed, throughput=5MB/sec 2024-09-13T00:21:17.887 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-OrriI7dJzeV_vya off=19121938 len=1230584 2024-09-13T00:21:17.890 INFO:tasks.workunit.client.0.smithi136.stdout:op 6054 completed, throughput=5MB/sec 2024-09-13T00:21:17.890 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-KqDYQrgOZPQPFse off=6683582 len=945154 2024-09-13T00:21:17.900 INFO:tasks.workunit.client.0.smithi136.stdout:op 6055 completed, throughput=5MB/sec 2024-09-13T00:21:17.900 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-a8xDCAeqnfq5ArX off=1127628 len=167437 2024-09-13T00:21:17.905 INFO:tasks.workunit.client.0.smithi136.stdout:op 6056 completed, throughput=5MB/sec 2024-09-13T00:21:17.905 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-wU20SCeUuXE1JU3 off=76452 len=1064014 2024-09-13T00:21:17.909 INFO:tasks.workunit.client.0.smithi136.stdout:op 6057 completed, throughput=5MB/sec 2024-09-13T00:21:17.909 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-w3tyqqm7X9plA9g off=8320320 len=1084805 2024-09-13T00:21:17.918 INFO:tasks.workunit.client.0.smithi136.stdout:op 6058 completed, throughput=5MB/sec 2024-09-13T00:21:18.919 INFO:tasks.workunit.client.0.smithi136.stdout: 1158: throughput=5MB/sec pending data=0 2024-09-13T00:21:18.919 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-_aSrh-ZqxONvT9_ off=633178 len=1213751 2024-09-13T00:21:18.922 INFO:tasks.workunit.client.0.smithi136.stdout:op 6059 completed, throughput=5MB/sec 2024-09-13T00:21:18.922 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-mZsYIBik8QwL8PQ off=6696284 len=109243 2024-09-13T00:21:18.923 INFO:tasks.workunit.client.0.smithi136.stdout:op 6060 completed, throughput=5MB/sec 2024-09-13T00:21:18.923 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-kj_s4t0fay3fEyX off=5681447 len=1876261 2024-09-13T00:21:18.936 INFO:tasks.workunit.client.0.smithi136.stdout:op 6061 completed, throughput=5MB/sec 2024-09-13T00:21:18.936 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-jcAhUadeUpqowIk off=9603997 len=187127 2024-09-13T00:21:18.937 INFO:tasks.workunit.client.0.smithi136.stdout:op 6062 completed, throughput=5MB/sec 2024-09-13T00:21:18.937 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-5HUYQyKvhTWceLQ off=285194 len=1063441 2024-09-13T00:21:18.939 INFO:tasks.workunit.client.0.smithi136.stdout:op 6063 completed, throughput=5MB/sec 2024-09-13T00:21:19.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:21:19 smithi136 bash[75164]: cluster 2024-09-13T00:21:17.738738+0000 mgr.y (mgr.24661) 1325 : cluster [DBG] pgmap v2008: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:21:19.509 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:21:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:21:19] "GET /metrics HTTP/1.1" 200 34142 "" "Prometheus/2.43.0" 2024-09-13T00:21:19.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:21:19 smithi136 bash[76371]: cluster 2024-09-13T00:21:17.738738+0000 mgr.y (mgr.24661) 1325 : cluster [DBG] pgmap v2008: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:21:19.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:21:19 smithi181 bash[67587]: cluster 2024-09-13T00:21:17.738738+0000 mgr.y (mgr.24661) 1325 : cluster [DBG] pgmap v2008: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.4 MiB/s wr, 4 op/s 2024-09-13T00:21:19.939 INFO:tasks.workunit.client.0.smithi136.stdout: 1159: throughput=5MB/sec pending data=0 2024-09-13T00:21:19.939 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-k4w-8XHoe14oLYV off=2825110 len=1171406 2024-09-13T00:21:19.944 INFO:tasks.workunit.client.0.smithi136.stdout:op 6064 completed, throughput=5MB/sec 2024-09-13T00:21:19.944 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-__5ULIRkS62mlSn off=3277803 len=276552 2024-09-13T00:21:19.949 INFO:tasks.workunit.client.0.smithi136.stdout:op 6065 completed, throughput=5MB/sec 2024-09-13T00:21:19.949 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-1bMTGJB5lju03f9 off=1114551 len=1498710 2024-09-13T00:21:19.953 INFO:tasks.workunit.client.0.smithi136.stdout:op 6066 completed, throughput=5MB/sec 2024-09-13T00:21:19.953 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-vWeQGPghgBNcStb off=14783918 len=1402280 2024-09-13T00:21:19.957 INFO:tasks.workunit.client.0.smithi136.stdout:op 6067 completed, throughput=5MB/sec 2024-09-13T00:21:19.957 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-yaXyUyrAo5CPd3X off=85023 len=714135 2024-09-13T00:21:19.959 INFO:tasks.workunit.client.0.smithi136.stdout:op 6068 completed, throughput=5MB/sec 2024-09-13T00:21:19.959 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-nElm29kUFhReOvz off=13996247 len=2055463 2024-09-13T00:21:19.968 INFO:tasks.workunit.client.0.smithi136.stdout:op 6069 completed, throughput=5MB/sec 2024-09-13T00:21:20.966 INFO:tasks.workunit.client.0.smithi136.stdout: 1160: throughput=5MB/sec pending data=0 2024-09-13T00:21:20.966 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-kOa3zKkubUWtKcB off=1234835 len=813339 2024-09-13T00:21:20.974 INFO:tasks.workunit.client.0.smithi136.stdout:op 6070 completed, throughput=5MB/sec 2024-09-13T00:21:20.974 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-gFI_EOy680od_jR off=7660165 len=1274691 2024-09-13T00:21:20.977 INFO:tasks.workunit.client.0.smithi136.stdout:op 6071 completed, throughput=5MB/sec 2024-09-13T00:21:20.977 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-Fid34LFAeff0vZY off=10370550 len=408156 2024-09-13T00:21:20.983 INFO:tasks.workunit.client.0.smithi136.stdout:op 6072 completed, throughput=5MB/sec 2024-09-13T00:21:20.983 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-phac-iJL0E2_s76 off=5091658 len=422806 2024-09-13T00:21:20.985 INFO:tasks.workunit.client.0.smithi136.stdout:op 6073 completed, throughput=5MB/sec 2024-09-13T00:21:20.985 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-ijzAGhM4i-HJesb off=1971461 len=620571 2024-09-13T00:21:20.988 INFO:tasks.workunit.client.0.smithi136.stdout:op 6074 completed, throughput=5MB/sec 2024-09-13T00:21:21.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:21:21 smithi136 bash[75164]: cluster 2024-09-13T00:21:19.739608+0000 mgr.y (mgr.24661) 1326 : cluster [DBG] pgmap v2009: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.7 MiB/s rd, 1.8 MiB/s wr, 4 op/s 2024-09-13T00:21:21.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:21:21 smithi136 bash[76371]: cluster 2024-09-13T00:21:19.739608+0000 mgr.y (mgr.24661) 1326 : cluster [DBG] pgmap v2009: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.7 MiB/s rd, 1.8 MiB/s wr, 4 op/s 2024-09-13T00:21:21.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:21:21 smithi181 bash[67587]: cluster 2024-09-13T00:21:19.739608+0000 mgr.y (mgr.24661) 1326 : cluster [DBG] pgmap v2009: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 2.7 MiB/s rd, 1.8 MiB/s wr, 4 op/s 2024-09-13T00:21:21.987 INFO:tasks.workunit.client.0.smithi136.stdout: 1161: throughput=5MB/sec pending data=0 2024-09-13T00:21:21.987 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-UFXQKn49P711msW off=17253201 len=644363 2024-09-13T00:21:21.989 INFO:tasks.workunit.client.0.smithi136.stdout:op 6075 completed, throughput=5MB/sec 2024-09-13T00:21:21.989 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-4DH0z-v_elGEzVY off=6905297 len=262777 2024-09-13T00:21:21.991 INFO:tasks.workunit.client.0.smithi136.stdout:op 6076 completed, throughput=5MB/sec 2024-09-13T00:21:21.991 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-83_prWDLahuoATJ off=21230771 len=142137 2024-09-13T00:21:21.996 INFO:tasks.workunit.client.0.smithi136.stdout:op 6077 completed, throughput=5MB/sec 2024-09-13T00:21:21.996 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-v6cLDT4FxTchIqH off=2330433 len=762971 2024-09-13T00:21:22.004 INFO:tasks.workunit.client.0.smithi136.stdout:op 6078 completed, throughput=5MB/sec 2024-09-13T00:21:22.004 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-1XMdLy3krAMvvCd off=2264888 len=2067854 2024-09-13T00:21:22.009 INFO:tasks.workunit.client.0.smithi136.stdout:op 6079 completed, throughput=5MB/sec 2024-09-13T00:21:22.009 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-KdBKgAVYSkhiQTk off=844805 len=1562388 2024-09-13T00:21:22.013 INFO:tasks.workunit.client.0.smithi136.stdout:op 6080 completed, throughput=5MB/sec 2024-09-13T00:21:22.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:21:22 smithi181 bash[95505]: ts=2024-09-13T00:21:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:21:23.013 INFO:tasks.workunit.client.0.smithi136.stdout: 1162: throughput=5MB/sec pending data=0 2024-09-13T00:21:23.013 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-azeqk6IGI2kFEFF off=4984798 len=1779641 2024-09-13T00:21:23.017 INFO:tasks.workunit.client.0.smithi136.stdout:op 6081 completed, throughput=5MB/sec 2024-09-13T00:21:23.017 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-DetdP7fxXwIL0H4 off=7743215 len=776633 2024-09-13T00:21:23.025 INFO:tasks.workunit.client.0.smithi136.stdout:op 6082 completed, throughput=5MB/sec 2024-09-13T00:21:23.025 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-VV_qd_BJj48Q91L off=14256179 len=1516792 2024-09-13T00:21:23.039 INFO:tasks.workunit.client.0.smithi136.stdout:op 6083 completed, throughput=5MB/sec 2024-09-13T00:21:23.039 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-fU_HLMdc_sin1qI off=16864772 len=94583 2024-09-13T00:21:23.045 INFO:tasks.workunit.client.0.smithi136.stdout:op 6084 completed, throughput=5MB/sec 2024-09-13T00:21:23.045 INFO:tasks.workunit.client.0.smithi136.stdout:READ : oid=obj-WBFjuw3R3RTobgK off=3983323 len=978200 2024-09-13T00:21:23.047 INFO:tasks.workunit.client.0.smithi136.stdout:op 6085 completed, throughput=5MB/sec 2024-09-13T00:21:23.047 INFO:tasks.workunit.client.0.smithi136.stdout:WRITE : oid=obj-P6yS3HWt60m32vU off=1905723 len=1132298 2024-09-13T00:21:23.058 INFO:tasks.workunit.client.0.smithi136.stdout:op 6086 completed, throughput=5MB/sec 2024-09-13T00:21:23.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:21:23 smithi136 bash[75164]: cluster 2024-09-13T00:21:21.740704+0000 mgr.y (mgr.24661) 1327 : cluster [DBG] pgmap v2010: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 3.4 MiB/s rd, 2.2 MiB/s wr, 5 op/s 2024-09-13T00:21:23.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:21:23 smithi136 bash[76371]: cluster 2024-09-13T00:21:21.740704+0000 mgr.y (mgr.24661) 1327 : cluster [DBG] pgmap v2010: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 3.4 MiB/s rd, 2.2 MiB/s wr, 5 op/s 2024-09-13T00:21:23.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:21:23 smithi181 bash[67587]: cluster 2024-09-13T00:21:21.740704+0000 mgr.y (mgr.24661) 1327 : cluster [DBG] pgmap v2010: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 3.4 MiB/s rd, 2.2 MiB/s wr, 5 op/s 2024-09-13T00:21:24.058 INFO:tasks.workunit.client.0.smithi136.stdout:waiting for all operations to complete 2024-09-13T00:21:24.058 INFO:tasks.workunit.client.0.smithi136.stdout:cleaning up objects 2024-09-13T00:21:24.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:21:24 smithi136 bash[76371]: audit 2024-09-13T00:21:23.805312+0000 mon.a (mon.0) 2294 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:21:24.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:21:24 smithi136 bash[75164]: audit 2024-09-13T00:21:23.805312+0000 mon.a (mon.0) 2294 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:21:24.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:21:24 smithi181 bash[67587]: audit 2024-09-13T00:21:23.805312+0000 mon.a (mon.0) 2294 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:21:25.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:21:25 smithi136 bash[76371]: cluster 2024-09-13T00:21:23.741589+0000 mgr.y (mgr.24661) 1328 : cluster [DBG] pgmap v2011: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.5 MiB/s wr, 4 op/s 2024-09-13T00:21:25.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:21:25 smithi136 bash[75164]: cluster 2024-09-13T00:21:23.741589+0000 mgr.y (mgr.24661) 1328 : cluster [DBG] pgmap v2011: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.5 MiB/s wr, 4 op/s 2024-09-13T00:21:25.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:21:25 smithi181 bash[67587]: cluster 2024-09-13T00:21:23.741589+0000 mgr.y (mgr.24661) 1328 : cluster [DBG] pgmap v2011: 105 pgs: 105 active+clean; 126 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.5 MiB/s wr, 4 op/s 2024-09-13T00:21:27.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:21:27 smithi136 bash[76371]: cluster 2024-09-13T00:21:25.742584+0000 mgr.y (mgr.24661) 1329 : cluster [DBG] pgmap v2012: 105 pgs: 105 active+clean; 125 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.3 MiB/s wr, 11 op/s 2024-09-13T00:21:27.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:21:27 smithi136 bash[75164]: cluster 2024-09-13T00:21:25.742584+0000 mgr.y (mgr.24661) 1329 : cluster [DBG] pgmap v2012: 105 pgs: 105 active+clean; 125 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.3 MiB/s wr, 11 op/s 2024-09-13T00:21:27.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:21:27 smithi181 bash[67587]: cluster 2024-09-13T00:21:25.742584+0000 mgr.y (mgr.24661) 1329 : cluster [DBG] pgmap v2012: 105 pgs: 105 active+clean; 125 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.3 MiB/s wr, 11 op/s 2024-09-13T00:21:29.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:21:29 smithi181 bash[67587]: cluster 2024-09-13T00:21:27.743689+0000 mgr.y (mgr.24661) 1330 : cluster [DBG] pgmap v2013: 105 pgs: 105 active+clean; 121 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.3 MiB/s wr, 39 op/s 2024-09-13T00:21:29.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:21:29 smithi136 bash[75164]: cluster 2024-09-13T00:21:27.743689+0000 mgr.y (mgr.24661) 1330 : cluster [DBG] pgmap v2013: 105 pgs: 105 active+clean; 121 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.3 MiB/s wr, 39 op/s 2024-09-13T00:21:29.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:21:29 smithi136 bash[76371]: cluster 2024-09-13T00:21:27.743689+0000 mgr.y (mgr.24661) 1330 : cluster [DBG] pgmap v2013: 105 pgs: 105 active+clean; 121 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.3 MiB/s wr, 39 op/s 2024-09-13T00:21:29.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:21:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:21:29] "GET /metrics HTTP/1.1" 200 34142 "" "Prometheus/2.43.0" 2024-09-13T00:21:30.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:21:30 smithi181 bash[67587]: cluster 2024-09-13T00:21:29.744618+0000 mgr.y (mgr.24661) 1331 : cluster [DBG] pgmap v2014: 105 pgs: 105 active+clean; 115 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.0 MiB/s wr, 75 op/s 2024-09-13T00:21:30.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:21:30 smithi136 bash[76371]: cluster 2024-09-13T00:21:29.744618+0000 mgr.y (mgr.24661) 1331 : cluster [DBG] pgmap v2014: 105 pgs: 105 active+clean; 115 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.0 MiB/s wr, 75 op/s 2024-09-13T00:21:30.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:21:30 smithi136 bash[75164]: cluster 2024-09-13T00:21:29.744618+0000 mgr.y (mgr.24661) 1331 : cluster [DBG] pgmap v2014: 105 pgs: 105 active+clean; 115 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.0 MiB/s wr, 75 op/s 2024-09-13T00:21:32.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:21:32 smithi181 bash[95505]: ts=2024-09-13T00:21:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:21:33.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:21:32 smithi181 bash[67587]: cluster 2024-09-13T00:21:31.745919+0000 mgr.y (mgr.24661) 1332 : cluster [DBG] pgmap v2015: 105 pgs: 105 active+clean; 104 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 1.9 MiB/s rd, 969 KiB/s wr, 147 op/s 2024-09-13T00:21:33.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:21:32 smithi136 bash[76371]: cluster 2024-09-13T00:21:31.745919+0000 mgr.y (mgr.24661) 1332 : cluster [DBG] pgmap v2015: 105 pgs: 105 active+clean; 104 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 1.9 MiB/s rd, 969 KiB/s wr, 147 op/s 2024-09-13T00:21:33.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:21:32 smithi136 bash[75164]: cluster 2024-09-13T00:21:31.745919+0000 mgr.y (mgr.24661) 1332 : cluster [DBG] pgmap v2015: 105 pgs: 105 active+clean; 104 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 1.9 MiB/s rd, 969 KiB/s wr, 147 op/s 2024-09-13T00:21:35.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:21:34 smithi181 bash[67587]: cluster 2024-09-13T00:21:33.746864+0000 mgr.y (mgr.24661) 1333 : cluster [DBG] pgmap v2016: 105 pgs: 105 active+clean; 100 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 711 KiB/s rd, 360 KiB/s wr, 174 op/s 2024-09-13T00:21:35.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:21:34 smithi136 bash[76371]: cluster 2024-09-13T00:21:33.746864+0000 mgr.y (mgr.24661) 1333 : cluster [DBG] pgmap v2016: 105 pgs: 105 active+clean; 100 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 711 KiB/s rd, 360 KiB/s wr, 174 op/s 2024-09-13T00:21:35.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:21:34 smithi136 bash[75164]: cluster 2024-09-13T00:21:33.746864+0000 mgr.y (mgr.24661) 1333 : cluster [DBG] pgmap v2016: 105 pgs: 105 active+clean; 100 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 711 KiB/s rd, 360 KiB/s wr, 174 op/s 2024-09-13T00:21:37.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:21:36 smithi181 bash[67587]: cluster 2024-09-13T00:21:35.747942+0000 mgr.y (mgr.24661) 1334 : cluster [DBG] pgmap v2017: 105 pgs: 105 active+clean; 96 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 372 KiB/s rd, 285 KiB/s wr, 202 op/s 2024-09-13T00:21:37.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:21:36 smithi136 bash[76371]: cluster 2024-09-13T00:21:35.747942+0000 mgr.y (mgr.24661) 1334 : cluster [DBG] pgmap v2017: 105 pgs: 105 active+clean; 96 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 372 KiB/s rd, 285 KiB/s wr, 202 op/s 2024-09-13T00:21:37.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:21:36 smithi136 bash[75164]: cluster 2024-09-13T00:21:35.747942+0000 mgr.y (mgr.24661) 1334 : cluster [DBG] pgmap v2017: 105 pgs: 105 active+clean; 96 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 372 KiB/s rd, 285 KiB/s wr, 202 op/s 2024-09-13T00:21:39.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:21:38 smithi181 bash[67587]: cluster 2024-09-13T00:21:37.749322+0000 mgr.y (mgr.24661) 1335 : cluster [DBG] pgmap v2018: 105 pgs: 105 active+clean; 88 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 154 KiB/s rd, 278 KiB/s wr, 248 op/s 2024-09-13T00:21:39.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:21:38 smithi181 bash[67587]: audit 2024-09-13T00:21:38.805768+0000 mon.a (mon.0) 2295 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:21:39.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:21:38 smithi136 bash[76371]: cluster 2024-09-13T00:21:37.749322+0000 mgr.y (mgr.24661) 1335 : cluster [DBG] pgmap v2018: 105 pgs: 105 active+clean; 88 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 154 KiB/s rd, 278 KiB/s wr, 248 op/s 2024-09-13T00:21:39.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:21:38 smithi136 bash[76371]: audit 2024-09-13T00:21:38.805768+0000 mon.a (mon.0) 2295 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:21:39.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:21:38 smithi136 bash[75164]: cluster 2024-09-13T00:21:37.749322+0000 mgr.y (mgr.24661) 1335 : cluster [DBG] pgmap v2018: 105 pgs: 105 active+clean; 88 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 154 KiB/s rd, 278 KiB/s wr, 248 op/s 2024-09-13T00:21:39.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:21:38 smithi136 bash[75164]: audit 2024-09-13T00:21:38.805768+0000 mon.a (mon.0) 2295 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:21:39.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:21:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:21:39] "GET /metrics HTTP/1.1" 200 34136 "" "Prometheus/2.43.0" 2024-09-13T00:21:41.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:21:40 smithi181 bash[67587]: cluster 2024-09-13T00:21:39.750242+0000 mgr.y (mgr.24661) 1336 : cluster [DBG] pgmap v2019: 105 pgs: 105 active+clean; 82 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 261 op/s 2024-09-13T00:21:41.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:21:40 smithi136 bash[75164]: cluster 2024-09-13T00:21:39.750242+0000 mgr.y (mgr.24661) 1336 : cluster [DBG] pgmap v2019: 105 pgs: 105 active+clean; 82 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 261 op/s 2024-09-13T00:21:41.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:21:40 smithi136 bash[76371]: cluster 2024-09-13T00:21:39.750242+0000 mgr.y (mgr.24661) 1336 : cluster [DBG] pgmap v2019: 105 pgs: 105 active+clean; 82 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 261 op/s 2024-09-13T00:21:42.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:21:42 smithi181 bash[95505]: ts=2024-09-13T00:21:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:21:43.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:21:42 smithi181 bash[67587]: cluster 2024-09-13T00:21:41.751329+0000 mgr.y (mgr.24661) 1337 : cluster [DBG] pgmap v2020: 105 pgs: 105 active+clean; 72 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 291 op/s 2024-09-13T00:21:43.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:21:42 smithi136 bash[75164]: cluster 2024-09-13T00:21:41.751329+0000 mgr.y (mgr.24661) 1337 : cluster [DBG] pgmap v2020: 105 pgs: 105 active+clean; 72 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 291 op/s 2024-09-13T00:21:43.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:21:42 smithi136 bash[76371]: cluster 2024-09-13T00:21:41.751329+0000 mgr.y (mgr.24661) 1337 : cluster [DBG] pgmap v2020: 105 pgs: 105 active+clean; 72 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 291 op/s 2024-09-13T00:21:45.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:21:44 smithi181 bash[67587]: cluster 2024-09-13T00:21:43.752215+0000 mgr.y (mgr.24661) 1338 : cluster [DBG] pgmap v2021: 105 pgs: 105 active+clean; 69 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 244 op/s 2024-09-13T00:21:45.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:21:44 smithi136 bash[75164]: cluster 2024-09-13T00:21:43.752215+0000 mgr.y (mgr.24661) 1338 : cluster [DBG] pgmap v2021: 105 pgs: 105 active+clean; 69 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 244 op/s 2024-09-13T00:21:45.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:21:44 smithi136 bash[76371]: cluster 2024-09-13T00:21:43.752215+0000 mgr.y (mgr.24661) 1338 : cluster [DBG] pgmap v2021: 105 pgs: 105 active+clean; 69 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 244 op/s 2024-09-13T00:21:47.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:21:46 smithi181 bash[67587]: cluster 2024-09-13T00:21:45.753172+0000 mgr.y (mgr.24661) 1339 : cluster [DBG] pgmap v2022: 105 pgs: 105 active+clean; 65 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 242 op/s 2024-09-13T00:21:47.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:21:46 smithi136 bash[75164]: cluster 2024-09-13T00:21:45.753172+0000 mgr.y (mgr.24661) 1339 : cluster [DBG] pgmap v2022: 105 pgs: 105 active+clean; 65 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 242 op/s 2024-09-13T00:21:47.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:21:46 smithi136 bash[76371]: cluster 2024-09-13T00:21:45.753172+0000 mgr.y (mgr.24661) 1339 : cluster [DBG] pgmap v2022: 105 pgs: 105 active+clean; 65 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 242 op/s 2024-09-13T00:21:49.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:21:48 smithi181 bash[67587]: cluster 2024-09-13T00:21:47.754321+0000 mgr.y (mgr.24661) 1340 : cluster [DBG] pgmap v2023: 105 pgs: 105 active+clean; 57 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 265 op/s 2024-09-13T00:21:49.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:21:48 smithi136 bash[75164]: cluster 2024-09-13T00:21:47.754321+0000 mgr.y (mgr.24661) 1340 : cluster [DBG] pgmap v2023: 105 pgs: 105 active+clean; 57 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 265 op/s 2024-09-13T00:21:49.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:21:48 smithi136 bash[76371]: cluster 2024-09-13T00:21:47.754321+0000 mgr.y (mgr.24661) 1340 : cluster [DBG] pgmap v2023: 105 pgs: 105 active+clean; 57 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 265 op/s 2024-09-13T00:21:49.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:21:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:21:49] "GET /metrics HTTP/1.1" 200 34136 "" "Prometheus/2.43.0" 2024-09-13T00:21:51.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:21:50 smithi181 bash[67587]: cluster 2024-09-13T00:21:49.755284+0000 mgr.y (mgr.24661) 1341 : cluster [DBG] pgmap v2024: 105 pgs: 105 active+clean; 52 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 249 op/s 2024-09-13T00:21:51.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:21:50 smithi136 bash[75164]: cluster 2024-09-13T00:21:49.755284+0000 mgr.y (mgr.24661) 1341 : cluster [DBG] pgmap v2024: 105 pgs: 105 active+clean; 52 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 249 op/s 2024-09-13T00:21:51.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:21:50 smithi136 bash[76371]: cluster 2024-09-13T00:21:49.755284+0000 mgr.y (mgr.24661) 1341 : cluster [DBG] pgmap v2024: 105 pgs: 105 active+clean; 52 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 249 op/s 2024-09-13T00:21:52.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:21:52 smithi181 bash[95505]: ts=2024-09-13T00:21:52.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:21:53.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:21:52 smithi181 bash[67587]: cluster 2024-09-13T00:21:51.756604+0000 mgr.y (mgr.24661) 1342 : cluster [DBG] pgmap v2025: 105 pgs: 105 active+clean; 43 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 270 op/s 2024-09-13T00:21:53.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:21:52 smithi136 bash[75164]: cluster 2024-09-13T00:21:51.756604+0000 mgr.y (mgr.24661) 1342 : cluster [DBG] pgmap v2025: 105 pgs: 105 active+clean; 43 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 270 op/s 2024-09-13T00:21:53.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:21:52 smithi136 bash[76371]: cluster 2024-09-13T00:21:51.756604+0000 mgr.y (mgr.24661) 1342 : cluster [DBG] pgmap v2025: 105 pgs: 105 active+clean; 43 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 270 op/s 2024-09-13T00:21:54.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:21:53 smithi136 bash[75164]: audit 2024-09-13T00:21:53.806062+0000 mon.a (mon.0) 2296 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:21:54.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:21:53 smithi136 bash[76371]: audit 2024-09-13T00:21:53.806062+0000 mon.a (mon.0) 2296 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:21:54.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:21:53 smithi181 bash[67587]: audit 2024-09-13T00:21:53.806062+0000 mon.a (mon.0) 2296 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:21:55.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:21:54 smithi136 bash[75164]: cluster 2024-09-13T00:21:53.757555+0000 mgr.y (mgr.24661) 1343 : cluster [DBG] pgmap v2026: 105 pgs: 105 active+clean; 39 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 226 op/s 2024-09-13T00:21:55.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:21:54 smithi136 bash[76371]: cluster 2024-09-13T00:21:53.757555+0000 mgr.y (mgr.24661) 1343 : cluster [DBG] pgmap v2026: 105 pgs: 105 active+clean; 39 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 226 op/s 2024-09-13T00:21:55.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:21:54 smithi181 bash[67587]: cluster 2024-09-13T00:21:53.757555+0000 mgr.y (mgr.24661) 1343 : cluster [DBG] pgmap v2026: 105 pgs: 105 active+clean; 39 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 226 op/s 2024-09-13T00:21:57.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:21:56 smithi136 bash[75164]: cluster 2024-09-13T00:21:55.758455+0000 mgr.y (mgr.24661) 1344 : cluster [DBG] pgmap v2027: 105 pgs: 105 active+clean; 35 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 226 op/s 2024-09-13T00:21:57.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:21:56 smithi136 bash[76371]: cluster 2024-09-13T00:21:55.758455+0000 mgr.y (mgr.24661) 1344 : cluster [DBG] pgmap v2027: 105 pgs: 105 active+clean; 35 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 226 op/s 2024-09-13T00:21:57.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:21:56 smithi181 bash[67587]: cluster 2024-09-13T00:21:55.758455+0000 mgr.y (mgr.24661) 1344 : cluster [DBG] pgmap v2027: 105 pgs: 105 active+clean; 35 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 226 op/s 2024-09-13T00:21:59.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:21:58 smithi136 bash[76371]: cluster 2024-09-13T00:21:57.759573+0000 mgr.y (mgr.24661) 1345 : cluster [DBG] pgmap v2028: 105 pgs: 105 active+clean; 28 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 250 op/s 2024-09-13T00:21:59.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:21:58 smithi136 bash[75164]: cluster 2024-09-13T00:21:57.759573+0000 mgr.y (mgr.24661) 1345 : cluster [DBG] pgmap v2028: 105 pgs: 105 active+clean; 28 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 250 op/s 2024-09-13T00:21:59.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:21:58 smithi181 bash[67587]: cluster 2024-09-13T00:21:57.759573+0000 mgr.y (mgr.24661) 1345 : cluster [DBG] pgmap v2028: 105 pgs: 105 active+clean; 28 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 250 op/s 2024-09-13T00:21:59.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:21:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:21:59] "GET /metrics HTTP/1.1" 200 34132 "" "Prometheus/2.43.0" 2024-09-13T00:22:01.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:00 smithi136 bash[75164]: cluster 2024-09-13T00:21:59.760556+0000 mgr.y (mgr.24661) 1346 : cluster [DBG] pgmap v2029: 105 pgs: 105 active+clean; 23 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 233 op/s 2024-09-13T00:22:01.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:00 smithi136 bash[76371]: cluster 2024-09-13T00:21:59.760556+0000 mgr.y (mgr.24661) 1346 : cluster [DBG] pgmap v2029: 105 pgs: 105 active+clean; 23 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 233 op/s 2024-09-13T00:22:01.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:00 smithi181 bash[67587]: cluster 2024-09-13T00:21:59.760556+0000 mgr.y (mgr.24661) 1346 : cluster [DBG] pgmap v2029: 105 pgs: 105 active+clean; 23 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 233 op/s 2024-09-13T00:22:02.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:22:02 smithi181 bash[95505]: ts=2024-09-13T00:22:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:22:03.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:02 smithi136 bash[76371]: cluster 2024-09-13T00:22:01.761670+0000 mgr.y (mgr.24661) 1347 : cluster [DBG] pgmap v2030: 105 pgs: 105 active+clean; 13 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 257 op/s 2024-09-13T00:22:03.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:02 smithi136 bash[75164]: cluster 2024-09-13T00:22:01.761670+0000 mgr.y (mgr.24661) 1347 : cluster [DBG] pgmap v2030: 105 pgs: 105 active+clean; 13 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 257 op/s 2024-09-13T00:22:03.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:02 smithi181 bash[67587]: cluster 2024-09-13T00:22:01.761670+0000 mgr.y (mgr.24661) 1347 : cluster [DBG] pgmap v2030: 105 pgs: 105 active+clean; 13 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 257 op/s 2024-09-13T00:22:05.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:04 smithi136 bash[76371]: cluster 2024-09-13T00:22:03.762528+0000 mgr.y (mgr.24661) 1348 : cluster [DBG] pgmap v2031: 105 pgs: 105 active+clean; 9.8 GiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 218 op/s 2024-09-13T00:22:05.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:04 smithi136 bash[75164]: cluster 2024-09-13T00:22:03.762528+0000 mgr.y (mgr.24661) 1348 : cluster [DBG] pgmap v2031: 105 pgs: 105 active+clean; 9.8 GiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 218 op/s 2024-09-13T00:22:05.309 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-13T00:22:05.309 DEBUG:teuthology.orchestra.run.smithi136:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-13T00:22:05.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:04 smithi181 bash[67587]: cluster 2024-09-13T00:22:03.762528+0000 mgr.y (mgr.24661) 1348 : cluster [DBG] pgmap v2031: 105 pgs: 105 active+clean; 9.8 GiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 218 op/s 2024-09-13T00:22:05.365 INFO:tasks.workunit:Stopping ['rados/load-gen-big.sh'] on client.0... 2024-09-13T00:22:05.365 DEBUG:teuthology.orchestra.run.smithi136:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2024-09-13T00:22:05.633 DEBUG:teuthology.parallel:result is None 2024-09-13T00:22:05.634 DEBUG:teuthology.orchestra.run.smithi136:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2024-09-13T00:22:05.646 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2024-09-13T00:22:05.646 DEBUG:teuthology.orchestra.run.smithi136:> rmdir -- /home/ubuntu/cephtest/mnt.0 2024-09-13T00:22:05.695 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2024-09-13T00:22:05.695 INFO:teuthology.task.full_sequential:In full_sequential, running task print... 2024-09-13T00:22:05.696 INFO:teuthology.task.print:**** done end rados_loadgenbig.yaml 2024-09-13T00:22:05.696 INFO:teuthology.task.full_sequential:In full_sequential, running task print... 2024-09-13T00:22:05.696 INFO:teuthology.task.print:**** done start rbd_import_export.yaml 2024-09-13T00:22:05.696 INFO:teuthology.task.full_sequential:In full_sequential, running task workunit... 2024-09-13T00:22:05.697 INFO:tasks.workunit:Pulling workunits from ref quincy 2024-09-13T00:22:05.697 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-09-13T00:22:05.697 DEBUG:teuthology.orchestra.run.smithi181:> stat -- /home/ubuntu/cephtest/mnt.1 2024-09-13T00:22:05.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T00:22:05.704 INFO:teuthology.orchestra.run.smithi181.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.1': No such file or directory 2024-09-13T00:22:05.704 DEBUG:teuthology.orchestra.run.smithi181:> mkdir -- /home/ubuntu/cephtest/mnt.1 2024-09-13T00:22:05.751 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.1 2024-09-13T00:22:05.751 DEBUG:teuthology.orchestra.run.smithi181:> cd -- /home/ubuntu/cephtest/mnt.1 && mkdir -- client.1 2024-09-13T00:22:05.798 INFO:tasks.workunit:timeout=3h 2024-09-13T00:22:05.798 INFO:tasks.workunit:cleanup=True 2024-09-13T00:22:05.799 DEBUG:teuthology.orchestra.run.smithi181:> rm -rf /home/ubuntu/cephtest/clone.client.1 && git clone --depth 1 --branch quincy https://git.ceph.com/ceph.git /home/ubuntu/cephtest/clone.client.1 && cd /home/ubuntu/cephtest/clone.client.1 2024-09-13T00:22:05.848 INFO:tasks.workunit.client.1.smithi181.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.1'... 2024-09-13T00:22:07.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:06 smithi136 bash[76371]: cluster 2024-09-13T00:22:05.763458+0000 mgr.y (mgr.24661) 1349 : cluster [DBG] pgmap v2032: 105 pgs: 105 active+clean; 6.7 GiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 215 op/s 2024-09-13T00:22:07.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:06 smithi136 bash[75164]: cluster 2024-09-13T00:22:05.763458+0000 mgr.y (mgr.24661) 1349 : cluster [DBG] pgmap v2032: 105 pgs: 105 active+clean; 6.7 GiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 215 op/s 2024-09-13T00:22:07.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:06 smithi181 bash[67587]: cluster 2024-09-13T00:22:05.763458+0000 mgr.y (mgr.24661) 1349 : cluster [DBG] pgmap v2032: 105 pgs: 105 active+clean; 6.7 GiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 215 op/s 2024-09-13T00:22:09.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:08 smithi136 bash[76371]: cluster 2024-09-13T00:22:07.764433+0000 mgr.y (mgr.24661) 1350 : cluster [DBG] pgmap v2033: 105 pgs: 105 active+clean; 1.6 GiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 227 op/s 2024-09-13T00:22:09.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:08 smithi136 bash[76371]: audit 2024-09-13T00:22:08.806514+0000 mon.a (mon.0) 2297 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:22:09.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:08 smithi136 bash[75164]: cluster 2024-09-13T00:22:07.764433+0000 mgr.y (mgr.24661) 1350 : cluster [DBG] pgmap v2033: 105 pgs: 105 active+clean; 1.6 GiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 227 op/s 2024-09-13T00:22:09.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:08 smithi136 bash[75164]: audit 2024-09-13T00:22:08.806514+0000 mon.a (mon.0) 2297 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:22:09.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:08 smithi181 bash[67587]: cluster 2024-09-13T00:22:07.764433+0000 mgr.y (mgr.24661) 1350 : cluster [DBG] pgmap v2033: 105 pgs: 105 active+clean; 1.6 GiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 227 op/s 2024-09-13T00:22:09.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:08 smithi181 bash[67587]: audit 2024-09-13T00:22:08.806514+0000 mon.a (mon.0) 2297 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:22:09.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:22:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:22:09] "GET /metrics HTTP/1.1" 200 34131 "" "Prometheus/2.43.0" 2024-09-13T00:22:11.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:10 smithi136 bash[76371]: cluster 2024-09-13T00:22:09.765261+0000 mgr.y (mgr.24661) 1351 : cluster [DBG] pgmap v2034: 105 pgs: 105 active+clean; 583 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 187 op/s 2024-09-13T00:22:11.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:10 smithi136 bash[75164]: cluster 2024-09-13T00:22:09.765261+0000 mgr.y (mgr.24661) 1351 : cluster [DBG] pgmap v2034: 105 pgs: 105 active+clean; 583 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 187 op/s 2024-09-13T00:22:11.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:10 smithi181 bash[67587]: cluster 2024-09-13T00:22:09.765261+0000 mgr.y (mgr.24661) 1351 : cluster [DBG] pgmap v2034: 105 pgs: 105 active+clean; 583 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 187 op/s 2024-09-13T00:22:11.970 DEBUG:teuthology.orchestra.run.smithi181:> cd -- /home/ubuntu/cephtest/clone.client.1/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.1 2024-09-13T00:22:12.018 INFO:tasks.workunit.client.1.smithi181.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-09-13T00:22:12.022 INFO:tasks.workunit.client.1.smithi181.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.1/qa/workunits/direct_io' 2024-09-13T00:22:12.022 INFO:tasks.workunit.client.1.smithi181.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-09-13T00:22:12.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:22:12 smithi181 bash[95505]: ts=2024-09-13T00:22:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:22:12.983 INFO:tasks.workunit.client.1.smithi181.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-09-13T00:22:13.060 INFO:tasks.workunit.client.1.smithi181.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-09-13T00:22:13.109 INFO:tasks.workunit.client.1.smithi181.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.1/qa/workunits/direct_io' 2024-09-13T00:22:13.110 INFO:tasks.workunit.client.1.smithi181.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.1/qa/workunits/fs' 2024-09-13T00:22:13.110 INFO:tasks.workunit.client.1.smithi181.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-09-13T00:22:13.156 INFO:tasks.workunit.client.1.smithi181.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.1/qa/workunits/fs' 2024-09-13T00:22:13.160 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2024-09-13T00:22:13.160 DEBUG:teuthology.orchestra.run.smithi181:> dd if=/home/ubuntu/cephtest/workunits.list.client.1 of=/dev/stdout 2024-09-13T00:22:13.207 INFO:tasks.workunit:Running workunits matching rbd/import_export.sh on client.1... 2024-09-13T00:22:13.208 INFO:tasks.workunit:Running workunit rbd/import_export.sh... 2024-09-13T00:22:13.208 DEBUG:teuthology.orchestra.run.smithi181:workunit test rbd/import_export.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && cd -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="1" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.1 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.1 CEPH_MNT=/home/ubuntu/cephtest/mnt.1 RBD_CREATE_ARGS=--new-format adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.1/qa/workunits/rbd/import_export.sh 2024-09-13T00:22:13.258 INFO:tasks.workunit.client.1.smithi181.stderr:+ export RBD_FORCE_ALLOW_V1=1 2024-09-13T00:22:13.258 INFO:tasks.workunit.client.1.smithi181.stderr:+ TMPDIR=/tmp/rbd_import_export_106423 2024-09-13T00:22:13.258 INFO:tasks.workunit.client.1.smithi181.stderr:+ rm -rf /tmp/rbd_import_export_106423 2024-09-13T00:22:13.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:12 smithi136 bash[75164]: cluster 2024-09-13T00:22:11.766320+0000 mgr.y (mgr.24661) 1352 : cluster [DBG] pgmap v2035: 105 pgs: 105 active+clean; 583 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 152 op/s 2024-09-13T00:22:13.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:12 smithi136 bash[76371]: cluster 2024-09-13T00:22:11.766320+0000 mgr.y (mgr.24661) 1352 : cluster [DBG] pgmap v2035: 105 pgs: 105 active+clean; 583 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 152 op/s 2024-09-13T00:22:13.260 INFO:tasks.workunit.client.1.smithi181.stderr:+ mkdir /tmp/rbd_import_export_106423 2024-09-13T00:22:13.260 INFO:tasks.workunit.client.1.smithi181.stderr:+ trap rm -rf /tmp/rbd_import_export_106423 INT TERM EXIT 2024-09-13T00:22:13.260 INFO:tasks.workunit.client.1.smithi181.stderr:+ mkdir foo.106423 2024-09-13T00:22:13.260 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd import foo.106423 foo.dir 2024-09-13T00:22:13.328 INFO:tasks.workunit.client.1.smithi181.stderr:rbd: cannot import a directory 2024-09-13T00:22:13.328 INFO:tasks.workunit.client.1.smithi181.stderr: Importing image: 0% complete...failed. 2024-09-13T00:22:13.328 INFO:tasks.workunit.client.1.smithi181.stderr:rbd: import failed: (21) Is a directory 2024-09-13T00:22:13.332 INFO:tasks.workunit.client.1.smithi181.stderr:+ true 2024-09-13T00:22:13.332 INFO:tasks.workunit.client.1.smithi181.stderr:+ rmdir foo.106423 2024-09-13T00:22:13.336 INFO:tasks.workunit.client.1.smithi181.stderr:+ dd if=/bin/sh of=/tmp/rbd_import_export_106423/img bs=1k count=1 seek=10 2024-09-13T00:22:13.337 INFO:tasks.workunit.client.1.smithi181.stderr:1+0 records in 2024-09-13T00:22:13.337 INFO:tasks.workunit.client.1.smithi181.stderr:1+0 records out 2024-09-13T00:22:13.337 INFO:tasks.workunit.client.1.smithi181.stderr:1024 bytes (1.0 kB, 1.0 KiB) copied, 9.3523e-05 s, 10.9 MB/s 2024-09-13T00:22:13.337 INFO:tasks.workunit.client.1.smithi181.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_106423/img bs=1k count=10 seek=100 2024-09-13T00:22:13.337 INFO:tasks.workunit.client.1.smithi181.stderr:10+0 records in 2024-09-13T00:22:13.338 INFO:tasks.workunit.client.1.smithi181.stderr:10+0 records out 2024-09-13T00:22:13.338 INFO:tasks.workunit.client.1.smithi181.stderr:10240 bytes (10 kB, 10 KiB) copied, 0.00021492 s, 47.6 MB/s 2024-09-13T00:22:13.338 INFO:tasks.workunit.client.1.smithi181.stderr:+ dd if=/bin/rm of=/tmp/rbd_import_export_106423/img bs=1k count=100 seek=1000 2024-09-13T00:22:13.338 INFO:tasks.workunit.client.1.smithi181.stderr:58+1 records in 2024-09-13T00:22:13.338 INFO:tasks.workunit.client.1.smithi181.stderr:58+1 records out 2024-09-13T00:22:13.338 INFO:tasks.workunit.client.1.smithi181.stderr:59904 bytes (60 kB, 58 KiB) copied, 0.000231973 s, 258 MB/s 2024-09-13T00:22:13.338 INFO:tasks.workunit.client.1.smithi181.stderr:+ dd if=/bin/ls of=/tmp/rbd_import_export_106423/img bs=1k seek=10000 2024-09-13T00:22:13.339 INFO:tasks.workunit.client.1.smithi181.stderr:134+1 records in 2024-09-13T00:22:13.339 INFO:tasks.workunit.client.1.smithi181.stderr:134+1 records out 2024-09-13T00:22:13.340 INFO:tasks.workunit.client.1.smithi181.stderr:138208 bytes (138 kB, 135 KiB) copied, 0.000546451 s, 253 MB/s 2024-09-13T00:22:13.340 INFO:tasks.workunit.client.1.smithi181.stderr:+ dd if=/bin/ln of=/tmp/rbd_import_export_106423/img bs=1k seek=100000 2024-09-13T00:22:13.340 INFO:tasks.workunit.client.1.smithi181.stderr:58+1 records in 2024-09-13T00:22:13.340 INFO:tasks.workunit.client.1.smithi181.stderr:58+1 records out 2024-09-13T00:22:13.340 INFO:tasks.workunit.client.1.smithi181.stderr:59904 bytes (60 kB, 58 KiB) copied, 0.000223909 s, 268 MB/s 2024-09-13T00:22:13.340 INFO:tasks.workunit.client.1.smithi181.stderr:+ dd if=/bin/grep of=/tmp/rbd_import_export_106423/img bs=1k seek=1000000 2024-09-13T00:22:13.341 INFO:tasks.workunit.client.1.smithi181.stderr:178+1 records in 2024-09-13T00:22:13.341 INFO:tasks.workunit.client.1.smithi181.stderr:178+1 records out 2024-09-13T00:22:13.341 INFO:tasks.workunit.client.1.smithi181.stderr:182728 bytes (183 kB, 178 KiB) copied, 0.00052522 s, 348 MB/s 2024-09-13T00:22:13.341 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd rm testimg 2024-09-13T00:22:13.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:12 smithi181 bash[67587]: cluster 2024-09-13T00:22:11.766320+0000 mgr.y (mgr.24661) 1352 : cluster [DBG] pgmap v2035: 105 pgs: 105 active+clean; 583 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 152 op/s 2024-09-13T00:22:13.412 INFO:tasks.workunit.client.1.smithi181.stderr: Removing image: 0% complete...failed. 2024-09-13T00:22:13.412 INFO:tasks.workunit.client.1.smithi181.stderr:rbd: delete error: (2) No such file or directory 2024-09-13T00:22:13.416 INFO:tasks.workunit.client.1.smithi181.stderr:+ true 2024-09-13T00:22:13.416 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd import --new-format /tmp/rbd_import_export_106423/img testimg 2024-09-13T00:22:14.015 INFO:tasks.workunit.client.1.smithi181.stderr: Importing image: 1% complete... Importing image: 2% complete... Importing image: 3% complete... Importing image: 4% complete... Importing image: 5% complete... Importing image: 6% complete... Importing image: 7% complete... Importing image: 8% complete... Importing image: 9% complete... Importing image: 10% complete... Importing image: 11% complete... Importing image: 12% complete... Importing image: 13% complete... Importing image: 14% complete... Importing image: 15% complete... Importing image: 16% complete... Importing image: 17% complete... Importing image: 18% complete... Importing image: 19% complete... Importing image: 20% complete... Importing image: 21% complete... Importing image: 22% complete... Importing image: 23% complete... Importing image: 24% complete... Importing image: 25% complete... Importing image: 26% complete... Importing image: 27% complete... Importing image: 28% complete... Importing image: 29% complete... Importing image: 30% complete... Importing image: 31% complete... Importing image: 32% complete... Importing image: 33% complete... Importing image: 34% complete... Importing image: 35% complete... Importing image: 36% complete... Importing image: 37% complete... Importing image: 38% complete... Importing image: 39% complete... Importing image: 40% complete... Importing image: 41% complete... Importing image: 42% complete... Importing image: 43% complete... Importing image: 44% complete... Importing image: 45% complete... Importing image: 46% complete... Importing image: 47% complete... Importing image: 48% complete... Importing image: 49% complete... Importing image: 50% complete... Importing image: 51% complete... Importing image: 52% complete... Importing image: 53% complete... Importing image: 54% complete... Importing image: 55% complete... Importing image: 56% complete... Importing image: 57% complete... Importing image: 58% complete... Importing image: 59% complete... Importing image: 60% complete... Importing image: 61% complete... Importing image: 62% complete... Importing image: 63% complete... Importing image: 64% complete... Importing image: 65% complete... Importing image: 66% complete... Importing image: 67% complete... Importing image: 68% complete... Importing image: 69% complete... Importing image: 70% complete... Importing image: 71% complete... Importing image: 72% complete... Importing image: 73% complete... Importing image: 74% complete... Importing image: 75% complete... Importing image: 76% complete... Importing image: 77% complete... Importing image: 78% complete... Importing image: 79% complete... Importing image: 80% complete... Importing image: 81% complete... Importing image: 82% complete... Importing image: 83% complete... Importing image: 84% complete... Importing image: 85% complete... Importing image: 86% complete... Importing image: 87% complete... Importing image: 88% complete... Importing image: 89% complete... Importing image: 90% complete... Importing image: 91% complete... Importing image: 92% complete... Importing image: 93% complete... Importing image: 94% complete... Importing image: 95% complete... Importing image: 96% complete... Importing image: 97% complete... Importing image: 98% complete... Importing image: 99% complete... Importing image: 100% complete...done. 2024-09-13T00:22:14.015 INFO:tasks.workunit.client.1.smithi181.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-09-13T00:22:14.018 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd export testimg /tmp/rbd_import_export_106423/img2 2024-09-13T00:22:14.277 INFO:tasks.workunit.client.1.smithi181.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-13T00:22:14.281 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd export testimg - 2024-09-13T00:22:15.108 INFO:tasks.workunit.client.1.smithi181.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-13T00:22:15.112 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd rm testimg 2024-09-13T00:22:15.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:14 smithi136 bash[75164]: cluster 2024-09-13T00:22:13.767181+0000 mgr.y (mgr.24661) 1353 : cluster [DBG] pgmap v2036: 105 pgs: 105 active+clean; 583 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 85 B/s rd, 0 B/s wr, 91 op/s 2024-09-13T00:22:15.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:14 smithi136 bash[76371]: cluster 2024-09-13T00:22:13.767181+0000 mgr.y (mgr.24661) 1353 : cluster [DBG] pgmap v2036: 105 pgs: 105 active+clean; 583 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 85 B/s rd, 0 B/s wr, 91 op/s 2024-09-13T00:22:15.261 INFO:tasks.workunit.client.1.smithi181.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-13T00:22:15.265 INFO:tasks.workunit.client.1.smithi181.stderr:+ cmp /tmp/rbd_import_export_106423/img /tmp/rbd_import_export_106423/img2 2024-09-13T00:22:15.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:14 smithi181 bash[67587]: cluster 2024-09-13T00:22:13.767181+0000 mgr.y (mgr.24661) 1353 : cluster [DBG] pgmap v2036: 105 pgs: 105 active+clean; 583 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 85 B/s rd, 0 B/s wr, 91 op/s 2024-09-13T00:22:15.942 INFO:tasks.workunit.client.1.smithi181.stderr:+ cmp /tmp/rbd_import_export_106423/img /tmp/rbd_import_export_106423/img3 2024-09-13T00:22:16.437 INFO:tasks.workunit.client.1.smithi181.stderr:+ rm /tmp/rbd_import_export_106423/img2 /tmp/rbd_import_export_106423/img3 2024-09-13T00:22:16.612 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd import --new-format - testimg 2024-09-13T00:22:17.195 INFO:tasks.workunit.client.1.smithi181.stderr: Importing image: 100% complete...done. 2024-09-13T00:22:17.196 INFO:tasks.workunit.client.1.smithi181.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-09-13T00:22:17.199 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd export testimg /tmp/rbd_import_export_106423/img2 2024-09-13T00:22:17.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:16 smithi136 bash[76371]: cluster 2024-09-13T00:22:15.768066+0000 mgr.y (mgr.24661) 1354 : cluster [DBG] pgmap v2037: 105 pgs: 105 active+clean; 583 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 53 KiB/s rd, 26 KiB/s wr, 69 op/s 2024-09-13T00:22:17.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:16 smithi136 bash[76371]: audit 2024-09-13T00:22:16.480704+0000 mon.a (mon.0) 2298 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:22:17.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:16 smithi136 bash[75164]: cluster 2024-09-13T00:22:15.768066+0000 mgr.y (mgr.24661) 1354 : cluster [DBG] pgmap v2037: 105 pgs: 105 active+clean; 583 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 53 KiB/s rd, 26 KiB/s wr, 69 op/s 2024-09-13T00:22:17.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:16 smithi136 bash[75164]: audit 2024-09-13T00:22:16.480704+0000 mon.a (mon.0) 2298 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:22:17.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:16 smithi181 bash[67587]: cluster 2024-09-13T00:22:15.768066+0000 mgr.y (mgr.24661) 1354 : cluster [DBG] pgmap v2037: 105 pgs: 105 active+clean; 583 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 53 KiB/s rd, 26 KiB/s wr, 69 op/s 2024-09-13T00:22:17.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:16 smithi181 bash[67587]: audit 2024-09-13T00:22:16.480704+0000 mon.a (mon.0) 2298 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:22:17.510 INFO:tasks.workunit.client.1.smithi181.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-13T00:22:17.516 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd export testimg - 2024-09-13T00:22:18.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:17 smithi136 bash[75164]: audit 2024-09-13T00:22:17.028530+0000 mon.a (mon.0) 2299 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:22:18.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:17 smithi136 bash[75164]: audit 2024-09-13T00:22:17.029952+0000 mon.a (mon.0) 2300 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:22:18.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:17 smithi136 bash[75164]: audit 2024-09-13T00:22:17.039961+0000 mon.a (mon.0) 2301 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:22:18.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:17 smithi136 bash[76371]: audit 2024-09-13T00:22:17.028530+0000 mon.a (mon.0) 2299 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:22:18.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:17 smithi136 bash[76371]: audit 2024-09-13T00:22:17.029952+0000 mon.a (mon.0) 2300 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:22:18.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:17 smithi136 bash[76371]: audit 2024-09-13T00:22:17.039961+0000 mon.a (mon.0) 2301 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:22:18.347 INFO:tasks.workunit.client.1.smithi181.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-13T00:22:18.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:17 smithi181 bash[67587]: audit 2024-09-13T00:22:17.028530+0000 mon.a (mon.0) 2299 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:22:18.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:17 smithi181 bash[67587]: audit 2024-09-13T00:22:17.029952+0000 mon.a (mon.0) 2300 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:22:18.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:17 smithi181 bash[67587]: audit 2024-09-13T00:22:17.039961+0000 mon.a (mon.0) 2301 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:22:18.351 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd rm testimg 2024-09-13T00:22:18.588 INFO:tasks.workunit.client.1.smithi181.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-13T00:22:18.595 INFO:tasks.workunit.client.1.smithi181.stderr:+ cmp /tmp/rbd_import_export_106423/img /tmp/rbd_import_export_106423/img2 2024-09-13T00:22:19.290 INFO:tasks.workunit.client.1.smithi181.stderr:+ cmp /tmp/rbd_import_export_106423/img /tmp/rbd_import_export_106423/img3 2024-09-13T00:22:19.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:19 smithi181 bash[67587]: cluster 2024-09-13T00:22:17.769107+0000 mgr.y (mgr.24661) 1355 : cluster [DBG] pgmap v2038: 105 pgs: 105 active+clean; 1.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 86 KiB/s rd, 41 KiB/s wr, 77 op/s 2024-09-13T00:22:19.412 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:19 smithi136 bash[75164]: cluster 2024-09-13T00:22:17.769107+0000 mgr.y (mgr.24661) 1355 : cluster [DBG] pgmap v2038: 105 pgs: 105 active+clean; 1.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 86 KiB/s rd, 41 KiB/s wr, 77 op/s 2024-09-13T00:22:19.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:19 smithi136 bash[76371]: cluster 2024-09-13T00:22:17.769107+0000 mgr.y (mgr.24661) 1355 : cluster [DBG] pgmap v2038: 105 pgs: 105 active+clean; 1.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 86 KiB/s rd, 41 KiB/s wr, 77 op/s 2024-09-13T00:22:19.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:22:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:22:19] "GET /metrics HTTP/1.1" 200 34131 "" "Prometheus/2.43.0" 2024-09-13T00:22:19.793 INFO:tasks.workunit.client.1.smithi181.stderr:+ rm /tmp/rbd_import_export_106423/img /tmp/rbd_import_export_106423/img2 /tmp/rbd_import_export_106423/img3 2024-09-13T00:22:20.020 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd help export 2024-09-13T00:22:20.020 INFO:tasks.workunit.client.1.smithi181.stderr:+ grep -q export-format 2024-09-13T00:22:20.037 INFO:tasks.workunit.client.1.smithi181.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_106423/img bs=1k count=10 seek=100 2024-09-13T00:22:20.037 INFO:tasks.workunit.client.1.smithi181.stderr:10+0 records in 2024-09-13T00:22:20.038 INFO:tasks.workunit.client.1.smithi181.stderr:10+0 records out 2024-09-13T00:22:20.038 INFO:tasks.workunit.client.1.smithi181.stderr:10240 bytes (10 kB, 10 KiB) copied, 9.799e-05 s, 105 MB/s 2024-09-13T00:22:20.038 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd import --new-format /tmp/rbd_import_export_106423/img testimg 2024-09-13T00:22:20.128 INFO:tasks.workunit.client.1.smithi181.stderr: Importing image: 100% complete...done. 2024-09-13T00:22:20.129 INFO:tasks.workunit.client.1.smithi181.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-09-13T00:22:20.131 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd snap create testimg@snap 2024-09-13T00:22:21.085 INFO:tasks.workunit.client.1.smithi181.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2024-09-13T00:22:21.106 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd image-meta set testimg key1 value1 2024-09-13T00:22:21.186 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd image-meta list testimg 2024-09-13T00:22:21.243 INFO:tasks.workunit.client.1.smithi181.stderr:+ IMAGEMETA_BEFORE=There is 1 metadatum on this image: 2024-09-13T00:22:21.243 INFO:tasks.workunit.client.1.smithi181.stderr: 2024-09-13T00:22:21.243 INFO:tasks.workunit.client.1.smithi181.stderr:Key Value 2024-09-13T00:22:21.243 INFO:tasks.workunit.client.1.smithi181.stderr:key1 value1 2024-09-13T00:22:21.244 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_106423/img_v2 2024-09-13T00:22:21.290 INFO:tasks.workunit.client.1.smithi181.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2024-09-13T00:22:21.293 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_106423/img_v2 testimg_import 2024-09-13T00:22:21.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:21 smithi181 bash[67587]: cluster 2024-09-13T00:22:19.769870+0000 mgr.y (mgr.24661) 1356 : cluster [DBG] pgmap v2039: 105 pgs: 105 active+clean; 6.0 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 159 KiB/s rd, 77 KiB/s wr, 43 op/s 2024-09-13T00:22:21.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:21 smithi136 bash[75164]: cluster 2024-09-13T00:22:19.769870+0000 mgr.y (mgr.24661) 1356 : cluster [DBG] pgmap v2039: 105 pgs: 105 active+clean; 6.0 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 159 KiB/s rd, 77 KiB/s wr, 43 op/s 2024-09-13T00:22:21.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:21 smithi136 bash[76371]: cluster 2024-09-13T00:22:19.769870+0000 mgr.y (mgr.24661) 1356 : cluster [DBG] pgmap v2039: 105 pgs: 105 active+clean; 6.0 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 159 KiB/s rd, 77 KiB/s wr, 43 op/s 2024-09-13T00:22:22.121 INFO:tasks.workunit.client.1.smithi181.stderr: Importing image: 50% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2024-09-13T00:22:22.128 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd info testimg_import 2024-09-13T00:22:22.179 INFO:tasks.workunit.client.1.smithi181.stdout:rbd image 'testimg_import': 2024-09-13T00:22:22.179 INFO:tasks.workunit.client.1.smithi181.stdout: size 110 KiB in 1 objects 2024-09-13T00:22:22.179 INFO:tasks.workunit.client.1.smithi181.stdout: order 22 (4 MiB objects) 2024-09-13T00:22:22.179 INFO:tasks.workunit.client.1.smithi181.stdout: snapshot_count: 1 2024-09-13T00:22:22.179 INFO:tasks.workunit.client.1.smithi181.stdout: id: d54971cfa613 2024-09-13T00:22:22.179 INFO:tasks.workunit.client.1.smithi181.stdout: block_name_prefix: rbd_data.d54971cfa613 2024-09-13T00:22:22.179 INFO:tasks.workunit.client.1.smithi181.stdout: format: 2 2024-09-13T00:22:22.179 INFO:tasks.workunit.client.1.smithi181.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2024-09-13T00:22:22.180 INFO:tasks.workunit.client.1.smithi181.stdout: op_features: 2024-09-13T00:22:22.180 INFO:tasks.workunit.client.1.smithi181.stdout: flags: 2024-09-13T00:22:22.180 INFO:tasks.workunit.client.1.smithi181.stdout: create_timestamp: Fri Sep 13 00:22:21 2024 2024-09-13T00:22:22.180 INFO:tasks.workunit.client.1.smithi181.stdout: access_timestamp: Fri Sep 13 00:22:21 2024 2024-09-13T00:22:22.180 INFO:tasks.workunit.client.1.smithi181.stdout: modify_timestamp: Fri Sep 13 00:22:21 2024 2024-09-13T00:22:22.183 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd info testimg_import@snap 2024-09-13T00:22:22.231 INFO:tasks.workunit.client.1.smithi181.stdout:rbd image 'testimg_import': 2024-09-13T00:22:22.231 INFO:tasks.workunit.client.1.smithi181.stdout: size 110 KiB in 1 objects 2024-09-13T00:22:22.231 INFO:tasks.workunit.client.1.smithi181.stdout: order 22 (4 MiB objects) 2024-09-13T00:22:22.231 INFO:tasks.workunit.client.1.smithi181.stdout: snapshot_count: 1 2024-09-13T00:22:22.231 INFO:tasks.workunit.client.1.smithi181.stdout: id: d54971cfa613 2024-09-13T00:22:22.231 INFO:tasks.workunit.client.1.smithi181.stdout: block_name_prefix: rbd_data.d54971cfa613 2024-09-13T00:22:22.231 INFO:tasks.workunit.client.1.smithi181.stdout: format: 2 2024-09-13T00:22:22.231 INFO:tasks.workunit.client.1.smithi181.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2024-09-13T00:22:22.231 INFO:tasks.workunit.client.1.smithi181.stdout: op_features: 2024-09-13T00:22:22.232 INFO:tasks.workunit.client.1.smithi181.stdout: flags: 2024-09-13T00:22:22.232 INFO:tasks.workunit.client.1.smithi181.stdout: create_timestamp: Fri Sep 13 00:22:21 2024 2024-09-13T00:22:22.232 INFO:tasks.workunit.client.1.smithi181.stdout: access_timestamp: Fri Sep 13 00:22:21 2024 2024-09-13T00:22:22.232 INFO:tasks.workunit.client.1.smithi181.stdout: modify_timestamp: Fri Sep 13 00:22:21 2024 2024-09-13T00:22:22.232 INFO:tasks.workunit.client.1.smithi181.stdout: protected: False 2024-09-13T00:22:22.234 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd image-meta list testimg_import 2024-09-13T00:22:22.288 INFO:tasks.workunit.client.1.smithi181.stderr:+ IMAGEMETA_AFTER=There is 1 metadatum on this image: 2024-09-13T00:22:22.289 INFO:tasks.workunit.client.1.smithi181.stderr: 2024-09-13T00:22:22.289 INFO:tasks.workunit.client.1.smithi181.stderr:Key Value 2024-09-13T00:22:22.289 INFO:tasks.workunit.client.1.smithi181.stderr:key1 value1 2024-09-13T00:22:22.289 INFO:tasks.workunit.client.1.smithi181.stderr:+ [ There is 1 metadatum on this image: 2024-09-13T00:22:22.289 INFO:tasks.workunit.client.1.smithi181.stderr: 2024-09-13T00:22:22.289 INFO:tasks.workunit.client.1.smithi181.stderr:Key Value 2024-09-13T00:22:22.289 INFO:tasks.workunit.client.1.smithi181.stderr:key1 value1 = There is 1 metadatum on this image: 2024-09-13T00:22:22.289 INFO:tasks.workunit.client.1.smithi181.stderr: 2024-09-13T00:22:22.289 INFO:tasks.workunit.client.1.smithi181.stderr:Key Value 2024-09-13T00:22:22.289 INFO:tasks.workunit.client.1.smithi181.stderr:key1 value1 ] 2024-09-13T00:22:22.289 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd export testimg_import /tmp/rbd_import_export_106423/img_import 2024-09-13T00:22:22.313 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:22 smithi181 bash[67587]: cluster 2024-09-13T00:22:21.057972+0000 mon.a (mon.0) 2302 : cluster [DBG] osdmap e1123: 8 total, 8 up, 8 in 2024-09-13T00:22:22.334 INFO:tasks.workunit.client.1.smithi181.stderr: Exporting image: 100% complete...done. 2024-09-13T00:22:22.337 INFO:tasks.workunit.client.1.smithi181.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_106423/img /tmp/rbd_import_export_106423/img_import 2024-09-13T00:22:22.337 INFO:tasks.workunit.client.1.smithi181.stderr:+ cmp -l /tmp/rbd_import_export_106423/img /tmp/rbd_import_export_106423/img_import 2024-09-13T00:22:22.338 INFO:tasks.workunit.client.1.smithi181.stderr:+ stat /tmp/rbd_import_export_106423/img --format %b 2024-09-13T00:22:22.339 INFO:tasks.workunit.client.1.smithi181.stderr:+ origsize=24 2024-09-13T00:22:22.339 INFO:tasks.workunit.client.1.smithi181.stderr:+ stat /tmp/rbd_import_export_106423/img_import --format %b 2024-09-13T00:22:22.339 INFO:tasks.workunit.client.1.smithi181.stderr:+ exportsize=224 2024-09-13T00:22:22.339 INFO:tasks.workunit.client.1.smithi181.stderr:+ difference=200 2024-09-13T00:22:22.340 INFO:tasks.workunit.client.1.smithi181.stderr:+ difference=200 2024-09-13T00:22:22.340 INFO:tasks.workunit.client.1.smithi181.stderr:+ test 200 -ge 0 -a 200 -lt 4096 2024-09-13T00:22:22.340 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd export testimg@snap /tmp/rbd_import_export_106423/img_snap 2024-09-13T00:22:22.383 INFO:tasks.workunit.client.1.smithi181.stderr: Exporting image: 100% complete...done. 2024-09-13T00:22:22.386 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd export testimg_import@snap /tmp/rbd_import_export_106423/img_snap_import 2024-09-13T00:22:22.429 INFO:tasks.workunit.client.1.smithi181.stderr: Exporting image: 100% complete...done. 2024-09-13T00:22:22.433 INFO:tasks.workunit.client.1.smithi181.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_106423/img_snap /tmp/rbd_import_export_106423/img_snap_import 2024-09-13T00:22:22.433 INFO:tasks.workunit.client.1.smithi181.stderr:+ cmp -l /tmp/rbd_import_export_106423/img_snap /tmp/rbd_import_export_106423/img_snap_import 2024-09-13T00:22:22.433 INFO:tasks.workunit.client.1.smithi181.stderr:+ stat /tmp/rbd_import_export_106423/img_snap --format %b 2024-09-13T00:22:22.434 INFO:tasks.workunit.client.1.smithi181.stderr:+ origsize=224 2024-09-13T00:22:22.435 INFO:tasks.workunit.client.1.smithi181.stderr:+ stat /tmp/rbd_import_export_106423/img_snap_import --format %b 2024-09-13T00:22:22.435 INFO:tasks.workunit.client.1.smithi181.stderr:+ exportsize=224 2024-09-13T00:22:22.435 INFO:tasks.workunit.client.1.smithi181.stderr:+ difference=0 2024-09-13T00:22:22.436 INFO:tasks.workunit.client.1.smithi181.stderr:+ difference=0 2024-09-13T00:22:22.436 INFO:tasks.workunit.client.1.smithi181.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2024-09-13T00:22:22.436 INFO:tasks.workunit.client.1.smithi181.stderr:+ rm /tmp/rbd_import_export_106423/img_v2 2024-09-13T00:22:22.436 INFO:tasks.workunit.client.1.smithi181.stderr:+ rm /tmp/rbd_import_export_106423/img_import 2024-09-13T00:22:22.436 INFO:tasks.workunit.client.1.smithi181.stderr:+ rm /tmp/rbd_import_export_106423/img_snap 2024-09-13T00:22:22.436 INFO:tasks.workunit.client.1.smithi181.stderr:+ rm /tmp/rbd_import_export_106423/img_snap_import 2024-09-13T00:22:22.437 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd snap rm testimg_import@snap 2024-09-13T00:22:22.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:22 smithi136 bash[75164]: cluster 2024-09-13T00:22:21.057972+0000 mon.a (mon.0) 2302 : cluster [DBG] osdmap e1123: 8 total, 8 up, 8 in 2024-09-13T00:22:22.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:22 smithi136 bash[76371]: cluster 2024-09-13T00:22:21.057972+0000 mon.a (mon.0) 2302 : cluster [DBG] osdmap e1123: 8 total, 8 up, 8 in 2024-09-13T00:22:22.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:22:22 smithi181 bash[95505]: ts=2024-09-13T00:22:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:22:23.097 INFO:tasks.workunit.client.1.smithi181.stderr: Removing snap: 100% complete...done. 2024-09-13T00:22:23.127 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd remove testimg_import 2024-09-13T00:22:23.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:23 smithi181 bash[67587]: cluster 2024-09-13T00:22:21.770674+0000 mgr.y (mgr.24661) 1357 : cluster [DBG] pgmap v2041: 105 pgs: 105 active+clean; 5.0 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 257 KiB/s rd, 97 KiB/s wr, 104 op/s 2024-09-13T00:22:23.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:23 smithi181 bash[67587]: cluster 2024-09-13T00:22:22.070966+0000 mon.a (mon.0) 2303 : cluster [DBG] osdmap e1124: 8 total, 8 up, 8 in 2024-09-13T00:22:23.378 INFO:tasks.workunit.client.1.smithi181.stderr: Removing image: 100% complete...done. 2024-09-13T00:22:23.386 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd snap rm testimg@snap 2024-09-13T00:22:23.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:23 smithi136 bash[76371]: cluster 2024-09-13T00:22:21.770674+0000 mgr.y (mgr.24661) 1357 : cluster [DBG] pgmap v2041: 105 pgs: 105 active+clean; 5.0 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 257 KiB/s rd, 97 KiB/s wr, 104 op/s 2024-09-13T00:22:23.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:23 smithi136 bash[76371]: cluster 2024-09-13T00:22:22.070966+0000 mon.a (mon.0) 2303 : cluster [DBG] osdmap e1124: 8 total, 8 up, 8 in 2024-09-13T00:22:23.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:23 smithi136 bash[75164]: cluster 2024-09-13T00:22:21.770674+0000 mgr.y (mgr.24661) 1357 : cluster [DBG] pgmap v2041: 105 pgs: 105 active+clean; 5.0 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 257 KiB/s rd, 97 KiB/s wr, 104 op/s 2024-09-13T00:22:23.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:23 smithi136 bash[75164]: cluster 2024-09-13T00:22:22.070966+0000 mon.a (mon.0) 2303 : cluster [DBG] osdmap e1124: 8 total, 8 up, 8 in 2024-09-13T00:22:24.112 INFO:tasks.workunit.client.1.smithi181.stderr: Removing snap: 100% complete...done. 2024-09-13T00:22:24.134 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd rm testimg 2024-09-13T00:22:24.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:24 smithi181 bash[67587]: cluster 2024-09-13T00:22:23.079788+0000 mon.a (mon.0) 2304 : cluster [DBG] osdmap e1125: 8 total, 8 up, 8 in 2024-09-13T00:22:24.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:24 smithi181 bash[67587]: audit 2024-09-13T00:22:23.806955+0000 mon.a (mon.0) 2305 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:22:24.374 INFO:tasks.workunit.client.1.smithi181.stderr: Removing image: 100% complete...done. 2024-09-13T00:22:24.381 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd import --order 20 /tmp/rbd_import_export_106423/img testimg 2024-09-13T00:22:24.500 INFO:tasks.workunit.client.1.smithi181.stderr: Importing image: 100% complete...done. 2024-09-13T00:22:24.500 INFO:tasks.workunit.client.1.smithi181.stderr:rbd: --order is deprecated, use --object-size 2024-09-13T00:22:24.504 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_106423/img_v2 2024-09-13T00:22:24.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:24 smithi136 bash[76371]: cluster 2024-09-13T00:22:23.079788+0000 mon.a (mon.0) 2304 : cluster [DBG] osdmap e1125: 8 total, 8 up, 8 in 2024-09-13T00:22:24.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:24 smithi136 bash[76371]: audit 2024-09-13T00:22:23.806955+0000 mon.a (mon.0) 2305 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:22:24.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:24 smithi136 bash[75164]: cluster 2024-09-13T00:22:23.079788+0000 mon.a (mon.0) 2304 : cluster [DBG] osdmap e1125: 8 total, 8 up, 8 in 2024-09-13T00:22:24.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:24 smithi136 bash[75164]: audit 2024-09-13T00:22:23.806955+0000 mon.a (mon.0) 2305 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:22:24.556 INFO:tasks.workunit.client.1.smithi181.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2024-09-13T00:22:24.560 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_106423/img_v2 testimg_import 2024-09-13T00:22:24.686 INFO:tasks.workunit.client.1.smithi181.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2024-09-13T00:22:24.691 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd info testimg_import 2024-09-13T00:22:24.691 INFO:tasks.workunit.client.1.smithi181.stderr:+ grep order 2024-09-13T00:22:24.691 INFO:tasks.workunit.client.1.smithi181.stderr:+ awk {print $2} 2024-09-13T00:22:24.692 INFO:tasks.workunit.client.1.smithi181.stderr:+ grep 20 2024-09-13T00:22:24.746 INFO:tasks.workunit.client.1.smithi181.stdout:20 2024-09-13T00:22:24.746 INFO:tasks.workunit.client.1.smithi181.stderr:+ rm /tmp/rbd_import_export_106423/img_v2 2024-09-13T00:22:24.747 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd remove testimg_import 2024-09-13T00:22:24.993 INFO:tasks.workunit.client.1.smithi181.stderr: Removing image: 100% complete...done. 2024-09-13T00:22:25.000 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd remove testimg 2024-09-13T00:22:25.235 INFO:tasks.workunit.client.1.smithi181.stderr: Removing image: 100% complete...done. 2024-09-13T00:22:25.242 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd import --image-feature layering /tmp/rbd_import_export_106423/img testimg 2024-09-13T00:22:25.325 INFO:tasks.workunit.client.1.smithi181.stderr: Importing image: 100% complete...done. 2024-09-13T00:22:25.329 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd info testimg 2024-09-13T00:22:25.329 INFO:tasks.workunit.client.1.smithi181.stderr:+ grep features 2024-09-13T00:22:25.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:25 smithi181 bash[67587]: cluster 2024-09-13T00:22:23.771473+0000 mgr.y (mgr.24661) 1358 : cluster [DBG] pgmap v2044: 105 pgs: 105 active+clean; 803 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 266 KiB/s rd, 84 KiB/s wr, 128 op/s 2024-09-13T00:22:25.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:25 smithi181 bash[67587]: cluster 2024-09-13T00:22:24.094416+0000 mon.a (mon.0) 2306 : cluster [DBG] osdmap e1126: 8 total, 8 up, 8 in 2024-09-13T00:22:25.380 INFO:tasks.workunit.client.1.smithi181.stderr:+ FEATURES_BEFORE= features: layering 2024-09-13T00:22:25.380 INFO:tasks.workunit.client.1.smithi181.stderr: op_features: 2024-09-13T00:22:25.380 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_106423/img_v2 2024-09-13T00:22:25.426 INFO:tasks.workunit.client.1.smithi181.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2024-09-13T00:22:25.429 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_106423/img_v2 testimg_import 2024-09-13T00:22:25.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:25 smithi136 bash[76371]: cluster 2024-09-13T00:22:23.771473+0000 mgr.y (mgr.24661) 1358 : cluster [DBG] pgmap v2044: 105 pgs: 105 active+clean; 803 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 266 KiB/s rd, 84 KiB/s wr, 128 op/s 2024-09-13T00:22:25.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:25 smithi136 bash[76371]: cluster 2024-09-13T00:22:24.094416+0000 mon.a (mon.0) 2306 : cluster [DBG] osdmap e1126: 8 total, 8 up, 8 in 2024-09-13T00:22:25.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:25 smithi136 bash[75164]: cluster 2024-09-13T00:22:23.771473+0000 mgr.y (mgr.24661) 1358 : cluster [DBG] pgmap v2044: 105 pgs: 105 active+clean; 803 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 266 KiB/s rd, 84 KiB/s wr, 128 op/s 2024-09-13T00:22:25.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:25 smithi136 bash[75164]: cluster 2024-09-13T00:22:24.094416+0000 mon.a (mon.0) 2306 : cluster [DBG] osdmap e1126: 8 total, 8 up, 8 in 2024-09-13T00:22:25.532 INFO:tasks.workunit.client.1.smithi181.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2024-09-13T00:22:25.539 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd info testimg_import 2024-09-13T00:22:25.539 INFO:tasks.workunit.client.1.smithi181.stderr:+ grep features 2024-09-13T00:22:25.589 INFO:tasks.workunit.client.1.smithi181.stderr:+ FEATURES_AFTER= features: layering 2024-09-13T00:22:25.589 INFO:tasks.workunit.client.1.smithi181.stderr: op_features: 2024-09-13T00:22:25.589 INFO:tasks.workunit.client.1.smithi181.stderr:+ [ features: layering 2024-09-13T00:22:25.589 INFO:tasks.workunit.client.1.smithi181.stderr: op_features: != features: layering 2024-09-13T00:22:25.589 INFO:tasks.workunit.client.1.smithi181.stderr: op_features: ] 2024-09-13T00:22:25.589 INFO:tasks.workunit.client.1.smithi181.stderr:+ rm /tmp/rbd_import_export_106423/img_v2 2024-09-13T00:22:25.590 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd remove testimg_import 2024-09-13T00:22:25.750 INFO:tasks.workunit.client.1.smithi181.stderr: Removing image: 100% complete...done. 2024-09-13T00:22:25.757 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd remove testimg 2024-09-13T00:22:25.926 INFO:tasks.workunit.client.1.smithi181.stderr: Removing image: 100% complete...done. 2024-09-13T00:22:25.933 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd import --stripe-count 1000 --stripe-unit 4096 /tmp/rbd_import_export_106423/img testimg 2024-09-13T00:22:26.063 INFO:tasks.workunit.client.1.smithi181.stderr: Importing image: 100% complete...done. 2024-09-13T00:22:26.068 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_106423/img_v2 2024-09-13T00:22:26.124 INFO:tasks.workunit.client.1.smithi181.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2024-09-13T00:22:26.128 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_106423/img_v2 testimg_import 2024-09-13T00:22:26.245 INFO:tasks.workunit.client.1.smithi181.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2024-09-13T00:22:26.252 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd info testimg_import 2024-09-13T00:22:26.252 INFO:tasks.workunit.client.1.smithi181.stderr:+ grep stripe unit 2024-09-13T00:22:26.252 INFO:tasks.workunit.client.1.smithi181.stderr:+ grep -Ei (4 KiB|4096) 2024-09-13T00:22:26.299 INFO:tasks.workunit.client.1.smithi181.stdout: stripe unit: 4 KiB 2024-09-13T00:22:26.300 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd info testimg_import 2024-09-13T00:22:26.300 INFO:tasks.workunit.client.1.smithi181.stderr:+ grep stripe count 2024-09-13T00:22:26.300 INFO:tasks.workunit.client.1.smithi181.stderr:+ awk {print $3} 2024-09-13T00:22:26.300 INFO:tasks.workunit.client.1.smithi181.stderr:+ grep 1000 2024-09-13T00:22:26.341 INFO:tasks.workunit.client.1.smithi181.stdout:1000 2024-09-13T00:22:26.341 INFO:tasks.workunit.client.1.smithi181.stderr:+ rm /tmp/rbd_import_export_106423/img_v2 2024-09-13T00:22:26.342 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd remove testimg_import 2024-09-13T00:22:26.537 INFO:tasks.workunit.client.1.smithi181.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-13T00:22:26.543 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd remove testimg 2024-09-13T00:22:26.796 INFO:tasks.workunit.client.1.smithi181.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-13T00:22:26.804 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd import --image-format=2 /tmp/rbd_import_export_106423/img testimg 2024-09-13T00:22:26.899 INFO:tasks.workunit.client.1.smithi181.stderr: Importing image: 100% complete...done. 2024-09-13T00:22:26.903 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd snap create testimg@snap1 2024-09-13T00:22:27.141 INFO:tasks.workunit.client.1.smithi181.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2024-09-13T00:22:27.161 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd snap create testimg@snap2 2024-09-13T00:22:27.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:27 smithi181 bash[67587]: cluster 2024-09-13T00:22:25.772413+0000 mgr.y (mgr.24661) 1359 : cluster [DBG] pgmap v2046: 105 pgs: 105 active+clean; 913 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 178 KiB/s rd, 19 KiB/s wr, 194 op/s 2024-09-13T00:22:27.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:27 smithi136 bash[76371]: cluster 2024-09-13T00:22:25.772413+0000 mgr.y (mgr.24661) 1359 : cluster [DBG] pgmap v2046: 105 pgs: 105 active+clean; 913 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 178 KiB/s rd, 19 KiB/s wr, 194 op/s 2024-09-13T00:22:27.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:27 smithi136 bash[75164]: cluster 2024-09-13T00:22:25.772413+0000 mgr.y (mgr.24661) 1359 : cluster [DBG] pgmap v2046: 105 pgs: 105 active+clean; 913 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 178 KiB/s rd, 19 KiB/s wr, 194 op/s 2024-09-13T00:22:28.155 INFO:tasks.workunit.client.1.smithi181.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2024-09-13T00:22:28.179 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd snap protect testimg@snap2 2024-09-13T00:22:28.234 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_106423/snap_protect 2024-09-13T00:22:28.281 INFO:tasks.workunit.client.1.smithi181.stderr: Exporting image: 33% complete... Exporting image: 100% complete... Exporting image: 100% complete...done. 2024-09-13T00:22:28.285 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_106423/snap_protect testimg_import 2024-09-13T00:22:28.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:28 smithi136 bash[76371]: cluster 2024-09-13T00:22:27.109736+0000 mon.a (mon.0) 2307 : cluster [DBG] osdmap e1127: 8 total, 8 up, 8 in 2024-09-13T00:22:28.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:28 smithi136 bash[75164]: cluster 2024-09-13T00:22:27.109736+0000 mon.a (mon.0) 2307 : cluster [DBG] osdmap e1127: 8 total, 8 up, 8 in 2024-09-13T00:22:28.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:28 smithi181 bash[67587]: cluster 2024-09-13T00:22:27.109736+0000 mon.a (mon.0) 2307 : cluster [DBG] osdmap e1127: 8 total, 8 up, 8 in 2024-09-13T00:22:29.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:29 smithi136 bash[75164]: cluster 2024-09-13T00:22:27.773308+0000 mgr.y (mgr.24661) 1360 : cluster [DBG] pgmap v2048: 105 pgs: 105 active+clean; 803 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 166 KiB/s rd, 16 KiB/s wr, 212 op/s 2024-09-13T00:22:29.452 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:29 smithi136 bash[75164]: cluster 2024-09-13T00:22:28.122479+0000 mon.a (mon.0) 2308 : cluster [DBG] osdmap e1128: 8 total, 8 up, 8 in 2024-09-13T00:22:29.452 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:29 smithi136 bash[75164]: cluster 2024-09-13T00:22:28.899189+0000 mon.a (mon.0) 2309 : cluster [DBG] osdmap e1129: 8 total, 8 up, 8 in 2024-09-13T00:22:29.452 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:29 smithi136 bash[76371]: cluster 2024-09-13T00:22:27.773308+0000 mgr.y (mgr.24661) 1360 : cluster [DBG] pgmap v2048: 105 pgs: 105 active+clean; 803 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 166 KiB/s rd, 16 KiB/s wr, 212 op/s 2024-09-13T00:22:29.453 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:29 smithi136 bash[76371]: cluster 2024-09-13T00:22:28.122479+0000 mon.a (mon.0) 2308 : cluster [DBG] osdmap e1128: 8 total, 8 up, 8 in 2024-09-13T00:22:29.453 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:29 smithi136 bash[76371]: cluster 2024-09-13T00:22:28.899189+0000 mon.a (mon.0) 2309 : cluster [DBG] osdmap e1129: 8 total, 8 up, 8 in 2024-09-13T00:22:29.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:29 smithi181 bash[67587]: cluster 2024-09-13T00:22:27.773308+0000 mgr.y (mgr.24661) 1360 : cluster [DBG] pgmap v2048: 105 pgs: 105 active+clean; 803 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 166 KiB/s rd, 16 KiB/s wr, 212 op/s 2024-09-13T00:22:29.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:29 smithi181 bash[67587]: cluster 2024-09-13T00:22:28.122479+0000 mon.a (mon.0) 2308 : cluster [DBG] osdmap e1128: 8 total, 8 up, 8 in 2024-09-13T00:22:29.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:29 smithi181 bash[67587]: cluster 2024-09-13T00:22:28.899189+0000 mon.a (mon.0) 2309 : cluster [DBG] osdmap e1129: 8 total, 8 up, 8 in 2024-09-13T00:22:29.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:22:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:22:29] "GET /metrics HTTP/1.1" 200 34123 "" "Prometheus/2.43.0" 2024-09-13T00:22:29.970 INFO:tasks.workunit.client.1.smithi181.stderr: Importing image: 33% complete... Importing image: 66% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2024-09-13T00:22:29.978 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd info testimg_import@snap1 2024-09-13T00:22:29.978 INFO:tasks.workunit.client.1.smithi181.stderr:+ grep protected: False 2024-09-13T00:22:30.037 INFO:tasks.workunit.client.1.smithi181.stdout: protected: False 2024-09-13T00:22:30.038 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd info testimg_import@snap2 2024-09-13T00:22:30.038 INFO:tasks.workunit.client.1.smithi181.stderr:+ grep protected: True 2024-09-13T00:22:30.088 INFO:tasks.workunit.client.1.smithi181.stdout: protected: True 2024-09-13T00:22:30.089 INFO:tasks.workunit.client.1.smithi181.stderr:+ rm /tmp/rbd_import_export_106423/snap_protect 2024-09-13T00:22:30.089 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd snap unprotect testimg@snap2 2024-09-13T00:22:30.143 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd snap unprotect testimg_import@snap2 2024-09-13T00:22:30.209 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd snap purge testimg 2024-09-13T00:22:31.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:30 smithi136 bash[75164]: cluster 2024-09-13T00:22:29.773998+0000 mgr.y (mgr.24661) 1361 : cluster [DBG] pgmap v2051: 105 pgs: 105 active+clean; 693 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 230 KiB/s rd, 27 KiB/s wr, 314 op/s 2024-09-13T00:22:31.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:30 smithi136 bash[75164]: cluster 2024-09-13T00:22:29.901972+0000 mon.a (mon.0) 2310 : cluster [DBG] osdmap e1130: 8 total, 8 up, 8 in 2024-09-13T00:22:31.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:30 smithi136 bash[76371]: cluster 2024-09-13T00:22:29.773998+0000 mgr.y (mgr.24661) 1361 : cluster [DBG] pgmap v2051: 105 pgs: 105 active+clean; 693 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 230 KiB/s rd, 27 KiB/s wr, 314 op/s 2024-09-13T00:22:31.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:30 smithi136 bash[76371]: cluster 2024-09-13T00:22:29.901972+0000 mon.a (mon.0) 2310 : cluster [DBG] osdmap e1130: 8 total, 8 up, 8 in 2024-09-13T00:22:31.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:30 smithi181 bash[67587]: cluster 2024-09-13T00:22:29.773998+0000 mgr.y (mgr.24661) 1361 : cluster [DBG] pgmap v2051: 105 pgs: 105 active+clean; 693 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 230 KiB/s rd, 27 KiB/s wr, 314 op/s 2024-09-13T00:22:31.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:30 smithi181 bash[67587]: cluster 2024-09-13T00:22:29.901972+0000 mon.a (mon.0) 2310 : cluster [DBG] osdmap e1130: 8 total, 8 up, 8 in 2024-09-13T00:22:32.153 INFO:tasks.workunit.client.1.smithi181.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2024-09-13T00:22:32.179 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd snap purge testimg_import 2024-09-13T00:22:32.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:32 smithi136 bash[75164]: cluster 2024-09-13T00:22:31.127907+0000 mon.a (mon.0) 2311 : cluster [DBG] osdmap e1131: 8 total, 8 up, 8 in 2024-09-13T00:22:32.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:32 smithi136 bash[76371]: cluster 2024-09-13T00:22:31.127907+0000 mon.a (mon.0) 2311 : cluster [DBG] osdmap e1131: 8 total, 8 up, 8 in 2024-09-13T00:22:32.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:32 smithi181 bash[67587]: cluster 2024-09-13T00:22:31.127907+0000 mon.a (mon.0) 2311 : cluster [DBG] osdmap e1131: 8 total, 8 up, 8 in 2024-09-13T00:22:32.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:22:32 smithi181 bash[95505]: ts=2024-09-13T00:22:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:22:33.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:33 smithi136 bash[75164]: cluster 2024-09-13T00:22:31.774847+0000 mgr.y (mgr.24661) 1362 : cluster [DBG] pgmap v2054: 105 pgs: 105 active+clean; 803 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 331 KiB/s rd, 39 KiB/s wr, 448 op/s 2024-09-13T00:22:33.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:33 smithi136 bash[75164]: cluster 2024-09-13T00:22:32.137969+0000 mon.a (mon.0) 2312 : cluster [DBG] osdmap e1132: 8 total, 8 up, 8 in 2024-09-13T00:22:33.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:33 smithi136 bash[76371]: cluster 2024-09-13T00:22:31.774847+0000 mgr.y (mgr.24661) 1362 : cluster [DBG] pgmap v2054: 105 pgs: 105 active+clean; 803 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 331 KiB/s rd, 39 KiB/s wr, 448 op/s 2024-09-13T00:22:33.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:33 smithi136 bash[76371]: cluster 2024-09-13T00:22:32.137969+0000 mon.a (mon.0) 2312 : cluster [DBG] osdmap e1132: 8 total, 8 up, 8 in 2024-09-13T00:22:33.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:33 smithi181 bash[67587]: cluster 2024-09-13T00:22:31.774847+0000 mgr.y (mgr.24661) 1362 : cluster [DBG] pgmap v2054: 105 pgs: 105 active+clean; 803 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 331 KiB/s rd, 39 KiB/s wr, 448 op/s 2024-09-13T00:22:33.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:33 smithi181 bash[67587]: cluster 2024-09-13T00:22:32.137969+0000 mon.a (mon.0) 2312 : cluster [DBG] osdmap e1132: 8 total, 8 up, 8 in 2024-09-13T00:22:33.920 INFO:tasks.workunit.client.1.smithi181.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2024-09-13T00:22:33.944 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd remove testimg 2024-09-13T00:22:34.174 INFO:tasks.workunit.client.1.smithi181.stderr: Removing image: 100% complete...done. 2024-09-13T00:22:34.180 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd remove testimg_import 2024-09-13T00:22:34.413 INFO:tasks.workunit.client.1.smithi181.stderr: Removing image: 100% complete...done. 2024-09-13T00:22:34.418 INFO:tasks.workunit.client.1.smithi181.stderr:+ tiered=0 2024-09-13T00:22:34.419 INFO:tasks.workunit.client.1.smithi181.stderr:+ + grep ^pool 2024-09-13T00:22:34.419 INFO:tasks.workunit.client.1.smithi181.stderr:ceph osd dump 2024-09-13T00:22:34.419 INFO:tasks.workunit.client.1.smithi181.stderr:+ grep 'rbd' 2024-09-13T00:22:34.419 INFO:tasks.workunit.client.1.smithi181.stderr:+ grep tier 2024-09-13T00:22:34.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:34 smithi136 bash[76371]: cluster 2024-09-13T00:22:33.146295+0000 mon.a (mon.0) 2313 : cluster [DBG] osdmap e1133: 8 total, 8 up, 8 in 2024-09-13T00:22:34.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:34 smithi136 bash[76371]: cluster 2024-09-13T00:22:33.901334+0000 mon.a (mon.0) 2314 : cluster [DBG] osdmap e1134: 8 total, 8 up, 8 in 2024-09-13T00:22:34.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:34 smithi136 bash[75164]: cluster 2024-09-13T00:22:33.146295+0000 mon.a (mon.0) 2313 : cluster [DBG] osdmap e1133: 8 total, 8 up, 8 in 2024-09-13T00:22:34.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:34 smithi136 bash[75164]: cluster 2024-09-13T00:22:33.901334+0000 mon.a (mon.0) 2314 : cluster [DBG] osdmap e1134: 8 total, 8 up, 8 in 2024-09-13T00:22:34.600 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:34 smithi181 bash[67587]: cluster 2024-09-13T00:22:33.146295+0000 mon.a (mon.0) 2313 : cluster [DBG] osdmap e1133: 8 total, 8 up, 8 in 2024-09-13T00:22:34.600 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:34 smithi181 bash[67587]: cluster 2024-09-13T00:22:33.901334+0000 mon.a (mon.0) 2314 : cluster [DBG] osdmap e1134: 8 total, 8 up, 8 in 2024-09-13T00:22:34.841 INFO:tasks.workunit.client.1.smithi181.stderr:+ dd if=/dev/urandom bs=1M seek=1 count=1 of=/tmp/rbd_import_export_106423/sparse1 2024-09-13T00:22:34.846 INFO:tasks.workunit.client.1.smithi181.stderr:1+0 records in 2024-09-13T00:22:34.847 INFO:tasks.workunit.client.1.smithi181.stderr:1+0 records out 2024-09-13T00:22:34.847 INFO:tasks.workunit.client.1.smithi181.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00510027 s, 206 MB/s 2024-09-13T00:22:34.847 INFO:tasks.workunit.client.1.smithi181.stderr:+ dd if=/dev/urandom bs=1M count=1 of=/tmp/rbd_import_export_106423/sparse2 2024-09-13T00:22:34.852 INFO:tasks.workunit.client.1.smithi181.stderr:1+0 records in 2024-09-13T00:22:34.852 INFO:tasks.workunit.client.1.smithi181.stderr:1+0 records out 2024-09-13T00:22:34.852 INFO:tasks.workunit.client.1.smithi181.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00512833 s, 204 MB/s 2024-09-13T00:22:34.853 INFO:tasks.workunit.client.1.smithi181.stderr:+ truncate /tmp/rbd_import_export_106423/sparse2 -s 2M 2024-09-13T00:22:34.864 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd rm sparse1 2024-09-13T00:22:34.944 INFO:tasks.workunit.client.1.smithi181.stderr: Removing image: 0% complete...failed. 2024-09-13T00:22:34.944 INFO:tasks.workunit.client.1.smithi181.stderr:rbd: delete error: (2) No such file or directory 2024-09-13T00:22:34.948 INFO:tasks.workunit.client.1.smithi181.stderr:+ true 2024-09-13T00:22:34.948 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_106423/sparse1 2024-09-13T00:22:35.056 INFO:tasks.workunit.client.1.smithi181.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2024-09-13T00:22:35.056 INFO:tasks.workunit.client.1.smithi181.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-09-13T00:22:35.057 INFO:tasks.workunit.client.1.smithi181.stderr:rbd: --order is deprecated, use --object-size 2024-09-13T00:22:35.060 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd ls -l 2024-09-13T00:22:35.060 INFO:tasks.workunit.client.1.smithi181.stderr:+ grep sparse1 2024-09-13T00:22:35.060 INFO:tasks.workunit.client.1.smithi181.stderr:+ grep -Ei (2 MiB|2048k) 2024-09-13T00:22:35.105 INFO:tasks.workunit.client.1.smithi181.stdout:sparse1 2 MiB 2 2024-09-13T00:22:35.106 INFO:tasks.workunit.client.1.smithi181.stderr:+ objects sparse1 2024-09-13T00:22:35.106 INFO:tasks.workunit.client.1.smithi181.stderr:+ image=sparse1 2024-09-13T00:22:35.106 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd info sparse1 2024-09-13T00:22:35.106 INFO:tasks.workunit.client.1.smithi181.stderr:+ grep block_name_prefix 2024-09-13T00:22:35.106 INFO:tasks.workunit.client.1.smithi181.stderr:+ awk {print $NF;} 2024-09-13T00:22:35.149 INFO:tasks.workunit.client.1.smithi181.stderr:+ prefix=rbd_data.d663daf40368 2024-09-13T00:22:35.151 INFO:tasks.workunit.client.1.smithi181.stderr:+ grep rbd_data.d663daf40368 2024-09-13T00:22:35.151 INFO:tasks.workunit.client.1.smithi181.stderr:+ get_image_data_pool sparse1 2024-09-13T00:22:35.151 INFO:tasks.workunit.client.1.smithi181.stderr:+ image=sparse1 2024-09-13T00:22:35.151 INFO:tasks.workunit.client.1.smithi181.stderr:+ sed -e s/rbd_data.d663daf40368\.// -e s/^0*\([0-9a-f]\)/\1/ 2024-09-13T00:22:35.151 INFO:tasks.workunit.client.1.smithi181.stderr:+ sort -u 2024-09-13T00:22:35.151 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd info sparse1 2024-09-13T00:22:35.151 INFO:tasks.workunit.client.1.smithi181.stderr:+ grep data_pool: 2024-09-13T00:22:35.151 INFO:tasks.workunit.client.1.smithi181.stderr:+ awk -F: { print $NF } 2024-09-13T00:22:35.193 INFO:tasks.workunit.client.1.smithi181.stderr:+ data_pool= 2024-09-13T00:22:35.194 INFO:tasks.workunit.client.1.smithi181.stderr:+ [ -z ] 2024-09-13T00:22:35.194 INFO:tasks.workunit.client.1.smithi181.stderr:+ data_pool=rbd 2024-09-13T00:22:35.194 INFO:tasks.workunit.client.1.smithi181.stderr:+ echo rbd 2024-09-13T00:22:35.194 INFO:tasks.workunit.client.1.smithi181.stderr:+ rados ls -p rbd 2024-09-13T00:22:35.245 INFO:tasks.workunit.client.1.smithi181.stderr:+ objects=1 2024-09-13T00:22:35.245 INFO:tasks.workunit.client.1.smithi181.stderr:+ echo 1 2024-09-13T00:22:35.245 INFO:tasks.workunit.client.1.smithi181.stderr:+ [ 0 -eq 1 -o 1 = 1 ] 2024-09-13T00:22:35.245 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd export sparse1 /tmp/rbd_import_export_106423/sparse1.out 2024-09-13T00:22:35.287 INFO:tasks.workunit.client.1.smithi181.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2024-09-13T00:22:35.291 INFO:tasks.workunit.client.1.smithi181.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_106423/sparse1 /tmp/rbd_import_export_106423/sparse1.out 2024-09-13T00:22:35.291 INFO:tasks.workunit.client.1.smithi181.stderr:+ cmp -l /tmp/rbd_import_export_106423/sparse1 /tmp/rbd_import_export_106423/sparse1.out 2024-09-13T00:22:35.293 INFO:tasks.workunit.client.1.smithi181.stderr:+ stat /tmp/rbd_import_export_106423/sparse1 --format %b 2024-09-13T00:22:35.293 INFO:tasks.workunit.client.1.smithi181.stderr:+ origsize=2048 2024-09-13T00:22:35.293 INFO:tasks.workunit.client.1.smithi181.stderr:+ stat /tmp/rbd_import_export_106423/sparse1.out --format %b 2024-09-13T00:22:35.294 INFO:tasks.workunit.client.1.smithi181.stderr:+ exportsize=2048 2024-09-13T00:22:35.294 INFO:tasks.workunit.client.1.smithi181.stderr:+ difference=0 2024-09-13T00:22:35.294 INFO:tasks.workunit.client.1.smithi181.stderr:+ difference=0 2024-09-13T00:22:35.294 INFO:tasks.workunit.client.1.smithi181.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2024-09-13T00:22:35.294 INFO:tasks.workunit.client.1.smithi181.stderr:+ rm /tmp/rbd_import_export_106423/sparse1.out 2024-09-13T00:22:35.295 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd rm sparse1 2024-09-13T00:22:35.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:35 smithi136 bash[76371]: cluster 2024-09-13T00:22:33.775631+0000 mgr.y (mgr.24661) 1363 : cluster [DBG] pgmap v2057: 105 pgs: 105 active+clean; 803 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 231 KiB/s rd, 20 KiB/s wr, 294 op/s 2024-09-13T00:22:35.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:35 smithi136 bash[76371]: audit 2024-09-13T00:22:34.719422+0000 mon.b (mon.1) 14 : audit [DBG] from='client.? 172.21.15.181:0/3880123344' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-09-13T00:22:35.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:35 smithi136 bash[76371]: audit 2024-09-13T00:22:34.823051+0000 mon.b (mon.1) 15 : audit [DBG] from='client.? 172.21.15.181:0/3880123344' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-09-13T00:22:35.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:35 smithi136 bash[75164]: cluster 2024-09-13T00:22:33.775631+0000 mgr.y (mgr.24661) 1363 : cluster [DBG] pgmap v2057: 105 pgs: 105 active+clean; 803 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 231 KiB/s rd, 20 KiB/s wr, 294 op/s 2024-09-13T00:22:35.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:35 smithi136 bash[75164]: audit 2024-09-13T00:22:34.719422+0000 mon.b (mon.1) 14 : audit [DBG] from='client.? 172.21.15.181:0/3880123344' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-09-13T00:22:35.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:35 smithi136 bash[75164]: audit 2024-09-13T00:22:34.823051+0000 mon.b (mon.1) 15 : audit [DBG] from='client.? 172.21.15.181:0/3880123344' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-09-13T00:22:35.512 INFO:tasks.workunit.client.1.smithi181.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2024-09-13T00:22:35.519 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd rm sparse2 2024-09-13T00:22:35.594 INFO:tasks.workunit.client.1.smithi181.stderr: Removing image: 0% complete...failed. 2024-09-13T00:22:35.594 INFO:tasks.workunit.client.1.smithi181.stderr:rbd: delete error: (2) No such file or directory 2024-09-13T00:22:35.597 INFO:tasks.workunit.client.1.smithi181.stderr:+ true 2024-09-13T00:22:35.597 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_106423/sparse2 2024-09-13T00:22:35.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:35 smithi181 bash[67587]: cluster 2024-09-13T00:22:33.775631+0000 mgr.y (mgr.24661) 1363 : cluster [DBG] pgmap v2057: 105 pgs: 105 active+clean; 803 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 231 KiB/s rd, 20 KiB/s wr, 294 op/s 2024-09-13T00:22:35.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:35 smithi181 bash[67587]: audit 2024-09-13T00:22:34.719422+0000 mon.b (mon.1) 14 : audit [DBG] from='client.? 172.21.15.181:0/3880123344' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-09-13T00:22:35.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:35 smithi181 bash[67587]: audit 2024-09-13T00:22:34.823051+0000 mon.b (mon.1) 15 : audit [DBG] from='client.? 172.21.15.181:0/3880123344' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-09-13T00:22:35.685 INFO:tasks.workunit.client.1.smithi181.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2024-09-13T00:22:35.686 INFO:tasks.workunit.client.1.smithi181.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-09-13T00:22:35.686 INFO:tasks.workunit.client.1.smithi181.stderr:rbd: --order is deprecated, use --object-size 2024-09-13T00:22:35.689 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd ls -l 2024-09-13T00:22:35.690 INFO:tasks.workunit.client.1.smithi181.stderr:+ grep sparse2 2024-09-13T00:22:35.690 INFO:tasks.workunit.client.1.smithi181.stderr:+ grep -Ei (2 MiB|2048k) 2024-09-13T00:22:35.727 INFO:tasks.workunit.client.1.smithi181.stdout:sparse2 2 MiB 2 2024-09-13T00:22:35.728 INFO:tasks.workunit.client.1.smithi181.stderr:+ objects sparse2 2024-09-13T00:22:35.728 INFO:tasks.workunit.client.1.smithi181.stderr:+ image=sparse2 2024-09-13T00:22:35.728 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd info sparse2 2024-09-13T00:22:35.729 INFO:tasks.workunit.client.1.smithi181.stderr:+ grep block_name_prefix 2024-09-13T00:22:35.729 INFO:tasks.workunit.client.1.smithi181.stderr:+ awk {print $NF;} 2024-09-13T00:22:35.767 INFO:tasks.workunit.client.1.smithi181.stderr:+ prefix=rbd_data.d6968871c4da 2024-09-13T00:22:35.767 INFO:tasks.workunit.client.1.smithi181.stderr:+ grep rbd_data.d6968871c4da 2024-09-13T00:22:35.767 INFO:tasks.workunit.client.1.smithi181.stderr:+ get_image_data_pool sparse2 2024-09-13T00:22:35.767 INFO:tasks.workunit.client.1.smithi181.stderr:+ image=sparse2 2024-09-13T00:22:35.767 INFO:tasks.workunit.client.1.smithi181.stderr:+ sed -e s/rbd_data.d6968871c4da\.// -e s/^0*\([0-9a-f]\)/\1/ 2024-09-13T00:22:35.767 INFO:tasks.workunit.client.1.smithi181.stderr:+ sort -u 2024-09-13T00:22:35.767 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd info sparse2 2024-09-13T00:22:35.767 INFO:tasks.workunit.client.1.smithi181.stderr:+ grep data_pool: 2024-09-13T00:22:35.767 INFO:tasks.workunit.client.1.smithi181.stderr:+ awk -F: { print $NF } 2024-09-13T00:22:35.804 INFO:tasks.workunit.client.1.smithi181.stderr:+ data_pool= 2024-09-13T00:22:35.804 INFO:tasks.workunit.client.1.smithi181.stderr:+ [ -z ] 2024-09-13T00:22:35.804 INFO:tasks.workunit.client.1.smithi181.stderr:+ data_pool=rbd 2024-09-13T00:22:35.804 INFO:tasks.workunit.client.1.smithi181.stderr:+ echo rbd 2024-09-13T00:22:35.804 INFO:tasks.workunit.client.1.smithi181.stderr:+ rados ls -p rbd 2024-09-13T00:22:35.855 INFO:tasks.workunit.client.1.smithi181.stderr:+ objects=0 2024-09-13T00:22:35.856 INFO:tasks.workunit.client.1.smithi181.stderr:+ echo 0 2024-09-13T00:22:35.856 INFO:tasks.workunit.client.1.smithi181.stderr:+ [ 0 -eq 1 -o 0 = 0 ] 2024-09-13T00:22:35.856 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd export sparse2 /tmp/rbd_import_export_106423/sparse2.out 2024-09-13T00:22:35.893 INFO:tasks.workunit.client.1.smithi181.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2024-09-13T00:22:35.897 INFO:tasks.workunit.client.1.smithi181.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_106423/sparse2 /tmp/rbd_import_export_106423/sparse2.out 2024-09-13T00:22:35.897 INFO:tasks.workunit.client.1.smithi181.stderr:+ cmp -l /tmp/rbd_import_export_106423/sparse2 /tmp/rbd_import_export_106423/sparse2.out 2024-09-13T00:22:35.898 INFO:tasks.workunit.client.1.smithi181.stderr:+ stat /tmp/rbd_import_export_106423/sparse2 --format %b 2024-09-13T00:22:35.899 INFO:tasks.workunit.client.1.smithi181.stderr:+ origsize=2048 2024-09-13T00:22:35.899 INFO:tasks.workunit.client.1.smithi181.stderr:+ stat /tmp/rbd_import_export_106423/sparse2.out --format %b 2024-09-13T00:22:35.900 INFO:tasks.workunit.client.1.smithi181.stderr:+ exportsize=2048 2024-09-13T00:22:35.900 INFO:tasks.workunit.client.1.smithi181.stderr:+ difference=0 2024-09-13T00:22:35.900 INFO:tasks.workunit.client.1.smithi181.stderr:+ difference=0 2024-09-13T00:22:35.900 INFO:tasks.workunit.client.1.smithi181.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2024-09-13T00:22:35.900 INFO:tasks.workunit.client.1.smithi181.stderr:+ rm /tmp/rbd_import_export_106423/sparse2.out 2024-09-13T00:22:35.900 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd rm sparse2 2024-09-13T00:22:36.065 INFO:tasks.workunit.client.1.smithi181.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2024-09-13T00:22:36.071 INFO:tasks.workunit.client.1.smithi181.stderr:+ truncate /tmp/rbd_import_export_106423/sparse1 -s 10M 2024-09-13T00:22:36.072 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd import --new-format --order 20 - sparse1 2024-09-13T00:22:36.235 INFO:tasks.workunit.client.1.smithi181.stderr: Importing image: 100% complete...done. 2024-09-13T00:22:36.236 INFO:tasks.workunit.client.1.smithi181.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-09-13T00:22:36.236 INFO:tasks.workunit.client.1.smithi181.stderr:rbd: --order is deprecated, use --object-size 2024-09-13T00:22:36.241 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd ls -l 2024-09-13T00:22:36.241 INFO:tasks.workunit.client.1.smithi181.stderr:+ grep sparse1 2024-09-13T00:22:36.242 INFO:tasks.workunit.client.1.smithi181.stderr:+ grep -Ei (10 MiB|10240k) 2024-09-13T00:22:36.283 INFO:tasks.workunit.client.1.smithi181.stdout:sparse1 10 MiB 2 2024-09-13T00:22:36.284 INFO:tasks.workunit.client.1.smithi181.stderr:+ objects sparse1 2024-09-13T00:22:36.284 INFO:tasks.workunit.client.1.smithi181.stderr:+ image=sparse1 2024-09-13T00:22:36.284 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd info sparse1 2024-09-13T00:22:36.284 INFO:tasks.workunit.client.1.smithi181.stderr:+ grep block_name_prefix 2024-09-13T00:22:36.284 INFO:tasks.workunit.client.1.smithi181.stderr:+ awk {print $NF;} 2024-09-13T00:22:36.326 INFO:tasks.workunit.client.1.smithi181.stderr:+ prefix=rbd_data.d6c3f4acf7ea 2024-09-13T00:22:36.326 INFO:tasks.workunit.client.1.smithi181.stderr:+ grep rbd_data.d6c3f4acf7ea 2024-09-13T00:22:36.326 INFO:tasks.workunit.client.1.smithi181.stderr:+ + get_image_data_pool sparse1 2024-09-13T00:22:36.326 INFO:tasks.workunit.client.1.smithi181.stderr:sed -e s/rbd_data.d6c3f4acf7ea\.// -e s/^0*\([0-9a-f]\)/\1/ 2024-09-13T00:22:36.326 INFO:tasks.workunit.client.1.smithi181.stderr:+ image=sparse1 2024-09-13T00:22:36.326 INFO:tasks.workunit.client.1.smithi181.stderr:+ sort -u 2024-09-13T00:22:36.326 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd info sparse1 2024-09-13T00:22:36.326 INFO:tasks.workunit.client.1.smithi181.stderr:+ grep data_pool: 2024-09-13T00:22:36.327 INFO:tasks.workunit.client.1.smithi181.stderr:+ awk -F: { print $NF } 2024-09-13T00:22:36.366 INFO:tasks.workunit.client.1.smithi181.stderr:+ data_pool= 2024-09-13T00:22:36.366 INFO:tasks.workunit.client.1.smithi181.stderr:+ [ -z ] 2024-09-13T00:22:36.366 INFO:tasks.workunit.client.1.smithi181.stderr:+ data_pool=rbd 2024-09-13T00:22:36.366 INFO:tasks.workunit.client.1.smithi181.stderr:+ echo rbd 2024-09-13T00:22:36.366 INFO:tasks.workunit.client.1.smithi181.stderr:+ rados ls -p rbd 2024-09-13T00:22:36.420 INFO:tasks.workunit.client.1.smithi181.stderr:+ objects=1 2024-09-13T00:22:36.420 INFO:tasks.workunit.client.1.smithi181.stderr:+ echo 1 2024-09-13T00:22:36.421 INFO:tasks.workunit.client.1.smithi181.stderr:+ [ 0 -eq 1 -o 1 = 1 ] 2024-09-13T00:22:36.421 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd export sparse1 /tmp/rbd_import_export_106423/sparse1.out 2024-09-13T00:22:36.463 INFO:tasks.workunit.client.1.smithi181.stderr: Exporting image: 10% complete... Exporting image: 20% complete... Exporting image: 30% complete... Exporting image: 40% complete... Exporting image: 50% complete... Exporting image: 60% complete... Exporting image: 70% complete... Exporting image: 80% complete... Exporting image: 90% complete... Exporting image: 100% complete...done. 2024-09-13T00:22:36.467 INFO:tasks.workunit.client.1.smithi181.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_106423/sparse1 /tmp/rbd_import_export_106423/sparse1.out 2024-09-13T00:22:36.467 INFO:tasks.workunit.client.1.smithi181.stderr:+ cmp -l /tmp/rbd_import_export_106423/sparse1 /tmp/rbd_import_export_106423/sparse1.out 2024-09-13T00:22:36.475 INFO:tasks.workunit.client.1.smithi181.stderr:+ stat /tmp/rbd_import_export_106423/sparse1 --format %b 2024-09-13T00:22:36.476 INFO:tasks.workunit.client.1.smithi181.stderr:+ origsize=2048 2024-09-13T00:22:36.476 INFO:tasks.workunit.client.1.smithi181.stderr:+ stat /tmp/rbd_import_export_106423/sparse1.out --format %b 2024-09-13T00:22:36.477 INFO:tasks.workunit.client.1.smithi181.stderr:+ exportsize=2048 2024-09-13T00:22:36.477 INFO:tasks.workunit.client.1.smithi181.stderr:+ difference=0 2024-09-13T00:22:36.477 INFO:tasks.workunit.client.1.smithi181.stderr:+ difference=0 2024-09-13T00:22:36.477 INFO:tasks.workunit.client.1.smithi181.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2024-09-13T00:22:36.477 INFO:tasks.workunit.client.1.smithi181.stderr:+ rm /tmp/rbd_import_export_106423/sparse1.out 2024-09-13T00:22:36.478 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd rm sparse1 2024-09-13T00:22:36.664 INFO:tasks.workunit.client.1.smithi181.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2024-09-13T00:22:36.670 INFO:tasks.workunit.client.1.smithi181.stderr:+ dd if=/dev/urandom bs=2M count=1 of=/tmp/rbd_import_export_106423/sparse2 oflag=append conv=notrunc 2024-09-13T00:22:36.678 INFO:tasks.workunit.client.1.smithi181.stderr:1+0 records in 2024-09-13T00:22:36.678 INFO:tasks.workunit.client.1.smithi181.stderr:1+0 records out 2024-09-13T00:22:36.678 INFO:tasks.workunit.client.1.smithi181.stderr:2097152 bytes (2.1 MB, 2.0 MiB) copied, 0.00767926 s, 273 MB/s 2024-09-13T00:22:36.678 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd import --new-format --order 20 - sparse2 2024-09-13T00:22:36.861 INFO:tasks.workunit.client.1.smithi181.stderr: Importing image: 100% complete...done. 2024-09-13T00:22:36.863 INFO:tasks.workunit.client.1.smithi181.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-09-13T00:22:36.863 INFO:tasks.workunit.client.1.smithi181.stderr:rbd: --order is deprecated, use --object-size 2024-09-13T00:22:36.868 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd ls -l 2024-09-13T00:22:36.869 INFO:tasks.workunit.client.1.smithi181.stderr:+ + grep -Ei (4 MiB|4096k) 2024-09-13T00:22:36.869 INFO:tasks.workunit.client.1.smithi181.stderr:grep sparse2 2024-09-13T00:22:36.923 INFO:tasks.workunit.client.1.smithi181.stdout:sparse2 4 MiB 2 2024-09-13T00:22:36.924 INFO:tasks.workunit.client.1.smithi181.stderr:+ objects sparse2 2024-09-13T00:22:36.924 INFO:tasks.workunit.client.1.smithi181.stderr:+ image=sparse2 2024-09-13T00:22:36.924 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd info sparse2 2024-09-13T00:22:36.925 INFO:tasks.workunit.client.1.smithi181.stderr:+ grep block_name_prefix 2024-09-13T00:22:36.925 INFO:tasks.workunit.client.1.smithi181.stderr:+ awk {print $NF;} 2024-09-13T00:22:36.971 INFO:tasks.workunit.client.1.smithi181.stderr:+ prefix=rbd_data.d6f0744c9fa5 2024-09-13T00:22:36.971 INFO:tasks.workunit.client.1.smithi181.stderr:+ grep rbd_data.d6f0744c9fa5 2024-09-13T00:22:36.971 INFO:tasks.workunit.client.1.smithi181.stderr:+ + sed -e s/rbd_data.d6f0744c9fa5\.// -e s/^0*\([0-9a-f]\)/\1/ 2024-09-13T00:22:36.971 INFO:tasks.workunit.client.1.smithi181.stderr:get_image_data_pool sparse2 2024-09-13T00:22:36.971 INFO:tasks.workunit.client.1.smithi181.stderr:+ image=sparse2 2024-09-13T00:22:36.971 INFO:tasks.workunit.client.1.smithi181.stderr:+ sort -u 2024-09-13T00:22:36.971 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd info sparse2 2024-09-13T00:22:36.971 INFO:tasks.workunit.client.1.smithi181.stderr:+ grep data_pool: 2024-09-13T00:22:36.972 INFO:tasks.workunit.client.1.smithi181.stderr:+ awk -F: { print $NF } 2024-09-13T00:22:37.018 INFO:tasks.workunit.client.1.smithi181.stderr:+ data_pool= 2024-09-13T00:22:37.018 INFO:tasks.workunit.client.1.smithi181.stderr:+ [ -z ] 2024-09-13T00:22:37.018 INFO:tasks.workunit.client.1.smithi181.stderr:+ data_pool=rbd 2024-09-13T00:22:37.018 INFO:tasks.workunit.client.1.smithi181.stderr:+ echo rbd 2024-09-13T00:22:37.018 INFO:tasks.workunit.client.1.smithi181.stderr:+ rados ls -p rbd 2024-09-13T00:22:37.073 INFO:tasks.workunit.client.1.smithi181.stderr:+ objects=0 2024-09-13T00:22:37.073 INFO:tasks.workunit.client.1.smithi181.stderr:2 2024-09-13T00:22:37.073 INFO:tasks.workunit.client.1.smithi181.stderr:3 2024-09-13T00:22:37.073 INFO:tasks.workunit.client.1.smithi181.stderr:+ echo 0 2 3 2024-09-13T00:22:37.073 INFO:tasks.workunit.client.1.smithi181.stderr:+ [ 0 -eq 1 -o 0 2 3 = 0 2 3 ] 2024-09-13T00:22:37.073 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd export sparse2 /tmp/rbd_import_export_106423/sparse2.out 2024-09-13T00:22:37.120 INFO:tasks.workunit.client.1.smithi181.stderr: Exporting image: 25% complete... Exporting image: 50% complete... Exporting image: 75% complete... Exporting image: 100% complete...done. 2024-09-13T00:22:37.124 INFO:tasks.workunit.client.1.smithi181.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_106423/sparse2 /tmp/rbd_import_export_106423/sparse2.out 2024-09-13T00:22:37.125 INFO:tasks.workunit.client.1.smithi181.stderr:+ cmp -l /tmp/rbd_import_export_106423/sparse2 /tmp/rbd_import_export_106423/sparse2.out 2024-09-13T00:22:37.128 INFO:tasks.workunit.client.1.smithi181.stderr:+ stat /tmp/rbd_import_export_106423/sparse2 --format %b 2024-09-13T00:22:37.129 INFO:tasks.workunit.client.1.smithi181.stderr:+ origsize=6144 2024-09-13T00:22:37.129 INFO:tasks.workunit.client.1.smithi181.stderr:+ stat /tmp/rbd_import_export_106423/sparse2.out --format %b 2024-09-13T00:22:37.130 INFO:tasks.workunit.client.1.smithi181.stderr:+ exportsize=6144 2024-09-13T00:22:37.130 INFO:tasks.workunit.client.1.smithi181.stderr:+ difference=0 2024-09-13T00:22:37.130 INFO:tasks.workunit.client.1.smithi181.stderr:+ difference=0 2024-09-13T00:22:37.130 INFO:tasks.workunit.client.1.smithi181.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2024-09-13T00:22:37.130 INFO:tasks.workunit.client.1.smithi181.stderr:+ rm /tmp/rbd_import_export_106423/sparse2.out 2024-09-13T00:22:37.130 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd rm sparse2 2024-09-13T00:22:37.381 INFO:tasks.workunit.client.1.smithi181.stderr: Removing image: 25% complete... Removing image: 50% complete... Removing image: 75% complete... Removing image: 100% complete...done. 2024-09-13T00:22:37.389 INFO:tasks.workunit.client.1.smithi181.stdout:partially-sparse file imports to partially-sparse image 2024-09-13T00:22:37.389 INFO:tasks.workunit.client.1.smithi181.stderr:+ echo partially-sparse file imports to partially-sparse image 2024-09-13T00:22:37.389 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_106423/sparse1 sparse 2024-09-13T00:22:37.504 INFO:tasks.workunit.client.1.smithi181.stderr: Importing image: 10% complete... Importing image: 20% complete... Importing image: 30% complete... Importing image: 40% complete... Importing image: 50% complete... Importing image: 60% complete... Importing image: 70% complete... Importing image: 80% complete... Importing image: 90% complete... Importing image: 100% complete...done. 2024-09-13T00:22:37.505 INFO:tasks.workunit.client.1.smithi181.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-09-13T00:22:37.505 INFO:tasks.workunit.client.1.smithi181.stderr:rbd: --order is deprecated, use --object-size 2024-09-13T00:22:37.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:37 smithi136 bash[76371]: cluster 2024-09-13T00:22:35.776548+0000 mgr.y (mgr.24661) 1364 : cluster [DBG] pgmap v2059: 105 pgs: 105 active+clean; 803 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 63 KiB/s rd, 6.7 KiB/s wr, 87 op/s 2024-09-13T00:22:37.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:37 smithi136 bash[75164]: cluster 2024-09-13T00:22:35.776548+0000 mgr.y (mgr.24661) 1364 : cluster [DBG] pgmap v2059: 105 pgs: 105 active+clean; 803 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 63 KiB/s rd, 6.7 KiB/s wr, 87 op/s 2024-09-13T00:22:37.510 INFO:tasks.workunit.client.1.smithi181.stderr:+ objects sparse 2024-09-13T00:22:37.510 INFO:tasks.workunit.client.1.smithi181.stderr:+ image=sparse 2024-09-13T00:22:37.510 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd info sparse 2024-09-13T00:22:37.510 INFO:tasks.workunit.client.1.smithi181.stderr:+ grep block_name_prefix 2024-09-13T00:22:37.510 INFO:tasks.workunit.client.1.smithi181.stderr:+ awk {print $NF;} 2024-09-13T00:22:37.553 INFO:tasks.workunit.client.1.smithi181.stderr:+ prefix=rbd_data.d71daec96ad5 2024-09-13T00:22:37.554 INFO:tasks.workunit.client.1.smithi181.stderr:+ grep rbd_data.d71daec96ad5 2024-09-13T00:22:37.554 INFO:tasks.workunit.client.1.smithi181.stderr:+ + get_image_data_pool sparse 2024-09-13T00:22:37.555 INFO:tasks.workunit.client.1.smithi181.stderr:sed -e s/rbd_data.d71daec96ad5\.// -e s/^0*\([0-9a-f]\)/\1/ 2024-09-13T00:22:37.555 INFO:tasks.workunit.client.1.smithi181.stderr:+ image=sparse 2024-09-13T00:22:37.555 INFO:tasks.workunit.client.1.smithi181.stderr:+ sort -u 2024-09-13T00:22:37.555 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd info sparse 2024-09-13T00:22:37.555 INFO:tasks.workunit.client.1.smithi181.stderr:+ grep data_pool: 2024-09-13T00:22:37.555 INFO:tasks.workunit.client.1.smithi181.stderr:+ awk -F: { print $NF } 2024-09-13T00:22:37.596 INFO:tasks.workunit.client.1.smithi181.stderr:+ data_pool= 2024-09-13T00:22:37.596 INFO:tasks.workunit.client.1.smithi181.stderr:+ [ -z ] 2024-09-13T00:22:37.596 INFO:tasks.workunit.client.1.smithi181.stderr:+ data_pool=rbd 2024-09-13T00:22:37.596 INFO:tasks.workunit.client.1.smithi181.stderr:+ echo rbd 2024-09-13T00:22:37.596 INFO:tasks.workunit.client.1.smithi181.stderr:+ rados ls -p rbd 2024-09-13T00:22:37.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:37 smithi181 bash[67587]: cluster 2024-09-13T00:22:35.776548+0000 mgr.y (mgr.24661) 1364 : cluster [DBG] pgmap v2059: 105 pgs: 105 active+clean; 803 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 63 KiB/s rd, 6.7 KiB/s wr, 87 op/s 2024-09-13T00:22:37.649 INFO:tasks.workunit.client.1.smithi181.stderr:+ objects=1 2024-09-13T00:22:37.649 INFO:tasks.workunit.client.1.smithi181.stderr:+ echo 1 2024-09-13T00:22:37.649 INFO:tasks.workunit.client.1.smithi181.stderr:+ [ 0 -eq 1 -o 1 = 1 ] 2024-09-13T00:22:37.649 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd rm sparse 2024-09-13T00:22:37.847 INFO:tasks.workunit.client.1.smithi181.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2024-09-13T00:22:37.853 INFO:tasks.workunit.client.1.smithi181.stderr:+ echo zeros import through stdin to sparse image 2024-09-13T00:22:37.853 INFO:tasks.workunit.client.1.smithi181.stdout:zeros import through stdin to sparse image 2024-09-13T00:22:37.853 INFO:tasks.workunit.client.1.smithi181.stderr:+ dd if=/dev/zero bs=1M count=4 2024-09-13T00:22:37.854 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd import --new-format - sparse 2024-09-13T00:22:37.920 INFO:tasks.workunit.client.1.smithi181.stderr:4+0 records in 2024-09-13T00:22:37.921 INFO:tasks.workunit.client.1.smithi181.stderr:4+0 records out 2024-09-13T00:22:37.921 INFO:tasks.workunit.client.1.smithi181.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0667281 s, 62.9 MB/s 2024-09-13T00:22:37.938 INFO:tasks.workunit.client.1.smithi181.stderr: Importing image: 100% complete...done. 2024-09-13T00:22:37.938 INFO:tasks.workunit.client.1.smithi181.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-09-13T00:22:37.942 INFO:tasks.workunit.client.1.smithi181.stderr:+ objects sparse 2024-09-13T00:22:37.943 INFO:tasks.workunit.client.1.smithi181.stderr:+ image=sparse 2024-09-13T00:22:37.943 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd info sparse 2024-09-13T00:22:37.943 INFO:tasks.workunit.client.1.smithi181.stderr:+ grep block_name_prefix 2024-09-13T00:22:37.943 INFO:tasks.workunit.client.1.smithi181.stderr:+ awk {print $NF;} 2024-09-13T00:22:37.986 INFO:tasks.workunit.client.1.smithi181.stderr:+ prefix=rbd_data.d73e999cb1f1 2024-09-13T00:22:37.986 INFO:tasks.workunit.client.1.smithi181.stderr:+ grep rbd_data.d73e999cb1f1 2024-09-13T00:22:37.986 INFO:tasks.workunit.client.1.smithi181.stderr:+ get_image_data_pool sparse 2024-09-13T00:22:37.987 INFO:tasks.workunit.client.1.smithi181.stderr:+ image=sparse 2024-09-13T00:22:37.987 INFO:tasks.workunit.client.1.smithi181.stderr:+ sed -e s/rbd_data.d73e999cb1f1\.// -e s/^0*\([0-9a-f]\)/\1/ 2024-09-13T00:22:37.987 INFO:tasks.workunit.client.1.smithi181.stderr:+ sort -u 2024-09-13T00:22:37.987 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd info sparse 2024-09-13T00:22:37.987 INFO:tasks.workunit.client.1.smithi181.stderr:+ grep data_pool: 2024-09-13T00:22:37.987 INFO:tasks.workunit.client.1.smithi181.stderr:+ awk -F: { print $NF } 2024-09-13T00:22:38.027 INFO:tasks.workunit.client.1.smithi181.stderr:+ data_pool= 2024-09-13T00:22:38.028 INFO:tasks.workunit.client.1.smithi181.stderr:+ [ -z ] 2024-09-13T00:22:38.028 INFO:tasks.workunit.client.1.smithi181.stderr:+ data_pool=rbd 2024-09-13T00:22:38.028 INFO:tasks.workunit.client.1.smithi181.stderr:+ echo rbd 2024-09-13T00:22:38.028 INFO:tasks.workunit.client.1.smithi181.stderr:+ rados ls -p rbd 2024-09-13T00:22:38.081 INFO:tasks.workunit.client.1.smithi181.stderr:+ objects= 2024-09-13T00:22:38.081 INFO:tasks.workunit.client.1.smithi181.stderr:+ echo 2024-09-13T00:22:38.081 INFO:tasks.workunit.client.1.smithi181.stderr:+ [ 0 -eq 1 -o = ] 2024-09-13T00:22:38.082 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd rm sparse 2024-09-13T00:22:38.228 INFO:tasks.workunit.client.1.smithi181.stderr: Removing image: 100% complete...done. 2024-09-13T00:22:38.233 INFO:tasks.workunit.client.1.smithi181.stderr:+ echo zeros export to sparse file 2024-09-13T00:22:38.233 INFO:tasks.workunit.client.1.smithi181.stdout:zeros export to sparse file 2024-09-13T00:22:38.233 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd create --new-format sparse --size 4 2024-09-13T00:22:38.290 INFO:tasks.workunit.client.1.smithi181.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-09-13T00:22:38.294 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd info sparse 2024-09-13T00:22:38.294 INFO:tasks.workunit.client.1.smithi181.stderr:+ grep block_name_prefix 2024-09-13T00:22:38.294 INFO:tasks.workunit.client.1.smithi181.stderr:+ awk {print $NF;} 2024-09-13T00:22:38.337 INFO:tasks.workunit.client.1.smithi181.stderr:+ prefix=rbd_data.d75ff0d34a1f 2024-09-13T00:22:38.337 INFO:tasks.workunit.client.1.smithi181.stderr:+ dd if=/dev/zero bs=4M count=1 2024-09-13T00:22:38.337 INFO:tasks.workunit.client.1.smithi181.stderr:+ get_image_data_pool sparse 2024-09-13T00:22:38.337 INFO:tasks.workunit.client.1.smithi181.stderr:+ image=sparse 2024-09-13T00:22:38.338 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd info sparse 2024-09-13T00:22:38.338 INFO:tasks.workunit.client.1.smithi181.stderr:+ grep data_pool: 2024-09-13T00:22:38.338 INFO:tasks.workunit.client.1.smithi181.stderr:+ awk -F: { print $NF } 2024-09-13T00:22:38.378 INFO:tasks.workunit.client.1.smithi181.stderr:+ data_pool= 2024-09-13T00:22:38.378 INFO:tasks.workunit.client.1.smithi181.stderr:+ [ -z ] 2024-09-13T00:22:38.378 INFO:tasks.workunit.client.1.smithi181.stderr:+ data_pool=rbd 2024-09-13T00:22:38.378 INFO:tasks.workunit.client.1.smithi181.stderr:+ echo rbd 2024-09-13T00:22:38.378 INFO:tasks.workunit.client.1.smithi181.stderr:+ rados -p rbd put rbd_data.d75ff0d34a1f.000000000000 - 2024-09-13T00:22:38.410 INFO:tasks.workunit.client.1.smithi181.stderr:1+0 records in 2024-09-13T00:22:38.410 INFO:tasks.workunit.client.1.smithi181.stderr:1+0 records out 2024-09-13T00:22:38.410 INFO:tasks.workunit.client.1.smithi181.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0727328 s, 57.7 MB/s 2024-09-13T00:22:38.442 INFO:tasks.workunit.client.1.smithi181.stderr:+ objects sparse 2024-09-13T00:22:38.442 INFO:tasks.workunit.client.1.smithi181.stderr:+ image=sparse 2024-09-13T00:22:38.443 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd info sparse 2024-09-13T00:22:38.443 INFO:tasks.workunit.client.1.smithi181.stderr:+ grep block_name_prefix 2024-09-13T00:22:38.443 INFO:tasks.workunit.client.1.smithi181.stderr:+ awk {print $NF;} 2024-09-13T00:22:38.486 INFO:tasks.workunit.client.1.smithi181.stderr:+ prefix=rbd_data.d75ff0d34a1f 2024-09-13T00:22:38.486 INFO:tasks.workunit.client.1.smithi181.stderr:+ grep rbd_data.d75ff0d34a1f 2024-09-13T00:22:38.486 INFO:tasks.workunit.client.1.smithi181.stderr:+ get_image_data_pool sparse 2024-09-13T00:22:38.486 INFO:tasks.workunit.client.1.smithi181.stderr:+ image=sparse 2024-09-13T00:22:38.486 INFO:tasks.workunit.client.1.smithi181.stderr:+ sed -e s/rbd_data.d75ff0d34a1f\.// -e s/^0*\([0-9a-f]\)/\1/ 2024-09-13T00:22:38.486 INFO:tasks.workunit.client.1.smithi181.stderr:+ sort -u 2024-09-13T00:22:38.487 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd info sparse 2024-09-13T00:22:38.487 INFO:tasks.workunit.client.1.smithi181.stderr:+ grep data_pool: 2024-09-13T00:22:38.487 INFO:tasks.workunit.client.1.smithi181.stderr:+ awk -F: { print $NF } 2024-09-13T00:22:38.526 INFO:tasks.workunit.client.1.smithi181.stderr:+ data_pool= 2024-09-13T00:22:38.526 INFO:tasks.workunit.client.1.smithi181.stderr:+ [ -z ] 2024-09-13T00:22:38.526 INFO:tasks.workunit.client.1.smithi181.stderr:+ data_pool=rbd 2024-09-13T00:22:38.526 INFO:tasks.workunit.client.1.smithi181.stderr:+ echo rbd 2024-09-13T00:22:38.526 INFO:tasks.workunit.client.1.smithi181.stderr:+ rados ls -p rbd 2024-09-13T00:22:38.579 INFO:tasks.workunit.client.1.smithi181.stderr:+ objects=0 2024-09-13T00:22:38.579 INFO:tasks.workunit.client.1.smithi181.stderr:+ echo 0 2024-09-13T00:22:38.579 INFO:tasks.workunit.client.1.smithi181.stderr:+ [ 0 -eq 1 -o 0 = 0 ] 2024-09-13T00:22:38.580 INFO:tasks.workunit.client.1.smithi181.stderr:+ rm /tmp/rbd_import_export_106423/sparse 2024-09-13T00:22:38.580 INFO:tasks.workunit.client.1.smithi181.stderr:rm: cannot remove '/tmp/rbd_import_export_106423/sparse': No such file or directory 2024-09-13T00:22:38.580 INFO:tasks.workunit.client.1.smithi181.stderr:+ true 2024-09-13T00:22:38.580 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd export sparse /tmp/rbd_import_export_106423/sparse 2024-09-13T00:22:38.621 INFO:tasks.workunit.client.1.smithi181.stderr: Exporting image: 100% complete...done. 2024-09-13T00:22:38.625 INFO:tasks.workunit.client.1.smithi181.stderr:+ stat /tmp/rbd_import_export_106423/sparse --format=%b 2024-09-13T00:22:38.626 INFO:tasks.workunit.client.1.smithi181.stderr:+ [ 0 = 0 ] 2024-09-13T00:22:38.626 INFO:tasks.workunit.client.1.smithi181.stderr:+ rbd rm sparse 2024-09-13T00:22:38.804 INFO:tasks.workunit.client.1.smithi181.stderr: Removing image: 100% complete...done. 2024-09-13T00:22:38.809 INFO:tasks.workunit.client.1.smithi181.stderr:+ rm /tmp/rbd_import_export_106423/sparse /tmp/rbd_import_export_106423/sparse1 /tmp/rbd_import_export_106423/sparse2 /tmp/rbd_import_export_106423/sparse3 2024-09-13T00:22:38.812 INFO:tasks.workunit.client.1.smithi181.stderr:rm: cannot remove '/tmp/rbd_import_export_106423/sparse3': No such file or directory 2024-09-13T00:22:38.812 INFO:tasks.workunit.client.1.smithi181.stderr:+ true 2024-09-13T00:22:38.813 INFO:tasks.workunit.client.1.smithi181.stdout:OK 2024-09-13T00:22:38.814 INFO:tasks.workunit.client.1.smithi181.stderr:+ echo OK 2024-09-13T00:22:38.814 INFO:tasks.workunit.client.1.smithi181.stderr:+ rm -rf /tmp/rbd_import_export_106423 2024-09-13T00:22:38.814 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-13T00:22:38.815 DEBUG:teuthology.orchestra.run.smithi181:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.1/client.1/tmp 2024-09-13T00:22:38.874 INFO:tasks.workunit:Stopping ['rbd/import_export.sh'] on client.1... 2024-09-13T00:22:38.874 DEBUG:teuthology.orchestra.run.smithi181:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.1 /home/ubuntu/cephtest/clone.client.1 2024-09-13T00:22:39.145 DEBUG:teuthology.parallel:result is None 2024-09-13T00:22:39.146 DEBUG:teuthology.orchestra.run.smithi181:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.1/client.1 2024-09-13T00:22:39.158 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.1/client.1 2024-09-13T00:22:39.158 DEBUG:teuthology.orchestra.run.smithi181:> rmdir -- /home/ubuntu/cephtest/mnt.1 2024-09-13T00:22:39.206 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.1/client.1 2024-09-13T00:22:39.206 INFO:teuthology.task.full_sequential:In full_sequential, running task print... 2024-09-13T00:22:39.207 INFO:teuthology.task.print:**** done end rbd_import_export.yaml 2024-09-13T00:22:39.207 INFO:teuthology.task.full_sequential:In full_sequential, running task print... 2024-09-13T00:22:39.207 INFO:teuthology.task.print:**** done start test_rbd_api.yaml 2024-09-13T00:22:39.207 INFO:teuthology.task.full_sequential:In full_sequential, running task workunit... 2024-09-13T00:22:39.208 INFO:tasks.workunit:Pulling workunits from ref quincy 2024-09-13T00:22:39.208 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-09-13T00:22:39.208 DEBUG:teuthology.orchestra.run.smithi136:> stat -- /home/ubuntu/cephtest/mnt.0 2024-09-13T00:22:39.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T00:22:39.215 INFO:teuthology.orchestra.run.smithi136.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2024-09-13T00:22:39.215 DEBUG:teuthology.orchestra.run.smithi136:> mkdir -- /home/ubuntu/cephtest/mnt.0 2024-09-13T00:22:39.264 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2024-09-13T00:22:39.264 DEBUG:teuthology.orchestra.run.smithi136:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2024-09-13T00:22:39.312 INFO:tasks.workunit:timeout=3h 2024-09-13T00:22:39.313 INFO:tasks.workunit:cleanup=True 2024-09-13T00:22:39.316 DEBUG:teuthology.orchestra.run.smithi136:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://git.ceph.com/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2024-09-13T00:22:39.363 INFO:tasks.workunit.client.0.smithi136.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-09-13T00:22:39.411 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:39 smithi136 bash[76371]: cluster 2024-09-13T00:22:37.777574+0000 mgr.y (mgr.24661) 1365 : cluster [DBG] pgmap v2060: 105 pgs: 105 active+clean; 583 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 282 KiB/s rd, 180 KiB/s wr, 156 op/s 2024-09-13T00:22:39.412 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:39 smithi136 bash[76371]: audit 2024-09-13T00:22:38.807484+0000 mon.a (mon.0) 2315 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:22:39.412 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:39 smithi136 bash[76371]: cluster 2024-09-13T00:22:38.903765+0000 mon.a (mon.0) 2316 : cluster [DBG] osdmap e1135: 8 total, 8 up, 8 in 2024-09-13T00:22:39.412 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:39 smithi136 bash[75164]: cluster 2024-09-13T00:22:37.777574+0000 mgr.y (mgr.24661) 1365 : cluster [DBG] pgmap v2060: 105 pgs: 105 active+clean; 583 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 282 KiB/s rd, 180 KiB/s wr, 156 op/s 2024-09-13T00:22:39.412 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:39 smithi136 bash[75164]: audit 2024-09-13T00:22:38.807484+0000 mon.a (mon.0) 2315 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:22:39.412 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:39 smithi136 bash[75164]: cluster 2024-09-13T00:22:38.903765+0000 mon.a (mon.0) 2316 : cluster [DBG] osdmap e1135: 8 total, 8 up, 8 in 2024-09-13T00:22:39.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:39 smithi181 bash[67587]: cluster 2024-09-13T00:22:37.777574+0000 mgr.y (mgr.24661) 1365 : cluster [DBG] pgmap v2060: 105 pgs: 105 active+clean; 583 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 282 KiB/s rd, 180 KiB/s wr, 156 op/s 2024-09-13T00:22:39.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:39 smithi181 bash[67587]: audit 2024-09-13T00:22:38.807484+0000 mon.a (mon.0) 2315 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:22:39.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:39 smithi181 bash[67587]: cluster 2024-09-13T00:22:38.903765+0000 mon.a (mon.0) 2316 : cluster [DBG] osdmap e1135: 8 total, 8 up, 8 in 2024-09-13T00:22:39.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:22:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:22:39] "GET /metrics HTTP/1.1" 200 34124 "" "Prometheus/2.43.0" 2024-09-13T00:22:41.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:41 smithi136 bash[75164]: cluster 2024-09-13T00:22:39.778367+0000 mgr.y (mgr.24661) 1366 : cluster [DBG] pgmap v2062: 105 pgs: 105 active+clean; 583 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 538 KiB/s rd, 324 KiB/s wr, 308 op/s 2024-09-13T00:22:41.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:41 smithi136 bash[76371]: cluster 2024-09-13T00:22:39.778367+0000 mgr.y (mgr.24661) 1366 : cluster [DBG] pgmap v2062: 105 pgs: 105 active+clean; 583 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 538 KiB/s rd, 324 KiB/s wr, 308 op/s 2024-09-13T00:22:41.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:41 smithi181 bash[67587]: cluster 2024-09-13T00:22:39.778367+0000 mgr.y (mgr.24661) 1366 : cluster [DBG] pgmap v2062: 105 pgs: 105 active+clean; 583 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 538 KiB/s rd, 324 KiB/s wr, 308 op/s 2024-09-13T00:22:42.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:22:42 smithi181 bash[95505]: ts=2024-09-13T00:22:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:22:43.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:43 smithi136 bash[76371]: cluster 2024-09-13T00:22:41.779501+0000 mgr.y (mgr.24661) 1367 : cluster [DBG] pgmap v2063: 105 pgs: 105 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 1023 KiB/s rd, 1.4 MiB/s wr, 341 op/s 2024-09-13T00:22:43.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:43 smithi136 bash[75164]: cluster 2024-09-13T00:22:41.779501+0000 mgr.y (mgr.24661) 1367 : cluster [DBG] pgmap v2063: 105 pgs: 105 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 1023 KiB/s rd, 1.4 MiB/s wr, 341 op/s 2024-09-13T00:22:43.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:43 smithi181 bash[67587]: cluster 2024-09-13T00:22:41.779501+0000 mgr.y (mgr.24661) 1367 : cluster [DBG] pgmap v2063: 105 pgs: 105 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 1023 KiB/s rd, 1.4 MiB/s wr, 341 op/s 2024-09-13T00:22:45.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:44 smithi136 bash[75164]: cluster 2024-09-13T00:22:43.780309+0000 mgr.y (mgr.24661) 1368 : cluster [DBG] pgmap v2064: 105 pgs: 105 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 829 KiB/s rd, 1.1 MiB/s wr, 277 op/s 2024-09-13T00:22:45.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:44 smithi136 bash[75164]: cluster 2024-09-13T00:22:43.906108+0000 mon.a (mon.0) 2317 : cluster [DBG] osdmap e1136: 8 total, 8 up, 8 in 2024-09-13T00:22:45.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:44 smithi136 bash[76371]: cluster 2024-09-13T00:22:43.780309+0000 mgr.y (mgr.24661) 1368 : cluster [DBG] pgmap v2064: 105 pgs: 105 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 829 KiB/s rd, 1.1 MiB/s wr, 277 op/s 2024-09-13T00:22:45.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:44 smithi136 bash[76371]: cluster 2024-09-13T00:22:43.906108+0000 mon.a (mon.0) 2317 : cluster [DBG] osdmap e1136: 8 total, 8 up, 8 in 2024-09-13T00:22:45.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:44 smithi181 bash[67587]: cluster 2024-09-13T00:22:43.780309+0000 mgr.y (mgr.24661) 1368 : cluster [DBG] pgmap v2064: 105 pgs: 105 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 829 KiB/s rd, 1.1 MiB/s wr, 277 op/s 2024-09-13T00:22:45.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:44 smithi181 bash[67587]: cluster 2024-09-13T00:22:43.906108+0000 mon.a (mon.0) 2317 : cluster [DBG] osdmap e1136: 8 total, 8 up, 8 in 2024-09-13T00:22:45.567 DEBUG:teuthology.orchestra.run.smithi136:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-09-13T00:22:45.620 INFO:tasks.workunit.client.0.smithi136.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-09-13T00:22:45.623 INFO:tasks.workunit.client.0.smithi136.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-13T00:22:45.624 INFO:tasks.workunit.client.0.smithi136.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-09-13T00:22:45.680 INFO:tasks.workunit.client.0.smithi136.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-09-13T00:22:45.740 INFO:tasks.workunit.client.0.smithi136.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-09-13T00:22:45.783 INFO:tasks.workunit.client.0.smithi136.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-13T00:22:45.785 INFO:tasks.workunit.client.0.smithi136.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-13T00:22:45.785 INFO:tasks.workunit.client.0.smithi136.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-09-13T00:22:45.825 INFO:tasks.workunit.client.0.smithi136.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-13T00:22:45.829 DEBUG:teuthology.orchestra.run.smithi136:> set -ex 2024-09-13T00:22:45.829 DEBUG:teuthology.orchestra.run.smithi136:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-09-13T00:22:45.876 INFO:tasks.workunit:Running workunits matching rbd/test_librbd.sh on client.0... 2024-09-13T00:22:45.877 INFO:tasks.workunit:Running workunit rbd/test_librbd.sh... 2024-09-13T00:22:45.877 DEBUG:teuthology.orchestra.run.smithi136:workunit test rbd/test_librbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_FEATURES=61 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/test_librbd.sh 2024-09-13T00:22:45.987 INFO:tasks.workunit.client.0.smithi136.stderr:warning: failed to enable lockdep 2024-09-13T00:22:45.988 INFO:tasks.workunit.client.0.smithi136.stdout:seed 114798 2024-09-13T00:22:45.988 INFO:tasks.workunit.client.0.smithi136.stdout:[==========] Running 406 tests from 15 test suites. 2024-09-13T00:22:45.988 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] Global test environment set-up. 2024-09-13T00:22:45.988 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 16 tests from TestJournalReplay 2024-09-13T00:22:47.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:46 smithi136 bash[75164]: cluster 2024-09-13T00:22:45.781124+0000 mgr.y (mgr.24661) 1369 : cluster [DBG] pgmap v2066: 105 pgs: 105 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 814 KiB/s rd, 1.3 MiB/s wr, 230 op/s 2024-09-13T00:22:47.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:46 smithi136 bash[76371]: cluster 2024-09-13T00:22:45.781124+0000 mgr.y (mgr.24661) 1369 : cluster [DBG] pgmap v2066: 105 pgs: 105 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 814 KiB/s rd, 1.3 MiB/s wr, 230 op/s 2024-09-13T00:22:47.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:46 smithi181 bash[67587]: cluster 2024-09-13T00:22:45.781124+0000 mgr.y (mgr.24661) 1369 : cluster [DBG] pgmap v2066: 105 pgs: 105 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 814 KiB/s rd, 1.3 MiB/s wr, 230 op/s 2024-09-13T00:22:47.937 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestJournalReplay.AioDiscardEvent 2024-09-13T00:22:48.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:47 smithi136 bash[76371]: cluster 2024-09-13T00:22:46.922747+0000 mon.a (mon.0) 2318 : cluster [DBG] osdmap e1137: 8 total, 8 up, 8 in 2024-09-13T00:22:48.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:47 smithi136 bash[76371]: audit 2024-09-13T00:22:46.926199+0000 mon.c (mon.2) 95 : audit [INF] from='client.? 172.21.15.136:0/1451855576' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:22:48.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:47 smithi136 bash[76371]: audit 2024-09-13T00:22:46.927068+0000 mon.a (mon.0) 2319 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:22:48.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:47 smithi136 bash[75164]: cluster 2024-09-13T00:22:46.922747+0000 mon.a (mon.0) 2318 : cluster [DBG] osdmap e1137: 8 total, 8 up, 8 in 2024-09-13T00:22:48.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:47 smithi136 bash[75164]: audit 2024-09-13T00:22:46.926199+0000 mon.c (mon.2) 95 : audit [INF] from='client.? 172.21.15.136:0/1451855576' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:22:48.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:47 smithi136 bash[75164]: audit 2024-09-13T00:22:46.927068+0000 mon.a (mon.0) 2319 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:22:48.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:47 smithi181 bash[67587]: cluster 2024-09-13T00:22:46.922747+0000 mon.a (mon.0) 2318 : cluster [DBG] osdmap e1137: 8 total, 8 up, 8 in 2024-09-13T00:22:48.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:47 smithi181 bash[67587]: audit 2024-09-13T00:22:46.926199+0000 mon.c (mon.2) 95 : audit [INF] from='client.? 172.21.15.136:0/1451855576' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:22:48.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:47 smithi181 bash[67587]: audit 2024-09-13T00:22:46.927068+0000 mon.a (mon.0) 2319 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:22:49.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:48 smithi136 bash[75164]: cluster 2024-09-13T00:22:47.781960+0000 mgr.y (mgr.24661) 1370 : cluster [DBG] pgmap v2068: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 579 KiB/s rd, 1.1 MiB/s wr, 92 op/s 2024-09-13T00:22:49.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:48 smithi136 bash[75164]: cluster 2024-09-13T00:22:47.915475+0000 mon.a (mon.0) 2320 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:22:49.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:48 smithi136 bash[75164]: audit 2024-09-13T00:22:47.935502+0000 mon.a (mon.0) 2321 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:22:49.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:48 smithi136 bash[75164]: cluster 2024-09-13T00:22:47.935970+0000 mon.a (mon.0) 2322 : cluster [DBG] osdmap e1138: 8 total, 8 up, 8 in 2024-09-13T00:22:49.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:48 smithi136 bash[76371]: cluster 2024-09-13T00:22:47.781960+0000 mgr.y (mgr.24661) 1370 : cluster [DBG] pgmap v2068: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 579 KiB/s rd, 1.1 MiB/s wr, 92 op/s 2024-09-13T00:22:49.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:48 smithi136 bash[76371]: cluster 2024-09-13T00:22:47.915475+0000 mon.a (mon.0) 2320 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:22:49.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:48 smithi136 bash[76371]: audit 2024-09-13T00:22:47.935502+0000 mon.a (mon.0) 2321 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:22:49.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:48 smithi136 bash[76371]: cluster 2024-09-13T00:22:47.935970+0000 mon.a (mon.0) 2322 : cluster [DBG] osdmap e1138: 8 total, 8 up, 8 in 2024-09-13T00:22:49.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:48 smithi181 bash[67587]: cluster 2024-09-13T00:22:47.781960+0000 mgr.y (mgr.24661) 1370 : cluster [DBG] pgmap v2068: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 579 KiB/s rd, 1.1 MiB/s wr, 92 op/s 2024-09-13T00:22:49.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:48 smithi181 bash[67587]: cluster 2024-09-13T00:22:47.915475+0000 mon.a (mon.0) 2320 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:22:49.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:48 smithi181 bash[67587]: audit 2024-09-13T00:22:47.935502+0000 mon.a (mon.0) 2321 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:22:49.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:48 smithi181 bash[67587]: cluster 2024-09-13T00:22:47.935970+0000 mon.a (mon.0) 2322 : cluster [DBG] osdmap e1138: 8 total, 8 up, 8 in 2024-09-13T00:22:49.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:22:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:22:49] "GET /metrics HTTP/1.1" 200 34124 "" "Prometheus/2.43.0" 2024-09-13T00:22:49.994 INFO:tasks.workunit.client.0.smithi136.stdout:SKIPPING 2024-09-13T00:22:49.994 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestJournalReplay.AioDiscardEvent (2057 ms) 2024-09-13T00:22:49.994 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestJournalReplay.AioWriteEvent 2024-09-13T00:22:50.011 INFO:tasks.workunit.client.0.smithi136.stdout:SKIPPING 2024-09-13T00:22:50.011 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestJournalReplay.AioWriteEvent (17 ms) 2024-09-13T00:22:50.011 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestJournalReplay.AioFlushEvent 2024-09-13T00:22:50.041 INFO:tasks.workunit.client.0.smithi136.stdout:SKIPPING 2024-09-13T00:22:50.041 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestJournalReplay.AioFlushEvent (31 ms) 2024-09-13T00:22:50.041 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestJournalReplay.SnapCreate 2024-09-13T00:22:50.061 INFO:tasks.workunit.client.0.smithi136.stdout:SKIPPING 2024-09-13T00:22:50.061 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestJournalReplay.SnapCreate (20 ms) 2024-09-13T00:22:50.061 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestJournalReplay.SnapProtect 2024-09-13T00:22:50.081 INFO:tasks.workunit.client.0.smithi136.stdout:SKIPPING 2024-09-13T00:22:50.081 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestJournalReplay.SnapProtect (19 ms) 2024-09-13T00:22:50.081 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestJournalReplay.SnapUnprotect 2024-09-13T00:22:50.101 INFO:tasks.workunit.client.0.smithi136.stdout:SKIPPING 2024-09-13T00:22:50.101 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestJournalReplay.SnapUnprotect (20 ms) 2024-09-13T00:22:50.101 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestJournalReplay.SnapRename 2024-09-13T00:22:50.120 INFO:tasks.workunit.client.0.smithi136.stdout:SKIPPING 2024-09-13T00:22:50.120 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestJournalReplay.SnapRename (19 ms) 2024-09-13T00:22:50.120 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestJournalReplay.SnapRollback 2024-09-13T00:22:50.140 INFO:tasks.workunit.client.0.smithi136.stdout:SKIPPING 2024-09-13T00:22:50.140 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestJournalReplay.SnapRollback (20 ms) 2024-09-13T00:22:50.140 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestJournalReplay.SnapRemove 2024-09-13T00:22:50.160 INFO:tasks.workunit.client.0.smithi136.stdout:SKIPPING 2024-09-13T00:22:50.160 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestJournalReplay.SnapRemove (20 ms) 2024-09-13T00:22:50.160 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestJournalReplay.Rename 2024-09-13T00:22:50.178 INFO:tasks.workunit.client.0.smithi136.stdout:SKIPPING 2024-09-13T00:22:50.178 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestJournalReplay.Rename (19 ms) 2024-09-13T00:22:50.178 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestJournalReplay.Resize 2024-09-13T00:22:50.197 INFO:tasks.workunit.client.0.smithi136.stdout:SKIPPING 2024-09-13T00:22:50.197 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestJournalReplay.Resize (18 ms) 2024-09-13T00:22:50.197 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestJournalReplay.Flatten 2024-09-13T00:22:50.219 INFO:tasks.workunit.client.0.smithi136.stdout:SKIPPING 2024-09-13T00:22:50.219 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestJournalReplay.Flatten (22 ms) 2024-09-13T00:22:50.219 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestJournalReplay.UpdateFeatures 2024-09-13T00:22:50.237 INFO:tasks.workunit.client.0.smithi136.stdout:SKIPPING 2024-09-13T00:22:50.237 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestJournalReplay.UpdateFeatures (18 ms) 2024-09-13T00:22:50.237 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestJournalReplay.MetadataSet 2024-09-13T00:22:50.255 INFO:tasks.workunit.client.0.smithi136.stdout:SKIPPING 2024-09-13T00:22:50.255 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestJournalReplay.MetadataSet (18 ms) 2024-09-13T00:22:50.255 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestJournalReplay.MetadataRemove 2024-09-13T00:22:50.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:49 smithi136 bash[75164]: cluster 2024-09-13T00:22:48.938711+0000 mon.a (mon.0) 2323 : cluster [DBG] osdmap e1139: 8 total, 8 up, 8 in 2024-09-13T00:22:50.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:49 smithi136 bash[76371]: cluster 2024-09-13T00:22:48.938711+0000 mon.a (mon.0) 2323 : cluster [DBG] osdmap e1139: 8 total, 8 up, 8 in 2024-09-13T00:22:50.275 INFO:tasks.workunit.client.0.smithi136.stdout:SKIPPING 2024-09-13T00:22:50.275 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestJournalReplay.MetadataRemove (21 ms) 2024-09-13T00:22:50.276 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestJournalReplay.ObjectPosition 2024-09-13T00:22:50.294 INFO:tasks.workunit.client.0.smithi136.stdout:SKIPPING 2024-09-13T00:22:50.294 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestJournalReplay.ObjectPosition (19 ms) 2024-09-13T00:22:50.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:49 smithi181 bash[67587]: cluster 2024-09-13T00:22:48.938711+0000 mon.a (mon.0) 2323 : cluster [DBG] osdmap e1139: 8 total, 8 up, 8 in 2024-09-13T00:22:50.967 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 16 tests from TestJournalReplay (2358 ms total) 2024-09-13T00:22:50.968 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:22:50.968 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 3 tests from TestJournalEntries 2024-09-13T00:22:51.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:50 smithi136 bash[75164]: cluster 2024-09-13T00:22:49.782714+0000 mgr.y (mgr.24661) 1371 : cluster [DBG] pgmap v2071: 137 pgs: 6 creating+peering, 20 unknown, 111 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:22:51.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:50 smithi136 bash[75164]: cluster 2024-09-13T00:22:49.942216+0000 mon.a (mon.0) 2324 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:22:51.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:50 smithi136 bash[75164]: cluster 2024-09-13T00:22:49.942264+0000 mon.a (mon.0) 2325 : cluster [INF] Cluster is now healthy 2024-09-13T00:22:51.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:50 smithi136 bash[75164]: cluster 2024-09-13T00:22:49.960572+0000 mon.a (mon.0) 2326 : cluster [DBG] osdmap e1140: 8 total, 8 up, 8 in 2024-09-13T00:22:51.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:50 smithi136 bash[76371]: cluster 2024-09-13T00:22:49.782714+0000 mgr.y (mgr.24661) 1371 : cluster [DBG] pgmap v2071: 137 pgs: 6 creating+peering, 20 unknown, 111 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:22:51.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:50 smithi136 bash[76371]: cluster 2024-09-13T00:22:49.942216+0000 mon.a (mon.0) 2324 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:22:51.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:50 smithi136 bash[76371]: cluster 2024-09-13T00:22:49.942264+0000 mon.a (mon.0) 2325 : cluster [INF] Cluster is now healthy 2024-09-13T00:22:51.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:50 smithi136 bash[76371]: cluster 2024-09-13T00:22:49.960572+0000 mon.a (mon.0) 2326 : cluster [DBG] osdmap e1140: 8 total, 8 up, 8 in 2024-09-13T00:22:51.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:50 smithi181 bash[67587]: cluster 2024-09-13T00:22:49.782714+0000 mgr.y (mgr.24661) 1371 : cluster [DBG] pgmap v2071: 137 pgs: 6 creating+peering, 20 unknown, 111 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:22:51.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:50 smithi181 bash[67587]: cluster 2024-09-13T00:22:49.942216+0000 mon.a (mon.0) 2324 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:22:51.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:50 smithi181 bash[67587]: cluster 2024-09-13T00:22:49.942264+0000 mon.a (mon.0) 2325 : cluster [INF] Cluster is now healthy 2024-09-13T00:22:51.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:50 smithi181 bash[67587]: cluster 2024-09-13T00:22:49.960572+0000 mon.a (mon.0) 2326 : cluster [DBG] osdmap e1140: 8 total, 8 up, 8 in 2024-09-13T00:22:52.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:51 smithi136 bash[76371]: cluster 2024-09-13T00:22:50.961091+0000 mon.a (mon.0) 2327 : cluster [DBG] osdmap e1141: 8 total, 8 up, 8 in 2024-09-13T00:22:52.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:51 smithi136 bash[75164]: cluster 2024-09-13T00:22:50.961091+0000 mon.a (mon.0) 2327 : cluster [DBG] osdmap e1141: 8 total, 8 up, 8 in 2024-09-13T00:22:52.314 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:51 smithi181 bash[67587]: cluster 2024-09-13T00:22:50.961091+0000 mon.a (mon.0) 2327 : cluster [DBG] osdmap e1141: 8 total, 8 up, 8 in 2024-09-13T00:22:52.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:22:52 smithi181 bash[95505]: ts=2024-09-13T00:22:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:22:53.019 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestJournalEntries.AioWrite 2024-09-13T00:22:53.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:53 smithi136 bash[75164]: cluster 2024-09-13T00:22:51.783604+0000 mgr.y (mgr.24661) 1372 : cluster [DBG] pgmap v2074: 105 pgs: 105 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:22:53.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:53 smithi136 bash[75164]: cluster 2024-09-13T00:22:51.995190+0000 mon.a (mon.0) 2328 : cluster [DBG] osdmap e1142: 8 total, 8 up, 8 in 2024-09-13T00:22:53.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:53 smithi136 bash[75164]: audit 2024-09-13T00:22:52.001218+0000 mon.c (mon.2) 96 : audit [INF] from='client.? 172.21.15.136:0/2010156916' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:22:53.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:53 smithi136 bash[75164]: audit 2024-09-13T00:22:52.002063+0000 mon.a (mon.0) 2329 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:22:53.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:53 smithi136 bash[76371]: cluster 2024-09-13T00:22:51.783604+0000 mgr.y (mgr.24661) 1372 : cluster [DBG] pgmap v2074: 105 pgs: 105 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:22:53.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:53 smithi136 bash[76371]: cluster 2024-09-13T00:22:51.995190+0000 mon.a (mon.0) 2328 : cluster [DBG] osdmap e1142: 8 total, 8 up, 8 in 2024-09-13T00:22:53.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:53 smithi136 bash[76371]: audit 2024-09-13T00:22:52.001218+0000 mon.c (mon.2) 96 : audit [INF] from='client.? 172.21.15.136:0/2010156916' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:22:53.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:53 smithi136 bash[76371]: audit 2024-09-13T00:22:52.002063+0000 mon.a (mon.0) 2329 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:22:53.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:53 smithi181 bash[67587]: cluster 2024-09-13T00:22:51.783604+0000 mgr.y (mgr.24661) 1372 : cluster [DBG] pgmap v2074: 105 pgs: 105 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:22:53.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:53 smithi181 bash[67587]: cluster 2024-09-13T00:22:51.995190+0000 mon.a (mon.0) 2328 : cluster [DBG] osdmap e1142: 8 total, 8 up, 8 in 2024-09-13T00:22:53.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:53 smithi181 bash[67587]: audit 2024-09-13T00:22:52.001218+0000 mon.c (mon.2) 96 : audit [INF] from='client.? 172.21.15.136:0/2010156916' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:22:53.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:53 smithi181 bash[67587]: audit 2024-09-13T00:22:52.002063+0000 mon.a (mon.0) 2329 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:22:54.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:54 smithi181 bash[67587]: audit 2024-09-13T00:22:52.994521+0000 mon.a (mon.0) 2330 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:22:54.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:54 smithi181 bash[67587]: cluster 2024-09-13T00:22:52.994846+0000 mon.a (mon.0) 2331 : cluster [DBG] osdmap e1143: 8 total, 8 up, 8 in 2024-09-13T00:22:54.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:54 smithi181 bash[67587]: audit 2024-09-13T00:22:53.808015+0000 mon.a (mon.0) 2332 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:22:54.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:54 smithi136 bash[75164]: audit 2024-09-13T00:22:52.994521+0000 mon.a (mon.0) 2330 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:22:54.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:54 smithi136 bash[75164]: cluster 2024-09-13T00:22:52.994846+0000 mon.a (mon.0) 2331 : cluster [DBG] osdmap e1143: 8 total, 8 up, 8 in 2024-09-13T00:22:54.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:54 smithi136 bash[75164]: audit 2024-09-13T00:22:53.808015+0000 mon.a (mon.0) 2332 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:22:54.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:54 smithi136 bash[76371]: audit 2024-09-13T00:22:52.994521+0000 mon.a (mon.0) 2330 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:22:54.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:54 smithi136 bash[76371]: cluster 2024-09-13T00:22:52.994846+0000 mon.a (mon.0) 2331 : cluster [DBG] osdmap e1143: 8 total, 8 up, 8 in 2024-09-13T00:22:54.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:54 smithi136 bash[76371]: audit 2024-09-13T00:22:53.808015+0000 mon.a (mon.0) 2332 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:22:55.086 INFO:tasks.workunit.client.0.smithi136.stdout:SKIPPING 2024-09-13T00:22:55.087 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestJournalEntries.AioWrite (2068 ms) 2024-09-13T00:22:55.087 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestJournalEntries.AioDiscard 2024-09-13T00:22:55.108 INFO:tasks.workunit.client.0.smithi136.stdout:SKIPPING 2024-09-13T00:22:55.108 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestJournalEntries.AioDiscard (21 ms) 2024-09-13T00:22:55.108 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestJournalEntries.AioFlush 2024-09-13T00:22:55.129 INFO:tasks.workunit.client.0.smithi136.stdout:SKIPPING 2024-09-13T00:22:55.129 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestJournalEntries.AioFlush (21 ms) 2024-09-13T00:22:55.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:55 smithi181 bash[67587]: cluster 2024-09-13T00:22:53.784430+0000 mgr.y (mgr.24661) 1373 : cluster [DBG] pgmap v2077: 137 pgs: 6 creating+peering, 20 unknown, 111 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:22:55.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:55 smithi181 bash[67587]: cluster 2024-09-13T00:22:54.020966+0000 mon.a (mon.0) 2333 : cluster [DBG] osdmap e1144: 8 total, 8 up, 8 in 2024-09-13T00:22:55.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:55 smithi136 bash[75164]: cluster 2024-09-13T00:22:53.784430+0000 mgr.y (mgr.24661) 1373 : cluster [DBG] pgmap v2077: 137 pgs: 6 creating+peering, 20 unknown, 111 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:22:55.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:55 smithi136 bash[75164]: cluster 2024-09-13T00:22:54.020966+0000 mon.a (mon.0) 2333 : cluster [DBG] osdmap e1144: 8 total, 8 up, 8 in 2024-09-13T00:22:55.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:55 smithi136 bash[76371]: cluster 2024-09-13T00:22:53.784430+0000 mgr.y (mgr.24661) 1373 : cluster [DBG] pgmap v2077: 137 pgs: 6 creating+peering, 20 unknown, 111 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:22:55.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:55 smithi136 bash[76371]: cluster 2024-09-13T00:22:54.020966+0000 mon.a (mon.0) 2333 : cluster [DBG] osdmap e1144: 8 total, 8 up, 8 in 2024-09-13T00:22:56.045 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 3 tests from TestJournalEntries (2110 ms total) 2024-09-13T00:22:56.045 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:22:56.045 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 2 tests from TestTrash 2024-09-13T00:22:56.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:56 smithi181 bash[67587]: cluster 2024-09-13T00:22:55.035506+0000 mon.a (mon.0) 2334 : cluster [DBG] osdmap e1145: 8 total, 8 up, 8 in 2024-09-13T00:22:56.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:56 smithi136 bash[75164]: cluster 2024-09-13T00:22:55.035506+0000 mon.a (mon.0) 2334 : cluster [DBG] osdmap e1145: 8 total, 8 up, 8 in 2024-09-13T00:22:56.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:56 smithi136 bash[76371]: cluster 2024-09-13T00:22:55.035506+0000 mon.a (mon.0) 2334 : cluster [DBG] osdmap e1145: 8 total, 8 up, 8 in 2024-09-13T00:22:57.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:57 smithi181 bash[67587]: cluster 2024-09-13T00:22:55.785380+0000 mgr.y (mgr.24661) 1374 : cluster [DBG] pgmap v2080: 137 pgs: 6 creating+peering, 12 unknown, 119 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 767 B/s wr, 1 op/s 2024-09-13T00:22:57.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:57 smithi181 bash[67587]: cluster 2024-09-13T00:22:56.038700+0000 mon.a (mon.0) 2335 : cluster [DBG] osdmap e1146: 8 total, 8 up, 8 in 2024-09-13T00:22:57.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:57 smithi136 bash[75164]: cluster 2024-09-13T00:22:55.785380+0000 mgr.y (mgr.24661) 1374 : cluster [DBG] pgmap v2080: 137 pgs: 6 creating+peering, 12 unknown, 119 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 767 B/s wr, 1 op/s 2024-09-13T00:22:57.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:57 smithi136 bash[75164]: cluster 2024-09-13T00:22:56.038700+0000 mon.a (mon.0) 2335 : cluster [DBG] osdmap e1146: 8 total, 8 up, 8 in 2024-09-13T00:22:57.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:57 smithi136 bash[76371]: cluster 2024-09-13T00:22:55.785380+0000 mgr.y (mgr.24661) 1374 : cluster [DBG] pgmap v2080: 137 pgs: 6 creating+peering, 12 unknown, 119 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 767 B/s wr, 1 op/s 2024-09-13T00:22:57.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:57 smithi136 bash[76371]: cluster 2024-09-13T00:22:56.038700+0000 mon.a (mon.0) 2335 : cluster [DBG] osdmap e1146: 8 total, 8 up, 8 in 2024-09-13T00:22:58.087 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestTrash.UserRemovingSource 2024-09-13T00:22:58.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:58 smithi181 bash[67587]: cluster 2024-09-13T00:22:57.067514+0000 mon.a (mon.0) 2336 : cluster [DBG] osdmap e1147: 8 total, 8 up, 8 in 2024-09-13T00:22:58.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:58 smithi181 bash[67587]: audit 2024-09-13T00:22:57.071251+0000 mon.c (mon.2) 97 : audit [INF] from='client.? 172.21.15.136:0/862997171' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:22:58.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:58 smithi181 bash[67587]: audit 2024-09-13T00:22:57.072083+0000 mon.a (mon.0) 2337 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:22:58.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:58 smithi136 bash[75164]: cluster 2024-09-13T00:22:57.067514+0000 mon.a (mon.0) 2336 : cluster [DBG] osdmap e1147: 8 total, 8 up, 8 in 2024-09-13T00:22:58.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:58 smithi136 bash[75164]: audit 2024-09-13T00:22:57.071251+0000 mon.c (mon.2) 97 : audit [INF] from='client.? 172.21.15.136:0/862997171' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:22:58.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:58 smithi136 bash[75164]: audit 2024-09-13T00:22:57.072083+0000 mon.a (mon.0) 2337 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:22:58.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:58 smithi136 bash[76371]: cluster 2024-09-13T00:22:57.067514+0000 mon.a (mon.0) 2336 : cluster [DBG] osdmap e1147: 8 total, 8 up, 8 in 2024-09-13T00:22:58.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:58 smithi136 bash[76371]: audit 2024-09-13T00:22:57.071251+0000 mon.c (mon.2) 97 : audit [INF] from='client.? 172.21.15.136:0/862997171' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:22:58.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:58 smithi136 bash[76371]: audit 2024-09-13T00:22:57.072083+0000 mon.a (mon.0) 2337 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:22:59.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:59 smithi181 bash[67587]: cluster 2024-09-13T00:22:57.786230+0000 mgr.y (mgr.24661) 1375 : cluster [DBG] pgmap v2083: 137 pgs: 2 creating+peering, 30 unknown, 105 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:22:59.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:59 smithi181 bash[67587]: cluster 2024-09-13T00:22:58.058603+0000 mon.a (mon.0) 2338 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:22:59.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:59 smithi181 bash[67587]: audit 2024-09-13T00:22:58.084445+0000 mon.a (mon.0) 2339 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:22:59.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:22:59 smithi181 bash[67587]: cluster 2024-09-13T00:22:58.084925+0000 mon.a (mon.0) 2340 : cluster [DBG] osdmap e1148: 8 total, 8 up, 8 in 2024-09-13T00:22:59.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:59 smithi136 bash[75164]: cluster 2024-09-13T00:22:57.786230+0000 mgr.y (mgr.24661) 1375 : cluster [DBG] pgmap v2083: 137 pgs: 2 creating+peering, 30 unknown, 105 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:22:59.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:59 smithi136 bash[75164]: cluster 2024-09-13T00:22:58.058603+0000 mon.a (mon.0) 2338 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:22:59.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:59 smithi136 bash[75164]: audit 2024-09-13T00:22:58.084445+0000 mon.a (mon.0) 2339 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:22:59.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:22:59 smithi136 bash[75164]: cluster 2024-09-13T00:22:58.084925+0000 mon.a (mon.0) 2340 : cluster [DBG] osdmap e1148: 8 total, 8 up, 8 in 2024-09-13T00:22:59.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:59 smithi136 bash[76371]: cluster 2024-09-13T00:22:57.786230+0000 mgr.y (mgr.24661) 1375 : cluster [DBG] pgmap v2083: 137 pgs: 2 creating+peering, 30 unknown, 105 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:22:59.414 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:59 smithi136 bash[76371]: cluster 2024-09-13T00:22:58.058603+0000 mon.a (mon.0) 2338 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:22:59.414 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:59 smithi136 bash[76371]: audit 2024-09-13T00:22:58.084445+0000 mon.a (mon.0) 2339 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:22:59.414 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:22:59 smithi136 bash[76371]: cluster 2024-09-13T00:22:58.084925+0000 mon.a (mon.0) 2340 : cluster [DBG] osdmap e1148: 8 total, 8 up, 8 in 2024-09-13T00:22:59.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:22:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:22:59] "GET /metrics HTTP/1.1" 200 36491 "" "Prometheus/2.43.0" 2024-09-13T00:23:00.302 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestTrash.UserRemovingSource (2215 ms) 2024-09-13T00:23:00.302 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestTrash.RestoreMirroringSource 2024-09-13T00:23:00.404 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestTrash.RestoreMirroringSource (102 ms) 2024-09-13T00:23:00.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:00 smithi136 bash[75164]: cluster 2024-09-13T00:22:59.081657+0000 mon.a (mon.0) 2341 : cluster [DBG] osdmap e1149: 8 total, 8 up, 8 in 2024-09-13T00:23:00.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:00 smithi136 bash[76371]: cluster 2024-09-13T00:22:59.081657+0000 mon.a (mon.0) 2341 : cluster [DBG] osdmap e1149: 8 total, 8 up, 8 in 2024-09-13T00:23:00.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:00 smithi181 bash[67587]: cluster 2024-09-13T00:22:59.081657+0000 mon.a (mon.0) 2341 : cluster [DBG] osdmap e1149: 8 total, 8 up, 8 in 2024-09-13T00:23:01.125 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 2 tests from TestTrash (2317 ms total) 2024-09-13T00:23:01.126 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:23:01.126 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 1 test from TestOperations 2024-09-13T00:23:01.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:01 smithi136 bash[75164]: cluster 2024-09-13T00:22:59.786976+0000 mgr.y (mgr.24661) 1376 : cluster [DBG] pgmap v2086: 137 pgs: 2 creating+peering, 20 unknown, 115 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:23:01.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:01 smithi136 bash[75164]: cluster 2024-09-13T00:23:00.086499+0000 mon.a (mon.0) 2342 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:23:01.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:01 smithi136 bash[75164]: cluster 2024-09-13T00:23:00.086549+0000 mon.a (mon.0) 2343 : cluster [INF] Cluster is now healthy 2024-09-13T00:23:01.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:01 smithi136 bash[75164]: cluster 2024-09-13T00:23:00.108882+0000 mon.a (mon.0) 2344 : cluster [DBG] osdmap e1150: 8 total, 8 up, 8 in 2024-09-13T00:23:01.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:01 smithi136 bash[76371]: cluster 2024-09-13T00:22:59.786976+0000 mgr.y (mgr.24661) 1376 : cluster [DBG] pgmap v2086: 137 pgs: 2 creating+peering, 20 unknown, 115 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:23:01.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:01 smithi136 bash[76371]: cluster 2024-09-13T00:23:00.086499+0000 mon.a (mon.0) 2342 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:23:01.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:01 smithi136 bash[76371]: cluster 2024-09-13T00:23:00.086549+0000 mon.a (mon.0) 2343 : cluster [INF] Cluster is now healthy 2024-09-13T00:23:01.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:01 smithi136 bash[76371]: cluster 2024-09-13T00:23:00.108882+0000 mon.a (mon.0) 2344 : cluster [DBG] osdmap e1150: 8 total, 8 up, 8 in 2024-09-13T00:23:01.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:01 smithi181 bash[67587]: cluster 2024-09-13T00:22:59.786976+0000 mgr.y (mgr.24661) 1376 : cluster [DBG] pgmap v2086: 137 pgs: 2 creating+peering, 20 unknown, 115 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:23:01.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:01 smithi181 bash[67587]: cluster 2024-09-13T00:23:00.086499+0000 mon.a (mon.0) 2342 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:23:01.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:01 smithi181 bash[67587]: cluster 2024-09-13T00:23:00.086549+0000 mon.a (mon.0) 2343 : cluster [INF] Cluster is now healthy 2024-09-13T00:23:01.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:01 smithi181 bash[67587]: cluster 2024-09-13T00:23:00.108882+0000 mon.a (mon.0) 2344 : cluster [DBG] osdmap e1150: 8 total, 8 up, 8 in 2024-09-13T00:23:02.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:02 smithi136 bash[75164]: cluster 2024-09-13T00:23:01.116540+0000 mon.a (mon.0) 2345 : cluster [DBG] osdmap e1151: 8 total, 8 up, 8 in 2024-09-13T00:23:02.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:02 smithi136 bash[76371]: cluster 2024-09-13T00:23:01.116540+0000 mon.a (mon.0) 2345 : cluster [DBG] osdmap e1151: 8 total, 8 up, 8 in 2024-09-13T00:23:02.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:02 smithi181 bash[67587]: cluster 2024-09-13T00:23:01.116540+0000 mon.a (mon.0) 2345 : cluster [DBG] osdmap e1151: 8 total, 8 up, 8 in 2024-09-13T00:23:02.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:23:02 smithi181 bash[95505]: ts=2024-09-13T00:23:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:23:03.151 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestOperations.DisableJournalingCorrupt 2024-09-13T00:23:03.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:03 smithi136 bash[75164]: cluster 2024-09-13T00:23:01.787758+0000 mgr.y (mgr.24661) 1377 : cluster [DBG] pgmap v2089: 105 pgs: 105 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:23:03.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:03 smithi136 bash[75164]: cluster 2024-09-13T00:23:02.136223+0000 mon.a (mon.0) 2346 : cluster [DBG] osdmap e1152: 8 total, 8 up, 8 in 2024-09-13T00:23:03.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:03 smithi136 bash[75164]: audit 2024-09-13T00:23:02.139312+0000 mon.c (mon.2) 98 : audit [INF] from='client.? 172.21.15.136:0/1012677033' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:23:03.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:03 smithi136 bash[75164]: audit 2024-09-13T00:23:02.141237+0000 mon.a (mon.0) 2347 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:23:03.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:03 smithi136 bash[76371]: cluster 2024-09-13T00:23:01.787758+0000 mgr.y (mgr.24661) 1377 : cluster [DBG] pgmap v2089: 105 pgs: 105 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:23:03.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:03 smithi136 bash[76371]: cluster 2024-09-13T00:23:02.136223+0000 mon.a (mon.0) 2346 : cluster [DBG] osdmap e1152: 8 total, 8 up, 8 in 2024-09-13T00:23:03.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:03 smithi136 bash[76371]: audit 2024-09-13T00:23:02.139312+0000 mon.c (mon.2) 98 : audit [INF] from='client.? 172.21.15.136:0/1012677033' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:23:03.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:03 smithi136 bash[76371]: audit 2024-09-13T00:23:02.141237+0000 mon.a (mon.0) 2347 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:23:03.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:03 smithi181 bash[67587]: cluster 2024-09-13T00:23:01.787758+0000 mgr.y (mgr.24661) 1377 : cluster [DBG] pgmap v2089: 105 pgs: 105 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:23:03.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:03 smithi181 bash[67587]: cluster 2024-09-13T00:23:02.136223+0000 mon.a (mon.0) 2346 : cluster [DBG] osdmap e1152: 8 total, 8 up, 8 in 2024-09-13T00:23:03.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:03 smithi181 bash[67587]: audit 2024-09-13T00:23:02.139312+0000 mon.c (mon.2) 98 : audit [INF] from='client.? 172.21.15.136:0/1012677033' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:23:03.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:03 smithi181 bash[67587]: audit 2024-09-13T00:23:02.141237+0000 mon.a (mon.0) 2347 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:23:04.512 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:04 smithi136 bash[75164]: audit 2024-09-13T00:23:03.135404+0000 mon.a (mon.0) 2348 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:23:04.513 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:04 smithi136 bash[75164]: cluster 2024-09-13T00:23:03.135827+0000 mon.a (mon.0) 2349 : cluster [DBG] osdmap e1153: 8 total, 8 up, 8 in 2024-09-13T00:23:04.513 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:04 smithi136 bash[76371]: audit 2024-09-13T00:23:03.135404+0000 mon.a (mon.0) 2348 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:23:04.513 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:04 smithi136 bash[76371]: cluster 2024-09-13T00:23:03.135827+0000 mon.a (mon.0) 2349 : cluster [DBG] osdmap e1153: 8 total, 8 up, 8 in 2024-09-13T00:23:04.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:04 smithi181 bash[67587]: audit 2024-09-13T00:23:03.135404+0000 mon.a (mon.0) 2348 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:23:04.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:04 smithi181 bash[67587]: cluster 2024-09-13T00:23:03.135827+0000 mon.a (mon.0) 2349 : cluster [DBG] osdmap e1153: 8 total, 8 up, 8 in 2024-09-13T00:23:05.194 INFO:tasks.workunit.client.0.smithi136.stdout:SKIPPING 2024-09-13T00:23:05.194 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestOperations.DisableJournalingCorrupt (2043 ms) 2024-09-13T00:23:05.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:05 smithi136 bash[75164]: cluster 2024-09-13T00:23:03.788621+0000 mgr.y (mgr.24661) 1378 : cluster [DBG] pgmap v2092: 137 pgs: 4 creating+activating, 2 creating+peering, 22 unknown, 109 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:23:05.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:05 smithi136 bash[75164]: cluster 2024-09-13T00:23:04.156958+0000 mon.a (mon.0) 2350 : cluster [DBG] osdmap e1154: 8 total, 8 up, 8 in 2024-09-13T00:23:05.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:05 smithi136 bash[76371]: cluster 2024-09-13T00:23:03.788621+0000 mgr.y (mgr.24661) 1378 : cluster [DBG] pgmap v2092: 137 pgs: 4 creating+activating, 2 creating+peering, 22 unknown, 109 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:23:05.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:05 smithi136 bash[76371]: cluster 2024-09-13T00:23:04.156958+0000 mon.a (mon.0) 2350 : cluster [DBG] osdmap e1154: 8 total, 8 up, 8 in 2024-09-13T00:23:05.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:05 smithi181 bash[67587]: cluster 2024-09-13T00:23:03.788621+0000 mgr.y (mgr.24661) 1378 : cluster [DBG] pgmap v2092: 137 pgs: 4 creating+activating, 2 creating+peering, 22 unknown, 109 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:23:05.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:05 smithi181 bash[67587]: cluster 2024-09-13T00:23:04.156958+0000 mon.a (mon.0) 2350 : cluster [DBG] osdmap e1154: 8 total, 8 up, 8 in 2024-09-13T00:23:06.192 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 1 test from TestOperations (2043 ms total) 2024-09-13T00:23:06.193 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:23:06.193 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 4 tests from TestObjectMap 2024-09-13T00:23:06.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:06 smithi136 bash[75164]: cluster 2024-09-13T00:23:05.158924+0000 mon.a (mon.0) 2351 : cluster [DBG] osdmap e1155: 8 total, 8 up, 8 in 2024-09-13T00:23:06.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:06 smithi136 bash[76371]: cluster 2024-09-13T00:23:05.158924+0000 mon.a (mon.0) 2351 : cluster [DBG] osdmap e1155: 8 total, 8 up, 8 in 2024-09-13T00:23:06.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:06 smithi181 bash[67587]: cluster 2024-09-13T00:23:05.158924+0000 mon.a (mon.0) 2351 : cluster [DBG] osdmap e1155: 8 total, 8 up, 8 in 2024-09-13T00:23:07.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:07 smithi136 bash[75164]: cluster 2024-09-13T00:23:05.789565+0000 mgr.y (mgr.24661) 1379 : cluster [DBG] pgmap v2095: 137 pgs: 4 creating+activating, 2 creating+peering, 13 unknown, 118 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:23:07.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:07 smithi136 bash[75164]: cluster 2024-09-13T00:23:06.185316+0000 mon.a (mon.0) 2352 : cluster [DBG] osdmap e1156: 8 total, 8 up, 8 in 2024-09-13T00:23:07.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:07 smithi136 bash[76371]: cluster 2024-09-13T00:23:05.789565+0000 mgr.y (mgr.24661) 1379 : cluster [DBG] pgmap v2095: 137 pgs: 4 creating+activating, 2 creating+peering, 13 unknown, 118 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:23:07.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:07 smithi136 bash[76371]: cluster 2024-09-13T00:23:06.185316+0000 mon.a (mon.0) 2352 : cluster [DBG] osdmap e1156: 8 total, 8 up, 8 in 2024-09-13T00:23:07.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:07 smithi181 bash[67587]: cluster 2024-09-13T00:23:05.789565+0000 mgr.y (mgr.24661) 1379 : cluster [DBG] pgmap v2095: 137 pgs: 4 creating+activating, 2 creating+peering, 13 unknown, 118 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:23:07.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:07 smithi181 bash[67587]: cluster 2024-09-13T00:23:06.185316+0000 mon.a (mon.0) 2352 : cluster [DBG] osdmap e1156: 8 total, 8 up, 8 in 2024-09-13T00:23:08.215 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestObjectMap.RefreshInvalidatesWhenCorrupt 2024-09-13T00:23:08.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:08 smithi136 bash[75164]: cluster 2024-09-13T00:23:07.199107+0000 mon.a (mon.0) 2353 : cluster [DBG] osdmap e1157: 8 total, 8 up, 8 in 2024-09-13T00:23:08.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:08 smithi136 bash[75164]: audit 2024-09-13T00:23:07.203284+0000 mon.a (mon.0) 2354 : audit [INF] from='client.? 172.21.15.136:0/2846698260' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:23:08.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:08 smithi136 bash[76371]: cluster 2024-09-13T00:23:07.199107+0000 mon.a (mon.0) 2353 : cluster [DBG] osdmap e1157: 8 total, 8 up, 8 in 2024-09-13T00:23:08.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:08 smithi136 bash[76371]: audit 2024-09-13T00:23:07.203284+0000 mon.a (mon.0) 2354 : audit [INF] from='client.? 172.21.15.136:0/2846698260' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:23:08.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:08 smithi181 bash[67587]: cluster 2024-09-13T00:23:07.199107+0000 mon.a (mon.0) 2353 : cluster [DBG] osdmap e1157: 8 total, 8 up, 8 in 2024-09-13T00:23:08.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:08 smithi181 bash[67587]: audit 2024-09-13T00:23:07.203284+0000 mon.a (mon.0) 2354 : audit [INF] from='client.? 172.21.15.136:0/2846698260' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:23:09.509 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:23:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:23:09] "GET /metrics HTTP/1.1" 200 36493 "" "Prometheus/2.43.0" 2024-09-13T00:23:09.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:09 smithi136 bash[76371]: cluster 2024-09-13T00:23:07.790486+0000 mgr.y (mgr.24661) 1380 : cluster [DBG] pgmap v2098: 137 pgs: 10 creating+peering, 22 unknown, 105 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:23:09.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:09 smithi136 bash[76371]: cluster 2024-09-13T00:23:08.192502+0000 mon.a (mon.0) 2355 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:23:09.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:09 smithi136 bash[76371]: audit 2024-09-13T00:23:08.213316+0000 mon.a (mon.0) 2356 : audit [INF] from='client.? 172.21.15.136:0/2846698260' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:23:09.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:09 smithi136 bash[76371]: cluster 2024-09-13T00:23:08.213797+0000 mon.a (mon.0) 2357 : cluster [DBG] osdmap e1158: 8 total, 8 up, 8 in 2024-09-13T00:23:09.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:09 smithi136 bash[76371]: audit 2024-09-13T00:23:08.814858+0000 mon.a (mon.0) 2358 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:23:09.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:09 smithi136 bash[76371]: audit 2024-09-13T00:23:08.816859+0000 mon.a (mon.0) 2359 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:23:09.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:09 smithi136 bash[75164]: cluster 2024-09-13T00:23:07.790486+0000 mgr.y (mgr.24661) 1380 : cluster [DBG] pgmap v2098: 137 pgs: 10 creating+peering, 22 unknown, 105 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:23:09.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:09 smithi136 bash[75164]: cluster 2024-09-13T00:23:08.192502+0000 mon.a (mon.0) 2355 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:23:09.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:09 smithi136 bash[75164]: audit 2024-09-13T00:23:08.213316+0000 mon.a (mon.0) 2356 : audit [INF] from='client.? 172.21.15.136:0/2846698260' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:23:09.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:09 smithi136 bash[75164]: cluster 2024-09-13T00:23:08.213797+0000 mon.a (mon.0) 2357 : cluster [DBG] osdmap e1158: 8 total, 8 up, 8 in 2024-09-13T00:23:09.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:09 smithi136 bash[75164]: audit 2024-09-13T00:23:08.814858+0000 mon.a (mon.0) 2358 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:23:09.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:09 smithi136 bash[75164]: audit 2024-09-13T00:23:08.816859+0000 mon.a (mon.0) 2359 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:23:09.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:09 smithi181 bash[67587]: cluster 2024-09-13T00:23:07.790486+0000 mgr.y (mgr.24661) 1380 : cluster [DBG] pgmap v2098: 137 pgs: 10 creating+peering, 22 unknown, 105 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:23:09.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:09 smithi181 bash[67587]: cluster 2024-09-13T00:23:08.192502+0000 mon.a (mon.0) 2355 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:23:09.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:09 smithi181 bash[67587]: audit 2024-09-13T00:23:08.213316+0000 mon.a (mon.0) 2356 : audit [INF] from='client.? 172.21.15.136:0/2846698260' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:23:09.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:09 smithi181 bash[67587]: cluster 2024-09-13T00:23:08.213797+0000 mon.a (mon.0) 2357 : cluster [DBG] osdmap e1158: 8 total, 8 up, 8 in 2024-09-13T00:23:09.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:09 smithi181 bash[67587]: audit 2024-09-13T00:23:08.814858+0000 mon.a (mon.0) 2358 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:23:09.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:09 smithi181 bash[67587]: audit 2024-09-13T00:23:08.816859+0000 mon.a (mon.0) 2359 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:23:10.302 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestObjectMap.RefreshInvalidatesWhenCorrupt (2086 ms) 2024-09-13T00:23:10.302 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestObjectMap.RefreshInvalidatesWhenTooSmall 2024-09-13T00:23:10.347 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestObjectMap.RefreshInvalidatesWhenTooSmall (45 ms) 2024-09-13T00:23:10.347 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestObjectMap.InvalidateFlagOnDisk 2024-09-13T00:23:10.423 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestObjectMap.InvalidateFlagOnDisk (76 ms) 2024-09-13T00:23:10.423 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestObjectMap.AcquireLockInvalidatesWhenTooSmall 2024-09-13T00:23:10.491 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestObjectMap.AcquireLockInvalidatesWhenTooSmall (68 ms) 2024-09-13T00:23:10.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:10 smithi136 bash[76371]: cluster 2024-09-13T00:23:09.218793+0000 mon.a (mon.0) 2360 : cluster [DBG] osdmap e1159: 8 total, 8 up, 8 in 2024-09-13T00:23:10.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:10 smithi136 bash[75164]: cluster 2024-09-13T00:23:09.218793+0000 mon.a (mon.0) 2360 : cluster [DBG] osdmap e1159: 8 total, 8 up, 8 in 2024-09-13T00:23:10.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:10 smithi181 bash[67587]: cluster 2024-09-13T00:23:09.218793+0000 mon.a (mon.0) 2360 : cluster [DBG] osdmap e1159: 8 total, 8 up, 8 in 2024-09-13T00:23:11.236 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 4 tests from TestObjectMap (2275 ms total) 2024-09-13T00:23:11.236 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:23:11.236 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 2 tests from TestMirroringWatcher 2024-09-13T00:23:11.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:11 smithi136 bash[76371]: cluster 2024-09-13T00:23:09.791159+0000 mgr.y (mgr.24661) 1381 : cluster [DBG] pgmap v2101: 137 pgs: 10 creating+peering, 16 unknown, 111 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:23:11.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:11 smithi136 bash[76371]: cluster 2024-09-13T00:23:10.212165+0000 mon.a (mon.0) 2361 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:23:11.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:11 smithi136 bash[76371]: cluster 2024-09-13T00:23:10.212211+0000 mon.a (mon.0) 2362 : cluster [INF] Cluster is now healthy 2024-09-13T00:23:11.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:11 smithi136 bash[76371]: cluster 2024-09-13T00:23:10.229939+0000 mon.a (mon.0) 2363 : cluster [DBG] osdmap e1160: 8 total, 8 up, 8 in 2024-09-13T00:23:11.511 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:11 smithi136 bash[75164]: cluster 2024-09-13T00:23:09.791159+0000 mgr.y (mgr.24661) 1381 : cluster [DBG] pgmap v2101: 137 pgs: 10 creating+peering, 16 unknown, 111 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:23:11.511 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:11 smithi136 bash[75164]: cluster 2024-09-13T00:23:10.212165+0000 mon.a (mon.0) 2361 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:23:11.512 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:11 smithi136 bash[75164]: cluster 2024-09-13T00:23:10.212211+0000 mon.a (mon.0) 2362 : cluster [INF] Cluster is now healthy 2024-09-13T00:23:11.512 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:11 smithi136 bash[75164]: cluster 2024-09-13T00:23:10.229939+0000 mon.a (mon.0) 2363 : cluster [DBG] osdmap e1160: 8 total, 8 up, 8 in 2024-09-13T00:23:11.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:11 smithi181 bash[67587]: cluster 2024-09-13T00:23:09.791159+0000 mgr.y (mgr.24661) 1381 : cluster [DBG] pgmap v2101: 137 pgs: 10 creating+peering, 16 unknown, 111 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:23:11.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:11 smithi181 bash[67587]: cluster 2024-09-13T00:23:10.212165+0000 mon.a (mon.0) 2361 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:23:11.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:11 smithi181 bash[67587]: cluster 2024-09-13T00:23:10.212211+0000 mon.a (mon.0) 2362 : cluster [INF] Cluster is now healthy 2024-09-13T00:23:11.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:11 smithi181 bash[67587]: cluster 2024-09-13T00:23:10.229939+0000 mon.a (mon.0) 2363 : cluster [DBG] osdmap e1160: 8 total, 8 up, 8 in 2024-09-13T00:23:12.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:12 smithi136 bash[76371]: cluster 2024-09-13T00:23:11.230799+0000 mon.a (mon.0) 2364 : cluster [DBG] osdmap e1161: 8 total, 8 up, 8 in 2024-09-13T00:23:12.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:12 smithi136 bash[75164]: cluster 2024-09-13T00:23:11.230799+0000 mon.a (mon.0) 2364 : cluster [DBG] osdmap e1161: 8 total, 8 up, 8 in 2024-09-13T00:23:12.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:12 smithi181 bash[67587]: cluster 2024-09-13T00:23:11.230799+0000 mon.a (mon.0) 2364 : cluster [DBG] osdmap e1161: 8 total, 8 up, 8 in 2024-09-13T00:23:12.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:23:12 smithi181 bash[95505]: ts=2024-09-13T00:23:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:23:13.266 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMirroringWatcher.ModeUpdated 2024-09-13T00:23:13.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:13 smithi181 bash[67587]: cluster 2024-09-13T00:23:11.791951+0000 mgr.y (mgr.24661) 1382 : cluster [DBG] pgmap v2104: 105 pgs: 105 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:23:13.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:13 smithi181 bash[67587]: cluster 2024-09-13T00:23:12.259600+0000 mon.a (mon.0) 2365 : cluster [DBG] osdmap e1162: 8 total, 8 up, 8 in 2024-09-13T00:23:13.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:13 smithi181 bash[67587]: audit 2024-09-13T00:23:12.261512+0000 mon.a (mon.0) 2366 : audit [INF] from='client.? 172.21.15.136:0/1747854044' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:23:13.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:13 smithi136 bash[76371]: cluster 2024-09-13T00:23:11.791951+0000 mgr.y (mgr.24661) 1382 : cluster [DBG] pgmap v2104: 105 pgs: 105 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:23:13.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:13 smithi136 bash[76371]: cluster 2024-09-13T00:23:12.259600+0000 mon.a (mon.0) 2365 : cluster [DBG] osdmap e1162: 8 total, 8 up, 8 in 2024-09-13T00:23:13.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:13 smithi136 bash[76371]: audit 2024-09-13T00:23:12.261512+0000 mon.a (mon.0) 2366 : audit [INF] from='client.? 172.21.15.136:0/1747854044' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:23:13.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:13 smithi136 bash[75164]: cluster 2024-09-13T00:23:11.791951+0000 mgr.y (mgr.24661) 1382 : cluster [DBG] pgmap v2104: 105 pgs: 105 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:23:13.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:13 smithi136 bash[75164]: cluster 2024-09-13T00:23:12.259600+0000 mon.a (mon.0) 2365 : cluster [DBG] osdmap e1162: 8 total, 8 up, 8 in 2024-09-13T00:23:13.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:13 smithi136 bash[75164]: audit 2024-09-13T00:23:12.261512+0000 mon.a (mon.0) 2366 : audit [INF] from='client.? 172.21.15.136:0/1747854044' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:23:14.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:14 smithi181 bash[67587]: audit 2024-09-13T00:23:13.262067+0000 mon.a (mon.0) 2367 : audit [INF] from='client.? 172.21.15.136:0/1747854044' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:23:14.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:14 smithi181 bash[67587]: cluster 2024-09-13T00:23:13.264291+0000 mon.a (mon.0) 2368 : cluster [DBG] osdmap e1163: 8 total, 8 up, 8 in 2024-09-13T00:23:14.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:14 smithi136 bash[76371]: audit 2024-09-13T00:23:13.262067+0000 mon.a (mon.0) 2367 : audit [INF] from='client.? 172.21.15.136:0/1747854044' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:23:14.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:14 smithi136 bash[76371]: cluster 2024-09-13T00:23:13.264291+0000 mon.a (mon.0) 2368 : cluster [DBG] osdmap e1163: 8 total, 8 up, 8 in 2024-09-13T00:23:14.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:14 smithi136 bash[75164]: audit 2024-09-13T00:23:13.262067+0000 mon.a (mon.0) 2367 : audit [INF] from='client.? 172.21.15.136:0/1747854044' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:23:14.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:14 smithi136 bash[75164]: cluster 2024-09-13T00:23:13.264291+0000 mon.a (mon.0) 2368 : cluster [DBG] osdmap e1163: 8 total, 8 up, 8 in 2024-09-13T00:23:15.374 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMirroringWatcher.ModeUpdated (2108 ms) 2024-09-13T00:23:15.374 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMirroringWatcher.ImageStatusUpdated 2024-09-13T00:23:15.413 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMirroringWatcher.ImageStatusUpdated (39 ms) 2024-09-13T00:23:15.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:15 smithi181 bash[67587]: cluster 2024-09-13T00:23:13.792832+0000 mgr.y (mgr.24661) 1383 : cluster [DBG] pgmap v2107: 137 pgs: 4 creating+peering, 22 unknown, 111 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:23:15.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:15 smithi181 bash[67587]: cluster 2024-09-13T00:23:14.290589+0000 mon.a (mon.0) 2369 : cluster [DBG] osdmap e1164: 8 total, 8 up, 8 in 2024-09-13T00:23:15.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:15 smithi136 bash[75164]: cluster 2024-09-13T00:23:13.792832+0000 mgr.y (mgr.24661) 1383 : cluster [DBG] pgmap v2107: 137 pgs: 4 creating+peering, 22 unknown, 111 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:23:15.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:15 smithi136 bash[75164]: cluster 2024-09-13T00:23:14.290589+0000 mon.a (mon.0) 2369 : cluster [DBG] osdmap e1164: 8 total, 8 up, 8 in 2024-09-13T00:23:15.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:15 smithi136 bash[76371]: cluster 2024-09-13T00:23:13.792832+0000 mgr.y (mgr.24661) 1383 : cluster [DBG] pgmap v2107: 137 pgs: 4 creating+peering, 22 unknown, 111 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:23:15.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:15 smithi136 bash[76371]: cluster 2024-09-13T00:23:14.290589+0000 mon.a (mon.0) 2369 : cluster [DBG] osdmap e1164: 8 total, 8 up, 8 in 2024-09-13T00:23:16.317 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 2 tests from TestMirroringWatcher (2147 ms total) 2024-09-13T00:23:16.317 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:23:16.317 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 57 tests from TestMigration 2024-09-13T00:23:16.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:16 smithi181 bash[67587]: cluster 2024-09-13T00:23:15.299710+0000 mon.a (mon.0) 2370 : cluster [DBG] osdmap e1165: 8 total, 8 up, 8 in 2024-09-13T00:23:16.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:16 smithi181 bash[67587]: cluster 2024-09-13T00:23:15.793688+0000 mgr.y (mgr.24661) 1384 : cluster [DBG] pgmap v2110: 137 pgs: 4 creating+peering, 15 unknown, 118 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:23:16.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:16 smithi136 bash[75164]: cluster 2024-09-13T00:23:15.299710+0000 mon.a (mon.0) 2370 : cluster [DBG] osdmap e1165: 8 total, 8 up, 8 in 2024-09-13T00:23:16.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:16 smithi136 bash[75164]: cluster 2024-09-13T00:23:15.793688+0000 mgr.y (mgr.24661) 1384 : cluster [DBG] pgmap v2110: 137 pgs: 4 creating+peering, 15 unknown, 118 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:23:16.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:16 smithi136 bash[76371]: cluster 2024-09-13T00:23:15.299710+0000 mon.a (mon.0) 2370 : cluster [DBG] osdmap e1165: 8 total, 8 up, 8 in 2024-09-13T00:23:16.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:16 smithi136 bash[76371]: cluster 2024-09-13T00:23:15.793688+0000 mgr.y (mgr.24661) 1384 : cluster [DBG] pgmap v2110: 137 pgs: 4 creating+peering, 15 unknown, 118 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:23:17.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:17 smithi181 bash[67587]: cluster 2024-09-13T00:23:16.309473+0000 mon.a (mon.0) 2371 : cluster [DBG] osdmap e1166: 8 total, 8 up, 8 in 2024-09-13T00:23:17.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:17 smithi181 bash[67587]: audit 2024-09-13T00:23:17.135985+0000 mon.a (mon.0) 2372 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:23:17.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:17 smithi136 bash[76371]: cluster 2024-09-13T00:23:16.309473+0000 mon.a (mon.0) 2371 : cluster [DBG] osdmap e1166: 8 total, 8 up, 8 in 2024-09-13T00:23:17.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:17 smithi136 bash[76371]: audit 2024-09-13T00:23:17.135985+0000 mon.a (mon.0) 2372 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:23:17.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:17 smithi136 bash[75164]: cluster 2024-09-13T00:23:16.309473+0000 mon.a (mon.0) 2371 : cluster [DBG] osdmap e1166: 8 total, 8 up, 8 in 2024-09-13T00:23:17.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:17 smithi136 bash[75164]: audit 2024-09-13T00:23:17.135985+0000 mon.a (mon.0) 2372 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:23:19.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:18 smithi181 bash[67587]: cluster 2024-09-13T00:23:17.319024+0000 mon.a (mon.0) 2373 : cluster [DBG] osdmap e1167: 8 total, 8 up, 8 in 2024-09-13T00:23:19.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:18 smithi181 bash[67587]: audit 2024-09-13T00:23:17.322191+0000 mon.a (mon.0) 2374 : audit [INF] from='client.? 172.21.15.136:0/1823895149' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:23:19.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:18 smithi181 bash[67587]: audit 2024-09-13T00:23:17.724557+0000 mon.a (mon.0) 2375 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:23:19.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:18 smithi181 bash[67587]: audit 2024-09-13T00:23:17.726198+0000 mon.a (mon.0) 2376 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:23:19.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:18 smithi181 bash[67587]: audit 2024-09-13T00:23:17.735363+0000 mon.a (mon.0) 2377 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:23:19.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:18 smithi181 bash[67587]: cluster 2024-09-13T00:23:17.794531+0000 mgr.y (mgr.24661) 1385 : cluster [DBG] pgmap v2113: 137 pgs: 2 creating+peering, 30 unknown, 105 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:23:19.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:18 smithi136 bash[75164]: cluster 2024-09-13T00:23:17.319024+0000 mon.a (mon.0) 2373 : cluster [DBG] osdmap e1167: 8 total, 8 up, 8 in 2024-09-13T00:23:19.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:18 smithi136 bash[75164]: audit 2024-09-13T00:23:17.322191+0000 mon.a (mon.0) 2374 : audit [INF] from='client.? 172.21.15.136:0/1823895149' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:23:19.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:18 smithi136 bash[75164]: audit 2024-09-13T00:23:17.724557+0000 mon.a (mon.0) 2375 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:23:19.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:18 smithi136 bash[75164]: audit 2024-09-13T00:23:17.726198+0000 mon.a (mon.0) 2376 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:23:19.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:18 smithi136 bash[75164]: audit 2024-09-13T00:23:17.735363+0000 mon.a (mon.0) 2377 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:23:19.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:18 smithi136 bash[75164]: cluster 2024-09-13T00:23:17.794531+0000 mgr.y (mgr.24661) 1385 : cluster [DBG] pgmap v2113: 137 pgs: 2 creating+peering, 30 unknown, 105 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:23:19.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:18 smithi136 bash[76371]: cluster 2024-09-13T00:23:17.319024+0000 mon.a (mon.0) 2373 : cluster [DBG] osdmap e1167: 8 total, 8 up, 8 in 2024-09-13T00:23:19.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:18 smithi136 bash[76371]: audit 2024-09-13T00:23:17.322191+0000 mon.a (mon.0) 2374 : audit [INF] from='client.? 172.21.15.136:0/1823895149' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:23:19.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:18 smithi136 bash[76371]: audit 2024-09-13T00:23:17.724557+0000 mon.a (mon.0) 2375 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:23:19.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:18 smithi136 bash[76371]: audit 2024-09-13T00:23:17.726198+0000 mon.a (mon.0) 2376 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:23:19.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:18 smithi136 bash[76371]: audit 2024-09-13T00:23:17.735363+0000 mon.a (mon.0) 2377 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:23:19.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:18 smithi136 bash[76371]: cluster 2024-09-13T00:23:17.794531+0000 mgr.y (mgr.24661) 1385 : cluster [DBG] pgmap v2113: 137 pgs: 2 creating+peering, 30 unknown, 105 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:23:19.328 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.Empty 2024-09-13T00:23:19.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:23:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:23:19] "GET /metrics HTTP/1.1" 200 36493 "" "Prometheus/2.43.0" 2024-09-13T00:23:20.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:19 smithi181 bash[67587]: audit 2024-09-13T00:23:18.325695+0000 mon.a (mon.0) 2378 : audit [INF] from='client.? 172.21.15.136:0/1823895149' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:23:20.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:19 smithi181 bash[67587]: cluster 2024-09-13T00:23:18.326120+0000 mon.a (mon.0) 2379 : cluster [DBG] osdmap e1168: 8 total, 8 up, 8 in 2024-09-13T00:23:20.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:19 smithi181 bash[67587]: cluster 2024-09-13T00:23:18.777510+0000 mon.a (mon.0) 2380 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:23:20.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:19 smithi181 bash[67587]: cluster 2024-09-13T00:23:19.327130+0000 mon.a (mon.0) 2381 : cluster [DBG] osdmap e1169: 8 total, 8 up, 8 in 2024-09-13T00:23:20.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:19 smithi136 bash[75164]: audit 2024-09-13T00:23:18.325695+0000 mon.a (mon.0) 2378 : audit [INF] from='client.? 172.21.15.136:0/1823895149' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:23:20.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:19 smithi136 bash[75164]: cluster 2024-09-13T00:23:18.326120+0000 mon.a (mon.0) 2379 : cluster [DBG] osdmap e1168: 8 total, 8 up, 8 in 2024-09-13T00:23:20.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:19 smithi136 bash[75164]: cluster 2024-09-13T00:23:18.777510+0000 mon.a (mon.0) 2380 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:23:20.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:19 smithi136 bash[75164]: cluster 2024-09-13T00:23:19.327130+0000 mon.a (mon.0) 2381 : cluster [DBG] osdmap e1169: 8 total, 8 up, 8 in 2024-09-13T00:23:20.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:19 smithi136 bash[76371]: audit 2024-09-13T00:23:18.325695+0000 mon.a (mon.0) 2378 : audit [INF] from='client.? 172.21.15.136:0/1823895149' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:23:20.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:19 smithi136 bash[76371]: cluster 2024-09-13T00:23:18.326120+0000 mon.a (mon.0) 2379 : cluster [DBG] osdmap e1168: 8 total, 8 up, 8 in 2024-09-13T00:23:20.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:19 smithi136 bash[76371]: cluster 2024-09-13T00:23:18.777510+0000 mon.a (mon.0) 2380 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:23:20.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:19 smithi136 bash[76371]: cluster 2024-09-13T00:23:19.327130+0000 mon.a (mon.0) 2381 : cluster [DBG] osdmap e1169: 8 total, 8 up, 8 in 2024-09-13T00:23:21.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:20 smithi181 bash[67587]: cluster 2024-09-13T00:23:19.795378+0000 mgr.y (mgr.24661) 1386 : cluster [DBG] pgmap v2116: 169 pgs: 4 creating+peering, 48 unknown, 117 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:23:21.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:20 smithi181 bash[67587]: cluster 2024-09-13T00:23:20.329832+0000 mon.a (mon.0) 2382 : cluster [DBG] osdmap e1170: 8 total, 8 up, 8 in 2024-09-13T00:23:21.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:20 smithi136 bash[75164]: cluster 2024-09-13T00:23:19.795378+0000 mgr.y (mgr.24661) 1386 : cluster [DBG] pgmap v2116: 169 pgs: 4 creating+peering, 48 unknown, 117 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:23:21.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:20 smithi136 bash[75164]: cluster 2024-09-13T00:23:20.329832+0000 mon.a (mon.0) 2382 : cluster [DBG] osdmap e1170: 8 total, 8 up, 8 in 2024-09-13T00:23:21.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:20 smithi136 bash[76371]: cluster 2024-09-13T00:23:19.795378+0000 mgr.y (mgr.24661) 1386 : cluster [DBG] pgmap v2116: 169 pgs: 4 creating+peering, 48 unknown, 117 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:23:21.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:20 smithi136 bash[76371]: cluster 2024-09-13T00:23:20.329832+0000 mon.a (mon.0) 2382 : cluster [DBG] osdmap e1170: 8 total, 8 up, 8 in 2024-09-13T00:23:21.421 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:23:21.445 INFO:tasks.workunit.client.0.smithi136.stdout:migration_prepare 2024-09-13T00:23:21.813 INFO:tasks.workunit.client.0.smithi136.stdout:migration_execute 2024-09-13T00:23:22.016 INFO:tasks.workunit.client.0.smithi136.stdout:migration_commit 2024-09-13T00:23:22.216 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.Empty (2888 ms) 2024-09-13T00:23:22.216 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.OtherName 2024-09-13T00:23:22.294 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:23:22.321 INFO:tasks.workunit.client.0.smithi136.stdout:migration_prepare 2024-09-13T00:23:22.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:22 smithi181 bash[67587]: cluster 2024-09-13T00:23:21.332600+0000 mon.a (mon.0) 2383 : cluster [DBG] osdmap e1171: 8 total, 8 up, 8 in 2024-09-13T00:23:22.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:22 smithi181 bash[67587]: cluster 2024-09-13T00:23:21.796487+0000 mgr.y (mgr.24661) 1387 : cluster [DBG] pgmap v2119: 169 pgs: 4 creating+peering, 12 unknown, 153 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 2.7 KiB/s rd, 1.2 KiB/s wr, 7 op/s 2024-09-13T00:23:22.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:23:22 smithi181 bash[95505]: ts=2024-09-13T00:23:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:23:22.664 INFO:tasks.workunit.client.0.smithi136.stdout:migration_execute 2024-09-13T00:23:22.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:22 smithi136 bash[75164]: cluster 2024-09-13T00:23:21.332600+0000 mon.a (mon.0) 2383 : cluster [DBG] osdmap e1171: 8 total, 8 up, 8 in 2024-09-13T00:23:22.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:22 smithi136 bash[75164]: cluster 2024-09-13T00:23:21.796487+0000 mgr.y (mgr.24661) 1387 : cluster [DBG] pgmap v2119: 169 pgs: 4 creating+peering, 12 unknown, 153 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 2.7 KiB/s rd, 1.2 KiB/s wr, 7 op/s 2024-09-13T00:23:22.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:22 smithi136 bash[76371]: cluster 2024-09-13T00:23:21.332600+0000 mon.a (mon.0) 2383 : cluster [DBG] osdmap e1171: 8 total, 8 up, 8 in 2024-09-13T00:23:22.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:22 smithi136 bash[76371]: cluster 2024-09-13T00:23:21.796487+0000 mgr.y (mgr.24661) 1387 : cluster [DBG] pgmap v2119: 169 pgs: 4 creating+peering, 12 unknown, 153 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 2.7 KiB/s rd, 1.2 KiB/s wr, 7 op/s 2024-09-13T00:23:22.839 INFO:tasks.workunit.client.0.smithi136.stdout:migration_commit 2024-09-13T00:23:23.034 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.OtherName (818 ms) 2024-09-13T00:23:23.035 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.OtherPool 2024-09-13T00:23:23.114 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:23:23.142 INFO:tasks.workunit.client.0.smithi136.stdout:migration_prepare 2024-09-13T00:23:24.647 INFO:tasks.workunit.client.0.smithi136.stdout:migration_execute 2024-09-13T00:23:24.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:24 smithi136 bash[75164]: cluster 2024-09-13T00:23:23.346781+0000 mon.a (mon.0) 2384 : cluster [DBG] osdmap e1172: 8 total, 8 up, 8 in 2024-09-13T00:23:24.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:24 smithi136 bash[75164]: cluster 2024-09-13T00:23:23.797455+0000 mgr.y (mgr.24661) 1388 : cluster [DBG] pgmap v2121: 169 pgs: 2 creating+peering, 167 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 122 KiB/s rd, 6.7 KiB/s wr, 155 op/s 2024-09-13T00:23:24.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:24 smithi136 bash[75164]: audit 2024-09-13T00:23:23.808810+0000 mon.a (mon.0) 2385 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:23:24.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:24 smithi136 bash[76371]: cluster 2024-09-13T00:23:23.346781+0000 mon.a (mon.0) 2384 : cluster [DBG] osdmap e1172: 8 total, 8 up, 8 in 2024-09-13T00:23:24.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:24 smithi136 bash[76371]: cluster 2024-09-13T00:23:23.797455+0000 mgr.y (mgr.24661) 1388 : cluster [DBG] pgmap v2121: 169 pgs: 2 creating+peering, 167 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 122 KiB/s rd, 6.7 KiB/s wr, 155 op/s 2024-09-13T00:23:24.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:24 smithi136 bash[76371]: audit 2024-09-13T00:23:23.808810+0000 mon.a (mon.0) 2385 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:23:24.828 INFO:tasks.workunit.client.0.smithi136.stdout:migration_commit 2024-09-13T00:23:24.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:24 smithi181 bash[67587]: cluster 2024-09-13T00:23:23.346781+0000 mon.a (mon.0) 2384 : cluster [DBG] osdmap e1172: 8 total, 8 up, 8 in 2024-09-13T00:23:24.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:24 smithi181 bash[67587]: cluster 2024-09-13T00:23:23.797455+0000 mgr.y (mgr.24661) 1388 : cluster [DBG] pgmap v2121: 169 pgs: 2 creating+peering, 167 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 122 KiB/s rd, 6.7 KiB/s wr, 155 op/s 2024-09-13T00:23:24.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:24 smithi181 bash[67587]: audit 2024-09-13T00:23:23.808810+0000 mon.a (mon.0) 2385 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:23:25.034 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.OtherPool (1999 ms) 2024-09-13T00:23:25.034 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.OtherNamespace 2024-09-13T00:23:25.116 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:23:25.149 INFO:tasks.workunit.client.0.smithi136.stdout:migration_prepare 2024-09-13T00:23:25.496 INFO:tasks.workunit.client.0.smithi136.stdout:migration_execute 2024-09-13T00:23:25.688 INFO:tasks.workunit.client.0.smithi136.stdout:migration_commit 2024-09-13T00:23:25.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:25 smithi136 bash[75164]: cluster 2024-09-13T00:23:24.358309+0000 mon.a (mon.0) 2386 : cluster [DBG] osdmap e1173: 8 total, 8 up, 8 in 2024-09-13T00:23:25.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:25 smithi136 bash[76371]: cluster 2024-09-13T00:23:24.358309+0000 mon.a (mon.0) 2386 : cluster [DBG] osdmap e1173: 8 total, 8 up, 8 in 2024-09-13T00:23:25.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:25 smithi181 bash[67587]: cluster 2024-09-13T00:23:24.358309+0000 mon.a (mon.0) 2386 : cluster [DBG] osdmap e1173: 8 total, 8 up, 8 in 2024-09-13T00:23:25.897 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.OtherNamespace (863 ms) 2024-09-13T00:23:25.897 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.DataPool 2024-09-13T00:23:25.982 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:23:26.011 INFO:tasks.workunit.client.0.smithi136.stdout:migration_prepare 2024-09-13T00:23:26.395 INFO:tasks.workunit.client.0.smithi136.stdout:migration_execute 2024-09-13T00:23:26.599 INFO:tasks.workunit.client.0.smithi136.stdout:migration_commit 2024-09-13T00:23:26.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:26 smithi136 bash[75164]: cluster 2024-09-13T00:23:25.798641+0000 mgr.y (mgr.24661) 1389 : cluster [DBG] pgmap v2123: 169 pgs: 169 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 223 KiB/s rd, 13 KiB/s wr, 281 op/s 2024-09-13T00:23:26.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:26 smithi136 bash[76371]: cluster 2024-09-13T00:23:25.798641+0000 mgr.y (mgr.24661) 1389 : cluster [DBG] pgmap v2123: 169 pgs: 169 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 223 KiB/s rd, 13 KiB/s wr, 281 op/s 2024-09-13T00:23:26.806 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.DataPool (909 ms) 2024-09-13T00:23:26.806 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.AbortAfterPrepare 2024-09-13T00:23:26.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:26 smithi181 bash[67587]: cluster 2024-09-13T00:23:25.798641+0000 mgr.y (mgr.24661) 1389 : cluster [DBG] pgmap v2123: 169 pgs: 169 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 223 KiB/s rd, 13 KiB/s wr, 281 op/s 2024-09-13T00:23:26.888 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:23:26.918 INFO:tasks.workunit.client.0.smithi136.stdout:migration_prepare 2024-09-13T00:23:27.293 INFO:tasks.workunit.client.0.smithi136.stdout:migration_abort 2024-09-13T00:23:27.540 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.AbortAfterPrepare (735 ms) 2024-09-13T00:23:27.541 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.AbortAfterFailedPrepare 2024-09-13T00:23:27.619 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:23:27.647 INFO:tasks.workunit.client.0.smithi136.stdout:migration_prepare 2024-09-13T00:23:27.906 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.AbortAfterFailedPrepare (365 ms) 2024-09-13T00:23:27.906 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.AbortAfterExecute 2024-09-13T00:23:27.988 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:23:28.015 INFO:tasks.workunit.client.0.smithi136.stdout:migration_prepare 2024-09-13T00:23:28.388 INFO:tasks.workunit.client.0.smithi136.stdout:migration_execute 2024-09-13T00:23:28.588 INFO:tasks.workunit.client.0.smithi136.stdout:migration_abort 2024-09-13T00:23:28.834 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.AbortAfterExecute (928 ms) 2024-09-13T00:23:28.834 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.OtherPoolAbortAfterExecute 2024-09-13T00:23:28.952 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:23:28.985 INFO:tasks.workunit.client.0.smithi136.stdout:migration_prepare 2024-09-13T00:23:29.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:28 smithi136 bash[75164]: cluster 2024-09-13T00:23:27.800248+0000 mgr.y (mgr.24661) 1390 : cluster [DBG] pgmap v2124: 169 pgs: 169 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 374 KiB/s rd, 21 KiB/s wr, 470 op/s 2024-09-13T00:23:29.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:28 smithi136 bash[76371]: cluster 2024-09-13T00:23:27.800248+0000 mgr.y (mgr.24661) 1390 : cluster [DBG] pgmap v2124: 169 pgs: 169 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 374 KiB/s rd, 21 KiB/s wr, 470 op/s 2024-09-13T00:23:29.325 INFO:tasks.workunit.client.0.smithi136.stdout:migration_execute 2024-09-13T00:23:29.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:28 smithi181 bash[67587]: cluster 2024-09-13T00:23:27.800248+0000 mgr.y (mgr.24661) 1390 : cluster [DBG] pgmap v2124: 169 pgs: 169 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 374 KiB/s rd, 21 KiB/s wr, 470 op/s 2024-09-13T00:23:29.511 INFO:tasks.workunit.client.0.smithi136.stdout:migration_abort 2024-09-13T00:23:29.754 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.OtherPoolAbortAfterExecute (920 ms) 2024-09-13T00:23:29.754 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.OtherNamespaceAbortAfterExecute 2024-09-13T00:23:29.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:23:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:23:29] "GET /metrics HTTP/1.1" 200 38899 "" "Prometheus/2.43.0" 2024-09-13T00:23:29.840 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:23:29.873 INFO:tasks.workunit.client.0.smithi136.stdout:migration_prepare 2024-09-13T00:23:30.213 INFO:tasks.workunit.client.0.smithi136.stdout:migration_execute 2024-09-13T00:23:30.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:29 smithi136 bash[75164]: cluster 2024-09-13T00:23:28.919090+0000 mon.a (mon.0) 2387 : cluster [DBG] osdmap e1174: 8 total, 8 up, 8 in 2024-09-13T00:23:30.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:29 smithi136 bash[76371]: cluster 2024-09-13T00:23:28.919090+0000 mon.a (mon.0) 2387 : cluster [DBG] osdmap e1174: 8 total, 8 up, 8 in 2024-09-13T00:23:30.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:29 smithi181 bash[67587]: cluster 2024-09-13T00:23:28.919090+0000 mon.a (mon.0) 2387 : cluster [DBG] osdmap e1174: 8 total, 8 up, 8 in 2024-09-13T00:23:30.386 INFO:tasks.workunit.client.0.smithi136.stdout:migration_abort 2024-09-13T00:23:30.659 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.OtherNamespaceAbortAfterExecute (905 ms) 2024-09-13T00:23:30.659 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.MirroringSamePool 2024-09-13T00:23:30.741 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:23:30.771 INFO:tasks.workunit.client.0.smithi136.stdout:SKIPPING 2024-09-13T00:23:30.790 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.MirroringSamePool (131 ms) 2024-09-13T00:23:30.790 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.MirroringAbort 2024-09-13T00:23:30.870 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:23:30.898 INFO:tasks.workunit.client.0.smithi136.stdout:SKIPPING 2024-09-13T00:23:30.914 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.MirroringAbort (124 ms) 2024-09-13T00:23:30.915 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.MirroringOtherPoolDisabled 2024-09-13T00:23:30.998 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:23:31.025 INFO:tasks.workunit.client.0.smithi136.stdout:SKIPPING 2024-09-13T00:23:31.043 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.MirroringOtherPoolDisabled (128 ms) 2024-09-13T00:23:31.043 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.MirroringOtherPoolEnabled 2024-09-13T00:23:31.123 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:23:31.151 INFO:tasks.workunit.client.0.smithi136.stdout:SKIPPING 2024-09-13T00:23:31.168 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.MirroringOtherPoolEnabled (125 ms) 2024-09-13T00:23:31.168 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.MirroringPool 2024-09-13T00:23:31.250 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:23:31.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:30 smithi136 bash[75164]: cluster 2024-09-13T00:23:29.801196+0000 mgr.y (mgr.24661) 1391 : cluster [DBG] pgmap v2126: 169 pgs: 169 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 513 KiB/s rd, 29 KiB/s wr, 643 op/s 2024-09-13T00:23:31.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:30 smithi136 bash[76371]: cluster 2024-09-13T00:23:29.801196+0000 mgr.y (mgr.24661) 1391 : cluster [DBG] pgmap v2126: 169 pgs: 169 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 513 KiB/s rd, 29 KiB/s wr, 643 op/s 2024-09-13T00:23:31.279 INFO:tasks.workunit.client.0.smithi136.stdout:SKIPPING 2024-09-13T00:23:31.295 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.MirroringPool (128 ms) 2024-09-13T00:23:31.295 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.Group 2024-09-13T00:23:31.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:30 smithi181 bash[67587]: cluster 2024-09-13T00:23:29.801196+0000 mgr.y (mgr.24661) 1391 : cluster [DBG] pgmap v2126: 169 pgs: 169 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 513 KiB/s rd, 29 KiB/s wr, 643 op/s 2024-09-13T00:23:31.378 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:23:31.429 INFO:tasks.workunit.client.0.smithi136.stdout:migration_prepare 2024-09-13T00:23:31.801 INFO:tasks.workunit.client.0.smithi136.stdout:migration_execute 2024-09-13T00:23:31.985 INFO:tasks.workunit.client.0.smithi136.stdout:migration_commit 2024-09-13T00:23:32.209 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.Group (913 ms) 2024-09-13T00:23:32.209 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.GroupAbort 2024-09-13T00:23:32.287 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:23:32.338 INFO:tasks.workunit.client.0.smithi136.stdout:migration_prepare 2024-09-13T00:23:32.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:23:32 smithi181 bash[95505]: ts=2024-09-13T00:23:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:23:32.705 INFO:tasks.workunit.client.0.smithi136.stdout:migration_abort 2024-09-13T00:23:32.984 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.GroupAbort (775 ms) 2024-09-13T00:23:32.984 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.NoSnaps 2024-09-13T00:23:33.064 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:23:33.093 INFO:tasks.workunit.client.0.smithi136.stdout:write: 1 0~8388609 2024-09-13T00:23:33.206 INFO:tasks.workunit.client.0.smithi136.stdout:write: 1 16777218~8388609 2024-09-13T00:23:33.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:32 smithi136 bash[75164]: cluster 2024-09-13T00:23:31.802914+0000 mgr.y (mgr.24661) 1392 : cluster [DBG] pgmap v2127: 169 pgs: 169 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 651 KiB/s rd, 38 KiB/s wr, 820 op/s 2024-09-13T00:23:33.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:32 smithi136 bash[76371]: cluster 2024-09-13T00:23:31.802914+0000 mgr.y (mgr.24661) 1392 : cluster [DBG] pgmap v2127: 169 pgs: 169 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 651 KiB/s rd, 38 KiB/s wr, 820 op/s 2024-09-13T00:23:33.292 INFO:tasks.workunit.client.0.smithi136.stdout:migration_prepare 2024-09-13T00:23:33.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:32 smithi181 bash[67587]: cluster 2024-09-13T00:23:31.802914+0000 mgr.y (mgr.24661) 1392 : cluster [DBG] pgmap v2127: 169 pgs: 169 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 651 KiB/s rd, 38 KiB/s wr, 820 op/s 2024-09-13T00:23:33.753 INFO:tasks.workunit.client.0.smithi136.stdout:migration_execute 2024-09-13T00:23:34.022 INFO:tasks.workunit.client.0.smithi136.stdout:migration_commit 2024-09-13T00:23:34.300 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.NoSnaps (1316 ms) 2024-09-13T00:23:34.300 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.NoSnapsOtherPool 2024-09-13T00:23:34.382 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:23:34.410 INFO:tasks.workunit.client.0.smithi136.stdout:write: 1 0~8388609 2024-09-13T00:23:34.501 INFO:tasks.workunit.client.0.smithi136.stdout:write: 1 16777218~8388609 2024-09-13T00:23:34.565 INFO:tasks.workunit.client.0.smithi136.stdout:write: 1 0~8388609 2024-09-13T00:23:34.580 INFO:tasks.workunit.client.0.smithi136.stdout:write: 1 16777218~8388609 2024-09-13T00:23:34.640 INFO:tasks.workunit.client.0.smithi136.stdout:migration_prepare 2024-09-13T00:23:35.031 INFO:tasks.workunit.client.0.smithi136.stdout:migration_execute 2024-09-13T00:23:35.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:34 smithi136 bash[76371]: cluster 2024-09-13T00:23:33.804174+0000 mgr.y (mgr.24661) 1393 : cluster [DBG] pgmap v2128: 169 pgs: 169 active+clean; 8.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 606 KiB/s rd, 470 KiB/s wr, 777 op/s 2024-09-13T00:23:35.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:34 smithi136 bash[76371]: cluster 2024-09-13T00:23:33.930700+0000 mon.a (mon.0) 2388 : cluster [DBG] osdmap e1175: 8 total, 8 up, 8 in 2024-09-13T00:23:35.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:34 smithi136 bash[75164]: cluster 2024-09-13T00:23:33.804174+0000 mgr.y (mgr.24661) 1393 : cluster [DBG] pgmap v2128: 169 pgs: 169 active+clean; 8.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 606 KiB/s rd, 470 KiB/s wr, 777 op/s 2024-09-13T00:23:35.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:34 smithi136 bash[75164]: cluster 2024-09-13T00:23:33.930700+0000 mon.a (mon.0) 2388 : cluster [DBG] osdmap e1175: 8 total, 8 up, 8 in 2024-09-13T00:23:35.268 INFO:tasks.workunit.client.0.smithi136.stdout:migration_commit 2024-09-13T00:23:35.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:34 smithi181 bash[67587]: cluster 2024-09-13T00:23:33.804174+0000 mgr.y (mgr.24661) 1393 : cluster [DBG] pgmap v2128: 169 pgs: 169 active+clean; 8.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 606 KiB/s rd, 470 KiB/s wr, 777 op/s 2024-09-13T00:23:35.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:34 smithi181 bash[67587]: cluster 2024-09-13T00:23:33.930700+0000 mon.a (mon.0) 2388 : cluster [DBG] osdmap e1175: 8 total, 8 up, 8 in 2024-09-13T00:23:35.534 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.NoSnapsOtherPool (1234 ms) 2024-09-13T00:23:35.534 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.NoSnapsDataPool 2024-09-13T00:23:35.613 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:23:35.641 INFO:tasks.workunit.client.0.smithi136.stdout:write: 1 0~8388609 2024-09-13T00:23:35.751 INFO:tasks.workunit.client.0.smithi136.stdout:write: 1 16777218~8388609 2024-09-13T00:23:35.818 INFO:tasks.workunit.client.0.smithi136.stdout:migration_prepare 2024-09-13T00:23:36.261 INFO:tasks.workunit.client.0.smithi136.stdout:migration_execute 2024-09-13T00:23:36.509 INFO:tasks.workunit.client.0.smithi136.stdout:migration_commit 2024-09-13T00:23:36.771 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.NoSnapsDataPool (1237 ms) 2024-09-13T00:23:36.771 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.NoSnapsShrinkAfterPrepare 2024-09-13T00:23:36.852 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:23:36.879 INFO:tasks.workunit.client.0.smithi136.stdout:write: 1 0~8388609 2024-09-13T00:23:36.966 INFO:tasks.workunit.client.0.smithi136.stdout:write: 1 16777218~8388609 2024-09-13T00:23:37.027 INFO:tasks.workunit.client.0.smithi136.stdout:migration_prepare 2024-09-13T00:23:37.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:36 smithi136 bash[76371]: cluster 2024-09-13T00:23:35.806323+0000 mgr.y (mgr.24661) 1394 : cluster [DBG] pgmap v2130: 169 pgs: 169 active+clean; 41 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 6.6 MiB/s rd, 6.5 MiB/s wr, 846 op/s 2024-09-13T00:23:37.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:36 smithi136 bash[75164]: cluster 2024-09-13T00:23:35.806323+0000 mgr.y (mgr.24661) 1394 : cluster [DBG] pgmap v2130: 169 pgs: 169 active+clean; 41 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 6.6 MiB/s rd, 6.5 MiB/s wr, 846 op/s 2024-09-13T00:23:37.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:36 smithi181 bash[67587]: cluster 2024-09-13T00:23:35.806323+0000 mgr.y (mgr.24661) 1394 : cluster [DBG] pgmap v2130: 169 pgs: 169 active+clean; 41 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 6.6 MiB/s rd, 6.5 MiB/s wr, 846 op/s 2024-09-13T00:23:37.473 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 41943040 2024-09-13T00:23:37.496 INFO:tasks.workunit.client.0.smithi136.stdout:migration_execute 2024-09-13T00:23:37.729 INFO:tasks.workunit.client.0.smithi136.stdout:migration_commit 2024-09-13T00:23:37.965 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.NoSnapsShrinkAfterPrepare (1194 ms) 2024-09-13T00:23:37.965 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.NoSnapsShrinkToZeroBeforePrepare 2024-09-13T00:23:38.051 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:23:38.080 INFO:tasks.workunit.client.0.smithi136.stdout:write: 1 0~8388609 2024-09-13T00:23:38.167 INFO:tasks.workunit.client.0.smithi136.stdout:write: 1 16777218~8388609 2024-09-13T00:23:38.246 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 0 2024-09-13T00:23:38.283 INFO:tasks.workunit.client.0.smithi136.stdout:migration_prepare 2024-09-13T00:23:38.577 INFO:tasks.workunit.client.0.smithi136.stdout:migration_execute 2024-09-13T00:23:38.771 INFO:tasks.workunit.client.0.smithi136.stdout:migration_commit 2024-09-13T00:23:38.909 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.NoSnapsShrinkToZeroBeforePrepare (945 ms) 2024-09-13T00:23:38.909 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.NoSnapsShrinkToZeroAfterPrepare 2024-09-13T00:23:38.997 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:23:39.027 INFO:tasks.workunit.client.0.smithi136.stdout:write: 1 0~8388609 2024-09-13T00:23:39.128 INFO:tasks.workunit.client.0.smithi136.stdout:write: 1 16777218~8388609 2024-09-13T00:23:39.215 INFO:tasks.workunit.client.0.smithi136.stdout:migration_prepare 2024-09-13T00:23:39.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:38 smithi136 bash[76371]: cluster 2024-09-13T00:23:37.808850+0000 mgr.y (mgr.24661) 1395 : cluster [DBG] pgmap v2131: 169 pgs: 169 active+clean; 89 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 19 MiB/s rd, 15 MiB/s wr, 1.06k op/s 2024-09-13T00:23:39.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:38 smithi136 bash[76371]: audit 2024-09-13T00:23:38.808945+0000 mon.a (mon.0) 2389 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:23:39.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:38 smithi136 bash[75164]: cluster 2024-09-13T00:23:37.808850+0000 mgr.y (mgr.24661) 1395 : cluster [DBG] pgmap v2131: 169 pgs: 169 active+clean; 89 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 19 MiB/s rd, 15 MiB/s wr, 1.06k op/s 2024-09-13T00:23:39.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:38 smithi136 bash[75164]: audit 2024-09-13T00:23:38.808945+0000 mon.a (mon.0) 2389 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:23:39.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:38 smithi181 bash[67587]: cluster 2024-09-13T00:23:37.808850+0000 mgr.y (mgr.24661) 1395 : cluster [DBG] pgmap v2131: 169 pgs: 169 active+clean; 89 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 19 MiB/s rd, 15 MiB/s wr, 1.06k op/s 2024-09-13T00:23:39.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:38 smithi181 bash[67587]: audit 2024-09-13T00:23:38.808945+0000 mon.a (mon.0) 2389 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:23:39.656 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 0 2024-09-13T00:23:39.689 INFO:tasks.workunit.client.0.smithi136.stdout:migration_execute 2024-09-13T00:23:39.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:23:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:23:39] "GET /metrics HTTP/1.1" 200 38980 "" "Prometheus/2.43.0" 2024-09-13T00:23:39.870 INFO:tasks.workunit.client.0.smithi136.stdout:migration_commit 2024-09-13T00:23:40.026 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.NoSnapsShrinkToZeroAfterPrepare (1116 ms) 2024-09-13T00:23:40.026 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.NoSnapsExpandAfterPrepare 2024-09-13T00:23:40.108 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:23:40.139 INFO:tasks.workunit.client.0.smithi136.stdout:write: 1 0~8388609 2024-09-13T00:23:40.228 INFO:tasks.workunit.client.0.smithi136.stdout:write: 1 16777218~8388609 2024-09-13T00:23:40.298 INFO:tasks.workunit.client.0.smithi136.stdout:migration_prepare 2024-09-13T00:23:40.775 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 167772160 2024-09-13T00:23:40.796 INFO:tasks.workunit.client.0.smithi136.stdout:migration_execute 2024-09-13T00:23:41.074 INFO:tasks.workunit.client.0.smithi136.stdout:migration_commit 2024-09-13T00:23:41.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:40 smithi136 bash[76371]: cluster 2024-09-13T00:23:39.810068+0000 mgr.y (mgr.24661) 1396 : cluster [DBG] pgmap v2132: 169 pgs: 169 active+clean; 121 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 27 MiB/s rd, 21 MiB/s wr, 979 op/s 2024-09-13T00:23:41.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:40 smithi136 bash[75164]: cluster 2024-09-13T00:23:39.810068+0000 mgr.y (mgr.24661) 1396 : cluster [DBG] pgmap v2132: 169 pgs: 169 active+clean; 121 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 27 MiB/s rd, 21 MiB/s wr, 979 op/s 2024-09-13T00:23:41.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:40 smithi181 bash[67587]: cluster 2024-09-13T00:23:39.810068+0000 mgr.y (mgr.24661) 1396 : cluster [DBG] pgmap v2132: 169 pgs: 169 active+clean; 121 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 27 MiB/s rd, 21 MiB/s wr, 979 op/s 2024-09-13T00:23:41.386 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.NoSnapsExpandAfterPrepare (1360 ms) 2024-09-13T00:23:41.386 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.NoSnapsSnapAfterPrepare 2024-09-13T00:23:41.470 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:23:41.499 INFO:tasks.workunit.client.0.smithi136.stdout:write: 1 0~8388609 2024-09-13T00:23:41.579 INFO:tasks.workunit.client.0.smithi136.stdout:write: 1 16777218~8388609 2024-09-13T00:23:41.651 INFO:tasks.workunit.client.0.smithi136.stdout:migration_prepare 2024-09-13T00:23:42.097 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: after_prepare_snap 2024-09-13T00:23:42.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:23:42 smithi181 bash[95505]: ts=2024-09-13T00:23:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:23:43.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:42 smithi136 bash[76371]: cluster 2024-09-13T00:23:41.811693+0000 mgr.y (mgr.24661) 1397 : cluster [DBG] pgmap v2133: 169 pgs: 169 active+clean; 157 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 38 MiB/s rd, 30 MiB/s wr, 905 op/s 2024-09-13T00:23:43.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:42 smithi136 bash[75164]: cluster 2024-09-13T00:23:41.811693+0000 mgr.y (mgr.24661) 1397 : cluster [DBG] pgmap v2133: 169 pgs: 169 active+clean; 157 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 38 MiB/s rd, 30 MiB/s wr, 905 op/s 2024-09-13T00:23:43.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:42 smithi181 bash[67587]: cluster 2024-09-13T00:23:41.811693+0000 mgr.y (mgr.24661) 1397 : cluster [DBG] pgmap v2133: 169 pgs: 169 active+clean; 157 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 38 MiB/s rd, 30 MiB/s wr, 905 op/s 2024-09-13T00:23:43.998 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 41943040 2024-09-13T00:23:44.047 INFO:tasks.workunit.client.0.smithi136.stdout:write: * 0~1000 2024-09-13T00:23:44.060 INFO:tasks.workunit.client.0.smithi136.stdout:migration_execute 2024-09-13T00:23:44.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:43 smithi136 bash[75164]: cluster 2024-09-13T00:23:42.959592+0000 mon.a (mon.0) 2390 : cluster [DBG] osdmap e1176: 8 total, 8 up, 8 in 2024-09-13T00:23:44.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:43 smithi136 bash[76371]: cluster 2024-09-13T00:23:42.959592+0000 mon.a (mon.0) 2390 : cluster [DBG] osdmap e1176: 8 total, 8 up, 8 in 2024-09-13T00:23:44.290 INFO:tasks.workunit.client.0.smithi136.stdout:migration_commit 2024-09-13T00:23:44.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:43 smithi181 bash[67587]: cluster 2024-09-13T00:23:42.959592+0000 mon.a (mon.0) 2390 : cluster [DBG] osdmap e1176: 8 total, 8 up, 8 in 2024-09-13T00:23:44.618 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.NoSnapsSnapAfterPrepare (3232 ms) 2024-09-13T00:23:44.618 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.Snaps 2024-09-13T00:23:44.698 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:23:44.729 INFO:tasks.workunit.client.0.smithi136.stdout:write: 1 0~8388609 2024-09-13T00:23:44.821 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap1 2024-09-13T00:23:45.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:44 smithi136 bash[75164]: cluster 2024-09-13T00:23:43.812717+0000 mgr.y (mgr.24661) 1398 : cluster [DBG] pgmap v2135: 169 pgs: 169 active+clean; 173 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 43 MiB/s rd, 34 MiB/s wr, 926 op/s 2024-09-13T00:23:45.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:44 smithi136 bash[75164]: cluster 2024-09-13T00:23:43.972125+0000 mon.a (mon.0) 2391 : cluster [DBG] osdmap e1177: 8 total, 8 up, 8 in 2024-09-13T00:23:45.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:44 smithi136 bash[76371]: cluster 2024-09-13T00:23:43.812717+0000 mgr.y (mgr.24661) 1398 : cluster [DBG] pgmap v2135: 169 pgs: 169 active+clean; 173 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 43 MiB/s rd, 34 MiB/s wr, 926 op/s 2024-09-13T00:23:45.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:44 smithi136 bash[76371]: cluster 2024-09-13T00:23:43.972125+0000 mon.a (mon.0) 2391 : cluster [DBG] osdmap e1177: 8 total, 8 up, 8 in 2024-09-13T00:23:45.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:44 smithi181 bash[67587]: cluster 2024-09-13T00:23:43.812717+0000 mgr.y (mgr.24661) 1398 : cluster [DBG] pgmap v2135: 169 pgs: 169 active+clean; 173 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 43 MiB/s rd, 34 MiB/s wr, 926 op/s 2024-09-13T00:23:45.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:44 smithi181 bash[67587]: cluster 2024-09-13T00:23:43.972125+0000 mon.a (mon.0) 2391 : cluster [DBG] osdmap e1177: 8 total, 8 up, 8 in 2024-09-13T00:23:46.014 INFO:tasks.workunit.client.0.smithi136.stdout:write: 1 8388609~8388609 2024-09-13T00:23:46.058 INFO:tasks.workunit.client.0.smithi136.stdout:write: X 0~1000 2024-09-13T00:23:46.058 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 1010~1000 2024-09-13T00:23:46.059 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap2 2024-09-13T00:23:46.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:45 smithi136 bash[75164]: cluster 2024-09-13T00:23:44.982667+0000 mon.a (mon.0) 2392 : cluster [DBG] osdmap e1178: 8 total, 8 up, 8 in 2024-09-13T00:23:46.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:45 smithi136 bash[76371]: cluster 2024-09-13T00:23:44.982667+0000 mon.a (mon.0) 2392 : cluster [DBG] osdmap e1178: 8 total, 8 up, 8 in 2024-09-13T00:23:46.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:45 smithi181 bash[67587]: cluster 2024-09-13T00:23:44.982667+0000 mon.a (mon.0) 2392 : cluster [DBG] osdmap e1178: 8 total, 8 up, 8 in 2024-09-13T00:23:47.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:46 smithi136 bash[75164]: cluster 2024-09-13T00:23:45.813695+0000 mgr.y (mgr.24661) 1399 : cluster [DBG] pgmap v2138: 169 pgs: 169 active+clean; 189 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 36 MiB/s rd, 27 MiB/s wr, 678 op/s 2024-09-13T00:23:47.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:46 smithi136 bash[75164]: cluster 2024-09-13T00:23:45.992433+0000 mon.a (mon.0) 2393 : cluster [DBG] osdmap e1179: 8 total, 8 up, 8 in 2024-09-13T00:23:47.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:46 smithi136 bash[76371]: cluster 2024-09-13T00:23:45.813695+0000 mgr.y (mgr.24661) 1399 : cluster [DBG] pgmap v2138: 169 pgs: 169 active+clean; 189 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 36 MiB/s rd, 27 MiB/s wr, 678 op/s 2024-09-13T00:23:47.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:46 smithi136 bash[76371]: cluster 2024-09-13T00:23:45.992433+0000 mon.a (mon.0) 2393 : cluster [DBG] osdmap e1179: 8 total, 8 up, 8 in 2024-09-13T00:23:47.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:46 smithi181 bash[67587]: cluster 2024-09-13T00:23:45.813695+0000 mgr.y (mgr.24661) 1399 : cluster [DBG] pgmap v2138: 169 pgs: 169 active+clean; 189 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 36 MiB/s rd, 27 MiB/s wr, 678 op/s 2024-09-13T00:23:47.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:46 smithi181 bash[67587]: cluster 2024-09-13T00:23:45.992433+0000 mon.a (mon.0) 2393 : cluster [DBG] osdmap e1179: 8 total, 8 up, 8 in 2024-09-13T00:23:48.046 INFO:tasks.workunit.client.0.smithi136.stdout:write: X 8388609~1000 2024-09-13T00:23:48.061 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 16777228~1000 2024-09-13T00:23:48.061 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 167772160 2024-09-13T00:23:48.083 INFO:tasks.workunit.client.0.smithi136.stdout:write: 2 83886079~8388609 2024-09-13T00:23:48.137 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap3 2024-09-13T00:23:48.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:48 smithi136 bash[76371]: cluster 2024-09-13T00:23:47.001907+0000 mon.a (mon.0) 2394 : cluster [DBG] osdmap e1180: 8 total, 8 up, 8 in 2024-09-13T00:23:48.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:48 smithi136 bash[75164]: cluster 2024-09-13T00:23:47.001907+0000 mon.a (mon.0) 2394 : cluster [DBG] osdmap e1180: 8 total, 8 up, 8 in 2024-09-13T00:23:48.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:48 smithi181 bash[67587]: cluster 2024-09-13T00:23:47.001907+0000 mon.a (mon.0) 2394 : cluster [DBG] osdmap e1180: 8 total, 8 up, 8 in 2024-09-13T00:23:49.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:49 smithi136 bash[75164]: cluster 2024-09-13T00:23:47.814616+0000 mgr.y (mgr.24661) 1400 : cluster [DBG] pgmap v2141: 169 pgs: 169 active+clean; 205 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 26 MiB/s rd, 13 MiB/s wr, 457 op/s 2024-09-13T00:23:49.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:49 smithi136 bash[75164]: cluster 2024-09-13T00:23:48.012508+0000 mon.a (mon.0) 2395 : cluster [DBG] osdmap e1181: 8 total, 8 up, 8 in 2024-09-13T00:23:49.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:49 smithi136 bash[76371]: cluster 2024-09-13T00:23:47.814616+0000 mgr.y (mgr.24661) 1400 : cluster [DBG] pgmap v2141: 169 pgs: 169 active+clean; 205 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 26 MiB/s rd, 13 MiB/s wr, 457 op/s 2024-09-13T00:23:49.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:49 smithi136 bash[76371]: cluster 2024-09-13T00:23:48.012508+0000 mon.a (mon.0) 2395 : cluster [DBG] osdmap e1181: 8 total, 8 up, 8 in 2024-09-13T00:23:49.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:49 smithi181 bash[67587]: cluster 2024-09-13T00:23:47.814616+0000 mgr.y (mgr.24661) 1400 : cluster [DBG] pgmap v2141: 169 pgs: 169 active+clean; 205 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 26 MiB/s rd, 13 MiB/s wr, 457 op/s 2024-09-13T00:23:49.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:49 smithi181 bash[67587]: cluster 2024-09-13T00:23:48.012508+0000 mon.a (mon.0) 2395 : cluster [DBG] osdmap e1181: 8 total, 8 up, 8 in 2024-09-13T00:23:49.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:23:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:23:49] "GET /metrics HTTP/1.1" 200 38980 "" "Prometheus/2.43.0" 2024-09-13T00:23:50.063 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:23:50.111 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 83886079~1 2024-09-13T00:23:50.133 INFO:tasks.workunit.client.0.smithi136.stdout:migration_prepare 2024-09-13T00:23:50.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:50 smithi181 bash[67587]: cluster 2024-09-13T00:23:49.023530+0000 mon.a (mon.0) 2396 : cluster [DBG] osdmap e1182: 8 total, 8 up, 8 in 2024-09-13T00:23:50.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:50 smithi136 bash[75164]: cluster 2024-09-13T00:23:49.023530+0000 mon.a (mon.0) 2396 : cluster [DBG] osdmap e1182: 8 total, 8 up, 8 in 2024-09-13T00:23:50.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:50 smithi136 bash[76371]: cluster 2024-09-13T00:23:49.023530+0000 mon.a (mon.0) 2396 : cluster [DBG] osdmap e1182: 8 total, 8 up, 8 in 2024-09-13T00:23:51.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:51 smithi181 bash[67587]: cluster 2024-09-13T00:23:49.815492+0000 mgr.y (mgr.24661) 1401 : cluster [DBG] pgmap v2144: 169 pgs: 169 active+clean; 229 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 16 MiB/s rd, 12 MiB/s wr, 300 op/s 2024-09-13T00:23:51.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:51 smithi181 bash[67587]: cluster 2024-09-13T00:23:50.031704+0000 mon.a (mon.0) 2397 : cluster [DBG] osdmap e1183: 8 total, 8 up, 8 in 2024-09-13T00:23:51.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:51 smithi136 bash[75164]: cluster 2024-09-13T00:23:49.815492+0000 mgr.y (mgr.24661) 1401 : cluster [DBG] pgmap v2144: 169 pgs: 169 active+clean; 229 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 16 MiB/s rd, 12 MiB/s wr, 300 op/s 2024-09-13T00:23:51.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:51 smithi136 bash[75164]: cluster 2024-09-13T00:23:50.031704+0000 mon.a (mon.0) 2397 : cluster [DBG] osdmap e1183: 8 total, 8 up, 8 in 2024-09-13T00:23:51.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:51 smithi136 bash[76371]: cluster 2024-09-13T00:23:49.815492+0000 mgr.y (mgr.24661) 1401 : cluster [DBG] pgmap v2144: 169 pgs: 169 active+clean; 229 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 16 MiB/s rd, 12 MiB/s wr, 300 op/s 2024-09-13T00:23:51.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:51 smithi136 bash[76371]: cluster 2024-09-13T00:23:50.031704+0000 mon.a (mon.0) 2397 : cluster [DBG] osdmap e1183: 8 total, 8 up, 8 in 2024-09-13T00:23:52.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:52 smithi181 bash[67587]: cluster 2024-09-13T00:23:51.044530+0000 mon.a (mon.0) 2398 : cluster [DBG] osdmap e1184: 8 total, 8 up, 8 in 2024-09-13T00:23:52.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:52 smithi136 bash[75164]: cluster 2024-09-13T00:23:51.044530+0000 mon.a (mon.0) 2398 : cluster [DBG] osdmap e1184: 8 total, 8 up, 8 in 2024-09-13T00:23:52.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:52 smithi136 bash[76371]: cluster 2024-09-13T00:23:51.044530+0000 mon.a (mon.0) 2398 : cluster [DBG] osdmap e1184: 8 total, 8 up, 8 in 2024-09-13T00:23:52.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:23:52 smithi181 bash[95505]: ts=2024-09-13T00:23:52.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:23:53.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:53 smithi181 bash[67587]: cluster 2024-09-13T00:23:51.816556+0000 mgr.y (mgr.24661) 1402 : cluster [DBG] pgmap v2147: 169 pgs: 169 active+clean; 249 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 5.1 MiB/s rd, 10 MiB/s wr, 199 op/s 2024-09-13T00:23:53.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:53 smithi181 bash[67587]: cluster 2024-09-13T00:23:52.055711+0000 mon.a (mon.0) 2399 : cluster [DBG] osdmap e1185: 8 total, 8 up, 8 in 2024-09-13T00:23:53.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:53 smithi136 bash[75164]: cluster 2024-09-13T00:23:51.816556+0000 mgr.y (mgr.24661) 1402 : cluster [DBG] pgmap v2147: 169 pgs: 169 active+clean; 249 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 5.1 MiB/s rd, 10 MiB/s wr, 199 op/s 2024-09-13T00:23:53.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:53 smithi136 bash[75164]: cluster 2024-09-13T00:23:52.055711+0000 mon.a (mon.0) 2399 : cluster [DBG] osdmap e1185: 8 total, 8 up, 8 in 2024-09-13T00:23:53.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:53 smithi136 bash[76371]: cluster 2024-09-13T00:23:51.816556+0000 mgr.y (mgr.24661) 1402 : cluster [DBG] pgmap v2147: 169 pgs: 169 active+clean; 249 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 5.1 MiB/s rd, 10 MiB/s wr, 199 op/s 2024-09-13T00:23:53.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:53 smithi136 bash[76371]: cluster 2024-09-13T00:23:52.055711+0000 mon.a (mon.0) 2399 : cluster [DBG] osdmap e1185: 8 total, 8 up, 8 in 2024-09-13T00:23:53.969 INFO:tasks.workunit.client.0.smithi136.stdout:migration_execute 2024-09-13T00:23:54.265 INFO:tasks.workunit.client.0.smithi136.stdout:migration_commit 2024-09-13T00:23:54.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:54 smithi181 bash[67587]: cluster 2024-09-13T00:23:53.066157+0000 mon.a (mon.0) 2400 : cluster [DBG] osdmap e1186: 8 total, 8 up, 8 in 2024-09-13T00:23:54.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:54 smithi181 bash[67587]: audit 2024-09-13T00:23:53.809525+0000 mon.a (mon.0) 2401 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:23:54.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:54 smithi136 bash[75164]: cluster 2024-09-13T00:23:53.066157+0000 mon.a (mon.0) 2400 : cluster [DBG] osdmap e1186: 8 total, 8 up, 8 in 2024-09-13T00:23:54.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:54 smithi136 bash[75164]: audit 2024-09-13T00:23:53.809525+0000 mon.a (mon.0) 2401 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:23:54.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:54 smithi136 bash[76371]: cluster 2024-09-13T00:23:53.066157+0000 mon.a (mon.0) 2400 : cluster [DBG] osdmap e1186: 8 total, 8 up, 8 in 2024-09-13T00:23:54.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:54 smithi136 bash[76371]: audit 2024-09-13T00:23:53.809525+0000 mon.a (mon.0) 2401 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:23:55.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:55 smithi181 bash[67587]: cluster 2024-09-13T00:23:53.820370+0000 mgr.y (mgr.24661) 1403 : cluster [DBG] pgmap v2150: 169 pgs: 169 active+clean; 253 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.1 MiB/s rd, 4.0 MiB/s wr, 140 op/s 2024-09-13T00:23:55.544 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:55 smithi136 bash[75164]: cluster 2024-09-13T00:23:53.820370+0000 mgr.y (mgr.24661) 1403 : cluster [DBG] pgmap v2150: 169 pgs: 169 active+clean; 253 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.1 MiB/s rd, 4.0 MiB/s wr, 140 op/s 2024-09-13T00:23:55.545 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:55 smithi136 bash[76371]: cluster 2024-09-13T00:23:53.820370+0000 mgr.y (mgr.24661) 1403 : cluster [DBG] pgmap v2150: 169 pgs: 169 active+clean; 253 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.1 MiB/s rd, 4.0 MiB/s wr, 140 op/s 2024-09-13T00:23:56.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:56 smithi136 bash[75164]: cluster 2024-09-13T00:23:55.079928+0000 mon.a (mon.0) 2402 : cluster [DBG] osdmap e1187: 8 total, 8 up, 8 in 2024-09-13T00:23:56.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:56 smithi136 bash[76371]: cluster 2024-09-13T00:23:55.079928+0000 mon.a (mon.0) 2402 : cluster [DBG] osdmap e1187: 8 total, 8 up, 8 in 2024-09-13T00:23:56.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:56 smithi181 bash[67587]: cluster 2024-09-13T00:23:55.079928+0000 mon.a (mon.0) 2402 : cluster [DBG] osdmap e1187: 8 total, 8 up, 8 in 2024-09-13T00:23:57.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:57 smithi136 bash[76371]: cluster 2024-09-13T00:23:55.821432+0000 mgr.y (mgr.24661) 1404 : cluster [DBG] pgmap v2152: 169 pgs: 169 active+clean; 273 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 13 MiB/s rd, 4.2 MiB/s wr, 76 op/s 2024-09-13T00:23:57.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:57 smithi136 bash[76371]: cluster 2024-09-13T00:23:56.107297+0000 mon.a (mon.0) 2403 : cluster [DBG] osdmap e1188: 8 total, 8 up, 8 in 2024-09-13T00:23:57.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:57 smithi136 bash[75164]: cluster 2024-09-13T00:23:55.821432+0000 mgr.y (mgr.24661) 1404 : cluster [DBG] pgmap v2152: 169 pgs: 169 active+clean; 273 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 13 MiB/s rd, 4.2 MiB/s wr, 76 op/s 2024-09-13T00:23:57.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:57 smithi136 bash[75164]: cluster 2024-09-13T00:23:56.107297+0000 mon.a (mon.0) 2403 : cluster [DBG] osdmap e1188: 8 total, 8 up, 8 in 2024-09-13T00:23:57.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:57 smithi181 bash[67587]: cluster 2024-09-13T00:23:55.821432+0000 mgr.y (mgr.24661) 1404 : cluster [DBG] pgmap v2152: 169 pgs: 169 active+clean; 273 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 13 MiB/s rd, 4.2 MiB/s wr, 76 op/s 2024-09-13T00:23:57.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:57 smithi181 bash[67587]: cluster 2024-09-13T00:23:56.107297+0000 mon.a (mon.0) 2403 : cluster [DBG] osdmap e1188: 8 total, 8 up, 8 in 2024-09-13T00:23:57.699 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.Snaps (13081 ms) 2024-09-13T00:23:57.699 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.SnapsOtherPool 2024-09-13T00:23:57.785 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:23:57.814 INFO:tasks.workunit.client.0.smithi136.stdout:write: 1 0~8388609 2024-09-13T00:23:57.919 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap1 2024-09-13T00:23:58.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:58 smithi136 bash[75164]: cluster 2024-09-13T00:23:57.114211+0000 mon.a (mon.0) 2404 : cluster [DBG] osdmap e1189: 8 total, 8 up, 8 in 2024-09-13T00:23:58.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:58 smithi136 bash[76371]: cluster 2024-09-13T00:23:57.114211+0000 mon.a (mon.0) 2404 : cluster [DBG] osdmap e1189: 8 total, 8 up, 8 in 2024-09-13T00:23:58.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:58 smithi181 bash[67587]: cluster 2024-09-13T00:23:57.114211+0000 mon.a (mon.0) 2404 : cluster [DBG] osdmap e1189: 8 total, 8 up, 8 in 2024-09-13T00:23:59.160 INFO:tasks.workunit.client.0.smithi136.stdout:write: 1 8388609~8388609 2024-09-13T00:23:59.201 INFO:tasks.workunit.client.0.smithi136.stdout:write: X 0~1000 2024-09-13T00:23:59.201 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 1010~1000 2024-09-13T00:23:59.201 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap2 2024-09-13T00:23:59.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:59 smithi136 bash[75164]: cluster 2024-09-13T00:23:57.822387+0000 mgr.y (mgr.24661) 1405 : cluster [DBG] pgmap v2155: 169 pgs: 169 active+clean; 269 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 26 MiB/s rd, 3.4 MiB/s wr, 347 op/s 2024-09-13T00:23:59.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:23:59 smithi136 bash[75164]: cluster 2024-09-13T00:23:58.123467+0000 mon.a (mon.0) 2405 : cluster [DBG] osdmap e1190: 8 total, 8 up, 8 in 2024-09-13T00:23:59.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:59 smithi136 bash[76371]: cluster 2024-09-13T00:23:57.822387+0000 mgr.y (mgr.24661) 1405 : cluster [DBG] pgmap v2155: 169 pgs: 169 active+clean; 269 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 26 MiB/s rd, 3.4 MiB/s wr, 347 op/s 2024-09-13T00:23:59.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:23:59 smithi136 bash[76371]: cluster 2024-09-13T00:23:58.123467+0000 mon.a (mon.0) 2405 : cluster [DBG] osdmap e1190: 8 total, 8 up, 8 in 2024-09-13T00:23:59.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:59 smithi181 bash[67587]: cluster 2024-09-13T00:23:57.822387+0000 mgr.y (mgr.24661) 1405 : cluster [DBG] pgmap v2155: 169 pgs: 169 active+clean; 269 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 26 MiB/s rd, 3.4 MiB/s wr, 347 op/s 2024-09-13T00:23:59.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:23:59 smithi181 bash[67587]: cluster 2024-09-13T00:23:58.123467+0000 mon.a (mon.0) 2405 : cluster [DBG] osdmap e1190: 8 total, 8 up, 8 in 2024-09-13T00:23:59.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:23:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:23:59] "GET /metrics HTTP/1.1" 200 38985 "" "Prometheus/2.43.0" 2024-09-13T00:24:00.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:00 smithi136 bash[75164]: cluster 2024-09-13T00:23:59.133116+0000 mon.a (mon.0) 2406 : cluster [DBG] osdmap e1191: 8 total, 8 up, 8 in 2024-09-13T00:24:00.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:00 smithi136 bash[76371]: cluster 2024-09-13T00:23:59.133116+0000 mon.a (mon.0) 2406 : cluster [DBG] osdmap e1191: 8 total, 8 up, 8 in 2024-09-13T00:24:00.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:00 smithi181 bash[67587]: cluster 2024-09-13T00:23:59.133116+0000 mon.a (mon.0) 2406 : cluster [DBG] osdmap e1191: 8 total, 8 up, 8 in 2024-09-13T00:24:01.176 INFO:tasks.workunit.client.0.smithi136.stdout:write: X 8388609~1000 2024-09-13T00:24:01.194 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 16777228~1000 2024-09-13T00:24:01.194 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 167772160 2024-09-13T00:24:01.215 INFO:tasks.workunit.client.0.smithi136.stdout:write: 2 83886079~8388609 2024-09-13T00:24:01.271 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap3 2024-09-13T00:24:01.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:01 smithi136 bash[75164]: cluster 2024-09-13T00:23:59.823395+0000 mgr.y (mgr.24661) 1406 : cluster [DBG] pgmap v2158: 169 pgs: 169 active+clean; 265 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 44 MiB/s rd, 3.0 MiB/s wr, 394 op/s 2024-09-13T00:24:01.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:01 smithi136 bash[75164]: cluster 2024-09-13T00:24:00.147969+0000 mon.a (mon.0) 2407 : cluster [DBG] osdmap e1192: 8 total, 8 up, 8 in 2024-09-13T00:24:01.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:01 smithi136 bash[76371]: cluster 2024-09-13T00:23:59.823395+0000 mgr.y (mgr.24661) 1406 : cluster [DBG] pgmap v2158: 169 pgs: 169 active+clean; 265 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 44 MiB/s rd, 3.0 MiB/s wr, 394 op/s 2024-09-13T00:24:01.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:01 smithi136 bash[76371]: cluster 2024-09-13T00:24:00.147969+0000 mon.a (mon.0) 2407 : cluster [DBG] osdmap e1192: 8 total, 8 up, 8 in 2024-09-13T00:24:01.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:01 smithi181 bash[67587]: cluster 2024-09-13T00:23:59.823395+0000 mgr.y (mgr.24661) 1406 : cluster [DBG] pgmap v2158: 169 pgs: 169 active+clean; 265 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 44 MiB/s rd, 3.0 MiB/s wr, 394 op/s 2024-09-13T00:24:01.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:01 smithi181 bash[67587]: cluster 2024-09-13T00:24:00.147969+0000 mon.a (mon.0) 2407 : cluster [DBG] osdmap e1192: 8 total, 8 up, 8 in 2024-09-13T00:24:02.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:02 smithi136 bash[76371]: cluster 2024-09-13T00:24:01.146840+0000 mon.a (mon.0) 2408 : cluster [DBG] osdmap e1193: 8 total, 8 up, 8 in 2024-09-13T00:24:02.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:02 smithi136 bash[75164]: cluster 2024-09-13T00:24:01.146840+0000 mon.a (mon.0) 2408 : cluster [DBG] osdmap e1193: 8 total, 8 up, 8 in 2024-09-13T00:24:02.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:02 smithi181 bash[67587]: cluster 2024-09-13T00:24:01.146840+0000 mon.a (mon.0) 2408 : cluster [DBG] osdmap e1193: 8 total, 8 up, 8 in 2024-09-13T00:24:02.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:24:02 smithi181 bash[95505]: ts=2024-09-13T00:24:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:24:03.206 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:24:03.259 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 83886079~1 2024-09-13T00:24:03.282 INFO:tasks.workunit.client.0.smithi136.stdout:write: 1 0~8388609 2024-09-13T00:24:03.314 INFO:tasks.workunit.client.0.smithi136.stdout:write: 1 16777218~8388609 2024-09-13T00:24:03.387 INFO:tasks.workunit.client.0.smithi136.stdout:migration_prepare 2024-09-13T00:24:03.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:03 smithi136 bash[76371]: cluster 2024-09-13T00:24:01.824539+0000 mgr.y (mgr.24661) 1407 : cluster [DBG] pgmap v2161: 169 pgs: 169 active+clean; 285 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 39 MiB/s rd, 10 MiB/s wr, 164 op/s 2024-09-13T00:24:03.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:03 smithi136 bash[76371]: cluster 2024-09-13T00:24:02.177179+0000 mon.a (mon.0) 2409 : cluster [DBG] osdmap e1194: 8 total, 8 up, 8 in 2024-09-13T00:24:03.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:03 smithi136 bash[75164]: cluster 2024-09-13T00:24:01.824539+0000 mgr.y (mgr.24661) 1407 : cluster [DBG] pgmap v2161: 169 pgs: 169 active+clean; 285 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 39 MiB/s rd, 10 MiB/s wr, 164 op/s 2024-09-13T00:24:03.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:03 smithi136 bash[75164]: cluster 2024-09-13T00:24:02.177179+0000 mon.a (mon.0) 2409 : cluster [DBG] osdmap e1194: 8 total, 8 up, 8 in 2024-09-13T00:24:03.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:03 smithi181 bash[67587]: cluster 2024-09-13T00:24:01.824539+0000 mgr.y (mgr.24661) 1407 : cluster [DBG] pgmap v2161: 169 pgs: 169 active+clean; 285 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 39 MiB/s rd, 10 MiB/s wr, 164 op/s 2024-09-13T00:24:03.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:03 smithi181 bash[67587]: cluster 2024-09-13T00:24:02.177179+0000 mon.a (mon.0) 2409 : cluster [DBG] osdmap e1194: 8 total, 8 up, 8 in 2024-09-13T00:24:04.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:04 smithi136 bash[75164]: cluster 2024-09-13T00:24:03.171966+0000 mon.a (mon.0) 2410 : cluster [DBG] osdmap e1195: 8 total, 8 up, 8 in 2024-09-13T00:24:04.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:04 smithi136 bash[75164]: cluster 2024-09-13T00:24:03.934897+0000 mon.a (mon.0) 2411 : cluster [DBG] osdmap e1196: 8 total, 8 up, 8 in 2024-09-13T00:24:04.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:04 smithi136 bash[76371]: cluster 2024-09-13T00:24:03.171966+0000 mon.a (mon.0) 2410 : cluster [DBG] osdmap e1195: 8 total, 8 up, 8 in 2024-09-13T00:24:04.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:04 smithi136 bash[76371]: cluster 2024-09-13T00:24:03.934897+0000 mon.a (mon.0) 2411 : cluster [DBG] osdmap e1196: 8 total, 8 up, 8 in 2024-09-13T00:24:04.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:04 smithi181 bash[67587]: cluster 2024-09-13T00:24:03.171966+0000 mon.a (mon.0) 2410 : cluster [DBG] osdmap e1195: 8 total, 8 up, 8 in 2024-09-13T00:24:04.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:04 smithi181 bash[67587]: cluster 2024-09-13T00:24:03.934897+0000 mon.a (mon.0) 2411 : cluster [DBG] osdmap e1196: 8 total, 8 up, 8 in 2024-09-13T00:24:05.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:05 smithi136 bash[75164]: cluster 2024-09-13T00:24:03.825549+0000 mgr.y (mgr.24661) 1408 : cluster [DBG] pgmap v2164: 169 pgs: 169 active+clean; 301 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 11 MiB/s rd, 10 MiB/s wr, 129 op/s 2024-09-13T00:24:05.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:05 smithi136 bash[75164]: cluster 2024-09-13T00:24:04.938678+0000 mon.a (mon.0) 2412 : cluster [DBG] osdmap e1197: 8 total, 8 up, 8 in 2024-09-13T00:24:05.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:05 smithi136 bash[76371]: cluster 2024-09-13T00:24:03.825549+0000 mgr.y (mgr.24661) 1408 : cluster [DBG] pgmap v2164: 169 pgs: 169 active+clean; 301 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 11 MiB/s rd, 10 MiB/s wr, 129 op/s 2024-09-13T00:24:05.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:05 smithi136 bash[76371]: cluster 2024-09-13T00:24:04.938678+0000 mon.a (mon.0) 2412 : cluster [DBG] osdmap e1197: 8 total, 8 up, 8 in 2024-09-13T00:24:05.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:05 smithi181 bash[67587]: cluster 2024-09-13T00:24:03.825549+0000 mgr.y (mgr.24661) 1408 : cluster [DBG] pgmap v2164: 169 pgs: 169 active+clean; 301 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 11 MiB/s rd, 10 MiB/s wr, 129 op/s 2024-09-13T00:24:05.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:05 smithi181 bash[67587]: cluster 2024-09-13T00:24:04.938678+0000 mon.a (mon.0) 2412 : cluster [DBG] osdmap e1197: 8 total, 8 up, 8 in 2024-09-13T00:24:06.880 INFO:tasks.workunit.client.0.smithi136.stdout:migration_execute 2024-09-13T00:24:07.191 INFO:tasks.workunit.client.0.smithi136.stdout:migration_commit 2024-09-13T00:24:07.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:06 smithi136 bash[75164]: cluster 2024-09-13T00:24:05.826592+0000 mgr.y (mgr.24661) 1409 : cluster [DBG] pgmap v2167: 169 pgs: 169 active+clean; 313 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 35 KiB/s rd, 5.0 MiB/s wr, 48 op/s 2024-09-13T00:24:07.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:06 smithi136 bash[75164]: cluster 2024-09-13T00:24:05.942044+0000 mon.a (mon.0) 2413 : cluster [DBG] osdmap e1198: 8 total, 8 up, 8 in 2024-09-13T00:24:07.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:06 smithi136 bash[76371]: cluster 2024-09-13T00:24:05.826592+0000 mgr.y (mgr.24661) 1409 : cluster [DBG] pgmap v2167: 169 pgs: 169 active+clean; 313 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 35 KiB/s rd, 5.0 MiB/s wr, 48 op/s 2024-09-13T00:24:07.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:06 smithi136 bash[76371]: cluster 2024-09-13T00:24:05.942044+0000 mon.a (mon.0) 2413 : cluster [DBG] osdmap e1198: 8 total, 8 up, 8 in 2024-09-13T00:24:07.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:06 smithi181 bash[67587]: cluster 2024-09-13T00:24:05.826592+0000 mgr.y (mgr.24661) 1409 : cluster [DBG] pgmap v2167: 169 pgs: 169 active+clean; 313 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 35 KiB/s rd, 5.0 MiB/s wr, 48 op/s 2024-09-13T00:24:07.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:06 smithi181 bash[67587]: cluster 2024-09-13T00:24:05.942044+0000 mon.a (mon.0) 2413 : cluster [DBG] osdmap e1198: 8 total, 8 up, 8 in 2024-09-13T00:24:09.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:08 smithi136 bash[75164]: cluster 2024-09-13T00:24:07.827950+0000 mgr.y (mgr.24661) 1410 : cluster [DBG] pgmap v2169: 169 pgs: 169 active+clean; 349 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 20 MiB/s rd, 11 MiB/s wr, 336 op/s 2024-09-13T00:24:09.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:08 smithi136 bash[75164]: cluster 2024-09-13T00:24:07.956894+0000 mon.a (mon.0) 2414 : cluster [DBG] osdmap e1199: 8 total, 8 up, 8 in 2024-09-13T00:24:09.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:08 smithi136 bash[75164]: audit 2024-09-13T00:24:08.809975+0000 mon.a (mon.0) 2415 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:24:09.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:08 smithi136 bash[76371]: cluster 2024-09-13T00:24:07.827950+0000 mgr.y (mgr.24661) 1410 : cluster [DBG] pgmap v2169: 169 pgs: 169 active+clean; 349 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 20 MiB/s rd, 11 MiB/s wr, 336 op/s 2024-09-13T00:24:09.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:08 smithi136 bash[76371]: cluster 2024-09-13T00:24:07.956894+0000 mon.a (mon.0) 2414 : cluster [DBG] osdmap e1199: 8 total, 8 up, 8 in 2024-09-13T00:24:09.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:08 smithi136 bash[76371]: audit 2024-09-13T00:24:08.809975+0000 mon.a (mon.0) 2415 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:24:09.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:08 smithi181 bash[67587]: cluster 2024-09-13T00:24:07.827950+0000 mgr.y (mgr.24661) 1410 : cluster [DBG] pgmap v2169: 169 pgs: 169 active+clean; 349 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 20 MiB/s rd, 11 MiB/s wr, 336 op/s 2024-09-13T00:24:09.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:08 smithi181 bash[67587]: cluster 2024-09-13T00:24:07.956894+0000 mon.a (mon.0) 2414 : cluster [DBG] osdmap e1199: 8 total, 8 up, 8 in 2024-09-13T00:24:09.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:08 smithi181 bash[67587]: audit 2024-09-13T00:24:08.809975+0000 mon.a (mon.0) 2415 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:24:09.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:24:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:24:09] "GET /metrics HTTP/1.1" 200 38983 "" "Prometheus/2.43.0" 2024-09-13T00:24:10.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:09 smithi136 bash[75164]: cluster 2024-09-13T00:24:08.955663+0000 mon.a (mon.0) 2416 : cluster [DBG] osdmap e1200: 8 total, 8 up, 8 in 2024-09-13T00:24:10.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:09 smithi136 bash[76371]: cluster 2024-09-13T00:24:08.955663+0000 mon.a (mon.0) 2416 : cluster [DBG] osdmap e1200: 8 total, 8 up, 8 in 2024-09-13T00:24:10.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:09 smithi181 bash[67587]: cluster 2024-09-13T00:24:08.955663+0000 mon.a (mon.0) 2416 : cluster [DBG] osdmap e1200: 8 total, 8 up, 8 in 2024-09-13T00:24:10.562 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.SnapsOtherPool (12864 ms) 2024-09-13T00:24:10.563 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.SnapsDataPool 2024-09-13T00:24:10.649 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:24:10.677 INFO:tasks.workunit.client.0.smithi136.stdout:write: 1 0~8388609 2024-09-13T00:24:10.791 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap1 2024-09-13T00:24:11.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:10 smithi136 bash[75164]: cluster 2024-09-13T00:24:09.828843+0000 mgr.y (mgr.24661) 1411 : cluster [DBG] pgmap v2172: 169 pgs: 169 active+clean; 357 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 27 MiB/s rd, 9.8 MiB/s wr, 304 op/s 2024-09-13T00:24:11.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:10 smithi136 bash[75164]: cluster 2024-09-13T00:24:09.975342+0000 mon.a (mon.0) 2417 : cluster [DBG] osdmap e1201: 8 total, 8 up, 8 in 2024-09-13T00:24:11.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:10 smithi136 bash[76371]: cluster 2024-09-13T00:24:09.828843+0000 mgr.y (mgr.24661) 1411 : cluster [DBG] pgmap v2172: 169 pgs: 169 active+clean; 357 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 27 MiB/s rd, 9.8 MiB/s wr, 304 op/s 2024-09-13T00:24:11.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:10 smithi136 bash[76371]: cluster 2024-09-13T00:24:09.975342+0000 mon.a (mon.0) 2417 : cluster [DBG] osdmap e1201: 8 total, 8 up, 8 in 2024-09-13T00:24:11.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:10 smithi181 bash[67587]: cluster 2024-09-13T00:24:09.828843+0000 mgr.y (mgr.24661) 1411 : cluster [DBG] pgmap v2172: 169 pgs: 169 active+clean; 357 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 27 MiB/s rd, 9.8 MiB/s wr, 304 op/s 2024-09-13T00:24:11.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:10 smithi181 bash[67587]: cluster 2024-09-13T00:24:09.975342+0000 mon.a (mon.0) 2417 : cluster [DBG] osdmap e1201: 8 total, 8 up, 8 in 2024-09-13T00:24:12.017 INFO:tasks.workunit.client.0.smithi136.stdout:write: 1 8388609~8388609 2024-09-13T00:24:12.048 INFO:tasks.workunit.client.0.smithi136.stdout:write: X 0~1000 2024-09-13T00:24:12.048 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 1010~1000 2024-09-13T00:24:12.049 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap2 2024-09-13T00:24:12.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:11 smithi136 bash[75164]: cluster 2024-09-13T00:24:10.987759+0000 mon.a (mon.0) 2418 : cluster [DBG] osdmap e1202: 8 total, 8 up, 8 in 2024-09-13T00:24:12.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:11 smithi136 bash[76371]: cluster 2024-09-13T00:24:10.987759+0000 mon.a (mon.0) 2418 : cluster [DBG] osdmap e1202: 8 total, 8 up, 8 in 2024-09-13T00:24:12.314 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:11 smithi181 bash[67587]: cluster 2024-09-13T00:24:10.987759+0000 mon.a (mon.0) 2418 : cluster [DBG] osdmap e1202: 8 total, 8 up, 8 in 2024-09-13T00:24:12.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:24:12 smithi181 bash[95505]: ts=2024-09-13T00:24:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:24:13.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:12 smithi136 bash[75164]: cluster 2024-09-13T00:24:11.830047+0000 mgr.y (mgr.24661) 1412 : cluster [DBG] pgmap v2175: 169 pgs: 169 active+clean; 369 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 48 MiB/s rd, 10 MiB/s wr, 185 op/s 2024-09-13T00:24:13.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:12 smithi136 bash[75164]: cluster 2024-09-13T00:24:11.997380+0000 mon.a (mon.0) 2419 : cluster [DBG] osdmap e1203: 8 total, 8 up, 8 in 2024-09-13T00:24:13.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:12 smithi136 bash[76371]: cluster 2024-09-13T00:24:11.830047+0000 mgr.y (mgr.24661) 1412 : cluster [DBG] pgmap v2175: 169 pgs: 169 active+clean; 369 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 48 MiB/s rd, 10 MiB/s wr, 185 op/s 2024-09-13T00:24:13.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:12 smithi136 bash[76371]: cluster 2024-09-13T00:24:11.997380+0000 mon.a (mon.0) 2419 : cluster [DBG] osdmap e1203: 8 total, 8 up, 8 in 2024-09-13T00:24:13.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:12 smithi181 bash[67587]: cluster 2024-09-13T00:24:11.830047+0000 mgr.y (mgr.24661) 1412 : cluster [DBG] pgmap v2175: 169 pgs: 169 active+clean; 369 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 48 MiB/s rd, 10 MiB/s wr, 185 op/s 2024-09-13T00:24:13.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:12 smithi181 bash[67587]: cluster 2024-09-13T00:24:11.997380+0000 mon.a (mon.0) 2419 : cluster [DBG] osdmap e1203: 8 total, 8 up, 8 in 2024-09-13T00:24:14.041 INFO:tasks.workunit.client.0.smithi136.stdout:write: X 8388609~1000 2024-09-13T00:24:14.058 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 16777228~1000 2024-09-13T00:24:14.058 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 167772160 2024-09-13T00:24:14.078 INFO:tasks.workunit.client.0.smithi136.stdout:write: 2 83886079~8388609 2024-09-13T00:24:14.133 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap3 2024-09-13T00:24:14.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:14 smithi136 bash[75164]: cluster 2024-09-13T00:24:13.004933+0000 mon.a (mon.0) 2420 : cluster [DBG] osdmap e1204: 8 total, 8 up, 8 in 2024-09-13T00:24:14.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:14 smithi136 bash[76371]: cluster 2024-09-13T00:24:13.004933+0000 mon.a (mon.0) 2420 : cluster [DBG] osdmap e1204: 8 total, 8 up, 8 in 2024-09-13T00:24:14.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:14 smithi181 bash[67587]: cluster 2024-09-13T00:24:13.004933+0000 mon.a (mon.0) 2420 : cluster [DBG] osdmap e1204: 8 total, 8 up, 8 in 2024-09-13T00:24:15.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:15 smithi136 bash[75164]: cluster 2024-09-13T00:24:13.831101+0000 mgr.y (mgr.24661) 1413 : cluster [DBG] pgmap v2178: 169 pgs: 169 active+clean; 357 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 46 MiB/s rd, 9.0 MiB/s wr, 210 op/s 2024-09-13T00:24:15.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:15 smithi136 bash[75164]: cluster 2024-09-13T00:24:14.014028+0000 mon.a (mon.0) 2421 : cluster [DBG] osdmap e1205: 8 total, 8 up, 8 in 2024-09-13T00:24:15.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:15 smithi136 bash[76371]: cluster 2024-09-13T00:24:13.831101+0000 mgr.y (mgr.24661) 1413 : cluster [DBG] pgmap v2178: 169 pgs: 169 active+clean; 357 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 46 MiB/s rd, 9.0 MiB/s wr, 210 op/s 2024-09-13T00:24:15.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:15 smithi136 bash[76371]: cluster 2024-09-13T00:24:14.014028+0000 mon.a (mon.0) 2421 : cluster [DBG] osdmap e1205: 8 total, 8 up, 8 in 2024-09-13T00:24:15.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:15 smithi181 bash[67587]: cluster 2024-09-13T00:24:13.831101+0000 mgr.y (mgr.24661) 1413 : cluster [DBG] pgmap v2178: 169 pgs: 169 active+clean; 357 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 46 MiB/s rd, 9.0 MiB/s wr, 210 op/s 2024-09-13T00:24:15.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:15 smithi181 bash[67587]: cluster 2024-09-13T00:24:14.014028+0000 mon.a (mon.0) 2421 : cluster [DBG] osdmap e1205: 8 total, 8 up, 8 in 2024-09-13T00:24:16.063 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:24:16.121 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 83886079~1 2024-09-13T00:24:16.143 INFO:tasks.workunit.client.0.smithi136.stdout:migration_prepare 2024-09-13T00:24:16.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:16 smithi181 bash[67587]: cluster 2024-09-13T00:24:15.022796+0000 mon.a (mon.0) 2422 : cluster [DBG] osdmap e1206: 8 total, 8 up, 8 in 2024-09-13T00:24:16.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:16 smithi136 bash[75164]: cluster 2024-09-13T00:24:15.022796+0000 mon.a (mon.0) 2422 : cluster [DBG] osdmap e1206: 8 total, 8 up, 8 in 2024-09-13T00:24:16.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:16 smithi136 bash[76371]: cluster 2024-09-13T00:24:15.022796+0000 mon.a (mon.0) 2422 : cluster [DBG] osdmap e1206: 8 total, 8 up, 8 in 2024-09-13T00:24:17.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:17 smithi181 bash[67587]: cluster 2024-09-13T00:24:15.832222+0000 mgr.y (mgr.24661) 1414 : cluster [DBG] pgmap v2181: 169 pgs: 169 active+clean; 377 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 11 MiB/s rd, 6.0 MiB/s wr, 99 op/s 2024-09-13T00:24:17.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:17 smithi181 bash[67587]: cluster 2024-09-13T00:24:16.033152+0000 mon.a (mon.0) 2423 : cluster [DBG] osdmap e1207: 8 total, 8 up, 8 in 2024-09-13T00:24:17.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:17 smithi136 bash[76371]: cluster 2024-09-13T00:24:15.832222+0000 mgr.y (mgr.24661) 1414 : cluster [DBG] pgmap v2181: 169 pgs: 169 active+clean; 377 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 11 MiB/s rd, 6.0 MiB/s wr, 99 op/s 2024-09-13T00:24:17.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:17 smithi136 bash[76371]: cluster 2024-09-13T00:24:16.033152+0000 mon.a (mon.0) 2423 : cluster [DBG] osdmap e1207: 8 total, 8 up, 8 in 2024-09-13T00:24:17.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:17 smithi136 bash[75164]: cluster 2024-09-13T00:24:15.832222+0000 mgr.y (mgr.24661) 1414 : cluster [DBG] pgmap v2181: 169 pgs: 169 active+clean; 377 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 11 MiB/s rd, 6.0 MiB/s wr, 99 op/s 2024-09-13T00:24:17.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:17 smithi136 bash[75164]: cluster 2024-09-13T00:24:16.033152+0000 mon.a (mon.0) 2423 : cluster [DBG] osdmap e1207: 8 total, 8 up, 8 in 2024-09-13T00:24:18.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:18 smithi181 bash[67587]: cluster 2024-09-13T00:24:17.047284+0000 mon.a (mon.0) 2424 : cluster [DBG] osdmap e1208: 8 total, 8 up, 8 in 2024-09-13T00:24:18.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:18 smithi181 bash[67587]: audit 2024-09-13T00:24:17.862890+0000 mon.a (mon.0) 2425 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:24:18.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:18 smithi136 bash[76371]: cluster 2024-09-13T00:24:17.047284+0000 mon.a (mon.0) 2424 : cluster [DBG] osdmap e1208: 8 total, 8 up, 8 in 2024-09-13T00:24:18.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:18 smithi136 bash[76371]: audit 2024-09-13T00:24:17.862890+0000 mon.a (mon.0) 2425 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:24:18.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:18 smithi136 bash[75164]: cluster 2024-09-13T00:24:17.047284+0000 mon.a (mon.0) 2424 : cluster [DBG] osdmap e1208: 8 total, 8 up, 8 in 2024-09-13T00:24:18.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:18 smithi136 bash[75164]: audit 2024-09-13T00:24:17.862890+0000 mon.a (mon.0) 2425 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:24:19.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:19 smithi181 bash[67587]: cluster 2024-09-13T00:24:17.833197+0000 mgr.y (mgr.24661) 1415 : cluster [DBG] pgmap v2184: 169 pgs: 169 active+clean; 393 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 3.1 MiB/s rd, 7.0 MiB/s wr, 146 op/s 2024-09-13T00:24:19.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:19 smithi181 bash[67587]: cluster 2024-09-13T00:24:18.056918+0000 mon.a (mon.0) 2426 : cluster [DBG] osdmap e1209: 8 total, 8 up, 8 in 2024-09-13T00:24:19.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:19 smithi181 bash[67587]: audit 2024-09-13T00:24:18.413126+0000 mon.a (mon.0) 2427 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:24:19.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:19 smithi181 bash[67587]: audit 2024-09-13T00:24:18.414674+0000 mon.a (mon.0) 2428 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:24:19.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:19 smithi181 bash[67587]: audit 2024-09-13T00:24:18.424276+0000 mon.a (mon.0) 2429 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:24:19.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:19 smithi181 bash[67587]: cluster 2024-09-13T00:24:18.940959+0000 mon.a (mon.0) 2430 : cluster [DBG] osdmap e1210: 8 total, 8 up, 8 in 2024-09-13T00:24:19.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:19 smithi136 bash[75164]: cluster 2024-09-13T00:24:17.833197+0000 mgr.y (mgr.24661) 1415 : cluster [DBG] pgmap v2184: 169 pgs: 169 active+clean; 393 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 3.1 MiB/s rd, 7.0 MiB/s wr, 146 op/s 2024-09-13T00:24:19.414 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:19 smithi136 bash[75164]: cluster 2024-09-13T00:24:18.056918+0000 mon.a (mon.0) 2426 : cluster [DBG] osdmap e1209: 8 total, 8 up, 8 in 2024-09-13T00:24:19.414 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:19 smithi136 bash[75164]: audit 2024-09-13T00:24:18.413126+0000 mon.a (mon.0) 2427 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:24:19.414 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:19 smithi136 bash[75164]: audit 2024-09-13T00:24:18.414674+0000 mon.a (mon.0) 2428 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:24:19.414 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:19 smithi136 bash[75164]: audit 2024-09-13T00:24:18.424276+0000 mon.a (mon.0) 2429 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:24:19.414 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:19 smithi136 bash[75164]: cluster 2024-09-13T00:24:18.940959+0000 mon.a (mon.0) 2430 : cluster [DBG] osdmap e1210: 8 total, 8 up, 8 in 2024-09-13T00:24:19.414 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:19 smithi136 bash[76371]: cluster 2024-09-13T00:24:17.833197+0000 mgr.y (mgr.24661) 1415 : cluster [DBG] pgmap v2184: 169 pgs: 169 active+clean; 393 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 3.1 MiB/s rd, 7.0 MiB/s wr, 146 op/s 2024-09-13T00:24:19.414 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:19 smithi136 bash[76371]: cluster 2024-09-13T00:24:18.056918+0000 mon.a (mon.0) 2426 : cluster [DBG] osdmap e1209: 8 total, 8 up, 8 in 2024-09-13T00:24:19.414 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:19 smithi136 bash[76371]: audit 2024-09-13T00:24:18.413126+0000 mon.a (mon.0) 2427 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:24:19.414 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:19 smithi136 bash[76371]: audit 2024-09-13T00:24:18.414674+0000 mon.a (mon.0) 2428 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:24:19.415 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:19 smithi136 bash[76371]: audit 2024-09-13T00:24:18.424276+0000 mon.a (mon.0) 2429 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:24:19.415 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:19 smithi136 bash[76371]: cluster 2024-09-13T00:24:18.940959+0000 mon.a (mon.0) 2430 : cluster [DBG] osdmap e1210: 8 total, 8 up, 8 in 2024-09-13T00:24:19.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:24:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:24:19] "GET /metrics HTTP/1.1" 200 38983 "" "Prometheus/2.43.0" 2024-09-13T00:24:19.855 INFO:tasks.workunit.client.0.smithi136.stdout:migration_execute 2024-09-13T00:24:20.119 INFO:tasks.workunit.client.0.smithi136.stdout:migration_commit 2024-09-13T00:24:21.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:21 smithi181 bash[67587]: cluster 2024-09-13T00:24:19.834107+0000 mgr.y (mgr.24661) 1416 : cluster [DBG] pgmap v2187: 169 pgs: 169 active+clean; 397 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 5.1 MiB/s rd, 4.0 MiB/s wr, 152 op/s 2024-09-13T00:24:21.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:21 smithi136 bash[75164]: cluster 2024-09-13T00:24:19.834107+0000 mgr.y (mgr.24661) 1416 : cluster [DBG] pgmap v2187: 169 pgs: 169 active+clean; 397 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 5.1 MiB/s rd, 4.0 MiB/s wr, 152 op/s 2024-09-13T00:24:21.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:21 smithi136 bash[76371]: cluster 2024-09-13T00:24:19.834107+0000 mgr.y (mgr.24661) 1416 : cluster [DBG] pgmap v2187: 169 pgs: 169 active+clean; 397 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 5.1 MiB/s rd, 4.0 MiB/s wr, 152 op/s 2024-09-13T00:24:22.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:22 smithi181 bash[67587]: cluster 2024-09-13T00:24:21.061258+0000 mon.a (mon.0) 2431 : cluster [DBG] osdmap e1211: 8 total, 8 up, 8 in 2024-09-13T00:24:22.348 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:24:22 smithi181 bash[95505]: ts=2024-09-13T00:24:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:24:22.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:22 smithi136 bash[75164]: cluster 2024-09-13T00:24:21.061258+0000 mon.a (mon.0) 2431 : cluster [DBG] osdmap e1211: 8 total, 8 up, 8 in 2024-09-13T00:24:22.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:22 smithi136 bash[76371]: cluster 2024-09-13T00:24:21.061258+0000 mon.a (mon.0) 2431 : cluster [DBG] osdmap e1211: 8 total, 8 up, 8 in 2024-09-13T00:24:23.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:23 smithi181 bash[67587]: cluster 2024-09-13T00:24:21.835204+0000 mgr.y (mgr.24661) 1417 : cluster [DBG] pgmap v2189: 169 pgs: 169 active+clean; 409 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 22 MiB/s rd, 4.2 MiB/s wr, 346 op/s 2024-09-13T00:24:23.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:23 smithi181 bash[67587]: cluster 2024-09-13T00:24:22.090319+0000 mon.a (mon.0) 2432 : cluster [DBG] osdmap e1212: 8 total, 8 up, 8 in 2024-09-13T00:24:23.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:23 smithi136 bash[75164]: cluster 2024-09-13T00:24:21.835204+0000 mgr.y (mgr.24661) 1417 : cluster [DBG] pgmap v2189: 169 pgs: 169 active+clean; 409 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 22 MiB/s rd, 4.2 MiB/s wr, 346 op/s 2024-09-13T00:24:23.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:23 smithi136 bash[75164]: cluster 2024-09-13T00:24:22.090319+0000 mon.a (mon.0) 2432 : cluster [DBG] osdmap e1212: 8 total, 8 up, 8 in 2024-09-13T00:24:23.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:23 smithi136 bash[76371]: cluster 2024-09-13T00:24:21.835204+0000 mgr.y (mgr.24661) 1417 : cluster [DBG] pgmap v2189: 169 pgs: 169 active+clean; 409 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 22 MiB/s rd, 4.2 MiB/s wr, 346 op/s 2024-09-13T00:24:23.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:23 smithi136 bash[76371]: cluster 2024-09-13T00:24:22.090319+0000 mon.a (mon.0) 2432 : cluster [DBG] osdmap e1212: 8 total, 8 up, 8 in 2024-09-13T00:24:23.713 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.SnapsDataPool (13150 ms) 2024-09-13T00:24:23.713 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.SnapsShrinkAfterPrepare 2024-09-13T00:24:23.794 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:24:23.826 INFO:tasks.workunit.client.0.smithi136.stdout:write: 1 0~8388609 2024-09-13T00:24:23.941 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap1 2024-09-13T00:24:24.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:24 smithi136 bash[75164]: cluster 2024-09-13T00:24:23.099486+0000 mon.a (mon.0) 2433 : cluster [DBG] osdmap e1213: 8 total, 8 up, 8 in 2024-09-13T00:24:24.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:24 smithi136 bash[75164]: audit 2024-09-13T00:24:23.810474+0000 mon.a (mon.0) 2434 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:24:24.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:24 smithi136 bash[76371]: cluster 2024-09-13T00:24:23.099486+0000 mon.a (mon.0) 2433 : cluster [DBG] osdmap e1213: 8 total, 8 up, 8 in 2024-09-13T00:24:24.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:24 smithi136 bash[76371]: audit 2024-09-13T00:24:23.810474+0000 mon.a (mon.0) 2434 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:24:24.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:24 smithi181 bash[67587]: cluster 2024-09-13T00:24:23.099486+0000 mon.a (mon.0) 2433 : cluster [DBG] osdmap e1213: 8 total, 8 up, 8 in 2024-09-13T00:24:24.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:24 smithi181 bash[67587]: audit 2024-09-13T00:24:23.810474+0000 mon.a (mon.0) 2434 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:24:25.153 INFO:tasks.workunit.client.0.smithi136.stdout:write: 1 8388609~8388609 2024-09-13T00:24:25.182 INFO:tasks.workunit.client.0.smithi136.stdout:write: X 0~1000 2024-09-13T00:24:25.182 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 1010~1000 2024-09-13T00:24:25.182 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap2 2024-09-13T00:24:25.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:25 smithi136 bash[75164]: cluster 2024-09-13T00:24:23.836425+0000 mgr.y (mgr.24661) 1418 : cluster [DBG] pgmap v2192: 169 pgs: 169 active+clean; 413 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 28 MiB/s rd, 4.9 MiB/s wr, 327 op/s 2024-09-13T00:24:25.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:25 smithi136 bash[75164]: cluster 2024-09-13T00:24:24.102457+0000 mon.a (mon.0) 2435 : cluster [DBG] osdmap e1214: 8 total, 8 up, 8 in 2024-09-13T00:24:25.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:25 smithi136 bash[76371]: cluster 2024-09-13T00:24:23.836425+0000 mgr.y (mgr.24661) 1418 : cluster [DBG] pgmap v2192: 169 pgs: 169 active+clean; 413 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 28 MiB/s rd, 4.9 MiB/s wr, 327 op/s 2024-09-13T00:24:25.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:25 smithi136 bash[76371]: cluster 2024-09-13T00:24:24.102457+0000 mon.a (mon.0) 2435 : cluster [DBG] osdmap e1214: 8 total, 8 up, 8 in 2024-09-13T00:24:25.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:25 smithi181 bash[67587]: cluster 2024-09-13T00:24:23.836425+0000 mgr.y (mgr.24661) 1418 : cluster [DBG] pgmap v2192: 169 pgs: 169 active+clean; 413 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 28 MiB/s rd, 4.9 MiB/s wr, 327 op/s 2024-09-13T00:24:25.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:25 smithi181 bash[67587]: cluster 2024-09-13T00:24:24.102457+0000 mon.a (mon.0) 2435 : cluster [DBG] osdmap e1214: 8 total, 8 up, 8 in 2024-09-13T00:24:26.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:26 smithi136 bash[75164]: cluster 2024-09-13T00:24:25.126232+0000 mon.a (mon.0) 2436 : cluster [DBG] osdmap e1215: 8 total, 8 up, 8 in 2024-09-13T00:24:26.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:26 smithi136 bash[76371]: cluster 2024-09-13T00:24:25.126232+0000 mon.a (mon.0) 2436 : cluster [DBG] osdmap e1215: 8 total, 8 up, 8 in 2024-09-13T00:24:26.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:26 smithi181 bash[67587]: cluster 2024-09-13T00:24:25.126232+0000 mon.a (mon.0) 2436 : cluster [DBG] osdmap e1215: 8 total, 8 up, 8 in 2024-09-13T00:24:27.165 INFO:tasks.workunit.client.0.smithi136.stdout:write: X 8388609~1000 2024-09-13T00:24:27.181 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 16777228~1000 2024-09-13T00:24:27.182 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 167772160 2024-09-13T00:24:27.204 INFO:tasks.workunit.client.0.smithi136.stdout:write: 2 83886079~8388609 2024-09-13T00:24:27.261 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap3 2024-09-13T00:24:27.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:27 smithi136 bash[75164]: cluster 2024-09-13T00:24:25.837532+0000 mgr.y (mgr.24661) 1419 : cluster [DBG] pgmap v2195: 169 pgs: 169 active+clean; 409 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 19 MiB/s rd, 4.0 MiB/s wr, 81 op/s 2024-09-13T00:24:27.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:27 smithi136 bash[75164]: cluster 2024-09-13T00:24:26.136010+0000 mon.a (mon.0) 2437 : cluster [DBG] osdmap e1216: 8 total, 8 up, 8 in 2024-09-13T00:24:27.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:27 smithi136 bash[76371]: cluster 2024-09-13T00:24:25.837532+0000 mgr.y (mgr.24661) 1419 : cluster [DBG] pgmap v2195: 169 pgs: 169 active+clean; 409 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 19 MiB/s rd, 4.0 MiB/s wr, 81 op/s 2024-09-13T00:24:27.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:27 smithi136 bash[76371]: cluster 2024-09-13T00:24:26.136010+0000 mon.a (mon.0) 2437 : cluster [DBG] osdmap e1216: 8 total, 8 up, 8 in 2024-09-13T00:24:27.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:27 smithi181 bash[67587]: cluster 2024-09-13T00:24:25.837532+0000 mgr.y (mgr.24661) 1419 : cluster [DBG] pgmap v2195: 169 pgs: 169 active+clean; 409 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 19 MiB/s rd, 4.0 MiB/s wr, 81 op/s 2024-09-13T00:24:27.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:27 smithi181 bash[67587]: cluster 2024-09-13T00:24:26.136010+0000 mon.a (mon.0) 2437 : cluster [DBG] osdmap e1216: 8 total, 8 up, 8 in 2024-09-13T00:24:28.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:28 smithi136 bash[75164]: cluster 2024-09-13T00:24:27.139134+0000 mon.a (mon.0) 2438 : cluster [DBG] osdmap e1217: 8 total, 8 up, 8 in 2024-09-13T00:24:28.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:28 smithi136 bash[76371]: cluster 2024-09-13T00:24:27.139134+0000 mon.a (mon.0) 2438 : cluster [DBG] osdmap e1217: 8 total, 8 up, 8 in 2024-09-13T00:24:28.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:28 smithi181 bash[67587]: cluster 2024-09-13T00:24:27.139134+0000 mon.a (mon.0) 2438 : cluster [DBG] osdmap e1217: 8 total, 8 up, 8 in 2024-09-13T00:24:28.972 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:24:29.032 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 83886079~1 2024-09-13T00:24:29.053 INFO:tasks.workunit.client.0.smithi136.stdout:migration_prepare 2024-09-13T00:24:29.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:29 smithi136 bash[75164]: cluster 2024-09-13T00:24:27.838489+0000 mgr.y (mgr.24661) 1420 : cluster [DBG] pgmap v2198: 169 pgs: 169 active+clean; 417 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 34 MiB/s rd, 5.0 MiB/s wr, 134 op/s 2024-09-13T00:24:29.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:29 smithi136 bash[75164]: cluster 2024-09-13T00:24:28.164172+0000 mon.a (mon.0) 2439 : cluster [DBG] osdmap e1218: 8 total, 8 up, 8 in 2024-09-13T00:24:29.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:29 smithi136 bash[75164]: cluster 2024-09-13T00:24:28.945174+0000 mon.a (mon.0) 2440 : cluster [DBG] osdmap e1219: 8 total, 8 up, 8 in 2024-09-13T00:24:29.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:29 smithi136 bash[76371]: cluster 2024-09-13T00:24:27.838489+0000 mgr.y (mgr.24661) 1420 : cluster [DBG] pgmap v2198: 169 pgs: 169 active+clean; 417 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 34 MiB/s rd, 5.0 MiB/s wr, 134 op/s 2024-09-13T00:24:29.414 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:29 smithi136 bash[76371]: cluster 2024-09-13T00:24:28.164172+0000 mon.a (mon.0) 2439 : cluster [DBG] osdmap e1218: 8 total, 8 up, 8 in 2024-09-13T00:24:29.414 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:29 smithi136 bash[76371]: cluster 2024-09-13T00:24:28.945174+0000 mon.a (mon.0) 2440 : cluster [DBG] osdmap e1219: 8 total, 8 up, 8 in 2024-09-13T00:24:29.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:29 smithi181 bash[67587]: cluster 2024-09-13T00:24:27.838489+0000 mgr.y (mgr.24661) 1420 : cluster [DBG] pgmap v2198: 169 pgs: 169 active+clean; 417 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 34 MiB/s rd, 5.0 MiB/s wr, 134 op/s 2024-09-13T00:24:29.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:29 smithi181 bash[67587]: cluster 2024-09-13T00:24:28.164172+0000 mon.a (mon.0) 2439 : cluster [DBG] osdmap e1218: 8 total, 8 up, 8 in 2024-09-13T00:24:29.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:29 smithi181 bash[67587]: cluster 2024-09-13T00:24:28.945174+0000 mon.a (mon.0) 2440 : cluster [DBG] osdmap e1219: 8 total, 8 up, 8 in 2024-09-13T00:24:29.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:24:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:24:29] "GET /metrics HTTP/1.1" 200 38988 "" "Prometheus/2.43.0" 2024-09-13T00:24:31.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:31 smithi136 bash[75164]: cluster 2024-09-13T00:24:29.839531+0000 mgr.y (mgr.24661) 1421 : cluster [DBG] pgmap v2201: 169 pgs: 169 active+clean; 437 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 25 MiB/s rd, 7.0 MiB/s wr, 160 op/s 2024-09-13T00:24:31.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:31 smithi136 bash[75164]: cluster 2024-09-13T00:24:30.175193+0000 mon.a (mon.0) 2441 : cluster [DBG] osdmap e1220: 8 total, 8 up, 8 in 2024-09-13T00:24:31.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:31 smithi136 bash[76371]: cluster 2024-09-13T00:24:29.839531+0000 mgr.y (mgr.24661) 1421 : cluster [DBG] pgmap v2201: 169 pgs: 169 active+clean; 437 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 25 MiB/s rd, 7.0 MiB/s wr, 160 op/s 2024-09-13T00:24:31.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:31 smithi136 bash[76371]: cluster 2024-09-13T00:24:30.175193+0000 mon.a (mon.0) 2441 : cluster [DBG] osdmap e1220: 8 total, 8 up, 8 in 2024-09-13T00:24:31.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:31 smithi181 bash[67587]: cluster 2024-09-13T00:24:29.839531+0000 mgr.y (mgr.24661) 1421 : cluster [DBG] pgmap v2201: 169 pgs: 169 active+clean; 437 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 25 MiB/s rd, 7.0 MiB/s wr, 160 op/s 2024-09-13T00:24:31.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:31 smithi181 bash[67587]: cluster 2024-09-13T00:24:30.175193+0000 mon.a (mon.0) 2441 : cluster [DBG] osdmap e1220: 8 total, 8 up, 8 in 2024-09-13T00:24:32.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:32 smithi136 bash[75164]: cluster 2024-09-13T00:24:31.175722+0000 mon.a (mon.0) 2442 : cluster [DBG] osdmap e1221: 8 total, 8 up, 8 in 2024-09-13T00:24:32.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:32 smithi136 bash[76371]: cluster 2024-09-13T00:24:31.175722+0000 mon.a (mon.0) 2442 : cluster [DBG] osdmap e1221: 8 total, 8 up, 8 in 2024-09-13T00:24:32.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:32 smithi181 bash[67587]: cluster 2024-09-13T00:24:31.175722+0000 mon.a (mon.0) 2442 : cluster [DBG] osdmap e1221: 8 total, 8 up, 8 in 2024-09-13T00:24:32.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:24:32 smithi181 bash[95505]: ts=2024-09-13T00:24:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:24:33.128 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 41943040 2024-09-13T00:24:33.189 INFO:tasks.workunit.client.0.smithi136.stdout:migration_execute 2024-09-13T00:24:33.468 INFO:tasks.workunit.client.0.smithi136.stdout:migration_commit 2024-09-13T00:24:33.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:33 smithi136 bash[75164]: cluster 2024-09-13T00:24:31.840635+0000 mgr.y (mgr.24661) 1422 : cluster [DBG] pgmap v2204: 169 pgs: 169 active+clean; 453 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 2.1 MiB/s rd, 7.0 MiB/s wr, 159 op/s 2024-09-13T00:24:33.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:33 smithi136 bash[75164]: cluster 2024-09-13T00:24:32.207854+0000 mon.a (mon.0) 2443 : cluster [DBG] osdmap e1222: 8 total, 8 up, 8 in 2024-09-13T00:24:33.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:33 smithi136 bash[76371]: cluster 2024-09-13T00:24:31.840635+0000 mgr.y (mgr.24661) 1422 : cluster [DBG] pgmap v2204: 169 pgs: 169 active+clean; 453 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 2.1 MiB/s rd, 7.0 MiB/s wr, 159 op/s 2024-09-13T00:24:33.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:33 smithi136 bash[76371]: cluster 2024-09-13T00:24:32.207854+0000 mon.a (mon.0) 2443 : cluster [DBG] osdmap e1222: 8 total, 8 up, 8 in 2024-09-13T00:24:33.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:33 smithi181 bash[67587]: cluster 2024-09-13T00:24:31.840635+0000 mgr.y (mgr.24661) 1422 : cluster [DBG] pgmap v2204: 169 pgs: 169 active+clean; 453 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 2.1 MiB/s rd, 7.0 MiB/s wr, 159 op/s 2024-09-13T00:24:33.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:33 smithi181 bash[67587]: cluster 2024-09-13T00:24:32.207854+0000 mon.a (mon.0) 2443 : cluster [DBG] osdmap e1222: 8 total, 8 up, 8 in 2024-09-13T00:24:35.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:35 smithi136 bash[75164]: cluster 2024-09-13T00:24:33.841800+0000 mgr.y (mgr.24661) 1423 : cluster [DBG] pgmap v2206: 169 pgs: 169 active+clean; 461 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 8.2 MiB/s rd, 4.1 MiB/s wr, 109 op/s 2024-09-13T00:24:35.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:35 smithi136 bash[75164]: cluster 2024-09-13T00:24:34.211868+0000 mon.a (mon.0) 2444 : cluster [DBG] osdmap e1223: 8 total, 8 up, 8 in 2024-09-13T00:24:35.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:35 smithi136 bash[76371]: cluster 2024-09-13T00:24:33.841800+0000 mgr.y (mgr.24661) 1423 : cluster [DBG] pgmap v2206: 169 pgs: 169 active+clean; 461 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 8.2 MiB/s rd, 4.1 MiB/s wr, 109 op/s 2024-09-13T00:24:35.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:35 smithi136 bash[76371]: cluster 2024-09-13T00:24:34.211868+0000 mon.a (mon.0) 2444 : cluster [DBG] osdmap e1223: 8 total, 8 up, 8 in 2024-09-13T00:24:35.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:35 smithi181 bash[67587]: cluster 2024-09-13T00:24:33.841800+0000 mgr.y (mgr.24661) 1423 : cluster [DBG] pgmap v2206: 169 pgs: 169 active+clean; 461 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 8.2 MiB/s rd, 4.1 MiB/s wr, 109 op/s 2024-09-13T00:24:35.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:35 smithi181 bash[67587]: cluster 2024-09-13T00:24:34.211868+0000 mon.a (mon.0) 2444 : cluster [DBG] osdmap e1223: 8 total, 8 up, 8 in 2024-09-13T00:24:36.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:36 smithi136 bash[75164]: cluster 2024-09-13T00:24:35.222878+0000 mon.a (mon.0) 2445 : cluster [DBG] osdmap e1224: 8 total, 8 up, 8 in 2024-09-13T00:24:36.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:36 smithi136 bash[76371]: cluster 2024-09-13T00:24:35.222878+0000 mon.a (mon.0) 2445 : cluster [DBG] osdmap e1224: 8 total, 8 up, 8 in 2024-09-13T00:24:36.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:36 smithi181 bash[67587]: cluster 2024-09-13T00:24:35.222878+0000 mon.a (mon.0) 2445 : cluster [DBG] osdmap e1224: 8 total, 8 up, 8 in 2024-09-13T00:24:36.790 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.SnapsShrinkAfterPrepare (13077 ms) 2024-09-13T00:24:36.790 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.SnapsShrinkToZeroBeforePrepare 2024-09-13T00:24:36.875 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:24:36.903 INFO:tasks.workunit.client.0.smithi136.stdout:write: 1 0~8388609 2024-09-13T00:24:37.019 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap1 2024-09-13T00:24:37.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:37 smithi136 bash[75164]: cluster 2024-09-13T00:24:35.842911+0000 mgr.y (mgr.24661) 1424 : cluster [DBG] pgmap v2209: 169 pgs: 169 active+clean; 465 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 19 MiB/s rd, 4.3 MiB/s wr, 79 op/s 2024-09-13T00:24:37.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:37 smithi136 bash[75164]: cluster 2024-09-13T00:24:36.238173+0000 mon.a (mon.0) 2446 : cluster [DBG] osdmap e1225: 8 total, 8 up, 8 in 2024-09-13T00:24:37.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:37 smithi136 bash[76371]: cluster 2024-09-13T00:24:35.842911+0000 mgr.y (mgr.24661) 1424 : cluster [DBG] pgmap v2209: 169 pgs: 169 active+clean; 465 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 19 MiB/s rd, 4.3 MiB/s wr, 79 op/s 2024-09-13T00:24:37.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:37 smithi136 bash[76371]: cluster 2024-09-13T00:24:36.238173+0000 mon.a (mon.0) 2446 : cluster [DBG] osdmap e1225: 8 total, 8 up, 8 in 2024-09-13T00:24:37.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:37 smithi181 bash[67587]: cluster 2024-09-13T00:24:35.842911+0000 mgr.y (mgr.24661) 1424 : cluster [DBG] pgmap v2209: 169 pgs: 169 active+clean; 465 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 19 MiB/s rd, 4.3 MiB/s wr, 79 op/s 2024-09-13T00:24:37.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:37 smithi181 bash[67587]: cluster 2024-09-13T00:24:36.238173+0000 mon.a (mon.0) 2446 : cluster [DBG] osdmap e1225: 8 total, 8 up, 8 in 2024-09-13T00:24:38.289 INFO:tasks.workunit.client.0.smithi136.stdout:write: 1 8388609~8388609 2024-09-13T00:24:38.327 INFO:tasks.workunit.client.0.smithi136.stdout:write: X 0~1000 2024-09-13T00:24:38.327 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 1010~1000 2024-09-13T00:24:38.327 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap2 2024-09-13T00:24:38.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:38 smithi136 bash[75164]: cluster 2024-09-13T00:24:37.238373+0000 mon.a (mon.0) 2447 : cluster [DBG] osdmap e1226: 8 total, 8 up, 8 in 2024-09-13T00:24:38.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:38 smithi136 bash[76371]: cluster 2024-09-13T00:24:37.238373+0000 mon.a (mon.0) 2447 : cluster [DBG] osdmap e1226: 8 total, 8 up, 8 in 2024-09-13T00:24:38.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:38 smithi181 bash[67587]: cluster 2024-09-13T00:24:37.238373+0000 mon.a (mon.0) 2447 : cluster [DBG] osdmap e1226: 8 total, 8 up, 8 in 2024-09-13T00:24:39.509 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:24:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:24:39] "GET /metrics HTTP/1.1" 200 38987 "" "Prometheus/2.43.0" 2024-09-13T00:24:39.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:39 smithi136 bash[75164]: cluster 2024-09-13T00:24:37.843980+0000 mgr.y (mgr.24661) 1425 : cluster [DBG] pgmap v2212: 169 pgs: 169 active+clean; 477 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 45 MiB/s rd, 5.0 MiB/s wr, 422 op/s 2024-09-13T00:24:39.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:39 smithi136 bash[75164]: cluster 2024-09-13T00:24:38.261785+0000 mon.a (mon.0) 2448 : cluster [DBG] osdmap e1227: 8 total, 8 up, 8 in 2024-09-13T00:24:39.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:39 smithi136 bash[75164]: audit 2024-09-13T00:24:38.811146+0000 mon.a (mon.0) 2449 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:24:39.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:39 smithi136 bash[76371]: cluster 2024-09-13T00:24:37.843980+0000 mgr.y (mgr.24661) 1425 : cluster [DBG] pgmap v2212: 169 pgs: 169 active+clean; 477 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 45 MiB/s rd, 5.0 MiB/s wr, 422 op/s 2024-09-13T00:24:39.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:39 smithi136 bash[76371]: cluster 2024-09-13T00:24:38.261785+0000 mon.a (mon.0) 2448 : cluster [DBG] osdmap e1227: 8 total, 8 up, 8 in 2024-09-13T00:24:39.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:39 smithi136 bash[76371]: audit 2024-09-13T00:24:38.811146+0000 mon.a (mon.0) 2449 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:24:39.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:39 smithi181 bash[67587]: cluster 2024-09-13T00:24:37.843980+0000 mgr.y (mgr.24661) 1425 : cluster [DBG] pgmap v2212: 169 pgs: 169 active+clean; 477 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 45 MiB/s rd, 5.0 MiB/s wr, 422 op/s 2024-09-13T00:24:39.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:39 smithi181 bash[67587]: cluster 2024-09-13T00:24:38.261785+0000 mon.a (mon.0) 2448 : cluster [DBG] osdmap e1227: 8 total, 8 up, 8 in 2024-09-13T00:24:39.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:39 smithi181 bash[67587]: audit 2024-09-13T00:24:38.811146+0000 mon.a (mon.0) 2449 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:24:40.309 INFO:tasks.workunit.client.0.smithi136.stdout:write: X 8388609~1000 2024-09-13T00:24:40.325 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 16777228~1000 2024-09-13T00:24:40.325 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 167772160 2024-09-13T00:24:40.347 INFO:tasks.workunit.client.0.smithi136.stdout:write: 2 83886079~8388609 2024-09-13T00:24:40.408 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap3 2024-09-13T00:24:40.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:40 smithi181 bash[67587]: cluster 2024-09-13T00:24:39.273834+0000 mon.a (mon.0) 2450 : cluster [DBG] osdmap e1228: 8 total, 8 up, 8 in 2024-09-13T00:24:40.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:40 smithi136 bash[75164]: cluster 2024-09-13T00:24:39.273834+0000 mon.a (mon.0) 2450 : cluster [DBG] osdmap e1228: 8 total, 8 up, 8 in 2024-09-13T00:24:40.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:40 smithi136 bash[76371]: cluster 2024-09-13T00:24:39.273834+0000 mon.a (mon.0) 2450 : cluster [DBG] osdmap e1228: 8 total, 8 up, 8 in 2024-09-13T00:24:41.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:41 smithi181 bash[67587]: cluster 2024-09-13T00:24:39.844953+0000 mgr.y (mgr.24661) 1426 : cluster [DBG] pgmap v2215: 169 pgs: 169 active+clean; 465 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 447 op/s 2024-09-13T00:24:41.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:41 smithi181 bash[67587]: cluster 2024-09-13T00:24:40.284026+0000 mon.a (mon.0) 2451 : cluster [DBG] osdmap e1229: 8 total, 8 up, 8 in 2024-09-13T00:24:41.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:41 smithi136 bash[75164]: cluster 2024-09-13T00:24:39.844953+0000 mgr.y (mgr.24661) 1426 : cluster [DBG] pgmap v2215: 169 pgs: 169 active+clean; 465 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 447 op/s 2024-09-13T00:24:41.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:41 smithi136 bash[75164]: cluster 2024-09-13T00:24:40.284026+0000 mon.a (mon.0) 2451 : cluster [DBG] osdmap e1229: 8 total, 8 up, 8 in 2024-09-13T00:24:41.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:41 smithi136 bash[76371]: cluster 2024-09-13T00:24:39.844953+0000 mgr.y (mgr.24661) 1426 : cluster [DBG] pgmap v2215: 169 pgs: 169 active+clean; 465 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 447 op/s 2024-09-13T00:24:41.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:41 smithi136 bash[76371]: cluster 2024-09-13T00:24:40.284026+0000 mon.a (mon.0) 2451 : cluster [DBG] osdmap e1229: 8 total, 8 up, 8 in 2024-09-13T00:24:42.331 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:24:42.382 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 83886079~1 2024-09-13T00:24:42.406 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 0 2024-09-13T00:24:42.448 INFO:tasks.workunit.client.0.smithi136.stdout:migration_prepare 2024-09-13T00:24:42.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:42 smithi181 bash[67587]: cluster 2024-09-13T00:24:41.294078+0000 mon.a (mon.0) 2452 : cluster [DBG] osdmap e1230: 8 total, 8 up, 8 in 2024-09-13T00:24:42.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:24:42 smithi181 bash[95505]: ts=2024-09-13T00:24:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:24:42.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:42 smithi136 bash[76371]: cluster 2024-09-13T00:24:41.294078+0000 mon.a (mon.0) 2452 : cluster [DBG] osdmap e1230: 8 total, 8 up, 8 in 2024-09-13T00:24:42.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:42 smithi136 bash[75164]: cluster 2024-09-13T00:24:41.294078+0000 mon.a (mon.0) 2452 : cluster [DBG] osdmap e1230: 8 total, 8 up, 8 in 2024-09-13T00:24:43.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:43 smithi181 bash[67587]: cluster 2024-09-13T00:24:41.846055+0000 mgr.y (mgr.24661) 1427 : cluster [DBG] pgmap v2218: 169 pgs: 169 active+clean; 501 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 15 MiB/s rd, 11 MiB/s wr, 161 op/s 2024-09-13T00:24:43.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:43 smithi181 bash[67587]: cluster 2024-09-13T00:24:42.305501+0000 mon.a (mon.0) 2453 : cluster [DBG] osdmap e1231: 8 total, 8 up, 8 in 2024-09-13T00:24:43.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:43 smithi136 bash[75164]: cluster 2024-09-13T00:24:41.846055+0000 mgr.y (mgr.24661) 1427 : cluster [DBG] pgmap v2218: 169 pgs: 169 active+clean; 501 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 15 MiB/s rd, 11 MiB/s wr, 161 op/s 2024-09-13T00:24:43.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:43 smithi136 bash[75164]: cluster 2024-09-13T00:24:42.305501+0000 mon.a (mon.0) 2453 : cluster [DBG] osdmap e1231: 8 total, 8 up, 8 in 2024-09-13T00:24:43.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:43 smithi136 bash[76371]: cluster 2024-09-13T00:24:41.846055+0000 mgr.y (mgr.24661) 1427 : cluster [DBG] pgmap v2218: 169 pgs: 169 active+clean; 501 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 15 MiB/s rd, 11 MiB/s wr, 161 op/s 2024-09-13T00:24:43.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:43 smithi136 bash[76371]: cluster 2024-09-13T00:24:42.305501+0000 mon.a (mon.0) 2453 : cluster [DBG] osdmap e1231: 8 total, 8 up, 8 in 2024-09-13T00:24:44.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:44 smithi181 bash[67587]: cluster 2024-09-13T00:24:43.313088+0000 mon.a (mon.0) 2454 : cluster [DBG] osdmap e1232: 8 total, 8 up, 8 in 2024-09-13T00:24:44.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:44 smithi181 bash[67587]: cluster 2024-09-13T00:24:43.847067+0000 mgr.y (mgr.24661) 1428 : cluster [DBG] pgmap v2221: 169 pgs: 169 active+clean; 501 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 10 MiB/s rd, 9.0 MiB/s wr, 159 op/s 2024-09-13T00:24:44.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:44 smithi181 bash[67587]: cluster 2024-09-13T00:24:43.949315+0000 mon.a (mon.0) 2455 : cluster [DBG] osdmap e1233: 8 total, 8 up, 8 in 2024-09-13T00:24:44.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:44 smithi136 bash[75164]: cluster 2024-09-13T00:24:43.313088+0000 mon.a (mon.0) 2454 : cluster [DBG] osdmap e1232: 8 total, 8 up, 8 in 2024-09-13T00:24:44.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:44 smithi136 bash[75164]: cluster 2024-09-13T00:24:43.847067+0000 mgr.y (mgr.24661) 1428 : cluster [DBG] pgmap v2221: 169 pgs: 169 active+clean; 501 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 10 MiB/s rd, 9.0 MiB/s wr, 159 op/s 2024-09-13T00:24:44.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:44 smithi136 bash[75164]: cluster 2024-09-13T00:24:43.949315+0000 mon.a (mon.0) 2455 : cluster [DBG] osdmap e1233: 8 total, 8 up, 8 in 2024-09-13T00:24:44.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:44 smithi136 bash[76371]: cluster 2024-09-13T00:24:43.313088+0000 mon.a (mon.0) 2454 : cluster [DBG] osdmap e1232: 8 total, 8 up, 8 in 2024-09-13T00:24:44.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:44 smithi136 bash[76371]: cluster 2024-09-13T00:24:43.847067+0000 mgr.y (mgr.24661) 1428 : cluster [DBG] pgmap v2221: 169 pgs: 169 active+clean; 501 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 10 MiB/s rd, 9.0 MiB/s wr, 159 op/s 2024-09-13T00:24:44.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:44 smithi136 bash[76371]: cluster 2024-09-13T00:24:43.949315+0000 mon.a (mon.0) 2455 : cluster [DBG] osdmap e1233: 8 total, 8 up, 8 in 2024-09-13T00:24:45.738 INFO:tasks.workunit.client.0.smithi136.stdout:migration_execute 2024-09-13T00:24:46.055 INFO:tasks.workunit.client.0.smithi136.stdout:migration_commit 2024-09-13T00:24:46.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:45 smithi136 bash[76371]: cluster 2024-09-13T00:24:44.954218+0000 mon.a (mon.0) 2456 : cluster [DBG] osdmap e1234: 8 total, 8 up, 8 in 2024-09-13T00:24:46.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:45 smithi136 bash[75164]: cluster 2024-09-13T00:24:44.954218+0000 mon.a (mon.0) 2456 : cluster [DBG] osdmap e1234: 8 total, 8 up, 8 in 2024-09-13T00:24:46.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:45 smithi181 bash[67587]: cluster 2024-09-13T00:24:44.954218+0000 mon.a (mon.0) 2456 : cluster [DBG] osdmap e1234: 8 total, 8 up, 8 in 2024-09-13T00:24:47.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:46 smithi136 bash[75164]: cluster 2024-09-13T00:24:45.848127+0000 mgr.y (mgr.24661) 1429 : cluster [DBG] pgmap v2224: 169 pgs: 169 active+clean; 509 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.0 MiB/s wr, 195 op/s 2024-09-13T00:24:47.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:46 smithi136 bash[76371]: cluster 2024-09-13T00:24:45.848127+0000 mgr.y (mgr.24661) 1429 : cluster [DBG] pgmap v2224: 169 pgs: 169 active+clean; 509 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.0 MiB/s wr, 195 op/s 2024-09-13T00:24:47.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:46 smithi181 bash[67587]: cluster 2024-09-13T00:24:45.848127+0000 mgr.y (mgr.24661) 1429 : cluster [DBG] pgmap v2224: 169 pgs: 169 active+clean; 509 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.0 MiB/s wr, 195 op/s 2024-09-13T00:24:48.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:47 smithi136 bash[75164]: cluster 2024-09-13T00:24:46.965091+0000 mon.a (mon.0) 2457 : cluster [DBG] osdmap e1235: 8 total, 8 up, 8 in 2024-09-13T00:24:48.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:47 smithi136 bash[76371]: cluster 2024-09-13T00:24:46.965091+0000 mon.a (mon.0) 2457 : cluster [DBG] osdmap e1235: 8 total, 8 up, 8 in 2024-09-13T00:24:48.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:47 smithi181 bash[67587]: cluster 2024-09-13T00:24:46.965091+0000 mon.a (mon.0) 2457 : cluster [DBG] osdmap e1235: 8 total, 8 up, 8 in 2024-09-13T00:24:49.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:48 smithi136 bash[75164]: cluster 2024-09-13T00:24:47.849074+0000 mgr.y (mgr.24661) 1430 : cluster [DBG] pgmap v2226: 169 pgs: 169 active+clean; 525 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 21 MiB/s rd, 4.4 MiB/s wr, 357 op/s 2024-09-13T00:24:49.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:48 smithi136 bash[75164]: cluster 2024-09-13T00:24:47.987608+0000 mon.a (mon.0) 2458 : cluster [DBG] osdmap e1236: 8 total, 8 up, 8 in 2024-09-13T00:24:49.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:48 smithi136 bash[76371]: cluster 2024-09-13T00:24:47.849074+0000 mgr.y (mgr.24661) 1430 : cluster [DBG] pgmap v2226: 169 pgs: 169 active+clean; 525 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 21 MiB/s rd, 4.4 MiB/s wr, 357 op/s 2024-09-13T00:24:49.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:48 smithi136 bash[76371]: cluster 2024-09-13T00:24:47.987608+0000 mon.a (mon.0) 2458 : cluster [DBG] osdmap e1236: 8 total, 8 up, 8 in 2024-09-13T00:24:49.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:48 smithi181 bash[67587]: cluster 2024-09-13T00:24:47.849074+0000 mgr.y (mgr.24661) 1430 : cluster [DBG] pgmap v2226: 169 pgs: 169 active+clean; 525 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 21 MiB/s rd, 4.4 MiB/s wr, 357 op/s 2024-09-13T00:24:49.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:48 smithi181 bash[67587]: cluster 2024-09-13T00:24:47.987608+0000 mon.a (mon.0) 2458 : cluster [DBG] osdmap e1236: 8 total, 8 up, 8 in 2024-09-13T00:24:49.437 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.SnapsShrinkToZeroBeforePrepare (12647 ms) 2024-09-13T00:24:49.437 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.SnapsShrinkToZeroAfterPrepare 2024-09-13T00:24:49.521 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:24:49.550 INFO:tasks.workunit.client.0.smithi136.stdout:write: 1 0~8388609 2024-09-13T00:24:49.665 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap1 2024-09-13T00:24:49.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:24:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:24:49] "GET /metrics HTTP/1.1" 200 38987 "" "Prometheus/2.43.0" 2024-09-13T00:24:50.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:50 smithi136 bash[76371]: cluster 2024-09-13T00:24:48.995461+0000 mon.a (mon.0) 2459 : cluster [DBG] osdmap e1237: 8 total, 8 up, 8 in 2024-09-13T00:24:50.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:50 smithi136 bash[75164]: cluster 2024-09-13T00:24:48.995461+0000 mon.a (mon.0) 2459 : cluster [DBG] osdmap e1237: 8 total, 8 up, 8 in 2024-09-13T00:24:50.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:50 smithi181 bash[67587]: cluster 2024-09-13T00:24:48.995461+0000 mon.a (mon.0) 2459 : cluster [DBG] osdmap e1237: 8 total, 8 up, 8 in 2024-09-13T00:24:51.037 INFO:tasks.workunit.client.0.smithi136.stdout:write: 1 8388609~8388609 2024-09-13T00:24:51.084 INFO:tasks.workunit.client.0.smithi136.stdout:write: X 0~1000 2024-09-13T00:24:51.084 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 1010~1000 2024-09-13T00:24:51.084 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap2 2024-09-13T00:24:51.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:51 smithi181 bash[67587]: cluster 2024-09-13T00:24:49.849993+0000 mgr.y (mgr.24661) 1431 : cluster [DBG] pgmap v2229: 169 pgs: 169 active+clean; 533 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 30 MiB/s rd, 4.9 MiB/s wr, 318 op/s 2024-09-13T00:24:51.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:51 smithi181 bash[67587]: cluster 2024-09-13T00:24:50.008297+0000 mon.a (mon.0) 2460 : cluster [DBG] osdmap e1238: 8 total, 8 up, 8 in 2024-09-13T00:24:51.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:51 smithi136 bash[75164]: cluster 2024-09-13T00:24:49.849993+0000 mgr.y (mgr.24661) 1431 : cluster [DBG] pgmap v2229: 169 pgs: 169 active+clean; 533 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 30 MiB/s rd, 4.9 MiB/s wr, 318 op/s 2024-09-13T00:24:51.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:51 smithi136 bash[75164]: cluster 2024-09-13T00:24:50.008297+0000 mon.a (mon.0) 2460 : cluster [DBG] osdmap e1238: 8 total, 8 up, 8 in 2024-09-13T00:24:51.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:51 smithi136 bash[76371]: cluster 2024-09-13T00:24:49.849993+0000 mgr.y (mgr.24661) 1431 : cluster [DBG] pgmap v2229: 169 pgs: 169 active+clean; 533 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 30 MiB/s rd, 4.9 MiB/s wr, 318 op/s 2024-09-13T00:24:51.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:51 smithi136 bash[76371]: cluster 2024-09-13T00:24:50.008297+0000 mon.a (mon.0) 2460 : cluster [DBG] osdmap e1238: 8 total, 8 up, 8 in 2024-09-13T00:24:52.314 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:52 smithi181 bash[67587]: cluster 2024-09-13T00:24:51.009329+0000 mon.a (mon.0) 2461 : cluster [DBG] osdmap e1239: 8 total, 8 up, 8 in 2024-09-13T00:24:52.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:52 smithi136 bash[75164]: cluster 2024-09-13T00:24:51.009329+0000 mon.a (mon.0) 2461 : cluster [DBG] osdmap e1239: 8 total, 8 up, 8 in 2024-09-13T00:24:52.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:52 smithi136 bash[76371]: cluster 2024-09-13T00:24:51.009329+0000 mon.a (mon.0) 2461 : cluster [DBG] osdmap e1239: 8 total, 8 up, 8 in 2024-09-13T00:24:52.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:24:52 smithi181 bash[95505]: ts=2024-09-13T00:24:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:24:53.057 INFO:tasks.workunit.client.0.smithi136.stdout:write: X 8388609~1000 2024-09-13T00:24:53.076 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 16777228~1000 2024-09-13T00:24:53.076 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 167772160 2024-09-13T00:24:53.097 INFO:tasks.workunit.client.0.smithi136.stdout:write: 2 83886079~8388609 2024-09-13T00:24:53.146 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap3 2024-09-13T00:24:53.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:53 smithi181 bash[67587]: cluster 2024-09-13T00:24:51.851197+0000 mgr.y (mgr.24661) 1432 : cluster [DBG] pgmap v2232: 169 pgs: 169 active+clean; 517 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 27 MiB/s rd, 4.0 MiB/s wr, 230 op/s 2024-09-13T00:24:53.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:53 smithi181 bash[67587]: cluster 2024-09-13T00:24:52.030097+0000 mon.a (mon.0) 2462 : cluster [DBG] osdmap e1240: 8 total, 8 up, 8 in 2024-09-13T00:24:53.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:53 smithi136 bash[75164]: cluster 2024-09-13T00:24:51.851197+0000 mgr.y (mgr.24661) 1432 : cluster [DBG] pgmap v2232: 169 pgs: 169 active+clean; 517 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 27 MiB/s rd, 4.0 MiB/s wr, 230 op/s 2024-09-13T00:24:53.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:53 smithi136 bash[75164]: cluster 2024-09-13T00:24:52.030097+0000 mon.a (mon.0) 2462 : cluster [DBG] osdmap e1240: 8 total, 8 up, 8 in 2024-09-13T00:24:53.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:53 smithi136 bash[76371]: cluster 2024-09-13T00:24:51.851197+0000 mgr.y (mgr.24661) 1432 : cluster [DBG] pgmap v2232: 169 pgs: 169 active+clean; 517 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 27 MiB/s rd, 4.0 MiB/s wr, 230 op/s 2024-09-13T00:24:53.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:53 smithi136 bash[76371]: cluster 2024-09-13T00:24:52.030097+0000 mon.a (mon.0) 2462 : cluster [DBG] osdmap e1240: 8 total, 8 up, 8 in 2024-09-13T00:24:54.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:54 smithi181 bash[67587]: cluster 2024-09-13T00:24:53.031198+0000 mon.a (mon.0) 2463 : cluster [DBG] osdmap e1241: 8 total, 8 up, 8 in 2024-09-13T00:24:54.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:54 smithi181 bash[67587]: audit 2024-09-13T00:24:53.811378+0000 mon.a (mon.0) 2464 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:24:54.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:54 smithi181 bash[67587]: cluster 2024-09-13T00:24:53.953495+0000 mon.a (mon.0) 2465 : cluster [DBG] osdmap e1242: 8 total, 8 up, 8 in 2024-09-13T00:24:54.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:54 smithi136 bash[75164]: cluster 2024-09-13T00:24:53.031198+0000 mon.a (mon.0) 2463 : cluster [DBG] osdmap e1241: 8 total, 8 up, 8 in 2024-09-13T00:24:54.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:54 smithi136 bash[75164]: audit 2024-09-13T00:24:53.811378+0000 mon.a (mon.0) 2464 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:24:54.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:54 smithi136 bash[75164]: cluster 2024-09-13T00:24:53.953495+0000 mon.a (mon.0) 2465 : cluster [DBG] osdmap e1242: 8 total, 8 up, 8 in 2024-09-13T00:24:54.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:54 smithi136 bash[76371]: cluster 2024-09-13T00:24:53.031198+0000 mon.a (mon.0) 2463 : cluster [DBG] osdmap e1241: 8 total, 8 up, 8 in 2024-09-13T00:24:54.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:54 smithi136 bash[76371]: audit 2024-09-13T00:24:53.811378+0000 mon.a (mon.0) 2464 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:24:54.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:54 smithi136 bash[76371]: cluster 2024-09-13T00:24:53.953495+0000 mon.a (mon.0) 2465 : cluster [DBG] osdmap e1242: 8 total, 8 up, 8 in 2024-09-13T00:24:54.977 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:24:55.034 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 83886079~1 2024-09-13T00:24:55.056 INFO:tasks.workunit.client.0.smithi136.stdout:migration_prepare 2024-09-13T00:24:55.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:55 smithi181 bash[67587]: cluster 2024-09-13T00:24:53.853384+0000 mgr.y (mgr.24661) 1433 : cluster [DBG] pgmap v2235: 169 pgs: 169 active+clean; 533 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 18 MiB/s rd, 5.0 MiB/s wr, 172 op/s 2024-09-13T00:24:55.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:55 smithi181 bash[67587]: cluster 2024-09-13T00:24:54.955317+0000 mon.a (mon.0) 2466 : cluster [DBG] osdmap e1243: 8 total, 8 up, 8 in 2024-09-13T00:24:55.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:55 smithi136 bash[75164]: cluster 2024-09-13T00:24:53.853384+0000 mgr.y (mgr.24661) 1433 : cluster [DBG] pgmap v2235: 169 pgs: 169 active+clean; 533 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 18 MiB/s rd, 5.0 MiB/s wr, 172 op/s 2024-09-13T00:24:55.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:55 smithi136 bash[75164]: cluster 2024-09-13T00:24:54.955317+0000 mon.a (mon.0) 2466 : cluster [DBG] osdmap e1243: 8 total, 8 up, 8 in 2024-09-13T00:24:55.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:55 smithi136 bash[76371]: cluster 2024-09-13T00:24:53.853384+0000 mgr.y (mgr.24661) 1433 : cluster [DBG] pgmap v2235: 169 pgs: 169 active+clean; 533 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 18 MiB/s rd, 5.0 MiB/s wr, 172 op/s 2024-09-13T00:24:55.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:55 smithi136 bash[76371]: cluster 2024-09-13T00:24:54.955317+0000 mon.a (mon.0) 2466 : cluster [DBG] osdmap e1243: 8 total, 8 up, 8 in 2024-09-13T00:24:57.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:57 smithi181 bash[67587]: cluster 2024-09-13T00:24:55.854456+0000 mgr.y (mgr.24661) 1434 : cluster [DBG] pgmap v2238: 169 pgs: 169 active+clean; 549 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 3.0 MiB/s rd, 6.0 MiB/s wr, 35 op/s 2024-09-13T00:24:57.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:57 smithi181 bash[67587]: cluster 2024-09-13T00:24:56.063034+0000 mon.a (mon.0) 2467 : cluster [DBG] osdmap e1244: 8 total, 8 up, 8 in 2024-09-13T00:24:57.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:57 smithi136 bash[75164]: cluster 2024-09-13T00:24:55.854456+0000 mgr.y (mgr.24661) 1434 : cluster [DBG] pgmap v2238: 169 pgs: 169 active+clean; 549 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 3.0 MiB/s rd, 6.0 MiB/s wr, 35 op/s 2024-09-13T00:24:57.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:57 smithi136 bash[75164]: cluster 2024-09-13T00:24:56.063034+0000 mon.a (mon.0) 2467 : cluster [DBG] osdmap e1244: 8 total, 8 up, 8 in 2024-09-13T00:24:57.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:57 smithi136 bash[76371]: cluster 2024-09-13T00:24:55.854456+0000 mgr.y (mgr.24661) 1434 : cluster [DBG] pgmap v2238: 169 pgs: 169 active+clean; 549 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 3.0 MiB/s rd, 6.0 MiB/s wr, 35 op/s 2024-09-13T00:24:57.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:57 smithi136 bash[76371]: cluster 2024-09-13T00:24:56.063034+0000 mon.a (mon.0) 2467 : cluster [DBG] osdmap e1244: 8 total, 8 up, 8 in 2024-09-13T00:24:58.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:58 smithi181 bash[67587]: cluster 2024-09-13T00:24:57.069070+0000 mon.a (mon.0) 2468 : cluster [DBG] osdmap e1245: 8 total, 8 up, 8 in 2024-09-13T00:24:58.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:58 smithi136 bash[75164]: cluster 2024-09-13T00:24:57.069070+0000 mon.a (mon.0) 2468 : cluster [DBG] osdmap e1245: 8 total, 8 up, 8 in 2024-09-13T00:24:58.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:58 smithi136 bash[76371]: cluster 2024-09-13T00:24:57.069070+0000 mon.a (mon.0) 2468 : cluster [DBG] osdmap e1245: 8 total, 8 up, 8 in 2024-09-13T00:24:59.049 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 0 2024-09-13T00:24:59.181 INFO:tasks.workunit.client.0.smithi136.stdout:migration_execute 2024-09-13T00:24:59.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:59 smithi181 bash[67587]: cluster 2024-09-13T00:24:57.855404+0000 mgr.y (mgr.24661) 1435 : cluster [DBG] pgmap v2241: 169 pgs: 169 active+clean; 565 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 110 KiB/s rd, 7.0 MiB/s wr, 159 op/s 2024-09-13T00:24:59.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:59 smithi181 bash[67587]: cluster 2024-09-13T00:24:58.090166+0000 mon.a (mon.0) 2469 : cluster [DBG] osdmap e1246: 8 total, 8 up, 8 in 2024-09-13T00:24:59.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:24:59 smithi181 bash[67587]: cluster 2024-09-13T00:24:58.955429+0000 mon.a (mon.0) 2470 : cluster [DBG] osdmap e1247: 8 total, 8 up, 8 in 2024-09-13T00:24:59.367 INFO:tasks.workunit.client.0.smithi136.stdout:migration_commit 2024-09-13T00:24:59.412 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:59 smithi136 bash[76371]: cluster 2024-09-13T00:24:57.855404+0000 mgr.y (mgr.24661) 1435 : cluster [DBG] pgmap v2241: 169 pgs: 169 active+clean; 565 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 110 KiB/s rd, 7.0 MiB/s wr, 159 op/s 2024-09-13T00:24:59.412 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:59 smithi136 bash[76371]: cluster 2024-09-13T00:24:58.090166+0000 mon.a (mon.0) 2469 : cluster [DBG] osdmap e1246: 8 total, 8 up, 8 in 2024-09-13T00:24:59.412 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:24:59 smithi136 bash[76371]: cluster 2024-09-13T00:24:58.955429+0000 mon.a (mon.0) 2470 : cluster [DBG] osdmap e1247: 8 total, 8 up, 8 in 2024-09-13T00:24:59.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:59 smithi136 bash[75164]: cluster 2024-09-13T00:24:57.855404+0000 mgr.y (mgr.24661) 1435 : cluster [DBG] pgmap v2241: 169 pgs: 169 active+clean; 565 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 110 KiB/s rd, 7.0 MiB/s wr, 159 op/s 2024-09-13T00:24:59.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:59 smithi136 bash[75164]: cluster 2024-09-13T00:24:58.090166+0000 mon.a (mon.0) 2469 : cluster [DBG] osdmap e1246: 8 total, 8 up, 8 in 2024-09-13T00:24:59.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:24:59 smithi136 bash[75164]: cluster 2024-09-13T00:24:58.955429+0000 mon.a (mon.0) 2470 : cluster [DBG] osdmap e1247: 8 total, 8 up, 8 in 2024-09-13T00:24:59.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:24:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:24:59] "GET /metrics HTTP/1.1" 200 38989 "" "Prometheus/2.43.0" 2024-09-13T00:25:01.442 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:01 smithi181 bash[67587]: cluster 2024-09-13T00:24:59.856295+0000 mgr.y (mgr.24661) 1436 : cluster [DBG] pgmap v2244: 169 pgs: 169 active+clean; 573 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 14 MiB/s rd, 5.0 MiB/s wr, 240 op/s 2024-09-13T00:25:01.442 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:01 smithi181 bash[67587]: cluster 2024-09-13T00:25:00.098663+0000 mon.a (mon.0) 2471 : cluster [DBG] osdmap e1248: 8 total, 8 up, 8 in 2024-09-13T00:25:01.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:01 smithi136 bash[75164]: cluster 2024-09-13T00:24:59.856295+0000 mgr.y (mgr.24661) 1436 : cluster [DBG] pgmap v2244: 169 pgs: 169 active+clean; 573 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 14 MiB/s rd, 5.0 MiB/s wr, 240 op/s 2024-09-13T00:25:01.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:01 smithi136 bash[75164]: cluster 2024-09-13T00:25:00.098663+0000 mon.a (mon.0) 2471 : cluster [DBG] osdmap e1248: 8 total, 8 up, 8 in 2024-09-13T00:25:01.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:01 smithi136 bash[76371]: cluster 2024-09-13T00:24:59.856295+0000 mgr.y (mgr.24661) 1436 : cluster [DBG] pgmap v2244: 169 pgs: 169 active+clean; 573 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 14 MiB/s rd, 5.0 MiB/s wr, 240 op/s 2024-09-13T00:25:01.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:01 smithi136 bash[76371]: cluster 2024-09-13T00:25:00.098663+0000 mon.a (mon.0) 2471 : cluster [DBG] osdmap e1248: 8 total, 8 up, 8 in 2024-09-13T00:25:02.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:02 smithi136 bash[76371]: cluster 2024-09-13T00:25:01.102978+0000 mon.a (mon.0) 2472 : cluster [DBG] osdmap e1249: 8 total, 8 up, 8 in 2024-09-13T00:25:02.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:02 smithi136 bash[75164]: cluster 2024-09-13T00:25:01.102978+0000 mon.a (mon.0) 2472 : cluster [DBG] osdmap e1249: 8 total, 8 up, 8 in 2024-09-13T00:25:02.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:02 smithi181 bash[67587]: cluster 2024-09-13T00:25:01.102978+0000 mon.a (mon.0) 2472 : cluster [DBG] osdmap e1249: 8 total, 8 up, 8 in 2024-09-13T00:25:02.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:25:02 smithi181 bash[95505]: ts=2024-09-13T00:25:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:25:02.614 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.SnapsShrinkToZeroAfterPrepare (13177 ms) 2024-09-13T00:25:02.614 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.SnapsExpandAfterPrepare 2024-09-13T00:25:02.701 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:25:02.730 INFO:tasks.workunit.client.0.smithi136.stdout:write: 1 0~8388609 2024-09-13T00:25:02.846 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap1 2024-09-13T00:25:03.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:03 smithi136 bash[75164]: cluster 2024-09-13T00:25:01.857395+0000 mgr.y (mgr.24661) 1437 : cluster [DBG] pgmap v2247: 169 pgs: 169 active+clean; 585 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 29 MiB/s rd, 4.0 MiB/s wr, 316 op/s 2024-09-13T00:25:03.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:03 smithi136 bash[75164]: cluster 2024-09-13T00:25:02.136151+0000 mon.a (mon.0) 2473 : cluster [DBG] osdmap e1250: 8 total, 8 up, 8 in 2024-09-13T00:25:03.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:03 smithi136 bash[76371]: cluster 2024-09-13T00:25:01.857395+0000 mgr.y (mgr.24661) 1437 : cluster [DBG] pgmap v2247: 169 pgs: 169 active+clean; 585 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 29 MiB/s rd, 4.0 MiB/s wr, 316 op/s 2024-09-13T00:25:03.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:03 smithi136 bash[76371]: cluster 2024-09-13T00:25:02.136151+0000 mon.a (mon.0) 2473 : cluster [DBG] osdmap e1250: 8 total, 8 up, 8 in 2024-09-13T00:25:03.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:03 smithi181 bash[67587]: cluster 2024-09-13T00:25:01.857395+0000 mgr.y (mgr.24661) 1437 : cluster [DBG] pgmap v2247: 169 pgs: 169 active+clean; 585 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 29 MiB/s rd, 4.0 MiB/s wr, 316 op/s 2024-09-13T00:25:03.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:03 smithi181 bash[67587]: cluster 2024-09-13T00:25:02.136151+0000 mon.a (mon.0) 2473 : cluster [DBG] osdmap e1250: 8 total, 8 up, 8 in 2024-09-13T00:25:04.178 INFO:tasks.workunit.client.0.smithi136.stdout:write: 1 8388609~8388609 2024-09-13T00:25:04.212 INFO:tasks.workunit.client.0.smithi136.stdout:write: X 0~1000 2024-09-13T00:25:04.212 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 1010~1000 2024-09-13T00:25:04.212 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap2 2024-09-13T00:25:04.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:04 smithi136 bash[75164]: cluster 2024-09-13T00:25:03.141146+0000 mon.a (mon.0) 2474 : cluster [DBG] osdmap e1251: 8 total, 8 up, 8 in 2024-09-13T00:25:04.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:04 smithi136 bash[76371]: cluster 2024-09-13T00:25:03.141146+0000 mon.a (mon.0) 2474 : cluster [DBG] osdmap e1251: 8 total, 8 up, 8 in 2024-09-13T00:25:04.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:04 smithi181 bash[67587]: cluster 2024-09-13T00:25:03.141146+0000 mon.a (mon.0) 2474 : cluster [DBG] osdmap e1251: 8 total, 8 up, 8 in 2024-09-13T00:25:05.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:05 smithi136 bash[76371]: cluster 2024-09-13T00:25:03.858406+0000 mgr.y (mgr.24661) 1438 : cluster [DBG] pgmap v2250: 169 pgs: 169 active+clean; 577 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 33 MiB/s rd, 6.0 MiB/s wr, 379 op/s 2024-09-13T00:25:05.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:05 smithi136 bash[76371]: cluster 2024-09-13T00:25:04.151767+0000 mon.a (mon.0) 2475 : cluster [DBG] osdmap e1252: 8 total, 8 up, 8 in 2024-09-13T00:25:05.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:05 smithi136 bash[75164]: cluster 2024-09-13T00:25:03.858406+0000 mgr.y (mgr.24661) 1438 : cluster [DBG] pgmap v2250: 169 pgs: 169 active+clean; 577 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 33 MiB/s rd, 6.0 MiB/s wr, 379 op/s 2024-09-13T00:25:05.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:05 smithi136 bash[75164]: cluster 2024-09-13T00:25:04.151767+0000 mon.a (mon.0) 2475 : cluster [DBG] osdmap e1252: 8 total, 8 up, 8 in 2024-09-13T00:25:05.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:05 smithi181 bash[67587]: cluster 2024-09-13T00:25:03.858406+0000 mgr.y (mgr.24661) 1438 : cluster [DBG] pgmap v2250: 169 pgs: 169 active+clean; 577 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 33 MiB/s rd, 6.0 MiB/s wr, 379 op/s 2024-09-13T00:25:05.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:05 smithi181 bash[67587]: cluster 2024-09-13T00:25:04.151767+0000 mon.a (mon.0) 2475 : cluster [DBG] osdmap e1252: 8 total, 8 up, 8 in 2024-09-13T00:25:06.193 INFO:tasks.workunit.client.0.smithi136.stdout:write: X 8388609~1000 2024-09-13T00:25:06.214 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 16777228~1000 2024-09-13T00:25:06.214 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 167772160 2024-09-13T00:25:06.237 INFO:tasks.workunit.client.0.smithi136.stdout:write: 2 83886079~8388609 2024-09-13T00:25:06.285 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap3 2024-09-13T00:25:06.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:06 smithi136 bash[75164]: cluster 2024-09-13T00:25:05.169000+0000 mon.a (mon.0) 2476 : cluster [DBG] osdmap e1253: 8 total, 8 up, 8 in 2024-09-13T00:25:06.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:06 smithi136 bash[76371]: cluster 2024-09-13T00:25:05.169000+0000 mon.a (mon.0) 2476 : cluster [DBG] osdmap e1253: 8 total, 8 up, 8 in 2024-09-13T00:25:06.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:06 smithi181 bash[67587]: cluster 2024-09-13T00:25:05.169000+0000 mon.a (mon.0) 2476 : cluster [DBG] osdmap e1253: 8 total, 8 up, 8 in 2024-09-13T00:25:07.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:07 smithi136 bash[75164]: cluster 2024-09-13T00:25:05.859466+0000 mgr.y (mgr.24661) 1439 : cluster [DBG] pgmap v2253: 169 pgs: 169 active+clean; 577 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 26 MiB/s rd, 5.0 MiB/s wr, 256 op/s 2024-09-13T00:25:07.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:07 smithi136 bash[75164]: cluster 2024-09-13T00:25:06.174621+0000 mon.a (mon.0) 2477 : cluster [DBG] osdmap e1254: 8 total, 8 up, 8 in 2024-09-13T00:25:07.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:07 smithi136 bash[76371]: cluster 2024-09-13T00:25:05.859466+0000 mgr.y (mgr.24661) 1439 : cluster [DBG] pgmap v2253: 169 pgs: 169 active+clean; 577 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 26 MiB/s rd, 5.0 MiB/s wr, 256 op/s 2024-09-13T00:25:07.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:07 smithi136 bash[76371]: cluster 2024-09-13T00:25:06.174621+0000 mon.a (mon.0) 2477 : cluster [DBG] osdmap e1254: 8 total, 8 up, 8 in 2024-09-13T00:25:07.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:07 smithi181 bash[67587]: cluster 2024-09-13T00:25:05.859466+0000 mgr.y (mgr.24661) 1439 : cluster [DBG] pgmap v2253: 169 pgs: 169 active+clean; 577 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 26 MiB/s rd, 5.0 MiB/s wr, 256 op/s 2024-09-13T00:25:07.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:07 smithi181 bash[67587]: cluster 2024-09-13T00:25:06.174621+0000 mon.a (mon.0) 2477 : cluster [DBG] osdmap e1254: 8 total, 8 up, 8 in 2024-09-13T00:25:08.214 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:25:08.273 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 83886079~1 2024-09-13T00:25:08.294 INFO:tasks.workunit.client.0.smithi136.stdout:migration_prepare 2024-09-13T00:25:08.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:08 smithi136 bash[75164]: cluster 2024-09-13T00:25:07.185866+0000 mon.a (mon.0) 2478 : cluster [DBG] osdmap e1255: 8 total, 8 up, 8 in 2024-09-13T00:25:08.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:08 smithi136 bash[76371]: cluster 2024-09-13T00:25:07.185866+0000 mon.a (mon.0) 2478 : cluster [DBG] osdmap e1255: 8 total, 8 up, 8 in 2024-09-13T00:25:08.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:08 smithi181 bash[67587]: cluster 2024-09-13T00:25:07.185866+0000 mon.a (mon.0) 2478 : cluster [DBG] osdmap e1255: 8 total, 8 up, 8 in 2024-09-13T00:25:09.509 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:25:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:25:09] "GET /metrics HTTP/1.1" 200 38991 "" "Prometheus/2.43.0" 2024-09-13T00:25:09.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:09 smithi136 bash[76371]: cluster 2024-09-13T00:25:07.860459+0000 mgr.y (mgr.24661) 1440 : cluster [DBG] pgmap v2256: 169 pgs: 169 active+clean; 601 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 19 MiB/s rd, 8.0 MiB/s wr, 108 op/s 2024-09-13T00:25:09.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:09 smithi136 bash[76371]: cluster 2024-09-13T00:25:08.195837+0000 mon.a (mon.0) 2479 : cluster [DBG] osdmap e1256: 8 total, 8 up, 8 in 2024-09-13T00:25:09.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:09 smithi136 bash[76371]: audit 2024-09-13T00:25:08.811549+0000 mon.a (mon.0) 2480 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:25:09.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:09 smithi136 bash[76371]: cluster 2024-09-13T00:25:08.962931+0000 mon.a (mon.0) 2481 : cluster [DBG] osdmap e1257: 8 total, 8 up, 8 in 2024-09-13T00:25:09.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:09 smithi136 bash[75164]: cluster 2024-09-13T00:25:07.860459+0000 mgr.y (mgr.24661) 1440 : cluster [DBG] pgmap v2256: 169 pgs: 169 active+clean; 601 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 19 MiB/s rd, 8.0 MiB/s wr, 108 op/s 2024-09-13T00:25:09.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:09 smithi136 bash[75164]: cluster 2024-09-13T00:25:08.195837+0000 mon.a (mon.0) 2479 : cluster [DBG] osdmap e1256: 8 total, 8 up, 8 in 2024-09-13T00:25:09.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:09 smithi136 bash[75164]: audit 2024-09-13T00:25:08.811549+0000 mon.a (mon.0) 2480 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:25:09.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:09 smithi136 bash[75164]: cluster 2024-09-13T00:25:08.962931+0000 mon.a (mon.0) 2481 : cluster [DBG] osdmap e1257: 8 total, 8 up, 8 in 2024-09-13T00:25:09.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:09 smithi181 bash[67587]: cluster 2024-09-13T00:25:07.860459+0000 mgr.y (mgr.24661) 1440 : cluster [DBG] pgmap v2256: 169 pgs: 169 active+clean; 601 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 19 MiB/s rd, 8.0 MiB/s wr, 108 op/s 2024-09-13T00:25:09.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:09 smithi181 bash[67587]: cluster 2024-09-13T00:25:08.195837+0000 mon.a (mon.0) 2479 : cluster [DBG] osdmap e1256: 8 total, 8 up, 8 in 2024-09-13T00:25:09.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:09 smithi181 bash[67587]: audit 2024-09-13T00:25:08.811549+0000 mon.a (mon.0) 2480 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:25:09.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:09 smithi181 bash[67587]: cluster 2024-09-13T00:25:08.962931+0000 mon.a (mon.0) 2481 : cluster [DBG] osdmap e1257: 8 total, 8 up, 8 in 2024-09-13T00:25:11.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:10 smithi136 bash[75164]: cluster 2024-09-13T00:25:09.861473+0000 mgr.y (mgr.24661) 1441 : cluster [DBG] pgmap v2259: 169 pgs: 169 active+clean; 617 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 11 MiB/s rd, 8.0 MiB/s wr, 159 op/s 2024-09-13T00:25:11.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:10 smithi136 bash[75164]: cluster 2024-09-13T00:25:09.977742+0000 mon.a (mon.0) 2482 : cluster [DBG] osdmap e1258: 8 total, 8 up, 8 in 2024-09-13T00:25:11.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:10 smithi136 bash[76371]: cluster 2024-09-13T00:25:09.861473+0000 mgr.y (mgr.24661) 1441 : cluster [DBG] pgmap v2259: 169 pgs: 169 active+clean; 617 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 11 MiB/s rd, 8.0 MiB/s wr, 159 op/s 2024-09-13T00:25:11.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:10 smithi136 bash[76371]: cluster 2024-09-13T00:25:09.977742+0000 mon.a (mon.0) 2482 : cluster [DBG] osdmap e1258: 8 total, 8 up, 8 in 2024-09-13T00:25:11.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:10 smithi181 bash[67587]: cluster 2024-09-13T00:25:09.861473+0000 mgr.y (mgr.24661) 1441 : cluster [DBG] pgmap v2259: 169 pgs: 169 active+clean; 617 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 11 MiB/s rd, 8.0 MiB/s wr, 159 op/s 2024-09-13T00:25:11.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:10 smithi181 bash[67587]: cluster 2024-09-13T00:25:09.977742+0000 mon.a (mon.0) 2482 : cluster [DBG] osdmap e1258: 8 total, 8 up, 8 in 2024-09-13T00:25:11.888 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 167772160 2024-09-13T00:25:11.917 INFO:tasks.workunit.client.0.smithi136.stdout:write: * 83886080~1000 2024-09-13T00:25:11.980 INFO:tasks.workunit.client.0.smithi136.stdout:migration_execute 2024-09-13T00:25:12.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:11 smithi136 bash[75164]: cluster 2024-09-13T00:25:10.980866+0000 mon.a (mon.0) 2483 : cluster [DBG] osdmap e1259: 8 total, 8 up, 8 in 2024-09-13T00:25:12.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:11 smithi136 bash[76371]: cluster 2024-09-13T00:25:10.980866+0000 mon.a (mon.0) 2483 : cluster [DBG] osdmap e1259: 8 total, 8 up, 8 in 2024-09-13T00:25:12.270 INFO:tasks.workunit.client.0.smithi136.stdout:migration_commit 2024-09-13T00:25:12.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:11 smithi181 bash[67587]: cluster 2024-09-13T00:25:10.980866+0000 mon.a (mon.0) 2483 : cluster [DBG] osdmap e1259: 8 total, 8 up, 8 in 2024-09-13T00:25:12.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:25:12 smithi181 bash[95505]: ts=2024-09-13T00:25:12.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:25:13.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:12 smithi136 bash[75164]: cluster 2024-09-13T00:25:11.862805+0000 mgr.y (mgr.24661) 1442 : cluster [DBG] pgmap v2262: 169 pgs: 169 active+clean; 621 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 12 MiB/s rd, 3.0 MiB/s wr, 181 op/s 2024-09-13T00:25:13.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:12 smithi136 bash[76371]: cluster 2024-09-13T00:25:11.862805+0000 mgr.y (mgr.24661) 1442 : cluster [DBG] pgmap v2262: 169 pgs: 169 active+clean; 621 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 12 MiB/s rd, 3.0 MiB/s wr, 181 op/s 2024-09-13T00:25:13.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:12 smithi181 bash[67587]: cluster 2024-09-13T00:25:11.862805+0000 mgr.y (mgr.24661) 1442 : cluster [DBG] pgmap v2262: 169 pgs: 169 active+clean; 621 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 12 MiB/s rd, 3.0 MiB/s wr, 181 op/s 2024-09-13T00:25:14.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:14 smithi181 bash[67587]: cluster 2024-09-13T00:25:13.008978+0000 mon.a (mon.0) 2484 : cluster [DBG] osdmap e1260: 8 total, 8 up, 8 in 2024-09-13T00:25:14.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:14 smithi136 bash[75164]: cluster 2024-09-13T00:25:13.008978+0000 mon.a (mon.0) 2484 : cluster [DBG] osdmap e1260: 8 total, 8 up, 8 in 2024-09-13T00:25:14.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:14 smithi136 bash[76371]: cluster 2024-09-13T00:25:13.008978+0000 mon.a (mon.0) 2484 : cluster [DBG] osdmap e1260: 8 total, 8 up, 8 in 2024-09-13T00:25:15.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:15 smithi181 bash[67587]: cluster 2024-09-13T00:25:13.863835+0000 mgr.y (mgr.24661) 1443 : cluster [DBG] pgmap v2264: 169 pgs: 169 active+clean; 629 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 18 MiB/s rd, 4.1 MiB/s wr, 236 op/s 2024-09-13T00:25:15.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:15 smithi181 bash[67587]: cluster 2024-09-13T00:25:14.023701+0000 mon.a (mon.0) 2485 : cluster [DBG] osdmap e1261: 8 total, 8 up, 8 in 2024-09-13T00:25:15.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:15 smithi136 bash[75164]: cluster 2024-09-13T00:25:13.863835+0000 mgr.y (mgr.24661) 1443 : cluster [DBG] pgmap v2264: 169 pgs: 169 active+clean; 629 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 18 MiB/s rd, 4.1 MiB/s wr, 236 op/s 2024-09-13T00:25:15.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:15 smithi136 bash[75164]: cluster 2024-09-13T00:25:14.023701+0000 mon.a (mon.0) 2485 : cluster [DBG] osdmap e1261: 8 total, 8 up, 8 in 2024-09-13T00:25:15.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:15 smithi136 bash[76371]: cluster 2024-09-13T00:25:13.863835+0000 mgr.y (mgr.24661) 1443 : cluster [DBG] pgmap v2264: 169 pgs: 169 active+clean; 629 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 18 MiB/s rd, 4.1 MiB/s wr, 236 op/s 2024-09-13T00:25:15.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:15 smithi136 bash[76371]: cluster 2024-09-13T00:25:14.023701+0000 mon.a (mon.0) 2485 : cluster [DBG] osdmap e1261: 8 total, 8 up, 8 in 2024-09-13T00:25:15.711 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.SnapsExpandAfterPrepare (13097 ms) 2024-09-13T00:25:15.711 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.SnapsExpandAfterPrepare2 2024-09-13T00:25:15.795 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:25:15.824 INFO:tasks.workunit.client.0.smithi136.stdout:write: X 41943040~10 2024-09-13T00:25:15.846 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap1 2024-09-13T00:25:16.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:16 smithi181 bash[67587]: cluster 2024-09-13T00:25:15.030174+0000 mon.a (mon.0) 2486 : cluster [DBG] osdmap e1262: 8 total, 8 up, 8 in 2024-09-13T00:25:16.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:16 smithi136 bash[75164]: cluster 2024-09-13T00:25:15.030174+0000 mon.a (mon.0) 2486 : cluster [DBG] osdmap e1262: 8 total, 8 up, 8 in 2024-09-13T00:25:16.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:16 smithi136 bash[76371]: cluster 2024-09-13T00:25:15.030174+0000 mon.a (mon.0) 2486 : cluster [DBG] osdmap e1262: 8 total, 8 up, 8 in 2024-09-13T00:25:17.077 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 41943040 2024-09-13T00:25:17.133 INFO:tasks.workunit.client.0.smithi136.stdout:migration_prepare 2024-09-13T00:25:17.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:17 smithi181 bash[67587]: cluster 2024-09-13T00:25:15.865077+0000 mgr.y (mgr.24661) 1444 : cluster [DBG] pgmap v2267: 169 pgs: 169 active+clean; 637 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 27 MiB/s rd, 4.1 MiB/s wr, 334 op/s 2024-09-13T00:25:17.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:17 smithi181 bash[67587]: cluster 2024-09-13T00:25:16.045133+0000 mon.a (mon.0) 2487 : cluster [DBG] osdmap e1263: 8 total, 8 up, 8 in 2024-09-13T00:25:17.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:17 smithi136 bash[75164]: cluster 2024-09-13T00:25:15.865077+0000 mgr.y (mgr.24661) 1444 : cluster [DBG] pgmap v2267: 169 pgs: 169 active+clean; 637 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 27 MiB/s rd, 4.1 MiB/s wr, 334 op/s 2024-09-13T00:25:17.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:17 smithi136 bash[75164]: cluster 2024-09-13T00:25:16.045133+0000 mon.a (mon.0) 2487 : cluster [DBG] osdmap e1263: 8 total, 8 up, 8 in 2024-09-13T00:25:17.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:17 smithi136 bash[76371]: cluster 2024-09-13T00:25:15.865077+0000 mgr.y (mgr.24661) 1444 : cluster [DBG] pgmap v2267: 169 pgs: 169 active+clean; 637 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 27 MiB/s rd, 4.1 MiB/s wr, 334 op/s 2024-09-13T00:25:17.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:17 smithi136 bash[76371]: cluster 2024-09-13T00:25:16.045133+0000 mon.a (mon.0) 2487 : cluster [DBG] osdmap e1263: 8 total, 8 up, 8 in 2024-09-13T00:25:18.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:18 smithi181 bash[67587]: cluster 2024-09-13T00:25:17.052718+0000 mon.a (mon.0) 2488 : cluster [DBG] osdmap e1264: 8 total, 8 up, 8 in 2024-09-13T00:25:18.463 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:25:18.492 INFO:tasks.workunit.client.0.smithi136.stdout:write: Y 41943040~5 2024-09-13T00:25:18.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:18 smithi136 bash[75164]: cluster 2024-09-13T00:25:17.052718+0000 mon.a (mon.0) 2488 : cluster [DBG] osdmap e1264: 8 total, 8 up, 8 in 2024-09-13T00:25:18.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:18 smithi136 bash[76371]: cluster 2024-09-13T00:25:17.052718+0000 mon.a (mon.0) 2488 : cluster [DBG] osdmap e1264: 8 total, 8 up, 8 in 2024-09-13T00:25:18.731 INFO:tasks.workunit.client.0.smithi136.stdout:migration_execute 2024-09-13T00:25:18.903 INFO:tasks.workunit.client.0.smithi136.stdout:migration_commit 2024-09-13T00:25:19.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:19 smithi181 bash[67587]: cluster 2024-09-13T00:25:17.866027+0000 mgr.y (mgr.24661) 1445 : cluster [DBG] pgmap v2270: 169 pgs: 169 active+clean; 633 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 37 MiB/s rd, 3.0 MiB/s wr, 270 op/s 2024-09-13T00:25:19.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:19 smithi181 bash[67587]: cluster 2024-09-13T00:25:18.064162+0000 mon.a (mon.0) 2489 : cluster [DBG] osdmap e1265: 8 total, 8 up, 8 in 2024-09-13T00:25:19.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:19 smithi181 bash[67587]: audit 2024-09-13T00:25:18.554158+0000 mon.a (mon.0) 2490 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:25:19.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:19 smithi181 bash[67587]: cluster 2024-09-13T00:25:18.962150+0000 mon.a (mon.0) 2491 : cluster [DBG] osdmap e1266: 8 total, 8 up, 8 in 2024-09-13T00:25:19.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:19 smithi136 bash[76371]: cluster 2024-09-13T00:25:17.866027+0000 mgr.y (mgr.24661) 1445 : cluster [DBG] pgmap v2270: 169 pgs: 169 active+clean; 633 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 37 MiB/s rd, 3.0 MiB/s wr, 270 op/s 2024-09-13T00:25:19.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:19 smithi136 bash[76371]: cluster 2024-09-13T00:25:18.064162+0000 mon.a (mon.0) 2489 : cluster [DBG] osdmap e1265: 8 total, 8 up, 8 in 2024-09-13T00:25:19.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:19 smithi136 bash[76371]: audit 2024-09-13T00:25:18.554158+0000 mon.a (mon.0) 2490 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:25:19.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:19 smithi136 bash[76371]: cluster 2024-09-13T00:25:18.962150+0000 mon.a (mon.0) 2491 : cluster [DBG] osdmap e1266: 8 total, 8 up, 8 in 2024-09-13T00:25:19.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:19 smithi136 bash[75164]: cluster 2024-09-13T00:25:17.866027+0000 mgr.y (mgr.24661) 1445 : cluster [DBG] pgmap v2270: 169 pgs: 169 active+clean; 633 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 37 MiB/s rd, 3.0 MiB/s wr, 270 op/s 2024-09-13T00:25:19.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:19 smithi136 bash[75164]: cluster 2024-09-13T00:25:18.064162+0000 mon.a (mon.0) 2489 : cluster [DBG] osdmap e1265: 8 total, 8 up, 8 in 2024-09-13T00:25:19.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:19 smithi136 bash[75164]: audit 2024-09-13T00:25:18.554158+0000 mon.a (mon.0) 2490 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:25:19.414 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:19 smithi136 bash[75164]: cluster 2024-09-13T00:25:18.962150+0000 mon.a (mon.0) 2491 : cluster [DBG] osdmap e1266: 8 total, 8 up, 8 in 2024-09-13T00:25:19.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:25:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:25:19] "GET /metrics HTTP/1.1" 200 38991 "" "Prometheus/2.43.0" 2024-09-13T00:25:20.207 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.SnapsExpandAfterPrepare2 (4496 ms) 2024-09-13T00:25:20.207 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.SnapsSnapAfterPrepare 2024-09-13T00:25:20.293 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:25:20.322 INFO:tasks.workunit.client.0.smithi136.stdout:write: 1 0~8388609 2024-09-13T00:25:20.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:20 smithi181 bash[67587]: audit 2024-09-13T00:25:19.127357+0000 mon.a (mon.0) 2492 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:25:20.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:20 smithi181 bash[67587]: audit 2024-09-13T00:25:19.129204+0000 mon.a (mon.0) 2493 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:25:20.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:20 smithi181 bash[67587]: audit 2024-09-13T00:25:19.138979+0000 mon.a (mon.0) 2494 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:25:20.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:20 smithi181 bash[67587]: cluster 2024-09-13T00:25:19.970341+0000 mon.a (mon.0) 2495 : cluster [DBG] osdmap e1267: 8 total, 8 up, 8 in 2024-09-13T00:25:20.438 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap1 2024-09-13T00:25:20.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:20 smithi136 bash[75164]: audit 2024-09-13T00:25:19.127357+0000 mon.a (mon.0) 2492 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:25:20.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:20 smithi136 bash[75164]: audit 2024-09-13T00:25:19.129204+0000 mon.a (mon.0) 2493 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:25:20.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:20 smithi136 bash[75164]: audit 2024-09-13T00:25:19.138979+0000 mon.a (mon.0) 2494 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:25:20.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:20 smithi136 bash[75164]: cluster 2024-09-13T00:25:19.970341+0000 mon.a (mon.0) 2495 : cluster [DBG] osdmap e1267: 8 total, 8 up, 8 in 2024-09-13T00:25:20.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:20 smithi136 bash[76371]: audit 2024-09-13T00:25:19.127357+0000 mon.a (mon.0) 2492 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:25:20.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:20 smithi136 bash[76371]: audit 2024-09-13T00:25:19.129204+0000 mon.a (mon.0) 2493 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:25:20.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:20 smithi136 bash[76371]: audit 2024-09-13T00:25:19.138979+0000 mon.a (mon.0) 2494 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:25:20.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:20 smithi136 bash[76371]: cluster 2024-09-13T00:25:19.970341+0000 mon.a (mon.0) 2495 : cluster [DBG] osdmap e1267: 8 total, 8 up, 8 in 2024-09-13T00:25:21.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:21 smithi136 bash[75164]: cluster 2024-09-13T00:25:19.866965+0000 mgr.y (mgr.24661) 1446 : cluster [DBG] pgmap v2273: 169 pgs: 169 active+clean; 621 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 36 MiB/s rd, 2.0 MiB/s wr, 242 op/s 2024-09-13T00:25:21.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:21 smithi136 bash[76371]: cluster 2024-09-13T00:25:19.866965+0000 mgr.y (mgr.24661) 1446 : cluster [DBG] pgmap v2273: 169 pgs: 169 active+clean; 621 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 36 MiB/s rd, 2.0 MiB/s wr, 242 op/s 2024-09-13T00:25:21.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:21 smithi181 bash[67587]: cluster 2024-09-13T00:25:19.866965+0000 mgr.y (mgr.24661) 1446 : cluster [DBG] pgmap v2273: 169 pgs: 169 active+clean; 621 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 36 MiB/s rd, 2.0 MiB/s wr, 242 op/s 2024-09-13T00:25:22.193 INFO:tasks.workunit.client.0.smithi136.stdout:write: 1 8388609~8388609 2024-09-13T00:25:22.228 INFO:tasks.workunit.client.0.smithi136.stdout:write: X 0~1000 2024-09-13T00:25:22.229 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 1010~1000 2024-09-13T00:25:22.229 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap2 2024-09-13T00:25:22.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:22 smithi136 bash[75164]: cluster 2024-09-13T00:25:21.148330+0000 mon.a (mon.0) 2496 : cluster [DBG] osdmap e1268: 8 total, 8 up, 8 in 2024-09-13T00:25:22.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:22 smithi136 bash[76371]: cluster 2024-09-13T00:25:21.148330+0000 mon.a (mon.0) 2496 : cluster [DBG] osdmap e1268: 8 total, 8 up, 8 in 2024-09-13T00:25:22.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:22 smithi181 bash[67587]: cluster 2024-09-13T00:25:21.148330+0000 mon.a (mon.0) 2496 : cluster [DBG] osdmap e1268: 8 total, 8 up, 8 in 2024-09-13T00:25:22.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:25:22 smithi181 bash[95505]: ts=2024-09-13T00:25:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:25:23.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:23 smithi136 bash[75164]: cluster 2024-09-13T00:25:21.868053+0000 mgr.y (mgr.24661) 1447 : cluster [DBG] pgmap v2276: 169 pgs: 169 active+clean; 633 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 11 MiB/s rd, 3.0 MiB/s wr, 424 op/s 2024-09-13T00:25:23.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:23 smithi136 bash[75164]: cluster 2024-09-13T00:25:22.168365+0000 mon.a (mon.0) 2497 : cluster [DBG] osdmap e1269: 8 total, 8 up, 8 in 2024-09-13T00:25:23.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:23 smithi136 bash[76371]: cluster 2024-09-13T00:25:21.868053+0000 mgr.y (mgr.24661) 1447 : cluster [DBG] pgmap v2276: 169 pgs: 169 active+clean; 633 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 11 MiB/s rd, 3.0 MiB/s wr, 424 op/s 2024-09-13T00:25:23.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:23 smithi136 bash[76371]: cluster 2024-09-13T00:25:22.168365+0000 mon.a (mon.0) 2497 : cluster [DBG] osdmap e1269: 8 total, 8 up, 8 in 2024-09-13T00:25:23.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:23 smithi181 bash[67587]: cluster 2024-09-13T00:25:21.868053+0000 mgr.y (mgr.24661) 1447 : cluster [DBG] pgmap v2276: 169 pgs: 169 active+clean; 633 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 11 MiB/s rd, 3.0 MiB/s wr, 424 op/s 2024-09-13T00:25:23.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:23 smithi181 bash[67587]: cluster 2024-09-13T00:25:22.168365+0000 mon.a (mon.0) 2497 : cluster [DBG] osdmap e1269: 8 total, 8 up, 8 in 2024-09-13T00:25:24.005 INFO:tasks.workunit.client.0.smithi136.stdout:write: X 8388609~1000 2024-09-13T00:25:24.026 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 16777228~1000 2024-09-13T00:25:24.026 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 167772160 2024-09-13T00:25:24.046 INFO:tasks.workunit.client.0.smithi136.stdout:write: 2 83886079~8388609 2024-09-13T00:25:24.113 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap3 2024-09-13T00:25:24.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:24 smithi136 bash[75164]: cluster 2024-09-13T00:25:23.182514+0000 mon.a (mon.0) 2498 : cluster [DBG] osdmap e1270: 8 total, 8 up, 8 in 2024-09-13T00:25:24.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:24 smithi136 bash[75164]: audit 2024-09-13T00:25:23.811891+0000 mon.a (mon.0) 2499 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:25:24.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:24 smithi136 bash[75164]: cluster 2024-09-13T00:25:23.973262+0000 mon.a (mon.0) 2500 : cluster [DBG] osdmap e1271: 8 total, 8 up, 8 in 2024-09-13T00:25:24.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:24 smithi136 bash[76371]: cluster 2024-09-13T00:25:23.182514+0000 mon.a (mon.0) 2498 : cluster [DBG] osdmap e1270: 8 total, 8 up, 8 in 2024-09-13T00:25:24.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:24 smithi136 bash[76371]: audit 2024-09-13T00:25:23.811891+0000 mon.a (mon.0) 2499 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:25:24.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:24 smithi136 bash[76371]: cluster 2024-09-13T00:25:23.973262+0000 mon.a (mon.0) 2500 : cluster [DBG] osdmap e1271: 8 total, 8 up, 8 in 2024-09-13T00:25:24.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:24 smithi181 bash[67587]: cluster 2024-09-13T00:25:23.182514+0000 mon.a (mon.0) 2498 : cluster [DBG] osdmap e1270: 8 total, 8 up, 8 in 2024-09-13T00:25:24.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:24 smithi181 bash[67587]: audit 2024-09-13T00:25:23.811891+0000 mon.a (mon.0) 2499 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:25:24.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:24 smithi181 bash[67587]: cluster 2024-09-13T00:25:23.973262+0000 mon.a (mon.0) 2500 : cluster [DBG] osdmap e1271: 8 total, 8 up, 8 in 2024-09-13T00:25:25.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:25 smithi136 bash[75164]: cluster 2024-09-13T00:25:23.869132+0000 mgr.y (mgr.24661) 1448 : cluster [DBG] pgmap v2279: 169 pgs: 169 active+clean; 637 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 1.2 MiB/s rd, 4.0 MiB/s wr, 257 op/s 2024-09-13T00:25:25.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:25 smithi136 bash[75164]: cluster 2024-09-13T00:25:24.970539+0000 mon.a (mon.0) 2501 : cluster [DBG] osdmap e1272: 8 total, 8 up, 8 in 2024-09-13T00:25:25.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:25 smithi136 bash[76371]: cluster 2024-09-13T00:25:23.869132+0000 mgr.y (mgr.24661) 1448 : cluster [DBG] pgmap v2279: 169 pgs: 169 active+clean; 637 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 1.2 MiB/s rd, 4.0 MiB/s wr, 257 op/s 2024-09-13T00:25:25.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:25 smithi136 bash[76371]: cluster 2024-09-13T00:25:24.970539+0000 mon.a (mon.0) 2501 : cluster [DBG] osdmap e1272: 8 total, 8 up, 8 in 2024-09-13T00:25:25.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:25 smithi181 bash[67587]: cluster 2024-09-13T00:25:23.869132+0000 mgr.y (mgr.24661) 1448 : cluster [DBG] pgmap v2279: 169 pgs: 169 active+clean; 637 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 1.2 MiB/s rd, 4.0 MiB/s wr, 257 op/s 2024-09-13T00:25:25.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:25 smithi181 bash[67587]: cluster 2024-09-13T00:25:24.970539+0000 mon.a (mon.0) 2501 : cluster [DBG] osdmap e1272: 8 total, 8 up, 8 in 2024-09-13T00:25:25.999 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:25:26.061 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 83886079~1 2024-09-13T00:25:26.084 INFO:tasks.workunit.client.0.smithi136.stdout:migration_prepare 2024-09-13T00:25:27.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:26 smithi136 bash[75164]: cluster 2024-09-13T00:25:25.870261+0000 mgr.y (mgr.24661) 1449 : cluster [DBG] pgmap v2282: 169 pgs: 169 active+clean; 645 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 18 KiB/s rd, 3.0 MiB/s wr, 31 op/s 2024-09-13T00:25:27.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:26 smithi136 bash[75164]: cluster 2024-09-13T00:25:25.977620+0000 mon.a (mon.0) 2502 : cluster [DBG] osdmap e1273: 8 total, 8 up, 8 in 2024-09-13T00:25:27.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:26 smithi136 bash[76371]: cluster 2024-09-13T00:25:25.870261+0000 mgr.y (mgr.24661) 1449 : cluster [DBG] pgmap v2282: 169 pgs: 169 active+clean; 645 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 18 KiB/s rd, 3.0 MiB/s wr, 31 op/s 2024-09-13T00:25:27.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:26 smithi136 bash[76371]: cluster 2024-09-13T00:25:25.977620+0000 mon.a (mon.0) 2502 : cluster [DBG] osdmap e1273: 8 total, 8 up, 8 in 2024-09-13T00:25:27.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:26 smithi181 bash[67587]: cluster 2024-09-13T00:25:25.870261+0000 mgr.y (mgr.24661) 1449 : cluster [DBG] pgmap v2282: 169 pgs: 169 active+clean; 645 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 18 KiB/s rd, 3.0 MiB/s wr, 31 op/s 2024-09-13T00:25:27.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:26 smithi181 bash[67587]: cluster 2024-09-13T00:25:25.977620+0000 mon.a (mon.0) 2502 : cluster [DBG] osdmap e1273: 8 total, 8 up, 8 in 2024-09-13T00:25:28.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:28 smithi136 bash[75164]: cluster 2024-09-13T00:25:27.196248+0000 mon.a (mon.0) 2503 : cluster [DBG] osdmap e1274: 8 total, 8 up, 8 in 2024-09-13T00:25:28.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:28 smithi136 bash[76371]: cluster 2024-09-13T00:25:27.196248+0000 mon.a (mon.0) 2503 : cluster [DBG] osdmap e1274: 8 total, 8 up, 8 in 2024-09-13T00:25:28.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:28 smithi181 bash[67587]: cluster 2024-09-13T00:25:27.196248+0000 mon.a (mon.0) 2503 : cluster [DBG] osdmap e1274: 8 total, 8 up, 8 in 2024-09-13T00:25:29.509 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:25:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:25:29] "GET /metrics HTTP/1.1" 200 38993 "" "Prometheus/2.43.0" 2024-09-13T00:25:29.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:29 smithi136 bash[75164]: cluster 2024-09-13T00:25:27.871202+0000 mgr.y (mgr.24661) 1450 : cluster [DBG] pgmap v2285: 169 pgs: 169 active+clean; 677 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 95 KiB/s rd, 8.0 MiB/s wr, 139 op/s 2024-09-13T00:25:29.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:29 smithi136 bash[75164]: cluster 2024-09-13T00:25:28.205017+0000 mon.a (mon.0) 2504 : cluster [DBG] osdmap e1275: 8 total, 8 up, 8 in 2024-09-13T00:25:29.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:29 smithi136 bash[75164]: cluster 2024-09-13T00:25:28.971355+0000 mon.a (mon.0) 2505 : cluster [DBG] osdmap e1276: 8 total, 8 up, 8 in 2024-09-13T00:25:29.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:29 smithi136 bash[76371]: cluster 2024-09-13T00:25:27.871202+0000 mgr.y (mgr.24661) 1450 : cluster [DBG] pgmap v2285: 169 pgs: 169 active+clean; 677 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 95 KiB/s rd, 8.0 MiB/s wr, 139 op/s 2024-09-13T00:25:29.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:29 smithi136 bash[76371]: cluster 2024-09-13T00:25:28.205017+0000 mon.a (mon.0) 2504 : cluster [DBG] osdmap e1275: 8 total, 8 up, 8 in 2024-09-13T00:25:29.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:29 smithi136 bash[76371]: cluster 2024-09-13T00:25:28.971355+0000 mon.a (mon.0) 2505 : cluster [DBG] osdmap e1276: 8 total, 8 up, 8 in 2024-09-13T00:25:29.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:29 smithi181 bash[67587]: cluster 2024-09-13T00:25:27.871202+0000 mgr.y (mgr.24661) 1450 : cluster [DBG] pgmap v2285: 169 pgs: 169 active+clean; 677 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 95 KiB/s rd, 8.0 MiB/s wr, 139 op/s 2024-09-13T00:25:29.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:29 smithi181 bash[67587]: cluster 2024-09-13T00:25:28.205017+0000 mon.a (mon.0) 2504 : cluster [DBG] osdmap e1275: 8 total, 8 up, 8 in 2024-09-13T00:25:29.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:29 smithi181 bash[67587]: cluster 2024-09-13T00:25:28.971355+0000 mon.a (mon.0) 2505 : cluster [DBG] osdmap e1276: 8 total, 8 up, 8 in 2024-09-13T00:25:30.173 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: after_prepare_snap 2024-09-13T00:25:31.244 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 41943040 2024-09-13T00:25:31.325 INFO:tasks.workunit.client.0.smithi136.stdout:write: * 0~1000 2024-09-13T00:25:31.344 INFO:tasks.workunit.client.0.smithi136.stdout:migration_execute 2024-09-13T00:25:31.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:31 smithi136 bash[75164]: cluster 2024-09-13T00:25:29.872101+0000 mgr.y (mgr.24661) 1451 : cluster [DBG] pgmap v2288: 169 pgs: 169 active+clean; 677 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 5.1 MiB/s rd, 6.0 MiB/s wr, 142 op/s 2024-09-13T00:25:31.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:31 smithi136 bash[75164]: cluster 2024-09-13T00:25:30.213280+0000 mon.a (mon.0) 2506 : cluster [DBG] osdmap e1277: 8 total, 8 up, 8 in 2024-09-13T00:25:31.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:31 smithi136 bash[76371]: cluster 2024-09-13T00:25:29.872101+0000 mgr.y (mgr.24661) 1451 : cluster [DBG] pgmap v2288: 169 pgs: 169 active+clean; 677 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 5.1 MiB/s rd, 6.0 MiB/s wr, 142 op/s 2024-09-13T00:25:31.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:31 smithi136 bash[76371]: cluster 2024-09-13T00:25:30.213280+0000 mon.a (mon.0) 2506 : cluster [DBG] osdmap e1277: 8 total, 8 up, 8 in 2024-09-13T00:25:31.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:31 smithi181 bash[67587]: cluster 2024-09-13T00:25:29.872101+0000 mgr.y (mgr.24661) 1451 : cluster [DBG] pgmap v2288: 169 pgs: 169 active+clean; 677 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 5.1 MiB/s rd, 6.0 MiB/s wr, 142 op/s 2024-09-13T00:25:31.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:31 smithi181 bash[67587]: cluster 2024-09-13T00:25:30.213280+0000 mon.a (mon.0) 2506 : cluster [DBG] osdmap e1277: 8 total, 8 up, 8 in 2024-09-13T00:25:31.610 INFO:tasks.workunit.client.0.smithi136.stdout:migration_commit 2024-09-13T00:25:32.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:32 smithi136 bash[75164]: cluster 2024-09-13T00:25:31.216336+0000 mon.a (mon.0) 2507 : cluster [DBG] osdmap e1278: 8 total, 8 up, 8 in 2024-09-13T00:25:32.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:32 smithi136 bash[76371]: cluster 2024-09-13T00:25:31.216336+0000 mon.a (mon.0) 2507 : cluster [DBG] osdmap e1278: 8 total, 8 up, 8 in 2024-09-13T00:25:32.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:32 smithi181 bash[67587]: cluster 2024-09-13T00:25:31.216336+0000 mon.a (mon.0) 2507 : cluster [DBG] osdmap e1278: 8 total, 8 up, 8 in 2024-09-13T00:25:32.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:25:32 smithi181 bash[95505]: ts=2024-09-13T00:25:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:25:33.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:33 smithi136 bash[76371]: cluster 2024-09-13T00:25:31.873442+0000 mgr.y (mgr.24661) 1452 : cluster [DBG] pgmap v2291: 169 pgs: 169 active+clean; 685 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 37 MiB/s rd, 2.0 MiB/s wr, 326 op/s 2024-09-13T00:25:33.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:33 smithi136 bash[76371]: cluster 2024-09-13T00:25:32.239583+0000 mon.a (mon.0) 2508 : cluster [DBG] osdmap e1279: 8 total, 8 up, 8 in 2024-09-13T00:25:33.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:33 smithi136 bash[75164]: cluster 2024-09-13T00:25:31.873442+0000 mgr.y (mgr.24661) 1452 : cluster [DBG] pgmap v2291: 169 pgs: 169 active+clean; 685 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 37 MiB/s rd, 2.0 MiB/s wr, 326 op/s 2024-09-13T00:25:33.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:33 smithi136 bash[75164]: cluster 2024-09-13T00:25:32.239583+0000 mon.a (mon.0) 2508 : cluster [DBG] osdmap e1279: 8 total, 8 up, 8 in 2024-09-13T00:25:33.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:33 smithi181 bash[67587]: cluster 2024-09-13T00:25:31.873442+0000 mgr.y (mgr.24661) 1452 : cluster [DBG] pgmap v2291: 169 pgs: 169 active+clean; 685 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 37 MiB/s rd, 2.0 MiB/s wr, 326 op/s 2024-09-13T00:25:33.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:33 smithi181 bash[67587]: cluster 2024-09-13T00:25:32.239583+0000 mon.a (mon.0) 2508 : cluster [DBG] osdmap e1279: 8 total, 8 up, 8 in 2024-09-13T00:25:34.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:34 smithi136 bash[76371]: cluster 2024-09-13T00:25:33.239669+0000 mon.a (mon.0) 2509 : cluster [DBG] osdmap e1280: 8 total, 8 up, 8 in 2024-09-13T00:25:34.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:34 smithi136 bash[75164]: cluster 2024-09-13T00:25:33.239669+0000 mon.a (mon.0) 2509 : cluster [DBG] osdmap e1280: 8 total, 8 up, 8 in 2024-09-13T00:25:34.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:34 smithi181 bash[67587]: cluster 2024-09-13T00:25:33.239669+0000 mon.a (mon.0) 2509 : cluster [DBG] osdmap e1280: 8 total, 8 up, 8 in 2024-09-13T00:25:34.878 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.SnapsSnapAfterPrepare (14671 ms) 2024-09-13T00:25:34.879 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.SnapsSnapExpandAfterPrepare 2024-09-13T00:25:34.959 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:25:34.988 INFO:tasks.workunit.client.0.smithi136.stdout:write: 1 0~8388609 2024-09-13T00:25:35.098 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap1 2024-09-13T00:25:35.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:35 smithi181 bash[67587]: cluster 2024-09-13T00:25:33.874594+0000 mgr.y (mgr.24661) 1453 : cluster [DBG] pgmap v2294: 169 pgs: 169 active+clean; 701 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 39 MiB/s rd, 5.0 MiB/s wr, 331 op/s 2024-09-13T00:25:35.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:35 smithi181 bash[67587]: cluster 2024-09-13T00:25:34.267396+0000 mon.a (mon.0) 2510 : cluster [DBG] osdmap e1281: 8 total, 8 up, 8 in 2024-09-13T00:25:35.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:35 smithi136 bash[75164]: cluster 2024-09-13T00:25:33.874594+0000 mgr.y (mgr.24661) 1453 : cluster [DBG] pgmap v2294: 169 pgs: 169 active+clean; 701 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 39 MiB/s rd, 5.0 MiB/s wr, 331 op/s 2024-09-13T00:25:35.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:35 smithi136 bash[75164]: cluster 2024-09-13T00:25:34.267396+0000 mon.a (mon.0) 2510 : cluster [DBG] osdmap e1281: 8 total, 8 up, 8 in 2024-09-13T00:25:35.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:35 smithi136 bash[76371]: cluster 2024-09-13T00:25:33.874594+0000 mgr.y (mgr.24661) 1453 : cluster [DBG] pgmap v2294: 169 pgs: 169 active+clean; 701 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 39 MiB/s rd, 5.0 MiB/s wr, 331 op/s 2024-09-13T00:25:35.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:35 smithi136 bash[76371]: cluster 2024-09-13T00:25:34.267396+0000 mon.a (mon.0) 2510 : cluster [DBG] osdmap e1281: 8 total, 8 up, 8 in 2024-09-13T00:25:36.317 INFO:tasks.workunit.client.0.smithi136.stdout:write: 1 8388609~8388609 2024-09-13T00:25:36.364 INFO:tasks.workunit.client.0.smithi136.stdout:write: X 0~1000 2024-09-13T00:25:36.365 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 1010~1000 2024-09-13T00:25:36.365 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap2 2024-09-13T00:25:36.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:36 smithi181 bash[67587]: cluster 2024-09-13T00:25:35.278746+0000 mon.a (mon.0) 2511 : cluster [DBG] osdmap e1282: 8 total, 8 up, 8 in 2024-09-13T00:25:36.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:36 smithi136 bash[75164]: cluster 2024-09-13T00:25:35.278746+0000 mon.a (mon.0) 2511 : cluster [DBG] osdmap e1282: 8 total, 8 up, 8 in 2024-09-13T00:25:36.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:36 smithi136 bash[76371]: cluster 2024-09-13T00:25:35.278746+0000 mon.a (mon.0) 2511 : cluster [DBG] osdmap e1282: 8 total, 8 up, 8 in 2024-09-13T00:25:37.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:37 smithi181 bash[67587]: cluster 2024-09-13T00:25:35.875683+0000 mgr.y (mgr.24661) 1454 : cluster [DBG] pgmap v2297: 169 pgs: 169 active+clean; 705 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 13 MiB/s rd, 6.0 MiB/s wr, 232 op/s 2024-09-13T00:25:37.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:37 smithi181 bash[67587]: cluster 2024-09-13T00:25:36.290303+0000 mon.a (mon.0) 2512 : cluster [DBG] osdmap e1283: 8 total, 8 up, 8 in 2024-09-13T00:25:37.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:37 smithi136 bash[75164]: cluster 2024-09-13T00:25:35.875683+0000 mgr.y (mgr.24661) 1454 : cluster [DBG] pgmap v2297: 169 pgs: 169 active+clean; 705 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 13 MiB/s rd, 6.0 MiB/s wr, 232 op/s 2024-09-13T00:25:37.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:37 smithi136 bash[75164]: cluster 2024-09-13T00:25:36.290303+0000 mon.a (mon.0) 2512 : cluster [DBG] osdmap e1283: 8 total, 8 up, 8 in 2024-09-13T00:25:37.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:37 smithi136 bash[76371]: cluster 2024-09-13T00:25:35.875683+0000 mgr.y (mgr.24661) 1454 : cluster [DBG] pgmap v2297: 169 pgs: 169 active+clean; 705 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 13 MiB/s rd, 6.0 MiB/s wr, 232 op/s 2024-09-13T00:25:37.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:37 smithi136 bash[76371]: cluster 2024-09-13T00:25:36.290303+0000 mon.a (mon.0) 2512 : cluster [DBG] osdmap e1283: 8 total, 8 up, 8 in 2024-09-13T00:25:38.339 INFO:tasks.workunit.client.0.smithi136.stdout:write: X 8388609~1000 2024-09-13T00:25:38.363 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 16777228~1000 2024-09-13T00:25:38.363 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 167772160 2024-09-13T00:25:38.383 INFO:tasks.workunit.client.0.smithi136.stdout:write: 2 83886079~8388609 2024-09-13T00:25:38.433 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap3 2024-09-13T00:25:38.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:38 smithi181 bash[67587]: cluster 2024-09-13T00:25:37.299315+0000 mon.a (mon.0) 2513 : cluster [DBG] osdmap e1284: 8 total, 8 up, 8 in 2024-09-13T00:25:38.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:38 smithi136 bash[75164]: cluster 2024-09-13T00:25:37.299315+0000 mon.a (mon.0) 2513 : cluster [DBG] osdmap e1284: 8 total, 8 up, 8 in 2024-09-13T00:25:38.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:38 smithi136 bash[76371]: cluster 2024-09-13T00:25:37.299315+0000 mon.a (mon.0) 2513 : cluster [DBG] osdmap e1284: 8 total, 8 up, 8 in 2024-09-13T00:25:39.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:39 smithi181 bash[67587]: cluster 2024-09-13T00:25:37.876766+0000 mgr.y (mgr.24661) 1455 : cluster [DBG] pgmap v2300: 169 pgs: 169 active+clean; 701 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 34 MiB/s rd, 6.0 MiB/s wr, 379 op/s 2024-09-13T00:25:39.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:39 smithi181 bash[67587]: cluster 2024-09-13T00:25:38.314349+0000 mon.a (mon.0) 2514 : cluster [DBG] osdmap e1285: 8 total, 8 up, 8 in 2024-09-13T00:25:39.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:39 smithi181 bash[67587]: audit 2024-09-13T00:25:38.812314+0000 mon.a (mon.0) 2515 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:25:39.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:39 smithi181 bash[67587]: cluster 2024-09-13T00:25:38.973743+0000 mon.a (mon.0) 2516 : cluster [DBG] osdmap e1286: 8 total, 8 up, 8 in 2024-09-13T00:25:39.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:25:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:25:39] "GET /metrics HTTP/1.1" 200 38991 "" "Prometheus/2.43.0" 2024-09-13T00:25:39.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:39 smithi136 bash[76371]: cluster 2024-09-13T00:25:37.876766+0000 mgr.y (mgr.24661) 1455 : cluster [DBG] pgmap v2300: 169 pgs: 169 active+clean; 701 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 34 MiB/s rd, 6.0 MiB/s wr, 379 op/s 2024-09-13T00:25:39.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:39 smithi136 bash[76371]: cluster 2024-09-13T00:25:38.314349+0000 mon.a (mon.0) 2514 : cluster [DBG] osdmap e1285: 8 total, 8 up, 8 in 2024-09-13T00:25:39.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:39 smithi136 bash[76371]: audit 2024-09-13T00:25:38.812314+0000 mon.a (mon.0) 2515 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:25:39.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:39 smithi136 bash[76371]: cluster 2024-09-13T00:25:38.973743+0000 mon.a (mon.0) 2516 : cluster [DBG] osdmap e1286: 8 total, 8 up, 8 in 2024-09-13T00:25:39.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:39 smithi136 bash[75164]: cluster 2024-09-13T00:25:37.876766+0000 mgr.y (mgr.24661) 1455 : cluster [DBG] pgmap v2300: 169 pgs: 169 active+clean; 701 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 34 MiB/s rd, 6.0 MiB/s wr, 379 op/s 2024-09-13T00:25:39.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:39 smithi136 bash[75164]: cluster 2024-09-13T00:25:38.314349+0000 mon.a (mon.0) 2514 : cluster [DBG] osdmap e1285: 8 total, 8 up, 8 in 2024-09-13T00:25:39.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:39 smithi136 bash[75164]: audit 2024-09-13T00:25:38.812314+0000 mon.a (mon.0) 2515 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:25:39.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:39 smithi136 bash[75164]: cluster 2024-09-13T00:25:38.973743+0000 mon.a (mon.0) 2516 : cluster [DBG] osdmap e1286: 8 total, 8 up, 8 in 2024-09-13T00:25:39.997 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:25:40.056 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 83886079~1 2024-09-13T00:25:40.077 INFO:tasks.workunit.client.0.smithi136.stdout:migration_prepare 2024-09-13T00:25:41.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:40 smithi136 bash[75164]: cluster 2024-09-13T00:25:39.877693+0000 mgr.y (mgr.24661) 1456 : cluster [DBG] pgmap v2303: 169 pgs: 169 active+clean; 717 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 40 MiB/s rd, 7.0 MiB/s wr, 176 op/s 2024-09-13T00:25:41.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:40 smithi136 bash[75164]: cluster 2024-09-13T00:25:39.976405+0000 mon.a (mon.0) 2517 : cluster [DBG] osdmap e1287: 8 total, 8 up, 8 in 2024-09-13T00:25:41.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:40 smithi136 bash[76371]: cluster 2024-09-13T00:25:39.877693+0000 mgr.y (mgr.24661) 1456 : cluster [DBG] pgmap v2303: 169 pgs: 169 active+clean; 717 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 40 MiB/s rd, 7.0 MiB/s wr, 176 op/s 2024-09-13T00:25:41.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:40 smithi136 bash[76371]: cluster 2024-09-13T00:25:39.976405+0000 mon.a (mon.0) 2517 : cluster [DBG] osdmap e1287: 8 total, 8 up, 8 in 2024-09-13T00:25:41.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:40 smithi181 bash[67587]: cluster 2024-09-13T00:25:39.877693+0000 mgr.y (mgr.24661) 1456 : cluster [DBG] pgmap v2303: 169 pgs: 169 active+clean; 717 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 40 MiB/s rd, 7.0 MiB/s wr, 176 op/s 2024-09-13T00:25:41.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:40 smithi181 bash[67587]: cluster 2024-09-13T00:25:39.976405+0000 mon.a (mon.0) 2517 : cluster [DBG] osdmap e1287: 8 total, 8 up, 8 in 2024-09-13T00:25:42.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:41 smithi136 bash[75164]: cluster 2024-09-13T00:25:40.987335+0000 mon.a (mon.0) 2518 : cluster [DBG] osdmap e1288: 8 total, 8 up, 8 in 2024-09-13T00:25:42.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:41 smithi136 bash[76371]: cluster 2024-09-13T00:25:40.987335+0000 mon.a (mon.0) 2518 : cluster [DBG] osdmap e1288: 8 total, 8 up, 8 in 2024-09-13T00:25:42.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:41 smithi181 bash[67587]: cluster 2024-09-13T00:25:40.987335+0000 mon.a (mon.0) 2518 : cluster [DBG] osdmap e1288: 8 total, 8 up, 8 in 2024-09-13T00:25:42.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:25:42 smithi181 bash[95505]: ts=2024-09-13T00:25:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:25:43.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:43 smithi136 bash[75164]: cluster 2024-09-13T00:25:41.878990+0000 mgr.y (mgr.24661) 1457 : cluster [DBG] pgmap v2306: 169 pgs: 169 active+clean; 729 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 12 MiB/s rd, 6.0 MiB/s wr, 183 op/s 2024-09-13T00:25:43.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:43 smithi136 bash[75164]: cluster 2024-09-13T00:25:42.001817+0000 mon.a (mon.0) 2519 : cluster [DBG] osdmap e1289: 8 total, 8 up, 8 in 2024-09-13T00:25:43.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:43 smithi136 bash[76371]: cluster 2024-09-13T00:25:41.878990+0000 mgr.y (mgr.24661) 1457 : cluster [DBG] pgmap v2306: 169 pgs: 169 active+clean; 729 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 12 MiB/s rd, 6.0 MiB/s wr, 183 op/s 2024-09-13T00:25:43.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:43 smithi136 bash[76371]: cluster 2024-09-13T00:25:42.001817+0000 mon.a (mon.0) 2519 : cluster [DBG] osdmap e1289: 8 total, 8 up, 8 in 2024-09-13T00:25:43.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:43 smithi181 bash[67587]: cluster 2024-09-13T00:25:41.878990+0000 mgr.y (mgr.24661) 1457 : cluster [DBG] pgmap v2306: 169 pgs: 169 active+clean; 729 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 12 MiB/s rd, 6.0 MiB/s wr, 183 op/s 2024-09-13T00:25:43.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:43 smithi181 bash[67587]: cluster 2024-09-13T00:25:42.001817+0000 mon.a (mon.0) 2519 : cluster [DBG] osdmap e1289: 8 total, 8 up, 8 in 2024-09-13T00:25:43.936 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: after_prepare_snap 2024-09-13T00:25:44.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:44 smithi136 bash[75164]: cluster 2024-09-13T00:25:43.001441+0000 mon.a (mon.0) 2520 : cluster [DBG] osdmap e1290: 8 total, 8 up, 8 in 2024-09-13T00:25:44.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:44 smithi136 bash[75164]: cluster 2024-09-13T00:25:43.977323+0000 mon.a (mon.0) 2521 : cluster [DBG] osdmap e1291: 8 total, 8 up, 8 in 2024-09-13T00:25:44.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:44 smithi136 bash[76371]: cluster 2024-09-13T00:25:43.001441+0000 mon.a (mon.0) 2520 : cluster [DBG] osdmap e1290: 8 total, 8 up, 8 in 2024-09-13T00:25:44.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:44 smithi136 bash[76371]: cluster 2024-09-13T00:25:43.977323+0000 mon.a (mon.0) 2521 : cluster [DBG] osdmap e1291: 8 total, 8 up, 8 in 2024-09-13T00:25:44.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:44 smithi181 bash[67587]: cluster 2024-09-13T00:25:43.001441+0000 mon.a (mon.0) 2520 : cluster [DBG] osdmap e1290: 8 total, 8 up, 8 in 2024-09-13T00:25:44.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:44 smithi181 bash[67587]: cluster 2024-09-13T00:25:43.977323+0000 mon.a (mon.0) 2521 : cluster [DBG] osdmap e1291: 8 total, 8 up, 8 in 2024-09-13T00:25:45.048 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 167772160 2024-09-13T00:25:45.093 INFO:tasks.workunit.client.0.smithi136.stdout:write: * 83886080~1000 2024-09-13T00:25:45.109 INFO:tasks.workunit.client.0.smithi136.stdout:migration_execute 2024-09-13T00:25:45.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:45 smithi181 bash[67587]: cluster 2024-09-13T00:25:43.880179+0000 mgr.y (mgr.24661) 1458 : cluster [DBG] pgmap v2309: 169 pgs: 169 active+clean; 733 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 120 KiB/s rd, 3.0 MiB/s wr, 172 op/s 2024-09-13T00:25:45.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:45 smithi181 bash[67587]: cluster 2024-09-13T00:25:45.017388+0000 mon.a (mon.0) 2522 : cluster [DBG] osdmap e1292: 8 total, 8 up, 8 in 2024-09-13T00:25:45.381 INFO:tasks.workunit.client.0.smithi136.stdout:migration_commit 2024-09-13T00:25:45.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:45 smithi136 bash[75164]: cluster 2024-09-13T00:25:43.880179+0000 mgr.y (mgr.24661) 1458 : cluster [DBG] pgmap v2309: 169 pgs: 169 active+clean; 733 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 120 KiB/s rd, 3.0 MiB/s wr, 172 op/s 2024-09-13T00:25:45.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:45 smithi136 bash[75164]: cluster 2024-09-13T00:25:45.017388+0000 mon.a (mon.0) 2522 : cluster [DBG] osdmap e1292: 8 total, 8 up, 8 in 2024-09-13T00:25:45.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:45 smithi136 bash[76371]: cluster 2024-09-13T00:25:43.880179+0000 mgr.y (mgr.24661) 1458 : cluster [DBG] pgmap v2309: 169 pgs: 169 active+clean; 733 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 120 KiB/s rd, 3.0 MiB/s wr, 172 op/s 2024-09-13T00:25:45.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:45 smithi136 bash[76371]: cluster 2024-09-13T00:25:45.017388+0000 mon.a (mon.0) 2522 : cluster [DBG] osdmap e1292: 8 total, 8 up, 8 in 2024-09-13T00:25:47.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:47 smithi181 bash[67587]: cluster 2024-09-13T00:25:45.881307+0000 mgr.y (mgr.24661) 1459 : cluster [DBG] pgmap v2312: 169 pgs: 169 active+clean; 741 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 19 MiB/s rd, 2.0 MiB/s wr, 46 op/s 2024-09-13T00:25:47.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:47 smithi181 bash[67587]: cluster 2024-09-13T00:25:46.041344+0000 mon.a (mon.0) 2523 : cluster [DBG] osdmap e1293: 8 total, 8 up, 8 in 2024-09-13T00:25:47.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:47 smithi136 bash[75164]: cluster 2024-09-13T00:25:45.881307+0000 mgr.y (mgr.24661) 1459 : cluster [DBG] pgmap v2312: 169 pgs: 169 active+clean; 741 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 19 MiB/s rd, 2.0 MiB/s wr, 46 op/s 2024-09-13T00:25:47.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:47 smithi136 bash[75164]: cluster 2024-09-13T00:25:46.041344+0000 mon.a (mon.0) 2523 : cluster [DBG] osdmap e1293: 8 total, 8 up, 8 in 2024-09-13T00:25:47.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:47 smithi136 bash[76371]: cluster 2024-09-13T00:25:45.881307+0000 mgr.y (mgr.24661) 1459 : cluster [DBG] pgmap v2312: 169 pgs: 169 active+clean; 741 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 19 MiB/s rd, 2.0 MiB/s wr, 46 op/s 2024-09-13T00:25:47.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:47 smithi136 bash[76371]: cluster 2024-09-13T00:25:46.041344+0000 mon.a (mon.0) 2523 : cluster [DBG] osdmap e1293: 8 total, 8 up, 8 in 2024-09-13T00:25:48.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:48 smithi181 bash[67587]: cluster 2024-09-13T00:25:47.054941+0000 mon.a (mon.0) 2524 : cluster [DBG] osdmap e1294: 8 total, 8 up, 8 in 2024-09-13T00:25:48.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:48 smithi136 bash[75164]: cluster 2024-09-13T00:25:47.054941+0000 mon.a (mon.0) 2524 : cluster [DBG] osdmap e1294: 8 total, 8 up, 8 in 2024-09-13T00:25:48.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:48 smithi136 bash[76371]: cluster 2024-09-13T00:25:47.054941+0000 mon.a (mon.0) 2524 : cluster [DBG] osdmap e1294: 8 total, 8 up, 8 in 2024-09-13T00:25:48.842 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.SnapsSnapExpandAfterPrepare (13964 ms) 2024-09-13T00:25:48.842 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.Clone 2024-09-13T00:25:48.925 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:25:48.956 INFO:tasks.workunit.client.0.smithi136.stdout:write: X 0~8388609 2024-09-13T00:25:49.052 INFO:tasks.workunit.client.0.smithi136.stdout:write: X 16777218~8388609 2024-09-13T00:25:49.131 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap 2024-09-13T00:25:49.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:49 smithi181 bash[67587]: cluster 2024-09-13T00:25:47.882233+0000 mgr.y (mgr.24661) 1460 : cluster [DBG] pgmap v2315: 169 pgs: 169 active+clean; 749 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 37 MiB/s rd, 4.0 MiB/s wr, 468 op/s 2024-09-13T00:25:49.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:49 smithi181 bash[67587]: cluster 2024-09-13T00:25:48.063362+0000 mon.a (mon.0) 2525 : cluster [DBG] osdmap e1295: 8 total, 8 up, 8 in 2024-09-13T00:25:49.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:49 smithi136 bash[75164]: cluster 2024-09-13T00:25:47.882233+0000 mgr.y (mgr.24661) 1460 : cluster [DBG] pgmap v2315: 169 pgs: 169 active+clean; 749 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 37 MiB/s rd, 4.0 MiB/s wr, 468 op/s 2024-09-13T00:25:49.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:49 smithi136 bash[75164]: cluster 2024-09-13T00:25:48.063362+0000 mon.a (mon.0) 2525 : cluster [DBG] osdmap e1295: 8 total, 8 up, 8 in 2024-09-13T00:25:49.414 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:49 smithi136 bash[76371]: cluster 2024-09-13T00:25:47.882233+0000 mgr.y (mgr.24661) 1460 : cluster [DBG] pgmap v2315: 169 pgs: 169 active+clean; 749 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 37 MiB/s rd, 4.0 MiB/s wr, 468 op/s 2024-09-13T00:25:49.414 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:49 smithi136 bash[76371]: cluster 2024-09-13T00:25:48.063362+0000 mon.a (mon.0) 2525 : cluster [DBG] osdmap e1295: 8 total, 8 up, 8 in 2024-09-13T00:25:49.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:25:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:25:49] "GET /metrics HTTP/1.1" 200 38991 "" "Prometheus/2.43.0" 2024-09-13T00:25:51.108 INFO:tasks.workunit.client.0.smithi136.stdout:snap_protect: snap 2024-09-13T00:25:51.141 INFO:tasks.workunit.client.0.smithi136.stdout:clone image103 -> image106 2024-09-13T00:25:51.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:51 smithi181 bash[67587]: cluster 2024-09-13T00:25:49.883323+0000 mgr.y (mgr.24661) 1461 : cluster [DBG] pgmap v2317: 169 pgs: 169 active+clean; 757 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 36 MiB/s rd, 5.8 MiB/s wr, 417 op/s 2024-09-13T00:25:51.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:51 smithi181 bash[67587]: cluster 2024-09-13T00:25:50.077999+0000 mon.a (mon.0) 2526 : cluster [DBG] osdmap e1296: 8 total, 8 up, 8 in 2024-09-13T00:25:51.411 INFO:tasks.workunit.client.0.smithi136.stdout:write: X 0~1000 2024-09-13T00:25:51.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:51 smithi136 bash[75164]: cluster 2024-09-13T00:25:49.883323+0000 mgr.y (mgr.24661) 1461 : cluster [DBG] pgmap v2317: 169 pgs: 169 active+clean; 757 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 36 MiB/s rd, 5.8 MiB/s wr, 417 op/s 2024-09-13T00:25:51.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:51 smithi136 bash[75164]: cluster 2024-09-13T00:25:50.077999+0000 mon.a (mon.0) 2526 : cluster [DBG] osdmap e1296: 8 total, 8 up, 8 in 2024-09-13T00:25:51.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:51 smithi136 bash[76371]: cluster 2024-09-13T00:25:49.883323+0000 mgr.y (mgr.24661) 1461 : cluster [DBG] pgmap v2317: 169 pgs: 169 active+clean; 757 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 36 MiB/s rd, 5.8 MiB/s wr, 417 op/s 2024-09-13T00:25:51.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:51 smithi136 bash[76371]: cluster 2024-09-13T00:25:50.077999+0000 mon.a (mon.0) 2526 : cluster [DBG] osdmap e1296: 8 total, 8 up, 8 in 2024-09-13T00:25:51.510 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 1010~1000 2024-09-13T00:25:51.516 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap 2024-09-13T00:25:52.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:52 smithi181 bash[67587]: cluster 2024-09-13T00:25:51.076050+0000 mon.a (mon.0) 2527 : cluster [DBG] osdmap e1297: 8 total, 8 up, 8 in 2024-09-13T00:25:52.349 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:25:52 smithi181 bash[95505]: ts=2024-09-13T00:25:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:25:52.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:52 smithi136 bash[75164]: cluster 2024-09-13T00:25:51.076050+0000 mon.a (mon.0) 2527 : cluster [DBG] osdmap e1297: 8 total, 8 up, 8 in 2024-09-13T00:25:52.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:52 smithi136 bash[76371]: cluster 2024-09-13T00:25:51.076050+0000 mon.a (mon.0) 2527 : cluster [DBG] osdmap e1297: 8 total, 8 up, 8 in 2024-09-13T00:25:53.142 INFO:tasks.workunit.client.0.smithi136.stdout:snap_protect: snap 2024-09-13T00:25:53.173 INFO:tasks.workunit.client.0.smithi136.stdout:clone image106 -> image108 2024-09-13T00:25:53.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:53 smithi181 bash[67587]: cluster 2024-09-13T00:25:51.884390+0000 mgr.y (mgr.24661) 1462 : cluster [DBG] pgmap v2320: 169 pgs: 169 active+clean; 761 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 32 MiB/s rd, 8.3 MiB/s wr, 146 op/s 2024-09-13T00:25:53.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:53 smithi181 bash[67587]: cluster 2024-09-13T00:25:52.102596+0000 mon.a (mon.0) 2528 : cluster [DBG] osdmap e1298: 8 total, 8 up, 8 in 2024-09-13T00:25:53.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:53 smithi136 bash[75164]: cluster 2024-09-13T00:25:51.884390+0000 mgr.y (mgr.24661) 1462 : cluster [DBG] pgmap v2320: 169 pgs: 169 active+clean; 761 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 32 MiB/s rd, 8.3 MiB/s wr, 146 op/s 2024-09-13T00:25:53.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:53 smithi136 bash[75164]: cluster 2024-09-13T00:25:52.102596+0000 mon.a (mon.0) 2528 : cluster [DBG] osdmap e1298: 8 total, 8 up, 8 in 2024-09-13T00:25:53.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:53 smithi136 bash[76371]: cluster 2024-09-13T00:25:51.884390+0000 mgr.y (mgr.24661) 1462 : cluster [DBG] pgmap v2320: 169 pgs: 169 active+clean; 761 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 32 MiB/s rd, 8.3 MiB/s wr, 146 op/s 2024-09-13T00:25:53.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:53 smithi136 bash[76371]: cluster 2024-09-13T00:25:52.102596+0000 mon.a (mon.0) 2528 : cluster [DBG] osdmap e1298: 8 total, 8 up, 8 in 2024-09-13T00:25:53.519 INFO:tasks.workunit.client.0.smithi136.stdout:write: X 1000~1000 2024-09-13T00:25:53.614 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 2010~1000 2024-09-13T00:25:53.619 INFO:tasks.workunit.client.0.smithi136.stdout:migration_prepare 2024-09-13T00:25:54.418 INFO:tasks.workunit.client.0.smithi136.stdout:migration_execute 2024-09-13T00:25:54.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:54 smithi136 bash[76371]: cluster 2024-09-13T00:25:53.115318+0000 mon.a (mon.0) 2529 : cluster [DBG] osdmap e1299: 8 total, 8 up, 8 in 2024-09-13T00:25:54.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:54 smithi136 bash[76371]: audit 2024-09-13T00:25:53.812795+0000 mon.a (mon.0) 2530 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:25:54.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:54 smithi136 bash[76371]: cluster 2024-09-13T00:25:53.988442+0000 mon.a (mon.0) 2531 : cluster [DBG] osdmap e1300: 8 total, 8 up, 8 in 2024-09-13T00:25:54.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:54 smithi136 bash[75164]: cluster 2024-09-13T00:25:53.115318+0000 mon.a (mon.0) 2529 : cluster [DBG] osdmap e1299: 8 total, 8 up, 8 in 2024-09-13T00:25:54.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:54 smithi136 bash[75164]: audit 2024-09-13T00:25:53.812795+0000 mon.a (mon.0) 2530 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:25:54.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:54 smithi136 bash[75164]: cluster 2024-09-13T00:25:53.988442+0000 mon.a (mon.0) 2531 : cluster [DBG] osdmap e1300: 8 total, 8 up, 8 in 2024-09-13T00:25:54.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:54 smithi181 bash[67587]: cluster 2024-09-13T00:25:53.115318+0000 mon.a (mon.0) 2529 : cluster [DBG] osdmap e1299: 8 total, 8 up, 8 in 2024-09-13T00:25:54.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:54 smithi181 bash[67587]: audit 2024-09-13T00:25:53.812795+0000 mon.a (mon.0) 2530 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:25:54.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:54 smithi181 bash[67587]: cluster 2024-09-13T00:25:53.988442+0000 mon.a (mon.0) 2531 : cluster [DBG] osdmap e1300: 8 total, 8 up, 8 in 2024-09-13T00:25:54.883 INFO:tasks.workunit.client.0.smithi136.stdout:migration_commit 2024-09-13T00:25:55.303 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.Clone (6461 ms) 2024-09-13T00:25:55.303 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.CloneParent 2024-09-13T00:25:55.386 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:25:55.415 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap 2024-09-13T00:25:55.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:55 smithi136 bash[75164]: cluster 2024-09-13T00:25:53.885358+0000 mgr.y (mgr.24661) 1463 : cluster [DBG] pgmap v2323: 169 pgs: 169 active+clean; 773 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 25 MiB/s rd, 9.0 MiB/s wr, 135 op/s 2024-09-13T00:25:55.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:55 smithi136 bash[76371]: cluster 2024-09-13T00:25:53.885358+0000 mgr.y (mgr.24661) 1463 : cluster [DBG] pgmap v2323: 169 pgs: 169 active+clean; 773 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 25 MiB/s rd, 9.0 MiB/s wr, 135 op/s 2024-09-13T00:25:55.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:55 smithi181 bash[67587]: cluster 2024-09-13T00:25:53.885358+0000 mgr.y (mgr.24661) 1463 : cluster [DBG] pgmap v2323: 169 pgs: 169 active+clean; 773 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 25 MiB/s rd, 9.0 MiB/s wr, 135 op/s 2024-09-13T00:25:57.164 INFO:tasks.workunit.client.0.smithi136.stdout:snap_protect: snap 2024-09-13T00:25:57.191 INFO:tasks.workunit.client.0.smithi136.stdout:clone image109 -> image112 2024-09-13T00:25:57.442 INFO:tasks.workunit.client.0.smithi136.stdout:migration_prepare 2024-09-13T00:25:57.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:57 smithi136 bash[75164]: cluster 2024-09-13T00:25:55.886492+0000 mgr.y (mgr.24661) 1464 : cluster [DBG] pgmap v2325: 169 pgs: 169 active+clean; 773 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 23 MiB/s rd, 5.8 MiB/s wr, 312 op/s 2024-09-13T00:25:57.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:57 smithi136 bash[75164]: cluster 2024-09-13T00:25:56.132266+0000 mon.a (mon.0) 2532 : cluster [DBG] osdmap e1301: 8 total, 8 up, 8 in 2024-09-13T00:25:57.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:57 smithi136 bash[76371]: cluster 2024-09-13T00:25:55.886492+0000 mgr.y (mgr.24661) 1464 : cluster [DBG] pgmap v2325: 169 pgs: 169 active+clean; 773 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 23 MiB/s rd, 5.8 MiB/s wr, 312 op/s 2024-09-13T00:25:57.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:57 smithi136 bash[76371]: cluster 2024-09-13T00:25:56.132266+0000 mon.a (mon.0) 2532 : cluster [DBG] osdmap e1301: 8 total, 8 up, 8 in 2024-09-13T00:25:57.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:57 smithi181 bash[67587]: cluster 2024-09-13T00:25:55.886492+0000 mgr.y (mgr.24661) 1464 : cluster [DBG] pgmap v2325: 169 pgs: 169 active+clean; 773 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 23 MiB/s rd, 5.8 MiB/s wr, 312 op/s 2024-09-13T00:25:57.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:57 smithi181 bash[67587]: cluster 2024-09-13T00:25:56.132266+0000 mon.a (mon.0) 2532 : cluster [DBG] osdmap e1301: 8 total, 8 up, 8 in 2024-09-13T00:25:57.863 INFO:tasks.workunit.client.0.smithi136.stdout:migration_abort 2024-09-13T00:25:58.213 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.CloneParent (2910 ms) 2024-09-13T00:25:58.213 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.CloneUpdateAfterPrepare 2024-09-13T00:25:58.297 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:25:58.327 INFO:tasks.workunit.client.0.smithi136.stdout:write: X 0~10 2024-09-13T00:25:58.351 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap 2024-09-13T00:25:58.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:58 smithi136 bash[75164]: cluster 2024-09-13T00:25:57.135489+0000 mon.a (mon.0) 2533 : cluster [DBG] osdmap e1302: 8 total, 8 up, 8 in 2024-09-13T00:25:58.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:58 smithi136 bash[76371]: cluster 2024-09-13T00:25:57.135489+0000 mon.a (mon.0) 2533 : cluster [DBG] osdmap e1302: 8 total, 8 up, 8 in 2024-09-13T00:25:58.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:58 smithi181 bash[67587]: cluster 2024-09-13T00:25:57.135489+0000 mon.a (mon.0) 2533 : cluster [DBG] osdmap e1302: 8 total, 8 up, 8 in 2024-09-13T00:25:59.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:59 smithi136 bash[75164]: cluster 2024-09-13T00:25:57.887484+0000 mgr.y (mgr.24661) 1465 : cluster [DBG] pgmap v2328: 169 pgs: 169 active+clean; 777 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 17 MiB/s rd, 3.4 MiB/s wr, 592 op/s 2024-09-13T00:25:59.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:25:59 smithi136 bash[75164]: cluster 2024-09-13T00:25:58.984361+0000 mon.a (mon.0) 2534 : cluster [DBG] osdmap e1303: 8 total, 8 up, 8 in 2024-09-13T00:25:59.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:59 smithi136 bash[76371]: cluster 2024-09-13T00:25:57.887484+0000 mgr.y (mgr.24661) 1465 : cluster [DBG] pgmap v2328: 169 pgs: 169 active+clean; 777 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 17 MiB/s rd, 3.4 MiB/s wr, 592 op/s 2024-09-13T00:25:59.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:25:59 smithi136 bash[76371]: cluster 2024-09-13T00:25:58.984361+0000 mon.a (mon.0) 2534 : cluster [DBG] osdmap e1303: 8 total, 8 up, 8 in 2024-09-13T00:25:59.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:59 smithi181 bash[67587]: cluster 2024-09-13T00:25:57.887484+0000 mgr.y (mgr.24661) 1465 : cluster [DBG] pgmap v2328: 169 pgs: 169 active+clean; 777 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 17 MiB/s rd, 3.4 MiB/s wr, 592 op/s 2024-09-13T00:25:59.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:25:59 smithi181 bash[67587]: cluster 2024-09-13T00:25:58.984361+0000 mon.a (mon.0) 2534 : cluster [DBG] osdmap e1303: 8 total, 8 up, 8 in 2024-09-13T00:25:59.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:25:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:25:59] "GET /metrics HTTP/1.1" 200 38994 "" "Prometheus/2.43.0" 2024-09-13T00:26:00.008 INFO:tasks.workunit.client.0.smithi136.stdout:snap_protect: snap 2024-09-13T00:26:00.037 INFO:tasks.workunit.client.0.smithi136.stdout:clone image113 -> image116 2024-09-13T00:26:00.286 INFO:tasks.workunit.client.0.smithi136.stdout:migration_prepare 2024-09-13T00:26:00.683 INFO:tasks.workunit.client.0.smithi136.stdout:write: Y 0~1 2024-09-13T00:26:00.748 INFO:tasks.workunit.client.0.smithi136.stdout:migration_execute 2024-09-13T00:26:00.948 INFO:tasks.workunit.client.0.smithi136.stdout:migration_commit 2024-09-13T00:26:01.222 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.CloneUpdateAfterPrepare (3009 ms) 2024-09-13T00:26:01.222 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.TriggerAssertSnapcSeq 2024-09-13T00:26:01.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:00 smithi136 bash[76371]: cluster 2024-09-13T00:25:59.888419+0000 mgr.y (mgr.24661) 1466 : cluster [DBG] pgmap v2330: 169 pgs: 169 active+clean; 781 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 15 MiB/s rd, 2.0 MiB/s wr, 617 op/s 2024-09-13T00:26:01.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:00 smithi136 bash[76371]: cluster 2024-09-13T00:25:59.985863+0000 mon.a (mon.0) 2535 : cluster [DBG] osdmap e1304: 8 total, 8 up, 8 in 2024-09-13T00:26:01.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:00 smithi136 bash[75164]: cluster 2024-09-13T00:25:59.888419+0000 mgr.y (mgr.24661) 1466 : cluster [DBG] pgmap v2330: 169 pgs: 169 active+clean; 781 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 15 MiB/s rd, 2.0 MiB/s wr, 617 op/s 2024-09-13T00:26:01.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:00 smithi136 bash[75164]: cluster 2024-09-13T00:25:59.985863+0000 mon.a (mon.0) 2535 : cluster [DBG] osdmap e1304: 8 total, 8 up, 8 in 2024-09-13T00:26:01.304 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:26:01.336 INFO:tasks.workunit.client.0.smithi136.stdout:write: A 41943040~10 2024-09-13T00:26:01.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:00 smithi181 bash[67587]: cluster 2024-09-13T00:25:59.888419+0000 mgr.y (mgr.24661) 1466 : cluster [DBG] pgmap v2330: 169 pgs: 169 active+clean; 781 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 15 MiB/s rd, 2.0 MiB/s wr, 617 op/s 2024-09-13T00:26:01.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:00 smithi181 bash[67587]: cluster 2024-09-13T00:25:59.985863+0000 mon.a (mon.0) 2535 : cluster [DBG] osdmap e1304: 8 total, 8 up, 8 in 2024-09-13T00:26:01.359 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap1 2024-09-13T00:26:02.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:26:02 smithi181 bash[95505]: ts=2024-09-13T00:26:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:26:03.025 INFO:tasks.workunit.client.0.smithi136.stdout:write: B 41943041~10 2024-09-13T00:26:03.046 INFO:tasks.workunit.client.0.smithi136.stdout:migration_prepare 2024-09-13T00:26:03.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:02 smithi136 bash[75164]: cluster 2024-09-13T00:26:01.889847+0000 mgr.y (mgr.24661) 1467 : cluster [DBG] pgmap v2332: 169 pgs: 169 active+clean; 785 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 11 MiB/s rd, 2.1 MiB/s wr, 617 op/s 2024-09-13T00:26:03.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:02 smithi136 bash[75164]: cluster 2024-09-13T00:26:01.996120+0000 mon.a (mon.0) 2536 : cluster [DBG] osdmap e1305: 8 total, 8 up, 8 in 2024-09-13T00:26:03.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:02 smithi136 bash[76371]: cluster 2024-09-13T00:26:01.889847+0000 mgr.y (mgr.24661) 1467 : cluster [DBG] pgmap v2332: 169 pgs: 169 active+clean; 785 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 11 MiB/s rd, 2.1 MiB/s wr, 617 op/s 2024-09-13T00:26:03.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:02 smithi136 bash[76371]: cluster 2024-09-13T00:26:01.996120+0000 mon.a (mon.0) 2536 : cluster [DBG] osdmap e1305: 8 total, 8 up, 8 in 2024-09-13T00:26:03.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:02 smithi181 bash[67587]: cluster 2024-09-13T00:26:01.889847+0000 mgr.y (mgr.24661) 1467 : cluster [DBG] pgmap v2332: 169 pgs: 169 active+clean; 785 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 11 MiB/s rd, 2.1 MiB/s wr, 617 op/s 2024-09-13T00:26:03.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:02 smithi181 bash[67587]: cluster 2024-09-13T00:26:01.996120+0000 mon.a (mon.0) 2536 : cluster [DBG] osdmap e1305: 8 total, 8 up, 8 in 2024-09-13T00:26:04.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:04 smithi136 bash[75164]: cluster 2024-09-13T00:26:03.005424+0000 mon.a (mon.0) 2537 : cluster [DBG] osdmap e1306: 8 total, 8 up, 8 in 2024-09-13T00:26:04.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:04 smithi136 bash[76371]: cluster 2024-09-13T00:26:03.005424+0000 mon.a (mon.0) 2537 : cluster [DBG] osdmap e1306: 8 total, 8 up, 8 in 2024-09-13T00:26:04.348 INFO:tasks.workunit.client.0.smithi136.stdout:write: C 41943042~10 2024-09-13T00:26:04.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:04 smithi181 bash[67587]: cluster 2024-09-13T00:26:03.005424+0000 mon.a (mon.0) 2537 : cluster [DBG] osdmap e1306: 8 total, 8 up, 8 in 2024-09-13T00:26:04.384 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap2 2024-09-13T00:26:05.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:05 smithi136 bash[75164]: cluster 2024-09-13T00:26:03.890792+0000 mgr.y (mgr.24661) 1468 : cluster [DBG] pgmap v2335: 169 pgs: 169 active+clean; 785 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.7 MiB/s wr, 581 op/s 2024-09-13T00:26:05.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:05 smithi136 bash[75164]: cluster 2024-09-13T00:26:04.016815+0000 mon.a (mon.0) 2538 : cluster [DBG] osdmap e1307: 8 total, 8 up, 8 in 2024-09-13T00:26:05.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:05 smithi136 bash[76371]: cluster 2024-09-13T00:26:03.890792+0000 mgr.y (mgr.24661) 1468 : cluster [DBG] pgmap v2335: 169 pgs: 169 active+clean; 785 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.7 MiB/s wr, 581 op/s 2024-09-13T00:26:05.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:05 smithi136 bash[76371]: cluster 2024-09-13T00:26:04.016815+0000 mon.a (mon.0) 2538 : cluster [DBG] osdmap e1307: 8 total, 8 up, 8 in 2024-09-13T00:26:05.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:05 smithi181 bash[67587]: cluster 2024-09-13T00:26:03.890792+0000 mgr.y (mgr.24661) 1468 : cluster [DBG] pgmap v2335: 169 pgs: 169 active+clean; 785 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.7 MiB/s wr, 581 op/s 2024-09-13T00:26:05.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:05 smithi181 bash[67587]: cluster 2024-09-13T00:26:04.016815+0000 mon.a (mon.0) 2538 : cluster [DBG] osdmap e1307: 8 total, 8 up, 8 in 2024-09-13T00:26:06.053 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 41943040 2024-09-13T00:26:06.135 INFO:tasks.workunit.client.0.smithi136.stdout:migration_execute 2024-09-13T00:26:06.308 INFO:tasks.workunit.client.0.smithi136.stdout:migration_commit 2024-09-13T00:26:06.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:06 smithi181 bash[67587]: cluster 2024-09-13T00:26:05.029864+0000 mon.a (mon.0) 2539 : cluster [DBG] osdmap e1308: 8 total, 8 up, 8 in 2024-09-13T00:26:06.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:06 smithi136 bash[75164]: cluster 2024-09-13T00:26:05.029864+0000 mon.a (mon.0) 2539 : cluster [DBG] osdmap e1308: 8 total, 8 up, 8 in 2024-09-13T00:26:06.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:06 smithi136 bash[76371]: cluster 2024-09-13T00:26:05.029864+0000 mon.a (mon.0) 2539 : cluster [DBG] osdmap e1308: 8 total, 8 up, 8 in 2024-09-13T00:26:07.305 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.TriggerAssertSnapcSeq (6083 ms) 2024-09-13T00:26:07.306 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.SnapTrimBeforePrepare 2024-09-13T00:26:07.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:07 smithi181 bash[67587]: cluster 2024-09-13T00:26:05.891838+0000 mgr.y (mgr.24661) 1469 : cluster [DBG] pgmap v2338: 169 pgs: 169 active+clean; 785 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 242 KiB/s rd, 12 KiB/s wr, 306 op/s 2024-09-13T00:26:07.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:07 smithi181 bash[67587]: cluster 2024-09-13T00:26:06.035962+0000 mon.a (mon.0) 2540 : cluster [DBG] osdmap e1309: 8 total, 8 up, 8 in 2024-09-13T00:26:07.384 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:26:07.413 INFO:tasks.workunit.client.0.smithi136.stdout:write: A 41943040~10 2024-09-13T00:26:07.436 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap1 2024-09-13T00:26:07.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:07 smithi136 bash[75164]: cluster 2024-09-13T00:26:05.891838+0000 mgr.y (mgr.24661) 1469 : cluster [DBG] pgmap v2338: 169 pgs: 169 active+clean; 785 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 242 KiB/s rd, 12 KiB/s wr, 306 op/s 2024-09-13T00:26:07.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:07 smithi136 bash[75164]: cluster 2024-09-13T00:26:06.035962+0000 mon.a (mon.0) 2540 : cluster [DBG] osdmap e1309: 8 total, 8 up, 8 in 2024-09-13T00:26:07.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:07 smithi136 bash[76371]: cluster 2024-09-13T00:26:05.891838+0000 mgr.y (mgr.24661) 1469 : cluster [DBG] pgmap v2338: 169 pgs: 169 active+clean; 785 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 242 KiB/s rd, 12 KiB/s wr, 306 op/s 2024-09-13T00:26:07.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:07 smithi136 bash[76371]: cluster 2024-09-13T00:26:06.035962+0000 mon.a (mon.0) 2540 : cluster [DBG] osdmap e1309: 8 total, 8 up, 8 in 2024-09-13T00:26:08.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:08 smithi181 bash[67587]: cluster 2024-09-13T00:26:07.044240+0000 mon.a (mon.0) 2541 : cluster [DBG] osdmap e1310: 8 total, 8 up, 8 in 2024-09-13T00:26:08.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:08 smithi136 bash[75164]: cluster 2024-09-13T00:26:07.044240+0000 mon.a (mon.0) 2541 : cluster [DBG] osdmap e1310: 8 total, 8 up, 8 in 2024-09-13T00:26:08.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:08 smithi136 bash[76371]: cluster 2024-09-13T00:26:07.044240+0000 mon.a (mon.0) 2541 : cluster [DBG] osdmap e1310: 8 total, 8 up, 8 in 2024-09-13T00:26:09.101 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 41943040 2024-09-13T00:26:09.148 INFO:tasks.workunit.client.0.smithi136.stdout:migration_prepare 2024-09-13T00:26:09.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:09 smithi181 bash[67587]: cluster 2024-09-13T00:26:07.892840+0000 mgr.y (mgr.24661) 1470 : cluster [DBG] pgmap v2341: 169 pgs: 169 active+clean; 785 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 154 KiB/s rd, 13 KiB/s wr, 202 op/s 2024-09-13T00:26:09.409 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:09 smithi181 bash[67587]: cluster 2024-09-13T00:26:08.054703+0000 mon.a (mon.0) 2542 : cluster [DBG] osdmap e1311: 8 total, 8 up, 8 in 2024-09-13T00:26:09.410 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:09 smithi181 bash[67587]: audit 2024-09-13T00:26:08.813107+0000 mon.a (mon.0) 2543 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:26:09.414 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:09 smithi136 bash[76371]: cluster 2024-09-13T00:26:07.892840+0000 mgr.y (mgr.24661) 1470 : cluster [DBG] pgmap v2341: 169 pgs: 169 active+clean; 785 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 154 KiB/s rd, 13 KiB/s wr, 202 op/s 2024-09-13T00:26:09.414 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:09 smithi136 bash[76371]: cluster 2024-09-13T00:26:08.054703+0000 mon.a (mon.0) 2542 : cluster [DBG] osdmap e1311: 8 total, 8 up, 8 in 2024-09-13T00:26:09.414 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:09 smithi136 bash[76371]: audit 2024-09-13T00:26:08.813107+0000 mon.a (mon.0) 2543 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:26:09.414 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:09 smithi136 bash[75164]: cluster 2024-09-13T00:26:07.892840+0000 mgr.y (mgr.24661) 1470 : cluster [DBG] pgmap v2341: 169 pgs: 169 active+clean; 785 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 154 KiB/s rd, 13 KiB/s wr, 202 op/s 2024-09-13T00:26:09.414 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:09 smithi136 bash[75164]: cluster 2024-09-13T00:26:08.054703+0000 mon.a (mon.0) 2542 : cluster [DBG] osdmap e1311: 8 total, 8 up, 8 in 2024-09-13T00:26:09.414 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:09 smithi136 bash[75164]: audit 2024-09-13T00:26:08.813107+0000 mon.a (mon.0) 2543 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:26:09.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:26:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:26:09] "GET /metrics HTTP/1.1" 200 38992 "" "Prometheus/2.43.0" 2024-09-13T00:26:10.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:10 smithi181 bash[67587]: cluster 2024-09-13T00:26:09.062214+0000 mon.a (mon.0) 2544 : cluster [DBG] osdmap e1312: 8 total, 8 up, 8 in 2024-09-13T00:26:10.351 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:26:10.381 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap3 2024-09-13T00:26:10.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:10 smithi136 bash[75164]: cluster 2024-09-13T00:26:09.062214+0000 mon.a (mon.0) 2544 : cluster [DBG] osdmap e1312: 8 total, 8 up, 8 in 2024-09-13T00:26:10.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:10 smithi136 bash[76371]: cluster 2024-09-13T00:26:09.062214+0000 mon.a (mon.0) 2544 : cluster [DBG] osdmap e1312: 8 total, 8 up, 8 in 2024-09-13T00:26:11.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:11 smithi181 bash[67587]: cluster 2024-09-13T00:26:09.893651+0000 mgr.y (mgr.24661) 1471 : cluster [DBG] pgmap v2344: 169 pgs: 169 active+clean; 785 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 140 KiB/s rd, 14 KiB/s wr, 191 op/s 2024-09-13T00:26:11.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:11 smithi181 bash[67587]: cluster 2024-09-13T00:26:10.071954+0000 mon.a (mon.0) 2545 : cluster [DBG] osdmap e1313: 8 total, 8 up, 8 in 2024-09-13T00:26:11.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:11 smithi136 bash[75164]: cluster 2024-09-13T00:26:09.893651+0000 mgr.y (mgr.24661) 1471 : cluster [DBG] pgmap v2344: 169 pgs: 169 active+clean; 785 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 140 KiB/s rd, 14 KiB/s wr, 191 op/s 2024-09-13T00:26:11.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:11 smithi136 bash[75164]: cluster 2024-09-13T00:26:10.071954+0000 mon.a (mon.0) 2545 : cluster [DBG] osdmap e1313: 8 total, 8 up, 8 in 2024-09-13T00:26:11.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:11 smithi136 bash[76371]: cluster 2024-09-13T00:26:09.893651+0000 mgr.y (mgr.24661) 1471 : cluster [DBG] pgmap v2344: 169 pgs: 169 active+clean; 785 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 140 KiB/s rd, 14 KiB/s wr, 191 op/s 2024-09-13T00:26:11.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:11 smithi136 bash[76371]: cluster 2024-09-13T00:26:10.071954+0000 mon.a (mon.0) 2545 : cluster [DBG] osdmap e1313: 8 total, 8 up, 8 in 2024-09-13T00:26:12.115 INFO:tasks.workunit.client.0.smithi136.stdout:write: B 41943040~10 2024-09-13T00:26:12.130 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap4 2024-09-13T00:26:12.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:12 smithi181 bash[67587]: cluster 2024-09-13T00:26:11.079752+0000 mon.a (mon.0) 2546 : cluster [DBG] osdmap e1314: 8 total, 8 up, 8 in 2024-09-13T00:26:12.348 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:26:12 smithi181 bash[95505]: ts=2024-09-13T00:26:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:26:12.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:12 smithi136 bash[75164]: cluster 2024-09-13T00:26:11.079752+0000 mon.a (mon.0) 2546 : cluster [DBG] osdmap e1314: 8 total, 8 up, 8 in 2024-09-13T00:26:12.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:12 smithi136 bash[76371]: cluster 2024-09-13T00:26:11.079752+0000 mon.a (mon.0) 2546 : cluster [DBG] osdmap e1314: 8 total, 8 up, 8 in 2024-09-13T00:26:13.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:13 smithi181 bash[67587]: cluster 2024-09-13T00:26:11.894782+0000 mgr.y (mgr.24661) 1472 : cluster [DBG] pgmap v2347: 169 pgs: 169 active+clean; 785 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 272 KiB/s rd, 23 KiB/s wr, 359 op/s 2024-09-13T00:26:13.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:13 smithi181 bash[67587]: cluster 2024-09-13T00:26:12.091584+0000 mon.a (mon.0) 2547 : cluster [DBG] osdmap e1315: 8 total, 8 up, 8 in 2024-09-13T00:26:13.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:13 smithi136 bash[75164]: cluster 2024-09-13T00:26:11.894782+0000 mgr.y (mgr.24661) 1472 : cluster [DBG] pgmap v2347: 169 pgs: 169 active+clean; 785 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 272 KiB/s rd, 23 KiB/s wr, 359 op/s 2024-09-13T00:26:13.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:13 smithi136 bash[75164]: cluster 2024-09-13T00:26:12.091584+0000 mon.a (mon.0) 2547 : cluster [DBG] osdmap e1315: 8 total, 8 up, 8 in 2024-09-13T00:26:13.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:13 smithi136 bash[76371]: cluster 2024-09-13T00:26:11.894782+0000 mgr.y (mgr.24661) 1472 : cluster [DBG] pgmap v2347: 169 pgs: 169 active+clean; 785 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 272 KiB/s rd, 23 KiB/s wr, 359 op/s 2024-09-13T00:26:13.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:13 smithi136 bash[76371]: cluster 2024-09-13T00:26:12.091584+0000 mon.a (mon.0) 2547 : cluster [DBG] osdmap e1315: 8 total, 8 up, 8 in 2024-09-13T00:26:14.015 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 41943040 2024-09-13T00:26:14.071 INFO:tasks.workunit.client.0.smithi136.stdout:migration_execute 2024-09-13T00:26:14.240 INFO:tasks.workunit.client.0.smithi136.stdout:migration_commit 2024-09-13T00:26:14.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:14 smithi136 bash[75164]: cluster 2024-09-13T00:26:13.097593+0000 mon.a (mon.0) 2548 : cluster [DBG] osdmap e1316: 8 total, 8 up, 8 in 2024-09-13T00:26:14.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:14 smithi136 bash[75164]: cluster 2024-09-13T00:26:13.986416+0000 mon.a (mon.0) 2549 : cluster [DBG] osdmap e1317: 8 total, 8 up, 8 in 2024-09-13T00:26:14.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:14 smithi136 bash[76371]: cluster 2024-09-13T00:26:13.097593+0000 mon.a (mon.0) 2548 : cluster [DBG] osdmap e1316: 8 total, 8 up, 8 in 2024-09-13T00:26:14.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:14 smithi136 bash[76371]: cluster 2024-09-13T00:26:13.986416+0000 mon.a (mon.0) 2549 : cluster [DBG] osdmap e1317: 8 total, 8 up, 8 in 2024-09-13T00:26:14.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:14 smithi181 bash[67587]: cluster 2024-09-13T00:26:13.097593+0000 mon.a (mon.0) 2548 : cluster [DBG] osdmap e1316: 8 total, 8 up, 8 in 2024-09-13T00:26:14.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:14 smithi181 bash[67587]: cluster 2024-09-13T00:26:13.986416+0000 mon.a (mon.0) 2549 : cluster [DBG] osdmap e1317: 8 total, 8 up, 8 in 2024-09-13T00:26:15.414 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.SnapTrimBeforePrepare (8109 ms) 2024-09-13T00:26:15.414 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.AbortInUseImage 2024-09-13T00:26:15.499 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:26:15.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:15 smithi136 bash[75164]: cluster 2024-09-13T00:26:13.895737+0000 mgr.y (mgr.24661) 1473 : cluster [DBG] pgmap v2350: 169 pgs: 169 active+clean; 785 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 170 KiB/s rd, 17 KiB/s wr, 231 op/s 2024-09-13T00:26:15.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:15 smithi136 bash[76371]: cluster 2024-09-13T00:26:13.895737+0000 mgr.y (mgr.24661) 1473 : cluster [DBG] pgmap v2350: 169 pgs: 169 active+clean; 785 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 170 KiB/s rd, 17 KiB/s wr, 231 op/s 2024-09-13T00:26:15.528 INFO:tasks.workunit.client.0.smithi136.stdout:migration_prepare 2024-09-13T00:26:15.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:15 smithi181 bash[67587]: cluster 2024-09-13T00:26:13.895737+0000 mgr.y (mgr.24661) 1473 : cluster [DBG] pgmap v2350: 169 pgs: 169 active+clean; 785 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 170 KiB/s rd, 17 KiB/s wr, 231 op/s 2024-09-13T00:26:15.995 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.AbortInUseImage (582 ms) 2024-09-13T00:26:15.996 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.AbortWithoutSnapshots 2024-09-13T00:26:16.079 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:26:16.108 INFO:tasks.workunit.client.0.smithi136.stdout:write: 1 0~8388609 2024-09-13T00:26:16.213 INFO:tasks.workunit.client.0.smithi136.stdout:write: 1 16777218~8388609 2024-09-13T00:26:16.294 INFO:tasks.workunit.client.0.smithi136.stdout:migration_prepare 2024-09-13T00:26:16.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:16 smithi136 bash[75164]: cluster 2024-09-13T00:26:15.113731+0000 mon.a (mon.0) 2550 : cluster [DBG] osdmap e1318: 8 total, 8 up, 8 in 2024-09-13T00:26:16.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:16 smithi136 bash[76371]: cluster 2024-09-13T00:26:15.113731+0000 mon.a (mon.0) 2550 : cluster [DBG] osdmap e1318: 8 total, 8 up, 8 in 2024-09-13T00:26:16.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:16 smithi181 bash[67587]: cluster 2024-09-13T00:26:15.113731+0000 mon.a (mon.0) 2550 : cluster [DBG] osdmap e1318: 8 total, 8 up, 8 in 2024-09-13T00:26:16.750 INFO:tasks.workunit.client.0.smithi136.stdout:write: 1 0~8388609 2024-09-13T00:26:16.819 INFO:tasks.workunit.client.0.smithi136.stdout:write: 1 16777218~8388609 2024-09-13T00:26:16.914 INFO:tasks.workunit.client.0.smithi136.stdout:migration_abort 2024-09-13T00:26:17.308 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.AbortWithoutSnapshots (1312 ms) 2024-09-13T00:26:17.308 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.AbortWithSnapshots 2024-09-13T00:26:17.383 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:26:17.412 INFO:tasks.workunit.client.0.smithi136.stdout:write: 1 0~8388609 2024-09-13T00:26:17.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:17 smithi136 bash[75164]: cluster 2024-09-13T00:26:15.896883+0000 mgr.y (mgr.24661) 1474 : cluster [DBG] pgmap v2353: 169 pgs: 169 active+clean; 785 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 116 KiB/s rd, 8.0 KiB/s wr, 153 op/s 2024-09-13T00:26:17.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:17 smithi136 bash[76371]: cluster 2024-09-13T00:26:15.896883+0000 mgr.y (mgr.24661) 1474 : cluster [DBG] pgmap v2353: 169 pgs: 169 active+clean; 785 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 116 KiB/s rd, 8.0 KiB/s wr, 153 op/s 2024-09-13T00:26:17.515 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap1 2024-09-13T00:26:17.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:17 smithi181 bash[67587]: cluster 2024-09-13T00:26:15.896883+0000 mgr.y (mgr.24661) 1474 : cluster [DBG] pgmap v2353: 169 pgs: 169 active+clean; 785 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 116 KiB/s rd, 8.0 KiB/s wr, 153 op/s 2024-09-13T00:26:19.182 INFO:tasks.workunit.client.0.smithi136.stdout:write: 1 8388609~8388609 2024-09-13T00:26:19.234 INFO:tasks.workunit.client.0.smithi136.stdout:write: X 0~1000 2024-09-13T00:26:19.234 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 1010~1000 2024-09-13T00:26:19.235 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap2 2024-09-13T00:26:19.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:19 smithi136 bash[75164]: cluster 2024-09-13T00:26:17.898203+0000 mgr.y (mgr.24661) 1475 : cluster [DBG] pgmap v2354: 169 pgs: 169 active+clean; 801 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.8 MiB/s wr, 227 op/s 2024-09-13T00:26:19.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:19 smithi136 bash[75164]: cluster 2024-09-13T00:26:18.154129+0000 mon.a (mon.0) 2551 : cluster [DBG] osdmap e1319: 8 total, 8 up, 8 in 2024-09-13T00:26:19.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:19 smithi136 bash[76371]: cluster 2024-09-13T00:26:17.898203+0000 mgr.y (mgr.24661) 1475 : cluster [DBG] pgmap v2354: 169 pgs: 169 active+clean; 801 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.8 MiB/s wr, 227 op/s 2024-09-13T00:26:19.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:19 smithi136 bash[76371]: cluster 2024-09-13T00:26:18.154129+0000 mon.a (mon.0) 2551 : cluster [DBG] osdmap e1319: 8 total, 8 up, 8 in 2024-09-13T00:26:19.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:19 smithi181 bash[67587]: cluster 2024-09-13T00:26:17.898203+0000 mgr.y (mgr.24661) 1475 : cluster [DBG] pgmap v2354: 169 pgs: 169 active+clean; 801 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.8 MiB/s wr, 227 op/s 2024-09-13T00:26:19.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:19 smithi181 bash[67587]: cluster 2024-09-13T00:26:18.154129+0000 mon.a (mon.0) 2551 : cluster [DBG] osdmap e1319: 8 total, 8 up, 8 in 2024-09-13T00:26:19.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:26:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:26:19] "GET /metrics HTTP/1.1" 200 38992 "" "Prometheus/2.43.0" 2024-09-13T00:26:20.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:20 smithi136 bash[75164]: cluster 2024-09-13T00:26:19.157971+0000 mon.a (mon.0) 2552 : cluster [DBG] osdmap e1320: 8 total, 8 up, 8 in 2024-09-13T00:26:20.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:20 smithi136 bash[75164]: audit 2024-09-13T00:26:19.267713+0000 mon.a (mon.0) 2553 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:26:20.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:20 smithi136 bash[75164]: audit 2024-09-13T00:26:19.608246+0000 mon.a (mon.0) 2554 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:26:20.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:20 smithi136 bash[75164]: audit 2024-09-13T00:26:19.615287+0000 mon.a (mon.0) 2555 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:26:20.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:20 smithi136 bash[75164]: audit 2024-09-13T00:26:20.013547+0000 mon.a (mon.0) 2556 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:26:20.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:20 smithi136 bash[75164]: audit 2024-09-13T00:26:20.014523+0000 mon.a (mon.0) 2557 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:26:20.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:20 smithi136 bash[75164]: audit 2024-09-13T00:26:20.021890+0000 mon.a (mon.0) 2558 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:26:20.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:20 smithi136 bash[76371]: cluster 2024-09-13T00:26:19.157971+0000 mon.a (mon.0) 2552 : cluster [DBG] osdmap e1320: 8 total, 8 up, 8 in 2024-09-13T00:26:20.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:20 smithi136 bash[76371]: audit 2024-09-13T00:26:19.267713+0000 mon.a (mon.0) 2553 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:26:20.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:20 smithi136 bash[76371]: audit 2024-09-13T00:26:19.608246+0000 mon.a (mon.0) 2554 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:26:20.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:20 smithi136 bash[76371]: audit 2024-09-13T00:26:19.615287+0000 mon.a (mon.0) 2555 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:26:20.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:20 smithi136 bash[76371]: audit 2024-09-13T00:26:20.013547+0000 mon.a (mon.0) 2556 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:26:20.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:20 smithi136 bash[76371]: audit 2024-09-13T00:26:20.014523+0000 mon.a (mon.0) 2557 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:26:20.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:20 smithi136 bash[76371]: audit 2024-09-13T00:26:20.021890+0000 mon.a (mon.0) 2558 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:26:20.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:20 smithi181 bash[67587]: cluster 2024-09-13T00:26:19.157971+0000 mon.a (mon.0) 2552 : cluster [DBG] osdmap e1320: 8 total, 8 up, 8 in 2024-09-13T00:26:20.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:20 smithi181 bash[67587]: audit 2024-09-13T00:26:19.267713+0000 mon.a (mon.0) 2553 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:26:20.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:20 smithi181 bash[67587]: audit 2024-09-13T00:26:19.608246+0000 mon.a (mon.0) 2554 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:26:20.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:20 smithi181 bash[67587]: audit 2024-09-13T00:26:19.615287+0000 mon.a (mon.0) 2555 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:26:20.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:20 smithi181 bash[67587]: audit 2024-09-13T00:26:20.013547+0000 mon.a (mon.0) 2556 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:26:20.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:20 smithi181 bash[67587]: audit 2024-09-13T00:26:20.014523+0000 mon.a (mon.0) 2557 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:26:20.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:20 smithi181 bash[67587]: audit 2024-09-13T00:26:20.021890+0000 mon.a (mon.0) 2558 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:26:21.224 INFO:tasks.workunit.client.0.smithi136.stdout:write: X 8388609~1000 2024-09-13T00:26:21.243 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 16777228~1000 2024-09-13T00:26:21.243 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 167772160 2024-09-13T00:26:21.262 INFO:tasks.workunit.client.0.smithi136.stdout:write: 2 83886079~8388609 2024-09-13T00:26:21.308 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap3 2024-09-13T00:26:21.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:21 smithi136 bash[75164]: cluster 2024-09-13T00:26:19.899038+0000 mgr.y (mgr.24661) 1476 : cluster [DBG] pgmap v2357: 169 pgs: 169 active+clean; 829 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 5.7 MiB/s rd, 8.8 MiB/s wr, 381 op/s 2024-09-13T00:26:21.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:21 smithi136 bash[75164]: cluster 2024-09-13T00:26:20.185049+0000 mon.a (mon.0) 2559 : cluster [DBG] osdmap e1321: 8 total, 8 up, 8 in 2024-09-13T00:26:21.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:21 smithi136 bash[76371]: cluster 2024-09-13T00:26:19.899038+0000 mgr.y (mgr.24661) 1476 : cluster [DBG] pgmap v2357: 169 pgs: 169 active+clean; 829 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 5.7 MiB/s rd, 8.8 MiB/s wr, 381 op/s 2024-09-13T00:26:21.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:21 smithi136 bash[76371]: cluster 2024-09-13T00:26:20.185049+0000 mon.a (mon.0) 2559 : cluster [DBG] osdmap e1321: 8 total, 8 up, 8 in 2024-09-13T00:26:21.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:21 smithi181 bash[67587]: cluster 2024-09-13T00:26:19.899038+0000 mgr.y (mgr.24661) 1476 : cluster [DBG] pgmap v2357: 169 pgs: 169 active+clean; 829 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 5.7 MiB/s rd, 8.8 MiB/s wr, 381 op/s 2024-09-13T00:26:21.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:21 smithi181 bash[67587]: cluster 2024-09-13T00:26:20.185049+0000 mon.a (mon.0) 2559 : cluster [DBG] osdmap e1321: 8 total, 8 up, 8 in 2024-09-13T00:26:22.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:22 smithi136 bash[75164]: cluster 2024-09-13T00:26:21.189461+0000 mon.a (mon.0) 2560 : cluster [DBG] osdmap e1322: 8 total, 8 up, 8 in 2024-09-13T00:26:22.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:22 smithi136 bash[76371]: cluster 2024-09-13T00:26:21.189461+0000 mon.a (mon.0) 2560 : cluster [DBG] osdmap e1322: 8 total, 8 up, 8 in 2024-09-13T00:26:22.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:22 smithi181 bash[67587]: cluster 2024-09-13T00:26:21.189461+0000 mon.a (mon.0) 2560 : cluster [DBG] osdmap e1322: 8 total, 8 up, 8 in 2024-09-13T00:26:22.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:26:22 smithi181 bash[95505]: ts=2024-09-13T00:26:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:26:23.251 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:26:23.311 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 83886079~1 2024-09-13T00:26:23.331 INFO:tasks.workunit.client.0.smithi136.stdout:migration_prepare 2024-09-13T00:26:23.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:23 smithi136 bash[75164]: cluster 2024-09-13T00:26:21.900215+0000 mgr.y (mgr.24661) 1477 : cluster [DBG] pgmap v2360: 169 pgs: 169 active+clean; 849 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 17 MiB/s rd, 25 MiB/s wr, 381 op/s 2024-09-13T00:26:23.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:23 smithi136 bash[75164]: cluster 2024-09-13T00:26:22.212168+0000 mon.a (mon.0) 2561 : cluster [DBG] osdmap e1323: 8 total, 8 up, 8 in 2024-09-13T00:26:23.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:23 smithi136 bash[76371]: cluster 2024-09-13T00:26:21.900215+0000 mgr.y (mgr.24661) 1477 : cluster [DBG] pgmap v2360: 169 pgs: 169 active+clean; 849 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 17 MiB/s rd, 25 MiB/s wr, 381 op/s 2024-09-13T00:26:23.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:23 smithi136 bash[76371]: cluster 2024-09-13T00:26:22.212168+0000 mon.a (mon.0) 2561 : cluster [DBG] osdmap e1323: 8 total, 8 up, 8 in 2024-09-13T00:26:23.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:23 smithi181 bash[67587]: cluster 2024-09-13T00:26:21.900215+0000 mgr.y (mgr.24661) 1477 : cluster [DBG] pgmap v2360: 169 pgs: 169 active+clean; 849 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 17 MiB/s rd, 25 MiB/s wr, 381 op/s 2024-09-13T00:26:23.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:23 smithi181 bash[67587]: cluster 2024-09-13T00:26:22.212168+0000 mon.a (mon.0) 2561 : cluster [DBG] osdmap e1323: 8 total, 8 up, 8 in 2024-09-13T00:26:24.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:24 smithi136 bash[75164]: cluster 2024-09-13T00:26:23.219759+0000 mon.a (mon.0) 2562 : cluster [DBG] osdmap e1324: 8 total, 8 up, 8 in 2024-09-13T00:26:24.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:24 smithi136 bash[75164]: audit 2024-09-13T00:26:23.813590+0000 mon.a (mon.0) 2563 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:26:24.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:24 smithi136 bash[75164]: cluster 2024-09-13T00:26:23.991370+0000 mon.a (mon.0) 2564 : cluster [DBG] osdmap e1325: 8 total, 8 up, 8 in 2024-09-13T00:26:24.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:24 smithi136 bash[76371]: cluster 2024-09-13T00:26:23.219759+0000 mon.a (mon.0) 2562 : cluster [DBG] osdmap e1324: 8 total, 8 up, 8 in 2024-09-13T00:26:24.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:24 smithi136 bash[76371]: audit 2024-09-13T00:26:23.813590+0000 mon.a (mon.0) 2563 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:26:24.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:24 smithi136 bash[76371]: cluster 2024-09-13T00:26:23.991370+0000 mon.a (mon.0) 2564 : cluster [DBG] osdmap e1325: 8 total, 8 up, 8 in 2024-09-13T00:26:24.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:24 smithi181 bash[67587]: cluster 2024-09-13T00:26:23.219759+0000 mon.a (mon.0) 2562 : cluster [DBG] osdmap e1324: 8 total, 8 up, 8 in 2024-09-13T00:26:24.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:24 smithi181 bash[67587]: audit 2024-09-13T00:26:23.813590+0000 mon.a (mon.0) 2563 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:26:24.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:24 smithi181 bash[67587]: cluster 2024-09-13T00:26:23.991370+0000 mon.a (mon.0) 2564 : cluster [DBG] osdmap e1325: 8 total, 8 up, 8 in 2024-09-13T00:26:25.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:25 smithi136 bash[75164]: cluster 2024-09-13T00:26:23.901253+0000 mgr.y (mgr.24661) 1478 : cluster [DBG] pgmap v2363: 169 pgs: 169 active+clean; 853 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 13 MiB/s rd, 17 MiB/s wr, 168 op/s 2024-09-13T00:26:25.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:25 smithi136 bash[75164]: cluster 2024-09-13T00:26:24.992849+0000 mon.a (mon.0) 2565 : cluster [DBG] osdmap e1326: 8 total, 8 up, 8 in 2024-09-13T00:26:25.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:25 smithi136 bash[76371]: cluster 2024-09-13T00:26:23.901253+0000 mgr.y (mgr.24661) 1478 : cluster [DBG] pgmap v2363: 169 pgs: 169 active+clean; 853 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 13 MiB/s rd, 17 MiB/s wr, 168 op/s 2024-09-13T00:26:25.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:25 smithi136 bash[76371]: cluster 2024-09-13T00:26:24.992849+0000 mon.a (mon.0) 2565 : cluster [DBG] osdmap e1326: 8 total, 8 up, 8 in 2024-09-13T00:26:25.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:25 smithi181 bash[67587]: cluster 2024-09-13T00:26:23.901253+0000 mgr.y (mgr.24661) 1478 : cluster [DBG] pgmap v2363: 169 pgs: 169 active+clean; 853 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 13 MiB/s rd, 17 MiB/s wr, 168 op/s 2024-09-13T00:26:25.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:25 smithi181 bash[67587]: cluster 2024-09-13T00:26:24.992849+0000 mon.a (mon.0) 2565 : cluster [DBG] osdmap e1326: 8 total, 8 up, 8 in 2024-09-13T00:26:26.937 INFO:tasks.workunit.client.0.smithi136.stdout:write: 1 0~8388609 2024-09-13T00:26:27.118 INFO:tasks.workunit.client.0.smithi136.stdout:write: 1 16777218~8388609 2024-09-13T00:26:27.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:26 smithi136 bash[75164]: cluster 2024-09-13T00:26:25.902302+0000 mgr.y (mgr.24661) 1479 : cluster [DBG] pgmap v2366: 169 pgs: 169 active+clean; 865 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 51 KiB/s rd, 2.0 MiB/s wr, 74 op/s 2024-09-13T00:26:27.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:26 smithi136 bash[75164]: cluster 2024-09-13T00:26:25.998055+0000 mon.a (mon.0) 2566 : cluster [DBG] osdmap e1327: 8 total, 8 up, 8 in 2024-09-13T00:26:27.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:26 smithi136 bash[76371]: cluster 2024-09-13T00:26:25.902302+0000 mgr.y (mgr.24661) 1479 : cluster [DBG] pgmap v2366: 169 pgs: 169 active+clean; 865 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 51 KiB/s rd, 2.0 MiB/s wr, 74 op/s 2024-09-13T00:26:27.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:26 smithi136 bash[76371]: cluster 2024-09-13T00:26:25.998055+0000 mon.a (mon.0) 2566 : cluster [DBG] osdmap e1327: 8 total, 8 up, 8 in 2024-09-13T00:26:27.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:26 smithi181 bash[67587]: cluster 2024-09-13T00:26:25.902302+0000 mgr.y (mgr.24661) 1479 : cluster [DBG] pgmap v2366: 169 pgs: 169 active+clean; 865 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 51 KiB/s rd, 2.0 MiB/s wr, 74 op/s 2024-09-13T00:26:27.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:26 smithi181 bash[67587]: cluster 2024-09-13T00:26:25.998055+0000 mon.a (mon.0) 2566 : cluster [DBG] osdmap e1327: 8 total, 8 up, 8 in 2024-09-13T00:26:28.028 INFO:tasks.workunit.client.0.smithi136.stdout:write: 1 0~8388609 2024-09-13T00:26:28.063 INFO:tasks.workunit.client.0.smithi136.stdout:write: 1 16777218~8388609 2024-09-13T00:26:28.189 INFO:tasks.workunit.client.0.smithi136.stdout:migration_abort 2024-09-13T00:26:29.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:29 smithi181 bash[67587]: cluster 2024-09-13T00:26:27.903640+0000 mgr.y (mgr.24661) 1480 : cluster [DBG] pgmap v2368: 169 pgs: 169 active+clean; 873 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 6.1 MiB/s rd, 3.4 MiB/s wr, 213 op/s 2024-09-13T00:26:29.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:29 smithi181 bash[67587]: cluster 2024-09-13T00:26:28.006877+0000 mon.a (mon.0) 2567 : cluster [DBG] osdmap e1328: 8 total, 8 up, 8 in 2024-09-13T00:26:29.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:29 smithi136 bash[75164]: cluster 2024-09-13T00:26:27.903640+0000 mgr.y (mgr.24661) 1480 : cluster [DBG] pgmap v2368: 169 pgs: 169 active+clean; 873 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 6.1 MiB/s rd, 3.4 MiB/s wr, 213 op/s 2024-09-13T00:26:29.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:29 smithi136 bash[75164]: cluster 2024-09-13T00:26:28.006877+0000 mon.a (mon.0) 2567 : cluster [DBG] osdmap e1328: 8 total, 8 up, 8 in 2024-09-13T00:26:29.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:29 smithi136 bash[76371]: cluster 2024-09-13T00:26:27.903640+0000 mgr.y (mgr.24661) 1480 : cluster [DBG] pgmap v2368: 169 pgs: 169 active+clean; 873 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 6.1 MiB/s rd, 3.4 MiB/s wr, 213 op/s 2024-09-13T00:26:29.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:29 smithi136 bash[76371]: cluster 2024-09-13T00:26:28.006877+0000 mon.a (mon.0) 2567 : cluster [DBG] osdmap e1328: 8 total, 8 up, 8 in 2024-09-13T00:26:29.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:26:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:26:29] "GET /metrics HTTP/1.1" 200 38989 "" "Prometheus/2.43.0" 2024-09-13T00:26:30.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:30 smithi181 bash[67587]: cluster 2024-09-13T00:26:29.012310+0000 mon.a (mon.0) 2568 : cluster [DBG] osdmap e1329: 8 total, 8 up, 8 in 2024-09-13T00:26:30.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:30 smithi136 bash[75164]: cluster 2024-09-13T00:26:29.012310+0000 mon.a (mon.0) 2568 : cluster [DBG] osdmap e1329: 8 total, 8 up, 8 in 2024-09-13T00:26:30.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:30 smithi136 bash[76371]: cluster 2024-09-13T00:26:29.012310+0000 mon.a (mon.0) 2568 : cluster [DBG] osdmap e1329: 8 total, 8 up, 8 in 2024-09-13T00:26:31.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:31 smithi181 bash[67587]: cluster 2024-09-13T00:26:29.904501+0000 mgr.y (mgr.24661) 1481 : cluster [DBG] pgmap v2371: 169 pgs: 169 active+clean; 913 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 29 MiB/s rd, 12 MiB/s wr, 226 op/s 2024-09-13T00:26:31.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:31 smithi181 bash[67587]: cluster 2024-09-13T00:26:30.035686+0000 mon.a (mon.0) 2569 : cluster [DBG] osdmap e1330: 8 total, 8 up, 8 in 2024-09-13T00:26:31.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:31 smithi136 bash[75164]: cluster 2024-09-13T00:26:29.904501+0000 mgr.y (mgr.24661) 1481 : cluster [DBG] pgmap v2371: 169 pgs: 169 active+clean; 913 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 29 MiB/s rd, 12 MiB/s wr, 226 op/s 2024-09-13T00:26:31.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:31 smithi136 bash[75164]: cluster 2024-09-13T00:26:30.035686+0000 mon.a (mon.0) 2569 : cluster [DBG] osdmap e1330: 8 total, 8 up, 8 in 2024-09-13T00:26:31.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:31 smithi136 bash[76371]: cluster 2024-09-13T00:26:29.904501+0000 mgr.y (mgr.24661) 1481 : cluster [DBG] pgmap v2371: 169 pgs: 169 active+clean; 913 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 29 MiB/s rd, 12 MiB/s wr, 226 op/s 2024-09-13T00:26:31.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:31 smithi136 bash[76371]: cluster 2024-09-13T00:26:30.035686+0000 mon.a (mon.0) 2569 : cluster [DBG] osdmap e1330: 8 total, 8 up, 8 in 2024-09-13T00:26:32.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:32 smithi181 bash[67587]: cluster 2024-09-13T00:26:31.043428+0000 mon.a (mon.0) 2570 : cluster [DBG] osdmap e1331: 8 total, 8 up, 8 in 2024-09-13T00:26:32.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:32 smithi136 bash[75164]: cluster 2024-09-13T00:26:31.043428+0000 mon.a (mon.0) 2570 : cluster [DBG] osdmap e1331: 8 total, 8 up, 8 in 2024-09-13T00:26:32.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:32 smithi136 bash[76371]: cluster 2024-09-13T00:26:31.043428+0000 mon.a (mon.0) 2570 : cluster [DBG] osdmap e1331: 8 total, 8 up, 8 in 2024-09-13T00:26:32.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:26:32 smithi181 bash[95505]: ts=2024-09-13T00:26:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:26:32.704 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.AbortWithSnapshots (15397 ms) 2024-09-13T00:26:32.705 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.CloneV1Parent 2024-09-13T00:26:32.786 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:26:32.813 INFO:tasks.workunit.client.0.smithi136.stdout:write: A 0~10 2024-09-13T00:26:32.837 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap1 2024-09-13T00:26:33.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:33 smithi181 bash[67587]: cluster 2024-09-13T00:26:31.905696+0000 mgr.y (mgr.24661) 1482 : cluster [DBG] pgmap v2374: 169 pgs: 169 active+clean; 945 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 35 MiB/s rd, 22 MiB/s wr, 292 op/s 2024-09-13T00:26:33.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:33 smithi181 bash[67587]: cluster 2024-09-13T00:26:32.054987+0000 mon.a (mon.0) 2571 : cluster [DBG] osdmap e1332: 8 total, 8 up, 8 in 2024-09-13T00:26:33.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:33 smithi136 bash[75164]: cluster 2024-09-13T00:26:31.905696+0000 mgr.y (mgr.24661) 1482 : cluster [DBG] pgmap v2374: 169 pgs: 169 active+clean; 945 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 35 MiB/s rd, 22 MiB/s wr, 292 op/s 2024-09-13T00:26:33.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:33 smithi136 bash[75164]: cluster 2024-09-13T00:26:32.054987+0000 mon.a (mon.0) 2571 : cluster [DBG] osdmap e1332: 8 total, 8 up, 8 in 2024-09-13T00:26:33.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:33 smithi136 bash[76371]: cluster 2024-09-13T00:26:31.905696+0000 mgr.y (mgr.24661) 1482 : cluster [DBG] pgmap v2374: 169 pgs: 169 active+clean; 945 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 35 MiB/s rd, 22 MiB/s wr, 292 op/s 2024-09-13T00:26:33.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:33 smithi136 bash[76371]: cluster 2024-09-13T00:26:32.054987+0000 mon.a (mon.0) 2571 : cluster [DBG] osdmap e1332: 8 total, 8 up, 8 in 2024-09-13T00:26:34.097 INFO:tasks.workunit.client.0.smithi136.stdout:snap_protect: snap1 2024-09-13T00:26:34.237 INFO:tasks.workunit.client.0.smithi136.stdout:migration_prepare 2024-09-13T00:26:34.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:34 smithi181 bash[67587]: cluster 2024-09-13T00:26:33.062147+0000 mon.a (mon.0) 2572 : cluster [DBG] osdmap e1333: 8 total, 8 up, 8 in 2024-09-13T00:26:34.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:34 smithi136 bash[75164]: cluster 2024-09-13T00:26:33.062147+0000 mon.a (mon.0) 2572 : cluster [DBG] osdmap e1333: 8 total, 8 up, 8 in 2024-09-13T00:26:34.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:34 smithi136 bash[76371]: cluster 2024-09-13T00:26:33.062147+0000 mon.a (mon.0) 2572 : cluster [DBG] osdmap e1333: 8 total, 8 up, 8 in 2024-09-13T00:26:35.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:35 smithi181 bash[67587]: cluster 2024-09-13T00:26:33.906659+0000 mgr.y (mgr.24661) 1483 : cluster [DBG] pgmap v2377: 169 pgs: 169 active+clean; 929 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 9.2 MiB/s rd, 10 MiB/s wr, 245 op/s 2024-09-13T00:26:35.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:35 smithi181 bash[67587]: cluster 2024-09-13T00:26:34.071041+0000 mon.a (mon.0) 2573 : cluster [DBG] osdmap e1334: 8 total, 8 up, 8 in 2024-09-13T00:26:35.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:35 smithi136 bash[75164]: cluster 2024-09-13T00:26:33.906659+0000 mgr.y (mgr.24661) 1483 : cluster [DBG] pgmap v2377: 169 pgs: 169 active+clean; 929 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 9.2 MiB/s rd, 10 MiB/s wr, 245 op/s 2024-09-13T00:26:35.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:35 smithi136 bash[75164]: cluster 2024-09-13T00:26:34.071041+0000 mon.a (mon.0) 2573 : cluster [DBG] osdmap e1334: 8 total, 8 up, 8 in 2024-09-13T00:26:35.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:35 smithi136 bash[76371]: cluster 2024-09-13T00:26:33.906659+0000 mgr.y (mgr.24661) 1483 : cluster [DBG] pgmap v2377: 169 pgs: 169 active+clean; 929 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 9.2 MiB/s rd, 10 MiB/s wr, 245 op/s 2024-09-13T00:26:35.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:35 smithi136 bash[76371]: cluster 2024-09-13T00:26:34.071041+0000 mon.a (mon.0) 2573 : cluster [DBG] osdmap e1334: 8 total, 8 up, 8 in 2024-09-13T00:26:35.572 INFO:tasks.workunit.client.0.smithi136.stdout:migration_execute 2024-09-13T00:26:35.798 INFO:tasks.workunit.client.0.smithi136.stdout:migration_commit 2024-09-13T00:26:36.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:36 smithi181 bash[67587]: cluster 2024-09-13T00:26:35.082077+0000 mon.a (mon.0) 2574 : cluster [DBG] osdmap e1335: 8 total, 8 up, 8 in 2024-09-13T00:26:36.386 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.CloneV1Parent (3681 ms) 2024-09-13T00:26:36.386 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.CloneV2Parent 2024-09-13T00:26:36.473 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:26:36.502 INFO:tasks.workunit.client.0.smithi136.stdout:write: A 0~10 2024-09-13T00:26:36.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:36 smithi136 bash[76371]: cluster 2024-09-13T00:26:35.082077+0000 mon.a (mon.0) 2574 : cluster [DBG] osdmap e1335: 8 total, 8 up, 8 in 2024-09-13T00:26:36.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:36 smithi136 bash[75164]: cluster 2024-09-13T00:26:35.082077+0000 mon.a (mon.0) 2574 : cluster [DBG] osdmap e1335: 8 total, 8 up, 8 in 2024-09-13T00:26:36.526 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap1 2024-09-13T00:26:37.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:37 smithi181 bash[67587]: cluster 2024-09-13T00:26:35.907698+0000 mgr.y (mgr.24661) 1484 : cluster [DBG] pgmap v2380: 169 pgs: 169 active+clean; 913 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 25 MiB/s rd, 1.0 MiB/s wr, 54 op/s 2024-09-13T00:26:37.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:37 smithi181 bash[67587]: cluster 2024-09-13T00:26:36.094036+0000 mon.a (mon.0) 2575 : cluster [DBG] osdmap e1336: 8 total, 8 up, 8 in 2024-09-13T00:26:37.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:37 smithi136 bash[75164]: cluster 2024-09-13T00:26:35.907698+0000 mgr.y (mgr.24661) 1484 : cluster [DBG] pgmap v2380: 169 pgs: 169 active+clean; 913 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 25 MiB/s rd, 1.0 MiB/s wr, 54 op/s 2024-09-13T00:26:37.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:37 smithi136 bash[75164]: cluster 2024-09-13T00:26:36.094036+0000 mon.a (mon.0) 2575 : cluster [DBG] osdmap e1336: 8 total, 8 up, 8 in 2024-09-13T00:26:37.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:37 smithi136 bash[76371]: cluster 2024-09-13T00:26:35.907698+0000 mgr.y (mgr.24661) 1484 : cluster [DBG] pgmap v2380: 169 pgs: 169 active+clean; 913 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 25 MiB/s rd, 1.0 MiB/s wr, 54 op/s 2024-09-13T00:26:37.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:37 smithi136 bash[76371]: cluster 2024-09-13T00:26:36.094036+0000 mon.a (mon.0) 2575 : cluster [DBG] osdmap e1336: 8 total, 8 up, 8 in 2024-09-13T00:26:38.146 INFO:tasks.workunit.client.0.smithi136.stdout:snap_protect: snap1 2024-09-13T00:26:38.301 INFO:tasks.workunit.client.0.smithi136.stdout:migration_prepare 2024-09-13T00:26:38.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:38 smithi136 bash[75164]: cluster 2024-09-13T00:26:37.093144+0000 mon.a (mon.0) 2576 : cluster [DBG] osdmap e1337: 8 total, 8 up, 8 in 2024-09-13T00:26:38.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:38 smithi136 bash[76371]: cluster 2024-09-13T00:26:37.093144+0000 mon.a (mon.0) 2576 : cluster [DBG] osdmap e1337: 8 total, 8 up, 8 in 2024-09-13T00:26:38.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:38 smithi181 bash[67587]: cluster 2024-09-13T00:26:37.093144+0000 mon.a (mon.0) 2576 : cluster [DBG] osdmap e1337: 8 total, 8 up, 8 in 2024-09-13T00:26:39.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:39 smithi136 bash[76371]: cluster 2024-09-13T00:26:37.908714+0000 mgr.y (mgr.24661) 1485 : cluster [DBG] pgmap v2383: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 34 MiB/s rd, 20 KiB/s wr, 401 op/s 2024-09-13T00:26:39.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:39 smithi136 bash[76371]: cluster 2024-09-13T00:26:38.115868+0000 mon.a (mon.0) 2577 : cluster [DBG] osdmap e1338: 8 total, 8 up, 8 in 2024-09-13T00:26:39.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:39 smithi136 bash[76371]: audit 2024-09-13T00:26:38.814128+0000 mon.a (mon.0) 2578 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:26:39.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:39 smithi136 bash[76371]: cluster 2024-09-13T00:26:38.996730+0000 mon.a (mon.0) 2579 : cluster [DBG] osdmap e1339: 8 total, 8 up, 8 in 2024-09-13T00:26:39.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:39 smithi136 bash[75164]: cluster 2024-09-13T00:26:37.908714+0000 mgr.y (mgr.24661) 1485 : cluster [DBG] pgmap v2383: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 34 MiB/s rd, 20 KiB/s wr, 401 op/s 2024-09-13T00:26:39.414 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:39 smithi136 bash[75164]: cluster 2024-09-13T00:26:38.115868+0000 mon.a (mon.0) 2577 : cluster [DBG] osdmap e1338: 8 total, 8 up, 8 in 2024-09-13T00:26:39.414 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:39 smithi136 bash[75164]: audit 2024-09-13T00:26:38.814128+0000 mon.a (mon.0) 2578 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:26:39.414 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:39 smithi136 bash[75164]: cluster 2024-09-13T00:26:38.996730+0000 mon.a (mon.0) 2579 : cluster [DBG] osdmap e1339: 8 total, 8 up, 8 in 2024-09-13T00:26:39.500 INFO:tasks.workunit.client.0.smithi136.stdout:migration_execute 2024-09-13T00:26:39.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:39 smithi181 bash[67587]: cluster 2024-09-13T00:26:37.908714+0000 mgr.y (mgr.24661) 1485 : cluster [DBG] pgmap v2383: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 34 MiB/s rd, 20 KiB/s wr, 401 op/s 2024-09-13T00:26:39.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:39 smithi181 bash[67587]: cluster 2024-09-13T00:26:38.115868+0000 mon.a (mon.0) 2577 : cluster [DBG] osdmap e1338: 8 total, 8 up, 8 in 2024-09-13T00:26:39.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:39 smithi181 bash[67587]: audit 2024-09-13T00:26:38.814128+0000 mon.a (mon.0) 2578 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:26:39.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:39 smithi181 bash[67587]: cluster 2024-09-13T00:26:38.996730+0000 mon.a (mon.0) 2579 : cluster [DBG] osdmap e1339: 8 total, 8 up, 8 in 2024-09-13T00:26:39.722 INFO:tasks.workunit.client.0.smithi136.stdout:migration_commit 2024-09-13T00:26:39.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:26:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:26:39] "GET /metrics HTTP/1.1" 200 38993 "" "Prometheus/2.43.0" 2024-09-13T00:26:40.359 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.CloneV2Parent (3974 ms) 2024-09-13T00:26:40.360 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.CloneV1ParentAbort 2024-09-13T00:26:40.437 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:26:40.467 INFO:tasks.workunit.client.0.smithi136.stdout:write: A 0~10 2024-09-13T00:26:40.492 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap1 2024-09-13T00:26:41.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:41 smithi136 bash[75164]: cluster 2024-09-13T00:26:39.909742+0000 mgr.y (mgr.24661) 1486 : cluster [DBG] pgmap v2386: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 11 MiB/s rd, 25 KiB/s wr, 513 op/s 2024-09-13T00:26:41.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:41 smithi136 bash[75164]: cluster 2024-09-13T00:26:40.126620+0000 mon.a (mon.0) 2580 : cluster [DBG] osdmap e1340: 8 total, 8 up, 8 in 2024-09-13T00:26:41.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:41 smithi136 bash[76371]: cluster 2024-09-13T00:26:39.909742+0000 mgr.y (mgr.24661) 1486 : cluster [DBG] pgmap v2386: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 11 MiB/s rd, 25 KiB/s wr, 513 op/s 2024-09-13T00:26:41.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:41 smithi136 bash[76371]: cluster 2024-09-13T00:26:40.126620+0000 mon.a (mon.0) 2580 : cluster [DBG] osdmap e1340: 8 total, 8 up, 8 in 2024-09-13T00:26:41.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:41 smithi181 bash[67587]: cluster 2024-09-13T00:26:39.909742+0000 mgr.y (mgr.24661) 1486 : cluster [DBG] pgmap v2386: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 11 MiB/s rd, 25 KiB/s wr, 513 op/s 2024-09-13T00:26:41.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:41 smithi181 bash[67587]: cluster 2024-09-13T00:26:40.126620+0000 mon.a (mon.0) 2580 : cluster [DBG] osdmap e1340: 8 total, 8 up, 8 in 2024-09-13T00:26:42.171 INFO:tasks.workunit.client.0.smithi136.stdout:snap_protect: snap1 2024-09-13T00:26:42.335 INFO:tasks.workunit.client.0.smithi136.stdout:migration_prepare 2024-09-13T00:26:42.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:42 smithi136 bash[75164]: cluster 2024-09-13T00:26:41.137353+0000 mon.a (mon.0) 2581 : cluster [DBG] osdmap e1341: 8 total, 8 up, 8 in 2024-09-13T00:26:42.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:42 smithi136 bash[76371]: cluster 2024-09-13T00:26:41.137353+0000 mon.a (mon.0) 2581 : cluster [DBG] osdmap e1341: 8 total, 8 up, 8 in 2024-09-13T00:26:42.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:42 smithi181 bash[67587]: cluster 2024-09-13T00:26:41.137353+0000 mon.a (mon.0) 2581 : cluster [DBG] osdmap e1341: 8 total, 8 up, 8 in 2024-09-13T00:26:42.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:26:42 smithi181 bash[95505]: ts=2024-09-13T00:26:42.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:26:43.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:43 smithi136 bash[76371]: cluster 2024-09-13T00:26:41.910894+0000 mgr.y (mgr.24661) 1487 : cluster [DBG] pgmap v2389: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 386 KiB/s rd, 23 KiB/s wr, 485 op/s 2024-09-13T00:26:43.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:43 smithi136 bash[76371]: cluster 2024-09-13T00:26:42.148626+0000 mon.a (mon.0) 2582 : cluster [DBG] osdmap e1342: 8 total, 8 up, 8 in 2024-09-13T00:26:43.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:43 smithi136 bash[75164]: cluster 2024-09-13T00:26:41.910894+0000 mgr.y (mgr.24661) 1487 : cluster [DBG] pgmap v2389: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 386 KiB/s rd, 23 KiB/s wr, 485 op/s 2024-09-13T00:26:43.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:43 smithi136 bash[75164]: cluster 2024-09-13T00:26:42.148626+0000 mon.a (mon.0) 2582 : cluster [DBG] osdmap e1342: 8 total, 8 up, 8 in 2024-09-13T00:26:43.555 INFO:tasks.workunit.client.0.smithi136.stdout:migration_abort 2024-09-13T00:26:43.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:43 smithi181 bash[67587]: cluster 2024-09-13T00:26:41.910894+0000 mgr.y (mgr.24661) 1487 : cluster [DBG] pgmap v2389: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 386 KiB/s rd, 23 KiB/s wr, 485 op/s 2024-09-13T00:26:43.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:43 smithi181 bash[67587]: cluster 2024-09-13T00:26:42.148626+0000 mon.a (mon.0) 2582 : cluster [DBG] osdmap e1342: 8 total, 8 up, 8 in 2024-09-13T00:26:44.301 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.CloneV1ParentAbort (3941 ms) 2024-09-13T00:26:44.301 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.CloneV2ParentAbort 2024-09-13T00:26:44.386 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:26:44.419 INFO:tasks.workunit.client.0.smithi136.stdout:write: A 0~10 2024-09-13T00:26:44.444 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap1 2024-09-13T00:26:44.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:44 smithi136 bash[75164]: cluster 2024-09-13T00:26:43.155930+0000 mon.a (mon.0) 2583 : cluster [DBG] osdmap e1343: 8 total, 8 up, 8 in 2024-09-13T00:26:44.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:44 smithi136 bash[75164]: cluster 2024-09-13T00:26:43.998578+0000 mon.a (mon.0) 2584 : cluster [DBG] osdmap e1344: 8 total, 8 up, 8 in 2024-09-13T00:26:44.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:44 smithi136 bash[76371]: cluster 2024-09-13T00:26:43.155930+0000 mon.a (mon.0) 2583 : cluster [DBG] osdmap e1343: 8 total, 8 up, 8 in 2024-09-13T00:26:44.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:44 smithi136 bash[76371]: cluster 2024-09-13T00:26:43.998578+0000 mon.a (mon.0) 2584 : cluster [DBG] osdmap e1344: 8 total, 8 up, 8 in 2024-09-13T00:26:44.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:44 smithi181 bash[67587]: cluster 2024-09-13T00:26:43.155930+0000 mon.a (mon.0) 2583 : cluster [DBG] osdmap e1343: 8 total, 8 up, 8 in 2024-09-13T00:26:44.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:44 smithi181 bash[67587]: cluster 2024-09-13T00:26:43.998578+0000 mon.a (mon.0) 2584 : cluster [DBG] osdmap e1344: 8 total, 8 up, 8 in 2024-09-13T00:26:45.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:45 smithi136 bash[75164]: cluster 2024-09-13T00:26:43.911906+0000 mgr.y (mgr.24661) 1488 : cluster [DBG] pgmap v2392: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 293 KiB/s rd, 17 KiB/s wr, 369 op/s 2024-09-13T00:26:45.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:45 smithi136 bash[76371]: cluster 2024-09-13T00:26:43.911906+0000 mgr.y (mgr.24661) 1488 : cluster [DBG] pgmap v2392: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 293 KiB/s rd, 17 KiB/s wr, 369 op/s 2024-09-13T00:26:45.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:45 smithi181 bash[67587]: cluster 2024-09-13T00:26:43.911906+0000 mgr.y (mgr.24661) 1488 : cluster [DBG] pgmap v2392: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 293 KiB/s rd, 17 KiB/s wr, 369 op/s 2024-09-13T00:26:46.227 INFO:tasks.workunit.client.0.smithi136.stdout:snap_protect: snap1 2024-09-13T00:26:46.373 INFO:tasks.workunit.client.0.smithi136.stdout:migration_prepare 2024-09-13T00:26:46.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:46 smithi136 bash[75164]: cluster 2024-09-13T00:26:45.174778+0000 mon.a (mon.0) 2585 : cluster [DBG] osdmap e1345: 8 total, 8 up, 8 in 2024-09-13T00:26:46.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:46 smithi136 bash[76371]: cluster 2024-09-13T00:26:45.174778+0000 mon.a (mon.0) 2585 : cluster [DBG] osdmap e1345: 8 total, 8 up, 8 in 2024-09-13T00:26:46.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:46 smithi181 bash[67587]: cluster 2024-09-13T00:26:45.174778+0000 mon.a (mon.0) 2585 : cluster [DBG] osdmap e1345: 8 total, 8 up, 8 in 2024-09-13T00:26:47.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:47 smithi136 bash[75164]: cluster 2024-09-13T00:26:45.912964+0000 mgr.y (mgr.24661) 1489 : cluster [DBG] pgmap v2395: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 128 KiB/s rd, 13 KiB/s wr, 173 op/s 2024-09-13T00:26:47.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:47 smithi136 bash[75164]: cluster 2024-09-13T00:26:46.201482+0000 mon.a (mon.0) 2586 : cluster [DBG] osdmap e1346: 8 total, 8 up, 8 in 2024-09-13T00:26:47.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:47 smithi136 bash[76371]: cluster 2024-09-13T00:26:45.912964+0000 mgr.y (mgr.24661) 1489 : cluster [DBG] pgmap v2395: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 128 KiB/s rd, 13 KiB/s wr, 173 op/s 2024-09-13T00:26:47.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:47 smithi136 bash[76371]: cluster 2024-09-13T00:26:46.201482+0000 mon.a (mon.0) 2586 : cluster [DBG] osdmap e1346: 8 total, 8 up, 8 in 2024-09-13T00:26:47.576 INFO:tasks.workunit.client.0.smithi136.stdout:migration_abort 2024-09-13T00:26:47.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:47 smithi181 bash[67587]: cluster 2024-09-13T00:26:45.912964+0000 mgr.y (mgr.24661) 1489 : cluster [DBG] pgmap v2395: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 128 KiB/s rd, 13 KiB/s wr, 173 op/s 2024-09-13T00:26:47.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:47 smithi181 bash[67587]: cluster 2024-09-13T00:26:46.201482+0000 mon.a (mon.0) 2586 : cluster [DBG] osdmap e1346: 8 total, 8 up, 8 in 2024-09-13T00:26:48.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:48 smithi136 bash[76371]: cluster 2024-09-13T00:26:47.198707+0000 mon.a (mon.0) 2587 : cluster [DBG] osdmap e1347: 8 total, 8 up, 8 in 2024-09-13T00:26:48.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:48 smithi136 bash[75164]: cluster 2024-09-13T00:26:47.198707+0000 mon.a (mon.0) 2587 : cluster [DBG] osdmap e1347: 8 total, 8 up, 8 in 2024-09-13T00:26:48.512 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.CloneV2ParentAbort (4211 ms) 2024-09-13T00:26:48.512 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.CloneV1ParentAbortFixIncompleteChildReattach 2024-09-13T00:26:48.593 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:26:48.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:48 smithi181 bash[67587]: cluster 2024-09-13T00:26:47.198707+0000 mon.a (mon.0) 2587 : cluster [DBG] osdmap e1347: 8 total, 8 up, 8 in 2024-09-13T00:26:48.622 INFO:tasks.workunit.client.0.smithi136.stdout:write: A 0~10 2024-09-13T00:26:48.646 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap1 2024-09-13T00:26:49.509 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:26:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:26:49] "GET /metrics HTTP/1.1" 200 38993 "" "Prometheus/2.43.0" 2024-09-13T00:26:49.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:49 smithi136 bash[75164]: cluster 2024-09-13T00:26:47.913972+0000 mgr.y (mgr.24661) 1490 : cluster [DBG] pgmap v2398: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 317 KiB/s rd, 22 KiB/s wr, 411 op/s 2024-09-13T00:26:49.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:49 smithi136 bash[75164]: cluster 2024-09-13T00:26:48.220223+0000 mon.a (mon.0) 2588 : cluster [DBG] osdmap e1348: 8 total, 8 up, 8 in 2024-09-13T00:26:49.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:49 smithi136 bash[75164]: cluster 2024-09-13T00:26:49.000859+0000 mon.a (mon.0) 2589 : cluster [DBG] osdmap e1349: 8 total, 8 up, 8 in 2024-09-13T00:26:49.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:49 smithi136 bash[76371]: cluster 2024-09-13T00:26:47.913972+0000 mgr.y (mgr.24661) 1490 : cluster [DBG] pgmap v2398: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 317 KiB/s rd, 22 KiB/s wr, 411 op/s 2024-09-13T00:26:49.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:49 smithi136 bash[76371]: cluster 2024-09-13T00:26:48.220223+0000 mon.a (mon.0) 2588 : cluster [DBG] osdmap e1348: 8 total, 8 up, 8 in 2024-09-13T00:26:49.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:49 smithi136 bash[76371]: cluster 2024-09-13T00:26:49.000859+0000 mon.a (mon.0) 2589 : cluster [DBG] osdmap e1349: 8 total, 8 up, 8 in 2024-09-13T00:26:49.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:49 smithi181 bash[67587]: cluster 2024-09-13T00:26:47.913972+0000 mgr.y (mgr.24661) 1490 : cluster [DBG] pgmap v2398: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 317 KiB/s rd, 22 KiB/s wr, 411 op/s 2024-09-13T00:26:49.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:49 smithi181 bash[67587]: cluster 2024-09-13T00:26:48.220223+0000 mon.a (mon.0) 2588 : cluster [DBG] osdmap e1348: 8 total, 8 up, 8 in 2024-09-13T00:26:49.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:49 smithi181 bash[67587]: cluster 2024-09-13T00:26:49.000859+0000 mon.a (mon.0) 2589 : cluster [DBG] osdmap e1349: 8 total, 8 up, 8 in 2024-09-13T00:26:50.027 INFO:tasks.workunit.client.0.smithi136.stdout:snap_protect: snap1 2024-09-13T00:26:50.177 INFO:tasks.workunit.client.0.smithi136.stdout:migration_prepare 2024-09-13T00:26:51.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:51 smithi136 bash[76371]: cluster 2024-09-13T00:26:49.914888+0000 mgr.y (mgr.24661) 1491 : cluster [DBG] pgmap v2401: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 312 KiB/s rd, 21 KiB/s wr, 399 op/s 2024-09-13T00:26:51.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:51 smithi136 bash[76371]: cluster 2024-09-13T00:26:50.005003+0000 mon.a (mon.0) 2590 : cluster [DBG] osdmap e1350: 8 total, 8 up, 8 in 2024-09-13T00:26:51.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:50 smithi136 bash[75164]: cluster 2024-09-13T00:26:49.914888+0000 mgr.y (mgr.24661) 1491 : cluster [DBG] pgmap v2401: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 312 KiB/s rd, 21 KiB/s wr, 399 op/s 2024-09-13T00:26:51.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:50 smithi136 bash[75164]: cluster 2024-09-13T00:26:50.005003+0000 mon.a (mon.0) 2590 : cluster [DBG] osdmap e1350: 8 total, 8 up, 8 in 2024-09-13T00:26:51.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:50 smithi181 bash[67587]: cluster 2024-09-13T00:26:49.914888+0000 mgr.y (mgr.24661) 1491 : cluster [DBG] pgmap v2401: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 312 KiB/s rd, 21 KiB/s wr, 399 op/s 2024-09-13T00:26:51.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:50 smithi181 bash[67587]: cluster 2024-09-13T00:26:50.005003+0000 mon.a (mon.0) 2590 : cluster [DBG] osdmap e1350: 8 total, 8 up, 8 in 2024-09-13T00:26:51.671 INFO:tasks.workunit.client.0.smithi136.stdout:migration_abort 2024-09-13T00:26:52.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:52 smithi136 bash[75164]: cluster 2024-09-13T00:26:51.227139+0000 mon.a (mon.0) 2591 : cluster [DBG] osdmap e1351: 8 total, 8 up, 8 in 2024-09-13T00:26:52.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:52 smithi136 bash[76371]: cluster 2024-09-13T00:26:51.227139+0000 mon.a (mon.0) 2591 : cluster [DBG] osdmap e1351: 8 total, 8 up, 8 in 2024-09-13T00:26:52.525 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.CloneV1ParentAbortFixIncompleteChildReattach (4013 ms) 2024-09-13T00:26:52.525 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.CloneV1ParentAbortFixParentReattach 2024-09-13T00:26:52.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:52 smithi181 bash[67587]: cluster 2024-09-13T00:26:51.227139+0000 mon.a (mon.0) 2591 : cluster [DBG] osdmap e1351: 8 total, 8 up, 8 in 2024-09-13T00:26:52.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:26:52 smithi181 bash[95505]: ts=2024-09-13T00:26:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:26:52.610 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:26:52.642 INFO:tasks.workunit.client.0.smithi136.stdout:write: A 0~10 2024-09-13T00:26:52.667 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap1 2024-09-13T00:26:53.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:53 smithi136 bash[76371]: cluster 2024-09-13T00:26:51.915983+0000 mgr.y (mgr.24661) 1492 : cluster [DBG] pgmap v2404: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 230 KiB/s rd, 18 KiB/s wr, 300 op/s 2024-09-13T00:26:53.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:53 smithi136 bash[76371]: cluster 2024-09-13T00:26:52.236434+0000 mon.a (mon.0) 2592 : cluster [DBG] osdmap e1352: 8 total, 8 up, 8 in 2024-09-13T00:26:53.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:53 smithi136 bash[75164]: cluster 2024-09-13T00:26:51.915983+0000 mgr.y (mgr.24661) 1492 : cluster [DBG] pgmap v2404: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 230 KiB/s rd, 18 KiB/s wr, 300 op/s 2024-09-13T00:26:53.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:53 smithi136 bash[75164]: cluster 2024-09-13T00:26:52.236434+0000 mon.a (mon.0) 2592 : cluster [DBG] osdmap e1352: 8 total, 8 up, 8 in 2024-09-13T00:26:53.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:53 smithi181 bash[67587]: cluster 2024-09-13T00:26:51.915983+0000 mgr.y (mgr.24661) 1492 : cluster [DBG] pgmap v2404: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 230 KiB/s rd, 18 KiB/s wr, 300 op/s 2024-09-13T00:26:53.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:53 smithi181 bash[67587]: cluster 2024-09-13T00:26:52.236434+0000 mon.a (mon.0) 2592 : cluster [DBG] osdmap e1352: 8 total, 8 up, 8 in 2024-09-13T00:26:54.035 INFO:tasks.workunit.client.0.smithi136.stdout:snap_protect: snap1 2024-09-13T00:26:54.191 INFO:tasks.workunit.client.0.smithi136.stdout:migration_prepare 2024-09-13T00:26:54.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:54 smithi136 bash[75164]: cluster 2024-09-13T00:26:53.245727+0000 mon.a (mon.0) 2593 : cluster [DBG] osdmap e1353: 8 total, 8 up, 8 in 2024-09-13T00:26:54.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:54 smithi136 bash[75164]: audit 2024-09-13T00:26:53.814553+0000 mon.a (mon.0) 2594 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:26:54.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:54 smithi136 bash[75164]: cluster 2024-09-13T00:26:54.001991+0000 mon.a (mon.0) 2595 : cluster [DBG] osdmap e1354: 8 total, 8 up, 8 in 2024-09-13T00:26:54.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:54 smithi136 bash[76371]: cluster 2024-09-13T00:26:53.245727+0000 mon.a (mon.0) 2593 : cluster [DBG] osdmap e1353: 8 total, 8 up, 8 in 2024-09-13T00:26:54.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:54 smithi136 bash[76371]: audit 2024-09-13T00:26:53.814553+0000 mon.a (mon.0) 2594 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:26:54.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:54 smithi136 bash[76371]: cluster 2024-09-13T00:26:54.001991+0000 mon.a (mon.0) 2595 : cluster [DBG] osdmap e1354: 8 total, 8 up, 8 in 2024-09-13T00:26:54.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:54 smithi181 bash[67587]: cluster 2024-09-13T00:26:53.245727+0000 mon.a (mon.0) 2593 : cluster [DBG] osdmap e1353: 8 total, 8 up, 8 in 2024-09-13T00:26:54.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:54 smithi181 bash[67587]: audit 2024-09-13T00:26:53.814553+0000 mon.a (mon.0) 2594 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:26:54.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:54 smithi181 bash[67587]: cluster 2024-09-13T00:26:54.001991+0000 mon.a (mon.0) 2595 : cluster [DBG] osdmap e1354: 8 total, 8 up, 8 in 2024-09-13T00:26:55.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:55 smithi181 bash[67587]: cluster 2024-09-13T00:26:53.916981+0000 mgr.y (mgr.24661) 1493 : cluster [DBG] pgmap v2407: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 139 KiB/s rd, 12 KiB/s wr, 188 op/s 2024-09-13T00:26:55.721 INFO:tasks.workunit.client.0.smithi136.stdout:migration_abort 2024-09-13T00:26:55.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:55 smithi136 bash[75164]: cluster 2024-09-13T00:26:53.916981+0000 mgr.y (mgr.24661) 1493 : cluster [DBG] pgmap v2407: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 139 KiB/s rd, 12 KiB/s wr, 188 op/s 2024-09-13T00:26:55.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:55 smithi136 bash[76371]: cluster 2024-09-13T00:26:53.916981+0000 mgr.y (mgr.24661) 1493 : cluster [DBG] pgmap v2407: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 139 KiB/s rd, 12 KiB/s wr, 188 op/s 2024-09-13T00:26:56.582 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.CloneV1ParentAbortFixParentReattach (4057 ms) 2024-09-13T00:26:56.582 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.CloneV1ParentAbortRelinkNotNeeded 2024-09-13T00:26:56.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:56 smithi181 bash[67587]: cluster 2024-09-13T00:26:55.260160+0000 mon.a (mon.0) 2596 : cluster [DBG] osdmap e1355: 8 total, 8 up, 8 in 2024-09-13T00:26:56.664 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:26:56.697 INFO:tasks.workunit.client.0.smithi136.stdout:write: A 0~10 2024-09-13T00:26:56.722 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap1 2024-09-13T00:26:56.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:56 smithi136 bash[75164]: cluster 2024-09-13T00:26:55.260160+0000 mon.a (mon.0) 2596 : cluster [DBG] osdmap e1355: 8 total, 8 up, 8 in 2024-09-13T00:26:56.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:56 smithi136 bash[76371]: cluster 2024-09-13T00:26:55.260160+0000 mon.a (mon.0) 2596 : cluster [DBG] osdmap e1355: 8 total, 8 up, 8 in 2024-09-13T00:26:57.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:57 smithi181 bash[67587]: cluster 2024-09-13T00:26:55.918181+0000 mgr.y (mgr.24661) 1494 : cluster [DBG] pgmap v2410: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 201 KiB/s rd, 11 KiB/s wr, 249 op/s 2024-09-13T00:26:57.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:57 smithi181 bash[67587]: cluster 2024-09-13T00:26:56.285575+0000 mon.a (mon.0) 2597 : cluster [DBG] osdmap e1356: 8 total, 8 up, 8 in 2024-09-13T00:26:57.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:57 smithi136 bash[75164]: cluster 2024-09-13T00:26:55.918181+0000 mgr.y (mgr.24661) 1494 : cluster [DBG] pgmap v2410: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 201 KiB/s rd, 11 KiB/s wr, 249 op/s 2024-09-13T00:26:57.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:57 smithi136 bash[75164]: cluster 2024-09-13T00:26:56.285575+0000 mon.a (mon.0) 2597 : cluster [DBG] osdmap e1356: 8 total, 8 up, 8 in 2024-09-13T00:26:57.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:57 smithi136 bash[76371]: cluster 2024-09-13T00:26:55.918181+0000 mgr.y (mgr.24661) 1494 : cluster [DBG] pgmap v2410: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 201 KiB/s rd, 11 KiB/s wr, 249 op/s 2024-09-13T00:26:57.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:57 smithi136 bash[76371]: cluster 2024-09-13T00:26:56.285575+0000 mon.a (mon.0) 2597 : cluster [DBG] osdmap e1356: 8 total, 8 up, 8 in 2024-09-13T00:26:58.323 INFO:tasks.workunit.client.0.smithi136.stdout:snap_protect: snap1 2024-09-13T00:26:58.479 INFO:tasks.workunit.client.0.smithi136.stdout:migration_prepare 2024-09-13T00:26:58.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:58 smithi181 bash[67587]: cluster 2024-09-13T00:26:57.290713+0000 mon.a (mon.0) 2598 : cluster [DBG] osdmap e1357: 8 total, 8 up, 8 in 2024-09-13T00:26:58.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:58 smithi136 bash[75164]: cluster 2024-09-13T00:26:57.290713+0000 mon.a (mon.0) 2598 : cluster [DBG] osdmap e1357: 8 total, 8 up, 8 in 2024-09-13T00:26:58.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:58 smithi136 bash[76371]: cluster 2024-09-13T00:26:57.290713+0000 mon.a (mon.0) 2598 : cluster [DBG] osdmap e1357: 8 total, 8 up, 8 in 2024-09-13T00:26:59.458 INFO:tasks.workunit.client.0.smithi136.stdout:migration_abort 2024-09-13T00:26:59.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:59 smithi181 bash[67587]: cluster 2024-09-13T00:26:57.919179+0000 mgr.y (mgr.24661) 1495 : cluster [DBG] pgmap v2413: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 434 KiB/s rd, 21 KiB/s wr, 541 op/s 2024-09-13T00:26:59.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:59 smithi181 bash[67587]: cluster 2024-09-13T00:26:58.302954+0000 mon.a (mon.0) 2599 : cluster [DBG] osdmap e1358: 8 total, 8 up, 8 in 2024-09-13T00:26:59.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:26:59 smithi181 bash[67587]: cluster 2024-09-13T00:26:59.009591+0000 mon.a (mon.0) 2600 : cluster [DBG] osdmap e1359: 8 total, 8 up, 8 in 2024-09-13T00:26:59.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:26:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:26:59] "GET /metrics HTTP/1.1" 200 38991 "" "Prometheus/2.43.0" 2024-09-13T00:26:59.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:59 smithi136 bash[76371]: cluster 2024-09-13T00:26:57.919179+0000 mgr.y (mgr.24661) 1495 : cluster [DBG] pgmap v2413: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 434 KiB/s rd, 21 KiB/s wr, 541 op/s 2024-09-13T00:26:59.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:59 smithi136 bash[76371]: cluster 2024-09-13T00:26:58.302954+0000 mon.a (mon.0) 2599 : cluster [DBG] osdmap e1358: 8 total, 8 up, 8 in 2024-09-13T00:26:59.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:26:59 smithi136 bash[76371]: cluster 2024-09-13T00:26:59.009591+0000 mon.a (mon.0) 2600 : cluster [DBG] osdmap e1359: 8 total, 8 up, 8 in 2024-09-13T00:26:59.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:59 smithi136 bash[75164]: cluster 2024-09-13T00:26:57.919179+0000 mgr.y (mgr.24661) 1495 : cluster [DBG] pgmap v2413: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 434 KiB/s rd, 21 KiB/s wr, 541 op/s 2024-09-13T00:26:59.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:59 smithi136 bash[75164]: cluster 2024-09-13T00:26:58.302954+0000 mon.a (mon.0) 2599 : cluster [DBG] osdmap e1358: 8 total, 8 up, 8 in 2024-09-13T00:26:59.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:26:59 smithi136 bash[75164]: cluster 2024-09-13T00:26:59.009591+0000 mon.a (mon.0) 2600 : cluster [DBG] osdmap e1359: 8 total, 8 up, 8 in 2024-09-13T00:27:00.601 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.CloneV1ParentAbortRelinkNotNeeded (4019 ms) 2024-09-13T00:27:00.601 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.CloneV2ParentAbortFixIncompleteChildReattach 2024-09-13T00:27:00.687 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:27:00.716 INFO:tasks.workunit.client.0.smithi136.stdout:write: A 0~10 2024-09-13T00:27:00.736 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap1 2024-09-13T00:27:01.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:01 smithi181 bash[67587]: cluster 2024-09-13T00:26:59.920077+0000 mgr.y (mgr.24661) 1496 : cluster [DBG] pgmap v2416: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 391 KiB/s rd, 23 KiB/s wr, 499 op/s 2024-09-13T00:27:01.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:01 smithi181 bash[67587]: cluster 2024-09-13T00:27:00.310513+0000 mon.a (mon.0) 2601 : cluster [DBG] osdmap e1360: 8 total, 8 up, 8 in 2024-09-13T00:27:01.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:01 smithi136 bash[76371]: cluster 2024-09-13T00:26:59.920077+0000 mgr.y (mgr.24661) 1496 : cluster [DBG] pgmap v2416: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 391 KiB/s rd, 23 KiB/s wr, 499 op/s 2024-09-13T00:27:01.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:01 smithi136 bash[76371]: cluster 2024-09-13T00:27:00.310513+0000 mon.a (mon.0) 2601 : cluster [DBG] osdmap e1360: 8 total, 8 up, 8 in 2024-09-13T00:27:01.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:01 smithi136 bash[75164]: cluster 2024-09-13T00:26:59.920077+0000 mgr.y (mgr.24661) 1496 : cluster [DBG] pgmap v2416: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 391 KiB/s rd, 23 KiB/s wr, 499 op/s 2024-09-13T00:27:01.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:01 smithi136 bash[75164]: cluster 2024-09-13T00:27:00.310513+0000 mon.a (mon.0) 2601 : cluster [DBG] osdmap e1360: 8 total, 8 up, 8 in 2024-09-13T00:27:02.361 INFO:tasks.workunit.client.0.smithi136.stdout:snap_protect: snap1 2024-09-13T00:27:02.507 INFO:tasks.workunit.client.0.smithi136.stdout:migration_prepare 2024-09-13T00:27:02.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:02 smithi181 bash[67587]: cluster 2024-09-13T00:27:01.319656+0000 mon.a (mon.0) 2602 : cluster [DBG] osdmap e1361: 8 total, 8 up, 8 in 2024-09-13T00:27:02.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:27:02 smithi181 bash[95505]: ts=2024-09-13T00:27:02.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:27:02.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:02 smithi136 bash[75164]: cluster 2024-09-13T00:27:01.319656+0000 mon.a (mon.0) 2602 : cluster [DBG] osdmap e1361: 8 total, 8 up, 8 in 2024-09-13T00:27:02.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:02 smithi136 bash[76371]: cluster 2024-09-13T00:27:01.319656+0000 mon.a (mon.0) 2602 : cluster [DBG] osdmap e1361: 8 total, 8 up, 8 in 2024-09-13T00:27:03.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:03 smithi181 bash[67587]: cluster 2024-09-13T00:27:01.921192+0000 mgr.y (mgr.24661) 1497 : cluster [DBG] pgmap v2419: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 215 KiB/s rd, 16 KiB/s wr, 280 op/s 2024-09-13T00:27:03.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:03 smithi181 bash[67587]: cluster 2024-09-13T00:27:02.334000+0000 mon.a (mon.0) 2603 : cluster [DBG] osdmap e1362: 8 total, 8 up, 8 in 2024-09-13T00:27:03.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:03 smithi136 bash[75164]: cluster 2024-09-13T00:27:01.921192+0000 mgr.y (mgr.24661) 1497 : cluster [DBG] pgmap v2419: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 215 KiB/s rd, 16 KiB/s wr, 280 op/s 2024-09-13T00:27:03.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:03 smithi136 bash[75164]: cluster 2024-09-13T00:27:02.334000+0000 mon.a (mon.0) 2603 : cluster [DBG] osdmap e1362: 8 total, 8 up, 8 in 2024-09-13T00:27:03.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:03 smithi136 bash[76371]: cluster 2024-09-13T00:27:01.921192+0000 mgr.y (mgr.24661) 1497 : cluster [DBG] pgmap v2419: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 215 KiB/s rd, 16 KiB/s wr, 280 op/s 2024-09-13T00:27:03.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:03 smithi136 bash[76371]: cluster 2024-09-13T00:27:02.334000+0000 mon.a (mon.0) 2603 : cluster [DBG] osdmap e1362: 8 total, 8 up, 8 in 2024-09-13T00:27:03.765 INFO:tasks.workunit.client.0.smithi136.stdout:migration_abort 2024-09-13T00:27:04.313 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.CloneV2ParentAbortFixIncompleteChildReattach (3712 ms) 2024-09-13T00:27:04.313 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.CloneV2ParentAbortFixParentReattach 2024-09-13T00:27:04.401 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:27:04.431 INFO:tasks.workunit.client.0.smithi136.stdout:write: A 0~10 2024-09-13T00:27:04.456 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap1 2024-09-13T00:27:04.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:04 smithi181 bash[67587]: cluster 2024-09-13T00:27:03.340810+0000 mon.a (mon.0) 2604 : cluster [DBG] osdmap e1363: 8 total, 8 up, 8 in 2024-09-13T00:27:04.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:04 smithi181 bash[67587]: cluster 2024-09-13T00:27:03.922347+0000 mgr.y (mgr.24661) 1498 : cluster [DBG] pgmap v2422: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 219 KiB/s rd, 14 KiB/s wr, 282 op/s 2024-09-13T00:27:04.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:04 smithi181 bash[67587]: cluster 2024-09-13T00:27:04.008646+0000 mon.a (mon.0) 2605 : cluster [DBG] osdmap e1364: 8 total, 8 up, 8 in 2024-09-13T00:27:04.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:04 smithi136 bash[76371]: cluster 2024-09-13T00:27:03.340810+0000 mon.a (mon.0) 2604 : cluster [DBG] osdmap e1363: 8 total, 8 up, 8 in 2024-09-13T00:27:04.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:04 smithi136 bash[76371]: cluster 2024-09-13T00:27:03.922347+0000 mgr.y (mgr.24661) 1498 : cluster [DBG] pgmap v2422: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 219 KiB/s rd, 14 KiB/s wr, 282 op/s 2024-09-13T00:27:04.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:04 smithi136 bash[76371]: cluster 2024-09-13T00:27:04.008646+0000 mon.a (mon.0) 2605 : cluster [DBG] osdmap e1364: 8 total, 8 up, 8 in 2024-09-13T00:27:04.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:04 smithi136 bash[75164]: cluster 2024-09-13T00:27:03.340810+0000 mon.a (mon.0) 2604 : cluster [DBG] osdmap e1363: 8 total, 8 up, 8 in 2024-09-13T00:27:04.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:04 smithi136 bash[75164]: cluster 2024-09-13T00:27:03.922347+0000 mgr.y (mgr.24661) 1498 : cluster [DBG] pgmap v2422: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 219 KiB/s rd, 14 KiB/s wr, 282 op/s 2024-09-13T00:27:04.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:04 smithi136 bash[75164]: cluster 2024-09-13T00:27:04.008646+0000 mon.a (mon.0) 2605 : cluster [DBG] osdmap e1364: 8 total, 8 up, 8 in 2024-09-13T00:27:06.405 INFO:tasks.workunit.client.0.smithi136.stdout:snap_protect: snap1 2024-09-13T00:27:06.533 INFO:tasks.workunit.client.0.smithi136.stdout:migration_prepare 2024-09-13T00:27:06.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:06 smithi136 bash[76371]: cluster 2024-09-13T00:27:05.354664+0000 mon.a (mon.0) 2606 : cluster [DBG] osdmap e1365: 8 total, 8 up, 8 in 2024-09-13T00:27:06.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:06 smithi136 bash[76371]: cluster 2024-09-13T00:27:05.923405+0000 mgr.y (mgr.24661) 1499 : cluster [DBG] pgmap v2425: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 207 KiB/s rd, 14 KiB/s wr, 261 op/s 2024-09-13T00:27:06.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:06 smithi136 bash[75164]: cluster 2024-09-13T00:27:05.354664+0000 mon.a (mon.0) 2606 : cluster [DBG] osdmap e1365: 8 total, 8 up, 8 in 2024-09-13T00:27:06.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:06 smithi136 bash[75164]: cluster 2024-09-13T00:27:05.923405+0000 mgr.y (mgr.24661) 1499 : cluster [DBG] pgmap v2425: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 207 KiB/s rd, 14 KiB/s wr, 261 op/s 2024-09-13T00:27:06.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:06 smithi181 bash[67587]: cluster 2024-09-13T00:27:05.354664+0000 mon.a (mon.0) 2606 : cluster [DBG] osdmap e1365: 8 total, 8 up, 8 in 2024-09-13T00:27:06.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:06 smithi181 bash[67587]: cluster 2024-09-13T00:27:05.923405+0000 mgr.y (mgr.24661) 1499 : cluster [DBG] pgmap v2425: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 207 KiB/s rd, 14 KiB/s wr, 261 op/s 2024-09-13T00:27:07.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:07 smithi136 bash[75164]: cluster 2024-09-13T00:27:06.378863+0000 mon.a (mon.0) 2607 : cluster [DBG] osdmap e1366: 8 total, 8 up, 8 in 2024-09-13T00:27:07.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:07 smithi136 bash[76371]: cluster 2024-09-13T00:27:06.378863+0000 mon.a (mon.0) 2607 : cluster [DBG] osdmap e1366: 8 total, 8 up, 8 in 2024-09-13T00:27:07.795 INFO:tasks.workunit.client.0.smithi136.stdout:migration_abort 2024-09-13T00:27:07.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:07 smithi181 bash[67587]: cluster 2024-09-13T00:27:06.378863+0000 mon.a (mon.0) 2607 : cluster [DBG] osdmap e1366: 8 total, 8 up, 8 in 2024-09-13T00:27:08.676 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.CloneV2ParentAbortFixParentReattach (4363 ms) 2024-09-13T00:27:08.676 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.CloneV2ParentAbortRelinkNotNeeded 2024-09-13T00:27:08.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:08 smithi136 bash[76371]: cluster 2024-09-13T00:27:07.371997+0000 mon.a (mon.0) 2608 : cluster [DBG] osdmap e1367: 8 total, 8 up, 8 in 2024-09-13T00:27:08.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:08 smithi136 bash[76371]: cluster 2024-09-13T00:27:07.924359+0000 mgr.y (mgr.24661) 1500 : cluster [DBG] pgmap v2428: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 179 KiB/s rd, 19 KiB/s wr, 242 op/s 2024-09-13T00:27:08.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:08 smithi136 bash[75164]: cluster 2024-09-13T00:27:07.371997+0000 mon.a (mon.0) 2608 : cluster [DBG] osdmap e1367: 8 total, 8 up, 8 in 2024-09-13T00:27:08.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:08 smithi136 bash[75164]: cluster 2024-09-13T00:27:07.924359+0000 mgr.y (mgr.24661) 1500 : cluster [DBG] pgmap v2428: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 179 KiB/s rd, 19 KiB/s wr, 242 op/s 2024-09-13T00:27:08.763 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:27:08.794 INFO:tasks.workunit.client.0.smithi136.stdout:write: A 0~10 2024-09-13T00:27:08.819 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap1 2024-09-13T00:27:08.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:08 smithi181 bash[67587]: cluster 2024-09-13T00:27:07.371997+0000 mon.a (mon.0) 2608 : cluster [DBG] osdmap e1367: 8 total, 8 up, 8 in 2024-09-13T00:27:08.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:08 smithi181 bash[67587]: cluster 2024-09-13T00:27:07.924359+0000 mgr.y (mgr.24661) 1500 : cluster [DBG] pgmap v2428: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 179 KiB/s rd, 19 KiB/s wr, 242 op/s 2024-09-13T00:27:09.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:27:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:27:09] "GET /metrics HTTP/1.1" 200 38991 "" "Prometheus/2.43.0" 2024-09-13T00:27:09.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:09 smithi136 bash[75164]: cluster 2024-09-13T00:27:08.383190+0000 mon.a (mon.0) 2609 : cluster [DBG] osdmap e1368: 8 total, 8 up, 8 in 2024-09-13T00:27:09.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:09 smithi136 bash[75164]: audit 2024-09-13T00:27:08.814819+0000 mon.a (mon.0) 2610 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:27:09.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:09 smithi136 bash[75164]: cluster 2024-09-13T00:27:09.009539+0000 mon.a (mon.0) 2611 : cluster [DBG] osdmap e1369: 8 total, 8 up, 8 in 2024-09-13T00:27:09.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:09 smithi136 bash[76371]: cluster 2024-09-13T00:27:08.383190+0000 mon.a (mon.0) 2609 : cluster [DBG] osdmap e1368: 8 total, 8 up, 8 in 2024-09-13T00:27:09.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:09 smithi136 bash[76371]: audit 2024-09-13T00:27:08.814819+0000 mon.a (mon.0) 2610 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:27:09.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:09 smithi136 bash[76371]: cluster 2024-09-13T00:27:09.009539+0000 mon.a (mon.0) 2611 : cluster [DBG] osdmap e1369: 8 total, 8 up, 8 in 2024-09-13T00:27:09.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:09 smithi181 bash[67587]: cluster 2024-09-13T00:27:08.383190+0000 mon.a (mon.0) 2609 : cluster [DBG] osdmap e1368: 8 total, 8 up, 8 in 2024-09-13T00:27:09.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:09 smithi181 bash[67587]: audit 2024-09-13T00:27:08.814819+0000 mon.a (mon.0) 2610 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:27:09.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:09 smithi181 bash[67587]: cluster 2024-09-13T00:27:09.009539+0000 mon.a (mon.0) 2611 : cluster [DBG] osdmap e1369: 8 total, 8 up, 8 in 2024-09-13T00:27:10.035 INFO:tasks.workunit.client.0.smithi136.stdout:snap_protect: snap1 2024-09-13T00:27:10.187 INFO:tasks.workunit.client.0.smithi136.stdout:migration_prepare 2024-09-13T00:27:11.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:11 smithi181 bash[67587]: cluster 2024-09-13T00:27:09.925269+0000 mgr.y (mgr.24661) 1501 : cluster [DBG] pgmap v2431: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 204 KiB/s rd, 20 KiB/s wr, 275 op/s 2024-09-13T00:27:11.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:11 smithi181 bash[67587]: cluster 2024-09-13T00:27:10.016219+0000 mon.a (mon.0) 2612 : cluster [DBG] osdmap e1370: 8 total, 8 up, 8 in 2024-09-13T00:27:11.446 INFO:tasks.workunit.client.0.smithi136.stdout:migration_abort 2024-09-13T00:27:11.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:11 smithi136 bash[75164]: cluster 2024-09-13T00:27:09.925269+0000 mgr.y (mgr.24661) 1501 : cluster [DBG] pgmap v2431: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 204 KiB/s rd, 20 KiB/s wr, 275 op/s 2024-09-13T00:27:11.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:11 smithi136 bash[75164]: cluster 2024-09-13T00:27:10.016219+0000 mon.a (mon.0) 2612 : cluster [DBG] osdmap e1370: 8 total, 8 up, 8 in 2024-09-13T00:27:11.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:11 smithi136 bash[76371]: cluster 2024-09-13T00:27:09.925269+0000 mgr.y (mgr.24661) 1501 : cluster [DBG] pgmap v2431: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 204 KiB/s rd, 20 KiB/s wr, 275 op/s 2024-09-13T00:27:11.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:11 smithi136 bash[76371]: cluster 2024-09-13T00:27:10.016219+0000 mon.a (mon.0) 2612 : cluster [DBG] osdmap e1370: 8 total, 8 up, 8 in 2024-09-13T00:27:12.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:12 smithi181 bash[67587]: cluster 2024-09-13T00:27:11.014169+0000 mon.a (mon.0) 2613 : cluster [DBG] osdmap e1371: 8 total, 8 up, 8 in 2024-09-13T00:27:12.327 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.CloneV2ParentAbortRelinkNotNeeded (3651 ms) 2024-09-13T00:27:12.327 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.StressNoMigrate 2024-09-13T00:27:12.416 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:27:12.447 INFO:tasks.workunit.client.0.smithi136.stdout:write: A 14077678~2863184 2024-09-13T00:27:12.505 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 12011517~2279598 2024-09-13T00:27:12.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:12 smithi136 bash[75164]: cluster 2024-09-13T00:27:11.014169+0000 mon.a (mon.0) 2613 : cluster [DBG] osdmap e1371: 8 total, 8 up, 8 in 2024-09-13T00:27:12.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:12 smithi136 bash[76371]: cluster 2024-09-13T00:27:11.014169+0000 mon.a (mon.0) 2613 : cluster [DBG] osdmap e1371: 8 total, 8 up, 8 in 2024-09-13T00:27:12.510 INFO:tasks.workunit.client.0.smithi136.stdout:write: A 33003494~3464130 2024-09-13T00:27:12.554 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 53046868~6439942 2024-09-13T00:27:12.554 INFO:tasks.workunit.client.0.smithi136.stdout:write: A 44855126~2934581 2024-09-13T00:27:12.590 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 8982574~4185311 2024-09-13T00:27:12.595 INFO:tasks.workunit.client.0.smithi136.stdout:write: A 55758395~8142118 2024-09-13T00:27:12.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:27:12 smithi181 bash[95505]: ts=2024-09-13T00:27:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:27:12.672 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 53059617~154497 2024-09-13T00:27:12.673 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap0 2024-09-13T00:27:13.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:13 smithi181 bash[67587]: cluster 2024-09-13T00:27:11.926498+0000 mgr.y (mgr.24661) 1502 : cluster [DBG] pgmap v2434: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 400 KiB/s rd, 29 KiB/s wr, 507 op/s 2024-09-13T00:27:13.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:13 smithi181 bash[67587]: cluster 2024-09-13T00:27:12.039221+0000 mon.a (mon.0) 2614 : cluster [DBG] osdmap e1372: 8 total, 8 up, 8 in 2024-09-13T00:27:13.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:13 smithi136 bash[75164]: cluster 2024-09-13T00:27:11.926498+0000 mgr.y (mgr.24661) 1502 : cluster [DBG] pgmap v2434: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 400 KiB/s rd, 29 KiB/s wr, 507 op/s 2024-09-13T00:27:13.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:13 smithi136 bash[75164]: cluster 2024-09-13T00:27:12.039221+0000 mon.a (mon.0) 2614 : cluster [DBG] osdmap e1372: 8 total, 8 up, 8 in 2024-09-13T00:27:13.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:13 smithi136 bash[76371]: cluster 2024-09-13T00:27:11.926498+0000 mgr.y (mgr.24661) 1502 : cluster [DBG] pgmap v2434: 169 pgs: 169 active+clean; 905 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 400 KiB/s rd, 29 KiB/s wr, 507 op/s 2024-09-13T00:27:13.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:13 smithi136 bash[76371]: cluster 2024-09-13T00:27:12.039221+0000 mon.a (mon.0) 2614 : cluster [DBG] osdmap e1372: 8 total, 8 up, 8 in 2024-09-13T00:27:14.042 INFO:tasks.workunit.client.0.smithi136.stdout:write: B 14493002~1146446 2024-09-13T00:27:14.063 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 51105033~1698853 2024-09-13T00:27:14.063 INFO:tasks.workunit.client.0.smithi136.stdout:write: B 62240144~6780979 2024-09-13T00:27:14.079 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 70905129~7907418 2024-09-13T00:27:14.079 INFO:tasks.workunit.client.0.smithi136.stdout:write: B 54753666~5216127 2024-09-13T00:27:14.095 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 44074137~4975841 2024-09-13T00:27:14.096 INFO:tasks.workunit.client.0.smithi136.stdout:write: B 16227634~8360574 2024-09-13T00:27:14.117 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 2251565~7820167 2024-09-13T00:27:14.128 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap1 2024-09-13T00:27:14.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:14 smithi181 bash[67587]: cluster 2024-09-13T00:27:13.052091+0000 mon.a (mon.0) 2615 : cluster [DBG] osdmap e1373: 8 total, 8 up, 8 in 2024-09-13T00:27:14.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:14 smithi181 bash[67587]: cluster 2024-09-13T00:27:14.013407+0000 mon.a (mon.0) 2616 : cluster [DBG] osdmap e1374: 8 total, 8 up, 8 in 2024-09-13T00:27:14.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:14 smithi136 bash[75164]: cluster 2024-09-13T00:27:13.052091+0000 mon.a (mon.0) 2615 : cluster [DBG] osdmap e1373: 8 total, 8 up, 8 in 2024-09-13T00:27:14.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:14 smithi136 bash[75164]: cluster 2024-09-13T00:27:14.013407+0000 mon.a (mon.0) 2616 : cluster [DBG] osdmap e1374: 8 total, 8 up, 8 in 2024-09-13T00:27:14.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:14 smithi136 bash[76371]: cluster 2024-09-13T00:27:13.052091+0000 mon.a (mon.0) 2615 : cluster [DBG] osdmap e1373: 8 total, 8 up, 8 in 2024-09-13T00:27:14.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:14 smithi136 bash[76371]: cluster 2024-09-13T00:27:14.013407+0000 mon.a (mon.0) 2616 : cluster [DBG] osdmap e1374: 8 total, 8 up, 8 in 2024-09-13T00:27:15.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:15 smithi181 bash[67587]: cluster 2024-09-13T00:27:13.927487+0000 mgr.y (mgr.24661) 1503 : cluster [DBG] pgmap v2437: 169 pgs: 169 active+clean; 914 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 338 KiB/s rd, 2.1 MiB/s wr, 440 op/s 2024-09-13T00:27:15.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:15 smithi136 bash[75164]: cluster 2024-09-13T00:27:13.927487+0000 mgr.y (mgr.24661) 1503 : cluster [DBG] pgmap v2437: 169 pgs: 169 active+clean; 914 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 338 KiB/s rd, 2.1 MiB/s wr, 440 op/s 2024-09-13T00:27:15.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:15 smithi136 bash[76371]: cluster 2024-09-13T00:27:13.927487+0000 mgr.y (mgr.24661) 1503 : cluster [DBG] pgmap v2437: 169 pgs: 169 active+clean; 914 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 338 KiB/s rd, 2.1 MiB/s wr, 440 op/s 2024-09-13T00:27:16.108 INFO:tasks.workunit.client.0.smithi136.stdout:snap_protect: snap1 2024-09-13T00:27:16.140 INFO:tasks.workunit.client.0.smithi136.stdout:clone image157 -> image160 2024-09-13T00:27:16.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:16 smithi181 bash[67587]: cluster 2024-09-13T00:27:15.062277+0000 mon.a (mon.0) 2617 : cluster [DBG] osdmap e1375: 8 total, 8 up, 8 in 2024-09-13T00:27:16.405 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 152695571 2024-09-13T00:27:16.436 INFO:tasks.workunit.client.0.smithi136.stdout:write: C 47553949~1124661 2024-09-13T00:27:16.477 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 25686790~4635208 2024-09-13T00:27:16.504 INFO:tasks.workunit.client.0.smithi136.stdout:write: C 122555160~1407612 2024-09-13T00:27:16.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:16 smithi136 bash[76371]: cluster 2024-09-13T00:27:15.062277+0000 mon.a (mon.0) 2617 : cluster [DBG] osdmap e1375: 8 total, 8 up, 8 in 2024-09-13T00:27:16.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:16 smithi136 bash[75164]: cluster 2024-09-13T00:27:15.062277+0000 mon.a (mon.0) 2617 : cluster [DBG] osdmap e1375: 8 total, 8 up, 8 in 2024-09-13T00:27:16.532 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 97179367~3790258 2024-09-13T00:27:16.532 INFO:tasks.workunit.client.0.smithi136.stdout:write: C 110353723~5383672 2024-09-13T00:27:16.601 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 139723022~1007900 2024-09-13T00:27:16.602 INFO:tasks.workunit.client.0.smithi136.stdout:write: C 90998374~7664673 2024-09-13T00:27:16.676 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 105471046~7204147 2024-09-13T00:27:16.682 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap2 2024-09-13T00:27:17.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:17 smithi181 bash[67587]: cluster 2024-09-13T00:27:15.928685+0000 mgr.y (mgr.24661) 1504 : cluster [DBG] pgmap v2440: 169 pgs: 169 active+clean; 945 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 103 KiB/s rd, 8.2 MiB/s wr, 155 op/s 2024-09-13T00:27:17.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:17 smithi181 bash[67587]: cluster 2024-09-13T00:27:16.083171+0000 mon.a (mon.0) 2618 : cluster [DBG] osdmap e1376: 8 total, 8 up, 8 in 2024-09-13T00:27:17.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:17 smithi136 bash[75164]: cluster 2024-09-13T00:27:15.928685+0000 mgr.y (mgr.24661) 1504 : cluster [DBG] pgmap v2440: 169 pgs: 169 active+clean; 945 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 103 KiB/s rd, 8.2 MiB/s wr, 155 op/s 2024-09-13T00:27:17.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:17 smithi136 bash[75164]: cluster 2024-09-13T00:27:16.083171+0000 mon.a (mon.0) 2618 : cluster [DBG] osdmap e1376: 8 total, 8 up, 8 in 2024-09-13T00:27:17.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:17 smithi136 bash[76371]: cluster 2024-09-13T00:27:15.928685+0000 mgr.y (mgr.24661) 1504 : cluster [DBG] pgmap v2440: 169 pgs: 169 active+clean; 945 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 103 KiB/s rd, 8.2 MiB/s wr, 155 op/s 2024-09-13T00:27:17.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:17 smithi136 bash[76371]: cluster 2024-09-13T00:27:16.083171+0000 mon.a (mon.0) 2618 : cluster [DBG] osdmap e1376: 8 total, 8 up, 8 in 2024-09-13T00:27:18.124 INFO:tasks.workunit.client.0.smithi136.stdout:snap_protect: snap2 2024-09-13T00:27:18.150 INFO:tasks.workunit.client.0.smithi136.stdout:clone image160 -> image162 2024-09-13T00:27:18.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:18 smithi181 bash[67587]: cluster 2024-09-13T00:27:17.090803+0000 mon.a (mon.0) 2619 : cluster [DBG] osdmap e1377: 8 total, 8 up, 8 in 2024-09-13T00:27:18.493 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 173913874 2024-09-13T00:27:18.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:18 smithi136 bash[76371]: cluster 2024-09-13T00:27:17.090803+0000 mon.a (mon.0) 2619 : cluster [DBG] osdmap e1377: 8 total, 8 up, 8 in 2024-09-13T00:27:18.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:18 smithi136 bash[75164]: cluster 2024-09-13T00:27:17.090803+0000 mon.a (mon.0) 2619 : cluster [DBG] osdmap e1377: 8 total, 8 up, 8 in 2024-09-13T00:27:18.524 INFO:tasks.workunit.client.0.smithi136.stdout:write: D 95931422~912323 2024-09-13T00:27:18.628 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 15474708~3990278 2024-09-13T00:27:18.688 INFO:tasks.workunit.client.0.smithi136.stdout:write: D 123649651~1294044 2024-09-13T00:27:18.735 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 104833729~5338582 2024-09-13T00:27:18.759 INFO:tasks.workunit.client.0.smithi136.stdout:write: D 67731133~6213810 2024-09-13T00:27:18.833 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 119812203~3621293 2024-09-13T00:27:18.846 INFO:tasks.workunit.client.0.smithi136.stdout:write: D 3689456~1147963 2024-09-13T00:27:18.872 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 6382646~4161038 2024-09-13T00:27:18.887 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap3 2024-09-13T00:27:19.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:19 smithi181 bash[67587]: cluster 2024-09-13T00:27:17.929639+0000 mgr.y (mgr.24661) 1505 : cluster [DBG] pgmap v2443: 169 pgs: 169 active+clean; 1018 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 318 KiB/s rd, 19 MiB/s wr, 259 op/s 2024-09-13T00:27:19.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:19 smithi181 bash[67587]: cluster 2024-09-13T00:27:18.100876+0000 mon.a (mon.0) 2620 : cluster [DBG] osdmap e1378: 8 total, 8 up, 8 in 2024-09-13T00:27:19.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:19 smithi181 bash[67587]: cluster 2024-09-13T00:27:19.016513+0000 mon.a (mon.0) 2621 : cluster [DBG] osdmap e1379: 8 total, 8 up, 8 in 2024-09-13T00:27:19.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:19 smithi136 bash[76371]: cluster 2024-09-13T00:27:17.929639+0000 mgr.y (mgr.24661) 1505 : cluster [DBG] pgmap v2443: 169 pgs: 169 active+clean; 1018 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 318 KiB/s rd, 19 MiB/s wr, 259 op/s 2024-09-13T00:27:19.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:19 smithi136 bash[76371]: cluster 2024-09-13T00:27:18.100876+0000 mon.a (mon.0) 2620 : cluster [DBG] osdmap e1378: 8 total, 8 up, 8 in 2024-09-13T00:27:19.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:19 smithi136 bash[76371]: cluster 2024-09-13T00:27:19.016513+0000 mon.a (mon.0) 2621 : cluster [DBG] osdmap e1379: 8 total, 8 up, 8 in 2024-09-13T00:27:19.414 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:19 smithi136 bash[75164]: cluster 2024-09-13T00:27:17.929639+0000 mgr.y (mgr.24661) 1505 : cluster [DBG] pgmap v2443: 169 pgs: 169 active+clean; 1018 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 318 KiB/s rd, 19 MiB/s wr, 259 op/s 2024-09-13T00:27:19.414 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:19 smithi136 bash[75164]: cluster 2024-09-13T00:27:18.100876+0000 mon.a (mon.0) 2620 : cluster [DBG] osdmap e1378: 8 total, 8 up, 8 in 2024-09-13T00:27:19.414 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:19 smithi136 bash[75164]: cluster 2024-09-13T00:27:19.016513+0000 mon.a (mon.0) 2621 : cluster [DBG] osdmap e1379: 8 total, 8 up, 8 in 2024-09-13T00:27:19.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:27:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:27:19] "GET /metrics HTTP/1.1" 200 38991 "" "Prometheus/2.43.0" 2024-09-13T00:27:20.049 INFO:tasks.workunit.client.0.smithi136.stdout:snap_protect: snap3 2024-09-13T00:27:20.076 INFO:tasks.workunit.client.0.smithi136.stdout:clone image162 -> image164 2024-09-13T00:27:20.408 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 86269344 2024-09-13T00:27:20.639 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.StressNoMigrate (8312 ms) 2024-09-13T00:27:20.639 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.Stress 2024-09-13T00:27:20.707 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:27:20.730 INFO:tasks.workunit.client.0.smithi136.stdout:write: A 31551101~696959 2024-09-13T00:27:20.760 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 1704859~7104587 2024-09-13T00:27:20.760 INFO:tasks.workunit.client.0.smithi136.stdout:write: A 56712300~3501964 2024-09-13T00:27:20.798 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 61037759~428545 2024-09-13T00:27:20.801 INFO:tasks.workunit.client.0.smithi136.stdout:write: A 5933210~6430102 2024-09-13T00:27:20.854 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 47941403~4652931 2024-09-13T00:27:20.854 INFO:tasks.workunit.client.0.smithi136.stdout:write: A 37370309~6845533 2024-09-13T00:27:20.910 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 54676738~1600033 2024-09-13T00:27:20.914 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap0 2024-09-13T00:27:21.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:21 smithi181 bash[67587]: cluster 2024-09-13T00:27:19.930529+0000 mgr.y (mgr.24661) 1506 : cluster [DBG] pgmap v2446: 169 pgs: 169 active+clean; 1.0 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 1.8 MiB/s rd, 17 MiB/s wr, 310 op/s 2024-09-13T00:27:21.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:21 smithi181 bash[67587]: cluster 2024-09-13T00:27:20.016999+0000 mon.a (mon.0) 2622 : cluster [DBG] osdmap e1380: 8 total, 8 up, 8 in 2024-09-13T00:27:21.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:21 smithi181 bash[67587]: audit 2024-09-13T00:27:20.143737+0000 mon.a (mon.0) 2623 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:27:21.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:21 smithi181 bash[67587]: audit 2024-09-13T00:27:20.528751+0000 mon.a (mon.0) 2624 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:27:21.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:21 smithi181 bash[67587]: audit 2024-09-13T00:27:20.535283+0000 mon.a (mon.0) 2625 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:27:21.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:21 smithi136 bash[75164]: cluster 2024-09-13T00:27:19.930529+0000 mgr.y (mgr.24661) 1506 : cluster [DBG] pgmap v2446: 169 pgs: 169 active+clean; 1.0 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 1.8 MiB/s rd, 17 MiB/s wr, 310 op/s 2024-09-13T00:27:21.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:21 smithi136 bash[75164]: cluster 2024-09-13T00:27:20.016999+0000 mon.a (mon.0) 2622 : cluster [DBG] osdmap e1380: 8 total, 8 up, 8 in 2024-09-13T00:27:21.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:21 smithi136 bash[75164]: audit 2024-09-13T00:27:20.143737+0000 mon.a (mon.0) 2623 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:27:21.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:21 smithi136 bash[75164]: audit 2024-09-13T00:27:20.528751+0000 mon.a (mon.0) 2624 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:27:21.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:21 smithi136 bash[75164]: audit 2024-09-13T00:27:20.535283+0000 mon.a (mon.0) 2625 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:27:21.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:21 smithi136 bash[76371]: cluster 2024-09-13T00:27:19.930529+0000 mgr.y (mgr.24661) 1506 : cluster [DBG] pgmap v2446: 169 pgs: 169 active+clean; 1.0 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 1.8 MiB/s rd, 17 MiB/s wr, 310 op/s 2024-09-13T00:27:21.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:21 smithi136 bash[76371]: cluster 2024-09-13T00:27:20.016999+0000 mon.a (mon.0) 2622 : cluster [DBG] osdmap e1380: 8 total, 8 up, 8 in 2024-09-13T00:27:21.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:21 smithi136 bash[76371]: audit 2024-09-13T00:27:20.143737+0000 mon.a (mon.0) 2623 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:27:21.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:21 smithi136 bash[76371]: audit 2024-09-13T00:27:20.528751+0000 mon.a (mon.0) 2624 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:27:21.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:21 smithi136 bash[76371]: audit 2024-09-13T00:27:20.535283+0000 mon.a (mon.0) 2625 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:27:22.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:22 smithi181 bash[67587]: audit 2024-09-13T00:27:21.083359+0000 mon.a (mon.0) 2626 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:27:22.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:22 smithi181 bash[67587]: audit 2024-09-13T00:27:21.085262+0000 mon.a (mon.0) 2627 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:27:22.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:22 smithi181 bash[67587]: audit 2024-09-13T00:27:21.097594+0000 mon.a (mon.0) 2628 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:27:22.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:22 smithi181 bash[67587]: cluster 2024-09-13T00:27:21.544498+0000 mon.a (mon.0) 2629 : cluster [DBG] osdmap e1381: 8 total, 8 up, 8 in 2024-09-13T00:27:22.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:22 smithi136 bash[75164]: audit 2024-09-13T00:27:21.083359+0000 mon.a (mon.0) 2626 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:27:22.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:22 smithi136 bash[75164]: audit 2024-09-13T00:27:21.085262+0000 mon.a (mon.0) 2627 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:27:22.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:22 smithi136 bash[75164]: audit 2024-09-13T00:27:21.097594+0000 mon.a (mon.0) 2628 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:27:22.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:22 smithi136 bash[75164]: cluster 2024-09-13T00:27:21.544498+0000 mon.a (mon.0) 2629 : cluster [DBG] osdmap e1381: 8 total, 8 up, 8 in 2024-09-13T00:27:22.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:22 smithi136 bash[76371]: audit 2024-09-13T00:27:21.083359+0000 mon.a (mon.0) 2626 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:27:22.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:22 smithi136 bash[76371]: audit 2024-09-13T00:27:21.085262+0000 mon.a (mon.0) 2627 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:27:22.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:22 smithi136 bash[76371]: audit 2024-09-13T00:27:21.097594+0000 mon.a (mon.0) 2628 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:27:22.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:22 smithi136 bash[76371]: cluster 2024-09-13T00:27:21.544498+0000 mon.a (mon.0) 2629 : cluster [DBG] osdmap e1381: 8 total, 8 up, 8 in 2024-09-13T00:27:22.582 INFO:tasks.workunit.client.0.smithi136.stdout:snap_protect: snap0 2024-09-13T00:27:22.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:27:22 smithi181 bash[95505]: ts=2024-09-13T00:27:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:27:22.611 INFO:tasks.workunit.client.0.smithi136.stdout:clone image165 -> image168 2024-09-13T00:27:22.865 INFO:tasks.workunit.client.0.smithi136.stdout:write: B 47778565~1295064 2024-09-13T00:27:22.908 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 43261406~3387480 2024-09-13T00:27:22.946 INFO:tasks.workunit.client.0.smithi136.stdout:write: B 28079134~1068210 2024-09-13T00:27:22.969 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 18689020~2466329 2024-09-13T00:27:22.985 INFO:tasks.workunit.client.0.smithi136.stdout:write: B 16039647~7074348 2024-09-13T00:27:23.039 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 33241777~4163011 2024-09-13T00:27:23.064 INFO:tasks.workunit.client.0.smithi136.stdout:write: B 4859970~8237400 2024-09-13T00:27:23.146 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 57285027~6072622 2024-09-13T00:27:23.174 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap1 2024-09-13T00:27:23.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:23 smithi181 bash[67587]: cluster 2024-09-13T00:27:21.931663+0000 mgr.y (mgr.24661) 1507 : cluster [DBG] pgmap v2449: 169 pgs: 169 active+clean; 1.1 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 18 MiB/s rd, 17 MiB/s wr, 518 op/s 2024-09-13T00:27:23.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:23 smithi181 bash[67587]: cluster 2024-09-13T00:27:22.553877+0000 mon.a (mon.0) 2630 : cluster [DBG] osdmap e1382: 8 total, 8 up, 8 in 2024-09-13T00:27:23.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:23 smithi136 bash[75164]: cluster 2024-09-13T00:27:21.931663+0000 mgr.y (mgr.24661) 1507 : cluster [DBG] pgmap v2449: 169 pgs: 169 active+clean; 1.1 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 18 MiB/s rd, 17 MiB/s wr, 518 op/s 2024-09-13T00:27:23.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:23 smithi136 bash[75164]: cluster 2024-09-13T00:27:22.553877+0000 mon.a (mon.0) 2630 : cluster [DBG] osdmap e1382: 8 total, 8 up, 8 in 2024-09-13T00:27:23.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:23 smithi136 bash[76371]: cluster 2024-09-13T00:27:21.931663+0000 mgr.y (mgr.24661) 1507 : cluster [DBG] pgmap v2449: 169 pgs: 169 active+clean; 1.1 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 18 MiB/s rd, 17 MiB/s wr, 518 op/s 2024-09-13T00:27:23.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:23 smithi136 bash[76371]: cluster 2024-09-13T00:27:22.553877+0000 mon.a (mon.0) 2630 : cluster [DBG] osdmap e1382: 8 total, 8 up, 8 in 2024-09-13T00:27:24.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:24 smithi181 bash[67587]: audit 2024-09-13T00:27:23.815401+0000 mon.a (mon.0) 2631 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:27:24.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:24 smithi136 bash[76371]: audit 2024-09-13T00:27:23.815401+0000 mon.a (mon.0) 2631 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:27:24.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:24 smithi136 bash[75164]: audit 2024-09-13T00:27:23.815401+0000 mon.a (mon.0) 2631 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:27:25.066 INFO:tasks.workunit.client.0.smithi136.stdout:snap_protect: snap1 2024-09-13T00:27:25.095 INFO:tasks.workunit.client.0.smithi136.stdout:clone image168 -> image170 2024-09-13T00:27:25.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:25 smithi181 bash[67587]: cluster 2024-09-13T00:27:23.932792+0000 mgr.y (mgr.24661) 1508 : cluster [DBG] pgmap v2451: 169 pgs: 169 active+clean; 1.1 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 18 MiB/s rd, 18 MiB/s wr, 437 op/s 2024-09-13T00:27:25.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:25 smithi181 bash[67587]: cluster 2024-09-13T00:27:24.036940+0000 mon.a (mon.0) 2632 : cluster [DBG] osdmap e1383: 8 total, 8 up, 8 in 2024-09-13T00:27:25.426 INFO:tasks.workunit.client.0.smithi136.stdout:write: C 78940656~4945424 2024-09-13T00:27:25.506 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 28152742~3503482 2024-09-13T00:27:25.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:25 smithi136 bash[75164]: cluster 2024-09-13T00:27:23.932792+0000 mgr.y (mgr.24661) 1508 : cluster [DBG] pgmap v2451: 169 pgs: 169 active+clean; 1.1 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 18 MiB/s rd, 18 MiB/s wr, 437 op/s 2024-09-13T00:27:25.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:25 smithi136 bash[75164]: cluster 2024-09-13T00:27:24.036940+0000 mon.a (mon.0) 2632 : cluster [DBG] osdmap e1383: 8 total, 8 up, 8 in 2024-09-13T00:27:25.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:25 smithi136 bash[76371]: cluster 2024-09-13T00:27:23.932792+0000 mgr.y (mgr.24661) 1508 : cluster [DBG] pgmap v2451: 169 pgs: 169 active+clean; 1.1 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 18 MiB/s rd, 18 MiB/s wr, 437 op/s 2024-09-13T00:27:25.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:25 smithi136 bash[76371]: cluster 2024-09-13T00:27:24.036940+0000 mon.a (mon.0) 2632 : cluster [DBG] osdmap e1383: 8 total, 8 up, 8 in 2024-09-13T00:27:25.534 INFO:tasks.workunit.client.0.smithi136.stdout:write: C 41710845~4537977 2024-09-13T00:27:25.618 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 78492374~596673 2024-09-13T00:27:25.624 INFO:tasks.workunit.client.0.smithi136.stdout:write: C 77694179~3583682 2024-09-13T00:27:25.655 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 50636644~7339992 2024-09-13T00:27:25.672 INFO:tasks.workunit.client.0.smithi136.stdout:write: C 33800881~7758985 2024-09-13T00:27:25.741 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 2757858~6140521 2024-09-13T00:27:25.808 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap2 2024-09-13T00:27:26.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:26 smithi181 bash[67587]: cluster 2024-09-13T00:27:25.037794+0000 mon.a (mon.0) 2633 : cluster [DBG] osdmap e1384: 8 total, 8 up, 8 in 2024-09-13T00:27:26.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:26 smithi136 bash[75164]: cluster 2024-09-13T00:27:25.037794+0000 mon.a (mon.0) 2633 : cluster [DBG] osdmap e1384: 8 total, 8 up, 8 in 2024-09-13T00:27:26.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:26 smithi136 bash[76371]: cluster 2024-09-13T00:27:25.037794+0000 mon.a (mon.0) 2633 : cluster [DBG] osdmap e1384: 8 total, 8 up, 8 in 2024-09-13T00:27:27.089 INFO:tasks.workunit.client.0.smithi136.stdout:snap_protect: snap2 2024-09-13T00:27:27.121 INFO:tasks.workunit.client.0.smithi136.stdout:clone image170 -> image172 2024-09-13T00:27:27.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:27 smithi181 bash[67587]: cluster 2024-09-13T00:27:25.934001+0000 mgr.y (mgr.24661) 1509 : cluster [DBG] pgmap v2454: 169 pgs: 169 active+clean; 1.2 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 15 MiB/s rd, 16 MiB/s wr, 480 op/s 2024-09-13T00:27:27.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:27 smithi181 bash[67587]: cluster 2024-09-13T00:27:26.064572+0000 mon.a (mon.0) 2634 : cluster [DBG] osdmap e1385: 8 total, 8 up, 8 in 2024-09-13T00:27:27.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:27 smithi136 bash[75164]: cluster 2024-09-13T00:27:25.934001+0000 mgr.y (mgr.24661) 1509 : cluster [DBG] pgmap v2454: 169 pgs: 169 active+clean; 1.2 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 15 MiB/s rd, 16 MiB/s wr, 480 op/s 2024-09-13T00:27:27.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:27 smithi136 bash[75164]: cluster 2024-09-13T00:27:26.064572+0000 mon.a (mon.0) 2634 : cluster [DBG] osdmap e1385: 8 total, 8 up, 8 in 2024-09-13T00:27:27.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:27 smithi136 bash[76371]: cluster 2024-09-13T00:27:25.934001+0000 mgr.y (mgr.24661) 1509 : cluster [DBG] pgmap v2454: 169 pgs: 169 active+clean; 1.2 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 15 MiB/s rd, 16 MiB/s wr, 480 op/s 2024-09-13T00:27:27.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:27 smithi136 bash[76371]: cluster 2024-09-13T00:27:26.064572+0000 mon.a (mon.0) 2634 : cluster [DBG] osdmap e1385: 8 total, 8 up, 8 in 2024-09-13T00:27:27.521 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 114723072 2024-09-13T00:27:27.549 INFO:tasks.workunit.client.0.smithi136.stdout:write: D 44683543~4031144 2024-09-13T00:27:27.650 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 109391751~4356908 2024-09-13T00:27:27.650 INFO:tasks.workunit.client.0.smithi136.stdout:write: D 11015133~4894939 2024-09-13T00:27:27.739 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 40355613~3142367 2024-09-13T00:27:27.795 INFO:tasks.workunit.client.0.smithi136.stdout:write: D 96462979~1728267 2024-09-13T00:27:27.825 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 57499334~7931131 2024-09-13T00:27:27.844 INFO:tasks.workunit.client.0.smithi136.stdout:write: D 108070509~6652563 2024-09-13T00:27:27.911 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 108305344~6417728 2024-09-13T00:27:27.929 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap3 2024-09-13T00:27:28.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:28 smithi181 bash[67587]: cluster 2024-09-13T00:27:27.062883+0000 mon.a (mon.0) 2635 : cluster [DBG] osdmap e1386: 8 total, 8 up, 8 in 2024-09-13T00:27:28.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:28 smithi136 bash[75164]: cluster 2024-09-13T00:27:27.062883+0000 mon.a (mon.0) 2635 : cluster [DBG] osdmap e1386: 8 total, 8 up, 8 in 2024-09-13T00:27:28.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:28 smithi136 bash[76371]: cluster 2024-09-13T00:27:27.062883+0000 mon.a (mon.0) 2635 : cluster [DBG] osdmap e1386: 8 total, 8 up, 8 in 2024-09-13T00:27:29.126 INFO:tasks.workunit.client.0.smithi136.stdout:snap_protect: snap3 2024-09-13T00:27:29.154 INFO:tasks.workunit.client.0.smithi136.stdout:clone image172 -> image174 2024-09-13T00:27:29.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:29 smithi181 bash[67587]: cluster 2024-09-13T00:27:27.934925+0000 mgr.y (mgr.24661) 1510 : cluster [DBG] pgmap v2457: 169 pgs: 169 active+clean; 1.3 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 9.5 MiB/s rd, 20 MiB/s wr, 391 op/s 2024-09-13T00:27:29.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:29 smithi181 bash[67587]: cluster 2024-09-13T00:27:28.085860+0000 mon.a (mon.0) 2636 : cluster [DBG] osdmap e1387: 8 total, 8 up, 8 in 2024-09-13T00:27:29.414 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:29 smithi136 bash[76371]: cluster 2024-09-13T00:27:27.934925+0000 mgr.y (mgr.24661) 1510 : cluster [DBG] pgmap v2457: 169 pgs: 169 active+clean; 1.3 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 9.5 MiB/s rd, 20 MiB/s wr, 391 op/s 2024-09-13T00:27:29.414 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:29 smithi136 bash[76371]: cluster 2024-09-13T00:27:28.085860+0000 mon.a (mon.0) 2636 : cluster [DBG] osdmap e1387: 8 total, 8 up, 8 in 2024-09-13T00:27:29.414 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:29 smithi136 bash[75164]: cluster 2024-09-13T00:27:27.934925+0000 mgr.y (mgr.24661) 1510 : cluster [DBG] pgmap v2457: 169 pgs: 169 active+clean; 1.3 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 9.5 MiB/s rd, 20 MiB/s wr, 391 op/s 2024-09-13T00:27:29.414 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:29 smithi136 bash[75164]: cluster 2024-09-13T00:27:28.085860+0000 mon.a (mon.0) 2636 : cluster [DBG] osdmap e1387: 8 total, 8 up, 8 in 2024-09-13T00:27:29.619 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 107000571 2024-09-13T00:27:29.675 INFO:tasks.workunit.client.0.smithi136.stdout:migration_prepare 2024-09-13T00:27:29.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:27:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:27:29] "GET /metrics HTTP/1.1" 200 38992 "" "Prometheus/2.43.0" 2024-09-13T00:27:30.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:30 smithi181 bash[67587]: cluster 2024-09-13T00:27:29.095618+0000 mon.a (mon.0) 2637 : cluster [DBG] osdmap e1388: 8 total, 8 up, 8 in 2024-09-13T00:27:30.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:30 smithi136 bash[75164]: cluster 2024-09-13T00:27:29.095618+0000 mon.a (mon.0) 2637 : cluster [DBG] osdmap e1388: 8 total, 8 up, 8 in 2024-09-13T00:27:30.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:30 smithi136 bash[76371]: cluster 2024-09-13T00:27:29.095618+0000 mon.a (mon.0) 2637 : cluster [DBG] osdmap e1388: 8 total, 8 up, 8 in 2024-09-13T00:27:30.834 INFO:tasks.workunit.client.0.smithi136.stdout:migration_execute 2024-09-13T00:27:31.483 INFO:tasks.workunit.client.0.smithi136.stdout:migration_commit 2024-09-13T00:27:31.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:31 smithi136 bash[76371]: cluster 2024-09-13T00:27:29.935842+0000 mgr.y (mgr.24661) 1511 : cluster [DBG] pgmap v2460: 169 pgs: 169 active+clean; 1.3 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 8.8 MiB/s rd, 19 MiB/s wr, 393 op/s 2024-09-13T00:27:31.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:31 smithi136 bash[75164]: cluster 2024-09-13T00:27:29.935842+0000 mgr.y (mgr.24661) 1511 : cluster [DBG] pgmap v2460: 169 pgs: 169 active+clean; 1.3 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 8.8 MiB/s rd, 19 MiB/s wr, 393 op/s 2024-09-13T00:27:31.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:31 smithi181 bash[67587]: cluster 2024-09-13T00:27:29.935842+0000 mgr.y (mgr.24661) 1511 : cluster [DBG] pgmap v2460: 169 pgs: 169 active+clean; 1.3 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 8.8 MiB/s rd, 19 MiB/s wr, 393 op/s 2024-09-13T00:27:32.136 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.Stress (11498 ms) 2024-09-13T00:27:32.137 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.Stress2 2024-09-13T00:27:32.222 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:27:32.250 INFO:tasks.workunit.client.0.smithi136.stdout:write: A 43323647~5680169 2024-09-13T00:27:32.321 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 71740417~4044254 2024-09-13T00:27:32.321 INFO:tasks.workunit.client.0.smithi136.stdout:write: A 53746279~1685603 2024-09-13T00:27:32.347 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 58157772~4878002 2024-09-13T00:27:32.352 INFO:tasks.workunit.client.0.smithi136.stdout:write: A 77713591~6172489 2024-09-13T00:27:32.413 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 37009481~6646247 2024-09-13T00:27:32.418 INFO:tasks.workunit.client.0.smithi136.stdout:write: A 24155391~2085353 2024-09-13T00:27:32.446 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 72487793~7811957 2024-09-13T00:27:32.465 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap0 2024-09-13T00:27:32.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:27:32 smithi181 bash[95505]: ts=2024-09-13T00:27:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:27:33.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:33 smithi136 bash[75164]: cluster 2024-09-13T00:27:31.937376+0000 mgr.y (mgr.24661) 1512 : cluster [DBG] pgmap v2461: 169 pgs: 169 active+clean; 1.3 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 18 MiB/s rd, 21 MiB/s wr, 640 op/s 2024-09-13T00:27:33.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:33 smithi136 bash[76371]: cluster 2024-09-13T00:27:31.937376+0000 mgr.y (mgr.24661) 1512 : cluster [DBG] pgmap v2461: 169 pgs: 169 active+clean; 1.3 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 18 MiB/s rd, 21 MiB/s wr, 640 op/s 2024-09-13T00:27:33.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:33 smithi181 bash[67587]: cluster 2024-09-13T00:27:31.937376+0000 mgr.y (mgr.24661) 1512 : cluster [DBG] pgmap v2461: 169 pgs: 169 active+clean; 1.3 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 18 MiB/s rd, 21 MiB/s wr, 640 op/s 2024-09-13T00:27:34.153 INFO:tasks.workunit.client.0.smithi136.stdout:snap_protect: snap0 2024-09-13T00:27:34.186 INFO:tasks.workunit.client.0.smithi136.stdout:clone image175 -> image178 2024-09-13T00:27:34.440 INFO:tasks.workunit.client.0.smithi136.stdout:write: B 22028023~132688 2024-09-13T00:27:34.491 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 35394259~5195870 2024-09-13T00:27:34.507 INFO:tasks.workunit.client.0.smithi136.stdout:write: B 79092750~4793330 2024-09-13T00:27:34.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:34 smithi136 bash[75164]: cluster 2024-09-13T00:27:33.120561+0000 mon.a (mon.0) 2638 : cluster [DBG] osdmap e1389: 8 total, 8 up, 8 in 2024-09-13T00:27:34.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:34 smithi136 bash[76371]: cluster 2024-09-13T00:27:33.120561+0000 mon.a (mon.0) 2638 : cluster [DBG] osdmap e1389: 8 total, 8 up, 8 in 2024-09-13T00:27:34.571 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 36495052~103782 2024-09-13T00:27:34.578 INFO:tasks.workunit.client.0.smithi136.stdout:write: B 60074912~6139236 2024-09-13T00:27:34.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:34 smithi181 bash[67587]: cluster 2024-09-13T00:27:33.120561+0000 mon.a (mon.0) 2638 : cluster [DBG] osdmap e1389: 8 total, 8 up, 8 in 2024-09-13T00:27:34.641 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 20713377~5625886 2024-09-13T00:27:34.662 INFO:tasks.workunit.client.0.smithi136.stdout:write: B 70026311~3202203 2024-09-13T00:27:34.699 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 24023673~5316768 2024-09-13T00:27:34.705 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap1 2024-09-13T00:27:35.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:35 smithi136 bash[76371]: cluster 2024-09-13T00:27:33.938363+0000 mgr.y (mgr.24661) 1513 : cluster [DBG] pgmap v2463: 169 pgs: 169 active+clean; 1.4 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 17 MiB/s rd, 12 MiB/s wr, 744 op/s 2024-09-13T00:27:35.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:35 smithi136 bash[76371]: cluster 2024-09-13T00:27:34.121751+0000 mon.a (mon.0) 2639 : cluster [DBG] osdmap e1390: 8 total, 8 up, 8 in 2024-09-13T00:27:35.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:35 smithi136 bash[75164]: cluster 2024-09-13T00:27:33.938363+0000 mgr.y (mgr.24661) 1513 : cluster [DBG] pgmap v2463: 169 pgs: 169 active+clean; 1.4 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 17 MiB/s rd, 12 MiB/s wr, 744 op/s 2024-09-13T00:27:35.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:35 smithi136 bash[75164]: cluster 2024-09-13T00:27:34.121751+0000 mon.a (mon.0) 2639 : cluster [DBG] osdmap e1390: 8 total, 8 up, 8 in 2024-09-13T00:27:35.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:35 smithi181 bash[67587]: cluster 2024-09-13T00:27:33.938363+0000 mgr.y (mgr.24661) 1513 : cluster [DBG] pgmap v2463: 169 pgs: 169 active+clean; 1.4 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 17 MiB/s rd, 12 MiB/s wr, 744 op/s 2024-09-13T00:27:35.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:35 smithi181 bash[67587]: cluster 2024-09-13T00:27:34.121751+0000 mon.a (mon.0) 2639 : cluster [DBG] osdmap e1390: 8 total, 8 up, 8 in 2024-09-13T00:27:36.179 INFO:tasks.workunit.client.0.smithi136.stdout:snap_protect: snap1 2024-09-13T00:27:36.214 INFO:tasks.workunit.client.0.smithi136.stdout:clone image178 -> image180 2024-09-13T00:27:36.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:36 smithi136 bash[75164]: cluster 2024-09-13T00:27:35.146086+0000 mon.a (mon.0) 2640 : cluster [DBG] osdmap e1391: 8 total, 8 up, 8 in 2024-09-13T00:27:36.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:36 smithi136 bash[76371]: cluster 2024-09-13T00:27:35.146086+0000 mon.a (mon.0) 2640 : cluster [DBG] osdmap e1391: 8 total, 8 up, 8 in 2024-09-13T00:27:36.546 INFO:tasks.workunit.client.0.smithi136.stdout:write: C 62758651~2272480 2024-09-13T00:27:36.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:36 smithi181 bash[67587]: cluster 2024-09-13T00:27:35.146086+0000 mon.a (mon.0) 2640 : cluster [DBG] osdmap e1391: 8 total, 8 up, 8 in 2024-09-13T00:27:36.645 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 56362~6439888 2024-09-13T00:27:36.664 INFO:tasks.workunit.client.0.smithi136.stdout:write: C 38251960~2092605 2024-09-13T00:27:36.697 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 79675430~3511411 2024-09-13T00:27:36.736 INFO:tasks.workunit.client.0.smithi136.stdout:write: C 53266409~3687095 2024-09-13T00:27:36.787 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 15960209~1168279 2024-09-13T00:27:36.807 INFO:tasks.workunit.client.0.smithi136.stdout:write: C 51632616~6402363 2024-09-13T00:27:36.858 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 53541273~4433800 2024-09-13T00:27:36.864 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap2 2024-09-13T00:27:37.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:37 smithi136 bash[76371]: cluster 2024-09-13T00:27:35.939401+0000 mgr.y (mgr.24661) 1514 : cluster [DBG] pgmap v2466: 169 pgs: 169 active+clean; 1.4 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 22 MiB/s rd, 12 MiB/s wr, 836 op/s 2024-09-13T00:27:37.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:37 smithi136 bash[76371]: cluster 2024-09-13T00:27:36.155044+0000 mon.a (mon.0) 2641 : cluster [DBG] osdmap e1392: 8 total, 8 up, 8 in 2024-09-13T00:27:37.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:37 smithi136 bash[75164]: cluster 2024-09-13T00:27:35.939401+0000 mgr.y (mgr.24661) 1514 : cluster [DBG] pgmap v2466: 169 pgs: 169 active+clean; 1.4 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 22 MiB/s rd, 12 MiB/s wr, 836 op/s 2024-09-13T00:27:37.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:37 smithi136 bash[75164]: cluster 2024-09-13T00:27:36.155044+0000 mon.a (mon.0) 2641 : cluster [DBG] osdmap e1392: 8 total, 8 up, 8 in 2024-09-13T00:27:37.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:37 smithi181 bash[67587]: cluster 2024-09-13T00:27:35.939401+0000 mgr.y (mgr.24661) 1514 : cluster [DBG] pgmap v2466: 169 pgs: 169 active+clean; 1.4 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 22 MiB/s rd, 12 MiB/s wr, 836 op/s 2024-09-13T00:27:37.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:37 smithi181 bash[67587]: cluster 2024-09-13T00:27:36.155044+0000 mon.a (mon.0) 2641 : cluster [DBG] osdmap e1392: 8 total, 8 up, 8 in 2024-09-13T00:27:38.196 INFO:tasks.workunit.client.0.smithi136.stdout:snap_protect: snap2 2024-09-13T00:27:38.228 INFO:tasks.workunit.client.0.smithi136.stdout:clone image180 -> image182 2024-09-13T00:27:38.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:38 smithi136 bash[75164]: cluster 2024-09-13T00:27:37.163421+0000 mon.a (mon.0) 2642 : cluster [DBG] osdmap e1393: 8 total, 8 up, 8 in 2024-09-13T00:27:38.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:38 smithi136 bash[76371]: cluster 2024-09-13T00:27:37.163421+0000 mon.a (mon.0) 2642 : cluster [DBG] osdmap e1393: 8 total, 8 up, 8 in 2024-09-13T00:27:38.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:38 smithi181 bash[67587]: cluster 2024-09-13T00:27:37.163421+0000 mon.a (mon.0) 2642 : cluster [DBG] osdmap e1393: 8 total, 8 up, 8 in 2024-09-13T00:27:38.623 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 86817213 2024-09-13T00:27:38.652 INFO:tasks.workunit.client.0.smithi136.stdout:write: D 67147292~3244578 2024-09-13T00:27:38.737 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 15109393~681761 2024-09-13T00:27:38.752 INFO:tasks.workunit.client.0.smithi136.stdout:write: D 57475912~1016421 2024-09-13T00:27:38.778 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 69423569~7801437 2024-09-13T00:27:38.801 INFO:tasks.workunit.client.0.smithi136.stdout:write: D 78416043~1546083 2024-09-13T00:27:38.831 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 83722951~3094262 2024-09-13T00:27:38.837 INFO:tasks.workunit.client.0.smithi136.stdout:write: D 29464944~543158 2024-09-13T00:27:38.857 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 30818028~6599735 2024-09-13T00:27:38.872 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap3 2024-09-13T00:27:39.509 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:27:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:27:39] "GET /metrics HTTP/1.1" 200 38991 "" "Prometheus/2.43.0" 2024-09-13T00:27:39.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:39 smithi136 bash[75164]: cluster 2024-09-13T00:27:37.940384+0000 mgr.y (mgr.24661) 1515 : cluster [DBG] pgmap v2469: 169 pgs: 169 active+clean; 1.5 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 21 MiB/s rd, 13 MiB/s wr, 613 op/s 2024-09-13T00:27:39.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:39 smithi136 bash[75164]: cluster 2024-09-13T00:27:38.172879+0000 mon.a (mon.0) 2643 : cluster [DBG] osdmap e1394: 8 total, 8 up, 8 in 2024-09-13T00:27:39.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:39 smithi136 bash[75164]: audit 2024-09-13T00:27:38.816054+0000 mon.a (mon.0) 2644 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:27:39.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:39 smithi136 bash[76371]: cluster 2024-09-13T00:27:37.940384+0000 mgr.y (mgr.24661) 1515 : cluster [DBG] pgmap v2469: 169 pgs: 169 active+clean; 1.5 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 21 MiB/s rd, 13 MiB/s wr, 613 op/s 2024-09-13T00:27:39.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:39 smithi136 bash[76371]: cluster 2024-09-13T00:27:38.172879+0000 mon.a (mon.0) 2643 : cluster [DBG] osdmap e1394: 8 total, 8 up, 8 in 2024-09-13T00:27:39.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:39 smithi136 bash[76371]: audit 2024-09-13T00:27:38.816054+0000 mon.a (mon.0) 2644 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:27:39.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:39 smithi181 bash[67587]: cluster 2024-09-13T00:27:37.940384+0000 mgr.y (mgr.24661) 1515 : cluster [DBG] pgmap v2469: 169 pgs: 169 active+clean; 1.5 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 21 MiB/s rd, 13 MiB/s wr, 613 op/s 2024-09-13T00:27:39.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:39 smithi181 bash[67587]: cluster 2024-09-13T00:27:38.172879+0000 mon.a (mon.0) 2643 : cluster [DBG] osdmap e1394: 8 total, 8 up, 8 in 2024-09-13T00:27:39.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:39 smithi181 bash[67587]: audit 2024-09-13T00:27:38.816054+0000 mon.a (mon.0) 2644 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:27:40.212 INFO:tasks.workunit.client.0.smithi136.stdout:snap_protect: snap3 2024-09-13T00:27:40.241 INFO:tasks.workunit.client.0.smithi136.stdout:clone image182 -> image184 2024-09-13T00:27:40.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:40 smithi136 bash[76371]: cluster 2024-09-13T00:27:39.183382+0000 mon.a (mon.0) 2645 : cluster [DBG] osdmap e1395: 8 total, 8 up, 8 in 2024-09-13T00:27:40.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:40 smithi136 bash[75164]: cluster 2024-09-13T00:27:39.183382+0000 mon.a (mon.0) 2645 : cluster [DBG] osdmap e1395: 8 total, 8 up, 8 in 2024-09-13T00:27:40.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:40 smithi181 bash[67587]: cluster 2024-09-13T00:27:39.183382+0000 mon.a (mon.0) 2645 : cluster [DBG] osdmap e1395: 8 total, 8 up, 8 in 2024-09-13T00:27:40.715 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 104369437 2024-09-13T00:27:40.754 INFO:tasks.workunit.client.0.smithi136.stdout:migration_prepare 2024-09-13T00:27:41.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:41 smithi136 bash[76371]: cluster 2024-09-13T00:27:39.941311+0000 mgr.y (mgr.24661) 1516 : cluster [DBG] pgmap v2472: 169 pgs: 169 active+clean; 1.5 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 14 MiB/s rd, 17 MiB/s wr, 446 op/s 2024-09-13T00:27:41.583 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:41 smithi136 bash[76371]: cluster 2024-09-13T00:27:40.195463+0000 mon.a (mon.0) 2646 : cluster [DBG] osdmap e1396: 8 total, 8 up, 8 in 2024-09-13T00:27:41.584 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:41 smithi136 bash[75164]: cluster 2024-09-13T00:27:39.941311+0000 mgr.y (mgr.24661) 1516 : cluster [DBG] pgmap v2472: 169 pgs: 169 active+clean; 1.5 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 14 MiB/s rd, 17 MiB/s wr, 446 op/s 2024-09-13T00:27:41.584 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:41 smithi136 bash[75164]: cluster 2024-09-13T00:27:40.195463+0000 mon.a (mon.0) 2646 : cluster [DBG] osdmap e1396: 8 total, 8 up, 8 in 2024-09-13T00:27:41.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:41 smithi181 bash[67587]: cluster 2024-09-13T00:27:39.941311+0000 mgr.y (mgr.24661) 1516 : cluster [DBG] pgmap v2472: 169 pgs: 169 active+clean; 1.5 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 14 MiB/s rd, 17 MiB/s wr, 446 op/s 2024-09-13T00:27:41.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:41 smithi181 bash[67587]: cluster 2024-09-13T00:27:40.195463+0000 mon.a (mon.0) 2646 : cluster [DBG] osdmap e1396: 8 total, 8 up, 8 in 2024-09-13T00:27:41.860 INFO:tasks.workunit.client.0.smithi136.stdout:write: a 65515169~5014613 2024-09-13T00:27:41.947 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 37206672~7045693 2024-09-13T00:27:41.996 INFO:tasks.workunit.client.0.smithi136.stdout:write: a 56566585~5778743 2024-09-13T00:27:42.068 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 93520624~7483860 2024-09-13T00:27:42.076 INFO:tasks.workunit.client.0.smithi136.stdout:write: a 91780921~8269074 2024-09-13T00:27:42.124 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 64659996~2662459 2024-09-13T00:27:42.129 INFO:tasks.workunit.client.0.smithi136.stdout:write: a 85067215~5480626 2024-09-13T00:27:42.189 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 95141429~3493361 2024-09-13T00:27:42.196 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: user0 2024-09-13T00:27:42.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:27:42 smithi181 bash[95505]: ts=2024-09-13T00:27:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:27:43.512 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:43 smithi136 bash[75164]: cluster 2024-09-13T00:27:41.942199+0000 mgr.y (mgr.24661) 1517 : cluster [DBG] pgmap v2474: 169 pgs: 169 active+clean; 1.5 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 9.1 MiB/s rd, 11 MiB/s wr, 524 op/s 2024-09-13T00:27:43.513 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:43 smithi136 bash[76371]: cluster 2024-09-13T00:27:41.942199+0000 mgr.y (mgr.24661) 1517 : cluster [DBG] pgmap v2474: 169 pgs: 169 active+clean; 1.5 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 9.1 MiB/s rd, 11 MiB/s wr, 524 op/s 2024-09-13T00:27:43.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:43 smithi181 bash[67587]: cluster 2024-09-13T00:27:41.942199+0000 mgr.y (mgr.24661) 1517 : cluster [DBG] pgmap v2474: 169 pgs: 169 active+clean; 1.5 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 9.1 MiB/s rd, 11 MiB/s wr, 524 op/s 2024-09-13T00:27:44.253 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 139021196 2024-09-13T00:27:44.284 INFO:tasks.workunit.client.0.smithi136.stdout:write: b 8955236~1104081 2024-09-13T00:27:44.302 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 113090443~1268758 2024-09-13T00:27:44.303 INFO:tasks.workunit.client.0.smithi136.stdout:write: b 32476537~6097546 2024-09-13T00:27:44.314 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 82636036~8306525 2024-09-13T00:27:44.315 INFO:tasks.workunit.client.0.smithi136.stdout:write: b 59098006~8028341 2024-09-13T00:27:44.353 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 112748776~5678250 2024-09-13T00:27:44.391 INFO:tasks.workunit.client.0.smithi136.stdout:write: b 63391323~4628370 2024-09-13T00:27:44.450 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 3285455~755531 2024-09-13T00:27:44.450 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: user1 2024-09-13T00:27:44.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:44 smithi136 bash[76371]: cluster 2024-09-13T00:27:43.216605+0000 mon.a (mon.0) 2647 : cluster [DBG] osdmap e1397: 8 total, 8 up, 8 in 2024-09-13T00:27:44.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:44 smithi136 bash[75164]: cluster 2024-09-13T00:27:43.216605+0000 mon.a (mon.0) 2647 : cluster [DBG] osdmap e1397: 8 total, 8 up, 8 in 2024-09-13T00:27:44.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:44 smithi181 bash[67587]: cluster 2024-09-13T00:27:43.216605+0000 mon.a (mon.0) 2647 : cluster [DBG] osdmap e1397: 8 total, 8 up, 8 in 2024-09-13T00:27:45.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:45 smithi136 bash[76371]: cluster 2024-09-13T00:27:43.943166+0000 mgr.y (mgr.24661) 1518 : cluster [DBG] pgmap v2476: 169 pgs: 169 active+clean; 1.5 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 7.7 MiB/s rd, 11 MiB/s wr, 525 op/s 2024-09-13T00:27:45.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:45 smithi136 bash[76371]: cluster 2024-09-13T00:27:44.233775+0000 mon.a (mon.0) 2648 : cluster [DBG] osdmap e1398: 8 total, 8 up, 8 in 2024-09-13T00:27:45.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:45 smithi136 bash[75164]: cluster 2024-09-13T00:27:43.943166+0000 mgr.y (mgr.24661) 1518 : cluster [DBG] pgmap v2476: 169 pgs: 169 active+clean; 1.5 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 7.7 MiB/s rd, 11 MiB/s wr, 525 op/s 2024-09-13T00:27:45.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:45 smithi136 bash[75164]: cluster 2024-09-13T00:27:44.233775+0000 mon.a (mon.0) 2648 : cluster [DBG] osdmap e1398: 8 total, 8 up, 8 in 2024-09-13T00:27:45.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:45 smithi181 bash[67587]: cluster 2024-09-13T00:27:43.943166+0000 mgr.y (mgr.24661) 1518 : cluster [DBG] pgmap v2476: 169 pgs: 169 active+clean; 1.5 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 7.7 MiB/s rd, 11 MiB/s wr, 525 op/s 2024-09-13T00:27:45.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:45 smithi181 bash[67587]: cluster 2024-09-13T00:27:44.233775+0000 mon.a (mon.0) 2648 : cluster [DBG] osdmap e1398: 8 total, 8 up, 8 in 2024-09-13T00:27:46.278 INFO:tasks.workunit.client.0.smithi136.stdout:write: c 95746279~6534274 2024-09-13T00:27:46.329 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 110169990~7582400 2024-09-13T00:27:46.330 INFO:tasks.workunit.client.0.smithi136.stdout:write: c 91267872~8225369 2024-09-13T00:27:46.385 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 7312141~2632061 2024-09-13T00:27:46.385 INFO:tasks.workunit.client.0.smithi136.stdout:write: c 32951315~6910466 2024-09-13T00:27:46.391 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 120020434~6141722 2024-09-13T00:27:46.391 INFO:tasks.workunit.client.0.smithi136.stdout:write: c 3357897~8143079 2024-09-13T00:27:46.459 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 51801231~1317191 2024-09-13T00:27:46.459 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: user2 2024-09-13T00:27:46.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:46 smithi136 bash[76371]: cluster 2024-09-13T00:27:45.232312+0000 mon.a (mon.0) 2649 : cluster [DBG] osdmap e1399: 8 total, 8 up, 8 in 2024-09-13T00:27:46.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:46 smithi136 bash[75164]: cluster 2024-09-13T00:27:45.232312+0000 mon.a (mon.0) 2649 : cluster [DBG] osdmap e1399: 8 total, 8 up, 8 in 2024-09-13T00:27:46.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:46 smithi181 bash[67587]: cluster 2024-09-13T00:27:45.232312+0000 mon.a (mon.0) 2649 : cluster [DBG] osdmap e1399: 8 total, 8 up, 8 in 2024-09-13T00:27:47.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:47 smithi181 bash[67587]: cluster 2024-09-13T00:27:45.944246+0000 mgr.y (mgr.24661) 1519 : cluster [DBG] pgmap v2479: 169 pgs: 169 active+clean; 1.6 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 12 MiB/s rd, 10 MiB/s wr, 548 op/s 2024-09-13T00:27:47.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:47 smithi181 bash[67587]: cluster 2024-09-13T00:27:46.254715+0000 mon.a (mon.0) 2650 : cluster [DBG] osdmap e1400: 8 total, 8 up, 8 in 2024-09-13T00:27:47.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:47 smithi136 bash[76371]: cluster 2024-09-13T00:27:45.944246+0000 mgr.y (mgr.24661) 1519 : cluster [DBG] pgmap v2479: 169 pgs: 169 active+clean; 1.6 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 12 MiB/s rd, 10 MiB/s wr, 548 op/s 2024-09-13T00:27:47.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:47 smithi136 bash[76371]: cluster 2024-09-13T00:27:46.254715+0000 mon.a (mon.0) 2650 : cluster [DBG] osdmap e1400: 8 total, 8 up, 8 in 2024-09-13T00:27:47.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:47 smithi136 bash[75164]: cluster 2024-09-13T00:27:45.944246+0000 mgr.y (mgr.24661) 1519 : cluster [DBG] pgmap v2479: 169 pgs: 169 active+clean; 1.6 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 12 MiB/s rd, 10 MiB/s wr, 548 op/s 2024-09-13T00:27:47.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:47 smithi136 bash[75164]: cluster 2024-09-13T00:27:46.254715+0000 mon.a (mon.0) 2650 : cluster [DBG] osdmap e1400: 8 total, 8 up, 8 in 2024-09-13T00:27:48.308 INFO:tasks.workunit.client.0.smithi136.stdout:write: d 89086398~691950 2024-09-13T00:27:48.330 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 50871672~173153 2024-09-13T00:27:48.330 INFO:tasks.workunit.client.0.smithi136.stdout:write: d 36882614~5308290 2024-09-13T00:27:48.356 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 44471128~1127306 2024-09-13T00:27:48.356 INFO:tasks.workunit.client.0.smithi136.stdout:write: d 47628872~2361275 2024-09-13T00:27:48.357 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 92884994~4902188 2024-09-13T00:27:48.360 INFO:tasks.workunit.client.0.smithi136.stdout:write: d 8516024~652523 2024-09-13T00:27:48.361 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 84494898~1262472 2024-09-13T00:27:48.361 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: user3 2024-09-13T00:27:48.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:48 smithi181 bash[67587]: cluster 2024-09-13T00:27:47.253485+0000 mon.a (mon.0) 2651 : cluster [DBG] osdmap e1401: 8 total, 8 up, 8 in 2024-09-13T00:27:48.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:48 smithi136 bash[75164]: cluster 2024-09-13T00:27:47.253485+0000 mon.a (mon.0) 2651 : cluster [DBG] osdmap e1401: 8 total, 8 up, 8 in 2024-09-13T00:27:48.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:48 smithi136 bash[76371]: cluster 2024-09-13T00:27:47.253485+0000 mon.a (mon.0) 2651 : cluster [DBG] osdmap e1401: 8 total, 8 up, 8 in 2024-09-13T00:27:49.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:49 smithi181 bash[67587]: cluster 2024-09-13T00:27:47.945226+0000 mgr.y (mgr.24661) 1520 : cluster [DBG] pgmap v2482: 169 pgs: 169 active+clean; 1.7 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 14 MiB/s rd, 43 MiB/s wr, 366 op/s 2024-09-13T00:27:49.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:49 smithi181 bash[67587]: cluster 2024-09-13T00:27:48.285168+0000 mon.a (mon.0) 2652 : cluster [DBG] osdmap e1402: 8 total, 8 up, 8 in 2024-09-13T00:27:49.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:27:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:27:49] "GET /metrics HTTP/1.1" 200 38991 "" "Prometheus/2.43.0" 2024-09-13T00:27:49.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:49 smithi136 bash[75164]: cluster 2024-09-13T00:27:47.945226+0000 mgr.y (mgr.24661) 1520 : cluster [DBG] pgmap v2482: 169 pgs: 169 active+clean; 1.7 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 14 MiB/s rd, 43 MiB/s wr, 366 op/s 2024-09-13T00:27:49.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:49 smithi136 bash[75164]: cluster 2024-09-13T00:27:48.285168+0000 mon.a (mon.0) 2652 : cluster [DBG] osdmap e1402: 8 total, 8 up, 8 in 2024-09-13T00:27:49.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:49 smithi136 bash[76371]: cluster 2024-09-13T00:27:47.945226+0000 mgr.y (mgr.24661) 1520 : cluster [DBG] pgmap v2482: 169 pgs: 169 active+clean; 1.7 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 14 MiB/s rd, 43 MiB/s wr, 366 op/s 2024-09-13T00:27:49.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:49 smithi136 bash[76371]: cluster 2024-09-13T00:27:48.285168+0000 mon.a (mon.0) 2652 : cluster [DBG] osdmap e1402: 8 total, 8 up, 8 in 2024-09-13T00:27:50.350 INFO:tasks.workunit.client.0.smithi136.stdout:write: 1 13902119~3475529 2024-09-13T00:27:50.353 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 14771001~1737764 2024-09-13T00:27:50.432 INFO:tasks.workunit.client.0.smithi136.stdout:write: 2 27804239~3475529 2024-09-13T00:27:50.434 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 28673121~1737764 2024-09-13T00:27:50.494 INFO:tasks.workunit.client.0.smithi136.stdout:write: 3 41706358~3475529 2024-09-13T00:27:50.496 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 42575240~1737764 2024-09-13T00:27:50.557 INFO:tasks.workunit.client.0.smithi136.stdout:write: 4 55608478~3475529 2024-09-13T00:27:50.559 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 56477360~1737764 2024-09-13T00:27:50.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:50 smithi181 bash[67587]: cluster 2024-09-13T00:27:49.280747+0000 mon.a (mon.0) 2653 : cluster [DBG] osdmap e1403: 8 total, 8 up, 8 in 2024-09-13T00:27:50.605 INFO:tasks.workunit.client.0.smithi136.stdout:write: 5 69510598~3475529 2024-09-13T00:27:50.607 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 70379480~1737764 2024-09-13T00:27:50.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:50 smithi136 bash[76371]: cluster 2024-09-13T00:27:49.280747+0000 mon.a (mon.0) 2653 : cluster [DBG] osdmap e1403: 8 total, 8 up, 8 in 2024-09-13T00:27:50.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:50 smithi136 bash[75164]: cluster 2024-09-13T00:27:49.280747+0000 mon.a (mon.0) 2653 : cluster [DBG] osdmap e1403: 8 total, 8 up, 8 in 2024-09-13T00:27:51.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:51 smithi181 bash[67587]: cluster 2024-09-13T00:27:49.946112+0000 mgr.y (mgr.24661) 1521 : cluster [DBG] pgmap v2485: 169 pgs: 169 active+clean; 1.7 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 8.3 MiB/s rd, 43 MiB/s wr, 356 op/s 2024-09-13T00:27:51.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:51 smithi181 bash[67587]: cluster 2024-09-13T00:27:50.304133+0000 mon.a (mon.0) 2654 : cluster [DBG] osdmap e1404: 8 total, 8 up, 8 in 2024-09-13T00:27:51.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:51 smithi136 bash[75164]: cluster 2024-09-13T00:27:49.946112+0000 mgr.y (mgr.24661) 1521 : cluster [DBG] pgmap v2485: 169 pgs: 169 active+clean; 1.7 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 8.3 MiB/s rd, 43 MiB/s wr, 356 op/s 2024-09-13T00:27:51.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:51 smithi136 bash[75164]: cluster 2024-09-13T00:27:50.304133+0000 mon.a (mon.0) 2654 : cluster [DBG] osdmap e1404: 8 total, 8 up, 8 in 2024-09-13T00:27:51.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:51 smithi136 bash[76371]: cluster 2024-09-13T00:27:49.946112+0000 mgr.y (mgr.24661) 1521 : cluster [DBG] pgmap v2485: 169 pgs: 169 active+clean; 1.7 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 8.3 MiB/s rd, 43 MiB/s wr, 356 op/s 2024-09-13T00:27:51.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:51 smithi136 bash[76371]: cluster 2024-09-13T00:27:50.304133+0000 mon.a (mon.0) 2654 : cluster [DBG] osdmap e1404: 8 total, 8 up, 8 in 2024-09-13T00:27:52.290 INFO:tasks.workunit.client.0.smithi136.stdout:migration_execute 2024-09-13T00:27:52.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:27:52 smithi181 bash[95505]: ts=2024-09-13T00:27:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:27:52.975 INFO:tasks.workunit.client.0.smithi136.stdout:migration_commit 2024-09-13T00:27:53.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:53 smithi181 bash[67587]: cluster 2024-09-13T00:27:51.947723+0000 mgr.y (mgr.24661) 1522 : cluster [DBG] pgmap v2487: 169 pgs: 169 active+clean; 1.8 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 63 MiB/s rd, 17 MiB/s wr, 314 op/s 2024-09-13T00:27:53.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:53 smithi136 bash[75164]: cluster 2024-09-13T00:27:51.947723+0000 mgr.y (mgr.24661) 1522 : cluster [DBG] pgmap v2487: 169 pgs: 169 active+clean; 1.8 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 63 MiB/s rd, 17 MiB/s wr, 314 op/s 2024-09-13T00:27:53.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:53 smithi136 bash[76371]: cluster 2024-09-13T00:27:51.947723+0000 mgr.y (mgr.24661) 1522 : cluster [DBG] pgmap v2487: 169 pgs: 169 active+clean; 1.8 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 63 MiB/s rd, 17 MiB/s wr, 314 op/s 2024-09-13T00:27:54.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:54 smithi181 bash[67587]: audit 2024-09-13T00:27:53.816237+0000 mon.a (mon.0) 2655 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:27:54.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:54 smithi136 bash[76371]: audit 2024-09-13T00:27:53.816237+0000 mon.a (mon.0) 2655 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:27:54.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:54 smithi136 bash[75164]: audit 2024-09-13T00:27:53.816237+0000 mon.a (mon.0) 2655 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:27:54.983 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.Stress2 (22846 ms) 2024-09-13T00:27:54.983 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMigration.StressLive 2024-09-13T00:27:55.067 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 83886080 2024-09-13T00:27:55.098 INFO:tasks.workunit.client.0.smithi136.stdout:write: A 17222796~1232075 2024-09-13T00:27:55.135 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 12252744~5624361 2024-09-13T00:27:55.141 INFO:tasks.workunit.client.0.smithi136.stdout:write: A 46089259~3093941 2024-09-13T00:27:55.189 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 67955919~4549192 2024-09-13T00:27:55.189 INFO:tasks.workunit.client.0.smithi136.stdout:write: A 54635311~4359330 2024-09-13T00:27:55.246 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 39230953~6711452 2024-09-13T00:27:55.252 INFO:tasks.workunit.client.0.smithi136.stdout:write: A 64176256~7061418 2024-09-13T00:27:55.312 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 49471152~6368472 2024-09-13T00:27:55.322 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap0 2024-09-13T00:27:55.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:55 smithi181 bash[67587]: cluster 2024-09-13T00:27:53.948960+0000 mgr.y (mgr.24661) 1523 : cluster [DBG] pgmap v2488: 169 pgs: 169 active+clean; 1.8 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 60 MiB/s rd, 13 MiB/s wr, 364 op/s 2024-09-13T00:27:55.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:55 smithi136 bash[75164]: cluster 2024-09-13T00:27:53.948960+0000 mgr.y (mgr.24661) 1523 : cluster [DBG] pgmap v2488: 169 pgs: 169 active+clean; 1.8 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 60 MiB/s rd, 13 MiB/s wr, 364 op/s 2024-09-13T00:27:55.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:55 smithi136 bash[76371]: cluster 2024-09-13T00:27:53.948960+0000 mgr.y (mgr.24661) 1523 : cluster [DBG] pgmap v2488: 169 pgs: 169 active+clean; 1.8 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 60 MiB/s rd, 13 MiB/s wr, 364 op/s 2024-09-13T00:27:57.371 INFO:tasks.workunit.client.0.smithi136.stdout:snap_protect: snap0 2024-09-13T00:27:57.400 INFO:tasks.workunit.client.0.smithi136.stdout:clone image185 -> image188 2024-09-13T00:27:57.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:57 smithi181 bash[67587]: cluster 2024-09-13T00:27:55.950211+0000 mgr.y (mgr.24661) 1524 : cluster [DBG] pgmap v2489: 169 pgs: 169 active+clean; 1.8 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 82 MiB/s rd, 12 MiB/s wr, 439 op/s 2024-09-13T00:27:57.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:57 smithi181 bash[67587]: cluster 2024-09-13T00:27:56.337046+0000 mon.a (mon.0) 2656 : cluster [DBG] osdmap e1405: 8 total, 8 up, 8 in 2024-09-13T00:27:57.658 INFO:tasks.workunit.client.0.smithi136.stdout:write: B 13830648~4447794 2024-09-13T00:27:57.730 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 7045727~8105646 2024-09-13T00:27:57.746 INFO:tasks.workunit.client.0.smithi136.stdout:write: B 47415754~7803315 2024-09-13T00:27:57.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:57 smithi136 bash[75164]: cluster 2024-09-13T00:27:55.950211+0000 mgr.y (mgr.24661) 1524 : cluster [DBG] pgmap v2489: 169 pgs: 169 active+clean; 1.8 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 82 MiB/s rd, 12 MiB/s wr, 439 op/s 2024-09-13T00:27:57.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:57 smithi136 bash[75164]: cluster 2024-09-13T00:27:56.337046+0000 mon.a (mon.0) 2656 : cluster [DBG] osdmap e1405: 8 total, 8 up, 8 in 2024-09-13T00:27:57.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:57 smithi136 bash[76371]: cluster 2024-09-13T00:27:55.950211+0000 mgr.y (mgr.24661) 1524 : cluster [DBG] pgmap v2489: 169 pgs: 169 active+clean; 1.8 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 82 MiB/s rd, 12 MiB/s wr, 439 op/s 2024-09-13T00:27:57.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:57 smithi136 bash[76371]: cluster 2024-09-13T00:27:56.337046+0000 mon.a (mon.0) 2656 : cluster [DBG] osdmap e1405: 8 total, 8 up, 8 in 2024-09-13T00:27:57.847 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 52863161~3559307 2024-09-13T00:27:57.854 INFO:tasks.workunit.client.0.smithi136.stdout:write: B 58891683~6125237 2024-09-13T00:27:57.933 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 15122663~3793986 2024-09-13T00:27:57.940 INFO:tasks.workunit.client.0.smithi136.stdout:write: B 38580493~4365947 2024-09-13T00:27:57.990 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 26767524~7179635 2024-09-13T00:27:58.009 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap1 2024-09-13T00:27:58.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:58 smithi181 bash[67587]: cluster 2024-09-13T00:27:57.345759+0000 mon.a (mon.0) 2657 : cluster [DBG] osdmap e1406: 8 total, 8 up, 8 in 2024-09-13T00:27:58.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:58 smithi181 bash[67587]: cluster 2024-09-13T00:27:57.951259+0000 mgr.y (mgr.24661) 1525 : cluster [DBG] pgmap v2492: 169 pgs: 169 active+clean; 1.8 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 147 MiB/s rd, 9.6 MiB/s wr, 494 op/s 2024-09-13T00:27:58.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:58 smithi136 bash[75164]: cluster 2024-09-13T00:27:57.345759+0000 mon.a (mon.0) 2657 : cluster [DBG] osdmap e1406: 8 total, 8 up, 8 in 2024-09-13T00:27:58.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:58 smithi136 bash[75164]: cluster 2024-09-13T00:27:57.951259+0000 mgr.y (mgr.24661) 1525 : cluster [DBG] pgmap v2492: 169 pgs: 169 active+clean; 1.8 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 147 MiB/s rd, 9.6 MiB/s wr, 494 op/s 2024-09-13T00:27:58.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:58 smithi136 bash[76371]: cluster 2024-09-13T00:27:57.345759+0000 mon.a (mon.0) 2657 : cluster [DBG] osdmap e1406: 8 total, 8 up, 8 in 2024-09-13T00:27:58.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:58 smithi136 bash[76371]: cluster 2024-09-13T00:27:57.951259+0000 mgr.y (mgr.24661) 1525 : cluster [DBG] pgmap v2492: 169 pgs: 169 active+clean; 1.8 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 147 MiB/s rd, 9.6 MiB/s wr, 494 op/s 2024-09-13T00:27:59.387 INFO:tasks.workunit.client.0.smithi136.stdout:snap_protect: snap1 2024-09-13T00:27:59.418 INFO:tasks.workunit.client.0.smithi136.stdout:clone image188 -> image190 2024-09-13T00:27:59.749 INFO:tasks.workunit.client.0.smithi136.stdout:write: C 38605213~5747920 2024-09-13T00:27:59.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:27:59 smithi136 bash[75164]: cluster 2024-09-13T00:27:58.354446+0000 mon.a (mon.0) 2658 : cluster [DBG] osdmap e1407: 8 total, 8 up, 8 in 2024-09-13T00:27:59.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:27:59 smithi136 bash[76371]: cluster 2024-09-13T00:27:58.354446+0000 mon.a (mon.0) 2658 : cluster [DBG] osdmap e1407: 8 total, 8 up, 8 in 2024-09-13T00:27:59.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:27:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:27:59] "GET /metrics HTTP/1.1" 200 38997 "" "Prometheus/2.43.0" 2024-09-13T00:27:59.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:27:59 smithi181 bash[67587]: cluster 2024-09-13T00:27:58.354446+0000 mon.a (mon.0) 2658 : cluster [DBG] osdmap e1407: 8 total, 8 up, 8 in 2024-09-13T00:27:59.854 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 52050290~2732023 2024-09-13T00:27:59.906 INFO:tasks.workunit.client.0.smithi136.stdout:write: C 26220692~965836 2024-09-13T00:27:59.930 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 8027255~7395163 2024-09-13T00:27:59.952 INFO:tasks.workunit.client.0.smithi136.stdout:write: C 64095283~6510868 2024-09-13T00:28:00.060 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 76719321~7166759 2024-09-13T00:28:00.075 INFO:tasks.workunit.client.0.smithi136.stdout:write: C 53557594~3964759 2024-09-13T00:28:00.104 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 3681797~804847 2024-09-13T00:28:00.117 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap2 2024-09-13T00:28:00.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:00 smithi136 bash[75164]: cluster 2024-09-13T00:27:59.356547+0000 mon.a (mon.0) 2659 : cluster [DBG] osdmap e1408: 8 total, 8 up, 8 in 2024-09-13T00:28:00.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:00 smithi136 bash[75164]: cluster 2024-09-13T00:27:59.952389+0000 mgr.y (mgr.24661) 1526 : cluster [DBG] pgmap v2495: 169 pgs: 169 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 144 MiB/s rd, 16 MiB/s wr, 510 op/s 2024-09-13T00:28:00.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:00 smithi136 bash[76371]: cluster 2024-09-13T00:27:59.356547+0000 mon.a (mon.0) 2659 : cluster [DBG] osdmap e1408: 8 total, 8 up, 8 in 2024-09-13T00:28:00.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:00 smithi136 bash[76371]: cluster 2024-09-13T00:27:59.952389+0000 mgr.y (mgr.24661) 1526 : cluster [DBG] pgmap v2495: 169 pgs: 169 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 144 MiB/s rd, 16 MiB/s wr, 510 op/s 2024-09-13T00:28:00.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:00 smithi181 bash[67587]: cluster 2024-09-13T00:27:59.356547+0000 mon.a (mon.0) 2659 : cluster [DBG] osdmap e1408: 8 total, 8 up, 8 in 2024-09-13T00:28:00.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:00 smithi181 bash[67587]: cluster 2024-09-13T00:27:59.952389+0000 mgr.y (mgr.24661) 1526 : cluster [DBG] pgmap v2495: 169 pgs: 169 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 144 MiB/s rd, 16 MiB/s wr, 510 op/s 2024-09-13T00:28:01.412 INFO:tasks.workunit.client.0.smithi136.stdout:snap_protect: snap2 2024-09-13T00:28:01.440 INFO:tasks.workunit.client.0.smithi136.stdout:clone image190 -> image192 2024-09-13T00:28:01.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:01 smithi136 bash[75164]: cluster 2024-09-13T00:28:00.380215+0000 mon.a (mon.0) 2660 : cluster [DBG] osdmap e1409: 8 total, 8 up, 8 in 2024-09-13T00:28:01.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:01 smithi136 bash[76371]: cluster 2024-09-13T00:28:00.380215+0000 mon.a (mon.0) 2660 : cluster [DBG] osdmap e1409: 8 total, 8 up, 8 in 2024-09-13T00:28:01.848 INFO:tasks.workunit.client.0.smithi136.stdout:write: D 5442372~765903 2024-09-13T00:28:01.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:01 smithi181 bash[67587]: cluster 2024-09-13T00:28:00.380215+0000 mon.a (mon.0) 2660 : cluster [DBG] osdmap e1409: 8 total, 8 up, 8 in 2024-09-13T00:28:01.887 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 15279748~2751068 2024-09-13T00:28:01.905 INFO:tasks.workunit.client.0.smithi136.stdout:write: D 48488094~5613799 2024-09-13T00:28:02.007 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 9979746~5236588 2024-09-13T00:28:02.021 INFO:tasks.workunit.client.0.smithi136.stdout:write: D 12416223~3182507 2024-09-13T00:28:02.059 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 3684096~3192839 2024-09-13T00:28:02.074 INFO:tasks.workunit.client.0.smithi136.stdout:write: D 8940759~5912584 2024-09-13T00:28:02.121 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 50587647~346270 2024-09-13T00:28:02.128 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: snap3 2024-09-13T00:28:02.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:02 smithi181 bash[67587]: cluster 2024-09-13T00:28:01.389461+0000 mon.a (mon.0) 2661 : cluster [DBG] osdmap e1410: 8 total, 8 up, 8 in 2024-09-13T00:28:02.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:02 smithi181 bash[67587]: cluster 2024-09-13T00:28:01.953543+0000 mgr.y (mgr.24661) 1527 : cluster [DBG] pgmap v2498: 169 pgs: 169 active+clean; 2.0 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 18 MiB/s rd, 28 MiB/s wr, 332 op/s 2024-09-13T00:28:02.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:28:02 smithi181 bash[95505]: ts=2024-09-13T00:28:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:28:02.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:02 smithi136 bash[75164]: cluster 2024-09-13T00:28:01.389461+0000 mon.a (mon.0) 2661 : cluster [DBG] osdmap e1410: 8 total, 8 up, 8 in 2024-09-13T00:28:02.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:02 smithi136 bash[75164]: cluster 2024-09-13T00:28:01.953543+0000 mgr.y (mgr.24661) 1527 : cluster [DBG] pgmap v2498: 169 pgs: 169 active+clean; 2.0 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 18 MiB/s rd, 28 MiB/s wr, 332 op/s 2024-09-13T00:28:02.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:02 smithi136 bash[76371]: cluster 2024-09-13T00:28:01.389461+0000 mon.a (mon.0) 2661 : cluster [DBG] osdmap e1410: 8 total, 8 up, 8 in 2024-09-13T00:28:02.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:02 smithi136 bash[76371]: cluster 2024-09-13T00:28:01.953543+0000 mgr.y (mgr.24661) 1527 : cluster [DBG] pgmap v2498: 169 pgs: 169 active+clean; 2.0 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 18 MiB/s rd, 28 MiB/s wr, 332 op/s 2024-09-13T00:28:03.444 INFO:tasks.workunit.client.0.smithi136.stdout:snap_protect: snap3 2024-09-13T00:28:03.473 INFO:tasks.workunit.client.0.smithi136.stdout:clone image192 -> image194 2024-09-13T00:28:03.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:03 smithi136 bash[75164]: cluster 2024-09-13T00:28:02.400044+0000 mon.a (mon.0) 2662 : cluster [DBG] osdmap e1411: 8 total, 8 up, 8 in 2024-09-13T00:28:03.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:03 smithi136 bash[76371]: cluster 2024-09-13T00:28:02.400044+0000 mon.a (mon.0) 2662 : cluster [DBG] osdmap e1411: 8 total, 8 up, 8 in 2024-09-13T00:28:03.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:03 smithi181 bash[67587]: cluster 2024-09-13T00:28:02.400044+0000 mon.a (mon.0) 2662 : cluster [DBG] osdmap e1411: 8 total, 8 up, 8 in 2024-09-13T00:28:03.958 INFO:tasks.workunit.client.0.smithi136.stdout:migration_prepare 2024-09-13T00:28:04.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:04 smithi136 bash[75164]: cluster 2024-09-13T00:28:03.401304+0000 mon.a (mon.0) 2663 : cluster [DBG] osdmap e1412: 8 total, 8 up, 8 in 2024-09-13T00:28:04.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:04 smithi136 bash[75164]: cluster 2024-09-13T00:28:03.954599+0000 mgr.y (mgr.24661) 1528 : cluster [DBG] pgmap v2501: 169 pgs: 169 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 6.7 MiB/s rd, 16 MiB/s wr, 290 op/s 2024-09-13T00:28:04.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:04 smithi136 bash[76371]: cluster 2024-09-13T00:28:03.401304+0000 mon.a (mon.0) 2663 : cluster [DBG] osdmap e1412: 8 total, 8 up, 8 in 2024-09-13T00:28:04.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:04 smithi136 bash[76371]: cluster 2024-09-13T00:28:03.954599+0000 mgr.y (mgr.24661) 1528 : cluster [DBG] pgmap v2501: 169 pgs: 169 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 6.7 MiB/s rd, 16 MiB/s wr, 290 op/s 2024-09-13T00:28:04.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:04 smithi181 bash[67587]: cluster 2024-09-13T00:28:03.401304+0000 mon.a (mon.0) 2663 : cluster [DBG] osdmap e1412: 8 total, 8 up, 8 in 2024-09-13T00:28:04.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:04 smithi181 bash[67587]: cluster 2024-09-13T00:28:03.954599+0000 mgr.y (mgr.24661) 1528 : cluster [DBG] pgmap v2501: 169 pgs: 169 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 6.7 MiB/s rd, 16 MiB/s wr, 290 op/s 2024-09-13T00:28:05.073 INFO:tasks.workunit.client.0.smithi136.stdout:migration_execute 2024-09-13T00:28:05.073 INFO:tasks.workunit.client.0.smithi136.stdout:write: a 51608996~1310867 2024-09-13T00:28:05.141 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 66541992~950753 2024-09-13T00:28:05.195 INFO:tasks.workunit.client.0.smithi136.stdout:write: a 50060553~6652376 2024-09-13T00:28:05.268 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 2228527~3184542 2024-09-13T00:28:05.308 INFO:tasks.workunit.client.0.smithi136.stdout:write: a 20766605~2954851 2024-09-13T00:28:05.355 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 30003740~5910325 2024-09-13T00:28:05.393 INFO:tasks.workunit.client.0.smithi136.stdout:write: a 40230660~4208944 2024-09-13T00:28:05.461 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 57291660~1891680 2024-09-13T00:28:05.492 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: user0 2024-09-13T00:28:06.633 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 133334599 2024-09-13T00:28:06.660 INFO:tasks.workunit.client.0.smithi136.stdout:write: b 122999769~5116458 2024-09-13T00:28:06.686 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 62163129~708009 2024-09-13T00:28:06.687 INFO:tasks.workunit.client.0.smithi136.stdout:write: b 108539342~6054356 2024-09-13T00:28:06.699 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 115279623~3594453 2024-09-13T00:28:06.700 INFO:tasks.workunit.client.0.smithi136.stdout:write: b 101073738~4452999 2024-09-13T00:28:06.712 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 77807854~3834332 2024-09-13T00:28:06.718 INFO:tasks.workunit.client.0.smithi136.stdout:write: b 107574753~5252830 2024-09-13T00:28:06.754 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 49912881~8001141 2024-09-13T00:28:06.754 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: user1 2024-09-13T00:28:07.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:06 smithi136 bash[75164]: cluster 2024-09-13T00:28:05.594272+0000 mon.a (mon.0) 2664 : cluster [DBG] osdmap e1413: 8 total, 8 up, 8 in 2024-09-13T00:28:07.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:06 smithi136 bash[75164]: cluster 2024-09-13T00:28:05.955602+0000 mgr.y (mgr.24661) 1529 : cluster [DBG] pgmap v2503: 169 pgs: 169 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 17 MiB/s rd, 13 MiB/s wr, 438 op/s 2024-09-13T00:28:07.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:06 smithi136 bash[76371]: cluster 2024-09-13T00:28:05.594272+0000 mon.a (mon.0) 2664 : cluster [DBG] osdmap e1413: 8 total, 8 up, 8 in 2024-09-13T00:28:07.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:06 smithi136 bash[76371]: cluster 2024-09-13T00:28:05.955602+0000 mgr.y (mgr.24661) 1529 : cluster [DBG] pgmap v2503: 169 pgs: 169 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 17 MiB/s rd, 13 MiB/s wr, 438 op/s 2024-09-13T00:28:07.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:06 smithi181 bash[67587]: cluster 2024-09-13T00:28:05.594272+0000 mon.a (mon.0) 2664 : cluster [DBG] osdmap e1413: 8 total, 8 up, 8 in 2024-09-13T00:28:07.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:06 smithi181 bash[67587]: cluster 2024-09-13T00:28:05.955602+0000 mgr.y (mgr.24661) 1529 : cluster [DBG] pgmap v2503: 169 pgs: 169 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 17 MiB/s rd, 13 MiB/s wr, 438 op/s 2024-09-13T00:28:08.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:07 smithi136 bash[75164]: cluster 2024-09-13T00:28:06.606685+0000 mon.a (mon.0) 2665 : cluster [DBG] osdmap e1414: 8 total, 8 up, 8 in 2024-09-13T00:28:08.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:07 smithi136 bash[76371]: cluster 2024-09-13T00:28:06.606685+0000 mon.a (mon.0) 2665 : cluster [DBG] osdmap e1414: 8 total, 8 up, 8 in 2024-09-13T00:28:08.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:07 smithi181 bash[67587]: cluster 2024-09-13T00:28:06.606685+0000 mon.a (mon.0) 2665 : cluster [DBG] osdmap e1414: 8 total, 8 up, 8 in 2024-09-13T00:28:08.653 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 91476800 2024-09-13T00:28:08.717 INFO:tasks.workunit.client.0.smithi136.stdout:write: c 48498294~5932488 2024-09-13T00:28:08.740 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 84772971~104828 2024-09-13T00:28:08.740 INFO:tasks.workunit.client.0.smithi136.stdout:write: c 65533297~3363651 2024-09-13T00:28:08.752 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 81064563~5336647 2024-09-13T00:28:08.753 INFO:tasks.workunit.client.0.smithi136.stdout:write: c 31186535~4302774 2024-09-13T00:28:08.770 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 7476031~6143999 2024-09-13T00:28:08.772 INFO:tasks.workunit.client.0.smithi136.stdout:write: c 14042281~6345198 2024-09-13T00:28:08.791 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 66571352~3119668 2024-09-13T00:28:08.798 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: user2 2024-09-13T00:28:09.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:08 smithi136 bash[75164]: cluster 2024-09-13T00:28:07.613544+0000 mon.a (mon.0) 2666 : cluster [DBG] osdmap e1415: 8 total, 8 up, 8 in 2024-09-13T00:28:09.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:08 smithi136 bash[75164]: cluster 2024-09-13T00:28:07.956585+0000 mgr.y (mgr.24661) 1530 : cluster [DBG] pgmap v2506: 169 pgs: 169 active+clean; 2.1 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 33 MiB/s rd, 22 MiB/s wr, 742 op/s 2024-09-13T00:28:09.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:08 smithi136 bash[76371]: cluster 2024-09-13T00:28:07.613544+0000 mon.a (mon.0) 2666 : cluster [DBG] osdmap e1415: 8 total, 8 up, 8 in 2024-09-13T00:28:09.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:08 smithi136 bash[76371]: cluster 2024-09-13T00:28:07.956585+0000 mgr.y (mgr.24661) 1530 : cluster [DBG] pgmap v2506: 169 pgs: 169 active+clean; 2.1 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 33 MiB/s rd, 22 MiB/s wr, 742 op/s 2024-09-13T00:28:09.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:08 smithi181 bash[67587]: cluster 2024-09-13T00:28:07.613544+0000 mon.a (mon.0) 2666 : cluster [DBG] osdmap e1415: 8 total, 8 up, 8 in 2024-09-13T00:28:09.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:08 smithi181 bash[67587]: cluster 2024-09-13T00:28:07.956585+0000 mgr.y (mgr.24661) 1530 : cluster [DBG] pgmap v2506: 169 pgs: 169 active+clean; 2.1 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 33 MiB/s rd, 22 MiB/s wr, 742 op/s 2024-09-13T00:28:09.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:28:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:28:09] "GET /metrics HTTP/1.1" 200 38989 "" "Prometheus/2.43.0" 2024-09-13T00:28:09.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:09 smithi136 bash[76371]: cluster 2024-09-13T00:28:08.624191+0000 mon.a (mon.0) 2667 : cluster [DBG] osdmap e1416: 8 total, 8 up, 8 in 2024-09-13T00:28:09.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:09 smithi136 bash[76371]: audit 2024-09-13T00:28:08.818502+0000 mon.a (mon.0) 2668 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:28:09.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:09 smithi136 bash[75164]: cluster 2024-09-13T00:28:08.624191+0000 mon.a (mon.0) 2667 : cluster [DBG] osdmap e1416: 8 total, 8 up, 8 in 2024-09-13T00:28:09.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:09 smithi136 bash[75164]: audit 2024-09-13T00:28:08.818502+0000 mon.a (mon.0) 2668 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:28:10.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:09 smithi181 bash[67587]: cluster 2024-09-13T00:28:08.624191+0000 mon.a (mon.0) 2667 : cluster [DBG] osdmap e1416: 8 total, 8 up, 8 in 2024-09-13T00:28:10.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:09 smithi181 bash[67587]: audit 2024-09-13T00:28:08.818502+0000 mon.a (mon.0) 2668 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:28:10.672 INFO:tasks.workunit.client.0.smithi136.stdout:write: d 29305480~785416 2024-09-13T00:28:10.692 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 13140517~6376660 2024-09-13T00:28:10.692 INFO:tasks.workunit.client.0.smithi136.stdout:write: d 21487815~7018448 2024-09-13T00:28:10.722 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 8933344~8198081 2024-09-13T00:28:10.729 INFO:tasks.workunit.client.0.smithi136.stdout:write: d 41290928~7181605 2024-09-13T00:28:10.751 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 16576741~4966165 2024-09-13T00:28:10.772 INFO:tasks.workunit.client.0.smithi136.stdout:write: d 68743714~1144392 2024-09-13T00:28:10.773 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 30062078~2540461 2024-09-13T00:28:10.773 INFO:tasks.workunit.client.0.smithi136.stdout:snap_create: user3 2024-09-13T00:28:11.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:10 smithi136 bash[75164]: cluster 2024-09-13T00:28:09.627079+0000 mon.a (mon.0) 2669 : cluster [DBG] osdmap e1417: 8 total, 8 up, 8 in 2024-09-13T00:28:11.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:10 smithi136 bash[75164]: cluster 2024-09-13T00:28:09.957361+0000 mgr.y (mgr.24661) 1531 : cluster [DBG] pgmap v2509: 169 pgs: 169 active+clean; 2.1 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 26 MiB/s rd, 39 MiB/s wr, 839 op/s 2024-09-13T00:28:11.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:10 smithi136 bash[76371]: cluster 2024-09-13T00:28:09.627079+0000 mon.a (mon.0) 2669 : cluster [DBG] osdmap e1417: 8 total, 8 up, 8 in 2024-09-13T00:28:11.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:10 smithi136 bash[76371]: cluster 2024-09-13T00:28:09.957361+0000 mgr.y (mgr.24661) 1531 : cluster [DBG] pgmap v2509: 169 pgs: 169 active+clean; 2.1 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 26 MiB/s rd, 39 MiB/s wr, 839 op/s 2024-09-13T00:28:11.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:10 smithi181 bash[67587]: cluster 2024-09-13T00:28:09.627079+0000 mon.a (mon.0) 2669 : cluster [DBG] osdmap e1417: 8 total, 8 up, 8 in 2024-09-13T00:28:11.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:10 smithi181 bash[67587]: cluster 2024-09-13T00:28:09.957361+0000 mgr.y (mgr.24661) 1531 : cluster [DBG] pgmap v2509: 169 pgs: 169 active+clean; 2.1 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 26 MiB/s rd, 39 MiB/s wr, 839 op/s 2024-09-13T00:28:12.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:11 smithi136 bash[76371]: cluster 2024-09-13T00:28:10.649196+0000 mon.a (mon.0) 2670 : cluster [DBG] osdmap e1418: 8 total, 8 up, 8 in 2024-09-13T00:28:12.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:11 smithi136 bash[75164]: cluster 2024-09-13T00:28:10.649196+0000 mon.a (mon.0) 2670 : cluster [DBG] osdmap e1418: 8 total, 8 up, 8 in 2024-09-13T00:28:12.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:11 smithi181 bash[67587]: cluster 2024-09-13T00:28:10.649196+0000 mon.a (mon.0) 2670 : cluster [DBG] osdmap e1418: 8 total, 8 up, 8 in 2024-09-13T00:28:12.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:28:12 smithi181 bash[95505]: ts=2024-09-13T00:28:12.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:28:12.705 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 142842137 2024-09-13T00:28:12.755 INFO:tasks.workunit.client.0.smithi136.stdout:write: 1 14284213~3571053 2024-09-13T00:28:12.765 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 15176976~1785526 2024-09-13T00:28:12.788 INFO:tasks.workunit.client.0.smithi136.stdout:write: 2 28568427~3571053 2024-09-13T00:28:12.791 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 29461190~1785526 2024-09-13T00:28:12.818 INFO:tasks.workunit.client.0.smithi136.stdout:write: 3 42852641~3571053 2024-09-13T00:28:12.820 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 43745404~1785526 2024-09-13T00:28:12.849 INFO:tasks.workunit.client.0.smithi136.stdout:write: 4 57136854~3571053 2024-09-13T00:28:12.851 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 58029617~1785526 2024-09-13T00:28:12.876 INFO:tasks.workunit.client.0.smithi136.stdout:write: 5 71421068~3571053 2024-09-13T00:28:12.878 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 72313831~1785526 2024-09-13T00:28:13.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:12 smithi136 bash[76371]: cluster 2024-09-13T00:28:11.649232+0000 mon.a (mon.0) 2671 : cluster [DBG] osdmap e1419: 8 total, 8 up, 8 in 2024-09-13T00:28:13.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:12 smithi136 bash[76371]: cluster 2024-09-13T00:28:11.958445+0000 mgr.y (mgr.24661) 1532 : cluster [DBG] pgmap v2512: 169 pgs: 169 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 8.0 MiB/s rd, 31 MiB/s wr, 519 op/s 2024-09-13T00:28:13.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:12 smithi136 bash[75164]: cluster 2024-09-13T00:28:11.649232+0000 mon.a (mon.0) 2671 : cluster [DBG] osdmap e1419: 8 total, 8 up, 8 in 2024-09-13T00:28:13.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:12 smithi136 bash[75164]: cluster 2024-09-13T00:28:11.958445+0000 mgr.y (mgr.24661) 1532 : cluster [DBG] pgmap v2512: 169 pgs: 169 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 8.0 MiB/s rd, 31 MiB/s wr, 519 op/s 2024-09-13T00:28:13.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:12 smithi181 bash[67587]: cluster 2024-09-13T00:28:11.649232+0000 mon.a (mon.0) 2671 : cluster [DBG] osdmap e1419: 8 total, 8 up, 8 in 2024-09-13T00:28:13.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:12 smithi181 bash[67587]: cluster 2024-09-13T00:28:11.958445+0000 mgr.y (mgr.24661) 1532 : cluster [DBG] pgmap v2512: 169 pgs: 169 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 8.0 MiB/s rd, 31 MiB/s wr, 519 op/s 2024-09-13T00:28:13.190 INFO:tasks.workunit.client.0.smithi136.stdout:migration_commit 2024-09-13T00:28:14.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:13 smithi136 bash[76371]: cluster 2024-09-13T00:28:12.677278+0000 mon.a (mon.0) 2672 : cluster [DBG] osdmap e1420: 8 total, 8 up, 8 in 2024-09-13T00:28:14.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:13 smithi136 bash[75164]: cluster 2024-09-13T00:28:12.677278+0000 mon.a (mon.0) 2672 : cluster [DBG] osdmap e1420: 8 total, 8 up, 8 in 2024-09-13T00:28:14.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:13 smithi181 bash[67587]: cluster 2024-09-13T00:28:12.677278+0000 mon.a (mon.0) 2672 : cluster [DBG] osdmap e1420: 8 total, 8 up, 8 in 2024-09-13T00:28:14.968 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMigration.StressLive (19985 ms) 2024-09-13T00:28:15.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:14 smithi136 bash[75164]: cluster 2024-09-13T00:28:13.959549+0000 mgr.y (mgr.24661) 1533 : cluster [DBG] pgmap v2514: 169 pgs: 169 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 3.5 MiB/s rd, 19 MiB/s wr, 305 op/s 2024-09-13T00:28:15.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:14 smithi136 bash[76371]: cluster 2024-09-13T00:28:13.959549+0000 mgr.y (mgr.24661) 1533 : cluster [DBG] pgmap v2514: 169 pgs: 169 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 3.5 MiB/s rd, 19 MiB/s wr, 305 op/s 2024-09-13T00:28:15.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:14 smithi181 bash[67587]: cluster 2024-09-13T00:28:13.959549+0000 mgr.y (mgr.24661) 1533 : cluster [DBG] pgmap v2514: 169 pgs: 169 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 3.5 MiB/s rd, 19 MiB/s wr, 305 op/s 2024-09-13T00:28:16.718 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 57 tests from TestMigration (295640 ms total) 2024-09-13T00:28:16.718 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:28:16.719 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 6 tests from TestGroup 2024-09-13T00:28:17.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:16 smithi136 bash[76371]: cluster 2024-09-13T00:28:15.698304+0000 mon.a (mon.0) 2673 : cluster [DBG] osdmap e1421: 8 total, 8 up, 8 in 2024-09-13T00:28:17.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:16 smithi136 bash[76371]: cluster 2024-09-13T00:28:15.960402+0000 mgr.y (mgr.24661) 1534 : cluster [DBG] pgmap v2516: 137 pgs: 137 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail 2024-09-13T00:28:17.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:16 smithi136 bash[75164]: cluster 2024-09-13T00:28:15.698304+0000 mon.a (mon.0) 2673 : cluster [DBG] osdmap e1421: 8 total, 8 up, 8 in 2024-09-13T00:28:17.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:16 smithi136 bash[75164]: cluster 2024-09-13T00:28:15.960402+0000 mgr.y (mgr.24661) 1534 : cluster [DBG] pgmap v2516: 137 pgs: 137 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail 2024-09-13T00:28:17.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:16 smithi181 bash[67587]: cluster 2024-09-13T00:28:15.698304+0000 mon.a (mon.0) 2673 : cluster [DBG] osdmap e1421: 8 total, 8 up, 8 in 2024-09-13T00:28:17.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:16 smithi181 bash[67587]: cluster 2024-09-13T00:28:15.960402+0000 mgr.y (mgr.24661) 1534 : cluster [DBG] pgmap v2516: 137 pgs: 137 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail 2024-09-13T00:28:18.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:17 smithi136 bash[75164]: cluster 2024-09-13T00:28:16.686667+0000 mon.a (mon.0) 2674 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:28:18.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:17 smithi136 bash[75164]: cluster 2024-09-13T00:28:16.686721+0000 mon.a (mon.0) 2675 : cluster [INF] Cluster is now healthy 2024-09-13T00:28:18.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:17 smithi136 bash[75164]: cluster 2024-09-13T00:28:16.704549+0000 mon.a (mon.0) 2676 : cluster [DBG] osdmap e1422: 8 total, 8 up, 8 in 2024-09-13T00:28:18.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:17 smithi136 bash[76371]: cluster 2024-09-13T00:28:16.686667+0000 mon.a (mon.0) 2674 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:28:18.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:17 smithi136 bash[76371]: cluster 2024-09-13T00:28:16.686721+0000 mon.a (mon.0) 2675 : cluster [INF] Cluster is now healthy 2024-09-13T00:28:18.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:17 smithi136 bash[76371]: cluster 2024-09-13T00:28:16.704549+0000 mon.a (mon.0) 2676 : cluster [DBG] osdmap e1422: 8 total, 8 up, 8 in 2024-09-13T00:28:18.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:17 smithi181 bash[67587]: cluster 2024-09-13T00:28:16.686667+0000 mon.a (mon.0) 2674 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:28:18.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:17 smithi181 bash[67587]: cluster 2024-09-13T00:28:16.686721+0000 mon.a (mon.0) 2675 : cluster [INF] Cluster is now healthy 2024-09-13T00:28:18.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:17 smithi181 bash[67587]: cluster 2024-09-13T00:28:16.704549+0000 mon.a (mon.0) 2676 : cluster [DBG] osdmap e1422: 8 total, 8 up, 8 in 2024-09-13T00:28:18.713 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestGroup.group_create 2024-09-13T00:28:19.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:18 smithi136 bash[75164]: cluster 2024-09-13T00:28:17.708588+0000 mon.a (mon.0) 2677 : cluster [DBG] osdmap e1423: 8 total, 8 up, 8 in 2024-09-13T00:28:19.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:18 smithi136 bash[75164]: audit 2024-09-13T00:28:17.711107+0000 mon.a (mon.0) 2678 : audit [INF] from='client.? 172.21.15.136:0/2944495787' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-9","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:28:19.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:18 smithi136 bash[75164]: cluster 2024-09-13T00:28:17.961144+0000 mgr.y (mgr.24661) 1535 : cluster [DBG] pgmap v2519: 137 pgs: 32 unknown, 105 active+clean; 4.6 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail 2024-09-13T00:28:19.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:18 smithi136 bash[75164]: audit 2024-09-13T00:28:18.711024+0000 mon.a (mon.0) 2679 : audit [INF] from='client.? 172.21.15.136:0/2944495787' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-9","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:28:19.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:18 smithi136 bash[75164]: cluster 2024-09-13T00:28:18.711890+0000 mon.a (mon.0) 2680 : cluster [DBG] osdmap e1424: 8 total, 8 up, 8 in 2024-09-13T00:28:19.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:18 smithi136 bash[76371]: cluster 2024-09-13T00:28:17.708588+0000 mon.a (mon.0) 2677 : cluster [DBG] osdmap e1423: 8 total, 8 up, 8 in 2024-09-13T00:28:19.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:18 smithi136 bash[76371]: audit 2024-09-13T00:28:17.711107+0000 mon.a (mon.0) 2678 : audit [INF] from='client.? 172.21.15.136:0/2944495787' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-9","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:28:19.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:18 smithi136 bash[76371]: cluster 2024-09-13T00:28:17.961144+0000 mgr.y (mgr.24661) 1535 : cluster [DBG] pgmap v2519: 137 pgs: 32 unknown, 105 active+clean; 4.6 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail 2024-09-13T00:28:19.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:18 smithi136 bash[76371]: audit 2024-09-13T00:28:18.711024+0000 mon.a (mon.0) 2679 : audit [INF] from='client.? 172.21.15.136:0/2944495787' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-9","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:28:19.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:18 smithi136 bash[76371]: cluster 2024-09-13T00:28:18.711890+0000 mon.a (mon.0) 2680 : cluster [DBG] osdmap e1424: 8 total, 8 up, 8 in 2024-09-13T00:28:19.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:18 smithi181 bash[67587]: cluster 2024-09-13T00:28:17.708588+0000 mon.a (mon.0) 2677 : cluster [DBG] osdmap e1423: 8 total, 8 up, 8 in 2024-09-13T00:28:19.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:18 smithi181 bash[67587]: audit 2024-09-13T00:28:17.711107+0000 mon.a (mon.0) 2678 : audit [INF] from='client.? 172.21.15.136:0/2944495787' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-9","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:28:19.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:18 smithi181 bash[67587]: cluster 2024-09-13T00:28:17.961144+0000 mgr.y (mgr.24661) 1535 : cluster [DBG] pgmap v2519: 137 pgs: 32 unknown, 105 active+clean; 4.6 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail 2024-09-13T00:28:19.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:18 smithi181 bash[67587]: audit 2024-09-13T00:28:18.711024+0000 mon.a (mon.0) 2679 : audit [INF] from='client.? 172.21.15.136:0/2944495787' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-9","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:28:19.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:18 smithi181 bash[67587]: cluster 2024-09-13T00:28:18.711890+0000 mon.a (mon.0) 2680 : cluster [DBG] osdmap e1424: 8 total, 8 up, 8 in 2024-09-13T00:28:19.752 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:28:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:28:19] "GET /metrics HTTP/1.1" 200 38989 "" "Prometheus/2.43.0" 2024-09-13T00:28:20.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:19 smithi136 bash[75164]: cluster 2024-09-13T00:28:18.717656+0000 mon.a (mon.0) 2681 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:28:20.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:19 smithi136 bash[76371]: cluster 2024-09-13T00:28:18.717656+0000 mon.a (mon.0) 2681 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:28:20.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:19 smithi181 bash[67587]: cluster 2024-09-13T00:28:18.717656+0000 mon.a (mon.0) 2681 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:28:20.831 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestGroup.group_create (2119 ms) 2024-09-13T00:28:20.832 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestGroup.group_createPP 2024-09-13T00:28:20.873 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestGroup.group_createPP (41 ms) 2024-09-13T00:28:20.873 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestGroup.add_image 2024-09-13T00:28:20.963 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestGroup.add_image (90 ms) 2024-09-13T00:28:20.963 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestGroup.add_imagePP 2024-09-13T00:28:21.055 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestGroup.add_imagePP (92 ms) 2024-09-13T00:28:21.055 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestGroup.add_snapshot 2024-09-13T00:28:21.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:20 smithi181 bash[67587]: cluster 2024-09-13T00:28:19.748702+0000 mon.a (mon.0) 2682 : cluster [DBG] osdmap e1425: 8 total, 8 up, 8 in 2024-09-13T00:28:21.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:20 smithi181 bash[67587]: cluster 2024-09-13T00:28:19.961856+0000 mgr.y (mgr.24661) 1536 : cluster [DBG] pgmap v2522: 137 pgs: 2 creating+peering, 25 unknown, 110 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2024-09-13T00:28:21.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:20 smithi136 bash[76371]: cluster 2024-09-13T00:28:19.748702+0000 mon.a (mon.0) 2682 : cluster [DBG] osdmap e1425: 8 total, 8 up, 8 in 2024-09-13T00:28:21.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:20 smithi136 bash[76371]: cluster 2024-09-13T00:28:19.961856+0000 mgr.y (mgr.24661) 1536 : cluster [DBG] pgmap v2522: 137 pgs: 2 creating+peering, 25 unknown, 110 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2024-09-13T00:28:21.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:20 smithi136 bash[75164]: cluster 2024-09-13T00:28:19.748702+0000 mon.a (mon.0) 2682 : cluster [DBG] osdmap e1425: 8 total, 8 up, 8 in 2024-09-13T00:28:21.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:20 smithi136 bash[75164]: cluster 2024-09-13T00:28:19.961856+0000 mgr.y (mgr.24661) 1536 : cluster [DBG] pgmap v2522: 137 pgs: 2 creating+peering, 25 unknown, 110 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2024-09-13T00:28:22.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:21 smithi181 bash[67587]: cluster 2024-09-13T00:28:20.752033+0000 mon.a (mon.0) 2683 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:28:22.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:21 smithi181 bash[67587]: cluster 2024-09-13T00:28:20.752075+0000 mon.a (mon.0) 2684 : cluster [INF] Cluster is now healthy 2024-09-13T00:28:22.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:21 smithi181 bash[67587]: cluster 2024-09-13T00:28:20.771146+0000 mon.a (mon.0) 2685 : cluster [DBG] osdmap e1426: 8 total, 8 up, 8 in 2024-09-13T00:28:22.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:21 smithi181 bash[67587]: audit 2024-09-13T00:28:21.230696+0000 mon.a (mon.0) 2686 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:28:22.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:21 smithi181 bash[67587]: audit 2024-09-13T00:28:21.755195+0000 mon.a (mon.0) 2687 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:28:22.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:21 smithi181 bash[67587]: audit 2024-09-13T00:28:21.756722+0000 mon.a (mon.0) 2688 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:28:22.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:21 smithi136 bash[76371]: cluster 2024-09-13T00:28:20.752033+0000 mon.a (mon.0) 2683 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:28:22.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:21 smithi136 bash[76371]: cluster 2024-09-13T00:28:20.752075+0000 mon.a (mon.0) 2684 : cluster [INF] Cluster is now healthy 2024-09-13T00:28:22.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:21 smithi136 bash[76371]: cluster 2024-09-13T00:28:20.771146+0000 mon.a (mon.0) 2685 : cluster [DBG] osdmap e1426: 8 total, 8 up, 8 in 2024-09-13T00:28:22.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:21 smithi136 bash[76371]: audit 2024-09-13T00:28:21.230696+0000 mon.a (mon.0) 2686 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:28:22.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:21 smithi136 bash[76371]: audit 2024-09-13T00:28:21.755195+0000 mon.a (mon.0) 2687 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:28:22.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:21 smithi136 bash[76371]: audit 2024-09-13T00:28:21.756722+0000 mon.a (mon.0) 2688 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:28:22.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:21 smithi136 bash[75164]: cluster 2024-09-13T00:28:20.752033+0000 mon.a (mon.0) 2683 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:28:22.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:21 smithi136 bash[75164]: cluster 2024-09-13T00:28:20.752075+0000 mon.a (mon.0) 2684 : cluster [INF] Cluster is now healthy 2024-09-13T00:28:22.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:21 smithi136 bash[75164]: cluster 2024-09-13T00:28:20.771146+0000 mon.a (mon.0) 2685 : cluster [DBG] osdmap e1426: 8 total, 8 up, 8 in 2024-09-13T00:28:22.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:21 smithi136 bash[75164]: audit 2024-09-13T00:28:21.230696+0000 mon.a (mon.0) 2686 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:28:22.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:21 smithi136 bash[75164]: audit 2024-09-13T00:28:21.755195+0000 mon.a (mon.0) 2687 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:28:22.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:21 smithi136 bash[75164]: audit 2024-09-13T00:28:21.756722+0000 mon.a (mon.0) 2688 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:28:22.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:28:22 smithi181 bash[95505]: ts=2024-09-13T00:28:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:28:23.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:22 smithi181 bash[67587]: audit 2024-09-13T00:28:21.766573+0000 mon.a (mon.0) 2689 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:28:23.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:22 smithi181 bash[67587]: cluster 2024-09-13T00:28:21.784278+0000 mon.a (mon.0) 2690 : cluster [DBG] osdmap e1427: 8 total, 8 up, 8 in 2024-09-13T00:28:23.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:22 smithi181 bash[67587]: cluster 2024-09-13T00:28:21.962855+0000 mgr.y (mgr.24661) 1537 : cluster [DBG] pgmap v2525: 137 pgs: 2 creating+peering, 6 unknown, 129 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 1023 B/s wr, 6 op/s 2024-09-13T00:28:23.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:22 smithi136 bash[75164]: audit 2024-09-13T00:28:21.766573+0000 mon.a (mon.0) 2689 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:28:23.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:22 smithi136 bash[75164]: cluster 2024-09-13T00:28:21.784278+0000 mon.a (mon.0) 2690 : cluster [DBG] osdmap e1427: 8 total, 8 up, 8 in 2024-09-13T00:28:23.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:22 smithi136 bash[75164]: cluster 2024-09-13T00:28:21.962855+0000 mgr.y (mgr.24661) 1537 : cluster [DBG] pgmap v2525: 137 pgs: 2 creating+peering, 6 unknown, 129 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 1023 B/s wr, 6 op/s 2024-09-13T00:28:23.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:22 smithi136 bash[76371]: audit 2024-09-13T00:28:21.766573+0000 mon.a (mon.0) 2689 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:28:23.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:22 smithi136 bash[76371]: cluster 2024-09-13T00:28:21.784278+0000 mon.a (mon.0) 2690 : cluster [DBG] osdmap e1427: 8 total, 8 up, 8 in 2024-09-13T00:28:23.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:22 smithi136 bash[76371]: cluster 2024-09-13T00:28:21.962855+0000 mgr.y (mgr.24661) 1537 : cluster [DBG] pgmap v2525: 137 pgs: 2 creating+peering, 6 unknown, 129 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 1023 B/s wr, 6 op/s 2024-09-13T00:28:24.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:23 smithi181 bash[67587]: cluster 2024-09-13T00:28:22.793481+0000 mon.a (mon.0) 2691 : cluster [DBG] osdmap e1428: 8 total, 8 up, 8 in 2024-09-13T00:28:24.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:23 smithi136 bash[75164]: cluster 2024-09-13T00:28:22.793481+0000 mon.a (mon.0) 2691 : cluster [DBG] osdmap e1428: 8 total, 8 up, 8 in 2024-09-13T00:28:24.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:23 smithi136 bash[76371]: cluster 2024-09-13T00:28:22.793481+0000 mon.a (mon.0) 2691 : cluster [DBG] osdmap e1428: 8 total, 8 up, 8 in 2024-09-13T00:28:25.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:24 smithi181 bash[67587]: cluster 2024-09-13T00:28:23.797740+0000 mon.a (mon.0) 2692 : cluster [DBG] osdmap e1429: 8 total, 8 up, 8 in 2024-09-13T00:28:25.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:24 smithi181 bash[67587]: audit 2024-09-13T00:28:23.826706+0000 mon.a (mon.0) 2693 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:28:25.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:24 smithi181 bash[67587]: audit 2024-09-13T00:28:23.828845+0000 mon.a (mon.0) 2694 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:28:25.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:24 smithi181 bash[67587]: cluster 2024-09-13T00:28:23.963550+0000 mgr.y (mgr.24661) 1538 : cluster [DBG] pgmap v2528: 137 pgs: 137 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 85 KiB/s rd, 5.2 KiB/s wr, 106 op/s 2024-09-13T00:28:25.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:24 smithi136 bash[75164]: cluster 2024-09-13T00:28:23.797740+0000 mon.a (mon.0) 2692 : cluster [DBG] osdmap e1429: 8 total, 8 up, 8 in 2024-09-13T00:28:25.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:24 smithi136 bash[75164]: audit 2024-09-13T00:28:23.826706+0000 mon.a (mon.0) 2693 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:28:25.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:24 smithi136 bash[75164]: audit 2024-09-13T00:28:23.828845+0000 mon.a (mon.0) 2694 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:28:25.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:24 smithi136 bash[75164]: cluster 2024-09-13T00:28:23.963550+0000 mgr.y (mgr.24661) 1538 : cluster [DBG] pgmap v2528: 137 pgs: 137 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 85 KiB/s rd, 5.2 KiB/s wr, 106 op/s 2024-09-13T00:28:25.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:24 smithi136 bash[76371]: cluster 2024-09-13T00:28:23.797740+0000 mon.a (mon.0) 2692 : cluster [DBG] osdmap e1429: 8 total, 8 up, 8 in 2024-09-13T00:28:25.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:24 smithi136 bash[76371]: audit 2024-09-13T00:28:23.826706+0000 mon.a (mon.0) 2693 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:28:25.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:24 smithi136 bash[76371]: audit 2024-09-13T00:28:23.828845+0000 mon.a (mon.0) 2694 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:28:25.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:24 smithi136 bash[76371]: cluster 2024-09-13T00:28:23.963550+0000 mgr.y (mgr.24661) 1538 : cluster [DBG] pgmap v2528: 137 pgs: 137 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 85 KiB/s rd, 5.2 KiB/s wr, 106 op/s 2024-09-13T00:28:26.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:25 smithi136 bash[75164]: cluster 2024-09-13T00:28:24.840766+0000 mon.a (mon.0) 2695 : cluster [DBG] osdmap e1430: 8 total, 8 up, 8 in 2024-09-13T00:28:26.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:25 smithi136 bash[76371]: cluster 2024-09-13T00:28:24.840766+0000 mon.a (mon.0) 2695 : cluster [DBG] osdmap e1430: 8 total, 8 up, 8 in 2024-09-13T00:28:26.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:25 smithi181 bash[67587]: cluster 2024-09-13T00:28:24.840766+0000 mon.a (mon.0) 2695 : cluster [DBG] osdmap e1430: 8 total, 8 up, 8 in 2024-09-13T00:28:26.932 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestGroup.add_snapshot (5877 ms) 2024-09-13T00:28:26.932 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestGroup.add_snapshotPP 2024-09-13T00:28:27.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:26 smithi136 bash[75164]: cluster 2024-09-13T00:28:25.846474+0000 mon.a (mon.0) 2696 : cluster [DBG] osdmap e1431: 8 total, 8 up, 8 in 2024-09-13T00:28:27.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:26 smithi136 bash[75164]: cluster 2024-09-13T00:28:25.964312+0000 mgr.y (mgr.24661) 1539 : cluster [DBG] pgmap v2531: 137 pgs: 137 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 135 KiB/s rd, 16 KiB/s wr, 187 op/s 2024-09-13T00:28:27.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:26 smithi136 bash[76371]: cluster 2024-09-13T00:28:25.846474+0000 mon.a (mon.0) 2696 : cluster [DBG] osdmap e1431: 8 total, 8 up, 8 in 2024-09-13T00:28:27.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:26 smithi136 bash[76371]: cluster 2024-09-13T00:28:25.964312+0000 mgr.y (mgr.24661) 1539 : cluster [DBG] pgmap v2531: 137 pgs: 137 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 135 KiB/s rd, 16 KiB/s wr, 187 op/s 2024-09-13T00:28:27.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:26 smithi181 bash[67587]: cluster 2024-09-13T00:28:25.846474+0000 mon.a (mon.0) 2696 : cluster [DBG] osdmap e1431: 8 total, 8 up, 8 in 2024-09-13T00:28:27.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:26 smithi181 bash[67587]: cluster 2024-09-13T00:28:25.964312+0000 mgr.y (mgr.24661) 1539 : cluster [DBG] pgmap v2531: 137 pgs: 137 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 135 KiB/s rd, 16 KiB/s wr, 187 op/s 2024-09-13T00:28:28.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:27 smithi136 bash[75164]: cluster 2024-09-13T00:28:26.869967+0000 mon.a (mon.0) 2697 : cluster [DBG] osdmap e1432: 8 total, 8 up, 8 in 2024-09-13T00:28:28.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:27 smithi136 bash[76371]: cluster 2024-09-13T00:28:26.869967+0000 mon.a (mon.0) 2697 : cluster [DBG] osdmap e1432: 8 total, 8 up, 8 in 2024-09-13T00:28:28.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:27 smithi181 bash[67587]: cluster 2024-09-13T00:28:26.869967+0000 mon.a (mon.0) 2697 : cluster [DBG] osdmap e1432: 8 total, 8 up, 8 in 2024-09-13T00:28:29.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:28 smithi136 bash[75164]: cluster 2024-09-13T00:28:27.882347+0000 mon.a (mon.0) 2698 : cluster [DBG] osdmap e1433: 8 total, 8 up, 8 in 2024-09-13T00:28:29.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:28 smithi136 bash[75164]: cluster 2024-09-13T00:28:27.965092+0000 mgr.y (mgr.24661) 1540 : cluster [DBG] pgmap v2534: 137 pgs: 137 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 66 KiB/s rd, 13 KiB/s wr, 102 op/s 2024-09-13T00:28:29.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:28 smithi136 bash[76371]: cluster 2024-09-13T00:28:27.882347+0000 mon.a (mon.0) 2698 : cluster [DBG] osdmap e1433: 8 total, 8 up, 8 in 2024-09-13T00:28:29.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:28 smithi136 bash[76371]: cluster 2024-09-13T00:28:27.965092+0000 mgr.y (mgr.24661) 1540 : cluster [DBG] pgmap v2534: 137 pgs: 137 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 66 KiB/s rd, 13 KiB/s wr, 102 op/s 2024-09-13T00:28:29.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:28 smithi181 bash[67587]: cluster 2024-09-13T00:28:27.882347+0000 mon.a (mon.0) 2698 : cluster [DBG] osdmap e1433: 8 total, 8 up, 8 in 2024-09-13T00:28:29.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:28 smithi181 bash[67587]: cluster 2024-09-13T00:28:27.965092+0000 mgr.y (mgr.24661) 1540 : cluster [DBG] pgmap v2534: 137 pgs: 137 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 66 KiB/s rd, 13 KiB/s wr, 102 op/s 2024-09-13T00:28:29.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:28:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:28:29] "GET /metrics HTTP/1.1" 200 36527 "" "Prometheus/2.43.0" 2024-09-13T00:28:30.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:29 smithi136 bash[75164]: cluster 2024-09-13T00:28:28.897509+0000 mon.a (mon.0) 2699 : cluster [DBG] osdmap e1434: 8 total, 8 up, 8 in 2024-09-13T00:28:30.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:29 smithi136 bash[75164]: cluster 2024-09-13T00:28:29.054558+0000 mon.a (mon.0) 2700 : cluster [DBG] osdmap e1435: 8 total, 8 up, 8 in 2024-09-13T00:28:30.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:29 smithi136 bash[76371]: cluster 2024-09-13T00:28:28.897509+0000 mon.a (mon.0) 2699 : cluster [DBG] osdmap e1434: 8 total, 8 up, 8 in 2024-09-13T00:28:30.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:29 smithi136 bash[76371]: cluster 2024-09-13T00:28:29.054558+0000 mon.a (mon.0) 2700 : cluster [DBG] osdmap e1435: 8 total, 8 up, 8 in 2024-09-13T00:28:30.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:29 smithi181 bash[67587]: cluster 2024-09-13T00:28:28.897509+0000 mon.a (mon.0) 2699 : cluster [DBG] osdmap e1434: 8 total, 8 up, 8 in 2024-09-13T00:28:30.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:29 smithi181 bash[67587]: cluster 2024-09-13T00:28:29.054558+0000 mon.a (mon.0) 2700 : cluster [DBG] osdmap e1435: 8 total, 8 up, 8 in 2024-09-13T00:28:31.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:31 smithi181 bash[67587]: cluster 2024-09-13T00:28:29.966043+0000 mgr.y (mgr.24661) 1541 : cluster [DBG] pgmap v2537: 137 pgs: 137 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 45 KiB/s rd, 5.5 KiB/s wr, 58 op/s 2024-09-13T00:28:31.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:31 smithi181 bash[67587]: cluster 2024-09-13T00:28:30.053429+0000 mon.a (mon.0) 2701 : cluster [DBG] osdmap e1436: 8 total, 8 up, 8 in 2024-09-13T00:28:31.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:31 smithi136 bash[75164]: cluster 2024-09-13T00:28:29.966043+0000 mgr.y (mgr.24661) 1541 : cluster [DBG] pgmap v2537: 137 pgs: 137 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 45 KiB/s rd, 5.5 KiB/s wr, 58 op/s 2024-09-13T00:28:31.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:31 smithi136 bash[75164]: cluster 2024-09-13T00:28:30.053429+0000 mon.a (mon.0) 2701 : cluster [DBG] osdmap e1436: 8 total, 8 up, 8 in 2024-09-13T00:28:31.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:31 smithi136 bash[76371]: cluster 2024-09-13T00:28:29.966043+0000 mgr.y (mgr.24661) 1541 : cluster [DBG] pgmap v2537: 137 pgs: 137 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 45 KiB/s rd, 5.5 KiB/s wr, 58 op/s 2024-09-13T00:28:31.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:31 smithi136 bash[76371]: cluster 2024-09-13T00:28:30.053429+0000 mon.a (mon.0) 2701 : cluster [DBG] osdmap e1436: 8 total, 8 up, 8 in 2024-09-13T00:28:32.139 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestGroup.add_snapshotPP (5207 ms) 2024-09-13T00:28:32.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:32 smithi181 bash[67587]: cluster 2024-09-13T00:28:31.063229+0000 mon.a (mon.0) 2702 : cluster [DBG] osdmap e1437: 8 total, 8 up, 8 in 2024-09-13T00:28:32.348 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:28:32 smithi181 bash[95505]: ts=2024-09-13T00:28:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:28:32.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:32 smithi136 bash[75164]: cluster 2024-09-13T00:28:31.063229+0000 mon.a (mon.0) 2702 : cluster [DBG] osdmap e1437: 8 total, 8 up, 8 in 2024-09-13T00:28:32.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:32 smithi136 bash[76371]: cluster 2024-09-13T00:28:31.063229+0000 mon.a (mon.0) 2702 : cluster [DBG] osdmap e1437: 8 total, 8 up, 8 in 2024-09-13T00:28:33.089 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 6 tests from TestGroup (13426 ms total) 2024-09-13T00:28:33.089 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:28:33.090 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 31 tests from TestDeepCopy 2024-09-13T00:28:33.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:33 smithi181 bash[67587]: cluster 2024-09-13T00:28:31.967067+0000 mgr.y (mgr.24661) 1542 : cluster [DBG] pgmap v2540: 137 pgs: 137 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 117 KiB/s rd, 10 KiB/s wr, 150 op/s 2024-09-13T00:28:33.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:33 smithi181 bash[67587]: cluster 2024-09-13T00:28:32.074074+0000 mon.a (mon.0) 2703 : cluster [DBG] osdmap e1438: 8 total, 8 up, 8 in 2024-09-13T00:28:33.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:33 smithi136 bash[75164]: cluster 2024-09-13T00:28:31.967067+0000 mgr.y (mgr.24661) 1542 : cluster [DBG] pgmap v2540: 137 pgs: 137 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 117 KiB/s rd, 10 KiB/s wr, 150 op/s 2024-09-13T00:28:33.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:33 smithi136 bash[75164]: cluster 2024-09-13T00:28:32.074074+0000 mon.a (mon.0) 2703 : cluster [DBG] osdmap e1438: 8 total, 8 up, 8 in 2024-09-13T00:28:33.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:33 smithi136 bash[76371]: cluster 2024-09-13T00:28:31.967067+0000 mgr.y (mgr.24661) 1542 : cluster [DBG] pgmap v2540: 137 pgs: 137 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 117 KiB/s rd, 10 KiB/s wr, 150 op/s 2024-09-13T00:28:33.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:33 smithi136 bash[76371]: cluster 2024-09-13T00:28:32.074074+0000 mon.a (mon.0) 2703 : cluster [DBG] osdmap e1438: 8 total, 8 up, 8 in 2024-09-13T00:28:34.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:34 smithi181 bash[67587]: cluster 2024-09-13T00:28:33.083705+0000 mon.a (mon.0) 2704 : cluster [DBG] osdmap e1439: 8 total, 8 up, 8 in 2024-09-13T00:28:34.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:34 smithi136 bash[75164]: cluster 2024-09-13T00:28:33.083705+0000 mon.a (mon.0) 2704 : cluster [DBG] osdmap e1439: 8 total, 8 up, 8 in 2024-09-13T00:28:34.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:34 smithi136 bash[76371]: cluster 2024-09-13T00:28:33.083705+0000 mon.a (mon.0) 2704 : cluster [DBG] osdmap e1439: 8 total, 8 up, 8 in 2024-09-13T00:28:35.105 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestDeepCopy.Empty 2024-09-13T00:28:35.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:35 smithi136 bash[75164]: cluster 2024-09-13T00:28:33.967857+0000 mgr.y (mgr.24661) 1543 : cluster [DBG] pgmap v2543: 105 pgs: 105 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:28:35.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:35 smithi136 bash[75164]: cluster 2024-09-13T00:28:34.097676+0000 mon.a (mon.0) 2705 : cluster [DBG] osdmap e1440: 8 total, 8 up, 8 in 2024-09-13T00:28:35.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:35 smithi136 bash[75164]: audit 2024-09-13T00:28:34.101419+0000 mon.a (mon.0) 2706 : audit [INF] from='client.? 172.21.15.136:0/459741781' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:28:35.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:35 smithi136 bash[76371]: cluster 2024-09-13T00:28:33.967857+0000 mgr.y (mgr.24661) 1543 : cluster [DBG] pgmap v2543: 105 pgs: 105 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:28:35.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:35 smithi136 bash[76371]: cluster 2024-09-13T00:28:34.097676+0000 mon.a (mon.0) 2705 : cluster [DBG] osdmap e1440: 8 total, 8 up, 8 in 2024-09-13T00:28:35.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:35 smithi136 bash[76371]: audit 2024-09-13T00:28:34.101419+0000 mon.a (mon.0) 2706 : audit [INF] from='client.? 172.21.15.136:0/459741781' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:28:35.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:35 smithi181 bash[67587]: cluster 2024-09-13T00:28:33.967857+0000 mgr.y (mgr.24661) 1543 : cluster [DBG] pgmap v2543: 105 pgs: 105 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:28:35.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:35 smithi181 bash[67587]: cluster 2024-09-13T00:28:34.097676+0000 mon.a (mon.0) 2705 : cluster [DBG] osdmap e1440: 8 total, 8 up, 8 in 2024-09-13T00:28:35.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:35 smithi181 bash[67587]: audit 2024-09-13T00:28:34.101419+0000 mon.a (mon.0) 2706 : audit [INF] from='client.? 172.21.15.136:0/459741781' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:28:36.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:36 smithi136 bash[75164]: audit 2024-09-13T00:28:35.100400+0000 mon.a (mon.0) 2707 : audit [INF] from='client.? 172.21.15.136:0/459741781' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:28:36.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:36 smithi136 bash[75164]: cluster 2024-09-13T00:28:35.100827+0000 mon.a (mon.0) 2708 : cluster [DBG] osdmap e1441: 8 total, 8 up, 8 in 2024-09-13T00:28:36.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:36 smithi136 bash[76371]: audit 2024-09-13T00:28:35.100400+0000 mon.a (mon.0) 2707 : audit [INF] from='client.? 172.21.15.136:0/459741781' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:28:36.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:36 smithi136 bash[76371]: cluster 2024-09-13T00:28:35.100827+0000 mon.a (mon.0) 2708 : cluster [DBG] osdmap e1441: 8 total, 8 up, 8 in 2024-09-13T00:28:36.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:36 smithi181 bash[67587]: audit 2024-09-13T00:28:35.100400+0000 mon.a (mon.0) 2707 : audit [INF] from='client.? 172.21.15.136:0/459741781' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:28:36.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:36 smithi181 bash[67587]: cluster 2024-09-13T00:28:35.100827+0000 mon.a (mon.0) 2708 : cluster [DBG] osdmap e1441: 8 total, 8 up, 8 in 2024-09-13T00:28:37.367 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestDeepCopy.Empty (2262 ms) 2024-09-13T00:28:37.367 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestDeepCopy.NoSnaps 2024-09-13T00:28:37.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:37 smithi136 bash[75164]: cluster 2024-09-13T00:28:35.968803+0000 mgr.y (mgr.24661) 1544 : cluster [DBG] pgmap v2546: 137 pgs: 3 creating+peering, 23 unknown, 111 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:28:37.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:37 smithi136 bash[75164]: cluster 2024-09-13T00:28:36.125272+0000 mon.a (mon.0) 2709 : cluster [DBG] osdmap e1442: 8 total, 8 up, 8 in 2024-09-13T00:28:37.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:37 smithi136 bash[76371]: cluster 2024-09-13T00:28:35.968803+0000 mgr.y (mgr.24661) 1544 : cluster [DBG] pgmap v2546: 137 pgs: 3 creating+peering, 23 unknown, 111 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:28:37.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:37 smithi136 bash[76371]: cluster 2024-09-13T00:28:36.125272+0000 mon.a (mon.0) 2709 : cluster [DBG] osdmap e1442: 8 total, 8 up, 8 in 2024-09-13T00:28:37.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:37 smithi181 bash[67587]: cluster 2024-09-13T00:28:35.968803+0000 mgr.y (mgr.24661) 1544 : cluster [DBG] pgmap v2546: 137 pgs: 3 creating+peering, 23 unknown, 111 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:28:37.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:37 smithi181 bash[67587]: cluster 2024-09-13T00:28:36.125272+0000 mon.a (mon.0) 2709 : cluster [DBG] osdmap e1442: 8 total, 8 up, 8 in 2024-09-13T00:28:37.817 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestDeepCopy.NoSnaps (450 ms) 2024-09-13T00:28:37.817 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestDeepCopy.Snaps 2024-09-13T00:28:38.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:38 smithi136 bash[75164]: cluster 2024-09-13T00:28:37.128628+0000 mon.a (mon.0) 2710 : cluster [DBG] osdmap e1443: 8 total, 8 up, 8 in 2024-09-13T00:28:38.512 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:38 smithi136 bash[76371]: cluster 2024-09-13T00:28:37.128628+0000 mon.a (mon.0) 2710 : cluster [DBG] osdmap e1443: 8 total, 8 up, 8 in 2024-09-13T00:28:38.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:38 smithi181 bash[67587]: cluster 2024-09-13T00:28:37.128628+0000 mon.a (mon.0) 2710 : cluster [DBG] osdmap e1443: 8 total, 8 up, 8 in 2024-09-13T00:28:39.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:39 smithi136 bash[76371]: cluster 2024-09-13T00:28:37.969667+0000 mgr.y (mgr.24661) 1545 : cluster [DBG] pgmap v2549: 137 pgs: 3 creating+peering, 4 unknown, 130 active+clean; 8.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4.5 KiB/s rd, 1.0 MiB/s wr, 7 op/s 2024-09-13T00:28:39.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:39 smithi136 bash[76371]: cluster 2024-09-13T00:28:38.151074+0000 mon.a (mon.0) 2711 : cluster [DBG] osdmap e1444: 8 total, 8 up, 8 in 2024-09-13T00:28:39.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:39 smithi136 bash[76371]: audit 2024-09-13T00:28:38.819417+0000 mon.a (mon.0) 2712 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:28:39.414 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:39 smithi136 bash[75164]: cluster 2024-09-13T00:28:37.969667+0000 mgr.y (mgr.24661) 1545 : cluster [DBG] pgmap v2549: 137 pgs: 3 creating+peering, 4 unknown, 130 active+clean; 8.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4.5 KiB/s rd, 1.0 MiB/s wr, 7 op/s 2024-09-13T00:28:39.414 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:39 smithi136 bash[75164]: cluster 2024-09-13T00:28:38.151074+0000 mon.a (mon.0) 2711 : cluster [DBG] osdmap e1444: 8 total, 8 up, 8 in 2024-09-13T00:28:39.414 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:39 smithi136 bash[75164]: audit 2024-09-13T00:28:38.819417+0000 mon.a (mon.0) 2712 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:28:39.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:39 smithi181 bash[67587]: cluster 2024-09-13T00:28:37.969667+0000 mgr.y (mgr.24661) 1545 : cluster [DBG] pgmap v2549: 137 pgs: 3 creating+peering, 4 unknown, 130 active+clean; 8.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4.5 KiB/s rd, 1.0 MiB/s wr, 7 op/s 2024-09-13T00:28:39.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:39 smithi181 bash[67587]: cluster 2024-09-13T00:28:38.151074+0000 mon.a (mon.0) 2711 : cluster [DBG] osdmap e1444: 8 total, 8 up, 8 in 2024-09-13T00:28:39.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:39 smithi181 bash[67587]: audit 2024-09-13T00:28:38.819417+0000 mon.a (mon.0) 2712 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:28:39.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:28:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:28:39] "GET /metrics HTTP/1.1" 200 36540 "" "Prometheus/2.43.0" 2024-09-13T00:28:40.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:40 smithi136 bash[75164]: cluster 2024-09-13T00:28:39.150619+0000 mon.a (mon.0) 2713 : cluster [DBG] osdmap e1445: 8 total, 8 up, 8 in 2024-09-13T00:28:40.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:40 smithi136 bash[76371]: cluster 2024-09-13T00:28:39.150619+0000 mon.a (mon.0) 2713 : cluster [DBG] osdmap e1445: 8 total, 8 up, 8 in 2024-09-13T00:28:40.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:40 smithi181 bash[67587]: cluster 2024-09-13T00:28:39.150619+0000 mon.a (mon.0) 2713 : cluster [DBG] osdmap e1445: 8 total, 8 up, 8 in 2024-09-13T00:28:41.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:41 smithi136 bash[76371]: cluster 2024-09-13T00:28:39.970477+0000 mgr.y (mgr.24661) 1546 : cluster [DBG] pgmap v2552: 137 pgs: 137 active+clean; 25 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.0 MiB/s rd, 5.0 MiB/s wr, 46 op/s 2024-09-13T00:28:41.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:41 smithi136 bash[76371]: cluster 2024-09-13T00:28:40.174688+0000 mon.a (mon.0) 2714 : cluster [DBG] osdmap e1446: 8 total, 8 up, 8 in 2024-09-13T00:28:41.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:41 smithi136 bash[75164]: cluster 2024-09-13T00:28:39.970477+0000 mgr.y (mgr.24661) 1546 : cluster [DBG] pgmap v2552: 137 pgs: 137 active+clean; 25 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.0 MiB/s rd, 5.0 MiB/s wr, 46 op/s 2024-09-13T00:28:41.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:41 smithi136 bash[75164]: cluster 2024-09-13T00:28:40.174688+0000 mon.a (mon.0) 2714 : cluster [DBG] osdmap e1446: 8 total, 8 up, 8 in 2024-09-13T00:28:41.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:41 smithi181 bash[67587]: cluster 2024-09-13T00:28:39.970477+0000 mgr.y (mgr.24661) 1546 : cluster [DBG] pgmap v2552: 137 pgs: 137 active+clean; 25 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.0 MiB/s rd, 5.0 MiB/s wr, 46 op/s 2024-09-13T00:28:41.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:41 smithi181 bash[67587]: cluster 2024-09-13T00:28:40.174688+0000 mon.a (mon.0) 2714 : cluster [DBG] osdmap e1446: 8 total, 8 up, 8 in 2024-09-13T00:28:41.683 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestDeepCopy.Snaps (3866 ms) 2024-09-13T00:28:41.683 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestDeepCopy.SnapDiscard 2024-09-13T00:28:42.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:42 smithi136 bash[75164]: cluster 2024-09-13T00:28:41.174325+0000 mon.a (mon.0) 2715 : cluster [DBG] osdmap e1447: 8 total, 8 up, 8 in 2024-09-13T00:28:42.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:42 smithi136 bash[76371]: cluster 2024-09-13T00:28:41.174325+0000 mon.a (mon.0) 2715 : cluster [DBG] osdmap e1447: 8 total, 8 up, 8 in 2024-09-13T00:28:42.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:42 smithi181 bash[67587]: cluster 2024-09-13T00:28:41.174325+0000 mon.a (mon.0) 2715 : cluster [DBG] osdmap e1447: 8 total, 8 up, 8 in 2024-09-13T00:28:42.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:28:42 smithi181 bash[95505]: ts=2024-09-13T00:28:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:28:43.432 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestDeepCopy.SnapDiscard (1749 ms) 2024-09-13T00:28:43.432 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestDeepCopy.CloneDiscard 2024-09-13T00:28:43.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:43 smithi136 bash[76371]: cluster 2024-09-13T00:28:41.971433+0000 mgr.y (mgr.24661) 1547 : cluster [DBG] pgmap v2555: 137 pgs: 137 active+clean; 65 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 26 MiB/s rd, 14 MiB/s wr, 200 op/s 2024-09-13T00:28:43.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:43 smithi136 bash[76371]: cluster 2024-09-13T00:28:42.197882+0000 mon.a (mon.0) 2716 : cluster [DBG] osdmap e1448: 8 total, 8 up, 8 in 2024-09-13T00:28:43.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:43 smithi136 bash[75164]: cluster 2024-09-13T00:28:41.971433+0000 mgr.y (mgr.24661) 1547 : cluster [DBG] pgmap v2555: 137 pgs: 137 active+clean; 65 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 26 MiB/s rd, 14 MiB/s wr, 200 op/s 2024-09-13T00:28:43.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:43 smithi136 bash[75164]: cluster 2024-09-13T00:28:42.197882+0000 mon.a (mon.0) 2716 : cluster [DBG] osdmap e1448: 8 total, 8 up, 8 in 2024-09-13T00:28:43.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:43 smithi181 bash[67587]: cluster 2024-09-13T00:28:41.971433+0000 mgr.y (mgr.24661) 1547 : cluster [DBG] pgmap v2555: 137 pgs: 137 active+clean; 65 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 26 MiB/s rd, 14 MiB/s wr, 200 op/s 2024-09-13T00:28:43.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:43 smithi181 bash[67587]: cluster 2024-09-13T00:28:42.197882+0000 mon.a (mon.0) 2716 : cluster [DBG] osdmap e1448: 8 total, 8 up, 8 in 2024-09-13T00:28:44.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:44 smithi136 bash[76371]: cluster 2024-09-13T00:28:43.206762+0000 mon.a (mon.0) 2717 : cluster [DBG] osdmap e1449: 8 total, 8 up, 8 in 2024-09-13T00:28:44.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:44 smithi136 bash[76371]: cluster 2024-09-13T00:28:44.056588+0000 mon.a (mon.0) 2718 : cluster [DBG] osdmap e1450: 8 total, 8 up, 8 in 2024-09-13T00:28:44.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:44 smithi136 bash[75164]: cluster 2024-09-13T00:28:43.206762+0000 mon.a (mon.0) 2717 : cluster [DBG] osdmap e1449: 8 total, 8 up, 8 in 2024-09-13T00:28:44.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:44 smithi136 bash[75164]: cluster 2024-09-13T00:28:44.056588+0000 mon.a (mon.0) 2718 : cluster [DBG] osdmap e1450: 8 total, 8 up, 8 in 2024-09-13T00:28:44.514 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestDeepCopy.CloneDiscard (1083 ms) 2024-09-13T00:28:44.514 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestDeepCopy.CloneShrink 2024-09-13T00:28:44.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:44 smithi181 bash[67587]: cluster 2024-09-13T00:28:43.206762+0000 mon.a (mon.0) 2717 : cluster [DBG] osdmap e1449: 8 total, 8 up, 8 in 2024-09-13T00:28:44.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:44 smithi181 bash[67587]: cluster 2024-09-13T00:28:44.056588+0000 mon.a (mon.0) 2718 : cluster [DBG] osdmap e1450: 8 total, 8 up, 8 in 2024-09-13T00:28:45.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:45 smithi136 bash[75164]: cluster 2024-09-13T00:28:43.972303+0000 mgr.y (mgr.24661) 1548 : cluster [DBG] pgmap v2558: 137 pgs: 137 active+clean; 65 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 22 MiB/s rd, 10 MiB/s wr, 181 op/s 2024-09-13T00:28:45.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:45 smithi136 bash[76371]: cluster 2024-09-13T00:28:43.972303+0000 mgr.y (mgr.24661) 1548 : cluster [DBG] pgmap v2558: 137 pgs: 137 active+clean; 65 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 22 MiB/s rd, 10 MiB/s wr, 181 op/s 2024-09-13T00:28:45.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:45 smithi181 bash[67587]: cluster 2024-09-13T00:28:43.972303+0000 mgr.y (mgr.24661) 1548 : cluster [DBG] pgmap v2558: 137 pgs: 137 active+clean; 65 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 22 MiB/s rd, 10 MiB/s wr, 181 op/s 2024-09-13T00:28:45.627 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestDeepCopy.CloneShrink (1112 ms) 2024-09-13T00:28:45.627 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestDeepCopy.CloneExpand 2024-09-13T00:28:46.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:46 smithi136 bash[75164]: cluster 2024-09-13T00:28:45.219880+0000 mon.a (mon.0) 2719 : cluster [DBG] osdmap e1451: 8 total, 8 up, 8 in 2024-09-13T00:28:46.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:46 smithi136 bash[76371]: cluster 2024-09-13T00:28:45.219880+0000 mon.a (mon.0) 2719 : cluster [DBG] osdmap e1451: 8 total, 8 up, 8 in 2024-09-13T00:28:46.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:46 smithi181 bash[67587]: cluster 2024-09-13T00:28:45.219880+0000 mon.a (mon.0) 2719 : cluster [DBG] osdmap e1451: 8 total, 8 up, 8 in 2024-09-13T00:28:46.758 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestDeepCopy.CloneExpand (1131 ms) 2024-09-13T00:28:46.758 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestDeepCopy.CloneHideParent 2024-09-13T00:28:47.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:47 smithi136 bash[75164]: cluster 2024-09-13T00:28:45.973270+0000 mgr.y (mgr.24661) 1549 : cluster [DBG] pgmap v2561: 137 pgs: 137 active+clean; 65 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 4.1 MiB/s rd, 9.7 KiB/s wr, 88 op/s 2024-09-13T00:28:47.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:47 smithi136 bash[75164]: cluster 2024-09-13T00:28:46.241571+0000 mon.a (mon.0) 2720 : cluster [DBG] osdmap e1452: 8 total, 8 up, 8 in 2024-09-13T00:28:47.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:47 smithi136 bash[76371]: cluster 2024-09-13T00:28:45.973270+0000 mgr.y (mgr.24661) 1549 : cluster [DBG] pgmap v2561: 137 pgs: 137 active+clean; 65 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 4.1 MiB/s rd, 9.7 KiB/s wr, 88 op/s 2024-09-13T00:28:47.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:47 smithi136 bash[76371]: cluster 2024-09-13T00:28:46.241571+0000 mon.a (mon.0) 2720 : cluster [DBG] osdmap e1452: 8 total, 8 up, 8 in 2024-09-13T00:28:47.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:47 smithi181 bash[67587]: cluster 2024-09-13T00:28:45.973270+0000 mgr.y (mgr.24661) 1549 : cluster [DBG] pgmap v2561: 137 pgs: 137 active+clean; 65 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 4.1 MiB/s rd, 9.7 KiB/s wr, 88 op/s 2024-09-13T00:28:47.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:47 smithi181 bash[67587]: cluster 2024-09-13T00:28:46.241571+0000 mon.a (mon.0) 2720 : cluster [DBG] osdmap e1452: 8 total, 8 up, 8 in 2024-09-13T00:28:48.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:48 smithi181 bash[67587]: cluster 2024-09-13T00:28:47.247456+0000 mon.a (mon.0) 2721 : cluster [DBG] osdmap e1453: 8 total, 8 up, 8 in 2024-09-13T00:28:48.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:48 smithi136 bash[75164]: cluster 2024-09-13T00:28:47.247456+0000 mon.a (mon.0) 2721 : cluster [DBG] osdmap e1453: 8 total, 8 up, 8 in 2024-09-13T00:28:48.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:48 smithi136 bash[76371]: cluster 2024-09-13T00:28:47.247456+0000 mon.a (mon.0) 2721 : cluster [DBG] osdmap e1453: 8 total, 8 up, 8 in 2024-09-13T00:28:49.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:49 smithi181 bash[67587]: cluster 2024-09-13T00:28:47.974249+0000 mgr.y (mgr.24661) 1550 : cluster [DBG] pgmap v2564: 137 pgs: 137 active+clean; 69 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 9.2 MiB/s rd, 1.0 MiB/s wr, 334 op/s 2024-09-13T00:28:49.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:49 smithi181 bash[67587]: cluster 2024-09-13T00:28:48.269310+0000 mon.a (mon.0) 2722 : cluster [DBG] osdmap e1454: 8 total, 8 up, 8 in 2024-09-13T00:28:49.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:28:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:28:49] "GET /metrics HTTP/1.1" 200 36540 "" "Prometheus/2.43.0" 2024-09-13T00:28:49.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:49 smithi136 bash[75164]: cluster 2024-09-13T00:28:47.974249+0000 mgr.y (mgr.24661) 1550 : cluster [DBG] pgmap v2564: 137 pgs: 137 active+clean; 69 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 9.2 MiB/s rd, 1.0 MiB/s wr, 334 op/s 2024-09-13T00:28:49.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:49 smithi136 bash[75164]: cluster 2024-09-13T00:28:48.269310+0000 mon.a (mon.0) 2722 : cluster [DBG] osdmap e1454: 8 total, 8 up, 8 in 2024-09-13T00:28:49.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:49 smithi136 bash[76371]: cluster 2024-09-13T00:28:47.974249+0000 mgr.y (mgr.24661) 1550 : cluster [DBG] pgmap v2564: 137 pgs: 137 active+clean; 69 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 9.2 MiB/s rd, 1.0 MiB/s wr, 334 op/s 2024-09-13T00:28:49.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:49 smithi136 bash[76371]: cluster 2024-09-13T00:28:48.269310+0000 mon.a (mon.0) 2722 : cluster [DBG] osdmap e1454: 8 total, 8 up, 8 in 2024-09-13T00:28:50.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:50 smithi181 bash[67587]: cluster 2024-09-13T00:28:49.276209+0000 mon.a (mon.0) 2723 : cluster [DBG] osdmap e1455: 8 total, 8 up, 8 in 2024-09-13T00:28:50.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:50 smithi136 bash[75164]: cluster 2024-09-13T00:28:49.276209+0000 mon.a (mon.0) 2723 : cluster [DBG] osdmap e1455: 8 total, 8 up, 8 in 2024-09-13T00:28:50.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:50 smithi136 bash[76371]: cluster 2024-09-13T00:28:49.276209+0000 mon.a (mon.0) 2723 : cluster [DBG] osdmap e1455: 8 total, 8 up, 8 in 2024-09-13T00:28:51.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:51 smithi181 bash[67587]: cluster 2024-09-13T00:28:49.975078+0000 mgr.y (mgr.24661) 1551 : cluster [DBG] pgmap v2567: 137 pgs: 137 active+clean; 69 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.3 MiB/s rd, 1.0 MiB/s wr, 427 op/s 2024-09-13T00:28:51.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:51 smithi181 bash[67587]: cluster 2024-09-13T00:28:50.285784+0000 mon.a (mon.0) 2724 : cluster [DBG] osdmap e1456: 8 total, 8 up, 8 in 2024-09-13T00:28:51.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:51 smithi136 bash[76371]: cluster 2024-09-13T00:28:49.975078+0000 mgr.y (mgr.24661) 1551 : cluster [DBG] pgmap v2567: 137 pgs: 137 active+clean; 69 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.3 MiB/s rd, 1.0 MiB/s wr, 427 op/s 2024-09-13T00:28:51.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:51 smithi136 bash[76371]: cluster 2024-09-13T00:28:50.285784+0000 mon.a (mon.0) 2724 : cluster [DBG] osdmap e1456: 8 total, 8 up, 8 in 2024-09-13T00:28:51.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:51 smithi136 bash[75164]: cluster 2024-09-13T00:28:49.975078+0000 mgr.y (mgr.24661) 1551 : cluster [DBG] pgmap v2567: 137 pgs: 137 active+clean; 69 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.3 MiB/s rd, 1.0 MiB/s wr, 427 op/s 2024-09-13T00:28:51.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:51 smithi136 bash[75164]: cluster 2024-09-13T00:28:50.285784+0000 mon.a (mon.0) 2724 : cluster [DBG] osdmap e1456: 8 total, 8 up, 8 in 2024-09-13T00:28:52.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:52 smithi181 bash[67587]: cluster 2024-09-13T00:28:51.288458+0000 mon.a (mon.0) 2725 : cluster [DBG] osdmap e1457: 8 total, 8 up, 8 in 2024-09-13T00:28:52.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:28:52 smithi181 bash[95505]: ts=2024-09-13T00:28:52.317Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:28:52.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:52 smithi136 bash[76371]: cluster 2024-09-13T00:28:51.288458+0000 mon.a (mon.0) 2725 : cluster [DBG] osdmap e1457: 8 total, 8 up, 8 in 2024-09-13T00:28:52.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:52 smithi136 bash[75164]: cluster 2024-09-13T00:28:51.288458+0000 mon.a (mon.0) 2725 : cluster [DBG] osdmap e1457: 8 total, 8 up, 8 in 2024-09-13T00:28:53.552 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestDeepCopy.CloneHideParent (6794 ms) 2024-09-13T00:28:53.552 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestDeepCopy.Clone 2024-09-13T00:28:53.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:53 smithi181 bash[67587]: cluster 2024-09-13T00:28:51.975988+0000 mgr.y (mgr.24661) 1552 : cluster [DBG] pgmap v2570: 137 pgs: 137 active+clean; 69 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 259 KiB/s rd, 20 KiB/s wr, 348 op/s 2024-09-13T00:28:53.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:53 smithi181 bash[67587]: cluster 2024-09-13T00:28:52.309411+0000 mon.a (mon.0) 2726 : cluster [DBG] osdmap e1458: 8 total, 8 up, 8 in 2024-09-13T00:28:53.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:53 smithi136 bash[76371]: cluster 2024-09-13T00:28:51.975988+0000 mgr.y (mgr.24661) 1552 : cluster [DBG] pgmap v2570: 137 pgs: 137 active+clean; 69 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 259 KiB/s rd, 20 KiB/s wr, 348 op/s 2024-09-13T00:28:53.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:53 smithi136 bash[76371]: cluster 2024-09-13T00:28:52.309411+0000 mon.a (mon.0) 2726 : cluster [DBG] osdmap e1458: 8 total, 8 up, 8 in 2024-09-13T00:28:53.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:53 smithi136 bash[75164]: cluster 2024-09-13T00:28:51.975988+0000 mgr.y (mgr.24661) 1552 : cluster [DBG] pgmap v2570: 137 pgs: 137 active+clean; 69 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 259 KiB/s rd, 20 KiB/s wr, 348 op/s 2024-09-13T00:28:53.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:53 smithi136 bash[75164]: cluster 2024-09-13T00:28:52.309411+0000 mon.a (mon.0) 2726 : cluster [DBG] osdmap e1458: 8 total, 8 up, 8 in 2024-09-13T00:28:54.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:54 smithi181 bash[67587]: cluster 2024-09-13T00:28:53.316536+0000 mon.a (mon.0) 2727 : cluster [DBG] osdmap e1459: 8 total, 8 up, 8 in 2024-09-13T00:28:54.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:54 smithi181 bash[67587]: audit 2024-09-13T00:28:53.819700+0000 mon.a (mon.0) 2728 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:28:54.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:54 smithi136 bash[75164]: cluster 2024-09-13T00:28:53.316536+0000 mon.a (mon.0) 2727 : cluster [DBG] osdmap e1459: 8 total, 8 up, 8 in 2024-09-13T00:28:54.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:54 smithi136 bash[75164]: audit 2024-09-13T00:28:53.819700+0000 mon.a (mon.0) 2728 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:28:54.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:54 smithi136 bash[76371]: cluster 2024-09-13T00:28:53.316536+0000 mon.a (mon.0) 2727 : cluster [DBG] osdmap e1459: 8 total, 8 up, 8 in 2024-09-13T00:28:54.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:54 smithi136 bash[76371]: audit 2024-09-13T00:28:53.819700+0000 mon.a (mon.0) 2728 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:28:55.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:55 smithi181 bash[67587]: cluster 2024-09-13T00:28:53.976939+0000 mgr.y (mgr.24661) 1553 : cluster [DBG] pgmap v2573: 137 pgs: 137 active+clean; 69 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 139 KiB/s rd, 12 KiB/s wr, 197 op/s 2024-09-13T00:28:55.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:55 smithi181 bash[67587]: cluster 2024-09-13T00:28:54.327370+0000 mon.a (mon.0) 2729 : cluster [DBG] osdmap e1460: 8 total, 8 up, 8 in 2024-09-13T00:28:55.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:55 smithi136 bash[75164]: cluster 2024-09-13T00:28:53.976939+0000 mgr.y (mgr.24661) 1553 : cluster [DBG] pgmap v2573: 137 pgs: 137 active+clean; 69 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 139 KiB/s rd, 12 KiB/s wr, 197 op/s 2024-09-13T00:28:55.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:55 smithi136 bash[75164]: cluster 2024-09-13T00:28:54.327370+0000 mon.a (mon.0) 2729 : cluster [DBG] osdmap e1460: 8 total, 8 up, 8 in 2024-09-13T00:28:55.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:55 smithi136 bash[76371]: cluster 2024-09-13T00:28:53.976939+0000 mgr.y (mgr.24661) 1553 : cluster [DBG] pgmap v2573: 137 pgs: 137 active+clean; 69 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 139 KiB/s rd, 12 KiB/s wr, 197 op/s 2024-09-13T00:28:55.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:55 smithi136 bash[76371]: cluster 2024-09-13T00:28:54.327370+0000 mon.a (mon.0) 2729 : cluster [DBG] osdmap e1460: 8 total, 8 up, 8 in 2024-09-13T00:28:56.027 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestDeepCopy.Clone (2474 ms) 2024-09-13T00:28:56.027 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestDeepCopy.CloneFlatten 2024-09-13T00:28:56.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:56 smithi181 bash[67587]: cluster 2024-09-13T00:28:55.330324+0000 mon.a (mon.0) 2730 : cluster [DBG] osdmap e1461: 8 total, 8 up, 8 in 2024-09-13T00:28:56.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:56 smithi136 bash[75164]: cluster 2024-09-13T00:28:55.330324+0000 mon.a (mon.0) 2730 : cluster [DBG] osdmap e1461: 8 total, 8 up, 8 in 2024-09-13T00:28:56.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:56 smithi136 bash[76371]: cluster 2024-09-13T00:28:55.330324+0000 mon.a (mon.0) 2730 : cluster [DBG] osdmap e1461: 8 total, 8 up, 8 in 2024-09-13T00:28:57.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:57 smithi181 bash[67587]: cluster 2024-09-13T00:28:55.977942+0000 mgr.y (mgr.24661) 1554 : cluster [DBG] pgmap v2576: 137 pgs: 137 active+clean; 81 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 44 KiB/s rd, 3.0 MiB/s wr, 71 op/s 2024-09-13T00:28:57.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:57 smithi181 bash[67587]: cluster 2024-09-13T00:28:56.354086+0000 mon.a (mon.0) 2731 : cluster [DBG] osdmap e1462: 8 total, 8 up, 8 in 2024-09-13T00:28:57.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:57 smithi136 bash[75164]: cluster 2024-09-13T00:28:55.977942+0000 mgr.y (mgr.24661) 1554 : cluster [DBG] pgmap v2576: 137 pgs: 137 active+clean; 81 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 44 KiB/s rd, 3.0 MiB/s wr, 71 op/s 2024-09-13T00:28:57.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:57 smithi136 bash[75164]: cluster 2024-09-13T00:28:56.354086+0000 mon.a (mon.0) 2731 : cluster [DBG] osdmap e1462: 8 total, 8 up, 8 in 2024-09-13T00:28:57.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:57 smithi136 bash[76371]: cluster 2024-09-13T00:28:55.977942+0000 mgr.y (mgr.24661) 1554 : cluster [DBG] pgmap v2576: 137 pgs: 137 active+clean; 81 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 44 KiB/s rd, 3.0 MiB/s wr, 71 op/s 2024-09-13T00:28:57.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:57 smithi136 bash[76371]: cluster 2024-09-13T00:28:56.354086+0000 mon.a (mon.0) 2731 : cluster [DBG] osdmap e1462: 8 total, 8 up, 8 in 2024-09-13T00:28:58.152 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestDeepCopy.CloneFlatten (2125 ms) 2024-09-13T00:28:58.152 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestDeepCopy.Stress 2024-09-13T00:28:58.219 INFO:tasks.workunit.client.0.smithi136.stdout:write: A 14790807~1486018 2024-09-13T00:28:58.237 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 56181415~4463792 2024-09-13T00:28:58.238 INFO:tasks.workunit.client.0.smithi136.stdout:write: A 6392568~5032039 2024-09-13T00:28:58.265 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 77494535~5659240 2024-09-13T00:28:58.267 INFO:tasks.workunit.client.0.smithi136.stdout:write: A 10613912~5185565 2024-09-13T00:28:58.284 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 35347970~6963229 2024-09-13T00:28:58.285 INFO:tasks.workunit.client.0.smithi136.stdout:write: A 51748735~3369696 2024-09-13T00:28:58.304 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 62608606~4334000 2024-09-13T00:28:58.304 INFO:tasks.workunit.client.0.smithi136.stdout:snap: snap0 2024-09-13T00:28:58.417 INFO:tasks.workunit.client.0.smithi136.stdout:clone image240 -> image241 2024-09-13T00:28:58.542 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 167154208 2024-09-13T00:28:58.558 INFO:tasks.workunit.client.0.smithi136.stdout:write: B 135043401~4938223 2024-09-13T00:28:58.590 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 23765013~2448341 2024-09-13T00:28:58.600 INFO:tasks.workunit.client.0.smithi136.stdout:write: B 55455529~6187377 2024-09-13T00:28:58.638 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 118820296~7395771 2024-09-13T00:28:58.641 INFO:tasks.workunit.client.0.smithi136.stdout:write: B 50039412~6963559 2024-09-13T00:28:58.675 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 25535620~6807670 2024-09-13T00:28:58.682 INFO:tasks.workunit.client.0.smithi136.stdout:write: B 22834381~1193782 2024-09-13T00:28:58.688 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 156447357~1125876 2024-09-13T00:28:58.688 INFO:tasks.workunit.client.0.smithi136.stdout:snap: snap1 2024-09-13T00:28:58.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:58 smithi136 bash[75164]: cluster 2024-09-13T00:28:57.363016+0000 mon.a (mon.0) 2732 : cluster [DBG] osdmap e1463: 8 total, 8 up, 8 in 2024-09-13T00:28:58.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:58 smithi136 bash[75164]: cluster 2024-09-13T00:28:57.978823+0000 mgr.y (mgr.24661) 1555 : cluster [DBG] pgmap v2579: 137 pgs: 137 active+clean; 113 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 7.2 MiB/s rd, 11 MiB/s wr, 307 op/s 2024-09-13T00:28:58.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:58 smithi136 bash[76371]: cluster 2024-09-13T00:28:57.363016+0000 mon.a (mon.0) 2732 : cluster [DBG] osdmap e1463: 8 total, 8 up, 8 in 2024-09-13T00:28:58.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:58 smithi136 bash[76371]: cluster 2024-09-13T00:28:57.978823+0000 mgr.y (mgr.24661) 1555 : cluster [DBG] pgmap v2579: 137 pgs: 137 active+clean; 113 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 7.2 MiB/s rd, 11 MiB/s wr, 307 op/s 2024-09-13T00:28:58.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:58 smithi181 bash[67587]: cluster 2024-09-13T00:28:57.363016+0000 mon.a (mon.0) 2732 : cluster [DBG] osdmap e1463: 8 total, 8 up, 8 in 2024-09-13T00:28:58.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:58 smithi181 bash[67587]: cluster 2024-09-13T00:28:57.978823+0000 mgr.y (mgr.24661) 1555 : cluster [DBG] pgmap v2579: 137 pgs: 137 active+clean; 113 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 7.2 MiB/s rd, 11 MiB/s wr, 307 op/s 2024-09-13T00:28:59.412 INFO:tasks.workunit.client.0.smithi136.stdout:write: C 141387196~4314181 2024-09-13T00:28:59.418 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 63050654~4315402 2024-09-13T00:28:59.419 INFO:tasks.workunit.client.0.smithi136.stdout:write: C 33128620~2036377 2024-09-13T00:28:59.419 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 56874872~1520574 2024-09-13T00:28:59.420 INFO:tasks.workunit.client.0.smithi136.stdout:write: C 68454845~2054332 2024-09-13T00:28:59.420 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 86160901~7817847 2024-09-13T00:28:59.428 INFO:tasks.workunit.client.0.smithi136.stdout:write: C 27324198~4790973 2024-09-13T00:28:59.428 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 91125394~10735 2024-09-13T00:28:59.460 INFO:tasks.workunit.client.0.smithi136.stdout:snap: snap2 2024-09-13T00:28:59.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:28:59 smithi136 bash[75164]: cluster 2024-09-13T00:28:58.378888+0000 mon.a (mon.0) 2733 : cluster [DBG] osdmap e1464: 8 total, 8 up, 8 in 2024-09-13T00:28:59.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:28:59 smithi136 bash[76371]: cluster 2024-09-13T00:28:58.378888+0000 mon.a (mon.0) 2733 : cluster [DBG] osdmap e1464: 8 total, 8 up, 8 in 2024-09-13T00:28:59.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:28:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:28:59] "GET /metrics HTTP/1.1" 200 36552 "" "Prometheus/2.43.0" 2024-09-13T00:28:59.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:28:59 smithi181 bash[67587]: cluster 2024-09-13T00:28:58.378888+0000 mon.a (mon.0) 2733 : cluster [DBG] osdmap e1464: 8 total, 8 up, 8 in 2024-09-13T00:29:00.444 INFO:tasks.workunit.client.0.smithi136.stdout:clone image241 -> image242 2024-09-13T00:29:00.610 INFO:tasks.workunit.client.0.smithi136.stdout:write: D 737910~2120731 2024-09-13T00:29:00.636 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 161315139~780424 2024-09-13T00:29:00.648 INFO:tasks.workunit.client.0.smithi136.stdout:write: D 143659679~4508619 2024-09-13T00:29:00.684 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 62213569~5746375 2024-09-13T00:29:00.722 INFO:tasks.workunit.client.0.smithi136.stdout:write: D 118395963~4020748 2024-09-13T00:29:00.745 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 59524801~4357343 2024-09-13T00:29:00.753 INFO:tasks.workunit.client.0.smithi136.stdout:write: D 40016868~6002088 2024-09-13T00:29:00.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:00 smithi136 bash[75164]: cluster 2024-09-13T00:28:59.385773+0000 mon.a (mon.0) 2734 : cluster [DBG] osdmap e1465: 8 total, 8 up, 8 in 2024-09-13T00:29:00.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:00 smithi136 bash[75164]: cluster 2024-09-13T00:28:59.979610+0000 mgr.y (mgr.24661) 1556 : cluster [DBG] pgmap v2582: 137 pgs: 137 active+clean; 151 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 17 MiB/s rd, 17 MiB/s wr, 437 op/s 2024-09-13T00:29:00.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:00 smithi136 bash[76371]: cluster 2024-09-13T00:28:59.385773+0000 mon.a (mon.0) 2734 : cluster [DBG] osdmap e1465: 8 total, 8 up, 8 in 2024-09-13T00:29:00.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:00 smithi136 bash[76371]: cluster 2024-09-13T00:28:59.979610+0000 mgr.y (mgr.24661) 1556 : cluster [DBG] pgmap v2582: 137 pgs: 137 active+clean; 151 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 17 MiB/s rd, 17 MiB/s wr, 437 op/s 2024-09-13T00:29:00.782 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 136146610~558589 2024-09-13T00:29:00.804 INFO:tasks.workunit.client.0.smithi136.stdout:snap: snap3 2024-09-13T00:29:00.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:00 smithi181 bash[67587]: cluster 2024-09-13T00:28:59.385773+0000 mon.a (mon.0) 2734 : cluster [DBG] osdmap e1465: 8 total, 8 up, 8 in 2024-09-13T00:29:00.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:00 smithi181 bash[67587]: cluster 2024-09-13T00:28:59.979610+0000 mgr.y (mgr.24661) 1556 : cluster [DBG] pgmap v2582: 137 pgs: 137 active+clean; 151 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 17 MiB/s rd, 17 MiB/s wr, 437 op/s 2024-09-13T00:29:01.442 INFO:tasks.workunit.client.0.smithi136.stdout:clone image242 -> image243 2024-09-13T00:29:01.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:01 smithi136 bash[75164]: cluster 2024-09-13T00:29:00.394061+0000 mon.a (mon.0) 2735 : cluster [DBG] osdmap e1466: 8 total, 8 up, 8 in 2024-09-13T00:29:01.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:01 smithi136 bash[76371]: cluster 2024-09-13T00:29:00.394061+0000 mon.a (mon.0) 2735 : cluster [DBG] osdmap e1466: 8 total, 8 up, 8 in 2024-09-13T00:29:01.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:01 smithi181 bash[67587]: cluster 2024-09-13T00:29:00.394061+0000 mon.a (mon.0) 2735 : cluster [DBG] osdmap e1466: 8 total, 8 up, 8 in 2024-09-13T00:29:02.278 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestDeepCopy.Stress (4126 ms) 2024-09-13T00:29:02.278 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestDeepCopy.NoSnaps_LargerDstObjSize 2024-09-13T00:29:02.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:02 smithi181 bash[67587]: cluster 2024-09-13T00:29:01.405748+0000 mon.a (mon.0) 2736 : cluster [DBG] osdmap e1467: 8 total, 8 up, 8 in 2024-09-13T00:29:02.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:02 smithi181 bash[67587]: cluster 2024-09-13T00:29:01.980728+0000 mgr.y (mgr.24661) 1557 : cluster [DBG] pgmap v2585: 137 pgs: 137 active+clean; 256 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 37 MiB/s rd, 33 MiB/s wr, 615 op/s 2024-09-13T00:29:02.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:29:02 smithi181 bash[95505]: ts=2024-09-13T00:29:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:29:02.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:02 smithi136 bash[76371]: cluster 2024-09-13T00:29:01.405748+0000 mon.a (mon.0) 2736 : cluster [DBG] osdmap e1467: 8 total, 8 up, 8 in 2024-09-13T00:29:02.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:02 smithi136 bash[76371]: cluster 2024-09-13T00:29:01.980728+0000 mgr.y (mgr.24661) 1557 : cluster [DBG] pgmap v2585: 137 pgs: 137 active+clean; 256 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 37 MiB/s rd, 33 MiB/s wr, 615 op/s 2024-09-13T00:29:02.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:02 smithi136 bash[75164]: cluster 2024-09-13T00:29:01.405748+0000 mon.a (mon.0) 2736 : cluster [DBG] osdmap e1467: 8 total, 8 up, 8 in 2024-09-13T00:29:02.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:02 smithi136 bash[75164]: cluster 2024-09-13T00:29:01.980728+0000 mgr.y (mgr.24661) 1557 : cluster [DBG] pgmap v2585: 137 pgs: 137 active+clean; 256 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 37 MiB/s rd, 33 MiB/s wr, 615 op/s 2024-09-13T00:29:02.763 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestDeepCopy.NoSnaps_LargerDstObjSize (486 ms) 2024-09-13T00:29:02.764 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestDeepCopy.Snaps_LargerDstObjSize 2024-09-13T00:29:04.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:04 smithi136 bash[75164]: cluster 2024-09-13T00:29:03.419786+0000 mon.a (mon.0) 2737 : cluster [DBG] osdmap e1468: 8 total, 8 up, 8 in 2024-09-13T00:29:04.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:04 smithi136 bash[75164]: cluster 2024-09-13T00:29:03.981749+0000 mgr.y (mgr.24661) 1558 : cluster [DBG] pgmap v2587: 137 pgs: 137 active+clean; 312 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 43 MiB/s rd, 35 MiB/s wr, 515 op/s 2024-09-13T00:29:04.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:04 smithi136 bash[76371]: cluster 2024-09-13T00:29:03.419786+0000 mon.a (mon.0) 2737 : cluster [DBG] osdmap e1468: 8 total, 8 up, 8 in 2024-09-13T00:29:04.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:04 smithi136 bash[76371]: cluster 2024-09-13T00:29:03.981749+0000 mgr.y (mgr.24661) 1558 : cluster [DBG] pgmap v2587: 137 pgs: 137 active+clean; 312 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 43 MiB/s rd, 35 MiB/s wr, 515 op/s 2024-09-13T00:29:04.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:04 smithi181 bash[67587]: cluster 2024-09-13T00:29:03.419786+0000 mon.a (mon.0) 2737 : cluster [DBG] osdmap e1468: 8 total, 8 up, 8 in 2024-09-13T00:29:04.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:04 smithi181 bash[67587]: cluster 2024-09-13T00:29:03.981749+0000 mgr.y (mgr.24661) 1558 : cluster [DBG] pgmap v2587: 137 pgs: 137 active+clean; 312 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 43 MiB/s rd, 35 MiB/s wr, 515 op/s 2024-09-13T00:29:05.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:05 smithi136 bash[76371]: cluster 2024-09-13T00:29:04.424962+0000 mon.a (mon.0) 2738 : cluster [DBG] osdmap e1469: 8 total, 8 up, 8 in 2024-09-13T00:29:05.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:05 smithi136 bash[75164]: cluster 2024-09-13T00:29:04.424962+0000 mon.a (mon.0) 2738 : cluster [DBG] osdmap e1469: 8 total, 8 up, 8 in 2024-09-13T00:29:05.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:05 smithi181 bash[67587]: cluster 2024-09-13T00:29:04.424962+0000 mon.a (mon.0) 2738 : cluster [DBG] osdmap e1469: 8 total, 8 up, 8 in 2024-09-13T00:29:06.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:06 smithi136 bash[76371]: cluster 2024-09-13T00:29:05.426930+0000 mon.a (mon.0) 2739 : cluster [DBG] osdmap e1470: 8 total, 8 up, 8 in 2024-09-13T00:29:06.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:06 smithi136 bash[76371]: cluster 2024-09-13T00:29:05.982685+0000 mgr.y (mgr.24661) 1559 : cluster [DBG] pgmap v2590: 137 pgs: 137 active+clean; 332 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 35 MiB/s rd, 29 MiB/s wr, 469 op/s 2024-09-13T00:29:06.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:06 smithi136 bash[75164]: cluster 2024-09-13T00:29:05.426930+0000 mon.a (mon.0) 2739 : cluster [DBG] osdmap e1470: 8 total, 8 up, 8 in 2024-09-13T00:29:06.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:06 smithi136 bash[75164]: cluster 2024-09-13T00:29:05.982685+0000 mgr.y (mgr.24661) 1559 : cluster [DBG] pgmap v2590: 137 pgs: 137 active+clean; 332 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 35 MiB/s rd, 29 MiB/s wr, 469 op/s 2024-09-13T00:29:06.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:06 smithi181 bash[67587]: cluster 2024-09-13T00:29:05.426930+0000 mon.a (mon.0) 2739 : cluster [DBG] osdmap e1470: 8 total, 8 up, 8 in 2024-09-13T00:29:06.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:06 smithi181 bash[67587]: cluster 2024-09-13T00:29:05.982685+0000 mgr.y (mgr.24661) 1559 : cluster [DBG] pgmap v2590: 137 pgs: 137 active+clean; 332 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 35 MiB/s rd, 29 MiB/s wr, 469 op/s 2024-09-13T00:29:06.933 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestDeepCopy.Snaps_LargerDstObjSize (4169 ms) 2024-09-13T00:29:06.933 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestDeepCopy.Clone_LargerDstObjSize 2024-09-13T00:29:07.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:07 smithi136 bash[75164]: cluster 2024-09-13T00:29:06.448252+0000 mon.a (mon.0) 2740 : cluster [DBG] osdmap e1471: 8 total, 8 up, 8 in 2024-09-13T00:29:07.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:07 smithi136 bash[76371]: cluster 2024-09-13T00:29:06.448252+0000 mon.a (mon.0) 2740 : cluster [DBG] osdmap e1471: 8 total, 8 up, 8 in 2024-09-13T00:29:07.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:07 smithi181 bash[67587]: cluster 2024-09-13T00:29:06.448252+0000 mon.a (mon.0) 2740 : cluster [DBG] osdmap e1471: 8 total, 8 up, 8 in 2024-09-13T00:29:08.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:08 smithi136 bash[76371]: cluster 2024-09-13T00:29:07.452007+0000 mon.a (mon.0) 2741 : cluster [DBG] osdmap e1472: 8 total, 8 up, 8 in 2024-09-13T00:29:08.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:08 smithi136 bash[76371]: cluster 2024-09-13T00:29:07.983552+0000 mgr.y (mgr.24661) 1560 : cluster [DBG] pgmap v2593: 137 pgs: 137 active+clean; 364 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 32 MiB/s rd, 13 MiB/s wr, 209 op/s 2024-09-13T00:29:08.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:08 smithi136 bash[75164]: cluster 2024-09-13T00:29:07.452007+0000 mon.a (mon.0) 2741 : cluster [DBG] osdmap e1472: 8 total, 8 up, 8 in 2024-09-13T00:29:08.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:08 smithi136 bash[75164]: cluster 2024-09-13T00:29:07.983552+0000 mgr.y (mgr.24661) 1560 : cluster [DBG] pgmap v2593: 137 pgs: 137 active+clean; 364 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 32 MiB/s rd, 13 MiB/s wr, 209 op/s 2024-09-13T00:29:08.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:08 smithi181 bash[67587]: cluster 2024-09-13T00:29:07.452007+0000 mon.a (mon.0) 2741 : cluster [DBG] osdmap e1472: 8 total, 8 up, 8 in 2024-09-13T00:29:08.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:08 smithi181 bash[67587]: cluster 2024-09-13T00:29:07.983552+0000 mgr.y (mgr.24661) 1560 : cluster [DBG] pgmap v2593: 137 pgs: 137 active+clean; 364 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 32 MiB/s rd, 13 MiB/s wr, 209 op/s 2024-09-13T00:29:09.229 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestDeepCopy.Clone_LargerDstObjSize (2296 ms) 2024-09-13T00:29:09.229 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestDeepCopy.CloneFlatten_LargerDstObjSize 2024-09-13T00:29:09.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:09 smithi136 bash[75164]: cluster 2024-09-13T00:29:08.474374+0000 mon.a (mon.0) 2742 : cluster [DBG] osdmap e1473: 8 total, 8 up, 8 in 2024-09-13T00:29:09.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:09 smithi136 bash[75164]: audit 2024-09-13T00:29:08.820003+0000 mon.a (mon.0) 2743 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:29:09.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:29:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:29:09] "GET /metrics HTTP/1.1" 200 36557 "" "Prometheus/2.43.0" 2024-09-13T00:29:09.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:09 smithi136 bash[76371]: cluster 2024-09-13T00:29:08.474374+0000 mon.a (mon.0) 2742 : cluster [DBG] osdmap e1473: 8 total, 8 up, 8 in 2024-09-13T00:29:09.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:09 smithi136 bash[76371]: audit 2024-09-13T00:29:08.820003+0000 mon.a (mon.0) 2743 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:29:09.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:09 smithi181 bash[67587]: cluster 2024-09-13T00:29:08.474374+0000 mon.a (mon.0) 2742 : cluster [DBG] osdmap e1473: 8 total, 8 up, 8 in 2024-09-13T00:29:09.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:09 smithi181 bash[67587]: audit 2024-09-13T00:29:08.820003+0000 mon.a (mon.0) 2743 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:29:10.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:10 smithi136 bash[75164]: cluster 2024-09-13T00:29:09.487415+0000 mon.a (mon.0) 2744 : cluster [DBG] osdmap e1474: 8 total, 8 up, 8 in 2024-09-13T00:29:10.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:10 smithi136 bash[75164]: cluster 2024-09-13T00:29:09.984431+0000 mgr.y (mgr.24661) 1561 : cluster [DBG] pgmap v2596: 137 pgs: 137 active+clean; 380 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 30 MiB/s rd, 12 MiB/s wr, 228 op/s 2024-09-13T00:29:10.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:10 smithi136 bash[76371]: cluster 2024-09-13T00:29:09.487415+0000 mon.a (mon.0) 2744 : cluster [DBG] osdmap e1474: 8 total, 8 up, 8 in 2024-09-13T00:29:10.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:10 smithi136 bash[76371]: cluster 2024-09-13T00:29:09.984431+0000 mgr.y (mgr.24661) 1561 : cluster [DBG] pgmap v2596: 137 pgs: 137 active+clean; 380 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 30 MiB/s rd, 12 MiB/s wr, 228 op/s 2024-09-13T00:29:10.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:10 smithi181 bash[67587]: cluster 2024-09-13T00:29:09.487415+0000 mon.a (mon.0) 2744 : cluster [DBG] osdmap e1474: 8 total, 8 up, 8 in 2024-09-13T00:29:10.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:10 smithi181 bash[67587]: cluster 2024-09-13T00:29:09.984431+0000 mgr.y (mgr.24661) 1561 : cluster [DBG] pgmap v2596: 137 pgs: 137 active+clean; 380 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 30 MiB/s rd, 12 MiB/s wr, 228 op/s 2024-09-13T00:29:11.353 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestDeepCopy.CloneFlatten_LargerDstObjSize (2125 ms) 2024-09-13T00:29:11.353 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestDeepCopy.Stress_LargerDstObjSize 2024-09-13T00:29:11.420 INFO:tasks.workunit.client.0.smithi136.stdout:write: A 55694942~1703547 2024-09-13T00:29:11.442 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 44685504~5477669 2024-09-13T00:29:11.443 INFO:tasks.workunit.client.0.smithi136.stdout:write: A 60176779~1765659 2024-09-13T00:29:11.455 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 42559405~2753199 2024-09-13T00:29:11.458 INFO:tasks.workunit.client.0.smithi136.stdout:write: A 34117136~7250215 2024-09-13T00:29:11.491 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 79159933~2737096 2024-09-13T00:29:11.491 INFO:tasks.workunit.client.0.smithi136.stdout:write: A 45375819~1343128 2024-09-13T00:29:11.504 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 59262871~8171080 2024-09-13T00:29:11.506 INFO:tasks.workunit.client.0.smithi136.stdout:snap: snap0 2024-09-13T00:29:11.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:11 smithi136 bash[75164]: cluster 2024-09-13T00:29:10.501556+0000 mon.a (mon.0) 2745 : cluster [DBG] osdmap e1475: 8 total, 8 up, 8 in 2024-09-13T00:29:11.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:11 smithi136 bash[76371]: cluster 2024-09-13T00:29:10.501556+0000 mon.a (mon.0) 2745 : cluster [DBG] osdmap e1475: 8 total, 8 up, 8 in 2024-09-13T00:29:11.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:11 smithi181 bash[67587]: cluster 2024-09-13T00:29:10.501556+0000 mon.a (mon.0) 2745 : cluster [DBG] osdmap e1475: 8 total, 8 up, 8 in 2024-09-13T00:29:12.549 INFO:tasks.workunit.client.0.smithi136.stdout:clone image262 -> image263 2024-09-13T00:29:12.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:12 smithi181 bash[67587]: cluster 2024-09-13T00:29:11.985897+0000 mgr.y (mgr.24661) 1562 : cluster [DBG] pgmap v2598: 137 pgs: 137 active+clean; 464 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 41 MiB/s rd, 22 MiB/s wr, 317 op/s 2024-09-13T00:29:12.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:29:12 smithi181 bash[95505]: ts=2024-09-13T00:29:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:29:12.673 INFO:tasks.workunit.client.0.smithi136.stdout:write: B 41217230~4563364 2024-09-13T00:29:12.719 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 60380453~1453609 2024-09-13T00:29:12.727 INFO:tasks.workunit.client.0.smithi136.stdout:write: B 52068173~5276278 2024-09-13T00:29:12.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:12 smithi136 bash[75164]: cluster 2024-09-13T00:29:11.985897+0000 mgr.y (mgr.24661) 1562 : cluster [DBG] pgmap v2598: 137 pgs: 137 active+clean; 464 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 41 MiB/s rd, 22 MiB/s wr, 317 op/s 2024-09-13T00:29:12.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:12 smithi136 bash[76371]: cluster 2024-09-13T00:29:11.985897+0000 mgr.y (mgr.24661) 1562 : cluster [DBG] pgmap v2598: 137 pgs: 137 active+clean; 464 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 41 MiB/s rd, 22 MiB/s wr, 317 op/s 2024-09-13T00:29:12.761 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 15593768~2218786 2024-09-13T00:29:12.768 INFO:tasks.workunit.client.0.smithi136.stdout:write: B 13202360~170320 2024-09-13T00:29:12.771 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 124689~6854374 2024-09-13T00:29:12.779 INFO:tasks.workunit.client.0.smithi136.stdout:write: B 8557921~2473574 2024-09-13T00:29:12.798 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 78398097~5487983 2024-09-13T00:29:12.807 INFO:tasks.workunit.client.0.smithi136.stdout:snap: snap1 2024-09-13T00:29:13.584 INFO:tasks.workunit.client.0.smithi136.stdout:clone image263 -> image264 2024-09-13T00:29:13.743 INFO:tasks.workunit.client.0.smithi136.stdout:write: C 14053585~1019159 2024-09-13T00:29:13.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:13 smithi136 bash[76371]: cluster 2024-09-13T00:29:12.510635+0000 mon.a (mon.0) 2746 : cluster [DBG] osdmap e1476: 8 total, 8 up, 8 in 2024-09-13T00:29:13.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:13 smithi136 bash[75164]: cluster 2024-09-13T00:29:12.510635+0000 mon.a (mon.0) 2746 : cluster [DBG] osdmap e1476: 8 total, 8 up, 8 in 2024-09-13T00:29:13.764 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 25046590~7870007 2024-09-13T00:29:13.776 INFO:tasks.workunit.client.0.smithi136.stdout:write: C 2218495~6227681 2024-09-13T00:29:13.813 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 80342852~3543228 2024-09-13T00:29:13.822 INFO:tasks.workunit.client.0.smithi136.stdout:write: C 11714308~5651275 2024-09-13T00:29:13.848 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 56773469~8113425 2024-09-13T00:29:13.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:13 smithi181 bash[67587]: cluster 2024-09-13T00:29:12.510635+0000 mon.a (mon.0) 2746 : cluster [DBG] osdmap e1476: 8 total, 8 up, 8 in 2024-09-13T00:29:13.875 INFO:tasks.workunit.client.0.smithi136.stdout:write: C 46231221~421966 2024-09-13T00:29:13.907 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 77373047~5716011 2024-09-13T00:29:13.914 INFO:tasks.workunit.client.0.smithi136.stdout:snap: snap2 2024-09-13T00:29:14.584 INFO:tasks.workunit.client.0.smithi136.stdout:clone image264 -> image265 2024-09-13T00:29:14.779 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 129441221 2024-09-13T00:29:14.795 INFO:tasks.workunit.client.0.smithi136.stdout:write: D 52678521~8167164 2024-09-13T00:29:14.841 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 49909683~3782421 2024-09-13T00:29:14.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:14 smithi181 bash[67587]: cluster 2024-09-13T00:29:13.531964+0000 mon.a (mon.0) 2747 : cluster [DBG] osdmap e1477: 8 total, 8 up, 8 in 2024-09-13T00:29:14.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:14 smithi181 bash[67587]: cluster 2024-09-13T00:29:13.986809+0000 mgr.y (mgr.24661) 1563 : cluster [DBG] pgmap v2601: 137 pgs: 137 active+clean; 495 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 43 MiB/s rd, 28 MiB/s wr, 422 op/s 2024-09-13T00:29:14.869 INFO:tasks.workunit.client.0.smithi136.stdout:write: D 121166613~8274608 2024-09-13T00:29:14.903 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 122375338~7065883 2024-09-13T00:29:14.906 INFO:tasks.workunit.client.0.smithi136.stdout:write: D 74664023~1006485 2024-09-13T00:29:14.918 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 47755472~2967091 2024-09-13T00:29:14.925 INFO:tasks.workunit.client.0.smithi136.stdout:write: D 43350907~7647678 2024-09-13T00:29:14.966 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 109356147~7741526 2024-09-13T00:29:14.966 INFO:tasks.workunit.client.0.smithi136.stdout:snap: snap3 2024-09-13T00:29:15.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:14 smithi136 bash[75164]: cluster 2024-09-13T00:29:13.531964+0000 mon.a (mon.0) 2747 : cluster [DBG] osdmap e1477: 8 total, 8 up, 8 in 2024-09-13T00:29:15.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:14 smithi136 bash[75164]: cluster 2024-09-13T00:29:13.986809+0000 mgr.y (mgr.24661) 1563 : cluster [DBG] pgmap v2601: 137 pgs: 137 active+clean; 495 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 43 MiB/s rd, 28 MiB/s wr, 422 op/s 2024-09-13T00:29:15.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:14 smithi136 bash[76371]: cluster 2024-09-13T00:29:13.531964+0000 mon.a (mon.0) 2747 : cluster [DBG] osdmap e1477: 8 total, 8 up, 8 in 2024-09-13T00:29:15.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:14 smithi136 bash[76371]: cluster 2024-09-13T00:29:13.986809+0000 mgr.y (mgr.24661) 1563 : cluster [DBG] pgmap v2601: 137 pgs: 137 active+clean; 495 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 43 MiB/s rd, 28 MiB/s wr, 422 op/s 2024-09-13T00:29:15.594 INFO:tasks.workunit.client.0.smithi136.stdout:clone image265 -> image266 2024-09-13T00:29:15.836 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 163231023 2024-09-13T00:29:15.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:15 smithi181 bash[67587]: cluster 2024-09-13T00:29:14.537728+0000 mon.a (mon.0) 2748 : cluster [DBG] osdmap e1478: 8 total, 8 up, 8 in 2024-09-13T00:29:16.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:15 smithi136 bash[75164]: cluster 2024-09-13T00:29:14.537728+0000 mon.a (mon.0) 2748 : cluster [DBG] osdmap e1478: 8 total, 8 up, 8 in 2024-09-13T00:29:16.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:15 smithi136 bash[76371]: cluster 2024-09-13T00:29:14.537728+0000 mon.a (mon.0) 2748 : cluster [DBG] osdmap e1478: 8 total, 8 up, 8 in 2024-09-13T00:29:16.501 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestDeepCopy.Stress_LargerDstObjSize (5147 ms) 2024-09-13T00:29:16.501 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestDeepCopy.NoSnaps_SmallerDstObjSize 2024-09-13T00:29:16.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:16 smithi181 bash[67587]: cluster 2024-09-13T00:29:15.546659+0000 mon.a (mon.0) 2749 : cluster [DBG] osdmap e1479: 8 total, 8 up, 8 in 2024-09-13T00:29:16.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:16 smithi181 bash[67587]: cluster 2024-09-13T00:29:15.987787+0000 mgr.y (mgr.24661) 1564 : cluster [DBG] pgmap v2604: 137 pgs: 137 active+clean; 553 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 22 MiB/s rd, 20 MiB/s wr, 242 op/s 2024-09-13T00:29:16.912 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestDeepCopy.NoSnaps_SmallerDstObjSize (411 ms) 2024-09-13T00:29:16.912 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestDeepCopy.Snaps_SmallerDstObjSize 2024-09-13T00:29:17.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:16 smithi136 bash[75164]: cluster 2024-09-13T00:29:15.546659+0000 mon.a (mon.0) 2749 : cluster [DBG] osdmap e1479: 8 total, 8 up, 8 in 2024-09-13T00:29:17.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:16 smithi136 bash[75164]: cluster 2024-09-13T00:29:15.987787+0000 mgr.y (mgr.24661) 1564 : cluster [DBG] pgmap v2604: 137 pgs: 137 active+clean; 553 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 22 MiB/s rd, 20 MiB/s wr, 242 op/s 2024-09-13T00:29:17.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:16 smithi136 bash[76371]: cluster 2024-09-13T00:29:15.546659+0000 mon.a (mon.0) 2749 : cluster [DBG] osdmap e1479: 8 total, 8 up, 8 in 2024-09-13T00:29:17.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:16 smithi136 bash[76371]: cluster 2024-09-13T00:29:15.987787+0000 mgr.y (mgr.24661) 1564 : cluster [DBG] pgmap v2604: 137 pgs: 137 active+clean; 553 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 22 MiB/s rd, 20 MiB/s wr, 242 op/s 2024-09-13T00:29:18.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:18 smithi181 bash[67587]: cluster 2024-09-13T00:29:17.558307+0000 mon.a (mon.0) 2750 : cluster [DBG] osdmap e1480: 8 total, 8 up, 8 in 2024-09-13T00:29:18.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:18 smithi181 bash[67587]: cluster 2024-09-13T00:29:17.988724+0000 mgr.y (mgr.24661) 1565 : cluster [DBG] pgmap v2606: 137 pgs: 137 active+clean; 618 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 37 MiB/s rd, 31 MiB/s wr, 698 op/s 2024-09-13T00:29:19.012 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:18 smithi136 bash[75164]: cluster 2024-09-13T00:29:17.558307+0000 mon.a (mon.0) 2750 : cluster [DBG] osdmap e1480: 8 total, 8 up, 8 in 2024-09-13T00:29:19.012 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:18 smithi136 bash[75164]: cluster 2024-09-13T00:29:17.988724+0000 mgr.y (mgr.24661) 1565 : cluster [DBG] pgmap v2606: 137 pgs: 137 active+clean; 618 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 37 MiB/s rd, 31 MiB/s wr, 698 op/s 2024-09-13T00:29:19.012 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:18 smithi136 bash[76371]: cluster 2024-09-13T00:29:17.558307+0000 mon.a (mon.0) 2750 : cluster [DBG] osdmap e1480: 8 total, 8 up, 8 in 2024-09-13T00:29:19.012 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:18 smithi136 bash[76371]: cluster 2024-09-13T00:29:17.988724+0000 mgr.y (mgr.24661) 1565 : cluster [DBG] pgmap v2606: 137 pgs: 137 active+clean; 618 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 37 MiB/s rd, 31 MiB/s wr, 698 op/s 2024-09-13T00:29:19.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:19 smithi136 bash[75164]: cluster 2024-09-13T00:29:18.570863+0000 mon.a (mon.0) 2751 : cluster [DBG] osdmap e1481: 8 total, 8 up, 8 in 2024-09-13T00:29:19.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:29:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:29:19] "GET /metrics HTTP/1.1" 200 36557 "" "Prometheus/2.43.0" 2024-09-13T00:29:19.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:19 smithi136 bash[76371]: cluster 2024-09-13T00:29:18.570863+0000 mon.a (mon.0) 2751 : cluster [DBG] osdmap e1481: 8 total, 8 up, 8 in 2024-09-13T00:29:19.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:19 smithi181 bash[67587]: cluster 2024-09-13T00:29:18.570863+0000 mon.a (mon.0) 2751 : cluster [DBG] osdmap e1481: 8 total, 8 up, 8 in 2024-09-13T00:29:20.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:20 smithi181 bash[67587]: cluster 2024-09-13T00:29:19.578621+0000 mon.a (mon.0) 2752 : cluster [DBG] osdmap e1482: 8 total, 8 up, 8 in 2024-09-13T00:29:20.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:20 smithi181 bash[67587]: cluster 2024-09-13T00:29:19.989559+0000 mgr.y (mgr.24661) 1566 : cluster [DBG] pgmap v2609: 137 pgs: 137 active+clean; 644 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 30 MiB/s rd, 27 MiB/s wr, 610 op/s 2024-09-13T00:29:21.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:20 smithi136 bash[76371]: cluster 2024-09-13T00:29:19.578621+0000 mon.a (mon.0) 2752 : cluster [DBG] osdmap e1482: 8 total, 8 up, 8 in 2024-09-13T00:29:21.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:20 smithi136 bash[76371]: cluster 2024-09-13T00:29:19.989559+0000 mgr.y (mgr.24661) 1566 : cluster [DBG] pgmap v2609: 137 pgs: 137 active+clean; 644 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 30 MiB/s rd, 27 MiB/s wr, 610 op/s 2024-09-13T00:29:21.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:20 smithi136 bash[75164]: cluster 2024-09-13T00:29:19.578621+0000 mon.a (mon.0) 2752 : cluster [DBG] osdmap e1482: 8 total, 8 up, 8 in 2024-09-13T00:29:21.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:20 smithi136 bash[75164]: cluster 2024-09-13T00:29:19.989559+0000 mgr.y (mgr.24661) 1566 : cluster [DBG] pgmap v2609: 137 pgs: 137 active+clean; 644 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 30 MiB/s rd, 27 MiB/s wr, 610 op/s 2024-09-13T00:29:21.100 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestDeepCopy.Snaps_SmallerDstObjSize (4189 ms) 2024-09-13T00:29:21.101 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestDeepCopy.Clone_SmallerDstObjSize 2024-09-13T00:29:21.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:21 smithi181 bash[67587]: cluster 2024-09-13T00:29:20.589002+0000 mon.a (mon.0) 2753 : cluster [DBG] osdmap e1483: 8 total, 8 up, 8 in 2024-09-13T00:29:22.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:21 smithi136 bash[75164]: cluster 2024-09-13T00:29:20.589002+0000 mon.a (mon.0) 2753 : cluster [DBG] osdmap e1483: 8 total, 8 up, 8 in 2024-09-13T00:29:22.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:21 smithi136 bash[76371]: cluster 2024-09-13T00:29:20.589002+0000 mon.a (mon.0) 2753 : cluster [DBG] osdmap e1483: 8 total, 8 up, 8 in 2024-09-13T00:29:22.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:29:22 smithi181 bash[95505]: ts=2024-09-13T00:29:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:29:23.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:22 smithi136 bash[75164]: cluster 2024-09-13T00:29:21.596621+0000 mon.a (mon.0) 2754 : cluster [DBG] osdmap e1484: 8 total, 8 up, 8 in 2024-09-13T00:29:23.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:22 smithi136 bash[75164]: audit 2024-09-13T00:29:21.880901+0000 mon.a (mon.0) 2755 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:29:23.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:22 smithi136 bash[75164]: cluster 2024-09-13T00:29:21.990293+0000 mgr.y (mgr.24661) 1567 : cluster [DBG] pgmap v2612: 137 pgs: 137 active+clean; 676 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 27 MiB/s rd, 15 MiB/s wr, 161 op/s 2024-09-13T00:29:23.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:22 smithi136 bash[75164]: audit 2024-09-13T00:29:22.432746+0000 mon.a (mon.0) 2756 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:29:23.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:22 smithi136 bash[75164]: audit 2024-09-13T00:29:22.434055+0000 mon.a (mon.0) 2757 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:29:23.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:22 smithi136 bash[75164]: audit 2024-09-13T00:29:22.442221+0000 mon.a (mon.0) 2758 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:29:23.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:22 smithi136 bash[76371]: cluster 2024-09-13T00:29:21.596621+0000 mon.a (mon.0) 2754 : cluster [DBG] osdmap e1484: 8 total, 8 up, 8 in 2024-09-13T00:29:23.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:22 smithi136 bash[76371]: audit 2024-09-13T00:29:21.880901+0000 mon.a (mon.0) 2755 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:29:23.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:22 smithi136 bash[76371]: cluster 2024-09-13T00:29:21.990293+0000 mgr.y (mgr.24661) 1567 : cluster [DBG] pgmap v2612: 137 pgs: 137 active+clean; 676 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 27 MiB/s rd, 15 MiB/s wr, 161 op/s 2024-09-13T00:29:23.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:22 smithi136 bash[76371]: audit 2024-09-13T00:29:22.432746+0000 mon.a (mon.0) 2756 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:29:23.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:22 smithi136 bash[76371]: audit 2024-09-13T00:29:22.434055+0000 mon.a (mon.0) 2757 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:29:23.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:22 smithi136 bash[76371]: audit 2024-09-13T00:29:22.442221+0000 mon.a (mon.0) 2758 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:29:23.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:22 smithi181 bash[67587]: cluster 2024-09-13T00:29:21.596621+0000 mon.a (mon.0) 2754 : cluster [DBG] osdmap e1484: 8 total, 8 up, 8 in 2024-09-13T00:29:23.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:22 smithi181 bash[67587]: audit 2024-09-13T00:29:21.880901+0000 mon.a (mon.0) 2755 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:29:23.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:22 smithi181 bash[67587]: cluster 2024-09-13T00:29:21.990293+0000 mgr.y (mgr.24661) 1567 : cluster [DBG] pgmap v2612: 137 pgs: 137 active+clean; 676 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 27 MiB/s rd, 15 MiB/s wr, 161 op/s 2024-09-13T00:29:23.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:22 smithi181 bash[67587]: audit 2024-09-13T00:29:22.432746+0000 mon.a (mon.0) 2756 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:29:23.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:22 smithi181 bash[67587]: audit 2024-09-13T00:29:22.434055+0000 mon.a (mon.0) 2757 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:29:23.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:22 smithi181 bash[67587]: audit 2024-09-13T00:29:22.442221+0000 mon.a (mon.0) 2758 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:29:23.290 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestDeepCopy.Clone_SmallerDstObjSize (2189 ms) 2024-09-13T00:29:23.290 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestDeepCopy.CloneFlatten_SmallerDstObjSize 2024-09-13T00:29:24.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:23 smithi136 bash[75164]: cluster 2024-09-13T00:29:22.607042+0000 mon.a (mon.0) 2759 : cluster [DBG] osdmap e1485: 8 total, 8 up, 8 in 2024-09-13T00:29:24.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:23 smithi136 bash[76371]: cluster 2024-09-13T00:29:22.607042+0000 mon.a (mon.0) 2759 : cluster [DBG] osdmap e1485: 8 total, 8 up, 8 in 2024-09-13T00:29:24.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:23 smithi181 bash[67587]: cluster 2024-09-13T00:29:22.607042+0000 mon.a (mon.0) 2759 : cluster [DBG] osdmap e1485: 8 total, 8 up, 8 in 2024-09-13T00:29:25.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:24 smithi136 bash[75164]: cluster 2024-09-13T00:29:23.621245+0000 mon.a (mon.0) 2760 : cluster [DBG] osdmap e1486: 8 total, 8 up, 8 in 2024-09-13T00:29:25.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:24 smithi136 bash[75164]: audit 2024-09-13T00:29:23.820694+0000 mon.a (mon.0) 2761 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:29:25.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:24 smithi136 bash[75164]: cluster 2024-09-13T00:29:23.990968+0000 mgr.y (mgr.24661) 1568 : cluster [DBG] pgmap v2615: 137 pgs: 137 active+clean; 686 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 27 MiB/s rd, 11 MiB/s wr, 203 op/s 2024-09-13T00:29:25.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:24 smithi136 bash[76371]: cluster 2024-09-13T00:29:23.621245+0000 mon.a (mon.0) 2760 : cluster [DBG] osdmap e1486: 8 total, 8 up, 8 in 2024-09-13T00:29:25.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:24 smithi136 bash[76371]: audit 2024-09-13T00:29:23.820694+0000 mon.a (mon.0) 2761 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:29:25.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:24 smithi136 bash[76371]: cluster 2024-09-13T00:29:23.990968+0000 mgr.y (mgr.24661) 1568 : cluster [DBG] pgmap v2615: 137 pgs: 137 active+clean; 686 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 27 MiB/s rd, 11 MiB/s wr, 203 op/s 2024-09-13T00:29:25.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:24 smithi181 bash[67587]: cluster 2024-09-13T00:29:23.621245+0000 mon.a (mon.0) 2760 : cluster [DBG] osdmap e1486: 8 total, 8 up, 8 in 2024-09-13T00:29:25.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:24 smithi181 bash[67587]: audit 2024-09-13T00:29:23.820694+0000 mon.a (mon.0) 2761 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:29:25.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:24 smithi181 bash[67587]: cluster 2024-09-13T00:29:23.990968+0000 mgr.y (mgr.24661) 1568 : cluster [DBG] pgmap v2615: 137 pgs: 137 active+clean; 686 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 27 MiB/s rd, 11 MiB/s wr, 203 op/s 2024-09-13T00:29:25.307 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestDeepCopy.CloneFlatten_SmallerDstObjSize (2017 ms) 2024-09-13T00:29:25.307 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestDeepCopy.Stress_SmallerDstObjSize 2024-09-13T00:29:25.368 INFO:tasks.workunit.client.0.smithi136.stdout:write: A 22053118~5378162 2024-09-13T00:29:25.395 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 51609383~6430732 2024-09-13T00:29:25.396 INFO:tasks.workunit.client.0.smithi136.stdout:write: A 50249347~2603305 2024-09-13T00:29:25.415 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 30372811~7226114 2024-09-13T00:29:25.416 INFO:tasks.workunit.client.0.smithi136.stdout:write: A 15393278~3529800 2024-09-13T00:29:25.435 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 15700829~6627220 2024-09-13T00:29:25.445 INFO:tasks.workunit.client.0.smithi136.stdout:write: A 31679044~3208371 2024-09-13T00:29:25.460 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 68994510~2830723 2024-09-13T00:29:25.460 INFO:tasks.workunit.client.0.smithi136.stdout:snap: snap0 2024-09-13T00:29:25.678 INFO:tasks.workunit.client.0.smithi136.stdout:clone image285 -> image286 2024-09-13T00:29:25.802 INFO:tasks.workunit.client.0.smithi136.stdout:write: B 68255881~125557 2024-09-13T00:29:25.815 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 41327667~3708727 2024-09-13T00:29:25.823 INFO:tasks.workunit.client.0.smithi136.stdout:write: B 11450253~1390478 2024-09-13T00:29:25.842 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 76973008~6913072 2024-09-13T00:29:25.851 INFO:tasks.workunit.client.0.smithi136.stdout:write: B 26373353~4633059 2024-09-13T00:29:25.890 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 28632963~5126357 2024-09-13T00:29:25.909 INFO:tasks.workunit.client.0.smithi136.stdout:write: B 35670343~7216295 2024-09-13T00:29:25.934 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 47201459~354433 2024-09-13T00:29:25.942 INFO:tasks.workunit.client.0.smithi136.stdout:snap: snap1 2024-09-13T00:29:26.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:25 smithi136 bash[75164]: cluster 2024-09-13T00:29:24.628242+0000 mon.a (mon.0) 2762 : cluster [DBG] osdmap e1487: 8 total, 8 up, 8 in 2024-09-13T00:29:26.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:25 smithi136 bash[76371]: cluster 2024-09-13T00:29:24.628242+0000 mon.a (mon.0) 2762 : cluster [DBG] osdmap e1487: 8 total, 8 up, 8 in 2024-09-13T00:29:26.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:25 smithi181 bash[67587]: cluster 2024-09-13T00:29:24.628242+0000 mon.a (mon.0) 2762 : cluster [DBG] osdmap e1487: 8 total, 8 up, 8 in 2024-09-13T00:29:26.687 INFO:tasks.workunit.client.0.smithi136.stdout:clone image286 -> image287 2024-09-13T00:29:26.848 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 97450806 2024-09-13T00:29:26.863 INFO:tasks.workunit.client.0.smithi136.stdout:write: C 9413588~2231153 2024-09-13T00:29:26.893 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 3879035~317311 2024-09-13T00:29:26.903 INFO:tasks.workunit.client.0.smithi136.stdout:write: C 81163376~6403338 2024-09-13T00:29:26.938 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 9523399~4055586 2024-09-13T00:29:26.949 INFO:tasks.workunit.client.0.smithi136.stdout:write: C 80723938~2071648 2024-09-13T00:29:26.963 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 38748522~2707877 2024-09-13T00:29:26.993 INFO:tasks.workunit.client.0.smithi136.stdout:write: C 32052538~6066789 2024-09-13T00:29:27.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:26 smithi136 bash[75164]: cluster 2024-09-13T00:29:25.635931+0000 mon.a (mon.0) 2763 : cluster [DBG] osdmap e1488: 8 total, 8 up, 8 in 2024-09-13T00:29:27.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:26 smithi136 bash[75164]: cluster 2024-09-13T00:29:25.991845+0000 mgr.y (mgr.24661) 1569 : cluster [DBG] pgmap v2618: 137 pgs: 137 active+clean; 730 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 27 MiB/s rd, 14 MiB/s wr, 418 op/s 2024-09-13T00:29:27.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:26 smithi136 bash[76371]: cluster 2024-09-13T00:29:25.635931+0000 mon.a (mon.0) 2763 : cluster [DBG] osdmap e1488: 8 total, 8 up, 8 in 2024-09-13T00:29:27.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:26 smithi136 bash[76371]: cluster 2024-09-13T00:29:25.991845+0000 mgr.y (mgr.24661) 1569 : cluster [DBG] pgmap v2618: 137 pgs: 137 active+clean; 730 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 27 MiB/s rd, 14 MiB/s wr, 418 op/s 2024-09-13T00:29:27.019 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 5804843~1228976 2024-09-13T00:29:27.022 INFO:tasks.workunit.client.0.smithi136.stdout:snap: snap2 2024-09-13T00:29:27.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:26 smithi181 bash[67587]: cluster 2024-09-13T00:29:25.635931+0000 mon.a (mon.0) 2763 : cluster [DBG] osdmap e1488: 8 total, 8 up, 8 in 2024-09-13T00:29:27.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:26 smithi181 bash[67587]: cluster 2024-09-13T00:29:25.991845+0000 mgr.y (mgr.24661) 1569 : cluster [DBG] pgmap v2618: 137 pgs: 137 active+clean; 730 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 27 MiB/s rd, 14 MiB/s wr, 418 op/s 2024-09-13T00:29:27.697 INFO:tasks.workunit.client.0.smithi136.stdout:clone image287 -> image288 2024-09-13T00:29:27.899 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 153007197 2024-09-13T00:29:27.915 INFO:tasks.workunit.client.0.smithi136.stdout:write: D 3566113~4718386 2024-09-13T00:29:27.952 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 142389814~5183411 2024-09-13T00:29:27.953 INFO:tasks.workunit.client.0.smithi136.stdout:write: D 63410546~4002212 2024-09-13T00:29:27.986 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 115760398~3572047 2024-09-13T00:29:27.986 INFO:tasks.workunit.client.0.smithi136.stdout:write: D 149911626~880909 2024-09-13T00:29:27.996 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 83906348~7314384 2024-09-13T00:29:28.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:27 smithi136 bash[75164]: cluster 2024-09-13T00:29:26.648349+0000 mon.a (mon.0) 2764 : cluster [DBG] osdmap e1489: 8 total, 8 up, 8 in 2024-09-13T00:29:28.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:27 smithi136 bash[76371]: cluster 2024-09-13T00:29:26.648349+0000 mon.a (mon.0) 2764 : cluster [DBG] osdmap e1489: 8 total, 8 up, 8 in 2024-09-13T00:29:28.009 INFO:tasks.workunit.client.0.smithi136.stdout:write: D 86952076~808239 2024-09-13T00:29:28.015 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 12735507~6374338 2024-09-13T00:29:28.023 INFO:tasks.workunit.client.0.smithi136.stdout:snap: snap3 2024-09-13T00:29:28.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:27 smithi181 bash[67587]: cluster 2024-09-13T00:29:26.648349+0000 mon.a (mon.0) 2764 : cluster [DBG] osdmap e1489: 8 total, 8 up, 8 in 2024-09-13T00:29:28.715 INFO:tasks.workunit.client.0.smithi136.stdout:clone image288 -> image289 2024-09-13T00:29:28.945 INFO:tasks.workunit.client.0.smithi136.stdout:resize: 87584868 2024-09-13T00:29:29.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:28 smithi136 bash[75164]: cluster 2024-09-13T00:29:27.652455+0000 mon.a (mon.0) 2765 : cluster [DBG] osdmap e1490: 8 total, 8 up, 8 in 2024-09-13T00:29:29.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:28 smithi136 bash[75164]: cluster 2024-09-13T00:29:27.992523+0000 mgr.y (mgr.24661) 1570 : cluster [DBG] pgmap v2621: 137 pgs: 137 active+clean; 825 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 48 MiB/s rd, 33 MiB/s wr, 665 op/s 2024-09-13T00:29:29.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:28 smithi136 bash[76371]: cluster 2024-09-13T00:29:27.652455+0000 mon.a (mon.0) 2765 : cluster [DBG] osdmap e1490: 8 total, 8 up, 8 in 2024-09-13T00:29:29.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:28 smithi136 bash[76371]: cluster 2024-09-13T00:29:27.992523+0000 mgr.y (mgr.24661) 1570 : cluster [DBG] pgmap v2621: 137 pgs: 137 active+clean; 825 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 48 MiB/s rd, 33 MiB/s wr, 665 op/s 2024-09-13T00:29:29.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:28 smithi181 bash[67587]: cluster 2024-09-13T00:29:27.652455+0000 mon.a (mon.0) 2765 : cluster [DBG] osdmap e1490: 8 total, 8 up, 8 in 2024-09-13T00:29:29.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:28 smithi181 bash[67587]: cluster 2024-09-13T00:29:27.992523+0000 mgr.y (mgr.24661) 1570 : cluster [DBG] pgmap v2621: 137 pgs: 137 active+clean; 825 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 48 MiB/s rd, 33 MiB/s wr, 665 op/s 2024-09-13T00:29:29.550 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestDeepCopy.Stress_SmallerDstObjSize (4243 ms) 2024-09-13T00:29:29.550 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestDeepCopy.NoSnaps_StrippingLargerDstObjSize 2024-09-13T00:29:29.605 INFO:tasks.workunit.client.0.smithi136.stdout:SKIPPING 2024-09-13T00:29:29.666 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:29:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:29:29] "GET /metrics HTTP/1.1" 200 36562 "" "Prometheus/2.43.0" 2024-09-13T00:29:29.776 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestDeepCopy.NoSnaps_StrippingLargerDstObjSize (226 ms) 2024-09-13T00:29:29.776 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestDeepCopy.Snaps_StrippingLargerDstObjSize 2024-09-13T00:29:29.837 INFO:tasks.workunit.client.0.smithi136.stdout:SKIPPING 2024-09-13T00:29:30.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:29 smithi136 bash[75164]: cluster 2024-09-13T00:29:28.672484+0000 mon.a (mon.0) 2766 : cluster [DBG] osdmap e1491: 8 total, 8 up, 8 in 2024-09-13T00:29:30.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:29 smithi136 bash[76371]: cluster 2024-09-13T00:29:28.672484+0000 mon.a (mon.0) 2766 : cluster [DBG] osdmap e1491: 8 total, 8 up, 8 in 2024-09-13T00:29:30.010 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestDeepCopy.Snaps_StrippingLargerDstObjSize (234 ms) 2024-09-13T00:29:30.010 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestDeepCopy.Clone_StrippingLargerDstObjSize 2024-09-13T00:29:30.072 INFO:tasks.workunit.client.0.smithi136.stdout:SKIPPING 2024-09-13T00:29:30.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:29 smithi181 bash[67587]: cluster 2024-09-13T00:29:28.672484+0000 mon.a (mon.0) 2766 : cluster [DBG] osdmap e1491: 8 total, 8 up, 8 in 2024-09-13T00:29:30.252 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestDeepCopy.Clone_StrippingLargerDstObjSize (241 ms) 2024-09-13T00:29:30.252 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestDeepCopy.CloneFlatten_StrippingLargerDstObjSize 2024-09-13T00:29:30.309 INFO:tasks.workunit.client.0.smithi136.stdout:SKIPPING 2024-09-13T00:29:30.486 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestDeepCopy.CloneFlatten_StrippingLargerDstObjSize (235 ms) 2024-09-13T00:29:30.487 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestDeepCopy.Stress_StrippingLargerDstObjSize 2024-09-13T00:29:30.547 INFO:tasks.workunit.client.0.smithi136.stdout:SKIPPING 2024-09-13T00:29:30.721 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestDeepCopy.Stress_StrippingLargerDstObjSize (235 ms) 2024-09-13T00:29:30.722 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestDeepCopy.NoSnaps_StrippingSmallerDstObjSize 2024-09-13T00:29:30.783 INFO:tasks.workunit.client.0.smithi136.stdout:SKIPPING 2024-09-13T00:29:30.953 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestDeepCopy.NoSnaps_StrippingSmallerDstObjSize (231 ms) 2024-09-13T00:29:30.953 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestDeepCopy.Snaps_StrippingSmallerDstObjSize 2024-09-13T00:29:31.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:30 smithi136 bash[75164]: cluster 2024-09-13T00:29:29.993518+0000 mgr.y (mgr.24661) 1571 : cluster [DBG] pgmap v2623: 137 pgs: 137 active+clean; 877 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 37 MiB/s rd, 32 MiB/s wr, 484 op/s 2024-09-13T00:29:31.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:30 smithi136 bash[76371]: cluster 2024-09-13T00:29:29.993518+0000 mgr.y (mgr.24661) 1571 : cluster [DBG] pgmap v2623: 137 pgs: 137 active+clean; 877 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 37 MiB/s rd, 32 MiB/s wr, 484 op/s 2024-09-13T00:29:31.012 INFO:tasks.workunit.client.0.smithi136.stdout:SKIPPING 2024-09-13T00:29:31.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:30 smithi181 bash[67587]: cluster 2024-09-13T00:29:29.993518+0000 mgr.y (mgr.24661) 1571 : cluster [DBG] pgmap v2623: 137 pgs: 137 active+clean; 877 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 37 MiB/s rd, 32 MiB/s wr, 484 op/s 2024-09-13T00:29:31.187 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestDeepCopy.Snaps_StrippingSmallerDstObjSize (235 ms) 2024-09-13T00:29:31.188 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestDeepCopy.Clone_StrippingSmallerDstObjSize 2024-09-13T00:29:31.249 INFO:tasks.workunit.client.0.smithi136.stdout:SKIPPING 2024-09-13T00:29:31.422 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestDeepCopy.Clone_StrippingSmallerDstObjSize (235 ms) 2024-09-13T00:29:31.423 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestDeepCopy.CloneFlatten_StrippingSmallerDstObjSize 2024-09-13T00:29:31.483 INFO:tasks.workunit.client.0.smithi136.stdout:SKIPPING 2024-09-13T00:29:31.657 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestDeepCopy.CloneFlatten_StrippingSmallerDstObjSize (235 ms) 2024-09-13T00:29:31.658 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestDeepCopy.Stress_StrippingSmallerDstObjSize 2024-09-13T00:29:31.719 INFO:tasks.workunit.client.0.smithi136.stdout:SKIPPING 2024-09-13T00:29:31.897 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestDeepCopy.Stress_StrippingSmallerDstObjSize (240 ms) 2024-09-13T00:29:31.968 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 31 tests from TestDeepCopy (56793 ms total) 2024-09-13T00:29:31.968 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:29:31.969 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 51 tests from TestMirroring 2024-09-13T00:29:32.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:29:32 smithi181 bash[95505]: ts=2024-09-13T00:29:32.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:29:33.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:32 smithi136 bash[75164]: cluster 2024-09-13T00:29:31.961781+0000 mon.a (mon.0) 2767 : cluster [DBG] osdmap e1492: 8 total, 8 up, 8 in 2024-09-13T00:29:33.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:32 smithi136 bash[75164]: cluster 2024-09-13T00:29:31.994333+0000 mgr.y (mgr.24661) 1572 : cluster [DBG] pgmap v2625: 105 pgs: 105 active+clean; 4.6 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2024-09-13T00:29:33.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:32 smithi136 bash[76371]: cluster 2024-09-13T00:29:31.961781+0000 mon.a (mon.0) 2767 : cluster [DBG] osdmap e1492: 8 total, 8 up, 8 in 2024-09-13T00:29:33.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:32 smithi136 bash[76371]: cluster 2024-09-13T00:29:31.994333+0000 mgr.y (mgr.24661) 1572 : cluster [DBG] pgmap v2625: 105 pgs: 105 active+clean; 4.6 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2024-09-13T00:29:33.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:32 smithi181 bash[67587]: cluster 2024-09-13T00:29:31.961781+0000 mon.a (mon.0) 2767 : cluster [DBG] osdmap e1492: 8 total, 8 up, 8 in 2024-09-13T00:29:33.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:32 smithi181 bash[67587]: cluster 2024-09-13T00:29:31.994333+0000 mgr.y (mgr.24661) 1572 : cluster [DBG] pgmap v2625: 105 pgs: 105 active+clean; 4.6 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2024-09-13T00:29:33.985 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModeImage 2024-09-13T00:29:34.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:33 smithi136 bash[75164]: cluster 2024-09-13T00:29:32.982083+0000 mon.a (mon.0) 2768 : cluster [DBG] osdmap e1493: 8 total, 8 up, 8 in 2024-09-13T00:29:34.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:33 smithi136 bash[75164]: audit 2024-09-13T00:29:32.984822+0000 mon.c (mon.2) 99 : audit [INF] from='client.? 172.21.15.136:0/2010575863' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:29:34.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:33 smithi136 bash[75164]: audit 2024-09-13T00:29:32.985902+0000 mon.a (mon.0) 2769 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:29:34.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:33 smithi136 bash[76371]: cluster 2024-09-13T00:29:32.982083+0000 mon.a (mon.0) 2768 : cluster [DBG] osdmap e1493: 8 total, 8 up, 8 in 2024-09-13T00:29:34.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:33 smithi136 bash[76371]: audit 2024-09-13T00:29:32.984822+0000 mon.c (mon.2) 99 : audit [INF] from='client.? 172.21.15.136:0/2010575863' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:29:34.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:33 smithi136 bash[76371]: audit 2024-09-13T00:29:32.985902+0000 mon.a (mon.0) 2769 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:29:34.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:33 smithi181 bash[67587]: cluster 2024-09-13T00:29:32.982083+0000 mon.a (mon.0) 2768 : cluster [DBG] osdmap e1493: 8 total, 8 up, 8 in 2024-09-13T00:29:34.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:33 smithi181 bash[67587]: audit 2024-09-13T00:29:32.984822+0000 mon.c (mon.2) 99 : audit [INF] from='client.? 172.21.15.136:0/2010575863' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:29:34.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:33 smithi181 bash[67587]: audit 2024-09-13T00:29:32.985902+0000 mon.a (mon.0) 2769 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:29:35.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:34 smithi136 bash[76371]: audit 2024-09-13T00:29:33.984384+0000 mon.a (mon.0) 2770 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:29:35.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:34 smithi136 bash[76371]: cluster 2024-09-13T00:29:33.984787+0000 mon.a (mon.0) 2771 : cluster [DBG] osdmap e1494: 8 total, 8 up, 8 in 2024-09-13T00:29:35.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:34 smithi136 bash[76371]: cluster 2024-09-13T00:29:33.994926+0000 mgr.y (mgr.24661) 1573 : cluster [DBG] pgmap v2628: 137 pgs: 2 creating+peering, 30 unknown, 105 active+clean; 4.6 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail 2024-09-13T00:29:35.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:34 smithi136 bash[75164]: audit 2024-09-13T00:29:33.984384+0000 mon.a (mon.0) 2770 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:29:35.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:34 smithi136 bash[75164]: cluster 2024-09-13T00:29:33.984787+0000 mon.a (mon.0) 2771 : cluster [DBG] osdmap e1494: 8 total, 8 up, 8 in 2024-09-13T00:29:35.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:34 smithi136 bash[75164]: cluster 2024-09-13T00:29:33.994926+0000 mgr.y (mgr.24661) 1573 : cluster [DBG] pgmap v2628: 137 pgs: 2 creating+peering, 30 unknown, 105 active+clean; 4.6 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail 2024-09-13T00:29:35.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:34 smithi181 bash[67587]: audit 2024-09-13T00:29:33.984384+0000 mon.a (mon.0) 2770 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:29:35.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:34 smithi181 bash[67587]: cluster 2024-09-13T00:29:33.984787+0000 mon.a (mon.0) 2771 : cluster [DBG] osdmap e1494: 8 total, 8 up, 8 in 2024-09-13T00:29:35.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:34 smithi181 bash[67587]: cluster 2024-09-13T00:29:33.994926+0000 mgr.y (mgr.24661) 1573 : cluster [DBG] pgmap v2628: 137 pgs: 2 creating+peering, 30 unknown, 105 active+clean; 4.6 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail 2024-09-13T00:29:36.232 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModeImage (2246 ms) 2024-09-13T00:29:36.232 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModePool 2024-09-13T00:29:36.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:36 smithi181 bash[67587]: cluster 2024-09-13T00:29:35.009335+0000 mon.a (mon.0) 2772 : cluster [DBG] osdmap e1495: 8 total, 8 up, 8 in 2024-09-13T00:29:36.472 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModePool (240 ms) 2024-09-13T00:29:36.472 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModeDisabled 2024-09-13T00:29:36.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:36 smithi136 bash[75164]: cluster 2024-09-13T00:29:35.009335+0000 mon.a (mon.0) 2772 : cluster [DBG] osdmap e1495: 8 total, 8 up, 8 in 2024-09-13T00:29:36.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:36 smithi136 bash[76371]: cluster 2024-09-13T00:29:35.009335+0000 mon.a (mon.0) 2772 : cluster [DBG] osdmap e1495: 8 total, 8 up, 8 in 2024-09-13T00:29:36.683 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModeDisabled (211 ms) 2024-09-13T00:29:36.683 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMirroring.DisableImageMirror_In_MirrorModeImage 2024-09-13T00:29:36.946 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMirroring.DisableImageMirror_In_MirrorModeImage (264 ms) 2024-09-13T00:29:36.947 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMirroring.DisableImageMirror_In_MirrorModeImage_NoObjectMap 2024-09-13T00:29:37.145 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMirroring.DisableImageMirror_In_MirrorModeImage_NoObjectMap (199 ms) 2024-09-13T00:29:37.145 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMirroring.DisableImageMirror_In_MirrorModePool 2024-09-13T00:29:37.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:37 smithi181 bash[67587]: cluster 2024-09-13T00:29:35.995830+0000 mgr.y (mgr.24661) 1574 : cluster [DBG] pgmap v2630: 137 pgs: 2 creating+peering, 18 unknown, 117 active+clean; 4.6 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2024-09-13T00:29:37.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:37 smithi181 bash[67587]: cluster 2024-09-13T00:29:36.018701+0000 mon.a (mon.0) 2773 : cluster [DBG] osdmap e1496: 8 total, 8 up, 8 in 2024-09-13T00:29:37.362 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMirroring.DisableImageMirror_In_MirrorModePool (216 ms) 2024-09-13T00:29:37.362 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMirroring.DisableImageMirror_In_MirrorModeDisabled 2024-09-13T00:29:37.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:37 smithi136 bash[76371]: cluster 2024-09-13T00:29:35.995830+0000 mgr.y (mgr.24661) 1574 : cluster [DBG] pgmap v2630: 137 pgs: 2 creating+peering, 18 unknown, 117 active+clean; 4.6 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2024-09-13T00:29:37.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:37 smithi136 bash[76371]: cluster 2024-09-13T00:29:36.018701+0000 mon.a (mon.0) 2773 : cluster [DBG] osdmap e1496: 8 total, 8 up, 8 in 2024-09-13T00:29:37.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:37 smithi136 bash[75164]: cluster 2024-09-13T00:29:35.995830+0000 mgr.y (mgr.24661) 1574 : cluster [DBG] pgmap v2630: 137 pgs: 2 creating+peering, 18 unknown, 117 active+clean; 4.6 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2024-09-13T00:29:37.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:37 smithi136 bash[75164]: cluster 2024-09-13T00:29:36.018701+0000 mon.a (mon.0) 2773 : cluster [DBG] osdmap e1496: 8 total, 8 up, 8 in 2024-09-13T00:29:37.580 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMirroring.DisableImageMirror_In_MirrorModeDisabled (219 ms) 2024-09-13T00:29:37.581 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMirroring.DisableImageMirrorWithPeer 2024-09-13T00:29:37.581 INFO:tasks.workunit.client.0.smithi136.stdout:SKIPPING 2024-09-13T00:29:37.581 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMirroring.DisableImageMirrorWithPeer (0 ms) 2024-09-13T00:29:37.581 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMirroring.DisableJournalingWithPeer 2024-09-13T00:29:37.581 INFO:tasks.workunit.client.0.smithi136.stdout:SKIPPING 2024-09-13T00:29:37.581 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMirroring.DisableJournalingWithPeer (0 ms) 2024-09-13T00:29:37.581 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModeDisabled_WithoutJournaling 2024-09-13T00:29:37.726 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModeDisabled_WithoutJournaling (146 ms) 2024-09-13T00:29:37.726 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModePool_WithoutJournaling 2024-09-13T00:29:37.938 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModePool_WithoutJournaling (211 ms) 2024-09-13T00:29:37.938 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModeImage_WithoutJournaling 2024-09-13T00:29:39.153 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModeImage_WithoutJournaling (1215 ms) 2024-09-13T00:29:39.153 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModeImage_WithoutExclusiveLock 2024-09-13T00:29:39.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:39 smithi181 bash[67587]: cluster 2024-09-13T00:29:37.997079+0000 mgr.y (mgr.24661) 1575 : cluster [DBG] pgmap v2632: 137 pgs: 2 creating+peering, 135 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 56 KiB/s rd, 12 KiB/s wr, 83 op/s 2024-09-13T00:29:39.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:39 smithi181 bash[67587]: cluster 2024-09-13T00:29:38.027796+0000 mon.a (mon.0) 2774 : cluster [DBG] osdmap e1497: 8 total, 8 up, 8 in 2024-09-13T00:29:39.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:39 smithi181 bash[67587]: audit 2024-09-13T00:29:38.820737+0000 mon.a (mon.0) 2775 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:29:39.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:39 smithi136 bash[76371]: cluster 2024-09-13T00:29:37.997079+0000 mgr.y (mgr.24661) 1575 : cluster [DBG] pgmap v2632: 137 pgs: 2 creating+peering, 135 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 56 KiB/s rd, 12 KiB/s wr, 83 op/s 2024-09-13T00:29:39.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:39 smithi136 bash[76371]: cluster 2024-09-13T00:29:38.027796+0000 mon.a (mon.0) 2774 : cluster [DBG] osdmap e1497: 8 total, 8 up, 8 in 2024-09-13T00:29:39.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:39 smithi136 bash[76371]: audit 2024-09-13T00:29:38.820737+0000 mon.a (mon.0) 2775 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:29:39.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:39 smithi136 bash[75164]: cluster 2024-09-13T00:29:37.997079+0000 mgr.y (mgr.24661) 1575 : cluster [DBG] pgmap v2632: 137 pgs: 2 creating+peering, 135 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 56 KiB/s rd, 12 KiB/s wr, 83 op/s 2024-09-13T00:29:39.414 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:39 smithi136 bash[75164]: cluster 2024-09-13T00:29:38.027796+0000 mon.a (mon.0) 2774 : cluster [DBG] osdmap e1497: 8 total, 8 up, 8 in 2024-09-13T00:29:39.414 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:39 smithi136 bash[75164]: audit 2024-09-13T00:29:38.820737+0000 mon.a (mon.0) 2775 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:29:39.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:29:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:29:39] "GET /metrics HTTP/1.1" 200 36531 "" "Prometheus/2.43.0" 2024-09-13T00:29:40.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:40 smithi181 bash[67587]: cluster 2024-09-13T00:29:39.045169+0000 mon.a (mon.0) 2776 : cluster [DBG] osdmap e1498: 8 total, 8 up, 8 in 2024-09-13T00:29:40.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:40 smithi136 bash[75164]: cluster 2024-09-13T00:29:39.045169+0000 mon.a (mon.0) 2776 : cluster [DBG] osdmap e1498: 8 total, 8 up, 8 in 2024-09-13T00:29:40.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:40 smithi136 bash[76371]: cluster 2024-09-13T00:29:39.045169+0000 mon.a (mon.0) 2776 : cluster [DBG] osdmap e1498: 8 total, 8 up, 8 in 2024-09-13T00:29:41.154 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModeImage_WithoutExclusiveLock (2001 ms) 2024-09-13T00:29:41.154 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMirroring.CreateImage_In_MirrorModeDisabled 2024-09-13T00:29:41.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:41 smithi181 bash[67587]: cluster 2024-09-13T00:29:39.997927+0000 mgr.y (mgr.24661) 1576 : cluster [DBG] pgmap v2635: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 116 KiB/s rd, 21 KiB/s wr, 167 op/s 2024-09-13T00:29:41.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:41 smithi181 bash[67587]: cluster 2024-09-13T00:29:40.053468+0000 mon.a (mon.0) 2777 : cluster [DBG] osdmap e1499: 8 total, 8 up, 8 in 2024-09-13T00:29:41.400 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMirroring.CreateImage_In_MirrorModeDisabled (246 ms) 2024-09-13T00:29:41.400 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMirroring.CreateImage_In_MirrorModeImage 2024-09-13T00:29:41.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:41 smithi136 bash[76371]: cluster 2024-09-13T00:29:39.997927+0000 mgr.y (mgr.24661) 1576 : cluster [DBG] pgmap v2635: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 116 KiB/s rd, 21 KiB/s wr, 167 op/s 2024-09-13T00:29:41.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:41 smithi136 bash[76371]: cluster 2024-09-13T00:29:40.053468+0000 mon.a (mon.0) 2777 : cluster [DBG] osdmap e1499: 8 total, 8 up, 8 in 2024-09-13T00:29:41.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:41 smithi136 bash[75164]: cluster 2024-09-13T00:29:39.997927+0000 mgr.y (mgr.24661) 1576 : cluster [DBG] pgmap v2635: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 116 KiB/s rd, 21 KiB/s wr, 167 op/s 2024-09-13T00:29:41.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:41 smithi136 bash[75164]: cluster 2024-09-13T00:29:40.053468+0000 mon.a (mon.0) 2777 : cluster [DBG] osdmap e1499: 8 total, 8 up, 8 in 2024-09-13T00:29:41.652 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMirroring.CreateImage_In_MirrorModeImage (253 ms) 2024-09-13T00:29:41.652 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMirroring.CreateImage_In_MirrorModePool 2024-09-13T00:29:41.902 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMirroring.CreateImage_In_MirrorModePool (250 ms) 2024-09-13T00:29:41.902 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMirroring.CreateImage_In_MirrorModePool_WithoutJournaling 2024-09-13T00:29:42.129 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMirroring.CreateImage_In_MirrorModePool_WithoutJournaling (227 ms) 2024-09-13T00:29:42.129 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMirroring.CreateImage_In_MirrorModeImage_WithoutJournaling 2024-09-13T00:29:42.337 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMirroring.CreateImage_In_MirrorModeImage_WithoutJournaling (207 ms) 2024-09-13T00:29:42.337 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMirroring.EnableJournaling_In_MirrorModeDisabled 2024-09-13T00:29:42.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:42 smithi181 bash[67587]: cluster 2024-09-13T00:29:41.053437+0000 mon.a (mon.0) 2778 : cluster [DBG] osdmap e1500: 8 total, 8 up, 8 in 2024-09-13T00:29:42.348 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:29:42 smithi181 bash[95505]: ts=2024-09-13T00:29:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:29:42.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:42 smithi136 bash[76371]: cluster 2024-09-13T00:29:41.053437+0000 mon.a (mon.0) 2778 : cluster [DBG] osdmap e1500: 8 total, 8 up, 8 in 2024-09-13T00:29:42.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:42 smithi136 bash[75164]: cluster 2024-09-13T00:29:41.053437+0000 mon.a (mon.0) 2778 : cluster [DBG] osdmap e1500: 8 total, 8 up, 8 in 2024-09-13T00:29:42.598 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMirroring.EnableJournaling_In_MirrorModeDisabled (262 ms) 2024-09-13T00:29:42.598 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMirroring.EnableJournaling_In_MirrorModeImage 2024-09-13T00:29:42.859 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMirroring.EnableJournaling_In_MirrorModeImage (260 ms) 2024-09-13T00:29:42.859 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMirroring.EnableJournaling_In_MirrorModeImage_SnapshotMirroringEnabled 2024-09-13T00:29:43.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:43 smithi181 bash[67587]: cluster 2024-09-13T00:29:41.999197+0000 mgr.y (mgr.24661) 1577 : cluster [DBG] pgmap v2638: 137 pgs: 137 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 372 KiB/s rd, 45 KiB/s wr, 538 op/s 2024-09-13T00:29:43.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:43 smithi136 bash[75164]: cluster 2024-09-13T00:29:41.999197+0000 mgr.y (mgr.24661) 1577 : cluster [DBG] pgmap v2638: 137 pgs: 137 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 372 KiB/s rd, 45 KiB/s wr, 538 op/s 2024-09-13T00:29:43.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:43 smithi136 bash[76371]: cluster 2024-09-13T00:29:41.999197+0000 mgr.y (mgr.24661) 1577 : cluster [DBG] pgmap v2638: 137 pgs: 137 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 372 KiB/s rd, 45 KiB/s wr, 538 op/s 2024-09-13T00:29:44.215 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMirroring.EnableJournaling_In_MirrorModeImage_SnapshotMirroringEnabled (1356 ms) 2024-09-13T00:29:44.215 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMirroring.EnableJournaling_In_MirrorModePool 2024-09-13T00:29:44.489 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMirroring.EnableJournaling_In_MirrorModePool (274 ms) 2024-09-13T00:29:44.489 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMirroring.DisableJournaling_In_MirrorModePool 2024-09-13T00:29:44.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:44 smithi136 bash[75164]: cluster 2024-09-13T00:29:43.077076+0000 mon.a (mon.0) 2779 : cluster [DBG] osdmap e1501: 8 total, 8 up, 8 in 2024-09-13T00:29:44.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:44 smithi136 bash[75164]: cluster 2024-09-13T00:29:44.070366+0000 mon.a (mon.0) 2780 : cluster [DBG] osdmap e1502: 8 total, 8 up, 8 in 2024-09-13T00:29:44.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:44 smithi136 bash[76371]: cluster 2024-09-13T00:29:43.077076+0000 mon.a (mon.0) 2779 : cluster [DBG] osdmap e1501: 8 total, 8 up, 8 in 2024-09-13T00:29:44.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:44 smithi136 bash[76371]: cluster 2024-09-13T00:29:44.070366+0000 mon.a (mon.0) 2780 : cluster [DBG] osdmap e1502: 8 total, 8 up, 8 in 2024-09-13T00:29:44.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:44 smithi181 bash[67587]: cluster 2024-09-13T00:29:43.077076+0000 mon.a (mon.0) 2779 : cluster [DBG] osdmap e1501: 8 total, 8 up, 8 in 2024-09-13T00:29:44.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:44 smithi181 bash[67587]: cluster 2024-09-13T00:29:44.070366+0000 mon.a (mon.0) 2780 : cluster [DBG] osdmap e1502: 8 total, 8 up, 8 in 2024-09-13T00:29:44.721 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMirroring.DisableJournaling_In_MirrorModePool (233 ms) 2024-09-13T00:29:44.722 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMirroring.DisableJournaling_In_MirrorModeImage 2024-09-13T00:29:44.932 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMirroring.DisableJournaling_In_MirrorModeImage (210 ms) 2024-09-13T00:29:44.932 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMirroring.MirrorModeSet_DisabledMode_To_PoolMode 2024-09-13T00:29:45.299 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMirroring.MirrorModeSet_DisabledMode_To_PoolMode (367 ms) 2024-09-13T00:29:45.299 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMirroring.MirrorModeSet_PoolMode_To_DisabledMode 2024-09-13T00:29:45.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:45 smithi136 bash[76371]: cluster 2024-09-13T00:29:44.000147+0000 mgr.y (mgr.24661) 1578 : cluster [DBG] pgmap v2640: 137 pgs: 137 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 317 KiB/s rd, 35 KiB/s wr, 457 op/s 2024-09-13T00:29:45.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:45 smithi136 bash[75164]: cluster 2024-09-13T00:29:44.000147+0000 mgr.y (mgr.24661) 1578 : cluster [DBG] pgmap v2640: 137 pgs: 137 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 317 KiB/s rd, 35 KiB/s wr, 457 op/s 2024-09-13T00:29:45.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:45 smithi181 bash[67587]: cluster 2024-09-13T00:29:44.000147+0000 mgr.y (mgr.24661) 1578 : cluster [DBG] pgmap v2640: 137 pgs: 137 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 317 KiB/s rd, 35 KiB/s wr, 457 op/s 2024-09-13T00:29:45.721 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMirroring.MirrorModeSet_PoolMode_To_DisabledMode (422 ms) 2024-09-13T00:29:45.721 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMirroring.MirrorModeSet_DisabledMode_To_ImageMode 2024-09-13T00:29:46.129 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMirroring.MirrorModeSet_DisabledMode_To_ImageMode (408 ms) 2024-09-13T00:29:46.129 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMirroring.MirrorModeSet_PoolMode_To_ImageMode 2024-09-13T00:29:46.491 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMirroring.MirrorModeSet_PoolMode_To_ImageMode (363 ms) 2024-09-13T00:29:46.492 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMirroring.MirrorModeSet_ImageMode_To_PoolMode 2024-09-13T00:29:46.826 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMirroring.MirrorModeSet_ImageMode_To_PoolMode (334 ms) 2024-09-13T00:29:46.826 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMirroring.MirrorModeSet_ImageMode_To_DisabledMode 2024-09-13T00:29:47.295 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMirroring.MirrorModeSet_ImageMode_To_DisabledMode (469 ms) 2024-09-13T00:29:47.295 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMirroring.RemoveImage_With_MirrorImageEnabled 2024-09-13T00:29:47.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:47 smithi136 bash[75164]: cluster 2024-09-13T00:29:46.001304+0000 mgr.y (mgr.24661) 1579 : cluster [DBG] pgmap v2642: 137 pgs: 137 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 391 KiB/s rd, 45 KiB/s wr, 574 op/s 2024-09-13T00:29:47.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:47 smithi136 bash[76371]: cluster 2024-09-13T00:29:46.001304+0000 mgr.y (mgr.24661) 1579 : cluster [DBG] pgmap v2642: 137 pgs: 137 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 391 KiB/s rd, 45 KiB/s wr, 574 op/s 2024-09-13T00:29:47.516 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMirroring.RemoveImage_With_MirrorImageEnabled (221 ms) 2024-09-13T00:29:47.516 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMirroring.RemoveImage_With_MirrorImageDisabled 2024-09-13T00:29:47.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:47 smithi181 bash[67587]: cluster 2024-09-13T00:29:46.001304+0000 mgr.y (mgr.24661) 1579 : cluster [DBG] pgmap v2642: 137 pgs: 137 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 391 KiB/s rd, 45 KiB/s wr, 574 op/s 2024-09-13T00:29:47.740 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMirroring.RemoveImage_With_MirrorImageDisabled (224 ms) 2024-09-13T00:29:47.740 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMirroring.RemoveImage_With_ImageWithoutJournal 2024-09-13T00:29:47.921 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMirroring.RemoveImage_With_ImageWithoutJournal (182 ms) 2024-09-13T00:29:47.922 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMirroring.RemoveImage_With_MirrorImageDemoted 2024-09-13T00:29:48.167 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMirroring.RemoveImage_With_MirrorImageDemoted (245 ms) 2024-09-13T00:29:48.167 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMirroring.TrashMoveRestore_PoolMode 2024-09-13T00:29:48.485 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMirroring.TrashMoveRestore_PoolMode (318 ms) 2024-09-13T00:29:48.485 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMirroring.TrashMoveRestore_ImageMode_MirroringDisabled 2024-09-13T00:29:48.791 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMirroring.TrashMoveRestore_ImageMode_MirroringDisabled (307 ms) 2024-09-13T00:29:48.791 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMirroring.TrashMoveRestore_ImageMode_MirroringEnabled 2024-09-13T00:29:49.093 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMirroring.TrashMoveRestore_ImageMode_MirroringEnabled (301 ms) 2024-09-13T00:29:49.093 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMirroring.MirrorStatusList 2024-09-13T00:29:49.258 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMirroring.MirrorStatusList (165 ms) 2024-09-13T00:29:49.258 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMirroring.RemoveBootstrapped 2024-09-13T00:29:49.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:49 smithi136 bash[75164]: cluster 2024-09-13T00:29:48.002128+0000 mgr.y (mgr.24661) 1580 : cluster [DBG] pgmap v2643: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 486 KiB/s rd, 61 KiB/s wr, 693 op/s 2024-09-13T00:29:49.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:49 smithi136 bash[75164]: cluster 2024-09-13T00:29:49.072847+0000 mon.a (mon.0) 2781 : cluster [DBG] osdmap e1503: 8 total, 8 up, 8 in 2024-09-13T00:29:49.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:49 smithi136 bash[76371]: cluster 2024-09-13T00:29:48.002128+0000 mgr.y (mgr.24661) 1580 : cluster [DBG] pgmap v2643: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 486 KiB/s rd, 61 KiB/s wr, 693 op/s 2024-09-13T00:29:49.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:49 smithi136 bash[76371]: cluster 2024-09-13T00:29:49.072847+0000 mon.a (mon.0) 2781 : cluster [DBG] osdmap e1503: 8 total, 8 up, 8 in 2024-09-13T00:29:49.501 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMirroring.RemoveBootstrapped (243 ms) 2024-09-13T00:29:49.501 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMirroring.AioPromoteDemote 2024-09-13T00:29:49.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:49 smithi181 bash[67587]: cluster 2024-09-13T00:29:48.002128+0000 mgr.y (mgr.24661) 1580 : cluster [DBG] pgmap v2643: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 486 KiB/s rd, 61 KiB/s wr, 693 op/s 2024-09-13T00:29:49.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:49 smithi181 bash[67587]: cluster 2024-09-13T00:29:49.072847+0000 mon.a (mon.0) 2781 : cluster [DBG] osdmap e1503: 8 total, 8 up, 8 in 2024-09-13T00:29:49.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:29:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:29:49] "GET /metrics HTTP/1.1" 200 36531 "" "Prometheus/2.43.0" 2024-09-13T00:29:50.192 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMirroring.AioPromoteDemote (691 ms) 2024-09-13T00:29:50.192 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMirroring.AioGetInfo 2024-09-13T00:29:50.926 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMirroring.AioGetInfo (734 ms) 2024-09-13T00:29:50.926 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMirroring.AioGetStatus 2024-09-13T00:29:51.344 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMirroring.AioGetStatus (418 ms) 2024-09-13T00:29:51.344 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMirroring.SiteName 2024-09-13T00:29:51.355 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMirroring.SiteName (12 ms) 2024-09-13T00:29:51.355 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMirroring.Bootstrap 2024-09-13T00:29:51.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:51 smithi136 bash[75164]: cluster 2024-09-13T00:29:50.002795+0000 mgr.y (mgr.24661) 1581 : cluster [DBG] pgmap v2645: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 409 KiB/s rd, 62 KiB/s wr, 585 op/s 2024-09-13T00:29:51.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:51 smithi136 bash[76371]: cluster 2024-09-13T00:29:50.002795+0000 mgr.y (mgr.24661) 1581 : cluster [DBG] pgmap v2645: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 409 KiB/s rd, 62 KiB/s wr, 585 op/s 2024-09-13T00:29:51.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:51 smithi181 bash[67587]: cluster 2024-09-13T00:29:50.002795+0000 mgr.y (mgr.24661) 1581 : cluster [DBG] pgmap v2645: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 409 KiB/s rd, 62 KiB/s wr, 585 op/s 2024-09-13T00:29:52.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:29:52 smithi181 bash[95505]: ts=2024-09-13T00:29:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:29:52.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:52 smithi181 bash[67587]: audit 2024-09-13T00:29:51.350133+0000 mon.a (mon.0) 2782 : audit [INF] from='client.? ' entity='client.admin' 2024-09-13T00:29:52.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:52 smithi181 bash[67587]: audit 2024-09-13T00:29:51.350672+0000 mon.c (mon.2) 100 : audit [DBG] from='client.? 172.21.15.136:0/2010575863' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-09-13T00:29:52.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:52 smithi181 bash[67587]: audit 2024-09-13T00:29:51.350980+0000 mon.c (mon.2) 101 : audit [INF] from='client.? 172.21.15.136:0/2010575863' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]: dispatch 2024-09-13T00:29:52.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:52 smithi181 bash[67587]: audit 2024-09-13T00:29:51.351204+0000 mon.a (mon.0) 2783 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]: dispatch 2024-09-13T00:29:52.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:52 smithi181 bash[67587]: audit 2024-09-13T00:29:51.356577+0000 mon.a (mon.0) 2784 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]': finished 2024-09-13T00:29:52.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:52 smithi181 bash[67587]: audit 2024-09-13T00:29:51.357126+0000 mon.c (mon.2) 102 : audit [DBG] from='client.? 172.21.15.136:0/2010575863' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-09-13T00:29:52.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:52 smithi136 bash[75164]: audit 2024-09-13T00:29:51.350133+0000 mon.a (mon.0) 2782 : audit [INF] from='client.? ' entity='client.admin' 2024-09-13T00:29:52.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:52 smithi136 bash[75164]: audit 2024-09-13T00:29:51.350672+0000 mon.c (mon.2) 100 : audit [DBG] from='client.? 172.21.15.136:0/2010575863' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-09-13T00:29:52.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:52 smithi136 bash[75164]: audit 2024-09-13T00:29:51.350980+0000 mon.c (mon.2) 101 : audit [INF] from='client.? 172.21.15.136:0/2010575863' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]: dispatch 2024-09-13T00:29:52.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:52 smithi136 bash[75164]: audit 2024-09-13T00:29:51.351204+0000 mon.a (mon.0) 2783 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]: dispatch 2024-09-13T00:29:52.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:52 smithi136 bash[75164]: audit 2024-09-13T00:29:51.356577+0000 mon.a (mon.0) 2784 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]': finished 2024-09-13T00:29:52.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:52 smithi136 bash[75164]: audit 2024-09-13T00:29:51.357126+0000 mon.c (mon.2) 102 : audit [DBG] from='client.? 172.21.15.136:0/2010575863' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-09-13T00:29:52.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:52 smithi136 bash[76371]: audit 2024-09-13T00:29:51.350133+0000 mon.a (mon.0) 2782 : audit [INF] from='client.? ' entity='client.admin' 2024-09-13T00:29:52.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:52 smithi136 bash[76371]: audit 2024-09-13T00:29:51.350672+0000 mon.c (mon.2) 100 : audit [DBG] from='client.? 172.21.15.136:0/2010575863' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-09-13T00:29:52.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:52 smithi136 bash[76371]: audit 2024-09-13T00:29:51.350980+0000 mon.c (mon.2) 101 : audit [INF] from='client.? 172.21.15.136:0/2010575863' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]: dispatch 2024-09-13T00:29:52.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:52 smithi136 bash[76371]: audit 2024-09-13T00:29:51.351204+0000 mon.a (mon.0) 2783 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]: dispatch 2024-09-13T00:29:52.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:52 smithi136 bash[76371]: audit 2024-09-13T00:29:51.356577+0000 mon.a (mon.0) 2784 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]': finished 2024-09-13T00:29:52.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:52 smithi136 bash[76371]: audit 2024-09-13T00:29:51.357126+0000 mon.c (mon.2) 102 : audit [DBG] from='client.? 172.21.15.136:0/2010575863' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-09-13T00:29:53.058 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMirroring.Bootstrap (1702 ms) 2024-09-13T00:29:53.058 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMirroring.PeerDirection 2024-09-13T00:29:53.071 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMirroring.PeerDirection (13 ms) 2024-09-13T00:29:53.071 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMirroring.Snapshot 2024-09-13T00:29:53.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:53 smithi136 bash[75164]: cluster 2024-09-13T00:29:52.003659+0000 mgr.y (mgr.24661) 1582 : cluster [DBG] pgmap v2646: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 911 KiB/s rd, 133 KiB/s wr, 1.25k op/s 2024-09-13T00:29:53.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:53 smithi136 bash[75164]: audit 2024-09-13T00:29:53.035347+0000 mon.c (mon.2) 103 : audit [DBG] from='client.? 172.21.15.136:0/2010575863' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer_client_id"}]: dispatch 2024-09-13T00:29:53.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:53 smithi136 bash[75164]: audit 2024-09-13T00:29:53.040676+0000 mon.a (mon.0) 2785 : audit [INF] from='client.? ' entity='client.admin' 2024-09-13T00:29:53.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:53 smithi136 bash[75164]: audit 2024-09-13T00:29:53.041888+0000 mon.c (mon.2) 104 : audit [INF] from='client.? 172.21.15.136:0/2010575863' entity='client.admin' cmd=[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]: dispatch 2024-09-13T00:29:53.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:53 smithi136 bash[75164]: audit 2024-09-13T00:29:53.042242+0000 mon.a (mon.0) 2786 : audit [INF] from='client.? ' entity='client.admin' cmd=[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]: dispatch 2024-09-13T00:29:53.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:53 smithi136 bash[75164]: audit 2024-09-13T00:29:53.046836+0000 mon.a (mon.0) 2787 : audit [INF] from='client.? ' entity='client.admin' cmd='[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]': finished 2024-09-13T00:29:53.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:53 smithi136 bash[75164]: audit 2024-09-13T00:29:53.048136+0000 mon.c (mon.2) 105 : audit [DBG] from='client.? 172.21.15.136:0/2010575863' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-09-13T00:29:53.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:53 smithi136 bash[75164]: audit 2024-09-13T00:29:53.065298+0000 mon.c (mon.2) 106 : audit [INF] from='client.? 172.21.15.136:0/2010575863' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/d68919d3-5ccc-4ff9-867c-068d79574f2d"}]: dispatch 2024-09-13T00:29:53.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:53 smithi136 bash[75164]: audit 2024-09-13T00:29:53.065604+0000 mon.a (mon.0) 2788 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/d68919d3-5ccc-4ff9-867c-068d79574f2d"}]: dispatch 2024-09-13T00:29:53.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:53 smithi136 bash[75164]: audit 2024-09-13T00:29:53.069315+0000 mon.a (mon.0) 2789 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/d68919d3-5ccc-4ff9-867c-068d79574f2d"}]': finished 2024-09-13T00:29:53.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:53 smithi136 bash[76371]: cluster 2024-09-13T00:29:52.003659+0000 mgr.y (mgr.24661) 1582 : cluster [DBG] pgmap v2646: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 911 KiB/s rd, 133 KiB/s wr, 1.25k op/s 2024-09-13T00:29:53.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:53 smithi136 bash[76371]: audit 2024-09-13T00:29:53.035347+0000 mon.c (mon.2) 103 : audit [DBG] from='client.? 172.21.15.136:0/2010575863' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer_client_id"}]: dispatch 2024-09-13T00:29:53.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:53 smithi136 bash[76371]: audit 2024-09-13T00:29:53.040676+0000 mon.a (mon.0) 2785 : audit [INF] from='client.? ' entity='client.admin' 2024-09-13T00:29:53.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:53 smithi136 bash[76371]: audit 2024-09-13T00:29:53.041888+0000 mon.c (mon.2) 104 : audit [INF] from='client.? 172.21.15.136:0/2010575863' entity='client.admin' cmd=[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]: dispatch 2024-09-13T00:29:53.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:53 smithi136 bash[76371]: audit 2024-09-13T00:29:53.042242+0000 mon.a (mon.0) 2786 : audit [INF] from='client.? ' entity='client.admin' cmd=[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]: dispatch 2024-09-13T00:29:53.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:53 smithi136 bash[76371]: audit 2024-09-13T00:29:53.046836+0000 mon.a (mon.0) 2787 : audit [INF] from='client.? ' entity='client.admin' cmd='[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]': finished 2024-09-13T00:29:53.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:53 smithi136 bash[76371]: audit 2024-09-13T00:29:53.048136+0000 mon.c (mon.2) 105 : audit [DBG] from='client.? 172.21.15.136:0/2010575863' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-09-13T00:29:53.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:53 smithi136 bash[76371]: audit 2024-09-13T00:29:53.065298+0000 mon.c (mon.2) 106 : audit [INF] from='client.? 172.21.15.136:0/2010575863' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/d68919d3-5ccc-4ff9-867c-068d79574f2d"}]: dispatch 2024-09-13T00:29:53.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:53 smithi136 bash[76371]: audit 2024-09-13T00:29:53.065604+0000 mon.a (mon.0) 2788 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/d68919d3-5ccc-4ff9-867c-068d79574f2d"}]: dispatch 2024-09-13T00:29:53.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:53 smithi136 bash[76371]: audit 2024-09-13T00:29:53.069315+0000 mon.a (mon.0) 2789 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/d68919d3-5ccc-4ff9-867c-068d79574f2d"}]': finished 2024-09-13T00:29:53.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:53 smithi181 bash[67587]: cluster 2024-09-13T00:29:52.003659+0000 mgr.y (mgr.24661) 1582 : cluster [DBG] pgmap v2646: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 911 KiB/s rd, 133 KiB/s wr, 1.25k op/s 2024-09-13T00:29:53.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:53 smithi181 bash[67587]: audit 2024-09-13T00:29:53.035347+0000 mon.c (mon.2) 103 : audit [DBG] from='client.? 172.21.15.136:0/2010575863' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer_client_id"}]: dispatch 2024-09-13T00:29:53.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:53 smithi181 bash[67587]: audit 2024-09-13T00:29:53.040676+0000 mon.a (mon.0) 2785 : audit [INF] from='client.? ' entity='client.admin' 2024-09-13T00:29:53.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:53 smithi181 bash[67587]: audit 2024-09-13T00:29:53.041888+0000 mon.c (mon.2) 104 : audit [INF] from='client.? 172.21.15.136:0/2010575863' entity='client.admin' cmd=[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]: dispatch 2024-09-13T00:29:53.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:53 smithi181 bash[67587]: audit 2024-09-13T00:29:53.042242+0000 mon.a (mon.0) 2786 : audit [INF] from='client.? ' entity='client.admin' cmd=[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]: dispatch 2024-09-13T00:29:53.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:53 smithi181 bash[67587]: audit 2024-09-13T00:29:53.046836+0000 mon.a (mon.0) 2787 : audit [INF] from='client.? ' entity='client.admin' cmd='[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]': finished 2024-09-13T00:29:53.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:53 smithi181 bash[67587]: audit 2024-09-13T00:29:53.048136+0000 mon.c (mon.2) 105 : audit [DBG] from='client.? 172.21.15.136:0/2010575863' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-09-13T00:29:53.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:53 smithi181 bash[67587]: audit 2024-09-13T00:29:53.065298+0000 mon.c (mon.2) 106 : audit [INF] from='client.? 172.21.15.136:0/2010575863' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/d68919d3-5ccc-4ff9-867c-068d79574f2d"}]: dispatch 2024-09-13T00:29:53.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:53 smithi181 bash[67587]: audit 2024-09-13T00:29:53.065604+0000 mon.a (mon.0) 2788 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/d68919d3-5ccc-4ff9-867c-068d79574f2d"}]: dispatch 2024-09-13T00:29:53.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:53 smithi181 bash[67587]: audit 2024-09-13T00:29:53.069315+0000 mon.a (mon.0) 2789 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/d68919d3-5ccc-4ff9-867c-068d79574f2d"}]': finished 2024-09-13T00:29:54.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:54 smithi136 bash[75164]: audit 2024-09-13T00:29:53.821421+0000 mon.a (mon.0) 2790 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:29:54.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:54 smithi136 bash[75164]: cluster 2024-09-13T00:29:54.079860+0000 mon.a (mon.0) 2791 : cluster [DBG] osdmap e1504: 8 total, 8 up, 8 in 2024-09-13T00:29:54.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:54 smithi136 bash[76371]: audit 2024-09-13T00:29:53.821421+0000 mon.a (mon.0) 2790 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:29:54.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:54 smithi136 bash[76371]: cluster 2024-09-13T00:29:54.079860+0000 mon.a (mon.0) 2791 : cluster [DBG] osdmap e1504: 8 total, 8 up, 8 in 2024-09-13T00:29:54.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:54 smithi181 bash[67587]: audit 2024-09-13T00:29:53.821421+0000 mon.a (mon.0) 2790 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:29:54.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:54 smithi181 bash[67587]: cluster 2024-09-13T00:29:54.079860+0000 mon.a (mon.0) 2791 : cluster [DBG] osdmap e1504: 8 total, 8 up, 8 in 2024-09-13T00:29:55.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:55 smithi136 bash[75164]: cluster 2024-09-13T00:29:54.004569+0000 mgr.y (mgr.24661) 1583 : cluster [DBG] pgmap v2647: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 792 KiB/s rd, 119 KiB/s wr, 1.09k op/s 2024-09-13T00:29:55.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:55 smithi136 bash[75164]: cluster 2024-09-13T00:29:55.093169+0000 mon.a (mon.0) 2792 : cluster [DBG] osdmap e1505: 8 total, 8 up, 8 in 2024-09-13T00:29:55.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:55 smithi136 bash[76371]: cluster 2024-09-13T00:29:54.004569+0000 mgr.y (mgr.24661) 1583 : cluster [DBG] pgmap v2647: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 792 KiB/s rd, 119 KiB/s wr, 1.09k op/s 2024-09-13T00:29:55.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:55 smithi136 bash[76371]: cluster 2024-09-13T00:29:55.093169+0000 mon.a (mon.0) 2792 : cluster [DBG] osdmap e1505: 8 total, 8 up, 8 in 2024-09-13T00:29:55.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:55 smithi181 bash[67587]: cluster 2024-09-13T00:29:54.004569+0000 mgr.y (mgr.24661) 1583 : cluster [DBG] pgmap v2647: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 792 KiB/s rd, 119 KiB/s wr, 1.09k op/s 2024-09-13T00:29:55.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:55 smithi181 bash[67587]: cluster 2024-09-13T00:29:55.093169+0000 mon.a (mon.0) 2792 : cluster [DBG] osdmap e1505: 8 total, 8 up, 8 in 2024-09-13T00:29:57.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:57 smithi136 bash[75164]: cluster 2024-09-13T00:29:56.005478+0000 mgr.y (mgr.24661) 1584 : cluster [DBG] pgmap v2650: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 851 KiB/s rd, 129 KiB/s wr, 1.15k op/s 2024-09-13T00:29:57.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:57 smithi136 bash[75164]: cluster 2024-09-13T00:29:56.097840+0000 mon.a (mon.0) 2793 : cluster [DBG] osdmap e1506: 8 total, 8 up, 8 in 2024-09-13T00:29:57.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:57 smithi136 bash[76371]: cluster 2024-09-13T00:29:56.005478+0000 mgr.y (mgr.24661) 1584 : cluster [DBG] pgmap v2650: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 851 KiB/s rd, 129 KiB/s wr, 1.15k op/s 2024-09-13T00:29:57.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:57 smithi136 bash[76371]: cluster 2024-09-13T00:29:56.097840+0000 mon.a (mon.0) 2793 : cluster [DBG] osdmap e1506: 8 total, 8 up, 8 in 2024-09-13T00:29:57.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:57 smithi181 bash[67587]: cluster 2024-09-13T00:29:56.005478+0000 mgr.y (mgr.24661) 1584 : cluster [DBG] pgmap v2650: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 851 KiB/s rd, 129 KiB/s wr, 1.15k op/s 2024-09-13T00:29:57.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:57 smithi181 bash[67587]: cluster 2024-09-13T00:29:56.097840+0000 mon.a (mon.0) 2793 : cluster [DBG] osdmap e1506: 8 total, 8 up, 8 in 2024-09-13T00:29:58.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:58 smithi136 bash[75164]: cluster 2024-09-13T00:29:57.099763+0000 mon.a (mon.0) 2794 : cluster [DBG] osdmap e1507: 8 total, 8 up, 8 in 2024-09-13T00:29:58.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:58 smithi136 bash[76371]: cluster 2024-09-13T00:29:57.099763+0000 mon.a (mon.0) 2794 : cluster [DBG] osdmap e1507: 8 total, 8 up, 8 in 2024-09-13T00:29:58.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:58 smithi181 bash[67587]: cluster 2024-09-13T00:29:57.099763+0000 mon.a (mon.0) 2794 : cluster [DBG] osdmap e1507: 8 total, 8 up, 8 in 2024-09-13T00:29:59.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:59 smithi136 bash[75164]: cluster 2024-09-13T00:29:58.006423+0000 mgr.y (mgr.24661) 1585 : cluster [DBG] pgmap v2653: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 392 KiB/s rd, 47 KiB/s wr, 537 op/s 2024-09-13T00:29:59.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:29:59 smithi136 bash[75164]: cluster 2024-09-13T00:29:58.123853+0000 mon.a (mon.0) 2795 : cluster [DBG] osdmap e1508: 8 total, 8 up, 8 in 2024-09-13T00:29:59.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:59 smithi136 bash[76371]: cluster 2024-09-13T00:29:58.006423+0000 mgr.y (mgr.24661) 1585 : cluster [DBG] pgmap v2653: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 392 KiB/s rd, 47 KiB/s wr, 537 op/s 2024-09-13T00:29:59.414 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:29:59 smithi136 bash[76371]: cluster 2024-09-13T00:29:58.123853+0000 mon.a (mon.0) 2795 : cluster [DBG] osdmap e1508: 8 total, 8 up, 8 in 2024-09-13T00:29:59.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:59 smithi181 bash[67587]: cluster 2024-09-13T00:29:58.006423+0000 mgr.y (mgr.24661) 1585 : cluster [DBG] pgmap v2653: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 392 KiB/s rd, 47 KiB/s wr, 537 op/s 2024-09-13T00:29:59.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:29:59 smithi181 bash[67587]: cluster 2024-09-13T00:29:58.123853+0000 mon.a (mon.0) 2795 : cluster [DBG] osdmap e1508: 8 total, 8 up, 8 in 2024-09-13T00:29:59.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:29:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:29:59] "GET /metrics HTTP/1.1" 200 36540 "" "Prometheus/2.43.0" 2024-09-13T00:30:00.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:00 smithi136 bash[76371]: cluster 2024-09-13T00:29:59.131970+0000 mon.a (mon.0) 2796 : cluster [DBG] osdmap e1509: 8 total, 8 up, 8 in 2024-09-13T00:30:00.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:00 smithi136 bash[76371]: cluster 2024-09-13T00:30:00.000149+0000 mon.a (mon.0) 2797 : cluster [INF] overall HEALTH_OK 2024-09-13T00:30:00.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:00 smithi136 bash[75164]: cluster 2024-09-13T00:29:59.131970+0000 mon.a (mon.0) 2796 : cluster [DBG] osdmap e1509: 8 total, 8 up, 8 in 2024-09-13T00:30:00.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:00 smithi136 bash[75164]: cluster 2024-09-13T00:30:00.000149+0000 mon.a (mon.0) 2797 : cluster [INF] overall HEALTH_OK 2024-09-13T00:30:00.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:00 smithi181 bash[67587]: cluster 2024-09-13T00:29:59.131970+0000 mon.a (mon.0) 2796 : cluster [DBG] osdmap e1509: 8 total, 8 up, 8 in 2024-09-13T00:30:00.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:00 smithi181 bash[67587]: cluster 2024-09-13T00:30:00.000149+0000 mon.a (mon.0) 2797 : cluster [INF] overall HEALTH_OK 2024-09-13T00:30:01.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:01 smithi136 bash[76371]: cluster 2024-09-13T00:30:00.007275+0000 mgr.y (mgr.24661) 1586 : cluster [DBG] pgmap v2656: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 225 KiB/s rd, 26 KiB/s wr, 313 op/s 2024-09-13T00:30:01.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:01 smithi136 bash[76371]: cluster 2024-09-13T00:30:00.143395+0000 mon.a (mon.0) 2798 : cluster [DBG] osdmap e1510: 8 total, 8 up, 8 in 2024-09-13T00:30:01.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:01 smithi136 bash[75164]: cluster 2024-09-13T00:30:00.007275+0000 mgr.y (mgr.24661) 1586 : cluster [DBG] pgmap v2656: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 225 KiB/s rd, 26 KiB/s wr, 313 op/s 2024-09-13T00:30:01.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:01 smithi136 bash[75164]: cluster 2024-09-13T00:30:00.143395+0000 mon.a (mon.0) 2798 : cluster [DBG] osdmap e1510: 8 total, 8 up, 8 in 2024-09-13T00:30:01.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:01 smithi181 bash[67587]: cluster 2024-09-13T00:30:00.007275+0000 mgr.y (mgr.24661) 1586 : cluster [DBG] pgmap v2656: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 225 KiB/s rd, 26 KiB/s wr, 313 op/s 2024-09-13T00:30:01.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:01 smithi181 bash[67587]: cluster 2024-09-13T00:30:00.143395+0000 mon.a (mon.0) 2798 : cluster [DBG] osdmap e1510: 8 total, 8 up, 8 in 2024-09-13T00:30:02.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:02 smithi136 bash[76371]: cluster 2024-09-13T00:30:01.143536+0000 mon.a (mon.0) 2799 : cluster [DBG] osdmap e1511: 8 total, 8 up, 8 in 2024-09-13T00:30:02.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:02 smithi136 bash[75164]: cluster 2024-09-13T00:30:01.143536+0000 mon.a (mon.0) 2799 : cluster [DBG] osdmap e1511: 8 total, 8 up, 8 in 2024-09-13T00:30:02.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:02 smithi181 bash[67587]: cluster 2024-09-13T00:30:01.143536+0000 mon.a (mon.0) 2799 : cluster [DBG] osdmap e1511: 8 total, 8 up, 8 in 2024-09-13T00:30:02.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:30:02 smithi181 bash[95505]: ts=2024-09-13T00:30:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:30:03.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:03 smithi136 bash[76371]: cluster 2024-09-13T00:30:02.008201+0000 mgr.y (mgr.24661) 1587 : cluster [DBG] pgmap v2659: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 63 KiB/s rd, 6.2 KiB/s wr, 81 op/s 2024-09-13T00:30:03.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:03 smithi136 bash[76371]: cluster 2024-09-13T00:30:02.167496+0000 mon.a (mon.0) 2800 : cluster [DBG] osdmap e1512: 8 total, 8 up, 8 in 2024-09-13T00:30:03.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:03 smithi136 bash[75164]: cluster 2024-09-13T00:30:02.008201+0000 mgr.y (mgr.24661) 1587 : cluster [DBG] pgmap v2659: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 63 KiB/s rd, 6.2 KiB/s wr, 81 op/s 2024-09-13T00:30:03.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:03 smithi136 bash[75164]: cluster 2024-09-13T00:30:02.167496+0000 mon.a (mon.0) 2800 : cluster [DBG] osdmap e1512: 8 total, 8 up, 8 in 2024-09-13T00:30:03.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:03 smithi181 bash[67587]: cluster 2024-09-13T00:30:02.008201+0000 mgr.y (mgr.24661) 1587 : cluster [DBG] pgmap v2659: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 63 KiB/s rd, 6.2 KiB/s wr, 81 op/s 2024-09-13T00:30:03.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:03 smithi181 bash[67587]: cluster 2024-09-13T00:30:02.167496+0000 mon.a (mon.0) 2800 : cluster [DBG] osdmap e1512: 8 total, 8 up, 8 in 2024-09-13T00:30:04.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:04 smithi136 bash[76371]: cluster 2024-09-13T00:30:03.174760+0000 mon.a (mon.0) 2801 : cluster [DBG] osdmap e1513: 8 total, 8 up, 8 in 2024-09-13T00:30:04.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:04 smithi136 bash[76371]: cluster 2024-09-13T00:30:04.083042+0000 mon.a (mon.0) 2802 : cluster [DBG] osdmap e1514: 8 total, 8 up, 8 in 2024-09-13T00:30:04.511 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:04 smithi136 bash[75164]: cluster 2024-09-13T00:30:03.174760+0000 mon.a (mon.0) 2801 : cluster [DBG] osdmap e1513: 8 total, 8 up, 8 in 2024-09-13T00:30:04.511 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:04 smithi136 bash[75164]: cluster 2024-09-13T00:30:04.083042+0000 mon.a (mon.0) 2802 : cluster [DBG] osdmap e1514: 8 total, 8 up, 8 in 2024-09-13T00:30:04.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:04 smithi181 bash[67587]: cluster 2024-09-13T00:30:03.174760+0000 mon.a (mon.0) 2801 : cluster [DBG] osdmap e1513: 8 total, 8 up, 8 in 2024-09-13T00:30:04.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:04 smithi181 bash[67587]: cluster 2024-09-13T00:30:04.083042+0000 mon.a (mon.0) 2802 : cluster [DBG] osdmap e1514: 8 total, 8 up, 8 in 2024-09-13T00:30:05.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:05 smithi136 bash[75164]: cluster 2024-09-13T00:30:04.009041+0000 mgr.y (mgr.24661) 1588 : cluster [DBG] pgmap v2662: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 63 KiB/s rd, 6.0 KiB/s wr, 82 op/s 2024-09-13T00:30:05.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:05 smithi136 bash[75164]: cluster 2024-09-13T00:30:05.085313+0000 mon.a (mon.0) 2803 : cluster [DBG] osdmap e1515: 8 total, 8 up, 8 in 2024-09-13T00:30:05.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:05 smithi136 bash[76371]: cluster 2024-09-13T00:30:04.009041+0000 mgr.y (mgr.24661) 1588 : cluster [DBG] pgmap v2662: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 63 KiB/s rd, 6.0 KiB/s wr, 82 op/s 2024-09-13T00:30:05.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:05 smithi136 bash[76371]: cluster 2024-09-13T00:30:05.085313+0000 mon.a (mon.0) 2803 : cluster [DBG] osdmap e1515: 8 total, 8 up, 8 in 2024-09-13T00:30:05.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:05 smithi181 bash[67587]: cluster 2024-09-13T00:30:04.009041+0000 mgr.y (mgr.24661) 1588 : cluster [DBG] pgmap v2662: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 63 KiB/s rd, 6.0 KiB/s wr, 82 op/s 2024-09-13T00:30:05.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:05 smithi181 bash[67587]: cluster 2024-09-13T00:30:05.085313+0000 mon.a (mon.0) 2803 : cluster [DBG] osdmap e1515: 8 total, 8 up, 8 in 2024-09-13T00:30:07.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:07 smithi181 bash[67587]: cluster 2024-09-13T00:30:06.010052+0000 mgr.y (mgr.24661) 1589 : cluster [DBG] pgmap v2665: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 3.7 KiB/s rd, 767 B/s wr, 7 op/s 2024-09-13T00:30:07.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:07 smithi181 bash[67587]: cluster 2024-09-13T00:30:06.087592+0000 mon.a (mon.0) 2804 : cluster [DBG] osdmap e1516: 8 total, 8 up, 8 in 2024-09-13T00:30:07.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:07 smithi136 bash[75164]: cluster 2024-09-13T00:30:06.010052+0000 mgr.y (mgr.24661) 1589 : cluster [DBG] pgmap v2665: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 3.7 KiB/s rd, 767 B/s wr, 7 op/s 2024-09-13T00:30:07.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:07 smithi136 bash[75164]: cluster 2024-09-13T00:30:06.087592+0000 mon.a (mon.0) 2804 : cluster [DBG] osdmap e1516: 8 total, 8 up, 8 in 2024-09-13T00:30:07.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:07 smithi136 bash[76371]: cluster 2024-09-13T00:30:06.010052+0000 mgr.y (mgr.24661) 1589 : cluster [DBG] pgmap v2665: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 3.7 KiB/s rd, 767 B/s wr, 7 op/s 2024-09-13T00:30:07.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:07 smithi136 bash[76371]: cluster 2024-09-13T00:30:06.087592+0000 mon.a (mon.0) 2804 : cluster [DBG] osdmap e1516: 8 total, 8 up, 8 in 2024-09-13T00:30:08.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:08 smithi136 bash[75164]: cluster 2024-09-13T00:30:07.101590+0000 mon.a (mon.0) 2805 : cluster [DBG] osdmap e1517: 8 total, 8 up, 8 in 2024-09-13T00:30:08.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:08 smithi136 bash[76371]: cluster 2024-09-13T00:30:07.101590+0000 mon.a (mon.0) 2805 : cluster [DBG] osdmap e1517: 8 total, 8 up, 8 in 2024-09-13T00:30:08.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:08 smithi181 bash[67587]: cluster 2024-09-13T00:30:07.101590+0000 mon.a (mon.0) 2805 : cluster [DBG] osdmap e1517: 8 total, 8 up, 8 in 2024-09-13T00:30:09.318 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMirroring.Snapshot (16248 ms) 2024-09-13T00:30:09.319 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMirroring.SnapshotRemoveOnDisable 2024-09-13T00:30:09.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:09 smithi136 bash[76371]: cluster 2024-09-13T00:30:08.011038+0000 mgr.y (mgr.24661) 1590 : cluster [DBG] pgmap v2668: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 66 KiB/s rd, 5.7 KiB/s wr, 89 op/s 2024-09-13T00:30:09.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:09 smithi136 bash[76371]: cluster 2024-09-13T00:30:08.114614+0000 mon.a (mon.0) 2806 : cluster [DBG] osdmap e1518: 8 total, 8 up, 8 in 2024-09-13T00:30:09.414 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:09 smithi136 bash[76371]: audit 2024-09-13T00:30:08.821696+0000 mon.a (mon.0) 2807 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:30:09.414 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:09 smithi136 bash[76371]: cluster 2024-09-13T00:30:09.085425+0000 mon.a (mon.0) 2808 : cluster [DBG] osdmap e1519: 8 total, 8 up, 8 in 2024-09-13T00:30:09.414 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:09 smithi136 bash[75164]: cluster 2024-09-13T00:30:08.011038+0000 mgr.y (mgr.24661) 1590 : cluster [DBG] pgmap v2668: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 66 KiB/s rd, 5.7 KiB/s wr, 89 op/s 2024-09-13T00:30:09.414 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:09 smithi136 bash[75164]: cluster 2024-09-13T00:30:08.114614+0000 mon.a (mon.0) 2806 : cluster [DBG] osdmap e1518: 8 total, 8 up, 8 in 2024-09-13T00:30:09.414 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:09 smithi136 bash[75164]: audit 2024-09-13T00:30:08.821696+0000 mon.a (mon.0) 2807 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:30:09.414 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:09 smithi136 bash[75164]: cluster 2024-09-13T00:30:09.085425+0000 mon.a (mon.0) 2808 : cluster [DBG] osdmap e1519: 8 total, 8 up, 8 in 2024-09-13T00:30:09.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:09 smithi181 bash[67587]: cluster 2024-09-13T00:30:08.011038+0000 mgr.y (mgr.24661) 1590 : cluster [DBG] pgmap v2668: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 66 KiB/s rd, 5.7 KiB/s wr, 89 op/s 2024-09-13T00:30:09.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:09 smithi181 bash[67587]: cluster 2024-09-13T00:30:08.114614+0000 mon.a (mon.0) 2806 : cluster [DBG] osdmap e1518: 8 total, 8 up, 8 in 2024-09-13T00:30:09.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:09 smithi181 bash[67587]: audit 2024-09-13T00:30:08.821696+0000 mon.a (mon.0) 2807 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:30:09.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:09 smithi181 bash[67587]: cluster 2024-09-13T00:30:09.085425+0000 mon.a (mon.0) 2808 : cluster [DBG] osdmap e1519: 8 total, 8 up, 8 in 2024-09-13T00:30:09.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:30:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:30:09] "GET /metrics HTTP/1.1" 200 36542 "" "Prometheus/2.43.0" 2024-09-13T00:30:10.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:10 smithi136 bash[76371]: audit 2024-09-13T00:30:09.301273+0000 mon.c (mon.2) 107 : audit [INF] from='client.? 172.21.15.136:0/2010575863' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/a3ab8f7b-4b62-46fe-a826-5ad1dddb9b32"}]: dispatch 2024-09-13T00:30:10.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:10 smithi136 bash[76371]: audit 2024-09-13T00:30:09.301955+0000 mon.a (mon.0) 2809 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/a3ab8f7b-4b62-46fe-a826-5ad1dddb9b32"}]: dispatch 2024-09-13T00:30:10.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:10 smithi136 bash[76371]: audit 2024-09-13T00:30:09.308841+0000 mon.a (mon.0) 2810 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/a3ab8f7b-4b62-46fe-a826-5ad1dddb9b32"}]': finished 2024-09-13T00:30:10.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:10 smithi136 bash[75164]: audit 2024-09-13T00:30:09.301273+0000 mon.c (mon.2) 107 : audit [INF] from='client.? 172.21.15.136:0/2010575863' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/a3ab8f7b-4b62-46fe-a826-5ad1dddb9b32"}]: dispatch 2024-09-13T00:30:10.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:10 smithi136 bash[75164]: audit 2024-09-13T00:30:09.301955+0000 mon.a (mon.0) 2809 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/a3ab8f7b-4b62-46fe-a826-5ad1dddb9b32"}]: dispatch 2024-09-13T00:30:10.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:10 smithi136 bash[75164]: audit 2024-09-13T00:30:09.308841+0000 mon.a (mon.0) 2810 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/a3ab8f7b-4b62-46fe-a826-5ad1dddb9b32"}]': finished 2024-09-13T00:30:10.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:10 smithi181 bash[67587]: audit 2024-09-13T00:30:09.301273+0000 mon.c (mon.2) 107 : audit [INF] from='client.? 172.21.15.136:0/2010575863' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/a3ab8f7b-4b62-46fe-a826-5ad1dddb9b32"}]: dispatch 2024-09-13T00:30:10.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:10 smithi181 bash[67587]: audit 2024-09-13T00:30:09.301955+0000 mon.a (mon.0) 2809 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/a3ab8f7b-4b62-46fe-a826-5ad1dddb9b32"}]: dispatch 2024-09-13T00:30:10.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:10 smithi181 bash[67587]: audit 2024-09-13T00:30:09.308841+0000 mon.a (mon.0) 2810 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/a3ab8f7b-4b62-46fe-a826-5ad1dddb9b32"}]': finished 2024-09-13T00:30:11.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:11 smithi181 bash[67587]: cluster 2024-09-13T00:30:10.011917+0000 mgr.y (mgr.24661) 1591 : cluster [DBG] pgmap v2671: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 63 KiB/s rd, 5.2 KiB/s wr, 84 op/s 2024-09-13T00:30:11.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:11 smithi181 bash[67587]: cluster 2024-09-13T00:30:10.322347+0000 mon.a (mon.0) 2811 : cluster [DBG] osdmap e1520: 8 total, 8 up, 8 in 2024-09-13T00:30:11.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:11 smithi136 bash[75164]: cluster 2024-09-13T00:30:10.011917+0000 mgr.y (mgr.24661) 1591 : cluster [DBG] pgmap v2671: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 63 KiB/s rd, 5.2 KiB/s wr, 84 op/s 2024-09-13T00:30:11.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:11 smithi136 bash[75164]: cluster 2024-09-13T00:30:10.322347+0000 mon.a (mon.0) 2811 : cluster [DBG] osdmap e1520: 8 total, 8 up, 8 in 2024-09-13T00:30:11.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:11 smithi136 bash[76371]: cluster 2024-09-13T00:30:10.011917+0000 mgr.y (mgr.24661) 1591 : cluster [DBG] pgmap v2671: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 63 KiB/s rd, 5.2 KiB/s wr, 84 op/s 2024-09-13T00:30:11.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:11 smithi136 bash[76371]: cluster 2024-09-13T00:30:10.322347+0000 mon.a (mon.0) 2811 : cluster [DBG] osdmap e1520: 8 total, 8 up, 8 in 2024-09-13T00:30:12.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:12 smithi181 bash[67587]: cluster 2024-09-13T00:30:11.325609+0000 mon.a (mon.0) 2812 : cluster [DBG] osdmap e1521: 8 total, 8 up, 8 in 2024-09-13T00:30:12.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:30:12 smithi181 bash[95505]: ts=2024-09-13T00:30:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:30:12.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:12 smithi136 bash[75164]: cluster 2024-09-13T00:30:11.325609+0000 mon.a (mon.0) 2812 : cluster [DBG] osdmap e1521: 8 total, 8 up, 8 in 2024-09-13T00:30:12.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:12 smithi136 bash[76371]: cluster 2024-09-13T00:30:11.325609+0000 mon.a (mon.0) 2812 : cluster [DBG] osdmap e1521: 8 total, 8 up, 8 in 2024-09-13T00:30:13.579 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMirroring.SnapshotRemoveOnDisable (4260 ms) 2024-09-13T00:30:13.579 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMirroring.SnapshotUnlinkPeer 2024-09-13T00:30:13.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:13 smithi136 bash[75164]: cluster 2024-09-13T00:30:12.012941+0000 mgr.y (mgr.24661) 1592 : cluster [DBG] pgmap v2674: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 54 KiB/s rd, 5.5 KiB/s wr, 77 op/s 2024-09-13T00:30:13.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:13 smithi136 bash[75164]: cluster 2024-09-13T00:30:12.348508+0000 mon.a (mon.0) 2813 : cluster [DBG] osdmap e1522: 8 total, 8 up, 8 in 2024-09-13T00:30:13.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:13 smithi136 bash[76371]: cluster 2024-09-13T00:30:12.012941+0000 mgr.y (mgr.24661) 1592 : cluster [DBG] pgmap v2674: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 54 KiB/s rd, 5.5 KiB/s wr, 77 op/s 2024-09-13T00:30:13.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:13 smithi136 bash[76371]: cluster 2024-09-13T00:30:12.348508+0000 mon.a (mon.0) 2813 : cluster [DBG] osdmap e1522: 8 total, 8 up, 8 in 2024-09-13T00:30:13.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:13 smithi181 bash[67587]: cluster 2024-09-13T00:30:12.012941+0000 mgr.y (mgr.24661) 1592 : cluster [DBG] pgmap v2674: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 54 KiB/s rd, 5.5 KiB/s wr, 77 op/s 2024-09-13T00:30:13.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:13 smithi181 bash[67587]: cluster 2024-09-13T00:30:12.348508+0000 mon.a (mon.0) 2813 : cluster [DBG] osdmap e1522: 8 total, 8 up, 8 in 2024-09-13T00:30:14.348 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 13 00:30:13 smithi181 bash[98820]: logger=cleanup t=2024-09-13T00:30:13.888040645Z level=info msg="Completed cleanup jobs" duration=193.954678ms 2024-09-13T00:30:14.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:14 smithi136 bash[75164]: cluster 2024-09-13T00:30:13.348747+0000 mon.a (mon.0) 2814 : cluster [DBG] osdmap e1523: 8 total, 8 up, 8 in 2024-09-13T00:30:14.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:14 smithi136 bash[75164]: audit 2024-09-13T00:30:13.561954+0000 mon.c (mon.2) 108 : audit [INF] from='client.? 172.21.15.136:0/2010575863' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/49b58ab3-7865-4266-beda-f6bbdc20af27"}]: dispatch 2024-09-13T00:30:14.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:14 smithi136 bash[75164]: audit 2024-09-13T00:30:13.562685+0000 mon.a (mon.0) 2815 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/49b58ab3-7865-4266-beda-f6bbdc20af27"}]: dispatch 2024-09-13T00:30:14.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:14 smithi136 bash[75164]: audit 2024-09-13T00:30:13.569125+0000 mon.a (mon.0) 2816 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/49b58ab3-7865-4266-beda-f6bbdc20af27"}]': finished 2024-09-13T00:30:14.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:14 smithi136 bash[75164]: cluster 2024-09-13T00:30:14.088506+0000 mon.a (mon.0) 2817 : cluster [DBG] osdmap e1524: 8 total, 8 up, 8 in 2024-09-13T00:30:14.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:14 smithi136 bash[76371]: cluster 2024-09-13T00:30:13.348747+0000 mon.a (mon.0) 2814 : cluster [DBG] osdmap e1523: 8 total, 8 up, 8 in 2024-09-13T00:30:14.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:14 smithi136 bash[76371]: audit 2024-09-13T00:30:13.561954+0000 mon.c (mon.2) 108 : audit [INF] from='client.? 172.21.15.136:0/2010575863' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/49b58ab3-7865-4266-beda-f6bbdc20af27"}]: dispatch 2024-09-13T00:30:14.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:14 smithi136 bash[76371]: audit 2024-09-13T00:30:13.562685+0000 mon.a (mon.0) 2815 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/49b58ab3-7865-4266-beda-f6bbdc20af27"}]: dispatch 2024-09-13T00:30:14.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:14 smithi136 bash[76371]: audit 2024-09-13T00:30:13.569125+0000 mon.a (mon.0) 2816 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/49b58ab3-7865-4266-beda-f6bbdc20af27"}]': finished 2024-09-13T00:30:14.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:14 smithi136 bash[76371]: cluster 2024-09-13T00:30:14.088506+0000 mon.a (mon.0) 2817 : cluster [DBG] osdmap e1524: 8 total, 8 up, 8 in 2024-09-13T00:30:14.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:14 smithi181 bash[67587]: cluster 2024-09-13T00:30:13.348747+0000 mon.a (mon.0) 2814 : cluster [DBG] osdmap e1523: 8 total, 8 up, 8 in 2024-09-13T00:30:14.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:14 smithi181 bash[67587]: audit 2024-09-13T00:30:13.561954+0000 mon.c (mon.2) 108 : audit [INF] from='client.? 172.21.15.136:0/2010575863' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/49b58ab3-7865-4266-beda-f6bbdc20af27"}]: dispatch 2024-09-13T00:30:14.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:14 smithi181 bash[67587]: audit 2024-09-13T00:30:13.562685+0000 mon.a (mon.0) 2815 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/49b58ab3-7865-4266-beda-f6bbdc20af27"}]: dispatch 2024-09-13T00:30:14.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:14 smithi181 bash[67587]: audit 2024-09-13T00:30:13.569125+0000 mon.a (mon.0) 2816 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/49b58ab3-7865-4266-beda-f6bbdc20af27"}]': finished 2024-09-13T00:30:14.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:14 smithi181 bash[67587]: cluster 2024-09-13T00:30:14.088506+0000 mon.a (mon.0) 2817 : cluster [DBG] osdmap e1524: 8 total, 8 up, 8 in 2024-09-13T00:30:15.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:15 smithi136 bash[75164]: cluster 2024-09-13T00:30:14.013872+0000 mgr.y (mgr.24661) 1593 : cluster [DBG] pgmap v2677: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 85 KiB/s rd, 10 KiB/s wr, 120 op/s 2024-09-13T00:30:15.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:15 smithi136 bash[75164]: cluster 2024-09-13T00:30:15.090890+0000 mon.a (mon.0) 2818 : cluster [DBG] osdmap e1525: 8 total, 8 up, 8 in 2024-09-13T00:30:15.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:15 smithi136 bash[76371]: cluster 2024-09-13T00:30:14.013872+0000 mgr.y (mgr.24661) 1593 : cluster [DBG] pgmap v2677: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 85 KiB/s rd, 10 KiB/s wr, 120 op/s 2024-09-13T00:30:15.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:15 smithi136 bash[76371]: cluster 2024-09-13T00:30:15.090890+0000 mon.a (mon.0) 2818 : cluster [DBG] osdmap e1525: 8 total, 8 up, 8 in 2024-09-13T00:30:15.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:15 smithi181 bash[67587]: cluster 2024-09-13T00:30:14.013872+0000 mgr.y (mgr.24661) 1593 : cluster [DBG] pgmap v2677: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 85 KiB/s rd, 10 KiB/s wr, 120 op/s 2024-09-13T00:30:15.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:15 smithi181 bash[67587]: cluster 2024-09-13T00:30:15.090890+0000 mon.a (mon.0) 2818 : cluster [DBG] osdmap e1525: 8 total, 8 up, 8 in 2024-09-13T00:30:17.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:17 smithi181 bash[67587]: cluster 2024-09-13T00:30:16.014879+0000 mgr.y (mgr.24661) 1594 : cluster [DBG] pgmap v2680: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 39 KiB/s rd, 5.7 KiB/s wr, 57 op/s 2024-09-13T00:30:17.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:17 smithi181 bash[67587]: cluster 2024-09-13T00:30:16.092875+0000 mon.a (mon.0) 2819 : cluster [DBG] osdmap e1526: 8 total, 8 up, 8 in 2024-09-13T00:30:17.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:17 smithi136 bash[76371]: cluster 2024-09-13T00:30:16.014879+0000 mgr.y (mgr.24661) 1594 : cluster [DBG] pgmap v2680: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 39 KiB/s rd, 5.7 KiB/s wr, 57 op/s 2024-09-13T00:30:17.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:17 smithi136 bash[76371]: cluster 2024-09-13T00:30:16.092875+0000 mon.a (mon.0) 2819 : cluster [DBG] osdmap e1526: 8 total, 8 up, 8 in 2024-09-13T00:30:17.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:17 smithi136 bash[75164]: cluster 2024-09-13T00:30:16.014879+0000 mgr.y (mgr.24661) 1594 : cluster [DBG] pgmap v2680: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 39 KiB/s rd, 5.7 KiB/s wr, 57 op/s 2024-09-13T00:30:17.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:17 smithi136 bash[75164]: cluster 2024-09-13T00:30:16.092875+0000 mon.a (mon.0) 2819 : cluster [DBG] osdmap e1526: 8 total, 8 up, 8 in 2024-09-13T00:30:18.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:18 smithi181 bash[67587]: cluster 2024-09-13T00:30:17.103016+0000 mon.a (mon.0) 2820 : cluster [DBG] osdmap e1527: 8 total, 8 up, 8 in 2024-09-13T00:30:18.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:18 smithi136 bash[76371]: cluster 2024-09-13T00:30:17.103016+0000 mon.a (mon.0) 2820 : cluster [DBG] osdmap e1527: 8 total, 8 up, 8 in 2024-09-13T00:30:18.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:18 smithi136 bash[75164]: cluster 2024-09-13T00:30:17.103016+0000 mon.a (mon.0) 2820 : cluster [DBG] osdmap e1527: 8 total, 8 up, 8 in 2024-09-13T00:30:19.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:19 smithi136 bash[75164]: cluster 2024-09-13T00:30:18.015713+0000 mgr.y (mgr.24661) 1595 : cluster [DBG] pgmap v2683: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 75 KiB/s rd, 7.2 KiB/s wr, 106 op/s 2024-09-13T00:30:19.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:19 smithi136 bash[75164]: cluster 2024-09-13T00:30:18.113558+0000 mon.a (mon.0) 2821 : cluster [DBG] osdmap e1528: 8 total, 8 up, 8 in 2024-09-13T00:30:19.414 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:19 smithi136 bash[75164]: audit 2024-09-13T00:30:18.164962+0000 mon.c (mon.2) 109 : audit [INF] from='client.? 172.21.15.136:0/2010575863' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/f71e09b5-b118-4040-8ce7-a8fa074735fe"}]: dispatch 2024-09-13T00:30:19.414 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:19 smithi136 bash[75164]: audit 2024-09-13T00:30:18.165646+0000 mon.a (mon.0) 2822 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/f71e09b5-b118-4040-8ce7-a8fa074735fe"}]: dispatch 2024-09-13T00:30:19.414 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:19 smithi136 bash[75164]: audit 2024-09-13T00:30:18.172134+0000 mon.a (mon.0) 2823 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/f71e09b5-b118-4040-8ce7-a8fa074735fe"}]': finished 2024-09-13T00:30:19.414 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:19 smithi136 bash[75164]: cluster 2024-09-13T00:30:19.089598+0000 mon.a (mon.0) 2824 : cluster [DBG] osdmap e1529: 8 total, 8 up, 8 in 2024-09-13T00:30:19.414 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:19 smithi136 bash[76371]: cluster 2024-09-13T00:30:18.015713+0000 mgr.y (mgr.24661) 1595 : cluster [DBG] pgmap v2683: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 75 KiB/s rd, 7.2 KiB/s wr, 106 op/s 2024-09-13T00:30:19.414 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:19 smithi136 bash[76371]: cluster 2024-09-13T00:30:18.113558+0000 mon.a (mon.0) 2821 : cluster [DBG] osdmap e1528: 8 total, 8 up, 8 in 2024-09-13T00:30:19.414 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:19 smithi136 bash[76371]: audit 2024-09-13T00:30:18.164962+0000 mon.c (mon.2) 109 : audit [INF] from='client.? 172.21.15.136:0/2010575863' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/f71e09b5-b118-4040-8ce7-a8fa074735fe"}]: dispatch 2024-09-13T00:30:19.414 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:19 smithi136 bash[76371]: audit 2024-09-13T00:30:18.165646+0000 mon.a (mon.0) 2822 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/f71e09b5-b118-4040-8ce7-a8fa074735fe"}]: dispatch 2024-09-13T00:30:19.414 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:19 smithi136 bash[76371]: audit 2024-09-13T00:30:18.172134+0000 mon.a (mon.0) 2823 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/f71e09b5-b118-4040-8ce7-a8fa074735fe"}]': finished 2024-09-13T00:30:19.415 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:19 smithi136 bash[76371]: cluster 2024-09-13T00:30:19.089598+0000 mon.a (mon.0) 2824 : cluster [DBG] osdmap e1529: 8 total, 8 up, 8 in 2024-09-13T00:30:19.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:19 smithi181 bash[67587]: cluster 2024-09-13T00:30:18.015713+0000 mgr.y (mgr.24661) 1595 : cluster [DBG] pgmap v2683: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 75 KiB/s rd, 7.2 KiB/s wr, 106 op/s 2024-09-13T00:30:19.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:19 smithi181 bash[67587]: cluster 2024-09-13T00:30:18.113558+0000 mon.a (mon.0) 2821 : cluster [DBG] osdmap e1528: 8 total, 8 up, 8 in 2024-09-13T00:30:19.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:19 smithi181 bash[67587]: audit 2024-09-13T00:30:18.164962+0000 mon.c (mon.2) 109 : audit [INF] from='client.? 172.21.15.136:0/2010575863' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/f71e09b5-b118-4040-8ce7-a8fa074735fe"}]: dispatch 2024-09-13T00:30:19.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:19 smithi181 bash[67587]: audit 2024-09-13T00:30:18.165646+0000 mon.a (mon.0) 2822 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/f71e09b5-b118-4040-8ce7-a8fa074735fe"}]: dispatch 2024-09-13T00:30:19.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:19 smithi181 bash[67587]: audit 2024-09-13T00:30:18.172134+0000 mon.a (mon.0) 2823 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/f71e09b5-b118-4040-8ce7-a8fa074735fe"}]': finished 2024-09-13T00:30:19.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:19 smithi181 bash[67587]: cluster 2024-09-13T00:30:19.089598+0000 mon.a (mon.0) 2824 : cluster [DBG] osdmap e1529: 8 total, 8 up, 8 in 2024-09-13T00:30:19.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:30:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:30:19] "GET /metrics HTTP/1.1" 200 36542 "" "Prometheus/2.43.0" 2024-09-13T00:30:21.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:21 smithi136 bash[76371]: cluster 2024-09-13T00:30:20.016492+0000 mgr.y (mgr.24661) 1596 : cluster [DBG] pgmap v2686: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 95 KiB/s rd, 8.0 KiB/s wr, 126 op/s 2024-09-13T00:30:21.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:21 smithi136 bash[76371]: cluster 2024-09-13T00:30:20.128375+0000 mon.a (mon.0) 2825 : cluster [DBG] osdmap e1530: 8 total, 8 up, 8 in 2024-09-13T00:30:21.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:21 smithi136 bash[75164]: cluster 2024-09-13T00:30:20.016492+0000 mgr.y (mgr.24661) 1596 : cluster [DBG] pgmap v2686: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 95 KiB/s rd, 8.0 KiB/s wr, 126 op/s 2024-09-13T00:30:21.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:21 smithi136 bash[75164]: cluster 2024-09-13T00:30:20.128375+0000 mon.a (mon.0) 2825 : cluster [DBG] osdmap e1530: 8 total, 8 up, 8 in 2024-09-13T00:30:21.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:21 smithi181 bash[67587]: cluster 2024-09-13T00:30:20.016492+0000 mgr.y (mgr.24661) 1596 : cluster [DBG] pgmap v2686: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 95 KiB/s rd, 8.0 KiB/s wr, 126 op/s 2024-09-13T00:30:21.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:21 smithi181 bash[67587]: cluster 2024-09-13T00:30:20.128375+0000 mon.a (mon.0) 2825 : cluster [DBG] osdmap e1530: 8 total, 8 up, 8 in 2024-09-13T00:30:22.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:22 smithi136 bash[75164]: cluster 2024-09-13T00:30:21.137390+0000 mon.a (mon.0) 2826 : cluster [DBG] osdmap e1531: 8 total, 8 up, 8 in 2024-09-13T00:30:22.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:22 smithi136 bash[76371]: cluster 2024-09-13T00:30:21.137390+0000 mon.a (mon.0) 2826 : cluster [DBG] osdmap e1531: 8 total, 8 up, 8 in 2024-09-13T00:30:22.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:22 smithi181 bash[67587]: cluster 2024-09-13T00:30:21.137390+0000 mon.a (mon.0) 2826 : cluster [DBG] osdmap e1531: 8 total, 8 up, 8 in 2024-09-13T00:30:22.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:30:22 smithi181 bash[95505]: ts=2024-09-13T00:30:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:30:23.342 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMirroring.SnapshotUnlinkPeer (9763 ms) 2024-09-13T00:30:23.342 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMirroring.SnapshotImageState 2024-09-13T00:30:23.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:23 smithi136 bash[75164]: cluster 2024-09-13T00:30:22.017533+0000 mgr.y (mgr.24661) 1597 : cluster [DBG] pgmap v2689: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 174 KiB/s rd, 10 KiB/s wr, 228 op/s 2024-09-13T00:30:23.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:23 smithi136 bash[75164]: cluster 2024-09-13T00:30:22.153406+0000 mon.a (mon.0) 2827 : cluster [DBG] osdmap e1532: 8 total, 8 up, 8 in 2024-09-13T00:30:23.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:23 smithi136 bash[75164]: audit 2024-09-13T00:30:22.531774+0000 mon.a (mon.0) 2828 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:30:23.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:23 smithi136 bash[75164]: audit 2024-09-13T00:30:23.055369+0000 mon.a (mon.0) 2829 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:30:23.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:23 smithi136 bash[75164]: audit 2024-09-13T00:30:23.056753+0000 mon.a (mon.0) 2830 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:30:23.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:23 smithi136 bash[75164]: audit 2024-09-13T00:30:23.066299+0000 mon.a (mon.0) 2831 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:30:23.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:23 smithi136 bash[76371]: cluster 2024-09-13T00:30:22.017533+0000 mgr.y (mgr.24661) 1597 : cluster [DBG] pgmap v2689: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 174 KiB/s rd, 10 KiB/s wr, 228 op/s 2024-09-13T00:30:23.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:23 smithi136 bash[76371]: cluster 2024-09-13T00:30:22.153406+0000 mon.a (mon.0) 2827 : cluster [DBG] osdmap e1532: 8 total, 8 up, 8 in 2024-09-13T00:30:23.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:23 smithi136 bash[76371]: audit 2024-09-13T00:30:22.531774+0000 mon.a (mon.0) 2828 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:30:23.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:23 smithi136 bash[76371]: audit 2024-09-13T00:30:23.055369+0000 mon.a (mon.0) 2829 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:30:23.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:23 smithi136 bash[76371]: audit 2024-09-13T00:30:23.056753+0000 mon.a (mon.0) 2830 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:30:23.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:23 smithi136 bash[76371]: audit 2024-09-13T00:30:23.066299+0000 mon.a (mon.0) 2831 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:30:23.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:23 smithi181 bash[67587]: cluster 2024-09-13T00:30:22.017533+0000 mgr.y (mgr.24661) 1597 : cluster [DBG] pgmap v2689: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 174 KiB/s rd, 10 KiB/s wr, 228 op/s 2024-09-13T00:30:23.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:23 smithi181 bash[67587]: cluster 2024-09-13T00:30:22.153406+0000 mon.a (mon.0) 2827 : cluster [DBG] osdmap e1532: 8 total, 8 up, 8 in 2024-09-13T00:30:23.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:23 smithi181 bash[67587]: audit 2024-09-13T00:30:22.531774+0000 mon.a (mon.0) 2828 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:30:23.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:23 smithi181 bash[67587]: audit 2024-09-13T00:30:23.055369+0000 mon.a (mon.0) 2829 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:30:23.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:23 smithi181 bash[67587]: audit 2024-09-13T00:30:23.056753+0000 mon.a (mon.0) 2830 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:30:23.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:23 smithi181 bash[67587]: audit 2024-09-13T00:30:23.066299+0000 mon.a (mon.0) 2831 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:30:24.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:24 smithi136 bash[75164]: cluster 2024-09-13T00:30:23.161725+0000 mon.a (mon.0) 2832 : cluster [DBG] osdmap e1533: 8 total, 8 up, 8 in 2024-09-13T00:30:24.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:24 smithi136 bash[75164]: audit 2024-09-13T00:30:23.307171+0000 mon.c (mon.2) 110 : audit [INF] from='client.? 172.21.15.136:0/2010575863' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/8e1d2f30-2661-4e97-b229-29f2cb83fcd2"}]: dispatch 2024-09-13T00:30:24.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:24 smithi136 bash[75164]: audit 2024-09-13T00:30:23.307869+0000 mon.a (mon.0) 2833 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/8e1d2f30-2661-4e97-b229-29f2cb83fcd2"}]: dispatch 2024-09-13T00:30:24.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:24 smithi136 bash[75164]: audit 2024-09-13T00:30:23.316183+0000 mon.a (mon.0) 2834 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/8e1d2f30-2661-4e97-b229-29f2cb83fcd2"}]': finished 2024-09-13T00:30:24.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:24 smithi136 bash[75164]: audit 2024-09-13T00:30:23.322645+0000 mon.c (mon.2) 111 : audit [INF] from='client.? 172.21.15.136:0/2010575863' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/6572737c-910a-45e7-bc7d-59d7d4492e6a"}]: dispatch 2024-09-13T00:30:24.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:24 smithi136 bash[75164]: audit 2024-09-13T00:30:23.323242+0000 mon.a (mon.0) 2835 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/6572737c-910a-45e7-bc7d-59d7d4492e6a"}]: dispatch 2024-09-13T00:30:24.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:24 smithi136 bash[75164]: audit 2024-09-13T00:30:23.331454+0000 mon.a (mon.0) 2836 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/6572737c-910a-45e7-bc7d-59d7d4492e6a"}]': finished 2024-09-13T00:30:24.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:24 smithi136 bash[75164]: audit 2024-09-13T00:30:23.822148+0000 mon.a (mon.0) 2837 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:30:24.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:24 smithi136 bash[76371]: cluster 2024-09-13T00:30:23.161725+0000 mon.a (mon.0) 2832 : cluster [DBG] osdmap e1533: 8 total, 8 up, 8 in 2024-09-13T00:30:24.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:24 smithi136 bash[76371]: audit 2024-09-13T00:30:23.307171+0000 mon.c (mon.2) 110 : audit [INF] from='client.? 172.21.15.136:0/2010575863' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/8e1d2f30-2661-4e97-b229-29f2cb83fcd2"}]: dispatch 2024-09-13T00:30:24.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:24 smithi136 bash[76371]: audit 2024-09-13T00:30:23.307869+0000 mon.a (mon.0) 2833 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/8e1d2f30-2661-4e97-b229-29f2cb83fcd2"}]: dispatch 2024-09-13T00:30:24.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:24 smithi136 bash[76371]: audit 2024-09-13T00:30:23.316183+0000 mon.a (mon.0) 2834 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/8e1d2f30-2661-4e97-b229-29f2cb83fcd2"}]': finished 2024-09-13T00:30:24.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:24 smithi136 bash[76371]: audit 2024-09-13T00:30:23.322645+0000 mon.c (mon.2) 111 : audit [INF] from='client.? 172.21.15.136:0/2010575863' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/6572737c-910a-45e7-bc7d-59d7d4492e6a"}]: dispatch 2024-09-13T00:30:24.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:24 smithi136 bash[76371]: audit 2024-09-13T00:30:23.323242+0000 mon.a (mon.0) 2835 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/6572737c-910a-45e7-bc7d-59d7d4492e6a"}]: dispatch 2024-09-13T00:30:24.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:24 smithi136 bash[76371]: audit 2024-09-13T00:30:23.331454+0000 mon.a (mon.0) 2836 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/6572737c-910a-45e7-bc7d-59d7d4492e6a"}]': finished 2024-09-13T00:30:24.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:24 smithi136 bash[76371]: audit 2024-09-13T00:30:23.822148+0000 mon.a (mon.0) 2837 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:30:24.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:24 smithi181 bash[67587]: cluster 2024-09-13T00:30:23.161725+0000 mon.a (mon.0) 2832 : cluster [DBG] osdmap e1533: 8 total, 8 up, 8 in 2024-09-13T00:30:24.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:24 smithi181 bash[67587]: audit 2024-09-13T00:30:23.307171+0000 mon.c (mon.2) 110 : audit [INF] from='client.? 172.21.15.136:0/2010575863' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/8e1d2f30-2661-4e97-b229-29f2cb83fcd2"}]: dispatch 2024-09-13T00:30:24.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:24 smithi181 bash[67587]: audit 2024-09-13T00:30:23.307869+0000 mon.a (mon.0) 2833 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/8e1d2f30-2661-4e97-b229-29f2cb83fcd2"}]: dispatch 2024-09-13T00:30:24.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:24 smithi181 bash[67587]: audit 2024-09-13T00:30:23.316183+0000 mon.a (mon.0) 2834 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/8e1d2f30-2661-4e97-b229-29f2cb83fcd2"}]': finished 2024-09-13T00:30:24.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:24 smithi181 bash[67587]: audit 2024-09-13T00:30:23.322645+0000 mon.c (mon.2) 111 : audit [INF] from='client.? 172.21.15.136:0/2010575863' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/6572737c-910a-45e7-bc7d-59d7d4492e6a"}]: dispatch 2024-09-13T00:30:24.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:24 smithi181 bash[67587]: audit 2024-09-13T00:30:23.323242+0000 mon.a (mon.0) 2835 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/6572737c-910a-45e7-bc7d-59d7d4492e6a"}]: dispatch 2024-09-13T00:30:24.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:24 smithi181 bash[67587]: audit 2024-09-13T00:30:23.331454+0000 mon.a (mon.0) 2836 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/6572737c-910a-45e7-bc7d-59d7d4492e6a"}]': finished 2024-09-13T00:30:24.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:24 smithi181 bash[67587]: audit 2024-09-13T00:30:23.822148+0000 mon.a (mon.0) 2837 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:30:25.600 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:25 smithi181 bash[67587]: cluster 2024-09-13T00:30:24.018401+0000 mgr.y (mgr.24661) 1598 : cluster [DBG] pgmap v2692: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 153 KiB/s rd, 9.5 KiB/s wr, 204 op/s 2024-09-13T00:30:25.601 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:25 smithi181 bash[67587]: cluster 2024-09-13T00:30:24.340800+0000 mon.a (mon.0) 2838 : cluster [DBG] osdmap e1534: 8 total, 8 up, 8 in 2024-09-13T00:30:25.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:25 smithi136 bash[75164]: cluster 2024-09-13T00:30:24.018401+0000 mgr.y (mgr.24661) 1598 : cluster [DBG] pgmap v2692: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 153 KiB/s rd, 9.5 KiB/s wr, 204 op/s 2024-09-13T00:30:25.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:25 smithi136 bash[75164]: cluster 2024-09-13T00:30:24.340800+0000 mon.a (mon.0) 2838 : cluster [DBG] osdmap e1534: 8 total, 8 up, 8 in 2024-09-13T00:30:25.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:25 smithi136 bash[76371]: cluster 2024-09-13T00:30:24.018401+0000 mgr.y (mgr.24661) 1598 : cluster [DBG] pgmap v2692: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 153 KiB/s rd, 9.5 KiB/s wr, 204 op/s 2024-09-13T00:30:25.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:25 smithi136 bash[76371]: cluster 2024-09-13T00:30:24.340800+0000 mon.a (mon.0) 2838 : cluster [DBG] osdmap e1534: 8 total, 8 up, 8 in 2024-09-13T00:30:26.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:26 smithi136 bash[75164]: cluster 2024-09-13T00:30:25.356540+0000 mon.a (mon.0) 2839 : cluster [DBG] osdmap e1535: 8 total, 8 up, 8 in 2024-09-13T00:30:26.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:26 smithi136 bash[76371]: cluster 2024-09-13T00:30:25.356540+0000 mon.a (mon.0) 2839 : cluster [DBG] osdmap e1535: 8 total, 8 up, 8 in 2024-09-13T00:30:26.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:26 smithi181 bash[67587]: cluster 2024-09-13T00:30:25.356540+0000 mon.a (mon.0) 2839 : cluster [DBG] osdmap e1535: 8 total, 8 up, 8 in 2024-09-13T00:30:27.479 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMirroring.SnapshotImageState (4137 ms) 2024-09-13T00:30:27.479 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMirroring.SnapshotPromoteDemote 2024-09-13T00:30:27.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:27 smithi136 bash[75164]: cluster 2024-09-13T00:30:26.019345+0000 mgr.y (mgr.24661) 1599 : cluster [DBG] pgmap v2695: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 39 KiB/s rd, 6.7 KiB/s wr, 60 op/s 2024-09-13T00:30:27.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:27 smithi136 bash[75164]: cluster 2024-09-13T00:30:26.364114+0000 mon.a (mon.0) 2840 : cluster [DBG] osdmap e1536: 8 total, 8 up, 8 in 2024-09-13T00:30:27.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:27 smithi136 bash[76371]: cluster 2024-09-13T00:30:26.019345+0000 mgr.y (mgr.24661) 1599 : cluster [DBG] pgmap v2695: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 39 KiB/s rd, 6.7 KiB/s wr, 60 op/s 2024-09-13T00:30:27.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:27 smithi136 bash[76371]: cluster 2024-09-13T00:30:26.364114+0000 mon.a (mon.0) 2840 : cluster [DBG] osdmap e1536: 8 total, 8 up, 8 in 2024-09-13T00:30:27.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:27 smithi181 bash[67587]: cluster 2024-09-13T00:30:26.019345+0000 mgr.y (mgr.24661) 1599 : cluster [DBG] pgmap v2695: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 39 KiB/s rd, 6.7 KiB/s wr, 60 op/s 2024-09-13T00:30:27.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:27 smithi181 bash[67587]: cluster 2024-09-13T00:30:26.364114+0000 mon.a (mon.0) 2840 : cluster [DBG] osdmap e1536: 8 total, 8 up, 8 in 2024-09-13T00:30:28.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:28 smithi136 bash[76371]: cluster 2024-09-13T00:30:27.371387+0000 mon.a (mon.0) 2841 : cluster [DBG] osdmap e1537: 8 total, 8 up, 8 in 2024-09-13T00:30:28.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:28 smithi136 bash[76371]: cluster 2024-09-13T00:30:28.020198+0000 mgr.y (mgr.24661) 1600 : cluster [DBG] pgmap v2698: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 152 KiB/s rd, 23 KiB/s wr, 215 op/s 2024-09-13T00:30:28.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:28 smithi136 bash[75164]: cluster 2024-09-13T00:30:27.371387+0000 mon.a (mon.0) 2841 : cluster [DBG] osdmap e1537: 8 total, 8 up, 8 in 2024-09-13T00:30:28.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:28 smithi136 bash[75164]: cluster 2024-09-13T00:30:28.020198+0000 mgr.y (mgr.24661) 1600 : cluster [DBG] pgmap v2698: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 152 KiB/s rd, 23 KiB/s wr, 215 op/s 2024-09-13T00:30:28.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:28 smithi181 bash[67587]: cluster 2024-09-13T00:30:27.371387+0000 mon.a (mon.0) 2841 : cluster [DBG] osdmap e1537: 8 total, 8 up, 8 in 2024-09-13T00:30:28.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:28 smithi181 bash[67587]: cluster 2024-09-13T00:30:28.020198+0000 mgr.y (mgr.24661) 1600 : cluster [DBG] pgmap v2698: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 152 KiB/s rd, 23 KiB/s wr, 215 op/s 2024-09-13T00:30:29.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:30:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:30:29] "GET /metrics HTTP/1.1" 200 36541 "" "Prometheus/2.43.0" 2024-09-13T00:30:29.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:29 smithi136 bash[76371]: cluster 2024-09-13T00:30:28.383804+0000 mon.a (mon.0) 2842 : cluster [DBG] osdmap e1538: 8 total, 8 up, 8 in 2024-09-13T00:30:29.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:29 smithi136 bash[76371]: cluster 2024-09-13T00:30:29.092253+0000 mon.a (mon.0) 2843 : cluster [DBG] osdmap e1539: 8 total, 8 up, 8 in 2024-09-13T00:30:29.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:29 smithi136 bash[75164]: cluster 2024-09-13T00:30:28.383804+0000 mon.a (mon.0) 2842 : cluster [DBG] osdmap e1538: 8 total, 8 up, 8 in 2024-09-13T00:30:29.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:29 smithi136 bash[75164]: cluster 2024-09-13T00:30:29.092253+0000 mon.a (mon.0) 2843 : cluster [DBG] osdmap e1539: 8 total, 8 up, 8 in 2024-09-13T00:30:29.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:29 smithi181 bash[67587]: cluster 2024-09-13T00:30:28.383804+0000 mon.a (mon.0) 2842 : cluster [DBG] osdmap e1538: 8 total, 8 up, 8 in 2024-09-13T00:30:29.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:29 smithi181 bash[67587]: cluster 2024-09-13T00:30:29.092253+0000 mon.a (mon.0) 2843 : cluster [DBG] osdmap e1539: 8 total, 8 up, 8 in 2024-09-13T00:30:31.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:31 smithi181 bash[67587]: cluster 2024-09-13T00:30:30.021131+0000 mgr.y (mgr.24661) 1601 : cluster [DBG] pgmap v2701: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 120 KiB/s rd, 19 KiB/s wr, 169 op/s 2024-09-13T00:30:31.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:31 smithi181 bash[67587]: cluster 2024-09-13T00:30:30.096277+0000 mon.a (mon.0) 2844 : cluster [DBG] osdmap e1540: 8 total, 8 up, 8 in 2024-09-13T00:30:31.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:31 smithi136 bash[75164]: cluster 2024-09-13T00:30:30.021131+0000 mgr.y (mgr.24661) 1601 : cluster [DBG] pgmap v2701: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 120 KiB/s rd, 19 KiB/s wr, 169 op/s 2024-09-13T00:30:31.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:31 smithi136 bash[75164]: cluster 2024-09-13T00:30:30.096277+0000 mon.a (mon.0) 2844 : cluster [DBG] osdmap e1540: 8 total, 8 up, 8 in 2024-09-13T00:30:31.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:31 smithi136 bash[76371]: cluster 2024-09-13T00:30:30.021131+0000 mgr.y (mgr.24661) 1601 : cluster [DBG] pgmap v2701: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 120 KiB/s rd, 19 KiB/s wr, 169 op/s 2024-09-13T00:30:31.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:31 smithi136 bash[76371]: cluster 2024-09-13T00:30:30.096277+0000 mon.a (mon.0) 2844 : cluster [DBG] osdmap e1540: 8 total, 8 up, 8 in 2024-09-13T00:30:32.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:32 smithi136 bash[75164]: cluster 2024-09-13T00:30:31.105009+0000 mon.a (mon.0) 2845 : cluster [DBG] osdmap e1541: 8 total, 8 up, 8 in 2024-09-13T00:30:32.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:32 smithi136 bash[76371]: cluster 2024-09-13T00:30:31.105009+0000 mon.a (mon.0) 2845 : cluster [DBG] osdmap e1541: 8 total, 8 up, 8 in 2024-09-13T00:30:32.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:32 smithi181 bash[67587]: cluster 2024-09-13T00:30:31.105009+0000 mon.a (mon.0) 2845 : cluster [DBG] osdmap e1541: 8 total, 8 up, 8 in 2024-09-13T00:30:32.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:30:32 smithi181 bash[95505]: ts=2024-09-13T00:30:32.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:30:33.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:33 smithi136 bash[75164]: cluster 2024-09-13T00:30:32.022243+0000 mgr.y (mgr.24661) 1602 : cluster [DBG] pgmap v2704: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 121 KiB/s rd, 10 KiB/s wr, 168 op/s 2024-09-13T00:30:33.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:33 smithi136 bash[75164]: cluster 2024-09-13T00:30:32.114864+0000 mon.a (mon.0) 2846 : cluster [DBG] osdmap e1542: 8 total, 8 up, 8 in 2024-09-13T00:30:33.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:33 smithi136 bash[76371]: cluster 2024-09-13T00:30:32.022243+0000 mgr.y (mgr.24661) 1602 : cluster [DBG] pgmap v2704: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 121 KiB/s rd, 10 KiB/s wr, 168 op/s 2024-09-13T00:30:33.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:33 smithi136 bash[76371]: cluster 2024-09-13T00:30:32.114864+0000 mon.a (mon.0) 2846 : cluster [DBG] osdmap e1542: 8 total, 8 up, 8 in 2024-09-13T00:30:33.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:33 smithi181 bash[67587]: cluster 2024-09-13T00:30:32.022243+0000 mgr.y (mgr.24661) 1602 : cluster [DBG] pgmap v2704: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 121 KiB/s rd, 10 KiB/s wr, 168 op/s 2024-09-13T00:30:33.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:33 smithi181 bash[67587]: cluster 2024-09-13T00:30:32.114864+0000 mon.a (mon.0) 2846 : cluster [DBG] osdmap e1542: 8 total, 8 up, 8 in 2024-09-13T00:30:34.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:34 smithi136 bash[76371]: cluster 2024-09-13T00:30:33.123228+0000 mon.a (mon.0) 2847 : cluster [DBG] osdmap e1543: 8 total, 8 up, 8 in 2024-09-13T00:30:34.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:34 smithi136 bash[76371]: cluster 2024-09-13T00:30:34.095684+0000 mon.a (mon.0) 2848 : cluster [DBG] osdmap e1544: 8 total, 8 up, 8 in 2024-09-13T00:30:34.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:34 smithi136 bash[75164]: cluster 2024-09-13T00:30:33.123228+0000 mon.a (mon.0) 2847 : cluster [DBG] osdmap e1543: 8 total, 8 up, 8 in 2024-09-13T00:30:34.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:34 smithi136 bash[75164]: cluster 2024-09-13T00:30:34.095684+0000 mon.a (mon.0) 2848 : cluster [DBG] osdmap e1544: 8 total, 8 up, 8 in 2024-09-13T00:30:34.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:34 smithi181 bash[67587]: cluster 2024-09-13T00:30:33.123228+0000 mon.a (mon.0) 2847 : cluster [DBG] osdmap e1543: 8 total, 8 up, 8 in 2024-09-13T00:30:34.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:34 smithi181 bash[67587]: cluster 2024-09-13T00:30:34.095684+0000 mon.a (mon.0) 2848 : cluster [DBG] osdmap e1544: 8 total, 8 up, 8 in 2024-09-13T00:30:35.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:35 smithi136 bash[75164]: cluster 2024-09-13T00:30:34.023106+0000 mgr.y (mgr.24661) 1603 : cluster [DBG] pgmap v2707: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 119 KiB/s rd, 8.5 KiB/s wr, 163 op/s 2024-09-13T00:30:35.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:35 smithi136 bash[76371]: cluster 2024-09-13T00:30:34.023106+0000 mgr.y (mgr.24661) 1603 : cluster [DBG] pgmap v2707: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 119 KiB/s rd, 8.5 KiB/s wr, 163 op/s 2024-09-13T00:30:35.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:35 smithi181 bash[67587]: cluster 2024-09-13T00:30:34.023106+0000 mgr.y (mgr.24661) 1603 : cluster [DBG] pgmap v2707: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 119 KiB/s rd, 8.5 KiB/s wr, 163 op/s 2024-09-13T00:30:36.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:36 smithi136 bash[75164]: cluster 2024-09-13T00:30:35.153694+0000 mon.a (mon.0) 2849 : cluster [DBG] osdmap e1545: 8 total, 8 up, 8 in 2024-09-13T00:30:36.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:36 smithi136 bash[76371]: cluster 2024-09-13T00:30:35.153694+0000 mon.a (mon.0) 2849 : cluster [DBG] osdmap e1545: 8 total, 8 up, 8 in 2024-09-13T00:30:36.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:36 smithi181 bash[67587]: cluster 2024-09-13T00:30:35.153694+0000 mon.a (mon.0) 2849 : cluster [DBG] osdmap e1545: 8 total, 8 up, 8 in 2024-09-13T00:30:37.323 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMirroring.SnapshotPromoteDemote (9844 ms) 2024-09-13T00:30:37.323 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestMirroring.AioSnapshotCreate 2024-09-13T00:30:37.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:37 smithi136 bash[76371]: cluster 2024-09-13T00:30:36.024031+0000 mgr.y (mgr.24661) 1604 : cluster [DBG] pgmap v2710: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 2.5 KiB/s rd, 767 B/s wr, 4 op/s 2024-09-13T00:30:37.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:37 smithi136 bash[76371]: cluster 2024-09-13T00:30:36.171402+0000 mon.a (mon.0) 2850 : cluster [DBG] osdmap e1546: 8 total, 8 up, 8 in 2024-09-13T00:30:37.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:37 smithi136 bash[75164]: cluster 2024-09-13T00:30:36.024031+0000 mgr.y (mgr.24661) 1604 : cluster [DBG] pgmap v2710: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 2.5 KiB/s rd, 767 B/s wr, 4 op/s 2024-09-13T00:30:37.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:37 smithi136 bash[75164]: cluster 2024-09-13T00:30:36.171402+0000 mon.a (mon.0) 2850 : cluster [DBG] osdmap e1546: 8 total, 8 up, 8 in 2024-09-13T00:30:37.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:37 smithi181 bash[67587]: cluster 2024-09-13T00:30:36.024031+0000 mgr.y (mgr.24661) 1604 : cluster [DBG] pgmap v2710: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 2.5 KiB/s rd, 767 B/s wr, 4 op/s 2024-09-13T00:30:37.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:37 smithi181 bash[67587]: cluster 2024-09-13T00:30:36.171402+0000 mon.a (mon.0) 2850 : cluster [DBG] osdmap e1546: 8 total, 8 up, 8 in 2024-09-13T00:30:38.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:38 smithi136 bash[75164]: cluster 2024-09-13T00:30:37.182252+0000 mon.a (mon.0) 2851 : cluster [DBG] osdmap e1547: 8 total, 8 up, 8 in 2024-09-13T00:30:38.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:38 smithi136 bash[75164]: audit 2024-09-13T00:30:37.305866+0000 mon.c (mon.2) 112 : audit [INF] from='client.? 172.21.15.136:0/2010575863' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/668da36e-5476-43f2-8c57-848ec2eb2a13"}]: dispatch 2024-09-13T00:30:38.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:38 smithi136 bash[75164]: audit 2024-09-13T00:30:37.306534+0000 mon.a (mon.0) 2852 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/668da36e-5476-43f2-8c57-848ec2eb2a13"}]: dispatch 2024-09-13T00:30:38.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:38 smithi136 bash[75164]: audit 2024-09-13T00:30:37.312621+0000 mon.a (mon.0) 2853 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/668da36e-5476-43f2-8c57-848ec2eb2a13"}]': finished 2024-09-13T00:30:38.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:38 smithi136 bash[76371]: cluster 2024-09-13T00:30:37.182252+0000 mon.a (mon.0) 2851 : cluster [DBG] osdmap e1547: 8 total, 8 up, 8 in 2024-09-13T00:30:38.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:38 smithi136 bash[76371]: audit 2024-09-13T00:30:37.305866+0000 mon.c (mon.2) 112 : audit [INF] from='client.? 172.21.15.136:0/2010575863' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/668da36e-5476-43f2-8c57-848ec2eb2a13"}]: dispatch 2024-09-13T00:30:38.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:38 smithi136 bash[76371]: audit 2024-09-13T00:30:37.306534+0000 mon.a (mon.0) 2852 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/668da36e-5476-43f2-8c57-848ec2eb2a13"}]: dispatch 2024-09-13T00:30:38.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:38 smithi136 bash[76371]: audit 2024-09-13T00:30:37.312621+0000 mon.a (mon.0) 2853 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/668da36e-5476-43f2-8c57-848ec2eb2a13"}]': finished 2024-09-13T00:30:38.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:38 smithi181 bash[67587]: cluster 2024-09-13T00:30:37.182252+0000 mon.a (mon.0) 2851 : cluster [DBG] osdmap e1547: 8 total, 8 up, 8 in 2024-09-13T00:30:38.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:38 smithi181 bash[67587]: audit 2024-09-13T00:30:37.305866+0000 mon.c (mon.2) 112 : audit [INF] from='client.? 172.21.15.136:0/2010575863' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/668da36e-5476-43f2-8c57-848ec2eb2a13"}]: dispatch 2024-09-13T00:30:38.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:38 smithi181 bash[67587]: audit 2024-09-13T00:30:37.306534+0000 mon.a (mon.0) 2852 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/668da36e-5476-43f2-8c57-848ec2eb2a13"}]: dispatch 2024-09-13T00:30:38.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:38 smithi181 bash[67587]: audit 2024-09-13T00:30:37.312621+0000 mon.a (mon.0) 2853 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/668da36e-5476-43f2-8c57-848ec2eb2a13"}]': finished 2024-09-13T00:30:39.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:39 smithi181 bash[67587]: cluster 2024-09-13T00:30:38.025027+0000 mgr.y (mgr.24661) 1605 : cluster [DBG] pgmap v2713: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 58 KiB/s rd, 7.2 KiB/s wr, 83 op/s 2024-09-13T00:30:39.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:39 smithi181 bash[67587]: cluster 2024-09-13T00:30:38.326269+0000 mon.a (mon.0) 2854 : cluster [DBG] osdmap e1548: 8 total, 8 up, 8 in 2024-09-13T00:30:39.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:39 smithi181 bash[67587]: audit 2024-09-13T00:30:38.822530+0000 mon.a (mon.0) 2855 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:30:39.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:39 smithi181 bash[67587]: cluster 2024-09-13T00:30:39.098212+0000 mon.a (mon.0) 2856 : cluster [DBG] osdmap e1549: 8 total, 8 up, 8 in 2024-09-13T00:30:39.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:30:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:30:39] "GET /metrics HTTP/1.1" 200 36539 "" "Prometheus/2.43.0" 2024-09-13T00:30:39.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:39 smithi136 bash[75164]: cluster 2024-09-13T00:30:38.025027+0000 mgr.y (mgr.24661) 1605 : cluster [DBG] pgmap v2713: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 58 KiB/s rd, 7.2 KiB/s wr, 83 op/s 2024-09-13T00:30:39.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:39 smithi136 bash[75164]: cluster 2024-09-13T00:30:38.326269+0000 mon.a (mon.0) 2854 : cluster [DBG] osdmap e1548: 8 total, 8 up, 8 in 2024-09-13T00:30:39.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:39 smithi136 bash[75164]: audit 2024-09-13T00:30:38.822530+0000 mon.a (mon.0) 2855 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:30:39.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:39 smithi136 bash[75164]: cluster 2024-09-13T00:30:39.098212+0000 mon.a (mon.0) 2856 : cluster [DBG] osdmap e1549: 8 total, 8 up, 8 in 2024-09-13T00:30:39.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:39 smithi136 bash[76371]: cluster 2024-09-13T00:30:38.025027+0000 mgr.y (mgr.24661) 1605 : cluster [DBG] pgmap v2713: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 58 KiB/s rd, 7.2 KiB/s wr, 83 op/s 2024-09-13T00:30:39.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:39 smithi136 bash[76371]: cluster 2024-09-13T00:30:38.326269+0000 mon.a (mon.0) 2854 : cluster [DBG] osdmap e1548: 8 total, 8 up, 8 in 2024-09-13T00:30:39.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:39 smithi136 bash[76371]: audit 2024-09-13T00:30:38.822530+0000 mon.a (mon.0) 2855 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:30:39.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:39 smithi136 bash[76371]: cluster 2024-09-13T00:30:39.098212+0000 mon.a (mon.0) 2856 : cluster [DBG] osdmap e1549: 8 total, 8 up, 8 in 2024-09-13T00:30:41.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:41 smithi136 bash[75164]: cluster 2024-09-13T00:30:40.025899+0000 mgr.y (mgr.24661) 1606 : cluster [DBG] pgmap v2716: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 58 KiB/s rd, 7.7 KiB/s wr, 84 op/s 2024-09-13T00:30:41.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:41 smithi136 bash[75164]: cluster 2024-09-13T00:30:40.101330+0000 mon.a (mon.0) 2857 : cluster [DBG] osdmap e1550: 8 total, 8 up, 8 in 2024-09-13T00:30:41.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:41 smithi136 bash[76371]: cluster 2024-09-13T00:30:40.025899+0000 mgr.y (mgr.24661) 1606 : cluster [DBG] pgmap v2716: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 58 KiB/s rd, 7.7 KiB/s wr, 84 op/s 2024-09-13T00:30:41.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:41 smithi136 bash[76371]: cluster 2024-09-13T00:30:40.101330+0000 mon.a (mon.0) 2857 : cluster [DBG] osdmap e1550: 8 total, 8 up, 8 in 2024-09-13T00:30:41.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:41 smithi181 bash[67587]: cluster 2024-09-13T00:30:40.025899+0000 mgr.y (mgr.24661) 1606 : cluster [DBG] pgmap v2716: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 58 KiB/s rd, 7.7 KiB/s wr, 84 op/s 2024-09-13T00:30:41.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:41 smithi181 bash[67587]: cluster 2024-09-13T00:30:40.101330+0000 mon.a (mon.0) 2857 : cluster [DBG] osdmap e1550: 8 total, 8 up, 8 in 2024-09-13T00:30:42.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:42 smithi136 bash[75164]: cluster 2024-09-13T00:30:41.103454+0000 mon.a (mon.0) 2858 : cluster [DBG] osdmap e1551: 8 total, 8 up, 8 in 2024-09-13T00:30:42.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:42 smithi136 bash[76371]: cluster 2024-09-13T00:30:41.103454+0000 mon.a (mon.0) 2858 : cluster [DBG] osdmap e1551: 8 total, 8 up, 8 in 2024-09-13T00:30:42.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:42 smithi181 bash[67587]: cluster 2024-09-13T00:30:41.103454+0000 mon.a (mon.0) 2858 : cluster [DBG] osdmap e1551: 8 total, 8 up, 8 in 2024-09-13T00:30:42.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:30:42 smithi181 bash[95505]: ts=2024-09-13T00:30:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:30:43.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:43 smithi136 bash[75164]: cluster 2024-09-13T00:30:42.027031+0000 mgr.y (mgr.24661) 1607 : cluster [DBG] pgmap v2719: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 57 KiB/s rd, 10 KiB/s wr, 87 op/s 2024-09-13T00:30:43.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:43 smithi136 bash[75164]: cluster 2024-09-13T00:30:42.125680+0000 mon.a (mon.0) 2859 : cluster [DBG] osdmap e1552: 8 total, 8 up, 8 in 2024-09-13T00:30:43.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:43 smithi136 bash[76371]: cluster 2024-09-13T00:30:42.027031+0000 mgr.y (mgr.24661) 1607 : cluster [DBG] pgmap v2719: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 57 KiB/s rd, 10 KiB/s wr, 87 op/s 2024-09-13T00:30:43.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:43 smithi136 bash[76371]: cluster 2024-09-13T00:30:42.125680+0000 mon.a (mon.0) 2859 : cluster [DBG] osdmap e1552: 8 total, 8 up, 8 in 2024-09-13T00:30:43.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:43 smithi181 bash[67587]: cluster 2024-09-13T00:30:42.027031+0000 mgr.y (mgr.24661) 1607 : cluster [DBG] pgmap v2719: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 57 KiB/s rd, 10 KiB/s wr, 87 op/s 2024-09-13T00:30:43.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:43 smithi181 bash[67587]: cluster 2024-09-13T00:30:42.125680+0000 mon.a (mon.0) 2859 : cluster [DBG] osdmap e1552: 8 total, 8 up, 8 in 2024-09-13T00:30:44.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:44 smithi136 bash[75164]: cluster 2024-09-13T00:30:43.125720+0000 mon.a (mon.0) 2860 : cluster [DBG] osdmap e1553: 8 total, 8 up, 8 in 2024-09-13T00:30:44.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:44 smithi136 bash[75164]: cluster 2024-09-13T00:30:44.099438+0000 mon.a (mon.0) 2861 : cluster [DBG] osdmap e1554: 8 total, 8 up, 8 in 2024-09-13T00:30:44.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:44 smithi136 bash[76371]: cluster 2024-09-13T00:30:43.125720+0000 mon.a (mon.0) 2860 : cluster [DBG] osdmap e1553: 8 total, 8 up, 8 in 2024-09-13T00:30:44.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:44 smithi136 bash[76371]: cluster 2024-09-13T00:30:44.099438+0000 mon.a (mon.0) 2861 : cluster [DBG] osdmap e1554: 8 total, 8 up, 8 in 2024-09-13T00:30:44.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:44 smithi181 bash[67587]: cluster 2024-09-13T00:30:43.125720+0000 mon.a (mon.0) 2860 : cluster [DBG] osdmap e1553: 8 total, 8 up, 8 in 2024-09-13T00:30:44.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:44 smithi181 bash[67587]: cluster 2024-09-13T00:30:44.099438+0000 mon.a (mon.0) 2861 : cluster [DBG] osdmap e1554: 8 total, 8 up, 8 in 2024-09-13T00:30:45.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:45 smithi136 bash[75164]: cluster 2024-09-13T00:30:44.027938+0000 mgr.y (mgr.24661) 1608 : cluster [DBG] pgmap v2722: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 79 KiB/s rd, 12 KiB/s wr, 115 op/s 2024-09-13T00:30:45.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:45 smithi136 bash[76371]: cluster 2024-09-13T00:30:44.027938+0000 mgr.y (mgr.24661) 1608 : cluster [DBG] pgmap v2722: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 79 KiB/s rd, 12 KiB/s wr, 115 op/s 2024-09-13T00:30:45.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:45 smithi181 bash[67587]: cluster 2024-09-13T00:30:44.027938+0000 mgr.y (mgr.24661) 1608 : cluster [DBG] pgmap v2722: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 79 KiB/s rd, 12 KiB/s wr, 115 op/s 2024-09-13T00:30:46.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:46 smithi136 bash[75164]: cluster 2024-09-13T00:30:45.154187+0000 mon.a (mon.0) 2862 : cluster [DBG] osdmap e1555: 8 total, 8 up, 8 in 2024-09-13T00:30:46.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:46 smithi136 bash[76371]: cluster 2024-09-13T00:30:45.154187+0000 mon.a (mon.0) 2862 : cluster [DBG] osdmap e1555: 8 total, 8 up, 8 in 2024-09-13T00:30:46.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:46 smithi181 bash[67587]: cluster 2024-09-13T00:30:45.154187+0000 mon.a (mon.0) 2862 : cluster [DBG] osdmap e1555: 8 total, 8 up, 8 in 2024-09-13T00:30:47.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:47 smithi136 bash[75164]: cluster 2024-09-13T00:30:46.028903+0000 mgr.y (mgr.24661) 1609 : cluster [DBG] pgmap v2725: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 45 KiB/s rd, 7.2 KiB/s wr, 68 op/s 2024-09-13T00:30:47.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:47 smithi136 bash[75164]: cluster 2024-09-13T00:30:46.167930+0000 mon.a (mon.0) 2863 : cluster [DBG] osdmap e1556: 8 total, 8 up, 8 in 2024-09-13T00:30:47.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:47 smithi136 bash[76371]: cluster 2024-09-13T00:30:46.028903+0000 mgr.y (mgr.24661) 1609 : cluster [DBG] pgmap v2725: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 45 KiB/s rd, 7.2 KiB/s wr, 68 op/s 2024-09-13T00:30:47.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:47 smithi136 bash[76371]: cluster 2024-09-13T00:30:46.167930+0000 mon.a (mon.0) 2863 : cluster [DBG] osdmap e1556: 8 total, 8 up, 8 in 2024-09-13T00:30:47.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:47 smithi181 bash[67587]: cluster 2024-09-13T00:30:46.028903+0000 mgr.y (mgr.24661) 1609 : cluster [DBG] pgmap v2725: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 45 KiB/s rd, 7.2 KiB/s wr, 68 op/s 2024-09-13T00:30:47.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:47 smithi181 bash[67587]: cluster 2024-09-13T00:30:46.167930+0000 mon.a (mon.0) 2863 : cluster [DBG] osdmap e1556: 8 total, 8 up, 8 in 2024-09-13T00:30:48.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:48 smithi136 bash[75164]: cluster 2024-09-13T00:30:47.188047+0000 mon.a (mon.0) 2864 : cluster [DBG] osdmap e1557: 8 total, 8 up, 8 in 2024-09-13T00:30:48.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:48 smithi136 bash[76371]: cluster 2024-09-13T00:30:47.188047+0000 mon.a (mon.0) 2864 : cluster [DBG] osdmap e1557: 8 total, 8 up, 8 in 2024-09-13T00:30:48.600 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:48 smithi181 bash[67587]: cluster 2024-09-13T00:30:47.188047+0000 mon.a (mon.0) 2864 : cluster [DBG] osdmap e1557: 8 total, 8 up, 8 in 2024-09-13T00:30:49.509 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:30:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:30:49] "GET /metrics HTTP/1.1" 200 36539 "" "Prometheus/2.43.0" 2024-09-13T00:30:49.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:49 smithi136 bash[75164]: cluster 2024-09-13T00:30:48.029740+0000 mgr.y (mgr.24661) 1610 : cluster [DBG] pgmap v2728: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 88 KiB/s rd, 14 KiB/s wr, 126 op/s 2024-09-13T00:30:49.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:49 smithi136 bash[75164]: cluster 2024-09-13T00:30:48.197210+0000 mon.a (mon.0) 2865 : cluster [DBG] osdmap e1558: 8 total, 8 up, 8 in 2024-09-13T00:30:49.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:49 smithi136 bash[76371]: cluster 2024-09-13T00:30:48.029740+0000 mgr.y (mgr.24661) 1610 : cluster [DBG] pgmap v2728: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 88 KiB/s rd, 14 KiB/s wr, 126 op/s 2024-09-13T00:30:49.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:49 smithi136 bash[76371]: cluster 2024-09-13T00:30:48.197210+0000 mon.a (mon.0) 2865 : cluster [DBG] osdmap e1558: 8 total, 8 up, 8 in 2024-09-13T00:30:49.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:49 smithi181 bash[67587]: cluster 2024-09-13T00:30:48.029740+0000 mgr.y (mgr.24661) 1610 : cluster [DBG] pgmap v2728: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 88 KiB/s rd, 14 KiB/s wr, 126 op/s 2024-09-13T00:30:49.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:49 smithi181 bash[67587]: cluster 2024-09-13T00:30:48.197210+0000 mon.a (mon.0) 2865 : cluster [DBG] osdmap e1558: 8 total, 8 up, 8 in 2024-09-13T00:30:50.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:50 smithi136 bash[75164]: cluster 2024-09-13T00:30:49.204914+0000 mon.a (mon.0) 2866 : cluster [DBG] osdmap e1559: 8 total, 8 up, 8 in 2024-09-13T00:30:50.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:50 smithi136 bash[76371]: cluster 2024-09-13T00:30:49.204914+0000 mon.a (mon.0) 2866 : cluster [DBG] osdmap e1559: 8 total, 8 up, 8 in 2024-09-13T00:30:50.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:50 smithi181 bash[67587]: cluster 2024-09-13T00:30:49.204914+0000 mon.a (mon.0) 2866 : cluster [DBG] osdmap e1559: 8 total, 8 up, 8 in 2024-09-13T00:30:51.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:51 smithi136 bash[75164]: cluster 2024-09-13T00:30:50.030535+0000 mgr.y (mgr.24661) 1611 : cluster [DBG] pgmap v2731: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 71 KiB/s rd, 11 KiB/s wr, 98 op/s 2024-09-13T00:30:51.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:51 smithi136 bash[75164]: cluster 2024-09-13T00:30:50.215097+0000 mon.a (mon.0) 2867 : cluster [DBG] osdmap e1560: 8 total, 8 up, 8 in 2024-09-13T00:30:51.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:51 smithi136 bash[76371]: cluster 2024-09-13T00:30:50.030535+0000 mgr.y (mgr.24661) 1611 : cluster [DBG] pgmap v2731: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 71 KiB/s rd, 11 KiB/s wr, 98 op/s 2024-09-13T00:30:51.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:51 smithi136 bash[76371]: cluster 2024-09-13T00:30:50.215097+0000 mon.a (mon.0) 2867 : cluster [DBG] osdmap e1560: 8 total, 8 up, 8 in 2024-09-13T00:30:51.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:51 smithi181 bash[67587]: cluster 2024-09-13T00:30:50.030535+0000 mgr.y (mgr.24661) 1611 : cluster [DBG] pgmap v2731: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 71 KiB/s rd, 11 KiB/s wr, 98 op/s 2024-09-13T00:30:51.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:51 smithi181 bash[67587]: cluster 2024-09-13T00:30:50.215097+0000 mon.a (mon.0) 2867 : cluster [DBG] osdmap e1560: 8 total, 8 up, 8 in 2024-09-13T00:30:52.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:52 smithi136 bash[75164]: cluster 2024-09-13T00:30:51.227651+0000 mon.a (mon.0) 2868 : cluster [DBG] osdmap e1561: 8 total, 8 up, 8 in 2024-09-13T00:30:52.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:52 smithi136 bash[76371]: cluster 2024-09-13T00:30:51.227651+0000 mon.a (mon.0) 2868 : cluster [DBG] osdmap e1561: 8 total, 8 up, 8 in 2024-09-13T00:30:52.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:52 smithi181 bash[67587]: cluster 2024-09-13T00:30:51.227651+0000 mon.a (mon.0) 2868 : cluster [DBG] osdmap e1561: 8 total, 8 up, 8 in 2024-09-13T00:30:52.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:30:52 smithi181 bash[95505]: ts=2024-09-13T00:30:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:30:53.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:53 smithi136 bash[75164]: cluster 2024-09-13T00:30:52.031491+0000 mgr.y (mgr.24661) 1612 : cluster [DBG] pgmap v2734: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 116 KiB/s rd, 14 KiB/s wr, 158 op/s 2024-09-13T00:30:53.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:53 smithi136 bash[75164]: cluster 2024-09-13T00:30:52.241989+0000 mon.a (mon.0) 2869 : cluster [DBG] osdmap e1562: 8 total, 8 up, 8 in 2024-09-13T00:30:53.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:53 smithi136 bash[76371]: cluster 2024-09-13T00:30:52.031491+0000 mgr.y (mgr.24661) 1612 : cluster [DBG] pgmap v2734: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 116 KiB/s rd, 14 KiB/s wr, 158 op/s 2024-09-13T00:30:53.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:53 smithi136 bash[76371]: cluster 2024-09-13T00:30:52.241989+0000 mon.a (mon.0) 2869 : cluster [DBG] osdmap e1562: 8 total, 8 up, 8 in 2024-09-13T00:30:53.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:53 smithi181 bash[67587]: cluster 2024-09-13T00:30:52.031491+0000 mgr.y (mgr.24661) 1612 : cluster [DBG] pgmap v2734: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 116 KiB/s rd, 14 KiB/s wr, 158 op/s 2024-09-13T00:30:53.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:53 smithi181 bash[67587]: cluster 2024-09-13T00:30:52.241989+0000 mon.a (mon.0) 2869 : cluster [DBG] osdmap e1562: 8 total, 8 up, 8 in 2024-09-13T00:30:54.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:54 smithi181 bash[67587]: cluster 2024-09-13T00:30:53.241004+0000 mon.a (mon.0) 2870 : cluster [DBG] osdmap e1563: 8 total, 8 up, 8 in 2024-09-13T00:30:54.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:54 smithi181 bash[67587]: audit 2024-09-13T00:30:53.823110+0000 mon.a (mon.0) 2871 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:30:54.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:54 smithi136 bash[75164]: cluster 2024-09-13T00:30:53.241004+0000 mon.a (mon.0) 2870 : cluster [DBG] osdmap e1563: 8 total, 8 up, 8 in 2024-09-13T00:30:54.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:54 smithi136 bash[75164]: audit 2024-09-13T00:30:53.823110+0000 mon.a (mon.0) 2871 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:30:54.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:54 smithi136 bash[76371]: cluster 2024-09-13T00:30:53.241004+0000 mon.a (mon.0) 2870 : cluster [DBG] osdmap e1563: 8 total, 8 up, 8 in 2024-09-13T00:30:54.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:54 smithi136 bash[76371]: audit 2024-09-13T00:30:53.823110+0000 mon.a (mon.0) 2871 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:30:55.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:55 smithi181 bash[67587]: cluster 2024-09-13T00:30:54.032281+0000 mgr.y (mgr.24661) 1613 : cluster [DBG] pgmap v2737: 137 pgs: 137 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 198 KiB/s rd, 21 KiB/s wr, 263 op/s 2024-09-13T00:30:55.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:55 smithi181 bash[67587]: cluster 2024-09-13T00:30:54.277691+0000 mon.a (mon.0) 2872 : cluster [DBG] osdmap e1564: 8 total, 8 up, 8 in 2024-09-13T00:30:55.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:55 smithi136 bash[75164]: cluster 2024-09-13T00:30:54.032281+0000 mgr.y (mgr.24661) 1613 : cluster [DBG] pgmap v2737: 137 pgs: 137 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 198 KiB/s rd, 21 KiB/s wr, 263 op/s 2024-09-13T00:30:55.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:55 smithi136 bash[75164]: cluster 2024-09-13T00:30:54.277691+0000 mon.a (mon.0) 2872 : cluster [DBG] osdmap e1564: 8 total, 8 up, 8 in 2024-09-13T00:30:55.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:55 smithi136 bash[76371]: cluster 2024-09-13T00:30:54.032281+0000 mgr.y (mgr.24661) 1613 : cluster [DBG] pgmap v2737: 137 pgs: 137 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 198 KiB/s rd, 21 KiB/s wr, 263 op/s 2024-09-13T00:30:55.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:55 smithi136 bash[76371]: cluster 2024-09-13T00:30:54.277691+0000 mon.a (mon.0) 2872 : cluster [DBG] osdmap e1564: 8 total, 8 up, 8 in 2024-09-13T00:30:56.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:56 smithi181 bash[67587]: cluster 2024-09-13T00:30:55.288166+0000 mon.a (mon.0) 2873 : cluster [DBG] osdmap e1565: 8 total, 8 up, 8 in 2024-09-13T00:30:56.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:56 smithi136 bash[75164]: cluster 2024-09-13T00:30:55.288166+0000 mon.a (mon.0) 2873 : cluster [DBG] osdmap e1565: 8 total, 8 up, 8 in 2024-09-13T00:30:56.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:56 smithi136 bash[76371]: cluster 2024-09-13T00:30:55.288166+0000 mon.a (mon.0) 2873 : cluster [DBG] osdmap e1565: 8 total, 8 up, 8 in 2024-09-13T00:30:57.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:57 smithi181 bash[67587]: cluster 2024-09-13T00:30:56.033321+0000 mgr.y (mgr.24661) 1614 : cluster [DBG] pgmap v2740: 137 pgs: 137 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 97 KiB/s rd, 8.5 KiB/s wr, 135 op/s 2024-09-13T00:30:57.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:57 smithi181 bash[67587]: cluster 2024-09-13T00:30:56.295514+0000 mon.a (mon.0) 2874 : cluster [DBG] osdmap e1566: 8 total, 8 up, 8 in 2024-09-13T00:30:57.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:57 smithi136 bash[75164]: cluster 2024-09-13T00:30:56.033321+0000 mgr.y (mgr.24661) 1614 : cluster [DBG] pgmap v2740: 137 pgs: 137 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 97 KiB/s rd, 8.5 KiB/s wr, 135 op/s 2024-09-13T00:30:57.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:57 smithi136 bash[75164]: cluster 2024-09-13T00:30:56.295514+0000 mon.a (mon.0) 2874 : cluster [DBG] osdmap e1566: 8 total, 8 up, 8 in 2024-09-13T00:30:57.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:57 smithi136 bash[76371]: cluster 2024-09-13T00:30:56.033321+0000 mgr.y (mgr.24661) 1614 : cluster [DBG] pgmap v2740: 137 pgs: 137 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 97 KiB/s rd, 8.5 KiB/s wr, 135 op/s 2024-09-13T00:30:57.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:57 smithi136 bash[76371]: cluster 2024-09-13T00:30:56.295514+0000 mon.a (mon.0) 2874 : cluster [DBG] osdmap e1566: 8 total, 8 up, 8 in 2024-09-13T00:30:58.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:58 smithi181 bash[67587]: cluster 2024-09-13T00:30:57.300708+0000 mon.a (mon.0) 2875 : cluster [DBG] osdmap e1567: 8 total, 8 up, 8 in 2024-09-13T00:30:58.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:58 smithi136 bash[75164]: cluster 2024-09-13T00:30:57.300708+0000 mon.a (mon.0) 2875 : cluster [DBG] osdmap e1567: 8 total, 8 up, 8 in 2024-09-13T00:30:58.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:58 smithi136 bash[76371]: cluster 2024-09-13T00:30:57.300708+0000 mon.a (mon.0) 2875 : cluster [DBG] osdmap e1567: 8 total, 8 up, 8 in 2024-09-13T00:30:59.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:59 smithi181 bash[67587]: cluster 2024-09-13T00:30:58.034182+0000 mgr.y (mgr.24661) 1615 : cluster [DBG] pgmap v2743: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 59 KiB/s rd, 4.7 KiB/s wr, 90 op/s 2024-09-13T00:30:59.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:59 smithi181 bash[67587]: cluster 2024-09-13T00:30:58.324094+0000 mon.a (mon.0) 2876 : cluster [DBG] osdmap e1568: 8 total, 8 up, 8 in 2024-09-13T00:30:59.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:30:59 smithi181 bash[67587]: cluster 2024-09-13T00:30:59.106105+0000 mon.a (mon.0) 2877 : cluster [DBG] osdmap e1569: 8 total, 8 up, 8 in 2024-09-13T00:30:59.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:30:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:30:59] "GET /metrics HTTP/1.1" 200 36547 "" "Prometheus/2.43.0" 2024-09-13T00:30:59.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:59 smithi136 bash[75164]: cluster 2024-09-13T00:30:58.034182+0000 mgr.y (mgr.24661) 1615 : cluster [DBG] pgmap v2743: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 59 KiB/s rd, 4.7 KiB/s wr, 90 op/s 2024-09-13T00:30:59.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:59 smithi136 bash[75164]: cluster 2024-09-13T00:30:58.324094+0000 mon.a (mon.0) 2876 : cluster [DBG] osdmap e1568: 8 total, 8 up, 8 in 2024-09-13T00:30:59.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:30:59 smithi136 bash[75164]: cluster 2024-09-13T00:30:59.106105+0000 mon.a (mon.0) 2877 : cluster [DBG] osdmap e1569: 8 total, 8 up, 8 in 2024-09-13T00:30:59.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:59 smithi136 bash[76371]: cluster 2024-09-13T00:30:58.034182+0000 mgr.y (mgr.24661) 1615 : cluster [DBG] pgmap v2743: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 59 KiB/s rd, 4.7 KiB/s wr, 90 op/s 2024-09-13T00:30:59.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:59 smithi136 bash[76371]: cluster 2024-09-13T00:30:58.324094+0000 mon.a (mon.0) 2876 : cluster [DBG] osdmap e1568: 8 total, 8 up, 8 in 2024-09-13T00:30:59.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:30:59 smithi136 bash[76371]: cluster 2024-09-13T00:30:59.106105+0000 mon.a (mon.0) 2877 : cluster [DBG] osdmap e1569: 8 total, 8 up, 8 in 2024-09-13T00:31:01.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:01 smithi136 bash[76371]: cluster 2024-09-13T00:31:00.034979+0000 mgr.y (mgr.24661) 1616 : cluster [DBG] pgmap v2746: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 50 KiB/s rd, 6.2 KiB/s wr, 73 op/s 2024-09-13T00:31:01.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:01 smithi136 bash[76371]: cluster 2024-09-13T00:31:00.108100+0000 mon.a (mon.0) 2878 : cluster [DBG] osdmap e1570: 8 total, 8 up, 8 in 2024-09-13T00:31:01.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:01 smithi136 bash[75164]: cluster 2024-09-13T00:31:00.034979+0000 mgr.y (mgr.24661) 1616 : cluster [DBG] pgmap v2746: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 50 KiB/s rd, 6.2 KiB/s wr, 73 op/s 2024-09-13T00:31:01.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:01 smithi136 bash[75164]: cluster 2024-09-13T00:31:00.108100+0000 mon.a (mon.0) 2878 : cluster [DBG] osdmap e1570: 8 total, 8 up, 8 in 2024-09-13T00:31:01.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:01 smithi181 bash[67587]: cluster 2024-09-13T00:31:00.034979+0000 mgr.y (mgr.24661) 1616 : cluster [DBG] pgmap v2746: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 50 KiB/s rd, 6.2 KiB/s wr, 73 op/s 2024-09-13T00:31:01.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:01 smithi181 bash[67587]: cluster 2024-09-13T00:31:00.108100+0000 mon.a (mon.0) 2878 : cluster [DBG] osdmap e1570: 8 total, 8 up, 8 in 2024-09-13T00:31:02.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:02 smithi181 bash[67587]: cluster 2024-09-13T00:31:01.331722+0000 mon.a (mon.0) 2879 : cluster [DBG] osdmap e1571: 8 total, 8 up, 8 in 2024-09-13T00:31:02.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:31:02 smithi181 bash[95505]: ts=2024-09-13T00:31:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:31:02.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:02 smithi136 bash[76371]: cluster 2024-09-13T00:31:01.331722+0000 mon.a (mon.0) 2879 : cluster [DBG] osdmap e1571: 8 total, 8 up, 8 in 2024-09-13T00:31:02.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:02 smithi136 bash[75164]: cluster 2024-09-13T00:31:01.331722+0000 mon.a (mon.0) 2879 : cluster [DBG] osdmap e1571: 8 total, 8 up, 8 in 2024-09-13T00:31:03.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:03 smithi181 bash[67587]: cluster 2024-09-13T00:31:02.035898+0000 mgr.y (mgr.24661) 1617 : cluster [DBG] pgmap v2749: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 79 KiB/s rd, 6.5 KiB/s wr, 116 op/s 2024-09-13T00:31:03.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:03 smithi181 bash[67587]: cluster 2024-09-13T00:31:02.347580+0000 mon.a (mon.0) 2880 : cluster [DBG] osdmap e1572: 8 total, 8 up, 8 in 2024-09-13T00:31:03.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:03 smithi136 bash[75164]: cluster 2024-09-13T00:31:02.035898+0000 mgr.y (mgr.24661) 1617 : cluster [DBG] pgmap v2749: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 79 KiB/s rd, 6.5 KiB/s wr, 116 op/s 2024-09-13T00:31:03.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:03 smithi136 bash[75164]: cluster 2024-09-13T00:31:02.347580+0000 mon.a (mon.0) 2880 : cluster [DBG] osdmap e1572: 8 total, 8 up, 8 in 2024-09-13T00:31:03.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:03 smithi136 bash[76371]: cluster 2024-09-13T00:31:02.035898+0000 mgr.y (mgr.24661) 1617 : cluster [DBG] pgmap v2749: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 79 KiB/s rd, 6.5 KiB/s wr, 116 op/s 2024-09-13T00:31:03.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:03 smithi136 bash[76371]: cluster 2024-09-13T00:31:02.347580+0000 mon.a (mon.0) 2880 : cluster [DBG] osdmap e1572: 8 total, 8 up, 8 in 2024-09-13T00:31:04.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:04 smithi136 bash[76371]: cluster 2024-09-13T00:31:03.351573+0000 mon.a (mon.0) 2881 : cluster [DBG] osdmap e1573: 8 total, 8 up, 8 in 2024-09-13T00:31:04.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:04 smithi136 bash[76371]: cluster 2024-09-13T00:31:04.107418+0000 mon.a (mon.0) 2882 : cluster [DBG] osdmap e1574: 8 total, 8 up, 8 in 2024-09-13T00:31:04.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:04 smithi136 bash[75164]: cluster 2024-09-13T00:31:03.351573+0000 mon.a (mon.0) 2881 : cluster [DBG] osdmap e1573: 8 total, 8 up, 8 in 2024-09-13T00:31:04.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:04 smithi136 bash[75164]: cluster 2024-09-13T00:31:04.107418+0000 mon.a (mon.0) 2882 : cluster [DBG] osdmap e1574: 8 total, 8 up, 8 in 2024-09-13T00:31:04.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:04 smithi181 bash[67587]: cluster 2024-09-13T00:31:03.351573+0000 mon.a (mon.0) 2881 : cluster [DBG] osdmap e1573: 8 total, 8 up, 8 in 2024-09-13T00:31:04.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:04 smithi181 bash[67587]: cluster 2024-09-13T00:31:04.107418+0000 mon.a (mon.0) 2882 : cluster [DBG] osdmap e1574: 8 total, 8 up, 8 in 2024-09-13T00:31:05.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:05 smithi136 bash[75164]: cluster 2024-09-13T00:31:04.036878+0000 mgr.y (mgr.24661) 1618 : cluster [DBG] pgmap v2752: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 132 KiB/s rd, 10 KiB/s wr, 187 op/s 2024-09-13T00:31:05.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:05 smithi136 bash[75164]: cluster 2024-09-13T00:31:05.110599+0000 mon.a (mon.0) 2883 : cluster [DBG] osdmap e1575: 8 total, 8 up, 8 in 2024-09-13T00:31:05.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:05 smithi136 bash[76371]: cluster 2024-09-13T00:31:04.036878+0000 mgr.y (mgr.24661) 1618 : cluster [DBG] pgmap v2752: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 132 KiB/s rd, 10 KiB/s wr, 187 op/s 2024-09-13T00:31:05.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:05 smithi136 bash[76371]: cluster 2024-09-13T00:31:05.110599+0000 mon.a (mon.0) 2883 : cluster [DBG] osdmap e1575: 8 total, 8 up, 8 in 2024-09-13T00:31:05.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:05 smithi181 bash[67587]: cluster 2024-09-13T00:31:04.036878+0000 mgr.y (mgr.24661) 1618 : cluster [DBG] pgmap v2752: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 132 KiB/s rd, 10 KiB/s wr, 187 op/s 2024-09-13T00:31:05.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:05 smithi181 bash[67587]: cluster 2024-09-13T00:31:05.110599+0000 mon.a (mon.0) 2883 : cluster [DBG] osdmap e1575: 8 total, 8 up, 8 in 2024-09-13T00:31:07.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:07 smithi136 bash[75164]: cluster 2024-09-13T00:31:06.037988+0000 mgr.y (mgr.24661) 1619 : cluster [DBG] pgmap v2755: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 64 KiB/s rd, 5.0 KiB/s wr, 92 op/s 2024-09-13T00:31:07.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:07 smithi136 bash[75164]: cluster 2024-09-13T00:31:06.117644+0000 mon.a (mon.0) 2884 : cluster [DBG] osdmap e1576: 8 total, 8 up, 8 in 2024-09-13T00:31:07.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:07 smithi136 bash[76371]: cluster 2024-09-13T00:31:06.037988+0000 mgr.y (mgr.24661) 1619 : cluster [DBG] pgmap v2755: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 64 KiB/s rd, 5.0 KiB/s wr, 92 op/s 2024-09-13T00:31:07.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:07 smithi136 bash[76371]: cluster 2024-09-13T00:31:06.117644+0000 mon.a (mon.0) 2884 : cluster [DBG] osdmap e1576: 8 total, 8 up, 8 in 2024-09-13T00:31:07.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:07 smithi181 bash[67587]: cluster 2024-09-13T00:31:06.037988+0000 mgr.y (mgr.24661) 1619 : cluster [DBG] pgmap v2755: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 64 KiB/s rd, 5.0 KiB/s wr, 92 op/s 2024-09-13T00:31:07.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:07 smithi181 bash[67587]: cluster 2024-09-13T00:31:06.117644+0000 mon.a (mon.0) 2884 : cluster [DBG] osdmap e1576: 8 total, 8 up, 8 in 2024-09-13T00:31:08.274 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestMirroring.AioSnapshotCreate (30951 ms) 2024-09-13T00:31:08.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:08 smithi136 bash[75164]: cluster 2024-09-13T00:31:07.123410+0000 mon.a (mon.0) 2885 : cluster [DBG] osdmap e1577: 8 total, 8 up, 8 in 2024-09-13T00:31:08.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:08 smithi136 bash[76371]: cluster 2024-09-13T00:31:07.123410+0000 mon.a (mon.0) 2885 : cluster [DBG] osdmap e1577: 8 total, 8 up, 8 in 2024-09-13T00:31:08.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:08 smithi181 bash[67587]: cluster 2024-09-13T00:31:07.123410+0000 mon.a (mon.0) 2885 : cluster [DBG] osdmap e1577: 8 total, 8 up, 8 in 2024-09-13T00:31:09.120 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 51 tests from TestMirroring (94289 ms total) 2024-09-13T00:31:09.120 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:31:09.120 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 34 tests from TestInternal 2024-09-13T00:31:09.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:09 smithi136 bash[75164]: cluster 2024-09-13T00:31:08.038868+0000 mgr.y (mgr.24661) 1620 : cluster [DBG] pgmap v2758: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 76 KiB/s rd, 5.5 KiB/s wr, 111 op/s 2024-09-13T00:31:09.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:09 smithi136 bash[75164]: cluster 2024-09-13T00:31:08.133445+0000 mon.a (mon.0) 2886 : cluster [DBG] osdmap e1578: 8 total, 8 up, 8 in 2024-09-13T00:31:09.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:09 smithi136 bash[75164]: audit 2024-09-13T00:31:08.256713+0000 mon.c (mon.2) 113 : audit [INF] from='client.? 172.21.15.136:0/2010575863' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/6793075f-d3a7-415b-a32e-e8c0c9d49c28"}]: dispatch 2024-09-13T00:31:09.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:09 smithi136 bash[75164]: audit 2024-09-13T00:31:08.257429+0000 mon.a (mon.0) 2887 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/6793075f-d3a7-415b-a32e-e8c0c9d49c28"}]: dispatch 2024-09-13T00:31:09.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:09 smithi136 bash[75164]: audit 2024-09-13T00:31:08.264301+0000 mon.a (mon.0) 2888 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/6793075f-d3a7-415b-a32e-e8c0c9d49c28"}]': finished 2024-09-13T00:31:09.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:09 smithi136 bash[75164]: audit 2024-09-13T00:31:08.823625+0000 mon.a (mon.0) 2889 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:31:09.414 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:09 smithi136 bash[75164]: cluster 2024-09-13T00:31:09.109799+0000 mon.a (mon.0) 2890 : cluster [DBG] osdmap e1579: 8 total, 8 up, 8 in 2024-09-13T00:31:09.414 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:09 smithi136 bash[76371]: cluster 2024-09-13T00:31:08.038868+0000 mgr.y (mgr.24661) 1620 : cluster [DBG] pgmap v2758: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 76 KiB/s rd, 5.5 KiB/s wr, 111 op/s 2024-09-13T00:31:09.414 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:09 smithi136 bash[76371]: cluster 2024-09-13T00:31:08.133445+0000 mon.a (mon.0) 2886 : cluster [DBG] osdmap e1578: 8 total, 8 up, 8 in 2024-09-13T00:31:09.414 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:09 smithi136 bash[76371]: audit 2024-09-13T00:31:08.256713+0000 mon.c (mon.2) 113 : audit [INF] from='client.? 172.21.15.136:0/2010575863' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/6793075f-d3a7-415b-a32e-e8c0c9d49c28"}]: dispatch 2024-09-13T00:31:09.414 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:09 smithi136 bash[76371]: audit 2024-09-13T00:31:08.257429+0000 mon.a (mon.0) 2887 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/6793075f-d3a7-415b-a32e-e8c0c9d49c28"}]: dispatch 2024-09-13T00:31:09.414 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:09 smithi136 bash[76371]: audit 2024-09-13T00:31:08.264301+0000 mon.a (mon.0) 2888 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/6793075f-d3a7-415b-a32e-e8c0c9d49c28"}]': finished 2024-09-13T00:31:09.414 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:09 smithi136 bash[76371]: audit 2024-09-13T00:31:08.823625+0000 mon.a (mon.0) 2889 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:31:09.414 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:09 smithi136 bash[76371]: cluster 2024-09-13T00:31:09.109799+0000 mon.a (mon.0) 2890 : cluster [DBG] osdmap e1579: 8 total, 8 up, 8 in 2024-09-13T00:31:09.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:09 smithi181 bash[67587]: cluster 2024-09-13T00:31:08.038868+0000 mgr.y (mgr.24661) 1620 : cluster [DBG] pgmap v2758: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 76 KiB/s rd, 5.5 KiB/s wr, 111 op/s 2024-09-13T00:31:09.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:09 smithi181 bash[67587]: cluster 2024-09-13T00:31:08.133445+0000 mon.a (mon.0) 2886 : cluster [DBG] osdmap e1578: 8 total, 8 up, 8 in 2024-09-13T00:31:09.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:09 smithi181 bash[67587]: audit 2024-09-13T00:31:08.256713+0000 mon.c (mon.2) 113 : audit [INF] from='client.? 172.21.15.136:0/2010575863' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/6793075f-d3a7-415b-a32e-e8c0c9d49c28"}]: dispatch 2024-09-13T00:31:09.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:09 smithi181 bash[67587]: audit 2024-09-13T00:31:08.257429+0000 mon.a (mon.0) 2887 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/6793075f-d3a7-415b-a32e-e8c0c9d49c28"}]: dispatch 2024-09-13T00:31:09.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:09 smithi181 bash[67587]: audit 2024-09-13T00:31:08.264301+0000 mon.a (mon.0) 2888 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/87/6793075f-d3a7-415b-a32e-e8c0c9d49c28"}]': finished 2024-09-13T00:31:09.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:09 smithi181 bash[67587]: audit 2024-09-13T00:31:08.823625+0000 mon.a (mon.0) 2889 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:31:09.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:09 smithi181 bash[67587]: cluster 2024-09-13T00:31:09.109799+0000 mon.a (mon.0) 2890 : cluster [DBG] osdmap e1579: 8 total, 8 up, 8 in 2024-09-13T00:31:09.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:31:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:31:09] "GET /metrics HTTP/1.1" 200 36540 "" "Prometheus/2.43.0" 2024-09-13T00:31:11.167 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestInternal.OpenByID 2024-09-13T00:31:11.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:11 smithi136 bash[75164]: cluster 2024-09-13T00:31:10.039536+0000 mgr.y (mgr.24661) 1621 : cluster [DBG] pgmap v2761: 105 pgs: 105 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:31:11.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:11 smithi136 bash[75164]: cluster 2024-09-13T00:31:10.157867+0000 mon.a (mon.0) 2891 : cluster [DBG] osdmap e1580: 8 total, 8 up, 8 in 2024-09-13T00:31:11.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:11 smithi136 bash[75164]: audit 2024-09-13T00:31:10.161622+0000 mon.a (mon.0) 2892 : audit [INF] from='client.? 172.21.15.136:0/2085910915' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-12","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:31:11.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:11 smithi136 bash[76371]: cluster 2024-09-13T00:31:10.039536+0000 mgr.y (mgr.24661) 1621 : cluster [DBG] pgmap v2761: 105 pgs: 105 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:31:11.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:11 smithi136 bash[76371]: cluster 2024-09-13T00:31:10.157867+0000 mon.a (mon.0) 2891 : cluster [DBG] osdmap e1580: 8 total, 8 up, 8 in 2024-09-13T00:31:11.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:11 smithi136 bash[76371]: audit 2024-09-13T00:31:10.161622+0000 mon.a (mon.0) 2892 : audit [INF] from='client.? 172.21.15.136:0/2085910915' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-12","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:31:11.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:11 smithi181 bash[67587]: cluster 2024-09-13T00:31:10.039536+0000 mgr.y (mgr.24661) 1621 : cluster [DBG] pgmap v2761: 105 pgs: 105 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:31:11.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:11 smithi181 bash[67587]: cluster 2024-09-13T00:31:10.157867+0000 mon.a (mon.0) 2891 : cluster [DBG] osdmap e1580: 8 total, 8 up, 8 in 2024-09-13T00:31:11.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:11 smithi181 bash[67587]: audit 2024-09-13T00:31:10.161622+0000 mon.a (mon.0) 2892 : audit [INF] from='client.? 172.21.15.136:0/2085910915' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-12","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:31:12.512 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:12 smithi136 bash[76371]: audit 2024-09-13T00:31:11.163151+0000 mon.a (mon.0) 2893 : audit [INF] from='client.? 172.21.15.136:0/2085910915' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-12","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:31:12.512 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:12 smithi136 bash[76371]: cluster 2024-09-13T00:31:11.163688+0000 mon.a (mon.0) 2894 : cluster [DBG] osdmap e1581: 8 total, 8 up, 8 in 2024-09-13T00:31:12.513 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:12 smithi136 bash[75164]: audit 2024-09-13T00:31:11.163151+0000 mon.a (mon.0) 2893 : audit [INF] from='client.? 172.21.15.136:0/2085910915' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-12","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:31:12.513 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:12 smithi136 bash[75164]: cluster 2024-09-13T00:31:11.163688+0000 mon.a (mon.0) 2894 : cluster [DBG] osdmap e1581: 8 total, 8 up, 8 in 2024-09-13T00:31:12.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:12 smithi181 bash[67587]: audit 2024-09-13T00:31:11.163151+0000 mon.a (mon.0) 2893 : audit [INF] from='client.? 172.21.15.136:0/2085910915' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-12","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:31:12.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:12 smithi181 bash[67587]: cluster 2024-09-13T00:31:11.163688+0000 mon.a (mon.0) 2894 : cluster [DBG] osdmap e1581: 8 total, 8 up, 8 in 2024-09-13T00:31:12.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:31:12 smithi181 bash[95505]: ts=2024-09-13T00:31:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:31:13.279 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestInternal.OpenByID (2112 ms) 2024-09-13T00:31:13.279 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestInternal.OpenSnapDNE 2024-09-13T00:31:13.332 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestInternal.OpenSnapDNE (53 ms) 2024-09-13T00:31:13.332 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestInternal.IsExclusiveLockOwner 2024-09-13T00:31:13.385 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestInternal.IsExclusiveLockOwner (53 ms) 2024-09-13T00:31:13.385 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestInternal.ResizeLocksImage 2024-09-13T00:31:13.451 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestInternal.ResizeLocksImage (66 ms) 2024-09-13T00:31:13.451 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestInternal.ResizeFailsToLockImage 2024-09-13T00:31:13.498 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestInternal.ResizeFailsToLockImage (47 ms) 2024-09-13T00:31:13.498 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestInternal.SnapCreateLocksImage 2024-09-13T00:31:13.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:13 smithi136 bash[76371]: cluster 2024-09-13T00:31:12.040460+0000 mgr.y (mgr.24661) 1622 : cluster [DBG] pgmap v2764: 137 pgs: 6 creating+peering, 15 unknown, 116 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:31:13.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:13 smithi136 bash[76371]: cluster 2024-09-13T00:31:12.201453+0000 mon.a (mon.0) 2895 : cluster [DBG] osdmap e1582: 8 total, 8 up, 8 in 2024-09-13T00:31:13.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:13 smithi136 bash[75164]: cluster 2024-09-13T00:31:12.040460+0000 mgr.y (mgr.24661) 1622 : cluster [DBG] pgmap v2764: 137 pgs: 6 creating+peering, 15 unknown, 116 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:31:13.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:13 smithi136 bash[75164]: cluster 2024-09-13T00:31:12.201453+0000 mon.a (mon.0) 2895 : cluster [DBG] osdmap e1582: 8 total, 8 up, 8 in 2024-09-13T00:31:13.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:13 smithi181 bash[67587]: cluster 2024-09-13T00:31:12.040460+0000 mgr.y (mgr.24661) 1622 : cluster [DBG] pgmap v2764: 137 pgs: 6 creating+peering, 15 unknown, 116 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:31:13.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:13 smithi181 bash[67587]: cluster 2024-09-13T00:31:12.201453+0000 mon.a (mon.0) 2895 : cluster [DBG] osdmap e1582: 8 total, 8 up, 8 in 2024-09-13T00:31:14.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:14 smithi136 bash[75164]: cluster 2024-09-13T00:31:13.203043+0000 mon.a (mon.0) 2896 : cluster [DBG] osdmap e1583: 8 total, 8 up, 8 in 2024-09-13T00:31:14.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:14 smithi136 bash[76371]: cluster 2024-09-13T00:31:13.203043+0000 mon.a (mon.0) 2896 : cluster [DBG] osdmap e1583: 8 total, 8 up, 8 in 2024-09-13T00:31:14.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:14 smithi181 bash[67587]: cluster 2024-09-13T00:31:13.203043+0000 mon.a (mon.0) 2896 : cluster [DBG] osdmap e1583: 8 total, 8 up, 8 in 2024-09-13T00:31:15.263 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestInternal.SnapCreateLocksImage (1766 ms) 2024-09-13T00:31:15.264 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestInternal.SnapCreateFailsToLockImage 2024-09-13T00:31:15.308 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestInternal.SnapCreateFailsToLockImage (45 ms) 2024-09-13T00:31:15.308 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestInternal.SnapRollbackLocksImage 2024-09-13T00:31:15.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:15 smithi136 bash[75164]: cluster 2024-09-13T00:31:14.041315+0000 mgr.y (mgr.24661) 1623 : cluster [DBG] pgmap v2767: 137 pgs: 6 creating+peering, 1 unknown, 130 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 511 B/s wr, 1 op/s 2024-09-13T00:31:15.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:15 smithi136 bash[75164]: cluster 2024-09-13T00:31:14.218415+0000 mon.a (mon.0) 2897 : cluster [DBG] osdmap e1584: 8 total, 8 up, 8 in 2024-09-13T00:31:15.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:15 smithi136 bash[76371]: cluster 2024-09-13T00:31:14.041315+0000 mgr.y (mgr.24661) 1623 : cluster [DBG] pgmap v2767: 137 pgs: 6 creating+peering, 1 unknown, 130 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 511 B/s wr, 1 op/s 2024-09-13T00:31:15.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:15 smithi136 bash[76371]: cluster 2024-09-13T00:31:14.218415+0000 mon.a (mon.0) 2897 : cluster [DBG] osdmap e1584: 8 total, 8 up, 8 in 2024-09-13T00:31:15.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:15 smithi181 bash[67587]: cluster 2024-09-13T00:31:14.041315+0000 mgr.y (mgr.24661) 1623 : cluster [DBG] pgmap v2767: 137 pgs: 6 creating+peering, 1 unknown, 130 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 511 B/s wr, 1 op/s 2024-09-13T00:31:15.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:15 smithi181 bash[67587]: cluster 2024-09-13T00:31:14.218415+0000 mon.a (mon.0) 2897 : cluster [DBG] osdmap e1584: 8 total, 8 up, 8 in 2024-09-13T00:31:16.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:16 smithi136 bash[75164]: cluster 2024-09-13T00:31:15.228045+0000 mon.a (mon.0) 2898 : cluster [DBG] osdmap e1585: 8 total, 8 up, 8 in 2024-09-13T00:31:16.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:16 smithi136 bash[76371]: cluster 2024-09-13T00:31:15.228045+0000 mon.a (mon.0) 2898 : cluster [DBG] osdmap e1585: 8 total, 8 up, 8 in 2024-09-13T00:31:16.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:16 smithi181 bash[67587]: cluster 2024-09-13T00:31:15.228045+0000 mon.a (mon.0) 2898 : cluster [DBG] osdmap e1585: 8 total, 8 up, 8 in 2024-09-13T00:31:17.291 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestInternal.SnapRollbackLocksImage (1982 ms) 2024-09-13T00:31:17.291 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestInternal.SnapRollbackFailsToLockImage 2024-09-13T00:31:17.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:17 smithi136 bash[75164]: cluster 2024-09-13T00:31:16.042245+0000 mgr.y (mgr.24661) 1624 : cluster [DBG] pgmap v2770: 137 pgs: 3 creating+peering, 134 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 1.7 KiB/s wr, 13 op/s 2024-09-13T00:31:17.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:17 smithi136 bash[75164]: cluster 2024-09-13T00:31:16.236693+0000 mon.a (mon.0) 2899 : cluster [DBG] osdmap e1586: 8 total, 8 up, 8 in 2024-09-13T00:31:17.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:17 smithi136 bash[76371]: cluster 2024-09-13T00:31:16.042245+0000 mgr.y (mgr.24661) 1624 : cluster [DBG] pgmap v2770: 137 pgs: 3 creating+peering, 134 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 1.7 KiB/s wr, 13 op/s 2024-09-13T00:31:17.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:17 smithi136 bash[76371]: cluster 2024-09-13T00:31:16.236693+0000 mon.a (mon.0) 2899 : cluster [DBG] osdmap e1586: 8 total, 8 up, 8 in 2024-09-13T00:31:17.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:17 smithi181 bash[67587]: cluster 2024-09-13T00:31:16.042245+0000 mgr.y (mgr.24661) 1624 : cluster [DBG] pgmap v2770: 137 pgs: 3 creating+peering, 134 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 1.7 KiB/s wr, 13 op/s 2024-09-13T00:31:17.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:17 smithi181 bash[67587]: cluster 2024-09-13T00:31:16.236693+0000 mon.a (mon.0) 2899 : cluster [DBG] osdmap e1586: 8 total, 8 up, 8 in 2024-09-13T00:31:18.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:18 smithi136 bash[75164]: cluster 2024-09-13T00:31:17.247563+0000 mon.a (mon.0) 2900 : cluster [DBG] osdmap e1587: 8 total, 8 up, 8 in 2024-09-13T00:31:18.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:18 smithi136 bash[76371]: cluster 2024-09-13T00:31:17.247563+0000 mon.a (mon.0) 2900 : cluster [DBG] osdmap e1587: 8 total, 8 up, 8 in 2024-09-13T00:31:18.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:18 smithi181 bash[67587]: cluster 2024-09-13T00:31:17.247563+0000 mon.a (mon.0) 2900 : cluster [DBG] osdmap e1587: 8 total, 8 up, 8 in 2024-09-13T00:31:19.148 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestInternal.SnapRollbackFailsToLockImage (1858 ms) 2024-09-13T00:31:19.148 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestInternal.SnapSetReleasesLock 2024-09-13T00:31:19.509 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:31:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:31:19] "GET /metrics HTTP/1.1" 200 36540 "" "Prometheus/2.43.0" 2024-09-13T00:31:19.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:19 smithi136 bash[76371]: cluster 2024-09-13T00:31:18.043253+0000 mgr.y (mgr.24661) 1625 : cluster [DBG] pgmap v2773: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 107 KiB/s rd, 12 KiB/s wr, 152 op/s 2024-09-13T00:31:19.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:19 smithi136 bash[76371]: cluster 2024-09-13T00:31:18.255881+0000 mon.a (mon.0) 2901 : cluster [DBG] osdmap e1588: 8 total, 8 up, 8 in 2024-09-13T00:31:19.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:19 smithi136 bash[76371]: cluster 2024-09-13T00:31:19.114500+0000 mon.a (mon.0) 2902 : cluster [DBG] osdmap e1589: 8 total, 8 up, 8 in 2024-09-13T00:31:19.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:19 smithi136 bash[75164]: cluster 2024-09-13T00:31:18.043253+0000 mgr.y (mgr.24661) 1625 : cluster [DBG] pgmap v2773: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 107 KiB/s rd, 12 KiB/s wr, 152 op/s 2024-09-13T00:31:19.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:19 smithi136 bash[75164]: cluster 2024-09-13T00:31:18.255881+0000 mon.a (mon.0) 2901 : cluster [DBG] osdmap e1588: 8 total, 8 up, 8 in 2024-09-13T00:31:19.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:19 smithi136 bash[75164]: cluster 2024-09-13T00:31:19.114500+0000 mon.a (mon.0) 2902 : cluster [DBG] osdmap e1589: 8 total, 8 up, 8 in 2024-09-13T00:31:19.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:19 smithi181 bash[67587]: cluster 2024-09-13T00:31:18.043253+0000 mgr.y (mgr.24661) 1625 : cluster [DBG] pgmap v2773: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 107 KiB/s rd, 12 KiB/s wr, 152 op/s 2024-09-13T00:31:19.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:19 smithi181 bash[67587]: cluster 2024-09-13T00:31:18.255881+0000 mon.a (mon.0) 2901 : cluster [DBG] osdmap e1588: 8 total, 8 up, 8 in 2024-09-13T00:31:19.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:19 smithi181 bash[67587]: cluster 2024-09-13T00:31:19.114500+0000 mon.a (mon.0) 2902 : cluster [DBG] osdmap e1589: 8 total, 8 up, 8 in 2024-09-13T00:31:21.172 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestInternal.SnapSetReleasesLock (2023 ms) 2024-09-13T00:31:21.172 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestInternal.FlattenLocksImage 2024-09-13T00:31:21.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:21 smithi136 bash[75164]: cluster 2024-09-13T00:31:20.044102+0000 mgr.y (mgr.24661) 1626 : cluster [DBG] pgmap v2776: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 114 KiB/s rd, 12 KiB/s wr, 163 op/s 2024-09-13T00:31:21.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:21 smithi136 bash[75164]: cluster 2024-09-13T00:31:20.116792+0000 mon.a (mon.0) 2903 : cluster [DBG] osdmap e1590: 8 total, 8 up, 8 in 2024-09-13T00:31:21.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:21 smithi136 bash[76371]: cluster 2024-09-13T00:31:20.044102+0000 mgr.y (mgr.24661) 1626 : cluster [DBG] pgmap v2776: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 114 KiB/s rd, 12 KiB/s wr, 163 op/s 2024-09-13T00:31:21.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:21 smithi136 bash[76371]: cluster 2024-09-13T00:31:20.116792+0000 mon.a (mon.0) 2903 : cluster [DBG] osdmap e1590: 8 total, 8 up, 8 in 2024-09-13T00:31:21.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:21 smithi181 bash[67587]: cluster 2024-09-13T00:31:20.044102+0000 mgr.y (mgr.24661) 1626 : cluster [DBG] pgmap v2776: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 114 KiB/s rd, 12 KiB/s wr, 163 op/s 2024-09-13T00:31:21.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:21 smithi181 bash[67587]: cluster 2024-09-13T00:31:20.116792+0000 mon.a (mon.0) 2903 : cluster [DBG] osdmap e1590: 8 total, 8 up, 8 in 2024-09-13T00:31:22.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:22 smithi136 bash[75164]: cluster 2024-09-13T00:31:21.126823+0000 mon.a (mon.0) 2904 : cluster [DBG] osdmap e1591: 8 total, 8 up, 8 in 2024-09-13T00:31:22.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:22 smithi136 bash[76371]: cluster 2024-09-13T00:31:21.126823+0000 mon.a (mon.0) 2904 : cluster [DBG] osdmap e1591: 8 total, 8 up, 8 in 2024-09-13T00:31:22.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:22 smithi181 bash[67587]: cluster 2024-09-13T00:31:21.126823+0000 mon.a (mon.0) 2904 : cluster [DBG] osdmap e1591: 8 total, 8 up, 8 in 2024-09-13T00:31:22.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:31:22 smithi181 bash[95505]: ts=2024-09-13T00:31:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:31:23.196 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestInternal.FlattenLocksImage (2023 ms) 2024-09-13T00:31:23.196 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestInternal.FlattenFailsToLockImage 2024-09-13T00:31:23.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:23 smithi136 bash[75164]: cluster 2024-09-13T00:31:22.045143+0000 mgr.y (mgr.24661) 1627 : cluster [DBG] pgmap v2779: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 22 KiB/s rd, 2.5 KiB/s wr, 38 op/s 2024-09-13T00:31:23.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:23 smithi136 bash[75164]: cluster 2024-09-13T00:31:22.141540+0000 mon.a (mon.0) 2905 : cluster [DBG] osdmap e1592: 8 total, 8 up, 8 in 2024-09-13T00:31:23.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:23 smithi136 bash[76371]: cluster 2024-09-13T00:31:22.045143+0000 mgr.y (mgr.24661) 1627 : cluster [DBG] pgmap v2779: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 22 KiB/s rd, 2.5 KiB/s wr, 38 op/s 2024-09-13T00:31:23.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:23 smithi136 bash[76371]: cluster 2024-09-13T00:31:22.141540+0000 mon.a (mon.0) 2905 : cluster [DBG] osdmap e1592: 8 total, 8 up, 8 in 2024-09-13T00:31:23.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:23 smithi181 bash[67587]: cluster 2024-09-13T00:31:22.045143+0000 mgr.y (mgr.24661) 1627 : cluster [DBG] pgmap v2779: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 22 KiB/s rd, 2.5 KiB/s wr, 38 op/s 2024-09-13T00:31:23.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:23 smithi181 bash[67587]: cluster 2024-09-13T00:31:22.141540+0000 mon.a (mon.0) 2905 : cluster [DBG] osdmap e1592: 8 total, 8 up, 8 in 2024-09-13T00:31:24.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:24 smithi136 bash[75164]: cluster 2024-09-13T00:31:23.147303+0000 mon.a (mon.0) 2906 : cluster [DBG] osdmap e1593: 8 total, 8 up, 8 in 2024-09-13T00:31:24.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:24 smithi136 bash[75164]: audit 2024-09-13T00:31:23.205860+0000 mon.a (mon.0) 2907 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:31:24.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:24 smithi136 bash[75164]: audit 2024-09-13T00:31:23.823857+0000 mon.a (mon.0) 2908 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:31:24.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:24 smithi136 bash[75164]: cluster 2024-09-13T00:31:24.116452+0000 mon.a (mon.0) 2909 : cluster [DBG] osdmap e1594: 8 total, 8 up, 8 in 2024-09-13T00:31:24.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:24 smithi136 bash[76371]: cluster 2024-09-13T00:31:23.147303+0000 mon.a (mon.0) 2906 : cluster [DBG] osdmap e1593: 8 total, 8 up, 8 in 2024-09-13T00:31:24.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:24 smithi136 bash[76371]: audit 2024-09-13T00:31:23.205860+0000 mon.a (mon.0) 2907 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:31:24.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:24 smithi136 bash[76371]: audit 2024-09-13T00:31:23.823857+0000 mon.a (mon.0) 2908 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:31:24.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:24 smithi136 bash[76371]: cluster 2024-09-13T00:31:24.116452+0000 mon.a (mon.0) 2909 : cluster [DBG] osdmap e1594: 8 total, 8 up, 8 in 2024-09-13T00:31:24.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:24 smithi181 bash[67587]: cluster 2024-09-13T00:31:23.147303+0000 mon.a (mon.0) 2906 : cluster [DBG] osdmap e1593: 8 total, 8 up, 8 in 2024-09-13T00:31:24.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:24 smithi181 bash[67587]: audit 2024-09-13T00:31:23.205860+0000 mon.a (mon.0) 2907 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:31:24.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:24 smithi181 bash[67587]: audit 2024-09-13T00:31:23.823857+0000 mon.a (mon.0) 2908 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:31:24.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:24 smithi181 bash[67587]: cluster 2024-09-13T00:31:24.116452+0000 mon.a (mon.0) 2909 : cluster [DBG] osdmap e1594: 8 total, 8 up, 8 in 2024-09-13T00:31:25.213 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestInternal.FlattenFailsToLockImage (2017 ms) 2024-09-13T00:31:25.213 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestInternal.WriteFailsToLockImageBlocklisted 2024-09-13T00:31:25.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:25 smithi136 bash[75164]: cluster 2024-09-13T00:31:24.046040+0000 mgr.y (mgr.24661) 1628 : cluster [DBG] pgmap v2782: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 100 KiB/s rd, 7.0 KiB/s wr, 135 op/s 2024-09-13T00:31:25.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:25 smithi136 bash[76371]: cluster 2024-09-13T00:31:24.046040+0000 mgr.y (mgr.24661) 1628 : cluster [DBG] pgmap v2782: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 100 KiB/s rd, 7.0 KiB/s wr, 135 op/s 2024-09-13T00:31:25.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:25 smithi181 bash[67587]: cluster 2024-09-13T00:31:24.046040+0000 mgr.y (mgr.24661) 1628 : cluster [DBG] pgmap v2782: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 100 KiB/s rd, 7.0 KiB/s wr, 135 op/s 2024-09-13T00:31:26.200 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestInternal.WriteFailsToLockImageBlocklisted (987 ms) 2024-09-13T00:31:26.200 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestInternal.WriteFailsToLockImageBlocklistedWatch 2024-09-13T00:31:26.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:26 smithi136 bash[75164]: cluster 2024-09-13T00:31:25.161431+0000 mon.a (mon.0) 2910 : cluster [DBG] osdmap e1595: 8 total, 8 up, 8 in 2024-09-13T00:31:26.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:26 smithi136 bash[75164]: audit 2024-09-13T00:31:25.304645+0000 mon.a (mon.0) 2911 : audit [INF] from='client.? 172.21.15.136:0/895274419' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.136:0/895274419"}]: dispatch 2024-09-13T00:31:26.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:26 smithi136 bash[76371]: cluster 2024-09-13T00:31:25.161431+0000 mon.a (mon.0) 2910 : cluster [DBG] osdmap e1595: 8 total, 8 up, 8 in 2024-09-13T00:31:26.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:26 smithi136 bash[76371]: audit 2024-09-13T00:31:25.304645+0000 mon.a (mon.0) 2911 : audit [INF] from='client.? 172.21.15.136:0/895274419' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.136:0/895274419"}]: dispatch 2024-09-13T00:31:26.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:26 smithi181 bash[67587]: cluster 2024-09-13T00:31:25.161431+0000 mon.a (mon.0) 2910 : cluster [DBG] osdmap e1595: 8 total, 8 up, 8 in 2024-09-13T00:31:26.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:26 smithi181 bash[67587]: audit 2024-09-13T00:31:25.304645+0000 mon.a (mon.0) 2911 : audit [INF] from='client.? 172.21.15.136:0/895274419' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.136:0/895274419"}]: dispatch 2024-09-13T00:31:27.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:27 smithi136 bash[75164]: cluster 2024-09-13T00:31:26.046998+0000 mgr.y (mgr.24661) 1629 : cluster [DBG] pgmap v2785: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 113 KiB/s rd, 8.7 KiB/s wr, 152 op/s 2024-09-13T00:31:27.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:27 smithi136 bash[75164]: audit 2024-09-13T00:31:26.177606+0000 mon.a (mon.0) 2912 : audit [INF] from='client.? 172.21.15.136:0/895274419' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.136:0/895274419"}]': finished 2024-09-13T00:31:27.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:27 smithi136 bash[75164]: cluster 2024-09-13T00:31:26.177758+0000 mon.a (mon.0) 2913 : cluster [DBG] osdmap e1596: 8 total, 8 up, 8 in 2024-09-13T00:31:27.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:27 smithi136 bash[75164]: audit 2024-09-13T00:31:26.284335+0000 mon.a (mon.0) 2914 : audit [INF] from='client.? 172.21.15.136:0/2437398169' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.136:0/2437398169"}]: dispatch 2024-09-13T00:31:27.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:27 smithi136 bash[76371]: cluster 2024-09-13T00:31:26.046998+0000 mgr.y (mgr.24661) 1629 : cluster [DBG] pgmap v2785: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 113 KiB/s rd, 8.7 KiB/s wr, 152 op/s 2024-09-13T00:31:27.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:27 smithi136 bash[76371]: audit 2024-09-13T00:31:26.177606+0000 mon.a (mon.0) 2912 : audit [INF] from='client.? 172.21.15.136:0/895274419' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.136:0/895274419"}]': finished 2024-09-13T00:31:27.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:27 smithi136 bash[76371]: cluster 2024-09-13T00:31:26.177758+0000 mon.a (mon.0) 2913 : cluster [DBG] osdmap e1596: 8 total, 8 up, 8 in 2024-09-13T00:31:27.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:27 smithi136 bash[76371]: audit 2024-09-13T00:31:26.284335+0000 mon.a (mon.0) 2914 : audit [INF] from='client.? 172.21.15.136:0/2437398169' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.136:0/2437398169"}]: dispatch 2024-09-13T00:31:27.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:27 smithi181 bash[67587]: cluster 2024-09-13T00:31:26.046998+0000 mgr.y (mgr.24661) 1629 : cluster [DBG] pgmap v2785: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 113 KiB/s rd, 8.7 KiB/s wr, 152 op/s 2024-09-13T00:31:27.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:27 smithi181 bash[67587]: audit 2024-09-13T00:31:26.177606+0000 mon.a (mon.0) 2912 : audit [INF] from='client.? 172.21.15.136:0/895274419' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.136:0/895274419"}]': finished 2024-09-13T00:31:27.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:27 smithi181 bash[67587]: cluster 2024-09-13T00:31:26.177758+0000 mon.a (mon.0) 2913 : cluster [DBG] osdmap e1596: 8 total, 8 up, 8 in 2024-09-13T00:31:27.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:27 smithi181 bash[67587]: audit 2024-09-13T00:31:26.284335+0000 mon.a (mon.0) 2914 : audit [INF] from='client.? 172.21.15.136:0/2437398169' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.136:0/2437398169"}]: dispatch 2024-09-13T00:31:28.205 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestInternal.WriteFailsToLockImageBlocklistedWatch (2005 ms) 2024-09-13T00:31:28.205 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestInternal.AioWriteRequestsLock 2024-09-13T00:31:28.240 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestInternal.AioWriteRequestsLock (35 ms) 2024-09-13T00:31:28.240 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestInternal.AioDiscardRequestsLock 2024-09-13T00:31:28.269 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestInternal.AioDiscardRequestsLock (30 ms) 2024-09-13T00:31:28.270 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestInternal.CancelAsyncResize 2024-09-13T00:31:28.322 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestInternal.CancelAsyncResize (52 ms) 2024-09-13T00:31:28.322 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestInternal.MultipleResize 2024-09-13T00:31:28.391 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestInternal.MultipleResize (70 ms) 2024-09-13T00:31:28.391 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestInternal.Metadata 2024-09-13T00:31:28.443 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestInternal.Metadata (51 ms) 2024-09-13T00:31:28.443 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestInternal.MetadataConfApply 2024-09-13T00:31:28.479 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestInternal.MetadataConfApply (36 ms) 2024-09-13T00:31:28.479 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestInternal.SnapshotCopyup 2024-09-13T00:31:28.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:28 smithi136 bash[75164]: audit 2024-09-13T00:31:27.198589+0000 mon.a (mon.0) 2915 : audit [INF] from='client.? 172.21.15.136:0/2437398169' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.136:0/2437398169"}]': finished 2024-09-13T00:31:28.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:28 smithi136 bash[75164]: cluster 2024-09-13T00:31:27.201697+0000 mon.a (mon.0) 2916 : cluster [DBG] osdmap e1597: 8 total, 8 up, 8 in 2024-09-13T00:31:28.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:28 smithi136 bash[76371]: audit 2024-09-13T00:31:27.198589+0000 mon.a (mon.0) 2915 : audit [INF] from='client.? 172.21.15.136:0/2437398169' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.136:0/2437398169"}]': finished 2024-09-13T00:31:28.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:28 smithi136 bash[76371]: cluster 2024-09-13T00:31:27.201697+0000 mon.a (mon.0) 2916 : cluster [DBG] osdmap e1597: 8 total, 8 up, 8 in 2024-09-13T00:31:28.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:28 smithi181 bash[67587]: audit 2024-09-13T00:31:27.198589+0000 mon.a (mon.0) 2915 : audit [INF] from='client.? 172.21.15.136:0/2437398169' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.136:0/2437398169"}]': finished 2024-09-13T00:31:28.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:28 smithi181 bash[67587]: cluster 2024-09-13T00:31:27.201697+0000 mon.a (mon.0) 2916 : cluster [DBG] osdmap e1597: 8 total, 8 up, 8 in 2024-09-13T00:31:29.509 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:31:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:31:29] "GET /metrics HTTP/1.1" 200 36537 "" "Prometheus/2.43.0" 2024-09-13T00:31:29.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:29 smithi136 bash[75164]: cluster 2024-09-13T00:31:28.047904+0000 mgr.y (mgr.24661) 1630 : cluster [DBG] pgmap v2788: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 132 KiB/s rd, 10 KiB/s wr, 182 op/s 2024-09-13T00:31:29.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:29 smithi136 bash[75164]: audit 2024-09-13T00:31:28.682159+0000 mon.a (mon.0) 2917 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:31:29.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:29 smithi136 bash[75164]: audit 2024-09-13T00:31:28.687900+0000 mon.a (mon.0) 2918 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:31:29.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:29 smithi136 bash[75164]: audit 2024-09-13T00:31:28.926424+0000 mon.a (mon.0) 2919 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:31:29.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:29 smithi136 bash[75164]: audit 2024-09-13T00:31:28.936068+0000 mon.a (mon.0) 2920 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:31:29.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:29 smithi136 bash[75164]: cluster 2024-09-13T00:31:29.121604+0000 mon.a (mon.0) 2921 : cluster [DBG] osdmap e1598: 8 total, 8 up, 8 in 2024-09-13T00:31:29.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:29 smithi136 bash[76371]: cluster 2024-09-13T00:31:28.047904+0000 mgr.y (mgr.24661) 1630 : cluster [DBG] pgmap v2788: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 132 KiB/s rd, 10 KiB/s wr, 182 op/s 2024-09-13T00:31:29.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:29 smithi136 bash[76371]: audit 2024-09-13T00:31:28.682159+0000 mon.a (mon.0) 2917 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:31:29.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:29 smithi136 bash[76371]: audit 2024-09-13T00:31:28.687900+0000 mon.a (mon.0) 2918 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:31:29.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:29 smithi136 bash[76371]: audit 2024-09-13T00:31:28.926424+0000 mon.a (mon.0) 2919 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:31:29.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:29 smithi136 bash[76371]: audit 2024-09-13T00:31:28.936068+0000 mon.a (mon.0) 2920 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:31:29.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:29 smithi136 bash[76371]: cluster 2024-09-13T00:31:29.121604+0000 mon.a (mon.0) 2921 : cluster [DBG] osdmap e1598: 8 total, 8 up, 8 in 2024-09-13T00:31:29.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:29 smithi181 bash[67587]: cluster 2024-09-13T00:31:28.047904+0000 mgr.y (mgr.24661) 1630 : cluster [DBG] pgmap v2788: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 132 KiB/s rd, 10 KiB/s wr, 182 op/s 2024-09-13T00:31:29.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:29 smithi181 bash[67587]: audit 2024-09-13T00:31:28.682159+0000 mon.a (mon.0) 2917 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:31:29.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:29 smithi181 bash[67587]: audit 2024-09-13T00:31:28.687900+0000 mon.a (mon.0) 2918 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:31:29.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:29 smithi181 bash[67587]: audit 2024-09-13T00:31:28.926424+0000 mon.a (mon.0) 2919 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:31:29.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:29 smithi181 bash[67587]: audit 2024-09-13T00:31:28.936068+0000 mon.a (mon.0) 2920 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:31:29.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:29 smithi181 bash[67587]: cluster 2024-09-13T00:31:29.121604+0000 mon.a (mon.0) 2921 : cluster [DBG] osdmap e1598: 8 total, 8 up, 8 in 2024-09-13T00:31:30.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:30 smithi136 bash[75164]: audit 2024-09-13T00:31:29.360429+0000 mon.a (mon.0) 2922 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:31:30.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:30 smithi136 bash[75164]: audit 2024-09-13T00:31:29.361555+0000 mon.a (mon.0) 2923 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:31:30.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:30 smithi136 bash[75164]: audit 2024-09-13T00:31:29.369752+0000 mon.a (mon.0) 2924 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:31:30.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:30 smithi136 bash[76371]: audit 2024-09-13T00:31:29.360429+0000 mon.a (mon.0) 2922 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:31:30.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:30 smithi136 bash[76371]: audit 2024-09-13T00:31:29.361555+0000 mon.a (mon.0) 2923 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:31:30.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:30 smithi136 bash[76371]: audit 2024-09-13T00:31:29.369752+0000 mon.a (mon.0) 2924 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:31:30.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:30 smithi181 bash[67587]: audit 2024-09-13T00:31:29.360429+0000 mon.a (mon.0) 2922 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:31:30.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:30 smithi181 bash[67587]: audit 2024-09-13T00:31:29.361555+0000 mon.a (mon.0) 2923 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:31:30.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:30 smithi181 bash[67587]: audit 2024-09-13T00:31:29.369752+0000 mon.a (mon.0) 2924 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:31:31.447 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestInternal.SnapshotCopyup (2968 ms) 2024-09-13T00:31:31.447 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestInternal.SnapshotCopyupZeros 2024-09-13T00:31:31.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:31 smithi136 bash[75164]: cluster 2024-09-13T00:31:30.048707+0000 mgr.y (mgr.24661) 1631 : cluster [DBG] pgmap v2790: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 102 KiB/s rd, 8.4 KiB/s wr, 147 op/s 2024-09-13T00:31:31.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:31 smithi136 bash[75164]: cluster 2024-09-13T00:31:30.224345+0000 mon.a (mon.0) 2925 : cluster [DBG] osdmap e1599: 8 total, 8 up, 8 in 2024-09-13T00:31:31.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:31 smithi136 bash[76371]: cluster 2024-09-13T00:31:30.048707+0000 mgr.y (mgr.24661) 1631 : cluster [DBG] pgmap v2790: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 102 KiB/s rd, 8.4 KiB/s wr, 147 op/s 2024-09-13T00:31:31.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:31 smithi136 bash[76371]: cluster 2024-09-13T00:31:30.224345+0000 mon.a (mon.0) 2925 : cluster [DBG] osdmap e1599: 8 total, 8 up, 8 in 2024-09-13T00:31:31.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:31 smithi181 bash[67587]: cluster 2024-09-13T00:31:30.048707+0000 mgr.y (mgr.24661) 1631 : cluster [DBG] pgmap v2790: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 102 KiB/s rd, 8.4 KiB/s wr, 147 op/s 2024-09-13T00:31:31.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:31 smithi181 bash[67587]: cluster 2024-09-13T00:31:30.224345+0000 mon.a (mon.0) 2925 : cluster [DBG] osdmap e1599: 8 total, 8 up, 8 in 2024-09-13T00:31:32.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:32 smithi136 bash[75164]: cluster 2024-09-13T00:31:31.244157+0000 mon.a (mon.0) 2926 : cluster [DBG] osdmap e1600: 8 total, 8 up, 8 in 2024-09-13T00:31:32.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:32 smithi136 bash[76371]: cluster 2024-09-13T00:31:31.244157+0000 mon.a (mon.0) 2926 : cluster [DBG] osdmap e1600: 8 total, 8 up, 8 in 2024-09-13T00:31:32.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:32 smithi181 bash[67587]: cluster 2024-09-13T00:31:31.244157+0000 mon.a (mon.0) 2926 : cluster [DBG] osdmap e1600: 8 total, 8 up, 8 in 2024-09-13T00:31:32.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:31:32 smithi181 bash[95505]: ts=2024-09-13T00:31:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:31:33.331 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestInternal.SnapshotCopyupZeros (1885 ms) 2024-09-13T00:31:33.332 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestInternal.SnapshotCopyupZerosMigration 2024-09-13T00:31:33.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:33 smithi136 bash[75164]: cluster 2024-09-13T00:31:32.049680+0000 mgr.y (mgr.24661) 1632 : cluster [DBG] pgmap v2793: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 146 KiB/s rd, 16 KiB/s wr, 207 op/s 2024-09-13T00:31:33.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:33 smithi136 bash[75164]: cluster 2024-09-13T00:31:32.260543+0000 mon.a (mon.0) 2927 : cluster [DBG] osdmap e1601: 8 total, 8 up, 8 in 2024-09-13T00:31:33.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:33 smithi136 bash[76371]: cluster 2024-09-13T00:31:32.049680+0000 mgr.y (mgr.24661) 1632 : cluster [DBG] pgmap v2793: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 146 KiB/s rd, 16 KiB/s wr, 207 op/s 2024-09-13T00:31:33.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:33 smithi136 bash[76371]: cluster 2024-09-13T00:31:32.260543+0000 mon.a (mon.0) 2927 : cluster [DBG] osdmap e1601: 8 total, 8 up, 8 in 2024-09-13T00:31:33.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:33 smithi181 bash[67587]: cluster 2024-09-13T00:31:32.049680+0000 mgr.y (mgr.24661) 1632 : cluster [DBG] pgmap v2793: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 146 KiB/s rd, 16 KiB/s wr, 207 op/s 2024-09-13T00:31:33.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:33 smithi181 bash[67587]: cluster 2024-09-13T00:31:32.260543+0000 mon.a (mon.0) 2927 : cluster [DBG] osdmap e1601: 8 total, 8 up, 8 in 2024-09-13T00:31:34.187 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestInternal.SnapshotCopyupZerosMigration (855 ms) 2024-09-13T00:31:34.187 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestInternal.ResizeCopyup 2024-09-13T00:31:34.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:34 smithi181 bash[67587]: cluster 2024-09-13T00:31:33.267002+0000 mon.a (mon.0) 2928 : cluster [DBG] osdmap e1602: 8 total, 8 up, 8 in 2024-09-13T00:31:34.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:34 smithi181 bash[67587]: cluster 2024-09-13T00:31:34.121883+0000 mon.a (mon.0) 2929 : cluster [DBG] osdmap e1603: 8 total, 8 up, 8 in 2024-09-13T00:31:34.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:34 smithi136 bash[75164]: cluster 2024-09-13T00:31:33.267002+0000 mon.a (mon.0) 2928 : cluster [DBG] osdmap e1602: 8 total, 8 up, 8 in 2024-09-13T00:31:34.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:34 smithi136 bash[75164]: cluster 2024-09-13T00:31:34.121883+0000 mon.a (mon.0) 2929 : cluster [DBG] osdmap e1603: 8 total, 8 up, 8 in 2024-09-13T00:31:34.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:34 smithi136 bash[76371]: cluster 2024-09-13T00:31:33.267002+0000 mon.a (mon.0) 2928 : cluster [DBG] osdmap e1602: 8 total, 8 up, 8 in 2024-09-13T00:31:34.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:34 smithi136 bash[76371]: cluster 2024-09-13T00:31:34.121883+0000 mon.a (mon.0) 2929 : cluster [DBG] osdmap e1603: 8 total, 8 up, 8 in 2024-09-13T00:31:35.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:35 smithi181 bash[67587]: cluster 2024-09-13T00:31:34.050580+0000 mgr.y (mgr.24661) 1633 : cluster [DBG] pgmap v2796: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 208 KiB/s rd, 26 KiB/s wr, 294 op/s 2024-09-13T00:31:35.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:35 smithi136 bash[75164]: cluster 2024-09-13T00:31:34.050580+0000 mgr.y (mgr.24661) 1633 : cluster [DBG] pgmap v2796: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 208 KiB/s rd, 26 KiB/s wr, 294 op/s 2024-09-13T00:31:35.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:35 smithi136 bash[76371]: cluster 2024-09-13T00:31:34.050580+0000 mgr.y (mgr.24661) 1633 : cluster [DBG] pgmap v2796: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 208 KiB/s rd, 26 KiB/s wr, 294 op/s 2024-09-13T00:31:36.418 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestInternal.ResizeCopyup (2231 ms) 2024-09-13T00:31:36.418 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestInternal.DiscardCopyup 2024-09-13T00:31:36.438 INFO:tasks.workunit.client.0.smithi136.stdout:SKIPPING 2024-09-13T00:31:36.438 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestInternal.DiscardCopyup (20 ms) 2024-09-13T00:31:36.438 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestInternal.ImageOptions 2024-09-13T00:31:36.458 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestInternal.ImageOptions (20 ms) 2024-09-13T00:31:36.458 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestInternal.WriteFullCopyup 2024-09-13T00:31:36.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:36 smithi181 bash[67587]: cluster 2024-09-13T00:31:35.290598+0000 mon.a (mon.0) 2930 : cluster [DBG] osdmap e1604: 8 total, 8 up, 8 in 2024-09-13T00:31:36.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:36 smithi136 bash[75164]: cluster 2024-09-13T00:31:35.290598+0000 mon.a (mon.0) 2930 : cluster [DBG] osdmap e1604: 8 total, 8 up, 8 in 2024-09-13T00:31:36.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:36 smithi136 bash[76371]: cluster 2024-09-13T00:31:35.290598+0000 mon.a (mon.0) 2930 : cluster [DBG] osdmap e1604: 8 total, 8 up, 8 in 2024-09-13T00:31:37.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:37 smithi181 bash[67587]: cluster 2024-09-13T00:31:36.051526+0000 mgr.y (mgr.24661) 1634 : cluster [DBG] pgmap v2799: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 176 KiB/s rd, 18 KiB/s wr, 243 op/s 2024-09-13T00:31:37.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:37 smithi181 bash[67587]: cluster 2024-09-13T00:31:36.298861+0000 mon.a (mon.0) 2931 : cluster [DBG] osdmap e1605: 8 total, 8 up, 8 in 2024-09-13T00:31:37.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:37 smithi136 bash[75164]: cluster 2024-09-13T00:31:36.051526+0000 mgr.y (mgr.24661) 1634 : cluster [DBG] pgmap v2799: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 176 KiB/s rd, 18 KiB/s wr, 243 op/s 2024-09-13T00:31:37.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:37 smithi136 bash[75164]: cluster 2024-09-13T00:31:36.298861+0000 mon.a (mon.0) 2931 : cluster [DBG] osdmap e1605: 8 total, 8 up, 8 in 2024-09-13T00:31:37.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:37 smithi136 bash[76371]: cluster 2024-09-13T00:31:36.051526+0000 mgr.y (mgr.24661) 1634 : cluster [DBG] pgmap v2799: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 176 KiB/s rd, 18 KiB/s wr, 243 op/s 2024-09-13T00:31:37.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:37 smithi136 bash[76371]: cluster 2024-09-13T00:31:36.298861+0000 mon.a (mon.0) 2931 : cluster [DBG] osdmap e1605: 8 total, 8 up, 8 in 2024-09-13T00:31:38.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:38 smithi181 bash[67587]: cluster 2024-09-13T00:31:37.302178+0000 mon.a (mon.0) 2932 : cluster [DBG] osdmap e1606: 8 total, 8 up, 8 in 2024-09-13T00:31:38.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:38 smithi136 bash[75164]: cluster 2024-09-13T00:31:37.302178+0000 mon.a (mon.0) 2932 : cluster [DBG] osdmap e1606: 8 total, 8 up, 8 in 2024-09-13T00:31:38.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:38 smithi136 bash[76371]: cluster 2024-09-13T00:31:37.302178+0000 mon.a (mon.0) 2932 : cluster [DBG] osdmap e1606: 8 total, 8 up, 8 in 2024-09-13T00:31:39.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:39 smithi181 bash[67587]: cluster 2024-09-13T00:31:38.052429+0000 mgr.y (mgr.24661) 1635 : cluster [DBG] pgmap v2802: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 205 KiB/s rd, 22 KiB/s wr, 277 op/s 2024-09-13T00:31:39.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:39 smithi181 bash[67587]: cluster 2024-09-13T00:31:38.330017+0000 mon.a (mon.0) 2933 : cluster [DBG] osdmap e1607: 8 total, 8 up, 8 in 2024-09-13T00:31:39.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:39 smithi181 bash[67587]: audit 2024-09-13T00:31:38.824248+0000 mon.a (mon.0) 2934 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:31:39.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:31:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:31:39] "GET /metrics HTTP/1.1" 200 36541 "" "Prometheus/2.43.0" 2024-09-13T00:31:39.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:39 smithi136 bash[75164]: cluster 2024-09-13T00:31:38.052429+0000 mgr.y (mgr.24661) 1635 : cluster [DBG] pgmap v2802: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 205 KiB/s rd, 22 KiB/s wr, 277 op/s 2024-09-13T00:31:39.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:39 smithi136 bash[75164]: cluster 2024-09-13T00:31:38.330017+0000 mon.a (mon.0) 2933 : cluster [DBG] osdmap e1607: 8 total, 8 up, 8 in 2024-09-13T00:31:39.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:39 smithi136 bash[75164]: audit 2024-09-13T00:31:38.824248+0000 mon.a (mon.0) 2934 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:31:39.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:39 smithi136 bash[76371]: cluster 2024-09-13T00:31:38.052429+0000 mgr.y (mgr.24661) 1635 : cluster [DBG] pgmap v2802: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 205 KiB/s rd, 22 KiB/s wr, 277 op/s 2024-09-13T00:31:39.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:39 smithi136 bash[76371]: cluster 2024-09-13T00:31:38.330017+0000 mon.a (mon.0) 2933 : cluster [DBG] osdmap e1607: 8 total, 8 up, 8 in 2024-09-13T00:31:39.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:39 smithi136 bash[76371]: audit 2024-09-13T00:31:38.824248+0000 mon.a (mon.0) 2934 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:31:40.389 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestInternal.WriteFullCopyup (3932 ms) 2024-09-13T00:31:40.389 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestInternal.DiffIterateCloneOverwrite 2024-09-13T00:31:40.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:40 smithi181 bash[67587]: cluster 2024-09-13T00:31:39.340726+0000 mon.a (mon.0) 2935 : cluster [DBG] osdmap e1608: 8 total, 8 up, 8 in 2024-09-13T00:31:40.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:40 smithi136 bash[75164]: cluster 2024-09-13T00:31:39.340726+0000 mon.a (mon.0) 2935 : cluster [DBG] osdmap e1608: 8 total, 8 up, 8 in 2024-09-13T00:31:40.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:40 smithi136 bash[76371]: cluster 2024-09-13T00:31:39.340726+0000 mon.a (mon.0) 2935 : cluster [DBG] osdmap e1608: 8 total, 8 up, 8 in 2024-09-13T00:31:41.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:41 smithi181 bash[67587]: cluster 2024-09-13T00:31:40.053228+0000 mgr.y (mgr.24661) 1636 : cluster [DBG] pgmap v2805: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 170 KiB/s rd, 24 KiB/s wr, 237 op/s 2024-09-13T00:31:41.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:41 smithi181 bash[67587]: cluster 2024-09-13T00:31:40.351190+0000 mon.a (mon.0) 2936 : cluster [DBG] osdmap e1609: 8 total, 8 up, 8 in 2024-09-13T00:31:41.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:41 smithi136 bash[75164]: cluster 2024-09-13T00:31:40.053228+0000 mgr.y (mgr.24661) 1636 : cluster [DBG] pgmap v2805: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 170 KiB/s rd, 24 KiB/s wr, 237 op/s 2024-09-13T00:31:41.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:41 smithi136 bash[75164]: cluster 2024-09-13T00:31:40.351190+0000 mon.a (mon.0) 2936 : cluster [DBG] osdmap e1609: 8 total, 8 up, 8 in 2024-09-13T00:31:41.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:41 smithi136 bash[76371]: cluster 2024-09-13T00:31:40.053228+0000 mgr.y (mgr.24661) 1636 : cluster [DBG] pgmap v2805: 137 pgs: 137 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 170 KiB/s rd, 24 KiB/s wr, 237 op/s 2024-09-13T00:31:41.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:41 smithi136 bash[76371]: cluster 2024-09-13T00:31:40.351190+0000 mon.a (mon.0) 2936 : cluster [DBG] osdmap e1609: 8 total, 8 up, 8 in 2024-09-13T00:31:42.423 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestInternal.DiffIterateCloneOverwrite (2033 ms) 2024-09-13T00:31:42.423 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestInternal.TestCoR 2024-09-13T00:31:42.446 INFO:tasks.workunit.client.0.smithi136.stdout:SKIPPING due to disabled rbd_copy_on_read 2024-09-13T00:31:42.446 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestInternal.TestCoR (23 ms) 2024-09-13T00:31:42.446 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestInternal.FlattenNoEmptyObjects 2024-09-13T00:31:42.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:42 smithi181 bash[67587]: cluster 2024-09-13T00:31:41.361799+0000 mon.a (mon.0) 2937 : cluster [DBG] osdmap e1610: 8 total, 8 up, 8 in 2024-09-13T00:31:42.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:31:42 smithi181 bash[95505]: ts=2024-09-13T00:31:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:31:42.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:42 smithi136 bash[76371]: cluster 2024-09-13T00:31:41.361799+0000 mon.a (mon.0) 2937 : cluster [DBG] osdmap e1610: 8 total, 8 up, 8 in 2024-09-13T00:31:42.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:42 smithi136 bash[75164]: cluster 2024-09-13T00:31:41.361799+0000 mon.a (mon.0) 2937 : cluster [DBG] osdmap e1610: 8 total, 8 up, 8 in 2024-09-13T00:31:43.016 INFO:tasks.workunit.client.0.smithi136.stdout:made parent image "image400": 4194304K (1024 * 4K) 2024-09-13T00:31:43.016 INFO:tasks.workunit.client.0.smithi136.stdout:generated random write map: 2024-09-13T00:31:43.016 INFO:tasks.workunit.client.0.smithi136.stdout: [11 , 47804 ] 2024-09-13T00:31:43.016 INFO:tasks.workunit.client.0.smithi136.stdout: [13 , 54194 ] 2024-09-13T00:31:43.017 INFO:tasks.workunit.client.0.smithi136.stdout: [47 , 194698 ] 2024-09-13T00:31:43.017 INFO:tasks.workunit.client.0.smithi136.stdout: [53 , 220261 ] 2024-09-13T00:31:43.017 INFO:tasks.workunit.client.0.smithi136.stdout: [68 , 279058 ] 2024-09-13T00:31:43.017 INFO:tasks.workunit.client.0.smithi136.stdout: [69 , 282840 ] 2024-09-13T00:31:43.017 INFO:tasks.workunit.client.0.smithi136.stdout: [76 , 313218 ] 2024-09-13T00:31:43.017 INFO:tasks.workunit.client.0.smithi136.stdout: [79 , 325406 ] 2024-09-13T00:31:43.017 INFO:tasks.workunit.client.0.smithi136.stdout: [86 , 354862 ] 2024-09-13T00:31:43.017 INFO:tasks.workunit.client.0.smithi136.stdout: [87 , 358895 ] 2024-09-13T00:31:43.017 INFO:tasks.workunit.client.0.smithi136.stdout: [96 , 394140 ] 2024-09-13T00:31:43.017 INFO:tasks.workunit.client.0.smithi136.stdout: [99 , 408626 ] 2024-09-13T00:31:43.017 INFO:tasks.workunit.client.0.smithi136.stdout: [117 , 482037 ] 2024-09-13T00:31:43.017 INFO:tasks.workunit.client.0.smithi136.stdout: [135 , 554564 ] 2024-09-13T00:31:43.017 INFO:tasks.workunit.client.0.smithi136.stdout: [136 , 560307 ] 2024-09-13T00:31:43.018 INFO:tasks.workunit.client.0.smithi136.stdout: [176 , 724241 ] 2024-09-13T00:31:43.018 INFO:tasks.workunit.client.0.smithi136.stdout: [197 , 808459 ] 2024-09-13T00:31:43.018 INFO:tasks.workunit.client.0.smithi136.stdout: [198 , 811813 ] 2024-09-13T00:31:43.018 INFO:tasks.workunit.client.0.smithi136.stdout: [209 , 857730 ] 2024-09-13T00:31:43.018 INFO:tasks.workunit.client.0.smithi136.stdout: [213 , 874853 ] 2024-09-13T00:31:43.018 INFO:tasks.workunit.client.0.smithi136.stdout: [216 , 887530 ] 2024-09-13T00:31:43.018 INFO:tasks.workunit.client.0.smithi136.stdout: [221 , 905675 ] 2024-09-13T00:31:43.018 INFO:tasks.workunit.client.0.smithi136.stdout: [224 , 920423 ] 2024-09-13T00:31:43.018 INFO:tasks.workunit.client.0.smithi136.stdout: [229 , 940385 ] 2024-09-13T00:31:43.018 INFO:tasks.workunit.client.0.smithi136.stdout: [232 , 952376 ] 2024-09-13T00:31:43.018 INFO:tasks.workunit.client.0.smithi136.stdout: [248 , 1019122 ] 2024-09-13T00:31:43.018 INFO:tasks.workunit.client.0.smithi136.stdout: [260 , 1067419 ] 2024-09-13T00:31:43.018 INFO:tasks.workunit.client.0.smithi136.stdout: [265 , 1086027 ] 2024-09-13T00:31:43.019 INFO:tasks.workunit.client.0.smithi136.stdout: [275 , 1127888 ] 2024-09-13T00:31:43.019 INFO:tasks.workunit.client.0.smithi136.stdout: [332 , 1360693 ] 2024-09-13T00:31:43.019 INFO:tasks.workunit.client.0.smithi136.stdout: [345 , 1415296 ] 2024-09-13T00:31:43.019 INFO:tasks.workunit.client.0.smithi136.stdout: [346 , 1420789 ] 2024-09-13T00:31:43.019 INFO:tasks.workunit.client.0.smithi136.stdout: [351 , 1441200 ] 2024-09-13T00:31:43.019 INFO:tasks.workunit.client.0.smithi136.stdout: [356 , 1459918 ] 2024-09-13T00:31:43.019 INFO:tasks.workunit.client.0.smithi136.stdout: [360 , 1476321 ] 2024-09-13T00:31:43.019 INFO:tasks.workunit.client.0.smithi136.stdout: [373 , 1529309 ] 2024-09-13T00:31:43.019 INFO:tasks.workunit.client.0.smithi136.stdout: [379 , 1553686 ] 2024-09-13T00:31:43.019 INFO:tasks.workunit.client.0.smithi136.stdout: [382 , 1567032 ] 2024-09-13T00:31:43.019 INFO:tasks.workunit.client.0.smithi136.stdout: [383 , 1570796 ] 2024-09-13T00:31:43.019 INFO:tasks.workunit.client.0.smithi136.stdout: [420 , 1723623 ] 2024-09-13T00:31:43.019 INFO:tasks.workunit.client.0.smithi136.stdout: [423 , 1734637 ] 2024-09-13T00:31:43.019 INFO:tasks.workunit.client.0.smithi136.stdout: [425 , 1744011 ] 2024-09-13T00:31:43.020 INFO:tasks.workunit.client.0.smithi136.stdout: [431 , 1766732 ] 2024-09-13T00:31:43.020 INFO:tasks.workunit.client.0.smithi136.stdout: [433 , 1774494 ] 2024-09-13T00:31:43.020 INFO:tasks.workunit.client.0.smithi136.stdout: [440 , 1802583 ] 2024-09-13T00:31:43.020 INFO:tasks.workunit.client.0.smithi136.stdout: [455 , 1866168 ] 2024-09-13T00:31:43.020 INFO:tasks.workunit.client.0.smithi136.stdout: [458 , 1879392 ] 2024-09-13T00:31:43.020 INFO:tasks.workunit.client.0.smithi136.stdout: [461 , 1889201 ] 2024-09-13T00:31:43.020 INFO:tasks.workunit.client.0.smithi136.stdout: [462 , 1894297 ] 2024-09-13T00:31:43.020 INFO:tasks.workunit.client.0.smithi136.stdout: [468 , 1919074 ] 2024-09-13T00:31:43.020 INFO:tasks.workunit.client.0.smithi136.stdout: [477 , 1953937 ] 2024-09-13T00:31:43.020 INFO:tasks.workunit.client.0.smithi136.stdout: [485 , 1989065 ] 2024-09-13T00:31:43.020 INFO:tasks.workunit.client.0.smithi136.stdout: [489 , 2004500 ] 2024-09-13T00:31:43.020 INFO:tasks.workunit.client.0.smithi136.stdout: [509 , 2087511 ] 2024-09-13T00:31:43.020 INFO:tasks.workunit.client.0.smithi136.stdout: [512 , 2099930 ] 2024-09-13T00:31:43.020 INFO:tasks.workunit.client.0.smithi136.stdout: [519 , 2128038 ] 2024-09-13T00:31:43.021 INFO:tasks.workunit.client.0.smithi136.stdout: [542 , 2222269 ] 2024-09-13T00:31:43.021 INFO:tasks.workunit.client.0.smithi136.stdout: [548 , 2247444 ] 2024-09-13T00:31:43.021 INFO:tasks.workunit.client.0.smithi136.stdout: [576 , 2361589 ] 2024-09-13T00:31:43.021 INFO:tasks.workunit.client.0.smithi136.stdout: [581 , 2382351 ] 2024-09-13T00:31:43.021 INFO:tasks.workunit.client.0.smithi136.stdout: [582 , 2384836 ] 2024-09-13T00:31:43.021 INFO:tasks.workunit.client.0.smithi136.stdout: [589 , 2415959 ] 2024-09-13T00:31:43.021 INFO:tasks.workunit.client.0.smithi136.stdout: [590 , 2418697 ] 2024-09-13T00:31:43.021 INFO:tasks.workunit.client.0.smithi136.stdout: [600 , 2459560 ] 2024-09-13T00:31:43.021 INFO:tasks.workunit.client.0.smithi136.stdout: [605 , 2479663 ] 2024-09-13T00:31:43.021 INFO:tasks.workunit.client.0.smithi136.stdout: [607 , 2488528 ] 2024-09-13T00:31:43.021 INFO:tasks.workunit.client.0.smithi136.stdout: [622 , 2551021 ] 2024-09-13T00:31:43.021 INFO:tasks.workunit.client.0.smithi136.stdout: [625 , 2562938 ] 2024-09-13T00:31:43.021 INFO:tasks.workunit.client.0.smithi136.stdout: [669 , 2740892 ] 2024-09-13T00:31:43.021 INFO:tasks.workunit.client.0.smithi136.stdout: [682 , 2796281 ] 2024-09-13T00:31:43.022 INFO:tasks.workunit.client.0.smithi136.stdout: [695 , 2850233 ] 2024-09-13T00:31:43.022 INFO:tasks.workunit.client.0.smithi136.stdout: [703 , 2880250 ] 2024-09-13T00:31:43.022 INFO:tasks.workunit.client.0.smithi136.stdout: [707 , 2896793 ] 2024-09-13T00:31:43.022 INFO:tasks.workunit.client.0.smithi136.stdout: [719 , 2948575 ] 2024-09-13T00:31:43.022 INFO:tasks.workunit.client.0.smithi136.stdout: [726 , 2974249 ] 2024-09-13T00:31:43.022 INFO:tasks.workunit.client.0.smithi136.stdout: [750 , 3073238 ] 2024-09-13T00:31:43.022 INFO:tasks.workunit.client.0.smithi136.stdout: [776 , 3180132 ] 2024-09-13T00:31:43.022 INFO:tasks.workunit.client.0.smithi136.stdout: [777 , 3185238 ] 2024-09-13T00:31:43.022 INFO:tasks.workunit.client.0.smithi136.stdout: [798 , 3269673 ] 2024-09-13T00:31:43.022 INFO:tasks.workunit.client.0.smithi136.stdout: [807 , 3307813 ] 2024-09-13T00:31:43.022 INFO:tasks.workunit.client.0.smithi136.stdout: [825 , 3382743 ] 2024-09-13T00:31:43.022 INFO:tasks.workunit.client.0.smithi136.stdout: [849 , 3478903 ] 2024-09-13T00:31:43.022 INFO:tasks.workunit.client.0.smithi136.stdout: [893 , 3659374 ] 2024-09-13T00:31:43.022 INFO:tasks.workunit.client.0.smithi136.stdout: [911 , 3733364 ] 2024-09-13T00:31:43.023 INFO:tasks.workunit.client.0.smithi136.stdout: [925 , 3789862 ] 2024-09-13T00:31:43.023 INFO:tasks.workunit.client.0.smithi136.stdout: [936 , 3835637 ] 2024-09-13T00:31:43.023 INFO:tasks.workunit.client.0.smithi136.stdout: [962 , 3941079 ] 2024-09-13T00:31:43.023 INFO:tasks.workunit.client.0.smithi136.stdout: [965 , 3955609 ] 2024-09-13T00:31:43.023 INFO:tasks.workunit.client.0.smithi136.stdout: [970 , 3975325 ] 2024-09-13T00:31:43.023 INFO:tasks.workunit.client.0.smithi136.stdout: [986 , 4042039 ] 2024-09-13T00:31:43.023 INFO:tasks.workunit.client.0.smithi136.stdout: [988 , 4048883 ] 2024-09-13T00:31:43.023 INFO:tasks.workunit.client.0.smithi136.stdout: [1012 , 4148617 ] 2024-09-13T00:31:43.023 INFO:tasks.workunit.client.0.smithi136.stdout: [1020 , 4180521 ] 2024-09-13T00:31:43.023 INFO:tasks.workunit.client.0.smithi136.stdout:write data based on random map 2024-09-13T00:31:43.023 INFO:tasks.workunit.client.0.smithi136.stdout: write object-11 2024-09-13T00:31:43.023 INFO:tasks.workunit.client.0.smithi136.stdout: write object-13 2024-09-13T00:31:43.023 INFO:tasks.workunit.client.0.smithi136.stdout: write object-47 2024-09-13T00:31:43.023 INFO:tasks.workunit.client.0.smithi136.stdout: write object-53 2024-09-13T00:31:43.023 INFO:tasks.workunit.client.0.smithi136.stdout: write object-68 2024-09-13T00:31:43.024 INFO:tasks.workunit.client.0.smithi136.stdout: write object-69 2024-09-13T00:31:43.024 INFO:tasks.workunit.client.0.smithi136.stdout: write object-76 2024-09-13T00:31:43.024 INFO:tasks.workunit.client.0.smithi136.stdout: write object-79 2024-09-13T00:31:43.024 INFO:tasks.workunit.client.0.smithi136.stdout: write object-86 2024-09-13T00:31:43.024 INFO:tasks.workunit.client.0.smithi136.stdout: write object-87 2024-09-13T00:31:43.024 INFO:tasks.workunit.client.0.smithi136.stdout: write object-96 2024-09-13T00:31:43.024 INFO:tasks.workunit.client.0.smithi136.stdout: write object-99 2024-09-13T00:31:43.024 INFO:tasks.workunit.client.0.smithi136.stdout: write object-117 2024-09-13T00:31:43.024 INFO:tasks.workunit.client.0.smithi136.stdout: write object-135 2024-09-13T00:31:43.024 INFO:tasks.workunit.client.0.smithi136.stdout: write object-136 2024-09-13T00:31:43.025 INFO:tasks.workunit.client.0.smithi136.stdout: write object-176 2024-09-13T00:31:43.025 INFO:tasks.workunit.client.0.smithi136.stdout: write object-197 2024-09-13T00:31:43.025 INFO:tasks.workunit.client.0.smithi136.stdout: write object-198 2024-09-13T00:31:43.025 INFO:tasks.workunit.client.0.smithi136.stdout: write object-209 2024-09-13T00:31:43.025 INFO:tasks.workunit.client.0.smithi136.stdout: write object-213 2024-09-13T00:31:43.025 INFO:tasks.workunit.client.0.smithi136.stdout: write object-216 2024-09-13T00:31:43.025 INFO:tasks.workunit.client.0.smithi136.stdout: write object-221 2024-09-13T00:31:43.025 INFO:tasks.workunit.client.0.smithi136.stdout: write object-224 2024-09-13T00:31:43.025 INFO:tasks.workunit.client.0.smithi136.stdout: write object-229 2024-09-13T00:31:43.025 INFO:tasks.workunit.client.0.smithi136.stdout: write object-232 2024-09-13T00:31:43.025 INFO:tasks.workunit.client.0.smithi136.stdout: write object-248 2024-09-13T00:31:43.025 INFO:tasks.workunit.client.0.smithi136.stdout: write object-260 2024-09-13T00:31:43.025 INFO:tasks.workunit.client.0.smithi136.stdout: write object-265 2024-09-13T00:31:43.025 INFO:tasks.workunit.client.0.smithi136.stdout: write object-275 2024-09-13T00:31:43.026 INFO:tasks.workunit.client.0.smithi136.stdout: write object-332 2024-09-13T00:31:43.026 INFO:tasks.workunit.client.0.smithi136.stdout: write object-345 2024-09-13T00:31:43.026 INFO:tasks.workunit.client.0.smithi136.stdout: write object-346 2024-09-13T00:31:43.026 INFO:tasks.workunit.client.0.smithi136.stdout: write object-351 2024-09-13T00:31:43.026 INFO:tasks.workunit.client.0.smithi136.stdout: write object-356 2024-09-13T00:31:43.026 INFO:tasks.workunit.client.0.smithi136.stdout: write object-360 2024-09-13T00:31:43.026 INFO:tasks.workunit.client.0.smithi136.stdout: write object-373 2024-09-13T00:31:43.026 INFO:tasks.workunit.client.0.smithi136.stdout: write object-379 2024-09-13T00:31:43.026 INFO:tasks.workunit.client.0.smithi136.stdout: write object-382 2024-09-13T00:31:43.026 INFO:tasks.workunit.client.0.smithi136.stdout: write object-383 2024-09-13T00:31:43.026 INFO:tasks.workunit.client.0.smithi136.stdout: write object-420 2024-09-13T00:31:43.026 INFO:tasks.workunit.client.0.smithi136.stdout: write object-423 2024-09-13T00:31:43.026 INFO:tasks.workunit.client.0.smithi136.stdout: write object-425 2024-09-13T00:31:43.026 INFO:tasks.workunit.client.0.smithi136.stdout: write object-431 2024-09-13T00:31:43.027 INFO:tasks.workunit.client.0.smithi136.stdout: write object-433 2024-09-13T00:31:43.027 INFO:tasks.workunit.client.0.smithi136.stdout: write object-440 2024-09-13T00:31:43.027 INFO:tasks.workunit.client.0.smithi136.stdout: write object-455 2024-09-13T00:31:43.027 INFO:tasks.workunit.client.0.smithi136.stdout: write object-458 2024-09-13T00:31:43.027 INFO:tasks.workunit.client.0.smithi136.stdout: write object-461 2024-09-13T00:31:43.027 INFO:tasks.workunit.client.0.smithi136.stdout: write object-462 2024-09-13T00:31:43.027 INFO:tasks.workunit.client.0.smithi136.stdout: write object-468 2024-09-13T00:31:43.027 INFO:tasks.workunit.client.0.smithi136.stdout: write object-477 2024-09-13T00:31:43.027 INFO:tasks.workunit.client.0.smithi136.stdout: write object-485 2024-09-13T00:31:43.027 INFO:tasks.workunit.client.0.smithi136.stdout: write object-489 2024-09-13T00:31:43.027 INFO:tasks.workunit.client.0.smithi136.stdout: write object-509 2024-09-13T00:31:43.027 INFO:tasks.workunit.client.0.smithi136.stdout: write object-512 2024-09-13T00:31:43.027 INFO:tasks.workunit.client.0.smithi136.stdout: write object-519 2024-09-13T00:31:43.027 INFO:tasks.workunit.client.0.smithi136.stdout: write object-542 2024-09-13T00:31:43.028 INFO:tasks.workunit.client.0.smithi136.stdout: write object-548 2024-09-13T00:31:43.028 INFO:tasks.workunit.client.0.smithi136.stdout: write object-576 2024-09-13T00:31:43.028 INFO:tasks.workunit.client.0.smithi136.stdout: write object-581 2024-09-13T00:31:43.028 INFO:tasks.workunit.client.0.smithi136.stdout: write object-582 2024-09-13T00:31:43.028 INFO:tasks.workunit.client.0.smithi136.stdout: write object-589 2024-09-13T00:31:43.028 INFO:tasks.workunit.client.0.smithi136.stdout: write object-590 2024-09-13T00:31:43.028 INFO:tasks.workunit.client.0.smithi136.stdout: write object-600 2024-09-13T00:31:43.028 INFO:tasks.workunit.client.0.smithi136.stdout: write object-605 2024-09-13T00:31:43.028 INFO:tasks.workunit.client.0.smithi136.stdout: write object-607 2024-09-13T00:31:43.028 INFO:tasks.workunit.client.0.smithi136.stdout: write object-622 2024-09-13T00:31:43.028 INFO:tasks.workunit.client.0.smithi136.stdout: write object-625 2024-09-13T00:31:43.028 INFO:tasks.workunit.client.0.smithi136.stdout: write object-669 2024-09-13T00:31:43.028 INFO:tasks.workunit.client.0.smithi136.stdout: write object-682 2024-09-13T00:31:43.029 INFO:tasks.workunit.client.0.smithi136.stdout: write object-695 2024-09-13T00:31:43.029 INFO:tasks.workunit.client.0.smithi136.stdout: write object-703 2024-09-13T00:31:43.029 INFO:tasks.workunit.client.0.smithi136.stdout: write object-707 2024-09-13T00:31:43.029 INFO:tasks.workunit.client.0.smithi136.stdout: write object-719 2024-09-13T00:31:43.029 INFO:tasks.workunit.client.0.smithi136.stdout: write object-726 2024-09-13T00:31:43.029 INFO:tasks.workunit.client.0.smithi136.stdout: write object-750 2024-09-13T00:31:43.029 INFO:tasks.workunit.client.0.smithi136.stdout: write object-776 2024-09-13T00:31:43.029 INFO:tasks.workunit.client.0.smithi136.stdout: write object-777 2024-09-13T00:31:43.029 INFO:tasks.workunit.client.0.smithi136.stdout: write object-798 2024-09-13T00:31:43.029 INFO:tasks.workunit.client.0.smithi136.stdout: write object-807 2024-09-13T00:31:43.029 INFO:tasks.workunit.client.0.smithi136.stdout: write object-825 2024-09-13T00:31:43.030 INFO:tasks.workunit.client.0.smithi136.stdout: write object-849 2024-09-13T00:31:43.030 INFO:tasks.workunit.client.0.smithi136.stdout: write object-893 2024-09-13T00:31:43.030 INFO:tasks.workunit.client.0.smithi136.stdout: write object-911 2024-09-13T00:31:43.030 INFO:tasks.workunit.client.0.smithi136.stdout: write object-925 2024-09-13T00:31:43.030 INFO:tasks.workunit.client.0.smithi136.stdout: write object-936 2024-09-13T00:31:43.030 INFO:tasks.workunit.client.0.smithi136.stdout: write object-962 2024-09-13T00:31:43.030 INFO:tasks.workunit.client.0.smithi136.stdout: write object-965 2024-09-13T00:31:43.030 INFO:tasks.workunit.client.0.smithi136.stdout: write object-970 2024-09-13T00:31:43.030 INFO:tasks.workunit.client.0.smithi136.stdout: write object-986 2024-09-13T00:31:43.030 INFO:tasks.workunit.client.0.smithi136.stdout: write object-988 2024-09-13T00:31:43.030 INFO:tasks.workunit.client.0.smithi136.stdout: write object-1012 2024-09-13T00:31:43.734 INFO:tasks.workunit.client.0.smithi136.stdout: write object-1020 2024-09-13T00:31:43.734 INFO:tasks.workunit.client.0.smithi136.stdout:verify written data by reading 2024-09-13T00:31:43.734 INFO:tasks.workunit.client.0.smithi136.stdout: read object-11 2024-09-13T00:31:43.734 INFO:tasks.workunit.client.0.smithi136.stdout:made snapshot "image400@parent_snap" and protect it 2024-09-13T00:31:43.734 INFO:tasks.workunit.client.0.smithi136.stdout:made and opened clone "image401" 2024-09-13T00:31:43.735 INFO:tasks.workunit.client.0.smithi136.stdout:flattening clone: "image401" 2024-09-13T00:31:43.735 INFO:tasks.workunit.client.0.smithi136.stdout:check whether child image has the same set of objects as parent 2024-09-13T00:31:43.735 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestInternal.FlattenNoEmptyObjects (1288 ms) 2024-09-13T00:31:43.735 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestInternal.PoolMetadataConfApply 2024-09-13T00:31:43.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:43 smithi136 bash[75164]: cluster 2024-09-13T00:31:42.054193+0000 mgr.y (mgr.24661) 1637 : cluster [DBG] pgmap v2808: 137 pgs: 137 active+clean; 8.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 1.2 MiB/s rd, 1.0 MiB/s wr, 222 op/s 2024-09-13T00:31:43.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:43 smithi136 bash[75164]: cluster 2024-09-13T00:31:42.371717+0000 mon.a (mon.0) 2938 : cluster [DBG] osdmap e1611: 8 total, 8 up, 8 in 2024-09-13T00:31:43.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:43 smithi136 bash[76371]: cluster 2024-09-13T00:31:42.054193+0000 mgr.y (mgr.24661) 1637 : cluster [DBG] pgmap v2808: 137 pgs: 137 active+clean; 8.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 1.2 MiB/s rd, 1.0 MiB/s wr, 222 op/s 2024-09-13T00:31:43.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:43 smithi136 bash[76371]: cluster 2024-09-13T00:31:42.371717+0000 mon.a (mon.0) 2938 : cluster [DBG] osdmap e1611: 8 total, 8 up, 8 in 2024-09-13T00:31:43.835 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestInternal.PoolMetadataConfApply (101 ms) 2024-09-13T00:31:43.835 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestInternal.Sparsify 2024-09-13T00:31:43.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:43 smithi181 bash[67587]: cluster 2024-09-13T00:31:42.054193+0000 mgr.y (mgr.24661) 1637 : cluster [DBG] pgmap v2808: 137 pgs: 137 active+clean; 8.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 1.2 MiB/s rd, 1.0 MiB/s wr, 222 op/s 2024-09-13T00:31:43.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:43 smithi181 bash[67587]: cluster 2024-09-13T00:31:42.371717+0000 mon.a (mon.0) 2938 : cluster [DBG] osdmap e1611: 8 total, 8 up, 8 in 2024-09-13T00:31:43.862 INFO:tasks.workunit.client.0.smithi136.stdout:sparsify_supported=1 2024-09-13T00:31:43.862 INFO:tasks.workunit.client.0.smithi136.stdout:sparse_read_supported=1 2024-09-13T00:31:43.894 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestInternal.Sparsify (60 ms) 2024-09-13T00:31:43.894 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestInternal.SparsifyClone 2024-09-13T00:31:43.931 INFO:tasks.workunit.client.0.smithi136.stdout:sparsify_supported=1 2024-09-13T00:31:44.030 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:31:43 smithi136 bash[68787]: [13/Sep/2024:00:31:43] ENGINE Bus STOPPING 2024-09-13T00:31:44.030 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:31:43 smithi136 bash[68787]: [13/Sep/2024:00:31:43] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T00:31:44.030 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:31:43 smithi136 bash[68787]: [13/Sep/2024:00:31:43] ENGINE Bus STOPPED 2024-09-13T00:31:44.030 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:31:43 smithi136 bash[68787]: [13/Sep/2024:00:31:43] ENGINE Bus STARTING 2024-09-13T00:31:44.030 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:31:44 smithi136 bash[68787]: [13/Sep/2024:00:31:44] ENGINE Serving on http://:::9283 2024-09-13T00:31:44.375 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:31:44 smithi136 bash[68787]: [13/Sep/2024:00:31:44] ENGINE Bus STARTED 2024-09-13T00:31:44.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:44 smithi136 bash[75164]: cluster 2024-09-13T00:31:43.381691+0000 mon.a (mon.0) 2939 : cluster [DBG] osdmap e1612: 8 total, 8 up, 8 in 2024-09-13T00:31:44.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:44 smithi136 bash[75164]: audit 2024-09-13T00:31:43.765377+0000 mon.a (mon.0) 2940 : audit [INF] from='client.? 172.21.15.136:0/2085910915' entity='client.admin' 2024-09-13T00:31:44.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:44 smithi136 bash[75164]: audit 2024-09-13T00:31:43.779287+0000 mon.a (mon.0) 2941 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:31:44.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:44 smithi136 bash[75164]: audit 2024-09-13T00:31:43.782663+0000 mon.a (mon.0) 2942 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:31:44.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:44 smithi136 bash[75164]: audit 2024-09-13T00:31:43.784180+0000 mon.a (mon.0) 2943 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:31:44.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:44 smithi136 bash[75164]: audit 2024-09-13T00:31:43.785006+0000 mon.a (mon.0) 2944 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:31:44.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:44 smithi136 bash[75164]: audit 2024-09-13T00:31:43.792770+0000 mon.a (mon.0) 2945 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:31:44.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:44 smithi136 bash[76371]: cluster 2024-09-13T00:31:43.381691+0000 mon.a (mon.0) 2939 : cluster [DBG] osdmap e1612: 8 total, 8 up, 8 in 2024-09-13T00:31:44.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:44 smithi136 bash[76371]: audit 2024-09-13T00:31:43.765377+0000 mon.a (mon.0) 2940 : audit [INF] from='client.? 172.21.15.136:0/2085910915' entity='client.admin' 2024-09-13T00:31:44.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:44 smithi136 bash[76371]: audit 2024-09-13T00:31:43.779287+0000 mon.a (mon.0) 2941 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:31:44.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:44 smithi136 bash[76371]: audit 2024-09-13T00:31:43.782663+0000 mon.a (mon.0) 2942 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:31:44.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:44 smithi136 bash[76371]: audit 2024-09-13T00:31:43.784180+0000 mon.a (mon.0) 2943 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:31:44.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:44 smithi136 bash[76371]: audit 2024-09-13T00:31:43.785006+0000 mon.a (mon.0) 2944 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:31:44.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:44 smithi136 bash[76371]: audit 2024-09-13T00:31:43.792770+0000 mon.a (mon.0) 2945 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:31:44.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:44 smithi181 bash[67587]: cluster 2024-09-13T00:31:43.381691+0000 mon.a (mon.0) 2939 : cluster [DBG] osdmap e1612: 8 total, 8 up, 8 in 2024-09-13T00:31:44.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:44 smithi181 bash[67587]: audit 2024-09-13T00:31:43.765377+0000 mon.a (mon.0) 2940 : audit [INF] from='client.? 172.21.15.136:0/2085910915' entity='client.admin' 2024-09-13T00:31:44.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:44 smithi181 bash[67587]: audit 2024-09-13T00:31:43.779287+0000 mon.a (mon.0) 2941 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:31:44.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:44 smithi181 bash[67587]: audit 2024-09-13T00:31:43.782663+0000 mon.a (mon.0) 2942 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:31:44.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:44 smithi181 bash[67587]: audit 2024-09-13T00:31:43.784180+0000 mon.a (mon.0) 2943 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:31:44.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:44 smithi181 bash[67587]: audit 2024-09-13T00:31:43.785006+0000 mon.a (mon.0) 2944 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:31:44.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:44 smithi181 bash[67587]: audit 2024-09-13T00:31:43.792770+0000 mon.a (mon.0) 2945 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:31:45.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:45 smithi136 bash[75164]: cluster 2024-09-13T00:31:44.055177+0000 mgr.y (mgr.24661) 1638 : cluster [DBG] pgmap v2811: 137 pgs: 137 active+clean; 8.7 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 3.2 MiB/s rd, 3.0 MiB/s wr, 241 op/s 2024-09-13T00:31:45.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:45 smithi136 bash[75164]: cluster 2024-09-13T00:31:44.808887+0000 mon.a (mon.0) 2946 : cluster [DBG] osdmap e1613: 8 total, 8 up, 8 in 2024-09-13T00:31:45.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:45 smithi136 bash[76371]: cluster 2024-09-13T00:31:44.055177+0000 mgr.y (mgr.24661) 1638 : cluster [DBG] pgmap v2811: 137 pgs: 137 active+clean; 8.7 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 3.2 MiB/s rd, 3.0 MiB/s wr, 241 op/s 2024-09-13T00:31:45.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:45 smithi136 bash[76371]: cluster 2024-09-13T00:31:44.808887+0000 mon.a (mon.0) 2946 : cluster [DBG] osdmap e1613: 8 total, 8 up, 8 in 2024-09-13T00:31:45.841 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestInternal.SparsifyClone (1947 ms) 2024-09-13T00:31:45.841 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestInternal.MissingDataPool 2024-09-13T00:31:45.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:45 smithi181 bash[67587]: cluster 2024-09-13T00:31:44.055177+0000 mgr.y (mgr.24661) 1638 : cluster [DBG] pgmap v2811: 137 pgs: 137 active+clean; 8.7 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 3.2 MiB/s rd, 3.0 MiB/s wr, 241 op/s 2024-09-13T00:31:45.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:45 smithi181 bash[67587]: cluster 2024-09-13T00:31:44.808887+0000 mon.a (mon.0) 2946 : cluster [DBG] osdmap e1613: 8 total, 8 up, 8 in 2024-09-13T00:31:47.097 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestInternal.MissingDataPool (1255 ms) 2024-09-13T00:31:47.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:46 smithi181 bash[67587]: cluster 2024-09-13T00:31:45.815332+0000 mon.a (mon.0) 2947 : cluster [DBG] osdmap e1614: 8 total, 8 up, 8 in 2024-09-13T00:31:47.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:46 smithi181 bash[67587]: cluster 2024-09-13T00:31:46.055950+0000 mgr.y (mgr.24661) 1639 : cluster [DBG] pgmap v2814: 137 pgs: 137 active+clean; 8.7 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.0 MiB/s wr, 146 op/s 2024-09-13T00:31:47.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:46 smithi136 bash[76371]: cluster 2024-09-13T00:31:45.815332+0000 mon.a (mon.0) 2947 : cluster [DBG] osdmap e1614: 8 total, 8 up, 8 in 2024-09-13T00:31:47.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:46 smithi136 bash[76371]: cluster 2024-09-13T00:31:46.055950+0000 mgr.y (mgr.24661) 1639 : cluster [DBG] pgmap v2814: 137 pgs: 137 active+clean; 8.7 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.0 MiB/s wr, 146 op/s 2024-09-13T00:31:47.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:46 smithi136 bash[75164]: cluster 2024-09-13T00:31:45.815332+0000 mon.a (mon.0) 2947 : cluster [DBG] osdmap e1614: 8 total, 8 up, 8 in 2024-09-13T00:31:47.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:46 smithi136 bash[75164]: cluster 2024-09-13T00:31:46.055950+0000 mgr.y (mgr.24661) 1639 : cluster [DBG] pgmap v2814: 137 pgs: 137 active+clean; 8.7 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.0 MiB/s wr, 146 op/s 2024-09-13T00:31:47.835 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 34 tests from TestInternal (35930 ms total) 2024-09-13T00:31:47.835 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:31:47.835 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 17 tests from TestImageWatcher 2024-09-13T00:31:48.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:47 smithi181 bash[67587]: cluster 2024-09-13T00:31:46.822381+0000 mon.a (mon.0) 2948 : cluster [DBG] osdmap e1615: 8 total, 8 up, 8 in 2024-09-13T00:31:48.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:47 smithi136 bash[75164]: cluster 2024-09-13T00:31:46.822381+0000 mon.a (mon.0) 2948 : cluster [DBG] osdmap e1615: 8 total, 8 up, 8 in 2024-09-13T00:31:48.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:47 smithi136 bash[76371]: cluster 2024-09-13T00:31:46.822381+0000 mon.a (mon.0) 2948 : cluster [DBG] osdmap e1615: 8 total, 8 up, 8 in 2024-09-13T00:31:49.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:48 smithi181 bash[67587]: cluster 2024-09-13T00:31:47.829564+0000 mon.a (mon.0) 2949 : cluster [DBG] osdmap e1616: 8 total, 8 up, 8 in 2024-09-13T00:31:49.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:48 smithi181 bash[67587]: cluster 2024-09-13T00:31:48.056547+0000 mgr.y (mgr.24661) 1640 : cluster [DBG] pgmap v2817: 105 pgs: 105 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:31:49.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:48 smithi136 bash[75164]: cluster 2024-09-13T00:31:47.829564+0000 mon.a (mon.0) 2949 : cluster [DBG] osdmap e1616: 8 total, 8 up, 8 in 2024-09-13T00:31:49.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:48 smithi136 bash[75164]: cluster 2024-09-13T00:31:48.056547+0000 mgr.y (mgr.24661) 1640 : cluster [DBG] pgmap v2817: 105 pgs: 105 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:31:49.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:48 smithi136 bash[76371]: cluster 2024-09-13T00:31:47.829564+0000 mon.a (mon.0) 2949 : cluster [DBG] osdmap e1616: 8 total, 8 up, 8 in 2024-09-13T00:31:49.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:48 smithi136 bash[76371]: cluster 2024-09-13T00:31:48.056547+0000 mgr.y (mgr.24661) 1640 : cluster [DBG] pgmap v2817: 105 pgs: 105 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:31:49.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:31:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:31:49] "GET /metrics HTTP/1.1" 200 36541 "" "Prometheus/2.43.0" 2024-09-13T00:31:49.854 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestImageWatcher.NotifyHeaderUpdate 2024-09-13T00:31:50.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:49 smithi181 bash[67587]: cluster 2024-09-13T00:31:48.841513+0000 mon.a (mon.0) 2950 : cluster [DBG] osdmap e1617: 8 total, 8 up, 8 in 2024-09-13T00:31:50.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:49 smithi181 bash[67587]: audit 2024-09-13T00:31:48.845234+0000 mon.a (mon.0) 2951 : audit [INF] from='client.? 172.21.15.136:0/2525161772' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-13","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:31:50.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:49 smithi136 bash[75164]: cluster 2024-09-13T00:31:48.841513+0000 mon.a (mon.0) 2950 : cluster [DBG] osdmap e1617: 8 total, 8 up, 8 in 2024-09-13T00:31:50.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:49 smithi136 bash[75164]: audit 2024-09-13T00:31:48.845234+0000 mon.a (mon.0) 2951 : audit [INF] from='client.? 172.21.15.136:0/2525161772' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-13","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:31:50.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:49 smithi136 bash[76371]: cluster 2024-09-13T00:31:48.841513+0000 mon.a (mon.0) 2950 : cluster [DBG] osdmap e1617: 8 total, 8 up, 8 in 2024-09-13T00:31:50.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:49 smithi136 bash[76371]: audit 2024-09-13T00:31:48.845234+0000 mon.a (mon.0) 2951 : audit [INF] from='client.? 172.21.15.136:0/2525161772' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-13","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:31:51.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:50 smithi136 bash[75164]: audit 2024-09-13T00:31:49.852961+0000 mon.a (mon.0) 2952 : audit [INF] from='client.? 172.21.15.136:0/2525161772' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-13","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:31:51.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:50 smithi136 bash[75164]: cluster 2024-09-13T00:31:49.853513+0000 mon.a (mon.0) 2953 : cluster [DBG] osdmap e1618: 8 total, 8 up, 8 in 2024-09-13T00:31:51.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:50 smithi136 bash[75164]: cluster 2024-09-13T00:31:50.057287+0000 mgr.y (mgr.24661) 1641 : cluster [DBG] pgmap v2820: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:31:51.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:50 smithi136 bash[76371]: audit 2024-09-13T00:31:49.852961+0000 mon.a (mon.0) 2952 : audit [INF] from='client.? 172.21.15.136:0/2525161772' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-13","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:31:51.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:50 smithi136 bash[76371]: cluster 2024-09-13T00:31:49.853513+0000 mon.a (mon.0) 2953 : cluster [DBG] osdmap e1618: 8 total, 8 up, 8 in 2024-09-13T00:31:51.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:50 smithi136 bash[76371]: cluster 2024-09-13T00:31:50.057287+0000 mgr.y (mgr.24661) 1641 : cluster [DBG] pgmap v2820: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:31:51.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:50 smithi181 bash[67587]: audit 2024-09-13T00:31:49.852961+0000 mon.a (mon.0) 2952 : audit [INF] from='client.? 172.21.15.136:0/2525161772' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-13","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:31:51.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:50 smithi181 bash[67587]: cluster 2024-09-13T00:31:49.853513+0000 mon.a (mon.0) 2953 : cluster [DBG] osdmap e1618: 8 total, 8 up, 8 in 2024-09-13T00:31:51.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:50 smithi181 bash[67587]: cluster 2024-09-13T00:31:50.057287+0000 mgr.y (mgr.24661) 1641 : cluster [DBG] pgmap v2820: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:31:51.939 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestImageWatcher.NotifyHeaderUpdate (2086 ms) 2024-09-13T00:31:51.940 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestImageWatcher.NotifyFlatten 2024-09-13T00:31:51.977 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestImageWatcher.NotifyFlatten (37 ms) 2024-09-13T00:31:51.977 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestImageWatcher.NotifyResize 2024-09-13T00:31:52.023 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestImageWatcher.NotifyResize (47 ms) 2024-09-13T00:31:52.023 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestImageWatcher.NotifyRebuildObjectMap 2024-09-13T00:31:52.084 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestImageWatcher.NotifyRebuildObjectMap (60 ms) 2024-09-13T00:31:52.084 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestImageWatcher.NotifyUpdateFeatures 2024-09-13T00:31:52.141 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestImageWatcher.NotifyUpdateFeatures (57 ms) 2024-09-13T00:31:52.141 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestImageWatcher.NotifySnapCreate 2024-09-13T00:31:52.194 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestImageWatcher.NotifySnapCreate (53 ms) 2024-09-13T00:31:52.194 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestImageWatcher.NotifySnapCreateError 2024-09-13T00:31:52.250 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestImageWatcher.NotifySnapCreateError (56 ms) 2024-09-13T00:31:52.250 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestImageWatcher.NotifySnapRename 2024-09-13T00:31:52.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:51 smithi136 bash[75164]: cluster 2024-09-13T00:31:50.862829+0000 mon.a (mon.0) 2954 : cluster [DBG] osdmap e1619: 8 total, 8 up, 8 in 2024-09-13T00:31:52.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:51 smithi136 bash[76371]: cluster 2024-09-13T00:31:50.862829+0000 mon.a (mon.0) 2954 : cluster [DBG] osdmap e1619: 8 total, 8 up, 8 in 2024-09-13T00:31:52.304 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestImageWatcher.NotifySnapRename (54 ms) 2024-09-13T00:31:52.304 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestImageWatcher.NotifySnapRenameError 2024-09-13T00:31:52.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:51 smithi181 bash[67587]: cluster 2024-09-13T00:31:50.862829+0000 mon.a (mon.0) 2954 : cluster [DBG] osdmap e1619: 8 total, 8 up, 8 in 2024-09-13T00:31:52.355 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestImageWatcher.NotifySnapRenameError (51 ms) 2024-09-13T00:31:52.355 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestImageWatcher.NotifySnapRemove 2024-09-13T00:31:52.410 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestImageWatcher.NotifySnapRemove (55 ms) 2024-09-13T00:31:52.410 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestImageWatcher.NotifySnapProtect 2024-09-13T00:31:52.468 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestImageWatcher.NotifySnapProtect (59 ms) 2024-09-13T00:31:52.469 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestImageWatcher.NotifySnapUnprotect 2024-09-13T00:31:52.515 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestImageWatcher.NotifySnapUnprotect (46 ms) 2024-09-13T00:31:52.515 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestImageWatcher.NotifyRename 2024-09-13T00:31:52.567 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestImageWatcher.NotifyRename (52 ms) 2024-09-13T00:31:52.567 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestImageWatcher.NotifyAsyncTimedOut 2024-09-13T00:31:52.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:31:52 smithi181 bash[95505]: ts=2024-09-13T00:31:52.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:31:52.605 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestImageWatcher.NotifyAsyncTimedOut (39 ms) 2024-09-13T00:31:52.606 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestImageWatcher.NotifyAsyncError 2024-09-13T00:31:52.648 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestImageWatcher.NotifyAsyncError (42 ms) 2024-09-13T00:31:52.648 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestImageWatcher.NotifyAsyncCompleteError 2024-09-13T00:31:52.686 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestImageWatcher.NotifyAsyncCompleteError (37 ms) 2024-09-13T00:31:52.686 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestImageWatcher.NotifyAsyncRequestTimedOut 2024-09-13T00:31:52.726 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestImageWatcher.NotifyAsyncRequestTimedOut (40 ms) 2024-09-13T00:31:52.900 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 17 tests from TestImageWatcher (2872 ms total) 2024-09-13T00:31:52.900 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:31:52.901 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 160 tests from TestLibRBD 2024-09-13T00:31:52.954 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.CreateAndStat 2024-09-13T00:31:53.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:52 smithi136 bash[75164]: cluster 2024-09-13T00:31:51.865009+0000 mon.a (mon.0) 2955 : cluster [DBG] osdmap e1620: 8 total, 8 up, 8 in 2024-09-13T00:31:53.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:52 smithi136 bash[75164]: cluster 2024-09-13T00:31:52.057976+0000 mgr.y (mgr.24661) 1642 : cluster [DBG] pgmap v2823: 137 pgs: 3 creating+peering, 9 unknown, 125 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:31:53.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:52 smithi136 bash[76371]: cluster 2024-09-13T00:31:51.865009+0000 mon.a (mon.0) 2955 : cluster [DBG] osdmap e1620: 8 total, 8 up, 8 in 2024-09-13T00:31:53.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:52 smithi136 bash[76371]: cluster 2024-09-13T00:31:52.057976+0000 mgr.y (mgr.24661) 1642 : cluster [DBG] pgmap v2823: 137 pgs: 3 creating+peering, 9 unknown, 125 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:31:53.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:52 smithi181 bash[67587]: cluster 2024-09-13T00:31:51.865009+0000 mon.a (mon.0) 2955 : cluster [DBG] osdmap e1620: 8 total, 8 up, 8 in 2024-09-13T00:31:53.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:52 smithi181 bash[67587]: cluster 2024-09-13T00:31:52.057976+0000 mgr.y (mgr.24661) 1642 : cluster [DBG] pgmap v2823: 137 pgs: 3 creating+peering, 9 unknown, 125 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:31:54.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:53 smithi136 bash[75164]: cluster 2024-09-13T00:31:52.892974+0000 mon.a (mon.0) 2956 : cluster [DBG] osdmap e1621: 8 total, 8 up, 8 in 2024-09-13T00:31:54.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:53 smithi136 bash[75164]: audit 2024-09-13T00:31:53.824751+0000 mon.a (mon.0) 2957 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:31:54.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:53 smithi136 bash[76371]: cluster 2024-09-13T00:31:52.892974+0000 mon.a (mon.0) 2956 : cluster [DBG] osdmap e1621: 8 total, 8 up, 8 in 2024-09-13T00:31:54.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:53 smithi136 bash[76371]: audit 2024-09-13T00:31:53.824751+0000 mon.a (mon.0) 2957 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:31:54.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:53 smithi181 bash[67587]: cluster 2024-09-13T00:31:52.892974+0000 mon.a (mon.0) 2956 : cluster [DBG] osdmap e1621: 8 total, 8 up, 8 in 2024-09-13T00:31:54.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:53 smithi181 bash[67587]: audit 2024-09-13T00:31:53.824751+0000 mon.a (mon.0) 2957 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:31:54.907 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:31:55.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:54 smithi136 bash[75164]: cluster 2024-09-13T00:31:53.902879+0000 mon.a (mon.0) 2958 : cluster [DBG] osdmap e1622: 8 total, 8 up, 8 in 2024-09-13T00:31:55.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:54 smithi136 bash[75164]: audit 2024-09-13T00:31:53.906004+0000 mon.c (mon.2) 114 : audit [INF] from='client.? 172.21.15.136:0/3261449233' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-14","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:31:55.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:54 smithi136 bash[75164]: audit 2024-09-13T00:31:53.909570+0000 mon.a (mon.0) 2959 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-14","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:31:55.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:54 smithi136 bash[75164]: cluster 2024-09-13T00:31:54.058717+0000 mgr.y (mgr.24661) 1643 : cluster [DBG] pgmap v2826: 137 pgs: 32 unknown, 105 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:31:55.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:54 smithi136 bash[75164]: audit 2024-09-13T00:31:54.899878+0000 mon.a (mon.0) 2960 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-14","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:31:55.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:54 smithi136 bash[75164]: cluster 2024-09-13T00:31:54.900225+0000 mon.a (mon.0) 2961 : cluster [DBG] osdmap e1623: 8 total, 8 up, 8 in 2024-09-13T00:31:55.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:54 smithi136 bash[76371]: cluster 2024-09-13T00:31:53.902879+0000 mon.a (mon.0) 2958 : cluster [DBG] osdmap e1622: 8 total, 8 up, 8 in 2024-09-13T00:31:55.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:54 smithi136 bash[76371]: audit 2024-09-13T00:31:53.906004+0000 mon.c (mon.2) 114 : audit [INF] from='client.? 172.21.15.136:0/3261449233' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-14","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:31:55.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:54 smithi136 bash[76371]: audit 2024-09-13T00:31:53.909570+0000 mon.a (mon.0) 2959 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-14","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:31:55.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:54 smithi136 bash[76371]: cluster 2024-09-13T00:31:54.058717+0000 mgr.y (mgr.24661) 1643 : cluster [DBG] pgmap v2826: 137 pgs: 32 unknown, 105 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:31:55.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:54 smithi136 bash[76371]: audit 2024-09-13T00:31:54.899878+0000 mon.a (mon.0) 2960 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-14","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:31:55.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:54 smithi136 bash[76371]: cluster 2024-09-13T00:31:54.900225+0000 mon.a (mon.0) 2961 : cluster [DBG] osdmap e1623: 8 total, 8 up, 8 in 2024-09-13T00:31:55.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:54 smithi181 bash[67587]: cluster 2024-09-13T00:31:53.902879+0000 mon.a (mon.0) 2958 : cluster [DBG] osdmap e1622: 8 total, 8 up, 8 in 2024-09-13T00:31:55.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:54 smithi181 bash[67587]: audit 2024-09-13T00:31:53.906004+0000 mon.c (mon.2) 114 : audit [INF] from='client.? 172.21.15.136:0/3261449233' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-14","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:31:55.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:54 smithi181 bash[67587]: audit 2024-09-13T00:31:53.909570+0000 mon.a (mon.0) 2959 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-14","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:31:55.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:54 smithi181 bash[67587]: cluster 2024-09-13T00:31:54.058717+0000 mgr.y (mgr.24661) 1643 : cluster [DBG] pgmap v2826: 137 pgs: 32 unknown, 105 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:31:55.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:54 smithi181 bash[67587]: audit 2024-09-13T00:31:54.899878+0000 mon.a (mon.0) 2960 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-14","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:31:55.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:54 smithi181 bash[67587]: cluster 2024-09-13T00:31:54.900225+0000 mon.a (mon.0) 2961 : cluster [DBG] osdmap e1623: 8 total, 8 up, 8 in 2024-09-13T00:31:56.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:55 smithi136 bash[76371]: cluster 2024-09-13T00:31:54.910357+0000 mon.a (mon.0) 2962 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:31:56.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:55 smithi136 bash[76371]: cluster 2024-09-13T00:31:55.923029+0000 mon.a (mon.0) 2963 : cluster [DBG] osdmap e1624: 8 total, 8 up, 8 in 2024-09-13T00:31:56.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:55 smithi136 bash[75164]: cluster 2024-09-13T00:31:54.910357+0000 mon.a (mon.0) 2962 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:31:56.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:55 smithi136 bash[75164]: cluster 2024-09-13T00:31:55.923029+0000 mon.a (mon.0) 2963 : cluster [DBG] osdmap e1624: 8 total, 8 up, 8 in 2024-09-13T00:31:56.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:55 smithi181 bash[67587]: cluster 2024-09-13T00:31:54.910357+0000 mon.a (mon.0) 2962 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:31:56.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:55 smithi181 bash[67587]: cluster 2024-09-13T00:31:55.923029+0000 mon.a (mon.0) 2963 : cluster [DBG] osdmap e1624: 8 total, 8 up, 8 in 2024-09-13T00:31:57.022 INFO:tasks.workunit.client.0.smithi136.stdout:image has size 2097152 and order 22 2024-09-13T00:31:57.022 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.CreateAndStat (4068 ms) 2024-09-13T00:31:57.022 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.CreateWithSameDataPool 2024-09-13T00:31:57.022 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:31:57.056 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.CreateWithSameDataPool (34 ms) 2024-09-13T00:31:57.056 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.CreateAndStatPP 2024-09-13T00:31:57.057 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:31:57.092 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.CreateAndStatPP (37 ms) 2024-09-13T00:31:57.093 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.GetId 2024-09-13T00:31:57.093 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:31:57.135 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.GetId (43 ms) 2024-09-13T00:31:57.136 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.GetIdPP 2024-09-13T00:31:57.136 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:31:57.177 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.GetIdPP (42 ms) 2024-09-13T00:31:57.177 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.GetBlockNamePrefix 2024-09-13T00:31:57.177 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:31:57.215 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.GetBlockNamePrefix (38 ms) 2024-09-13T00:31:57.215 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.GetBlockNamePrefixPP 2024-09-13T00:31:57.216 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:31:57.251 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.GetBlockNamePrefixPP (35 ms) 2024-09-13T00:31:57.251 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestGetCreateTimestamp 2024-09-13T00:31:57.251 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:31:57.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:56 smithi136 bash[76371]: cluster 2024-09-13T00:31:56.059391+0000 mgr.y (mgr.24661) 1644 : cluster [DBG] pgmap v2829: 137 pgs: 6 creating+peering, 20 unknown, 111 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:31:57.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:56 smithi136 bash[76371]: audit 2024-09-13T00:31:56.103536+0000 mon.a (mon.0) 2964 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T00:31:57.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:56 smithi136 bash[76371]: audit 2024-09-13T00:31:56.295452+0000 mon.a (mon.0) 2965 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:31:57.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:56 smithi136 bash[76371]: audit 2024-09-13T00:31:56.295923+0000 mon.a (mon.0) 2966 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:31:57.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:56 smithi136 bash[76371]: audit 2024-09-13T00:31:56.296360+0000 mon.a (mon.0) 2967 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:31:57.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:56 smithi136 bash[76371]: audit 2024-09-13T00:31:56.299785+0000 mon.a (mon.0) 2968 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:31:57.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:56 smithi136 bash[76371]: audit 2024-09-13T00:31:56.300109+0000 mon.a (mon.0) 2969 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:31:57.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:56 smithi136 bash[76371]: audit 2024-09-13T00:31:56.300492+0000 mon.a (mon.0) 2970 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:31:57.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:56 smithi136 bash[76371]: audit 2024-09-13T00:31:56.303696+0000 mon.a (mon.0) 2971 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:31:57.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:56 smithi136 bash[76371]: audit 2024-09-13T00:31:56.304203+0000 mon.a (mon.0) 2972 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:31:57.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:56 smithi136 bash[76371]: audit 2024-09-13T00:31:56.304513+0000 mon.a (mon.0) 2973 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:31:57.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:56 smithi136 bash[76371]: audit 2024-09-13T00:31:56.326737+0000 mon.a (mon.0) 2974 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T00:31:57.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:56 smithi136 bash[76371]: audit 2024-09-13T00:31:56.326884+0000 mon.a (mon.0) 2975 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T00:31:57.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:56 smithi136 bash[76371]: audit 2024-09-13T00:31:56.327401+0000 mon.a (mon.0) 2976 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:31:57.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:56 smithi136 bash[76371]: audit 2024-09-13T00:31:56.327904+0000 mon.a (mon.0) 2977 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:31:57.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:56 smithi136 bash[76371]: audit 2024-09-13T00:31:56.328318+0000 mon.a (mon.0) 2978 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:31:57.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:56 smithi136 bash[76371]: audit 2024-09-13T00:31:56.330664+0000 mon.b (mon.1) 16 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T00:31:57.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:56 smithi136 bash[76371]: audit 2024-09-13T00:31:56.330791+0000 mon.b (mon.1) 17 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T00:31:57.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:56 smithi136 bash[76371]: audit 2024-09-13T00:31:56.331539+0000 mon.a (mon.0) 2979 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:31:57.262 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:56 smithi136 bash[75164]: cluster 2024-09-13T00:31:56.059391+0000 mgr.y (mgr.24661) 1644 : cluster [DBG] pgmap v2829: 137 pgs: 6 creating+peering, 20 unknown, 111 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:31:57.262 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:56 smithi136 bash[75164]: audit 2024-09-13T00:31:56.103536+0000 mon.a (mon.0) 2964 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T00:31:57.262 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:56 smithi136 bash[75164]: audit 2024-09-13T00:31:56.295452+0000 mon.a (mon.0) 2965 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:31:57.262 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:56 smithi136 bash[75164]: audit 2024-09-13T00:31:56.295923+0000 mon.a (mon.0) 2966 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:31:57.262 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:56 smithi136 bash[75164]: audit 2024-09-13T00:31:56.296360+0000 mon.a (mon.0) 2967 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:31:57.262 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:56 smithi136 bash[75164]: audit 2024-09-13T00:31:56.299785+0000 mon.a (mon.0) 2968 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:31:57.262 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:56 smithi136 bash[75164]: audit 2024-09-13T00:31:56.300109+0000 mon.a (mon.0) 2969 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:31:57.262 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:56 smithi136 bash[75164]: audit 2024-09-13T00:31:56.300492+0000 mon.a (mon.0) 2970 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:31:57.262 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:56 smithi136 bash[75164]: audit 2024-09-13T00:31:56.303696+0000 mon.a (mon.0) 2971 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:31:57.262 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:56 smithi136 bash[75164]: audit 2024-09-13T00:31:56.304203+0000 mon.a (mon.0) 2972 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:31:57.262 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:56 smithi136 bash[75164]: audit 2024-09-13T00:31:56.304513+0000 mon.a (mon.0) 2973 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:31:57.262 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:56 smithi136 bash[75164]: audit 2024-09-13T00:31:56.326737+0000 mon.a (mon.0) 2974 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T00:31:57.263 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:56 smithi136 bash[75164]: audit 2024-09-13T00:31:56.326884+0000 mon.a (mon.0) 2975 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T00:31:57.263 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:56 smithi136 bash[75164]: audit 2024-09-13T00:31:56.327401+0000 mon.a (mon.0) 2976 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:31:57.263 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:56 smithi136 bash[75164]: audit 2024-09-13T00:31:56.327904+0000 mon.a (mon.0) 2977 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:31:57.263 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:56 smithi136 bash[75164]: audit 2024-09-13T00:31:56.328318+0000 mon.a (mon.0) 2978 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:31:57.263 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:56 smithi136 bash[75164]: audit 2024-09-13T00:31:56.330664+0000 mon.b (mon.1) 16 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T00:31:57.263 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:56 smithi136 bash[75164]: audit 2024-09-13T00:31:56.330791+0000 mon.b (mon.1) 17 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T00:31:57.263 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:56 smithi136 bash[75164]: audit 2024-09-13T00:31:56.331539+0000 mon.a (mon.0) 2979 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:31:57.263 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:56 smithi136 bash[76371]: audit 2024-09-13T00:31:56.332212+0000 mon.a (mon.0) 2980 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:31:57.263 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:56 smithi136 bash[76371]: audit 2024-09-13T00:31:56.332504+0000 mon.a (mon.0) 2981 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:31:57.263 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:56 smithi136 bash[76371]: audit 2024-09-13T00:31:56.334954+0000 mon.c (mon.2) 115 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T00:31:57.264 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:56 smithi136 bash[76371]: audit 2024-09-13T00:31:56.335084+0000 mon.c (mon.2) 116 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T00:31:57.264 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:56 smithi136 bash[76371]: audit 2024-09-13T00:31:56.335357+0000 mon.a (mon.0) 2982 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:31:57.264 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:56 smithi136 bash[76371]: audit 2024-09-13T00:31:56.336315+0000 mon.a (mon.0) 2983 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:31:57.264 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:56 smithi136 bash[76371]: audit 2024-09-13T00:31:56.336759+0000 mon.a (mon.0) 2984 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:31:57.264 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:56 smithi136 bash[75164]: audit 2024-09-13T00:31:56.332212+0000 mon.a (mon.0) 2980 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:31:57.264 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:56 smithi136 bash[75164]: audit 2024-09-13T00:31:56.332504+0000 mon.a (mon.0) 2981 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:31:57.264 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:56 smithi136 bash[75164]: audit 2024-09-13T00:31:56.334954+0000 mon.c (mon.2) 115 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T00:31:57.264 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:56 smithi136 bash[75164]: audit 2024-09-13T00:31:56.335084+0000 mon.c (mon.2) 116 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T00:31:57.264 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:56 smithi136 bash[75164]: audit 2024-09-13T00:31:56.335357+0000 mon.a (mon.0) 2982 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:31:57.264 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:56 smithi136 bash[75164]: audit 2024-09-13T00:31:56.336315+0000 mon.a (mon.0) 2983 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:31:57.265 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:56 smithi136 bash[75164]: audit 2024-09-13T00:31:56.336759+0000 mon.a (mon.0) 2984 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:31:57.286 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestGetCreateTimestamp (35 ms) 2024-09-13T00:31:57.286 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.GetCreateTimestampPP 2024-09-13T00:31:57.287 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:31:57.321 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.GetCreateTimestampPP (34 ms) 2024-09-13T00:31:57.321 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.OpenAio 2024-09-13T00:31:57.321 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:31:57.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:56 smithi181 bash[67587]: cluster 2024-09-13T00:31:56.059391+0000 mgr.y (mgr.24661) 1644 : cluster [DBG] pgmap v2829: 137 pgs: 6 creating+peering, 20 unknown, 111 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:31:57.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:56 smithi181 bash[67587]: audit 2024-09-13T00:31:56.103536+0000 mon.a (mon.0) 2964 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T00:31:57.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:56 smithi181 bash[67587]: audit 2024-09-13T00:31:56.295452+0000 mon.a (mon.0) 2965 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:31:57.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:56 smithi181 bash[67587]: audit 2024-09-13T00:31:56.295923+0000 mon.a (mon.0) 2966 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:31:57.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:56 smithi181 bash[67587]: audit 2024-09-13T00:31:56.296360+0000 mon.a (mon.0) 2967 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:31:57.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:56 smithi181 bash[67587]: audit 2024-09-13T00:31:56.299785+0000 mon.a (mon.0) 2968 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:31:57.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:56 smithi181 bash[67587]: audit 2024-09-13T00:31:56.300109+0000 mon.a (mon.0) 2969 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:31:57.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:56 smithi181 bash[67587]: audit 2024-09-13T00:31:56.300492+0000 mon.a (mon.0) 2970 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:31:57.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:56 smithi181 bash[67587]: audit 2024-09-13T00:31:56.303696+0000 mon.a (mon.0) 2971 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:31:57.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:56 smithi181 bash[67587]: audit 2024-09-13T00:31:56.304203+0000 mon.a (mon.0) 2972 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:31:57.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:56 smithi181 bash[67587]: audit 2024-09-13T00:31:56.304513+0000 mon.a (mon.0) 2973 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:31:57.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:56 smithi181 bash[67587]: audit 2024-09-13T00:31:56.326737+0000 mon.a (mon.0) 2974 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T00:31:57.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:56 smithi181 bash[67587]: audit 2024-09-13T00:31:56.326884+0000 mon.a (mon.0) 2975 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T00:31:57.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:56 smithi181 bash[67587]: audit 2024-09-13T00:31:56.327401+0000 mon.a (mon.0) 2976 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:31:57.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:56 smithi181 bash[67587]: audit 2024-09-13T00:31:56.327904+0000 mon.a (mon.0) 2977 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:31:57.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:56 smithi181 bash[67587]: audit 2024-09-13T00:31:56.328318+0000 mon.a (mon.0) 2978 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:31:57.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:56 smithi181 bash[67587]: audit 2024-09-13T00:31:56.330664+0000 mon.b (mon.1) 16 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T00:31:57.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:56 smithi181 bash[67587]: audit 2024-09-13T00:31:56.330791+0000 mon.b (mon.1) 17 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T00:31:57.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:56 smithi181 bash[67587]: audit 2024-09-13T00:31:56.331539+0000 mon.a (mon.0) 2979 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:31:57.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:56 smithi181 bash[67587]: audit 2024-09-13T00:31:56.332212+0000 mon.a (mon.0) 2980 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:31:57.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:56 smithi181 bash[67587]: audit 2024-09-13T00:31:56.332504+0000 mon.a (mon.0) 2981 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:31:57.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:56 smithi181 bash[67587]: audit 2024-09-13T00:31:56.334954+0000 mon.c (mon.2) 115 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T00:31:57.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:56 smithi181 bash[67587]: audit 2024-09-13T00:31:56.335084+0000 mon.c (mon.2) 116 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T00:31:57.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:56 smithi181 bash[67587]: audit 2024-09-13T00:31:56.335357+0000 mon.a (mon.0) 2982 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:31:57.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:56 smithi181 bash[67587]: audit 2024-09-13T00:31:56.336315+0000 mon.a (mon.0) 2983 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:31:57.352 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:56 smithi181 bash[67587]: audit 2024-09-13T00:31:56.336759+0000 mon.a (mon.0) 2984 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:31:57.356 INFO:tasks.workunit.client.0.smithi136.stdout:image has size 2097152 and order 22 2024-09-13T00:31:57.357 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.OpenAio (36 ms) 2024-09-13T00:31:57.357 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.OpenAioFail 2024-09-13T00:31:57.361 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.OpenAioFail (5 ms) 2024-09-13T00:31:57.361 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.OpenAioPP 2024-09-13T00:31:57.361 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:31:57.412 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.OpenAioPP (50 ms) 2024-09-13T00:31:57.412 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.OpenAioFailPP 2024-09-13T00:31:57.419 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.OpenAioFailPP (7 ms) 2024-09-13T00:31:57.419 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.ResizeAndStat 2024-09-13T00:31:57.419 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:31:57.494 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.ResizeAndStat (75 ms) 2024-09-13T00:31:57.494 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.ResizeAndStatPP 2024-09-13T00:31:57.494 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:31:57.573 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.ResizeAndStatPP (79 ms) 2024-09-13T00:31:57.573 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.UpdateWatchAndResize 2024-09-13T00:31:57.573 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:31:57.664 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.UpdateWatchAndResize (92 ms) 2024-09-13T00:31:57.665 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.UpdateWatchAndResizePP 2024-09-13T00:31:57.665 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:31:57.747 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.UpdateWatchAndResizePP (82 ms) 2024-09-13T00:31:57.747 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestCreateLsDelete 2024-09-13T00:31:58.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:58 smithi136 bash[75164]: cluster 2024-09-13T00:31:56.932716+0000 mon.a (mon.0) 2985 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:31:58.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:58 smithi136 bash[75164]: cluster 2024-09-13T00:31:56.932762+0000 mon.a (mon.0) 2986 : cluster [INF] Cluster is now healthy 2024-09-13T00:31:58.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:58 smithi136 bash[75164]: cluster 2024-09-13T00:31:56.960704+0000 mon.a (mon.0) 2987 : cluster [DBG] osdmap e1625: 8 total, 8 up, 8 in 2024-09-13T00:31:58.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:58 smithi136 bash[76371]: cluster 2024-09-13T00:31:56.932716+0000 mon.a (mon.0) 2985 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:31:58.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:58 smithi136 bash[76371]: cluster 2024-09-13T00:31:56.932762+0000 mon.a (mon.0) 2986 : cluster [INF] Cluster is now healthy 2024-09-13T00:31:58.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:58 smithi136 bash[76371]: cluster 2024-09-13T00:31:56.960704+0000 mon.a (mon.0) 2987 : cluster [DBG] osdmap e1625: 8 total, 8 up, 8 in 2024-09-13T00:31:58.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:58 smithi181 bash[67587]: cluster 2024-09-13T00:31:56.932716+0000 mon.a (mon.0) 2985 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:31:58.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:58 smithi181 bash[67587]: cluster 2024-09-13T00:31:56.932762+0000 mon.a (mon.0) 2986 : cluster [INF] Cluster is now healthy 2024-09-13T00:31:58.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:58 smithi181 bash[67587]: cluster 2024-09-13T00:31:56.960704+0000 mon.a (mon.0) 2987 : cluster [DBG] osdmap e1625: 8 total, 8 up, 8 in 2024-09-13T00:31:59.330 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:31:59.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:59 smithi181 bash[67587]: cluster 2024-09-13T00:31:58.060201+0000 mgr.y (mgr.24661) 1645 : cluster [DBG] pgmap v2831: 137 pgs: 6 creating+peering, 6 unknown, 125 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 3.8 KiB/s wr, 19 op/s 2024-09-13T00:31:59.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:59 smithi181 bash[67587]: cluster 2024-09-13T00:31:58.272777+0000 mon.a (mon.0) 2988 : cluster [DBG] osdmap e1626: 8 total, 8 up, 8 in 2024-09-13T00:31:59.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:31:59 smithi181 bash[67587]: audit 2024-09-13T00:31:58.275272+0000 mon.a (mon.0) 2989 : audit [INF] from='client.? 172.21.15.136:0/1446178036' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-15","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:31:59.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:31:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:31:59] "GET /metrics HTTP/1.1" 200 34127 "" "Prometheus/2.43.0" 2024-09-13T00:31:59.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:59 smithi136 bash[75164]: cluster 2024-09-13T00:31:58.060201+0000 mgr.y (mgr.24661) 1645 : cluster [DBG] pgmap v2831: 137 pgs: 6 creating+peering, 6 unknown, 125 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 3.8 KiB/s wr, 19 op/s 2024-09-13T00:31:59.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:59 smithi136 bash[75164]: cluster 2024-09-13T00:31:58.272777+0000 mon.a (mon.0) 2988 : cluster [DBG] osdmap e1626: 8 total, 8 up, 8 in 2024-09-13T00:31:59.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:31:59 smithi136 bash[75164]: audit 2024-09-13T00:31:58.275272+0000 mon.a (mon.0) 2989 : audit [INF] from='client.? 172.21.15.136:0/1446178036' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-15","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:31:59.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:59 smithi136 bash[76371]: cluster 2024-09-13T00:31:58.060201+0000 mgr.y (mgr.24661) 1645 : cluster [DBG] pgmap v2831: 137 pgs: 6 creating+peering, 6 unknown, 125 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 3.8 KiB/s wr, 19 op/s 2024-09-13T00:31:59.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:59 smithi136 bash[76371]: cluster 2024-09-13T00:31:58.272777+0000 mon.a (mon.0) 2988 : cluster [DBG] osdmap e1626: 8 total, 8 up, 8 in 2024-09-13T00:31:59.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:31:59 smithi136 bash[76371]: audit 2024-09-13T00:31:58.275272+0000 mon.a (mon.0) 2989 : audit [INF] from='client.? 172.21.15.136:0/1446178036' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-15","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:32:00.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:00 smithi181 bash[67587]: audit 2024-09-13T00:31:59.282634+0000 mon.a (mon.0) 2990 : audit [INF] from='client.? 172.21.15.136:0/1446178036' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-15","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:32:00.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:00 smithi181 bash[67587]: cluster 2024-09-13T00:31:59.283292+0000 mon.a (mon.0) 2991 : cluster [DBG] osdmap e1627: 8 total, 8 up, 8 in 2024-09-13T00:32:00.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:00 smithi136 bash[75164]: audit 2024-09-13T00:31:59.282634+0000 mon.a (mon.0) 2990 : audit [INF] from='client.? 172.21.15.136:0/1446178036' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-15","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:32:00.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:00 smithi136 bash[75164]: cluster 2024-09-13T00:31:59.283292+0000 mon.a (mon.0) 2991 : cluster [DBG] osdmap e1627: 8 total, 8 up, 8 in 2024-09-13T00:32:00.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:00 smithi136 bash[76371]: audit 2024-09-13T00:31:59.282634+0000 mon.a (mon.0) 2990 : audit [INF] from='client.? 172.21.15.136:0/1446178036' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-15","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:32:00.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:00 smithi136 bash[76371]: cluster 2024-09-13T00:31:59.283292+0000 mon.a (mon.0) 2991 : cluster [DBG] osdmap e1627: 8 total, 8 up, 8 in 2024-09-13T00:32:01.380 INFO:tasks.workunit.client.0.smithi136.stdout:image: image18 2024-09-13T00:32:01.380 INFO:tasks.workunit.client.0.smithi136.stdout:expected = image18 2024-09-13T00:32:01.380 INFO:tasks.workunit.client.0.smithi136.stdout:found image18 2024-09-13T00:32:01.380 INFO:tasks.workunit.client.0.smithi136.stdout:erased image18 2024-09-13T00:32:01.380 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:32:01.582 INFO:tasks.workunit.client.0.smithi136.stdout:image: image18 2024-09-13T00:32:01.582 INFO:tasks.workunit.client.0.smithi136.stdout:image: image19 2024-09-13T00:32:01.582 INFO:tasks.workunit.client.0.smithi136.stdout:expected = image18 2024-09-13T00:32:01.582 INFO:tasks.workunit.client.0.smithi136.stdout:found image18 2024-09-13T00:32:01.582 INFO:tasks.workunit.client.0.smithi136.stdout:erased image18 2024-09-13T00:32:01.582 INFO:tasks.workunit.client.0.smithi136.stdout:expected = image19 2024-09-13T00:32:01.582 INFO:tasks.workunit.client.0.smithi136.stdout:found image19 2024-09-13T00:32:01.582 INFO:tasks.workunit.client.0.smithi136.stdout:erased image19 2024-09-13T00:32:01.582 INFO:tasks.workunit.client.0.smithi136.stdout:image: image19 2024-09-13T00:32:01.582 INFO:tasks.workunit.client.0.smithi136.stdout:expected = image19 2024-09-13T00:32:01.583 INFO:tasks.workunit.client.0.smithi136.stdout:found image19 2024-09-13T00:32:01.583 INFO:tasks.workunit.client.0.smithi136.stdout:erased image19 2024-09-13T00:32:01.583 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestCreateLsDelete (3835 ms) 2024-09-13T00:32:01.583 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestCreateLsDeletePP 2024-09-13T00:32:01.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:01 smithi181 bash[67587]: cluster 2024-09-13T00:32:00.061209+0000 mgr.y (mgr.24661) 1646 : cluster [DBG] pgmap v2834: 169 pgs: 4 creating+peering, 24 unknown, 141 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 56 KiB/s rd, 10 KiB/s wr, 87 op/s 2024-09-13T00:32:01.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:01 smithi181 bash[67587]: cluster 2024-09-13T00:32:00.315815+0000 mon.a (mon.0) 2992 : cluster [DBG] osdmap e1628: 8 total, 8 up, 8 in 2024-09-13T00:32:01.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:01 smithi136 bash[75164]: cluster 2024-09-13T00:32:00.061209+0000 mgr.y (mgr.24661) 1646 : cluster [DBG] pgmap v2834: 169 pgs: 4 creating+peering, 24 unknown, 141 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 56 KiB/s rd, 10 KiB/s wr, 87 op/s 2024-09-13T00:32:01.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:01 smithi136 bash[75164]: cluster 2024-09-13T00:32:00.315815+0000 mon.a (mon.0) 2992 : cluster [DBG] osdmap e1628: 8 total, 8 up, 8 in 2024-09-13T00:32:01.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:01 smithi136 bash[76371]: cluster 2024-09-13T00:32:00.061209+0000 mgr.y (mgr.24661) 1646 : cluster [DBG] pgmap v2834: 169 pgs: 4 creating+peering, 24 unknown, 141 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 56 KiB/s rd, 10 KiB/s wr, 87 op/s 2024-09-13T00:32:01.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:01 smithi136 bash[76371]: cluster 2024-09-13T00:32:00.315815+0000 mon.a (mon.0) 2992 : cluster [DBG] osdmap e1628: 8 total, 8 up, 8 in 2024-09-13T00:32:02.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:02 smithi181 bash[67587]: cluster 2024-09-13T00:32:01.317526+0000 mon.a (mon.0) 2993 : cluster [DBG] osdmap e1629: 8 total, 8 up, 8 in 2024-09-13T00:32:02.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:32:02 smithi181 bash[95505]: ts=2024-09-13T00:32:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:32:02.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:02 smithi136 bash[75164]: cluster 2024-09-13T00:32:01.317526+0000 mon.a (mon.0) 2993 : cluster [DBG] osdmap e1629: 8 total, 8 up, 8 in 2024-09-13T00:32:02.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:02 smithi136 bash[76371]: cluster 2024-09-13T00:32:01.317526+0000 mon.a (mon.0) 2993 : cluster [DBG] osdmap e1629: 8 total, 8 up, 8 in 2024-09-13T00:32:03.367 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:32:03.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:03 smithi136 bash[75164]: cluster 2024-09-13T00:32:02.062200+0000 mgr.y (mgr.24661) 1647 : cluster [DBG] pgmap v2837: 169 pgs: 4 creating+peering, 4 unknown, 161 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 173 KiB/s rd, 16 KiB/s wr, 240 op/s 2024-09-13T00:32:03.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:03 smithi136 bash[75164]: cluster 2024-09-13T00:32:02.345881+0000 mon.a (mon.0) 2994 : cluster [DBG] osdmap e1630: 8 total, 8 up, 8 in 2024-09-13T00:32:03.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:03 smithi136 bash[75164]: audit 2024-09-13T00:32:02.351269+0000 mon.c (mon.2) 117 : audit [INF] from='client.? 172.21.15.136:0/2110304576' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-16","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:32:03.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:03 smithi136 bash[75164]: audit 2024-09-13T00:32:02.353675+0000 mon.a (mon.0) 2995 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-16","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:32:03.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:03 smithi136 bash[76371]: cluster 2024-09-13T00:32:02.062200+0000 mgr.y (mgr.24661) 1647 : cluster [DBG] pgmap v2837: 169 pgs: 4 creating+peering, 4 unknown, 161 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 173 KiB/s rd, 16 KiB/s wr, 240 op/s 2024-09-13T00:32:03.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:03 smithi136 bash[76371]: cluster 2024-09-13T00:32:02.345881+0000 mon.a (mon.0) 2994 : cluster [DBG] osdmap e1630: 8 total, 8 up, 8 in 2024-09-13T00:32:03.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:03 smithi136 bash[76371]: audit 2024-09-13T00:32:02.351269+0000 mon.c (mon.2) 117 : audit [INF] from='client.? 172.21.15.136:0/2110304576' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-16","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:32:03.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:03 smithi136 bash[76371]: audit 2024-09-13T00:32:02.353675+0000 mon.a (mon.0) 2995 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-16","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:32:03.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:03 smithi181 bash[67587]: cluster 2024-09-13T00:32:02.062200+0000 mgr.y (mgr.24661) 1647 : cluster [DBG] pgmap v2837: 169 pgs: 4 creating+peering, 4 unknown, 161 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 173 KiB/s rd, 16 KiB/s wr, 240 op/s 2024-09-13T00:32:03.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:03 smithi181 bash[67587]: cluster 2024-09-13T00:32:02.345881+0000 mon.a (mon.0) 2994 : cluster [DBG] osdmap e1630: 8 total, 8 up, 8 in 2024-09-13T00:32:03.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:03 smithi181 bash[67587]: audit 2024-09-13T00:32:02.351269+0000 mon.c (mon.2) 117 : audit [INF] from='client.? 172.21.15.136:0/2110304576' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-16","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:32:03.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:03 smithi181 bash[67587]: audit 2024-09-13T00:32:02.353675+0000 mon.a (mon.0) 2995 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-16","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:32:04.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:04 smithi136 bash[75164]: audit 2024-09-13T00:32:03.346440+0000 mon.a (mon.0) 2996 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-16","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:32:04.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:04 smithi136 bash[75164]: cluster 2024-09-13T00:32:03.346859+0000 mon.a (mon.0) 2997 : cluster [DBG] osdmap e1631: 8 total, 8 up, 8 in 2024-09-13T00:32:04.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:04 smithi136 bash[75164]: cluster 2024-09-13T00:32:04.130243+0000 mon.a (mon.0) 2998 : cluster [DBG] osdmap e1632: 8 total, 8 up, 8 in 2024-09-13T00:32:04.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:04 smithi136 bash[76371]: audit 2024-09-13T00:32:03.346440+0000 mon.a (mon.0) 2996 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-16","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:32:04.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:04 smithi136 bash[76371]: cluster 2024-09-13T00:32:03.346859+0000 mon.a (mon.0) 2997 : cluster [DBG] osdmap e1631: 8 total, 8 up, 8 in 2024-09-13T00:32:04.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:04 smithi136 bash[76371]: cluster 2024-09-13T00:32:04.130243+0000 mon.a (mon.0) 2998 : cluster [DBG] osdmap e1632: 8 total, 8 up, 8 in 2024-09-13T00:32:04.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:04 smithi181 bash[67587]: audit 2024-09-13T00:32:03.346440+0000 mon.a (mon.0) 2996 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-16","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:32:04.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:04 smithi181 bash[67587]: cluster 2024-09-13T00:32:03.346859+0000 mon.a (mon.0) 2997 : cluster [DBG] osdmap e1631: 8 total, 8 up, 8 in 2024-09-13T00:32:04.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:04 smithi181 bash[67587]: cluster 2024-09-13T00:32:04.130243+0000 mon.a (mon.0) 2998 : cluster [DBG] osdmap e1632: 8 total, 8 up, 8 in 2024-09-13T00:32:05.183 INFO:tasks.workunit.client.0.smithi136.stdout:num images is: 1 2024-09-13T00:32:05.184 INFO:tasks.workunit.client.0.smithi136.stdout:expected: 1 2024-09-13T00:32:05.184 INFO:tasks.workunit.client.0.smithi136.stdout:image: image20 2024-09-13T00:32:05.184 INFO:tasks.workunit.client.0.smithi136.stdout:expected = image20 2024-09-13T00:32:05.184 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:32:05.207 INFO:tasks.workunit.client.0.smithi136.stdout:num images is: 2 2024-09-13T00:32:05.207 INFO:tasks.workunit.client.0.smithi136.stdout:expected: 2 2024-09-13T00:32:05.207 INFO:tasks.workunit.client.0.smithi136.stdout:image: image20 2024-09-13T00:32:05.207 INFO:tasks.workunit.client.0.smithi136.stdout:image: image21 2024-09-13T00:32:05.207 INFO:tasks.workunit.client.0.smithi136.stdout:expected = image20 2024-09-13T00:32:05.207 INFO:tasks.workunit.client.0.smithi136.stdout:expected = image21 2024-09-13T00:32:05.360 INFO:tasks.workunit.client.0.smithi136.stdout:num images is: 1 2024-09-13T00:32:05.360 INFO:tasks.workunit.client.0.smithi136.stdout:expected: 1 2024-09-13T00:32:05.360 INFO:tasks.workunit.client.0.smithi136.stdout:image: image21 2024-09-13T00:32:05.361 INFO:tasks.workunit.client.0.smithi136.stdout:expected = image21 2024-09-13T00:32:05.361 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestCreateLsDeletePP (3778 ms) 2024-09-13T00:32:05.361 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestCopy 2024-09-13T00:32:05.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:05 smithi136 bash[75164]: cluster 2024-09-13T00:32:04.063216+0000 mgr.y (mgr.24661) 1648 : cluster [DBG] pgmap v2840: 201 pgs: 4 creating+peering, 25 unknown, 172 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 154 KiB/s rd, 11 KiB/s wr, 203 op/s 2024-09-13T00:32:05.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:05 smithi136 bash[75164]: cluster 2024-09-13T00:32:05.132596+0000 mon.a (mon.0) 2999 : cluster [DBG] osdmap e1633: 8 total, 8 up, 8 in 2024-09-13T00:32:05.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:05 smithi136 bash[76371]: cluster 2024-09-13T00:32:04.063216+0000 mgr.y (mgr.24661) 1648 : cluster [DBG] pgmap v2840: 201 pgs: 4 creating+peering, 25 unknown, 172 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 154 KiB/s rd, 11 KiB/s wr, 203 op/s 2024-09-13T00:32:05.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:05 smithi136 bash[76371]: cluster 2024-09-13T00:32:05.132596+0000 mon.a (mon.0) 2999 : cluster [DBG] osdmap e1633: 8 total, 8 up, 8 in 2024-09-13T00:32:05.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:05 smithi181 bash[67587]: cluster 2024-09-13T00:32:04.063216+0000 mgr.y (mgr.24661) 1648 : cluster [DBG] pgmap v2840: 201 pgs: 4 creating+peering, 25 unknown, 172 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 154 KiB/s rd, 11 KiB/s wr, 203 op/s 2024-09-13T00:32:05.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:05 smithi181 bash[67587]: cluster 2024-09-13T00:32:05.132596+0000 mon.a (mon.0) 2999 : cluster [DBG] osdmap e1633: 8 total, 8 up, 8 in 2024-09-13T00:32:07.406 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:32:07.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:07 smithi136 bash[75164]: cluster 2024-09-13T00:32:06.064315+0000 mgr.y (mgr.24661) 1649 : cluster [DBG] pgmap v2843: 201 pgs: 4 creating+peering, 11 unknown, 186 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 28 KiB/s rd, 4.0 KiB/s wr, 39 op/s 2024-09-13T00:32:07.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:07 smithi136 bash[75164]: cluster 2024-09-13T00:32:06.383800+0000 mon.a (mon.0) 3000 : cluster [DBG] osdmap e1634: 8 total, 8 up, 8 in 2024-09-13T00:32:07.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:07 smithi136 bash[75164]: audit 2024-09-13T00:32:06.400914+0000 mon.c (mon.2) 118 : audit [INF] from='client.? 172.21.15.136:0/1184660616' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-17","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:32:07.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:07 smithi136 bash[75164]: audit 2024-09-13T00:32:06.402060+0000 mon.a (mon.0) 3001 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-17","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:32:07.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:07 smithi136 bash[76371]: cluster 2024-09-13T00:32:06.064315+0000 mgr.y (mgr.24661) 1649 : cluster [DBG] pgmap v2843: 201 pgs: 4 creating+peering, 11 unknown, 186 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 28 KiB/s rd, 4.0 KiB/s wr, 39 op/s 2024-09-13T00:32:07.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:07 smithi136 bash[76371]: cluster 2024-09-13T00:32:06.383800+0000 mon.a (mon.0) 3000 : cluster [DBG] osdmap e1634: 8 total, 8 up, 8 in 2024-09-13T00:32:07.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:07 smithi136 bash[76371]: audit 2024-09-13T00:32:06.400914+0000 mon.c (mon.2) 118 : audit [INF] from='client.? 172.21.15.136:0/1184660616' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-17","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:32:07.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:07 smithi136 bash[76371]: audit 2024-09-13T00:32:06.402060+0000 mon.a (mon.0) 3001 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-17","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:32:07.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:07 smithi181 bash[67587]: cluster 2024-09-13T00:32:06.064315+0000 mgr.y (mgr.24661) 1649 : cluster [DBG] pgmap v2843: 201 pgs: 4 creating+peering, 11 unknown, 186 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 28 KiB/s rd, 4.0 KiB/s wr, 39 op/s 2024-09-13T00:32:07.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:07 smithi181 bash[67587]: cluster 2024-09-13T00:32:06.383800+0000 mon.a (mon.0) 3000 : cluster [DBG] osdmap e1634: 8 total, 8 up, 8 in 2024-09-13T00:32:07.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:07 smithi181 bash[67587]: audit 2024-09-13T00:32:06.400914+0000 mon.c (mon.2) 118 : audit [INF] from='client.? 172.21.15.136:0/1184660616' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-17","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:32:07.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:07 smithi181 bash[67587]: audit 2024-09-13T00:32:06.402060+0000 mon.a (mon.0) 3001 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-17","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:32:08.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:08 smithi136 bash[75164]: audit 2024-09-13T00:32:07.394495+0000 mon.a (mon.0) 3002 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-17","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:32:08.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:08 smithi136 bash[75164]: cluster 2024-09-13T00:32:07.394720+0000 mon.a (mon.0) 3003 : cluster [DBG] osdmap e1635: 8 total, 8 up, 8 in 2024-09-13T00:32:08.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:08 smithi136 bash[75164]: cluster 2024-09-13T00:32:08.065373+0000 mgr.y (mgr.24661) 1650 : cluster [DBG] pgmap v2846: 233 pgs: 11 creating+peering, 18 unknown, 204 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 25 KiB/s rd, 3.0 KiB/s wr, 38 op/s 2024-09-13T00:32:08.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:08 smithi136 bash[76371]: audit 2024-09-13T00:32:07.394495+0000 mon.a (mon.0) 3002 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-17","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:32:08.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:08 smithi136 bash[76371]: cluster 2024-09-13T00:32:07.394720+0000 mon.a (mon.0) 3003 : cluster [DBG] osdmap e1635: 8 total, 8 up, 8 in 2024-09-13T00:32:08.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:08 smithi136 bash[76371]: cluster 2024-09-13T00:32:08.065373+0000 mgr.y (mgr.24661) 1650 : cluster [DBG] pgmap v2846: 233 pgs: 11 creating+peering, 18 unknown, 204 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 25 KiB/s rd, 3.0 KiB/s wr, 38 op/s 2024-09-13T00:32:08.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:08 smithi181 bash[67587]: audit 2024-09-13T00:32:07.394495+0000 mon.a (mon.0) 3002 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-17","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:32:08.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:08 smithi181 bash[67587]: cluster 2024-09-13T00:32:07.394720+0000 mon.a (mon.0) 3003 : cluster [DBG] osdmap e1635: 8 total, 8 up, 8 in 2024-09-13T00:32:08.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:08 smithi181 bash[67587]: cluster 2024-09-13T00:32:08.065373+0000 mgr.y (mgr.24661) 1650 : cluster [DBG] pgmap v2846: 233 pgs: 11 creating+peering, 18 unknown, 204 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 25 KiB/s rd, 3.0 KiB/s wr, 38 op/s 2024-09-13T00:32:09.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:32:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:32:09] "GET /metrics HTTP/1.1" 200 43704 "" "Prometheus/2.43.0" 2024-09-13T00:32:09.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:09 smithi136 bash[75164]: cluster 2024-09-13T00:32:08.411396+0000 mon.a (mon.0) 3004 : cluster [DBG] osdmap e1636: 8 total, 8 up, 8 in 2024-09-13T00:32:09.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:09 smithi136 bash[75164]: audit 2024-09-13T00:32:08.825340+0000 mon.a (mon.0) 3005 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:32:09.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:09 smithi136 bash[75164]: cluster 2024-09-13T00:32:09.133489+0000 mon.a (mon.0) 3006 : cluster [DBG] osdmap e1637: 8 total, 8 up, 8 in 2024-09-13T00:32:09.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:09 smithi136 bash[76371]: cluster 2024-09-13T00:32:08.411396+0000 mon.a (mon.0) 3004 : cluster [DBG] osdmap e1636: 8 total, 8 up, 8 in 2024-09-13T00:32:09.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:09 smithi136 bash[76371]: audit 2024-09-13T00:32:08.825340+0000 mon.a (mon.0) 3005 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:32:09.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:09 smithi136 bash[76371]: cluster 2024-09-13T00:32:09.133489+0000 mon.a (mon.0) 3006 : cluster [DBG] osdmap e1637: 8 total, 8 up, 8 in 2024-09-13T00:32:09.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:09 smithi181 bash[67587]: cluster 2024-09-13T00:32:08.411396+0000 mon.a (mon.0) 3004 : cluster [DBG] osdmap e1636: 8 total, 8 up, 8 in 2024-09-13T00:32:09.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:09 smithi181 bash[67587]: audit 2024-09-13T00:32:08.825340+0000 mon.a (mon.0) 3005 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:32:09.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:09 smithi181 bash[67587]: cluster 2024-09-13T00:32:09.133489+0000 mon.a (mon.0) 3006 : cluster [DBG] osdmap e1637: 8 total, 8 up, 8 in 2024-09-13T00:32:10.139 INFO:tasks.workunit.client.0.smithi136.stdout:image: image22 2024-09-13T00:32:10.139 INFO:tasks.workunit.client.0.smithi136.stdout:expected = image22 2024-09-13T00:32:10.139 INFO:tasks.workunit.client.0.smithi136.stdout:found image22 2024-09-13T00:32:10.140 INFO:tasks.workunit.client.0.smithi136.stdout:erased image22 2024-09-13T00:32:10.140 INFO:tasks.workunit.client.0.smithi136.stdout:image: image22 2024-09-13T00:32:10.140 INFO:tasks.workunit.client.0.smithi136.stdout:image: image23 2024-09-13T00:32:10.140 INFO:tasks.workunit.client.0.smithi136.stdout:expected = image22 2024-09-13T00:32:10.140 INFO:tasks.workunit.client.0.smithi136.stdout:found image22 2024-09-13T00:32:10.140 INFO:tasks.workunit.client.0.smithi136.stdout:erased image22 2024-09-13T00:32:10.140 INFO:tasks.workunit.client.0.smithi136.stdout:expected = image23 2024-09-13T00:32:10.140 INFO:tasks.workunit.client.0.smithi136.stdout:found image23 2024-09-13T00:32:10.140 INFO:tasks.workunit.client.0.smithi136.stdout:erased image23 2024-09-13T00:32:10.140 INFO:tasks.workunit.client.0.smithi136.stdout:100.00% done 2024-09-13T00:32:10.140 INFO:tasks.workunit.client.0.smithi136.stdout:image: image22 2024-09-13T00:32:10.140 INFO:tasks.workunit.client.0.smithi136.stdout:image: image23 2024-09-13T00:32:10.140 INFO:tasks.workunit.client.0.smithi136.stdout:image: image24 2024-09-13T00:32:10.141 INFO:tasks.workunit.client.0.smithi136.stdout:expected = image22 2024-09-13T00:32:10.141 INFO:tasks.workunit.client.0.smithi136.stdout:found image22 2024-09-13T00:32:10.141 INFO:tasks.workunit.client.0.smithi136.stdout:erased image22 2024-09-13T00:32:10.141 INFO:tasks.workunit.client.0.smithi136.stdout:expected = image23 2024-09-13T00:32:10.141 INFO:tasks.workunit.client.0.smithi136.stdout:found image23 2024-09-13T00:32:10.141 INFO:tasks.workunit.client.0.smithi136.stdout:erased image23 2024-09-13T00:32:10.141 INFO:tasks.workunit.client.0.smithi136.stdout:expected = image24 2024-09-13T00:32:10.141 INFO:tasks.workunit.client.0.smithi136.stdout:found image24 2024-09-13T00:32:10.141 INFO:tasks.workunit.client.0.smithi136.stdout:erased image24 2024-09-13T00:32:10.141 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestCopy (4778 ms) 2024-09-13T00:32:10.141 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestCopyPP 2024-09-13T00:32:11.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:10 smithi136 bash[75164]: cluster 2024-09-13T00:32:10.066265+0000 mgr.y (mgr.24661) 1651 : cluster [DBG] pgmap v2849: 233 pgs: 11 creating+peering, 12 unknown, 210 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 26 KiB/s rd, 2.7 KiB/s wr, 40 op/s 2024-09-13T00:32:11.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:10 smithi136 bash[76371]: cluster 2024-09-13T00:32:10.066265+0000 mgr.y (mgr.24661) 1651 : cluster [DBG] pgmap v2849: 233 pgs: 11 creating+peering, 12 unknown, 210 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 26 KiB/s rd, 2.7 KiB/s wr, 40 op/s 2024-09-13T00:32:11.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:10 smithi181 bash[67587]: cluster 2024-09-13T00:32:10.066265+0000 mgr.y (mgr.24661) 1651 : cluster [DBG] pgmap v2849: 233 pgs: 11 creating+peering, 12 unknown, 210 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 26 KiB/s rd, 2.7 KiB/s wr, 40 op/s 2024-09-13T00:32:11.693 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:32:12.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:11 smithi136 bash[75164]: cluster 2024-09-13T00:32:10.681314+0000 mon.a (mon.0) 3007 : cluster [DBG] osdmap e1638: 8 total, 8 up, 8 in 2024-09-13T00:32:12.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:11 smithi136 bash[75164]: audit 2024-09-13T00:32:10.685704+0000 mon.c (mon.2) 119 : audit [INF] from='client.? 172.21.15.136:0/3667525983' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:32:12.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:11 smithi136 bash[75164]: audit 2024-09-13T00:32:10.692638+0000 mon.a (mon.0) 3008 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:32:12.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:11 smithi136 bash[75164]: audit 2024-09-13T00:32:11.684590+0000 mon.a (mon.0) 3009 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-18","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:32:12.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:11 smithi136 bash[75164]: cluster 2024-09-13T00:32:11.685750+0000 mon.a (mon.0) 3010 : cluster [DBG] osdmap e1639: 8 total, 8 up, 8 in 2024-09-13T00:32:12.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:11 smithi136 bash[76371]: cluster 2024-09-13T00:32:10.681314+0000 mon.a (mon.0) 3007 : cluster [DBG] osdmap e1638: 8 total, 8 up, 8 in 2024-09-13T00:32:12.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:11 smithi136 bash[76371]: audit 2024-09-13T00:32:10.685704+0000 mon.c (mon.2) 119 : audit [INF] from='client.? 172.21.15.136:0/3667525983' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:32:12.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:11 smithi136 bash[76371]: audit 2024-09-13T00:32:10.692638+0000 mon.a (mon.0) 3008 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:32:12.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:11 smithi136 bash[76371]: audit 2024-09-13T00:32:11.684590+0000 mon.a (mon.0) 3009 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-18","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:32:12.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:11 smithi136 bash[76371]: cluster 2024-09-13T00:32:11.685750+0000 mon.a (mon.0) 3010 : cluster [DBG] osdmap e1639: 8 total, 8 up, 8 in 2024-09-13T00:32:12.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:11 smithi181 bash[67587]: cluster 2024-09-13T00:32:10.681314+0000 mon.a (mon.0) 3007 : cluster [DBG] osdmap e1638: 8 total, 8 up, 8 in 2024-09-13T00:32:12.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:11 smithi181 bash[67587]: audit 2024-09-13T00:32:10.685704+0000 mon.c (mon.2) 119 : audit [INF] from='client.? 172.21.15.136:0/3667525983' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:32:12.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:11 smithi181 bash[67587]: audit 2024-09-13T00:32:10.692638+0000 mon.a (mon.0) 3008 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:32:12.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:11 smithi181 bash[67587]: audit 2024-09-13T00:32:11.684590+0000 mon.a (mon.0) 3009 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-18","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:32:12.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:11 smithi181 bash[67587]: cluster 2024-09-13T00:32:11.685750+0000 mon.a (mon.0) 3010 : cluster [DBG] osdmap e1639: 8 total, 8 up, 8 in 2024-09-13T00:32:12.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:32:12 smithi181 bash[95505]: ts=2024-09-13T00:32:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:32:13.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:12 smithi136 bash[75164]: cluster 2024-09-13T00:32:12.067052+0000 mgr.y (mgr.24661) 1652 : cluster [DBG] pgmap v2852: 265 pgs: 16 creating+peering, 16 unknown, 233 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 66 KiB/s rd, 4.7 KiB/s wr, 80 op/s 2024-09-13T00:32:13.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:12 smithi136 bash[76371]: cluster 2024-09-13T00:32:12.067052+0000 mgr.y (mgr.24661) 1652 : cluster [DBG] pgmap v2852: 265 pgs: 16 creating+peering, 16 unknown, 233 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 66 KiB/s rd, 4.7 KiB/s wr, 80 op/s 2024-09-13T00:32:13.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:12 smithi181 bash[67587]: cluster 2024-09-13T00:32:12.067052+0000 mgr.y (mgr.24661) 1652 : cluster [DBG] pgmap v2852: 265 pgs: 16 creating+peering, 16 unknown, 233 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 66 KiB/s rd, 4.7 KiB/s wr, 80 op/s 2024-09-13T00:32:14.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:13 smithi136 bash[76371]: cluster 2024-09-13T00:32:12.706506+0000 mon.a (mon.0) 3011 : cluster [DBG] osdmap e1640: 8 total, 8 up, 8 in 2024-09-13T00:32:14.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:13 smithi136 bash[75164]: cluster 2024-09-13T00:32:12.706506+0000 mon.a (mon.0) 3011 : cluster [DBG] osdmap e1640: 8 total, 8 up, 8 in 2024-09-13T00:32:14.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:13 smithi181 bash[67587]: cluster 2024-09-13T00:32:12.706506+0000 mon.a (mon.0) 3011 : cluster [DBG] osdmap e1640: 8 total, 8 up, 8 in 2024-09-13T00:32:14.474 INFO:tasks.workunit.client.0.smithi136.stdout:num images is: 1 2024-09-13T00:32:14.475 INFO:tasks.workunit.client.0.smithi136.stdout:expected: 1 2024-09-13T00:32:14.475 INFO:tasks.workunit.client.0.smithi136.stdout:image: image25 2024-09-13T00:32:14.475 INFO:tasks.workunit.client.0.smithi136.stdout:expected = image25 2024-09-13T00:32:14.530 INFO:tasks.workunit.client.0.smithi136.stdout:num images is: 2 2024-09-13T00:32:14.530 INFO:tasks.workunit.client.0.smithi136.stdout:expected: 2 2024-09-13T00:32:14.531 INFO:tasks.workunit.client.0.smithi136.stdout:image: image25 2024-09-13T00:32:14.531 INFO:tasks.workunit.client.0.smithi136.stdout:image: image26 2024-09-13T00:32:14.531 INFO:tasks.workunit.client.0.smithi136.stdout:expected = image25 2024-09-13T00:32:14.531 INFO:tasks.workunit.client.0.smithi136.stdout:expected = image26 2024-09-13T00:32:14.665 INFO:tasks.workunit.client.0.smithi136.stdout:100.00% done 2024-09-13T00:32:14.665 INFO:tasks.workunit.client.0.smithi136.stdout:num images is: 3 2024-09-13T00:32:14.665 INFO:tasks.workunit.client.0.smithi136.stdout:expected: 3 2024-09-13T00:32:14.665 INFO:tasks.workunit.client.0.smithi136.stdout:image: image25 2024-09-13T00:32:14.665 INFO:tasks.workunit.client.0.smithi136.stdout:image: image26 2024-09-13T00:32:14.665 INFO:tasks.workunit.client.0.smithi136.stdout:image: image27 2024-09-13T00:32:14.665 INFO:tasks.workunit.client.0.smithi136.stdout:expected = image25 2024-09-13T00:32:14.666 INFO:tasks.workunit.client.0.smithi136.stdout:expected = image26 2024-09-13T00:32:14.666 INFO:tasks.workunit.client.0.smithi136.stdout:expected = image27 2024-09-13T00:32:14.750 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestCopyPP (4610 ms) 2024-09-13T00:32:14.750 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestDeepCopy 2024-09-13T00:32:15.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:14 smithi136 bash[76371]: cluster 2024-09-13T00:32:13.724641+0000 mon.a (mon.0) 3012 : cluster [DBG] osdmap e1641: 8 total, 8 up, 8 in 2024-09-13T00:32:15.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:14 smithi136 bash[76371]: cluster 2024-09-13T00:32:14.068042+0000 mgr.y (mgr.24661) 1653 : cluster [DBG] pgmap v2855: 265 pgs: 16 creating+peering, 7 unknown, 242 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 341 KiB/s rd, 22 KiB/s wr, 398 op/s 2024-09-13T00:32:15.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:14 smithi136 bash[76371]: cluster 2024-09-13T00:32:14.136529+0000 mon.a (mon.0) 3013 : cluster [DBG] osdmap e1642: 8 total, 8 up, 8 in 2024-09-13T00:32:15.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:14 smithi136 bash[75164]: cluster 2024-09-13T00:32:13.724641+0000 mon.a (mon.0) 3012 : cluster [DBG] osdmap e1641: 8 total, 8 up, 8 in 2024-09-13T00:32:15.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:14 smithi136 bash[75164]: cluster 2024-09-13T00:32:14.068042+0000 mgr.y (mgr.24661) 1653 : cluster [DBG] pgmap v2855: 265 pgs: 16 creating+peering, 7 unknown, 242 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 341 KiB/s rd, 22 KiB/s wr, 398 op/s 2024-09-13T00:32:15.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:14 smithi136 bash[75164]: cluster 2024-09-13T00:32:14.136529+0000 mon.a (mon.0) 3013 : cluster [DBG] osdmap e1642: 8 total, 8 up, 8 in 2024-09-13T00:32:15.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:14 smithi181 bash[67587]: cluster 2024-09-13T00:32:13.724641+0000 mon.a (mon.0) 3012 : cluster [DBG] osdmap e1641: 8 total, 8 up, 8 in 2024-09-13T00:32:15.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:14 smithi181 bash[67587]: cluster 2024-09-13T00:32:14.068042+0000 mgr.y (mgr.24661) 1653 : cluster [DBG] pgmap v2855: 265 pgs: 16 creating+peering, 7 unknown, 242 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 341 KiB/s rd, 22 KiB/s wr, 398 op/s 2024-09-13T00:32:15.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:14 smithi181 bash[67587]: cluster 2024-09-13T00:32:14.136529+0000 mon.a (mon.0) 3013 : cluster [DBG] osdmap e1642: 8 total, 8 up, 8 in 2024-09-13T00:32:16.784 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:32:17.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:16 smithi181 bash[67587]: cluster 2024-09-13T00:32:15.769374+0000 mon.a (mon.0) 3014 : cluster [DBG] osdmap e1643: 8 total, 8 up, 8 in 2024-09-13T00:32:17.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:16 smithi181 bash[67587]: audit 2024-09-13T00:32:15.771892+0000 mon.c (mon.2) 120 : audit [INF] from='client.? 172.21.15.136:0/1164141046' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:32:17.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:16 smithi181 bash[67587]: audit 2024-09-13T00:32:15.779634+0000 mon.a (mon.0) 3015 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:32:17.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:16 smithi181 bash[67587]: cluster 2024-09-13T00:32:16.069364+0000 mgr.y (mgr.24661) 1654 : cluster [DBG] pgmap v2858: 297 pgs: 32 unknown, 12 creating+peering, 253 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 309 KiB/s rd, 20 KiB/s wr, 361 op/s 2024-09-13T00:32:17.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:16 smithi136 bash[75164]: cluster 2024-09-13T00:32:15.769374+0000 mon.a (mon.0) 3014 : cluster [DBG] osdmap e1643: 8 total, 8 up, 8 in 2024-09-13T00:32:17.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:16 smithi136 bash[75164]: audit 2024-09-13T00:32:15.771892+0000 mon.c (mon.2) 120 : audit [INF] from='client.? 172.21.15.136:0/1164141046' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:32:17.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:16 smithi136 bash[75164]: audit 2024-09-13T00:32:15.779634+0000 mon.a (mon.0) 3015 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:32:17.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:16 smithi136 bash[75164]: cluster 2024-09-13T00:32:16.069364+0000 mgr.y (mgr.24661) 1654 : cluster [DBG] pgmap v2858: 297 pgs: 32 unknown, 12 creating+peering, 253 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 309 KiB/s rd, 20 KiB/s wr, 361 op/s 2024-09-13T00:32:17.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:16 smithi136 bash[76371]: cluster 2024-09-13T00:32:15.769374+0000 mon.a (mon.0) 3014 : cluster [DBG] osdmap e1643: 8 total, 8 up, 8 in 2024-09-13T00:32:17.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:16 smithi136 bash[76371]: audit 2024-09-13T00:32:15.771892+0000 mon.c (mon.2) 120 : audit [INF] from='client.? 172.21.15.136:0/1164141046' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:32:17.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:16 smithi136 bash[76371]: audit 2024-09-13T00:32:15.779634+0000 mon.a (mon.0) 3015 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:32:17.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:16 smithi136 bash[76371]: cluster 2024-09-13T00:32:16.069364+0000 mgr.y (mgr.24661) 1654 : cluster [DBG] pgmap v2858: 297 pgs: 32 unknown, 12 creating+peering, 253 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 309 KiB/s rd, 20 KiB/s wr, 361 op/s 2024-09-13T00:32:18.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:17 smithi181 bash[67587]: cluster 2024-09-13T00:32:16.756764+0000 mon.a (mon.0) 3016 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:32:18.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:17 smithi181 bash[67587]: audit 2024-09-13T00:32:16.775478+0000 mon.a (mon.0) 3017 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-19","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:32:18.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:17 smithi181 bash[67587]: cluster 2024-09-13T00:32:16.775956+0000 mon.a (mon.0) 3018 : cluster [DBG] osdmap e1644: 8 total, 8 up, 8 in 2024-09-13T00:32:18.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:17 smithi136 bash[75164]: cluster 2024-09-13T00:32:16.756764+0000 mon.a (mon.0) 3016 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:32:18.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:17 smithi136 bash[75164]: audit 2024-09-13T00:32:16.775478+0000 mon.a (mon.0) 3017 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-19","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:32:18.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:17 smithi136 bash[75164]: cluster 2024-09-13T00:32:16.775956+0000 mon.a (mon.0) 3018 : cluster [DBG] osdmap e1644: 8 total, 8 up, 8 in 2024-09-13T00:32:18.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:17 smithi136 bash[76371]: cluster 2024-09-13T00:32:16.756764+0000 mon.a (mon.0) 3016 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:32:18.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:17 smithi136 bash[76371]: audit 2024-09-13T00:32:16.775478+0000 mon.a (mon.0) 3017 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-19","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:32:18.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:17 smithi136 bash[76371]: cluster 2024-09-13T00:32:16.775956+0000 mon.a (mon.0) 3018 : cluster [DBG] osdmap e1644: 8 total, 8 up, 8 in 2024-09-13T00:32:19.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:18 smithi181 bash[67587]: cluster 2024-09-13T00:32:17.776844+0000 mon.a (mon.0) 3019 : cluster [DBG] osdmap e1645: 8 total, 8 up, 8 in 2024-09-13T00:32:19.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:18 smithi181 bash[67587]: cluster 2024-09-13T00:32:18.070247+0000 mgr.y (mgr.24661) 1655 : cluster [DBG] pgmap v2861: 297 pgs: 12 creating+peering, 13 unknown, 272 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 82 KiB/s rd, 3.7 KiB/s wr, 92 op/s 2024-09-13T00:32:19.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:18 smithi136 bash[76371]: cluster 2024-09-13T00:32:17.776844+0000 mon.a (mon.0) 3019 : cluster [DBG] osdmap e1645: 8 total, 8 up, 8 in 2024-09-13T00:32:19.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:18 smithi136 bash[76371]: cluster 2024-09-13T00:32:18.070247+0000 mgr.y (mgr.24661) 1655 : cluster [DBG] pgmap v2861: 297 pgs: 12 creating+peering, 13 unknown, 272 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 82 KiB/s rd, 3.7 KiB/s wr, 92 op/s 2024-09-13T00:32:19.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:18 smithi136 bash[75164]: cluster 2024-09-13T00:32:17.776844+0000 mon.a (mon.0) 3019 : cluster [DBG] osdmap e1645: 8 total, 8 up, 8 in 2024-09-13T00:32:19.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:18 smithi136 bash[75164]: cluster 2024-09-13T00:32:18.070247+0000 mgr.y (mgr.24661) 1655 : cluster [DBG] pgmap v2861: 297 pgs: 12 creating+peering, 13 unknown, 272 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 82 KiB/s rd, 3.7 KiB/s wr, 92 op/s 2024-09-13T00:32:19.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:32:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:32:19] "GET /metrics HTTP/1.1" 200 43704 "" "Prometheus/2.43.0" 2024-09-13T00:32:20.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:19 smithi181 bash[67587]: cluster 2024-09-13T00:32:18.780820+0000 mon.a (mon.0) 3020 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:32:20.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:19 smithi181 bash[67587]: cluster 2024-09-13T00:32:18.780865+0000 mon.a (mon.0) 3021 : cluster [INF] Cluster is now healthy 2024-09-13T00:32:20.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:19 smithi181 bash[67587]: cluster 2024-09-13T00:32:18.800263+0000 mon.a (mon.0) 3022 : cluster [DBG] osdmap e1646: 8 total, 8 up, 8 in 2024-09-13T00:32:20.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:19 smithi181 bash[67587]: cluster 2024-09-13T00:32:19.151325+0000 mon.a (mon.0) 3023 : cluster [DBG] osdmap e1647: 8 total, 8 up, 8 in 2024-09-13T00:32:20.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:19 smithi136 bash[75164]: cluster 2024-09-13T00:32:18.780820+0000 mon.a (mon.0) 3020 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:32:20.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:19 smithi136 bash[75164]: cluster 2024-09-13T00:32:18.780865+0000 mon.a (mon.0) 3021 : cluster [INF] Cluster is now healthy 2024-09-13T00:32:20.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:19 smithi136 bash[75164]: cluster 2024-09-13T00:32:18.800263+0000 mon.a (mon.0) 3022 : cluster [DBG] osdmap e1646: 8 total, 8 up, 8 in 2024-09-13T00:32:20.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:19 smithi136 bash[75164]: cluster 2024-09-13T00:32:19.151325+0000 mon.a (mon.0) 3023 : cluster [DBG] osdmap e1647: 8 total, 8 up, 8 in 2024-09-13T00:32:20.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:19 smithi136 bash[76371]: cluster 2024-09-13T00:32:18.780820+0000 mon.a (mon.0) 3020 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:32:20.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:19 smithi136 bash[76371]: cluster 2024-09-13T00:32:18.780865+0000 mon.a (mon.0) 3021 : cluster [INF] Cluster is now healthy 2024-09-13T00:32:20.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:19 smithi136 bash[76371]: cluster 2024-09-13T00:32:18.800263+0000 mon.a (mon.0) 3022 : cluster [DBG] osdmap e1646: 8 total, 8 up, 8 in 2024-09-13T00:32:20.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:19 smithi136 bash[76371]: cluster 2024-09-13T00:32:19.151325+0000 mon.a (mon.0) 3023 : cluster [DBG] osdmap e1647: 8 total, 8 up, 8 in 2024-09-13T00:32:21.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:20 smithi181 bash[67587]: cluster 2024-09-13T00:32:20.071414+0000 mgr.y (mgr.24661) 1656 : cluster [DBG] pgmap v2864: 297 pgs: 12 creating+peering, 8 unknown, 277 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 328 KiB/s rd, 21 KiB/s wr, 380 op/s 2024-09-13T00:32:21.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:20 smithi136 bash[75164]: cluster 2024-09-13T00:32:20.071414+0000 mgr.y (mgr.24661) 1656 : cluster [DBG] pgmap v2864: 297 pgs: 12 creating+peering, 8 unknown, 277 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 328 KiB/s rd, 21 KiB/s wr, 380 op/s 2024-09-13T00:32:21.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:20 smithi136 bash[76371]: cluster 2024-09-13T00:32:20.071414+0000 mgr.y (mgr.24661) 1656 : cluster [DBG] pgmap v2864: 297 pgs: 12 creating+peering, 8 unknown, 277 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 328 KiB/s rd, 21 KiB/s wr, 380 op/s 2024-09-13T00:32:22.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:21 smithi181 bash[67587]: cluster 2024-09-13T00:32:20.812324+0000 mon.a (mon.0) 3024 : cluster [DBG] osdmap e1648: 8 total, 8 up, 8 in 2024-09-13T00:32:22.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:21 smithi136 bash[75164]: cluster 2024-09-13T00:32:20.812324+0000 mon.a (mon.0) 3024 : cluster [DBG] osdmap e1648: 8 total, 8 up, 8 in 2024-09-13T00:32:22.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:21 smithi136 bash[76371]: cluster 2024-09-13T00:32:20.812324+0000 mon.a (mon.0) 3024 : cluster [DBG] osdmap e1648: 8 total, 8 up, 8 in 2024-09-13T00:32:22.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:32:22 smithi181 bash[95505]: ts=2024-09-13T00:32:22.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:32:23.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:22 smithi181 bash[67587]: cluster 2024-09-13T00:32:21.819812+0000 mon.a (mon.0) 3025 : cluster [DBG] osdmap e1649: 8 total, 8 up, 8 in 2024-09-13T00:32:23.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:22 smithi181 bash[67587]: cluster 2024-09-13T00:32:22.072283+0000 mgr.y (mgr.24661) 1657 : cluster [DBG] pgmap v2867: 297 pgs: 297 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 649 KiB/s rd, 42 KiB/s wr, 767 op/s 2024-09-13T00:32:23.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:22 smithi136 bash[75164]: cluster 2024-09-13T00:32:21.819812+0000 mon.a (mon.0) 3025 : cluster [DBG] osdmap e1649: 8 total, 8 up, 8 in 2024-09-13T00:32:23.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:22 smithi136 bash[75164]: cluster 2024-09-13T00:32:22.072283+0000 mgr.y (mgr.24661) 1657 : cluster [DBG] pgmap v2867: 297 pgs: 297 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 649 KiB/s rd, 42 KiB/s wr, 767 op/s 2024-09-13T00:32:23.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:22 smithi136 bash[76371]: cluster 2024-09-13T00:32:21.819812+0000 mon.a (mon.0) 3025 : cluster [DBG] osdmap e1649: 8 total, 8 up, 8 in 2024-09-13T00:32:23.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:22 smithi136 bash[76371]: cluster 2024-09-13T00:32:22.072283+0000 mgr.y (mgr.24661) 1657 : cluster [DBG] pgmap v2867: 297 pgs: 297 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 649 KiB/s rd, 42 KiB/s wr, 767 op/s 2024-09-13T00:32:24.044 INFO:tasks.workunit.client.0.smithi136.stdout:image: image28 2024-09-13T00:32:24.044 INFO:tasks.workunit.client.0.smithi136.stdout:expected = image28 2024-09-13T00:32:24.044 INFO:tasks.workunit.client.0.smithi136.stdout:found image28 2024-09-13T00:32:24.044 INFO:tasks.workunit.client.0.smithi136.stdout:erased image28 2024-09-13T00:32:24.044 INFO:tasks.workunit.client.0.smithi136.stdout:image: image28 2024-09-13T00:32:24.044 INFO:tasks.workunit.client.0.smithi136.stdout:image: image29 2024-09-13T00:32:24.044 INFO:tasks.workunit.client.0.smithi136.stdout:expected = image28 2024-09-13T00:32:24.044 INFO:tasks.workunit.client.0.smithi136.stdout:found image28 2024-09-13T00:32:24.044 INFO:tasks.workunit.client.0.smithi136.stdout:erased image28 2024-09-13T00:32:24.044 INFO:tasks.workunit.client.0.smithi136.stdout:expected = image29 2024-09-13T00:32:24.044 INFO:tasks.workunit.client.0.smithi136.stdout:found image29 2024-09-13T00:32:24.044 INFO:tasks.workunit.client.0.smithi136.stdout:erased image29 2024-09-13T00:32:24.045 INFO:tasks.workunit.client.0.smithi136.stdout:100.00% done 2024-09-13T00:32:24.045 INFO:tasks.workunit.client.0.smithi136.stdout:image: image28 2024-09-13T00:32:24.045 INFO:tasks.workunit.client.0.smithi136.stdout:image: image29 2024-09-13T00:32:24.045 INFO:tasks.workunit.client.0.smithi136.stdout:image: image30 2024-09-13T00:32:24.045 INFO:tasks.workunit.client.0.smithi136.stdout:expected = image28 2024-09-13T00:32:24.045 INFO:tasks.workunit.client.0.smithi136.stdout:found image28 2024-09-13T00:32:24.045 INFO:tasks.workunit.client.0.smithi136.stdout:erased image28 2024-09-13T00:32:24.045 INFO:tasks.workunit.client.0.smithi136.stdout:expected = image29 2024-09-13T00:32:24.045 INFO:tasks.workunit.client.0.smithi136.stdout:found image29 2024-09-13T00:32:24.045 INFO:tasks.workunit.client.0.smithi136.stdout:erased image29 2024-09-13T00:32:24.045 INFO:tasks.workunit.client.0.smithi136.stdout:expected = image30 2024-09-13T00:32:24.045 INFO:tasks.workunit.client.0.smithi136.stdout:found image30 2024-09-13T00:32:24.045 INFO:tasks.workunit.client.0.smithi136.stdout:erased image30 2024-09-13T00:32:24.046 INFO:tasks.workunit.client.0.smithi136.stdout:image: image28 2024-09-13T00:32:24.046 INFO:tasks.workunit.client.0.smithi136.stdout:image: image29 2024-09-13T00:32:24.046 INFO:tasks.workunit.client.0.smithi136.stdout:image: image30 2024-09-13T00:32:24.046 INFO:tasks.workunit.client.0.smithi136.stdout:image: image31 2024-09-13T00:32:24.046 INFO:tasks.workunit.client.0.smithi136.stdout:expected = image28 2024-09-13T00:32:24.046 INFO:tasks.workunit.client.0.smithi136.stdout:found image28 2024-09-13T00:32:24.046 INFO:tasks.workunit.client.0.smithi136.stdout:erased image28 2024-09-13T00:32:24.046 INFO:tasks.workunit.client.0.smithi136.stdout:expected = image29 2024-09-13T00:32:24.046 INFO:tasks.workunit.client.0.smithi136.stdout:found image29 2024-09-13T00:32:24.046 INFO:tasks.workunit.client.0.smithi136.stdout:erased image29 2024-09-13T00:32:24.046 INFO:tasks.workunit.client.0.smithi136.stdout:expected = image30 2024-09-13T00:32:24.046 INFO:tasks.workunit.client.0.smithi136.stdout:found image30 2024-09-13T00:32:24.046 INFO:tasks.workunit.client.0.smithi136.stdout:erased image30 2024-09-13T00:32:24.047 INFO:tasks.workunit.client.0.smithi136.stdout:expected = image31 2024-09-13T00:32:24.047 INFO:tasks.workunit.client.0.smithi136.stdout:found image31 2024-09-13T00:32:24.047 INFO:tasks.workunit.client.0.smithi136.stdout:erased image31 2024-09-13T00:32:24.047 INFO:tasks.workunit.client.0.smithi136.stdout:image: image28 2024-09-13T00:32:24.047 INFO:tasks.workunit.client.0.smithi136.stdout:image: image29 2024-09-13T00:32:24.047 INFO:tasks.workunit.client.0.smithi136.stdout:image: image30 2024-09-13T00:32:24.047 INFO:tasks.workunit.client.0.smithi136.stdout:image: image31 2024-09-13T00:32:24.047 INFO:tasks.workunit.client.0.smithi136.stdout:image: image32 2024-09-13T00:32:24.047 INFO:tasks.workunit.client.0.smithi136.stdout:expected = image28 2024-09-13T00:32:24.047 INFO:tasks.workunit.client.0.smithi136.stdout:found image28 2024-09-13T00:32:24.047 INFO:tasks.workunit.client.0.smithi136.stdout:erased image28 2024-09-13T00:32:24.047 INFO:tasks.workunit.client.0.smithi136.stdout:expected = image29 2024-09-13T00:32:24.047 INFO:tasks.workunit.client.0.smithi136.stdout:found image29 2024-09-13T00:32:24.047 INFO:tasks.workunit.client.0.smithi136.stdout:erased image29 2024-09-13T00:32:24.047 INFO:tasks.workunit.client.0.smithi136.stdout:expected = image30 2024-09-13T00:32:24.048 INFO:tasks.workunit.client.0.smithi136.stdout:found image30 2024-09-13T00:32:24.048 INFO:tasks.workunit.client.0.smithi136.stdout:erased image30 2024-09-13T00:32:24.048 INFO:tasks.workunit.client.0.smithi136.stdout:expected = image31 2024-09-13T00:32:24.048 INFO:tasks.workunit.client.0.smithi136.stdout:found image31 2024-09-13T00:32:24.048 INFO:tasks.workunit.client.0.smithi136.stdout:erased image31 2024-09-13T00:32:24.048 INFO:tasks.workunit.client.0.smithi136.stdout:expected = image32 2024-09-13T00:32:24.048 INFO:tasks.workunit.client.0.smithi136.stdout:found image32 2024-09-13T00:32:24.048 INFO:tasks.workunit.client.0.smithi136.stdout:erased image32 2024-09-13T00:32:24.048 INFO:tasks.workunit.client.0.smithi136.stdout:100.00% done 2024-09-13T00:32:24.048 INFO:tasks.workunit.client.0.smithi136.stdout:image: image28 2024-09-13T00:32:24.048 INFO:tasks.workunit.client.0.smithi136.stdout:image: image29 2024-09-13T00:32:24.048 INFO:tasks.workunit.client.0.smithi136.stdout:image: image30 2024-09-13T00:32:24.048 INFO:tasks.workunit.client.0.smithi136.stdout:image: image31 2024-09-13T00:32:24.048 INFO:tasks.workunit.client.0.smithi136.stdout:image: image32 2024-09-13T00:32:24.048 INFO:tasks.workunit.client.0.smithi136.stdout:image: image33 2024-09-13T00:32:24.049 INFO:tasks.workunit.client.0.smithi136.stdout:expected = image28 2024-09-13T00:32:24.049 INFO:tasks.workunit.client.0.smithi136.stdout:found image28 2024-09-13T00:32:24.049 INFO:tasks.workunit.client.0.smithi136.stdout:erased image28 2024-09-13T00:32:24.049 INFO:tasks.workunit.client.0.smithi136.stdout:expected = image29 2024-09-13T00:32:24.049 INFO:tasks.workunit.client.0.smithi136.stdout:found image29 2024-09-13T00:32:24.049 INFO:tasks.workunit.client.0.smithi136.stdout:erased image29 2024-09-13T00:32:24.049 INFO:tasks.workunit.client.0.smithi136.stdout:expected = image30 2024-09-13T00:32:24.049 INFO:tasks.workunit.client.0.smithi136.stdout:found image30 2024-09-13T00:32:24.049 INFO:tasks.workunit.client.0.smithi136.stdout:erased image30 2024-09-13T00:32:24.049 INFO:tasks.workunit.client.0.smithi136.stdout:expected = image31 2024-09-13T00:32:24.049 INFO:tasks.workunit.client.0.smithi136.stdout:found image31 2024-09-13T00:32:24.049 INFO:tasks.workunit.client.0.smithi136.stdout:erased image31 2024-09-13T00:32:24.049 INFO:tasks.workunit.client.0.smithi136.stdout:expected = image32 2024-09-13T00:32:24.050 INFO:tasks.workunit.client.0.smithi136.stdout:found image32 2024-09-13T00:32:24.050 INFO:tasks.workunit.client.0.smithi136.stdout:erased image32 2024-09-13T00:32:24.050 INFO:tasks.workunit.client.0.smithi136.stdout:expected = image33 2024-09-13T00:32:24.050 INFO:tasks.workunit.client.0.smithi136.stdout:found image33 2024-09-13T00:32:24.050 INFO:tasks.workunit.client.0.smithi136.stdout:erased image33 2024-09-13T00:32:24.050 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestDeepCopy (9294 ms) 2024-09-13T00:32:24.050 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestDeepCopyPP 2024-09-13T00:32:24.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:23 smithi181 bash[67587]: cluster 2024-09-13T00:32:22.829949+0000 mon.a (mon.0) 3026 : cluster [DBG] osdmap e1650: 8 total, 8 up, 8 in 2024-09-13T00:32:24.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:23 smithi136 bash[75164]: cluster 2024-09-13T00:32:22.829949+0000 mon.a (mon.0) 3026 : cluster [DBG] osdmap e1650: 8 total, 8 up, 8 in 2024-09-13T00:32:24.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:23 smithi136 bash[76371]: cluster 2024-09-13T00:32:22.829949+0000 mon.a (mon.0) 3026 : cluster [DBG] osdmap e1650: 8 total, 8 up, 8 in 2024-09-13T00:32:25.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:24 smithi181 bash[67587]: cluster 2024-09-13T00:32:23.833029+0000 mon.a (mon.0) 3027 : cluster [DBG] osdmap e1651: 8 total, 8 up, 8 in 2024-09-13T00:32:25.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:24 smithi181 bash[67587]: audit 2024-09-13T00:32:23.836833+0000 mon.a (mon.0) 3028 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:32:25.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:24 smithi181 bash[67587]: cluster 2024-09-13T00:32:24.073158+0000 mgr.y (mgr.24661) 1658 : cluster [DBG] pgmap v2870: 297 pgs: 297 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 386 KiB/s rd, 25 KiB/s wr, 461 op/s 2024-09-13T00:32:25.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:24 smithi181 bash[67587]: cluster 2024-09-13T00:32:24.527763+0000 mon.a (mon.0) 3029 : cluster [DBG] osdmap e1652: 8 total, 8 up, 8 in 2024-09-13T00:32:25.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:24 smithi181 bash[67587]: audit 2024-09-13T00:32:24.534005+0000 mon.a (mon.0) 3030 : audit [INF] from='client.? 172.21.15.136:0/1973331762' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:32:25.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:24 smithi136 bash[75164]: cluster 2024-09-13T00:32:23.833029+0000 mon.a (mon.0) 3027 : cluster [DBG] osdmap e1651: 8 total, 8 up, 8 in 2024-09-13T00:32:25.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:24 smithi136 bash[75164]: audit 2024-09-13T00:32:23.836833+0000 mon.a (mon.0) 3028 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:32:25.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:24 smithi136 bash[75164]: cluster 2024-09-13T00:32:24.073158+0000 mgr.y (mgr.24661) 1658 : cluster [DBG] pgmap v2870: 297 pgs: 297 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 386 KiB/s rd, 25 KiB/s wr, 461 op/s 2024-09-13T00:32:25.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:24 smithi136 bash[75164]: cluster 2024-09-13T00:32:24.527763+0000 mon.a (mon.0) 3029 : cluster [DBG] osdmap e1652: 8 total, 8 up, 8 in 2024-09-13T00:32:25.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:24 smithi136 bash[75164]: audit 2024-09-13T00:32:24.534005+0000 mon.a (mon.0) 3030 : audit [INF] from='client.? 172.21.15.136:0/1973331762' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:32:25.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:24 smithi136 bash[76371]: cluster 2024-09-13T00:32:23.833029+0000 mon.a (mon.0) 3027 : cluster [DBG] osdmap e1651: 8 total, 8 up, 8 in 2024-09-13T00:32:25.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:24 smithi136 bash[76371]: audit 2024-09-13T00:32:23.836833+0000 mon.a (mon.0) 3028 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:32:25.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:24 smithi136 bash[76371]: cluster 2024-09-13T00:32:24.073158+0000 mgr.y (mgr.24661) 1658 : cluster [DBG] pgmap v2870: 297 pgs: 297 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 386 KiB/s rd, 25 KiB/s wr, 461 op/s 2024-09-13T00:32:25.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:24 smithi136 bash[76371]: cluster 2024-09-13T00:32:24.527763+0000 mon.a (mon.0) 3029 : cluster [DBG] osdmap e1652: 8 total, 8 up, 8 in 2024-09-13T00:32:25.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:24 smithi136 bash[76371]: audit 2024-09-13T00:32:24.534005+0000 mon.a (mon.0) 3030 : audit [INF] from='client.? 172.21.15.136:0/1973331762' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:32:25.541 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:32:26.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:26 smithi181 bash[67587]: audit 2024-09-13T00:32:25.533167+0000 mon.a (mon.0) 3031 : audit [INF] from='client.? 172.21.15.136:0/1973331762' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-20","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:32:26.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:26 smithi181 bash[67587]: cluster 2024-09-13T00:32:25.533663+0000 mon.a (mon.0) 3032 : cluster [DBG] osdmap e1653: 8 total, 8 up, 8 in 2024-09-13T00:32:26.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:26 smithi181 bash[67587]: cluster 2024-09-13T00:32:26.074372+0000 mgr.y (mgr.24661) 1659 : cluster [DBG] pgmap v2873: 329 pgs: 5 creating+peering, 22 unknown, 302 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 27 KiB/s rd, 2.5 KiB/s wr, 39 op/s 2024-09-13T00:32:27.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:26 smithi136 bash[75164]: audit 2024-09-13T00:32:25.533167+0000 mon.a (mon.0) 3031 : audit [INF] from='client.? 172.21.15.136:0/1973331762' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-20","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:32:27.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:26 smithi136 bash[75164]: cluster 2024-09-13T00:32:25.533663+0000 mon.a (mon.0) 3032 : cluster [DBG] osdmap e1653: 8 total, 8 up, 8 in 2024-09-13T00:32:27.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:26 smithi136 bash[75164]: cluster 2024-09-13T00:32:26.074372+0000 mgr.y (mgr.24661) 1659 : cluster [DBG] pgmap v2873: 329 pgs: 5 creating+peering, 22 unknown, 302 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 27 KiB/s rd, 2.5 KiB/s wr, 39 op/s 2024-09-13T00:32:27.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:26 smithi136 bash[76371]: audit 2024-09-13T00:32:25.533167+0000 mon.a (mon.0) 3031 : audit [INF] from='client.? 172.21.15.136:0/1973331762' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-20","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:32:27.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:26 smithi136 bash[76371]: cluster 2024-09-13T00:32:25.533663+0000 mon.a (mon.0) 3032 : cluster [DBG] osdmap e1653: 8 total, 8 up, 8 in 2024-09-13T00:32:27.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:26 smithi136 bash[76371]: cluster 2024-09-13T00:32:26.074372+0000 mgr.y (mgr.24661) 1659 : cluster [DBG] pgmap v2873: 329 pgs: 5 creating+peering, 22 unknown, 302 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 27 KiB/s rd, 2.5 KiB/s wr, 39 op/s 2024-09-13T00:32:27.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:27 smithi181 bash[67587]: cluster 2024-09-13T00:32:26.542466+0000 mon.a (mon.0) 3033 : cluster [DBG] osdmap e1654: 8 total, 8 up, 8 in 2024-09-13T00:32:28.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:27 smithi136 bash[76371]: cluster 2024-09-13T00:32:26.542466+0000 mon.a (mon.0) 3033 : cluster [DBG] osdmap e1654: 8 total, 8 up, 8 in 2024-09-13T00:32:28.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:27 smithi136 bash[75164]: cluster 2024-09-13T00:32:26.542466+0000 mon.a (mon.0) 3033 : cluster [DBG] osdmap e1654: 8 total, 8 up, 8 in 2024-09-13T00:32:28.193 INFO:tasks.workunit.client.0.smithi136.stdout:num images is: 1 2024-09-13T00:32:28.194 INFO:tasks.workunit.client.0.smithi136.stdout:expected: 1 2024-09-13T00:32:28.194 INFO:tasks.workunit.client.0.smithi136.stdout:image: image34 2024-09-13T00:32:28.194 INFO:tasks.workunit.client.0.smithi136.stdout:expected = image34 2024-09-13T00:32:28.267 INFO:tasks.workunit.client.0.smithi136.stdout:num images is: 2 2024-09-13T00:32:28.267 INFO:tasks.workunit.client.0.smithi136.stdout:expected: 2 2024-09-13T00:32:28.268 INFO:tasks.workunit.client.0.smithi136.stdout:image: image34 2024-09-13T00:32:28.268 INFO:tasks.workunit.client.0.smithi136.stdout:image: image35 2024-09-13T00:32:28.268 INFO:tasks.workunit.client.0.smithi136.stdout:expected = image34 2024-09-13T00:32:28.268 INFO:tasks.workunit.client.0.smithi136.stdout:expected = image35 2024-09-13T00:32:28.412 INFO:tasks.workunit.client.0.smithi136.stdout:100.00% done 2024-09-13T00:32:28.413 INFO:tasks.workunit.client.0.smithi136.stdout:num images is: 3 2024-09-13T00:32:28.413 INFO:tasks.workunit.client.0.smithi136.stdout:expected: 3 2024-09-13T00:32:28.413 INFO:tasks.workunit.client.0.smithi136.stdout:image: image34 2024-09-13T00:32:28.413 INFO:tasks.workunit.client.0.smithi136.stdout:image: image35 2024-09-13T00:32:28.413 INFO:tasks.workunit.client.0.smithi136.stdout:image: image36 2024-09-13T00:32:28.413 INFO:tasks.workunit.client.0.smithi136.stdout:expected = image34 2024-09-13T00:32:28.413 INFO:tasks.workunit.client.0.smithi136.stdout:expected = image35 2024-09-13T00:32:28.413 INFO:tasks.workunit.client.0.smithi136.stdout:expected = image36 2024-09-13T00:32:28.521 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestDeepCopyPP (4477 ms) 2024-09-13T00:32:28.521 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestCreateLsDeleteSnap 2024-09-13T00:32:28.521 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:32:28.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:28 smithi181 bash[67587]: cluster 2024-09-13T00:32:27.545066+0000 mon.a (mon.0) 3034 : cluster [DBG] osdmap e1655: 8 total, 8 up, 8 in 2024-09-13T00:32:28.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:28 smithi181 bash[67587]: cluster 2024-09-13T00:32:28.075226+0000 mgr.y (mgr.24661) 1660 : cluster [DBG] pgmap v2876: 329 pgs: 5 creating+peering, 6 unknown, 318 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 145 KiB/s rd, 6.2 KiB/s wr, 172 op/s 2024-09-13T00:32:29.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:28 smithi136 bash[75164]: cluster 2024-09-13T00:32:27.545066+0000 mon.a (mon.0) 3034 : cluster [DBG] osdmap e1655: 8 total, 8 up, 8 in 2024-09-13T00:32:29.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:28 smithi136 bash[75164]: cluster 2024-09-13T00:32:28.075226+0000 mgr.y (mgr.24661) 1660 : cluster [DBG] pgmap v2876: 329 pgs: 5 creating+peering, 6 unknown, 318 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 145 KiB/s rd, 6.2 KiB/s wr, 172 op/s 2024-09-13T00:32:29.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:28 smithi136 bash[76371]: cluster 2024-09-13T00:32:27.545066+0000 mon.a (mon.0) 3034 : cluster [DBG] osdmap e1655: 8 total, 8 up, 8 in 2024-09-13T00:32:29.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:28 smithi136 bash[76371]: cluster 2024-09-13T00:32:28.075226+0000 mgr.y (mgr.24661) 1660 : cluster [DBG] pgmap v2876: 329 pgs: 5 creating+peering, 6 unknown, 318 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 145 KiB/s rd, 6.2 KiB/s wr, 172 op/s 2024-09-13T00:32:29.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:32:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:32:29] "GET /metrics HTTP/1.1" 200 48560 "" "Prometheus/2.43.0" 2024-09-13T00:32:30.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:30 smithi181 bash[67587]: cluster 2024-09-13T00:32:29.532896+0000 mon.a (mon.0) 3035 : cluster [DBG] osdmap e1656: 8 total, 8 up, 8 in 2024-09-13T00:32:30.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:30 smithi181 bash[67587]: cluster 2024-09-13T00:32:30.076211+0000 mgr.y (mgr.24661) 1661 : cluster [DBG] pgmap v2878: 329 pgs: 5 creating+peering, 324 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 165 KiB/s rd, 6.8 KiB/s wr, 189 op/s 2024-09-13T00:32:31.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:30 smithi136 bash[75164]: cluster 2024-09-13T00:32:29.532896+0000 mon.a (mon.0) 3035 : cluster [DBG] osdmap e1656: 8 total, 8 up, 8 in 2024-09-13T00:32:31.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:30 smithi136 bash[75164]: cluster 2024-09-13T00:32:30.076211+0000 mgr.y (mgr.24661) 1661 : cluster [DBG] pgmap v2878: 329 pgs: 5 creating+peering, 324 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 165 KiB/s rd, 6.8 KiB/s wr, 189 op/s 2024-09-13T00:32:31.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:30 smithi136 bash[76371]: cluster 2024-09-13T00:32:29.532896+0000 mon.a (mon.0) 3035 : cluster [DBG] osdmap e1656: 8 total, 8 up, 8 in 2024-09-13T00:32:31.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:30 smithi136 bash[76371]: cluster 2024-09-13T00:32:30.076211+0000 mgr.y (mgr.24661) 1661 : cluster [DBG] pgmap v2878: 329 pgs: 5 creating+peering, 324 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 165 KiB/s rd, 6.8 KiB/s wr, 189 op/s 2024-09-13T00:32:31.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:31 smithi181 bash[67587]: cluster 2024-09-13T00:32:30.545670+0000 mon.a (mon.0) 3036 : cluster [DBG] osdmap e1657: 8 total, 8 up, 8 in 2024-09-13T00:32:32.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:31 smithi136 bash[75164]: cluster 2024-09-13T00:32:30.545670+0000 mon.a (mon.0) 3036 : cluster [DBG] osdmap e1657: 8 total, 8 up, 8 in 2024-09-13T00:32:32.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:31 smithi136 bash[76371]: cluster 2024-09-13T00:32:30.545670+0000 mon.a (mon.0) 3036 : cluster [DBG] osdmap e1657: 8 total, 8 up, 8 in 2024-09-13T00:32:32.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:32 smithi181 bash[67587]: cluster 2024-09-13T00:32:31.553519+0000 mon.a (mon.0) 3037 : cluster [DBG] osdmap e1658: 8 total, 8 up, 8 in 2024-09-13T00:32:32.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:32 smithi181 bash[67587]: cluster 2024-09-13T00:32:32.077419+0000 mgr.y (mgr.24661) 1662 : cluster [DBG] pgmap v2881: 329 pgs: 329 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 310 KiB/s rd, 20 KiB/s wr, 367 op/s 2024-09-13T00:32:32.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:32:32 smithi181 bash[95505]: ts=2024-09-13T00:32:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:32:32.610 INFO:tasks.workunit.client.0.smithi136.stdout:num snaps is: 1 2024-09-13T00:32:32.610 INFO:tasks.workunit.client.0.smithi136.stdout:expected: 1 2024-09-13T00:32:32.611 INFO:tasks.workunit.client.0.smithi136.stdout:snap: snap1 2024-09-13T00:32:32.611 INFO:tasks.workunit.client.0.smithi136.stdout:found snap1 with size 2097152 2024-09-13T00:32:32.611 INFO:tasks.workunit.client.0.smithi136.stdout:num snaps is: 2 2024-09-13T00:32:32.611 INFO:tasks.workunit.client.0.smithi136.stdout:expected: 2 2024-09-13T00:32:32.611 INFO:tasks.workunit.client.0.smithi136.stdout:snap: snap1 2024-09-13T00:32:32.611 INFO:tasks.workunit.client.0.smithi136.stdout:snap: snap2 2024-09-13T00:32:32.611 INFO:tasks.workunit.client.0.smithi136.stdout:found snap1 with size 2097152 2024-09-13T00:32:32.611 INFO:tasks.workunit.client.0.smithi136.stdout:found snap2 with size 4194304 2024-09-13T00:32:32.611 INFO:tasks.workunit.client.0.smithi136.stdout:num snaps is: 1 2024-09-13T00:32:32.611 INFO:tasks.workunit.client.0.smithi136.stdout:expected: 1 2024-09-13T00:32:32.611 INFO:tasks.workunit.client.0.smithi136.stdout:snap: snap2 2024-09-13T00:32:32.611 INFO:tasks.workunit.client.0.smithi136.stdout:found snap2 with size 4194304 2024-09-13T00:32:32.611 INFO:tasks.workunit.client.0.smithi136.stdout:num snaps is: 0 2024-09-13T00:32:32.612 INFO:tasks.workunit.client.0.smithi136.stdout:expected: 0 2024-09-13T00:32:32.612 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestCreateLsDeleteSnap (4090 ms) 2024-09-13T00:32:32.612 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestGetSnapShotTimeStamp 2024-09-13T00:32:32.612 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:32:33.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:32 smithi136 bash[75164]: cluster 2024-09-13T00:32:31.553519+0000 mon.a (mon.0) 3037 : cluster [DBG] osdmap e1658: 8 total, 8 up, 8 in 2024-09-13T00:32:33.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:32 smithi136 bash[75164]: cluster 2024-09-13T00:32:32.077419+0000 mgr.y (mgr.24661) 1662 : cluster [DBG] pgmap v2881: 329 pgs: 329 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 310 KiB/s rd, 20 KiB/s wr, 367 op/s 2024-09-13T00:32:33.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:32 smithi136 bash[76371]: cluster 2024-09-13T00:32:31.553519+0000 mon.a (mon.0) 3037 : cluster [DBG] osdmap e1658: 8 total, 8 up, 8 in 2024-09-13T00:32:33.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:32 smithi136 bash[76371]: cluster 2024-09-13T00:32:32.077419+0000 mgr.y (mgr.24661) 1662 : cluster [DBG] pgmap v2881: 329 pgs: 329 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 310 KiB/s rd, 20 KiB/s wr, 367 op/s 2024-09-13T00:32:33.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:33 smithi181 bash[67587]: cluster 2024-09-13T00:32:32.563274+0000 mon.a (mon.0) 3038 : cluster [DBG] osdmap e1659: 8 total, 8 up, 8 in 2024-09-13T00:32:34.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:33 smithi136 bash[76371]: cluster 2024-09-13T00:32:32.563274+0000 mon.a (mon.0) 3038 : cluster [DBG] osdmap e1659: 8 total, 8 up, 8 in 2024-09-13T00:32:34.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:33 smithi136 bash[75164]: cluster 2024-09-13T00:32:32.563274+0000 mon.a (mon.0) 3038 : cluster [DBG] osdmap e1659: 8 total, 8 up, 8 in 2024-09-13T00:32:34.640 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestGetSnapShotTimeStamp (2029 ms) 2024-09-13T00:32:34.640 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestCreateLsDeleteSnapPP 2024-09-13T00:32:34.640 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:32:34.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:34 smithi181 bash[67587]: cluster 2024-09-13T00:32:33.564168+0000 mon.a (mon.0) 3039 : cluster [DBG] osdmap e1660: 8 total, 8 up, 8 in 2024-09-13T00:32:34.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:34 smithi181 bash[67587]: cluster 2024-09-13T00:32:34.078274+0000 mgr.y (mgr.24661) 1663 : cluster [DBG] pgmap v2884: 329 pgs: 329 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 296 KiB/s rd, 21 KiB/s wr, 353 op/s 2024-09-13T00:32:35.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:34 smithi136 bash[76371]: cluster 2024-09-13T00:32:33.564168+0000 mon.a (mon.0) 3039 : cluster [DBG] osdmap e1660: 8 total, 8 up, 8 in 2024-09-13T00:32:35.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:34 smithi136 bash[76371]: cluster 2024-09-13T00:32:34.078274+0000 mgr.y (mgr.24661) 1663 : cluster [DBG] pgmap v2884: 329 pgs: 329 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 296 KiB/s rd, 21 KiB/s wr, 353 op/s 2024-09-13T00:32:35.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:34 smithi136 bash[75164]: cluster 2024-09-13T00:32:33.564168+0000 mon.a (mon.0) 3039 : cluster [DBG] osdmap e1660: 8 total, 8 up, 8 in 2024-09-13T00:32:35.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:34 smithi136 bash[75164]: cluster 2024-09-13T00:32:34.078274+0000 mgr.y (mgr.24661) 1663 : cluster [DBG] pgmap v2884: 329 pgs: 329 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 296 KiB/s rd, 21 KiB/s wr, 353 op/s 2024-09-13T00:32:35.640 INFO:tasks.workunit.client.0.smithi136.stdout:num snaps is: 1 2024-09-13T00:32:35.640 INFO:tasks.workunit.client.0.smithi136.stdout:expected: 1 2024-09-13T00:32:35.640 INFO:tasks.workunit.client.0.smithi136.stdout:snap: snap1 2024-09-13T00:32:35.641 INFO:tasks.workunit.client.0.smithi136.stdout:found snap1 with size 2097152 2024-09-13T00:32:35.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:35 smithi181 bash[67587]: cluster 2024-09-13T00:32:34.585555+0000 mon.a (mon.0) 3040 : cluster [DBG] osdmap e1661: 8 total, 8 up, 8 in 2024-09-13T00:32:36.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:35 smithi136 bash[76371]: cluster 2024-09-13T00:32:34.585555+0000 mon.a (mon.0) 3040 : cluster [DBG] osdmap e1661: 8 total, 8 up, 8 in 2024-09-13T00:32:36.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:35 smithi136 bash[75164]: cluster 2024-09-13T00:32:34.585555+0000 mon.a (mon.0) 3040 : cluster [DBG] osdmap e1661: 8 total, 8 up, 8 in 2024-09-13T00:32:36.651 INFO:tasks.workunit.client.0.smithi136.stdout:num snaps is: 2 2024-09-13T00:32:36.651 INFO:tasks.workunit.client.0.smithi136.stdout:expected: 2 2024-09-13T00:32:36.651 INFO:tasks.workunit.client.0.smithi136.stdout:snap: snap1 2024-09-13T00:32:36.652 INFO:tasks.workunit.client.0.smithi136.stdout:snap: snap2 2024-09-13T00:32:36.652 INFO:tasks.workunit.client.0.smithi136.stdout:found snap1 with size 2097152 2024-09-13T00:32:36.652 INFO:tasks.workunit.client.0.smithi136.stdout:found snap2 with size 4194304 2024-09-13T00:32:37.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:36 smithi136 bash[75164]: cluster 2024-09-13T00:32:35.596890+0000 mon.a (mon.0) 3041 : cluster [DBG] osdmap e1662: 8 total, 8 up, 8 in 2024-09-13T00:32:37.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:36 smithi136 bash[75164]: cluster 2024-09-13T00:32:36.079629+0000 mgr.y (mgr.24661) 1664 : cluster [DBG] pgmap v2887: 329 pgs: 329 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 40 KiB/s rd, 5.5 KiB/s wr, 58 op/s 2024-09-13T00:32:37.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:36 smithi136 bash[76371]: cluster 2024-09-13T00:32:35.596890+0000 mon.a (mon.0) 3041 : cluster [DBG] osdmap e1662: 8 total, 8 up, 8 in 2024-09-13T00:32:37.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:36 smithi136 bash[76371]: cluster 2024-09-13T00:32:36.079629+0000 mgr.y (mgr.24661) 1664 : cluster [DBG] pgmap v2887: 329 pgs: 329 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 40 KiB/s rd, 5.5 KiB/s wr, 58 op/s 2024-09-13T00:32:37.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:36 smithi181 bash[67587]: cluster 2024-09-13T00:32:35.596890+0000 mon.a (mon.0) 3041 : cluster [DBG] osdmap e1662: 8 total, 8 up, 8 in 2024-09-13T00:32:37.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:36 smithi181 bash[67587]: cluster 2024-09-13T00:32:36.079629+0000 mgr.y (mgr.24661) 1664 : cluster [DBG] pgmap v2887: 329 pgs: 329 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 40 KiB/s rd, 5.5 KiB/s wr, 58 op/s 2024-09-13T00:32:37.660 INFO:tasks.workunit.client.0.smithi136.stdout:num snaps is: 1 2024-09-13T00:32:37.660 INFO:tasks.workunit.client.0.smithi136.stdout:expected: 1 2024-09-13T00:32:37.660 INFO:tasks.workunit.client.0.smithi136.stdout:snap: snap2 2024-09-13T00:32:37.660 INFO:tasks.workunit.client.0.smithi136.stdout:found snap2 with size 4194304 2024-09-13T00:32:38.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:37 smithi136 bash[75164]: cluster 2024-09-13T00:32:36.609056+0000 mon.a (mon.0) 3042 : cluster [DBG] osdmap e1663: 8 total, 8 up, 8 in 2024-09-13T00:32:38.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:37 smithi136 bash[76371]: cluster 2024-09-13T00:32:36.609056+0000 mon.a (mon.0) 3042 : cluster [DBG] osdmap e1663: 8 total, 8 up, 8 in 2024-09-13T00:32:38.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:37 smithi181 bash[67587]: cluster 2024-09-13T00:32:36.609056+0000 mon.a (mon.0) 3042 : cluster [DBG] osdmap e1663: 8 total, 8 up, 8 in 2024-09-13T00:32:38.672 INFO:tasks.workunit.client.0.smithi136.stdout:num snaps is: 0 2024-09-13T00:32:38.673 INFO:tasks.workunit.client.0.smithi136.stdout:expected: 0 2024-09-13T00:32:38.693 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestCreateLsDeleteSnapPP (4053 ms) 2024-09-13T00:32:38.693 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestGetNameIdSnapPP 2024-09-13T00:32:38.693 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:32:39.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:38 smithi136 bash[75164]: cluster 2024-09-13T00:32:37.616376+0000 mon.a (mon.0) 3043 : cluster [DBG] osdmap e1664: 8 total, 8 up, 8 in 2024-09-13T00:32:39.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:38 smithi136 bash[75164]: cluster 2024-09-13T00:32:38.080550+0000 mgr.y (mgr.24661) 1665 : cluster [DBG] pgmap v2890: 329 pgs: 329 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 85 KiB/s rd, 9.5 KiB/s wr, 116 op/s 2024-09-13T00:32:39.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:38 smithi136 bash[76371]: cluster 2024-09-13T00:32:37.616376+0000 mon.a (mon.0) 3043 : cluster [DBG] osdmap e1664: 8 total, 8 up, 8 in 2024-09-13T00:32:39.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:38 smithi136 bash[76371]: cluster 2024-09-13T00:32:38.080550+0000 mgr.y (mgr.24661) 1665 : cluster [DBG] pgmap v2890: 329 pgs: 329 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 85 KiB/s rd, 9.5 KiB/s wr, 116 op/s 2024-09-13T00:32:39.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:38 smithi181 bash[67587]: cluster 2024-09-13T00:32:37.616376+0000 mon.a (mon.0) 3043 : cluster [DBG] osdmap e1664: 8 total, 8 up, 8 in 2024-09-13T00:32:39.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:38 smithi181 bash[67587]: cluster 2024-09-13T00:32:38.080550+0000 mgr.y (mgr.24661) 1665 : cluster [DBG] pgmap v2890: 329 pgs: 329 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 85 KiB/s rd, 9.5 KiB/s wr, 116 op/s 2024-09-13T00:32:39.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:39 smithi136 bash[75164]: cluster 2024-09-13T00:32:38.625370+0000 mon.a (mon.0) 3044 : cluster [DBG] osdmap e1665: 8 total, 8 up, 8 in 2024-09-13T00:32:39.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:39 smithi136 bash[75164]: audit 2024-09-13T00:32:38.825968+0000 mon.a (mon.0) 3045 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:32:39.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:39 smithi136 bash[75164]: cluster 2024-09-13T00:32:39.535196+0000 mon.a (mon.0) 3046 : cluster [DBG] osdmap e1666: 8 total, 8 up, 8 in 2024-09-13T00:32:39.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:32:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:32:39] "GET /metrics HTTP/1.1" 200 50977 "" "Prometheus/2.43.0" 2024-09-13T00:32:39.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:39 smithi136 bash[76371]: cluster 2024-09-13T00:32:38.625370+0000 mon.a (mon.0) 3044 : cluster [DBG] osdmap e1665: 8 total, 8 up, 8 in 2024-09-13T00:32:39.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:39 smithi136 bash[76371]: audit 2024-09-13T00:32:38.825968+0000 mon.a (mon.0) 3045 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:32:39.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:39 smithi136 bash[76371]: cluster 2024-09-13T00:32:39.535196+0000 mon.a (mon.0) 3046 : cluster [DBG] osdmap e1666: 8 total, 8 up, 8 in 2024-09-13T00:32:40.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:39 smithi181 bash[67587]: cluster 2024-09-13T00:32:38.625370+0000 mon.a (mon.0) 3044 : cluster [DBG] osdmap e1665: 8 total, 8 up, 8 in 2024-09-13T00:32:40.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:39 smithi181 bash[67587]: audit 2024-09-13T00:32:38.825968+0000 mon.a (mon.0) 3045 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:32:40.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:39 smithi181 bash[67587]: cluster 2024-09-13T00:32:39.535196+0000 mon.a (mon.0) 3046 : cluster [DBG] osdmap e1666: 8 total, 8 up, 8 in 2024-09-13T00:32:41.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:40 smithi136 bash[75164]: cluster 2024-09-13T00:32:40.081403+0000 mgr.y (mgr.24661) 1666 : cluster [DBG] pgmap v2893: 329 pgs: 329 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 48 KiB/s rd, 5.0 KiB/s wr, 65 op/s 2024-09-13T00:32:41.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:40 smithi136 bash[75164]: cluster 2024-09-13T00:32:40.539681+0000 mon.a (mon.0) 3047 : cluster [DBG] osdmap e1667: 8 total, 8 up, 8 in 2024-09-13T00:32:41.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:40 smithi136 bash[76371]: cluster 2024-09-13T00:32:40.081403+0000 mgr.y (mgr.24661) 1666 : cluster [DBG] pgmap v2893: 329 pgs: 329 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 48 KiB/s rd, 5.0 KiB/s wr, 65 op/s 2024-09-13T00:32:41.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:40 smithi136 bash[76371]: cluster 2024-09-13T00:32:40.539681+0000 mon.a (mon.0) 3047 : cluster [DBG] osdmap e1667: 8 total, 8 up, 8 in 2024-09-13T00:32:41.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:40 smithi181 bash[67587]: cluster 2024-09-13T00:32:40.081403+0000 mgr.y (mgr.24661) 1666 : cluster [DBG] pgmap v2893: 329 pgs: 329 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 48 KiB/s rd, 5.0 KiB/s wr, 65 op/s 2024-09-13T00:32:41.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:40 smithi181 bash[67587]: cluster 2024-09-13T00:32:40.539681+0000 mon.a (mon.0) 3047 : cluster [DBG] osdmap e1667: 8 total, 8 up, 8 in 2024-09-13T00:32:42.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:32:42 smithi181 bash[95505]: ts=2024-09-13T00:32:42.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:32:42.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:42 smithi181 bash[67587]: cluster 2024-09-13T00:32:41.541192+0000 mon.a (mon.0) 3048 : cluster [DBG] osdmap e1668: 8 total, 8 up, 8 in 2024-09-13T00:32:42.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:42 smithi181 bash[67587]: cluster 2024-09-13T00:32:42.082571+0000 mgr.y (mgr.24661) 1667 : cluster [DBG] pgmap v2896: 329 pgs: 329 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 19 KiB/s rd, 2.0 KiB/s wr, 28 op/s 2024-09-13T00:32:43.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:42 smithi136 bash[75164]: cluster 2024-09-13T00:32:41.541192+0000 mon.a (mon.0) 3048 : cluster [DBG] osdmap e1668: 8 total, 8 up, 8 in 2024-09-13T00:32:43.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:42 smithi136 bash[75164]: cluster 2024-09-13T00:32:42.082571+0000 mgr.y (mgr.24661) 1667 : cluster [DBG] pgmap v2896: 329 pgs: 329 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 19 KiB/s rd, 2.0 KiB/s wr, 28 op/s 2024-09-13T00:32:43.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:42 smithi136 bash[76371]: cluster 2024-09-13T00:32:41.541192+0000 mon.a (mon.0) 3048 : cluster [DBG] osdmap e1668: 8 total, 8 up, 8 in 2024-09-13T00:32:43.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:42 smithi136 bash[76371]: cluster 2024-09-13T00:32:42.082571+0000 mgr.y (mgr.24661) 1667 : cluster [DBG] pgmap v2896: 329 pgs: 329 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 19 KiB/s rd, 2.0 KiB/s wr, 28 op/s 2024-09-13T00:32:43.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:43 smithi181 bash[67587]: cluster 2024-09-13T00:32:42.551540+0000 mon.a (mon.0) 3049 : cluster [DBG] osdmap e1669: 8 total, 8 up, 8 in 2024-09-13T00:32:44.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:43 smithi136 bash[75164]: cluster 2024-09-13T00:32:42.551540+0000 mon.a (mon.0) 3049 : cluster [DBG] osdmap e1669: 8 total, 8 up, 8 in 2024-09-13T00:32:44.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:43 smithi136 bash[76371]: cluster 2024-09-13T00:32:42.551540+0000 mon.a (mon.0) 3049 : cluster [DBG] osdmap e1669: 8 total, 8 up, 8 in 2024-09-13T00:32:44.596 INFO:tasks.workunit.client.0.smithi136.stdout:num snaps is: 0 2024-09-13T00:32:44.597 INFO:tasks.workunit.client.0.smithi136.stdout:expected: 0 2024-09-13T00:32:44.616 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestGetNameIdSnapPP (5923 ms) 2024-09-13T00:32:44.616 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestCreateLsRenameSnapPP 2024-09-13T00:32:44.616 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:32:44.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:44 smithi181 bash[67587]: cluster 2024-09-13T00:32:43.559265+0000 mon.a (mon.0) 3050 : cluster [DBG] osdmap e1670: 8 total, 8 up, 8 in 2024-09-13T00:32:44.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:44 smithi181 bash[67587]: audit 2024-09-13T00:32:43.877399+0000 mon.a (mon.0) 3051 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:32:44.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:44 smithi181 bash[67587]: cluster 2024-09-13T00:32:44.083435+0000 mgr.y (mgr.24661) 1668 : cluster [DBG] pgmap v2899: 329 pgs: 329 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 29 KiB/s rd, 3.0 KiB/s wr, 40 op/s 2024-09-13T00:32:44.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:44 smithi181 bash[67587]: audit 2024-09-13T00:32:44.407158+0000 mon.a (mon.0) 3052 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:32:44.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:44 smithi181 bash[67587]: audit 2024-09-13T00:32:44.408755+0000 mon.a (mon.0) 3053 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:32:44.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:44 smithi181 bash[67587]: audit 2024-09-13T00:32:44.418571+0000 mon.a (mon.0) 3054 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:32:45.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:44 smithi136 bash[75164]: cluster 2024-09-13T00:32:43.559265+0000 mon.a (mon.0) 3050 : cluster [DBG] osdmap e1670: 8 total, 8 up, 8 in 2024-09-13T00:32:45.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:44 smithi136 bash[75164]: audit 2024-09-13T00:32:43.877399+0000 mon.a (mon.0) 3051 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:32:45.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:44 smithi136 bash[75164]: cluster 2024-09-13T00:32:44.083435+0000 mgr.y (mgr.24661) 1668 : cluster [DBG] pgmap v2899: 329 pgs: 329 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 29 KiB/s rd, 3.0 KiB/s wr, 40 op/s 2024-09-13T00:32:45.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:44 smithi136 bash[75164]: audit 2024-09-13T00:32:44.407158+0000 mon.a (mon.0) 3052 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:32:45.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:44 smithi136 bash[75164]: audit 2024-09-13T00:32:44.408755+0000 mon.a (mon.0) 3053 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:32:45.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:44 smithi136 bash[75164]: audit 2024-09-13T00:32:44.418571+0000 mon.a (mon.0) 3054 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:32:45.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:44 smithi136 bash[76371]: cluster 2024-09-13T00:32:43.559265+0000 mon.a (mon.0) 3050 : cluster [DBG] osdmap e1670: 8 total, 8 up, 8 in 2024-09-13T00:32:45.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:44 smithi136 bash[76371]: audit 2024-09-13T00:32:43.877399+0000 mon.a (mon.0) 3051 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:32:45.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:44 smithi136 bash[76371]: cluster 2024-09-13T00:32:44.083435+0000 mgr.y (mgr.24661) 1668 : cluster [DBG] pgmap v2899: 329 pgs: 329 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 29 KiB/s rd, 3.0 KiB/s wr, 40 op/s 2024-09-13T00:32:45.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:44 smithi136 bash[76371]: audit 2024-09-13T00:32:44.407158+0000 mon.a (mon.0) 3052 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:32:45.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:44 smithi136 bash[76371]: audit 2024-09-13T00:32:44.408755+0000 mon.a (mon.0) 3053 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:32:45.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:44 smithi136 bash[76371]: audit 2024-09-13T00:32:44.418571+0000 mon.a (mon.0) 3054 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:32:45.621 INFO:tasks.workunit.client.0.smithi136.stdout:num snaps is: 1 2024-09-13T00:32:45.621 INFO:tasks.workunit.client.0.smithi136.stdout:expected: 1 2024-09-13T00:32:45.621 INFO:tasks.workunit.client.0.smithi136.stdout:snap: snap1 2024-09-13T00:32:45.621 INFO:tasks.workunit.client.0.smithi136.stdout:found snap1 with size 2097152 2024-09-13T00:32:45.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:45 smithi181 bash[67587]: cluster 2024-09-13T00:32:44.574838+0000 mon.a (mon.0) 3055 : cluster [DBG] osdmap e1671: 8 total, 8 up, 8 in 2024-09-13T00:32:46.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:45 smithi136 bash[75164]: cluster 2024-09-13T00:32:44.574838+0000 mon.a (mon.0) 3055 : cluster [DBG] osdmap e1671: 8 total, 8 up, 8 in 2024-09-13T00:32:46.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:45 smithi136 bash[76371]: cluster 2024-09-13T00:32:44.574838+0000 mon.a (mon.0) 3055 : cluster [DBG] osdmap e1671: 8 total, 8 up, 8 in 2024-09-13T00:32:46.639 INFO:tasks.workunit.client.0.smithi136.stdout:num snaps is: 2 2024-09-13T00:32:46.639 INFO:tasks.workunit.client.0.smithi136.stdout:expected: 2 2024-09-13T00:32:46.639 INFO:tasks.workunit.client.0.smithi136.stdout:snap: snap1 2024-09-13T00:32:46.639 INFO:tasks.workunit.client.0.smithi136.stdout:snap: snap2 2024-09-13T00:32:46.639 INFO:tasks.workunit.client.0.smithi136.stdout:found snap1 with size 2097152 2024-09-13T00:32:46.639 INFO:tasks.workunit.client.0.smithi136.stdout:found snap2 with size 4194304 2024-09-13T00:32:46.651 INFO:tasks.workunit.client.0.smithi136.stdout:num snaps is: 2 2024-09-13T00:32:46.651 INFO:tasks.workunit.client.0.smithi136.stdout:expected: 2 2024-09-13T00:32:46.651 INFO:tasks.workunit.client.0.smithi136.stdout:snap: snap1-rename 2024-09-13T00:32:46.651 INFO:tasks.workunit.client.0.smithi136.stdout:snap: snap2 2024-09-13T00:32:46.651 INFO:tasks.workunit.client.0.smithi136.stdout:found snap1-rename with size 2097152 2024-09-13T00:32:46.651 INFO:tasks.workunit.client.0.smithi136.stdout:found snap2 with size 4194304 2024-09-13T00:32:47.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:46 smithi136 bash[75164]: cluster 2024-09-13T00:32:45.574694+0000 mon.a (mon.0) 3056 : cluster [DBG] osdmap e1672: 8 total, 8 up, 8 in 2024-09-13T00:32:47.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:46 smithi136 bash[75164]: cluster 2024-09-13T00:32:46.084722+0000 mgr.y (mgr.24661) 1669 : cluster [DBG] pgmap v2902: 329 pgs: 329 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 55 KiB/s rd, 5.7 KiB/s wr, 74 op/s 2024-09-13T00:32:47.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:46 smithi136 bash[76371]: cluster 2024-09-13T00:32:45.574694+0000 mon.a (mon.0) 3056 : cluster [DBG] osdmap e1672: 8 total, 8 up, 8 in 2024-09-13T00:32:47.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:46 smithi136 bash[76371]: cluster 2024-09-13T00:32:46.084722+0000 mgr.y (mgr.24661) 1669 : cluster [DBG] pgmap v2902: 329 pgs: 329 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 55 KiB/s rd, 5.7 KiB/s wr, 74 op/s 2024-09-13T00:32:47.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:46 smithi181 bash[67587]: cluster 2024-09-13T00:32:45.574694+0000 mon.a (mon.0) 3056 : cluster [DBG] osdmap e1672: 8 total, 8 up, 8 in 2024-09-13T00:32:47.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:46 smithi181 bash[67587]: cluster 2024-09-13T00:32:46.084722+0000 mgr.y (mgr.24661) 1669 : cluster [DBG] pgmap v2902: 329 pgs: 329 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 55 KiB/s rd, 5.7 KiB/s wr, 74 op/s 2024-09-13T00:32:47.668 INFO:tasks.workunit.client.0.smithi136.stdout:num snaps is: 1 2024-09-13T00:32:47.669 INFO:tasks.workunit.client.0.smithi136.stdout:expected: 1 2024-09-13T00:32:47.669 INFO:tasks.workunit.client.0.smithi136.stdout:snap: snap2-rename 2024-09-13T00:32:47.669 INFO:tasks.workunit.client.0.smithi136.stdout:found snap2-rename with size 4194304 2024-09-13T00:32:48.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:47 smithi136 bash[75164]: cluster 2024-09-13T00:32:46.594540+0000 mon.a (mon.0) 3057 : cluster [DBG] osdmap e1673: 8 total, 8 up, 8 in 2024-09-13T00:32:48.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:47 smithi136 bash[76371]: cluster 2024-09-13T00:32:46.594540+0000 mon.a (mon.0) 3057 : cluster [DBG] osdmap e1673: 8 total, 8 up, 8 in 2024-09-13T00:32:48.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:47 smithi181 bash[67587]: cluster 2024-09-13T00:32:46.594540+0000 mon.a (mon.0) 3057 : cluster [DBG] osdmap e1673: 8 total, 8 up, 8 in 2024-09-13T00:32:48.687 INFO:tasks.workunit.client.0.smithi136.stdout:num snaps is: 0 2024-09-13T00:32:48.687 INFO:tasks.workunit.client.0.smithi136.stdout:expected: 0 2024-09-13T00:32:48.698 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestCreateLsRenameSnapPP (4082 ms) 2024-09-13T00:32:48.698 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.ConcurrentCreatesUnvalidatedPool 2024-09-13T00:32:49.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:48 smithi136 bash[75164]: cluster 2024-09-13T00:32:47.619153+0000 mon.a (mon.0) 3058 : cluster [DBG] osdmap e1674: 8 total, 8 up, 8 in 2024-09-13T00:32:49.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:48 smithi136 bash[75164]: cluster 2024-09-13T00:32:48.085654+0000 mgr.y (mgr.24661) 1670 : cluster [DBG] pgmap v2905: 329 pgs: 329 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 76 KiB/s rd, 7.7 KiB/s wr, 102 op/s 2024-09-13T00:32:49.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:48 smithi136 bash[76371]: cluster 2024-09-13T00:32:47.619153+0000 mon.a (mon.0) 3058 : cluster [DBG] osdmap e1674: 8 total, 8 up, 8 in 2024-09-13T00:32:49.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:48 smithi136 bash[76371]: cluster 2024-09-13T00:32:48.085654+0000 mgr.y (mgr.24661) 1670 : cluster [DBG] pgmap v2905: 329 pgs: 329 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 76 KiB/s rd, 7.7 KiB/s wr, 102 op/s 2024-09-13T00:32:49.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:48 smithi181 bash[67587]: cluster 2024-09-13T00:32:47.619153+0000 mon.a (mon.0) 3058 : cluster [DBG] osdmap e1674: 8 total, 8 up, 8 in 2024-09-13T00:32:49.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:48 smithi181 bash[67587]: cluster 2024-09-13T00:32:48.085654+0000 mgr.y (mgr.24661) 1670 : cluster [DBG] pgmap v2905: 329 pgs: 329 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 76 KiB/s rd, 7.7 KiB/s wr, 102 op/s 2024-09-13T00:32:49.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:32:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:32:49] "GET /metrics HTTP/1.1" 200 50977 "" "Prometheus/2.43.0" 2024-09-13T00:32:49.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:49 smithi136 bash[75164]: cluster 2024-09-13T00:32:48.643223+0000 mon.a (mon.0) 3059 : cluster [DBG] osdmap e1675: 8 total, 8 up, 8 in 2024-09-13T00:32:49.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:49 smithi136 bash[75164]: cluster 2024-09-13T00:32:49.543123+0000 mon.a (mon.0) 3060 : cluster [DBG] osdmap e1676: 8 total, 8 up, 8 in 2024-09-13T00:32:49.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:49 smithi136 bash[75164]: audit 2024-09-13T00:32:49.547521+0000 mon.c (mon.2) 121 : audit [INF] from='client.? 172.21.15.136:0/1927298351' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-21","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:32:49.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:49 smithi136 bash[75164]: audit 2024-09-13T00:32:49.548731+0000 mon.a (mon.0) 3061 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-21","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:32:49.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:49 smithi136 bash[76371]: cluster 2024-09-13T00:32:48.643223+0000 mon.a (mon.0) 3059 : cluster [DBG] osdmap e1675: 8 total, 8 up, 8 in 2024-09-13T00:32:49.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:49 smithi136 bash[76371]: cluster 2024-09-13T00:32:49.543123+0000 mon.a (mon.0) 3060 : cluster [DBG] osdmap e1676: 8 total, 8 up, 8 in 2024-09-13T00:32:49.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:49 smithi136 bash[76371]: audit 2024-09-13T00:32:49.547521+0000 mon.c (mon.2) 121 : audit [INF] from='client.? 172.21.15.136:0/1927298351' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-21","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:32:49.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:49 smithi136 bash[76371]: audit 2024-09-13T00:32:49.548731+0000 mon.a (mon.0) 3061 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-21","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:32:50.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:49 smithi181 bash[67587]: cluster 2024-09-13T00:32:48.643223+0000 mon.a (mon.0) 3059 : cluster [DBG] osdmap e1675: 8 total, 8 up, 8 in 2024-09-13T00:32:50.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:49 smithi181 bash[67587]: cluster 2024-09-13T00:32:49.543123+0000 mon.a (mon.0) 3060 : cluster [DBG] osdmap e1676: 8 total, 8 up, 8 in 2024-09-13T00:32:50.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:49 smithi181 bash[67587]: audit 2024-09-13T00:32:49.547521+0000 mon.c (mon.2) 121 : audit [INF] from='client.? 172.21.15.136:0/1927298351' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-21","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:32:50.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:49 smithi181 bash[67587]: audit 2024-09-13T00:32:49.548731+0000 mon.a (mon.0) 3061 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-21","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:32:50.563 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:32:50.563 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:32:50.563 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:32:50.563 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:32:51.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:50 smithi136 bash[75164]: cluster 2024-09-13T00:32:50.086585+0000 mgr.y (mgr.24661) 1671 : cluster [DBG] pgmap v2908: 361 pgs: 32 unknown, 329 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 37 KiB/s rd, 4.2 KiB/s wr, 52 op/s 2024-09-13T00:32:51.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:50 smithi136 bash[75164]: audit 2024-09-13T00:32:50.545427+0000 mon.a (mon.0) 3062 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-21","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:32:51.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:50 smithi136 bash[75164]: cluster 2024-09-13T00:32:50.546101+0000 mon.a (mon.0) 3063 : cluster [DBG] osdmap e1677: 8 total, 8 up, 8 in 2024-09-13T00:32:51.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:50 smithi136 bash[76371]: cluster 2024-09-13T00:32:50.086585+0000 mgr.y (mgr.24661) 1671 : cluster [DBG] pgmap v2908: 361 pgs: 32 unknown, 329 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 37 KiB/s rd, 4.2 KiB/s wr, 52 op/s 2024-09-13T00:32:51.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:50 smithi136 bash[76371]: audit 2024-09-13T00:32:50.545427+0000 mon.a (mon.0) 3062 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-21","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:32:51.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:50 smithi136 bash[76371]: cluster 2024-09-13T00:32:50.546101+0000 mon.a (mon.0) 3063 : cluster [DBG] osdmap e1677: 8 total, 8 up, 8 in 2024-09-13T00:32:51.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:50 smithi181 bash[67587]: cluster 2024-09-13T00:32:50.086585+0000 mgr.y (mgr.24661) 1671 : cluster [DBG] pgmap v2908: 361 pgs: 32 unknown, 329 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 37 KiB/s rd, 4.2 KiB/s wr, 52 op/s 2024-09-13T00:32:51.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:50 smithi181 bash[67587]: audit 2024-09-13T00:32:50.545427+0000 mon.a (mon.0) 3062 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-21","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:32:51.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:50 smithi181 bash[67587]: cluster 2024-09-13T00:32:50.546101+0000 mon.a (mon.0) 3063 : cluster [DBG] osdmap e1677: 8 total, 8 up, 8 in 2024-09-13T00:32:52.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:51 smithi136 bash[75164]: cluster 2024-09-13T00:32:50.636445+0000 mon.a (mon.0) 3064 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:32:52.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:51 smithi136 bash[75164]: cluster 2024-09-13T00:32:51.545873+0000 mon.a (mon.0) 3065 : cluster [DBG] osdmap e1678: 8 total, 8 up, 8 in 2024-09-13T00:32:52.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:51 smithi136 bash[76371]: cluster 2024-09-13T00:32:50.636445+0000 mon.a (mon.0) 3064 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:32:52.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:51 smithi136 bash[76371]: cluster 2024-09-13T00:32:51.545873+0000 mon.a (mon.0) 3065 : cluster [DBG] osdmap e1678: 8 total, 8 up, 8 in 2024-09-13T00:32:52.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:51 smithi181 bash[67587]: cluster 2024-09-13T00:32:50.636445+0000 mon.a (mon.0) 3064 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:32:52.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:51 smithi181 bash[67587]: cluster 2024-09-13T00:32:51.545873+0000 mon.a (mon.0) 3065 : cluster [DBG] osdmap e1678: 8 total, 8 up, 8 in 2024-09-13T00:32:52.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:32:52 smithi181 bash[95505]: ts=2024-09-13T00:32:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:32:53.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:52 smithi136 bash[76371]: cluster 2024-09-13T00:32:52.087954+0000 mgr.y (mgr.24661) 1672 : cluster [DBG] pgmap v2911: 361 pgs: 4 creating+peering, 16 unknown, 341 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.2 KiB/s rd, 767 B/s wr, 8 op/s 2024-09-13T00:32:53.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:52 smithi136 bash[76371]: cluster 2024-09-13T00:32:52.550619+0000 mon.a (mon.0) 3066 : cluster [DBG] osdmap e1679: 8 total, 8 up, 8 in 2024-09-13T00:32:53.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:52 smithi136 bash[75164]: cluster 2024-09-13T00:32:52.087954+0000 mgr.y (mgr.24661) 1672 : cluster [DBG] pgmap v2911: 361 pgs: 4 creating+peering, 16 unknown, 341 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.2 KiB/s rd, 767 B/s wr, 8 op/s 2024-09-13T00:32:53.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:52 smithi136 bash[75164]: cluster 2024-09-13T00:32:52.550619+0000 mon.a (mon.0) 3066 : cluster [DBG] osdmap e1679: 8 total, 8 up, 8 in 2024-09-13T00:32:53.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:52 smithi181 bash[67587]: cluster 2024-09-13T00:32:52.087954+0000 mgr.y (mgr.24661) 1672 : cluster [DBG] pgmap v2911: 361 pgs: 4 creating+peering, 16 unknown, 341 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.2 KiB/s rd, 767 B/s wr, 8 op/s 2024-09-13T00:32:53.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:52 smithi181 bash[67587]: cluster 2024-09-13T00:32:52.550619+0000 mon.a (mon.0) 3066 : cluster [DBG] osdmap e1679: 8 total, 8 up, 8 in 2024-09-13T00:32:53.150 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.ConcurrentCreatesUnvalidatedPool (4452 ms) 2024-09-13T00:32:53.150 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.RemoveFullTry 2024-09-13T00:32:54.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:53 smithi136 bash[75164]: cluster 2024-09-13T00:32:52.647478+0000 mon.a (mon.0) 3067 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:32:54.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:53 smithi136 bash[75164]: cluster 2024-09-13T00:32:52.647523+0000 mon.a (mon.0) 3068 : cluster [INF] Cluster is now healthy 2024-09-13T00:32:54.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:53 smithi136 bash[76371]: cluster 2024-09-13T00:32:52.647478+0000 mon.a (mon.0) 3067 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:32:54.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:53 smithi136 bash[76371]: cluster 2024-09-13T00:32:52.647523+0000 mon.a (mon.0) 3068 : cluster [INF] Cluster is now healthy 2024-09-13T00:32:54.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:53 smithi181 bash[67587]: cluster 2024-09-13T00:32:52.647478+0000 mon.a (mon.0) 3067 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:32:54.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:53 smithi181 bash[67587]: cluster 2024-09-13T00:32:52.647523+0000 mon.a (mon.0) 3068 : cluster [INF] Cluster is now healthy 2024-09-13T00:32:54.609 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:32:54.928 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:32:54 smithi136 bash[68787]: [13/Sep/2024:00:32:54] ENGINE Bus STOPPING 2024-09-13T00:32:54.929 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:54 smithi136 bash[75164]: cluster 2024-09-13T00:32:53.673489+0000 mon.a (mon.0) 3069 : cluster [DBG] osdmap e1680: 8 total, 8 up, 8 in 2024-09-13T00:32:54.929 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:54 smithi136 bash[75164]: audit 2024-09-13T00:32:53.678254+0000 mon.b (mon.1) 18 : audit [INF] from='client.? 172.21.15.136:0/3076646475' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-22","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:32:54.929 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:54 smithi136 bash[75164]: audit 2024-09-13T00:32:53.684983+0000 mon.a (mon.0) 3070 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-22","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:32:54.929 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:54 smithi136 bash[75164]: audit 2024-09-13T00:32:53.826269+0000 mon.a (mon.0) 3071 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:32:54.929 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:54 smithi136 bash[75164]: cluster 2024-09-13T00:32:54.088945+0000 mgr.y (mgr.24661) 1673 : cluster [DBG] pgmap v2914: 393 pgs: 4 creating+peering, 39 unknown, 350 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 42 KiB/s rd, 3.0 KiB/s wr, 53 op/s 2024-09-13T00:32:54.929 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:54 smithi136 bash[75164]: audit 2024-09-13T00:32:54.542432+0000 mon.a (mon.0) 3072 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-22","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:32:54.929 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:54 smithi136 bash[75164]: cluster 2024-09-13T00:32:54.542882+0000 mon.a (mon.0) 3073 : cluster [DBG] osdmap e1681: 8 total, 8 up, 8 in 2024-09-13T00:32:54.929 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:54 smithi136 bash[75164]: audit 2024-09-13T00:32:54.608242+0000 mon.a (mon.0) 3074 : audit [INF] from='client.? ' entity='client.admin' 2024-09-13T00:32:54.929 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:54 smithi136 bash[75164]: audit 2024-09-13T00:32:54.626644+0000 mon.a (mon.0) 3075 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:32:54.930 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:54 smithi136 bash[75164]: audit 2024-09-13T00:32:54.630345+0000 mon.a (mon.0) 3076 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:32:54.930 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:54 smithi136 bash[75164]: audit 2024-09-13T00:32:54.632565+0000 mon.a (mon.0) 3077 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:32:54.930 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:54 smithi136 bash[75164]: audit 2024-09-13T00:32:54.633551+0000 mon.a (mon.0) 3078 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:32:54.930 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:54 smithi136 bash[75164]: audit 2024-09-13T00:32:54.639996+0000 mon.a (mon.0) 3079 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:32:54.930 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:54 smithi136 bash[75164]: audit 2024-09-13T00:32:54.640253+0000 mon.a (mon.0) 3080 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T00:32:54.930 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:54 smithi136 bash[76371]: cluster 2024-09-13T00:32:53.673489+0000 mon.a (mon.0) 3069 : cluster [DBG] osdmap e1680: 8 total, 8 up, 8 in 2024-09-13T00:32:54.930 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:54 smithi136 bash[76371]: audit 2024-09-13T00:32:53.678254+0000 mon.b (mon.1) 18 : audit [INF] from='client.? 172.21.15.136:0/3076646475' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-22","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:32:54.930 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:54 smithi136 bash[76371]: audit 2024-09-13T00:32:53.684983+0000 mon.a (mon.0) 3070 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-22","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:32:54.930 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:54 smithi136 bash[76371]: audit 2024-09-13T00:32:53.826269+0000 mon.a (mon.0) 3071 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:32:54.931 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:54 smithi136 bash[76371]: cluster 2024-09-13T00:32:54.088945+0000 mgr.y (mgr.24661) 1673 : cluster [DBG] pgmap v2914: 393 pgs: 4 creating+peering, 39 unknown, 350 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 42 KiB/s rd, 3.0 KiB/s wr, 53 op/s 2024-09-13T00:32:54.931 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:54 smithi136 bash[76371]: audit 2024-09-13T00:32:54.542432+0000 mon.a (mon.0) 3072 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-22","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:32:54.931 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:54 smithi136 bash[76371]: cluster 2024-09-13T00:32:54.542882+0000 mon.a (mon.0) 3073 : cluster [DBG] osdmap e1681: 8 total, 8 up, 8 in 2024-09-13T00:32:54.931 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:54 smithi136 bash[76371]: audit 2024-09-13T00:32:54.608242+0000 mon.a (mon.0) 3074 : audit [INF] from='client.? ' entity='client.admin' 2024-09-13T00:32:54.931 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:54 smithi136 bash[76371]: audit 2024-09-13T00:32:54.626644+0000 mon.a (mon.0) 3075 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:32:54.931 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:54 smithi136 bash[76371]: audit 2024-09-13T00:32:54.630345+0000 mon.a (mon.0) 3076 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:32:54.931 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:54 smithi136 bash[76371]: audit 2024-09-13T00:32:54.632565+0000 mon.a (mon.0) 3077 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:32:54.931 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:54 smithi136 bash[76371]: audit 2024-09-13T00:32:54.633551+0000 mon.a (mon.0) 3078 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:32:54.931 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:54 smithi136 bash[76371]: audit 2024-09-13T00:32:54.639996+0000 mon.a (mon.0) 3079 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:32:54.931 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:54 smithi136 bash[76371]: audit 2024-09-13T00:32:54.640253+0000 mon.a (mon.0) 3080 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T00:32:55.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:54 smithi181 bash[67587]: cluster 2024-09-13T00:32:53.673489+0000 mon.a (mon.0) 3069 : cluster [DBG] osdmap e1680: 8 total, 8 up, 8 in 2024-09-13T00:32:55.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:54 smithi181 bash[67587]: audit 2024-09-13T00:32:53.678254+0000 mon.b (mon.1) 18 : audit [INF] from='client.? 172.21.15.136:0/3076646475' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-22","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:32:55.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:54 smithi181 bash[67587]: audit 2024-09-13T00:32:53.684983+0000 mon.a (mon.0) 3070 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-22","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:32:55.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:54 smithi181 bash[67587]: audit 2024-09-13T00:32:53.826269+0000 mon.a (mon.0) 3071 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:32:55.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:54 smithi181 bash[67587]: cluster 2024-09-13T00:32:54.088945+0000 mgr.y (mgr.24661) 1673 : cluster [DBG] pgmap v2914: 393 pgs: 4 creating+peering, 39 unknown, 350 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 42 KiB/s rd, 3.0 KiB/s wr, 53 op/s 2024-09-13T00:32:55.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:54 smithi181 bash[67587]: audit 2024-09-13T00:32:54.542432+0000 mon.a (mon.0) 3072 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-22","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:32:55.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:54 smithi181 bash[67587]: cluster 2024-09-13T00:32:54.542882+0000 mon.a (mon.0) 3073 : cluster [DBG] osdmap e1681: 8 total, 8 up, 8 in 2024-09-13T00:32:55.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:54 smithi181 bash[67587]: audit 2024-09-13T00:32:54.608242+0000 mon.a (mon.0) 3074 : audit [INF] from='client.? ' entity='client.admin' 2024-09-13T00:32:55.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:54 smithi181 bash[67587]: audit 2024-09-13T00:32:54.626644+0000 mon.a (mon.0) 3075 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:32:55.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:54 smithi181 bash[67587]: audit 2024-09-13T00:32:54.630345+0000 mon.a (mon.0) 3076 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:32:55.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:54 smithi181 bash[67587]: audit 2024-09-13T00:32:54.632565+0000 mon.a (mon.0) 3077 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:32:55.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:54 smithi181 bash[67587]: audit 2024-09-13T00:32:54.633551+0000 mon.a (mon.0) 3078 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:32:55.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:54 smithi181 bash[67587]: audit 2024-09-13T00:32:54.639996+0000 mon.a (mon.0) 3079 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:32:55.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:54 smithi181 bash[67587]: audit 2024-09-13T00:32:54.640253+0000 mon.a (mon.0) 3080 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T00:32:55.259 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:32:54 smithi136 bash[68787]: [13/Sep/2024:00:32:54] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T00:32:55.259 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:32:54 smithi136 bash[68787]: [13/Sep/2024:00:32:54] ENGINE Bus STOPPED 2024-09-13T00:32:55.259 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:32:54 smithi136 bash[68787]: [13/Sep/2024:00:32:54] ENGINE Bus STARTING 2024-09-13T00:32:55.260 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[68787]: [13/Sep/2024:00:32:55] ENGINE Serving on http://:::9283 2024-09-13T00:32:55.260 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[68787]: [13/Sep/2024:00:32:55] ENGINE Bus STARTED 2024-09-13T00:32:56.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[75164]: cluster 2024-09-13T00:32:54.689792+0000 mon.a (mon.0) 3081 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:32:56.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[75164]: audit 2024-09-13T00:32:54.825395+0000 mon.a (mon.0) 3082 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:32:56.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[75164]: audit 2024-09-13T00:32:54.825820+0000 mon.a (mon.0) 3083 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:32:56.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[75164]: audit 2024-09-13T00:32:54.826119+0000 mon.a (mon.0) 3084 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:32:56.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[75164]: audit 2024-09-13T00:32:54.829395+0000 mon.a (mon.0) 3085 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:32:56.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[75164]: audit 2024-09-13T00:32:54.829982+0000 mon.a (mon.0) 3086 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:32:56.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[75164]: audit 2024-09-13T00:32:54.830360+0000 mon.a (mon.0) 3087 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:32:56.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[75164]: audit 2024-09-13T00:32:54.832742+0000 mon.a (mon.0) 3088 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:32:56.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[75164]: audit 2024-09-13T00:32:54.833242+0000 mon.a (mon.0) 3089 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:32:56.012 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[75164]: audit 2024-09-13T00:32:54.833582+0000 mon.a (mon.0) 3090 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:32:56.012 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[75164]: audit 2024-09-13T00:32:54.852173+0000 mon.a (mon.0) 3091 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T00:32:56.012 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[75164]: audit 2024-09-13T00:32:54.852254+0000 mon.a (mon.0) 3092 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T00:32:56.012 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[75164]: audit 2024-09-13T00:32:54.852499+0000 mon.a (mon.0) 3093 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:32:56.012 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[75164]: audit 2024-09-13T00:32:54.852730+0000 mon.a (mon.0) 3094 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:32:56.012 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[75164]: audit 2024-09-13T00:32:54.852983+0000 mon.a (mon.0) 3095 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:32:56.013 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[75164]: audit 2024-09-13T00:32:54.854848+0000 mon.b (mon.1) 19 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T00:32:56.013 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[75164]: audit 2024-09-13T00:32:54.854954+0000 mon.b (mon.1) 20 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T00:32:56.013 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[75164]: audit 2024-09-13T00:32:54.855519+0000 mon.a (mon.0) 3096 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:32:56.013 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[75164]: audit 2024-09-13T00:32:54.855721+0000 mon.a (mon.0) 3097 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:32:56.014 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[76371]: cluster 2024-09-13T00:32:54.689792+0000 mon.a (mon.0) 3081 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:32:56.014 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[76371]: audit 2024-09-13T00:32:54.825395+0000 mon.a (mon.0) 3082 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:32:56.014 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[76371]: audit 2024-09-13T00:32:54.825820+0000 mon.a (mon.0) 3083 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:32:56.014 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[76371]: audit 2024-09-13T00:32:54.826119+0000 mon.a (mon.0) 3084 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:32:56.014 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[76371]: audit 2024-09-13T00:32:54.829395+0000 mon.a (mon.0) 3085 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:32:56.015 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[76371]: audit 2024-09-13T00:32:54.829982+0000 mon.a (mon.0) 3086 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:32:56.015 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[76371]: audit 2024-09-13T00:32:54.830360+0000 mon.a (mon.0) 3087 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:32:56.015 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[76371]: audit 2024-09-13T00:32:54.832742+0000 mon.a (mon.0) 3088 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:32:56.015 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[76371]: audit 2024-09-13T00:32:54.833242+0000 mon.a (mon.0) 3089 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:32:56.015 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[76371]: audit 2024-09-13T00:32:54.833582+0000 mon.a (mon.0) 3090 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:32:56.015 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[76371]: audit 2024-09-13T00:32:54.852173+0000 mon.a (mon.0) 3091 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T00:32:56.016 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[76371]: audit 2024-09-13T00:32:54.852254+0000 mon.a (mon.0) 3092 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T00:32:56.016 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[76371]: audit 2024-09-13T00:32:54.852499+0000 mon.a (mon.0) 3093 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:32:56.016 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[76371]: audit 2024-09-13T00:32:54.852730+0000 mon.a (mon.0) 3094 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:32:56.016 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[76371]: audit 2024-09-13T00:32:54.852983+0000 mon.a (mon.0) 3095 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:32:56.016 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[76371]: audit 2024-09-13T00:32:54.854848+0000 mon.b (mon.1) 19 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T00:32:56.016 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[76371]: audit 2024-09-13T00:32:54.854954+0000 mon.b (mon.1) 20 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T00:32:56.016 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[76371]: audit 2024-09-13T00:32:54.855519+0000 mon.a (mon.0) 3096 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:32:56.016 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[76371]: audit 2024-09-13T00:32:54.855721+0000 mon.a (mon.0) 3097 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:32:56.016 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[76371]: audit 2024-09-13T00:32:54.856105+0000 mon.a (mon.0) 3098 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:32:56.017 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[76371]: audit 2024-09-13T00:32:54.857939+0000 mon.c (mon.2) 122 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T00:32:56.017 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[76371]: audit 2024-09-13T00:32:54.858072+0000 mon.c (mon.2) 123 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T00:32:56.017 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[76371]: audit 2024-09-13T00:32:54.858089+0000 mon.a (mon.0) 3099 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:32:56.017 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[76371]: audit 2024-09-13T00:32:54.858322+0000 mon.a (mon.0) 3100 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:32:56.017 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[76371]: audit 2024-09-13T00:32:54.858845+0000 mon.a (mon.0) 3101 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:32:56.017 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[76371]: cluster 2024-09-13T00:32:55.546138+0000 mon.a (mon.0) 3102 : cluster [DBG] osdmap e1682: 8 total, 8 up, 8 in 2024-09-13T00:32:56.017 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[75164]: audit 2024-09-13T00:32:54.856105+0000 mon.a (mon.0) 3098 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:32:56.017 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[75164]: audit 2024-09-13T00:32:54.857939+0000 mon.c (mon.2) 122 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T00:32:56.017 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[75164]: audit 2024-09-13T00:32:54.858072+0000 mon.c (mon.2) 123 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T00:32:56.017 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[75164]: audit 2024-09-13T00:32:54.858089+0000 mon.a (mon.0) 3099 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:32:56.017 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[75164]: audit 2024-09-13T00:32:54.858322+0000 mon.a (mon.0) 3100 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:32:56.017 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[75164]: audit 2024-09-13T00:32:54.858845+0000 mon.a (mon.0) 3101 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:32:56.018 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:55 smithi136 bash[75164]: cluster 2024-09-13T00:32:55.546138+0000 mon.a (mon.0) 3102 : cluster [DBG] osdmap e1682: 8 total, 8 up, 8 in 2024-09-13T00:32:56.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:55 smithi181 bash[67587]: cluster 2024-09-13T00:32:54.689792+0000 mon.a (mon.0) 3081 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:32:56.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:55 smithi181 bash[67587]: audit 2024-09-13T00:32:54.825395+0000 mon.a (mon.0) 3082 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:32:56.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:55 smithi181 bash[67587]: audit 2024-09-13T00:32:54.825820+0000 mon.a (mon.0) 3083 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:32:56.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:55 smithi181 bash[67587]: audit 2024-09-13T00:32:54.826119+0000 mon.a (mon.0) 3084 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:32:56.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:55 smithi181 bash[67587]: audit 2024-09-13T00:32:54.829395+0000 mon.a (mon.0) 3085 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:32:56.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:55 smithi181 bash[67587]: audit 2024-09-13T00:32:54.829982+0000 mon.a (mon.0) 3086 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:32:56.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:55 smithi181 bash[67587]: audit 2024-09-13T00:32:54.830360+0000 mon.a (mon.0) 3087 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:32:56.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:55 smithi181 bash[67587]: audit 2024-09-13T00:32:54.832742+0000 mon.a (mon.0) 3088 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:32:56.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:55 smithi181 bash[67587]: audit 2024-09-13T00:32:54.833242+0000 mon.a (mon.0) 3089 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:32:56.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:55 smithi181 bash[67587]: audit 2024-09-13T00:32:54.833582+0000 mon.a (mon.0) 3090 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:32:56.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:55 smithi181 bash[67587]: audit 2024-09-13T00:32:54.852173+0000 mon.a (mon.0) 3091 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T00:32:56.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:55 smithi181 bash[67587]: audit 2024-09-13T00:32:54.852254+0000 mon.a (mon.0) 3092 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T00:32:56.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:55 smithi181 bash[67587]: audit 2024-09-13T00:32:54.852499+0000 mon.a (mon.0) 3093 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:32:56.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:55 smithi181 bash[67587]: audit 2024-09-13T00:32:54.852730+0000 mon.a (mon.0) 3094 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:32:56.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:55 smithi181 bash[67587]: audit 2024-09-13T00:32:54.852983+0000 mon.a (mon.0) 3095 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:32:56.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:55 smithi181 bash[67587]: audit 2024-09-13T00:32:54.854848+0000 mon.b (mon.1) 19 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T00:32:56.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:55 smithi181 bash[67587]: audit 2024-09-13T00:32:54.854954+0000 mon.b (mon.1) 20 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T00:32:56.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:55 smithi181 bash[67587]: audit 2024-09-13T00:32:54.855519+0000 mon.a (mon.0) 3096 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:32:56.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:55 smithi181 bash[67587]: audit 2024-09-13T00:32:54.855721+0000 mon.a (mon.0) 3097 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:32:56.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:55 smithi181 bash[67587]: audit 2024-09-13T00:32:54.856105+0000 mon.a (mon.0) 3098 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:32:56.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:55 smithi181 bash[67587]: audit 2024-09-13T00:32:54.857939+0000 mon.c (mon.2) 122 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T00:32:56.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:55 smithi181 bash[67587]: audit 2024-09-13T00:32:54.858072+0000 mon.c (mon.2) 123 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T00:32:56.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:55 smithi181 bash[67587]: audit 2024-09-13T00:32:54.858089+0000 mon.a (mon.0) 3099 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:32:56.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:55 smithi181 bash[67587]: audit 2024-09-13T00:32:54.858322+0000 mon.a (mon.0) 3100 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:32:56.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:55 smithi181 bash[67587]: audit 2024-09-13T00:32:54.858845+0000 mon.a (mon.0) 3101 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:32:56.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:55 smithi181 bash[67587]: cluster 2024-09-13T00:32:55.546138+0000 mon.a (mon.0) 3102 : cluster [DBG] osdmap e1682: 8 total, 8 up, 8 in 2024-09-13T00:32:56.735 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:32:57.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:56 smithi136 bash[75164]: cluster 2024-09-13T00:32:56.090259+0000 mgr.y (mgr.24661) 1674 : cluster [DBG] pgmap v2917: 393 pgs: 4 creating+peering, 21 unknown, 368 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 92 KiB/s rd, 7.5 KiB/s wr, 127 op/s 2024-09-13T00:32:57.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:56 smithi136 bash[75164]: cluster 2024-09-13T00:32:56.550284+0000 mon.a (mon.0) 3103 : cluster [DBG] osdmap e1683: 8 total, 8 up, 8 in 2024-09-13T00:32:57.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:56 smithi136 bash[76371]: cluster 2024-09-13T00:32:56.090259+0000 mgr.y (mgr.24661) 1674 : cluster [DBG] pgmap v2917: 393 pgs: 4 creating+peering, 21 unknown, 368 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 92 KiB/s rd, 7.5 KiB/s wr, 127 op/s 2024-09-13T00:32:57.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:56 smithi136 bash[76371]: cluster 2024-09-13T00:32:56.550284+0000 mon.a (mon.0) 3103 : cluster [DBG] osdmap e1683: 8 total, 8 up, 8 in 2024-09-13T00:32:57.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:56 smithi181 bash[67587]: cluster 2024-09-13T00:32:56.090259+0000 mgr.y (mgr.24661) 1674 : cluster [DBG] pgmap v2917: 393 pgs: 4 creating+peering, 21 unknown, 368 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 92 KiB/s rd, 7.5 KiB/s wr, 127 op/s 2024-09-13T00:32:57.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:56 smithi181 bash[67587]: cluster 2024-09-13T00:32:56.550284+0000 mon.a (mon.0) 3103 : cluster [DBG] osdmap e1683: 8 total, 8 up, 8 in 2024-09-13T00:32:58.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:57 smithi136 bash[75164]: cluster 2024-09-13T00:32:56.693637+0000 mon.a (mon.0) 3104 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:32:58.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:57 smithi136 bash[75164]: cluster 2024-09-13T00:32:56.693692+0000 mon.a (mon.0) 3105 : cluster [INF] Cluster is now healthy 2024-09-13T00:32:58.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:57 smithi136 bash[75164]: audit 2024-09-13T00:32:56.754327+0000 mon.c (mon.2) 124 : audit [INF] from='client.? 172.21.15.136:0/1086627647' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi136-114798-22", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-09-13T00:32:58.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:57 smithi136 bash[75164]: audit 2024-09-13T00:32:56.754983+0000 mon.a (mon.0) 3106 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi136-114798-22", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-09-13T00:32:58.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:57 smithi136 bash[76371]: cluster 2024-09-13T00:32:56.693637+0000 mon.a (mon.0) 3104 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:32:58.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:57 smithi136 bash[76371]: cluster 2024-09-13T00:32:56.693692+0000 mon.a (mon.0) 3105 : cluster [INF] Cluster is now healthy 2024-09-13T00:32:58.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:57 smithi136 bash[76371]: audit 2024-09-13T00:32:56.754327+0000 mon.c (mon.2) 124 : audit [INF] from='client.? 172.21.15.136:0/1086627647' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi136-114798-22", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-09-13T00:32:58.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:57 smithi136 bash[76371]: audit 2024-09-13T00:32:56.754983+0000 mon.a (mon.0) 3106 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi136-114798-22", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-09-13T00:32:58.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:57 smithi181 bash[67587]: cluster 2024-09-13T00:32:56.693637+0000 mon.a (mon.0) 3104 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:32:58.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:57 smithi181 bash[67587]: cluster 2024-09-13T00:32:56.693692+0000 mon.a (mon.0) 3105 : cluster [INF] Cluster is now healthy 2024-09-13T00:32:58.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:57 smithi181 bash[67587]: audit 2024-09-13T00:32:56.754327+0000 mon.c (mon.2) 124 : audit [INF] from='client.? 172.21.15.136:0/1086627647' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi136-114798-22", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-09-13T00:32:58.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:57 smithi181 bash[67587]: audit 2024-09-13T00:32:56.754983+0000 mon.a (mon.0) 3106 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi136-114798-22", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-09-13T00:32:59.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:58 smithi136 bash[75164]: audit 2024-09-13T00:32:57.725745+0000 mon.a (mon.0) 3107 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi136-114798-22", "field": "max_bytes", "val": "10485760"}]': finished 2024-09-13T00:32:59.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:58 smithi136 bash[75164]: cluster 2024-09-13T00:32:57.725962+0000 mon.a (mon.0) 3108 : cluster [DBG] osdmap e1684: 8 total, 8 up, 8 in 2024-09-13T00:32:59.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:32:58 smithi136 bash[75164]: cluster 2024-09-13T00:32:58.091197+0000 mgr.y (mgr.24661) 1675 : cluster [DBG] pgmap v2920: 393 pgs: 4 creating+peering, 8 unknown, 381 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 82 KiB/s rd, 7.0 KiB/s wr, 123 op/s 2024-09-13T00:32:59.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:58 smithi136 bash[76371]: audit 2024-09-13T00:32:57.725745+0000 mon.a (mon.0) 3107 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi136-114798-22", "field": "max_bytes", "val": "10485760"}]': finished 2024-09-13T00:32:59.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:58 smithi136 bash[76371]: cluster 2024-09-13T00:32:57.725962+0000 mon.a (mon.0) 3108 : cluster [DBG] osdmap e1684: 8 total, 8 up, 8 in 2024-09-13T00:32:59.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:32:58 smithi136 bash[76371]: cluster 2024-09-13T00:32:58.091197+0000 mgr.y (mgr.24661) 1675 : cluster [DBG] pgmap v2920: 393 pgs: 4 creating+peering, 8 unknown, 381 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 82 KiB/s rd, 7.0 KiB/s wr, 123 op/s 2024-09-13T00:32:59.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:58 smithi181 bash[67587]: audit 2024-09-13T00:32:57.725745+0000 mon.a (mon.0) 3107 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi136-114798-22", "field": "max_bytes", "val": "10485760"}]': finished 2024-09-13T00:32:59.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:58 smithi181 bash[67587]: cluster 2024-09-13T00:32:57.725962+0000 mon.a (mon.0) 3108 : cluster [DBG] osdmap e1684: 8 total, 8 up, 8 in 2024-09-13T00:32:59.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:32:58 smithi181 bash[67587]: cluster 2024-09-13T00:32:58.091197+0000 mgr.y (mgr.24661) 1675 : cluster [DBG] pgmap v2920: 393 pgs: 4 creating+peering, 8 unknown, 381 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 82 KiB/s rd, 7.0 KiB/s wr, 123 op/s 2024-09-13T00:32:59.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:32:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:32:59] "GET /metrics HTTP/1.1" 200 55746 "" "Prometheus/2.43.0" 2024-09-13T00:33:00.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:00 smithi181 bash[67587]: cluster 2024-09-13T00:32:59.546381+0000 mon.a (mon.0) 3109 : cluster [DBG] osdmap e1685: 8 total, 8 up, 8 in 2024-09-13T00:33:00.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:00 smithi181 bash[67587]: cluster 2024-09-13T00:33:00.092246+0000 mgr.y (mgr.24661) 1676 : cluster [DBG] pgmap v2922: 393 pgs: 393 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 69 KiB/s rd, 4.6 KiB/s wr, 101 op/s 2024-09-13T00:33:01.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:00 smithi136 bash[75164]: cluster 2024-09-13T00:32:59.546381+0000 mon.a (mon.0) 3109 : cluster [DBG] osdmap e1685: 8 total, 8 up, 8 in 2024-09-13T00:33:01.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:00 smithi136 bash[75164]: cluster 2024-09-13T00:33:00.092246+0000 mgr.y (mgr.24661) 1676 : cluster [DBG] pgmap v2922: 393 pgs: 393 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 69 KiB/s rd, 4.6 KiB/s wr, 101 op/s 2024-09-13T00:33:01.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:00 smithi136 bash[76371]: cluster 2024-09-13T00:32:59.546381+0000 mon.a (mon.0) 3109 : cluster [DBG] osdmap e1685: 8 total, 8 up, 8 in 2024-09-13T00:33:01.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:00 smithi136 bash[76371]: cluster 2024-09-13T00:33:00.092246+0000 mgr.y (mgr.24661) 1676 : cluster [DBG] pgmap v2922: 393 pgs: 393 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 69 KiB/s rd, 4.6 KiB/s wr, 101 op/s 2024-09-13T00:33:02.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:33:02 smithi181 bash[95505]: ts=2024-09-13T00:33:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:33:03.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:03 smithi136 bash[75164]: cluster 2024-09-13T00:33:02.094923+0000 mgr.y (mgr.24661) 1677 : cluster [DBG] pgmap v2923: 393 pgs: 393 active+clean; 8.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 42 KiB/s rd, 686 KiB/s wr, 65 op/s 2024-09-13T00:33:03.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:03 smithi136 bash[76371]: cluster 2024-09-13T00:33:02.094923+0000 mgr.y (mgr.24661) 1677 : cluster [DBG] pgmap v2923: 393 pgs: 393 active+clean; 8.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 42 KiB/s rd, 686 KiB/s wr, 65 op/s 2024-09-13T00:33:03.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:03 smithi181 bash[67587]: cluster 2024-09-13T00:33:02.094923+0000 mgr.y (mgr.24661) 1677 : cluster [DBG] pgmap v2923: 393 pgs: 393 active+clean; 8.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 42 KiB/s rd, 686 KiB/s wr, 65 op/s 2024-09-13T00:33:05.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:05 smithi136 bash[75164]: cluster 2024-09-13T00:33:04.096863+0000 mgr.y (mgr.24661) 1678 : cluster [DBG] pgmap v2924: 393 pgs: 393 active+clean; 8.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 23 KiB/s rd, 545 KiB/s wr, 36 op/s 2024-09-13T00:33:05.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:05 smithi136 bash[75164]: cluster 2024-09-13T00:33:04.550099+0000 mon.a (mon.0) 3110 : cluster [DBG] osdmap e1686: 8 total, 8 up, 8 in 2024-09-13T00:33:05.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:05 smithi136 bash[76371]: cluster 2024-09-13T00:33:04.096863+0000 mgr.y (mgr.24661) 1678 : cluster [DBG] pgmap v2924: 393 pgs: 393 active+clean; 8.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 23 KiB/s rd, 545 KiB/s wr, 36 op/s 2024-09-13T00:33:05.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:05 smithi136 bash[76371]: cluster 2024-09-13T00:33:04.550099+0000 mon.a (mon.0) 3110 : cluster [DBG] osdmap e1686: 8 total, 8 up, 8 in 2024-09-13T00:33:05.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:05 smithi181 bash[67587]: cluster 2024-09-13T00:33:04.096863+0000 mgr.y (mgr.24661) 1678 : cluster [DBG] pgmap v2924: 393 pgs: 393 active+clean; 8.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 23 KiB/s rd, 545 KiB/s wr, 36 op/s 2024-09-13T00:33:05.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:05 smithi181 bash[67587]: cluster 2024-09-13T00:33:04.550099+0000 mon.a (mon.0) 3110 : cluster [DBG] osdmap e1686: 8 total, 8 up, 8 in 2024-09-13T00:33:07.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:07 smithi136 bash[75164]: cluster 2024-09-13T00:33:06.098643+0000 mgr.y (mgr.24661) 1679 : cluster [DBG] pgmap v2926: 393 pgs: 393 active+clean; 12 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 17 KiB/s rd, 897 KiB/s wr, 25 op/s 2024-09-13T00:33:07.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:07 smithi136 bash[76371]: cluster 2024-09-13T00:33:06.098643+0000 mgr.y (mgr.24661) 1679 : cluster [DBG] pgmap v2926: 393 pgs: 393 active+clean; 12 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 17 KiB/s rd, 897 KiB/s wr, 25 op/s 2024-09-13T00:33:07.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:07 smithi181 bash[67587]: cluster 2024-09-13T00:33:06.098643+0000 mgr.y (mgr.24661) 1679 : cluster [DBG] pgmap v2926: 393 pgs: 393 active+clean; 12 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 17 KiB/s rd, 897 KiB/s wr, 25 op/s 2024-09-13T00:33:09.509 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:33:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:33:09] "GET /metrics HTTP/1.1" 200 55810 "" "Prometheus/2.43.0" 2024-09-13T00:33:09.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:09 smithi136 bash[75164]: cluster 2024-09-13T00:33:08.100923+0000 mgr.y (mgr.24661) 1680 : cluster [DBG] pgmap v2927: 393 pgs: 393 active+clean; 12 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 839 KiB/s wr, 11 op/s 2024-09-13T00:33:09.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:09 smithi136 bash[75164]: audit 2024-09-13T00:33:08.826713+0000 mon.a (mon.0) 3111 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:33:09.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:09 smithi136 bash[76371]: cluster 2024-09-13T00:33:08.100923+0000 mgr.y (mgr.24661) 1680 : cluster [DBG] pgmap v2927: 393 pgs: 393 active+clean; 12 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 839 KiB/s wr, 11 op/s 2024-09-13T00:33:09.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:09 smithi136 bash[76371]: audit 2024-09-13T00:33:08.826713+0000 mon.a (mon.0) 3111 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:33:09.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:09 smithi181 bash[67587]: cluster 2024-09-13T00:33:08.100923+0000 mgr.y (mgr.24661) 1680 : cluster [DBG] pgmap v2927: 393 pgs: 393 active+clean; 12 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 839 KiB/s wr, 11 op/s 2024-09-13T00:33:09.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:09 smithi181 bash[67587]: audit 2024-09-13T00:33:08.826713+0000 mon.a (mon.0) 3111 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:33:10.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:10 smithi136 bash[75164]: audit 2024-09-13T00:33:10.132188+0000 mon.a (mon.0) 3112 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "90.9", "id": [4, 5]}]: dispatch 2024-09-13T00:33:10.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:10 smithi136 bash[75164]: audit 2024-09-13T00:33:10.132883+0000 mon.a (mon.0) 3113 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "90.b", "id": [4, 6]}]: dispatch 2024-09-13T00:33:10.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:10 smithi136 bash[75164]: audit 2024-09-13T00:33:10.133468+0000 mon.a (mon.0) 3114 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "90.10", "id": [1, 5]}]: dispatch 2024-09-13T00:33:10.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:10 smithi136 bash[75164]: audit 2024-09-13T00:33:10.134044+0000 mon.a (mon.0) 3115 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "90.16", "id": [3, 2]}]: dispatch 2024-09-13T00:33:10.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:10 smithi136 bash[75164]: audit 2024-09-13T00:33:10.134625+0000 mon.a (mon.0) 3116 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "90.1d", "id": [1, 6]}]: dispatch 2024-09-13T00:33:10.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:10 smithi136 bash[75164]: audit 2024-09-13T00:33:10.135190+0000 mon.a (mon.0) 3117 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "93.9", "id": [2, 4]}]: dispatch 2024-09-13T00:33:10.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:10 smithi136 bash[75164]: audit 2024-09-13T00:33:10.135753+0000 mon.a (mon.0) 3118 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "93.13", "id": [2, 4]}]: dispatch 2024-09-13T00:33:10.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:10 smithi136 bash[75164]: audit 2024-09-13T00:33:10.136320+0000 mon.a (mon.0) 3119 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "93.1d", "id": [2, 3]}]: dispatch 2024-09-13T00:33:10.511 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:10 smithi136 bash[75164]: audit 2024-09-13T00:33:10.136893+0000 mon.a (mon.0) 3120 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "95.10", "id": [1, 2]}]: dispatch 2024-09-13T00:33:10.511 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:10 smithi136 bash[75164]: audit 2024-09-13T00:33:10.137459+0000 mon.a (mon.0) 3121 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "96.1", "id": [0, 3]}]: dispatch 2024-09-13T00:33:10.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:10 smithi136 bash[76371]: audit 2024-09-13T00:33:10.132188+0000 mon.a (mon.0) 3112 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "90.9", "id": [4, 5]}]: dispatch 2024-09-13T00:33:10.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:10 smithi136 bash[76371]: audit 2024-09-13T00:33:10.132883+0000 mon.a (mon.0) 3113 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "90.b", "id": [4, 6]}]: dispatch 2024-09-13T00:33:10.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:10 smithi136 bash[76371]: audit 2024-09-13T00:33:10.133468+0000 mon.a (mon.0) 3114 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "90.10", "id": [1, 5]}]: dispatch 2024-09-13T00:33:10.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:10 smithi136 bash[76371]: audit 2024-09-13T00:33:10.134044+0000 mon.a (mon.0) 3115 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "90.16", "id": [3, 2]}]: dispatch 2024-09-13T00:33:10.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:10 smithi136 bash[76371]: audit 2024-09-13T00:33:10.134625+0000 mon.a (mon.0) 3116 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "90.1d", "id": [1, 6]}]: dispatch 2024-09-13T00:33:10.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:10 smithi136 bash[76371]: audit 2024-09-13T00:33:10.135190+0000 mon.a (mon.0) 3117 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "93.9", "id": [2, 4]}]: dispatch 2024-09-13T00:33:10.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:10 smithi136 bash[76371]: audit 2024-09-13T00:33:10.135753+0000 mon.a (mon.0) 3118 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "93.13", "id": [2, 4]}]: dispatch 2024-09-13T00:33:10.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:10 smithi136 bash[76371]: audit 2024-09-13T00:33:10.136320+0000 mon.a (mon.0) 3119 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "93.1d", "id": [2, 3]}]: dispatch 2024-09-13T00:33:10.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:10 smithi136 bash[76371]: audit 2024-09-13T00:33:10.136893+0000 mon.a (mon.0) 3120 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "95.10", "id": [1, 2]}]: dispatch 2024-09-13T00:33:10.512 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:10 smithi136 bash[76371]: audit 2024-09-13T00:33:10.137459+0000 mon.a (mon.0) 3121 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "96.1", "id": [0, 3]}]: dispatch 2024-09-13T00:33:10.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:10 smithi181 bash[67587]: audit 2024-09-13T00:33:10.132188+0000 mon.a (mon.0) 3112 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "90.9", "id": [4, 5]}]: dispatch 2024-09-13T00:33:10.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:10 smithi181 bash[67587]: audit 2024-09-13T00:33:10.132883+0000 mon.a (mon.0) 3113 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "90.b", "id": [4, 6]}]: dispatch 2024-09-13T00:33:10.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:10 smithi181 bash[67587]: audit 2024-09-13T00:33:10.133468+0000 mon.a (mon.0) 3114 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "90.10", "id": [1, 5]}]: dispatch 2024-09-13T00:33:10.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:10 smithi181 bash[67587]: audit 2024-09-13T00:33:10.134044+0000 mon.a (mon.0) 3115 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "90.16", "id": [3, 2]}]: dispatch 2024-09-13T00:33:10.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:10 smithi181 bash[67587]: audit 2024-09-13T00:33:10.134625+0000 mon.a (mon.0) 3116 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "90.1d", "id": [1, 6]}]: dispatch 2024-09-13T00:33:10.600 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:10 smithi181 bash[67587]: audit 2024-09-13T00:33:10.135190+0000 mon.a (mon.0) 3117 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "93.9", "id": [2, 4]}]: dispatch 2024-09-13T00:33:10.600 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:10 smithi181 bash[67587]: audit 2024-09-13T00:33:10.135753+0000 mon.a (mon.0) 3118 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "93.13", "id": [2, 4]}]: dispatch 2024-09-13T00:33:10.600 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:10 smithi181 bash[67587]: audit 2024-09-13T00:33:10.136320+0000 mon.a (mon.0) 3119 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "93.1d", "id": [2, 3]}]: dispatch 2024-09-13T00:33:10.600 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:10 smithi181 bash[67587]: audit 2024-09-13T00:33:10.136893+0000 mon.a (mon.0) 3120 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "95.10", "id": [1, 2]}]: dispatch 2024-09-13T00:33:10.600 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:10 smithi181 bash[67587]: audit 2024-09-13T00:33:10.137459+0000 mon.a (mon.0) 3121 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "96.1", "id": [0, 3]}]: dispatch 2024-09-13T00:33:11.198 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:11 smithi136 bash[76371]: cluster 2024-09-13T00:33:10.102667+0000 mgr.y (mgr.24661) 1681 : cluster [DBG] pgmap v2928: 393 pgs: 393 active+clean; 13 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 5.3 KiB/s rd, 819 KiB/s wr, 9 op/s 2024-09-13T00:33:11.198 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:11 smithi136 bash[76371]: audit 2024-09-13T00:33:10.182969+0000 mon.a (mon.0) 3122 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "90.9", "id": [4, 5]}]': finished 2024-09-13T00:33:11.199 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:11 smithi136 bash[75164]: cluster 2024-09-13T00:33:10.102667+0000 mgr.y (mgr.24661) 1681 : cluster [DBG] pgmap v2928: 393 pgs: 393 active+clean; 13 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 5.3 KiB/s rd, 819 KiB/s wr, 9 op/s 2024-09-13T00:33:11.199 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:11 smithi136 bash[75164]: audit 2024-09-13T00:33:10.182969+0000 mon.a (mon.0) 3122 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "90.9", "id": [4, 5]}]': finished 2024-09-13T00:33:11.199 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:11 smithi136 bash[75164]: audit 2024-09-13T00:33:10.183150+0000 mon.a (mon.0) 3123 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "90.b", "id": [4, 6]}]': finished 2024-09-13T00:33:11.199 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:11 smithi136 bash[75164]: audit 2024-09-13T00:33:10.183308+0000 mon.a (mon.0) 3124 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "90.10", "id": [1, 5]}]': finished 2024-09-13T00:33:11.199 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:11 smithi136 bash[75164]: audit 2024-09-13T00:33:10.183470+0000 mon.a (mon.0) 3125 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "90.16", "id": [3, 2]}]': finished 2024-09-13T00:33:11.199 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:11 smithi136 bash[75164]: audit 2024-09-13T00:33:10.183634+0000 mon.a (mon.0) 3126 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "90.1d", "id": [1, 6]}]': finished 2024-09-13T00:33:11.199 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:11 smithi136 bash[75164]: audit 2024-09-13T00:33:10.183792+0000 mon.a (mon.0) 3127 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "93.9", "id": [2, 4]}]': finished 2024-09-13T00:33:11.199 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:11 smithi136 bash[75164]: audit 2024-09-13T00:33:10.183928+0000 mon.a (mon.0) 3128 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "93.13", "id": [2, 4]}]': finished 2024-09-13T00:33:11.199 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:11 smithi136 bash[75164]: audit 2024-09-13T00:33:10.184079+0000 mon.a (mon.0) 3129 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "93.1d", "id": [2, 3]}]': finished 2024-09-13T00:33:11.200 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:11 smithi136 bash[75164]: audit 2024-09-13T00:33:10.184225+0000 mon.a (mon.0) 3130 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "95.10", "id": [1, 2]}]': finished 2024-09-13T00:33:11.200 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:11 smithi136 bash[75164]: audit 2024-09-13T00:33:10.184377+0000 mon.a (mon.0) 3131 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "96.1", "id": [0, 3]}]': finished 2024-09-13T00:33:11.200 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:11 smithi136 bash[75164]: cluster 2024-09-13T00:33:10.184505+0000 mon.a (mon.0) 3132 : cluster [DBG] osdmap e1687: 8 total, 8 up, 8 in 2024-09-13T00:33:11.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:11 smithi136 bash[76371]: audit 2024-09-13T00:33:10.183150+0000 mon.a (mon.0) 3123 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "90.b", "id": [4, 6]}]': finished 2024-09-13T00:33:11.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:11 smithi136 bash[76371]: audit 2024-09-13T00:33:10.183308+0000 mon.a (mon.0) 3124 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "90.10", "id": [1, 5]}]': finished 2024-09-13T00:33:11.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:11 smithi136 bash[76371]: audit 2024-09-13T00:33:10.183470+0000 mon.a (mon.0) 3125 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "90.16", "id": [3, 2]}]': finished 2024-09-13T00:33:11.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:11 smithi136 bash[76371]: audit 2024-09-13T00:33:10.183634+0000 mon.a (mon.0) 3126 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "90.1d", "id": [1, 6]}]': finished 2024-09-13T00:33:11.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:11 smithi136 bash[76371]: audit 2024-09-13T00:33:10.183792+0000 mon.a (mon.0) 3127 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "93.9", "id": [2, 4]}]': finished 2024-09-13T00:33:11.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:11 smithi136 bash[76371]: audit 2024-09-13T00:33:10.183928+0000 mon.a (mon.0) 3128 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "93.13", "id": [2, 4]}]': finished 2024-09-13T00:33:11.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:11 smithi136 bash[76371]: audit 2024-09-13T00:33:10.184079+0000 mon.a (mon.0) 3129 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "93.1d", "id": [2, 3]}]': finished 2024-09-13T00:33:11.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:11 smithi136 bash[76371]: audit 2024-09-13T00:33:10.184225+0000 mon.a (mon.0) 3130 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "95.10", "id": [1, 2]}]': finished 2024-09-13T00:33:11.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:11 smithi136 bash[76371]: audit 2024-09-13T00:33:10.184377+0000 mon.a (mon.0) 3131 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "96.1", "id": [0, 3]}]': finished 2024-09-13T00:33:11.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:11 smithi136 bash[76371]: cluster 2024-09-13T00:33:10.184505+0000 mon.a (mon.0) 3132 : cluster [DBG] osdmap e1687: 8 total, 8 up, 8 in 2024-09-13T00:33:11.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:11 smithi181 bash[67587]: cluster 2024-09-13T00:33:10.102667+0000 mgr.y (mgr.24661) 1681 : cluster [DBG] pgmap v2928: 393 pgs: 393 active+clean; 13 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 5.3 KiB/s rd, 819 KiB/s wr, 9 op/s 2024-09-13T00:33:11.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:11 smithi181 bash[67587]: audit 2024-09-13T00:33:10.182969+0000 mon.a (mon.0) 3122 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "90.9", "id": [4, 5]}]': finished 2024-09-13T00:33:11.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:11 smithi181 bash[67587]: audit 2024-09-13T00:33:10.183150+0000 mon.a (mon.0) 3123 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "90.b", "id": [4, 6]}]': finished 2024-09-13T00:33:11.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:11 smithi181 bash[67587]: audit 2024-09-13T00:33:10.183308+0000 mon.a (mon.0) 3124 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "90.10", "id": [1, 5]}]': finished 2024-09-13T00:33:11.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:11 smithi181 bash[67587]: audit 2024-09-13T00:33:10.183470+0000 mon.a (mon.0) 3125 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "90.16", "id": [3, 2]}]': finished 2024-09-13T00:33:11.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:11 smithi181 bash[67587]: audit 2024-09-13T00:33:10.183634+0000 mon.a (mon.0) 3126 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "90.1d", "id": [1, 6]}]': finished 2024-09-13T00:33:11.600 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:11 smithi181 bash[67587]: audit 2024-09-13T00:33:10.183792+0000 mon.a (mon.0) 3127 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "93.9", "id": [2, 4]}]': finished 2024-09-13T00:33:11.600 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:11 smithi181 bash[67587]: audit 2024-09-13T00:33:10.183928+0000 mon.a (mon.0) 3128 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "93.13", "id": [2, 4]}]': finished 2024-09-13T00:33:11.600 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:11 smithi181 bash[67587]: audit 2024-09-13T00:33:10.184079+0000 mon.a (mon.0) 3129 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "93.1d", "id": [2, 3]}]': finished 2024-09-13T00:33:11.600 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:11 smithi181 bash[67587]: audit 2024-09-13T00:33:10.184225+0000 mon.a (mon.0) 3130 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "95.10", "id": [1, 2]}]': finished 2024-09-13T00:33:11.600 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:11 smithi181 bash[67587]: audit 2024-09-13T00:33:10.184377+0000 mon.a (mon.0) 3131 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "96.1", "id": [0, 3]}]': finished 2024-09-13T00:33:11.600 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:11 smithi181 bash[67587]: cluster 2024-09-13T00:33:10.184505+0000 mon.a (mon.0) 3132 : cluster [DBG] osdmap e1687: 8 total, 8 up, 8 in 2024-09-13T00:33:12.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:12 smithi136 bash[76371]: cluster 2024-09-13T00:33:11.183955+0000 mon.a (mon.0) 3133 : cluster [DBG] osdmap e1688: 8 total, 8 up, 8 in 2024-09-13T00:33:12.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:12 smithi136 bash[75164]: cluster 2024-09-13T00:33:11.183955+0000 mon.a (mon.0) 3133 : cluster [DBG] osdmap e1688: 8 total, 8 up, 8 in 2024-09-13T00:33:12.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:12 smithi181 bash[67587]: cluster 2024-09-13T00:33:11.183955+0000 mon.a (mon.0) 3133 : cluster [DBG] osdmap e1688: 8 total, 8 up, 8 in 2024-09-13T00:33:12.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:33:12 smithi181 bash[95505]: ts=2024-09-13T00:33:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:33:13.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:13 smithi136 bash[76371]: cluster 2024-09-13T00:33:12.104342+0000 mgr.y (mgr.24661) 1682 : cluster [DBG] pgmap v2931: 393 pgs: 4 peering, 389 active+clean; 17 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 2.0 KiB/s rd, 678 KiB/s wr, 3 op/s; 2 B/s, 0 objects/s recovering 2024-09-13T00:33:13.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:13 smithi136 bash[76371]: cluster 2024-09-13T00:33:12.191406+0000 mon.a (mon.0) 3134 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-13T00:33:13.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:13 smithi136 bash[75164]: cluster 2024-09-13T00:33:12.104342+0000 mgr.y (mgr.24661) 1682 : cluster [DBG] pgmap v2931: 393 pgs: 4 peering, 389 active+clean; 17 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 2.0 KiB/s rd, 678 KiB/s wr, 3 op/s; 2 B/s, 0 objects/s recovering 2024-09-13T00:33:13.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:13 smithi136 bash[75164]: cluster 2024-09-13T00:33:12.191406+0000 mon.a (mon.0) 3134 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-13T00:33:13.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:13 smithi181 bash[67587]: cluster 2024-09-13T00:33:12.104342+0000 mgr.y (mgr.24661) 1682 : cluster [DBG] pgmap v2931: 393 pgs: 4 peering, 389 active+clean; 17 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 2.0 KiB/s rd, 678 KiB/s wr, 3 op/s; 2 B/s, 0 objects/s recovering 2024-09-13T00:33:13.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:13 smithi181 bash[67587]: cluster 2024-09-13T00:33:12.191406+0000 mon.a (mon.0) 3134 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-13T00:33:15.121 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.RemoveFullTry (21971 ms) 2024-09-13T00:33:15.121 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.RemoveFullTryDataPool 2024-09-13T00:33:15.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:15 smithi181 bash[67587]: cluster 2024-09-13T00:33:14.106221+0000 mgr.y (mgr.24661) 1683 : cluster [DBG] pgmap v2932: 393 pgs: 4 peering, 389 active+clean; 21 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 1.7 KiB/s rd, 1.1 MiB/s wr, 3 op/s; 7 B/s, 0 keys/s, 0 objects/s recovering 2024-09-13T00:33:15.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:15 smithi181 bash[67587]: cluster 2024-09-13T00:33:14.541515+0000 mon.a (mon.0) 3135 : cluster [WRN] pool 'test-librbd-smithi136-114798-22' is full (reached quota's max_bytes: 10 MiB) 2024-09-13T00:33:15.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:15 smithi181 bash[67587]: cluster 2024-09-13T00:33:14.542199+0000 mon.a (mon.0) 3136 : cluster [WRN] Health check failed: 1 pool(s) full (POOL_FULL) 2024-09-13T00:33:15.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:15 smithi181 bash[67587]: cluster 2024-09-13T00:33:14.552508+0000 mon.a (mon.0) 3137 : cluster [DBG] osdmap e1689: 8 total, 8 up, 8 in 2024-09-13T00:33:16.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:15 smithi136 bash[75164]: cluster 2024-09-13T00:33:14.106221+0000 mgr.y (mgr.24661) 1683 : cluster [DBG] pgmap v2932: 393 pgs: 4 peering, 389 active+clean; 21 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 1.7 KiB/s rd, 1.1 MiB/s wr, 3 op/s; 7 B/s, 0 keys/s, 0 objects/s recovering 2024-09-13T00:33:16.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:15 smithi136 bash[75164]: cluster 2024-09-13T00:33:14.541515+0000 mon.a (mon.0) 3135 : cluster [WRN] pool 'test-librbd-smithi136-114798-22' is full (reached quota's max_bytes: 10 MiB) 2024-09-13T00:33:16.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:15 smithi136 bash[75164]: cluster 2024-09-13T00:33:14.542199+0000 mon.a (mon.0) 3136 : cluster [WRN] Health check failed: 1 pool(s) full (POOL_FULL) 2024-09-13T00:33:16.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:15 smithi136 bash[75164]: cluster 2024-09-13T00:33:14.552508+0000 mon.a (mon.0) 3137 : cluster [DBG] osdmap e1689: 8 total, 8 up, 8 in 2024-09-13T00:33:16.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:15 smithi136 bash[76371]: cluster 2024-09-13T00:33:14.106221+0000 mgr.y (mgr.24661) 1683 : cluster [DBG] pgmap v2932: 393 pgs: 4 peering, 389 active+clean; 21 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 1.7 KiB/s rd, 1.1 MiB/s wr, 3 op/s; 7 B/s, 0 keys/s, 0 objects/s recovering 2024-09-13T00:33:16.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:15 smithi136 bash[76371]: cluster 2024-09-13T00:33:14.541515+0000 mon.a (mon.0) 3135 : cluster [WRN] pool 'test-librbd-smithi136-114798-22' is full (reached quota's max_bytes: 10 MiB) 2024-09-13T00:33:16.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:15 smithi136 bash[76371]: cluster 2024-09-13T00:33:14.542199+0000 mon.a (mon.0) 3136 : cluster [WRN] Health check failed: 1 pool(s) full (POOL_FULL) 2024-09-13T00:33:16.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:15 smithi136 bash[76371]: cluster 2024-09-13T00:33:14.552508+0000 mon.a (mon.0) 3137 : cluster [DBG] osdmap e1689: 8 total, 8 up, 8 in 2024-09-13T00:33:17.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:16 smithi136 bash[75164]: cluster 2024-09-13T00:33:15.658950+0000 mon.a (mon.0) 3138 : cluster [DBG] osdmap e1690: 8 total, 8 up, 8 in 2024-09-13T00:33:17.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:16 smithi136 bash[75164]: audit 2024-09-13T00:33:15.676700+0000 mon.a (mon.0) 3139 : audit [INF] from='client.? 172.21.15.136:0/2689665424' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-23","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:33:17.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:16 smithi136 bash[75164]: cluster 2024-09-13T00:33:16.107295+0000 mgr.y (mgr.24661) 1684 : cluster [DBG] pgmap v2935: 425 pgs: 32 unknown, 1 peering, 392 active+clean; 18 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 14 KiB/s rd, 1.5 MiB/s wr, 19 op/s; 13 B/s, 2 keys/s, 1 objects/s recovering 2024-09-13T00:33:17.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:16 smithi136 bash[76371]: cluster 2024-09-13T00:33:15.658950+0000 mon.a (mon.0) 3138 : cluster [DBG] osdmap e1690: 8 total, 8 up, 8 in 2024-09-13T00:33:17.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:16 smithi136 bash[76371]: audit 2024-09-13T00:33:15.676700+0000 mon.a (mon.0) 3139 : audit [INF] from='client.? 172.21.15.136:0/2689665424' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-23","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:33:17.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:16 smithi136 bash[76371]: cluster 2024-09-13T00:33:16.107295+0000 mgr.y (mgr.24661) 1684 : cluster [DBG] pgmap v2935: 425 pgs: 32 unknown, 1 peering, 392 active+clean; 18 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 14 KiB/s rd, 1.5 MiB/s wr, 19 op/s; 13 B/s, 2 keys/s, 1 objects/s recovering 2024-09-13T00:33:17.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:16 smithi181 bash[67587]: cluster 2024-09-13T00:33:15.658950+0000 mon.a (mon.0) 3138 : cluster [DBG] osdmap e1690: 8 total, 8 up, 8 in 2024-09-13T00:33:17.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:16 smithi181 bash[67587]: audit 2024-09-13T00:33:15.676700+0000 mon.a (mon.0) 3139 : audit [INF] from='client.? 172.21.15.136:0/2689665424' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-23","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:33:17.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:16 smithi181 bash[67587]: cluster 2024-09-13T00:33:16.107295+0000 mgr.y (mgr.24661) 1684 : cluster [DBG] pgmap v2935: 425 pgs: 32 unknown, 1 peering, 392 active+clean; 18 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 14 KiB/s rd, 1.5 MiB/s wr, 19 op/s; 13 B/s, 2 keys/s, 1 objects/s recovering 2024-09-13T00:33:18.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:17 smithi136 bash[75164]: cluster 2024-09-13T00:33:16.649362+0000 mon.a (mon.0) 3140 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:33:18.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:17 smithi136 bash[75164]: cluster 2024-09-13T00:33:16.649613+0000 mon.a (mon.0) 3141 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-13T00:33:18.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:17 smithi136 bash[75164]: audit 2024-09-13T00:33:16.660439+0000 mon.a (mon.0) 3142 : audit [INF] from='client.? 172.21.15.136:0/2689665424' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-23","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:33:18.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:17 smithi136 bash[75164]: cluster 2024-09-13T00:33:16.661221+0000 mon.a (mon.0) 3143 : cluster [DBG] osdmap e1691: 8 total, 8 up, 8 in 2024-09-13T00:33:18.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:17 smithi136 bash[76371]: cluster 2024-09-13T00:33:16.649362+0000 mon.a (mon.0) 3140 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:33:18.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:17 smithi136 bash[76371]: cluster 2024-09-13T00:33:16.649613+0000 mon.a (mon.0) 3141 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-13T00:33:18.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:17 smithi136 bash[76371]: audit 2024-09-13T00:33:16.660439+0000 mon.a (mon.0) 3142 : audit [INF] from='client.? 172.21.15.136:0/2689665424' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-23","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:33:18.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:17 smithi136 bash[76371]: cluster 2024-09-13T00:33:16.661221+0000 mon.a (mon.0) 3143 : cluster [DBG] osdmap e1691: 8 total, 8 up, 8 in 2024-09-13T00:33:18.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:17 smithi181 bash[67587]: cluster 2024-09-13T00:33:16.649362+0000 mon.a (mon.0) 3140 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:33:18.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:17 smithi181 bash[67587]: cluster 2024-09-13T00:33:16.649613+0000 mon.a (mon.0) 3141 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-13T00:33:18.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:17 smithi181 bash[67587]: audit 2024-09-13T00:33:16.660439+0000 mon.a (mon.0) 3142 : audit [INF] from='client.? 172.21.15.136:0/2689665424' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-23","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:33:18.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:17 smithi181 bash[67587]: cluster 2024-09-13T00:33:16.661221+0000 mon.a (mon.0) 3143 : cluster [DBG] osdmap e1691: 8 total, 8 up, 8 in 2024-09-13T00:33:18.742 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:33:19.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:18 smithi136 bash[76371]: cluster 2024-09-13T00:33:17.681947+0000 mon.a (mon.0) 3144 : cluster [DBG] osdmap e1692: 8 total, 8 up, 8 in 2024-09-13T00:33:19.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:18 smithi136 bash[76371]: audit 2024-09-13T00:33:17.683082+0000 mon.c (mon.2) 125 : audit [INF] from='client.? 172.21.15.136:0/3316243620' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-24","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:33:19.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:18 smithi136 bash[76371]: audit 2024-09-13T00:33:17.685145+0000 mon.a (mon.0) 3145 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-24","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:33:19.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:18 smithi136 bash[76371]: cluster 2024-09-13T00:33:18.108389+0000 mgr.y (mgr.24661) 1685 : cluster [DBG] pgmap v2938: 457 pgs: 6 creating+activating, 8 creating+peering, 48 unknown, 395 active+clean; 14 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 27 KiB/s rd, 257 KiB/s wr, 36 op/s; 3 B/s, 1 keys/s, 0 objects/s recovering 2024-09-13T00:33:19.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:18 smithi136 bash[76371]: audit 2024-09-13T00:33:18.684037+0000 mon.a (mon.0) 3146 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-24","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:33:19.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:18 smithi136 bash[76371]: cluster 2024-09-13T00:33:18.687152+0000 mon.a (mon.0) 3147 : cluster [DBG] osdmap e1693: 8 total, 8 up, 8 in 2024-09-13T00:33:19.010 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:33:18 smithi136 bash[68787]: [13/Sep/2024:00:33:18] ENGINE Bus STOPPING 2024-09-13T00:33:19.010 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:33:18 smithi136 bash[68787]: [13/Sep/2024:00:33:18] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T00:33:19.010 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:33:18 smithi136 bash[68787]: [13/Sep/2024:00:33:18] ENGINE Bus STOPPED 2024-09-13T00:33:19.010 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:33:18 smithi136 bash[68787]: [13/Sep/2024:00:33:18] ENGINE Bus STARTING 2024-09-13T00:33:19.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:18 smithi136 bash[75164]: cluster 2024-09-13T00:33:17.681947+0000 mon.a (mon.0) 3144 : cluster [DBG] osdmap e1692: 8 total, 8 up, 8 in 2024-09-13T00:33:19.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:18 smithi136 bash[75164]: audit 2024-09-13T00:33:17.683082+0000 mon.c (mon.2) 125 : audit [INF] from='client.? 172.21.15.136:0/3316243620' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-24","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:33:19.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:18 smithi136 bash[75164]: audit 2024-09-13T00:33:17.685145+0000 mon.a (mon.0) 3145 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-24","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:33:19.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:18 smithi136 bash[75164]: cluster 2024-09-13T00:33:18.108389+0000 mgr.y (mgr.24661) 1685 : cluster [DBG] pgmap v2938: 457 pgs: 6 creating+activating, 8 creating+peering, 48 unknown, 395 active+clean; 14 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 27 KiB/s rd, 257 KiB/s wr, 36 op/s; 3 B/s, 1 keys/s, 0 objects/s recovering 2024-09-13T00:33:19.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:18 smithi136 bash[75164]: audit 2024-09-13T00:33:18.684037+0000 mon.a (mon.0) 3146 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-24","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:33:19.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:18 smithi136 bash[75164]: cluster 2024-09-13T00:33:18.687152+0000 mon.a (mon.0) 3147 : cluster [DBG] osdmap e1693: 8 total, 8 up, 8 in 2024-09-13T00:33:19.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:18 smithi181 bash[67587]: cluster 2024-09-13T00:33:17.681947+0000 mon.a (mon.0) 3144 : cluster [DBG] osdmap e1692: 8 total, 8 up, 8 in 2024-09-13T00:33:19.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:18 smithi181 bash[67587]: audit 2024-09-13T00:33:17.683082+0000 mon.c (mon.2) 125 : audit [INF] from='client.? 172.21.15.136:0/3316243620' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-24","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:33:19.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:18 smithi181 bash[67587]: audit 2024-09-13T00:33:17.685145+0000 mon.a (mon.0) 3145 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-24","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:33:19.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:18 smithi181 bash[67587]: cluster 2024-09-13T00:33:18.108389+0000 mgr.y (mgr.24661) 1685 : cluster [DBG] pgmap v2938: 457 pgs: 6 creating+activating, 8 creating+peering, 48 unknown, 395 active+clean; 14 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 27 KiB/s rd, 257 KiB/s wr, 36 op/s; 3 B/s, 1 keys/s, 0 objects/s recovering 2024-09-13T00:33:19.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:18 smithi181 bash[67587]: audit 2024-09-13T00:33:18.684037+0000 mon.a (mon.0) 3146 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-24","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:33:19.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:18 smithi181 bash[67587]: cluster 2024-09-13T00:33:18.687152+0000 mon.a (mon.0) 3147 : cluster [DBG] osdmap e1693: 8 total, 8 up, 8 in 2024-09-13T00:33:19.416 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[68787]: [13/Sep/2024:00:33:19] ENGINE Serving on http://:::9283 2024-09-13T00:33:19.416 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[68787]: [13/Sep/2024:00:33:19] ENGINE Bus STARTED 2024-09-13T00:33:19.741 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:33:19] "GET /metrics HTTP/1.1" 200 55810 "" "Prometheus/2.43.0" 2024-09-13T00:33:20.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[75164]: audit 2024-09-13T00:33:18.742545+0000 mon.a (mon.0) 3148 : audit [INF] from='client.? ' entity='client.admin' 2024-09-13T00:33:20.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[75164]: audit 2024-09-13T00:33:18.758226+0000 mon.a (mon.0) 3149 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:33:20.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[75164]: audit 2024-09-13T00:33:18.760637+0000 mon.a (mon.0) 3150 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:33:20.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[75164]: audit 2024-09-13T00:33:18.763643+0000 mon.a (mon.0) 3151 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:33:20.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[75164]: audit 2024-09-13T00:33:18.764510+0000 mon.a (mon.0) 3152 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:33:20.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[75164]: audit 2024-09-13T00:33:18.764800+0000 mon.a (mon.0) 3153 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T00:33:20.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[75164]: audit 2024-09-13T00:33:18.776649+0000 mon.a (mon.0) 3154 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:33:20.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[75164]: audit 2024-09-13T00:33:18.949796+0000 mon.a (mon.0) 3155 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:33:20.012 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[75164]: audit 2024-09-13T00:33:18.950077+0000 mon.a (mon.0) 3156 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:33:20.012 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[75164]: audit 2024-09-13T00:33:18.950302+0000 mon.a (mon.0) 3157 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:33:20.012 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[75164]: audit 2024-09-13T00:33:18.952711+0000 mon.a (mon.0) 3158 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:33:20.012 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[75164]: audit 2024-09-13T00:33:18.952929+0000 mon.a (mon.0) 3159 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:33:20.012 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[75164]: audit 2024-09-13T00:33:18.953141+0000 mon.a (mon.0) 3160 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:33:20.012 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[75164]: audit 2024-09-13T00:33:18.954646+0000 mon.a (mon.0) 3161 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:33:20.013 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[75164]: audit 2024-09-13T00:33:18.954946+0000 mon.a (mon.0) 3162 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:33:20.013 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[75164]: audit 2024-09-13T00:33:18.955071+0000 mon.a (mon.0) 3163 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:33:20.013 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[75164]: audit 2024-09-13T00:33:18.967947+0000 mon.a (mon.0) 3164 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T00:33:20.013 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[75164]: audit 2024-09-13T00:33:18.967997+0000 mon.a (mon.0) 3165 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T00:33:20.013 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[75164]: audit 2024-09-13T00:33:18.968129+0000 mon.a (mon.0) 3166 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:33:20.014 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[76371]: audit 2024-09-13T00:33:18.742545+0000 mon.a (mon.0) 3148 : audit [INF] from='client.? ' entity='client.admin' 2024-09-13T00:33:20.014 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[76371]: audit 2024-09-13T00:33:18.758226+0000 mon.a (mon.0) 3149 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:33:20.014 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[76371]: audit 2024-09-13T00:33:18.760637+0000 mon.a (mon.0) 3150 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:33:20.014 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[76371]: audit 2024-09-13T00:33:18.763643+0000 mon.a (mon.0) 3151 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:33:20.014 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[76371]: audit 2024-09-13T00:33:18.764510+0000 mon.a (mon.0) 3152 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:33:20.014 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[76371]: audit 2024-09-13T00:33:18.764800+0000 mon.a (mon.0) 3153 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T00:33:20.014 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[76371]: audit 2024-09-13T00:33:18.776649+0000 mon.a (mon.0) 3154 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:33:20.015 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[76371]: audit 2024-09-13T00:33:18.949796+0000 mon.a (mon.0) 3155 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:33:20.015 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[76371]: audit 2024-09-13T00:33:18.950077+0000 mon.a (mon.0) 3156 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:33:20.015 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[76371]: audit 2024-09-13T00:33:18.950302+0000 mon.a (mon.0) 3157 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:33:20.015 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[76371]: audit 2024-09-13T00:33:18.952711+0000 mon.a (mon.0) 3158 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:33:20.015 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[76371]: audit 2024-09-13T00:33:18.952929+0000 mon.a (mon.0) 3159 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:33:20.015 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[76371]: audit 2024-09-13T00:33:18.953141+0000 mon.a (mon.0) 3160 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:33:20.016 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[76371]: audit 2024-09-13T00:33:18.954646+0000 mon.a (mon.0) 3161 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:33:20.016 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[76371]: audit 2024-09-13T00:33:18.954946+0000 mon.a (mon.0) 3162 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:33:20.016 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[76371]: audit 2024-09-13T00:33:18.955071+0000 mon.a (mon.0) 3163 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:33:20.016 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[76371]: audit 2024-09-13T00:33:18.967947+0000 mon.a (mon.0) 3164 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T00:33:20.016 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[76371]: audit 2024-09-13T00:33:18.967997+0000 mon.a (mon.0) 3165 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T00:33:20.016 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[76371]: audit 2024-09-13T00:33:18.968129+0000 mon.a (mon.0) 3166 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:33:20.016 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[75164]: audit 2024-09-13T00:33:18.968273+0000 mon.a (mon.0) 3167 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:33:20.017 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[75164]: audit 2024-09-13T00:33:18.968424+0000 mon.a (mon.0) 3168 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:33:20.017 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[75164]: audit 2024-09-13T00:33:18.969896+0000 mon.b (mon.1) 21 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T00:33:20.017 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[75164]: audit 2024-09-13T00:33:18.969999+0000 mon.b (mon.1) 22 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T00:33:20.017 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[75164]: audit 2024-09-13T00:33:18.970460+0000 mon.a (mon.0) 3169 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:33:20.017 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[75164]: audit 2024-09-13T00:33:18.970577+0000 mon.a (mon.0) 3170 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:33:20.017 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[75164]: audit 2024-09-13T00:33:18.970685+0000 mon.a (mon.0) 3171 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:33:20.017 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[75164]: audit 2024-09-13T00:33:18.971798+0000 mon.c (mon.2) 126 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T00:33:20.017 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[75164]: audit 2024-09-13T00:33:18.971849+0000 mon.c (mon.2) 127 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T00:33:20.017 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[75164]: audit 2024-09-13T00:33:18.971974+0000 mon.a (mon.0) 3172 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:33:20.017 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[75164]: audit 2024-09-13T00:33:18.972264+0000 mon.a (mon.0) 3173 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:33:20.018 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[75164]: audit 2024-09-13T00:33:18.972375+0000 mon.a (mon.0) 3174 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:33:20.018 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[75164]: cluster 2024-09-13T00:33:19.543862+0000 mon.a (mon.0) 3175 : cluster [INF] pool 'test-librbd-smithi136-114798-22' no longer out of quota; removing NO_QUOTA flag 2024-09-13T00:33:20.018 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[75164]: cluster 2024-09-13T00:33:19.544645+0000 mon.a (mon.0) 3176 : cluster [INF] Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-09-13T00:33:20.018 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[75164]: cluster 2024-09-13T00:33:19.555435+0000 mon.a (mon.0) 3177 : cluster [DBG] osdmap e1694: 8 total, 8 up, 8 in 2024-09-13T00:33:20.018 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[76371]: audit 2024-09-13T00:33:18.968273+0000 mon.a (mon.0) 3167 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:33:20.018 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[76371]: audit 2024-09-13T00:33:18.968424+0000 mon.a (mon.0) 3168 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:33:20.018 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[76371]: audit 2024-09-13T00:33:18.969896+0000 mon.b (mon.1) 21 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T00:33:20.019 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[76371]: audit 2024-09-13T00:33:18.969999+0000 mon.b (mon.1) 22 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T00:33:20.019 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[76371]: audit 2024-09-13T00:33:18.970460+0000 mon.a (mon.0) 3169 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:33:20.019 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[76371]: audit 2024-09-13T00:33:18.970577+0000 mon.a (mon.0) 3170 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:33:20.019 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[76371]: audit 2024-09-13T00:33:18.970685+0000 mon.a (mon.0) 3171 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:33:20.019 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[76371]: audit 2024-09-13T00:33:18.971798+0000 mon.c (mon.2) 126 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T00:33:20.019 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[76371]: audit 2024-09-13T00:33:18.971849+0000 mon.c (mon.2) 127 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T00:33:20.019 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[76371]: audit 2024-09-13T00:33:18.971974+0000 mon.a (mon.0) 3172 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:33:20.019 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[76371]: audit 2024-09-13T00:33:18.972264+0000 mon.a (mon.0) 3173 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:33:20.019 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[76371]: audit 2024-09-13T00:33:18.972375+0000 mon.a (mon.0) 3174 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:33:20.019 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[76371]: cluster 2024-09-13T00:33:19.543862+0000 mon.a (mon.0) 3175 : cluster [INF] pool 'test-librbd-smithi136-114798-22' no longer out of quota; removing NO_QUOTA flag 2024-09-13T00:33:20.019 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[76371]: cluster 2024-09-13T00:33:19.544645+0000 mon.a (mon.0) 3176 : cluster [INF] Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-09-13T00:33:20.019 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:19 smithi136 bash[76371]: cluster 2024-09-13T00:33:19.555435+0000 mon.a (mon.0) 3177 : cluster [DBG] osdmap e1694: 8 total, 8 up, 8 in 2024-09-13T00:33:20.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:19 smithi181 bash[67587]: audit 2024-09-13T00:33:18.742545+0000 mon.a (mon.0) 3148 : audit [INF] from='client.? ' entity='client.admin' 2024-09-13T00:33:20.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:19 smithi181 bash[67587]: audit 2024-09-13T00:33:18.758226+0000 mon.a (mon.0) 3149 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:33:20.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:19 smithi181 bash[67587]: audit 2024-09-13T00:33:18.760637+0000 mon.a (mon.0) 3150 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:33:20.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:19 smithi181 bash[67587]: audit 2024-09-13T00:33:18.763643+0000 mon.a (mon.0) 3151 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:33:20.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:19 smithi181 bash[67587]: audit 2024-09-13T00:33:18.764510+0000 mon.a (mon.0) 3152 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:33:20.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:19 smithi181 bash[67587]: audit 2024-09-13T00:33:18.764800+0000 mon.a (mon.0) 3153 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T00:33:20.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:19 smithi181 bash[67587]: audit 2024-09-13T00:33:18.776649+0000 mon.a (mon.0) 3154 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:33:20.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:19 smithi181 bash[67587]: audit 2024-09-13T00:33:18.949796+0000 mon.a (mon.0) 3155 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:33:20.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:19 smithi181 bash[67587]: audit 2024-09-13T00:33:18.950077+0000 mon.a (mon.0) 3156 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:33:20.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:19 smithi181 bash[67587]: audit 2024-09-13T00:33:18.950302+0000 mon.a (mon.0) 3157 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:33:20.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:19 smithi181 bash[67587]: audit 2024-09-13T00:33:18.952711+0000 mon.a (mon.0) 3158 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:33:20.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:19 smithi181 bash[67587]: audit 2024-09-13T00:33:18.952929+0000 mon.a (mon.0) 3159 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:33:20.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:19 smithi181 bash[67587]: audit 2024-09-13T00:33:18.953141+0000 mon.a (mon.0) 3160 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:33:20.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:19 smithi181 bash[67587]: audit 2024-09-13T00:33:18.954646+0000 mon.a (mon.0) 3161 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:33:20.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:19 smithi181 bash[67587]: audit 2024-09-13T00:33:18.954946+0000 mon.a (mon.0) 3162 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:33:20.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:19 smithi181 bash[67587]: audit 2024-09-13T00:33:18.955071+0000 mon.a (mon.0) 3163 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:33:20.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:19 smithi181 bash[67587]: audit 2024-09-13T00:33:18.967947+0000 mon.a (mon.0) 3164 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T00:33:20.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:19 smithi181 bash[67587]: audit 2024-09-13T00:33:18.967997+0000 mon.a (mon.0) 3165 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T00:33:20.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:19 smithi181 bash[67587]: audit 2024-09-13T00:33:18.968129+0000 mon.a (mon.0) 3166 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:33:20.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:19 smithi181 bash[67587]: audit 2024-09-13T00:33:18.968273+0000 mon.a (mon.0) 3167 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:33:20.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:19 smithi181 bash[67587]: audit 2024-09-13T00:33:18.968424+0000 mon.a (mon.0) 3168 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:33:20.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:19 smithi181 bash[67587]: audit 2024-09-13T00:33:18.969896+0000 mon.b (mon.1) 21 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T00:33:20.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:19 smithi181 bash[67587]: audit 2024-09-13T00:33:18.969999+0000 mon.b (mon.1) 22 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T00:33:20.102 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:19 smithi181 bash[67587]: audit 2024-09-13T00:33:18.970460+0000 mon.a (mon.0) 3169 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:33:20.102 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:19 smithi181 bash[67587]: audit 2024-09-13T00:33:18.970577+0000 mon.a (mon.0) 3170 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:33:20.102 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:19 smithi181 bash[67587]: audit 2024-09-13T00:33:18.970685+0000 mon.a (mon.0) 3171 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:33:20.102 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:19 smithi181 bash[67587]: audit 2024-09-13T00:33:18.971798+0000 mon.c (mon.2) 126 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T00:33:20.102 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:19 smithi181 bash[67587]: audit 2024-09-13T00:33:18.971849+0000 mon.c (mon.2) 127 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T00:33:20.102 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:19 smithi181 bash[67587]: audit 2024-09-13T00:33:18.971974+0000 mon.a (mon.0) 3172 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:33:20.102 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:19 smithi181 bash[67587]: audit 2024-09-13T00:33:18.972264+0000 mon.a (mon.0) 3173 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:33:20.102 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:19 smithi181 bash[67587]: audit 2024-09-13T00:33:18.972375+0000 mon.a (mon.0) 3174 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:33:20.102 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:19 smithi181 bash[67587]: cluster 2024-09-13T00:33:19.543862+0000 mon.a (mon.0) 3175 : cluster [INF] pool 'test-librbd-smithi136-114798-22' no longer out of quota; removing NO_QUOTA flag 2024-09-13T00:33:20.102 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:19 smithi181 bash[67587]: cluster 2024-09-13T00:33:19.544645+0000 mon.a (mon.0) 3176 : cluster [INF] Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-09-13T00:33:20.102 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:19 smithi181 bash[67587]: cluster 2024-09-13T00:33:19.555435+0000 mon.a (mon.0) 3177 : cluster [DBG] osdmap e1694: 8 total, 8 up, 8 in 2024-09-13T00:33:21.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:20 smithi136 bash[76371]: cluster 2024-09-13T00:33:20.109417+0000 mgr.y (mgr.24661) 1686 : cluster [DBG] pgmap v2941: 457 pgs: 6 creating+activating, 10 creating+peering, 31 unknown, 410 active+clean; 4.6 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.0 KiB/s rd, 2.2 KiB/s wr, 16 op/s 2024-09-13T00:33:21.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:20 smithi136 bash[76371]: cluster 2024-09-13T00:33:20.557449+0000 mon.a (mon.0) 3178 : cluster [DBG] osdmap e1695: 8 total, 8 up, 8 in 2024-09-13T00:33:21.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:20 smithi136 bash[76371]: audit 2024-09-13T00:33:20.615726+0000 mon.c (mon.2) 128 : audit [INF] from='client.? 172.21.15.136:0/1086627647' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi136-114798-24", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-09-13T00:33:21.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:20 smithi136 bash[76371]: audit 2024-09-13T00:33:20.616405+0000 mon.a (mon.0) 3179 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi136-114798-24", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-09-13T00:33:21.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:20 smithi136 bash[75164]: cluster 2024-09-13T00:33:20.109417+0000 mgr.y (mgr.24661) 1686 : cluster [DBG] pgmap v2941: 457 pgs: 6 creating+activating, 10 creating+peering, 31 unknown, 410 active+clean; 4.6 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.0 KiB/s rd, 2.2 KiB/s wr, 16 op/s 2024-09-13T00:33:21.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:20 smithi136 bash[75164]: cluster 2024-09-13T00:33:20.557449+0000 mon.a (mon.0) 3178 : cluster [DBG] osdmap e1695: 8 total, 8 up, 8 in 2024-09-13T00:33:21.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:20 smithi136 bash[75164]: audit 2024-09-13T00:33:20.615726+0000 mon.c (mon.2) 128 : audit [INF] from='client.? 172.21.15.136:0/1086627647' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi136-114798-24", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-09-13T00:33:21.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:20 smithi136 bash[75164]: audit 2024-09-13T00:33:20.616405+0000 mon.a (mon.0) 3179 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi136-114798-24", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-09-13T00:33:21.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:20 smithi181 bash[67587]: cluster 2024-09-13T00:33:20.109417+0000 mgr.y (mgr.24661) 1686 : cluster [DBG] pgmap v2941: 457 pgs: 6 creating+activating, 10 creating+peering, 31 unknown, 410 active+clean; 4.6 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.0 KiB/s rd, 2.2 KiB/s wr, 16 op/s 2024-09-13T00:33:21.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:20 smithi181 bash[67587]: cluster 2024-09-13T00:33:20.557449+0000 mon.a (mon.0) 3178 : cluster [DBG] osdmap e1695: 8 total, 8 up, 8 in 2024-09-13T00:33:21.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:20 smithi181 bash[67587]: audit 2024-09-13T00:33:20.615726+0000 mon.c (mon.2) 128 : audit [INF] from='client.? 172.21.15.136:0/1086627647' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi136-114798-24", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-09-13T00:33:21.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:20 smithi181 bash[67587]: audit 2024-09-13T00:33:20.616405+0000 mon.a (mon.0) 3179 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi136-114798-24", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-09-13T00:33:22.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:21 smithi136 bash[75164]: cluster 2024-09-13T00:33:20.741612+0000 mon.a (mon.0) 3180 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:33:22.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:21 smithi136 bash[75164]: cluster 2024-09-13T00:33:20.741660+0000 mon.a (mon.0) 3181 : cluster [INF] Cluster is now healthy 2024-09-13T00:33:22.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:21 smithi136 bash[75164]: audit 2024-09-13T00:33:21.561264+0000 mon.a (mon.0) 3182 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi136-114798-24", "field": "max_bytes", "val": "10485760"}]': finished 2024-09-13T00:33:22.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:21 smithi136 bash[75164]: cluster 2024-09-13T00:33:21.561490+0000 mon.a (mon.0) 3183 : cluster [DBG] osdmap e1696: 8 total, 8 up, 8 in 2024-09-13T00:33:22.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:21 smithi136 bash[76371]: cluster 2024-09-13T00:33:20.741612+0000 mon.a (mon.0) 3180 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:33:22.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:21 smithi136 bash[76371]: cluster 2024-09-13T00:33:20.741660+0000 mon.a (mon.0) 3181 : cluster [INF] Cluster is now healthy 2024-09-13T00:33:22.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:21 smithi136 bash[76371]: audit 2024-09-13T00:33:21.561264+0000 mon.a (mon.0) 3182 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi136-114798-24", "field": "max_bytes", "val": "10485760"}]': finished 2024-09-13T00:33:22.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:21 smithi136 bash[76371]: cluster 2024-09-13T00:33:21.561490+0000 mon.a (mon.0) 3183 : cluster [DBG] osdmap e1696: 8 total, 8 up, 8 in 2024-09-13T00:33:22.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:21 smithi181 bash[67587]: cluster 2024-09-13T00:33:20.741612+0000 mon.a (mon.0) 3180 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:33:22.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:21 smithi181 bash[67587]: cluster 2024-09-13T00:33:20.741660+0000 mon.a (mon.0) 3181 : cluster [INF] Cluster is now healthy 2024-09-13T00:33:22.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:21 smithi181 bash[67587]: audit 2024-09-13T00:33:21.561264+0000 mon.a (mon.0) 3182 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi136-114798-24", "field": "max_bytes", "val": "10485760"}]': finished 2024-09-13T00:33:22.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:21 smithi181 bash[67587]: cluster 2024-09-13T00:33:21.561490+0000 mon.a (mon.0) 3183 : cluster [DBG] osdmap e1696: 8 total, 8 up, 8 in 2024-09-13T00:33:22.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:33:22 smithi181 bash[95505]: ts=2024-09-13T00:33:22.318Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:33:23.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:22 smithi181 bash[67587]: cluster 2024-09-13T00:33:22.110901+0000 mgr.y (mgr.24661) 1687 : cluster [DBG] pgmap v2944: 457 pgs: 2 creating+peering, 3 unknown, 452 active+clean; 5.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 8.5 KiB/s rd, 258 KiB/s wr, 16 op/s 2024-09-13T00:33:23.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:22 smithi136 bash[75164]: cluster 2024-09-13T00:33:22.110901+0000 mgr.y (mgr.24661) 1687 : cluster [DBG] pgmap v2944: 457 pgs: 2 creating+peering, 3 unknown, 452 active+clean; 5.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 8.5 KiB/s rd, 258 KiB/s wr, 16 op/s 2024-09-13T00:33:23.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:22 smithi136 bash[76371]: cluster 2024-09-13T00:33:22.110901+0000 mgr.y (mgr.24661) 1687 : cluster [DBG] pgmap v2944: 457 pgs: 2 creating+peering, 3 unknown, 452 active+clean; 5.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 8.5 KiB/s rd, 258 KiB/s wr, 16 op/s 2024-09-13T00:33:24.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:23 smithi136 bash[76371]: audit 2024-09-13T00:33:23.827364+0000 mon.a (mon.0) 3184 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:33:24.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:23 smithi136 bash[75164]: audit 2024-09-13T00:33:23.827364+0000 mon.a (mon.0) 3184 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:33:24.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:23 smithi181 bash[67587]: audit 2024-09-13T00:33:23.827364+0000 mon.a (mon.0) 3184 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:33:25.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:24 smithi136 bash[76371]: cluster 2024-09-13T00:33:24.112731+0000 mgr.y (mgr.24661) 1688 : cluster [DBG] pgmap v2945: 457 pgs: 457 active+clean; 5.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 9.6 KiB/s rd, 190 KiB/s wr, 16 op/s 2024-09-13T00:33:25.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:24 smithi136 bash[75164]: cluster 2024-09-13T00:33:24.112731+0000 mgr.y (mgr.24661) 1688 : cluster [DBG] pgmap v2945: 457 pgs: 457 active+clean; 5.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 9.6 KiB/s rd, 190 KiB/s wr, 16 op/s 2024-09-13T00:33:25.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:24 smithi181 bash[67587]: cluster 2024-09-13T00:33:24.112731+0000 mgr.y (mgr.24661) 1688 : cluster [DBG] pgmap v2945: 457 pgs: 457 active+clean; 5.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 9.6 KiB/s rd, 190 KiB/s wr, 16 op/s 2024-09-13T00:33:27.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:27 smithi136 bash[75164]: cluster 2024-09-13T00:33:26.115012+0000 mgr.y (mgr.24661) 1689 : cluster [DBG] pgmap v2946: 457 pgs: 457 active+clean; 5.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 8.1 KiB/s rd, 157 KiB/s wr, 13 op/s 2024-09-13T00:33:27.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:27 smithi136 bash[76371]: cluster 2024-09-13T00:33:26.115012+0000 mgr.y (mgr.24661) 1689 : cluster [DBG] pgmap v2946: 457 pgs: 457 active+clean; 5.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 8.1 KiB/s rd, 157 KiB/s wr, 13 op/s 2024-09-13T00:33:27.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:27 smithi181 bash[67587]: cluster 2024-09-13T00:33:26.115012+0000 mgr.y (mgr.24661) 1689 : cluster [DBG] pgmap v2946: 457 pgs: 457 active+clean; 5.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 8.1 KiB/s rd, 157 KiB/s wr, 13 op/s 2024-09-13T00:33:29.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:29 smithi136 bash[75164]: cluster 2024-09-13T00:33:28.117521+0000 mgr.y (mgr.24661) 1690 : cluster [DBG] pgmap v2947: 457 pgs: 457 active+clean; 8.6 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.6 KiB/s rd, 513 KiB/s wr, 11 op/s 2024-09-13T00:33:29.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:29 smithi136 bash[76371]: cluster 2024-09-13T00:33:28.117521+0000 mgr.y (mgr.24661) 1690 : cluster [DBG] pgmap v2947: 457 pgs: 457 active+clean; 8.6 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.6 KiB/s rd, 513 KiB/s wr, 11 op/s 2024-09-13T00:33:29.510 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:33:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:33:29] "GET /metrics HTTP/1.1" 200 60663 "" "Prometheus/2.43.0" 2024-09-13T00:33:29.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:29 smithi181 bash[67587]: cluster 2024-09-13T00:33:28.117521+0000 mgr.y (mgr.24661) 1690 : cluster [DBG] pgmap v2947: 457 pgs: 457 active+clean; 8.6 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.6 KiB/s rd, 513 KiB/s wr, 11 op/s 2024-09-13T00:33:30.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:30 smithi181 bash[67587]: cluster 2024-09-13T00:33:29.562261+0000 mon.a (mon.0) 3185 : cluster [DBG] osdmap e1697: 8 total, 8 up, 8 in 2024-09-13T00:33:30.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:30 smithi181 bash[67587]: cluster 2024-09-13T00:33:30.118498+0000 mgr.y (mgr.24661) 1691 : cluster [DBG] pgmap v2949: 457 pgs: 457 active+clean; 13 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 3.7 KiB/s rd, 958 KiB/s wr, 7 op/s 2024-09-13T00:33:31.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:30 smithi136 bash[75164]: cluster 2024-09-13T00:33:29.562261+0000 mon.a (mon.0) 3185 : cluster [DBG] osdmap e1697: 8 total, 8 up, 8 in 2024-09-13T00:33:31.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:30 smithi136 bash[75164]: cluster 2024-09-13T00:33:30.118498+0000 mgr.y (mgr.24661) 1691 : cluster [DBG] pgmap v2949: 457 pgs: 457 active+clean; 13 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 3.7 KiB/s rd, 958 KiB/s wr, 7 op/s 2024-09-13T00:33:31.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:30 smithi136 bash[76371]: cluster 2024-09-13T00:33:29.562261+0000 mon.a (mon.0) 3185 : cluster [DBG] osdmap e1697: 8 total, 8 up, 8 in 2024-09-13T00:33:31.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:30 smithi136 bash[76371]: cluster 2024-09-13T00:33:30.118498+0000 mgr.y (mgr.24661) 1691 : cluster [DBG] pgmap v2949: 457 pgs: 457 active+clean; 13 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 3.7 KiB/s rd, 958 KiB/s wr, 7 op/s 2024-09-13T00:33:32.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:33:32 smithi181 bash[95505]: ts=2024-09-13T00:33:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:33:33.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:33 smithi136 bash[75164]: cluster 2024-09-13T00:33:32.121401+0000 mgr.y (mgr.24661) 1692 : cluster [DBG] pgmap v2950: 457 pgs: 457 active+clean; 15 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 4.0 KiB/s rd, 922 KiB/s wr, 6 op/s 2024-09-13T00:33:33.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:33 smithi136 bash[76371]: cluster 2024-09-13T00:33:32.121401+0000 mgr.y (mgr.24661) 1692 : cluster [DBG] pgmap v2950: 457 pgs: 457 active+clean; 15 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 4.0 KiB/s rd, 922 KiB/s wr, 6 op/s 2024-09-13T00:33:33.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:33 smithi181 bash[67587]: cluster 2024-09-13T00:33:32.121401+0000 mgr.y (mgr.24661) 1692 : cluster [DBG] pgmap v2950: 457 pgs: 457 active+clean; 15 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 4.0 KiB/s rd, 922 KiB/s wr, 6 op/s 2024-09-13T00:33:34.933 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.RemoveFullTryDataPool (19812 ms) 2024-09-13T00:33:34.933 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestIO 2024-09-13T00:33:34.933 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:33:35.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:35 smithi136 bash[75164]: cluster 2024-09-13T00:33:34.123293+0000 mgr.y (mgr.24661) 1693 : cluster [DBG] pgmap v2951: 457 pgs: 457 active+clean; 15 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 1.7 KiB/s rd, 921 KiB/s wr, 3 op/s 2024-09-13T00:33:35.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:35 smithi136 bash[75164]: cluster 2024-09-13T00:33:34.550103+0000 mon.a (mon.0) 3186 : cluster [WRN] pool 'test-librbd-smithi136-114798-24' is full (reached quota's max_bytes: 10 MiB) 2024-09-13T00:33:35.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:35 smithi136 bash[75164]: cluster 2024-09-13T00:33:34.550818+0000 mon.a (mon.0) 3187 : cluster [WRN] Health check failed: 1 pool(s) full (POOL_FULL) 2024-09-13T00:33:35.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:35 smithi136 bash[75164]: cluster 2024-09-13T00:33:34.562514+0000 mon.a (mon.0) 3188 : cluster [DBG] osdmap e1698: 8 total, 8 up, 8 in 2024-09-13T00:33:35.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:35 smithi136 bash[76371]: cluster 2024-09-13T00:33:34.123293+0000 mgr.y (mgr.24661) 1693 : cluster [DBG] pgmap v2951: 457 pgs: 457 active+clean; 15 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 1.7 KiB/s rd, 921 KiB/s wr, 3 op/s 2024-09-13T00:33:35.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:35 smithi136 bash[76371]: cluster 2024-09-13T00:33:34.550103+0000 mon.a (mon.0) 3186 : cluster [WRN] pool 'test-librbd-smithi136-114798-24' is full (reached quota's max_bytes: 10 MiB) 2024-09-13T00:33:35.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:35 smithi136 bash[76371]: cluster 2024-09-13T00:33:34.550818+0000 mon.a (mon.0) 3187 : cluster [WRN] Health check failed: 1 pool(s) full (POOL_FULL) 2024-09-13T00:33:35.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:35 smithi136 bash[76371]: cluster 2024-09-13T00:33:34.562514+0000 mon.a (mon.0) 3188 : cluster [DBG] osdmap e1698: 8 total, 8 up, 8 in 2024-09-13T00:33:35.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:35 smithi181 bash[67587]: cluster 2024-09-13T00:33:34.123293+0000 mgr.y (mgr.24661) 1693 : cluster [DBG] pgmap v2951: 457 pgs: 457 active+clean; 15 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 1.7 KiB/s rd, 921 KiB/s wr, 3 op/s 2024-09-13T00:33:35.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:35 smithi181 bash[67587]: cluster 2024-09-13T00:33:34.550103+0000 mon.a (mon.0) 3186 : cluster [WRN] pool 'test-librbd-smithi136-114798-24' is full (reached quota's max_bytes: 10 MiB) 2024-09-13T00:33:35.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:35 smithi181 bash[67587]: cluster 2024-09-13T00:33:34.550818+0000 mon.a (mon.0) 3187 : cluster [WRN] Health check failed: 1 pool(s) full (POOL_FULL) 2024-09-13T00:33:35.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:35 smithi181 bash[67587]: cluster 2024-09-13T00:33:34.562514+0000 mon.a (mon.0) 3188 : cluster [DBG] osdmap e1698: 8 total, 8 up, 8 in 2024-09-13T00:33:37.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:37 smithi136 bash[76371]: cluster 2024-09-13T00:33:36.125427+0000 mgr.y (mgr.24661) 1694 : cluster [DBG] pgmap v2953: 457 pgs: 457 active+clean; 15 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 769 KiB/s wr, 12 op/s 2024-09-13T00:33:37.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:37 smithi136 bash[75164]: cluster 2024-09-13T00:33:36.125427+0000 mgr.y (mgr.24661) 1694 : cluster [DBG] pgmap v2953: 457 pgs: 457 active+clean; 15 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 769 KiB/s wr, 12 op/s 2024-09-13T00:33:37.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:37 smithi181 bash[67587]: cluster 2024-09-13T00:33:36.125427+0000 mgr.y (mgr.24661) 1694 : cluster [DBG] pgmap v2953: 457 pgs: 457 active+clean; 15 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 769 KiB/s wr, 12 op/s 2024-09-13T00:33:39.509 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:33:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:33:39] "GET /metrics HTTP/1.1" 200 60742 "" "Prometheus/2.43.0" 2024-09-13T00:33:39.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:39 smithi136 bash[76371]: cluster 2024-09-13T00:33:38.127885+0000 mgr.y (mgr.24661) 1695 : cluster [DBG] pgmap v2954: 457 pgs: 457 active+clean; 8.6 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 98 KiB/s rd, 641 KiB/s wr, 183 op/s 2024-09-13T00:33:39.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:39 smithi136 bash[76371]: audit 2024-09-13T00:33:38.835537+0000 mon.a (mon.0) 3189 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:33:39.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:39 smithi136 bash[76371]: audit 2024-09-13T00:33:38.837962+0000 mon.a (mon.0) 3190 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:33:39.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:39 smithi136 bash[75164]: cluster 2024-09-13T00:33:38.127885+0000 mgr.y (mgr.24661) 1695 : cluster [DBG] pgmap v2954: 457 pgs: 457 active+clean; 8.6 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 98 KiB/s rd, 641 KiB/s wr, 183 op/s 2024-09-13T00:33:39.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:39 smithi136 bash[75164]: audit 2024-09-13T00:33:38.835537+0000 mon.a (mon.0) 3189 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:33:39.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:39 smithi136 bash[75164]: audit 2024-09-13T00:33:38.837962+0000 mon.a (mon.0) 3190 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:33:39.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:39 smithi181 bash[67587]: cluster 2024-09-13T00:33:38.127885+0000 mgr.y (mgr.24661) 1695 : cluster [DBG] pgmap v2954: 457 pgs: 457 active+clean; 8.6 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 98 KiB/s rd, 641 KiB/s wr, 183 op/s 2024-09-13T00:33:39.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:39 smithi181 bash[67587]: audit 2024-09-13T00:33:38.835537+0000 mon.a (mon.0) 3189 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:33:39.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:39 smithi181 bash[67587]: audit 2024-09-13T00:33:38.837962+0000 mon.a (mon.0) 3190 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:33:40.142 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 512 2024-09-13T00:33:40.142 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 512 2024-09-13T00:33:40.142 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 512 2024-09-13T00:33:40.142 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 512 2024-09-13T00:33:40.143 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 512 2024-09-13T00:33:40.143 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:40.143 INFO:tasks.workunit.client.0.smithi136.stdout:started write 2024-09-13T00:33:40.143 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:33:40.143 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:33:40.143 INFO:tasks.workunit.client.0.smithi136.stdout:finished write 2024-09-13T00:33:40.143 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:40.143 INFO:tasks.workunit.client.0.smithi136.stdout:started write 2024-09-13T00:33:40.143 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:33:40.143 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:33:40.143 INFO:tasks.workunit.client.0.smithi136.stdout:finished write 2024-09-13T00:33:40.143 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:40.144 INFO:tasks.workunit.client.0.smithi136.stdout:started write 2024-09-13T00:33:40.144 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:33:40.144 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:33:40.144 INFO:tasks.workunit.client.0.smithi136.stdout:finished write 2024-09-13T00:33:40.144 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:40.144 INFO:tasks.workunit.client.0.smithi136.stdout:started write 2024-09-13T00:33:40.144 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:33:40.144 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:33:40.144 INFO:tasks.workunit.client.0.smithi136.stdout:finished write 2024-09-13T00:33:40.144 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:40.144 INFO:tasks.workunit.client.0.smithi136.stdout:started write 2024-09-13T00:33:40.144 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:33:40.144 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:33:40.145 INFO:tasks.workunit.client.0.smithi136.stdout:finished write 2024-09-13T00:33:40.145 INFO:tasks.workunit.client.0.smithi136.stdout:start compare and write 2024-09-13T00:33:40.145 INFO:tasks.workunit.client.0.smithi136.stdout:compare and wrote: 512 2024-09-13T00:33:40.145 INFO:tasks.workunit.client.0.smithi136.stdout:start compare and write 2024-09-13T00:33:40.145 INFO:tasks.workunit.client.0.smithi136.stdout:compare and wrote: 512 2024-09-13T00:33:40.145 INFO:tasks.workunit.client.0.smithi136.stdout:start compare and write 2024-09-13T00:33:40.145 INFO:tasks.workunit.client.0.smithi136.stdout:compare and wrote: 512 2024-09-13T00:33:40.145 INFO:tasks.workunit.client.0.smithi136.stdout:start compare and write 2024-09-13T00:33:40.145 INFO:tasks.workunit.client.0.smithi136.stdout:compare and wrote: 512 2024-09-13T00:33:40.145 INFO:tasks.workunit.client.0.smithi136.stdout:start compare and write 2024-09-13T00:33:40.145 INFO:tasks.workunit.client.0.smithi136.stdout:compare and wrote: 512 2024-09-13T00:33:40.145 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:40.146 INFO:tasks.workunit.client.0.smithi136.stdout:started aio compare and write 2024-09-13T00:33:40.146 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:33:40.146 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:33:40.146 INFO:tasks.workunit.client.0.smithi136.stdout:finished aio compare and write 2024-09-13T00:33:40.146 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:40.146 INFO:tasks.workunit.client.0.smithi136.stdout:started aio compare and write 2024-09-13T00:33:40.146 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:33:40.146 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:33:40.146 INFO:tasks.workunit.client.0.smithi136.stdout:finished aio compare and write 2024-09-13T00:33:40.146 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:40.146 INFO:tasks.workunit.client.0.smithi136.stdout:started aio compare and write 2024-09-13T00:33:40.146 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:33:40.146 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:33:40.147 INFO:tasks.workunit.client.0.smithi136.stdout:finished aio compare and write 2024-09-13T00:33:40.147 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:40.147 INFO:tasks.workunit.client.0.smithi136.stdout:started aio compare and write 2024-09-13T00:33:40.147 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:33:40.147 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:33:40.147 INFO:tasks.workunit.client.0.smithi136.stdout:finished aio compare and write 2024-09-13T00:33:40.147 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:40.147 INFO:tasks.workunit.client.0.smithi136.stdout:started aio compare and write 2024-09-13T00:33:40.147 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:33:40.147 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:33:40.147 INFO:tasks.workunit.client.0.smithi136.stdout:finished aio compare and write 2024-09-13T00:33:40.147 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:33:40.147 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:33:40.147 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:33:40.148 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:33:40.148 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:33:40.148 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:40.148 INFO:tasks.workunit.client.0.smithi136.stdout:started read 2024-09-13T00:33:40.148 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:33:40.148 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 512 2024-09-13T00:33:40.148 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:40.148 INFO:tasks.workunit.client.0.smithi136.stdout:started read 2024-09-13T00:33:40.148 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:33:40.148 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 512 2024-09-13T00:33:40.148 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:40.148 INFO:tasks.workunit.client.0.smithi136.stdout:started read 2024-09-13T00:33:40.148 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:33:40.149 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 512 2024-09-13T00:33:40.149 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:40.149 INFO:tasks.workunit.client.0.smithi136.stdout:started read 2024-09-13T00:33:40.149 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:33:40.149 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 512 2024-09-13T00:33:40.149 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:40.149 INFO:tasks.workunit.client.0.smithi136.stdout:started read 2024-09-13T00:33:40.149 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:33:40.149 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 512 2024-09-13T00:33:40.149 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 512~512 = 512 2024-09-13T00:33:40.149 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:33:40.149 INFO:tasks.workunit.client.0.smithi136.stdout:aio discard: 1536~512 = 0 2024-09-13T00:33:40.149 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:33:40.150 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:33:40.150 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:33:40.150 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:33:40.150 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:33:40.150 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 0 2024-09-13T00:33:40.150 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:40.150 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:40.150 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 0 2024-09-13T00:33:40.150 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:40.150 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:40.150 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 16384 2024-09-13T00:33:40.150 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:40.150 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:40.151 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 16384 2024-09-13T00:33:40.151 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:40.151 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:40.151 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:33:40.151 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:33:40.151 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 49152 2024-09-13T00:33:40.151 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:40.151 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:40.151 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 49152 2024-09-13T00:33:40.151 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:40.151 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:40.151 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 65536 2024-09-13T00:33:40.151 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:40.151 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:40.152 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 65536 2024-09-13T00:33:40.152 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:40.152 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:40.152 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:33:40.152 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:33:40.152 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 98304 2024-09-13T00:33:40.152 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:40.152 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:40.152 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 98304 2024-09-13T00:33:40.152 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:40.152 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:40.152 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 114688 2024-09-13T00:33:40.152 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:40.153 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:40.153 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 114688 2024-09-13T00:33:40.153 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:40.153 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:40.153 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:33:40.153 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:33:40.153 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 147456 2024-09-13T00:33:40.153 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:40.153 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:40.153 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 147456 2024-09-13T00:33:40.153 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:40.153 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:40.153 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 163840 2024-09-13T00:33:40.154 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:40.154 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:40.154 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 163840 2024-09-13T00:33:40.154 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:40.154 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:40.154 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:33:40.154 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:33:40.154 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 196608 2024-09-13T00:33:40.154 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:40.154 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:40.154 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 196608 2024-09-13T00:33:40.154 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:40.154 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:40.155 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 212992 2024-09-13T00:33:40.155 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:40.155 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:40.155 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 212992 2024-09-13T00:33:40.155 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:40.155 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:40.155 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:33:40.155 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:33:40.155 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:40.155 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:33:40.155 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:33:40.155 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:33:40.155 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:33:40.156 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:40.156 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:40.156 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:40.156 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:33:40.156 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:33:40.156 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:33:40.156 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:33:40.156 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:40.156 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:40.156 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:40.156 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:33:40.156 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:33:40.156 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:33:40.157 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:33:40.157 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:40.157 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:40.157 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:40.157 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:33:40.157 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:33:40.157 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:33:40.157 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:33:40.157 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:40.157 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:40.157 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:40.157 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:33:40.157 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:33:40.158 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:40.158 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:33:40.158 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:33:40.158 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:40.158 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:33:40.158 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:33:40.158 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:33:40.158 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:33:40.158 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:40.158 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:40.158 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:40.158 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:33:40.158 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:33:40.158 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:33:40.159 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:33:40.159 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:40.159 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:40.159 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:40.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:40 smithi136 bash[75164]: cluster 2024-09-13T00:33:39.552499+0000 mon.a (mon.0) 3191 : cluster [INF] pool 'test-librbd-smithi136-114798-24' no longer out of quota; removing NO_QUOTA flag 2024-09-13T00:33:40.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:40 smithi136 bash[75164]: cluster 2024-09-13T00:33:39.553355+0000 mon.a (mon.0) 3192 : cluster [INF] Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-09-13T00:33:40.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:40 smithi136 bash[75164]: cluster 2024-09-13T00:33:39.553412+0000 mon.a (mon.0) 3193 : cluster [INF] Cluster is now healthy 2024-09-13T00:33:40.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:40 smithi136 bash[75164]: cluster 2024-09-13T00:33:39.569374+0000 mon.a (mon.0) 3194 : cluster [DBG] osdmap e1699: 8 total, 8 up, 8 in 2024-09-13T00:33:40.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:40 smithi136 bash[76371]: cluster 2024-09-13T00:33:39.552499+0000 mon.a (mon.0) 3191 : cluster [INF] pool 'test-librbd-smithi136-114798-24' no longer out of quota; removing NO_QUOTA flag 2024-09-13T00:33:40.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:40 smithi136 bash[76371]: cluster 2024-09-13T00:33:39.553355+0000 mon.a (mon.0) 3192 : cluster [INF] Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-09-13T00:33:40.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:40 smithi136 bash[76371]: cluster 2024-09-13T00:33:39.553412+0000 mon.a (mon.0) 3193 : cluster [INF] Cluster is now healthy 2024-09-13T00:33:40.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:40 smithi136 bash[76371]: cluster 2024-09-13T00:33:39.569374+0000 mon.a (mon.0) 3194 : cluster [DBG] osdmap e1699: 8 total, 8 up, 8 in 2024-09-13T00:33:40.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:40 smithi181 bash[67587]: cluster 2024-09-13T00:33:39.552499+0000 mon.a (mon.0) 3191 : cluster [INF] pool 'test-librbd-smithi136-114798-24' no longer out of quota; removing NO_QUOTA flag 2024-09-13T00:33:40.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:40 smithi181 bash[67587]: cluster 2024-09-13T00:33:39.553355+0000 mon.a (mon.0) 3192 : cluster [INF] Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-09-13T00:33:40.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:40 smithi181 bash[67587]: cluster 2024-09-13T00:33:39.553412+0000 mon.a (mon.0) 3193 : cluster [INF] Cluster is now healthy 2024-09-13T00:33:40.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:40 smithi181 bash[67587]: cluster 2024-09-13T00:33:39.569374+0000 mon.a (mon.0) 3194 : cluster [DBG] osdmap e1699: 8 total, 8 up, 8 in 2024-09-13T00:33:41.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:41 smithi136 bash[75164]: cluster 2024-09-13T00:33:40.129029+0000 mgr.y (mgr.24661) 1696 : cluster [DBG] pgmap v2956: 457 pgs: 457 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 106 KiB/s rd, 430 KiB/s wr, 198 op/s 2024-09-13T00:33:41.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:41 smithi136 bash[76371]: cluster 2024-09-13T00:33:40.129029+0000 mgr.y (mgr.24661) 1696 : cluster [DBG] pgmap v2956: 457 pgs: 457 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 106 KiB/s rd, 430 KiB/s wr, 198 op/s 2024-09-13T00:33:41.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:41 smithi181 bash[67587]: cluster 2024-09-13T00:33:40.129029+0000 mgr.y (mgr.24661) 1696 : cluster [DBG] pgmap v2956: 457 pgs: 457 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 106 KiB/s rd, 430 KiB/s wr, 198 op/s 2024-09-13T00:33:42.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:33:42 smithi181 bash[95505]: ts=2024-09-13T00:33:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:33:43.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:43 smithi136 bash[75164]: cluster 2024-09-13T00:33:42.132337+0000 mgr.y (mgr.24661) 1697 : cluster [DBG] pgmap v2957: 457 pgs: 457 active+clean; 4.7 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 380 KiB/s rd, 578 KiB/s wr, 738 op/s 2024-09-13T00:33:43.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:43 smithi136 bash[76371]: cluster 2024-09-13T00:33:42.132337+0000 mgr.y (mgr.24661) 1697 : cluster [DBG] pgmap v2957: 457 pgs: 457 active+clean; 4.7 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 380 KiB/s rd, 578 KiB/s wr, 738 op/s 2024-09-13T00:33:43.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:43 smithi181 bash[67587]: cluster 2024-09-13T00:33:42.132337+0000 mgr.y (mgr.24661) 1697 : cluster [DBG] pgmap v2957: 457 pgs: 457 active+clean; 4.7 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 380 KiB/s rd, 578 KiB/s wr, 738 op/s 2024-09-13T00:33:44.780 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:33:44.780 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:33:44.780 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:33:44.780 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:33:44.781 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:44.781 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:44.781 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:44.781 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:33:44.781 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:33:44.781 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:33:44.781 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:33:44.781 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:44.781 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:44.781 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:44.781 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:33:44.781 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:33:44.781 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:44.782 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:33:44.782 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:33:44.782 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:44.782 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:33:44.782 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:33:44.782 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:33:44.782 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:33:44.782 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:44.782 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:44.782 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:44.782 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:33:44.782 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:33:44.782 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:33:44.783 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:33:44.783 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:44.783 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:44.783 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:44.783 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:33:44.783 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:33:44.783 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:33:44.783 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:33:44.783 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:44.783 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:44.783 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:44.783 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:33:44.783 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:33:44.783 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:33:44.784 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:33:44.784 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:44.784 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:44.784 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:44.784 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:33:44.784 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:33:44.784 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:44.784 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:33:44.784 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:33:44.784 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:44.784 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:33:44.784 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:33:44.785 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:33:44.785 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:33:44.785 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:44.785 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:44.785 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:44.785 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:33:44.785 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:33:44.785 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:33:44.785 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:33:44.785 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:44.785 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:44.785 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:44.785 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:33:44.786 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:33:44.786 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:33:44.786 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:33:44.786 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:44.786 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:44.786 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:44.786 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:33:44.786 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:33:44.786 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:33:44.786 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:33:44.786 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:44.786 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:44.786 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:44.786 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:33:44.787 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:33:44.787 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:44.787 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:33:44.787 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:33:44.787 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:44.787 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:33:44.787 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:33:44.787 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:33:44.787 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:33:44.787 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:44.787 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:44.787 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:44.787 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:33:44.787 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:33:44.788 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:33:44.788 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:33:44.788 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:44.788 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:44.788 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:44.788 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:33:44.788 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:33:44.788 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:33:44.788 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:33:44.788 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:44.788 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:44.788 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:44.788 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:33:44.789 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:33:44.789 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:33:44.789 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:33:44.789 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:44.789 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:44.789 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:44.789 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:33:44.789 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:33:44.789 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:44.789 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:33:44.789 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:33:44.789 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:33:44.789 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:33:44.790 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 512 2024-09-13T00:33:44.790 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:33:44.790 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestIO (9847 ms) 2024-09-13T00:33:44.790 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestEncryptionLUKS1 2024-09-13T00:33:44.790 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:33:45.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:45 smithi136 bash[75164]: cluster 2024-09-13T00:33:44.134269+0000 mgr.y (mgr.24661) 1698 : cluster [DBG] pgmap v2958: 457 pgs: 457 active+clean; 4.7 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 318 KiB/s rd, 483 KiB/s wr, 617 op/s 2024-09-13T00:33:45.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:45 smithi136 bash[76371]: cluster 2024-09-13T00:33:44.134269+0000 mgr.y (mgr.24661) 1698 : cluster [DBG] pgmap v2958: 457 pgs: 457 active+clean; 4.7 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 318 KiB/s rd, 483 KiB/s wr, 617 op/s 2024-09-13T00:33:45.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:45 smithi181 bash[67587]: cluster 2024-09-13T00:33:44.134269+0000 mgr.y (mgr.24661) 1698 : cluster [DBG] pgmap v2958: 457 pgs: 457 active+clean; 4.7 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 318 KiB/s rd, 483 KiB/s wr, 617 op/s 2024-09-13T00:33:47.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:47 smithi136 bash[75164]: cluster 2024-09-13T00:33:46.135467+0000 mgr.y (mgr.24661) 1699 : cluster [DBG] pgmap v2959: 457 pgs: 457 active+clean; 4.7 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 300 KiB/s rd, 461 KiB/s wr, 583 op/s 2024-09-13T00:33:47.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:47 smithi136 bash[76371]: cluster 2024-09-13T00:33:46.135467+0000 mgr.y (mgr.24661) 1699 : cluster [DBG] pgmap v2959: 457 pgs: 457 active+clean; 4.7 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 300 KiB/s rd, 461 KiB/s wr, 583 op/s 2024-09-13T00:33:47.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:47 smithi181 bash[67587]: cluster 2024-09-13T00:33:46.135467+0000 mgr.y (mgr.24661) 1699 : cluster [DBG] pgmap v2959: 457 pgs: 457 active+clean; 4.7 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 300 KiB/s rd, 461 KiB/s wr, 583 op/s 2024-09-13T00:33:49.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:49 smithi136 bash[75164]: cluster 2024-09-13T00:33:48.137813+0000 mgr.y (mgr.24661) 1700 : cluster [DBG] pgmap v2960: 457 pgs: 457 active+clean; 6.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 351 KiB/s rd, 175 KiB/s wr, 707 op/s 2024-09-13T00:33:49.509 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:33:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:33:49] "GET /metrics HTTP/1.1" 200 60742 "" "Prometheus/2.43.0" 2024-09-13T00:33:49.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:49 smithi136 bash[76371]: cluster 2024-09-13T00:33:48.137813+0000 mgr.y (mgr.24661) 1700 : cluster [DBG] pgmap v2960: 457 pgs: 457 active+clean; 6.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 351 KiB/s rd, 175 KiB/s wr, 707 op/s 2024-09-13T00:33:49.600 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:49 smithi181 bash[67587]: cluster 2024-09-13T00:33:48.137813+0000 mgr.y (mgr.24661) 1700 : cluster [DBG] pgmap v2960: 457 pgs: 457 active+clean; 6.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 351 KiB/s rd, 175 KiB/s wr, 707 op/s 2024-09-13T00:33:51.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:51 smithi136 bash[75164]: cluster 2024-09-13T00:33:50.139731+0000 mgr.y (mgr.24661) 1701 : cluster [DBG] pgmap v2961: 457 pgs: 457 active+clean; 6.7 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 385 KiB/s rd, 201 KiB/s wr, 724 op/s 2024-09-13T00:33:51.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:51 smithi136 bash[76371]: cluster 2024-09-13T00:33:50.139731+0000 mgr.y (mgr.24661) 1701 : cluster [DBG] pgmap v2961: 457 pgs: 457 active+clean; 6.7 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 385 KiB/s rd, 201 KiB/s wr, 724 op/s 2024-09-13T00:33:51.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:51 smithi181 bash[67587]: cluster 2024-09-13T00:33:50.139731+0000 mgr.y (mgr.24661) 1701 : cluster [DBG] pgmap v2961: 457 pgs: 457 active+clean; 6.7 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 385 KiB/s rd, 201 KiB/s wr, 724 op/s 2024-09-13T00:33:52.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:33:52 smithi181 bash[95505]: ts=2024-09-13T00:33:52.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:33:53.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:53 smithi136 bash[75164]: cluster 2024-09-13T00:33:52.142693+0000 mgr.y (mgr.24661) 1702 : cluster [DBG] pgmap v2962: 457 pgs: 457 active+clean; 11 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 340 KiB/s rd, 518 KiB/s wr, 640 op/s 2024-09-13T00:33:53.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:53 smithi136 bash[76371]: cluster 2024-09-13T00:33:52.142693+0000 mgr.y (mgr.24661) 1702 : cluster [DBG] pgmap v2962: 457 pgs: 457 active+clean; 11 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 340 KiB/s rd, 518 KiB/s wr, 640 op/s 2024-09-13T00:33:53.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:53 smithi181 bash[67587]: cluster 2024-09-13T00:33:52.142693+0000 mgr.y (mgr.24661) 1702 : cluster [DBG] pgmap v2962: 457 pgs: 457 active+clean; 11 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 340 KiB/s rd, 518 KiB/s wr, 640 op/s 2024-09-13T00:33:55.037 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 512 2024-09-13T00:33:55.037 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 512 2024-09-13T00:33:55.037 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 512 2024-09-13T00:33:55.037 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 512 2024-09-13T00:33:55.037 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 512 2024-09-13T00:33:55.037 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:55.037 INFO:tasks.workunit.client.0.smithi136.stdout:started write 2024-09-13T00:33:55.037 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:33:55.037 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:33:55.037 INFO:tasks.workunit.client.0.smithi136.stdout:finished write 2024-09-13T00:33:55.037 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:55.038 INFO:tasks.workunit.client.0.smithi136.stdout:started write 2024-09-13T00:33:55.038 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:33:55.038 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:33:55.038 INFO:tasks.workunit.client.0.smithi136.stdout:finished write 2024-09-13T00:33:55.038 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:55.038 INFO:tasks.workunit.client.0.smithi136.stdout:started write 2024-09-13T00:33:55.038 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:33:55.038 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:33:55.038 INFO:tasks.workunit.client.0.smithi136.stdout:finished write 2024-09-13T00:33:55.038 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:55.038 INFO:tasks.workunit.client.0.smithi136.stdout:started write 2024-09-13T00:33:55.038 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:33:55.038 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:33:55.038 INFO:tasks.workunit.client.0.smithi136.stdout:finished write 2024-09-13T00:33:55.039 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:55.039 INFO:tasks.workunit.client.0.smithi136.stdout:started write 2024-09-13T00:33:55.039 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:33:55.039 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:33:55.039 INFO:tasks.workunit.client.0.smithi136.stdout:finished write 2024-09-13T00:33:55.039 INFO:tasks.workunit.client.0.smithi136.stdout:start compare and write 2024-09-13T00:33:55.039 INFO:tasks.workunit.client.0.smithi136.stdout:compare and wrote: 512 2024-09-13T00:33:55.039 INFO:tasks.workunit.client.0.smithi136.stdout:start compare and write 2024-09-13T00:33:55.039 INFO:tasks.workunit.client.0.smithi136.stdout:compare and wrote: 512 2024-09-13T00:33:55.039 INFO:tasks.workunit.client.0.smithi136.stdout:start compare and write 2024-09-13T00:33:55.039 INFO:tasks.workunit.client.0.smithi136.stdout:compare and wrote: 512 2024-09-13T00:33:55.039 INFO:tasks.workunit.client.0.smithi136.stdout:start compare and write 2024-09-13T00:33:55.039 INFO:tasks.workunit.client.0.smithi136.stdout:compare and wrote: 512 2024-09-13T00:33:55.039 INFO:tasks.workunit.client.0.smithi136.stdout:start compare and write 2024-09-13T00:33:55.039 INFO:tasks.workunit.client.0.smithi136.stdout:compare and wrote: 512 2024-09-13T00:33:55.040 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:55.040 INFO:tasks.workunit.client.0.smithi136.stdout:started aio compare and write 2024-09-13T00:33:55.040 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:33:55.040 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:33:55.040 INFO:tasks.workunit.client.0.smithi136.stdout:finished aio compare and write 2024-09-13T00:33:55.040 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:55.040 INFO:tasks.workunit.client.0.smithi136.stdout:started aio compare and write 2024-09-13T00:33:55.040 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:33:55.040 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:33:55.040 INFO:tasks.workunit.client.0.smithi136.stdout:finished aio compare and write 2024-09-13T00:33:55.040 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:55.040 INFO:tasks.workunit.client.0.smithi136.stdout:started aio compare and write 2024-09-13T00:33:55.040 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:33:55.041 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:33:55.041 INFO:tasks.workunit.client.0.smithi136.stdout:finished aio compare and write 2024-09-13T00:33:55.041 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:55.041 INFO:tasks.workunit.client.0.smithi136.stdout:started aio compare and write 2024-09-13T00:33:55.041 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:33:55.041 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:33:55.041 INFO:tasks.workunit.client.0.smithi136.stdout:finished aio compare and write 2024-09-13T00:33:55.041 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:55.041 INFO:tasks.workunit.client.0.smithi136.stdout:started aio compare and write 2024-09-13T00:33:55.041 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:33:55.041 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:33:55.041 INFO:tasks.workunit.client.0.smithi136.stdout:finished aio compare and write 2024-09-13T00:33:55.041 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:33:55.041 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:33:55.041 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:33:55.042 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:33:55.042 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:33:55.042 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:55.042 INFO:tasks.workunit.client.0.smithi136.stdout:started read 2024-09-13T00:33:55.042 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:33:55.042 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 512 2024-09-13T00:33:55.042 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:55.042 INFO:tasks.workunit.client.0.smithi136.stdout:started read 2024-09-13T00:33:55.042 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:33:55.042 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 512 2024-09-13T00:33:55.042 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:55.042 INFO:tasks.workunit.client.0.smithi136.stdout:started read 2024-09-13T00:33:55.042 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:33:55.042 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 512 2024-09-13T00:33:55.043 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:55.043 INFO:tasks.workunit.client.0.smithi136.stdout:started read 2024-09-13T00:33:55.043 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:33:55.043 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 512 2024-09-13T00:33:55.043 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:55.043 INFO:tasks.workunit.client.0.smithi136.stdout:started read 2024-09-13T00:33:55.043 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:33:55.043 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 512 2024-09-13T00:33:55.043 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 512~512 = 512 2024-09-13T00:33:55.043 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:33:55.043 INFO:tasks.workunit.client.0.smithi136.stdout:aio discard: 1536~512 = 0 2024-09-13T00:33:55.043 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:33:55.043 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:33:55.043 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:33:55.044 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:33:55.044 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:33:55.044 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 0 2024-09-13T00:33:55.044 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:55.044 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:55.044 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 0 2024-09-13T00:33:55.044 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:55.044 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:55.044 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 16384 2024-09-13T00:33:55.044 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:55.044 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:55.044 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 16384 2024-09-13T00:33:55.044 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:55.044 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:55.044 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:33:55.045 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:33:55.045 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 49152 2024-09-13T00:33:55.045 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:55.045 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:55.045 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 49152 2024-09-13T00:33:55.045 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:55.045 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:55.045 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 65536 2024-09-13T00:33:55.045 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:55.045 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:55.045 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 65536 2024-09-13T00:33:55.045 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:55.045 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:55.045 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:33:55.046 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:33:55.046 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 98304 2024-09-13T00:33:55.046 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:55.046 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:55.046 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 98304 2024-09-13T00:33:55.046 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:55.046 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:55.046 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 114688 2024-09-13T00:33:55.046 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:55.046 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:55.046 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 114688 2024-09-13T00:33:55.046 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:55.046 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:55.046 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:33:55.047 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:33:55.047 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 147456 2024-09-13T00:33:55.047 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:55.047 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:55.047 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 147456 2024-09-13T00:33:55.047 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:55.047 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:55.047 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 163840 2024-09-13T00:33:55.047 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:55.047 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:55.047 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 163840 2024-09-13T00:33:55.047 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:55.047 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:55.047 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:33:55.047 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:33:55.048 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 196608 2024-09-13T00:33:55.048 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:55.048 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:55.048 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 196608 2024-09-13T00:33:55.048 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:55.048 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:55.048 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 212992 2024-09-13T00:33:55.048 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:55.049 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:55.049 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 212992 2024-09-13T00:33:55.049 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:55.049 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:55.049 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:33:55.049 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:33:55.049 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:55.049 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:33:55.049 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:33:55.049 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:33:55.049 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:33:55.050 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:55.050 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:55.050 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:55.050 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:33:55.050 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:33:55.050 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:33:55.050 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:33:55.050 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:55.050 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:55.050 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:55.050 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:33:55.050 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:33:55.050 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:33:55.050 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:33:55.050 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:55.051 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:55.051 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:55.051 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:33:55.051 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:33:55.051 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:33:55.051 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:33:55.051 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:55.051 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:55.051 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:55.051 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:33:55.051 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:33:55.051 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:55.051 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:33:55.051 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:33:55.052 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:55.052 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:33:55.052 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:33:55.052 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:33:55.052 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:33:55.052 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:55.052 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:55.052 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:55.052 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:33:55.052 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:33:55.052 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:33:55.052 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:33:55.052 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:33:55.053 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:33:55.053 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:33:55.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:54 smithi181 bash[67587]: audit 2024-09-13T00:33:53.836019+0000 mon.a (mon.0) 3195 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:33:55.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:54 smithi181 bash[67587]: audit 2024-09-13T00:33:53.838358+0000 mon.a (mon.0) 3196 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:33:55.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:54 smithi181 bash[67587]: cluster 2024-09-13T00:33:54.144546+0000 mgr.y (mgr.24661) 1703 : cluster [DBG] pgmap v2963: 457 pgs: 457 active+clean; 11 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 157 KiB/s rd, 420 KiB/s wr, 279 op/s 2024-09-13T00:33:55.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:54 smithi136 bash[75164]: audit 2024-09-13T00:33:53.836019+0000 mon.a (mon.0) 3195 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:33:55.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:54 smithi136 bash[75164]: audit 2024-09-13T00:33:53.838358+0000 mon.a (mon.0) 3196 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:33:55.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:54 smithi136 bash[75164]: cluster 2024-09-13T00:33:54.144546+0000 mgr.y (mgr.24661) 1703 : cluster [DBG] pgmap v2963: 457 pgs: 457 active+clean; 11 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 157 KiB/s rd, 420 KiB/s wr, 279 op/s 2024-09-13T00:33:55.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:54 smithi136 bash[76371]: audit 2024-09-13T00:33:53.836019+0000 mon.a (mon.0) 3195 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:33:55.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:54 smithi136 bash[76371]: audit 2024-09-13T00:33:53.838358+0000 mon.a (mon.0) 3196 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:33:55.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:54 smithi136 bash[76371]: cluster 2024-09-13T00:33:54.144546+0000 mgr.y (mgr.24661) 1703 : cluster [DBG] pgmap v2963: 457 pgs: 457 active+clean; 11 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 157 KiB/s rd, 420 KiB/s wr, 279 op/s 2024-09-13T00:33:57.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:57 smithi136 bash[75164]: cluster 2024-09-13T00:33:56.146894+0000 mgr.y (mgr.24661) 1704 : cluster [DBG] pgmap v2964: 457 pgs: 457 active+clean; 11 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 452 KiB/s rd, 564 KiB/s wr, 575 op/s 2024-09-13T00:33:57.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:57 smithi136 bash[76371]: cluster 2024-09-13T00:33:56.146894+0000 mgr.y (mgr.24661) 1704 : cluster [DBG] pgmap v2964: 457 pgs: 457 active+clean; 11 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 452 KiB/s rd, 564 KiB/s wr, 575 op/s 2024-09-13T00:33:57.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:57 smithi181 bash[67587]: cluster 2024-09-13T00:33:56.146894+0000 mgr.y (mgr.24661) 1704 : cluster [DBG] pgmap v2964: 457 pgs: 457 active+clean; 11 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 452 KiB/s rd, 564 KiB/s wr, 575 op/s 2024-09-13T00:33:59.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:33:59 smithi136 bash[75164]: cluster 2024-09-13T00:33:58.149573+0000 mgr.y (mgr.24661) 1705 : cluster [DBG] pgmap v2965: 457 pgs: 457 active+clean; 11 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 452 KiB/s rd, 564 KiB/s wr, 575 op/s 2024-09-13T00:33:59.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:33:59 smithi136 bash[76371]: cluster 2024-09-13T00:33:58.149573+0000 mgr.y (mgr.24661) 1705 : cluster [DBG] pgmap v2965: 457 pgs: 457 active+clean; 11 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 452 KiB/s rd, 564 KiB/s wr, 575 op/s 2024-09-13T00:33:59.510 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:33:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:33:59] "GET /metrics HTTP/1.1" 200 60711 "" "Prometheus/2.43.0" 2024-09-13T00:33:59.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:33:59 smithi181 bash[67587]: cluster 2024-09-13T00:33:58.149573+0000 mgr.y (mgr.24661) 1705 : cluster [DBG] pgmap v2965: 457 pgs: 457 active+clean; 11 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 452 KiB/s rd, 564 KiB/s wr, 575 op/s 2024-09-13T00:34:01.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:01 smithi136 bash[75164]: cluster 2024-09-13T00:34:00.151567+0000 mgr.y (mgr.24661) 1706 : cluster [DBG] pgmap v2966: 457 pgs: 457 active+clean; 11 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 638 KiB/s rd, 674 KiB/s wr, 644 op/s 2024-09-13T00:34:01.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:01 smithi136 bash[76371]: cluster 2024-09-13T00:34:00.151567+0000 mgr.y (mgr.24661) 1706 : cluster [DBG] pgmap v2966: 457 pgs: 457 active+clean; 11 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 638 KiB/s rd, 674 KiB/s wr, 644 op/s 2024-09-13T00:34:01.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:01 smithi181 bash[67587]: cluster 2024-09-13T00:34:00.151567+0000 mgr.y (mgr.24661) 1706 : cluster [DBG] pgmap v2966: 457 pgs: 457 active+clean; 11 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 638 KiB/s rd, 674 KiB/s wr, 644 op/s 2024-09-13T00:34:02.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:34:02 smithi181 bash[95505]: ts=2024-09-13T00:34:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:34:02.764 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:02.764 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:02.764 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:02.764 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:02.764 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:02.764 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:02.764 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:02.764 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:02.764 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:02.764 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:02.764 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:02.765 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:02.765 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:02.765 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:02.765 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:02.765 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:02.765 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:02.765 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:02.765 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:02.765 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:02.765 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:02.765 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:02.765 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:02.765 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:02.766 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:02.766 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:02.766 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:02.766 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:02.766 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:02.766 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:02.766 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:02.766 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:02.766 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:02.766 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:02.766 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:02.766 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:02.766 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:02.766 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:02.767 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:02.767 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:02.767 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:02.767 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:02.767 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:02.767 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:02.767 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:02.767 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:02.767 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:02.767 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:02.767 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:02.767 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:02.767 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:02.767 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:02.767 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:02.768 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:02.768 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:02.768 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:02.768 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:02.768 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:02.768 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:02.768 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:02.768 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:02.768 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:02.768 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:02.768 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:02.768 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:02.768 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:02.768 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:02.769 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:02.769 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:02.769 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:02.769 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:02.769 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:02.769 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:02.769 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:02.769 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:02.769 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:02.769 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:02.769 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:02.769 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:02.769 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:02.769 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:02.770 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:02.770 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:02.770 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:02.770 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:02.770 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:02.770 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:02.770 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:02.770 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:02.770 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:02.770 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:02.770 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:02.770 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:02.770 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:02.770 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:02.771 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:02.771 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:02.771 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:02.771 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:02.771 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:02.771 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:02.771 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:02.771 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:02.771 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:02.771 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:02.771 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:02.771 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:02.772 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:02.772 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:02.772 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:02.772 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:02.772 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:02.772 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:02.772 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:02.772 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:02.772 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:02.772 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:02.772 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:02.773 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:02.773 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:02.773 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:02.773 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:34:02.773 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:34:02.773 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 512 2024-09-13T00:34:02.773 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:34:02.773 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 4 2024-09-13T00:34:02.773 INFO:tasks.workunit.client.0.smithi136.stdout:read: 4 2024-09-13T00:34:02.773 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestEncryptionLUKS1 (17984 ms) 2024-09-13T00:34:02.773 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestEncryptionLUKS2 2024-09-13T00:34:02.773 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:34:03.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:03 smithi136 bash[75164]: cluster 2024-09-13T00:34:02.154278+0000 mgr.y (mgr.24661) 1707 : cluster [DBG] pgmap v2967: 457 pgs: 457 active+clean; 15 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 614 KiB/s rd, 642 KiB/s wr, 594 op/s 2024-09-13T00:34:03.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:03 smithi136 bash[76371]: cluster 2024-09-13T00:34:02.154278+0000 mgr.y (mgr.24661) 1707 : cluster [DBG] pgmap v2967: 457 pgs: 457 active+clean; 15 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 614 KiB/s rd, 642 KiB/s wr, 594 op/s 2024-09-13T00:34:03.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:03 smithi181 bash[67587]: cluster 2024-09-13T00:34:02.154278+0000 mgr.y (mgr.24661) 1707 : cluster [DBG] pgmap v2967: 457 pgs: 457 active+clean; 15 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 614 KiB/s rd, 642 KiB/s wr, 594 op/s 2024-09-13T00:34:05.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:05 smithi136 bash[75164]: cluster 2024-09-13T00:34:04.155703+0000 mgr.y (mgr.24661) 1708 : cluster [DBG] pgmap v2968: 457 pgs: 457 active+clean; 15 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 615 KiB/s rd, 301 KiB/s wr, 594 op/s 2024-09-13T00:34:05.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:05 smithi136 bash[76371]: cluster 2024-09-13T00:34:04.155703+0000 mgr.y (mgr.24661) 1708 : cluster [DBG] pgmap v2968: 457 pgs: 457 active+clean; 15 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 615 KiB/s rd, 301 KiB/s wr, 594 op/s 2024-09-13T00:34:05.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:05 smithi181 bash[67587]: cluster 2024-09-13T00:34:04.155703+0000 mgr.y (mgr.24661) 1708 : cluster [DBG] pgmap v2968: 457 pgs: 457 active+clean; 15 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 615 KiB/s rd, 301 KiB/s wr, 594 op/s 2024-09-13T00:34:07.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:07 smithi136 bash[75164]: cluster 2024-09-13T00:34:06.157159+0000 mgr.y (mgr.24661) 1709 : cluster [DBG] pgmap v2969: 457 pgs: 457 active+clean; 15 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 678 KiB/s rd, 319 KiB/s wr, 658 op/s 2024-09-13T00:34:07.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:07 smithi136 bash[76371]: cluster 2024-09-13T00:34:06.157159+0000 mgr.y (mgr.24661) 1709 : cluster [DBG] pgmap v2969: 457 pgs: 457 active+clean; 15 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 678 KiB/s rd, 319 KiB/s wr, 658 op/s 2024-09-13T00:34:07.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:07 smithi181 bash[67587]: cluster 2024-09-13T00:34:06.157159+0000 mgr.y (mgr.24661) 1709 : cluster [DBG] pgmap v2969: 457 pgs: 457 active+clean; 15 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 678 KiB/s rd, 319 KiB/s wr, 658 op/s 2024-09-13T00:34:09.509 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:34:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:34:09] "GET /metrics HTTP/1.1" 200 60714 "" "Prometheus/2.43.0" 2024-09-13T00:34:09.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:09 smithi136 bash[76371]: cluster 2024-09-13T00:34:08.158658+0000 mgr.y (mgr.24661) 1710 : cluster [DBG] pgmap v2970: 457 pgs: 457 active+clean; 15 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 386 KiB/s rd, 175 KiB/s wr, 365 op/s 2024-09-13T00:34:09.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:09 smithi136 bash[76371]: audit 2024-09-13T00:34:08.828064+0000 mon.a (mon.0) 3197 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:34:09.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:09 smithi136 bash[75164]: cluster 2024-09-13T00:34:08.158658+0000 mgr.y (mgr.24661) 1710 : cluster [DBG] pgmap v2970: 457 pgs: 457 active+clean; 15 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 386 KiB/s rd, 175 KiB/s wr, 365 op/s 2024-09-13T00:34:09.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:09 smithi136 bash[75164]: audit 2024-09-13T00:34:08.828064+0000 mon.a (mon.0) 3197 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:34:09.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:09 smithi181 bash[67587]: cluster 2024-09-13T00:34:08.158658+0000 mgr.y (mgr.24661) 1710 : cluster [DBG] pgmap v2970: 457 pgs: 457 active+clean; 15 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 386 KiB/s rd, 175 KiB/s wr, 365 op/s 2024-09-13T00:34:09.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:09 smithi181 bash[67587]: audit 2024-09-13T00:34:08.828064+0000 mon.a (mon.0) 3197 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:34:10.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:10 smithi136 bash[75164]: audit 2024-09-13T00:34:10.212826+0000 mon.a (mon.0) 3198 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "95.15", "id": [1, 4]}]: dispatch 2024-09-13T00:34:10.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:10 smithi136 bash[75164]: audit 2024-09-13T00:34:10.213505+0000 mon.a (mon.0) 3199 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "97.d", "id": [0, 7]}]: dispatch 2024-09-13T00:34:10.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:10 smithi136 bash[75164]: audit 2024-09-13T00:34:10.213983+0000 mon.a (mon.0) 3200 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "99.19", "id": [4, 3]}]: dispatch 2024-09-13T00:34:10.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:10 smithi136 bash[76371]: audit 2024-09-13T00:34:10.212826+0000 mon.a (mon.0) 3198 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "95.15", "id": [1, 4]}]: dispatch 2024-09-13T00:34:10.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:10 smithi136 bash[76371]: audit 2024-09-13T00:34:10.213505+0000 mon.a (mon.0) 3199 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "97.d", "id": [0, 7]}]: dispatch 2024-09-13T00:34:10.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:10 smithi136 bash[76371]: audit 2024-09-13T00:34:10.213983+0000 mon.a (mon.0) 3200 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "99.19", "id": [4, 3]}]: dispatch 2024-09-13T00:34:10.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:10 smithi181 bash[67587]: audit 2024-09-13T00:34:10.212826+0000 mon.a (mon.0) 3198 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "95.15", "id": [1, 4]}]: dispatch 2024-09-13T00:34:10.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:10 smithi181 bash[67587]: audit 2024-09-13T00:34:10.213505+0000 mon.a (mon.0) 3199 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "97.d", "id": [0, 7]}]: dispatch 2024-09-13T00:34:10.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:10 smithi181 bash[67587]: audit 2024-09-13T00:34:10.213983+0000 mon.a (mon.0) 3200 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "99.19", "id": [4, 3]}]: dispatch 2024-09-13T00:34:11.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:11 smithi181 bash[67587]: cluster 2024-09-13T00:34:10.160828+0000 mgr.y (mgr.24661) 1711 : cluster [DBG] pgmap v2971: 457 pgs: 457 active+clean; 15 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 386 KiB/s rd, 175 KiB/s wr, 365 op/s 2024-09-13T00:34:11.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:11 smithi181 bash[67587]: audit 2024-09-13T00:34:10.247973+0000 mon.a (mon.0) 3201 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "95.15", "id": [1, 4]}]': finished 2024-09-13T00:34:11.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:11 smithi181 bash[67587]: audit 2024-09-13T00:34:10.248153+0000 mon.a (mon.0) 3202 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "97.d", "id": [0, 7]}]': finished 2024-09-13T00:34:11.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:11 smithi181 bash[67587]: audit 2024-09-13T00:34:10.248302+0000 mon.a (mon.0) 3203 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "99.19", "id": [4, 3]}]': finished 2024-09-13T00:34:11.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:11 smithi181 bash[67587]: cluster 2024-09-13T00:34:10.248427+0000 mon.a (mon.0) 3204 : cluster [DBG] osdmap e1700: 8 total, 8 up, 8 in 2024-09-13T00:34:11.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:11 smithi136 bash[75164]: cluster 2024-09-13T00:34:10.160828+0000 mgr.y (mgr.24661) 1711 : cluster [DBG] pgmap v2971: 457 pgs: 457 active+clean; 15 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 386 KiB/s rd, 175 KiB/s wr, 365 op/s 2024-09-13T00:34:11.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:11 smithi136 bash[75164]: audit 2024-09-13T00:34:10.247973+0000 mon.a (mon.0) 3201 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "95.15", "id": [1, 4]}]': finished 2024-09-13T00:34:11.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:11 smithi136 bash[75164]: audit 2024-09-13T00:34:10.248153+0000 mon.a (mon.0) 3202 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "97.d", "id": [0, 7]}]': finished 2024-09-13T00:34:11.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:11 smithi136 bash[75164]: audit 2024-09-13T00:34:10.248302+0000 mon.a (mon.0) 3203 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "99.19", "id": [4, 3]}]': finished 2024-09-13T00:34:11.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:11 smithi136 bash[75164]: cluster 2024-09-13T00:34:10.248427+0000 mon.a (mon.0) 3204 : cluster [DBG] osdmap e1700: 8 total, 8 up, 8 in 2024-09-13T00:34:11.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:11 smithi136 bash[76371]: cluster 2024-09-13T00:34:10.160828+0000 mgr.y (mgr.24661) 1711 : cluster [DBG] pgmap v2971: 457 pgs: 457 active+clean; 15 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 386 KiB/s rd, 175 KiB/s wr, 365 op/s 2024-09-13T00:34:11.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:11 smithi136 bash[76371]: audit 2024-09-13T00:34:10.247973+0000 mon.a (mon.0) 3201 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "95.15", "id": [1, 4]}]': finished 2024-09-13T00:34:11.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:11 smithi136 bash[76371]: audit 2024-09-13T00:34:10.248153+0000 mon.a (mon.0) 3202 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "97.d", "id": [0, 7]}]': finished 2024-09-13T00:34:11.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:11 smithi136 bash[76371]: audit 2024-09-13T00:34:10.248302+0000 mon.a (mon.0) 3203 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "99.19", "id": [4, 3]}]': finished 2024-09-13T00:34:11.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:11 smithi136 bash[76371]: cluster 2024-09-13T00:34:10.248427+0000 mon.a (mon.0) 3204 : cluster [DBG] osdmap e1700: 8 total, 8 up, 8 in 2024-09-13T00:34:12.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:12 smithi181 bash[67587]: cluster 2024-09-13T00:34:11.249841+0000 mon.a (mon.0) 3205 : cluster [DBG] osdmap e1701: 8 total, 8 up, 8 in 2024-09-13T00:34:12.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:34:12 smithi181 bash[95505]: ts=2024-09-13T00:34:12.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:34:12.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:12 smithi136 bash[75164]: cluster 2024-09-13T00:34:11.249841+0000 mon.a (mon.0) 3205 : cluster [DBG] osdmap e1701: 8 total, 8 up, 8 in 2024-09-13T00:34:12.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:12 smithi136 bash[76371]: cluster 2024-09-13T00:34:11.249841+0000 mon.a (mon.0) 3205 : cluster [DBG] osdmap e1701: 8 total, 8 up, 8 in 2024-09-13T00:34:13.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:13 smithi181 bash[67587]: cluster 2024-09-13T00:34:12.162621+0000 mgr.y (mgr.24661) 1712 : cluster [DBG] pgmap v2974: 457 pgs: 2 peering, 455 active+clean; 27 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 101 KiB/s rd, 1.5 MiB/s wr, 105 op/s 2024-09-13T00:34:13.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:13 smithi136 bash[75164]: cluster 2024-09-13T00:34:12.162621+0000 mgr.y (mgr.24661) 1712 : cluster [DBG] pgmap v2974: 457 pgs: 2 peering, 455 active+clean; 27 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 101 KiB/s rd, 1.5 MiB/s wr, 105 op/s 2024-09-13T00:34:13.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:13 smithi136 bash[76371]: cluster 2024-09-13T00:34:12.162621+0000 mgr.y (mgr.24661) 1712 : cluster [DBG] pgmap v2974: 457 pgs: 2 peering, 455 active+clean; 27 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 101 KiB/s rd, 1.5 MiB/s wr, 105 op/s 2024-09-13T00:34:15.516 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 512 2024-09-13T00:34:15.516 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 512 2024-09-13T00:34:15.516 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 512 2024-09-13T00:34:15.517 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 512 2024-09-13T00:34:15.517 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 512 2024-09-13T00:34:15.517 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:15.517 INFO:tasks.workunit.client.0.smithi136.stdout:started write 2024-09-13T00:34:15.517 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:15.517 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:15.517 INFO:tasks.workunit.client.0.smithi136.stdout:finished write 2024-09-13T00:34:15.517 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:15.517 INFO:tasks.workunit.client.0.smithi136.stdout:started write 2024-09-13T00:34:15.518 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:15.518 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:15.518 INFO:tasks.workunit.client.0.smithi136.stdout:finished write 2024-09-13T00:34:15.518 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:15.518 INFO:tasks.workunit.client.0.smithi136.stdout:started write 2024-09-13T00:34:15.518 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:15.518 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:15.518 INFO:tasks.workunit.client.0.smithi136.stdout:finished write 2024-09-13T00:34:15.518 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:15.518 INFO:tasks.workunit.client.0.smithi136.stdout:started write 2024-09-13T00:34:15.518 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:15.518 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:15.518 INFO:tasks.workunit.client.0.smithi136.stdout:finished write 2024-09-13T00:34:15.518 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:15.518 INFO:tasks.workunit.client.0.smithi136.stdout:started write 2024-09-13T00:34:15.519 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:15.519 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:15.519 INFO:tasks.workunit.client.0.smithi136.stdout:finished write 2024-09-13T00:34:15.519 INFO:tasks.workunit.client.0.smithi136.stdout:start compare and write 2024-09-13T00:34:15.519 INFO:tasks.workunit.client.0.smithi136.stdout:compare and wrote: 512 2024-09-13T00:34:15.519 INFO:tasks.workunit.client.0.smithi136.stdout:start compare and write 2024-09-13T00:34:15.519 INFO:tasks.workunit.client.0.smithi136.stdout:compare and wrote: 512 2024-09-13T00:34:15.519 INFO:tasks.workunit.client.0.smithi136.stdout:start compare and write 2024-09-13T00:34:15.519 INFO:tasks.workunit.client.0.smithi136.stdout:compare and wrote: 512 2024-09-13T00:34:15.519 INFO:tasks.workunit.client.0.smithi136.stdout:start compare and write 2024-09-13T00:34:15.519 INFO:tasks.workunit.client.0.smithi136.stdout:compare and wrote: 512 2024-09-13T00:34:15.519 INFO:tasks.workunit.client.0.smithi136.stdout:start compare and write 2024-09-13T00:34:15.519 INFO:tasks.workunit.client.0.smithi136.stdout:compare and wrote: 512 2024-09-13T00:34:15.519 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:15.520 INFO:tasks.workunit.client.0.smithi136.stdout:started aio compare and write 2024-09-13T00:34:15.520 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:15.520 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:15.520 INFO:tasks.workunit.client.0.smithi136.stdout:finished aio compare and write 2024-09-13T00:34:15.520 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:15.520 INFO:tasks.workunit.client.0.smithi136.stdout:started aio compare and write 2024-09-13T00:34:15.520 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:15.520 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:15.520 INFO:tasks.workunit.client.0.smithi136.stdout:finished aio compare and write 2024-09-13T00:34:15.520 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:15.520 INFO:tasks.workunit.client.0.smithi136.stdout:started aio compare and write 2024-09-13T00:34:15.520 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:15.520 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:15.520 INFO:tasks.workunit.client.0.smithi136.stdout:finished aio compare and write 2024-09-13T00:34:15.521 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:15.521 INFO:tasks.workunit.client.0.smithi136.stdout:started aio compare and write 2024-09-13T00:34:15.521 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:15.521 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:15.521 INFO:tasks.workunit.client.0.smithi136.stdout:finished aio compare and write 2024-09-13T00:34:15.521 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:15.521 INFO:tasks.workunit.client.0.smithi136.stdout:started aio compare and write 2024-09-13T00:34:15.521 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:15.521 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:15.521 INFO:tasks.workunit.client.0.smithi136.stdout:finished aio compare and write 2024-09-13T00:34:15.521 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:34:15.521 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:34:15.521 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:34:15.521 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:34:15.522 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:34:15.522 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:15.522 INFO:tasks.workunit.client.0.smithi136.stdout:started read 2024-09-13T00:34:15.522 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:34:15.522 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 512 2024-09-13T00:34:15.522 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:15.522 INFO:tasks.workunit.client.0.smithi136.stdout:started read 2024-09-13T00:34:15.522 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:34:15.522 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 512 2024-09-13T00:34:15.522 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:15.522 INFO:tasks.workunit.client.0.smithi136.stdout:started read 2024-09-13T00:34:15.522 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:34:15.522 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 512 2024-09-13T00:34:15.522 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:15.523 INFO:tasks.workunit.client.0.smithi136.stdout:started read 2024-09-13T00:34:15.523 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:34:15.523 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 512 2024-09-13T00:34:15.523 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:15.523 INFO:tasks.workunit.client.0.smithi136.stdout:started read 2024-09-13T00:34:15.523 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:34:15.523 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 512 2024-09-13T00:34:15.523 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 512~512 = 512 2024-09-13T00:34:15.523 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:15.523 INFO:tasks.workunit.client.0.smithi136.stdout:aio discard: 1536~512 = 0 2024-09-13T00:34:15.523 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:34:15.523 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:34:15.523 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:34:15.523 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:34:15.523 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:34:15.524 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 0 2024-09-13T00:34:15.524 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:15.524 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:15.524 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 0 2024-09-13T00:34:15.524 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:15.524 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:15.524 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 16384 2024-09-13T00:34:15.524 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:15.525 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:15.525 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 16384 2024-09-13T00:34:15.525 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:15.525 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:15.525 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:15.525 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:15.526 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 49152 2024-09-13T00:34:15.526 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:15.526 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:15.526 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 49152 2024-09-13T00:34:15.526 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:15.526 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:15.526 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 65536 2024-09-13T00:34:15.526 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:15.526 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:15.527 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 65536 2024-09-13T00:34:15.527 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:15.527 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:15.527 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:15.527 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:15.528 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 98304 2024-09-13T00:34:15.528 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:15.528 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:15.528 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 98304 2024-09-13T00:34:15.528 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:15.528 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:15.528 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 114688 2024-09-13T00:34:15.529 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:15.529 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:15.529 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 114688 2024-09-13T00:34:15.529 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:15.529 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:15.529 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:15.529 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:15.529 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 147456 2024-09-13T00:34:15.529 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:15.529 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:15.529 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 147456 2024-09-13T00:34:15.529 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:15.529 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:15.529 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 163840 2024-09-13T00:34:15.529 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:15.530 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:15.530 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 163840 2024-09-13T00:34:15.530 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:15.530 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:15.530 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:15.530 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:15.530 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 196608 2024-09-13T00:34:15.530 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:15.530 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:15.530 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 196608 2024-09-13T00:34:15.530 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:15.530 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:15.530 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 212992 2024-09-13T00:34:15.530 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:15.530 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:15.531 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 212992 2024-09-13T00:34:15.531 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:15.531 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:15.531 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:15.531 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:15.531 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:15.531 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:15.531 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:15.531 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:15.531 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:15.531 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:15.531 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:15.531 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:15.531 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:15.531 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:15.531 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:15.532 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:15.532 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:15.532 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:15.532 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:15.532 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:15.532 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:15.532 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:15.532 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:15.532 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:15.532 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:15.532 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:15.532 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:15.532 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:15.532 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:15.532 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:15.533 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:15.533 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:15.533 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:15.533 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:15.533 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:15.533 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:15.533 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:15.533 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:15.533 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:15.533 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:15.533 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:15.533 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:15.533 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:15.533 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:15.534 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:15.534 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:15.534 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:15.534 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:15.534 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:15.534 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:15.534 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:15.534 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:15.534 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:15.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:15 smithi181 bash[67587]: cluster 2024-09-13T00:34:14.164776+0000 mgr.y (mgr.24661) 1713 : cluster [DBG] pgmap v2975: 457 pgs: 2 peering, 455 active+clean; 31 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 1.5 MiB/s rd, 2.2 MiB/s wr, 372 op/s 2024-09-13T00:34:15.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:15 smithi136 bash[75164]: cluster 2024-09-13T00:34:14.164776+0000 mgr.y (mgr.24661) 1713 : cluster [DBG] pgmap v2975: 457 pgs: 2 peering, 455 active+clean; 31 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 1.5 MiB/s rd, 2.2 MiB/s wr, 372 op/s 2024-09-13T00:34:15.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:15 smithi136 bash[76371]: cluster 2024-09-13T00:34:14.164776+0000 mgr.y (mgr.24661) 1713 : cluster [DBG] pgmap v2975: 457 pgs: 2 peering, 455 active+clean; 31 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 1.5 MiB/s rd, 2.2 MiB/s wr, 372 op/s 2024-09-13T00:34:17.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:17 smithi181 bash[67587]: cluster 2024-09-13T00:34:16.166990+0000 mgr.y (mgr.24661) 1714 : cluster [DBG] pgmap v2976: 457 pgs: 457 active+clean; 31 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 1.5 MiB/s rd, 2.2 MiB/s wr, 368 op/s; 4 B/s, 9 keys/s, 0 objects/s recovering 2024-09-13T00:34:17.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:17 smithi136 bash[75164]: cluster 2024-09-13T00:34:16.166990+0000 mgr.y (mgr.24661) 1714 : cluster [DBG] pgmap v2976: 457 pgs: 457 active+clean; 31 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 1.5 MiB/s rd, 2.2 MiB/s wr, 368 op/s; 4 B/s, 9 keys/s, 0 objects/s recovering 2024-09-13T00:34:17.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:17 smithi136 bash[76371]: cluster 2024-09-13T00:34:16.166990+0000 mgr.y (mgr.24661) 1714 : cluster [DBG] pgmap v2976: 457 pgs: 457 active+clean; 31 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 1.5 MiB/s rd, 2.2 MiB/s wr, 368 op/s; 4 B/s, 9 keys/s, 0 objects/s recovering 2024-09-13T00:34:19.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:19 smithi181 bash[67587]: cluster 2024-09-13T00:34:18.169695+0000 mgr.y (mgr.24661) 1715 : cluster [DBG] pgmap v2977: 457 pgs: 457 active+clean; 31 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 1.5 MiB/s rd, 2.2 MiB/s wr, 368 op/s; 4 B/s, 9 keys/s, 0 objects/s recovering 2024-09-13T00:34:19.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:19 smithi181 bash[67587]: audit 2024-09-13T00:34:18.968155+0000 mon.a (mon.0) 3206 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:34:19.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:34:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:34:19] "GET /metrics HTTP/1.1" 200 60714 "" "Prometheus/2.43.0" 2024-09-13T00:34:19.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:19 smithi136 bash[76371]: cluster 2024-09-13T00:34:18.169695+0000 mgr.y (mgr.24661) 1715 : cluster [DBG] pgmap v2977: 457 pgs: 457 active+clean; 31 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 1.5 MiB/s rd, 2.2 MiB/s wr, 368 op/s; 4 B/s, 9 keys/s, 0 objects/s recovering 2024-09-13T00:34:19.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:19 smithi136 bash[76371]: audit 2024-09-13T00:34:18.968155+0000 mon.a (mon.0) 3206 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:34:19.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:19 smithi136 bash[75164]: cluster 2024-09-13T00:34:18.169695+0000 mgr.y (mgr.24661) 1715 : cluster [DBG] pgmap v2977: 457 pgs: 457 active+clean; 31 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 1.5 MiB/s rd, 2.2 MiB/s wr, 368 op/s; 4 B/s, 9 keys/s, 0 objects/s recovering 2024-09-13T00:34:19.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:19 smithi136 bash[75164]: audit 2024-09-13T00:34:18.968155+0000 mon.a (mon.0) 3206 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:34:20.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:20 smithi181 bash[67587]: audit 2024-09-13T00:34:19.527810+0000 mon.a (mon.0) 3207 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:34:20.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:20 smithi181 bash[67587]: audit 2024-09-13T00:34:19.528795+0000 mon.a (mon.0) 3208 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:34:20.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:20 smithi181 bash[67587]: audit 2024-09-13T00:34:19.534839+0000 mon.a (mon.0) 3209 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:34:20.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:20 smithi136 bash[75164]: audit 2024-09-13T00:34:19.527810+0000 mon.a (mon.0) 3207 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:34:20.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:20 smithi136 bash[75164]: audit 2024-09-13T00:34:19.528795+0000 mon.a (mon.0) 3208 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:34:20.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:20 smithi136 bash[75164]: audit 2024-09-13T00:34:19.534839+0000 mon.a (mon.0) 3209 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:34:20.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:20 smithi136 bash[76371]: audit 2024-09-13T00:34:19.527810+0000 mon.a (mon.0) 3207 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:34:20.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:20 smithi136 bash[76371]: audit 2024-09-13T00:34:19.528795+0000 mon.a (mon.0) 3208 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:34:20.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:20 smithi136 bash[76371]: audit 2024-09-13T00:34:19.534839+0000 mon.a (mon.0) 3209 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:34:21.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:21 smithi181 bash[67587]: cluster 2024-09-13T00:34:20.170999+0000 mgr.y (mgr.24661) 1716 : cluster [DBG] pgmap v2978: 457 pgs: 457 active+clean; 31 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.0 MiB/s wr, 710 op/s; 3 B/s, 8 keys/s, 0 objects/s recovering 2024-09-13T00:34:21.706 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:21.707 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:21.707 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:21.707 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:21.707 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:21.707 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:21.708 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:21.708 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:21.708 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:21.708 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:21.708 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:21.709 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:21.709 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:21.709 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:21.709 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:21.709 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:21.709 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:21.709 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:21.709 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:21.709 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:21.710 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:21.710 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:21.710 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:21.710 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:21.710 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:21.710 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:21.710 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:21.710 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:21.710 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:21.710 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:21.710 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:21.710 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:21.710 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:21.710 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:21.711 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:21.711 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:21.711 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:21.711 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:21.711 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:21.711 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:21.711 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:21.711 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:21.711 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:21.711 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:21.711 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:21.711 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:21.711 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:21.712 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:21.712 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:21.712 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:21.712 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:21.712 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:21.712 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:21.712 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:21.712 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:21.712 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:21.712 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:21.712 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:21.712 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:21.713 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:21.713 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:21.713 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:21.713 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:21.713 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:21.713 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:21.713 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:21.713 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:21.713 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:21.713 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:21.713 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:21.713 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:21.713 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:21.714 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:21.714 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:21.714 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:21.714 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:21.714 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:21.714 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:21.714 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:21.715 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:21.715 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:21.715 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:21.715 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:21.715 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:21.715 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:21.716 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:21.716 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:21.716 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:21.716 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:21.716 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:21.717 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:21.717 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:21.717 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:21.717 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:21.717 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:21.717 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:21.717 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:21.717 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:21.717 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:21.717 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:21.717 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:21.718 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:21.718 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:21.718 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:21.718 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:21.718 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:21.718 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:21.718 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:21.718 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:21.718 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:21.718 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:21.718 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:21.718 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:21.719 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:21.719 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:21.719 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:21.719 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:21.719 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:21.719 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:21.719 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:21.719 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:21.719 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:34:21.720 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:34:21.720 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 512 2024-09-13T00:34:21.720 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:34:21.720 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 4 2024-09-13T00:34:21.720 INFO:tasks.workunit.client.0.smithi136.stdout:read: 4 2024-09-13T00:34:21.720 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestEncryptionLUKS2 (18943 ms) 2024-09-13T00:34:21.720 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestIOWithIOHint 2024-09-13T00:34:21.720 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:34:21.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:21 smithi136 bash[75164]: cluster 2024-09-13T00:34:20.170999+0000 mgr.y (mgr.24661) 1716 : cluster [DBG] pgmap v2978: 457 pgs: 457 active+clean; 31 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.0 MiB/s wr, 710 op/s; 3 B/s, 8 keys/s, 0 objects/s recovering 2024-09-13T00:34:21.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:21 smithi136 bash[76371]: cluster 2024-09-13T00:34:20.170999+0000 mgr.y (mgr.24661) 1716 : cluster [DBG] pgmap v2978: 457 pgs: 457 active+clean; 31 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.0 MiB/s wr, 710 op/s; 3 B/s, 8 keys/s, 0 objects/s recovering 2024-09-13T00:34:22.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:34:22 smithi181 bash[95505]: ts=2024-09-13T00:34:22.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:34:23.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:23 smithi181 bash[67587]: cluster 2024-09-13T00:34:22.174007+0000 mgr.y (mgr.24661) 1717 : cluster [DBG] pgmap v2979: 457 pgs: 457 active+clean; 31 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.8 MiB/s wr, 642 op/s; 3 B/s, 7 keys/s, 0 objects/s recovering 2024-09-13T00:34:23.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:23 smithi136 bash[76371]: cluster 2024-09-13T00:34:22.174007+0000 mgr.y (mgr.24661) 1717 : cluster [DBG] pgmap v2979: 457 pgs: 457 active+clean; 31 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.8 MiB/s wr, 642 op/s; 3 B/s, 7 keys/s, 0 objects/s recovering 2024-09-13T00:34:23.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:23 smithi136 bash[75164]: cluster 2024-09-13T00:34:22.174007+0000 mgr.y (mgr.24661) 1717 : cluster [DBG] pgmap v2979: 457 pgs: 457 active+clean; 31 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.8 MiB/s wr, 642 op/s; 3 B/s, 7 keys/s, 0 objects/s recovering 2024-09-13T00:34:24.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:24 smithi181 bash[67587]: audit 2024-09-13T00:34:23.829121+0000 mon.a (mon.0) 3210 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:34:24.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:24 smithi136 bash[76371]: audit 2024-09-13T00:34:23.829121+0000 mon.a (mon.0) 3210 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:34:24.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:24 smithi136 bash[75164]: audit 2024-09-13T00:34:23.829121+0000 mon.a (mon.0) 3210 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:34:25.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:25 smithi181 bash[67587]: cluster 2024-09-13T00:34:24.175923+0000 mgr.y (mgr.24661) 1718 : cluster [DBG] pgmap v2980: 457 pgs: 457 active+clean; 35 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 3.0 MiB/s rd, 720 KiB/s wr, 763 op/s; 2 B/s, 6 keys/s, 0 objects/s recovering 2024-09-13T00:34:25.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:25 smithi136 bash[75164]: cluster 2024-09-13T00:34:24.175923+0000 mgr.y (mgr.24661) 1718 : cluster [DBG] pgmap v2980: 457 pgs: 457 active+clean; 35 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 3.0 MiB/s rd, 720 KiB/s wr, 763 op/s; 2 B/s, 6 keys/s, 0 objects/s recovering 2024-09-13T00:34:25.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:25 smithi136 bash[76371]: cluster 2024-09-13T00:34:24.175923+0000 mgr.y (mgr.24661) 1718 : cluster [DBG] pgmap v2980: 457 pgs: 457 active+clean; 35 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 3.0 MiB/s rd, 720 KiB/s wr, 763 op/s; 2 B/s, 6 keys/s, 0 objects/s recovering 2024-09-13T00:34:27.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:27 smithi181 bash[67587]: cluster 2024-09-13T00:34:26.178643+0000 mgr.y (mgr.24661) 1719 : cluster [DBG] pgmap v2981: 457 pgs: 457 active+clean; 35 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 2.0 MiB/s rd, 245 KiB/s wr, 525 op/s; 2 B/s, 6 keys/s, 0 objects/s recovering 2024-09-13T00:34:27.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:27 smithi136 bash[75164]: cluster 2024-09-13T00:34:26.178643+0000 mgr.y (mgr.24661) 1719 : cluster [DBG] pgmap v2981: 457 pgs: 457 active+clean; 35 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 2.0 MiB/s rd, 245 KiB/s wr, 525 op/s; 2 B/s, 6 keys/s, 0 objects/s recovering 2024-09-13T00:34:27.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:27 smithi136 bash[76371]: cluster 2024-09-13T00:34:26.178643+0000 mgr.y (mgr.24661) 1719 : cluster [DBG] pgmap v2981: 457 pgs: 457 active+clean; 35 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 2.0 MiB/s rd, 245 KiB/s wr, 525 op/s; 2 B/s, 6 keys/s, 0 objects/s recovering 2024-09-13T00:34:27.794 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 512 2024-09-13T00:34:27.794 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 512 2024-09-13T00:34:27.794 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 512 2024-09-13T00:34:27.795 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 512 2024-09-13T00:34:27.795 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 512 2024-09-13T00:34:27.795 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:27.795 INFO:tasks.workunit.client.0.smithi136.stdout:started write 2024-09-13T00:34:27.795 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:27.795 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:27.795 INFO:tasks.workunit.client.0.smithi136.stdout:finished write 2024-09-13T00:34:27.795 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:27.795 INFO:tasks.workunit.client.0.smithi136.stdout:started write 2024-09-13T00:34:27.795 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:27.795 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:27.795 INFO:tasks.workunit.client.0.smithi136.stdout:finished write 2024-09-13T00:34:27.796 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:27.796 INFO:tasks.workunit.client.0.smithi136.stdout:started write 2024-09-13T00:34:27.796 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:27.796 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:27.796 INFO:tasks.workunit.client.0.smithi136.stdout:finished write 2024-09-13T00:34:27.796 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:27.796 INFO:tasks.workunit.client.0.smithi136.stdout:started write 2024-09-13T00:34:27.796 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:27.796 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:27.796 INFO:tasks.workunit.client.0.smithi136.stdout:finished write 2024-09-13T00:34:27.796 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:27.796 INFO:tasks.workunit.client.0.smithi136.stdout:started write 2024-09-13T00:34:27.796 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:27.797 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:27.797 INFO:tasks.workunit.client.0.smithi136.stdout:finished write 2024-09-13T00:34:27.797 INFO:tasks.workunit.client.0.smithi136.stdout:start compare and write 2024-09-13T00:34:27.797 INFO:tasks.workunit.client.0.smithi136.stdout:compare and wrote: 512 2024-09-13T00:34:27.797 INFO:tasks.workunit.client.0.smithi136.stdout:start compare and write 2024-09-13T00:34:27.797 INFO:tasks.workunit.client.0.smithi136.stdout:compare and wrote: 512 2024-09-13T00:34:27.797 INFO:tasks.workunit.client.0.smithi136.stdout:start compare and write 2024-09-13T00:34:27.797 INFO:tasks.workunit.client.0.smithi136.stdout:compare and wrote: 512 2024-09-13T00:34:27.797 INFO:tasks.workunit.client.0.smithi136.stdout:start compare and write 2024-09-13T00:34:27.797 INFO:tasks.workunit.client.0.smithi136.stdout:compare and wrote: 512 2024-09-13T00:34:27.797 INFO:tasks.workunit.client.0.smithi136.stdout:start compare and write 2024-09-13T00:34:27.797 INFO:tasks.workunit.client.0.smithi136.stdout:compare and wrote: 512 2024-09-13T00:34:27.798 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:27.798 INFO:tasks.workunit.client.0.smithi136.stdout:started aio compare and write 2024-09-13T00:34:27.798 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:27.798 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:27.798 INFO:tasks.workunit.client.0.smithi136.stdout:finished aio compare and write 2024-09-13T00:34:27.798 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:27.798 INFO:tasks.workunit.client.0.smithi136.stdout:started aio compare and write 2024-09-13T00:34:27.798 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:27.798 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:27.798 INFO:tasks.workunit.client.0.smithi136.stdout:finished aio compare and write 2024-09-13T00:34:27.798 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:27.798 INFO:tasks.workunit.client.0.smithi136.stdout:started aio compare and write 2024-09-13T00:34:27.799 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:27.799 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:27.799 INFO:tasks.workunit.client.0.smithi136.stdout:finished aio compare and write 2024-09-13T00:34:27.799 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:27.799 INFO:tasks.workunit.client.0.smithi136.stdout:started aio compare and write 2024-09-13T00:34:27.799 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:27.799 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:27.799 INFO:tasks.workunit.client.0.smithi136.stdout:finished aio compare and write 2024-09-13T00:34:27.799 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:27.799 INFO:tasks.workunit.client.0.smithi136.stdout:started aio compare and write 2024-09-13T00:34:27.799 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:27.799 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:27.799 INFO:tasks.workunit.client.0.smithi136.stdout:finished aio compare and write 2024-09-13T00:34:27.800 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:34:27.800 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:34:27.800 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:34:27.800 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:34:27.800 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:34:27.800 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:27.800 INFO:tasks.workunit.client.0.smithi136.stdout:started read 2024-09-13T00:34:27.800 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:34:27.800 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 512 2024-09-13T00:34:27.800 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:27.800 INFO:tasks.workunit.client.0.smithi136.stdout:started read 2024-09-13T00:34:27.801 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:34:27.801 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 512 2024-09-13T00:34:27.801 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:27.801 INFO:tasks.workunit.client.0.smithi136.stdout:started read 2024-09-13T00:34:27.801 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:34:27.801 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 512 2024-09-13T00:34:27.801 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:27.801 INFO:tasks.workunit.client.0.smithi136.stdout:started read 2024-09-13T00:34:27.801 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:34:27.801 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 512 2024-09-13T00:34:27.801 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:27.801 INFO:tasks.workunit.client.0.smithi136.stdout:started read 2024-09-13T00:34:27.802 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:34:27.802 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 512 2024-09-13T00:34:27.802 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 512~512 = 512 2024-09-13T00:34:27.802 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:27.802 INFO:tasks.workunit.client.0.smithi136.stdout:aio discard: 1536~512 = 0 2024-09-13T00:34:27.802 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:34:27.802 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:34:27.802 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:34:27.802 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:34:27.802 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:34:27.802 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 0 2024-09-13T00:34:27.802 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:27.803 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:27.803 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 0 2024-09-13T00:34:27.803 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:27.803 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:27.803 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 16384 2024-09-13T00:34:27.803 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:27.803 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:27.803 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 16384 2024-09-13T00:34:27.803 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:27.803 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:27.803 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:27.803 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:27.803 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 49152 2024-09-13T00:34:27.804 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:27.804 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:27.804 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 49152 2024-09-13T00:34:27.804 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:27.804 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:27.804 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 65536 2024-09-13T00:34:27.804 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:27.804 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:27.804 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 65536 2024-09-13T00:34:27.804 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:27.804 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:27.804 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:27.804 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:27.804 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 98304 2024-09-13T00:34:27.805 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:27.805 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:27.805 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 98304 2024-09-13T00:34:27.805 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:27.805 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:27.805 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 114688 2024-09-13T00:34:27.805 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:27.805 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:27.805 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 114688 2024-09-13T00:34:27.805 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:27.805 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:27.805 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:27.805 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:27.806 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 147456 2024-09-13T00:34:27.806 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:27.806 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:27.806 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 147456 2024-09-13T00:34:27.806 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:27.806 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:27.806 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 163840 2024-09-13T00:34:27.806 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:27.806 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:27.806 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 163840 2024-09-13T00:34:27.806 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:27.806 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:27.807 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:27.807 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:27.807 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 196608 2024-09-13T00:34:27.807 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:27.807 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:27.807 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 196608 2024-09-13T00:34:27.807 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:27.807 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:27.807 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 212992 2024-09-13T00:34:27.807 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:27.807 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:27.807 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 212992 2024-09-13T00:34:27.807 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:27.807 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:27.808 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:27.808 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:27.808 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:27.808 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:27.808 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:27.808 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:27.808 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:27.808 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:27.808 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:27.808 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:27.808 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:27.808 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:27.809 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:27.809 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:27.809 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:27.809 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:27.809 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:27.809 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:27.809 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:27.809 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:27.809 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:27.809 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:27.809 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:27.809 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:27.810 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:27.810 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:27.810 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:27.810 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:27.810 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:27.810 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:27.810 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:27.810 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:27.810 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:27.810 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:27.810 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:27.810 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:27.811 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:27.811 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:27.811 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:27.811 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:27.811 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:27.811 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:27.811 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:27.811 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:27.811 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:27.811 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:27.811 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:27.811 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:27.811 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:27.812 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:27.812 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:29.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:29 smithi181 bash[67587]: cluster 2024-09-13T00:34:28.180793+0000 mgr.y (mgr.24661) 1720 : cluster [DBG] pgmap v2982: 457 pgs: 457 active+clean; 35 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 2.1 MiB/s rd, 322 KiB/s wr, 836 op/s 2024-09-13T00:34:29.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:29 smithi136 bash[75164]: cluster 2024-09-13T00:34:28.180793+0000 mgr.y (mgr.24661) 1720 : cluster [DBG] pgmap v2982: 457 pgs: 457 active+clean; 35 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 2.1 MiB/s rd, 322 KiB/s wr, 836 op/s 2024-09-13T00:34:29.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:29 smithi136 bash[76371]: cluster 2024-09-13T00:34:28.180793+0000 mgr.y (mgr.24661) 1720 : cluster [DBG] pgmap v2982: 457 pgs: 457 active+clean; 35 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 2.1 MiB/s rd, 322 KiB/s wr, 836 op/s 2024-09-13T00:34:29.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:34:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:34:29] "GET /metrics HTTP/1.1" 200 60714 "" "Prometheus/2.43.0" 2024-09-13T00:34:31.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:31 smithi181 bash[67587]: cluster 2024-09-13T00:34:30.182818+0000 mgr.y (mgr.24661) 1721 : cluster [DBG] pgmap v2983: 457 pgs: 457 active+clean; 35 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 2.1 MiB/s rd, 322 KiB/s wr, 836 op/s 2024-09-13T00:34:31.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:31 smithi136 bash[76371]: cluster 2024-09-13T00:34:30.182818+0000 mgr.y (mgr.24661) 1721 : cluster [DBG] pgmap v2983: 457 pgs: 457 active+clean; 35 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 2.1 MiB/s rd, 322 KiB/s wr, 836 op/s 2024-09-13T00:34:31.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:31 smithi136 bash[75164]: cluster 2024-09-13T00:34:30.182818+0000 mgr.y (mgr.24661) 1721 : cluster [DBG] pgmap v2983: 457 pgs: 457 active+clean; 35 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 2.1 MiB/s rd, 322 KiB/s wr, 836 op/s 2024-09-13T00:34:32.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:34:32 smithi181 bash[95505]: ts=2024-09-13T00:34:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:34:33.296 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:33.297 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:33.297 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:33.297 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:33.297 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:33.297 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:33.297 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:33.297 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:33.297 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:33.297 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:33.297 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:33.297 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:33.297 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:33.297 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:33.298 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:33.298 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:33.298 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:33.298 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:33.298 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:33.298 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:33.298 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:33.298 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:33.298 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:33.298 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:33.298 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:33.298 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:33.298 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:33.299 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:33.299 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:33.299 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:33.299 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:33.299 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:33.299 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:33.299 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:33.299 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:33.299 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:33.299 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:33.299 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:33.299 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:33.299 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:33.299 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:33.299 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:33.300 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:33.300 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:33.300 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:33.300 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:33.300 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:33.300 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:33.300 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:33.300 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:33.300 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:33.300 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:33.300 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:33.300 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:33.300 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:33.300 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:33.301 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:33.301 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:33.301 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:33.301 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:33.301 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:33.301 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:33.301 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:33.301 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:33.301 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:33.301 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:33.301 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:33.301 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:33.301 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:33.301 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:33.302 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:33.302 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:33.302 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:33.302 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:33.302 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:33.302 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:33.302 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:33.302 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:33.302 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:33.302 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:33.302 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:33.302 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:33.302 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:33.302 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:33.303 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:33.303 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:33.303 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:33.303 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:33.303 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:33.303 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:33.303 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:33.303 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:33.303 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:33.303 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:33.303 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:33.303 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:33.303 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:33.304 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:33.304 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:33.304 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:33.304 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:33.304 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:33.304 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:33.304 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:33.304 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:33.304 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:33.304 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:33.304 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:33.304 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:33.304 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:33.305 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:33.305 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:33.305 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:33.305 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:33.305 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:33.305 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:33.305 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:33.305 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:33.305 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:33.305 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:33.305 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:33.305 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:34:33.305 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 512 2024-09-13T00:34:33.305 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:34:33.306 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestIOWithIOHint (11590 ms) 2024-09-13T00:34:33.306 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestDataPoolIO 2024-09-13T00:34:33.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:33 smithi181 bash[67587]: cluster 2024-09-13T00:34:32.185712+0000 mgr.y (mgr.24661) 1722 : cluster [DBG] pgmap v2984: 457 pgs: 457 active+clean; 35 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 736 KiB/s rd, 143 KiB/s wr, 494 op/s 2024-09-13T00:34:33.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:33 smithi136 bash[75164]: cluster 2024-09-13T00:34:32.185712+0000 mgr.y (mgr.24661) 1722 : cluster [DBG] pgmap v2984: 457 pgs: 457 active+clean; 35 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 736 KiB/s rd, 143 KiB/s wr, 494 op/s 2024-09-13T00:34:33.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:33 smithi136 bash[76371]: cluster 2024-09-13T00:34:32.185712+0000 mgr.y (mgr.24661) 1722 : cluster [DBG] pgmap v2984: 457 pgs: 457 active+clean; 35 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 736 KiB/s rd, 143 KiB/s wr, 494 op/s 2024-09-13T00:34:35.360 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:34:35.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:35 smithi181 bash[67587]: cluster 2024-09-13T00:34:34.187603+0000 mgr.y (mgr.24661) 1723 : cluster [DBG] pgmap v2985: 457 pgs: 457 active+clean; 35 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 892 KiB/s rd, 228 KiB/s wr, 788 op/s 2024-09-13T00:34:35.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:35 smithi181 bash[67587]: cluster 2024-09-13T00:34:34.342185+0000 mon.a (mon.0) 3211 : cluster [DBG] osdmap e1702: 8 total, 8 up, 8 in 2024-09-13T00:34:35.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:35 smithi181 bash[67587]: audit 2024-09-13T00:34:34.346111+0000 mon.a (mon.0) 3212 : audit [INF] from='client.? 172.21.15.136:0/1723785274' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-25","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:34:35.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:35 smithi136 bash[75164]: cluster 2024-09-13T00:34:34.187603+0000 mgr.y (mgr.24661) 1723 : cluster [DBG] pgmap v2985: 457 pgs: 457 active+clean; 35 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 892 KiB/s rd, 228 KiB/s wr, 788 op/s 2024-09-13T00:34:35.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:35 smithi136 bash[75164]: cluster 2024-09-13T00:34:34.342185+0000 mon.a (mon.0) 3211 : cluster [DBG] osdmap e1702: 8 total, 8 up, 8 in 2024-09-13T00:34:35.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:35 smithi136 bash[75164]: audit 2024-09-13T00:34:34.346111+0000 mon.a (mon.0) 3212 : audit [INF] from='client.? 172.21.15.136:0/1723785274' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-25","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:34:35.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:35 smithi136 bash[76371]: cluster 2024-09-13T00:34:34.187603+0000 mgr.y (mgr.24661) 1723 : cluster [DBG] pgmap v2985: 457 pgs: 457 active+clean; 35 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 892 KiB/s rd, 228 KiB/s wr, 788 op/s 2024-09-13T00:34:35.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:35 smithi136 bash[76371]: cluster 2024-09-13T00:34:34.342185+0000 mon.a (mon.0) 3211 : cluster [DBG] osdmap e1702: 8 total, 8 up, 8 in 2024-09-13T00:34:35.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:35 smithi136 bash[76371]: audit 2024-09-13T00:34:34.346111+0000 mon.a (mon.0) 3212 : audit [INF] from='client.? 172.21.15.136:0/1723785274' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-25","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:34:36.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:36 smithi136 bash[76371]: audit 2024-09-13T00:34:35.351714+0000 mon.a (mon.0) 3213 : audit [INF] from='client.? 172.21.15.136:0/1723785274' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-25","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:34:36.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:36 smithi136 bash[76371]: cluster 2024-09-13T00:34:35.352161+0000 mon.a (mon.0) 3214 : cluster [DBG] osdmap e1703: 8 total, 8 up, 8 in 2024-09-13T00:34:36.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:36 smithi136 bash[75164]: audit 2024-09-13T00:34:35.351714+0000 mon.a (mon.0) 3213 : audit [INF] from='client.? 172.21.15.136:0/1723785274' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-25","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:34:36.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:36 smithi136 bash[75164]: cluster 2024-09-13T00:34:35.352161+0000 mon.a (mon.0) 3214 : cluster [DBG] osdmap e1703: 8 total, 8 up, 8 in 2024-09-13T00:34:36.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:36 smithi181 bash[67587]: audit 2024-09-13T00:34:35.351714+0000 mon.a (mon.0) 3213 : audit [INF] from='client.? 172.21.15.136:0/1723785274' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-25","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:34:36.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:36 smithi181 bash[67587]: cluster 2024-09-13T00:34:35.352161+0000 mon.a (mon.0) 3214 : cluster [DBG] osdmap e1703: 8 total, 8 up, 8 in 2024-09-13T00:34:37.503 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 512 2024-09-13T00:34:37.503 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 512 2024-09-13T00:34:37.503 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 512 2024-09-13T00:34:37.503 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 512 2024-09-13T00:34:37.503 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 512 2024-09-13T00:34:37.503 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:37.503 INFO:tasks.workunit.client.0.smithi136.stdout:started write 2024-09-13T00:34:37.503 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:37.504 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:37.504 INFO:tasks.workunit.client.0.smithi136.stdout:finished write 2024-09-13T00:34:37.504 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:37.504 INFO:tasks.workunit.client.0.smithi136.stdout:started write 2024-09-13T00:34:37.504 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:37.504 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:37.504 INFO:tasks.workunit.client.0.smithi136.stdout:finished write 2024-09-13T00:34:37.504 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:37.504 INFO:tasks.workunit.client.0.smithi136.stdout:started write 2024-09-13T00:34:37.504 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:37.504 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:37.504 INFO:tasks.workunit.client.0.smithi136.stdout:finished write 2024-09-13T00:34:37.505 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:37.505 INFO:tasks.workunit.client.0.smithi136.stdout:started write 2024-09-13T00:34:37.505 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:37.505 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:37.505 INFO:tasks.workunit.client.0.smithi136.stdout:finished write 2024-09-13T00:34:37.505 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:37.505 INFO:tasks.workunit.client.0.smithi136.stdout:started write 2024-09-13T00:34:37.505 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:37.505 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:37.505 INFO:tasks.workunit.client.0.smithi136.stdout:finished write 2024-09-13T00:34:37.505 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:34:37.505 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:34:37.505 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:34:37.505 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:34:37.506 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:34:37.506 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:37.506 INFO:tasks.workunit.client.0.smithi136.stdout:started read 2024-09-13T00:34:37.506 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:34:37.506 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 512 2024-09-13T00:34:37.506 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:37.506 INFO:tasks.workunit.client.0.smithi136.stdout:started read 2024-09-13T00:34:37.506 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:34:37.506 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 512 2024-09-13T00:34:37.506 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:37.506 INFO:tasks.workunit.client.0.smithi136.stdout:started read 2024-09-13T00:34:37.506 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:34:37.506 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 512 2024-09-13T00:34:37.506 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:37.507 INFO:tasks.workunit.client.0.smithi136.stdout:started read 2024-09-13T00:34:37.507 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:34:37.507 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 512 2024-09-13T00:34:37.507 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:37.507 INFO:tasks.workunit.client.0.smithi136.stdout:started read 2024-09-13T00:34:37.507 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:34:37.507 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 512 2024-09-13T00:34:37.507 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 512~512 = 512 2024-09-13T00:34:37.507 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:37.507 INFO:tasks.workunit.client.0.smithi136.stdout:aio discard: 1536~512 = 0 2024-09-13T00:34:37.507 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:34:37.507 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:34:37.507 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:34:37.508 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:34:37.508 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:34:37.508 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:34:37.508 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:34:37.508 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestDataPoolIO (4206 ms) 2024-09-13T00:34:37.508 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteMismatch 2024-09-13T00:34:37.508 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:34:37.565 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestCompareAndWriteMismatch (63 ms) 2024-09-13T00:34:37.565 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteMismatch 2024-09-13T00:34:37.566 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:34:37.633 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteMismatch (67 ms) 2024-09-13T00:34:37.633 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteSuccess 2024-09-13T00:34:37.633 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:34:37.696 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestCompareAndWriteSuccess (63 ms) 2024-09-13T00:34:37.696 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteSuccess 2024-09-13T00:34:37.696 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:34:37.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:37 smithi136 bash[76371]: cluster 2024-09-13T00:34:36.189899+0000 mgr.y (mgr.24661) 1724 : cluster [DBG] pgmap v2988: 489 pgs: 4 creating+peering, 17 unknown, 468 active+clean; 35 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 236 KiB/s rd, 128 KiB/s wr, 442 op/s 2024-09-13T00:34:37.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:37 smithi136 bash[76371]: cluster 2024-09-13T00:34:36.364588+0000 mon.a (mon.0) 3215 : cluster [DBG] osdmap e1704: 8 total, 8 up, 8 in 2024-09-13T00:34:37.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:37 smithi136 bash[75164]: cluster 2024-09-13T00:34:36.189899+0000 mgr.y (mgr.24661) 1724 : cluster [DBG] pgmap v2988: 489 pgs: 4 creating+peering, 17 unknown, 468 active+clean; 35 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 236 KiB/s rd, 128 KiB/s wr, 442 op/s 2024-09-13T00:34:37.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:37 smithi136 bash[75164]: cluster 2024-09-13T00:34:36.364588+0000 mon.a (mon.0) 3215 : cluster [DBG] osdmap e1704: 8 total, 8 up, 8 in 2024-09-13T00:34:37.763 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteSuccess (68 ms) 2024-09-13T00:34:37.763 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteStripeUnitUnaligned 2024-09-13T00:34:37.763 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:34:37.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:37 smithi181 bash[67587]: cluster 2024-09-13T00:34:36.189899+0000 mgr.y (mgr.24661) 1724 : cluster [DBG] pgmap v2988: 489 pgs: 4 creating+peering, 17 unknown, 468 active+clean; 35 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 236 KiB/s rd, 128 KiB/s wr, 442 op/s 2024-09-13T00:34:37.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:37 smithi181 bash[67587]: cluster 2024-09-13T00:34:36.364588+0000 mon.a (mon.0) 3215 : cluster [DBG] osdmap e1704: 8 total, 8 up, 8 in 2024-09-13T00:34:37.905 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestCompareAndWriteStripeUnitUnaligned (141 ms) 2024-09-13T00:34:37.905 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteStripeUnitUnaligned 2024-09-13T00:34:37.905 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:34:38.025 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteStripeUnitUnaligned (121 ms) 2024-09-13T00:34:38.026 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteTooLarge 2024-09-13T00:34:38.026 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:34:38.157 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestCompareAndWriteTooLarge (131 ms) 2024-09-13T00:34:38.157 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteTooLarge 2024-09-13T00:34:38.157 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:34:38.294 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteTooLarge (137 ms) 2024-09-13T00:34:38.294 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteStripeUnitSuccess 2024-09-13T00:34:38.294 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:34:38.711 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestCompareAndWriteStripeUnitSuccess (418 ms) 2024-09-13T00:34:38.712 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteStripeUnitSuccess 2024-09-13T00:34:38.712 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:34:38.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:38 smithi136 bash[76371]: cluster 2024-09-13T00:34:37.373041+0000 mon.a (mon.0) 3216 : cluster [DBG] osdmap e1705: 8 total, 8 up, 8 in 2024-09-13T00:34:38.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:38 smithi136 bash[75164]: cluster 2024-09-13T00:34:37.373041+0000 mon.a (mon.0) 3216 : cluster [DBG] osdmap e1705: 8 total, 8 up, 8 in 2024-09-13T00:34:38.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:38 smithi181 bash[67587]: cluster 2024-09-13T00:34:37.373041+0000 mon.a (mon.0) 3216 : cluster [DBG] osdmap e1705: 8 total, 8 up, 8 in 2024-09-13T00:34:39.215 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteStripeUnitSuccess (503 ms) 2024-09-13T00:34:39.215 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteVIovecLenDiffers 2024-09-13T00:34:39.215 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:34:39.277 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteVIovecLenDiffers (62 ms) 2024-09-13T00:34:39.277 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteVMismatch 2024-09-13T00:34:39.277 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:34:39.339 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteVMismatch (63 ms) 2024-09-13T00:34:39.340 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteVSuccess 2024-09-13T00:34:39.340 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:34:39.403 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteVSuccess (63 ms) 2024-09-13T00:34:39.403 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestScatterGatherIO 2024-09-13T00:34:39.403 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:34:39.459 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestScatterGatherIO (57 ms) 2024-09-13T00:34:39.459 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestEmptyDiscard 2024-09-13T00:34:39.459 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:34:39.508 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:39.508 INFO:tasks.workunit.client.0.smithi136.stdout:aio discard: 0~1048576 = 0 2024-09-13T00:34:39.508 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:39.508 INFO:tasks.workunit.client.0.smithi136.stdout:aio discard: 0~4194304 = 0 2024-09-13T00:34:39.508 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:39.508 INFO:tasks.workunit.client.0.smithi136.stdout:aio discard: 3145728~1048576 = 0 2024-09-13T00:34:39.508 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestEmptyDiscard (48 ms) 2024-09-13T00:34:39.508 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestFUA 2024-09-13T00:34:39.509 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:34:39.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:39 smithi136 bash[76371]: cluster 2024-09-13T00:34:38.191366+0000 mgr.y (mgr.24661) 1725 : cluster [DBG] pgmap v2991: 489 pgs: 4 creating+peering, 5 unknown, 480 active+clean; 37 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 41 KiB/s rd, 1.5 KiB/s wr, 148 op/s 2024-09-13T00:34:39.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:39 smithi136 bash[76371]: audit 2024-09-13T00:34:38.829408+0000 mon.a (mon.0) 3217 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:34:39.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:34:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:34:39] "GET /metrics HTTP/1.1" 200 63174 "" "Prometheus/2.43.0" 2024-09-13T00:34:39.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:39 smithi136 bash[75164]: cluster 2024-09-13T00:34:38.191366+0000 mgr.y (mgr.24661) 1725 : cluster [DBG] pgmap v2991: 489 pgs: 4 creating+peering, 5 unknown, 480 active+clean; 37 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 41 KiB/s rd, 1.5 KiB/s wr, 148 op/s 2024-09-13T00:34:39.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:39 smithi136 bash[75164]: audit 2024-09-13T00:34:38.829408+0000 mon.a (mon.0) 3217 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:34:39.797 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 512 2024-09-13T00:34:39.798 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 512 2024-09-13T00:34:39.798 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 512 2024-09-13T00:34:39.798 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 512 2024-09-13T00:34:39.798 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 512 2024-09-13T00:34:39.798 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:34:39.798 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:34:39.798 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:34:39.798 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:34:39.798 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:34:39.798 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:39.798 INFO:tasks.workunit.client.0.smithi136.stdout:started write 2024-09-13T00:34:39.799 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:39.799 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:39.799 INFO:tasks.workunit.client.0.smithi136.stdout:finished write 2024-09-13T00:34:39.799 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:39.799 INFO:tasks.workunit.client.0.smithi136.stdout:started write 2024-09-13T00:34:39.799 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:39.799 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:39.799 INFO:tasks.workunit.client.0.smithi136.stdout:finished write 2024-09-13T00:34:39.799 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:39.799 INFO:tasks.workunit.client.0.smithi136.stdout:started write 2024-09-13T00:34:39.799 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:39.799 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:39.799 INFO:tasks.workunit.client.0.smithi136.stdout:finished write 2024-09-13T00:34:39.799 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:39.800 INFO:tasks.workunit.client.0.smithi136.stdout:started write 2024-09-13T00:34:39.800 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:39.800 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:39.800 INFO:tasks.workunit.client.0.smithi136.stdout:finished write 2024-09-13T00:34:39.800 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:39.800 INFO:tasks.workunit.client.0.smithi136.stdout:started write 2024-09-13T00:34:39.800 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:39.800 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:39.800 INFO:tasks.workunit.client.0.smithi136.stdout:finished write 2024-09-13T00:34:39.800 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:39.800 INFO:tasks.workunit.client.0.smithi136.stdout:started read 2024-09-13T00:34:39.800 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:34:39.800 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 512 2024-09-13T00:34:39.801 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:39.801 INFO:tasks.workunit.client.0.smithi136.stdout:started read 2024-09-13T00:34:39.801 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:34:39.801 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 512 2024-09-13T00:34:39.801 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:39.801 INFO:tasks.workunit.client.0.smithi136.stdout:started read 2024-09-13T00:34:39.801 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:34:39.801 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 512 2024-09-13T00:34:39.801 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:39.801 INFO:tasks.workunit.client.0.smithi136.stdout:started read 2024-09-13T00:34:39.801 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:34:39.801 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 512 2024-09-13T00:34:39.801 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:39.802 INFO:tasks.workunit.client.0.smithi136.stdout:started read 2024-09-13T00:34:39.802 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:34:39.802 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 512 2024-09-13T00:34:39.802 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestFUA (290 ms) 2024-09-13T00:34:39.802 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestIOPP 2024-09-13T00:34:39.802 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:34:39.848 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 512 2024-09-13T00:34:39.848 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 512 2024-09-13T00:34:39.848 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 512 2024-09-13T00:34:39.848 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 512 2024-09-13T00:34:39.848 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 512 2024-09-13T00:34:39.848 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:39.848 INFO:tasks.workunit.client.0.smithi136.stdout:started write 2024-09-13T00:34:39.848 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:39.848 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:39.848 INFO:tasks.workunit.client.0.smithi136.stdout:finished write 2024-09-13T00:34:39.848 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:39.849 INFO:tasks.workunit.client.0.smithi136.stdout:started write 2024-09-13T00:34:39.849 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:39.849 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:39.849 INFO:tasks.workunit.client.0.smithi136.stdout:finished write 2024-09-13T00:34:39.849 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:39.849 INFO:tasks.workunit.client.0.smithi136.stdout:started write 2024-09-13T00:34:39.849 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:39.849 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:39.849 INFO:tasks.workunit.client.0.smithi136.stdout:finished write 2024-09-13T00:34:39.849 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:39.849 INFO:tasks.workunit.client.0.smithi136.stdout:started write 2024-09-13T00:34:39.849 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:39.849 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:39.850 INFO:tasks.workunit.client.0.smithi136.stdout:finished write 2024-09-13T00:34:39.850 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:39.850 INFO:tasks.workunit.client.0.smithi136.stdout:started write 2024-09-13T00:34:39.850 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:39.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:39 smithi181 bash[67587]: cluster 2024-09-13T00:34:38.191366+0000 mgr.y (mgr.24661) 1725 : cluster [DBG] pgmap v2991: 489 pgs: 4 creating+peering, 5 unknown, 480 active+clean; 37 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 41 KiB/s rd, 1.5 KiB/s wr, 148 op/s 2024-09-13T00:34:39.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:39 smithi181 bash[67587]: audit 2024-09-13T00:34:38.829408+0000 mon.a (mon.0) 3217 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:34:39.885 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:39.885 INFO:tasks.workunit.client.0.smithi136.stdout:finished write 2024-09-13T00:34:39.885 INFO:tasks.workunit.client.0.smithi136.stdout:start compare and write 2024-09-13T00:34:39.886 INFO:tasks.workunit.client.0.smithi136.stdout:compare and wrote: 512 2024-09-13T00:34:39.886 INFO:tasks.workunit.client.0.smithi136.stdout:start compare and write 2024-09-13T00:34:39.886 INFO:tasks.workunit.client.0.smithi136.stdout:compare and wrote: 512 2024-09-13T00:34:39.886 INFO:tasks.workunit.client.0.smithi136.stdout:start compare and write 2024-09-13T00:34:39.886 INFO:tasks.workunit.client.0.smithi136.stdout:compare and wrote: 512 2024-09-13T00:34:39.886 INFO:tasks.workunit.client.0.smithi136.stdout:start compare and write 2024-09-13T00:34:39.886 INFO:tasks.workunit.client.0.smithi136.stdout:compare and wrote: 512 2024-09-13T00:34:39.886 INFO:tasks.workunit.client.0.smithi136.stdout:start compare and write 2024-09-13T00:34:39.886 INFO:tasks.workunit.client.0.smithi136.stdout:compare and wrote: 512 2024-09-13T00:34:39.886 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:39.886 INFO:tasks.workunit.client.0.smithi136.stdout:started aio compare and write 2024-09-13T00:34:39.886 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:39.888 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:39.888 INFO:tasks.workunit.client.0.smithi136.stdout:finished aio compare and write 2024-09-13T00:34:39.888 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:39.888 INFO:tasks.workunit.client.0.smithi136.stdout:started aio compare and write 2024-09-13T00:34:39.889 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:39.890 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:39.891 INFO:tasks.workunit.client.0.smithi136.stdout:finished aio compare and write 2024-09-13T00:34:39.891 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:39.891 INFO:tasks.workunit.client.0.smithi136.stdout:started aio compare and write 2024-09-13T00:34:39.891 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:39.894 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:39.894 INFO:tasks.workunit.client.0.smithi136.stdout:finished aio compare and write 2024-09-13T00:34:39.894 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:39.894 INFO:tasks.workunit.client.0.smithi136.stdout:started aio compare and write 2024-09-13T00:34:39.894 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:39.897 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:39.897 INFO:tasks.workunit.client.0.smithi136.stdout:finished aio compare and write 2024-09-13T00:34:39.897 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:39.897 INFO:tasks.workunit.client.0.smithi136.stdout:started aio compare and write 2024-09-13T00:34:39.897 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:39.901 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:39.901 INFO:tasks.workunit.client.0.smithi136.stdout:finished aio compare and write 2024-09-13T00:34:39.901 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:34:39.901 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:34:39.902 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:34:39.902 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:34:39.902 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:34:39.902 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:39.902 INFO:tasks.workunit.client.0.smithi136.stdout:started read 2024-09-13T00:34:39.902 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:34:39.902 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 512 2024-09-13T00:34:39.902 INFO:tasks.workunit.client.0.smithi136.stdout:finished read 2024-09-13T00:34:39.902 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:39.902 INFO:tasks.workunit.client.0.smithi136.stdout:started read 2024-09-13T00:34:39.902 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:34:39.903 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 512 2024-09-13T00:34:39.903 INFO:tasks.workunit.client.0.smithi136.stdout:finished read 2024-09-13T00:34:39.903 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:39.903 INFO:tasks.workunit.client.0.smithi136.stdout:started read 2024-09-13T00:34:39.903 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:34:39.903 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 512 2024-09-13T00:34:39.904 INFO:tasks.workunit.client.0.smithi136.stdout:finished read 2024-09-13T00:34:39.904 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:39.904 INFO:tasks.workunit.client.0.smithi136.stdout:started read 2024-09-13T00:34:39.904 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:34:39.904 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 512 2024-09-13T00:34:39.905 INFO:tasks.workunit.client.0.smithi136.stdout:finished read 2024-09-13T00:34:39.905 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:39.905 INFO:tasks.workunit.client.0.smithi136.stdout:started read 2024-09-13T00:34:39.905 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:34:39.905 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 512 2024-09-13T00:34:39.905 INFO:tasks.workunit.client.0.smithi136.stdout:finished read 2024-09-13T00:34:39.905 INFO:tasks.workunit.client.0.smithi136.stdout:discard: 512~512 2024-09-13T00:34:39.905 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:41.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:41 smithi136 bash[76371]: cluster 2024-09-13T00:34:40.193373+0000 mgr.y (mgr.24661) 1726 : cluster [DBG] pgmap v2992: 489 pgs: 489 active+clean; 53 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.7 MiB/s wr, 157 op/s 2024-09-13T00:34:41.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:41 smithi136 bash[75164]: cluster 2024-09-13T00:34:40.193373+0000 mgr.y (mgr.24661) 1726 : cluster [DBG] pgmap v2992: 489 pgs: 489 active+clean; 53 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.7 MiB/s wr, 157 op/s 2024-09-13T00:34:41.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:41 smithi181 bash[67587]: cluster 2024-09-13T00:34:40.193373+0000 mgr.y (mgr.24661) 1726 : cluster [DBG] pgmap v2992: 489 pgs: 489 active+clean; 53 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.7 MiB/s wr, 157 op/s 2024-09-13T00:34:42.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:34:42 smithi181 bash[95505]: ts=2024-09-13T00:34:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:34:42.692 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:34:42.692 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:34:42.692 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:34:42.692 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:34:42.693 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:34:42.693 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 0 2024-09-13T00:34:42.693 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:42.693 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:42.693 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 0 2024-09-13T00:34:42.693 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:42.693 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:42.693 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 16384 2024-09-13T00:34:42.693 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:42.693 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:42.693 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 16384 2024-09-13T00:34:42.693 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:42.693 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:42.693 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:42.694 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:42.694 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 49152 2024-09-13T00:34:42.694 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:42.694 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:42.694 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 49152 2024-09-13T00:34:42.694 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:42.694 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:42.694 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 65536 2024-09-13T00:34:42.694 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:42.694 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:42.694 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 65536 2024-09-13T00:34:42.694 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:42.695 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:42.695 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:42.695 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:42.695 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 98304 2024-09-13T00:34:42.695 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:42.695 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:42.695 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 98304 2024-09-13T00:34:42.695 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:42.695 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:42.695 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 114688 2024-09-13T00:34:42.695 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:42.695 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:42.695 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 114688 2024-09-13T00:34:42.695 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:42.696 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:42.696 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:42.696 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:42.696 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 147456 2024-09-13T00:34:42.696 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:42.696 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:42.696 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 147456 2024-09-13T00:34:42.696 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:42.696 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:42.696 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 163840 2024-09-13T00:34:42.696 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:42.696 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:42.696 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 163840 2024-09-13T00:34:42.697 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:42.697 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:42.697 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:42.697 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:42.697 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 196608 2024-09-13T00:34:42.697 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:42.697 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:42.697 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 196608 2024-09-13T00:34:42.697 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:42.697 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:42.697 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 212992 2024-09-13T00:34:42.698 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:42.698 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:42.698 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 212992 2024-09-13T00:34:42.698 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:42.698 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:42.698 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:42.698 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:42.698 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:42.698 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:42.698 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:42.698 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:42.698 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:42.698 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:42.698 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:42.699 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:42.699 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:42.699 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:42.699 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:42.699 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:42.699 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:42.699 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:42.699 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:42.699 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:42.699 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:42.716 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:42.716 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:42.716 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:42.716 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:42.716 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:42.716 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:42.716 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:42.739 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:42.739 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:42.739 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:42.739 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:42.739 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:42.739 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:42.740 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:42.740 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:42.740 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:42.740 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:42.740 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:42.740 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:42.740 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:42.803 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:42.804 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:42.804 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:42.804 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:42.804 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:42.804 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:42.804 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:42.870 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:42.871 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:42.871 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:42.871 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:42.871 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:42.871 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:42.871 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:42.958 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:42.958 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:42.958 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:42.958 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:42.958 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:42.958 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:42.958 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:43.053 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:43.053 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:43.053 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:43.053 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:43.054 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:43.054 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:43.054 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:43.054 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:43.054 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:43.054 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:43.054 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:43.054 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:43.054 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:43.180 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:43.181 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:43.181 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:43.181 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:43.181 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:43.181 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:43.181 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:43.305 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:43.306 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:43.306 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:43.306 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:43.306 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:43.306 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:43.306 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:43.451 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:43.451 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:43.451 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:43.451 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:43.451 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:43.452 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:43.452 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:43.574 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:43.574 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:43.574 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:43.574 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:43.574 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:43.574 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:43.574 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:43.574 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:43.575 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:43.575 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:43.575 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:43.575 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:43.575 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:43.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:43 smithi136 bash[75164]: cluster 2024-09-13T00:34:42.196415+0000 mgr.y (mgr.24661) 1727 : cluster [DBG] pgmap v2993: 489 pgs: 489 active+clean; 83 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 7.2 MiB/s rd, 7.1 MiB/s wr, 359 op/s 2024-09-13T00:34:43.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:43 smithi136 bash[76371]: cluster 2024-09-13T00:34:42.196415+0000 mgr.y (mgr.24661) 1727 : cluster [DBG] pgmap v2993: 489 pgs: 489 active+clean; 83 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 7.2 MiB/s rd, 7.1 MiB/s wr, 359 op/s 2024-09-13T00:34:43.762 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:43.763 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:43.763 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:43.763 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:43.763 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:43.763 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:43.763 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:43.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:43 smithi181 bash[67587]: cluster 2024-09-13T00:34:42.196415+0000 mgr.y (mgr.24661) 1727 : cluster [DBG] pgmap v2993: 489 pgs: 489 active+clean; 83 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 7.2 MiB/s rd, 7.1 MiB/s wr, 359 op/s 2024-09-13T00:34:43.901 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:43.901 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:43.901 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:43.901 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:43.901 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:43.901 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:43.901 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:44.115 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:44.115 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:44.115 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:44.115 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:44.116 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:44.116 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:44.116 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:44.315 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:44.315 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:44.315 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:44.315 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:44.315 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:44.315 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:44.315 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:44.315 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:44.316 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:44.316 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:44.316 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:44.316 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:44.316 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:44.615 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:44.615 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:44.616 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:44.616 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:44.616 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:44.616 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:44.616 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:44.808 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:44.808 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:44.808 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:44.808 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:44.808 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:44.808 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:44.808 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:45.010 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:45.010 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:45.010 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:45.010 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:45.011 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:45.011 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:45.011 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:45.217 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:45.217 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:45.218 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:45.218 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:45.218 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:45.218 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:45.218 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:45.218 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:45.218 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:45.218 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:45.218 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestIOPP (5420 ms) 2024-09-13T00:34:45.218 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteCompareTooSmallPP 2024-09-13T00:34:45.218 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:34:45.285 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestCompareAndWriteCompareTooSmallPP (67 ms) 2024-09-13T00:34:45.285 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteCompareTooSmallPP 2024-09-13T00:34:45.285 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:34:45.348 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteCompareTooSmallPP (63 ms) 2024-09-13T00:34:45.348 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteWriteTooSmallPP 2024-09-13T00:34:45.348 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:34:45.413 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestCompareAndWriteWriteTooSmallPP (66 ms) 2024-09-13T00:34:45.414 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteWriteTooSmallPP 2024-09-13T00:34:45.414 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:34:45.481 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteWriteTooSmallPP (67 ms) 2024-09-13T00:34:45.481 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteMismatchPP 2024-09-13T00:34:45.481 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:34:45.546 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestCompareAndWriteMismatchPP (65 ms) 2024-09-13T00:34:45.546 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteMismatchPP 2024-09-13T00:34:45.546 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:34:45.609 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:45.619 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteMismatchPP (74 ms) 2024-09-13T00:34:45.619 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteMismatchBufferlistGreaterLenPP 2024-09-13T00:34:45.619 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:34:45.688 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestCompareAndWriteMismatchBufferlistGreaterLenPP (68 ms) 2024-09-13T00:34:45.688 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteMismatchBufferlistGreaterLenPP 2024-09-13T00:34:45.688 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:34:45.746 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:45.757 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteMismatchBufferlistGreaterLenPP (69 ms) 2024-09-13T00:34:45.757 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteSuccessPP 2024-09-13T00:34:45.757 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:34:45.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:45 smithi136 bash[75164]: cluster 2024-09-13T00:34:44.198578+0000 mgr.y (mgr.24661) 1728 : cluster [DBG] pgmap v2994: 489 pgs: 489 active+clean; 87 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 7.2 MiB/s rd, 7.0 MiB/s wr, 333 op/s 2024-09-13T00:34:45.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:45 smithi136 bash[75164]: cluster 2024-09-13T00:34:44.578215+0000 mon.a (mon.0) 3218 : cluster [DBG] osdmap e1706: 8 total, 8 up, 8 in 2024-09-13T00:34:45.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:45 smithi136 bash[76371]: cluster 2024-09-13T00:34:44.198578+0000 mgr.y (mgr.24661) 1728 : cluster [DBG] pgmap v2994: 489 pgs: 489 active+clean; 87 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 7.2 MiB/s rd, 7.0 MiB/s wr, 333 op/s 2024-09-13T00:34:45.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:45 smithi136 bash[76371]: cluster 2024-09-13T00:34:44.578215+0000 mon.a (mon.0) 3218 : cluster [DBG] osdmap e1706: 8 total, 8 up, 8 in 2024-09-13T00:34:45.828 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestCompareAndWriteSuccessPP (71 ms) 2024-09-13T00:34:45.828 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteSuccessPP 2024-09-13T00:34:45.828 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:34:45.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:45 smithi181 bash[67587]: cluster 2024-09-13T00:34:44.198578+0000 mgr.y (mgr.24661) 1728 : cluster [DBG] pgmap v2994: 489 pgs: 489 active+clean; 87 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 7.2 MiB/s rd, 7.0 MiB/s wr, 333 op/s 2024-09-13T00:34:45.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:45 smithi181 bash[67587]: cluster 2024-09-13T00:34:44.578215+0000 mon.a (mon.0) 3218 : cluster [DBG] osdmap e1706: 8 total, 8 up, 8 in 2024-09-13T00:34:45.886 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:45.896 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteSuccessPP (68 ms) 2024-09-13T00:34:45.896 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteSuccessBufferlistGreaterLenPP 2024-09-13T00:34:45.896 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:34:45.968 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestCompareAndWriteSuccessBufferlistGreaterLenPP (71 ms) 2024-09-13T00:34:45.968 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteSuccessBufferlistGreaterLenPP 2024-09-13T00:34:45.968 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:34:46.031 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:46.042 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteSuccessBufferlistGreaterLenPP (74 ms) 2024-09-13T00:34:46.042 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteStripeUnitUnalignedPP 2024-09-13T00:34:46.042 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:34:46.186 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestCompareAndWriteStripeUnitUnalignedPP (144 ms) 2024-09-13T00:34:46.186 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteStripeUnitUnalignedPP 2024-09-13T00:34:46.186 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:34:46.275 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:46.329 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteStripeUnitUnalignedPP (144 ms) 2024-09-13T00:34:46.330 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteTooLargePP 2024-09-13T00:34:46.330 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:34:46.491 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestCompareAndWriteTooLargePP (161 ms) 2024-09-13T00:34:46.491 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteTooLargePP 2024-09-13T00:34:46.491 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:34:46.597 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:46.664 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteTooLargePP (174 ms) 2024-09-13T00:34:46.664 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteStripeUnitSuccessPP 2024-09-13T00:34:46.665 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:34:47.091 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestCompareAndWriteStripeUnitSuccessPP (427 ms) 2024-09-13T00:34:47.092 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteStripeUnitSuccessPP 2024-09-13T00:34:47.092 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:34:47.546 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:47.584 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteStripeUnitSuccessPP (493 ms) 2024-09-13T00:34:47.584 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestIOPPWithIOHint 2024-09-13T00:34:47.585 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:34:47.641 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 512 2024-09-13T00:34:47.641 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 512 2024-09-13T00:34:47.642 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 512 2024-09-13T00:34:47.642 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 512 2024-09-13T00:34:47.642 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 512 2024-09-13T00:34:47.642 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:47.642 INFO:tasks.workunit.client.0.smithi136.stdout:started write 2024-09-13T00:34:47.642 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:47.644 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:47.644 INFO:tasks.workunit.client.0.smithi136.stdout:finished write 2024-09-13T00:34:47.644 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:47.644 INFO:tasks.workunit.client.0.smithi136.stdout:started write 2024-09-13T00:34:47.644 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:47.646 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:47.646 INFO:tasks.workunit.client.0.smithi136.stdout:finished write 2024-09-13T00:34:47.646 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:47.646 INFO:tasks.workunit.client.0.smithi136.stdout:started write 2024-09-13T00:34:47.647 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:47.649 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:47.649 INFO:tasks.workunit.client.0.smithi136.stdout:finished write 2024-09-13T00:34:47.649 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:47.649 INFO:tasks.workunit.client.0.smithi136.stdout:started write 2024-09-13T00:34:47.649 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:47.652 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:47.652 INFO:tasks.workunit.client.0.smithi136.stdout:finished write 2024-09-13T00:34:47.652 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:47.653 INFO:tasks.workunit.client.0.smithi136.stdout:started write 2024-09-13T00:34:47.653 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:47.654 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:47.654 INFO:tasks.workunit.client.0.smithi136.stdout:finished write 2024-09-13T00:34:47.654 INFO:tasks.workunit.client.0.smithi136.stdout:read: 512 2024-09-13T00:34:47.654 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:47.654 INFO:tasks.workunit.client.0.smithi136.stdout:started read 2024-09-13T00:34:47.655 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:34:47.655 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 512 2024-09-13T00:34:47.655 INFO:tasks.workunit.client.0.smithi136.stdout:finished read 2024-09-13T00:34:47.655 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:47.656 INFO:tasks.workunit.client.0.smithi136.stdout:started read 2024-09-13T00:34:47.656 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:34:47.656 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 512 2024-09-13T00:34:47.656 INFO:tasks.workunit.client.0.smithi136.stdout:finished read 2024-09-13T00:34:47.656 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:47.656 INFO:tasks.workunit.client.0.smithi136.stdout:started read 2024-09-13T00:34:47.656 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:34:47.657 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 512 2024-09-13T00:34:47.657 INFO:tasks.workunit.client.0.smithi136.stdout:finished read 2024-09-13T00:34:47.657 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:47.657 INFO:tasks.workunit.client.0.smithi136.stdout:started read 2024-09-13T00:34:47.657 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:34:47.657 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 512 2024-09-13T00:34:47.657 INFO:tasks.workunit.client.0.smithi136.stdout:finished read 2024-09-13T00:34:47.658 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:47.658 INFO:tasks.workunit.client.0.smithi136.stdout:started read 2024-09-13T00:34:47.658 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:34:47.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:47 smithi136 bash[75164]: cluster 2024-09-13T00:34:46.201638+0000 mgr.y (mgr.24661) 1729 : cluster [DBG] pgmap v2996: 489 pgs: 489 active+clean; 87 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 6.9 MiB/s rd, 6.6 MiB/s wr, 1.19k op/s 2024-09-13T00:34:47.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:47 smithi136 bash[76371]: cluster 2024-09-13T00:34:46.201638+0000 mgr.y (mgr.24661) 1729 : cluster [DBG] pgmap v2996: 489 pgs: 489 active+clean; 87 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 6.9 MiB/s rd, 6.6 MiB/s wr, 1.19k op/s 2024-09-13T00:34:47.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:47 smithi181 bash[67587]: cluster 2024-09-13T00:34:46.201638+0000 mgr.y (mgr.24661) 1729 : cluster [DBG] pgmap v2996: 489 pgs: 489 active+clean; 87 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 6.9 MiB/s rd, 6.6 MiB/s wr, 1.19k op/s 2024-09-13T00:34:49.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:49 smithi136 bash[75164]: cluster 2024-09-13T00:34:48.204021+0000 mgr.y (mgr.24661) 1730 : cluster [DBG] pgmap v2997: 489 pgs: 489 active+clean; 115 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 9.3 MiB/s rd, 9.0 MiB/s wr, 1.03k op/s 2024-09-13T00:34:49.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:34:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:34:49] "GET /metrics HTTP/1.1" 200 63174 "" "Prometheus/2.43.0" 2024-09-13T00:34:49.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:49 smithi136 bash[76371]: cluster 2024-09-13T00:34:48.204021+0000 mgr.y (mgr.24661) 1730 : cluster [DBG] pgmap v2997: 489 pgs: 489 active+clean; 115 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 9.3 MiB/s rd, 9.0 MiB/s wr, 1.03k op/s 2024-09-13T00:34:49.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:49 smithi181 bash[67587]: cluster 2024-09-13T00:34:48.204021+0000 mgr.y (mgr.24661) 1730 : cluster [DBG] pgmap v2997: 489 pgs: 489 active+clean; 115 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 9.3 MiB/s rd, 9.0 MiB/s wr, 1.03k op/s 2024-09-13T00:34:51.573 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 512 2024-09-13T00:34:51.573 INFO:tasks.workunit.client.0.smithi136.stdout:finished read 2024-09-13T00:34:51.573 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 0 2024-09-13T00:34:51.573 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:51.573 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:51.574 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 0 2024-09-13T00:34:51.574 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:51.574 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:51.574 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 16384 2024-09-13T00:34:51.574 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:51.574 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:51.574 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 16384 2024-09-13T00:34:51.574 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:51.574 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:51.574 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:51.574 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:51.574 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 49152 2024-09-13T00:34:51.575 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:51.575 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:51.575 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 49152 2024-09-13T00:34:51.575 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:51.575 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:51.575 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 65536 2024-09-13T00:34:51.575 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:51.575 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:51.575 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 65536 2024-09-13T00:34:51.575 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:51.575 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:51.576 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:51.576 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:51.576 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 98304 2024-09-13T00:34:51.576 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:51.576 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:51.576 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 98304 2024-09-13T00:34:51.576 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:51.576 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:51.576 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 114688 2024-09-13T00:34:51.576 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:51.576 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:51.576 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 114688 2024-09-13T00:34:51.577 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:51.577 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:51.577 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:51.577 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:51.577 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 147456 2024-09-13T00:34:51.577 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:51.577 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:51.577 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 147456 2024-09-13T00:34:51.577 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:51.577 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:51.578 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 163840 2024-09-13T00:34:51.578 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:51.578 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:51.578 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 163840 2024-09-13T00:34:51.578 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:51.578 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:51.578 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:51.578 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:51.578 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 196608 2024-09-13T00:34:51.578 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:51.578 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:51.578 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 196608 2024-09-13T00:34:51.579 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:51.579 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:51.579 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 212992 2024-09-13T00:34:51.579 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:51.579 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:51.579 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 212992 2024-09-13T00:34:51.579 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:51.579 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:51.579 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:51.579 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:51.579 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:51.579 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:51.580 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:51.580 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:51.580 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:51.580 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:51.580 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:51.580 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:51.580 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:51.580 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:51.580 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:51.581 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:51.581 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:51.581 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:51.581 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:51.581 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:51.581 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:51.608 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:51.608 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:51.608 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:51.608 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:51.609 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:51.609 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:51.609 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:51.640 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:51.640 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:51.640 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:51.641 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:51.641 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:51.641 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:51.641 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:51.641 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:51.641 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:51.641 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:51.641 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:51.641 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:51.641 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:51.728 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:51.728 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:51.729 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:51.729 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:51.729 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:51.729 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:51.729 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:51.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:51 smithi136 bash[75164]: cluster 2024-09-13T00:34:50.205342+0000 mgr.y (mgr.24661) 1731 : cluster [DBG] pgmap v2998: 489 pgs: 489 active+clean; 127 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 9.0 MiB/s rd, 8.7 MiB/s wr, 1.24k op/s 2024-09-13T00:34:51.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:51 smithi136 bash[76371]: cluster 2024-09-13T00:34:50.205342+0000 mgr.y (mgr.24661) 1731 : cluster [DBG] pgmap v2998: 489 pgs: 489 active+clean; 127 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 9.0 MiB/s rd, 8.7 MiB/s wr, 1.24k op/s 2024-09-13T00:34:51.803 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:51.803 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:51.803 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:51.803 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:51.803 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:51.804 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:51.804 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:51.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:51 smithi181 bash[67587]: cluster 2024-09-13T00:34:50.205342+0000 mgr.y (mgr.24661) 1731 : cluster [DBG] pgmap v2998: 489 pgs: 489 active+clean; 127 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 9.0 MiB/s rd, 8.7 MiB/s wr, 1.24k op/s 2024-09-13T00:34:51.926 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:51.926 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:51.926 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:51.926 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:51.926 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:51.926 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:51.926 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:52.050 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:52.050 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:52.050 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:52.050 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:52.050 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:52.050 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:52.050 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:52.051 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:52.051 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:52.051 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:52.051 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:52.051 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:52.051 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:52.237 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:52.237 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:52.237 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:52.237 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:52.237 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:52.237 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:52.238 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:52.420 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:52.420 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:52.420 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:52.421 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:52.421 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:52.421 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:52.421 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:52.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:34:52 smithi181 bash[95505]: ts=2024-09-13T00:34:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:34:52.654 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:52.654 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:52.655 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:52.655 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:52.655 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:52.655 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:52.655 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:52.841 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:52.841 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:52.841 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:52.841 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:52.841 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:52.841 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:52.841 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:52.841 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:52.842 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:52.842 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:52.842 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:52.842 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:52.842 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:53.118 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:53.119 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:53.119 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:53.119 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:53.119 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:53.119 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:53.119 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:53.386 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:53.387 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:53.387 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:53.387 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:53.387 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:53.387 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:53.387 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:53.676 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:53.676 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:53.676 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:53.676 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:53.676 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:53.676 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:53.676 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:53.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:53 smithi136 bash[75164]: cluster 2024-09-13T00:34:52.208305+0000 mgr.y (mgr.24661) 1732 : cluster [DBG] pgmap v2999: 489 pgs: 489 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 7.0 MiB/s rd, 6.7 MiB/s wr, 1.11k op/s 2024-09-13T00:34:53.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:53 smithi136 bash[76371]: cluster 2024-09-13T00:34:52.208305+0000 mgr.y (mgr.24661) 1732 : cluster [DBG] pgmap v2999: 489 pgs: 489 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 7.0 MiB/s rd, 6.7 MiB/s wr, 1.11k op/s 2024-09-13T00:34:53.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:53 smithi181 bash[67587]: cluster 2024-09-13T00:34:52.208305+0000 mgr.y (mgr.24661) 1732 : cluster [DBG] pgmap v2999: 489 pgs: 489 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 7.0 MiB/s rd, 6.7 MiB/s wr, 1.11k op/s 2024-09-13T00:34:53.951 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:53.952 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:53.952 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:53.952 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:53.952 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:53.952 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:53.953 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:53.953 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:53.953 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:53.953 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:53.953 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:53.953 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:53.953 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:54.321 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:54.321 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:54.321 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:54.322 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:54.322 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:54.322 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:54.322 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:54.675 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:54.676 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:54.676 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:54.676 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:54.676 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:54.676 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:54.676 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:54.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:54 smithi136 bash[75164]: audit 2024-09-13T00:34:53.829770+0000 mon.a (mon.0) 3219 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:34:54.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:54 smithi136 bash[75164]: cluster 2024-09-13T00:34:54.210294+0000 mgr.y (mgr.24661) 1733 : cluster [DBG] pgmap v3000: 489 pgs: 489 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 6.1 MiB/s rd, 5.9 MiB/s wr, 1.09k op/s 2024-09-13T00:34:54.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:54 smithi136 bash[76371]: audit 2024-09-13T00:34:53.829770+0000 mon.a (mon.0) 3219 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:34:54.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:54 smithi136 bash[76371]: cluster 2024-09-13T00:34:54.210294+0000 mgr.y (mgr.24661) 1733 : cluster [DBG] pgmap v3000: 489 pgs: 489 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 6.1 MiB/s rd, 5.9 MiB/s wr, 1.09k op/s 2024-09-13T00:34:54.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:54 smithi181 bash[67587]: audit 2024-09-13T00:34:53.829770+0000 mon.a (mon.0) 3219 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:34:54.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:54 smithi181 bash[67587]: cluster 2024-09-13T00:34:54.210294+0000 mgr.y (mgr.24661) 1733 : cluster [DBG] pgmap v3000: 489 pgs: 489 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 6.1 MiB/s rd, 5.9 MiB/s wr, 1.09k op/s 2024-09-13T00:34:55.059 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:55.059 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:55.059 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:55.059 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:55.059 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:55.059 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:55.059 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:34:55.422 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:34:55.422 INFO:tasks.workunit.client.0.smithi136.stdout:finished writesame 2024-09-13T00:34:55.422 INFO:tasks.workunit.client.0.smithi136.stdout:to verify the data 2024-09-13T00:34:55.422 INFO:tasks.workunit.client.0.smithi136.stdout:verified 2024-09-13T00:34:55.422 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:55.422 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:55.422 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:55.422 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:34:55.422 INFO:tasks.workunit.client.0.smithi136.stdout:started writesame 2024-09-13T00:34:55.422 INFO:tasks.workunit.client.0.smithi136.stdout:expected fail, finished writesame 2024-09-13T00:34:55.422 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestIOPPWithIOHint (7837 ms) 2024-09-13T00:34:55.423 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestIOToSnapshot 2024-09-13T00:34:55.423 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:34:56.617 INFO:tasks.workunit.client.0.smithi136.stdout:num snaps is: 0 2024-09-13T00:34:56.618 INFO:tasks.workunit.client.0.smithi136.stdout:expected: 0 2024-09-13T00:34:56.618 INFO:tasks.workunit.client.0.smithi136.stdout:num snaps is: 1 2024-09-13T00:34:56.618 INFO:tasks.workunit.client.0.smithi136.stdout:expected: 1 2024-09-13T00:34:56.618 INFO:tasks.workunit.client.0.smithi136.stdout:snap: orig 2024-09-13T00:34:56.618 INFO:tasks.workunit.client.0.smithi136.stdout:found orig with size 2097152 2024-09-13T00:34:56.618 INFO:tasks.workunit.client.0.smithi136.stdout:read: 80 2024-09-13T00:34:56.618 INFO:tasks.workunit.client.0.smithi136.stdout:write test data! 2024-09-13T00:34:56.618 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 80 2024-09-13T00:34:56.618 INFO:tasks.workunit.client.0.smithi136.stdout:num snaps is: 2 2024-09-13T00:34:56.618 INFO:tasks.workunit.client.0.smithi136.stdout:expected: 2 2024-09-13T00:34:56.618 INFO:tasks.workunit.client.0.smithi136.stdout:snap: orig 2024-09-13T00:34:56.618 INFO:tasks.workunit.client.0.smithi136.stdout:snap: written 2024-09-13T00:34:56.619 INFO:tasks.workunit.client.0.smithi136.stdout:found orig with size 2097152 2024-09-13T00:34:56.619 INFO:tasks.workunit.client.0.smithi136.stdout:found written with size 2097152 2024-09-13T00:34:56.619 INFO:tasks.workunit.client.0.smithi136.stdout:read: 80 2024-09-13T00:34:56.619 INFO:tasks.workunit.client.0.smithi136.stdout:read: 80 2024-09-13T00:34:56.619 INFO:tasks.workunit.client.0.smithi136.stdout:read: 80 2024-09-13T00:34:56.619 INFO:tasks.workunit.client.0.smithi136.stdout:write to snapshot returned -30 2024-09-13T00:34:56.619 INFO:tasks.workunit.client.0.smithi136.stdout:Read-only file system 2024-09-13T00:34:56.685 INFO:tasks.workunit.client.0.smithi136.stdout:read: 80 2024-09-13T00:34:56.686 INFO:tasks.workunit.client.0.smithi136.stdout:read: 80 2024-09-13T00:34:56.686 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 80 2024-09-13T00:34:56.686 INFO:tasks.workunit.client.0.smithi136.stdout:opening testimg@orig 2024-09-13T00:34:56.686 INFO:tasks.workunit.client.0.smithi136.stdout:read: 80 2024-09-13T00:34:56.686 INFO:tasks.workunit.client.0.smithi136.stdout:write to snapshot returned -30 2024-09-13T00:34:56.686 INFO:tasks.workunit.client.0.smithi136.stdout:Read-only file system 2024-09-13T00:34:56.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:56 smithi181 bash[67587]: cluster 2024-09-13T00:34:55.540192+0000 mon.a (mon.0) 3220 : cluster [DBG] osdmap e1707: 8 total, 8 up, 8 in 2024-09-13T00:34:56.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:56 smithi181 bash[67587]: cluster 2024-09-13T00:34:56.212222+0000 mgr.y (mgr.24661) 1734 : cluster [DBG] pgmap v3002: 489 pgs: 489 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 6.0 MiB/s rd, 5.8 MiB/s wr, 854 op/s 2024-09-13T00:34:57.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:56 smithi136 bash[75164]: cluster 2024-09-13T00:34:55.540192+0000 mon.a (mon.0) 3220 : cluster [DBG] osdmap e1707: 8 total, 8 up, 8 in 2024-09-13T00:34:57.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:56 smithi136 bash[75164]: cluster 2024-09-13T00:34:56.212222+0000 mgr.y (mgr.24661) 1734 : cluster [DBG] pgmap v3002: 489 pgs: 489 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 6.0 MiB/s rd, 5.8 MiB/s wr, 854 op/s 2024-09-13T00:34:57.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:56 smithi136 bash[76371]: cluster 2024-09-13T00:34:55.540192+0000 mon.a (mon.0) 3220 : cluster [DBG] osdmap e1707: 8 total, 8 up, 8 in 2024-09-13T00:34:57.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:56 smithi136 bash[76371]: cluster 2024-09-13T00:34:56.212222+0000 mgr.y (mgr.24661) 1734 : cluster [DBG] pgmap v3002: 489 pgs: 489 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 6.0 MiB/s rd, 5.8 MiB/s wr, 854 op/s 2024-09-13T00:34:57.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:57 smithi181 bash[67587]: cluster 2024-09-13T00:34:56.552145+0000 mon.a (mon.0) 3221 : cluster [DBG] osdmap e1708: 8 total, 8 up, 8 in 2024-09-13T00:34:58.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:57 smithi136 bash[75164]: cluster 2024-09-13T00:34:56.552145+0000 mon.a (mon.0) 3221 : cluster [DBG] osdmap e1708: 8 total, 8 up, 8 in 2024-09-13T00:34:58.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:57 smithi136 bash[76371]: cluster 2024-09-13T00:34:56.552145+0000 mon.a (mon.0) 3221 : cluster [DBG] osdmap e1708: 8 total, 8 up, 8 in 2024-09-13T00:34:58.650 INFO:tasks.workunit.client.0.smithi136.stdout:num snaps is: 2 2024-09-13T00:34:58.650 INFO:tasks.workunit.client.0.smithi136.stdout:expected: 2 2024-09-13T00:34:58.650 INFO:tasks.workunit.client.0.smithi136.stdout:snap: orig 2024-09-13T00:34:58.650 INFO:tasks.workunit.client.0.smithi136.stdout:snap: written 2024-09-13T00:34:58.650 INFO:tasks.workunit.client.0.smithi136.stdout:found orig with size 2097152 2024-09-13T00:34:58.650 INFO:tasks.workunit.client.0.smithi136.stdout:found written with size 2097152 2024-09-13T00:34:58.650 INFO:tasks.workunit.client.0.smithi136.stdout:num snaps is: 1 2024-09-13T00:34:58.651 INFO:tasks.workunit.client.0.smithi136.stdout:expected: 1 2024-09-13T00:34:58.651 INFO:tasks.workunit.client.0.smithi136.stdout:snap: orig 2024-09-13T00:34:58.651 INFO:tasks.workunit.client.0.smithi136.stdout:found orig with size 2097152 2024-09-13T00:34:58.651 INFO:tasks.workunit.client.0.smithi136.stdout:num snaps is: 0 2024-09-13T00:34:58.651 INFO:tasks.workunit.client.0.smithi136.stdout:expected: 0 2024-09-13T00:34:58.651 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestIOToSnapshot (3228 ms) 2024-09-13T00:34:58.651 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestSnapshotDeletedIo 2024-09-13T00:34:58.651 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:34:58.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:58 smithi181 bash[67587]: cluster 2024-09-13T00:34:57.552069+0000 mon.a (mon.0) 3222 : cluster [DBG] osdmap e1709: 8 total, 8 up, 8 in 2024-09-13T00:34:58.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:58 smithi181 bash[67587]: cluster 2024-09-13T00:34:58.213250+0000 mgr.y (mgr.24661) 1735 : cluster [DBG] pgmap v3005: 489 pgs: 489 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 465 KiB/s rd, 208 KiB/s wr, 917 op/s 2024-09-13T00:34:59.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:58 smithi136 bash[75164]: cluster 2024-09-13T00:34:57.552069+0000 mon.a (mon.0) 3222 : cluster [DBG] osdmap e1709: 8 total, 8 up, 8 in 2024-09-13T00:34:59.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:58 smithi136 bash[75164]: cluster 2024-09-13T00:34:58.213250+0000 mgr.y (mgr.24661) 1735 : cluster [DBG] pgmap v3005: 489 pgs: 489 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 465 KiB/s rd, 208 KiB/s wr, 917 op/s 2024-09-13T00:34:59.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:58 smithi136 bash[76371]: cluster 2024-09-13T00:34:57.552069+0000 mon.a (mon.0) 3222 : cluster [DBG] osdmap e1709: 8 total, 8 up, 8 in 2024-09-13T00:34:59.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:58 smithi136 bash[76371]: cluster 2024-09-13T00:34:58.213250+0000 mgr.y (mgr.24661) 1735 : cluster [DBG] pgmap v3005: 489 pgs: 489 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 465 KiB/s rd, 208 KiB/s wr, 917 op/s 2024-09-13T00:34:59.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:34:59 smithi136 bash[75164]: cluster 2024-09-13T00:34:58.588426+0000 mon.a (mon.0) 3223 : cluster [DBG] osdmap e1710: 8 total, 8 up, 8 in 2024-09-13T00:34:59.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:34:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:34:59] "GET /metrics HTTP/1.1" 200 63184 "" "Prometheus/2.43.0" 2024-09-13T00:34:59.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:34:59 smithi136 bash[76371]: cluster 2024-09-13T00:34:58.588426+0000 mon.a (mon.0) 3223 : cluster [DBG] osdmap e1710: 8 total, 8 up, 8 in 2024-09-13T00:34:59.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:34:59 smithi181 bash[67587]: cluster 2024-09-13T00:34:58.588426+0000 mon.a (mon.0) 3223 : cluster [DBG] osdmap e1710: 8 total, 8 up, 8 in 2024-09-13T00:35:00.663 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestSnapshotDeletedIo (2014 ms) 2024-09-13T00:35:00.663 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestClone 2024-09-13T00:35:00.664 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:35:01.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:00 smithi136 bash[75164]: cluster 2024-09-13T00:34:59.594638+0000 mon.a (mon.0) 3224 : cluster [DBG] osdmap e1711: 8 total, 8 up, 8 in 2024-09-13T00:35:01.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:00 smithi136 bash[75164]: cluster 2024-09-13T00:35:00.214242+0000 mgr.y (mgr.24661) 1736 : cluster [DBG] pgmap v3008: 489 pgs: 489 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 113 KiB/s rd, 55 KiB/s wr, 259 op/s 2024-09-13T00:35:01.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:00 smithi136 bash[76371]: cluster 2024-09-13T00:34:59.594638+0000 mon.a (mon.0) 3224 : cluster [DBG] osdmap e1711: 8 total, 8 up, 8 in 2024-09-13T00:35:01.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:00 smithi136 bash[76371]: cluster 2024-09-13T00:35:00.214242+0000 mgr.y (mgr.24661) 1736 : cluster [DBG] pgmap v3008: 489 pgs: 489 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 113 KiB/s rd, 55 KiB/s wr, 259 op/s 2024-09-13T00:35:01.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:00 smithi181 bash[67587]: cluster 2024-09-13T00:34:59.594638+0000 mon.a (mon.0) 3224 : cluster [DBG] osdmap e1711: 8 total, 8 up, 8 in 2024-09-13T00:35:01.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:00 smithi181 bash[67587]: cluster 2024-09-13T00:35:00.214242+0000 mgr.y (mgr.24661) 1736 : cluster [DBG] pgmap v3008: 489 pgs: 489 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 113 KiB/s rd, 55 KiB/s wr, 259 op/s 2024-09-13T00:35:01.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:01 smithi181 bash[67587]: cluster 2024-09-13T00:35:00.604353+0000 mon.a (mon.0) 3225 : cluster [DBG] osdmap e1712: 8 total, 8 up, 8 in 2024-09-13T00:35:02.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:01 smithi136 bash[75164]: cluster 2024-09-13T00:35:00.604353+0000 mon.a (mon.0) 3225 : cluster [DBG] osdmap e1712: 8 total, 8 up, 8 in 2024-09-13T00:35:02.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:01 smithi136 bash[76371]: cluster 2024-09-13T00:35:00.604353+0000 mon.a (mon.0) 3225 : cluster [DBG] osdmap e1712: 8 total, 8 up, 8 in 2024-09-13T00:35:02.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:35:02 smithi181 bash[95505]: ts=2024-09-13T00:35:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:35:02.680 INFO:tasks.workunit.client.0.smithi136.stdout:made parent image "parent" 2024-09-13T00:35:02.681 INFO:tasks.workunit.client.0.smithi136.stdout:parent has no parent info 2024-09-13T00:35:02.681 INFO:tasks.workunit.client.0.smithi136.stdout:made snapshot "parent@parent_snap" 2024-09-13T00:35:02.681 INFO:tasks.workunit.client.0.smithi136.stdout:can't unprotect an unprotected snap 2024-09-13T00:35:02.681 INFO:tasks.workunit.client.0.smithi136.stdout:can't protect a protected snap 2024-09-13T00:35:02.681 INFO:tasks.workunit.client.0.smithi136.stdout:made and opened clone "child" 2024-09-13T00:35:02.681 INFO:tasks.workunit.client.0.smithi136.stdout:read: 8 2024-09-13T00:35:02.682 INFO:tasks.workunit.client.0.smithi136.stdout:read: 8 2024-09-13T00:35:02.682 INFO:tasks.workunit.client.0.smithi136.stdout:read: 8 2024-09-13T00:35:02.682 INFO:tasks.workunit.client.0.smithi136.stdout:sizes and overlaps are good between parent and child 2024-09-13T00:35:02.682 INFO:tasks.workunit.client.0.smithi136.stdout:child image successfully cloned all image-meta pairs 2024-09-13T00:35:02.682 INFO:tasks.workunit.client.0.smithi136.stdout:sized down clone, changed overlap 2024-09-13T00:35:02.683 INFO:tasks.workunit.client.0.smithi136.stdout:parent info: size 4194304 obj_size 4194304 parent_pool 18446744073709551615 2024-09-13T00:35:02.683 INFO:tasks.workunit.client.0.smithi136.stdout:sized up clone, changed size but not overlap or parent's size 2024-09-13T00:35:02.683 INFO:tasks.workunit.client.0.smithi136.stdout:can't remove parent while child still exists 2024-09-13T00:35:02.683 INFO:tasks.workunit.client.0.smithi136.stdout:can't remove parent while still protected 2024-09-13T00:35:02.683 INFO:tasks.workunit.client.0.smithi136.stdout:removed parent snap after unprotecting 2024-09-13T00:35:02.683 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestClone (2016 ms) 2024-09-13T00:35:02.684 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestClone2 2024-09-13T00:35:02.684 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:35:03.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:02 smithi136 bash[76371]: cluster 2024-09-13T00:35:01.609240+0000 mon.a (mon.0) 3226 : cluster [DBG] osdmap e1713: 8 total, 8 up, 8 in 2024-09-13T00:35:03.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:02 smithi136 bash[76371]: cluster 2024-09-13T00:35:02.215767+0000 mgr.y (mgr.24661) 1737 : cluster [DBG] pgmap v3011: 489 pgs: 489 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 147 KiB/s rd, 56 KiB/s wr, 303 op/s 2024-09-13T00:35:03.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:02 smithi136 bash[75164]: cluster 2024-09-13T00:35:01.609240+0000 mon.a (mon.0) 3226 : cluster [DBG] osdmap e1713: 8 total, 8 up, 8 in 2024-09-13T00:35:03.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:02 smithi136 bash[75164]: cluster 2024-09-13T00:35:02.215767+0000 mgr.y (mgr.24661) 1737 : cluster [DBG] pgmap v3011: 489 pgs: 489 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 147 KiB/s rd, 56 KiB/s wr, 303 op/s 2024-09-13T00:35:03.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:02 smithi181 bash[67587]: cluster 2024-09-13T00:35:01.609240+0000 mon.a (mon.0) 3226 : cluster [DBG] osdmap e1713: 8 total, 8 up, 8 in 2024-09-13T00:35:03.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:02 smithi181 bash[67587]: cluster 2024-09-13T00:35:02.215767+0000 mgr.y (mgr.24661) 1737 : cluster [DBG] pgmap v3011: 489 pgs: 489 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 147 KiB/s rd, 56 KiB/s wr, 303 op/s 2024-09-13T00:35:03.964 INFO:tasks.workunit.client.0.smithi136.stdout:made parent image "parent" 2024-09-13T00:35:03.965 INFO:tasks.workunit.client.0.smithi136.stdout:parent has no parent info 2024-09-13T00:35:03.965 INFO:tasks.workunit.client.0.smithi136.stdout:made snapshot "parent@parent_snap" 2024-09-13T00:35:03.965 INFO:tasks.workunit.client.0.smithi136.stdout:made and opened clone "child" 2024-09-13T00:35:03.965 INFO:tasks.workunit.client.0.smithi136.stdout:child image successfully cloned all image-meta pairs 2024-09-13T00:35:03.965 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestClone2 (1285 ms) 2024-09-13T00:35:03.965 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.ListChildren 2024-09-13T00:35:04.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:03 smithi136 bash[75164]: cluster 2024-09-13T00:35:02.638181+0000 mon.a (mon.0) 3227 : cluster [DBG] osdmap e1714: 8 total, 8 up, 8 in 2024-09-13T00:35:04.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:03 smithi136 bash[76371]: cluster 2024-09-13T00:35:02.638181+0000 mon.a (mon.0) 3227 : cluster [DBG] osdmap e1714: 8 total, 8 up, 8 in 2024-09-13T00:35:04.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:03 smithi181 bash[67587]: cluster 2024-09-13T00:35:02.638181+0000 mon.a (mon.0) 3227 : cluster [DBG] osdmap e1714: 8 total, 8 up, 8 in 2024-09-13T00:35:05.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:04 smithi136 bash[75164]: cluster 2024-09-13T00:35:03.639396+0000 mon.a (mon.0) 3228 : cluster [DBG] osdmap e1715: 8 total, 8 up, 8 in 2024-09-13T00:35:05.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:04 smithi136 bash[75164]: cluster 2024-09-13T00:35:04.216756+0000 mgr.y (mgr.24661) 1738 : cluster [DBG] pgmap v3014: 489 pgs: 489 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 644 KiB/s rd, 47 KiB/s wr, 777 op/s 2024-09-13T00:35:05.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:04 smithi136 bash[76371]: cluster 2024-09-13T00:35:03.639396+0000 mon.a (mon.0) 3228 : cluster [DBG] osdmap e1715: 8 total, 8 up, 8 in 2024-09-13T00:35:05.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:04 smithi136 bash[76371]: cluster 2024-09-13T00:35:04.216756+0000 mgr.y (mgr.24661) 1738 : cluster [DBG] pgmap v3014: 489 pgs: 489 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 644 KiB/s rd, 47 KiB/s wr, 777 op/s 2024-09-13T00:35:05.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:04 smithi181 bash[67587]: cluster 2024-09-13T00:35:03.639396+0000 mon.a (mon.0) 3228 : cluster [DBG] osdmap e1715: 8 total, 8 up, 8 in 2024-09-13T00:35:05.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:04 smithi181 bash[67587]: cluster 2024-09-13T00:35:04.216756+0000 mgr.y (mgr.24661) 1738 : cluster [DBG] pgmap v3014: 489 pgs: 489 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 644 KiB/s rd, 47 KiB/s wr, 777 op/s 2024-09-13T00:35:06.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:05 smithi136 bash[75164]: cluster 2024-09-13T00:35:04.669475+0000 mon.a (mon.0) 3229 : cluster [DBG] osdmap e1716: 8 total, 8 up, 8 in 2024-09-13T00:35:06.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:05 smithi136 bash[75164]: audit 2024-09-13T00:35:04.674430+0000 mon.c (mon.2) 129 : audit [INF] from='client.? 172.21.15.136:0/3682926904' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-26","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:35:06.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:05 smithi136 bash[75164]: audit 2024-09-13T00:35:04.675352+0000 mon.a (mon.0) 3230 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-26","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:35:06.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:05 smithi136 bash[76371]: cluster 2024-09-13T00:35:04.669475+0000 mon.a (mon.0) 3229 : cluster [DBG] osdmap e1716: 8 total, 8 up, 8 in 2024-09-13T00:35:06.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:05 smithi136 bash[76371]: audit 2024-09-13T00:35:04.674430+0000 mon.c (mon.2) 129 : audit [INF] from='client.? 172.21.15.136:0/3682926904' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-26","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:35:06.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:05 smithi136 bash[76371]: audit 2024-09-13T00:35:04.675352+0000 mon.a (mon.0) 3230 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-26","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:35:06.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:05 smithi181 bash[67587]: cluster 2024-09-13T00:35:04.669475+0000 mon.a (mon.0) 3229 : cluster [DBG] osdmap e1716: 8 total, 8 up, 8 in 2024-09-13T00:35:06.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:05 smithi181 bash[67587]: audit 2024-09-13T00:35:04.674430+0000 mon.c (mon.2) 129 : audit [INF] from='client.? 172.21.15.136:0/3682926904' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-26","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:35:06.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:05 smithi181 bash[67587]: audit 2024-09-13T00:35:04.675352+0000 mon.a (mon.0) 3230 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-26","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:35:07.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:06 smithi136 bash[76371]: audit 2024-09-13T00:35:05.669781+0000 mon.a (mon.0) 3231 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-26","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:35:07.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:06 smithi136 bash[76371]: cluster 2024-09-13T00:35:05.670276+0000 mon.a (mon.0) 3232 : cluster [DBG] osdmap e1717: 8 total, 8 up, 8 in 2024-09-13T00:35:07.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:06 smithi136 bash[76371]: cluster 2024-09-13T00:35:06.218116+0000 mgr.y (mgr.24661) 1739 : cluster [DBG] pgmap v3017: 521 pgs: 5 creating+peering, 23 unknown, 493 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 718 KiB/s rd, 50 KiB/s wr, 855 op/s 2024-09-13T00:35:07.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:06 smithi136 bash[75164]: audit 2024-09-13T00:35:05.669781+0000 mon.a (mon.0) 3231 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-26","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:35:07.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:06 smithi136 bash[75164]: cluster 2024-09-13T00:35:05.670276+0000 mon.a (mon.0) 3232 : cluster [DBG] osdmap e1717: 8 total, 8 up, 8 in 2024-09-13T00:35:07.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:06 smithi136 bash[75164]: cluster 2024-09-13T00:35:06.218116+0000 mgr.y (mgr.24661) 1739 : cluster [DBG] pgmap v3017: 521 pgs: 5 creating+peering, 23 unknown, 493 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 718 KiB/s rd, 50 KiB/s wr, 855 op/s 2024-09-13T00:35:07.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:06 smithi181 bash[67587]: audit 2024-09-13T00:35:05.669781+0000 mon.a (mon.0) 3231 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-26","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:35:07.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:06 smithi181 bash[67587]: cluster 2024-09-13T00:35:05.670276+0000 mon.a (mon.0) 3232 : cluster [DBG] osdmap e1717: 8 total, 8 up, 8 in 2024-09-13T00:35:07.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:06 smithi181 bash[67587]: cluster 2024-09-13T00:35:06.218116+0000 mgr.y (mgr.24661) 1739 : cluster [DBG] pgmap v3017: 521 pgs: 5 creating+peering, 23 unknown, 493 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 718 KiB/s rd, 50 KiB/s wr, 855 op/s 2024-09-13T00:35:07.725 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:35:08.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:07 smithi136 bash[75164]: cluster 2024-09-13T00:35:06.699067+0000 mon.a (mon.0) 3233 : cluster [DBG] osdmap e1718: 8 total, 8 up, 8 in 2024-09-13T00:35:08.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:07 smithi136 bash[75164]: audit 2024-09-13T00:35:06.701170+0000 mon.c (mon.2) 130 : audit [INF] from='client.? 172.21.15.136:0/1073219475' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-27","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:35:08.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:07 smithi136 bash[75164]: audit 2024-09-13T00:35:06.702861+0000 mon.a (mon.0) 3234 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-27","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:35:08.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:07 smithi136 bash[76371]: cluster 2024-09-13T00:35:06.699067+0000 mon.a (mon.0) 3233 : cluster [DBG] osdmap e1718: 8 total, 8 up, 8 in 2024-09-13T00:35:08.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:07 smithi136 bash[76371]: audit 2024-09-13T00:35:06.701170+0000 mon.c (mon.2) 130 : audit [INF] from='client.? 172.21.15.136:0/1073219475' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-27","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:35:08.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:07 smithi136 bash[76371]: audit 2024-09-13T00:35:06.702861+0000 mon.a (mon.0) 3234 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-27","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:35:08.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:07 smithi181 bash[67587]: cluster 2024-09-13T00:35:06.699067+0000 mon.a (mon.0) 3233 : cluster [DBG] osdmap e1718: 8 total, 8 up, 8 in 2024-09-13T00:35:08.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:07 smithi181 bash[67587]: audit 2024-09-13T00:35:06.701170+0000 mon.c (mon.2) 130 : audit [INF] from='client.? 172.21.15.136:0/1073219475' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-27","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:35:08.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:07 smithi181 bash[67587]: audit 2024-09-13T00:35:06.702861+0000 mon.a (mon.0) 3234 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-27","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:35:09.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:08 smithi136 bash[76371]: audit 2024-09-13T00:35:07.702447+0000 mon.a (mon.0) 3235 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-27","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:35:09.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:08 smithi136 bash[76371]: cluster 2024-09-13T00:35:07.703037+0000 mon.a (mon.0) 3236 : cluster [DBG] osdmap e1719: 8 total, 8 up, 8 in 2024-09-13T00:35:09.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:08 smithi136 bash[76371]: cluster 2024-09-13T00:35:08.219249+0000 mgr.y (mgr.24661) 1740 : cluster [DBG] pgmap v3020: 553 pgs: 15 creating+peering, 28 unknown, 510 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 154 KiB/s rd, 9.7 KiB/s wr, 185 op/s 2024-09-13T00:35:09.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:08 smithi136 bash[75164]: audit 2024-09-13T00:35:07.702447+0000 mon.a (mon.0) 3235 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-27","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:35:09.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:08 smithi136 bash[75164]: cluster 2024-09-13T00:35:07.703037+0000 mon.a (mon.0) 3236 : cluster [DBG] osdmap e1719: 8 total, 8 up, 8 in 2024-09-13T00:35:09.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:08 smithi136 bash[75164]: cluster 2024-09-13T00:35:08.219249+0000 mgr.y (mgr.24661) 1740 : cluster [DBG] pgmap v3020: 553 pgs: 15 creating+peering, 28 unknown, 510 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 154 KiB/s rd, 9.7 KiB/s wr, 185 op/s 2024-09-13T00:35:09.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:08 smithi181 bash[67587]: audit 2024-09-13T00:35:07.702447+0000 mon.a (mon.0) 3235 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-27","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:35:09.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:08 smithi181 bash[67587]: cluster 2024-09-13T00:35:07.703037+0000 mon.a (mon.0) 3236 : cluster [DBG] osdmap e1719: 8 total, 8 up, 8 in 2024-09-13T00:35:09.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:08 smithi181 bash[67587]: cluster 2024-09-13T00:35:08.219249+0000 mgr.y (mgr.24661) 1740 : cluster [DBG] pgmap v3020: 553 pgs: 15 creating+peering, 28 unknown, 510 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 154 KiB/s rd, 9.7 KiB/s wr, 185 op/s 2024-09-13T00:35:09.729 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:35:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:35:09] "GET /metrics HTTP/1.1" 200 68035 "" "Prometheus/2.43.0" 2024-09-13T00:35:10.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:09 smithi136 bash[76371]: cluster 2024-09-13T00:35:08.717418+0000 mon.a (mon.0) 3237 : cluster [DBG] osdmap e1720: 8 total, 8 up, 8 in 2024-09-13T00:35:10.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:09 smithi136 bash[76371]: audit 2024-09-13T00:35:08.830242+0000 mon.a (mon.0) 3238 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:35:10.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:09 smithi136 bash[76371]: cluster 2024-09-13T00:35:09.594209+0000 mon.a (mon.0) 3239 : cluster [DBG] osdmap e1721: 8 total, 8 up, 8 in 2024-09-13T00:35:10.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:09 smithi136 bash[75164]: cluster 2024-09-13T00:35:08.717418+0000 mon.a (mon.0) 3237 : cluster [DBG] osdmap e1720: 8 total, 8 up, 8 in 2024-09-13T00:35:10.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:09 smithi136 bash[75164]: audit 2024-09-13T00:35:08.830242+0000 mon.a (mon.0) 3238 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:35:10.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:09 smithi136 bash[75164]: cluster 2024-09-13T00:35:09.594209+0000 mon.a (mon.0) 3239 : cluster [DBG] osdmap e1721: 8 total, 8 up, 8 in 2024-09-13T00:35:10.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:09 smithi181 bash[67587]: cluster 2024-09-13T00:35:08.717418+0000 mon.a (mon.0) 3237 : cluster [DBG] osdmap e1720: 8 total, 8 up, 8 in 2024-09-13T00:35:10.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:09 smithi181 bash[67587]: audit 2024-09-13T00:35:08.830242+0000 mon.a (mon.0) 3238 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:35:10.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:09 smithi181 bash[67587]: cluster 2024-09-13T00:35:09.594209+0000 mon.a (mon.0) 3239 : cluster [DBG] osdmap e1721: 8 total, 8 up, 8 in 2024-09-13T00:35:11.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:10 smithi136 bash[75164]: cluster 2024-09-13T00:35:10.220283+0000 mgr.y (mgr.24661) 1741 : cluster [DBG] pgmap v3023: 553 pgs: 15 creating+peering, 17 unknown, 521 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 72 KiB/s rd, 6.2 KiB/s wr, 88 op/s 2024-09-13T00:35:11.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:10 smithi136 bash[76371]: cluster 2024-09-13T00:35:10.220283+0000 mgr.y (mgr.24661) 1741 : cluster [DBG] pgmap v3023: 553 pgs: 15 creating+peering, 17 unknown, 521 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 72 KiB/s rd, 6.2 KiB/s wr, 88 op/s 2024-09-13T00:35:11.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:10 smithi181 bash[67587]: cluster 2024-09-13T00:35:10.220283+0000 mgr.y (mgr.24661) 1741 : cluster [DBG] pgmap v3023: 553 pgs: 15 creating+peering, 17 unknown, 521 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 72 KiB/s rd, 6.2 KiB/s wr, 88 op/s 2024-09-13T00:35:12.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:11 smithi181 bash[67587]: cluster 2024-09-13T00:35:10.741314+0000 mon.a (mon.0) 3240 : cluster [DBG] osdmap e1722: 8 total, 8 up, 8 in 2024-09-13T00:35:12.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:11 smithi136 bash[75164]: cluster 2024-09-13T00:35:10.741314+0000 mon.a (mon.0) 3240 : cluster [DBG] osdmap e1722: 8 total, 8 up, 8 in 2024-09-13T00:35:12.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:11 smithi136 bash[76371]: cluster 2024-09-13T00:35:10.741314+0000 mon.a (mon.0) 3240 : cluster [DBG] osdmap e1722: 8 total, 8 up, 8 in 2024-09-13T00:35:12.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:35:12 smithi181 bash[95505]: ts=2024-09-13T00:35:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:35:13.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:12 smithi181 bash[67587]: cluster 2024-09-13T00:35:11.760693+0000 mon.a (mon.0) 3241 : cluster [DBG] osdmap e1723: 8 total, 8 up, 8 in 2024-09-13T00:35:13.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:12 smithi181 bash[67587]: cluster 2024-09-13T00:35:12.222082+0000 mgr.y (mgr.24661) 1742 : cluster [DBG] pgmap v3026: 553 pgs: 5 creating+peering, 548 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 37 KiB/s rd, 4.0 KiB/s wr, 47 op/s 2024-09-13T00:35:13.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:12 smithi136 bash[75164]: cluster 2024-09-13T00:35:11.760693+0000 mon.a (mon.0) 3241 : cluster [DBG] osdmap e1723: 8 total, 8 up, 8 in 2024-09-13T00:35:13.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:12 smithi136 bash[75164]: cluster 2024-09-13T00:35:12.222082+0000 mgr.y (mgr.24661) 1742 : cluster [DBG] pgmap v3026: 553 pgs: 5 creating+peering, 548 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 37 KiB/s rd, 4.0 KiB/s wr, 47 op/s 2024-09-13T00:35:13.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:12 smithi136 bash[76371]: cluster 2024-09-13T00:35:11.760693+0000 mon.a (mon.0) 3241 : cluster [DBG] osdmap e1723: 8 total, 8 up, 8 in 2024-09-13T00:35:13.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:12 smithi136 bash[76371]: cluster 2024-09-13T00:35:12.222082+0000 mgr.y (mgr.24661) 1742 : cluster [DBG] pgmap v3026: 553 pgs: 5 creating+peering, 548 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 37 KiB/s rd, 4.0 KiB/s wr, 47 op/s 2024-09-13T00:35:14.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:13 smithi181 bash[67587]: cluster 2024-09-13T00:35:12.790614+0000 mon.a (mon.0) 3242 : cluster [DBG] osdmap e1724: 8 total, 8 up, 8 in 2024-09-13T00:35:14.110 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:14.110 INFO:tasks.workunit.client.0.smithi136.stdout:trying to find test-librbd-smithi136-114798-27/image96 2024-09-13T00:35:14.111 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-27/image96 2024-09-13T00:35:14.111 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-27/image96 2024-09-13T00:35:14.111 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:14.111 INFO:tasks.workunit.client.0.smithi136.stdout:num children is: 1 2024-09-13T00:35:14.111 INFO:tasks.workunit.client.0.smithi136.stdout:expected: 1 2024-09-13T00:35:14.111 INFO:tasks.workunit.client.0.smithi136.stdout:child: image96 2024-09-13T00:35:14.111 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-27/image96/b0ffe535f3fc 2024-09-13T00:35:14.111 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:14.111 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:14.111 INFO:tasks.workunit.client.0.smithi136.stdout:trying to find test-librbd-smithi136-114798-27/image96 2024-09-13T00:35:14.111 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-26/image97 2024-09-13T00:35:14.111 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-27/image96 2024-09-13T00:35:14.111 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-27/image96 2024-09-13T00:35:14.111 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:14.112 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:14.112 INFO:tasks.workunit.client.0.smithi136.stdout:trying to find test-librbd-smithi136-114798-26/image97 2024-09-13T00:35:14.112 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-26/image97 2024-09-13T00:35:14.112 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-26/image97 2024-09-13T00:35:14.112 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:14.112 INFO:tasks.workunit.client.0.smithi136.stdout:num children is: 2 2024-09-13T00:35:14.112 INFO:tasks.workunit.client.0.smithi136.stdout:expected: 2 2024-09-13T00:35:14.112 INFO:tasks.workunit.client.0.smithi136.stdout:child: image97 2024-09-13T00:35:14.112 INFO:tasks.workunit.client.0.smithi136.stdout:child: image96 2024-09-13T00:35:14.112 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-27/image96/b0ffe535f3fc 2024-09-13T00:35:14.112 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:14.112 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-26/image97/b0ff5e705198 2024-09-13T00:35:14.112 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:14.112 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:14.113 INFO:tasks.workunit.client.0.smithi136.stdout:trying to find test-librbd-smithi136-114798-27/image96 2024-09-13T00:35:14.113 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-26/image97 2024-09-13T00:35:14.113 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-27/image96 2024-09-13T00:35:14.113 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-27/image96 2024-09-13T00:35:14.113 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:14.113 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:14.113 INFO:tasks.workunit.client.0.smithi136.stdout:trying to find test-librbd-smithi136-114798-26/image97 2024-09-13T00:35:14.113 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-26/image97 2024-09-13T00:35:14.113 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-26/image97 2024-09-13T00:35:14.113 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:14.113 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:14.113 INFO:tasks.workunit.client.0.smithi136.stdout:trying to find test-librbd-smithi136-114798-27/image98 2024-09-13T00:35:14.113 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-26/image97 2024-09-13T00:35:14.114 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-27/image96 2024-09-13T00:35:14.114 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-27/image98 2024-09-13T00:35:14.114 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-27/image98 2024-09-13T00:35:14.114 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:14.114 INFO:tasks.workunit.client.0.smithi136.stdout:num children is: 3 2024-09-13T00:35:14.114 INFO:tasks.workunit.client.0.smithi136.stdout:expected: 3 2024-09-13T00:35:14.114 INFO:tasks.workunit.client.0.smithi136.stdout:child: image97 2024-09-13T00:35:14.114 INFO:tasks.workunit.client.0.smithi136.stdout:child: image96 2024-09-13T00:35:14.114 INFO:tasks.workunit.client.0.smithi136.stdout:child: image98 2024-09-13T00:35:14.114 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-27/image96/b0ffe535f3fc 2024-09-13T00:35:14.114 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:14.114 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-26/image97/b0ff5e705198 2024-09-13T00:35:14.114 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:14.114 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-27/image98/b0ffa35c2ffc 2024-09-13T00:35:14.115 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:14.115 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:14.115 INFO:tasks.workunit.client.0.smithi136.stdout:trying to find test-librbd-smithi136-114798-27/image96 2024-09-13T00:35:14.115 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-26/image97 2024-09-13T00:35:14.115 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-27/image96 2024-09-13T00:35:14.115 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-27/image96 2024-09-13T00:35:14.115 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:14.115 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:14.115 INFO:tasks.workunit.client.0.smithi136.stdout:trying to find test-librbd-smithi136-114798-26/image97 2024-09-13T00:35:14.115 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-26/image97 2024-09-13T00:35:14.115 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-26/image97 2024-09-13T00:35:14.115 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:14.115 INFO:tasks.workunit.client.0.smithi136.stdout:num children is: 3 2024-09-13T00:35:14.116 INFO:tasks.workunit.client.0.smithi136.stdout:expected: 3 2024-09-13T00:35:14.116 INFO:tasks.workunit.client.0.smithi136.stdout:child: image97 2024-09-13T00:35:14.116 INFO:tasks.workunit.client.0.smithi136.stdout:child: image96 2024-09-13T00:35:14.116 INFO:tasks.workunit.client.0.smithi136.stdout:child: image98 2024-09-13T00:35:14.116 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-27/image96/b0ffe535f3fc 2024-09-13T00:35:14.116 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:14.116 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-26/image97/b0ff5e705198 2024-09-13T00:35:14.116 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:14.116 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-27/image98/b0ffa35c2ffc 2024-09-13T00:35:14.116 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:14.116 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:14.116 INFO:tasks.workunit.client.0.smithi136.stdout:trying to find test-librbd-smithi136-114798-27/image96 2024-09-13T00:35:14.116 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-26/image97 2024-09-13T00:35:14.116 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-27/image96 2024-09-13T00:35:14.116 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-27/image96 2024-09-13T00:35:14.116 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:14.117 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:14.117 INFO:tasks.workunit.client.0.smithi136.stdout:trying to find test-librbd-smithi136-114798-26/image97 2024-09-13T00:35:14.117 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-26/image97 2024-09-13T00:35:14.117 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-26/image97 2024-09-13T00:35:14.117 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:14.117 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:14.117 INFO:tasks.workunit.client.0.smithi136.stdout:trying to find test-librbd-smithi136-114798-27/image99 2024-09-13T00:35:14.117 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-26/image97 2024-09-13T00:35:14.117 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-27/image96 2024-09-13T00:35:14.117 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-27/image99 2024-09-13T00:35:14.117 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-27/image99 2024-09-13T00:35:14.117 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:14.117 INFO:tasks.workunit.client.0.smithi136.stdout:num children is: 4 2024-09-13T00:35:14.117 INFO:tasks.workunit.client.0.smithi136.stdout:expected: 4 2024-09-13T00:35:14.118 INFO:tasks.workunit.client.0.smithi136.stdout:child: image97 2024-09-13T00:35:14.118 INFO:tasks.workunit.client.0.smithi136.stdout:child: image96 2024-09-13T00:35:14.118 INFO:tasks.workunit.client.0.smithi136.stdout:child: image98 2024-09-13T00:35:14.118 INFO:tasks.workunit.client.0.smithi136.stdout:child: image99 2024-09-13T00:35:14.118 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-27/image96/b0ffe535f3fc 2024-09-13T00:35:14.118 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:14.118 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-26/image97/b0ff5e705198 2024-09-13T00:35:14.118 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:14.118 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-27/image98/b0ffa35c2ffc 2024-09-13T00:35:14.118 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:14.118 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-27/image99/b0ff52900975 2024-09-13T00:35:14.118 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:14.118 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:14.118 INFO:tasks.workunit.client.0.smithi136.stdout:trying to find test-librbd-smithi136-114798-27/image96 2024-09-13T00:35:14.119 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-26/image97 2024-09-13T00:35:14.119 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-27/image96 2024-09-13T00:35:14.119 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-27/image96 2024-09-13T00:35:14.119 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:14.119 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:14.119 INFO:tasks.workunit.client.0.smithi136.stdout:trying to find test-librbd-smithi136-114798-26/image97 2024-09-13T00:35:14.119 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-26/image97 2024-09-13T00:35:14.119 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-26/image97 2024-09-13T00:35:14.119 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:14.119 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:14.119 INFO:tasks.workunit.client.0.smithi136.stdout:trying to find test-librbd-smithi136-114798-27/image98 2024-09-13T00:35:14.119 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-26/image97 2024-09-13T00:35:14.119 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-27/image96 2024-09-13T00:35:14.119 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-27/image98 2024-09-13T00:35:14.119 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-27/image98 2024-09-13T00:35:14.119 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:14.120 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:14.120 INFO:tasks.workunit.client.0.smithi136.stdout:trying to find test-librbd-smithi136-114798-27/image99 2024-09-13T00:35:14.120 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-26/image97 2024-09-13T00:35:14.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:13 smithi136 bash[76371]: cluster 2024-09-13T00:35:12.790614+0000 mon.a (mon.0) 3242 : cluster [DBG] osdmap e1724: 8 total, 8 up, 8 in 2024-09-13T00:35:14.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:13 smithi136 bash[75164]: cluster 2024-09-13T00:35:12.790614+0000 mon.a (mon.0) 3242 : cluster [DBG] osdmap e1724: 8 total, 8 up, 8 in 2024-09-13T00:35:15.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:14 smithi181 bash[67587]: cluster 2024-09-13T00:35:14.223920+0000 mgr.y (mgr.24661) 1743 : cluster [DBG] pgmap v3028: 553 pgs: 553 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 26 KiB/s rd, 1.5 KiB/s wr, 33 op/s 2024-09-13T00:35:15.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:14 smithi136 bash[75164]: cluster 2024-09-13T00:35:14.223920+0000 mgr.y (mgr.24661) 1743 : cluster [DBG] pgmap v3028: 553 pgs: 553 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 26 KiB/s rd, 1.5 KiB/s wr, 33 op/s 2024-09-13T00:35:15.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:14 smithi136 bash[76371]: cluster 2024-09-13T00:35:14.223920+0000 mgr.y (mgr.24661) 1743 : cluster [DBG] pgmap v3028: 553 pgs: 553 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 26 KiB/s rd, 1.5 KiB/s wr, 33 op/s 2024-09-13T00:35:16.042 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-27/image96 2024-09-13T00:35:16.043 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-27/image98 2024-09-13T00:35:16.043 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-27/image99 2024-09-13T00:35:16.043 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-27/image99 2024-09-13T00:35:16.043 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:16.043 INFO:tasks.workunit.client.0.smithi136.stdout:num children is: 4 2024-09-13T00:35:16.043 INFO:tasks.workunit.client.0.smithi136.stdout:expected: 4 2024-09-13T00:35:16.043 INFO:tasks.workunit.client.0.smithi136.stdout:child: image97 2024-09-13T00:35:16.043 INFO:tasks.workunit.client.0.smithi136.stdout:child: image96 2024-09-13T00:35:16.043 INFO:tasks.workunit.client.0.smithi136.stdout:child: image98 2024-09-13T00:35:16.043 INFO:tasks.workunit.client.0.smithi136.stdout:child: image99 2024-09-13T00:35:16.043 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-27/image96/b0ffe535f3fc 2024-09-13T00:35:16.043 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:16.044 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-26/image97/b0ff5e705198 2024-09-13T00:35:16.044 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:16.044 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-27/image98/b0ffa35c2ffc 2024-09-13T00:35:16.044 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:16.044 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-27/image99/b0ff52900975 2024-09-13T00:35:16.044 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:16.044 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:16.044 INFO:tasks.workunit.client.0.smithi136.stdout:trying to find test-librbd-smithi136-114798-26/image97 2024-09-13T00:35:16.044 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-26/image97 2024-09-13T00:35:16.044 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-26/image97 2024-09-13T00:35:16.044 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:16.044 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:16.045 INFO:tasks.workunit.client.0.smithi136.stdout:trying to find test-librbd-smithi136-114798-27/image98 2024-09-13T00:35:16.045 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-26/image97 2024-09-13T00:35:16.045 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-27/image98 2024-09-13T00:35:16.045 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-27/image98 2024-09-13T00:35:16.045 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:16.045 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:16.045 INFO:tasks.workunit.client.0.smithi136.stdout:trying to find test-librbd-smithi136-114798-27/image99 2024-09-13T00:35:16.045 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-26/image97 2024-09-13T00:35:16.045 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-27/image98 2024-09-13T00:35:16.045 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-27/image99 2024-09-13T00:35:16.045 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-27/image99 2024-09-13T00:35:16.045 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:16.046 INFO:tasks.workunit.client.0.smithi136.stdout:num children is: 3 2024-09-13T00:35:16.046 INFO:tasks.workunit.client.0.smithi136.stdout:expected: 3 2024-09-13T00:35:16.046 INFO:tasks.workunit.client.0.smithi136.stdout:child: image97 2024-09-13T00:35:16.046 INFO:tasks.workunit.client.0.smithi136.stdout:child: image98 2024-09-13T00:35:16.046 INFO:tasks.workunit.client.0.smithi136.stdout:child: image99 2024-09-13T00:35:16.046 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-26/image97/b0ff5e705198 2024-09-13T00:35:16.046 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:16.046 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-27/image98/b0ffa35c2ffc 2024-09-13T00:35:16.046 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:16.046 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-27/image99/b0ff52900975 2024-09-13T00:35:16.046 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:16.046 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:16.047 INFO:tasks.workunit.client.0.smithi136.stdout:trying to find test-librbd-smithi136-114798-26/image97 2024-09-13T00:35:16.047 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-26/image97 2024-09-13T00:35:16.047 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-26/image97 2024-09-13T00:35:16.047 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:16.047 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:16.047 INFO:tasks.workunit.client.0.smithi136.stdout:trying to find test-librbd-smithi136-114798-27/image99 2024-09-13T00:35:16.047 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-26/image97 2024-09-13T00:35:16.047 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-27/image99 2024-09-13T00:35:16.047 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-27/image99 2024-09-13T00:35:16.047 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:16.047 INFO:tasks.workunit.client.0.smithi136.stdout:num children is: 2 2024-09-13T00:35:16.047 INFO:tasks.workunit.client.0.smithi136.stdout:expected: 2 2024-09-13T00:35:16.048 INFO:tasks.workunit.client.0.smithi136.stdout:child: image97 2024-09-13T00:35:16.048 INFO:tasks.workunit.client.0.smithi136.stdout:child: image99 2024-09-13T00:35:16.048 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-26/image97/b0ff5e705198 2024-09-13T00:35:16.048 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:16.048 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-27/image99/b0ff52900975 2024-09-13T00:35:16.048 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:16.048 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:16.048 INFO:tasks.workunit.client.0.smithi136.stdout:trying to find test-librbd-smithi136-114798-26/image97 2024-09-13T00:35:16.048 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-26/image97 2024-09-13T00:35:16.048 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-26/image97 2024-09-13T00:35:16.048 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:16.048 INFO:tasks.workunit.client.0.smithi136.stdout:num children is: 1 2024-09-13T00:35:16.049 INFO:tasks.workunit.client.0.smithi136.stdout:expected: 1 2024-09-13T00:35:16.049 INFO:tasks.workunit.client.0.smithi136.stdout:child: image97 2024-09-13T00:35:16.049 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-26/image97/b0ff5e705198 2024-09-13T00:35:16.049 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:16.049 INFO:tasks.workunit.client.0.smithi136.stdout:num children is: 0 2024-09-13T00:35:16.049 INFO:tasks.workunit.client.0.smithi136.stdout:expected: 0 2024-09-13T00:35:16.049 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.ListChildren (12078 ms) 2024-09-13T00:35:16.049 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.ListChildrenTiered 2024-09-13T00:35:17.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:16 smithi181 bash[67587]: cluster 2024-09-13T00:35:15.802596+0000 mon.a (mon.0) 3243 : cluster [DBG] osdmap e1725: 8 total, 8 up, 8 in 2024-09-13T00:35:17.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:16 smithi181 bash[67587]: cluster 2024-09-13T00:35:16.225503+0000 mgr.y (mgr.24661) 1744 : cluster [DBG] pgmap v3030: 553 pgs: 553 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 330 KiB/s rd, 6.2 KiB/s wr, 401 op/s 2024-09-13T00:35:17.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:16 smithi136 bash[76371]: cluster 2024-09-13T00:35:15.802596+0000 mon.a (mon.0) 3243 : cluster [DBG] osdmap e1725: 8 total, 8 up, 8 in 2024-09-13T00:35:17.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:16 smithi136 bash[76371]: cluster 2024-09-13T00:35:16.225503+0000 mgr.y (mgr.24661) 1744 : cluster [DBG] pgmap v3030: 553 pgs: 553 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 330 KiB/s rd, 6.2 KiB/s wr, 401 op/s 2024-09-13T00:35:17.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:16 smithi136 bash[75164]: cluster 2024-09-13T00:35:15.802596+0000 mon.a (mon.0) 3243 : cluster [DBG] osdmap e1725: 8 total, 8 up, 8 in 2024-09-13T00:35:17.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:16 smithi136 bash[75164]: cluster 2024-09-13T00:35:16.225503+0000 mgr.y (mgr.24661) 1744 : cluster [DBG] pgmap v3030: 553 pgs: 553 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 330 KiB/s rd, 6.2 KiB/s wr, 401 op/s 2024-09-13T00:35:18.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:17 smithi181 bash[67587]: cluster 2024-09-13T00:35:16.818967+0000 mon.a (mon.0) 3244 : cluster [DBG] osdmap e1726: 8 total, 8 up, 8 in 2024-09-13T00:35:18.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:17 smithi181 bash[67587]: audit 2024-09-13T00:35:16.824103+0000 mon.c (mon.2) 131 : audit [INF] from='client.? 172.21.15.136:0/1826370758' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-28","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:35:18.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:17 smithi181 bash[67587]: audit 2024-09-13T00:35:16.829020+0000 mon.a (mon.0) 3245 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-28","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:35:18.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:17 smithi136 bash[76371]: cluster 2024-09-13T00:35:16.818967+0000 mon.a (mon.0) 3244 : cluster [DBG] osdmap e1726: 8 total, 8 up, 8 in 2024-09-13T00:35:18.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:17 smithi136 bash[76371]: audit 2024-09-13T00:35:16.824103+0000 mon.c (mon.2) 131 : audit [INF] from='client.? 172.21.15.136:0/1826370758' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-28","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:35:18.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:17 smithi136 bash[76371]: audit 2024-09-13T00:35:16.829020+0000 mon.a (mon.0) 3245 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-28","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:35:18.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:17 smithi136 bash[75164]: cluster 2024-09-13T00:35:16.818967+0000 mon.a (mon.0) 3244 : cluster [DBG] osdmap e1726: 8 total, 8 up, 8 in 2024-09-13T00:35:18.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:17 smithi136 bash[75164]: audit 2024-09-13T00:35:16.824103+0000 mon.c (mon.2) 131 : audit [INF] from='client.? 172.21.15.136:0/1826370758' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-28","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:35:18.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:17 smithi136 bash[75164]: audit 2024-09-13T00:35:16.829020+0000 mon.a (mon.0) 3245 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-28","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:35:19.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:18 smithi181 bash[67587]: audit 2024-09-13T00:35:17.818009+0000 mon.a (mon.0) 3246 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-28","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:35:19.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:18 smithi181 bash[67587]: cluster 2024-09-13T00:35:17.818604+0000 mon.a (mon.0) 3247 : cluster [DBG] osdmap e1727: 8 total, 8 up, 8 in 2024-09-13T00:35:19.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:18 smithi181 bash[67587]: cluster 2024-09-13T00:35:18.226523+0000 mgr.y (mgr.24661) 1745 : cluster [DBG] pgmap v3033: 585 pgs: 5 creating+peering, 27 unknown, 553 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 362 KiB/s rd, 8.6 KiB/s wr, 443 op/s 2024-09-13T00:35:19.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:18 smithi136 bash[75164]: audit 2024-09-13T00:35:17.818009+0000 mon.a (mon.0) 3246 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-28","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:35:19.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:18 smithi136 bash[75164]: cluster 2024-09-13T00:35:17.818604+0000 mon.a (mon.0) 3247 : cluster [DBG] osdmap e1727: 8 total, 8 up, 8 in 2024-09-13T00:35:19.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:18 smithi136 bash[75164]: cluster 2024-09-13T00:35:18.226523+0000 mgr.y (mgr.24661) 1745 : cluster [DBG] pgmap v3033: 585 pgs: 5 creating+peering, 27 unknown, 553 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 362 KiB/s rd, 8.6 KiB/s wr, 443 op/s 2024-09-13T00:35:19.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:18 smithi136 bash[76371]: audit 2024-09-13T00:35:17.818009+0000 mon.a (mon.0) 3246 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-28","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:35:19.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:18 smithi136 bash[76371]: cluster 2024-09-13T00:35:17.818604+0000 mon.a (mon.0) 3247 : cluster [DBG] osdmap e1727: 8 total, 8 up, 8 in 2024-09-13T00:35:19.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:18 smithi136 bash[76371]: cluster 2024-09-13T00:35:18.226523+0000 mgr.y (mgr.24661) 1745 : cluster [DBG] pgmap v3033: 585 pgs: 5 creating+peering, 27 unknown, 553 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 362 KiB/s rd, 8.6 KiB/s wr, 443 op/s 2024-09-13T00:35:19.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:35:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:35:19] "GET /metrics HTTP/1.1" 200 68035 "" "Prometheus/2.43.0" 2024-09-13T00:35:20.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:20 smithi136 bash[75164]: cluster 2024-09-13T00:35:18.859698+0000 mon.a (mon.0) 3248 : cluster [DBG] osdmap e1728: 8 total, 8 up, 8 in 2024-09-13T00:35:20.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:20 smithi136 bash[75164]: audit 2024-09-13T00:35:18.864490+0000 mon.c (mon.2) 132 : audit [INF] from='client.? 172.21.15.136:0/972936309' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-29","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:35:20.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:20 smithi136 bash[75164]: audit 2024-09-13T00:35:18.865327+0000 mon.a (mon.0) 3249 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-29","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:35:20.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:20 smithi136 bash[75164]: audit 2024-09-13T00:35:19.599261+0000 mon.a (mon.0) 3250 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-29","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:35:20.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:20 smithi136 bash[75164]: cluster 2024-09-13T00:35:19.600148+0000 mon.a (mon.0) 3251 : cluster [DBG] osdmap e1729: 8 total, 8 up, 8 in 2024-09-13T00:35:20.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:20 smithi136 bash[75164]: audit 2024-09-13T00:35:19.626774+0000 mon.a (mon.0) 3252 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:35:20.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:20 smithi136 bash[76371]: cluster 2024-09-13T00:35:18.859698+0000 mon.a (mon.0) 3248 : cluster [DBG] osdmap e1728: 8 total, 8 up, 8 in 2024-09-13T00:35:20.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:20 smithi136 bash[76371]: audit 2024-09-13T00:35:18.864490+0000 mon.c (mon.2) 132 : audit [INF] from='client.? 172.21.15.136:0/972936309' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-29","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:35:20.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:20 smithi136 bash[76371]: audit 2024-09-13T00:35:18.865327+0000 mon.a (mon.0) 3249 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-29","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:35:20.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:20 smithi136 bash[76371]: audit 2024-09-13T00:35:19.599261+0000 mon.a (mon.0) 3250 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-29","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:35:20.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:20 smithi136 bash[76371]: cluster 2024-09-13T00:35:19.600148+0000 mon.a (mon.0) 3251 : cluster [DBG] osdmap e1729: 8 total, 8 up, 8 in 2024-09-13T00:35:20.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:20 smithi136 bash[76371]: audit 2024-09-13T00:35:19.626774+0000 mon.a (mon.0) 3252 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:35:20.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:20 smithi181 bash[67587]: cluster 2024-09-13T00:35:18.859698+0000 mon.a (mon.0) 3248 : cluster [DBG] osdmap e1728: 8 total, 8 up, 8 in 2024-09-13T00:35:20.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:20 smithi181 bash[67587]: audit 2024-09-13T00:35:18.864490+0000 mon.c (mon.2) 132 : audit [INF] from='client.? 172.21.15.136:0/972936309' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-29","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:35:20.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:20 smithi181 bash[67587]: audit 2024-09-13T00:35:18.865327+0000 mon.a (mon.0) 3249 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-29","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:35:20.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:20 smithi181 bash[67587]: audit 2024-09-13T00:35:19.599261+0000 mon.a (mon.0) 3250 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-29","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:35:20.600 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:20 smithi181 bash[67587]: cluster 2024-09-13T00:35:19.600148+0000 mon.a (mon.0) 3251 : cluster [DBG] osdmap e1729: 8 total, 8 up, 8 in 2024-09-13T00:35:20.600 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:20 smithi181 bash[67587]: audit 2024-09-13T00:35:19.626774+0000 mon.a (mon.0) 3252 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:35:21.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:21 smithi136 bash[76371]: audit 2024-09-13T00:35:20.152726+0000 mon.a (mon.0) 3253 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:35:21.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:21 smithi136 bash[76371]: audit 2024-09-13T00:35:20.154223+0000 mon.a (mon.0) 3254 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:35:21.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:21 smithi136 bash[76371]: audit 2024-09-13T00:35:20.163308+0000 mon.a (mon.0) 3255 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:35:21.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:21 smithi136 bash[76371]: cluster 2024-09-13T00:35:20.227237+0000 mgr.y (mgr.24661) 1746 : cluster [DBG] pgmap v3036: 617 pgs: 11 creating+peering, 46 unknown, 560 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 143 KiB/s rd, 7.7 KiB/s wr, 179 op/s 2024-09-13T00:35:21.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:21 smithi136 bash[76371]: cluster 2024-09-13T00:35:20.604403+0000 mon.a (mon.0) 3256 : cluster [DBG] osdmap e1730: 8 total, 8 up, 8 in 2024-09-13T00:35:21.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:21 smithi136 bash[76371]: audit 2024-09-13T00:35:20.611285+0000 mon.c (mon.2) 133 : audit [INF] from='client.? 172.21.15.136:0/936488091' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-30","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:35:21.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:21 smithi136 bash[76371]: audit 2024-09-13T00:35:20.613686+0000 mon.a (mon.0) 3257 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-30","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:35:21.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:21 smithi136 bash[75164]: audit 2024-09-13T00:35:20.152726+0000 mon.a (mon.0) 3253 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:35:21.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:21 smithi136 bash[75164]: audit 2024-09-13T00:35:20.154223+0000 mon.a (mon.0) 3254 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:35:21.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:21 smithi136 bash[75164]: audit 2024-09-13T00:35:20.163308+0000 mon.a (mon.0) 3255 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:35:21.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:21 smithi136 bash[75164]: cluster 2024-09-13T00:35:20.227237+0000 mgr.y (mgr.24661) 1746 : cluster [DBG] pgmap v3036: 617 pgs: 11 creating+peering, 46 unknown, 560 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 143 KiB/s rd, 7.7 KiB/s wr, 179 op/s 2024-09-13T00:35:21.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:21 smithi136 bash[75164]: cluster 2024-09-13T00:35:20.604403+0000 mon.a (mon.0) 3256 : cluster [DBG] osdmap e1730: 8 total, 8 up, 8 in 2024-09-13T00:35:21.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:21 smithi136 bash[75164]: audit 2024-09-13T00:35:20.611285+0000 mon.c (mon.2) 133 : audit [INF] from='client.? 172.21.15.136:0/936488091' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-30","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:35:21.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:21 smithi136 bash[75164]: audit 2024-09-13T00:35:20.613686+0000 mon.a (mon.0) 3257 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-30","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:35:21.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:21 smithi181 bash[67587]: audit 2024-09-13T00:35:20.152726+0000 mon.a (mon.0) 3253 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:35:21.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:21 smithi181 bash[67587]: audit 2024-09-13T00:35:20.154223+0000 mon.a (mon.0) 3254 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:35:21.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:21 smithi181 bash[67587]: audit 2024-09-13T00:35:20.163308+0000 mon.a (mon.0) 3255 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:35:21.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:21 smithi181 bash[67587]: cluster 2024-09-13T00:35:20.227237+0000 mgr.y (mgr.24661) 1746 : cluster [DBG] pgmap v3036: 617 pgs: 11 creating+peering, 46 unknown, 560 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 143 KiB/s rd, 7.7 KiB/s wr, 179 op/s 2024-09-13T00:35:21.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:21 smithi181 bash[67587]: cluster 2024-09-13T00:35:20.604403+0000 mon.a (mon.0) 3256 : cluster [DBG] osdmap e1730: 8 total, 8 up, 8 in 2024-09-13T00:35:21.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:21 smithi181 bash[67587]: audit 2024-09-13T00:35:20.611285+0000 mon.c (mon.2) 133 : audit [INF] from='client.? 172.21.15.136:0/936488091' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-30","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:35:21.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:21 smithi181 bash[67587]: audit 2024-09-13T00:35:20.613686+0000 mon.a (mon.0) 3257 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-30","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:35:22.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:35:22 smithi181 bash[95505]: ts=2024-09-13T00:35:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:35:23.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:22 smithi136 bash[76371]: audit 2024-09-13T00:35:21.608688+0000 mon.a (mon.0) 3258 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-30","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:35:23.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:22 smithi136 bash[76371]: cluster 2024-09-13T00:35:21.609203+0000 mon.a (mon.0) 3259 : cluster [DBG] osdmap e1731: 8 total, 8 up, 8 in 2024-09-13T00:35:23.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:22 smithi136 bash[76371]: audit 2024-09-13T00:35:21.616161+0000 mon.c (mon.2) 134 : audit [INF] from='client.? 172.21.15.136:0/1086627647' entity='client.admin' cmd=[{"prefix": "osd tier add", "pool": "test-librbd-smithi136-114798-28", "tierpool":"test-librbd-smithi136-114798-30", "force_nonempty":""}]: dispatch 2024-09-13T00:35:23.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:22 smithi136 bash[76371]: audit 2024-09-13T00:35:21.617077+0000 mon.a (mon.0) 3260 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd tier add", "pool": "test-librbd-smithi136-114798-28", "tierpool":"test-librbd-smithi136-114798-30", "force_nonempty":""}]: dispatch 2024-09-13T00:35:23.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:22 smithi136 bash[76371]: cluster 2024-09-13T00:35:22.229052+0000 mgr.y (mgr.24661) 1747 : cluster [DBG] pgmap v3039: 649 pgs: 28 creating+peering, 18 unknown, 603 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 99 KiB/s rd, 3.2 KiB/s wr, 119 op/s 2024-09-13T00:35:23.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:22 smithi136 bash[75164]: audit 2024-09-13T00:35:21.608688+0000 mon.a (mon.0) 3258 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-30","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:35:23.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:22 smithi136 bash[75164]: cluster 2024-09-13T00:35:21.609203+0000 mon.a (mon.0) 3259 : cluster [DBG] osdmap e1731: 8 total, 8 up, 8 in 2024-09-13T00:35:23.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:22 smithi136 bash[75164]: audit 2024-09-13T00:35:21.616161+0000 mon.c (mon.2) 134 : audit [INF] from='client.? 172.21.15.136:0/1086627647' entity='client.admin' cmd=[{"prefix": "osd tier add", "pool": "test-librbd-smithi136-114798-28", "tierpool":"test-librbd-smithi136-114798-30", "force_nonempty":""}]: dispatch 2024-09-13T00:35:23.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:22 smithi136 bash[75164]: audit 2024-09-13T00:35:21.617077+0000 mon.a (mon.0) 3260 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd tier add", "pool": "test-librbd-smithi136-114798-28", "tierpool":"test-librbd-smithi136-114798-30", "force_nonempty":""}]: dispatch 2024-09-13T00:35:23.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:22 smithi136 bash[75164]: cluster 2024-09-13T00:35:22.229052+0000 mgr.y (mgr.24661) 1747 : cluster [DBG] pgmap v3039: 649 pgs: 28 creating+peering, 18 unknown, 603 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 99 KiB/s rd, 3.2 KiB/s wr, 119 op/s 2024-09-13T00:35:23.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:22 smithi181 bash[67587]: audit 2024-09-13T00:35:21.608688+0000 mon.a (mon.0) 3258 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-30","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:35:23.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:22 smithi181 bash[67587]: cluster 2024-09-13T00:35:21.609203+0000 mon.a (mon.0) 3259 : cluster [DBG] osdmap e1731: 8 total, 8 up, 8 in 2024-09-13T00:35:23.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:22 smithi181 bash[67587]: audit 2024-09-13T00:35:21.616161+0000 mon.c (mon.2) 134 : audit [INF] from='client.? 172.21.15.136:0/1086627647' entity='client.admin' cmd=[{"prefix": "osd tier add", "pool": "test-librbd-smithi136-114798-28", "tierpool":"test-librbd-smithi136-114798-30", "force_nonempty":""}]: dispatch 2024-09-13T00:35:23.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:22 smithi181 bash[67587]: audit 2024-09-13T00:35:21.617077+0000 mon.a (mon.0) 3260 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd tier add", "pool": "test-librbd-smithi136-114798-28", "tierpool":"test-librbd-smithi136-114798-30", "force_nonempty":""}]: dispatch 2024-09-13T00:35:23.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:22 smithi181 bash[67587]: cluster 2024-09-13T00:35:22.229052+0000 mgr.y (mgr.24661) 1747 : cluster [DBG] pgmap v3039: 649 pgs: 28 creating+peering, 18 unknown, 603 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 99 KiB/s rd, 3.2 KiB/s wr, 119 op/s 2024-09-13T00:35:24.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:23 smithi136 bash[75164]: audit 2024-09-13T00:35:22.612241+0000 mon.a (mon.0) 3261 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier add", "pool": "test-librbd-smithi136-114798-28", "tierpool":"test-librbd-smithi136-114798-30", "force_nonempty":""}]': finished 2024-09-13T00:35:24.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:23 smithi136 bash[75164]: cluster 2024-09-13T00:35:22.612470+0000 mon.a (mon.0) 3262 : cluster [DBG] osdmap e1732: 8 total, 8 up, 8 in 2024-09-13T00:35:24.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:23 smithi136 bash[75164]: audit 2024-09-13T00:35:22.614754+0000 mon.c (mon.2) 135 : audit [INF] from='client.? 172.21.15.136:0/1086627647' entity='client.admin' cmd=[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi136-114798-30", "mode":"writeback"}]: dispatch 2024-09-13T00:35:24.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:23 smithi136 bash[75164]: audit 2024-09-13T00:35:22.623068+0000 mon.a (mon.0) 3263 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi136-114798-30", "mode":"writeback"}]: dispatch 2024-09-13T00:35:24.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:23 smithi136 bash[75164]: cluster 2024-09-13T00:35:23.603693+0000 mon.a (mon.0) 3264 : cluster [WRN] Health check failed: 1 cache pools are missing hit_sets (CACHE_POOL_NO_HIT_SET) 2024-09-13T00:35:24.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:23 smithi136 bash[75164]: audit 2024-09-13T00:35:23.613785+0000 mon.a (mon.0) 3265 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi136-114798-30", "mode":"writeback"}]': finished 2024-09-13T00:35:24.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:23 smithi136 bash[75164]: cluster 2024-09-13T00:35:23.613967+0000 mon.a (mon.0) 3266 : cluster [DBG] osdmap e1733: 8 total, 8 up, 8 in 2024-09-13T00:35:24.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:23 smithi136 bash[75164]: audit 2024-09-13T00:35:23.623451+0000 mon.c (mon.2) 136 : audit [INF] from='client.? 172.21.15.136:0/1086627647' entity='client.admin' cmd=[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi136-114798-28", "overlaypool":"test-librbd-smithi136-114798-30"}]: dispatch 2024-09-13T00:35:24.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:23 smithi136 bash[76371]: audit 2024-09-13T00:35:22.612241+0000 mon.a (mon.0) 3261 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier add", "pool": "test-librbd-smithi136-114798-28", "tierpool":"test-librbd-smithi136-114798-30", "force_nonempty":""}]': finished 2024-09-13T00:35:24.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:23 smithi136 bash[76371]: cluster 2024-09-13T00:35:22.612470+0000 mon.a (mon.0) 3262 : cluster [DBG] osdmap e1732: 8 total, 8 up, 8 in 2024-09-13T00:35:24.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:23 smithi136 bash[76371]: audit 2024-09-13T00:35:22.614754+0000 mon.c (mon.2) 135 : audit [INF] from='client.? 172.21.15.136:0/1086627647' entity='client.admin' cmd=[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi136-114798-30", "mode":"writeback"}]: dispatch 2024-09-13T00:35:24.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:23 smithi136 bash[76371]: audit 2024-09-13T00:35:22.623068+0000 mon.a (mon.0) 3263 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi136-114798-30", "mode":"writeback"}]: dispatch 2024-09-13T00:35:24.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:23 smithi136 bash[76371]: cluster 2024-09-13T00:35:23.603693+0000 mon.a (mon.0) 3264 : cluster [WRN] Health check failed: 1 cache pools are missing hit_sets (CACHE_POOL_NO_HIT_SET) 2024-09-13T00:35:24.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:23 smithi136 bash[76371]: audit 2024-09-13T00:35:23.613785+0000 mon.a (mon.0) 3265 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi136-114798-30", "mode":"writeback"}]': finished 2024-09-13T00:35:24.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:23 smithi136 bash[76371]: cluster 2024-09-13T00:35:23.613967+0000 mon.a (mon.0) 3266 : cluster [DBG] osdmap e1733: 8 total, 8 up, 8 in 2024-09-13T00:35:24.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:23 smithi136 bash[76371]: audit 2024-09-13T00:35:23.623451+0000 mon.c (mon.2) 136 : audit [INF] from='client.? 172.21.15.136:0/1086627647' entity='client.admin' cmd=[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi136-114798-28", "overlaypool":"test-librbd-smithi136-114798-30"}]: dispatch 2024-09-13T00:35:24.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:23 smithi181 bash[67587]: audit 2024-09-13T00:35:22.612241+0000 mon.a (mon.0) 3261 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier add", "pool": "test-librbd-smithi136-114798-28", "tierpool":"test-librbd-smithi136-114798-30", "force_nonempty":""}]': finished 2024-09-13T00:35:24.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:23 smithi181 bash[67587]: cluster 2024-09-13T00:35:22.612470+0000 mon.a (mon.0) 3262 : cluster [DBG] osdmap e1732: 8 total, 8 up, 8 in 2024-09-13T00:35:24.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:23 smithi181 bash[67587]: audit 2024-09-13T00:35:22.614754+0000 mon.c (mon.2) 135 : audit [INF] from='client.? 172.21.15.136:0/1086627647' entity='client.admin' cmd=[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi136-114798-30", "mode":"writeback"}]: dispatch 2024-09-13T00:35:24.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:23 smithi181 bash[67587]: audit 2024-09-13T00:35:22.623068+0000 mon.a (mon.0) 3263 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi136-114798-30", "mode":"writeback"}]: dispatch 2024-09-13T00:35:24.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:23 smithi181 bash[67587]: cluster 2024-09-13T00:35:23.603693+0000 mon.a (mon.0) 3264 : cluster [WRN] Health check failed: 1 cache pools are missing hit_sets (CACHE_POOL_NO_HIT_SET) 2024-09-13T00:35:24.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:23 smithi181 bash[67587]: audit 2024-09-13T00:35:23.613785+0000 mon.a (mon.0) 3265 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi136-114798-30", "mode":"writeback"}]': finished 2024-09-13T00:35:24.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:23 smithi181 bash[67587]: cluster 2024-09-13T00:35:23.613967+0000 mon.a (mon.0) 3266 : cluster [DBG] osdmap e1733: 8 total, 8 up, 8 in 2024-09-13T00:35:24.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:23 smithi181 bash[67587]: audit 2024-09-13T00:35:23.623451+0000 mon.c (mon.2) 136 : audit [INF] from='client.? 172.21.15.136:0/1086627647' entity='client.admin' cmd=[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi136-114798-28", "overlaypool":"test-librbd-smithi136-114798-30"}]: dispatch 2024-09-13T00:35:24.615 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:35:25.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:24 smithi136 bash[76371]: audit 2024-09-13T00:35:23.624467+0000 mon.a (mon.0) 3267 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi136-114798-28", "overlaypool":"test-librbd-smithi136-114798-30"}]: dispatch 2024-09-13T00:35:25.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:24 smithi136 bash[76371]: audit 2024-09-13T00:35:23.837772+0000 mon.a (mon.0) 3268 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:35:25.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:24 smithi136 bash[76371]: audit 2024-09-13T00:35:23.840790+0000 mon.a (mon.0) 3269 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:35:25.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:24 smithi136 bash[76371]: cluster 2024-09-13T00:35:24.230239+0000 mgr.y (mgr.24661) 1748 : cluster [DBG] pgmap v3042: 649 pgs: 23 creating+peering, 2 unknown, 624 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:35:25.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:24 smithi136 bash[76371]: audit 2024-09-13T00:35:24.602414+0000 mon.a (mon.0) 3270 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi136-114798-28", "overlaypool":"test-librbd-smithi136-114798-30"}]': finished 2024-09-13T00:35:25.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:24 smithi136 bash[76371]: cluster 2024-09-13T00:35:24.602638+0000 mon.a (mon.0) 3271 : cluster [DBG] osdmap e1734: 8 total, 8 up, 8 in 2024-09-13T00:35:25.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:24 smithi136 bash[75164]: audit 2024-09-13T00:35:23.624467+0000 mon.a (mon.0) 3267 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi136-114798-28", "overlaypool":"test-librbd-smithi136-114798-30"}]: dispatch 2024-09-13T00:35:25.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:24 smithi136 bash[75164]: audit 2024-09-13T00:35:23.837772+0000 mon.a (mon.0) 3268 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:35:25.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:24 smithi136 bash[75164]: audit 2024-09-13T00:35:23.840790+0000 mon.a (mon.0) 3269 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:35:25.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:24 smithi136 bash[75164]: cluster 2024-09-13T00:35:24.230239+0000 mgr.y (mgr.24661) 1748 : cluster [DBG] pgmap v3042: 649 pgs: 23 creating+peering, 2 unknown, 624 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:35:25.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:24 smithi136 bash[75164]: audit 2024-09-13T00:35:24.602414+0000 mon.a (mon.0) 3270 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi136-114798-28", "overlaypool":"test-librbd-smithi136-114798-30"}]': finished 2024-09-13T00:35:25.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:24 smithi136 bash[75164]: cluster 2024-09-13T00:35:24.602638+0000 mon.a (mon.0) 3271 : cluster [DBG] osdmap e1734: 8 total, 8 up, 8 in 2024-09-13T00:35:25.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:24 smithi181 bash[67587]: audit 2024-09-13T00:35:23.624467+0000 mon.a (mon.0) 3267 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi136-114798-28", "overlaypool":"test-librbd-smithi136-114798-30"}]: dispatch 2024-09-13T00:35:25.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:24 smithi181 bash[67587]: audit 2024-09-13T00:35:23.837772+0000 mon.a (mon.0) 3268 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:35:25.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:24 smithi181 bash[67587]: audit 2024-09-13T00:35:23.840790+0000 mon.a (mon.0) 3269 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:35:25.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:24 smithi181 bash[67587]: cluster 2024-09-13T00:35:24.230239+0000 mgr.y (mgr.24661) 1748 : cluster [DBG] pgmap v3042: 649 pgs: 23 creating+peering, 2 unknown, 624 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:35:25.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:24 smithi181 bash[67587]: audit 2024-09-13T00:35:24.602414+0000 mon.a (mon.0) 3270 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi136-114798-28", "overlaypool":"test-librbd-smithi136-114798-30"}]': finished 2024-09-13T00:35:25.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:24 smithi181 bash[67587]: cluster 2024-09-13T00:35:24.602638+0000 mon.a (mon.0) 3271 : cluster [DBG] osdmap e1734: 8 total, 8 up, 8 in 2024-09-13T00:35:27.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:26 smithi136 bash[75164]: cluster 2024-09-13T00:35:25.643689+0000 mon.a (mon.0) 3272 : cluster [DBG] osdmap e1735: 8 total, 8 up, 8 in 2024-09-13T00:35:27.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:26 smithi136 bash[75164]: cluster 2024-09-13T00:35:26.232623+0000 mgr.y (mgr.24661) 1749 : cluster [DBG] pgmap v3045: 649 pgs: 6 creating+peering, 643 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:35:27.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:26 smithi136 bash[76371]: cluster 2024-09-13T00:35:25.643689+0000 mon.a (mon.0) 3272 : cluster [DBG] osdmap e1735: 8 total, 8 up, 8 in 2024-09-13T00:35:27.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:26 smithi136 bash[76371]: cluster 2024-09-13T00:35:26.232623+0000 mgr.y (mgr.24661) 1749 : cluster [DBG] pgmap v3045: 649 pgs: 6 creating+peering, 643 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:35:27.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:26 smithi181 bash[67587]: cluster 2024-09-13T00:35:25.643689+0000 mon.a (mon.0) 3272 : cluster [DBG] osdmap e1735: 8 total, 8 up, 8 in 2024-09-13T00:35:27.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:26 smithi181 bash[67587]: cluster 2024-09-13T00:35:26.232623+0000 mgr.y (mgr.24661) 1749 : cluster [DBG] pgmap v3045: 649 pgs: 6 creating+peering, 643 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:35:28.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:27 smithi136 bash[75164]: cluster 2024-09-13T00:35:26.656448+0000 mon.a (mon.0) 3273 : cluster [DBG] osdmap e1736: 8 total, 8 up, 8 in 2024-09-13T00:35:28.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:27 smithi136 bash[76371]: cluster 2024-09-13T00:35:26.656448+0000 mon.a (mon.0) 3273 : cluster [DBG] osdmap e1736: 8 total, 8 up, 8 in 2024-09-13T00:35:28.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:27 smithi181 bash[67587]: cluster 2024-09-13T00:35:26.656448+0000 mon.a (mon.0) 3273 : cluster [DBG] osdmap e1736: 8 total, 8 up, 8 in 2024-09-13T00:35:29.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:28 smithi136 bash[75164]: cluster 2024-09-13T00:35:27.658216+0000 mon.a (mon.0) 3274 : cluster [DBG] osdmap e1737: 8 total, 8 up, 8 in 2024-09-13T00:35:29.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:28 smithi136 bash[75164]: cluster 2024-09-13T00:35:28.233800+0000 mgr.y (mgr.24661) 1750 : cluster [DBG] pgmap v3048: 649 pgs: 649 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 767 B/s wr, 0 op/s 2024-09-13T00:35:29.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:28 smithi136 bash[76371]: cluster 2024-09-13T00:35:27.658216+0000 mon.a (mon.0) 3274 : cluster [DBG] osdmap e1737: 8 total, 8 up, 8 in 2024-09-13T00:35:29.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:28 smithi136 bash[76371]: cluster 2024-09-13T00:35:28.233800+0000 mgr.y (mgr.24661) 1750 : cluster [DBG] pgmap v3048: 649 pgs: 649 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 767 B/s wr, 0 op/s 2024-09-13T00:35:29.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:28 smithi181 bash[67587]: cluster 2024-09-13T00:35:27.658216+0000 mon.a (mon.0) 3274 : cluster [DBG] osdmap e1737: 8 total, 8 up, 8 in 2024-09-13T00:35:29.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:28 smithi181 bash[67587]: cluster 2024-09-13T00:35:28.233800+0000 mgr.y (mgr.24661) 1750 : cluster [DBG] pgmap v3048: 649 pgs: 649 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 767 B/s wr, 0 op/s 2024-09-13T00:35:29.675 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:35:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:35:29] "GET /metrics HTTP/1.1" 200 75464 "" "Prometheus/2.43.0" 2024-09-13T00:35:30.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:29 smithi136 bash[75164]: cluster 2024-09-13T00:35:28.682642+0000 mon.a (mon.0) 3275 : cluster [DBG] osdmap e1738: 8 total, 8 up, 8 in 2024-09-13T00:35:30.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:29 smithi136 bash[76371]: cluster 2024-09-13T00:35:28.682642+0000 mon.a (mon.0) 3275 : cluster [DBG] osdmap e1738: 8 total, 8 up, 8 in 2024-09-13T00:35:30.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:29 smithi181 bash[67587]: cluster 2024-09-13T00:35:28.682642+0000 mon.a (mon.0) 3275 : cluster [DBG] osdmap e1738: 8 total, 8 up, 8 in 2024-09-13T00:35:31.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:30 smithi136 bash[76371]: cluster 2024-09-13T00:35:29.689393+0000 mon.a (mon.0) 3276 : cluster [DBG] osdmap e1739: 8 total, 8 up, 8 in 2024-09-13T00:35:31.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:30 smithi136 bash[76371]: cluster 2024-09-13T00:35:30.234874+0000 mgr.y (mgr.24661) 1751 : cluster [DBG] pgmap v3051: 649 pgs: 649 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 1023 B/s wr, 1 op/s 2024-09-13T00:35:31.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:30 smithi136 bash[75164]: cluster 2024-09-13T00:35:29.689393+0000 mon.a (mon.0) 3276 : cluster [DBG] osdmap e1739: 8 total, 8 up, 8 in 2024-09-13T00:35:31.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:30 smithi136 bash[75164]: cluster 2024-09-13T00:35:30.234874+0000 mgr.y (mgr.24661) 1751 : cluster [DBG] pgmap v3051: 649 pgs: 649 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 1023 B/s wr, 1 op/s 2024-09-13T00:35:31.045 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:31.045 INFO:tasks.workunit.client.0.smithi136.stdout:trying to find test-librbd-smithi136-114798-29/image101 2024-09-13T00:35:31.045 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-29/image101 2024-09-13T00:35:31.045 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-29/image101 2024-09-13T00:35:31.045 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:31.045 INFO:tasks.workunit.client.0.smithi136.stdout:num children is: 1 2024-09-13T00:35:31.045 INFO:tasks.workunit.client.0.smithi136.stdout:expected: 1 2024-09-13T00:35:31.045 INFO:tasks.workunit.client.0.smithi136.stdout:child: image101 2024-09-13T00:35:31.045 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-29/image101/b0ff6cf93950 2024-09-13T00:35:31.045 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:31.045 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:31.046 INFO:tasks.workunit.client.0.smithi136.stdout:trying to find test-librbd-smithi136-114798-29/image101 2024-09-13T00:35:31.046 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-28/image102 2024-09-13T00:35:31.046 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-29/image101 2024-09-13T00:35:31.046 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-29/image101 2024-09-13T00:35:31.046 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:31.046 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:31.046 INFO:tasks.workunit.client.0.smithi136.stdout:trying to find test-librbd-smithi136-114798-28/image102 2024-09-13T00:35:31.046 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-28/image102 2024-09-13T00:35:31.046 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-28/image102 2024-09-13T00:35:31.046 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:31.046 INFO:tasks.workunit.client.0.smithi136.stdout:num children is: 2 2024-09-13T00:35:31.046 INFO:tasks.workunit.client.0.smithi136.stdout:expected: 2 2024-09-13T00:35:31.046 INFO:tasks.workunit.client.0.smithi136.stdout:child: image102 2024-09-13T00:35:31.046 INFO:tasks.workunit.client.0.smithi136.stdout:child: image101 2024-09-13T00:35:31.046 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-29/image101/b0ff6cf93950 2024-09-13T00:35:31.047 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:31.047 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-28/image102/b0ffd258e7ed 2024-09-13T00:35:31.047 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:31.047 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:31.047 INFO:tasks.workunit.client.0.smithi136.stdout:trying to find test-librbd-smithi136-114798-29/image101 2024-09-13T00:35:31.047 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-28/image102 2024-09-13T00:35:31.047 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-29/image101 2024-09-13T00:35:31.047 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-29/image101 2024-09-13T00:35:31.047 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:31.047 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:31.047 INFO:tasks.workunit.client.0.smithi136.stdout:trying to find test-librbd-smithi136-114798-28/image102 2024-09-13T00:35:31.047 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-28/image102 2024-09-13T00:35:31.047 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-28/image102 2024-09-13T00:35:31.048 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:31.048 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:31.048 INFO:tasks.workunit.client.0.smithi136.stdout:trying to find test-librbd-smithi136-114798-29/image103 2024-09-13T00:35:31.048 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-28/image102 2024-09-13T00:35:31.048 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-29/image101 2024-09-13T00:35:31.048 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-29/image103 2024-09-13T00:35:31.048 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-29/image103 2024-09-13T00:35:31.048 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:31.048 INFO:tasks.workunit.client.0.smithi136.stdout:num children is: 3 2024-09-13T00:35:31.048 INFO:tasks.workunit.client.0.smithi136.stdout:expected: 3 2024-09-13T00:35:31.048 INFO:tasks.workunit.client.0.smithi136.stdout:child: image102 2024-09-13T00:35:31.048 INFO:tasks.workunit.client.0.smithi136.stdout:child: image101 2024-09-13T00:35:31.048 INFO:tasks.workunit.client.0.smithi136.stdout:child: image103 2024-09-13T00:35:31.048 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-29/image101/b0ff6cf93950 2024-09-13T00:35:31.049 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:31.049 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-28/image102/b0ffd258e7ed 2024-09-13T00:35:31.049 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:31.049 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-29/image103/b0ff31ab5c88 2024-09-13T00:35:31.049 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:31.049 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:31.049 INFO:tasks.workunit.client.0.smithi136.stdout:trying to find test-librbd-smithi136-114798-29/image101 2024-09-13T00:35:31.049 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-28/image102 2024-09-13T00:35:31.049 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-29/image101 2024-09-13T00:35:31.049 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-29/image101 2024-09-13T00:35:31.049 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:31.049 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:31.049 INFO:tasks.workunit.client.0.smithi136.stdout:trying to find test-librbd-smithi136-114798-28/image102 2024-09-13T00:35:31.049 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-28/image102 2024-09-13T00:35:31.049 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-28/image102 2024-09-13T00:35:31.050 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:31.050 INFO:tasks.workunit.client.0.smithi136.stdout:num children is: 3 2024-09-13T00:35:31.050 INFO:tasks.workunit.client.0.smithi136.stdout:expected: 3 2024-09-13T00:35:31.050 INFO:tasks.workunit.client.0.smithi136.stdout:child: image102 2024-09-13T00:35:31.050 INFO:tasks.workunit.client.0.smithi136.stdout:child: image101 2024-09-13T00:35:31.050 INFO:tasks.workunit.client.0.smithi136.stdout:child: image103 2024-09-13T00:35:31.050 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-29/image101/b0ff6cf93950 2024-09-13T00:35:31.050 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:31.050 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-28/image102/b0ffd258e7ed 2024-09-13T00:35:31.050 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:31.050 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-29/image103/b0ff31ab5c88 2024-09-13T00:35:31.050 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:31.050 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:31.051 INFO:tasks.workunit.client.0.smithi136.stdout:trying to find test-librbd-smithi136-114798-29/image101 2024-09-13T00:35:31.051 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-28/image102 2024-09-13T00:35:31.051 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-29/image101 2024-09-13T00:35:31.051 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-29/image101 2024-09-13T00:35:31.051 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:31.051 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:31.051 INFO:tasks.workunit.client.0.smithi136.stdout:trying to find test-librbd-smithi136-114798-28/image102 2024-09-13T00:35:31.051 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-28/image102 2024-09-13T00:35:31.051 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-28/image102 2024-09-13T00:35:31.051 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:31.051 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:31.051 INFO:tasks.workunit.client.0.smithi136.stdout:trying to find test-librbd-smithi136-114798-29/image104 2024-09-13T00:35:31.051 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-28/image102 2024-09-13T00:35:31.051 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-29/image101 2024-09-13T00:35:31.052 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-29/image104 2024-09-13T00:35:31.052 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-29/image104 2024-09-13T00:35:31.052 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:31.052 INFO:tasks.workunit.client.0.smithi136.stdout:num children is: 4 2024-09-13T00:35:31.052 INFO:tasks.workunit.client.0.smithi136.stdout:expected: 4 2024-09-13T00:35:31.052 INFO:tasks.workunit.client.0.smithi136.stdout:child: image102 2024-09-13T00:35:31.052 INFO:tasks.workunit.client.0.smithi136.stdout:child: image101 2024-09-13T00:35:31.052 INFO:tasks.workunit.client.0.smithi136.stdout:child: image103 2024-09-13T00:35:31.052 INFO:tasks.workunit.client.0.smithi136.stdout:child: image104 2024-09-13T00:35:31.052 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-29/image101/b0ff6cf93950 2024-09-13T00:35:31.052 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:31.053 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-28/image102/b0ffd258e7ed 2024-09-13T00:35:31.053 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:31.053 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-29/image103/b0ff31ab5c88 2024-09-13T00:35:31.053 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:31.053 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-29/image104/b0ffc55e9a38 2024-09-13T00:35:31.053 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:31.053 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:31.053 INFO:tasks.workunit.client.0.smithi136.stdout:trying to find test-librbd-smithi136-114798-29/image101 2024-09-13T00:35:31.053 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-28/image102 2024-09-13T00:35:31.053 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-29/image101 2024-09-13T00:35:31.053 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-29/image101 2024-09-13T00:35:31.053 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:31.053 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:31.053 INFO:tasks.workunit.client.0.smithi136.stdout:trying to find test-librbd-smithi136-114798-28/image102 2024-09-13T00:35:31.054 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-28/image102 2024-09-13T00:35:31.054 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-28/image102 2024-09-13T00:35:31.054 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:31.054 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:31.054 INFO:tasks.workunit.client.0.smithi136.stdout:trying to find test-librbd-smithi136-114798-29/image103 2024-09-13T00:35:31.054 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-28/image102 2024-09-13T00:35:31.054 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-29/image101 2024-09-13T00:35:31.054 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-29/image103 2024-09-13T00:35:31.054 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-29/image103 2024-09-13T00:35:31.054 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:31.054 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:31.054 INFO:tasks.workunit.client.0.smithi136.stdout:trying to find test-librbd-smithi136-114798-29/image104 2024-09-13T00:35:31.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:30 smithi181 bash[67587]: cluster 2024-09-13T00:35:29.689393+0000 mon.a (mon.0) 3276 : cluster [DBG] osdmap e1739: 8 total, 8 up, 8 in 2024-09-13T00:35:31.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:30 smithi181 bash[67587]: cluster 2024-09-13T00:35:30.234874+0000 mgr.y (mgr.24661) 1751 : cluster [DBG] pgmap v3051: 649 pgs: 649 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 1023 B/s wr, 1 op/s 2024-09-13T00:35:32.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:35:32 smithi181 bash[95505]: ts=2024-09-13T00:35:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:35:33.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:33 smithi136 bash[76371]: cluster 2024-09-13T00:35:32.201240+0000 mon.a (mon.0) 3277 : cluster [DBG] osdmap e1740: 8 total, 8 up, 8 in 2024-09-13T00:35:33.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:33 smithi136 bash[76371]: cluster 2024-09-13T00:35:32.238882+0000 mgr.y (mgr.24661) 1752 : cluster [DBG] pgmap v3053: 649 pgs: 649 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 325 KiB/s rd, 8.5 KiB/s wr, 388 op/s 2024-09-13T00:35:33.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:33 smithi136 bash[76371]: audit 2024-09-13T00:35:32.458821+0000 mon.c (mon.2) 137 : audit [INF] from='client.? 172.21.15.136:0/1086627647' entity='client.admin' cmd=[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi136-114798-28"}]: dispatch 2024-09-13T00:35:33.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:33 smithi136 bash[76371]: audit 2024-09-13T00:35:32.459387+0000 mon.a (mon.0) 3278 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi136-114798-28"}]: dispatch 2024-09-13T00:35:33.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:33 smithi136 bash[75164]: cluster 2024-09-13T00:35:32.201240+0000 mon.a (mon.0) 3277 : cluster [DBG] osdmap e1740: 8 total, 8 up, 8 in 2024-09-13T00:35:33.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:33 smithi136 bash[75164]: cluster 2024-09-13T00:35:32.238882+0000 mgr.y (mgr.24661) 1752 : cluster [DBG] pgmap v3053: 649 pgs: 649 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 325 KiB/s rd, 8.5 KiB/s wr, 388 op/s 2024-09-13T00:35:33.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:33 smithi136 bash[75164]: audit 2024-09-13T00:35:32.458821+0000 mon.c (mon.2) 137 : audit [INF] from='client.? 172.21.15.136:0/1086627647' entity='client.admin' cmd=[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi136-114798-28"}]: dispatch 2024-09-13T00:35:33.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:33 smithi136 bash[75164]: audit 2024-09-13T00:35:32.459387+0000 mon.a (mon.0) 3278 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi136-114798-28"}]: dispatch 2024-09-13T00:35:33.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:33 smithi181 bash[67587]: cluster 2024-09-13T00:35:32.201240+0000 mon.a (mon.0) 3277 : cluster [DBG] osdmap e1740: 8 total, 8 up, 8 in 2024-09-13T00:35:33.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:33 smithi181 bash[67587]: cluster 2024-09-13T00:35:32.238882+0000 mgr.y (mgr.24661) 1752 : cluster [DBG] pgmap v3053: 649 pgs: 649 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 325 KiB/s rd, 8.5 KiB/s wr, 388 op/s 2024-09-13T00:35:33.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:33 smithi181 bash[67587]: audit 2024-09-13T00:35:32.458821+0000 mon.c (mon.2) 137 : audit [INF] from='client.? 172.21.15.136:0/1086627647' entity='client.admin' cmd=[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi136-114798-28"}]: dispatch 2024-09-13T00:35:33.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:33 smithi181 bash[67587]: audit 2024-09-13T00:35:32.459387+0000 mon.a (mon.0) 3278 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi136-114798-28"}]: dispatch 2024-09-13T00:35:34.211 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-28/image102 2024-09-13T00:35:34.211 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-29/image101 2024-09-13T00:35:34.211 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-29/image103 2024-09-13T00:35:34.211 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-29/image104 2024-09-13T00:35:34.211 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-29/image104 2024-09-13T00:35:34.211 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:34.211 INFO:tasks.workunit.client.0.smithi136.stdout:num children is: 4 2024-09-13T00:35:34.211 INFO:tasks.workunit.client.0.smithi136.stdout:expected: 4 2024-09-13T00:35:34.212 INFO:tasks.workunit.client.0.smithi136.stdout:child: image102 2024-09-13T00:35:34.212 INFO:tasks.workunit.client.0.smithi136.stdout:child: image101 2024-09-13T00:35:34.212 INFO:tasks.workunit.client.0.smithi136.stdout:child: image103 2024-09-13T00:35:34.212 INFO:tasks.workunit.client.0.smithi136.stdout:child: image104 2024-09-13T00:35:34.212 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-29/image101/b0ff6cf93950 2024-09-13T00:35:34.212 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:34.212 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-28/image102/b0ffd258e7ed 2024-09-13T00:35:34.212 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:34.212 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-29/image103/b0ff31ab5c88 2024-09-13T00:35:34.212 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:34.212 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-29/image104/b0ffc55e9a38 2024-09-13T00:35:34.212 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:34.212 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:34.212 INFO:tasks.workunit.client.0.smithi136.stdout:trying to find test-librbd-smithi136-114798-28/image102 2024-09-13T00:35:34.213 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-28/image102 2024-09-13T00:35:34.213 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-28/image102 2024-09-13T00:35:34.213 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:34.213 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:34.213 INFO:tasks.workunit.client.0.smithi136.stdout:trying to find test-librbd-smithi136-114798-29/image103 2024-09-13T00:35:34.213 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-28/image102 2024-09-13T00:35:34.213 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-29/image103 2024-09-13T00:35:34.213 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-29/image103 2024-09-13T00:35:34.213 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:34.213 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:34.213 INFO:tasks.workunit.client.0.smithi136.stdout:trying to find test-librbd-smithi136-114798-29/image104 2024-09-13T00:35:34.213 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-28/image102 2024-09-13T00:35:34.213 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-29/image103 2024-09-13T00:35:34.213 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-29/image104 2024-09-13T00:35:34.214 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-29/image104 2024-09-13T00:35:34.214 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:34.214 INFO:tasks.workunit.client.0.smithi136.stdout:num children is: 3 2024-09-13T00:35:34.214 INFO:tasks.workunit.client.0.smithi136.stdout:expected: 3 2024-09-13T00:35:34.214 INFO:tasks.workunit.client.0.smithi136.stdout:child: image102 2024-09-13T00:35:34.214 INFO:tasks.workunit.client.0.smithi136.stdout:child: image103 2024-09-13T00:35:34.214 INFO:tasks.workunit.client.0.smithi136.stdout:child: image104 2024-09-13T00:35:34.214 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-28/image102/b0ffd258e7ed 2024-09-13T00:35:34.214 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:34.214 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-29/image103/b0ff31ab5c88 2024-09-13T00:35:34.214 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:34.214 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-29/image104/b0ffc55e9a38 2024-09-13T00:35:34.214 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:34.215 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:34.215 INFO:tasks.workunit.client.0.smithi136.stdout:trying to find test-librbd-smithi136-114798-28/image102 2024-09-13T00:35:34.215 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-28/image102 2024-09-13T00:35:34.215 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-28/image102 2024-09-13T00:35:34.215 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:34.215 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:34.215 INFO:tasks.workunit.client.0.smithi136.stdout:trying to find test-librbd-smithi136-114798-29/image104 2024-09-13T00:35:34.215 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-28/image102 2024-09-13T00:35:34.215 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-29/image104 2024-09-13T00:35:34.215 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-29/image104 2024-09-13T00:35:34.215 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:34.215 INFO:tasks.workunit.client.0.smithi136.stdout:num children is: 2 2024-09-13T00:35:34.215 INFO:tasks.workunit.client.0.smithi136.stdout:expected: 2 2024-09-13T00:35:34.215 INFO:tasks.workunit.client.0.smithi136.stdout:child: image102 2024-09-13T00:35:34.216 INFO:tasks.workunit.client.0.smithi136.stdout:child: image104 2024-09-13T00:35:34.216 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-28/image102/b0ffd258e7ed 2024-09-13T00:35:34.216 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:34.216 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-29/image104/b0ffc55e9a38 2024-09-13T00:35:34.216 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:34.216 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:34.216 INFO:tasks.workunit.client.0.smithi136.stdout:trying to find test-librbd-smithi136-114798-28/image102 2024-09-13T00:35:34.216 INFO:tasks.workunit.client.0.smithi136.stdout:checking test-librbd-smithi136-114798-28/image102 2024-09-13T00:35:34.216 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-28/image102 2024-09-13T00:35:34.216 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:34.216 INFO:tasks.workunit.client.0.smithi136.stdout:num children is: 1 2024-09-13T00:35:34.216 INFO:tasks.workunit.client.0.smithi136.stdout:expected: 1 2024-09-13T00:35:34.216 INFO:tasks.workunit.client.0.smithi136.stdout:child: image102 2024-09-13T00:35:34.217 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-28/image102/b0ffd258e7ed 2024-09-13T00:35:34.217 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:35:34.217 INFO:tasks.workunit.client.0.smithi136.stdout:num children is: 0 2024-09-13T00:35:34.217 INFO:tasks.workunit.client.0.smithi136.stdout:expected: 0 2024-09-13T00:35:34.217 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.ListChildrenTiered (18168 ms) 2024-09-13T00:35:34.217 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.LockingPP 2024-09-13T00:35:34.217 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:35:34.376 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.LockingPP (165 ms) 2024-09-13T00:35:34.376 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.FlushAio 2024-09-13T00:35:34.376 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:35:34.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:34 smithi136 bash[75164]: audit 2024-09-13T00:35:33.202408+0000 mon.a (mon.0) 3279 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi136-114798-28"}]': finished 2024-09-13T00:35:34.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:34 smithi136 bash[75164]: cluster 2024-09-13T00:35:33.202589+0000 mon.a (mon.0) 3280 : cluster [DBG] osdmap e1741: 8 total, 8 up, 8 in 2024-09-13T00:35:34.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:34 smithi136 bash[75164]: audit 2024-09-13T00:35:33.204828+0000 mon.c (mon.2) 138 : audit [INF] from='client.? 172.21.15.136:0/1086627647' entity='client.admin' cmd=[{"prefix": "osd tier remove", "pool": "test-librbd-smithi136-114798-28", "tierpool":"test-librbd-smithi136-114798-30"}]: dispatch 2024-09-13T00:35:34.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:34 smithi136 bash[75164]: audit 2024-09-13T00:35:33.208167+0000 mon.a (mon.0) 3281 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd tier remove", "pool": "test-librbd-smithi136-114798-28", "tierpool":"test-librbd-smithi136-114798-30"}]: dispatch 2024-09-13T00:35:34.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:34 smithi136 bash[75164]: cluster 2024-09-13T00:35:34.195929+0000 mon.a (mon.0) 3282 : cluster [INF] Health check cleared: CACHE_POOL_NO_HIT_SET (was: 1 cache pools are missing hit_sets) 2024-09-13T00:35:34.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:34 smithi136 bash[75164]: cluster 2024-09-13T00:35:34.195975+0000 mon.a (mon.0) 3283 : cluster [INF] Cluster is now healthy 2024-09-13T00:35:34.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:34 smithi136 bash[75164]: audit 2024-09-13T00:35:34.208379+0000 mon.a (mon.0) 3284 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier remove", "pool": "test-librbd-smithi136-114798-28", "tierpool":"test-librbd-smithi136-114798-30"}]': finished 2024-09-13T00:35:34.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:34 smithi136 bash[75164]: cluster 2024-09-13T00:35:34.208840+0000 mon.a (mon.0) 3285 : cluster [DBG] osdmap e1742: 8 total, 8 up, 8 in 2024-09-13T00:35:34.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:34 smithi136 bash[76371]: audit 2024-09-13T00:35:33.202408+0000 mon.a (mon.0) 3279 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi136-114798-28"}]': finished 2024-09-13T00:35:34.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:34 smithi136 bash[76371]: cluster 2024-09-13T00:35:33.202589+0000 mon.a (mon.0) 3280 : cluster [DBG] osdmap e1741: 8 total, 8 up, 8 in 2024-09-13T00:35:34.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:34 smithi136 bash[76371]: audit 2024-09-13T00:35:33.204828+0000 mon.c (mon.2) 138 : audit [INF] from='client.? 172.21.15.136:0/1086627647' entity='client.admin' cmd=[{"prefix": "osd tier remove", "pool": "test-librbd-smithi136-114798-28", "tierpool":"test-librbd-smithi136-114798-30"}]: dispatch 2024-09-13T00:35:34.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:34 smithi136 bash[76371]: audit 2024-09-13T00:35:33.208167+0000 mon.a (mon.0) 3281 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd tier remove", "pool": "test-librbd-smithi136-114798-28", "tierpool":"test-librbd-smithi136-114798-30"}]: dispatch 2024-09-13T00:35:34.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:34 smithi136 bash[76371]: cluster 2024-09-13T00:35:34.195929+0000 mon.a (mon.0) 3282 : cluster [INF] Health check cleared: CACHE_POOL_NO_HIT_SET (was: 1 cache pools are missing hit_sets) 2024-09-13T00:35:34.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:34 smithi136 bash[76371]: cluster 2024-09-13T00:35:34.195975+0000 mon.a (mon.0) 3283 : cluster [INF] Cluster is now healthy 2024-09-13T00:35:34.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:34 smithi136 bash[76371]: audit 2024-09-13T00:35:34.208379+0000 mon.a (mon.0) 3284 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier remove", "pool": "test-librbd-smithi136-114798-28", "tierpool":"test-librbd-smithi136-114798-30"}]': finished 2024-09-13T00:35:34.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:34 smithi136 bash[76371]: cluster 2024-09-13T00:35:34.208840+0000 mon.a (mon.0) 3285 : cluster [DBG] osdmap e1742: 8 total, 8 up, 8 in 2024-09-13T00:35:34.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:34 smithi181 bash[67587]: audit 2024-09-13T00:35:33.202408+0000 mon.a (mon.0) 3279 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi136-114798-28"}]': finished 2024-09-13T00:35:34.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:34 smithi181 bash[67587]: cluster 2024-09-13T00:35:33.202589+0000 mon.a (mon.0) 3280 : cluster [DBG] osdmap e1741: 8 total, 8 up, 8 in 2024-09-13T00:35:34.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:34 smithi181 bash[67587]: audit 2024-09-13T00:35:33.204828+0000 mon.c (mon.2) 138 : audit [INF] from='client.? 172.21.15.136:0/1086627647' entity='client.admin' cmd=[{"prefix": "osd tier remove", "pool": "test-librbd-smithi136-114798-28", "tierpool":"test-librbd-smithi136-114798-30"}]: dispatch 2024-09-13T00:35:34.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:34 smithi181 bash[67587]: audit 2024-09-13T00:35:33.208167+0000 mon.a (mon.0) 3281 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd tier remove", "pool": "test-librbd-smithi136-114798-28", "tierpool":"test-librbd-smithi136-114798-30"}]: dispatch 2024-09-13T00:35:34.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:34 smithi181 bash[67587]: cluster 2024-09-13T00:35:34.195929+0000 mon.a (mon.0) 3282 : cluster [INF] Health check cleared: CACHE_POOL_NO_HIT_SET (was: 1 cache pools are missing hit_sets) 2024-09-13T00:35:34.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:34 smithi181 bash[67587]: cluster 2024-09-13T00:35:34.195975+0000 mon.a (mon.0) 3283 : cluster [INF] Cluster is now healthy 2024-09-13T00:35:34.600 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:34 smithi181 bash[67587]: audit 2024-09-13T00:35:34.208379+0000 mon.a (mon.0) 3284 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier remove", "pool": "test-librbd-smithi136-114798-28", "tierpool":"test-librbd-smithi136-114798-30"}]': finished 2024-09-13T00:35:34.600 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:34 smithi181 bash[67587]: cluster 2024-09-13T00:35:34.208840+0000 mon.a (mon.0) 3285 : cluster [DBG] osdmap e1742: 8 total, 8 up, 8 in 2024-09-13T00:35:34.803 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.FlushAio (427 ms) 2024-09-13T00:35:34.803 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.FlushAioPP 2024-09-13T00:35:34.803 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:35:35.212 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.FlushAioPP (409 ms) 2024-09-13T00:35:35.212 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.SnapDiff 2024-09-13T00:35:35.212 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:35:35.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:35 smithi136 bash[75164]: cluster 2024-09-13T00:35:34.240378+0000 mgr.y (mgr.24661) 1753 : cluster [DBG] pgmap v3056: 649 pgs: 649 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 395 KiB/s rd, 9.2 KiB/s wr, 474 op/s 2024-09-13T00:35:35.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:35 smithi136 bash[75164]: cluster 2024-09-13T00:35:34.602453+0000 mon.a (mon.0) 3286 : cluster [DBG] osdmap e1743: 8 total, 8 up, 8 in 2024-09-13T00:35:35.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:35 smithi136 bash[76371]: cluster 2024-09-13T00:35:34.240378+0000 mgr.y (mgr.24661) 1753 : cluster [DBG] pgmap v3056: 649 pgs: 649 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 395 KiB/s rd, 9.2 KiB/s wr, 474 op/s 2024-09-13T00:35:35.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:35 smithi136 bash[76371]: cluster 2024-09-13T00:35:34.602453+0000 mon.a (mon.0) 3286 : cluster [DBG] osdmap e1743: 8 total, 8 up, 8 in 2024-09-13T00:35:35.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:35 smithi181 bash[67587]: cluster 2024-09-13T00:35:34.240378+0000 mgr.y (mgr.24661) 1753 : cluster [DBG] pgmap v3056: 649 pgs: 649 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 395 KiB/s rd, 9.2 KiB/s wr, 474 op/s 2024-09-13T00:35:35.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:35 smithi181 bash[67587]: cluster 2024-09-13T00:35:34.602453+0000 mon.a (mon.0) 3286 : cluster [DBG] osdmap e1743: 8 total, 8 up, 8 in 2024-09-13T00:35:37.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:37 smithi136 bash[75164]: cluster 2024-09-13T00:35:36.239659+0000 mon.a (mon.0) 3287 : cluster [DBG] osdmap e1744: 8 total, 8 up, 8 in 2024-09-13T00:35:37.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:37 smithi136 bash[75164]: cluster 2024-09-13T00:35:36.241911+0000 mgr.y (mgr.24661) 1754 : cluster [DBG] pgmap v3059: 649 pgs: 649 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 160 KiB/s rd, 2.5 KiB/s wr, 198 op/s 2024-09-13T00:35:37.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:37 smithi136 bash[76371]: cluster 2024-09-13T00:35:36.239659+0000 mon.a (mon.0) 3287 : cluster [DBG] osdmap e1744: 8 total, 8 up, 8 in 2024-09-13T00:35:37.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:37 smithi136 bash[76371]: cluster 2024-09-13T00:35:36.241911+0000 mgr.y (mgr.24661) 1754 : cluster [DBG] pgmap v3059: 649 pgs: 649 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 160 KiB/s rd, 2.5 KiB/s wr, 198 op/s 2024-09-13T00:35:37.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:37 smithi181 bash[67587]: cluster 2024-09-13T00:35:36.239659+0000 mon.a (mon.0) 3287 : cluster [DBG] osdmap e1744: 8 total, 8 up, 8 in 2024-09-13T00:35:37.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:37 smithi181 bash[67587]: cluster 2024-09-13T00:35:36.241911+0000 mgr.y (mgr.24661) 1754 : cluster [DBG] pgmap v3059: 649 pgs: 649 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 160 KiB/s rd, 2.5 KiB/s wr, 198 op/s 2024-09-13T00:35:38.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:38 smithi181 bash[67587]: cluster 2024-09-13T00:35:37.251064+0000 mon.a (mon.0) 3288 : cluster [DBG] osdmap e1745: 8 total, 8 up, 8 in 2024-09-13T00:35:38.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:38 smithi136 bash[75164]: cluster 2024-09-13T00:35:37.251064+0000 mon.a (mon.0) 3288 : cluster [DBG] osdmap e1745: 8 total, 8 up, 8 in 2024-09-13T00:35:38.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:38 smithi136 bash[76371]: cluster 2024-09-13T00:35:37.251064+0000 mon.a (mon.0) 3288 : cluster [DBG] osdmap e1745: 8 total, 8 up, 8 in 2024-09-13T00:35:39.500 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 512 2024-09-13T00:35:39.500 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.SnapDiff (4288 ms) 2024-09-13T00:35:39.500 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.ZeroLengthWrite 2024-09-13T00:35:39.500 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:35:39.554 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.ZeroLengthWrite (55 ms) 2024-09-13T00:35:39.554 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.ZeroLengthDiscard 2024-09-13T00:35:39.554 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:35:39.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:39 smithi181 bash[67587]: cluster 2024-09-13T00:35:38.243815+0000 mgr.y (mgr.24661) 1755 : cluster [DBG] pgmap v3061: 649 pgs: 649 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 173 KiB/s rd, 8.9 KiB/s wr, 230 op/s 2024-09-13T00:35:39.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:39 smithi181 bash[67587]: cluster 2024-09-13T00:35:38.254380+0000 mon.a (mon.0) 3289 : cluster [DBG] osdmap e1746: 8 total, 8 up, 8 in 2024-09-13T00:35:39.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:39 smithi181 bash[67587]: audit 2024-09-13T00:35:38.837690+0000 mon.a (mon.0) 3290 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:35:39.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:39 smithi181 bash[67587]: audit 2024-09-13T00:35:38.840517+0000 mon.a (mon.0) 3291 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:35:39.674 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.ZeroLengthDiscard (120 ms) 2024-09-13T00:35:39.674 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.ZeroLengthRead 2024-09-13T00:35:39.675 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:35:39.727 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.ZeroLengthRead (52 ms) 2024-09-13T00:35:39.727 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.LargeCacheRead 2024-09-13T00:35:39.727 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:35:39.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:35:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:35:39] "GET /metrics HTTP/1.1" 200 75533 "" "Prometheus/2.43.0" 2024-09-13T00:35:39.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:39 smithi136 bash[75164]: cluster 2024-09-13T00:35:38.243815+0000 mgr.y (mgr.24661) 1755 : cluster [DBG] pgmap v3061: 649 pgs: 649 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 173 KiB/s rd, 8.9 KiB/s wr, 230 op/s 2024-09-13T00:35:39.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:39 smithi136 bash[75164]: cluster 2024-09-13T00:35:38.254380+0000 mon.a (mon.0) 3289 : cluster [DBG] osdmap e1746: 8 total, 8 up, 8 in 2024-09-13T00:35:39.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:39 smithi136 bash[75164]: audit 2024-09-13T00:35:38.837690+0000 mon.a (mon.0) 3290 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:35:39.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:39 smithi136 bash[75164]: audit 2024-09-13T00:35:38.840517+0000 mon.a (mon.0) 3291 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:35:39.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:39 smithi136 bash[76371]: cluster 2024-09-13T00:35:38.243815+0000 mgr.y (mgr.24661) 1755 : cluster [DBG] pgmap v3061: 649 pgs: 649 active+clean; 135 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 173 KiB/s rd, 8.9 KiB/s wr, 230 op/s 2024-09-13T00:35:39.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:39 smithi136 bash[76371]: cluster 2024-09-13T00:35:38.254380+0000 mon.a (mon.0) 3289 : cluster [DBG] osdmap e1746: 8 total, 8 up, 8 in 2024-09-13T00:35:39.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:39 smithi136 bash[76371]: audit 2024-09-13T00:35:38.837690+0000 mon.a (mon.0) 3290 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:35:39.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:39 smithi136 bash[76371]: audit 2024-09-13T00:35:38.840517+0000 mon.a (mon.0) 3291 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:35:39.805 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.LargeCacheRead (79 ms) 2024-09-13T00:35:39.806 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestPendingAio 2024-09-13T00:35:39.806 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:35:40.130 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestPendingAio (325 ms) 2024-09-13T00:35:40.130 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.CompareAndWriteCopyup 2024-09-13T00:35:40.131 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:35:40.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:40 smithi181 bash[67587]: cluster 2024-09-13T00:35:39.308700+0000 mon.a (mon.0) 3292 : cluster [DBG] osdmap e1747: 8 total, 8 up, 8 in 2024-09-13T00:35:40.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:40 smithi181 bash[67587]: cluster 2024-09-13T00:35:39.609514+0000 mon.a (mon.0) 3293 : cluster [DBG] osdmap e1748: 8 total, 8 up, 8 in 2024-09-13T00:35:40.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:40 smithi136 bash[75164]: cluster 2024-09-13T00:35:39.308700+0000 mon.a (mon.0) 3292 : cluster [DBG] osdmap e1747: 8 total, 8 up, 8 in 2024-09-13T00:35:40.764 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:40 smithi136 bash[75164]: cluster 2024-09-13T00:35:39.609514+0000 mon.a (mon.0) 3293 : cluster [DBG] osdmap e1748: 8 total, 8 up, 8 in 2024-09-13T00:35:40.764 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:40 smithi136 bash[76371]: cluster 2024-09-13T00:35:39.308700+0000 mon.a (mon.0) 3292 : cluster [DBG] osdmap e1747: 8 total, 8 up, 8 in 2024-09-13T00:35:40.764 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:40 smithi136 bash[76371]: cluster 2024-09-13T00:35:39.609514+0000 mon.a (mon.0) 3293 : cluster [DBG] osdmap e1748: 8 total, 8 up, 8 in 2024-09-13T00:35:40.861 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:35:41.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:41 smithi181 bash[67587]: cluster 2024-09-13T00:35:40.245007+0000 mgr.y (mgr.24661) 1756 : cluster [DBG] pgmap v3065: 649 pgs: 649 active+clean; 137 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 201 KiB/s rd, 144 KiB/s wr, 404 op/s 2024-09-13T00:35:41.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:41 smithi181 bash[67587]: cluster 2024-09-13T00:35:40.620711+0000 mon.a (mon.0) 3294 : cluster [DBG] osdmap e1749: 8 total, 8 up, 8 in 2024-09-13T00:35:41.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:41 smithi136 bash[75164]: cluster 2024-09-13T00:35:40.245007+0000 mgr.y (mgr.24661) 1756 : cluster [DBG] pgmap v3065: 649 pgs: 649 active+clean; 137 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 201 KiB/s rd, 144 KiB/s wr, 404 op/s 2024-09-13T00:35:41.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:41 smithi136 bash[75164]: cluster 2024-09-13T00:35:40.620711+0000 mon.a (mon.0) 3294 : cluster [DBG] osdmap e1749: 8 total, 8 up, 8 in 2024-09-13T00:35:41.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:41 smithi136 bash[76371]: cluster 2024-09-13T00:35:40.245007+0000 mgr.y (mgr.24661) 1756 : cluster [DBG] pgmap v3065: 649 pgs: 649 active+clean; 137 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 201 KiB/s rd, 144 KiB/s wr, 404 op/s 2024-09-13T00:35:41.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:41 smithi136 bash[76371]: cluster 2024-09-13T00:35:40.620711+0000 mon.a (mon.0) 3294 : cluster [DBG] osdmap e1749: 8 total, 8 up, 8 in 2024-09-13T00:35:42.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:35:42 smithi181 bash[95505]: ts=2024-09-13T00:35:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:35:42.731 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.CompareAndWriteCopyup (2600 ms) 2024-09-13T00:35:42.731 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.CompareAndWriteCopyupMismatch 2024-09-13T00:35:42.731 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:35:43.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:42 smithi136 bash[75164]: cluster 2024-09-13T00:35:41.624181+0000 mon.a (mon.0) 3295 : cluster [DBG] osdmap e1750: 8 total, 8 up, 8 in 2024-09-13T00:35:43.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:42 smithi136 bash[75164]: cluster 2024-09-13T00:35:42.246968+0000 mgr.y (mgr.24661) 1757 : cluster [DBG] pgmap v3068: 649 pgs: 649 active+clean; 141 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 201 KiB/s rd, 206 KiB/s wr, 448 op/s 2024-09-13T00:35:43.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:42 smithi136 bash[76371]: cluster 2024-09-13T00:35:41.624181+0000 mon.a (mon.0) 3295 : cluster [DBG] osdmap e1750: 8 total, 8 up, 8 in 2024-09-13T00:35:43.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:42 smithi136 bash[76371]: cluster 2024-09-13T00:35:42.246968+0000 mgr.y (mgr.24661) 1757 : cluster [DBG] pgmap v3068: 649 pgs: 649 active+clean; 141 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 201 KiB/s rd, 206 KiB/s wr, 448 op/s 2024-09-13T00:35:43.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:42 smithi181 bash[67587]: cluster 2024-09-13T00:35:41.624181+0000 mon.a (mon.0) 3295 : cluster [DBG] osdmap e1750: 8 total, 8 up, 8 in 2024-09-13T00:35:43.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:42 smithi181 bash[67587]: cluster 2024-09-13T00:35:42.246968+0000 mgr.y (mgr.24661) 1757 : cluster [DBG] pgmap v3068: 649 pgs: 649 active+clean; 141 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 201 KiB/s rd, 206 KiB/s wr, 448 op/s 2024-09-13T00:35:43.883 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:35:44.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:43 smithi136 bash[75164]: cluster 2024-09-13T00:35:42.634301+0000 mon.a (mon.0) 3296 : cluster [DBG] osdmap e1751: 8 total, 8 up, 8 in 2024-09-13T00:35:44.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:43 smithi136 bash[76371]: cluster 2024-09-13T00:35:42.634301+0000 mon.a (mon.0) 3296 : cluster [DBG] osdmap e1751: 8 total, 8 up, 8 in 2024-09-13T00:35:44.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:43 smithi181 bash[67587]: cluster 2024-09-13T00:35:42.634301+0000 mon.a (mon.0) 3296 : cluster [DBG] osdmap e1751: 8 total, 8 up, 8 in 2024-09-13T00:35:45.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:44 smithi136 bash[75164]: cluster 2024-09-13T00:35:43.633964+0000 mon.a (mon.0) 3297 : cluster [DBG] osdmap e1752: 8 total, 8 up, 8 in 2024-09-13T00:35:45.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:44 smithi136 bash[75164]: cluster 2024-09-13T00:35:44.248119+0000 mgr.y (mgr.24661) 1758 : cluster [DBG] pgmap v3071: 649 pgs: 649 active+clean; 141 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 186 KiB/s rd, 80 KiB/s wr, 301 op/s 2024-09-13T00:35:45.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:44 smithi136 bash[76371]: cluster 2024-09-13T00:35:43.633964+0000 mon.a (mon.0) 3297 : cluster [DBG] osdmap e1752: 8 total, 8 up, 8 in 2024-09-13T00:35:45.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:44 smithi136 bash[76371]: cluster 2024-09-13T00:35:44.248119+0000 mgr.y (mgr.24661) 1758 : cluster [DBG] pgmap v3071: 649 pgs: 649 active+clean; 141 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 186 KiB/s rd, 80 KiB/s wr, 301 op/s 2024-09-13T00:35:45.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:44 smithi181 bash[67587]: cluster 2024-09-13T00:35:43.633964+0000 mon.a (mon.0) 3297 : cluster [DBG] osdmap e1752: 8 total, 8 up, 8 in 2024-09-13T00:35:45.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:44 smithi181 bash[67587]: cluster 2024-09-13T00:35:44.248119+0000 mgr.y (mgr.24661) 1758 : cluster [DBG] pgmap v3071: 649 pgs: 649 active+clean; 141 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 186 KiB/s rd, 80 KiB/s wr, 301 op/s 2024-09-13T00:35:45.760 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.CompareAndWriteCopyupMismatch (3028 ms) 2024-09-13T00:35:45.760 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.Flatten 2024-09-13T00:35:45.760 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:35:46.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:45 smithi136 bash[75164]: cluster 2024-09-13T00:35:44.663728+0000 mon.a (mon.0) 3298 : cluster [DBG] osdmap e1753: 8 total, 8 up, 8 in 2024-09-13T00:35:46.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:45 smithi136 bash[76371]: cluster 2024-09-13T00:35:44.663728+0000 mon.a (mon.0) 3298 : cluster [DBG] osdmap e1753: 8 total, 8 up, 8 in 2024-09-13T00:35:46.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:45 smithi181 bash[67587]: cluster 2024-09-13T00:35:44.663728+0000 mon.a (mon.0) 3298 : cluster [DBG] osdmap e1753: 8 total, 8 up, 8 in 2024-09-13T00:35:46.916 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.Flatten (1157 ms) 2024-09-13T00:35:46.916 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.Sparsify 2024-09-13T00:35:46.916 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:35:47.000 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 32768 2024-09-13T00:35:47.000 INFO:tasks.workunit.client.0.smithi136.stdout:read: 32768 2024-09-13T00:35:47.001 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.Sparsify (85 ms) 2024-09-13T00:35:47.001 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.SparsifyPP 2024-09-13T00:35:47.001 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:35:47.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:46 smithi136 bash[75164]: cluster 2024-09-13T00:35:45.672549+0000 mon.a (mon.0) 3299 : cluster [DBG] osdmap e1754: 8 total, 8 up, 8 in 2024-09-13T00:35:47.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:46 smithi136 bash[75164]: cluster 2024-09-13T00:35:46.250655+0000 mgr.y (mgr.24661) 1759 : cluster [DBG] pgmap v3074: 649 pgs: 649 active+clean; 143 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 717 KiB/s rd, 530 KiB/s wr, 275 op/s 2024-09-13T00:35:47.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:46 smithi136 bash[76371]: cluster 2024-09-13T00:35:45.672549+0000 mon.a (mon.0) 3299 : cluster [DBG] osdmap e1754: 8 total, 8 up, 8 in 2024-09-13T00:35:47.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:46 smithi136 bash[76371]: cluster 2024-09-13T00:35:46.250655+0000 mgr.y (mgr.24661) 1759 : cluster [DBG] pgmap v3074: 649 pgs: 649 active+clean; 143 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 717 KiB/s rd, 530 KiB/s wr, 275 op/s 2024-09-13T00:35:47.083 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.SparsifyPP (83 ms) 2024-09-13T00:35:47.084 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.SnapshotLimit 2024-09-13T00:35:47.084 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:35:47.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:46 smithi181 bash[67587]: cluster 2024-09-13T00:35:45.672549+0000 mon.a (mon.0) 3299 : cluster [DBG] osdmap e1754: 8 total, 8 up, 8 in 2024-09-13T00:35:47.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:46 smithi181 bash[67587]: cluster 2024-09-13T00:35:46.250655+0000 mgr.y (mgr.24661) 1759 : cluster [DBG] pgmap v3074: 649 pgs: 649 active+clean; 143 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 717 KiB/s rd, 530 KiB/s wr, 275 op/s 2024-09-13T00:35:48.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:47 smithi136 bash[75164]: cluster 2024-09-13T00:35:46.691325+0000 mon.a (mon.0) 3300 : cluster [DBG] osdmap e1755: 8 total, 8 up, 8 in 2024-09-13T00:35:48.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:47 smithi136 bash[76371]: cluster 2024-09-13T00:35:46.691325+0000 mon.a (mon.0) 3300 : cluster [DBG] osdmap e1755: 8 total, 8 up, 8 in 2024-09-13T00:35:48.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:47 smithi181 bash[67587]: cluster 2024-09-13T00:35:46.691325+0000 mon.a (mon.0) 3300 : cluster [DBG] osdmap e1755: 8 total, 8 up, 8 in 2024-09-13T00:35:49.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:48 smithi136 bash[75164]: cluster 2024-09-13T00:35:47.691723+0000 mon.a (mon.0) 3301 : cluster [DBG] osdmap e1756: 8 total, 8 up, 8 in 2024-09-13T00:35:49.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:48 smithi136 bash[75164]: cluster 2024-09-13T00:35:48.251829+0000 mgr.y (mgr.24661) 1760 : cluster [DBG] pgmap v3077: 649 pgs: 649 active+clean; 143 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 689 KiB/s rd, 534 KiB/s wr, 245 op/s 2024-09-13T00:35:49.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:48 smithi136 bash[76371]: cluster 2024-09-13T00:35:47.691723+0000 mon.a (mon.0) 3301 : cluster [DBG] osdmap e1756: 8 total, 8 up, 8 in 2024-09-13T00:35:49.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:48 smithi136 bash[76371]: cluster 2024-09-13T00:35:48.251829+0000 mgr.y (mgr.24661) 1760 : cluster [DBG] pgmap v3077: 649 pgs: 649 active+clean; 143 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 689 KiB/s rd, 534 KiB/s wr, 245 op/s 2024-09-13T00:35:49.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:48 smithi181 bash[67587]: cluster 2024-09-13T00:35:47.691723+0000 mon.a (mon.0) 3301 : cluster [DBG] osdmap e1756: 8 total, 8 up, 8 in 2024-09-13T00:35:49.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:48 smithi181 bash[67587]: cluster 2024-09-13T00:35:48.251829+0000 mgr.y (mgr.24661) 1760 : cluster [DBG] pgmap v3077: 649 pgs: 649 active+clean; 143 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 689 KiB/s rd, 534 KiB/s wr, 245 op/s 2024-09-13T00:35:49.696 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:35:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:35:49] "GET /metrics HTTP/1.1" 200 75533 "" "Prometheus/2.43.0" 2024-09-13T00:35:50.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:49 smithi136 bash[75164]: cluster 2024-09-13T00:35:48.703190+0000 mon.a (mon.0) 3302 : cluster [DBG] osdmap e1757: 8 total, 8 up, 8 in 2024-09-13T00:35:50.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:49 smithi136 bash[75164]: cluster 2024-09-13T00:35:49.613321+0000 mon.a (mon.0) 3303 : cluster [DBG] osdmap e1758: 8 total, 8 up, 8 in 2024-09-13T00:35:50.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:49 smithi136 bash[76371]: cluster 2024-09-13T00:35:48.703190+0000 mon.a (mon.0) 3302 : cluster [DBG] osdmap e1757: 8 total, 8 up, 8 in 2024-09-13T00:35:50.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:49 smithi136 bash[76371]: cluster 2024-09-13T00:35:49.613321+0000 mon.a (mon.0) 3303 : cluster [DBG] osdmap e1758: 8 total, 8 up, 8 in 2024-09-13T00:35:50.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:49 smithi181 bash[67587]: cluster 2024-09-13T00:35:48.703190+0000 mon.a (mon.0) 3302 : cluster [DBG] osdmap e1757: 8 total, 8 up, 8 in 2024-09-13T00:35:50.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:49 smithi181 bash[67587]: cluster 2024-09-13T00:35:49.613321+0000 mon.a (mon.0) 3303 : cluster [DBG] osdmap e1758: 8 total, 8 up, 8 in 2024-09-13T00:35:51.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:50 smithi136 bash[75164]: cluster 2024-09-13T00:35:50.253003+0000 mgr.y (mgr.24661) 1761 : cluster [DBG] pgmap v3080: 649 pgs: 649 active+clean; 143 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 167 KiB/s rd, 34 KiB/s wr, 213 op/s 2024-09-13T00:35:51.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:50 smithi136 bash[75164]: cluster 2024-09-13T00:35:50.617294+0000 mon.a (mon.0) 3304 : cluster [DBG] osdmap e1759: 8 total, 8 up, 8 in 2024-09-13T00:35:51.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:50 smithi136 bash[76371]: cluster 2024-09-13T00:35:50.253003+0000 mgr.y (mgr.24661) 1761 : cluster [DBG] pgmap v3080: 649 pgs: 649 active+clean; 143 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 167 KiB/s rd, 34 KiB/s wr, 213 op/s 2024-09-13T00:35:51.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:50 smithi136 bash[76371]: cluster 2024-09-13T00:35:50.617294+0000 mon.a (mon.0) 3304 : cluster [DBG] osdmap e1759: 8 total, 8 up, 8 in 2024-09-13T00:35:51.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:50 smithi181 bash[67587]: cluster 2024-09-13T00:35:50.253003+0000 mgr.y (mgr.24661) 1761 : cluster [DBG] pgmap v3080: 649 pgs: 649 active+clean; 143 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 167 KiB/s rd, 34 KiB/s wr, 213 op/s 2024-09-13T00:35:51.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:50 smithi181 bash[67587]: cluster 2024-09-13T00:35:50.617294+0000 mon.a (mon.0) 3304 : cluster [DBG] osdmap e1759: 8 total, 8 up, 8 in 2024-09-13T00:35:51.688 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.SnapshotLimit (4605 ms) 2024-09-13T00:35:51.689 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.SnapshotLimitPP 2024-09-13T00:35:51.689 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:35:52.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:35:52 smithi181 bash[95505]: ts=2024-09-13T00:35:52.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:35:53.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:52 smithi136 bash[75164]: cluster 2024-09-13T00:35:51.619659+0000 mon.a (mon.0) 3305 : cluster [DBG] osdmap e1760: 8 total, 8 up, 8 in 2024-09-13T00:35:53.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:52 smithi136 bash[75164]: cluster 2024-09-13T00:35:52.254849+0000 mgr.y (mgr.24661) 1762 : cluster [DBG] pgmap v3083: 649 pgs: 649 active+clean; 143 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 138 KiB/s rd, 28 KiB/s wr, 164 op/s 2024-09-13T00:35:53.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:52 smithi136 bash[76371]: cluster 2024-09-13T00:35:51.619659+0000 mon.a (mon.0) 3305 : cluster [DBG] osdmap e1760: 8 total, 8 up, 8 in 2024-09-13T00:35:53.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:52 smithi136 bash[76371]: cluster 2024-09-13T00:35:52.254849+0000 mgr.y (mgr.24661) 1762 : cluster [DBG] pgmap v3083: 649 pgs: 649 active+clean; 143 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 138 KiB/s rd, 28 KiB/s wr, 164 op/s 2024-09-13T00:35:53.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:52 smithi181 bash[67587]: cluster 2024-09-13T00:35:51.619659+0000 mon.a (mon.0) 3305 : cluster [DBG] osdmap e1760: 8 total, 8 up, 8 in 2024-09-13T00:35:53.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:52 smithi181 bash[67587]: cluster 2024-09-13T00:35:52.254849+0000 mgr.y (mgr.24661) 1762 : cluster [DBG] pgmap v3083: 649 pgs: 649 active+clean; 143 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 138 KiB/s rd, 28 KiB/s wr, 164 op/s 2024-09-13T00:35:54.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:53 smithi136 bash[75164]: cluster 2024-09-13T00:35:52.645566+0000 mon.a (mon.0) 3306 : cluster [DBG] osdmap e1761: 8 total, 8 up, 8 in 2024-09-13T00:35:54.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:53 smithi136 bash[76371]: cluster 2024-09-13T00:35:52.645566+0000 mon.a (mon.0) 3306 : cluster [DBG] osdmap e1761: 8 total, 8 up, 8 in 2024-09-13T00:35:54.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:53 smithi181 bash[67587]: cluster 2024-09-13T00:35:52.645566+0000 mon.a (mon.0) 3306 : cluster [DBG] osdmap e1761: 8 total, 8 up, 8 in 2024-09-13T00:35:55.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:54 smithi136 bash[75164]: cluster 2024-09-13T00:35:53.665313+0000 mon.a (mon.0) 3307 : cluster [DBG] osdmap e1762: 8 total, 8 up, 8 in 2024-09-13T00:35:55.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:54 smithi136 bash[75164]: audit 2024-09-13T00:35:53.831633+0000 mon.a (mon.0) 3308 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:35:55.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:54 smithi136 bash[75164]: cluster 2024-09-13T00:35:54.255973+0000 mgr.y (mgr.24661) 1763 : cluster [DBG] pgmap v3086: 649 pgs: 649 active+clean; 143 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 58 KiB/s rd, 12 KiB/s wr, 79 op/s 2024-09-13T00:35:55.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:54 smithi136 bash[76371]: cluster 2024-09-13T00:35:53.665313+0000 mon.a (mon.0) 3307 : cluster [DBG] osdmap e1762: 8 total, 8 up, 8 in 2024-09-13T00:35:55.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:54 smithi136 bash[76371]: audit 2024-09-13T00:35:53.831633+0000 mon.a (mon.0) 3308 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:35:55.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:54 smithi136 bash[76371]: cluster 2024-09-13T00:35:54.255973+0000 mgr.y (mgr.24661) 1763 : cluster [DBG] pgmap v3086: 649 pgs: 649 active+clean; 143 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 58 KiB/s rd, 12 KiB/s wr, 79 op/s 2024-09-13T00:35:55.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:54 smithi181 bash[67587]: cluster 2024-09-13T00:35:53.665313+0000 mon.a (mon.0) 3307 : cluster [DBG] osdmap e1762: 8 total, 8 up, 8 in 2024-09-13T00:35:55.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:54 smithi181 bash[67587]: audit 2024-09-13T00:35:53.831633+0000 mon.a (mon.0) 3308 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:35:55.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:54 smithi181 bash[67587]: cluster 2024-09-13T00:35:54.255973+0000 mgr.y (mgr.24661) 1763 : cluster [DBG] pgmap v3086: 649 pgs: 649 active+clean; 143 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 58 KiB/s rd, 12 KiB/s wr, 79 op/s 2024-09-13T00:35:56.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:55 smithi136 bash[75164]: cluster 2024-09-13T00:35:54.682473+0000 mon.a (mon.0) 3309 : cluster [DBG] osdmap e1763: 8 total, 8 up, 8 in 2024-09-13T00:35:56.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:55 smithi136 bash[76371]: cluster 2024-09-13T00:35:54.682473+0000 mon.a (mon.0) 3309 : cluster [DBG] osdmap e1763: 8 total, 8 up, 8 in 2024-09-13T00:35:56.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:55 smithi181 bash[67587]: cluster 2024-09-13T00:35:54.682473+0000 mon.a (mon.0) 3309 : cluster [DBG] osdmap e1763: 8 total, 8 up, 8 in 2024-09-13T00:35:56.755 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.SnapshotLimitPP (5067 ms) 2024-09-13T00:35:56.755 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.RebuildObjectMapViaLockOwner 2024-09-13T00:35:56.756 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:35:56.920 INFO:tasks.workunit.client.0.smithi136.stdout:0.00% done 2024-09-13T00:35:56.921 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.RebuildObjectMapViaLockOwner (165 ms) 2024-09-13T00:35:56.921 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.RenameViaLockOwner 2024-09-13T00:35:56.921 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:35:57.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:56 smithi136 bash[75164]: cluster 2024-09-13T00:35:55.681532+0000 mon.a (mon.0) 3310 : cluster [DBG] osdmap e1764: 8 total, 8 up, 8 in 2024-09-13T00:35:57.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:56 smithi136 bash[75164]: cluster 2024-09-13T00:35:56.258353+0000 mgr.y (mgr.24661) 1764 : cluster [DBG] pgmap v3089: 649 pgs: 649 active+clean; 143 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 20 KiB/s rd, 3.2 KiB/s wr, 30 op/s 2024-09-13T00:35:57.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:56 smithi136 bash[76371]: cluster 2024-09-13T00:35:55.681532+0000 mon.a (mon.0) 3310 : cluster [DBG] osdmap e1764: 8 total, 8 up, 8 in 2024-09-13T00:35:57.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:56 smithi136 bash[76371]: cluster 2024-09-13T00:35:56.258353+0000 mgr.y (mgr.24661) 1764 : cluster [DBG] pgmap v3089: 649 pgs: 649 active+clean; 143 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 20 KiB/s rd, 3.2 KiB/s wr, 30 op/s 2024-09-13T00:35:57.098 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.RenameViaLockOwner (177 ms) 2024-09-13T00:35:57.098 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.SnapCreateViaLockOwner 2024-09-13T00:35:57.098 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:35:57.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:56 smithi181 bash[67587]: cluster 2024-09-13T00:35:55.681532+0000 mon.a (mon.0) 3310 : cluster [DBG] osdmap e1764: 8 total, 8 up, 8 in 2024-09-13T00:35:57.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:56 smithi181 bash[67587]: cluster 2024-09-13T00:35:56.258353+0000 mgr.y (mgr.24661) 1764 : cluster [DBG] pgmap v3089: 649 pgs: 649 active+clean; 143 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 20 KiB/s rd, 3.2 KiB/s wr, 30 op/s 2024-09-13T00:35:57.816 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.SnapCreateViaLockOwner (718 ms) 2024-09-13T00:35:57.816 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.SnapRemoveViaLockOwner 2024-09-13T00:35:57.816 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:35:58.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:57 smithi136 bash[75164]: cluster 2024-09-13T00:35:56.707938+0000 mon.a (mon.0) 3311 : cluster [DBG] osdmap e1765: 8 total, 8 up, 8 in 2024-09-13T00:35:58.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:57 smithi136 bash[76371]: cluster 2024-09-13T00:35:56.707938+0000 mon.a (mon.0) 3311 : cluster [DBG] osdmap e1765: 8 total, 8 up, 8 in 2024-09-13T00:35:58.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:57 smithi181 bash[67587]: cluster 2024-09-13T00:35:56.707938+0000 mon.a (mon.0) 3311 : cluster [DBG] osdmap e1765: 8 total, 8 up, 8 in 2024-09-13T00:35:59.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:58 smithi136 bash[75164]: cluster 2024-09-13T00:35:57.716465+0000 mon.a (mon.0) 3312 : cluster [DBG] osdmap e1766: 8 total, 8 up, 8 in 2024-09-13T00:35:59.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:58 smithi136 bash[75164]: cluster 2024-09-13T00:35:58.259531+0000 mgr.y (mgr.24661) 1765 : cluster [DBG] pgmap v3092: 649 pgs: 649 active+clean; 143 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 27 KiB/s rd, 5.7 KiB/s wr, 43 op/s 2024-09-13T00:35:59.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:58 smithi136 bash[76371]: cluster 2024-09-13T00:35:57.716465+0000 mon.a (mon.0) 3312 : cluster [DBG] osdmap e1766: 8 total, 8 up, 8 in 2024-09-13T00:35:59.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:58 smithi136 bash[76371]: cluster 2024-09-13T00:35:58.259531+0000 mgr.y (mgr.24661) 1765 : cluster [DBG] pgmap v3092: 649 pgs: 649 active+clean; 143 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 27 KiB/s rd, 5.7 KiB/s wr, 43 op/s 2024-09-13T00:35:59.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:58 smithi181 bash[67587]: cluster 2024-09-13T00:35:57.716465+0000 mon.a (mon.0) 3312 : cluster [DBG] osdmap e1766: 8 total, 8 up, 8 in 2024-09-13T00:35:59.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:58 smithi181 bash[67587]: cluster 2024-09-13T00:35:58.259531+0000 mgr.y (mgr.24661) 1765 : cluster [DBG] pgmap v3092: 649 pgs: 649 active+clean; 143 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 27 KiB/s rd, 5.7 KiB/s wr, 43 op/s 2024-09-13T00:35:59.722 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:35:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:35:59] "GET /metrics HTTP/1.1" 200 75543 "" "Prometheus/2.43.0" 2024-09-13T00:35:59.737 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.SnapRemoveViaLockOwner (1921 ms) 2024-09-13T00:35:59.737 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.UpdateFeaturesViaLockOwner 2024-09-13T00:35:59.737 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:35:59.871 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.UpdateFeaturesViaLockOwner (133 ms) 2024-09-13T00:35:59.871 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.EnableJournalingViaLockOwner 2024-09-13T00:35:59.871 INFO:tasks.workunit.client.0.smithi136.stdout:SKIPPING 2024-09-13T00:35:59.871 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.EnableJournalingViaLockOwner (0 ms) 2024-09-13T00:35:59.871 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.SnapRemove2 2024-09-13T00:35:59.871 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:36:00.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:59 smithi136 bash[76371]: cluster 2024-09-13T00:35:58.729332+0000 mon.a (mon.0) 3313 : cluster [DBG] osdmap e1767: 8 total, 8 up, 8 in 2024-09-13T00:36:00.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:35:59 smithi136 bash[76371]: cluster 2024-09-13T00:35:59.617527+0000 mon.a (mon.0) 3314 : cluster [DBG] osdmap e1768: 8 total, 8 up, 8 in 2024-09-13T00:36:00.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:59 smithi136 bash[75164]: cluster 2024-09-13T00:35:58.729332+0000 mon.a (mon.0) 3313 : cluster [DBG] osdmap e1767: 8 total, 8 up, 8 in 2024-09-13T00:36:00.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:35:59 smithi136 bash[75164]: cluster 2024-09-13T00:35:59.617527+0000 mon.a (mon.0) 3314 : cluster [DBG] osdmap e1768: 8 total, 8 up, 8 in 2024-09-13T00:36:00.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:59 smithi181 bash[67587]: cluster 2024-09-13T00:35:58.729332+0000 mon.a (mon.0) 3313 : cluster [DBG] osdmap e1767: 8 total, 8 up, 8 in 2024-09-13T00:36:00.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:35:59 smithi181 bash[67587]: cluster 2024-09-13T00:35:59.617527+0000 mon.a (mon.0) 3314 : cluster [DBG] osdmap e1768: 8 total, 8 up, 8 in 2024-09-13T00:36:01.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:00 smithi136 bash[75164]: cluster 2024-09-13T00:36:00.260692+0000 mgr.y (mgr.24661) 1766 : cluster [DBG] pgmap v3095: 649 pgs: 649 active+clean; 143 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 49 KiB/s rd, 6.5 KiB/s wr, 70 op/s 2024-09-13T00:36:01.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:00 smithi136 bash[76371]: cluster 2024-09-13T00:36:00.260692+0000 mgr.y (mgr.24661) 1766 : cluster [DBG] pgmap v3095: 649 pgs: 649 active+clean; 143 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 49 KiB/s rd, 6.5 KiB/s wr, 70 op/s 2024-09-13T00:36:01.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:00 smithi181 bash[67587]: cluster 2024-09-13T00:36:00.260692+0000 mgr.y (mgr.24661) 1766 : cluster [DBG] pgmap v3095: 649 pgs: 649 active+clean; 143 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 49 KiB/s rd, 6.5 KiB/s wr, 70 op/s 2024-09-13T00:36:01.818 INFO:tasks.workunit.client.0.smithi136.stdout:100.00% done 2024-09-13T00:36:01.818 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.SnapRemove2 (1948 ms) 2024-09-13T00:36:01.818 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.SnapRenameViaLockOwner 2024-09-13T00:36:01.818 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:36:02.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:01 smithi136 bash[75164]: cluster 2024-09-13T00:36:00.741499+0000 mon.a (mon.0) 3315 : cluster [DBG] osdmap e1769: 8 total, 8 up, 8 in 2024-09-13T00:36:02.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:01 smithi136 bash[76371]: cluster 2024-09-13T00:36:00.741499+0000 mon.a (mon.0) 3315 : cluster [DBG] osdmap e1769: 8 total, 8 up, 8 in 2024-09-13T00:36:02.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:01 smithi181 bash[67587]: cluster 2024-09-13T00:36:00.741499+0000 mon.a (mon.0) 3315 : cluster [DBG] osdmap e1769: 8 total, 8 up, 8 in 2024-09-13T00:36:02.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:36:02 smithi181 bash[95505]: ts=2024-09-13T00:36:02.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:36:02.890 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.SnapRenameViaLockOwner (1071 ms) 2024-09-13T00:36:02.890 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.SnapProtectViaLockOwner 2024-09-13T00:36:02.890 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:36:03.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:02 smithi136 bash[75164]: cluster 2024-09-13T00:36:01.751919+0000 mon.a (mon.0) 3316 : cluster [DBG] osdmap e1770: 8 total, 8 up, 8 in 2024-09-13T00:36:03.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:02 smithi136 bash[75164]: cluster 2024-09-13T00:36:02.262513+0000 mgr.y (mgr.24661) 1767 : cluster [DBG] pgmap v3098: 649 pgs: 649 active+clean; 143 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 156 KiB/s rd, 8.5 KiB/s wr, 200 op/s 2024-09-13T00:36:03.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:02 smithi136 bash[76371]: cluster 2024-09-13T00:36:01.751919+0000 mon.a (mon.0) 3316 : cluster [DBG] osdmap e1770: 8 total, 8 up, 8 in 2024-09-13T00:36:03.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:02 smithi136 bash[76371]: cluster 2024-09-13T00:36:02.262513+0000 mgr.y (mgr.24661) 1767 : cluster [DBG] pgmap v3098: 649 pgs: 649 active+clean; 143 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 156 KiB/s rd, 8.5 KiB/s wr, 200 op/s 2024-09-13T00:36:03.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:02 smithi181 bash[67587]: cluster 2024-09-13T00:36:01.751919+0000 mon.a (mon.0) 3316 : cluster [DBG] osdmap e1770: 8 total, 8 up, 8 in 2024-09-13T00:36:03.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:02 smithi181 bash[67587]: cluster 2024-09-13T00:36:02.262513+0000 mgr.y (mgr.24661) 1767 : cluster [DBG] pgmap v3098: 649 pgs: 649 active+clean; 143 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 156 KiB/s rd, 8.5 KiB/s wr, 200 op/s 2024-09-13T00:36:03.887 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.SnapProtectViaLockOwner (998 ms) 2024-09-13T00:36:03.887 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.SnapUnprotectViaLockOwner 2024-09-13T00:36:03.888 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:36:04.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:03 smithi136 bash[75164]: cluster 2024-09-13T00:36:02.764480+0000 mon.a (mon.0) 3317 : cluster [DBG] osdmap e1771: 8 total, 8 up, 8 in 2024-09-13T00:36:04.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:03 smithi136 bash[76371]: cluster 2024-09-13T00:36:02.764480+0000 mon.a (mon.0) 3317 : cluster [DBG] osdmap e1771: 8 total, 8 up, 8 in 2024-09-13T00:36:04.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:03 smithi181 bash[67587]: cluster 2024-09-13T00:36:02.764480+0000 mon.a (mon.0) 3317 : cluster [DBG] osdmap e1771: 8 total, 8 up, 8 in 2024-09-13T00:36:04.755 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.SnapUnprotectViaLockOwner (868 ms) 2024-09-13T00:36:04.755 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.FlattenViaLockOwner 2024-09-13T00:36:04.756 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:36:05.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:04 smithi181 bash[67587]: cluster 2024-09-13T00:36:03.773404+0000 mon.a (mon.0) 3318 : cluster [DBG] osdmap e1772: 8 total, 8 up, 8 in 2024-09-13T00:36:05.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:04 smithi181 bash[67587]: cluster 2024-09-13T00:36:04.263640+0000 mgr.y (mgr.24661) 1768 : cluster [DBG] pgmap v3101: 649 pgs: 649 active+clean; 143 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 168 KiB/s rd, 12 KiB/s wr, 227 op/s 2024-09-13T00:36:05.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:04 smithi181 bash[67587]: cluster 2024-09-13T00:36:04.620144+0000 mon.a (mon.0) 3319 : cluster [DBG] osdmap e1773: 8 total, 8 up, 8 in 2024-09-13T00:36:05.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:04 smithi136 bash[75164]: cluster 2024-09-13T00:36:03.773404+0000 mon.a (mon.0) 3318 : cluster [DBG] osdmap e1772: 8 total, 8 up, 8 in 2024-09-13T00:36:05.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:04 smithi136 bash[75164]: cluster 2024-09-13T00:36:04.263640+0000 mgr.y (mgr.24661) 1768 : cluster [DBG] pgmap v3101: 649 pgs: 649 active+clean; 143 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 168 KiB/s rd, 12 KiB/s wr, 227 op/s 2024-09-13T00:36:05.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:04 smithi136 bash[75164]: cluster 2024-09-13T00:36:04.620144+0000 mon.a (mon.0) 3319 : cluster [DBG] osdmap e1773: 8 total, 8 up, 8 in 2024-09-13T00:36:05.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:04 smithi136 bash[76371]: cluster 2024-09-13T00:36:03.773404+0000 mon.a (mon.0) 3318 : cluster [DBG] osdmap e1772: 8 total, 8 up, 8 in 2024-09-13T00:36:05.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:04 smithi136 bash[76371]: cluster 2024-09-13T00:36:04.263640+0000 mgr.y (mgr.24661) 1768 : cluster [DBG] pgmap v3101: 649 pgs: 649 active+clean; 143 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 168 KiB/s rd, 12 KiB/s wr, 227 op/s 2024-09-13T00:36:05.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:04 smithi136 bash[76371]: cluster 2024-09-13T00:36:04.620144+0000 mon.a (mon.0) 3319 : cluster [DBG] osdmap e1773: 8 total, 8 up, 8 in 2024-09-13T00:36:06.090 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.FlattenViaLockOwner (1335 ms) 2024-09-13T00:36:06.090 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.ResizeViaLockOwner 2024-09-13T00:36:06.090 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:36:06.196 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.ResizeViaLockOwner (106 ms) 2024-09-13T00:36:06.196 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.SparsifyViaLockOwner 2024-09-13T00:36:06.197 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:36:06.297 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.SparsifyViaLockOwner (101 ms) 2024-09-13T00:36:06.298 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.ObjectMapConsistentSnap 2024-09-13T00:36:06.298 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:36:07.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:06 smithi181 bash[67587]: cluster 2024-09-13T00:36:05.786761+0000 mon.a (mon.0) 3320 : cluster [DBG] osdmap e1774: 8 total, 8 up, 8 in 2024-09-13T00:36:07.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:06 smithi181 bash[67587]: cluster 2024-09-13T00:36:06.265594+0000 mgr.y (mgr.24661) 1769 : cluster [DBG] pgmap v3104: 649 pgs: 649 active+clean; 143 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 90 KiB/s rd, 8.7 KiB/s wr, 129 op/s 2024-09-13T00:36:07.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:06 smithi136 bash[75164]: cluster 2024-09-13T00:36:05.786761+0000 mon.a (mon.0) 3320 : cluster [DBG] osdmap e1774: 8 total, 8 up, 8 in 2024-09-13T00:36:07.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:06 smithi136 bash[75164]: cluster 2024-09-13T00:36:06.265594+0000 mgr.y (mgr.24661) 1769 : cluster [DBG] pgmap v3104: 649 pgs: 649 active+clean; 143 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 90 KiB/s rd, 8.7 KiB/s wr, 129 op/s 2024-09-13T00:36:07.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:06 smithi136 bash[76371]: cluster 2024-09-13T00:36:05.786761+0000 mon.a (mon.0) 3320 : cluster [DBG] osdmap e1774: 8 total, 8 up, 8 in 2024-09-13T00:36:07.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:06 smithi136 bash[76371]: cluster 2024-09-13T00:36:06.265594+0000 mgr.y (mgr.24661) 1769 : cluster [DBG] pgmap v3104: 649 pgs: 649 active+clean; 143 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 90 KiB/s rd, 8.7 KiB/s wr, 129 op/s 2024-09-13T00:36:08.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:07 smithi181 bash[67587]: cluster 2024-09-13T00:36:06.796035+0000 mon.a (mon.0) 3321 : cluster [DBG] osdmap e1775: 8 total, 8 up, 8 in 2024-09-13T00:36:08.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:07 smithi136 bash[75164]: cluster 2024-09-13T00:36:06.796035+0000 mon.a (mon.0) 3321 : cluster [DBG] osdmap e1775: 8 total, 8 up, 8 in 2024-09-13T00:36:08.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:07 smithi136 bash[76371]: cluster 2024-09-13T00:36:06.796035+0000 mon.a (mon.0) 3321 : cluster [DBG] osdmap e1775: 8 total, 8 up, 8 in 2024-09-13T00:36:09.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:08 smithi181 bash[67587]: cluster 2024-09-13T00:36:07.798450+0000 mon.a (mon.0) 3322 : cluster [DBG] osdmap e1776: 8 total, 8 up, 8 in 2024-09-13T00:36:09.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:08 smithi181 bash[67587]: cluster 2024-09-13T00:36:08.266761+0000 mgr.y (mgr.24661) 1770 : cluster [DBG] pgmap v3107: 649 pgs: 649 active+clean; 143 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 202 KiB/s rd, 11 KiB/s wr, 261 op/s 2024-09-13T00:36:09.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:08 smithi136 bash[76371]: cluster 2024-09-13T00:36:07.798450+0000 mon.a (mon.0) 3322 : cluster [DBG] osdmap e1776: 8 total, 8 up, 8 in 2024-09-13T00:36:09.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:08 smithi136 bash[76371]: cluster 2024-09-13T00:36:08.266761+0000 mgr.y (mgr.24661) 1770 : cluster [DBG] pgmap v3107: 649 pgs: 649 active+clean; 143 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 202 KiB/s rd, 11 KiB/s wr, 261 op/s 2024-09-13T00:36:09.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:08 smithi136 bash[75164]: cluster 2024-09-13T00:36:07.798450+0000 mon.a (mon.0) 3322 : cluster [DBG] osdmap e1776: 8 total, 8 up, 8 in 2024-09-13T00:36:09.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:08 smithi136 bash[75164]: cluster 2024-09-13T00:36:08.266761+0000 mgr.y (mgr.24661) 1770 : cluster [DBG] pgmap v3107: 649 pgs: 649 active+clean; 143 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 202 KiB/s rd, 11 KiB/s wr, 261 op/s 2024-09-13T00:36:09.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:36:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:36:09] "GET /metrics HTTP/1.1" 200 75535 "" "Prometheus/2.43.0" 2024-09-13T00:36:10.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:09 smithi181 bash[67587]: cluster 2024-09-13T00:36:08.831584+0000 mon.a (mon.0) 3323 : cluster [DBG] osdmap e1777: 8 total, 8 up, 8 in 2024-09-13T00:36:10.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:09 smithi181 bash[67587]: audit 2024-09-13T00:36:08.834259+0000 mon.a (mon.0) 3324 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:36:10.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:09 smithi181 bash[67587]: cluster 2024-09-13T00:36:09.625388+0000 mon.a (mon.0) 3325 : cluster [DBG] osdmap e1778: 8 total, 8 up, 8 in 2024-09-13T00:36:10.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:09 smithi136 bash[75164]: cluster 2024-09-13T00:36:08.831584+0000 mon.a (mon.0) 3323 : cluster [DBG] osdmap e1777: 8 total, 8 up, 8 in 2024-09-13T00:36:10.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:09 smithi136 bash[75164]: audit 2024-09-13T00:36:08.834259+0000 mon.a (mon.0) 3324 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:36:10.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:09 smithi136 bash[75164]: cluster 2024-09-13T00:36:09.625388+0000 mon.a (mon.0) 3325 : cluster [DBG] osdmap e1778: 8 total, 8 up, 8 in 2024-09-13T00:36:10.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:09 smithi136 bash[76371]: cluster 2024-09-13T00:36:08.831584+0000 mon.a (mon.0) 3323 : cluster [DBG] osdmap e1777: 8 total, 8 up, 8 in 2024-09-13T00:36:10.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:09 smithi136 bash[76371]: audit 2024-09-13T00:36:08.834259+0000 mon.a (mon.0) 3324 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:36:10.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:09 smithi136 bash[76371]: cluster 2024-09-13T00:36:09.625388+0000 mon.a (mon.0) 3325 : cluster [DBG] osdmap e1778: 8 total, 8 up, 8 in 2024-09-13T00:36:11.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:10 smithi181 bash[67587]: cluster 2024-09-13T00:36:10.267868+0000 mgr.y (mgr.24661) 1771 : cluster [DBG] pgmap v3110: 649 pgs: 649 active+clean; 143 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 KiB/s rd, 10 KiB/s wr, 222 op/s 2024-09-13T00:36:11.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:10 smithi181 bash[67587]: audit 2024-09-13T00:36:10.304750+0000 mon.a (mon.0) 3326 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "102.0", "id": [6, 7]}]: dispatch 2024-09-13T00:36:11.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:10 smithi181 bash[67587]: audit 2024-09-13T00:36:10.305389+0000 mon.a (mon.0) 3327 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "106.15", "id": [2, 5]}]: dispatch 2024-09-13T00:36:11.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:10 smithi181 bash[67587]: audit 2024-09-13T00:36:10.627603+0000 mon.a (mon.0) 3328 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "102.0", "id": [6, 7]}]': finished 2024-09-13T00:36:11.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:10 smithi181 bash[67587]: audit 2024-09-13T00:36:10.627803+0000 mon.a (mon.0) 3329 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "106.15", "id": [2, 5]}]': finished 2024-09-13T00:36:11.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:10 smithi181 bash[67587]: cluster 2024-09-13T00:36:10.627932+0000 mon.a (mon.0) 3330 : cluster [DBG] osdmap e1779: 8 total, 8 up, 8 in 2024-09-13T00:36:11.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:10 smithi136 bash[75164]: cluster 2024-09-13T00:36:10.267868+0000 mgr.y (mgr.24661) 1771 : cluster [DBG] pgmap v3110: 649 pgs: 649 active+clean; 143 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 KiB/s rd, 10 KiB/s wr, 222 op/s 2024-09-13T00:36:11.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:10 smithi136 bash[75164]: audit 2024-09-13T00:36:10.304750+0000 mon.a (mon.0) 3326 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "102.0", "id": [6, 7]}]: dispatch 2024-09-13T00:36:11.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:10 smithi136 bash[75164]: audit 2024-09-13T00:36:10.305389+0000 mon.a (mon.0) 3327 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "106.15", "id": [2, 5]}]: dispatch 2024-09-13T00:36:11.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:10 smithi136 bash[75164]: audit 2024-09-13T00:36:10.627603+0000 mon.a (mon.0) 3328 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "102.0", "id": [6, 7]}]': finished 2024-09-13T00:36:11.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:10 smithi136 bash[75164]: audit 2024-09-13T00:36:10.627803+0000 mon.a (mon.0) 3329 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "106.15", "id": [2, 5]}]': finished 2024-09-13T00:36:11.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:10 smithi136 bash[75164]: cluster 2024-09-13T00:36:10.627932+0000 mon.a (mon.0) 3330 : cluster [DBG] osdmap e1779: 8 total, 8 up, 8 in 2024-09-13T00:36:11.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:10 smithi136 bash[76371]: cluster 2024-09-13T00:36:10.267868+0000 mgr.y (mgr.24661) 1771 : cluster [DBG] pgmap v3110: 649 pgs: 649 active+clean; 143 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 KiB/s rd, 10 KiB/s wr, 222 op/s 2024-09-13T00:36:11.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:10 smithi136 bash[76371]: audit 2024-09-13T00:36:10.304750+0000 mon.a (mon.0) 3326 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "102.0", "id": [6, 7]}]: dispatch 2024-09-13T00:36:11.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:10 smithi136 bash[76371]: audit 2024-09-13T00:36:10.305389+0000 mon.a (mon.0) 3327 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "106.15", "id": [2, 5]}]: dispatch 2024-09-13T00:36:11.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:10 smithi136 bash[76371]: audit 2024-09-13T00:36:10.627603+0000 mon.a (mon.0) 3328 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "102.0", "id": [6, 7]}]': finished 2024-09-13T00:36:11.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:10 smithi136 bash[76371]: audit 2024-09-13T00:36:10.627803+0000 mon.a (mon.0) 3329 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "106.15", "id": [2, 5]}]': finished 2024-09-13T00:36:11.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:10 smithi136 bash[76371]: cluster 2024-09-13T00:36:10.627932+0000 mon.a (mon.0) 3330 : cluster [DBG] osdmap e1779: 8 total, 8 up, 8 in 2024-09-13T00:36:12.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:36:12 smithi181 bash[95505]: ts=2024-09-13T00:36:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:36:13.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:12 smithi136 bash[75164]: cluster 2024-09-13T00:36:11.630272+0000 mon.a (mon.0) 3331 : cluster [DBG] osdmap e1780: 8 total, 8 up, 8 in 2024-09-13T00:36:13.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:12 smithi136 bash[75164]: cluster 2024-09-13T00:36:12.269716+0000 mgr.y (mgr.24661) 1772 : cluster [DBG] pgmap v3113: 649 pgs: 1 peering, 648 active+clean; 143 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 52 KiB/s rd, 5.2 KiB/s wr, 76 op/s 2024-09-13T00:36:13.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:12 smithi136 bash[76371]: cluster 2024-09-13T00:36:11.630272+0000 mon.a (mon.0) 3331 : cluster [DBG] osdmap e1780: 8 total, 8 up, 8 in 2024-09-13T00:36:13.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:12 smithi136 bash[76371]: cluster 2024-09-13T00:36:12.269716+0000 mgr.y (mgr.24661) 1772 : cluster [DBG] pgmap v3113: 649 pgs: 1 peering, 648 active+clean; 143 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 52 KiB/s rd, 5.2 KiB/s wr, 76 op/s 2024-09-13T00:36:13.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:12 smithi181 bash[67587]: cluster 2024-09-13T00:36:11.630272+0000 mon.a (mon.0) 3331 : cluster [DBG] osdmap e1780: 8 total, 8 up, 8 in 2024-09-13T00:36:13.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:12 smithi181 bash[67587]: cluster 2024-09-13T00:36:12.269716+0000 mgr.y (mgr.24661) 1772 : cluster [DBG] pgmap v3113: 649 pgs: 1 peering, 648 active+clean; 143 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 52 KiB/s rd, 5.2 KiB/s wr, 76 op/s 2024-09-13T00:36:14.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:13 smithi136 bash[75164]: cluster 2024-09-13T00:36:12.623795+0000 mon.a (mon.0) 3332 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-13T00:36:14.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:13 smithi136 bash[75164]: cluster 2024-09-13T00:36:12.642456+0000 mon.a (mon.0) 3333 : cluster [DBG] osdmap e1781: 8 total, 8 up, 8 in 2024-09-13T00:36:14.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:13 smithi136 bash[76371]: cluster 2024-09-13T00:36:12.623795+0000 mon.a (mon.0) 3332 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-13T00:36:14.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:13 smithi136 bash[76371]: cluster 2024-09-13T00:36:12.642456+0000 mon.a (mon.0) 3333 : cluster [DBG] osdmap e1781: 8 total, 8 up, 8 in 2024-09-13T00:36:14.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:13 smithi181 bash[67587]: cluster 2024-09-13T00:36:12.623795+0000 mon.a (mon.0) 3332 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-13T00:36:14.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:13 smithi181 bash[67587]: cluster 2024-09-13T00:36:12.642456+0000 mon.a (mon.0) 3333 : cluster [DBG] osdmap e1781: 8 total, 8 up, 8 in 2024-09-13T00:36:15.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:14 smithi136 bash[75164]: cluster 2024-09-13T00:36:13.649272+0000 mon.a (mon.0) 3334 : cluster [DBG] osdmap e1782: 8 total, 8 up, 8 in 2024-09-13T00:36:15.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:14 smithi136 bash[75164]: cluster 2024-09-13T00:36:14.271002+0000 mgr.y (mgr.24661) 1773 : cluster [DBG] pgmap v3116: 649 pgs: 1 peering, 648 active+clean; 143 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 86 KiB/s rd, 4.7 KiB/s wr, 113 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T00:36:15.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:14 smithi136 bash[76371]: cluster 2024-09-13T00:36:13.649272+0000 mon.a (mon.0) 3334 : cluster [DBG] osdmap e1782: 8 total, 8 up, 8 in 2024-09-13T00:36:15.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:14 smithi136 bash[76371]: cluster 2024-09-13T00:36:14.271002+0000 mgr.y (mgr.24661) 1773 : cluster [DBG] pgmap v3116: 649 pgs: 1 peering, 648 active+clean; 143 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 86 KiB/s rd, 4.7 KiB/s wr, 113 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T00:36:15.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:14 smithi181 bash[67587]: cluster 2024-09-13T00:36:13.649272+0000 mon.a (mon.0) 3334 : cluster [DBG] osdmap e1782: 8 total, 8 up, 8 in 2024-09-13T00:36:15.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:14 smithi181 bash[67587]: cluster 2024-09-13T00:36:14.271002+0000 mgr.y (mgr.24661) 1773 : cluster [DBG] pgmap v3116: 649 pgs: 1 peering, 648 active+clean; 143 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 86 KiB/s rd, 4.7 KiB/s wr, 113 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T00:36:16.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:15 smithi136 bash[75164]: cluster 2024-09-13T00:36:14.661173+0000 mon.a (mon.0) 3335 : cluster [DBG] osdmap e1783: 8 total, 8 up, 8 in 2024-09-13T00:36:16.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:15 smithi136 bash[76371]: cluster 2024-09-13T00:36:14.661173+0000 mon.a (mon.0) 3335 : cluster [DBG] osdmap e1783: 8 total, 8 up, 8 in 2024-09-13T00:36:16.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:15 smithi181 bash[67587]: cluster 2024-09-13T00:36:14.661173+0000 mon.a (mon.0) 3335 : cluster [DBG] osdmap e1783: 8 total, 8 up, 8 in 2024-09-13T00:36:17.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:16 smithi136 bash[75164]: cluster 2024-09-13T00:36:15.673473+0000 mon.a (mon.0) 3336 : cluster [DBG] osdmap e1784: 8 total, 8 up, 8 in 2024-09-13T00:36:17.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:16 smithi136 bash[75164]: cluster 2024-09-13T00:36:16.273712+0000 mgr.y (mgr.24661) 1774 : cluster [DBG] pgmap v3119: 649 pgs: 649 active+clean; 143 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 160 KiB/s rd, 43 KiB/s wr, 239 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T00:36:17.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:16 smithi136 bash[76371]: cluster 2024-09-13T00:36:15.673473+0000 mon.a (mon.0) 3336 : cluster [DBG] osdmap e1784: 8 total, 8 up, 8 in 2024-09-13T00:36:17.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:16 smithi136 bash[76371]: cluster 2024-09-13T00:36:16.273712+0000 mgr.y (mgr.24661) 1774 : cluster [DBG] pgmap v3119: 649 pgs: 649 active+clean; 143 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 160 KiB/s rd, 43 KiB/s wr, 239 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T00:36:17.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:16 smithi181 bash[67587]: cluster 2024-09-13T00:36:15.673473+0000 mon.a (mon.0) 3336 : cluster [DBG] osdmap e1784: 8 total, 8 up, 8 in 2024-09-13T00:36:17.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:16 smithi181 bash[67587]: cluster 2024-09-13T00:36:16.273712+0000 mgr.y (mgr.24661) 1774 : cluster [DBG] pgmap v3119: 649 pgs: 649 active+clean; 143 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 160 KiB/s rd, 43 KiB/s wr, 239 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T00:36:17.297 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.ObjectMapConsistentSnap (10999 ms) 2024-09-13T00:36:17.297 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.Metadata 2024-09-13T00:36:17.297 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:36:17.952 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.Metadata (656 ms) 2024-09-13T00:36:17.952 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.MetadataPP 2024-09-13T00:36:17.953 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:36:18.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:17 smithi136 bash[75164]: cluster 2024-09-13T00:36:16.665077+0000 mon.a (mon.0) 3337 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-13T00:36:18.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:17 smithi136 bash[75164]: cluster 2024-09-13T00:36:16.665154+0000 mon.a (mon.0) 3338 : cluster [INF] Cluster is now healthy 2024-09-13T00:36:18.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:17 smithi136 bash[76371]: cluster 2024-09-13T00:36:16.665077+0000 mon.a (mon.0) 3337 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-13T00:36:18.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:17 smithi136 bash[76371]: cluster 2024-09-13T00:36:16.665154+0000 mon.a (mon.0) 3338 : cluster [INF] Cluster is now healthy 2024-09-13T00:36:18.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:17 smithi181 bash[67587]: cluster 2024-09-13T00:36:16.665077+0000 mon.a (mon.0) 3337 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-13T00:36:18.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:17 smithi181 bash[67587]: cluster 2024-09-13T00:36:16.665154+0000 mon.a (mon.0) 3338 : cluster [INF] Cluster is now healthy 2024-09-13T00:36:19.004 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.MetadataPP (1051 ms) 2024-09-13T00:36:19.004 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.UpdateFeatures 2024-09-13T00:36:19.004 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:36:19.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:18 smithi136 bash[75164]: cluster 2024-09-13T00:36:17.684914+0000 mon.a (mon.0) 3339 : cluster [DBG] osdmap e1785: 8 total, 8 up, 8 in 2024-09-13T00:36:19.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:18 smithi136 bash[75164]: cluster 2024-09-13T00:36:18.274896+0000 mgr.y (mgr.24661) 1775 : cluster [DBG] pgmap v3121: 649 pgs: 649 active+clean; 143 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 90 KiB/s rd, 52 KiB/s wr, 161 op/s 2024-09-13T00:36:19.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:18 smithi136 bash[76371]: cluster 2024-09-13T00:36:17.684914+0000 mon.a (mon.0) 3339 : cluster [DBG] osdmap e1785: 8 total, 8 up, 8 in 2024-09-13T00:36:19.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:18 smithi136 bash[76371]: cluster 2024-09-13T00:36:18.274896+0000 mgr.y (mgr.24661) 1775 : cluster [DBG] pgmap v3121: 649 pgs: 649 active+clean; 143 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 90 KiB/s rd, 52 KiB/s wr, 161 op/s 2024-09-13T00:36:19.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:18 smithi181 bash[67587]: cluster 2024-09-13T00:36:17.684914+0000 mon.a (mon.0) 3339 : cluster [DBG] osdmap e1785: 8 total, 8 up, 8 in 2024-09-13T00:36:19.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:18 smithi181 bash[67587]: cluster 2024-09-13T00:36:18.274896+0000 mgr.y (mgr.24661) 1775 : cluster [DBG] pgmap v3121: 649 pgs: 649 active+clean; 143 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 90 KiB/s rd, 52 KiB/s wr, 161 op/s 2024-09-13T00:36:19.286 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.UpdateFeatures (282 ms) 2024-09-13T00:36:19.286 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.FeaturesBitmaskString 2024-09-13T00:36:19.286 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.FeaturesBitmaskString (0 ms) 2024-09-13T00:36:19.286 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.RebuildObjectMap 2024-09-13T00:36:19.286 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:36:19.724 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:36:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:36:19] "GET /metrics HTTP/1.1" 200 75535 "" "Prometheus/2.43.0" 2024-09-13T00:36:19.903 INFO:tasks.workunit.client.0.smithi136.stdout:0.00% done 2024-09-13T00:36:19.904 INFO:tasks.workunit.client.0.smithi136.stdout:25.00% done 2024-09-13T00:36:19.904 INFO:tasks.workunit.client.0.smithi136.stdout:50.00% done 2024-09-13T00:36:19.904 INFO:tasks.workunit.client.0.smithi136.stdout:75.00% done 2024-09-13T00:36:19.904 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.RebuildObjectMap (618 ms) 2024-09-13T00:36:19.904 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.RebuildNewObjectMap 2024-09-13T00:36:19.990 INFO:tasks.workunit.client.0.smithi136.stdout:0.00% done 2024-09-13T00:36:19.990 INFO:tasks.workunit.client.0.smithi136.stdout:25.00% done 2024-09-13T00:36:19.990 INFO:tasks.workunit.client.0.smithi136.stdout:50.00% done 2024-09-13T00:36:19.990 INFO:tasks.workunit.client.0.smithi136.stdout:75.00% done 2024-09-13T00:36:19.990 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.RebuildNewObjectMap (86 ms) 2024-09-13T00:36:19.990 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.CheckObjectMap 2024-09-13T00:36:19.990 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:36:20.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:19 smithi136 bash[75164]: cluster 2024-09-13T00:36:18.715807+0000 mon.a (mon.0) 3340 : cluster [DBG] osdmap e1786: 8 total, 8 up, 8 in 2024-09-13T00:36:20.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:19 smithi136 bash[76371]: cluster 2024-09-13T00:36:18.715807+0000 mon.a (mon.0) 3340 : cluster [DBG] osdmap e1786: 8 total, 8 up, 8 in 2024-09-13T00:36:20.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:19 smithi181 bash[67587]: cluster 2024-09-13T00:36:18.715807+0000 mon.a (mon.0) 3340 : cluster [DBG] osdmap e1786: 8 total, 8 up, 8 in 2024-09-13T00:36:20.855 INFO:tasks.workunit.client.0.smithi136.stdout:0.00% done 2024-09-13T00:36:20.855 INFO:tasks.workunit.client.0.smithi136.stdout:25.00% done 2024-09-13T00:36:20.855 INFO:tasks.workunit.client.0.smithi136.stdout:50.00% done 2024-09-13T00:36:20.856 INFO:tasks.workunit.client.0.smithi136.stdout:75.00% done 2024-09-13T00:36:20.856 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.CheckObjectMap (866 ms) 2024-09-13T00:36:20.856 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.BlockingAIO 2024-09-13T00:36:20.856 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:36:20.905 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.BlockingAIO (49 ms) 2024-09-13T00:36:20.905 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.ExclusiveLockTransition 2024-09-13T00:36:20.905 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:36:21.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:20 smithi136 bash[75164]: cluster 2024-09-13T00:36:19.715172+0000 mon.a (mon.0) 3341 : cluster [DBG] osdmap e1787: 8 total, 8 up, 8 in 2024-09-13T00:36:21.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:20 smithi136 bash[75164]: cluster 2024-09-13T00:36:20.276126+0000 mgr.y (mgr.24661) 1776 : cluster [DBG] pgmap v3124: 649 pgs: 649 active+clean; 143 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 198 KiB/s rd, 81 KiB/s wr, 324 op/s 2024-09-13T00:36:21.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:20 smithi136 bash[75164]: audit 2024-09-13T00:36:20.290551+0000 mon.a (mon.0) 3342 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:36:21.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:20 smithi136 bash[75164]: audit 2024-09-13T00:36:20.659927+0000 mon.a (mon.0) 3343 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:36:21.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:20 smithi136 bash[75164]: audit 2024-09-13T00:36:20.666487+0000 mon.a (mon.0) 3344 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:36:21.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:20 smithi136 bash[76371]: cluster 2024-09-13T00:36:19.715172+0000 mon.a (mon.0) 3341 : cluster [DBG] osdmap e1787: 8 total, 8 up, 8 in 2024-09-13T00:36:21.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:20 smithi136 bash[76371]: cluster 2024-09-13T00:36:20.276126+0000 mgr.y (mgr.24661) 1776 : cluster [DBG] pgmap v3124: 649 pgs: 649 active+clean; 143 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 198 KiB/s rd, 81 KiB/s wr, 324 op/s 2024-09-13T00:36:21.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:20 smithi136 bash[76371]: audit 2024-09-13T00:36:20.290551+0000 mon.a (mon.0) 3342 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:36:21.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:20 smithi136 bash[76371]: audit 2024-09-13T00:36:20.659927+0000 mon.a (mon.0) 3343 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:36:21.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:20 smithi136 bash[76371]: audit 2024-09-13T00:36:20.666487+0000 mon.a (mon.0) 3344 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:36:21.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:20 smithi181 bash[67587]: cluster 2024-09-13T00:36:19.715172+0000 mon.a (mon.0) 3341 : cluster [DBG] osdmap e1787: 8 total, 8 up, 8 in 2024-09-13T00:36:21.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:20 smithi181 bash[67587]: cluster 2024-09-13T00:36:20.276126+0000 mgr.y (mgr.24661) 1776 : cluster [DBG] pgmap v3124: 649 pgs: 649 active+clean; 143 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 198 KiB/s rd, 81 KiB/s wr, 324 op/s 2024-09-13T00:36:21.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:20 smithi181 bash[67587]: audit 2024-09-13T00:36:20.290551+0000 mon.a (mon.0) 3342 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:36:21.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:20 smithi181 bash[67587]: audit 2024-09-13T00:36:20.659927+0000 mon.a (mon.0) 3343 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:36:21.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:20 smithi181 bash[67587]: audit 2024-09-13T00:36:20.666487+0000 mon.a (mon.0) 3344 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:36:21.138 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.ExclusiveLockTransition (233 ms) 2024-09-13T00:36:21.138 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.ExclusiveLockReadTransition 2024-09-13T00:36:21.138 INFO:tasks.workunit.client.0.smithi136.stdout:SKIPPING 2024-09-13T00:36:21.138 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.ExclusiveLockReadTransition (0 ms) 2024-09-13T00:36:21.138 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.CacheMayCopyOnWrite 2024-09-13T00:36:21.138 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:36:21.913 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.CacheMayCopyOnWrite (776 ms) 2024-09-13T00:36:21.913 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.FlushEmptyOpsOnExternalSnapshot 2024-09-13T00:36:21.913 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:36:22.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:21 smithi136 bash[75164]: cluster 2024-09-13T00:36:20.738446+0000 mon.a (mon.0) 3345 : cluster [DBG] osdmap e1788: 8 total, 8 up, 8 in 2024-09-13T00:36:22.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:21 smithi136 bash[75164]: audit 2024-09-13T00:36:21.161029+0000 mon.a (mon.0) 3346 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:36:22.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:21 smithi136 bash[75164]: audit 2024-09-13T00:36:21.161825+0000 mon.a (mon.0) 3347 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:36:22.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:21 smithi136 bash[75164]: audit 2024-09-13T00:36:21.167054+0000 mon.a (mon.0) 3348 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:36:22.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:21 smithi136 bash[76371]: cluster 2024-09-13T00:36:20.738446+0000 mon.a (mon.0) 3345 : cluster [DBG] osdmap e1788: 8 total, 8 up, 8 in 2024-09-13T00:36:22.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:21 smithi136 bash[76371]: audit 2024-09-13T00:36:21.161029+0000 mon.a (mon.0) 3346 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:36:22.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:21 smithi136 bash[76371]: audit 2024-09-13T00:36:21.161825+0000 mon.a (mon.0) 3347 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:36:22.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:21 smithi136 bash[76371]: audit 2024-09-13T00:36:21.167054+0000 mon.a (mon.0) 3348 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:36:22.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:21 smithi181 bash[67587]: cluster 2024-09-13T00:36:20.738446+0000 mon.a (mon.0) 3345 : cluster [DBG] osdmap e1788: 8 total, 8 up, 8 in 2024-09-13T00:36:22.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:21 smithi181 bash[67587]: audit 2024-09-13T00:36:21.161029+0000 mon.a (mon.0) 3346 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:36:22.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:21 smithi181 bash[67587]: audit 2024-09-13T00:36:21.161825+0000 mon.a (mon.0) 3347 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:36:22.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:21 smithi181 bash[67587]: audit 2024-09-13T00:36:21.167054+0000 mon.a (mon.0) 3348 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:36:22.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:36:22 smithi181 bash[95505]: ts=2024-09-13T00:36:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:36:22.860 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.FlushEmptyOpsOnExternalSnapshot (947 ms) 2024-09-13T00:36:22.861 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestImageOptions 2024-09-13T00:36:23.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:22 smithi136 bash[75164]: cluster 2024-09-13T00:36:21.742459+0000 mon.a (mon.0) 3349 : cluster [DBG] osdmap e1789: 8 total, 8 up, 8 in 2024-09-13T00:36:23.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:22 smithi136 bash[75164]: cluster 2024-09-13T00:36:22.278013+0000 mgr.y (mgr.24661) 1777 : cluster [DBG] pgmap v3127: 649 pgs: 649 active+clean; 143 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 609 KiB/s rd, 179 KiB/s wr, 891 op/s 2024-09-13T00:36:23.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:22 smithi136 bash[76371]: cluster 2024-09-13T00:36:21.742459+0000 mon.a (mon.0) 3349 : cluster [DBG] osdmap e1789: 8 total, 8 up, 8 in 2024-09-13T00:36:23.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:22 smithi136 bash[76371]: cluster 2024-09-13T00:36:22.278013+0000 mgr.y (mgr.24661) 1777 : cluster [DBG] pgmap v3127: 649 pgs: 649 active+clean; 143 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 609 KiB/s rd, 179 KiB/s wr, 891 op/s 2024-09-13T00:36:23.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:22 smithi181 bash[67587]: cluster 2024-09-13T00:36:21.742459+0000 mon.a (mon.0) 3349 : cluster [DBG] osdmap e1789: 8 total, 8 up, 8 in 2024-09-13T00:36:23.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:22 smithi181 bash[67587]: cluster 2024-09-13T00:36:22.278013+0000 mgr.y (mgr.24661) 1777 : cluster [DBG] pgmap v3127: 649 pgs: 649 active+clean; 143 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 609 KiB/s rd, 179 KiB/s wr, 891 op/s 2024-09-13T00:36:24.083 INFO:tasks.workunit.client.0.smithi136.stdout:0.00% done 2024-09-13T00:36:24.083 INFO:tasks.workunit.client.0.smithi136.stdout:100.00% done 2024-09-13T00:36:24.083 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestImageOptions (1222 ms) 2024-09-13T00:36:24.083 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestImageOptionsPP 2024-09-13T00:36:24.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:23 smithi181 bash[67587]: cluster 2024-09-13T00:36:22.764944+0000 mon.a (mon.0) 3350 : cluster [DBG] osdmap e1790: 8 total, 8 up, 8 in 2024-09-13T00:36:24.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:23 smithi136 bash[75164]: cluster 2024-09-13T00:36:22.764944+0000 mon.a (mon.0) 3350 : cluster [DBG] osdmap e1790: 8 total, 8 up, 8 in 2024-09-13T00:36:24.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:23 smithi136 bash[76371]: cluster 2024-09-13T00:36:22.764944+0000 mon.a (mon.0) 3350 : cluster [DBG] osdmap e1790: 8 total, 8 up, 8 in 2024-09-13T00:36:25.053 INFO:tasks.workunit.client.0.smithi136.stdout:0.00% done 2024-09-13T00:36:25.053 INFO:tasks.workunit.client.0.smithi136.stdout:100.00% done 2024-09-13T00:36:25.053 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestImageOptionsPP (969 ms) 2024-09-13T00:36:25.053 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.EventSocketPipe 2024-09-13T00:36:25.053 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.EventSocketPipe (0 ms) 2024-09-13T00:36:25.053 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.EventSocketEventfd 2024-09-13T00:36:25.054 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.EventSocketEventfd (0 ms) 2024-09-13T00:36:25.054 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.ImagePollIO 2024-09-13T00:36:25.054 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:36:25.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:24 smithi181 bash[67587]: cluster 2024-09-13T00:36:23.770220+0000 mon.a (mon.0) 3351 : cluster [DBG] osdmap e1791: 8 total, 8 up, 8 in 2024-09-13T00:36:25.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:24 smithi181 bash[67587]: audit 2024-09-13T00:36:23.832217+0000 mon.a (mon.0) 3352 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:36:25.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:24 smithi181 bash[67587]: cluster 2024-09-13T00:36:24.279126+0000 mgr.y (mgr.24661) 1778 : cluster [DBG] pgmap v3130: 649 pgs: 649 active+clean; 143 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 562 KiB/s rd, 179 KiB/s wr, 800 op/s 2024-09-13T00:36:25.151 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 512 2024-09-13T00:36:25.151 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 512 2024-09-13T00:36:25.151 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 512 2024-09-13T00:36:25.151 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 512 2024-09-13T00:36:25.151 INFO:tasks.workunit.client.0.smithi136.stdout:wrote: 512 2024-09-13T00:36:25.151 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:36:25.151 INFO:tasks.workunit.client.0.smithi136.stdout:started write 2024-09-13T00:36:25.151 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:36:25.151 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:36:25.151 INFO:tasks.workunit.client.0.smithi136.stdout:finished write 2024-09-13T00:36:25.152 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:36:25.152 INFO:tasks.workunit.client.0.smithi136.stdout:started write 2024-09-13T00:36:25.152 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:36:25.152 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:36:25.152 INFO:tasks.workunit.client.0.smithi136.stdout:finished write 2024-09-13T00:36:25.152 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:36:25.152 INFO:tasks.workunit.client.0.smithi136.stdout:started write 2024-09-13T00:36:25.152 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:36:25.152 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:36:25.152 INFO:tasks.workunit.client.0.smithi136.stdout:finished write 2024-09-13T00:36:25.152 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:36:25.152 INFO:tasks.workunit.client.0.smithi136.stdout:started write 2024-09-13T00:36:25.152 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:36:25.153 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:36:25.153 INFO:tasks.workunit.client.0.smithi136.stdout:finished write 2024-09-13T00:36:25.153 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:36:25.153 INFO:tasks.workunit.client.0.smithi136.stdout:started write 2024-09-13T00:36:25.153 INFO:tasks.workunit.client.0.smithi136.stdout:write completion cb called! 2024-09-13T00:36:25.153 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 0 2024-09-13T00:36:25.153 INFO:tasks.workunit.client.0.smithi136.stdout:finished write 2024-09-13T00:36:25.153 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:36:25.153 INFO:tasks.workunit.client.0.smithi136.stdout:started read 2024-09-13T00:36:25.153 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:36:25.153 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 512 2024-09-13T00:36:25.153 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:36:25.153 INFO:tasks.workunit.client.0.smithi136.stdout:started read 2024-09-13T00:36:25.153 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:36:25.154 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 512 2024-09-13T00:36:25.154 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:36:25.154 INFO:tasks.workunit.client.0.smithi136.stdout:started read 2024-09-13T00:36:25.154 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:36:25.154 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 512 2024-09-13T00:36:25.154 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:36:25.154 INFO:tasks.workunit.client.0.smithi136.stdout:started read 2024-09-13T00:36:25.154 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:36:25.154 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 512 2024-09-13T00:36:25.154 INFO:tasks.workunit.client.0.smithi136.stdout:created completion 2024-09-13T00:36:25.154 INFO:tasks.workunit.client.0.smithi136.stdout:started read 2024-09-13T00:36:25.154 INFO:tasks.workunit.client.0.smithi136.stdout:read completion cb called! 2024-09-13T00:36:25.154 INFO:tasks.workunit.client.0.smithi136.stdout:return value is: 512 2024-09-13T00:36:25.155 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.ImagePollIO (98 ms) 2024-09-13T00:36:25.155 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.Mirror 2024-09-13T00:36:25.161 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:36:25.183 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:36:25.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:24 smithi136 bash[75164]: cluster 2024-09-13T00:36:23.770220+0000 mon.a (mon.0) 3351 : cluster [DBG] osdmap e1791: 8 total, 8 up, 8 in 2024-09-13T00:36:25.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:24 smithi136 bash[75164]: audit 2024-09-13T00:36:23.832217+0000 mon.a (mon.0) 3352 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:36:25.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:24 smithi136 bash[75164]: cluster 2024-09-13T00:36:24.279126+0000 mgr.y (mgr.24661) 1778 : cluster [DBG] pgmap v3130: 649 pgs: 649 active+clean; 143 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 562 KiB/s rd, 179 KiB/s wr, 800 op/s 2024-09-13T00:36:25.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:24 smithi136 bash[76371]: cluster 2024-09-13T00:36:23.770220+0000 mon.a (mon.0) 3351 : cluster [DBG] osdmap e1791: 8 total, 8 up, 8 in 2024-09-13T00:36:25.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:24 smithi136 bash[76371]: audit 2024-09-13T00:36:23.832217+0000 mon.a (mon.0) 3352 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:36:25.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:24 smithi136 bash[76371]: cluster 2024-09-13T00:36:24.279126+0000 mgr.y (mgr.24661) 1778 : cluster [DBG] pgmap v3130: 649 pgs: 649 active+clean; 143 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 562 KiB/s rd, 179 KiB/s wr, 800 op/s 2024-09-13T00:36:26.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:25 smithi181 bash[67587]: cluster 2024-09-13T00:36:24.793541+0000 mon.a (mon.0) 3353 : cluster [DBG] osdmap e1792: 8 total, 8 up, 8 in 2024-09-13T00:36:26.195 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.Mirror (1044 ms) 2024-09-13T00:36:26.195 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.MirrorPeerAttributes 2024-09-13T00:36:26.235 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.MirrorPeerAttributes (40 ms) 2024-09-13T00:36:26.235 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.CreateWithMirrorEnabled 2024-09-13T00:36:26.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:25 smithi136 bash[75164]: cluster 2024-09-13T00:36:24.793541+0000 mon.a (mon.0) 3353 : cluster [DBG] osdmap e1792: 8 total, 8 up, 8 in 2024-09-13T00:36:26.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:25 smithi136 bash[76371]: cluster 2024-09-13T00:36:24.793541+0000 mon.a (mon.0) 3353 : cluster [DBG] osdmap e1792: 8 total, 8 up, 8 in 2024-09-13T00:36:27.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:26 smithi181 bash[67587]: cluster 2024-09-13T00:36:25.795287+0000 mon.a (mon.0) 3354 : cluster [DBG] osdmap e1793: 8 total, 8 up, 8 in 2024-09-13T00:36:27.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:26 smithi181 bash[67587]: audit 2024-09-13T00:36:26.132219+0000 mon.a (mon.0) 3355 : audit [INF] from='client.? 172.21.15.136:0/983750906' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/90/uuid4"}]: dispatch 2024-09-13T00:36:27.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:26 smithi181 bash[67587]: audit 2024-09-13T00:36:26.138815+0000 mon.a (mon.0) 3356 : audit [INF] from='client.? 172.21.15.136:0/983750906' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/90/uuid4"}]': finished 2024-09-13T00:36:27.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:26 smithi181 bash[67587]: audit 2024-09-13T00:36:26.143901+0000 mon.a (mon.0) 3357 : audit [INF] from='client.? 172.21.15.136:0/983750906' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/90/3de556e7-c076-4108-9f2a-3afe851c9ceb"}]: dispatch 2024-09-13T00:36:27.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:26 smithi181 bash[67587]: audit 2024-09-13T00:36:26.149989+0000 mon.a (mon.0) 3358 : audit [INF] from='client.? 172.21.15.136:0/983750906' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/90/3de556e7-c076-4108-9f2a-3afe851c9ceb"}]': finished 2024-09-13T00:36:27.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:26 smithi181 bash[67587]: audit 2024-09-13T00:36:26.166678+0000 mon.a (mon.0) 3359 : audit [INF] from='client.? 172.21.15.136:0/983750906' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/90/a1377219-c7f0-48c8-bdd7-467429ee99f4"}]: dispatch 2024-09-13T00:36:27.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:26 smithi181 bash[67587]: audit 2024-09-13T00:36:26.172965+0000 mon.a (mon.0) 3360 : audit [INF] from='client.? 172.21.15.136:0/983750906' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/90/a1377219-c7f0-48c8-bdd7-467429ee99f4"}]': finished 2024-09-13T00:36:27.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:26 smithi181 bash[67587]: audit 2024-09-13T00:36:26.178053+0000 mon.a (mon.0) 3361 : audit [INF] from='client.? 172.21.15.136:0/983750906' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/90/b4dbc6dd-65b8-40e0-b11c-ce37a171ce21"}]: dispatch 2024-09-13T00:36:27.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:26 smithi181 bash[67587]: audit 2024-09-13T00:36:26.184535+0000 mon.a (mon.0) 3362 : audit [INF] from='client.? 172.21.15.136:0/983750906' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/90/b4dbc6dd-65b8-40e0-b11c-ce37a171ce21"}]': finished 2024-09-13T00:36:27.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:26 smithi181 bash[67587]: audit 2024-09-13T00:36:26.206987+0000 mon.a (mon.0) 3363 : audit [DBG] from='client.? 172.21.15.136:0/983750906' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer/90/2e8c66f3-9dc2-4c42-9dc6-c7ce6624f52b"}]: dispatch 2024-09-13T00:36:27.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:26 smithi181 bash[67587]: audit 2024-09-13T00:36:26.217045+0000 mon.a (mon.0) 3364 : audit [INF] from='client.? 172.21.15.136:0/983750906' entity='client.admin' 2024-09-13T00:36:27.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:26 smithi181 bash[67587]: audit 2024-09-13T00:36:26.217995+0000 mon.a (mon.0) 3365 : audit [DBG] from='client.? 172.21.15.136:0/983750906' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer/90/2e8c66f3-9dc2-4c42-9dc6-c7ce6624f52b"}]: dispatch 2024-09-13T00:36:27.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:26 smithi181 bash[67587]: audit 2024-09-13T00:36:26.218898+0000 mon.a (mon.0) 3366 : audit [INF] from='client.? 172.21.15.136:0/983750906' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/90/2e8c66f3-9dc2-4c42-9dc6-c7ce6624f52b"}]: dispatch 2024-09-13T00:36:27.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:26 smithi181 bash[67587]: audit 2024-09-13T00:36:26.227322+0000 mon.a (mon.0) 3367 : audit [INF] from='client.? 172.21.15.136:0/983750906' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/90/2e8c66f3-9dc2-4c42-9dc6-c7ce6624f52b"}]': finished 2024-09-13T00:36:27.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:26 smithi181 bash[67587]: cluster 2024-09-13T00:36:26.281347+0000 mgr.y (mgr.24661) 1779 : cluster [DBG] pgmap v3133: 649 pgs: 649 active+clean; 148 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 302 KiB/s rd, 1.2 MiB/s wr, 412 op/s 2024-09-13T00:36:27.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:26 smithi136 bash[76371]: cluster 2024-09-13T00:36:25.795287+0000 mon.a (mon.0) 3354 : cluster [DBG] osdmap e1793: 8 total, 8 up, 8 in 2024-09-13T00:36:27.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:26 smithi136 bash[76371]: audit 2024-09-13T00:36:26.132219+0000 mon.a (mon.0) 3355 : audit [INF] from='client.? 172.21.15.136:0/983750906' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/90/uuid4"}]: dispatch 2024-09-13T00:36:27.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:26 smithi136 bash[76371]: audit 2024-09-13T00:36:26.138815+0000 mon.a (mon.0) 3356 : audit [INF] from='client.? 172.21.15.136:0/983750906' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/90/uuid4"}]': finished 2024-09-13T00:36:27.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:26 smithi136 bash[76371]: audit 2024-09-13T00:36:26.143901+0000 mon.a (mon.0) 3357 : audit [INF] from='client.? 172.21.15.136:0/983750906' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/90/3de556e7-c076-4108-9f2a-3afe851c9ceb"}]: dispatch 2024-09-13T00:36:27.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:26 smithi136 bash[76371]: audit 2024-09-13T00:36:26.149989+0000 mon.a (mon.0) 3358 : audit [INF] from='client.? 172.21.15.136:0/983750906' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/90/3de556e7-c076-4108-9f2a-3afe851c9ceb"}]': finished 2024-09-13T00:36:27.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:26 smithi136 bash[76371]: audit 2024-09-13T00:36:26.166678+0000 mon.a (mon.0) 3359 : audit [INF] from='client.? 172.21.15.136:0/983750906' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/90/a1377219-c7f0-48c8-bdd7-467429ee99f4"}]: dispatch 2024-09-13T00:36:27.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:26 smithi136 bash[76371]: audit 2024-09-13T00:36:26.172965+0000 mon.a (mon.0) 3360 : audit [INF] from='client.? 172.21.15.136:0/983750906' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/90/a1377219-c7f0-48c8-bdd7-467429ee99f4"}]': finished 2024-09-13T00:36:27.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:26 smithi136 bash[76371]: audit 2024-09-13T00:36:26.178053+0000 mon.a (mon.0) 3361 : audit [INF] from='client.? 172.21.15.136:0/983750906' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/90/b4dbc6dd-65b8-40e0-b11c-ce37a171ce21"}]: dispatch 2024-09-13T00:36:27.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:26 smithi136 bash[76371]: audit 2024-09-13T00:36:26.184535+0000 mon.a (mon.0) 3362 : audit [INF] from='client.? 172.21.15.136:0/983750906' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/90/b4dbc6dd-65b8-40e0-b11c-ce37a171ce21"}]': finished 2024-09-13T00:36:27.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:26 smithi136 bash[76371]: audit 2024-09-13T00:36:26.206987+0000 mon.a (mon.0) 3363 : audit [DBG] from='client.? 172.21.15.136:0/983750906' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer/90/2e8c66f3-9dc2-4c42-9dc6-c7ce6624f52b"}]: dispatch 2024-09-13T00:36:27.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:26 smithi136 bash[76371]: audit 2024-09-13T00:36:26.217045+0000 mon.a (mon.0) 3364 : audit [INF] from='client.? 172.21.15.136:0/983750906' entity='client.admin' 2024-09-13T00:36:27.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:26 smithi136 bash[76371]: audit 2024-09-13T00:36:26.217995+0000 mon.a (mon.0) 3365 : audit [DBG] from='client.? 172.21.15.136:0/983750906' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer/90/2e8c66f3-9dc2-4c42-9dc6-c7ce6624f52b"}]: dispatch 2024-09-13T00:36:27.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:26 smithi136 bash[76371]: audit 2024-09-13T00:36:26.218898+0000 mon.a (mon.0) 3366 : audit [INF] from='client.? 172.21.15.136:0/983750906' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/90/2e8c66f3-9dc2-4c42-9dc6-c7ce6624f52b"}]: dispatch 2024-09-13T00:36:27.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:26 smithi136 bash[76371]: audit 2024-09-13T00:36:26.227322+0000 mon.a (mon.0) 3367 : audit [INF] from='client.? 172.21.15.136:0/983750906' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/90/2e8c66f3-9dc2-4c42-9dc6-c7ce6624f52b"}]': finished 2024-09-13T00:36:27.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:26 smithi136 bash[76371]: cluster 2024-09-13T00:36:26.281347+0000 mgr.y (mgr.24661) 1779 : cluster [DBG] pgmap v3133: 649 pgs: 649 active+clean; 148 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 302 KiB/s rd, 1.2 MiB/s wr, 412 op/s 2024-09-13T00:36:27.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:26 smithi136 bash[75164]: cluster 2024-09-13T00:36:25.795287+0000 mon.a (mon.0) 3354 : cluster [DBG] osdmap e1793: 8 total, 8 up, 8 in 2024-09-13T00:36:27.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:26 smithi136 bash[75164]: audit 2024-09-13T00:36:26.132219+0000 mon.a (mon.0) 3355 : audit [INF] from='client.? 172.21.15.136:0/983750906' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/90/uuid4"}]: dispatch 2024-09-13T00:36:27.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:26 smithi136 bash[75164]: audit 2024-09-13T00:36:26.138815+0000 mon.a (mon.0) 3356 : audit [INF] from='client.? 172.21.15.136:0/983750906' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/90/uuid4"}]': finished 2024-09-13T00:36:27.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:26 smithi136 bash[75164]: audit 2024-09-13T00:36:26.143901+0000 mon.a (mon.0) 3357 : audit [INF] from='client.? 172.21.15.136:0/983750906' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/90/3de556e7-c076-4108-9f2a-3afe851c9ceb"}]: dispatch 2024-09-13T00:36:27.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:26 smithi136 bash[75164]: audit 2024-09-13T00:36:26.149989+0000 mon.a (mon.0) 3358 : audit [INF] from='client.? 172.21.15.136:0/983750906' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/90/3de556e7-c076-4108-9f2a-3afe851c9ceb"}]': finished 2024-09-13T00:36:27.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:26 smithi136 bash[75164]: audit 2024-09-13T00:36:26.166678+0000 mon.a (mon.0) 3359 : audit [INF] from='client.? 172.21.15.136:0/983750906' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/90/a1377219-c7f0-48c8-bdd7-467429ee99f4"}]: dispatch 2024-09-13T00:36:27.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:26 smithi136 bash[75164]: audit 2024-09-13T00:36:26.172965+0000 mon.a (mon.0) 3360 : audit [INF] from='client.? 172.21.15.136:0/983750906' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/90/a1377219-c7f0-48c8-bdd7-467429ee99f4"}]': finished 2024-09-13T00:36:27.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:26 smithi136 bash[75164]: audit 2024-09-13T00:36:26.178053+0000 mon.a (mon.0) 3361 : audit [INF] from='client.? 172.21.15.136:0/983750906' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/90/b4dbc6dd-65b8-40e0-b11c-ce37a171ce21"}]: dispatch 2024-09-13T00:36:27.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:26 smithi136 bash[75164]: audit 2024-09-13T00:36:26.184535+0000 mon.a (mon.0) 3362 : audit [INF] from='client.? 172.21.15.136:0/983750906' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/90/b4dbc6dd-65b8-40e0-b11c-ce37a171ce21"}]': finished 2024-09-13T00:36:27.262 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:26 smithi136 bash[75164]: audit 2024-09-13T00:36:26.206987+0000 mon.a (mon.0) 3363 : audit [DBG] from='client.? 172.21.15.136:0/983750906' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer/90/2e8c66f3-9dc2-4c42-9dc6-c7ce6624f52b"}]: dispatch 2024-09-13T00:36:27.262 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:26 smithi136 bash[75164]: audit 2024-09-13T00:36:26.217045+0000 mon.a (mon.0) 3364 : audit [INF] from='client.? 172.21.15.136:0/983750906' entity='client.admin' 2024-09-13T00:36:27.262 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:26 smithi136 bash[75164]: audit 2024-09-13T00:36:26.217995+0000 mon.a (mon.0) 3365 : audit [DBG] from='client.? 172.21.15.136:0/983750906' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer/90/2e8c66f3-9dc2-4c42-9dc6-c7ce6624f52b"}]: dispatch 2024-09-13T00:36:27.262 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:26 smithi136 bash[75164]: audit 2024-09-13T00:36:26.218898+0000 mon.a (mon.0) 3366 : audit [INF] from='client.? 172.21.15.136:0/983750906' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/90/2e8c66f3-9dc2-4c42-9dc6-c7ce6624f52b"}]: dispatch 2024-09-13T00:36:27.262 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:26 smithi136 bash[75164]: audit 2024-09-13T00:36:26.227322+0000 mon.a (mon.0) 3367 : audit [INF] from='client.? 172.21.15.136:0/983750906' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/90/2e8c66f3-9dc2-4c42-9dc6-c7ce6624f52b"}]': finished 2024-09-13T00:36:27.262 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:26 smithi136 bash[75164]: cluster 2024-09-13T00:36:26.281347+0000 mgr.y (mgr.24661) 1779 : cluster [DBG] pgmap v3133: 649 pgs: 649 active+clean; 148 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 302 KiB/s rd, 1.2 MiB/s wr, 412 op/s 2024-09-13T00:36:28.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:28 smithi136 bash[75164]: cluster 2024-09-13T00:36:27.237058+0000 mon.a (mon.0) 3368 : cluster [DBG] osdmap e1794: 8 total, 8 up, 8 in 2024-09-13T00:36:28.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:28 smithi136 bash[76371]: cluster 2024-09-13T00:36:27.237058+0000 mon.a (mon.0) 3368 : cluster [DBG] osdmap e1794: 8 total, 8 up, 8 in 2024-09-13T00:36:28.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:28 smithi181 bash[67587]: cluster 2024-09-13T00:36:27.237058+0000 mon.a (mon.0) 3368 : cluster [DBG] osdmap e1794: 8 total, 8 up, 8 in 2024-09-13T00:36:29.512 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:36:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:36:29] "GET /metrics HTTP/1.1" 200 75541 "" "Prometheus/2.43.0" 2024-09-13T00:36:29.512 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:29 smithi136 bash[75164]: cluster 2024-09-13T00:36:28.241186+0000 mon.a (mon.0) 3369 : cluster [DBG] osdmap e1795: 8 total, 8 up, 8 in 2024-09-13T00:36:29.512 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:29 smithi136 bash[75164]: cluster 2024-09-13T00:36:28.282618+0000 mgr.y (mgr.24661) 1780 : cluster [DBG] pgmap v3136: 649 pgs: 649 active+clean; 149 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 274 KiB/s rd, 1.6 MiB/s wr, 385 op/s 2024-09-13T00:36:29.513 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:29 smithi136 bash[76371]: cluster 2024-09-13T00:36:28.241186+0000 mon.a (mon.0) 3369 : cluster [DBG] osdmap e1795: 8 total, 8 up, 8 in 2024-09-13T00:36:29.513 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:29 smithi136 bash[76371]: cluster 2024-09-13T00:36:28.282618+0000 mgr.y (mgr.24661) 1780 : cluster [DBG] pgmap v3136: 649 pgs: 649 active+clean; 149 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 274 KiB/s rd, 1.6 MiB/s wr, 385 op/s 2024-09-13T00:36:29.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:29 smithi181 bash[67587]: cluster 2024-09-13T00:36:28.241186+0000 mon.a (mon.0) 3369 : cluster [DBG] osdmap e1795: 8 total, 8 up, 8 in 2024-09-13T00:36:29.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:29 smithi181 bash[67587]: cluster 2024-09-13T00:36:28.282618+0000 mgr.y (mgr.24661) 1780 : cluster [DBG] pgmap v3136: 649 pgs: 649 active+clean; 149 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 274 KiB/s rd, 1.6 MiB/s wr, 385 op/s 2024-09-13T00:36:30.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:30 smithi181 bash[67587]: cluster 2024-09-13T00:36:29.264631+0000 mon.a (mon.0) 3370 : cluster [DBG] osdmap e1796: 8 total, 8 up, 8 in 2024-09-13T00:36:30.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:30 smithi136 bash[75164]: cluster 2024-09-13T00:36:29.264631+0000 mon.a (mon.0) 3370 : cluster [DBG] osdmap e1796: 8 total, 8 up, 8 in 2024-09-13T00:36:30.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:30 smithi136 bash[76371]: cluster 2024-09-13T00:36:29.264631+0000 mon.a (mon.0) 3370 : cluster [DBG] osdmap e1796: 8 total, 8 up, 8 in 2024-09-13T00:36:31.374 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.CreateWithMirrorEnabled (5140 ms) 2024-09-13T00:36:31.374 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.FlushCacheWithCopyupOnExternalSnapshot 2024-09-13T00:36:31.374 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:36:31.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:31 smithi181 bash[67587]: cluster 2024-09-13T00:36:30.272323+0000 mon.a (mon.0) 3371 : cluster [DBG] osdmap e1797: 8 total, 8 up, 8 in 2024-09-13T00:36:31.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:31 smithi181 bash[67587]: cluster 2024-09-13T00:36:30.284161+0000 mgr.y (mgr.24661) 1781 : cluster [DBG] pgmap v3139: 649 pgs: 649 active+clean; 151 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 192 KiB/s rd, 984 KiB/s wr, 262 op/s 2024-09-13T00:36:31.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:31 smithi136 bash[75164]: cluster 2024-09-13T00:36:30.272323+0000 mon.a (mon.0) 3371 : cluster [DBG] osdmap e1797: 8 total, 8 up, 8 in 2024-09-13T00:36:31.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:31 smithi136 bash[75164]: cluster 2024-09-13T00:36:30.284161+0000 mgr.y (mgr.24661) 1781 : cluster [DBG] pgmap v3139: 649 pgs: 649 active+clean; 151 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 192 KiB/s rd, 984 KiB/s wr, 262 op/s 2024-09-13T00:36:31.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:31 smithi136 bash[76371]: cluster 2024-09-13T00:36:30.272323+0000 mon.a (mon.0) 3371 : cluster [DBG] osdmap e1797: 8 total, 8 up, 8 in 2024-09-13T00:36:31.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:31 smithi136 bash[76371]: cluster 2024-09-13T00:36:30.284161+0000 mgr.y (mgr.24661) 1781 : cluster [DBG] pgmap v3139: 649 pgs: 649 active+clean; 151 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 192 KiB/s rd, 984 KiB/s wr, 262 op/s 2024-09-13T00:36:32.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:32 smithi181 bash[67587]: cluster 2024-09-13T00:36:31.286684+0000 mon.a (mon.0) 3372 : cluster [DBG] osdmap e1798: 8 total, 8 up, 8 in 2024-09-13T00:36:32.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:36:32 smithi181 bash[95505]: ts=2024-09-13T00:36:32.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:36:32.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:32 smithi136 bash[75164]: cluster 2024-09-13T00:36:31.286684+0000 mon.a (mon.0) 3372 : cluster [DBG] osdmap e1798: 8 total, 8 up, 8 in 2024-09-13T00:36:32.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:32 smithi136 bash[76371]: cluster 2024-09-13T00:36:31.286684+0000 mon.a (mon.0) 3372 : cluster [DBG] osdmap e1798: 8 total, 8 up, 8 in 2024-09-13T00:36:33.378 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.FlushCacheWithCopyupOnExternalSnapshot (2004 ms) 2024-09-13T00:36:33.378 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.ExclusiveLock 2024-09-13T00:36:33.378 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:36:33.527 INFO:tasks.workunit.client.0.smithi136.stdout:1: acquiring exclusive lock 2024-09-13T00:36:33.527 INFO:tasks.workunit.client.0.smithi136.stdout:2: acquiring exclusive lock 2024-09-13T00:36:33.535 INFO:tasks.workunit.client.0.smithi136.stdout:2: exclusive lock acquired 2024-09-13T00:36:33.537 INFO:tasks.workunit.client.0.smithi136.stdout:2: releasing exclusive lock 2024-09-13T00:36:33.542 INFO:tasks.workunit.client.0.smithi136.stdout:2: exclusive lock released 2024-09-13T00:36:33.542 INFO:tasks.workunit.client.0.smithi136.stdout:2: acquiring exclusive lock 2024-09-13T00:36:33.550 INFO:tasks.workunit.client.0.smithi136.stdout:1: exclusive lock acquired 2024-09-13T00:36:33.590 INFO:tasks.workunit.client.0.smithi136.stdout:1: releasing exclusive lock 2024-09-13T00:36:33.594 INFO:tasks.workunit.client.0.smithi136.stdout:1: exclusive lock released 2024-09-13T00:36:33.595 INFO:tasks.workunit.client.0.smithi136.stdout:1: acquiring exclusive lock 2024-09-13T00:36:33.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:33 smithi181 bash[67587]: cluster 2024-09-13T00:36:32.286499+0000 mgr.y (mgr.24661) 1782 : cluster [DBG] pgmap v3141: 649 pgs: 649 active+clean; 151 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 235 KiB/s rd, 524 KiB/s wr, 311 op/s 2024-09-13T00:36:33.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:33 smithi181 bash[67587]: cluster 2024-09-13T00:36:32.314771+0000 mon.a (mon.0) 3373 : cluster [DBG] osdmap e1799: 8 total, 8 up, 8 in 2024-09-13T00:36:33.601 INFO:tasks.workunit.client.0.smithi136.stdout:2: exclusive lock acquired 2024-09-13T00:36:33.630 INFO:tasks.workunit.client.0.smithi136.stdout:2: releasing exclusive lock 2024-09-13T00:36:33.634 INFO:tasks.workunit.client.0.smithi136.stdout:2: exclusive lock released 2024-09-13T00:36:33.634 INFO:tasks.workunit.client.0.smithi136.stdout:2: acquiring exclusive lock 2024-09-13T00:36:33.641 INFO:tasks.workunit.client.0.smithi136.stdout:1: exclusive lock acquired 2024-09-13T00:36:33.663 INFO:tasks.workunit.client.0.smithi136.stdout:1: releasing exclusive lock 2024-09-13T00:36:33.667 INFO:tasks.workunit.client.0.smithi136.stdout:1: exclusive lock released 2024-09-13T00:36:33.667 INFO:tasks.workunit.client.0.smithi136.stdout:1: acquiring exclusive lock 2024-09-13T00:36:33.673 INFO:tasks.workunit.client.0.smithi136.stdout:2: exclusive lock acquired 2024-09-13T00:36:33.675 INFO:tasks.workunit.client.0.smithi136.stdout:2: releasing exclusive lock 2024-09-13T00:36:33.679 INFO:tasks.workunit.client.0.smithi136.stdout:2: exclusive lock released 2024-09-13T00:36:33.679 INFO:tasks.workunit.client.0.smithi136.stdout:2: acquiring exclusive lock 2024-09-13T00:36:33.685 INFO:tasks.workunit.client.0.smithi136.stdout:1: exclusive lock acquired 2024-09-13T00:36:33.713 INFO:tasks.workunit.client.0.smithi136.stdout:1: releasing exclusive lock 2024-09-13T00:36:33.718 INFO:tasks.workunit.client.0.smithi136.stdout:1: exclusive lock released 2024-09-13T00:36:33.718 INFO:tasks.workunit.client.0.smithi136.stdout:1: acquiring exclusive lock 2024-09-13T00:36:33.723 INFO:tasks.workunit.client.0.smithi136.stdout:2: exclusive lock acquired 2024-09-13T00:36:33.758 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:33 smithi136 bash[75164]: cluster 2024-09-13T00:36:32.286499+0000 mgr.y (mgr.24661) 1782 : cluster [DBG] pgmap v3141: 649 pgs: 649 active+clean; 151 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 235 KiB/s rd, 524 KiB/s wr, 311 op/s 2024-09-13T00:36:33.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:33 smithi136 bash[75164]: cluster 2024-09-13T00:36:32.314771+0000 mon.a (mon.0) 3373 : cluster [DBG] osdmap e1799: 8 total, 8 up, 8 in 2024-09-13T00:36:33.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:33 smithi136 bash[76371]: cluster 2024-09-13T00:36:32.286499+0000 mgr.y (mgr.24661) 1782 : cluster [DBG] pgmap v3141: 649 pgs: 649 active+clean; 151 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 235 KiB/s rd, 524 KiB/s wr, 311 op/s 2024-09-13T00:36:33.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:33 smithi136 bash[76371]: cluster 2024-09-13T00:36:32.314771+0000 mon.a (mon.0) 3373 : cluster [DBG] osdmap e1799: 8 total, 8 up, 8 in 2024-09-13T00:36:33.773 INFO:tasks.workunit.client.0.smithi136.stdout:2: releasing exclusive lock 2024-09-13T00:36:33.778 INFO:tasks.workunit.client.0.smithi136.stdout:2: exclusive lock released 2024-09-13T00:36:33.778 INFO:tasks.workunit.client.0.smithi136.stdout:2: acquiring exclusive lock 2024-09-13T00:36:33.784 INFO:tasks.workunit.client.0.smithi136.stdout:2: exclusive lock acquired 2024-09-13T00:36:33.832 INFO:tasks.workunit.client.0.smithi136.stdout:2: releasing exclusive lock 2024-09-13T00:36:33.836 INFO:tasks.workunit.client.0.smithi136.stdout:2: exclusive lock released 2024-09-13T00:36:33.842 INFO:tasks.workunit.client.0.smithi136.stdout:1: exclusive lock acquired 2024-09-13T00:36:33.878 INFO:tasks.workunit.client.0.smithi136.stdout:1: releasing exclusive lock 2024-09-13T00:36:33.885 INFO:tasks.workunit.client.0.smithi136.stdout:1: exclusive lock released 2024-09-13T00:36:33.885 INFO:tasks.workunit.client.0.smithi136.stdout:1: acquiring exclusive lock 2024-09-13T00:36:33.894 INFO:tasks.workunit.client.0.smithi136.stdout:1: exclusive lock acquired 2024-09-13T00:36:33.943 INFO:tasks.workunit.client.0.smithi136.stdout:1: releasing exclusive lock 2024-09-13T00:36:33.949 INFO:tasks.workunit.client.0.smithi136.stdout:1: exclusive lock released 2024-09-13T00:36:33.989 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.ExclusiveLock (610 ms) 2024-09-13T00:36:33.989 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.BreakLock 2024-09-13T00:36:34.027 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:36:34.436 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.BreakLock (447 ms) 2024-09-13T00:36:34.436 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.DiscardAfterWrite 2024-09-13T00:36:34.436 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:36:34.493 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.DiscardAfterWrite (58 ms) 2024-09-13T00:36:34.494 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.DefaultFeatures 2024-09-13T00:36:34.494 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.DefaultFeatures (0 ms) 2024-09-13T00:36:34.494 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestTrashMoveAndPurge 2024-09-13T00:36:34.494 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:36:34.656 INFO:tasks.workunit.client.0.smithi136.stdout:0.00% done 2024-09-13T00:36:34.656 INFO:tasks.workunit.client.0.smithi136.stdout:1.56% done 2024-09-13T00:36:34.656 INFO:tasks.workunit.client.0.smithi136.stdout:3.12% done 2024-09-13T00:36:34.656 INFO:tasks.workunit.client.0.smithi136.stdout:4.69% done 2024-09-13T00:36:34.656 INFO:tasks.workunit.client.0.smithi136.stdout:6.25% done 2024-09-13T00:36:34.656 INFO:tasks.workunit.client.0.smithi136.stdout:7.81% done 2024-09-13T00:36:34.656 INFO:tasks.workunit.client.0.smithi136.stdout:9.38% done 2024-09-13T00:36:34.656 INFO:tasks.workunit.client.0.smithi136.stdout:10.94% done 2024-09-13T00:36:34.656 INFO:tasks.workunit.client.0.smithi136.stdout:12.50% done 2024-09-13T00:36:34.656 INFO:tasks.workunit.client.0.smithi136.stdout:14.06% done 2024-09-13T00:36:34.657 INFO:tasks.workunit.client.0.smithi136.stdout:15.62% done 2024-09-13T00:36:34.657 INFO:tasks.workunit.client.0.smithi136.stdout:17.19% done 2024-09-13T00:36:34.657 INFO:tasks.workunit.client.0.smithi136.stdout:18.75% done 2024-09-13T00:36:34.657 INFO:tasks.workunit.client.0.smithi136.stdout:20.31% done 2024-09-13T00:36:34.657 INFO:tasks.workunit.client.0.smithi136.stdout:21.88% done 2024-09-13T00:36:34.657 INFO:tasks.workunit.client.0.smithi136.stdout:23.44% done 2024-09-13T00:36:34.657 INFO:tasks.workunit.client.0.smithi136.stdout:25.00% done 2024-09-13T00:36:34.657 INFO:tasks.workunit.client.0.smithi136.stdout:26.56% done 2024-09-13T00:36:34.657 INFO:tasks.workunit.client.0.smithi136.stdout:28.12% done 2024-09-13T00:36:34.657 INFO:tasks.workunit.client.0.smithi136.stdout:29.69% done 2024-09-13T00:36:34.657 INFO:tasks.workunit.client.0.smithi136.stdout:31.25% done 2024-09-13T00:36:34.657 INFO:tasks.workunit.client.0.smithi136.stdout:32.81% done 2024-09-13T00:36:34.658 INFO:tasks.workunit.client.0.smithi136.stdout:34.38% done 2024-09-13T00:36:34.658 INFO:tasks.workunit.client.0.smithi136.stdout:35.94% done 2024-09-13T00:36:34.658 INFO:tasks.workunit.client.0.smithi136.stdout:37.50% done 2024-09-13T00:36:34.658 INFO:tasks.workunit.client.0.smithi136.stdout:39.06% done 2024-09-13T00:36:34.658 INFO:tasks.workunit.client.0.smithi136.stdout:40.62% done 2024-09-13T00:36:34.658 INFO:tasks.workunit.client.0.smithi136.stdout:42.19% done 2024-09-13T00:36:34.658 INFO:tasks.workunit.client.0.smithi136.stdout:43.75% done 2024-09-13T00:36:34.658 INFO:tasks.workunit.client.0.smithi136.stdout:45.31% done 2024-09-13T00:36:34.658 INFO:tasks.workunit.client.0.smithi136.stdout:46.88% done 2024-09-13T00:36:34.658 INFO:tasks.workunit.client.0.smithi136.stdout:48.44% done 2024-09-13T00:36:34.658 INFO:tasks.workunit.client.0.smithi136.stdout:50.00% done 2024-09-13T00:36:34.658 INFO:tasks.workunit.client.0.smithi136.stdout:51.56% done 2024-09-13T00:36:34.659 INFO:tasks.workunit.client.0.smithi136.stdout:53.12% done 2024-09-13T00:36:34.659 INFO:tasks.workunit.client.0.smithi136.stdout:54.69% done 2024-09-13T00:36:34.659 INFO:tasks.workunit.client.0.smithi136.stdout:56.25% done 2024-09-13T00:36:34.659 INFO:tasks.workunit.client.0.smithi136.stdout:57.81% done 2024-09-13T00:36:34.659 INFO:tasks.workunit.client.0.smithi136.stdout:59.38% done 2024-09-13T00:36:34.659 INFO:tasks.workunit.client.0.smithi136.stdout:60.94% done 2024-09-13T00:36:34.659 INFO:tasks.workunit.client.0.smithi136.stdout:62.50% done 2024-09-13T00:36:34.659 INFO:tasks.workunit.client.0.smithi136.stdout:64.06% done 2024-09-13T00:36:34.659 INFO:tasks.workunit.client.0.smithi136.stdout:65.62% done 2024-09-13T00:36:34.659 INFO:tasks.workunit.client.0.smithi136.stdout:67.19% done 2024-09-13T00:36:34.659 INFO:tasks.workunit.client.0.smithi136.stdout:68.75% done 2024-09-13T00:36:34.659 INFO:tasks.workunit.client.0.smithi136.stdout:70.31% done 2024-09-13T00:36:34.660 INFO:tasks.workunit.client.0.smithi136.stdout:71.88% done 2024-09-13T00:36:34.660 INFO:tasks.workunit.client.0.smithi136.stdout:73.44% done 2024-09-13T00:36:34.660 INFO:tasks.workunit.client.0.smithi136.stdout:75.00% done 2024-09-13T00:36:34.660 INFO:tasks.workunit.client.0.smithi136.stdout:76.56% done 2024-09-13T00:36:34.660 INFO:tasks.workunit.client.0.smithi136.stdout:78.12% done 2024-09-13T00:36:34.660 INFO:tasks.workunit.client.0.smithi136.stdout:79.69% done 2024-09-13T00:36:34.660 INFO:tasks.workunit.client.0.smithi136.stdout:81.25% done 2024-09-13T00:36:34.660 INFO:tasks.workunit.client.0.smithi136.stdout:82.81% done 2024-09-13T00:36:34.660 INFO:tasks.workunit.client.0.smithi136.stdout:84.38% done 2024-09-13T00:36:34.661 INFO:tasks.workunit.client.0.smithi136.stdout:85.94% done 2024-09-13T00:36:34.661 INFO:tasks.workunit.client.0.smithi136.stdout:87.50% done 2024-09-13T00:36:34.661 INFO:tasks.workunit.client.0.smithi136.stdout:89.06% done 2024-09-13T00:36:34.661 INFO:tasks.workunit.client.0.smithi136.stdout:90.62% done 2024-09-13T00:36:34.661 INFO:tasks.workunit.client.0.smithi136.stdout:92.19% done 2024-09-13T00:36:34.661 INFO:tasks.workunit.client.0.smithi136.stdout:93.75% done 2024-09-13T00:36:34.661 INFO:tasks.workunit.client.0.smithi136.stdout:95.31% done 2024-09-13T00:36:34.661 INFO:tasks.workunit.client.0.smithi136.stdout:96.88% done 2024-09-13T00:36:34.661 INFO:tasks.workunit.client.0.smithi136.stdout:98.44% done 2024-09-13T00:36:34.661 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestTrashMoveAndPurge (162 ms) 2024-09-13T00:36:34.661 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestTrashMoveAndPurgeNonExpiredDelay 2024-09-13T00:36:34.661 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:36:34.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:34 smithi136 bash[75164]: cluster 2024-09-13T00:36:33.316741+0000 mon.a (mon.0) 3374 : cluster [DBG] osdmap e1800: 8 total, 8 up, 8 in 2024-09-13T00:36:34.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:34 smithi136 bash[75164]: audit 2024-09-13T00:36:34.121531+0000 mon.c (mon.2) 139 : audit [INF] from='client.? 172.21.15.136:0/1086627647' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.136:0/1211818073"}]: dispatch 2024-09-13T00:36:34.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:34 smithi136 bash[75164]: audit 2024-09-13T00:36:34.122050+0000 mon.a (mon.0) 3375 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.136:0/1211818073"}]: dispatch 2024-09-13T00:36:34.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:34 smithi136 bash[76371]: cluster 2024-09-13T00:36:33.316741+0000 mon.a (mon.0) 3374 : cluster [DBG] osdmap e1800: 8 total, 8 up, 8 in 2024-09-13T00:36:34.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:34 smithi136 bash[76371]: audit 2024-09-13T00:36:34.121531+0000 mon.c (mon.2) 139 : audit [INF] from='client.? 172.21.15.136:0/1086627647' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.136:0/1211818073"}]: dispatch 2024-09-13T00:36:34.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:34 smithi136 bash[76371]: audit 2024-09-13T00:36:34.122050+0000 mon.a (mon.0) 3375 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.136:0/1211818073"}]: dispatch 2024-09-13T00:36:34.817 INFO:tasks.workunit.client.0.smithi136.stdout:0.00% done 2024-09-13T00:36:34.817 INFO:tasks.workunit.client.0.smithi136.stdout:1.56% done 2024-09-13T00:36:34.817 INFO:tasks.workunit.client.0.smithi136.stdout:3.12% done 2024-09-13T00:36:34.817 INFO:tasks.workunit.client.0.smithi136.stdout:4.69% done 2024-09-13T00:36:34.817 INFO:tasks.workunit.client.0.smithi136.stdout:6.25% done 2024-09-13T00:36:34.817 INFO:tasks.workunit.client.0.smithi136.stdout:7.81% done 2024-09-13T00:36:34.817 INFO:tasks.workunit.client.0.smithi136.stdout:9.38% done 2024-09-13T00:36:34.817 INFO:tasks.workunit.client.0.smithi136.stdout:10.94% done 2024-09-13T00:36:34.817 INFO:tasks.workunit.client.0.smithi136.stdout:12.50% done 2024-09-13T00:36:34.818 INFO:tasks.workunit.client.0.smithi136.stdout:14.06% done 2024-09-13T00:36:34.818 INFO:tasks.workunit.client.0.smithi136.stdout:15.62% done 2024-09-13T00:36:34.818 INFO:tasks.workunit.client.0.smithi136.stdout:17.19% done 2024-09-13T00:36:34.818 INFO:tasks.workunit.client.0.smithi136.stdout:18.75% done 2024-09-13T00:36:34.818 INFO:tasks.workunit.client.0.smithi136.stdout:20.31% done 2024-09-13T00:36:34.818 INFO:tasks.workunit.client.0.smithi136.stdout:21.88% done 2024-09-13T00:36:34.818 INFO:tasks.workunit.client.0.smithi136.stdout:23.44% done 2024-09-13T00:36:34.818 INFO:tasks.workunit.client.0.smithi136.stdout:25.00% done 2024-09-13T00:36:34.818 INFO:tasks.workunit.client.0.smithi136.stdout:26.56% done 2024-09-13T00:36:34.818 INFO:tasks.workunit.client.0.smithi136.stdout:28.12% done 2024-09-13T00:36:34.818 INFO:tasks.workunit.client.0.smithi136.stdout:29.69% done 2024-09-13T00:36:34.818 INFO:tasks.workunit.client.0.smithi136.stdout:31.25% done 2024-09-13T00:36:34.818 INFO:tasks.workunit.client.0.smithi136.stdout:32.81% done 2024-09-13T00:36:34.819 INFO:tasks.workunit.client.0.smithi136.stdout:34.38% done 2024-09-13T00:36:34.819 INFO:tasks.workunit.client.0.smithi136.stdout:35.94% done 2024-09-13T00:36:34.819 INFO:tasks.workunit.client.0.smithi136.stdout:37.50% done 2024-09-13T00:36:34.819 INFO:tasks.workunit.client.0.smithi136.stdout:39.06% done 2024-09-13T00:36:34.819 INFO:tasks.workunit.client.0.smithi136.stdout:40.62% done 2024-09-13T00:36:34.819 INFO:tasks.workunit.client.0.smithi136.stdout:42.19% done 2024-09-13T00:36:34.819 INFO:tasks.workunit.client.0.smithi136.stdout:43.75% done 2024-09-13T00:36:34.819 INFO:tasks.workunit.client.0.smithi136.stdout:45.31% done 2024-09-13T00:36:34.819 INFO:tasks.workunit.client.0.smithi136.stdout:46.88% done 2024-09-13T00:36:34.819 INFO:tasks.workunit.client.0.smithi136.stdout:48.44% done 2024-09-13T00:36:34.819 INFO:tasks.workunit.client.0.smithi136.stdout:50.00% done 2024-09-13T00:36:34.819 INFO:tasks.workunit.client.0.smithi136.stdout:51.56% done 2024-09-13T00:36:34.819 INFO:tasks.workunit.client.0.smithi136.stdout:53.12% done 2024-09-13T00:36:34.819 INFO:tasks.workunit.client.0.smithi136.stdout:54.69% done 2024-09-13T00:36:34.820 INFO:tasks.workunit.client.0.smithi136.stdout:56.25% done 2024-09-13T00:36:34.820 INFO:tasks.workunit.client.0.smithi136.stdout:57.81% done 2024-09-13T00:36:34.820 INFO:tasks.workunit.client.0.smithi136.stdout:59.38% done 2024-09-13T00:36:34.820 INFO:tasks.workunit.client.0.smithi136.stdout:60.94% done 2024-09-13T00:36:34.820 INFO:tasks.workunit.client.0.smithi136.stdout:62.50% done 2024-09-13T00:36:34.820 INFO:tasks.workunit.client.0.smithi136.stdout:64.06% done 2024-09-13T00:36:34.820 INFO:tasks.workunit.client.0.smithi136.stdout:65.62% done 2024-09-13T00:36:34.820 INFO:tasks.workunit.client.0.smithi136.stdout:67.19% done 2024-09-13T00:36:34.820 INFO:tasks.workunit.client.0.smithi136.stdout:68.75% done 2024-09-13T00:36:34.820 INFO:tasks.workunit.client.0.smithi136.stdout:70.31% done 2024-09-13T00:36:34.820 INFO:tasks.workunit.client.0.smithi136.stdout:71.88% done 2024-09-13T00:36:34.820 INFO:tasks.workunit.client.0.smithi136.stdout:73.44% done 2024-09-13T00:36:34.820 INFO:tasks.workunit.client.0.smithi136.stdout:75.00% done 2024-09-13T00:36:34.820 INFO:tasks.workunit.client.0.smithi136.stdout:76.56% done 2024-09-13T00:36:34.821 INFO:tasks.workunit.client.0.smithi136.stdout:78.12% done 2024-09-13T00:36:34.821 INFO:tasks.workunit.client.0.smithi136.stdout:79.69% done 2024-09-13T00:36:34.821 INFO:tasks.workunit.client.0.smithi136.stdout:81.25% done 2024-09-13T00:36:34.821 INFO:tasks.workunit.client.0.smithi136.stdout:82.81% done 2024-09-13T00:36:34.821 INFO:tasks.workunit.client.0.smithi136.stdout:84.38% done 2024-09-13T00:36:34.821 INFO:tasks.workunit.client.0.smithi136.stdout:85.94% done 2024-09-13T00:36:34.821 INFO:tasks.workunit.client.0.smithi136.stdout:87.50% done 2024-09-13T00:36:34.821 INFO:tasks.workunit.client.0.smithi136.stdout:89.06% done 2024-09-13T00:36:34.821 INFO:tasks.workunit.client.0.smithi136.stdout:90.62% done 2024-09-13T00:36:34.821 INFO:tasks.workunit.client.0.smithi136.stdout:92.19% done 2024-09-13T00:36:34.821 INFO:tasks.workunit.client.0.smithi136.stdout:93.75% done 2024-09-13T00:36:34.821 INFO:tasks.workunit.client.0.smithi136.stdout:95.31% done 2024-09-13T00:36:34.821 INFO:tasks.workunit.client.0.smithi136.stdout:96.88% done 2024-09-13T00:36:34.822 INFO:tasks.workunit.client.0.smithi136.stdout:98.44% done 2024-09-13T00:36:34.822 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestTrashMoveAndPurgeNonExpiredDelay (161 ms) 2024-09-13T00:36:34.822 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestTrashPurge 2024-09-13T00:36:34.822 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:36:34.838 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:36:34.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:34 smithi181 bash[67587]: cluster 2024-09-13T00:36:33.316741+0000 mon.a (mon.0) 3374 : cluster [DBG] osdmap e1800: 8 total, 8 up, 8 in 2024-09-13T00:36:34.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:34 smithi181 bash[67587]: audit 2024-09-13T00:36:34.121531+0000 mon.c (mon.2) 139 : audit [INF] from='client.? 172.21.15.136:0/1086627647' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.136:0/1211818073"}]: dispatch 2024-09-13T00:36:34.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:34 smithi181 bash[67587]: audit 2024-09-13T00:36:34.122050+0000 mon.a (mon.0) 3375 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.136:0/1211818073"}]: dispatch 2024-09-13T00:36:35.159 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestTrashPurge (342 ms) 2024-09-13T00:36:35.159 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestTrashMoveAndRestore 2024-09-13T00:36:35.159 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:36:35.292 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestTrashMoveAndRestore (133 ms) 2024-09-13T00:36:35.292 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestListWatchers 2024-09-13T00:36:35.292 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:36:35.359 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestListWatchers (68 ms) 2024-09-13T00:36:35.359 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestSetSnapById 2024-09-13T00:36:35.360 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:36:35.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:35 smithi136 bash[75164]: cluster 2024-09-13T00:36:34.288361+0000 mgr.y (mgr.24661) 1783 : cluster [DBG] pgmap v3144: 649 pgs: 649 active+clean; 152 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 164 KiB/s rd, 74 KiB/s wr, 213 op/s 2024-09-13T00:36:35.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:35 smithi136 bash[75164]: audit 2024-09-13T00:36:34.344281+0000 mon.a (mon.0) 3376 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.136:0/1211818073"}]': finished 2024-09-13T00:36:35.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:35 smithi136 bash[75164]: cluster 2024-09-13T00:36:34.344528+0000 mon.a (mon.0) 3377 : cluster [DBG] osdmap e1801: 8 total, 8 up, 8 in 2024-09-13T00:36:35.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:35 smithi136 bash[76371]: cluster 2024-09-13T00:36:34.288361+0000 mgr.y (mgr.24661) 1783 : cluster [DBG] pgmap v3144: 649 pgs: 649 active+clean; 152 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 164 KiB/s rd, 74 KiB/s wr, 213 op/s 2024-09-13T00:36:35.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:35 smithi136 bash[76371]: audit 2024-09-13T00:36:34.344281+0000 mon.a (mon.0) 3376 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.136:0/1211818073"}]': finished 2024-09-13T00:36:35.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:35 smithi136 bash[76371]: cluster 2024-09-13T00:36:34.344528+0000 mon.a (mon.0) 3377 : cluster [DBG] osdmap e1801: 8 total, 8 up, 8 in 2024-09-13T00:36:35.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:35 smithi181 bash[67587]: cluster 2024-09-13T00:36:34.288361+0000 mgr.y (mgr.24661) 1783 : cluster [DBG] pgmap v3144: 649 pgs: 649 active+clean; 152 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 164 KiB/s rd, 74 KiB/s wr, 213 op/s 2024-09-13T00:36:35.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:35 smithi181 bash[67587]: audit 2024-09-13T00:36:34.344281+0000 mon.a (mon.0) 3376 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.136:0/1211818073"}]': finished 2024-09-13T00:36:35.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:35 smithi181 bash[67587]: cluster 2024-09-13T00:36:34.344528+0000 mon.a (mon.0) 3377 : cluster [DBG] osdmap e1801: 8 total, 8 up, 8 in 2024-09-13T00:36:36.443 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestSetSnapById (1084 ms) 2024-09-13T00:36:36.444 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.Namespaces 2024-09-13T00:36:36.494 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.Namespaces (51 ms) 2024-09-13T00:36:36.494 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.NamespacesPP 2024-09-13T00:36:36.569 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:36:36.581 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:36:36.783 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.NamespacesPP (289 ms) 2024-09-13T00:36:36.784 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.Migration 2024-09-13T00:36:36.784 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:36:36.784 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:36:37.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:37 smithi136 bash[75164]: cluster 2024-09-13T00:36:36.291715+0000 mgr.y (mgr.24661) 1784 : cluster [DBG] pgmap v3146: 649 pgs: 649 active+clean; 152 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 182 KiB/s rd, 112 KiB/s wr, 184 op/s 2024-09-13T00:36:37.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:37 smithi136 bash[75164]: cluster 2024-09-13T00:36:36.368319+0000 mon.a (mon.0) 3378 : cluster [DBG] osdmap e1802: 8 total, 8 up, 8 in 2024-09-13T00:36:37.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:37 smithi136 bash[76371]: cluster 2024-09-13T00:36:36.291715+0000 mgr.y (mgr.24661) 1784 : cluster [DBG] pgmap v3146: 649 pgs: 649 active+clean; 152 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 182 KiB/s rd, 112 KiB/s wr, 184 op/s 2024-09-13T00:36:37.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:37 smithi136 bash[76371]: cluster 2024-09-13T00:36:36.368319+0000 mon.a (mon.0) 3378 : cluster [DBG] osdmap e1802: 8 total, 8 up, 8 in 2024-09-13T00:36:37.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:37 smithi181 bash[67587]: cluster 2024-09-13T00:36:36.291715+0000 mgr.y (mgr.24661) 1784 : cluster [DBG] pgmap v3146: 649 pgs: 649 active+clean; 152 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 182 KiB/s rd, 112 KiB/s wr, 184 op/s 2024-09-13T00:36:37.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:37 smithi181 bash[67587]: cluster 2024-09-13T00:36:36.368319+0000 mon.a (mon.0) 3378 : cluster [DBG] osdmap e1802: 8 total, 8 up, 8 in 2024-09-13T00:36:37.937 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.Migration (1153 ms) 2024-09-13T00:36:37.937 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.MigrationPP 2024-09-13T00:36:37.937 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:36:37.937 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:36:39.059 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.MigrationPP (1123 ms) 2024-09-13T00:36:39.059 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestGetAccessTimestamp 2024-09-13T00:36:39.059 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:36:39.103 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestGetAccessTimestamp (43 ms) 2024-09-13T00:36:39.103 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.TestGetModifyTimestamp 2024-09-13T00:36:39.103 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:36:39.149 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.TestGetModifyTimestamp (46 ms) 2024-09-13T00:36:39.149 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.ZeroOverlapFlatten 2024-09-13T00:36:39.149 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:36:39.610 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.ZeroOverlapFlatten (461 ms) 2024-09-13T00:36:39.610 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.PoolMetadata 2024-09-13T00:36:39.689 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:36:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:36:39] "GET /metrics HTTP/1.1" 200 75537 "" "Prometheus/2.43.0" 2024-09-13T00:36:39.690 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:39 smithi136 bash[75164]: cluster 2024-09-13T00:36:38.294609+0000 mgr.y (mgr.24661) 1785 : cluster [DBG] pgmap v3148: 649 pgs: 649 active+clean; 152 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 255 KiB/s rd, 101 KiB/s wr, 307 op/s 2024-09-13T00:36:39.690 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:39 smithi136 bash[75164]: audit 2024-09-13T00:36:38.833518+0000 mon.a (mon.0) 3379 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:36:39.690 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:39 smithi136 bash[76371]: cluster 2024-09-13T00:36:38.294609+0000 mgr.y (mgr.24661) 1785 : cluster [DBG] pgmap v3148: 649 pgs: 649 active+clean; 152 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 255 KiB/s rd, 101 KiB/s wr, 307 op/s 2024-09-13T00:36:39.690 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:39 smithi136 bash[76371]: audit 2024-09-13T00:36:38.833518+0000 mon.a (mon.0) 3379 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:36:39.702 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.PoolMetadata (93 ms) 2024-09-13T00:36:39.702 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.PoolMetadataPP 2024-09-13T00:36:39.741 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.PoolMetadataPP (38 ms) 2024-09-13T00:36:39.741 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.Config 2024-09-13T00:36:39.751 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:36:39.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:39 smithi181 bash[67587]: cluster 2024-09-13T00:36:38.294609+0000 mgr.y (mgr.24661) 1785 : cluster [DBG] pgmap v3148: 649 pgs: 649 active+clean; 152 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 255 KiB/s rd, 101 KiB/s wr, 307 op/s 2024-09-13T00:36:39.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:39 smithi181 bash[67587]: audit 2024-09-13T00:36:38.833518+0000 mon.a (mon.0) 3379 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:36:39.886 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.Config (146 ms) 2024-09-13T00:36:39.886 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.ConfigPP 2024-09-13T00:36:39.895 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:36:39.967 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.ConfigPP (81 ms) 2024-09-13T00:36:39.967 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.PoolStatsPP 2024-09-13T00:36:40.009 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:36:39 smithi136 bash[68787]: [13/Sep/2024:00:36:39] ENGINE Bus STOPPING 2024-09-13T00:36:40.009 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:36:39 smithi136 bash[68787]: [13/Sep/2024:00:36:39] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T00:36:40.009 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:36:39 smithi136 bash[68787]: [13/Sep/2024:00:36:39] ENGINE Bus STOPPED 2024-09-13T00:36:40.009 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:36:39 smithi136 bash[68787]: [13/Sep/2024:00:36:39] ENGINE Bus STARTING 2024-09-13T00:36:40.448 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[68787]: [13/Sep/2024:00:36:40] ENGINE Serving on http://:::9283 2024-09-13T00:36:40.449 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[68787]: [13/Sep/2024:00:36:40] ENGINE Bus STARTED 2024-09-13T00:36:40.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[76371]: cluster 2024-09-13T00:36:39.388214+0000 mon.a (mon.0) 3380 : cluster [DBG] osdmap e1803: 8 total, 8 up, 8 in 2024-09-13T00:36:40.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[76371]: cluster 2024-09-13T00:36:39.633793+0000 mon.a (mon.0) 3381 : cluster [DBG] osdmap e1804: 8 total, 8 up, 8 in 2024-09-13T00:36:40.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[76371]: audit 2024-09-13T00:36:39.670084+0000 mon.a (mon.0) 3382 : audit [INF] from='client.? ' entity='client.admin' 2024-09-13T00:36:40.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[76371]: audit 2024-09-13T00:36:39.699723+0000 mon.a (mon.0) 3383 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:36:40.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[76371]: audit 2024-09-13T00:36:39.705471+0000 mon.a (mon.0) 3384 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:36:40.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[76371]: audit 2024-09-13T00:36:39.708838+0000 mon.a (mon.0) 3385 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:36:40.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[76371]: audit 2024-09-13T00:36:39.709863+0000 mon.a (mon.0) 3386 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T00:36:40.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[76371]: audit 2024-09-13T00:36:39.712232+0000 mon.a (mon.0) 3387 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:36:40.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[76371]: audit 2024-09-13T00:36:39.722969+0000 mon.a (mon.0) 3388 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:36:40.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[76371]: audit 2024-09-13T00:36:39.940164+0000 mon.a (mon.0) 3389 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:36:40.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[76371]: audit 2024-09-13T00:36:39.940511+0000 mon.a (mon.0) 3390 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:36:40.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[76371]: audit 2024-09-13T00:36:39.940852+0000 mon.a (mon.0) 3391 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:36:40.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[76371]: audit 2024-09-13T00:36:39.943317+0000 mon.a (mon.0) 3392 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:36:40.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[76371]: audit 2024-09-13T00:36:39.943637+0000 mon.a (mon.0) 3393 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:36:40.762 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[76371]: audit 2024-09-13T00:36:39.943946+0000 mon.a (mon.0) 3394 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:36:40.762 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[76371]: audit 2024-09-13T00:36:39.946119+0000 mon.a (mon.0) 3395 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:36:40.762 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[76371]: audit 2024-09-13T00:36:39.946480+0000 mon.a (mon.0) 3396 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:36:40.762 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[76371]: audit 2024-09-13T00:36:39.946810+0000 mon.a (mon.0) 3397 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:36:40.762 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[76371]: audit 2024-09-13T00:36:39.960436+0000 mon.a (mon.0) 3398 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T00:36:40.763 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[76371]: audit 2024-09-13T00:36:39.960508+0000 mon.a (mon.0) 3399 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T00:36:40.763 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[76371]: audit 2024-09-13T00:36:39.960830+0000 mon.a (mon.0) 3400 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:36:40.763 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[76371]: audit 2024-09-13T00:36:39.961142+0000 mon.a (mon.0) 3401 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:36:40.763 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[76371]: audit 2024-09-13T00:36:39.961419+0000 mon.a (mon.0) 3402 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:36:40.763 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[76371]: audit 2024-09-13T00:36:39.962977+0000 mon.b (mon.1) 23 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T00:36:40.763 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[76371]: audit 2024-09-13T00:36:39.963085+0000 mon.b (mon.1) 24 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T00:36:40.763 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[76371]: audit 2024-09-13T00:36:39.963762+0000 mon.a (mon.0) 3403 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:36:40.763 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[76371]: audit 2024-09-13T00:36:39.964051+0000 mon.a (mon.0) 3404 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:36:40.763 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[76371]: audit 2024-09-13T00:36:39.964523+0000 mon.a (mon.0) 3405 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:36:40.764 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[76371]: audit 2024-09-13T00:36:39.966322+0000 mon.c (mon.2) 140 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T00:36:40.764 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[76371]: audit 2024-09-13T00:36:39.966453+0000 mon.c (mon.2) 141 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T00:36:40.764 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[76371]: audit 2024-09-13T00:36:39.966550+0000 mon.a (mon.0) 3406 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:36:40.764 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[76371]: audit 2024-09-13T00:36:39.966830+0000 mon.a (mon.0) 3407 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:36:40.764 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[76371]: audit 2024-09-13T00:36:39.967097+0000 mon.a (mon.0) 3408 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:36:40.764 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[76371]: cluster 2024-09-13T00:36:40.295704+0000 mgr.y (mgr.24661) 1786 : cluster [DBG] pgmap v3151: 649 pgs: 649 active+clean; 152 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 405 KiB/s rd, 69 KiB/s wr, 501 op/s 2024-09-13T00:36:40.764 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[75164]: cluster 2024-09-13T00:36:39.388214+0000 mon.a (mon.0) 3380 : cluster [DBG] osdmap e1803: 8 total, 8 up, 8 in 2024-09-13T00:36:40.765 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[75164]: cluster 2024-09-13T00:36:39.633793+0000 mon.a (mon.0) 3381 : cluster [DBG] osdmap e1804: 8 total, 8 up, 8 in 2024-09-13T00:36:40.765 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[75164]: audit 2024-09-13T00:36:39.670084+0000 mon.a (mon.0) 3382 : audit [INF] from='client.? ' entity='client.admin' 2024-09-13T00:36:40.765 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[75164]: audit 2024-09-13T00:36:39.699723+0000 mon.a (mon.0) 3383 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:36:40.765 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[75164]: audit 2024-09-13T00:36:39.705471+0000 mon.a (mon.0) 3384 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:36:40.765 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[75164]: audit 2024-09-13T00:36:39.708838+0000 mon.a (mon.0) 3385 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:36:40.765 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[75164]: audit 2024-09-13T00:36:39.709863+0000 mon.a (mon.0) 3386 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T00:36:40.765 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[75164]: audit 2024-09-13T00:36:39.712232+0000 mon.a (mon.0) 3387 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:36:40.765 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[75164]: audit 2024-09-13T00:36:39.722969+0000 mon.a (mon.0) 3388 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:36:40.765 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[75164]: audit 2024-09-13T00:36:39.940164+0000 mon.a (mon.0) 3389 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:36:40.765 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[75164]: audit 2024-09-13T00:36:39.940511+0000 mon.a (mon.0) 3390 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:36:40.766 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[75164]: audit 2024-09-13T00:36:39.940852+0000 mon.a (mon.0) 3391 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:36:40.766 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[75164]: audit 2024-09-13T00:36:39.943317+0000 mon.a (mon.0) 3392 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:36:40.766 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[75164]: audit 2024-09-13T00:36:39.943637+0000 mon.a (mon.0) 3393 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:36:40.766 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[75164]: audit 2024-09-13T00:36:39.943946+0000 mon.a (mon.0) 3394 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:36:40.766 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[75164]: audit 2024-09-13T00:36:39.946119+0000 mon.a (mon.0) 3395 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:36:40.766 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[75164]: audit 2024-09-13T00:36:39.946480+0000 mon.a (mon.0) 3396 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:36:40.766 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[75164]: audit 2024-09-13T00:36:39.946810+0000 mon.a (mon.0) 3397 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:36:40.766 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[75164]: audit 2024-09-13T00:36:39.960436+0000 mon.a (mon.0) 3398 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T00:36:40.766 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[75164]: audit 2024-09-13T00:36:39.960508+0000 mon.a (mon.0) 3399 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T00:36:40.766 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[75164]: audit 2024-09-13T00:36:39.960830+0000 mon.a (mon.0) 3400 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:36:40.766 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[75164]: audit 2024-09-13T00:36:39.961142+0000 mon.a (mon.0) 3401 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:36:40.766 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[75164]: audit 2024-09-13T00:36:39.961419+0000 mon.a (mon.0) 3402 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:36:40.767 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[75164]: audit 2024-09-13T00:36:39.962977+0000 mon.b (mon.1) 23 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T00:36:40.767 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[75164]: audit 2024-09-13T00:36:39.963085+0000 mon.b (mon.1) 24 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T00:36:40.767 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[75164]: audit 2024-09-13T00:36:39.963762+0000 mon.a (mon.0) 3403 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:36:40.767 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[75164]: audit 2024-09-13T00:36:39.964051+0000 mon.a (mon.0) 3404 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:36:40.767 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[75164]: audit 2024-09-13T00:36:39.964523+0000 mon.a (mon.0) 3405 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:36:40.767 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[75164]: audit 2024-09-13T00:36:39.966322+0000 mon.c (mon.2) 140 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T00:36:40.767 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[75164]: audit 2024-09-13T00:36:39.966453+0000 mon.c (mon.2) 141 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T00:36:40.767 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[75164]: audit 2024-09-13T00:36:39.966550+0000 mon.a (mon.0) 3406 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:36:40.767 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[75164]: audit 2024-09-13T00:36:39.966830+0000 mon.a (mon.0) 3407 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:36:40.767 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[75164]: audit 2024-09-13T00:36:39.967097+0000 mon.a (mon.0) 3408 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:36:40.767 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:40 smithi136 bash[75164]: cluster 2024-09-13T00:36:40.295704+0000 mgr.y (mgr.24661) 1786 : cluster [DBG] pgmap v3151: 649 pgs: 649 active+clean; 152 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 405 KiB/s rd, 69 KiB/s wr, 501 op/s 2024-09-13T00:36:40.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:40 smithi181 bash[67587]: cluster 2024-09-13T00:36:39.388214+0000 mon.a (mon.0) 3380 : cluster [DBG] osdmap e1803: 8 total, 8 up, 8 in 2024-09-13T00:36:40.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:40 smithi181 bash[67587]: cluster 2024-09-13T00:36:39.633793+0000 mon.a (mon.0) 3381 : cluster [DBG] osdmap e1804: 8 total, 8 up, 8 in 2024-09-13T00:36:40.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:40 smithi181 bash[67587]: audit 2024-09-13T00:36:39.670084+0000 mon.a (mon.0) 3382 : audit [INF] from='client.? ' entity='client.admin' 2024-09-13T00:36:40.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:40 smithi181 bash[67587]: audit 2024-09-13T00:36:39.699723+0000 mon.a (mon.0) 3383 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:36:40.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:40 smithi181 bash[67587]: audit 2024-09-13T00:36:39.705471+0000 mon.a (mon.0) 3384 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:36:40.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:40 smithi181 bash[67587]: audit 2024-09-13T00:36:39.708838+0000 mon.a (mon.0) 3385 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:36:40.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:40 smithi181 bash[67587]: audit 2024-09-13T00:36:39.709863+0000 mon.a (mon.0) 3386 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T00:36:40.851 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:40 smithi181 bash[67587]: audit 2024-09-13T00:36:39.712232+0000 mon.a (mon.0) 3387 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:36:40.851 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:40 smithi181 bash[67587]: audit 2024-09-13T00:36:39.722969+0000 mon.a (mon.0) 3388 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:36:40.851 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:40 smithi181 bash[67587]: audit 2024-09-13T00:36:39.940164+0000 mon.a (mon.0) 3389 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:36:40.851 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:40 smithi181 bash[67587]: audit 2024-09-13T00:36:39.940511+0000 mon.a (mon.0) 3390 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:36:40.851 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:40 smithi181 bash[67587]: audit 2024-09-13T00:36:39.940852+0000 mon.a (mon.0) 3391 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:36:40.851 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:40 smithi181 bash[67587]: audit 2024-09-13T00:36:39.943317+0000 mon.a (mon.0) 3392 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:36:40.851 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:40 smithi181 bash[67587]: audit 2024-09-13T00:36:39.943637+0000 mon.a (mon.0) 3393 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:36:40.852 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:40 smithi181 bash[67587]: audit 2024-09-13T00:36:39.943946+0000 mon.a (mon.0) 3394 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:36:40.852 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:40 smithi181 bash[67587]: audit 2024-09-13T00:36:39.946119+0000 mon.a (mon.0) 3395 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:36:40.852 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:40 smithi181 bash[67587]: audit 2024-09-13T00:36:39.946480+0000 mon.a (mon.0) 3396 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:36:40.852 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:40 smithi181 bash[67587]: audit 2024-09-13T00:36:39.946810+0000 mon.a (mon.0) 3397 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:36:40.852 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:40 smithi181 bash[67587]: audit 2024-09-13T00:36:39.960436+0000 mon.a (mon.0) 3398 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T00:36:40.852 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:40 smithi181 bash[67587]: audit 2024-09-13T00:36:39.960508+0000 mon.a (mon.0) 3399 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T00:36:40.852 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:40 smithi181 bash[67587]: audit 2024-09-13T00:36:39.960830+0000 mon.a (mon.0) 3400 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:36:40.852 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:40 smithi181 bash[67587]: audit 2024-09-13T00:36:39.961142+0000 mon.a (mon.0) 3401 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:36:40.852 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:40 smithi181 bash[67587]: audit 2024-09-13T00:36:39.961419+0000 mon.a (mon.0) 3402 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:36:40.852 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:40 smithi181 bash[67587]: audit 2024-09-13T00:36:39.962977+0000 mon.b (mon.1) 23 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T00:36:40.852 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:40 smithi181 bash[67587]: audit 2024-09-13T00:36:39.963085+0000 mon.b (mon.1) 24 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T00:36:40.852 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:40 smithi181 bash[67587]: audit 2024-09-13T00:36:39.963762+0000 mon.a (mon.0) 3403 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:36:40.853 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:40 smithi181 bash[67587]: audit 2024-09-13T00:36:39.964051+0000 mon.a (mon.0) 3404 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:36:40.853 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:40 smithi181 bash[67587]: audit 2024-09-13T00:36:39.964523+0000 mon.a (mon.0) 3405 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:36:40.853 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:40 smithi181 bash[67587]: audit 2024-09-13T00:36:39.966322+0000 mon.c (mon.2) 140 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T00:36:40.853 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:40 smithi181 bash[67587]: audit 2024-09-13T00:36:39.966453+0000 mon.c (mon.2) 141 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T00:36:40.853 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:40 smithi181 bash[67587]: audit 2024-09-13T00:36:39.966550+0000 mon.a (mon.0) 3406 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:36:40.853 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:40 smithi181 bash[67587]: audit 2024-09-13T00:36:39.966830+0000 mon.a (mon.0) 3407 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:36:40.853 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:40 smithi181 bash[67587]: audit 2024-09-13T00:36:39.967097+0000 mon.a (mon.0) 3408 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:36:40.853 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:40 smithi181 bash[67587]: cluster 2024-09-13T00:36:40.295704+0000 mgr.y (mgr.24661) 1786 : cluster [DBG] pgmap v3151: 649 pgs: 649 active+clean; 152 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 405 KiB/s rd, 69 KiB/s wr, 501 op/s 2024-09-13T00:36:41.754 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:36:42.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:41 smithi136 bash[75164]: cluster 2024-09-13T00:36:40.737836+0000 mon.a (mon.0) 3409 : cluster [DBG] osdmap e1805: 8 total, 8 up, 8 in 2024-09-13T00:36:42.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:41 smithi136 bash[75164]: audit 2024-09-13T00:36:40.749509+0000 mon.a (mon.0) 3410 : audit [INF] from='client.? 172.21.15.136:0/4108725519' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-31","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:36:42.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:41 smithi136 bash[76371]: cluster 2024-09-13T00:36:40.737836+0000 mon.a (mon.0) 3409 : cluster [DBG] osdmap e1805: 8 total, 8 up, 8 in 2024-09-13T00:36:42.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:41 smithi136 bash[76371]: audit 2024-09-13T00:36:40.749509+0000 mon.a (mon.0) 3410 : audit [INF] from='client.? 172.21.15.136:0/4108725519' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-31","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:36:42.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:41 smithi181 bash[67587]: cluster 2024-09-13T00:36:40.737836+0000 mon.a (mon.0) 3409 : cluster [DBG] osdmap e1805: 8 total, 8 up, 8 in 2024-09-13T00:36:42.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:41 smithi181 bash[67587]: audit 2024-09-13T00:36:40.749509+0000 mon.a (mon.0) 3410 : audit [INF] from='client.? 172.21.15.136:0/4108725519' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-31","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:36:42.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:36:42 smithi181 bash[95505]: ts=2024-09-13T00:36:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:36:43.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:42 smithi136 bash[75164]: audit 2024-09-13T00:36:41.741778+0000 mon.a (mon.0) 3411 : audit [INF] from='client.? 172.21.15.136:0/4108725519' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-31","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:36:43.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:42 smithi136 bash[75164]: cluster 2024-09-13T00:36:41.742464+0000 mon.a (mon.0) 3412 : cluster [DBG] osdmap e1806: 8 total, 8 up, 8 in 2024-09-13T00:36:43.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:42 smithi136 bash[75164]: cluster 2024-09-13T00:36:42.297556+0000 mgr.y (mgr.24661) 1787 : cluster [DBG] pgmap v3154: 681 pgs: 4 creating+peering, 24 unknown, 653 active+clean; 152 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 613 KiB/s rd, 34 KiB/s wr, 776 op/s 2024-09-13T00:36:43.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:42 smithi136 bash[76371]: audit 2024-09-13T00:36:41.741778+0000 mon.a (mon.0) 3411 : audit [INF] from='client.? 172.21.15.136:0/4108725519' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-31","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:36:43.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:42 smithi136 bash[76371]: cluster 2024-09-13T00:36:41.742464+0000 mon.a (mon.0) 3412 : cluster [DBG] osdmap e1806: 8 total, 8 up, 8 in 2024-09-13T00:36:43.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:42 smithi136 bash[76371]: cluster 2024-09-13T00:36:42.297556+0000 mgr.y (mgr.24661) 1787 : cluster [DBG] pgmap v3154: 681 pgs: 4 creating+peering, 24 unknown, 653 active+clean; 152 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 613 KiB/s rd, 34 KiB/s wr, 776 op/s 2024-09-13T00:36:43.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:42 smithi181 bash[67587]: audit 2024-09-13T00:36:41.741778+0000 mon.a (mon.0) 3411 : audit [INF] from='client.? 172.21.15.136:0/4108725519' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-31","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:36:43.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:42 smithi181 bash[67587]: cluster 2024-09-13T00:36:41.742464+0000 mon.a (mon.0) 3412 : cluster [DBG] osdmap e1806: 8 total, 8 up, 8 in 2024-09-13T00:36:43.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:42 smithi181 bash[67587]: cluster 2024-09-13T00:36:42.297556+0000 mgr.y (mgr.24661) 1787 : cluster [DBG] pgmap v3154: 681 pgs: 4 creating+peering, 24 unknown, 653 active+clean; 152 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 613 KiB/s rd, 34 KiB/s wr, 776 op/s 2024-09-13T00:36:43.866 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:36:43.885 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:36:43.897 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:36:44.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:43 smithi181 bash[67587]: cluster 2024-09-13T00:36:42.766473+0000 mon.a (mon.0) 3413 : cluster [DBG] osdmap e1807: 8 total, 8 up, 8 in 2024-09-13T00:36:44.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:43 smithi136 bash[76371]: cluster 2024-09-13T00:36:42.766473+0000 mon.a (mon.0) 3413 : cluster [DBG] osdmap e1807: 8 total, 8 up, 8 in 2024-09-13T00:36:44.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:43 smithi136 bash[75164]: cluster 2024-09-13T00:36:42.766473+0000 mon.a (mon.0) 3413 : cluster [DBG] osdmap e1807: 8 total, 8 up, 8 in 2024-09-13T00:36:44.974 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.PoolStatsPP (5007 ms) 2024-09-13T00:36:44.974 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.ImageSpec 2024-09-13T00:36:45.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:44 smithi181 bash[67587]: cluster 2024-09-13T00:36:43.782695+0000 mon.a (mon.0) 3414 : cluster [DBG] osdmap e1808: 8 total, 8 up, 8 in 2024-09-13T00:36:45.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:44 smithi181 bash[67587]: cluster 2024-09-13T00:36:44.298791+0000 mgr.y (mgr.24661) 1788 : cluster [DBG] pgmap v3157: 681 pgs: 4 creating+peering, 12 unknown, 665 active+clean; 152 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 354 KiB/s rd, 21 KiB/s wr, 462 op/s 2024-09-13T00:36:45.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:44 smithi136 bash[75164]: cluster 2024-09-13T00:36:43.782695+0000 mon.a (mon.0) 3414 : cluster [DBG] osdmap e1808: 8 total, 8 up, 8 in 2024-09-13T00:36:45.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:44 smithi136 bash[75164]: cluster 2024-09-13T00:36:44.298791+0000 mgr.y (mgr.24661) 1788 : cluster [DBG] pgmap v3157: 681 pgs: 4 creating+peering, 12 unknown, 665 active+clean; 152 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 354 KiB/s rd, 21 KiB/s wr, 462 op/s 2024-09-13T00:36:45.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:44 smithi136 bash[76371]: cluster 2024-09-13T00:36:43.782695+0000 mon.a (mon.0) 3414 : cluster [DBG] osdmap e1808: 8 total, 8 up, 8 in 2024-09-13T00:36:45.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:44 smithi136 bash[76371]: cluster 2024-09-13T00:36:44.298791+0000 mgr.y (mgr.24661) 1788 : cluster [DBG] pgmap v3157: 681 pgs: 4 creating+peering, 12 unknown, 665 active+clean; 152 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 354 KiB/s rd, 21 KiB/s wr, 462 op/s 2024-09-13T00:36:46.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:45 smithi181 bash[67587]: cluster 2024-09-13T00:36:44.793482+0000 mon.a (mon.0) 3415 : cluster [DBG] osdmap e1809: 8 total, 8 up, 8 in 2024-09-13T00:36:46.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:45 smithi136 bash[75164]: cluster 2024-09-13T00:36:44.793482+0000 mon.a (mon.0) 3415 : cluster [DBG] osdmap e1809: 8 total, 8 up, 8 in 2024-09-13T00:36:46.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:45 smithi136 bash[76371]: cluster 2024-09-13T00:36:44.793482+0000 mon.a (mon.0) 3415 : cluster [DBG] osdmap e1809: 8 total, 8 up, 8 in 2024-09-13T00:36:46.824 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:36:47.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:46 smithi181 bash[67587]: cluster 2024-09-13T00:36:45.803543+0000 mon.a (mon.0) 3416 : cluster [DBG] osdmap e1810: 8 total, 8 up, 8 in 2024-09-13T00:36:47.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:46 smithi181 bash[67587]: audit 2024-09-13T00:36:45.807299+0000 mon.a (mon.0) 3417 : audit [INF] from='client.? 172.21.15.136:0/2250095473' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-32","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:36:47.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:46 smithi181 bash[67587]: cluster 2024-09-13T00:36:46.300735+0000 mgr.y (mgr.24661) 1789 : cluster [DBG] pgmap v3160: 713 pgs: 30 unknown, 4 creating+peering, 679 active+clean; 152 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 53 KiB/s rd, 9.0 KiB/s wr, 89 op/s 2024-09-13T00:36:47.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:46 smithi136 bash[75164]: cluster 2024-09-13T00:36:45.803543+0000 mon.a (mon.0) 3416 : cluster [DBG] osdmap e1810: 8 total, 8 up, 8 in 2024-09-13T00:36:47.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:46 smithi136 bash[75164]: audit 2024-09-13T00:36:45.807299+0000 mon.a (mon.0) 3417 : audit [INF] from='client.? 172.21.15.136:0/2250095473' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-32","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:36:47.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:46 smithi136 bash[75164]: cluster 2024-09-13T00:36:46.300735+0000 mgr.y (mgr.24661) 1789 : cluster [DBG] pgmap v3160: 713 pgs: 30 unknown, 4 creating+peering, 679 active+clean; 152 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 53 KiB/s rd, 9.0 KiB/s wr, 89 op/s 2024-09-13T00:36:47.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:46 smithi136 bash[76371]: cluster 2024-09-13T00:36:45.803543+0000 mon.a (mon.0) 3416 : cluster [DBG] osdmap e1810: 8 total, 8 up, 8 in 2024-09-13T00:36:47.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:46 smithi136 bash[76371]: audit 2024-09-13T00:36:45.807299+0000 mon.a (mon.0) 3417 : audit [INF] from='client.? 172.21.15.136:0/2250095473' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-32","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:36:47.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:46 smithi136 bash[76371]: cluster 2024-09-13T00:36:46.300735+0000 mgr.y (mgr.24661) 1789 : cluster [DBG] pgmap v3160: 713 pgs: 30 unknown, 4 creating+peering, 679 active+clean; 152 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 53 KiB/s rd, 9.0 KiB/s wr, 89 op/s 2024-09-13T00:36:48.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:47 smithi181 bash[67587]: cluster 2024-09-13T00:36:46.796138+0000 mon.a (mon.0) 3418 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:36:48.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:47 smithi181 bash[67587]: audit 2024-09-13T00:36:46.815014+0000 mon.a (mon.0) 3419 : audit [INF] from='client.? 172.21.15.136:0/2250095473' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-32","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:36:48.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:47 smithi181 bash[67587]: cluster 2024-09-13T00:36:46.815441+0000 mon.a (mon.0) 3420 : cluster [DBG] osdmap e1811: 8 total, 8 up, 8 in 2024-09-13T00:36:48.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:47 smithi136 bash[75164]: cluster 2024-09-13T00:36:46.796138+0000 mon.a (mon.0) 3418 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:36:48.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:47 smithi136 bash[75164]: audit 2024-09-13T00:36:46.815014+0000 mon.a (mon.0) 3419 : audit [INF] from='client.? 172.21.15.136:0/2250095473' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-32","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:36:48.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:47 smithi136 bash[75164]: cluster 2024-09-13T00:36:46.815441+0000 mon.a (mon.0) 3420 : cluster [DBG] osdmap e1811: 8 total, 8 up, 8 in 2024-09-13T00:36:48.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:47 smithi136 bash[76371]: cluster 2024-09-13T00:36:46.796138+0000 mon.a (mon.0) 3418 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:36:48.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:47 smithi136 bash[76371]: audit 2024-09-13T00:36:46.815014+0000 mon.a (mon.0) 3419 : audit [INF] from='client.? 172.21.15.136:0/2250095473' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-32","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:36:48.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:47 smithi136 bash[76371]: cluster 2024-09-13T00:36:46.815441+0000 mon.a (mon.0) 3420 : cluster [DBG] osdmap e1811: 8 total, 8 up, 8 in 2024-09-13T00:36:49.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:48 smithi181 bash[67587]: cluster 2024-09-13T00:36:47.815559+0000 mon.a (mon.0) 3421 : cluster [DBG] osdmap e1812: 8 total, 8 up, 8 in 2024-09-13T00:36:49.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:48 smithi181 bash[67587]: cluster 2024-09-13T00:36:48.301888+0000 mgr.y (mgr.24661) 1790 : cluster [DBG] pgmap v3163: 713 pgs: 16 unknown, 13 creating+peering, 684 active+clean; 152 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 33 KiB/s rd, 5.7 KiB/s wr, 48 op/s 2024-09-13T00:36:49.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:48 smithi136 bash[75164]: cluster 2024-09-13T00:36:47.815559+0000 mon.a (mon.0) 3421 : cluster [DBG] osdmap e1812: 8 total, 8 up, 8 in 2024-09-13T00:36:49.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:48 smithi136 bash[75164]: cluster 2024-09-13T00:36:48.301888+0000 mgr.y (mgr.24661) 1790 : cluster [DBG] pgmap v3163: 713 pgs: 16 unknown, 13 creating+peering, 684 active+clean; 152 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 33 KiB/s rd, 5.7 KiB/s wr, 48 op/s 2024-09-13T00:36:49.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:48 smithi136 bash[76371]: cluster 2024-09-13T00:36:47.815559+0000 mon.a (mon.0) 3421 : cluster [DBG] osdmap e1812: 8 total, 8 up, 8 in 2024-09-13T00:36:49.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:48 smithi136 bash[76371]: cluster 2024-09-13T00:36:48.301888+0000 mgr.y (mgr.24661) 1790 : cluster [DBG] pgmap v3163: 713 pgs: 16 unknown, 13 creating+peering, 684 active+clean; 152 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 33 KiB/s rd, 5.7 KiB/s wr, 48 op/s 2024-09-13T00:36:49.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:36:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:36:49] "GET /metrics HTTP/1.1" 200 75537 "" "Prometheus/2.43.0" 2024-09-13T00:36:50.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:49 smithi136 bash[75164]: cluster 2024-09-13T00:36:48.821763+0000 mon.a (mon.0) 3422 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:36:50.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:49 smithi136 bash[75164]: cluster 2024-09-13T00:36:48.821816+0000 mon.a (mon.0) 3423 : cluster [INF] Cluster is now healthy 2024-09-13T00:36:50.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:49 smithi136 bash[75164]: cluster 2024-09-13T00:36:48.844196+0000 mon.a (mon.0) 3424 : cluster [DBG] osdmap e1813: 8 total, 8 up, 8 in 2024-09-13T00:36:50.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:49 smithi136 bash[76371]: cluster 2024-09-13T00:36:48.821763+0000 mon.a (mon.0) 3422 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:36:50.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:49 smithi136 bash[76371]: cluster 2024-09-13T00:36:48.821816+0000 mon.a (mon.0) 3423 : cluster [INF] Cluster is now healthy 2024-09-13T00:36:50.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:49 smithi136 bash[76371]: cluster 2024-09-13T00:36:48.844196+0000 mon.a (mon.0) 3424 : cluster [DBG] osdmap e1813: 8 total, 8 up, 8 in 2024-09-13T00:36:50.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:49 smithi181 bash[67587]: cluster 2024-09-13T00:36:48.821763+0000 mon.a (mon.0) 3422 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:36:50.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:49 smithi181 bash[67587]: cluster 2024-09-13T00:36:48.821816+0000 mon.a (mon.0) 3423 : cluster [INF] Cluster is now healthy 2024-09-13T00:36:50.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:49 smithi181 bash[67587]: cluster 2024-09-13T00:36:48.844196+0000 mon.a (mon.0) 3424 : cluster [DBG] osdmap e1813: 8 total, 8 up, 8 in 2024-09-13T00:36:51.177 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.ImageSpec (6202 ms) 2024-09-13T00:36:51.177 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.SnapRemoveWithChildMissing 2024-09-13T00:36:51.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:50 smithi136 bash[76371]: cluster 2024-09-13T00:36:49.842513+0000 mon.a (mon.0) 3425 : cluster [DBG] osdmap e1814: 8 total, 8 up, 8 in 2024-09-13T00:36:51.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:50 smithi136 bash[76371]: cluster 2024-09-13T00:36:50.303111+0000 mgr.y (mgr.24661) 1791 : cluster [DBG] pgmap v3166: 713 pgs: 12 unknown, 13 creating+peering, 688 active+clean; 152 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 40 KiB/s rd, 5.0 KiB/s wr, 54 op/s 2024-09-13T00:36:51.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:50 smithi136 bash[75164]: cluster 2024-09-13T00:36:49.842513+0000 mon.a (mon.0) 3425 : cluster [DBG] osdmap e1814: 8 total, 8 up, 8 in 2024-09-13T00:36:51.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:50 smithi136 bash[75164]: cluster 2024-09-13T00:36:50.303111+0000 mgr.y (mgr.24661) 1791 : cluster [DBG] pgmap v3166: 713 pgs: 12 unknown, 13 creating+peering, 688 active+clean; 152 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 40 KiB/s rd, 5.0 KiB/s wr, 54 op/s 2024-09-13T00:36:51.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:50 smithi181 bash[67587]: cluster 2024-09-13T00:36:49.842513+0000 mon.a (mon.0) 3425 : cluster [DBG] osdmap e1814: 8 total, 8 up, 8 in 2024-09-13T00:36:51.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:50 smithi181 bash[67587]: cluster 2024-09-13T00:36:50.303111+0000 mgr.y (mgr.24661) 1791 : cluster [DBG] pgmap v3166: 713 pgs: 12 unknown, 13 creating+peering, 688 active+clean; 152 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 40 KiB/s rd, 5.0 KiB/s wr, 54 op/s 2024-09-13T00:36:52.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:51 smithi136 bash[75164]: cluster 2024-09-13T00:36:50.871396+0000 mon.a (mon.0) 3426 : cluster [DBG] osdmap e1815: 8 total, 8 up, 8 in 2024-09-13T00:36:52.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:51 smithi136 bash[76371]: cluster 2024-09-13T00:36:50.871396+0000 mon.a (mon.0) 3426 : cluster [DBG] osdmap e1815: 8 total, 8 up, 8 in 2024-09-13T00:36:52.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:51 smithi181 bash[67587]: cluster 2024-09-13T00:36:50.871396+0000 mon.a (mon.0) 3426 : cluster [DBG] osdmap e1815: 8 total, 8 up, 8 in 2024-09-13T00:36:52.601 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:36:52 smithi181 bash[95505]: ts=2024-09-13T00:36:52.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:36:52.938 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:36:53.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:52 smithi136 bash[75164]: cluster 2024-09-13T00:36:51.881656+0000 mon.a (mon.0) 3427 : cluster [DBG] osdmap e1816: 8 total, 8 up, 8 in 2024-09-13T00:36:53.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:52 smithi136 bash[75164]: audit 2024-09-13T00:36:51.890846+0000 mon.c (mon.2) 142 : audit [INF] from='client.? 172.21.15.136:0/4001129217' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-33","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:36:53.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:52 smithi136 bash[75164]: audit 2024-09-13T00:36:51.891539+0000 mon.a (mon.0) 3428 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-33","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:36:53.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:52 smithi136 bash[75164]: cluster 2024-09-13T00:36:52.304377+0000 mgr.y (mgr.24661) 1792 : cluster [DBG] pgmap v3169: 745 pgs: 32 unknown, 713 active+clean; 152 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 103 KiB/s rd, 6.0 KiB/s wr, 131 op/s 2024-09-13T00:36:53.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:52 smithi136 bash[76371]: cluster 2024-09-13T00:36:51.881656+0000 mon.a (mon.0) 3427 : cluster [DBG] osdmap e1816: 8 total, 8 up, 8 in 2024-09-13T00:36:53.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:52 smithi136 bash[76371]: audit 2024-09-13T00:36:51.890846+0000 mon.c (mon.2) 142 : audit [INF] from='client.? 172.21.15.136:0/4001129217' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-33","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:36:53.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:52 smithi136 bash[76371]: audit 2024-09-13T00:36:51.891539+0000 mon.a (mon.0) 3428 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-33","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:36:53.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:52 smithi136 bash[76371]: cluster 2024-09-13T00:36:52.304377+0000 mgr.y (mgr.24661) 1792 : cluster [DBG] pgmap v3169: 745 pgs: 32 unknown, 713 active+clean; 152 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 103 KiB/s rd, 6.0 KiB/s wr, 131 op/s 2024-09-13T00:36:53.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:52 smithi181 bash[67587]: cluster 2024-09-13T00:36:51.881656+0000 mon.a (mon.0) 3427 : cluster [DBG] osdmap e1816: 8 total, 8 up, 8 in 2024-09-13T00:36:53.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:52 smithi181 bash[67587]: audit 2024-09-13T00:36:51.890846+0000 mon.c (mon.2) 142 : audit [INF] from='client.? 172.21.15.136:0/4001129217' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-33","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:36:53.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:52 smithi181 bash[67587]: audit 2024-09-13T00:36:51.891539+0000 mon.a (mon.0) 3428 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-33","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:36:53.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:52 smithi181 bash[67587]: cluster 2024-09-13T00:36:52.304377+0000 mgr.y (mgr.24661) 1792 : cluster [DBG] pgmap v3169: 745 pgs: 32 unknown, 713 active+clean; 152 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 103 KiB/s rd, 6.0 KiB/s wr, 131 op/s 2024-09-13T00:36:54.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:53 smithi136 bash[75164]: cluster 2024-09-13T00:36:52.874466+0000 mon.a (mon.0) 3429 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:36:54.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:53 smithi136 bash[75164]: audit 2024-09-13T00:36:52.896907+0000 mon.a (mon.0) 3430 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-33","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:36:54.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:53 smithi136 bash[75164]: cluster 2024-09-13T00:36:52.897463+0000 mon.a (mon.0) 3431 : cluster [DBG] osdmap e1817: 8 total, 8 up, 8 in 2024-09-13T00:36:54.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:53 smithi136 bash[75164]: audit 2024-09-13T00:36:53.841798+0000 mon.a (mon.0) 3432 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:36:54.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:53 smithi136 bash[75164]: audit 2024-09-13T00:36:53.844166+0000 mon.a (mon.0) 3433 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:36:54.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:53 smithi136 bash[76371]: cluster 2024-09-13T00:36:52.874466+0000 mon.a (mon.0) 3429 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:36:54.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:53 smithi136 bash[76371]: audit 2024-09-13T00:36:52.896907+0000 mon.a (mon.0) 3430 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-33","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:36:54.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:53 smithi136 bash[76371]: cluster 2024-09-13T00:36:52.897463+0000 mon.a (mon.0) 3431 : cluster [DBG] osdmap e1817: 8 total, 8 up, 8 in 2024-09-13T00:36:54.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:53 smithi136 bash[76371]: audit 2024-09-13T00:36:53.841798+0000 mon.a (mon.0) 3432 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:36:54.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:53 smithi136 bash[76371]: audit 2024-09-13T00:36:53.844166+0000 mon.a (mon.0) 3433 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:36:54.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:53 smithi181 bash[67587]: cluster 2024-09-13T00:36:52.874466+0000 mon.a (mon.0) 3429 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:36:54.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:53 smithi181 bash[67587]: audit 2024-09-13T00:36:52.896907+0000 mon.a (mon.0) 3430 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-33","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:36:54.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:53 smithi181 bash[67587]: cluster 2024-09-13T00:36:52.897463+0000 mon.a (mon.0) 3431 : cluster [DBG] osdmap e1817: 8 total, 8 up, 8 in 2024-09-13T00:36:54.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:53 smithi181 bash[67587]: audit 2024-09-13T00:36:53.841798+0000 mon.a (mon.0) 3432 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:36:54.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:53 smithi181 bash[67587]: audit 2024-09-13T00:36:53.844166+0000 mon.a (mon.0) 3433 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:36:55.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:54 smithi136 bash[75164]: cluster 2024-09-13T00:36:53.893739+0000 mon.a (mon.0) 3434 : cluster [DBG] osdmap e1818: 8 total, 8 up, 8 in 2024-09-13T00:36:55.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:54 smithi136 bash[75164]: cluster 2024-09-13T00:36:54.305609+0000 mgr.y (mgr.24661) 1793 : cluster [DBG] pgmap v3172: 745 pgs: 3 creating+peering, 27 unknown, 715 active+clean; 152 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 77 KiB/s rd, 4.2 KiB/s wr, 100 op/s 2024-09-13T00:36:55.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:54 smithi136 bash[75164]: cluster 2024-09-13T00:36:54.636305+0000 mon.a (mon.0) 3435 : cluster [DBG] osdmap e1819: 8 total, 8 up, 8 in 2024-09-13T00:36:55.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:54 smithi136 bash[76371]: cluster 2024-09-13T00:36:53.893739+0000 mon.a (mon.0) 3434 : cluster [DBG] osdmap e1818: 8 total, 8 up, 8 in 2024-09-13T00:36:55.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:54 smithi136 bash[76371]: cluster 2024-09-13T00:36:54.305609+0000 mgr.y (mgr.24661) 1793 : cluster [DBG] pgmap v3172: 745 pgs: 3 creating+peering, 27 unknown, 715 active+clean; 152 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 77 KiB/s rd, 4.2 KiB/s wr, 100 op/s 2024-09-13T00:36:55.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:54 smithi136 bash[76371]: cluster 2024-09-13T00:36:54.636305+0000 mon.a (mon.0) 3435 : cluster [DBG] osdmap e1819: 8 total, 8 up, 8 in 2024-09-13T00:36:55.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:54 smithi181 bash[67587]: cluster 2024-09-13T00:36:53.893739+0000 mon.a (mon.0) 3434 : cluster [DBG] osdmap e1818: 8 total, 8 up, 8 in 2024-09-13T00:36:55.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:54 smithi181 bash[67587]: cluster 2024-09-13T00:36:54.305609+0000 mgr.y (mgr.24661) 1793 : cluster [DBG] pgmap v3172: 745 pgs: 3 creating+peering, 27 unknown, 715 active+clean; 152 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 77 KiB/s rd, 4.2 KiB/s wr, 100 op/s 2024-09-13T00:36:55.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:54 smithi181 bash[67587]: cluster 2024-09-13T00:36:54.636305+0000 mon.a (mon.0) 3435 : cluster [DBG] osdmap e1819: 8 total, 8 up, 8 in 2024-09-13T00:36:56.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:55 smithi136 bash[76371]: cluster 2024-09-13T00:36:54.900530+0000 mon.a (mon.0) 3436 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:36:56.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:55 smithi136 bash[76371]: cluster 2024-09-13T00:36:54.900571+0000 mon.a (mon.0) 3437 : cluster [INF] Cluster is now healthy 2024-09-13T00:36:56.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:55 smithi136 bash[76371]: cluster 2024-09-13T00:36:55.640932+0000 mon.a (mon.0) 3438 : cluster [DBG] osdmap e1820: 8 total, 8 up, 8 in 2024-09-13T00:36:56.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:55 smithi136 bash[75164]: cluster 2024-09-13T00:36:54.900530+0000 mon.a (mon.0) 3436 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:36:56.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:55 smithi136 bash[75164]: cluster 2024-09-13T00:36:54.900571+0000 mon.a (mon.0) 3437 : cluster [INF] Cluster is now healthy 2024-09-13T00:36:56.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:55 smithi136 bash[75164]: cluster 2024-09-13T00:36:55.640932+0000 mon.a (mon.0) 3438 : cluster [DBG] osdmap e1820: 8 total, 8 up, 8 in 2024-09-13T00:36:56.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:55 smithi181 bash[67587]: cluster 2024-09-13T00:36:54.900530+0000 mon.a (mon.0) 3436 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:36:56.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:55 smithi181 bash[67587]: cluster 2024-09-13T00:36:54.900571+0000 mon.a (mon.0) 3437 : cluster [INF] Cluster is now healthy 2024-09-13T00:36:56.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:55 smithi181 bash[67587]: cluster 2024-09-13T00:36:55.640932+0000 mon.a (mon.0) 3438 : cluster [DBG] osdmap e1820: 8 total, 8 up, 8 in 2024-09-13T00:36:57.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:56 smithi136 bash[75164]: cluster 2024-09-13T00:36:56.308159+0000 mgr.y (mgr.24661) 1794 : cluster [DBG] pgmap v3175: 745 pgs: 3 creating+peering, 10 unknown, 732 active+clean; 152 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 62 KiB/s rd, 2.2 KiB/s wr, 77 op/s 2024-09-13T00:36:57.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:56 smithi136 bash[75164]: cluster 2024-09-13T00:36:56.645698+0000 mon.a (mon.0) 3439 : cluster [DBG] osdmap e1821: 8 total, 8 up, 8 in 2024-09-13T00:36:57.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:56 smithi136 bash[76371]: cluster 2024-09-13T00:36:56.308159+0000 mgr.y (mgr.24661) 1794 : cluster [DBG] pgmap v3175: 745 pgs: 3 creating+peering, 10 unknown, 732 active+clean; 152 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 62 KiB/s rd, 2.2 KiB/s wr, 77 op/s 2024-09-13T00:36:57.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:56 smithi136 bash[76371]: cluster 2024-09-13T00:36:56.645698+0000 mon.a (mon.0) 3439 : cluster [DBG] osdmap e1821: 8 total, 8 up, 8 in 2024-09-13T00:36:57.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:56 smithi181 bash[67587]: cluster 2024-09-13T00:36:56.308159+0000 mgr.y (mgr.24661) 1794 : cluster [DBG] pgmap v3175: 745 pgs: 3 creating+peering, 10 unknown, 732 active+clean; 152 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 62 KiB/s rd, 2.2 KiB/s wr, 77 op/s 2024-09-13T00:36:57.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:56 smithi181 bash[67587]: cluster 2024-09-13T00:36:56.645698+0000 mon.a (mon.0) 3439 : cluster [DBG] osdmap e1821: 8 total, 8 up, 8 in 2024-09-13T00:36:59.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:58 smithi136 bash[75164]: cluster 2024-09-13T00:36:57.923161+0000 mon.a (mon.0) 3440 : cluster [DBG] osdmap e1822: 8 total, 8 up, 8 in 2024-09-13T00:36:59.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:36:58 smithi136 bash[75164]: cluster 2024-09-13T00:36:58.309444+0000 mgr.y (mgr.24661) 1795 : cluster [DBG] pgmap v3178: 713 pgs: 713 active+clean; 6.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:36:59.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:58 smithi136 bash[76371]: cluster 2024-09-13T00:36:57.923161+0000 mon.a (mon.0) 3440 : cluster [DBG] osdmap e1822: 8 total, 8 up, 8 in 2024-09-13T00:36:59.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:36:58 smithi136 bash[76371]: cluster 2024-09-13T00:36:58.309444+0000 mgr.y (mgr.24661) 1795 : cluster [DBG] pgmap v3178: 713 pgs: 713 active+clean; 6.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:36:59.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:58 smithi181 bash[67587]: cluster 2024-09-13T00:36:57.923161+0000 mon.a (mon.0) 3440 : cluster [DBG] osdmap e1822: 8 total, 8 up, 8 in 2024-09-13T00:36:59.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:36:58 smithi181 bash[67587]: cluster 2024-09-13T00:36:58.309444+0000 mgr.y (mgr.24661) 1795 : cluster [DBG] pgmap v3178: 713 pgs: 713 active+clean; 6.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:36:59.708 INFO:tasks.workunit.client.0.smithi136.stdout:num children is: 3 2024-09-13T00:36:59.708 INFO:tasks.workunit.client.0.smithi136.stdout:expected: 3 2024-09-13T00:36:59.708 INFO:tasks.workunit.client.0.smithi136.stdout:child: image202 2024-09-13T00:36:59.709 INFO:tasks.workunit.client.0.smithi136.stdout:child: image204 2024-09-13T00:36:59.709 INFO:tasks.workunit.client.0.smithi136.stdout:child: image203 2024-09-13T00:36:59.709 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-14/image202/b0ff9b1c87d9 2024-09-13T00:36:59.709 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:36:59.709 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-33/image203/b0ff4e56a0d 2024-09-13T00:36:59.709 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:36:59.709 INFO:tasks.workunit.client.0.smithi136.stdout:found child test-librbd-smithi136-114798-14/image204/b0ffa1d9f120 2024-09-13T00:36:59.709 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:36:59.709 INFO:tasks.workunit.client.0.smithi136.stdout:num children is: 0 2024-09-13T00:36:59.709 INFO:tasks.workunit.client.0.smithi136.stdout:expected: 0 2024-09-13T00:36:59.709 INFO:tasks.workunit.client.0.smithi136.stdout:num snaps is: 0 2024-09-13T00:36:59.709 INFO:tasks.workunit.client.0.smithi136.stdout:expected: 0 2024-09-13T00:36:59.709 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.SnapRemoveWithChildMissing (8532 ms) 2024-09-13T00:36:59.710 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.QuiesceWatch 2024-09-13T00:36:59.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:36:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:36:59] "GET /metrics HTTP/1.1" 200 82892 "" "Prometheus/2.43.0" 2024-09-13T00:37:00.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:00 smithi136 bash[76371]: cluster 2024-09-13T00:36:58.935742+0000 mon.a (mon.0) 3441 : cluster [DBG] osdmap e1823: 8 total, 8 up, 8 in 2024-09-13T00:37:00.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:00 smithi136 bash[76371]: cluster 2024-09-13T00:36:59.641570+0000 mon.a (mon.0) 3442 : cluster [DBG] osdmap e1824: 8 total, 8 up, 8 in 2024-09-13T00:37:00.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:00 smithi136 bash[75164]: cluster 2024-09-13T00:36:58.935742+0000 mon.a (mon.0) 3441 : cluster [DBG] osdmap e1823: 8 total, 8 up, 8 in 2024-09-13T00:37:00.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:00 smithi136 bash[75164]: cluster 2024-09-13T00:36:59.641570+0000 mon.a (mon.0) 3442 : cluster [DBG] osdmap e1824: 8 total, 8 up, 8 in 2024-09-13T00:37:00.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:00 smithi181 bash[67587]: cluster 2024-09-13T00:36:58.935742+0000 mon.a (mon.0) 3441 : cluster [DBG] osdmap e1823: 8 total, 8 up, 8 in 2024-09-13T00:37:00.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:00 smithi181 bash[67587]: cluster 2024-09-13T00:36:59.641570+0000 mon.a (mon.0) 3442 : cluster [DBG] osdmap e1824: 8 total, 8 up, 8 in 2024-09-13T00:37:01.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:01 smithi136 bash[75164]: cluster 2024-09-13T00:37:00.310729+0000 mgr.y (mgr.24661) 1796 : cluster [DBG] pgmap v3181: 713 pgs: 713 active+clean; 6.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:37:01.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:01 smithi136 bash[75164]: cluster 2024-09-13T00:37:00.644832+0000 mon.a (mon.0) 3443 : cluster [DBG] osdmap e1825: 8 total, 8 up, 8 in 2024-09-13T00:37:01.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:01 smithi136 bash[75164]: audit 2024-09-13T00:37:00.660415+0000 mon.c (mon.2) 143 : audit [INF] from='client.? 172.21.15.136:0/1945377724' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-34","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:37:01.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:01 smithi136 bash[75164]: audit 2024-09-13T00:37:00.661321+0000 mon.a (mon.0) 3444 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-34","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:37:01.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:01 smithi136 bash[76371]: cluster 2024-09-13T00:37:00.310729+0000 mgr.y (mgr.24661) 1796 : cluster [DBG] pgmap v3181: 713 pgs: 713 active+clean; 6.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:37:01.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:01 smithi136 bash[76371]: cluster 2024-09-13T00:37:00.644832+0000 mon.a (mon.0) 3443 : cluster [DBG] osdmap e1825: 8 total, 8 up, 8 in 2024-09-13T00:37:01.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:01 smithi136 bash[76371]: audit 2024-09-13T00:37:00.660415+0000 mon.c (mon.2) 143 : audit [INF] from='client.? 172.21.15.136:0/1945377724' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-34","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:37:01.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:01 smithi136 bash[76371]: audit 2024-09-13T00:37:00.661321+0000 mon.a (mon.0) 3444 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-34","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:37:01.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:01 smithi181 bash[67587]: cluster 2024-09-13T00:37:00.310729+0000 mgr.y (mgr.24661) 1796 : cluster [DBG] pgmap v3181: 713 pgs: 713 active+clean; 6.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:37:01.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:01 smithi181 bash[67587]: cluster 2024-09-13T00:37:00.644832+0000 mon.a (mon.0) 3443 : cluster [DBG] osdmap e1825: 8 total, 8 up, 8 in 2024-09-13T00:37:01.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:01 smithi181 bash[67587]: audit 2024-09-13T00:37:00.660415+0000 mon.c (mon.2) 143 : audit [INF] from='client.? 172.21.15.136:0/1945377724' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-34","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:37:01.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:01 smithi181 bash[67587]: audit 2024-09-13T00:37:00.661321+0000 mon.a (mon.0) 3444 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-34","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:37:01.660 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:37:02.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:37:02 smithi181 bash[95505]: ts=2024-09-13T00:37:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:37:03.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:02 smithi136 bash[76371]: audit 2024-09-13T00:37:01.649844+0000 mon.a (mon.0) 3445 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-34","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:37:03.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:02 smithi136 bash[76371]: cluster 2024-09-13T00:37:01.650210+0000 mon.a (mon.0) 3446 : cluster [DBG] osdmap e1826: 8 total, 8 up, 8 in 2024-09-13T00:37:03.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:02 smithi136 bash[76371]: cluster 2024-09-13T00:37:02.312793+0000 mgr.y (mgr.24661) 1797 : cluster [DBG] pgmap v3184: 745 pgs: 14 creating+peering, 12 unknown, 719 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 136 KiB/s rd, 8.0 KiB/s wr, 173 op/s 2024-09-13T00:37:03.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:02 smithi136 bash[75164]: audit 2024-09-13T00:37:01.649844+0000 mon.a (mon.0) 3445 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-34","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:37:03.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:02 smithi136 bash[75164]: cluster 2024-09-13T00:37:01.650210+0000 mon.a (mon.0) 3446 : cluster [DBG] osdmap e1826: 8 total, 8 up, 8 in 2024-09-13T00:37:03.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:02 smithi136 bash[75164]: cluster 2024-09-13T00:37:02.312793+0000 mgr.y (mgr.24661) 1797 : cluster [DBG] pgmap v3184: 745 pgs: 14 creating+peering, 12 unknown, 719 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 136 KiB/s rd, 8.0 KiB/s wr, 173 op/s 2024-09-13T00:37:03.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:02 smithi181 bash[67587]: audit 2024-09-13T00:37:01.649844+0000 mon.a (mon.0) 3445 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-34","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:37:03.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:02 smithi181 bash[67587]: cluster 2024-09-13T00:37:01.650210+0000 mon.a (mon.0) 3446 : cluster [DBG] osdmap e1826: 8 total, 8 up, 8 in 2024-09-13T00:37:03.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:02 smithi181 bash[67587]: cluster 2024-09-13T00:37:02.312793+0000 mgr.y (mgr.24661) 1797 : cluster [DBG] pgmap v3184: 745 pgs: 14 creating+peering, 12 unknown, 719 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 136 KiB/s rd, 8.0 KiB/s wr, 173 op/s 2024-09-13T00:37:04.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:03 smithi136 bash[75164]: cluster 2024-09-13T00:37:02.662215+0000 mon.a (mon.0) 3447 : cluster [DBG] osdmap e1827: 8 total, 8 up, 8 in 2024-09-13T00:37:04.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:03 smithi136 bash[76371]: cluster 2024-09-13T00:37:02.662215+0000 mon.a (mon.0) 3447 : cluster [DBG] osdmap e1827: 8 total, 8 up, 8 in 2024-09-13T00:37:04.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:03 smithi181 bash[67587]: cluster 2024-09-13T00:37:02.662215+0000 mon.a (mon.0) 3447 : cluster [DBG] osdmap e1827: 8 total, 8 up, 8 in 2024-09-13T00:37:05.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:04 smithi136 bash[75164]: cluster 2024-09-13T00:37:03.662252+0000 mon.a (mon.0) 3448 : cluster [DBG] osdmap e1828: 8 total, 8 up, 8 in 2024-09-13T00:37:05.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:04 smithi136 bash[75164]: cluster 2024-09-13T00:37:04.314028+0000 mgr.y (mgr.24661) 1798 : cluster [DBG] pgmap v3187: 745 pgs: 14 creating+peering, 6 unknown, 725 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 17 KiB/s rd, 1.2 KiB/s wr, 26 op/s 2024-09-13T00:37:05.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:04 smithi136 bash[75164]: cluster 2024-09-13T00:37:04.642587+0000 mon.a (mon.0) 3449 : cluster [DBG] osdmap e1829: 8 total, 8 up, 8 in 2024-09-13T00:37:05.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:04 smithi136 bash[76371]: cluster 2024-09-13T00:37:03.662252+0000 mon.a (mon.0) 3448 : cluster [DBG] osdmap e1828: 8 total, 8 up, 8 in 2024-09-13T00:37:05.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:04 smithi136 bash[76371]: cluster 2024-09-13T00:37:04.314028+0000 mgr.y (mgr.24661) 1798 : cluster [DBG] pgmap v3187: 745 pgs: 14 creating+peering, 6 unknown, 725 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 17 KiB/s rd, 1.2 KiB/s wr, 26 op/s 2024-09-13T00:37:05.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:04 smithi136 bash[76371]: cluster 2024-09-13T00:37:04.642587+0000 mon.a (mon.0) 3449 : cluster [DBG] osdmap e1829: 8 total, 8 up, 8 in 2024-09-13T00:37:05.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:04 smithi181 bash[67587]: cluster 2024-09-13T00:37:03.662252+0000 mon.a (mon.0) 3448 : cluster [DBG] osdmap e1828: 8 total, 8 up, 8 in 2024-09-13T00:37:05.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:04 smithi181 bash[67587]: cluster 2024-09-13T00:37:04.314028+0000 mgr.y (mgr.24661) 1798 : cluster [DBG] pgmap v3187: 745 pgs: 14 creating+peering, 6 unknown, 725 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 17 KiB/s rd, 1.2 KiB/s wr, 26 op/s 2024-09-13T00:37:05.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:04 smithi181 bash[67587]: cluster 2024-09-13T00:37:04.642587+0000 mon.a (mon.0) 3449 : cluster [DBG] osdmap e1829: 8 total, 8 up, 8 in 2024-09-13T00:37:07.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:06 smithi136 bash[76371]: cluster 2024-09-13T00:37:05.702997+0000 mon.a (mon.0) 3450 : cluster [DBG] osdmap e1830: 8 total, 8 up, 8 in 2024-09-13T00:37:07.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:06 smithi136 bash[76371]: cluster 2024-09-13T00:37:06.316706+0000 mgr.y (mgr.24661) 1799 : cluster [DBG] pgmap v3190: 745 pgs: 7 creating+peering, 738 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 17 KiB/s rd, 1.2 KiB/s wr, 21 op/s 2024-09-13T00:37:07.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:06 smithi136 bash[75164]: cluster 2024-09-13T00:37:05.702997+0000 mon.a (mon.0) 3450 : cluster [DBG] osdmap e1830: 8 total, 8 up, 8 in 2024-09-13T00:37:07.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:06 smithi136 bash[75164]: cluster 2024-09-13T00:37:06.316706+0000 mgr.y (mgr.24661) 1799 : cluster [DBG] pgmap v3190: 745 pgs: 7 creating+peering, 738 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 17 KiB/s rd, 1.2 KiB/s wr, 21 op/s 2024-09-13T00:37:07.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:06 smithi181 bash[67587]: cluster 2024-09-13T00:37:05.702997+0000 mon.a (mon.0) 3450 : cluster [DBG] osdmap e1830: 8 total, 8 up, 8 in 2024-09-13T00:37:07.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:06 smithi181 bash[67587]: cluster 2024-09-13T00:37:06.316706+0000 mgr.y (mgr.24661) 1799 : cluster [DBG] pgmap v3190: 745 pgs: 7 creating+peering, 738 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 17 KiB/s rd, 1.2 KiB/s wr, 21 op/s 2024-09-13T00:37:08.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:07 smithi136 bash[75164]: cluster 2024-09-13T00:37:06.716874+0000 mon.a (mon.0) 3451 : cluster [DBG] osdmap e1831: 8 total, 8 up, 8 in 2024-09-13T00:37:08.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:07 smithi136 bash[76371]: cluster 2024-09-13T00:37:06.716874+0000 mon.a (mon.0) 3451 : cluster [DBG] osdmap e1831: 8 total, 8 up, 8 in 2024-09-13T00:37:08.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:07 smithi181 bash[67587]: cluster 2024-09-13T00:37:06.716874+0000 mon.a (mon.0) 3451 : cluster [DBG] osdmap e1831: 8 total, 8 up, 8 in 2024-09-13T00:37:09.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:08 smithi136 bash[75164]: cluster 2024-09-13T00:37:07.719306+0000 mon.a (mon.0) 3452 : cluster [DBG] osdmap e1832: 8 total, 8 up, 8 in 2024-09-13T00:37:09.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:08 smithi136 bash[75164]: cluster 2024-09-13T00:37:08.317960+0000 mgr.y (mgr.24661) 1800 : cluster [DBG] pgmap v3193: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 18 KiB/s rd, 1.7 KiB/s wr, 24 op/s 2024-09-13T00:37:09.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:08 smithi136 bash[76371]: cluster 2024-09-13T00:37:07.719306+0000 mon.a (mon.0) 3452 : cluster [DBG] osdmap e1832: 8 total, 8 up, 8 in 2024-09-13T00:37:09.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:08 smithi136 bash[76371]: cluster 2024-09-13T00:37:08.317960+0000 mgr.y (mgr.24661) 1800 : cluster [DBG] pgmap v3193: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 18 KiB/s rd, 1.7 KiB/s wr, 24 op/s 2024-09-13T00:37:09.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:08 smithi181 bash[67587]: cluster 2024-09-13T00:37:07.719306+0000 mon.a (mon.0) 3452 : cluster [DBG] osdmap e1832: 8 total, 8 up, 8 in 2024-09-13T00:37:09.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:08 smithi181 bash[67587]: cluster 2024-09-13T00:37:08.317960+0000 mgr.y (mgr.24661) 1800 : cluster [DBG] pgmap v3193: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 18 KiB/s rd, 1.7 KiB/s wr, 24 op/s 2024-09-13T00:37:09.736 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:37:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:37:09] "GET /metrics HTTP/1.1" 200 82943 "" "Prometheus/2.43.0" 2024-09-13T00:37:09.840 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.QuiesceWatch (10131 ms) 2024-09-13T00:37:09.840 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.QuiesceWatchPP 2024-09-13T00:37:09.842 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:37:10.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:09 smithi136 bash[75164]: cluster 2024-09-13T00:37:08.743780+0000 mon.a (mon.0) 3453 : cluster [DBG] osdmap e1833: 8 total, 8 up, 8 in 2024-09-13T00:37:10.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:09 smithi136 bash[75164]: audit 2024-09-13T00:37:08.837319+0000 mon.a (mon.0) 3454 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:37:10.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:09 smithi136 bash[75164]: audit 2024-09-13T00:37:08.838871+0000 mon.a (mon.0) 3455 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:37:10.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:09 smithi136 bash[75164]: cluster 2024-09-13T00:37:09.646133+0000 mon.a (mon.0) 3456 : cluster [DBG] osdmap e1834: 8 total, 8 up, 8 in 2024-09-13T00:37:10.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:09 smithi136 bash[76371]: cluster 2024-09-13T00:37:08.743780+0000 mon.a (mon.0) 3453 : cluster [DBG] osdmap e1833: 8 total, 8 up, 8 in 2024-09-13T00:37:10.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:09 smithi136 bash[76371]: audit 2024-09-13T00:37:08.837319+0000 mon.a (mon.0) 3454 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:37:10.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:09 smithi136 bash[76371]: audit 2024-09-13T00:37:08.838871+0000 mon.a (mon.0) 3455 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:37:10.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:09 smithi136 bash[76371]: cluster 2024-09-13T00:37:09.646133+0000 mon.a (mon.0) 3456 : cluster [DBG] osdmap e1834: 8 total, 8 up, 8 in 2024-09-13T00:37:10.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:09 smithi181 bash[67587]: cluster 2024-09-13T00:37:08.743780+0000 mon.a (mon.0) 3453 : cluster [DBG] osdmap e1833: 8 total, 8 up, 8 in 2024-09-13T00:37:10.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:09 smithi181 bash[67587]: audit 2024-09-13T00:37:08.837319+0000 mon.a (mon.0) 3454 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:37:10.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:09 smithi181 bash[67587]: audit 2024-09-13T00:37:08.838871+0000 mon.a (mon.0) 3455 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:37:10.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:09 smithi181 bash[67587]: cluster 2024-09-13T00:37:09.646133+0000 mon.a (mon.0) 3456 : cluster [DBG] osdmap e1834: 8 total, 8 up, 8 in 2024-09-13T00:37:11.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:10 smithi136 bash[75164]: cluster 2024-09-13T00:37:10.319109+0000 mgr.y (mgr.24661) 1801 : cluster [DBG] pgmap v3196: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 48 KiB/s rd, 5.2 KiB/s wr, 66 op/s 2024-09-13T00:37:11.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:10 smithi136 bash[75164]: audit 2024-09-13T00:37:10.662733+0000 mon.a (mon.0) 3457 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "107.c", "id": [3, 7]}]: dispatch 2024-09-13T00:37:11.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:10 smithi136 bash[75164]: audit 2024-09-13T00:37:10.663549+0000 mon.a (mon.0) 3458 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "109.d", "id": [3, 6]}]: dispatch 2024-09-13T00:37:11.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:10 smithi136 bash[75164]: audit 2024-09-13T00:37:10.663961+0000 mon.a (mon.0) 3459 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "109.11", "id": [1, 0]}]: dispatch 2024-09-13T00:37:11.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:10 smithi136 bash[75164]: audit 2024-09-13T00:37:10.664316+0000 mon.a (mon.0) 3460 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "109.14", "id": [3, 7]}]: dispatch 2024-09-13T00:37:11.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:10 smithi136 bash[75164]: audit 2024-09-13T00:37:10.664688+0000 mon.a (mon.0) 3461 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "109.16", "id": [3, 6]}]: dispatch 2024-09-13T00:37:11.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:10 smithi136 bash[76371]: cluster 2024-09-13T00:37:10.319109+0000 mgr.y (mgr.24661) 1801 : cluster [DBG] pgmap v3196: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 48 KiB/s rd, 5.2 KiB/s wr, 66 op/s 2024-09-13T00:37:11.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:10 smithi136 bash[76371]: audit 2024-09-13T00:37:10.662733+0000 mon.a (mon.0) 3457 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "107.c", "id": [3, 7]}]: dispatch 2024-09-13T00:37:11.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:10 smithi136 bash[76371]: audit 2024-09-13T00:37:10.663549+0000 mon.a (mon.0) 3458 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "109.d", "id": [3, 6]}]: dispatch 2024-09-13T00:37:11.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:10 smithi136 bash[76371]: audit 2024-09-13T00:37:10.663961+0000 mon.a (mon.0) 3459 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "109.11", "id": [1, 0]}]: dispatch 2024-09-13T00:37:11.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:10 smithi136 bash[76371]: audit 2024-09-13T00:37:10.664316+0000 mon.a (mon.0) 3460 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "109.14", "id": [3, 7]}]: dispatch 2024-09-13T00:37:11.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:10 smithi136 bash[76371]: audit 2024-09-13T00:37:10.664688+0000 mon.a (mon.0) 3461 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "109.16", "id": [3, 6]}]: dispatch 2024-09-13T00:37:11.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:10 smithi181 bash[67587]: cluster 2024-09-13T00:37:10.319109+0000 mgr.y (mgr.24661) 1801 : cluster [DBG] pgmap v3196: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 48 KiB/s rd, 5.2 KiB/s wr, 66 op/s 2024-09-13T00:37:11.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:10 smithi181 bash[67587]: audit 2024-09-13T00:37:10.662733+0000 mon.a (mon.0) 3457 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "107.c", "id": [3, 7]}]: dispatch 2024-09-13T00:37:11.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:10 smithi181 bash[67587]: audit 2024-09-13T00:37:10.663549+0000 mon.a (mon.0) 3458 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "109.d", "id": [3, 6]}]: dispatch 2024-09-13T00:37:11.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:10 smithi181 bash[67587]: audit 2024-09-13T00:37:10.663961+0000 mon.a (mon.0) 3459 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "109.11", "id": [1, 0]}]: dispatch 2024-09-13T00:37:11.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:10 smithi181 bash[67587]: audit 2024-09-13T00:37:10.664316+0000 mon.a (mon.0) 3460 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "109.14", "id": [3, 7]}]: dispatch 2024-09-13T00:37:11.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:10 smithi181 bash[67587]: audit 2024-09-13T00:37:10.664688+0000 mon.a (mon.0) 3461 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "109.16", "id": [3, 6]}]: dispatch 2024-09-13T00:37:12.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:11 smithi181 bash[67587]: audit 2024-09-13T00:37:10.757946+0000 mon.a (mon.0) 3462 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "107.c", "id": [3, 7]}]': finished 2024-09-13T00:37:12.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:11 smithi181 bash[67587]: audit 2024-09-13T00:37:10.758108+0000 mon.a (mon.0) 3463 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "109.d", "id": [3, 6]}]': finished 2024-09-13T00:37:12.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:11 smithi181 bash[67587]: audit 2024-09-13T00:37:10.758266+0000 mon.a (mon.0) 3464 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "109.11", "id": [1, 0]}]': finished 2024-09-13T00:37:12.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:11 smithi181 bash[67587]: audit 2024-09-13T00:37:10.758419+0000 mon.a (mon.0) 3465 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "109.14", "id": [3, 7]}]': finished 2024-09-13T00:37:12.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:11 smithi181 bash[67587]: audit 2024-09-13T00:37:10.758558+0000 mon.a (mon.0) 3466 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "109.16", "id": [3, 6]}]': finished 2024-09-13T00:37:12.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:11 smithi181 bash[67587]: cluster 2024-09-13T00:37:10.758679+0000 mon.a (mon.0) 3467 : cluster [DBG] osdmap e1835: 8 total, 8 up, 8 in 2024-09-13T00:37:12.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:11 smithi136 bash[75164]: audit 2024-09-13T00:37:10.757946+0000 mon.a (mon.0) 3462 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "107.c", "id": [3, 7]}]': finished 2024-09-13T00:37:12.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:11 smithi136 bash[75164]: audit 2024-09-13T00:37:10.758108+0000 mon.a (mon.0) 3463 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "109.d", "id": [3, 6]}]': finished 2024-09-13T00:37:12.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:11 smithi136 bash[75164]: audit 2024-09-13T00:37:10.758266+0000 mon.a (mon.0) 3464 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "109.11", "id": [1, 0]}]': finished 2024-09-13T00:37:12.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:11 smithi136 bash[75164]: audit 2024-09-13T00:37:10.758419+0000 mon.a (mon.0) 3465 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "109.14", "id": [3, 7]}]': finished 2024-09-13T00:37:12.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:11 smithi136 bash[75164]: audit 2024-09-13T00:37:10.758558+0000 mon.a (mon.0) 3466 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "109.16", "id": [3, 6]}]': finished 2024-09-13T00:37:12.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:11 smithi136 bash[75164]: cluster 2024-09-13T00:37:10.758679+0000 mon.a (mon.0) 3467 : cluster [DBG] osdmap e1835: 8 total, 8 up, 8 in 2024-09-13T00:37:12.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:11 smithi136 bash[76371]: audit 2024-09-13T00:37:10.757946+0000 mon.a (mon.0) 3462 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "107.c", "id": [3, 7]}]': finished 2024-09-13T00:37:12.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:11 smithi136 bash[76371]: audit 2024-09-13T00:37:10.758108+0000 mon.a (mon.0) 3463 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "109.d", "id": [3, 6]}]': finished 2024-09-13T00:37:12.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:11 smithi136 bash[76371]: audit 2024-09-13T00:37:10.758266+0000 mon.a (mon.0) 3464 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "109.11", "id": [1, 0]}]': finished 2024-09-13T00:37:12.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:11 smithi136 bash[76371]: audit 2024-09-13T00:37:10.758419+0000 mon.a (mon.0) 3465 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "109.14", "id": [3, 7]}]': finished 2024-09-13T00:37:12.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:11 smithi136 bash[76371]: audit 2024-09-13T00:37:10.758558+0000 mon.a (mon.0) 3466 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "109.16", "id": [3, 6]}]': finished 2024-09-13T00:37:12.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:11 smithi136 bash[76371]: cluster 2024-09-13T00:37:10.758679+0000 mon.a (mon.0) 3467 : cluster [DBG] osdmap e1835: 8 total, 8 up, 8 in 2024-09-13T00:37:12.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:37:12 smithi181 bash[95505]: ts=2024-09-13T00:37:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:37:13.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:12 smithi181 bash[67587]: cluster 2024-09-13T00:37:11.757947+0000 mon.a (mon.0) 3468 : cluster [DBG] osdmap e1836: 8 total, 8 up, 8 in 2024-09-13T00:37:13.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:12 smithi181 bash[67587]: cluster 2024-09-13T00:37:12.321165+0000 mgr.y (mgr.24661) 1802 : cluster [DBG] pgmap v3199: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 80 KiB/s rd, 7.7 KiB/s wr, 108 op/s 2024-09-13T00:37:13.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:12 smithi136 bash[75164]: cluster 2024-09-13T00:37:11.757947+0000 mon.a (mon.0) 3468 : cluster [DBG] osdmap e1836: 8 total, 8 up, 8 in 2024-09-13T00:37:13.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:12 smithi136 bash[75164]: cluster 2024-09-13T00:37:12.321165+0000 mgr.y (mgr.24661) 1802 : cluster [DBG] pgmap v3199: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 80 KiB/s rd, 7.7 KiB/s wr, 108 op/s 2024-09-13T00:37:13.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:12 smithi136 bash[76371]: cluster 2024-09-13T00:37:11.757947+0000 mon.a (mon.0) 3468 : cluster [DBG] osdmap e1836: 8 total, 8 up, 8 in 2024-09-13T00:37:13.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:12 smithi136 bash[76371]: cluster 2024-09-13T00:37:12.321165+0000 mgr.y (mgr.24661) 1802 : cluster [DBG] pgmap v3199: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 80 KiB/s rd, 7.7 KiB/s wr, 108 op/s 2024-09-13T00:37:14.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:13 smithi181 bash[67587]: cluster 2024-09-13T00:37:12.783373+0000 mon.a (mon.0) 3469 : cluster [DBG] osdmap e1837: 8 total, 8 up, 8 in 2024-09-13T00:37:14.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:13 smithi136 bash[75164]: cluster 2024-09-13T00:37:12.783373+0000 mon.a (mon.0) 3469 : cluster [DBG] osdmap e1837: 8 total, 8 up, 8 in 2024-09-13T00:37:14.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:13 smithi136 bash[76371]: cluster 2024-09-13T00:37:12.783373+0000 mon.a (mon.0) 3469 : cluster [DBG] osdmap e1837: 8 total, 8 up, 8 in 2024-09-13T00:37:15.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:14 smithi181 bash[67587]: cluster 2024-09-13T00:37:13.797183+0000 mon.a (mon.0) 3470 : cluster [DBG] osdmap e1838: 8 total, 8 up, 8 in 2024-09-13T00:37:15.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:14 smithi181 bash[67587]: cluster 2024-09-13T00:37:14.322370+0000 mgr.y (mgr.24661) 1803 : cluster [DBG] pgmap v3202: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 36 KiB/s rd, 3.2 KiB/s wr, 51 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T00:37:15.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:14 smithi181 bash[67587]: cluster 2024-09-13T00:37:14.647508+0000 mon.a (mon.0) 3471 : cluster [DBG] osdmap e1839: 8 total, 8 up, 8 in 2024-09-13T00:37:15.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:14 smithi136 bash[75164]: cluster 2024-09-13T00:37:13.797183+0000 mon.a (mon.0) 3470 : cluster [DBG] osdmap e1838: 8 total, 8 up, 8 in 2024-09-13T00:37:15.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:14 smithi136 bash[75164]: cluster 2024-09-13T00:37:14.322370+0000 mgr.y (mgr.24661) 1803 : cluster [DBG] pgmap v3202: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 36 KiB/s rd, 3.2 KiB/s wr, 51 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T00:37:15.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:14 smithi136 bash[75164]: cluster 2024-09-13T00:37:14.647508+0000 mon.a (mon.0) 3471 : cluster [DBG] osdmap e1839: 8 total, 8 up, 8 in 2024-09-13T00:37:15.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:14 smithi136 bash[76371]: cluster 2024-09-13T00:37:13.797183+0000 mon.a (mon.0) 3470 : cluster [DBG] osdmap e1838: 8 total, 8 up, 8 in 2024-09-13T00:37:15.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:14 smithi136 bash[76371]: cluster 2024-09-13T00:37:14.322370+0000 mgr.y (mgr.24661) 1803 : cluster [DBG] pgmap v3202: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 36 KiB/s rd, 3.2 KiB/s wr, 51 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T00:37:15.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:14 smithi136 bash[76371]: cluster 2024-09-13T00:37:14.647508+0000 mon.a (mon.0) 3471 : cluster [DBG] osdmap e1839: 8 total, 8 up, 8 in 2024-09-13T00:37:15.819 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.QuiesceWatchPP (5980 ms) 2024-09-13T00:37:15.819 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.QuiesceWatchError 2024-09-13T00:37:15.820 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:37:17.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:16 smithi136 bash[75164]: cluster 2024-09-13T00:37:15.650805+0000 mon.a (mon.0) 3472 : cluster [DBG] osdmap e1840: 8 total, 8 up, 8 in 2024-09-13T00:37:17.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:16 smithi136 bash[75164]: cluster 2024-09-13T00:37:16.325056+0000 mgr.y (mgr.24661) 1804 : cluster [DBG] pgmap v3205: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 27 KiB/s rd, 1023 B/s wr, 37 op/s; 3 B/s, 0 objects/s recovering 2024-09-13T00:37:17.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:16 smithi136 bash[76371]: cluster 2024-09-13T00:37:15.650805+0000 mon.a (mon.0) 3472 : cluster [DBG] osdmap e1840: 8 total, 8 up, 8 in 2024-09-13T00:37:17.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:16 smithi136 bash[76371]: cluster 2024-09-13T00:37:16.325056+0000 mgr.y (mgr.24661) 1804 : cluster [DBG] pgmap v3205: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 27 KiB/s rd, 1023 B/s wr, 37 op/s; 3 B/s, 0 objects/s recovering 2024-09-13T00:37:17.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:16 smithi181 bash[67587]: cluster 2024-09-13T00:37:15.650805+0000 mon.a (mon.0) 3472 : cluster [DBG] osdmap e1840: 8 total, 8 up, 8 in 2024-09-13T00:37:17.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:16 smithi181 bash[67587]: cluster 2024-09-13T00:37:16.325056+0000 mgr.y (mgr.24661) 1804 : cluster [DBG] pgmap v3205: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 27 KiB/s rd, 1023 B/s wr, 37 op/s; 3 B/s, 0 objects/s recovering 2024-09-13T00:37:18.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:17 smithi136 bash[75164]: cluster 2024-09-13T00:37:16.665574+0000 mon.a (mon.0) 3473 : cluster [DBG] osdmap e1841: 8 total, 8 up, 8 in 2024-09-13T00:37:18.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:17 smithi136 bash[76371]: cluster 2024-09-13T00:37:16.665574+0000 mon.a (mon.0) 3473 : cluster [DBG] osdmap e1841: 8 total, 8 up, 8 in 2024-09-13T00:37:18.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:17 smithi181 bash[67587]: cluster 2024-09-13T00:37:16.665574+0000 mon.a (mon.0) 3473 : cluster [DBG] osdmap e1841: 8 total, 8 up, 8 in 2024-09-13T00:37:19.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:18 smithi136 bash[75164]: cluster 2024-09-13T00:37:17.664118+0000 mon.a (mon.0) 3474 : cluster [DBG] osdmap e1842: 8 total, 8 up, 8 in 2024-09-13T00:37:19.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:18 smithi136 bash[75164]: cluster 2024-09-13T00:37:18.326269+0000 mgr.y (mgr.24661) 1805 : cluster [DBG] pgmap v3208: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 101 KiB/s rd, 6.0 KiB/s wr, 132 op/s; 3 B/s, 0 objects/s recovering 2024-09-13T00:37:19.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:18 smithi136 bash[76371]: cluster 2024-09-13T00:37:17.664118+0000 mon.a (mon.0) 3474 : cluster [DBG] osdmap e1842: 8 total, 8 up, 8 in 2024-09-13T00:37:19.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:18 smithi136 bash[76371]: cluster 2024-09-13T00:37:18.326269+0000 mgr.y (mgr.24661) 1805 : cluster [DBG] pgmap v3208: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 101 KiB/s rd, 6.0 KiB/s wr, 132 op/s; 3 B/s, 0 objects/s recovering 2024-09-13T00:37:19.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:18 smithi181 bash[67587]: cluster 2024-09-13T00:37:17.664118+0000 mon.a (mon.0) 3474 : cluster [DBG] osdmap e1842: 8 total, 8 up, 8 in 2024-09-13T00:37:19.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:18 smithi181 bash[67587]: cluster 2024-09-13T00:37:18.326269+0000 mgr.y (mgr.24661) 1805 : cluster [DBG] pgmap v3208: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 101 KiB/s rd, 6.0 KiB/s wr, 132 op/s; 3 B/s, 0 objects/s recovering 2024-09-13T00:37:19.687 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:37:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:37:19] "GET /metrics HTTP/1.1" 200 82943 "" "Prometheus/2.43.0" 2024-09-13T00:37:20.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:19 smithi136 bash[76371]: cluster 2024-09-13T00:37:18.697738+0000 mon.a (mon.0) 3475 : cluster [DBG] osdmap e1843: 8 total, 8 up, 8 in 2024-09-13T00:37:20.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:19 smithi136 bash[76371]: cluster 2024-09-13T00:37:19.653682+0000 mon.a (mon.0) 3476 : cluster [DBG] osdmap e1844: 8 total, 8 up, 8 in 2024-09-13T00:37:20.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:19 smithi136 bash[75164]: cluster 2024-09-13T00:37:18.697738+0000 mon.a (mon.0) 3475 : cluster [DBG] osdmap e1843: 8 total, 8 up, 8 in 2024-09-13T00:37:20.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:19 smithi136 bash[75164]: cluster 2024-09-13T00:37:19.653682+0000 mon.a (mon.0) 3476 : cluster [DBG] osdmap e1844: 8 total, 8 up, 8 in 2024-09-13T00:37:20.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:19 smithi181 bash[67587]: cluster 2024-09-13T00:37:18.697738+0000 mon.a (mon.0) 3475 : cluster [DBG] osdmap e1843: 8 total, 8 up, 8 in 2024-09-13T00:37:20.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:19 smithi181 bash[67587]: cluster 2024-09-13T00:37:19.653682+0000 mon.a (mon.0) 3476 : cluster [DBG] osdmap e1844: 8 total, 8 up, 8 in 2024-09-13T00:37:21.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:20 smithi136 bash[75164]: cluster 2024-09-13T00:37:20.327510+0000 mgr.y (mgr.24661) 1806 : cluster [DBG] pgmap v3211: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 79 KiB/s rd, 5.7 KiB/s wr, 107 op/s 2024-09-13T00:37:21.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:20 smithi136 bash[76371]: cluster 2024-09-13T00:37:20.327510+0000 mgr.y (mgr.24661) 1806 : cluster [DBG] pgmap v3211: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 79 KiB/s rd, 5.7 KiB/s wr, 107 op/s 2024-09-13T00:37:21.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:20 smithi181 bash[67587]: cluster 2024-09-13T00:37:20.327510+0000 mgr.y (mgr.24661) 1806 : cluster [DBG] pgmap v3211: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 79 KiB/s rd, 5.7 KiB/s wr, 107 op/s 2024-09-13T00:37:21.934 INFO:tasks.workunit.client.0.smithi136.stdout:10.00% done 2024-09-13T00:37:21.934 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.QuiesceWatchError (6114 ms) 2024-09-13T00:37:21.934 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.QuiesceWatchTimeout 2024-09-13T00:37:21.934 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:37:21.969 INFO:tasks.workunit.client.0.smithi136.stderr:test quiesce is not long enough to time out 2024-09-13T00:37:22.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:21 smithi136 bash[75164]: cluster 2024-09-13T00:37:20.705288+0000 mon.a (mon.0) 3477 : cluster [DBG] osdmap e1845: 8 total, 8 up, 8 in 2024-09-13T00:37:22.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:21 smithi136 bash[76371]: cluster 2024-09-13T00:37:20.705288+0000 mon.a (mon.0) 3477 : cluster [DBG] osdmap e1845: 8 total, 8 up, 8 in 2024-09-13T00:37:22.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:21 smithi181 bash[67587]: cluster 2024-09-13T00:37:20.705288+0000 mon.a (mon.0) 3477 : cluster [DBG] osdmap e1845: 8 total, 8 up, 8 in 2024-09-13T00:37:22.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:37:22 smithi181 bash[95505]: ts=2024-09-13T00:37:22.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:37:23.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:22 smithi136 bash[75164]: cluster 2024-09-13T00:37:21.705985+0000 mon.a (mon.0) 3478 : cluster [DBG] osdmap e1846: 8 total, 8 up, 8 in 2024-09-13T00:37:23.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:22 smithi136 bash[75164]: cluster 2024-09-13T00:37:22.329621+0000 mgr.y (mgr.24661) 1807 : cluster [DBG] pgmap v3214: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 51 KiB/s rd, 5.2 KiB/s wr, 70 op/s 2024-09-13T00:37:23.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:22 smithi136 bash[76371]: cluster 2024-09-13T00:37:21.705985+0000 mon.a (mon.0) 3478 : cluster [DBG] osdmap e1846: 8 total, 8 up, 8 in 2024-09-13T00:37:23.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:22 smithi136 bash[76371]: cluster 2024-09-13T00:37:22.329621+0000 mgr.y (mgr.24661) 1807 : cluster [DBG] pgmap v3214: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 51 KiB/s rd, 5.2 KiB/s wr, 70 op/s 2024-09-13T00:37:23.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:22 smithi181 bash[67587]: cluster 2024-09-13T00:37:21.705985+0000 mon.a (mon.0) 3478 : cluster [DBG] osdmap e1846: 8 total, 8 up, 8 in 2024-09-13T00:37:23.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:22 smithi181 bash[67587]: cluster 2024-09-13T00:37:22.329621+0000 mgr.y (mgr.24661) 1807 : cluster [DBG] pgmap v3214: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 51 KiB/s rd, 5.2 KiB/s wr, 70 op/s 2024-09-13T00:37:24.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:23 smithi136 bash[75164]: audit 2024-09-13T00:37:23.833701+0000 mon.a (mon.0) 3479 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:37:24.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:23 smithi136 bash[76371]: audit 2024-09-13T00:37:23.833701+0000 mon.a (mon.0) 3479 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:37:24.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:23 smithi181 bash[67587]: audit 2024-09-13T00:37:23.833701+0000 mon.a (mon.0) 3479 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:37:25.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:24 smithi136 bash[75164]: cluster 2024-09-13T00:37:24.332278+0000 mgr.y (mgr.24661) 1808 : cluster [DBG] pgmap v3215: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 37 KiB/s rd, 3.4 KiB/s wr, 51 op/s 2024-09-13T00:37:25.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:24 smithi136 bash[75164]: cluster 2024-09-13T00:37:24.653455+0000 mon.a (mon.0) 3480 : cluster [DBG] osdmap e1847: 8 total, 8 up, 8 in 2024-09-13T00:37:25.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:24 smithi136 bash[76371]: cluster 2024-09-13T00:37:24.332278+0000 mgr.y (mgr.24661) 1808 : cluster [DBG] pgmap v3215: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 37 KiB/s rd, 3.4 KiB/s wr, 51 op/s 2024-09-13T00:37:25.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:24 smithi136 bash[76371]: cluster 2024-09-13T00:37:24.653455+0000 mon.a (mon.0) 3480 : cluster [DBG] osdmap e1847: 8 total, 8 up, 8 in 2024-09-13T00:37:25.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:24 smithi181 bash[67587]: cluster 2024-09-13T00:37:24.332278+0000 mgr.y (mgr.24661) 1808 : cluster [DBG] pgmap v3215: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 37 KiB/s rd, 3.4 KiB/s wr, 51 op/s 2024-09-13T00:37:25.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:24 smithi181 bash[67587]: cluster 2024-09-13T00:37:24.653455+0000 mon.a (mon.0) 3480 : cluster [DBG] osdmap e1847: 8 total, 8 up, 8 in 2024-09-13T00:37:27.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:27 smithi136 bash[75164]: cluster 2024-09-13T00:37:26.335567+0000 mgr.y (mgr.24661) 1809 : cluster [DBG] pgmap v3217: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 40 KiB/s rd, 3.5 KiB/s wr, 55 op/s 2024-09-13T00:37:27.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:27 smithi136 bash[76371]: cluster 2024-09-13T00:37:26.335567+0000 mgr.y (mgr.24661) 1809 : cluster [DBG] pgmap v3217: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 40 KiB/s rd, 3.5 KiB/s wr, 55 op/s 2024-09-13T00:37:27.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:27 smithi181 bash[67587]: cluster 2024-09-13T00:37:26.335567+0000 mgr.y (mgr.24661) 1809 : cluster [DBG] pgmap v3217: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 40 KiB/s rd, 3.5 KiB/s wr, 55 op/s 2024-09-13T00:37:29.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:29 smithi136 bash[76371]: cluster 2024-09-13T00:37:28.338941+0000 mgr.y (mgr.24661) 1810 : cluster [DBG] pgmap v3218: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 40 KiB/s rd, 3.3 KiB/s wr, 55 op/s 2024-09-13T00:37:29.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:37:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:37:29] "GET /metrics HTTP/1.1" 200 82965 "" "Prometheus/2.43.0" 2024-09-13T00:37:29.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:29 smithi136 bash[75164]: cluster 2024-09-13T00:37:28.338941+0000 mgr.y (mgr.24661) 1810 : cluster [DBG] pgmap v3218: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 40 KiB/s rd, 3.3 KiB/s wr, 55 op/s 2024-09-13T00:37:29.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:29 smithi181 bash[67587]: cluster 2024-09-13T00:37:28.338941+0000 mgr.y (mgr.24661) 1810 : cluster [DBG] pgmap v3218: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 40 KiB/s rd, 3.3 KiB/s wr, 55 op/s 2024-09-13T00:37:30.714 INFO:tasks.workunit.client.0.smithi136.stderr:test quiesce is timed out 2024-09-13T00:37:30.735 INFO:tasks.workunit.client.0.smithi136.stderr:waiting for timed out ... 0 2024-09-13T00:37:31.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:30 smithi136 bash[75164]: cluster 2024-09-13T00:37:29.659600+0000 mon.a (mon.0) 3481 : cluster [DBG] osdmap e1848: 8 total, 8 up, 8 in 2024-09-13T00:37:31.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:30 smithi136 bash[75164]: cluster 2024-09-13T00:37:30.340146+0000 mgr.y (mgr.24661) 1811 : cluster [DBG] pgmap v3220: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 14 KiB/s rd, 894 B/s wr, 20 op/s 2024-09-13T00:37:31.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:30 smithi136 bash[76371]: cluster 2024-09-13T00:37:29.659600+0000 mon.a (mon.0) 3481 : cluster [DBG] osdmap e1848: 8 total, 8 up, 8 in 2024-09-13T00:37:31.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:30 smithi136 bash[76371]: cluster 2024-09-13T00:37:30.340146+0000 mgr.y (mgr.24661) 1811 : cluster [DBG] pgmap v3220: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 14 KiB/s rd, 894 B/s wr, 20 op/s 2024-09-13T00:37:31.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:30 smithi181 bash[67587]: cluster 2024-09-13T00:37:29.659600+0000 mon.a (mon.0) 3481 : cluster [DBG] osdmap e1848: 8 total, 8 up, 8 in 2024-09-13T00:37:31.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:30 smithi181 bash[67587]: cluster 2024-09-13T00:37:30.340146+0000 mgr.y (mgr.24661) 1811 : cluster [DBG] pgmap v3220: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 14 KiB/s rd, 894 B/s wr, 20 op/s 2024-09-13T00:37:31.735 INFO:tasks.workunit.client.0.smithi136.stderr:waiting for timed out ... 1 2024-09-13T00:37:32.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:31 smithi136 bash[75164]: cluster 2024-09-13T00:37:30.670426+0000 mon.a (mon.0) 3482 : cluster [DBG] osdmap e1849: 8 total, 8 up, 8 in 2024-09-13T00:37:32.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:31 smithi136 bash[76371]: cluster 2024-09-13T00:37:30.670426+0000 mon.a (mon.0) 3482 : cluster [DBG] osdmap e1849: 8 total, 8 up, 8 in 2024-09-13T00:37:32.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:31 smithi181 bash[67587]: cluster 2024-09-13T00:37:30.670426+0000 mon.a (mon.0) 3482 : cluster [DBG] osdmap e1849: 8 total, 8 up, 8 in 2024-09-13T00:37:32.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:37:32 smithi181 bash[95505]: ts=2024-09-13T00:37:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:37:32.736 INFO:tasks.workunit.client.0.smithi136.stderr:waiting for timed out ... 2 2024-09-13T00:37:33.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:32 smithi136 bash[75164]: cluster 2024-09-13T00:37:32.344132+0000 mgr.y (mgr.24661) 1812 : cluster [DBG] pgmap v3222: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 16 KiB/s rd, 1.3 KiB/s wr, 21 op/s 2024-09-13T00:37:33.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:32 smithi136 bash[76371]: cluster 2024-09-13T00:37:32.344132+0000 mgr.y (mgr.24661) 1812 : cluster [DBG] pgmap v3222: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 16 KiB/s rd, 1.3 KiB/s wr, 21 op/s 2024-09-13T00:37:33.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:32 smithi181 bash[67587]: cluster 2024-09-13T00:37:32.344132+0000 mgr.y (mgr.24661) 1812 : cluster [DBG] pgmap v3222: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 16 KiB/s rd, 1.3 KiB/s wr, 21 op/s 2024-09-13T00:37:33.736 INFO:tasks.workunit.client.0.smithi136.stderr:waiting for timed out ... 3 2024-09-13T00:37:34.736 INFO:tasks.workunit.client.0.smithi136.stderr:waiting for timed out ... 4 2024-09-13T00:37:35.736 INFO:tasks.workunit.client.0.smithi136.stderr:waiting for timed out ... 5 2024-09-13T00:37:35.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:35 smithi136 bash[75164]: cluster 2024-09-13T00:37:34.346865+0000 mgr.y (mgr.24661) 1813 : cluster [DBG] pgmap v3223: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 1022 B/s wr, 15 op/s 2024-09-13T00:37:35.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:35 smithi136 bash[76371]: cluster 2024-09-13T00:37:34.346865+0000 mgr.y (mgr.24661) 1813 : cluster [DBG] pgmap v3223: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 1022 B/s wr, 15 op/s 2024-09-13T00:37:35.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:35 smithi181 bash[67587]: cluster 2024-09-13T00:37:34.346865+0000 mgr.y (mgr.24661) 1813 : cluster [DBG] pgmap v3223: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 1022 B/s wr, 15 op/s 2024-09-13T00:37:36.736 INFO:tasks.workunit.client.0.smithi136.stderr:waiting for timed out ... 6 2024-09-13T00:37:37.736 INFO:tasks.workunit.client.0.smithi136.stderr:waiting for timed out ... 7 2024-09-13T00:37:37.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:37 smithi136 bash[75164]: cluster 2024-09-13T00:37:36.350678+0000 mgr.y (mgr.24661) 1814 : cluster [DBG] pgmap v3224: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 3.6 KiB/s rd, 383 B/s wr, 4 op/s 2024-09-13T00:37:37.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:37 smithi136 bash[76371]: cluster 2024-09-13T00:37:36.350678+0000 mgr.y (mgr.24661) 1814 : cluster [DBG] pgmap v3224: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 3.6 KiB/s rd, 383 B/s wr, 4 op/s 2024-09-13T00:37:37.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:37 smithi181 bash[67587]: cluster 2024-09-13T00:37:36.350678+0000 mgr.y (mgr.24661) 1814 : cluster [DBG] pgmap v3224: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 3.6 KiB/s rd, 383 B/s wr, 4 op/s 2024-09-13T00:37:38.736 INFO:tasks.workunit.client.0.smithi136.stderr:waiting for timed out ... 8 2024-09-13T00:37:39.737 INFO:tasks.workunit.client.0.smithi136.stderr:waiting for timed out ... 9 2024-09-13T00:37:39.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:37:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:37:39] "GET /metrics HTTP/1.1" 200 82966 "" "Prometheus/2.43.0" 2024-09-13T00:37:39.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:39 smithi136 bash[75164]: cluster 2024-09-13T00:37:38.354035+0000 mgr.y (mgr.24661) 1815 : cluster [DBG] pgmap v3225: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 3.3 KiB/s rd, 353 B/s wr, 4 op/s 2024-09-13T00:37:39.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:39 smithi136 bash[75164]: audit 2024-09-13T00:37:38.834087+0000 mon.a (mon.0) 3483 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:37:39.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:39 smithi136 bash[76371]: cluster 2024-09-13T00:37:38.354035+0000 mgr.y (mgr.24661) 1815 : cluster [DBG] pgmap v3225: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 3.3 KiB/s rd, 353 B/s wr, 4 op/s 2024-09-13T00:37:39.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:39 smithi136 bash[76371]: audit 2024-09-13T00:37:38.834087+0000 mon.a (mon.0) 3483 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:37:39.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:39 smithi181 bash[67587]: cluster 2024-09-13T00:37:38.354035+0000 mgr.y (mgr.24661) 1815 : cluster [DBG] pgmap v3225: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 3.3 KiB/s rd, 353 B/s wr, 4 op/s 2024-09-13T00:37:39.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:39 smithi181 bash[67587]: audit 2024-09-13T00:37:38.834087+0000 mon.a (mon.0) 3483 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:37:40.737 INFO:tasks.workunit.client.0.smithi136.stderr:waiting for timed out ... 10 2024-09-13T00:37:40.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:40 smithi136 bash[75164]: audit 2024-09-13T00:37:40.004698+0000 mon.a (mon.0) 3484 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:37:40.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:40 smithi136 bash[75164]: audit 2024-09-13T00:37:40.376372+0000 mon.a (mon.0) 3485 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:37:40.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:40 smithi136 bash[75164]: audit 2024-09-13T00:37:40.381696+0000 mon.a (mon.0) 3486 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:37:40.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:40 smithi136 bash[76371]: audit 2024-09-13T00:37:40.004698+0000 mon.a (mon.0) 3484 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:37:40.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:40 smithi136 bash[76371]: audit 2024-09-13T00:37:40.376372+0000 mon.a (mon.0) 3485 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:37:40.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:40 smithi136 bash[76371]: audit 2024-09-13T00:37:40.381696+0000 mon.a (mon.0) 3486 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:37:40.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:40 smithi181 bash[67587]: audit 2024-09-13T00:37:40.004698+0000 mon.a (mon.0) 3484 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:37:40.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:40 smithi181 bash[67587]: audit 2024-09-13T00:37:40.376372+0000 mon.a (mon.0) 3485 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:37:40.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:40 smithi181 bash[67587]: audit 2024-09-13T00:37:40.381696+0000 mon.a (mon.0) 3486 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:37:41.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:41 smithi136 bash[75164]: cluster 2024-09-13T00:37:40.356521+0000 mgr.y (mgr.24661) 1816 : cluster [DBG] pgmap v3226: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 2.9 KiB/s rd, 306 B/s wr, 3 op/s 2024-09-13T00:37:41.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:41 smithi136 bash[75164]: audit 2024-09-13T00:37:40.814137+0000 mon.a (mon.0) 3487 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:37:41.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:41 smithi136 bash[75164]: audit 2024-09-13T00:37:40.815675+0000 mon.a (mon.0) 3488 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:37:41.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:41 smithi136 bash[75164]: audit 2024-09-13T00:37:40.824636+0000 mon.a (mon.0) 3489 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:37:41.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:41 smithi136 bash[76371]: cluster 2024-09-13T00:37:40.356521+0000 mgr.y (mgr.24661) 1816 : cluster [DBG] pgmap v3226: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 2.9 KiB/s rd, 306 B/s wr, 3 op/s 2024-09-13T00:37:41.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:41 smithi136 bash[76371]: audit 2024-09-13T00:37:40.814137+0000 mon.a (mon.0) 3487 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:37:41.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:41 smithi136 bash[76371]: audit 2024-09-13T00:37:40.815675+0000 mon.a (mon.0) 3488 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:37:41.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:41 smithi136 bash[76371]: audit 2024-09-13T00:37:40.824636+0000 mon.a (mon.0) 3489 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:37:41.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:41 smithi181 bash[67587]: cluster 2024-09-13T00:37:40.356521+0000 mgr.y (mgr.24661) 1816 : cluster [DBG] pgmap v3226: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 2.9 KiB/s rd, 306 B/s wr, 3 op/s 2024-09-13T00:37:41.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:41 smithi181 bash[67587]: audit 2024-09-13T00:37:40.814137+0000 mon.a (mon.0) 3487 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:37:41.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:41 smithi181 bash[67587]: audit 2024-09-13T00:37:40.815675+0000 mon.a (mon.0) 3488 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:37:41.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:41 smithi181 bash[67587]: audit 2024-09-13T00:37:40.824636+0000 mon.a (mon.0) 3489 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:37:42.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:37:42 smithi181 bash[95505]: ts=2024-09-13T00:37:42.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:37:43.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:43 smithi136 bash[75164]: cluster 2024-09-13T00:37:42.361080+0000 mgr.y (mgr.24661) 1817 : cluster [DBG] pgmap v3227: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 2.5 KiB/s rd, 262 B/s wr, 3 op/s 2024-09-13T00:37:43.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:43 smithi136 bash[76371]: cluster 2024-09-13T00:37:42.361080+0000 mgr.y (mgr.24661) 1817 : cluster [DBG] pgmap v3227: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 2.5 KiB/s rd, 262 B/s wr, 3 op/s 2024-09-13T00:37:43.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:43 smithi181 bash[67587]: cluster 2024-09-13T00:37:42.361080+0000 mgr.y (mgr.24661) 1817 : cluster [DBG] pgmap v3227: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 2.5 KiB/s rd, 262 B/s wr, 3 op/s 2024-09-13T00:37:45.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:45 smithi136 bash[75164]: cluster 2024-09-13T00:37:44.363710+0000 mgr.y (mgr.24661) 1818 : cluster [DBG] pgmap v3228: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-09-13T00:37:45.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:45 smithi136 bash[76371]: cluster 2024-09-13T00:37:44.363710+0000 mgr.y (mgr.24661) 1818 : cluster [DBG] pgmap v3228: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-09-13T00:37:45.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:45 smithi181 bash[67587]: cluster 2024-09-13T00:37:44.363710+0000 mgr.y (mgr.24661) 1818 : cluster [DBG] pgmap v3228: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-09-13T00:37:47.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:47 smithi136 bash[75164]: cluster 2024-09-13T00:37:46.367572+0000 mgr.y (mgr.24661) 1819 : cluster [DBG] pgmap v3229: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-09-13T00:37:47.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:47 smithi136 bash[76371]: cluster 2024-09-13T00:37:46.367572+0000 mgr.y (mgr.24661) 1819 : cluster [DBG] pgmap v3229: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-09-13T00:37:47.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:47 smithi181 bash[67587]: cluster 2024-09-13T00:37:46.367572+0000 mgr.y (mgr.24661) 1819 : cluster [DBG] pgmap v3229: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-09-13T00:37:49.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:49 smithi136 bash[75164]: cluster 2024-09-13T00:37:48.370912+0000 mgr.y (mgr.24661) 1820 : cluster [DBG] pgmap v3230: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:37:49.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:37:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:37:49] "GET /metrics HTTP/1.1" 200 82966 "" "Prometheus/2.43.0" 2024-09-13T00:37:49.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:49 smithi136 bash[76371]: cluster 2024-09-13T00:37:48.370912+0000 mgr.y (mgr.24661) 1820 : cluster [DBG] pgmap v3230: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:37:49.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:49 smithi181 bash[67587]: cluster 2024-09-13T00:37:48.370912+0000 mgr.y (mgr.24661) 1820 : cluster [DBG] pgmap v3230: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:37:51.737 INFO:tasks.workunit.client.0.smithi136.stderr:test retry succeeds 2024-09-13T00:37:51.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:51 smithi136 bash[75164]: cluster 2024-09-13T00:37:50.373544+0000 mgr.y (mgr.24661) 1821 : cluster [DBG] pgmap v3231: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:37:51.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:51 smithi136 bash[76371]: cluster 2024-09-13T00:37:50.373544+0000 mgr.y (mgr.24661) 1821 : cluster [DBG] pgmap v3231: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:37:51.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:51 smithi181 bash[67587]: cluster 2024-09-13T00:37:50.373544+0000 mgr.y (mgr.24661) 1821 : cluster [DBG] pgmap v3231: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:37:52.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:37:52 smithi181 bash[95505]: ts=2024-09-13T00:37:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:37:52.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:52 smithi181 bash[67587]: cluster 2024-09-13T00:37:52.378066+0000 mgr.y (mgr.24661) 1822 : cluster [DBG] pgmap v3232: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:37:52.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:52 smithi181 bash[67587]: cluster 2024-09-13T00:37:52.459605+0000 mon.a (mon.0) 3490 : cluster [DBG] osdmap e1850: 8 total, 8 up, 8 in 2024-09-13T00:37:52.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:52 smithi136 bash[75164]: cluster 2024-09-13T00:37:52.378066+0000 mgr.y (mgr.24661) 1822 : cluster [DBG] pgmap v3232: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:37:52.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:52 smithi136 bash[75164]: cluster 2024-09-13T00:37:52.459605+0000 mon.a (mon.0) 3490 : cluster [DBG] osdmap e1850: 8 total, 8 up, 8 in 2024-09-13T00:37:52.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:52 smithi136 bash[76371]: cluster 2024-09-13T00:37:52.378066+0000 mgr.y (mgr.24661) 1822 : cluster [DBG] pgmap v3232: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:37:52.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:52 smithi136 bash[76371]: cluster 2024-09-13T00:37:52.459605+0000 mon.a (mon.0) 3490 : cluster [DBG] osdmap e1850: 8 total, 8 up, 8 in 2024-09-13T00:37:54.745 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.QuiesceWatchTimeout (32812 ms) 2024-09-13T00:37:54.746 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.WriteZeroes 2024-09-13T00:37:54.746 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:37:54.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:54 smithi181 bash[67587]: cluster 2024-09-13T00:37:53.508964+0000 mon.a (mon.0) 3491 : cluster [DBG] osdmap e1851: 8 total, 8 up, 8 in 2024-09-13T00:37:54.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:54 smithi181 bash[67587]: audit 2024-09-13T00:37:53.834523+0000 mon.a (mon.0) 3492 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:37:54.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:54 smithi181 bash[67587]: cluster 2024-09-13T00:37:54.379964+0000 mgr.y (mgr.24661) 1823 : cluster [DBG] pgmap v3235: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:37:54.883 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.WriteZeroes (137 ms) 2024-09-13T00:37:54.883 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.WriteZeroesThickProvision 2024-09-13T00:37:54.883 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:37:55.006 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.WriteZeroesThickProvision (124 ms) 2024-09-13T00:37:55.007 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] TestLibRBD.ConcurentOperations 2024-09-13T00:37:55.007 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:37:55.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:54 smithi136 bash[75164]: cluster 2024-09-13T00:37:53.508964+0000 mon.a (mon.0) 3491 : cluster [DBG] osdmap e1851: 8 total, 8 up, 8 in 2024-09-13T00:37:55.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:54 smithi136 bash[75164]: audit 2024-09-13T00:37:53.834523+0000 mon.a (mon.0) 3492 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:37:55.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:54 smithi136 bash[75164]: cluster 2024-09-13T00:37:54.379964+0000 mgr.y (mgr.24661) 1823 : cluster [DBG] pgmap v3235: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:37:55.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:54 smithi136 bash[76371]: cluster 2024-09-13T00:37:53.508964+0000 mon.a (mon.0) 3491 : cluster [DBG] osdmap e1851: 8 total, 8 up, 8 in 2024-09-13T00:37:55.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:54 smithi136 bash[76371]: audit 2024-09-13T00:37:53.834523+0000 mon.a (mon.0) 3492 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:37:55.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:54 smithi136 bash[76371]: cluster 2024-09-13T00:37:54.379964+0000 mgr.y (mgr.24661) 1823 : cluster [DBG] pgmap v3235: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:37:55.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:55 smithi181 bash[67587]: cluster 2024-09-13T00:37:54.510686+0000 mon.a (mon.0) 3493 : cluster [DBG] osdmap e1852: 8 total, 8 up, 8 in 2024-09-13T00:37:56.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:55 smithi136 bash[75164]: cluster 2024-09-13T00:37:54.510686+0000 mon.a (mon.0) 3493 : cluster [DBG] osdmap e1852: 8 total, 8 up, 8 in 2024-09-13T00:37:56.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:55 smithi136 bash[76371]: cluster 2024-09-13T00:37:54.510686+0000 mon.a (mon.0) 3493 : cluster [DBG] osdmap e1852: 8 total, 8 up, 8 in 2024-09-13T00:37:56.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:56 smithi181 bash[67587]: cluster 2024-09-13T00:37:55.527896+0000 mon.a (mon.0) 3494 : cluster [DBG] osdmap e1853: 8 total, 8 up, 8 in 2024-09-13T00:37:56.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:56 smithi181 bash[67587]: cluster 2024-09-13T00:37:56.383167+0000 mgr.y (mgr.24661) 1824 : cluster [DBG] pgmap v3238: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 42 KiB/s rd, 2.5 KiB/s wr, 52 op/s 2024-09-13T00:37:57.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:56 smithi136 bash[75164]: cluster 2024-09-13T00:37:55.527896+0000 mon.a (mon.0) 3494 : cluster [DBG] osdmap e1853: 8 total, 8 up, 8 in 2024-09-13T00:37:57.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:56 smithi136 bash[75164]: cluster 2024-09-13T00:37:56.383167+0000 mgr.y (mgr.24661) 1824 : cluster [DBG] pgmap v3238: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 42 KiB/s rd, 2.5 KiB/s wr, 52 op/s 2024-09-13T00:37:57.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:56 smithi136 bash[76371]: cluster 2024-09-13T00:37:55.527896+0000 mon.a (mon.0) 3494 : cluster [DBG] osdmap e1853: 8 total, 8 up, 8 in 2024-09-13T00:37:57.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:56 smithi136 bash[76371]: cluster 2024-09-13T00:37:56.383167+0000 mgr.y (mgr.24661) 1824 : cluster [DBG] pgmap v3238: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 42 KiB/s rd, 2.5 KiB/s wr, 52 op/s 2024-09-13T00:37:57.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:57 smithi181 bash[67587]: cluster 2024-09-13T00:37:56.549408+0000 mon.a (mon.0) 3495 : cluster [DBG] osdmap e1854: 8 total, 8 up, 8 in 2024-09-13T00:37:58.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:57 smithi136 bash[75164]: cluster 2024-09-13T00:37:56.549408+0000 mon.a (mon.0) 3495 : cluster [DBG] osdmap e1854: 8 total, 8 up, 8 in 2024-09-13T00:37:58.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:57 smithi136 bash[76371]: cluster 2024-09-13T00:37:56.549408+0000 mon.a (mon.0) 3495 : cluster [DBG] osdmap e1854: 8 total, 8 up, 8 in 2024-09-13T00:37:58.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:58 smithi181 bash[67587]: cluster 2024-09-13T00:37:57.559335+0000 mon.a (mon.0) 3496 : cluster [DBG] osdmap e1855: 8 total, 8 up, 8 in 2024-09-13T00:37:58.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:58 smithi181 bash[67587]: cluster 2024-09-13T00:37:58.385264+0000 mgr.y (mgr.24661) 1825 : cluster [DBG] pgmap v3241: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 119 KiB/s rd, 7.2 KiB/s wr, 162 op/s 2024-09-13T00:37:59.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:58 smithi136 bash[75164]: cluster 2024-09-13T00:37:57.559335+0000 mon.a (mon.0) 3496 : cluster [DBG] osdmap e1855: 8 total, 8 up, 8 in 2024-09-13T00:37:59.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:58 smithi136 bash[75164]: cluster 2024-09-13T00:37:58.385264+0000 mgr.y (mgr.24661) 1825 : cluster [DBG] pgmap v3241: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 119 KiB/s rd, 7.2 KiB/s wr, 162 op/s 2024-09-13T00:37:59.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:58 smithi136 bash[76371]: cluster 2024-09-13T00:37:57.559335+0000 mon.a (mon.0) 3496 : cluster [DBG] osdmap e1855: 8 total, 8 up, 8 in 2024-09-13T00:37:59.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:58 smithi136 bash[76371]: cluster 2024-09-13T00:37:58.385264+0000 mgr.y (mgr.24661) 1825 : cluster [DBG] pgmap v3241: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 119 KiB/s rd, 7.2 KiB/s wr, 162 op/s 2024-09-13T00:37:59.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:37:59 smithi136 bash[75164]: cluster 2024-09-13T00:37:58.572439+0000 mon.a (mon.0) 3497 : cluster [DBG] osdmap e1856: 8 total, 8 up, 8 in 2024-09-13T00:37:59.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:37:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:37:59] "GET /metrics HTTP/1.1" 200 82964 "" "Prometheus/2.43.0" 2024-09-13T00:37:59.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:37:59 smithi136 bash[76371]: cluster 2024-09-13T00:37:58.572439+0000 mon.a (mon.0) 3497 : cluster [DBG] osdmap e1856: 8 total, 8 up, 8 in 2024-09-13T00:37:59.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:37:59 smithi181 bash[67587]: cluster 2024-09-13T00:37:58.572439+0000 mon.a (mon.0) 3497 : cluster [DBG] osdmap e1856: 8 total, 8 up, 8 in 2024-09-13T00:38:00.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:00 smithi181 bash[67587]: cluster 2024-09-13T00:37:59.578665+0000 mon.a (mon.0) 3498 : cluster [DBG] osdmap e1857: 8 total, 8 up, 8 in 2024-09-13T00:38:00.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:00 smithi181 bash[67587]: cluster 2024-09-13T00:37:59.671918+0000 mon.a (mon.0) 3499 : cluster [DBG] osdmap e1858: 8 total, 8 up, 8 in 2024-09-13T00:38:00.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:00 smithi181 bash[67587]: cluster 2024-09-13T00:38:00.386452+0000 mgr.y (mgr.24661) 1826 : cluster [DBG] pgmap v3245: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 94 KiB/s rd, 8.3 KiB/s wr, 140 op/s 2024-09-13T00:38:01.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:00 smithi136 bash[75164]: cluster 2024-09-13T00:37:59.578665+0000 mon.a (mon.0) 3498 : cluster [DBG] osdmap e1857: 8 total, 8 up, 8 in 2024-09-13T00:38:01.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:00 smithi136 bash[75164]: cluster 2024-09-13T00:37:59.671918+0000 mon.a (mon.0) 3499 : cluster [DBG] osdmap e1858: 8 total, 8 up, 8 in 2024-09-13T00:38:01.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:00 smithi136 bash[75164]: cluster 2024-09-13T00:38:00.386452+0000 mgr.y (mgr.24661) 1826 : cluster [DBG] pgmap v3245: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 94 KiB/s rd, 8.3 KiB/s wr, 140 op/s 2024-09-13T00:38:01.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:00 smithi136 bash[76371]: cluster 2024-09-13T00:37:59.578665+0000 mon.a (mon.0) 3498 : cluster [DBG] osdmap e1857: 8 total, 8 up, 8 in 2024-09-13T00:38:01.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:00 smithi136 bash[76371]: cluster 2024-09-13T00:37:59.671918+0000 mon.a (mon.0) 3499 : cluster [DBG] osdmap e1858: 8 total, 8 up, 8 in 2024-09-13T00:38:01.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:00 smithi136 bash[76371]: cluster 2024-09-13T00:38:00.386452+0000 mgr.y (mgr.24661) 1826 : cluster [DBG] pgmap v3245: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 94 KiB/s rd, 8.3 KiB/s wr, 140 op/s 2024-09-13T00:38:02.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:01 smithi136 bash[75164]: cluster 2024-09-13T00:38:00.710013+0000 mon.a (mon.0) 3500 : cluster [DBG] osdmap e1859: 8 total, 8 up, 8 in 2024-09-13T00:38:02.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:01 smithi136 bash[76371]: cluster 2024-09-13T00:38:00.710013+0000 mon.a (mon.0) 3500 : cluster [DBG] osdmap e1859: 8 total, 8 up, 8 in 2024-09-13T00:38:02.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:01 smithi181 bash[67587]: cluster 2024-09-13T00:38:00.710013+0000 mon.a (mon.0) 3500 : cluster [DBG] osdmap e1859: 8 total, 8 up, 8 in 2024-09-13T00:38:02.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:38:02 smithi181 bash[95505]: ts=2024-09-13T00:38:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:38:03.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:02 smithi136 bash[75164]: cluster 2024-09-13T00:38:01.719571+0000 mon.a (mon.0) 3501 : cluster [DBG] osdmap e1860: 8 total, 8 up, 8 in 2024-09-13T00:38:03.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:02 smithi136 bash[75164]: cluster 2024-09-13T00:38:02.388514+0000 mgr.y (mgr.24661) 1827 : cluster [DBG] pgmap v3248: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 22 KiB/s rd, 6.0 KiB/s wr, 34 op/s 2024-09-13T00:38:03.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:02 smithi136 bash[76371]: cluster 2024-09-13T00:38:01.719571+0000 mon.a (mon.0) 3501 : cluster [DBG] osdmap e1860: 8 total, 8 up, 8 in 2024-09-13T00:38:03.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:02 smithi136 bash[76371]: cluster 2024-09-13T00:38:02.388514+0000 mgr.y (mgr.24661) 1827 : cluster [DBG] pgmap v3248: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 22 KiB/s rd, 6.0 KiB/s wr, 34 op/s 2024-09-13T00:38:03.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:02 smithi181 bash[67587]: cluster 2024-09-13T00:38:01.719571+0000 mon.a (mon.0) 3501 : cluster [DBG] osdmap e1860: 8 total, 8 up, 8 in 2024-09-13T00:38:03.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:02 smithi181 bash[67587]: cluster 2024-09-13T00:38:02.388514+0000 mgr.y (mgr.24661) 1827 : cluster [DBG] pgmap v3248: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 22 KiB/s rd, 6.0 KiB/s wr, 34 op/s 2024-09-13T00:38:04.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:03 smithi136 bash[75164]: cluster 2024-09-13T00:38:02.729662+0000 mon.a (mon.0) 3502 : cluster [DBG] osdmap e1861: 8 total, 8 up, 8 in 2024-09-13T00:38:04.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:03 smithi136 bash[76371]: cluster 2024-09-13T00:38:02.729662+0000 mon.a (mon.0) 3502 : cluster [DBG] osdmap e1861: 8 total, 8 up, 8 in 2024-09-13T00:38:04.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:03 smithi181 bash[67587]: cluster 2024-09-13T00:38:02.729662+0000 mon.a (mon.0) 3502 : cluster [DBG] osdmap e1861: 8 total, 8 up, 8 in 2024-09-13T00:38:05.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:04 smithi136 bash[75164]: cluster 2024-09-13T00:38:03.733434+0000 mon.a (mon.0) 3503 : cluster [DBG] osdmap e1862: 8 total, 8 up, 8 in 2024-09-13T00:38:05.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:04 smithi136 bash[75164]: cluster 2024-09-13T00:38:04.389695+0000 mgr.y (mgr.24661) 1828 : cluster [DBG] pgmap v3251: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 9.0 KiB/s rd, 4.5 KiB/s wr, 13 op/s 2024-09-13T00:38:05.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:04 smithi136 bash[75164]: cluster 2024-09-13T00:38:04.673958+0000 mon.a (mon.0) 3504 : cluster [DBG] osdmap e1863: 8 total, 8 up, 8 in 2024-09-13T00:38:05.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:04 smithi136 bash[76371]: cluster 2024-09-13T00:38:03.733434+0000 mon.a (mon.0) 3503 : cluster [DBG] osdmap e1862: 8 total, 8 up, 8 in 2024-09-13T00:38:05.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:04 smithi136 bash[76371]: cluster 2024-09-13T00:38:04.389695+0000 mgr.y (mgr.24661) 1828 : cluster [DBG] pgmap v3251: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 9.0 KiB/s rd, 4.5 KiB/s wr, 13 op/s 2024-09-13T00:38:05.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:04 smithi136 bash[76371]: cluster 2024-09-13T00:38:04.673958+0000 mon.a (mon.0) 3504 : cluster [DBG] osdmap e1863: 8 total, 8 up, 8 in 2024-09-13T00:38:05.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:04 smithi181 bash[67587]: cluster 2024-09-13T00:38:03.733434+0000 mon.a (mon.0) 3503 : cluster [DBG] osdmap e1862: 8 total, 8 up, 8 in 2024-09-13T00:38:05.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:04 smithi181 bash[67587]: cluster 2024-09-13T00:38:04.389695+0000 mgr.y (mgr.24661) 1828 : cluster [DBG] pgmap v3251: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 9.0 KiB/s rd, 4.5 KiB/s wr, 13 op/s 2024-09-13T00:38:05.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:04 smithi181 bash[67587]: cluster 2024-09-13T00:38:04.673958+0000 mon.a (mon.0) 3504 : cluster [DBG] osdmap e1863: 8 total, 8 up, 8 in 2024-09-13T00:38:07.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:06 smithi181 bash[67587]: cluster 2024-09-13T00:38:05.760905+0000 mon.a (mon.0) 3505 : cluster [DBG] osdmap e1864: 8 total, 8 up, 8 in 2024-09-13T00:38:07.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:06 smithi181 bash[67587]: cluster 2024-09-13T00:38:06.391584+0000 mgr.y (mgr.24661) 1829 : cluster [DBG] pgmap v3254: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 255 B/s wr, 9 op/s 2024-09-13T00:38:07.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:06 smithi136 bash[75164]: cluster 2024-09-13T00:38:05.760905+0000 mon.a (mon.0) 3505 : cluster [DBG] osdmap e1864: 8 total, 8 up, 8 in 2024-09-13T00:38:07.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:06 smithi136 bash[75164]: cluster 2024-09-13T00:38:06.391584+0000 mgr.y (mgr.24661) 1829 : cluster [DBG] pgmap v3254: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 255 B/s wr, 9 op/s 2024-09-13T00:38:07.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:06 smithi136 bash[76371]: cluster 2024-09-13T00:38:05.760905+0000 mon.a (mon.0) 3505 : cluster [DBG] osdmap e1864: 8 total, 8 up, 8 in 2024-09-13T00:38:07.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:06 smithi136 bash[76371]: cluster 2024-09-13T00:38:06.391584+0000 mgr.y (mgr.24661) 1829 : cluster [DBG] pgmap v3254: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 255 B/s wr, 9 op/s 2024-09-13T00:38:08.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:07 smithi181 bash[67587]: cluster 2024-09-13T00:38:06.771707+0000 mon.a (mon.0) 3506 : cluster [DBG] osdmap e1865: 8 total, 8 up, 8 in 2024-09-13T00:38:08.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:07 smithi136 bash[75164]: cluster 2024-09-13T00:38:06.771707+0000 mon.a (mon.0) 3506 : cluster [DBG] osdmap e1865: 8 total, 8 up, 8 in 2024-09-13T00:38:08.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:07 smithi136 bash[76371]: cluster 2024-09-13T00:38:06.771707+0000 mon.a (mon.0) 3506 : cluster [DBG] osdmap e1865: 8 total, 8 up, 8 in 2024-09-13T00:38:09.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:08 smithi181 bash[67587]: cluster 2024-09-13T00:38:07.772885+0000 mon.a (mon.0) 3507 : cluster [DBG] osdmap e1866: 8 total, 8 up, 8 in 2024-09-13T00:38:09.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:08 smithi181 bash[67587]: cluster 2024-09-13T00:38:08.392839+0000 mgr.y (mgr.24661) 1830 : cluster [DBG] pgmap v3257: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 181 KiB/s rd, 5.5 KiB/s wr, 248 op/s 2024-09-13T00:38:09.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:08 smithi136 bash[75164]: cluster 2024-09-13T00:38:07.772885+0000 mon.a (mon.0) 3507 : cluster [DBG] osdmap e1866: 8 total, 8 up, 8 in 2024-09-13T00:38:09.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:08 smithi136 bash[75164]: cluster 2024-09-13T00:38:08.392839+0000 mgr.y (mgr.24661) 1830 : cluster [DBG] pgmap v3257: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 181 KiB/s rd, 5.5 KiB/s wr, 248 op/s 2024-09-13T00:38:09.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:08 smithi136 bash[76371]: cluster 2024-09-13T00:38:07.772885+0000 mon.a (mon.0) 3507 : cluster [DBG] osdmap e1866: 8 total, 8 up, 8 in 2024-09-13T00:38:09.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:08 smithi136 bash[76371]: cluster 2024-09-13T00:38:08.392839+0000 mgr.y (mgr.24661) 1830 : cluster [DBG] pgmap v3257: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 181 KiB/s rd, 5.5 KiB/s wr, 248 op/s 2024-09-13T00:38:09.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:38:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:38:09] "GET /metrics HTTP/1.1" 200 82966 "" "Prometheus/2.43.0" 2024-09-13T00:38:10.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:09 smithi181 bash[67587]: cluster 2024-09-13T00:38:08.803548+0000 mon.a (mon.0) 3508 : cluster [DBG] osdmap e1867: 8 total, 8 up, 8 in 2024-09-13T00:38:10.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:09 smithi181 bash[67587]: audit 2024-09-13T00:38:08.835676+0000 mon.a (mon.0) 3509 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:38:10.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:09 smithi136 bash[75164]: cluster 2024-09-13T00:38:08.803548+0000 mon.a (mon.0) 3508 : cluster [DBG] osdmap e1867: 8 total, 8 up, 8 in 2024-09-13T00:38:10.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:09 smithi136 bash[75164]: audit 2024-09-13T00:38:08.835676+0000 mon.a (mon.0) 3509 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:38:10.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:09 smithi136 bash[76371]: cluster 2024-09-13T00:38:08.803548+0000 mon.a (mon.0) 3508 : cluster [DBG] osdmap e1867: 8 total, 8 up, 8 in 2024-09-13T00:38:10.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:09 smithi136 bash[76371]: audit 2024-09-13T00:38:08.835676+0000 mon.a (mon.0) 3509 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:38:11.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:10 smithi181 bash[67587]: cluster 2024-09-13T00:38:09.805244+0000 mon.a (mon.0) 3510 : cluster [DBG] osdmap e1868: 8 total, 8 up, 8 in 2024-09-13T00:38:11.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:10 smithi181 bash[67587]: cluster 2024-09-13T00:38:10.394104+0000 mgr.y (mgr.24661) 1831 : cluster [DBG] pgmap v3260: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 174 KiB/s rd, 5.5 KiB/s wr, 239 op/s 2024-09-13T00:38:11.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:10 smithi136 bash[75164]: cluster 2024-09-13T00:38:09.805244+0000 mon.a (mon.0) 3510 : cluster [DBG] osdmap e1868: 8 total, 8 up, 8 in 2024-09-13T00:38:11.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:10 smithi136 bash[75164]: cluster 2024-09-13T00:38:10.394104+0000 mgr.y (mgr.24661) 1831 : cluster [DBG] pgmap v3260: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 174 KiB/s rd, 5.5 KiB/s wr, 239 op/s 2024-09-13T00:38:11.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:10 smithi136 bash[76371]: cluster 2024-09-13T00:38:09.805244+0000 mon.a (mon.0) 3510 : cluster [DBG] osdmap e1868: 8 total, 8 up, 8 in 2024-09-13T00:38:11.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:10 smithi136 bash[76371]: cluster 2024-09-13T00:38:10.394104+0000 mgr.y (mgr.24661) 1831 : cluster [DBG] pgmap v3260: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 174 KiB/s rd, 5.5 KiB/s wr, 239 op/s 2024-09-13T00:38:12.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:11 smithi181 bash[67587]: cluster 2024-09-13T00:38:10.828202+0000 mon.a (mon.0) 3511 : cluster [DBG] osdmap e1869: 8 total, 8 up, 8 in 2024-09-13T00:38:12.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:11 smithi136 bash[75164]: cluster 2024-09-13T00:38:10.828202+0000 mon.a (mon.0) 3511 : cluster [DBG] osdmap e1869: 8 total, 8 up, 8 in 2024-09-13T00:38:12.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:11 smithi136 bash[76371]: cluster 2024-09-13T00:38:10.828202+0000 mon.a (mon.0) 3511 : cluster [DBG] osdmap e1869: 8 total, 8 up, 8 in 2024-09-13T00:38:12.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:38:12 smithi181 bash[95505]: ts=2024-09-13T00:38:12.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:38:13.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:12 smithi181 bash[67587]: cluster 2024-09-13T00:38:11.829369+0000 mon.a (mon.0) 3512 : cluster [DBG] osdmap e1870: 8 total, 8 up, 8 in 2024-09-13T00:38:13.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:12 smithi181 bash[67587]: cluster 2024-09-13T00:38:12.396175+0000 mgr.y (mgr.24661) 1832 : cluster [DBG] pgmap v3263: 745 pgs: 2 active+clean+snaptrim, 743 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 4.0 KiB/s wr, 20 op/s 2024-09-13T00:38:13.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:12 smithi136 bash[75164]: cluster 2024-09-13T00:38:11.829369+0000 mon.a (mon.0) 3512 : cluster [DBG] osdmap e1870: 8 total, 8 up, 8 in 2024-09-13T00:38:13.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:12 smithi136 bash[75164]: cluster 2024-09-13T00:38:12.396175+0000 mgr.y (mgr.24661) 1832 : cluster [DBG] pgmap v3263: 745 pgs: 2 active+clean+snaptrim, 743 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 4.0 KiB/s wr, 20 op/s 2024-09-13T00:38:13.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:12 smithi136 bash[76371]: cluster 2024-09-13T00:38:11.829369+0000 mon.a (mon.0) 3512 : cluster [DBG] osdmap e1870: 8 total, 8 up, 8 in 2024-09-13T00:38:13.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:12 smithi136 bash[76371]: cluster 2024-09-13T00:38:12.396175+0000 mgr.y (mgr.24661) 1832 : cluster [DBG] pgmap v3263: 745 pgs: 2 active+clean+snaptrim, 743 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 4.0 KiB/s wr, 20 op/s 2024-09-13T00:38:14.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:13 smithi136 bash[75164]: cluster 2024-09-13T00:38:12.855773+0000 mon.a (mon.0) 3513 : cluster [DBG] osdmap e1871: 8 total, 8 up, 8 in 2024-09-13T00:38:14.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:13 smithi136 bash[76371]: cluster 2024-09-13T00:38:12.855773+0000 mon.a (mon.0) 3513 : cluster [DBG] osdmap e1871: 8 total, 8 up, 8 in 2024-09-13T00:38:14.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:13 smithi181 bash[67587]: cluster 2024-09-13T00:38:12.855773+0000 mon.a (mon.0) 3513 : cluster [DBG] osdmap e1871: 8 total, 8 up, 8 in 2024-09-13T00:38:15.264 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:14 smithi136 bash[75164]: cluster 2024-09-13T00:38:13.859044+0000 mon.a (mon.0) 3514 : cluster [DBG] osdmap e1872: 8 total, 8 up, 8 in 2024-09-13T00:38:15.265 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:14 smithi136 bash[75164]: cluster 2024-09-13T00:38:14.397417+0000 mgr.y (mgr.24661) 1833 : cluster [DBG] pgmap v3266: 745 pgs: 2 active+clean+snaptrim, 743 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 4.0 KiB/s wr, 20 op/s 2024-09-13T00:38:15.265 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:14 smithi136 bash[75164]: cluster 2024-09-13T00:38:14.677448+0000 mon.a (mon.0) 3515 : cluster [DBG] osdmap e1873: 8 total, 8 up, 8 in 2024-09-13T00:38:15.265 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:14 smithi136 bash[76371]: cluster 2024-09-13T00:38:13.859044+0000 mon.a (mon.0) 3514 : cluster [DBG] osdmap e1872: 8 total, 8 up, 8 in 2024-09-13T00:38:15.265 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:14 smithi136 bash[76371]: cluster 2024-09-13T00:38:14.397417+0000 mgr.y (mgr.24661) 1833 : cluster [DBG] pgmap v3266: 745 pgs: 2 active+clean+snaptrim, 743 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 4.0 KiB/s wr, 20 op/s 2024-09-13T00:38:15.265 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:14 smithi136 bash[76371]: cluster 2024-09-13T00:38:14.677448+0000 mon.a (mon.0) 3515 : cluster [DBG] osdmap e1873: 8 total, 8 up, 8 in 2024-09-13T00:38:15.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:14 smithi181 bash[67587]: cluster 2024-09-13T00:38:13.859044+0000 mon.a (mon.0) 3514 : cluster [DBG] osdmap e1872: 8 total, 8 up, 8 in 2024-09-13T00:38:15.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:14 smithi181 bash[67587]: cluster 2024-09-13T00:38:14.397417+0000 mgr.y (mgr.24661) 1833 : cluster [DBG] pgmap v3266: 745 pgs: 2 active+clean+snaptrim, 743 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 4.0 KiB/s wr, 20 op/s 2024-09-13T00:38:15.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:14 smithi181 bash[67587]: cluster 2024-09-13T00:38:14.677448+0000 mon.a (mon.0) 3515 : cluster [DBG] osdmap e1873: 8 total, 8 up, 8 in 2024-09-13T00:38:17.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:16 smithi136 bash[75164]: cluster 2024-09-13T00:38:15.681681+0000 mon.a (mon.0) 3516 : cluster [DBG] osdmap e1874: 8 total, 8 up, 8 in 2024-09-13T00:38:17.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:16 smithi136 bash[75164]: cluster 2024-09-13T00:38:16.400057+0000 mgr.y (mgr.24661) 1834 : cluster [DBG] pgmap v3269: 745 pgs: 2 active+clean+snaptrim, 743 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 2.5 KiB/s rd, 0 B/s wr, 3 op/s 2024-09-13T00:38:17.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:16 smithi136 bash[76371]: cluster 2024-09-13T00:38:15.681681+0000 mon.a (mon.0) 3516 : cluster [DBG] osdmap e1874: 8 total, 8 up, 8 in 2024-09-13T00:38:17.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:16 smithi136 bash[76371]: cluster 2024-09-13T00:38:16.400057+0000 mgr.y (mgr.24661) 1834 : cluster [DBG] pgmap v3269: 745 pgs: 2 active+clean+snaptrim, 743 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 2.5 KiB/s rd, 0 B/s wr, 3 op/s 2024-09-13T00:38:17.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:16 smithi181 bash[67587]: cluster 2024-09-13T00:38:15.681681+0000 mon.a (mon.0) 3516 : cluster [DBG] osdmap e1874: 8 total, 8 up, 8 in 2024-09-13T00:38:17.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:16 smithi181 bash[67587]: cluster 2024-09-13T00:38:16.400057+0000 mgr.y (mgr.24661) 1834 : cluster [DBG] pgmap v3269: 745 pgs: 2 active+clean+snaptrim, 743 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 2.5 KiB/s rd, 0 B/s wr, 3 op/s 2024-09-13T00:38:18.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:17 smithi136 bash[75164]: cluster 2024-09-13T00:38:16.684876+0000 mon.a (mon.0) 3517 : cluster [DBG] osdmap e1875: 8 total, 8 up, 8 in 2024-09-13T00:38:18.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:17 smithi136 bash[76371]: cluster 2024-09-13T00:38:16.684876+0000 mon.a (mon.0) 3517 : cluster [DBG] osdmap e1875: 8 total, 8 up, 8 in 2024-09-13T00:38:18.105 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:17 smithi181 bash[67587]: cluster 2024-09-13T00:38:16.684876+0000 mon.a (mon.0) 3517 : cluster [DBG] osdmap e1875: 8 total, 8 up, 8 in 2024-09-13T00:38:19.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:18 smithi136 bash[75164]: cluster 2024-09-13T00:38:17.714189+0000 mon.a (mon.0) 3518 : cluster [DBG] osdmap e1876: 8 total, 8 up, 8 in 2024-09-13T00:38:19.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:18 smithi136 bash[75164]: cluster 2024-09-13T00:38:18.401312+0000 mgr.y (mgr.24661) 1835 : cluster [DBG] pgmap v3272: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 47 KiB/s rd, 4.5 KiB/s wr, 62 op/s 2024-09-13T00:38:19.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:18 smithi136 bash[76371]: cluster 2024-09-13T00:38:17.714189+0000 mon.a (mon.0) 3518 : cluster [DBG] osdmap e1876: 8 total, 8 up, 8 in 2024-09-13T00:38:19.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:18 smithi136 bash[76371]: cluster 2024-09-13T00:38:18.401312+0000 mgr.y (mgr.24661) 1835 : cluster [DBG] pgmap v3272: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 47 KiB/s rd, 4.5 KiB/s wr, 62 op/s 2024-09-13T00:38:19.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:18 smithi181 bash[67587]: cluster 2024-09-13T00:38:17.714189+0000 mon.a (mon.0) 3518 : cluster [DBG] osdmap e1876: 8 total, 8 up, 8 in 2024-09-13T00:38:19.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:18 smithi181 bash[67587]: cluster 2024-09-13T00:38:18.401312+0000 mgr.y (mgr.24661) 1835 : cluster [DBG] pgmap v3272: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 47 KiB/s rd, 4.5 KiB/s wr, 62 op/s 2024-09-13T00:38:19.720 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:38:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:38:19] "GET /metrics HTTP/1.1" 200 82966 "" "Prometheus/2.43.0" 2024-09-13T00:38:19.845 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] TestLibRBD.ConcurentOperations (24838 ms) 2024-09-13T00:38:20.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:19 smithi136 bash[76371]: cluster 2024-09-13T00:38:18.726597+0000 mon.a (mon.0) 3519 : cluster [DBG] osdmap e1877: 8 total, 8 up, 8 in 2024-09-13T00:38:20.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:19 smithi136 bash[76371]: cluster 2024-09-13T00:38:19.680591+0000 mon.a (mon.0) 3520 : cluster [DBG] osdmap e1878: 8 total, 8 up, 8 in 2024-09-13T00:38:20.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:19 smithi136 bash[75164]: cluster 2024-09-13T00:38:18.726597+0000 mon.a (mon.0) 3519 : cluster [DBG] osdmap e1877: 8 total, 8 up, 8 in 2024-09-13T00:38:20.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:19 smithi136 bash[75164]: cluster 2024-09-13T00:38:19.680591+0000 mon.a (mon.0) 3520 : cluster [DBG] osdmap e1878: 8 total, 8 up, 8 in 2024-09-13T00:38:20.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:19 smithi181 bash[67587]: cluster 2024-09-13T00:38:18.726597+0000 mon.a (mon.0) 3519 : cluster [DBG] osdmap e1877: 8 total, 8 up, 8 in 2024-09-13T00:38:20.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:19 smithi181 bash[67587]: cluster 2024-09-13T00:38:19.680591+0000 mon.a (mon.0) 3520 : cluster [DBG] osdmap e1878: 8 total, 8 up, 8 in 2024-09-13T00:38:21.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:20 smithi136 bash[76371]: cluster 2024-09-13T00:38:20.402522+0000 mgr.y (mgr.24661) 1836 : cluster [DBG] pgmap v3275: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 49 KiB/s rd, 5.0 KiB/s wr, 67 op/s 2024-09-13T00:38:21.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:20 smithi136 bash[75164]: cluster 2024-09-13T00:38:20.402522+0000 mgr.y (mgr.24661) 1836 : cluster [DBG] pgmap v3275: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 49 KiB/s rd, 5.0 KiB/s wr, 67 op/s 2024-09-13T00:38:21.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:20 smithi181 bash[67587]: cluster 2024-09-13T00:38:20.402522+0000 mgr.y (mgr.24661) 1836 : cluster [DBG] pgmap v3275: 745 pgs: 745 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 49 KiB/s rd, 5.0 KiB/s wr, 67 op/s 2024-09-13T00:38:22.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:21 smithi136 bash[76371]: cluster 2024-09-13T00:38:20.739296+0000 mon.a (mon.0) 3521 : cluster [DBG] osdmap e1879: 8 total, 8 up, 8 in 2024-09-13T00:38:22.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:21 smithi136 bash[75164]: cluster 2024-09-13T00:38:20.739296+0000 mon.a (mon.0) 3521 : cluster [DBG] osdmap e1879: 8 total, 8 up, 8 in 2024-09-13T00:38:22.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:21 smithi181 bash[67587]: cluster 2024-09-13T00:38:20.739296+0000 mon.a (mon.0) 3521 : cluster [DBG] osdmap e1879: 8 total, 8 up, 8 in 2024-09-13T00:38:22.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:38:22 smithi181 bash[95505]: ts=2024-09-13T00:38:22.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:38:23.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:22 smithi136 bash[75164]: cluster 2024-09-13T00:38:21.748179+0000 mon.a (mon.0) 3522 : cluster [DBG] osdmap e1880: 8 total, 8 up, 8 in 2024-09-13T00:38:23.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:22 smithi136 bash[75164]: cluster 2024-09-13T00:38:22.404503+0000 mgr.y (mgr.24661) 1837 : cluster [DBG] pgmap v3278: 681 pgs: 681 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:38:23.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:22 smithi136 bash[76371]: cluster 2024-09-13T00:38:21.748179+0000 mon.a (mon.0) 3522 : cluster [DBG] osdmap e1880: 8 total, 8 up, 8 in 2024-09-13T00:38:23.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:22 smithi136 bash[76371]: cluster 2024-09-13T00:38:22.404503+0000 mgr.y (mgr.24661) 1837 : cluster [DBG] pgmap v3278: 681 pgs: 681 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:38:23.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:22 smithi181 bash[67587]: cluster 2024-09-13T00:38:21.748179+0000 mon.a (mon.0) 3522 : cluster [DBG] osdmap e1880: 8 total, 8 up, 8 in 2024-09-13T00:38:23.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:22 smithi181 bash[67587]: cluster 2024-09-13T00:38:22.404503+0000 mgr.y (mgr.24661) 1837 : cluster [DBG] pgmap v3278: 681 pgs: 681 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:38:24.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:23 smithi181 bash[67587]: cluster 2024-09-13T00:38:22.761804+0000 mon.a (mon.0) 3523 : cluster [DBG] osdmap e1881: 8 total, 8 up, 8 in 2024-09-13T00:38:24.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:23 smithi136 bash[76371]: cluster 2024-09-13T00:38:22.761804+0000 mon.a (mon.0) 3523 : cluster [DBG] osdmap e1881: 8 total, 8 up, 8 in 2024-09-13T00:38:24.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:23 smithi136 bash[75164]: cluster 2024-09-13T00:38:22.761804+0000 mon.a (mon.0) 3523 : cluster [DBG] osdmap e1881: 8 total, 8 up, 8 in 2024-09-13T00:38:25.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:24 smithi181 bash[67587]: cluster 2024-09-13T00:38:23.763735+0000 mon.a (mon.0) 3524 : cluster [DBG] osdmap e1882: 8 total, 8 up, 8 in 2024-09-13T00:38:25.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:24 smithi181 bash[67587]: audit 2024-09-13T00:38:23.835067+0000 mon.a (mon.0) 3525 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:38:25.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:24 smithi181 bash[67587]: cluster 2024-09-13T00:38:24.405573+0000 mgr.y (mgr.24661) 1838 : cluster [DBG] pgmap v3281: 617 pgs: 617 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:38:25.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:24 smithi181 bash[67587]: cluster 2024-09-13T00:38:24.683379+0000 mon.a (mon.0) 3526 : cluster [DBG] osdmap e1883: 8 total, 8 up, 8 in 2024-09-13T00:38:25.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:24 smithi136 bash[75164]: cluster 2024-09-13T00:38:23.763735+0000 mon.a (mon.0) 3524 : cluster [DBG] osdmap e1882: 8 total, 8 up, 8 in 2024-09-13T00:38:25.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:24 smithi136 bash[75164]: audit 2024-09-13T00:38:23.835067+0000 mon.a (mon.0) 3525 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:38:25.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:24 smithi136 bash[75164]: cluster 2024-09-13T00:38:24.405573+0000 mgr.y (mgr.24661) 1838 : cluster [DBG] pgmap v3281: 617 pgs: 617 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:38:25.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:24 smithi136 bash[75164]: cluster 2024-09-13T00:38:24.683379+0000 mon.a (mon.0) 3526 : cluster [DBG] osdmap e1883: 8 total, 8 up, 8 in 2024-09-13T00:38:25.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:24 smithi136 bash[76371]: cluster 2024-09-13T00:38:23.763735+0000 mon.a (mon.0) 3524 : cluster [DBG] osdmap e1882: 8 total, 8 up, 8 in 2024-09-13T00:38:25.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:24 smithi136 bash[76371]: audit 2024-09-13T00:38:23.835067+0000 mon.a (mon.0) 3525 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:38:25.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:24 smithi136 bash[76371]: cluster 2024-09-13T00:38:24.405573+0000 mgr.y (mgr.24661) 1838 : cluster [DBG] pgmap v3281: 617 pgs: 617 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:38:25.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:24 smithi136 bash[76371]: cluster 2024-09-13T00:38:24.683379+0000 mon.a (mon.0) 3526 : cluster [DBG] osdmap e1883: 8 total, 8 up, 8 in 2024-09-13T00:38:27.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:26 smithi136 bash[75164]: cluster 2024-09-13T00:38:25.685722+0000 mon.a (mon.0) 3527 : cluster [DBG] osdmap e1884: 8 total, 8 up, 8 in 2024-09-13T00:38:27.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:26 smithi136 bash[75164]: cluster 2024-09-13T00:38:26.407564+0000 mgr.y (mgr.24661) 1839 : cluster [DBG] pgmap v3284: 553 pgs: 553 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:38:27.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:26 smithi136 bash[76371]: cluster 2024-09-13T00:38:25.685722+0000 mon.a (mon.0) 3527 : cluster [DBG] osdmap e1884: 8 total, 8 up, 8 in 2024-09-13T00:38:27.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:26 smithi136 bash[76371]: cluster 2024-09-13T00:38:26.407564+0000 mgr.y (mgr.24661) 1839 : cluster [DBG] pgmap v3284: 553 pgs: 553 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:38:27.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:26 smithi181 bash[67587]: cluster 2024-09-13T00:38:25.685722+0000 mon.a (mon.0) 3527 : cluster [DBG] osdmap e1884: 8 total, 8 up, 8 in 2024-09-13T00:38:27.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:26 smithi181 bash[67587]: cluster 2024-09-13T00:38:26.407564+0000 mgr.y (mgr.24661) 1839 : cluster [DBG] pgmap v3284: 553 pgs: 553 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:38:28.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:27 smithi136 bash[75164]: cluster 2024-09-13T00:38:26.698441+0000 mon.a (mon.0) 3528 : cluster [DBG] osdmap e1885: 8 total, 8 up, 8 in 2024-09-13T00:38:28.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:27 smithi136 bash[76371]: cluster 2024-09-13T00:38:26.698441+0000 mon.a (mon.0) 3528 : cluster [DBG] osdmap e1885: 8 total, 8 up, 8 in 2024-09-13T00:38:28.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:27 smithi181 bash[67587]: cluster 2024-09-13T00:38:26.698441+0000 mon.a (mon.0) 3528 : cluster [DBG] osdmap e1885: 8 total, 8 up, 8 in 2024-09-13T00:38:29.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:28 smithi136 bash[76371]: cluster 2024-09-13T00:38:27.698940+0000 mon.a (mon.0) 3529 : cluster [DBG] osdmap e1886: 8 total, 8 up, 8 in 2024-09-13T00:38:29.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:28 smithi136 bash[76371]: cluster 2024-09-13T00:38:28.408710+0000 mgr.y (mgr.24661) 1840 : cluster [DBG] pgmap v3287: 489 pgs: 489 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:38:29.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:28 smithi136 bash[76371]: cluster 2024-09-13T00:38:28.701895+0000 mon.a (mon.0) 3530 : cluster [DBG] osdmap e1887: 8 total, 8 up, 8 in 2024-09-13T00:38:29.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:28 smithi136 bash[75164]: cluster 2024-09-13T00:38:27.698940+0000 mon.a (mon.0) 3529 : cluster [DBG] osdmap e1886: 8 total, 8 up, 8 in 2024-09-13T00:38:29.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:28 smithi136 bash[75164]: cluster 2024-09-13T00:38:28.408710+0000 mgr.y (mgr.24661) 1840 : cluster [DBG] pgmap v3287: 489 pgs: 489 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:38:29.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:28 smithi136 bash[75164]: cluster 2024-09-13T00:38:28.701895+0000 mon.a (mon.0) 3530 : cluster [DBG] osdmap e1887: 8 total, 8 up, 8 in 2024-09-13T00:38:29.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:28 smithi181 bash[67587]: cluster 2024-09-13T00:38:27.698940+0000 mon.a (mon.0) 3529 : cluster [DBG] osdmap e1886: 8 total, 8 up, 8 in 2024-09-13T00:38:29.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:28 smithi181 bash[67587]: cluster 2024-09-13T00:38:28.408710+0000 mgr.y (mgr.24661) 1840 : cluster [DBG] pgmap v3287: 489 pgs: 489 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:38:29.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:28 smithi181 bash[67587]: cluster 2024-09-13T00:38:28.701895+0000 mon.a (mon.0) 3530 : cluster [DBG] osdmap e1887: 8 total, 8 up, 8 in 2024-09-13T00:38:29.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:38:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:38:29] "GET /metrics HTTP/1.1" 200 73346 "" "Prometheus/2.43.0" 2024-09-13T00:38:31.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:30 smithi136 bash[75164]: cluster 2024-09-13T00:38:29.689634+0000 mon.a (mon.0) 3531 : cluster [DBG] osdmap e1888: 8 total, 8 up, 8 in 2024-09-13T00:38:31.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:30 smithi136 bash[75164]: cluster 2024-09-13T00:38:30.409878+0000 mgr.y (mgr.24661) 1841 : cluster [DBG] pgmap v3290: 425 pgs: 425 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:38:31.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:30 smithi136 bash[76371]: cluster 2024-09-13T00:38:29.689634+0000 mon.a (mon.0) 3531 : cluster [DBG] osdmap e1888: 8 total, 8 up, 8 in 2024-09-13T00:38:31.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:30 smithi136 bash[76371]: cluster 2024-09-13T00:38:30.409878+0000 mgr.y (mgr.24661) 1841 : cluster [DBG] pgmap v3290: 425 pgs: 425 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:38:31.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:30 smithi181 bash[67587]: cluster 2024-09-13T00:38:29.689634+0000 mon.a (mon.0) 3531 : cluster [DBG] osdmap e1888: 8 total, 8 up, 8 in 2024-09-13T00:38:31.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:30 smithi181 bash[67587]: cluster 2024-09-13T00:38:30.409878+0000 mgr.y (mgr.24661) 1841 : cluster [DBG] pgmap v3290: 425 pgs: 425 active+clean; 6.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:38:32.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:31 smithi136 bash[76371]: cluster 2024-09-13T00:38:30.696299+0000 mon.a (mon.0) 3532 : cluster [DBG] osdmap e1889: 8 total, 8 up, 8 in 2024-09-13T00:38:32.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:31 smithi136 bash[75164]: cluster 2024-09-13T00:38:30.696299+0000 mon.a (mon.0) 3532 : cluster [DBG] osdmap e1889: 8 total, 8 up, 8 in 2024-09-13T00:38:32.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:31 smithi181 bash[67587]: cluster 2024-09-13T00:38:30.696299+0000 mon.a (mon.0) 3532 : cluster [DBG] osdmap e1889: 8 total, 8 up, 8 in 2024-09-13T00:38:32.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:38:32 smithi181 bash[95505]: ts=2024-09-13T00:38:32.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:38:33.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:32 smithi136 bash[75164]: cluster 2024-09-13T00:38:31.695992+0000 mon.a (mon.0) 3533 : cluster [DBG] osdmap e1890: 8 total, 8 up, 8 in 2024-09-13T00:38:33.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:32 smithi136 bash[75164]: cluster 2024-09-13T00:38:32.411175+0000 mgr.y (mgr.24661) 1842 : cluster [DBG] pgmap v3293: 361 pgs: 361 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:38:33.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:32 smithi136 bash[75164]: cluster 2024-09-13T00:38:32.700407+0000 mon.a (mon.0) 3534 : cluster [DBG] osdmap e1891: 8 total, 8 up, 8 in 2024-09-13T00:38:33.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:32 smithi136 bash[76371]: cluster 2024-09-13T00:38:31.695992+0000 mon.a (mon.0) 3533 : cluster [DBG] osdmap e1890: 8 total, 8 up, 8 in 2024-09-13T00:38:33.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:32 smithi136 bash[76371]: cluster 2024-09-13T00:38:32.411175+0000 mgr.y (mgr.24661) 1842 : cluster [DBG] pgmap v3293: 361 pgs: 361 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:38:33.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:32 smithi136 bash[76371]: cluster 2024-09-13T00:38:32.700407+0000 mon.a (mon.0) 3534 : cluster [DBG] osdmap e1891: 8 total, 8 up, 8 in 2024-09-13T00:38:33.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:32 smithi181 bash[67587]: cluster 2024-09-13T00:38:31.695992+0000 mon.a (mon.0) 3533 : cluster [DBG] osdmap e1890: 8 total, 8 up, 8 in 2024-09-13T00:38:33.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:32 smithi181 bash[67587]: cluster 2024-09-13T00:38:32.411175+0000 mgr.y (mgr.24661) 1842 : cluster [DBG] pgmap v3293: 361 pgs: 361 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:38:33.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:32 smithi181 bash[67587]: cluster 2024-09-13T00:38:32.700407+0000 mon.a (mon.0) 3534 : cluster [DBG] osdmap e1891: 8 total, 8 up, 8 in 2024-09-13T00:38:35.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:34 smithi136 bash[75164]: cluster 2024-09-13T00:38:33.702409+0000 mon.a (mon.0) 3535 : cluster [DBG] osdmap e1892: 8 total, 8 up, 8 in 2024-09-13T00:38:35.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:34 smithi136 bash[75164]: cluster 2024-09-13T00:38:34.411942+0000 mgr.y (mgr.24661) 1843 : cluster [DBG] pgmap v3296: 297 pgs: 297 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:38:35.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:34 smithi136 bash[76371]: cluster 2024-09-13T00:38:33.702409+0000 mon.a (mon.0) 3535 : cluster [DBG] osdmap e1892: 8 total, 8 up, 8 in 2024-09-13T00:38:35.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:34 smithi136 bash[76371]: cluster 2024-09-13T00:38:34.411942+0000 mgr.y (mgr.24661) 1843 : cluster [DBG] pgmap v3296: 297 pgs: 297 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:38:35.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:34 smithi181 bash[67587]: cluster 2024-09-13T00:38:33.702409+0000 mon.a (mon.0) 3535 : cluster [DBG] osdmap e1892: 8 total, 8 up, 8 in 2024-09-13T00:38:35.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:34 smithi181 bash[67587]: cluster 2024-09-13T00:38:34.411942+0000 mgr.y (mgr.24661) 1843 : cluster [DBG] pgmap v3296: 297 pgs: 297 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:38:36.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:35 smithi136 bash[75164]: cluster 2024-09-13T00:38:34.712997+0000 mon.a (mon.0) 3536 : cluster [DBG] osdmap e1893: 8 total, 8 up, 8 in 2024-09-13T00:38:36.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:35 smithi136 bash[76371]: cluster 2024-09-13T00:38:34.712997+0000 mon.a (mon.0) 3536 : cluster [DBG] osdmap e1893: 8 total, 8 up, 8 in 2024-09-13T00:38:36.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:35 smithi181 bash[67587]: cluster 2024-09-13T00:38:34.712997+0000 mon.a (mon.0) 3536 : cluster [DBG] osdmap e1893: 8 total, 8 up, 8 in 2024-09-13T00:38:37.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:36 smithi136 bash[75164]: cluster 2024-09-13T00:38:35.723940+0000 mon.a (mon.0) 3537 : cluster [DBG] osdmap e1894: 8 total, 8 up, 8 in 2024-09-13T00:38:37.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:36 smithi136 bash[75164]: cluster 2024-09-13T00:38:36.413129+0000 mgr.y (mgr.24661) 1844 : cluster [DBG] pgmap v3299: 233 pgs: 233 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:38:37.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:36 smithi136 bash[76371]: cluster 2024-09-13T00:38:35.723940+0000 mon.a (mon.0) 3537 : cluster [DBG] osdmap e1894: 8 total, 8 up, 8 in 2024-09-13T00:38:37.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:36 smithi136 bash[76371]: cluster 2024-09-13T00:38:36.413129+0000 mgr.y (mgr.24661) 1844 : cluster [DBG] pgmap v3299: 233 pgs: 233 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:38:37.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:36 smithi181 bash[67587]: cluster 2024-09-13T00:38:35.723940+0000 mon.a (mon.0) 3537 : cluster [DBG] osdmap e1894: 8 total, 8 up, 8 in 2024-09-13T00:38:37.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:36 smithi181 bash[67587]: cluster 2024-09-13T00:38:36.413129+0000 mgr.y (mgr.24661) 1844 : cluster [DBG] pgmap v3299: 233 pgs: 233 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:38:38.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:37 smithi136 bash[75164]: cluster 2024-09-13T00:38:36.734601+0000 mon.a (mon.0) 3538 : cluster [DBG] osdmap e1895: 8 total, 8 up, 8 in 2024-09-13T00:38:38.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:37 smithi136 bash[76371]: cluster 2024-09-13T00:38:36.734601+0000 mon.a (mon.0) 3538 : cluster [DBG] osdmap e1895: 8 total, 8 up, 8 in 2024-09-13T00:38:38.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:37 smithi181 bash[67587]: cluster 2024-09-13T00:38:36.734601+0000 mon.a (mon.0) 3538 : cluster [DBG] osdmap e1895: 8 total, 8 up, 8 in 2024-09-13T00:38:39.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:38 smithi181 bash[67587]: cluster 2024-09-13T00:38:37.737261+0000 mon.a (mon.0) 3539 : cluster [DBG] osdmap e1896: 8 total, 8 up, 8 in 2024-09-13T00:38:39.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:38 smithi181 bash[67587]: cluster 2024-09-13T00:38:38.413889+0000 mgr.y (mgr.24661) 1845 : cluster [DBG] pgmap v3302: 169 pgs: 169 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:38:39.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:38 smithi136 bash[75164]: cluster 2024-09-13T00:38:37.737261+0000 mon.a (mon.0) 3539 : cluster [DBG] osdmap e1896: 8 total, 8 up, 8 in 2024-09-13T00:38:39.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:38 smithi136 bash[75164]: cluster 2024-09-13T00:38:38.413889+0000 mgr.y (mgr.24661) 1845 : cluster [DBG] pgmap v3302: 169 pgs: 169 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:38:39.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:38 smithi136 bash[76371]: cluster 2024-09-13T00:38:37.737261+0000 mon.a (mon.0) 3539 : cluster [DBG] osdmap e1896: 8 total, 8 up, 8 in 2024-09-13T00:38:39.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:38 smithi136 bash[76371]: cluster 2024-09-13T00:38:38.413889+0000 mgr.y (mgr.24661) 1845 : cluster [DBG] pgmap v3302: 169 pgs: 169 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:38:39.747 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 160 tests from TestLibRBD (386891 ms total) 2024-09-13T00:38:39.748 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:38:39.748 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 11 tests from DiffIterateTest/0, where TypeParam = DiffIterateParams 2024-09-13T00:38:39.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:38:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:38:39] "GET /metrics HTTP/1.1" 200 36738 "" "Prometheus/2.43.0" 2024-09-13T00:38:39.802 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] DiffIterateTest/0.DiffIterate 2024-09-13T00:38:40.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:39 smithi181 bash[67587]: cluster 2024-09-13T00:38:38.739801+0000 mon.a (mon.0) 3540 : cluster [DBG] osdmap e1897: 8 total, 8 up, 8 in 2024-09-13T00:38:40.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:39 smithi181 bash[67587]: audit 2024-09-13T00:38:38.835358+0000 mon.a (mon.0) 3541 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:38:40.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:39 smithi181 bash[67587]: cluster 2024-09-13T00:38:39.741405+0000 mon.a (mon.0) 3542 : cluster [DBG] osdmap e1898: 8 total, 8 up, 8 in 2024-09-13T00:38:40.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:39 smithi136 bash[75164]: cluster 2024-09-13T00:38:38.739801+0000 mon.a (mon.0) 3540 : cluster [DBG] osdmap e1897: 8 total, 8 up, 8 in 2024-09-13T00:38:40.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:39 smithi136 bash[75164]: audit 2024-09-13T00:38:38.835358+0000 mon.a (mon.0) 3541 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:38:40.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:39 smithi136 bash[75164]: cluster 2024-09-13T00:38:39.741405+0000 mon.a (mon.0) 3542 : cluster [DBG] osdmap e1898: 8 total, 8 up, 8 in 2024-09-13T00:38:40.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:39 smithi136 bash[76371]: cluster 2024-09-13T00:38:38.739801+0000 mon.a (mon.0) 3540 : cluster [DBG] osdmap e1897: 8 total, 8 up, 8 in 2024-09-13T00:38:40.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:39 smithi136 bash[76371]: audit 2024-09-13T00:38:38.835358+0000 mon.a (mon.0) 3541 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:38:40.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:39 smithi136 bash[76371]: cluster 2024-09-13T00:38:39.741405+0000 mon.a (mon.0) 3542 : cluster [DBG] osdmap e1898: 8 total, 8 up, 8 in 2024-09-13T00:38:41.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:40 smithi181 bash[67587]: cluster 2024-09-13T00:38:40.414555+0000 mgr.y (mgr.24661) 1846 : cluster [DBG] pgmap v3305: 105 pgs: 105 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:38:41.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:40 smithi136 bash[75164]: cluster 2024-09-13T00:38:40.414555+0000 mgr.y (mgr.24661) 1846 : cluster [DBG] pgmap v3305: 105 pgs: 105 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:38:41.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:40 smithi136 bash[76371]: cluster 2024-09-13T00:38:40.414555+0000 mgr.y (mgr.24661) 1846 : cluster [DBG] pgmap v3305: 105 pgs: 105 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:38:41.803 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:38:42.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:41 smithi181 bash[67587]: cluster 2024-09-13T00:38:40.782464+0000 mon.a (mon.0) 3543 : cluster [DBG] osdmap e1899: 8 total, 8 up, 8 in 2024-09-13T00:38:42.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:41 smithi181 bash[67587]: audit 2024-09-13T00:38:40.789574+0000 mon.a (mon.0) 3544 : audit [INF] from='client.? 172.21.15.136:0/2140671540' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-35","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:38:42.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:41 smithi181 bash[67587]: audit 2024-09-13T00:38:40.957783+0000 mon.a (mon.0) 3545 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:38:42.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:41 smithi181 bash[67587]: audit 2024-09-13T00:38:41.489928+0000 mon.a (mon.0) 3546 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:38:42.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:41 smithi181 bash[67587]: audit 2024-09-13T00:38:41.491697+0000 mon.a (mon.0) 3547 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:38:42.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:41 smithi181 bash[67587]: audit 2024-09-13T00:38:41.499210+0000 mon.a (mon.0) 3548 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:38:42.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:41 smithi136 bash[75164]: cluster 2024-09-13T00:38:40.782464+0000 mon.a (mon.0) 3543 : cluster [DBG] osdmap e1899: 8 total, 8 up, 8 in 2024-09-13T00:38:42.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:41 smithi136 bash[75164]: audit 2024-09-13T00:38:40.789574+0000 mon.a (mon.0) 3544 : audit [INF] from='client.? 172.21.15.136:0/2140671540' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-35","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:38:42.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:41 smithi136 bash[75164]: audit 2024-09-13T00:38:40.957783+0000 mon.a (mon.0) 3545 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:38:42.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:41 smithi136 bash[75164]: audit 2024-09-13T00:38:41.489928+0000 mon.a (mon.0) 3546 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:38:42.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:41 smithi136 bash[75164]: audit 2024-09-13T00:38:41.491697+0000 mon.a (mon.0) 3547 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:38:42.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:41 smithi136 bash[75164]: audit 2024-09-13T00:38:41.499210+0000 mon.a (mon.0) 3548 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:38:42.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:41 smithi136 bash[76371]: cluster 2024-09-13T00:38:40.782464+0000 mon.a (mon.0) 3543 : cluster [DBG] osdmap e1899: 8 total, 8 up, 8 in 2024-09-13T00:38:42.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:41 smithi136 bash[76371]: audit 2024-09-13T00:38:40.789574+0000 mon.a (mon.0) 3544 : audit [INF] from='client.? 172.21.15.136:0/2140671540' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-35","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:38:42.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:41 smithi136 bash[76371]: audit 2024-09-13T00:38:40.957783+0000 mon.a (mon.0) 3545 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:38:42.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:41 smithi136 bash[76371]: audit 2024-09-13T00:38:41.489928+0000 mon.a (mon.0) 3546 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:38:42.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:41 smithi136 bash[76371]: audit 2024-09-13T00:38:41.491697+0000 mon.a (mon.0) 3547 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:38:42.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:41 smithi136 bash[76371]: audit 2024-09-13T00:38:41.499210+0000 mon.a (mon.0) 3548 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:38:42.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:38:42 smithi181 bash[95505]: ts=2024-09-13T00:38:42.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:38:43.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:42 smithi181 bash[67587]: audit 2024-09-13T00:38:41.791748+0000 mon.a (mon.0) 3549 : audit [INF] from='client.? 172.21.15.136:0/2140671540' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-35","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:38:43.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:42 smithi181 bash[67587]: cluster 2024-09-13T00:38:41.791983+0000 mon.a (mon.0) 3550 : cluster [DBG] osdmap e1900: 8 total, 8 up, 8 in 2024-09-13T00:38:43.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:42 smithi181 bash[67587]: cluster 2024-09-13T00:38:42.415363+0000 mgr.y (mgr.24661) 1847 : cluster [DBG] pgmap v3308: 137 pgs: 2 creating+peering, 26 unknown, 109 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:38:43.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:42 smithi136 bash[75164]: audit 2024-09-13T00:38:41.791748+0000 mon.a (mon.0) 3549 : audit [INF] from='client.? 172.21.15.136:0/2140671540' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-35","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:38:43.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:42 smithi136 bash[75164]: cluster 2024-09-13T00:38:41.791983+0000 mon.a (mon.0) 3550 : cluster [DBG] osdmap e1900: 8 total, 8 up, 8 in 2024-09-13T00:38:43.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:42 smithi136 bash[75164]: cluster 2024-09-13T00:38:42.415363+0000 mgr.y (mgr.24661) 1847 : cluster [DBG] pgmap v3308: 137 pgs: 2 creating+peering, 26 unknown, 109 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:38:43.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:42 smithi136 bash[76371]: audit 2024-09-13T00:38:41.791748+0000 mon.a (mon.0) 3549 : audit [INF] from='client.? 172.21.15.136:0/2140671540' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-35","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:38:43.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:42 smithi136 bash[76371]: cluster 2024-09-13T00:38:41.791983+0000 mon.a (mon.0) 3550 : cluster [DBG] osdmap e1900: 8 total, 8 up, 8 in 2024-09-13T00:38:43.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:42 smithi136 bash[76371]: cluster 2024-09-13T00:38:42.415363+0000 mgr.y (mgr.24661) 1847 : cluster [DBG] pgmap v3308: 137 pgs: 2 creating+peering, 26 unknown, 109 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:38:43.890 INFO:tasks.workunit.client.0.smithi136.stdout: wrote [165303~41788,1325885~84340,4703722~81746,5440711~59009,8500420~31149,14979933~52091,15762171~4213,16216691~31237,17925031~2424,19843901~46061] 2024-09-13T00:38:44.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:43 smithi181 bash[67587]: cluster 2024-09-13T00:38:42.814396+0000 mon.a (mon.0) 3551 : cluster [DBG] osdmap e1901: 8 total, 8 up, 8 in 2024-09-13T00:38:44.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:43 smithi136 bash[75164]: cluster 2024-09-13T00:38:42.814396+0000 mon.a (mon.0) 3551 : cluster [DBG] osdmap e1901: 8 total, 8 up, 8 in 2024-09-13T00:38:44.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:43 smithi136 bash[76371]: cluster 2024-09-13T00:38:42.814396+0000 mon.a (mon.0) 3551 : cluster [DBG] osdmap e1901: 8 total, 8 up, 8 in 2024-09-13T00:38:44.859 INFO:tasks.workunit.client.0.smithi136.stdout: wrote [2865464~54603,3426436~93061,7052206~92852,7386134~60773,8570241~82114,10393095~43370,11105384~94172,15284163~31247,16258789~30554,16465223~97115] 2024-09-13T00:38:44.878 INFO:tasks.workunit.client.0.smithi136.stdout: diff was [1410225~2109272,5499720~1947187,8531569~2667987,15284163~31247,16247928~314410] 2024-09-13T00:38:44.886 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] DiffIterateTest/0.DiffIterate (5083 ms) 2024-09-13T00:38:44.886 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] DiffIterateTest/0.DiffIterateDiscard 2024-09-13T00:38:44.886 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:38:44.938 INFO:tasks.workunit.client.0.smithi136.stdout:iterate_cb 0~256 2024-09-13T00:38:45.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:44 smithi181 bash[67587]: cluster 2024-09-13T00:38:43.821618+0000 mon.a (mon.0) 3552 : cluster [DBG] osdmap e1902: 8 total, 8 up, 8 in 2024-09-13T00:38:45.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:44 smithi181 bash[67587]: cluster 2024-09-13T00:38:44.416021+0000 mgr.y (mgr.24661) 1848 : cluster [DBG] pgmap v3311: 137 pgs: 2 creating+peering, 17 unknown, 118 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:38:45.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:44 smithi136 bash[75164]: cluster 2024-09-13T00:38:43.821618+0000 mon.a (mon.0) 3552 : cluster [DBG] osdmap e1902: 8 total, 8 up, 8 in 2024-09-13T00:38:45.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:44 smithi136 bash[75164]: cluster 2024-09-13T00:38:44.416021+0000 mgr.y (mgr.24661) 1848 : cluster [DBG] pgmap v3311: 137 pgs: 2 creating+peering, 17 unknown, 118 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:38:45.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:44 smithi136 bash[76371]: cluster 2024-09-13T00:38:43.821618+0000 mon.a (mon.0) 3552 : cluster [DBG] osdmap e1902: 8 total, 8 up, 8 in 2024-09-13T00:38:45.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:44 smithi136 bash[76371]: cluster 2024-09-13T00:38:44.416021+0000 mgr.y (mgr.24661) 1848 : cluster [DBG] pgmap v3311: 137 pgs: 2 creating+peering, 17 unknown, 118 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:38:45.879 INFO:tasks.workunit.client.0.smithi136.stdout:iterate_cb 0~256 2024-09-13T00:38:46.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:45 smithi181 bash[67587]: cluster 2024-09-13T00:38:44.831431+0000 mon.a (mon.0) 3553 : cluster [DBG] osdmap e1903: 8 total, 8 up, 8 in 2024-09-13T00:38:46.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:45 smithi136 bash[75164]: cluster 2024-09-13T00:38:44.831431+0000 mon.a (mon.0) 3553 : cluster [DBG] osdmap e1903: 8 total, 8 up, 8 in 2024-09-13T00:38:46.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:45 smithi136 bash[76371]: cluster 2024-09-13T00:38:44.831431+0000 mon.a (mon.0) 3553 : cluster [DBG] osdmap e1903: 8 total, 8 up, 8 in 2024-09-13T00:38:46.887 INFO:tasks.workunit.client.0.smithi136.stdout:iterate_cb 0~256 2024-09-13T00:38:47.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:46 smithi181 bash[67587]: cluster 2024-09-13T00:38:45.842479+0000 mon.a (mon.0) 3554 : cluster [DBG] osdmap e1904: 8 total, 8 up, 8 in 2024-09-13T00:38:47.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:46 smithi181 bash[67587]: cluster 2024-09-13T00:38:46.416875+0000 mgr.y (mgr.24661) 1849 : cluster [DBG] pgmap v3314: 137 pgs: 137 active+clean; 17 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 14 KiB/s rd, 179 KiB/s wr, 28 op/s 2024-09-13T00:38:47.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:46 smithi136 bash[75164]: cluster 2024-09-13T00:38:45.842479+0000 mon.a (mon.0) 3554 : cluster [DBG] osdmap e1904: 8 total, 8 up, 8 in 2024-09-13T00:38:47.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:46 smithi136 bash[75164]: cluster 2024-09-13T00:38:46.416875+0000 mgr.y (mgr.24661) 1849 : cluster [DBG] pgmap v3314: 137 pgs: 137 active+clean; 17 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 14 KiB/s rd, 179 KiB/s wr, 28 op/s 2024-09-13T00:38:47.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:46 smithi136 bash[76371]: cluster 2024-09-13T00:38:45.842479+0000 mon.a (mon.0) 3554 : cluster [DBG] osdmap e1904: 8 total, 8 up, 8 in 2024-09-13T00:38:47.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:46 smithi136 bash[76371]: cluster 2024-09-13T00:38:46.416875+0000 mgr.y (mgr.24661) 1849 : cluster [DBG] pgmap v3314: 137 pgs: 137 active+clean; 17 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 14 KiB/s rd, 179 KiB/s wr, 28 op/s 2024-09-13T00:38:47.901 INFO:tasks.workunit.client.0.smithi136.stdout:iterate_cb 0~256 2024-09-13T00:38:47.905 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] DiffIterateTest/0.DiffIterateDiscard (3019 ms) 2024-09-13T00:38:47.905 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] DiffIterateTest/0.DiffIterateStress 2024-09-13T00:38:47.905 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:38:48.036 INFO:tasks.workunit.client.0.smithi136.stdout: i=0 exists [97502456~4319494,111158409~7385747,137653545~6008102,204502697~1161534,291025513~1878807,301118059~2497110,329254671~7320057,356349735~2009351,367724830~4161459,405595440~7376810] wrote [97502456~4319494,111158409~7385747,137653545~6008102,204502697~1161534,291025513~1878807,301118059~2497110,329254671~7320057,356349735~2009351,367724830~4161459,405595440~7376810] 2024-09-13T00:38:48.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:47 smithi136 bash[75164]: cluster 2024-09-13T00:38:46.850954+0000 mon.a (mon.0) 3555 : cluster [DBG] osdmap e1905: 8 total, 8 up, 8 in 2024-09-13T00:38:48.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:47 smithi136 bash[76371]: cluster 2024-09-13T00:38:46.850954+0000 mon.a (mon.0) 3555 : cluster [DBG] osdmap e1905: 8 total, 8 up, 8 in 2024-09-13T00:38:48.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:47 smithi181 bash[67587]: cluster 2024-09-13T00:38:46.850954+0000 mon.a (mon.0) 3555 : cluster [DBG] osdmap e1905: 8 total, 8 up, 8 in 2024-09-13T00:38:48.972 INFO:tasks.workunit.client.0.smithi136.stdout: i=1 exists [6333791~894944,43286419~3567780,50397556~1503215,89250005~3131785,97502456~4319494,111158409~7385747,137653545~6008102,163394852~4718557,171656619~199342,204502697~1161534,240318840~976558,269960512~2117506,291025513~1878807,301118059~8395949,329254671~7320057,341594269~8118654,356349735~2009351,367724830~4161459,405595440~7376810] wrote [6333791~894944,43286419~3567780,50397556~1503215,89250005~3131785,163394852~4718557,171656619~199342,240318840~976558,269960512~2117506,302940867~6573141,341594269~8118654] 2024-09-13T00:38:49.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:48 smithi136 bash[75164]: cluster 2024-09-13T00:38:47.853222+0000 mon.a (mon.0) 3556 : cluster [DBG] osdmap e1906: 8 total, 8 up, 8 in 2024-09-13T00:38:49.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:48 smithi136 bash[75164]: cluster 2024-09-13T00:38:48.417535+0000 mgr.y (mgr.24661) 1850 : cluster [DBG] pgmap v3317: 137 pgs: 137 active+clean; 17 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 41 KiB/s rd, 185 KiB/s wr, 68 op/s 2024-09-13T00:38:49.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:48 smithi136 bash[76371]: cluster 2024-09-13T00:38:47.853222+0000 mon.a (mon.0) 3556 : cluster [DBG] osdmap e1906: 8 total, 8 up, 8 in 2024-09-13T00:38:49.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:48 smithi136 bash[76371]: cluster 2024-09-13T00:38:48.417535+0000 mgr.y (mgr.24661) 1850 : cluster [DBG] pgmap v3317: 137 pgs: 137 active+clean; 17 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 41 KiB/s rd, 185 KiB/s wr, 68 op/s 2024-09-13T00:38:49.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:48 smithi181 bash[67587]: cluster 2024-09-13T00:38:47.853222+0000 mon.a (mon.0) 3556 : cluster [DBG] osdmap e1906: 8 total, 8 up, 8 in 2024-09-13T00:38:49.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:48 smithi181 bash[67587]: cluster 2024-09-13T00:38:48.417535+0000 mgr.y (mgr.24661) 1850 : cluster [DBG] pgmap v3317: 137 pgs: 137 active+clean; 17 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 41 KiB/s rd, 185 KiB/s wr, 68 op/s 2024-09-13T00:38:49.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:38:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:38:49] "GET /metrics HTTP/1.1" 200 36738 "" "Prometheus/2.43.0" 2024-09-13T00:38:49.994 INFO:tasks.workunit.client.0.smithi136.stdout: i=2 exists [5507084~6002062,18624312~6440348,43286419~3567780,50397556~1503215,83314700~3016720,89250005~3131785,97502456~4319494,111158409~7385747,126152787~20381765,163394852~4718557,171656619~199342,204447798~1679405,240318840~976558,269960512~2117506,291025513~1878807,301118059~8395949,329254671~7320057,341594269~8118654,356349735~2009351,367724830~4161459,381456303~3508409,405595440~7376810] wrote [5507084~6002062,18624312~6440348,83314700~3016720,97623360~4189624,113921932~3493308,126152787~14073799,143604550~2930002,204447798~1679405,381456303~3508409] 2024-09-13T00:38:50.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:49 smithi136 bash[75164]: cluster 2024-09-13T00:38:48.875588+0000 mon.a (mon.0) 3557 : cluster [DBG] osdmap e1907: 8 total, 8 up, 8 in 2024-09-13T00:38:50.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:49 smithi136 bash[76371]: cluster 2024-09-13T00:38:48.875588+0000 mon.a (mon.0) 3557 : cluster [DBG] osdmap e1907: 8 total, 8 up, 8 in 2024-09-13T00:38:50.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:49 smithi181 bash[67587]: cluster 2024-09-13T00:38:48.875588+0000 mon.a (mon.0) 3557 : cluster [DBG] osdmap e1907: 8 total, 8 up, 8 in 2024-09-13T00:38:51.016 INFO:tasks.workunit.client.0.smithi136.stdout: i=3 exists [5507084~6002062,18624312~6440348,38721951~8132248,50397556~1503215,71349028~11331915,83314700~3016720,86936101~78870,89250005~12571945,111158409~7385747,126152787~20381765,163394852~4718557,171656619~199342,204447798~1679405,222904244~4325743,240318840~976558,269960512~2117506,291025513~1878807,301118059~8395949,317751762~7525150,329254671~7320057,341594269~8118654,356349735~2009351,367724830~4161459,373179324~6803307,381456303~3508409,401627021~3314153,405595440~7376810] wrote [38721951~5200138,71349028~11331915,86936101~78870,91915173~8177010,129165667~1581292,222904244~4325743,317751762~7525150,373179324~6803307,401627021~3314153] 2024-09-13T00:38:51.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:50 smithi136 bash[75164]: cluster 2024-09-13T00:38:49.887124+0000 mon.a (mon.0) 3558 : cluster [DBG] osdmap e1908: 8 total, 8 up, 8 in 2024-09-13T00:38:51.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:50 smithi136 bash[75164]: cluster 2024-09-13T00:38:50.418222+0000 mgr.y (mgr.24661) 1851 : cluster [DBG] pgmap v3320: 137 pgs: 137 active+clean; 54 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 40 KiB/s rd, 5.3 MiB/s wr, 65 op/s 2024-09-13T00:38:51.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:50 smithi136 bash[76371]: cluster 2024-09-13T00:38:49.887124+0000 mon.a (mon.0) 3558 : cluster [DBG] osdmap e1908: 8 total, 8 up, 8 in 2024-09-13T00:38:51.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:50 smithi136 bash[76371]: cluster 2024-09-13T00:38:50.418222+0000 mgr.y (mgr.24661) 1851 : cluster [DBG] pgmap v3320: 137 pgs: 137 active+clean; 54 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 40 KiB/s rd, 5.3 MiB/s wr, 65 op/s 2024-09-13T00:38:51.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:50 smithi181 bash[67587]: cluster 2024-09-13T00:38:49.887124+0000 mon.a (mon.0) 3558 : cluster [DBG] osdmap e1908: 8 total, 8 up, 8 in 2024-09-13T00:38:51.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:50 smithi181 bash[67587]: cluster 2024-09-13T00:38:50.418222+0000 mgr.y (mgr.24661) 1851 : cluster [DBG] pgmap v3320: 137 pgs: 137 active+clean; 54 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 40 KiB/s rd, 5.3 MiB/s wr, 65 op/s 2024-09-13T00:38:51.996 INFO:tasks.workunit.client.0.smithi136.stdout: i=4 exists [5507084~6002062,18624312~6440348,38721951~14483269,58515289~704557,61366356~6043690,71349028~11331915,83314700~3016720,86936101~78870,89250005~12571945,111158409~7385747,126152787~20381765,153773626~3477336,162728173~5385236,171656619~199342,204447798~1679405,222904244~4325743,240318840~976558,269960512~2117506,289226804~7918678,301118059~8395949,317751762~7525150,329254671~7320057,341594269~8118654,356349735~2009351,367724830~4161459,373179324~6803307,381456303~3508409,401627021~3314153,405595440~7376810] wrote [46526671~6678549,58515289~704557,61366356~6043690,130605682~5648521,153773626~3477336,162728173~1277847,289226804~7918678,405696667~5790944] 2024-09-13T00:38:52.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:51 smithi136 bash[75164]: cluster 2024-09-13T00:38:50.899078+0000 mon.a (mon.0) 3559 : cluster [DBG] osdmap e1909: 8 total, 8 up, 8 in 2024-09-13T00:38:52.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:51 smithi136 bash[76371]: cluster 2024-09-13T00:38:50.899078+0000 mon.a (mon.0) 3559 : cluster [DBG] osdmap e1909: 8 total, 8 up, 8 in 2024-09-13T00:38:52.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:51 smithi181 bash[67587]: cluster 2024-09-13T00:38:50.899078+0000 mon.a (mon.0) 3559 : cluster [DBG] osdmap e1909: 8 total, 8 up, 8 in 2024-09-13T00:38:52.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:38:52 smithi181 bash[95505]: ts=2024-09-13T00:38:52.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:38:53.004 INFO:tasks.workunit.client.0.smithi136.stdout: i=5 exists [5507084~6002062,18624312~6440348,26995819~3290536,38721951~14483269,58515289~704557,61366356~6043690,71349028~11331915,83314700~3016720,86936101~78870,89250005~12571945,111158409~7385747,126152787~20381765,147450241~4531765,153773626~3477336,162728173~5385236,171656619~199342,180807822~766370,204447798~1679405,208047485~1381668,222904244~4325743,240318840~976558,269960512~2117506,279969777~95145,289226804~7918678,301118059~8395949,317751762~7525150,329254671~7320057,339257207~1141979,341594269~8118654,356349735~2009351,367724830~4161459,373179324~11785388,390387020~7188584,401627021~3314153,405595440~7376810] wrote [26995819~3290536,147450241~4531765,180807822~766370,208047485~1381668,279969777~95145,339257207~1141979,373635351~8279985,390387020~7188584,409295430~1606555] 2024-09-13T00:38:53.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:52 smithi136 bash[75164]: cluster 2024-09-13T00:38:51.900445+0000 mon.a (mon.0) 3560 : cluster [DBG] osdmap e1910: 8 total, 8 up, 8 in 2024-09-13T00:38:53.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:52 smithi136 bash[75164]: cluster 2024-09-13T00:38:52.419024+0000 mgr.y (mgr.24661) 1852 : cluster [DBG] pgmap v3323: 137 pgs: 137 active+clean; 167 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 120 KiB/s rd, 28 MiB/s wr, 200 op/s 2024-09-13T00:38:53.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:52 smithi136 bash[76371]: cluster 2024-09-13T00:38:51.900445+0000 mon.a (mon.0) 3560 : cluster [DBG] osdmap e1910: 8 total, 8 up, 8 in 2024-09-13T00:38:53.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:52 smithi136 bash[76371]: cluster 2024-09-13T00:38:52.419024+0000 mgr.y (mgr.24661) 1852 : cluster [DBG] pgmap v3323: 137 pgs: 137 active+clean; 167 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 120 KiB/s rd, 28 MiB/s wr, 200 op/s 2024-09-13T00:38:53.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:52 smithi181 bash[67587]: cluster 2024-09-13T00:38:51.900445+0000 mon.a (mon.0) 3560 : cluster [DBG] osdmap e1910: 8 total, 8 up, 8 in 2024-09-13T00:38:53.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:52 smithi181 bash[67587]: cluster 2024-09-13T00:38:52.419024+0000 mgr.y (mgr.24661) 1852 : cluster [DBG] pgmap v3323: 137 pgs: 137 active+clean; 167 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 120 KiB/s rd, 28 MiB/s wr, 200 op/s 2024-09-13T00:38:54.023 INFO:tasks.workunit.client.0.smithi136.stdout: i=6 exists [2612746~8896400,18624312~6440348,26995819~3290536,32429128~3147149,38721951~14483269,58515289~704557,61366356~6043690,71349028~11331915,83314700~3016720,86936101~78870,89250005~12571945,111158409~7385747,126152787~20381765,147450241~4531765,153773626~3477336,162728173~5385236,171656619~199342,180807822~766370,204447798~1679405,208047485~1381668,222904244~4325743,240318840~976558,244107705~1444348,260842262~7844195,269960512~2117506,279969777~95145,289202665~7942817,301118059~8395949,317751762~7525150,329254671~7320057,339257207~1141979,341594269~8118654,356349735~2009351,367724830~4161459,373179324~11785388,390387020~7188584,401627021~3314153,405595440~7376810] wrote [2612746~3549385,27288947~289421,32429128~3147149,64536720~2354229,92243442~6230977,244107705~1444348,260842262~7844195,289202665~1293771,301411264~7201245] 2024-09-13T00:38:54.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:53 smithi136 bash[75164]: cluster 2024-09-13T00:38:52.924141+0000 mon.a (mon.0) 3561 : cluster [DBG] osdmap e1911: 8 total, 8 up, 8 in 2024-09-13T00:38:54.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:53 smithi136 bash[75164]: audit 2024-09-13T00:38:53.835990+0000 mon.a (mon.0) 3562 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:38:54.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:53 smithi136 bash[76371]: cluster 2024-09-13T00:38:52.924141+0000 mon.a (mon.0) 3561 : cluster [DBG] osdmap e1911: 8 total, 8 up, 8 in 2024-09-13T00:38:54.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:53 smithi136 bash[76371]: audit 2024-09-13T00:38:53.835990+0000 mon.a (mon.0) 3562 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:38:54.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:53 smithi181 bash[67587]: cluster 2024-09-13T00:38:52.924141+0000 mon.a (mon.0) 3561 : cluster [DBG] osdmap e1911: 8 total, 8 up, 8 in 2024-09-13T00:38:54.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:53 smithi181 bash[67587]: audit 2024-09-13T00:38:53.835990+0000 mon.a (mon.0) 3562 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:38:54.818 INFO:tasks.workunit.client.0.smithi136.stdout: i=7 exists [2612746~8896400,18624312~6440348,26995819~3290536,31904846~21300374,58515289~704557,61366356~6043690,71349028~11331915,83314700~3016720,86936101~78870,89250005~12571945,111158409~10067142,126152787~20381765,147450241~4531765,153773626~7600370,162728173~5385236,171656619~199342,180807822~766370,204447798~1679405,208047485~1381668,212320259~6420604,222904244~4325743,227825355~4871993,240318840~976558,244107705~1444348,260842262~7844195,269960512~2117506,276897564~6749766,289202665~7942817,301118059~8395949,317751762~7525150,329254671~7320057,339257207~1141979,341594269~8118654,356349735~2009351,367724830~4161459,373179324~11785388,387920372~9655232,401627021~3314153,405595440~7376810] wrote [31904846~7749369,113977873~7247678,155299497~6074499,212320259~6420604,227825355~4871993,276897564~6749766,343284705~1605783,378757665~2361378,387920372~6989975] 2024-09-13T00:38:55.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:54 smithi136 bash[75164]: cluster 2024-09-13T00:38:53.930513+0000 mon.a (mon.0) 3563 : cluster [DBG] osdmap e1912: 8 total, 8 up, 8 in 2024-09-13T00:38:55.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:54 smithi136 bash[75164]: cluster 2024-09-13T00:38:54.419819+0000 mgr.y (mgr.24661) 1853 : cluster [DBG] pgmap v3326: 137 pgs: 137 active+clean; 239 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 109 KiB/s rd, 35 MiB/s wr, 183 op/s 2024-09-13T00:38:55.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:54 smithi136 bash[75164]: cluster 2024-09-13T00:38:54.696119+0000 mon.a (mon.0) 3564 : cluster [DBG] osdmap e1913: 8 total, 8 up, 8 in 2024-09-13T00:38:55.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:54 smithi136 bash[76371]: cluster 2024-09-13T00:38:53.930513+0000 mon.a (mon.0) 3563 : cluster [DBG] osdmap e1912: 8 total, 8 up, 8 in 2024-09-13T00:38:55.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:54 smithi136 bash[76371]: cluster 2024-09-13T00:38:54.419819+0000 mgr.y (mgr.24661) 1853 : cluster [DBG] pgmap v3326: 137 pgs: 137 active+clean; 239 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 109 KiB/s rd, 35 MiB/s wr, 183 op/s 2024-09-13T00:38:55.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:54 smithi136 bash[76371]: cluster 2024-09-13T00:38:54.696119+0000 mon.a (mon.0) 3564 : cluster [DBG] osdmap e1913: 8 total, 8 up, 8 in 2024-09-13T00:38:55.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:54 smithi181 bash[67587]: cluster 2024-09-13T00:38:53.930513+0000 mon.a (mon.0) 3563 : cluster [DBG] osdmap e1912: 8 total, 8 up, 8 in 2024-09-13T00:38:55.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:54 smithi181 bash[67587]: cluster 2024-09-13T00:38:54.419819+0000 mgr.y (mgr.24661) 1853 : cluster [DBG] pgmap v3326: 137 pgs: 137 active+clean; 239 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 109 KiB/s rd, 35 MiB/s wr, 183 op/s 2024-09-13T00:38:55.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:54 smithi181 bash[67587]: cluster 2024-09-13T00:38:54.696119+0000 mon.a (mon.0) 3564 : cluster [DBG] osdmap e1913: 8 total, 8 up, 8 in 2024-09-13T00:38:55.790 INFO:tasks.workunit.client.0.smithi136.stdout: i=8 exists [2612746~8896400,18624312~6440348,26995819~4642300,31904846~21300374,58515289~704557,61366356~6043690,71349028~14982392,86936101~78870,89250005~12571945,111158409~10067142,126152787~20381765,147450241~4531765,153773626~7600370,162728173~5385236,171656619~199342,172123438~2775342,180155982~1465141,204447798~1679405,208047485~1381668,212320259~6420604,222904244~4325743,227825355~4871993,240318840~976558,244107705~8129772,260842262~7844195,269960512~2117506,276897564~6749766,289202665~7942817,301118059~8395949,317751762~7525150,329254671~7320057,339257207~1141979,341594269~8118654,356349735~2009351,367724830~4161459,373179324~11785388,387920372~9655232,401627021~3314153,405595440~7376810] wrote [30081629~1556490,81806467~4199395,111245499~4720159,172123438~2775342,180155982~1465141,244213343~8024134,322459502~1891157,330730451~2400977,373888833~3376207] 2024-09-13T00:38:56.802 INFO:tasks.workunit.client.0.smithi136.stdout: i=9 exists [2612746~8896400,18624312~6440348,26995819~4642300,31904846~21300374,58515289~704557,61366356~6043690,71349028~15665943,89250005~12571945,111158409~10067142,126152787~20381765,147450241~4531765,153773626~14339783,169300267~5598513,180155982~4105980,204447798~1679405,206691828~1354340,208047485~1381668,212320259~6420604,220044632~7185355,227825355~4871993,240318840~976558,244107705~8129772,260842262~7844195,269960512~2117506,273904834~1135501,276897564~6749766,284165576~12979906,301118059~8395949,317751762~7525150,329254671~7320057,339257207~1141979,341594269~8118654,356349735~2009351,367724830~4161459,373179324~11785388,387920372~9655232,401627021~3314153,405595440~7376810] wrote [84137493~2803411,159230705~5201250,169300267~4108516,180376314~3885648,206691828~1354340,220044632~5434875,246659731~4921744,266675149~244059,273904834~1135501,284165576~6885360] 2024-09-13T00:38:57.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:56 smithi136 bash[75164]: cluster 2024-09-13T00:38:55.695337+0000 mon.a (mon.0) 3565 : cluster [DBG] osdmap e1914: 8 total, 8 up, 8 in 2024-09-13T00:38:57.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:56 smithi136 bash[75164]: cluster 2024-09-13T00:38:56.420868+0000 mgr.y (mgr.24661) 1854 : cluster [DBG] pgmap v3329: 137 pgs: 137 active+clean; 372 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 176 KiB/s rd, 41 MiB/s wr, 286 op/s 2024-09-13T00:38:57.012 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:56 smithi136 bash[76371]: cluster 2024-09-13T00:38:55.695337+0000 mon.a (mon.0) 3565 : cluster [DBG] osdmap e1914: 8 total, 8 up, 8 in 2024-09-13T00:38:57.012 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:56 smithi136 bash[76371]: cluster 2024-09-13T00:38:56.420868+0000 mgr.y (mgr.24661) 1854 : cluster [DBG] pgmap v3329: 137 pgs: 137 active+clean; 372 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 176 KiB/s rd, 41 MiB/s wr, 286 op/s 2024-09-13T00:38:57.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:56 smithi181 bash[67587]: cluster 2024-09-13T00:38:55.695337+0000 mon.a (mon.0) 3565 : cluster [DBG] osdmap e1914: 8 total, 8 up, 8 in 2024-09-13T00:38:57.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:56 smithi181 bash[67587]: cluster 2024-09-13T00:38:56.420868+0000 mgr.y (mgr.24661) 1854 : cluster [DBG] pgmap v3329: 137 pgs: 137 active+clean; 372 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 176 KiB/s rd, 41 MiB/s wr, 286 op/s 2024-09-13T00:38:57.807 INFO:tasks.workunit.client.0.smithi136.stdout: i=10 exists [2612746~8896400,18624312~6440348,26995819~4642300,31904846~21300374,58515289~704557,61366356~25648615,89250005~12571945,111158409~10067142,126152787~20381765,147450241~4531765,153773626~14339783,169300267~5598513,180155982~4105980,204447798~1679405,206691828~1354340,208047485~1381668,212320259~6420604,220044632~12652716,240318840~976558,244107705~8129772,260842262~7844195,269960512~2117506,273904834~1135501,276897564~6749766,284165576~12979906,301118059~8395949,317751762~7525150,329254671~7320057,339257207~1141979,341594269~13892162,356349735~2009351,365997237~1542191,367724830~4161459,373179324~11785388,387920372~9655232,401627021~3314153,405595440~7376810] wrote [22666579~2303167,65727373~6163988,78799742~5691133,128451845~308073,222846411~6287708,348549356~6937075,365997237~1542191,392519231~4064300] 2024-09-13T00:38:58.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:57 smithi136 bash[75164]: cluster 2024-09-13T00:38:56.709843+0000 mon.a (mon.0) 3566 : cluster [DBG] osdmap e1915: 8 total, 8 up, 8 in 2024-09-13T00:38:58.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:57 smithi136 bash[76371]: cluster 2024-09-13T00:38:56.709843+0000 mon.a (mon.0) 3566 : cluster [DBG] osdmap e1915: 8 total, 8 up, 8 in 2024-09-13T00:38:58.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:57 smithi181 bash[67587]: cluster 2024-09-13T00:38:56.709843+0000 mon.a (mon.0) 3566 : cluster [DBG] osdmap e1915: 8 total, 8 up, 8 in 2024-09-13T00:38:58.843 INFO:tasks.workunit.client.0.smithi136.stdout: i=11 exists [2612746~8896400,18624312~6440348,26995819~4642300,31904846~21300374,58515289~704557,61366356~25648615,89250005~12571945,105459787~1424206,111158409~10067142,122043306~24491246,147450241~4531765,153773626~14339783,169300267~5598513,180155982~4105980,204447798~1679405,206691828~1354340,208047485~1381668,212320259~6420604,220044632~12652716,240318840~11918637,260842262~7844195,269960512~2117506,273904834~1135501,276897564~21900567,301118059~8395949,317751762~10401988,329254671~8494241,339257207~1141979,341594269~13892162,356349735~2009351,365997237~1542191,367724830~4161459,373179324~11785388,387920372~17020802,405595440~7376810] wrote [5335581~3339840,105459787~1424206,122043306~5052094,130261050~4316483,240828348~6357952,282726102~2562064,292988707~5809424,322862850~5290900,335934186~1814726,396670010~6408714] 2024-09-13T00:38:59.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:58 smithi136 bash[75164]: cluster 2024-09-13T00:38:57.707437+0000 mon.a (mon.0) 3567 : cluster [DBG] osdmap e1916: 8 total, 8 up, 8 in 2024-09-13T00:38:59.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:58 smithi136 bash[75164]: cluster 2024-09-13T00:38:58.421513+0000 mgr.y (mgr.24661) 1855 : cluster [DBG] pgmap v3332: 137 pgs: 137 active+clean; 414 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 176 KiB/s rd, 39 MiB/s wr, 285 op/s 2024-09-13T00:38:59.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:58 smithi136 bash[76371]: cluster 2024-09-13T00:38:57.707437+0000 mon.a (mon.0) 3567 : cluster [DBG] osdmap e1916: 8 total, 8 up, 8 in 2024-09-13T00:38:59.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:58 smithi136 bash[76371]: cluster 2024-09-13T00:38:58.421513+0000 mgr.y (mgr.24661) 1855 : cluster [DBG] pgmap v3332: 137 pgs: 137 active+clean; 414 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 176 KiB/s rd, 39 MiB/s wr, 285 op/s 2024-09-13T00:38:59.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:58 smithi181 bash[67587]: cluster 2024-09-13T00:38:57.707437+0000 mon.a (mon.0) 3567 : cluster [DBG] osdmap e1916: 8 total, 8 up, 8 in 2024-09-13T00:38:59.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:58 smithi181 bash[67587]: cluster 2024-09-13T00:38:58.421513+0000 mgr.y (mgr.24661) 1855 : cluster [DBG] pgmap v3332: 137 pgs: 137 active+clean; 414 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 176 KiB/s rd, 39 MiB/s wr, 285 op/s 2024-09-13T00:38:59.739 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:38:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:38:59] "GET /metrics HTTP/1.1" 200 36750 "" "Prometheus/2.43.0" 2024-09-13T00:38:59.873 INFO:tasks.workunit.client.0.smithi136.stdout: i=12 exists [2612746~10642447,18624312~34580908,58515289~704557,61366356~25648615,89250005~13234343,105459787~1424206,111158409~10067142,122043306~24491246,147450241~4531765,153773626~14339783,169300267~5598513,180155982~4105980,204447798~1679405,206691828~1354340,208047485~1381668,212320259~6420604,220044632~12652716,240318840~11918637,260842262~7844195,269960512~2117506,273904834~1135501,276897564~21900567,301118059~8395949,312566619~2948229,316301726~21447186,339257207~1141979,341594269~13892162,356349735~2009351,365997237~5889052,373179324~13170580,387920372~17020802,405595440~7376810] wrote [6779309~6475884,24769299~7548428,62948307~5586522,95048390~7435958,277779043~4483937,312566619~2948229,316301726~3511210,328085427~7228576,366458829~4720835,379083185~7266719] 2024-09-13T00:39:00.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:38:59 smithi136 bash[76371]: cluster 2024-09-13T00:38:58.738645+0000 mon.a (mon.0) 3568 : cluster [DBG] osdmap e1917: 8 total, 8 up, 8 in 2024-09-13T00:39:00.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:38:59 smithi136 bash[75164]: cluster 2024-09-13T00:38:58.738645+0000 mon.a (mon.0) 3568 : cluster [DBG] osdmap e1917: 8 total, 8 up, 8 in 2024-09-13T00:39:00.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:38:59 smithi181 bash[67587]: cluster 2024-09-13T00:38:58.738645+0000 mon.a (mon.0) 3568 : cluster [DBG] osdmap e1917: 8 total, 8 up, 8 in 2024-09-13T00:39:00.855 INFO:tasks.workunit.client.0.smithi136.stdout: i=13 exists [2612746~10642447,17113294~36091926,58515289~704557,61366356~25648615,89250005~13234343,105459787~1424206,111158409~10067142,122043306~24491246,147450241~4531765,153773626~14339783,169300267~5598513,180155982~4105980,187369573~871966,201556540~2015504,204447798~1679405,206691828~1354340,208047485~1381668,209974378~1903654,212320259~6420604,220044632~12652716,240318840~11918637,260842262~7844195,269960512~2117506,273904834~1135501,276897564~23715912,301118059~8395949,312566619~2948229,316301726~21447186,339257207~1141979,341594269~13892162,356349735~2009351,365997237~5889052,373179324~13170580,387920372~17020802,405595440~7376810] wrote [17113294~3323835,22837062~7110070,63473742~8112656,99390713~475647,159255557~7387112,173851851~84284,187369573~871966,201556540~2015504,209974378~1903654,294860984~5752492] 2024-09-13T00:39:01.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:00 smithi136 bash[75164]: cluster 2024-09-13T00:38:59.735520+0000 mon.a (mon.0) 3569 : cluster [DBG] osdmap e1918: 8 total, 8 up, 8 in 2024-09-13T00:39:01.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:00 smithi136 bash[75164]: cluster 2024-09-13T00:39:00.422196+0000 mgr.y (mgr.24661) 1856 : cluster [DBG] pgmap v3335: 137 pgs: 137 active+clean; 472 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 1.5 KiB/s rd, 23 MiB/s wr, 15 op/s 2024-09-13T00:39:01.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:00 smithi136 bash[76371]: cluster 2024-09-13T00:38:59.735520+0000 mon.a (mon.0) 3569 : cluster [DBG] osdmap e1918: 8 total, 8 up, 8 in 2024-09-13T00:39:01.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:00 smithi136 bash[76371]: cluster 2024-09-13T00:39:00.422196+0000 mgr.y (mgr.24661) 1856 : cluster [DBG] pgmap v3335: 137 pgs: 137 active+clean; 472 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 1.5 KiB/s rd, 23 MiB/s wr, 15 op/s 2024-09-13T00:39:01.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:00 smithi181 bash[67587]: cluster 2024-09-13T00:38:59.735520+0000 mon.a (mon.0) 3569 : cluster [DBG] osdmap e1918: 8 total, 8 up, 8 in 2024-09-13T00:39:01.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:00 smithi181 bash[67587]: cluster 2024-09-13T00:39:00.422196+0000 mgr.y (mgr.24661) 1856 : cluster [DBG] pgmap v3335: 137 pgs: 137 active+clean; 472 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 1.5 KiB/s rd, 23 MiB/s wr, 15 op/s 2024-09-13T00:39:01.889 INFO:tasks.workunit.client.0.smithi136.stdout: i=14 exists [978366~12276827,17113294~36091926,58515289~704557,61366356~25648615,89250005~13234343,105459787~1424206,111158409~10067142,122043306~24491246,147450241~4531765,153773626~14339783,169300267~5598513,180155982~4105980,187369573~871966,201556540~2015504,204447798~1679405,206691828~12049035,220044632~12652716,240318840~11918637,260842262~7844195,269960512~2117506,273904834~1135501,276897564~24043645,301118059~8395949,312566619~2948229,316301726~21447186,339257207~1141979,341594269~13892162,356349735~2009351,365490113~20859791,387920372~17020802,405595440~7376810] wrote [978366~8009759,28753760~4043921,63639003~3918459,207131072~5649534,297239089~3702120,365490113~5510962,371762046~1637913,379288747~1574174,389872420~4206259] 2024-09-13T00:39:02.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:01 smithi181 bash[67587]: cluster 2024-09-13T00:39:00.758449+0000 mon.a (mon.0) 3570 : cluster [DBG] osdmap e1919: 8 total, 8 up, 8 in 2024-09-13T00:39:02.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:01 smithi136 bash[75164]: cluster 2024-09-13T00:39:00.758449+0000 mon.a (mon.0) 3570 : cluster [DBG] osdmap e1919: 8 total, 8 up, 8 in 2024-09-13T00:39:02.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:01 smithi136 bash[76371]: cluster 2024-09-13T00:39:00.758449+0000 mon.a (mon.0) 3570 : cluster [DBG] osdmap e1919: 8 total, 8 up, 8 in 2024-09-13T00:39:02.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:39:02 smithi181 bash[95505]: ts=2024-09-13T00:39:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:39:02.919 INFO:tasks.workunit.client.0.smithi136.stdout: i=15 exists [978366~12276827,17113294~36091926,58515289~704557,61366356~25648615,89250005~13234343,105459787~1424206,111158409~10067142,122043306~24774781,147450241~20663168,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~2015504,204447798~1679405,206691828~12049035,220044632~12652716,240318840~11918637,260842262~7844195,269960512~2117506,273904834~1135501,276897564~24043645,301118059~8395949,312566619~2948229,316301726~21447186,339257207~1141979,341594269~13892162,356349735~2009351,365490113~20859791,387920372~17020802,405595440~7376810] wrote [144141427~2676660,148016038~10362739,171460898~6550730,192121569~7452224,207577196~1690477,248606422~493001,327427195~2026374,348299711~4603277,366010954~3024486] 2024-09-13T00:39:03.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:02 smithi181 bash[67587]: cluster 2024-09-13T00:39:01.761485+0000 mon.a (mon.0) 3571 : cluster [DBG] osdmap e1920: 8 total, 8 up, 8 in 2024-09-13T00:39:03.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:02 smithi181 bash[67587]: cluster 2024-09-13T00:39:02.423090+0000 mgr.y (mgr.24661) 1857 : cluster [DBG] pgmap v3338: 137 pgs: 137 active+clean; 585 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 208 KiB/s rd, 41 MiB/s wr, 331 op/s 2024-09-13T00:39:03.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:02 smithi136 bash[75164]: cluster 2024-09-13T00:39:01.761485+0000 mon.a (mon.0) 3571 : cluster [DBG] osdmap e1920: 8 total, 8 up, 8 in 2024-09-13T00:39:03.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:02 smithi136 bash[75164]: cluster 2024-09-13T00:39:02.423090+0000 mgr.y (mgr.24661) 1857 : cluster [DBG] pgmap v3338: 137 pgs: 137 active+clean; 585 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 208 KiB/s rd, 41 MiB/s wr, 331 op/s 2024-09-13T00:39:03.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:02 smithi136 bash[76371]: cluster 2024-09-13T00:39:01.761485+0000 mon.a (mon.0) 3571 : cluster [DBG] osdmap e1920: 8 total, 8 up, 8 in 2024-09-13T00:39:03.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:02 smithi136 bash[76371]: cluster 2024-09-13T00:39:02.423090+0000 mgr.y (mgr.24661) 1857 : cluster [DBG] pgmap v3338: 137 pgs: 137 active+clean; 585 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 208 KiB/s rd, 41 MiB/s wr, 331 op/s 2024-09-13T00:39:03.893 INFO:tasks.workunit.client.0.smithi136.stdout: i=16 exists [978366~12276827,17113294~39372164,58515289~704557,61366356~25648615,89250005~13234343,105459787~1424206,111158409~10067142,122043306~24774781,147450241~20663168,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~2015504,204447798~1679405,206691828~12049035,220044632~12652716,240318840~11918637,260842262~7844195,269960512~2117506,273904834~1135501,276897564~24043645,301118059~8395949,312566619~2948229,316301726~21447186,339257207~1141979,341594269~13892162,356349735~2009351,365490113~20859791,387920372~17020802,405595440~7376810] wrote [18131473~1232378,48635901~7849557,62637940~1049327,122781879~1514283,131583241~5698564,158290495~6974599,222829084~7034872,344823952~7059258,379003744~4970562] 2024-09-13T00:39:04.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:03 smithi181 bash[67587]: cluster 2024-09-13T00:39:02.783828+0000 mon.a (mon.0) 3572 : cluster [DBG] osdmap e1921: 8 total, 8 up, 8 in 2024-09-13T00:39:04.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:03 smithi136 bash[75164]: cluster 2024-09-13T00:39:02.783828+0000 mon.a (mon.0) 3572 : cluster [DBG] osdmap e1921: 8 total, 8 up, 8 in 2024-09-13T00:39:04.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:03 smithi136 bash[76371]: cluster 2024-09-13T00:39:02.783828+0000 mon.a (mon.0) 3572 : cluster [DBG] osdmap e1921: 8 total, 8 up, 8 in 2024-09-13T00:39:04.884 INFO:tasks.workunit.client.0.smithi136.stdout: i=17 exists [978366~12276827,17113294~39372164,58515289~28499682,89250005~13234343,105459787~1424206,111158409~10067142,122043306~24774781,147450241~20663168,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240318840~11918637,252886187~622714,260842262~7844195,269960512~2117506,273904834~1135501,276897564~24043645,301118059~8395949,312566619~2948229,316301726~21447186,339257207~1141979,341594269~13892162,356349735~2009351,365490113~20859791,387920372~17020802,405595440~7376810] wrote [7846533~846537,17650227~1831268,37199402~669117,58865372~5846796,201939997~6695813,215619334~2003239,227214911~7176352,252886187~622714,389458074~7853223,401222124~243609] 2024-09-13T00:39:05.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:04 smithi181 bash[67587]: cluster 2024-09-13T00:39:03.790433+0000 mon.a (mon.0) 3573 : cluster [DBG] osdmap e1922: 8 total, 8 up, 8 in 2024-09-13T00:39:05.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:04 smithi181 bash[67587]: cluster 2024-09-13T00:39:04.423831+0000 mgr.y (mgr.24661) 1858 : cluster [DBG] pgmap v3341: 137 pgs: 137 active+clean; 660 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 208 KiB/s rd, 46 MiB/s wr, 332 op/s 2024-09-13T00:39:05.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:04 smithi136 bash[76371]: cluster 2024-09-13T00:39:03.790433+0000 mon.a (mon.0) 3573 : cluster [DBG] osdmap e1922: 8 total, 8 up, 8 in 2024-09-13T00:39:05.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:04 smithi136 bash[76371]: cluster 2024-09-13T00:39:04.423831+0000 mgr.y (mgr.24661) 1858 : cluster [DBG] pgmap v3341: 137 pgs: 137 active+clean; 660 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 208 KiB/s rd, 46 MiB/s wr, 332 op/s 2024-09-13T00:39:05.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:04 smithi136 bash[75164]: cluster 2024-09-13T00:39:03.790433+0000 mon.a (mon.0) 3573 : cluster [DBG] osdmap e1922: 8 total, 8 up, 8 in 2024-09-13T00:39:05.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:04 smithi136 bash[75164]: cluster 2024-09-13T00:39:04.423831+0000 mgr.y (mgr.24661) 1858 : cluster [DBG] pgmap v3341: 137 pgs: 137 active+clean; 660 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 208 KiB/s rd, 46 MiB/s wr, 332 op/s 2024-09-13T00:39:05.913 INFO:tasks.workunit.client.0.smithi136.stdout: i=18 exists [978366~12276827,17113294~39372164,58515289~28499682,89250005~13234343,105459787~1424206,111158409~10067142,122043306~24774781,147450241~20663168,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240318840~11918637,252886187~622714,260842262~7844195,269960512~2117506,273904834~1135501,276897564~24043645,301118059~8395949,312566619~2948229,316301726~21447186,339257207~1141979,341594269~13892162,356349735~2009351,365490113~20859791,387920372~17020802,405595440~7376810] wrote [47180962~4366707,66638502~7190962,76198939~3215002,142230204~1250407,158551854~7251869,205819705~2852580,211120086~6863455,229610868~1211581,370509734~5125029] 2024-09-13T00:39:06.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:05 smithi181 bash[67587]: cluster 2024-09-13T00:39:04.801349+0000 mon.a (mon.0) 3574 : cluster [DBG] osdmap e1923: 8 total, 8 up, 8 in 2024-09-13T00:39:06.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:05 smithi136 bash[75164]: cluster 2024-09-13T00:39:04.801349+0000 mon.a (mon.0) 3574 : cluster [DBG] osdmap e1923: 8 total, 8 up, 8 in 2024-09-13T00:39:06.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:05 smithi136 bash[76371]: cluster 2024-09-13T00:39:04.801349+0000 mon.a (mon.0) 3574 : cluster [DBG] osdmap e1923: 8 total, 8 up, 8 in 2024-09-13T00:39:06.942 INFO:tasks.workunit.client.0.smithi136.stdout: i=19 exists [978366~12404130,17113294~39372164,58515289~28499682,89250005~13234343,105459787~1424206,111158409~10067142,122043306~24774781,147450241~20663168,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240318840~11918637,252886187~622714,260842262~7844195,269960512~2117506,273064912~36449096,312566619~2948229,316301726~21447186,339257207~1141979,341594269~18955574,362400302~23949602,387920372~17020802,405595440~7376810] wrote [5881326~7501170,80242387~800824,91032793~4865453,128928644~7456281,273064912~4591743,294714089~7671666,319038423~7560208,355040685~5509158,362400302~4428536,383843711~554618] 2024-09-13T00:39:07.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:06 smithi181 bash[67587]: cluster 2024-09-13T00:39:05.803259+0000 mon.a (mon.0) 3575 : cluster [DBG] osdmap e1924: 8 total, 8 up, 8 in 2024-09-13T00:39:07.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:06 smithi181 bash[67587]: cluster 2024-09-13T00:39:06.424688+0000 mgr.y (mgr.24661) 1859 : cluster [DBG] pgmap v3344: 137 pgs: 137 active+clean; 768 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 231 KiB/s rd, 46 MiB/s wr, 360 op/s 2024-09-13T00:39:07.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:06 smithi136 bash[75164]: cluster 2024-09-13T00:39:05.803259+0000 mon.a (mon.0) 3575 : cluster [DBG] osdmap e1924: 8 total, 8 up, 8 in 2024-09-13T00:39:07.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:06 smithi136 bash[75164]: cluster 2024-09-13T00:39:06.424688+0000 mgr.y (mgr.24661) 1859 : cluster [DBG] pgmap v3344: 137 pgs: 137 active+clean; 768 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 231 KiB/s rd, 46 MiB/s wr, 360 op/s 2024-09-13T00:39:07.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:06 smithi136 bash[76371]: cluster 2024-09-13T00:39:05.803259+0000 mon.a (mon.0) 3575 : cluster [DBG] osdmap e1924: 8 total, 8 up, 8 in 2024-09-13T00:39:07.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:06 smithi136 bash[76371]: cluster 2024-09-13T00:39:06.424688+0000 mgr.y (mgr.24661) 1859 : cluster [DBG] pgmap v3344: 137 pgs: 137 active+clean; 768 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 231 KiB/s rd, 46 MiB/s wr, 360 op/s 2024-09-13T00:39:07.859 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to 1 diff [6333791~894944,43286419~3567780,50397556~1503215,89250005~3131785,163394852~4718557,171656619~199342,240318840~976558,269960512~2117506,302940867~6573141,341594269~8118654] 2024-09-13T00:39:07.859 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [6333791~894944,43286419~3567780,50397556~1503215,89250005~3131785,163394852~4718557,171656619~199342,240318840~976558,269960512~2117506,302940867~6573141,341594269~8118654] 2024-09-13T00:39:07.899 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [4194304~3034431,41943040~4911159,50331648~1569123,88080384~4301406,159383552~12472409,239075328~2220070,268435456~3642562,302940867~6573141,339738624~9974299] 2024-09-13T00:39:07.899 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to 2 diff [5507084~6002062,18624312~6440348,43286419~3567780,50397556~1503215,83314700~3016720,89250005~3131785,97623360~4189624,113921932~3493308,126152787~14073799,143604550~2930002,163394852~4718557,171656619~199342,204447798~1679405,240318840~976558,269960512~2117506,302940867~6573141,341594269~8118654,381456303~3508409] 2024-09-13T00:39:07.899 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [5507084~6002062,18624312~6440348,43286419~3567780,50397556~1503215,83314700~3016720,89250005~3131785,97623360~4189624,113921932~3493308,126152787~14073799,143604550~2930002,163394852~4718557,171656619~199342,204447798~1679405,240318840~976558,269960512~2117506,302940867~6573141,341594269~8118654,381456303~3508409] 2024-09-13T00:39:07.914 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [4194304~7314842,16777216~8287444,41943040~4911159,50331648~1569123,79691776~6639644,88080384~4301406,97623360~4189624,113921932~3493308,125829120~14397466,143604550~2930002,159383552~12472409,204447798~1679405,239075328~2220070,268435456~3642562,302940867~6573141,339738624~9974299,377487360~7477352] 2024-09-13T00:39:07.914 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to 3 diff [5507084~6002062,18624312~6440348,38721951~8132248,50397556~1503215,71349028~11331915,83314700~3016720,86936101~78870,89250005~12562979,113921932~3493308,126152787~14073799,143604550~2930002,163394852~4718557,171656619~199342,204447798~1679405,222904244~4325743,240318840~976558,269960512~2117506,302940867~6573141,317751762~7525150,341594269~8118654,373179324~6803307,381456303~3508409,401627021~3314153] 2024-09-13T00:39:07.914 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [5507084~6002062,18624312~6440348,38721951~8132248,50397556~1503215,71349028~11331915,83314700~3016720,86936101~78870,89250005~12562979,113921932~3493308,126152787~14073799,143604550~2930002,163394852~4718557,171656619~199342,204447798~1679405,222904244~4325743,240318840~976558,269960512~2117506,302940867~6573141,317751762~7525150,341594269~8118654,373179324~6803307,381456303~3508409,401627021~3314153] 2024-09-13T00:39:07.929 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [4194304~7314842,16777216~8287444,37748736~9105463,50331648~1569123,71303168~15711803,88080384~13732600,113921932~3493308,125829120~14397466,143604550~2930002,159383552~12472409,204447798~1679405,222298112~4931875,239075328~2220070,268435456~3642562,302940867~6573141,314572800~10704112,339738624~9974299,371886289~13078423,398458880~6482294] 2024-09-13T00:39:07.929 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to 4 diff [5507084~6002062,18624312~6440348,38721951~14483269,58515289~704557,61366356~6043690,71349028~11331915,83314700~3016720,86936101~78870,89250005~12562979,113921932~3493308,126152787~14073799,143604550~2930002,153773626~3477336,162728173~5385236,171656619~199342,204447798~1679405,222904244~4325743,240318840~976558,269960512~2117506,289226804~7918678,302940867~6573141,317751762~7525150,341594269~8118654,373179324~6803307,381456303~3508409,401627021~3314153,405696667~5790944] 2024-09-13T00:39:07.929 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [5507084~6002062,18624312~6440348,38721951~14483269,58515289~704557,61366356~6043690,71349028~11331915,83314700~3016720,86936101~78870,89250005~12562979,113921932~3493308,126152787~14073799,143604550~2930002,153773626~3477336,162728173~5385236,171656619~199342,204447798~1679405,222904244~4325743,240318840~976558,269960512~2117506,289226804~7918678,302940867~6573141,317751762~7525150,341594269~8118654,373179324~6803307,381456303~3508409,401627021~3314153,405696667~5790944] 2024-09-13T00:39:07.943 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [4194304~7314842,16777216~8287444,37748736~15456484,54525952~12884094,71303168~15711803,88080384~13732600,113921932~3493308,125829120~14397466,143604550~2930002,150994944~6256018,159383552~12472409,204447798~1679405,222298112~4931875,239075328~2220070,268435456~3642562,285212672~11932810,302940867~6573141,314572800~10704112,339738624~9974299,371886289~13078423,398458880~6482294,405696667~5790944] 2024-09-13T00:39:07.943 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to 5 diff [5507084~6002062,18624312~6440348,26995819~3290536,38721951~14483269,58515289~704557,61366356~6043690,71349028~11331915,83314700~3016720,86936101~78870,89250005~12562979,113921932~3493308,126152787~14073799,143604550~2930002,147450241~4531765,153773626~3477336,162728173~5385236,171656619~199342,180807822~766370,204447798~1679405,208047485~1381668,222904244~4325743,240318840~976558,269960512~2117506,279969777~95145,289226804~7918678,302940867~6573141,317751762~7525150,339257207~1141979,341594269~8118654,373179324~11785388,390387020~7188584,401627021~3314153,405696667~5790944] 2024-09-13T00:39:07.943 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [5507084~6002062,18624312~6440348,26995819~3290536,38721951~14483269,58515289~704557,61366356~6043690,71349028~11331915,83314700~3016720,86936101~78870,89250005~12562979,113921932~3493308,126152787~14073799,143604550~2930002,147450241~4531765,153773626~3477336,162728173~5385236,171656619~199342,180807822~766370,204447798~1679405,208047485~1381668,222904244~4325743,240318840~976558,269960512~2117506,279969777~95145,289226804~7918678,302940867~6573141,317751762~7525150,339257207~1141979,341594269~8118654,373179324~11785388,390387020~7188584,401627021~3314153,405696667~5790944] 2024-09-13T00:39:07.956 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [4194304~7314842,16777216~8287444,25165824~5120531,37748736~15456484,54525952~12884094,71303168~15711803,88080384~13732600,113921932~3493308,125829120~14397466,143604550~2930002,146800640~10450322,159383552~12472409,180355072~1219120,204447798~4981355,222298112~4931875,239075328~2220070,268435456~3642562,276824064~3240858,285212672~11932810,302940867~6573141,314572800~10704112,336574728~13138195,371886289~13078423,390070272~7505332,398458880~6482294,405696667~5790944] 2024-09-13T00:39:07.956 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to 6 diff [2612746~8896400,18624312~6440348,26995819~3290536,32429128~3147149,38721951~14483269,58515289~704557,61366356~6043690,71349028~11331915,83314700~3016720,86936101~78870,89250005~12562979,113921932~3493308,126152787~14073799,143604550~2930002,147450241~4531765,153773626~3477336,162728173~5385236,171656619~199342,180807822~766370,204447798~1679405,208047485~1381668,222904244~4325743,240318840~976558,244107705~1444348,260842262~7844195,269960512~2117506,279969777~95145,289202665~7942817,301411264~8102744,317751762~7525150,339257207~1141979,341594269~8118654,373179324~11785388,390387020~7188584,401627021~3314153,405696667~5790944] 2024-09-13T00:39:07.956 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [2612746~8896400,18624312~6440348,26995819~3290536,32429128~3147149,38721951~14483269,58515289~704557,61366356~6043690,71349028~11331915,83314700~3016720,86936101~78870,89250005~12562979,113921932~3493308,126152787~14073799,143604550~2930002,147450241~4531765,153773626~3477336,162728173~5385236,171656619~199342,180807822~766370,204447798~1679405,208047485~1381668,222904244~4325743,240318840~976558,244107705~1444348,260842262~7844195,269960512~2117506,279969777~95145,289202665~7942817,301411264~8102744,317751762~7525150,339257207~1141979,341594269~8118654,373179324~11785388,390387020~7188584,401627021~3314153,405696667~5790944] 2024-09-13T00:39:07.967 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~11509146,16777216~8287444,25165824~10410453,37748736~15456484,54525952~12884094,71303168~15711803,88080384~13732600,113921932~3493308,125829120~14397466,143604550~2930002,146800640~10450322,159383552~12472409,180355072~1219120,204447798~4981355,222298112~4931875,239075328~2220070,243269632~2282421,260046848~12031170,276824064~3240858,285212672~11932810,301411264~8102744,314572800~10704112,336574728~13138195,371886289~13078423,390070272~7505332,398458880~6482294,405696667~5790944] 2024-09-13T00:39:07.967 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to 7 diff [2612746~8896400,18624312~6440348,26995819~3290536,31904846~21300374,58515289~704557,61366356~6043690,71349028~11331915,83314700~3016720,86936101~78870,89250005~12562979,113921932~7303619,126152787~14073799,143604550~2930002,147450241~4531765,153773626~7600370,162728173~5385236,171656619~199342,180807822~766370,204447798~1679405,208047485~1381668,212320259~6420604,222904244~4325743,227825355~4871993,240318840~976558,244107705~1444348,260842262~7844195,269960512~2117506,276897564~6749766,289202665~7942817,301411264~8102744,317751762~7525150,339257207~1141979,341594269~8118654,373179324~11785388,387920372~9655232,401627021~3314153,405696667~5790944] 2024-09-13T00:39:07.968 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [2612746~8896400,18624312~6440348,26995819~3290536,31904846~21300374,58515289~704557,61366356~6043690,71349028~11331915,83314700~3016720,86936101~78870,89250005~12562979,113921932~7303619,126152787~14073799,143604550~2930002,147450241~4531765,153773626~7600370,162728173~5385236,171656619~199342,180807822~766370,204447798~1679405,208047485~1381668,212320259~6420604,222904244~4325743,227825355~4871993,240318840~976558,244107705~1444348,260842262~7844195,269960512~2117506,276897564~6749766,289202665~7942817,301411264~8102744,317751762~7525150,339257207~1141979,341594269~8118654,373179324~11785388,387920372~9655232,401627021~3314153,405696667~5790944] 2024-09-13T00:39:07.977 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~11509146,16777216~8287444,25165824~28039396,54525952~12884094,71303168~15711803,88080384~13732600,113921932~7303619,125829120~14397466,143604550~2930002,146800640~25055321,180355072~1219120,204447798~4981355,209715200~9025663,222298112~10399236,239075328~2220070,243269632~2282421,260046848~12031170,276824064~6823266,285212672~11932810,301411264~8102744,314572800~10704112,336574728~13138195,371886289~13078423,385875968~11699636,398458880~6482294,405696667~5790944] 2024-09-13T00:39:07.978 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to 8 diff [2612746~8896400,18624312~6440348,26995819~4642300,31904846~21300374,58515289~704557,61366356~6043690,71349028~14982392,86936101~78870,89250005~12562979,111245499~9980052,126152787~14073799,143604550~2930002,147450241~4531765,153773626~7600370,162728173~5385236,171656619~199342,172123438~2775342,180155982~1465141,204447798~1679405,208047485~1381668,212320259~6420604,222904244~4325743,227825355~4871993,240318840~976558,244107705~8129772,260842262~7844195,269960512~2117506,276897564~6749766,289202665~7942817,301411264~8102744,317751762~7525150,330730451~2400977,339257207~1141979,341594269~8118654,373179324~11785388,387920372~9655232,401627021~3314153,405696667~5790944] 2024-09-13T00:39:07.978 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [2612746~8896400,18624312~6440348,26995819~4642300,31904846~21300374,58515289~704557,61366356~6043690,71349028~14982392,86936101~78870,89250005~12562979,111245499~9980052,126152787~14073799,143604550~2930002,147450241~4531765,153773626~7600370,162728173~5385236,171656619~199342,172123438~2775342,180155982~1465141,204447798~1679405,208047485~1381668,212320259~6420604,222904244~4325743,227825355~4871993,240318840~976558,244107705~8129772,260842262~7844195,269960512~2117506,276897564~6749766,289202665~7942817,301411264~8102744,317751762~7525150,330730451~2400977,339257207~1141979,341594269~8118654,373179324~11785388,387920372~9655232,401627021~3314153,405696667~5790944] 2024-09-13T00:39:07.987 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~11509146,16777216~8287444,25165824~28039396,54525952~12884094,71303168~15711803,88080384~13732600,111245499~9980052,125829120~14397466,143604550~2930002,146800640~25055321,171966464~2932316,176160768~5460355,204447798~4981355,209715200~9025663,222298112~10399236,239075328~2220070,243269632~8967845,260046848~12031170,276824064~6823266,285212672~11932810,301411264~8102744,314572800~10704112,330730451~2400977,336574728~13138195,371886289~13078423,385875968~11699636,398458880~6482294,405696667~5790944] 2024-09-13T00:39:07.987 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to 9 diff [2612746~8896400,18624312~6440348,26995819~4642300,31904846~21300374,58515289~704557,61366356~6043690,71349028~15665943,89250005~12562979,111245499~9980052,126152787~14073799,143604550~2930002,147450241~4531765,153773626~14339783,169300267~5598513,180155982~4105980,204447798~1679405,206691828~1354340,208047485~1381668,212320259~6420604,220044632~7185355,227825355~4871993,240318840~976558,244107705~8129772,260842262~7844195,269960512~2117506,273904834~1135501,276897564~6749766,284165576~12979906,301411264~8102744,317751762~7525150,330730451~2400977,339257207~1141979,341594269~8118654,373179324~11785388,387920372~9655232,401627021~3314153,405696667~5790944] 2024-09-13T00:39:07.988 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [2612746~8896400,18624312~6440348,26995819~4642300,31904846~21300374,58515289~704557,61366356~6043690,71349028~15665943,89250005~12562979,111245499~9980052,126152787~14073799,143604550~2930002,147450241~4531765,153773626~14339783,169300267~5598513,180155982~4105980,204447798~1679405,206691828~1354340,208047485~1381668,212320259~6420604,220044632~7185355,227825355~4871993,240318840~976558,244107705~8129772,260842262~7844195,269960512~2117506,273904834~1135501,276897564~6749766,284165576~12979906,301411264~8102744,317751762~7525150,330730451~2400977,339257207~1141979,341594269~8118654,373179324~11785388,387920372~9655232,401627021~3314153,405696667~5790944] 2024-09-13T00:39:07.996 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~11509146,16777216~8287444,25165824~28039396,54525952~12884094,71303168~15711803,88080384~13732600,111245499~9980052,125829120~14397466,143604550~2930002,146800640~28098140,176160768~8101194,204447798~4981355,209715200~22982148,239075328~2220070,243269632~8967845,260046848~12031170,272629760~2410575,276824064~20321418,301411264~8102744,314572800~10704112,330730451~2400977,336574728~13138195,371886289~13078423,385875968~11699636,398458880~6482294,405696667~5790944] 2024-09-13T00:39:07.997 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to 10 diff [2612746~8896400,18624312~6440348,26995819~4642300,31904846~21300374,58515289~704557,61366356~25648615,89250005~12562979,111245499~9980052,126152787~14073799,143604550~2930002,147450241~4531765,153773626~14339783,169300267~5598513,180155982~4105980,204447798~1679405,206691828~1354340,208047485~1381668,212320259~6420604,220044632~12652716,240318840~976558,244107705~8129772,260842262~7844195,269960512~2117506,273904834~1135501,276897564~6749766,284165576~12979906,301411264~8102744,317751762~7525150,330730451~2400977,339257207~1141979,341594269~13892162,365997237~1542191,373179324~11785388,387920372~9655232,401627021~3314153,405696667~5790944] 2024-09-13T00:39:07.997 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [2612746~8896400,18624312~6440348,26995819~4642300,31904846~21300374,58515289~704557,61366356~25648615,89250005~12562979,111245499~9980052,126152787~14073799,143604550~2930002,147450241~4531765,153773626~14339783,169300267~5598513,180155982~4105980,204447798~1679405,206691828~1354340,208047485~1381668,212320259~6420604,220044632~12652716,240318840~976558,244107705~8129772,260842262~7844195,269960512~2117506,273904834~1135501,276897564~6749766,284165576~12979906,301411264~8102744,317751762~7525150,330730451~2400977,339257207~1141979,341594269~13892162,365997237~1542191,373179324~11785388,387920372~9655232,401627021~3314153,405696667~5790944] 2024-09-13T00:39:08.005 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~11509146,16777216~8287444,25165824~28039396,54525952~32489019,88080384~13732600,111245499~9980052,125829120~14397466,143604550~2930002,146800640~28098140,176160768~8101194,204447798~4981355,209715200~22982148,239075328~2220070,243269632~8967845,260046848~12031170,272629760~2410575,276824064~20321418,301411264~8102744,314572800~10704112,330730451~2400977,336574728~18911703,365997237~1542191,371886289~13078423,385875968~11699636,398458880~6482294,405696667~5790944] 2024-09-13T00:39:08.005 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to 11 diff [2612746~8896400,18624312~6440348,26995819~4642300,31904846~21300374,58515289~704557,61366356~25648615,89250005~12562979,105459787~1424206,111245499~9980052,122043306~18183280,143604550~2930002,147450241~4531765,153773626~14339783,169300267~5598513,180155982~4105980,204447798~1679405,206691828~1354340,208047485~1381668,212320259~6420604,220044632~12652716,240318840~11918637,260842262~7844195,269960512~2117506,273904834~1135501,276897564~21900567,301411264~8102744,317751762~10401988,330730451~2400977,335934186~1814726,339257207~1141979,341594269~13892162,365997237~1542191,373179324~11785388,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.005 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [2612746~8896400,18624312~6440348,26995819~4642300,31904846~21300374,58515289~704557,61366356~25648615,89250005~12562979,105459787~1424206,111245499~9980052,122043306~18183280,143604550~2930002,147450241~4531765,153773626~14339783,169300267~5598513,180155982~4105980,204447798~1679405,206691828~1354340,208047485~1381668,212320259~6420604,220044632~12652716,240318840~11918637,260842262~7844195,269960512~2117506,273904834~1135501,276897564~21900567,301411264~8102744,317751762~10401988,330730451~2400977,335934186~1814726,339257207~1141979,341594269~13892162,365997237~1542191,373179324~11785388,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.012 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~11509146,16777216~8287444,25165824~28039396,54525952~32489019,88080384~13732600,104857600~2026393,111245499~9980052,121634816~18591770,143604550~2930002,146800640~28098140,176160768~8101194,204447798~4981355,209715200~22982148,239075328~13162149,260046848~12031170,272629760~2410575,276824064~21974067,301411264~8102744,314572800~13580950,330730451~2400977,335934186~19552245,365997237~1542191,371886289~13078423,385875968~19065206,405696667~5790944] 2024-09-13T00:39:08.013 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to 12 diff [2612746~10642447,18624312~34580908,58515289~704557,61366356~25648615,89250005~13234343,105459787~1424206,111245499~9980052,122043306~18183280,143604550~2930002,147450241~4531765,153773626~14339783,169300267~5598513,180155982~4105980,204447798~1679405,206691828~1354340,208047485~1381668,212320259~6420604,220044632~12652716,240318840~11918637,260842262~7844195,269960512~2117506,273904834~1135501,276897564~21900567,301411264~8102744,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,341594269~13892162,365997237~5182427,373179324~13170580,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.013 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [2612746~10642447,18624312~34580908,58515289~704557,61366356~25648615,89250005~13234343,105459787~1424206,111245499~9980052,122043306~18183280,143604550~2930002,147450241~4531765,153773626~14339783,169300267~5598513,180155982~4105980,204447798~1679405,206691828~1354340,208047485~1381668,212320259~6420604,220044632~12652716,240318840~11918637,260842262~7844195,269960512~2117506,273904834~1135501,276897564~21900567,301411264~8102744,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,341594269~13892162,365997237~5182427,373179324~13170580,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.020 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13255193,16777216~36428004,54525952~32489019,88080384~14403964,104857600~2026393,111245499~9980052,121634816~18591770,143604550~2930002,146800640~28098140,176160768~8101194,204447798~4981355,209715200~22982148,239075328~13162149,260046848~12031170,272629760~2410575,276824064~21974067,301411264~8102744,310378496~24935507,335934186~19552245,365997237~5182427,371886289~33054885,405696667~5790944] 2024-09-13T00:39:08.020 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to 13 diff [2612746~10642447,17113294~36091926,58515289~704557,61366356~25648615,89250005~13234343,105459787~1424206,111245499~9980052,122043306~18183280,143604550~2930002,147450241~4531765,153773626~14339783,169300267~5598513,180155982~4105980,187369573~871966,201556540~2015504,204447798~1679405,206691828~1354340,208047485~1381668,209974378~1903654,212320259~6420604,220044632~12652716,240318840~11918637,260842262~7844195,269960512~2117506,273904834~1135501,276897564~23715912,301411264~8102744,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,341594269~13892162,365997237~5182427,373179324~13170580,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.020 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [2612746~10642447,17113294~36091926,58515289~704557,61366356~25648615,89250005~13234343,105459787~1424206,111245499~9980052,122043306~18183280,143604550~2930002,147450241~4531765,153773626~14339783,169300267~5598513,180155982~4105980,187369573~871966,201556540~2015504,204447798~1679405,206691828~1354340,208047485~1381668,209974378~1903654,212320259~6420604,220044632~12652716,240318840~11918637,260842262~7844195,269960512~2117506,273904834~1135501,276897564~23715912,301411264~8102744,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,341594269~13892162,365997237~5182427,373179324~13170580,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.027 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13255193,16777216~36428004,54525952~32489019,88080384~14403964,104857600~2026393,111245499~9980052,121634816~18591770,143604550~2930002,146800640~28098140,176160768~8101194,184549376~3692163,201556540~2015504,204447798~4981355,209715200~22982148,239075328~13162149,260046848~12031170,272629760~2410575,276824064~23789412,301411264~8102744,310378496~24935507,335934186~19552245,365997237~5182427,371886289~33054885,405696667~5790944] 2024-09-13T00:39:08.027 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to 14 diff [978366~12276827,17113294~36091926,58515289~704557,61366356~25648615,89250005~13234343,105459787~1424206,111245499~9980052,122043306~18183280,143604550~2930002,147450241~4531765,153773626~14339783,169300267~5598513,180155982~4105980,187369573~871966,201556540~2015504,204447798~1679405,206691828~12049035,220044632~12652716,240318840~11918637,260842262~7844195,269960512~2117506,273904834~1135501,276897564~24043645,301411264~8102744,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,341594269~13892162,365490113~5689551,371762046~14587858,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.028 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~36091926,58515289~704557,61366356~25648615,89250005~13234343,105459787~1424206,111245499~9980052,122043306~18183280,143604550~2930002,147450241~4531765,153773626~14339783,169300267~5598513,180155982~4105980,187369573~871966,201556540~2015504,204447798~1679405,206691828~12049035,220044632~12652716,240318840~11918637,260842262~7844195,269960512~2117506,273904834~1135501,276897564~24043645,301411264~8102744,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,341594269~13892162,365490113~5689551,371762046~14587858,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.034 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13255193,16777216~36428004,54525952~32489019,88080384~14403964,104857600~2026393,111245499~9980052,121634816~18591770,143604550~2930002,146800640~28098140,176160768~8101194,184549376~3692163,201556540~2015504,204447798~28249550,239075328~13162149,260046848~12031170,272629760~2410575,276824064~24117145,301411264~8102744,310378496~24935507,335934186~19552245,365490113~5689551,371762046~33179128,405696667~5790944] 2024-09-13T00:39:08.035 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to 15 diff [978366~12276827,17113294~36091926,58515289~704557,61366356~25648615,89250005~13234343,105459787~1424206,111245499~9980052,122043306~18183280,143604550~3213537,147450241~20663168,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~2015504,204447798~1679405,206691828~12049035,220044632~12652716,240318840~11918637,260842262~7844195,269960512~2117506,273904834~1135501,276897564~24043645,301411264~8102744,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,341594269~13892162,365490113~5689551,371762046~14587858,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.035 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~36091926,58515289~704557,61366356~25648615,89250005~13234343,105459787~1424206,111245499~9980052,122043306~18183280,143604550~3213537,147450241~20663168,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~2015504,204447798~1679405,206691828~12049035,220044632~12652716,240318840~11918637,260842262~7844195,269960512~2117506,273904834~1135501,276897564~24043645,301411264~8102744,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,341594269~13892162,365490113~5689551,371762046~14587858,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.041 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13255193,16777216~36428004,54525952~32489019,88080384~14403964,104857600~2026393,111245499~9980052,121634816~18591770,143604550~40657412,184549376~3692163,188743680~10830113,201556540~2015504,204447798~28249550,239075328~13162149,260046848~12031170,272629760~2410575,276824064~24117145,301411264~8102744,310378496~24935507,335934186~19552245,365490113~5689551,371762046~33179128,405696667~5790944] 2024-09-13T00:39:08.041 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to 16 diff [978366~12276827,17113294~39372164,58515289~704557,61366356~25648615,89250005~13234343,105459787~1424206,111245499~9980052,122043306~18183280,143604550~3213537,147450241~20663168,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~2015504,204447798~1679405,206691828~12049035,220044632~12652716,240318840~11918637,260842262~7844195,269960512~2117506,273904834~1135501,276897564~24043645,301411264~8102744,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,341594269~13892162,365490113~5689551,371762046~14587858,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.042 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~39372164,58515289~704557,61366356~25648615,89250005~13234343,105459787~1424206,111245499~9980052,122043306~18183280,143604550~3213537,147450241~20663168,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~2015504,204447798~1679405,206691828~12049035,220044632~12652716,240318840~11918637,260842262~7844195,269960512~2117506,273904834~1135501,276897564~24043645,301411264~8102744,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,341594269~13892162,365490113~5689551,371762046~14587858,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.049 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13255193,16777216~70237755,88080384~14403964,104857600~2026393,111245499~9980052,121634816~18591770,143604550~40657412,184549376~3692163,188743680~10830113,201556540~2015504,204447798~28249550,239075328~13162149,260046848~12031170,272629760~2410575,276824064~24117145,301411264~8102744,310378496~24935507,335934186~19552245,365490113~5689551,371762046~33179128,405696667~5790944] 2024-09-13T00:39:08.049 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to 17 diff [978366~12276827,17113294~39372164,58515289~28499682,89250005~13234343,105459787~1424206,111245499~9980052,122043306~18183280,143604550~3213537,147450241~20663168,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240318840~11918637,252886187~622714,260842262~7844195,269960512~2117506,273904834~1135501,276897564~24043645,301411264~8102744,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,341594269~13892162,365490113~5689551,371762046~14587858,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.049 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~39372164,58515289~28499682,89250005~13234343,105459787~1424206,111245499~9980052,122043306~18183280,143604550~3213537,147450241~20663168,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240318840~11918637,252886187~622714,260842262~7844195,269960512~2117506,273904834~1135501,276897564~24043645,301411264~8102744,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,341594269~13892162,365490113~5689551,371762046~14587858,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.056 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13255193,16777216~70237755,88080384~14403964,104857600~2026393,111245499~9980052,121634816~18591770,143604550~40657412,184549376~3692163,188743680~10830113,201556540~32834723,239075328~14433573,260046848~12031170,272629760~2410575,276824064~24117145,301411264~8102744,310378496~24935507,335934186~19552245,365490113~5689551,371762046~33179128,405696667~5790944] 2024-09-13T00:39:08.056 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to 18 diff [978366~12276827,17113294~39372164,58515289~28499682,89250005~13234343,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~20663168,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240318840~11918637,252886187~622714,260842262~7844195,269960512~2117506,273904834~1135501,276897564~24043645,301411264~8102744,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,341594269~13892162,365490113~20859791,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.056 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~39372164,58515289~28499682,89250005~13234343,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~20663168,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240318840~11918637,252886187~622714,260842262~7844195,269960512~2117506,273904834~1135501,276897564~24043645,301411264~8102744,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,341594269~13892162,365490113~20859791,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.062 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13255193,16777216~70237755,88080384~14403964,104857600~2026393,111245499~9980052,121634816~18591770,142230204~1250407,143604550~40657412,184549376~3692163,188743680~10830113,201556540~32834723,239075328~14433573,260046848~12031170,272629760~2410575,276824064~24117145,301411264~8102744,310378496~24935507,335934186~19552245,365490113~39451061,405696667~5790944] 2024-09-13T00:39:08.062 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to 19 diff [978366~12404130,17113294~39372164,58515289~28499682,89250005~13234343,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~20663168,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240318840~11918637,252886187~622714,260842262~7844195,269960512~2117506,273064912~36449096,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,341594269~18955574,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.063 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~39372164,58515289~28499682,89250005~13234343,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~20663168,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240318840~11918637,252886187~622714,260842262~7844195,269960512~2117506,273064912~36449096,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,341594269~18955574,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.069 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,16777216~70237755,88080384~14403964,104857600~2026393,111245499~9980052,121634816~18591770,142230204~1250407,143604550~40657412,184549376~3692163,188743680~10830113,201556540~32834723,239075328~14433573,260046848~12031170,272629760~36884248,310378496~24935507,335934186~24615657,360710144~44231030,405696667~5790944] 2024-09-13T00:39:08.069 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to 2 diff [5507084~6002062,18624312~6440348,83314700~3016720,97623360~4189624,113921932~3493308,126152787~14073799,143604550~2930002,204447798~1679405,381456303~3508409] 2024-09-13T00:39:08.069 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [5507084~6002062,18624312~6440348,83314700~3016720,97623360~4189624,113921932~3493308,126152787~14073799,143604550~2930002,204447798~1679405,381456303~3508409] 2024-09-13T00:39:08.076 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [5507084~6002062,16777216~8287444,79691776~6639644,97623360~4189624,113921932~3493308,125829120~14397466,143604550~2930002,204447798~1679405,377487360~7477352] 2024-09-13T00:39:08.076 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to 3 diff [5507084~6002062,18624312~6440348,38721951~5200138,71349028~11331915,83314700~3016720,86936101~78870,91915173~9897811,113921932~3493308,126152787~14073799,143604550~2930002,204447798~1679405,222904244~4325743,317751762~7525150,373179324~6803307,381456303~3508409,401627021~3314153] 2024-09-13T00:39:08.076 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [5507084~6002062,18624312~6440348,38721951~5200138,71349028~11331915,83314700~3016720,86936101~78870,91915173~9897811,113921932~3493308,126152787~14073799,143604550~2930002,204447798~1679405,222904244~4325743,317751762~7525150,373179324~6803307,381456303~3508409,401627021~3314153] 2024-09-13T00:39:08.082 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [5507084~6002062,16777216~8287444,37748736~6173353,71303168~15711803,91915173~9897811,113921932~3493308,125829120~14397466,143604550~2930002,204447798~1679405,222298112~4931875,314572800~10704112,371886289~13078423,398458880~6482294] 2024-09-13T00:39:08.082 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to 4 diff [5507084~6002062,18624312~6440348,38721951~5200138,46526671~6678549,58515289~704557,61366356~6043690,71349028~11331915,83314700~3016720,86936101~78870,91915173~9897811,113921932~3493308,126152787~14073799,143604550~2930002,153773626~3477336,162728173~1277847,204447798~1679405,222904244~4325743,289226804~7918678,317751762~7525150,373179324~6803307,381456303~3508409,401627021~3314153,405696667~5790944] 2024-09-13T00:39:08.082 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [5507084~6002062,18624312~6440348,38721951~5200138,46526671~6678549,58515289~704557,61366356~6043690,71349028~11331915,83314700~3016720,86936101~78870,91915173~9897811,113921932~3493308,126152787~14073799,143604550~2930002,153773626~3477336,162728173~1277847,204447798~1679405,222904244~4325743,289226804~7918678,317751762~7525150,373179324~6803307,381456303~3508409,401627021~3314153,405696667~5790944] 2024-09-13T00:39:08.088 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [5507084~6002062,16777216~8287444,37748736~6173353,46526671~6678549,54525952~12884094,71303168~15711803,91915173~9897811,113921932~3493308,125829120~14397466,143604550~2930002,150994944~6256018,162728173~1277847,204447798~1679405,222298112~4931875,285212672~11932810,314572800~10704112,371886289~13078423,398458880~6482294,405696667~5790944] 2024-09-13T00:39:08.088 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to 5 diff [5507084~6002062,18624312~6440348,26995819~3290536,38721951~5200138,46526671~6678549,58515289~704557,61366356~6043690,71349028~11331915,83314700~3016720,86936101~78870,91915173~9897811,113921932~3493308,126152787~14073799,143604550~2930002,147450241~4531765,153773626~3477336,162728173~1277847,180807822~766370,204447798~1679405,208047485~1381668,222904244~4325743,279969777~95145,289226804~7918678,317751762~7525150,339257207~1141979,373179324~11785388,390387020~7188584,401627021~3314153,405696667~5790944] 2024-09-13T00:39:08.088 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [5507084~6002062,18624312~6440348,26995819~3290536,38721951~5200138,46526671~6678549,58515289~704557,61366356~6043690,71349028~11331915,83314700~3016720,86936101~78870,91915173~9897811,113921932~3493308,126152787~14073799,143604550~2930002,147450241~4531765,153773626~3477336,162728173~1277847,180807822~766370,204447798~1679405,208047485~1381668,222904244~4325743,279969777~95145,289226804~7918678,317751762~7525150,339257207~1141979,373179324~11785388,390387020~7188584,401627021~3314153,405696667~5790944] 2024-09-13T00:39:08.094 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [5507084~6002062,16777216~8287444,25165824~5120531,37748736~6173353,46526671~6678549,54525952~12884094,71303168~15711803,91915173~9897811,113921932~3493308,125829120~14397466,143604550~2930002,146800640~10450322,162728173~1277847,180355072~1219120,204447798~4981355,222298112~4931875,276824064~3240858,285212672~11932810,314572800~10704112,336574728~3824458,371886289~13078423,390070272~7505332,398458880~6482294,405696667~5790944] 2024-09-13T00:39:08.094 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to 6 diff [2612746~8896400,18624312~6440348,26995819~3290536,32429128~3147149,38721951~5200138,46526671~6678549,58515289~704557,61366356~6043690,71349028~11331915,83314700~3016720,86936101~78870,91915173~9897811,113921932~3493308,126152787~14073799,143604550~2930002,147450241~4531765,153773626~3477336,162728173~1277847,180807822~766370,204447798~1679405,208047485~1381668,222904244~4325743,244107705~1444348,260842262~7844195,279969777~95145,289202665~7942817,301411264~7201245,317751762~7525150,339257207~1141979,373179324~11785388,390387020~7188584,401627021~3314153,405696667~5790944] 2024-09-13T00:39:08.095 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [2612746~8896400,18624312~6440348,26995819~3290536,32429128~3147149,38721951~5200138,46526671~6678549,58515289~704557,61366356~6043690,71349028~11331915,83314700~3016720,86936101~78870,91915173~9897811,113921932~3493308,126152787~14073799,143604550~2930002,147450241~4531765,153773626~3477336,162728173~1277847,180807822~766370,204447798~1679405,208047485~1381668,222904244~4325743,244107705~1444348,260842262~7844195,279969777~95145,289202665~7942817,301411264~7201245,317751762~7525150,339257207~1141979,373179324~11785388,390387020~7188584,401627021~3314153,405696667~5790944] 2024-09-13T00:39:08.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:07 smithi181 bash[67587]: cluster 2024-09-13T00:39:06.824373+0000 mon.a (mon.0) 3576 : cluster [DBG] osdmap e1925: 8 total, 8 up, 8 in 2024-09-13T00:39:08.101 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~11509146,16777216~8287444,25165824~10410453,37748736~6173353,46526671~6678549,54525952~12884094,71303168~15711803,91915173~9897811,113921932~3493308,125829120~14397466,143604550~2930002,146800640~10450322,162728173~1277847,180355072~1219120,204447798~4981355,222298112~4931875,243269632~2282421,260046848~8639609,276824064~3240858,285212672~11932810,301411264~7201245,314572800~10704112,336574728~3824458,371886289~13078423,390070272~7505332,398458880~6482294,405696667~5790944] 2024-09-13T00:39:08.101 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to 7 diff [2612746~8896400,18624312~6440348,26995819~3290536,31904846~12017243,46526671~6678549,58515289~704557,61366356~6043690,71349028~11331915,83314700~3016720,86936101~78870,91915173~9897811,113921932~7303619,126152787~14073799,143604550~2930002,147450241~4531765,153773626~7600370,162728173~1277847,180807822~766370,204447798~1679405,208047485~1381668,212320259~6420604,222904244~4325743,227825355~4871993,244107705~1444348,260842262~7844195,276897564~6749766,289202665~7942817,301411264~7201245,317751762~7525150,339257207~1141979,343284705~1605783,373179324~11785388,387920372~9655232,401627021~3314153,405696667~5790944] 2024-09-13T00:39:08.101 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [2612746~8896400,18624312~6440348,26995819~3290536,31904846~12017243,46526671~6678549,58515289~704557,61366356~6043690,71349028~11331915,83314700~3016720,86936101~78870,91915173~9897811,113921932~7303619,126152787~14073799,143604550~2930002,147450241~4531765,153773626~7600370,162728173~1277847,180807822~766370,204447798~1679405,208047485~1381668,212320259~6420604,222904244~4325743,227825355~4871993,244107705~1444348,260842262~7844195,276897564~6749766,289202665~7942817,301411264~7201245,317751762~7525150,339257207~1141979,343284705~1605783,373179324~11785388,387920372~9655232,401627021~3314153,405696667~5790944] 2024-09-13T00:39:08.107 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~11509146,16777216~8287444,25165824~18756265,46526671~6678549,54525952~12884094,71303168~15711803,91915173~9897811,113921932~7303619,125829120~14397466,143604550~2930002,146800640~14573356,162728173~1277847,180355072~1219120,204447798~4981355,209715200~9025663,222298112~10399236,243269632~2282421,260046848~8639609,276824064~6823266,285212672~11932810,301411264~7201245,314572800~10704112,336574728~3824458,343284705~1605783,371886289~13078423,385875968~11699636,398458880~6482294,405696667~5790944] 2024-09-13T00:39:08.107 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to 8 diff [2612746~8896400,18624312~6440348,26995819~4642300,31904846~12017243,46526671~6678549,58515289~704557,61366356~6043690,71349028~14982392,86936101~78870,91915173~9897811,111245499~9980052,126152787~14073799,143604550~2930002,147450241~4531765,153773626~7600370,162728173~1277847,172123438~2775342,180155982~1465141,204447798~1679405,208047485~1381668,212320259~6420604,222904244~4325743,227825355~4871993,244107705~8129772,260842262~7844195,276897564~6749766,289202665~7942817,301411264~7201245,317751762~7525150,330730451~2400977,339257207~1141979,343284705~1605783,373179324~11785388,387920372~9655232,401627021~3314153,405696667~5790944] 2024-09-13T00:39:08.107 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [2612746~8896400,18624312~6440348,26995819~4642300,31904846~12017243,46526671~6678549,58515289~704557,61366356~6043690,71349028~14982392,86936101~78870,91915173~9897811,111245499~9980052,126152787~14073799,143604550~2930002,147450241~4531765,153773626~7600370,162728173~1277847,172123438~2775342,180155982~1465141,204447798~1679405,208047485~1381668,212320259~6420604,222904244~4325743,227825355~4871993,244107705~8129772,260842262~7844195,276897564~6749766,289202665~7942817,301411264~7201245,317751762~7525150,330730451~2400977,339257207~1141979,343284705~1605783,373179324~11785388,387920372~9655232,401627021~3314153,405696667~5790944] 2024-09-13T00:39:08.113 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~11509146,16777216~8287444,25165824~18756265,46526671~6678549,54525952~12884094,71303168~15711803,91915173~9897811,111245499~9980052,125829120~14397466,143604550~2930002,146800640~14573356,162728173~1277847,171966464~2932316,176160768~5460355,204447798~4981355,209715200~9025663,222298112~10399236,243269632~8967845,260046848~8639609,276824064~6823266,285212672~11932810,301411264~7201245,314572800~10704112,330730451~2400977,336574728~3824458,343284705~1605783,371886289~13078423,385875968~11699636,398458880~6482294,405696667~5790944] 2024-09-13T00:39:08.113 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to 9 diff [2612746~8896400,18624312~6440348,26995819~4642300,31904846~12017243,46526671~6678549,58515289~704557,61366356~6043690,71349028~15665943,91915173~9897811,111245499~9980052,126152787~14073799,143604550~2930002,147450241~4531765,153773626~10658329,169300267~5598513,180155982~4105980,204447798~1679405,206691828~1354340,208047485~1381668,212320259~6420604,220044632~7185355,227825355~4871993,244107705~8129772,260842262~7844195,273904834~1135501,276897564~6749766,284165576~12979906,301411264~7201245,317751762~7525150,330730451~2400977,339257207~1141979,343284705~1605783,373179324~11785388,387920372~9655232,401627021~3314153,405696667~5790944] 2024-09-13T00:39:08.113 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [2612746~8896400,18624312~6440348,26995819~4642300,31904846~12017243,46526671~6678549,58515289~704557,61366356~6043690,71349028~15665943,91915173~9897811,111245499~9980052,126152787~14073799,143604550~2930002,147450241~4531765,153773626~10658329,169300267~5598513,180155982~4105980,204447798~1679405,206691828~1354340,208047485~1381668,212320259~6420604,220044632~7185355,227825355~4871993,244107705~8129772,260842262~7844195,273904834~1135501,276897564~6749766,284165576~12979906,301411264~7201245,317751762~7525150,330730451~2400977,339257207~1141979,343284705~1605783,373179324~11785388,387920372~9655232,401627021~3314153,405696667~5790944] 2024-09-13T00:39:08.120 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~11509146,16777216~8287444,25165824~18756265,46526671~6678549,54525952~12884094,71303168~15711803,91915173~9897811,111245499~9980052,125829120~14397466,143604550~2930002,146800640~17631315,169300267~5598513,176160768~8101194,204447798~4981355,209715200~22982148,243269632~8967845,260046848~8639609,272629760~2410575,276824064~20321418,301411264~7201245,314572800~10704112,330730451~2400977,336574728~3824458,343284705~1605783,371886289~13078423,385875968~11699636,398458880~6482294,405696667~5790944] 2024-09-13T00:39:08.120 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to 10 diff [2612746~8896400,18624312~6440348,26995819~4642300,31904846~12017243,46526671~6678549,58515289~704557,61366356~25648615,91915173~9897811,111245499~9980052,126152787~14073799,143604550~2930002,147450241~4531765,153773626~10658329,169300267~5598513,180155982~4105980,204447798~1679405,206691828~1354340,208047485~1381668,212320259~6420604,220044632~12652716,244107705~8129772,260842262~7844195,273904834~1135501,276897564~6749766,284165576~12979906,301411264~7201245,317751762~7525150,330730451~2400977,339257207~1141979,343284705~1605783,348549356~6937075,365997237~1542191,373179324~11785388,387920372~9655232,401627021~3314153,405696667~5790944] 2024-09-13T00:39:08.120 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [2612746~8896400,18624312~6440348,26995819~4642300,31904846~12017243,46526671~6678549,58515289~704557,61366356~25648615,91915173~9897811,111245499~9980052,126152787~14073799,143604550~2930002,147450241~4531765,153773626~10658329,169300267~5598513,180155982~4105980,204447798~1679405,206691828~1354340,208047485~1381668,212320259~6420604,220044632~12652716,244107705~8129772,260842262~7844195,273904834~1135501,276897564~6749766,284165576~12979906,301411264~7201245,317751762~7525150,330730451~2400977,339257207~1141979,343284705~1605783,348549356~6937075,365997237~1542191,373179324~11785388,387920372~9655232,401627021~3314153,405696667~5790944] 2024-09-13T00:39:08.126 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~11509146,16777216~8287444,25165824~18756265,46526671~6678549,54525952~32489019,91915173~9897811,111245499~9980052,125829120~14397466,143604550~2930002,146800640~17631315,169300267~5598513,176160768~8101194,204447798~4981355,209715200~22982148,243269632~8967845,260046848~8639609,272629760~2410575,276824064~20321418,301411264~7201245,314572800~10704112,330730451~2400977,336574728~3824458,343284705~1605783,348549356~6937075,365997237~1542191,371886289~13078423,385875968~11699636,398458880~6482294,405696667~5790944] 2024-09-13T00:39:08.126 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to 11 diff [2612746~8896400,18624312~6440348,26995819~4642300,31904846~12017243,46526671~6678549,58515289~704557,61366356~25648615,91915173~9897811,105459787~1424206,111245499~9980052,122043306~18183280,143604550~2930002,147450241~4531765,153773626~10658329,169300267~5598513,180155982~4105980,204447798~1679405,206691828~1354340,208047485~1381668,212320259~6420604,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~21900567,301411264~7201245,317751762~10401988,330730451~2400977,335934186~1814726,339257207~1141979,343284705~1605783,348549356~6937075,365997237~1542191,373179324~11785388,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.126 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [2612746~8896400,18624312~6440348,26995819~4642300,31904846~12017243,46526671~6678549,58515289~704557,61366356~25648615,91915173~9897811,105459787~1424206,111245499~9980052,122043306~18183280,143604550~2930002,147450241~4531765,153773626~10658329,169300267~5598513,180155982~4105980,204447798~1679405,206691828~1354340,208047485~1381668,212320259~6420604,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~21900567,301411264~7201245,317751762~10401988,330730451~2400977,335934186~1814726,339257207~1141979,343284705~1605783,348549356~6937075,365997237~1542191,373179324~11785388,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.132 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~11509146,16777216~8287444,25165824~18756265,46526671~6678549,54525952~32489019,91915173~9897811,104857600~2026393,111245499~9980052,121634816~18591770,143604550~2930002,146800640~17631315,169300267~5598513,176160768~8101194,204447798~4981355,209715200~22982148,240828348~11409129,260046848~8639609,272629760~2410575,276824064~21974067,301411264~7201245,314572800~13580950,330730451~2400977,335934186~4465000,343284705~1605783,348549356~6937075,365997237~1542191,371886289~13078423,385875968~19065206,405696667~5790944] 2024-09-13T00:39:08.132 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to 12 diff [2612746~10642447,18624312~25297777,46526671~6678549,58515289~704557,61366356~25648615,91915173~10569175,105459787~1424206,111245499~9980052,122043306~18183280,143604550~2930002,147450241~4531765,153773626~10658329,169300267~5598513,180155982~4105980,204447798~1679405,206691828~1354340,208047485~1381668,212320259~6420604,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~21900567,301411264~7201245,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~1605783,348549356~6937075,365997237~5182427,373179324~13170580,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.132 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [2612746~10642447,18624312~25297777,46526671~6678549,58515289~704557,61366356~25648615,91915173~10569175,105459787~1424206,111245499~9980052,122043306~18183280,143604550~2930002,147450241~4531765,153773626~10658329,169300267~5598513,180155982~4105980,204447798~1679405,206691828~1354340,208047485~1381668,212320259~6420604,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~21900567,301411264~7201245,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~1605783,348549356~6937075,365997237~5182427,373179324~13170580,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.139 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13255193,16777216~27144873,46526671~6678549,54525952~32489019,91915173~10569175,104857600~2026393,111245499~9980052,121634816~18591770,143604550~2930002,146800640~17631315,169300267~5598513,176160768~8101194,204447798~4981355,209715200~22982148,240828348~11409129,260046848~8639609,272629760~2410575,276824064~21974067,301411264~7201245,310378496~24935507,335934186~4465000,343284705~1605783,348549356~6937075,365997237~5182427,371886289~33054885,405696667~5790944] 2024-09-13T00:39:08.139 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to 13 diff [2612746~10642447,17113294~26808795,46526671~6678549,58515289~704557,61366356~25648615,91915173~10569175,105459787~1424206,111245499~9980052,122043306~18183280,143604550~2930002,147450241~4531765,153773626~12869043,169300267~5598513,180155982~4105980,187369573~871966,201556540~2015504,204447798~1679405,206691828~1354340,208047485~1381668,209974378~1903654,212320259~6420604,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~23715912,301411264~7201245,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~1605783,348549356~6937075,365997237~5182427,373179324~13170580,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.139 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [2612746~10642447,17113294~26808795,46526671~6678549,58515289~704557,61366356~25648615,91915173~10569175,105459787~1424206,111245499~9980052,122043306~18183280,143604550~2930002,147450241~4531765,153773626~12869043,169300267~5598513,180155982~4105980,187369573~871966,201556540~2015504,204447798~1679405,206691828~1354340,208047485~1381668,209974378~1903654,212320259~6420604,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~23715912,301411264~7201245,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~1605783,348549356~6937075,365997237~5182427,373179324~13170580,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.145 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13255193,16777216~27144873,46526671~6678549,54525952~32489019,91915173~10569175,104857600~2026393,111245499~9980052,121634816~18591770,143604550~2930002,146800640~19842029,169300267~5598513,176160768~8101194,184549376~3692163,201556540~2015504,204447798~4981355,209715200~22982148,240828348~11409129,260046848~8639609,272629760~2410575,276824064~23789412,301411264~7201245,310378496~24935507,335934186~4465000,343284705~1605783,348549356~6937075,365997237~5182427,371886289~33054885,405696667~5790944] 2024-09-13T00:39:08.145 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to 14 diff [978366~12276827,17113294~26808795,46526671~6678549,58515289~704557,61366356~25648615,91915173~10569175,105459787~1424206,111245499~9980052,122043306~18183280,143604550~2930002,147450241~4531765,153773626~12869043,169300267~5598513,180155982~4105980,187369573~871966,201556540~2015504,204447798~1679405,206691828~12049035,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~24043645,301411264~7201245,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~1605783,348549356~6937075,365490113~5689551,371762046~14587858,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.146 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~26808795,46526671~6678549,58515289~704557,61366356~25648615,91915173~10569175,105459787~1424206,111245499~9980052,122043306~18183280,143604550~2930002,147450241~4531765,153773626~12869043,169300267~5598513,180155982~4105980,187369573~871966,201556540~2015504,204447798~1679405,206691828~12049035,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~24043645,301411264~7201245,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~1605783,348549356~6937075,365490113~5689551,371762046~14587858,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.152 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13255193,16777216~27144873,46526671~6678549,54525952~32489019,91915173~10569175,104857600~2026393,111245499~9980052,121634816~18591770,143604550~2930002,146800640~19842029,169300267~5598513,176160768~8101194,184549376~3692163,201556540~2015504,204447798~28249550,240828348~11409129,260046848~8639609,272629760~2410575,276824064~24117145,301411264~7201245,310378496~24935507,335934186~4465000,343284705~1605783,348549356~6937075,365490113~5689551,371762046~33179128,405696667~5790944] 2024-09-13T00:39:08.152 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to 15 diff [978366~12276827,17113294~26808795,46526671~6678549,58515289~704557,61366356~25648615,91915173~10569175,105459787~1424206,111245499~9980052,122043306~18183280,143604550~3213537,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~2015504,204447798~1679405,206691828~12049035,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~24043645,301411264~7201245,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~1605783,348299711~7186720,365490113~5689551,371762046~14587858,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.153 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~26808795,46526671~6678549,58515289~704557,61366356~25648615,91915173~10569175,105459787~1424206,111245499~9980052,122043306~18183280,143604550~3213537,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~2015504,204447798~1679405,206691828~12049035,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~24043645,301411264~7201245,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~1605783,348299711~7186720,365490113~5689551,371762046~14587858,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.158 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13255193,16777216~27144873,46526671~6678549,54525952~32489019,91915173~10569175,104857600~2026393,111245499~9980052,121634816~18591770,143604550~23038119,169300267~14961695,184549376~3692163,188743680~10830113,201556540~2015504,204447798~28249550,240828348~11409129,260046848~8639609,272629760~2410575,276824064~24117145,301411264~7201245,310378496~24935507,335934186~4465000,343284705~1605783,348299711~7186720,365490113~5689551,371762046~33179128,405696667~5790944] 2024-09-13T00:39:08.158 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to 16 diff [978366~12276827,17113294~26808795,46526671~9958787,58515289~704557,61366356~25648615,91915173~10569175,105459787~1424206,111245499~9980052,122043306~18183280,143604550~3213537,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~2015504,204447798~1679405,206691828~12049035,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~24043645,301411264~7201245,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~12201726,365490113~5689551,371762046~14587858,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.158 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~26808795,46526671~9958787,58515289~704557,61366356~25648615,91915173~10569175,105459787~1424206,111245499~9980052,122043306~18183280,143604550~3213537,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~2015504,204447798~1679405,206691828~12049035,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~24043645,301411264~7201245,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~12201726,365490113~5689551,371762046~14587858,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.165 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13255193,16777216~27144873,46526671~40488300,91915173~10569175,104857600~2026393,111245499~9980052,121634816~18591770,143604550~23038119,169300267~14961695,184549376~3692163,188743680~10830113,201556540~2015504,204447798~28249550,240828348~11409129,260046848~8639609,272629760~2410575,276824064~24117145,301411264~7201245,310378496~24935507,335934186~4465000,343284705~12201726,365490113~5689551,371762046~33179128,405696667~5790944] 2024-09-13T00:39:08.165 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to 17 diff [978366~12276827,17113294~26808795,46526671~9958787,58515289~28499682,91915173~10569175,105459787~1424206,111245499~9980052,122043306~18183280,143604550~3213537,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273904834~1135501,276897564~24043645,301411264~7201245,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~12201726,365490113~5689551,371762046~14587858,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.165 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~26808795,46526671~9958787,58515289~28499682,91915173~10569175,105459787~1424206,111245499~9980052,122043306~18183280,143604550~3213537,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273904834~1135501,276897564~24043645,301411264~7201245,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~12201726,365490113~5689551,371762046~14587858,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.171 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13255193,16777216~27144873,46526671~40488300,91915173~10569175,104857600~2026393,111245499~9980052,121634816~18591770,143604550~23038119,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~2410575,276824064~24117145,301411264~7201245,310378496~24935507,335934186~4465000,343284705~12201726,365490113~5689551,371762046~33179128,405696667~5790944] 2024-09-13T00:39:08.171 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to 18 diff [978366~12276827,17113294~26808795,46526671~9958787,58515289~28499682,91915173~10569175,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273904834~1135501,276897564~24043645,301411264~7201245,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~12201726,365490113~20859791,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.171 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~26808795,46526671~9958787,58515289~28499682,91915173~10569175,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273904834~1135501,276897564~24043645,301411264~7201245,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~12201726,365490113~20859791,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.177 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13255193,16777216~27144873,46526671~40488300,91915173~10569175,104857600~2026393,111245499~9980052,121634816~18591770,142230204~1250407,143604550~23038119,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~2410575,276824064~24117145,301411264~7201245,310378496~24935507,335934186~4465000,343284705~12201726,365490113~39451061,405696667~5790944] 2024-09-13T00:39:08.177 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to 19 diff [978366~12404130,17113294~26808795,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.178 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~26808795,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.184 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,16777216~27144873,46526671~40488300,91032793~11451555,104857600~2026393,111245499~9980052,121634816~18591770,142230204~1250407,143604550~23038119,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~35982749,310378496~24935507,335934186~4465000,343284705~17265138,360710144~44231030,405696667~5790944] 2024-09-13T00:39:08.184 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to 3 diff [38721951~5200138,71349028~11331915,86936101~78870,91915173~8177010,129165667~1581292,222904244~4325743,317751762~7525150,373179324~6803307,401627021~3314153] 2024-09-13T00:39:08.184 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [38721951~5200138,71349028~11331915,86936101~78870,91915173~8177010,129165667~1581292,222904244~4325743,317751762~7525150,373179324~6803307,401627021~3314153] 2024-09-13T00:39:08.191 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [37748736~6173353,71303168~11377775,86331420~683551,91915173~8177010,129165667~1581292,222298112~4931875,314572800~10704112,371886289~8096342,398458880~6482294] 2024-09-13T00:39:08.191 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to 4 diff [38721951~5200138,46526671~6678549,58515289~704557,61366356~6043690,71349028~11331915,86936101~78870,91915173~8177010,129165667~7088536,153773626~3477336,162728173~1277847,222904244~4325743,289226804~7918678,317751762~7525150,373179324~6803307,401627021~3314153,405696667~5790944] 2024-09-13T00:39:08.191 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [38721951~5200138,46526671~6678549,58515289~704557,61366356~6043690,71349028~11331915,86936101~78870,91915173~8177010,129165667~7088536,153773626~3477336,162728173~1277847,222904244~4325743,289226804~7918678,317751762~7525150,373179324~6803307,401627021~3314153,405696667~5790944] 2024-09-13T00:39:08.197 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [37748736~6173353,46526671~6678549,54525952~12884094,71303168~11377775,86331420~683551,91915173~8177010,129165667~7088536,150994944~6256018,162728173~1277847,222298112~4931875,285212672~11932810,314572800~10704112,371886289~8096342,398458880~6482294,405696667~5790944] 2024-09-13T00:39:08.197 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to 5 diff [26995819~3290536,38721951~5200138,46526671~6678549,58515289~704557,61366356~6043690,71349028~11331915,86936101~78870,91915173~8177010,129165667~7088536,147450241~4531765,153773626~3477336,162728173~1277847,180807822~766370,208047485~1381668,222904244~4325743,279969777~95145,289226804~7918678,317751762~7525150,339257207~1141979,373179324~8736012,390387020~7188584,401627021~3314153,405696667~5790944] 2024-09-13T00:39:08.197 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [26995819~3290536,38721951~5200138,46526671~6678549,58515289~704557,61366356~6043690,71349028~11331915,86936101~78870,91915173~8177010,129165667~7088536,147450241~4531765,153773626~3477336,162728173~1277847,180807822~766370,208047485~1381668,222904244~4325743,279969777~95145,289226804~7918678,317751762~7525150,339257207~1141979,373179324~8736012,390387020~7188584,401627021~3314153,405696667~5790944] 2024-09-13T00:39:08.203 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [25165824~5120531,37748736~6173353,46526671~6678549,54525952~12884094,71303168~11377775,86331420~683551,91915173~8177010,129165667~7088536,146800640~10450322,162728173~1277847,180355072~1219120,206127203~3301950,222298112~4931875,276824064~3240858,285212672~11932810,314572800~10704112,336574728~3824458,371886289~10029047,390070272~7505332,398458880~6482294,405696667~5790944] 2024-09-13T00:39:08.203 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to 6 diff [2612746~3549385,26995819~3290536,32429128~3147149,38721951~5200138,46526671~6678549,58515289~704557,61366356~6043690,71349028~11331915,86936101~78870,91915173~8177010,129165667~7088536,147450241~4531765,153773626~3477336,162728173~1277847,180807822~766370,208047485~1381668,222904244~4325743,244107705~1444348,260842262~7844195,279969777~95145,289202665~7942817,301411264~7201245,317751762~7525150,339257207~1141979,373179324~8736012,390387020~7188584,401627021~3314153,405696667~5790944] 2024-09-13T00:39:08.203 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [2612746~3549385,26995819~3290536,32429128~3147149,38721951~5200138,46526671~6678549,58515289~704557,61366356~6043690,71349028~11331915,86936101~78870,91915173~8177010,129165667~7088536,147450241~4531765,153773626~3477336,162728173~1277847,180807822~766370,208047485~1381668,222904244~4325743,244107705~1444348,260842262~7844195,279969777~95145,289202665~7942817,301411264~7201245,317751762~7525150,339257207~1141979,373179324~8736012,390387020~7188584,401627021~3314153,405696667~5790944] 2024-09-13T00:39:08.210 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~6162131,25165824~10410453,37748736~6173353,46526671~6678549,54525952~12884094,71303168~11377775,86331420~683551,91915173~8177010,129165667~7088536,146800640~10450322,162728173~1277847,180355072~1219120,206127203~3301950,222298112~4931875,243269632~2282421,260046848~8639609,276824064~3240858,285212672~11932810,301411264~7201245,314572800~10704112,336574728~3824458,371886289~10029047,390070272~7505332,398458880~6482294,405696667~5790944] 2024-09-13T00:39:08.210 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to 7 diff [2612746~3549385,26995819~3290536,31904846~12017243,46526671~6678549,58515289~704557,61366356~6043690,71349028~11331915,86936101~78870,91915173~8177010,113977873~7247678,129165667~7088536,147450241~4531765,153773626~7600370,162728173~1277847,180807822~766370,208047485~1381668,212320259~6420604,222904244~4325743,227825355~4871993,244107705~1444348,260842262~7844195,276897564~6749766,289202665~7942817,301411264~7201245,317751762~7525150,339257207~1141979,343284705~1605783,373179324~8736012,387920372~9655232,401627021~3314153,405696667~5790944] 2024-09-13T00:39:08.211 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [2612746~3549385,26995819~3290536,31904846~12017243,46526671~6678549,58515289~704557,61366356~6043690,71349028~11331915,86936101~78870,91915173~8177010,113977873~7247678,129165667~7088536,147450241~4531765,153773626~7600370,162728173~1277847,180807822~766370,208047485~1381668,212320259~6420604,222904244~4325743,227825355~4871993,244107705~1444348,260842262~7844195,276897564~6749766,289202665~7942817,301411264~7201245,317751762~7525150,339257207~1141979,343284705~1605783,373179324~8736012,387920372~9655232,401627021~3314153,405696667~5790944] 2024-09-13T00:39:08.217 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~6162131,25165824~18756265,46526671~6678549,54525952~12884094,71303168~11377775,86331420~683551,91915173~8177010,113977873~7247678,129165667~7088536,146800640~14573356,162728173~1277847,180355072~1219120,206127203~3301950,209715200~9025663,222298112~10399236,243269632~2282421,260046848~8639609,276824064~6823266,285212672~11932810,301411264~7201245,314572800~10704112,336574728~3824458,343284705~1605783,371886289~10029047,385875968~11699636,398458880~6482294,405696667~5790944] 2024-09-13T00:39:08.217 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to 8 diff [2612746~3549385,26995819~4642300,31904846~12017243,46526671~6678549,58515289~704557,61366356~6043690,71349028~14656834,86936101~78870,91915173~8177010,111245499~9980052,129165667~7088536,147450241~4531765,153773626~7600370,162728173~1277847,172123438~2775342,180155982~1465141,208047485~1381668,212320259~6420604,222904244~4325743,227825355~4871993,244107705~8129772,260842262~7844195,276897564~6749766,289202665~7942817,301411264~7201245,317751762~7525150,330730451~2400977,339257207~1141979,343284705~1605783,373179324~8736012,387920372~9655232,401627021~3314153,405696667~5790944] 2024-09-13T00:39:08.217 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [2612746~3549385,26995819~4642300,31904846~12017243,46526671~6678549,58515289~704557,61366356~6043690,71349028~14656834,86936101~78870,91915173~8177010,111245499~9980052,129165667~7088536,147450241~4531765,153773626~7600370,162728173~1277847,172123438~2775342,180155982~1465141,208047485~1381668,212320259~6420604,222904244~4325743,227825355~4871993,244107705~8129772,260842262~7844195,276897564~6749766,289202665~7942817,301411264~7201245,317751762~7525150,330730451~2400977,339257207~1141979,343284705~1605783,373179324~8736012,387920372~9655232,401627021~3314153,405696667~5790944] 2024-09-13T00:39:08.223 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~6162131,25165824~18756265,46526671~6678549,54525952~12884094,71303168~14702694,86331420~683551,91915173~8177010,111245499~9980052,129165667~7088536,146800640~14573356,162728173~1277847,171966464~2932316,176160768~5460355,206127203~3301950,209715200~9025663,222298112~10399236,243269632~8967845,260046848~8639609,276824064~6823266,285212672~11932810,301411264~7201245,314572800~10704112,330730451~2400977,336574728~3824458,343284705~1605783,371886289~10029047,385875968~11699636,398458880~6482294,405696667~5790944] 2024-09-13T00:39:08.223 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to 9 diff [2612746~3549385,26995819~4642300,31904846~12017243,46526671~6678549,58515289~704557,61366356~6043690,71349028~15665943,91915173~8177010,111245499~9980052,129165667~7088536,147450241~4531765,153773626~10658329,169300267~5598513,180155982~4105980,206691828~1354340,208047485~1381668,212320259~6420604,220044632~7185355,227825355~4871993,244107705~8129772,260842262~7844195,273904834~1135501,276897564~6749766,284165576~12979906,301411264~7201245,317751762~7525150,330730451~2400977,339257207~1141979,343284705~1605783,373179324~8736012,387920372~9655232,401627021~3314153,405696667~5790944] 2024-09-13T00:39:08.223 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [2612746~3549385,26995819~4642300,31904846~12017243,46526671~6678549,58515289~704557,61366356~6043690,71349028~15665943,91915173~8177010,111245499~9980052,129165667~7088536,147450241~4531765,153773626~10658329,169300267~5598513,180155982~4105980,206691828~1354340,208047485~1381668,212320259~6420604,220044632~7185355,227825355~4871993,244107705~8129772,260842262~7844195,273904834~1135501,276897564~6749766,284165576~12979906,301411264~7201245,317751762~7525150,330730451~2400977,339257207~1141979,343284705~1605783,373179324~8736012,387920372~9655232,401627021~3314153,405696667~5790944] 2024-09-13T00:39:08.229 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~6162131,25165824~18756265,46526671~6678549,54525952~12884094,71303168~15711803,91915173~8177010,111245499~9980052,129165667~7088536,146800640~17631315,169300267~5598513,176160768~8101194,206127203~3301950,209715200~22982148,243269632~8967845,260046848~8639609,272629760~2410575,276824064~20321418,301411264~7201245,314572800~10704112,330730451~2400977,336574728~3824458,343284705~1605783,371886289~10029047,385875968~11699636,398458880~6482294,405696667~5790944] 2024-09-13T00:39:08.230 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to 10 diff [2612746~3549385,22666579~2303167,26995819~4642300,31904846~12017243,46526671~6678549,58515289~704557,61366356~25648615,91915173~8177010,111245499~9980052,128451845~308073,129165667~7088536,147450241~4531765,153773626~10658329,169300267~5598513,180155982~4105980,206691828~1354340,208047485~1381668,212320259~6420604,220044632~12652716,244107705~8129772,260842262~7844195,273904834~1135501,276897564~6749766,284165576~12979906,301411264~7201245,317751762~7525150,330730451~2400977,339257207~1141979,343284705~1605783,348549356~6937075,365997237~1542191,373179324~8736012,387920372~9655232,401627021~3314153,405696667~5790944] 2024-09-13T00:39:08.230 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [2612746~3549385,22666579~2303167,26995819~4642300,31904846~12017243,46526671~6678549,58515289~704557,61366356~25648615,91915173~8177010,111245499~9980052,128451845~308073,129165667~7088536,147450241~4531765,153773626~10658329,169300267~5598513,180155982~4105980,206691828~1354340,208047485~1381668,212320259~6420604,220044632~12652716,244107705~8129772,260842262~7844195,273904834~1135501,276897564~6749766,284165576~12979906,301411264~7201245,317751762~7525150,330730451~2400977,339257207~1141979,343284705~1605783,348549356~6937075,365997237~1542191,373179324~8736012,387920372~9655232,401627021~3314153,405696667~5790944] 2024-09-13T00:39:08.236 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~6162131,22666579~2303167,25165824~18756265,46526671~6678549,54525952~32489019,91915173~8177010,111245499~9980052,128451845~308073,129165667~7088536,146800640~17631315,169300267~5598513,176160768~8101194,206127203~3301950,209715200~22982148,243269632~8967845,260046848~8639609,272629760~2410575,276824064~20321418,301411264~7201245,314572800~10704112,330730451~2400977,336574728~3824458,343284705~1605783,348549356~6937075,365997237~1542191,371886289~10029047,385875968~11699636,398458880~6482294,405696667~5790944] 2024-09-13T00:39:08.236 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to 11 diff [2612746~6062675,22666579~2303167,26995819~4642300,31904846~12017243,46526671~6678549,58515289~704557,61366356~25648615,91915173~8177010,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,129165667~7088536,147450241~4531765,153773626~10658329,169300267~5598513,180155982~4105980,206691828~1354340,208047485~1381668,212320259~6420604,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~21900567,301411264~7201245,317751762~10401988,330730451~2400977,335934186~1814726,339257207~1141979,343284705~1605783,348549356~6937075,365997237~1542191,373179324~8736012,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.237 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [2612746~6062675,22666579~2303167,26995819~4642300,31904846~12017243,46526671~6678549,58515289~704557,61366356~25648615,91915173~8177010,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,129165667~7088536,147450241~4531765,153773626~10658329,169300267~5598513,180155982~4105980,206691828~1354340,208047485~1381668,212320259~6420604,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~21900567,301411264~7201245,317751762~10401988,330730451~2400977,335934186~1814726,339257207~1141979,343284705~1605783,348549356~6937075,365997237~1542191,373179324~8736012,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.242 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~8675421,22666579~2303167,25165824~18756265,46526671~6678549,54525952~32489019,91915173~8177010,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,129165667~7088536,146800640~17631315,169300267~5598513,176160768~8101194,206127203~3301950,209715200~22982148,240828348~11409129,260046848~8639609,272629760~2410575,276824064~21974067,301411264~7201245,314572800~13580950,330730451~2400977,335934186~4465000,343284705~1605783,348549356~6937075,365997237~1542191,371886289~10029047,385875968~19065206,405696667~5790944] 2024-09-13T00:39:08.243 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to 12 diff [2612746~10642447,22666579~21255510,46526671~6678549,58515289~704557,61366356~25648615,91915173~10569175,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,129165667~7088536,147450241~4531765,153773626~10658329,169300267~5598513,180155982~4105980,206691828~1354340,208047485~1381668,212320259~6420604,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~21900567,301411264~7201245,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~1605783,348549356~6937075,365997237~5182427,373179324~13170580,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.243 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [2612746~10642447,22666579~21255510,46526671~6678549,58515289~704557,61366356~25648615,91915173~10569175,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,129165667~7088536,147450241~4531765,153773626~10658329,169300267~5598513,180155982~4105980,206691828~1354340,208047485~1381668,212320259~6420604,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~21900567,301411264~7201245,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~1605783,348549356~6937075,365997237~5182427,373179324~13170580,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.249 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13255193,22666579~21255510,46526671~6678549,54525952~32489019,91915173~10569175,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,129165667~7088536,146800640~17631315,169300267~5598513,176160768~8101194,206127203~3301950,209715200~22982148,240828348~11409129,260046848~8639609,272629760~2410575,276824064~21974067,301411264~7201245,310378496~24935507,335934186~4465000,343284705~1605783,348549356~6937075,365997237~5182427,371886289~33054885,405696667~5790944] 2024-09-13T00:39:08.249 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to 13 diff [2612746~10642447,17113294~3323835,22666579~21255510,46526671~6678549,58515289~704557,61366356~25648615,91915173~10569175,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,129165667~7088536,147450241~4531765,153773626~12869043,169300267~5598513,180155982~4105980,187369573~871966,201556540~2015504,206691828~1354340,208047485~1381668,209974378~1903654,212320259~6420604,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~23715912,301411264~7201245,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~1605783,348549356~6937075,365997237~5182427,373179324~13170580,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.249 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [2612746~10642447,17113294~3323835,22666579~21255510,46526671~6678549,58515289~704557,61366356~25648615,91915173~10569175,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,129165667~7088536,147450241~4531765,153773626~12869043,169300267~5598513,180155982~4105980,187369573~871966,201556540~2015504,206691828~1354340,208047485~1381668,209974378~1903654,212320259~6420604,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~23715912,301411264~7201245,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~1605783,348549356~6937075,365997237~5182427,373179324~13170580,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.255 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13255193,17113294~3323835,22666579~21255510,46526671~6678549,54525952~32489019,91915173~10569175,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,129165667~7088536,146800640~19842029,169300267~5598513,176160768~8101194,184549376~3692163,201556540~2015504,206127203~3301950,209715200~22982148,240828348~11409129,260046848~8639609,272629760~2410575,276824064~23789412,301411264~7201245,310378496~24935507,335934186~4465000,343284705~1605783,348549356~6937075,365997237~5182427,371886289~33054885,405696667~5790944] 2024-09-13T00:39:08.255 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to 14 diff [978366~12276827,17113294~3323835,22666579~21255510,46526671~6678549,58515289~704557,61366356~25648615,91915173~10569175,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,129165667~7088536,147450241~4531765,153773626~12869043,169300267~5598513,180155982~4105980,187369573~871966,201556540~2015504,206691828~12049035,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~24043645,301411264~7201245,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~1605783,348549356~6937075,365490113~5689551,371762046~14587858,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.255 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~3323835,22666579~21255510,46526671~6678549,58515289~704557,61366356~25648615,91915173~10569175,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,129165667~7088536,147450241~4531765,153773626~12869043,169300267~5598513,180155982~4105980,187369573~871966,201556540~2015504,206691828~12049035,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~24043645,301411264~7201245,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~1605783,348549356~6937075,365490113~5689551,371762046~14587858,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:07 smithi136 bash[75164]: cluster 2024-09-13T00:39:06.824373+0000 mon.a (mon.0) 3576 : cluster [DBG] osdmap e1925: 8 total, 8 up, 8 in 2024-09-13T00:39:08.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:07 smithi136 bash[76371]: cluster 2024-09-13T00:39:06.824373+0000 mon.a (mon.0) 3576 : cluster [DBG] osdmap e1925: 8 total, 8 up, 8 in 2024-09-13T00:39:08.262 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13255193,17113294~3323835,22666579~21255510,46526671~6678549,54525952~32489019,91915173~10569175,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,129165667~7088536,146800640~19842029,169300267~5598513,176160768~8101194,184549376~3692163,201556540~2015504,206127203~26570145,240828348~11409129,260046848~8639609,272629760~2410575,276824064~24117145,301411264~7201245,310378496~24935507,335934186~4465000,343284705~1605783,348549356~6937075,365490113~5689551,371762046~33179128,405696667~5790944] 2024-09-13T00:39:08.262 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to 15 diff [978366~12276827,17113294~3323835,22666579~21255510,46526671~6678549,58515289~704557,61366356~25648615,91915173~10569175,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,129165667~7088536,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~2015504,206691828~12049035,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~24043645,301411264~7201245,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~1605783,348299711~7186720,365490113~5689551,371762046~14587858,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.262 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~3323835,22666579~21255510,46526671~6678549,58515289~704557,61366356~25648615,91915173~10569175,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,129165667~7088536,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~2015504,206691828~12049035,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~24043645,301411264~7201245,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~1605783,348299711~7186720,365490113~5689551,371762046~14587858,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.268 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13255193,17113294~3323835,22666579~21255510,46526671~6678549,54525952~32489019,91915173~10569175,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,129165667~7088536,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~2015504,206127203~26570145,240828348~11409129,260046848~8639609,272629760~2410575,276824064~24117145,301411264~7201245,310378496~24935507,335934186~4465000,343284705~1605783,348299711~7186720,365490113~5689551,371762046~33179128,405696667~5790944] 2024-09-13T00:39:08.269 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to 16 diff [978366~12276827,17113294~3323835,22666579~21255510,46526671~9958787,58515289~704557,61366356~25648615,91915173~10569175,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,129165667~8116138,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~2015504,206691828~12049035,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~24043645,301411264~7201245,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~12201726,365490113~5689551,371762046~14587858,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.269 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~3323835,22666579~21255510,46526671~9958787,58515289~704557,61366356~25648615,91915173~10569175,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,129165667~8116138,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~2015504,206691828~12049035,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~24043645,301411264~7201245,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~12201726,365490113~5689551,371762046~14587858,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.275 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13255193,17113294~3323835,22666579~21255510,46526671~40488300,91915173~10569175,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,129165667~8116138,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~2015504,206127203~26570145,240828348~11409129,260046848~8639609,272629760~2410575,276824064~24117145,301411264~7201245,310378496~24935507,335934186~4465000,343284705~12201726,365490113~5689551,371762046~33179128,405696667~5790944] 2024-09-13T00:39:08.275 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to 17 diff [978366~12276827,17113294~3323835,22666579~21255510,46526671~9958787,58515289~28499682,91915173~10569175,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,129165667~8116138,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273904834~1135501,276897564~24043645,301411264~7201245,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~12201726,365490113~5689551,371762046~14587858,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.275 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~3323835,22666579~21255510,46526671~9958787,58515289~28499682,91915173~10569175,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,129165667~8116138,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273904834~1135501,276897564~24043645,301411264~7201245,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~12201726,365490113~5689551,371762046~14587858,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.281 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13255193,17113294~3323835,22666579~21255510,46526671~40488300,91915173~10569175,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,129165667~8116138,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~2410575,276824064~24117145,301411264~7201245,310378496~24935507,335934186~4465000,343284705~12201726,365490113~5689551,371762046~33179128,405696667~5790944] 2024-09-13T00:39:08.281 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to 18 diff [978366~12276827,17113294~3323835,22666579~21255510,46526671~9958787,58515289~28499682,91915173~10569175,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,129165667~8116138,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273904834~1135501,276897564~24043645,301411264~7201245,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~12201726,365490113~20859791,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.282 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~3323835,22666579~21255510,46526671~9958787,58515289~28499682,91915173~10569175,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,129165667~8116138,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273904834~1135501,276897564~24043645,301411264~7201245,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~12201726,365490113~20859791,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.288 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13255193,17113294~3323835,22666579~21255510,46526671~40488300,91915173~10569175,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,129165667~8116138,142230204~1250407,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~2410575,276824064~24117145,301411264~7201245,310378496~24935507,335934186~4465000,343284705~12201726,365490113~39451061,405696667~5790944] 2024-09-13T00:39:08.288 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to 19 diff [978366~12404130,17113294~3323835,22666579~21255510,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.288 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~21255510,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:08.295 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~21255510,46526671~40488300,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~35982749,310378496~24935507,335934186~4465000,343284705~17265138,360710144~44231030,405696667~5790944] 2024-09-13T00:39:08.295 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to 4 diff [46526671~6678549,58515289~704557,61366356~6043690,130605682~5648521,153773626~3477336,162728173~1277847,289226804~7918678,405696667~5790944] 2024-09-13T00:39:08.295 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [46526671~6678549,58515289~704557,61366356~6043690,130605682~5648521,153773626~3477336,162728173~1277847,289226804~7918678,405696667~5790944] 2024-09-13T00:39:08.301 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [46526671~6678549,54525952~12884094,130605682~5648521,150994944~6256018,162728173~1277847,285212672~11932810,405696667~5790944] 2024-09-13T00:39:08.302 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to 5 diff [26995819~3290536,46526671~6678549,58515289~704557,61366356~6043690,130605682~5648521,147450241~4531765,153773626~3477336,162728173~1277847,180807822~766370,208047485~1381668,279969777~95145,289226804~7918678,339257207~1141979,373635351~8279985,390387020~7188584,405696667~5790944] 2024-09-13T00:39:08.302 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [26995819~3290536,46526671~6678549,58515289~704557,61366356~6043690,130605682~5648521,147450241~4531765,153773626~3477336,162728173~1277847,180807822~766370,208047485~1381668,279969777~95145,289226804~7918678,339257207~1141979,373635351~8279985,390387020~7188584,405696667~5790944] 2024-09-13T00:39:08.308 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [25165824~5120531,46526671~6678549,54525952~12884094,130605682~5648521,146800640~10450322,162728173~1277847,180355072~1219120,206127203~3301950,276824064~3240858,285212672~11932810,336574728~3824458,373635351~8279985,390070272~7505332,405696667~5790944] 2024-09-13T00:39:08.308 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to 6 diff [2612746~3549385,26995819~3290536,32429128~3147149,46526671~6678549,58515289~704557,61366356~6043690,92243442~6230977,130605682~5648521,147450241~4531765,153773626~3477336,162728173~1277847,180807822~766370,208047485~1381668,244107705~1444348,260842262~7844195,279969777~95145,289202665~7942817,301411264~7201245,339257207~1141979,373635351~8279985,390387020~7188584,405696667~5790944] 2024-09-13T00:39:08.308 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [2612746~3549385,26995819~3290536,32429128~3147149,46526671~6678549,58515289~704557,61366356~6043690,92243442~6230977,130605682~5648521,147450241~4531765,153773626~3477336,162728173~1277847,180807822~766370,208047485~1381668,244107705~1444348,260842262~7844195,279969777~95145,289202665~7942817,301411264~7201245,339257207~1141979,373635351~8279985,390387020~7188584,405696667~5790944] 2024-09-13T00:39:08.314 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~6162131,25165824~10410453,46526671~6678549,54525952~12884094,92243442~6230977,130605682~5648521,146800640~10450322,162728173~1277847,180355072~1219120,206127203~3301950,243269632~2282421,260046848~8639609,276824064~3240858,285212672~11932810,301411264~7201245,336574728~3824458,373635351~8279985,390070272~7505332,405696667~5790944] 2024-09-13T00:39:08.314 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to 7 diff [2612746~3549385,26995819~3290536,31904846~7749369,46526671~6678549,58515289~704557,61366356~6043690,92243442~6230977,113977873~7247678,130605682~5648521,147450241~4531765,153773626~7600370,162728173~1277847,180807822~766370,208047485~1381668,212320259~6420604,227825355~4871993,244107705~1444348,260842262~7844195,276897564~6749766,289202665~7942817,301411264~7201245,339257207~1141979,343284705~1605783,373635351~8279985,387920372~9655232,405696667~5790944] 2024-09-13T00:39:08.314 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [2612746~3549385,26995819~3290536,31904846~7749369,46526671~6678549,58515289~704557,61366356~6043690,92243442~6230977,113977873~7247678,130605682~5648521,147450241~4531765,153773626~7600370,162728173~1277847,180807822~766370,208047485~1381668,212320259~6420604,227825355~4871993,244107705~1444348,260842262~7844195,276897564~6749766,289202665~7942817,301411264~7201245,339257207~1141979,343284705~1605783,373635351~8279985,387920372~9655232,405696667~5790944] 2024-09-13T00:39:08.320 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~6162131,25165824~14488391,46526671~6678549,54525952~12884094,92243442~6230977,113977873~7247678,130605682~5648521,146800640~14573356,162728173~1277847,180355072~1219120,206127203~3301950,209715200~9025663,227229987~5467361,243269632~2282421,260046848~8639609,276824064~6823266,285212672~11932810,301411264~7201245,336574728~3824458,343284705~1605783,373635351~8279985,385875968~11699636,405696667~5790944] 2024-09-13T00:39:08.320 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to 8 diff [2612746~3549385,26995819~4642300,31904846~7749369,46526671~6678549,58515289~704557,61366356~6043690,81806467~4199395,92243442~6230977,111245499~9980052,130605682~5648521,147450241~4531765,153773626~7600370,162728173~1277847,172123438~2775342,180155982~1465141,208047485~1381668,212320259~6420604,227825355~4871993,244107705~8129772,260842262~7844195,276897564~6749766,289202665~7942817,301411264~7201245,322459502~1891157,330730451~2400977,339257207~1141979,343284705~1605783,373635351~8279985,387920372~9655232,405696667~5790944] 2024-09-13T00:39:08.320 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [2612746~3549385,26995819~4642300,31904846~7749369,46526671~6678549,58515289~704557,61366356~6043690,81806467~4199395,92243442~6230977,111245499~9980052,130605682~5648521,147450241~4531765,153773626~7600370,162728173~1277847,172123438~2775342,180155982~1465141,208047485~1381668,212320259~6420604,227825355~4871993,244107705~8129772,260842262~7844195,276897564~6749766,289202665~7942817,301411264~7201245,322459502~1891157,330730451~2400977,339257207~1141979,343284705~1605783,373635351~8279985,387920372~9655232,405696667~5790944] 2024-09-13T00:39:08.326 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~6162131,25165824~14488391,46526671~6678549,54525952~12884094,81806467~4199395,92243442~6230977,111245499~9980052,130605682~5648521,146800640~14573356,162728173~1277847,171966464~2932316,176160768~5460355,206127203~3301950,209715200~9025663,227229987~5467361,243269632~8967845,260046848~8639609,276824064~6823266,285212672~11932810,301411264~7201245,322459502~1891157,330730451~2400977,336574728~3824458,343284705~1605783,373635351~8279985,385875968~11699636,405696667~5790944] 2024-09-13T00:39:08.326 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to 9 diff [2612746~3549385,26995819~4642300,31904846~7749369,46526671~6678549,58515289~704557,61366356~6043690,81806467~5134437,92243442~6230977,111245499~9980052,130605682~5648521,147450241~4531765,153773626~10658329,169300267~5598513,180155982~4105980,206691828~1354340,208047485~1381668,212320259~6420604,220044632~5434875,227825355~4871993,244107705~8129772,260842262~7844195,273904834~1135501,276897564~6749766,284165576~12979906,301411264~7201245,322459502~1891157,330730451~2400977,339257207~1141979,343284705~1605783,373635351~8279985,387920372~9655232,405696667~5790944] 2024-09-13T00:39:08.326 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [2612746~3549385,26995819~4642300,31904846~7749369,46526671~6678549,58515289~704557,61366356~6043690,81806467~5134437,92243442~6230977,111245499~9980052,130605682~5648521,147450241~4531765,153773626~10658329,169300267~5598513,180155982~4105980,206691828~1354340,208047485~1381668,212320259~6420604,220044632~5434875,227825355~4871993,244107705~8129772,260842262~7844195,273904834~1135501,276897564~6749766,284165576~12979906,301411264~7201245,322459502~1891157,330730451~2400977,339257207~1141979,343284705~1605783,373635351~8279985,387920372~9655232,405696667~5790944] 2024-09-13T00:39:08.333 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~6162131,25165824~14488391,46526671~6678549,54525952~12884094,81806467~5134437,92243442~6230977,111245499~9980052,130605682~5648521,146800640~17631315,169300267~5598513,176160768~8101194,206127203~3301950,209715200~15764307,227229987~5467361,243269632~8967845,260046848~8639609,272629760~2410575,276824064~20321418,301411264~7201245,322459502~1891157,330730451~2400977,336574728~3824458,343284705~1605783,373635351~8279985,385875968~11699636,405696667~5790944] 2024-09-13T00:39:08.333 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to 10 diff [2612746~3549385,22666579~2303167,26995819~4642300,31904846~7749369,46526671~6678549,58515289~704557,61366356~10525005,78799742~8141162,92243442~6230977,111245499~9980052,128451845~308073,130605682~5648521,147450241~4531765,153773626~10658329,169300267~5598513,180155982~4105980,206691828~1354340,208047485~1381668,212320259~6420604,220044632~12652716,244107705~8129772,260842262~7844195,273904834~1135501,276897564~6749766,284165576~12979906,301411264~7201245,322459502~1891157,330730451~2400977,339257207~1141979,343284705~1605783,348549356~6937075,365997237~1542191,373635351~8279985,387920372~9655232,405696667~5790944] 2024-09-13T00:39:08.333 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [2612746~3549385,22666579~2303167,26995819~4642300,31904846~7749369,46526671~6678549,58515289~704557,61366356~10525005,78799742~8141162,92243442~6230977,111245499~9980052,128451845~308073,130605682~5648521,147450241~4531765,153773626~10658329,169300267~5598513,180155982~4105980,206691828~1354340,208047485~1381668,212320259~6420604,220044632~12652716,244107705~8129772,260842262~7844195,273904834~1135501,276897564~6749766,284165576~12979906,301411264~7201245,322459502~1891157,330730451~2400977,339257207~1141979,343284705~1605783,348549356~6937075,365997237~1542191,373635351~8279985,387920372~9655232,405696667~5790944] 2024-09-13T00:39:08.339 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~6162131,22666579~2303167,25165824~14488391,46526671~6678549,54525952~17365409,78799742~8141162,92243442~6230977,111245499~9980052,128451845~308073,130605682~5648521,146800640~17631315,169300267~5598513,176160768~8101194,206127203~3301950,209715200~22982148,243269632~8967845,260046848~8639609,272629760~2410575,276824064~20321418,301411264~7201245,322459502~1891157,330730451~2400977,336574728~3824458,343284705~1605783,348549356~6937075,365997237~1542191,373635351~8279985,385875968~11699636,405696667~5790944] 2024-09-13T00:39:08.339 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to 11 diff [2612746~6062675,22666579~2303167,26995819~4642300,31904846~7749369,46526671~6678549,58515289~704557,61366356~10525005,78799742~8141162,92243442~6230977,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~5993153,147450241~4531765,153773626~10658329,169300267~5598513,180155982~4105980,206691828~1354340,208047485~1381668,212320259~6420604,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~21900567,301411264~7201245,322459502~5694248,330730451~2400977,335934186~1814726,339257207~1141979,343284705~1605783,348549356~6937075,365997237~1542191,373635351~8279985,387920372~15158352,405696667~5790944] 2024-09-13T00:39:08.339 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [2612746~6062675,22666579~2303167,26995819~4642300,31904846~7749369,46526671~6678549,58515289~704557,61366356~10525005,78799742~8141162,92243442~6230977,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~5993153,147450241~4531765,153773626~10658329,169300267~5598513,180155982~4105980,206691828~1354340,208047485~1381668,212320259~6420604,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~21900567,301411264~7201245,322459502~5694248,330730451~2400977,335934186~1814726,339257207~1141979,343284705~1605783,348549356~6937075,365997237~1542191,373635351~8279985,387920372~15158352,405696667~5790944] 2024-09-13T00:39:08.345 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~8675421,22666579~2303167,25165824~14488391,46526671~6678549,54525952~17365409,78799742~8141162,92243442~6230977,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,130261050~5993153,146800640~17631315,169300267~5598513,176160768~8101194,206127203~3301950,209715200~22982148,240828348~11409129,260046848~8639609,272629760~2410575,276824064~21974067,301411264~7201245,322459502~5694248,330730451~2400977,335934186~4465000,343284705~1605783,348549356~6937075,365997237~1542191,373635351~8279985,385875968~17202756,405696667~5790944] 2024-09-13T00:39:08.345 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to 12 diff [2612746~10642447,22666579~16987636,46526671~6678549,58515289~704557,61366356~10525005,78799742~8141162,92243442~10240906,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~5993153,147450241~4531765,153773626~10658329,169300267~5598513,180155982~4105980,206691828~1354340,208047485~1381668,212320259~6420604,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~21900567,301411264~7201245,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,339257207~1141979,343284705~1605783,348549356~6937075,365997237~5182427,373635351~12714553,387920372~15158352,405696667~5790944] 2024-09-13T00:39:08.345 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [2612746~10642447,22666579~16987636,46526671~6678549,58515289~704557,61366356~10525005,78799742~8141162,92243442~10240906,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~5993153,147450241~4531765,153773626~10658329,169300267~5598513,180155982~4105980,206691828~1354340,208047485~1381668,212320259~6420604,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~21900567,301411264~7201245,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,339257207~1141979,343284705~1605783,348549356~6937075,365997237~5182427,373635351~12714553,387920372~15158352,405696667~5790944] 2024-09-13T00:39:08.352 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13255193,22666579~16987636,46526671~6678549,54525952~17365409,78799742~8141162,92243442~10240906,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,130261050~5993153,146800640~17631315,169300267~5598513,176160768~8101194,206127203~3301950,209715200~22982148,240828348~11409129,260046848~8639609,272629760~2410575,276824064~21974067,301411264~7201245,310378496~5136352,316301726~3511210,322459502~12854501,335934186~4465000,343284705~1605783,348549356~6937075,365997237~5182427,373635351~29443373,405696667~5790944] 2024-09-13T00:39:08.353 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to 13 diff [2612746~10642447,17113294~3323835,22666579~16987636,46526671~6678549,58515289~704557,61366356~10525005,78799742~8141162,92243442~10240906,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~5993153,147450241~4531765,153773626~12869043,169300267~5598513,180155982~4105980,187369573~871966,201556540~2015504,206691828~1354340,208047485~1381668,209974378~1903654,212320259~6420604,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~23715912,301411264~7201245,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,339257207~1141979,343284705~1605783,348549356~6937075,365997237~5182427,373635351~12714553,387920372~15158352,405696667~5790944] 2024-09-13T00:39:08.353 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [2612746~10642447,17113294~3323835,22666579~16987636,46526671~6678549,58515289~704557,61366356~10525005,78799742~8141162,92243442~10240906,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~5993153,147450241~4531765,153773626~12869043,169300267~5598513,180155982~4105980,187369573~871966,201556540~2015504,206691828~1354340,208047485~1381668,209974378~1903654,212320259~6420604,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~23715912,301411264~7201245,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,339257207~1141979,343284705~1605783,348549356~6937075,365997237~5182427,373635351~12714553,387920372~15158352,405696667~5790944] 2024-09-13T00:39:08.359 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13255193,17113294~3323835,22666579~16987636,46526671~6678549,54525952~17365409,78799742~8141162,92243442~10240906,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,130261050~5993153,146800640~19842029,169300267~5598513,176160768~8101194,184549376~3692163,201556540~2015504,206127203~3301950,209715200~22982148,240828348~11409129,260046848~8639609,272629760~2410575,276824064~23789412,301411264~7201245,310378496~5136352,316301726~3511210,322459502~12854501,335934186~4465000,343284705~1605783,348549356~6937075,365997237~5182427,373635351~29443373,405696667~5790944] 2024-09-13T00:39:08.359 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to 14 diff [978366~12276827,17113294~3323835,22666579~16987636,46526671~6678549,58515289~704557,61366356~10525005,78799742~8141162,92243442~10240906,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~5993153,147450241~4531765,153773626~12869043,169300267~5598513,180155982~4105980,187369573~871966,201556540~2015504,206691828~12049035,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~24043645,301411264~7201245,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,339257207~1141979,343284705~1605783,348549356~6937075,365490113~5689551,371762046~1637913,373635351~12714553,387920372~15158352,405696667~5790944] 2024-09-13T00:39:08.359 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~3323835,22666579~16987636,46526671~6678549,58515289~704557,61366356~10525005,78799742~8141162,92243442~10240906,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~5993153,147450241~4531765,153773626~12869043,169300267~5598513,180155982~4105980,187369573~871966,201556540~2015504,206691828~12049035,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~24043645,301411264~7201245,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,339257207~1141979,343284705~1605783,348549356~6937075,365490113~5689551,371762046~1637913,373635351~12714553,387920372~15158352,405696667~5790944] 2024-09-13T00:39:08.366 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13255193,17113294~3323835,22666579~16987636,46526671~6678549,54525952~17365409,78799742~8141162,92243442~10240906,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,130261050~5993153,146800640~19842029,169300267~5598513,176160768~8101194,184549376~3692163,201556540~2015504,206127203~26570145,240828348~11409129,260046848~8639609,272629760~2410575,276824064~24117145,301411264~7201245,310378496~5136352,316301726~3511210,322459502~12854501,335934186~4465000,343284705~1605783,348549356~6937075,365490113~5689551,371762046~1637913,373635351~29443373,405696667~5790944] 2024-09-13T00:39:08.366 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to 15 diff [978366~12276827,17113294~3323835,22666579~16987636,46526671~6678549,58515289~704557,61366356~10525005,78799742~8141162,92243442~10240906,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~5993153,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~2015504,206691828~12049035,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~24043645,301411264~7201245,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,339257207~1141979,343284705~1605783,348299711~7186720,365490113~5689551,371762046~1637913,373635351~12714553,387920372~15158352,405696667~5790944] 2024-09-13T00:39:08.366 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~3323835,22666579~16987636,46526671~6678549,58515289~704557,61366356~10525005,78799742~8141162,92243442~10240906,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~5993153,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~2015504,206691828~12049035,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~24043645,301411264~7201245,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,339257207~1141979,343284705~1605783,348299711~7186720,365490113~5689551,371762046~1637913,373635351~12714553,387920372~15158352,405696667~5790944] 2024-09-13T00:39:08.372 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13255193,17113294~3323835,22666579~16987636,46526671~6678549,54525952~17365409,78799742~8141162,92243442~10240906,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,130261050~5993153,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~2015504,206127203~26570145,240828348~11409129,260046848~8639609,272629760~2410575,276824064~24117145,301411264~7201245,310378496~5136352,316301726~3511210,322459502~12854501,335934186~4465000,343284705~1605783,348299711~7186720,365490113~5689551,371762046~1637913,373635351~29443373,405696667~5790944] 2024-09-13T00:39:08.372 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to 16 diff [978366~12276827,17113294~3323835,22666579~16987636,46526671~9958787,58515289~704557,61366356~10525005,78799742~8141162,92243442~10240906,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~7020755,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~2015504,206691828~12049035,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~24043645,301411264~7201245,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,339257207~1141979,343284705~12201726,365490113~5689551,371762046~1637913,373635351~12714553,387920372~15158352,405696667~5790944] 2024-09-13T00:39:08.372 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~3323835,22666579~16987636,46526671~9958787,58515289~704557,61366356~10525005,78799742~8141162,92243442~10240906,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~7020755,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~2015504,206691828~12049035,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~24043645,301411264~7201245,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,339257207~1141979,343284705~12201726,365490113~5689551,371762046~1637913,373635351~12714553,387920372~15158352,405696667~5790944] 2024-09-13T00:39:08.378 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13255193,17113294~3323835,22666579~16987636,46526671~25364690,78799742~8141162,92243442~10240906,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,130261050~7020755,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~2015504,206127203~26570145,240828348~11409129,260046848~8639609,272629760~2410575,276824064~24117145,301411264~7201245,310378496~5136352,316301726~3511210,322459502~12854501,335934186~4465000,343284705~12201726,365490113~5689551,371762046~1637913,373635351~29443373,405696667~5790944] 2024-09-13T00:39:08.378 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to 17 diff [978366~12276827,17113294~3323835,22666579~16987636,46526671~9958787,58515289~13376072,78799742~8141162,92243442~10240906,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~7020755,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273904834~1135501,276897564~24043645,301411264~7201245,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,339257207~1141979,343284705~12201726,365490113~5689551,371762046~1637913,373635351~12714553,387920372~15158352,405696667~5790944] 2024-09-13T00:39:08.378 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~3323835,22666579~16987636,46526671~9958787,58515289~13376072,78799742~8141162,92243442~10240906,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~7020755,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273904834~1135501,276897564~24043645,301411264~7201245,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,339257207~1141979,343284705~12201726,365490113~5689551,371762046~1637913,373635351~12714553,387920372~15158352,405696667~5790944] 2024-09-13T00:39:08.385 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13255193,17113294~3323835,22666579~16987636,46526671~25364690,78799742~8141162,92243442~10240906,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,130261050~7020755,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~2410575,276824064~24117145,301411264~7201245,310378496~5136352,316301726~3511210,322459502~12854501,335934186~4465000,343284705~12201726,365490113~5689551,371762046~1637913,373635351~29443373,405696667~5790944] 2024-09-13T00:39:08.385 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to 18 diff [978366~12276827,17113294~3323835,22666579~16987636,46526671~9958787,58515289~15314175,76198939~10741965,92243442~10240906,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~7020755,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273904834~1135501,276897564~24043645,301411264~7201245,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,339257207~1141979,343284705~12201726,365490113~20859791,387920372~15158352,405696667~5790944] 2024-09-13T00:39:08.385 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~3323835,22666579~16987636,46526671~9958787,58515289~15314175,76198939~10741965,92243442~10240906,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~7020755,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273904834~1135501,276897564~24043645,301411264~7201245,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,339257207~1141979,343284705~12201726,365490113~20859791,387920372~15158352,405696667~5790944] 2024-09-13T00:39:08.392 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13255193,17113294~3323835,22666579~16987636,46526671~27302793,76198939~10741965,92243442~10240906,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,130261050~7020755,142230204~1250407,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~2410575,276824064~24117145,301411264~7201245,310378496~5136352,316301726~3511210,322459502~12854501,335934186~4465000,343284705~12201726,365490113~37588611,405696667~5790944] 2024-09-13T00:39:08.392 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to 19 diff [978366~12404130,17113294~3323835,22666579~16987636,46526671~9958787,58515289~15314175,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,405696667~5790944] 2024-09-13T00:39:08.392 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,46526671~9958787,58515289~15314175,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,405696667~5790944] 2024-09-13T00:39:08.398 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~16987636,46526671~27302793,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~35982749,310378496~5136352,316301726~19012277,335934186~4465000,343284705~17265138,360710144~42368580,405696667~5790944] 2024-09-13T00:39:08.398 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to 5 diff [26995819~3290536,147450241~4531765,180807822~766370,208047485~1381668,279969777~95145,339257207~1141979,373635351~8279985,390387020~7188584,409295430~1606555] 2024-09-13T00:39:08.398 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [26995819~3290536,147450241~4531765,180807822~766370,208047485~1381668,279969777~95145,339257207~1141979,373635351~8279985,390387020~7188584,409295430~1606555] 2024-09-13T00:39:08.404 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [25165824~5120531,146800640~5181366,180355072~1219120,206127203~3301950,276824064~3240858,336574728~3824458,373635351~8279985,390070272~7505332,409295430~1606555] 2024-09-13T00:39:08.404 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to 6 diff [2612746~3549385,26995819~3290536,32429128~3147149,64536720~2354229,92243442~6230977,147450241~4531765,180807822~766370,208047485~1381668,244107705~1444348,260842262~7844195,279969777~95145,289202665~1293771,301411264~7201245,339257207~1141979,373635351~8279985,390387020~7188584,409295430~1606555] 2024-09-13T00:39:08.404 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [2612746~3549385,26995819~3290536,32429128~3147149,64536720~2354229,92243442~6230977,147450241~4531765,180807822~766370,208047485~1381668,244107705~1444348,260842262~7844195,279969777~95145,289202665~1293771,301411264~7201245,339257207~1141979,373635351~8279985,390387020~7188584,409295430~1606555] 2024-09-13T00:39:08.411 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~6162131,25165824~10410453,64536720~2354229,92243442~6230977,146800640~5181366,180355072~1219120,206127203~3301950,243269632~2282421,260046848~8639609,276824064~3240858,289202665~1293771,301411264~7201245,336574728~3824458,373635351~8279985,390070272~7505332,409295430~1606555] 2024-09-13T00:39:08.411 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to 7 diff [2612746~3549385,26995819~3290536,31904846~7749369,64536720~2354229,92243442~6230977,113977873~7247678,147450241~4531765,155299497~6074499,180807822~766370,208047485~1381668,212320259~6420604,227825355~4871993,244107705~1444348,260842262~7844195,276897564~6749766,289202665~1293771,301411264~7201245,339257207~1141979,343284705~1605783,373635351~8279985,387920372~9655232,409295430~1606555] 2024-09-13T00:39:08.411 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [2612746~3549385,26995819~3290536,31904846~7749369,64536720~2354229,92243442~6230977,113977873~7247678,147450241~4531765,155299497~6074499,180807822~766370,208047485~1381668,212320259~6420604,227825355~4871993,244107705~1444348,260842262~7844195,276897564~6749766,289202665~1293771,301411264~7201245,339257207~1141979,343284705~1605783,373635351~8279985,387920372~9655232,409295430~1606555] 2024-09-13T00:39:08.417 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~6162131,25165824~14488391,64536720~2354229,92243442~6230977,113977873~7247678,146800640~5181366,155299497~6074499,180355072~1219120,206127203~3301950,209715200~9025663,227229987~5467361,243269632~2282421,260046848~8639609,276824064~6823266,289202665~1293771,301411264~7201245,336574728~3824458,343284705~1605783,373635351~8279985,385875968~11699636,409295430~1606555] 2024-09-13T00:39:08.418 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to 8 diff [2612746~3549385,26995819~4642300,31904846~7749369,64536720~2354229,81806467~4199395,92243442~6230977,111245499~9980052,147450241~4531765,155299497~6074499,172123438~2775342,180155982~1465141,208047485~1381668,212320259~6420604,227825355~4871993,244107705~8129772,260842262~7844195,276897564~6749766,289202665~1293771,301411264~7201245,322459502~1891157,330730451~2400977,339257207~1141979,343284705~1605783,373635351~8279985,387920372~9655232,409295430~1606555] 2024-09-13T00:39:08.418 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [2612746~3549385,26995819~4642300,31904846~7749369,64536720~2354229,81806467~4199395,92243442~6230977,111245499~9980052,147450241~4531765,155299497~6074499,172123438~2775342,180155982~1465141,208047485~1381668,212320259~6420604,227825355~4871993,244107705~8129772,260842262~7844195,276897564~6749766,289202665~1293771,301411264~7201245,322459502~1891157,330730451~2400977,339257207~1141979,343284705~1605783,373635351~8279985,387920372~9655232,409295430~1606555] 2024-09-13T00:39:08.424 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~6162131,25165824~14488391,64536720~2354229,81806467~4199395,92243442~6230977,111245499~9980052,146800640~5181366,155299497~6074499,171966464~2932316,176160768~5460355,206127203~3301950,209715200~9025663,227229987~5467361,243269632~8967845,260046848~8639609,276824064~6823266,289202665~1293771,301411264~7201245,322459502~1891157,330730451~2400977,336574728~3824458,343284705~1605783,373635351~8279985,385875968~11699636,409295430~1606555] 2024-09-13T00:39:08.424 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to 9 diff [2612746~3549385,26995819~4642300,31904846~7749369,64536720~2354229,81806467~5134437,92243442~6230977,111245499~9980052,147450241~4531765,155299497~9132458,169300267~5598513,180155982~4105980,206691828~1354340,208047485~1381668,212320259~6420604,220044632~5434875,227825355~4871993,244107705~8129772,260842262~7844195,273904834~1135501,276897564~6749766,284165576~6885360,301411264~7201245,322459502~1891157,330730451~2400977,339257207~1141979,343284705~1605783,373635351~8279985,387920372~9655232,409295430~1606555] 2024-09-13T00:39:08.424 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [2612746~3549385,26995819~4642300,31904846~7749369,64536720~2354229,81806467~5134437,92243442~6230977,111245499~9980052,147450241~4531765,155299497~9132458,169300267~5598513,180155982~4105980,206691828~1354340,208047485~1381668,212320259~6420604,220044632~5434875,227825355~4871993,244107705~8129772,260842262~7844195,273904834~1135501,276897564~6749766,284165576~6885360,301411264~7201245,322459502~1891157,330730451~2400977,339257207~1141979,343284705~1605783,373635351~8279985,387920372~9655232,409295430~1606555] 2024-09-13T00:39:08.430 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~6162131,25165824~14488391,64536720~2354229,81806467~5134437,92243442~6230977,111245499~9980052,146800640~5181366,155299497~9132458,169300267~5598513,176160768~8101194,206127203~3301950,209715200~15764307,227229987~5467361,243269632~8967845,260046848~8639609,272629760~2410575,276824064~14226872,301411264~7201245,322459502~1891157,330730451~2400977,336574728~3824458,343284705~1605783,373635351~8279985,385875968~11699636,409295430~1606555] 2024-09-13T00:39:08.430 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to 10 diff [2612746~3549385,22666579~2303167,26995819~4642300,31904846~7749369,64536720~7354641,78799742~8141162,92243442~6230977,111245499~9980052,128451845~308073,147450241~4531765,155299497~9132458,169300267~5598513,180155982~4105980,206691828~1354340,208047485~1381668,212320259~6420604,220044632~12652716,244107705~8129772,260842262~7844195,273904834~1135501,276897564~6749766,284165576~6885360,301411264~7201245,322459502~1891157,330730451~2400977,339257207~1141979,343284705~1605783,348549356~6937075,365997237~1542191,373635351~8279985,387920372~9655232,409295430~1606555] 2024-09-13T00:39:08.430 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [2612746~3549385,22666579~2303167,26995819~4642300,31904846~7749369,64536720~7354641,78799742~8141162,92243442~6230977,111245499~9980052,128451845~308073,147450241~4531765,155299497~9132458,169300267~5598513,180155982~4105980,206691828~1354340,208047485~1381668,212320259~6420604,220044632~12652716,244107705~8129772,260842262~7844195,273904834~1135501,276897564~6749766,284165576~6885360,301411264~7201245,322459502~1891157,330730451~2400977,339257207~1141979,343284705~1605783,348549356~6937075,365997237~1542191,373635351~8279985,387920372~9655232,409295430~1606555] 2024-09-13T00:39:08.437 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~6162131,22666579~2303167,25165824~14488391,64536720~7354641,78799742~8141162,92243442~6230977,111245499~9980052,128451845~308073,146800640~5181366,155299497~9132458,169300267~5598513,176160768~8101194,206127203~3301950,209715200~22982148,243269632~8967845,260046848~8639609,272629760~2410575,276824064~14226872,301411264~7201245,322459502~1891157,330730451~2400977,336574728~3824458,343284705~1605783,348549356~6937075,365997237~1542191,373635351~8279985,385875968~11699636,409295430~1606555] 2024-09-13T00:39:08.437 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to 11 diff [2612746~6062675,22666579~2303167,26995819~4642300,31904846~7749369,64536720~7354641,78799742~8141162,92243442~6230977,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~4316483,147450241~4531765,155299497~9132458,169300267~5598513,180155982~4105980,206691828~1354340,208047485~1381668,212320259~6420604,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~14153372,292988707~5809424,301411264~7201245,322459502~5694248,330730451~2400977,335934186~1814726,339257207~1141979,343284705~1605783,348549356~6937075,365997237~1542191,373635351~8279985,387920372~15158352,409295430~1606555] 2024-09-13T00:39:08.437 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [2612746~6062675,22666579~2303167,26995819~4642300,31904846~7749369,64536720~7354641,78799742~8141162,92243442~6230977,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~4316483,147450241~4531765,155299497~9132458,169300267~5598513,180155982~4105980,206691828~1354340,208047485~1381668,212320259~6420604,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~14153372,292988707~5809424,301411264~7201245,322459502~5694248,330730451~2400977,335934186~1814726,339257207~1141979,343284705~1605783,348549356~6937075,365997237~1542191,373635351~8279985,387920372~15158352,409295430~1606555] 2024-09-13T00:39:08.443 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~8675421,22666579~2303167,25165824~14488391,64536720~7354641,78799742~8141162,92243442~6230977,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,130261050~4316483,146800640~5181366,155299497~9132458,169300267~5598513,176160768~8101194,206127203~3301950,209715200~22982148,240828348~11409129,260046848~8639609,272629760~2410575,276824064~14226872,292988707~5809424,301411264~7201245,322459502~5694248,330730451~2400977,335934186~4465000,343284705~1605783,348549356~6937075,365997237~1542191,373635351~8279985,385875968~17202756,409295430~1606555] 2024-09-13T00:39:08.443 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to 12 diff [2612746~10642447,22666579~16987636,62948307~8943054,78799742~8141162,92243442~10240906,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~4316483,147450241~4531765,155299497~9132458,169300267~5598513,180155982~4105980,206691828~1354340,208047485~1381668,212320259~6420604,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~14153372,292988707~5809424,301411264~7201245,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,339257207~1141979,343284705~1605783,348549356~6937075,365997237~5182427,373635351~12714553,387920372~15158352,409295430~1606555] 2024-09-13T00:39:08.443 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [2612746~10642447,22666579~16987636,62948307~8943054,78799742~8141162,92243442~10240906,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~4316483,147450241~4531765,155299497~9132458,169300267~5598513,180155982~4105980,206691828~1354340,208047485~1381668,212320259~6420604,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~14153372,292988707~5809424,301411264~7201245,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,339257207~1141979,343284705~1605783,348549356~6937075,365997237~5182427,373635351~12714553,387920372~15158352,409295430~1606555] 2024-09-13T00:39:08.450 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13255193,22666579~16987636,62948307~8943054,78799742~8141162,92243442~10240906,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,130261050~4316483,146800640~5181366,155299497~9132458,169300267~5598513,176160768~8101194,206127203~3301950,209715200~22982148,240828348~11409129,260046848~8639609,272629760~2410575,276824064~14226872,292988707~5809424,301411264~7201245,310378496~5136352,316301726~3511210,322459502~12854501,335934186~4465000,343284705~1605783,348549356~6937075,365997237~5182427,373635351~29443373,409295430~1606555] 2024-09-13T00:39:08.450 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to 13 diff [2612746~10642447,17113294~3323835,22666579~16987636,62948307~8943054,78799742~8141162,92243442~10240906,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~4316483,147450241~4531765,155299497~11343172,169300267~5598513,180155982~4105980,187369573~871966,201556540~2015504,206691828~1354340,208047485~1381668,209974378~1903654,212320259~6420604,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~14153372,292988707~7624769,301411264~7201245,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,339257207~1141979,343284705~1605783,348549356~6937075,365997237~5182427,373635351~12714553,387920372~15158352,409295430~1606555] 2024-09-13T00:39:08.450 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [2612746~10642447,17113294~3323835,22666579~16987636,62948307~8943054,78799742~8141162,92243442~10240906,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~4316483,147450241~4531765,155299497~11343172,169300267~5598513,180155982~4105980,187369573~871966,201556540~2015504,206691828~1354340,208047485~1381668,209974378~1903654,212320259~6420604,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~14153372,292988707~7624769,301411264~7201245,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,339257207~1141979,343284705~1605783,348549356~6937075,365997237~5182427,373635351~12714553,387920372~15158352,409295430~1606555] 2024-09-13T00:39:08.456 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13255193,17113294~3323835,22666579~16987636,62948307~8943054,78799742~8141162,92243442~10240906,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,130261050~4316483,146800640~5181366,155299497~11343172,169300267~5598513,176160768~8101194,184549376~3692163,201556540~2015504,206127203~3301950,209715200~22982148,240828348~11409129,260046848~8639609,272629760~2410575,276824064~14226872,292988707~7624769,301411264~7201245,310378496~5136352,316301726~3511210,322459502~12854501,335934186~4465000,343284705~1605783,348549356~6937075,365997237~5182427,373635351~29443373,409295430~1606555] 2024-09-13T00:39:08.456 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to 14 diff [978366~12276827,17113294~3323835,22666579~16987636,62948307~8943054,78799742~8141162,92243442~10240906,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~4316483,147450241~4531765,155299497~11343172,169300267~5598513,180155982~4105980,187369573~871966,201556540~2015504,206691828~12049035,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~14153372,292988707~7952502,301411264~7201245,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,339257207~1141979,343284705~1605783,348549356~6937075,365490113~5689551,371762046~1637913,373635351~12714553,387920372~15158352,409295430~1606555] 2024-09-13T00:39:08.456 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~3323835,22666579~16987636,62948307~8943054,78799742~8141162,92243442~10240906,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~4316483,147450241~4531765,155299497~11343172,169300267~5598513,180155982~4105980,187369573~871966,201556540~2015504,206691828~12049035,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~14153372,292988707~7952502,301411264~7201245,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,339257207~1141979,343284705~1605783,348549356~6937075,365490113~5689551,371762046~1637913,373635351~12714553,387920372~15158352,409295430~1606555] 2024-09-13T00:39:08.463 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13255193,17113294~3323835,22666579~16987636,62948307~8943054,78799742~8141162,92243442~10240906,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,130261050~4316483,146800640~5181366,155299497~11343172,169300267~5598513,176160768~8101194,184549376~3692163,201556540~2015504,206127203~26570145,240828348~11409129,260046848~8639609,272629760~2410575,276824064~14226872,292988707~7952502,301411264~7201245,310378496~5136352,316301726~3511210,322459502~12854501,335934186~4465000,343284705~1605783,348549356~6937075,365490113~5689551,371762046~1637913,373635351~29443373,409295430~1606555] 2024-09-13T00:39:08.463 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to 15 diff [978366~12276827,17113294~3323835,22666579~16987636,62948307~8943054,78799742~8141162,92243442~10240906,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~4316483,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~2015504,206691828~12049035,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~14153372,292988707~7952502,301411264~7201245,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,339257207~1141979,343284705~1605783,348299711~7186720,365490113~5689551,371762046~1637913,373635351~12714553,387920372~15158352,409295430~1606555] 2024-09-13T00:39:08.463 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~3323835,22666579~16987636,62948307~8943054,78799742~8141162,92243442~10240906,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~4316483,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~2015504,206691828~12049035,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~14153372,292988707~7952502,301411264~7201245,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,339257207~1141979,343284705~1605783,348299711~7186720,365490113~5689551,371762046~1637913,373635351~12714553,387920372~15158352,409295430~1606555] 2024-09-13T00:39:08.470 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13255193,17113294~3323835,22666579~16987636,62948307~8943054,78799742~8141162,92243442~10240906,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,130261050~4316483,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~2015504,206127203~26570145,240828348~11409129,260046848~8639609,272629760~2410575,276824064~14226872,292988707~7952502,301411264~7201245,310378496~5136352,316301726~3511210,322459502~12854501,335934186~4465000,343284705~1605783,348299711~7186720,365490113~5689551,371762046~1637913,373635351~29443373,409295430~1606555] 2024-09-13T00:39:08.470 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to 16 diff [978366~12276827,17113294~3323835,22666579~16987636,48635901~7849557,62637940~9253421,78799742~8141162,92243442~10240906,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~7020755,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~2015504,206691828~12049035,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~14153372,292988707~7952502,301411264~7201245,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,339257207~1141979,343284705~12201726,365490113~5689551,371762046~1637913,373635351~12714553,387920372~15158352,409295430~1606555] 2024-09-13T00:39:08.470 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~3323835,22666579~16987636,48635901~7849557,62637940~9253421,78799742~8141162,92243442~10240906,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~7020755,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~2015504,206691828~12049035,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~14153372,292988707~7952502,301411264~7201245,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,339257207~1141979,343284705~12201726,365490113~5689551,371762046~1637913,373635351~12714553,387920372~15158352,409295430~1606555] 2024-09-13T00:39:08.476 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13255193,17113294~3323835,22666579~16987636,48635901~7849557,62637940~9253421,78799742~8141162,92243442~10240906,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,130261050~7020755,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~2015504,206127203~26570145,240828348~11409129,260046848~8639609,272629760~2410575,276824064~14226872,292988707~7952502,301411264~7201245,310378496~5136352,316301726~3511210,322459502~12854501,335934186~4465000,343284705~12201726,365490113~5689551,371762046~1637913,373635351~29443373,409295430~1606555] 2024-09-13T00:39:08.476 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to 17 diff [978366~12276827,17113294~3323835,22666579~16987636,48635901~7849557,58865372~13025989,78799742~8141162,92243442~10240906,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~7020755,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273904834~1135501,276897564~14153372,292988707~7952502,301411264~7201245,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,339257207~1141979,343284705~12201726,365490113~5689551,371762046~1637913,373635351~12714553,387920372~15158352,409295430~1606555] 2024-09-13T00:39:08.476 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~3323835,22666579~16987636,48635901~7849557,58865372~13025989,78799742~8141162,92243442~10240906,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~7020755,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273904834~1135501,276897564~14153372,292988707~7952502,301411264~7201245,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,339257207~1141979,343284705~12201726,365490113~5689551,371762046~1637913,373635351~12714553,387920372~15158352,409295430~1606555] 2024-09-13T00:39:08.482 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13255193,17113294~3323835,22666579~16987636,48635901~7849557,58865372~13025989,78799742~8141162,92243442~10240906,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,130261050~7020755,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~2410575,276824064~14226872,292988707~7952502,301411264~7201245,310378496~5136352,316301726~3511210,322459502~12854501,335934186~4465000,343284705~12201726,365490113~5689551,371762046~1637913,373635351~29443373,409295430~1606555] 2024-09-13T00:39:08.483 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to 18 diff [978366~12276827,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,92243442~10240906,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~7020755,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273904834~1135501,276897564~14153372,292988707~7952502,301411264~7201245,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,339257207~1141979,343284705~12201726,365490113~20859791,387920372~15158352,409295430~1606555] 2024-09-13T00:39:08.483 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,92243442~10240906,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~7020755,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273904834~1135501,276897564~14153372,292988707~7952502,301411264~7201245,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,339257207~1141979,343284705~12201726,365490113~20859791,387920372~15158352,409295430~1606555] 2024-09-13T00:39:08.489 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13255193,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,92243442~10240906,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,130261050~7020755,142230204~1250407,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~2410575,276824064~14226872,292988707~7952502,301411264~7201245,310378496~5136352,316301726~3511210,322459502~12854501,335934186~4465000,343284705~12201726,365490113~37588611,409295430~1606555] 2024-09-13T00:39:08.489 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to 19 diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,409295430~1606555] 2024-09-13T00:39:08.489 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,409295430~1606555] 2024-09-13T00:39:08.495 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~18421176,292988707~15623802,310378496~5136352,316301726~19012277,335934186~4465000,343284705~17265138,360710144~42368580,409295430~1606555] 2024-09-13T00:39:08.495 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to 6 diff [2612746~3549385,27288947~289421,32429128~3147149,64536720~2354229,92243442~6230977,244107705~1444348,260842262~7844195,289202665~1293771,301411264~7201245] 2024-09-13T00:39:08.496 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [2612746~3549385,27288947~289421,32429128~3147149,64536720~2354229,92243442~6230977,244107705~1444348,260842262~7844195,289202665~1293771,301411264~7201245] 2024-09-13T00:39:08.502 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~6162131,27288947~289421,30286355~5289922,64536720~2354229,92243442~6230977,243269632~2282421,260046848~8639609,289202665~1293771,301411264~7201245] 2024-09-13T00:39:08.502 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to 7 diff [2612746~3549385,27288947~289421,31904846~7749369,64536720~2354229,92243442~6230977,113977873~7247678,155299497~6074499,212320259~6420604,227825355~4871993,244107705~1444348,260842262~7844195,276897564~6749766,289202665~1293771,301411264~7201245,343284705~1605783,378757665~2361378,387920372~6989975] 2024-09-13T00:39:08.502 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [2612746~3549385,27288947~289421,31904846~7749369,64536720~2354229,92243442~6230977,113977873~7247678,155299497~6074499,212320259~6420604,227825355~4871993,244107705~1444348,260842262~7844195,276897564~6749766,289202665~1293771,301411264~7201245,343284705~1605783,378757665~2361378,387920372~6989975] 2024-09-13T00:39:08.508 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~6162131,27288947~289421,30286355~9367860,64536720~2354229,92243442~6230977,113977873~7247678,155299497~6074499,209715200~9025663,227229987~5467361,243269632~2282421,260046848~8639609,276897564~6749766,289202665~1293771,301411264~7201245,343284705~1605783,378757665~2361378,385875968~9034379] 2024-09-13T00:39:08.508 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to 8 diff [2612746~3549385,27288947~289421,30081629~1556490,31904846~7749369,64536720~2354229,81806467~4199395,92243442~6230977,111245499~9980052,155299497~6074499,172123438~2775342,180155982~1465141,212320259~6420604,227825355~4871993,244107705~8129772,260842262~7844195,276897564~6749766,289202665~1293771,301411264~7201245,322459502~1891157,330730451~2400977,343284705~1605783,373888833~3376207,378757665~2361378,387920372~6989975] 2024-09-13T00:39:08.508 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [2612746~3549385,27288947~289421,30081629~1556490,31904846~7749369,64536720~2354229,81806467~4199395,92243442~6230977,111245499~9980052,155299497~6074499,172123438~2775342,180155982~1465141,212320259~6420604,227825355~4871993,244107705~8129772,260842262~7844195,276897564~6749766,289202665~1293771,301411264~7201245,322459502~1891157,330730451~2400977,343284705~1605783,373888833~3376207,378757665~2361378,387920372~6989975] 2024-09-13T00:39:08.514 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~6162131,27288947~289421,30081629~9572586,64536720~2354229,81806467~4199395,92243442~6230977,111245499~9980052,155299497~6074499,171966464~2932316,176160768~5460355,209715200~9025663,227229987~5467361,243269632~8967845,260046848~8639609,276897564~6749766,289202665~1293771,301411264~7201245,322459502~1891157,330730451~2400977,343284705~1605783,373888833~3376207,378757665~2361378,385875968~9034379] 2024-09-13T00:39:08.514 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to 9 diff [2612746~3549385,27288947~289421,30081629~1556490,31904846~7749369,64536720~2354229,81806467~5134437,92243442~6230977,111245499~9980052,155299497~9132458,169300267~5598513,180155982~4105980,206691828~1354340,212320259~6420604,220044632~5434875,227825355~4871993,244107705~8129772,260842262~7844195,273904834~1135501,276897564~6749766,284165576~6885360,301411264~7201245,322459502~1891157,330730451~2400977,343284705~1605783,373888833~3376207,378757665~2361378,387920372~6989975] 2024-09-13T00:39:08.514 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [2612746~3549385,27288947~289421,30081629~1556490,31904846~7749369,64536720~2354229,81806467~5134437,92243442~6230977,111245499~9980052,155299497~9132458,169300267~5598513,180155982~4105980,206691828~1354340,212320259~6420604,220044632~5434875,227825355~4871993,244107705~8129772,260842262~7844195,273904834~1135501,276897564~6749766,284165576~6885360,301411264~7201245,322459502~1891157,330730451~2400977,343284705~1605783,373888833~3376207,378757665~2361378,387920372~6989975] 2024-09-13T00:39:08.520 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~6162131,27288947~289421,30081629~9572586,64536720~2354229,81806467~5134437,92243442~6230977,111245499~9980052,155299497~9132458,169300267~5598513,176160768~8101194,206691828~1354340,209715200~15764307,227229987~5467361,243269632~8967845,260046848~8639609,272629760~2410575,276897564~14153372,301411264~7201245,322459502~1891157,330730451~2400977,343284705~1605783,373888833~3376207,378757665~2361378,385875968~9034379] 2024-09-13T00:39:08.520 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to 10 diff [2612746~3549385,22666579~2303167,27288947~289421,30081629~1556490,31904846~7749369,64536720~7354641,78799742~8141162,92243442~6230977,111245499~9980052,128451845~308073,155299497~9132458,169300267~5598513,180155982~4105980,206691828~1354340,212320259~6420604,220044632~12652716,244107705~8129772,260842262~7844195,273904834~1135501,276897564~6749766,284165576~6885360,301411264~7201245,322459502~1891157,330730451~2400977,343284705~1605783,348549356~6937075,365997237~1542191,373888833~3376207,378757665~2361378,387920372~8663159] 2024-09-13T00:39:08.520 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [2612746~3549385,22666579~2303167,27288947~289421,30081629~1556490,31904846~7749369,64536720~7354641,78799742~8141162,92243442~6230977,111245499~9980052,128451845~308073,155299497~9132458,169300267~5598513,180155982~4105980,206691828~1354340,212320259~6420604,220044632~12652716,244107705~8129772,260842262~7844195,273904834~1135501,276897564~6749766,284165576~6885360,301411264~7201245,322459502~1891157,330730451~2400977,343284705~1605783,348549356~6937075,365997237~1542191,373888833~3376207,378757665~2361378,387920372~8663159] 2024-09-13T00:39:08.527 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~6162131,22666579~2303167,27288947~289421,30081629~9572586,64536720~7354641,78799742~8141162,92243442~6230977,111245499~9980052,128451845~308073,155299497~9132458,169300267~5598513,176160768~8101194,206691828~1354340,209715200~22982148,243269632~8967845,260046848~8639609,272629760~2410575,276897564~14153372,301411264~7201245,322459502~1891157,330730451~2400977,343284705~1605783,348549356~6937075,365997237~1542191,373888833~3376207,378757665~2361378,385875968~10707563] 2024-09-13T00:39:08.527 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to 11 diff [2612746~6062675,22666579~2303167,27288947~289421,30081629~1556490,31904846~7749369,64536720~7354641,78799742~8141162,92243442~6230977,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~4316483,155299497~9132458,169300267~5598513,180155982~4105980,206691828~1354340,212320259~6420604,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~14153372,292988707~5809424,301411264~7201245,322459502~5694248,330730451~2400977,335934186~1814726,343284705~1605783,348549356~6937075,365997237~1542191,373888833~3376207,378757665~2361378,387920372~8663159,396670010~6408714] 2024-09-13T00:39:08.527 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [2612746~6062675,22666579~2303167,27288947~289421,30081629~1556490,31904846~7749369,64536720~7354641,78799742~8141162,92243442~6230977,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~4316483,155299497~9132458,169300267~5598513,180155982~4105980,206691828~1354340,212320259~6420604,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~14153372,292988707~5809424,301411264~7201245,322459502~5694248,330730451~2400977,335934186~1814726,343284705~1605783,348549356~6937075,365997237~1542191,373888833~3376207,378757665~2361378,387920372~8663159,396670010~6408714] 2024-09-13T00:39:08.533 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~8675421,22666579~2303167,27288947~289421,30081629~9572586,64536720~7354641,78799742~8141162,92243442~6230977,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,130261050~4316483,155299497~9132458,169300267~5598513,176160768~8101194,206691828~1354340,209715200~22982148,240828348~11409129,260046848~8639609,272629760~2410575,276897564~14153372,292988707~5809424,301411264~7201245,322459502~5694248,330730451~2400977,335934186~1814726,343284705~1605783,348549356~6937075,365997237~1542191,373888833~3376207,378757665~2361378,385875968~10707563,396670010~6408714] 2024-09-13T00:39:08.533 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to 12 diff [2612746~10642447,22666579~16987636,62948307~8943054,78799742~8141162,92243442~10240906,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~4316483,155299497~9132458,169300267~5598513,180155982~4105980,206691828~1354340,212320259~6420604,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~14153372,292988707~5809424,301411264~7201245,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,343284705~1605783,348549356~6937075,365997237~5182427,373888833~3376207,378757665~7592239,387920372~8663159,396670010~6408714] 2024-09-13T00:39:08.533 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [2612746~10642447,22666579~16987636,62948307~8943054,78799742~8141162,92243442~10240906,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~4316483,155299497~9132458,169300267~5598513,180155982~4105980,206691828~1354340,212320259~6420604,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~14153372,292988707~5809424,301411264~7201245,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,343284705~1605783,348549356~6937075,365997237~5182427,373888833~3376207,378757665~7592239,387920372~8663159,396670010~6408714] 2024-09-13T00:39:08.539 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13255193,22666579~16987636,62948307~8943054,78799742~8141162,92243442~10240906,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,130261050~4316483,155299497~9132458,169300267~5598513,176160768~8101194,206691828~1354340,209715200~22982148,240828348~11409129,260046848~8639609,272629760~2410575,276897564~14153372,292988707~5809424,301411264~7201245,310378496~5136352,316301726~3511210,322459502~12854501,335934186~1814726,343284705~1605783,348549356~6937075,365997237~5182427,373888833~3376207,378757665~17825866,396670010~6408714] 2024-09-13T00:39:08.539 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to 13 diff [2612746~10642447,17113294~3323835,22666579~16987636,62948307~8943054,78799742~8141162,92243442~10240906,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~4316483,155299497~11343172,169300267~5598513,180155982~4105980,187369573~871966,201556540~2015504,206691828~1354340,209974378~1903654,212320259~6420604,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~14153372,292988707~7624769,301411264~7201245,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,343284705~1605783,348549356~6937075,365997237~5182427,373888833~3376207,378757665~7592239,387920372~8663159,396670010~6408714] 2024-09-13T00:39:08.539 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [2612746~10642447,17113294~3323835,22666579~16987636,62948307~8943054,78799742~8141162,92243442~10240906,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~4316483,155299497~11343172,169300267~5598513,180155982~4105980,187369573~871966,201556540~2015504,206691828~1354340,209974378~1903654,212320259~6420604,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~14153372,292988707~7624769,301411264~7201245,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,343284705~1605783,348549356~6937075,365997237~5182427,373888833~3376207,378757665~7592239,387920372~8663159,396670010~6408714] 2024-09-13T00:39:08.546 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13255193,17113294~3323835,22666579~16987636,62948307~8943054,78799742~8141162,92243442~10240906,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,130261050~4316483,155299497~11343172,169300267~5598513,176160768~8101194,184549376~3692163,201556540~2015504,206691828~1354340,209715200~22982148,240828348~11409129,260046848~8639609,272629760~2410575,276897564~14153372,292988707~7624769,301411264~7201245,310378496~5136352,316301726~3511210,322459502~12854501,335934186~1814726,343284705~1605783,348549356~6937075,365997237~5182427,373888833~3376207,378757665~17825866,396670010~6408714] 2024-09-13T00:39:08.546 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to 14 diff [978366~12276827,17113294~3323835,22666579~16987636,62948307~8943054,78799742~8141162,92243442~10240906,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~4316483,155299497~11343172,169300267~5598513,180155982~4105980,187369573~871966,201556540~2015504,206691828~12049035,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~14153372,292988707~7952502,301411264~7201245,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,343284705~1605783,348549356~6937075,365490113~5689551,371762046~1637913,373888833~3376207,378757665~7592239,387920372~8663159,396670010~6408714] 2024-09-13T00:39:08.546 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~3323835,22666579~16987636,62948307~8943054,78799742~8141162,92243442~10240906,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~4316483,155299497~11343172,169300267~5598513,180155982~4105980,187369573~871966,201556540~2015504,206691828~12049035,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~14153372,292988707~7952502,301411264~7201245,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,343284705~1605783,348549356~6937075,365490113~5689551,371762046~1637913,373888833~3376207,378757665~7592239,387920372~8663159,396670010~6408714] 2024-09-13T00:39:08.552 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13255193,17113294~3323835,22666579~16987636,62948307~8943054,78799742~8141162,92243442~10240906,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,130261050~4316483,155299497~11343172,169300267~5598513,176160768~8101194,184549376~3692163,201556540~2015504,206691828~26005520,240828348~11409129,260046848~8639609,272629760~2410575,276897564~14153372,292988707~7952502,301411264~7201245,310378496~5136352,316301726~3511210,322459502~12854501,335934186~1814726,343284705~1605783,348549356~6937075,365490113~5689551,371762046~1637913,373888833~3376207,378757665~17825866,396670010~6408714] 2024-09-13T00:39:08.552 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to 15 diff [978366~12276827,17113294~3323835,22666579~16987636,62948307~8943054,78799742~8141162,92243442~10240906,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~4316483,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~2015504,206691828~12049035,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~14153372,292988707~7952502,301411264~7201245,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,343284705~1605783,348299711~7186720,365490113~5689551,371762046~1637913,373888833~3376207,378757665~7592239,387920372~8663159,396670010~6408714] 2024-09-13T00:39:08.553 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~3323835,22666579~16987636,62948307~8943054,78799742~8141162,92243442~10240906,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~4316483,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~2015504,206691828~12049035,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~14153372,292988707~7952502,301411264~7201245,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,343284705~1605783,348299711~7186720,365490113~5689551,371762046~1637913,373888833~3376207,378757665~7592239,387920372~8663159,396670010~6408714] 2024-09-13T00:39:08.559 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13255193,17113294~3323835,22666579~16987636,62948307~8943054,78799742~8141162,92243442~10240906,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,130261050~4316483,144141427~2676660,148016038~18626631,169300267~14961695,184549376~3692163,188743680~10830113,201556540~2015504,206691828~26005520,240828348~11409129,260046848~8639609,272629760~2410575,276897564~14153372,292988707~7952502,301411264~7201245,310378496~5136352,316301726~3511210,322459502~12854501,335934186~1814726,343284705~1605783,348299711~7186720,365490113~5689551,371762046~1637913,373888833~3376207,378757665~17825866,396670010~6408714] 2024-09-13T00:39:08.559 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to 16 diff [978366~12276827,17113294~3323835,22666579~16987636,48635901~7849557,62637940~9253421,78799742~8141162,92243442~10240906,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~7020755,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~2015504,206691828~12049035,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~14153372,292988707~7952502,301411264~7201245,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,343284705~12201726,365490113~5689551,371762046~1637913,373888833~3376207,378757665~7592239,387920372~8663159,396670010~6408714] 2024-09-13T00:39:08.559 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~3323835,22666579~16987636,48635901~7849557,62637940~9253421,78799742~8141162,92243442~10240906,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~7020755,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~2015504,206691828~12049035,220044632~12652716,240828348~11409129,260842262~7844195,273904834~1135501,276897564~14153372,292988707~7952502,301411264~7201245,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,343284705~12201726,365490113~5689551,371762046~1637913,373888833~3376207,378757665~7592239,387920372~8663159,396670010~6408714] 2024-09-13T00:39:08.565 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13255193,17113294~3323835,22666579~16987636,48635901~7849557,62637940~9253421,78799742~8141162,92243442~10240906,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,130261050~7020755,144141427~2676660,148016038~18626631,169300267~14961695,184549376~3692163,188743680~10830113,201556540~2015504,206691828~26005520,240828348~11409129,260046848~8639609,272629760~2410575,276897564~14153372,292988707~7952502,301411264~7201245,310378496~5136352,316301726~3511210,322459502~12854501,335934186~1814726,343284705~12201726,365490113~5689551,371762046~1637913,373888833~3376207,378757665~17825866,396670010~6408714] 2024-09-13T00:39:08.565 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to 17 diff [978366~12276827,17113294~3323835,22666579~16987636,48635901~7849557,58865372~13025989,78799742~8141162,92243442~10240906,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~7020755,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273904834~1135501,276897564~14153372,292988707~7952502,301411264~7201245,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,343284705~12201726,365490113~5689551,371762046~1637913,373888833~3376207,378757665~7592239,387920372~15158352] 2024-09-13T00:39:08.566 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~3323835,22666579~16987636,48635901~7849557,58865372~13025989,78799742~8141162,92243442~10240906,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~7020755,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273904834~1135501,276897564~14153372,292988707~7952502,301411264~7201245,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,343284705~12201726,365490113~5689551,371762046~1637913,373888833~3376207,378757665~7592239,387920372~15158352] 2024-09-13T00:39:08.572 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13255193,17113294~3323835,22666579~16987636,48635901~7849557,58865372~13025989,78799742~8141162,92243442~10240906,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,130261050~7020755,144141427~2676660,148016038~18626631,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~2410575,276897564~14153372,292988707~7952502,301411264~7201245,310378496~5136352,316301726~3511210,322459502~12854501,335934186~1814726,343284705~12201726,365490113~5689551,371762046~1637913,373888833~3376207,378757665~24321059] 2024-09-13T00:39:08.572 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to 18 diff [978366~12276827,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,92243442~10240906,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~7020755,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273904834~1135501,276897564~14153372,292988707~7952502,301411264~7201245,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,343284705~12201726,365490113~11774927,378757665~7592239,387920372~15158352] 2024-09-13T00:39:08.572 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,92243442~10240906,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~7020755,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273904834~1135501,276897564~14153372,292988707~7952502,301411264~7201245,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,343284705~12201726,365490113~11774927,378757665~7592239,387920372~15158352] 2024-09-13T00:39:08.579 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13255193,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,92243442~10240906,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,130261050~7020755,142230204~1250407,144141427~2676660,148016038~18626631,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~2410575,276897564~14153372,292988707~7952502,301411264~7201245,310378496~5136352,316301726~3511210,322459502~12854501,335934186~1814726,343284705~12201726,365490113~11774927,378757665~24321059] 2024-09-13T00:39:08.579 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to 19 diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:08.579 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:08.585 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~18421176,292988707~15623802,310378496~5136352,316301726~19012277,335934186~1814726,343284705~17265138,360710144~16554896,378757665~24321059] 2024-09-13T00:39:08.585 INFO:tasks.workunit.client.0.smithi136.stdout:from 6 to 7 diff [31904846~7749369,113977873~7247678,155299497~6074499,212320259~6420604,227825355~4871993,276897564~6749766,343284705~1605783,378757665~2361378,387920372~6989975] 2024-09-13T00:39:08.585 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [31904846~7749369,113977873~7247678,155299497~6074499,212320259~6420604,227825355~4871993,276897564~6749766,343284705~1605783,378757665~2361378,387920372~6989975] 2024-09-13T00:39:08.592 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [31904846~7749369,113977873~7247678,155299497~6074499,209715200~9025663,227229987~5467361,276897564~6749766,343284705~1605783,378757665~2361378,385875968~9034379] 2024-09-13T00:39:08.592 INFO:tasks.workunit.client.0.smithi136.stdout:from 6 to 8 diff [30081629~1556490,31904846~7749369,81806467~4199395,111245499~9980052,155299497~6074499,172123438~2775342,180155982~1465141,212320259~6420604,227825355~4871993,244213343~8024134,276897564~6749766,322459502~1891157,330730451~2400977,343284705~1605783,373888833~3376207,378757665~2361378,387920372~6989975] 2024-09-13T00:39:08.592 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [30081629~1556490,31904846~7749369,81806467~4199395,111245499~9980052,155299497~6074499,172123438~2775342,180155982~1465141,212320259~6420604,227825355~4871993,244213343~8024134,276897564~6749766,322459502~1891157,330730451~2400977,343284705~1605783,373888833~3376207,378757665~2361378,387920372~6989975] 2024-09-13T00:39:08.598 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [30081629~1556490,31904846~7749369,81806467~4199395,111245499~9980052,155299497~6074499,171966464~2932316,176160768~5460355,209715200~9025663,227229987~5467361,244213343~8024134,276897564~6749766,322459502~1891157,330730451~2400977,343284705~1605783,373888833~3376207,378757665~2361378,385875968~9034379] 2024-09-13T00:39:08.599 INFO:tasks.workunit.client.0.smithi136.stdout:from 6 to 9 diff [30081629~1556490,31904846~7749369,81806467~5134437,111245499~9980052,155299497~9132458,169300267~5598513,180155982~4105980,206691828~1354340,212320259~6420604,220044632~5434875,227825355~4871993,244213343~8024134,266675149~244059,273904834~1135501,276897564~6749766,284165576~6885360,322459502~1891157,330730451~2400977,343284705~1605783,373888833~3376207,378757665~2361378,387920372~6989975] 2024-09-13T00:39:08.599 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [30081629~1556490,31904846~7749369,81806467~5134437,111245499~9980052,155299497~9132458,169300267~5598513,180155982~4105980,206691828~1354340,212320259~6420604,220044632~5434875,227825355~4871993,244213343~8024134,266675149~244059,273904834~1135501,276897564~6749766,284165576~6885360,322459502~1891157,330730451~2400977,343284705~1605783,373888833~3376207,378757665~2361378,387920372~6989975] 2024-09-13T00:39:08.604 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [30081629~1556490,31904846~7749369,81806467~5134437,111245499~9980052,155299497~9132458,169300267~5598513,176160768~8101194,206691828~1354340,209715200~15764307,227229987~5467361,244213343~8024134,266675149~244059,272629760~2410575,276897564~14153372,322459502~1891157,330730451~2400977,343284705~1605783,373888833~3376207,378757665~2361378,385875968~9034379] 2024-09-13T00:39:08.605 INFO:tasks.workunit.client.0.smithi136.stdout:from 6 to 10 diff [22666579~2303167,30081629~1556490,31904846~7749369,65727373~6163988,78799742~8141162,111245499~9980052,128451845~308073,155299497~9132458,169300267~5598513,180155982~4105980,206691828~1354340,212320259~6420604,220044632~12652716,244213343~8024134,266675149~244059,273904834~1135501,276897564~6749766,284165576~6885360,322459502~1891157,330730451~2400977,343284705~1605783,348549356~6937075,365997237~1542191,373888833~3376207,378757665~2361378,387920372~8663159] 2024-09-13T00:39:08.605 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [22666579~2303167,30081629~1556490,31904846~7749369,65727373~6163988,78799742~8141162,111245499~9980052,128451845~308073,155299497~9132458,169300267~5598513,180155982~4105980,206691828~1354340,212320259~6420604,220044632~12652716,244213343~8024134,266675149~244059,273904834~1135501,276897564~6749766,284165576~6885360,322459502~1891157,330730451~2400977,343284705~1605783,348549356~6937075,365997237~1542191,373888833~3376207,378757665~2361378,387920372~8663159] 2024-09-13T00:39:08.611 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [22666579~2303167,30081629~1556490,31904846~7749369,65727373~6163988,78799742~8141162,111245499~9980052,128451845~308073,155299497~9132458,169300267~5598513,176160768~8101194,206691828~1354340,209715200~22982148,244213343~8024134,266675149~244059,272629760~2410575,276897564~14153372,322459502~1891157,330730451~2400977,343284705~1605783,348549356~6937075,365997237~1542191,373888833~3376207,378757665~2361378,385875968~10707563] 2024-09-13T00:39:08.611 INFO:tasks.workunit.client.0.smithi136.stdout:from 6 to 11 diff [5335581~3339840,22666579~2303167,30081629~1556490,31904846~7749369,65727373~6163988,78799742~8141162,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~4316483,155299497~9132458,169300267~5598513,180155982~4105980,206691828~1354340,212320259~6420604,220044632~12652716,240828348~11409129,266675149~244059,273904834~1135501,276897564~14153372,292988707~5809424,322459502~5694248,330730451~2400977,335934186~1814726,343284705~1605783,348549356~6937075,365997237~1542191,373888833~3376207,378757665~2361378,387920372~8663159,396670010~6408714] 2024-09-13T00:39:08.611 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [5335581~3339840,22666579~2303167,30081629~1556490,31904846~7749369,65727373~6163988,78799742~8141162,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~4316483,155299497~9132458,169300267~5598513,180155982~4105980,206691828~1354340,212320259~6420604,220044632~12652716,240828348~11409129,266675149~244059,273904834~1135501,276897564~14153372,292988707~5809424,322459502~5694248,330730451~2400977,335934186~1814726,343284705~1605783,348549356~6937075,365997237~1542191,373888833~3376207,378757665~2361378,387920372~8663159,396670010~6408714] 2024-09-13T00:39:08.617 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [5335581~3339840,22666579~2303167,30081629~1556490,31904846~7749369,65727373~6163988,78799742~8141162,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,130261050~4316483,155299497~9132458,169300267~5598513,176160768~8101194,206691828~1354340,209715200~22982148,240828348~11409129,266675149~244059,272629760~2410575,276897564~14153372,292988707~5809424,322459502~5694248,330730451~2400977,335934186~1814726,343284705~1605783,348549356~6937075,365997237~1542191,373888833~3376207,378757665~2361378,385875968~10707563,396670010~6408714] 2024-09-13T00:39:08.618 INFO:tasks.workunit.client.0.smithi136.stdout:from 6 to 12 diff [5335581~7919612,22666579~16987636,62948307~8943054,78799742~8141162,95048390~7435958,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~4316483,155299497~9132458,169300267~5598513,180155982~4105980,206691828~1354340,212320259~6420604,220044632~12652716,240828348~11409129,266675149~244059,273904834~1135501,276897564~14153372,292988707~5809424,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,343284705~1605783,348549356~6937075,365997237~5182427,373888833~3376207,378757665~7592239,387920372~8663159,396670010~6408714] 2024-09-13T00:39:08.618 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [5335581~7919612,22666579~16987636,62948307~8943054,78799742~8141162,95048390~7435958,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~4316483,155299497~9132458,169300267~5598513,180155982~4105980,206691828~1354340,212320259~6420604,220044632~12652716,240828348~11409129,266675149~244059,273904834~1135501,276897564~14153372,292988707~5809424,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,343284705~1605783,348549356~6937075,365997237~5182427,373888833~3376207,378757665~7592239,387920372~8663159,396670010~6408714] 2024-09-13T00:39:08.624 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [5335581~7919612,22666579~16987636,62948307~8943054,78799742~8141162,95048390~7435958,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,130261050~4316483,155299497~9132458,169300267~5598513,176160768~8101194,206691828~1354340,209715200~22982148,240828348~11409129,266675149~244059,272629760~2410575,276897564~14153372,292988707~5809424,310378496~5136352,316301726~3511210,322459502~12854501,335934186~1814726,343284705~1605783,348549356~6937075,365997237~5182427,373888833~3376207,378757665~17825866,396670010~6408714] 2024-09-13T00:39:08.624 INFO:tasks.workunit.client.0.smithi136.stdout:from 6 to 13 diff [5335581~7919612,17113294~3323835,22666579~16987636,62948307~8943054,78799742~8141162,95048390~7435958,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~4316483,155299497~11343172,169300267~5598513,180155982~4105980,187369573~871966,201556540~2015504,206691828~1354340,209974378~1903654,212320259~6420604,220044632~12652716,240828348~11409129,266675149~244059,273904834~1135501,276897564~14153372,292988707~7624769,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,343284705~1605783,348549356~6937075,365997237~5182427,373888833~3376207,378757665~7592239,387920372~8663159,396670010~6408714] 2024-09-13T00:39:08.624 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [5335581~7919612,17113294~3323835,22666579~16987636,62948307~8943054,78799742~8141162,95048390~7435958,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~4316483,155299497~11343172,169300267~5598513,180155982~4105980,187369573~871966,201556540~2015504,206691828~1354340,209974378~1903654,212320259~6420604,220044632~12652716,240828348~11409129,266675149~244059,273904834~1135501,276897564~14153372,292988707~7624769,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,343284705~1605783,348549356~6937075,365997237~5182427,373888833~3376207,378757665~7592239,387920372~8663159,396670010~6408714] 2024-09-13T00:39:08.631 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [5335581~7919612,17113294~3323835,22666579~16987636,62948307~8943054,78799742~8141162,95048390~7435958,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,130261050~4316483,155299497~11343172,169300267~5598513,176160768~8101194,184549376~3692163,201556540~2015504,206691828~1354340,209715200~22982148,240828348~11409129,266675149~244059,272629760~2410575,276897564~14153372,292988707~7624769,310378496~5136352,316301726~3511210,322459502~12854501,335934186~1814726,343284705~1605783,348549356~6937075,365997237~5182427,373888833~3376207,378757665~17825866,396670010~6408714] 2024-09-13T00:39:08.631 INFO:tasks.workunit.client.0.smithi136.stdout:from 6 to 14 diff [978366~12276827,17113294~3323835,22666579~16987636,62948307~8943054,78799742~8141162,95048390~7435958,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~4316483,155299497~11343172,169300267~5598513,180155982~4105980,187369573~871966,201556540~2015504,206691828~12049035,220044632~12652716,240828348~11409129,266675149~244059,273904834~1135501,276897564~14153372,292988707~7952502,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,343284705~1605783,348549356~6937075,365490113~5689551,371762046~1637913,373888833~3376207,378757665~7592239,387920372~8663159,396670010~6408714] 2024-09-13T00:39:08.631 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~3323835,22666579~16987636,62948307~8943054,78799742~8141162,95048390~7435958,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~4316483,155299497~11343172,169300267~5598513,180155982~4105980,187369573~871966,201556540~2015504,206691828~12049035,220044632~12652716,240828348~11409129,266675149~244059,273904834~1135501,276897564~14153372,292988707~7952502,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,343284705~1605783,348549356~6937075,365490113~5689551,371762046~1637913,373888833~3376207,378757665~7592239,387920372~8663159,396670010~6408714] 2024-09-13T00:39:08.637 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12276827,17113294~3323835,22666579~16987636,62948307~8943054,78799742~8141162,95048390~7435958,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,130261050~4316483,155299497~11343172,169300267~5598513,176160768~8101194,184549376~3692163,201556540~2015504,206691828~26005520,240828348~11409129,266675149~244059,272629760~2410575,276897564~14153372,292988707~7952502,310378496~5136352,316301726~3511210,322459502~12854501,335934186~1814726,343284705~1605783,348549356~6937075,365490113~5689551,371762046~1637913,373888833~3376207,378757665~17825866,396670010~6408714] 2024-09-13T00:39:08.637 INFO:tasks.workunit.client.0.smithi136.stdout:from 6 to 15 diff [978366~12276827,17113294~3323835,22666579~16987636,62948307~8943054,78799742~8141162,95048390~7435958,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~4316483,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~2015504,206691828~12049035,220044632~12652716,240828348~11409129,266675149~244059,273904834~1135501,276897564~14153372,292988707~7952502,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,343284705~1605783,348299711~7186720,365490113~5689551,371762046~1637913,373888833~3376207,378757665~7592239,387920372~8663159,396670010~6408714] 2024-09-13T00:39:08.637 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~3323835,22666579~16987636,62948307~8943054,78799742~8141162,95048390~7435958,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~4316483,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~2015504,206691828~12049035,220044632~12652716,240828348~11409129,266675149~244059,273904834~1135501,276897564~14153372,292988707~7952502,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,343284705~1605783,348299711~7186720,365490113~5689551,371762046~1637913,373888833~3376207,378757665~7592239,387920372~8663159,396670010~6408714] 2024-09-13T00:39:08.644 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12276827,17113294~3323835,22666579~16987636,62948307~8943054,78799742~8141162,95048390~7435958,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,130261050~4316483,144141427~2676660,148016038~18626631,169300267~14961695,184549376~3692163,188743680~10830113,201556540~2015504,206691828~26005520,240828348~11409129,266675149~244059,272629760~2410575,276897564~14153372,292988707~7952502,310378496~5136352,316301726~3511210,322459502~12854501,335934186~1814726,343284705~1605783,348299711~7186720,365490113~5689551,371762046~1637913,373888833~3376207,378757665~17825866,396670010~6408714] 2024-09-13T00:39:08.644 INFO:tasks.workunit.client.0.smithi136.stdout:from 6 to 16 diff [978366~12276827,17113294~3323835,22666579~16987636,48635901~7849557,62637940~9253421,78799742~8141162,95048390~7435958,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~7020755,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~2015504,206691828~12049035,220044632~12652716,240828348~11409129,266675149~244059,273904834~1135501,276897564~14153372,292988707~7952502,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,343284705~12201726,365490113~5689551,371762046~1637913,373888833~3376207,378757665~7592239,387920372~8663159,396670010~6408714] 2024-09-13T00:39:08.644 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~3323835,22666579~16987636,48635901~7849557,62637940~9253421,78799742~8141162,95048390~7435958,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~7020755,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~2015504,206691828~12049035,220044632~12652716,240828348~11409129,266675149~244059,273904834~1135501,276897564~14153372,292988707~7952502,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,343284705~12201726,365490113~5689551,371762046~1637913,373888833~3376207,378757665~7592239,387920372~8663159,396670010~6408714] 2024-09-13T00:39:08.650 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12276827,17113294~3323835,22666579~16987636,48635901~7849557,62637940~9253421,78799742~8141162,95048390~7435958,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,130261050~7020755,144141427~2676660,148016038~18626631,169300267~14961695,184549376~3692163,188743680~10830113,201556540~2015504,206691828~26005520,240828348~11409129,266675149~244059,272629760~2410575,276897564~14153372,292988707~7952502,310378496~5136352,316301726~3511210,322459502~12854501,335934186~1814726,343284705~12201726,365490113~5689551,371762046~1637913,373888833~3376207,378757665~17825866,396670010~6408714] 2024-09-13T00:39:08.650 INFO:tasks.workunit.client.0.smithi136.stdout:from 6 to 17 diff [978366~12276827,17113294~3323835,22666579~16987636,48635901~7849557,58865372~13025989,78799742~8141162,95048390~7435958,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~7020755,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273904834~1135501,276897564~14153372,292988707~7952502,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,343284705~12201726,365490113~5689551,371762046~1637913,373888833~3376207,378757665~7592239,387920372~15158352] 2024-09-13T00:39:08.650 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~3323835,22666579~16987636,48635901~7849557,58865372~13025989,78799742~8141162,95048390~7435958,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~7020755,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273904834~1135501,276897564~14153372,292988707~7952502,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,343284705~12201726,365490113~5689551,371762046~1637913,373888833~3376207,378757665~7592239,387920372~15158352] 2024-09-13T00:39:08.657 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12276827,17113294~3323835,22666579~16987636,48635901~7849557,58865372~13025989,78799742~8141162,95048390~7435958,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,130261050~7020755,144141427~2676660,148016038~18626631,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,266675149~244059,272629760~2410575,276897564~14153372,292988707~7952502,310378496~5136352,316301726~3511210,322459502~12854501,335934186~1814726,343284705~12201726,365490113~5689551,371762046~1637913,373888833~3376207,378757665~24321059] 2024-09-13T00:39:08.657 INFO:tasks.workunit.client.0.smithi136.stdout:from 6 to 18 diff [978366~12276827,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,95048390~7435958,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~7020755,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273904834~1135501,276897564~14153372,292988707~7952502,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,343284705~12201726,365490113~11774927,378757665~7592239,387920372~15158352] 2024-09-13T00:39:08.657 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,95048390~7435958,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,130261050~7020755,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273904834~1135501,276897564~14153372,292988707~7952502,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,343284705~12201726,365490113~11774927,378757665~7592239,387920372~15158352] 2024-09-13T00:39:08.664 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12276827,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,95048390~7435958,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,130261050~7020755,142230204~1250407,144141427~2676660,148016038~18626631,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,266675149~244059,272629760~2410575,276897564~14153372,292988707~7952502,310378496~5136352,316301726~3511210,322459502~12854501,335934186~1814726,343284705~12201726,365490113~11774927,378757665~24321059] 2024-09-13T00:39:08.664 INFO:tasks.workunit.client.0.smithi136.stdout:from 6 to 19 diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273064912~17986024,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:08.664 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273064912~17986024,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:08.673 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,266675149~244059,272629760~18421176,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,343284705~17265138,360710144~16554896,378757665~24321059] 2024-09-13T00:39:08.673 INFO:tasks.workunit.client.0.smithi136.stdout:from 7 to 8 diff [30081629~1556490,81806467~4199395,111245499~4720159,172123438~2775342,180155982~1465141,244213343~8024134,322459502~1891157,330730451~2400977,373888833~3376207] 2024-09-13T00:39:08.673 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [30081629~1556490,81806467~4199395,111245499~4720159,172123438~2775342,180155982~1465141,244213343~8024134,322459502~1891157,330730451~2400977,373888833~3376207] 2024-09-13T00:39:08.677 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [30081629~1556490,81806467~4199395,111245499~4720159,171966464~2932316,176160768~5460355,244213343~8024134,322459502~1891157,330730451~2400977,373888833~3376207] 2024-09-13T00:39:08.677 INFO:tasks.workunit.client.0.smithi136.stdout:from 7 to 9 diff [30081629~1556490,81806467~5134437,111245499~4720159,159230705~5201250,169300267~5598513,180155982~4105980,206691828~1354340,220044632~5434875,244213343~8024134,266675149~244059,273904834~1135501,284165576~6885360,322459502~1891157,330730451~2400977,373888833~3376207] 2024-09-13T00:39:08.677 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [30081629~1556490,81806467~5134437,111245499~4720159,159230705~5201250,169300267~5598513,180155982~4105980,206691828~1354340,220044632~5434875,244213343~8024134,266675149~244059,273904834~1135501,284165576~6885360,322459502~1891157,330730451~2400977,373888833~3376207] 2024-09-13T00:39:08.684 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [30081629~1556490,81806467~5134437,111245499~4720159,159230705~5201250,169300267~5598513,176160768~8101194,206691828~1354340,218740863~6738644,244213343~8024134,266675149~244059,272629760~2410575,283647330~7403606,322459502~1891157,330730451~2400977,373888833~3376207] 2024-09-13T00:39:08.684 INFO:tasks.workunit.client.0.smithi136.stdout:from 7 to 10 diff [22666579~2303167,30081629~1556490,65727373~6163988,78799742~8141162,111245499~4720159,128451845~308073,159230705~5201250,169300267~5598513,180155982~4105980,206691828~1354340,220044632~9089487,244213343~8024134,266675149~244059,273904834~1135501,284165576~6885360,322459502~1891157,330730451~2400977,348549356~6937075,365997237~1542191,373888833~3376207,392519231~4064300] 2024-09-13T00:39:08.684 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [22666579~2303167,30081629~1556490,65727373~6163988,78799742~8141162,111245499~4720159,128451845~308073,159230705~5201250,169300267~5598513,180155982~4105980,206691828~1354340,220044632~9089487,244213343~8024134,266675149~244059,273904834~1135501,284165576~6885360,322459502~1891157,330730451~2400977,348549356~6937075,365997237~1542191,373888833~3376207,392519231~4064300] 2024-09-13T00:39:08.690 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [22666579~2303167,30081629~1556490,65727373~6163988,78799742~8141162,111245499~4720159,128451845~308073,159230705~5201250,169300267~5598513,176160768~8101194,206691828~1354340,218740863~10393256,244213343~8024134,266675149~244059,272629760~2410575,283647330~7403606,322459502~1891157,330730451~2400977,348549356~6937075,365997237~1542191,373888833~3376207,392519231~4064300] 2024-09-13T00:39:08.691 INFO:tasks.workunit.client.0.smithi136.stdout:from 7 to 11 diff [5335581~3339840,22666579~2303167,30081629~1556490,65727373~6163988,78799742~8141162,105459787~1424206,111245499~4720159,122043306~5052094,128451845~308073,130261050~4316483,159230705~5201250,169300267~5598513,180155982~4105980,206691828~1354340,220044632~9089487,240828348~11409129,266675149~244059,273904834~1135501,282726102~8324834,292988707~5809424,322459502~5694248,330730451~2400977,335934186~1814726,348549356~6937075,365997237~1542191,373888833~3376207,392519231~4064300,396670010~6408714] 2024-09-13T00:39:08.691 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [5335581~3339840,22666579~2303167,30081629~1556490,65727373~6163988,78799742~8141162,105459787~1424206,111245499~4720159,122043306~5052094,128451845~308073,130261050~4316483,159230705~5201250,169300267~5598513,180155982~4105980,206691828~1354340,220044632~9089487,240828348~11409129,266675149~244059,273904834~1135501,282726102~8324834,292988707~5809424,322459502~5694248,330730451~2400977,335934186~1814726,348549356~6937075,365997237~1542191,373888833~3376207,392519231~4064300,396670010~6408714] 2024-09-13T00:39:08.697 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [5335581~3339840,22666579~2303167,30081629~1556490,65727373~6163988,78799742~8141162,104857600~2026393,111245499~4720159,121634816~5460584,128451845~308073,130261050~4316483,159230705~5201250,169300267~5598513,176160768~8101194,206691828~1354340,218740863~10393256,240828348~11409129,266675149~244059,272629760~2410575,282726102~8324834,292988707~5809424,322459502~5694248,330730451~2400977,335934186~1814726,348549356~6937075,365997237~1542191,373888833~3376207,392519231~4064300,396670010~6408714] 2024-09-13T00:39:08.697 INFO:tasks.workunit.client.0.smithi136.stdout:from 7 to 12 diff [5335581~7919612,22666579~9651148,62948307~8943054,78799742~8141162,95048390~7435958,105459787~1424206,111245499~4720159,122043306~5052094,128451845~308073,130261050~4316483,159230705~5201250,169300267~5598513,180155982~4105980,206691828~1354340,220044632~9089487,240828348~11409129,266675149~244059,273904834~1135501,277779043~4483937,282726102~8324834,292988707~5809424,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,348549356~6937075,365997237~5182427,373888833~3376207,379083185~7266719,392519231~4064300,396670010~6408714] 2024-09-13T00:39:08.697 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [5335581~7919612,22666579~9651148,62948307~8943054,78799742~8141162,95048390~7435958,105459787~1424206,111245499~4720159,122043306~5052094,128451845~308073,130261050~4316483,159230705~5201250,169300267~5598513,180155982~4105980,206691828~1354340,220044632~9089487,240828348~11409129,266675149~244059,273904834~1135501,277779043~4483937,282726102~8324834,292988707~5809424,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,348549356~6937075,365997237~5182427,373888833~3376207,379083185~7266719,392519231~4064300,396670010~6408714] 2024-09-13T00:39:08.703 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [5335581~7919612,22666579~9651148,62948307~8943054,78799742~8141162,95048390~7435958,104857600~2026393,111245499~4720159,121634816~5460584,128451845~308073,130261050~4316483,159230705~5201250,169300267~5598513,176160768~8101194,206691828~1354340,218740863~10393256,240828348~11409129,266675149~244059,272629760~2410575,277779043~4483937,282726102~8324834,292988707~5809424,310378496~5136352,316301726~3511210,322459502~12854501,335934186~1814726,348549356~6937075,365997237~5182427,373888833~3376207,379083185~7266719,392519231~4064300,396670010~6408714] 2024-09-13T00:39:08.703 INFO:tasks.workunit.client.0.smithi136.stdout:from 7 to 13 diff [5335581~7919612,17113294~3323835,22666579~9651148,62948307~8943054,78799742~8141162,95048390~7435958,105459787~1424206,111245499~4720159,122043306~5052094,128451845~308073,130261050~4316483,159230705~7411964,169300267~5598513,180155982~4105980,187369573~871966,201556540~2015504,206691828~1354340,209974378~1903654,220044632~9089487,240828348~11409129,266675149~244059,273904834~1135501,277779043~4483937,282726102~8324834,292988707~7624769,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,348549356~6937075,365997237~5182427,373888833~3376207,379083185~7266719,392519231~4064300,396670010~6408714] 2024-09-13T00:39:08.703 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [5335581~7919612,17113294~3323835,22666579~9651148,62948307~8943054,78799742~8141162,95048390~7435958,105459787~1424206,111245499~4720159,122043306~5052094,128451845~308073,130261050~4316483,159230705~7411964,169300267~5598513,180155982~4105980,187369573~871966,201556540~2015504,206691828~1354340,209974378~1903654,220044632~9089487,240828348~11409129,266675149~244059,273904834~1135501,277779043~4483937,282726102~8324834,292988707~7624769,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,348549356~6937075,365997237~5182427,373888833~3376207,379083185~7266719,392519231~4064300,396670010~6408714] 2024-09-13T00:39:08.710 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [5335581~7919612,17113294~3323835,22666579~9651148,62948307~8943054,78799742~8141162,95048390~7435958,104857600~2026393,111245499~4720159,121634816~5460584,128451845~308073,130261050~4316483,159230705~7411964,169300267~5598513,176160768~8101194,184549376~3692163,201556540~2015504,206691828~1354340,209974378~1903654,218740863~10393256,240828348~11409129,266675149~244059,272629760~2410575,277779043~4483937,282726102~8324834,292988707~7624769,310378496~5136352,316301726~3511210,322459502~12854501,335934186~1814726,348549356~6937075,365997237~5182427,373888833~3376207,379083185~7266719,392519231~4064300,396670010~6408714] 2024-09-13T00:39:08.710 INFO:tasks.workunit.client.0.smithi136.stdout:from 7 to 14 diff [978366~12276827,17113294~3323835,22666579~10131102,62948307~8943054,78799742~8141162,95048390~7435958,105459787~1424206,111245499~4720159,122043306~5052094,128451845~308073,130261050~4316483,159230705~7411964,169300267~5598513,180155982~4105980,187369573~871966,201556540~2015504,206691828~6088778,220044632~9089487,240828348~11409129,266675149~244059,273904834~1135501,277779043~4483937,282726102~8324834,292988707~7952502,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,348549356~6937075,365490113~5689551,371762046~1637913,373888833~3376207,379083185~7266719,389872420~6711111,396670010~6408714] 2024-09-13T00:39:08.710 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~3323835,22666579~10131102,62948307~8943054,78799742~8141162,95048390~7435958,105459787~1424206,111245499~4720159,122043306~5052094,128451845~308073,130261050~4316483,159230705~7411964,169300267~5598513,180155982~4105980,187369573~871966,201556540~2015504,206691828~6088778,220044632~9089487,240828348~11409129,266675149~244059,273904834~1135501,277779043~4483937,282726102~8324834,292988707~7952502,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,348549356~6937075,365490113~5689551,371762046~1637913,373888833~3376207,379083185~7266719,389872420~6711111,396670010~6408714] 2024-09-13T00:39:08.716 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12276827,17113294~3323835,22666579~10131102,62948307~8943054,78799742~8141162,95048390~7435958,104857600~2026393,111245499~4720159,121634816~5460584,128451845~308073,130261050~4316483,159230705~7411964,169300267~5598513,176160768~8101194,184549376~3692163,201556540~2015504,206691828~6088778,218740863~10393256,240828348~11409129,266675149~244059,272629760~2410575,277779043~4483937,282726102~8324834,292988707~7952502,310378496~5136352,316301726~3511210,322459502~12854501,335934186~1814726,348549356~6937075,365490113~5689551,371762046~1637913,373888833~3376207,379083185~7266719,389872420~6711111,396670010~6408714] 2024-09-13T00:39:08.716 INFO:tasks.workunit.client.0.smithi136.stdout:from 7 to 15 diff [978366~12276827,17113294~3323835,22666579~10131102,62948307~8943054,78799742~8141162,95048390~7435958,105459787~1424206,111245499~4720159,122043306~5052094,128451845~308073,130261050~4316483,144141427~2676660,148016038~10362739,159230705~7411964,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~2015504,206691828~6088778,220044632~9089487,240828348~11409129,266675149~244059,273904834~1135501,277779043~4483937,282726102~8324834,292988707~7952502,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,348299711~7186720,365490113~5689551,371762046~1637913,373888833~3376207,379083185~7266719,389872420~6711111,396670010~6408714] 2024-09-13T00:39:08.716 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~3323835,22666579~10131102,62948307~8943054,78799742~8141162,95048390~7435958,105459787~1424206,111245499~4720159,122043306~5052094,128451845~308073,130261050~4316483,144141427~2676660,148016038~10362739,159230705~7411964,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~2015504,206691828~6088778,220044632~9089487,240828348~11409129,266675149~244059,273904834~1135501,277779043~4483937,282726102~8324834,292988707~7952502,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,348299711~7186720,365490113~5689551,371762046~1637913,373888833~3376207,379083185~7266719,389872420~6711111,396670010~6408714] 2024-09-13T00:39:08.723 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12276827,17113294~3323835,22666579~10131102,62948307~8943054,78799742~8141162,95048390~7435958,104857600~2026393,111245499~4720159,121634816~5460584,128451845~308073,130261050~4316483,144141427~2676660,148016038~10362739,159230705~7411964,169300267~14961695,184549376~3692163,188743680~10830113,201556540~2015504,206691828~6088778,218740863~10393256,240828348~11409129,266675149~244059,272629760~2410575,277779043~4483937,282726102~8324834,292988707~7952502,310378496~5136352,316301726~3511210,322459502~12854501,335934186~1814726,348299711~7186720,365490113~5689551,371762046~1637913,373888833~3376207,379083185~7266719,389872420~6711111,396670010~6408714] 2024-09-13T00:39:08.723 INFO:tasks.workunit.client.0.smithi136.stdout:from 7 to 16 diff [978366~12276827,17113294~3323835,22666579~10131102,48635901~7849557,62637940~9253421,78799742~8141162,95048390~7435958,105459787~1424206,111245499~4720159,122043306~5052094,128451845~308073,130261050~7020755,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~2015504,206691828~6088778,220044632~9819324,240828348~11409129,266675149~244059,273904834~1135501,277779043~4483937,282726102~8324834,292988707~7952502,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,344823952~10662479,365490113~5689551,371762046~1637913,373888833~3376207,379003744~7346160,389872420~6711111,396670010~6408714] 2024-09-13T00:39:08.723 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~3323835,22666579~10131102,48635901~7849557,62637940~9253421,78799742~8141162,95048390~7435958,105459787~1424206,111245499~4720159,122043306~5052094,128451845~308073,130261050~7020755,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~2015504,206691828~6088778,220044632~9819324,240828348~11409129,266675149~244059,273904834~1135501,277779043~4483937,282726102~8324834,292988707~7952502,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,344823952~10662479,365490113~5689551,371762046~1637913,373888833~3376207,379003744~7346160,389872420~6711111,396670010~6408714] 2024-09-13T00:39:08.729 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12276827,17113294~3323835,22666579~10131102,48635901~7849557,62637940~9253421,78799742~8141162,95048390~7435958,104857600~2026393,111245499~4720159,121634816~5460584,128451845~308073,130261050~7020755,144141427~2676660,148016038~18626631,169300267~14961695,184549376~3692163,188743680~10830113,201556540~2015504,206691828~6088778,218740863~11123093,240828348~11409129,266675149~244059,272629760~2410575,277779043~4483937,282726102~8324834,292988707~7952502,310378496~5136352,316301726~3511210,322459502~12854501,335934186~1814726,344823952~10662479,365490113~5689551,371762046~1637913,373888833~3376207,379003744~7346160,389872420~6711111,396670010~6408714] 2024-09-13T00:39:08.729 INFO:tasks.workunit.client.0.smithi136.stdout:from 7 to 17 diff [978366~12276827,17113294~3323835,22666579~10131102,37199402~669117,48635901~7849557,58865372~13025989,78799742~8141162,95048390~7435958,105459787~1424206,111245499~4720159,122043306~5052094,128451845~308073,130261050~7020755,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~11224066,215619334~2003239,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273904834~1135501,277779043~4483937,282726102~8324834,292988707~7952502,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,344823952~10662479,365490113~5689551,371762046~1637913,373888833~3376207,379003744~7346160,389458074~13620650] 2024-09-13T00:39:08.729 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~3323835,22666579~10131102,37199402~669117,48635901~7849557,58865372~13025989,78799742~8141162,95048390~7435958,105459787~1424206,111245499~4720159,122043306~5052094,128451845~308073,130261050~7020755,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~11224066,215619334~2003239,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273904834~1135501,277779043~4483937,282726102~8324834,292988707~7952502,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,344823952~10662479,365490113~5689551,371762046~1637913,373888833~3376207,379003744~7346160,389458074~13620650] 2024-09-13T00:39:08.736 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12276827,17113294~3323835,22666579~10131102,37199402~669117,48635901~7849557,58865372~13025989,78799742~8141162,95048390~7435958,104857600~2026393,111245499~4720159,121634816~5460584,128451845~308073,130261050~7020755,144141427~2676660,148016038~18626631,169300267~14961695,184549376~3692163,188743680~10830113,201556540~11224066,215619334~2003239,218740863~15650400,240828348~12680553,266675149~244059,272629760~2410575,277779043~4483937,282726102~8324834,292988707~7952502,310378496~5136352,316301726~3511210,322459502~12854501,335934186~1814726,344823952~10662479,365490113~5689551,371762046~1637913,373888833~3376207,379003744~7346160,389458074~13620650] 2024-09-13T00:39:08.736 INFO:tasks.workunit.client.0.smithi136.stdout:from 7 to 18 diff [978366~12276827,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,95048390~7435958,105459787~1424206,111245499~4720159,122043306~5052094,128451845~308073,130261050~7020755,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~16427001,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273904834~1135501,277779043~4483937,282726102~8324834,292988707~7952502,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,344823952~10662479,365490113~11774927,379003744~7346160,389458074~13620650] 2024-09-13T00:39:08.736 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,95048390~7435958,105459787~1424206,111245499~4720159,122043306~5052094,128451845~308073,130261050~7020755,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~16427001,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273904834~1135501,277779043~4483937,282726102~8324834,292988707~7952502,312566619~2948229,316301726~3511210,322459502~12854501,335934186~1814726,344823952~10662479,365490113~11774927,379003744~7346160,389458074~13620650] 2024-09-13T00:39:08.742 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12276827,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,95048390~7435958,104857600~2026393,111245499~4720159,121634816~5460584,128451845~308073,130261050~7020755,142230204~1250407,144141427~2676660,148016038~18626631,169300267~14961695,184549376~3692163,188743680~10830113,201556540~16427001,218740863~15650400,240828348~12680553,266675149~244059,272629760~2410575,277779043~4483937,282726102~8324834,292988707~7952502,310378496~5136352,316301726~3511210,322459502~12854501,335934186~1814726,344823952~10662479,365490113~11774927,379003744~7346160,389458074~13620650] 2024-09-13T00:39:08.742 INFO:tasks.workunit.client.0.smithi136.stdout:from 7 to 19 diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~4720159,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~16427001,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273064912~4591743,277779043~4483937,282726102~8324834,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~14864738,379003744~7346160,389458074~13620650] 2024-09-13T00:39:08.742 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~4720159,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~16427001,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273064912~4591743,277779043~4483937,282726102~8324834,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~14864738,379003744~7346160,389458074~13620650] 2024-09-13T00:39:08.749 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~4720159,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~14961695,184549376~3692163,188743680~10830113,201556540~16427001,218740863~15650400,240828348~12680553,266675149~244059,272629760~5026895,277779043~4483937,282726102~8324834,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,344823952~15725891,360710144~16554896,379003744~7346160,389458074~13620650] 2024-09-13T00:39:08.749 INFO:tasks.workunit.client.0.smithi136.stdout:from 8 to 9 diff [84137493~2803411,159230705~5201250,169300267~4108516,180376314~3885648,206691828~1354340,220044632~5434875,246659731~4921744,266675149~244059,273904834~1135501,284165576~6885360] 2024-09-13T00:39:08.749 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [84137493~2803411,159230705~5201250,169300267~4108516,180376314~3885648,206691828~1354340,220044632~5434875,246659731~4921744,266675149~244059,273904834~1135501,284165576~6885360] 2024-09-13T00:39:08.755 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [84137493~2803411,159230705~5201250,169300267~4108516,180376314~3885648,206691828~1354340,218740863~6738644,246659731~4921744,266675149~244059,272629760~2410575,283647330~7403606] 2024-09-13T00:39:08.755 INFO:tasks.workunit.client.0.smithi136.stdout:from 8 to 10 diff [22666579~2303167,65727373~6163988,78799742~8141162,128451845~308073,159230705~5201250,169300267~4108516,180376314~3885648,206691828~1354340,220044632~9089487,246659731~4921744,266675149~244059,273904834~1135501,284165576~6885360,348549356~6937075,365997237~1542191,392519231~4064300] 2024-09-13T00:39:08.755 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [22666579~2303167,65727373~6163988,78799742~8141162,128451845~308073,159230705~5201250,169300267~4108516,180376314~3885648,206691828~1354340,220044632~9089487,246659731~4921744,266675149~244059,273904834~1135501,284165576~6885360,348549356~6937075,365997237~1542191,392519231~4064300] 2024-09-13T00:39:08.761 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [22666579~2303167,65727373~6163988,78799742~8141162,128451845~308073,159230705~5201250,169300267~4108516,180376314~3885648,206691828~1354340,218740863~10393256,246659731~4921744,266675149~244059,272629760~2410575,283647330~7403606,348549356~6937075,365997237~1542191,392519231~4064300] 2024-09-13T00:39:08.762 INFO:tasks.workunit.client.0.smithi136.stdout:from 8 to 11 diff [5335581~3339840,22666579~2303167,65727373~6163988,78799742~8141162,105459787~1424206,122043306~5052094,128451845~308073,130261050~4316483,159230705~5201250,169300267~4108516,180376314~3885648,206691828~1354340,220044632~9089487,240828348~10753127,266675149~244059,273904834~1135501,282726102~8324834,292988707~5809424,322862850~5290900,335934186~1814726,348549356~6937075,365997237~1542191,392519231~4064300,396670010~6408714] 2024-09-13T00:39:08.762 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [5335581~3339840,22666579~2303167,65727373~6163988,78799742~8141162,105459787~1424206,122043306~5052094,128451845~308073,130261050~4316483,159230705~5201250,169300267~4108516,180376314~3885648,206691828~1354340,220044632~9089487,240828348~10753127,266675149~244059,273904834~1135501,282726102~8324834,292988707~5809424,322862850~5290900,335934186~1814726,348549356~6937075,365997237~1542191,392519231~4064300,396670010~6408714] 2024-09-13T00:39:08.767 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [5335581~3339840,22666579~2303167,65727373~6163988,78799742~8141162,104857600~2026393,121634816~5460584,128451845~308073,130261050~4316483,159230705~5201250,169300267~4108516,180376314~3885648,206691828~1354340,218740863~10393256,240828348~10753127,266675149~244059,272629760~2410575,282726102~8324834,292988707~5809424,322862850~5290900,335934186~1814726,348549356~6937075,365997237~1542191,392519231~4064300,396670010~6408714] 2024-09-13T00:39:08.767 INFO:tasks.workunit.client.0.smithi136.stdout:from 8 to 12 diff [5335581~7919612,22666579~9651148,62948307~8943054,78799742~8141162,95048390~7435958,105459787~1424206,122043306~5052094,128451845~308073,130261050~4316483,159230705~5201250,169300267~4108516,180376314~3885648,206691828~1354340,220044632~9089487,240828348~10753127,266675149~244059,273904834~1135501,277779043~4483937,282726102~8324834,292988707~5809424,312566619~2948229,316301726~3511210,322862850~12451153,335934186~1814726,348549356~6937075,365997237~5182427,379083185~7266719,392519231~4064300,396670010~6408714] 2024-09-13T00:39:08.768 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [5335581~7919612,22666579~9651148,62948307~8943054,78799742~8141162,95048390~7435958,105459787~1424206,122043306~5052094,128451845~308073,130261050~4316483,159230705~5201250,169300267~4108516,180376314~3885648,206691828~1354340,220044632~9089487,240828348~10753127,266675149~244059,273904834~1135501,277779043~4483937,282726102~8324834,292988707~5809424,312566619~2948229,316301726~3511210,322862850~12451153,335934186~1814726,348549356~6937075,365997237~5182427,379083185~7266719,392519231~4064300,396670010~6408714] 2024-09-13T00:39:08.774 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [5335581~7919612,22666579~9651148,62948307~8943054,78799742~8141162,95048390~7435958,104857600~2026393,121634816~5460584,128451845~308073,130261050~4316483,159230705~5201250,169300267~4108516,180376314~3885648,206691828~1354340,218740863~10393256,240828348~10753127,266675149~244059,272629760~2410575,277779043~4483937,282726102~8324834,292988707~5809424,310378496~5136352,316301726~3511210,322862850~12451153,335934186~1814726,348549356~6937075,365997237~5182427,379083185~7266719,392519231~4064300,396670010~6408714] 2024-09-13T00:39:08.774 INFO:tasks.workunit.client.0.smithi136.stdout:from 8 to 13 diff [5335581~7919612,17113294~3323835,22666579~9651148,62948307~8943054,78799742~8141162,95048390~7435958,105459787~1424206,122043306~5052094,128451845~308073,130261050~4316483,159230705~7411964,169300267~4108516,173851851~84284,180376314~3885648,187369573~871966,201556540~2015504,206691828~1354340,209974378~1903654,220044632~9089487,240828348~10753127,266675149~244059,273904834~1135501,277779043~4483937,282726102~8324834,292988707~7624769,312566619~2948229,316301726~3511210,322862850~12451153,335934186~1814726,348549356~6937075,365997237~5182427,379083185~7266719,392519231~4064300,396670010~6408714] 2024-09-13T00:39:08.774 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [5335581~7919612,17113294~3323835,22666579~9651148,62948307~8943054,78799742~8141162,95048390~7435958,105459787~1424206,122043306~5052094,128451845~308073,130261050~4316483,159230705~7411964,169300267~4108516,173851851~84284,180376314~3885648,187369573~871966,201556540~2015504,206691828~1354340,209974378~1903654,220044632~9089487,240828348~10753127,266675149~244059,273904834~1135501,277779043~4483937,282726102~8324834,292988707~7624769,312566619~2948229,316301726~3511210,322862850~12451153,335934186~1814726,348549356~6937075,365997237~5182427,379083185~7266719,392519231~4064300,396670010~6408714] 2024-09-13T00:39:08.780 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [5335581~7919612,17113294~3323835,22666579~9651148,62948307~8943054,78799742~8141162,95048390~7435958,104857600~2026393,121634816~5460584,128451845~308073,130261050~4316483,159230705~7411964,169300267~4108516,173851851~84284,180376314~3885648,184549376~3692163,201556540~2015504,206691828~1354340,209974378~1903654,218740863~10393256,240828348~10753127,266675149~244059,272629760~2410575,277779043~4483937,282726102~8324834,292988707~7624769,310378496~5136352,316301726~3511210,322862850~12451153,335934186~1814726,348549356~6937075,365997237~5182427,379083185~7266719,392519231~4064300,396670010~6408714] 2024-09-13T00:39:08.780 INFO:tasks.workunit.client.0.smithi136.stdout:from 8 to 14 diff [978366~12276827,17113294~3323835,22666579~10131102,62948307~8943054,78799742~8141162,95048390~7435958,105459787~1424206,122043306~5052094,128451845~308073,130261050~4316483,159230705~7411964,169300267~4108516,173851851~84284,180376314~3885648,187369573~871966,201556540~2015504,206691828~6088778,220044632~9089487,240828348~10753127,266675149~244059,273904834~1135501,277779043~4483937,282726102~8324834,292988707~7952502,312566619~2948229,316301726~3511210,322862850~12451153,335934186~1814726,348549356~6937075,365490113~5689551,371762046~1637913,379083185~7266719,389872420~6711111,396670010~6408714] 2024-09-13T00:39:08.780 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~3323835,22666579~10131102,62948307~8943054,78799742~8141162,95048390~7435958,105459787~1424206,122043306~5052094,128451845~308073,130261050~4316483,159230705~7411964,169300267~4108516,173851851~84284,180376314~3885648,187369573~871966,201556540~2015504,206691828~6088778,220044632~9089487,240828348~10753127,266675149~244059,273904834~1135501,277779043~4483937,282726102~8324834,292988707~7952502,312566619~2948229,316301726~3511210,322862850~12451153,335934186~1814726,348549356~6937075,365490113~5689551,371762046~1637913,379083185~7266719,389872420~6711111,396670010~6408714] 2024-09-13T00:39:08.786 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12276827,17113294~3323835,22666579~10131102,62948307~8943054,78799742~8141162,95048390~7435958,104857600~2026393,121634816~5460584,128451845~308073,130261050~4316483,159230705~7411964,169300267~4108516,173851851~84284,180376314~3885648,184549376~3692163,201556540~2015504,206691828~6088778,218740863~10393256,240828348~10753127,266675149~244059,272629760~2410575,277779043~4483937,282726102~8324834,292988707~7952502,310378496~5136352,316301726~3511210,322862850~12451153,335934186~1814726,348549356~6937075,365490113~5689551,371762046~1637913,379083185~7266719,389872420~6711111,396670010~6408714] 2024-09-13T00:39:08.786 INFO:tasks.workunit.client.0.smithi136.stdout:from 8 to 15 diff [978366~12276827,17113294~3323835,22666579~10131102,62948307~8943054,78799742~8141162,95048390~7435958,105459787~1424206,122043306~5052094,128451845~308073,130261050~4316483,144141427~2676660,148016038~10362739,159230705~7411964,169300267~8711361,180376314~3885648,187369573~871966,192121569~7452224,201556540~2015504,206691828~6088778,220044632~9089487,240828348~10753127,266675149~244059,273904834~1135501,277779043~4483937,282726102~8324834,292988707~7952502,312566619~2948229,316301726~3511210,322862850~12451153,335934186~1814726,348299711~7186720,365490113~5689551,371762046~1637913,379083185~7266719,389872420~6711111,396670010~6408714] 2024-09-13T00:39:08.786 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~3323835,22666579~10131102,62948307~8943054,78799742~8141162,95048390~7435958,105459787~1424206,122043306~5052094,128451845~308073,130261050~4316483,144141427~2676660,148016038~10362739,159230705~7411964,169300267~8711361,180376314~3885648,187369573~871966,192121569~7452224,201556540~2015504,206691828~6088778,220044632~9089487,240828348~10753127,266675149~244059,273904834~1135501,277779043~4483937,282726102~8324834,292988707~7952502,312566619~2948229,316301726~3511210,322862850~12451153,335934186~1814726,348299711~7186720,365490113~5689551,371762046~1637913,379083185~7266719,389872420~6711111,396670010~6408714] 2024-09-13T00:39:08.793 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12276827,17113294~3323835,22666579~10131102,62948307~8943054,78799742~8141162,95048390~7435958,104857600~2026393,121634816~5460584,128451845~308073,130261050~4316483,144141427~2676660,148016038~10362739,159230705~7411964,169300267~8711361,180376314~3885648,184549376~3692163,188743680~10830113,201556540~2015504,206691828~6088778,218740863~10393256,240828348~10753127,266675149~244059,272629760~2410575,277779043~4483937,282726102~8324834,292988707~7952502,310378496~5136352,316301726~3511210,322862850~12451153,335934186~1814726,348299711~7186720,365490113~5689551,371762046~1637913,379083185~7266719,389872420~6711111,396670010~6408714] 2024-09-13T00:39:08.793 INFO:tasks.workunit.client.0.smithi136.stdout:from 8 to 16 diff [978366~12276827,17113294~3323835,22666579~10131102,48635901~7849557,62637940~9253421,78799742~8141162,95048390~7435958,105459787~1424206,122043306~5052094,128451845~308073,130261050~7020755,144141427~2676660,148016038~18626631,169300267~8711361,180376314~3885648,187369573~871966,192121569~7452224,201556540~2015504,206691828~6088778,220044632~9819324,240828348~10753127,266675149~244059,273904834~1135501,277779043~4483937,282726102~8324834,292988707~7952502,312566619~2948229,316301726~3511210,322862850~12451153,335934186~1814726,344823952~10662479,365490113~5689551,371762046~1637913,379003744~7346160,389872420~6711111,396670010~6408714] 2024-09-13T00:39:08.793 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~3323835,22666579~10131102,48635901~7849557,62637940~9253421,78799742~8141162,95048390~7435958,105459787~1424206,122043306~5052094,128451845~308073,130261050~7020755,144141427~2676660,148016038~18626631,169300267~8711361,180376314~3885648,187369573~871966,192121569~7452224,201556540~2015504,206691828~6088778,220044632~9819324,240828348~10753127,266675149~244059,273904834~1135501,277779043~4483937,282726102~8324834,292988707~7952502,312566619~2948229,316301726~3511210,322862850~12451153,335934186~1814726,344823952~10662479,365490113~5689551,371762046~1637913,379003744~7346160,389872420~6711111,396670010~6408714] 2024-09-13T00:39:08.800 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12276827,17113294~3323835,22666579~10131102,48635901~7849557,62637940~9253421,78799742~8141162,95048390~7435958,104857600~2026393,121634816~5460584,128451845~308073,130261050~7020755,144141427~2676660,148016038~18626631,169300267~8711361,180376314~3885648,184549376~3692163,188743680~10830113,201556540~2015504,206691828~6088778,218740863~11123093,240828348~10753127,266675149~244059,272629760~2410575,277779043~4483937,282726102~8324834,292988707~7952502,310378496~5136352,316301726~3511210,322862850~12451153,335934186~1814726,344823952~10662479,365490113~5689551,371762046~1637913,379003744~7346160,389872420~6711111,396670010~6408714] 2024-09-13T00:39:08.800 INFO:tasks.workunit.client.0.smithi136.stdout:from 8 to 17 diff [978366~12276827,17113294~3323835,22666579~10131102,37199402~669117,48635901~7849557,58865372~13025989,78799742~8141162,95048390~7435958,105459787~1424206,122043306~5052094,128451845~308073,130261050~7020755,144141427~2676660,148016038~18626631,169300267~8711361,180376314~3885648,187369573~871966,192121569~7452224,201556540~11224066,215619334~2003239,220044632~14346631,240828348~10753127,252886187~622714,266675149~244059,273904834~1135501,277779043~4483937,282726102~8324834,292988707~7952502,312566619~2948229,316301726~3511210,322862850~12451153,335934186~1814726,344823952~10662479,365490113~5689551,371762046~1637913,379003744~7346160,389458074~13620650] 2024-09-13T00:39:08.800 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~3323835,22666579~10131102,37199402~669117,48635901~7849557,58865372~13025989,78799742~8141162,95048390~7435958,105459787~1424206,122043306~5052094,128451845~308073,130261050~7020755,144141427~2676660,148016038~18626631,169300267~8711361,180376314~3885648,187369573~871966,192121569~7452224,201556540~11224066,215619334~2003239,220044632~14346631,240828348~10753127,252886187~622714,266675149~244059,273904834~1135501,277779043~4483937,282726102~8324834,292988707~7952502,312566619~2948229,316301726~3511210,322862850~12451153,335934186~1814726,344823952~10662479,365490113~5689551,371762046~1637913,379003744~7346160,389458074~13620650] 2024-09-13T00:39:08.806 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12276827,17113294~3323835,22666579~10131102,37199402~669117,48635901~7849557,58865372~13025989,78799742~8141162,95048390~7435958,104857600~2026393,121634816~5460584,128451845~308073,130261050~7020755,144141427~2676660,148016038~18626631,169300267~8711361,180376314~3885648,184549376~3692163,188743680~10830113,201556540~11224066,215619334~2003239,218740863~15650400,240828348~10753127,252237477~1271424,266675149~244059,272629760~2410575,277779043~4483937,282726102~8324834,292988707~7952502,310378496~5136352,316301726~3511210,322862850~12451153,335934186~1814726,344823952~10662479,365490113~5689551,371762046~1637913,379003744~7346160,389458074~13620650] 2024-09-13T00:39:08.806 INFO:tasks.workunit.client.0.smithi136.stdout:from 8 to 18 diff [978366~12276827,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,95048390~7435958,105459787~1424206,122043306~5052094,128451845~308073,130261050~7020755,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180376314~3885648,187369573~871966,192121569~7452224,201556540~16427001,220044632~14346631,240828348~10753127,252886187~622714,266675149~244059,273904834~1135501,277779043~4483937,282726102~8324834,292988707~7952502,312566619~2948229,316301726~3511210,322862850~12451153,335934186~1814726,344823952~10662479,365490113~10144650,379003744~7346160,389458074~13620650] 2024-09-13T00:39:08.806 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,95048390~7435958,105459787~1424206,122043306~5052094,128451845~308073,130261050~7020755,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180376314~3885648,187369573~871966,192121569~7452224,201556540~16427001,220044632~14346631,240828348~10753127,252886187~622714,266675149~244059,273904834~1135501,277779043~4483937,282726102~8324834,292988707~7952502,312566619~2948229,316301726~3511210,322862850~12451153,335934186~1814726,344823952~10662479,365490113~10144650,379003744~7346160,389458074~13620650] 2024-09-13T00:39:08.813 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12276827,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,95048390~7435958,104857600~2026393,121634816~5460584,128451845~308073,130261050~7020755,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180376314~3885648,184549376~3692163,188743680~10830113,201556540~16427001,218740863~15650400,240828348~10753127,252237477~1271424,266675149~244059,272629760~2410575,277779043~4483937,282726102~8324834,292988707~7952502,310378496~5136352,316301726~3511210,322862850~12451153,335934186~1814726,344823952~10662479,365490113~10144650,379003744~7346160,389458074~13620650] 2024-09-13T00:39:08.813 INFO:tasks.workunit.client.0.smithi136.stdout:from 8 to 19 diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180376314~3885648,187369573~871966,192121569~7452224,201556540~16427001,220044632~14346631,240828348~10753127,252886187~622714,266675149~244059,273064912~4591743,277779043~4483937,282726102~8324834,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:08.813 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180376314~3885648,187369573~871966,192121569~7452224,201556540~16427001,220044632~14346631,240828348~10753127,252886187~622714,266675149~244059,273064912~4591743,277779043~4483937,282726102~8324834,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:08.820 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180376314~3885648,184549376~3692163,188743680~10830113,201556540~16427001,218740863~15650400,240828348~10753127,252237477~1271424,266675149~244059,272629760~5026895,277779043~4483937,282726102~8324834,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,344823952~15725891,360710144~14924619,379003744~7346160,389458074~13620650] 2024-09-13T00:39:08.820 INFO:tasks.workunit.client.0.smithi136.stdout:from 9 to 10 diff [22666579~2303167,65727373~6163988,78799742~5691133,128451845~308073,222846411~6287708,348549356~6937075,365997237~1542191,392519231~4064300] 2024-09-13T00:39:08.820 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [22666579~2303167,65727373~6163988,78799742~5691133,128451845~308073,222846411~6287708,348549356~6937075,365997237~1542191,392519231~4064300] 2024-09-13T00:39:08.826 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [22666579~2303167,65727373~6163988,78799742~5691133,128451845~308073,222846411~6287708,348549356~6937075,365997237~1542191,392519231~4064300] 2024-09-13T00:39:08.826 INFO:tasks.workunit.client.0.smithi136.stdout:from 9 to 11 diff [5335581~3339840,22666579~2303167,65727373~6163988,78799742~5691133,105459787~1424206,122043306~5052094,128451845~308073,130261050~4316483,222846411~6287708,240828348~6357952,282726102~2562064,292988707~5809424,322862850~5290900,335934186~1814726,348549356~6937075,365997237~1542191,392519231~4064300,396670010~6408714] 2024-09-13T00:39:08.826 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [5335581~3339840,22666579~2303167,65727373~6163988,78799742~5691133,105459787~1424206,122043306~5052094,128451845~308073,130261050~4316483,222846411~6287708,240828348~6357952,282726102~2562064,292988707~5809424,322862850~5290900,335934186~1814726,348549356~6937075,365997237~1542191,392519231~4064300,396670010~6408714] 2024-09-13T00:39:08.836 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [5335581~3339840,22666579~2303167,65727373~6163988,78799742~5691133,104857600~2026393,121634816~5460584,128451845~308073,130261050~4316483,222846411~6287708,240828348~6357952,282726102~2562064,292988707~5809424,322862850~5290900,335934186~1814726,348549356~6937075,365997237~1542191,392519231~4064300,396670010~6408714] 2024-09-13T00:39:08.836 INFO:tasks.workunit.client.0.smithi136.stdout:from 9 to 12 diff [5335581~7919612,22666579~9651148,62948307~8943054,78799742~5691133,95048390~7435958,105459787~1424206,122043306~5052094,128451845~308073,130261050~4316483,222846411~6287708,240828348~6357952,277779043~4483937,282726102~2562064,292988707~5809424,312566619~2948229,316301726~3511210,322862850~12451153,335934186~1814726,348549356~6937075,365997237~5182427,379083185~7266719,392519231~4064300,396670010~6408714] 2024-09-13T00:39:08.836 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [5335581~7919612,22666579~9651148,62948307~8943054,78799742~5691133,95048390~7435958,105459787~1424206,122043306~5052094,128451845~308073,130261050~4316483,222846411~6287708,240828348~6357952,277779043~4483937,282726102~2562064,292988707~5809424,312566619~2948229,316301726~3511210,322862850~12451153,335934186~1814726,348549356~6937075,365997237~5182427,379083185~7266719,392519231~4064300,396670010~6408714] 2024-09-13T00:39:08.843 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [5335581~7919612,22666579~9651148,62948307~8943054,78799742~5691133,95048390~7435958,104857600~2026393,121634816~5460584,128451845~308073,130261050~4316483,222846411~6287708,240828348~6357952,277779043~4483937,282726102~2562064,292988707~5809424,310378496~5136352,316301726~3511210,322862850~12451153,335934186~1814726,348549356~6937075,365997237~5182427,379083185~7266719,392519231~4064300,396670010~6408714] 2024-09-13T00:39:08.843 INFO:tasks.workunit.client.0.smithi136.stdout:from 9 to 13 diff [5335581~7919612,17113294~3323835,22666579~9651148,62948307~8943054,78799742~5691133,95048390~7435958,105459787~1424206,122043306~5052094,128451845~308073,130261050~4316483,159255557~7387112,173851851~84284,187369573~871966,201556540~2015504,209974378~1903654,222846411~6287708,240828348~6357952,277779043~4483937,282726102~2562064,292988707~7624769,312566619~2948229,316301726~3511210,322862850~12451153,335934186~1814726,348549356~6937075,365997237~5182427,379083185~7266719,392519231~4064300,396670010~6408714] 2024-09-13T00:39:08.843 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [5335581~7919612,17113294~3323835,22666579~9651148,62948307~8943054,78799742~5691133,95048390~7435958,105459787~1424206,122043306~5052094,128451845~308073,130261050~4316483,159255557~7387112,173851851~84284,187369573~871966,201556540~2015504,209974378~1903654,222846411~6287708,240828348~6357952,277779043~4483937,282726102~2562064,292988707~7624769,312566619~2948229,316301726~3511210,322862850~12451153,335934186~1814726,348549356~6937075,365997237~5182427,379083185~7266719,392519231~4064300,396670010~6408714] 2024-09-13T00:39:08.854 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [5335581~7919612,17113294~3323835,22666579~9651148,62948307~8943054,78799742~5691133,95048390~7435958,104857600~2026393,121634816~5460584,128451845~308073,130261050~4316483,159255557~7387112,173851851~84284,184549376~3692163,201556540~2015504,209974378~1903654,222846411~6287708,240828348~6357952,277779043~4483937,282726102~2562064,292988707~7624769,310378496~5136352,316301726~3511210,322862850~12451153,335934186~1814726,348549356~6937075,365997237~5182427,379083185~7266719,392519231~4064300,396670010~6408714] 2024-09-13T00:39:08.854 INFO:tasks.workunit.client.0.smithi136.stdout:from 9 to 14 diff [978366~12276827,17113294~3323835,22666579~10131102,62948307~8943054,78799742~5691133,95048390~7435958,105459787~1424206,122043306~5052094,128451845~308073,130261050~4316483,159255557~7387112,173851851~84284,187369573~871966,201556540~2015504,207131072~5649534,222846411~6287708,240828348~6357952,277779043~4483937,282726102~2562064,292988707~7952502,312566619~2948229,316301726~3511210,322862850~12451153,335934186~1814726,348549356~6937075,365490113~5689551,371762046~1637913,379083185~7266719,389872420~6711111,396670010~6408714] 2024-09-13T00:39:08.854 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~3323835,22666579~10131102,62948307~8943054,78799742~5691133,95048390~7435958,105459787~1424206,122043306~5052094,128451845~308073,130261050~4316483,159255557~7387112,173851851~84284,187369573~871966,201556540~2015504,207131072~5649534,222846411~6287708,240828348~6357952,277779043~4483937,282726102~2562064,292988707~7952502,312566619~2948229,316301726~3511210,322862850~12451153,335934186~1814726,348549356~6937075,365490113~5689551,371762046~1637913,379083185~7266719,389872420~6711111,396670010~6408714] 2024-09-13T00:39:08.861 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12276827,17113294~3323835,22666579~10131102,62948307~8943054,78799742~5691133,95048390~7435958,104857600~2026393,121634816~5460584,128451845~308073,130261050~4316483,159255557~7387112,173851851~84284,184549376~3692163,201556540~2015504,207131072~5649534,222846411~6287708,240828348~6357952,277779043~4483937,282726102~2562064,292988707~7952502,310378496~5136352,316301726~3511210,322862850~12451153,335934186~1814726,348549356~6937075,365490113~5689551,371762046~1637913,379083185~7266719,389872420~6711111,396670010~6408714] 2024-09-13T00:39:08.861 INFO:tasks.workunit.client.0.smithi136.stdout:from 9 to 15 diff [978366~12276827,17113294~3323835,22666579~10131102,62948307~8943054,78799742~5691133,95048390~7435958,105459787~1424206,122043306~5052094,128451845~308073,130261050~4316483,144141427~2676660,148016038~10362739,159255557~7387112,171460898~6550730,187369573~871966,192121569~7452224,201556540~2015504,207131072~5649534,222846411~6287708,240828348~6357952,248606422~493001,277779043~4483937,282726102~2562064,292988707~7952502,312566619~2948229,316301726~3511210,322862850~12451153,335934186~1814726,348299711~7186720,365490113~5689551,371762046~1637913,379083185~7266719,389872420~6711111,396670010~6408714] 2024-09-13T00:39:08.861 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~3323835,22666579~10131102,62948307~8943054,78799742~5691133,95048390~7435958,105459787~1424206,122043306~5052094,128451845~308073,130261050~4316483,144141427~2676660,148016038~10362739,159255557~7387112,171460898~6550730,187369573~871966,192121569~7452224,201556540~2015504,207131072~5649534,222846411~6287708,240828348~6357952,248606422~493001,277779043~4483937,282726102~2562064,292988707~7952502,312566619~2948229,316301726~3511210,322862850~12451153,335934186~1814726,348299711~7186720,365490113~5689551,371762046~1637913,379083185~7266719,389872420~6711111,396670010~6408714] 2024-09-13T00:39:08.867 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12276827,17113294~3323835,22666579~10131102,62948307~8943054,78799742~5691133,95048390~7435958,104857600~2026393,121634816~5460584,128451845~308073,130261050~4316483,144141427~2676660,148016038~10362739,159255557~7387112,171460898~6550730,184549376~3692163,188743680~10830113,201556540~2015504,207131072~5649534,222846411~6287708,240828348~6357952,248606422~493001,277779043~4483937,282726102~2562064,292988707~7952502,310378496~5136352,316301726~3511210,322862850~12451153,335934186~1814726,348299711~7186720,365490113~5689551,371762046~1637913,379083185~7266719,389872420~6711111,396670010~6408714] 2024-09-13T00:39:08.867 INFO:tasks.workunit.client.0.smithi136.stdout:from 9 to 16 diff [978366~12276827,17113294~3323835,22666579~10131102,48635901~7849557,62637940~9253421,78799742~5691133,95048390~7435958,105459787~1424206,122043306~5052094,128451845~308073,130261050~7020755,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~2015504,207131072~5649534,222829084~7034872,240828348~6357952,248606422~493001,277779043~4483937,282726102~2562064,292988707~7952502,312566619~2948229,316301726~3511210,322862850~12451153,335934186~1814726,344823952~10662479,365490113~5689551,371762046~1637913,379003744~7346160,389872420~6711111,396670010~6408714] 2024-09-13T00:39:08.867 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~3323835,22666579~10131102,48635901~7849557,62637940~9253421,78799742~5691133,95048390~7435958,105459787~1424206,122043306~5052094,128451845~308073,130261050~7020755,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~2015504,207131072~5649534,222829084~7034872,240828348~6357952,248606422~493001,277779043~4483937,282726102~2562064,292988707~7952502,312566619~2948229,316301726~3511210,322862850~12451153,335934186~1814726,344823952~10662479,365490113~5689551,371762046~1637913,379003744~7346160,389872420~6711111,396670010~6408714] 2024-09-13T00:39:08.877 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12276827,17113294~3323835,22666579~10131102,48635901~7849557,62637940~9253421,78799742~5691133,95048390~7435958,104857600~2026393,121634816~5460584,128451845~308073,130261050~7020755,144141427~2676660,148016038~18626631,171460898~6550730,184549376~3692163,188743680~10830113,201556540~2015504,207131072~5649534,222829084~7034872,240828348~6357952,248606422~493001,277779043~4483937,282726102~2562064,292988707~7952502,310378496~5136352,316301726~3511210,322862850~12451153,335934186~1814726,344823952~10662479,365490113~5689551,371762046~1637913,379003744~7346160,389872420~6711111,396670010~6408714] 2024-09-13T00:39:08.877 INFO:tasks.workunit.client.0.smithi136.stdout:from 9 to 17 diff [978366~12276827,17113294~3323835,22666579~10131102,37199402~669117,48635901~7849557,58865372~13025989,78799742~5691133,95048390~7435958,105459787~1424206,122043306~5052094,128451845~308073,130261050~7020755,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~11224066,215619334~2003239,222829084~11562179,240828348~6357952,248606422~493001,252886187~622714,277779043~4483937,282726102~2562064,292988707~7952502,312566619~2948229,316301726~3511210,322862850~12451153,335934186~1814726,344823952~10662479,365490113~5689551,371762046~1637913,379003744~7346160,389458074~13620650] 2024-09-13T00:39:08.877 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~3323835,22666579~10131102,37199402~669117,48635901~7849557,58865372~13025989,78799742~5691133,95048390~7435958,105459787~1424206,122043306~5052094,128451845~308073,130261050~7020755,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~11224066,215619334~2003239,222829084~11562179,240828348~6357952,248606422~493001,252886187~622714,277779043~4483937,282726102~2562064,292988707~7952502,312566619~2948229,316301726~3511210,322862850~12451153,335934186~1814726,344823952~10662479,365490113~5689551,371762046~1637913,379003744~7346160,389458074~13620650] 2024-09-13T00:39:08.880 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12276827,17113294~3323835,22666579~10131102,37199402~669117,48635901~7849557,58865372~13025989,78799742~5691133,95048390~7435958,104857600~2026393,121634816~5460584,128451845~308073,130261050~7020755,144141427~2676660,148016038~18626631,171460898~6550730,184549376~3692163,188743680~10830113,201556540~11224066,215619334~2003239,222829084~11562179,240828348~6357952,248606422~493001,252237477~1271424,277779043~4483937,282726102~2562064,292988707~7952502,310378496~5136352,316301726~3511210,322862850~12451153,335934186~1814726,344823952~10662479,365490113~5689551,371762046~1637913,379003744~7346160,389458074~13620650] 2024-09-13T00:39:08.880 INFO:tasks.workunit.client.0.smithi136.stdout:from 9 to 18 diff [978366~12276827,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~8291936,95048390~7435958,105459787~1424206,122043306~5052094,128451845~308073,130261050~7020755,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252886187~622714,277779043~4483937,282726102~2562064,292988707~7952502,312566619~2948229,316301726~3511210,322862850~12451153,335934186~1814726,344823952~10662479,365490113~10144650,379003744~7346160,389458074~13620650] 2024-09-13T00:39:08.880 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~8291936,95048390~7435958,105459787~1424206,122043306~5052094,128451845~308073,130261050~7020755,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252886187~622714,277779043~4483937,282726102~2562064,292988707~7952502,312566619~2948229,316301726~3511210,322862850~12451153,335934186~1814726,344823952~10662479,365490113~10144650,379003744~7346160,389458074~13620650] 2024-09-13T00:39:08.886 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12276827,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~8291936,95048390~7435958,104857600~2026393,121634816~5460584,128451845~308073,130261050~7020755,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,184549376~3692163,188743680~10830113,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252237477~1271424,277779043~4483937,282726102~2562064,292988707~7952502,310378496~5136352,316301726~3511210,322862850~12451153,335934186~1814726,344823952~10662479,365490113~10144650,379003744~7346160,389458074~13620650] 2024-09-13T00:39:08.886 INFO:tasks.workunit.client.0.smithi136.stdout:from 9 to 19 diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~8291936,91032793~11451555,105459787~1424206,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:08.886 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~8291936,91032793~11451555,105459787~1424206,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:08.892 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~8291936,91032793~11451555,104857600~2026393,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,184549376~3692163,188743680~10830113,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252237477~1271424,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,344823952~15725891,360710144~14924619,379003744~7346160,389458074~13620650] 2024-09-13T00:39:08.892 INFO:tasks.workunit.client.0.smithi136.stdout:from 10 to 11 diff [5335581~3339840,105459787~1424206,122043306~5052094,130261050~4316483,240828348~6357952,282726102~2562064,292988707~5809424,322862850~5290900,335934186~1814726,396670010~6408714] 2024-09-13T00:39:08.893 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [5335581~3339840,105459787~1424206,122043306~5052094,130261050~4316483,240828348~6357952,282726102~2562064,292988707~5809424,322862850~5290900,335934186~1814726,396670010~6408714] 2024-09-13T00:39:08.898 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [5335581~3339840,104857600~2026393,121634816~5460584,130261050~4316483,240828348~6357952,282726102~2562064,292988707~5809424,322862850~5290900,335934186~1814726,396670010~6408714] 2024-09-13T00:39:08.898 INFO:tasks.workunit.client.0.smithi136.stdout:from 10 to 12 diff [5335581~7919612,24769299~7548428,62948307~5586522,95048390~7435958,105459787~1424206,122043306~5052094,130261050~4316483,240828348~6357952,277779043~4483937,282726102~2562064,292988707~5809424,312566619~2948229,316301726~3511210,322862850~12451153,335934186~1814726,366458829~4720835,379083185~7266719,396670010~6408714] 2024-09-13T00:39:08.898 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [5335581~7919612,24769299~7548428,62948307~5586522,95048390~7435958,105459787~1424206,122043306~5052094,130261050~4316483,240828348~6357952,277779043~4483937,282726102~2562064,292988707~5809424,312566619~2948229,316301726~3511210,322862850~12451153,335934186~1814726,366458829~4720835,379083185~7266719,396670010~6408714] 2024-09-13T00:39:08.904 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [5335581~7919612,24769299~7548428,62948307~5586522,95048390~7435958,104857600~2026393,121634816~5460584,130261050~4316483,240828348~6357952,277779043~4483937,282726102~2562064,292988707~5809424,310378496~5136352,316301726~3511210,322862850~12451153,335934186~1814726,366458829~4720835,379083185~7266719,396670010~6408714] 2024-09-13T00:39:08.904 INFO:tasks.workunit.client.0.smithi136.stdout:from 10 to 13 diff [5335581~7919612,17113294~3323835,22837062~9480665,62948307~8638091,95048390~7435958,105459787~1424206,122043306~5052094,130261050~4316483,159255557~7387112,173851851~84284,187369573~871966,201556540~2015504,209974378~1903654,240828348~6357952,277779043~4483937,282726102~2562064,292988707~7624769,312566619~2948229,316301726~3511210,322862850~12451153,335934186~1814726,366458829~4720835,379083185~7266719,396670010~6408714] 2024-09-13T00:39:08.904 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [5335581~7919612,17113294~3323835,22837062~9480665,62948307~8638091,95048390~7435958,105459787~1424206,122043306~5052094,130261050~4316483,159255557~7387112,173851851~84284,187369573~871966,201556540~2015504,209974378~1903654,240828348~6357952,277779043~4483937,282726102~2562064,292988707~7624769,312566619~2948229,316301726~3511210,322862850~12451153,335934186~1814726,366458829~4720835,379083185~7266719,396670010~6408714] 2024-09-13T00:39:08.910 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [5335581~7919612,17113294~3323835,22837062~9480665,62948307~8638091,95048390~7435958,104857600~2026393,121634816~5460584,130261050~4316483,159255557~7387112,173851851~84284,184549376~3692163,201556540~2015504,209974378~1903654,240828348~6357952,277779043~4483937,282726102~2562064,292988707~7624769,310378496~5136352,316301726~3511210,322862850~12451153,335934186~1814726,366458829~4720835,379083185~7266719,396670010~6408714] 2024-09-13T00:39:08.910 INFO:tasks.workunit.client.0.smithi136.stdout:from 10 to 14 diff [978366~12276827,17113294~3323835,22837062~9960619,62948307~8638091,95048390~7435958,105459787~1424206,122043306~5052094,130261050~4316483,159255557~7387112,173851851~84284,187369573~871966,201556540~2015504,207131072~5649534,240828348~6357952,277779043~4483937,282726102~2562064,292988707~7952502,312566619~2948229,316301726~3511210,322862850~12451153,335934186~1814726,365490113~5689551,371762046~1637913,379083185~7266719,389872420~4206259,396670010~6408714] 2024-09-13T00:39:08.910 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~3323835,22837062~9960619,62948307~8638091,95048390~7435958,105459787~1424206,122043306~5052094,130261050~4316483,159255557~7387112,173851851~84284,187369573~871966,201556540~2015504,207131072~5649534,240828348~6357952,277779043~4483937,282726102~2562064,292988707~7952502,312566619~2948229,316301726~3511210,322862850~12451153,335934186~1814726,365490113~5689551,371762046~1637913,379083185~7266719,389872420~4206259,396670010~6408714] 2024-09-13T00:39:08.916 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12276827,17113294~3323835,22837062~9960619,62948307~8638091,95048390~7435958,104857600~2026393,121634816~5460584,130261050~4316483,159255557~7387112,173851851~84284,184549376~3692163,201556540~2015504,207131072~5649534,240828348~6357952,277779043~4483937,282726102~2562064,292988707~7952502,310378496~5136352,316301726~3511210,322862850~12451153,335934186~1814726,365490113~5689551,371762046~1637913,379083185~7266719,389872420~4206259,396670010~6408714] 2024-09-13T00:39:08.916 INFO:tasks.workunit.client.0.smithi136.stdout:from 10 to 15 diff [978366~12276827,17113294~3323835,22837062~9960619,62948307~8638091,95048390~7435958,105459787~1424206,122043306~5052094,130261050~4316483,144141427~2676660,148016038~10362739,159255557~7387112,171460898~6550730,187369573~871966,192121569~7452224,201556540~2015504,207131072~5649534,240828348~6357952,248606422~493001,277779043~4483937,282726102~2562064,292988707~7952502,312566619~2948229,316301726~3511210,322862850~12451153,335934186~1814726,348299711~4603277,365490113~5689551,371762046~1637913,379083185~7266719,389872420~4206259,396670010~6408714] 2024-09-13T00:39:08.916 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~3323835,22837062~9960619,62948307~8638091,95048390~7435958,105459787~1424206,122043306~5052094,130261050~4316483,144141427~2676660,148016038~10362739,159255557~7387112,171460898~6550730,187369573~871966,192121569~7452224,201556540~2015504,207131072~5649534,240828348~6357952,248606422~493001,277779043~4483937,282726102~2562064,292988707~7952502,312566619~2948229,316301726~3511210,322862850~12451153,335934186~1814726,348299711~4603277,365490113~5689551,371762046~1637913,379083185~7266719,389872420~4206259,396670010~6408714] 2024-09-13T00:39:08.922 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12276827,17113294~3323835,22837062~9960619,62948307~8638091,95048390~7435958,104857600~2026393,121634816~5460584,130261050~4316483,144141427~2676660,148016038~10362739,159255557~7387112,171460898~6550730,184549376~3692163,188743680~10830113,201556540~2015504,207131072~5649534,240828348~6357952,248606422~493001,277779043~4483937,282726102~2562064,292988707~7952502,310378496~5136352,316301726~3511210,322862850~12451153,335934186~1814726,348299711~4603277,365490113~5689551,371762046~1637913,379083185~7266719,389872420~4206259,396670010~6408714] 2024-09-13T00:39:08.922 INFO:tasks.workunit.client.0.smithi136.stdout:from 10 to 16 diff [978366~12276827,17113294~3323835,22837062~9960619,48635901~7849557,62637940~8948458,95048390~7435958,105459787~1424206,122043306~5052094,130261050~7020755,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~2015504,207131072~5649534,222829084~7034872,240828348~6357952,248606422~493001,277779043~4483937,282726102~2562064,292988707~7952502,312566619~2948229,316301726~3511210,322862850~12451153,335934186~1814726,344823952~8079036,365490113~5689551,371762046~1637913,379003744~7346160,389872420~4206259,396670010~6408714] 2024-09-13T00:39:08.922 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~3323835,22837062~9960619,48635901~7849557,62637940~8948458,95048390~7435958,105459787~1424206,122043306~5052094,130261050~7020755,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~2015504,207131072~5649534,222829084~7034872,240828348~6357952,248606422~493001,277779043~4483937,282726102~2562064,292988707~7952502,312566619~2948229,316301726~3511210,322862850~12451153,335934186~1814726,344823952~8079036,365490113~5689551,371762046~1637913,379003744~7346160,389872420~4206259,396670010~6408714] 2024-09-13T00:39:08.928 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12276827,17113294~3323835,22837062~9960619,48635901~7849557,62637940~8948458,95048390~7435958,104857600~2026393,121634816~5460584,130261050~7020755,144141427~2676660,148016038~18626631,171460898~6550730,184549376~3692163,188743680~10830113,201556540~2015504,207131072~5649534,222829084~7034872,240828348~6357952,248606422~493001,277779043~4483937,282726102~2562064,292988707~7952502,310378496~5136352,316301726~3511210,322862850~12451153,335934186~1814726,344823952~8079036,365490113~5689551,371762046~1637913,379003744~7346160,389872420~4206259,396670010~6408714] 2024-09-13T00:39:08.928 INFO:tasks.workunit.client.0.smithi136.stdout:from 10 to 17 diff [978366~12276827,17113294~3323835,22837062~9960619,37199402~669117,48635901~7849557,58865372~12721026,95048390~7435958,105459787~1424206,122043306~5052094,130261050~7020755,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~11224066,215619334~2003239,222829084~11562179,240828348~6357952,248606422~493001,252886187~622714,277779043~4483937,282726102~2562064,292988707~7952502,312566619~2948229,316301726~3511210,322862850~12451153,335934186~1814726,344823952~8079036,365490113~5689551,371762046~1637913,379003744~7346160,389458074~13620650] 2024-09-13T00:39:08.928 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~3323835,22837062~9960619,37199402~669117,48635901~7849557,58865372~12721026,95048390~7435958,105459787~1424206,122043306~5052094,130261050~7020755,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~11224066,215619334~2003239,222829084~11562179,240828348~6357952,248606422~493001,252886187~622714,277779043~4483937,282726102~2562064,292988707~7952502,312566619~2948229,316301726~3511210,322862850~12451153,335934186~1814726,344823952~8079036,365490113~5689551,371762046~1637913,379003744~7346160,389458074~13620650] 2024-09-13T00:39:08.934 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12276827,17113294~3323835,22837062~9960619,37199402~669117,48635901~7849557,58865372~12721026,95048390~7435958,104857600~2026393,121634816~5460584,130261050~7020755,144141427~2676660,148016038~18626631,171460898~6550730,184549376~3692163,188743680~10830113,201556540~11224066,215619334~2003239,222829084~11562179,240828348~6357952,248606422~493001,252237477~1271424,277779043~4483937,282726102~2562064,292988707~7952502,310378496~5136352,316301726~3511210,322862850~12451153,335934186~1814726,344823952~8079036,365490113~5689551,371762046~1637913,379003744~7346160,389458074~13620650] 2024-09-13T00:39:08.934 INFO:tasks.workunit.client.0.smithi136.stdout:from 10 to 18 diff [978366~12276827,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,95048390~7435958,105459787~1424206,122043306~5052094,130261050~7020755,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252886187~622714,277779043~4483937,282726102~2562064,292988707~7952502,312566619~2948229,316301726~3511210,322862850~12451153,335934186~1814726,344823952~8079036,365490113~10144650,379003744~7346160,389458074~13620650] 2024-09-13T00:39:08.934 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,95048390~7435958,105459787~1424206,122043306~5052094,130261050~7020755,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252886187~622714,277779043~4483937,282726102~2562064,292988707~7952502,312566619~2948229,316301726~3511210,322862850~12451153,335934186~1814726,344823952~8079036,365490113~10144650,379003744~7346160,389458074~13620650] 2024-09-13T00:39:08.940 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12276827,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,95048390~7435958,104857600~2026393,121634816~5460584,130261050~7020755,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,184549376~3692163,188743680~10830113,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252237477~1271424,277779043~4483937,282726102~2562064,292988707~7952502,310378496~5136352,316301726~3511210,322862850~12451153,335934186~1814726,344823952~8079036,365490113~10144650,379003744~7346160,389458074~13620650] 2024-09-13T00:39:08.940 INFO:tasks.workunit.client.0.smithi136.stdout:from 10 to 19 diff [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~11451555,105459787~1424206,122043306~5052094,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~8079036,355040685~5509158,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:08.940 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~11451555,105459787~1424206,122043306~5052094,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~8079036,355040685~5509158,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:08.945 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~11451555,104857600~2026393,121634816~5460584,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,184549376~3692163,188743680~10830113,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252237477~1271424,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,344823952~8079036,355040685~5509158,360710144~14924619,379003744~7346160,389458074~13620650] 2024-09-13T00:39:08.946 INFO:tasks.workunit.client.0.smithi136.stdout:from 11 to 12 diff [6779309~6475884,24769299~7548428,62948307~5586522,95048390~7435958,277779043~4483937,312566619~2948229,316301726~3511210,328085427~7228576,366458829~4720835,379083185~7266719] 2024-09-13T00:39:08.946 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [6779309~6475884,24769299~7548428,62948307~5586522,95048390~7435958,277779043~4483937,312566619~2948229,316301726~3511210,328085427~7228576,366458829~4720835,379083185~7266719] 2024-09-13T00:39:08.952 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [6779309~6475884,24769299~7548428,62948307~5586522,95048390~7435958,277779043~4483937,310378496~5136352,316301726~3511210,328085427~7228576,366458829~4720835,379083185~7266719] 2024-09-13T00:39:08.952 INFO:tasks.workunit.client.0.smithi136.stdout:from 11 to 13 diff [6779309~6475884,17113294~3323835,22837062~9480665,62948307~8638091,95048390~7435958,159255557~7387112,173851851~84284,187369573~871966,201556540~2015504,209974378~1903654,277779043~4483937,294860984~5752492,312566619~2948229,316301726~3511210,328085427~7228576,366458829~4720835,379083185~7266719] 2024-09-13T00:39:08.952 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [6779309~6475884,17113294~3323835,22837062~9480665,62948307~8638091,95048390~7435958,159255557~7387112,173851851~84284,187369573~871966,201556540~2015504,209974378~1903654,277779043~4483937,294860984~5752492,312566619~2948229,316301726~3511210,328085427~7228576,366458829~4720835,379083185~7266719] 2024-09-13T00:39:08.957 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [6779309~6475884,17113294~3323835,22837062~9480665,62948307~8638091,95048390~7435958,159255557~7387112,173851851~84284,184549376~3692163,201556540~2015504,209974378~1903654,277779043~4483937,294860984~5752492,310378496~5136352,316301726~3511210,328085427~7228576,366458829~4720835,379083185~7266719] 2024-09-13T00:39:08.957 INFO:tasks.workunit.client.0.smithi136.stdout:from 11 to 14 diff [978366~12276827,17113294~3323835,22837062~9960619,62948307~8638091,95048390~7435958,159255557~7387112,173851851~84284,187369573~871966,201556540~2015504,207131072~5649534,277779043~4483937,294860984~6080225,312566619~2948229,316301726~3511210,328085427~7228576,365490113~5689551,371762046~1637913,379083185~7266719,389872420~4206259] 2024-09-13T00:39:08.957 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~3323835,22837062~9960619,62948307~8638091,95048390~7435958,159255557~7387112,173851851~84284,187369573~871966,201556540~2015504,207131072~5649534,277779043~4483937,294860984~6080225,312566619~2948229,316301726~3511210,328085427~7228576,365490113~5689551,371762046~1637913,379083185~7266719,389872420~4206259] 2024-09-13T00:39:08.963 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12276827,17113294~3323835,22837062~9960619,62948307~8638091,95048390~7435958,159255557~7387112,173851851~84284,184549376~3692163,201556540~2015504,207131072~5649534,277779043~4483937,294860984~6080225,310378496~5136352,316301726~3511210,328085427~7228576,365490113~5689551,371762046~1637913,379083185~7266719,389872420~4206259] 2024-09-13T00:39:08.963 INFO:tasks.workunit.client.0.smithi136.stdout:from 11 to 15 diff [978366~12276827,17113294~3323835,22837062~9960619,62948307~8638091,95048390~7435958,144141427~2676660,148016038~10362739,159255557~7387112,171460898~6550730,187369573~871966,192121569~7452224,201556540~2015504,207131072~5649534,248606422~493001,277779043~4483937,294860984~6080225,312566619~2948229,316301726~3511210,327427195~7886808,348299711~4603277,365490113~5689551,371762046~1637913,379083185~7266719,389872420~4206259] 2024-09-13T00:39:08.963 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~3323835,22837062~9960619,62948307~8638091,95048390~7435958,144141427~2676660,148016038~10362739,159255557~7387112,171460898~6550730,187369573~871966,192121569~7452224,201556540~2015504,207131072~5649534,248606422~493001,277779043~4483937,294860984~6080225,312566619~2948229,316301726~3511210,327427195~7886808,348299711~4603277,365490113~5689551,371762046~1637913,379083185~7266719,389872420~4206259] 2024-09-13T00:39:08.969 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12276827,17113294~3323835,22837062~9960619,62948307~8638091,95048390~7435958,144141427~2676660,148016038~10362739,159255557~7387112,171460898~6550730,184549376~3692163,188743680~10830113,201556540~2015504,207131072~5649534,248606422~493001,277779043~4483937,294860984~6080225,310378496~5136352,316301726~3511210,327427195~7886808,348299711~4603277,365490113~5689551,371762046~1637913,379083185~7266719,389872420~4206259] 2024-09-13T00:39:08.969 INFO:tasks.workunit.client.0.smithi136.stdout:from 11 to 16 diff [978366~12276827,17113294~3323835,22837062~9960619,48635901~7849557,62637940~8948458,95048390~7435958,122781879~1514283,131583241~5698564,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~2015504,207131072~5649534,222829084~7034872,248606422~493001,277779043~4483937,294860984~6080225,312566619~2948229,316301726~3511210,327427195~7886808,344823952~8079036,365490113~5689551,371762046~1637913,379003744~7346160,389872420~4206259] 2024-09-13T00:39:08.969 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~3323835,22837062~9960619,48635901~7849557,62637940~8948458,95048390~7435958,122781879~1514283,131583241~5698564,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~2015504,207131072~5649534,222829084~7034872,248606422~493001,277779043~4483937,294860984~6080225,312566619~2948229,316301726~3511210,327427195~7886808,344823952~8079036,365490113~5689551,371762046~1637913,379003744~7346160,389872420~4206259] 2024-09-13T00:39:08.974 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12276827,17113294~3323835,22837062~9960619,48635901~7849557,62637940~8948458,95048390~7435958,122781879~1514283,131583241~5698564,144141427~2676660,148016038~18626631,171460898~6550730,184549376~3692163,188743680~10830113,201556540~2015504,207131072~5649534,222829084~7034872,248606422~493001,277779043~4483937,294860984~6080225,310378496~5136352,316301726~3511210,327427195~7886808,344823952~8079036,365490113~5689551,371762046~1637913,379003744~7346160,389872420~4206259] 2024-09-13T00:39:08.975 INFO:tasks.workunit.client.0.smithi136.stdout:from 11 to 17 diff [978366~12276827,17113294~3323835,22837062~9960619,37199402~669117,48635901~7849557,58865372~12721026,95048390~7435958,122781879~1514283,131583241~5698564,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~11224066,215619334~2003239,222829084~11562179,248606422~493001,252886187~622714,277779043~4483937,294860984~6080225,312566619~2948229,316301726~3511210,327427195~7886808,344823952~8079036,365490113~5689551,371762046~1637913,379003744~7346160,389458074~7853223,401222124~243609] 2024-09-13T00:39:08.975 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~3323835,22837062~9960619,37199402~669117,48635901~7849557,58865372~12721026,95048390~7435958,122781879~1514283,131583241~5698564,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~11224066,215619334~2003239,222829084~11562179,248606422~493001,252886187~622714,277779043~4483937,294860984~6080225,312566619~2948229,316301726~3511210,327427195~7886808,344823952~8079036,365490113~5689551,371762046~1637913,379003744~7346160,389458074~7853223,401222124~243609] 2024-09-13T00:39:08.981 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12276827,17113294~3323835,22837062~9960619,37199402~669117,48635901~7849557,58865372~12721026,95048390~7435958,122781879~1514283,131583241~5698564,144141427~2676660,148016038~18626631,171460898~6550730,184549376~3692163,188743680~10830113,201556540~11224066,215619334~2003239,222829084~11562179,248606422~493001,252237477~1271424,277779043~4483937,294860984~6080225,310378496~5136352,316301726~3511210,327427195~7886808,344823952~8079036,365490113~5689551,371762046~1637913,379003744~7346160,389458074~7853223,401222124~243609] 2024-09-13T00:39:08.981 INFO:tasks.workunit.client.0.smithi136.stdout:from 11 to 18 diff [978366~12276827,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,95048390~7435958,122781879~1514283,131583241~5698564,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,248606422~493001,252886187~622714,277779043~4483937,294860984~6080225,312566619~2948229,316301726~3511210,327427195~7886808,344823952~8079036,365490113~10144650,379003744~7346160,389458074~7853223,401222124~243609] 2024-09-13T00:39:08.981 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12276827,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,95048390~7435958,122781879~1514283,131583241~5698564,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,248606422~493001,252886187~622714,277779043~4483937,294860984~6080225,312566619~2948229,316301726~3511210,327427195~7886808,344823952~8079036,365490113~10144650,379003744~7346160,389458074~7853223,401222124~243609] 2024-09-13T00:39:08.987 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12276827,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,95048390~7435958,122781879~1514283,131583241~5698564,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,184549376~3692163,188743680~10830113,201556540~16427001,222829084~11562179,248606422~493001,252237477~1271424,277779043~4483937,294860984~6080225,310378496~5136352,316301726~3511210,327427195~7886808,344823952~8079036,365490113~10144650,379003744~7346160,389458074~7853223,401222124~243609] 2024-09-13T00:39:08.987 INFO:tasks.workunit.client.0.smithi136.stdout:from 11 to 19 diff [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~11451555,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,294714089~7671666,312566619~2948229,316301726~10296905,327427195~7886808,344823952~8079036,355040685~5509158,362400302~13234461,379003744~7346160,389458074~7853223,401222124~243609] 2024-09-13T00:39:08.987 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~11451555,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,294714089~7671666,312566619~2948229,316301726~10296905,327427195~7886808,344823952~8079036,355040685~5509158,362400302~13234461,379003744~7346160,389458074~7853223,401222124~243609] 2024-09-13T00:39:08.992 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~11451555,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,184549376~3692163,188743680~10830113,201556540~16427001,222829084~11562179,248606422~493001,252237477~1271424,273064912~4591743,277779043~4483937,294714089~7671666,310378496~5136352,316301726~10296905,327427195~7886808,344823952~8079036,355040685~5509158,360710144~14924619,379003744~7346160,389458074~7853223,401222124~243609] 2024-09-13T00:39:08.992 INFO:tasks.workunit.client.0.smithi136.stdout:from 12 to 13 diff [17113294~3323835,22837062~7110070,63473742~8112656,99390713~475647,159255557~7387112,173851851~84284,187369573~871966,201556540~2015504,209974378~1903654,294860984~5752492] 2024-09-13T00:39:08.992 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [17113294~3323835,22837062~7110070,63473742~8112656,99390713~475647,159255557~7387112,173851851~84284,187369573~871966,201556540~2015504,209974378~1903654,294860984~5752492] 2024-09-13T00:39:08.998 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [17113294~3323835,22837062~7110070,63473742~8112656,99390713~475647,159255557~7387112,173851851~84284,184549376~3692163,201556540~2015504,209974378~1903654,294860984~5752492] 2024-09-13T00:39:08.998 INFO:tasks.workunit.client.0.smithi136.stdout:from 12 to 14 diff [978366~8009759,17113294~3323835,22837062~9960619,63473742~8112656,99390713~475647,159255557~7387112,173851851~84284,187369573~871966,201556540~2015504,207131072~5649534,294860984~6080225,365490113~5510962,371762046~1637913,379288747~1574174,389872420~4206259] 2024-09-13T00:39:08.998 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~8009759,17113294~3323835,22837062~9960619,63473742~8112656,99390713~475647,159255557~7387112,173851851~84284,187369573~871966,201556540~2015504,207131072~5649534,294860984~6080225,365490113~5510962,371762046~1637913,379288747~1574174,389872420~4206259] 2024-09-13T00:39:09.005 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~8009759,17113294~3323835,22837062~9960619,63473742~8112656,99390713~475647,159255557~7387112,173851851~84284,184549376~3692163,201556540~2015504,207131072~5649534,294860984~6080225,365490113~5510962,371762046~1637913,379288747~1574174,389872420~4206259] 2024-09-13T00:39:09.005 INFO:tasks.workunit.client.0.smithi136.stdout:from 12 to 15 diff [978366~8009759,17113294~3323835,22837062~9960619,63473742~8112656,99390713~475647,144141427~2676660,148016038~10362739,159255557~7387112,171460898~6550730,187369573~871966,192121569~7452224,201556540~2015504,207131072~5649534,248606422~493001,294860984~6080225,327427195~2026374,348299711~4603277,365490113~5510962,371762046~1637913,379288747~1574174,389872420~4206259] 2024-09-13T00:39:09.005 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~8009759,17113294~3323835,22837062~9960619,63473742~8112656,99390713~475647,144141427~2676660,148016038~10362739,159255557~7387112,171460898~6550730,187369573~871966,192121569~7452224,201556540~2015504,207131072~5649534,248606422~493001,294860984~6080225,327427195~2026374,348299711~4603277,365490113~5510962,371762046~1637913,379288747~1574174,389872420~4206259] 2024-09-13T00:39:09.011 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~8009759,17113294~3323835,22837062~9960619,63473742~8112656,99390713~475647,144141427~2676660,148016038~10362739,159255557~7387112,171460898~6550730,184549376~3692163,188743680~10830113,201556540~2015504,207131072~5649534,248606422~493001,294860984~6080225,327427195~2026374,348299711~4603277,365490113~5510962,371762046~1637913,379288747~1574174,389872420~4206259] 2024-09-13T00:39:09.011 INFO:tasks.workunit.client.0.smithi136.stdout:from 12 to 16 diff [978366~8009759,17113294~3323835,22837062~9960619,48635901~7849557,62637940~8948458,99390713~475647,122781879~1514283,131583241~5698564,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~2015504,207131072~5649534,222829084~7034872,248606422~493001,294860984~6080225,327427195~2026374,344823952~8079036,365490113~5510962,371762046~1637913,379003744~4970562,389872420~4206259] 2024-09-13T00:39:09.011 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~8009759,17113294~3323835,22837062~9960619,48635901~7849557,62637940~8948458,99390713~475647,122781879~1514283,131583241~5698564,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~2015504,207131072~5649534,222829084~7034872,248606422~493001,294860984~6080225,327427195~2026374,344823952~8079036,365490113~5510962,371762046~1637913,379003744~4970562,389872420~4206259] 2024-09-13T00:39:09.016 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~8009759,17113294~3323835,22837062~9960619,48635901~7849557,62637940~8948458,99390713~475647,122781879~1514283,131583241~5698564,144141427~2676660,148016038~18626631,171460898~6550730,184549376~3692163,188743680~10830113,201556540~2015504,207131072~5649534,222829084~7034872,248606422~493001,294860984~6080225,327427195~2026374,344823952~8079036,365490113~5510962,371762046~1637913,379003744~4970562,389872420~4206259] 2024-09-13T00:39:09.016 INFO:tasks.workunit.client.0.smithi136.stdout:from 12 to 17 diff [978366~8009759,17113294~3323835,22837062~9960619,37199402~669117,48635901~7849557,58865372~12721026,99390713~475647,122781879~1514283,131583241~5698564,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~11224066,215619334~2003239,222829084~11562179,248606422~493001,252886187~622714,294860984~6080225,327427195~2026374,344823952~8079036,365490113~5510962,371762046~1637913,379003744~4970562,389458074~7853223,401222124~243609] 2024-09-13T00:39:09.017 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~8009759,17113294~3323835,22837062~9960619,37199402~669117,48635901~7849557,58865372~12721026,99390713~475647,122781879~1514283,131583241~5698564,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~11224066,215619334~2003239,222829084~11562179,248606422~493001,252886187~622714,294860984~6080225,327427195~2026374,344823952~8079036,365490113~5510962,371762046~1637913,379003744~4970562,389458074~7853223,401222124~243609] 2024-09-13T00:39:09.022 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~8009759,17113294~3323835,22837062~9960619,37199402~669117,48635901~7849557,58865372~12721026,99390713~475647,122781879~1514283,131583241~5698564,144141427~2676660,148016038~18626631,171460898~6550730,184549376~3692163,188743680~10830113,201556540~11224066,215619334~2003239,222829084~11562179,248606422~493001,252237477~1271424,294860984~6080225,327427195~2026374,344823952~8079036,365490113~5510962,371762046~1637913,379003744~4970562,389458074~7853223,401222124~243609] 2024-09-13T00:39:09.022 INFO:tasks.workunit.client.0.smithi136.stdout:from 12 to 18 diff [978366~8009759,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,99390713~475647,122781879~1514283,131583241~5698564,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,248606422~493001,252886187~622714,294860984~6080225,327427195~2026374,344823952~8079036,365490113~10144650,379003744~4970562,389458074~7853223,401222124~243609] 2024-09-13T00:39:09.022 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~8009759,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,99390713~475647,122781879~1514283,131583241~5698564,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,248606422~493001,252886187~622714,294860984~6080225,327427195~2026374,344823952~8079036,365490113~10144650,379003744~4970562,389458074~7853223,401222124~243609] 2024-09-13T00:39:09.027 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~8009759,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,99390713~475647,122781879~1514283,131583241~5698564,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,184549376~3692163,188743680~10830113,201556540~16427001,222829084~11562179,248606422~493001,252237477~1271424,294860984~6080225,327427195~2026374,344823952~8079036,365490113~10144650,379003744~4970562,389458074~7853223,401222124~243609] 2024-09-13T00:39:09.027 INFO:tasks.workunit.client.0.smithi136.stdout:from 12 to 19 diff [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~4865453,99390713~475647,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,248606422~493001,252886187~622714,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,362400302~13234461,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:09.028 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~4865453,99390713~475647,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,248606422~493001,252886187~622714,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,362400302~13234461,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:09.033 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~4865453,99390713~475647,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,184549376~3692163,188743680~10830113,201556540~16427001,222829084~11562179,248606422~493001,252237477~1271424,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,360710144~14924619,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:09.033 INFO:tasks.workunit.client.0.smithi136.stdout:from 13 to 14 diff [978366~8009759,28753760~4043921,63639003~3918459,207131072~5649534,297239089~3702120,365490113~5510962,371762046~1637913,379288747~1574174,389872420~4206259] 2024-09-13T00:39:09.033 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~8009759,28753760~4043921,63639003~3918459,207131072~5649534,297239089~3702120,365490113~5510962,371762046~1637913,379288747~1574174,389872420~4206259] 2024-09-13T00:39:09.043 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~8009759,28753760~4043921,63639003~3918459,207131072~5649534,297239089~3702120,365490113~5510962,371762046~1637913,379288747~1574174,389872420~4206259] 2024-09-13T00:39:09.043 INFO:tasks.workunit.client.0.smithi136.stdout:from 13 to 15 diff [978366~8009759,28753760~4043921,63639003~3918459,144141427~2676660,148016038~10362739,171460898~6550730,192121569~7452224,207131072~5649534,248606422~493001,297239089~3702120,327427195~2026374,348299711~4603277,365490113~5510962,371762046~1637913,379288747~1574174,389872420~4206259] 2024-09-13T00:39:09.043 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~8009759,28753760~4043921,63639003~3918459,144141427~2676660,148016038~10362739,171460898~6550730,192121569~7452224,207131072~5649534,248606422~493001,297239089~3702120,327427195~2026374,348299711~4603277,365490113~5510962,371762046~1637913,379288747~1574174,389872420~4206259] 2024-09-13T00:39:09.048 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~8009759,28753760~4043921,63639003~3918459,144141427~2676660,148016038~10362739,171460898~6550730,188743680~10830113,207131072~5649534,248606422~493001,297239089~3702120,327427195~2026374,348299711~4603277,365490113~5510962,371762046~1637913,379288747~1574174,389872420~4206259] 2024-09-13T00:39:09.048 INFO:tasks.workunit.client.0.smithi136.stdout:from 13 to 16 diff [978366~8009759,18131473~1232378,28753760~4043921,48635901~7849557,62637940~4919522,122781879~1514283,131583241~5698564,144141427~2676660,148016038~17249056,171460898~6550730,192121569~7452224,207131072~5649534,222829084~7034872,248606422~493001,297239089~3702120,327427195~2026374,344823952~8079036,365490113~5510962,371762046~1637913,379003744~4970562,389872420~4206259] 2024-09-13T00:39:09.048 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~8009759,18131473~1232378,28753760~4043921,48635901~7849557,62637940~4919522,122781879~1514283,131583241~5698564,144141427~2676660,148016038~17249056,171460898~6550730,192121569~7452224,207131072~5649534,222829084~7034872,248606422~493001,297239089~3702120,327427195~2026374,344823952~8079036,365490113~5510962,371762046~1637913,379003744~4970562,389872420~4206259] 2024-09-13T00:39:09.054 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~8009759,18131473~1232378,28753760~4043921,48635901~7849557,62637940~4919522,122781879~1514283,131583241~5698564,144141427~2676660,148016038~17249056,171460898~6550730,188743680~10830113,207131072~5649534,222829084~7034872,248606422~493001,297239089~3702120,327427195~2026374,344823952~8079036,365490113~5510962,371762046~1637913,379003744~4970562,389872420~4206259] 2024-09-13T00:39:09.054 INFO:tasks.workunit.client.0.smithi136.stdout:from 13 to 17 diff [978366~8009759,17650227~1831268,28753760~4043921,37199402~669117,48635901~7849557,58865372~8692090,122781879~1514283,131583241~5698564,144141427~2676660,148016038~17249056,171460898~6550730,192121569~7452224,201939997~10840609,215619334~2003239,222829084~11562179,248606422~493001,252886187~622714,297239089~3702120,327427195~2026374,344823952~8079036,365490113~5510962,371762046~1637913,379003744~4970562,389458074~7853223,401222124~243609] 2024-09-13T00:39:09.054 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~8009759,17650227~1831268,28753760~4043921,37199402~669117,48635901~7849557,58865372~8692090,122781879~1514283,131583241~5698564,144141427~2676660,148016038~17249056,171460898~6550730,192121569~7452224,201939997~10840609,215619334~2003239,222829084~11562179,248606422~493001,252886187~622714,297239089~3702120,327427195~2026374,344823952~8079036,365490113~5510962,371762046~1637913,379003744~4970562,389458074~7853223,401222124~243609] 2024-09-13T00:39:09.060 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~8009759,17650227~1831268,28753760~4043921,37199402~669117,48635901~7849557,58865372~8692090,122781879~1514283,131583241~5698564,144141427~2676660,148016038~17249056,171460898~6550730,188743680~10830113,201939997~10840609,215619334~2003239,222829084~11562179,248606422~493001,252237477~1271424,297239089~3702120,327427195~2026374,344823952~8079036,365490113~5510962,371762046~1637913,379003744~4970562,389458074~7853223,401222124~243609] 2024-09-13T00:39:09.060 INFO:tasks.workunit.client.0.smithi136.stdout:from 13 to 18 diff [978366~8009759,17650227~1831268,28753760~4043921,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,122781879~1514283,131583241~5698564,142230204~1250407,144141427~2676660,148016038~17787685,171460898~6550730,192121569~7452224,201939997~16043544,222829084~11562179,248606422~493001,252886187~622714,297239089~3702120,327427195~2026374,344823952~8079036,365490113~10144650,379003744~4970562,389458074~7853223,401222124~243609] 2024-09-13T00:39:09.060 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~8009759,17650227~1831268,28753760~4043921,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,122781879~1514283,131583241~5698564,142230204~1250407,144141427~2676660,148016038~17787685,171460898~6550730,192121569~7452224,201939997~16043544,222829084~11562179,248606422~493001,252886187~622714,297239089~3702120,327427195~2026374,344823952~8079036,365490113~10144650,379003744~4970562,389458074~7853223,401222124~243609] 2024-09-13T00:39:09.065 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~8009759,17650227~1831268,28753760~4043921,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,122781879~1514283,131583241~5698564,142230204~1250407,144141427~2676660,148016038~17787685,171460898~6550730,188743680~10830113,201939997~16043544,222829084~11562179,248606422~493001,252237477~1271424,297239089~3702120,327427195~2026374,344823952~8079036,365490113~10144650,379003744~4970562,389458074~7853223,401222124~243609] 2024-09-13T00:39:09.066 INFO:tasks.workunit.client.0.smithi136.stdout:from 13 to 19 diff [978366~12404130,17650227~1831268,28753760~4043921,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~4865453,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~17787685,171460898~6550730,192121569~7452224,201939997~16043544,222829084~11562179,248606422~493001,252886187~622714,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,362400302~13234461,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:09.066 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17650227~1831268,28753760~4043921,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~4865453,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~17787685,171460898~6550730,192121569~7452224,201939997~16043544,222829084~11562179,248606422~493001,252886187~622714,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,362400302~13234461,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:09.071 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17650227~1831268,28753760~4043921,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~4865453,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~17787685,171460898~6550730,188743680~10830113,201939997~16043544,222829084~11562179,248606422~493001,252237477~1271424,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,360710144~14924619,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:09.071 INFO:tasks.workunit.client.0.smithi136.stdout:from 14 to 15 diff [144141427~2676660,148016038~10362739,171460898~6550730,192121569~7452224,207577196~1690477,248606422~493001,327427195~2026374,348299711~4603277,366010954~3024486] 2024-09-13T00:39:09.071 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [144141427~2676660,148016038~10362739,171460898~6550730,192121569~7452224,207577196~1690477,248606422~493001,327427195~2026374,348299711~4603277,366010954~3024486] 2024-09-13T00:39:09.076 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [144141427~2676660,148016038~10362739,171460898~6550730,188743680~10830113,207577196~1690477,248606422~493001,327427195~2026374,348299711~4603277,366010954~3024486] 2024-09-13T00:39:09.077 INFO:tasks.workunit.client.0.smithi136.stdout:from 14 to 16 diff [18131473~1232378,48635901~7849557,62637940~1049327,122781879~1514283,131583241~5698564,144141427~2676660,148016038~17249056,171460898~6550730,192121569~7452224,207577196~1690477,222829084~7034872,248606422~493001,327427195~2026374,344823952~8079036,366010954~3024486,379003744~4970562] 2024-09-13T00:39:09.077 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [18131473~1232378,48635901~7849557,62637940~1049327,122781879~1514283,131583241~5698564,144141427~2676660,148016038~17249056,171460898~6550730,192121569~7452224,207577196~1690477,222829084~7034872,248606422~493001,327427195~2026374,344823952~8079036,366010954~3024486,379003744~4970562] 2024-09-13T00:39:09.082 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [18131473~1232378,48635901~7849557,62637940~1049327,122781879~1514283,131583241~5698564,144141427~2676660,148016038~17249056,171460898~6550730,188743680~10830113,207577196~1690477,222829084~7034872,248606422~493001,327427195~2026374,344823952~8079036,366010954~3024486,379003744~4970562] 2024-09-13T00:39:09.082 INFO:tasks.workunit.client.0.smithi136.stdout:from 14 to 17 diff [7846533~846537,17650227~1831268,37199402~669117,48635901~7849557,58865372~5846796,122781879~1514283,131583241~5698564,144141427~2676660,148016038~17249056,171460898~6550730,192121569~7452224,201939997~7327676,215619334~2003239,222829084~11562179,248606422~493001,252886187~622714,327427195~2026374,344823952~8079036,366010954~3024486,379003744~4970562,389458074~7853223,401222124~243609] 2024-09-13T00:39:09.082 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [7846533~846537,17650227~1831268,37199402~669117,48635901~7849557,58865372~5846796,122781879~1514283,131583241~5698564,144141427~2676660,148016038~17249056,171460898~6550730,192121569~7452224,201939997~7327676,215619334~2003239,222829084~11562179,248606422~493001,252886187~622714,327427195~2026374,344823952~8079036,366010954~3024486,379003744~4970562,389458074~7853223,401222124~243609] 2024-09-13T00:39:09.088 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [7846533~846537,17650227~1831268,37199402~669117,48635901~7849557,58865372~5846796,122781879~1514283,131583241~5698564,144141427~2676660,148016038~17249056,171460898~6550730,188743680~10830113,201939997~7327676,215619334~2003239,222829084~11562179,248606422~493001,252237477~1271424,327427195~2026374,344823952~8079036,366010954~3024486,379003744~4970562,389458074~7853223,401222124~243609] 2024-09-13T00:39:09.088 INFO:tasks.workunit.client.0.smithi136.stdout:from 14 to 18 diff [7846533~846537,17650227~1831268,37199402~669117,47180962~9304496,58865372~5846796,66638502~7190962,76198939~3215002,122781879~1514283,131583241~5698564,142230204~1250407,144141427~2676660,148016038~17787685,171460898~6550730,192121569~7452224,201939997~7327676,211120086~6863455,222829084~11562179,248606422~493001,252886187~622714,327427195~2026374,344823952~8079036,366010954~3024486,370509734~5125029,379003744~4970562,389458074~7853223,401222124~243609] 2024-09-13T00:39:09.088 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [7846533~846537,17650227~1831268,37199402~669117,47180962~9304496,58865372~5846796,66638502~7190962,76198939~3215002,122781879~1514283,131583241~5698564,142230204~1250407,144141427~2676660,148016038~17787685,171460898~6550730,192121569~7452224,201939997~7327676,211120086~6863455,222829084~11562179,248606422~493001,252886187~622714,327427195~2026374,344823952~8079036,366010954~3024486,370509734~5125029,379003744~4970562,389458074~7853223,401222124~243609] 2024-09-13T00:39:09.095 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [7846533~846537,17650227~1831268,37199402~669117,47180962~9304496,58865372~5846796,66638502~7190962,76198939~3215002,122781879~1514283,131583241~5698564,142230204~1250407,144141427~2676660,148016038~17787685,171460898~6550730,188743680~10830113,201939997~7327676,211120086~6863455,222829084~11562179,248606422~493001,252237477~1271424,327427195~2026374,344823952~8079036,366010954~3024486,370509734~5125029,379003744~4970562,389458074~7853223,401222124~243609] 2024-09-13T00:39:09.095 INFO:tasks.workunit.client.0.smithi136.stdout:from 14 to 19 diff [5881326~7501170,17650227~1831268,37199402~669117,47180962~9304496,58865372~5846796,66638502~7190962,76198939~3215002,80242387~800824,91032793~4865453,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~17787685,171460898~6550730,192121569~7452224,201939997~7327676,211120086~6863455,222829084~11562179,248606422~493001,252886187~622714,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,362400302~6635138,370509734~5125029,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:09.095 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [5881326~7501170,17650227~1831268,37199402~669117,47180962~9304496,58865372~5846796,66638502~7190962,76198939~3215002,80242387~800824,91032793~4865453,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~17787685,171460898~6550730,192121569~7452224,201939997~7327676,211120086~6863455,222829084~11562179,248606422~493001,252886187~622714,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,362400302~6635138,370509734~5125029,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:09.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:08 smithi181 bash[67587]: cluster 2024-09-13T00:39:07.835815+0000 mon.a (mon.0) 3577 : cluster [DBG] osdmap e1926: 8 total, 8 up, 8 in 2024-09-13T00:39:09.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:08 smithi181 bash[67587]: cluster 2024-09-13T00:39:08.425154+0000 mgr.y (mgr.24661) 1860 : cluster [DBG] pgmap v3347: 137 pgs: 137 active+clean; 817 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 232 KiB/s rd, 40 MiB/s wr, 358 op/s 2024-09-13T00:39:09.101 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [5881326~7501170,17650227~1831268,37199402~669117,47180962~9304496,58865372~5846796,66638502~7190962,76198939~3215002,80242387~800824,91032793~4865453,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~17787685,171460898~6550730,188743680~10830113,201939997~7327676,211120086~6863455,222829084~11562179,248606422~493001,252237477~1271424,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,360710144~8325296,370509734~5125029,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:09.101 INFO:tasks.workunit.client.0.smithi136.stdout:from 15 to 16 diff [18131473~1232378,48635901~7849557,62637940~1049327,122781879~1514283,131583241~5698564,158290495~6974599,222829084~7034872,344823952~7059258,379003744~4970562] 2024-09-13T00:39:09.101 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [18131473~1232378,48635901~7849557,62637940~1049327,122781879~1514283,131583241~5698564,158290495~6974599,222829084~7034872,344823952~7059258,379003744~4970562] 2024-09-13T00:39:09.106 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [18131473~1232378,48635901~7849557,62637940~1049327,122781879~1514283,131583241~5698564,158290495~6974599,222829084~7034872,344823952~7059258,379003744~4970562] 2024-09-13T00:39:09.106 INFO:tasks.workunit.client.0.smithi136.stdout:from 15 to 17 diff [7846533~846537,17650227~1831268,37199402~669117,48635901~7849557,58865372~5846796,122781879~1514283,131583241~5698564,158290495~6974599,201939997~6695813,215619334~2003239,222829084~11562179,252886187~622714,344823952~7059258,379003744~4970562,389458074~7853223,401222124~243609] 2024-09-13T00:39:09.106 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [7846533~846537,17650227~1831268,37199402~669117,48635901~7849557,58865372~5846796,122781879~1514283,131583241~5698564,158290495~6974599,201939997~6695813,215619334~2003239,222829084~11562179,252886187~622714,344823952~7059258,379003744~4970562,389458074~7853223,401222124~243609] 2024-09-13T00:39:09.112 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [7846533~846537,17650227~1831268,37199402~669117,48635901~7849557,58865372~5846796,122781879~1514283,131583241~5698564,158290495~6974599,201939997~6695813,215619334~2003239,222829084~11562179,252237477~1271424,344823952~7059258,379003744~4970562,389458074~7853223,401222124~243609] 2024-09-13T00:39:09.112 INFO:tasks.workunit.client.0.smithi136.stdout:from 15 to 18 diff [7846533~846537,17650227~1831268,37199402~669117,47180962~9304496,58865372~5846796,66638502~7190962,76198939~3215002,122781879~1514283,131583241~5698564,142230204~1250407,158290495~7513228,201939997~6732288,211120086~6863455,222829084~11562179,252886187~622714,344823952~7059258,370509734~5125029,379003744~4970562,389458074~7853223,401222124~243609] 2024-09-13T00:39:09.112 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [7846533~846537,17650227~1831268,37199402~669117,47180962~9304496,58865372~5846796,66638502~7190962,76198939~3215002,122781879~1514283,131583241~5698564,142230204~1250407,158290495~7513228,201939997~6732288,211120086~6863455,222829084~11562179,252886187~622714,344823952~7059258,370509734~5125029,379003744~4970562,389458074~7853223,401222124~243609] 2024-09-13T00:39:09.118 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [7846533~846537,17650227~1831268,37199402~669117,47180962~9304496,58865372~5846796,66638502~7190962,76198939~3215002,122781879~1514283,131583241~5698564,142230204~1250407,158290495~7513228,201939997~6732288,211120086~6863455,222829084~11562179,252237477~1271424,344823952~7059258,370509734~5125029,379003744~4970562,389458074~7853223,401222124~243609] 2024-09-13T00:39:09.118 INFO:tasks.workunit.client.0.smithi136.stdout:from 15 to 19 diff [5881326~7501170,17650227~1831268,37199402~669117,47180962~9304496,58865372~5846796,66638502~7190962,76198939~3215002,80242387~800824,91032793~4865453,122781879~1514283,128928644~8353161,142230204~1250407,158290495~7513228,201939997~6732288,211120086~6863455,222829084~11562179,252886187~622714,273064912~4591743,294714089~7671666,319038423~7560208,344823952~7059258,355040685~5509158,362400302~4428536,370509734~5125029,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:09.118 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [5881326~7501170,17650227~1831268,37199402~669117,47180962~9304496,58865372~5846796,66638502~7190962,76198939~3215002,80242387~800824,91032793~4865453,122781879~1514283,128928644~8353161,142230204~1250407,158290495~7513228,201939997~6732288,211120086~6863455,222829084~11562179,252886187~622714,273064912~4591743,294714089~7671666,319038423~7560208,344823952~7059258,355040685~5509158,362400302~4428536,370509734~5125029,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:09.124 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [5881326~7501170,17650227~1831268,37199402~669117,47180962~9304496,58865372~5846796,66638502~7190962,76198939~3215002,80242387~800824,91032793~4865453,122781879~1514283,128928644~8353161,142230204~1250407,158290495~7513228,201939997~6732288,211120086~6863455,222829084~11562179,252237477~1271424,273064912~4591743,294714089~7671666,319038423~7560208,344823952~7059258,355040685~5509158,360710144~6118694,370509734~5125029,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:09.124 INFO:tasks.workunit.client.0.smithi136.stdout:from 16 to 17 diff [7846533~846537,17650227~1831268,37199402~669117,58865372~5846796,201939997~6695813,215619334~2003239,227214911~7176352,252886187~622714,389458074~7853223,401222124~243609] 2024-09-13T00:39:09.124 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [7846533~846537,17650227~1831268,37199402~669117,58865372~5846796,201939997~6695813,215619334~2003239,227214911~7176352,252886187~622714,389458074~7853223,401222124~243609] 2024-09-13T00:39:09.129 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [7846533~846537,17650227~1831268,37199402~669117,58865372~5846796,201939997~6695813,215619334~2003239,227214911~7176352,252237477~1271424,389458074~7853223,401222124~243609] 2024-09-13T00:39:09.129 INFO:tasks.workunit.client.0.smithi136.stdout:from 16 to 18 diff [7846533~846537,17650227~1831268,37199402~669117,47180962~4366707,58865372~5846796,66638502~7190962,76198939~3215002,142230204~1250407,158551854~7251869,201939997~6732288,211120086~6863455,227214911~7176352,252886187~622714,370509734~5125029,389458074~7853223,401222124~243609] 2024-09-13T00:39:09.129 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [7846533~846537,17650227~1831268,37199402~669117,47180962~4366707,58865372~5846796,66638502~7190962,76198939~3215002,142230204~1250407,158551854~7251869,201939997~6732288,211120086~6863455,227214911~7176352,252886187~622714,370509734~5125029,389458074~7853223,401222124~243609] 2024-09-13T00:39:09.135 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [7846533~846537,17650227~1831268,37199402~669117,47180962~4366707,58865372~5846796,66638502~7190962,76198939~3215002,142230204~1250407,158551854~7251869,201939997~6732288,211120086~6863455,227214911~7176352,252237477~1271424,370509734~5125029,389458074~7853223,401222124~243609] 2024-09-13T00:39:09.135 INFO:tasks.workunit.client.0.smithi136.stdout:from 16 to 19 diff [5881326~7501170,17650227~1831268,37199402~669117,47180962~4366707,58865372~5846796,66638502~7190962,76198939~3215002,80242387~800824,91032793~4865453,128928644~7456281,142230204~1250407,158551854~7251869,201939997~6732288,211120086~6863455,227214911~7176352,252886187~622714,273064912~4591743,294714089~7671666,319038423~7560208,355040685~5509158,362400302~4428536,370509734~5125029,383843711~554618,389458074~7853223,401222124~243609] 2024-09-13T00:39:09.135 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [5881326~7501170,17650227~1831268,37199402~669117,47180962~4366707,58865372~5846796,66638502~7190962,76198939~3215002,80242387~800824,91032793~4865453,128928644~7456281,142230204~1250407,158551854~7251869,201939997~6732288,211120086~6863455,227214911~7176352,252886187~622714,273064912~4591743,294714089~7671666,319038423~7560208,355040685~5509158,362400302~4428536,370509734~5125029,383843711~554618,389458074~7853223,401222124~243609] 2024-09-13T00:39:09.141 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [5881326~7501170,17650227~1831268,37199402~669117,47180962~4366707,58865372~5846796,66638502~7190962,76198939~3215002,80242387~800824,91032793~4865453,128928644~7456281,142230204~1250407,158551854~7251869,201939997~6732288,211120086~6863455,227214911~7176352,252237477~1271424,273064912~4591743,294714089~7671666,319038423~7560208,355040685~5509158,360710144~6118694,370509734~5125029,383843711~554618,389458074~7853223,401222124~243609] 2024-09-13T00:39:09.141 INFO:tasks.workunit.client.0.smithi136.stdout:from 17 to 18 diff [47180962~4366707,66638502~7190962,76198939~3215002,142230204~1250407,158551854~7251869,205819705~2852580,211120086~6863455,229610868~1211581,370509734~5125029] 2024-09-13T00:39:09.141 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [47180962~4366707,66638502~7190962,76198939~3215002,142230204~1250407,158551854~7251869,205819705~2852580,211120086~6863455,229610868~1211581,370509734~5125029] 2024-09-13T00:39:09.148 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [47180962~4366707,66638502~7190962,76198939~3215002,142230204~1250407,158551854~7251869,205819705~2852580,211120086~6863455,229610868~1211581,370509734~5125029] 2024-09-13T00:39:09.148 INFO:tasks.workunit.client.0.smithi136.stdout:from 17 to 19 diff [5881326~7501170,47180962~4366707,66638502~7190962,76198939~3215002,80242387~800824,91032793~4865453,128928644~7456281,142230204~1250407,158551854~7251869,205819705~2852580,211120086~6863455,229610868~1211581,273064912~4591743,294714089~7671666,319038423~7560208,355040685~5509158,362400302~4428536,370509734~5125029,383843711~554618] 2024-09-13T00:39:09.148 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [5881326~7501170,47180962~4366707,66638502~7190962,76198939~3215002,80242387~800824,91032793~4865453,128928644~7456281,142230204~1250407,158551854~7251869,205819705~2852580,211120086~6863455,229610868~1211581,273064912~4591743,294714089~7671666,319038423~7560208,355040685~5509158,362400302~4428536,370509734~5125029,383843711~554618] 2024-09-13T00:39:09.154 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [5881326~7501170,47180962~4366707,66638502~7190962,76198939~3215002,80242387~800824,91032793~4865453,128928644~7456281,142230204~1250407,158551854~7251869,205819705~2852580,211120086~6863455,229610868~1211581,273064912~4591743,294714089~7671666,319038423~7560208,355040685~5509158,360710144~6118694,370509734~5125029,383843711~554618] 2024-09-13T00:39:09.154 INFO:tasks.workunit.client.0.smithi136.stdout:from 18 to 19 diff [5881326~7501170,80242387~800824,91032793~4865453,128928644~7456281,273064912~4591743,294714089~7671666,319038423~7560208,355040685~5509158,362400302~4428536,383843711~554618] 2024-09-13T00:39:09.154 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [5881326~7501170,80242387~800824,91032793~4865453,128928644~7456281,273064912~4591743,294714089~7671666,319038423~7560208,355040685~5509158,362400302~4428536,383843711~554618] 2024-09-13T00:39:09.159 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [5881326~7501170,80242387~800824,91032793~4865453,128928644~7456281,273064912~4591743,294714089~7671666,319038423~7560208,355040685~5509158,360710144~6118694,383843711~554618] 2024-09-13T00:39:09.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:08 smithi136 bash[75164]: cluster 2024-09-13T00:39:07.835815+0000 mon.a (mon.0) 3577 : cluster [DBG] osdmap e1926: 8 total, 8 up, 8 in 2024-09-13T00:39:09.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:08 smithi136 bash[75164]: cluster 2024-09-13T00:39:08.425154+0000 mgr.y (mgr.24661) 1860 : cluster [DBG] pgmap v3347: 137 pgs: 137 active+clean; 817 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 232 KiB/s rd, 40 MiB/s wr, 358 op/s 2024-09-13T00:39:09.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:08 smithi136 bash[76371]: cluster 2024-09-13T00:39:07.835815+0000 mon.a (mon.0) 3577 : cluster [DBG] osdmap e1926: 8 total, 8 up, 8 in 2024-09-13T00:39:09.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:08 smithi136 bash[76371]: cluster 2024-09-13T00:39:08.425154+0000 mgr.y (mgr.24661) 1860 : cluster [DBG] pgmap v3347: 137 pgs: 137 active+clean; 817 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 232 KiB/s rd, 40 MiB/s wr, 358 op/s 2024-09-13T00:39:09.761 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:39:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:39:09] "GET /metrics HTTP/1.1" 200 36757 "" "Prometheus/2.43.0" 2024-09-13T00:39:09.879 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to HEAD diff [978366~12404130,17113294~39372164,58515289~28499682,89250005~13234343,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~20663168,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240318840~11918637,252886187~622714,260842262~7844195,269960512~2117506,273064912~36449096,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,341594269~18955574,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:09.879 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~39372164,58515289~28499682,89250005~13234343,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~20663168,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240318840~11918637,252886187~622714,260842262~7844195,269960512~2117506,273064912~36449096,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,341594269~18955574,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:09.907 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,16777216~70237755,88080384~14403964,104857600~2026393,111245499~9980052,121634816~18591770,142230204~1250407,143604550~40657412,184549376~3692163,188743680~10830113,201556540~32834723,239075328~14433573,260046848~12031170,272629760~36884248,310378496~24935507,335934186~24615657,360710144~44231030,405696667~5790944] 2024-09-13T00:39:09.908 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to HEAD diff [978366~12404130,17113294~26808795,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:09.908 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~26808795,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:09.920 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,16777216~27144873,46526671~40488300,91032793~11451555,104857600~2026393,111245499~9980052,121634816~18591770,142230204~1250407,143604550~23038119,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~35982749,310378496~24935507,335934186~4465000,343284705~17265138,360710144~44231030,405696667~5790944] 2024-09-13T00:39:09.920 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to HEAD diff [978366~12404130,17113294~3323835,22666579~21255510,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:09.921 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~21255510,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:09.933 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~21255510,46526671~40488300,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~35982749,310378496~24935507,335934186~4465000,343284705~17265138,360710144~44231030,405696667~5790944] 2024-09-13T00:39:09.933 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,46526671~9958787,58515289~15314175,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,405696667~5790944] 2024-09-13T00:39:09.934 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,46526671~9958787,58515289~15314175,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,405696667~5790944] 2024-09-13T00:39:09.945 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~16987636,46526671~27302793,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~35982749,310378496~5136352,316301726~19012277,335934186~4465000,343284705~17265138,360710144~42368580,405696667~5790944] 2024-09-13T00:39:09.945 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,409295430~1606555] 2024-09-13T00:39:09.945 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,409295430~1606555] 2024-09-13T00:39:09.956 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~18421176,292988707~15623802,310378496~5136352,316301726~19012277,335934186~4465000,343284705~17265138,360710144~42368580,409295430~1606555] 2024-09-13T00:39:09.956 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:09.956 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:09.965 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~18421176,292988707~15623802,310378496~5136352,316301726~19012277,335934186~1814726,343284705~17265138,360710144~16554896,378757665~24321059] 2024-09-13T00:39:09.965 INFO:tasks.workunit.client.0.smithi136.stdout:from 6 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273064912~17986024,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:09.965 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273064912~17986024,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:09.973 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,266675149~244059,272629760~18421176,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,343284705~17265138,360710144~16554896,378757665~24321059] 2024-09-13T00:39:09.973 INFO:tasks.workunit.client.0.smithi136.stdout:from 7 to HEAD diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~4720159,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~16427001,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273064912~4591743,277779043~4483937,282726102~8324834,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~14864738,379003744~7346160,389458074~13620650] 2024-09-13T00:39:09.974 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~4720159,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~16427001,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273064912~4591743,277779043~4483937,282726102~8324834,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~14864738,379003744~7346160,389458074~13620650] 2024-09-13T00:39:09.981 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~4720159,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~14961695,184549376~3692163,188743680~10830113,201556540~16427001,218740863~15650400,240828348~12680553,266675149~244059,272629760~5026895,277779043~4483937,282726102~8324834,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,344823952~15725891,360710144~16554896,379003744~7346160,389458074~13620650] 2024-09-13T00:39:09.981 INFO:tasks.workunit.client.0.smithi136.stdout:from 8 to HEAD diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180376314~3885648,187369573~871966,192121569~7452224,201556540~16427001,220044632~14346631,240828348~10753127,252886187~622714,266675149~244059,273064912~4591743,277779043~4483937,282726102~8324834,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:09.981 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180376314~3885648,187369573~871966,192121569~7452224,201556540~16427001,220044632~14346631,240828348~10753127,252886187~622714,266675149~244059,273064912~4591743,277779043~4483937,282726102~8324834,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:09.989 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180376314~3885648,184549376~3692163,188743680~10830113,201556540~16427001,218740863~15650400,240828348~10753127,252237477~1271424,266675149~244059,272629760~5026895,277779043~4483937,282726102~8324834,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,344823952~15725891,360710144~14924619,379003744~7346160,389458074~13620650] 2024-09-13T00:39:09.989 INFO:tasks.workunit.client.0.smithi136.stdout:from 9 to HEAD diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~8291936,91032793~11451555,105459787~1424206,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:09.989 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~8291936,91032793~11451555,105459787~1424206,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:09.995 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~8291936,91032793~11451555,104857600~2026393,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,184549376~3692163,188743680~10830113,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252237477~1271424,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,344823952~15725891,360710144~14924619,379003744~7346160,389458074~13620650] 2024-09-13T00:39:09.995 INFO:tasks.workunit.client.0.smithi136.stdout:from 10 to HEAD diff [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~11451555,105459787~1424206,122043306~5052094,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~8079036,355040685~5509158,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:09.995 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~11451555,105459787~1424206,122043306~5052094,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~8079036,355040685~5509158,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:10.002 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~11451555,104857600~2026393,121634816~5460584,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,184549376~3692163,188743680~10830113,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252237477~1271424,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,344823952~8079036,355040685~5509158,360710144~14924619,379003744~7346160,389458074~13620650] 2024-09-13T00:39:10.002 INFO:tasks.workunit.client.0.smithi136.stdout:from 11 to HEAD diff [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~11451555,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,294714089~7671666,312566619~2948229,316301726~10296905,327427195~7886808,344823952~8079036,355040685~5509158,362400302~13234461,379003744~7346160,389458074~7853223,401222124~243609] 2024-09-13T00:39:10.002 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~11451555,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,294714089~7671666,312566619~2948229,316301726~10296905,327427195~7886808,344823952~8079036,355040685~5509158,362400302~13234461,379003744~7346160,389458074~7853223,401222124~243609] 2024-09-13T00:39:10.009 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~11451555,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,184549376~3692163,188743680~10830113,201556540~16427001,222829084~11562179,248606422~493001,252237477~1271424,273064912~4591743,277779043~4483937,294714089~7671666,310378496~5136352,316301726~10296905,327427195~7886808,344823952~8079036,355040685~5509158,360710144~14924619,379003744~7346160,389458074~7853223,401222124~243609] 2024-09-13T00:39:10.009 INFO:tasks.workunit.client.0.smithi136.stdout:from 12 to HEAD diff [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~4865453,99390713~475647,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,248606422~493001,252886187~622714,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,362400302~13234461,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:10.009 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~4865453,99390713~475647,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,248606422~493001,252886187~622714,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,362400302~13234461,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:10.017 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~4865453,99390713~475647,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,184549376~3692163,188743680~10830113,201556540~16427001,222829084~11562179,248606422~493001,252237477~1271424,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,360710144~14924619,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:10.017 INFO:tasks.workunit.client.0.smithi136.stdout:from 13 to HEAD diff [978366~12404130,17650227~1831268,28753760~4043921,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~4865453,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~17787685,171460898~6550730,192121569~7452224,201939997~16043544,222829084~11562179,248606422~493001,252886187~622714,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,362400302~13234461,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:10.017 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17650227~1831268,28753760~4043921,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~4865453,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~17787685,171460898~6550730,192121569~7452224,201939997~16043544,222829084~11562179,248606422~493001,252886187~622714,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,362400302~13234461,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:10.022 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17650227~1831268,28753760~4043921,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~4865453,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~17787685,171460898~6550730,188743680~10830113,201939997~16043544,222829084~11562179,248606422~493001,252237477~1271424,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,360710144~14924619,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:10.022 INFO:tasks.workunit.client.0.smithi136.stdout:from 14 to HEAD diff [5881326~7501170,17650227~1831268,37199402~669117,47180962~9304496,58865372~5846796,66638502~7190962,76198939~3215002,80242387~800824,91032793~4865453,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~17787685,171460898~6550730,192121569~7452224,201939997~7327676,211120086~6863455,222829084~11562179,248606422~493001,252886187~622714,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,362400302~6635138,370509734~5125029,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:10.022 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [5881326~7501170,17650227~1831268,37199402~669117,47180962~9304496,58865372~5846796,66638502~7190962,76198939~3215002,80242387~800824,91032793~4865453,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~17787685,171460898~6550730,192121569~7452224,201939997~7327676,211120086~6863455,222829084~11562179,248606422~493001,252886187~622714,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,362400302~6635138,370509734~5125029,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:10.028 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [5881326~7501170,17650227~1831268,37199402~669117,47180962~9304496,58865372~5846796,66638502~7190962,76198939~3215002,80242387~800824,91032793~4865453,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~17787685,171460898~6550730,188743680~10830113,201939997~7327676,211120086~6863455,222829084~11562179,248606422~493001,252237477~1271424,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,360710144~8325296,370509734~5125029,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:10.028 INFO:tasks.workunit.client.0.smithi136.stdout:from 15 to HEAD diff [5881326~7501170,17650227~1831268,37199402~669117,47180962~9304496,58865372~5846796,66638502~7190962,76198939~3215002,80242387~800824,91032793~4865453,122781879~1514283,128928644~8353161,142230204~1250407,158290495~7513228,201939997~6732288,211120086~6863455,222829084~11562179,252886187~622714,273064912~4591743,294714089~7671666,319038423~7560208,344823952~7059258,355040685~5509158,362400302~4428536,370509734~5125029,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:10.028 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [5881326~7501170,17650227~1831268,37199402~669117,47180962~9304496,58865372~5846796,66638502~7190962,76198939~3215002,80242387~800824,91032793~4865453,122781879~1514283,128928644~8353161,142230204~1250407,158290495~7513228,201939997~6732288,211120086~6863455,222829084~11562179,252886187~622714,273064912~4591743,294714089~7671666,319038423~7560208,344823952~7059258,355040685~5509158,362400302~4428536,370509734~5125029,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:10.033 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [5881326~7501170,17650227~1831268,37199402~669117,47180962~9304496,58865372~5846796,66638502~7190962,76198939~3215002,80242387~800824,91032793~4865453,122781879~1514283,128928644~8353161,142230204~1250407,158290495~7513228,201939997~6732288,211120086~6863455,222829084~11562179,252237477~1271424,273064912~4591743,294714089~7671666,319038423~7560208,344823952~7059258,355040685~5509158,360710144~6118694,370509734~5125029,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:10.034 INFO:tasks.workunit.client.0.smithi136.stdout:from 16 to HEAD diff [5881326~7501170,17650227~1831268,37199402~669117,47180962~4366707,58865372~5846796,66638502~7190962,76198939~3215002,80242387~800824,91032793~4865453,128928644~7456281,142230204~1250407,158551854~7251869,201939997~6732288,211120086~6863455,227214911~7176352,252886187~622714,273064912~4591743,294714089~7671666,319038423~7560208,355040685~5509158,362400302~4428536,370509734~5125029,383843711~554618,389458074~7853223,401222124~243609] 2024-09-13T00:39:10.034 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [5881326~7501170,17650227~1831268,37199402~669117,47180962~4366707,58865372~5846796,66638502~7190962,76198939~3215002,80242387~800824,91032793~4865453,128928644~7456281,142230204~1250407,158551854~7251869,201939997~6732288,211120086~6863455,227214911~7176352,252886187~622714,273064912~4591743,294714089~7671666,319038423~7560208,355040685~5509158,362400302~4428536,370509734~5125029,383843711~554618,389458074~7853223,401222124~243609] 2024-09-13T00:39:10.040 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [5881326~7501170,17650227~1831268,37199402~669117,47180962~4366707,58865372~5846796,66638502~7190962,76198939~3215002,80242387~800824,91032793~4865453,128928644~7456281,142230204~1250407,158551854~7251869,201939997~6732288,211120086~6863455,227214911~7176352,252237477~1271424,273064912~4591743,294714089~7671666,319038423~7560208,355040685~5509158,360710144~6118694,370509734~5125029,383843711~554618,389458074~7853223,401222124~243609] 2024-09-13T00:39:10.040 INFO:tasks.workunit.client.0.smithi136.stdout:from 17 to HEAD diff [5881326~7501170,47180962~4366707,66638502~7190962,76198939~3215002,80242387~800824,91032793~4865453,128928644~7456281,142230204~1250407,158551854~7251869,205819705~2852580,211120086~6863455,229610868~1211581,273064912~4591743,294714089~7671666,319038423~7560208,355040685~5509158,362400302~4428536,370509734~5125029,383843711~554618] 2024-09-13T00:39:10.040 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [5881326~7501170,47180962~4366707,66638502~7190962,76198939~3215002,80242387~800824,91032793~4865453,128928644~7456281,142230204~1250407,158551854~7251869,205819705~2852580,211120086~6863455,229610868~1211581,273064912~4591743,294714089~7671666,319038423~7560208,355040685~5509158,362400302~4428536,370509734~5125029,383843711~554618] 2024-09-13T00:39:10.045 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [5881326~7501170,47180962~4366707,66638502~7190962,76198939~3215002,80242387~800824,91032793~4865453,128928644~7456281,142230204~1250407,158551854~7251869,205819705~2852580,211120086~6863455,229610868~1211581,273064912~4591743,294714089~7671666,319038423~7560208,355040685~5509158,360710144~6118694,370509734~5125029,383843711~554618] 2024-09-13T00:39:10.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:09 smithi181 bash[67587]: audit 2024-09-13T00:39:08.843226+0000 mon.a (mon.0) 3578 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:39:10.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:09 smithi136 bash[75164]: audit 2024-09-13T00:39:08.843226+0000 mon.a (mon.0) 3578 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:39:10.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:09 smithi136 bash[76371]: audit 2024-09-13T00:39:08.843226+0000 mon.a (mon.0) 3578 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:39:10.888 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to HEAD diff [978366~12404130,17113294~39372164,58515289~28499682,89250005~13234343,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~20663168,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240318840~11918637,252886187~622714,260842262~7844195,269960512~2117506,273064912~36449096,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,341594269~18955574,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:10.889 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~39372164,58515289~28499682,89250005~13234343,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~20663168,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240318840~11918637,252886187~622714,260842262~7844195,269960512~2117506,273064912~36449096,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,341594269~18955574,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:10.947 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,16777216~70237755,88080384~14403964,104857600~2026393,111245499~9980052,121634816~18591770,142230204~1250407,143604550~40657412,184549376~3692163,188743680~10830113,201556540~32834723,239075328~14433573,260046848~12031170,272629760~36884248,310378496~24935507,335934186~24615657,360710144~44231030,405696667~5790944] 2024-09-13T00:39:10.947 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to HEAD diff [978366~12404130,17113294~26808795,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:10.947 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~26808795,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:10.955 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,16777216~27144873,46526671~40488300,91032793~11451555,104857600~2026393,111245499~9980052,121634816~18591770,142230204~1250407,143604550~23038119,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~35982749,310378496~24935507,335934186~4465000,343284705~17265138,360710144~44231030,405696667~5790944] 2024-09-13T00:39:10.955 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to HEAD diff [978366~12404130,17113294~3323835,22666579~21255510,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:10.955 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~21255510,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:10.963 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~21255510,46526671~40488300,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~35982749,310378496~24935507,335934186~4465000,343284705~17265138,360710144~44231030,405696667~5790944] 2024-09-13T00:39:10.963 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,46526671~9958787,58515289~15314175,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,405696667~5790944] 2024-09-13T00:39:10.963 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,46526671~9958787,58515289~15314175,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,405696667~5790944] 2024-09-13T00:39:10.970 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~16987636,46526671~27302793,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~35982749,310378496~5136352,316301726~19012277,335934186~4465000,343284705~17265138,360710144~42368580,405696667~5790944] 2024-09-13T00:39:10.970 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,409295430~1606555] 2024-09-13T00:39:10.971 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,409295430~1606555] 2024-09-13T00:39:10.977 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~18421176,292988707~15623802,310378496~5136352,316301726~19012277,335934186~4465000,343284705~17265138,360710144~42368580,409295430~1606555] 2024-09-13T00:39:10.977 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:10.977 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:10.984 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~18421176,292988707~15623802,310378496~5136352,316301726~19012277,335934186~1814726,343284705~17265138,360710144~16554896,378757665~24321059] 2024-09-13T00:39:10.984 INFO:tasks.workunit.client.0.smithi136.stdout:from 6 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273064912~17986024,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:10.984 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273064912~17986024,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:10.990 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,266675149~244059,272629760~18421176,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,343284705~17265138,360710144~16554896,378757665~24321059] 2024-09-13T00:39:10.990 INFO:tasks.workunit.client.0.smithi136.stdout:from 7 to HEAD diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~4720159,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~16427001,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273064912~4591743,277779043~4483937,282726102~8324834,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~14864738,379003744~7346160,389458074~13620650] 2024-09-13T00:39:10.990 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~4720159,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~16427001,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273064912~4591743,277779043~4483937,282726102~8324834,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~14864738,379003744~7346160,389458074~13620650] 2024-09-13T00:39:10.996 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~4720159,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~14961695,184549376~3692163,188743680~10830113,201556540~16427001,218740863~15650400,240828348~12680553,266675149~244059,272629760~5026895,277779043~4483937,282726102~8324834,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,344823952~15725891,360710144~16554896,379003744~7346160,389458074~13620650] 2024-09-13T00:39:10.996 INFO:tasks.workunit.client.0.smithi136.stdout:from 8 to HEAD diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180376314~3885648,187369573~871966,192121569~7452224,201556540~16427001,220044632~14346631,240828348~10753127,252886187~622714,266675149~244059,273064912~4591743,277779043~4483937,282726102~8324834,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:10.996 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180376314~3885648,187369573~871966,192121569~7452224,201556540~16427001,220044632~14346631,240828348~10753127,252886187~622714,266675149~244059,273064912~4591743,277779043~4483937,282726102~8324834,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:11.002 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180376314~3885648,184549376~3692163,188743680~10830113,201556540~16427001,218740863~15650400,240828348~10753127,252237477~1271424,266675149~244059,272629760~5026895,277779043~4483937,282726102~8324834,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,344823952~15725891,360710144~14924619,379003744~7346160,389458074~13620650] 2024-09-13T00:39:11.002 INFO:tasks.workunit.client.0.smithi136.stdout:from 9 to HEAD diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~8291936,91032793~11451555,105459787~1424206,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:11.002 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~8291936,91032793~11451555,105459787~1424206,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:11.008 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~8291936,91032793~11451555,104857600~2026393,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,184549376~3692163,188743680~10830113,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252237477~1271424,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,344823952~15725891,360710144~14924619,379003744~7346160,389458074~13620650] 2024-09-13T00:39:11.008 INFO:tasks.workunit.client.0.smithi136.stdout:from 10 to HEAD diff [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~11451555,105459787~1424206,122043306~5052094,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~8079036,355040685~5509158,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:11.008 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~11451555,105459787~1424206,122043306~5052094,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~8079036,355040685~5509158,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:11.014 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~11451555,104857600~2026393,121634816~5460584,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,184549376~3692163,188743680~10830113,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252237477~1271424,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,344823952~8079036,355040685~5509158,360710144~14924619,379003744~7346160,389458074~13620650] 2024-09-13T00:39:11.014 INFO:tasks.workunit.client.0.smithi136.stdout:from 11 to HEAD diff [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~11451555,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,294714089~7671666,312566619~2948229,316301726~10296905,327427195~7886808,344823952~8079036,355040685~5509158,362400302~13234461,379003744~7346160,389458074~7853223,401222124~243609] 2024-09-13T00:39:11.014 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~11451555,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,294714089~7671666,312566619~2948229,316301726~10296905,327427195~7886808,344823952~8079036,355040685~5509158,362400302~13234461,379003744~7346160,389458074~7853223,401222124~243609] 2024-09-13T00:39:11.020 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~11451555,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,184549376~3692163,188743680~10830113,201556540~16427001,222829084~11562179,248606422~493001,252237477~1271424,273064912~4591743,277779043~4483937,294714089~7671666,310378496~5136352,316301726~10296905,327427195~7886808,344823952~8079036,355040685~5509158,360710144~14924619,379003744~7346160,389458074~7853223,401222124~243609] 2024-09-13T00:39:11.020 INFO:tasks.workunit.client.0.smithi136.stdout:from 12 to HEAD diff [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~4865453,99390713~475647,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,248606422~493001,252886187~622714,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,362400302~13234461,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:11.020 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~4865453,99390713~475647,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,248606422~493001,252886187~622714,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,362400302~13234461,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:11.025 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~4865453,99390713~475647,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,184549376~3692163,188743680~10830113,201556540~16427001,222829084~11562179,248606422~493001,252237477~1271424,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,360710144~14924619,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:11.026 INFO:tasks.workunit.client.0.smithi136.stdout:from 13 to HEAD diff [978366~12404130,17650227~1831268,28753760~4043921,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~4865453,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~17787685,171460898~6550730,192121569~7452224,201939997~16043544,222829084~11562179,248606422~493001,252886187~622714,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,362400302~13234461,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:11.026 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17650227~1831268,28753760~4043921,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~4865453,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~17787685,171460898~6550730,192121569~7452224,201939997~16043544,222829084~11562179,248606422~493001,252886187~622714,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,362400302~13234461,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:11.031 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17650227~1831268,28753760~4043921,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~4865453,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~17787685,171460898~6550730,188743680~10830113,201939997~16043544,222829084~11562179,248606422~493001,252237477~1271424,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,360710144~14924619,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:11.031 INFO:tasks.workunit.client.0.smithi136.stdout:from 14 to HEAD diff [5881326~7501170,17650227~1831268,37199402~669117,47180962~9304496,58865372~5846796,66638502~7190962,76198939~3215002,80242387~800824,91032793~4865453,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~17787685,171460898~6550730,192121569~7452224,201939997~7327676,211120086~6863455,222829084~11562179,248606422~493001,252886187~622714,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,362400302~6635138,370509734~5125029,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:11.031 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [5881326~7501170,17650227~1831268,37199402~669117,47180962~9304496,58865372~5846796,66638502~7190962,76198939~3215002,80242387~800824,91032793~4865453,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~17787685,171460898~6550730,192121569~7452224,201939997~7327676,211120086~6863455,222829084~11562179,248606422~493001,252886187~622714,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,362400302~6635138,370509734~5125029,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:11.037 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [5881326~7501170,17650227~1831268,37199402~669117,47180962~9304496,58865372~5846796,66638502~7190962,76198939~3215002,80242387~800824,91032793~4865453,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~17787685,171460898~6550730,188743680~10830113,201939997~7327676,211120086~6863455,222829084~11562179,248606422~493001,252237477~1271424,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,360710144~8325296,370509734~5125029,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:11.037 INFO:tasks.workunit.client.0.smithi136.stdout:from 15 to HEAD diff [5881326~7501170,17650227~1831268,37199402~669117,47180962~9304496,58865372~5846796,66638502~7190962,76198939~3215002,80242387~800824,91032793~4865453,122781879~1514283,128928644~8353161,142230204~1250407,158290495~7513228,201939997~6732288,211120086~6863455,222829084~11562179,252886187~622714,273064912~4591743,294714089~7671666,319038423~7560208,344823952~7059258,355040685~5509158,362400302~4428536,370509734~5125029,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:11.037 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [5881326~7501170,17650227~1831268,37199402~669117,47180962~9304496,58865372~5846796,66638502~7190962,76198939~3215002,80242387~800824,91032793~4865453,122781879~1514283,128928644~8353161,142230204~1250407,158290495~7513228,201939997~6732288,211120086~6863455,222829084~11562179,252886187~622714,273064912~4591743,294714089~7671666,319038423~7560208,344823952~7059258,355040685~5509158,362400302~4428536,370509734~5125029,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:11.043 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [5881326~7501170,17650227~1831268,37199402~669117,47180962~9304496,58865372~5846796,66638502~7190962,76198939~3215002,80242387~800824,91032793~4865453,122781879~1514283,128928644~8353161,142230204~1250407,158290495~7513228,201939997~6732288,211120086~6863455,222829084~11562179,252237477~1271424,273064912~4591743,294714089~7671666,319038423~7560208,344823952~7059258,355040685~5509158,360710144~6118694,370509734~5125029,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:11.043 INFO:tasks.workunit.client.0.smithi136.stdout:from 16 to HEAD diff [5881326~7501170,17650227~1831268,37199402~669117,47180962~4366707,58865372~5846796,66638502~7190962,76198939~3215002,80242387~800824,91032793~4865453,128928644~7456281,142230204~1250407,158551854~7251869,201939997~6732288,211120086~6863455,227214911~7176352,252886187~622714,273064912~4591743,294714089~7671666,319038423~7560208,355040685~5509158,362400302~4428536,370509734~5125029,383843711~554618,389458074~7853223,401222124~243609] 2024-09-13T00:39:11.043 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [5881326~7501170,17650227~1831268,37199402~669117,47180962~4366707,58865372~5846796,66638502~7190962,76198939~3215002,80242387~800824,91032793~4865453,128928644~7456281,142230204~1250407,158551854~7251869,201939997~6732288,211120086~6863455,227214911~7176352,252886187~622714,273064912~4591743,294714089~7671666,319038423~7560208,355040685~5509158,362400302~4428536,370509734~5125029,383843711~554618,389458074~7853223,401222124~243609] 2024-09-13T00:39:11.049 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [5881326~7501170,17650227~1831268,37199402~669117,47180962~4366707,58865372~5846796,66638502~7190962,76198939~3215002,80242387~800824,91032793~4865453,128928644~7456281,142230204~1250407,158551854~7251869,201939997~6732288,211120086~6863455,227214911~7176352,252237477~1271424,273064912~4591743,294714089~7671666,319038423~7560208,355040685~5509158,360710144~6118694,370509734~5125029,383843711~554618,389458074~7853223,401222124~243609] 2024-09-13T00:39:11.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:10 smithi136 bash[75164]: cluster 2024-09-13T00:39:09.852768+0000 mon.a (mon.0) 3579 : cluster [DBG] osdmap e1927: 8 total, 8 up, 8 in 2024-09-13T00:39:11.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:10 smithi136 bash[75164]: cluster 2024-09-13T00:39:10.425753+0000 mgr.y (mgr.24661) 1861 : cluster [DBG] pgmap v3349: 137 pgs: 137 active+clean; 865 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 17 KiB/s rd, 23 MiB/s wr, 1.02k op/s 2024-09-13T00:39:11.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:10 smithi136 bash[76371]: cluster 2024-09-13T00:39:09.852768+0000 mon.a (mon.0) 3579 : cluster [DBG] osdmap e1927: 8 total, 8 up, 8 in 2024-09-13T00:39:11.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:10 smithi136 bash[76371]: cluster 2024-09-13T00:39:10.425753+0000 mgr.y (mgr.24661) 1861 : cluster [DBG] pgmap v3349: 137 pgs: 137 active+clean; 865 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 17 KiB/s rd, 23 MiB/s wr, 1.02k op/s 2024-09-13T00:39:11.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:10 smithi181 bash[67587]: cluster 2024-09-13T00:39:09.852768+0000 mon.a (mon.0) 3579 : cluster [DBG] osdmap e1927: 8 total, 8 up, 8 in 2024-09-13T00:39:11.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:10 smithi181 bash[67587]: cluster 2024-09-13T00:39:10.425753+0000 mgr.y (mgr.24661) 1861 : cluster [DBG] pgmap v3349: 137 pgs: 137 active+clean; 865 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 17 KiB/s rd, 23 MiB/s wr, 1.02k op/s 2024-09-13T00:39:11.913 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to HEAD diff [978366~12404130,17113294~39372164,58515289~28499682,89250005~13234343,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~20663168,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240318840~11918637,252886187~622714,260842262~7844195,269960512~2117506,273064912~36449096,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,341594269~18955574,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:11.913 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~39372164,58515289~28499682,89250005~13234343,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~20663168,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240318840~11918637,252886187~622714,260842262~7844195,269960512~2117506,273064912~36449096,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,341594269~18955574,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:11.975 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,16777216~70237755,88080384~14403964,104857600~2026393,111245499~9980052,121634816~18591770,142230204~1250407,143604550~40657412,184549376~3692163,188743680~10830113,201556540~32834723,239075328~14433573,260046848~12031170,272629760~36884248,310378496~24935507,335934186~24615657,360710144~44231030,405696667~5790944] 2024-09-13T00:39:11.975 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to HEAD diff [978366~12404130,17113294~26808795,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:11.975 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~26808795,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:11.983 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,16777216~27144873,46526671~40488300,91032793~11451555,104857600~2026393,111245499~9980052,121634816~18591770,142230204~1250407,143604550~23038119,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~35982749,310378496~24935507,335934186~4465000,343284705~17265138,360710144~44231030,405696667~5790944] 2024-09-13T00:39:11.983 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to HEAD diff [978366~12404130,17113294~3323835,22666579~21255510,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:11.983 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~21255510,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:11.990 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~21255510,46526671~40488300,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~35982749,310378496~24935507,335934186~4465000,343284705~17265138,360710144~44231030,405696667~5790944] 2024-09-13T00:39:11.990 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,46526671~9958787,58515289~15314175,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,405696667~5790944] 2024-09-13T00:39:11.991 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,46526671~9958787,58515289~15314175,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,405696667~5790944] 2024-09-13T00:39:11.997 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~16987636,46526671~27302793,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~35982749,310378496~5136352,316301726~19012277,335934186~4465000,343284705~17265138,360710144~42368580,405696667~5790944] 2024-09-13T00:39:11.997 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,409295430~1606555] 2024-09-13T00:39:11.997 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,409295430~1606555] 2024-09-13T00:39:12.004 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~18421176,292988707~15623802,310378496~5136352,316301726~19012277,335934186~4465000,343284705~17265138,360710144~42368580,409295430~1606555] 2024-09-13T00:39:12.004 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:12.004 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:12.010 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~18421176,292988707~15623802,310378496~5136352,316301726~19012277,335934186~1814726,343284705~17265138,360710144~16554896,378757665~24321059] 2024-09-13T00:39:12.010 INFO:tasks.workunit.client.0.smithi136.stdout:from 6 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273064912~17986024,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:12.010 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273064912~17986024,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:12.016 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,266675149~244059,272629760~18421176,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,343284705~17265138,360710144~16554896,378757665~24321059] 2024-09-13T00:39:12.016 INFO:tasks.workunit.client.0.smithi136.stdout:from 7 to HEAD diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~4720159,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~16427001,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273064912~4591743,277779043~4483937,282726102~8324834,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~14864738,379003744~7346160,389458074~13620650] 2024-09-13T00:39:12.016 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~4720159,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~16427001,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273064912~4591743,277779043~4483937,282726102~8324834,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~14864738,379003744~7346160,389458074~13620650] 2024-09-13T00:39:12.022 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~4720159,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~14961695,184549376~3692163,188743680~10830113,201556540~16427001,218740863~15650400,240828348~12680553,266675149~244059,272629760~5026895,277779043~4483937,282726102~8324834,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,344823952~15725891,360710144~16554896,379003744~7346160,389458074~13620650] 2024-09-13T00:39:12.022 INFO:tasks.workunit.client.0.smithi136.stdout:from 8 to HEAD diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180376314~3885648,187369573~871966,192121569~7452224,201556540~16427001,220044632~14346631,240828348~10753127,252886187~622714,266675149~244059,273064912~4591743,277779043~4483937,282726102~8324834,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:12.022 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180376314~3885648,187369573~871966,192121569~7452224,201556540~16427001,220044632~14346631,240828348~10753127,252886187~622714,266675149~244059,273064912~4591743,277779043~4483937,282726102~8324834,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:12.028 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180376314~3885648,184549376~3692163,188743680~10830113,201556540~16427001,218740863~15650400,240828348~10753127,252237477~1271424,266675149~244059,272629760~5026895,277779043~4483937,282726102~8324834,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,344823952~15725891,360710144~14924619,379003744~7346160,389458074~13620650] 2024-09-13T00:39:12.028 INFO:tasks.workunit.client.0.smithi136.stdout:from 9 to HEAD diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~8291936,91032793~11451555,105459787~1424206,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:12.028 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~8291936,91032793~11451555,105459787~1424206,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:12.034 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~8291936,91032793~11451555,104857600~2026393,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,184549376~3692163,188743680~10830113,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252237477~1271424,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,344823952~15725891,360710144~14924619,379003744~7346160,389458074~13620650] 2024-09-13T00:39:12.034 INFO:tasks.workunit.client.0.smithi136.stdout:from 10 to HEAD diff [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~11451555,105459787~1424206,122043306~5052094,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~8079036,355040685~5509158,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:12.034 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~11451555,105459787~1424206,122043306~5052094,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~8079036,355040685~5509158,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:12.040 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~11451555,104857600~2026393,121634816~5460584,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,184549376~3692163,188743680~10830113,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252237477~1271424,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,344823952~8079036,355040685~5509158,360710144~14924619,379003744~7346160,389458074~13620650] 2024-09-13T00:39:12.040 INFO:tasks.workunit.client.0.smithi136.stdout:from 11 to HEAD diff [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~11451555,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,294714089~7671666,312566619~2948229,316301726~10296905,327427195~7886808,344823952~8079036,355040685~5509158,362400302~13234461,379003744~7346160,389458074~7853223,401222124~243609] 2024-09-13T00:39:12.040 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~11451555,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,294714089~7671666,312566619~2948229,316301726~10296905,327427195~7886808,344823952~8079036,355040685~5509158,362400302~13234461,379003744~7346160,389458074~7853223,401222124~243609] 2024-09-13T00:39:12.046 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~11451555,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,184549376~3692163,188743680~10830113,201556540~16427001,222829084~11562179,248606422~493001,252237477~1271424,273064912~4591743,277779043~4483937,294714089~7671666,310378496~5136352,316301726~10296905,327427195~7886808,344823952~8079036,355040685~5509158,360710144~14924619,379003744~7346160,389458074~7853223,401222124~243609] 2024-09-13T00:39:12.046 INFO:tasks.workunit.client.0.smithi136.stdout:from 12 to HEAD diff [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~4865453,99390713~475647,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,248606422~493001,252886187~622714,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,362400302~13234461,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:12.046 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~4865453,99390713~475647,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,248606422~493001,252886187~622714,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,362400302~13234461,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:12.052 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~4865453,99390713~475647,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,184549376~3692163,188743680~10830113,201556540~16427001,222829084~11562179,248606422~493001,252237477~1271424,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,360710144~14924619,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:12.052 INFO:tasks.workunit.client.0.smithi136.stdout:from 13 to HEAD diff [978366~12404130,17650227~1831268,28753760~4043921,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~4865453,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~17787685,171460898~6550730,192121569~7452224,201939997~16043544,222829084~11562179,248606422~493001,252886187~622714,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,362400302~13234461,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:12.052 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17650227~1831268,28753760~4043921,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~4865453,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~17787685,171460898~6550730,192121569~7452224,201939997~16043544,222829084~11562179,248606422~493001,252886187~622714,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,362400302~13234461,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:12.058 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17650227~1831268,28753760~4043921,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~4865453,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~17787685,171460898~6550730,188743680~10830113,201939997~16043544,222829084~11562179,248606422~493001,252237477~1271424,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,360710144~14924619,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:12.058 INFO:tasks.workunit.client.0.smithi136.stdout:from 14 to HEAD diff [5881326~7501170,17650227~1831268,37199402~669117,47180962~9304496,58865372~5846796,66638502~7190962,76198939~3215002,80242387~800824,91032793~4865453,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~17787685,171460898~6550730,192121569~7452224,201939997~7327676,211120086~6863455,222829084~11562179,248606422~493001,252886187~622714,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,362400302~6635138,370509734~5125029,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:12.058 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [5881326~7501170,17650227~1831268,37199402~669117,47180962~9304496,58865372~5846796,66638502~7190962,76198939~3215002,80242387~800824,91032793~4865453,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~17787685,171460898~6550730,192121569~7452224,201939997~7327676,211120086~6863455,222829084~11562179,248606422~493001,252886187~622714,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,362400302~6635138,370509734~5125029,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:12.063 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [5881326~7501170,17650227~1831268,37199402~669117,47180962~9304496,58865372~5846796,66638502~7190962,76198939~3215002,80242387~800824,91032793~4865453,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~17787685,171460898~6550730,188743680~10830113,201939997~7327676,211120086~6863455,222829084~11562179,248606422~493001,252237477~1271424,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,360710144~8325296,370509734~5125029,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:12.064 INFO:tasks.workunit.client.0.smithi136.stdout:from 15 to HEAD diff [5881326~7501170,17650227~1831268,37199402~669117,47180962~9304496,58865372~5846796,66638502~7190962,76198939~3215002,80242387~800824,91032793~4865453,122781879~1514283,128928644~8353161,142230204~1250407,158290495~7513228,201939997~6732288,211120086~6863455,222829084~11562179,252886187~622714,273064912~4591743,294714089~7671666,319038423~7560208,344823952~7059258,355040685~5509158,362400302~4428536,370509734~5125029,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:12.064 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [5881326~7501170,17650227~1831268,37199402~669117,47180962~9304496,58865372~5846796,66638502~7190962,76198939~3215002,80242387~800824,91032793~4865453,122781879~1514283,128928644~8353161,142230204~1250407,158290495~7513228,201939997~6732288,211120086~6863455,222829084~11562179,252886187~622714,273064912~4591743,294714089~7671666,319038423~7560208,344823952~7059258,355040685~5509158,362400302~4428536,370509734~5125029,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:12.069 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [5881326~7501170,17650227~1831268,37199402~669117,47180962~9304496,58865372~5846796,66638502~7190962,76198939~3215002,80242387~800824,91032793~4865453,122781879~1514283,128928644~8353161,142230204~1250407,158290495~7513228,201939997~6732288,211120086~6863455,222829084~11562179,252237477~1271424,273064912~4591743,294714089~7671666,319038423~7560208,344823952~7059258,355040685~5509158,360710144~6118694,370509734~5125029,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:12.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:11 smithi136 bash[75164]: cluster 2024-09-13T00:39:10.871168+0000 mon.a (mon.0) 3580 : cluster [DBG] osdmap e1928: 8 total, 8 up, 8 in 2024-09-13T00:39:12.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:11 smithi136 bash[76371]: cluster 2024-09-13T00:39:10.871168+0000 mon.a (mon.0) 3580 : cluster [DBG] osdmap e1928: 8 total, 8 up, 8 in 2024-09-13T00:39:12.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:11 smithi181 bash[67587]: cluster 2024-09-13T00:39:10.871168+0000 mon.a (mon.0) 3580 : cluster [DBG] osdmap e1928: 8 total, 8 up, 8 in 2024-09-13T00:39:12.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:39:12 smithi181 bash[95505]: ts=2024-09-13T00:39:12.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:39:12.919 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to HEAD diff [978366~12404130,17113294~39372164,58515289~28499682,89250005~13234343,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~20663168,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240318840~11918637,252886187~622714,260842262~7844195,269960512~2117506,273064912~36449096,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,341594269~18955574,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:12.920 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~39372164,58515289~28499682,89250005~13234343,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~20663168,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240318840~11918637,252886187~622714,260842262~7844195,269960512~2117506,273064912~36449096,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,341594269~18955574,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:12.988 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,16777216~70237755,88080384~14403964,104857600~2026393,111245499~9980052,121634816~18591770,142230204~1250407,143604550~40657412,184549376~3692163,188743680~10830113,201556540~32834723,239075328~14433573,260046848~12031170,272629760~36884248,310378496~24935507,335934186~24615657,360710144~44231030,405696667~5790944] 2024-09-13T00:39:12.988 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to HEAD diff [978366~12404130,17113294~26808795,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:12.988 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~26808795,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:12.996 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,16777216~27144873,46526671~40488300,91032793~11451555,104857600~2026393,111245499~9980052,121634816~18591770,142230204~1250407,143604550~23038119,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~35982749,310378496~24935507,335934186~4465000,343284705~17265138,360710144~44231030,405696667~5790944] 2024-09-13T00:39:12.996 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to HEAD diff [978366~12404130,17113294~3323835,22666579~21255510,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:12.996 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~21255510,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:13.002 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~21255510,46526671~40488300,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~35982749,310378496~24935507,335934186~4465000,343284705~17265138,360710144~44231030,405696667~5790944] 2024-09-13T00:39:13.002 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,46526671~9958787,58515289~15314175,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,405696667~5790944] 2024-09-13T00:39:13.003 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,46526671~9958787,58515289~15314175,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,405696667~5790944] 2024-09-13T00:39:13.009 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~16987636,46526671~27302793,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~35982749,310378496~5136352,316301726~19012277,335934186~4465000,343284705~17265138,360710144~42368580,405696667~5790944] 2024-09-13T00:39:13.009 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,409295430~1606555] 2024-09-13T00:39:13.009 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,409295430~1606555] 2024-09-13T00:39:13.015 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~18421176,292988707~15623802,310378496~5136352,316301726~19012277,335934186~4465000,343284705~17265138,360710144~42368580,409295430~1606555] 2024-09-13T00:39:13.015 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:13.015 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:13.021 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~18421176,292988707~15623802,310378496~5136352,316301726~19012277,335934186~1814726,343284705~17265138,360710144~16554896,378757665~24321059] 2024-09-13T00:39:13.021 INFO:tasks.workunit.client.0.smithi136.stdout:from 6 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273064912~17986024,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:13.022 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273064912~17986024,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:13.027 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,266675149~244059,272629760~18421176,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,343284705~17265138,360710144~16554896,378757665~24321059] 2024-09-13T00:39:13.028 INFO:tasks.workunit.client.0.smithi136.stdout:from 7 to HEAD diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~4720159,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~16427001,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273064912~4591743,277779043~4483937,282726102~8324834,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~14864738,379003744~7346160,389458074~13620650] 2024-09-13T00:39:13.028 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~4720159,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~16427001,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273064912~4591743,277779043~4483937,282726102~8324834,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~14864738,379003744~7346160,389458074~13620650] 2024-09-13T00:39:13.033 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~4720159,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~14961695,184549376~3692163,188743680~10830113,201556540~16427001,218740863~15650400,240828348~12680553,266675149~244059,272629760~5026895,277779043~4483937,282726102~8324834,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,344823952~15725891,360710144~16554896,379003744~7346160,389458074~13620650] 2024-09-13T00:39:13.033 INFO:tasks.workunit.client.0.smithi136.stdout:from 8 to HEAD diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180376314~3885648,187369573~871966,192121569~7452224,201556540~16427001,220044632~14346631,240828348~10753127,252886187~622714,266675149~244059,273064912~4591743,277779043~4483937,282726102~8324834,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:13.033 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180376314~3885648,187369573~871966,192121569~7452224,201556540~16427001,220044632~14346631,240828348~10753127,252886187~622714,266675149~244059,273064912~4591743,277779043~4483937,282726102~8324834,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:13.039 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180376314~3885648,184549376~3692163,188743680~10830113,201556540~16427001,218740863~15650400,240828348~10753127,252237477~1271424,266675149~244059,272629760~5026895,277779043~4483937,282726102~8324834,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,344823952~15725891,360710144~14924619,379003744~7346160,389458074~13620650] 2024-09-13T00:39:13.039 INFO:tasks.workunit.client.0.smithi136.stdout:from 9 to HEAD diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~8291936,91032793~11451555,105459787~1424206,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:13.039 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~8291936,91032793~11451555,105459787~1424206,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:13.045 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~8291936,91032793~11451555,104857600~2026393,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,184549376~3692163,188743680~10830113,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252237477~1271424,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,344823952~15725891,360710144~14924619,379003744~7346160,389458074~13620650] 2024-09-13T00:39:13.045 INFO:tasks.workunit.client.0.smithi136.stdout:from 10 to HEAD diff [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~11451555,105459787~1424206,122043306~5052094,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~8079036,355040685~5509158,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:13.045 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~11451555,105459787~1424206,122043306~5052094,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~8079036,355040685~5509158,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:13.051 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~11451555,104857600~2026393,121634816~5460584,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,184549376~3692163,188743680~10830113,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252237477~1271424,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,344823952~8079036,355040685~5509158,360710144~14924619,379003744~7346160,389458074~13620650] 2024-09-13T00:39:13.051 INFO:tasks.workunit.client.0.smithi136.stdout:from 11 to HEAD diff [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~11451555,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,294714089~7671666,312566619~2948229,316301726~10296905,327427195~7886808,344823952~8079036,355040685~5509158,362400302~13234461,379003744~7346160,389458074~7853223,401222124~243609] 2024-09-13T00:39:13.051 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~11451555,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,294714089~7671666,312566619~2948229,316301726~10296905,327427195~7886808,344823952~8079036,355040685~5509158,362400302~13234461,379003744~7346160,389458074~7853223,401222124~243609] 2024-09-13T00:39:13.057 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~11451555,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,184549376~3692163,188743680~10830113,201556540~16427001,222829084~11562179,248606422~493001,252237477~1271424,273064912~4591743,277779043~4483937,294714089~7671666,310378496~5136352,316301726~10296905,327427195~7886808,344823952~8079036,355040685~5509158,360710144~14924619,379003744~7346160,389458074~7853223,401222124~243609] 2024-09-13T00:39:13.057 INFO:tasks.workunit.client.0.smithi136.stdout:from 12 to HEAD diff [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~4865453,99390713~475647,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,248606422~493001,252886187~622714,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,362400302~13234461,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:13.057 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~4865453,99390713~475647,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,248606422~493001,252886187~622714,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,362400302~13234461,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:13.063 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~4865453,99390713~475647,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,184549376~3692163,188743680~10830113,201556540~16427001,222829084~11562179,248606422~493001,252237477~1271424,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,360710144~14924619,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:13.063 INFO:tasks.workunit.client.0.smithi136.stdout:from 13 to HEAD diff [978366~12404130,17650227~1831268,28753760~4043921,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~4865453,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~17787685,171460898~6550730,192121569~7452224,201939997~16043544,222829084~11562179,248606422~493001,252886187~622714,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,362400302~13234461,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:13.063 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17650227~1831268,28753760~4043921,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~4865453,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~17787685,171460898~6550730,192121569~7452224,201939997~16043544,222829084~11562179,248606422~493001,252886187~622714,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,362400302~13234461,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:13.069 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17650227~1831268,28753760~4043921,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~4865453,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~17787685,171460898~6550730,188743680~10830113,201939997~16043544,222829084~11562179,248606422~493001,252237477~1271424,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,360710144~14924619,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:13.069 INFO:tasks.workunit.client.0.smithi136.stdout:from 14 to HEAD diff [5881326~7501170,17650227~1831268,37199402~669117,47180962~9304496,58865372~5846796,66638502~7190962,76198939~3215002,80242387~800824,91032793~4865453,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~17787685,171460898~6550730,192121569~7452224,201939997~7327676,211120086~6863455,222829084~11562179,248606422~493001,252886187~622714,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,362400302~6635138,370509734~5125029,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:13.069 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [5881326~7501170,17650227~1831268,37199402~669117,47180962~9304496,58865372~5846796,66638502~7190962,76198939~3215002,80242387~800824,91032793~4865453,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~17787685,171460898~6550730,192121569~7452224,201939997~7327676,211120086~6863455,222829084~11562179,248606422~493001,252886187~622714,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,362400302~6635138,370509734~5125029,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:13.074 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [5881326~7501170,17650227~1831268,37199402~669117,47180962~9304496,58865372~5846796,66638502~7190962,76198939~3215002,80242387~800824,91032793~4865453,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~17787685,171460898~6550730,188743680~10830113,201939997~7327676,211120086~6863455,222829084~11562179,248606422~493001,252237477~1271424,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,360710144~8325296,370509734~5125029,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:13.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:12 smithi136 bash[75164]: cluster 2024-09-13T00:39:11.868702+0000 mon.a (mon.0) 3581 : cluster [DBG] osdmap e1929: 8 total, 8 up, 8 in 2024-09-13T00:39:13.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:12 smithi136 bash[75164]: cluster 2024-09-13T00:39:12.426621+0000 mgr.y (mgr.24661) 1862 : cluster [DBG] pgmap v3352: 137 pgs: 137 active+clean; 895 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 185 KiB/s rd, 18 MiB/s wr, 3.69k op/s 2024-09-13T00:39:13.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:12 smithi136 bash[76371]: cluster 2024-09-13T00:39:11.868702+0000 mon.a (mon.0) 3581 : cluster [DBG] osdmap e1929: 8 total, 8 up, 8 in 2024-09-13T00:39:13.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:12 smithi136 bash[76371]: cluster 2024-09-13T00:39:12.426621+0000 mgr.y (mgr.24661) 1862 : cluster [DBG] pgmap v3352: 137 pgs: 137 active+clean; 895 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 185 KiB/s rd, 18 MiB/s wr, 3.69k op/s 2024-09-13T00:39:13.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:12 smithi181 bash[67587]: cluster 2024-09-13T00:39:11.868702+0000 mon.a (mon.0) 3581 : cluster [DBG] osdmap e1929: 8 total, 8 up, 8 in 2024-09-13T00:39:13.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:12 smithi181 bash[67587]: cluster 2024-09-13T00:39:12.426621+0000 mgr.y (mgr.24661) 1862 : cluster [DBG] pgmap v3352: 137 pgs: 137 active+clean; 895 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 185 KiB/s rd, 18 MiB/s wr, 3.69k op/s 2024-09-13T00:39:13.936 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to HEAD diff [978366~12404130,17113294~39372164,58515289~28499682,89250005~13234343,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~20663168,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240318840~11918637,252886187~622714,260842262~7844195,269960512~2117506,273064912~36449096,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,341594269~18955574,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:13.936 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~39372164,58515289~28499682,89250005~13234343,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~20663168,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240318840~11918637,252886187~622714,260842262~7844195,269960512~2117506,273064912~36449096,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,341594269~18955574,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:13.999 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,16777216~70237755,88080384~14403964,104857600~2026393,111245499~9980052,121634816~18591770,142230204~1250407,143604550~40657412,184549376~3692163,188743680~10830113,201556540~32834723,239075328~14433573,260046848~12031170,272629760~36884248,310378496~24935507,335934186~24615657,360710144~44231030,405696667~5790944] 2024-09-13T00:39:14.000 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to HEAD diff [978366~12404130,17113294~26808795,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:14.000 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~26808795,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:14.007 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,16777216~27144873,46526671~40488300,91032793~11451555,104857600~2026393,111245499~9980052,121634816~18591770,142230204~1250407,143604550~23038119,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~35982749,310378496~24935507,335934186~4465000,343284705~17265138,360710144~44231030,405696667~5790944] 2024-09-13T00:39:14.007 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to HEAD diff [978366~12404130,17113294~3323835,22666579~21255510,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:14.007 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~21255510,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:14.013 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~21255510,46526671~40488300,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~35982749,310378496~24935507,335934186~4465000,343284705~17265138,360710144~44231030,405696667~5790944] 2024-09-13T00:39:14.013 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,46526671~9958787,58515289~15314175,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,405696667~5790944] 2024-09-13T00:39:14.013 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,46526671~9958787,58515289~15314175,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,405696667~5790944] 2024-09-13T00:39:14.019 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~16987636,46526671~27302793,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~35982749,310378496~5136352,316301726~19012277,335934186~4465000,343284705~17265138,360710144~42368580,405696667~5790944] 2024-09-13T00:39:14.019 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,409295430~1606555] 2024-09-13T00:39:14.019 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,409295430~1606555] 2024-09-13T00:39:14.025 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~18421176,292988707~15623802,310378496~5136352,316301726~19012277,335934186~4465000,343284705~17265138,360710144~42368580,409295430~1606555] 2024-09-13T00:39:14.025 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:14.025 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:14.031 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~18421176,292988707~15623802,310378496~5136352,316301726~19012277,335934186~1814726,343284705~17265138,360710144~16554896,378757665~24321059] 2024-09-13T00:39:14.031 INFO:tasks.workunit.client.0.smithi136.stdout:from 6 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273064912~17986024,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:14.031 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273064912~17986024,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:14.037 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,266675149~244059,272629760~18421176,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,343284705~17265138,360710144~16554896,378757665~24321059] 2024-09-13T00:39:14.037 INFO:tasks.workunit.client.0.smithi136.stdout:from 7 to HEAD diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~4720159,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~16427001,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273064912~4591743,277779043~4483937,282726102~8324834,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~14864738,379003744~7346160,389458074~13620650] 2024-09-13T00:39:14.037 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~4720159,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~16427001,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273064912~4591743,277779043~4483937,282726102~8324834,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~14864738,379003744~7346160,389458074~13620650] 2024-09-13T00:39:14.042 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~4720159,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~14961695,184549376~3692163,188743680~10830113,201556540~16427001,218740863~15650400,240828348~12680553,266675149~244059,272629760~5026895,277779043~4483937,282726102~8324834,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,344823952~15725891,360710144~16554896,379003744~7346160,389458074~13620650] 2024-09-13T00:39:14.042 INFO:tasks.workunit.client.0.smithi136.stdout:from 8 to HEAD diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180376314~3885648,187369573~871966,192121569~7452224,201556540~16427001,220044632~14346631,240828348~10753127,252886187~622714,266675149~244059,273064912~4591743,277779043~4483937,282726102~8324834,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:14.042 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180376314~3885648,187369573~871966,192121569~7452224,201556540~16427001,220044632~14346631,240828348~10753127,252886187~622714,266675149~244059,273064912~4591743,277779043~4483937,282726102~8324834,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:14.048 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180376314~3885648,184549376~3692163,188743680~10830113,201556540~16427001,218740863~15650400,240828348~10753127,252237477~1271424,266675149~244059,272629760~5026895,277779043~4483937,282726102~8324834,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,344823952~15725891,360710144~14924619,379003744~7346160,389458074~13620650] 2024-09-13T00:39:14.048 INFO:tasks.workunit.client.0.smithi136.stdout:from 9 to HEAD diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~8291936,91032793~11451555,105459787~1424206,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:14.048 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~8291936,91032793~11451555,105459787~1424206,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:14.054 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~8291936,91032793~11451555,104857600~2026393,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,184549376~3692163,188743680~10830113,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252237477~1271424,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,344823952~15725891,360710144~14924619,379003744~7346160,389458074~13620650] 2024-09-13T00:39:14.054 INFO:tasks.workunit.client.0.smithi136.stdout:from 10 to HEAD diff [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~11451555,105459787~1424206,122043306~5052094,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~8079036,355040685~5509158,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:14.054 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~11451555,105459787~1424206,122043306~5052094,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~8079036,355040685~5509158,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:14.060 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~11451555,104857600~2026393,121634816~5460584,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,184549376~3692163,188743680~10830113,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252237477~1271424,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,344823952~8079036,355040685~5509158,360710144~14924619,379003744~7346160,389458074~13620650] 2024-09-13T00:39:14.060 INFO:tasks.workunit.client.0.smithi136.stdout:from 11 to HEAD diff [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~11451555,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,294714089~7671666,312566619~2948229,316301726~10296905,327427195~7886808,344823952~8079036,355040685~5509158,362400302~13234461,379003744~7346160,389458074~7853223,401222124~243609] 2024-09-13T00:39:14.060 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~11451555,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,294714089~7671666,312566619~2948229,316301726~10296905,327427195~7886808,344823952~8079036,355040685~5509158,362400302~13234461,379003744~7346160,389458074~7853223,401222124~243609] 2024-09-13T00:39:14.066 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~11451555,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,184549376~3692163,188743680~10830113,201556540~16427001,222829084~11562179,248606422~493001,252237477~1271424,273064912~4591743,277779043~4483937,294714089~7671666,310378496~5136352,316301726~10296905,327427195~7886808,344823952~8079036,355040685~5509158,360710144~14924619,379003744~7346160,389458074~7853223,401222124~243609] 2024-09-13T00:39:14.066 INFO:tasks.workunit.client.0.smithi136.stdout:from 12 to HEAD diff [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~4865453,99390713~475647,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,248606422~493001,252886187~622714,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,362400302~13234461,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:14.066 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~4865453,99390713~475647,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,248606422~493001,252886187~622714,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,362400302~13234461,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:14.072 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~4865453,99390713~475647,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,184549376~3692163,188743680~10830113,201556540~16427001,222829084~11562179,248606422~493001,252237477~1271424,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,360710144~14924619,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:14.072 INFO:tasks.workunit.client.0.smithi136.stdout:from 13 to HEAD diff [978366~12404130,17650227~1831268,28753760~4043921,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~4865453,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~17787685,171460898~6550730,192121569~7452224,201939997~16043544,222829084~11562179,248606422~493001,252886187~622714,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,362400302~13234461,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:14.072 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17650227~1831268,28753760~4043921,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~4865453,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~17787685,171460898~6550730,192121569~7452224,201939997~16043544,222829084~11562179,248606422~493001,252886187~622714,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,362400302~13234461,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:14.078 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17650227~1831268,28753760~4043921,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~4865453,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~17787685,171460898~6550730,188743680~10830113,201939997~16043544,222829084~11562179,248606422~493001,252237477~1271424,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,360710144~14924619,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:14.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:13 smithi136 bash[75164]: cluster 2024-09-13T00:39:12.882056+0000 mon.a (mon.0) 3582 : cluster [DBG] osdmap e1930: 8 total, 8 up, 8 in 2024-09-13T00:39:14.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:13 smithi136 bash[76371]: cluster 2024-09-13T00:39:12.882056+0000 mon.a (mon.0) 3582 : cluster [DBG] osdmap e1930: 8 total, 8 up, 8 in 2024-09-13T00:39:14.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:13 smithi181 bash[67587]: cluster 2024-09-13T00:39:12.882056+0000 mon.a (mon.0) 3582 : cluster [DBG] osdmap e1930: 8 total, 8 up, 8 in 2024-09-13T00:39:14.961 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to HEAD diff [978366~12404130,17113294~39372164,58515289~28499682,89250005~13234343,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~20663168,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240318840~11918637,252886187~622714,260842262~7844195,269960512~2117506,273064912~36449096,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,341594269~18955574,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:14.961 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~39372164,58515289~28499682,89250005~13234343,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~20663168,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240318840~11918637,252886187~622714,260842262~7844195,269960512~2117506,273064912~36449096,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,341594269~18955574,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:15.035 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,16777216~70237755,88080384~14403964,104857600~2026393,111245499~9980052,121634816~18591770,142230204~1250407,143604550~40657412,184549376~3692163,188743680~10830113,201556540~32834723,239075328~14433573,260046848~12031170,272629760~36884248,310378496~24935507,335934186~24615657,360710144~44231030,405696667~5790944] 2024-09-13T00:39:15.036 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to HEAD diff [978366~12404130,17113294~26808795,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:15.036 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~26808795,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:15.042 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,16777216~27144873,46526671~40488300,91032793~11451555,104857600~2026393,111245499~9980052,121634816~18591770,142230204~1250407,143604550~23038119,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~35982749,310378496~24935507,335934186~4465000,343284705~17265138,360710144~44231030,405696667~5790944] 2024-09-13T00:39:15.042 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to HEAD diff [978366~12404130,17113294~3323835,22666579~21255510,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:15.042 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~21255510,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:15.048 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~21255510,46526671~40488300,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~35982749,310378496~24935507,335934186~4465000,343284705~17265138,360710144~44231030,405696667~5790944] 2024-09-13T00:39:15.048 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,46526671~9958787,58515289~15314175,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,405696667~5790944] 2024-09-13T00:39:15.048 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,46526671~9958787,58515289~15314175,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,405696667~5790944] 2024-09-13T00:39:15.054 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~16987636,46526671~27302793,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~35982749,310378496~5136352,316301726~19012277,335934186~4465000,343284705~17265138,360710144~42368580,405696667~5790944] 2024-09-13T00:39:15.054 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,409295430~1606555] 2024-09-13T00:39:15.054 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,409295430~1606555] 2024-09-13T00:39:15.060 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~18421176,292988707~15623802,310378496~5136352,316301726~19012277,335934186~4465000,343284705~17265138,360710144~42368580,409295430~1606555] 2024-09-13T00:39:15.060 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:15.060 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:15.066 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~18421176,292988707~15623802,310378496~5136352,316301726~19012277,335934186~1814726,343284705~17265138,360710144~16554896,378757665~24321059] 2024-09-13T00:39:15.066 INFO:tasks.workunit.client.0.smithi136.stdout:from 6 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273064912~17986024,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:15.066 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273064912~17986024,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:15.071 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,266675149~244059,272629760~18421176,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,343284705~17265138,360710144~16554896,378757665~24321059] 2024-09-13T00:39:15.072 INFO:tasks.workunit.client.0.smithi136.stdout:from 7 to HEAD diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~4720159,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~16427001,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273064912~4591743,277779043~4483937,282726102~8324834,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~14864738,379003744~7346160,389458074~13620650] 2024-09-13T00:39:15.072 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~4720159,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~16427001,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273064912~4591743,277779043~4483937,282726102~8324834,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~14864738,379003744~7346160,389458074~13620650] 2024-09-13T00:39:15.078 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~4720159,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~14961695,184549376~3692163,188743680~10830113,201556540~16427001,218740863~15650400,240828348~12680553,266675149~244059,272629760~5026895,277779043~4483937,282726102~8324834,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,344823952~15725891,360710144~16554896,379003744~7346160,389458074~13620650] 2024-09-13T00:39:15.078 INFO:tasks.workunit.client.0.smithi136.stdout:from 8 to HEAD diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180376314~3885648,187369573~871966,192121569~7452224,201556540~16427001,220044632~14346631,240828348~10753127,252886187~622714,266675149~244059,273064912~4591743,277779043~4483937,282726102~8324834,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:15.078 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180376314~3885648,187369573~871966,192121569~7452224,201556540~16427001,220044632~14346631,240828348~10753127,252886187~622714,266675149~244059,273064912~4591743,277779043~4483937,282726102~8324834,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:15.084 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180376314~3885648,184549376~3692163,188743680~10830113,201556540~16427001,218740863~15650400,240828348~10753127,252237477~1271424,266675149~244059,272629760~5026895,277779043~4483937,282726102~8324834,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,344823952~15725891,360710144~14924619,379003744~7346160,389458074~13620650] 2024-09-13T00:39:15.084 INFO:tasks.workunit.client.0.smithi136.stdout:from 9 to HEAD diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~8291936,91032793~11451555,105459787~1424206,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:15.084 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~8291936,91032793~11451555,105459787~1424206,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:15.089 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~8291936,91032793~11451555,104857600~2026393,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,184549376~3692163,188743680~10830113,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252237477~1271424,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,344823952~15725891,360710144~14924619,379003744~7346160,389458074~13620650] 2024-09-13T00:39:15.089 INFO:tasks.workunit.client.0.smithi136.stdout:from 10 to HEAD diff [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~11451555,105459787~1424206,122043306~5052094,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~8079036,355040685~5509158,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:15.090 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~11451555,105459787~1424206,122043306~5052094,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~8079036,355040685~5509158,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:15.095 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~11451555,104857600~2026393,121634816~5460584,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,184549376~3692163,188743680~10830113,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252237477~1271424,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,344823952~8079036,355040685~5509158,360710144~14924619,379003744~7346160,389458074~13620650] 2024-09-13T00:39:15.095 INFO:tasks.workunit.client.0.smithi136.stdout:from 11 to HEAD diff [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~11451555,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,294714089~7671666,312566619~2948229,316301726~10296905,327427195~7886808,344823952~8079036,355040685~5509158,362400302~13234461,379003744~7346160,389458074~7853223,401222124~243609] 2024-09-13T00:39:15.095 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~11451555,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,294714089~7671666,312566619~2948229,316301726~10296905,327427195~7886808,344823952~8079036,355040685~5509158,362400302~13234461,379003744~7346160,389458074~7853223,401222124~243609] 2024-09-13T00:39:15.101 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~11451555,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,184549376~3692163,188743680~10830113,201556540~16427001,222829084~11562179,248606422~493001,252237477~1271424,273064912~4591743,277779043~4483937,294714089~7671666,310378496~5136352,316301726~10296905,327427195~7886808,344823952~8079036,355040685~5509158,360710144~14924619,379003744~7346160,389458074~7853223,401222124~243609] 2024-09-13T00:39:15.101 INFO:tasks.workunit.client.0.smithi136.stdout:from 12 to HEAD diff [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~4865453,99390713~475647,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,248606422~493001,252886187~622714,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,362400302~13234461,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:15.101 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~4865453,99390713~475647,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,248606422~493001,252886187~622714,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,362400302~13234461,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:15.107 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~4865453,99390713~475647,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,184549376~3692163,188743680~10830113,201556540~16427001,222829084~11562179,248606422~493001,252237477~1271424,273064912~4591743,294714089~7671666,319038423~7560208,327427195~2026374,344823952~8079036,355040685~5509158,360710144~14924619,379003744~5394585,389458074~7853223,401222124~243609] 2024-09-13T00:39:15.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:14 smithi136 bash[75164]: cluster 2024-09-13T00:39:13.899769+0000 mon.a (mon.0) 3583 : cluster [DBG] osdmap e1931: 8 total, 8 up, 8 in 2024-09-13T00:39:15.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:14 smithi136 bash[75164]: cluster 2024-09-13T00:39:14.427323+0000 mgr.y (mgr.24661) 1863 : cluster [DBG] pgmap v3355: 137 pgs: 137 active+clean; 885 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 203 KiB/s rd, 7.3 MiB/s wr, 4.52k op/s 2024-09-13T00:39:15.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:14 smithi136 bash[76371]: cluster 2024-09-13T00:39:13.899769+0000 mon.a (mon.0) 3583 : cluster [DBG] osdmap e1931: 8 total, 8 up, 8 in 2024-09-13T00:39:15.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:14 smithi136 bash[76371]: cluster 2024-09-13T00:39:14.427323+0000 mgr.y (mgr.24661) 1863 : cluster [DBG] pgmap v3355: 137 pgs: 137 active+clean; 885 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 203 KiB/s rd, 7.3 MiB/s wr, 4.52k op/s 2024-09-13T00:39:15.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:14 smithi181 bash[67587]: cluster 2024-09-13T00:39:13.899769+0000 mon.a (mon.0) 3583 : cluster [DBG] osdmap e1931: 8 total, 8 up, 8 in 2024-09-13T00:39:15.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:14 smithi181 bash[67587]: cluster 2024-09-13T00:39:14.427323+0000 mgr.y (mgr.24661) 1863 : cluster [DBG] pgmap v3355: 137 pgs: 137 active+clean; 885 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 203 KiB/s rd, 7.3 MiB/s wr, 4.52k op/s 2024-09-13T00:39:15.977 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to HEAD diff [978366~12404130,17113294~39372164,58515289~28499682,89250005~13234343,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~20663168,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240318840~11918637,252886187~622714,260842262~7844195,269960512~2117506,273064912~36449096,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,341594269~18955574,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:15.977 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~39372164,58515289~28499682,89250005~13234343,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~20663168,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240318840~11918637,252886187~622714,260842262~7844195,269960512~2117506,273064912~36449096,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,341594269~18955574,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:16.036 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,16777216~70237755,88080384~14403964,104857600~2026393,111245499~9980052,121634816~18591770,142230204~1250407,143604550~40657412,184549376~3692163,188743680~10830113,201556540~32834723,239075328~14433573,260046848~12031170,272629760~36884248,310378496~24935507,335934186~24615657,360710144~44231030,405696667~5790944] 2024-09-13T00:39:16.037 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to HEAD diff [978366~12404130,17113294~26808795,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:16.037 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~26808795,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:16.043 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,16777216~27144873,46526671~40488300,91032793~11451555,104857600~2026393,111245499~9980052,121634816~18591770,142230204~1250407,143604550~23038119,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~35982749,310378496~24935507,335934186~4465000,343284705~17265138,360710144~44231030,405696667~5790944] 2024-09-13T00:39:16.043 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to HEAD diff [978366~12404130,17113294~3323835,22666579~21255510,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:16.043 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~21255510,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:16.049 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~21255510,46526671~40488300,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~35982749,310378496~24935507,335934186~4465000,343284705~17265138,360710144~44231030,405696667~5790944] 2024-09-13T00:39:16.049 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,46526671~9958787,58515289~15314175,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,405696667~5790944] 2024-09-13T00:39:16.049 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,46526671~9958787,58515289~15314175,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,405696667~5790944] 2024-09-13T00:39:16.056 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~16987636,46526671~27302793,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~35982749,310378496~5136352,316301726~19012277,335934186~4465000,343284705~17265138,360710144~42368580,405696667~5790944] 2024-09-13T00:39:16.056 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,409295430~1606555] 2024-09-13T00:39:16.056 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,409295430~1606555] 2024-09-13T00:39:16.061 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~18421176,292988707~15623802,310378496~5136352,316301726~19012277,335934186~4465000,343284705~17265138,360710144~42368580,409295430~1606555] 2024-09-13T00:39:16.061 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:16.061 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:16.068 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~18421176,292988707~15623802,310378496~5136352,316301726~19012277,335934186~1814726,343284705~17265138,360710144~16554896,378757665~24321059] 2024-09-13T00:39:16.068 INFO:tasks.workunit.client.0.smithi136.stdout:from 6 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273064912~17986024,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:16.068 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273064912~17986024,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:16.074 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,266675149~244059,272629760~18421176,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,343284705~17265138,360710144~16554896,378757665~24321059] 2024-09-13T00:39:16.074 INFO:tasks.workunit.client.0.smithi136.stdout:from 7 to HEAD diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~4720159,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~16427001,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273064912~4591743,277779043~4483937,282726102~8324834,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~14864738,379003744~7346160,389458074~13620650] 2024-09-13T00:39:16.074 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~4720159,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~16427001,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273064912~4591743,277779043~4483937,282726102~8324834,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~14864738,379003744~7346160,389458074~13620650] 2024-09-13T00:39:16.079 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~4720159,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~14961695,184549376~3692163,188743680~10830113,201556540~16427001,218740863~15650400,240828348~12680553,266675149~244059,272629760~5026895,277779043~4483937,282726102~8324834,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,344823952~15725891,360710144~16554896,379003744~7346160,389458074~13620650] 2024-09-13T00:39:16.080 INFO:tasks.workunit.client.0.smithi136.stdout:from 8 to HEAD diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180376314~3885648,187369573~871966,192121569~7452224,201556540~16427001,220044632~14346631,240828348~10753127,252886187~622714,266675149~244059,273064912~4591743,277779043~4483937,282726102~8324834,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:16.080 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180376314~3885648,187369573~871966,192121569~7452224,201556540~16427001,220044632~14346631,240828348~10753127,252886187~622714,266675149~244059,273064912~4591743,277779043~4483937,282726102~8324834,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:16.085 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180376314~3885648,184549376~3692163,188743680~10830113,201556540~16427001,218740863~15650400,240828348~10753127,252237477~1271424,266675149~244059,272629760~5026895,277779043~4483937,282726102~8324834,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,344823952~15725891,360710144~14924619,379003744~7346160,389458074~13620650] 2024-09-13T00:39:16.085 INFO:tasks.workunit.client.0.smithi136.stdout:from 9 to HEAD diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~8291936,91032793~11451555,105459787~1424206,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:16.085 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~8291936,91032793~11451555,105459787~1424206,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:16.091 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~8291936,91032793~11451555,104857600~2026393,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,184549376~3692163,188743680~10830113,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252237477~1271424,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,344823952~15725891,360710144~14924619,379003744~7346160,389458074~13620650] 2024-09-13T00:39:16.091 INFO:tasks.workunit.client.0.smithi136.stdout:from 10 to HEAD diff [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~11451555,105459787~1424206,122043306~5052094,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~8079036,355040685~5509158,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:16.091 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~11451555,105459787~1424206,122043306~5052094,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~8079036,355040685~5509158,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:16.097 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~11451555,104857600~2026393,121634816~5460584,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,184549376~3692163,188743680~10830113,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252237477~1271424,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,344823952~8079036,355040685~5509158,360710144~14924619,379003744~7346160,389458074~13620650] 2024-09-13T00:39:16.097 INFO:tasks.workunit.client.0.smithi136.stdout:from 11 to HEAD diff [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~11451555,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,294714089~7671666,312566619~2948229,316301726~10296905,327427195~7886808,344823952~8079036,355040685~5509158,362400302~13234461,379003744~7346160,389458074~7853223,401222124~243609] 2024-09-13T00:39:16.097 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~11451555,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,294714089~7671666,312566619~2948229,316301726~10296905,327427195~7886808,344823952~8079036,355040685~5509158,362400302~13234461,379003744~7346160,389458074~7853223,401222124~243609] 2024-09-13T00:39:16.102 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~11451555,122781879~1514283,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,184549376~3692163,188743680~10830113,201556540~16427001,222829084~11562179,248606422~493001,252237477~1271424,273064912~4591743,277779043~4483937,294714089~7671666,310378496~5136352,316301726~10296905,327427195~7886808,344823952~8079036,355040685~5509158,360710144~14924619,379003744~7346160,389458074~7853223,401222124~243609] 2024-09-13T00:39:16.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:15 smithi136 bash[75164]: cluster 2024-09-13T00:39:14.934836+0000 mon.a (mon.0) 3584 : cluster [DBG] osdmap e1932: 8 total, 8 up, 8 in 2024-09-13T00:39:16.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:15 smithi136 bash[76371]: cluster 2024-09-13T00:39:14.934836+0000 mon.a (mon.0) 3584 : cluster [DBG] osdmap e1932: 8 total, 8 up, 8 in 2024-09-13T00:39:16.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:15 smithi181 bash[67587]: cluster 2024-09-13T00:39:14.934836+0000 mon.a (mon.0) 3584 : cluster [DBG] osdmap e1932: 8 total, 8 up, 8 in 2024-09-13T00:39:16.986 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to HEAD diff [978366~12404130,17113294~39372164,58515289~28499682,89250005~13234343,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~20663168,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240318840~11918637,252886187~622714,260842262~7844195,269960512~2117506,273064912~36449096,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,341594269~18955574,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:16.986 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~39372164,58515289~28499682,89250005~13234343,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~20663168,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240318840~11918637,252886187~622714,260842262~7844195,269960512~2117506,273064912~36449096,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,341594269~18955574,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:17.062 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,16777216~70237755,88080384~14403964,104857600~2026393,111245499~9980052,121634816~18591770,142230204~1250407,143604550~40657412,184549376~3692163,188743680~10830113,201556540~32834723,239075328~14433573,260046848~12031170,272629760~36884248,310378496~24935507,335934186~24615657,360710144~44231030,405696667~5790944] 2024-09-13T00:39:17.062 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to HEAD diff [978366~12404130,17113294~26808795,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:17.062 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~26808795,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:17.068 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,16777216~27144873,46526671~40488300,91032793~11451555,104857600~2026393,111245499~9980052,121634816~18591770,142230204~1250407,143604550~23038119,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~35982749,310378496~24935507,335934186~4465000,343284705~17265138,360710144~44231030,405696667~5790944] 2024-09-13T00:39:17.068 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to HEAD diff [978366~12404130,17113294~3323835,22666579~21255510,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:17.068 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~21255510,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:17.074 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~21255510,46526671~40488300,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~35982749,310378496~24935507,335934186~4465000,343284705~17265138,360710144~44231030,405696667~5790944] 2024-09-13T00:39:17.074 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,46526671~9958787,58515289~15314175,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,405696667~5790944] 2024-09-13T00:39:17.074 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,46526671~9958787,58515289~15314175,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,405696667~5790944] 2024-09-13T00:39:17.080 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~16987636,46526671~27302793,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~35982749,310378496~5136352,316301726~19012277,335934186~4465000,343284705~17265138,360710144~42368580,405696667~5790944] 2024-09-13T00:39:17.081 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,409295430~1606555] 2024-09-13T00:39:17.081 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,409295430~1606555] 2024-09-13T00:39:17.086 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~18421176,292988707~15623802,310378496~5136352,316301726~19012277,335934186~4465000,343284705~17265138,360710144~42368580,409295430~1606555] 2024-09-13T00:39:17.086 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:17.086 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:17.092 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~18421176,292988707~15623802,310378496~5136352,316301726~19012277,335934186~1814726,343284705~17265138,360710144~16554896,378757665~24321059] 2024-09-13T00:39:17.092 INFO:tasks.workunit.client.0.smithi136.stdout:from 6 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273064912~17986024,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:17.092 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273064912~17986024,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:17.097 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,266675149~244059,272629760~18421176,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,343284705~17265138,360710144~16554896,378757665~24321059] 2024-09-13T00:39:17.098 INFO:tasks.workunit.client.0.smithi136.stdout:from 7 to HEAD diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~4720159,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~16427001,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273064912~4591743,277779043~4483937,282726102~8324834,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~14864738,379003744~7346160,389458074~13620650] 2024-09-13T00:39:17.098 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~4720159,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~16427001,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273064912~4591743,277779043~4483937,282726102~8324834,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~14864738,379003744~7346160,389458074~13620650] 2024-09-13T00:39:17.103 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~4720159,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~14961695,184549376~3692163,188743680~10830113,201556540~16427001,218740863~15650400,240828348~12680553,266675149~244059,272629760~5026895,277779043~4483937,282726102~8324834,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,344823952~15725891,360710144~16554896,379003744~7346160,389458074~13620650] 2024-09-13T00:39:17.103 INFO:tasks.workunit.client.0.smithi136.stdout:from 8 to HEAD diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180376314~3885648,187369573~871966,192121569~7452224,201556540~16427001,220044632~14346631,240828348~10753127,252886187~622714,266675149~244059,273064912~4591743,277779043~4483937,282726102~8324834,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:17.103 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180376314~3885648,187369573~871966,192121569~7452224,201556540~16427001,220044632~14346631,240828348~10753127,252886187~622714,266675149~244059,273064912~4591743,277779043~4483937,282726102~8324834,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:17.109 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180376314~3885648,184549376~3692163,188743680~10830113,201556540~16427001,218740863~15650400,240828348~10753127,252237477~1271424,266675149~244059,272629760~5026895,277779043~4483937,282726102~8324834,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,344823952~15725891,360710144~14924619,379003744~7346160,389458074~13620650] 2024-09-13T00:39:17.109 INFO:tasks.workunit.client.0.smithi136.stdout:from 9 to HEAD diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~8291936,91032793~11451555,105459787~1424206,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:17.110 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~8291936,91032793~11451555,105459787~1424206,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:17.115 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~8291936,91032793~11451555,104857600~2026393,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,184549376~3692163,188743680~10830113,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252237477~1271424,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,344823952~15725891,360710144~14924619,379003744~7346160,389458074~13620650] 2024-09-13T00:39:17.115 INFO:tasks.workunit.client.0.smithi136.stdout:from 10 to HEAD diff [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~11451555,105459787~1424206,122043306~5052094,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~8079036,355040685~5509158,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:17.115 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~11451555,105459787~1424206,122043306~5052094,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~8079036,355040685~5509158,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:17.121 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22837062~9960619,37199402~669117,47180962~9304496,58865372~14964092,76198939~3215002,80242387~800824,91032793~11451555,104857600~2026393,121634816~5460584,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,184549376~3692163,188743680~10830113,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252237477~1271424,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,344823952~8079036,355040685~5509158,360710144~14924619,379003744~7346160,389458074~13620650] 2024-09-13T00:39:17.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:16 smithi136 bash[75164]: cluster 2024-09-13T00:39:15.935425+0000 mon.a (mon.0) 3585 : cluster [DBG] osdmap e1933: 8 total, 8 up, 8 in 2024-09-13T00:39:17.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:16 smithi136 bash[75164]: cluster 2024-09-13T00:39:16.428155+0000 mgr.y (mgr.24661) 1864 : cluster [DBG] pgmap v3358: 137 pgs: 137 active+clean; 862 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 143 KiB/s rd, 6.2 KiB/s wr, 2.74k op/s 2024-09-13T00:39:17.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:16 smithi136 bash[76371]: cluster 2024-09-13T00:39:15.935425+0000 mon.a (mon.0) 3585 : cluster [DBG] osdmap e1933: 8 total, 8 up, 8 in 2024-09-13T00:39:17.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:16 smithi136 bash[76371]: cluster 2024-09-13T00:39:16.428155+0000 mgr.y (mgr.24661) 1864 : cluster [DBG] pgmap v3358: 137 pgs: 137 active+clean; 862 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 143 KiB/s rd, 6.2 KiB/s wr, 2.74k op/s 2024-09-13T00:39:17.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:16 smithi181 bash[67587]: cluster 2024-09-13T00:39:15.935425+0000 mon.a (mon.0) 3585 : cluster [DBG] osdmap e1933: 8 total, 8 up, 8 in 2024-09-13T00:39:17.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:16 smithi181 bash[67587]: cluster 2024-09-13T00:39:16.428155+0000 mgr.y (mgr.24661) 1864 : cluster [DBG] pgmap v3358: 137 pgs: 137 active+clean; 862 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 143 KiB/s rd, 6.2 KiB/s wr, 2.74k op/s 2024-09-13T00:39:17.994 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to HEAD diff [978366~12404130,17113294~39372164,58515289~28499682,89250005~13234343,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~20663168,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240318840~11918637,252886187~622714,260842262~7844195,269960512~2117506,273064912~36449096,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,341594269~18955574,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:17.994 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~39372164,58515289~28499682,89250005~13234343,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~20663168,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240318840~11918637,252886187~622714,260842262~7844195,269960512~2117506,273064912~36449096,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,341594269~18955574,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:18.066 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,16777216~70237755,88080384~14403964,104857600~2026393,111245499~9980052,121634816~18591770,142230204~1250407,143604550~40657412,184549376~3692163,188743680~10830113,201556540~32834723,239075328~14433573,260046848~12031170,272629760~36884248,310378496~24935507,335934186~24615657,360710144~44231030,405696667~5790944] 2024-09-13T00:39:18.067 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to HEAD diff [978366~12404130,17113294~26808795,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:18.067 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~26808795,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:18.074 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,16777216~27144873,46526671~40488300,91032793~11451555,104857600~2026393,111245499~9980052,121634816~18591770,142230204~1250407,143604550~23038119,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~35982749,310378496~24935507,335934186~4465000,343284705~17265138,360710144~44231030,405696667~5790944] 2024-09-13T00:39:18.074 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to HEAD diff [978366~12404130,17113294~3323835,22666579~21255510,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:18.075 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~21255510,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:18.080 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~21255510,46526671~40488300,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~35982749,310378496~24935507,335934186~4465000,343284705~17265138,360710144~44231030,405696667~5790944] 2024-09-13T00:39:18.080 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,46526671~9958787,58515289~15314175,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,405696667~5790944] 2024-09-13T00:39:18.081 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,46526671~9958787,58515289~15314175,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,405696667~5790944] 2024-09-13T00:39:18.086 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~16987636,46526671~27302793,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~35982749,310378496~5136352,316301726~19012277,335934186~4465000,343284705~17265138,360710144~42368580,405696667~5790944] 2024-09-13T00:39:18.086 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,409295430~1606555] 2024-09-13T00:39:18.086 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,409295430~1606555] 2024-09-13T00:39:18.092 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~18421176,292988707~15623802,310378496~5136352,316301726~19012277,335934186~4465000,343284705~17265138,360710144~42368580,409295430~1606555] 2024-09-13T00:39:18.092 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:18.092 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:18.098 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~18421176,292988707~15623802,310378496~5136352,316301726~19012277,335934186~1814726,343284705~17265138,360710144~16554896,378757665~24321059] 2024-09-13T00:39:18.098 INFO:tasks.workunit.client.0.smithi136.stdout:from 6 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273064912~17986024,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:18.098 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273064912~17986024,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:18.103 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,266675149~244059,272629760~18421176,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,343284705~17265138,360710144~16554896,378757665~24321059] 2024-09-13T00:39:18.104 INFO:tasks.workunit.client.0.smithi136.stdout:from 7 to HEAD diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~4720159,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~16427001,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273064912~4591743,277779043~4483937,282726102~8324834,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~14864738,379003744~7346160,389458074~13620650] 2024-09-13T00:39:18.104 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~4720159,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~16427001,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273064912~4591743,277779043~4483937,282726102~8324834,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~14864738,379003744~7346160,389458074~13620650] 2024-09-13T00:39:18.109 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~4720159,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~14961695,184549376~3692163,188743680~10830113,201556540~16427001,218740863~15650400,240828348~12680553,266675149~244059,272629760~5026895,277779043~4483937,282726102~8324834,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,344823952~15725891,360710144~16554896,379003744~7346160,389458074~13620650] 2024-09-13T00:39:18.110 INFO:tasks.workunit.client.0.smithi136.stdout:from 8 to HEAD diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180376314~3885648,187369573~871966,192121569~7452224,201556540~16427001,220044632~14346631,240828348~10753127,252886187~622714,266675149~244059,273064912~4591743,277779043~4483937,282726102~8324834,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:18.110 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180376314~3885648,187369573~871966,192121569~7452224,201556540~16427001,220044632~14346631,240828348~10753127,252886187~622714,266675149~244059,273064912~4591743,277779043~4483937,282726102~8324834,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:18.115 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180376314~3885648,184549376~3692163,188743680~10830113,201556540~16427001,218740863~15650400,240828348~10753127,252237477~1271424,266675149~244059,272629760~5026895,277779043~4483937,282726102~8324834,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,344823952~15725891,360710144~14924619,379003744~7346160,389458074~13620650] 2024-09-13T00:39:18.115 INFO:tasks.workunit.client.0.smithi136.stdout:from 9 to HEAD diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~8291936,91032793~11451555,105459787~1424206,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:18.115 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~8291936,91032793~11451555,105459787~1424206,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,187369573~871966,192121569~7452224,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252886187~622714,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:18.121 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~8291936,91032793~11451555,104857600~2026393,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,171460898~6550730,184549376~3692163,188743680~10830113,201556540~16427001,222829084~11562179,240828348~6357952,248606422~493001,252237477~1271424,273064912~4591743,277779043~4483937,282726102~2562064,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,344823952~15725891,360710144~14924619,379003744~7346160,389458074~13620650] 2024-09-13T00:39:18.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:17 smithi136 bash[75164]: cluster 2024-09-13T00:39:16.960284+0000 mon.a (mon.0) 3586 : cluster [DBG] osdmap e1934: 8 total, 8 up, 8 in 2024-09-13T00:39:18.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:17 smithi136 bash[76371]: cluster 2024-09-13T00:39:16.960284+0000 mon.a (mon.0) 3586 : cluster [DBG] osdmap e1934: 8 total, 8 up, 8 in 2024-09-13T00:39:18.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:17 smithi181 bash[67587]: cluster 2024-09-13T00:39:16.960284+0000 mon.a (mon.0) 3586 : cluster [DBG] osdmap e1934: 8 total, 8 up, 8 in 2024-09-13T00:39:18.997 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to HEAD diff [978366~12404130,17113294~39372164,58515289~28499682,89250005~13234343,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~20663168,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240318840~11918637,252886187~622714,260842262~7844195,269960512~2117506,273064912~36449096,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,341594269~18955574,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:18.997 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~39372164,58515289~28499682,89250005~13234343,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~20663168,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240318840~11918637,252886187~622714,260842262~7844195,269960512~2117506,273064912~36449096,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,341594269~18955574,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:19.079 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,16777216~70237755,88080384~14403964,104857600~2026393,111245499~9980052,121634816~18591770,142230204~1250407,143604550~40657412,184549376~3692163,188743680~10830113,201556540~32834723,239075328~14433573,260046848~12031170,272629760~36884248,310378496~24935507,335934186~24615657,360710144~44231030,405696667~5790944] 2024-09-13T00:39:19.079 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to HEAD diff [978366~12404130,17113294~26808795,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:19.079 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~26808795,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:19.086 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,16777216~27144873,46526671~40488300,91032793~11451555,104857600~2026393,111245499~9980052,121634816~18591770,142230204~1250407,143604550~23038119,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~35982749,310378496~24935507,335934186~4465000,343284705~17265138,360710144~44231030,405696667~5790944] 2024-09-13T00:39:19.087 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to HEAD diff [978366~12404130,17113294~3323835,22666579~21255510,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:19.087 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~21255510,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:19.092 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~21255510,46526671~40488300,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~35982749,310378496~24935507,335934186~4465000,343284705~17265138,360710144~44231030,405696667~5790944] 2024-09-13T00:39:19.093 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,46526671~9958787,58515289~15314175,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,405696667~5790944] 2024-09-13T00:39:19.093 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,46526671~9958787,58515289~15314175,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,405696667~5790944] 2024-09-13T00:39:19.098 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~16987636,46526671~27302793,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~35982749,310378496~5136352,316301726~19012277,335934186~4465000,343284705~17265138,360710144~42368580,405696667~5790944] 2024-09-13T00:39:19.098 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,409295430~1606555] 2024-09-13T00:39:19.098 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,409295430~1606555] 2024-09-13T00:39:19.104 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~18421176,292988707~15623802,310378496~5136352,316301726~19012277,335934186~4465000,343284705~17265138,360710144~42368580,409295430~1606555] 2024-09-13T00:39:19.104 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:19.104 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:19.110 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~18421176,292988707~15623802,310378496~5136352,316301726~19012277,335934186~1814726,343284705~17265138,360710144~16554896,378757665~24321059] 2024-09-13T00:39:19.110 INFO:tasks.workunit.client.0.smithi136.stdout:from 6 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273064912~17986024,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:19.110 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273064912~17986024,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:19.116 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,266675149~244059,272629760~18421176,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,343284705~17265138,360710144~16554896,378757665~24321059] 2024-09-13T00:39:19.116 INFO:tasks.workunit.client.0.smithi136.stdout:from 7 to HEAD diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~4720159,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~16427001,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273064912~4591743,277779043~4483937,282726102~8324834,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~14864738,379003744~7346160,389458074~13620650] 2024-09-13T00:39:19.116 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~4720159,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~16427001,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273064912~4591743,277779043~4483937,282726102~8324834,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~14864738,379003744~7346160,389458074~13620650] 2024-09-13T00:39:19.121 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~4720159,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~14961695,184549376~3692163,188743680~10830113,201556540~16427001,218740863~15650400,240828348~12680553,266675149~244059,272629760~5026895,277779043~4483937,282726102~8324834,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,344823952~15725891,360710144~16554896,379003744~7346160,389458074~13620650] 2024-09-13T00:39:19.121 INFO:tasks.workunit.client.0.smithi136.stdout:from 8 to HEAD diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180376314~3885648,187369573~871966,192121569~7452224,201556540~16427001,220044632~14346631,240828348~10753127,252886187~622714,266675149~244059,273064912~4591743,277779043~4483937,282726102~8324834,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:19.121 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180376314~3885648,187369573~871966,192121569~7452224,201556540~16427001,220044632~14346631,240828348~10753127,252886187~622714,266675149~244059,273064912~4591743,277779043~4483937,282726102~8324834,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~13234461,379003744~7346160,389458074~13620650] 2024-09-13T00:39:19.127 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180376314~3885648,184549376~3692163,188743680~10830113,201556540~16427001,218740863~15650400,240828348~10753127,252237477~1271424,266675149~244059,272629760~5026895,277779043~4483937,282726102~8324834,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,344823952~15725891,360710144~14924619,379003744~7346160,389458074~13620650] 2024-09-13T00:39:19.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:18 smithi136 bash[75164]: cluster 2024-09-13T00:39:17.961288+0000 mon.a (mon.0) 3587 : cluster [DBG] osdmap e1935: 8 total, 8 up, 8 in 2024-09-13T00:39:19.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:18 smithi136 bash[75164]: cluster 2024-09-13T00:39:18.428839+0000 mgr.y (mgr.24661) 1865 : cluster [DBG] pgmap v3361: 137 pgs: 137 active+clean; 835 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 136 KiB/s rd, 6.2 KiB/s wr, 1.69k op/s 2024-09-13T00:39:19.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:18 smithi136 bash[76371]: cluster 2024-09-13T00:39:17.961288+0000 mon.a (mon.0) 3587 : cluster [DBG] osdmap e1935: 8 total, 8 up, 8 in 2024-09-13T00:39:19.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:18 smithi136 bash[76371]: cluster 2024-09-13T00:39:18.428839+0000 mgr.y (mgr.24661) 1865 : cluster [DBG] pgmap v3361: 137 pgs: 137 active+clean; 835 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 136 KiB/s rd, 6.2 KiB/s wr, 1.69k op/s 2024-09-13T00:39:19.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:18 smithi181 bash[67587]: cluster 2024-09-13T00:39:17.961288+0000 mon.a (mon.0) 3587 : cluster [DBG] osdmap e1935: 8 total, 8 up, 8 in 2024-09-13T00:39:19.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:18 smithi181 bash[67587]: cluster 2024-09-13T00:39:18.428839+0000 mgr.y (mgr.24661) 1865 : cluster [DBG] pgmap v3361: 137 pgs: 137 active+clean; 835 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 136 KiB/s rd, 6.2 KiB/s wr, 1.69k op/s 2024-09-13T00:39:19.729 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to HEAD diff [978366~12404130,17113294~39372164,58515289~28499682,89250005~13234343,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~20663168,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240318840~11918637,252886187~622714,260842262~7844195,269960512~2117506,273064912~36449096,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,341594269~18955574,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:19.729 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~39372164,58515289~28499682,89250005~13234343,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~20663168,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240318840~11918637,252886187~622714,260842262~7844195,269960512~2117506,273064912~36449096,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,341594269~18955574,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:19.762 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:39:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:39:19] "GET /metrics HTTP/1.1" 200 36757 "" "Prometheus/2.43.0" 2024-09-13T00:39:19.809 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,16777216~70237755,88080384~14403964,104857600~2026393,111245499~9980052,121634816~18591770,142230204~1250407,143604550~40657412,184549376~3692163,188743680~10830113,201556540~32834723,239075328~14433573,260046848~12031170,272629760~36884248,310378496~24935507,335934186~24615657,360710144~44231030,405696667~5790944] 2024-09-13T00:39:19.809 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to HEAD diff [978366~12404130,17113294~26808795,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:19.809 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~26808795,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:19.815 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,16777216~27144873,46526671~40488300,91032793~11451555,104857600~2026393,111245499~9980052,121634816~18591770,142230204~1250407,143604550~23038119,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~35982749,310378496~24935507,335934186~4465000,343284705~17265138,360710144~44231030,405696667~5790944] 2024-09-13T00:39:19.815 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to HEAD diff [978366~12404130,17113294~3323835,22666579~21255510,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:19.815 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~21255510,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:19.821 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~21255510,46526671~40488300,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~35982749,310378496~24935507,335934186~4465000,343284705~17265138,360710144~44231030,405696667~5790944] 2024-09-13T00:39:19.821 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,46526671~9958787,58515289~15314175,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,405696667~5790944] 2024-09-13T00:39:19.821 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,46526671~9958787,58515289~15314175,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,405696667~5790944] 2024-09-13T00:39:19.827 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~16987636,46526671~27302793,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~35982749,310378496~5136352,316301726~19012277,335934186~4465000,343284705~17265138,360710144~42368580,405696667~5790944] 2024-09-13T00:39:19.827 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,409295430~1606555] 2024-09-13T00:39:19.828 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,409295430~1606555] 2024-09-13T00:39:19.835 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~18421176,292988707~15623802,310378496~5136352,316301726~19012277,335934186~4465000,343284705~17265138,360710144~42368580,409295430~1606555] 2024-09-13T00:39:19.836 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:19.836 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:19.842 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~18421176,292988707~15623802,310378496~5136352,316301726~19012277,335934186~1814726,343284705~17265138,360710144~16554896,378757665~24321059] 2024-09-13T00:39:19.842 INFO:tasks.workunit.client.0.smithi136.stdout:from 6 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273064912~17986024,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:19.842 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273064912~17986024,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:19.848 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,266675149~244059,272629760~18421176,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,343284705~17265138,360710144~16554896,378757665~24321059] 2024-09-13T00:39:19.848 INFO:tasks.workunit.client.0.smithi136.stdout:from 7 to HEAD diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~4720159,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~16427001,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273064912~4591743,277779043~4483937,282726102~8324834,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~14864738,379003744~7346160,389458074~13620650] 2024-09-13T00:39:19.848 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~4720159,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~16427001,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273064912~4591743,277779043~4483937,282726102~8324834,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,344823952~15725891,362400302~14864738,379003744~7346160,389458074~13620650] 2024-09-13T00:39:19.853 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22666579~10131102,37199402~669117,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~4720159,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~14961695,184549376~3692163,188743680~10830113,201556540~16427001,218740863~15650400,240828348~12680553,266675149~244059,272629760~5026895,277779043~4483937,282726102~8324834,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,344823952~15725891,360710144~16554896,379003744~7346160,389458074~13620650] 2024-09-13T00:39:20.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:19 smithi136 bash[75164]: cluster 2024-09-13T00:39:18.981447+0000 mon.a (mon.0) 3588 : cluster [DBG] osdmap e1936: 8 total, 8 up, 8 in 2024-09-13T00:39:20.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:19 smithi136 bash[75164]: cluster 2024-09-13T00:39:19.698703+0000 mon.a (mon.0) 3589 : cluster [DBG] osdmap e1937: 8 total, 8 up, 8 in 2024-09-13T00:39:20.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:19 smithi136 bash[76371]: cluster 2024-09-13T00:39:18.981447+0000 mon.a (mon.0) 3588 : cluster [DBG] osdmap e1936: 8 total, 8 up, 8 in 2024-09-13T00:39:20.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:19 smithi136 bash[76371]: cluster 2024-09-13T00:39:19.698703+0000 mon.a (mon.0) 3589 : cluster [DBG] osdmap e1937: 8 total, 8 up, 8 in 2024-09-13T00:39:20.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:19 smithi181 bash[67587]: cluster 2024-09-13T00:39:18.981447+0000 mon.a (mon.0) 3588 : cluster [DBG] osdmap e1936: 8 total, 8 up, 8 in 2024-09-13T00:39:20.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:19 smithi181 bash[67587]: cluster 2024-09-13T00:39:19.698703+0000 mon.a (mon.0) 3589 : cluster [DBG] osdmap e1937: 8 total, 8 up, 8 in 2024-09-13T00:39:20.749 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to HEAD diff [978366~12404130,17113294~39372164,58515289~28499682,89250005~13234343,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~20663168,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240318840~11918637,252886187~622714,260842262~7844195,269960512~2117506,273064912~36449096,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,341594269~18955574,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:20.749 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~39372164,58515289~28499682,89250005~13234343,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~20663168,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240318840~11918637,252886187~622714,260842262~7844195,269960512~2117506,273064912~36449096,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,341594269~18955574,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:20.811 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,16777216~70237755,88080384~14403964,104857600~2026393,111245499~9980052,121634816~18591770,142230204~1250407,143604550~40657412,184549376~3692163,188743680~10830113,201556540~32834723,239075328~14433573,260046848~12031170,272629760~36884248,310378496~24935507,335934186~24615657,360710144~44231030,405696667~5790944] 2024-09-13T00:39:20.811 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to HEAD diff [978366~12404130,17113294~26808795,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:20.811 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~26808795,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:20.818 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,16777216~27144873,46526671~40488300,91032793~11451555,104857600~2026393,111245499~9980052,121634816~18591770,142230204~1250407,143604550~23038119,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~35982749,310378496~24935507,335934186~4465000,343284705~17265138,360710144~44231030,405696667~5790944] 2024-09-13T00:39:20.818 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to HEAD diff [978366~12404130,17113294~3323835,22666579~21255510,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:20.818 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~21255510,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:20.824 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~21255510,46526671~40488300,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~35982749,310378496~24935507,335934186~4465000,343284705~17265138,360710144~44231030,405696667~5790944] 2024-09-13T00:39:20.824 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,46526671~9958787,58515289~15314175,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,405696667~5790944] 2024-09-13T00:39:20.824 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,46526671~9958787,58515289~15314175,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,405696667~5790944] 2024-09-13T00:39:20.830 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~16987636,46526671~27302793,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~35982749,310378496~5136352,316301726~19012277,335934186~4465000,343284705~17265138,360710144~42368580,405696667~5790944] 2024-09-13T00:39:20.830 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,409295430~1606555] 2024-09-13T00:39:20.830 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,409295430~1606555] 2024-09-13T00:39:20.836 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~18421176,292988707~15623802,310378496~5136352,316301726~19012277,335934186~4465000,343284705~17265138,360710144~42368580,409295430~1606555] 2024-09-13T00:39:20.836 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:20.836 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:20.842 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~18421176,292988707~15623802,310378496~5136352,316301726~19012277,335934186~1814726,343284705~17265138,360710144~16554896,378757665~24321059] 2024-09-13T00:39:20.842 INFO:tasks.workunit.client.0.smithi136.stdout:from 6 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273064912~17986024,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:20.842 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,266675149~244059,273064912~17986024,292988707~9397048,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:20.847 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,266675149~244059,272629760~18421176,292988707~9397048,310378496~5136352,316301726~19012277,335934186~1814726,343284705~17265138,360710144~16554896,378757665~24321059] 2024-09-13T00:39:21.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:20 smithi136 bash[75164]: cluster 2024-09-13T00:39:20.429476+0000 mgr.y (mgr.24661) 1866 : cluster [DBG] pgmap v3364: 137 pgs: 137 active+clean; 803 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 2.2 KiB/s rd, 0 B/s wr, 851 op/s 2024-09-13T00:39:21.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:20 smithi136 bash[75164]: cluster 2024-09-13T00:39:20.701453+0000 mon.a (mon.0) 3590 : cluster [DBG] osdmap e1938: 8 total, 8 up, 8 in 2024-09-13T00:39:21.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:20 smithi136 bash[76371]: cluster 2024-09-13T00:39:20.429476+0000 mgr.y (mgr.24661) 1866 : cluster [DBG] pgmap v3364: 137 pgs: 137 active+clean; 803 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 2.2 KiB/s rd, 0 B/s wr, 851 op/s 2024-09-13T00:39:21.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:20 smithi136 bash[76371]: cluster 2024-09-13T00:39:20.701453+0000 mon.a (mon.0) 3590 : cluster [DBG] osdmap e1938: 8 total, 8 up, 8 in 2024-09-13T00:39:21.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:20 smithi181 bash[67587]: cluster 2024-09-13T00:39:20.429476+0000 mgr.y (mgr.24661) 1866 : cluster [DBG] pgmap v3364: 137 pgs: 137 active+clean; 803 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 2.2 KiB/s rd, 0 B/s wr, 851 op/s 2024-09-13T00:39:21.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:20 smithi181 bash[67587]: cluster 2024-09-13T00:39:20.701453+0000 mon.a (mon.0) 3590 : cluster [DBG] osdmap e1938: 8 total, 8 up, 8 in 2024-09-13T00:39:21.722 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to HEAD diff [978366~12404130,17113294~39372164,58515289~28499682,89250005~13234343,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~20663168,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240318840~11918637,252886187~622714,260842262~7844195,269960512~2117506,273064912~36449096,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,341594269~18955574,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:21.722 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~39372164,58515289~28499682,89250005~13234343,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~20663168,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240318840~11918637,252886187~622714,260842262~7844195,269960512~2117506,273064912~36449096,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,341594269~18955574,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:21.804 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,16777216~70237755,88080384~14403964,104857600~2026393,111245499~9980052,121634816~18591770,142230204~1250407,143604550~40657412,184549376~3692163,188743680~10830113,201556540~32834723,239075328~14433573,260046848~12031170,272629760~36884248,310378496~24935507,335934186~24615657,360710144~44231030,405696667~5790944] 2024-09-13T00:39:21.804 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to HEAD diff [978366~12404130,17113294~26808795,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:21.804 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~26808795,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:21.810 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,16777216~27144873,46526671~40488300,91032793~11451555,104857600~2026393,111245499~9980052,121634816~18591770,142230204~1250407,143604550~23038119,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~35982749,310378496~24935507,335934186~4465000,343284705~17265138,360710144~44231030,405696667~5790944] 2024-09-13T00:39:21.811 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to HEAD diff [978366~12404130,17113294~3323835,22666579~21255510,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:21.811 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~21255510,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:21.817 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~21255510,46526671~40488300,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~35982749,310378496~24935507,335934186~4465000,343284705~17265138,360710144~44231030,405696667~5790944] 2024-09-13T00:39:21.817 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,46526671~9958787,58515289~15314175,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,405696667~5790944] 2024-09-13T00:39:21.817 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,46526671~9958787,58515289~15314175,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,405696667~5790944] 2024-09-13T00:39:21.823 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~16987636,46526671~27302793,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~35982749,310378496~5136352,316301726~19012277,335934186~4465000,343284705~17265138,360710144~42368580,405696667~5790944] 2024-09-13T00:39:21.823 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,409295430~1606555] 2024-09-13T00:39:21.823 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,409295430~1606555] 2024-09-13T00:39:21.829 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~18421176,292988707~15623802,310378496~5136352,316301726~19012277,335934186~4465000,343284705~17265138,360710144~42368580,409295430~1606555] 2024-09-13T00:39:21.829 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:21.829 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,343284705~17265138,362400302~14864738,378757665~7592239,387920372~15158352] 2024-09-13T00:39:21.834 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,148016038~18626631,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~18421176,292988707~15623802,310378496~5136352,316301726~19012277,335934186~1814726,343284705~17265138,360710144~16554896,378757665~24321059] 2024-09-13T00:39:22.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:39:22 smithi181 bash[95505]: ts=2024-09-13T00:39:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:39:22.746 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to HEAD diff [978366~12404130,17113294~39372164,58515289~28499682,89250005~13234343,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~20663168,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240318840~11918637,252886187~622714,260842262~7844195,269960512~2117506,273064912~36449096,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,341594269~18955574,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:22.746 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~39372164,58515289~28499682,89250005~13234343,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~20663168,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240318840~11918637,252886187~622714,260842262~7844195,269960512~2117506,273064912~36449096,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,341594269~18955574,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:22.812 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,16777216~70237755,88080384~14403964,104857600~2026393,111245499~9980052,121634816~18591770,142230204~1250407,143604550~40657412,184549376~3692163,188743680~10830113,201556540~32834723,239075328~14433573,260046848~12031170,272629760~36884248,310378496~24935507,335934186~24615657,360710144~44231030,405696667~5790944] 2024-09-13T00:39:22.812 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to HEAD diff [978366~12404130,17113294~26808795,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:22.812 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~26808795,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:22.817 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,16777216~27144873,46526671~40488300,91032793~11451555,104857600~2026393,111245499~9980052,121634816~18591770,142230204~1250407,143604550~23038119,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~35982749,310378496~24935507,335934186~4465000,343284705~17265138,360710144~44231030,405696667~5790944] 2024-09-13T00:39:22.817 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to HEAD diff [978366~12404130,17113294~3323835,22666579~21255510,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:22.817 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~21255510,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:22.823 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~21255510,46526671~40488300,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~35982749,310378496~24935507,335934186~4465000,343284705~17265138,360710144~44231030,405696667~5790944] 2024-09-13T00:39:22.823 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,46526671~9958787,58515289~15314175,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,405696667~5790944] 2024-09-13T00:39:22.823 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,46526671~9958787,58515289~15314175,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,405696667~5790944] 2024-09-13T00:39:22.831 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~16987636,46526671~27302793,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~35982749,310378496~5136352,316301726~19012277,335934186~4465000,343284705~17265138,360710144~42368580,405696667~5790944] 2024-09-13T00:39:22.831 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,409295430~1606555] 2024-09-13T00:39:22.831 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~17986024,292988707~15623802,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,409295430~1606555] 2024-09-13T00:39:22.841 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~16987636,47180962~9304496,58865372~14964092,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~18421176,292988707~15623802,310378496~5136352,316301726~19012277,335934186~4465000,343284705~17265138,360710144~42368580,409295430~1606555] 2024-09-13T00:39:23.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:22 smithi136 bash[75164]: cluster 2024-09-13T00:39:21.703419+0000 mon.a (mon.0) 3591 : cluster [DBG] osdmap e1939: 8 total, 8 up, 8 in 2024-09-13T00:39:23.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:22 smithi136 bash[75164]: cluster 2024-09-13T00:39:22.430358+0000 mgr.y (mgr.24661) 1867 : cluster [DBG] pgmap v3367: 137 pgs: 2 active+clean+snaptrim, 3 active+clean+snaptrim_wait, 132 active+clean; 740 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 104 KiB/s rd, 6.2 KiB/s wr, 1.23k op/s 2024-09-13T00:39:23.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:22 smithi136 bash[76371]: cluster 2024-09-13T00:39:21.703419+0000 mon.a (mon.0) 3591 : cluster [DBG] osdmap e1939: 8 total, 8 up, 8 in 2024-09-13T00:39:23.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:22 smithi136 bash[76371]: cluster 2024-09-13T00:39:22.430358+0000 mgr.y (mgr.24661) 1867 : cluster [DBG] pgmap v3367: 137 pgs: 2 active+clean+snaptrim, 3 active+clean+snaptrim_wait, 132 active+clean; 740 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 104 KiB/s rd, 6.2 KiB/s wr, 1.23k op/s 2024-09-13T00:39:23.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:22 smithi181 bash[67587]: cluster 2024-09-13T00:39:21.703419+0000 mon.a (mon.0) 3591 : cluster [DBG] osdmap e1939: 8 total, 8 up, 8 in 2024-09-13T00:39:23.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:22 smithi181 bash[67587]: cluster 2024-09-13T00:39:22.430358+0000 mgr.y (mgr.24661) 1867 : cluster [DBG] pgmap v3367: 137 pgs: 2 active+clean+snaptrim, 3 active+clean+snaptrim_wait, 132 active+clean; 740 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 104 KiB/s rd, 6.2 KiB/s wr, 1.23k op/s 2024-09-13T00:39:23.752 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to HEAD diff [978366~12404130,17113294~39372164,58515289~28499682,89250005~13234343,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~20663168,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240318840~11918637,252886187~622714,260842262~7844195,269960512~2117506,273064912~36449096,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,341594269~18955574,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:23.753 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~39372164,58515289~28499682,89250005~13234343,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~20663168,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240318840~11918637,252886187~622714,260842262~7844195,269960512~2117506,273064912~36449096,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,341594269~18955574,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:23.831 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,16777216~70237755,88080384~14403964,104857600~2026393,111245499~9980052,121634816~18591770,142230204~1250407,143604550~40657412,184549376~3692163,188743680~10830113,201556540~32834723,239075328~14433573,260046848~12031170,272629760~36884248,310378496~24935507,335934186~24615657,360710144~44231030,405696667~5790944] 2024-09-13T00:39:23.831 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to HEAD diff [978366~12404130,17113294~26808795,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:23.831 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~26808795,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:23.839 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,16777216~27144873,46526671~40488300,91032793~11451555,104857600~2026393,111245499~9980052,121634816~18591770,142230204~1250407,143604550~23038119,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~35982749,310378496~24935507,335934186~4465000,343284705~17265138,360710144~44231030,405696667~5790944] 2024-09-13T00:39:23.839 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to HEAD diff [978366~12404130,17113294~3323835,22666579~21255510,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:23.839 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~21255510,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:23.847 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~21255510,46526671~40488300,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~35982749,310378496~24935507,335934186~4465000,343284705~17265138,360710144~44231030,405696667~5790944] 2024-09-13T00:39:23.847 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to HEAD diff [978366~12404130,17113294~3323835,22666579~16987636,46526671~9958787,58515289~15314175,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,405696667~5790944] 2024-09-13T00:39:23.847 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~16987636,46526671~9958787,58515289~15314175,76198939~10741965,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~15158352,405696667~5790944] 2024-09-13T00:39:23.853 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~16987636,46526671~27302793,76198939~10741965,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~35982749,310378496~5136352,316301726~19012277,335934186~4465000,343284705~17265138,360710144~42368580,405696667~5790944] 2024-09-13T00:39:24.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:23 smithi136 bash[75164]: cluster 2024-09-13T00:39:22.714161+0000 mon.a (mon.0) 3592 : cluster [DBG] osdmap e1940: 8 total, 8 up, 8 in 2024-09-13T00:39:24.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:23 smithi136 bash[76371]: cluster 2024-09-13T00:39:22.714161+0000 mon.a (mon.0) 3592 : cluster [DBG] osdmap e1940: 8 total, 8 up, 8 in 2024-09-13T00:39:24.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:23 smithi181 bash[67587]: cluster 2024-09-13T00:39:22.714161+0000 mon.a (mon.0) 3592 : cluster [DBG] osdmap e1940: 8 total, 8 up, 8 in 2024-09-13T00:39:24.735 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to HEAD diff [978366~12404130,17113294~39372164,58515289~28499682,89250005~13234343,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~20663168,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240318840~11918637,252886187~622714,260842262~7844195,269960512~2117506,273064912~36449096,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,341594269~18955574,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:24.736 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~39372164,58515289~28499682,89250005~13234343,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~20663168,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240318840~11918637,252886187~622714,260842262~7844195,269960512~2117506,273064912~36449096,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,341594269~18955574,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:24.809 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,16777216~70237755,88080384~14403964,104857600~2026393,111245499~9980052,121634816~18591770,142230204~1250407,143604550~40657412,184549376~3692163,188743680~10830113,201556540~32834723,239075328~14433573,260046848~12031170,272629760~36884248,310378496~24935507,335934186~24615657,360710144~44231030,405696667~5790944] 2024-09-13T00:39:24.809 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to HEAD diff [978366~12404130,17113294~26808795,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:24.810 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~26808795,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:24.816 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,16777216~27144873,46526671~40488300,91032793~11451555,104857600~2026393,111245499~9980052,121634816~18591770,142230204~1250407,143604550~23038119,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~35982749,310378496~24935507,335934186~4465000,343284705~17265138,360710144~44231030,405696667~5790944] 2024-09-13T00:39:24.817 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to HEAD diff [978366~12404130,17113294~3323835,22666579~21255510,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:24.817 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~3323835,22666579~21255510,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~5052094,128451845~308073,128928644~8353161,142230204~1250407,144141427~2676660,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:24.821 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,17113294~3323835,22666579~21255510,46526671~40488300,91032793~11451555,104857600~2026393,111245499~9980052,121634816~5460584,128451845~308073,128928644~8353161,142230204~1250407,144141427~22501242,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~35982749,310378496~24935507,335934186~4465000,343284705~17265138,360710144~44231030,405696667~5790944] 2024-09-13T00:39:25.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:24 smithi136 bash[75164]: cluster 2024-09-13T00:39:23.716748+0000 mon.a (mon.0) 3593 : cluster [DBG] osdmap e1941: 8 total, 8 up, 8 in 2024-09-13T00:39:25.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:24 smithi136 bash[75164]: audit 2024-09-13T00:39:23.836352+0000 mon.a (mon.0) 3594 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:39:25.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:24 smithi136 bash[75164]: cluster 2024-09-13T00:39:24.431067+0000 mgr.y (mgr.24661) 1868 : cluster [DBG] pgmap v3370: 137 pgs: 2 active+clean+snaptrim, 3 active+clean+snaptrim_wait, 132 active+clean; 696 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 104 KiB/s rd, 6.2 KiB/s wr, 1.06k op/s 2024-09-13T00:39:25.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:24 smithi136 bash[75164]: cluster 2024-09-13T00:39:24.700903+0000 mon.a (mon.0) 3595 : cluster [DBG] osdmap e1942: 8 total, 8 up, 8 in 2024-09-13T00:39:25.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:24 smithi136 bash[76371]: cluster 2024-09-13T00:39:23.716748+0000 mon.a (mon.0) 3593 : cluster [DBG] osdmap e1941: 8 total, 8 up, 8 in 2024-09-13T00:39:25.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:24 smithi136 bash[76371]: audit 2024-09-13T00:39:23.836352+0000 mon.a (mon.0) 3594 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:39:25.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:24 smithi136 bash[76371]: cluster 2024-09-13T00:39:24.431067+0000 mgr.y (mgr.24661) 1868 : cluster [DBG] pgmap v3370: 137 pgs: 2 active+clean+snaptrim, 3 active+clean+snaptrim_wait, 132 active+clean; 696 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 104 KiB/s rd, 6.2 KiB/s wr, 1.06k op/s 2024-09-13T00:39:25.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:24 smithi136 bash[76371]: cluster 2024-09-13T00:39:24.700903+0000 mon.a (mon.0) 3595 : cluster [DBG] osdmap e1942: 8 total, 8 up, 8 in 2024-09-13T00:39:25.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:24 smithi181 bash[67587]: cluster 2024-09-13T00:39:23.716748+0000 mon.a (mon.0) 3593 : cluster [DBG] osdmap e1941: 8 total, 8 up, 8 in 2024-09-13T00:39:25.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:24 smithi181 bash[67587]: audit 2024-09-13T00:39:23.836352+0000 mon.a (mon.0) 3594 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:39:25.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:24 smithi181 bash[67587]: cluster 2024-09-13T00:39:24.431067+0000 mgr.y (mgr.24661) 1868 : cluster [DBG] pgmap v3370: 137 pgs: 2 active+clean+snaptrim, 3 active+clean+snaptrim_wait, 132 active+clean; 696 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 104 KiB/s rd, 6.2 KiB/s wr, 1.06k op/s 2024-09-13T00:39:25.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:24 smithi181 bash[67587]: cluster 2024-09-13T00:39:24.700903+0000 mon.a (mon.0) 3595 : cluster [DBG] osdmap e1942: 8 total, 8 up, 8 in 2024-09-13T00:39:25.814 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to HEAD diff [978366~12404130,17113294~39372164,58515289~28499682,89250005~13234343,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~20663168,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240318840~11918637,252886187~622714,260842262~7844195,269960512~2117506,273064912~36449096,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,341594269~18955574,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:25.814 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~39372164,58515289~28499682,89250005~13234343,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~20663168,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240318840~11918637,252886187~622714,260842262~7844195,269960512~2117506,273064912~36449096,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,341594269~18955574,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:25.868 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,16777216~70237755,88080384~14403964,104857600~2026393,111245499~9980052,121634816~18591770,142230204~1250407,143604550~40657412,184549376~3692163,188743680~10830113,201556540~32834723,239075328~14433573,260046848~12031170,272629760~36884248,310378496~24935507,335934186~24615657,360710144~44231030,405696667~5790944] 2024-09-13T00:39:25.868 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to HEAD diff [978366~12404130,17113294~26808795,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:25.868 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~26808795,46526671~9958787,58515289~28499682,91032793~11451555,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~19192428,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240828348~11409129,252886187~622714,260842262~7844195,273064912~35547597,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,343284705~17265138,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:25.874 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,16777216~27144873,46526671~40488300,91032793~11451555,104857600~2026393,111245499~9980052,121634816~18591770,142230204~1250407,143604550~23038119,169300267~14961695,184549376~3692163,188743680~10830113,201556540~32834723,240828348~12680553,260046848~8639609,272629760~35982749,310378496~24935507,335934186~4465000,343284705~17265138,360710144~44231030,405696667~5790944] 2024-09-13T00:39:26.809 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to HEAD diff [978366~12404130,17113294~39372164,58515289~28499682,89250005~13234343,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~20663168,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240318840~11918637,252886187~622714,260842262~7844195,269960512~2117506,273064912~36449096,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,341594269~18955574,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:26.809 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [978366~12404130,17113294~39372164,58515289~28499682,89250005~13234343,105459787~1424206,111245499~9980052,122043306~18183280,142230204~1250407,143604550~3213537,147450241~20663168,169300267~8711361,180155982~4105980,187369573~871966,192121569~7452224,201556540~17184323,220044632~14346631,240318840~11918637,252886187~622714,260842262~7844195,269960512~2117506,273064912~36449096,312566619~2948229,316301726~19012277,335934186~1814726,339257207~1141979,341594269~18955574,362400302~23949602,387920372~17020802,405696667~5790944] 2024-09-13T00:39:26.865 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~13382496,16777216~70237755,88080384~14403964,104857600~2026393,111245499~9980052,121634816~18591770,142230204~1250407,143604550~40657412,184549376~3692163,188743680~10830113,201556540~32834723,239075328~14433573,260046848~12031170,272629760~36884248,310378496~24935507,335934186~24615657,360710144~44231030,405696667~5790944] 2024-09-13T00:39:27.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:26 smithi181 bash[67587]: cluster 2024-09-13T00:39:25.764936+0000 mon.a (mon.0) 3596 : cluster [DBG] osdmap e1943: 8 total, 8 up, 8 in 2024-09-13T00:39:27.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:26 smithi181 bash[67587]: cluster 2024-09-13T00:39:26.431989+0000 mgr.y (mgr.24661) 1869 : cluster [DBG] pgmap v3373: 137 pgs: 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 134 active+clean; 608 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 69 KiB/s rd, 6.2 KiB/s wr, 741 op/s 2024-09-13T00:39:27.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:26 smithi136 bash[75164]: cluster 2024-09-13T00:39:25.764936+0000 mon.a (mon.0) 3596 : cluster [DBG] osdmap e1943: 8 total, 8 up, 8 in 2024-09-13T00:39:27.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:26 smithi136 bash[75164]: cluster 2024-09-13T00:39:26.431989+0000 mgr.y (mgr.24661) 1869 : cluster [DBG] pgmap v3373: 137 pgs: 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 134 active+clean; 608 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 69 KiB/s rd, 6.2 KiB/s wr, 741 op/s 2024-09-13T00:39:27.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:26 smithi136 bash[76371]: cluster 2024-09-13T00:39:25.764936+0000 mon.a (mon.0) 3596 : cluster [DBG] osdmap e1943: 8 total, 8 up, 8 in 2024-09-13T00:39:27.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:26 smithi136 bash[76371]: cluster 2024-09-13T00:39:26.431989+0000 mgr.y (mgr.24661) 1869 : cluster [DBG] pgmap v3373: 137 pgs: 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 134 active+clean; 608 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 69 KiB/s rd, 6.2 KiB/s wr, 741 op/s 2024-09-13T00:39:27.852 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] DiffIterateTest/0.DiffIterateStress (39948 ms) 2024-09-13T00:39:27.852 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] DiffIterateTest/0.DiffIterateRegression6926 2024-09-13T00:39:27.853 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:39:28.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:27 smithi181 bash[67587]: cluster 2024-09-13T00:39:26.771913+0000 mon.a (mon.0) 3597 : cluster [DBG] osdmap e1944: 8 total, 8 up, 8 in 2024-09-13T00:39:28.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:27 smithi136 bash[75164]: cluster 2024-09-13T00:39:26.771913+0000 mon.a (mon.0) 3597 : cluster [DBG] osdmap e1944: 8 total, 8 up, 8 in 2024-09-13T00:39:28.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:27 smithi136 bash[76371]: cluster 2024-09-13T00:39:26.771913+0000 mon.a (mon.0) 3597 : cluster [DBG] osdmap e1944: 8 total, 8 up, 8 in 2024-09-13T00:39:28.831 INFO:tasks.workunit.client.0.smithi136.stdout:iterate_cb 0~256 2024-09-13T00:39:28.849 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] DiffIterateTest/0.DiffIterateRegression6926 (996 ms) 2024-09-13T00:39:28.849 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] DiffIterateTest/0.DiffIterateParent 2024-09-13T00:39:28.849 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:39:29.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:28 smithi181 bash[67587]: cluster 2024-09-13T00:39:27.778991+0000 mon.a (mon.0) 3598 : cluster [DBG] osdmap e1945: 8 total, 8 up, 8 in 2024-09-13T00:39:29.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:28 smithi181 bash[67587]: cluster 2024-09-13T00:39:28.432775+0000 mgr.y (mgr.24661) 1870 : cluster [DBG] pgmap v3376: 137 pgs: 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 134 active+clean; 560 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 70 KiB/s rd, 6.2 KiB/s wr, 561 op/s 2024-09-13T00:39:29.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:28 smithi136 bash[75164]: cluster 2024-09-13T00:39:27.778991+0000 mon.a (mon.0) 3598 : cluster [DBG] osdmap e1945: 8 total, 8 up, 8 in 2024-09-13T00:39:29.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:28 smithi136 bash[75164]: cluster 2024-09-13T00:39:28.432775+0000 mgr.y (mgr.24661) 1870 : cluster [DBG] pgmap v3376: 137 pgs: 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 134 active+clean; 560 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 70 KiB/s rd, 6.2 KiB/s wr, 561 op/s 2024-09-13T00:39:29.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:28 smithi136 bash[76371]: cluster 2024-09-13T00:39:27.778991+0000 mon.a (mon.0) 3598 : cluster [DBG] osdmap e1945: 8 total, 8 up, 8 in 2024-09-13T00:39:29.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:28 smithi136 bash[76371]: cluster 2024-09-13T00:39:28.432775+0000 mgr.y (mgr.24661) 1870 : cluster [DBG] pgmap v3376: 137 pgs: 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 134 active+clean; 560 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 70 KiB/s rd, 6.2 KiB/s wr, 561 op/s 2024-09-13T00:39:29.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:39:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:39:29] "GET /metrics HTTP/1.1" 200 36756 "" "Prometheus/2.43.0" 2024-09-13T00:39:29.862 INFO:tasks.workunit.client.0.smithi136.stdout:iterate_cb 0~4194304 2024-09-13T00:39:29.862 INFO:tasks.workunit.client.0.smithi136.stdout:iterate_cb 4194304~4194304 2024-09-13T00:39:29.863 INFO:tasks.workunit.client.0.smithi136.stdout:iterate_cb 8388608~4194304 2024-09-13T00:39:29.863 INFO:tasks.workunit.client.0.smithi136.stdout:iterate_cb 12582912~4194304 2024-09-13T00:39:29.863 INFO:tasks.workunit.client.0.smithi136.stdout:iterate_cb 16777216~4194304 2024-09-13T00:39:29.937 INFO:tasks.workunit.client.0.smithi136.stdout:iterate_cb 0~4194304 2024-09-13T00:39:29.937 INFO:tasks.workunit.client.0.smithi136.stdout:iterate_cb 4194304~4194304 2024-09-13T00:39:29.937 INFO:tasks.workunit.client.0.smithi136.stdout:iterate_cb 8388608~2097152 2024-09-13T00:39:29.937 INFO:tasks.workunit.client.0.smithi136.stdout:iterate_cb 16777216~4194304 2024-09-13T00:39:29.965 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] DiffIterateTest/0.DiffIterateParent (1116 ms) 2024-09-13T00:39:29.965 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] DiffIterateTest/0.DiffIterateIgnoreParent 2024-09-13T00:39:29.965 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:39:30.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:29 smithi181 bash[67587]: cluster 2024-09-13T00:39:28.793639+0000 mon.a (mon.0) 3599 : cluster [DBG] osdmap e1946: 8 total, 8 up, 8 in 2024-09-13T00:39:30.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:29 smithi181 bash[67587]: cluster 2024-09-13T00:39:29.705626+0000 mon.a (mon.0) 3600 : cluster [DBG] osdmap e1947: 8 total, 8 up, 8 in 2024-09-13T00:39:30.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:29 smithi136 bash[75164]: cluster 2024-09-13T00:39:28.793639+0000 mon.a (mon.0) 3599 : cluster [DBG] osdmap e1946: 8 total, 8 up, 8 in 2024-09-13T00:39:30.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:29 smithi136 bash[75164]: cluster 2024-09-13T00:39:29.705626+0000 mon.a (mon.0) 3600 : cluster [DBG] osdmap e1947: 8 total, 8 up, 8 in 2024-09-13T00:39:30.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:29 smithi136 bash[76371]: cluster 2024-09-13T00:39:28.793639+0000 mon.a (mon.0) 3599 : cluster [DBG] osdmap e1946: 8 total, 8 up, 8 in 2024-09-13T00:39:30.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:29 smithi136 bash[76371]: cluster 2024-09-13T00:39:29.705626+0000 mon.a (mon.0) 3600 : cluster [DBG] osdmap e1947: 8 total, 8 up, 8 in 2024-09-13T00:39:31.094 INFO:tasks.workunit.client.0.smithi136.stdout: wrote [894443~38729,1207455~158,1936054~36067,7547238~64221,7856894~82395,9000571~5550,11195915~43706,12559360~34314,18726767~46575,19879126~41857] to clone 2024-09-13T00:39:31.095 INFO:tasks.workunit.client.0.smithi136.stdout: diff was [0~1972121,4194304~3744985,8388608~4205066,16777216~3143767] 2024-09-13T00:39:31.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:30 smithi181 bash[67587]: cluster 2024-09-13T00:39:30.433509+0000 mgr.y (mgr.24661) 1871 : cluster [DBG] pgmap v3379: 137 pgs: 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 134 active+clean; 511 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 1.7 KiB/s rd, 2.0 MiB/s wr, 155 op/s 2024-09-13T00:39:31.105 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] DiffIterateTest/0.DiffIterateIgnoreParent (1141 ms) 2024-09-13T00:39:31.105 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] DiffIterateTest/0.DiffIterateCallbackError 2024-09-13T00:39:31.105 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:39:31.157 INFO:tasks.workunit.client.0.smithi136.stdout: wrote [4305800~36907,6569006~68701,8332921~70370,14392468~49026,16199783~95727,16342497~27353,16672914~81748,16771258~74010,17242633~72463,18389037~10091] 2024-09-13T00:39:31.276 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] DiffIterateTest/0.DiffIterateCallbackError (75 ms) 2024-09-13T00:39:31.276 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] DiffIterateTest/0.DiffIterateParentDiscard 2024-09-13T00:39:31.276 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:39:31.276 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:30 smithi136 bash[75164]: cluster 2024-09-13T00:39:30.433509+0000 mgr.y (mgr.24661) 1871 : cluster [DBG] pgmap v3379: 137 pgs: 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 134 active+clean; 511 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 1.7 KiB/s rd, 2.0 MiB/s wr, 155 op/s 2024-09-13T00:39:31.277 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:30 smithi136 bash[76371]: cluster 2024-09-13T00:39:30.433509+0000 mgr.y (mgr.24661) 1871 : cluster [DBG] pgmap v3379: 137 pgs: 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 134 active+clean; 511 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 1.7 KiB/s rd, 2.0 MiB/s wr, 155 op/s 2024-09-13T00:39:32.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:31 smithi181 bash[67587]: cluster 2024-09-13T00:39:30.816047+0000 mon.a (mon.0) 3601 : cluster [DBG] osdmap e1948: 8 total, 8 up, 8 in 2024-09-13T00:39:32.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:31 smithi136 bash[75164]: cluster 2024-09-13T00:39:30.816047+0000 mon.a (mon.0) 3601 : cluster [DBG] osdmap e1948: 8 total, 8 up, 8 in 2024-09-13T00:39:32.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:31 smithi136 bash[76371]: cluster 2024-09-13T00:39:30.816047+0000 mon.a (mon.0) 3601 : cluster [DBG] osdmap e1948: 8 total, 8 up, 8 in 2024-09-13T00:39:32.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:39:32 smithi181 bash[95505]: ts=2024-09-13T00:39:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:39:33.074 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] DiffIterateTest/0.DiffIterateParentDiscard (1893 ms) 2024-09-13T00:39:33.074 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] DiffIterateTest/0.DiffIterateUnalignedSmall 2024-09-13T00:39:33.074 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:39:33.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:32 smithi181 bash[67587]: cluster 2024-09-13T00:39:31.829378+0000 mon.a (mon.0) 3602 : cluster [DBG] osdmap e1949: 8 total, 8 up, 8 in 2024-09-13T00:39:33.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:32 smithi181 bash[67587]: cluster 2024-09-13T00:39:32.434365+0000 mgr.y (mgr.24661) 1872 : cluster [DBG] pgmap v3382: 137 pgs: 137 active+clean; 502 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 4.1 MiB/s rd, 9.1 MiB/s wr, 265 op/s 2024-09-13T00:39:33.169 INFO:tasks.workunit.client.0.smithi136.stdout:iterate_cb 5000005~1234 2024-09-13T00:39:33.178 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] DiffIterateTest/0.DiffIterateUnalignedSmall (105 ms) 2024-09-13T00:39:33.178 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] DiffIterateTest/0.DiffIterateUnaligned 2024-09-13T00:39:33.179 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:39:33.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:32 smithi136 bash[75164]: cluster 2024-09-13T00:39:31.829378+0000 mon.a (mon.0) 3602 : cluster [DBG] osdmap e1949: 8 total, 8 up, 8 in 2024-09-13T00:39:33.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:32 smithi136 bash[75164]: cluster 2024-09-13T00:39:32.434365+0000 mgr.y (mgr.24661) 1872 : cluster [DBG] pgmap v3382: 137 pgs: 137 active+clean; 502 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 4.1 MiB/s rd, 9.1 MiB/s wr, 265 op/s 2024-09-13T00:39:33.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:32 smithi136 bash[76371]: cluster 2024-09-13T00:39:31.829378+0000 mon.a (mon.0) 3602 : cluster [DBG] osdmap e1949: 8 total, 8 up, 8 in 2024-09-13T00:39:33.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:32 smithi136 bash[76371]: cluster 2024-09-13T00:39:32.434365+0000 mgr.y (mgr.24661) 1872 : cluster [DBG] pgmap v3382: 137 pgs: 137 active+clean; 502 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 4.1 MiB/s rd, 9.1 MiB/s wr, 265 op/s 2024-09-13T00:39:33.327 INFO:tasks.workunit.client.0.smithi136.stdout:iterate_cb 8376263~12345 2024-09-13T00:39:33.327 INFO:tasks.workunit.client.0.smithi136.stdout:iterate_cb 8388608~4194304 2024-09-13T00:39:33.327 INFO:tasks.workunit.client.0.smithi136.stdout:iterate_cb 12582912~54321 2024-09-13T00:39:33.339 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] DiffIterateTest/0.DiffIterateUnaligned (161 ms) 2024-09-13T00:39:33.339 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] DiffIterateTest/0.DiffIterateStriping 2024-09-13T00:39:33.339 INFO:tasks.workunit.client.0.smithi136.stdout:SKIPPING 2024-09-13T00:39:33.340 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] DiffIterateTest/0.DiffIterateStriping (0 ms) 2024-09-13T00:39:33.857 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 11 tests from DiffIterateTest/0 (53537 ms total) 2024-09-13T00:39:33.857 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:39:33.857 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 11 tests from DiffIterateTest/1, where TypeParam = DiffIterateParams 2024-09-13T00:39:33.934 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] DiffIterateTest/1.DiffIterate 2024-09-13T00:39:34.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:33 smithi181 bash[67587]: cluster 2024-09-13T00:39:32.840069+0000 mon.a (mon.0) 3603 : cluster [DBG] osdmap e1950: 8 total, 8 up, 8 in 2024-09-13T00:39:34.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:33 smithi136 bash[75164]: cluster 2024-09-13T00:39:32.840069+0000 mon.a (mon.0) 3603 : cluster [DBG] osdmap e1950: 8 total, 8 up, 8 in 2024-09-13T00:39:34.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:33 smithi136 bash[76371]: cluster 2024-09-13T00:39:32.840069+0000 mon.a (mon.0) 3603 : cluster [DBG] osdmap e1950: 8 total, 8 up, 8 in 2024-09-13T00:39:35.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:34 smithi136 bash[75164]: cluster 2024-09-13T00:39:33.849557+0000 mon.a (mon.0) 3604 : cluster [DBG] osdmap e1951: 8 total, 8 up, 8 in 2024-09-13T00:39:35.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:34 smithi136 bash[75164]: cluster 2024-09-13T00:39:34.434970+0000 mgr.y (mgr.24661) 1873 : cluster [DBG] pgmap v3385: 105 pgs: 105 active+clean; 4.6 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2024-09-13T00:39:35.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:34 smithi136 bash[76371]: cluster 2024-09-13T00:39:33.849557+0000 mon.a (mon.0) 3604 : cluster [DBG] osdmap e1951: 8 total, 8 up, 8 in 2024-09-13T00:39:35.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:34 smithi136 bash[76371]: cluster 2024-09-13T00:39:34.434970+0000 mgr.y (mgr.24661) 1873 : cluster [DBG] pgmap v3385: 105 pgs: 105 active+clean; 4.6 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2024-09-13T00:39:35.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:34 smithi181 bash[67587]: cluster 2024-09-13T00:39:33.849557+0000 mon.a (mon.0) 3604 : cluster [DBG] osdmap e1951: 8 total, 8 up, 8 in 2024-09-13T00:39:35.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:34 smithi181 bash[67587]: cluster 2024-09-13T00:39:34.434970+0000 mgr.y (mgr.24661) 1873 : cluster [DBG] pgmap v3385: 105 pgs: 105 active+clean; 4.6 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2024-09-13T00:39:35.873 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:39:36.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:35 smithi136 bash[75164]: cluster 2024-09-13T00:39:34.860977+0000 mon.a (mon.0) 3605 : cluster [DBG] osdmap e1952: 8 total, 8 up, 8 in 2024-09-13T00:39:36.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:35 smithi136 bash[75164]: audit 2024-09-13T00:39:34.869405+0000 mon.c (mon.2) 144 : audit [INF] from='client.? 172.21.15.136:0/2397509528' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-36","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:39:36.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:35 smithi136 bash[75164]: audit 2024-09-13T00:39:34.871058+0000 mon.a (mon.0) 3606 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-36","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:39:36.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:35 smithi136 bash[76371]: cluster 2024-09-13T00:39:34.860977+0000 mon.a (mon.0) 3605 : cluster [DBG] osdmap e1952: 8 total, 8 up, 8 in 2024-09-13T00:39:36.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:35 smithi136 bash[76371]: audit 2024-09-13T00:39:34.869405+0000 mon.c (mon.2) 144 : audit [INF] from='client.? 172.21.15.136:0/2397509528' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-36","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:39:36.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:35 smithi136 bash[76371]: audit 2024-09-13T00:39:34.871058+0000 mon.a (mon.0) 3606 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-36","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:39:36.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:35 smithi181 bash[67587]: cluster 2024-09-13T00:39:34.860977+0000 mon.a (mon.0) 3605 : cluster [DBG] osdmap e1952: 8 total, 8 up, 8 in 2024-09-13T00:39:36.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:35 smithi181 bash[67587]: audit 2024-09-13T00:39:34.869405+0000 mon.c (mon.2) 144 : audit [INF] from='client.? 172.21.15.136:0/2397509528' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-36","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:39:36.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:35 smithi181 bash[67587]: audit 2024-09-13T00:39:34.871058+0000 mon.a (mon.0) 3606 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-36","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:39:37.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:36 smithi136 bash[75164]: audit 2024-09-13T00:39:35.862202+0000 mon.a (mon.0) 3607 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-36","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:39:37.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:36 smithi136 bash[75164]: cluster 2024-09-13T00:39:35.862617+0000 mon.a (mon.0) 3608 : cluster [DBG] osdmap e1953: 8 total, 8 up, 8 in 2024-09-13T00:39:37.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:36 smithi136 bash[75164]: cluster 2024-09-13T00:39:36.435797+0000 mgr.y (mgr.24661) 1874 : cluster [DBG] pgmap v3388: 137 pgs: 6 creating+peering, 22 unknown, 109 active+clean; 4.6 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2024-09-13T00:39:37.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:36 smithi136 bash[76371]: audit 2024-09-13T00:39:35.862202+0000 mon.a (mon.0) 3607 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-36","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:39:37.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:36 smithi136 bash[76371]: cluster 2024-09-13T00:39:35.862617+0000 mon.a (mon.0) 3608 : cluster [DBG] osdmap e1953: 8 total, 8 up, 8 in 2024-09-13T00:39:37.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:36 smithi136 bash[76371]: cluster 2024-09-13T00:39:36.435797+0000 mgr.y (mgr.24661) 1874 : cluster [DBG] pgmap v3388: 137 pgs: 6 creating+peering, 22 unknown, 109 active+clean; 4.6 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2024-09-13T00:39:37.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:36 smithi181 bash[67587]: audit 2024-09-13T00:39:35.862202+0000 mon.a (mon.0) 3607 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi136-114798-36","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:39:37.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:36 smithi181 bash[67587]: cluster 2024-09-13T00:39:35.862617+0000 mon.a (mon.0) 3608 : cluster [DBG] osdmap e1953: 8 total, 8 up, 8 in 2024-09-13T00:39:37.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:36 smithi181 bash[67587]: cluster 2024-09-13T00:39:36.435797+0000 mgr.y (mgr.24661) 1874 : cluster [DBG] pgmap v3388: 137 pgs: 6 creating+peering, 22 unknown, 109 active+clean; 4.6 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2024-09-13T00:39:37.972 INFO:tasks.workunit.client.0.smithi136.stdout: wrote [3072674~2865,5296988~19926,6168954~63784,6339509~12096,8302981~40933,12338529~61383,12594536~16217,15585896~91970,16413539~89569,18193444~37627] 2024-09-13T00:39:38.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:37 smithi136 bash[75164]: cluster 2024-09-13T00:39:36.887914+0000 mon.a (mon.0) 3609 : cluster [DBG] osdmap e1954: 8 total, 8 up, 8 in 2024-09-13T00:39:38.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:37 smithi136 bash[76371]: cluster 2024-09-13T00:39:36.887914+0000 mon.a (mon.0) 3609 : cluster [DBG] osdmap e1954: 8 total, 8 up, 8 in 2024-09-13T00:39:38.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:37 smithi181 bash[67587]: cluster 2024-09-13T00:39:36.887914+0000 mon.a (mon.0) 3609 : cluster [DBG] osdmap e1954: 8 total, 8 up, 8 in 2024-09-13T00:39:38.941 INFO:tasks.workunit.client.0.smithi136.stdout: wrote [0~4194304,8388608~12582912] 2024-09-13T00:39:38.962 INFO:tasks.workunit.client.0.smithi136.stdout: diff was [0~4194304,8388608~12582912] 2024-09-13T00:39:38.971 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] DiffIterateTest/1.DiffIterate (5037 ms) 2024-09-13T00:39:38.971 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] DiffIterateTest/1.DiffIterateDiscard 2024-09-13T00:39:38.971 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:39:39.026 INFO:tasks.workunit.client.0.smithi136.stdout:iterate_cb 0~4194304 2024-09-13T00:39:39.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:38 smithi136 bash[75164]: cluster 2024-09-13T00:39:37.897576+0000 mon.a (mon.0) 3610 : cluster [DBG] osdmap e1955: 8 total, 8 up, 8 in 2024-09-13T00:39:39.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:38 smithi136 bash[75164]: cluster 2024-09-13T00:39:38.436444+0000 mgr.y (mgr.24661) 1875 : cluster [DBG] pgmap v3391: 137 pgs: 6 creating+peering, 10 unknown, 121 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:39:39.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:38 smithi136 bash[75164]: audit 2024-09-13T00:39:38.836791+0000 mon.a (mon.0) 3611 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:39:39.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:38 smithi136 bash[76371]: cluster 2024-09-13T00:39:37.897576+0000 mon.a (mon.0) 3610 : cluster [DBG] osdmap e1955: 8 total, 8 up, 8 in 2024-09-13T00:39:39.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:38 smithi136 bash[76371]: cluster 2024-09-13T00:39:38.436444+0000 mgr.y (mgr.24661) 1875 : cluster [DBG] pgmap v3391: 137 pgs: 6 creating+peering, 10 unknown, 121 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:39:39.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:38 smithi136 bash[76371]: audit 2024-09-13T00:39:38.836791+0000 mon.a (mon.0) 3611 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:39:39.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:38 smithi181 bash[67587]: cluster 2024-09-13T00:39:37.897576+0000 mon.a (mon.0) 3610 : cluster [DBG] osdmap e1955: 8 total, 8 up, 8 in 2024-09-13T00:39:39.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:38 smithi181 bash[67587]: cluster 2024-09-13T00:39:38.436444+0000 mgr.y (mgr.24661) 1875 : cluster [DBG] pgmap v3391: 137 pgs: 6 creating+peering, 10 unknown, 121 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:39:39.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:38 smithi181 bash[67587]: audit 2024-09-13T00:39:38.836791+0000 mon.a (mon.0) 3611 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:39:39.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:39:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:39:39] "GET /metrics HTTP/1.1" 200 36691 "" "Prometheus/2.43.0" 2024-09-13T00:39:39.957 INFO:tasks.workunit.client.0.smithi136.stdout:iterate_cb 0~4194304 2024-09-13T00:39:40.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:39 smithi136 bash[75164]: cluster 2024-09-13T00:39:38.913767+0000 mon.a (mon.0) 3612 : cluster [DBG] osdmap e1956: 8 total, 8 up, 8 in 2024-09-13T00:39:40.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:39 smithi136 bash[76371]: cluster 2024-09-13T00:39:38.913767+0000 mon.a (mon.0) 3612 : cluster [DBG] osdmap e1956: 8 total, 8 up, 8 in 2024-09-13T00:39:40.352 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:39 smithi181 bash[67587]: cluster 2024-09-13T00:39:38.913767+0000 mon.a (mon.0) 3612 : cluster [DBG] osdmap e1956: 8 total, 8 up, 8 in 2024-09-13T00:39:40.971 INFO:tasks.workunit.client.0.smithi136.stdout:iterate_cb 0~4194304 2024-09-13T00:39:41.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:40 smithi136 bash[76371]: cluster 2024-09-13T00:39:39.918972+0000 mon.a (mon.0) 3613 : cluster [DBG] osdmap e1957: 8 total, 8 up, 8 in 2024-09-13T00:39:41.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:40 smithi136 bash[76371]: cluster 2024-09-13T00:39:40.437240+0000 mgr.y (mgr.24661) 1876 : cluster [DBG] pgmap v3394: 137 pgs: 6 creating+peering, 131 active+clean; 16 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 177 KiB/s wr, 3 op/s 2024-09-13T00:39:41.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:40 smithi136 bash[75164]: cluster 2024-09-13T00:39:39.918972+0000 mon.a (mon.0) 3613 : cluster [DBG] osdmap e1957: 8 total, 8 up, 8 in 2024-09-13T00:39:41.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:40 smithi136 bash[75164]: cluster 2024-09-13T00:39:40.437240+0000 mgr.y (mgr.24661) 1876 : cluster [DBG] pgmap v3394: 137 pgs: 6 creating+peering, 131 active+clean; 16 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 177 KiB/s wr, 3 op/s 2024-09-13T00:39:41.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:40 smithi181 bash[67587]: cluster 2024-09-13T00:39:39.918972+0000 mon.a (mon.0) 3613 : cluster [DBG] osdmap e1957: 8 total, 8 up, 8 in 2024-09-13T00:39:41.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:40 smithi181 bash[67587]: cluster 2024-09-13T00:39:40.437240+0000 mgr.y (mgr.24661) 1876 : cluster [DBG] pgmap v3394: 137 pgs: 6 creating+peering, 131 active+clean; 16 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 177 KiB/s wr, 3 op/s 2024-09-13T00:39:41.955 INFO:tasks.workunit.client.0.smithi136.stdout:iterate_cb 0~4194304 2024-09-13T00:39:41.956 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] DiffIterateTest/1.DiffIterateDiscard (2986 ms) 2024-09-13T00:39:41.956 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] DiffIterateTest/1.DiffIterateStress 2024-09-13T00:39:41.956 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:39:42.021 INFO:tasks.workunit.client.0.smithi136.stdout: i=0 exists [4560292~5391508,19722387~2891244,34486600~1452745,60928352~2701375,73403820~3661240,90557685~1665327,162621364~5262343,236916784~4439194,290690539~1825214,318974959~3880458] wrote [4560292~5391508,19722387~2891244,34486600~1452745,60928352~2701375,73403820~3661240,90557685~1665327,162621364~5262343,236916784~4439194,290690539~1825214,318974959~3880458] 2024-09-13T00:39:42.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:41 smithi136 bash[76371]: cluster 2024-09-13T00:39:40.927912+0000 mon.a (mon.0) 3614 : cluster [DBG] osdmap e1958: 8 total, 8 up, 8 in 2024-09-13T00:39:42.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:41 smithi136 bash[76371]: audit 2024-09-13T00:39:41.584955+0000 mon.a (mon.0) 3615 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:39:42.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:41 smithi136 bash[75164]: cluster 2024-09-13T00:39:40.927912+0000 mon.a (mon.0) 3614 : cluster [DBG] osdmap e1958: 8 total, 8 up, 8 in 2024-09-13T00:39:42.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:41 smithi136 bash[75164]: audit 2024-09-13T00:39:41.584955+0000 mon.a (mon.0) 3615 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:39:42.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:41 smithi181 bash[67587]: cluster 2024-09-13T00:39:40.927912+0000 mon.a (mon.0) 3614 : cluster [DBG] osdmap e1958: 8 total, 8 up, 8 in 2024-09-13T00:39:42.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:41 smithi181 bash[67587]: audit 2024-09-13T00:39:41.584955+0000 mon.a (mon.0) 3615 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:39:42.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:39:42 smithi181 bash[95505]: ts=2024-09-13T00:39:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:39:43.049 INFO:tasks.workunit.client.0.smithi136.stdout: i=1 exists [4560292~5391508,19722387~2891244,26328864~816092,34486600~1452745,57809452~6942239,73403820~3661240,89862541~2360471,102357207~1790956,125900802~4428394,152569586~81016,162621364~5262343,236916784~4439194,288100454~6839410,314708100~8147317] wrote [26328864~816092,57809452~6942239,89862541~1763596,102357207~1790956,125900802~4428394,152569586~81016,288100454~6839410,314708100~5186089] 2024-09-13T00:39:43.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:42 smithi136 bash[75164]: cluster 2024-09-13T00:39:41.927632+0000 mon.a (mon.0) 3616 : cluster [DBG] osdmap e1959: 8 total, 8 up, 8 in 2024-09-13T00:39:43.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:42 smithi136 bash[75164]: audit 2024-09-13T00:39:42.139018+0000 mon.a (mon.0) 3617 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:39:43.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:42 smithi136 bash[75164]: audit 2024-09-13T00:39:42.140611+0000 mon.a (mon.0) 3618 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:39:43.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:42 smithi136 bash[75164]: audit 2024-09-13T00:39:42.147953+0000 mon.a (mon.0) 3619 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:39:43.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:42 smithi136 bash[75164]: cluster 2024-09-13T00:39:42.437965+0000 mgr.y (mgr.24661) 1877 : cluster [DBG] pgmap v3397: 137 pgs: 137 active+clean; 19 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 56 KiB/s rd, 210 KiB/s wr, 90 op/s 2024-09-13T00:39:43.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:42 smithi136 bash[76371]: cluster 2024-09-13T00:39:41.927632+0000 mon.a (mon.0) 3616 : cluster [DBG] osdmap e1959: 8 total, 8 up, 8 in 2024-09-13T00:39:43.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:42 smithi136 bash[76371]: audit 2024-09-13T00:39:42.139018+0000 mon.a (mon.0) 3617 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:39:43.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:42 smithi136 bash[76371]: audit 2024-09-13T00:39:42.140611+0000 mon.a (mon.0) 3618 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:39:43.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:42 smithi136 bash[76371]: audit 2024-09-13T00:39:42.147953+0000 mon.a (mon.0) 3619 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:39:43.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:42 smithi136 bash[76371]: cluster 2024-09-13T00:39:42.437965+0000 mgr.y (mgr.24661) 1877 : cluster [DBG] pgmap v3397: 137 pgs: 137 active+clean; 19 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 56 KiB/s rd, 210 KiB/s wr, 90 op/s 2024-09-13T00:39:43.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:42 smithi181 bash[67587]: cluster 2024-09-13T00:39:41.927632+0000 mon.a (mon.0) 3616 : cluster [DBG] osdmap e1959: 8 total, 8 up, 8 in 2024-09-13T00:39:43.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:42 smithi181 bash[67587]: audit 2024-09-13T00:39:42.139018+0000 mon.a (mon.0) 3617 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:39:43.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:42 smithi181 bash[67587]: audit 2024-09-13T00:39:42.140611+0000 mon.a (mon.0) 3618 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:39:43.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:42 smithi181 bash[67587]: audit 2024-09-13T00:39:42.147953+0000 mon.a (mon.0) 3619 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:39:43.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:42 smithi181 bash[67587]: cluster 2024-09-13T00:39:42.437965+0000 mgr.y (mgr.24661) 1877 : cluster [DBG] pgmap v3397: 137 pgs: 137 active+clean; 19 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 56 KiB/s rd, 210 KiB/s wr, 90 op/s 2024-09-13T00:39:44.078 INFO:tasks.workunit.client.0.smithi136.stdout: i=2 exists [4560292~5391508,17018890~5594741,26328864~816092,34486600~1452745,45428232~4919038,51373144~13378547,73403820~3661240,89862541~2360471,102357207~3481149,125900802~4428394,152569586~81016,162621364~5262343,169549666~5434611,184529373~1069310,202501730~5826341,219229250~6123315,236916784~4439194,288100454~6839410,314708100~8147317,364591348~3939750] wrote [17018890~5029211,45428232~4919038,51373144~7767833,104093515~1744841,169549666~5434611,184529373~1069310,202501730~5826341,219229250~6123315,364591348~3939750] 2024-09-13T00:39:44.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:43 smithi136 bash[75164]: cluster 2024-09-13T00:39:42.942211+0000 mon.a (mon.0) 3620 : cluster [DBG] osdmap e1960: 8 total, 8 up, 8 in 2024-09-13T00:39:44.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:43 smithi136 bash[76371]: cluster 2024-09-13T00:39:42.942211+0000 mon.a (mon.0) 3620 : cluster [DBG] osdmap e1960: 8 total, 8 up, 8 in 2024-09-13T00:39:44.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:43 smithi181 bash[67587]: cluster 2024-09-13T00:39:42.942211+0000 mon.a (mon.0) 3620 : cluster [DBG] osdmap e1960: 8 total, 8 up, 8 in 2024-09-13T00:39:45.094 INFO:tasks.workunit.client.0.smithi136.stdout: i=3 exists [4560292~5391508,17018890~5594741,25151502~1993454,34486600~1452745,45428232~4919038,51373144~18717800,73403820~3661240,89862541~2360471,102357207~3481149,125900802~4428394,146607183~2524721,152569586~81016,161732654~6151053,169549666~5434611,184529373~1069310,202501730~5826341,218401583~6950982,236916784~4439194,255893971~7076069,275808252~10949571,288100454~6839410,314708100~8147317,364591348~3939750,383783846~3273310] wrote [25151502~1835533,62292416~7798528,146607183~2524721,161732654~1825270,218401583~6315933,255893971~7076069,275808252~10949571,383783846~3273310] 2024-09-13T00:39:45.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:44 smithi136 bash[75164]: cluster 2024-09-13T00:39:43.963468+0000 mon.a (mon.0) 3621 : cluster [DBG] osdmap e1961: 8 total, 8 up, 8 in 2024-09-13T00:39:45.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:44 smithi136 bash[75164]: cluster 2024-09-13T00:39:44.438676+0000 mgr.y (mgr.24661) 1878 : cluster [DBG] pgmap v3400: 137 pgs: 137 active+clean; 40 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 56 KiB/s rd, 2.4 MiB/s wr, 89 op/s 2024-09-13T00:39:45.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:44 smithi136 bash[76371]: cluster 2024-09-13T00:39:43.963468+0000 mon.a (mon.0) 3621 : cluster [DBG] osdmap e1961: 8 total, 8 up, 8 in 2024-09-13T00:39:45.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:44 smithi136 bash[76371]: cluster 2024-09-13T00:39:44.438676+0000 mgr.y (mgr.24661) 1878 : cluster [DBG] pgmap v3400: 137 pgs: 137 active+clean; 40 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 56 KiB/s rd, 2.4 MiB/s wr, 89 op/s 2024-09-13T00:39:45.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:44 smithi181 bash[67587]: cluster 2024-09-13T00:39:43.963468+0000 mon.a (mon.0) 3621 : cluster [DBG] osdmap e1961: 8 total, 8 up, 8 in 2024-09-13T00:39:45.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:44 smithi181 bash[67587]: cluster 2024-09-13T00:39:44.438676+0000 mgr.y (mgr.24661) 1878 : cluster [DBG] pgmap v3400: 137 pgs: 137 active+clean; 40 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 56 KiB/s rd, 2.4 MiB/s wr, 89 op/s 2024-09-13T00:39:46.074 INFO:tasks.workunit.client.0.smithi136.stdout: i=4 exists [4560292~5936017,17018890~5594741,25151502~1993454,34486600~1452745,45428232~4919038,51373144~18717800,73403820~3661240,86462653~5760359,102357207~3481149,125900802~4428394,146607183~2524721,152569586~81016,161732654~6151053,169549666~5434611,184529373~1069310,202501730~5826341,218401583~6950982,236916784~4439194,255893971~7076069,275808252~10949571,288100454~6839410,299055426~9280621,314708100~9101289,364591348~3939750,379265209~1757828,383783846~3273310] wrote [4796489~5699820,19199008~3326602,86462653~5545120,221575630~1503523,299055426~9280621,316577351~3101669,321432696~2376693,379265209~1757828] 2024-09-13T00:39:46.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:45 smithi136 bash[75164]: cluster 2024-09-13T00:39:44.977149+0000 mon.a (mon.0) 3622 : cluster [DBG] osdmap e1962: 8 total, 8 up, 8 in 2024-09-13T00:39:46.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:45 smithi136 bash[76371]: cluster 2024-09-13T00:39:44.977149+0000 mon.a (mon.0) 3622 : cluster [DBG] osdmap e1962: 8 total, 8 up, 8 in 2024-09-13T00:39:46.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:45 smithi181 bash[67587]: cluster 2024-09-13T00:39:44.977149+0000 mon.a (mon.0) 3622 : cluster [DBG] osdmap e1962: 8 total, 8 up, 8 in 2024-09-13T00:39:47.130 INFO:tasks.workunit.client.0.smithi136.stdout: i=5 exists [4560292~5936017,17018890~5594741,25151502~1993454,30436866~3685803,34486600~1452745,45428232~4919038,51373144~20150674,73403820~3661240,85233139~6989873,102357207~3481149,125900802~4428394,146607183~2524721,152569586~81016,161732654~6151053,169549666~5434611,184529373~1069310,202501730~5826341,218401583~6950982,236916784~4439194,255893971~7076069,275808252~10949571,288100454~20235593,314708100~9101289,357229957~5050843,364591348~3939750,379265209~1757828,381044224~7521482] wrote [30436866~3685803,57304505~811949,67026347~4497471,85233139~6240299,258009925~2978957,293245930~14374763,357229957~5050843,381044224~7521482] 2024-09-13T00:39:47.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:46 smithi136 bash[75164]: cluster 2024-09-13T00:39:45.981380+0000 mon.a (mon.0) 3623 : cluster [DBG] osdmap e1963: 8 total, 8 up, 8 in 2024-09-13T00:39:47.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:46 smithi136 bash[75164]: cluster 2024-09-13T00:39:46.439502+0000 mgr.y (mgr.24661) 1879 : cluster [DBG] pgmap v3403: 137 pgs: 137 active+clean; 119 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 5.2 KiB/s rd, 19 MiB/s wr, 21 op/s 2024-09-13T00:39:47.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:46 smithi136 bash[76371]: cluster 2024-09-13T00:39:45.981380+0000 mon.a (mon.0) 3623 : cluster [DBG] osdmap e1963: 8 total, 8 up, 8 in 2024-09-13T00:39:47.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:46 smithi136 bash[76371]: cluster 2024-09-13T00:39:46.439502+0000 mgr.y (mgr.24661) 1879 : cluster [DBG] pgmap v3403: 137 pgs: 137 active+clean; 119 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 5.2 KiB/s rd, 19 MiB/s wr, 21 op/s 2024-09-13T00:39:47.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:46 smithi181 bash[67587]: cluster 2024-09-13T00:39:45.981380+0000 mon.a (mon.0) 3623 : cluster [DBG] osdmap e1963: 8 total, 8 up, 8 in 2024-09-13T00:39:47.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:46 smithi181 bash[67587]: cluster 2024-09-13T00:39:46.439502+0000 mgr.y (mgr.24661) 1879 : cluster [DBG] pgmap v3403: 137 pgs: 137 active+clean; 119 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 5.2 KiB/s rd, 19 MiB/s wr, 21 op/s 2024-09-13T00:39:48.117 INFO:tasks.workunit.client.0.smithi136.stdout: i=6 exists [4560292~5936017,17018890~5594741,25151502~1993454,30436866~3685803,34486600~1452745,42963561~28560257,73403820~3661240,85233139~6989873,96803318~2541368,102357207~3481149,125900802~4428394,146607183~2524721,152569586~81016,157648610~10235097,169549666~5434611,184529373~1069310,202501730~5826341,218401583~6950982,236916784~11883992,255893971~7076069,275808252~10949571,288100454~20235593,314708100~9101289,357229957~5050843,363576621~7957175,379265209~1757828,381044224~7521482] wrote [42963561~8471555,96803318~2541368,157648610~7336492,240729397~8071379,259484867~1032734,278779605~3926451,314924605~6468295,363576621~7957175] 2024-09-13T00:39:48.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:48 smithi181 bash[67587]: cluster 2024-09-13T00:39:47.002811+0000 mon.a (mon.0) 3624 : cluster [DBG] osdmap e1964: 8 total, 8 up, 8 in 2024-09-13T00:39:48.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:48 smithi136 bash[75164]: cluster 2024-09-13T00:39:47.002811+0000 mon.a (mon.0) 3624 : cluster [DBG] osdmap e1964: 8 total, 8 up, 8 in 2024-09-13T00:39:48.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:48 smithi136 bash[76371]: cluster 2024-09-13T00:39:47.002811+0000 mon.a (mon.0) 3624 : cluster [DBG] osdmap e1964: 8 total, 8 up, 8 in 2024-09-13T00:39:49.134 INFO:tasks.workunit.client.0.smithi136.stdout: i=7 exists [349554~3879452,4560292~5936017,17018890~5594741,25151502~1993454,30436866~3685803,34486600~1452745,42963561~28560257,73403820~3661240,85233139~6989873,96803318~2541368,102357207~3481149,125900802~4428394,132586105~717255,146607183~2524721,152569586~81016,152938762~3662694,157648610~10235097,169549666~5434611,184529373~1069310,202501730~5826341,218401583~6950982,236916784~11883992,255893971~7076069,275808252~10949571,288100454~20235593,314708100~9101289,341716408~2925347,357229957~5050843,363576621~7957175,379265209~1757828,381044224~15753763] wrote [349554~3879452,58632858~8356522,132586105~717255,152938762~3662694,160232382~294968,161088860~6187763,341716408~2925347,384926271~11871716] 2024-09-13T00:39:49.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:49 smithi181 bash[67587]: cluster 2024-09-13T00:39:48.006349+0000 mon.a (mon.0) 3625 : cluster [DBG] osdmap e1965: 8 total, 8 up, 8 in 2024-09-13T00:39:49.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:49 smithi181 bash[67587]: cluster 2024-09-13T00:39:48.440223+0000 mgr.y (mgr.24661) 1880 : cluster [DBG] pgmap v3406: 137 pgs: 137 active+clean; 224 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 163 KiB/s rd, 39 MiB/s wr, 268 op/s 2024-09-13T00:39:49.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:49 smithi136 bash[75164]: cluster 2024-09-13T00:39:48.006349+0000 mon.a (mon.0) 3625 : cluster [DBG] osdmap e1965: 8 total, 8 up, 8 in 2024-09-13T00:39:49.414 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:49 smithi136 bash[75164]: cluster 2024-09-13T00:39:48.440223+0000 mgr.y (mgr.24661) 1880 : cluster [DBG] pgmap v3406: 137 pgs: 137 active+clean; 224 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 163 KiB/s rd, 39 MiB/s wr, 268 op/s 2024-09-13T00:39:49.414 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:49 smithi136 bash[76371]: cluster 2024-09-13T00:39:48.006349+0000 mon.a (mon.0) 3625 : cluster [DBG] osdmap e1965: 8 total, 8 up, 8 in 2024-09-13T00:39:49.414 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:49 smithi136 bash[76371]: cluster 2024-09-13T00:39:48.440223+0000 mgr.y (mgr.24661) 1880 : cluster [DBG] pgmap v3406: 137 pgs: 137 active+clean; 224 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 163 KiB/s rd, 39 MiB/s wr, 268 op/s 2024-09-13T00:39:49.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:39:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:39:49] "GET /metrics HTTP/1.1" 200 36691 "" "Prometheus/2.43.0" 2024-09-13T00:39:49.835 INFO:tasks.workunit.client.0.smithi136.stdout: i=8 exists [349554~3879452,4560292~5936017,17018890~5594741,25151502~1993454,30436866~3685803,34486600~1452745,42963561~28560257,73403820~3661240,85233139~8112806,96803318~2541368,102357207~3481149,111990084~1888886,125900802~6251198,132586105~717255,139547811~3881147,146607183~9994273,157648610~10235097,169549666~5434611,184529373~1069310,202501730~5826341,218401583~6950982,236916784~11883992,255893971~8978287,275808252~32527795,314708100~9101289,341716408~2925347,357229957~5994276,363576621~7957175,379265209~1757828,381044224~15753763] wrote [8117709~1155886,62889550~635620,89501542~3844403,111990084~1888886,128487094~3664906,139547811~3881147,148874507~5867035,262629832~2242426,284420990~6366518,359672166~3552067] 2024-09-13T00:39:50.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:50 smithi181 bash[67587]: cluster 2024-09-13T00:39:49.033261+0000 mon.a (mon.0) 3626 : cluster [DBG] osdmap e1966: 8 total, 8 up, 8 in 2024-09-13T00:39:50.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:50 smithi181 bash[67587]: cluster 2024-09-13T00:39:49.737264+0000 mon.a (mon.0) 3627 : cluster [DBG] osdmap e1967: 8 total, 8 up, 8 in 2024-09-13T00:39:50.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:50 smithi136 bash[75164]: cluster 2024-09-13T00:39:49.033261+0000 mon.a (mon.0) 3626 : cluster [DBG] osdmap e1966: 8 total, 8 up, 8 in 2024-09-13T00:39:50.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:50 smithi136 bash[75164]: cluster 2024-09-13T00:39:49.737264+0000 mon.a (mon.0) 3627 : cluster [DBG] osdmap e1967: 8 total, 8 up, 8 in 2024-09-13T00:39:50.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:50 smithi136 bash[76371]: cluster 2024-09-13T00:39:49.033261+0000 mon.a (mon.0) 3626 : cluster [DBG] osdmap e1966: 8 total, 8 up, 8 in 2024-09-13T00:39:50.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:50 smithi136 bash[76371]: cluster 2024-09-13T00:39:49.737264+0000 mon.a (mon.0) 3627 : cluster [DBG] osdmap e1967: 8 total, 8 up, 8 in 2024-09-13T00:39:50.839 INFO:tasks.workunit.client.0.smithi136.stdout: i=9 exists [349554~10146755,17018890~6070249,25151502~1993454,30436866~3685803,34486600~1452745,38263469~4355551,42963561~28560257,73403820~3661240,77423716~5467230,85233139~8112806,94049957~5294729,99633293~6718151,111990084~1888886,125900802~6251198,132586105~717255,139547811~3881147,146607183~9994273,157648610~10235097,169549666~5434611,184529373~1069310,202501730~5826341,218401583~6950982,236916784~11883992,255893971~8978287,270216735~405595,275808252~32527795,314708100~9101289,341716408~2925347,357229957~14303839,379265209~1757828,381044224~15753763] wrote [1206012~6800177,19491092~3598047,38263469~4355551,77423716~5467230,94049957~3424142,99633293~6718151,259822965~4047173,270216735~405595,291769361~4774586,361706919~4084885] 2024-09-13T00:39:51.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:51 smithi181 bash[67587]: cluster 2024-09-13T00:39:50.440953+0000 mgr.y (mgr.24661) 1881 : cluster [DBG] pgmap v3409: 137 pgs: 137 active+clean; 290 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 157 KiB/s rd, 38 MiB/s wr, 258 op/s 2024-09-13T00:39:51.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:51 smithi181 bash[67587]: cluster 2024-09-13T00:39:50.739701+0000 mon.a (mon.0) 3628 : cluster [DBG] osdmap e1968: 8 total, 8 up, 8 in 2024-09-13T00:39:51.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:51 smithi136 bash[75164]: cluster 2024-09-13T00:39:50.440953+0000 mgr.y (mgr.24661) 1881 : cluster [DBG] pgmap v3409: 137 pgs: 137 active+clean; 290 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 157 KiB/s rd, 38 MiB/s wr, 258 op/s 2024-09-13T00:39:51.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:51 smithi136 bash[75164]: cluster 2024-09-13T00:39:50.739701+0000 mon.a (mon.0) 3628 : cluster [DBG] osdmap e1968: 8 total, 8 up, 8 in 2024-09-13T00:39:51.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:51 smithi136 bash[76371]: cluster 2024-09-13T00:39:50.440953+0000 mgr.y (mgr.24661) 1881 : cluster [DBG] pgmap v3409: 137 pgs: 137 active+clean; 290 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 157 KiB/s rd, 38 MiB/s wr, 258 op/s 2024-09-13T00:39:51.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:51 smithi136 bash[76371]: cluster 2024-09-13T00:39:50.739701+0000 mon.a (mon.0) 3628 : cluster [DBG] osdmap e1968: 8 total, 8 up, 8 in 2024-09-13T00:39:51.844 INFO:tasks.workunit.client.0.smithi136.stdout: i=10 exists [349554~10146755,17018890~6070249,25151502~1993454,30436866~3685803,34486600~1452745,38263469~4355551,42963561~28560257,73403820~3661240,77423716~5467230,85233139~8112806,94049957~5294729,99633293~6718151,108597807~1730921,111990084~1888886,125900802~6251198,132586105~717255,139547811~3881147,146607183~10816027,157648610~10235097,169549666~5434611,184529373~1069310,202501730~5826341,213480181~11872384,236916784~11883992,255893971~8978287,270216735~405595,275085725~33250322,314708100~9101289,336532120~1851319,341716408~2925347,357229957~14303839,379265209~1757828,381044224~15753763] wrote [108597807~1730921,151349022~6074188,173958345~343045,203755127~3246497,213480181~7879188,256003324~4232516,275085725~4043241,288186851~5226910,336532120~1851319] 2024-09-13T00:39:52.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:39:52 smithi181 bash[95505]: ts=2024-09-13T00:39:52.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:39:52.852 INFO:tasks.workunit.client.0.smithi136.stdout: i=11 exists [349554~10146755,15629417~7459722,25151502~1993454,30436866~3685803,34486600~1452745,38263469~4355551,42963561~28560257,73403820~3661240,77423716~5467230,85233139~8112806,94049957~5294729,99633293~10695435,111990084~1888886,125900802~6251198,132586105~717255,139547811~17875399,157648610~10235097,169549666~5434611,184529373~1069310,190394373~7279350,202501730~5826341,213480181~11872384,228850744~2814978,236916784~11883992,254009698~1204295,255893971~8978287,267976284~801439,270216735~405595,275085725~33250322,314708100~9101289,336532120~1851319,341716408~2925347,357229957~14303839,379265209~1757828,381044224~15753763] wrote [15629417~6573492,104736873~4880382,140287308~6391938,190394373~7279350,228850744~2814978,237508616~1394875,254009698~1204295,258646008~1859848,267976284~801439,319414177~2200975] 2024-09-13T00:39:53.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:52 smithi136 bash[75164]: cluster 2024-09-13T00:39:51.743637+0000 mon.a (mon.0) 3629 : cluster [DBG] osdmap e1969: 8 total, 8 up, 8 in 2024-09-13T00:39:53.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:52 smithi136 bash[75164]: cluster 2024-09-13T00:39:52.441830+0000 mgr.y (mgr.24661) 1882 : cluster [DBG] pgmap v3412: 137 pgs: 137 active+clean; 421 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 192 KiB/s rd, 48 MiB/s wr, 314 op/s 2024-09-13T00:39:53.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:52 smithi136 bash[76371]: cluster 2024-09-13T00:39:51.743637+0000 mon.a (mon.0) 3629 : cluster [DBG] osdmap e1969: 8 total, 8 up, 8 in 2024-09-13T00:39:53.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:52 smithi136 bash[76371]: cluster 2024-09-13T00:39:52.441830+0000 mgr.y (mgr.24661) 1882 : cluster [DBG] pgmap v3412: 137 pgs: 137 active+clean; 421 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 192 KiB/s rd, 48 MiB/s wr, 314 op/s 2024-09-13T00:39:53.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:52 smithi181 bash[67587]: cluster 2024-09-13T00:39:51.743637+0000 mon.a (mon.0) 3629 : cluster [DBG] osdmap e1969: 8 total, 8 up, 8 in 2024-09-13T00:39:53.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:52 smithi181 bash[67587]: cluster 2024-09-13T00:39:52.441830+0000 mgr.y (mgr.24661) 1882 : cluster [DBG] pgmap v3412: 137 pgs: 137 active+clean; 421 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 192 KiB/s rd, 48 MiB/s wr, 314 op/s 2024-09-13T00:39:53.865 INFO:tasks.workunit.client.0.smithi136.stdout: i=12 exists [349554~10146755,15629417~7459722,25151502~1993454,30436866~3685803,34486600~1452745,38263469~4355551,42963561~34101499,77423716~5467230,85233139~8112806,94049957~5294729,99633293~10695435,111990084~1888886,119463127~3632382,125900802~6251198,132586105~717255,139547811~17875399,157648610~10235097,169549666~5434611,184529373~1069310,185834642~3478716,190394373~7279350,202501730~5826341,213480181~11872384,228850744~2814978,236916784~11883992,254009698~1204295,255893971~9305901,267976284~801439,270216735~405595,275085725~33250322,314708100~9101289,334290642~4092797,341716408~2925347,355197550~1435514,357229957~14303839,372438365~5654930,379265209~17532778] wrote [70309861~3274462,94935831~4210304,119463127~3632382,185834642~3478716,257246349~7953523,315987651~2572893,334290642~3438187,355197550~1435514,372438365~5654930,380485412~1201240] 2024-09-13T00:39:54.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:53 smithi136 bash[75164]: cluster 2024-09-13T00:39:52.756316+0000 mon.a (mon.0) 3630 : cluster [DBG] osdmap e1970: 8 total, 8 up, 8 in 2024-09-13T00:39:54.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:53 smithi136 bash[76371]: cluster 2024-09-13T00:39:52.756316+0000 mon.a (mon.0) 3630 : cluster [DBG] osdmap e1970: 8 total, 8 up, 8 in 2024-09-13T00:39:54.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:53 smithi181 bash[67587]: cluster 2024-09-13T00:39:52.756316+0000 mon.a (mon.0) 3630 : cluster [DBG] osdmap e1970: 8 total, 8 up, 8 in 2024-09-13T00:39:54.825 INFO:tasks.workunit.client.0.smithi136.stdout: i=13 exists [349554~10146755,15629417~8020558,25151502~1993454,30436866~3685803,34486600~1452745,36284882~6334138,42963561~39927385,85233139~8112806,94049957~5294729,99633293~10695435,111990084~1888886,118298712~4796797,125900802~6292620,132586105~717255,139547811~17875399,157648610~10235097,169549666~5434611,184529373~1069310,185834642~3478716,190394373~7279350,202501730~5826341,210416821~14935744,228850744~2814978,236916784~11883992,254009698~1204295,255893971~9305901,267976284~801439,270216735~405595,275085725~33250322,314708100~9101289,334290642~4092797,341716408~2925347,349780859~1090521,355197550~1435514,357229957~14303839,372438365~5654930,379265209~17532778] wrote [19334362~4315613,36284882~2677627,75492970~3317754,118298712~2919173,130970235~1223187,196793454~777016,210416821~5372620,349780859~1090521,386221147~1313091,392734617~701911] 2024-09-13T00:39:55.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:54 smithi181 bash[67587]: cluster 2024-09-13T00:39:53.755371+0000 mon.a (mon.0) 3631 : cluster [DBG] osdmap e1971: 8 total, 8 up, 8 in 2024-09-13T00:39:55.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:54 smithi181 bash[67587]: audit 2024-09-13T00:39:53.837845+0000 mon.a (mon.0) 3632 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:39:55.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:54 smithi181 bash[67587]: cluster 2024-09-13T00:39:54.442542+0000 mgr.y (mgr.24661) 1883 : cluster [DBG] pgmap v3415: 137 pgs: 137 active+clean; 473 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 192 KiB/s rd, 45 MiB/s wr, 313 op/s 2024-09-13T00:39:55.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:54 smithi136 bash[75164]: cluster 2024-09-13T00:39:53.755371+0000 mon.a (mon.0) 3631 : cluster [DBG] osdmap e1971: 8 total, 8 up, 8 in 2024-09-13T00:39:55.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:54 smithi136 bash[75164]: audit 2024-09-13T00:39:53.837845+0000 mon.a (mon.0) 3632 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:39:55.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:54 smithi136 bash[75164]: cluster 2024-09-13T00:39:54.442542+0000 mgr.y (mgr.24661) 1883 : cluster [DBG] pgmap v3415: 137 pgs: 137 active+clean; 473 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 192 KiB/s rd, 45 MiB/s wr, 313 op/s 2024-09-13T00:39:55.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:54 smithi136 bash[76371]: cluster 2024-09-13T00:39:53.755371+0000 mon.a (mon.0) 3631 : cluster [DBG] osdmap e1971: 8 total, 8 up, 8 in 2024-09-13T00:39:55.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:54 smithi136 bash[76371]: audit 2024-09-13T00:39:53.837845+0000 mon.a (mon.0) 3632 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:39:55.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:54 smithi136 bash[76371]: cluster 2024-09-13T00:39:54.442542+0000 mgr.y (mgr.24661) 1883 : cluster [DBG] pgmap v3415: 137 pgs: 137 active+clean; 473 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 192 KiB/s rd, 45 MiB/s wr, 313 op/s 2024-09-13T00:39:55.899 INFO:tasks.workunit.client.0.smithi136.stdout: i=14 exists [349554~10146755,15629417~8020558,25151502~1993454,30436866~3685803,34486600~1452745,36284882~6334138,42963561~40546716,85233139~8112806,94049957~5294729,99633293~10695435,111990084~1888886,118298712~4796797,125900802~6292620,132586105~717255,137422775~30460932,169549666~5434611,184275136~5796665,190394373~7279350,202501730~5826341,209855877~15496688,228850744~2814978,236916784~11883992,254009698~1204295,255893971~9305901,267976284~801439,270216735~405595,271194284~37141763,312658844~1077171,314708100~11326954,334290642~4092797,341716408~2925347,349780859~1090521,355197550~1435514,357229957~14303839,372438365~5654930,378542603~18255384] wrote [77299628~6210649,137422775~7478256,156080958~4453769,184275136~5796665,209855877~4852798,271194284~7102112,297528430~4530993,312658844~1077171,321918734~4116320,378542603~7225041] 2024-09-13T00:39:56.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:55 smithi181 bash[67587]: cluster 2024-09-13T00:39:54.782479+0000 mon.a (mon.0) 3633 : cluster [DBG] osdmap e1972: 8 total, 8 up, 8 in 2024-09-13T00:39:56.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:55 smithi136 bash[75164]: cluster 2024-09-13T00:39:54.782479+0000 mon.a (mon.0) 3633 : cluster [DBG] osdmap e1972: 8 total, 8 up, 8 in 2024-09-13T00:39:56.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:55 smithi136 bash[76371]: cluster 2024-09-13T00:39:54.782479+0000 mon.a (mon.0) 3633 : cluster [DBG] osdmap e1972: 8 total, 8 up, 8 in 2024-09-13T00:39:56.952 INFO:tasks.workunit.client.0.smithi136.stdout: i=15 exists [349554~10146755,15629417~8020558,25151502~1993454,30436866~3685803,34486600~1452745,36284882~6334138,42963561~40546716,85233139~8112806,94049957~5294729,99633293~10695435,111990084~1888886,118298712~4796797,125900802~6292620,132586105~717255,137422775~30460932,169549666~5434611,181648361~8423440,190394373~7279350,202501730~5826341,209855877~15496688,228850744~2814978,236916784~11883992,254009698~1204295,255893971~9305901,267976284~801439,270216735~405595,271194284~37141763,312658844~1077171,314708100~11326954,334290642~4092797,341716408~2925347,349780859~1090521,355197550~1435514,357229957~14303839,372438365~5654930,378542603~18255384,399154333~7398128] wrote [69936108~351383,152443566~7602368,181648361~7586200,288210017~10491237,315280243~15871,367058150~2688799,399154333~7398128] 2024-09-13T00:39:57.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:56 smithi181 bash[67587]: cluster 2024-09-13T00:39:55.781637+0000 mon.a (mon.0) 3634 : cluster [DBG] osdmap e1973: 8 total, 8 up, 8 in 2024-09-13T00:39:57.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:56 smithi181 bash[67587]: cluster 2024-09-13T00:39:56.443485+0000 mgr.y (mgr.24661) 1884 : cluster [DBG] pgmap v3418: 137 pgs: 137 active+clean; 561 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 30 MiB/s wr, 18 op/s 2024-09-13T00:39:57.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:56 smithi136 bash[75164]: cluster 2024-09-13T00:39:55.781637+0000 mon.a (mon.0) 3634 : cluster [DBG] osdmap e1973: 8 total, 8 up, 8 in 2024-09-13T00:39:57.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:56 smithi136 bash[75164]: cluster 2024-09-13T00:39:56.443485+0000 mgr.y (mgr.24661) 1884 : cluster [DBG] pgmap v3418: 137 pgs: 137 active+clean; 561 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 30 MiB/s wr, 18 op/s 2024-09-13T00:39:57.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:56 smithi136 bash[76371]: cluster 2024-09-13T00:39:55.781637+0000 mon.a (mon.0) 3634 : cluster [DBG] osdmap e1973: 8 total, 8 up, 8 in 2024-09-13T00:39:57.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:56 smithi136 bash[76371]: cluster 2024-09-13T00:39:56.443485+0000 mgr.y (mgr.24661) 1884 : cluster [DBG] pgmap v3418: 137 pgs: 137 active+clean; 561 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 30 MiB/s wr, 18 op/s 2024-09-13T00:39:57.900 INFO:tasks.workunit.client.0.smithi136.stdout: i=16 exists [349554~10146755,15629417~8020558,25151502~1993454,30436866~3685803,34486600~8132420,42963561~40546716,85233139~14111547,99633293~10695435,111990084~1888886,118298712~4796797,125900802~6292620,132586105~717255,137422775~30460932,169549666~5434611,176921408~892517,181648361~8423440,190394373~7279350,202501730~5826341,209855877~15496688,228850744~2814978,236916784~11883992,254009698~1204295,255893971~9305901,267976284~801439,270216735~405595,271194284~37141763,312658844~1077171,314708100~12971245,334290642~4092797,341716408~2925347,349780859~1090521,355197550~1435514,357229957~14303839,372438365~5654930,378542603~18255384,399154333~7398128] wrote [35681415~772700,92199793~5688292,156150107~7886250,176921408~892517,187872938~1241731,281871891~120246,314746458~7252592,324025079~3654266,383310252~7301178] 2024-09-13T00:39:58.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:57 smithi181 bash[67587]: cluster 2024-09-13T00:39:56.803400+0000 mon.a (mon.0) 3635 : cluster [DBG] osdmap e1974: 8 total, 8 up, 8 in 2024-09-13T00:39:58.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:57 smithi136 bash[75164]: cluster 2024-09-13T00:39:56.803400+0000 mon.a (mon.0) 3635 : cluster [DBG] osdmap e1974: 8 total, 8 up, 8 in 2024-09-13T00:39:58.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:57 smithi136 bash[76371]: cluster 2024-09-13T00:39:56.803400+0000 mon.a (mon.0) 3635 : cluster [DBG] osdmap e1974: 8 total, 8 up, 8 in 2024-09-13T00:39:58.957 INFO:tasks.workunit.client.0.smithi136.stdout: i=17 exists [349554~10146755,12672196~10977779,25151502~1993454,30436866~3685803,34486600~49023677,85233139~14111547,99633293~10695435,111990084~1888886,118298712~4796797,125900802~6292620,132586105~717255,137422775~31570866,169549666~5434611,176921408~892517,181648361~8423440,190394373~7279350,202501730~5826341,209855877~15496688,228850744~2814978,236916784~11883992,254009698~1204295,255893971~9305901,266440832~2884976,270216735~405595,271194284~37141763,312658844~1077171,314708100~12971245,334290642~4092797,340862758~3778997,349780859~1090521,355197550~1435514,357229957~14303839,372438365~5654930,378542603~18255384,399154333~7398128] wrote [4044008~1500692,12672196~4646698,41298424~7573603,164046117~4947524,239452985~7571016,266440832~2884976,320605280~1229253,340862758~2115375,373744348~731198] 2024-09-13T00:39:59.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:58 smithi181 bash[67587]: cluster 2024-09-13T00:39:57.804691+0000 mon.a (mon.0) 3636 : cluster [DBG] osdmap e1975: 8 total, 8 up, 8 in 2024-09-13T00:39:59.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:58 smithi181 bash[67587]: cluster 2024-09-13T00:39:58.444167+0000 mgr.y (mgr.24661) 1885 : cluster [DBG] pgmap v3421: 137 pgs: 137 active+clean; 649 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 225 KiB/s rd, 38 MiB/s wr, 354 op/s 2024-09-13T00:39:59.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:58 smithi136 bash[75164]: cluster 2024-09-13T00:39:57.804691+0000 mon.a (mon.0) 3636 : cluster [DBG] osdmap e1975: 8 total, 8 up, 8 in 2024-09-13T00:39:59.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:58 smithi136 bash[75164]: cluster 2024-09-13T00:39:58.444167+0000 mgr.y (mgr.24661) 1885 : cluster [DBG] pgmap v3421: 137 pgs: 137 active+clean; 649 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 225 KiB/s rd, 38 MiB/s wr, 354 op/s 2024-09-13T00:39:59.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:58 smithi136 bash[76371]: cluster 2024-09-13T00:39:57.804691+0000 mon.a (mon.0) 3636 : cluster [DBG] osdmap e1975: 8 total, 8 up, 8 in 2024-09-13T00:39:59.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:58 smithi136 bash[76371]: cluster 2024-09-13T00:39:58.444167+0000 mgr.y (mgr.24661) 1885 : cluster [DBG] pgmap v3421: 137 pgs: 137 active+clean; 649 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 225 KiB/s rd, 38 MiB/s wr, 354 op/s 2024-09-13T00:39:59.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:39:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:39:59] "GET /metrics HTTP/1.1" 200 36752 "" "Prometheus/2.43.0" 2024-09-13T00:39:59.912 INFO:tasks.workunit.client.0.smithi136.stdout: i=18 exists [349554~10146755,12672196~10977779,25151502~1993454,30436866~3685803,34486600~49023677,85233139~14111547,99633293~10695435,111990084~1888886,118298712~4796797,125900802~7402558,137422775~31570866,169549666~5434611,176921408~892517,181648361~8423440,190394373~7279350,202501730~5826341,209855877~15496688,228850744~2814978,236916784~11883992,253976929~1237064,255893971~9305901,266440832~2884976,270216735~405595,271194284~37141763,312658844~1077171,314708100~12971245,334290642~4092797,340862758~3778997,349780859~1090521,355197550~1435514,357229957~14303839,372438365~5654930,378542603~18255384,399154333~7398128] wrote [779877~6463238,46280656~4369903,131817333~1228329,238014328~4184368,253976929~692918,277079440~2295397,284326362~934484,365150358~258762,391120616~285214] 2024-09-13T00:40:00.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:39:59 smithi136 bash[76371]: cluster 2024-09-13T00:39:58.826948+0000 mon.a (mon.0) 3637 : cluster [DBG] osdmap e1976: 8 total, 8 up, 8 in 2024-09-13T00:40:00.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:39:59 smithi136 bash[75164]: cluster 2024-09-13T00:39:58.826948+0000 mon.a (mon.0) 3637 : cluster [DBG] osdmap e1976: 8 total, 8 up, 8 in 2024-09-13T00:40:00.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:39:59 smithi181 bash[67587]: cluster 2024-09-13T00:39:58.826948+0000 mon.a (mon.0) 3637 : cluster [DBG] osdmap e1976: 8 total, 8 up, 8 in 2024-09-13T00:40:00.997 INFO:tasks.workunit.client.0.smithi136.stdout: i=19 exists [349554~11632857,12672196~14472760,30436866~3685803,34486600~49023677,85233139~14111547,99633293~10695435,111990084~1888886,118298712~4796797,125900802~7402558,137422775~31570866,169549666~5434611,176921408~892517,181648361~16025362,202501730~5826341,209855877~15496688,228850744~2814978,234483769~14317007,253976929~1237064,255893971~9305901,266440832~2884976,270216735~405595,271194284~37141763,312658844~1077171,314708100~12971245,334290642~4092797,340862758~3778997,349780859~1090521,355197550~1435514,357229957~14303839,372438365~5654930,378542603~18255384,399154333~7398128,409964598~4126707] wrote [6512841~5469570,20240601~6851271,67866250~1818956,147173517~3311657,185454789~6973791,234483769~7495500,303917827~2792906,358988383~1545760,409964598~4126707] 2024-09-13T00:40:01.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:00 smithi136 bash[75164]: cluster 2024-09-13T00:39:59.858196+0000 mon.a (mon.0) 3638 : cluster [DBG] osdmap e1977: 8 total, 8 up, 8 in 2024-09-13T00:40:01.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:00 smithi136 bash[75164]: cluster 2024-09-13T00:40:00.000149+0000 mon.a (mon.0) 3639 : cluster [INF] overall HEALTH_OK 2024-09-13T00:40:01.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:00 smithi136 bash[75164]: cluster 2024-09-13T00:40:00.444867+0000 mgr.y (mgr.24661) 1886 : cluster [DBG] pgmap v3424: 137 pgs: 137 active+clean; 713 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 225 KiB/s rd, 35 MiB/s wr, 352 op/s 2024-09-13T00:40:01.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:00 smithi136 bash[76371]: cluster 2024-09-13T00:39:59.858196+0000 mon.a (mon.0) 3638 : cluster [DBG] osdmap e1977: 8 total, 8 up, 8 in 2024-09-13T00:40:01.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:00 smithi136 bash[76371]: cluster 2024-09-13T00:40:00.000149+0000 mon.a (mon.0) 3639 : cluster [INF] overall HEALTH_OK 2024-09-13T00:40:01.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:00 smithi136 bash[76371]: cluster 2024-09-13T00:40:00.444867+0000 mgr.y (mgr.24661) 1886 : cluster [DBG] pgmap v3424: 137 pgs: 137 active+clean; 713 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 225 KiB/s rd, 35 MiB/s wr, 352 op/s 2024-09-13T00:40:01.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:00 smithi181 bash[67587]: cluster 2024-09-13T00:39:59.858196+0000 mon.a (mon.0) 3638 : cluster [DBG] osdmap e1977: 8 total, 8 up, 8 in 2024-09-13T00:40:01.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:00 smithi181 bash[67587]: cluster 2024-09-13T00:40:00.000149+0000 mon.a (mon.0) 3639 : cluster [INF] overall HEALTH_OK 2024-09-13T00:40:01.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:00 smithi181 bash[67587]: cluster 2024-09-13T00:40:00.444867+0000 mgr.y (mgr.24661) 1886 : cluster [DBG] pgmap v3424: 137 pgs: 137 active+clean; 713 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 225 KiB/s rd, 35 MiB/s wr, 352 op/s 2024-09-13T00:40:01.923 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to 1 diff [25165824~4194304,54525952~8388608,88080384~4194304,100663296~4194304,125829120~8388608,150994944~4194304,285212672~8388608,314572800~8388608] 2024-09-13T00:40:01.923 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [25165824~4194304,54525952~8388608,88080384~4194304,100663296~4194304,125829120~8388608,150994944~4194304,285212672~8388608,314572800~8388608] 2024-09-13T00:40:01.942 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [25165824~4194304,54525952~12582912,88080384~4194304,100663296~4194304,125829120~8388608,150994944~4194304,285212672~12582912,314572800~8388608] 2024-09-13T00:40:01.942 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to 2 diff [16777216~12582912,41943040~25165824,88080384~4194304,100663296~4194304,125829120~8388608,150994944~4194304,167772160~8388608,180355072~4194304,201326592~8388608,218103808~8388608,285212672~8388608,314572800~8388608,360710144~4194304] 2024-09-13T00:40:01.942 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [16777216~12582912,41943040~25165824,88080384~4194304,100663296~4194304,125829120~8388608,150994944~4194304,167772160~8388608,180355072~4194304,201326592~8388608,218103808~8388608,285212672~8388608,314572800~8388608,360710144~4194304] 2024-09-13T00:40:01.946 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [16777216~12582912,41943040~25165824,88080384~4194304,100663296~8388608,125829120~8388608,150994944~4194304,167772160~8388608,180355072~8388608,201326592~8388608,218103808~8388608,285212672~12582912,314572800~8388608,360710144~8388608] 2024-09-13T00:40:01.946 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to 3 diff [16777216~8388608,41943040~29360128,88080384~4194304,100663296~4194304,125829120~8388608,142606336~4194304,150994944~4194304,159383552~4194304,167772160~8388608,180355072~4194304,201326592~8388608,218103808~8388608,255852544~8388608,272629760~20971520,314572800~8388608,360710144~4194304,381681664~4194304] 2024-09-13T00:40:01.947 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [16777216~8388608,41943040~29360128,88080384~4194304,100663296~4194304,125829120~8388608,142606336~4194304,150994944~4194304,159383552~4194304,167772160~8388608,180355072~4194304,201326592~8388608,218103808~8388608,255852544~8388608,272629760~20971520,314572800~8388608,360710144~4194304,381681664~4194304] 2024-09-13T00:40:01.952 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [16777216~12582912,41943040~29360128,88080384~4194304,100663296~8388608,125829120~8388608,142606336~12582912,159383552~4194304,167772160~8388608,180355072~8388608,201326592~8388608,218103808~8388608,255852544~8388608,272629760~25165824,314572800~8388608,360710144~8388608,381681664~8388608] 2024-09-13T00:40:01.952 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to 4 diff [4194304~8388608,16777216~8388608,41943040~29360128,83886080~8388608,100663296~4194304,125829120~8388608,142606336~4194304,150994944~4194304,159383552~4194304,167772160~8388608,180355072~4194304,201326592~8388608,218103808~8388608,255852544~8388608,272629760~20971520,297795584~12582912,314572800~8388608,360710144~4194304,377487360~8388608] 2024-09-13T00:40:01.952 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [4194304~8388608,16777216~8388608,41943040~29360128,83886080~8388608,100663296~4194304,125829120~8388608,142606336~4194304,150994944~4194304,159383552~4194304,167772160~8388608,180355072~4194304,201326592~8388608,218103808~8388608,255852544~8388608,272629760~20971520,297795584~12582912,314572800~8388608,360710144~4194304,377487360~8388608] 2024-09-13T00:40:01.959 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [4194304~8388608,16777216~12582912,41943040~29360128,83886080~8388608,100663296~8388608,125829120~8388608,142606336~12582912,159383552~4194304,167772160~8388608,180355072~8388608,201326592~8388608,218103808~8388608,255852544~8388608,272629760~37748736,314572800~12582912,360710144~8388608,377487360~12582912] 2024-09-13T00:40:01.959 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to 5 diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~29360128,83886080~8388608,100663296~4194304,125829120~8388608,142606336~4194304,150994944~4194304,159383552~4194304,167772160~8388608,180355072~4194304,201326592~8388608,218103808~8388608,255852544~8388608,272629760~33554432,314572800~8388608,356515840~8388608,377487360~8388608] 2024-09-13T00:40:01.959 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~29360128,83886080~8388608,100663296~4194304,125829120~8388608,142606336~4194304,150994944~4194304,159383552~4194304,167772160~8388608,180355072~4194304,201326592~8388608,218103808~8388608,255852544~8388608,272629760~33554432,314572800~8388608,356515840~8388608,377487360~8388608] 2024-09-13T00:40:01.966 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [4194304~8388608,16777216~20971520,41943040~33554432,83886080~8388608,100663296~8388608,125829120~8388608,142606336~12582912,159383552~4194304,167772160~8388608,180355072~8388608,201326592~8388608,218103808~8388608,255852544~8388608,272629760~37748736,314572800~12582912,356515840~12582912,377487360~12582912] 2024-09-13T00:40:01.966 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to 6 diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~29360128,83886080~8388608,96468992~8388608,125829120~8388608,142606336~4194304,150994944~12582912,167772160~8388608,180355072~4194304,201326592~8388608,218103808~8388608,239075328~8388608,255852544~8388608,272629760~33554432,314572800~8388608,356515840~12582912,377487360~8388608] 2024-09-13T00:40:01.966 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~29360128,83886080~8388608,96468992~8388608,125829120~8388608,142606336~4194304,150994944~12582912,167772160~8388608,180355072~4194304,201326592~8388608,218103808~8388608,239075328~8388608,255852544~8388608,272629760~33554432,314572800~8388608,356515840~12582912,377487360~8388608] 2024-09-13T00:40:01.976 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [4194304~8388608,16777216~20971520,41943040~33554432,83886080~8388608,96468992~12582912,125829120~8388608,142606336~33554432,180355072~8388608,201326592~8388608,218103808~8388608,239075328~12582912,255852544~8388608,272629760~37748736,314572800~12582912,356515840~16777216,377487360~12582912] 2024-09-13T00:40:01.976 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to 7 diff [0~12582912,16777216~8388608,29360128~4194304,41943040~29360128,83886080~8388608,96468992~8388608,125829120~8388608,142606336~4194304,150994944~25165824,180355072~4194304,201326592~8388608,218103808~8388608,239075328~8388608,255852544~8388608,272629760~33554432,314572800~8388608,339738624~4194304,356515840~12582912,377487360~16777216] 2024-09-13T00:40:01.976 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~29360128,83886080~8388608,96468992~8388608,125829120~8388608,142606336~4194304,150994944~25165824,180355072~4194304,201326592~8388608,218103808~8388608,239075328~8388608,255852544~8388608,272629760~33554432,314572800~8388608,339738624~4194304,356515840~12582912,377487360~16777216] 2024-09-13T00:40:01.988 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~12582912,16777216~20971520,41943040~33554432,83886080~8388608,96468992~12582912,125829120~8388608,142606336~33554432,180355072~8388608,201326592~8388608,218103808~8388608,239075328~12582912,255852544~8388608,272629760~37748736,314572800~12582912,339738624~8388608,356515840~16777216,377487360~20971520] 2024-09-13T00:40:01.988 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to 8 diff [0~12582912,16777216~8388608,29360128~4194304,41943040~29360128,83886080~8388608,96468992~8388608,109051904~4194304,125829120~8388608,138412032~37748736,180355072~4194304,201326592~8388608,218103808~8388608,239075328~8388608,255852544~12582912,272629760~33554432,314572800~8388608,339738624~4194304,356515840~12582912,377487360~16777216] 2024-09-13T00:40:01.988 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~29360128,83886080~8388608,96468992~8388608,109051904~4194304,125829120~8388608,138412032~37748736,180355072~4194304,201326592~8388608,218103808~8388608,239075328~8388608,255852544~12582912,272629760~33554432,314572800~8388608,339738624~4194304,356515840~12582912,377487360~16777216] 2024-09-13T00:40:01.998 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~12582912,16777216~20971520,41943040~33554432,83886080~33554432,125829120~8388608,138412032~37748736,180355072~8388608,201326592~8388608,218103808~8388608,239075328~12582912,255852544~12582912,272629760~37748736,314572800~12582912,339738624~8388608,356515840~16777216,377487360~20971520] 2024-09-13T00:40:01.998 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to 9 diff [0~12582912,16777216~8388608,29360128~4194304,37748736~33554432,75497472~29360128,109051904~4194304,125829120~8388608,138412032~37748736,180355072~4194304,201326592~8388608,218103808~8388608,239075328~8388608,255852544~50331648,314572800~8388608,339738624~4194304,356515840~16777216,377487360~16777216] 2024-09-13T00:40:01.998 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,37748736~33554432,75497472~29360128,109051904~4194304,125829120~8388608,138412032~37748736,180355072~4194304,201326592~8388608,218103808~8388608,239075328~8388608,255852544~50331648,314572800~8388608,339738624~4194304,356515840~16777216,377487360~16777216] 2024-09-13T00:40:02.012 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~12582912,16777216~100663296,125829120~8388608,138412032~37748736,180355072~8388608,201326592~8388608,218103808~8388608,239075328~12582912,255852544~54525952,314572800~12582912,339738624~8388608,356515840~16777216,377487360~20971520] 2024-09-13T00:40:02.012 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to 10 diff [0~12582912,16777216~8388608,29360128~4194304,37748736~33554432,75497472~37748736,125829120~8388608,138412032~37748736,180355072~4194304,201326592~20971520,239075328~8388608,255852544~50331648,314572800~8388608,335544320~8388608,356515840~16777216,377487360~16777216] 2024-09-13T00:40:02.012 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,37748736~33554432,75497472~37748736,125829120~8388608,138412032~37748736,180355072~4194304,201326592~20971520,239075328~8388608,255852544~50331648,314572800~8388608,335544320~8388608,356515840~16777216,377487360~16777216] 2024-09-13T00:40:02.021 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~12582912,16777216~100663296,125829120~8388608,138412032~37748736,180355072~8388608,201326592~25165824,239075328~12582912,255852544~54525952,314572800~12582912,335544320~12582912,356515840~16777216,377487360~20971520] 2024-09-13T00:40:02.021 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to 11 diff [0~25165824,29360128~4194304,37748736~33554432,75497472~37748736,125829120~8388608,138412032~37748736,180355072~4194304,188743680~8388608,201326592~20971520,226492416~4194304,234881024~12582912,251658240~54525952,314572800~12582912,335544320~8388608,356515840~16777216,377487360~16777216] 2024-09-13T00:40:02.021 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,29360128~4194304,37748736~33554432,75497472~37748736,125829120~8388608,138412032~37748736,180355072~4194304,188743680~8388608,201326592~20971520,226492416~4194304,234881024~12582912,251658240~54525952,314572800~12582912,335544320~8388608,356515840~16777216,377487360~16777216] 2024-09-13T00:40:02.032 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~117440512,125829120~8388608,138412032~37748736,180355072~130023424,314572800~12582912,335544320~12582912,356515840~16777216,377487360~20971520] 2024-09-13T00:40:02.032 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to 12 diff [0~25165824,29360128~4194304,37748736~75497472,117440512~4194304,125829120~8388608,138412032~37748736,180355072~16777216,201326592~20971520,226492416~4194304,234881024~12582912,251658240~54525952,314572800~12582912,331350016~4194304,339738624~4194304,352321536~46137344] 2024-09-13T00:40:02.032 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,29360128~4194304,37748736~75497472,117440512~4194304,125829120~8388608,138412032~37748736,180355072~16777216,201326592~20971520,226492416~4194304,234881024~12582912,251658240~54525952,314572800~12582912,331350016~4194304,339738624~4194304,352321536~46137344] 2024-09-13T00:40:02.044 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~134217728,138412032~37748736,180355072~130023424,314572800~12582912,331350016~16777216,352321536~46137344] 2024-09-13T00:40:02.044 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to 13 diff [0~25165824,29360128~50331648,83886080~29360128,117440512~16777216,138412032~37748736,180355072~16777216,201326592~29360128,234881024~12582912,251658240~54525952,314572800~12582912,331350016~4194304,339738624~4194304,348127232~50331648] 2024-09-13T00:40:02.045 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,29360128~50331648,83886080~29360128,117440512~16777216,138412032~37748736,180355072~16777216,201326592~29360128,234881024~12582912,251658240~54525952,314572800~12582912,331350016~4194304,339738624~4194304,348127232~50331648] 2024-09-13T00:40:02.058 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~134217728,138412032~37748736,180355072~130023424,314572800~12582912,331350016~67108864] 2024-09-13T00:40:02.058 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to 14 diff [0~25165824,29360128~50331648,83886080~29360128,117440512~58720256,180355072~16777216,201326592~29360128,234881024~12582912,251658240~54525952,310378496~16777216,331350016~4194304,339738624~4194304,348127232~50331648] 2024-09-13T00:40:02.058 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,29360128~50331648,83886080~29360128,117440512~58720256,180355072~16777216,201326592~29360128,234881024~12582912,251658240~54525952,310378496~16777216,331350016~4194304,339738624~4194304,348127232~50331648] 2024-09-13T00:40:02.074 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~176160768,180355072~146800640,331350016~67108864] 2024-09-13T00:40:02.074 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to 15 diff [0~25165824,29360128~50331648,83886080~29360128,117440512~58720256,180355072~16777216,201326592~29360128,234881024~12582912,251658240~54525952,310378496~16777216,331350016~4194304,339738624~4194304,348127232~58720256] 2024-09-13T00:40:02.074 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,29360128~50331648,83886080~29360128,117440512~58720256,180355072~16777216,201326592~29360128,234881024~12582912,251658240~54525952,310378496~16777216,331350016~4194304,339738624~4194304,348127232~58720256] 2024-09-13T00:40:02.091 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~176160768,180355072~146800640,331350016~75497472] 2024-09-13T00:40:02.091 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to 16 diff [0~25165824,29360128~50331648,83886080~29360128,117440512~79691776,201326592~29360128,234881024~12582912,251658240~54525952,310378496~25165824,339738624~4194304,348127232~58720256] 2024-09-13T00:40:02.091 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,29360128~50331648,83886080~29360128,117440512~79691776,201326592~29360128,234881024~12582912,251658240~54525952,310378496~25165824,339738624~4194304,348127232~58720256] 2024-09-13T00:40:02.108 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~406847488] 2024-09-13T00:40:02.108 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to 17 diff [0~25165824,29360128~50331648,83886080~29360128,117440512~79691776,201326592~29360128,234881024~71303168,310378496~25165824,339738624~4194304,348127232~58720256] 2024-09-13T00:40:02.108 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,29360128~50331648,83886080~29360128,117440512~79691776,201326592~29360128,234881024~71303168,310378496~25165824,339738624~4194304,348127232~58720256] 2024-09-13T00:40:02.125 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~406847488] 2024-09-13T00:40:02.125 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to 18 diff [0~25165824,29360128~50331648,83886080~29360128,117440512~79691776,201326592~29360128,234881024~12582912,251658240~54525952,310378496~25165824,339738624~4194304,348127232~58720256] 2024-09-13T00:40:02.125 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,29360128~50331648,83886080~29360128,117440512~79691776,201326592~29360128,234881024~12582912,251658240~54525952,310378496~25165824,339738624~4194304,348127232~58720256] 2024-09-13T00:40:02.145 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~406847488] 2024-09-13T00:40:02.145 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to 19 diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:02.145 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:02.165 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:02.165 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to 2 diff [16777216~8388608,41943040~16777216,100663296~4194304,167772160~8388608,180355072~4194304,201326592~8388608,218103808~8388608,360710144~4194304] 2024-09-13T00:40:02.165 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [16777216~8388608,41943040~16777216,100663296~4194304,167772160~8388608,180355072~4194304,201326592~8388608,218103808~8388608,360710144~4194304] 2024-09-13T00:40:02.168 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [16777216~8388608,41943040~20971520,100663296~8388608,167772160~8388608,180355072~8388608,201326592~8388608,218103808~8388608,360710144~8388608] 2024-09-13T00:40:02.168 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to 3 diff [16777216~8388608,41943040~25165824,100663296~4194304,142606336~4194304,159383552~4194304,167772160~8388608,180355072~4194304,201326592~8388608,218103808~8388608,255852544~8388608,272629760~12582912,360710144~4194304,381681664~4194304] 2024-09-13T00:40:02.169 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [16777216~8388608,41943040~25165824,100663296~4194304,142606336~4194304,159383552~4194304,167772160~8388608,180355072~4194304,201326592~8388608,218103808~8388608,255852544~8388608,272629760~12582912,360710144~4194304,381681664~4194304] 2024-09-13T00:40:02.173 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [16777216~12582912,41943040~29360128,100663296~8388608,142606336~8388608,159383552~4194304,167772160~8388608,180355072~8388608,201326592~8388608,218103808~8388608,255852544~8388608,272629760~16777216,360710144~8388608,381681664~8388608] 2024-09-13T00:40:02.177 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to 4 diff [4194304~8388608,16777216~8388608,41943040~25165824,83886080~8388608,100663296~4194304,142606336~4194304,159383552~4194304,167772160~8388608,180355072~4194304,201326592~8388608,218103808~8388608,255852544~8388608,272629760~12582912,297795584~12582912,314572800~8388608,360710144~4194304,377487360~8388608] 2024-09-13T00:40:02.177 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [4194304~8388608,16777216~8388608,41943040~25165824,83886080~8388608,100663296~4194304,142606336~4194304,159383552~4194304,167772160~8388608,180355072~4194304,201326592~8388608,218103808~8388608,255852544~8388608,272629760~12582912,297795584~12582912,314572800~8388608,360710144~4194304,377487360~8388608] 2024-09-13T00:40:02.178 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [4194304~8388608,16777216~12582912,41943040~29360128,83886080~8388608,100663296~8388608,142606336~8388608,159383552~4194304,167772160~8388608,180355072~8388608,201326592~8388608,218103808~8388608,255852544~8388608,272629760~16777216,297795584~12582912,314572800~12582912,360710144~8388608,377487360~12582912] 2024-09-13T00:40:02.178 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to 5 diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~29360128,83886080~8388608,100663296~4194304,142606336~4194304,159383552~4194304,167772160~8388608,180355072~4194304,201326592~8388608,218103808~8388608,255852544~8388608,272629760~12582912,289406976~16777216,314572800~8388608,356515840~8388608,377487360~8388608] 2024-09-13T00:40:02.178 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~29360128,83886080~8388608,100663296~4194304,142606336~4194304,159383552~4194304,167772160~8388608,180355072~4194304,201326592~8388608,218103808~8388608,255852544~8388608,272629760~12582912,289406976~16777216,314572800~8388608,356515840~8388608,377487360~8388608] 2024-09-13T00:40:02.184 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [4194304~8388608,16777216~20971520,41943040~33554432,83886080~8388608,100663296~8388608,142606336~8388608,159383552~4194304,167772160~8388608,180355072~8388608,201326592~8388608,218103808~8388608,255852544~8388608,272629760~37748736,314572800~12582912,356515840~12582912,377487360~12582912] 2024-09-13T00:40:02.184 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to 6 diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~29360128,83886080~8388608,96468992~8388608,142606336~4194304,155189248~8388608,167772160~8388608,180355072~4194304,201326592~8388608,218103808~8388608,239075328~8388608,255852544~8388608,272629760~12582912,289406976~16777216,314572800~8388608,356515840~12582912,377487360~8388608] 2024-09-13T00:40:02.184 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~29360128,83886080~8388608,96468992~8388608,142606336~4194304,155189248~8388608,167772160~8388608,180355072~4194304,201326592~8388608,218103808~8388608,239075328~8388608,255852544~8388608,272629760~12582912,289406976~16777216,314572800~8388608,356515840~12582912,377487360~8388608] 2024-09-13T00:40:02.191 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [4194304~8388608,16777216~20971520,41943040~33554432,83886080~8388608,96468992~12582912,142606336~8388608,155189248~20971520,180355072~8388608,201326592~8388608,218103808~8388608,239075328~12582912,255852544~8388608,272629760~37748736,314572800~12582912,356515840~16777216,377487360~12582912] 2024-09-13T00:40:02.191 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to 7 diff [0~12582912,16777216~8388608,29360128~4194304,41943040~29360128,83886080~8388608,96468992~8388608,130023424~4194304,142606336~4194304,150994944~25165824,180355072~4194304,201326592~8388608,218103808~8388608,239075328~8388608,255852544~8388608,272629760~12582912,289406976~16777216,314572800~8388608,339738624~4194304,356515840~12582912,377487360~16777216] 2024-09-13T00:40:02.191 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~29360128,83886080~8388608,96468992~8388608,130023424~4194304,142606336~4194304,150994944~25165824,180355072~4194304,201326592~8388608,218103808~8388608,239075328~8388608,255852544~8388608,272629760~12582912,289406976~16777216,314572800~8388608,339738624~4194304,356515840~12582912,377487360~16777216] 2024-09-13T00:40:02.198 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~12582912,16777216~20971520,41943040~33554432,83886080~8388608,96468992~12582912,130023424~4194304,142606336~33554432,180355072~8388608,201326592~8388608,218103808~8388608,239075328~12582912,255852544~8388608,272629760~37748736,314572800~12582912,339738624~8388608,356515840~16777216,377487360~20971520] 2024-09-13T00:40:02.198 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to 8 diff [0~12582912,16777216~8388608,29360128~4194304,41943040~29360128,83886080~8388608,96468992~8388608,109051904~4194304,125829120~8388608,138412032~37748736,180355072~4194304,201326592~8388608,218103808~8388608,239075328~8388608,255852544~12582912,272629760~33554432,314572800~8388608,339738624~4194304,356515840~12582912,377487360~16777216] 2024-09-13T00:40:02.198 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~29360128,83886080~8388608,96468992~8388608,109051904~4194304,125829120~8388608,138412032~37748736,180355072~4194304,201326592~8388608,218103808~8388608,239075328~8388608,255852544~12582912,272629760~33554432,314572800~8388608,339738624~4194304,356515840~12582912,377487360~16777216] 2024-09-13T00:40:02.207 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~12582912,16777216~20971520,41943040~33554432,83886080~33554432,125829120~8388608,138412032~37748736,180355072~8388608,201326592~8388608,218103808~8388608,239075328~12582912,255852544~12582912,272629760~37748736,314572800~12582912,339738624~8388608,356515840~16777216,377487360~20971520] 2024-09-13T00:40:02.207 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to 9 diff [0~12582912,16777216~8388608,29360128~4194304,37748736~33554432,75497472~29360128,109051904~4194304,125829120~8388608,138412032~37748736,180355072~4194304,201326592~8388608,218103808~8388608,239075328~8388608,255852544~50331648,314572800~8388608,339738624~4194304,356515840~16777216,377487360~16777216] 2024-09-13T00:40:02.207 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,37748736~33554432,75497472~29360128,109051904~4194304,125829120~8388608,138412032~37748736,180355072~4194304,201326592~8388608,218103808~8388608,239075328~8388608,255852544~50331648,314572800~8388608,339738624~4194304,356515840~16777216,377487360~16777216] 2024-09-13T00:40:02.216 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~12582912,16777216~100663296,125829120~8388608,138412032~37748736,180355072~8388608,201326592~8388608,218103808~8388608,239075328~12582912,255852544~54525952,314572800~12582912,339738624~8388608,356515840~16777216,377487360~20971520] 2024-09-13T00:40:02.216 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to 10 diff [0~12582912,16777216~8388608,29360128~4194304,37748736~33554432,75497472~37748736,125829120~8388608,138412032~37748736,180355072~4194304,201326592~20971520,239075328~8388608,255852544~50331648,314572800~8388608,335544320~8388608,356515840~16777216,377487360~16777216] 2024-09-13T00:40:02.217 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,37748736~33554432,75497472~37748736,125829120~8388608,138412032~37748736,180355072~4194304,201326592~20971520,239075328~8388608,255852544~50331648,314572800~8388608,335544320~8388608,356515840~16777216,377487360~16777216] 2024-09-13T00:40:02.227 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~12582912,16777216~100663296,125829120~8388608,138412032~37748736,180355072~8388608,201326592~25165824,239075328~12582912,255852544~54525952,314572800~12582912,335544320~12582912,356515840~16777216,377487360~20971520] 2024-09-13T00:40:02.227 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to 11 diff [0~25165824,29360128~4194304,37748736~33554432,75497472~37748736,125829120~8388608,138412032~37748736,180355072~4194304,188743680~8388608,201326592~20971520,226492416~4194304,234881024~12582912,251658240~54525952,314572800~12582912,335544320~8388608,356515840~16777216,377487360~16777216] 2024-09-13T00:40:02.227 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,29360128~4194304,37748736~33554432,75497472~37748736,125829120~8388608,138412032~37748736,180355072~4194304,188743680~8388608,201326592~20971520,226492416~4194304,234881024~12582912,251658240~54525952,314572800~12582912,335544320~8388608,356515840~16777216,377487360~16777216] 2024-09-13T00:40:02.238 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~117440512,125829120~8388608,138412032~37748736,180355072~130023424,314572800~12582912,335544320~12582912,356515840~16777216,377487360~20971520] 2024-09-13T00:40:02.238 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to 12 diff [0~25165824,29360128~4194304,37748736~75497472,117440512~4194304,125829120~8388608,138412032~37748736,180355072~16777216,201326592~20971520,226492416~4194304,234881024~12582912,251658240~54525952,314572800~12582912,331350016~4194304,339738624~4194304,352321536~46137344] 2024-09-13T00:40:02.239 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,29360128~4194304,37748736~75497472,117440512~4194304,125829120~8388608,138412032~37748736,180355072~16777216,201326592~20971520,226492416~4194304,234881024~12582912,251658240~54525952,314572800~12582912,331350016~4194304,339738624~4194304,352321536~46137344] 2024-09-13T00:40:02.250 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~134217728,138412032~37748736,180355072~130023424,314572800~12582912,331350016~16777216,352321536~46137344] 2024-09-13T00:40:02.250 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to 13 diff [0~25165824,29360128~50331648,83886080~29360128,117440512~16777216,138412032~37748736,180355072~16777216,201326592~29360128,234881024~12582912,251658240~54525952,314572800~12582912,331350016~4194304,339738624~4194304,348127232~50331648] 2024-09-13T00:40:02.250 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,29360128~50331648,83886080~29360128,117440512~16777216,138412032~37748736,180355072~16777216,201326592~29360128,234881024~12582912,251658240~54525952,314572800~12582912,331350016~4194304,339738624~4194304,348127232~50331648] 2024-09-13T00:40:02.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:01 smithi136 bash[75164]: cluster 2024-09-13T00:40:00.892007+0000 mon.a (mon.0) 3640 : cluster [DBG] osdmap e1978: 8 total, 8 up, 8 in 2024-09-13T00:40:02.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:01 smithi136 bash[76371]: cluster 2024-09-13T00:40:00.892007+0000 mon.a (mon.0) 3640 : cluster [DBG] osdmap e1978: 8 total, 8 up, 8 in 2024-09-13T00:40:02.264 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~134217728,138412032~37748736,180355072~130023424,314572800~12582912,331350016~67108864] 2024-09-13T00:40:02.264 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to 14 diff [0~25165824,29360128~50331648,83886080~29360128,117440512~58720256,180355072~16777216,201326592~29360128,234881024~12582912,251658240~54525952,310378496~16777216,331350016~4194304,339738624~4194304,348127232~50331648] 2024-09-13T00:40:02.264 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,29360128~50331648,83886080~29360128,117440512~58720256,180355072~16777216,201326592~29360128,234881024~12582912,251658240~54525952,310378496~16777216,331350016~4194304,339738624~4194304,348127232~50331648] 2024-09-13T00:40:02.278 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~176160768,180355072~146800640,331350016~67108864] 2024-09-13T00:40:02.278 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to 15 diff [0~25165824,29360128~50331648,83886080~29360128,117440512~58720256,180355072~16777216,201326592~29360128,234881024~12582912,251658240~54525952,310378496~16777216,331350016~4194304,339738624~4194304,348127232~58720256] 2024-09-13T00:40:02.278 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,29360128~50331648,83886080~29360128,117440512~58720256,180355072~16777216,201326592~29360128,234881024~12582912,251658240~54525952,310378496~16777216,331350016~4194304,339738624~4194304,348127232~58720256] 2024-09-13T00:40:02.294 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~176160768,180355072~146800640,331350016~75497472] 2024-09-13T00:40:02.294 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to 16 diff [0~25165824,29360128~50331648,83886080~29360128,117440512~79691776,201326592~29360128,234881024~12582912,251658240~54525952,310378496~25165824,339738624~4194304,348127232~58720256] 2024-09-13T00:40:02.294 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,29360128~50331648,83886080~29360128,117440512~79691776,201326592~29360128,234881024~12582912,251658240~54525952,310378496~25165824,339738624~4194304,348127232~58720256] 2024-09-13T00:40:02.311 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~406847488] 2024-09-13T00:40:02.311 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to 17 diff [0~25165824,29360128~50331648,83886080~29360128,117440512~79691776,201326592~29360128,234881024~71303168,310378496~25165824,339738624~4194304,348127232~58720256] 2024-09-13T00:40:02.311 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,29360128~50331648,83886080~29360128,117440512~79691776,201326592~29360128,234881024~71303168,310378496~25165824,339738624~4194304,348127232~58720256] 2024-09-13T00:40:02.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:01 smithi181 bash[67587]: cluster 2024-09-13T00:40:00.892007+0000 mon.a (mon.0) 3640 : cluster [DBG] osdmap e1978: 8 total, 8 up, 8 in 2024-09-13T00:40:02.328 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~406847488] 2024-09-13T00:40:02.328 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to 18 diff [0~25165824,29360128~50331648,83886080~29360128,117440512~79691776,201326592~29360128,234881024~12582912,251658240~54525952,310378496~25165824,339738624~4194304,348127232~58720256] 2024-09-13T00:40:02.329 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,29360128~50331648,83886080~29360128,117440512~79691776,201326592~29360128,234881024~12582912,251658240~54525952,310378496~25165824,339738624~4194304,348127232~58720256] 2024-09-13T00:40:02.348 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~406847488] 2024-09-13T00:40:02.348 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to 19 diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:02.349 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:02.368 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:02.368 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to 3 diff [20971520~4194304,58720256~8388608,142606336~4194304,159383552~4194304,218103808~8388608,255852544~8388608,272629760~12582912,381681664~4194304] 2024-09-13T00:40:02.368 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [20971520~4194304,58720256~8388608,142606336~4194304,159383552~4194304,218103808~8388608,255852544~8388608,272629760~12582912,381681664~4194304] 2024-09-13T00:40:02.371 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [20971520~8388608,58720256~12582912,142606336~8388608,159383552~4194304,218103808~8388608,255852544~8388608,272629760~16777216,381681664~8388608] 2024-09-13T00:40:02.371 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to 4 diff [4194304~8388608,16777216~8388608,58720256~8388608,83886080~8388608,142606336~4194304,159383552~4194304,218103808~8388608,255852544~8388608,272629760~12582912,297795584~12582912,314572800~8388608,377487360~8388608] 2024-09-13T00:40:02.372 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [4194304~8388608,16777216~8388608,58720256~8388608,83886080~8388608,142606336~4194304,159383552~4194304,218103808~8388608,255852544~8388608,272629760~12582912,297795584~12582912,314572800~8388608,377487360~8388608] 2024-09-13T00:40:02.375 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [4194304~8388608,16777216~12582912,58720256~12582912,83886080~8388608,142606336~8388608,159383552~4194304,218103808~8388608,255852544~8388608,272629760~16777216,297795584~12582912,314572800~12582912,377487360~12582912] 2024-09-13T00:40:02.376 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to 5 diff [4194304~8388608,16777216~8388608,29360128~4194304,54525952~16777216,83886080~8388608,142606336~4194304,159383552~4194304,218103808~8388608,255852544~8388608,272629760~12582912,289406976~16777216,314572800~8388608,356515840~8388608,377487360~8388608] 2024-09-13T00:40:02.376 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [4194304~8388608,16777216~8388608,29360128~4194304,54525952~16777216,83886080~8388608,142606336~4194304,159383552~4194304,218103808~8388608,255852544~8388608,272629760~12582912,289406976~16777216,314572800~8388608,356515840~8388608,377487360~8388608] 2024-09-13T00:40:02.380 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [4194304~8388608,16777216~20971520,54525952~20971520,83886080~8388608,142606336~8388608,159383552~4194304,218103808~8388608,255852544~8388608,272629760~37748736,314572800~12582912,356515840~8388608,377487360~12582912] 2024-09-13T00:40:02.380 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to 6 diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~29360128,83886080~8388608,96468992~4194304,142606336~4194304,155189248~8388608,218103808~8388608,239075328~8388608,255852544~8388608,272629760~12582912,289406976~16777216,314572800~8388608,356515840~12582912,377487360~8388608] 2024-09-13T00:40:02.381 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~29360128,83886080~8388608,96468992~4194304,142606336~4194304,155189248~8388608,218103808~8388608,239075328~8388608,255852544~8388608,272629760~12582912,289406976~16777216,314572800~8388608,356515840~12582912,377487360~8388608] 2024-09-13T00:40:02.387 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [4194304~8388608,16777216~20971520,41943040~33554432,83886080~8388608,96468992~4194304,142606336~8388608,155189248~12582912,218103808~8388608,239075328~12582912,255852544~8388608,272629760~37748736,314572800~12582912,356515840~16777216,377487360~12582912] 2024-09-13T00:40:02.387 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to 7 diff [0~12582912,16777216~8388608,29360128~4194304,41943040~29360128,83886080~8388608,96468992~4194304,130023424~4194304,142606336~4194304,150994944~16777216,218103808~8388608,239075328~8388608,255852544~8388608,272629760~12582912,289406976~16777216,314572800~8388608,339738624~4194304,356515840~12582912,377487360~16777216] 2024-09-13T00:40:02.387 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~29360128,83886080~8388608,96468992~4194304,130023424~4194304,142606336~4194304,150994944~16777216,218103808~8388608,239075328~8388608,255852544~8388608,272629760~12582912,289406976~16777216,314572800~8388608,339738624~4194304,356515840~12582912,377487360~16777216] 2024-09-13T00:40:02.393 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~12582912,16777216~20971520,41943040~33554432,83886080~8388608,96468992~4194304,130023424~4194304,142606336~25165824,218103808~8388608,239075328~12582912,255852544~8388608,272629760~37748736,314572800~12582912,339738624~8388608,356515840~16777216,377487360~20971520] 2024-09-13T00:40:02.394 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to 8 diff [0~12582912,16777216~8388608,29360128~4194304,41943040~29360128,83886080~8388608,96468992~4194304,109051904~4194304,125829120~8388608,138412032~29360128,218103808~8388608,239075328~8388608,255852544~12582912,272629760~33554432,314572800~8388608,339738624~4194304,356515840~12582912,377487360~16777216] 2024-09-13T00:40:02.394 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~29360128,83886080~8388608,96468992~4194304,109051904~4194304,125829120~8388608,138412032~29360128,218103808~8388608,239075328~8388608,255852544~12582912,272629760~33554432,314572800~8388608,339738624~4194304,356515840~12582912,377487360~16777216] 2024-09-13T00:40:02.401 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~12582912,16777216~20971520,41943040~33554432,83886080~16777216,109051904~8388608,125829120~8388608,138412032~29360128,218103808~8388608,239075328~12582912,255852544~12582912,272629760~37748736,314572800~12582912,339738624~8388608,356515840~16777216,377487360~20971520] 2024-09-13T00:40:02.401 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to 9 diff [0~12582912,16777216~8388608,29360128~4194304,37748736~33554432,75497472~29360128,109051904~4194304,125829120~8388608,138412032~29360128,218103808~8388608,239075328~8388608,255852544~50331648,314572800~8388608,339738624~4194304,356515840~16777216,377487360~16777216] 2024-09-13T00:40:02.401 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,37748736~33554432,75497472~29360128,109051904~4194304,125829120~8388608,138412032~29360128,218103808~8388608,239075328~8388608,255852544~50331648,314572800~8388608,339738624~4194304,356515840~16777216,377487360~16777216] 2024-09-13T00:40:02.410 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~12582912,16777216~100663296,125829120~8388608,138412032~29360128,218103808~8388608,239075328~12582912,255852544~54525952,314572800~12582912,339738624~8388608,356515840~16777216,377487360~20971520] 2024-09-13T00:40:02.410 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to 10 diff [0~12582912,16777216~8388608,29360128~4194304,37748736~33554432,75497472~37748736,125829120~8388608,138412032~29360128,171966464~4194304,201326592~4194304,209715200~12582912,239075328~8388608,255852544~50331648,314572800~8388608,335544320~8388608,356515840~16777216,377487360~16777216] 2024-09-13T00:40:02.410 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,37748736~33554432,75497472~37748736,125829120~8388608,138412032~29360128,171966464~4194304,201326592~4194304,209715200~12582912,239075328~8388608,255852544~50331648,314572800~8388608,335544320~8388608,356515840~16777216,377487360~16777216] 2024-09-13T00:40:02.419 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~12582912,16777216~100663296,125829120~8388608,138412032~29360128,171966464~4194304,201326592~25165824,239075328~12582912,255852544~54525952,314572800~12582912,335544320~12582912,356515840~16777216,377487360~20971520] 2024-09-13T00:40:02.419 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to 11 diff [0~25165824,29360128~4194304,37748736~33554432,75497472~37748736,125829120~8388608,138412032~29360128,171966464~4194304,188743680~8388608,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~54525952,314572800~12582912,335544320~8388608,356515840~16777216,377487360~16777216] 2024-09-13T00:40:02.420 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,29360128~4194304,37748736~33554432,75497472~37748736,125829120~8388608,138412032~29360128,171966464~4194304,188743680~8388608,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~54525952,314572800~12582912,335544320~8388608,356515840~16777216,377487360~16777216] 2024-09-13T00:40:02.430 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~117440512,125829120~8388608,138412032~29360128,171966464~4194304,188743680~121634816,314572800~12582912,335544320~12582912,356515840~16777216,377487360~20971520] 2024-09-13T00:40:02.430 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to 12 diff [0~25165824,29360128~4194304,37748736~33554432,75497472~37748736,117440512~4194304,125829120~8388608,138412032~29360128,171966464~4194304,184549376~12582912,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~54525952,314572800~12582912,331350016~4194304,339738624~4194304,352321536~46137344] 2024-09-13T00:40:02.430 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,29360128~4194304,37748736~33554432,75497472~37748736,117440512~4194304,125829120~8388608,138412032~29360128,171966464~4194304,184549376~12582912,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~54525952,314572800~12582912,331350016~4194304,339738624~4194304,352321536~46137344] 2024-09-13T00:40:02.442 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~134217728,138412032~29360128,171966464~4194304,184549376~125829120,314572800~12582912,331350016~16777216,352321536~46137344] 2024-09-13T00:40:02.443 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to 13 diff [0~25165824,29360128~50331648,83886080~29360128,117440512~16777216,138412032~29360128,171966464~4194304,184549376~12582912,201326592~4194304,209715200~20971520,234881024~12582912,251658240~54525952,314572800~12582912,331350016~4194304,339738624~4194304,348127232~50331648] 2024-09-13T00:40:02.443 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,29360128~50331648,83886080~29360128,117440512~16777216,138412032~29360128,171966464~4194304,184549376~12582912,201326592~4194304,209715200~20971520,234881024~12582912,251658240~54525952,314572800~12582912,331350016~4194304,339738624~4194304,348127232~50331648] 2024-09-13T00:40:02.455 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~134217728,138412032~29360128,171966464~4194304,184549376~125829120,314572800~12582912,331350016~67108864] 2024-09-13T00:40:02.455 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to 14 diff [0~25165824,29360128~50331648,83886080~29360128,117440512~50331648,171966464~4194304,180355072~16777216,201326592~4194304,209715200~20971520,234881024~12582912,251658240~54525952,310378496~16777216,331350016~4194304,339738624~4194304,348127232~50331648] 2024-09-13T00:40:02.456 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,29360128~50331648,83886080~29360128,117440512~50331648,171966464~4194304,180355072~16777216,201326592~4194304,209715200~20971520,234881024~12582912,251658240~54525952,310378496~16777216,331350016~4194304,339738624~4194304,348127232~50331648] 2024-09-13T00:40:02.468 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~167772160,171966464~4194304,180355072~146800640,331350016~67108864] 2024-09-13T00:40:02.469 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to 15 diff [0~25165824,29360128~50331648,83886080~29360128,117440512~50331648,171966464~4194304,180355072~16777216,201326592~4194304,209715200~20971520,234881024~12582912,251658240~54525952,310378496~16777216,331350016~4194304,339738624~4194304,348127232~58720256] 2024-09-13T00:40:02.469 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,29360128~50331648,83886080~29360128,117440512~50331648,171966464~4194304,180355072~16777216,201326592~4194304,209715200~20971520,234881024~12582912,251658240~54525952,310378496~16777216,331350016~4194304,339738624~4194304,348127232~58720256] 2024-09-13T00:40:02.482 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~167772160,171966464~4194304,180355072~146800640,331350016~75497472] 2024-09-13T00:40:02.483 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to 16 diff [0~25165824,29360128~50331648,83886080~29360128,117440512~50331648,171966464~25165824,201326592~4194304,209715200~20971520,234881024~12582912,251658240~54525952,310378496~25165824,339738624~4194304,348127232~58720256] 2024-09-13T00:40:02.483 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,29360128~50331648,83886080~29360128,117440512~50331648,171966464~25165824,201326592~4194304,209715200~20971520,234881024~12582912,251658240~54525952,310378496~25165824,339738624~4194304,348127232~58720256] 2024-09-13T00:40:02.499 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~167772160,171966464~234881024] 2024-09-13T00:40:02.499 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to 17 diff [0~25165824,29360128~20971520,54525952~25165824,83886080~29360128,117440512~50331648,171966464~25165824,201326592~4194304,209715200~20971520,234881024~71303168,310378496~25165824,339738624~4194304,348127232~58720256] 2024-09-13T00:40:02.499 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,29360128~20971520,54525952~25165824,83886080~29360128,117440512~50331648,171966464~25165824,201326592~4194304,209715200~20971520,234881024~71303168,310378496~25165824,339738624~4194304,348127232~58720256] 2024-09-13T00:40:02.515 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~406847488] 2024-09-13T00:40:02.515 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to 18 diff [0~25165824,29360128~20971520,54525952~25165824,83886080~29360128,117440512~50331648,171966464~25165824,201326592~4194304,209715200~20971520,234881024~12582912,251658240~54525952,310378496~25165824,339738624~4194304,348127232~58720256] 2024-09-13T00:40:02.515 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,29360128~20971520,54525952~25165824,83886080~29360128,117440512~50331648,171966464~25165824,201326592~4194304,209715200~20971520,234881024~12582912,251658240~54525952,310378496~25165824,339738624~4194304,348127232~58720256] 2024-09-13T00:40:02.533 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~406847488] 2024-09-13T00:40:02.533 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to 19 diff [0~50331648,54525952~25165824,83886080~29360128,117440512~50331648,171966464~25165824,201326592~4194304,209715200~37748736,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:02.534 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~50331648,54525952~25165824,83886080~29360128,117440512~50331648,171966464~25165824,201326592~4194304,209715200~37748736,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:02.552 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:02.553 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to 4 diff [4194304~8388608,16777216~4194304,83886080~8388608,218103808~4194304,297795584~12582912,314572800~8388608,377487360~4194304] 2024-09-13T00:40:02.553 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [4194304~8388608,16777216~4194304,83886080~8388608,218103808~4194304,297795584~12582912,314572800~8388608,377487360~4194304] 2024-09-13T00:40:02.556 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [4194304~8388608,16777216~8388608,83886080~8388608,218103808~8388608,297795584~12582912,314572800~12582912,377487360~4194304] 2024-09-13T00:40:02.556 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to 5 diff [4194304~8388608,16777216~4194304,29360128~4194304,54525952~4194304,62914560~8388608,83886080~8388608,218103808~4194304,255852544~4194304,289406976~16777216,314572800~8388608,356515840~8388608,377487360~8388608] 2024-09-13T00:40:02.556 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [4194304~8388608,16777216~4194304,29360128~4194304,54525952~4194304,62914560~8388608,83886080~8388608,218103808~4194304,255852544~4194304,289406976~16777216,314572800~8388608,356515840~8388608,377487360~8388608] 2024-09-13T00:40:02.560 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [4194304~8388608,16777216~8388608,29360128~8388608,54525952~4194304,62914560~12582912,83886080~8388608,218103808~8388608,255852544~8388608,289406976~20971520,314572800~12582912,356515840~8388608,377487360~12582912] 2024-09-13T00:40:02.560 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to 6 diff [4194304~8388608,16777216~4194304,29360128~4194304,41943040~16777216,62914560~8388608,83886080~8388608,96468992~4194304,155189248~8388608,218103808~4194304,239075328~8388608,255852544~4194304,276824064~4194304,289406976~16777216,314572800~8388608,356515840~12582912,377487360~8388608] 2024-09-13T00:40:02.560 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [4194304~8388608,16777216~4194304,29360128~4194304,41943040~16777216,62914560~8388608,83886080~8388608,96468992~4194304,155189248~8388608,218103808~4194304,239075328~8388608,255852544~4194304,276824064~4194304,289406976~16777216,314572800~8388608,356515840~12582912,377487360~8388608] 2024-09-13T00:40:02.565 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [4194304~8388608,16777216~8388608,29360128~8388608,41943040~16777216,62914560~12582912,83886080~8388608,96468992~4194304,155189248~12582912,218103808~8388608,239075328~12582912,255852544~8388608,276824064~8388608,289406976~20971520,314572800~12582912,356515840~16777216,377487360~12582912] 2024-09-13T00:40:02.565 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to 7 diff [0~12582912,16777216~4194304,29360128~4194304,41943040~29360128,83886080~8388608,96468992~4194304,130023424~4194304,150994944~16777216,218103808~4194304,239075328~8388608,255852544~4194304,276824064~4194304,289406976~16777216,314572800~8388608,339738624~4194304,356515840~12582912,377487360~16777216] 2024-09-13T00:40:02.565 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~12582912,16777216~4194304,29360128~4194304,41943040~29360128,83886080~8388608,96468992~4194304,130023424~4194304,150994944~16777216,218103808~4194304,239075328~8388608,255852544~4194304,276824064~4194304,289406976~16777216,314572800~8388608,339738624~4194304,356515840~12582912,377487360~16777216] 2024-09-13T00:40:02.571 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~12582912,16777216~8388608,29360128~8388608,41943040~33554432,83886080~8388608,96468992~4194304,130023424~4194304,150994944~16777216,218103808~8388608,239075328~12582912,255852544~8388608,276824064~8388608,289406976~20971520,314572800~12582912,339738624~8388608,356515840~16777216,377487360~20971520] 2024-09-13T00:40:02.571 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to 8 diff [0~12582912,16777216~4194304,29360128~4194304,41943040~29360128,83886080~8388608,96468992~4194304,109051904~4194304,125829120~8388608,138412032~4194304,146800640~20971520,218103808~4194304,239075328~8388608,255852544~8388608,276824064~29360128,314572800~8388608,339738624~4194304,356515840~12582912,377487360~16777216] 2024-09-13T00:40:02.571 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~12582912,16777216~4194304,29360128~4194304,41943040~29360128,83886080~8388608,96468992~4194304,109051904~4194304,125829120~8388608,138412032~4194304,146800640~20971520,218103808~4194304,239075328~8388608,255852544~8388608,276824064~29360128,314572800~8388608,339738624~4194304,356515840~12582912,377487360~16777216] 2024-09-13T00:40:02.578 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~12582912,16777216~8388608,29360128~8388608,41943040~33554432,83886080~16777216,109051904~8388608,125829120~8388608,138412032~29360128,218103808~8388608,239075328~12582912,255852544~12582912,276824064~33554432,314572800~12582912,339738624~8388608,356515840~16777216,377487360~20971520] 2024-09-13T00:40:02.578 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to 9 diff [0~12582912,16777216~4194304,29360128~4194304,37748736~33554432,75497472~29360128,109051904~4194304,125829120~8388608,138412032~4194304,146800640~20971520,218103808~4194304,239075328~8388608,255852544~8388608,268435456~4194304,276824064~29360128,314572800~8388608,339738624~4194304,356515840~16777216,377487360~16777216] 2024-09-13T00:40:02.578 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~12582912,16777216~4194304,29360128~4194304,37748736~33554432,75497472~29360128,109051904~4194304,125829120~8388608,138412032~4194304,146800640~20971520,218103808~4194304,239075328~8388608,255852544~8388608,268435456~4194304,276824064~29360128,314572800~8388608,339738624~4194304,356515840~16777216,377487360~16777216] 2024-09-13T00:40:02.586 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~12582912,16777216~8388608,29360128~88080384,125829120~8388608,138412032~29360128,218103808~8388608,239075328~12582912,255852544~16777216,276824064~33554432,314572800~12582912,339738624~8388608,356515840~16777216,377487360~20971520] 2024-09-13T00:40:02.586 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to 10 diff [0~12582912,16777216~4194304,29360128~4194304,37748736~33554432,75497472~37748736,125829120~8388608,138412032~4194304,146800640~20971520,171966464~4194304,201326592~4194304,209715200~12582912,239075328~8388608,255852544~50331648,314572800~8388608,335544320~8388608,356515840~16777216,377487360~16777216] 2024-09-13T00:40:02.586 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~12582912,16777216~4194304,29360128~4194304,37748736~33554432,75497472~37748736,125829120~8388608,138412032~4194304,146800640~20971520,171966464~4194304,201326592~4194304,209715200~12582912,239075328~8388608,255852544~50331648,314572800~8388608,335544320~8388608,356515840~16777216,377487360~16777216] 2024-09-13T00:40:02.594 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~12582912,16777216~8388608,29360128~88080384,125829120~8388608,138412032~29360128,171966464~4194304,201326592~25165824,239075328~12582912,255852544~54525952,314572800~12582912,335544320~12582912,356515840~16777216,377487360~20971520] 2024-09-13T00:40:02.595 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to 11 diff [0~20971520,29360128~4194304,37748736~33554432,75497472~37748736,125829120~8388608,138412032~29360128,171966464~4194304,188743680~8388608,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~54525952,314572800~12582912,335544320~8388608,356515840~16777216,377487360~16777216] 2024-09-13T00:40:02.595 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~20971520,29360128~4194304,37748736~33554432,75497472~37748736,125829120~8388608,138412032~29360128,171966464~4194304,188743680~8388608,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~54525952,314572800~12582912,335544320~8388608,356515840~16777216,377487360~16777216] 2024-09-13T00:40:02.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:40:02 smithi181 bash[95505]: ts=2024-09-13T00:40:02.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:40:02.606 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~25165824,29360128~88080384,125829120~8388608,138412032~29360128,171966464~4194304,188743680~121634816,314572800~12582912,335544320~12582912,356515840~16777216,377487360~20971520] 2024-09-13T00:40:02.606 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to 12 diff [0~20971520,29360128~4194304,37748736~33554432,75497472~37748736,117440512~4194304,125829120~8388608,138412032~29360128,171966464~4194304,184549376~12582912,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~54525952,314572800~12582912,331350016~4194304,339738624~4194304,352321536~46137344] 2024-09-13T00:40:02.606 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~20971520,29360128~4194304,37748736~33554432,75497472~37748736,117440512~4194304,125829120~8388608,138412032~29360128,171966464~4194304,184549376~12582912,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~54525952,314572800~12582912,331350016~4194304,339738624~4194304,352321536~46137344] 2024-09-13T00:40:02.618 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~25165824,29360128~104857600,138412032~29360128,171966464~4194304,184549376~125829120,314572800~12582912,331350016~16777216,352321536~46137344] 2024-09-13T00:40:02.618 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to 13 diff [0~20971520,29360128~50331648,83886080~29360128,117440512~16777216,138412032~29360128,171966464~4194304,184549376~12582912,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~54525952,314572800~12582912,331350016~4194304,339738624~4194304,348127232~50331648] 2024-09-13T00:40:02.618 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~20971520,29360128~50331648,83886080~29360128,117440512~16777216,138412032~29360128,171966464~4194304,184549376~12582912,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~54525952,314572800~12582912,331350016~4194304,339738624~4194304,348127232~50331648] 2024-09-13T00:40:02.631 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~25165824,29360128~104857600,138412032~29360128,171966464~4194304,184549376~125829120,314572800~12582912,331350016~67108864] 2024-09-13T00:40:02.631 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to 14 diff [0~20971520,29360128~50331648,83886080~29360128,117440512~50331648,171966464~4194304,180355072~16777216,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~54525952,310378496~16777216,331350016~4194304,339738624~4194304,348127232~50331648] 2024-09-13T00:40:02.632 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~20971520,29360128~50331648,83886080~29360128,117440512~50331648,171966464~4194304,180355072~16777216,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~54525952,310378496~16777216,331350016~4194304,339738624~4194304,348127232~50331648] 2024-09-13T00:40:02.646 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~25165824,29360128~138412032,171966464~4194304,180355072~146800640,331350016~67108864] 2024-09-13T00:40:02.647 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to 15 diff [0~20971520,29360128~50331648,83886080~29360128,117440512~50331648,171966464~4194304,180355072~16777216,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~54525952,310378496~16777216,331350016~4194304,339738624~4194304,348127232~58720256] 2024-09-13T00:40:02.647 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~20971520,29360128~50331648,83886080~29360128,117440512~50331648,171966464~4194304,180355072~16777216,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~54525952,310378496~16777216,331350016~4194304,339738624~4194304,348127232~58720256] 2024-09-13T00:40:02.661 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~25165824,29360128~138412032,171966464~4194304,180355072~146800640,331350016~75497472] 2024-09-13T00:40:02.661 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to 16 diff [0~20971520,29360128~50331648,83886080~29360128,117440512~50331648,171966464~25165824,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~54525952,310378496~25165824,339738624~4194304,348127232~58720256] 2024-09-13T00:40:02.662 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~20971520,29360128~50331648,83886080~29360128,117440512~50331648,171966464~25165824,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~54525952,310378496~25165824,339738624~4194304,348127232~58720256] 2024-09-13T00:40:02.676 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~25165824,29360128~138412032,171966464~234881024] 2024-09-13T00:40:02.677 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to 17 diff [0~25165824,29360128~20971520,54525952~25165824,83886080~29360128,117440512~50331648,171966464~25165824,201326592~4194304,209715200~12582912,226492416~4194304,234881024~71303168,310378496~25165824,339738624~4194304,348127232~58720256] 2024-09-13T00:40:02.677 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,29360128~20971520,54525952~25165824,83886080~29360128,117440512~50331648,171966464~25165824,201326592~4194304,209715200~12582912,226492416~4194304,234881024~71303168,310378496~25165824,339738624~4194304,348127232~58720256] 2024-09-13T00:40:02.694 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~25165824,29360128~377487360] 2024-09-13T00:40:02.694 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to 18 diff [0~25165824,29360128~20971520,54525952~25165824,83886080~29360128,117440512~50331648,171966464~25165824,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~54525952,310378496~25165824,339738624~4194304,348127232~58720256] 2024-09-13T00:40:02.694 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,29360128~20971520,54525952~25165824,83886080~29360128,117440512~50331648,171966464~25165824,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~54525952,310378496~25165824,339738624~4194304,348127232~58720256] 2024-09-13T00:40:02.712 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~25165824,29360128~377487360] 2024-09-13T00:40:02.713 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to 19 diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:02.713 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:02.732 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:02.732 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to 5 diff [29360128~4194304,54525952~4194304,62914560~8388608,83886080~8388608,255852544~4194304,289406976~16777216,356515840~8388608,377487360~8388608] 2024-09-13T00:40:02.733 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [29360128~4194304,54525952~4194304,62914560~8388608,83886080~8388608,255852544~4194304,289406976~16777216,356515840~8388608,377487360~8388608] 2024-09-13T00:40:02.735 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [29360128~8388608,54525952~4194304,62914560~12582912,83886080~8388608,255852544~8388608,289406976~20971520,356515840~8388608,377487360~12582912] 2024-09-13T00:40:02.736 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to 6 diff [29360128~4194304,41943040~16777216,62914560~8388608,83886080~8388608,96468992~4194304,155189248~8388608,239075328~8388608,255852544~4194304,276824064~4194304,289406976~16777216,314572800~8388608,356515840~12582912,377487360~8388608] 2024-09-13T00:40:02.736 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [29360128~4194304,41943040~16777216,62914560~8388608,83886080~8388608,96468992~4194304,155189248~8388608,239075328~8388608,255852544~4194304,276824064~4194304,289406976~16777216,314572800~8388608,356515840~12582912,377487360~8388608] 2024-09-13T00:40:02.740 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [29360128~8388608,41943040~16777216,62914560~12582912,83886080~8388608,96468992~4194304,155189248~12582912,239075328~12582912,255852544~8388608,276824064~8388608,289406976~20971520,314572800~8388608,356515840~16777216,377487360~12582912] 2024-09-13T00:40:02.740 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to 7 diff [0~4194304,29360128~4194304,41943040~29360128,83886080~8388608,96468992~4194304,130023424~4194304,150994944~16777216,239075328~8388608,255852544~4194304,276824064~4194304,289406976~16777216,314572800~8388608,339738624~4194304,356515840~12582912,377487360~16777216] 2024-09-13T00:40:02.740 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~4194304,29360128~4194304,41943040~29360128,83886080~8388608,96468992~4194304,130023424~4194304,150994944~16777216,239075328~8388608,255852544~4194304,276824064~4194304,289406976~16777216,314572800~8388608,339738624~4194304,356515840~12582912,377487360~16777216] 2024-09-13T00:40:02.745 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~8388608,29360128~8388608,41943040~33554432,83886080~8388608,96468992~4194304,130023424~4194304,150994944~16777216,239075328~12582912,255852544~8388608,276824064~8388608,289406976~20971520,314572800~8388608,339738624~8388608,356515840~16777216,377487360~20971520] 2024-09-13T00:40:02.745 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to 8 diff [0~8388608,29360128~4194304,41943040~29360128,83886080~8388608,96468992~4194304,109051904~4194304,125829120~8388608,138412032~4194304,146800640~20971520,239075328~8388608,255852544~8388608,276824064~29360128,314572800~8388608,339738624~4194304,356515840~12582912,377487360~16777216] 2024-09-13T00:40:02.745 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,29360128~4194304,41943040~29360128,83886080~8388608,96468992~4194304,109051904~4194304,125829120~8388608,138412032~4194304,146800640~20971520,239075328~8388608,255852544~8388608,276824064~29360128,314572800~8388608,339738624~4194304,356515840~12582912,377487360~16777216] 2024-09-13T00:40:02.753 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~12582912,29360128~8388608,41943040~33554432,83886080~16777216,109051904~8388608,125829120~8388608,138412032~29360128,239075328~12582912,255852544~12582912,276824064~33554432,314572800~8388608,339738624~8388608,356515840~16777216,377487360~20971520] 2024-09-13T00:40:02.753 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to 9 diff [0~8388608,16777216~4194304,29360128~4194304,37748736~33554432,75497472~29360128,109051904~4194304,125829120~8388608,138412032~4194304,146800640~20971520,239075328~8388608,255852544~8388608,268435456~4194304,276824064~29360128,314572800~8388608,339738624~4194304,356515840~16777216,377487360~16777216] 2024-09-13T00:40:02.753 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,16777216~4194304,29360128~4194304,37748736~33554432,75497472~29360128,109051904~4194304,125829120~8388608,138412032~4194304,146800640~20971520,239075328~8388608,255852544~8388608,268435456~4194304,276824064~29360128,314572800~8388608,339738624~4194304,356515840~16777216,377487360~16777216] 2024-09-13T00:40:02.761 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~12582912,16777216~8388608,29360128~88080384,125829120~8388608,138412032~29360128,239075328~12582912,255852544~16777216,276824064~33554432,314572800~8388608,339738624~8388608,356515840~16777216,377487360~20971520] 2024-09-13T00:40:02.761 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to 10 diff [0~8388608,16777216~4194304,29360128~4194304,37748736~33554432,75497472~37748736,125829120~8388608,138412032~4194304,146800640~20971520,171966464~4194304,201326592~4194304,209715200~8388608,239075328~8388608,255852544~50331648,314572800~8388608,335544320~8388608,356515840~16777216,377487360~16777216] 2024-09-13T00:40:02.761 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,16777216~4194304,29360128~4194304,37748736~33554432,75497472~37748736,125829120~8388608,138412032~4194304,146800640~20971520,171966464~4194304,201326592~4194304,209715200~8388608,239075328~8388608,255852544~50331648,314572800~8388608,335544320~8388608,356515840~16777216,377487360~16777216] 2024-09-13T00:40:02.770 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~12582912,16777216~8388608,29360128~88080384,125829120~8388608,138412032~29360128,171966464~4194304,201326592~20971520,239075328~12582912,255852544~54525952,314572800~8388608,335544320~12582912,356515840~16777216,377487360~20971520] 2024-09-13T00:40:02.770 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to 11 diff [0~8388608,12582912~8388608,29360128~4194304,37748736~33554432,75497472~37748736,125829120~8388608,138412032~29360128,171966464~4194304,188743680~8388608,201326592~4194304,209715200~8388608,226492416~4194304,234881024~12582912,251658240~54525952,314572800~8388608,335544320~8388608,356515840~16777216,377487360~16777216] 2024-09-13T00:40:02.770 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,12582912~8388608,29360128~4194304,37748736~33554432,75497472~37748736,125829120~8388608,138412032~29360128,171966464~4194304,188743680~8388608,201326592~4194304,209715200~8388608,226492416~4194304,234881024~12582912,251658240~54525952,314572800~8388608,335544320~8388608,356515840~16777216,377487360~16777216] 2024-09-13T00:40:02.780 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~25165824,29360128~88080384,125829120~8388608,138412032~29360128,171966464~4194304,188743680~33554432,226492416~83886080,314572800~8388608,335544320~12582912,356515840~16777216,377487360~20971520] 2024-09-13T00:40:02.780 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to 12 diff [0~8388608,12582912~8388608,29360128~4194304,37748736~33554432,75497472~37748736,117440512~4194304,125829120~8388608,138412032~29360128,171966464~4194304,184549376~12582912,201326592~4194304,209715200~8388608,226492416~4194304,234881024~12582912,251658240~54525952,314572800~8388608,331350016~4194304,339738624~4194304,352321536~41943040] 2024-09-13T00:40:02.780 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,12582912~8388608,29360128~4194304,37748736~33554432,75497472~37748736,117440512~4194304,125829120~8388608,138412032~29360128,171966464~4194304,184549376~12582912,201326592~4194304,209715200~8388608,226492416~4194304,234881024~12582912,251658240~54525952,314572800~8388608,331350016~4194304,339738624~4194304,352321536~41943040] 2024-09-13T00:40:02.792 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~25165824,29360128~104857600,138412032~29360128,171966464~4194304,184549376~37748736,226492416~83886080,314572800~8388608,331350016~16777216,352321536~46137344] 2024-09-13T00:40:02.792 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to 13 diff [0~8388608,12582912~8388608,29360128~50331648,83886080~29360128,117440512~16777216,138412032~29360128,171966464~4194304,184549376~12582912,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~54525952,314572800~8388608,331350016~4194304,339738624~4194304,348127232~46137344] 2024-09-13T00:40:02.792 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,12582912~8388608,29360128~50331648,83886080~29360128,117440512~16777216,138412032~29360128,171966464~4194304,184549376~12582912,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~54525952,314572800~8388608,331350016~4194304,339738624~4194304,348127232~46137344] 2024-09-13T00:40:02.804 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~25165824,29360128~104857600,138412032~29360128,171966464~4194304,184549376~37748736,226492416~83886080,314572800~8388608,331350016~67108864] 2024-09-13T00:40:02.804 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to 14 diff [0~8388608,12582912~8388608,29360128~50331648,83886080~29360128,117440512~50331648,171966464~4194304,180355072~16777216,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~54525952,310378496~12582912,331350016~4194304,339738624~4194304,348127232~50331648] 2024-09-13T00:40:02.805 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,12582912~8388608,29360128~50331648,83886080~29360128,117440512~50331648,171966464~4194304,180355072~16777216,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~54525952,310378496~12582912,331350016~4194304,339738624~4194304,348127232~50331648] 2024-09-13T00:40:02.818 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~25165824,29360128~138412032,171966464~4194304,180355072~41943040,226492416~100663296,331350016~67108864] 2024-09-13T00:40:02.818 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to 15 diff [0~8388608,12582912~8388608,29360128~50331648,83886080~29360128,117440512~50331648,171966464~4194304,180355072~16777216,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~54525952,310378496~12582912,331350016~4194304,339738624~4194304,348127232~58720256] 2024-09-13T00:40:02.818 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,12582912~8388608,29360128~50331648,83886080~29360128,117440512~50331648,171966464~4194304,180355072~16777216,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~54525952,310378496~12582912,331350016~4194304,339738624~4194304,348127232~58720256] 2024-09-13T00:40:02.831 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~25165824,29360128~138412032,171966464~4194304,180355072~41943040,226492416~100663296,331350016~75497472] 2024-09-13T00:40:02.832 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to 16 diff [0~8388608,12582912~8388608,29360128~50331648,83886080~29360128,117440512~50331648,171966464~25165824,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~54525952,310378496~25165824,339738624~4194304,348127232~58720256] 2024-09-13T00:40:02.832 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,12582912~8388608,29360128~50331648,83886080~29360128,117440512~50331648,171966464~25165824,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~54525952,310378496~25165824,339738624~4194304,348127232~58720256] 2024-09-13T00:40:02.845 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~25165824,29360128~138412032,171966464~50331648,226492416~180355072] 2024-09-13T00:40:02.846 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to 17 diff [0~8388608,12582912~12582912,29360128~20971520,54525952~25165824,83886080~29360128,117440512~50331648,171966464~25165824,201326592~4194304,209715200~12582912,226492416~4194304,234881024~71303168,310378496~25165824,339738624~4194304,348127232~58720256] 2024-09-13T00:40:02.846 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,12582912~12582912,29360128~20971520,54525952~25165824,83886080~29360128,117440512~50331648,171966464~25165824,201326592~4194304,209715200~12582912,226492416~4194304,234881024~71303168,310378496~25165824,339738624~4194304,348127232~58720256] 2024-09-13T00:40:02.861 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~25165824,29360128~192937984,226492416~180355072] 2024-09-13T00:40:02.861 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to 18 diff [0~8388608,12582912~12582912,29360128~20971520,54525952~25165824,83886080~29360128,117440512~50331648,171966464~25165824,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~54525952,310378496~25165824,339738624~4194304,348127232~58720256] 2024-09-13T00:40:02.861 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,12582912~12582912,29360128~20971520,54525952~25165824,83886080~29360128,117440512~50331648,171966464~25165824,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~54525952,310378496~25165824,339738624~4194304,348127232~58720256] 2024-09-13T00:40:02.878 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~25165824,29360128~192937984,226492416~180355072] 2024-09-13T00:40:02.878 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to 19 diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:02.878 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:02.895 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~222298112,226492416~188743680] 2024-09-13T00:40:02.895 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to 6 diff [41943040~12582912,96468992~4194304,155189248~8388608,239075328~8388608,255852544~4194304,276824064~4194304,314572800~8388608,360710144~8388608] 2024-09-13T00:40:02.896 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [41943040~12582912,96468992~4194304,155189248~8388608,239075328~8388608,255852544~4194304,276824064~4194304,314572800~8388608,360710144~8388608] 2024-09-13T00:40:02.899 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [41943040~12582912,96468992~4194304,155189248~12582912,239075328~12582912,255852544~8388608,276824064~8388608,314572800~8388608,360710144~12582912] 2024-09-13T00:40:02.899 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to 7 diff [0~4194304,41943040~20971520,96468992~4194304,130023424~4194304,150994944~16777216,239075328~8388608,255852544~4194304,276824064~4194304,314572800~8388608,339738624~4194304,360710144~8388608,381681664~12582912] 2024-09-13T00:40:02.899 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~4194304,41943040~20971520,96468992~4194304,130023424~4194304,150994944~16777216,239075328~8388608,255852544~4194304,276824064~4194304,314572800~8388608,339738624~4194304,360710144~8388608,381681664~12582912] 2024-09-13T00:40:02.904 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~8388608,41943040~25165824,96468992~4194304,130023424~4194304,150994944~16777216,239075328~12582912,255852544~8388608,276824064~8388608,314572800~8388608,339738624~8388608,360710144~12582912,381681664~16777216] 2024-09-13T00:40:02.904 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to 8 diff [0~8388608,41943040~20971520,88080384~4194304,96468992~4194304,109051904~4194304,125829120~8388608,138412032~4194304,146800640~20971520,239075328~8388608,255852544~8388608,276824064~12582912,314572800~8388608,339738624~4194304,356515840~12582912,381681664~12582912] 2024-09-13T00:40:02.904 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,41943040~20971520,88080384~4194304,96468992~4194304,109051904~4194304,125829120~8388608,138412032~4194304,146800640~20971520,239075328~8388608,255852544~8388608,276824064~12582912,314572800~8388608,339738624~4194304,356515840~12582912,381681664~12582912] 2024-09-13T00:40:02.909 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~12582912,41943040~25165824,88080384~12582912,109051904~8388608,125829120~8388608,138412032~29360128,239075328~12582912,255852544~12582912,276824064~16777216,314572800~8388608,339738624~8388608,356515840~16777216,381681664~16777216] 2024-09-13T00:40:02.909 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to 9 diff [0~8388608,16777216~4194304,37748736~25165824,75497472~8388608,88080384~16777216,109051904~4194304,125829120~8388608,138412032~4194304,146800640~20971520,239075328~8388608,255852544~8388608,268435456~4194304,276824064~20971520,314572800~8388608,339738624~4194304,356515840~12582912,381681664~12582912] 2024-09-13T00:40:02.910 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,16777216~4194304,37748736~25165824,75497472~8388608,88080384~16777216,109051904~4194304,125829120~8388608,138412032~4194304,146800640~20971520,239075328~8388608,255852544~8388608,268435456~4194304,276824064~20971520,314572800~8388608,339738624~4194304,356515840~12582912,381681664~12582912] 2024-09-13T00:40:02.916 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~12582912,16777216~8388608,37748736~29360128,75497472~8388608,88080384~29360128,125829120~8388608,138412032~29360128,239075328~12582912,255852544~16777216,276824064~20971520,314572800~8388608,339738624~8388608,356515840~16777216,381681664~16777216] 2024-09-13T00:40:02.916 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to 10 diff [0~8388608,16777216~4194304,37748736~25165824,75497472~8388608,88080384~25165824,125829120~8388608,138412032~4194304,146800640~20971520,171966464~4194304,201326592~4194304,209715200~8388608,239075328~8388608,255852544~37748736,314572800~8388608,335544320~8388608,356515840~12582912,381681664~12582912] 2024-09-13T00:40:02.916 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,16777216~4194304,37748736~25165824,75497472~8388608,88080384~25165824,125829120~8388608,138412032~4194304,146800640~20971520,171966464~4194304,201326592~4194304,209715200~8388608,239075328~8388608,255852544~37748736,314572800~8388608,335544320~8388608,356515840~12582912,381681664~12582912] 2024-09-13T00:40:02.923 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~12582912,16777216~8388608,37748736~29360128,75497472~8388608,88080384~29360128,125829120~8388608,138412032~29360128,171966464~4194304,201326592~20971520,239075328~12582912,255852544~41943040,314572800~8388608,335544320~12582912,356515840~16777216,381681664~16777216] 2024-09-13T00:40:02.923 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to 11 diff [0~8388608,12582912~8388608,37748736~25165824,75497472~8388608,88080384~25165824,125829120~8388608,138412032~29360128,171966464~4194304,188743680~8388608,201326592~4194304,209715200~8388608,226492416~4194304,234881024~12582912,251658240~41943040,314572800~8388608,335544320~8388608,356515840~12582912,381681664~12582912] 2024-09-13T00:40:02.923 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,12582912~8388608,37748736~25165824,75497472~8388608,88080384~25165824,125829120~8388608,138412032~29360128,171966464~4194304,188743680~8388608,201326592~4194304,209715200~8388608,226492416~4194304,234881024~12582912,251658240~41943040,314572800~8388608,335544320~8388608,356515840~12582912,381681664~12582912] 2024-09-13T00:40:02.932 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~25165824,37748736~29360128,75497472~8388608,88080384~29360128,125829120~8388608,138412032~29360128,171966464~4194304,188743680~33554432,226492416~71303168,314572800~8388608,335544320~12582912,356515840~16777216,381681664~16777216] 2024-09-13T00:40:02.932 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to 12 diff [0~8388608,12582912~8388608,37748736~25165824,67108864~4194304,75497472~8388608,88080384~25165824,117440512~4194304,125829120~8388608,138412032~29360128,171966464~4194304,184549376~12582912,201326592~4194304,209715200~8388608,226492416~4194304,234881024~12582912,251658240~41943040,314572800~8388608,331350016~4194304,339738624~4194304,352321536~41943040] 2024-09-13T00:40:02.932 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,12582912~8388608,37748736~25165824,67108864~4194304,75497472~8388608,88080384~25165824,117440512~4194304,125829120~8388608,138412032~29360128,171966464~4194304,184549376~12582912,201326592~4194304,209715200~8388608,226492416~4194304,234881024~12582912,251658240~41943040,314572800~8388608,331350016~4194304,339738624~4194304,352321536~41943040] 2024-09-13T00:40:02.942 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~25165824,37748736~46137344,88080384~46137344,138412032~29360128,171966464~4194304,184549376~37748736,226492416~71303168,314572800~8388608,331350016~16777216,352321536~46137344] 2024-09-13T00:40:02.942 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to 13 diff [0~8388608,12582912~8388608,33554432~29360128,67108864~12582912,88080384~25165824,117440512~16777216,138412032~29360128,171966464~4194304,184549376~12582912,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~41943040,314572800~8388608,331350016~4194304,339738624~4194304,348127232~46137344] 2024-09-13T00:40:02.942 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,12582912~8388608,33554432~29360128,67108864~12582912,88080384~25165824,117440512~16777216,138412032~29360128,171966464~4194304,184549376~12582912,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~41943040,314572800~8388608,331350016~4194304,339738624~4194304,348127232~46137344] 2024-09-13T00:40:02.953 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~25165824,33554432~50331648,88080384~46137344,138412032~29360128,171966464~4194304,184549376~37748736,226492416~71303168,314572800~8388608,331350016~67108864] 2024-09-13T00:40:02.953 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to 14 diff [0~8388608,12582912~8388608,33554432~29360128,67108864~12582912,88080384~25165824,117440512~50331648,171966464~4194304,180355072~16777216,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~50331648,310378496~12582912,331350016~4194304,339738624~4194304,348127232~50331648] 2024-09-13T00:40:02.954 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,12582912~8388608,33554432~29360128,67108864~12582912,88080384~25165824,117440512~50331648,171966464~4194304,180355072~16777216,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~50331648,310378496~12582912,331350016~4194304,339738624~4194304,348127232~50331648] 2024-09-13T00:40:02.965 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~25165824,33554432~50331648,88080384~79691776,171966464~4194304,180355072~41943040,226492416~79691776,310378496~16777216,331350016~67108864] 2024-09-13T00:40:02.965 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to 15 diff [0~8388608,12582912~8388608,33554432~29360128,67108864~12582912,88080384~25165824,117440512~50331648,171966464~4194304,180355072~16777216,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~50331648,310378496~12582912,331350016~4194304,339738624~4194304,348127232~58720256] 2024-09-13T00:40:02.965 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,12582912~8388608,33554432~29360128,67108864~12582912,88080384~25165824,117440512~50331648,171966464~4194304,180355072~16777216,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~50331648,310378496~12582912,331350016~4194304,339738624~4194304,348127232~58720256] 2024-09-13T00:40:02.977 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~25165824,33554432~50331648,88080384~79691776,171966464~4194304,180355072~41943040,226492416~79691776,310378496~16777216,331350016~75497472] 2024-09-13T00:40:02.978 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to 16 diff [0~8388608,12582912~8388608,33554432~29360128,67108864~12582912,88080384~25165824,117440512~50331648,171966464~25165824,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~50331648,310378496~25165824,339738624~4194304,348127232~58720256] 2024-09-13T00:40:02.978 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,12582912~8388608,33554432~29360128,67108864~12582912,88080384~25165824,117440512~50331648,171966464~25165824,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~50331648,310378496~25165824,339738624~4194304,348127232~58720256] 2024-09-13T00:40:02.990 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~25165824,33554432~50331648,88080384~79691776,171966464~50331648,226492416~79691776,310378496~96468992] 2024-09-13T00:40:02.990 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to 17 diff [0~8388608,12582912~12582912,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~50331648,171966464~25165824,201326592~4194304,209715200~12582912,226492416~4194304,234881024~67108864,310378496~25165824,339738624~4194304,348127232~58720256] 2024-09-13T00:40:02.990 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,12582912~12582912,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~50331648,171966464~25165824,201326592~4194304,209715200~12582912,226492416~4194304,234881024~67108864,310378496~25165824,339738624~4194304,348127232~58720256] 2024-09-13T00:40:03.004 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~25165824,33554432~50331648,88080384~134217728,226492416~79691776,310378496~96468992] 2024-09-13T00:40:03.004 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to 18 diff [0~8388608,12582912~12582912,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~50331648,171966464~25165824,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~50331648,310378496~25165824,339738624~4194304,348127232~58720256] 2024-09-13T00:40:03.004 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,12582912~12582912,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~50331648,171966464~25165824,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~50331648,310378496~25165824,339738624~4194304,348127232~58720256] 2024-09-13T00:40:03.018 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~25165824,33554432~50331648,88080384~134217728,226492416~79691776,310378496~96468992] 2024-09-13T00:40:03.018 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to 19 diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:03.018 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:03.033 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~50331648,88080384~134217728,226492416~188743680] 2024-09-13T00:40:03.033 INFO:tasks.workunit.client.0.smithi136.stdout:from 6 to 7 diff [0~4194304,54525952~8388608,130023424~4194304,150994944~4194304,159383552~8388608,339738624~4194304,381681664~12582912] 2024-09-13T00:40:03.033 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~4194304,54525952~8388608,130023424~4194304,150994944~4194304,159383552~8388608,339738624~4194304,381681664~12582912] 2024-09-13T00:40:03.036 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~8388608,54525952~12582912,130023424~4194304,150994944~16777216,339738624~8388608,381681664~16777216] 2024-09-13T00:40:03.036 INFO:tasks.workunit.client.0.smithi136.stdout:from 6 to 8 diff [0~8388608,54525952~8388608,88080384~4194304,109051904~4194304,125829120~8388608,138412032~4194304,146800640~8388608,159383552~8388608,260046848~4194304,281018368~8388608,339738624~4194304,356515840~4194304,381681664~12582912] 2024-09-13T00:40:03.036 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,54525952~8388608,88080384~4194304,109051904~4194304,125829120~8388608,138412032~4194304,146800640~8388608,159383552~8388608,260046848~4194304,281018368~8388608,339738624~4194304,356515840~4194304,381681664~12582912] 2024-09-13T00:40:03.041 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~12582912,54525952~12582912,88080384~8388608,109051904~8388608,125829120~8388608,138412032~29360128,260046848~8388608,281018368~12582912,339738624~8388608,356515840~8388608,381681664~16777216] 2024-09-13T00:40:03.041 INFO:tasks.workunit.client.0.smithi136.stdout:from 6 to 9 diff [0~8388608,16777216~4194304,37748736~8388608,54525952~8388608,75497472~8388608,88080384~16777216,109051904~4194304,125829120~8388608,138412032~4194304,146800640~8388608,159383552~8388608,255852544~8388608,268435456~4194304,281018368~16777216,339738624~4194304,356515840~8388608,381681664~12582912] 2024-09-13T00:40:03.041 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,16777216~4194304,37748736~8388608,54525952~8388608,75497472~8388608,88080384~16777216,109051904~4194304,125829120~8388608,138412032~4194304,146800640~8388608,159383552~8388608,255852544~8388608,268435456~4194304,281018368~16777216,339738624~4194304,356515840~8388608,381681664~12582912] 2024-09-13T00:40:03.046 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~12582912,16777216~8388608,37748736~8388608,54525952~12582912,75497472~8388608,88080384~29360128,125829120~8388608,138412032~29360128,255852544~16777216,281018368~16777216,339738624~8388608,356515840~12582912,381681664~16777216] 2024-09-13T00:40:03.046 INFO:tasks.workunit.client.0.smithi136.stdout:from 6 to 10 diff [0~8388608,16777216~4194304,37748736~8388608,54525952~8388608,75497472~8388608,88080384~25165824,125829120~8388608,138412032~4194304,146800640~20971520,171966464~4194304,201326592~4194304,209715200~8388608,255852544~20971520,281018368~12582912,335544320~8388608,356515840~8388608,381681664~12582912] 2024-09-13T00:40:03.046 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,16777216~4194304,37748736~8388608,54525952~8388608,75497472~8388608,88080384~25165824,125829120~8388608,138412032~4194304,146800640~20971520,171966464~4194304,201326592~4194304,209715200~8388608,255852544~20971520,281018368~12582912,335544320~8388608,356515840~8388608,381681664~12582912] 2024-09-13T00:40:03.052 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~12582912,16777216~8388608,37748736~8388608,54525952~12582912,75497472~8388608,88080384~29360128,125829120~8388608,138412032~29360128,171966464~4194304,201326592~20971520,255852544~41943040,335544320~12582912,356515840~12582912,381681664~16777216] 2024-09-13T00:40:03.052 INFO:tasks.workunit.client.0.smithi136.stdout:from 6 to 11 diff [0~8388608,12582912~8388608,37748736~8388608,54525952~8388608,75497472~8388608,88080384~25165824,125829120~8388608,138412032~29360128,171966464~4194304,188743680~8388608,201326592~4194304,209715200~8388608,226492416~4194304,234881024~4194304,251658240~25165824,281018368~12582912,318767104~4194304,335544320~8388608,356515840~8388608,381681664~12582912] 2024-09-13T00:40:03.052 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,12582912~8388608,37748736~8388608,54525952~8388608,75497472~8388608,88080384~25165824,125829120~8388608,138412032~29360128,171966464~4194304,188743680~8388608,201326592~4194304,209715200~8388608,226492416~4194304,234881024~4194304,251658240~25165824,281018368~12582912,318767104~4194304,335544320~8388608,356515840~8388608,381681664~12582912] 2024-09-13T00:40:03.060 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~25165824,37748736~8388608,54525952~12582912,75497472~8388608,88080384~29360128,125829120~8388608,138412032~29360128,171966464~4194304,188743680~33554432,226492416~12582912,251658240~46137344,318767104~4194304,335544320~12582912,356515840~12582912,381681664~16777216] 2024-09-13T00:40:03.060 INFO:tasks.workunit.client.0.smithi136.stdout:from 6 to 12 diff [0~8388608,12582912~8388608,37748736~8388608,54525952~8388608,67108864~4194304,75497472~8388608,88080384~25165824,117440512~4194304,125829120~8388608,138412032~29360128,171966464~4194304,184549376~12582912,201326592~4194304,209715200~8388608,226492416~4194304,234881024~4194304,251658240~25165824,281018368~12582912,314572800~8388608,331350016~4194304,339738624~4194304,352321536~12582912,369098752~25165824] 2024-09-13T00:40:03.060 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,12582912~8388608,37748736~8388608,54525952~8388608,67108864~4194304,75497472~8388608,88080384~25165824,117440512~4194304,125829120~8388608,138412032~29360128,171966464~4194304,184549376~12582912,201326592~4194304,209715200~8388608,226492416~4194304,234881024~4194304,251658240~25165824,281018368~12582912,314572800~8388608,331350016~4194304,339738624~4194304,352321536~12582912,369098752~25165824] 2024-09-13T00:40:03.068 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~25165824,37748736~8388608,54525952~29360128,88080384~46137344,138412032~29360128,171966464~4194304,184549376~37748736,226492416~12582912,251658240~46137344,314572800~8388608,331350016~16777216,352321536~46137344] 2024-09-13T00:40:03.068 INFO:tasks.workunit.client.0.smithi136.stdout:from 6 to 13 diff [0~8388608,12582912~8388608,33554432~8388608,54525952~8388608,67108864~12582912,88080384~25165824,117440512~16777216,138412032~29360128,171966464~4194304,184549376~12582912,201326592~4194304,209715200~12582912,226492416~4194304,234881024~4194304,251658240~25165824,281018368~12582912,314572800~8388608,331350016~4194304,339738624~4194304,348127232~16777216,369098752~25165824] 2024-09-13T00:40:03.069 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,12582912~8388608,33554432~8388608,54525952~8388608,67108864~12582912,88080384~25165824,117440512~16777216,138412032~29360128,171966464~4194304,184549376~12582912,201326592~4194304,209715200~12582912,226492416~4194304,234881024~4194304,251658240~25165824,281018368~12582912,314572800~8388608,331350016~4194304,339738624~4194304,348127232~16777216,369098752~25165824] 2024-09-13T00:40:03.077 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~25165824,33554432~12582912,54525952~29360128,88080384~46137344,138412032~29360128,171966464~4194304,184549376~37748736,226492416~12582912,251658240~46137344,314572800~8388608,331350016~67108864] 2024-09-13T00:40:03.078 INFO:tasks.workunit.client.0.smithi136.stdout:from 6 to 14 diff [0~8388608,12582912~8388608,33554432~8388608,54525952~8388608,67108864~12582912,88080384~25165824,117440512~50331648,171966464~4194304,180355072~16777216,201326592~4194304,209715200~12582912,226492416~4194304,234881024~4194304,251658240~25165824,281018368~20971520,310378496~12582912,331350016~4194304,339738624~4194304,348127232~16777216,369098752~29360128] 2024-09-13T00:40:03.078 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,12582912~8388608,33554432~8388608,54525952~8388608,67108864~12582912,88080384~25165824,117440512~50331648,171966464~4194304,180355072~16777216,201326592~4194304,209715200~12582912,226492416~4194304,234881024~4194304,251658240~25165824,281018368~20971520,310378496~12582912,331350016~4194304,339738624~4194304,348127232~16777216,369098752~29360128] 2024-09-13T00:40:03.088 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~25165824,33554432~12582912,54525952~29360128,88080384~79691776,171966464~4194304,180355072~41943040,226492416~12582912,251658240~54525952,310378496~16777216,331350016~67108864] 2024-09-13T00:40:03.088 INFO:tasks.workunit.client.0.smithi136.stdout:from 6 to 15 diff [0~8388608,12582912~8388608,33554432~8388608,54525952~8388608,67108864~12582912,88080384~25165824,117440512~50331648,171966464~4194304,180355072~16777216,201326592~4194304,209715200~12582912,226492416~4194304,234881024~4194304,251658240~25165824,281018368~20971520,310378496~12582912,331350016~4194304,339738624~4194304,348127232~58720256] 2024-09-13T00:40:03.088 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,12582912~8388608,33554432~8388608,54525952~8388608,67108864~12582912,88080384~25165824,117440512~50331648,171966464~4194304,180355072~16777216,201326592~4194304,209715200~12582912,226492416~4194304,234881024~4194304,251658240~25165824,281018368~20971520,310378496~12582912,331350016~4194304,339738624~4194304,348127232~58720256] 2024-09-13T00:40:03.099 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~25165824,33554432~12582912,54525952~29360128,88080384~79691776,171966464~4194304,180355072~41943040,226492416~12582912,251658240~54525952,310378496~16777216,331350016~75497472] 2024-09-13T00:40:03.099 INFO:tasks.workunit.client.0.smithi136.stdout:from 6 to 16 diff [0~8388608,12582912~8388608,33554432~8388608,54525952~8388608,67108864~12582912,88080384~25165824,117440512~50331648,171966464~25165824,201326592~4194304,209715200~12582912,226492416~4194304,234881024~4194304,251658240~25165824,281018368~20971520,310378496~25165824,339738624~4194304,348127232~58720256] 2024-09-13T00:40:03.099 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,12582912~8388608,33554432~8388608,54525952~8388608,67108864~12582912,88080384~25165824,117440512~50331648,171966464~25165824,201326592~4194304,209715200~12582912,226492416~4194304,234881024~4194304,251658240~25165824,281018368~20971520,310378496~25165824,339738624~4194304,348127232~58720256] 2024-09-13T00:40:03.112 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~25165824,33554432~12582912,54525952~29360128,88080384~79691776,171966464~50331648,226492416~12582912,251658240~54525952,310378496~96468992] 2024-09-13T00:40:03.112 INFO:tasks.workunit.client.0.smithi136.stdout:from 6 to 17 diff [0~8388608,12582912~12582912,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~50331648,171966464~25165824,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~25165824,281018368~20971520,310378496~25165824,339738624~4194304,348127232~58720256] 2024-09-13T00:40:03.112 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,12582912~12582912,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~50331648,171966464~25165824,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~25165824,281018368~20971520,310378496~25165824,339738624~4194304,348127232~58720256] 2024-09-13T00:40:03.125 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~25165824,33554432~16777216,54525952~29360128,88080384~134217728,226492416~20971520,251658240~54525952,310378496~96468992] 2024-09-13T00:40:03.125 INFO:tasks.workunit.client.0.smithi136.stdout:from 6 to 18 diff [0~8388608,12582912~12582912,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~50331648,171966464~25165824,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~25165824,281018368~20971520,310378496~25165824,339738624~4194304,348127232~58720256] 2024-09-13T00:40:03.125 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,12582912~12582912,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~50331648,171966464~25165824,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~25165824,281018368~20971520,310378496~25165824,339738624~4194304,348127232~58720256] 2024-09-13T00:40:03.140 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~25165824,33554432~50331648,88080384~134217728,226492416~20971520,251658240~54525952,310378496~96468992] 2024-09-13T00:40:03.140 INFO:tasks.workunit.client.0.smithi136.stdout:from 6 to 19 diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:03.140 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:03.155 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~50331648,88080384~134217728,226492416~20971520,251658240~163577856] 2024-09-13T00:40:03.156 INFO:tasks.workunit.client.0.smithi136.stdout:from 7 to 8 diff [4194304~4194304,58720256~4194304,88080384~4194304,109051904~4194304,125829120~4194304,138412032~4194304,146800640~8388608,260046848~4194304,281018368~8388608,356515840~4194304] 2024-09-13T00:40:03.156 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [4194304~4194304,58720256~4194304,88080384~4194304,109051904~4194304,125829120~4194304,138412032~4194304,146800640~8388608,260046848~4194304,281018368~8388608,356515840~4194304] 2024-09-13T00:40:03.159 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [4194304~8388608,58720256~8388608,88080384~8388608,109051904~8388608,125829120~8388608,138412032~16777216,260046848~8388608,281018368~12582912,356515840~8388608] 2024-09-13T00:40:03.159 INFO:tasks.workunit.client.0.smithi136.stdout:from 7 to 9 diff [0~8388608,16777216~4194304,37748736~8388608,58720256~4194304,75497472~8388608,88080384~16777216,109051904~4194304,125829120~4194304,138412032~4194304,146800640~8388608,255852544~8388608,268435456~4194304,281018368~16777216,356515840~8388608] 2024-09-13T00:40:03.159 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,16777216~4194304,37748736~8388608,58720256~4194304,75497472~8388608,88080384~16777216,109051904~4194304,125829120~4194304,138412032~4194304,146800640~8388608,255852544~8388608,268435456~4194304,281018368~16777216,356515840~8388608] 2024-09-13T00:40:03.163 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~12582912,16777216~8388608,37748736~8388608,58720256~8388608,75497472~8388608,88080384~29360128,125829120~8388608,138412032~16777216,255852544~16777216,281018368~16777216,356515840~12582912] 2024-09-13T00:40:03.163 INFO:tasks.workunit.client.0.smithi136.stdout:from 7 to 10 diff [0~8388608,16777216~4194304,37748736~8388608,58720256~4194304,75497472~8388608,88080384~25165824,125829120~4194304,138412032~4194304,146800640~12582912,171966464~4194304,201326592~4194304,209715200~8388608,255852544~20971520,281018368~12582912,335544320~4194304,356515840~8388608] 2024-09-13T00:40:03.164 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,16777216~4194304,37748736~8388608,58720256~4194304,75497472~8388608,88080384~25165824,125829120~4194304,138412032~4194304,146800640~12582912,171966464~4194304,201326592~4194304,209715200~8388608,255852544~20971520,281018368~12582912,335544320~4194304,356515840~8388608] 2024-09-13T00:40:03.168 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~12582912,16777216~8388608,37748736~8388608,58720256~8388608,75497472~8388608,88080384~29360128,125829120~8388608,138412032~20971520,171966464~4194304,201326592~20971520,255852544~41943040,335544320~4194304,356515840~12582912] 2024-09-13T00:40:03.168 INFO:tasks.workunit.client.0.smithi136.stdout:from 7 to 11 diff [0~8388608,12582912~8388608,37748736~8388608,58720256~4194304,75497472~8388608,88080384~25165824,125829120~4194304,138412032~20971520,171966464~4194304,188743680~8388608,201326592~4194304,209715200~8388608,226492416~4194304,234881024~4194304,251658240~25165824,281018368~12582912,318767104~4194304,335544320~4194304,356515840~8388608] 2024-09-13T00:40:03.169 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,12582912~8388608,37748736~8388608,58720256~4194304,75497472~8388608,88080384~25165824,125829120~4194304,138412032~20971520,171966464~4194304,188743680~8388608,201326592~4194304,209715200~8388608,226492416~4194304,234881024~4194304,251658240~25165824,281018368~12582912,318767104~4194304,335544320~4194304,356515840~8388608] 2024-09-13T00:40:03.175 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~25165824,37748736~8388608,58720256~8388608,75497472~8388608,88080384~29360128,125829120~8388608,138412032~20971520,171966464~4194304,188743680~33554432,226492416~12582912,251658240~46137344,318767104~4194304,335544320~4194304,356515840~12582912] 2024-09-13T00:40:03.175 INFO:tasks.workunit.client.0.smithi136.stdout:from 7 to 12 diff [0~8388608,12582912~8388608,37748736~8388608,58720256~4194304,67108864~4194304,75497472~8388608,88080384~25165824,117440512~4194304,125829120~4194304,138412032~20971520,171966464~4194304,184549376~12582912,201326592~4194304,209715200~8388608,226492416~4194304,234881024~4194304,251658240~25165824,281018368~12582912,314572800~8388608,331350016~4194304,352321536~12582912,369098752~12582912] 2024-09-13T00:40:03.175 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,12582912~8388608,37748736~8388608,58720256~4194304,67108864~4194304,75497472~8388608,88080384~25165824,117440512~4194304,125829120~4194304,138412032~20971520,171966464~4194304,184549376~12582912,201326592~4194304,209715200~8388608,226492416~4194304,234881024~4194304,251658240~25165824,281018368~12582912,314572800~8388608,331350016~4194304,352321536~12582912,369098752~12582912] 2024-09-13T00:40:03.183 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~25165824,37748736~8388608,58720256~25165824,88080384~46137344,138412032~20971520,171966464~4194304,184549376~37748736,226492416~12582912,251658240~46137344,314572800~8388608,331350016~8388608,352321536~33554432] 2024-09-13T00:40:03.183 INFO:tasks.workunit.client.0.smithi136.stdout:from 7 to 13 diff [0~8388608,12582912~8388608,33554432~8388608,58720256~4194304,67108864~12582912,88080384~25165824,117440512~12582912,138412032~20971520,171966464~4194304,184549376~12582912,201326592~4194304,209715200~12582912,226492416~4194304,234881024~4194304,251658240~25165824,281018368~12582912,314572800~8388608,331350016~4194304,348127232~16777216,369098752~12582912,385875968~8388608] 2024-09-13T00:40:03.183 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,12582912~8388608,33554432~8388608,58720256~4194304,67108864~12582912,88080384~25165824,117440512~12582912,138412032~20971520,171966464~4194304,184549376~12582912,201326592~4194304,209715200~12582912,226492416~4194304,234881024~4194304,251658240~25165824,281018368~12582912,314572800~8388608,331350016~4194304,348127232~16777216,369098752~12582912,385875968~8388608] 2024-09-13T00:40:03.192 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~25165824,33554432~12582912,58720256~25165824,88080384~46137344,138412032~20971520,171966464~4194304,184549376~37748736,226492416~12582912,251658240~46137344,314572800~8388608,331350016~8388608,348127232~46137344] 2024-09-13T00:40:03.192 INFO:tasks.workunit.client.0.smithi136.stdout:from 7 to 14 diff [0~8388608,12582912~8388608,33554432~8388608,58720256~4194304,67108864~12582912,88080384~25165824,117440512~12582912,134217728~25165824,171966464~4194304,180355072~16777216,201326592~4194304,209715200~12582912,226492416~4194304,234881024~4194304,251658240~25165824,281018368~20971520,310378496~12582912,331350016~4194304,348127232~16777216,369098752~25165824] 2024-09-13T00:40:03.192 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,12582912~8388608,33554432~8388608,58720256~4194304,67108864~12582912,88080384~25165824,117440512~12582912,134217728~25165824,171966464~4194304,180355072~16777216,201326592~4194304,209715200~12582912,226492416~4194304,234881024~4194304,251658240~25165824,281018368~20971520,310378496~12582912,331350016~4194304,348127232~16777216,369098752~25165824] 2024-09-13T00:40:03.201 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~25165824,33554432~12582912,58720256~25165824,88080384~75497472,171966464~4194304,180355072~41943040,226492416~12582912,251658240~54525952,310378496~16777216,331350016~8388608,348127232~46137344] 2024-09-13T00:40:03.201 INFO:tasks.workunit.client.0.smithi136.stdout:from 7 to 15 diff [0~8388608,12582912~8388608,33554432~8388608,58720256~4194304,67108864~12582912,88080384~25165824,117440512~12582912,134217728~25165824,171966464~4194304,180355072~16777216,201326592~4194304,209715200~12582912,226492416~4194304,234881024~4194304,251658240~25165824,281018368~20971520,310378496~12582912,331350016~4194304,348127232~46137344,398458880~8388608] 2024-09-13T00:40:03.201 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,12582912~8388608,33554432~8388608,58720256~4194304,67108864~12582912,88080384~25165824,117440512~12582912,134217728~25165824,171966464~4194304,180355072~16777216,201326592~4194304,209715200~12582912,226492416~4194304,234881024~4194304,251658240~25165824,281018368~20971520,310378496~12582912,331350016~4194304,348127232~46137344,398458880~8388608] 2024-09-13T00:40:03.211 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~25165824,33554432~12582912,58720256~25165824,88080384~75497472,171966464~4194304,180355072~41943040,226492416~12582912,251658240~54525952,310378496~16777216,331350016~8388608,348127232~46137344,398458880~8388608] 2024-09-13T00:40:03.212 INFO:tasks.workunit.client.0.smithi136.stdout:from 7 to 16 diff [0~8388608,12582912~8388608,33554432~8388608,58720256~4194304,67108864~12582912,88080384~25165824,117440512~12582912,134217728~29360128,171966464~25165824,201326592~4194304,209715200~12582912,226492416~4194304,234881024~4194304,251658240~25165824,281018368~20971520,310378496~25165824,348127232~46137344,398458880~8388608] 2024-09-13T00:40:03.212 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,12582912~8388608,33554432~8388608,58720256~4194304,67108864~12582912,88080384~25165824,117440512~12582912,134217728~29360128,171966464~25165824,201326592~4194304,209715200~12582912,226492416~4194304,234881024~4194304,251658240~25165824,281018368~20971520,310378496~25165824,348127232~46137344,398458880~8388608] 2024-09-13T00:40:03.223 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~25165824,33554432~12582912,58720256~25165824,88080384~79691776,171966464~50331648,226492416~12582912,251658240~54525952,310378496~29360128,348127232~46137344,398458880~8388608] 2024-09-13T00:40:03.223 INFO:tasks.workunit.client.0.smithi136.stdout:from 7 to 17 diff [0~8388608,12582912~12582912,33554432~16777216,58720256~4194304,67108864~12582912,88080384~25165824,117440512~12582912,134217728~62914560,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~25165824,281018368~20971520,310378496~25165824,339738624~4194304,348127232~46137344,398458880~8388608] 2024-09-13T00:40:03.223 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,12582912~12582912,33554432~16777216,58720256~4194304,67108864~12582912,88080384~25165824,117440512~12582912,134217728~62914560,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~25165824,281018368~20971520,310378496~25165824,339738624~4194304,348127232~46137344,398458880~8388608] 2024-09-13T00:40:03.236 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~25165824,33554432~16777216,58720256~25165824,88080384~134217728,226492416~20971520,251658240~54525952,310378496~33554432,348127232~46137344,398458880~8388608] 2024-09-13T00:40:03.236 INFO:tasks.workunit.client.0.smithi136.stdout:from 7 to 18 diff [0~8388608,12582912~12582912,33554432~16777216,58720256~4194304,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~25165824,281018368~20971520,310378496~25165824,339738624~4194304,348127232~46137344,398458880~8388608] 2024-09-13T00:40:03.237 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,12582912~12582912,33554432~16777216,58720256~4194304,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~25165824,281018368~20971520,310378496~25165824,339738624~4194304,348127232~46137344,398458880~8388608] 2024-09-13T00:40:03.249 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~25165824,33554432~20971520,58720256~25165824,88080384~134217728,226492416~20971520,251658240~54525952,310378496~33554432,348127232~46137344,398458880~8388608] 2024-09-13T00:40:03.249 INFO:tasks.workunit.client.0.smithi136.stdout:from 7 to 19 diff [0~29360128,33554432~16777216,58720256~4194304,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~46137344,398458880~12582912] 2024-09-13T00:40:03.249 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,58720256~4194304,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~46137344,398458880~12582912] 2024-09-13T00:40:03.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:02 smithi136 bash[75164]: cluster 2024-09-13T00:40:01.893424+0000 mon.a (mon.0) 3641 : cluster [DBG] osdmap e1979: 8 total, 8 up, 8 in 2024-09-13T00:40:03.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:02 smithi136 bash[75164]: cluster 2024-09-13T00:40:02.445735+0000 mgr.y (mgr.24661) 1887 : cluster [DBG] pgmap v3427: 137 pgs: 137 active+clean; 792 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 239 KiB/s rd, 37 MiB/s wr, 366 op/s 2024-09-13T00:40:03.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:02 smithi136 bash[76371]: cluster 2024-09-13T00:40:01.893424+0000 mon.a (mon.0) 3641 : cluster [DBG] osdmap e1979: 8 total, 8 up, 8 in 2024-09-13T00:40:03.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:02 smithi136 bash[76371]: cluster 2024-09-13T00:40:02.445735+0000 mgr.y (mgr.24661) 1887 : cluster [DBG] pgmap v3427: 137 pgs: 137 active+clean; 792 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 239 KiB/s rd, 37 MiB/s wr, 366 op/s 2024-09-13T00:40:03.263 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,58720256~25165824,88080384~134217728,226492416~20971520,251658240~92274688,348127232~46137344,398458880~16777216] 2024-09-13T00:40:03.263 INFO:tasks.workunit.client.0.smithi136.stdout:from 8 to 9 diff [0~8388608,16777216~4194304,37748736~8388608,75497472~8388608,92274688~12582912,255852544~4194304,268435456~4194304,289406976~8388608,360710144~4194304] 2024-09-13T00:40:03.263 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,16777216~4194304,37748736~8388608,75497472~8388608,92274688~12582912,255852544~4194304,268435456~4194304,289406976~8388608,360710144~4194304] 2024-09-13T00:40:03.267 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~8388608,16777216~8388608,37748736~8388608,75497472~8388608,92274688~16777216,255852544~8388608,268435456~4194304,289406976~8388608,360710144~8388608] 2024-09-13T00:40:03.267 INFO:tasks.workunit.client.0.smithi136.stdout:from 8 to 10 diff [0~8388608,16777216~4194304,37748736~8388608,75497472~8388608,92274688~16777216,150994944~8388608,171966464~4194304,201326592~4194304,209715200~8388608,255852544~8388608,268435456~8388608,285212672~8388608,335544320~4194304,360710144~4194304] 2024-09-13T00:40:03.267 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,16777216~4194304,37748736~8388608,75497472~8388608,92274688~16777216,150994944~8388608,171966464~4194304,201326592~4194304,209715200~8388608,255852544~8388608,268435456~8388608,285212672~8388608,335544320~4194304,360710144~4194304] 2024-09-13T00:40:03.271 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~8388608,16777216~8388608,37748736~8388608,75497472~8388608,92274688~20971520,150994944~8388608,171966464~4194304,201326592~20971520,255852544~8388608,268435456~12582912,285212672~12582912,335544320~4194304,360710144~8388608] 2024-09-13T00:40:03.271 INFO:tasks.workunit.client.0.smithi136.stdout:from 8 to 11 diff [0~8388608,12582912~8388608,37748736~8388608,75497472~8388608,92274688~16777216,138412032~8388608,150994944~8388608,171966464~4194304,188743680~8388608,201326592~4194304,209715200~8388608,226492416~4194304,234881024~4194304,251658240~25165824,285212672~8388608,318767104~4194304,335544320~4194304,360710144~4194304] 2024-09-13T00:40:03.271 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,12582912~8388608,37748736~8388608,75497472~8388608,92274688~16777216,138412032~8388608,150994944~8388608,171966464~4194304,188743680~8388608,201326592~4194304,209715200~8388608,226492416~4194304,234881024~4194304,251658240~25165824,285212672~8388608,318767104~4194304,335544320~4194304,360710144~4194304] 2024-09-13T00:40:03.277 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~8388608,12582912~12582912,37748736~8388608,75497472~8388608,92274688~20971520,138412032~8388608,150994944~8388608,171966464~4194304,188743680~33554432,226492416~12582912,251658240~29360128,285212672~12582912,318767104~4194304,335544320~4194304,360710144~8388608] 2024-09-13T00:40:03.277 INFO:tasks.workunit.client.0.smithi136.stdout:from 8 to 12 diff [0~8388608,12582912~8388608,37748736~8388608,67108864~4194304,75497472~8388608,92274688~16777216,117440512~4194304,138412032~8388608,150994944~8388608,171966464~4194304,184549376~12582912,201326592~4194304,209715200~8388608,226492416~4194304,234881024~4194304,251658240~25165824,285212672~8388608,314572800~8388608,331350016~4194304,352321536~4194304,360710144~4194304,369098752~12582912] 2024-09-13T00:40:03.277 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,12582912~8388608,37748736~8388608,67108864~4194304,75497472~8388608,92274688~16777216,117440512~4194304,138412032~8388608,150994944~8388608,171966464~4194304,184549376~12582912,201326592~4194304,209715200~8388608,226492416~4194304,234881024~4194304,251658240~25165824,285212672~8388608,314572800~8388608,331350016~4194304,352321536~4194304,360710144~4194304,369098752~12582912] 2024-09-13T00:40:03.284 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~8388608,12582912~12582912,37748736~8388608,67108864~16777216,92274688~20971520,117440512~8388608,138412032~8388608,150994944~8388608,171966464~4194304,184549376~37748736,226492416~12582912,251658240~29360128,285212672~12582912,314572800~8388608,331350016~8388608,352321536~33554432] 2024-09-13T00:40:03.284 INFO:tasks.workunit.client.0.smithi136.stdout:from 8 to 13 diff [0~8388608,12582912~8388608,33554432~8388608,67108864~12582912,92274688~16777216,117440512~8388608,130023424~4194304,138412032~8388608,150994944~8388608,171966464~4194304,184549376~12582912,201326592~4194304,209715200~12582912,226492416~4194304,234881024~4194304,251658240~25165824,285212672~8388608,314572800~8388608,331350016~4194304,348127232~8388608,360710144~4194304,369098752~12582912,385875968~8388608] 2024-09-13T00:40:03.284 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,12582912~8388608,33554432~8388608,67108864~12582912,92274688~16777216,117440512~8388608,130023424~4194304,138412032~8388608,150994944~8388608,171966464~4194304,184549376~12582912,201326592~4194304,209715200~12582912,226492416~4194304,234881024~4194304,251658240~25165824,285212672~8388608,314572800~8388608,331350016~4194304,348127232~8388608,360710144~4194304,369098752~12582912,385875968~8388608] 2024-09-13T00:40:03.291 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~8388608,12582912~12582912,33554432~12582912,67108864~16777216,92274688~20971520,117440512~8388608,130023424~4194304,138412032~8388608,150994944~8388608,171966464~4194304,184549376~37748736,226492416~12582912,251658240~29360128,285212672~12582912,314572800~8388608,331350016~8388608,348127232~46137344] 2024-09-13T00:40:03.292 INFO:tasks.workunit.client.0.smithi136.stdout:from 8 to 14 diff [0~8388608,12582912~8388608,33554432~8388608,67108864~12582912,92274688~16777216,117440512~8388608,130023424~16777216,150994944~12582912,171966464~4194304,180355072~16777216,201326592~4194304,209715200~12582912,226492416~4194304,234881024~4194304,251658240~25165824,285212672~16777216,310378496~12582912,331350016~4194304,348127232~8388608,360710144~4194304,369098752~25165824] 2024-09-13T00:40:03.292 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,12582912~8388608,33554432~8388608,67108864~12582912,92274688~16777216,117440512~8388608,130023424~16777216,150994944~12582912,171966464~4194304,180355072~16777216,201326592~4194304,209715200~12582912,226492416~4194304,234881024~4194304,251658240~25165824,285212672~16777216,310378496~12582912,331350016~4194304,348127232~8388608,360710144~4194304,369098752~25165824] 2024-09-13T00:40:03.300 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~8388608,12582912~12582912,33554432~12582912,67108864~16777216,92274688~20971520,117440512~8388608,130023424~16777216,150994944~12582912,171966464~4194304,180355072~41943040,226492416~12582912,251658240~29360128,285212672~20971520,310378496~16777216,331350016~8388608,348127232~46137344] 2024-09-13T00:40:03.300 INFO:tasks.workunit.client.0.smithi136.stdout:from 8 to 15 diff [0~8388608,12582912~8388608,33554432~8388608,67108864~12582912,92274688~16777216,117440512~8388608,130023424~16777216,150994944~12582912,171966464~4194304,180355072~16777216,201326592~4194304,209715200~12582912,226492416~4194304,234881024~4194304,251658240~25165824,285212672~16777216,310378496~12582912,331350016~4194304,348127232~8388608,360710144~33554432,398458880~8388608] 2024-09-13T00:40:03.300 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,12582912~8388608,33554432~8388608,67108864~12582912,92274688~16777216,117440512~8388608,130023424~16777216,150994944~12582912,171966464~4194304,180355072~16777216,201326592~4194304,209715200~12582912,226492416~4194304,234881024~4194304,251658240~25165824,285212672~16777216,310378496~12582912,331350016~4194304,348127232~8388608,360710144~33554432,398458880~8388608] 2024-09-13T00:40:03.310 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~8388608,12582912~12582912,33554432~12582912,67108864~16777216,92274688~20971520,117440512~8388608,130023424~16777216,150994944~12582912,171966464~4194304,180355072~41943040,226492416~12582912,251658240~29360128,285212672~20971520,310378496~16777216,331350016~8388608,348127232~46137344,398458880~8388608] 2024-09-13T00:40:03.310 INFO:tasks.workunit.client.0.smithi136.stdout:from 8 to 16 diff [0~8388608,12582912~8388608,33554432~8388608,67108864~12582912,88080384~20971520,117440512~8388608,130023424~16777216,150994944~16777216,171966464~25165824,201326592~4194304,209715200~12582912,226492416~4194304,234881024~4194304,251658240~25165824,281018368~20971520,310378496~25165824,348127232~8388608,360710144~33554432,398458880~8388608] 2024-09-13T00:40:03.310 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,12582912~8388608,33554432~8388608,67108864~12582912,88080384~20971520,117440512~8388608,130023424~16777216,150994944~16777216,171966464~25165824,201326592~4194304,209715200~12582912,226492416~4194304,234881024~4194304,251658240~25165824,281018368~20971520,310378496~25165824,348127232~8388608,360710144~33554432,398458880~8388608] 2024-09-13T00:40:03.321 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~8388608,12582912~12582912,33554432~12582912,67108864~16777216,88080384~25165824,117440512~8388608,130023424~16777216,150994944~16777216,171966464~50331648,226492416~12582912,251658240~54525952,310378496~29360128,348127232~46137344,398458880~8388608] 2024-09-13T00:40:03.321 INFO:tasks.workunit.client.0.smithi136.stdout:from 8 to 17 diff [0~8388608,12582912~12582912,33554432~16777216,67108864~12582912,88080384~20971520,117440512~8388608,130023424~16777216,150994944~46137344,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~25165824,281018368~20971520,310378496~25165824,339738624~4194304,348127232~8388608,360710144~33554432,398458880~8388608] 2024-09-13T00:40:03.321 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,12582912~12582912,33554432~16777216,67108864~12582912,88080384~20971520,117440512~8388608,130023424~16777216,150994944~46137344,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~25165824,281018368~20971520,310378496~25165824,339738624~4194304,348127232~8388608,360710144~33554432,398458880~8388608] 2024-09-13T00:40:03.332 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~8388608,12582912~12582912,33554432~16777216,67108864~16777216,88080384~25165824,117440512~8388608,130023424~16777216,150994944~71303168,226492416~20971520,251658240~54525952,310378496~33554432,348127232~46137344,398458880~8388608] 2024-09-13T00:40:03.332 INFO:tasks.workunit.client.0.smithi136.stdout:from 8 to 18 diff [0~8388608,12582912~12582912,33554432~16777216,67108864~12582912,88080384~20971520,117440512~8388608,130023424~16777216,150994944~46137344,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~25165824,281018368~20971520,310378496~25165824,339738624~4194304,348127232~8388608,360710144~33554432,398458880~8388608] 2024-09-13T00:40:03.333 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,12582912~12582912,33554432~16777216,67108864~12582912,88080384~20971520,117440512~8388608,130023424~16777216,150994944~46137344,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~25165824,281018368~20971520,310378496~25165824,339738624~4194304,348127232~8388608,360710144~33554432,398458880~8388608] 2024-09-13T00:40:03.345 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~8388608,12582912~12582912,33554432~20971520,67108864~16777216,88080384~25165824,117440512~8388608,130023424~16777216,150994944~71303168,226492416~20971520,251658240~54525952,310378496~33554432,348127232~46137344,398458880~8388608] 2024-09-13T00:40:03.345 INFO:tasks.workunit.client.0.smithi136.stdout:from 8 to 19 diff [0~29360128,33554432~16777216,67108864~12582912,88080384~20971520,117440512~8388608,130023424~67108864,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~46137344,398458880~12582912] 2024-09-13T00:40:03.345 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,67108864~12582912,88080384~20971520,117440512~8388608,130023424~67108864,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~46137344,398458880~12582912] 2024-09-13T00:40:03.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:02 smithi181 bash[67587]: cluster 2024-09-13T00:40:01.893424+0000 mon.a (mon.0) 3641 : cluster [DBG] osdmap e1979: 8 total, 8 up, 8 in 2024-09-13T00:40:03.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:02 smithi181 bash[67587]: cluster 2024-09-13T00:40:02.445735+0000 mgr.y (mgr.24661) 1887 : cluster [DBG] pgmap v3427: 137 pgs: 137 active+clean; 792 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 239 KiB/s rd, 37 MiB/s wr, 366 op/s 2024-09-13T00:40:03.359 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~16777216,88080384~25165824,117440512~8388608,130023424~92274688,226492416~20971520,251658240~92274688,348127232~46137344,398458880~16777216] 2024-09-13T00:40:03.359 INFO:tasks.workunit.client.0.smithi136.stdout:from 9 to 10 diff [104857600~4194304,150994944~8388608,171966464~4194304,201326592~4194304,209715200~8388608,255852544~8388608,272629760~4194304,285212672~8388608,335544320~4194304] 2024-09-13T00:40:03.359 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [104857600~4194304,150994944~8388608,171966464~4194304,201326592~4194304,209715200~8388608,255852544~8388608,272629760~4194304,285212672~8388608,335544320~4194304] 2024-09-13T00:40:03.362 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [104857600~8388608,150994944~8388608,171966464~4194304,201326592~20971520,255852544~8388608,272629760~8388608,285212672~8388608,335544320~4194304] 2024-09-13T00:40:03.362 INFO:tasks.workunit.client.0.smithi136.stdout:from 9 to 11 diff [12582912~8388608,100663296~8388608,138412032~8388608,150994944~8388608,171966464~4194304,188743680~8388608,201326592~4194304,209715200~8388608,226492416~4194304,234881024~4194304,251658240~16777216,272629760~4194304,285212672~8388608,318767104~4194304,335544320~4194304] 2024-09-13T00:40:03.362 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [12582912~8388608,100663296~8388608,138412032~8388608,150994944~8388608,171966464~4194304,188743680~8388608,201326592~4194304,209715200~8388608,226492416~4194304,234881024~4194304,251658240~16777216,272629760~4194304,285212672~8388608,318767104~4194304,335544320~4194304] 2024-09-13T00:40:03.367 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [12582912~12582912,100663296~12582912,138412032~8388608,150994944~8388608,171966464~4194304,188743680~33554432,226492416~12582912,251658240~29360128,285212672~8388608,318767104~4194304,335544320~4194304] 2024-09-13T00:40:03.368 INFO:tasks.workunit.client.0.smithi136.stdout:from 9 to 12 diff [12582912~8388608,67108864~4194304,92274688~16777216,117440512~4194304,138412032~8388608,150994944~8388608,171966464~4194304,184549376~12582912,201326592~4194304,209715200~8388608,226492416~4194304,234881024~4194304,251658240~16777216,272629760~4194304,285212672~8388608,314572800~8388608,331350016~4194304,352321536~4194304,369098752~12582912] 2024-09-13T00:40:03.368 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [12582912~8388608,67108864~4194304,92274688~16777216,117440512~4194304,138412032~8388608,150994944~8388608,171966464~4194304,184549376~12582912,201326592~4194304,209715200~8388608,226492416~4194304,234881024~4194304,251658240~16777216,272629760~4194304,285212672~8388608,314572800~8388608,331350016~4194304,352321536~4194304,369098752~12582912] 2024-09-13T00:40:03.373 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [12582912~12582912,67108864~8388608,92274688~20971520,117440512~8388608,138412032~8388608,150994944~8388608,171966464~4194304,184549376~37748736,226492416~12582912,251658240~29360128,285212672~8388608,314572800~8388608,331350016~8388608,352321536~8388608,369098752~16777216] 2024-09-13T00:40:03.373 INFO:tasks.workunit.client.0.smithi136.stdout:from 9 to 13 diff [12582912~8388608,33554432~4194304,67108864~8388608,92274688~16777216,117440512~8388608,130023424~4194304,138412032~8388608,150994944~8388608,171966464~4194304,184549376~12582912,201326592~4194304,209715200~12582912,226492416~4194304,234881024~4194304,251658240~16777216,272629760~4194304,285212672~8388608,314572800~8388608,331350016~4194304,348127232~8388608,369098752~12582912,385875968~8388608] 2024-09-13T00:40:03.373 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [12582912~8388608,33554432~4194304,67108864~8388608,92274688~16777216,117440512~8388608,130023424~4194304,138412032~8388608,150994944~8388608,171966464~4194304,184549376~12582912,201326592~4194304,209715200~12582912,226492416~4194304,234881024~4194304,251658240~16777216,272629760~4194304,285212672~8388608,314572800~8388608,331350016~4194304,348127232~8388608,369098752~12582912,385875968~8388608] 2024-09-13T00:40:03.379 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [12582912~12582912,33554432~8388608,67108864~12582912,92274688~20971520,117440512~8388608,130023424~4194304,138412032~8388608,150994944~8388608,171966464~4194304,184549376~37748736,226492416~12582912,251658240~29360128,285212672~8388608,314572800~8388608,331350016~8388608,348127232~12582912,369098752~25165824] 2024-09-13T00:40:03.380 INFO:tasks.workunit.client.0.smithi136.stdout:from 9 to 14 diff [12582912~8388608,33554432~4194304,67108864~12582912,92274688~16777216,117440512~8388608,130023424~16777216,150994944~12582912,171966464~4194304,180355072~16777216,201326592~4194304,209715200~12582912,226492416~4194304,234881024~4194304,251658240~25165824,285212672~16777216,310378496~12582912,331350016~4194304,348127232~8388608,369098752~25165824] 2024-09-13T00:40:03.380 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [12582912~8388608,33554432~4194304,67108864~12582912,92274688~16777216,117440512~8388608,130023424~16777216,150994944~12582912,171966464~4194304,180355072~16777216,201326592~4194304,209715200~12582912,226492416~4194304,234881024~4194304,251658240~25165824,285212672~16777216,310378496~12582912,331350016~4194304,348127232~8388608,369098752~25165824] 2024-09-13T00:40:03.387 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [12582912~12582912,33554432~8388608,67108864~16777216,92274688~20971520,117440512~8388608,130023424~16777216,150994944~12582912,171966464~4194304,180355072~41943040,226492416~12582912,251658240~29360128,285212672~20971520,310378496~16777216,331350016~8388608,348127232~12582912,369098752~25165824] 2024-09-13T00:40:03.387 INFO:tasks.workunit.client.0.smithi136.stdout:from 9 to 15 diff [12582912~8388608,33554432~4194304,67108864~12582912,92274688~16777216,117440512~8388608,130023424~16777216,150994944~12582912,171966464~4194304,180355072~16777216,201326592~4194304,209715200~12582912,226492416~4194304,234881024~4194304,251658240~25165824,285212672~16777216,310378496~12582912,331350016~4194304,348127232~8388608,364904448~29360128,398458880~8388608] 2024-09-13T00:40:03.388 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [12582912~8388608,33554432~4194304,67108864~12582912,92274688~16777216,117440512~8388608,130023424~16777216,150994944~12582912,171966464~4194304,180355072~16777216,201326592~4194304,209715200~12582912,226492416~4194304,234881024~4194304,251658240~25165824,285212672~16777216,310378496~12582912,331350016~4194304,348127232~8388608,364904448~29360128,398458880~8388608] 2024-09-13T00:40:03.395 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [12582912~12582912,33554432~8388608,67108864~16777216,92274688~20971520,117440512~8388608,130023424~16777216,150994944~12582912,171966464~4194304,180355072~41943040,226492416~12582912,251658240~29360128,285212672~20971520,310378496~16777216,331350016~8388608,348127232~12582912,364904448~29360128,398458880~8388608] 2024-09-13T00:40:03.395 INFO:tasks.workunit.client.0.smithi136.stdout:from 9 to 16 diff [12582912~8388608,33554432~4194304,67108864~12582912,88080384~8388608,100663296~8388608,117440512~8388608,130023424~16777216,150994944~16777216,171966464~25165824,201326592~4194304,209715200~12582912,226492416~4194304,234881024~4194304,251658240~25165824,281018368~20971520,310378496~25165824,348127232~8388608,364904448~29360128,398458880~8388608] 2024-09-13T00:40:03.395 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [12582912~8388608,33554432~4194304,67108864~12582912,88080384~8388608,100663296~8388608,117440512~8388608,130023424~16777216,150994944~16777216,171966464~25165824,201326592~4194304,209715200~12582912,226492416~4194304,234881024~4194304,251658240~25165824,281018368~20971520,310378496~25165824,348127232~8388608,364904448~29360128,398458880~8388608] 2024-09-13T00:40:03.405 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [12582912~12582912,33554432~8388608,67108864~16777216,88080384~25165824,117440512~8388608,130023424~16777216,150994944~16777216,171966464~50331648,226492416~12582912,251658240~54525952,310378496~29360128,348127232~12582912,364904448~29360128,398458880~8388608] 2024-09-13T00:40:03.405 INFO:tasks.workunit.client.0.smithi136.stdout:from 9 to 17 diff [0~4194304,12582912~12582912,33554432~12582912,67108864~12582912,88080384~8388608,100663296~8388608,117440512~8388608,130023424~16777216,150994944~46137344,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~25165824,281018368~20971520,310378496~25165824,339738624~4194304,348127232~8388608,364904448~29360128,398458880~8388608] 2024-09-13T00:40:03.405 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~4194304,12582912~12582912,33554432~12582912,67108864~12582912,88080384~8388608,100663296~8388608,117440512~8388608,130023424~16777216,150994944~46137344,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~25165824,281018368~20971520,310378496~25165824,339738624~4194304,348127232~8388608,364904448~29360128,398458880~8388608] 2024-09-13T00:40:03.415 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~8388608,12582912~12582912,33554432~16777216,67108864~16777216,88080384~25165824,117440512~8388608,130023424~16777216,150994944~71303168,226492416~20971520,251658240~54525952,310378496~33554432,348127232~12582912,364904448~29360128,398458880~8388608] 2024-09-13T00:40:03.415 INFO:tasks.workunit.client.0.smithi136.stdout:from 9 to 18 diff [0~8388608,12582912~12582912,33554432~16777216,67108864~12582912,88080384~8388608,100663296~8388608,117440512~8388608,130023424~16777216,150994944~46137344,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~25165824,281018368~20971520,310378496~25165824,339738624~4194304,348127232~8388608,364904448~29360128,398458880~8388608] 2024-09-13T00:40:03.415 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,12582912~12582912,33554432~16777216,67108864~12582912,88080384~8388608,100663296~8388608,117440512~8388608,130023424~16777216,150994944~46137344,201326592~4194304,209715200~12582912,226492416~4194304,234881024~12582912,251658240~25165824,281018368~20971520,310378496~25165824,339738624~4194304,348127232~8388608,364904448~29360128,398458880~8388608] 2024-09-13T00:40:03.426 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~8388608,12582912~12582912,33554432~20971520,67108864~16777216,88080384~25165824,117440512~8388608,130023424~16777216,150994944~71303168,226492416~20971520,251658240~54525952,310378496~33554432,348127232~12582912,364904448~29360128,398458880~8388608] 2024-09-13T00:40:03.426 INFO:tasks.workunit.client.0.smithi136.stdout:from 9 to 19 diff [0~29360128,33554432~16777216,67108864~12582912,88080384~8388608,100663296~8388608,117440512~8388608,130023424~67108864,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:03.426 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,67108864~12582912,88080384~8388608,100663296~8388608,117440512~8388608,130023424~67108864,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:03.439 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~16777216,88080384~25165824,117440512~8388608,130023424~92274688,226492416~20971520,251658240~92274688,348127232~12582912,364904448~29360128,398458880~16777216] 2024-09-13T00:40:03.439 INFO:tasks.workunit.client.0.smithi136.stdout:from 10 to 11 diff [12582912~8388608,100663296~8388608,138412032~8388608,188743680~8388608,226492416~4194304,234881024~4194304,251658240~8388608,264241152~4194304,318767104~4194304] 2024-09-13T00:40:03.439 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [12582912~8388608,100663296~8388608,138412032~8388608,188743680~8388608,226492416~4194304,234881024~4194304,251658240~8388608,264241152~4194304,318767104~4194304] 2024-09-13T00:40:03.442 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [12582912~12582912,100663296~12582912,138412032~8388608,188743680~12582912,226492416~12582912,251658240~20971520,318767104~4194304] 2024-09-13T00:40:03.442 INFO:tasks.workunit.client.0.smithi136.stdout:from 10 to 12 diff [12582912~8388608,67108864~4194304,92274688~16777216,117440512~4194304,138412032~8388608,184549376~12582912,226492416~4194304,234881024~4194304,251658240~16777216,314572800~8388608,331350016~4194304,352321536~4194304,369098752~12582912] 2024-09-13T00:40:03.442 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [12582912~8388608,67108864~4194304,92274688~16777216,117440512~4194304,138412032~8388608,184549376~12582912,226492416~4194304,234881024~4194304,251658240~16777216,314572800~8388608,331350016~4194304,352321536~4194304,369098752~12582912] 2024-09-13T00:40:03.446 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [12582912~12582912,67108864~8388608,92274688~20971520,117440512~8388608,138412032~8388608,184549376~16777216,226492416~12582912,251658240~20971520,314572800~8388608,331350016~8388608,352321536~8388608,369098752~16777216] 2024-09-13T00:40:03.446 INFO:tasks.workunit.client.0.smithi136.stdout:from 10 to 13 diff [12582912~8388608,33554432~4194304,67108864~8388608,92274688~16777216,117440512~8388608,130023424~4194304,138412032~8388608,184549376~12582912,209715200~8388608,226492416~4194304,234881024~4194304,251658240~16777216,314572800~8388608,331350016~4194304,348127232~8388608,369098752~12582912,385875968~8388608] 2024-09-13T00:40:03.446 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [12582912~8388608,33554432~4194304,67108864~8388608,92274688~16777216,117440512~8388608,130023424~4194304,138412032~8388608,184549376~12582912,209715200~8388608,226492416~4194304,234881024~4194304,251658240~16777216,314572800~8388608,331350016~4194304,348127232~8388608,369098752~12582912,385875968~8388608] 2024-09-13T00:40:03.451 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [12582912~12582912,33554432~8388608,67108864~12582912,92274688~20971520,117440512~8388608,130023424~4194304,138412032~8388608,184549376~16777216,209715200~8388608,226492416~12582912,251658240~20971520,314572800~8388608,331350016~8388608,348127232~12582912,369098752~25165824] 2024-09-13T00:40:03.451 INFO:tasks.workunit.client.0.smithi136.stdout:from 10 to 14 diff [12582912~8388608,33554432~4194304,67108864~12582912,92274688~16777216,117440512~8388608,130023424~16777216,155189248~8388608,180355072~16777216,209715200~8388608,226492416~4194304,234881024~4194304,251658240~25165824,293601280~8388608,310378496~12582912,331350016~4194304,348127232~8388608,369098752~25165824] 2024-09-13T00:40:03.452 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [12582912~8388608,33554432~4194304,67108864~12582912,92274688~16777216,117440512~8388608,130023424~16777216,155189248~8388608,180355072~16777216,209715200~8388608,226492416~4194304,234881024~4194304,251658240~25165824,293601280~8388608,310378496~12582912,331350016~4194304,348127232~8388608,369098752~25165824] 2024-09-13T00:40:03.458 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [12582912~12582912,33554432~8388608,67108864~16777216,92274688~20971520,117440512~8388608,130023424~16777216,155189248~8388608,180355072~20971520,209715200~8388608,226492416~12582912,251658240~29360128,293601280~12582912,310378496~16777216,331350016~8388608,348127232~12582912,369098752~25165824] 2024-09-13T00:40:03.458 INFO:tasks.workunit.client.0.smithi136.stdout:from 10 to 15 diff [12582912~8388608,33554432~4194304,67108864~12582912,92274688~16777216,117440512~8388608,130023424~16777216,150994944~8388608,180355072~16777216,209715200~8388608,226492416~4194304,234881024~4194304,251658240~25165824,285212672~16777216,310378496~12582912,331350016~4194304,348127232~8388608,364904448~29360128,398458880~8388608] 2024-09-13T00:40:03.458 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [12582912~8388608,33554432~4194304,67108864~12582912,92274688~16777216,117440512~8388608,130023424~16777216,150994944~8388608,180355072~16777216,209715200~8388608,226492416~4194304,234881024~4194304,251658240~25165824,285212672~16777216,310378496~12582912,331350016~4194304,348127232~8388608,364904448~29360128,398458880~8388608] 2024-09-13T00:40:03.464 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [12582912~12582912,33554432~8388608,67108864~16777216,92274688~20971520,117440512~8388608,130023424~16777216,150994944~12582912,180355072~20971520,209715200~8388608,226492416~12582912,251658240~29360128,285212672~20971520,310378496~16777216,331350016~8388608,348127232~12582912,364904448~29360128,398458880~8388608] 2024-09-13T00:40:03.464 INFO:tasks.workunit.client.0.smithi136.stdout:from 10 to 16 diff [12582912~8388608,33554432~4194304,67108864~12582912,88080384~8388608,100663296~8388608,117440512~8388608,130023424~16777216,150994944~12582912,176160768~20971520,209715200~8388608,226492416~4194304,234881024~4194304,251658240~25165824,281018368~20971520,310378496~25165824,348127232~8388608,364904448~29360128,398458880~8388608] 2024-09-13T00:40:03.465 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [12582912~8388608,33554432~4194304,67108864~12582912,88080384~8388608,100663296~8388608,117440512~8388608,130023424~16777216,150994944~12582912,176160768~20971520,209715200~8388608,226492416~4194304,234881024~4194304,251658240~25165824,281018368~20971520,310378496~25165824,348127232~8388608,364904448~29360128,398458880~8388608] 2024-09-13T00:40:03.472 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [12582912~12582912,33554432~8388608,67108864~16777216,88080384~25165824,117440512~8388608,130023424~16777216,150994944~16777216,176160768~25165824,209715200~8388608,226492416~12582912,251658240~54525952,310378496~29360128,348127232~12582912,364904448~29360128,398458880~8388608] 2024-09-13T00:40:03.472 INFO:tasks.workunit.client.0.smithi136.stdout:from 10 to 17 diff [0~4194304,12582912~12582912,33554432~12582912,67108864~12582912,88080384~8388608,100663296~8388608,117440512~8388608,130023424~16777216,150994944~20971520,176160768~20971520,209715200~8388608,226492416~4194304,234881024~12582912,251658240~25165824,281018368~20971520,310378496~25165824,339738624~4194304,348127232~8388608,364904448~29360128,398458880~8388608] 2024-09-13T00:40:03.472 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~4194304,12582912~12582912,33554432~12582912,67108864~12582912,88080384~8388608,100663296~8388608,117440512~8388608,130023424~16777216,150994944~20971520,176160768~20971520,209715200~8388608,226492416~4194304,234881024~12582912,251658240~25165824,281018368~20971520,310378496~25165824,339738624~4194304,348127232~8388608,364904448~29360128,398458880~8388608] 2024-09-13T00:40:03.481 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~8388608,12582912~12582912,33554432~16777216,67108864~16777216,88080384~25165824,117440512~8388608,130023424~16777216,150994944~20971520,176160768~25165824,209715200~8388608,226492416~20971520,251658240~54525952,310378496~33554432,348127232~12582912,364904448~29360128,398458880~8388608] 2024-09-13T00:40:03.482 INFO:tasks.workunit.client.0.smithi136.stdout:from 10 to 18 diff [0~8388608,12582912~12582912,33554432~16777216,67108864~12582912,88080384~8388608,100663296~8388608,117440512~8388608,130023424~16777216,150994944~20971520,176160768~20971520,209715200~8388608,226492416~4194304,234881024~12582912,251658240~25165824,281018368~20971520,310378496~25165824,339738624~4194304,348127232~8388608,364904448~29360128,398458880~8388608] 2024-09-13T00:40:03.482 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,12582912~12582912,33554432~16777216,67108864~12582912,88080384~8388608,100663296~8388608,117440512~8388608,130023424~16777216,150994944~20971520,176160768~20971520,209715200~8388608,226492416~4194304,234881024~12582912,251658240~25165824,281018368~20971520,310378496~25165824,339738624~4194304,348127232~8388608,364904448~29360128,398458880~8388608] 2024-09-13T00:40:03.491 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~8388608,12582912~12582912,33554432~20971520,67108864~16777216,88080384~25165824,117440512~8388608,130023424~16777216,150994944~20971520,176160768~25165824,209715200~8388608,226492416~20971520,251658240~54525952,310378496~33554432,348127232~12582912,364904448~29360128,398458880~8388608] 2024-09-13T00:40:03.491 INFO:tasks.workunit.client.0.smithi136.stdout:from 10 to 19 diff [0~29360128,33554432~16777216,67108864~12582912,88080384~8388608,100663296~8388608,117440512~8388608,130023424~41943040,176160768~20971520,209715200~8388608,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:03.492 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,67108864~12582912,88080384~8388608,100663296~8388608,117440512~8388608,130023424~41943040,176160768~20971520,209715200~8388608,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:03.502 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~16777216,88080384~25165824,117440512~8388608,130023424~41943040,176160768~25165824,209715200~8388608,226492416~20971520,251658240~92274688,348127232~12582912,364904448~29360128,398458880~16777216] 2024-09-13T00:40:03.502 INFO:tasks.workunit.client.0.smithi136.stdout:from 11 to 12 diff [67108864~4194304,92274688~8388608,117440512~4194304,184549376~4194304,255852544~8388608,314572800~4194304,331350016~4194304,352321536~4194304,369098752~12582912] 2024-09-13T00:40:03.502 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [67108864~4194304,92274688~8388608,117440512~4194304,184549376~4194304,255852544~8388608,314572800~4194304,331350016~4194304,352321536~4194304,369098752~12582912] 2024-09-13T00:40:03.506 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [67108864~8388608,92274688~8388608,117440512~8388608,184549376~8388608,255852544~12582912,314572800~4194304,331350016~8388608,352321536~8388608,369098752~16777216] 2024-09-13T00:40:03.506 INFO:tasks.workunit.client.0.smithi136.stdout:from 11 to 13 diff [16777216~8388608,33554432~4194304,67108864~8388608,92274688~8388608,117440512~8388608,130023424~4194304,184549376~4194304,192937984~4194304,209715200~8388608,255852544~8388608,314572800~4194304,331350016~4194304,348127232~8388608,369098752~12582912,385875968~8388608] 2024-09-13T00:40:03.506 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [16777216~8388608,33554432~4194304,67108864~8388608,92274688~8388608,117440512~8388608,130023424~4194304,184549376~4194304,192937984~4194304,209715200~8388608,255852544~8388608,314572800~4194304,331350016~4194304,348127232~8388608,369098752~12582912,385875968~8388608] 2024-09-13T00:40:03.510 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [16777216~8388608,33554432~8388608,67108864~12582912,92274688~8388608,117440512~8388608,130023424~4194304,184549376~16777216,209715200~8388608,255852544~12582912,314572800~4194304,331350016~8388608,348127232~12582912,369098752~25165824] 2024-09-13T00:40:03.510 INFO:tasks.workunit.client.0.smithi136.stdout:from 11 to 14 diff [16777216~8388608,33554432~4194304,67108864~12582912,92274688~8388608,117440512~8388608,130023424~12582912,155189248~8388608,180355072~8388608,192937984~4194304,209715200~8388608,255852544~8388608,268435456~8388608,293601280~8388608,310378496~12582912,331350016~4194304,348127232~8388608,369098752~25165824] 2024-09-13T00:40:03.510 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [16777216~8388608,33554432~4194304,67108864~12582912,92274688~8388608,117440512~8388608,130023424~12582912,155189248~8388608,180355072~8388608,192937984~4194304,209715200~8388608,255852544~8388608,268435456~8388608,293601280~8388608,310378496~12582912,331350016~4194304,348127232~8388608,369098752~25165824] 2024-09-13T00:40:03.515 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [16777216~8388608,33554432~8388608,67108864~16777216,92274688~8388608,117440512~8388608,130023424~16777216,155189248~8388608,180355072~20971520,209715200~8388608,255852544~25165824,293601280~12582912,310378496~16777216,331350016~8388608,348127232~12582912,369098752~25165824] 2024-09-13T00:40:03.515 INFO:tasks.workunit.client.0.smithi136.stdout:from 11 to 15 diff [16777216~8388608,33554432~4194304,67108864~12582912,92274688~8388608,117440512~8388608,130023424~12582912,150994944~8388608,180355072~16777216,209715200~8388608,255852544~8388608,268435456~8388608,285212672~16777216,310378496~12582912,331350016~4194304,348127232~8388608,364904448~29360128,398458880~8388608] 2024-09-13T00:40:03.515 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [16777216~8388608,33554432~4194304,67108864~12582912,92274688~8388608,117440512~8388608,130023424~12582912,150994944~8388608,180355072~16777216,209715200~8388608,255852544~8388608,268435456~8388608,285212672~16777216,310378496~12582912,331350016~4194304,348127232~8388608,364904448~29360128,398458880~8388608] 2024-09-13T00:40:03.521 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [16777216~8388608,33554432~8388608,67108864~16777216,92274688~8388608,117440512~8388608,130023424~16777216,150994944~12582912,180355072~20971520,209715200~8388608,255852544~25165824,285212672~20971520,310378496~16777216,331350016~8388608,348127232~12582912,364904448~29360128,398458880~8388608] 2024-09-13T00:40:03.521 INFO:tasks.workunit.client.0.smithi136.stdout:from 11 to 16 diff [16777216~8388608,33554432~4194304,67108864~12582912,88080384~8388608,117440512~8388608,130023424~12582912,150994944~12582912,176160768~20971520,209715200~8388608,255852544~8388608,268435456~8388608,281018368~20971520,310378496~25165824,348127232~8388608,364904448~29360128,398458880~8388608] 2024-09-13T00:40:03.521 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [16777216~8388608,33554432~4194304,67108864~12582912,88080384~8388608,117440512~8388608,130023424~12582912,150994944~12582912,176160768~20971520,209715200~8388608,255852544~8388608,268435456~8388608,281018368~20971520,310378496~25165824,348127232~8388608,364904448~29360128,398458880~8388608] 2024-09-13T00:40:03.528 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [16777216~8388608,33554432~8388608,67108864~16777216,88080384~12582912,117440512~8388608,130023424~16777216,150994944~16777216,176160768~25165824,209715200~8388608,255852544~50331648,310378496~29360128,348127232~12582912,364904448~29360128,398458880~8388608] 2024-09-13T00:40:03.528 INFO:tasks.workunit.client.0.smithi136.stdout:from 11 to 17 diff [0~4194304,12582912~12582912,33554432~12582912,67108864~12582912,88080384~8388608,117440512~8388608,130023424~12582912,150994944~20971520,176160768~20971520,209715200~8388608,239075328~8388608,255852544~20971520,281018368~20971520,310378496~25165824,339738624~4194304,348127232~8388608,364904448~29360128,398458880~8388608] 2024-09-13T00:40:03.528 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~4194304,12582912~12582912,33554432~12582912,67108864~12582912,88080384~8388608,117440512~8388608,130023424~12582912,150994944~20971520,176160768~20971520,209715200~8388608,239075328~8388608,255852544~20971520,281018368~20971520,310378496~25165824,339738624~4194304,348127232~8388608,364904448~29360128,398458880~8388608] 2024-09-13T00:40:03.537 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~8388608,12582912~12582912,33554432~16777216,67108864~16777216,88080384~12582912,117440512~8388608,130023424~16777216,150994944~20971520,176160768~25165824,209715200~8388608,239075328~8388608,255852544~50331648,310378496~33554432,348127232~12582912,364904448~29360128,398458880~8388608] 2024-09-13T00:40:03.537 INFO:tasks.workunit.client.0.smithi136.stdout:from 11 to 18 diff [0~8388608,12582912~12582912,33554432~16777216,67108864~12582912,88080384~8388608,117440512~8388608,130023424~12582912,150994944~20971520,176160768~20971520,209715200~8388608,234881024~12582912,251658240~25165824,281018368~20971520,310378496~25165824,339738624~4194304,348127232~8388608,364904448~29360128,398458880~8388608] 2024-09-13T00:40:03.537 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,12582912~12582912,33554432~16777216,67108864~12582912,88080384~8388608,117440512~8388608,130023424~12582912,150994944~20971520,176160768~20971520,209715200~8388608,234881024~12582912,251658240~25165824,281018368~20971520,310378496~25165824,339738624~4194304,348127232~8388608,364904448~29360128,398458880~8388608] 2024-09-13T00:40:03.545 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~8388608,12582912~12582912,33554432~20971520,67108864~16777216,88080384~12582912,117440512~8388608,130023424~16777216,150994944~20971520,176160768~25165824,209715200~8388608,234881024~12582912,251658240~54525952,310378496~33554432,348127232~12582912,364904448~29360128,398458880~8388608] 2024-09-13T00:40:03.546 INFO:tasks.workunit.client.0.smithi136.stdout:from 11 to 19 diff [0~25165824,33554432~16777216,67108864~12582912,88080384~8388608,117440512~8388608,130023424~12582912,146800640~25165824,176160768~20971520,209715200~8388608,230686720~12582912,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:03.546 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,33554432~16777216,67108864~12582912,88080384~8388608,117440512~8388608,130023424~12582912,146800640~25165824,176160768~20971520,209715200~8388608,230686720~12582912,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:03.555 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~16777216,88080384~12582912,117440512~8388608,130023424~41943040,176160768~25165824,209715200~8388608,230686720~16777216,251658240~92274688,348127232~12582912,364904448~29360128,398458880~16777216] 2024-09-13T00:40:03.556 INFO:tasks.workunit.client.0.smithi136.stdout:from 12 to 13 diff [16777216~8388608,33554432~4194304,71303168~4194304,117440512~4194304,130023424~4194304,192937984~4194304,209715200~8388608,348127232~4194304,385875968~8388608] 2024-09-13T00:40:03.556 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [16777216~8388608,33554432~4194304,71303168~4194304,117440512~4194304,130023424~4194304,192937984~4194304,209715200~8388608,348127232~4194304,385875968~8388608] 2024-09-13T00:40:03.558 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [16777216~8388608,33554432~8388608,71303168~8388608,117440512~4194304,130023424~4194304,192937984~8388608,209715200~8388608,348127232~4194304,385875968~8388608] 2024-09-13T00:40:03.558 INFO:tasks.workunit.client.0.smithi136.stdout:from 12 to 14 diff [16777216~8388608,33554432~4194304,71303168~8388608,117440512~4194304,130023424~12582912,155189248~8388608,180355072~8388608,192937984~4194304,209715200~8388608,268435456~8388608,293601280~8388608,310378496~4194304,318767104~4194304,348127232~4194304,377487360~16777216] 2024-09-13T00:40:03.559 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [16777216~8388608,33554432~4194304,71303168~8388608,117440512~4194304,130023424~12582912,155189248~8388608,180355072~8388608,192937984~4194304,209715200~8388608,268435456~8388608,293601280~8388608,310378496~4194304,318767104~4194304,348127232~4194304,377487360~16777216] 2024-09-13T00:40:03.563 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [16777216~8388608,33554432~8388608,71303168~12582912,117440512~4194304,130023424~16777216,155189248~8388608,180355072~20971520,209715200~8388608,268435456~12582912,293601280~12582912,310378496~4194304,318767104~8388608,348127232~4194304,377487360~16777216] 2024-09-13T00:40:03.563 INFO:tasks.workunit.client.0.smithi136.stdout:from 12 to 15 diff [16777216~8388608,33554432~4194304,67108864~12582912,117440512~4194304,130023424~12582912,150994944~8388608,180355072~16777216,209715200~8388608,268435456~8388608,285212672~16777216,310378496~12582912,348127232~4194304,364904448~4194304,377487360~16777216,398458880~8388608] 2024-09-13T00:40:03.563 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [16777216~8388608,33554432~4194304,67108864~12582912,117440512~4194304,130023424~12582912,150994944~8388608,180355072~16777216,209715200~8388608,268435456~8388608,285212672~16777216,310378496~12582912,348127232~4194304,364904448~4194304,377487360~16777216,398458880~8388608] 2024-09-13T00:40:03.568 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [16777216~8388608,33554432~8388608,67108864~16777216,117440512~4194304,130023424~16777216,150994944~12582912,180355072~20971520,209715200~8388608,268435456~12582912,285212672~20971520,310378496~16777216,348127232~4194304,364904448~8388608,377487360~16777216,398458880~8388608] 2024-09-13T00:40:03.569 INFO:tasks.workunit.client.0.smithi136.stdout:from 12 to 16 diff [16777216~8388608,33554432~4194304,67108864~12582912,88080384~8388608,117440512~4194304,130023424~12582912,150994944~12582912,176160768~20971520,209715200~8388608,268435456~8388608,281018368~20971520,310378496~20971520,348127232~4194304,364904448~4194304,377487360~16777216,398458880~8388608] 2024-09-13T00:40:03.569 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [16777216~8388608,33554432~4194304,67108864~12582912,88080384~8388608,117440512~4194304,130023424~12582912,150994944~12582912,176160768~20971520,209715200~8388608,268435456~8388608,281018368~20971520,310378496~20971520,348127232~4194304,364904448~4194304,377487360~16777216,398458880~8388608] 2024-09-13T00:40:03.574 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [16777216~8388608,33554432~8388608,67108864~16777216,88080384~12582912,117440512~4194304,130023424~16777216,150994944~16777216,176160768~25165824,209715200~8388608,268435456~37748736,310378496~20971520,348127232~4194304,364904448~8388608,377487360~16777216,398458880~8388608] 2024-09-13T00:40:03.574 INFO:tasks.workunit.client.0.smithi136.stdout:from 12 to 17 diff [0~4194304,12582912~12582912,33554432~12582912,67108864~12582912,88080384~8388608,117440512~4194304,130023424~12582912,150994944~20971520,176160768~20971520,209715200~8388608,239075328~8388608,264241152~12582912,281018368~20971520,310378496~20971520,339738624~4194304,348127232~4194304,364904448~4194304,373293056~20971520,398458880~8388608] 2024-09-13T00:40:03.574 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~4194304,12582912~12582912,33554432~12582912,67108864~12582912,88080384~8388608,117440512~4194304,130023424~12582912,150994944~20971520,176160768~20971520,209715200~8388608,239075328~8388608,264241152~12582912,281018368~20971520,310378496~20971520,339738624~4194304,348127232~4194304,364904448~4194304,373293056~20971520,398458880~8388608] 2024-09-13T00:40:03.581 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~8388608,12582912~12582912,33554432~16777216,67108864~16777216,88080384~12582912,117440512~4194304,130023424~16777216,150994944~20971520,176160768~25165824,209715200~8388608,239075328~8388608,264241152~41943040,310378496~20971520,339738624~4194304,348127232~4194304,364904448~29360128,398458880~8388608] 2024-09-13T00:40:03.581 INFO:tasks.workunit.client.0.smithi136.stdout:from 12 to 18 diff [0~8388608,12582912~12582912,33554432~16777216,67108864~12582912,88080384~8388608,117440512~4194304,130023424~12582912,150994944~20971520,176160768~20971520,209715200~8388608,234881024~12582912,251658240~4194304,264241152~12582912,281018368~20971520,310378496~20971520,339738624~4194304,348127232~4194304,364904448~4194304,373293056~20971520,398458880~8388608] 2024-09-13T00:40:03.581 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,12582912~12582912,33554432~16777216,67108864~12582912,88080384~8388608,117440512~4194304,130023424~12582912,150994944~20971520,176160768~20971520,209715200~8388608,234881024~12582912,251658240~4194304,264241152~12582912,281018368~20971520,310378496~20971520,339738624~4194304,348127232~4194304,364904448~4194304,373293056~20971520,398458880~8388608] 2024-09-13T00:40:03.589 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~8388608,12582912~12582912,33554432~20971520,67108864~16777216,88080384~12582912,117440512~4194304,130023424~16777216,150994944~20971520,176160768~25165824,209715200~8388608,234881024~12582912,251658240~4194304,264241152~41943040,310378496~20971520,339738624~4194304,348127232~4194304,364904448~29360128,398458880~8388608] 2024-09-13T00:40:03.589 INFO:tasks.workunit.client.0.smithi136.stdout:from 12 to 19 diff [0~25165824,33554432~16777216,67108864~12582912,88080384~8388608,117440512~4194304,130023424~12582912,146800640~25165824,176160768~20971520,209715200~8388608,230686720~12582912,251658240~4194304,264241152~12582912,281018368~25165824,310378496~20971520,339738624~4194304,348127232~4194304,356515840~4194304,364904448~4194304,373293056~20971520,398458880~12582912] 2024-09-13T00:40:03.589 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,33554432~16777216,67108864~12582912,88080384~8388608,117440512~4194304,130023424~12582912,146800640~25165824,176160768~20971520,209715200~8388608,230686720~12582912,251658240~4194304,264241152~12582912,281018368~25165824,310378496~20971520,339738624~4194304,348127232~4194304,356515840~4194304,364904448~4194304,373293056~20971520,398458880~12582912] 2024-09-13T00:40:03.598 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~16777216,88080384~12582912,117440512~4194304,130023424~41943040,176160768~25165824,209715200~8388608,230686720~16777216,251658240~4194304,264241152~67108864,339738624~4194304,348127232~4194304,356515840~4194304,364904448~29360128,398458880~16777216] 2024-09-13T00:40:03.598 INFO:tasks.workunit.client.0.smithi136.stdout:from 13 to 14 diff [75497472~8388608,134217728~8388608,155189248~8388608,180355072~8388608,209715200~8388608,268435456~8388608,293601280~8388608,310378496~4194304,318767104~4194304,377487360~8388608] 2024-09-13T00:40:03.598 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [75497472~8388608,134217728~8388608,155189248~8388608,180355072~8388608,209715200~8388608,268435456~8388608,293601280~8388608,310378496~4194304,318767104~4194304,377487360~8388608] 2024-09-13T00:40:03.601 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [75497472~8388608,134217728~12582912,155189248~8388608,180355072~12582912,209715200~8388608,268435456~12582912,293601280~12582912,310378496~4194304,318767104~8388608,377487360~8388608] 2024-09-13T00:40:03.601 INFO:tasks.workunit.client.0.smithi136.stdout:from 13 to 15 diff [67108864~4194304,75497472~8388608,134217728~8388608,150994944~8388608,180355072~12582912,209715200~8388608,268435456~8388608,285212672~16777216,310378496~12582912,364904448~4194304,377487360~8388608,398458880~8388608] 2024-09-13T00:40:03.601 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [67108864~4194304,75497472~8388608,134217728~8388608,150994944~8388608,180355072~12582912,209715200~8388608,268435456~8388608,285212672~16777216,310378496~12582912,364904448~4194304,377487360~8388608,398458880~8388608] 2024-09-13T00:40:03.605 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [67108864~4194304,75497472~8388608,134217728~12582912,150994944~12582912,180355072~12582912,209715200~8388608,268435456~12582912,285212672~20971520,310378496~16777216,364904448~8388608,377487360~8388608,398458880~8388608] 2024-09-13T00:40:03.606 INFO:tasks.workunit.client.0.smithi136.stdout:from 13 to 16 diff [33554432~4194304,67108864~4194304,75497472~8388608,88080384~8388608,134217728~8388608,150994944~12582912,176160768~16777216,209715200~8388608,268435456~8388608,281018368~20971520,310378496~20971520,364904448~4194304,377487360~12582912,398458880~8388608] 2024-09-13T00:40:03.606 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [33554432~4194304,67108864~4194304,75497472~8388608,88080384~8388608,134217728~8388608,150994944~12582912,176160768~16777216,209715200~8388608,268435456~8388608,281018368~20971520,310378496~20971520,364904448~4194304,377487360~12582912,398458880~8388608] 2024-09-13T00:40:03.610 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [33554432~4194304,67108864~4194304,75497472~8388608,88080384~12582912,134217728~12582912,150994944~16777216,176160768~16777216,209715200~8388608,268435456~37748736,310378496~20971520,364904448~8388608,377487360~16777216,398458880~8388608] 2024-09-13T00:40:03.610 INFO:tasks.workunit.client.0.smithi136.stdout:from 13 to 17 diff [0~4194304,12582912~8388608,33554432~12582912,67108864~4194304,75497472~8388608,88080384~8388608,134217728~8388608,150994944~20971520,176160768~16777216,209715200~8388608,239075328~8388608,264241152~12582912,281018368~20971520,310378496~20971520,339738624~4194304,364904448~4194304,373293056~16777216,398458880~8388608] 2024-09-13T00:40:03.610 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~4194304,12582912~8388608,33554432~12582912,67108864~4194304,75497472~8388608,88080384~8388608,134217728~8388608,150994944~20971520,176160768~16777216,209715200~8388608,239075328~8388608,264241152~12582912,281018368~20971520,310378496~20971520,339738624~4194304,364904448~4194304,373293056~16777216,398458880~8388608] 2024-09-13T00:40:03.616 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~8388608,12582912~8388608,33554432~16777216,67108864~4194304,75497472~8388608,88080384~12582912,134217728~12582912,150994944~20971520,176160768~16777216,209715200~8388608,239075328~8388608,264241152~41943040,310378496~20971520,339738624~4194304,364904448~29360128,398458880~8388608] 2024-09-13T00:40:03.616 INFO:tasks.workunit.client.0.smithi136.stdout:from 13 to 18 diff [0~8388608,12582912~8388608,33554432~16777216,67108864~4194304,75497472~8388608,88080384~8388608,130023424~12582912,150994944~20971520,176160768~16777216,209715200~8388608,234881024~12582912,251658240~4194304,264241152~12582912,281018368~20971520,310378496~20971520,339738624~4194304,364904448~4194304,373293056~20971520,398458880~8388608] 2024-09-13T00:40:03.616 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,12582912~8388608,33554432~16777216,67108864~4194304,75497472~8388608,88080384~8388608,130023424~12582912,150994944~20971520,176160768~16777216,209715200~8388608,234881024~12582912,251658240~4194304,264241152~12582912,281018368~20971520,310378496~20971520,339738624~4194304,364904448~4194304,373293056~20971520,398458880~8388608] 2024-09-13T00:40:03.623 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~8388608,12582912~8388608,33554432~20971520,67108864~4194304,75497472~8388608,88080384~12582912,130023424~16777216,150994944~20971520,176160768~16777216,209715200~8388608,234881024~12582912,251658240~4194304,264241152~41943040,310378496~20971520,339738624~4194304,364904448~29360128,398458880~8388608] 2024-09-13T00:40:03.623 INFO:tasks.workunit.client.0.smithi136.stdout:from 13 to 19 diff [0~25165824,33554432~16777216,67108864~4194304,75497472~8388608,88080384~8388608,130023424~12582912,146800640~25165824,176160768~16777216,209715200~8388608,230686720~12582912,251658240~4194304,264241152~12582912,281018368~25165824,310378496~20971520,339738624~4194304,356515840~4194304,364904448~4194304,373293056~20971520,398458880~12582912] 2024-09-13T00:40:03.623 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,33554432~16777216,67108864~4194304,75497472~8388608,88080384~8388608,130023424~12582912,146800640~25165824,176160768~16777216,209715200~8388608,230686720~12582912,251658240~4194304,264241152~12582912,281018368~25165824,310378496~20971520,339738624~4194304,356515840~4194304,364904448~4194304,373293056~20971520,398458880~12582912] 2024-09-13T00:40:03.630 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~4194304,75497472~8388608,88080384~12582912,130023424~41943040,176160768~16777216,209715200~8388608,230686720~16777216,251658240~4194304,264241152~67108864,339738624~4194304,356515840~4194304,364904448~29360128,398458880~16777216] 2024-09-13T00:40:03.631 INFO:tasks.workunit.client.0.smithi136.stdout:from 14 to 15 diff [67108864~4194304,150994944~8388608,180355072~8388608,285212672~12582912,314572800~4194304,364904448~4194304,398458880~8388608] 2024-09-13T00:40:03.631 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [67108864~4194304,150994944~8388608,180355072~8388608,285212672~12582912,314572800~4194304,364904448~4194304,398458880~8388608] 2024-09-13T00:40:03.633 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [67108864~4194304,150994944~12582912,180355072~12582912,285212672~16777216,314572800~4194304,364904448~8388608,398458880~8388608] 2024-09-13T00:40:03.633 INFO:tasks.workunit.client.0.smithi136.stdout:from 14 to 16 diff [33554432~4194304,67108864~4194304,88080384~8388608,150994944~12582912,176160768~12582912,281018368~16777216,314572800~12582912,364904448~4194304,381681664~8388608,398458880~8388608] 2024-09-13T00:40:03.633 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [33554432~4194304,67108864~4194304,88080384~8388608,150994944~12582912,176160768~12582912,281018368~16777216,314572800~12582912,364904448~4194304,381681664~8388608,398458880~8388608] 2024-09-13T00:40:03.637 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [33554432~4194304,67108864~4194304,88080384~12582912,150994944~16777216,176160768~16777216,281018368~20971520,314572800~16777216,364904448~8388608,381681664~12582912,398458880~8388608] 2024-09-13T00:40:03.637 INFO:tasks.workunit.client.0.smithi136.stdout:from 14 to 17 diff [0~4194304,12582912~8388608,33554432~12582912,67108864~4194304,88080384~8388608,150994944~20971520,176160768~12582912,239075328~8388608,264241152~4194304,281018368~16777216,314572800~12582912,339738624~4194304,364904448~4194304,373293056~4194304,381681664~8388608,398458880~8388608] 2024-09-13T00:40:03.637 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~4194304,12582912~8388608,33554432~12582912,67108864~4194304,88080384~8388608,150994944~20971520,176160768~12582912,239075328~8388608,264241152~4194304,281018368~16777216,314572800~12582912,339738624~4194304,364904448~4194304,373293056~4194304,381681664~8388608,398458880~8388608] 2024-09-13T00:40:03.642 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~8388608,12582912~8388608,33554432~16777216,67108864~4194304,88080384~12582912,150994944~20971520,176160768~16777216,239075328~8388608,264241152~8388608,281018368~20971520,314572800~16777216,339738624~4194304,364904448~12582912,381681664~12582912,398458880~8388608] 2024-09-13T00:40:03.642 INFO:tasks.workunit.client.0.smithi136.stdout:from 14 to 18 diff [0~8388608,12582912~8388608,33554432~16777216,67108864~4194304,88080384~8388608,130023424~4194304,150994944~20971520,176160768~12582912,234881024~12582912,251658240~4194304,264241152~4194304,276824064~20971520,314572800~12582912,339738624~4194304,364904448~4194304,373293056~4194304,381681664~12582912,398458880~8388608] 2024-09-13T00:40:03.643 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,12582912~8388608,33554432~16777216,67108864~4194304,88080384~8388608,130023424~4194304,150994944~20971520,176160768~12582912,234881024~12582912,251658240~4194304,264241152~4194304,276824064~20971520,314572800~12582912,339738624~4194304,364904448~4194304,373293056~4194304,381681664~12582912,398458880~8388608] 2024-09-13T00:40:03.648 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~8388608,12582912~8388608,33554432~20971520,67108864~4194304,88080384~12582912,130023424~4194304,150994944~20971520,176160768~16777216,234881024~12582912,251658240~4194304,264241152~8388608,276824064~25165824,314572800~16777216,339738624~4194304,364904448~12582912,381681664~12582912,398458880~8388608] 2024-09-13T00:40:03.648 INFO:tasks.workunit.client.0.smithi136.stdout:from 14 to 19 diff [0~25165824,33554432~16777216,67108864~4194304,88080384~8388608,130023424~4194304,146800640~25165824,176160768~16777216,230686720~12582912,251658240~4194304,264241152~4194304,276824064~20971520,301989888~4194304,314572800~12582912,339738624~4194304,356515840~4194304,364904448~4194304,373293056~4194304,381681664~12582912,398458880~12582912] 2024-09-13T00:40:03.648 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,33554432~16777216,67108864~4194304,88080384~8388608,130023424~4194304,146800640~25165824,176160768~16777216,230686720~12582912,251658240~4194304,264241152~4194304,276824064~20971520,301989888~4194304,314572800~12582912,339738624~4194304,356515840~4194304,364904448~4194304,373293056~4194304,381681664~12582912,398458880~12582912] 2024-09-13T00:40:03.655 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~4194304,88080384~12582912,130023424~4194304,146800640~25165824,176160768~16777216,230686720~16777216,251658240~4194304,264241152~8388608,276824064~33554432,314572800~16777216,339738624~4194304,356515840~4194304,364904448~12582912,381681664~12582912,398458880~16777216] 2024-09-13T00:40:03.655 INFO:tasks.workunit.client.0.smithi136.stdout:from 15 to 16 diff [33554432~4194304,88080384~8388608,155189248~8388608,176160768~4194304,184549376~4194304,281018368~4194304,314572800~12582912,381681664~8388608] 2024-09-13T00:40:03.655 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [33554432~4194304,88080384~8388608,155189248~8388608,176160768~4194304,184549376~4194304,281018368~4194304,314572800~12582912,381681664~8388608] 2024-09-13T00:40:03.657 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [33554432~4194304,88080384~12582912,155189248~12582912,176160768~4194304,184549376~8388608,281018368~4194304,314572800~16777216,381681664~12582912] 2024-09-13T00:40:03.657 INFO:tasks.workunit.client.0.smithi136.stdout:from 15 to 17 diff [0~4194304,12582912~8388608,33554432~12582912,88080384~8388608,155189248~16777216,176160768~4194304,184549376~4194304,239075328~8388608,264241152~4194304,281018368~4194304,314572800~12582912,339738624~4194304,373293056~4194304,381681664~8388608] 2024-09-13T00:40:03.658 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~4194304,12582912~8388608,33554432~12582912,88080384~8388608,155189248~16777216,176160768~4194304,184549376~4194304,239075328~8388608,264241152~4194304,281018368~4194304,314572800~12582912,339738624~4194304,373293056~4194304,381681664~8388608] 2024-09-13T00:40:03.662 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~8388608,12582912~8388608,33554432~16777216,88080384~12582912,155189248~16777216,176160768~4194304,184549376~8388608,239075328~8388608,264241152~8388608,281018368~4194304,314572800~16777216,339738624~4194304,373293056~4194304,381681664~12582912] 2024-09-13T00:40:03.662 INFO:tasks.workunit.client.0.smithi136.stdout:from 15 to 18 diff [0~8388608,12582912~8388608,33554432~16777216,88080384~8388608,130023424~4194304,155189248~16777216,176160768~4194304,184549376~4194304,234881024~12582912,251658240~4194304,264241152~4194304,276824064~8388608,314572800~12582912,339738624~4194304,364904448~4194304,373293056~4194304,381681664~12582912] 2024-09-13T00:40:03.662 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,12582912~8388608,33554432~16777216,88080384~8388608,130023424~4194304,155189248~16777216,176160768~4194304,184549376~4194304,234881024~12582912,251658240~4194304,264241152~4194304,276824064~8388608,314572800~12582912,339738624~4194304,364904448~4194304,373293056~4194304,381681664~12582912] 2024-09-13T00:40:03.666 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~8388608,12582912~8388608,33554432~20971520,88080384~12582912,130023424~4194304,155189248~16777216,176160768~4194304,184549376~8388608,234881024~12582912,251658240~4194304,264241152~8388608,276824064~12582912,314572800~16777216,339738624~4194304,364904448~4194304,373293056~4194304,381681664~12582912] 2024-09-13T00:40:03.667 INFO:tasks.workunit.client.0.smithi136.stdout:from 15 to 19 diff [0~25165824,33554432~16777216,67108864~4194304,88080384~8388608,130023424~4194304,146800640~4194304,155189248~16777216,176160768~4194304,184549376~8388608,230686720~12582912,251658240~4194304,264241152~4194304,276824064~8388608,301989888~4194304,314572800~12582912,339738624~4194304,356515840~4194304,364904448~4194304,373293056~4194304,381681664~12582912,406847488~4194304] 2024-09-13T00:40:03.667 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,33554432~16777216,67108864~4194304,88080384~8388608,130023424~4194304,146800640~4194304,155189248~16777216,176160768~4194304,184549376~8388608,230686720~12582912,251658240~4194304,264241152~4194304,276824064~8388608,301989888~4194304,314572800~12582912,339738624~4194304,356515840~4194304,364904448~4194304,373293056~4194304,381681664~12582912,406847488~4194304] 2024-09-13T00:40:03.672 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~4194304,88080384~12582912,130023424~4194304,146800640~4194304,155189248~16777216,176160768~4194304,184549376~8388608,230686720~16777216,251658240~4194304,264241152~8388608,276824064~12582912,301989888~8388608,314572800~16777216,339738624~4194304,356515840~4194304,364904448~4194304,373293056~4194304,381681664~12582912,406847488~8388608] 2024-09-13T00:40:03.673 INFO:tasks.workunit.client.0.smithi136.stdout:from 16 to 17 diff [0~4194304,12582912~8388608,37748736~8388608,163577856~8388608,239075328~8388608,264241152~4194304,318767104~4194304,339738624~4194304,373293056~4194304] 2024-09-13T00:40:03.673 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~4194304,12582912~8388608,37748736~8388608,163577856~8388608,239075328~8388608,264241152~4194304,318767104~4194304,339738624~4194304,373293056~4194304] 2024-09-13T00:40:03.676 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~8388608,12582912~8388608,37748736~12582912,163577856~8388608,239075328~8388608,264241152~8388608,318767104~4194304,339738624~4194304,373293056~4194304] 2024-09-13T00:40:03.676 INFO:tasks.workunit.client.0.smithi136.stdout:from 16 to 18 diff [0~8388608,12582912~8388608,37748736~12582912,130023424~4194304,163577856~8388608,234881024~12582912,251658240~4194304,264241152~4194304,276824064~8388608,318767104~4194304,339738624~4194304,364904448~4194304,373293056~4194304,390070272~4194304] 2024-09-13T00:40:03.676 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,12582912~8388608,37748736~12582912,130023424~4194304,163577856~8388608,234881024~12582912,251658240~4194304,264241152~4194304,276824064~8388608,318767104~4194304,339738624~4194304,364904448~4194304,373293056~4194304,390070272~4194304] 2024-09-13T00:40:03.680 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~8388608,12582912~8388608,37748736~16777216,130023424~4194304,163577856~8388608,234881024~12582912,251658240~4194304,264241152~8388608,276824064~12582912,318767104~4194304,339738624~4194304,364904448~4194304,373293056~4194304,390070272~4194304] 2024-09-13T00:40:03.680 INFO:tasks.workunit.client.0.smithi136.stdout:from 16 to 19 diff [0~25165824,37748736~12582912,67108864~4194304,130023424~4194304,146800640~4194304,163577856~8388608,184549376~8388608,230686720~12582912,251658240~4194304,264241152~4194304,276824064~8388608,301989888~4194304,318767104~4194304,339738624~4194304,356515840~4194304,364904448~4194304,373293056~4194304,390070272~4194304,406847488~4194304] 2024-09-13T00:40:03.680 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,37748736~12582912,67108864~4194304,130023424~4194304,146800640~4194304,163577856~8388608,184549376~8388608,230686720~12582912,251658240~4194304,264241152~4194304,276824064~8388608,301989888~4194304,318767104~4194304,339738624~4194304,356515840~4194304,364904448~4194304,373293056~4194304,390070272~4194304,406847488~4194304] 2024-09-13T00:40:03.687 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,37748736~16777216,67108864~4194304,130023424~4194304,146800640~4194304,163577856~8388608,184549376~8388608,230686720~16777216,251658240~4194304,264241152~8388608,276824064~12582912,301989888~8388608,318767104~4194304,339738624~4194304,356515840~4194304,364904448~4194304,373293056~4194304,390070272~4194304,406847488~8388608] 2024-09-13T00:40:03.687 INFO:tasks.workunit.client.0.smithi136.stdout:from 17 to 18 diff [0~8388608,46137344~8388608,130023424~4194304,234881024~4194304,251658240~4194304,276824064~8388608,364904448~4194304,390070272~4194304] 2024-09-13T00:40:03.687 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~8388608,46137344~8388608,130023424~4194304,234881024~4194304,251658240~4194304,276824064~8388608,364904448~4194304,390070272~4194304] 2024-09-13T00:40:03.688 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~8388608,46137344~8388608,130023424~4194304,234881024~8388608,251658240~4194304,276824064~12582912,364904448~4194304,390070272~4194304] 2024-09-13T00:40:03.688 INFO:tasks.workunit.client.0.smithi136.stdout:from 17 to 19 diff [0~12582912,16777216~8388608,46137344~8388608,67108864~4194304,130023424~4194304,146800640~4194304,184549376~8388608,230686720~8388608,251658240~4194304,276824064~8388608,301989888~4194304,356515840~4194304,364904448~4194304,390070272~4194304,406847488~4194304] 2024-09-13T00:40:03.689 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~12582912,16777216~8388608,46137344~8388608,67108864~4194304,130023424~4194304,146800640~4194304,184549376~8388608,230686720~8388608,251658240~4194304,276824064~8388608,301989888~4194304,356515840~4194304,364904448~4194304,390070272~4194304,406847488~4194304] 2024-09-13T00:40:03.693 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~12582912,16777216~12582912,46137344~8388608,67108864~4194304,130023424~4194304,146800640~4194304,184549376~8388608,230686720~12582912,251658240~4194304,276824064~12582912,301989888~8388608,356515840~4194304,364904448~4194304,390070272~4194304,406847488~8388608] 2024-09-13T00:40:03.693 INFO:tasks.workunit.client.0.smithi136.stdout:from 18 to 19 diff [4194304~8388608,16777216~8388608,67108864~4194304,146800640~4194304,184549376~8388608,230686720~8388608,301989888~4194304,356515840~4194304,406847488~4194304] 2024-09-13T00:40:03.693 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [4194304~8388608,16777216~8388608,67108864~4194304,146800640~4194304,184549376~8388608,230686720~8388608,301989888~4194304,356515840~4194304,406847488~4194304] 2024-09-13T00:40:03.696 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [4194304~8388608,16777216~12582912,67108864~4194304,146800640~4194304,184549376~8388608,230686720~12582912,301989888~8388608,356515840~4194304,406847488~8388608] 2024-09-13T00:40:03.939 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to HEAD diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:03.939 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:03.992 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:03.993 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to HEAD diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:03.993 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:04.011 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:04.011 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to HEAD diff [0~50331648,54525952~25165824,83886080~29360128,117440512~50331648,171966464~25165824,201326592~4194304,209715200~37748736,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:04.012 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~50331648,54525952~25165824,83886080~29360128,117440512~50331648,171966464~25165824,201326592~4194304,209715200~37748736,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:04.029 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:04.030 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to HEAD diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:04.030 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:04.047 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:04.048 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to HEAD diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:04.048 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:04.064 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~222298112,226492416~188743680] 2024-09-13T00:40:04.064 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to HEAD diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:04.064 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:04.079 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~50331648,88080384~134217728,226492416~188743680] 2024-09-13T00:40:04.080 INFO:tasks.workunit.client.0.smithi136.stdout:from 6 to HEAD diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:04.080 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:04.094 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~50331648,88080384~134217728,226492416~20971520,251658240~163577856] 2024-09-13T00:40:04.094 INFO:tasks.workunit.client.0.smithi136.stdout:from 7 to HEAD diff [0~29360128,33554432~16777216,58720256~4194304,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~46137344,398458880~12582912] 2024-09-13T00:40:04.094 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,58720256~4194304,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~46137344,398458880~12582912] 2024-09-13T00:40:04.107 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,58720256~25165824,88080384~134217728,226492416~20971520,251658240~92274688,348127232~46137344,398458880~16777216] 2024-09-13T00:40:04.107 INFO:tasks.workunit.client.0.smithi136.stdout:from 8 to HEAD diff [0~29360128,33554432~16777216,67108864~12582912,88080384~20971520,117440512~8388608,130023424~67108864,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~46137344,398458880~12582912] 2024-09-13T00:40:04.107 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,67108864~12582912,88080384~20971520,117440512~8388608,130023424~67108864,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~46137344,398458880~12582912] 2024-09-13T00:40:04.119 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~16777216,88080384~25165824,117440512~8388608,130023424~92274688,226492416~20971520,251658240~92274688,348127232~46137344,398458880~16777216] 2024-09-13T00:40:04.119 INFO:tasks.workunit.client.0.smithi136.stdout:from 9 to HEAD diff [0~29360128,33554432~16777216,67108864~12582912,88080384~8388608,100663296~8388608,117440512~8388608,130023424~67108864,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:04.120 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,67108864~12582912,88080384~8388608,100663296~8388608,117440512~8388608,130023424~67108864,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:04.131 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~16777216,88080384~25165824,117440512~8388608,130023424~92274688,226492416~20971520,251658240~92274688,348127232~12582912,364904448~29360128,398458880~16777216] 2024-09-13T00:40:04.131 INFO:tasks.workunit.client.0.smithi136.stdout:from 10 to HEAD diff [0~29360128,33554432~16777216,67108864~12582912,88080384~8388608,100663296~8388608,117440512~8388608,130023424~41943040,176160768~20971520,209715200~8388608,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:04.132 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,67108864~12582912,88080384~8388608,100663296~8388608,117440512~8388608,130023424~41943040,176160768~20971520,209715200~8388608,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:04.142 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~16777216,88080384~25165824,117440512~8388608,130023424~41943040,176160768~25165824,209715200~8388608,226492416~20971520,251658240~92274688,348127232~12582912,364904448~29360128,398458880~16777216] 2024-09-13T00:40:04.142 INFO:tasks.workunit.client.0.smithi136.stdout:from 11 to HEAD diff [0~25165824,33554432~16777216,67108864~12582912,88080384~8388608,117440512~8388608,130023424~12582912,146800640~25165824,176160768~20971520,209715200~8388608,230686720~12582912,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:04.142 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,33554432~16777216,67108864~12582912,88080384~8388608,117440512~8388608,130023424~12582912,146800640~25165824,176160768~20971520,209715200~8388608,230686720~12582912,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:04.152 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~16777216,88080384~12582912,117440512~8388608,130023424~41943040,176160768~25165824,209715200~8388608,230686720~16777216,251658240~92274688,348127232~12582912,364904448~29360128,398458880~16777216] 2024-09-13T00:40:04.152 INFO:tasks.workunit.client.0.smithi136.stdout:from 12 to HEAD diff [0~25165824,33554432~16777216,67108864~12582912,88080384~8388608,117440512~4194304,130023424~12582912,146800640~25165824,176160768~20971520,209715200~8388608,230686720~12582912,251658240~4194304,264241152~12582912,281018368~25165824,310378496~20971520,339738624~4194304,348127232~4194304,356515840~4194304,364904448~4194304,373293056~20971520,398458880~12582912] 2024-09-13T00:40:04.152 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,33554432~16777216,67108864~12582912,88080384~8388608,117440512~4194304,130023424~12582912,146800640~25165824,176160768~20971520,209715200~8388608,230686720~12582912,251658240~4194304,264241152~12582912,281018368~25165824,310378496~20971520,339738624~4194304,348127232~4194304,356515840~4194304,364904448~4194304,373293056~20971520,398458880~12582912] 2024-09-13T00:40:04.161 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~16777216,88080384~12582912,117440512~4194304,130023424~41943040,176160768~25165824,209715200~8388608,230686720~16777216,251658240~4194304,264241152~67108864,339738624~4194304,348127232~4194304,356515840~4194304,364904448~29360128,398458880~16777216] 2024-09-13T00:40:04.161 INFO:tasks.workunit.client.0.smithi136.stdout:from 13 to HEAD diff [0~25165824,33554432~16777216,67108864~4194304,75497472~8388608,88080384~8388608,130023424~12582912,146800640~25165824,176160768~16777216,209715200~8388608,230686720~12582912,251658240~4194304,264241152~12582912,281018368~25165824,310378496~20971520,339738624~4194304,356515840~4194304,364904448~4194304,373293056~20971520,398458880~12582912] 2024-09-13T00:40:04.161 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,33554432~16777216,67108864~4194304,75497472~8388608,88080384~8388608,130023424~12582912,146800640~25165824,176160768~16777216,209715200~8388608,230686720~12582912,251658240~4194304,264241152~12582912,281018368~25165824,310378496~20971520,339738624~4194304,356515840~4194304,364904448~4194304,373293056~20971520,398458880~12582912] 2024-09-13T00:40:04.168 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~4194304,75497472~8388608,88080384~12582912,130023424~41943040,176160768~16777216,209715200~8388608,230686720~16777216,251658240~4194304,264241152~67108864,339738624~4194304,356515840~4194304,364904448~29360128,398458880~16777216] 2024-09-13T00:40:04.168 INFO:tasks.workunit.client.0.smithi136.stdout:from 14 to HEAD diff [0~25165824,33554432~16777216,67108864~4194304,88080384~8388608,130023424~4194304,146800640~25165824,176160768~16777216,230686720~12582912,251658240~4194304,264241152~4194304,276824064~20971520,301989888~4194304,314572800~12582912,339738624~4194304,356515840~4194304,364904448~4194304,373293056~4194304,381681664~12582912,398458880~12582912] 2024-09-13T00:40:04.168 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,33554432~16777216,67108864~4194304,88080384~8388608,130023424~4194304,146800640~25165824,176160768~16777216,230686720~12582912,251658240~4194304,264241152~4194304,276824064~20971520,301989888~4194304,314572800~12582912,339738624~4194304,356515840~4194304,364904448~4194304,373293056~4194304,381681664~12582912,398458880~12582912] 2024-09-13T00:40:04.174 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~4194304,88080384~12582912,130023424~4194304,146800640~25165824,176160768~16777216,230686720~16777216,251658240~4194304,264241152~8388608,276824064~33554432,314572800~16777216,339738624~4194304,356515840~4194304,364904448~12582912,381681664~12582912,398458880~16777216] 2024-09-13T00:40:04.175 INFO:tasks.workunit.client.0.smithi136.stdout:from 15 to HEAD diff [0~25165824,33554432~16777216,67108864~4194304,88080384~8388608,130023424~4194304,146800640~4194304,155189248~16777216,176160768~4194304,184549376~8388608,230686720~12582912,251658240~4194304,264241152~4194304,276824064~8388608,301989888~4194304,314572800~12582912,339738624~4194304,356515840~4194304,364904448~4194304,373293056~4194304,381681664~12582912,406847488~4194304] 2024-09-13T00:40:04.175 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,33554432~16777216,67108864~4194304,88080384~8388608,130023424~4194304,146800640~4194304,155189248~16777216,176160768~4194304,184549376~8388608,230686720~12582912,251658240~4194304,264241152~4194304,276824064~8388608,301989888~4194304,314572800~12582912,339738624~4194304,356515840~4194304,364904448~4194304,373293056~4194304,381681664~12582912,406847488~4194304] 2024-09-13T00:40:04.180 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~4194304,88080384~12582912,130023424~4194304,146800640~4194304,155189248~16777216,176160768~4194304,184549376~8388608,230686720~16777216,251658240~4194304,264241152~8388608,276824064~12582912,301989888~8388608,314572800~16777216,339738624~4194304,356515840~4194304,364904448~4194304,373293056~4194304,381681664~12582912,406847488~8388608] 2024-09-13T00:40:04.180 INFO:tasks.workunit.client.0.smithi136.stdout:from 16 to HEAD diff [0~25165824,37748736~12582912,67108864~4194304,130023424~4194304,146800640~4194304,163577856~8388608,184549376~8388608,230686720~12582912,251658240~4194304,264241152~4194304,276824064~8388608,301989888~4194304,318767104~4194304,339738624~4194304,356515840~4194304,364904448~4194304,373293056~4194304,390070272~4194304,406847488~4194304] 2024-09-13T00:40:04.180 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,37748736~12582912,67108864~4194304,130023424~4194304,146800640~4194304,163577856~8388608,184549376~8388608,230686720~12582912,251658240~4194304,264241152~4194304,276824064~8388608,301989888~4194304,318767104~4194304,339738624~4194304,356515840~4194304,364904448~4194304,373293056~4194304,390070272~4194304,406847488~4194304] 2024-09-13T00:40:04.184 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,37748736~16777216,67108864~4194304,130023424~4194304,146800640~4194304,163577856~8388608,184549376~8388608,230686720~16777216,251658240~4194304,264241152~8388608,276824064~12582912,301989888~8388608,318767104~4194304,339738624~4194304,356515840~4194304,364904448~4194304,373293056~4194304,390070272~4194304,406847488~8388608] 2024-09-13T00:40:04.184 INFO:tasks.workunit.client.0.smithi136.stdout:from 17 to HEAD diff [0~12582912,16777216~8388608,46137344~8388608,67108864~4194304,130023424~4194304,146800640~4194304,184549376~8388608,230686720~8388608,251658240~4194304,276824064~8388608,301989888~4194304,356515840~4194304,364904448~4194304,390070272~4194304,406847488~4194304] 2024-09-13T00:40:04.184 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~12582912,16777216~8388608,46137344~8388608,67108864~4194304,130023424~4194304,146800640~4194304,184549376~8388608,230686720~8388608,251658240~4194304,276824064~8388608,301989888~4194304,356515840~4194304,364904448~4194304,390070272~4194304,406847488~4194304] 2024-09-13T00:40:04.187 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~12582912,16777216~12582912,46137344~8388608,67108864~4194304,130023424~4194304,146800640~4194304,184549376~8388608,230686720~12582912,251658240~4194304,276824064~12582912,301989888~8388608,356515840~4194304,364904448~4194304,390070272~4194304,406847488~8388608] 2024-09-13T00:40:04.939 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to HEAD diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:05.071 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:05.072 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:05.073 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to HEAD diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:05.073 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:05.073 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:05.073 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to HEAD diff [0~50331648,54525952~25165824,83886080~29360128,117440512~50331648,171966464~25165824,201326592~4194304,209715200~37748736,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:05.073 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~50331648,54525952~25165824,83886080~29360128,117440512~50331648,171966464~25165824,201326592~4194304,209715200~37748736,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:05.073 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:05.073 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to HEAD diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:05.073 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:05.073 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:05.073 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to HEAD diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:05.073 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:05.073 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~222298112,226492416~188743680] 2024-09-13T00:40:05.073 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to HEAD diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:05.073 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:05.082 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~50331648,88080384~134217728,226492416~188743680] 2024-09-13T00:40:05.082 INFO:tasks.workunit.client.0.smithi136.stdout:from 6 to HEAD diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:05.083 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:05.096 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~50331648,88080384~134217728,226492416~20971520,251658240~163577856] 2024-09-13T00:40:05.096 INFO:tasks.workunit.client.0.smithi136.stdout:from 7 to HEAD diff [0~29360128,33554432~16777216,58720256~4194304,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~46137344,398458880~12582912] 2024-09-13T00:40:05.097 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,58720256~4194304,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~46137344,398458880~12582912] 2024-09-13T00:40:05.106 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,58720256~25165824,88080384~134217728,226492416~20971520,251658240~92274688,348127232~46137344,398458880~16777216] 2024-09-13T00:40:05.106 INFO:tasks.workunit.client.0.smithi136.stdout:from 8 to HEAD diff [0~29360128,33554432~16777216,67108864~12582912,88080384~20971520,117440512~8388608,130023424~67108864,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~46137344,398458880~12582912] 2024-09-13T00:40:05.106 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,67108864~12582912,88080384~20971520,117440512~8388608,130023424~67108864,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~46137344,398458880~12582912] 2024-09-13T00:40:05.116 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~16777216,88080384~25165824,117440512~8388608,130023424~92274688,226492416~20971520,251658240~92274688,348127232~46137344,398458880~16777216] 2024-09-13T00:40:05.116 INFO:tasks.workunit.client.0.smithi136.stdout:from 9 to HEAD diff [0~29360128,33554432~16777216,67108864~12582912,88080384~8388608,100663296~8388608,117440512~8388608,130023424~67108864,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:05.116 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,67108864~12582912,88080384~8388608,100663296~8388608,117440512~8388608,130023424~67108864,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:05.126 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~16777216,88080384~25165824,117440512~8388608,130023424~92274688,226492416~20971520,251658240~92274688,348127232~12582912,364904448~29360128,398458880~16777216] 2024-09-13T00:40:05.126 INFO:tasks.workunit.client.0.smithi136.stdout:from 10 to HEAD diff [0~29360128,33554432~16777216,67108864~12582912,88080384~8388608,100663296~8388608,117440512~8388608,130023424~41943040,176160768~20971520,209715200~8388608,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:05.127 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,67108864~12582912,88080384~8388608,100663296~8388608,117440512~8388608,130023424~41943040,176160768~20971520,209715200~8388608,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:05.135 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~16777216,88080384~25165824,117440512~8388608,130023424~41943040,176160768~25165824,209715200~8388608,226492416~20971520,251658240~92274688,348127232~12582912,364904448~29360128,398458880~16777216] 2024-09-13T00:40:05.135 INFO:tasks.workunit.client.0.smithi136.stdout:from 11 to HEAD diff [0~25165824,33554432~16777216,67108864~12582912,88080384~8388608,117440512~8388608,130023424~12582912,146800640~25165824,176160768~20971520,209715200~8388608,230686720~12582912,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:05.135 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,33554432~16777216,67108864~12582912,88080384~8388608,117440512~8388608,130023424~12582912,146800640~25165824,176160768~20971520,209715200~8388608,230686720~12582912,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:05.142 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~16777216,88080384~12582912,117440512~8388608,130023424~41943040,176160768~25165824,209715200~8388608,230686720~16777216,251658240~92274688,348127232~12582912,364904448~29360128,398458880~16777216] 2024-09-13T00:40:05.143 INFO:tasks.workunit.client.0.smithi136.stdout:from 12 to HEAD diff [0~25165824,33554432~16777216,67108864~12582912,88080384~8388608,117440512~4194304,130023424~12582912,146800640~25165824,176160768~20971520,209715200~8388608,230686720~12582912,251658240~4194304,264241152~12582912,281018368~25165824,310378496~20971520,339738624~4194304,348127232~4194304,356515840~4194304,364904448~4194304,373293056~20971520,398458880~12582912] 2024-09-13T00:40:05.143 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,33554432~16777216,67108864~12582912,88080384~8388608,117440512~4194304,130023424~12582912,146800640~25165824,176160768~20971520,209715200~8388608,230686720~12582912,251658240~4194304,264241152~12582912,281018368~25165824,310378496~20971520,339738624~4194304,348127232~4194304,356515840~4194304,364904448~4194304,373293056~20971520,398458880~12582912] 2024-09-13T00:40:05.150 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~16777216,88080384~12582912,117440512~4194304,130023424~41943040,176160768~25165824,209715200~8388608,230686720~16777216,251658240~4194304,264241152~67108864,339738624~4194304,348127232~4194304,356515840~4194304,364904448~29360128,398458880~16777216] 2024-09-13T00:40:05.150 INFO:tasks.workunit.client.0.smithi136.stdout:from 13 to HEAD diff [0~25165824,33554432~16777216,67108864~4194304,75497472~8388608,88080384~8388608,130023424~12582912,146800640~25165824,176160768~16777216,209715200~8388608,230686720~12582912,251658240~4194304,264241152~12582912,281018368~25165824,310378496~20971520,339738624~4194304,356515840~4194304,364904448~4194304,373293056~20971520,398458880~12582912] 2024-09-13T00:40:05.150 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,33554432~16777216,67108864~4194304,75497472~8388608,88080384~8388608,130023424~12582912,146800640~25165824,176160768~16777216,209715200~8388608,230686720~12582912,251658240~4194304,264241152~12582912,281018368~25165824,310378496~20971520,339738624~4194304,356515840~4194304,364904448~4194304,373293056~20971520,398458880~12582912] 2024-09-13T00:40:05.155 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~4194304,75497472~8388608,88080384~12582912,130023424~41943040,176160768~16777216,209715200~8388608,230686720~16777216,251658240~4194304,264241152~67108864,339738624~4194304,356515840~4194304,364904448~29360128,398458880~16777216] 2024-09-13T00:40:05.156 INFO:tasks.workunit.client.0.smithi136.stdout:from 14 to HEAD diff [0~25165824,33554432~16777216,67108864~4194304,88080384~8388608,130023424~4194304,146800640~25165824,176160768~16777216,230686720~12582912,251658240~4194304,264241152~4194304,276824064~20971520,301989888~4194304,314572800~12582912,339738624~4194304,356515840~4194304,364904448~4194304,373293056~4194304,381681664~12582912,398458880~12582912] 2024-09-13T00:40:05.156 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,33554432~16777216,67108864~4194304,88080384~8388608,130023424~4194304,146800640~25165824,176160768~16777216,230686720~12582912,251658240~4194304,264241152~4194304,276824064~20971520,301989888~4194304,314572800~12582912,339738624~4194304,356515840~4194304,364904448~4194304,373293056~4194304,381681664~12582912,398458880~12582912] 2024-09-13T00:40:05.160 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~4194304,88080384~12582912,130023424~4194304,146800640~25165824,176160768~16777216,230686720~16777216,251658240~4194304,264241152~8388608,276824064~33554432,314572800~16777216,339738624~4194304,356515840~4194304,364904448~12582912,381681664~12582912,398458880~16777216] 2024-09-13T00:40:05.160 INFO:tasks.workunit.client.0.smithi136.stdout:from 15 to HEAD diff [0~25165824,33554432~16777216,67108864~4194304,88080384~8388608,130023424~4194304,146800640~4194304,155189248~16777216,176160768~4194304,184549376~8388608,230686720~12582912,251658240~4194304,264241152~4194304,276824064~8388608,301989888~4194304,314572800~12582912,339738624~4194304,356515840~4194304,364904448~4194304,373293056~4194304,381681664~12582912,406847488~4194304] 2024-09-13T00:40:05.160 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,33554432~16777216,67108864~4194304,88080384~8388608,130023424~4194304,146800640~4194304,155189248~16777216,176160768~4194304,184549376~8388608,230686720~12582912,251658240~4194304,264241152~4194304,276824064~8388608,301989888~4194304,314572800~12582912,339738624~4194304,356515840~4194304,364904448~4194304,373293056~4194304,381681664~12582912,406847488~4194304] 2024-09-13T00:40:05.164 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~4194304,88080384~12582912,130023424~4194304,146800640~4194304,155189248~16777216,176160768~4194304,184549376~8388608,230686720~16777216,251658240~4194304,264241152~8388608,276824064~12582912,301989888~8388608,314572800~16777216,339738624~4194304,356515840~4194304,364904448~4194304,373293056~4194304,381681664~12582912,406847488~8388608] 2024-09-13T00:40:05.164 INFO:tasks.workunit.client.0.smithi136.stdout:from 16 to HEAD diff [0~25165824,37748736~12582912,67108864~4194304,130023424~4194304,146800640~4194304,163577856~8388608,184549376~8388608,230686720~12582912,251658240~4194304,264241152~4194304,276824064~8388608,301989888~4194304,318767104~4194304,339738624~4194304,356515840~4194304,364904448~4194304,373293056~4194304,390070272~4194304,406847488~4194304] 2024-09-13T00:40:05.164 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,37748736~12582912,67108864~4194304,130023424~4194304,146800640~4194304,163577856~8388608,184549376~8388608,230686720~12582912,251658240~4194304,264241152~4194304,276824064~8388608,301989888~4194304,318767104~4194304,339738624~4194304,356515840~4194304,364904448~4194304,373293056~4194304,390070272~4194304,406847488~4194304] 2024-09-13T00:40:05.166 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,37748736~16777216,67108864~4194304,130023424~4194304,146800640~4194304,163577856~8388608,184549376~8388608,230686720~16777216,251658240~4194304,264241152~8388608,276824064~12582912,301989888~8388608,318767104~4194304,339738624~4194304,356515840~4194304,364904448~4194304,373293056~4194304,390070272~4194304,406847488~8388608] 2024-09-13T00:40:05.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:04 smithi136 bash[75164]: cluster 2024-09-13T00:40:03.918668+0000 mon.a (mon.0) 3642 : cluster [DBG] osdmap e1980: 8 total, 8 up, 8 in 2024-09-13T00:40:05.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:04 smithi136 bash[75164]: cluster 2024-09-13T00:40:04.446451+0000 mgr.y (mgr.24661) 1888 : cluster [DBG] pgmap v3429: 137 pgs: 137 active+clean; 809 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 261 KiB/s rd, 23 MiB/s wr, 420 op/s 2024-09-13T00:40:05.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:04 smithi136 bash[76371]: cluster 2024-09-13T00:40:03.918668+0000 mon.a (mon.0) 3642 : cluster [DBG] osdmap e1980: 8 total, 8 up, 8 in 2024-09-13T00:40:05.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:04 smithi136 bash[76371]: cluster 2024-09-13T00:40:04.446451+0000 mgr.y (mgr.24661) 1888 : cluster [DBG] pgmap v3429: 137 pgs: 137 active+clean; 809 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 261 KiB/s rd, 23 MiB/s wr, 420 op/s 2024-09-13T00:40:05.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:04 smithi181 bash[67587]: cluster 2024-09-13T00:40:03.918668+0000 mon.a (mon.0) 3642 : cluster [DBG] osdmap e1980: 8 total, 8 up, 8 in 2024-09-13T00:40:05.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:04 smithi181 bash[67587]: cluster 2024-09-13T00:40:04.446451+0000 mgr.y (mgr.24661) 1888 : cluster [DBG] pgmap v3429: 137 pgs: 137 active+clean; 809 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 261 KiB/s rd, 23 MiB/s wr, 420 op/s 2024-09-13T00:40:05.981 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to HEAD diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:05.981 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:06.021 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:06.021 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to HEAD diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:06.021 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:06.033 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:06.033 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to HEAD diff [0~50331648,54525952~25165824,83886080~29360128,117440512~50331648,171966464~25165824,201326592~4194304,209715200~37748736,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:06.033 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~50331648,54525952~25165824,83886080~29360128,117440512~50331648,171966464~25165824,201326592~4194304,209715200~37748736,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:06.043 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:06.043 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to HEAD diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:06.043 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:06.054 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:06.054 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to HEAD diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:06.055 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:06.064 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~222298112,226492416~188743680] 2024-09-13T00:40:06.064 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to HEAD diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:06.065 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:06.075 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~50331648,88080384~134217728,226492416~188743680] 2024-09-13T00:40:06.076 INFO:tasks.workunit.client.0.smithi136.stdout:from 6 to HEAD diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:06.076 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:06.086 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~50331648,88080384~134217728,226492416~20971520,251658240~163577856] 2024-09-13T00:40:06.086 INFO:tasks.workunit.client.0.smithi136.stdout:from 7 to HEAD diff [0~29360128,33554432~16777216,58720256~4194304,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~46137344,398458880~12582912] 2024-09-13T00:40:06.086 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,58720256~4194304,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~46137344,398458880~12582912] 2024-09-13T00:40:06.096 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,58720256~25165824,88080384~134217728,226492416~20971520,251658240~92274688,348127232~46137344,398458880~16777216] 2024-09-13T00:40:06.097 INFO:tasks.workunit.client.0.smithi136.stdout:from 8 to HEAD diff [0~29360128,33554432~16777216,67108864~12582912,88080384~20971520,117440512~8388608,130023424~67108864,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~46137344,398458880~12582912] 2024-09-13T00:40:06.097 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,67108864~12582912,88080384~20971520,117440512~8388608,130023424~67108864,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~46137344,398458880~12582912] 2024-09-13T00:40:06.106 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~16777216,88080384~25165824,117440512~8388608,130023424~92274688,226492416~20971520,251658240~92274688,348127232~46137344,398458880~16777216] 2024-09-13T00:40:06.107 INFO:tasks.workunit.client.0.smithi136.stdout:from 9 to HEAD diff [0~29360128,33554432~16777216,67108864~12582912,88080384~8388608,100663296~8388608,117440512~8388608,130023424~67108864,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:06.107 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,67108864~12582912,88080384~8388608,100663296~8388608,117440512~8388608,130023424~67108864,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:06.115 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~16777216,88080384~25165824,117440512~8388608,130023424~92274688,226492416~20971520,251658240~92274688,348127232~12582912,364904448~29360128,398458880~16777216] 2024-09-13T00:40:06.115 INFO:tasks.workunit.client.0.smithi136.stdout:from 10 to HEAD diff [0~29360128,33554432~16777216,67108864~12582912,88080384~8388608,100663296~8388608,117440512~8388608,130023424~41943040,176160768~20971520,209715200~8388608,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:06.115 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,67108864~12582912,88080384~8388608,100663296~8388608,117440512~8388608,130023424~41943040,176160768~20971520,209715200~8388608,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:06.122 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~16777216,88080384~25165824,117440512~8388608,130023424~41943040,176160768~25165824,209715200~8388608,226492416~20971520,251658240~92274688,348127232~12582912,364904448~29360128,398458880~16777216] 2024-09-13T00:40:06.122 INFO:tasks.workunit.client.0.smithi136.stdout:from 11 to HEAD diff [0~25165824,33554432~16777216,67108864~12582912,88080384~8388608,117440512~8388608,130023424~12582912,146800640~25165824,176160768~20971520,209715200~8388608,230686720~12582912,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:06.122 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,33554432~16777216,67108864~12582912,88080384~8388608,117440512~8388608,130023424~12582912,146800640~25165824,176160768~20971520,209715200~8388608,230686720~12582912,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:06.129 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~16777216,88080384~12582912,117440512~8388608,130023424~41943040,176160768~25165824,209715200~8388608,230686720~16777216,251658240~92274688,348127232~12582912,364904448~29360128,398458880~16777216] 2024-09-13T00:40:06.129 INFO:tasks.workunit.client.0.smithi136.stdout:from 12 to HEAD diff [0~25165824,33554432~16777216,67108864~12582912,88080384~8388608,117440512~4194304,130023424~12582912,146800640~25165824,176160768~20971520,209715200~8388608,230686720~12582912,251658240~4194304,264241152~12582912,281018368~25165824,310378496~20971520,339738624~4194304,348127232~4194304,356515840~4194304,364904448~4194304,373293056~20971520,398458880~12582912] 2024-09-13T00:40:06.129 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,33554432~16777216,67108864~12582912,88080384~8388608,117440512~4194304,130023424~12582912,146800640~25165824,176160768~20971520,209715200~8388608,230686720~12582912,251658240~4194304,264241152~12582912,281018368~25165824,310378496~20971520,339738624~4194304,348127232~4194304,356515840~4194304,364904448~4194304,373293056~20971520,398458880~12582912] 2024-09-13T00:40:06.134 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~16777216,88080384~12582912,117440512~4194304,130023424~41943040,176160768~25165824,209715200~8388608,230686720~16777216,251658240~4194304,264241152~67108864,339738624~4194304,348127232~4194304,356515840~4194304,364904448~29360128,398458880~16777216] 2024-09-13T00:40:06.135 INFO:tasks.workunit.client.0.smithi136.stdout:from 13 to HEAD diff [0~25165824,33554432~16777216,67108864~4194304,75497472~8388608,88080384~8388608,130023424~12582912,146800640~25165824,176160768~16777216,209715200~8388608,230686720~12582912,251658240~4194304,264241152~12582912,281018368~25165824,310378496~20971520,339738624~4194304,356515840~4194304,364904448~4194304,373293056~20971520,398458880~12582912] 2024-09-13T00:40:06.135 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,33554432~16777216,67108864~4194304,75497472~8388608,88080384~8388608,130023424~12582912,146800640~25165824,176160768~16777216,209715200~8388608,230686720~12582912,251658240~4194304,264241152~12582912,281018368~25165824,310378496~20971520,339738624~4194304,356515840~4194304,364904448~4194304,373293056~20971520,398458880~12582912] 2024-09-13T00:40:06.140 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~4194304,75497472~8388608,88080384~12582912,130023424~41943040,176160768~16777216,209715200~8388608,230686720~16777216,251658240~4194304,264241152~67108864,339738624~4194304,356515840~4194304,364904448~29360128,398458880~16777216] 2024-09-13T00:40:06.140 INFO:tasks.workunit.client.0.smithi136.stdout:from 14 to HEAD diff [0~25165824,33554432~16777216,67108864~4194304,88080384~8388608,130023424~4194304,146800640~25165824,176160768~16777216,230686720~12582912,251658240~4194304,264241152~4194304,276824064~20971520,301989888~4194304,314572800~12582912,339738624~4194304,356515840~4194304,364904448~4194304,373293056~4194304,381681664~12582912,398458880~12582912] 2024-09-13T00:40:06.140 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,33554432~16777216,67108864~4194304,88080384~8388608,130023424~4194304,146800640~25165824,176160768~16777216,230686720~12582912,251658240~4194304,264241152~4194304,276824064~20971520,301989888~4194304,314572800~12582912,339738624~4194304,356515840~4194304,364904448~4194304,373293056~4194304,381681664~12582912,398458880~12582912] 2024-09-13T00:40:06.144 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~4194304,88080384~12582912,130023424~4194304,146800640~25165824,176160768~16777216,230686720~16777216,251658240~4194304,264241152~8388608,276824064~33554432,314572800~16777216,339738624~4194304,356515840~4194304,364904448~12582912,381681664~12582912,398458880~16777216] 2024-09-13T00:40:06.144 INFO:tasks.workunit.client.0.smithi136.stdout:from 15 to HEAD diff [0~25165824,33554432~16777216,67108864~4194304,88080384~8388608,130023424~4194304,146800640~4194304,155189248~16777216,176160768~4194304,184549376~8388608,230686720~12582912,251658240~4194304,264241152~4194304,276824064~8388608,301989888~4194304,314572800~12582912,339738624~4194304,356515840~4194304,364904448~4194304,373293056~4194304,381681664~12582912,406847488~4194304] 2024-09-13T00:40:06.144 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,33554432~16777216,67108864~4194304,88080384~8388608,130023424~4194304,146800640~4194304,155189248~16777216,176160768~4194304,184549376~8388608,230686720~12582912,251658240~4194304,264241152~4194304,276824064~8388608,301989888~4194304,314572800~12582912,339738624~4194304,356515840~4194304,364904448~4194304,373293056~4194304,381681664~12582912,406847488~4194304] 2024-09-13T00:40:06.147 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~4194304,88080384~12582912,130023424~4194304,146800640~4194304,155189248~16777216,176160768~4194304,184549376~8388608,230686720~16777216,251658240~4194304,264241152~8388608,276824064~12582912,301989888~8388608,314572800~16777216,339738624~4194304,356515840~4194304,364904448~4194304,373293056~4194304,381681664~12582912,406847488~8388608] 2024-09-13T00:40:06.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:05 smithi136 bash[75164]: cluster 2024-09-13T00:40:04.930659+0000 mon.a (mon.0) 3643 : cluster [DBG] osdmap e1981: 8 total, 8 up, 8 in 2024-09-13T00:40:06.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:05 smithi136 bash[76371]: cluster 2024-09-13T00:40:04.930659+0000 mon.a (mon.0) 3643 : cluster [DBG] osdmap e1981: 8 total, 8 up, 8 in 2024-09-13T00:40:06.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:05 smithi181 bash[67587]: cluster 2024-09-13T00:40:04.930659+0000 mon.a (mon.0) 3643 : cluster [DBG] osdmap e1981: 8 total, 8 up, 8 in 2024-09-13T00:40:06.996 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to HEAD diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:06.996 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:07.051 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:07.052 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to HEAD diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:07.052 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:07.062 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:07.063 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to HEAD diff [0~50331648,54525952~25165824,83886080~29360128,117440512~50331648,171966464~25165824,201326592~4194304,209715200~37748736,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:07.063 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~50331648,54525952~25165824,83886080~29360128,117440512~50331648,171966464~25165824,201326592~4194304,209715200~37748736,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:07.073 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:07.073 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to HEAD diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:07.073 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:07.084 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:07.084 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to HEAD diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:07.085 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:07.094 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~222298112,226492416~188743680] 2024-09-13T00:40:07.094 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to HEAD diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:07.094 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:07.103 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~50331648,88080384~134217728,226492416~188743680] 2024-09-13T00:40:07.103 INFO:tasks.workunit.client.0.smithi136.stdout:from 6 to HEAD diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:07.103 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:07.111 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~50331648,88080384~134217728,226492416~20971520,251658240~163577856] 2024-09-13T00:40:07.111 INFO:tasks.workunit.client.0.smithi136.stdout:from 7 to HEAD diff [0~29360128,33554432~16777216,58720256~4194304,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~46137344,398458880~12582912] 2024-09-13T00:40:07.111 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,58720256~4194304,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~46137344,398458880~12582912] 2024-09-13T00:40:07.120 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,58720256~25165824,88080384~134217728,226492416~20971520,251658240~92274688,348127232~46137344,398458880~16777216] 2024-09-13T00:40:07.121 INFO:tasks.workunit.client.0.smithi136.stdout:from 8 to HEAD diff [0~29360128,33554432~16777216,67108864~12582912,88080384~20971520,117440512~8388608,130023424~67108864,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~46137344,398458880~12582912] 2024-09-13T00:40:07.121 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,67108864~12582912,88080384~20971520,117440512~8388608,130023424~67108864,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~46137344,398458880~12582912] 2024-09-13T00:40:07.127 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~16777216,88080384~25165824,117440512~8388608,130023424~92274688,226492416~20971520,251658240~92274688,348127232~46137344,398458880~16777216] 2024-09-13T00:40:07.127 INFO:tasks.workunit.client.0.smithi136.stdout:from 9 to HEAD diff [0~29360128,33554432~16777216,67108864~12582912,88080384~8388608,100663296~8388608,117440512~8388608,130023424~67108864,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:07.127 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,67108864~12582912,88080384~8388608,100663296~8388608,117440512~8388608,130023424~67108864,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:07.135 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~16777216,88080384~25165824,117440512~8388608,130023424~92274688,226492416~20971520,251658240~92274688,348127232~12582912,364904448~29360128,398458880~16777216] 2024-09-13T00:40:07.135 INFO:tasks.workunit.client.0.smithi136.stdout:from 10 to HEAD diff [0~29360128,33554432~16777216,67108864~12582912,88080384~8388608,100663296~8388608,117440512~8388608,130023424~41943040,176160768~20971520,209715200~8388608,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:07.135 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,67108864~12582912,88080384~8388608,100663296~8388608,117440512~8388608,130023424~41943040,176160768~20971520,209715200~8388608,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:07.140 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~16777216,88080384~25165824,117440512~8388608,130023424~41943040,176160768~25165824,209715200~8388608,226492416~20971520,251658240~92274688,348127232~12582912,364904448~29360128,398458880~16777216] 2024-09-13T00:40:07.140 INFO:tasks.workunit.client.0.smithi136.stdout:from 11 to HEAD diff [0~25165824,33554432~16777216,67108864~12582912,88080384~8388608,117440512~8388608,130023424~12582912,146800640~25165824,176160768~20971520,209715200~8388608,230686720~12582912,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:07.140 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,33554432~16777216,67108864~12582912,88080384~8388608,117440512~8388608,130023424~12582912,146800640~25165824,176160768~20971520,209715200~8388608,230686720~12582912,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:07.145 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~16777216,88080384~12582912,117440512~8388608,130023424~41943040,176160768~25165824,209715200~8388608,230686720~16777216,251658240~92274688,348127232~12582912,364904448~29360128,398458880~16777216] 2024-09-13T00:40:07.145 INFO:tasks.workunit.client.0.smithi136.stdout:from 12 to HEAD diff [0~25165824,33554432~16777216,67108864~12582912,88080384~8388608,117440512~4194304,130023424~12582912,146800640~25165824,176160768~20971520,209715200~8388608,230686720~12582912,251658240~4194304,264241152~12582912,281018368~25165824,310378496~20971520,339738624~4194304,348127232~4194304,356515840~4194304,364904448~4194304,373293056~20971520,398458880~12582912] 2024-09-13T00:40:07.145 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,33554432~16777216,67108864~12582912,88080384~8388608,117440512~4194304,130023424~12582912,146800640~25165824,176160768~20971520,209715200~8388608,230686720~12582912,251658240~4194304,264241152~12582912,281018368~25165824,310378496~20971520,339738624~4194304,348127232~4194304,356515840~4194304,364904448~4194304,373293056~20971520,398458880~12582912] 2024-09-13T00:40:07.150 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~16777216,88080384~12582912,117440512~4194304,130023424~41943040,176160768~25165824,209715200~8388608,230686720~16777216,251658240~4194304,264241152~67108864,339738624~4194304,348127232~4194304,356515840~4194304,364904448~29360128,398458880~16777216] 2024-09-13T00:40:07.150 INFO:tasks.workunit.client.0.smithi136.stdout:from 13 to HEAD diff [0~25165824,33554432~16777216,67108864~4194304,75497472~8388608,88080384~8388608,130023424~12582912,146800640~25165824,176160768~16777216,209715200~8388608,230686720~12582912,251658240~4194304,264241152~12582912,281018368~25165824,310378496~20971520,339738624~4194304,356515840~4194304,364904448~4194304,373293056~20971520,398458880~12582912] 2024-09-13T00:40:07.150 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,33554432~16777216,67108864~4194304,75497472~8388608,88080384~8388608,130023424~12582912,146800640~25165824,176160768~16777216,209715200~8388608,230686720~12582912,251658240~4194304,264241152~12582912,281018368~25165824,310378496~20971520,339738624~4194304,356515840~4194304,364904448~4194304,373293056~20971520,398458880~12582912] 2024-09-13T00:40:07.154 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~4194304,75497472~8388608,88080384~12582912,130023424~41943040,176160768~16777216,209715200~8388608,230686720~16777216,251658240~4194304,264241152~67108864,339738624~4194304,356515840~4194304,364904448~29360128,398458880~16777216] 2024-09-13T00:40:07.154 INFO:tasks.workunit.client.0.smithi136.stdout:from 14 to HEAD diff [0~25165824,33554432~16777216,67108864~4194304,88080384~8388608,130023424~4194304,146800640~25165824,176160768~16777216,230686720~12582912,251658240~4194304,264241152~4194304,276824064~20971520,301989888~4194304,314572800~12582912,339738624~4194304,356515840~4194304,364904448~4194304,373293056~4194304,381681664~12582912,398458880~12582912] 2024-09-13T00:40:07.154 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,33554432~16777216,67108864~4194304,88080384~8388608,130023424~4194304,146800640~25165824,176160768~16777216,230686720~12582912,251658240~4194304,264241152~4194304,276824064~20971520,301989888~4194304,314572800~12582912,339738624~4194304,356515840~4194304,364904448~4194304,373293056~4194304,381681664~12582912,398458880~12582912] 2024-09-13T00:40:07.156 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~4194304,88080384~12582912,130023424~4194304,146800640~25165824,176160768~16777216,230686720~16777216,251658240~4194304,264241152~8388608,276824064~33554432,314572800~16777216,339738624~4194304,356515840~4194304,364904448~12582912,381681664~12582912,398458880~16777216] 2024-09-13T00:40:07.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:06 smithi136 bash[75164]: cluster 2024-09-13T00:40:05.930611+0000 mon.a (mon.0) 3644 : cluster [DBG] osdmap e1982: 8 total, 8 up, 8 in 2024-09-13T00:40:07.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:06 smithi136 bash[75164]: cluster 2024-09-13T00:40:06.447307+0000 mgr.y (mgr.24661) 1889 : cluster [DBG] pgmap v3432: 137 pgs: 137 active+clean; 835 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 327 KiB/s rd, 12 MiB/s wr, 661 op/s 2024-09-13T00:40:07.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:06 smithi136 bash[76371]: cluster 2024-09-13T00:40:05.930611+0000 mon.a (mon.0) 3644 : cluster [DBG] osdmap e1982: 8 total, 8 up, 8 in 2024-09-13T00:40:07.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:06 smithi136 bash[76371]: cluster 2024-09-13T00:40:06.447307+0000 mgr.y (mgr.24661) 1889 : cluster [DBG] pgmap v3432: 137 pgs: 137 active+clean; 835 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 327 KiB/s rd, 12 MiB/s wr, 661 op/s 2024-09-13T00:40:07.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:06 smithi181 bash[67587]: cluster 2024-09-13T00:40:05.930611+0000 mon.a (mon.0) 3644 : cluster [DBG] osdmap e1982: 8 total, 8 up, 8 in 2024-09-13T00:40:07.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:06 smithi181 bash[67587]: cluster 2024-09-13T00:40:06.447307+0000 mgr.y (mgr.24661) 1889 : cluster [DBG] pgmap v3432: 137 pgs: 137 active+clean; 835 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 327 KiB/s rd, 12 MiB/s wr, 661 op/s 2024-09-13T00:40:08.030 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to HEAD diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:08.030 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:08.063 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:08.063 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to HEAD diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:08.064 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:08.076 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:08.076 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to HEAD diff [0~50331648,54525952~25165824,83886080~29360128,117440512~50331648,171966464~25165824,201326592~4194304,209715200~37748736,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:08.076 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~50331648,54525952~25165824,83886080~29360128,117440512~50331648,171966464~25165824,201326592~4194304,209715200~37748736,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:08.084 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:08.084 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to HEAD diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:08.084 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:08.092 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:08.092 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to HEAD diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:08.092 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:08.100 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~222298112,226492416~188743680] 2024-09-13T00:40:08.101 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to HEAD diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:08.101 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:08.108 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~50331648,88080384~134217728,226492416~188743680] 2024-09-13T00:40:08.108 INFO:tasks.workunit.client.0.smithi136.stdout:from 6 to HEAD diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:08.108 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:08.115 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~50331648,88080384~134217728,226492416~20971520,251658240~163577856] 2024-09-13T00:40:08.116 INFO:tasks.workunit.client.0.smithi136.stdout:from 7 to HEAD diff [0~29360128,33554432~16777216,58720256~4194304,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~46137344,398458880~12582912] 2024-09-13T00:40:08.116 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,58720256~4194304,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~46137344,398458880~12582912] 2024-09-13T00:40:08.122 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,58720256~25165824,88080384~134217728,226492416~20971520,251658240~92274688,348127232~46137344,398458880~16777216] 2024-09-13T00:40:08.122 INFO:tasks.workunit.client.0.smithi136.stdout:from 8 to HEAD diff [0~29360128,33554432~16777216,67108864~12582912,88080384~20971520,117440512~8388608,130023424~67108864,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~46137344,398458880~12582912] 2024-09-13T00:40:08.122 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,67108864~12582912,88080384~20971520,117440512~8388608,130023424~67108864,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~46137344,398458880~12582912] 2024-09-13T00:40:08.128 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~16777216,88080384~25165824,117440512~8388608,130023424~92274688,226492416~20971520,251658240~92274688,348127232~46137344,398458880~16777216] 2024-09-13T00:40:08.128 INFO:tasks.workunit.client.0.smithi136.stdout:from 9 to HEAD diff [0~29360128,33554432~16777216,67108864~12582912,88080384~8388608,100663296~8388608,117440512~8388608,130023424~67108864,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:08.128 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,67108864~12582912,88080384~8388608,100663296~8388608,117440512~8388608,130023424~67108864,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:08.132 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~16777216,88080384~25165824,117440512~8388608,130023424~92274688,226492416~20971520,251658240~92274688,348127232~12582912,364904448~29360128,398458880~16777216] 2024-09-13T00:40:08.132 INFO:tasks.workunit.client.0.smithi136.stdout:from 10 to HEAD diff [0~29360128,33554432~16777216,67108864~12582912,88080384~8388608,100663296~8388608,117440512~8388608,130023424~41943040,176160768~20971520,209715200~8388608,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:08.133 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,67108864~12582912,88080384~8388608,100663296~8388608,117440512~8388608,130023424~41943040,176160768~20971520,209715200~8388608,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:08.137 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~16777216,88080384~25165824,117440512~8388608,130023424~41943040,176160768~25165824,209715200~8388608,226492416~20971520,251658240~92274688,348127232~12582912,364904448~29360128,398458880~16777216] 2024-09-13T00:40:08.137 INFO:tasks.workunit.client.0.smithi136.stdout:from 11 to HEAD diff [0~25165824,33554432~16777216,67108864~12582912,88080384~8388608,117440512~8388608,130023424~12582912,146800640~25165824,176160768~20971520,209715200~8388608,230686720~12582912,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:08.137 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,33554432~16777216,67108864~12582912,88080384~8388608,117440512~8388608,130023424~12582912,146800640~25165824,176160768~20971520,209715200~8388608,230686720~12582912,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:08.141 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~16777216,88080384~12582912,117440512~8388608,130023424~41943040,176160768~25165824,209715200~8388608,230686720~16777216,251658240~92274688,348127232~12582912,364904448~29360128,398458880~16777216] 2024-09-13T00:40:08.141 INFO:tasks.workunit.client.0.smithi136.stdout:from 12 to HEAD diff [0~25165824,33554432~16777216,67108864~12582912,88080384~8388608,117440512~4194304,130023424~12582912,146800640~25165824,176160768~20971520,209715200~8388608,230686720~12582912,251658240~4194304,264241152~12582912,281018368~25165824,310378496~20971520,339738624~4194304,348127232~4194304,356515840~4194304,364904448~4194304,373293056~20971520,398458880~12582912] 2024-09-13T00:40:08.141 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,33554432~16777216,67108864~12582912,88080384~8388608,117440512~4194304,130023424~12582912,146800640~25165824,176160768~20971520,209715200~8388608,230686720~12582912,251658240~4194304,264241152~12582912,281018368~25165824,310378496~20971520,339738624~4194304,348127232~4194304,356515840~4194304,364904448~4194304,373293056~20971520,398458880~12582912] 2024-09-13T00:40:08.144 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~16777216,88080384~12582912,117440512~4194304,130023424~41943040,176160768~25165824,209715200~8388608,230686720~16777216,251658240~4194304,264241152~67108864,339738624~4194304,348127232~4194304,356515840~4194304,364904448~29360128,398458880~16777216] 2024-09-13T00:40:08.145 INFO:tasks.workunit.client.0.smithi136.stdout:from 13 to HEAD diff [0~25165824,33554432~16777216,67108864~4194304,75497472~8388608,88080384~8388608,130023424~12582912,146800640~25165824,176160768~16777216,209715200~8388608,230686720~12582912,251658240~4194304,264241152~12582912,281018368~25165824,310378496~20971520,339738624~4194304,356515840~4194304,364904448~4194304,373293056~20971520,398458880~12582912] 2024-09-13T00:40:08.145 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,33554432~16777216,67108864~4194304,75497472~8388608,88080384~8388608,130023424~12582912,146800640~25165824,176160768~16777216,209715200~8388608,230686720~12582912,251658240~4194304,264241152~12582912,281018368~25165824,310378496~20971520,339738624~4194304,356515840~4194304,364904448~4194304,373293056~20971520,398458880~12582912] 2024-09-13T00:40:08.147 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~4194304,75497472~8388608,88080384~12582912,130023424~41943040,176160768~16777216,209715200~8388608,230686720~16777216,251658240~4194304,264241152~67108864,339738624~4194304,356515840~4194304,364904448~29360128,398458880~16777216] 2024-09-13T00:40:08.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:07 smithi136 bash[75164]: cluster 2024-09-13T00:40:06.958815+0000 mon.a (mon.0) 3645 : cluster [DBG] osdmap e1983: 8 total, 8 up, 8 in 2024-09-13T00:40:08.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:07 smithi136 bash[76371]: cluster 2024-09-13T00:40:06.958815+0000 mon.a (mon.0) 3645 : cluster [DBG] osdmap e1983: 8 total, 8 up, 8 in 2024-09-13T00:40:08.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:07 smithi181 bash[67587]: cluster 2024-09-13T00:40:06.958815+0000 mon.a (mon.0) 3645 : cluster [DBG] osdmap e1983: 8 total, 8 up, 8 in 2024-09-13T00:40:08.998 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to HEAD diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:08.998 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:09.062 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:09.062 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to HEAD diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:09.062 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:09.069 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:09.069 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to HEAD diff [0~50331648,54525952~25165824,83886080~29360128,117440512~50331648,171966464~25165824,201326592~4194304,209715200~37748736,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:09.069 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~50331648,54525952~25165824,83886080~29360128,117440512~50331648,171966464~25165824,201326592~4194304,209715200~37748736,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:09.074 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:09.074 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to HEAD diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:09.074 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:09.080 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:09.080 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to HEAD diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:09.080 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:09.085 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~222298112,226492416~188743680] 2024-09-13T00:40:09.085 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to HEAD diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:09.085 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:09.090 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~50331648,88080384~134217728,226492416~188743680] 2024-09-13T00:40:09.090 INFO:tasks.workunit.client.0.smithi136.stdout:from 6 to HEAD diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:09.090 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:09.094 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~50331648,88080384~134217728,226492416~20971520,251658240~163577856] 2024-09-13T00:40:09.094 INFO:tasks.workunit.client.0.smithi136.stdout:from 7 to HEAD diff [0~29360128,33554432~16777216,58720256~4194304,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~46137344,398458880~12582912] 2024-09-13T00:40:09.094 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,58720256~4194304,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~46137344,398458880~12582912] 2024-09-13T00:40:09.098 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,58720256~25165824,88080384~134217728,226492416~20971520,251658240~92274688,348127232~46137344,398458880~16777216] 2024-09-13T00:40:09.098 INFO:tasks.workunit.client.0.smithi136.stdout:from 8 to HEAD diff [0~29360128,33554432~16777216,67108864~12582912,88080384~20971520,117440512~8388608,130023424~67108864,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~46137344,398458880~12582912] 2024-09-13T00:40:09.098 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,67108864~12582912,88080384~20971520,117440512~8388608,130023424~67108864,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~46137344,398458880~12582912] 2024-09-13T00:40:09.102 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~16777216,88080384~25165824,117440512~8388608,130023424~92274688,226492416~20971520,251658240~92274688,348127232~46137344,398458880~16777216] 2024-09-13T00:40:09.102 INFO:tasks.workunit.client.0.smithi136.stdout:from 9 to HEAD diff [0~29360128,33554432~16777216,67108864~12582912,88080384~8388608,100663296~8388608,117440512~8388608,130023424~67108864,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:09.103 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,67108864~12582912,88080384~8388608,100663296~8388608,117440512~8388608,130023424~67108864,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:09.106 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~16777216,88080384~25165824,117440512~8388608,130023424~92274688,226492416~20971520,251658240~92274688,348127232~12582912,364904448~29360128,398458880~16777216] 2024-09-13T00:40:09.107 INFO:tasks.workunit.client.0.smithi136.stdout:from 10 to HEAD diff [0~29360128,33554432~16777216,67108864~12582912,88080384~8388608,100663296~8388608,117440512~8388608,130023424~41943040,176160768~20971520,209715200~8388608,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:09.107 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,67108864~12582912,88080384~8388608,100663296~8388608,117440512~8388608,130023424~41943040,176160768~20971520,209715200~8388608,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:09.109 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~16777216,88080384~25165824,117440512~8388608,130023424~41943040,176160768~25165824,209715200~8388608,226492416~20971520,251658240~92274688,348127232~12582912,364904448~29360128,398458880~16777216] 2024-09-13T00:40:09.109 INFO:tasks.workunit.client.0.smithi136.stdout:from 11 to HEAD diff [0~25165824,33554432~16777216,67108864~12582912,88080384~8388608,117440512~8388608,130023424~12582912,146800640~25165824,176160768~20971520,209715200~8388608,230686720~12582912,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:09.109 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,33554432~16777216,67108864~12582912,88080384~8388608,117440512~8388608,130023424~12582912,146800640~25165824,176160768~20971520,209715200~8388608,230686720~12582912,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:09.112 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~16777216,88080384~12582912,117440512~8388608,130023424~41943040,176160768~25165824,209715200~8388608,230686720~16777216,251658240~92274688,348127232~12582912,364904448~29360128,398458880~16777216] 2024-09-13T00:40:09.112 INFO:tasks.workunit.client.0.smithi136.stdout:from 12 to HEAD diff [0~25165824,33554432~16777216,67108864~12582912,88080384~8388608,117440512~4194304,130023424~12582912,146800640~25165824,176160768~20971520,209715200~8388608,230686720~12582912,251658240~4194304,264241152~12582912,281018368~25165824,310378496~20971520,339738624~4194304,348127232~4194304,356515840~4194304,364904448~4194304,373293056~20971520,398458880~12582912] 2024-09-13T00:40:09.112 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,33554432~16777216,67108864~12582912,88080384~8388608,117440512~4194304,130023424~12582912,146800640~25165824,176160768~20971520,209715200~8388608,230686720~12582912,251658240~4194304,264241152~12582912,281018368~25165824,310378496~20971520,339738624~4194304,348127232~4194304,356515840~4194304,364904448~4194304,373293056~20971520,398458880~12582912] 2024-09-13T00:40:09.114 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~16777216,88080384~12582912,117440512~4194304,130023424~41943040,176160768~25165824,209715200~8388608,230686720~16777216,251658240~4194304,264241152~67108864,339738624~4194304,348127232~4194304,356515840~4194304,364904448~29360128,398458880~16777216] 2024-09-13T00:40:09.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:08 smithi136 bash[75164]: cluster 2024-09-13T00:40:07.970992+0000 mon.a (mon.0) 3646 : cluster [DBG] osdmap e1984: 8 total, 8 up, 8 in 2024-09-13T00:40:09.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:08 smithi136 bash[75164]: cluster 2024-09-13T00:40:08.447985+0000 mgr.y (mgr.24661) 1890 : cluster [DBG] pgmap v3435: 137 pgs: 137 active+clean; 833 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 589 KiB/s rd, 6.0 MiB/s wr, 1.10k op/s 2024-09-13T00:40:09.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:08 smithi136 bash[75164]: audit 2024-09-13T00:40:08.837819+0000 mon.a (mon.0) 3647 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:40:09.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:08 smithi136 bash[76371]: cluster 2024-09-13T00:40:07.970992+0000 mon.a (mon.0) 3646 : cluster [DBG] osdmap e1984: 8 total, 8 up, 8 in 2024-09-13T00:40:09.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:08 smithi136 bash[76371]: cluster 2024-09-13T00:40:08.447985+0000 mgr.y (mgr.24661) 1890 : cluster [DBG] pgmap v3435: 137 pgs: 137 active+clean; 833 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 589 KiB/s rd, 6.0 MiB/s wr, 1.10k op/s 2024-09-13T00:40:09.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:08 smithi136 bash[76371]: audit 2024-09-13T00:40:08.837819+0000 mon.a (mon.0) 3647 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:40:09.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:08 smithi181 bash[67587]: cluster 2024-09-13T00:40:07.970992+0000 mon.a (mon.0) 3646 : cluster [DBG] osdmap e1984: 8 total, 8 up, 8 in 2024-09-13T00:40:09.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:08 smithi181 bash[67587]: cluster 2024-09-13T00:40:08.447985+0000 mgr.y (mgr.24661) 1890 : cluster [DBG] pgmap v3435: 137 pgs: 137 active+clean; 833 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 589 KiB/s rd, 6.0 MiB/s wr, 1.10k op/s 2024-09-13T00:40:09.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:08 smithi181 bash[67587]: audit 2024-09-13T00:40:08.837819+0000 mon.a (mon.0) 3647 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:40:09.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:40:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:40:09] "GET /metrics HTTP/1.1" 200 36756 "" "Prometheus/2.43.0" 2024-09-13T00:40:10.013 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to HEAD diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:10.013 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:10.055 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:10.056 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to HEAD diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:10.056 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:10.063 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:10.063 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to HEAD diff [0~50331648,54525952~25165824,83886080~29360128,117440512~50331648,171966464~25165824,201326592~4194304,209715200~37748736,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:10.064 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~50331648,54525952~25165824,83886080~29360128,117440512~50331648,171966464~25165824,201326592~4194304,209715200~37748736,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:10.069 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:10.069 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to HEAD diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:10.069 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:10.077 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:10.078 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to HEAD diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:10.078 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:10.084 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~222298112,226492416~188743680] 2024-09-13T00:40:10.084 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to HEAD diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:10.084 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:10.089 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~50331648,88080384~134217728,226492416~188743680] 2024-09-13T00:40:10.089 INFO:tasks.workunit.client.0.smithi136.stdout:from 6 to HEAD diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:10.089 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:10.093 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~50331648,88080384~134217728,226492416~20971520,251658240~163577856] 2024-09-13T00:40:10.093 INFO:tasks.workunit.client.0.smithi136.stdout:from 7 to HEAD diff [0~29360128,33554432~16777216,58720256~4194304,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~46137344,398458880~12582912] 2024-09-13T00:40:10.093 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,58720256~4194304,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~46137344,398458880~12582912] 2024-09-13T00:40:10.097 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,58720256~25165824,88080384~134217728,226492416~20971520,251658240~92274688,348127232~46137344,398458880~16777216] 2024-09-13T00:40:10.097 INFO:tasks.workunit.client.0.smithi136.stdout:from 8 to HEAD diff [0~29360128,33554432~16777216,67108864~12582912,88080384~20971520,117440512~8388608,130023424~67108864,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~46137344,398458880~12582912] 2024-09-13T00:40:10.097 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,67108864~12582912,88080384~20971520,117440512~8388608,130023424~67108864,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~46137344,398458880~12582912] 2024-09-13T00:40:10.100 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~16777216,88080384~25165824,117440512~8388608,130023424~92274688,226492416~20971520,251658240~92274688,348127232~46137344,398458880~16777216] 2024-09-13T00:40:10.100 INFO:tasks.workunit.client.0.smithi136.stdout:from 9 to HEAD diff [0~29360128,33554432~16777216,67108864~12582912,88080384~8388608,100663296~8388608,117440512~8388608,130023424~67108864,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:10.100 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,67108864~12582912,88080384~8388608,100663296~8388608,117440512~8388608,130023424~67108864,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:10.103 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~16777216,88080384~25165824,117440512~8388608,130023424~92274688,226492416~20971520,251658240~92274688,348127232~12582912,364904448~29360128,398458880~16777216] 2024-09-13T00:40:10.103 INFO:tasks.workunit.client.0.smithi136.stdout:from 10 to HEAD diff [0~29360128,33554432~16777216,67108864~12582912,88080384~8388608,100663296~8388608,117440512~8388608,130023424~41943040,176160768~20971520,209715200~8388608,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:10.103 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,67108864~12582912,88080384~8388608,100663296~8388608,117440512~8388608,130023424~41943040,176160768~20971520,209715200~8388608,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:10.105 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~16777216,88080384~25165824,117440512~8388608,130023424~41943040,176160768~25165824,209715200~8388608,226492416~20971520,251658240~92274688,348127232~12582912,364904448~29360128,398458880~16777216] 2024-09-13T00:40:10.105 INFO:tasks.workunit.client.0.smithi136.stdout:from 11 to HEAD diff [0~25165824,33554432~16777216,67108864~12582912,88080384~8388608,117440512~8388608,130023424~12582912,146800640~25165824,176160768~20971520,209715200~8388608,230686720~12582912,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:10.105 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~25165824,33554432~16777216,67108864~12582912,88080384~8388608,117440512~8388608,130023424~12582912,146800640~25165824,176160768~20971520,209715200~8388608,230686720~12582912,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:10.106 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~16777216,88080384~12582912,117440512~8388608,130023424~41943040,176160768~25165824,209715200~8388608,230686720~16777216,251658240~92274688,348127232~12582912,364904448~29360128,398458880~16777216] 2024-09-13T00:40:10.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:09 smithi136 bash[75164]: cluster 2024-09-13T00:40:08.978222+0000 mon.a (mon.0) 3648 : cluster [DBG] osdmap e1985: 8 total, 8 up, 8 in 2024-09-13T00:40:10.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:09 smithi136 bash[76371]: cluster 2024-09-13T00:40:08.978222+0000 mon.a (mon.0) 3648 : cluster [DBG] osdmap e1985: 8 total, 8 up, 8 in 2024-09-13T00:40:10.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:09 smithi181 bash[67587]: cluster 2024-09-13T00:40:08.978222+0000 mon.a (mon.0) 3648 : cluster [DBG] osdmap e1985: 8 total, 8 up, 8 in 2024-09-13T00:40:11.018 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to HEAD diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:11.018 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:11.068 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:11.068 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to HEAD diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:11.068 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:11.088 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:11.088 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to HEAD diff [0~50331648,54525952~25165824,83886080~29360128,117440512~50331648,171966464~25165824,201326592~4194304,209715200~37748736,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:11.089 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~50331648,54525952~25165824,83886080~29360128,117440512~50331648,171966464~25165824,201326592~4194304,209715200~37748736,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:11.093 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:11.094 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to HEAD diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:11.094 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:11.101 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:11.101 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to HEAD diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:11.101 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:11.106 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~222298112,226492416~188743680] 2024-09-13T00:40:11.106 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to HEAD diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:11.106 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:11.110 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~50331648,88080384~134217728,226492416~188743680] 2024-09-13T00:40:11.110 INFO:tasks.workunit.client.0.smithi136.stdout:from 6 to HEAD diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:11.110 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:11.113 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~50331648,88080384~134217728,226492416~20971520,251658240~163577856] 2024-09-13T00:40:11.113 INFO:tasks.workunit.client.0.smithi136.stdout:from 7 to HEAD diff [0~29360128,33554432~16777216,58720256~4194304,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~46137344,398458880~12582912] 2024-09-13T00:40:11.113 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,58720256~4194304,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~46137344,398458880~12582912] 2024-09-13T00:40:11.115 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,58720256~25165824,88080384~134217728,226492416~20971520,251658240~92274688,348127232~46137344,398458880~16777216] 2024-09-13T00:40:11.115 INFO:tasks.workunit.client.0.smithi136.stdout:from 8 to HEAD diff [0~29360128,33554432~16777216,67108864~12582912,88080384~20971520,117440512~8388608,130023424~67108864,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~46137344,398458880~12582912] 2024-09-13T00:40:11.116 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,67108864~12582912,88080384~20971520,117440512~8388608,130023424~67108864,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~46137344,398458880~12582912] 2024-09-13T00:40:11.118 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~16777216,88080384~25165824,117440512~8388608,130023424~92274688,226492416~20971520,251658240~92274688,348127232~46137344,398458880~16777216] 2024-09-13T00:40:11.118 INFO:tasks.workunit.client.0.smithi136.stdout:from 9 to HEAD diff [0~29360128,33554432~16777216,67108864~12582912,88080384~8388608,100663296~8388608,117440512~8388608,130023424~67108864,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:11.118 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,67108864~12582912,88080384~8388608,100663296~8388608,117440512~8388608,130023424~67108864,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:11.120 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~16777216,88080384~25165824,117440512~8388608,130023424~92274688,226492416~20971520,251658240~92274688,348127232~12582912,364904448~29360128,398458880~16777216] 2024-09-13T00:40:11.120 INFO:tasks.workunit.client.0.smithi136.stdout:from 10 to HEAD diff [0~29360128,33554432~16777216,67108864~12582912,88080384~8388608,100663296~8388608,117440512~8388608,130023424~41943040,176160768~20971520,209715200~8388608,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:11.120 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,67108864~12582912,88080384~8388608,100663296~8388608,117440512~8388608,130023424~41943040,176160768~20971520,209715200~8388608,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:11.121 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~16777216,88080384~25165824,117440512~8388608,130023424~41943040,176160768~25165824,209715200~8388608,226492416~20971520,251658240~92274688,348127232~12582912,364904448~29360128,398458880~16777216] 2024-09-13T00:40:11.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:11 smithi136 bash[76371]: cluster 2024-09-13T00:40:09.983267+0000 mon.a (mon.0) 3649 : cluster [DBG] osdmap e1986: 8 total, 8 up, 8 in 2024-09-13T00:40:11.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:11 smithi136 bash[76371]: cluster 2024-09-13T00:40:10.448678+0000 mgr.y (mgr.24661) 1891 : cluster [DBG] pgmap v3438: 137 pgs: 137 active+clean; 828 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 355 KiB/s rd, 5.5 KiB/s wr, 634 op/s 2024-09-13T00:40:11.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:10 smithi136 bash[75164]: cluster 2024-09-13T00:40:09.983267+0000 mon.a (mon.0) 3649 : cluster [DBG] osdmap e1986: 8 total, 8 up, 8 in 2024-09-13T00:40:11.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:10 smithi136 bash[75164]: cluster 2024-09-13T00:40:10.448678+0000 mgr.y (mgr.24661) 1891 : cluster [DBG] pgmap v3438: 137 pgs: 137 active+clean; 828 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 355 KiB/s rd, 5.5 KiB/s wr, 634 op/s 2024-09-13T00:40:11.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:10 smithi181 bash[67587]: cluster 2024-09-13T00:40:09.983267+0000 mon.a (mon.0) 3649 : cluster [DBG] osdmap e1986: 8 total, 8 up, 8 in 2024-09-13T00:40:11.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:10 smithi181 bash[67587]: cluster 2024-09-13T00:40:10.448678+0000 mgr.y (mgr.24661) 1891 : cluster [DBG] pgmap v3438: 137 pgs: 137 active+clean; 828 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 355 KiB/s rd, 5.5 KiB/s wr, 634 op/s 2024-09-13T00:40:12.037 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to HEAD diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:12.037 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:12.104 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:12.104 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to HEAD diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:12.104 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:12.110 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:12.111 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to HEAD diff [0~50331648,54525952~25165824,83886080~29360128,117440512~50331648,171966464~25165824,201326592~4194304,209715200~37748736,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:12.111 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~50331648,54525952~25165824,83886080~29360128,117440512~50331648,171966464~25165824,201326592~4194304,209715200~37748736,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:12.121 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:12.121 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to HEAD diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:12.122 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:12.126 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:12.127 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to HEAD diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:12.127 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:12.131 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~222298112,226492416~188743680] 2024-09-13T00:40:12.131 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to HEAD diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:12.131 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:12.136 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~50331648,88080384~134217728,226492416~188743680] 2024-09-13T00:40:12.136 INFO:tasks.workunit.client.0.smithi136.stdout:from 6 to HEAD diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:12.136 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:12.139 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~50331648,88080384~134217728,226492416~20971520,251658240~163577856] 2024-09-13T00:40:12.140 INFO:tasks.workunit.client.0.smithi136.stdout:from 7 to HEAD diff [0~29360128,33554432~16777216,58720256~4194304,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~46137344,398458880~12582912] 2024-09-13T00:40:12.140 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,58720256~4194304,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~46137344,398458880~12582912] 2024-09-13T00:40:12.143 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,58720256~25165824,88080384~134217728,226492416~20971520,251658240~92274688,348127232~46137344,398458880~16777216] 2024-09-13T00:40:12.143 INFO:tasks.workunit.client.0.smithi136.stdout:from 8 to HEAD diff [0~29360128,33554432~16777216,67108864~12582912,88080384~20971520,117440512~8388608,130023424~67108864,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~46137344,398458880~12582912] 2024-09-13T00:40:12.143 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,67108864~12582912,88080384~20971520,117440512~8388608,130023424~67108864,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~46137344,398458880~12582912] 2024-09-13T00:40:12.146 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~16777216,88080384~25165824,117440512~8388608,130023424~92274688,226492416~20971520,251658240~92274688,348127232~46137344,398458880~16777216] 2024-09-13T00:40:12.146 INFO:tasks.workunit.client.0.smithi136.stdout:from 9 to HEAD diff [0~29360128,33554432~16777216,67108864~12582912,88080384~8388608,100663296~8388608,117440512~8388608,130023424~67108864,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:12.146 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,67108864~12582912,88080384~8388608,100663296~8388608,117440512~8388608,130023424~67108864,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~12582912,364904448~29360128,398458880~12582912] 2024-09-13T00:40:12.148 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~16777216,88080384~25165824,117440512~8388608,130023424~92274688,226492416~20971520,251658240~92274688,348127232~12582912,364904448~29360128,398458880~16777216] 2024-09-13T00:40:12.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:12 smithi181 bash[67587]: cluster 2024-09-13T00:40:11.005498+0000 mon.a (mon.0) 3650 : cluster [DBG] osdmap e1987: 8 total, 8 up, 8 in 2024-09-13T00:40:12.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:12 smithi136 bash[75164]: cluster 2024-09-13T00:40:11.005498+0000 mon.a (mon.0) 3650 : cluster [DBG] osdmap e1987: 8 total, 8 up, 8 in 2024-09-13T00:40:12.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:12 smithi136 bash[76371]: cluster 2024-09-13T00:40:11.005498+0000 mon.a (mon.0) 3650 : cluster [DBG] osdmap e1987: 8 total, 8 up, 8 in 2024-09-13T00:40:12.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:40:12 smithi181 bash[95505]: ts=2024-09-13T00:40:12.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:40:13.062 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to HEAD diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:13.062 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:13.121 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:13.121 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to HEAD diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:13.121 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:13.129 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:13.129 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to HEAD diff [0~50331648,54525952~25165824,83886080~29360128,117440512~50331648,171966464~25165824,201326592~4194304,209715200~37748736,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:13.130 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~50331648,54525952~25165824,83886080~29360128,117440512~50331648,171966464~25165824,201326592~4194304,209715200~37748736,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:13.135 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:13.135 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to HEAD diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:13.135 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:13.139 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:13.139 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to HEAD diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:13.139 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:13.142 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~222298112,226492416~188743680] 2024-09-13T00:40:13.142 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to HEAD diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:13.142 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:13.145 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~50331648,88080384~134217728,226492416~188743680] 2024-09-13T00:40:13.146 INFO:tasks.workunit.client.0.smithi136.stdout:from 6 to HEAD diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:13.146 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:13.148 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~50331648,88080384~134217728,226492416~20971520,251658240~163577856] 2024-09-13T00:40:13.148 INFO:tasks.workunit.client.0.smithi136.stdout:from 7 to HEAD diff [0~29360128,33554432~16777216,58720256~4194304,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~46137344,398458880~12582912] 2024-09-13T00:40:13.148 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,58720256~4194304,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~46137344,398458880~12582912] 2024-09-13T00:40:13.150 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,58720256~25165824,88080384~134217728,226492416~20971520,251658240~92274688,348127232~46137344,398458880~16777216] 2024-09-13T00:40:13.150 INFO:tasks.workunit.client.0.smithi136.stdout:from 8 to HEAD diff [0~29360128,33554432~16777216,67108864~12582912,88080384~20971520,117440512~8388608,130023424~67108864,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~46137344,398458880~12582912] 2024-09-13T00:40:13.150 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,67108864~12582912,88080384~20971520,117440512~8388608,130023424~67108864,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~46137344,398458880~12582912] 2024-09-13T00:40:13.151 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,67108864~16777216,88080384~25165824,117440512~8388608,130023424~92274688,226492416~20971520,251658240~92274688,348127232~46137344,398458880~16777216] 2024-09-13T00:40:13.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:13 smithi181 bash[67587]: cluster 2024-09-13T00:40:12.008329+0000 mon.a (mon.0) 3651 : cluster [DBG] osdmap e1988: 8 total, 8 up, 8 in 2024-09-13T00:40:13.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:13 smithi181 bash[67587]: cluster 2024-09-13T00:40:12.449374+0000 mgr.y (mgr.24661) 1892 : cluster [DBG] pgmap v3441: 137 pgs: 137 active+clean; 801 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 276 KiB/s rd, 6.2 KiB/s wr, 476 op/s 2024-09-13T00:40:13.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:13 smithi136 bash[76371]: cluster 2024-09-13T00:40:12.008329+0000 mon.a (mon.0) 3651 : cluster [DBG] osdmap e1988: 8 total, 8 up, 8 in 2024-09-13T00:40:13.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:13 smithi136 bash[76371]: cluster 2024-09-13T00:40:12.449374+0000 mgr.y (mgr.24661) 1892 : cluster [DBG] pgmap v3441: 137 pgs: 137 active+clean; 801 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 276 KiB/s rd, 6.2 KiB/s wr, 476 op/s 2024-09-13T00:40:13.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:13 smithi136 bash[75164]: cluster 2024-09-13T00:40:12.008329+0000 mon.a (mon.0) 3651 : cluster [DBG] osdmap e1988: 8 total, 8 up, 8 in 2024-09-13T00:40:13.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:13 smithi136 bash[75164]: cluster 2024-09-13T00:40:12.449374+0000 mgr.y (mgr.24661) 1892 : cluster [DBG] pgmap v3441: 137 pgs: 137 active+clean; 801 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 276 KiB/s rd, 6.2 KiB/s wr, 476 op/s 2024-09-13T00:40:14.037 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 13 00:40:13 smithi181 bash[98820]: logger=cleanup t=2024-09-13T00:40:13.740645497Z level=info msg="Completed cleanup jobs" duration=45.956942ms 2024-09-13T00:40:14.067 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to HEAD diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:14.067 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:14.100 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:14.101 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to HEAD diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:14.101 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:14.122 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:14.122 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to HEAD diff [0~50331648,54525952~25165824,83886080~29360128,117440512~50331648,171966464~25165824,201326592~4194304,209715200~37748736,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:14.122 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~50331648,54525952~25165824,83886080~29360128,117440512~50331648,171966464~25165824,201326592~4194304,209715200~37748736,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:14.130 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:14.130 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to HEAD diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:14.130 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:14.136 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:14.136 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to HEAD diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:14.136 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:14.144 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~222298112,226492416~188743680] 2024-09-13T00:40:14.145 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to HEAD diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:14.145 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:14.148 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~50331648,88080384~134217728,226492416~188743680] 2024-09-13T00:40:14.148 INFO:tasks.workunit.client.0.smithi136.stdout:from 6 to HEAD diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:14.148 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:14.150 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~50331648,88080384~134217728,226492416~20971520,251658240~163577856] 2024-09-13T00:40:14.150 INFO:tasks.workunit.client.0.smithi136.stdout:from 7 to HEAD diff [0~29360128,33554432~16777216,58720256~4194304,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~46137344,398458880~12582912] 2024-09-13T00:40:14.150 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,58720256~4194304,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~46137344,398458880~12582912] 2024-09-13T00:40:14.151 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~20971520,58720256~25165824,88080384~134217728,226492416~20971520,251658240~92274688,348127232~46137344,398458880~16777216] 2024-09-13T00:40:14.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:14 smithi181 bash[67587]: cluster 2024-09-13T00:40:13.034068+0000 mon.a (mon.0) 3652 : cluster [DBG] osdmap e1989: 8 total, 8 up, 8 in 2024-09-13T00:40:14.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:14 smithi136 bash[75164]: cluster 2024-09-13T00:40:13.034068+0000 mon.a (mon.0) 3652 : cluster [DBG] osdmap e1989: 8 total, 8 up, 8 in 2024-09-13T00:40:14.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:14 smithi136 bash[76371]: cluster 2024-09-13T00:40:13.034068+0000 mon.a (mon.0) 3652 : cluster [DBG] osdmap e1989: 8 total, 8 up, 8 in 2024-09-13T00:40:14.778 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to HEAD diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:14.778 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:14.815 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:14.815 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to HEAD diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:14.815 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:14.825 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:14.825 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to HEAD diff [0~50331648,54525952~25165824,83886080~29360128,117440512~50331648,171966464~25165824,201326592~4194304,209715200~37748736,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:14.826 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~50331648,54525952~25165824,83886080~29360128,117440512~50331648,171966464~25165824,201326592~4194304,209715200~37748736,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:14.835 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:14.836 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to HEAD diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:14.836 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:14.840 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:14.840 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to HEAD diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:14.841 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:14.843 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~222298112,226492416~188743680] 2024-09-13T00:40:14.843 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to HEAD diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:14.843 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:14.845 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~50331648,88080384~134217728,226492416~188743680] 2024-09-13T00:40:14.845 INFO:tasks.workunit.client.0.smithi136.stdout:from 6 to HEAD diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:14.845 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~16777216,251658240~25165824,281018368~25165824,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:14.846 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~50331648,88080384~134217728,226492416~20971520,251658240~163577856] 2024-09-13T00:40:15.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:15 smithi181 bash[67587]: cluster 2024-09-13T00:40:14.033887+0000 mon.a (mon.0) 3653 : cluster [DBG] osdmap e1990: 8 total, 8 up, 8 in 2024-09-13T00:40:15.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:15 smithi181 bash[67587]: cluster 2024-09-13T00:40:14.450068+0000 mgr.y (mgr.24661) 1893 : cluster [DBG] pgmap v3444: 137 pgs: 137 active+clean; 785 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 229 KiB/s rd, 6.2 KiB/s wr, 380 op/s 2024-09-13T00:40:15.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:15 smithi181 bash[67587]: cluster 2024-09-13T00:40:14.747798+0000 mon.a (mon.0) 3654 : cluster [DBG] osdmap e1991: 8 total, 8 up, 8 in 2024-09-13T00:40:15.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:15 smithi136 bash[75164]: cluster 2024-09-13T00:40:14.033887+0000 mon.a (mon.0) 3653 : cluster [DBG] osdmap e1990: 8 total, 8 up, 8 in 2024-09-13T00:40:15.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:15 smithi136 bash[75164]: cluster 2024-09-13T00:40:14.450068+0000 mgr.y (mgr.24661) 1893 : cluster [DBG] pgmap v3444: 137 pgs: 137 active+clean; 785 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 229 KiB/s rd, 6.2 KiB/s wr, 380 op/s 2024-09-13T00:40:15.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:15 smithi136 bash[75164]: cluster 2024-09-13T00:40:14.747798+0000 mon.a (mon.0) 3654 : cluster [DBG] osdmap e1991: 8 total, 8 up, 8 in 2024-09-13T00:40:15.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:15 smithi136 bash[76371]: cluster 2024-09-13T00:40:14.033887+0000 mon.a (mon.0) 3653 : cluster [DBG] osdmap e1990: 8 total, 8 up, 8 in 2024-09-13T00:40:15.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:15 smithi136 bash[76371]: cluster 2024-09-13T00:40:14.450068+0000 mgr.y (mgr.24661) 1893 : cluster [DBG] pgmap v3444: 137 pgs: 137 active+clean; 785 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 229 KiB/s rd, 6.2 KiB/s wr, 380 op/s 2024-09-13T00:40:15.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:15 smithi136 bash[76371]: cluster 2024-09-13T00:40:14.747798+0000 mon.a (mon.0) 3654 : cluster [DBG] osdmap e1991: 8 total, 8 up, 8 in 2024-09-13T00:40:15.776 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to HEAD diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:15.776 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:15.834 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:15.834 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to HEAD diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:15.834 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:15.840 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:15.840 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to HEAD diff [0~50331648,54525952~25165824,83886080~29360128,117440512~50331648,171966464~25165824,201326592~4194304,209715200~37748736,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:15.840 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~50331648,54525952~25165824,83886080~29360128,117440512~50331648,171966464~25165824,201326592~4194304,209715200~37748736,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:15.844 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:15.844 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to HEAD diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:15.844 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:15.847 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:15.847 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to HEAD diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:15.847 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:15.849 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~222298112,226492416~188743680] 2024-09-13T00:40:15.849 INFO:tasks.workunit.client.0.smithi136.stdout:from 5 to HEAD diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:15.849 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~29360128,33554432~16777216,54525952~8388608,67108864~12582912,88080384~25165824,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:15.851 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~29360128,33554432~50331648,88080384~134217728,226492416~188743680] 2024-09-13T00:40:16.796 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to HEAD diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:16.796 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:16.831 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:16.831 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to HEAD diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:16.831 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:16.837 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:16.837 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to HEAD diff [0~50331648,54525952~25165824,83886080~29360128,117440512~50331648,171966464~25165824,201326592~4194304,209715200~37748736,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:16.837 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~50331648,54525952~25165824,83886080~29360128,117440512~50331648,171966464~25165824,201326592~4194304,209715200~37748736,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:16.851 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:16.851 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to HEAD diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:16.851 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:16.854 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:16.854 INFO:tasks.workunit.client.0.smithi136.stdout:from 4 to HEAD diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:16.854 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:16.858 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~222298112,226492416~188743680] 2024-09-13T00:40:17.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:16 smithi136 bash[75164]: cluster 2024-09-13T00:40:15.752728+0000 mon.a (mon.0) 3655 : cluster [DBG] osdmap e1992: 8 total, 8 up, 8 in 2024-09-13T00:40:17.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:16 smithi136 bash[75164]: cluster 2024-09-13T00:40:16.451028+0000 mgr.y (mgr.24661) 1894 : cluster [DBG] pgmap v3447: 137 pgs: 1 active+clean+snaptrim, 136 active+clean; 721 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 71 KiB/s rd, 0 B/s wr, 143 op/s 2024-09-13T00:40:17.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:16 smithi136 bash[76371]: cluster 2024-09-13T00:40:15.752728+0000 mon.a (mon.0) 3655 : cluster [DBG] osdmap e1992: 8 total, 8 up, 8 in 2024-09-13T00:40:17.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:16 smithi136 bash[76371]: cluster 2024-09-13T00:40:16.451028+0000 mgr.y (mgr.24661) 1894 : cluster [DBG] pgmap v3447: 137 pgs: 1 active+clean+snaptrim, 136 active+clean; 721 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 71 KiB/s rd, 0 B/s wr, 143 op/s 2024-09-13T00:40:17.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:16 smithi181 bash[67587]: cluster 2024-09-13T00:40:15.752728+0000 mon.a (mon.0) 3655 : cluster [DBG] osdmap e1992: 8 total, 8 up, 8 in 2024-09-13T00:40:17.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:16 smithi181 bash[67587]: cluster 2024-09-13T00:40:16.451028+0000 mgr.y (mgr.24661) 1894 : cluster [DBG] pgmap v3447: 137 pgs: 1 active+clean+snaptrim, 136 active+clean; 721 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 71 KiB/s rd, 0 B/s wr, 143 op/s 2024-09-13T00:40:17.787 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to HEAD diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:17.787 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:17.835 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:17.835 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to HEAD diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:17.835 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:17.842 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:17.843 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to HEAD diff [0~50331648,54525952~25165824,83886080~29360128,117440512~50331648,171966464~25165824,201326592~4194304,209715200~37748736,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:17.843 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~50331648,54525952~25165824,83886080~29360128,117440512~50331648,171966464~25165824,201326592~4194304,209715200~37748736,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:17.848 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:17.848 INFO:tasks.workunit.client.0.smithi136.stdout:from 3 to HEAD diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:17.848 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~50331648,54525952~25165824,83886080~29360128,117440512~79691776,201326592~4194304,209715200~12582912,226492416~20971520,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:17.850 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:18.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:17 smithi181 bash[67587]: cluster 2024-09-13T00:40:16.761263+0000 mon.a (mon.0) 3656 : cluster [DBG] osdmap e1993: 8 total, 8 up, 8 in 2024-09-13T00:40:18.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:17 smithi136 bash[75164]: cluster 2024-09-13T00:40:16.761263+0000 mon.a (mon.0) 3656 : cluster [DBG] osdmap e1993: 8 total, 8 up, 8 in 2024-09-13T00:40:18.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:17 smithi136 bash[76371]: cluster 2024-09-13T00:40:16.761263+0000 mon.a (mon.0) 3656 : cluster [DBG] osdmap e1993: 8 total, 8 up, 8 in 2024-09-13T00:40:18.809 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to HEAD diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:18.810 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:18.837 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:18.837 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to HEAD diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:18.837 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:18.840 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:18.840 INFO:tasks.workunit.client.0.smithi136.stdout:from 2 to HEAD diff [0~50331648,54525952~25165824,83886080~29360128,117440512~50331648,171966464~25165824,201326592~4194304,209715200~37748736,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:18.840 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~50331648,54525952~25165824,83886080~29360128,117440512~50331648,171966464~25165824,201326592~4194304,209715200~37748736,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:18.847 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:19.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:18 smithi181 bash[67587]: cluster 2024-09-13T00:40:17.763785+0000 mon.a (mon.0) 3657 : cluster [DBG] osdmap e1994: 8 total, 8 up, 8 in 2024-09-13T00:40:19.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:18 smithi181 bash[67587]: cluster 2024-09-13T00:40:18.451821+0000 mgr.y (mgr.24661) 1895 : cluster [DBG] pgmap v3450: 137 pgs: 1 active+clean+snaptrim, 136 active+clean; 657 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 155 KiB/s rd, 6.2 KiB/s wr, 255 op/s 2024-09-13T00:40:19.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:18 smithi136 bash[75164]: cluster 2024-09-13T00:40:17.763785+0000 mon.a (mon.0) 3657 : cluster [DBG] osdmap e1994: 8 total, 8 up, 8 in 2024-09-13T00:40:19.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:18 smithi136 bash[75164]: cluster 2024-09-13T00:40:18.451821+0000 mgr.y (mgr.24661) 1895 : cluster [DBG] pgmap v3450: 137 pgs: 1 active+clean+snaptrim, 136 active+clean; 657 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 155 KiB/s rd, 6.2 KiB/s wr, 255 op/s 2024-09-13T00:40:19.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:18 smithi136 bash[76371]: cluster 2024-09-13T00:40:17.763785+0000 mon.a (mon.0) 3657 : cluster [DBG] osdmap e1994: 8 total, 8 up, 8 in 2024-09-13T00:40:19.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:18 smithi136 bash[76371]: cluster 2024-09-13T00:40:18.451821+0000 mgr.y (mgr.24661) 1895 : cluster [DBG] pgmap v3450: 137 pgs: 1 active+clean+snaptrim, 136 active+clean; 657 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 155 KiB/s rd, 6.2 KiB/s wr, 255 op/s 2024-09-13T00:40:19.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:40:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:40:19] "GET /metrics HTTP/1.1" 200 36756 "" "Prometheus/2.43.0" 2024-09-13T00:40:19.790 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to HEAD diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:19.790 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:19.813 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:19.813 INFO:tasks.workunit.client.0.smithi136.stdout:from 1 to HEAD diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:19.813 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:19.815 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:20.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:19 smithi181 bash[67587]: cluster 2024-09-13T00:40:18.784506+0000 mon.a (mon.0) 3658 : cluster [DBG] osdmap e1995: 8 total, 8 up, 8 in 2024-09-13T00:40:20.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:19 smithi181 bash[67587]: cluster 2024-09-13T00:40:19.751886+0000 mon.a (mon.0) 3659 : cluster [DBG] osdmap e1996: 8 total, 8 up, 8 in 2024-09-13T00:40:20.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:19 smithi136 bash[75164]: cluster 2024-09-13T00:40:18.784506+0000 mon.a (mon.0) 3658 : cluster [DBG] osdmap e1995: 8 total, 8 up, 8 in 2024-09-13T00:40:20.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:19 smithi136 bash[75164]: cluster 2024-09-13T00:40:19.751886+0000 mon.a (mon.0) 3659 : cluster [DBG] osdmap e1996: 8 total, 8 up, 8 in 2024-09-13T00:40:20.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:19 smithi136 bash[76371]: cluster 2024-09-13T00:40:18.784506+0000 mon.a (mon.0) 3658 : cluster [DBG] osdmap e1995: 8 total, 8 up, 8 in 2024-09-13T00:40:20.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:19 smithi136 bash[76371]: cluster 2024-09-13T00:40:19.751886+0000 mon.a (mon.0) 3659 : cluster [DBG] osdmap e1996: 8 total, 8 up, 8 in 2024-09-13T00:40:20.829 INFO:tasks.workunit.client.0.smithi136.stdout:from 0 to HEAD diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:20.829 INFO:tasks.workunit.client.0.smithi136.stdout: limited diff [0~79691776,83886080~29360128,117440512~79691776,201326592~46137344,251658240~54525952,310378496~25165824,339738624~4194304,348127232~62914560] 2024-09-13T00:40:20.841 INFO:tasks.workunit.client.0.smithi136.stdout: actual was [0~415236096] 2024-09-13T00:40:21.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:20 smithi181 bash[67587]: cluster 2024-09-13T00:40:20.452556+0000 mgr.y (mgr.24661) 1896 : cluster [DBG] pgmap v3453: 137 pgs: 1 active+clean+snaptrim, 136 active+clean; 614 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 125 KiB/s rd, 6.2 KiB/s wr, 195 op/s 2024-09-13T00:40:21.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:20 smithi136 bash[75164]: cluster 2024-09-13T00:40:20.452556+0000 mgr.y (mgr.24661) 1896 : cluster [DBG] pgmap v3453: 137 pgs: 1 active+clean+snaptrim, 136 active+clean; 614 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 125 KiB/s rd, 6.2 KiB/s wr, 195 op/s 2024-09-13T00:40:21.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:20 smithi136 bash[76371]: cluster 2024-09-13T00:40:20.452556+0000 mgr.y (mgr.24661) 1896 : cluster [DBG] pgmap v3453: 137 pgs: 1 active+clean+snaptrim, 136 active+clean; 614 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 125 KiB/s rd, 6.2 KiB/s wr, 195 op/s 2024-09-13T00:40:21.856 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] DiffIterateTest/1.DiffIterateStress (39900 ms) 2024-09-13T00:40:21.857 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] DiffIterateTest/1.DiffIterateRegression6926 2024-09-13T00:40:21.857 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:40:22.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:21 smithi181 bash[67587]: cluster 2024-09-13T00:40:20.797214+0000 mon.a (mon.0) 3660 : cluster [DBG] osdmap e1997: 8 total, 8 up, 8 in 2024-09-13T00:40:22.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:21 smithi136 bash[75164]: cluster 2024-09-13T00:40:20.797214+0000 mon.a (mon.0) 3660 : cluster [DBG] osdmap e1997: 8 total, 8 up, 8 in 2024-09-13T00:40:22.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:21 smithi136 bash[76371]: cluster 2024-09-13T00:40:20.797214+0000 mon.a (mon.0) 3660 : cluster [DBG] osdmap e1997: 8 total, 8 up, 8 in 2024-09-13T00:40:22.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:40:22 smithi181 bash[95505]: ts=2024-09-13T00:40:22.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:40:22.866 INFO:tasks.workunit.client.0.smithi136.stdout:iterate_cb 0~4194304 2024-09-13T00:40:22.874 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] DiffIterateTest/1.DiffIterateRegression6926 (1017 ms) 2024-09-13T00:40:22.874 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] DiffIterateTest/1.DiffIterateParent 2024-09-13T00:40:22.874 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:40:23.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:22 smithi181 bash[67587]: cluster 2024-09-13T00:40:21.798339+0000 mon.a (mon.0) 3661 : cluster [DBG] osdmap e1998: 8 total, 8 up, 8 in 2024-09-13T00:40:23.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:22 smithi181 bash[67587]: cluster 2024-09-13T00:40:22.453389+0000 mgr.y (mgr.24661) 1897 : cluster [DBG] pgmap v3456: 137 pgs: 2 active+clean+snaptrim, 135 active+clean; 521 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 84 KiB/s rd, 5.7 KiB/s wr, 133 op/s 2024-09-13T00:40:23.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:22 smithi136 bash[75164]: cluster 2024-09-13T00:40:21.798339+0000 mon.a (mon.0) 3661 : cluster [DBG] osdmap e1998: 8 total, 8 up, 8 in 2024-09-13T00:40:23.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:22 smithi136 bash[75164]: cluster 2024-09-13T00:40:22.453389+0000 mgr.y (mgr.24661) 1897 : cluster [DBG] pgmap v3456: 137 pgs: 2 active+clean+snaptrim, 135 active+clean; 521 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 84 KiB/s rd, 5.7 KiB/s wr, 133 op/s 2024-09-13T00:40:23.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:22 smithi136 bash[76371]: cluster 2024-09-13T00:40:21.798339+0000 mon.a (mon.0) 3661 : cluster [DBG] osdmap e1998: 8 total, 8 up, 8 in 2024-09-13T00:40:23.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:22 smithi136 bash[76371]: cluster 2024-09-13T00:40:22.453389+0000 mgr.y (mgr.24661) 1897 : cluster [DBG] pgmap v3456: 137 pgs: 2 active+clean+snaptrim, 135 active+clean; 521 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 84 KiB/s rd, 5.7 KiB/s wr, 133 op/s 2024-09-13T00:40:23.984 INFO:tasks.workunit.client.0.smithi136.stdout:iterate_cb 0~4194304 2024-09-13T00:40:23.984 INFO:tasks.workunit.client.0.smithi136.stdout:iterate_cb 4194304~4194304 2024-09-13T00:40:23.984 INFO:tasks.workunit.client.0.smithi136.stdout:iterate_cb 8388608~4194304 2024-09-13T00:40:23.984 INFO:tasks.workunit.client.0.smithi136.stdout:iterate_cb 12582912~4194304 2024-09-13T00:40:23.984 INFO:tasks.workunit.client.0.smithi136.stdout:iterate_cb 16777216~4194304 2024-09-13T00:40:24.060 INFO:tasks.workunit.client.0.smithi136.stdout:iterate_cb 0~4194304 2024-09-13T00:40:24.061 INFO:tasks.workunit.client.0.smithi136.stdout:iterate_cb 4194304~4194304 2024-09-13T00:40:24.061 INFO:tasks.workunit.client.0.smithi136.stdout:iterate_cb 8388608~4194304 2024-09-13T00:40:24.061 INFO:tasks.workunit.client.0.smithi136.stdout:iterate_cb 16777216~4194304 2024-09-13T00:40:24.091 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] DiffIterateTest/1.DiffIterateParent (1217 ms) 2024-09-13T00:40:24.091 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] DiffIterateTest/1.DiffIterateIgnoreParent 2024-09-13T00:40:24.091 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:40:24.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:23 smithi181 bash[67587]: cluster 2024-09-13T00:40:22.816254+0000 mon.a (mon.0) 3662 : cluster [DBG] osdmap e1999: 8 total, 8 up, 8 in 2024-09-13T00:40:24.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:23 smithi136 bash[75164]: cluster 2024-09-13T00:40:22.816254+0000 mon.a (mon.0) 3662 : cluster [DBG] osdmap e1999: 8 total, 8 up, 8 in 2024-09-13T00:40:24.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:23 smithi136 bash[76371]: cluster 2024-09-13T00:40:22.816254+0000 mon.a (mon.0) 3662 : cluster [DBG] osdmap e1999: 8 total, 8 up, 8 in 2024-09-13T00:40:25.037 INFO:tasks.workunit.client.0.smithi136.stdout: wrote [0~20971520] to clone 2024-09-13T00:40:25.037 INFO:tasks.workunit.client.0.smithi136.stdout: diff was [0~20971520] 2024-09-13T00:40:25.047 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] DiffIterateTest/1.DiffIterateIgnoreParent (956 ms) 2024-09-13T00:40:25.047 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] DiffIterateTest/1.DiffIterateCallbackError 2024-09-13T00:40:25.047 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:40:25.095 INFO:tasks.workunit.client.0.smithi136.stdout: wrote [707895~99384,3886109~23014,4903559~94744,5850471~77024,7737042~36122,8793230~89052,13020582~79318,17558243~66512,17804426~1578,20861268~75308] 2024-09-13T00:40:25.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:24 smithi181 bash[67587]: cluster 2024-09-13T00:40:23.840488+0000 mon.a (mon.0) 3663 : cluster [DBG] osdmap e2000: 8 total, 8 up, 8 in 2024-09-13T00:40:25.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:24 smithi181 bash[67587]: audit 2024-09-13T00:40:23.841226+0000 mon.a (mon.0) 3664 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:40:25.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:24 smithi181 bash[67587]: cluster 2024-09-13T00:40:24.454159+0000 mgr.y (mgr.24661) 1898 : cluster [DBG] pgmap v3459: 137 pgs: 2 active+clean+snaptrim, 135 active+clean; 474 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 78 KiB/s rd, 8.5 KiB/s wr, 120 op/s 2024-09-13T00:40:25.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:24 smithi181 bash[67587]: cluster 2024-09-13T00:40:24.753169+0000 mon.a (mon.0) 3665 : cluster [DBG] osdmap e2001: 8 total, 8 up, 8 in 2024-09-13T00:40:25.126 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] DiffIterateTest/1.DiffIterateCallbackError (79 ms) 2024-09-13T00:40:25.127 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] DiffIterateTest/1.DiffIterateParentDiscard 2024-09-13T00:40:25.127 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:40:25.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:24 smithi136 bash[75164]: cluster 2024-09-13T00:40:23.840488+0000 mon.a (mon.0) 3663 : cluster [DBG] osdmap e2000: 8 total, 8 up, 8 in 2024-09-13T00:40:25.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:24 smithi136 bash[75164]: audit 2024-09-13T00:40:23.841226+0000 mon.a (mon.0) 3664 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:40:25.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:24 smithi136 bash[75164]: cluster 2024-09-13T00:40:24.454159+0000 mgr.y (mgr.24661) 1898 : cluster [DBG] pgmap v3459: 137 pgs: 2 active+clean+snaptrim, 135 active+clean; 474 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 78 KiB/s rd, 8.5 KiB/s wr, 120 op/s 2024-09-13T00:40:25.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:24 smithi136 bash[75164]: cluster 2024-09-13T00:40:24.753169+0000 mon.a (mon.0) 3665 : cluster [DBG] osdmap e2001: 8 total, 8 up, 8 in 2024-09-13T00:40:25.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:24 smithi136 bash[76371]: cluster 2024-09-13T00:40:23.840488+0000 mon.a (mon.0) 3663 : cluster [DBG] osdmap e2000: 8 total, 8 up, 8 in 2024-09-13T00:40:25.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:24 smithi136 bash[76371]: audit 2024-09-13T00:40:23.841226+0000 mon.a (mon.0) 3664 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:40:25.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:24 smithi136 bash[76371]: cluster 2024-09-13T00:40:24.454159+0000 mgr.y (mgr.24661) 1898 : cluster [DBG] pgmap v3459: 137 pgs: 2 active+clean+snaptrim, 135 active+clean; 474 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 78 KiB/s rd, 8.5 KiB/s wr, 120 op/s 2024-09-13T00:40:25.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:24 smithi136 bash[76371]: cluster 2024-09-13T00:40:24.753169+0000 mon.a (mon.0) 3665 : cluster [DBG] osdmap e2001: 8 total, 8 up, 8 in 2024-09-13T00:40:27.112 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] DiffIterateTest/1.DiffIterateParentDiscard (1986 ms) 2024-09-13T00:40:27.112 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] DiffIterateTest/1.DiffIterateUnalignedSmall 2024-09-13T00:40:27.113 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:40:27.213 INFO:tasks.workunit.client.0.smithi136.stdout:iterate_cb 5000005~1234 2024-09-13T00:40:27.221 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] DiffIterateTest/1.DiffIterateUnalignedSmall (109 ms) 2024-09-13T00:40:27.221 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] DiffIterateTest/1.DiffIterateUnaligned 2024-09-13T00:40:27.221 INFO:tasks.workunit.client.0.smithi136.stdout:using new format! 2024-09-13T00:40:27.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:26 smithi136 bash[75164]: cluster 2024-09-13T00:40:25.856513+0000 mon.a (mon.0) 3666 : cluster [DBG] osdmap e2002: 8 total, 8 up, 8 in 2024-09-13T00:40:27.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:26 smithi136 bash[75164]: cluster 2024-09-13T00:40:26.455070+0000 mgr.y (mgr.24661) 1899 : cluster [DBG] pgmap v3462: 137 pgs: 137 active+clean; 466 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 3.1 MiB/s rd, 6.6 MiB/s wr, 169 op/s 2024-09-13T00:40:27.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:26 smithi136 bash[76371]: cluster 2024-09-13T00:40:25.856513+0000 mon.a (mon.0) 3666 : cluster [DBG] osdmap e2002: 8 total, 8 up, 8 in 2024-09-13T00:40:27.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:26 smithi136 bash[76371]: cluster 2024-09-13T00:40:26.455070+0000 mgr.y (mgr.24661) 1899 : cluster [DBG] pgmap v3462: 137 pgs: 137 active+clean; 466 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 3.1 MiB/s rd, 6.6 MiB/s wr, 169 op/s 2024-09-13T00:40:27.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:26 smithi181 bash[67587]: cluster 2024-09-13T00:40:25.856513+0000 mon.a (mon.0) 3666 : cluster [DBG] osdmap e2002: 8 total, 8 up, 8 in 2024-09-13T00:40:27.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:26 smithi181 bash[67587]: cluster 2024-09-13T00:40:26.455070+0000 mgr.y (mgr.24661) 1899 : cluster [DBG] pgmap v3462: 137 pgs: 137 active+clean; 466 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 3.1 MiB/s rd, 6.6 MiB/s wr, 169 op/s 2024-09-13T00:40:27.351 INFO:tasks.workunit.client.0.smithi136.stdout:iterate_cb 8376263~12345 2024-09-13T00:40:27.351 INFO:tasks.workunit.client.0.smithi136.stdout:iterate_cb 8388608~4194304 2024-09-13T00:40:27.351 INFO:tasks.workunit.client.0.smithi136.stdout:iterate_cb 12582912~54321 2024-09-13T00:40:27.362 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] DiffIterateTest/1.DiffIterateUnaligned (142 ms) 2024-09-13T00:40:27.362 INFO:tasks.workunit.client.0.smithi136.stdout:[ RUN ] DiffIterateTest/1.DiffIterateStriping 2024-09-13T00:40:27.363 INFO:tasks.workunit.client.0.smithi136.stdout:SKIPPING 2024-09-13T00:40:27.363 INFO:tasks.workunit.client.0.smithi136.stdout:[ OK ] DiffIterateTest/1.DiffIterateStriping (0 ms) 2024-09-13T00:40:27.882 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] 11 tests from DiffIterateTest/1 (53429 ms total) 2024-09-13T00:40:27.883 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:40:27.883 INFO:tasks.workunit.client.0.smithi136.stdout:[----------] Global test environment tear-down 2024-09-13T00:40:27.883 INFO:tasks.workunit.client.0.smithi136.stdout:[==========] 406 tests from 15 test suites ran. (1061894 ms total) 2024-09-13T00:40:27.883 INFO:tasks.workunit.client.0.smithi136.stdout:[ PASSED ] 406 tests. 2024-09-13T00:40:27.883 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:40:27.883 INFO:tasks.workunit.client.0.smithi136.stdout: YOU HAVE 2 DISABLED TESTS 2024-09-13T00:40:27.883 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:40:27.910 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-13T00:40:27.911 DEBUG:teuthology.orchestra.run.smithi136:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-13T00:40:27.963 INFO:tasks.workunit:Stopping ['rbd/test_librbd.sh'] on client.0... 2024-09-13T00:40:27.963 DEBUG:teuthology.orchestra.run.smithi136:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2024-09-13T00:40:28.233 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:27 smithi136 bash[75164]: cluster 2024-09-13T00:40:26.870693+0000 mon.a (mon.0) 3667 : cluster [DBG] osdmap e2003: 8 total, 8 up, 8 in 2024-09-13T00:40:28.234 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:27 smithi136 bash[76371]: cluster 2024-09-13T00:40:26.870693+0000 mon.a (mon.0) 3667 : cluster [DBG] osdmap e2003: 8 total, 8 up, 8 in 2024-09-13T00:40:28.235 DEBUG:teuthology.parallel:result is None 2024-09-13T00:40:28.235 DEBUG:teuthology.orchestra.run.smithi136:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2024-09-13T00:40:28.246 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2024-09-13T00:40:28.246 DEBUG:teuthology.orchestra.run.smithi136:> rmdir -- /home/ubuntu/cephtest/mnt.0 2024-09-13T00:40:28.292 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2024-09-13T00:40:28.292 INFO:teuthology.task.full_sequential:In full_sequential, running task print... 2024-09-13T00:40:28.292 INFO:teuthology.task.print:**** done end test_rbd_api.yaml 2024-09-13T00:40:28.292 INFO:teuthology.task.full_sequential:In full_sequential, running task print... 2024-09-13T00:40:28.293 INFO:teuthology.task.print:**** done start test_rbd_python.yaml 2024-09-13T00:40:28.293 INFO:teuthology.task.full_sequential:In full_sequential, running task workunit... 2024-09-13T00:40:28.293 INFO:tasks.workunit:Pulling workunits from ref quincy 2024-09-13T00:40:28.294 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-09-13T00:40:28.294 DEBUG:teuthology.orchestra.run.smithi136:> stat -- /home/ubuntu/cephtest/mnt.0 2024-09-13T00:40:28.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T00:40:28.340 INFO:teuthology.orchestra.run.smithi136.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2024-09-13T00:40:28.340 DEBUG:teuthology.orchestra.run.smithi136:> mkdir -- /home/ubuntu/cephtest/mnt.0 2024-09-13T00:40:28.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:27 smithi181 bash[67587]: cluster 2024-09-13T00:40:26.870693+0000 mon.a (mon.0) 3667 : cluster [DBG] osdmap e2003: 8 total, 8 up, 8 in 2024-09-13T00:40:28.389 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2024-09-13T00:40:28.389 DEBUG:teuthology.orchestra.run.smithi136:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2024-09-13T00:40:28.436 INFO:tasks.workunit:timeout=3h 2024-09-13T00:40:28.436 INFO:tasks.workunit:cleanup=True 2024-09-13T00:40:28.437 DEBUG:teuthology.orchestra.run.smithi136:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://git.ceph.com/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2024-09-13T00:40:28.486 INFO:tasks.workunit.client.0.smithi136.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-09-13T00:40:29.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:28 smithi136 bash[76371]: cluster 2024-09-13T00:40:27.876990+0000 mon.a (mon.0) 3668 : cluster [DBG] osdmap e2004: 8 total, 8 up, 8 in 2024-09-13T00:40:29.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:28 smithi136 bash[76371]: cluster 2024-09-13T00:40:28.455739+0000 mgr.y (mgr.24661) 1900 : cluster [DBG] pgmap v3465: 105 pgs: 105 active+clean; 4.6 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2024-09-13T00:40:29.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:28 smithi136 bash[75164]: cluster 2024-09-13T00:40:27.876990+0000 mon.a (mon.0) 3668 : cluster [DBG] osdmap e2004: 8 total, 8 up, 8 in 2024-09-13T00:40:29.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:28 smithi136 bash[75164]: cluster 2024-09-13T00:40:28.455739+0000 mgr.y (mgr.24661) 1900 : cluster [DBG] pgmap v3465: 105 pgs: 105 active+clean; 4.6 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2024-09-13T00:40:29.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:28 smithi181 bash[67587]: cluster 2024-09-13T00:40:27.876990+0000 mon.a (mon.0) 3668 : cluster [DBG] osdmap e2004: 8 total, 8 up, 8 in 2024-09-13T00:40:29.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:28 smithi181 bash[67587]: cluster 2024-09-13T00:40:28.455739+0000 mgr.y (mgr.24661) 1900 : cluster [DBG] pgmap v3465: 105 pgs: 105 active+clean; 4.6 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2024-09-13T00:40:29.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:40:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:40:29] "GET /metrics HTTP/1.1" 200 36753 "" "Prometheus/2.43.0" 2024-09-13T00:40:30.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:30 smithi181 bash[67587]: cluster 2024-09-13T00:40:30.456545+0000 mgr.y (mgr.24661) 1901 : cluster [DBG] pgmap v3466: 105 pgs: 105 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T00:40:31.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:30 smithi136 bash[75164]: cluster 2024-09-13T00:40:30.456545+0000 mgr.y (mgr.24661) 1901 : cluster [DBG] pgmap v3466: 105 pgs: 105 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T00:40:31.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:30 smithi136 bash[76371]: cluster 2024-09-13T00:40:30.456545+0000 mgr.y (mgr.24661) 1901 : cluster [DBG] pgmap v3466: 105 pgs: 105 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T00:40:32.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:32 smithi181 bash[67587]: cluster 2024-09-13T00:40:32.457901+0000 mgr.y (mgr.24661) 1902 : cluster [DBG] pgmap v3467: 105 pgs: 105 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:40:32.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:40:32 smithi181 bash[95505]: ts=2024-09-13T00:40:32.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:40:33.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:32 smithi136 bash[75164]: cluster 2024-09-13T00:40:32.457901+0000 mgr.y (mgr.24661) 1902 : cluster [DBG] pgmap v3467: 105 pgs: 105 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:40:33.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:32 smithi136 bash[76371]: cluster 2024-09-13T00:40:32.457901+0000 mgr.y (mgr.24661) 1902 : cluster [DBG] pgmap v3467: 105 pgs: 105 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:40:34.710 DEBUG:teuthology.orchestra.run.smithi136:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-09-13T00:40:34.756 INFO:tasks.workunit.client.0.smithi136.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-09-13T00:40:34.759 INFO:tasks.workunit.client.0.smithi136.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-13T00:40:34.759 INFO:tasks.workunit.client.0.smithi136.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-09-13T00:40:34.814 INFO:tasks.workunit.client.0.smithi136.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-09-13T00:40:34.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:34 smithi181 bash[67587]: cluster 2024-09-13T00:40:34.458794+0000 mgr.y (mgr.24661) 1903 : cluster [DBG] pgmap v3468: 105 pgs: 105 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:40:34.869 INFO:tasks.workunit.client.0.smithi136.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-09-13T00:40:34.915 INFO:tasks.workunit.client.0.smithi136.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-13T00:40:34.916 INFO:tasks.workunit.client.0.smithi136.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-13T00:40:34.916 INFO:tasks.workunit.client.0.smithi136.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-09-13T00:40:34.960 INFO:tasks.workunit.client.0.smithi136.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-13T00:40:34.965 DEBUG:teuthology.orchestra.run.smithi136:> set -ex 2024-09-13T00:40:34.965 DEBUG:teuthology.orchestra.run.smithi136:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-09-13T00:40:35.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:34 smithi136 bash[75164]: cluster 2024-09-13T00:40:34.458794+0000 mgr.y (mgr.24661) 1903 : cluster [DBG] pgmap v3468: 105 pgs: 105 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:40:35.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:34 smithi136 bash[76371]: cluster 2024-09-13T00:40:34.458794+0000 mgr.y (mgr.24661) 1903 : cluster [DBG] pgmap v3468: 105 pgs: 105 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:40:35.012 INFO:tasks.workunit:Running workunits matching rbd/test_librbd_python.sh on client.0... 2024-09-13T00:40:35.013 INFO:tasks.workunit:Running workunit rbd/test_librbd_python.sh... 2024-09-13T00:40:35.013 DEBUG:teuthology.orchestra.run.smithi136:workunit test rbd/test_librbd_python.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_FEATURES=61 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/test_librbd_python.sh 2024-09-13T00:40:35.064 INFO:tasks.workunit.client.0.smithi136.stderr:+ dirname /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/test_librbd_python.sh 2024-09-13T00:40:35.065 INFO:tasks.workunit.client.0.smithi136.stderr:+ relpath=/home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/../../../src/test/pybind 2024-09-13T00:40:35.065 INFO:tasks.workunit.client.0.smithi136.stderr:+ [ -n ] 2024-09-13T00:40:35.065 INFO:tasks.workunit.client.0.smithi136.stderr:+ python3 -m pytest -v /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/../../../src/test/pybind/test_rbd.py 2024-09-13T00:40:35.247 INFO:tasks.workunit.client.0.smithi136.stdout:============================= test session starts ============================== 2024-09-13T00:40:35.247 INFO:tasks.workunit.client.0.smithi136.stdout:platform linux -- Python 3.10.6, pytest-6.2.5, py-1.10.0, pluggy-0.13.0 -- /usr/bin/python3 2024-09-13T00:40:35.248 INFO:tasks.workunit.client.0.smithi136.stdout:cachedir: .pytest_cache 2024-09-13T00:40:35.248 INFO:tasks.workunit.client.0.smithi136.stdout:rootdir: /home/ubuntu/cephtest/clone.client.0/src/test/pybind, configfile: pytest.ini 2024-09-13T00:40:35.716 INFO:tasks.workunit.client.0.smithi136.stdout:collecting ... collected 182 items 2024-09-13T00:40:35.716 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:40:37.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:37 smithi136 bash[75164]: cluster 2024-09-13T00:40:36.204396+0000 mon.a (mon.0) 3669 : cluster [DBG] osdmap e2005: 8 total, 8 up, 8 in 2024-09-13T00:40:37.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:37 smithi136 bash[75164]: audit 2024-09-13T00:40:36.207407+0000 mon.c (mon.2) 145 : audit [INF] from='client.? 172.21.15.136:0/1130477352' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rbd-api-smithi136-117597-1","app": "rbd","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:40:37.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:37 smithi136 bash[75164]: audit 2024-09-13T00:40:36.208462+0000 mon.a (mon.0) 3670 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rbd-api-smithi136-117597-1","app": "rbd","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:40:37.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:37 smithi136 bash[75164]: cluster 2024-09-13T00:40:36.459649+0000 mgr.y (mgr.24661) 1904 : cluster [DBG] pgmap v3470: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:40:37.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:37 smithi136 bash[76371]: cluster 2024-09-13T00:40:36.204396+0000 mon.a (mon.0) 3669 : cluster [DBG] osdmap e2005: 8 total, 8 up, 8 in 2024-09-13T00:40:37.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:37 smithi136 bash[76371]: audit 2024-09-13T00:40:36.207407+0000 mon.c (mon.2) 145 : audit [INF] from='client.? 172.21.15.136:0/1130477352' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rbd-api-smithi136-117597-1","app": "rbd","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:40:37.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:37 smithi136 bash[76371]: audit 2024-09-13T00:40:36.208462+0000 mon.a (mon.0) 3670 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rbd-api-smithi136-117597-1","app": "rbd","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:40:37.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:37 smithi136 bash[76371]: cluster 2024-09-13T00:40:36.459649+0000 mgr.y (mgr.24661) 1904 : cluster [DBG] pgmap v3470: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:40:37.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:37 smithi181 bash[67587]: cluster 2024-09-13T00:40:36.204396+0000 mon.a (mon.0) 3669 : cluster [DBG] osdmap e2005: 8 total, 8 up, 8 in 2024-09-13T00:40:37.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:37 smithi181 bash[67587]: audit 2024-09-13T00:40:36.207407+0000 mon.c (mon.2) 145 : audit [INF] from='client.? 172.21.15.136:0/1130477352' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rbd-api-smithi136-117597-1","app": "rbd","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:40:37.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:37 smithi181 bash[67587]: audit 2024-09-13T00:40:36.208462+0000 mon.a (mon.0) 3670 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rbd-api-smithi136-117597-1","app": "rbd","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:40:37.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:37 smithi181 bash[67587]: cluster 2024-09-13T00:40:36.459649+0000 mgr.y (mgr.24661) 1904 : cluster [DBG] pgmap v3470: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:40:38.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:38 smithi136 bash[75164]: cluster 2024-09-13T00:40:37.195935+0000 mon.a (mon.0) 3671 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:40:38.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:38 smithi136 bash[75164]: audit 2024-09-13T00:40:37.205678+0000 mon.a (mon.0) 3672 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rbd-api-smithi136-117597-1","app": "rbd","yes_i_really_mean_it": true}]': finished 2024-09-13T00:40:38.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:38 smithi136 bash[75164]: cluster 2024-09-13T00:40:37.206551+0000 mon.a (mon.0) 3673 : cluster [DBG] osdmap e2006: 8 total, 8 up, 8 in 2024-09-13T00:40:38.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:38 smithi136 bash[76371]: cluster 2024-09-13T00:40:37.195935+0000 mon.a (mon.0) 3671 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:40:38.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:38 smithi136 bash[76371]: audit 2024-09-13T00:40:37.205678+0000 mon.a (mon.0) 3672 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rbd-api-smithi136-117597-1","app": "rbd","yes_i_really_mean_it": true}]': finished 2024-09-13T00:40:38.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:38 smithi136 bash[76371]: cluster 2024-09-13T00:40:37.206551+0000 mon.a (mon.0) 3673 : cluster [DBG] osdmap e2006: 8 total, 8 up, 8 in 2024-09-13T00:40:38.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:38 smithi181 bash[67587]: cluster 2024-09-13T00:40:37.195935+0000 mon.a (mon.0) 3671 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:40:38.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:38 smithi181 bash[67587]: audit 2024-09-13T00:40:37.205678+0000 mon.a (mon.0) 3672 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rbd-api-smithi136-117597-1","app": "rbd","yes_i_really_mean_it": true}]': finished 2024-09-13T00:40:38.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:38 smithi181 bash[67587]: cluster 2024-09-13T00:40:37.206551+0000 mon.a (mon.0) 3673 : cluster [DBG] osdmap e2006: 8 total, 8 up, 8 in 2024-09-13T00:40:39.275 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::test_version PASSED [ 0%] 2024-09-13T00:40:39.443 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::test_create PASSED [ 1%] 2024-09-13T00:40:39.509 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:40:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:40:39] "GET /metrics HTTP/1.1" 200 36687 "" "Prometheus/2.43.0" 2024-09-13T00:40:39.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:39 smithi136 bash[76371]: cluster 2024-09-13T00:40:38.231619+0000 mon.a (mon.0) 3674 : cluster [DBG] osdmap e2007: 8 total, 8 up, 8 in 2024-09-13T00:40:39.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:39 smithi136 bash[76371]: cluster 2024-09-13T00:40:38.460315+0000 mgr.y (mgr.24661) 1905 : cluster [DBG] pgmap v3473: 137 pgs: 8 creating+peering, 20 unknown, 109 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:40:39.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:39 smithi136 bash[76371]: audit 2024-09-13T00:40:38.845883+0000 mon.a (mon.0) 3675 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:40:39.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:39 smithi136 bash[76371]: audit 2024-09-13T00:40:38.848032+0000 mon.a (mon.0) 3676 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:40:39.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:39 smithi136 bash[75164]: cluster 2024-09-13T00:40:38.231619+0000 mon.a (mon.0) 3674 : cluster [DBG] osdmap e2007: 8 total, 8 up, 8 in 2024-09-13T00:40:39.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:39 smithi136 bash[75164]: cluster 2024-09-13T00:40:38.460315+0000 mgr.y (mgr.24661) 1905 : cluster [DBG] pgmap v3473: 137 pgs: 8 creating+peering, 20 unknown, 109 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:40:39.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:39 smithi136 bash[75164]: audit 2024-09-13T00:40:38.845883+0000 mon.a (mon.0) 3675 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:40:39.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:39 smithi136 bash[75164]: audit 2024-09-13T00:40:38.848032+0000 mon.a (mon.0) 3676 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:40:39.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:39 smithi181 bash[67587]: cluster 2024-09-13T00:40:38.231619+0000 mon.a (mon.0) 3674 : cluster [DBG] osdmap e2007: 8 total, 8 up, 8 in 2024-09-13T00:40:39.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:39 smithi181 bash[67587]: cluster 2024-09-13T00:40:38.460315+0000 mgr.y (mgr.24661) 1905 : cluster [DBG] pgmap v3473: 137 pgs: 8 creating+peering, 20 unknown, 109 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:40:39.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:39 smithi181 bash[67587]: audit 2024-09-13T00:40:38.845883+0000 mon.a (mon.0) 3675 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:40:39.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:39 smithi181 bash[67587]: audit 2024-09-13T00:40:38.848032+0000 mon.a (mon.0) 3676 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:40:40.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:40 smithi136 bash[75164]: cluster 2024-09-13T00:40:39.224883+0000 mon.a (mon.0) 3677 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:40:40.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:40 smithi136 bash[75164]: cluster 2024-09-13T00:40:39.224936+0000 mon.a (mon.0) 3678 : cluster [INF] Cluster is now healthy 2024-09-13T00:40:40.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:40 smithi136 bash[75164]: cluster 2024-09-13T00:40:39.249798+0000 mon.a (mon.0) 3679 : cluster [DBG] osdmap e2008: 8 total, 8 up, 8 in 2024-09-13T00:40:40.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:40 smithi136 bash[76371]: cluster 2024-09-13T00:40:39.224883+0000 mon.a (mon.0) 3677 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:40:40.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:40 smithi136 bash[76371]: cluster 2024-09-13T00:40:39.224936+0000 mon.a (mon.0) 3678 : cluster [INF] Cluster is now healthy 2024-09-13T00:40:40.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:40 smithi136 bash[76371]: cluster 2024-09-13T00:40:39.249798+0000 mon.a (mon.0) 3679 : cluster [DBG] osdmap e2008: 8 total, 8 up, 8 in 2024-09-13T00:40:40.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:40 smithi181 bash[67587]: cluster 2024-09-13T00:40:39.224883+0000 mon.a (mon.0) 3677 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:40:40.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:40 smithi181 bash[67587]: cluster 2024-09-13T00:40:39.224936+0000 mon.a (mon.0) 3678 : cluster [INF] Cluster is now healthy 2024-09-13T00:40:40.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:40 smithi181 bash[67587]: cluster 2024-09-13T00:40:39.249798+0000 mon.a (mon.0) 3679 : cluster [DBG] osdmap e2008: 8 total, 8 up, 8 in 2024-09-13T00:40:41.046 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::test_create_defaults PASSED [ 1%] 2024-09-13T00:40:41.278 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::test_context_manager PASSED [ 2%] 2024-09-13T00:40:41.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:41 smithi136 bash[75164]: cluster 2024-09-13T00:40:40.461626+0000 mgr.y (mgr.24661) 1906 : cluster [DBG] pgmap v3475: 137 pgs: 8 creating+peering, 10 unknown, 119 active+clean; 4.6 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 1.9 KiB/s rd, 240 B/s wr, 4 op/s 2024-09-13T00:40:41.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:41 smithi136 bash[76371]: cluster 2024-09-13T00:40:40.461626+0000 mgr.y (mgr.24661) 1906 : cluster [DBG] pgmap v3475: 137 pgs: 8 creating+peering, 10 unknown, 119 active+clean; 4.6 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 1.9 KiB/s rd, 240 B/s wr, 4 op/s 2024-09-13T00:40:41.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:41 smithi181 bash[67587]: cluster 2024-09-13T00:40:40.461626+0000 mgr.y (mgr.24661) 1906 : cluster [DBG] pgmap v3475: 137 pgs: 8 creating+peering, 10 unknown, 119 active+clean; 4.6 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 1.9 KiB/s rd, 240 B/s wr, 4 op/s 2024-09-13T00:40:42.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:40:42 smithi181 bash[95505]: ts=2024-09-13T00:40:42.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:40:43.482 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::test_open_read_only PASSED [ 2%] 2024-09-13T00:40:43.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:43 smithi181 bash[67587]: cluster 2024-09-13T00:40:42.259714+0000 mon.a (mon.0) 3680 : cluster [DBG] osdmap e2009: 8 total, 8 up, 8 in 2024-09-13T00:40:43.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:43 smithi181 bash[67587]: audit 2024-09-13T00:40:42.272473+0000 mon.a (mon.0) 3681 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:40:43.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:43 smithi181 bash[67587]: cluster 2024-09-13T00:40:42.462299+0000 mgr.y (mgr.24661) 1907 : cluster [DBG] pgmap v3477: 137 pgs: 137 active+clean; 4.6 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 177 KiB/s rd, 15 KiB/s wr, 240 op/s 2024-09-13T00:40:43.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:43 smithi181 bash[67587]: audit 2024-09-13T00:40:42.810629+0000 mon.a (mon.0) 3682 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:40:43.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:43 smithi181 bash[67587]: audit 2024-09-13T00:40:42.811834+0000 mon.a (mon.0) 3683 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:40:43.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:43 smithi181 bash[67587]: audit 2024-09-13T00:40:42.822339+0000 mon.a (mon.0) 3684 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:40:43.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:43 smithi136 bash[75164]: cluster 2024-09-13T00:40:42.259714+0000 mon.a (mon.0) 3680 : cluster [DBG] osdmap e2009: 8 total, 8 up, 8 in 2024-09-13T00:40:43.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:43 smithi136 bash[75164]: audit 2024-09-13T00:40:42.272473+0000 mon.a (mon.0) 3681 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:40:43.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:43 smithi136 bash[75164]: cluster 2024-09-13T00:40:42.462299+0000 mgr.y (mgr.24661) 1907 : cluster [DBG] pgmap v3477: 137 pgs: 137 active+clean; 4.6 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 177 KiB/s rd, 15 KiB/s wr, 240 op/s 2024-09-13T00:40:43.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:43 smithi136 bash[75164]: audit 2024-09-13T00:40:42.810629+0000 mon.a (mon.0) 3682 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:40:43.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:43 smithi136 bash[75164]: audit 2024-09-13T00:40:42.811834+0000 mon.a (mon.0) 3683 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:40:43.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:43 smithi136 bash[75164]: audit 2024-09-13T00:40:42.822339+0000 mon.a (mon.0) 3684 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:40:43.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:43 smithi136 bash[76371]: cluster 2024-09-13T00:40:42.259714+0000 mon.a (mon.0) 3680 : cluster [DBG] osdmap e2009: 8 total, 8 up, 8 in 2024-09-13T00:40:43.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:43 smithi136 bash[76371]: audit 2024-09-13T00:40:42.272473+0000 mon.a (mon.0) 3681 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:40:43.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:43 smithi136 bash[76371]: cluster 2024-09-13T00:40:42.462299+0000 mgr.y (mgr.24661) 1907 : cluster [DBG] pgmap v3477: 137 pgs: 137 active+clean; 4.6 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 177 KiB/s rd, 15 KiB/s wr, 240 op/s 2024-09-13T00:40:43.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:43 smithi136 bash[76371]: audit 2024-09-13T00:40:42.810629+0000 mon.a (mon.0) 3682 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:40:43.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:43 smithi136 bash[76371]: audit 2024-09-13T00:40:42.811834+0000 mon.a (mon.0) 3683 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:40:43.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:43 smithi136 bash[76371]: audit 2024-09-13T00:40:42.822339+0000 mon.a (mon.0) 3684 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:40:44.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:44 smithi181 bash[67587]: cluster 2024-09-13T00:40:43.276489+0000 mon.a (mon.0) 3685 : cluster [DBG] osdmap e2010: 8 total, 8 up, 8 in 2024-09-13T00:40:44.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:44 smithi136 bash[75164]: cluster 2024-09-13T00:40:43.276489+0000 mon.a (mon.0) 3685 : cluster [DBG] osdmap e2010: 8 total, 8 up, 8 in 2024-09-13T00:40:44.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:44 smithi136 bash[76371]: cluster 2024-09-13T00:40:43.276489+0000 mon.a (mon.0) 3685 : cluster [DBG] osdmap e2010: 8 total, 8 up, 8 in 2024-09-13T00:40:44.790 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::test_open_dne PASSED [ 3%] 2024-09-13T00:40:45.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:45 smithi181 bash[67587]: cluster 2024-09-13T00:40:44.463337+0000 mgr.y (mgr.24661) 1908 : cluster [DBG] pgmap v3479: 137 pgs: 137 active+clean; 4.6 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 215 KiB/s rd, 17 KiB/s wr, 293 op/s 2024-09-13T00:40:45.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:45 smithi136 bash[75164]: cluster 2024-09-13T00:40:44.463337+0000 mgr.y (mgr.24661) 1908 : cluster [DBG] pgmap v3479: 137 pgs: 137 active+clean; 4.6 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 215 KiB/s rd, 17 KiB/s wr, 293 op/s 2024-09-13T00:40:45.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:45 smithi136 bash[76371]: cluster 2024-09-13T00:40:44.463337+0000 mgr.y (mgr.24661) 1908 : cluster [DBG] pgmap v3479: 137 pgs: 137 active+clean; 4.6 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 215 KiB/s rd, 17 KiB/s wr, 293 op/s 2024-09-13T00:40:46.095 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::test_open_readonly_dne PASSED [ 3%] 2024-09-13T00:40:46.269 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::test_open_by_id PASSED [ 4%] 2024-09-13T00:40:46.400 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::test_aio_open PASSED [ 4%] 2024-09-13T00:40:46.420 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::test_remove_dne PASSED [ 5%] 2024-09-13T00:40:46.423 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::test_list_empty PASSED [ 6%] 2024-09-13T00:40:46.627 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::test_list PASSED [ 6%] 2024-09-13T00:40:46.808 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::test_remove_with_progress PASSED [ 7%] 2024-09-13T00:40:46.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:46 smithi181 bash[67587]: cluster 2024-09-13T00:40:46.464621+0000 mgr.y (mgr.24661) 1909 : cluster [DBG] pgmap v3480: 137 pgs: 137 active+clean; 4.6 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 200 KiB/s rd, 16 KiB/s wr, 273 op/s 2024-09-13T00:40:47.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:46 smithi136 bash[75164]: cluster 2024-09-13T00:40:46.464621+0000 mgr.y (mgr.24661) 1909 : cluster [DBG] pgmap v3480: 137 pgs: 137 active+clean; 4.6 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 200 KiB/s rd, 16 KiB/s wr, 273 op/s 2024-09-13T00:40:47.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:46 smithi136 bash[76371]: cluster 2024-09-13T00:40:46.464621+0000 mgr.y (mgr.24661) 1909 : cluster [DBG] pgmap v3480: 137 pgs: 137 active+clean; 4.6 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 200 KiB/s rd, 16 KiB/s wr, 273 op/s 2024-09-13T00:40:47.024 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::test_remove_canceled PASSED [ 7%] 2024-09-13T00:40:47.297 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::test_rename PASSED [ 8%] 2024-09-13T00:40:48.068 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::test_pool_metadata PASSED [ 8%] 2024-09-13T00:40:48.113 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::test_config_list PASSED [ 9%] 2024-09-13T00:40:48.148 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::test_pool_config_set_and_get_and_remove PASSED [ 9%] 2024-09-13T00:40:48.199 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::test_namespaces PASSED [ 10%] 2024-09-13T00:40:48.430 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:40:48 smithi136 bash[68787]: [13/Sep/2024:00:40:48] ENGINE Bus STOPPING 2024-09-13T00:40:48.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:40:48 smithi136 bash[68787]: [13/Sep/2024:00:40:48] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T00:40:48.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:40:48 smithi136 bash[68787]: [13/Sep/2024:00:40:48] ENGINE Bus STOPPED 2024-09-13T00:40:48.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:40:48 smithi136 bash[68787]: [13/Sep/2024:00:40:48] ENGINE Bus STARTING 2024-09-13T00:40:48.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:40:48 smithi136 bash[68787]: [13/Sep/2024:00:40:48] ENGINE Serving on http://:::9283 2024-09-13T00:40:48.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:40:48 smithi136 bash[68787]: [13/Sep/2024:00:40:48] ENGINE Bus STARTED 2024-09-13T00:40:49.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:49 smithi181 bash[67587]: audit 2024-09-13T00:40:48.096699+0000 mon.a (mon.0) 3686 : audit [INF] from='client.? ' entity='client.admin' 2024-09-13T00:40:49.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:49 smithi181 bash[67587]: audit 2024-09-13T00:40:48.118564+0000 mon.a (mon.0) 3687 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:40:49.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:49 smithi181 bash[67587]: audit 2024-09-13T00:40:48.125309+0000 mon.a (mon.0) 3688 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:40:49.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:49 smithi181 bash[67587]: audit 2024-09-13T00:40:48.129116+0000 mon.a (mon.0) 3689 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:40:49.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:49 smithi181 bash[67587]: audit 2024-09-13T00:40:48.130856+0000 mon.a (mon.0) 3690 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:40:49.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:49 smithi181 bash[67587]: audit 2024-09-13T00:40:48.138791+0000 mon.a (mon.0) 3691 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:40:49.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:49 smithi181 bash[67587]: audit 2024-09-13T00:40:48.162281+0000 mon.a (mon.0) 3692 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T00:40:49.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:49 smithi181 bash[67587]: audit 2024-09-13T00:40:48.327845+0000 mon.a (mon.0) 3693 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:40:49.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:49 smithi181 bash[67587]: audit 2024-09-13T00:40:48.328264+0000 mon.a (mon.0) 3694 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:40:49.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:49 smithi181 bash[67587]: audit 2024-09-13T00:40:48.328542+0000 mon.a (mon.0) 3695 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:40:49.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:49 smithi181 bash[67587]: audit 2024-09-13T00:40:48.332009+0000 mon.a (mon.0) 3696 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:40:49.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:49 smithi181 bash[67587]: audit 2024-09-13T00:40:48.332500+0000 mon.a (mon.0) 3697 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:40:49.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:49 smithi181 bash[67587]: audit 2024-09-13T00:40:48.332887+0000 mon.a (mon.0) 3698 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:40:49.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:49 smithi181 bash[67587]: audit 2024-09-13T00:40:48.335560+0000 mon.a (mon.0) 3699 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:40:49.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:49 smithi181 bash[67587]: audit 2024-09-13T00:40:48.336315+0000 mon.a (mon.0) 3700 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:40:49.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:49 smithi181 bash[67587]: audit 2024-09-13T00:40:48.336773+0000 mon.a (mon.0) 3701 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:40:49.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:49 smithi181 bash[67587]: audit 2024-09-13T00:40:48.354879+0000 mon.a (mon.0) 3702 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T00:40:49.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:49 smithi181 bash[67587]: audit 2024-09-13T00:40:48.355039+0000 mon.a (mon.0) 3703 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T00:40:49.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:49 smithi181 bash[67587]: audit 2024-09-13T00:40:48.355529+0000 mon.a (mon.0) 3704 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:40:49.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:49 smithi181 bash[67587]: audit 2024-09-13T00:40:48.355960+0000 mon.a (mon.0) 3705 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:40:49.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:49 smithi181 bash[67587]: audit 2024-09-13T00:40:48.356310+0000 mon.a (mon.0) 3706 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:40:49.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:49 smithi181 bash[67587]: audit 2024-09-13T00:40:48.359174+0000 mon.b (mon.1) 25 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T00:40:49.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:49 smithi181 bash[67587]: audit 2024-09-13T00:40:48.359272+0000 mon.b (mon.1) 26 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T00:40:49.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:49 smithi181 bash[67587]: audit 2024-09-13T00:40:48.359570+0000 mon.a (mon.0) 3707 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:40:49.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:49 smithi181 bash[67587]: audit 2024-09-13T00:40:48.360248+0000 mon.a (mon.0) 3708 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:40:49.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:49 smithi181 bash[67587]: audit 2024-09-13T00:40:48.360543+0000 mon.a (mon.0) 3709 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:40:49.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:49 smithi181 bash[67587]: audit 2024-09-13T00:40:48.363088+0000 mon.c (mon.2) 146 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T00:40:49.352 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:49 smithi181 bash[67587]: audit 2024-09-13T00:40:48.363225+0000 mon.c (mon.2) 147 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T00:40:49.352 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:49 smithi181 bash[67587]: audit 2024-09-13T00:40:48.363644+0000 mon.a (mon.0) 3710 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:40:49.352 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:49 smithi181 bash[67587]: audit 2024-09-13T00:40:48.364355+0000 mon.a (mon.0) 3711 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:40:49.352 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:49 smithi181 bash[67587]: audit 2024-09-13T00:40:48.364628+0000 mon.a (mon.0) 3712 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:40:49.352 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:49 smithi181 bash[67587]: cluster 2024-09-13T00:40:48.465309+0000 mgr.y (mgr.24661) 1910 : cluster [DBG] pgmap v3481: 137 pgs: 137 active+clean; 4.6 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 453 KiB/s rd, 32 KiB/s wr, 290 op/s 2024-09-13T00:40:49.416 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[76371]: audit 2024-09-13T00:40:48.096699+0000 mon.a (mon.0) 3686 : audit [INF] from='client.? ' entity='client.admin' 2024-09-13T00:40:49.416 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[76371]: audit 2024-09-13T00:40:48.118564+0000 mon.a (mon.0) 3687 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:40:49.416 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[76371]: audit 2024-09-13T00:40:48.125309+0000 mon.a (mon.0) 3688 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:40:49.416 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[76371]: audit 2024-09-13T00:40:48.129116+0000 mon.a (mon.0) 3689 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:40:49.416 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[76371]: audit 2024-09-13T00:40:48.130856+0000 mon.a (mon.0) 3690 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:40:49.416 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[76371]: audit 2024-09-13T00:40:48.138791+0000 mon.a (mon.0) 3691 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:40:49.416 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[76371]: audit 2024-09-13T00:40:48.162281+0000 mon.a (mon.0) 3692 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T00:40:49.417 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[76371]: audit 2024-09-13T00:40:48.327845+0000 mon.a (mon.0) 3693 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:40:49.417 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[76371]: audit 2024-09-13T00:40:48.328264+0000 mon.a (mon.0) 3694 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:40:49.417 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[76371]: audit 2024-09-13T00:40:48.328542+0000 mon.a (mon.0) 3695 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:40:49.417 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[76371]: audit 2024-09-13T00:40:48.332009+0000 mon.a (mon.0) 3696 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:40:49.417 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[76371]: audit 2024-09-13T00:40:48.332500+0000 mon.a (mon.0) 3697 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:40:49.417 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[76371]: audit 2024-09-13T00:40:48.332887+0000 mon.a (mon.0) 3698 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:40:49.417 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[76371]: audit 2024-09-13T00:40:48.335560+0000 mon.a (mon.0) 3699 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:40:49.417 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[76371]: audit 2024-09-13T00:40:48.336315+0000 mon.a (mon.0) 3700 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:40:49.417 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[76371]: audit 2024-09-13T00:40:48.336773+0000 mon.a (mon.0) 3701 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:40:49.417 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[76371]: audit 2024-09-13T00:40:48.354879+0000 mon.a (mon.0) 3702 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T00:40:49.417 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[76371]: audit 2024-09-13T00:40:48.355039+0000 mon.a (mon.0) 3703 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T00:40:49.417 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[76371]: audit 2024-09-13T00:40:48.355529+0000 mon.a (mon.0) 3704 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:40:49.418 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[75164]: audit 2024-09-13T00:40:48.096699+0000 mon.a (mon.0) 3686 : audit [INF] from='client.? ' entity='client.admin' 2024-09-13T00:40:49.418 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[75164]: audit 2024-09-13T00:40:48.118564+0000 mon.a (mon.0) 3687 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:40:49.418 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[75164]: audit 2024-09-13T00:40:48.125309+0000 mon.a (mon.0) 3688 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:40:49.418 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[75164]: audit 2024-09-13T00:40:48.129116+0000 mon.a (mon.0) 3689 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:40:49.418 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[75164]: audit 2024-09-13T00:40:48.130856+0000 mon.a (mon.0) 3690 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:40:49.418 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[75164]: audit 2024-09-13T00:40:48.138791+0000 mon.a (mon.0) 3691 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:40:49.418 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[75164]: audit 2024-09-13T00:40:48.162281+0000 mon.a (mon.0) 3692 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T00:40:49.418 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[75164]: audit 2024-09-13T00:40:48.327845+0000 mon.a (mon.0) 3693 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:40:49.419 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[75164]: audit 2024-09-13T00:40:48.328264+0000 mon.a (mon.0) 3694 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:40:49.419 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[75164]: audit 2024-09-13T00:40:48.328542+0000 mon.a (mon.0) 3695 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:40:49.419 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[75164]: audit 2024-09-13T00:40:48.332009+0000 mon.a (mon.0) 3696 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:40:49.419 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[75164]: audit 2024-09-13T00:40:48.332500+0000 mon.a (mon.0) 3697 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:40:49.419 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[75164]: audit 2024-09-13T00:40:48.332887+0000 mon.a (mon.0) 3698 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:40:49.419 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[75164]: audit 2024-09-13T00:40:48.335560+0000 mon.a (mon.0) 3699 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:40:49.419 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[75164]: audit 2024-09-13T00:40:48.336315+0000 mon.a (mon.0) 3700 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:40:49.419 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[75164]: audit 2024-09-13T00:40:48.336773+0000 mon.a (mon.0) 3701 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:40:49.419 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[75164]: audit 2024-09-13T00:40:48.354879+0000 mon.a (mon.0) 3702 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T00:40:49.419 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[75164]: audit 2024-09-13T00:40:48.355039+0000 mon.a (mon.0) 3703 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T00:40:49.419 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[75164]: audit 2024-09-13T00:40:48.355529+0000 mon.a (mon.0) 3704 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:40:49.419 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[75164]: audit 2024-09-13T00:40:48.355960+0000 mon.a (mon.0) 3705 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:40:49.419 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[75164]: audit 2024-09-13T00:40:48.356310+0000 mon.a (mon.0) 3706 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:40:49.419 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[75164]: audit 2024-09-13T00:40:48.359174+0000 mon.b (mon.1) 25 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T00:40:49.420 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[75164]: audit 2024-09-13T00:40:48.359272+0000 mon.b (mon.1) 26 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T00:40:49.420 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[75164]: audit 2024-09-13T00:40:48.359570+0000 mon.a (mon.0) 3707 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:40:49.420 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[75164]: audit 2024-09-13T00:40:48.360248+0000 mon.a (mon.0) 3708 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:40:49.420 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[75164]: audit 2024-09-13T00:40:48.360543+0000 mon.a (mon.0) 3709 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:40:49.420 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[75164]: audit 2024-09-13T00:40:48.363088+0000 mon.c (mon.2) 146 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T00:40:49.420 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[75164]: audit 2024-09-13T00:40:48.363225+0000 mon.c (mon.2) 147 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T00:40:49.420 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[75164]: audit 2024-09-13T00:40:48.363644+0000 mon.a (mon.0) 3710 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:40:49.420 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[75164]: audit 2024-09-13T00:40:48.364355+0000 mon.a (mon.0) 3711 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:40:49.420 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[75164]: audit 2024-09-13T00:40:48.364628+0000 mon.a (mon.0) 3712 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:40:49.420 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[75164]: cluster 2024-09-13T00:40:48.465309+0000 mgr.y (mgr.24661) 1910 : cluster [DBG] pgmap v3481: 137 pgs: 137 active+clean; 4.6 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 453 KiB/s rd, 32 KiB/s wr, 290 op/s 2024-09-13T00:40:49.420 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[76371]: audit 2024-09-13T00:40:48.355960+0000 mon.a (mon.0) 3705 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:40:49.420 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[76371]: audit 2024-09-13T00:40:48.356310+0000 mon.a (mon.0) 3706 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:40:49.421 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[76371]: audit 2024-09-13T00:40:48.359174+0000 mon.b (mon.1) 25 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T00:40:49.421 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[76371]: audit 2024-09-13T00:40:48.359272+0000 mon.b (mon.1) 26 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T00:40:49.421 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[76371]: audit 2024-09-13T00:40:48.359570+0000 mon.a (mon.0) 3707 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:40:49.421 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[76371]: audit 2024-09-13T00:40:48.360248+0000 mon.a (mon.0) 3708 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:40:49.421 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[76371]: audit 2024-09-13T00:40:48.360543+0000 mon.a (mon.0) 3709 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:40:49.421 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[76371]: audit 2024-09-13T00:40:48.363088+0000 mon.c (mon.2) 146 : audit [INF] from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T00:40:49.421 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[76371]: audit 2024-09-13T00:40:48.363225+0000 mon.c (mon.2) 147 : audit [INF] from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T00:40:49.421 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[76371]: audit 2024-09-13T00:40:48.363644+0000 mon.a (mon.0) 3710 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:40:49.421 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[76371]: audit 2024-09-13T00:40:48.364355+0000 mon.a (mon.0) 3711 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:40:49.421 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[76371]: audit 2024-09-13T00:40:48.364628+0000 mon.a (mon.0) 3712 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:40:49.421 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[76371]: cluster 2024-09-13T00:40:48.465309+0000 mgr.y (mgr.24661) 1910 : cluster [DBG] pgmap v3481: 137 pgs: 137 active+clean; 4.6 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 453 KiB/s rd, 32 KiB/s wr, 290 op/s 2024-09-13T00:40:49.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:40:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:40:49] "GET /metrics HTTP/1.1" 200 36687 "" "Prometheus/2.43.0" 2024-09-13T00:40:49.976 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::test_pool_stats PASSED [ 10%] 2024-09-13T00:40:49.979 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::test_features_to_string PASSED [ 11%] 2024-09-13T00:40:49.982 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::test_features_from_string PASSED [ 12%] 2024-09-13T00:40:50.177 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_update_features SKIPPED [ 12%] 2024-09-13T00:40:50.377 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_create_with_params SKIPPED [ 13%] 2024-09-13T00:40:50.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:50 smithi136 bash[75164]: cluster 2024-09-13T00:40:49.155941+0000 mon.a (mon.0) 3713 : cluster [DBG] osdmap e2011: 8 total, 8 up, 8 in 2024-09-13T00:40:50.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:50 smithi136 bash[75164]: cluster 2024-09-13T00:40:49.768482+0000 mon.a (mon.0) 3714 : cluster [DBG] osdmap e2012: 8 total, 8 up, 8 in 2024-09-13T00:40:50.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:50 smithi136 bash[76371]: cluster 2024-09-13T00:40:49.155941+0000 mon.a (mon.0) 3713 : cluster [DBG] osdmap e2011: 8 total, 8 up, 8 in 2024-09-13T00:40:50.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:50 smithi136 bash[76371]: cluster 2024-09-13T00:40:49.768482+0000 mon.a (mon.0) 3714 : cluster [DBG] osdmap e2012: 8 total, 8 up, 8 in 2024-09-13T00:40:50.579 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_id PASSED [ 13%] 2024-09-13T00:40:50.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:50 smithi181 bash[67587]: cluster 2024-09-13T00:40:49.155941+0000 mon.a (mon.0) 3713 : cluster [DBG] osdmap e2011: 8 total, 8 up, 8 in 2024-09-13T00:40:50.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:50 smithi181 bash[67587]: cluster 2024-09-13T00:40:49.768482+0000 mon.a (mon.0) 3714 : cluster [DBG] osdmap e2012: 8 total, 8 up, 8 in 2024-09-13T00:40:50.764 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_block_name_prefix PASSED [ 14%] 2024-09-13T00:40:50.963 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_data_pool_id PASSED [ 14%] 2024-09-13T00:40:51.154 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_create_timestamp PASSED [ 15%] 2024-09-13T00:40:51.349 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_access_timestamp PASSED [ 15%] 2024-09-13T00:40:51.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:51 smithi136 bash[75164]: cluster 2024-09-13T00:40:50.466063+0000 mgr.y (mgr.24661) 1911 : cluster [DBG] pgmap v3484: 137 pgs: 137 active+clean; 4.6 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 394 KiB/s rd, 25 KiB/s wr, 178 op/s 2024-09-13T00:40:51.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:51 smithi136 bash[76371]: cluster 2024-09-13T00:40:50.466063+0000 mgr.y (mgr.24661) 1911 : cluster [DBG] pgmap v3484: 137 pgs: 137 active+clean; 4.6 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 394 KiB/s rd, 25 KiB/s wr, 178 op/s 2024-09-13T00:40:51.549 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_modify_timestamp PASSED [ 16%] 2024-09-13T00:40:51.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:51 smithi181 bash[67587]: cluster 2024-09-13T00:40:50.466063+0000 mgr.y (mgr.24661) 1911 : cluster [DBG] pgmap v3484: 137 pgs: 137 active+clean; 4.6 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 394 KiB/s rd, 25 KiB/s wr, 178 op/s 2024-09-13T00:40:51.778 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_invalidate_cache PASSED [ 17%] 2024-09-13T00:40:51.965 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_stat PASSED [ 17%] 2024-09-13T00:40:52.171 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_flags PASSED [ 18%] 2024-09-13T00:40:52.382 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_image_auto_close PASSED [ 18%] 2024-09-13T00:40:52.598 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_use_after_close PASSED [ 19%] 2024-09-13T00:40:52.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:52 smithi181 bash[67587]: cluster 2024-09-13T00:40:52.467346+0000 mgr.y (mgr.24661) 1912 : cluster [DBG] pgmap v3485: 137 pgs: 137 active+clean; 4.6 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 462 KiB/s rd, 32 KiB/s wr, 314 op/s 2024-09-13T00:40:52.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:40:52 smithi181 bash[95505]: ts=2024-09-13T00:40:52.317Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:40:52.828 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_write PASSED [ 19%] 2024-09-13T00:40:53.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:52 smithi136 bash[75164]: cluster 2024-09-13T00:40:52.467346+0000 mgr.y (mgr.24661) 1912 : cluster [DBG] pgmap v3485: 137 pgs: 137 active+clean; 4.6 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 462 KiB/s rd, 32 KiB/s wr, 314 op/s 2024-09-13T00:40:53.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:52 smithi136 bash[76371]: cluster 2024-09-13T00:40:52.467346+0000 mgr.y (mgr.24661) 1912 : cluster [DBG] pgmap v3485: 137 pgs: 137 active+clean; 4.6 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 462 KiB/s rd, 32 KiB/s wr, 314 op/s 2024-09-13T00:40:53.050 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_write_with_fadvise_flags PASSED [ 20%] 2024-09-13T00:40:53.282 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_write_zeroes PASSED [ 20%] 2024-09-13T00:40:53.515 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_write_zeroes_thick_provision PASSED [ 21%] 2024-09-13T00:40:53.723 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_read PASSED [ 21%] 2024-09-13T00:40:53.929 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_read_with_fadvise_flags PASSED [ 22%] 2024-09-13T00:40:54.242 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_large_write PASSED [ 23%] 2024-09-13T00:40:54.440 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_large_read PASSED [ 23%] 2024-09-13T00:40:54.671 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_write_read PASSED [ 24%] 2024-09-13T00:40:54.875 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_read_bad_offset PASSED [ 24%] 2024-09-13T00:40:55.091 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_resize PASSED [ 25%] 2024-09-13T00:40:55.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:54 smithi136 bash[75164]: audit 2024-09-13T00:40:53.846957+0000 mon.a (mon.0) 3715 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:40:55.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:54 smithi136 bash[75164]: audit 2024-09-13T00:40:53.849296+0000 mon.a (mon.0) 3716 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:40:55.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:54 smithi136 bash[75164]: cluster 2024-09-13T00:40:54.468348+0000 mgr.y (mgr.24661) 1913 : cluster [DBG] pgmap v3486: 137 pgs: 137 active+clean; 4.6 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 934 KiB/s rd, 38 KiB/s wr, 547 op/s 2024-09-13T00:40:55.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:54 smithi136 bash[76371]: audit 2024-09-13T00:40:53.846957+0000 mon.a (mon.0) 3715 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:40:55.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:54 smithi136 bash[76371]: audit 2024-09-13T00:40:53.849296+0000 mon.a (mon.0) 3716 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:40:55.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:54 smithi136 bash[76371]: cluster 2024-09-13T00:40:54.468348+0000 mgr.y (mgr.24661) 1913 : cluster [DBG] pgmap v3486: 137 pgs: 137 active+clean; 4.6 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 934 KiB/s rd, 38 KiB/s wr, 547 op/s 2024-09-13T00:40:55.328 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_resize_allow_shrink_False PASSED [ 25%] 2024-09-13T00:40:55.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:54 smithi181 bash[67587]: audit 2024-09-13T00:40:53.846957+0000 mon.a (mon.0) 3715 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:40:55.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:54 smithi181 bash[67587]: audit 2024-09-13T00:40:53.849296+0000 mon.a (mon.0) 3716 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:40:55.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:54 smithi181 bash[67587]: cluster 2024-09-13T00:40:54.468348+0000 mgr.y (mgr.24661) 1913 : cluster [DBG] pgmap v3486: 137 pgs: 137 active+clean; 4.6 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 934 KiB/s rd, 38 KiB/s wr, 547 op/s 2024-09-13T00:40:57.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:56 smithi136 bash[75164]: cluster 2024-09-13T00:40:55.863064+0000 mon.a (mon.0) 3717 : cluster [DBG] osdmap e2013: 8 total, 8 up, 8 in 2024-09-13T00:40:57.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:56 smithi136 bash[75164]: cluster 2024-09-13T00:40:56.469371+0000 mgr.y (mgr.24661) 1914 : cluster [DBG] pgmap v3488: 137 pgs: 137 active+clean; 4.6 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 864 KiB/s rd, 36 KiB/s wr, 748 op/s 2024-09-13T00:40:57.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:56 smithi136 bash[76371]: cluster 2024-09-13T00:40:55.863064+0000 mon.a (mon.0) 3717 : cluster [DBG] osdmap e2013: 8 total, 8 up, 8 in 2024-09-13T00:40:57.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:56 smithi136 bash[76371]: cluster 2024-09-13T00:40:56.469371+0000 mgr.y (mgr.24661) 1914 : cluster [DBG] pgmap v3488: 137 pgs: 137 active+clean; 4.6 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 864 KiB/s rd, 36 KiB/s wr, 748 op/s 2024-09-13T00:40:57.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:56 smithi181 bash[67587]: cluster 2024-09-13T00:40:55.863064+0000 mon.a (mon.0) 3717 : cluster [DBG] osdmap e2013: 8 total, 8 up, 8 in 2024-09-13T00:40:57.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:56 smithi181 bash[67587]: cluster 2024-09-13T00:40:56.469371+0000 mgr.y (mgr.24661) 1914 : cluster [DBG] pgmap v3488: 137 pgs: 137 active+clean; 4.6 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 864 KiB/s rd, 36 KiB/s wr, 748 op/s 2024-09-13T00:40:58.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:57 smithi136 bash[75164]: cluster 2024-09-13T00:40:56.873325+0000 mon.a (mon.0) 3718 : cluster [DBG] osdmap e2014: 8 total, 8 up, 8 in 2024-09-13T00:40:58.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:57 smithi136 bash[76371]: cluster 2024-09-13T00:40:56.873325+0000 mon.a (mon.0) 3718 : cluster [DBG] osdmap e2014: 8 total, 8 up, 8 in 2024-09-13T00:40:58.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:57 smithi181 bash[67587]: cluster 2024-09-13T00:40:56.873325+0000 mon.a (mon.0) 3718 : cluster [DBG] osdmap e2014: 8 total, 8 up, 8 in 2024-09-13T00:40:59.102 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_size PASSED [ 26%] 2024-09-13T00:40:59.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:58 smithi136 bash[75164]: cluster 2024-09-13T00:40:57.878783+0000 mon.a (mon.0) 3719 : cluster [DBG] osdmap e2015: 8 total, 8 up, 8 in 2024-09-13T00:40:59.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:58 smithi136 bash[75164]: cluster 2024-09-13T00:40:58.470075+0000 mgr.y (mgr.24661) 1915 : cluster [DBG] pgmap v3491: 137 pgs: 137 active+clean; 4.6 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 972 KiB/s rd, 716 KiB/s wr, 786 op/s 2024-09-13T00:40:59.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:58 smithi136 bash[76371]: cluster 2024-09-13T00:40:57.878783+0000 mon.a (mon.0) 3719 : cluster [DBG] osdmap e2015: 8 total, 8 up, 8 in 2024-09-13T00:40:59.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:58 smithi136 bash[76371]: cluster 2024-09-13T00:40:58.470075+0000 mgr.y (mgr.24661) 1915 : cluster [DBG] pgmap v3491: 137 pgs: 137 active+clean; 4.6 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 972 KiB/s rd, 716 KiB/s wr, 786 op/s 2024-09-13T00:40:59.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:58 smithi181 bash[67587]: cluster 2024-09-13T00:40:57.878783+0000 mon.a (mon.0) 3719 : cluster [DBG] osdmap e2015: 8 total, 8 up, 8 in 2024-09-13T00:40:59.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:58 smithi181 bash[67587]: cluster 2024-09-13T00:40:58.470075+0000 mgr.y (mgr.24661) 1915 : cluster [DBG] pgmap v3491: 137 pgs: 137 active+clean; 4.6 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 972 KiB/s rd, 716 KiB/s wr, 786 op/s 2024-09-13T00:40:59.349 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_resize_down PASSED [ 26%] 2024-09-13T00:40:59.604 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_resize_bytes PASSED [ 27%] 2024-09-13T00:40:59.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:40:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:40:59] "GET /metrics HTTP/1.1" 200 36727 "" "Prometheus/2.43.0" 2024-09-13T00:41:00.157 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_copy PASSED [ 28%] 2024-09-13T00:41:00.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:59 smithi136 bash[76371]: cluster 2024-09-13T00:40:58.891243+0000 mon.a (mon.0) 3720 : cluster [DBG] osdmap e2016: 8 total, 8 up, 8 in 2024-09-13T00:41:00.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:40:59 smithi136 bash[76371]: cluster 2024-09-13T00:40:59.770864+0000 mon.a (mon.0) 3721 : cluster [DBG] osdmap e2017: 8 total, 8 up, 8 in 2024-09-13T00:41:00.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:59 smithi136 bash[75164]: cluster 2024-09-13T00:40:58.891243+0000 mon.a (mon.0) 3720 : cluster [DBG] osdmap e2016: 8 total, 8 up, 8 in 2024-09-13T00:41:00.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:40:59 smithi136 bash[75164]: cluster 2024-09-13T00:40:59.770864+0000 mon.a (mon.0) 3721 : cluster [DBG] osdmap e2017: 8 total, 8 up, 8 in 2024-09-13T00:41:00.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:59 smithi181 bash[67587]: cluster 2024-09-13T00:40:58.891243+0000 mon.a (mon.0) 3720 : cluster [DBG] osdmap e2016: 8 total, 8 up, 8 in 2024-09-13T00:41:00.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:40:59 smithi181 bash[67587]: cluster 2024-09-13T00:40:59.770864+0000 mon.a (mon.0) 3721 : cluster [DBG] osdmap e2017: 8 total, 8 up, 8 in 2024-09-13T00:41:00.665 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_copy2 PASSED [ 28%] 2024-09-13T00:41:00.871 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_copy3 SKIPPED [ 29%] 2024-09-13T00:41:01.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:00 smithi136 bash[75164]: cluster 2024-09-13T00:41:00.470858+0000 mgr.y (mgr.24661) 1916 : cluster [DBG] pgmap v3494: 137 pgs: 137 active+clean; 4.6 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 184 KiB/s rd, 2.0 MiB/s wr, 266 op/s 2024-09-13T00:41:01.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:00 smithi136 bash[76371]: cluster 2024-09-13T00:41:00.470858+0000 mgr.y (mgr.24661) 1916 : cluster [DBG] pgmap v3494: 137 pgs: 137 active+clean; 4.6 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 184 KiB/s rd, 2.0 MiB/s wr, 266 op/s 2024-09-13T00:41:01.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:00 smithi181 bash[67587]: cluster 2024-09-13T00:41:00.470858+0000 mgr.y (mgr.24661) 1916 : cluster [DBG] pgmap v3494: 137 pgs: 137 active+clean; 4.6 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 184 KiB/s rd, 2.0 MiB/s wr, 266 op/s 2024-09-13T00:41:02.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:41:02 smithi181 bash[95505]: ts=2024-09-13T00:41:02.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:41:03.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:02 smithi136 bash[75164]: cluster 2024-09-13T00:41:01.905466+0000 mon.a (mon.0) 3722 : cluster [DBG] osdmap e2018: 8 total, 8 up, 8 in 2024-09-13T00:41:03.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:02 smithi136 bash[75164]: cluster 2024-09-13T00:41:02.471796+0000 mgr.y (mgr.24661) 1917 : cluster [DBG] pgmap v3496: 137 pgs: 137 active+clean; 4.6 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 187 KiB/s rd, 1.8 MiB/s wr, 282 op/s 2024-09-13T00:41:03.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:02 smithi136 bash[76371]: cluster 2024-09-13T00:41:01.905466+0000 mon.a (mon.0) 3722 : cluster [DBG] osdmap e2018: 8 total, 8 up, 8 in 2024-09-13T00:41:03.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:02 smithi136 bash[76371]: cluster 2024-09-13T00:41:02.471796+0000 mgr.y (mgr.24661) 1917 : cluster [DBG] pgmap v3496: 137 pgs: 137 active+clean; 4.6 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 187 KiB/s rd, 1.8 MiB/s wr, 282 op/s 2024-09-13T00:41:03.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:02 smithi181 bash[67587]: cluster 2024-09-13T00:41:01.905466+0000 mon.a (mon.0) 3722 : cluster [DBG] osdmap e2018: 8 total, 8 up, 8 in 2024-09-13T00:41:03.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:02 smithi181 bash[67587]: cluster 2024-09-13T00:41:02.471796+0000 mgr.y (mgr.24661) 1917 : cluster [DBG] pgmap v3496: 137 pgs: 137 active+clean; 4.6 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 187 KiB/s rd, 1.8 MiB/s wr, 282 op/s 2024-09-13T00:41:04.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:03 smithi136 bash[75164]: cluster 2024-09-13T00:41:02.914639+0000 mon.a (mon.0) 3723 : cluster [DBG] osdmap e2019: 8 total, 8 up, 8 in 2024-09-13T00:41:04.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:03 smithi136 bash[76371]: cluster 2024-09-13T00:41:02.914639+0000 mon.a (mon.0) 3723 : cluster [DBG] osdmap e2019: 8 total, 8 up, 8 in 2024-09-13T00:41:04.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:03 smithi181 bash[67587]: cluster 2024-09-13T00:41:02.914639+0000 mon.a (mon.0) 3723 : cluster [DBG] osdmap e2019: 8 total, 8 up, 8 in 2024-09-13T00:41:05.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:04 smithi136 bash[76371]: cluster 2024-09-13T00:41:03.923378+0000 mon.a (mon.0) 3724 : cluster [DBG] osdmap e2020: 8 total, 8 up, 8 in 2024-09-13T00:41:05.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:04 smithi136 bash[76371]: cluster 2024-09-13T00:41:04.472479+0000 mgr.y (mgr.24661) 1918 : cluster [DBG] pgmap v3499: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 244 KiB/s rd, 895 KiB/s wr, 344 op/s 2024-09-13T00:41:05.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:04 smithi136 bash[75164]: cluster 2024-09-13T00:41:03.923378+0000 mon.a (mon.0) 3724 : cluster [DBG] osdmap e2020: 8 total, 8 up, 8 in 2024-09-13T00:41:05.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:04 smithi136 bash[75164]: cluster 2024-09-13T00:41:04.472479+0000 mgr.y (mgr.24661) 1918 : cluster [DBG] pgmap v3499: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 244 KiB/s rd, 895 KiB/s wr, 344 op/s 2024-09-13T00:41:05.332 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_deep_copy PASSED [ 29%] 2024-09-13T00:41:05.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:04 smithi181 bash[67587]: cluster 2024-09-13T00:41:03.923378+0000 mon.a (mon.0) 3724 : cluster [DBG] osdmap e2020: 8 total, 8 up, 8 in 2024-09-13T00:41:05.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:04 smithi181 bash[67587]: cluster 2024-09-13T00:41:04.472479+0000 mgr.y (mgr.24661) 1918 : cluster [DBG] pgmap v3499: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 244 KiB/s rd, 895 KiB/s wr, 344 op/s 2024-09-13T00:41:06.238 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_deep_copy_clone_v1_to_v1 FAILED [ 30%] 2024-09-13T00:41:06.238 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_deep_copy_clone_v1_to_v1 ERROR [ 30%] 2024-09-13T00:41:06.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:05 smithi136 bash[75164]: cluster 2024-09-13T00:41:04.934200+0000 mon.a (mon.0) 3725 : cluster [DBG] osdmap e2021: 8 total, 8 up, 8 in 2024-09-13T00:41:06.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:05 smithi136 bash[76371]: cluster 2024-09-13T00:41:04.934200+0000 mon.a (mon.0) 3725 : cluster [DBG] osdmap e2021: 8 total, 8 up, 8 in 2024-09-13T00:41:06.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:05 smithi181 bash[67587]: cluster 2024-09-13T00:41:04.934200+0000 mon.a (mon.0) 3725 : cluster [DBG] osdmap e2021: 8 total, 8 up, 8 in 2024-09-13T00:41:07.221 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_deep_copy_clone_v1_to_v2 FAILED [ 30%] 2024-09-13T00:41:07.222 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_deep_copy_clone_v1_to_v2 ERROR [ 30%] 2024-09-13T00:41:07.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:06 smithi136 bash[75164]: cluster 2024-09-13T00:41:05.943299+0000 mon.a (mon.0) 3726 : cluster [DBG] osdmap e2022: 8 total, 8 up, 8 in 2024-09-13T00:41:07.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:06 smithi136 bash[75164]: cluster 2024-09-13T00:41:06.473313+0000 mgr.y (mgr.24661) 1919 : cluster [DBG] pgmap v3502: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 235 KiB/s rd, 1.0 MiB/s wr, 326 op/s 2024-09-13T00:41:07.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:06 smithi136 bash[76371]: cluster 2024-09-13T00:41:05.943299+0000 mon.a (mon.0) 3726 : cluster [DBG] osdmap e2022: 8 total, 8 up, 8 in 2024-09-13T00:41:07.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:06 smithi136 bash[76371]: cluster 2024-09-13T00:41:06.473313+0000 mgr.y (mgr.24661) 1919 : cluster [DBG] pgmap v3502: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 235 KiB/s rd, 1.0 MiB/s wr, 326 op/s 2024-09-13T00:41:07.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:06 smithi181 bash[67587]: cluster 2024-09-13T00:41:05.943299+0000 mon.a (mon.0) 3726 : cluster [DBG] osdmap e2022: 8 total, 8 up, 8 in 2024-09-13T00:41:07.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:06 smithi181 bash[67587]: cluster 2024-09-13T00:41:06.473313+0000 mgr.y (mgr.24661) 1919 : cluster [DBG] pgmap v3502: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 235 KiB/s rd, 1.0 MiB/s wr, 326 op/s 2024-09-13T00:41:08.214 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_deep_copy_clone_v2_to_v1 FAILED [ 31%] 2024-09-13T00:41:08.215 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_deep_copy_clone_v2_to_v1 ERROR [ 31%] 2024-09-13T00:41:08.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:07 smithi136 bash[75164]: cluster 2024-09-13T00:41:06.953206+0000 mon.a (mon.0) 3727 : cluster [DBG] osdmap e2023: 8 total, 8 up, 8 in 2024-09-13T00:41:08.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:07 smithi136 bash[76371]: cluster 2024-09-13T00:41:06.953206+0000 mon.a (mon.0) 3727 : cluster [DBG] osdmap e2023: 8 total, 8 up, 8 in 2024-09-13T00:41:08.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:07 smithi181 bash[67587]: cluster 2024-09-13T00:41:06.953206+0000 mon.a (mon.0) 3727 : cluster [DBG] osdmap e2023: 8 total, 8 up, 8 in 2024-09-13T00:41:09.205 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_deep_copy_clone_v2_to_v2 FAILED [ 31%] 2024-09-13T00:41:09.205 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_deep_copy_clone_v2_to_v2 ERROR [ 31%] 2024-09-13T00:41:09.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:08 smithi136 bash[76371]: cluster 2024-09-13T00:41:07.955572+0000 mon.a (mon.0) 3728 : cluster [DBG] osdmap e2024: 8 total, 8 up, 8 in 2024-09-13T00:41:09.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:08 smithi136 bash[76371]: cluster 2024-09-13T00:41:08.474012+0000 mgr.y (mgr.24661) 1920 : cluster [DBG] pgmap v3505: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 175 KiB/s rd, 1.0 MiB/s wr, 244 op/s 2024-09-13T00:41:09.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:08 smithi136 bash[76371]: audit 2024-09-13T00:41:08.839670+0000 mon.a (mon.0) 3729 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:41:09.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:08 smithi136 bash[75164]: cluster 2024-09-13T00:41:07.955572+0000 mon.a (mon.0) 3728 : cluster [DBG] osdmap e2024: 8 total, 8 up, 8 in 2024-09-13T00:41:09.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:08 smithi136 bash[75164]: cluster 2024-09-13T00:41:08.474012+0000 mgr.y (mgr.24661) 1920 : cluster [DBG] pgmap v3505: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 175 KiB/s rd, 1.0 MiB/s wr, 244 op/s 2024-09-13T00:41:09.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:08 smithi136 bash[75164]: audit 2024-09-13T00:41:08.839670+0000 mon.a (mon.0) 3729 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:41:09.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:08 smithi181 bash[67587]: cluster 2024-09-13T00:41:07.955572+0000 mon.a (mon.0) 3728 : cluster [DBG] osdmap e2024: 8 total, 8 up, 8 in 2024-09-13T00:41:09.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:08 smithi181 bash[67587]: cluster 2024-09-13T00:41:08.474012+0000 mgr.y (mgr.24661) 1920 : cluster [DBG] pgmap v3505: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 175 KiB/s rd, 1.0 MiB/s wr, 244 op/s 2024-09-13T00:41:09.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:08 smithi181 bash[67587]: audit 2024-09-13T00:41:08.839670+0000 mon.a (mon.0) 3729 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:41:09.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:41:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:41:09] "GET /metrics HTTP/1.1" 200 36739 "" "Prometheus/2.43.0" 2024-09-13T00:41:10.047 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_deep_copy_clone_v1_flatten FAILED [ 32%] 2024-09-13T00:41:10.048 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_deep_copy_clone_v1_flatten ERROR [ 32%] 2024-09-13T00:41:10.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:09 smithi136 bash[75164]: cluster 2024-09-13T00:41:08.968446+0000 mon.a (mon.0) 3730 : cluster [DBG] osdmap e2025: 8 total, 8 up, 8 in 2024-09-13T00:41:10.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:09 smithi136 bash[75164]: cluster 2024-09-13T00:41:09.777679+0000 mon.a (mon.0) 3731 : cluster [DBG] osdmap e2026: 8 total, 8 up, 8 in 2024-09-13T00:41:10.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:09 smithi136 bash[76371]: cluster 2024-09-13T00:41:08.968446+0000 mon.a (mon.0) 3730 : cluster [DBG] osdmap e2025: 8 total, 8 up, 8 in 2024-09-13T00:41:10.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:09 smithi136 bash[76371]: cluster 2024-09-13T00:41:09.777679+0000 mon.a (mon.0) 3731 : cluster [DBG] osdmap e2026: 8 total, 8 up, 8 in 2024-09-13T00:41:10.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:09 smithi181 bash[67587]: cluster 2024-09-13T00:41:08.968446+0000 mon.a (mon.0) 3730 : cluster [DBG] osdmap e2025: 8 total, 8 up, 8 in 2024-09-13T00:41:10.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:09 smithi181 bash[67587]: cluster 2024-09-13T00:41:09.777679+0000 mon.a (mon.0) 3731 : cluster [DBG] osdmap e2026: 8 total, 8 up, 8 in 2024-09-13T00:41:11.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:10 smithi136 bash[75164]: cluster 2024-09-13T00:41:10.474697+0000 mgr.y (mgr.24661) 1921 : cluster [DBG] pgmap v3508: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 58 KiB/s rd, 4.5 KiB/s wr, 81 op/s 2024-09-13T00:41:11.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:10 smithi136 bash[76371]: cluster 2024-09-13T00:41:10.474697+0000 mgr.y (mgr.24661) 1921 : cluster [DBG] pgmap v3508: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 58 KiB/s rd, 4.5 KiB/s wr, 81 op/s 2024-09-13T00:41:11.267 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_deep_copy_clone_v2_flatten FAILED [ 32%] 2024-09-13T00:41:11.267 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_deep_copy_clone_v2_flatten ERROR [ 32%] 2024-09-13T00:41:11.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:10 smithi181 bash[67587]: cluster 2024-09-13T00:41:10.474697+0000 mgr.y (mgr.24661) 1921 : cluster [DBG] pgmap v3508: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 58 KiB/s rd, 4.5 KiB/s wr, 81 op/s 2024-09-13T00:41:12.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:11 smithi136 bash[75164]: cluster 2024-09-13T00:41:11.001461+0000 mon.a (mon.0) 3732 : cluster [DBG] osdmap e2027: 8 total, 8 up, 8 in 2024-09-13T00:41:12.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:11 smithi136 bash[76371]: cluster 2024-09-13T00:41:11.001461+0000 mon.a (mon.0) 3732 : cluster [DBG] osdmap e2027: 8 total, 8 up, 8 in 2024-09-13T00:41:12.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:11 smithi181 bash[67587]: cluster 2024-09-13T00:41:11.001461+0000 mon.a (mon.0) 3732 : cluster [DBG] osdmap e2027: 8 total, 8 up, 8 in 2024-09-13T00:41:12.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:41:12 smithi181 bash[95505]: ts=2024-09-13T00:41:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:41:13.232 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_create_snap PASSED [ 33%] 2024-09-13T00:41:13.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:13 smithi181 bash[67587]: cluster 2024-09-13T00:41:12.004137+0000 mon.a (mon.0) 3733 : cluster [DBG] osdmap e2028: 8 total, 8 up, 8 in 2024-09-13T00:41:13.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:13 smithi181 bash[67587]: cluster 2024-09-13T00:41:12.475555+0000 mgr.y (mgr.24661) 1922 : cluster [DBG] pgmap v3511: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 97 KiB/s rd, 13 KiB/s wr, 140 op/s 2024-09-13T00:41:13.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:13 smithi136 bash[75164]: cluster 2024-09-13T00:41:12.004137+0000 mon.a (mon.0) 3733 : cluster [DBG] osdmap e2028: 8 total, 8 up, 8 in 2024-09-13T00:41:13.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:13 smithi136 bash[75164]: cluster 2024-09-13T00:41:12.475555+0000 mgr.y (mgr.24661) 1922 : cluster [DBG] pgmap v3511: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 97 KiB/s rd, 13 KiB/s wr, 140 op/s 2024-09-13T00:41:13.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:13 smithi136 bash[76371]: cluster 2024-09-13T00:41:12.004137+0000 mon.a (mon.0) 3733 : cluster [DBG] osdmap e2028: 8 total, 8 up, 8 in 2024-09-13T00:41:13.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:13 smithi136 bash[76371]: cluster 2024-09-13T00:41:12.475555+0000 mgr.y (mgr.24661) 1922 : cluster [DBG] pgmap v3511: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 97 KiB/s rd, 13 KiB/s wr, 140 op/s 2024-09-13T00:41:14.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:14 smithi181 bash[67587]: cluster 2024-09-13T00:41:13.017745+0000 mon.a (mon.0) 3734 : cluster [DBG] osdmap e2029: 8 total, 8 up, 8 in 2024-09-13T00:41:14.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:14 smithi136 bash[75164]: cluster 2024-09-13T00:41:13.017745+0000 mon.a (mon.0) 3734 : cluster [DBG] osdmap e2029: 8 total, 8 up, 8 in 2024-09-13T00:41:14.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:14 smithi136 bash[76371]: cluster 2024-09-13T00:41:13.017745+0000 mon.a (mon.0) 3734 : cluster [DBG] osdmap e2029: 8 total, 8 up, 8 in 2024-09-13T00:41:14.989 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_create_snap_exists PASSED [ 34%] 2024-09-13T00:41:15.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:15 smithi181 bash[67587]: cluster 2024-09-13T00:41:14.018813+0000 mon.a (mon.0) 3735 : cluster [DBG] osdmap e2030: 8 total, 8 up, 8 in 2024-09-13T00:41:15.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:15 smithi181 bash[67587]: cluster 2024-09-13T00:41:14.476293+0000 mgr.y (mgr.24661) 1923 : cluster [DBG] pgmap v3514: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 121 KiB/s rd, 16 KiB/s wr, 171 op/s 2024-09-13T00:41:15.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:15 smithi181 bash[67587]: cluster 2024-09-13T00:41:14.777649+0000 mon.a (mon.0) 3736 : cluster [DBG] osdmap e2031: 8 total, 8 up, 8 in 2024-09-13T00:41:15.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:15 smithi136 bash[75164]: cluster 2024-09-13T00:41:14.018813+0000 mon.a (mon.0) 3735 : cluster [DBG] osdmap e2030: 8 total, 8 up, 8 in 2024-09-13T00:41:15.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:15 smithi136 bash[75164]: cluster 2024-09-13T00:41:14.476293+0000 mgr.y (mgr.24661) 1923 : cluster [DBG] pgmap v3514: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 121 KiB/s rd, 16 KiB/s wr, 171 op/s 2024-09-13T00:41:15.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:15 smithi136 bash[75164]: cluster 2024-09-13T00:41:14.777649+0000 mon.a (mon.0) 3736 : cluster [DBG] osdmap e2031: 8 total, 8 up, 8 in 2024-09-13T00:41:15.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:15 smithi136 bash[76371]: cluster 2024-09-13T00:41:14.018813+0000 mon.a (mon.0) 3735 : cluster [DBG] osdmap e2030: 8 total, 8 up, 8 in 2024-09-13T00:41:15.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:15 smithi136 bash[76371]: cluster 2024-09-13T00:41:14.476293+0000 mgr.y (mgr.24661) 1923 : cluster [DBG] pgmap v3514: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 121 KiB/s rd, 16 KiB/s wr, 171 op/s 2024-09-13T00:41:15.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:15 smithi136 bash[76371]: cluster 2024-09-13T00:41:14.777649+0000 mon.a (mon.0) 3736 : cluster [DBG] osdmap e2031: 8 total, 8 up, 8 in 2024-09-13T00:41:17.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:17 smithi181 bash[67587]: cluster 2024-09-13T00:41:16.042289+0000 mon.a (mon.0) 3737 : cluster [DBG] osdmap e2032: 8 total, 8 up, 8 in 2024-09-13T00:41:17.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:17 smithi181 bash[67587]: cluster 2024-09-13T00:41:16.477163+0000 mgr.y (mgr.24661) 1924 : cluster [DBG] pgmap v3517: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 124 KiB/s rd, 15 KiB/s wr, 178 op/s 2024-09-13T00:41:17.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:17 smithi136 bash[75164]: cluster 2024-09-13T00:41:16.042289+0000 mon.a (mon.0) 3737 : cluster [DBG] osdmap e2032: 8 total, 8 up, 8 in 2024-09-13T00:41:17.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:17 smithi136 bash[75164]: cluster 2024-09-13T00:41:16.477163+0000 mgr.y (mgr.24661) 1924 : cluster [DBG] pgmap v3517: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 124 KiB/s rd, 15 KiB/s wr, 178 op/s 2024-09-13T00:41:17.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:17 smithi136 bash[76371]: cluster 2024-09-13T00:41:16.042289+0000 mon.a (mon.0) 3737 : cluster [DBG] osdmap e2032: 8 total, 8 up, 8 in 2024-09-13T00:41:17.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:17 smithi136 bash[76371]: cluster 2024-09-13T00:41:16.477163+0000 mgr.y (mgr.24661) 1924 : cluster [DBG] pgmap v3517: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 124 KiB/s rd, 15 KiB/s wr, 178 op/s 2024-09-13T00:41:18.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:18 smithi181 bash[67587]: cluster 2024-09-13T00:41:17.053258+0000 mon.a (mon.0) 3738 : cluster [DBG] osdmap e2033: 8 total, 8 up, 8 in 2024-09-13T00:41:18.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:18 smithi136 bash[75164]: cluster 2024-09-13T00:41:17.053258+0000 mon.a (mon.0) 3738 : cluster [DBG] osdmap e2033: 8 total, 8 up, 8 in 2024-09-13T00:41:18.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:18 smithi136 bash[76371]: cluster 2024-09-13T00:41:17.053258+0000 mon.a (mon.0) 3738 : cluster [DBG] osdmap e2033: 8 total, 8 up, 8 in 2024-09-13T00:41:19.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:19 smithi181 bash[67587]: cluster 2024-09-13T00:41:18.059495+0000 mon.a (mon.0) 3739 : cluster [DBG] osdmap e2034: 8 total, 8 up, 8 in 2024-09-13T00:41:19.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:19 smithi181 bash[67587]: cluster 2024-09-13T00:41:18.477812+0000 mgr.y (mgr.24661) 1925 : cluster [DBG] pgmap v3520: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 93 KiB/s rd, 11 KiB/s wr, 134 op/s 2024-09-13T00:41:19.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:19 smithi136 bash[76371]: cluster 2024-09-13T00:41:18.059495+0000 mon.a (mon.0) 3739 : cluster [DBG] osdmap e2034: 8 total, 8 up, 8 in 2024-09-13T00:41:19.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:19 smithi136 bash[76371]: cluster 2024-09-13T00:41:18.477812+0000 mgr.y (mgr.24661) 1925 : cluster [DBG] pgmap v3520: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 93 KiB/s rd, 11 KiB/s wr, 134 op/s 2024-09-13T00:41:19.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:19 smithi136 bash[75164]: cluster 2024-09-13T00:41:18.059495+0000 mon.a (mon.0) 3739 : cluster [DBG] osdmap e2034: 8 total, 8 up, 8 in 2024-09-13T00:41:19.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:19 smithi136 bash[75164]: cluster 2024-09-13T00:41:18.477812+0000 mgr.y (mgr.24661) 1925 : cluster [DBG] pgmap v3520: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 93 KiB/s rd, 11 KiB/s wr, 134 op/s 2024-09-13T00:41:19.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:41:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:41:19] "GET /metrics HTTP/1.1" 200 36739 "" "Prometheus/2.43.0" 2024-09-13T00:41:20.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:20 smithi181 bash[67587]: cluster 2024-09-13T00:41:19.073837+0000 mon.a (mon.0) 3740 : cluster [DBG] osdmap e2035: 8 total, 8 up, 8 in 2024-09-13T00:41:20.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:20 smithi181 bash[67587]: cluster 2024-09-13T00:41:19.781247+0000 mon.a (mon.0) 3741 : cluster [DBG] osdmap e2036: 8 total, 8 up, 8 in 2024-09-13T00:41:20.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:20 smithi136 bash[75164]: cluster 2024-09-13T00:41:19.073837+0000 mon.a (mon.0) 3740 : cluster [DBG] osdmap e2035: 8 total, 8 up, 8 in 2024-09-13T00:41:20.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:20 smithi136 bash[75164]: cluster 2024-09-13T00:41:19.781247+0000 mon.a (mon.0) 3741 : cluster [DBG] osdmap e2036: 8 total, 8 up, 8 in 2024-09-13T00:41:20.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:20 smithi136 bash[76371]: cluster 2024-09-13T00:41:19.073837+0000 mon.a (mon.0) 3740 : cluster [DBG] osdmap e2035: 8 total, 8 up, 8 in 2024-09-13T00:41:20.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:20 smithi136 bash[76371]: cluster 2024-09-13T00:41:19.781247+0000 mon.a (mon.0) 3741 : cluster [DBG] osdmap e2036: 8 total, 8 up, 8 in 2024-09-13T00:41:20.980 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_create_snap_flags PASSED [ 34%] 2024-09-13T00:41:21.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:21 smithi181 bash[67587]: cluster 2024-09-13T00:41:20.478622+0000 mgr.y (mgr.24661) 1926 : cluster [DBG] pgmap v3523: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 40 KiB/s rd, 2.5 KiB/s wr, 52 op/s 2024-09-13T00:41:21.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:21 smithi181 bash[67587]: cluster 2024-09-13T00:41:20.782192+0000 mon.a (mon.0) 3742 : cluster [DBG] osdmap e2037: 8 total, 8 up, 8 in 2024-09-13T00:41:21.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:21 smithi136 bash[75164]: cluster 2024-09-13T00:41:20.478622+0000 mgr.y (mgr.24661) 1926 : cluster [DBG] pgmap v3523: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 40 KiB/s rd, 2.5 KiB/s wr, 52 op/s 2024-09-13T00:41:21.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:21 smithi136 bash[75164]: cluster 2024-09-13T00:41:20.782192+0000 mon.a (mon.0) 3742 : cluster [DBG] osdmap e2037: 8 total, 8 up, 8 in 2024-09-13T00:41:21.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:21 smithi136 bash[76371]: cluster 2024-09-13T00:41:20.478622+0000 mgr.y (mgr.24661) 1926 : cluster [DBG] pgmap v3523: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 40 KiB/s rd, 2.5 KiB/s wr, 52 op/s 2024-09-13T00:41:21.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:21 smithi136 bash[76371]: cluster 2024-09-13T00:41:20.782192+0000 mon.a (mon.0) 3742 : cluster [DBG] osdmap e2037: 8 total, 8 up, 8 in 2024-09-13T00:41:22.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:41:22 smithi181 bash[95505]: ts=2024-09-13T00:41:22.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:41:23.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:22 smithi181 bash[67587]: cluster 2024-09-13T00:41:21.785622+0000 mon.a (mon.0) 3743 : cluster [DBG] osdmap e2038: 8 total, 8 up, 8 in 2024-09-13T00:41:23.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:22 smithi181 bash[67587]: cluster 2024-09-13T00:41:22.479538+0000 mgr.y (mgr.24661) 1927 : cluster [DBG] pgmap v3526: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 73 KiB/s rd, 5.2 KiB/s wr, 98 op/s 2024-09-13T00:41:23.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:22 smithi136 bash[75164]: cluster 2024-09-13T00:41:21.785622+0000 mon.a (mon.0) 3743 : cluster [DBG] osdmap e2038: 8 total, 8 up, 8 in 2024-09-13T00:41:23.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:22 smithi136 bash[75164]: cluster 2024-09-13T00:41:22.479538+0000 mgr.y (mgr.24661) 1927 : cluster [DBG] pgmap v3526: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 73 KiB/s rd, 5.2 KiB/s wr, 98 op/s 2024-09-13T00:41:23.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:22 smithi136 bash[76371]: cluster 2024-09-13T00:41:21.785622+0000 mon.a (mon.0) 3743 : cluster [DBG] osdmap e2038: 8 total, 8 up, 8 in 2024-09-13T00:41:23.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:22 smithi136 bash[76371]: cluster 2024-09-13T00:41:22.479538+0000 mgr.y (mgr.24661) 1927 : cluster [DBG] pgmap v3526: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 73 KiB/s rd, 5.2 KiB/s wr, 98 op/s 2024-09-13T00:41:24.145 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:23 smithi181 bash[67587]: cluster 2024-09-13T00:41:22.796102+0000 mon.a (mon.0) 3744 : cluster [DBG] osdmap e2039: 8 total, 8 up, 8 in 2024-09-13T00:41:24.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:23 smithi136 bash[75164]: cluster 2024-09-13T00:41:22.796102+0000 mon.a (mon.0) 3744 : cluster [DBG] osdmap e2039: 8 total, 8 up, 8 in 2024-09-13T00:41:24.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:23 smithi136 bash[76371]: cluster 2024-09-13T00:41:22.796102+0000 mon.a (mon.0) 3744 : cluster [DBG] osdmap e2039: 8 total, 8 up, 8 in 2024-09-13T00:41:25.008 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_list_snaps PASSED [ 35%] 2024-09-13T00:41:25.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:24 smithi181 bash[67587]: cluster 2024-09-13T00:41:23.808735+0000 mon.a (mon.0) 3745 : cluster [DBG] osdmap e2040: 8 total, 8 up, 8 in 2024-09-13T00:41:25.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:24 smithi181 bash[67587]: audit 2024-09-13T00:41:23.840253+0000 mon.a (mon.0) 3746 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:41:25.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:24 smithi181 bash[67587]: cluster 2024-09-13T00:41:24.480181+0000 mgr.y (mgr.24661) 1928 : cluster [DBG] pgmap v3529: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 55 KiB/s rd, 5.0 KiB/s wr, 75 op/s 2024-09-13T00:41:25.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:24 smithi181 bash[67587]: cluster 2024-09-13T00:41:24.784275+0000 mon.a (mon.0) 3747 : cluster [DBG] osdmap e2041: 8 total, 8 up, 8 in 2024-09-13T00:41:25.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:24 smithi136 bash[75164]: cluster 2024-09-13T00:41:23.808735+0000 mon.a (mon.0) 3745 : cluster [DBG] osdmap e2040: 8 total, 8 up, 8 in 2024-09-13T00:41:25.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:24 smithi136 bash[75164]: audit 2024-09-13T00:41:23.840253+0000 mon.a (mon.0) 3746 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:41:25.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:24 smithi136 bash[75164]: cluster 2024-09-13T00:41:24.480181+0000 mgr.y (mgr.24661) 1928 : cluster [DBG] pgmap v3529: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 55 KiB/s rd, 5.0 KiB/s wr, 75 op/s 2024-09-13T00:41:25.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:24 smithi136 bash[75164]: cluster 2024-09-13T00:41:24.784275+0000 mon.a (mon.0) 3747 : cluster [DBG] osdmap e2041: 8 total, 8 up, 8 in 2024-09-13T00:41:25.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:24 smithi136 bash[76371]: cluster 2024-09-13T00:41:23.808735+0000 mon.a (mon.0) 3745 : cluster [DBG] osdmap e2040: 8 total, 8 up, 8 in 2024-09-13T00:41:25.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:24 smithi136 bash[76371]: audit 2024-09-13T00:41:23.840253+0000 mon.a (mon.0) 3746 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:41:25.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:24 smithi136 bash[76371]: cluster 2024-09-13T00:41:24.480181+0000 mgr.y (mgr.24661) 1928 : cluster [DBG] pgmap v3529: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 55 KiB/s rd, 5.0 KiB/s wr, 75 op/s 2024-09-13T00:41:25.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:24 smithi136 bash[76371]: cluster 2024-09-13T00:41:24.784275+0000 mon.a (mon.0) 3747 : cluster [DBG] osdmap e2041: 8 total, 8 up, 8 in 2024-09-13T00:41:27.043 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_list_snaps_iterator_auto_close PASSED [ 35%] 2024-09-13T00:41:27.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:26 smithi181 bash[67587]: cluster 2024-09-13T00:41:25.825203+0000 mon.a (mon.0) 3748 : cluster [DBG] osdmap e2042: 8 total, 8 up, 8 in 2024-09-13T00:41:27.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:26 smithi181 bash[67587]: cluster 2024-09-13T00:41:26.481170+0000 mgr.y (mgr.24661) 1929 : cluster [DBG] pgmap v3532: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 41 KiB/s rd, 5.2 KiB/s wr, 59 op/s 2024-09-13T00:41:27.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:26 smithi136 bash[75164]: cluster 2024-09-13T00:41:25.825203+0000 mon.a (mon.0) 3748 : cluster [DBG] osdmap e2042: 8 total, 8 up, 8 in 2024-09-13T00:41:27.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:26 smithi136 bash[75164]: cluster 2024-09-13T00:41:26.481170+0000 mgr.y (mgr.24661) 1929 : cluster [DBG] pgmap v3532: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 41 KiB/s rd, 5.2 KiB/s wr, 59 op/s 2024-09-13T00:41:27.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:26 smithi136 bash[76371]: cluster 2024-09-13T00:41:25.825203+0000 mon.a (mon.0) 3748 : cluster [DBG] osdmap e2042: 8 total, 8 up, 8 in 2024-09-13T00:41:27.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:26 smithi136 bash[76371]: cluster 2024-09-13T00:41:26.481170+0000 mgr.y (mgr.24661) 1929 : cluster [DBG] pgmap v3532: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 41 KiB/s rd, 5.2 KiB/s wr, 59 op/s 2024-09-13T00:41:28.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:27 smithi181 bash[67587]: cluster 2024-09-13T00:41:26.836093+0000 mon.a (mon.0) 3749 : cluster [DBG] osdmap e2043: 8 total, 8 up, 8 in 2024-09-13T00:41:28.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:27 smithi136 bash[75164]: cluster 2024-09-13T00:41:26.836093+0000 mon.a (mon.0) 3749 : cluster [DBG] osdmap e2043: 8 total, 8 up, 8 in 2024-09-13T00:41:28.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:27 smithi136 bash[76371]: cluster 2024-09-13T00:41:26.836093+0000 mon.a (mon.0) 3749 : cluster [DBG] osdmap e2043: 8 total, 8 up, 8 in 2024-09-13T00:41:29.064 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_remove_snap PASSED [ 36%] 2024-09-13T00:41:29.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:28 smithi136 bash[75164]: cluster 2024-09-13T00:41:27.843866+0000 mon.a (mon.0) 3750 : cluster [DBG] osdmap e2044: 8 total, 8 up, 8 in 2024-09-13T00:41:29.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:28 smithi136 bash[75164]: cluster 2024-09-13T00:41:28.481854+0000 mgr.y (mgr.24661) 1930 : cluster [DBG] pgmap v3535: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 75 KiB/s rd, 6.7 KiB/s wr, 101 op/s 2024-09-13T00:41:29.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:28 smithi136 bash[76371]: cluster 2024-09-13T00:41:27.843866+0000 mon.a (mon.0) 3750 : cluster [DBG] osdmap e2044: 8 total, 8 up, 8 in 2024-09-13T00:41:29.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:28 smithi136 bash[76371]: cluster 2024-09-13T00:41:28.481854+0000 mgr.y (mgr.24661) 1930 : cluster [DBG] pgmap v3535: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 75 KiB/s rd, 6.7 KiB/s wr, 101 op/s 2024-09-13T00:41:29.270 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_remove_snap_not_found PASSED [ 36%] 2024-09-13T00:41:29.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:28 smithi181 bash[67587]: cluster 2024-09-13T00:41:27.843866+0000 mon.a (mon.0) 3750 : cluster [DBG] osdmap e2044: 8 total, 8 up, 8 in 2024-09-13T00:41:29.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:28 smithi181 bash[67587]: cluster 2024-09-13T00:41:28.481854+0000 mgr.y (mgr.24661) 1930 : cluster [DBG] pgmap v3535: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 75 KiB/s rd, 6.7 KiB/s wr, 101 op/s 2024-09-13T00:41:29.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:41:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:41:29] "GET /metrics HTTP/1.1" 200 36743 "" "Prometheus/2.43.0" 2024-09-13T00:41:30.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:29 smithi136 bash[75164]: cluster 2024-09-13T00:41:28.862663+0000 mon.a (mon.0) 3751 : cluster [DBG] osdmap e2045: 8 total, 8 up, 8 in 2024-09-13T00:41:30.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:29 smithi136 bash[75164]: cluster 2024-09-13T00:41:29.784513+0000 mon.a (mon.0) 3752 : cluster [DBG] osdmap e2046: 8 total, 8 up, 8 in 2024-09-13T00:41:30.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:29 smithi136 bash[76371]: cluster 2024-09-13T00:41:28.862663+0000 mon.a (mon.0) 3751 : cluster [DBG] osdmap e2045: 8 total, 8 up, 8 in 2024-09-13T00:41:30.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:29 smithi136 bash[76371]: cluster 2024-09-13T00:41:29.784513+0000 mon.a (mon.0) 3752 : cluster [DBG] osdmap e2046: 8 total, 8 up, 8 in 2024-09-13T00:41:30.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:29 smithi181 bash[67587]: cluster 2024-09-13T00:41:28.862663+0000 mon.a (mon.0) 3751 : cluster [DBG] osdmap e2045: 8 total, 8 up, 8 in 2024-09-13T00:41:30.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:29 smithi181 bash[67587]: cluster 2024-09-13T00:41:29.784513+0000 mon.a (mon.0) 3752 : cluster [DBG] osdmap e2046: 8 total, 8 up, 8 in 2024-09-13T00:41:31.084 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_remove_snap2 PASSED [ 37%] 2024-09-13T00:41:32.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:31 smithi136 bash[75164]: cluster 2024-09-13T00:41:30.482567+0000 mgr.y (mgr.24661) 1931 : cluster [DBG] pgmap v3538: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 63 KiB/s rd, 2.5 KiB/s wr, 81 op/s 2024-09-13T00:41:32.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:31 smithi136 bash[75164]: cluster 2024-09-13T00:41:30.865409+0000 mon.a (mon.0) 3753 : cluster [DBG] osdmap e2047: 8 total, 8 up, 8 in 2024-09-13T00:41:32.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:31 smithi136 bash[76371]: cluster 2024-09-13T00:41:30.482567+0000 mgr.y (mgr.24661) 1931 : cluster [DBG] pgmap v3538: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 63 KiB/s rd, 2.5 KiB/s wr, 81 op/s 2024-09-13T00:41:32.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:31 smithi136 bash[76371]: cluster 2024-09-13T00:41:30.865409+0000 mon.a (mon.0) 3753 : cluster [DBG] osdmap e2047: 8 total, 8 up, 8 in 2024-09-13T00:41:32.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:31 smithi181 bash[67587]: cluster 2024-09-13T00:41:30.482567+0000 mgr.y (mgr.24661) 1931 : cluster [DBG] pgmap v3538: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 63 KiB/s rd, 2.5 KiB/s wr, 81 op/s 2024-09-13T00:41:32.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:31 smithi181 bash[67587]: cluster 2024-09-13T00:41:30.865409+0000 mon.a (mon.0) 3753 : cluster [DBG] osdmap e2047: 8 total, 8 up, 8 in 2024-09-13T00:41:32.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:41:32 smithi181 bash[95505]: ts=2024-09-13T00:41:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:41:33.099 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_remove_snap_by_id PASSED [ 37%] 2024-09-13T00:41:33.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:32 smithi136 bash[75164]: cluster 2024-09-13T00:41:31.867422+0000 mon.a (mon.0) 3754 : cluster [DBG] osdmap e2048: 8 total, 8 up, 8 in 2024-09-13T00:41:33.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:32 smithi136 bash[76371]: cluster 2024-09-13T00:41:31.867422+0000 mon.a (mon.0) 3754 : cluster [DBG] osdmap e2048: 8 total, 8 up, 8 in 2024-09-13T00:41:33.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:32 smithi181 bash[67587]: cluster 2024-09-13T00:41:31.867422+0000 mon.a (mon.0) 3754 : cluster [DBG] osdmap e2048: 8 total, 8 up, 8 in 2024-09-13T00:41:34.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:33 smithi136 bash[75164]: cluster 2024-09-13T00:41:32.483473+0000 mgr.y (mgr.24661) 1932 : cluster [DBG] pgmap v3541: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 158 KiB/s rd, 11 KiB/s wr, 218 op/s 2024-09-13T00:41:34.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:33 smithi136 bash[75164]: cluster 2024-09-13T00:41:32.888713+0000 mon.a (mon.0) 3755 : cluster [DBG] osdmap e2049: 8 total, 8 up, 8 in 2024-09-13T00:41:34.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:33 smithi136 bash[76371]: cluster 2024-09-13T00:41:32.483473+0000 mgr.y (mgr.24661) 1932 : cluster [DBG] pgmap v3541: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 158 KiB/s rd, 11 KiB/s wr, 218 op/s 2024-09-13T00:41:34.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:33 smithi136 bash[76371]: cluster 2024-09-13T00:41:32.888713+0000 mon.a (mon.0) 3755 : cluster [DBG] osdmap e2049: 8 total, 8 up, 8 in 2024-09-13T00:41:34.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:33 smithi181 bash[67587]: cluster 2024-09-13T00:41:32.483473+0000 mgr.y (mgr.24661) 1932 : cluster [DBG] pgmap v3541: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 158 KiB/s rd, 11 KiB/s wr, 218 op/s 2024-09-13T00:41:34.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:33 smithi181 bash[67587]: cluster 2024-09-13T00:41:32.888713+0000 mon.a (mon.0) 3755 : cluster [DBG] osdmap e2049: 8 total, 8 up, 8 in 2024-09-13T00:41:34.991 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_rename_snap PASSED [ 38%] 2024-09-13T00:41:35.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:34 smithi136 bash[75164]: cluster 2024-09-13T00:41:33.892239+0000 mon.a (mon.0) 3756 : cluster [DBG] osdmap e2050: 8 total, 8 up, 8 in 2024-09-13T00:41:35.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:34 smithi136 bash[75164]: cluster 2024-09-13T00:41:34.788547+0000 mon.a (mon.0) 3757 : cluster [DBG] osdmap e2051: 8 total, 8 up, 8 in 2024-09-13T00:41:35.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:34 smithi136 bash[76371]: cluster 2024-09-13T00:41:33.892239+0000 mon.a (mon.0) 3756 : cluster [DBG] osdmap e2050: 8 total, 8 up, 8 in 2024-09-13T00:41:35.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:34 smithi136 bash[76371]: cluster 2024-09-13T00:41:34.788547+0000 mon.a (mon.0) 3757 : cluster [DBG] osdmap e2051: 8 total, 8 up, 8 in 2024-09-13T00:41:35.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:34 smithi181 bash[67587]: cluster 2024-09-13T00:41:33.892239+0000 mon.a (mon.0) 3756 : cluster [DBG] osdmap e2050: 8 total, 8 up, 8 in 2024-09-13T00:41:35.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:34 smithi181 bash[67587]: cluster 2024-09-13T00:41:34.788547+0000 mon.a (mon.0) 3757 : cluster [DBG] osdmap e2051: 8 total, 8 up, 8 in 2024-09-13T00:41:36.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:35 smithi136 bash[75164]: cluster 2024-09-13T00:41:34.484302+0000 mgr.y (mgr.24661) 1933 : cluster [DBG] pgmap v3544: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 144 KiB/s rd, 11 KiB/s wr, 199 op/s 2024-09-13T00:41:36.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:35 smithi136 bash[76371]: cluster 2024-09-13T00:41:34.484302+0000 mgr.y (mgr.24661) 1933 : cluster [DBG] pgmap v3544: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 144 KiB/s rd, 11 KiB/s wr, 199 op/s 2024-09-13T00:41:36.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:35 smithi181 bash[67587]: cluster 2024-09-13T00:41:34.484302+0000 mgr.y (mgr.24661) 1933 : cluster [DBG] pgmap v3544: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 144 KiB/s rd, 11 KiB/s wr, 199 op/s 2024-09-13T00:41:37.148 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_protect_snap PASSED [ 39%] 2024-09-13T00:41:37.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:36 smithi136 bash[75164]: cluster 2024-09-13T00:41:35.924683+0000 mon.a (mon.0) 3758 : cluster [DBG] osdmap e2052: 8 total, 8 up, 8 in 2024-09-13T00:41:37.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:36 smithi136 bash[76371]: cluster 2024-09-13T00:41:35.924683+0000 mon.a (mon.0) 3758 : cluster [DBG] osdmap e2052: 8 total, 8 up, 8 in 2024-09-13T00:41:37.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:36 smithi181 bash[67587]: cluster 2024-09-13T00:41:35.924683+0000 mon.a (mon.0) 3758 : cluster [DBG] osdmap e2052: 8 total, 8 up, 8 in 2024-09-13T00:41:38.264 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:37 smithi136 bash[75164]: cluster 2024-09-13T00:41:36.485218+0000 mgr.y (mgr.24661) 1934 : cluster [DBG] pgmap v3547: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 101 KiB/s rd, 9.2 KiB/s wr, 143 op/s 2024-09-13T00:41:38.264 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:37 smithi136 bash[75164]: cluster 2024-09-13T00:41:36.934628+0000 mon.a (mon.0) 3759 : cluster [DBG] osdmap e2053: 8 total, 8 up, 8 in 2024-09-13T00:41:38.264 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:37 smithi136 bash[76371]: cluster 2024-09-13T00:41:36.485218+0000 mgr.y (mgr.24661) 1934 : cluster [DBG] pgmap v3547: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 101 KiB/s rd, 9.2 KiB/s wr, 143 op/s 2024-09-13T00:41:38.265 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:37 smithi136 bash[76371]: cluster 2024-09-13T00:41:36.934628+0000 mon.a (mon.0) 3759 : cluster [DBG] osdmap e2053: 8 total, 8 up, 8 in 2024-09-13T00:41:38.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:37 smithi181 bash[67587]: cluster 2024-09-13T00:41:36.485218+0000 mgr.y (mgr.24661) 1934 : cluster [DBG] pgmap v3547: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 101 KiB/s rd, 9.2 KiB/s wr, 143 op/s 2024-09-13T00:41:38.827 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:37 smithi181 bash[67587]: cluster 2024-09-13T00:41:36.934628+0000 mon.a (mon.0) 3759 : cluster [DBG] osdmap e2053: 8 total, 8 up, 8 in 2024-09-13T00:41:39.169 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_snap_exists PASSED [ 39%] 2024-09-13T00:41:39.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:38 smithi136 bash[75164]: cluster 2024-09-13T00:41:37.941996+0000 mon.a (mon.0) 3760 : cluster [DBG] osdmap e2054: 8 total, 8 up, 8 in 2024-09-13T00:41:39.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:38 smithi136 bash[75164]: audit 2024-09-13T00:41:38.840460+0000 mon.a (mon.0) 3761 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:41:39.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:38 smithi136 bash[76371]: cluster 2024-09-13T00:41:37.941996+0000 mon.a (mon.0) 3760 : cluster [DBG] osdmap e2054: 8 total, 8 up, 8 in 2024-09-13T00:41:39.262 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:38 smithi136 bash[76371]: audit 2024-09-13T00:41:38.840460+0000 mon.a (mon.0) 3761 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:41:39.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:38 smithi181 bash[67587]: cluster 2024-09-13T00:41:37.941996+0000 mon.a (mon.0) 3760 : cluster [DBG] osdmap e2054: 8 total, 8 up, 8 in 2024-09-13T00:41:39.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:38 smithi181 bash[67587]: audit 2024-09-13T00:41:38.840460+0000 mon.a (mon.0) 3761 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:41:39.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:41:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:41:39] "GET /metrics HTTP/1.1" 200 36744 "" "Prometheus/2.43.0" 2024-09-13T00:41:40.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:39 smithi136 bash[75164]: cluster 2024-09-13T00:41:38.486031+0000 mgr.y (mgr.24661) 1935 : cluster [DBG] pgmap v3550: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 98 KiB/s rd, 9.5 KiB/s wr, 137 op/s 2024-09-13T00:41:40.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:39 smithi136 bash[75164]: cluster 2024-09-13T00:41:38.961692+0000 mon.a (mon.0) 3762 : cluster [DBG] osdmap e2055: 8 total, 8 up, 8 in 2024-09-13T00:41:40.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:39 smithi136 bash[75164]: cluster 2024-09-13T00:41:39.789295+0000 mon.a (mon.0) 3763 : cluster [DBG] osdmap e2056: 8 total, 8 up, 8 in 2024-09-13T00:41:40.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:39 smithi136 bash[76371]: cluster 2024-09-13T00:41:38.486031+0000 mgr.y (mgr.24661) 1935 : cluster [DBG] pgmap v3550: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 98 KiB/s rd, 9.5 KiB/s wr, 137 op/s 2024-09-13T00:41:40.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:39 smithi136 bash[76371]: cluster 2024-09-13T00:41:38.961692+0000 mon.a (mon.0) 3762 : cluster [DBG] osdmap e2055: 8 total, 8 up, 8 in 2024-09-13T00:41:40.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:39 smithi136 bash[76371]: cluster 2024-09-13T00:41:39.789295+0000 mon.a (mon.0) 3763 : cluster [DBG] osdmap e2056: 8 total, 8 up, 8 in 2024-09-13T00:41:40.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:39 smithi181 bash[67587]: cluster 2024-09-13T00:41:38.486031+0000 mgr.y (mgr.24661) 1935 : cluster [DBG] pgmap v3550: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 98 KiB/s rd, 9.5 KiB/s wr, 137 op/s 2024-09-13T00:41:40.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:39 smithi181 bash[67587]: cluster 2024-09-13T00:41:38.961692+0000 mon.a (mon.0) 3762 : cluster [DBG] osdmap e2055: 8 total, 8 up, 8 in 2024-09-13T00:41:40.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:39 smithi181 bash[67587]: cluster 2024-09-13T00:41:39.789295+0000 mon.a (mon.0) 3763 : cluster [DBG] osdmap e2056: 8 total, 8 up, 8 in 2024-09-13T00:41:40.997 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_snap_timestamp PASSED [ 40%] 2024-09-13T00:41:42.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:41 smithi181 bash[67587]: cluster 2024-09-13T00:41:40.486790+0000 mgr.y (mgr.24661) 1936 : cluster [DBG] pgmap v3553: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 36 KiB/s rd, 1.2 KiB/s wr, 48 op/s 2024-09-13T00:41:42.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:41 smithi181 bash[67587]: cluster 2024-09-13T00:41:40.792475+0000 mon.a (mon.0) 3764 : cluster [DBG] osdmap e2057: 8 total, 8 up, 8 in 2024-09-13T00:41:42.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:41 smithi136 bash[75164]: cluster 2024-09-13T00:41:40.486790+0000 mgr.y (mgr.24661) 1936 : cluster [DBG] pgmap v3553: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 36 KiB/s rd, 1.2 KiB/s wr, 48 op/s 2024-09-13T00:41:42.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:41 smithi136 bash[75164]: cluster 2024-09-13T00:41:40.792475+0000 mon.a (mon.0) 3764 : cluster [DBG] osdmap e2057: 8 total, 8 up, 8 in 2024-09-13T00:41:42.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:41 smithi136 bash[76371]: cluster 2024-09-13T00:41:40.486790+0000 mgr.y (mgr.24661) 1936 : cluster [DBG] pgmap v3553: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 36 KiB/s rd, 1.2 KiB/s wr, 48 op/s 2024-09-13T00:41:42.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:41 smithi136 bash[76371]: cluster 2024-09-13T00:41:40.792475+0000 mon.a (mon.0) 3764 : cluster [DBG] osdmap e2057: 8 total, 8 up, 8 in 2024-09-13T00:41:42.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:41:42 smithi181 bash[95505]: ts=2024-09-13T00:41:42.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:41:43.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:42 smithi136 bash[75164]: cluster 2024-09-13T00:41:41.958022+0000 mon.a (mon.0) 3765 : cluster [DBG] osdmap e2058: 8 total, 8 up, 8 in 2024-09-13T00:41:43.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:42 smithi136 bash[76371]: cluster 2024-09-13T00:41:41.958022+0000 mon.a (mon.0) 3765 : cluster [DBG] osdmap e2058: 8 total, 8 up, 8 in 2024-09-13T00:41:43.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:42 smithi181 bash[67587]: cluster 2024-09-13T00:41:41.958022+0000 mon.a (mon.0) 3765 : cluster [DBG] osdmap e2058: 8 total, 8 up, 8 in 2024-09-13T00:41:44.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:43 smithi136 bash[75164]: cluster 2024-09-13T00:41:42.487497+0000 mgr.y (mgr.24661) 1937 : cluster [DBG] pgmap v3556: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 132 KiB/s rd, 8.0 KiB/s wr, 173 op/s 2024-09-13T00:41:44.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:43 smithi136 bash[75164]: cluster 2024-09-13T00:41:42.975914+0000 mon.a (mon.0) 3766 : cluster [DBG] osdmap e2059: 8 total, 8 up, 8 in 2024-09-13T00:41:44.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:43 smithi136 bash[76371]: cluster 2024-09-13T00:41:42.487497+0000 mgr.y (mgr.24661) 1937 : cluster [DBG] pgmap v3556: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 132 KiB/s rd, 8.0 KiB/s wr, 173 op/s 2024-09-13T00:41:44.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:43 smithi136 bash[76371]: cluster 2024-09-13T00:41:42.975914+0000 mon.a (mon.0) 3766 : cluster [DBG] osdmap e2059: 8 total, 8 up, 8 in 2024-09-13T00:41:44.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:43 smithi181 bash[67587]: cluster 2024-09-13T00:41:42.487497+0000 mgr.y (mgr.24661) 1937 : cluster [DBG] pgmap v3556: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 132 KiB/s rd, 8.0 KiB/s wr, 173 op/s 2024-09-13T00:41:44.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:43 smithi181 bash[67587]: cluster 2024-09-13T00:41:42.975914+0000 mon.a (mon.0) 3766 : cluster [DBG] osdmap e2059: 8 total, 8 up, 8 in 2024-09-13T00:41:45.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:44 smithi136 bash[75164]: cluster 2024-09-13T00:41:43.975367+0000 mon.a (mon.0) 3767 : cluster [DBG] osdmap e2060: 8 total, 8 up, 8 in 2024-09-13T00:41:45.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:44 smithi136 bash[75164]: cluster 2024-09-13T00:41:44.792999+0000 mon.a (mon.0) 3768 : cluster [DBG] osdmap e2061: 8 total, 8 up, 8 in 2024-09-13T00:41:45.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:44 smithi136 bash[76371]: cluster 2024-09-13T00:41:43.975367+0000 mon.a (mon.0) 3767 : cluster [DBG] osdmap e2060: 8 total, 8 up, 8 in 2024-09-13T00:41:45.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:44 smithi136 bash[76371]: cluster 2024-09-13T00:41:44.792999+0000 mon.a (mon.0) 3768 : cluster [DBG] osdmap e2061: 8 total, 8 up, 8 in 2024-09-13T00:41:45.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:44 smithi181 bash[67587]: cluster 2024-09-13T00:41:43.975367+0000 mon.a (mon.0) 3767 : cluster [DBG] osdmap e2060: 8 total, 8 up, 8 in 2024-09-13T00:41:45.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:44 smithi181 bash[67587]: cluster 2024-09-13T00:41:44.792999+0000 mon.a (mon.0) 3768 : cluster [DBG] osdmap e2061: 8 total, 8 up, 8 in 2024-09-13T00:41:46.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:45 smithi136 bash[76371]: cluster 2024-09-13T00:41:44.488236+0000 mgr.y (mgr.24661) 1938 : cluster [DBG] pgmap v3559: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 137 KiB/s rd, 9.7 KiB/s wr, 185 op/s 2024-09-13T00:41:46.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:45 smithi136 bash[76371]: cluster 2024-09-13T00:41:45.794414+0000 mon.a (mon.0) 3769 : cluster [DBG] osdmap e2062: 8 total, 8 up, 8 in 2024-09-13T00:41:46.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:45 smithi136 bash[75164]: cluster 2024-09-13T00:41:44.488236+0000 mgr.y (mgr.24661) 1938 : cluster [DBG] pgmap v3559: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 137 KiB/s rd, 9.7 KiB/s wr, 185 op/s 2024-09-13T00:41:46.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:45 smithi136 bash[75164]: cluster 2024-09-13T00:41:45.794414+0000 mon.a (mon.0) 3769 : cluster [DBG] osdmap e2062: 8 total, 8 up, 8 in 2024-09-13T00:41:46.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:45 smithi181 bash[67587]: cluster 2024-09-13T00:41:44.488236+0000 mgr.y (mgr.24661) 1938 : cluster [DBG] pgmap v3559: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 137 KiB/s rd, 9.7 KiB/s wr, 185 op/s 2024-09-13T00:41:46.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:45 smithi181 bash[67587]: cluster 2024-09-13T00:41:45.794414+0000 mon.a (mon.0) 3769 : cluster [DBG] osdmap e2062: 8 total, 8 up, 8 in 2024-09-13T00:41:48.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:47 smithi181 bash[67587]: cluster 2024-09-13T00:41:46.489255+0000 mgr.y (mgr.24661) 1939 : cluster [DBG] pgmap v3562: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 36 KiB/s rd, 3.0 KiB/s wr, 54 op/s 2024-09-13T00:41:48.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:47 smithi181 bash[67587]: cluster 2024-09-13T00:41:46.799846+0000 mon.a (mon.0) 3770 : cluster [DBG] osdmap e2063: 8 total, 8 up, 8 in 2024-09-13T00:41:48.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:47 smithi136 bash[75164]: cluster 2024-09-13T00:41:46.489255+0000 mgr.y (mgr.24661) 1939 : cluster [DBG] pgmap v3562: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 36 KiB/s rd, 3.0 KiB/s wr, 54 op/s 2024-09-13T00:41:48.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:47 smithi136 bash[75164]: cluster 2024-09-13T00:41:46.799846+0000 mon.a (mon.0) 3770 : cluster [DBG] osdmap e2063: 8 total, 8 up, 8 in 2024-09-13T00:41:48.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:47 smithi136 bash[76371]: cluster 2024-09-13T00:41:46.489255+0000 mgr.y (mgr.24661) 1939 : cluster [DBG] pgmap v3562: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 36 KiB/s rd, 3.0 KiB/s wr, 54 op/s 2024-09-13T00:41:48.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:47 smithi136 bash[76371]: cluster 2024-09-13T00:41:46.799846+0000 mon.a (mon.0) 3770 : cluster [DBG] osdmap e2063: 8 total, 8 up, 8 in 2024-09-13T00:41:49.013 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_limit_snaps PASSED [ 40%] 2024-09-13T00:41:49.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:48 smithi181 bash[67587]: cluster 2024-09-13T00:41:47.806418+0000 mon.a (mon.0) 3771 : cluster [DBG] osdmap e2064: 8 total, 8 up, 8 in 2024-09-13T00:41:49.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:48 smithi181 bash[67587]: audit 2024-09-13T00:41:48.280714+0000 mon.a (mon.0) 3772 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:41:49.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:48 smithi136 bash[75164]: cluster 2024-09-13T00:41:47.806418+0000 mon.a (mon.0) 3771 : cluster [DBG] osdmap e2064: 8 total, 8 up, 8 in 2024-09-13T00:41:49.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:48 smithi136 bash[75164]: audit 2024-09-13T00:41:48.280714+0000 mon.a (mon.0) 3772 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:41:49.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:48 smithi136 bash[76371]: cluster 2024-09-13T00:41:47.806418+0000 mon.a (mon.0) 3771 : cluster [DBG] osdmap e2064: 8 total, 8 up, 8 in 2024-09-13T00:41:49.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:48 smithi136 bash[76371]: audit 2024-09-13T00:41:48.280714+0000 mon.a (mon.0) 3772 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:41:49.266 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_remove_with_exclusive_lock PASSED [ 41%] 2024-09-13T00:41:49.469 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_remove_with_snap SKIPPED [ 41%] 2024-09-13T00:41:49.667 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_remove_with_watcher SKIPPED [ 42%] 2024-09-13T00:41:49.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:41:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:41:49] "GET /metrics HTTP/1.1" 200 36744 "" "Prometheus/2.43.0" 2024-09-13T00:41:50.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:49 smithi181 bash[67587]: cluster 2024-09-13T00:41:48.489759+0000 mgr.y (mgr.24661) 1940 : cluster [DBG] pgmap v3565: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 3.2 KiB/s rd, 767 B/s wr, 6 op/s 2024-09-13T00:41:50.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:49 smithi181 bash[67587]: cluster 2024-09-13T00:41:48.820194+0000 mon.a (mon.0) 3773 : cluster [DBG] osdmap e2065: 8 total, 8 up, 8 in 2024-09-13T00:41:50.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:49 smithi181 bash[67587]: cluster 2024-09-13T00:41:49.795425+0000 mon.a (mon.0) 3774 : cluster [DBG] osdmap e2066: 8 total, 8 up, 8 in 2024-09-13T00:41:50.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:49 smithi136 bash[76371]: cluster 2024-09-13T00:41:48.489759+0000 mgr.y (mgr.24661) 1940 : cluster [DBG] pgmap v3565: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 3.2 KiB/s rd, 767 B/s wr, 6 op/s 2024-09-13T00:41:50.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:49 smithi136 bash[76371]: cluster 2024-09-13T00:41:48.820194+0000 mon.a (mon.0) 3773 : cluster [DBG] osdmap e2065: 8 total, 8 up, 8 in 2024-09-13T00:41:50.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:49 smithi136 bash[76371]: cluster 2024-09-13T00:41:49.795425+0000 mon.a (mon.0) 3774 : cluster [DBG] osdmap e2066: 8 total, 8 up, 8 in 2024-09-13T00:41:50.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:49 smithi136 bash[75164]: cluster 2024-09-13T00:41:48.489759+0000 mgr.y (mgr.24661) 1940 : cluster [DBG] pgmap v3565: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 3.2 KiB/s rd, 767 B/s wr, 6 op/s 2024-09-13T00:41:50.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:49 smithi136 bash[75164]: cluster 2024-09-13T00:41:48.820194+0000 mon.a (mon.0) 3773 : cluster [DBG] osdmap e2065: 8 total, 8 up, 8 in 2024-09-13T00:41:50.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:49 smithi136 bash[75164]: cluster 2024-09-13T00:41:49.795425+0000 mon.a (mon.0) 3774 : cluster [DBG] osdmap e2066: 8 total, 8 up, 8 in 2024-09-13T00:41:51.044 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_rollback_to_snap PASSED [ 42%] 2024-09-13T00:41:52.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:51 smithi181 bash[67587]: cluster 2024-09-13T00:41:50.490450+0000 mgr.y (mgr.24661) 1941 : cluster [DBG] pgmap v3568: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 54 KiB/s rd, 4.0 KiB/s wr, 73 op/s 2024-09-13T00:41:52.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:51 smithi181 bash[67587]: cluster 2024-09-13T00:41:50.830603+0000 mon.a (mon.0) 3775 : cluster [DBG] osdmap e2067: 8 total, 8 up, 8 in 2024-09-13T00:41:52.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:51 smithi136 bash[75164]: cluster 2024-09-13T00:41:50.490450+0000 mgr.y (mgr.24661) 1941 : cluster [DBG] pgmap v3568: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 54 KiB/s rd, 4.0 KiB/s wr, 73 op/s 2024-09-13T00:41:52.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:51 smithi136 bash[75164]: cluster 2024-09-13T00:41:50.830603+0000 mon.a (mon.0) 3775 : cluster [DBG] osdmap e2067: 8 total, 8 up, 8 in 2024-09-13T00:41:52.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:51 smithi136 bash[76371]: cluster 2024-09-13T00:41:50.490450+0000 mgr.y (mgr.24661) 1941 : cluster [DBG] pgmap v3568: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 54 KiB/s rd, 4.0 KiB/s wr, 73 op/s 2024-09-13T00:41:52.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:51 smithi136 bash[76371]: cluster 2024-09-13T00:41:50.830603+0000 mon.a (mon.0) 3775 : cluster [DBG] osdmap e2067: 8 total, 8 up, 8 in 2024-09-13T00:41:52.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:41:52 smithi181 bash[95505]: ts=2024-09-13T00:41:52.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:41:53.058 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_rollback_to_snap_sparse PASSED [ 43%] 2024-09-13T00:41:53.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:52 smithi136 bash[75164]: cluster 2024-09-13T00:41:51.843112+0000 mon.a (mon.0) 3776 : cluster [DBG] osdmap e2068: 8 total, 8 up, 8 in 2024-09-13T00:41:53.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:52 smithi136 bash[76371]: cluster 2024-09-13T00:41:51.843112+0000 mon.a (mon.0) 3776 : cluster [DBG] osdmap e2068: 8 total, 8 up, 8 in 2024-09-13T00:41:53.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:52 smithi181 bash[67587]: cluster 2024-09-13T00:41:51.843112+0000 mon.a (mon.0) 3776 : cluster [DBG] osdmap e2068: 8 total, 8 up, 8 in 2024-09-13T00:41:54.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:53 smithi136 bash[75164]: cluster 2024-09-13T00:41:52.491424+0000 mgr.y (mgr.24661) 1942 : cluster [DBG] pgmap v3571: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 140 KiB/s rd, 12 KiB/s wr, 192 op/s 2024-09-13T00:41:54.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:53 smithi136 bash[75164]: cluster 2024-09-13T00:41:52.857032+0000 mon.a (mon.0) 3777 : cluster [DBG] osdmap e2069: 8 total, 8 up, 8 in 2024-09-13T00:41:54.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:53 smithi136 bash[75164]: audit 2024-09-13T00:41:53.768776+0000 mon.a (mon.0) 3778 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:41:54.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:53 smithi136 bash[75164]: audit 2024-09-13T00:41:53.774995+0000 mon.a (mon.0) 3779 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:41:54.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:53 smithi136 bash[75164]: audit 2024-09-13T00:41:53.840472+0000 mon.a (mon.0) 3780 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:41:54.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:53 smithi136 bash[76371]: cluster 2024-09-13T00:41:52.491424+0000 mgr.y (mgr.24661) 1942 : cluster [DBG] pgmap v3571: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 140 KiB/s rd, 12 KiB/s wr, 192 op/s 2024-09-13T00:41:54.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:53 smithi136 bash[76371]: cluster 2024-09-13T00:41:52.857032+0000 mon.a (mon.0) 3777 : cluster [DBG] osdmap e2069: 8 total, 8 up, 8 in 2024-09-13T00:41:54.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:53 smithi136 bash[76371]: audit 2024-09-13T00:41:53.768776+0000 mon.a (mon.0) 3778 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:41:54.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:53 smithi136 bash[76371]: audit 2024-09-13T00:41:53.774995+0000 mon.a (mon.0) 3779 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:41:54.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:53 smithi136 bash[76371]: audit 2024-09-13T00:41:53.840472+0000 mon.a (mon.0) 3780 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:41:54.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:53 smithi181 bash[67587]: cluster 2024-09-13T00:41:52.491424+0000 mgr.y (mgr.24661) 1942 : cluster [DBG] pgmap v3571: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 140 KiB/s rd, 12 KiB/s wr, 192 op/s 2024-09-13T00:41:54.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:53 smithi181 bash[67587]: cluster 2024-09-13T00:41:52.857032+0000 mon.a (mon.0) 3777 : cluster [DBG] osdmap e2069: 8 total, 8 up, 8 in 2024-09-13T00:41:54.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:53 smithi181 bash[67587]: audit 2024-09-13T00:41:53.768776+0000 mon.a (mon.0) 3778 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:41:54.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:53 smithi181 bash[67587]: audit 2024-09-13T00:41:53.774995+0000 mon.a (mon.0) 3779 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:41:54.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:53 smithi181 bash[67587]: audit 2024-09-13T00:41:53.840472+0000 mon.a (mon.0) 3780 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:41:55.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:54 smithi136 bash[75164]: cluster 2024-09-13T00:41:53.859702+0000 mon.a (mon.0) 3781 : cluster [DBG] osdmap e2070: 8 total, 8 up, 8 in 2024-09-13T00:41:55.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:54 smithi136 bash[75164]: audit 2024-09-13T00:41:54.014616+0000 mon.a (mon.0) 3782 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:41:55.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:54 smithi136 bash[75164]: audit 2024-09-13T00:41:54.024905+0000 mon.a (mon.0) 3783 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:41:55.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:54 smithi136 bash[75164]: audit 2024-09-13T00:41:54.423628+0000 mon.a (mon.0) 3784 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:41:55.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:54 smithi136 bash[75164]: audit 2024-09-13T00:41:54.424849+0000 mon.a (mon.0) 3785 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:41:55.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:54 smithi136 bash[75164]: audit 2024-09-13T00:41:54.430669+0000 mon.a (mon.0) 3786 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:41:55.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:54 smithi136 bash[75164]: cluster 2024-09-13T00:41:54.800597+0000 mon.a (mon.0) 3787 : cluster [DBG] osdmap e2071: 8 total, 8 up, 8 in 2024-09-13T00:41:55.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:54 smithi136 bash[76371]: cluster 2024-09-13T00:41:53.859702+0000 mon.a (mon.0) 3781 : cluster [DBG] osdmap e2070: 8 total, 8 up, 8 in 2024-09-13T00:41:55.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:54 smithi136 bash[76371]: audit 2024-09-13T00:41:54.014616+0000 mon.a (mon.0) 3782 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:41:55.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:54 smithi136 bash[76371]: audit 2024-09-13T00:41:54.024905+0000 mon.a (mon.0) 3783 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:41:55.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:54 smithi136 bash[76371]: audit 2024-09-13T00:41:54.423628+0000 mon.a (mon.0) 3784 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:41:55.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:54 smithi136 bash[76371]: audit 2024-09-13T00:41:54.424849+0000 mon.a (mon.0) 3785 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:41:55.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:54 smithi136 bash[76371]: audit 2024-09-13T00:41:54.430669+0000 mon.a (mon.0) 3786 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:41:55.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:54 smithi136 bash[76371]: cluster 2024-09-13T00:41:54.800597+0000 mon.a (mon.0) 3787 : cluster [DBG] osdmap e2071: 8 total, 8 up, 8 in 2024-09-13T00:41:55.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:54 smithi181 bash[67587]: cluster 2024-09-13T00:41:53.859702+0000 mon.a (mon.0) 3781 : cluster [DBG] osdmap e2070: 8 total, 8 up, 8 in 2024-09-13T00:41:55.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:54 smithi181 bash[67587]: audit 2024-09-13T00:41:54.014616+0000 mon.a (mon.0) 3782 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:41:55.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:54 smithi181 bash[67587]: audit 2024-09-13T00:41:54.024905+0000 mon.a (mon.0) 3783 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:41:55.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:54 smithi181 bash[67587]: audit 2024-09-13T00:41:54.423628+0000 mon.a (mon.0) 3784 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:41:55.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:54 smithi181 bash[67587]: audit 2024-09-13T00:41:54.424849+0000 mon.a (mon.0) 3785 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:41:55.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:54 smithi181 bash[67587]: audit 2024-09-13T00:41:54.430669+0000 mon.a (mon.0) 3786 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:41:55.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:54 smithi181 bash[67587]: cluster 2024-09-13T00:41:54.800597+0000 mon.a (mon.0) 3787 : cluster [DBG] osdmap e2071: 8 total, 8 up, 8 in 2024-09-13T00:41:56.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:56 smithi136 bash[75164]: cluster 2024-09-13T00:41:54.491923+0000 mgr.y (mgr.24661) 1943 : cluster [DBG] pgmap v3574: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 159 KiB/s rd, 13 KiB/s wr, 222 op/s 2024-09-13T00:41:56.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:56 smithi136 bash[75164]: cluster 2024-09-13T00:41:55.875978+0000 mon.a (mon.0) 3788 : cluster [DBG] osdmap e2072: 8 total, 8 up, 8 in 2024-09-13T00:41:56.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:56 smithi136 bash[76371]: cluster 2024-09-13T00:41:54.491923+0000 mgr.y (mgr.24661) 1943 : cluster [DBG] pgmap v3574: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 159 KiB/s rd, 13 KiB/s wr, 222 op/s 2024-09-13T00:41:56.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:56 smithi136 bash[76371]: cluster 2024-09-13T00:41:55.875978+0000 mon.a (mon.0) 3788 : cluster [DBG] osdmap e2072: 8 total, 8 up, 8 in 2024-09-13T00:41:56.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:56 smithi181 bash[67587]: cluster 2024-09-13T00:41:54.491923+0000 mgr.y (mgr.24661) 1943 : cluster [DBG] pgmap v3574: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 159 KiB/s rd, 13 KiB/s wr, 222 op/s 2024-09-13T00:41:56.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:56 smithi181 bash[67587]: cluster 2024-09-13T00:41:55.875978+0000 mon.a (mon.0) 3788 : cluster [DBG] osdmap e2072: 8 total, 8 up, 8 in 2024-09-13T00:41:57.122 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_rollback_with_resize PASSED [ 43%] 2024-09-13T00:41:58.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:57 smithi136 bash[75164]: cluster 2024-09-13T00:41:56.492909+0000 mgr.y (mgr.24661) 1944 : cluster [DBG] pgmap v3577: 137 pgs: 2 active+clean+snaptrim_wait, 135 active+clean; 13 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 161 KiB/s rd, 13 KiB/s wr, 225 op/s 2024-09-13T00:41:58.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:57 smithi136 bash[75164]: cluster 2024-09-13T00:41:56.880397+0000 mon.a (mon.0) 3789 : cluster [DBG] osdmap e2073: 8 total, 8 up, 8 in 2024-09-13T00:41:58.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:57 smithi136 bash[76371]: cluster 2024-09-13T00:41:56.492909+0000 mgr.y (mgr.24661) 1944 : cluster [DBG] pgmap v3577: 137 pgs: 2 active+clean+snaptrim_wait, 135 active+clean; 13 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 161 KiB/s rd, 13 KiB/s wr, 225 op/s 2024-09-13T00:41:58.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:57 smithi136 bash[76371]: cluster 2024-09-13T00:41:56.880397+0000 mon.a (mon.0) 3789 : cluster [DBG] osdmap e2073: 8 total, 8 up, 8 in 2024-09-13T00:41:58.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:57 smithi181 bash[67587]: cluster 2024-09-13T00:41:56.492909+0000 mgr.y (mgr.24661) 1944 : cluster [DBG] pgmap v3577: 137 pgs: 2 active+clean+snaptrim_wait, 135 active+clean; 13 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 161 KiB/s rd, 13 KiB/s wr, 225 op/s 2024-09-13T00:41:58.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:57 smithi181 bash[67587]: cluster 2024-09-13T00:41:56.880397+0000 mon.a (mon.0) 3789 : cluster [DBG] osdmap e2073: 8 total, 8 up, 8 in 2024-09-13T00:41:59.114 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_set_snap PASSED [ 44%] 2024-09-13T00:41:59.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:58 smithi136 bash[75164]: cluster 2024-09-13T00:41:57.890648+0000 mon.a (mon.0) 3790 : cluster [DBG] osdmap e2074: 8 total, 8 up, 8 in 2024-09-13T00:41:59.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:58 smithi136 bash[76371]: cluster 2024-09-13T00:41:57.890648+0000 mon.a (mon.0) 3790 : cluster [DBG] osdmap e2074: 8 total, 8 up, 8 in 2024-09-13T00:41:59.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:58 smithi181 bash[67587]: cluster 2024-09-13T00:41:57.890648+0000 mon.a (mon.0) 3790 : cluster [DBG] osdmap e2074: 8 total, 8 up, 8 in 2024-09-13T00:41:59.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:41:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:41:59] "GET /metrics HTTP/1.1" 200 36740 "" "Prometheus/2.43.0" 2024-09-13T00:42:00.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:59 smithi136 bash[75164]: cluster 2024-09-13T00:41:58.493616+0000 mgr.y (mgr.24661) 1945 : cluster [DBG] pgmap v3580: 137 pgs: 2 active+clean+snaptrim_wait, 135 active+clean; 13 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 106 KiB/s rd, 13 KiB/s wr, 148 op/s 2024-09-13T00:42:00.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:59 smithi136 bash[75164]: cluster 2024-09-13T00:41:58.901153+0000 mon.a (mon.0) 3791 : cluster [DBG] osdmap e2075: 8 total, 8 up, 8 in 2024-09-13T00:42:00.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:41:59 smithi136 bash[75164]: cluster 2024-09-13T00:41:59.798210+0000 mon.a (mon.0) 3792 : cluster [DBG] osdmap e2076: 8 total, 8 up, 8 in 2024-09-13T00:42:00.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:59 smithi136 bash[76371]: cluster 2024-09-13T00:41:58.493616+0000 mgr.y (mgr.24661) 1945 : cluster [DBG] pgmap v3580: 137 pgs: 2 active+clean+snaptrim_wait, 135 active+clean; 13 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 106 KiB/s rd, 13 KiB/s wr, 148 op/s 2024-09-13T00:42:00.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:59 smithi136 bash[76371]: cluster 2024-09-13T00:41:58.901153+0000 mon.a (mon.0) 3791 : cluster [DBG] osdmap e2075: 8 total, 8 up, 8 in 2024-09-13T00:42:00.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:41:59 smithi136 bash[76371]: cluster 2024-09-13T00:41:59.798210+0000 mon.a (mon.0) 3792 : cluster [DBG] osdmap e2076: 8 total, 8 up, 8 in 2024-09-13T00:42:00.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:59 smithi181 bash[67587]: cluster 2024-09-13T00:41:58.493616+0000 mgr.y (mgr.24661) 1945 : cluster [DBG] pgmap v3580: 137 pgs: 2 active+clean+snaptrim_wait, 135 active+clean; 13 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 106 KiB/s rd, 13 KiB/s wr, 148 op/s 2024-09-13T00:42:00.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:59 smithi181 bash[67587]: cluster 2024-09-13T00:41:58.901153+0000 mon.a (mon.0) 3791 : cluster [DBG] osdmap e2075: 8 total, 8 up, 8 in 2024-09-13T00:42:00.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:41:59 smithi181 bash[67587]: cluster 2024-09-13T00:41:59.798210+0000 mon.a (mon.0) 3792 : cluster [DBG] osdmap e2076: 8 total, 8 up, 8 in 2024-09-13T00:42:01.005 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_set_no_snap PASSED [ 45%] 2024-09-13T00:42:02.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:01 smithi181 bash[67587]: cluster 2024-09-13T00:42:00.494356+0000 mgr.y (mgr.24661) 1946 : cluster [DBG] pgmap v3583: 137 pgs: 2 active+clean+snaptrim_wait, 135 active+clean; 13 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 16 KiB/s rd, 5.2 KiB/s wr, 27 op/s 2024-09-13T00:42:02.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:01 smithi181 bash[67587]: cluster 2024-09-13T00:42:00.802394+0000 mon.a (mon.0) 3793 : cluster [DBG] osdmap e2077: 8 total, 8 up, 8 in 2024-09-13T00:42:02.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:01 smithi136 bash[75164]: cluster 2024-09-13T00:42:00.494356+0000 mgr.y (mgr.24661) 1946 : cluster [DBG] pgmap v3583: 137 pgs: 2 active+clean+snaptrim_wait, 135 active+clean; 13 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 16 KiB/s rd, 5.2 KiB/s wr, 27 op/s 2024-09-13T00:42:02.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:01 smithi136 bash[75164]: cluster 2024-09-13T00:42:00.802394+0000 mon.a (mon.0) 3793 : cluster [DBG] osdmap e2077: 8 total, 8 up, 8 in 2024-09-13T00:42:02.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:01 smithi136 bash[76371]: cluster 2024-09-13T00:42:00.494356+0000 mgr.y (mgr.24661) 1946 : cluster [DBG] pgmap v3583: 137 pgs: 2 active+clean+snaptrim_wait, 135 active+clean; 13 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 16 KiB/s rd, 5.2 KiB/s wr, 27 op/s 2024-09-13T00:42:02.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:01 smithi136 bash[76371]: cluster 2024-09-13T00:42:00.802394+0000 mon.a (mon.0) 3793 : cluster [DBG] osdmap e2077: 8 total, 8 up, 8 in 2024-09-13T00:42:02.601 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:42:02 smithi181 bash[95505]: ts=2024-09-13T00:42:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:42:03.143 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_set_snap_by_id PASSED [ 45%] 2024-09-13T00:42:03.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:02 smithi136 bash[75164]: cluster 2024-09-13T00:42:01.905475+0000 mon.a (mon.0) 3794 : cluster [DBG] osdmap e2078: 8 total, 8 up, 8 in 2024-09-13T00:42:03.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:02 smithi136 bash[76371]: cluster 2024-09-13T00:42:01.905475+0000 mon.a (mon.0) 3794 : cluster [DBG] osdmap e2078: 8 total, 8 up, 8 in 2024-09-13T00:42:03.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:02 smithi181 bash[67587]: cluster 2024-09-13T00:42:01.905475+0000 mon.a (mon.0) 3794 : cluster [DBG] osdmap e2078: 8 total, 8 up, 8 in 2024-09-13T00:42:04.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:03 smithi136 bash[75164]: cluster 2024-09-13T00:42:02.495316+0000 mgr.y (mgr.24661) 1947 : cluster [DBG] pgmap v3586: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 85 KiB/s rd, 11 KiB/s wr, 125 op/s 2024-09-13T00:42:04.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:03 smithi136 bash[75164]: cluster 2024-09-13T00:42:02.917601+0000 mon.a (mon.0) 3795 : cluster [DBG] osdmap e2079: 8 total, 8 up, 8 in 2024-09-13T00:42:04.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:03 smithi136 bash[76371]: cluster 2024-09-13T00:42:02.495316+0000 mgr.y (mgr.24661) 1947 : cluster [DBG] pgmap v3586: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 85 KiB/s rd, 11 KiB/s wr, 125 op/s 2024-09-13T00:42:04.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:03 smithi136 bash[76371]: cluster 2024-09-13T00:42:02.917601+0000 mon.a (mon.0) 3795 : cluster [DBG] osdmap e2079: 8 total, 8 up, 8 in 2024-09-13T00:42:04.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:03 smithi181 bash[67587]: cluster 2024-09-13T00:42:02.495316+0000 mgr.y (mgr.24661) 1947 : cluster [DBG] pgmap v3586: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 85 KiB/s rd, 11 KiB/s wr, 125 op/s 2024-09-13T00:42:04.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:03 smithi181 bash[67587]: cluster 2024-09-13T00:42:02.917601+0000 mon.a (mon.0) 3795 : cluster [DBG] osdmap e2079: 8 total, 8 up, 8 in 2024-09-13T00:42:05.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:04 smithi136 bash[75164]: cluster 2024-09-13T00:42:03.924168+0000 mon.a (mon.0) 3796 : cluster [DBG] osdmap e2080: 8 total, 8 up, 8 in 2024-09-13T00:42:05.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:04 smithi136 bash[75164]: cluster 2024-09-13T00:42:04.802928+0000 mon.a (mon.0) 3797 : cluster [DBG] osdmap e2081: 8 total, 8 up, 8 in 2024-09-13T00:42:05.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:04 smithi136 bash[76371]: cluster 2024-09-13T00:42:03.924168+0000 mon.a (mon.0) 3796 : cluster [DBG] osdmap e2080: 8 total, 8 up, 8 in 2024-09-13T00:42:05.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:04 smithi136 bash[76371]: cluster 2024-09-13T00:42:04.802928+0000 mon.a (mon.0) 3797 : cluster [DBG] osdmap e2081: 8 total, 8 up, 8 in 2024-09-13T00:42:05.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:04 smithi181 bash[67587]: cluster 2024-09-13T00:42:03.924168+0000 mon.a (mon.0) 3796 : cluster [DBG] osdmap e2080: 8 total, 8 up, 8 in 2024-09-13T00:42:05.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:04 smithi181 bash[67587]: cluster 2024-09-13T00:42:04.802928+0000 mon.a (mon.0) 3797 : cluster [DBG] osdmap e2081: 8 total, 8 up, 8 in 2024-09-13T00:42:06.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:05 smithi136 bash[76371]: cluster 2024-09-13T00:42:04.496043+0000 mgr.y (mgr.24661) 1948 : cluster [DBG] pgmap v3589: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 136 KiB/s rd, 17 KiB/s wr, 195 op/s 2024-09-13T00:42:06.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:05 smithi136 bash[76371]: cluster 2024-09-13T00:42:05.805209+0000 mon.a (mon.0) 3798 : cluster [DBG] osdmap e2082: 8 total, 8 up, 8 in 2024-09-13T00:42:06.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:05 smithi136 bash[75164]: cluster 2024-09-13T00:42:04.496043+0000 mgr.y (mgr.24661) 1948 : cluster [DBG] pgmap v3589: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 136 KiB/s rd, 17 KiB/s wr, 195 op/s 2024-09-13T00:42:06.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:05 smithi136 bash[75164]: cluster 2024-09-13T00:42:05.805209+0000 mon.a (mon.0) 3798 : cluster [DBG] osdmap e2082: 8 total, 8 up, 8 in 2024-09-13T00:42:06.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:05 smithi181 bash[67587]: cluster 2024-09-13T00:42:04.496043+0000 mgr.y (mgr.24661) 1948 : cluster [DBG] pgmap v3589: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 136 KiB/s rd, 17 KiB/s wr, 195 op/s 2024-09-13T00:42:06.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:05 smithi181 bash[67587]: cluster 2024-09-13T00:42:05.805209+0000 mon.a (mon.0) 3798 : cluster [DBG] osdmap e2082: 8 total, 8 up, 8 in 2024-09-13T00:42:08.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:07 smithi181 bash[67587]: cluster 2024-09-13T00:42:06.497134+0000 mgr.y (mgr.24661) 1949 : cluster [DBG] pgmap v3592: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 119 KiB/s rd, 13 KiB/s wr, 168 op/s 2024-09-13T00:42:08.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:07 smithi181 bash[67587]: cluster 2024-09-13T00:42:06.807468+0000 mon.a (mon.0) 3799 : cluster [DBG] osdmap e2083: 8 total, 8 up, 8 in 2024-09-13T00:42:08.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:07 smithi136 bash[75164]: cluster 2024-09-13T00:42:06.497134+0000 mgr.y (mgr.24661) 1949 : cluster [DBG] pgmap v3592: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 119 KiB/s rd, 13 KiB/s wr, 168 op/s 2024-09-13T00:42:08.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:07 smithi136 bash[75164]: cluster 2024-09-13T00:42:06.807468+0000 mon.a (mon.0) 3799 : cluster [DBG] osdmap e2083: 8 total, 8 up, 8 in 2024-09-13T00:42:08.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:07 smithi136 bash[76371]: cluster 2024-09-13T00:42:06.497134+0000 mgr.y (mgr.24661) 1949 : cluster [DBG] pgmap v3592: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 119 KiB/s rd, 13 KiB/s wr, 168 op/s 2024-09-13T00:42:08.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:07 smithi136 bash[76371]: cluster 2024-09-13T00:42:06.807468+0000 mon.a (mon.0) 3799 : cluster [DBG] osdmap e2083: 8 total, 8 up, 8 in 2024-09-13T00:42:09.046 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_snap_get_name PASSED [ 46%] 2024-09-13T00:42:09.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:08 smithi181 bash[67587]: cluster 2024-09-13T00:42:07.811091+0000 mon.a (mon.0) 3800 : cluster [DBG] osdmap e2084: 8 total, 8 up, 8 in 2024-09-13T00:42:09.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:08 smithi136 bash[75164]: cluster 2024-09-13T00:42:07.811091+0000 mon.a (mon.0) 3800 : cluster [DBG] osdmap e2084: 8 total, 8 up, 8 in 2024-09-13T00:42:09.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:08 smithi136 bash[76371]: cluster 2024-09-13T00:42:07.811091+0000 mon.a (mon.0) 3800 : cluster [DBG] osdmap e2084: 8 total, 8 up, 8 in 2024-09-13T00:42:09.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:42:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:42:09] "GET /metrics HTTP/1.1" 200 36741 "" "Prometheus/2.43.0" 2024-09-13T00:42:10.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:09 smithi181 bash[67587]: cluster 2024-09-13T00:42:08.497819+0000 mgr.y (mgr.24661) 1950 : cluster [DBG] pgmap v3595: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 71 KiB/s rd, 7.0 KiB/s wr, 98 op/s 2024-09-13T00:42:10.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:09 smithi181 bash[67587]: cluster 2024-09-13T00:42:08.844423+0000 mon.a (mon.0) 3801 : cluster [DBG] osdmap e2085: 8 total, 8 up, 8 in 2024-09-13T00:42:10.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:09 smithi181 bash[67587]: audit 2024-09-13T00:42:08.846625+0000 mon.a (mon.0) 3802 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:42:10.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:09 smithi181 bash[67587]: cluster 2024-09-13T00:42:09.807970+0000 mon.a (mon.0) 3803 : cluster [DBG] osdmap e2086: 8 total, 8 up, 8 in 2024-09-13T00:42:10.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:09 smithi136 bash[75164]: cluster 2024-09-13T00:42:08.497819+0000 mgr.y (mgr.24661) 1950 : cluster [DBG] pgmap v3595: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 71 KiB/s rd, 7.0 KiB/s wr, 98 op/s 2024-09-13T00:42:10.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:09 smithi136 bash[75164]: cluster 2024-09-13T00:42:08.844423+0000 mon.a (mon.0) 3801 : cluster [DBG] osdmap e2085: 8 total, 8 up, 8 in 2024-09-13T00:42:10.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:09 smithi136 bash[75164]: audit 2024-09-13T00:42:08.846625+0000 mon.a (mon.0) 3802 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:42:10.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:09 smithi136 bash[75164]: cluster 2024-09-13T00:42:09.807970+0000 mon.a (mon.0) 3803 : cluster [DBG] osdmap e2086: 8 total, 8 up, 8 in 2024-09-13T00:42:10.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:09 smithi136 bash[76371]: cluster 2024-09-13T00:42:08.497819+0000 mgr.y (mgr.24661) 1950 : cluster [DBG] pgmap v3595: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 71 KiB/s rd, 7.0 KiB/s wr, 98 op/s 2024-09-13T00:42:10.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:09 smithi136 bash[76371]: cluster 2024-09-13T00:42:08.844423+0000 mon.a (mon.0) 3801 : cluster [DBG] osdmap e2085: 8 total, 8 up, 8 in 2024-09-13T00:42:10.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:09 smithi136 bash[76371]: audit 2024-09-13T00:42:08.846625+0000 mon.a (mon.0) 3802 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:42:10.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:09 smithi136 bash[76371]: cluster 2024-09-13T00:42:09.807970+0000 mon.a (mon.0) 3803 : cluster [DBG] osdmap e2086: 8 total, 8 up, 8 in 2024-09-13T00:42:12.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:11 smithi136 bash[75164]: cluster 2024-09-13T00:42:10.498491+0000 mgr.y (mgr.24661) 1951 : cluster [DBG] pgmap v3598: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 6.5 KiB/s rd, 511 B/s wr, 9 op/s 2024-09-13T00:42:12.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:11 smithi136 bash[75164]: cluster 2024-09-13T00:42:10.851105+0000 mon.a (mon.0) 3804 : cluster [DBG] osdmap e2087: 8 total, 8 up, 8 in 2024-09-13T00:42:12.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:11 smithi136 bash[76371]: cluster 2024-09-13T00:42:10.498491+0000 mgr.y (mgr.24661) 1951 : cluster [DBG] pgmap v3598: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 6.5 KiB/s rd, 511 B/s wr, 9 op/s 2024-09-13T00:42:12.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:11 smithi136 bash[76371]: cluster 2024-09-13T00:42:10.851105+0000 mon.a (mon.0) 3804 : cluster [DBG] osdmap e2087: 8 total, 8 up, 8 in 2024-09-13T00:42:12.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:11 smithi181 bash[67587]: cluster 2024-09-13T00:42:10.498491+0000 mgr.y (mgr.24661) 1951 : cluster [DBG] pgmap v3598: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 6.5 KiB/s rd, 511 B/s wr, 9 op/s 2024-09-13T00:42:12.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:11 smithi181 bash[67587]: cluster 2024-09-13T00:42:10.851105+0000 mon.a (mon.0) 3804 : cluster [DBG] osdmap e2087: 8 total, 8 up, 8 in 2024-09-13T00:42:12.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:42:12 smithi181 bash[95505]: ts=2024-09-13T00:42:12.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:42:13.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:12 smithi136 bash[75164]: cluster 2024-09-13T00:42:11.856074+0000 mon.a (mon.0) 3805 : cluster [DBG] osdmap e2088: 8 total, 8 up, 8 in 2024-09-13T00:42:13.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:12 smithi136 bash[76371]: cluster 2024-09-13T00:42:11.856074+0000 mon.a (mon.0) 3805 : cluster [DBG] osdmap e2088: 8 total, 8 up, 8 in 2024-09-13T00:42:13.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:12 smithi181 bash[67587]: cluster 2024-09-13T00:42:11.856074+0000 mon.a (mon.0) 3805 : cluster [DBG] osdmap e2088: 8 total, 8 up, 8 in 2024-09-13T00:42:14.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:13 smithi136 bash[75164]: cluster 2024-09-13T00:42:12.499371+0000 mgr.y (mgr.24661) 1952 : cluster [DBG] pgmap v3601: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 86 KiB/s rd, 8.0 KiB/s wr, 114 op/s 2024-09-13T00:42:14.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:13 smithi136 bash[75164]: cluster 2024-09-13T00:42:12.880514+0000 mon.a (mon.0) 3806 : cluster [DBG] osdmap e2089: 8 total, 8 up, 8 in 2024-09-13T00:42:14.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:13 smithi136 bash[76371]: cluster 2024-09-13T00:42:12.499371+0000 mgr.y (mgr.24661) 1952 : cluster [DBG] pgmap v3601: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 86 KiB/s rd, 8.0 KiB/s wr, 114 op/s 2024-09-13T00:42:14.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:13 smithi136 bash[76371]: cluster 2024-09-13T00:42:12.880514+0000 mon.a (mon.0) 3806 : cluster [DBG] osdmap e2089: 8 total, 8 up, 8 in 2024-09-13T00:42:14.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:13 smithi181 bash[67587]: cluster 2024-09-13T00:42:12.499371+0000 mgr.y (mgr.24661) 1952 : cluster [DBG] pgmap v3601: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 86 KiB/s rd, 8.0 KiB/s wr, 114 op/s 2024-09-13T00:42:14.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:13 smithi181 bash[67587]: cluster 2024-09-13T00:42:12.880514+0000 mon.a (mon.0) 3806 : cluster [DBG] osdmap e2089: 8 total, 8 up, 8 in 2024-09-13T00:42:15.018 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_snap_get_id PASSED [ 46%] 2024-09-13T00:42:15.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:14 smithi136 bash[75164]: cluster 2024-09-13T00:42:13.881470+0000 mon.a (mon.0) 3807 : cluster [DBG] osdmap e2090: 8 total, 8 up, 8 in 2024-09-13T00:42:15.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:14 smithi136 bash[75164]: cluster 2024-09-13T00:42:14.807680+0000 mon.a (mon.0) 3808 : cluster [DBG] osdmap e2091: 8 total, 8 up, 8 in 2024-09-13T00:42:15.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:14 smithi136 bash[76371]: cluster 2024-09-13T00:42:13.881470+0000 mon.a (mon.0) 3807 : cluster [DBG] osdmap e2090: 8 total, 8 up, 8 in 2024-09-13T00:42:15.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:14 smithi136 bash[76371]: cluster 2024-09-13T00:42:14.807680+0000 mon.a (mon.0) 3808 : cluster [DBG] osdmap e2091: 8 total, 8 up, 8 in 2024-09-13T00:42:15.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:14 smithi181 bash[67587]: cluster 2024-09-13T00:42:13.881470+0000 mon.a (mon.0) 3807 : cluster [DBG] osdmap e2090: 8 total, 8 up, 8 in 2024-09-13T00:42:15.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:14 smithi181 bash[67587]: cluster 2024-09-13T00:42:14.807680+0000 mon.a (mon.0) 3808 : cluster [DBG] osdmap e2091: 8 total, 8 up, 8 in 2024-09-13T00:42:16.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:15 smithi136 bash[75164]: cluster 2024-09-13T00:42:14.500050+0000 mgr.y (mgr.24661) 1953 : cluster [DBG] pgmap v3604: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 90 KiB/s rd, 8.0 KiB/s wr, 121 op/s 2024-09-13T00:42:16.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:15 smithi136 bash[76371]: cluster 2024-09-13T00:42:14.500050+0000 mgr.y (mgr.24661) 1953 : cluster [DBG] pgmap v3604: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 90 KiB/s rd, 8.0 KiB/s wr, 121 op/s 2024-09-13T00:42:16.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:15 smithi181 bash[67587]: cluster 2024-09-13T00:42:14.500050+0000 mgr.y (mgr.24661) 1953 : cluster [DBG] pgmap v3604: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 90 KiB/s rd, 8.0 KiB/s wr, 121 op/s 2024-09-13T00:42:17.137 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_set_snap_sparse PASSED [ 47%] 2024-09-13T00:42:17.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:16 smithi136 bash[75164]: cluster 2024-09-13T00:42:15.916994+0000 mon.a (mon.0) 3809 : cluster [DBG] osdmap e2092: 8 total, 8 up, 8 in 2024-09-13T00:42:17.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:16 smithi136 bash[76371]: cluster 2024-09-13T00:42:15.916994+0000 mon.a (mon.0) 3809 : cluster [DBG] osdmap e2092: 8 total, 8 up, 8 in 2024-09-13T00:42:17.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:16 smithi181 bash[67587]: cluster 2024-09-13T00:42:15.916994+0000 mon.a (mon.0) 3809 : cluster [DBG] osdmap e2092: 8 total, 8 up, 8 in 2024-09-13T00:42:18.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:17 smithi136 bash[75164]: cluster 2024-09-13T00:42:16.500905+0000 mgr.y (mgr.24661) 1954 : cluster [DBG] pgmap v3607: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 2.2 KiB/s wr, 21 op/s 2024-09-13T00:42:18.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:17 smithi136 bash[75164]: cluster 2024-09-13T00:42:16.927558+0000 mon.a (mon.0) 3810 : cluster [DBG] osdmap e2093: 8 total, 8 up, 8 in 2024-09-13T00:42:18.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:17 smithi136 bash[76371]: cluster 2024-09-13T00:42:16.500905+0000 mgr.y (mgr.24661) 1954 : cluster [DBG] pgmap v3607: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 2.2 KiB/s wr, 21 op/s 2024-09-13T00:42:18.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:17 smithi136 bash[76371]: cluster 2024-09-13T00:42:16.927558+0000 mon.a (mon.0) 3810 : cluster [DBG] osdmap e2093: 8 total, 8 up, 8 in 2024-09-13T00:42:18.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:17 smithi181 bash[67587]: cluster 2024-09-13T00:42:16.500905+0000 mgr.y (mgr.24661) 1954 : cluster [DBG] pgmap v3607: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 2.2 KiB/s wr, 21 op/s 2024-09-13T00:42:18.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:17 smithi181 bash[67587]: cluster 2024-09-13T00:42:16.927558+0000 mon.a (mon.0) 3810 : cluster [DBG] osdmap e2093: 8 total, 8 up, 8 in 2024-09-13T00:42:19.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:18 smithi136 bash[75164]: cluster 2024-09-13T00:42:17.938569+0000 mon.a (mon.0) 3811 : cluster [DBG] osdmap e2094: 8 total, 8 up, 8 in 2024-09-13T00:42:19.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:18 smithi136 bash[76371]: cluster 2024-09-13T00:42:17.938569+0000 mon.a (mon.0) 3811 : cluster [DBG] osdmap e2094: 8 total, 8 up, 8 in 2024-09-13T00:42:19.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:18 smithi181 bash[67587]: cluster 2024-09-13T00:42:17.938569+0000 mon.a (mon.0) 3811 : cluster [DBG] osdmap e2094: 8 total, 8 up, 8 in 2024-09-13T00:42:19.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:42:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:42:19] "GET /metrics HTTP/1.1" 200 36741 "" "Prometheus/2.43.0" 2024-09-13T00:42:20.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:19 smithi136 bash[75164]: cluster 2024-09-13T00:42:18.501565+0000 mgr.y (mgr.24661) 1955 : cluster [DBG] pgmap v3610: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 49 KiB/s rd, 4.5 KiB/s wr, 66 op/s 2024-09-13T00:42:20.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:19 smithi136 bash[75164]: cluster 2024-09-13T00:42:18.953194+0000 mon.a (mon.0) 3812 : cluster [DBG] osdmap e2095: 8 total, 8 up, 8 in 2024-09-13T00:42:20.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:19 smithi136 bash[75164]: cluster 2024-09-13T00:42:19.810700+0000 mon.a (mon.0) 3813 : cluster [DBG] osdmap e2096: 8 total, 8 up, 8 in 2024-09-13T00:42:20.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:19 smithi136 bash[76371]: cluster 2024-09-13T00:42:18.501565+0000 mgr.y (mgr.24661) 1955 : cluster [DBG] pgmap v3610: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 49 KiB/s rd, 4.5 KiB/s wr, 66 op/s 2024-09-13T00:42:20.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:19 smithi136 bash[76371]: cluster 2024-09-13T00:42:18.953194+0000 mon.a (mon.0) 3812 : cluster [DBG] osdmap e2095: 8 total, 8 up, 8 in 2024-09-13T00:42:20.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:19 smithi136 bash[76371]: cluster 2024-09-13T00:42:19.810700+0000 mon.a (mon.0) 3813 : cluster [DBG] osdmap e2096: 8 total, 8 up, 8 in 2024-09-13T00:42:20.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:19 smithi181 bash[67587]: cluster 2024-09-13T00:42:18.501565+0000 mgr.y (mgr.24661) 1955 : cluster [DBG] pgmap v3610: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 49 KiB/s rd, 4.5 KiB/s wr, 66 op/s 2024-09-13T00:42:20.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:19 smithi181 bash[67587]: cluster 2024-09-13T00:42:18.953194+0000 mon.a (mon.0) 3812 : cluster [DBG] osdmap e2095: 8 total, 8 up, 8 in 2024-09-13T00:42:20.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:19 smithi181 bash[67587]: cluster 2024-09-13T00:42:19.810700+0000 mon.a (mon.0) 3813 : cluster [DBG] osdmap e2096: 8 total, 8 up, 8 in 2024-09-13T00:42:22.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:21 smithi181 bash[67587]: cluster 2024-09-13T00:42:20.502258+0000 mgr.y (mgr.24661) 1956 : cluster [DBG] pgmap v3613: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 90 KiB/s rd, 7.0 KiB/s wr, 124 op/s 2024-09-13T00:42:22.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:21 smithi181 bash[67587]: cluster 2024-09-13T00:42:20.812730+0000 mon.a (mon.0) 3814 : cluster [DBG] osdmap e2097: 8 total, 8 up, 8 in 2024-09-13T00:42:22.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:21 smithi136 bash[75164]: cluster 2024-09-13T00:42:20.502258+0000 mgr.y (mgr.24661) 1956 : cluster [DBG] pgmap v3613: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 90 KiB/s rd, 7.0 KiB/s wr, 124 op/s 2024-09-13T00:42:22.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:21 smithi136 bash[75164]: cluster 2024-09-13T00:42:20.812730+0000 mon.a (mon.0) 3814 : cluster [DBG] osdmap e2097: 8 total, 8 up, 8 in 2024-09-13T00:42:22.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:21 smithi136 bash[76371]: cluster 2024-09-13T00:42:20.502258+0000 mgr.y (mgr.24661) 1956 : cluster [DBG] pgmap v3613: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 90 KiB/s rd, 7.0 KiB/s wr, 124 op/s 2024-09-13T00:42:22.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:21 smithi136 bash[76371]: cluster 2024-09-13T00:42:20.812730+0000 mon.a (mon.0) 3814 : cluster [DBG] osdmap e2097: 8 total, 8 up, 8 in 2024-09-13T00:42:22.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:42:22 smithi181 bash[95505]: ts=2024-09-13T00:42:22.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:42:23.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:22 smithi181 bash[67587]: cluster 2024-09-13T00:42:21.816773+0000 mon.a (mon.0) 3815 : cluster [DBG] osdmap e2098: 8 total, 8 up, 8 in 2024-09-13T00:42:23.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:22 smithi136 bash[76371]: cluster 2024-09-13T00:42:21.816773+0000 mon.a (mon.0) 3815 : cluster [DBG] osdmap e2098: 8 total, 8 up, 8 in 2024-09-13T00:42:23.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:22 smithi136 bash[75164]: cluster 2024-09-13T00:42:21.816773+0000 mon.a (mon.0) 3815 : cluster [DBG] osdmap e2098: 8 total, 8 up, 8 in 2024-09-13T00:42:24.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:23 smithi136 bash[76371]: cluster 2024-09-13T00:42:22.503105+0000 mgr.y (mgr.24661) 1957 : cluster [DBG] pgmap v3616: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 98 KiB/s rd, 8.0 KiB/s wr, 134 op/s 2024-09-13T00:42:24.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:23 smithi136 bash[76371]: cluster 2024-09-13T00:42:22.845175+0000 mon.a (mon.0) 3816 : cluster [DBG] osdmap e2099: 8 total, 8 up, 8 in 2024-09-13T00:42:24.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:23 smithi136 bash[75164]: cluster 2024-09-13T00:42:22.503105+0000 mgr.y (mgr.24661) 1957 : cluster [DBG] pgmap v3616: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 98 KiB/s rd, 8.0 KiB/s wr, 134 op/s 2024-09-13T00:42:24.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:23 smithi136 bash[75164]: cluster 2024-09-13T00:42:22.845175+0000 mon.a (mon.0) 3816 : cluster [DBG] osdmap e2099: 8 total, 8 up, 8 in 2024-09-13T00:42:24.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:23 smithi181 bash[67587]: cluster 2024-09-13T00:42:22.503105+0000 mgr.y (mgr.24661) 1957 : cluster [DBG] pgmap v3616: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 98 KiB/s rd, 8.0 KiB/s wr, 134 op/s 2024-09-13T00:42:24.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:23 smithi181 bash[67587]: cluster 2024-09-13T00:42:22.845175+0000 mon.a (mon.0) 3816 : cluster [DBG] osdmap e2099: 8 total, 8 up, 8 in 2024-09-13T00:42:25.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:24 smithi136 bash[76371]: cluster 2024-09-13T00:42:23.844238+0000 mon.a (mon.0) 3817 : cluster [DBG] osdmap e2100: 8 total, 8 up, 8 in 2024-09-13T00:42:25.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:24 smithi136 bash[76371]: audit 2024-09-13T00:42:23.846152+0000 mon.a (mon.0) 3818 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:42:25.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:24 smithi136 bash[76371]: cluster 2024-09-13T00:42:24.812663+0000 mon.a (mon.0) 3819 : cluster [DBG] osdmap e2101: 8 total, 8 up, 8 in 2024-09-13T00:42:25.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:24 smithi136 bash[75164]: cluster 2024-09-13T00:42:23.844238+0000 mon.a (mon.0) 3817 : cluster [DBG] osdmap e2100: 8 total, 8 up, 8 in 2024-09-13T00:42:25.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:24 smithi136 bash[75164]: audit 2024-09-13T00:42:23.846152+0000 mon.a (mon.0) 3818 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:42:25.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:24 smithi136 bash[75164]: cluster 2024-09-13T00:42:24.812663+0000 mon.a (mon.0) 3819 : cluster [DBG] osdmap e2101: 8 total, 8 up, 8 in 2024-09-13T00:42:25.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:24 smithi181 bash[67587]: cluster 2024-09-13T00:42:23.844238+0000 mon.a (mon.0) 3817 : cluster [DBG] osdmap e2100: 8 total, 8 up, 8 in 2024-09-13T00:42:25.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:24 smithi181 bash[67587]: audit 2024-09-13T00:42:23.846152+0000 mon.a (mon.0) 3818 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:42:25.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:24 smithi181 bash[67587]: cluster 2024-09-13T00:42:24.812663+0000 mon.a (mon.0) 3819 : cluster [DBG] osdmap e2101: 8 total, 8 up, 8 in 2024-09-13T00:42:26.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:25 smithi136 bash[76371]: cluster 2024-09-13T00:42:24.503793+0000 mgr.y (mgr.24661) 1958 : cluster [DBG] pgmap v3619: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 57 KiB/s rd, 4.5 KiB/s wr, 75 op/s 2024-09-13T00:42:26.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:25 smithi136 bash[75164]: cluster 2024-09-13T00:42:24.503793+0000 mgr.y (mgr.24661) 1958 : cluster [DBG] pgmap v3619: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 57 KiB/s rd, 4.5 KiB/s wr, 75 op/s 2024-09-13T00:42:26.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:25 smithi181 bash[67587]: cluster 2024-09-13T00:42:24.503793+0000 mgr.y (mgr.24661) 1958 : cluster [DBG] pgmap v3619: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 57 KiB/s rd, 4.5 KiB/s wr, 75 op/s 2024-09-13T00:42:27.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:26 smithi136 bash[75164]: cluster 2024-09-13T00:42:25.879817+0000 mon.a (mon.0) 3820 : cluster [DBG] osdmap e2102: 8 total, 8 up, 8 in 2024-09-13T00:42:27.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:26 smithi136 bash[76371]: cluster 2024-09-13T00:42:25.879817+0000 mon.a (mon.0) 3820 : cluster [DBG] osdmap e2102: 8 total, 8 up, 8 in 2024-09-13T00:42:27.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:26 smithi181 bash[67587]: cluster 2024-09-13T00:42:25.879817+0000 mon.a (mon.0) 3820 : cluster [DBG] osdmap e2102: 8 total, 8 up, 8 in 2024-09-13T00:42:28.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:27 smithi136 bash[75164]: cluster 2024-09-13T00:42:26.504801+0000 mgr.y (mgr.24661) 1959 : cluster [DBG] pgmap v3622: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 1.5 KiB/s wr, 7 op/s 2024-09-13T00:42:28.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:27 smithi136 bash[75164]: cluster 2024-09-13T00:42:26.890114+0000 mon.a (mon.0) 3821 : cluster [DBG] osdmap e2103: 8 total, 8 up, 8 in 2024-09-13T00:42:28.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:27 smithi136 bash[76371]: cluster 2024-09-13T00:42:26.504801+0000 mgr.y (mgr.24661) 1959 : cluster [DBG] pgmap v3622: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 1.5 KiB/s wr, 7 op/s 2024-09-13T00:42:28.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:27 smithi136 bash[76371]: cluster 2024-09-13T00:42:26.890114+0000 mon.a (mon.0) 3821 : cluster [DBG] osdmap e2103: 8 total, 8 up, 8 in 2024-09-13T00:42:28.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:27 smithi181 bash[67587]: cluster 2024-09-13T00:42:26.504801+0000 mgr.y (mgr.24661) 1959 : cluster [DBG] pgmap v3622: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 1.5 KiB/s wr, 7 op/s 2024-09-13T00:42:28.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:27 smithi181 bash[67587]: cluster 2024-09-13T00:42:26.890114+0000 mon.a (mon.0) 3821 : cluster [DBG] osdmap e2103: 8 total, 8 up, 8 in 2024-09-13T00:42:29.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:28 smithi136 bash[75164]: cluster 2024-09-13T00:42:27.890867+0000 mon.a (mon.0) 3822 : cluster [DBG] osdmap e2104: 8 total, 8 up, 8 in 2024-09-13T00:42:29.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:28 smithi136 bash[76371]: cluster 2024-09-13T00:42:27.890867+0000 mon.a (mon.0) 3822 : cluster [DBG] osdmap e2104: 8 total, 8 up, 8 in 2024-09-13T00:42:29.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:28 smithi181 bash[67587]: cluster 2024-09-13T00:42:27.890867+0000 mon.a (mon.0) 3822 : cluster [DBG] osdmap e2104: 8 total, 8 up, 8 in 2024-09-13T00:42:29.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:42:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:42:29] "GET /metrics HTTP/1.1" 200 36740 "" "Prometheus/2.43.0" 2024-09-13T00:42:30.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:29 smithi136 bash[76371]: cluster 2024-09-13T00:42:28.505451+0000 mgr.y (mgr.24661) 1960 : cluster [DBG] pgmap v3625: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 76 KiB/s rd, 3.5 KiB/s wr, 102 op/s 2024-09-13T00:42:30.262 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:29 smithi136 bash[76371]: cluster 2024-09-13T00:42:28.912556+0000 mon.a (mon.0) 3823 : cluster [DBG] osdmap e2105: 8 total, 8 up, 8 in 2024-09-13T00:42:30.262 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:29 smithi136 bash[75164]: cluster 2024-09-13T00:42:28.505451+0000 mgr.y (mgr.24661) 1960 : cluster [DBG] pgmap v3625: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 76 KiB/s rd, 3.5 KiB/s wr, 102 op/s 2024-09-13T00:42:30.262 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:29 smithi136 bash[75164]: cluster 2024-09-13T00:42:28.912556+0000 mon.a (mon.0) 3823 : cluster [DBG] osdmap e2105: 8 total, 8 up, 8 in 2024-09-13T00:42:30.359 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:29 smithi181 bash[67587]: cluster 2024-09-13T00:42:28.505451+0000 mgr.y (mgr.24661) 1960 : cluster [DBG] pgmap v3625: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 76 KiB/s rd, 3.5 KiB/s wr, 102 op/s 2024-09-13T00:42:30.359 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:29 smithi181 bash[67587]: cluster 2024-09-13T00:42:28.912556+0000 mon.a (mon.0) 3823 : cluster [DBG] osdmap e2105: 8 total, 8 up, 8 in 2024-09-13T00:42:31.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:30 smithi136 bash[75164]: cluster 2024-09-13T00:42:29.913411+0000 mon.a (mon.0) 3824 : cluster [DBG] osdmap e2106: 8 total, 8 up, 8 in 2024-09-13T00:42:31.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:30 smithi136 bash[76371]: cluster 2024-09-13T00:42:29.913411+0000 mon.a (mon.0) 3824 : cluster [DBG] osdmap e2106: 8 total, 8 up, 8 in 2024-09-13T00:42:31.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:30 smithi181 bash[67587]: cluster 2024-09-13T00:42:29.913411+0000 mon.a (mon.0) 3824 : cluster [DBG] osdmap e2106: 8 total, 8 up, 8 in 2024-09-13T00:42:32.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:31 smithi136 bash[75164]: cluster 2024-09-13T00:42:30.506157+0000 mgr.y (mgr.24661) 1961 : cluster [DBG] pgmap v3628: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 80 KiB/s rd, 3.0 KiB/s wr, 106 op/s 2024-09-13T00:42:32.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:31 smithi136 bash[75164]: cluster 2024-09-13T00:42:30.939653+0000 mon.a (mon.0) 3825 : cluster [DBG] osdmap e2107: 8 total, 8 up, 8 in 2024-09-13T00:42:32.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:31 smithi136 bash[76371]: cluster 2024-09-13T00:42:30.506157+0000 mgr.y (mgr.24661) 1961 : cluster [DBG] pgmap v3628: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 80 KiB/s rd, 3.0 KiB/s wr, 106 op/s 2024-09-13T00:42:32.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:31 smithi136 bash[76371]: cluster 2024-09-13T00:42:30.939653+0000 mon.a (mon.0) 3825 : cluster [DBG] osdmap e2107: 8 total, 8 up, 8 in 2024-09-13T00:42:32.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:31 smithi181 bash[67587]: cluster 2024-09-13T00:42:30.506157+0000 mgr.y (mgr.24661) 1961 : cluster [DBG] pgmap v3628: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 80 KiB/s rd, 3.0 KiB/s wr, 106 op/s 2024-09-13T00:42:32.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:31 smithi181 bash[67587]: cluster 2024-09-13T00:42:30.939653+0000 mon.a (mon.0) 3825 : cluster [DBG] osdmap e2107: 8 total, 8 up, 8 in 2024-09-13T00:42:32.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:42:32 smithi181 bash[95505]: ts=2024-09-13T00:42:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:42:33.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:32 smithi136 bash[75164]: cluster 2024-09-13T00:42:31.939904+0000 mon.a (mon.0) 3826 : cluster [DBG] osdmap e2108: 8 total, 8 up, 8 in 2024-09-13T00:42:33.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:32 smithi136 bash[76371]: cluster 2024-09-13T00:42:31.939904+0000 mon.a (mon.0) 3826 : cluster [DBG] osdmap e2108: 8 total, 8 up, 8 in 2024-09-13T00:42:33.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:32 smithi181 bash[67587]: cluster 2024-09-13T00:42:31.939904+0000 mon.a (mon.0) 3826 : cluster [DBG] osdmap e2108: 8 total, 8 up, 8 in 2024-09-13T00:42:34.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:33 smithi136 bash[75164]: cluster 2024-09-13T00:42:32.507006+0000 mgr.y (mgr.24661) 1962 : cluster [DBG] pgmap v3631: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 87 KiB/s rd, 3.0 KiB/s wr, 117 op/s 2024-09-13T00:42:34.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:33 smithi136 bash[75164]: cluster 2024-09-13T00:42:32.960275+0000 mon.a (mon.0) 3827 : cluster [DBG] osdmap e2109: 8 total, 8 up, 8 in 2024-09-13T00:42:34.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:33 smithi136 bash[76371]: cluster 2024-09-13T00:42:32.507006+0000 mgr.y (mgr.24661) 1962 : cluster [DBG] pgmap v3631: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 87 KiB/s rd, 3.0 KiB/s wr, 117 op/s 2024-09-13T00:42:34.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:33 smithi136 bash[76371]: cluster 2024-09-13T00:42:32.960275+0000 mon.a (mon.0) 3827 : cluster [DBG] osdmap e2109: 8 total, 8 up, 8 in 2024-09-13T00:42:34.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:33 smithi181 bash[67587]: cluster 2024-09-13T00:42:32.507006+0000 mgr.y (mgr.24661) 1962 : cluster [DBG] pgmap v3631: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 87 KiB/s rd, 3.0 KiB/s wr, 117 op/s 2024-09-13T00:42:34.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:33 smithi181 bash[67587]: cluster 2024-09-13T00:42:32.960275+0000 mon.a (mon.0) 3827 : cluster [DBG] osdmap e2109: 8 total, 8 up, 8 in 2024-09-13T00:42:35.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:34 smithi136 bash[75164]: cluster 2024-09-13T00:42:33.961580+0000 mon.a (mon.0) 3828 : cluster [DBG] osdmap e2110: 8 total, 8 up, 8 in 2024-09-13T00:42:35.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:34 smithi136 bash[76371]: cluster 2024-09-13T00:42:33.961580+0000 mon.a (mon.0) 3828 : cluster [DBG] osdmap e2110: 8 total, 8 up, 8 in 2024-09-13T00:42:35.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:34 smithi181 bash[67587]: cluster 2024-09-13T00:42:33.961580+0000 mon.a (mon.0) 3828 : cluster [DBG] osdmap e2110: 8 total, 8 up, 8 in 2024-09-13T00:42:36.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:35 smithi136 bash[76371]: cluster 2024-09-13T00:42:34.507694+0000 mgr.y (mgr.24661) 1963 : cluster [DBG] pgmap v3634: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 89 KiB/s rd, 3.0 KiB/s wr, 118 op/s 2024-09-13T00:42:36.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:35 smithi136 bash[76371]: cluster 2024-09-13T00:42:34.984051+0000 mon.a (mon.0) 3829 : cluster [DBG] osdmap e2111: 8 total, 8 up, 8 in 2024-09-13T00:42:36.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:35 smithi136 bash[75164]: cluster 2024-09-13T00:42:34.507694+0000 mgr.y (mgr.24661) 1963 : cluster [DBG] pgmap v3634: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 89 KiB/s rd, 3.0 KiB/s wr, 118 op/s 2024-09-13T00:42:36.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:35 smithi136 bash[75164]: cluster 2024-09-13T00:42:34.984051+0000 mon.a (mon.0) 3829 : cluster [DBG] osdmap e2111: 8 total, 8 up, 8 in 2024-09-13T00:42:36.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:35 smithi181 bash[67587]: cluster 2024-09-13T00:42:34.507694+0000 mgr.y (mgr.24661) 1963 : cluster [DBG] pgmap v3634: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 89 KiB/s rd, 3.0 KiB/s wr, 118 op/s 2024-09-13T00:42:36.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:35 smithi181 bash[67587]: cluster 2024-09-13T00:42:34.984051+0000 mon.a (mon.0) 3829 : cluster [DBG] osdmap e2111: 8 total, 8 up, 8 in 2024-09-13T00:42:37.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:37 smithi136 bash[75164]: cluster 2024-09-13T00:42:35.986814+0000 mon.a (mon.0) 3830 : cluster [DBG] osdmap e2112: 8 total, 8 up, 8 in 2024-09-13T00:42:37.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:37 smithi136 bash[76371]: cluster 2024-09-13T00:42:35.986814+0000 mon.a (mon.0) 3830 : cluster [DBG] osdmap e2112: 8 total, 8 up, 8 in 2024-09-13T00:42:37.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:37 smithi181 bash[67587]: cluster 2024-09-13T00:42:35.986814+0000 mon.a (mon.0) 3830 : cluster [DBG] osdmap e2112: 8 total, 8 up, 8 in 2024-09-13T00:42:38.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:38 smithi136 bash[75164]: cluster 2024-09-13T00:42:36.508558+0000 mgr.y (mgr.24661) 1964 : cluster [DBG] pgmap v3637: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 1.2 KiB/s wr, 7 op/s 2024-09-13T00:42:38.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:38 smithi136 bash[75164]: cluster 2024-09-13T00:42:37.007202+0000 mon.a (mon.0) 3831 : cluster [DBG] osdmap e2113: 8 total, 8 up, 8 in 2024-09-13T00:42:38.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:38 smithi136 bash[76371]: cluster 2024-09-13T00:42:36.508558+0000 mgr.y (mgr.24661) 1964 : cluster [DBG] pgmap v3637: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 1.2 KiB/s wr, 7 op/s 2024-09-13T00:42:38.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:38 smithi136 bash[76371]: cluster 2024-09-13T00:42:37.007202+0000 mon.a (mon.0) 3831 : cluster [DBG] osdmap e2113: 8 total, 8 up, 8 in 2024-09-13T00:42:38.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:38 smithi181 bash[67587]: cluster 2024-09-13T00:42:36.508558+0000 mgr.y (mgr.24661) 1964 : cluster [DBG] pgmap v3637: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 1.2 KiB/s wr, 7 op/s 2024-09-13T00:42:38.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:38 smithi181 bash[67587]: cluster 2024-09-13T00:42:37.007202+0000 mon.a (mon.0) 3831 : cluster [DBG] osdmap e2113: 8 total, 8 up, 8 in 2024-09-13T00:42:39.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:39 smithi181 bash[67587]: cluster 2024-09-13T00:42:38.017360+0000 mon.a (mon.0) 3832 : cluster [DBG] osdmap e2114: 8 total, 8 up, 8 in 2024-09-13T00:42:39.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:39 smithi181 bash[67587]: audit 2024-09-13T00:42:38.841966+0000 mon.a (mon.0) 3833 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:42:39.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:39 smithi136 bash[75164]: cluster 2024-09-13T00:42:38.017360+0000 mon.a (mon.0) 3832 : cluster [DBG] osdmap e2114: 8 total, 8 up, 8 in 2024-09-13T00:42:39.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:39 smithi136 bash[75164]: audit 2024-09-13T00:42:38.841966+0000 mon.a (mon.0) 3833 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:42:39.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:39 smithi136 bash[76371]: cluster 2024-09-13T00:42:38.017360+0000 mon.a (mon.0) 3832 : cluster [DBG] osdmap e2114: 8 total, 8 up, 8 in 2024-09-13T00:42:39.414 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:39 smithi136 bash[76371]: audit 2024-09-13T00:42:38.841966+0000 mon.a (mon.0) 3833 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:42:39.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:42:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:42:39] "GET /metrics HTTP/1.1" 200 36742 "" "Prometheus/2.43.0" 2024-09-13T00:42:40.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:40 smithi181 bash[67587]: cluster 2024-09-13T00:42:38.509254+0000 mgr.y (mgr.24661) 1965 : cluster [DBG] pgmap v3640: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 135 KiB/s rd, 3.7 KiB/s wr, 181 op/s 2024-09-13T00:42:40.360 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:40 smithi181 bash[67587]: cluster 2024-09-13T00:42:39.026994+0000 mon.a (mon.0) 3834 : cluster [DBG] osdmap e2115: 8 total, 8 up, 8 in 2024-09-13T00:42:40.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:40 smithi136 bash[75164]: cluster 2024-09-13T00:42:38.509254+0000 mgr.y (mgr.24661) 1965 : cluster [DBG] pgmap v3640: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 135 KiB/s rd, 3.7 KiB/s wr, 181 op/s 2024-09-13T00:42:40.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:40 smithi136 bash[75164]: cluster 2024-09-13T00:42:39.026994+0000 mon.a (mon.0) 3834 : cluster [DBG] osdmap e2115: 8 total, 8 up, 8 in 2024-09-13T00:42:40.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:40 smithi136 bash[76371]: cluster 2024-09-13T00:42:38.509254+0000 mgr.y (mgr.24661) 1965 : cluster [DBG] pgmap v3640: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 135 KiB/s rd, 3.7 KiB/s wr, 181 op/s 2024-09-13T00:42:40.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:40 smithi136 bash[76371]: cluster 2024-09-13T00:42:39.026994+0000 mon.a (mon.0) 3834 : cluster [DBG] osdmap e2115: 8 total, 8 up, 8 in 2024-09-13T00:42:41.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:41 smithi181 bash[67587]: cluster 2024-09-13T00:42:40.034916+0000 mon.a (mon.0) 3835 : cluster [DBG] osdmap e2116: 8 total, 8 up, 8 in 2024-09-13T00:42:41.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:41 smithi136 bash[75164]: cluster 2024-09-13T00:42:40.034916+0000 mon.a (mon.0) 3835 : cluster [DBG] osdmap e2116: 8 total, 8 up, 8 in 2024-09-13T00:42:41.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:41 smithi136 bash[76371]: cluster 2024-09-13T00:42:40.034916+0000 mon.a (mon.0) 3835 : cluster [DBG] osdmap e2116: 8 total, 8 up, 8 in 2024-09-13T00:42:42.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:42 smithi181 bash[67587]: cluster 2024-09-13T00:42:40.510049+0000 mgr.y (mgr.24661) 1966 : cluster [DBG] pgmap v3643: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 139 KiB/s rd, 3.0 KiB/s wr, 185 op/s 2024-09-13T00:42:42.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:42 smithi181 bash[67587]: cluster 2024-09-13T00:42:41.047427+0000 mon.a (mon.0) 3836 : cluster [DBG] osdmap e2117: 8 total, 8 up, 8 in 2024-09-13T00:42:42.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:42 smithi136 bash[75164]: cluster 2024-09-13T00:42:40.510049+0000 mgr.y (mgr.24661) 1966 : cluster [DBG] pgmap v3643: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 139 KiB/s rd, 3.0 KiB/s wr, 185 op/s 2024-09-13T00:42:42.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:42 smithi136 bash[75164]: cluster 2024-09-13T00:42:41.047427+0000 mon.a (mon.0) 3836 : cluster [DBG] osdmap e2117: 8 total, 8 up, 8 in 2024-09-13T00:42:42.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:42 smithi136 bash[76371]: cluster 2024-09-13T00:42:40.510049+0000 mgr.y (mgr.24661) 1966 : cluster [DBG] pgmap v3643: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 139 KiB/s rd, 3.0 KiB/s wr, 185 op/s 2024-09-13T00:42:42.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:42 smithi136 bash[76371]: cluster 2024-09-13T00:42:41.047427+0000 mon.a (mon.0) 3836 : cluster [DBG] osdmap e2117: 8 total, 8 up, 8 in 2024-09-13T00:42:42.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:42:42 smithi181 bash[95505]: ts=2024-09-13T00:42:42.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:42:43.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:43 smithi181 bash[67587]: cluster 2024-09-13T00:42:42.055090+0000 mon.a (mon.0) 3837 : cluster [DBG] osdmap e2118: 8 total, 8 up, 8 in 2024-09-13T00:42:43.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:43 smithi136 bash[75164]: cluster 2024-09-13T00:42:42.055090+0000 mon.a (mon.0) 3837 : cluster [DBG] osdmap e2118: 8 total, 8 up, 8 in 2024-09-13T00:42:43.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:43 smithi136 bash[76371]: cluster 2024-09-13T00:42:42.055090+0000 mon.a (mon.0) 3837 : cluster [DBG] osdmap e2118: 8 total, 8 up, 8 in 2024-09-13T00:42:44.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:44 smithi181 bash[67587]: cluster 2024-09-13T00:42:42.511021+0000 mgr.y (mgr.24661) 1967 : cluster [DBG] pgmap v3646: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 169 KiB/s rd, 4.0 KiB/s wr, 228 op/s 2024-09-13T00:42:44.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:44 smithi181 bash[67587]: cluster 2024-09-13T00:42:43.066190+0000 mon.a (mon.0) 3838 : cluster [DBG] osdmap e2119: 8 total, 8 up, 8 in 2024-09-13T00:42:44.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:44 smithi136 bash[75164]: cluster 2024-09-13T00:42:42.511021+0000 mgr.y (mgr.24661) 1967 : cluster [DBG] pgmap v3646: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 169 KiB/s rd, 4.0 KiB/s wr, 228 op/s 2024-09-13T00:42:44.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:44 smithi136 bash[75164]: cluster 2024-09-13T00:42:43.066190+0000 mon.a (mon.0) 3838 : cluster [DBG] osdmap e2119: 8 total, 8 up, 8 in 2024-09-13T00:42:44.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:44 smithi136 bash[76371]: cluster 2024-09-13T00:42:42.511021+0000 mgr.y (mgr.24661) 1967 : cluster [DBG] pgmap v3646: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 169 KiB/s rd, 4.0 KiB/s wr, 228 op/s 2024-09-13T00:42:44.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:44 smithi136 bash[76371]: cluster 2024-09-13T00:42:43.066190+0000 mon.a (mon.0) 3838 : cluster [DBG] osdmap e2119: 8 total, 8 up, 8 in 2024-09-13T00:42:45.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:45 smithi181 bash[67587]: cluster 2024-09-13T00:42:44.066816+0000 mon.a (mon.0) 3839 : cluster [DBG] osdmap e2120: 8 total, 8 up, 8 in 2024-09-13T00:42:45.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:45 smithi136 bash[75164]: cluster 2024-09-13T00:42:44.066816+0000 mon.a (mon.0) 3839 : cluster [DBG] osdmap e2120: 8 total, 8 up, 8 in 2024-09-13T00:42:45.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:45 smithi136 bash[76371]: cluster 2024-09-13T00:42:44.066816+0000 mon.a (mon.0) 3839 : cluster [DBG] osdmap e2120: 8 total, 8 up, 8 in 2024-09-13T00:42:46.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:46 smithi181 bash[67587]: cluster 2024-09-13T00:42:44.511711+0000 mgr.y (mgr.24661) 1968 : cluster [DBG] pgmap v3649: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 171 KiB/s rd, 4.0 KiB/s wr, 230 op/s 2024-09-13T00:42:46.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:46 smithi181 bash[67587]: cluster 2024-09-13T00:42:45.088446+0000 mon.a (mon.0) 3840 : cluster [DBG] osdmap e2121: 8 total, 8 up, 8 in 2024-09-13T00:42:46.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:46 smithi136 bash[76371]: cluster 2024-09-13T00:42:44.511711+0000 mgr.y (mgr.24661) 1968 : cluster [DBG] pgmap v3649: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 171 KiB/s rd, 4.0 KiB/s wr, 230 op/s 2024-09-13T00:42:46.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:46 smithi136 bash[76371]: cluster 2024-09-13T00:42:45.088446+0000 mon.a (mon.0) 3840 : cluster [DBG] osdmap e2121: 8 total, 8 up, 8 in 2024-09-13T00:42:46.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:46 smithi136 bash[75164]: cluster 2024-09-13T00:42:44.511711+0000 mgr.y (mgr.24661) 1968 : cluster [DBG] pgmap v3649: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 171 KiB/s rd, 4.0 KiB/s wr, 230 op/s 2024-09-13T00:42:46.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:46 smithi136 bash[75164]: cluster 2024-09-13T00:42:45.088446+0000 mon.a (mon.0) 3840 : cluster [DBG] osdmap e2121: 8 total, 8 up, 8 in 2024-09-13T00:42:47.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:47 smithi136 bash[75164]: cluster 2024-09-13T00:42:46.098217+0000 mon.a (mon.0) 3841 : cluster [DBG] osdmap e2122: 8 total, 8 up, 8 in 2024-09-13T00:42:47.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:47 smithi136 bash[76371]: cluster 2024-09-13T00:42:46.098217+0000 mon.a (mon.0) 3841 : cluster [DBG] osdmap e2122: 8 total, 8 up, 8 in 2024-09-13T00:42:47.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:47 smithi181 bash[67587]: cluster 2024-09-13T00:42:46.098217+0000 mon.a (mon.0) 3841 : cluster [DBG] osdmap e2122: 8 total, 8 up, 8 in 2024-09-13T00:42:48.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:48 smithi136 bash[75164]: cluster 2024-09-13T00:42:46.512555+0000 mgr.y (mgr.24661) 1969 : cluster [DBG] pgmap v3652: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 767 B/s wr, 6 op/s 2024-09-13T00:42:48.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:48 smithi136 bash[75164]: cluster 2024-09-13T00:42:47.111670+0000 mon.a (mon.0) 3842 : cluster [DBG] osdmap e2123: 8 total, 8 up, 8 in 2024-09-13T00:42:48.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:48 smithi136 bash[76371]: cluster 2024-09-13T00:42:46.512555+0000 mgr.y (mgr.24661) 1969 : cluster [DBG] pgmap v3652: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 767 B/s wr, 6 op/s 2024-09-13T00:42:48.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:48 smithi136 bash[76371]: cluster 2024-09-13T00:42:47.111670+0000 mon.a (mon.0) 3842 : cluster [DBG] osdmap e2123: 8 total, 8 up, 8 in 2024-09-13T00:42:48.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:48 smithi181 bash[67587]: cluster 2024-09-13T00:42:46.512555+0000 mgr.y (mgr.24661) 1969 : cluster [DBG] pgmap v3652: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 767 B/s wr, 6 op/s 2024-09-13T00:42:48.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:48 smithi181 bash[67587]: cluster 2024-09-13T00:42:47.111670+0000 mon.a (mon.0) 3842 : cluster [DBG] osdmap e2123: 8 total, 8 up, 8 in 2024-09-13T00:42:49.414 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:49 smithi136 bash[75164]: cluster 2024-09-13T00:42:48.109350+0000 mon.a (mon.0) 3843 : cluster [DBG] osdmap e2124: 8 total, 8 up, 8 in 2024-09-13T00:42:49.414 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:49 smithi136 bash[76371]: cluster 2024-09-13T00:42:48.109350+0000 mon.a (mon.0) 3843 : cluster [DBG] osdmap e2124: 8 total, 8 up, 8 in 2024-09-13T00:42:49.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:49 smithi181 bash[67587]: cluster 2024-09-13T00:42:48.109350+0000 mon.a (mon.0) 3843 : cluster [DBG] osdmap e2124: 8 total, 8 up, 8 in 2024-09-13T00:42:49.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:42:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:42:49] "GET /metrics HTTP/1.1" 200 36742 "" "Prometheus/2.43.0" 2024-09-13T00:42:50.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:50 smithi136 bash[75164]: cluster 2024-09-13T00:42:48.513350+0000 mgr.y (mgr.24661) 1970 : cluster [DBG] pgmap v3655: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 200 KiB/s rd, 3.2 KiB/s wr, 268 op/s 2024-09-13T00:42:50.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:50 smithi136 bash[75164]: cluster 2024-09-13T00:42:49.132046+0000 mon.a (mon.0) 3844 : cluster [DBG] osdmap e2125: 8 total, 8 up, 8 in 2024-09-13T00:42:50.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:50 smithi136 bash[76371]: cluster 2024-09-13T00:42:48.513350+0000 mgr.y (mgr.24661) 1970 : cluster [DBG] pgmap v3655: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 200 KiB/s rd, 3.2 KiB/s wr, 268 op/s 2024-09-13T00:42:50.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:50 smithi136 bash[76371]: cluster 2024-09-13T00:42:49.132046+0000 mon.a (mon.0) 3844 : cluster [DBG] osdmap e2125: 8 total, 8 up, 8 in 2024-09-13T00:42:50.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:50 smithi181 bash[67587]: cluster 2024-09-13T00:42:48.513350+0000 mgr.y (mgr.24661) 1970 : cluster [DBG] pgmap v3655: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 200 KiB/s rd, 3.2 KiB/s wr, 268 op/s 2024-09-13T00:42:50.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:50 smithi181 bash[67587]: cluster 2024-09-13T00:42:49.132046+0000 mon.a (mon.0) 3844 : cluster [DBG] osdmap e2125: 8 total, 8 up, 8 in 2024-09-13T00:42:51.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:51 smithi136 bash[75164]: cluster 2024-09-13T00:42:50.133148+0000 mon.a (mon.0) 3845 : cluster [DBG] osdmap e2126: 8 total, 8 up, 8 in 2024-09-13T00:42:51.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:51 smithi136 bash[76371]: cluster 2024-09-13T00:42:50.133148+0000 mon.a (mon.0) 3845 : cluster [DBG] osdmap e2126: 8 total, 8 up, 8 in 2024-09-13T00:42:51.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:51 smithi181 bash[67587]: cluster 2024-09-13T00:42:50.133148+0000 mon.a (mon.0) 3845 : cluster [DBG] osdmap e2126: 8 total, 8 up, 8 in 2024-09-13T00:42:52.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:52 smithi136 bash[75164]: cluster 2024-09-13T00:42:50.514028+0000 mgr.y (mgr.24661) 1971 : cluster [DBG] pgmap v3658: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 204 KiB/s rd, 3.0 KiB/s wr, 272 op/s 2024-09-13T00:42:52.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:52 smithi136 bash[75164]: cluster 2024-09-13T00:42:51.157917+0000 mon.a (mon.0) 3846 : cluster [DBG] osdmap e2127: 8 total, 8 up, 8 in 2024-09-13T00:42:52.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:52 smithi136 bash[76371]: cluster 2024-09-13T00:42:50.514028+0000 mgr.y (mgr.24661) 1971 : cluster [DBG] pgmap v3658: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 204 KiB/s rd, 3.0 KiB/s wr, 272 op/s 2024-09-13T00:42:52.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:52 smithi136 bash[76371]: cluster 2024-09-13T00:42:51.157917+0000 mon.a (mon.0) 3846 : cluster [DBG] osdmap e2127: 8 total, 8 up, 8 in 2024-09-13T00:42:52.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:42:52 smithi181 bash[95505]: ts=2024-09-13T00:42:52.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:42:52.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:52 smithi181 bash[67587]: cluster 2024-09-13T00:42:50.514028+0000 mgr.y (mgr.24661) 1971 : cluster [DBG] pgmap v3658: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 204 KiB/s rd, 3.0 KiB/s wr, 272 op/s 2024-09-13T00:42:52.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:52 smithi181 bash[67587]: cluster 2024-09-13T00:42:51.157917+0000 mon.a (mon.0) 3846 : cluster [DBG] osdmap e2127: 8 total, 8 up, 8 in 2024-09-13T00:42:53.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:53 smithi136 bash[75164]: cluster 2024-09-13T00:42:52.169994+0000 mon.a (mon.0) 3847 : cluster [DBG] osdmap e2128: 8 total, 8 up, 8 in 2024-09-13T00:42:53.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:53 smithi136 bash[76371]: cluster 2024-09-13T00:42:52.169994+0000 mon.a (mon.0) 3847 : cluster [DBG] osdmap e2128: 8 total, 8 up, 8 in 2024-09-13T00:42:53.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:53 smithi181 bash[67587]: cluster 2024-09-13T00:42:52.169994+0000 mon.a (mon.0) 3847 : cluster [DBG] osdmap e2128: 8 total, 8 up, 8 in 2024-09-13T00:42:54.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:54 smithi136 bash[75164]: cluster 2024-09-13T00:42:52.514717+0000 mgr.y (mgr.24661) 1972 : cluster [DBG] pgmap v3661: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 234 KiB/s rd, 3.2 KiB/s wr, 315 op/s 2024-09-13T00:42:54.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:54 smithi136 bash[75164]: cluster 2024-09-13T00:42:53.182276+0000 mon.a (mon.0) 3848 : cluster [DBG] osdmap e2129: 8 total, 8 up, 8 in 2024-09-13T00:42:54.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:54 smithi136 bash[75164]: audit 2024-09-13T00:42:53.842612+0000 mon.a (mon.0) 3849 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:42:54.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:54 smithi136 bash[76371]: cluster 2024-09-13T00:42:52.514717+0000 mgr.y (mgr.24661) 1972 : cluster [DBG] pgmap v3661: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 234 KiB/s rd, 3.2 KiB/s wr, 315 op/s 2024-09-13T00:42:54.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:54 smithi136 bash[76371]: cluster 2024-09-13T00:42:53.182276+0000 mon.a (mon.0) 3848 : cluster [DBG] osdmap e2129: 8 total, 8 up, 8 in 2024-09-13T00:42:54.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:54 smithi136 bash[76371]: audit 2024-09-13T00:42:53.842612+0000 mon.a (mon.0) 3849 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:42:54.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:54 smithi181 bash[67587]: cluster 2024-09-13T00:42:52.514717+0000 mgr.y (mgr.24661) 1972 : cluster [DBG] pgmap v3661: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 234 KiB/s rd, 3.2 KiB/s wr, 315 op/s 2024-09-13T00:42:54.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:54 smithi181 bash[67587]: cluster 2024-09-13T00:42:53.182276+0000 mon.a (mon.0) 3848 : cluster [DBG] osdmap e2129: 8 total, 8 up, 8 in 2024-09-13T00:42:54.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:54 smithi181 bash[67587]: audit 2024-09-13T00:42:53.842612+0000 mon.a (mon.0) 3849 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:42:55.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:55 smithi136 bash[75164]: cluster 2024-09-13T00:42:54.182568+0000 mon.a (mon.0) 3850 : cluster [DBG] osdmap e2130: 8 total, 8 up, 8 in 2024-09-13T00:42:55.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:55 smithi136 bash[75164]: audit 2024-09-13T00:42:54.553331+0000 mon.a (mon.0) 3851 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:42:55.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:55 smithi136 bash[75164]: audit 2024-09-13T00:42:55.097555+0000 mon.a (mon.0) 3852 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:42:55.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:55 smithi136 bash[75164]: audit 2024-09-13T00:42:55.099347+0000 mon.a (mon.0) 3853 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:42:55.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:55 smithi136 bash[75164]: audit 2024-09-13T00:42:55.109316+0000 mon.a (mon.0) 3854 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:42:55.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:55 smithi136 bash[76371]: cluster 2024-09-13T00:42:54.182568+0000 mon.a (mon.0) 3850 : cluster [DBG] osdmap e2130: 8 total, 8 up, 8 in 2024-09-13T00:42:55.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:55 smithi136 bash[76371]: audit 2024-09-13T00:42:54.553331+0000 mon.a (mon.0) 3851 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:42:55.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:55 smithi136 bash[76371]: audit 2024-09-13T00:42:55.097555+0000 mon.a (mon.0) 3852 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:42:55.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:55 smithi136 bash[76371]: audit 2024-09-13T00:42:55.099347+0000 mon.a (mon.0) 3853 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:42:55.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:55 smithi136 bash[76371]: audit 2024-09-13T00:42:55.109316+0000 mon.a (mon.0) 3854 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:42:55.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:55 smithi181 bash[67587]: cluster 2024-09-13T00:42:54.182568+0000 mon.a (mon.0) 3850 : cluster [DBG] osdmap e2130: 8 total, 8 up, 8 in 2024-09-13T00:42:55.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:55 smithi181 bash[67587]: audit 2024-09-13T00:42:54.553331+0000 mon.a (mon.0) 3851 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:42:55.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:55 smithi181 bash[67587]: audit 2024-09-13T00:42:55.097555+0000 mon.a (mon.0) 3852 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:42:55.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:55 smithi181 bash[67587]: audit 2024-09-13T00:42:55.099347+0000 mon.a (mon.0) 3853 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:42:55.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:55 smithi181 bash[67587]: audit 2024-09-13T00:42:55.109316+0000 mon.a (mon.0) 3854 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:42:56.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:56 smithi136 bash[76371]: cluster 2024-09-13T00:42:54.515408+0000 mgr.y (mgr.24661) 1973 : cluster [DBG] pgmap v3664: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 235 KiB/s rd, 3.7 KiB/s wr, 317 op/s 2024-09-13T00:42:56.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:56 smithi136 bash[76371]: cluster 2024-09-13T00:42:55.205816+0000 mon.a (mon.0) 3855 : cluster [DBG] osdmap e2131: 8 total, 8 up, 8 in 2024-09-13T00:42:56.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:56 smithi136 bash[75164]: cluster 2024-09-13T00:42:54.515408+0000 mgr.y (mgr.24661) 1973 : cluster [DBG] pgmap v3664: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 235 KiB/s rd, 3.7 KiB/s wr, 317 op/s 2024-09-13T00:42:56.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:56 smithi136 bash[75164]: cluster 2024-09-13T00:42:55.205816+0000 mon.a (mon.0) 3855 : cluster [DBG] osdmap e2131: 8 total, 8 up, 8 in 2024-09-13T00:42:56.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:56 smithi181 bash[67587]: cluster 2024-09-13T00:42:54.515408+0000 mgr.y (mgr.24661) 1973 : cluster [DBG] pgmap v3664: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 235 KiB/s rd, 3.7 KiB/s wr, 317 op/s 2024-09-13T00:42:56.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:56 smithi181 bash[67587]: cluster 2024-09-13T00:42:55.205816+0000 mon.a (mon.0) 3855 : cluster [DBG] osdmap e2131: 8 total, 8 up, 8 in 2024-09-13T00:42:57.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:57 smithi136 bash[75164]: cluster 2024-09-13T00:42:56.207021+0000 mon.a (mon.0) 3856 : cluster [DBG] osdmap e2132: 8 total, 8 up, 8 in 2024-09-13T00:42:57.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:57 smithi136 bash[76371]: cluster 2024-09-13T00:42:56.207021+0000 mon.a (mon.0) 3856 : cluster [DBG] osdmap e2132: 8 total, 8 up, 8 in 2024-09-13T00:42:57.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:57 smithi181 bash[67587]: cluster 2024-09-13T00:42:56.207021+0000 mon.a (mon.0) 3856 : cluster [DBG] osdmap e2132: 8 total, 8 up, 8 in 2024-09-13T00:42:58.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:58 smithi136 bash[75164]: cluster 2024-09-13T00:42:56.516340+0000 mgr.y (mgr.24661) 1974 : cluster [DBG] pgmap v3667: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 1.2 KiB/s wr, 7 op/s 2024-09-13T00:42:58.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:58 smithi136 bash[75164]: cluster 2024-09-13T00:42:57.230687+0000 mon.a (mon.0) 3857 : cluster [DBG] osdmap e2133: 8 total, 8 up, 8 in 2024-09-13T00:42:58.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:58 smithi136 bash[76371]: cluster 2024-09-13T00:42:56.516340+0000 mgr.y (mgr.24661) 1974 : cluster [DBG] pgmap v3667: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 1.2 KiB/s wr, 7 op/s 2024-09-13T00:42:58.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:58 smithi136 bash[76371]: cluster 2024-09-13T00:42:57.230687+0000 mon.a (mon.0) 3857 : cluster [DBG] osdmap e2133: 8 total, 8 up, 8 in 2024-09-13T00:42:58.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:58 smithi181 bash[67587]: cluster 2024-09-13T00:42:56.516340+0000 mgr.y (mgr.24661) 1974 : cluster [DBG] pgmap v3667: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 1.2 KiB/s wr, 7 op/s 2024-09-13T00:42:58.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:58 smithi181 bash[67587]: cluster 2024-09-13T00:42:57.230687+0000 mon.a (mon.0) 3857 : cluster [DBG] osdmap e2133: 8 total, 8 up, 8 in 2024-09-13T00:42:59.509 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:42:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:42:59] "GET /metrics HTTP/1.1" 200 36744 "" "Prometheus/2.43.0" 2024-09-13T00:42:59.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:42:59 smithi136 bash[76371]: cluster 2024-09-13T00:42:58.231732+0000 mon.a (mon.0) 3858 : cluster [DBG] osdmap e2134: 8 total, 8 up, 8 in 2024-09-13T00:42:59.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:42:59 smithi136 bash[75164]: cluster 2024-09-13T00:42:58.231732+0000 mon.a (mon.0) 3858 : cluster [DBG] osdmap e2134: 8 total, 8 up, 8 in 2024-09-13T00:42:59.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:42:59 smithi181 bash[67587]: cluster 2024-09-13T00:42:58.231732+0000 mon.a (mon.0) 3858 : cluster [DBG] osdmap e2134: 8 total, 8 up, 8 in 2024-09-13T00:43:00.513 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:00 smithi136 bash[76371]: cluster 2024-09-13T00:42:58.517091+0000 mgr.y (mgr.24661) 1975 : cluster [DBG] pgmap v3670: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 266 KiB/s rd, 3.5 KiB/s wr, 355 op/s 2024-09-13T00:43:00.513 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:00 smithi136 bash[76371]: cluster 2024-09-13T00:42:59.255071+0000 mon.a (mon.0) 3859 : cluster [DBG] osdmap e2135: 8 total, 8 up, 8 in 2024-09-13T00:43:00.514 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:00 smithi136 bash[75164]: cluster 2024-09-13T00:42:58.517091+0000 mgr.y (mgr.24661) 1975 : cluster [DBG] pgmap v3670: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 266 KiB/s rd, 3.5 KiB/s wr, 355 op/s 2024-09-13T00:43:00.514 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:00 smithi136 bash[75164]: cluster 2024-09-13T00:42:59.255071+0000 mon.a (mon.0) 3859 : cluster [DBG] osdmap e2135: 8 total, 8 up, 8 in 2024-09-13T00:43:00.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:00 smithi181 bash[67587]: cluster 2024-09-13T00:42:58.517091+0000 mgr.y (mgr.24661) 1975 : cluster [DBG] pgmap v3670: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 266 KiB/s rd, 3.5 KiB/s wr, 355 op/s 2024-09-13T00:43:00.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:00 smithi181 bash[67587]: cluster 2024-09-13T00:42:59.255071+0000 mon.a (mon.0) 3859 : cluster [DBG] osdmap e2135: 8 total, 8 up, 8 in 2024-09-13T00:43:01.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:01 smithi181 bash[67587]: cluster 2024-09-13T00:43:00.262572+0000 mon.a (mon.0) 3860 : cluster [DBG] osdmap e2136: 8 total, 8 up, 8 in 2024-09-13T00:43:01.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:01 smithi136 bash[75164]: cluster 2024-09-13T00:43:00.262572+0000 mon.a (mon.0) 3860 : cluster [DBG] osdmap e2136: 8 total, 8 up, 8 in 2024-09-13T00:43:01.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:01 smithi136 bash[76371]: cluster 2024-09-13T00:43:00.262572+0000 mon.a (mon.0) 3860 : cluster [DBG] osdmap e2136: 8 total, 8 up, 8 in 2024-09-13T00:43:02.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:02 smithi181 bash[67587]: cluster 2024-09-13T00:43:00.517730+0000 mgr.y (mgr.24661) 1976 : cluster [DBG] pgmap v3673: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 270 KiB/s rd, 3.5 KiB/s wr, 360 op/s 2024-09-13T00:43:02.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:02 smithi181 bash[67587]: cluster 2024-09-13T00:43:01.278517+0000 mon.a (mon.0) 3861 : cluster [DBG] osdmap e2137: 8 total, 8 up, 8 in 2024-09-13T00:43:02.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:43:02 smithi181 bash[95505]: ts=2024-09-13T00:43:02.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:43:02.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:02 smithi136 bash[75164]: cluster 2024-09-13T00:43:00.517730+0000 mgr.y (mgr.24661) 1976 : cluster [DBG] pgmap v3673: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 270 KiB/s rd, 3.5 KiB/s wr, 360 op/s 2024-09-13T00:43:02.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:02 smithi136 bash[75164]: cluster 2024-09-13T00:43:01.278517+0000 mon.a (mon.0) 3861 : cluster [DBG] osdmap e2137: 8 total, 8 up, 8 in 2024-09-13T00:43:02.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:02 smithi136 bash[76371]: cluster 2024-09-13T00:43:00.517730+0000 mgr.y (mgr.24661) 1976 : cluster [DBG] pgmap v3673: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 270 KiB/s rd, 3.5 KiB/s wr, 360 op/s 2024-09-13T00:43:02.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:02 smithi136 bash[76371]: cluster 2024-09-13T00:43:01.278517+0000 mon.a (mon.0) 3861 : cluster [DBG] osdmap e2137: 8 total, 8 up, 8 in 2024-09-13T00:43:03.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:03 smithi181 bash[67587]: cluster 2024-09-13T00:43:02.283291+0000 mon.a (mon.0) 3862 : cluster [DBG] osdmap e2138: 8 total, 8 up, 8 in 2024-09-13T00:43:03.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:03 smithi136 bash[75164]: cluster 2024-09-13T00:43:02.283291+0000 mon.a (mon.0) 3862 : cluster [DBG] osdmap e2138: 8 total, 8 up, 8 in 2024-09-13T00:43:03.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:03 smithi136 bash[76371]: cluster 2024-09-13T00:43:02.283291+0000 mon.a (mon.0) 3862 : cluster [DBG] osdmap e2138: 8 total, 8 up, 8 in 2024-09-13T00:43:04.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:04 smithi181 bash[67587]: cluster 2024-09-13T00:43:02.518427+0000 mgr.y (mgr.24661) 1977 : cluster [DBG] pgmap v3676: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 300 KiB/s rd, 3.5 KiB/s wr, 403 op/s 2024-09-13T00:43:04.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:04 smithi181 bash[67587]: cluster 2024-09-13T00:43:03.293886+0000 mon.a (mon.0) 3863 : cluster [DBG] osdmap e2139: 8 total, 8 up, 8 in 2024-09-13T00:43:04.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:04 smithi136 bash[76371]: cluster 2024-09-13T00:43:02.518427+0000 mgr.y (mgr.24661) 1977 : cluster [DBG] pgmap v3676: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 300 KiB/s rd, 3.5 KiB/s wr, 403 op/s 2024-09-13T00:43:04.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:04 smithi136 bash[76371]: cluster 2024-09-13T00:43:03.293886+0000 mon.a (mon.0) 3863 : cluster [DBG] osdmap e2139: 8 total, 8 up, 8 in 2024-09-13T00:43:04.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:04 smithi136 bash[75164]: cluster 2024-09-13T00:43:02.518427+0000 mgr.y (mgr.24661) 1977 : cluster [DBG] pgmap v3676: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 300 KiB/s rd, 3.5 KiB/s wr, 403 op/s 2024-09-13T00:43:04.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:04 smithi136 bash[75164]: cluster 2024-09-13T00:43:03.293886+0000 mon.a (mon.0) 3863 : cluster [DBG] osdmap e2139: 8 total, 8 up, 8 in 2024-09-13T00:43:05.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:05 smithi181 bash[67587]: cluster 2024-09-13T00:43:04.302399+0000 mon.a (mon.0) 3864 : cluster [DBG] osdmap e2140: 8 total, 8 up, 8 in 2024-09-13T00:43:05.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:05 smithi136 bash[75164]: cluster 2024-09-13T00:43:04.302399+0000 mon.a (mon.0) 3864 : cluster [DBG] osdmap e2140: 8 total, 8 up, 8 in 2024-09-13T00:43:05.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:05 smithi136 bash[76371]: cluster 2024-09-13T00:43:04.302399+0000 mon.a (mon.0) 3864 : cluster [DBG] osdmap e2140: 8 total, 8 up, 8 in 2024-09-13T00:43:06.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:06 smithi181 bash[67587]: cluster 2024-09-13T00:43:04.519157+0000 mgr.y (mgr.24661) 1978 : cluster [DBG] pgmap v3679: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 301 KiB/s rd, 3.5 KiB/s wr, 404 op/s 2024-09-13T00:43:06.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:06 smithi181 bash[67587]: cluster 2024-09-13T00:43:05.312482+0000 mon.a (mon.0) 3865 : cluster [DBG] osdmap e2141: 8 total, 8 up, 8 in 2024-09-13T00:43:06.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:06 smithi136 bash[75164]: cluster 2024-09-13T00:43:04.519157+0000 mgr.y (mgr.24661) 1978 : cluster [DBG] pgmap v3679: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 301 KiB/s rd, 3.5 KiB/s wr, 404 op/s 2024-09-13T00:43:06.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:06 smithi136 bash[75164]: cluster 2024-09-13T00:43:05.312482+0000 mon.a (mon.0) 3865 : cluster [DBG] osdmap e2141: 8 total, 8 up, 8 in 2024-09-13T00:43:06.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:06 smithi136 bash[76371]: cluster 2024-09-13T00:43:04.519157+0000 mgr.y (mgr.24661) 1978 : cluster [DBG] pgmap v3679: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 301 KiB/s rd, 3.5 KiB/s wr, 404 op/s 2024-09-13T00:43:06.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:06 smithi136 bash[76371]: cluster 2024-09-13T00:43:05.312482+0000 mon.a (mon.0) 3865 : cluster [DBG] osdmap e2141: 8 total, 8 up, 8 in 2024-09-13T00:43:07.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:07 smithi181 bash[67587]: cluster 2024-09-13T00:43:06.320678+0000 mon.a (mon.0) 3866 : cluster [DBG] osdmap e2142: 8 total, 8 up, 8 in 2024-09-13T00:43:07.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:07 smithi136 bash[75164]: cluster 2024-09-13T00:43:06.320678+0000 mon.a (mon.0) 3866 : cluster [DBG] osdmap e2142: 8 total, 8 up, 8 in 2024-09-13T00:43:07.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:07 smithi136 bash[76371]: cluster 2024-09-13T00:43:06.320678+0000 mon.a (mon.0) 3866 : cluster [DBG] osdmap e2142: 8 total, 8 up, 8 in 2024-09-13T00:43:08.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:08 smithi181 bash[67587]: cluster 2024-09-13T00:43:06.520032+0000 mgr.y (mgr.24661) 1979 : cluster [DBG] pgmap v3682: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 1.2 KiB/s wr, 7 op/s 2024-09-13T00:43:08.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:08 smithi181 bash[67587]: cluster 2024-09-13T00:43:07.332583+0000 mon.a (mon.0) 3867 : cluster [DBG] osdmap e2143: 8 total, 8 up, 8 in 2024-09-13T00:43:08.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:08 smithi136 bash[76371]: cluster 2024-09-13T00:43:06.520032+0000 mgr.y (mgr.24661) 1979 : cluster [DBG] pgmap v3682: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 1.2 KiB/s wr, 7 op/s 2024-09-13T00:43:08.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:08 smithi136 bash[76371]: cluster 2024-09-13T00:43:07.332583+0000 mon.a (mon.0) 3867 : cluster [DBG] osdmap e2143: 8 total, 8 up, 8 in 2024-09-13T00:43:08.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:08 smithi136 bash[75164]: cluster 2024-09-13T00:43:06.520032+0000 mgr.y (mgr.24661) 1979 : cluster [DBG] pgmap v3682: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 1.2 KiB/s wr, 7 op/s 2024-09-13T00:43:08.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:08 smithi136 bash[75164]: cluster 2024-09-13T00:43:07.332583+0000 mon.a (mon.0) 3867 : cluster [DBG] osdmap e2143: 8 total, 8 up, 8 in 2024-09-13T00:43:09.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:09 smithi181 bash[67587]: cluster 2024-09-13T00:43:08.342237+0000 mon.a (mon.0) 3868 : cluster [DBG] osdmap e2144: 8 total, 8 up, 8 in 2024-09-13T00:43:09.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:09 smithi181 bash[67587]: audit 2024-09-13T00:43:08.843082+0000 mon.a (mon.0) 3869 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:43:09.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:43:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:43:09] "GET /metrics HTTP/1.1" 200 36742 "" "Prometheus/2.43.0" 2024-09-13T00:43:09.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:09 smithi136 bash[75164]: cluster 2024-09-13T00:43:08.342237+0000 mon.a (mon.0) 3868 : cluster [DBG] osdmap e2144: 8 total, 8 up, 8 in 2024-09-13T00:43:09.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:09 smithi136 bash[75164]: audit 2024-09-13T00:43:08.843082+0000 mon.a (mon.0) 3869 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:43:09.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:09 smithi136 bash[76371]: cluster 2024-09-13T00:43:08.342237+0000 mon.a (mon.0) 3868 : cluster [DBG] osdmap e2144: 8 total, 8 up, 8 in 2024-09-13T00:43:09.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:09 smithi136 bash[76371]: audit 2024-09-13T00:43:08.843082+0000 mon.a (mon.0) 3869 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:43:10.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:10 smithi181 bash[67587]: cluster 2024-09-13T00:43:08.520782+0000 mgr.y (mgr.24661) 1980 : cluster [DBG] pgmap v3685: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 331 KiB/s rd, 3.5 KiB/s wr, 443 op/s 2024-09-13T00:43:10.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:10 smithi181 bash[67587]: cluster 2024-09-13T00:43:09.353287+0000 mon.a (mon.0) 3870 : cluster [DBG] osdmap e2145: 8 total, 8 up, 8 in 2024-09-13T00:43:10.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:10 smithi136 bash[76371]: cluster 2024-09-13T00:43:08.520782+0000 mgr.y (mgr.24661) 1980 : cluster [DBG] pgmap v3685: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 331 KiB/s rd, 3.5 KiB/s wr, 443 op/s 2024-09-13T00:43:10.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:10 smithi136 bash[76371]: cluster 2024-09-13T00:43:09.353287+0000 mon.a (mon.0) 3870 : cluster [DBG] osdmap e2145: 8 total, 8 up, 8 in 2024-09-13T00:43:10.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:10 smithi136 bash[75164]: cluster 2024-09-13T00:43:08.520782+0000 mgr.y (mgr.24661) 1980 : cluster [DBG] pgmap v3685: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 331 KiB/s rd, 3.5 KiB/s wr, 443 op/s 2024-09-13T00:43:10.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:10 smithi136 bash[75164]: cluster 2024-09-13T00:43:09.353287+0000 mon.a (mon.0) 3870 : cluster [DBG] osdmap e2145: 8 total, 8 up, 8 in 2024-09-13T00:43:11.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:11 smithi136 bash[75164]: cluster 2024-09-13T00:43:10.360072+0000 mon.a (mon.0) 3871 : cluster [DBG] osdmap e2146: 8 total, 8 up, 8 in 2024-09-13T00:43:11.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:11 smithi136 bash[76371]: cluster 2024-09-13T00:43:10.360072+0000 mon.a (mon.0) 3871 : cluster [DBG] osdmap e2146: 8 total, 8 up, 8 in 2024-09-13T00:43:11.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:11 smithi181 bash[67587]: cluster 2024-09-13T00:43:10.360072+0000 mon.a (mon.0) 3871 : cluster [DBG] osdmap e2146: 8 total, 8 up, 8 in 2024-09-13T00:43:12.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:12 smithi181 bash[67587]: cluster 2024-09-13T00:43:10.521443+0000 mgr.y (mgr.24661) 1981 : cluster [DBG] pgmap v3688: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 335 KiB/s rd, 3.0 KiB/s wr, 447 op/s 2024-09-13T00:43:12.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:12 smithi181 bash[67587]: cluster 2024-09-13T00:43:11.372520+0000 mon.a (mon.0) 3872 : cluster [DBG] osdmap e2147: 8 total, 8 up, 8 in 2024-09-13T00:43:12.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:43:12 smithi181 bash[95505]: ts=2024-09-13T00:43:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:43:12.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:12 smithi136 bash[75164]: cluster 2024-09-13T00:43:10.521443+0000 mgr.y (mgr.24661) 1981 : cluster [DBG] pgmap v3688: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 335 KiB/s rd, 3.0 KiB/s wr, 447 op/s 2024-09-13T00:43:12.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:12 smithi136 bash[75164]: cluster 2024-09-13T00:43:11.372520+0000 mon.a (mon.0) 3872 : cluster [DBG] osdmap e2147: 8 total, 8 up, 8 in 2024-09-13T00:43:12.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:12 smithi136 bash[76371]: cluster 2024-09-13T00:43:10.521443+0000 mgr.y (mgr.24661) 1981 : cluster [DBG] pgmap v3688: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 335 KiB/s rd, 3.0 KiB/s wr, 447 op/s 2024-09-13T00:43:12.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:12 smithi136 bash[76371]: cluster 2024-09-13T00:43:11.372520+0000 mon.a (mon.0) 3872 : cluster [DBG] osdmap e2147: 8 total, 8 up, 8 in 2024-09-13T00:43:13.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:13 smithi136 bash[76371]: cluster 2024-09-13T00:43:12.374164+0000 mon.a (mon.0) 3873 : cluster [DBG] osdmap e2148: 8 total, 8 up, 8 in 2024-09-13T00:43:13.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:13 smithi136 bash[75164]: cluster 2024-09-13T00:43:12.374164+0000 mon.a (mon.0) 3873 : cluster [DBG] osdmap e2148: 8 total, 8 up, 8 in 2024-09-13T00:43:13.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:13 smithi181 bash[67587]: cluster 2024-09-13T00:43:12.374164+0000 mon.a (mon.0) 3873 : cluster [DBG] osdmap e2148: 8 total, 8 up, 8 in 2024-09-13T00:43:14.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:14 smithi136 bash[75164]: cluster 2024-09-13T00:43:12.522153+0000 mgr.y (mgr.24661) 1982 : cluster [DBG] pgmap v3691: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 365 KiB/s rd, 3.7 KiB/s wr, 491 op/s 2024-09-13T00:43:14.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:14 smithi136 bash[75164]: cluster 2024-09-13T00:43:13.394329+0000 mon.a (mon.0) 3874 : cluster [DBG] osdmap e2149: 8 total, 8 up, 8 in 2024-09-13T00:43:14.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:14 smithi136 bash[76371]: cluster 2024-09-13T00:43:12.522153+0000 mgr.y (mgr.24661) 1982 : cluster [DBG] pgmap v3691: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 365 KiB/s rd, 3.7 KiB/s wr, 491 op/s 2024-09-13T00:43:14.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:14 smithi136 bash[76371]: cluster 2024-09-13T00:43:13.394329+0000 mon.a (mon.0) 3874 : cluster [DBG] osdmap e2149: 8 total, 8 up, 8 in 2024-09-13T00:43:14.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:14 smithi181 bash[67587]: cluster 2024-09-13T00:43:12.522153+0000 mgr.y (mgr.24661) 1982 : cluster [DBG] pgmap v3691: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 365 KiB/s rd, 3.7 KiB/s wr, 491 op/s 2024-09-13T00:43:14.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:14 smithi181 bash[67587]: cluster 2024-09-13T00:43:13.394329+0000 mon.a (mon.0) 3874 : cluster [DBG] osdmap e2149: 8 total, 8 up, 8 in 2024-09-13T00:43:15.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:15 smithi136 bash[75164]: cluster 2024-09-13T00:43:14.396756+0000 mon.a (mon.0) 3875 : cluster [DBG] osdmap e2150: 8 total, 8 up, 8 in 2024-09-13T00:43:15.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:15 smithi136 bash[76371]: cluster 2024-09-13T00:43:14.396756+0000 mon.a (mon.0) 3875 : cluster [DBG] osdmap e2150: 8 total, 8 up, 8 in 2024-09-13T00:43:15.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:15 smithi181 bash[67587]: cluster 2024-09-13T00:43:14.396756+0000 mon.a (mon.0) 3875 : cluster [DBG] osdmap e2150: 8 total, 8 up, 8 in 2024-09-13T00:43:16.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:16 smithi136 bash[75164]: cluster 2024-09-13T00:43:14.522858+0000 mgr.y (mgr.24661) 1983 : cluster [DBG] pgmap v3694: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 366 KiB/s rd, 4.0 KiB/s wr, 492 op/s 2024-09-13T00:43:16.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:16 smithi136 bash[75164]: cluster 2024-09-13T00:43:15.418239+0000 mon.a (mon.0) 3876 : cluster [DBG] osdmap e2151: 8 total, 8 up, 8 in 2024-09-13T00:43:16.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:16 smithi136 bash[76371]: cluster 2024-09-13T00:43:14.522858+0000 mgr.y (mgr.24661) 1983 : cluster [DBG] pgmap v3694: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 366 KiB/s rd, 4.0 KiB/s wr, 492 op/s 2024-09-13T00:43:16.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:16 smithi136 bash[76371]: cluster 2024-09-13T00:43:15.418239+0000 mon.a (mon.0) 3876 : cluster [DBG] osdmap e2151: 8 total, 8 up, 8 in 2024-09-13T00:43:16.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:16 smithi181 bash[67587]: cluster 2024-09-13T00:43:14.522858+0000 mgr.y (mgr.24661) 1983 : cluster [DBG] pgmap v3694: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 366 KiB/s rd, 4.0 KiB/s wr, 492 op/s 2024-09-13T00:43:16.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:16 smithi181 bash[67587]: cluster 2024-09-13T00:43:15.418239+0000 mon.a (mon.0) 3876 : cluster [DBG] osdmap e2151: 8 total, 8 up, 8 in 2024-09-13T00:43:17.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:17 smithi136 bash[76371]: cluster 2024-09-13T00:43:16.416910+0000 mon.a (mon.0) 3877 : cluster [DBG] osdmap e2152: 8 total, 8 up, 8 in 2024-09-13T00:43:17.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:17 smithi136 bash[75164]: cluster 2024-09-13T00:43:16.416910+0000 mon.a (mon.0) 3877 : cluster [DBG] osdmap e2152: 8 total, 8 up, 8 in 2024-09-13T00:43:17.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:17 smithi181 bash[67587]: cluster 2024-09-13T00:43:16.416910+0000 mon.a (mon.0) 3877 : cluster [DBG] osdmap e2152: 8 total, 8 up, 8 in 2024-09-13T00:43:18.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:18 smithi136 bash[75164]: cluster 2024-09-13T00:43:16.523557+0000 mgr.y (mgr.24661) 1984 : cluster [DBG] pgmap v3697: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 1023 B/s wr, 7 op/s 2024-09-13T00:43:18.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:18 smithi136 bash[75164]: cluster 2024-09-13T00:43:17.445433+0000 mon.a (mon.0) 3878 : cluster [DBG] osdmap e2153: 8 total, 8 up, 8 in 2024-09-13T00:43:18.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:18 smithi136 bash[76371]: cluster 2024-09-13T00:43:16.523557+0000 mgr.y (mgr.24661) 1984 : cluster [DBG] pgmap v3697: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 1023 B/s wr, 7 op/s 2024-09-13T00:43:18.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:18 smithi136 bash[76371]: cluster 2024-09-13T00:43:17.445433+0000 mon.a (mon.0) 3878 : cluster [DBG] osdmap e2153: 8 total, 8 up, 8 in 2024-09-13T00:43:18.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:18 smithi181 bash[67587]: cluster 2024-09-13T00:43:16.523557+0000 mgr.y (mgr.24661) 1984 : cluster [DBG] pgmap v3697: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 1023 B/s wr, 7 op/s 2024-09-13T00:43:18.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:18 smithi181 bash[67587]: cluster 2024-09-13T00:43:17.445433+0000 mon.a (mon.0) 3878 : cluster [DBG] osdmap e2153: 8 total, 8 up, 8 in 2024-09-13T00:43:19.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:19 smithi136 bash[75164]: cluster 2024-09-13T00:43:18.447982+0000 mon.a (mon.0) 3879 : cluster [DBG] osdmap e2154: 8 total, 8 up, 8 in 2024-09-13T00:43:19.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:43:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:43:19] "GET /metrics HTTP/1.1" 200 36742 "" "Prometheus/2.43.0" 2024-09-13T00:43:19.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:19 smithi136 bash[76371]: cluster 2024-09-13T00:43:18.447982+0000 mon.a (mon.0) 3879 : cluster [DBG] osdmap e2154: 8 total, 8 up, 8 in 2024-09-13T00:43:19.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:19 smithi181 bash[67587]: cluster 2024-09-13T00:43:18.447982+0000 mon.a (mon.0) 3879 : cluster [DBG] osdmap e2154: 8 total, 8 up, 8 in 2024-09-13T00:43:20.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:20 smithi136 bash[75164]: cluster 2024-09-13T00:43:18.524260+0000 mgr.y (mgr.24661) 1985 : cluster [DBG] pgmap v3700: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 395 KiB/s rd, 3.2 KiB/s wr, 530 op/s 2024-09-13T00:43:20.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:20 smithi136 bash[75164]: cluster 2024-09-13T00:43:19.460247+0000 mon.a (mon.0) 3880 : cluster [DBG] osdmap e2155: 8 total, 8 up, 8 in 2024-09-13T00:43:20.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:20 smithi136 bash[76371]: cluster 2024-09-13T00:43:18.524260+0000 mgr.y (mgr.24661) 1985 : cluster [DBG] pgmap v3700: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 395 KiB/s rd, 3.2 KiB/s wr, 530 op/s 2024-09-13T00:43:20.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:20 smithi136 bash[76371]: cluster 2024-09-13T00:43:19.460247+0000 mon.a (mon.0) 3880 : cluster [DBG] osdmap e2155: 8 total, 8 up, 8 in 2024-09-13T00:43:20.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:20 smithi181 bash[67587]: cluster 2024-09-13T00:43:18.524260+0000 mgr.y (mgr.24661) 1985 : cluster [DBG] pgmap v3700: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 395 KiB/s rd, 3.2 KiB/s wr, 530 op/s 2024-09-13T00:43:20.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:20 smithi181 bash[67587]: cluster 2024-09-13T00:43:19.460247+0000 mon.a (mon.0) 3880 : cluster [DBG] osdmap e2155: 8 total, 8 up, 8 in 2024-09-13T00:43:21.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:21 smithi136 bash[75164]: cluster 2024-09-13T00:43:20.466627+0000 mon.a (mon.0) 3881 : cluster [DBG] osdmap e2156: 8 total, 8 up, 8 in 2024-09-13T00:43:21.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:21 smithi136 bash[76371]: cluster 2024-09-13T00:43:20.466627+0000 mon.a (mon.0) 3881 : cluster [DBG] osdmap e2156: 8 total, 8 up, 8 in 2024-09-13T00:43:21.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:21 smithi181 bash[67587]: cluster 2024-09-13T00:43:20.466627+0000 mon.a (mon.0) 3881 : cluster [DBG] osdmap e2156: 8 total, 8 up, 8 in 2024-09-13T00:43:22.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:22 smithi181 bash[67587]: cluster 2024-09-13T00:43:20.524973+0000 mgr.y (mgr.24661) 1986 : cluster [DBG] pgmap v3703: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 399 KiB/s rd, 3.0 KiB/s wr, 535 op/s 2024-09-13T00:43:22.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:22 smithi181 bash[67587]: cluster 2024-09-13T00:43:21.483944+0000 mon.a (mon.0) 3882 : cluster [DBG] osdmap e2157: 8 total, 8 up, 8 in 2024-09-13T00:43:22.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:43:22 smithi181 bash[95505]: ts=2024-09-13T00:43:22.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:43:22.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:22 smithi136 bash[76371]: cluster 2024-09-13T00:43:20.524973+0000 mgr.y (mgr.24661) 1986 : cluster [DBG] pgmap v3703: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 399 KiB/s rd, 3.0 KiB/s wr, 535 op/s 2024-09-13T00:43:22.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:22 smithi136 bash[76371]: cluster 2024-09-13T00:43:21.483944+0000 mon.a (mon.0) 3882 : cluster [DBG] osdmap e2157: 8 total, 8 up, 8 in 2024-09-13T00:43:22.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:22 smithi136 bash[75164]: cluster 2024-09-13T00:43:20.524973+0000 mgr.y (mgr.24661) 1986 : cluster [DBG] pgmap v3703: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 399 KiB/s rd, 3.0 KiB/s wr, 535 op/s 2024-09-13T00:43:22.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:22 smithi136 bash[75164]: cluster 2024-09-13T00:43:21.483944+0000 mon.a (mon.0) 3882 : cluster [DBG] osdmap e2157: 8 total, 8 up, 8 in 2024-09-13T00:43:23.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:23 smithi136 bash[75164]: cluster 2024-09-13T00:43:22.482296+0000 mon.a (mon.0) 3883 : cluster [DBG] osdmap e2158: 8 total, 8 up, 8 in 2024-09-13T00:43:23.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:23 smithi136 bash[76371]: cluster 2024-09-13T00:43:22.482296+0000 mon.a (mon.0) 3883 : cluster [DBG] osdmap e2158: 8 total, 8 up, 8 in 2024-09-13T00:43:23.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:23 smithi181 bash[67587]: cluster 2024-09-13T00:43:22.482296+0000 mon.a (mon.0) 3883 : cluster [DBG] osdmap e2158: 8 total, 8 up, 8 in 2024-09-13T00:43:24.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:24 smithi136 bash[76371]: cluster 2024-09-13T00:43:22.525711+0000 mgr.y (mgr.24661) 1987 : cluster [DBG] pgmap v3706: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 430 KiB/s rd, 3.5 KiB/s wr, 578 op/s 2024-09-13T00:43:24.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:24 smithi136 bash[76371]: cluster 2024-09-13T00:43:23.502197+0000 mon.a (mon.0) 3884 : cluster [DBG] osdmap e2159: 8 total, 8 up, 8 in 2024-09-13T00:43:24.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:24 smithi136 bash[76371]: audit 2024-09-13T00:43:23.843532+0000 mon.a (mon.0) 3885 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:43:24.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:24 smithi136 bash[75164]: cluster 2024-09-13T00:43:22.525711+0000 mgr.y (mgr.24661) 1987 : cluster [DBG] pgmap v3706: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 430 KiB/s rd, 3.5 KiB/s wr, 578 op/s 2024-09-13T00:43:24.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:24 smithi136 bash[75164]: cluster 2024-09-13T00:43:23.502197+0000 mon.a (mon.0) 3884 : cluster [DBG] osdmap e2159: 8 total, 8 up, 8 in 2024-09-13T00:43:24.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:24 smithi136 bash[75164]: audit 2024-09-13T00:43:23.843532+0000 mon.a (mon.0) 3885 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:43:24.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:24 smithi181 bash[67587]: cluster 2024-09-13T00:43:22.525711+0000 mgr.y (mgr.24661) 1987 : cluster [DBG] pgmap v3706: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 430 KiB/s rd, 3.5 KiB/s wr, 578 op/s 2024-09-13T00:43:24.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:24 smithi181 bash[67587]: cluster 2024-09-13T00:43:23.502197+0000 mon.a (mon.0) 3884 : cluster [DBG] osdmap e2159: 8 total, 8 up, 8 in 2024-09-13T00:43:24.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:24 smithi181 bash[67587]: audit 2024-09-13T00:43:23.843532+0000 mon.a (mon.0) 3885 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:43:25.855 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:25 smithi181 bash[67587]: cluster 2024-09-13T00:43:24.523152+0000 mon.a (mon.0) 3886 : cluster [DBG] osdmap e2160: 8 total, 8 up, 8 in 2024-09-13T00:43:25.855 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:25 smithi181 bash[67587]: cluster 2024-09-13T00:43:24.526550+0000 mgr.y (mgr.24661) 1988 : cluster [DBG] pgmap v3709: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 431 KiB/s rd, 4.0 KiB/s wr, 580 op/s 2024-09-13T00:43:26.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:25 smithi136 bash[75164]: cluster 2024-09-13T00:43:24.523152+0000 mon.a (mon.0) 3886 : cluster [DBG] osdmap e2160: 8 total, 8 up, 8 in 2024-09-13T00:43:26.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:25 smithi136 bash[75164]: cluster 2024-09-13T00:43:24.526550+0000 mgr.y (mgr.24661) 1988 : cluster [DBG] pgmap v3709: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 431 KiB/s rd, 4.0 KiB/s wr, 580 op/s 2024-09-13T00:43:26.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:25 smithi136 bash[76371]: cluster 2024-09-13T00:43:24.523152+0000 mon.a (mon.0) 3886 : cluster [DBG] osdmap e2160: 8 total, 8 up, 8 in 2024-09-13T00:43:26.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:25 smithi136 bash[76371]: cluster 2024-09-13T00:43:24.526550+0000 mgr.y (mgr.24661) 1988 : cluster [DBG] pgmap v3709: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 431 KiB/s rd, 4.0 KiB/s wr, 580 op/s 2024-09-13T00:43:26.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:26 smithi181 bash[67587]: cluster 2024-09-13T00:43:25.534832+0000 mon.a (mon.0) 3887 : cluster [DBG] osdmap e2161: 8 total, 8 up, 8 in 2024-09-13T00:43:27.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:26 smithi136 bash[76371]: cluster 2024-09-13T00:43:25.534832+0000 mon.a (mon.0) 3887 : cluster [DBG] osdmap e2161: 8 total, 8 up, 8 in 2024-09-13T00:43:27.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:26 smithi136 bash[75164]: cluster 2024-09-13T00:43:25.534832+0000 mon.a (mon.0) 3887 : cluster [DBG] osdmap e2161: 8 total, 8 up, 8 in 2024-09-13T00:43:27.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:27 smithi181 bash[67587]: cluster 2024-09-13T00:43:26.527615+0000 mgr.y (mgr.24661) 1989 : cluster [DBG] pgmap v3711: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 4.9 KiB/s rd, 1011 B/s wr, 7 op/s 2024-09-13T00:43:27.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:27 smithi181 bash[67587]: cluster 2024-09-13T00:43:26.538899+0000 mon.a (mon.0) 3888 : cluster [DBG] osdmap e2162: 8 total, 8 up, 8 in 2024-09-13T00:43:28.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:27 smithi136 bash[75164]: cluster 2024-09-13T00:43:26.527615+0000 mgr.y (mgr.24661) 1989 : cluster [DBG] pgmap v3711: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 4.9 KiB/s rd, 1011 B/s wr, 7 op/s 2024-09-13T00:43:28.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:27 smithi136 bash[75164]: cluster 2024-09-13T00:43:26.538899+0000 mon.a (mon.0) 3888 : cluster [DBG] osdmap e2162: 8 total, 8 up, 8 in 2024-09-13T00:43:28.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:27 smithi136 bash[76371]: cluster 2024-09-13T00:43:26.527615+0000 mgr.y (mgr.24661) 1989 : cluster [DBG] pgmap v3711: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 4.9 KiB/s rd, 1011 B/s wr, 7 op/s 2024-09-13T00:43:28.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:27 smithi136 bash[76371]: cluster 2024-09-13T00:43:26.538899+0000 mon.a (mon.0) 3888 : cluster [DBG] osdmap e2162: 8 total, 8 up, 8 in 2024-09-13T00:43:28.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:28 smithi181 bash[67587]: cluster 2024-09-13T00:43:27.556276+0000 mon.a (mon.0) 3889 : cluster [DBG] osdmap e2163: 8 total, 8 up, 8 in 2024-09-13T00:43:29.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:28 smithi136 bash[75164]: cluster 2024-09-13T00:43:27.556276+0000 mon.a (mon.0) 3889 : cluster [DBG] osdmap e2163: 8 total, 8 up, 8 in 2024-09-13T00:43:29.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:28 smithi136 bash[76371]: cluster 2024-09-13T00:43:27.556276+0000 mon.a (mon.0) 3889 : cluster [DBG] osdmap e2163: 8 total, 8 up, 8 in 2024-09-13T00:43:29.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:43:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:43:29] "GET /metrics HTTP/1.1" 200 36743 "" "Prometheus/2.43.0" 2024-09-13T00:43:29.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:29 smithi136 bash[75164]: cluster 2024-09-13T00:43:28.528478+0000 mgr.y (mgr.24661) 1990 : cluster [DBG] pgmap v3714: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 461 KiB/s rd, 3.0 KiB/s wr, 619 op/s 2024-09-13T00:43:29.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:29 smithi136 bash[75164]: cluster 2024-09-13T00:43:28.585602+0000 mon.a (mon.0) 3890 : cluster [DBG] osdmap e2164: 8 total, 8 up, 8 in 2024-09-13T00:43:29.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:29 smithi136 bash[76371]: cluster 2024-09-13T00:43:28.528478+0000 mgr.y (mgr.24661) 1990 : cluster [DBG] pgmap v3714: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 461 KiB/s rd, 3.0 KiB/s wr, 619 op/s 2024-09-13T00:43:29.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:29 smithi136 bash[76371]: cluster 2024-09-13T00:43:28.585602+0000 mon.a (mon.0) 3890 : cluster [DBG] osdmap e2164: 8 total, 8 up, 8 in 2024-09-13T00:43:29.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:29 smithi181 bash[67587]: cluster 2024-09-13T00:43:28.528478+0000 mgr.y (mgr.24661) 1990 : cluster [DBG] pgmap v3714: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 461 KiB/s rd, 3.0 KiB/s wr, 619 op/s 2024-09-13T00:43:29.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:29 smithi181 bash[67587]: cluster 2024-09-13T00:43:28.585602+0000 mon.a (mon.0) 3890 : cluster [DBG] osdmap e2164: 8 total, 8 up, 8 in 2024-09-13T00:43:30.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:30 smithi181 bash[67587]: cluster 2024-09-13T00:43:29.591596+0000 mon.a (mon.0) 3891 : cluster [DBG] osdmap e2165: 8 total, 8 up, 8 in 2024-09-13T00:43:31.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:30 smithi136 bash[75164]: cluster 2024-09-13T00:43:29.591596+0000 mon.a (mon.0) 3891 : cluster [DBG] osdmap e2165: 8 total, 8 up, 8 in 2024-09-13T00:43:31.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:30 smithi136 bash[76371]: cluster 2024-09-13T00:43:29.591596+0000 mon.a (mon.0) 3891 : cluster [DBG] osdmap e2165: 8 total, 8 up, 8 in 2024-09-13T00:43:31.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:31 smithi181 bash[67587]: cluster 2024-09-13T00:43:30.529252+0000 mgr.y (mgr.24661) 1991 : cluster [DBG] pgmap v3717: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 466 KiB/s rd, 3.5 KiB/s wr, 626 op/s 2024-09-13T00:43:31.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:31 smithi181 bash[67587]: cluster 2024-09-13T00:43:30.602140+0000 mon.a (mon.0) 3892 : cluster [DBG] osdmap e2166: 8 total, 8 up, 8 in 2024-09-13T00:43:32.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:31 smithi136 bash[75164]: cluster 2024-09-13T00:43:30.529252+0000 mgr.y (mgr.24661) 1991 : cluster [DBG] pgmap v3717: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 466 KiB/s rd, 3.5 KiB/s wr, 626 op/s 2024-09-13T00:43:32.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:31 smithi136 bash[75164]: cluster 2024-09-13T00:43:30.602140+0000 mon.a (mon.0) 3892 : cluster [DBG] osdmap e2166: 8 total, 8 up, 8 in 2024-09-13T00:43:32.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:31 smithi136 bash[76371]: cluster 2024-09-13T00:43:30.529252+0000 mgr.y (mgr.24661) 1991 : cluster [DBG] pgmap v3717: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 466 KiB/s rd, 3.5 KiB/s wr, 626 op/s 2024-09-13T00:43:32.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:31 smithi136 bash[76371]: cluster 2024-09-13T00:43:30.602140+0000 mon.a (mon.0) 3892 : cluster [DBG] osdmap e2166: 8 total, 8 up, 8 in 2024-09-13T00:43:32.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:43:32 smithi181 bash[95505]: ts=2024-09-13T00:43:32.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:43:33.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:32 smithi136 bash[75164]: cluster 2024-09-13T00:43:31.610942+0000 mon.a (mon.0) 3893 : cluster [DBG] osdmap e2167: 8 total, 8 up, 8 in 2024-09-13T00:43:33.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:32 smithi136 bash[76371]: cluster 2024-09-13T00:43:31.610942+0000 mon.a (mon.0) 3893 : cluster [DBG] osdmap e2167: 8 total, 8 up, 8 in 2024-09-13T00:43:33.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:32 smithi181 bash[67587]: cluster 2024-09-13T00:43:31.610942+0000 mon.a (mon.0) 3893 : cluster [DBG] osdmap e2167: 8 total, 8 up, 8 in 2024-09-13T00:43:34.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:33 smithi136 bash[75164]: cluster 2024-09-13T00:43:32.530177+0000 mgr.y (mgr.24661) 1992 : cluster [DBG] pgmap v3720: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 496 KiB/s rd, 3.7 KiB/s wr, 669 op/s 2024-09-13T00:43:34.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:33 smithi136 bash[75164]: cluster 2024-09-13T00:43:32.620451+0000 mon.a (mon.0) 3894 : cluster [DBG] osdmap e2168: 8 total, 8 up, 8 in 2024-09-13T00:43:34.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:33 smithi136 bash[76371]: cluster 2024-09-13T00:43:32.530177+0000 mgr.y (mgr.24661) 1992 : cluster [DBG] pgmap v3720: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 496 KiB/s rd, 3.7 KiB/s wr, 669 op/s 2024-09-13T00:43:34.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:33 smithi136 bash[76371]: cluster 2024-09-13T00:43:32.620451+0000 mon.a (mon.0) 3894 : cluster [DBG] osdmap e2168: 8 total, 8 up, 8 in 2024-09-13T00:43:34.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:33 smithi181 bash[67587]: cluster 2024-09-13T00:43:32.530177+0000 mgr.y (mgr.24661) 1992 : cluster [DBG] pgmap v3720: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 496 KiB/s rd, 3.7 KiB/s wr, 669 op/s 2024-09-13T00:43:34.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:33 smithi181 bash[67587]: cluster 2024-09-13T00:43:32.620451+0000 mon.a (mon.0) 3894 : cluster [DBG] osdmap e2168: 8 total, 8 up, 8 in 2024-09-13T00:43:35.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:34 smithi136 bash[75164]: cluster 2024-09-13T00:43:33.621355+0000 mon.a (mon.0) 3895 : cluster [DBG] osdmap e2169: 8 total, 8 up, 8 in 2024-09-13T00:43:35.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:34 smithi136 bash[76371]: cluster 2024-09-13T00:43:33.621355+0000 mon.a (mon.0) 3895 : cluster [DBG] osdmap e2169: 8 total, 8 up, 8 in 2024-09-13T00:43:35.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:34 smithi181 bash[67587]: cluster 2024-09-13T00:43:33.621355+0000 mon.a (mon.0) 3895 : cluster [DBG] osdmap e2169: 8 total, 8 up, 8 in 2024-09-13T00:43:36.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:35 smithi136 bash[75164]: cluster 2024-09-13T00:43:34.531065+0000 mgr.y (mgr.24661) 1993 : cluster [DBG] pgmap v3723: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 497 KiB/s rd, 3.7 KiB/s wr, 669 op/s 2024-09-13T00:43:36.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:35 smithi136 bash[75164]: cluster 2024-09-13T00:43:34.642719+0000 mon.a (mon.0) 3896 : cluster [DBG] osdmap e2170: 8 total, 8 up, 8 in 2024-09-13T00:43:36.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:35 smithi136 bash[76371]: cluster 2024-09-13T00:43:34.531065+0000 mgr.y (mgr.24661) 1993 : cluster [DBG] pgmap v3723: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 497 KiB/s rd, 3.7 KiB/s wr, 669 op/s 2024-09-13T00:43:36.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:35 smithi136 bash[76371]: cluster 2024-09-13T00:43:34.642719+0000 mon.a (mon.0) 3896 : cluster [DBG] osdmap e2170: 8 total, 8 up, 8 in 2024-09-13T00:43:36.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:35 smithi181 bash[67587]: cluster 2024-09-13T00:43:34.531065+0000 mgr.y (mgr.24661) 1993 : cluster [DBG] pgmap v3723: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 497 KiB/s rd, 3.7 KiB/s wr, 669 op/s 2024-09-13T00:43:36.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:35 smithi181 bash[67587]: cluster 2024-09-13T00:43:34.642719+0000 mon.a (mon.0) 3896 : cluster [DBG] osdmap e2170: 8 total, 8 up, 8 in 2024-09-13T00:43:37.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:36 smithi136 bash[76371]: cluster 2024-09-13T00:43:35.649697+0000 mon.a (mon.0) 3897 : cluster [DBG] osdmap e2171: 8 total, 8 up, 8 in 2024-09-13T00:43:37.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:36 smithi136 bash[75164]: cluster 2024-09-13T00:43:35.649697+0000 mon.a (mon.0) 3897 : cluster [DBG] osdmap e2171: 8 total, 8 up, 8 in 2024-09-13T00:43:37.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:36 smithi181 bash[67587]: cluster 2024-09-13T00:43:35.649697+0000 mon.a (mon.0) 3897 : cluster [DBG] osdmap e2171: 8 total, 8 up, 8 in 2024-09-13T00:43:38.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:37 smithi136 bash[76371]: cluster 2024-09-13T00:43:36.532142+0000 mgr.y (mgr.24661) 1994 : cluster [DBG] pgmap v3726: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 4.2 KiB/s rd, 1.2 KiB/s wr, 6 op/s 2024-09-13T00:43:38.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:37 smithi136 bash[76371]: cluster 2024-09-13T00:43:36.660359+0000 mon.a (mon.0) 3898 : cluster [DBG] osdmap e2172: 8 total, 8 up, 8 in 2024-09-13T00:43:38.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:37 smithi136 bash[75164]: cluster 2024-09-13T00:43:36.532142+0000 mgr.y (mgr.24661) 1994 : cluster [DBG] pgmap v3726: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 4.2 KiB/s rd, 1.2 KiB/s wr, 6 op/s 2024-09-13T00:43:38.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:37 smithi136 bash[75164]: cluster 2024-09-13T00:43:36.660359+0000 mon.a (mon.0) 3898 : cluster [DBG] osdmap e2172: 8 total, 8 up, 8 in 2024-09-13T00:43:38.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:37 smithi181 bash[67587]: cluster 2024-09-13T00:43:36.532142+0000 mgr.y (mgr.24661) 1994 : cluster [DBG] pgmap v3726: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 4.2 KiB/s rd, 1.2 KiB/s wr, 6 op/s 2024-09-13T00:43:38.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:37 smithi181 bash[67587]: cluster 2024-09-13T00:43:36.660359+0000 mon.a (mon.0) 3898 : cluster [DBG] osdmap e2172: 8 total, 8 up, 8 in 2024-09-13T00:43:39.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:38 smithi136 bash[76371]: cluster 2024-09-13T00:43:37.662202+0000 mon.a (mon.0) 3899 : cluster [DBG] osdmap e2173: 8 total, 8 up, 8 in 2024-09-13T00:43:39.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:38 smithi136 bash[75164]: cluster 2024-09-13T00:43:37.662202+0000 mon.a (mon.0) 3899 : cluster [DBG] osdmap e2173: 8 total, 8 up, 8 in 2024-09-13T00:43:39.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:38 smithi181 bash[67587]: cluster 2024-09-13T00:43:37.662202+0000 mon.a (mon.0) 3899 : cluster [DBG] osdmap e2173: 8 total, 8 up, 8 in 2024-09-13T00:43:39.693 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:43:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:43:39] "GET /metrics HTTP/1.1" 200 36743 "" "Prometheus/2.43.0" 2024-09-13T00:43:40.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:39 smithi136 bash[75164]: cluster 2024-09-13T00:43:38.533011+0000 mgr.y (mgr.24661) 1995 : cluster [DBG] pgmap v3729: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 527 KiB/s rd, 3.0 KiB/s wr, 708 op/s 2024-09-13T00:43:40.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:39 smithi136 bash[75164]: cluster 2024-09-13T00:43:38.687177+0000 mon.a (mon.0) 3900 : cluster [DBG] osdmap e2174: 8 total, 8 up, 8 in 2024-09-13T00:43:40.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:39 smithi136 bash[75164]: audit 2024-09-13T00:43:38.843893+0000 mon.a (mon.0) 3901 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:43:40.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:39 smithi136 bash[76371]: cluster 2024-09-13T00:43:38.533011+0000 mgr.y (mgr.24661) 1995 : cluster [DBG] pgmap v3729: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 527 KiB/s rd, 3.0 KiB/s wr, 708 op/s 2024-09-13T00:43:40.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:39 smithi136 bash[76371]: cluster 2024-09-13T00:43:38.687177+0000 mon.a (mon.0) 3900 : cluster [DBG] osdmap e2174: 8 total, 8 up, 8 in 2024-09-13T00:43:40.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:39 smithi136 bash[76371]: audit 2024-09-13T00:43:38.843893+0000 mon.a (mon.0) 3901 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:43:40.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:39 smithi181 bash[67587]: cluster 2024-09-13T00:43:38.533011+0000 mgr.y (mgr.24661) 1995 : cluster [DBG] pgmap v3729: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 527 KiB/s rd, 3.0 KiB/s wr, 708 op/s 2024-09-13T00:43:40.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:39 smithi181 bash[67587]: cluster 2024-09-13T00:43:38.687177+0000 mon.a (mon.0) 3900 : cluster [DBG] osdmap e2174: 8 total, 8 up, 8 in 2024-09-13T00:43:40.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:39 smithi181 bash[67587]: audit 2024-09-13T00:43:38.843893+0000 mon.a (mon.0) 3901 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:43:41.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:40 smithi136 bash[75164]: cluster 2024-09-13T00:43:39.688612+0000 mon.a (mon.0) 3902 : cluster [DBG] osdmap e2175: 8 total, 8 up, 8 in 2024-09-13T00:43:41.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:40 smithi136 bash[76371]: cluster 2024-09-13T00:43:39.688612+0000 mon.a (mon.0) 3902 : cluster [DBG] osdmap e2175: 8 total, 8 up, 8 in 2024-09-13T00:43:41.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:40 smithi181 bash[67587]: cluster 2024-09-13T00:43:39.688612+0000 mon.a (mon.0) 3902 : cluster [DBG] osdmap e2175: 8 total, 8 up, 8 in 2024-09-13T00:43:42.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:41 smithi136 bash[76371]: cluster 2024-09-13T00:43:40.533788+0000 mgr.y (mgr.24661) 1996 : cluster [DBG] pgmap v3732: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 531 KiB/s rd, 3.0 KiB/s wr, 713 op/s 2024-09-13T00:43:42.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:41 smithi136 bash[76371]: cluster 2024-09-13T00:43:40.710064+0000 mon.a (mon.0) 3903 : cluster [DBG] osdmap e2176: 8 total, 8 up, 8 in 2024-09-13T00:43:42.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:41 smithi136 bash[75164]: cluster 2024-09-13T00:43:40.533788+0000 mgr.y (mgr.24661) 1996 : cluster [DBG] pgmap v3732: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 531 KiB/s rd, 3.0 KiB/s wr, 713 op/s 2024-09-13T00:43:42.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:41 smithi136 bash[75164]: cluster 2024-09-13T00:43:40.710064+0000 mon.a (mon.0) 3903 : cluster [DBG] osdmap e2176: 8 total, 8 up, 8 in 2024-09-13T00:43:42.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:41 smithi181 bash[67587]: cluster 2024-09-13T00:43:40.533788+0000 mgr.y (mgr.24661) 1996 : cluster [DBG] pgmap v3732: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 531 KiB/s rd, 3.0 KiB/s wr, 713 op/s 2024-09-13T00:43:42.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:41 smithi181 bash[67587]: cluster 2024-09-13T00:43:40.710064+0000 mon.a (mon.0) 3903 : cluster [DBG] osdmap e2176: 8 total, 8 up, 8 in 2024-09-13T00:43:42.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:43:42 smithi181 bash[95505]: ts=2024-09-13T00:43:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:43:43.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:42 smithi136 bash[75164]: cluster 2024-09-13T00:43:41.712909+0000 mon.a (mon.0) 3904 : cluster [DBG] osdmap e2177: 8 total, 8 up, 8 in 2024-09-13T00:43:43.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:42 smithi136 bash[76371]: cluster 2024-09-13T00:43:41.712909+0000 mon.a (mon.0) 3904 : cluster [DBG] osdmap e2177: 8 total, 8 up, 8 in 2024-09-13T00:43:43.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:42 smithi181 bash[67587]: cluster 2024-09-13T00:43:41.712909+0000 mon.a (mon.0) 3904 : cluster [DBG] osdmap e2177: 8 total, 8 up, 8 in 2024-09-13T00:43:44.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:43 smithi136 bash[75164]: cluster 2024-09-13T00:43:42.534715+0000 mgr.y (mgr.24661) 1997 : cluster [DBG] pgmap v3735: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 561 KiB/s rd, 4.0 KiB/s wr, 757 op/s 2024-09-13T00:43:44.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:43 smithi136 bash[75164]: cluster 2024-09-13T00:43:42.736756+0000 mon.a (mon.0) 3905 : cluster [DBG] osdmap e2178: 8 total, 8 up, 8 in 2024-09-13T00:43:44.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:43 smithi136 bash[76371]: cluster 2024-09-13T00:43:42.534715+0000 mgr.y (mgr.24661) 1997 : cluster [DBG] pgmap v3735: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 561 KiB/s rd, 4.0 KiB/s wr, 757 op/s 2024-09-13T00:43:44.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:43 smithi136 bash[76371]: cluster 2024-09-13T00:43:42.736756+0000 mon.a (mon.0) 3905 : cluster [DBG] osdmap e2178: 8 total, 8 up, 8 in 2024-09-13T00:43:44.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:43 smithi181 bash[67587]: cluster 2024-09-13T00:43:42.534715+0000 mgr.y (mgr.24661) 1997 : cluster [DBG] pgmap v3735: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 561 KiB/s rd, 4.0 KiB/s wr, 757 op/s 2024-09-13T00:43:44.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:43 smithi181 bash[67587]: cluster 2024-09-13T00:43:42.736756+0000 mon.a (mon.0) 3905 : cluster [DBG] osdmap e2178: 8 total, 8 up, 8 in 2024-09-13T00:43:45.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:44 smithi136 bash[75164]: cluster 2024-09-13T00:43:43.746672+0000 mon.a (mon.0) 3906 : cluster [DBG] osdmap e2179: 8 total, 8 up, 8 in 2024-09-13T00:43:45.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:44 smithi136 bash[76371]: cluster 2024-09-13T00:43:43.746672+0000 mon.a (mon.0) 3906 : cluster [DBG] osdmap e2179: 8 total, 8 up, 8 in 2024-09-13T00:43:45.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:44 smithi181 bash[67587]: cluster 2024-09-13T00:43:43.746672+0000 mon.a (mon.0) 3906 : cluster [DBG] osdmap e2179: 8 total, 8 up, 8 in 2024-09-13T00:43:46.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:45 smithi181 bash[67587]: cluster 2024-09-13T00:43:44.535398+0000 mgr.y (mgr.24661) 1998 : cluster [DBG] pgmap v3738: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 562 KiB/s rd, 3.5 KiB/s wr, 757 op/s 2024-09-13T00:43:46.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:45 smithi181 bash[67587]: cluster 2024-09-13T00:43:44.756182+0000 mon.a (mon.0) 3907 : cluster [DBG] osdmap e2180: 8 total, 8 up, 8 in 2024-09-13T00:43:46.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:45 smithi136 bash[75164]: cluster 2024-09-13T00:43:44.535398+0000 mgr.y (mgr.24661) 1998 : cluster [DBG] pgmap v3738: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 562 KiB/s rd, 3.5 KiB/s wr, 757 op/s 2024-09-13T00:43:46.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:45 smithi136 bash[75164]: cluster 2024-09-13T00:43:44.756182+0000 mon.a (mon.0) 3907 : cluster [DBG] osdmap e2180: 8 total, 8 up, 8 in 2024-09-13T00:43:46.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:45 smithi136 bash[76371]: cluster 2024-09-13T00:43:44.535398+0000 mgr.y (mgr.24661) 1998 : cluster [DBG] pgmap v3738: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 562 KiB/s rd, 3.5 KiB/s wr, 757 op/s 2024-09-13T00:43:46.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:45 smithi136 bash[76371]: cluster 2024-09-13T00:43:44.756182+0000 mon.a (mon.0) 3907 : cluster [DBG] osdmap e2180: 8 total, 8 up, 8 in 2024-09-13T00:43:47.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:46 smithi181 bash[67587]: cluster 2024-09-13T00:43:45.757717+0000 mon.a (mon.0) 3908 : cluster [DBG] osdmap e2181: 8 total, 8 up, 8 in 2024-09-13T00:43:47.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:46 smithi136 bash[75164]: cluster 2024-09-13T00:43:45.757717+0000 mon.a (mon.0) 3908 : cluster [DBG] osdmap e2181: 8 total, 8 up, 8 in 2024-09-13T00:43:47.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:46 smithi136 bash[76371]: cluster 2024-09-13T00:43:45.757717+0000 mon.a (mon.0) 3908 : cluster [DBG] osdmap e2181: 8 total, 8 up, 8 in 2024-09-13T00:43:48.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:47 smithi181 bash[67587]: cluster 2024-09-13T00:43:46.536329+0000 mgr.y (mgr.24661) 1999 : cluster [DBG] pgmap v3741: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 4.7 KiB/s rd, 767 B/s wr, 6 op/s 2024-09-13T00:43:48.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:47 smithi181 bash[67587]: cluster 2024-09-13T00:43:46.778410+0000 mon.a (mon.0) 3909 : cluster [DBG] osdmap e2182: 8 total, 8 up, 8 in 2024-09-13T00:43:48.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:47 smithi136 bash[75164]: cluster 2024-09-13T00:43:46.536329+0000 mgr.y (mgr.24661) 1999 : cluster [DBG] pgmap v3741: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 4.7 KiB/s rd, 767 B/s wr, 6 op/s 2024-09-13T00:43:48.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:47 smithi136 bash[75164]: cluster 2024-09-13T00:43:46.778410+0000 mon.a (mon.0) 3909 : cluster [DBG] osdmap e2182: 8 total, 8 up, 8 in 2024-09-13T00:43:48.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:47 smithi136 bash[76371]: cluster 2024-09-13T00:43:46.536329+0000 mgr.y (mgr.24661) 1999 : cluster [DBG] pgmap v3741: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 4.7 KiB/s rd, 767 B/s wr, 6 op/s 2024-09-13T00:43:48.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:47 smithi136 bash[76371]: cluster 2024-09-13T00:43:46.778410+0000 mon.a (mon.0) 3909 : cluster [DBG] osdmap e2182: 8 total, 8 up, 8 in 2024-09-13T00:43:49.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:48 smithi181 bash[67587]: cluster 2024-09-13T00:43:47.788633+0000 mon.a (mon.0) 3910 : cluster [DBG] osdmap e2183: 8 total, 8 up, 8 in 2024-09-13T00:43:49.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:48 smithi136 bash[75164]: cluster 2024-09-13T00:43:47.788633+0000 mon.a (mon.0) 3910 : cluster [DBG] osdmap e2183: 8 total, 8 up, 8 in 2024-09-13T00:43:49.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:48 smithi136 bash[76371]: cluster 2024-09-13T00:43:47.788633+0000 mon.a (mon.0) 3910 : cluster [DBG] osdmap e2183: 8 total, 8 up, 8 in 2024-09-13T00:43:49.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:43:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:43:49] "GET /metrics HTTP/1.1" 200 36743 "" "Prometheus/2.43.0" 2024-09-13T00:43:50.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:49 smithi181 bash[67587]: cluster 2024-09-13T00:43:48.537077+0000 mgr.y (mgr.24661) 2000 : cluster [DBG] pgmap v3744: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 592 KiB/s rd, 3.7 KiB/s wr, 797 op/s 2024-09-13T00:43:50.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:49 smithi181 bash[67587]: cluster 2024-09-13T00:43:48.798322+0000 mon.a (mon.0) 3911 : cluster [DBG] osdmap e2184: 8 total, 8 up, 8 in 2024-09-13T00:43:50.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:49 smithi136 bash[75164]: cluster 2024-09-13T00:43:48.537077+0000 mgr.y (mgr.24661) 2000 : cluster [DBG] pgmap v3744: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 592 KiB/s rd, 3.7 KiB/s wr, 797 op/s 2024-09-13T00:43:50.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:49 smithi136 bash[75164]: cluster 2024-09-13T00:43:48.798322+0000 mon.a (mon.0) 3911 : cluster [DBG] osdmap e2184: 8 total, 8 up, 8 in 2024-09-13T00:43:50.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:49 smithi136 bash[76371]: cluster 2024-09-13T00:43:48.537077+0000 mgr.y (mgr.24661) 2000 : cluster [DBG] pgmap v3744: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 592 KiB/s rd, 3.7 KiB/s wr, 797 op/s 2024-09-13T00:43:50.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:49 smithi136 bash[76371]: cluster 2024-09-13T00:43:48.798322+0000 mon.a (mon.0) 3911 : cluster [DBG] osdmap e2184: 8 total, 8 up, 8 in 2024-09-13T00:43:51.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:50 smithi181 bash[67587]: cluster 2024-09-13T00:43:49.799550+0000 mon.a (mon.0) 3912 : cluster [DBG] osdmap e2185: 8 total, 8 up, 8 in 2024-09-13T00:43:51.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:50 smithi136 bash[75164]: cluster 2024-09-13T00:43:49.799550+0000 mon.a (mon.0) 3912 : cluster [DBG] osdmap e2185: 8 total, 8 up, 8 in 2024-09-13T00:43:51.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:50 smithi136 bash[76371]: cluster 2024-09-13T00:43:49.799550+0000 mon.a (mon.0) 3912 : cluster [DBG] osdmap e2185: 8 total, 8 up, 8 in 2024-09-13T00:43:52.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:51 smithi181 bash[67587]: cluster 2024-09-13T00:43:50.537807+0000 mgr.y (mgr.24661) 2001 : cluster [DBG] pgmap v3747: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 595 KiB/s rd, 3.2 KiB/s wr, 801 op/s 2024-09-13T00:43:52.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:51 smithi181 bash[67587]: cluster 2024-09-13T00:43:50.824236+0000 mon.a (mon.0) 3913 : cluster [DBG] osdmap e2186: 8 total, 8 up, 8 in 2024-09-13T00:43:52.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:51 smithi136 bash[75164]: cluster 2024-09-13T00:43:50.537807+0000 mgr.y (mgr.24661) 2001 : cluster [DBG] pgmap v3747: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 595 KiB/s rd, 3.2 KiB/s wr, 801 op/s 2024-09-13T00:43:52.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:51 smithi136 bash[75164]: cluster 2024-09-13T00:43:50.824236+0000 mon.a (mon.0) 3913 : cluster [DBG] osdmap e2186: 8 total, 8 up, 8 in 2024-09-13T00:43:52.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:51 smithi136 bash[76371]: cluster 2024-09-13T00:43:50.537807+0000 mgr.y (mgr.24661) 2001 : cluster [DBG] pgmap v3747: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 595 KiB/s rd, 3.2 KiB/s wr, 801 op/s 2024-09-13T00:43:52.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:51 smithi136 bash[76371]: cluster 2024-09-13T00:43:50.824236+0000 mon.a (mon.0) 3913 : cluster [DBG] osdmap e2186: 8 total, 8 up, 8 in 2024-09-13T00:43:52.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:43:52 smithi181 bash[95505]: ts=2024-09-13T00:43:52.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:43:53.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:52 smithi181 bash[67587]: cluster 2024-09-13T00:43:51.826823+0000 mon.a (mon.0) 3914 : cluster [DBG] osdmap e2187: 8 total, 8 up, 8 in 2024-09-13T00:43:53.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:52 smithi136 bash[75164]: cluster 2024-09-13T00:43:51.826823+0000 mon.a (mon.0) 3914 : cluster [DBG] osdmap e2187: 8 total, 8 up, 8 in 2024-09-13T00:43:53.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:52 smithi136 bash[76371]: cluster 2024-09-13T00:43:51.826823+0000 mon.a (mon.0) 3914 : cluster [DBG] osdmap e2187: 8 total, 8 up, 8 in 2024-09-13T00:43:54.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:53 smithi181 bash[67587]: cluster 2024-09-13T00:43:52.538697+0000 mgr.y (mgr.24661) 2002 : cluster [DBG] pgmap v3750: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 626 KiB/s rd, 3.2 KiB/s wr, 844 op/s 2024-09-13T00:43:54.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:53 smithi181 bash[67587]: cluster 2024-09-13T00:43:52.849545+0000 mon.a (mon.0) 3915 : cluster [DBG] osdmap e2188: 8 total, 8 up, 8 in 2024-09-13T00:43:54.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:53 smithi136 bash[75164]: cluster 2024-09-13T00:43:52.538697+0000 mgr.y (mgr.24661) 2002 : cluster [DBG] pgmap v3750: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 626 KiB/s rd, 3.2 KiB/s wr, 844 op/s 2024-09-13T00:43:54.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:53 smithi136 bash[75164]: cluster 2024-09-13T00:43:52.849545+0000 mon.a (mon.0) 3915 : cluster [DBG] osdmap e2188: 8 total, 8 up, 8 in 2024-09-13T00:43:54.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:53 smithi136 bash[76371]: cluster 2024-09-13T00:43:52.538697+0000 mgr.y (mgr.24661) 2002 : cluster [DBG] pgmap v3750: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 626 KiB/s rd, 3.2 KiB/s wr, 844 op/s 2024-09-13T00:43:54.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:53 smithi136 bash[76371]: cluster 2024-09-13T00:43:52.849545+0000 mon.a (mon.0) 3915 : cluster [DBG] osdmap e2188: 8 total, 8 up, 8 in 2024-09-13T00:43:55.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:54 smithi181 bash[67587]: audit 2024-09-13T00:43:53.849763+0000 mon.a (mon.0) 3916 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:43:55.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:54 smithi181 bash[67587]: cluster 2024-09-13T00:43:53.858636+0000 mon.a (mon.0) 3917 : cluster [DBG] osdmap e2189: 8 total, 8 up, 8 in 2024-09-13T00:43:55.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:54 smithi136 bash[75164]: audit 2024-09-13T00:43:53.849763+0000 mon.a (mon.0) 3916 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:43:55.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:54 smithi136 bash[75164]: cluster 2024-09-13T00:43:53.858636+0000 mon.a (mon.0) 3917 : cluster [DBG] osdmap e2189: 8 total, 8 up, 8 in 2024-09-13T00:43:55.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:54 smithi136 bash[76371]: audit 2024-09-13T00:43:53.849763+0000 mon.a (mon.0) 3916 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:43:55.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:54 smithi136 bash[76371]: cluster 2024-09-13T00:43:53.858636+0000 mon.a (mon.0) 3917 : cluster [DBG] osdmap e2189: 8 total, 8 up, 8 in 2024-09-13T00:43:56.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:55 smithi136 bash[75164]: cluster 2024-09-13T00:43:54.539366+0000 mgr.y (mgr.24661) 2003 : cluster [DBG] pgmap v3753: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 627 KiB/s rd, 3.5 KiB/s wr, 845 op/s 2024-09-13T00:43:56.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:55 smithi136 bash[75164]: cluster 2024-09-13T00:43:54.859847+0000 mon.a (mon.0) 3918 : cluster [DBG] osdmap e2190: 8 total, 8 up, 8 in 2024-09-13T00:43:56.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:55 smithi136 bash[75164]: audit 2024-09-13T00:43:55.256146+0000 mon.a (mon.0) 3919 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:43:56.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:55 smithi136 bash[75164]: audit 2024-09-13T00:43:55.804195+0000 mon.a (mon.0) 3920 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:43:56.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:55 smithi136 bash[75164]: audit 2024-09-13T00:43:55.805727+0000 mon.a (mon.0) 3921 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:43:56.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:55 smithi136 bash[75164]: audit 2024-09-13T00:43:55.815482+0000 mon.a (mon.0) 3922 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:43:56.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:55 smithi136 bash[76371]: cluster 2024-09-13T00:43:54.539366+0000 mgr.y (mgr.24661) 2003 : cluster [DBG] pgmap v3753: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 627 KiB/s rd, 3.5 KiB/s wr, 845 op/s 2024-09-13T00:43:56.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:55 smithi136 bash[76371]: cluster 2024-09-13T00:43:54.859847+0000 mon.a (mon.0) 3918 : cluster [DBG] osdmap e2190: 8 total, 8 up, 8 in 2024-09-13T00:43:56.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:55 smithi136 bash[76371]: audit 2024-09-13T00:43:55.256146+0000 mon.a (mon.0) 3919 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:43:56.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:55 smithi136 bash[76371]: audit 2024-09-13T00:43:55.804195+0000 mon.a (mon.0) 3920 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:43:56.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:55 smithi136 bash[76371]: audit 2024-09-13T00:43:55.805727+0000 mon.a (mon.0) 3921 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:43:56.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:55 smithi136 bash[76371]: audit 2024-09-13T00:43:55.815482+0000 mon.a (mon.0) 3922 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:43:56.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:55 smithi181 bash[67587]: cluster 2024-09-13T00:43:54.539366+0000 mgr.y (mgr.24661) 2003 : cluster [DBG] pgmap v3753: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 627 KiB/s rd, 3.5 KiB/s wr, 845 op/s 2024-09-13T00:43:56.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:55 smithi181 bash[67587]: cluster 2024-09-13T00:43:54.859847+0000 mon.a (mon.0) 3918 : cluster [DBG] osdmap e2190: 8 total, 8 up, 8 in 2024-09-13T00:43:56.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:55 smithi181 bash[67587]: audit 2024-09-13T00:43:55.256146+0000 mon.a (mon.0) 3919 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:43:56.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:55 smithi181 bash[67587]: audit 2024-09-13T00:43:55.804195+0000 mon.a (mon.0) 3920 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:43:56.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:55 smithi181 bash[67587]: audit 2024-09-13T00:43:55.805727+0000 mon.a (mon.0) 3921 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:43:56.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:55 smithi181 bash[67587]: audit 2024-09-13T00:43:55.815482+0000 mon.a (mon.0) 3922 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:43:57.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:56 smithi136 bash[75164]: cluster 2024-09-13T00:43:55.873048+0000 mon.a (mon.0) 3923 : cluster [DBG] osdmap e2191: 8 total, 8 up, 8 in 2024-09-13T00:43:57.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:56 smithi136 bash[76371]: cluster 2024-09-13T00:43:55.873048+0000 mon.a (mon.0) 3923 : cluster [DBG] osdmap e2191: 8 total, 8 up, 8 in 2024-09-13T00:43:57.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:56 smithi181 bash[67587]: cluster 2024-09-13T00:43:55.873048+0000 mon.a (mon.0) 3923 : cluster [DBG] osdmap e2191: 8 total, 8 up, 8 in 2024-09-13T00:43:58.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:57 smithi136 bash[75164]: cluster 2024-09-13T00:43:56.540301+0000 mgr.y (mgr.24661) 2004 : cluster [DBG] pgmap v3756: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 1.5 KiB/s wr, 7 op/s 2024-09-13T00:43:58.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:57 smithi136 bash[75164]: cluster 2024-09-13T00:43:56.889523+0000 mon.a (mon.0) 3924 : cluster [DBG] osdmap e2192: 8 total, 8 up, 8 in 2024-09-13T00:43:58.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:57 smithi136 bash[76371]: cluster 2024-09-13T00:43:56.540301+0000 mgr.y (mgr.24661) 2004 : cluster [DBG] pgmap v3756: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 1.5 KiB/s wr, 7 op/s 2024-09-13T00:43:58.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:57 smithi136 bash[76371]: cluster 2024-09-13T00:43:56.889523+0000 mon.a (mon.0) 3924 : cluster [DBG] osdmap e2192: 8 total, 8 up, 8 in 2024-09-13T00:43:58.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:57 smithi181 bash[67587]: cluster 2024-09-13T00:43:56.540301+0000 mgr.y (mgr.24661) 2004 : cluster [DBG] pgmap v3756: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 1.5 KiB/s wr, 7 op/s 2024-09-13T00:43:58.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:57 smithi181 bash[67587]: cluster 2024-09-13T00:43:56.889523+0000 mon.a (mon.0) 3924 : cluster [DBG] osdmap e2192: 8 total, 8 up, 8 in 2024-09-13T00:43:59.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:58 smithi136 bash[75164]: cluster 2024-09-13T00:43:57.891166+0000 mon.a (mon.0) 3925 : cluster [DBG] osdmap e2193: 8 total, 8 up, 8 in 2024-09-13T00:43:59.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:58 smithi136 bash[76371]: cluster 2024-09-13T00:43:57.891166+0000 mon.a (mon.0) 3925 : cluster [DBG] osdmap e2193: 8 total, 8 up, 8 in 2024-09-13T00:43:59.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:58 smithi181 bash[67587]: cluster 2024-09-13T00:43:57.891166+0000 mon.a (mon.0) 3925 : cluster [DBG] osdmap e2193: 8 total, 8 up, 8 in 2024-09-13T00:43:59.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:43:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:43:59] "GET /metrics HTTP/1.1" 200 36744 "" "Prometheus/2.43.0" 2024-09-13T00:44:00.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:59 smithi136 bash[75164]: cluster 2024-09-13T00:43:58.541008+0000 mgr.y (mgr.24661) 2005 : cluster [DBG] pgmap v3759: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 656 KiB/s rd, 3.7 KiB/s wr, 884 op/s 2024-09-13T00:44:00.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:43:59 smithi136 bash[75164]: cluster 2024-09-13T00:43:58.910911+0000 mon.a (mon.0) 3926 : cluster [DBG] osdmap e2194: 8 total, 8 up, 8 in 2024-09-13T00:44:00.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:59 smithi136 bash[76371]: cluster 2024-09-13T00:43:58.541008+0000 mgr.y (mgr.24661) 2005 : cluster [DBG] pgmap v3759: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 656 KiB/s rd, 3.7 KiB/s wr, 884 op/s 2024-09-13T00:44:00.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:43:59 smithi136 bash[76371]: cluster 2024-09-13T00:43:58.910911+0000 mon.a (mon.0) 3926 : cluster [DBG] osdmap e2194: 8 total, 8 up, 8 in 2024-09-13T00:44:00.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:59 smithi181 bash[67587]: cluster 2024-09-13T00:43:58.541008+0000 mgr.y (mgr.24661) 2005 : cluster [DBG] pgmap v3759: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 656 KiB/s rd, 3.7 KiB/s wr, 884 op/s 2024-09-13T00:44:00.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:43:59 smithi181 bash[67587]: cluster 2024-09-13T00:43:58.910911+0000 mon.a (mon.0) 3926 : cluster [DBG] osdmap e2194: 8 total, 8 up, 8 in 2024-09-13T00:44:01.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:00 smithi136 bash[75164]: cluster 2024-09-13T00:43:59.919533+0000 mon.a (mon.0) 3927 : cluster [DBG] osdmap e2195: 8 total, 8 up, 8 in 2024-09-13T00:44:01.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:00 smithi136 bash[76371]: cluster 2024-09-13T00:43:59.919533+0000 mon.a (mon.0) 3927 : cluster [DBG] osdmap e2195: 8 total, 8 up, 8 in 2024-09-13T00:44:01.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:00 smithi181 bash[67587]: cluster 2024-09-13T00:43:59.919533+0000 mon.a (mon.0) 3927 : cluster [DBG] osdmap e2195: 8 total, 8 up, 8 in 2024-09-13T00:44:02.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:01 smithi136 bash[75164]: cluster 2024-09-13T00:44:00.541685+0000 mgr.y (mgr.24661) 2006 : cluster [DBG] pgmap v3762: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 660 KiB/s rd, 2.7 KiB/s wr, 888 op/s 2024-09-13T00:44:02.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:01 smithi136 bash[75164]: cluster 2024-09-13T00:44:00.934737+0000 mon.a (mon.0) 3928 : cluster [DBG] osdmap e2196: 8 total, 8 up, 8 in 2024-09-13T00:44:02.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:01 smithi136 bash[76371]: cluster 2024-09-13T00:44:00.541685+0000 mgr.y (mgr.24661) 2006 : cluster [DBG] pgmap v3762: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 660 KiB/s rd, 2.7 KiB/s wr, 888 op/s 2024-09-13T00:44:02.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:01 smithi136 bash[76371]: cluster 2024-09-13T00:44:00.934737+0000 mon.a (mon.0) 3928 : cluster [DBG] osdmap e2196: 8 total, 8 up, 8 in 2024-09-13T00:44:02.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:01 smithi181 bash[67587]: cluster 2024-09-13T00:44:00.541685+0000 mgr.y (mgr.24661) 2006 : cluster [DBG] pgmap v3762: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 660 KiB/s rd, 2.7 KiB/s wr, 888 op/s 2024-09-13T00:44:02.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:01 smithi181 bash[67587]: cluster 2024-09-13T00:44:00.934737+0000 mon.a (mon.0) 3928 : cluster [DBG] osdmap e2196: 8 total, 8 up, 8 in 2024-09-13T00:44:02.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:44:02 smithi181 bash[95505]: ts=2024-09-13T00:44:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:44:03.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:02 smithi136 bash[75164]: cluster 2024-09-13T00:44:01.935792+0000 mon.a (mon.0) 3929 : cluster [DBG] osdmap e2197: 8 total, 8 up, 8 in 2024-09-13T00:44:03.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:02 smithi136 bash[76371]: cluster 2024-09-13T00:44:01.935792+0000 mon.a (mon.0) 3929 : cluster [DBG] osdmap e2197: 8 total, 8 up, 8 in 2024-09-13T00:44:03.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:02 smithi181 bash[67587]: cluster 2024-09-13T00:44:01.935792+0000 mon.a (mon.0) 3929 : cluster [DBG] osdmap e2197: 8 total, 8 up, 8 in 2024-09-13T00:44:04.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:03 smithi136 bash[75164]: cluster 2024-09-13T00:44:02.542578+0000 mgr.y (mgr.24661) 2007 : cluster [DBG] pgmap v3765: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 692 KiB/s rd, 3.7 KiB/s wr, 932 op/s 2024-09-13T00:44:04.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:03 smithi136 bash[75164]: cluster 2024-09-13T00:44:02.963593+0000 mon.a (mon.0) 3930 : cluster [DBG] osdmap e2198: 8 total, 8 up, 8 in 2024-09-13T00:44:04.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:03 smithi136 bash[76371]: cluster 2024-09-13T00:44:02.542578+0000 mgr.y (mgr.24661) 2007 : cluster [DBG] pgmap v3765: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 692 KiB/s rd, 3.7 KiB/s wr, 932 op/s 2024-09-13T00:44:04.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:03 smithi136 bash[76371]: cluster 2024-09-13T00:44:02.963593+0000 mon.a (mon.0) 3930 : cluster [DBG] osdmap e2198: 8 total, 8 up, 8 in 2024-09-13T00:44:04.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:03 smithi181 bash[67587]: cluster 2024-09-13T00:44:02.542578+0000 mgr.y (mgr.24661) 2007 : cluster [DBG] pgmap v3765: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 692 KiB/s rd, 3.7 KiB/s wr, 932 op/s 2024-09-13T00:44:04.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:03 smithi181 bash[67587]: cluster 2024-09-13T00:44:02.963593+0000 mon.a (mon.0) 3930 : cluster [DBG] osdmap e2198: 8 total, 8 up, 8 in 2024-09-13T00:44:05.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:04 smithi136 bash[75164]: cluster 2024-09-13T00:44:03.967660+0000 mon.a (mon.0) 3931 : cluster [DBG] osdmap e2199: 8 total, 8 up, 8 in 2024-09-13T00:44:05.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:04 smithi136 bash[76371]: cluster 2024-09-13T00:44:03.967660+0000 mon.a (mon.0) 3931 : cluster [DBG] osdmap e2199: 8 total, 8 up, 8 in 2024-09-13T00:44:05.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:04 smithi181 bash[67587]: cluster 2024-09-13T00:44:03.967660+0000 mon.a (mon.0) 3931 : cluster [DBG] osdmap e2199: 8 total, 8 up, 8 in 2024-09-13T00:44:06.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:05 smithi136 bash[75164]: cluster 2024-09-13T00:44:04.543313+0000 mgr.y (mgr.24661) 2008 : cluster [DBG] pgmap v3768: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 693 KiB/s rd, 4.0 KiB/s wr, 933 op/s 2024-09-13T00:44:06.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:05 smithi136 bash[75164]: cluster 2024-09-13T00:44:04.982270+0000 mon.a (mon.0) 3932 : cluster [DBG] osdmap e2200: 8 total, 8 up, 8 in 2024-09-13T00:44:06.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:05 smithi136 bash[76371]: cluster 2024-09-13T00:44:04.543313+0000 mgr.y (mgr.24661) 2008 : cluster [DBG] pgmap v3768: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 693 KiB/s rd, 4.0 KiB/s wr, 933 op/s 2024-09-13T00:44:06.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:05 smithi136 bash[76371]: cluster 2024-09-13T00:44:04.982270+0000 mon.a (mon.0) 3932 : cluster [DBG] osdmap e2200: 8 total, 8 up, 8 in 2024-09-13T00:44:06.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:05 smithi181 bash[67587]: cluster 2024-09-13T00:44:04.543313+0000 mgr.y (mgr.24661) 2008 : cluster [DBG] pgmap v3768: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 693 KiB/s rd, 4.0 KiB/s wr, 933 op/s 2024-09-13T00:44:06.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:05 smithi181 bash[67587]: cluster 2024-09-13T00:44:04.982270+0000 mon.a (mon.0) 3932 : cluster [DBG] osdmap e2200: 8 total, 8 up, 8 in 2024-09-13T00:44:07.259 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:06 smithi136 bash[76371]: cluster 2024-09-13T00:44:05.988424+0000 mon.a (mon.0) 3933 : cluster [DBG] osdmap e2201: 8 total, 8 up, 8 in 2024-09-13T00:44:07.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:06 smithi136 bash[75164]: cluster 2024-09-13T00:44:05.988424+0000 mon.a (mon.0) 3933 : cluster [DBG] osdmap e2201: 8 total, 8 up, 8 in 2024-09-13T00:44:07.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:06 smithi181 bash[67587]: cluster 2024-09-13T00:44:05.988424+0000 mon.a (mon.0) 3933 : cluster [DBG] osdmap e2201: 8 total, 8 up, 8 in 2024-09-13T00:44:08.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:08 smithi136 bash[75164]: cluster 2024-09-13T00:44:06.544191+0000 mgr.y (mgr.24661) 2009 : cluster [DBG] pgmap v3771: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 511 B/s wr, 6 op/s 2024-09-13T00:44:08.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:08 smithi136 bash[75164]: cluster 2024-09-13T00:44:07.000260+0000 mon.a (mon.0) 3934 : cluster [DBG] osdmap e2202: 8 total, 8 up, 8 in 2024-09-13T00:44:08.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:08 smithi136 bash[76371]: cluster 2024-09-13T00:44:06.544191+0000 mgr.y (mgr.24661) 2009 : cluster [DBG] pgmap v3771: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 511 B/s wr, 6 op/s 2024-09-13T00:44:08.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:08 smithi136 bash[76371]: cluster 2024-09-13T00:44:07.000260+0000 mon.a (mon.0) 3934 : cluster [DBG] osdmap e2202: 8 total, 8 up, 8 in 2024-09-13T00:44:08.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:08 smithi181 bash[67587]: cluster 2024-09-13T00:44:06.544191+0000 mgr.y (mgr.24661) 2009 : cluster [DBG] pgmap v3771: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 511 B/s wr, 6 op/s 2024-09-13T00:44:08.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:08 smithi181 bash[67587]: cluster 2024-09-13T00:44:07.000260+0000 mon.a (mon.0) 3934 : cluster [DBG] osdmap e2202: 8 total, 8 up, 8 in 2024-09-13T00:44:09.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:09 smithi136 bash[75164]: cluster 2024-09-13T00:44:08.001638+0000 mon.a (mon.0) 3935 : cluster [DBG] osdmap e2203: 8 total, 8 up, 8 in 2024-09-13T00:44:09.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:09 smithi136 bash[75164]: audit 2024-09-13T00:44:08.844692+0000 mon.a (mon.0) 3936 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:44:09.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:09 smithi136 bash[76371]: cluster 2024-09-13T00:44:08.001638+0000 mon.a (mon.0) 3935 : cluster [DBG] osdmap e2203: 8 total, 8 up, 8 in 2024-09-13T00:44:09.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:09 smithi136 bash[76371]: audit 2024-09-13T00:44:08.844692+0000 mon.a (mon.0) 3936 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:44:09.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:09 smithi181 bash[67587]: cluster 2024-09-13T00:44:08.001638+0000 mon.a (mon.0) 3935 : cluster [DBG] osdmap e2203: 8 total, 8 up, 8 in 2024-09-13T00:44:09.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:09 smithi181 bash[67587]: audit 2024-09-13T00:44:08.844692+0000 mon.a (mon.0) 3936 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:44:09.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:44:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:44:09] "GET /metrics HTTP/1.1" 200 36744 "" "Prometheus/2.43.0" 2024-09-13T00:44:10.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:10 smithi181 bash[67587]: cluster 2024-09-13T00:44:08.544957+0000 mgr.y (mgr.24661) 2010 : cluster [DBG] pgmap v3774: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 575 KiB/s rd, 2.2 KiB/s wr, 773 op/s 2024-09-13T00:44:10.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:10 smithi181 bash[67587]: cluster 2024-09-13T00:44:09.020813+0000 mon.a (mon.0) 3937 : cluster [DBG] osdmap e2204: 8 total, 8 up, 8 in 2024-09-13T00:44:10.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:10 smithi136 bash[75164]: cluster 2024-09-13T00:44:08.544957+0000 mgr.y (mgr.24661) 2010 : cluster [DBG] pgmap v3774: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 575 KiB/s rd, 2.2 KiB/s wr, 773 op/s 2024-09-13T00:44:10.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:10 smithi136 bash[75164]: cluster 2024-09-13T00:44:09.020813+0000 mon.a (mon.0) 3937 : cluster [DBG] osdmap e2204: 8 total, 8 up, 8 in 2024-09-13T00:44:10.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:10 smithi136 bash[76371]: cluster 2024-09-13T00:44:08.544957+0000 mgr.y (mgr.24661) 2010 : cluster [DBG] pgmap v3774: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 575 KiB/s rd, 2.2 KiB/s wr, 773 op/s 2024-09-13T00:44:10.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:10 smithi136 bash[76371]: cluster 2024-09-13T00:44:09.020813+0000 mon.a (mon.0) 3937 : cluster [DBG] osdmap e2204: 8 total, 8 up, 8 in 2024-09-13T00:44:11.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:11 smithi181 bash[67587]: cluster 2024-09-13T00:44:10.025318+0000 mon.a (mon.0) 3938 : cluster [DBG] osdmap e2205: 8 total, 8 up, 8 in 2024-09-13T00:44:11.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:11 smithi136 bash[75164]: cluster 2024-09-13T00:44:10.025318+0000 mon.a (mon.0) 3938 : cluster [DBG] osdmap e2205: 8 total, 8 up, 8 in 2024-09-13T00:44:11.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:11 smithi136 bash[76371]: cluster 2024-09-13T00:44:10.025318+0000 mon.a (mon.0) 3938 : cluster [DBG] osdmap e2205: 8 total, 8 up, 8 in 2024-09-13T00:44:12.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:12 smithi181 bash[67587]: cluster 2024-09-13T00:44:10.545723+0000 mgr.y (mgr.24661) 2011 : cluster [DBG] pgmap v3777: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 579 KiB/s rd, 3.0 KiB/s wr, 779 op/s 2024-09-13T00:44:12.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:12 smithi181 bash[67587]: cluster 2024-09-13T00:44:11.041103+0000 mon.a (mon.0) 3939 : cluster [DBG] osdmap e2206: 8 total, 8 up, 8 in 2024-09-13T00:44:12.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:12 smithi136 bash[75164]: cluster 2024-09-13T00:44:10.545723+0000 mgr.y (mgr.24661) 2011 : cluster [DBG] pgmap v3777: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 579 KiB/s rd, 3.0 KiB/s wr, 779 op/s 2024-09-13T00:44:12.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:12 smithi136 bash[75164]: cluster 2024-09-13T00:44:11.041103+0000 mon.a (mon.0) 3939 : cluster [DBG] osdmap e2206: 8 total, 8 up, 8 in 2024-09-13T00:44:12.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:12 smithi136 bash[76371]: cluster 2024-09-13T00:44:10.545723+0000 mgr.y (mgr.24661) 2011 : cluster [DBG] pgmap v3777: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 579 KiB/s rd, 3.0 KiB/s wr, 779 op/s 2024-09-13T00:44:12.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:12 smithi136 bash[76371]: cluster 2024-09-13T00:44:11.041103+0000 mon.a (mon.0) 3939 : cluster [DBG] osdmap e2206: 8 total, 8 up, 8 in 2024-09-13T00:44:12.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:44:12 smithi181 bash[95505]: ts=2024-09-13T00:44:12.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:44:13.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:13 smithi181 bash[67587]: cluster 2024-09-13T00:44:12.069470+0000 mon.a (mon.0) 3940 : cluster [DBG] osdmap e2207: 8 total, 8 up, 8 in 2024-09-13T00:44:13.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:13 smithi136 bash[75164]: cluster 2024-09-13T00:44:12.069470+0000 mon.a (mon.0) 3940 : cluster [DBG] osdmap e2207: 8 total, 8 up, 8 in 2024-09-13T00:44:13.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:13 smithi136 bash[76371]: cluster 2024-09-13T00:44:12.069470+0000 mon.a (mon.0) 3940 : cluster [DBG] osdmap e2207: 8 total, 8 up, 8 in 2024-09-13T00:44:14.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:14 smithi181 bash[67587]: cluster 2024-09-13T00:44:12.546481+0000 mgr.y (mgr.24661) 2012 : cluster [DBG] pgmap v3780: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 751 KiB/s rd, 4.2 KiB/s wr, 1.01k op/s 2024-09-13T00:44:14.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:14 smithi181 bash[67587]: cluster 2024-09-13T00:44:13.078164+0000 mon.a (mon.0) 3941 : cluster [DBG] osdmap e2208: 8 total, 8 up, 8 in 2024-09-13T00:44:14.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:14 smithi136 bash[75164]: cluster 2024-09-13T00:44:12.546481+0000 mgr.y (mgr.24661) 2012 : cluster [DBG] pgmap v3780: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 751 KiB/s rd, 4.2 KiB/s wr, 1.01k op/s 2024-09-13T00:44:14.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:14 smithi136 bash[75164]: cluster 2024-09-13T00:44:13.078164+0000 mon.a (mon.0) 3941 : cluster [DBG] osdmap e2208: 8 total, 8 up, 8 in 2024-09-13T00:44:14.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:14 smithi136 bash[76371]: cluster 2024-09-13T00:44:12.546481+0000 mgr.y (mgr.24661) 2012 : cluster [DBG] pgmap v3780: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 751 KiB/s rd, 4.2 KiB/s wr, 1.01k op/s 2024-09-13T00:44:14.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:14 smithi136 bash[76371]: cluster 2024-09-13T00:44:13.078164+0000 mon.a (mon.0) 3941 : cluster [DBG] osdmap e2208: 8 total, 8 up, 8 in 2024-09-13T00:44:15.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:15 smithi181 bash[67587]: cluster 2024-09-13T00:44:14.077887+0000 mon.a (mon.0) 3942 : cluster [DBG] osdmap e2209: 8 total, 8 up, 8 in 2024-09-13T00:44:15.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:15 smithi136 bash[75164]: cluster 2024-09-13T00:44:14.077887+0000 mon.a (mon.0) 3942 : cluster [DBG] osdmap e2209: 8 total, 8 up, 8 in 2024-09-13T00:44:15.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:15 smithi136 bash[76371]: cluster 2024-09-13T00:44:14.077887+0000 mon.a (mon.0) 3942 : cluster [DBG] osdmap e2209: 8 total, 8 up, 8 in 2024-09-13T00:44:16.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:16 smithi136 bash[75164]: cluster 2024-09-13T00:44:14.547152+0000 mgr.y (mgr.24661) 2013 : cluster [DBG] pgmap v3783: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 752 KiB/s rd, 3.2 KiB/s wr, 1.01k op/s 2024-09-13T00:44:16.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:16 smithi136 bash[75164]: cluster 2024-09-13T00:44:15.099633+0000 mon.a (mon.0) 3943 : cluster [DBG] osdmap e2210: 8 total, 8 up, 8 in 2024-09-13T00:44:16.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:16 smithi136 bash[76371]: cluster 2024-09-13T00:44:14.547152+0000 mgr.y (mgr.24661) 2013 : cluster [DBG] pgmap v3783: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 752 KiB/s rd, 3.2 KiB/s wr, 1.01k op/s 2024-09-13T00:44:16.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:16 smithi136 bash[76371]: cluster 2024-09-13T00:44:15.099633+0000 mon.a (mon.0) 3943 : cluster [DBG] osdmap e2210: 8 total, 8 up, 8 in 2024-09-13T00:44:16.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:16 smithi181 bash[67587]: cluster 2024-09-13T00:44:14.547152+0000 mgr.y (mgr.24661) 2013 : cluster [DBG] pgmap v3783: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 752 KiB/s rd, 3.2 KiB/s wr, 1.01k op/s 2024-09-13T00:44:16.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:16 smithi181 bash[67587]: cluster 2024-09-13T00:44:15.099633+0000 mon.a (mon.0) 3943 : cluster [DBG] osdmap e2210: 8 total, 8 up, 8 in 2024-09-13T00:44:17.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:17 smithi136 bash[75164]: cluster 2024-09-13T00:44:16.102998+0000 mon.a (mon.0) 3944 : cluster [DBG] osdmap e2211: 8 total, 8 up, 8 in 2024-09-13T00:44:17.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:17 smithi136 bash[76371]: cluster 2024-09-13T00:44:16.102998+0000 mon.a (mon.0) 3944 : cluster [DBG] osdmap e2211: 8 total, 8 up, 8 in 2024-09-13T00:44:17.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:17 smithi181 bash[67587]: cluster 2024-09-13T00:44:16.102998+0000 mon.a (mon.0) 3944 : cluster [DBG] osdmap e2211: 8 total, 8 up, 8 in 2024-09-13T00:44:18.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:18 smithi136 bash[75164]: cluster 2024-09-13T00:44:16.547978+0000 mgr.y (mgr.24661) 2014 : cluster [DBG] pgmap v3786: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 1023 B/s wr, 7 op/s 2024-09-13T00:44:18.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:18 smithi136 bash[75164]: cluster 2024-09-13T00:44:17.125074+0000 mon.a (mon.0) 3945 : cluster [DBG] osdmap e2212: 8 total, 8 up, 8 in 2024-09-13T00:44:18.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:18 smithi136 bash[76371]: cluster 2024-09-13T00:44:16.547978+0000 mgr.y (mgr.24661) 2014 : cluster [DBG] pgmap v3786: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 1023 B/s wr, 7 op/s 2024-09-13T00:44:18.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:18 smithi136 bash[76371]: cluster 2024-09-13T00:44:17.125074+0000 mon.a (mon.0) 3945 : cluster [DBG] osdmap e2212: 8 total, 8 up, 8 in 2024-09-13T00:44:18.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:18 smithi181 bash[67587]: cluster 2024-09-13T00:44:16.547978+0000 mgr.y (mgr.24661) 2014 : cluster [DBG] pgmap v3786: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 1023 B/s wr, 7 op/s 2024-09-13T00:44:18.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:18 smithi181 bash[67587]: cluster 2024-09-13T00:44:17.125074+0000 mon.a (mon.0) 3945 : cluster [DBG] osdmap e2212: 8 total, 8 up, 8 in 2024-09-13T00:44:19.414 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:19 smithi136 bash[75164]: cluster 2024-09-13T00:44:18.125275+0000 mon.a (mon.0) 3946 : cluster [DBG] osdmap e2213: 8 total, 8 up, 8 in 2024-09-13T00:44:19.415 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:19 smithi136 bash[76371]: cluster 2024-09-13T00:44:18.125275+0000 mon.a (mon.0) 3946 : cluster [DBG] osdmap e2213: 8 total, 8 up, 8 in 2024-09-13T00:44:19.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:19 smithi181 bash[67587]: cluster 2024-09-13T00:44:18.125275+0000 mon.a (mon.0) 3946 : cluster [DBG] osdmap e2213: 8 total, 8 up, 8 in 2024-09-13T00:44:19.759 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:44:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:44:19] "GET /metrics HTTP/1.1" 200 36744 "" "Prometheus/2.43.0" 2024-09-13T00:44:20.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:20 smithi136 bash[75164]: cluster 2024-09-13T00:44:18.548880+0000 mgr.y (mgr.24661) 2015 : cluster [DBG] pgmap v3789: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 783 KiB/s rd, 4.2 KiB/s wr, 1.06k op/s 2024-09-13T00:44:20.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:20 smithi136 bash[75164]: cluster 2024-09-13T00:44:19.148100+0000 mon.a (mon.0) 3947 : cluster [DBG] osdmap e2214: 8 total, 8 up, 8 in 2024-09-13T00:44:20.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:20 smithi136 bash[76371]: cluster 2024-09-13T00:44:18.548880+0000 mgr.y (mgr.24661) 2015 : cluster [DBG] pgmap v3789: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 783 KiB/s rd, 4.2 KiB/s wr, 1.06k op/s 2024-09-13T00:44:20.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:20 smithi136 bash[76371]: cluster 2024-09-13T00:44:19.148100+0000 mon.a (mon.0) 3947 : cluster [DBG] osdmap e2214: 8 total, 8 up, 8 in 2024-09-13T00:44:20.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:20 smithi181 bash[67587]: cluster 2024-09-13T00:44:18.548880+0000 mgr.y (mgr.24661) 2015 : cluster [DBG] pgmap v3789: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 783 KiB/s rd, 4.2 KiB/s wr, 1.06k op/s 2024-09-13T00:44:20.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:20 smithi181 bash[67587]: cluster 2024-09-13T00:44:19.148100+0000 mon.a (mon.0) 3947 : cluster [DBG] osdmap e2214: 8 total, 8 up, 8 in 2024-09-13T00:44:21.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:21 smithi136 bash[75164]: cluster 2024-09-13T00:44:20.157775+0000 mon.a (mon.0) 3948 : cluster [DBG] osdmap e2215: 8 total, 8 up, 8 in 2024-09-13T00:44:21.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:21 smithi136 bash[76371]: cluster 2024-09-13T00:44:20.157775+0000 mon.a (mon.0) 3948 : cluster [DBG] osdmap e2215: 8 total, 8 up, 8 in 2024-09-13T00:44:21.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:21 smithi181 bash[67587]: cluster 2024-09-13T00:44:20.157775+0000 mon.a (mon.0) 3948 : cluster [DBG] osdmap e2215: 8 total, 8 up, 8 in 2024-09-13T00:44:22.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:22 smithi136 bash[75164]: cluster 2024-09-13T00:44:20.549718+0000 mgr.y (mgr.24661) 2016 : cluster [DBG] pgmap v3792: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 783 KiB/s rd, 3.2 KiB/s wr, 1.05k op/s 2024-09-13T00:44:22.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:22 smithi136 bash[75164]: cluster 2024-09-13T00:44:21.167776+0000 mon.a (mon.0) 3949 : cluster [DBG] osdmap e2216: 8 total, 8 up, 8 in 2024-09-13T00:44:22.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:22 smithi136 bash[76371]: cluster 2024-09-13T00:44:20.549718+0000 mgr.y (mgr.24661) 2016 : cluster [DBG] pgmap v3792: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 783 KiB/s rd, 3.2 KiB/s wr, 1.05k op/s 2024-09-13T00:44:22.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:22 smithi136 bash[76371]: cluster 2024-09-13T00:44:21.167776+0000 mon.a (mon.0) 3949 : cluster [DBG] osdmap e2216: 8 total, 8 up, 8 in 2024-09-13T00:44:22.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:22 smithi181 bash[67587]: cluster 2024-09-13T00:44:20.549718+0000 mgr.y (mgr.24661) 2016 : cluster [DBG] pgmap v3792: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 783 KiB/s rd, 3.2 KiB/s wr, 1.05k op/s 2024-09-13T00:44:22.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:22 smithi181 bash[67587]: cluster 2024-09-13T00:44:21.167776+0000 mon.a (mon.0) 3949 : cluster [DBG] osdmap e2216: 8 total, 8 up, 8 in 2024-09-13T00:44:22.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:44:22 smithi181 bash[95505]: ts=2024-09-13T00:44:22.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:44:23.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:23 smithi136 bash[75164]: cluster 2024-09-13T00:44:22.176819+0000 mon.a (mon.0) 3950 : cluster [DBG] osdmap e2217: 8 total, 8 up, 8 in 2024-09-13T00:44:23.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:23 smithi136 bash[76371]: cluster 2024-09-13T00:44:22.176819+0000 mon.a (mon.0) 3950 : cluster [DBG] osdmap e2217: 8 total, 8 up, 8 in 2024-09-13T00:44:23.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:23 smithi181 bash[67587]: cluster 2024-09-13T00:44:22.176819+0000 mon.a (mon.0) 3950 : cluster [DBG] osdmap e2217: 8 total, 8 up, 8 in 2024-09-13T00:44:24.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:24 smithi136 bash[75164]: cluster 2024-09-13T00:44:22.550459+0000 mgr.y (mgr.24661) 2017 : cluster [DBG] pgmap v3795: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 812 KiB/s rd, 3.5 KiB/s wr, 1.09k op/s 2024-09-13T00:44:24.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:24 smithi136 bash[75164]: cluster 2024-09-13T00:44:23.187128+0000 mon.a (mon.0) 3951 : cluster [DBG] osdmap e2218: 8 total, 8 up, 8 in 2024-09-13T00:44:24.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:24 smithi136 bash[75164]: audit 2024-09-13T00:44:23.845329+0000 mon.a (mon.0) 3952 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:44:24.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:24 smithi136 bash[76371]: cluster 2024-09-13T00:44:22.550459+0000 mgr.y (mgr.24661) 2017 : cluster [DBG] pgmap v3795: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 812 KiB/s rd, 3.5 KiB/s wr, 1.09k op/s 2024-09-13T00:44:24.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:24 smithi136 bash[76371]: cluster 2024-09-13T00:44:23.187128+0000 mon.a (mon.0) 3951 : cluster [DBG] osdmap e2218: 8 total, 8 up, 8 in 2024-09-13T00:44:24.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:24 smithi136 bash[76371]: audit 2024-09-13T00:44:23.845329+0000 mon.a (mon.0) 3952 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:44:24.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:24 smithi181 bash[67587]: cluster 2024-09-13T00:44:22.550459+0000 mgr.y (mgr.24661) 2017 : cluster [DBG] pgmap v3795: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 812 KiB/s rd, 3.5 KiB/s wr, 1.09k op/s 2024-09-13T00:44:24.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:24 smithi181 bash[67587]: cluster 2024-09-13T00:44:23.187128+0000 mon.a (mon.0) 3951 : cluster [DBG] osdmap e2218: 8 total, 8 up, 8 in 2024-09-13T00:44:24.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:24 smithi181 bash[67587]: audit 2024-09-13T00:44:23.845329+0000 mon.a (mon.0) 3952 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:44:25.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:25 smithi136 bash[75164]: cluster 2024-09-13T00:44:24.188008+0000 mon.a (mon.0) 3953 : cluster [DBG] osdmap e2219: 8 total, 8 up, 8 in 2024-09-13T00:44:25.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:25 smithi136 bash[76371]: cluster 2024-09-13T00:44:24.188008+0000 mon.a (mon.0) 3953 : cluster [DBG] osdmap e2219: 8 total, 8 up, 8 in 2024-09-13T00:44:25.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:25 smithi181 bash[67587]: cluster 2024-09-13T00:44:24.188008+0000 mon.a (mon.0) 3953 : cluster [DBG] osdmap e2219: 8 total, 8 up, 8 in 2024-09-13T00:44:26.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:26 smithi136 bash[75164]: cluster 2024-09-13T00:44:24.551106+0000 mgr.y (mgr.24661) 2018 : cluster [DBG] pgmap v3798: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 817 KiB/s rd, 3.7 KiB/s wr, 1.10k op/s 2024-09-13T00:44:26.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:26 smithi136 bash[75164]: cluster 2024-09-13T00:44:25.214333+0000 mon.a (mon.0) 3954 : cluster [DBG] osdmap e2220: 8 total, 8 up, 8 in 2024-09-13T00:44:26.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:26 smithi136 bash[76371]: cluster 2024-09-13T00:44:24.551106+0000 mgr.y (mgr.24661) 2018 : cluster [DBG] pgmap v3798: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 817 KiB/s rd, 3.7 KiB/s wr, 1.10k op/s 2024-09-13T00:44:26.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:26 smithi136 bash[76371]: cluster 2024-09-13T00:44:25.214333+0000 mon.a (mon.0) 3954 : cluster [DBG] osdmap e2220: 8 total, 8 up, 8 in 2024-09-13T00:44:26.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:26 smithi181 bash[67587]: cluster 2024-09-13T00:44:24.551106+0000 mgr.y (mgr.24661) 2018 : cluster [DBG] pgmap v3798: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 817 KiB/s rd, 3.7 KiB/s wr, 1.10k op/s 2024-09-13T00:44:26.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:26 smithi181 bash[67587]: cluster 2024-09-13T00:44:25.214333+0000 mon.a (mon.0) 3954 : cluster [DBG] osdmap e2220: 8 total, 8 up, 8 in 2024-09-13T00:44:27.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:27 smithi136 bash[75164]: cluster 2024-09-13T00:44:26.222943+0000 mon.a (mon.0) 3955 : cluster [DBG] osdmap e2221: 8 total, 8 up, 8 in 2024-09-13T00:44:27.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:27 smithi136 bash[76371]: cluster 2024-09-13T00:44:26.222943+0000 mon.a (mon.0) 3955 : cluster [DBG] osdmap e2221: 8 total, 8 up, 8 in 2024-09-13T00:44:27.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:27 smithi181 bash[67587]: cluster 2024-09-13T00:44:26.222943+0000 mon.a (mon.0) 3955 : cluster [DBG] osdmap e2221: 8 total, 8 up, 8 in 2024-09-13T00:44:28.509 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:28 smithi136 bash[76371]: cluster 2024-09-13T00:44:26.551868+0000 mgr.y (mgr.24661) 2019 : cluster [DBG] pgmap v3801: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 767 B/s wr, 6 op/s 2024-09-13T00:44:28.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:28 smithi136 bash[76371]: cluster 2024-09-13T00:44:27.233564+0000 mon.a (mon.0) 3956 : cluster [DBG] osdmap e2222: 8 total, 8 up, 8 in 2024-09-13T00:44:28.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:28 smithi136 bash[75164]: cluster 2024-09-13T00:44:26.551868+0000 mgr.y (mgr.24661) 2019 : cluster [DBG] pgmap v3801: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 767 B/s wr, 6 op/s 2024-09-13T00:44:28.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:28 smithi136 bash[75164]: cluster 2024-09-13T00:44:27.233564+0000 mon.a (mon.0) 3956 : cluster [DBG] osdmap e2222: 8 total, 8 up, 8 in 2024-09-13T00:44:28.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:28 smithi181 bash[67587]: cluster 2024-09-13T00:44:26.551868+0000 mgr.y (mgr.24661) 2019 : cluster [DBG] pgmap v3801: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 767 B/s wr, 6 op/s 2024-09-13T00:44:28.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:28 smithi181 bash[67587]: cluster 2024-09-13T00:44:27.233564+0000 mon.a (mon.0) 3956 : cluster [DBG] osdmap e2222: 8 total, 8 up, 8 in 2024-09-13T00:44:29.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:29 smithi136 bash[75164]: cluster 2024-09-13T00:44:28.240221+0000 mon.a (mon.0) 3957 : cluster [DBG] osdmap e2223: 8 total, 8 up, 8 in 2024-09-13T00:44:29.510 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:44:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:44:29] "GET /metrics HTTP/1.1" 200 36745 "" "Prometheus/2.43.0" 2024-09-13T00:44:29.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:29 smithi136 bash[76371]: cluster 2024-09-13T00:44:28.240221+0000 mon.a (mon.0) 3957 : cluster [DBG] osdmap e2223: 8 total, 8 up, 8 in 2024-09-13T00:44:29.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:29 smithi181 bash[67587]: cluster 2024-09-13T00:44:28.240221+0000 mon.a (mon.0) 3957 : cluster [DBG] osdmap e2223: 8 total, 8 up, 8 in 2024-09-13T00:44:30.509 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:30 smithi136 bash[75164]: cluster 2024-09-13T00:44:28.552743+0000 mgr.y (mgr.24661) 2020 : cluster [DBG] pgmap v3804: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 848 KiB/s rd, 3.7 KiB/s wr, 1.14k op/s 2024-09-13T00:44:30.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:30 smithi136 bash[75164]: cluster 2024-09-13T00:44:29.250197+0000 mon.a (mon.0) 3958 : cluster [DBG] osdmap e2224: 8 total, 8 up, 8 in 2024-09-13T00:44:30.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:30 smithi136 bash[76371]: cluster 2024-09-13T00:44:28.552743+0000 mgr.y (mgr.24661) 2020 : cluster [DBG] pgmap v3804: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 848 KiB/s rd, 3.7 KiB/s wr, 1.14k op/s 2024-09-13T00:44:30.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:30 smithi136 bash[76371]: cluster 2024-09-13T00:44:29.250197+0000 mon.a (mon.0) 3958 : cluster [DBG] osdmap e2224: 8 total, 8 up, 8 in 2024-09-13T00:44:30.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:30 smithi181 bash[67587]: cluster 2024-09-13T00:44:28.552743+0000 mgr.y (mgr.24661) 2020 : cluster [DBG] pgmap v3804: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 848 KiB/s rd, 3.7 KiB/s wr, 1.14k op/s 2024-09-13T00:44:30.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:30 smithi181 bash[67587]: cluster 2024-09-13T00:44:29.250197+0000 mon.a (mon.0) 3958 : cluster [DBG] osdmap e2224: 8 total, 8 up, 8 in 2024-09-13T00:44:31.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:31 smithi181 bash[67587]: cluster 2024-09-13T00:44:30.251668+0000 mon.a (mon.0) 3959 : cluster [DBG] osdmap e2225: 8 total, 8 up, 8 in 2024-09-13T00:44:31.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:31 smithi136 bash[75164]: cluster 2024-09-13T00:44:30.251668+0000 mon.a (mon.0) 3959 : cluster [DBG] osdmap e2225: 8 total, 8 up, 8 in 2024-09-13T00:44:31.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:31 smithi136 bash[76371]: cluster 2024-09-13T00:44:30.251668+0000 mon.a (mon.0) 3959 : cluster [DBG] osdmap e2225: 8 total, 8 up, 8 in 2024-09-13T00:44:32.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:32 smithi181 bash[67587]: cluster 2024-09-13T00:44:30.553386+0000 mgr.y (mgr.24661) 2021 : cluster [DBG] pgmap v3807: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 848 KiB/s rd, 3.2 KiB/s wr, 1.14k op/s 2024-09-13T00:44:32.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:32 smithi181 bash[67587]: cluster 2024-09-13T00:44:31.273438+0000 mon.a (mon.0) 3960 : cluster [DBG] osdmap e2226: 8 total, 8 up, 8 in 2024-09-13T00:44:32.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:44:32 smithi181 bash[95505]: ts=2024-09-13T00:44:32.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:44:32.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:32 smithi136 bash[75164]: cluster 2024-09-13T00:44:30.553386+0000 mgr.y (mgr.24661) 2021 : cluster [DBG] pgmap v3807: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 848 KiB/s rd, 3.2 KiB/s wr, 1.14k op/s 2024-09-13T00:44:32.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:32 smithi136 bash[75164]: cluster 2024-09-13T00:44:31.273438+0000 mon.a (mon.0) 3960 : cluster [DBG] osdmap e2226: 8 total, 8 up, 8 in 2024-09-13T00:44:32.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:32 smithi136 bash[76371]: cluster 2024-09-13T00:44:30.553386+0000 mgr.y (mgr.24661) 2021 : cluster [DBG] pgmap v3807: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 848 KiB/s rd, 3.2 KiB/s wr, 1.14k op/s 2024-09-13T00:44:32.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:32 smithi136 bash[76371]: cluster 2024-09-13T00:44:31.273438+0000 mon.a (mon.0) 3960 : cluster [DBG] osdmap e2226: 8 total, 8 up, 8 in 2024-09-13T00:44:33.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:33 smithi181 bash[67587]: cluster 2024-09-13T00:44:32.281056+0000 mon.a (mon.0) 3961 : cluster [DBG] osdmap e2227: 8 total, 8 up, 8 in 2024-09-13T00:44:33.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:33 smithi136 bash[75164]: cluster 2024-09-13T00:44:32.281056+0000 mon.a (mon.0) 3961 : cluster [DBG] osdmap e2227: 8 total, 8 up, 8 in 2024-09-13T00:44:33.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:33 smithi136 bash[76371]: cluster 2024-09-13T00:44:32.281056+0000 mon.a (mon.0) 3961 : cluster [DBG] osdmap e2227: 8 total, 8 up, 8 in 2024-09-13T00:44:34.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:34 smithi181 bash[67587]: cluster 2024-09-13T00:44:32.554065+0000 mgr.y (mgr.24661) 2022 : cluster [DBG] pgmap v3810: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 878 KiB/s rd, 3.0 KiB/s wr, 1.18k op/s 2024-09-13T00:44:34.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:34 smithi181 bash[67587]: cluster 2024-09-13T00:44:33.293133+0000 mon.a (mon.0) 3962 : cluster [DBG] osdmap e2228: 8 total, 8 up, 8 in 2024-09-13T00:44:34.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:34 smithi136 bash[75164]: cluster 2024-09-13T00:44:32.554065+0000 mgr.y (mgr.24661) 2022 : cluster [DBG] pgmap v3810: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 878 KiB/s rd, 3.0 KiB/s wr, 1.18k op/s 2024-09-13T00:44:34.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:34 smithi136 bash[75164]: cluster 2024-09-13T00:44:33.293133+0000 mon.a (mon.0) 3962 : cluster [DBG] osdmap e2228: 8 total, 8 up, 8 in 2024-09-13T00:44:34.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:34 smithi136 bash[76371]: cluster 2024-09-13T00:44:32.554065+0000 mgr.y (mgr.24661) 2022 : cluster [DBG] pgmap v3810: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 878 KiB/s rd, 3.0 KiB/s wr, 1.18k op/s 2024-09-13T00:44:34.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:34 smithi136 bash[76371]: cluster 2024-09-13T00:44:33.293133+0000 mon.a (mon.0) 3962 : cluster [DBG] osdmap e2228: 8 total, 8 up, 8 in 2024-09-13T00:44:35.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:35 smithi181 bash[67587]: cluster 2024-09-13T00:44:34.303984+0000 mon.a (mon.0) 3963 : cluster [DBG] osdmap e2229: 8 total, 8 up, 8 in 2024-09-13T00:44:35.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:35 smithi136 bash[75164]: cluster 2024-09-13T00:44:34.303984+0000 mon.a (mon.0) 3963 : cluster [DBG] osdmap e2229: 8 total, 8 up, 8 in 2024-09-13T00:44:35.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:35 smithi136 bash[76371]: cluster 2024-09-13T00:44:34.303984+0000 mon.a (mon.0) 3963 : cluster [DBG] osdmap e2229: 8 total, 8 up, 8 in 2024-09-13T00:44:36.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:36 smithi181 bash[67587]: cluster 2024-09-13T00:44:34.554742+0000 mgr.y (mgr.24661) 2023 : cluster [DBG] pgmap v3813: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 883 KiB/s rd, 3.7 KiB/s wr, 1.19k op/s 2024-09-13T00:44:36.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:36 smithi181 bash[67587]: cluster 2024-09-13T00:44:35.315733+0000 mon.a (mon.0) 3964 : cluster [DBG] osdmap e2230: 8 total, 8 up, 8 in 2024-09-13T00:44:36.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:36 smithi136 bash[75164]: cluster 2024-09-13T00:44:34.554742+0000 mgr.y (mgr.24661) 2023 : cluster [DBG] pgmap v3813: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 883 KiB/s rd, 3.7 KiB/s wr, 1.19k op/s 2024-09-13T00:44:36.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:36 smithi136 bash[75164]: cluster 2024-09-13T00:44:35.315733+0000 mon.a (mon.0) 3964 : cluster [DBG] osdmap e2230: 8 total, 8 up, 8 in 2024-09-13T00:44:36.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:36 smithi136 bash[76371]: cluster 2024-09-13T00:44:34.554742+0000 mgr.y (mgr.24661) 2023 : cluster [DBG] pgmap v3813: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 883 KiB/s rd, 3.7 KiB/s wr, 1.19k op/s 2024-09-13T00:44:36.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:36 smithi136 bash[76371]: cluster 2024-09-13T00:44:35.315733+0000 mon.a (mon.0) 3964 : cluster [DBG] osdmap e2230: 8 total, 8 up, 8 in 2024-09-13T00:44:37.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:37 smithi181 bash[67587]: cluster 2024-09-13T00:44:36.316928+0000 mon.a (mon.0) 3965 : cluster [DBG] osdmap e2231: 8 total, 8 up, 8 in 2024-09-13T00:44:37.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:37 smithi136 bash[75164]: cluster 2024-09-13T00:44:36.316928+0000 mon.a (mon.0) 3965 : cluster [DBG] osdmap e2231: 8 total, 8 up, 8 in 2024-09-13T00:44:37.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:37 smithi136 bash[76371]: cluster 2024-09-13T00:44:36.316928+0000 mon.a (mon.0) 3965 : cluster [DBG] osdmap e2231: 8 total, 8 up, 8 in 2024-09-13T00:44:38.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:38 smithi181 bash[67587]: cluster 2024-09-13T00:44:36.555577+0000 mgr.y (mgr.24661) 2024 : cluster [DBG] pgmap v3816: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 1.2 KiB/s wr, 7 op/s 2024-09-13T00:44:38.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:38 smithi181 bash[67587]: cluster 2024-09-13T00:44:37.342374+0000 mon.a (mon.0) 3966 : cluster [DBG] osdmap e2232: 8 total, 8 up, 8 in 2024-09-13T00:44:38.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:38 smithi136 bash[75164]: cluster 2024-09-13T00:44:36.555577+0000 mgr.y (mgr.24661) 2024 : cluster [DBG] pgmap v3816: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 1.2 KiB/s wr, 7 op/s 2024-09-13T00:44:38.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:38 smithi136 bash[75164]: cluster 2024-09-13T00:44:37.342374+0000 mon.a (mon.0) 3966 : cluster [DBG] osdmap e2232: 8 total, 8 up, 8 in 2024-09-13T00:44:38.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:38 smithi136 bash[76371]: cluster 2024-09-13T00:44:36.555577+0000 mgr.y (mgr.24661) 2024 : cluster [DBG] pgmap v3816: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 1.2 KiB/s wr, 7 op/s 2024-09-13T00:44:38.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:38 smithi136 bash[76371]: cluster 2024-09-13T00:44:37.342374+0000 mon.a (mon.0) 3966 : cluster [DBG] osdmap e2232: 8 total, 8 up, 8 in 2024-09-13T00:44:39.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:44:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:44:39] "GET /metrics HTTP/1.1" 200 36744 "" "Prometheus/2.43.0" 2024-09-13T00:44:39.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:39 smithi136 bash[76371]: cluster 2024-09-13T00:44:38.347462+0000 mon.a (mon.0) 3967 : cluster [DBG] osdmap e2233: 8 total, 8 up, 8 in 2024-09-13T00:44:39.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:39 smithi136 bash[76371]: audit 2024-09-13T00:44:38.845760+0000 mon.a (mon.0) 3968 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:44:39.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:39 smithi136 bash[75164]: cluster 2024-09-13T00:44:38.347462+0000 mon.a (mon.0) 3967 : cluster [DBG] osdmap e2233: 8 total, 8 up, 8 in 2024-09-13T00:44:39.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:39 smithi136 bash[75164]: audit 2024-09-13T00:44:38.845760+0000 mon.a (mon.0) 3968 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:44:39.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:39 smithi181 bash[67587]: cluster 2024-09-13T00:44:38.347462+0000 mon.a (mon.0) 3967 : cluster [DBG] osdmap e2233: 8 total, 8 up, 8 in 2024-09-13T00:44:39.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:39 smithi181 bash[67587]: audit 2024-09-13T00:44:38.845760+0000 mon.a (mon.0) 3968 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:44:40.759 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:40 smithi136 bash[76371]: cluster 2024-09-13T00:44:38.556450+0000 mgr.y (mgr.24661) 2025 : cluster [DBG] pgmap v3819: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 914 KiB/s rd, 3.2 KiB/s wr, 1.23k op/s 2024-09-13T00:44:40.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:40 smithi136 bash[76371]: cluster 2024-09-13T00:44:39.360377+0000 mon.a (mon.0) 3969 : cluster [DBG] osdmap e2234: 8 total, 8 up, 8 in 2024-09-13T00:44:40.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:40 smithi136 bash[75164]: cluster 2024-09-13T00:44:38.556450+0000 mgr.y (mgr.24661) 2025 : cluster [DBG] pgmap v3819: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 914 KiB/s rd, 3.2 KiB/s wr, 1.23k op/s 2024-09-13T00:44:40.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:40 smithi136 bash[75164]: cluster 2024-09-13T00:44:39.360377+0000 mon.a (mon.0) 3969 : cluster [DBG] osdmap e2234: 8 total, 8 up, 8 in 2024-09-13T00:44:40.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:40 smithi181 bash[67587]: cluster 2024-09-13T00:44:38.556450+0000 mgr.y (mgr.24661) 2025 : cluster [DBG] pgmap v3819: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 914 KiB/s rd, 3.2 KiB/s wr, 1.23k op/s 2024-09-13T00:44:40.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:40 smithi181 bash[67587]: cluster 2024-09-13T00:44:39.360377+0000 mon.a (mon.0) 3969 : cluster [DBG] osdmap e2234: 8 total, 8 up, 8 in 2024-09-13T00:44:41.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:41 smithi136 bash[75164]: cluster 2024-09-13T00:44:40.362821+0000 mon.a (mon.0) 3970 : cluster [DBG] osdmap e2235: 8 total, 8 up, 8 in 2024-09-13T00:44:41.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:41 smithi136 bash[76371]: cluster 2024-09-13T00:44:40.362821+0000 mon.a (mon.0) 3970 : cluster [DBG] osdmap e2235: 8 total, 8 up, 8 in 2024-09-13T00:44:41.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:41 smithi181 bash[67587]: cluster 2024-09-13T00:44:40.362821+0000 mon.a (mon.0) 3970 : cluster [DBG] osdmap e2235: 8 total, 8 up, 8 in 2024-09-13T00:44:42.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:42 smithi181 bash[67587]: cluster 2024-09-13T00:44:40.557147+0000 mgr.y (mgr.24661) 2026 : cluster [DBG] pgmap v3822: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 914 KiB/s rd, 3.0 KiB/s wr, 1.23k op/s 2024-09-13T00:44:42.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:42 smithi181 bash[67587]: cluster 2024-09-13T00:44:41.390966+0000 mon.a (mon.0) 3971 : cluster [DBG] osdmap e2236: 8 total, 8 up, 8 in 2024-09-13T00:44:42.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:44:42 smithi181 bash[95505]: ts=2024-09-13T00:44:42.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:44:42.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:42 smithi136 bash[76371]: cluster 2024-09-13T00:44:40.557147+0000 mgr.y (mgr.24661) 2026 : cluster [DBG] pgmap v3822: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 914 KiB/s rd, 3.0 KiB/s wr, 1.23k op/s 2024-09-13T00:44:42.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:42 smithi136 bash[76371]: cluster 2024-09-13T00:44:41.390966+0000 mon.a (mon.0) 3971 : cluster [DBG] osdmap e2236: 8 total, 8 up, 8 in 2024-09-13T00:44:42.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:42 smithi136 bash[75164]: cluster 2024-09-13T00:44:40.557147+0000 mgr.y (mgr.24661) 2026 : cluster [DBG] pgmap v3822: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 914 KiB/s rd, 3.0 KiB/s wr, 1.23k op/s 2024-09-13T00:44:42.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:42 smithi136 bash[75164]: cluster 2024-09-13T00:44:41.390966+0000 mon.a (mon.0) 3971 : cluster [DBG] osdmap e2236: 8 total, 8 up, 8 in 2024-09-13T00:44:43.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:43 smithi136 bash[75164]: cluster 2024-09-13T00:44:42.390640+0000 mon.a (mon.0) 3972 : cluster [DBG] osdmap e2237: 8 total, 8 up, 8 in 2024-09-13T00:44:43.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:43 smithi136 bash[76371]: cluster 2024-09-13T00:44:42.390640+0000 mon.a (mon.0) 3972 : cluster [DBG] osdmap e2237: 8 total, 8 up, 8 in 2024-09-13T00:44:43.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:43 smithi181 bash[67587]: cluster 2024-09-13T00:44:42.390640+0000 mon.a (mon.0) 3972 : cluster [DBG] osdmap e2237: 8 total, 8 up, 8 in 2024-09-13T00:44:44.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:44 smithi136 bash[75164]: cluster 2024-09-13T00:44:42.557798+0000 mgr.y (mgr.24661) 2027 : cluster [DBG] pgmap v3825: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 943 KiB/s rd, 4.0 KiB/s wr, 1.27k op/s 2024-09-13T00:44:44.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:44 smithi136 bash[75164]: cluster 2024-09-13T00:44:43.415109+0000 mon.a (mon.0) 3973 : cluster [DBG] osdmap e2238: 8 total, 8 up, 8 in 2024-09-13T00:44:44.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:44 smithi136 bash[76371]: cluster 2024-09-13T00:44:42.557798+0000 mgr.y (mgr.24661) 2027 : cluster [DBG] pgmap v3825: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 943 KiB/s rd, 4.0 KiB/s wr, 1.27k op/s 2024-09-13T00:44:44.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:44 smithi136 bash[76371]: cluster 2024-09-13T00:44:43.415109+0000 mon.a (mon.0) 3973 : cluster [DBG] osdmap e2238: 8 total, 8 up, 8 in 2024-09-13T00:44:44.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:44 smithi181 bash[67587]: cluster 2024-09-13T00:44:42.557798+0000 mgr.y (mgr.24661) 2027 : cluster [DBG] pgmap v3825: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 943 KiB/s rd, 4.0 KiB/s wr, 1.27k op/s 2024-09-13T00:44:44.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:44 smithi181 bash[67587]: cluster 2024-09-13T00:44:43.415109+0000 mon.a (mon.0) 3973 : cluster [DBG] osdmap e2238: 8 total, 8 up, 8 in 2024-09-13T00:44:45.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:45 smithi136 bash[75164]: cluster 2024-09-13T00:44:44.415494+0000 mon.a (mon.0) 3974 : cluster [DBG] osdmap e2239: 8 total, 8 up, 8 in 2024-09-13T00:44:45.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:45 smithi136 bash[76371]: cluster 2024-09-13T00:44:44.415494+0000 mon.a (mon.0) 3974 : cluster [DBG] osdmap e2239: 8 total, 8 up, 8 in 2024-09-13T00:44:45.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:45 smithi181 bash[67587]: cluster 2024-09-13T00:44:44.415494+0000 mon.a (mon.0) 3974 : cluster [DBG] osdmap e2239: 8 total, 8 up, 8 in 2024-09-13T00:44:46.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:46 smithi136 bash[75164]: cluster 2024-09-13T00:44:44.558486+0000 mgr.y (mgr.24661) 2028 : cluster [DBG] pgmap v3828: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 948 KiB/s rd, 3.7 KiB/s wr, 1.28k op/s 2024-09-13T00:44:46.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:46 smithi136 bash[75164]: cluster 2024-09-13T00:44:45.439210+0000 mon.a (mon.0) 3975 : cluster [DBG] osdmap e2240: 8 total, 8 up, 8 in 2024-09-13T00:44:46.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:46 smithi136 bash[76371]: cluster 2024-09-13T00:44:44.558486+0000 mgr.y (mgr.24661) 2028 : cluster [DBG] pgmap v3828: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 948 KiB/s rd, 3.7 KiB/s wr, 1.28k op/s 2024-09-13T00:44:46.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:46 smithi136 bash[76371]: cluster 2024-09-13T00:44:45.439210+0000 mon.a (mon.0) 3975 : cluster [DBG] osdmap e2240: 8 total, 8 up, 8 in 2024-09-13T00:44:46.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:46 smithi181 bash[67587]: cluster 2024-09-13T00:44:44.558486+0000 mgr.y (mgr.24661) 2028 : cluster [DBG] pgmap v3828: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 948 KiB/s rd, 3.7 KiB/s wr, 1.28k op/s 2024-09-13T00:44:46.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:46 smithi181 bash[67587]: cluster 2024-09-13T00:44:45.439210+0000 mon.a (mon.0) 3975 : cluster [DBG] osdmap e2240: 8 total, 8 up, 8 in 2024-09-13T00:44:47.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:47 smithi136 bash[75164]: cluster 2024-09-13T00:44:46.448535+0000 mon.a (mon.0) 3976 : cluster [DBG] osdmap e2241: 8 total, 8 up, 8 in 2024-09-13T00:44:47.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:47 smithi136 bash[76371]: cluster 2024-09-13T00:44:46.448535+0000 mon.a (mon.0) 3976 : cluster [DBG] osdmap e2241: 8 total, 8 up, 8 in 2024-09-13T00:44:47.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:47 smithi181 bash[67587]: cluster 2024-09-13T00:44:46.448535+0000 mon.a (mon.0) 3976 : cluster [DBG] osdmap e2241: 8 total, 8 up, 8 in 2024-09-13T00:44:48.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:48 smithi136 bash[75164]: cluster 2024-09-13T00:44:46.559186+0000 mgr.y (mgr.24661) 2029 : cluster [DBG] pgmap v3831: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 511 B/s wr, 6 op/s 2024-09-13T00:44:48.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:48 smithi136 bash[75164]: cluster 2024-09-13T00:44:47.448410+0000 mon.a (mon.0) 3977 : cluster [DBG] osdmap e2242: 8 total, 8 up, 8 in 2024-09-13T00:44:48.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:48 smithi136 bash[76371]: cluster 2024-09-13T00:44:46.559186+0000 mgr.y (mgr.24661) 2029 : cluster [DBG] pgmap v3831: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 511 B/s wr, 6 op/s 2024-09-13T00:44:48.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:48 smithi136 bash[76371]: cluster 2024-09-13T00:44:47.448410+0000 mon.a (mon.0) 3977 : cluster [DBG] osdmap e2242: 8 total, 8 up, 8 in 2024-09-13T00:44:48.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:48 smithi181 bash[67587]: cluster 2024-09-13T00:44:46.559186+0000 mgr.y (mgr.24661) 2029 : cluster [DBG] pgmap v3831: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 511 B/s wr, 6 op/s 2024-09-13T00:44:48.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:48 smithi181 bash[67587]: cluster 2024-09-13T00:44:47.448410+0000 mon.a (mon.0) 3977 : cluster [DBG] osdmap e2242: 8 total, 8 up, 8 in 2024-09-13T00:44:49.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:49 smithi136 bash[75164]: cluster 2024-09-13T00:44:48.471647+0000 mon.a (mon.0) 3978 : cluster [DBG] osdmap e2243: 8 total, 8 up, 8 in 2024-09-13T00:44:49.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:44:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:44:49] "GET /metrics HTTP/1.1" 200 36744 "" "Prometheus/2.43.0" 2024-09-13T00:44:49.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:49 smithi136 bash[76371]: cluster 2024-09-13T00:44:48.471647+0000 mon.a (mon.0) 3978 : cluster [DBG] osdmap e2243: 8 total, 8 up, 8 in 2024-09-13T00:44:49.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:49 smithi181 bash[67587]: cluster 2024-09-13T00:44:48.471647+0000 mon.a (mon.0) 3978 : cluster [DBG] osdmap e2243: 8 total, 8 up, 8 in 2024-09-13T00:44:50.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:50 smithi136 bash[75164]: cluster 2024-09-13T00:44:48.559843+0000 mgr.y (mgr.24661) 2030 : cluster [DBG] pgmap v3834: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 980 KiB/s rd, 3.7 KiB/s wr, 1.32k op/s 2024-09-13T00:44:50.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:50 smithi136 bash[75164]: cluster 2024-09-13T00:44:49.485019+0000 mon.a (mon.0) 3979 : cluster [DBG] osdmap e2244: 8 total, 8 up, 8 in 2024-09-13T00:44:50.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:50 smithi136 bash[76371]: cluster 2024-09-13T00:44:48.559843+0000 mgr.y (mgr.24661) 2030 : cluster [DBG] pgmap v3834: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 980 KiB/s rd, 3.7 KiB/s wr, 1.32k op/s 2024-09-13T00:44:50.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:50 smithi136 bash[76371]: cluster 2024-09-13T00:44:49.485019+0000 mon.a (mon.0) 3979 : cluster [DBG] osdmap e2244: 8 total, 8 up, 8 in 2024-09-13T00:44:50.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:50 smithi181 bash[67587]: cluster 2024-09-13T00:44:48.559843+0000 mgr.y (mgr.24661) 2030 : cluster [DBG] pgmap v3834: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 980 KiB/s rd, 3.7 KiB/s wr, 1.32k op/s 2024-09-13T00:44:50.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:50 smithi181 bash[67587]: cluster 2024-09-13T00:44:49.485019+0000 mon.a (mon.0) 3979 : cluster [DBG] osdmap e2244: 8 total, 8 up, 8 in 2024-09-13T00:44:51.759 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:51 smithi136 bash[75164]: cluster 2024-09-13T00:44:50.491134+0000 mon.a (mon.0) 3980 : cluster [DBG] osdmap e2245: 8 total, 8 up, 8 in 2024-09-13T00:44:51.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:51 smithi136 bash[76371]: cluster 2024-09-13T00:44:50.491134+0000 mon.a (mon.0) 3980 : cluster [DBG] osdmap e2245: 8 total, 8 up, 8 in 2024-09-13T00:44:51.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:51 smithi181 bash[67587]: cluster 2024-09-13T00:44:50.491134+0000 mon.a (mon.0) 3980 : cluster [DBG] osdmap e2245: 8 total, 8 up, 8 in 2024-09-13T00:44:52.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:52 smithi181 bash[67587]: cluster 2024-09-13T00:44:50.560728+0000 mgr.y (mgr.24661) 2031 : cluster [DBG] pgmap v3837: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 980 KiB/s rd, 3.2 KiB/s wr, 1.32k op/s 2024-09-13T00:44:52.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:52 smithi181 bash[67587]: cluster 2024-09-13T00:44:51.507722+0000 mon.a (mon.0) 3981 : cluster [DBG] osdmap e2246: 8 total, 8 up, 8 in 2024-09-13T00:44:52.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:44:52 smithi181 bash[95505]: ts=2024-09-13T00:44:52.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:44:53.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:52 smithi136 bash[75164]: cluster 2024-09-13T00:44:50.560728+0000 mgr.y (mgr.24661) 2031 : cluster [DBG] pgmap v3837: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 980 KiB/s rd, 3.2 KiB/s wr, 1.32k op/s 2024-09-13T00:44:53.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:52 smithi136 bash[75164]: cluster 2024-09-13T00:44:51.507722+0000 mon.a (mon.0) 3981 : cluster [DBG] osdmap e2246: 8 total, 8 up, 8 in 2024-09-13T00:44:53.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:52 smithi136 bash[76371]: cluster 2024-09-13T00:44:50.560728+0000 mgr.y (mgr.24661) 2031 : cluster [DBG] pgmap v3837: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 980 KiB/s rd, 3.2 KiB/s wr, 1.32k op/s 2024-09-13T00:44:53.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:52 smithi136 bash[76371]: cluster 2024-09-13T00:44:51.507722+0000 mon.a (mon.0) 3981 : cluster [DBG] osdmap e2246: 8 total, 8 up, 8 in 2024-09-13T00:44:53.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:53 smithi181 bash[67587]: cluster 2024-09-13T00:44:52.505374+0000 mon.a (mon.0) 3982 : cluster [DBG] osdmap e2247: 8 total, 8 up, 8 in 2024-09-13T00:44:54.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:53 smithi136 bash[75164]: cluster 2024-09-13T00:44:52.505374+0000 mon.a (mon.0) 3982 : cluster [DBG] osdmap e2247: 8 total, 8 up, 8 in 2024-09-13T00:44:54.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:53 smithi136 bash[76371]: cluster 2024-09-13T00:44:52.505374+0000 mon.a (mon.0) 3982 : cluster [DBG] osdmap e2247: 8 total, 8 up, 8 in 2024-09-13T00:44:54.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:54 smithi181 bash[67587]: cluster 2024-09-13T00:44:52.561630+0000 mgr.y (mgr.24661) 2032 : cluster [DBG] pgmap v3840: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1008 KiB/s rd, 3.2 KiB/s wr, 1.36k op/s 2024-09-13T00:44:54.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:54 smithi181 bash[67587]: cluster 2024-09-13T00:44:53.531211+0000 mon.a (mon.0) 3983 : cluster [DBG] osdmap e2248: 8 total, 8 up, 8 in 2024-09-13T00:44:54.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:54 smithi181 bash[67587]: audit 2024-09-13T00:44:53.846021+0000 mon.a (mon.0) 3984 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:44:55.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:54 smithi136 bash[75164]: cluster 2024-09-13T00:44:52.561630+0000 mgr.y (mgr.24661) 2032 : cluster [DBG] pgmap v3840: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1008 KiB/s rd, 3.2 KiB/s wr, 1.36k op/s 2024-09-13T00:44:55.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:54 smithi136 bash[75164]: cluster 2024-09-13T00:44:53.531211+0000 mon.a (mon.0) 3983 : cluster [DBG] osdmap e2248: 8 total, 8 up, 8 in 2024-09-13T00:44:55.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:54 smithi136 bash[75164]: audit 2024-09-13T00:44:53.846021+0000 mon.a (mon.0) 3984 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:44:55.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:54 smithi136 bash[76371]: cluster 2024-09-13T00:44:52.561630+0000 mgr.y (mgr.24661) 2032 : cluster [DBG] pgmap v3840: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1008 KiB/s rd, 3.2 KiB/s wr, 1.36k op/s 2024-09-13T00:44:55.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:54 smithi136 bash[76371]: cluster 2024-09-13T00:44:53.531211+0000 mon.a (mon.0) 3983 : cluster [DBG] osdmap e2248: 8 total, 8 up, 8 in 2024-09-13T00:44:55.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:54 smithi136 bash[76371]: audit 2024-09-13T00:44:53.846021+0000 mon.a (mon.0) 3984 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:44:55.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:55 smithi181 bash[67587]: cluster 2024-09-13T00:44:54.531014+0000 mon.a (mon.0) 3985 : cluster [DBG] osdmap e2249: 8 total, 8 up, 8 in 2024-09-13T00:44:55.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:55 smithi181 bash[67587]: cluster 2024-09-13T00:44:54.562323+0000 mgr.y (mgr.24661) 2033 : cluster [DBG] pgmap v3843: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1013 KiB/s rd, 4.0 KiB/s wr, 1.36k op/s 2024-09-13T00:44:56.009 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:55 smithi136 bash[76371]: cluster 2024-09-13T00:44:54.531014+0000 mon.a (mon.0) 3985 : cluster [DBG] osdmap e2249: 8 total, 8 up, 8 in 2024-09-13T00:44:56.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:55 smithi136 bash[76371]: cluster 2024-09-13T00:44:54.562323+0000 mgr.y (mgr.24661) 2033 : cluster [DBG] pgmap v3843: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1013 KiB/s rd, 4.0 KiB/s wr, 1.36k op/s 2024-09-13T00:44:56.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:55 smithi136 bash[75164]: cluster 2024-09-13T00:44:54.531014+0000 mon.a (mon.0) 3985 : cluster [DBG] osdmap e2249: 8 total, 8 up, 8 in 2024-09-13T00:44:56.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:55 smithi136 bash[75164]: cluster 2024-09-13T00:44:54.562323+0000 mgr.y (mgr.24661) 2033 : cluster [DBG] pgmap v3843: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1013 KiB/s rd, 4.0 KiB/s wr, 1.36k op/s 2024-09-13T00:44:56.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:56 smithi181 bash[67587]: cluster 2024-09-13T00:44:55.556773+0000 mon.a (mon.0) 3986 : cluster [DBG] osdmap e2250: 8 total, 8 up, 8 in 2024-09-13T00:44:56.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:56 smithi181 bash[67587]: audit 2024-09-13T00:44:55.947486+0000 mon.a (mon.0) 3987 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:44:56.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:56 smithi181 bash[67587]: audit 2024-09-13T00:44:56.474101+0000 mon.a (mon.0) 3988 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:44:56.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:56 smithi181 bash[67587]: audit 2024-09-13T00:44:56.475676+0000 mon.a (mon.0) 3989 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:44:56.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:56 smithi181 bash[67587]: audit 2024-09-13T00:44:56.483081+0000 mon.a (mon.0) 3990 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:44:57.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:56 smithi136 bash[75164]: cluster 2024-09-13T00:44:55.556773+0000 mon.a (mon.0) 3986 : cluster [DBG] osdmap e2250: 8 total, 8 up, 8 in 2024-09-13T00:44:57.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:56 smithi136 bash[75164]: audit 2024-09-13T00:44:55.947486+0000 mon.a (mon.0) 3987 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:44:57.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:56 smithi136 bash[75164]: audit 2024-09-13T00:44:56.474101+0000 mon.a (mon.0) 3988 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:44:57.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:56 smithi136 bash[75164]: audit 2024-09-13T00:44:56.475676+0000 mon.a (mon.0) 3989 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:44:57.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:56 smithi136 bash[75164]: audit 2024-09-13T00:44:56.483081+0000 mon.a (mon.0) 3990 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:44:57.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:56 smithi136 bash[76371]: cluster 2024-09-13T00:44:55.556773+0000 mon.a (mon.0) 3986 : cluster [DBG] osdmap e2250: 8 total, 8 up, 8 in 2024-09-13T00:44:57.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:56 smithi136 bash[76371]: audit 2024-09-13T00:44:55.947486+0000 mon.a (mon.0) 3987 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:44:57.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:56 smithi136 bash[76371]: audit 2024-09-13T00:44:56.474101+0000 mon.a (mon.0) 3988 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:44:57.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:56 smithi136 bash[76371]: audit 2024-09-13T00:44:56.475676+0000 mon.a (mon.0) 3989 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:44:57.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:56 smithi136 bash[76371]: audit 2024-09-13T00:44:56.483081+0000 mon.a (mon.0) 3990 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:44:57.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:57 smithi181 bash[67587]: cluster 2024-09-13T00:44:56.551947+0000 mon.a (mon.0) 3991 : cluster [DBG] osdmap e2251: 8 total, 8 up, 8 in 2024-09-13T00:44:57.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:57 smithi181 bash[67587]: cluster 2024-09-13T00:44:56.562833+0000 mgr.y (mgr.24661) 2034 : cluster [DBG] pgmap v3846: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 767 B/s wr, 6 op/s 2024-09-13T00:44:58.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:57 smithi136 bash[75164]: cluster 2024-09-13T00:44:56.551947+0000 mon.a (mon.0) 3991 : cluster [DBG] osdmap e2251: 8 total, 8 up, 8 in 2024-09-13T00:44:58.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:57 smithi136 bash[75164]: cluster 2024-09-13T00:44:56.562833+0000 mgr.y (mgr.24661) 2034 : cluster [DBG] pgmap v3846: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 767 B/s wr, 6 op/s 2024-09-13T00:44:58.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:57 smithi136 bash[76371]: cluster 2024-09-13T00:44:56.551947+0000 mon.a (mon.0) 3991 : cluster [DBG] osdmap e2251: 8 total, 8 up, 8 in 2024-09-13T00:44:58.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:57 smithi136 bash[76371]: cluster 2024-09-13T00:44:56.562833+0000 mgr.y (mgr.24661) 2034 : cluster [DBG] pgmap v3846: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 767 B/s wr, 6 op/s 2024-09-13T00:44:58.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:58 smithi181 bash[67587]: cluster 2024-09-13T00:44:57.582808+0000 mon.a (mon.0) 3992 : cluster [DBG] osdmap e2252: 8 total, 8 up, 8 in 2024-09-13T00:44:59.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:58 smithi136 bash[75164]: cluster 2024-09-13T00:44:57.582808+0000 mon.a (mon.0) 3992 : cluster [DBG] osdmap e2252: 8 total, 8 up, 8 in 2024-09-13T00:44:59.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:58 smithi136 bash[76371]: cluster 2024-09-13T00:44:57.582808+0000 mon.a (mon.0) 3992 : cluster [DBG] osdmap e2252: 8 total, 8 up, 8 in 2024-09-13T00:44:59.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:44:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:44:59] "GET /metrics HTTP/1.1" 200 36746 "" "Prometheus/2.43.0" 2024-09-13T00:44:59.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:59 smithi136 bash[75164]: cluster 2024-09-13T00:44:58.563878+0000 mgr.y (mgr.24661) 2035 : cluster [DBG] pgmap v3848: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.0 MiB/s rd, 3.5 KiB/s wr, 1.40k op/s 2024-09-13T00:44:59.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:44:59 smithi136 bash[75164]: cluster 2024-09-13T00:44:58.589884+0000 mon.a (mon.0) 3993 : cluster [DBG] osdmap e2253: 8 total, 8 up, 8 in 2024-09-13T00:44:59.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:59 smithi136 bash[76371]: cluster 2024-09-13T00:44:58.563878+0000 mgr.y (mgr.24661) 2035 : cluster [DBG] pgmap v3848: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.0 MiB/s rd, 3.5 KiB/s wr, 1.40k op/s 2024-09-13T00:44:59.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:44:59 smithi136 bash[76371]: cluster 2024-09-13T00:44:58.589884+0000 mon.a (mon.0) 3993 : cluster [DBG] osdmap e2253: 8 total, 8 up, 8 in 2024-09-13T00:44:59.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:59 smithi181 bash[67587]: cluster 2024-09-13T00:44:58.563878+0000 mgr.y (mgr.24661) 2035 : cluster [DBG] pgmap v3848: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.0 MiB/s rd, 3.5 KiB/s wr, 1.40k op/s 2024-09-13T00:44:59.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:44:59 smithi181 bash[67587]: cluster 2024-09-13T00:44:58.589884+0000 mon.a (mon.0) 3993 : cluster [DBG] osdmap e2253: 8 total, 8 up, 8 in 2024-09-13T00:45:01.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:00 smithi136 bash[75164]: cluster 2024-09-13T00:44:59.599778+0000 mon.a (mon.0) 3994 : cluster [DBG] osdmap e2254: 8 total, 8 up, 8 in 2024-09-13T00:45:01.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:00 smithi136 bash[76371]: cluster 2024-09-13T00:44:59.599778+0000 mon.a (mon.0) 3994 : cluster [DBG] osdmap e2254: 8 total, 8 up, 8 in 2024-09-13T00:45:01.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:00 smithi181 bash[67587]: cluster 2024-09-13T00:44:59.599778+0000 mon.a (mon.0) 3994 : cluster [DBG] osdmap e2254: 8 total, 8 up, 8 in 2024-09-13T00:45:01.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:01 smithi181 bash[67587]: cluster 2024-09-13T00:45:00.564705+0000 mgr.y (mgr.24661) 2036 : cluster [DBG] pgmap v3851: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.0 MiB/s rd, 3.5 KiB/s wr, 1.40k op/s 2024-09-13T00:45:01.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:01 smithi181 bash[67587]: cluster 2024-09-13T00:45:00.612304+0000 mon.a (mon.0) 3995 : cluster [DBG] osdmap e2255: 8 total, 8 up, 8 in 2024-09-13T00:45:02.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:01 smithi136 bash[75164]: cluster 2024-09-13T00:45:00.564705+0000 mgr.y (mgr.24661) 2036 : cluster [DBG] pgmap v3851: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.0 MiB/s rd, 3.5 KiB/s wr, 1.40k op/s 2024-09-13T00:45:02.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:01 smithi136 bash[75164]: cluster 2024-09-13T00:45:00.612304+0000 mon.a (mon.0) 3995 : cluster [DBG] osdmap e2255: 8 total, 8 up, 8 in 2024-09-13T00:45:02.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:01 smithi136 bash[76371]: cluster 2024-09-13T00:45:00.564705+0000 mgr.y (mgr.24661) 2036 : cluster [DBG] pgmap v3851: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.0 MiB/s rd, 3.5 KiB/s wr, 1.40k op/s 2024-09-13T00:45:02.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:01 smithi136 bash[76371]: cluster 2024-09-13T00:45:00.612304+0000 mon.a (mon.0) 3995 : cluster [DBG] osdmap e2255: 8 total, 8 up, 8 in 2024-09-13T00:45:02.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:45:02 smithi181 bash[95505]: ts=2024-09-13T00:45:02.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:45:03.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:02 smithi136 bash[75164]: cluster 2024-09-13T00:45:01.619944+0000 mon.a (mon.0) 3996 : cluster [DBG] osdmap e2256: 8 total, 8 up, 8 in 2024-09-13T00:45:03.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:02 smithi136 bash[76371]: cluster 2024-09-13T00:45:01.619944+0000 mon.a (mon.0) 3996 : cluster [DBG] osdmap e2256: 8 total, 8 up, 8 in 2024-09-13T00:45:03.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:02 smithi181 bash[67587]: cluster 2024-09-13T00:45:01.619944+0000 mon.a (mon.0) 3996 : cluster [DBG] osdmap e2256: 8 total, 8 up, 8 in 2024-09-13T00:45:04.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:03 smithi136 bash[75164]: cluster 2024-09-13T00:45:02.565601+0000 mgr.y (mgr.24661) 2037 : cluster [DBG] pgmap v3854: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 860 KiB/s rd, 3.5 KiB/s wr, 1.16k op/s 2024-09-13T00:45:04.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:03 smithi136 bash[75164]: cluster 2024-09-13T00:45:02.630695+0000 mon.a (mon.0) 3997 : cluster [DBG] osdmap e2257: 8 total, 8 up, 8 in 2024-09-13T00:45:04.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:03 smithi136 bash[76371]: cluster 2024-09-13T00:45:02.565601+0000 mgr.y (mgr.24661) 2037 : cluster [DBG] pgmap v3854: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 860 KiB/s rd, 3.5 KiB/s wr, 1.16k op/s 2024-09-13T00:45:04.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:03 smithi136 bash[76371]: cluster 2024-09-13T00:45:02.630695+0000 mon.a (mon.0) 3997 : cluster [DBG] osdmap e2257: 8 total, 8 up, 8 in 2024-09-13T00:45:04.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:03 smithi181 bash[67587]: cluster 2024-09-13T00:45:02.565601+0000 mgr.y (mgr.24661) 2037 : cluster [DBG] pgmap v3854: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 860 KiB/s rd, 3.5 KiB/s wr, 1.16k op/s 2024-09-13T00:45:04.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:03 smithi181 bash[67587]: cluster 2024-09-13T00:45:02.630695+0000 mon.a (mon.0) 3997 : cluster [DBG] osdmap e2257: 8 total, 8 up, 8 in 2024-09-13T00:45:05.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:04 smithi136 bash[75164]: cluster 2024-09-13T00:45:03.639445+0000 mon.a (mon.0) 3998 : cluster [DBG] osdmap e2258: 8 total, 8 up, 8 in 2024-09-13T00:45:05.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:04 smithi136 bash[76371]: cluster 2024-09-13T00:45:03.639445+0000 mon.a (mon.0) 3998 : cluster [DBG] osdmap e2258: 8 total, 8 up, 8 in 2024-09-13T00:45:05.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:04 smithi181 bash[67587]: cluster 2024-09-13T00:45:03.639445+0000 mon.a (mon.0) 3998 : cluster [DBG] osdmap e2258: 8 total, 8 up, 8 in 2024-09-13T00:45:06.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:05 smithi136 bash[75164]: cluster 2024-09-13T00:45:04.566262+0000 mgr.y (mgr.24661) 2038 : cluster [DBG] pgmap v3857: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 865 KiB/s rd, 3.5 KiB/s wr, 1.16k op/s 2024-09-13T00:45:06.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:05 smithi136 bash[75164]: cluster 2024-09-13T00:45:04.650173+0000 mon.a (mon.0) 3999 : cluster [DBG] osdmap e2259: 8 total, 8 up, 8 in 2024-09-13T00:45:06.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:05 smithi136 bash[76371]: cluster 2024-09-13T00:45:04.566262+0000 mgr.y (mgr.24661) 2038 : cluster [DBG] pgmap v3857: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 865 KiB/s rd, 3.5 KiB/s wr, 1.16k op/s 2024-09-13T00:45:06.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:05 smithi136 bash[76371]: cluster 2024-09-13T00:45:04.650173+0000 mon.a (mon.0) 3999 : cluster [DBG] osdmap e2259: 8 total, 8 up, 8 in 2024-09-13T00:45:06.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:05 smithi181 bash[67587]: cluster 2024-09-13T00:45:04.566262+0000 mgr.y (mgr.24661) 2038 : cluster [DBG] pgmap v3857: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 865 KiB/s rd, 3.5 KiB/s wr, 1.16k op/s 2024-09-13T00:45:06.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:05 smithi181 bash[67587]: cluster 2024-09-13T00:45:04.650173+0000 mon.a (mon.0) 3999 : cluster [DBG] osdmap e2259: 8 total, 8 up, 8 in 2024-09-13T00:45:07.009 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:06 smithi136 bash[75164]: cluster 2024-09-13T00:45:05.658548+0000 mon.a (mon.0) 4000 : cluster [DBG] osdmap e2260: 8 total, 8 up, 8 in 2024-09-13T00:45:07.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:06 smithi136 bash[76371]: cluster 2024-09-13T00:45:05.658548+0000 mon.a (mon.0) 4000 : cluster [DBG] osdmap e2260: 8 total, 8 up, 8 in 2024-09-13T00:45:07.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:06 smithi181 bash[67587]: cluster 2024-09-13T00:45:05.658548+0000 mon.a (mon.0) 4000 : cluster [DBG] osdmap e2260: 8 total, 8 up, 8 in 2024-09-13T00:45:08.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:07 smithi136 bash[75164]: cluster 2024-09-13T00:45:06.567417+0000 mgr.y (mgr.24661) 2039 : cluster [DBG] pgmap v3860: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 767 B/s wr, 6 op/s 2024-09-13T00:45:08.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:07 smithi136 bash[75164]: cluster 2024-09-13T00:45:06.667324+0000 mon.a (mon.0) 4001 : cluster [DBG] osdmap e2261: 8 total, 8 up, 8 in 2024-09-13T00:45:08.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:07 smithi136 bash[76371]: cluster 2024-09-13T00:45:06.567417+0000 mgr.y (mgr.24661) 2039 : cluster [DBG] pgmap v3860: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 767 B/s wr, 6 op/s 2024-09-13T00:45:08.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:07 smithi136 bash[76371]: cluster 2024-09-13T00:45:06.667324+0000 mon.a (mon.0) 4001 : cluster [DBG] osdmap e2261: 8 total, 8 up, 8 in 2024-09-13T00:45:08.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:07 smithi181 bash[67587]: cluster 2024-09-13T00:45:06.567417+0000 mgr.y (mgr.24661) 2039 : cluster [DBG] pgmap v3860: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 767 B/s wr, 6 op/s 2024-09-13T00:45:08.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:07 smithi181 bash[67587]: cluster 2024-09-13T00:45:06.667324+0000 mon.a (mon.0) 4001 : cluster [DBG] osdmap e2261: 8 total, 8 up, 8 in 2024-09-13T00:45:09.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:08 smithi136 bash[76371]: cluster 2024-09-13T00:45:07.670953+0000 mon.a (mon.0) 4002 : cluster [DBG] osdmap e2262: 8 total, 8 up, 8 in 2024-09-13T00:45:09.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:08 smithi136 bash[75164]: cluster 2024-09-13T00:45:07.670953+0000 mon.a (mon.0) 4002 : cluster [DBG] osdmap e2262: 8 total, 8 up, 8 in 2024-09-13T00:45:09.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:08 smithi181 bash[67587]: cluster 2024-09-13T00:45:07.670953+0000 mon.a (mon.0) 4002 : cluster [DBG] osdmap e2262: 8 total, 8 up, 8 in 2024-09-13T00:45:09.684 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:45:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:45:09] "GET /metrics HTTP/1.1" 200 36745 "" "Prometheus/2.43.0" 2024-09-13T00:45:10.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:09 smithi136 bash[75164]: cluster 2024-09-13T00:45:08.568733+0000 mgr.y (mgr.24661) 2040 : cluster [DBG] pgmap v3863: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.1 MiB/s rd, 3.0 KiB/s wr, 1.49k op/s 2024-09-13T00:45:10.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:09 smithi136 bash[75164]: cluster 2024-09-13T00:45:08.690721+0000 mon.a (mon.0) 4003 : cluster [DBG] osdmap e2263: 8 total, 8 up, 8 in 2024-09-13T00:45:10.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:09 smithi136 bash[75164]: audit 2024-09-13T00:45:08.846370+0000 mon.a (mon.0) 4004 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:45:10.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:09 smithi136 bash[76371]: cluster 2024-09-13T00:45:08.568733+0000 mgr.y (mgr.24661) 2040 : cluster [DBG] pgmap v3863: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.1 MiB/s rd, 3.0 KiB/s wr, 1.49k op/s 2024-09-13T00:45:10.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:09 smithi136 bash[76371]: cluster 2024-09-13T00:45:08.690721+0000 mon.a (mon.0) 4003 : cluster [DBG] osdmap e2263: 8 total, 8 up, 8 in 2024-09-13T00:45:10.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:09 smithi136 bash[76371]: audit 2024-09-13T00:45:08.846370+0000 mon.a (mon.0) 4004 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:45:10.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:09 smithi181 bash[67587]: cluster 2024-09-13T00:45:08.568733+0000 mgr.y (mgr.24661) 2040 : cluster [DBG] pgmap v3863: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.1 MiB/s rd, 3.0 KiB/s wr, 1.49k op/s 2024-09-13T00:45:10.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:09 smithi181 bash[67587]: cluster 2024-09-13T00:45:08.690721+0000 mon.a (mon.0) 4003 : cluster [DBG] osdmap e2263: 8 total, 8 up, 8 in 2024-09-13T00:45:10.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:09 smithi181 bash[67587]: audit 2024-09-13T00:45:08.846370+0000 mon.a (mon.0) 4004 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:45:11.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:10 smithi136 bash[75164]: cluster 2024-09-13T00:45:09.706312+0000 mon.a (mon.0) 4005 : cluster [DBG] osdmap e2264: 8 total, 8 up, 8 in 2024-09-13T00:45:11.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:10 smithi136 bash[76371]: cluster 2024-09-13T00:45:09.706312+0000 mon.a (mon.0) 4005 : cluster [DBG] osdmap e2264: 8 total, 8 up, 8 in 2024-09-13T00:45:11.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:10 smithi181 bash[67587]: cluster 2024-09-13T00:45:09.706312+0000 mon.a (mon.0) 4005 : cluster [DBG] osdmap e2264: 8 total, 8 up, 8 in 2024-09-13T00:45:12.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:11 smithi136 bash[76371]: cluster 2024-09-13T00:45:10.569553+0000 mgr.y (mgr.24661) 2041 : cluster [DBG] pgmap v3866: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.1 MiB/s rd, 2.2 KiB/s wr, 1.48k op/s 2024-09-13T00:45:12.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:11 smithi136 bash[76371]: cluster 2024-09-13T00:45:10.704175+0000 mon.a (mon.0) 4006 : cluster [DBG] osdmap e2265: 8 total, 8 up, 8 in 2024-09-13T00:45:12.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:11 smithi136 bash[75164]: cluster 2024-09-13T00:45:10.569553+0000 mgr.y (mgr.24661) 2041 : cluster [DBG] pgmap v3866: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.1 MiB/s rd, 2.2 KiB/s wr, 1.48k op/s 2024-09-13T00:45:12.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:11 smithi136 bash[75164]: cluster 2024-09-13T00:45:10.704175+0000 mon.a (mon.0) 4006 : cluster [DBG] osdmap e2265: 8 total, 8 up, 8 in 2024-09-13T00:45:12.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:11 smithi181 bash[67587]: cluster 2024-09-13T00:45:10.569553+0000 mgr.y (mgr.24661) 2041 : cluster [DBG] pgmap v3866: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.1 MiB/s rd, 2.2 KiB/s wr, 1.48k op/s 2024-09-13T00:45:12.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:11 smithi181 bash[67587]: cluster 2024-09-13T00:45:10.704175+0000 mon.a (mon.0) 4006 : cluster [DBG] osdmap e2265: 8 total, 8 up, 8 in 2024-09-13T00:45:12.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:45:12 smithi181 bash[95505]: ts=2024-09-13T00:45:12.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:45:13.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:12 smithi136 bash[76371]: cluster 2024-09-13T00:45:11.729187+0000 mon.a (mon.0) 4007 : cluster [DBG] osdmap e2266: 8 total, 8 up, 8 in 2024-09-13T00:45:13.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:12 smithi136 bash[75164]: cluster 2024-09-13T00:45:11.729187+0000 mon.a (mon.0) 4007 : cluster [DBG] osdmap e2266: 8 total, 8 up, 8 in 2024-09-13T00:45:13.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:12 smithi181 bash[67587]: cluster 2024-09-13T00:45:11.729187+0000 mon.a (mon.0) 4007 : cluster [DBG] osdmap e2266: 8 total, 8 up, 8 in 2024-09-13T00:45:14.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:13 smithi136 bash[75164]: cluster 2024-09-13T00:45:12.570322+0000 mgr.y (mgr.24661) 2042 : cluster [DBG] pgmap v3869: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.1 MiB/s rd, 3.7 KiB/s wr, 1.52k op/s 2024-09-13T00:45:14.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:13 smithi136 bash[75164]: cluster 2024-09-13T00:45:12.740646+0000 mon.a (mon.0) 4008 : cluster [DBG] osdmap e2267: 8 total, 8 up, 8 in 2024-09-13T00:45:14.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:13 smithi136 bash[76371]: cluster 2024-09-13T00:45:12.570322+0000 mgr.y (mgr.24661) 2042 : cluster [DBG] pgmap v3869: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.1 MiB/s rd, 3.7 KiB/s wr, 1.52k op/s 2024-09-13T00:45:14.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:13 smithi136 bash[76371]: cluster 2024-09-13T00:45:12.740646+0000 mon.a (mon.0) 4008 : cluster [DBG] osdmap e2267: 8 total, 8 up, 8 in 2024-09-13T00:45:14.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:13 smithi181 bash[67587]: cluster 2024-09-13T00:45:12.570322+0000 mgr.y (mgr.24661) 2042 : cluster [DBG] pgmap v3869: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.1 MiB/s rd, 3.7 KiB/s wr, 1.52k op/s 2024-09-13T00:45:14.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:13 smithi181 bash[67587]: cluster 2024-09-13T00:45:12.740646+0000 mon.a (mon.0) 4008 : cluster [DBG] osdmap e2267: 8 total, 8 up, 8 in 2024-09-13T00:45:15.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:14 smithi136 bash[75164]: cluster 2024-09-13T00:45:13.747821+0000 mon.a (mon.0) 4009 : cluster [DBG] osdmap e2268: 8 total, 8 up, 8 in 2024-09-13T00:45:15.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:14 smithi136 bash[76371]: cluster 2024-09-13T00:45:13.747821+0000 mon.a (mon.0) 4009 : cluster [DBG] osdmap e2268: 8 total, 8 up, 8 in 2024-09-13T00:45:15.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:14 smithi181 bash[67587]: cluster 2024-09-13T00:45:13.747821+0000 mon.a (mon.0) 4009 : cluster [DBG] osdmap e2268: 8 total, 8 up, 8 in 2024-09-13T00:45:16.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:15 smithi181 bash[67587]: cluster 2024-09-13T00:45:14.571013+0000 mgr.y (mgr.24661) 2043 : cluster [DBG] pgmap v3872: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.1 MiB/s rd, 4.0 KiB/s wr, 1.53k op/s 2024-09-13T00:45:16.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:15 smithi181 bash[67587]: cluster 2024-09-13T00:45:14.762180+0000 mon.a (mon.0) 4010 : cluster [DBG] osdmap e2269: 8 total, 8 up, 8 in 2024-09-13T00:45:16.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:15 smithi136 bash[75164]: cluster 2024-09-13T00:45:14.571013+0000 mgr.y (mgr.24661) 2043 : cluster [DBG] pgmap v3872: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.1 MiB/s rd, 4.0 KiB/s wr, 1.53k op/s 2024-09-13T00:45:16.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:15 smithi136 bash[75164]: cluster 2024-09-13T00:45:14.762180+0000 mon.a (mon.0) 4010 : cluster [DBG] osdmap e2269: 8 total, 8 up, 8 in 2024-09-13T00:45:16.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:15 smithi136 bash[76371]: cluster 2024-09-13T00:45:14.571013+0000 mgr.y (mgr.24661) 2043 : cluster [DBG] pgmap v3872: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.1 MiB/s rd, 4.0 KiB/s wr, 1.53k op/s 2024-09-13T00:45:16.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:15 smithi136 bash[76371]: cluster 2024-09-13T00:45:14.762180+0000 mon.a (mon.0) 4010 : cluster [DBG] osdmap e2269: 8 total, 8 up, 8 in 2024-09-13T00:45:17.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:16 smithi181 bash[67587]: cluster 2024-09-13T00:45:15.760194+0000 mon.a (mon.0) 4011 : cluster [DBG] osdmap e2270: 8 total, 8 up, 8 in 2024-09-13T00:45:17.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:16 smithi136 bash[76371]: cluster 2024-09-13T00:45:15.760194+0000 mon.a (mon.0) 4011 : cluster [DBG] osdmap e2270: 8 total, 8 up, 8 in 2024-09-13T00:45:17.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:16 smithi136 bash[75164]: cluster 2024-09-13T00:45:15.760194+0000 mon.a (mon.0) 4011 : cluster [DBG] osdmap e2270: 8 total, 8 up, 8 in 2024-09-13T00:45:18.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:17 smithi181 bash[67587]: cluster 2024-09-13T00:45:16.572021+0000 mgr.y (mgr.24661) 2044 : cluster [DBG] pgmap v3875: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 4.7 KiB/s rd, 1.2 KiB/s wr, 7 op/s 2024-09-13T00:45:18.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:17 smithi181 bash[67587]: cluster 2024-09-13T00:45:16.782552+0000 mon.a (mon.0) 4012 : cluster [DBG] osdmap e2271: 8 total, 8 up, 8 in 2024-09-13T00:45:18.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:17 smithi136 bash[75164]: cluster 2024-09-13T00:45:16.572021+0000 mgr.y (mgr.24661) 2044 : cluster [DBG] pgmap v3875: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 4.7 KiB/s rd, 1.2 KiB/s wr, 7 op/s 2024-09-13T00:45:18.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:17 smithi136 bash[75164]: cluster 2024-09-13T00:45:16.782552+0000 mon.a (mon.0) 4012 : cluster [DBG] osdmap e2271: 8 total, 8 up, 8 in 2024-09-13T00:45:18.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:17 smithi136 bash[76371]: cluster 2024-09-13T00:45:16.572021+0000 mgr.y (mgr.24661) 2044 : cluster [DBG] pgmap v3875: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 4.7 KiB/s rd, 1.2 KiB/s wr, 7 op/s 2024-09-13T00:45:18.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:17 smithi136 bash[76371]: cluster 2024-09-13T00:45:16.782552+0000 mon.a (mon.0) 4012 : cluster [DBG] osdmap e2271: 8 total, 8 up, 8 in 2024-09-13T00:45:19.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:18 smithi181 bash[67587]: cluster 2024-09-13T00:45:17.787479+0000 mon.a (mon.0) 4013 : cluster [DBG] osdmap e2272: 8 total, 8 up, 8 in 2024-09-13T00:45:19.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:18 smithi136 bash[75164]: cluster 2024-09-13T00:45:17.787479+0000 mon.a (mon.0) 4013 : cluster [DBG] osdmap e2272: 8 total, 8 up, 8 in 2024-09-13T00:45:19.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:18 smithi136 bash[76371]: cluster 2024-09-13T00:45:17.787479+0000 mon.a (mon.0) 4013 : cluster [DBG] osdmap e2272: 8 total, 8 up, 8 in 2024-09-13T00:45:19.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:45:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:45:19] "GET /metrics HTTP/1.1" 200 36745 "" "Prometheus/2.43.0" 2024-09-13T00:45:20.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:19 smithi181 bash[67587]: cluster 2024-09-13T00:45:18.572905+0000 mgr.y (mgr.24661) 2045 : cluster [DBG] pgmap v3878: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.1 MiB/s rd, 3.7 KiB/s wr, 1.57k op/s 2024-09-13T00:45:20.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:19 smithi181 bash[67587]: cluster 2024-09-13T00:45:18.809955+0000 mon.a (mon.0) 4014 : cluster [DBG] osdmap e2273: 8 total, 8 up, 8 in 2024-09-13T00:45:20.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:19 smithi136 bash[75164]: cluster 2024-09-13T00:45:18.572905+0000 mgr.y (mgr.24661) 2045 : cluster [DBG] pgmap v3878: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.1 MiB/s rd, 3.7 KiB/s wr, 1.57k op/s 2024-09-13T00:45:20.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:19 smithi136 bash[75164]: cluster 2024-09-13T00:45:18.809955+0000 mon.a (mon.0) 4014 : cluster [DBG] osdmap e2273: 8 total, 8 up, 8 in 2024-09-13T00:45:20.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:19 smithi136 bash[76371]: cluster 2024-09-13T00:45:18.572905+0000 mgr.y (mgr.24661) 2045 : cluster [DBG] pgmap v3878: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.1 MiB/s rd, 3.7 KiB/s wr, 1.57k op/s 2024-09-13T00:45:20.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:19 smithi136 bash[76371]: cluster 2024-09-13T00:45:18.809955+0000 mon.a (mon.0) 4014 : cluster [DBG] osdmap e2273: 8 total, 8 up, 8 in 2024-09-13T00:45:21.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:20 smithi181 bash[67587]: cluster 2024-09-13T00:45:19.816996+0000 mon.a (mon.0) 4015 : cluster [DBG] osdmap e2274: 8 total, 8 up, 8 in 2024-09-13T00:45:21.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:20 smithi136 bash[76371]: cluster 2024-09-13T00:45:19.816996+0000 mon.a (mon.0) 4015 : cluster [DBG] osdmap e2274: 8 total, 8 up, 8 in 2024-09-13T00:45:21.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:20 smithi136 bash[75164]: cluster 2024-09-13T00:45:19.816996+0000 mon.a (mon.0) 4015 : cluster [DBG] osdmap e2274: 8 total, 8 up, 8 in 2024-09-13T00:45:22.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:21 smithi181 bash[67587]: cluster 2024-09-13T00:45:20.573630+0000 mgr.y (mgr.24661) 2046 : cluster [DBG] pgmap v3881: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.1 MiB/s rd, 3.0 KiB/s wr, 1.57k op/s 2024-09-13T00:45:22.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:21 smithi181 bash[67587]: cluster 2024-09-13T00:45:20.827986+0000 mon.a (mon.0) 4016 : cluster [DBG] osdmap e2275: 8 total, 8 up, 8 in 2024-09-13T00:45:22.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:21 smithi136 bash[75164]: cluster 2024-09-13T00:45:20.573630+0000 mgr.y (mgr.24661) 2046 : cluster [DBG] pgmap v3881: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.1 MiB/s rd, 3.0 KiB/s wr, 1.57k op/s 2024-09-13T00:45:22.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:21 smithi136 bash[75164]: cluster 2024-09-13T00:45:20.827986+0000 mon.a (mon.0) 4016 : cluster [DBG] osdmap e2275: 8 total, 8 up, 8 in 2024-09-13T00:45:22.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:21 smithi136 bash[76371]: cluster 2024-09-13T00:45:20.573630+0000 mgr.y (mgr.24661) 2046 : cluster [DBG] pgmap v3881: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.1 MiB/s rd, 3.0 KiB/s wr, 1.57k op/s 2024-09-13T00:45:22.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:21 smithi136 bash[76371]: cluster 2024-09-13T00:45:20.827986+0000 mon.a (mon.0) 4016 : cluster [DBG] osdmap e2275: 8 total, 8 up, 8 in 2024-09-13T00:45:22.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:45:22 smithi181 bash[95505]: ts=2024-09-13T00:45:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:45:23.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:22 smithi136 bash[76371]: cluster 2024-09-13T00:45:21.830276+0000 mon.a (mon.0) 4017 : cluster [DBG] osdmap e2276: 8 total, 8 up, 8 in 2024-09-13T00:45:23.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:22 smithi136 bash[75164]: cluster 2024-09-13T00:45:21.830276+0000 mon.a (mon.0) 4017 : cluster [DBG] osdmap e2276: 8 total, 8 up, 8 in 2024-09-13T00:45:23.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:22 smithi181 bash[67587]: cluster 2024-09-13T00:45:21.830276+0000 mon.a (mon.0) 4017 : cluster [DBG] osdmap e2276: 8 total, 8 up, 8 in 2024-09-13T00:45:24.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:23 smithi136 bash[75164]: cluster 2024-09-13T00:45:22.574308+0000 mgr.y (mgr.24661) 2047 : cluster [DBG] pgmap v3884: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.2 MiB/s rd, 3.5 KiB/s wr, 1.61k op/s 2024-09-13T00:45:24.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:23 smithi136 bash[75164]: cluster 2024-09-13T00:45:22.859693+0000 mon.a (mon.0) 4018 : cluster [DBG] osdmap e2277: 8 total, 8 up, 8 in 2024-09-13T00:45:24.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:23 smithi136 bash[76371]: cluster 2024-09-13T00:45:22.574308+0000 mgr.y (mgr.24661) 2047 : cluster [DBG] pgmap v3884: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.2 MiB/s rd, 3.5 KiB/s wr, 1.61k op/s 2024-09-13T00:45:24.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:23 smithi136 bash[76371]: cluster 2024-09-13T00:45:22.859693+0000 mon.a (mon.0) 4018 : cluster [DBG] osdmap e2277: 8 total, 8 up, 8 in 2024-09-13T00:45:24.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:23 smithi181 bash[67587]: cluster 2024-09-13T00:45:22.574308+0000 mgr.y (mgr.24661) 2047 : cluster [DBG] pgmap v3884: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.2 MiB/s rd, 3.5 KiB/s wr, 1.61k op/s 2024-09-13T00:45:24.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:23 smithi181 bash[67587]: cluster 2024-09-13T00:45:22.859693+0000 mon.a (mon.0) 4018 : cluster [DBG] osdmap e2277: 8 total, 8 up, 8 in 2024-09-13T00:45:25.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:24 smithi136 bash[75164]: audit 2024-09-13T00:45:23.846991+0000 mon.a (mon.0) 4019 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:45:25.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:24 smithi136 bash[75164]: cluster 2024-09-13T00:45:23.868522+0000 mon.a (mon.0) 4020 : cluster [DBG] osdmap e2278: 8 total, 8 up, 8 in 2024-09-13T00:45:25.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:24 smithi136 bash[76371]: audit 2024-09-13T00:45:23.846991+0000 mon.a (mon.0) 4019 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:45:25.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:24 smithi136 bash[76371]: cluster 2024-09-13T00:45:23.868522+0000 mon.a (mon.0) 4020 : cluster [DBG] osdmap e2278: 8 total, 8 up, 8 in 2024-09-13T00:45:25.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:24 smithi181 bash[67587]: audit 2024-09-13T00:45:23.846991+0000 mon.a (mon.0) 4019 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:45:25.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:24 smithi181 bash[67587]: cluster 2024-09-13T00:45:23.868522+0000 mon.a (mon.0) 4020 : cluster [DBG] osdmap e2278: 8 total, 8 up, 8 in 2024-09-13T00:45:26.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:25 smithi136 bash[75164]: cluster 2024-09-13T00:45:24.574982+0000 mgr.y (mgr.24661) 2048 : cluster [DBG] pgmap v3887: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.2 MiB/s rd, 3.7 KiB/s wr, 1.62k op/s 2024-09-13T00:45:26.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:25 smithi136 bash[75164]: cluster 2024-09-13T00:45:24.878971+0000 mon.a (mon.0) 4021 : cluster [DBG] osdmap e2279: 8 total, 8 up, 8 in 2024-09-13T00:45:26.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:25 smithi136 bash[76371]: cluster 2024-09-13T00:45:24.574982+0000 mgr.y (mgr.24661) 2048 : cluster [DBG] pgmap v3887: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.2 MiB/s rd, 3.7 KiB/s wr, 1.62k op/s 2024-09-13T00:45:26.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:25 smithi136 bash[76371]: cluster 2024-09-13T00:45:24.878971+0000 mon.a (mon.0) 4021 : cluster [DBG] osdmap e2279: 8 total, 8 up, 8 in 2024-09-13T00:45:26.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:25 smithi181 bash[67587]: cluster 2024-09-13T00:45:24.574982+0000 mgr.y (mgr.24661) 2048 : cluster [DBG] pgmap v3887: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.2 MiB/s rd, 3.7 KiB/s wr, 1.62k op/s 2024-09-13T00:45:26.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:25 smithi181 bash[67587]: cluster 2024-09-13T00:45:24.878971+0000 mon.a (mon.0) 4021 : cluster [DBG] osdmap e2279: 8 total, 8 up, 8 in 2024-09-13T00:45:27.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:26 smithi136 bash[75164]: cluster 2024-09-13T00:45:25.882223+0000 mon.a (mon.0) 4022 : cluster [DBG] osdmap e2280: 8 total, 8 up, 8 in 2024-09-13T00:45:27.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:26 smithi136 bash[76371]: cluster 2024-09-13T00:45:25.882223+0000 mon.a (mon.0) 4022 : cluster [DBG] osdmap e2280: 8 total, 8 up, 8 in 2024-09-13T00:45:27.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:26 smithi181 bash[67587]: cluster 2024-09-13T00:45:25.882223+0000 mon.a (mon.0) 4022 : cluster [DBG] osdmap e2280: 8 total, 8 up, 8 in 2024-09-13T00:45:28.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:27 smithi136 bash[75164]: cluster 2024-09-13T00:45:26.575967+0000 mgr.y (mgr.24661) 2049 : cluster [DBG] pgmap v3890: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 1.5 KiB/s wr, 7 op/s 2024-09-13T00:45:28.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:27 smithi136 bash[75164]: cluster 2024-09-13T00:45:26.906658+0000 mon.a (mon.0) 4023 : cluster [DBG] osdmap e2281: 8 total, 8 up, 8 in 2024-09-13T00:45:28.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:27 smithi136 bash[76371]: cluster 2024-09-13T00:45:26.575967+0000 mgr.y (mgr.24661) 2049 : cluster [DBG] pgmap v3890: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 1.5 KiB/s wr, 7 op/s 2024-09-13T00:45:28.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:27 smithi136 bash[76371]: cluster 2024-09-13T00:45:26.906658+0000 mon.a (mon.0) 4023 : cluster [DBG] osdmap e2281: 8 total, 8 up, 8 in 2024-09-13T00:45:28.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:27 smithi181 bash[67587]: cluster 2024-09-13T00:45:26.575967+0000 mgr.y (mgr.24661) 2049 : cluster [DBG] pgmap v3890: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 1.5 KiB/s wr, 7 op/s 2024-09-13T00:45:28.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:27 smithi181 bash[67587]: cluster 2024-09-13T00:45:26.906658+0000 mon.a (mon.0) 4023 : cluster [DBG] osdmap e2281: 8 total, 8 up, 8 in 2024-09-13T00:45:29.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:28 smithi136 bash[75164]: cluster 2024-09-13T00:45:27.907659+0000 mon.a (mon.0) 4024 : cluster [DBG] osdmap e2282: 8 total, 8 up, 8 in 2024-09-13T00:45:29.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:28 smithi136 bash[76371]: cluster 2024-09-13T00:45:27.907659+0000 mon.a (mon.0) 4024 : cluster [DBG] osdmap e2282: 8 total, 8 up, 8 in 2024-09-13T00:45:29.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:28 smithi181 bash[67587]: cluster 2024-09-13T00:45:27.907659+0000 mon.a (mon.0) 4024 : cluster [DBG] osdmap e2282: 8 total, 8 up, 8 in 2024-09-13T00:45:29.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:45:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:45:29] "GET /metrics HTTP/1.1" 200 36744 "" "Prometheus/2.43.0" 2024-09-13T00:45:30.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:29 smithi136 bash[75164]: cluster 2024-09-13T00:45:28.576867+0000 mgr.y (mgr.24661) 2050 : cluster [DBG] pgmap v3893: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.2 MiB/s rd, 3.5 KiB/s wr, 1.66k op/s 2024-09-13T00:45:30.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:29 smithi136 bash[75164]: cluster 2024-09-13T00:45:28.930001+0000 mon.a (mon.0) 4025 : cluster [DBG] osdmap e2283: 8 total, 8 up, 8 in 2024-09-13T00:45:30.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:29 smithi136 bash[76371]: cluster 2024-09-13T00:45:28.576867+0000 mgr.y (mgr.24661) 2050 : cluster [DBG] pgmap v3893: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.2 MiB/s rd, 3.5 KiB/s wr, 1.66k op/s 2024-09-13T00:45:30.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:29 smithi136 bash[76371]: cluster 2024-09-13T00:45:28.930001+0000 mon.a (mon.0) 4025 : cluster [DBG] osdmap e2283: 8 total, 8 up, 8 in 2024-09-13T00:45:30.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:29 smithi181 bash[67587]: cluster 2024-09-13T00:45:28.576867+0000 mgr.y (mgr.24661) 2050 : cluster [DBG] pgmap v3893: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.2 MiB/s rd, 3.5 KiB/s wr, 1.66k op/s 2024-09-13T00:45:30.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:29 smithi181 bash[67587]: cluster 2024-09-13T00:45:28.930001+0000 mon.a (mon.0) 4025 : cluster [DBG] osdmap e2283: 8 total, 8 up, 8 in 2024-09-13T00:45:31.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:30 smithi136 bash[75164]: cluster 2024-09-13T00:45:29.941400+0000 mon.a (mon.0) 4026 : cluster [DBG] osdmap e2284: 8 total, 8 up, 8 in 2024-09-13T00:45:31.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:30 smithi136 bash[76371]: cluster 2024-09-13T00:45:29.941400+0000 mon.a (mon.0) 4026 : cluster [DBG] osdmap e2284: 8 total, 8 up, 8 in 2024-09-13T00:45:31.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:30 smithi181 bash[67587]: cluster 2024-09-13T00:45:29.941400+0000 mon.a (mon.0) 4026 : cluster [DBG] osdmap e2284: 8 total, 8 up, 8 in 2024-09-13T00:45:32.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:31 smithi136 bash[75164]: cluster 2024-09-13T00:45:30.577585+0000 mgr.y (mgr.24661) 2051 : cluster [DBG] pgmap v3896: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.2 MiB/s rd, 2.7 KiB/s wr, 1.66k op/s 2024-09-13T00:45:32.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:31 smithi136 bash[75164]: cluster 2024-09-13T00:45:30.953558+0000 mon.a (mon.0) 4027 : cluster [DBG] osdmap e2285: 8 total, 8 up, 8 in 2024-09-13T00:45:32.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:31 smithi136 bash[76371]: cluster 2024-09-13T00:45:30.577585+0000 mgr.y (mgr.24661) 2051 : cluster [DBG] pgmap v3896: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.2 MiB/s rd, 2.7 KiB/s wr, 1.66k op/s 2024-09-13T00:45:32.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:31 smithi136 bash[76371]: cluster 2024-09-13T00:45:30.953558+0000 mon.a (mon.0) 4027 : cluster [DBG] osdmap e2285: 8 total, 8 up, 8 in 2024-09-13T00:45:32.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:31 smithi181 bash[67587]: cluster 2024-09-13T00:45:30.577585+0000 mgr.y (mgr.24661) 2051 : cluster [DBG] pgmap v3896: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.2 MiB/s rd, 2.7 KiB/s wr, 1.66k op/s 2024-09-13T00:45:32.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:31 smithi181 bash[67587]: cluster 2024-09-13T00:45:30.953558+0000 mon.a (mon.0) 4027 : cluster [DBG] osdmap e2285: 8 total, 8 up, 8 in 2024-09-13T00:45:32.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:45:32 smithi181 bash[95505]: ts=2024-09-13T00:45:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:45:33.259 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:32 smithi136 bash[75164]: cluster 2024-09-13T00:45:31.952847+0000 mon.a (mon.0) 4028 : cluster [DBG] osdmap e2286: 8 total, 8 up, 8 in 2024-09-13T00:45:33.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:32 smithi136 bash[76371]: cluster 2024-09-13T00:45:31.952847+0000 mon.a (mon.0) 4028 : cluster [DBG] osdmap e2286: 8 total, 8 up, 8 in 2024-09-13T00:45:33.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:32 smithi181 bash[67587]: cluster 2024-09-13T00:45:31.952847+0000 mon.a (mon.0) 4028 : cluster [DBG] osdmap e2286: 8 total, 8 up, 8 in 2024-09-13T00:45:34.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:33 smithi136 bash[75164]: cluster 2024-09-13T00:45:32.578299+0000 mgr.y (mgr.24661) 2052 : cluster [DBG] pgmap v3899: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.2 MiB/s rd, 3.5 KiB/s wr, 1.70k op/s 2024-09-13T00:45:34.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:33 smithi136 bash[75164]: cluster 2024-09-13T00:45:32.975300+0000 mon.a (mon.0) 4029 : cluster [DBG] osdmap e2287: 8 total, 8 up, 8 in 2024-09-13T00:45:34.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:33 smithi136 bash[76371]: cluster 2024-09-13T00:45:32.578299+0000 mgr.y (mgr.24661) 2052 : cluster [DBG] pgmap v3899: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.2 MiB/s rd, 3.5 KiB/s wr, 1.70k op/s 2024-09-13T00:45:34.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:33 smithi136 bash[76371]: cluster 2024-09-13T00:45:32.975300+0000 mon.a (mon.0) 4029 : cluster [DBG] osdmap e2287: 8 total, 8 up, 8 in 2024-09-13T00:45:34.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:33 smithi181 bash[67587]: cluster 2024-09-13T00:45:32.578299+0000 mgr.y (mgr.24661) 2052 : cluster [DBG] pgmap v3899: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.2 MiB/s rd, 3.5 KiB/s wr, 1.70k op/s 2024-09-13T00:45:34.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:33 smithi181 bash[67587]: cluster 2024-09-13T00:45:32.975300+0000 mon.a (mon.0) 4029 : cluster [DBG] osdmap e2287: 8 total, 8 up, 8 in 2024-09-13T00:45:35.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:34 smithi136 bash[75164]: cluster 2024-09-13T00:45:33.976780+0000 mon.a (mon.0) 4030 : cluster [DBG] osdmap e2288: 8 total, 8 up, 8 in 2024-09-13T00:45:35.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:34 smithi136 bash[76371]: cluster 2024-09-13T00:45:33.976780+0000 mon.a (mon.0) 4030 : cluster [DBG] osdmap e2288: 8 total, 8 up, 8 in 2024-09-13T00:45:35.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:34 smithi181 bash[67587]: cluster 2024-09-13T00:45:33.976780+0000 mon.a (mon.0) 4030 : cluster [DBG] osdmap e2288: 8 total, 8 up, 8 in 2024-09-13T00:45:36.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:36 smithi136 bash[75164]: cluster 2024-09-13T00:45:34.578970+0000 mgr.y (mgr.24661) 2053 : cluster [DBG] pgmap v3902: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.2 MiB/s rd, 3.5 KiB/s wr, 1.71k op/s 2024-09-13T00:45:36.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:36 smithi136 bash[75164]: cluster 2024-09-13T00:45:34.997350+0000 mon.a (mon.0) 4031 : cluster [DBG] osdmap e2289: 8 total, 8 up, 8 in 2024-09-13T00:45:36.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:36 smithi136 bash[76371]: cluster 2024-09-13T00:45:34.578970+0000 mgr.y (mgr.24661) 2053 : cluster [DBG] pgmap v3902: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.2 MiB/s rd, 3.5 KiB/s wr, 1.71k op/s 2024-09-13T00:45:36.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:36 smithi136 bash[76371]: cluster 2024-09-13T00:45:34.997350+0000 mon.a (mon.0) 4031 : cluster [DBG] osdmap e2289: 8 total, 8 up, 8 in 2024-09-13T00:45:36.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:36 smithi181 bash[67587]: cluster 2024-09-13T00:45:34.578970+0000 mgr.y (mgr.24661) 2053 : cluster [DBG] pgmap v3902: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.2 MiB/s rd, 3.5 KiB/s wr, 1.71k op/s 2024-09-13T00:45:36.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:36 smithi181 bash[67587]: cluster 2024-09-13T00:45:34.997350+0000 mon.a (mon.0) 4031 : cluster [DBG] osdmap e2289: 8 total, 8 up, 8 in 2024-09-13T00:45:37.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:37 smithi181 bash[67587]: cluster 2024-09-13T00:45:35.999702+0000 mon.a (mon.0) 4032 : cluster [DBG] osdmap e2290: 8 total, 8 up, 8 in 2024-09-13T00:45:37.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:37 smithi136 bash[75164]: cluster 2024-09-13T00:45:35.999702+0000 mon.a (mon.0) 4032 : cluster [DBG] osdmap e2290: 8 total, 8 up, 8 in 2024-09-13T00:45:37.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:37 smithi136 bash[76371]: cluster 2024-09-13T00:45:35.999702+0000 mon.a (mon.0) 4032 : cluster [DBG] osdmap e2290: 8 total, 8 up, 8 in 2024-09-13T00:45:38.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:38 smithi181 bash[67587]: cluster 2024-09-13T00:45:36.579956+0000 mgr.y (mgr.24661) 2054 : cluster [DBG] pgmap v3905: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 1023 B/s wr, 7 op/s 2024-09-13T00:45:38.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:38 smithi181 bash[67587]: cluster 2024-09-13T00:45:37.021847+0000 mon.a (mon.0) 4033 : cluster [DBG] osdmap e2291: 8 total, 8 up, 8 in 2024-09-13T00:45:38.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:38 smithi136 bash[75164]: cluster 2024-09-13T00:45:36.579956+0000 mgr.y (mgr.24661) 2054 : cluster [DBG] pgmap v3905: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 1023 B/s wr, 7 op/s 2024-09-13T00:45:38.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:38 smithi136 bash[75164]: cluster 2024-09-13T00:45:37.021847+0000 mon.a (mon.0) 4033 : cluster [DBG] osdmap e2291: 8 total, 8 up, 8 in 2024-09-13T00:45:38.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:38 smithi136 bash[76371]: cluster 2024-09-13T00:45:36.579956+0000 mgr.y (mgr.24661) 2054 : cluster [DBG] pgmap v3905: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 1023 B/s wr, 7 op/s 2024-09-13T00:45:38.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:38 smithi136 bash[76371]: cluster 2024-09-13T00:45:37.021847+0000 mon.a (mon.0) 4033 : cluster [DBG] osdmap e2291: 8 total, 8 up, 8 in 2024-09-13T00:45:39.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:39 smithi181 bash[67587]: cluster 2024-09-13T00:45:38.030498+0000 mon.a (mon.0) 4034 : cluster [DBG] osdmap e2292: 8 total, 8 up, 8 in 2024-09-13T00:45:39.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:39 smithi181 bash[67587]: audit 2024-09-13T00:45:38.847058+0000 mon.a (mon.0) 4035 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:45:39.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:39 smithi136 bash[75164]: cluster 2024-09-13T00:45:38.030498+0000 mon.a (mon.0) 4034 : cluster [DBG] osdmap e2292: 8 total, 8 up, 8 in 2024-09-13T00:45:39.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:39 smithi136 bash[75164]: audit 2024-09-13T00:45:38.847058+0000 mon.a (mon.0) 4035 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:45:39.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:39 smithi136 bash[76371]: cluster 2024-09-13T00:45:38.030498+0000 mon.a (mon.0) 4034 : cluster [DBG] osdmap e2292: 8 total, 8 up, 8 in 2024-09-13T00:45:39.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:39 smithi136 bash[76371]: audit 2024-09-13T00:45:38.847058+0000 mon.a (mon.0) 4035 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:45:39.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:45:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:45:39] "GET /metrics HTTP/1.1" 200 36749 "" "Prometheus/2.43.0" 2024-09-13T00:45:40.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:40 smithi181 bash[67587]: cluster 2024-09-13T00:45:38.580995+0000 mgr.y (mgr.24661) 2055 : cluster [DBG] pgmap v3908: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.3 MiB/s rd, 3.7 KiB/s wr, 1.75k op/s 2024-09-13T00:45:40.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:40 smithi181 bash[67587]: cluster 2024-09-13T00:45:39.039956+0000 mon.a (mon.0) 4036 : cluster [DBG] osdmap e2293: 8 total, 8 up, 8 in 2024-09-13T00:45:40.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:40 smithi136 bash[75164]: cluster 2024-09-13T00:45:38.580995+0000 mgr.y (mgr.24661) 2055 : cluster [DBG] pgmap v3908: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.3 MiB/s rd, 3.7 KiB/s wr, 1.75k op/s 2024-09-13T00:45:40.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:40 smithi136 bash[75164]: cluster 2024-09-13T00:45:39.039956+0000 mon.a (mon.0) 4036 : cluster [DBG] osdmap e2293: 8 total, 8 up, 8 in 2024-09-13T00:45:40.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:40 smithi136 bash[76371]: cluster 2024-09-13T00:45:38.580995+0000 mgr.y (mgr.24661) 2055 : cluster [DBG] pgmap v3908: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.3 MiB/s rd, 3.7 KiB/s wr, 1.75k op/s 2024-09-13T00:45:40.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:40 smithi136 bash[76371]: cluster 2024-09-13T00:45:39.039956+0000 mon.a (mon.0) 4036 : cluster [DBG] osdmap e2293: 8 total, 8 up, 8 in 2024-09-13T00:45:41.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:41 smithi181 bash[67587]: cluster 2024-09-13T00:45:40.050183+0000 mon.a (mon.0) 4037 : cluster [DBG] osdmap e2294: 8 total, 8 up, 8 in 2024-09-13T00:45:41.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:41 smithi136 bash[75164]: cluster 2024-09-13T00:45:40.050183+0000 mon.a (mon.0) 4037 : cluster [DBG] osdmap e2294: 8 total, 8 up, 8 in 2024-09-13T00:45:41.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:41 smithi136 bash[76371]: cluster 2024-09-13T00:45:40.050183+0000 mon.a (mon.0) 4037 : cluster [DBG] osdmap e2294: 8 total, 8 up, 8 in 2024-09-13T00:45:42.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:42 smithi181 bash[67587]: cluster 2024-09-13T00:45:40.581708+0000 mgr.y (mgr.24661) 2056 : cluster [DBG] pgmap v3911: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.3 MiB/s rd, 3.0 KiB/s wr, 1.75k op/s 2024-09-13T00:45:42.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:42 smithi181 bash[67587]: cluster 2024-09-13T00:45:41.059848+0000 mon.a (mon.0) 4038 : cluster [DBG] osdmap e2295: 8 total, 8 up, 8 in 2024-09-13T00:45:42.348 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:45:42 smithi181 bash[95505]: ts=2024-09-13T00:45:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:45:42.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:42 smithi136 bash[75164]: cluster 2024-09-13T00:45:40.581708+0000 mgr.y (mgr.24661) 2056 : cluster [DBG] pgmap v3911: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.3 MiB/s rd, 3.0 KiB/s wr, 1.75k op/s 2024-09-13T00:45:42.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:42 smithi136 bash[75164]: cluster 2024-09-13T00:45:41.059848+0000 mon.a (mon.0) 4038 : cluster [DBG] osdmap e2295: 8 total, 8 up, 8 in 2024-09-13T00:45:42.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:42 smithi136 bash[76371]: cluster 2024-09-13T00:45:40.581708+0000 mgr.y (mgr.24661) 2056 : cluster [DBG] pgmap v3911: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.3 MiB/s rd, 3.0 KiB/s wr, 1.75k op/s 2024-09-13T00:45:42.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:42 smithi136 bash[76371]: cluster 2024-09-13T00:45:41.059848+0000 mon.a (mon.0) 4038 : cluster [DBG] osdmap e2295: 8 total, 8 up, 8 in 2024-09-13T00:45:43.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:43 smithi181 bash[67587]: cluster 2024-09-13T00:45:42.061067+0000 mon.a (mon.0) 4039 : cluster [DBG] osdmap e2296: 8 total, 8 up, 8 in 2024-09-13T00:45:43.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:43 smithi136 bash[75164]: cluster 2024-09-13T00:45:42.061067+0000 mon.a (mon.0) 4039 : cluster [DBG] osdmap e2296: 8 total, 8 up, 8 in 2024-09-13T00:45:43.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:43 smithi136 bash[76371]: cluster 2024-09-13T00:45:42.061067+0000 mon.a (mon.0) 4039 : cluster [DBG] osdmap e2296: 8 total, 8 up, 8 in 2024-09-13T00:45:44.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:44 smithi181 bash[67587]: cluster 2024-09-13T00:45:42.582399+0000 mgr.y (mgr.24661) 2057 : cluster [DBG] pgmap v3914: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.3 MiB/s rd, 5.0 KiB/s wr, 1.79k op/s 2024-09-13T00:45:44.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:44 smithi181 bash[67587]: cluster 2024-09-13T00:45:43.085541+0000 mon.a (mon.0) 4040 : cluster [DBG] osdmap e2297: 8 total, 8 up, 8 in 2024-09-13T00:45:44.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:44 smithi136 bash[75164]: cluster 2024-09-13T00:45:42.582399+0000 mgr.y (mgr.24661) 2057 : cluster [DBG] pgmap v3914: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.3 MiB/s rd, 5.0 KiB/s wr, 1.79k op/s 2024-09-13T00:45:44.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:44 smithi136 bash[75164]: cluster 2024-09-13T00:45:43.085541+0000 mon.a (mon.0) 4040 : cluster [DBG] osdmap e2297: 8 total, 8 up, 8 in 2024-09-13T00:45:44.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:44 smithi136 bash[76371]: cluster 2024-09-13T00:45:42.582399+0000 mgr.y (mgr.24661) 2057 : cluster [DBG] pgmap v3914: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.3 MiB/s rd, 5.0 KiB/s wr, 1.79k op/s 2024-09-13T00:45:44.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:44 smithi136 bash[76371]: cluster 2024-09-13T00:45:43.085541+0000 mon.a (mon.0) 4040 : cluster [DBG] osdmap e2297: 8 total, 8 up, 8 in 2024-09-13T00:45:45.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:45 smithi136 bash[75164]: cluster 2024-09-13T00:45:44.091979+0000 mon.a (mon.0) 4041 : cluster [DBG] osdmap e2298: 8 total, 8 up, 8 in 2024-09-13T00:45:45.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:45 smithi136 bash[76371]: cluster 2024-09-13T00:45:44.091979+0000 mon.a (mon.0) 4041 : cluster [DBG] osdmap e2298: 8 total, 8 up, 8 in 2024-09-13T00:45:45.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:45 smithi181 bash[67587]: cluster 2024-09-13T00:45:44.091979+0000 mon.a (mon.0) 4041 : cluster [DBG] osdmap e2298: 8 total, 8 up, 8 in 2024-09-13T00:45:46.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:46 smithi181 bash[67587]: cluster 2024-09-13T00:45:44.583094+0000 mgr.y (mgr.24661) 2058 : cluster [DBG] pgmap v3917: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.3 MiB/s rd, 5.0 KiB/s wr, 1.79k op/s 2024-09-13T00:45:46.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:46 smithi181 bash[67587]: cluster 2024-09-13T00:45:45.102848+0000 mon.a (mon.0) 4042 : cluster [DBG] osdmap e2299: 8 total, 8 up, 8 in 2024-09-13T00:45:46.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:46 smithi136 bash[75164]: cluster 2024-09-13T00:45:44.583094+0000 mgr.y (mgr.24661) 2058 : cluster [DBG] pgmap v3917: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.3 MiB/s rd, 5.0 KiB/s wr, 1.79k op/s 2024-09-13T00:45:46.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:46 smithi136 bash[75164]: cluster 2024-09-13T00:45:45.102848+0000 mon.a (mon.0) 4042 : cluster [DBG] osdmap e2299: 8 total, 8 up, 8 in 2024-09-13T00:45:46.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:46 smithi136 bash[76371]: cluster 2024-09-13T00:45:44.583094+0000 mgr.y (mgr.24661) 2058 : cluster [DBG] pgmap v3917: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.3 MiB/s rd, 5.0 KiB/s wr, 1.79k op/s 2024-09-13T00:45:46.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:46 smithi136 bash[76371]: cluster 2024-09-13T00:45:45.102848+0000 mon.a (mon.0) 4042 : cluster [DBG] osdmap e2299: 8 total, 8 up, 8 in 2024-09-13T00:45:47.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:47 smithi136 bash[75164]: cluster 2024-09-13T00:45:46.112611+0000 mon.a (mon.0) 4043 : cluster [DBG] osdmap e2300: 8 total, 8 up, 8 in 2024-09-13T00:45:47.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:47 smithi136 bash[76371]: cluster 2024-09-13T00:45:46.112611+0000 mon.a (mon.0) 4043 : cluster [DBG] osdmap e2300: 8 total, 8 up, 8 in 2024-09-13T00:45:47.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:47 smithi181 bash[67587]: cluster 2024-09-13T00:45:46.112611+0000 mon.a (mon.0) 4043 : cluster [DBG] osdmap e2300: 8 total, 8 up, 8 in 2024-09-13T00:45:48.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:48 smithi136 bash[75164]: cluster 2024-09-13T00:45:46.583987+0000 mgr.y (mgr.24661) 2059 : cluster [DBG] pgmap v3920: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.2 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T00:45:48.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:48 smithi136 bash[75164]: cluster 2024-09-13T00:45:47.127110+0000 mon.a (mon.0) 4044 : cluster [DBG] osdmap e2301: 8 total, 8 up, 8 in 2024-09-13T00:45:48.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:48 smithi136 bash[76371]: cluster 2024-09-13T00:45:46.583987+0000 mgr.y (mgr.24661) 2059 : cluster [DBG] pgmap v3920: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.2 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T00:45:48.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:48 smithi136 bash[76371]: cluster 2024-09-13T00:45:47.127110+0000 mon.a (mon.0) 4044 : cluster [DBG] osdmap e2301: 8 total, 8 up, 8 in 2024-09-13T00:45:48.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:48 smithi181 bash[67587]: cluster 2024-09-13T00:45:46.583987+0000 mgr.y (mgr.24661) 2059 : cluster [DBG] pgmap v3920: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.2 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T00:45:48.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:48 smithi181 bash[67587]: cluster 2024-09-13T00:45:47.127110+0000 mon.a (mon.0) 4044 : cluster [DBG] osdmap e2301: 8 total, 8 up, 8 in 2024-09-13T00:45:49.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:49 smithi136 bash[75164]: cluster 2024-09-13T00:45:48.136186+0000 mon.a (mon.0) 4045 : cluster [DBG] osdmap e2302: 8 total, 8 up, 8 in 2024-09-13T00:45:49.414 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:49 smithi136 bash[76371]: cluster 2024-09-13T00:45:48.136186+0000 mon.a (mon.0) 4045 : cluster [DBG] osdmap e2302: 8 total, 8 up, 8 in 2024-09-13T00:45:49.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:49 smithi181 bash[67587]: cluster 2024-09-13T00:45:48.136186+0000 mon.a (mon.0) 4045 : cluster [DBG] osdmap e2302: 8 total, 8 up, 8 in 2024-09-13T00:45:49.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:45:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:45:49] "GET /metrics HTTP/1.1" 200 36749 "" "Prometheus/2.43.0" 2024-09-13T00:45:50.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:50 smithi136 bash[75164]: cluster 2024-09-13T00:45:48.584912+0000 mgr.y (mgr.24661) 2060 : cluster [DBG] pgmap v3923: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.3 MiB/s rd, 5.0 KiB/s wr, 1.76k op/s 2024-09-13T00:45:50.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:50 smithi136 bash[75164]: cluster 2024-09-13T00:45:49.147800+0000 mon.a (mon.0) 4046 : cluster [DBG] osdmap e2303: 8 total, 8 up, 8 in 2024-09-13T00:45:50.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:50 smithi136 bash[75164]: cluster 2024-09-13T00:45:49.886975+0000 mon.a (mon.0) 4047 : cluster [DBG] osdmap e2304: 8 total, 8 up, 8 in 2024-09-13T00:45:50.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:50 smithi136 bash[76371]: cluster 2024-09-13T00:45:48.584912+0000 mgr.y (mgr.24661) 2060 : cluster [DBG] pgmap v3923: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.3 MiB/s rd, 5.0 KiB/s wr, 1.76k op/s 2024-09-13T00:45:50.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:50 smithi136 bash[76371]: cluster 2024-09-13T00:45:49.147800+0000 mon.a (mon.0) 4046 : cluster [DBG] osdmap e2303: 8 total, 8 up, 8 in 2024-09-13T00:45:50.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:50 smithi136 bash[76371]: cluster 2024-09-13T00:45:49.886975+0000 mon.a (mon.0) 4047 : cluster [DBG] osdmap e2304: 8 total, 8 up, 8 in 2024-09-13T00:45:50.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:50 smithi181 bash[67587]: cluster 2024-09-13T00:45:48.584912+0000 mgr.y (mgr.24661) 2060 : cluster [DBG] pgmap v3923: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.3 MiB/s rd, 5.0 KiB/s wr, 1.76k op/s 2024-09-13T00:45:50.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:50 smithi181 bash[67587]: cluster 2024-09-13T00:45:49.147800+0000 mon.a (mon.0) 4046 : cluster [DBG] osdmap e2303: 8 total, 8 up, 8 in 2024-09-13T00:45:50.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:50 smithi181 bash[67587]: cluster 2024-09-13T00:45:49.886975+0000 mon.a (mon.0) 4047 : cluster [DBG] osdmap e2304: 8 total, 8 up, 8 in 2024-09-13T00:45:52.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:51 smithi136 bash[75164]: cluster 2024-09-13T00:45:50.585768+0000 mgr.y (mgr.24661) 2061 : cluster [DBG] pgmap v3926: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.3 MiB/s rd, 5.0 KiB/s wr, 1.76k op/s 2024-09-13T00:45:52.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:51 smithi136 bash[75164]: cluster 2024-09-13T00:45:50.886741+0000 mon.a (mon.0) 4048 : cluster [DBG] osdmap e2305: 8 total, 8 up, 8 in 2024-09-13T00:45:52.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:51 smithi136 bash[76371]: cluster 2024-09-13T00:45:50.585768+0000 mgr.y (mgr.24661) 2061 : cluster [DBG] pgmap v3926: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.3 MiB/s rd, 5.0 KiB/s wr, 1.76k op/s 2024-09-13T00:45:52.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:51 smithi136 bash[76371]: cluster 2024-09-13T00:45:50.886741+0000 mon.a (mon.0) 4048 : cluster [DBG] osdmap e2305: 8 total, 8 up, 8 in 2024-09-13T00:45:52.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:51 smithi181 bash[67587]: cluster 2024-09-13T00:45:50.585768+0000 mgr.y (mgr.24661) 2061 : cluster [DBG] pgmap v3926: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.3 MiB/s rd, 5.0 KiB/s wr, 1.76k op/s 2024-09-13T00:45:52.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:51 smithi181 bash[67587]: cluster 2024-09-13T00:45:50.886741+0000 mon.a (mon.0) 4048 : cluster [DBG] osdmap e2305: 8 total, 8 up, 8 in 2024-09-13T00:45:52.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:45:52 smithi181 bash[95505]: ts=2024-09-13T00:45:52.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:45:53.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:52 smithi136 bash[75164]: cluster 2024-09-13T00:45:51.895323+0000 mon.a (mon.0) 4049 : cluster [DBG] osdmap e2306: 8 total, 8 up, 8 in 2024-09-13T00:45:53.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:52 smithi136 bash[76371]: cluster 2024-09-13T00:45:51.895323+0000 mon.a (mon.0) 4049 : cluster [DBG] osdmap e2306: 8 total, 8 up, 8 in 2024-09-13T00:45:53.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:52 smithi181 bash[67587]: cluster 2024-09-13T00:45:51.895323+0000 mon.a (mon.0) 4049 : cluster [DBG] osdmap e2306: 8 total, 8 up, 8 in 2024-09-13T00:45:54.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:53 smithi136 bash[75164]: cluster 2024-09-13T00:45:52.586492+0000 mgr.y (mgr.24661) 2062 : cluster [DBG] pgmap v3929: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.2 MiB/s rd, 5.0 KiB/s wr, 1.71k op/s 2024-09-13T00:45:54.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:53 smithi136 bash[75164]: cluster 2024-09-13T00:45:52.906550+0000 mon.a (mon.0) 4050 : cluster [DBG] osdmap e2307: 8 total, 8 up, 8 in 2024-09-13T00:45:54.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:53 smithi136 bash[75164]: audit 2024-09-13T00:45:53.847620+0000 mon.a (mon.0) 4051 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:45:54.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:53 smithi136 bash[76371]: cluster 2024-09-13T00:45:52.586492+0000 mgr.y (mgr.24661) 2062 : cluster [DBG] pgmap v3929: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.2 MiB/s rd, 5.0 KiB/s wr, 1.71k op/s 2024-09-13T00:45:54.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:53 smithi136 bash[76371]: cluster 2024-09-13T00:45:52.906550+0000 mon.a (mon.0) 4050 : cluster [DBG] osdmap e2307: 8 total, 8 up, 8 in 2024-09-13T00:45:54.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:53 smithi136 bash[76371]: audit 2024-09-13T00:45:53.847620+0000 mon.a (mon.0) 4051 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:45:54.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:53 smithi181 bash[67587]: cluster 2024-09-13T00:45:52.586492+0000 mgr.y (mgr.24661) 2062 : cluster [DBG] pgmap v3929: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.2 MiB/s rd, 5.0 KiB/s wr, 1.71k op/s 2024-09-13T00:45:54.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:53 smithi181 bash[67587]: cluster 2024-09-13T00:45:52.906550+0000 mon.a (mon.0) 4050 : cluster [DBG] osdmap e2307: 8 total, 8 up, 8 in 2024-09-13T00:45:54.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:53 smithi181 bash[67587]: audit 2024-09-13T00:45:53.847620+0000 mon.a (mon.0) 4051 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:45:55.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:54 smithi136 bash[75164]: cluster 2024-09-13T00:45:53.907018+0000 mon.a (mon.0) 4052 : cluster [DBG] osdmap e2308: 8 total, 8 up, 8 in 2024-09-13T00:45:55.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:54 smithi136 bash[75164]: cluster 2024-09-13T00:45:54.886279+0000 mon.a (mon.0) 4053 : cluster [DBG] osdmap e2309: 8 total, 8 up, 8 in 2024-09-13T00:45:55.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:54 smithi136 bash[76371]: cluster 2024-09-13T00:45:53.907018+0000 mon.a (mon.0) 4052 : cluster [DBG] osdmap e2308: 8 total, 8 up, 8 in 2024-09-13T00:45:55.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:54 smithi136 bash[76371]: cluster 2024-09-13T00:45:54.886279+0000 mon.a (mon.0) 4053 : cluster [DBG] osdmap e2309: 8 total, 8 up, 8 in 2024-09-13T00:45:55.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:54 smithi181 bash[67587]: cluster 2024-09-13T00:45:53.907018+0000 mon.a (mon.0) 4052 : cluster [DBG] osdmap e2308: 8 total, 8 up, 8 in 2024-09-13T00:45:55.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:54 smithi181 bash[67587]: cluster 2024-09-13T00:45:54.886279+0000 mon.a (mon.0) 4053 : cluster [DBG] osdmap e2309: 8 total, 8 up, 8 in 2024-09-13T00:45:56.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:55 smithi136 bash[75164]: cluster 2024-09-13T00:45:54.587156+0000 mgr.y (mgr.24661) 2063 : cluster [DBG] pgmap v3932: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.2 MiB/s rd, 5.0 KiB/s wr, 1.71k op/s 2024-09-13T00:45:56.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:55 smithi136 bash[76371]: cluster 2024-09-13T00:45:54.587156+0000 mgr.y (mgr.24661) 2063 : cluster [DBG] pgmap v3932: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.2 MiB/s rd, 5.0 KiB/s wr, 1.71k op/s 2024-09-13T00:45:56.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:55 smithi181 bash[67587]: cluster 2024-09-13T00:45:54.587156+0000 mgr.y (mgr.24661) 2063 : cluster [DBG] pgmap v3932: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.2 MiB/s rd, 5.0 KiB/s wr, 1.71k op/s 2024-09-13T00:45:57.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:56 smithi136 bash[76371]: cluster 2024-09-13T00:45:55.947807+0000 mon.a (mon.0) 4054 : cluster [DBG] osdmap e2310: 8 total, 8 up, 8 in 2024-09-13T00:45:57.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:56 smithi136 bash[76371]: audit 2024-09-13T00:45:56.616477+0000 mon.a (mon.0) 4055 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:45:57.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:56 smithi136 bash[75164]: cluster 2024-09-13T00:45:55.947807+0000 mon.a (mon.0) 4054 : cluster [DBG] osdmap e2310: 8 total, 8 up, 8 in 2024-09-13T00:45:57.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:56 smithi136 bash[75164]: audit 2024-09-13T00:45:56.616477+0000 mon.a (mon.0) 4055 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:45:57.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:56 smithi181 bash[67587]: cluster 2024-09-13T00:45:55.947807+0000 mon.a (mon.0) 4054 : cluster [DBG] osdmap e2310: 8 total, 8 up, 8 in 2024-09-13T00:45:57.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:56 smithi181 bash[67587]: audit 2024-09-13T00:45:56.616477+0000 mon.a (mon.0) 4055 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:45:58.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:57 smithi136 bash[76371]: cluster 2024-09-13T00:45:56.588168+0000 mgr.y (mgr.24661) 2064 : cluster [DBG] pgmap v3935: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 3.7 KiB/s rd, 0 B/s wr, 7 op/s 2024-09-13T00:45:58.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:57 smithi136 bash[76371]: cluster 2024-09-13T00:45:56.954114+0000 mon.a (mon.0) 4056 : cluster [DBG] osdmap e2311: 8 total, 8 up, 8 in 2024-09-13T00:45:58.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:57 smithi136 bash[76371]: audit 2024-09-13T00:45:57.143579+0000 mon.a (mon.0) 4057 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:45:58.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:57 smithi136 bash[76371]: audit 2024-09-13T00:45:57.145028+0000 mon.a (mon.0) 4058 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:45:58.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:57 smithi136 bash[76371]: audit 2024-09-13T00:45:57.154127+0000 mon.a (mon.0) 4059 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:45:58.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:57 smithi136 bash[75164]: cluster 2024-09-13T00:45:56.588168+0000 mgr.y (mgr.24661) 2064 : cluster [DBG] pgmap v3935: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 3.7 KiB/s rd, 0 B/s wr, 7 op/s 2024-09-13T00:45:58.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:57 smithi136 bash[75164]: cluster 2024-09-13T00:45:56.954114+0000 mon.a (mon.0) 4056 : cluster [DBG] osdmap e2311: 8 total, 8 up, 8 in 2024-09-13T00:45:58.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:57 smithi136 bash[75164]: audit 2024-09-13T00:45:57.143579+0000 mon.a (mon.0) 4057 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:45:58.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:57 smithi136 bash[75164]: audit 2024-09-13T00:45:57.145028+0000 mon.a (mon.0) 4058 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:45:58.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:57 smithi136 bash[75164]: audit 2024-09-13T00:45:57.154127+0000 mon.a (mon.0) 4059 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:45:58.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:57 smithi181 bash[67587]: cluster 2024-09-13T00:45:56.588168+0000 mgr.y (mgr.24661) 2064 : cluster [DBG] pgmap v3935: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 3.7 KiB/s rd, 0 B/s wr, 7 op/s 2024-09-13T00:45:58.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:57 smithi181 bash[67587]: cluster 2024-09-13T00:45:56.954114+0000 mon.a (mon.0) 4056 : cluster [DBG] osdmap e2311: 8 total, 8 up, 8 in 2024-09-13T00:45:58.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:57 smithi181 bash[67587]: audit 2024-09-13T00:45:57.143579+0000 mon.a (mon.0) 4057 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:45:58.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:57 smithi181 bash[67587]: audit 2024-09-13T00:45:57.145028+0000 mon.a (mon.0) 4058 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:45:58.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:57 smithi181 bash[67587]: audit 2024-09-13T00:45:57.154127+0000 mon.a (mon.0) 4059 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:45:59.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:58 smithi136 bash[75164]: cluster 2024-09-13T00:45:57.959185+0000 mon.a (mon.0) 4060 : cluster [DBG] osdmap e2312: 8 total, 8 up, 8 in 2024-09-13T00:45:59.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:58 smithi136 bash[76371]: cluster 2024-09-13T00:45:57.959185+0000 mon.a (mon.0) 4060 : cluster [DBG] osdmap e2312: 8 total, 8 up, 8 in 2024-09-13T00:45:59.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:58 smithi181 bash[67587]: cluster 2024-09-13T00:45:57.959185+0000 mon.a (mon.0) 4060 : cluster [DBG] osdmap e2312: 8 total, 8 up, 8 in 2024-09-13T00:45:59.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:45:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:45:59] "GET /metrics HTTP/1.1" 200 36746 "" "Prometheus/2.43.0" 2024-09-13T00:46:00.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:59 smithi136 bash[75164]: cluster 2024-09-13T00:45:58.589065+0000 mgr.y (mgr.24661) 2065 : cluster [DBG] pgmap v3938: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.2 MiB/s rd, 5.0 KiB/s wr, 1.67k op/s 2024-09-13T00:46:00.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:59 smithi136 bash[75164]: cluster 2024-09-13T00:45:58.985956+0000 mon.a (mon.0) 4061 : cluster [DBG] osdmap e2313: 8 total, 8 up, 8 in 2024-09-13T00:46:00.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:45:59 smithi136 bash[75164]: cluster 2024-09-13T00:45:59.889360+0000 mon.a (mon.0) 4062 : cluster [DBG] osdmap e2314: 8 total, 8 up, 8 in 2024-09-13T00:46:00.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:59 smithi136 bash[76371]: cluster 2024-09-13T00:45:58.589065+0000 mgr.y (mgr.24661) 2065 : cluster [DBG] pgmap v3938: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.2 MiB/s rd, 5.0 KiB/s wr, 1.67k op/s 2024-09-13T00:46:00.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:59 smithi136 bash[76371]: cluster 2024-09-13T00:45:58.985956+0000 mon.a (mon.0) 4061 : cluster [DBG] osdmap e2313: 8 total, 8 up, 8 in 2024-09-13T00:46:00.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:45:59 smithi136 bash[76371]: cluster 2024-09-13T00:45:59.889360+0000 mon.a (mon.0) 4062 : cluster [DBG] osdmap e2314: 8 total, 8 up, 8 in 2024-09-13T00:46:00.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:59 smithi181 bash[67587]: cluster 2024-09-13T00:45:58.589065+0000 mgr.y (mgr.24661) 2065 : cluster [DBG] pgmap v3938: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.2 MiB/s rd, 5.0 KiB/s wr, 1.67k op/s 2024-09-13T00:46:00.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:59 smithi181 bash[67587]: cluster 2024-09-13T00:45:58.985956+0000 mon.a (mon.0) 4061 : cluster [DBG] osdmap e2313: 8 total, 8 up, 8 in 2024-09-13T00:46:00.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:45:59 smithi181 bash[67587]: cluster 2024-09-13T00:45:59.889360+0000 mon.a (mon.0) 4062 : cluster [DBG] osdmap e2314: 8 total, 8 up, 8 in 2024-09-13T00:46:02.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:01 smithi136 bash[76371]: cluster 2024-09-13T00:46:00.589890+0000 mgr.y (mgr.24661) 2066 : cluster [DBG] pgmap v3941: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.2 MiB/s rd, 5.0 KiB/s wr, 1.67k op/s 2024-09-13T00:46:02.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:01 smithi136 bash[76371]: cluster 2024-09-13T00:46:01.001089+0000 mon.a (mon.0) 4063 : cluster [DBG] osdmap e2315: 8 total, 8 up, 8 in 2024-09-13T00:46:02.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:01 smithi136 bash[75164]: cluster 2024-09-13T00:46:00.589890+0000 mgr.y (mgr.24661) 2066 : cluster [DBG] pgmap v3941: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.2 MiB/s rd, 5.0 KiB/s wr, 1.67k op/s 2024-09-13T00:46:02.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:01 smithi136 bash[75164]: cluster 2024-09-13T00:46:01.001089+0000 mon.a (mon.0) 4063 : cluster [DBG] osdmap e2315: 8 total, 8 up, 8 in 2024-09-13T00:46:02.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:01 smithi181 bash[67587]: cluster 2024-09-13T00:46:00.589890+0000 mgr.y (mgr.24661) 2066 : cluster [DBG] pgmap v3941: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.2 MiB/s rd, 5.0 KiB/s wr, 1.67k op/s 2024-09-13T00:46:02.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:01 smithi181 bash[67587]: cluster 2024-09-13T00:46:01.001089+0000 mon.a (mon.0) 4063 : cluster [DBG] osdmap e2315: 8 total, 8 up, 8 in 2024-09-13T00:46:02.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:46:02 smithi181 bash[95505]: ts=2024-09-13T00:46:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:46:03.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:03 smithi181 bash[67587]: cluster 2024-09-13T00:46:02.009560+0000 mon.a (mon.0) 4064 : cluster [DBG] osdmap e2316: 8 total, 8 up, 8 in 2024-09-13T00:46:03.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:03 smithi136 bash[76371]: cluster 2024-09-13T00:46:02.009560+0000 mon.a (mon.0) 4064 : cluster [DBG] osdmap e2316: 8 total, 8 up, 8 in 2024-09-13T00:46:03.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:03 smithi136 bash[75164]: cluster 2024-09-13T00:46:02.009560+0000 mon.a (mon.0) 4064 : cluster [DBG] osdmap e2316: 8 total, 8 up, 8 in 2024-09-13T00:46:04.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:04 smithi181 bash[67587]: cluster 2024-09-13T00:46:02.590559+0000 mgr.y (mgr.24661) 2067 : cluster [DBG] pgmap v3944: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.2 MiB/s rd, 5.0 KiB/s wr, 1.63k op/s 2024-09-13T00:46:04.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:04 smithi181 bash[67587]: cluster 2024-09-13T00:46:03.018118+0000 mon.a (mon.0) 4065 : cluster [DBG] osdmap e2317: 8 total, 8 up, 8 in 2024-09-13T00:46:04.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:04 smithi136 bash[75164]: cluster 2024-09-13T00:46:02.590559+0000 mgr.y (mgr.24661) 2067 : cluster [DBG] pgmap v3944: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.2 MiB/s rd, 5.0 KiB/s wr, 1.63k op/s 2024-09-13T00:46:04.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:04 smithi136 bash[75164]: cluster 2024-09-13T00:46:03.018118+0000 mon.a (mon.0) 4065 : cluster [DBG] osdmap e2317: 8 total, 8 up, 8 in 2024-09-13T00:46:04.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:04 smithi136 bash[76371]: cluster 2024-09-13T00:46:02.590559+0000 mgr.y (mgr.24661) 2067 : cluster [DBG] pgmap v3944: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.2 MiB/s rd, 5.0 KiB/s wr, 1.63k op/s 2024-09-13T00:46:04.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:04 smithi136 bash[76371]: cluster 2024-09-13T00:46:03.018118+0000 mon.a (mon.0) 4065 : cluster [DBG] osdmap e2317: 8 total, 8 up, 8 in 2024-09-13T00:46:05.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:05 smithi181 bash[67587]: cluster 2024-09-13T00:46:04.028099+0000 mon.a (mon.0) 4066 : cluster [DBG] osdmap e2318: 8 total, 8 up, 8 in 2024-09-13T00:46:05.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:05 smithi181 bash[67587]: cluster 2024-09-13T00:46:04.890122+0000 mon.a (mon.0) 4067 : cluster [DBG] osdmap e2319: 8 total, 8 up, 8 in 2024-09-13T00:46:05.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:05 smithi136 bash[75164]: cluster 2024-09-13T00:46:04.028099+0000 mon.a (mon.0) 4066 : cluster [DBG] osdmap e2318: 8 total, 8 up, 8 in 2024-09-13T00:46:05.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:05 smithi136 bash[75164]: cluster 2024-09-13T00:46:04.890122+0000 mon.a (mon.0) 4067 : cluster [DBG] osdmap e2319: 8 total, 8 up, 8 in 2024-09-13T00:46:05.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:05 smithi136 bash[76371]: cluster 2024-09-13T00:46:04.028099+0000 mon.a (mon.0) 4066 : cluster [DBG] osdmap e2318: 8 total, 8 up, 8 in 2024-09-13T00:46:05.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:05 smithi136 bash[76371]: cluster 2024-09-13T00:46:04.890122+0000 mon.a (mon.0) 4067 : cluster [DBG] osdmap e2319: 8 total, 8 up, 8 in 2024-09-13T00:46:06.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:06 smithi181 bash[67587]: cluster 2024-09-13T00:46:04.591252+0000 mgr.y (mgr.24661) 2068 : cluster [DBG] pgmap v3947: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.2 MiB/s rd, 5.0 KiB/s wr, 1.63k op/s 2024-09-13T00:46:06.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:06 smithi181 bash[67587]: cluster 2024-09-13T00:46:05.898009+0000 mon.a (mon.0) 4068 : cluster [DBG] osdmap e2320: 8 total, 8 up, 8 in 2024-09-13T00:46:06.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:06 smithi136 bash[75164]: cluster 2024-09-13T00:46:04.591252+0000 mgr.y (mgr.24661) 2068 : cluster [DBG] pgmap v3947: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.2 MiB/s rd, 5.0 KiB/s wr, 1.63k op/s 2024-09-13T00:46:06.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:06 smithi136 bash[75164]: cluster 2024-09-13T00:46:05.898009+0000 mon.a (mon.0) 4068 : cluster [DBG] osdmap e2320: 8 total, 8 up, 8 in 2024-09-13T00:46:06.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:06 smithi136 bash[76371]: cluster 2024-09-13T00:46:04.591252+0000 mgr.y (mgr.24661) 2068 : cluster [DBG] pgmap v3947: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.2 MiB/s rd, 5.0 KiB/s wr, 1.63k op/s 2024-09-13T00:46:06.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:06 smithi136 bash[76371]: cluster 2024-09-13T00:46:05.898009+0000 mon.a (mon.0) 4068 : cluster [DBG] osdmap e2320: 8 total, 8 up, 8 in 2024-09-13T00:46:08.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:07 smithi136 bash[75164]: cluster 2024-09-13T00:46:06.592286+0000 mgr.y (mgr.24661) 2069 : cluster [DBG] pgmap v3950: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 2.5 KiB/s rd, 0 B/s wr, 4 op/s 2024-09-13T00:46:08.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:07 smithi136 bash[75164]: cluster 2024-09-13T00:46:06.896756+0000 mon.a (mon.0) 4069 : cluster [DBG] osdmap e2321: 8 total, 8 up, 8 in 2024-09-13T00:46:08.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:07 smithi136 bash[76371]: cluster 2024-09-13T00:46:06.592286+0000 mgr.y (mgr.24661) 2069 : cluster [DBG] pgmap v3950: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 2.5 KiB/s rd, 0 B/s wr, 4 op/s 2024-09-13T00:46:08.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:07 smithi136 bash[76371]: cluster 2024-09-13T00:46:06.896756+0000 mon.a (mon.0) 4069 : cluster [DBG] osdmap e2321: 8 total, 8 up, 8 in 2024-09-13T00:46:08.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:07 smithi181 bash[67587]: cluster 2024-09-13T00:46:06.592286+0000 mgr.y (mgr.24661) 2069 : cluster [DBG] pgmap v3950: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 2.5 KiB/s rd, 0 B/s wr, 4 op/s 2024-09-13T00:46:08.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:07 smithi181 bash[67587]: cluster 2024-09-13T00:46:06.896756+0000 mon.a (mon.0) 4069 : cluster [DBG] osdmap e2321: 8 total, 8 up, 8 in 2024-09-13T00:46:09.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:08 smithi136 bash[75164]: cluster 2024-09-13T00:46:07.907315+0000 mon.a (mon.0) 4070 : cluster [DBG] osdmap e2322: 8 total, 8 up, 8 in 2024-09-13T00:46:09.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:08 smithi136 bash[75164]: audit 2024-09-13T00:46:08.848106+0000 mon.a (mon.0) 4071 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:46:09.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:08 smithi136 bash[76371]: cluster 2024-09-13T00:46:07.907315+0000 mon.a (mon.0) 4070 : cluster [DBG] osdmap e2322: 8 total, 8 up, 8 in 2024-09-13T00:46:09.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:08 smithi136 bash[76371]: audit 2024-09-13T00:46:08.848106+0000 mon.a (mon.0) 4071 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:46:09.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:08 smithi181 bash[67587]: cluster 2024-09-13T00:46:07.907315+0000 mon.a (mon.0) 4070 : cluster [DBG] osdmap e2322: 8 total, 8 up, 8 in 2024-09-13T00:46:09.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:08 smithi181 bash[67587]: audit 2024-09-13T00:46:08.848106+0000 mon.a (mon.0) 4071 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:46:09.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:46:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:46:09] "GET /metrics HTTP/1.1" 200 36746 "" "Prometheus/2.43.0" 2024-09-13T00:46:10.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:09 smithi136 bash[76371]: cluster 2024-09-13T00:46:08.593160+0000 mgr.y (mgr.24661) 2070 : cluster [DBG] pgmap v3953: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.1 MiB/s rd, 5.0 KiB/s wr, 1.58k op/s 2024-09-13T00:46:10.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:09 smithi136 bash[76371]: cluster 2024-09-13T00:46:08.922731+0000 mon.a (mon.0) 4072 : cluster [DBG] osdmap e2323: 8 total, 8 up, 8 in 2024-09-13T00:46:10.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:09 smithi136 bash[76371]: cluster 2024-09-13T00:46:09.901733+0000 mon.a (mon.0) 4073 : cluster [DBG] osdmap e2324: 8 total, 8 up, 8 in 2024-09-13T00:46:10.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:09 smithi136 bash[75164]: cluster 2024-09-13T00:46:08.593160+0000 mgr.y (mgr.24661) 2070 : cluster [DBG] pgmap v3953: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.1 MiB/s rd, 5.0 KiB/s wr, 1.58k op/s 2024-09-13T00:46:10.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:09 smithi136 bash[75164]: cluster 2024-09-13T00:46:08.922731+0000 mon.a (mon.0) 4072 : cluster [DBG] osdmap e2323: 8 total, 8 up, 8 in 2024-09-13T00:46:10.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:09 smithi136 bash[75164]: cluster 2024-09-13T00:46:09.901733+0000 mon.a (mon.0) 4073 : cluster [DBG] osdmap e2324: 8 total, 8 up, 8 in 2024-09-13T00:46:10.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:09 smithi181 bash[67587]: cluster 2024-09-13T00:46:08.593160+0000 mgr.y (mgr.24661) 2070 : cluster [DBG] pgmap v3953: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.1 MiB/s rd, 5.0 KiB/s wr, 1.58k op/s 2024-09-13T00:46:10.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:09 smithi181 bash[67587]: cluster 2024-09-13T00:46:08.922731+0000 mon.a (mon.0) 4072 : cluster [DBG] osdmap e2323: 8 total, 8 up, 8 in 2024-09-13T00:46:10.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:09 smithi181 bash[67587]: cluster 2024-09-13T00:46:09.901733+0000 mon.a (mon.0) 4073 : cluster [DBG] osdmap e2324: 8 total, 8 up, 8 in 2024-09-13T00:46:12.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:11 smithi136 bash[75164]: cluster 2024-09-13T00:46:10.594014+0000 mgr.y (mgr.24661) 2071 : cluster [DBG] pgmap v3956: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.1 MiB/s rd, 5.0 KiB/s wr, 1.58k op/s 2024-09-13T00:46:12.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:11 smithi136 bash[75164]: cluster 2024-09-13T00:46:10.929869+0000 mon.a (mon.0) 4074 : cluster [DBG] osdmap e2325: 8 total, 8 up, 8 in 2024-09-13T00:46:12.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:11 smithi136 bash[76371]: cluster 2024-09-13T00:46:10.594014+0000 mgr.y (mgr.24661) 2071 : cluster [DBG] pgmap v3956: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.1 MiB/s rd, 5.0 KiB/s wr, 1.58k op/s 2024-09-13T00:46:12.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:11 smithi136 bash[76371]: cluster 2024-09-13T00:46:10.929869+0000 mon.a (mon.0) 4074 : cluster [DBG] osdmap e2325: 8 total, 8 up, 8 in 2024-09-13T00:46:12.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:11 smithi181 bash[67587]: cluster 2024-09-13T00:46:10.594014+0000 mgr.y (mgr.24661) 2071 : cluster [DBG] pgmap v3956: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.1 MiB/s rd, 5.0 KiB/s wr, 1.58k op/s 2024-09-13T00:46:12.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:11 smithi181 bash[67587]: cluster 2024-09-13T00:46:10.929869+0000 mon.a (mon.0) 4074 : cluster [DBG] osdmap e2325: 8 total, 8 up, 8 in 2024-09-13T00:46:12.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:46:12 smithi181 bash[95505]: ts=2024-09-13T00:46:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:46:13.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:12 smithi136 bash[75164]: cluster 2024-09-13T00:46:11.937792+0000 mon.a (mon.0) 4075 : cluster [DBG] osdmap e2326: 8 total, 8 up, 8 in 2024-09-13T00:46:13.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:12 smithi136 bash[76371]: cluster 2024-09-13T00:46:11.937792+0000 mon.a (mon.0) 4075 : cluster [DBG] osdmap e2326: 8 total, 8 up, 8 in 2024-09-13T00:46:13.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:12 smithi181 bash[67587]: cluster 2024-09-13T00:46:11.937792+0000 mon.a (mon.0) 4075 : cluster [DBG] osdmap e2326: 8 total, 8 up, 8 in 2024-09-13T00:46:14.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:13 smithi136 bash[75164]: cluster 2024-09-13T00:46:12.594788+0000 mgr.y (mgr.24661) 2072 : cluster [DBG] pgmap v3959: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.1 MiB/s rd, 5.0 KiB/s wr, 1.53k op/s 2024-09-13T00:46:14.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:13 smithi136 bash[75164]: cluster 2024-09-13T00:46:12.960543+0000 mon.a (mon.0) 4076 : cluster [DBG] osdmap e2327: 8 total, 8 up, 8 in 2024-09-13T00:46:14.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:13 smithi136 bash[76371]: cluster 2024-09-13T00:46:12.594788+0000 mgr.y (mgr.24661) 2072 : cluster [DBG] pgmap v3959: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.1 MiB/s rd, 5.0 KiB/s wr, 1.53k op/s 2024-09-13T00:46:14.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:13 smithi136 bash[76371]: cluster 2024-09-13T00:46:12.960543+0000 mon.a (mon.0) 4076 : cluster [DBG] osdmap e2327: 8 total, 8 up, 8 in 2024-09-13T00:46:14.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:13 smithi181 bash[67587]: cluster 2024-09-13T00:46:12.594788+0000 mgr.y (mgr.24661) 2072 : cluster [DBG] pgmap v3959: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.1 MiB/s rd, 5.0 KiB/s wr, 1.53k op/s 2024-09-13T00:46:14.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:13 smithi181 bash[67587]: cluster 2024-09-13T00:46:12.960543+0000 mon.a (mon.0) 4076 : cluster [DBG] osdmap e2327: 8 total, 8 up, 8 in 2024-09-13T00:46:15.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:14 smithi136 bash[75164]: cluster 2024-09-13T00:46:13.970832+0000 mon.a (mon.0) 4077 : cluster [DBG] osdmap e2328: 8 total, 8 up, 8 in 2024-09-13T00:46:15.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:14 smithi136 bash[75164]: cluster 2024-09-13T00:46:14.898053+0000 mon.a (mon.0) 4078 : cluster [DBG] osdmap e2329: 8 total, 8 up, 8 in 2024-09-13T00:46:15.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:14 smithi136 bash[76371]: cluster 2024-09-13T00:46:13.970832+0000 mon.a (mon.0) 4077 : cluster [DBG] osdmap e2328: 8 total, 8 up, 8 in 2024-09-13T00:46:15.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:14 smithi136 bash[76371]: cluster 2024-09-13T00:46:14.898053+0000 mon.a (mon.0) 4078 : cluster [DBG] osdmap e2329: 8 total, 8 up, 8 in 2024-09-13T00:46:15.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:14 smithi181 bash[67587]: cluster 2024-09-13T00:46:13.970832+0000 mon.a (mon.0) 4077 : cluster [DBG] osdmap e2328: 8 total, 8 up, 8 in 2024-09-13T00:46:15.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:14 smithi181 bash[67587]: cluster 2024-09-13T00:46:14.898053+0000 mon.a (mon.0) 4078 : cluster [DBG] osdmap e2329: 8 total, 8 up, 8 in 2024-09-13T00:46:16.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:15 smithi136 bash[75164]: cluster 2024-09-13T00:46:14.595611+0000 mgr.y (mgr.24661) 2073 : cluster [DBG] pgmap v3962: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.1 MiB/s rd, 5.0 KiB/s wr, 1.54k op/s 2024-09-13T00:46:16.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:15 smithi136 bash[76371]: cluster 2024-09-13T00:46:14.595611+0000 mgr.y (mgr.24661) 2073 : cluster [DBG] pgmap v3962: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.1 MiB/s rd, 5.0 KiB/s wr, 1.54k op/s 2024-09-13T00:46:16.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:15 smithi181 bash[67587]: cluster 2024-09-13T00:46:14.595611+0000 mgr.y (mgr.24661) 2073 : cluster [DBG] pgmap v3962: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.1 MiB/s rd, 5.0 KiB/s wr, 1.54k op/s 2024-09-13T00:46:17.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:16 smithi136 bash[75164]: cluster 2024-09-13T00:46:15.994828+0000 mon.a (mon.0) 4079 : cluster [DBG] osdmap e2330: 8 total, 8 up, 8 in 2024-09-13T00:46:17.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:17 smithi136 bash[76371]: cluster 2024-09-13T00:46:15.994828+0000 mon.a (mon.0) 4079 : cluster [DBG] osdmap e2330: 8 total, 8 up, 8 in 2024-09-13T00:46:17.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:17 smithi181 bash[67587]: cluster 2024-09-13T00:46:15.994828+0000 mon.a (mon.0) 4079 : cluster [DBG] osdmap e2330: 8 total, 8 up, 8 in 2024-09-13T00:46:18.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:18 smithi136 bash[75164]: cluster 2024-09-13T00:46:16.596547+0000 mgr.y (mgr.24661) 2074 : cluster [DBG] pgmap v3965: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 4.2 KiB/s rd, 0 B/s wr, 8 op/s 2024-09-13T00:46:18.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:18 smithi136 bash[75164]: cluster 2024-09-13T00:46:17.005685+0000 mon.a (mon.0) 4080 : cluster [DBG] osdmap e2331: 8 total, 8 up, 8 in 2024-09-13T00:46:18.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:18 smithi136 bash[76371]: cluster 2024-09-13T00:46:16.596547+0000 mgr.y (mgr.24661) 2074 : cluster [DBG] pgmap v3965: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 4.2 KiB/s rd, 0 B/s wr, 8 op/s 2024-09-13T00:46:18.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:18 smithi136 bash[76371]: cluster 2024-09-13T00:46:17.005685+0000 mon.a (mon.0) 4080 : cluster [DBG] osdmap e2331: 8 total, 8 up, 8 in 2024-09-13T00:46:18.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:18 smithi181 bash[67587]: cluster 2024-09-13T00:46:16.596547+0000 mgr.y (mgr.24661) 2074 : cluster [DBG] pgmap v3965: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 4.2 KiB/s rd, 0 B/s wr, 8 op/s 2024-09-13T00:46:18.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:18 smithi181 bash[67587]: cluster 2024-09-13T00:46:17.005685+0000 mon.a (mon.0) 4080 : cluster [DBG] osdmap e2331: 8 total, 8 up, 8 in 2024-09-13T00:46:19.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:19 smithi181 bash[67587]: cluster 2024-09-13T00:46:18.013311+0000 mon.a (mon.0) 4081 : cluster [DBG] osdmap e2332: 8 total, 8 up, 8 in 2024-09-13T00:46:19.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:19 smithi136 bash[75164]: cluster 2024-09-13T00:46:18.013311+0000 mon.a (mon.0) 4081 : cluster [DBG] osdmap e2332: 8 total, 8 up, 8 in 2024-09-13T00:46:19.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:19 smithi136 bash[76371]: cluster 2024-09-13T00:46:18.013311+0000 mon.a (mon.0) 4081 : cluster [DBG] osdmap e2332: 8 total, 8 up, 8 in 2024-09-13T00:46:19.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:46:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:46:19] "GET /metrics HTTP/1.1" 200 36746 "" "Prometheus/2.43.0" 2024-09-13T00:46:20.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:20 smithi181 bash[67587]: cluster 2024-09-13T00:46:18.597424+0000 mgr.y (mgr.24661) 2075 : cluster [DBG] pgmap v3968: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.1 MiB/s rd, 5.0 KiB/s wr, 1.50k op/s 2024-09-13T00:46:20.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:20 smithi181 bash[67587]: cluster 2024-09-13T00:46:19.023729+0000 mon.a (mon.0) 4082 : cluster [DBG] osdmap e2333: 8 total, 8 up, 8 in 2024-09-13T00:46:20.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:20 smithi181 bash[67587]: cluster 2024-09-13T00:46:19.899885+0000 mon.a (mon.0) 4083 : cluster [DBG] osdmap e2334: 8 total, 8 up, 8 in 2024-09-13T00:46:20.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:20 smithi136 bash[75164]: cluster 2024-09-13T00:46:18.597424+0000 mgr.y (mgr.24661) 2075 : cluster [DBG] pgmap v3968: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.1 MiB/s rd, 5.0 KiB/s wr, 1.50k op/s 2024-09-13T00:46:20.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:20 smithi136 bash[75164]: cluster 2024-09-13T00:46:19.023729+0000 mon.a (mon.0) 4082 : cluster [DBG] osdmap e2333: 8 total, 8 up, 8 in 2024-09-13T00:46:20.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:20 smithi136 bash[75164]: cluster 2024-09-13T00:46:19.899885+0000 mon.a (mon.0) 4083 : cluster [DBG] osdmap e2334: 8 total, 8 up, 8 in 2024-09-13T00:46:20.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:20 smithi136 bash[76371]: cluster 2024-09-13T00:46:18.597424+0000 mgr.y (mgr.24661) 2075 : cluster [DBG] pgmap v3968: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.1 MiB/s rd, 5.0 KiB/s wr, 1.50k op/s 2024-09-13T00:46:20.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:20 smithi136 bash[76371]: cluster 2024-09-13T00:46:19.023729+0000 mon.a (mon.0) 4082 : cluster [DBG] osdmap e2333: 8 total, 8 up, 8 in 2024-09-13T00:46:20.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:20 smithi136 bash[76371]: cluster 2024-09-13T00:46:19.899885+0000 mon.a (mon.0) 4083 : cluster [DBG] osdmap e2334: 8 total, 8 up, 8 in 2024-09-13T00:46:22.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:21 smithi136 bash[76371]: cluster 2024-09-13T00:46:20.598227+0000 mgr.y (mgr.24661) 2076 : cluster [DBG] pgmap v3971: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.1 MiB/s rd, 5.0 KiB/s wr, 1.50k op/s 2024-09-13T00:46:22.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:21 smithi136 bash[76371]: cluster 2024-09-13T00:46:20.903985+0000 mon.a (mon.0) 4084 : cluster [DBG] osdmap e2335: 8 total, 8 up, 8 in 2024-09-13T00:46:22.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:21 smithi136 bash[75164]: cluster 2024-09-13T00:46:20.598227+0000 mgr.y (mgr.24661) 2076 : cluster [DBG] pgmap v3971: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.1 MiB/s rd, 5.0 KiB/s wr, 1.50k op/s 2024-09-13T00:46:22.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:21 smithi136 bash[75164]: cluster 2024-09-13T00:46:20.903985+0000 mon.a (mon.0) 4084 : cluster [DBG] osdmap e2335: 8 total, 8 up, 8 in 2024-09-13T00:46:22.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:21 smithi181 bash[67587]: cluster 2024-09-13T00:46:20.598227+0000 mgr.y (mgr.24661) 2076 : cluster [DBG] pgmap v3971: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.1 MiB/s rd, 5.0 KiB/s wr, 1.50k op/s 2024-09-13T00:46:22.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:21 smithi181 bash[67587]: cluster 2024-09-13T00:46:20.903985+0000 mon.a (mon.0) 4084 : cluster [DBG] osdmap e2335: 8 total, 8 up, 8 in 2024-09-13T00:46:22.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:46:22 smithi181 bash[95505]: ts=2024-09-13T00:46:22.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:46:23.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:22 smithi136 bash[75164]: cluster 2024-09-13T00:46:21.906239+0000 mon.a (mon.0) 4085 : cluster [DBG] osdmap e2336: 8 total, 8 up, 8 in 2024-09-13T00:46:23.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:22 smithi136 bash[76371]: cluster 2024-09-13T00:46:21.906239+0000 mon.a (mon.0) 4085 : cluster [DBG] osdmap e2336: 8 total, 8 up, 8 in 2024-09-13T00:46:23.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:22 smithi181 bash[67587]: cluster 2024-09-13T00:46:21.906239+0000 mon.a (mon.0) 4085 : cluster [DBG] osdmap e2336: 8 total, 8 up, 8 in 2024-09-13T00:46:24.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:23 smithi136 bash[75164]: cluster 2024-09-13T00:46:22.598914+0000 mgr.y (mgr.24661) 2077 : cluster [DBG] pgmap v3974: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.0 MiB/s rd, 5.0 KiB/s wr, 1.45k op/s 2024-09-13T00:46:24.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:23 smithi136 bash[75164]: cluster 2024-09-13T00:46:22.930927+0000 mon.a (mon.0) 4086 : cluster [DBG] osdmap e2337: 8 total, 8 up, 8 in 2024-09-13T00:46:24.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:23 smithi136 bash[75164]: audit 2024-09-13T00:46:23.848572+0000 mon.a (mon.0) 4087 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:46:24.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:23 smithi136 bash[76371]: cluster 2024-09-13T00:46:22.598914+0000 mgr.y (mgr.24661) 2077 : cluster [DBG] pgmap v3974: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.0 MiB/s rd, 5.0 KiB/s wr, 1.45k op/s 2024-09-13T00:46:24.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:23 smithi136 bash[76371]: cluster 2024-09-13T00:46:22.930927+0000 mon.a (mon.0) 4086 : cluster [DBG] osdmap e2337: 8 total, 8 up, 8 in 2024-09-13T00:46:24.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:23 smithi136 bash[76371]: audit 2024-09-13T00:46:23.848572+0000 mon.a (mon.0) 4087 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:46:24.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:23 smithi181 bash[67587]: cluster 2024-09-13T00:46:22.598914+0000 mgr.y (mgr.24661) 2077 : cluster [DBG] pgmap v3974: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.0 MiB/s rd, 5.0 KiB/s wr, 1.45k op/s 2024-09-13T00:46:24.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:23 smithi181 bash[67587]: cluster 2024-09-13T00:46:22.930927+0000 mon.a (mon.0) 4086 : cluster [DBG] osdmap e2337: 8 total, 8 up, 8 in 2024-09-13T00:46:24.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:23 smithi181 bash[67587]: audit 2024-09-13T00:46:23.848572+0000 mon.a (mon.0) 4087 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:46:25.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:24 smithi136 bash[76371]: cluster 2024-09-13T00:46:23.932823+0000 mon.a (mon.0) 4088 : cluster [DBG] osdmap e2338: 8 total, 8 up, 8 in 2024-09-13T00:46:25.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:24 smithi136 bash[76371]: cluster 2024-09-13T00:46:24.902587+0000 mon.a (mon.0) 4089 : cluster [DBG] osdmap e2339: 8 total, 8 up, 8 in 2024-09-13T00:46:25.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:24 smithi136 bash[75164]: cluster 2024-09-13T00:46:23.932823+0000 mon.a (mon.0) 4088 : cluster [DBG] osdmap e2338: 8 total, 8 up, 8 in 2024-09-13T00:46:25.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:24 smithi136 bash[75164]: cluster 2024-09-13T00:46:24.902587+0000 mon.a (mon.0) 4089 : cluster [DBG] osdmap e2339: 8 total, 8 up, 8 in 2024-09-13T00:46:25.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:24 smithi181 bash[67587]: cluster 2024-09-13T00:46:23.932823+0000 mon.a (mon.0) 4088 : cluster [DBG] osdmap e2338: 8 total, 8 up, 8 in 2024-09-13T00:46:25.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:24 smithi181 bash[67587]: cluster 2024-09-13T00:46:24.902587+0000 mon.a (mon.0) 4089 : cluster [DBG] osdmap e2339: 8 total, 8 up, 8 in 2024-09-13T00:46:26.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:25 smithi136 bash[75164]: cluster 2024-09-13T00:46:24.599606+0000 mgr.y (mgr.24661) 2078 : cluster [DBG] pgmap v3977: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.0 MiB/s rd, 5.0 KiB/s wr, 1.45k op/s 2024-09-13T00:46:26.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:25 smithi136 bash[76371]: cluster 2024-09-13T00:46:24.599606+0000 mgr.y (mgr.24661) 2078 : cluster [DBG] pgmap v3977: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.0 MiB/s rd, 5.0 KiB/s wr, 1.45k op/s 2024-09-13T00:46:26.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:25 smithi181 bash[67587]: cluster 2024-09-13T00:46:24.599606+0000 mgr.y (mgr.24661) 2078 : cluster [DBG] pgmap v3977: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.0 MiB/s rd, 5.0 KiB/s wr, 1.45k op/s 2024-09-13T00:46:27.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:26 smithi136 bash[75164]: cluster 2024-09-13T00:46:25.965938+0000 mon.a (mon.0) 4090 : cluster [DBG] osdmap e2340: 8 total, 8 up, 8 in 2024-09-13T00:46:27.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:26 smithi136 bash[76371]: cluster 2024-09-13T00:46:25.965938+0000 mon.a (mon.0) 4090 : cluster [DBG] osdmap e2340: 8 total, 8 up, 8 in 2024-09-13T00:46:27.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:26 smithi181 bash[67587]: cluster 2024-09-13T00:46:25.965938+0000 mon.a (mon.0) 4090 : cluster [DBG] osdmap e2340: 8 total, 8 up, 8 in 2024-09-13T00:46:28.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:27 smithi136 bash[75164]: cluster 2024-09-13T00:46:26.600468+0000 mgr.y (mgr.24661) 2079 : cluster [DBG] pgmap v3980: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 3.7 KiB/s rd, 0 B/s wr, 7 op/s 2024-09-13T00:46:28.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:27 smithi136 bash[75164]: cluster 2024-09-13T00:46:26.992593+0000 mon.a (mon.0) 4091 : cluster [DBG] osdmap e2341: 8 total, 8 up, 8 in 2024-09-13T00:46:28.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:27 smithi136 bash[76371]: cluster 2024-09-13T00:46:26.600468+0000 mgr.y (mgr.24661) 2079 : cluster [DBG] pgmap v3980: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 3.7 KiB/s rd, 0 B/s wr, 7 op/s 2024-09-13T00:46:28.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:27 smithi136 bash[76371]: cluster 2024-09-13T00:46:26.992593+0000 mon.a (mon.0) 4091 : cluster [DBG] osdmap e2341: 8 total, 8 up, 8 in 2024-09-13T00:46:28.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:27 smithi181 bash[67587]: cluster 2024-09-13T00:46:26.600468+0000 mgr.y (mgr.24661) 2079 : cluster [DBG] pgmap v3980: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 3.7 KiB/s rd, 0 B/s wr, 7 op/s 2024-09-13T00:46:28.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:27 smithi181 bash[67587]: cluster 2024-09-13T00:46:26.992593+0000 mon.a (mon.0) 4091 : cluster [DBG] osdmap e2341: 8 total, 8 up, 8 in 2024-09-13T00:46:29.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:29 smithi181 bash[67587]: cluster 2024-09-13T00:46:28.006842+0000 mon.a (mon.0) 4092 : cluster [DBG] osdmap e2342: 8 total, 8 up, 8 in 2024-09-13T00:46:29.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:29 smithi136 bash[75164]: cluster 2024-09-13T00:46:28.006842+0000 mon.a (mon.0) 4092 : cluster [DBG] osdmap e2342: 8 total, 8 up, 8 in 2024-09-13T00:46:29.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:29 smithi136 bash[76371]: cluster 2024-09-13T00:46:28.006842+0000 mon.a (mon.0) 4092 : cluster [DBG] osdmap e2342: 8 total, 8 up, 8 in 2024-09-13T00:46:29.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:46:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:46:29] "GET /metrics HTTP/1.1" 200 36747 "" "Prometheus/2.43.0" 2024-09-13T00:46:30.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:30 smithi136 bash[75164]: cluster 2024-09-13T00:46:28.601288+0000 mgr.y (mgr.24661) 2080 : cluster [DBG] pgmap v3983: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.0 MiB/s rd, 5.0 KiB/s wr, 1.41k op/s 2024-09-13T00:46:30.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:30 smithi136 bash[75164]: cluster 2024-09-13T00:46:29.020521+0000 mon.a (mon.0) 4093 : cluster [DBG] osdmap e2343: 8 total, 8 up, 8 in 2024-09-13T00:46:30.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:30 smithi136 bash[75164]: cluster 2024-09-13T00:46:29.905274+0000 mon.a (mon.0) 4094 : cluster [DBG] osdmap e2344: 8 total, 8 up, 8 in 2024-09-13T00:46:30.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:30 smithi136 bash[76371]: cluster 2024-09-13T00:46:28.601288+0000 mgr.y (mgr.24661) 2080 : cluster [DBG] pgmap v3983: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.0 MiB/s rd, 5.0 KiB/s wr, 1.41k op/s 2024-09-13T00:46:30.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:30 smithi136 bash[76371]: cluster 2024-09-13T00:46:29.020521+0000 mon.a (mon.0) 4093 : cluster [DBG] osdmap e2343: 8 total, 8 up, 8 in 2024-09-13T00:46:30.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:30 smithi136 bash[76371]: cluster 2024-09-13T00:46:29.905274+0000 mon.a (mon.0) 4094 : cluster [DBG] osdmap e2344: 8 total, 8 up, 8 in 2024-09-13T00:46:30.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:30 smithi181 bash[67587]: cluster 2024-09-13T00:46:28.601288+0000 mgr.y (mgr.24661) 2080 : cluster [DBG] pgmap v3983: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.0 MiB/s rd, 5.0 KiB/s wr, 1.41k op/s 2024-09-13T00:46:30.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:30 smithi181 bash[67587]: cluster 2024-09-13T00:46:29.020521+0000 mon.a (mon.0) 4093 : cluster [DBG] osdmap e2343: 8 total, 8 up, 8 in 2024-09-13T00:46:30.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:30 smithi181 bash[67587]: cluster 2024-09-13T00:46:29.905274+0000 mon.a (mon.0) 4094 : cluster [DBG] osdmap e2344: 8 total, 8 up, 8 in 2024-09-13T00:46:32.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:32 smithi181 bash[67587]: cluster 2024-09-13T00:46:30.602125+0000 mgr.y (mgr.24661) 2081 : cluster [DBG] pgmap v3986: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.0 MiB/s rd, 5.0 KiB/s wr, 1.41k op/s 2024-09-13T00:46:32.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:32 smithi181 bash[67587]: cluster 2024-09-13T00:46:31.022486+0000 mon.a (mon.0) 4095 : cluster [DBG] osdmap e2345: 8 total, 8 up, 8 in 2024-09-13T00:46:32.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:32 smithi136 bash[75164]: cluster 2024-09-13T00:46:30.602125+0000 mgr.y (mgr.24661) 2081 : cluster [DBG] pgmap v3986: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.0 MiB/s rd, 5.0 KiB/s wr, 1.41k op/s 2024-09-13T00:46:32.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:32 smithi136 bash[75164]: cluster 2024-09-13T00:46:31.022486+0000 mon.a (mon.0) 4095 : cluster [DBG] osdmap e2345: 8 total, 8 up, 8 in 2024-09-13T00:46:32.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:32 smithi136 bash[76371]: cluster 2024-09-13T00:46:30.602125+0000 mgr.y (mgr.24661) 2081 : cluster [DBG] pgmap v3986: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.0 MiB/s rd, 5.0 KiB/s wr, 1.41k op/s 2024-09-13T00:46:32.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:32 smithi136 bash[76371]: cluster 2024-09-13T00:46:31.022486+0000 mon.a (mon.0) 4095 : cluster [DBG] osdmap e2345: 8 total, 8 up, 8 in 2024-09-13T00:46:32.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:46:32 smithi181 bash[95505]: ts=2024-09-13T00:46:32.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:46:33.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:33 smithi181 bash[67587]: cluster 2024-09-13T00:46:32.033275+0000 mon.a (mon.0) 4096 : cluster [DBG] osdmap e2346: 8 total, 8 up, 8 in 2024-09-13T00:46:33.513 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:33 smithi136 bash[75164]: cluster 2024-09-13T00:46:32.033275+0000 mon.a (mon.0) 4096 : cluster [DBG] osdmap e2346: 8 total, 8 up, 8 in 2024-09-13T00:46:33.513 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:33 smithi136 bash[76371]: cluster 2024-09-13T00:46:32.033275+0000 mon.a (mon.0) 4096 : cluster [DBG] osdmap e2346: 8 total, 8 up, 8 in 2024-09-13T00:46:34.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:34 smithi181 bash[67587]: cluster 2024-09-13T00:46:32.602849+0000 mgr.y (mgr.24661) 2082 : cluster [DBG] pgmap v3989: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1008 KiB/s rd, 5.0 KiB/s wr, 1.36k op/s 2024-09-13T00:46:34.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:34 smithi181 bash[67587]: cluster 2024-09-13T00:46:33.044459+0000 mon.a (mon.0) 4097 : cluster [DBG] osdmap e2347: 8 total, 8 up, 8 in 2024-09-13T00:46:34.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:34 smithi136 bash[75164]: cluster 2024-09-13T00:46:32.602849+0000 mgr.y (mgr.24661) 2082 : cluster [DBG] pgmap v3989: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1008 KiB/s rd, 5.0 KiB/s wr, 1.36k op/s 2024-09-13T00:46:34.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:34 smithi136 bash[75164]: cluster 2024-09-13T00:46:33.044459+0000 mon.a (mon.0) 4097 : cluster [DBG] osdmap e2347: 8 total, 8 up, 8 in 2024-09-13T00:46:34.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:34 smithi136 bash[76371]: cluster 2024-09-13T00:46:32.602849+0000 mgr.y (mgr.24661) 2082 : cluster [DBG] pgmap v3989: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1008 KiB/s rd, 5.0 KiB/s wr, 1.36k op/s 2024-09-13T00:46:34.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:34 smithi136 bash[76371]: cluster 2024-09-13T00:46:33.044459+0000 mon.a (mon.0) 4097 : cluster [DBG] osdmap e2347: 8 total, 8 up, 8 in 2024-09-13T00:46:35.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:35 smithi181 bash[67587]: cluster 2024-09-13T00:46:34.046205+0000 mon.a (mon.0) 4098 : cluster [DBG] osdmap e2348: 8 total, 8 up, 8 in 2024-09-13T00:46:35.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:35 smithi181 bash[67587]: cluster 2024-09-13T00:46:34.906381+0000 mon.a (mon.0) 4099 : cluster [DBG] osdmap e2349: 8 total, 8 up, 8 in 2024-09-13T00:46:35.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:35 smithi136 bash[75164]: cluster 2024-09-13T00:46:34.046205+0000 mon.a (mon.0) 4098 : cluster [DBG] osdmap e2348: 8 total, 8 up, 8 in 2024-09-13T00:46:35.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:35 smithi136 bash[75164]: cluster 2024-09-13T00:46:34.906381+0000 mon.a (mon.0) 4099 : cluster [DBG] osdmap e2349: 8 total, 8 up, 8 in 2024-09-13T00:46:35.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:35 smithi136 bash[76371]: cluster 2024-09-13T00:46:34.046205+0000 mon.a (mon.0) 4098 : cluster [DBG] osdmap e2348: 8 total, 8 up, 8 in 2024-09-13T00:46:35.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:35 smithi136 bash[76371]: cluster 2024-09-13T00:46:34.906381+0000 mon.a (mon.0) 4099 : cluster [DBG] osdmap e2349: 8 total, 8 up, 8 in 2024-09-13T00:46:36.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:36 smithi181 bash[67587]: cluster 2024-09-13T00:46:34.603568+0000 mgr.y (mgr.24661) 2083 : cluster [DBG] pgmap v3992: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1009 KiB/s rd, 5.0 KiB/s wr, 1.36k op/s 2024-09-13T00:46:36.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:36 smithi181 bash[67587]: cluster 2024-09-13T00:46:35.908006+0000 mon.a (mon.0) 4100 : cluster [DBG] osdmap e2350: 8 total, 8 up, 8 in 2024-09-13T00:46:36.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:36 smithi136 bash[75164]: cluster 2024-09-13T00:46:34.603568+0000 mgr.y (mgr.24661) 2083 : cluster [DBG] pgmap v3992: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1009 KiB/s rd, 5.0 KiB/s wr, 1.36k op/s 2024-09-13T00:46:36.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:36 smithi136 bash[75164]: cluster 2024-09-13T00:46:35.908006+0000 mon.a (mon.0) 4100 : cluster [DBG] osdmap e2350: 8 total, 8 up, 8 in 2024-09-13T00:46:36.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:36 smithi136 bash[76371]: cluster 2024-09-13T00:46:34.603568+0000 mgr.y (mgr.24661) 2083 : cluster [DBG] pgmap v3992: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1009 KiB/s rd, 5.0 KiB/s wr, 1.36k op/s 2024-09-13T00:46:36.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:36 smithi136 bash[76371]: cluster 2024-09-13T00:46:35.908006+0000 mon.a (mon.0) 4100 : cluster [DBG] osdmap e2350: 8 total, 8 up, 8 in 2024-09-13T00:46:38.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:37 smithi136 bash[76371]: cluster 2024-09-13T00:46:36.604579+0000 mgr.y (mgr.24661) 2084 : cluster [DBG] pgmap v3995: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 3.5 KiB/s rd, 0 B/s wr, 6 op/s 2024-09-13T00:46:38.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:37 smithi136 bash[76371]: cluster 2024-09-13T00:46:36.912642+0000 mon.a (mon.0) 4101 : cluster [DBG] osdmap e2351: 8 total, 8 up, 8 in 2024-09-13T00:46:38.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:37 smithi136 bash[75164]: cluster 2024-09-13T00:46:36.604579+0000 mgr.y (mgr.24661) 2084 : cluster [DBG] pgmap v3995: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 3.5 KiB/s rd, 0 B/s wr, 6 op/s 2024-09-13T00:46:38.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:37 smithi136 bash[75164]: cluster 2024-09-13T00:46:36.912642+0000 mon.a (mon.0) 4101 : cluster [DBG] osdmap e2351: 8 total, 8 up, 8 in 2024-09-13T00:46:38.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:37 smithi181 bash[67587]: cluster 2024-09-13T00:46:36.604579+0000 mgr.y (mgr.24661) 2084 : cluster [DBG] pgmap v3995: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 3.5 KiB/s rd, 0 B/s wr, 6 op/s 2024-09-13T00:46:38.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:37 smithi181 bash[67587]: cluster 2024-09-13T00:46:36.912642+0000 mon.a (mon.0) 4101 : cluster [DBG] osdmap e2351: 8 total, 8 up, 8 in 2024-09-13T00:46:39.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:38 smithi136 bash[75164]: cluster 2024-09-13T00:46:37.914678+0000 mon.a (mon.0) 4102 : cluster [DBG] osdmap e2352: 8 total, 8 up, 8 in 2024-09-13T00:46:39.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:38 smithi136 bash[75164]: audit 2024-09-13T00:46:38.849152+0000 mon.a (mon.0) 4103 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:46:39.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:38 smithi136 bash[76371]: cluster 2024-09-13T00:46:37.914678+0000 mon.a (mon.0) 4102 : cluster [DBG] osdmap e2352: 8 total, 8 up, 8 in 2024-09-13T00:46:39.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:38 smithi136 bash[76371]: audit 2024-09-13T00:46:38.849152+0000 mon.a (mon.0) 4103 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:46:39.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:38 smithi181 bash[67587]: cluster 2024-09-13T00:46:37.914678+0000 mon.a (mon.0) 4102 : cluster [DBG] osdmap e2352: 8 total, 8 up, 8 in 2024-09-13T00:46:39.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:38 smithi181 bash[67587]: audit 2024-09-13T00:46:38.849152+0000 mon.a (mon.0) 4103 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:46:39.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:46:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:46:39] "GET /metrics HTTP/1.1" 200 36747 "" "Prometheus/2.43.0" 2024-09-13T00:46:40.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:39 smithi136 bash[75164]: cluster 2024-09-13T00:46:38.605491+0000 mgr.y (mgr.24661) 2085 : cluster [DBG] pgmap v3998: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 975 KiB/s rd, 5.0 KiB/s wr, 1.32k op/s 2024-09-13T00:46:40.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:39 smithi136 bash[75164]: cluster 2024-09-13T00:46:38.942035+0000 mon.a (mon.0) 4104 : cluster [DBG] osdmap e2353: 8 total, 8 up, 8 in 2024-09-13T00:46:40.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:39 smithi136 bash[75164]: cluster 2024-09-13T00:46:39.909251+0000 mon.a (mon.0) 4105 : cluster [DBG] osdmap e2354: 8 total, 8 up, 8 in 2024-09-13T00:46:40.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:39 smithi136 bash[76371]: cluster 2024-09-13T00:46:38.605491+0000 mgr.y (mgr.24661) 2085 : cluster [DBG] pgmap v3998: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 975 KiB/s rd, 5.0 KiB/s wr, 1.32k op/s 2024-09-13T00:46:40.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:39 smithi136 bash[76371]: cluster 2024-09-13T00:46:38.942035+0000 mon.a (mon.0) 4104 : cluster [DBG] osdmap e2353: 8 total, 8 up, 8 in 2024-09-13T00:46:40.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:39 smithi136 bash[76371]: cluster 2024-09-13T00:46:39.909251+0000 mon.a (mon.0) 4105 : cluster [DBG] osdmap e2354: 8 total, 8 up, 8 in 2024-09-13T00:46:40.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:39 smithi181 bash[67587]: cluster 2024-09-13T00:46:38.605491+0000 mgr.y (mgr.24661) 2085 : cluster [DBG] pgmap v3998: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 975 KiB/s rd, 5.0 KiB/s wr, 1.32k op/s 2024-09-13T00:46:40.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:39 smithi181 bash[67587]: cluster 2024-09-13T00:46:38.942035+0000 mon.a (mon.0) 4104 : cluster [DBG] osdmap e2353: 8 total, 8 up, 8 in 2024-09-13T00:46:40.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:39 smithi181 bash[67587]: cluster 2024-09-13T00:46:39.909251+0000 mon.a (mon.0) 4105 : cluster [DBG] osdmap e2354: 8 total, 8 up, 8 in 2024-09-13T00:46:42.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:41 smithi136 bash[75164]: cluster 2024-09-13T00:46:40.606332+0000 mgr.y (mgr.24661) 2086 : cluster [DBG] pgmap v4001: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 974 KiB/s rd, 5.0 KiB/s wr, 1.32k op/s 2024-09-13T00:46:42.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:41 smithi136 bash[75164]: cluster 2024-09-13T00:46:40.952481+0000 mon.a (mon.0) 4106 : cluster [DBG] osdmap e2355: 8 total, 8 up, 8 in 2024-09-13T00:46:42.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:41 smithi136 bash[76371]: cluster 2024-09-13T00:46:40.606332+0000 mgr.y (mgr.24661) 2086 : cluster [DBG] pgmap v4001: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 974 KiB/s rd, 5.0 KiB/s wr, 1.32k op/s 2024-09-13T00:46:42.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:41 smithi136 bash[76371]: cluster 2024-09-13T00:46:40.952481+0000 mon.a (mon.0) 4106 : cluster [DBG] osdmap e2355: 8 total, 8 up, 8 in 2024-09-13T00:46:42.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:41 smithi181 bash[67587]: cluster 2024-09-13T00:46:40.606332+0000 mgr.y (mgr.24661) 2086 : cluster [DBG] pgmap v4001: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 974 KiB/s rd, 5.0 KiB/s wr, 1.32k op/s 2024-09-13T00:46:42.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:41 smithi181 bash[67587]: cluster 2024-09-13T00:46:40.952481+0000 mon.a (mon.0) 4106 : cluster [DBG] osdmap e2355: 8 total, 8 up, 8 in 2024-09-13T00:46:42.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:46:42 smithi181 bash[95505]: ts=2024-09-13T00:46:42.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:46:43.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:42 smithi136 bash[75164]: cluster 2024-09-13T00:46:41.955148+0000 mon.a (mon.0) 4107 : cluster [DBG] osdmap e2356: 8 total, 8 up, 8 in 2024-09-13T00:46:43.602 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:42 smithi136 bash[76371]: cluster 2024-09-13T00:46:41.955148+0000 mon.a (mon.0) 4107 : cluster [DBG] osdmap e2356: 8 total, 8 up, 8 in 2024-09-13T00:46:43.602 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:42 smithi181 bash[67587]: cluster 2024-09-13T00:46:41.955148+0000 mon.a (mon.0) 4107 : cluster [DBG] osdmap e2356: 8 total, 8 up, 8 in 2024-09-13T00:46:44.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:43 smithi136 bash[75164]: cluster 2024-09-13T00:46:42.607021+0000 mgr.y (mgr.24661) 2087 : cluster [DBG] pgmap v4004: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 942 KiB/s rd, 5.0 KiB/s wr, 1.27k op/s 2024-09-13T00:46:44.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:43 smithi136 bash[75164]: cluster 2024-09-13T00:46:42.980525+0000 mon.a (mon.0) 4108 : cluster [DBG] osdmap e2357: 8 total, 8 up, 8 in 2024-09-13T00:46:44.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:43 smithi136 bash[76371]: cluster 2024-09-13T00:46:42.607021+0000 mgr.y (mgr.24661) 2087 : cluster [DBG] pgmap v4004: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 942 KiB/s rd, 5.0 KiB/s wr, 1.27k op/s 2024-09-13T00:46:44.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:43 smithi136 bash[76371]: cluster 2024-09-13T00:46:42.980525+0000 mon.a (mon.0) 4108 : cluster [DBG] osdmap e2357: 8 total, 8 up, 8 in 2024-09-13T00:46:44.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:43 smithi181 bash[67587]: cluster 2024-09-13T00:46:42.607021+0000 mgr.y (mgr.24661) 2087 : cluster [DBG] pgmap v4004: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 942 KiB/s rd, 5.0 KiB/s wr, 1.27k op/s 2024-09-13T00:46:44.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:43 smithi181 bash[67587]: cluster 2024-09-13T00:46:42.980525+0000 mon.a (mon.0) 4108 : cluster [DBG] osdmap e2357: 8 total, 8 up, 8 in 2024-09-13T00:46:45.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:44 smithi136 bash[75164]: cluster 2024-09-13T00:46:43.982115+0000 mon.a (mon.0) 4109 : cluster [DBG] osdmap e2358: 8 total, 8 up, 8 in 2024-09-13T00:46:45.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:44 smithi136 bash[75164]: cluster 2024-09-13T00:46:44.910485+0000 mon.a (mon.0) 4110 : cluster [DBG] osdmap e2359: 8 total, 8 up, 8 in 2024-09-13T00:46:45.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:44 smithi136 bash[76371]: cluster 2024-09-13T00:46:43.982115+0000 mon.a (mon.0) 4109 : cluster [DBG] osdmap e2358: 8 total, 8 up, 8 in 2024-09-13T00:46:45.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:44 smithi136 bash[76371]: cluster 2024-09-13T00:46:44.910485+0000 mon.a (mon.0) 4110 : cluster [DBG] osdmap e2359: 8 total, 8 up, 8 in 2024-09-13T00:46:45.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:44 smithi181 bash[67587]: cluster 2024-09-13T00:46:43.982115+0000 mon.a (mon.0) 4109 : cluster [DBG] osdmap e2358: 8 total, 8 up, 8 in 2024-09-13T00:46:45.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:44 smithi181 bash[67587]: cluster 2024-09-13T00:46:44.910485+0000 mon.a (mon.0) 4110 : cluster [DBG] osdmap e2359: 8 total, 8 up, 8 in 2024-09-13T00:46:46.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:45 smithi136 bash[76371]: cluster 2024-09-13T00:46:44.607734+0000 mgr.y (mgr.24661) 2088 : cluster [DBG] pgmap v4007: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 945 KiB/s rd, 5.0 KiB/s wr, 1.28k op/s 2024-09-13T00:46:46.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:45 smithi136 bash[75164]: cluster 2024-09-13T00:46:44.607734+0000 mgr.y (mgr.24661) 2088 : cluster [DBG] pgmap v4007: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 945 KiB/s rd, 5.0 KiB/s wr, 1.28k op/s 2024-09-13T00:46:46.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:45 smithi181 bash[67587]: cluster 2024-09-13T00:46:44.607734+0000 mgr.y (mgr.24661) 2088 : cluster [DBG] pgmap v4007: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 945 KiB/s rd, 5.0 KiB/s wr, 1.28k op/s 2024-09-13T00:46:47.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:47 smithi181 bash[67587]: cluster 2024-09-13T00:46:46.011517+0000 mon.a (mon.0) 4111 : cluster [DBG] osdmap e2360: 8 total, 8 up, 8 in 2024-09-13T00:46:47.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:47 smithi136 bash[75164]: cluster 2024-09-13T00:46:46.011517+0000 mon.a (mon.0) 4111 : cluster [DBG] osdmap e2360: 8 total, 8 up, 8 in 2024-09-13T00:46:47.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:47 smithi136 bash[76371]: cluster 2024-09-13T00:46:46.011517+0000 mon.a (mon.0) 4111 : cluster [DBG] osdmap e2360: 8 total, 8 up, 8 in 2024-09-13T00:46:48.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:48 smithi181 bash[67587]: cluster 2024-09-13T00:46:46.608645+0000 mgr.y (mgr.24661) 2089 : cluster [DBG] pgmap v4010: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 3.5 KiB/s rd, 0 B/s wr, 6 op/s 2024-09-13T00:46:48.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:48 smithi181 bash[67587]: cluster 2024-09-13T00:46:47.021683+0000 mon.a (mon.0) 4112 : cluster [DBG] osdmap e2361: 8 total, 8 up, 8 in 2024-09-13T00:46:48.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:48 smithi136 bash[75164]: cluster 2024-09-13T00:46:46.608645+0000 mgr.y (mgr.24661) 2089 : cluster [DBG] pgmap v4010: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 3.5 KiB/s rd, 0 B/s wr, 6 op/s 2024-09-13T00:46:48.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:48 smithi136 bash[75164]: cluster 2024-09-13T00:46:47.021683+0000 mon.a (mon.0) 4112 : cluster [DBG] osdmap e2361: 8 total, 8 up, 8 in 2024-09-13T00:46:48.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:48 smithi136 bash[76371]: cluster 2024-09-13T00:46:46.608645+0000 mgr.y (mgr.24661) 2089 : cluster [DBG] pgmap v4010: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 3.5 KiB/s rd, 0 B/s wr, 6 op/s 2024-09-13T00:46:48.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:48 smithi136 bash[76371]: cluster 2024-09-13T00:46:47.021683+0000 mon.a (mon.0) 4112 : cluster [DBG] osdmap e2361: 8 total, 8 up, 8 in 2024-09-13T00:46:49.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:49 smithi181 bash[67587]: cluster 2024-09-13T00:46:48.032702+0000 mon.a (mon.0) 4113 : cluster [DBG] osdmap e2362: 8 total, 8 up, 8 in 2024-09-13T00:46:49.412 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:49 smithi136 bash[76371]: cluster 2024-09-13T00:46:48.032702+0000 mon.a (mon.0) 4113 : cluster [DBG] osdmap e2362: 8 total, 8 up, 8 in 2024-09-13T00:46:49.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:49 smithi136 bash[75164]: cluster 2024-09-13T00:46:48.032702+0000 mon.a (mon.0) 4113 : cluster [DBG] osdmap e2362: 8 total, 8 up, 8 in 2024-09-13T00:46:49.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:46:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:46:49] "GET /metrics HTTP/1.1" 200 36747 "" "Prometheus/2.43.0" 2024-09-13T00:46:50.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:50 smithi181 bash[67587]: cluster 2024-09-13T00:46:48.609556+0000 mgr.y (mgr.24661) 2090 : cluster [DBG] pgmap v4013: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 910 KiB/s rd, 5.0 KiB/s wr, 1.23k op/s 2024-09-13T00:46:50.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:50 smithi181 bash[67587]: cluster 2024-09-13T00:46:49.043838+0000 mon.a (mon.0) 4114 : cluster [DBG] osdmap e2363: 8 total, 8 up, 8 in 2024-09-13T00:46:50.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:50 smithi181 bash[67587]: cluster 2024-09-13T00:46:49.918143+0000 mon.a (mon.0) 4115 : cluster [DBG] osdmap e2364: 8 total, 8 up, 8 in 2024-09-13T00:46:50.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:50 smithi136 bash[75164]: cluster 2024-09-13T00:46:48.609556+0000 mgr.y (mgr.24661) 2090 : cluster [DBG] pgmap v4013: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 910 KiB/s rd, 5.0 KiB/s wr, 1.23k op/s 2024-09-13T00:46:50.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:50 smithi136 bash[75164]: cluster 2024-09-13T00:46:49.043838+0000 mon.a (mon.0) 4114 : cluster [DBG] osdmap e2363: 8 total, 8 up, 8 in 2024-09-13T00:46:50.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:50 smithi136 bash[75164]: cluster 2024-09-13T00:46:49.918143+0000 mon.a (mon.0) 4115 : cluster [DBG] osdmap e2364: 8 total, 8 up, 8 in 2024-09-13T00:46:50.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:50 smithi136 bash[76371]: cluster 2024-09-13T00:46:48.609556+0000 mgr.y (mgr.24661) 2090 : cluster [DBG] pgmap v4013: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 910 KiB/s rd, 5.0 KiB/s wr, 1.23k op/s 2024-09-13T00:46:50.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:50 smithi136 bash[76371]: cluster 2024-09-13T00:46:49.043838+0000 mon.a (mon.0) 4114 : cluster [DBG] osdmap e2363: 8 total, 8 up, 8 in 2024-09-13T00:46:50.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:50 smithi136 bash[76371]: cluster 2024-09-13T00:46:49.918143+0000 mon.a (mon.0) 4115 : cluster [DBG] osdmap e2364: 8 total, 8 up, 8 in 2024-09-13T00:46:52.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:52 smithi181 bash[67587]: cluster 2024-09-13T00:46:50.610391+0000 mgr.y (mgr.24661) 2091 : cluster [DBG] pgmap v4016: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 911 KiB/s rd, 5.0 KiB/s wr, 1.23k op/s 2024-09-13T00:46:52.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:52 smithi181 bash[67587]: cluster 2024-09-13T00:46:51.050290+0000 mon.a (mon.0) 4116 : cluster [DBG] osdmap e2365: 8 total, 8 up, 8 in 2024-09-13T00:46:52.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:52 smithi136 bash[76371]: cluster 2024-09-13T00:46:50.610391+0000 mgr.y (mgr.24661) 2091 : cluster [DBG] pgmap v4016: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 911 KiB/s rd, 5.0 KiB/s wr, 1.23k op/s 2024-09-13T00:46:52.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:52 smithi136 bash[76371]: cluster 2024-09-13T00:46:51.050290+0000 mon.a (mon.0) 4116 : cluster [DBG] osdmap e2365: 8 total, 8 up, 8 in 2024-09-13T00:46:52.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:52 smithi136 bash[75164]: cluster 2024-09-13T00:46:50.610391+0000 mgr.y (mgr.24661) 2091 : cluster [DBG] pgmap v4016: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 911 KiB/s rd, 5.0 KiB/s wr, 1.23k op/s 2024-09-13T00:46:52.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:52 smithi136 bash[75164]: cluster 2024-09-13T00:46:51.050290+0000 mon.a (mon.0) 4116 : cluster [DBG] osdmap e2365: 8 total, 8 up, 8 in 2024-09-13T00:46:52.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:46:52 smithi181 bash[95505]: ts=2024-09-13T00:46:52.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:46:53.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:53 smithi181 bash[67587]: cluster 2024-09-13T00:46:52.059586+0000 mon.a (mon.0) 4117 : cluster [DBG] osdmap e2366: 8 total, 8 up, 8 in 2024-09-13T00:46:53.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:53 smithi136 bash[75164]: cluster 2024-09-13T00:46:52.059586+0000 mon.a (mon.0) 4117 : cluster [DBG] osdmap e2366: 8 total, 8 up, 8 in 2024-09-13T00:46:53.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:53 smithi136 bash[76371]: cluster 2024-09-13T00:46:52.059586+0000 mon.a (mon.0) 4117 : cluster [DBG] osdmap e2366: 8 total, 8 up, 8 in 2024-09-13T00:46:54.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:54 smithi181 bash[67587]: cluster 2024-09-13T00:46:52.611121+0000 mgr.y (mgr.24661) 2092 : cluster [DBG] pgmap v4019: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 877 KiB/s rd, 5.0 KiB/s wr, 1.19k op/s 2024-09-13T00:46:54.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:54 smithi181 bash[67587]: cluster 2024-09-13T00:46:53.076953+0000 mon.a (mon.0) 4118 : cluster [DBG] osdmap e2367: 8 total, 8 up, 8 in 2024-09-13T00:46:54.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:54 smithi181 bash[67587]: audit 2024-09-13T00:46:53.849517+0000 mon.a (mon.0) 4119 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:46:54.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:54 smithi136 bash[75164]: cluster 2024-09-13T00:46:52.611121+0000 mgr.y (mgr.24661) 2092 : cluster [DBG] pgmap v4019: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 877 KiB/s rd, 5.0 KiB/s wr, 1.19k op/s 2024-09-13T00:46:54.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:54 smithi136 bash[75164]: cluster 2024-09-13T00:46:53.076953+0000 mon.a (mon.0) 4118 : cluster [DBG] osdmap e2367: 8 total, 8 up, 8 in 2024-09-13T00:46:54.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:54 smithi136 bash[75164]: audit 2024-09-13T00:46:53.849517+0000 mon.a (mon.0) 4119 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:46:54.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:54 smithi136 bash[76371]: cluster 2024-09-13T00:46:52.611121+0000 mgr.y (mgr.24661) 2092 : cluster [DBG] pgmap v4019: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 877 KiB/s rd, 5.0 KiB/s wr, 1.19k op/s 2024-09-13T00:46:54.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:54 smithi136 bash[76371]: cluster 2024-09-13T00:46:53.076953+0000 mon.a (mon.0) 4118 : cluster [DBG] osdmap e2367: 8 total, 8 up, 8 in 2024-09-13T00:46:54.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:54 smithi136 bash[76371]: audit 2024-09-13T00:46:53.849517+0000 mon.a (mon.0) 4119 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:46:55.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:55 smithi181 bash[67587]: cluster 2024-09-13T00:46:54.083129+0000 mon.a (mon.0) 4120 : cluster [DBG] osdmap e2368: 8 total, 8 up, 8 in 2024-09-13T00:46:55.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:55 smithi181 bash[67587]: cluster 2024-09-13T00:46:54.915661+0000 mon.a (mon.0) 4121 : cluster [DBG] osdmap e2369: 8 total, 8 up, 8 in 2024-09-13T00:46:55.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:55 smithi136 bash[76371]: cluster 2024-09-13T00:46:54.083129+0000 mon.a (mon.0) 4120 : cluster [DBG] osdmap e2368: 8 total, 8 up, 8 in 2024-09-13T00:46:55.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:55 smithi136 bash[76371]: cluster 2024-09-13T00:46:54.915661+0000 mon.a (mon.0) 4121 : cluster [DBG] osdmap e2369: 8 total, 8 up, 8 in 2024-09-13T00:46:55.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:55 smithi136 bash[75164]: cluster 2024-09-13T00:46:54.083129+0000 mon.a (mon.0) 4120 : cluster [DBG] osdmap e2368: 8 total, 8 up, 8 in 2024-09-13T00:46:55.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:55 smithi136 bash[75164]: cluster 2024-09-13T00:46:54.915661+0000 mon.a (mon.0) 4121 : cluster [DBG] osdmap e2369: 8 total, 8 up, 8 in 2024-09-13T00:46:56.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:56 smithi181 bash[67587]: cluster 2024-09-13T00:46:54.611833+0000 mgr.y (mgr.24661) 2093 : cluster [DBG] pgmap v4022: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 879 KiB/s rd, 5.0 KiB/s wr, 1.19k op/s 2024-09-13T00:46:56.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:56 smithi181 bash[67587]: cluster 2024-09-13T00:46:55.918045+0000 mon.a (mon.0) 4122 : cluster [DBG] osdmap e2370: 8 total, 8 up, 8 in 2024-09-13T00:46:56.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:56 smithi136 bash[76371]: cluster 2024-09-13T00:46:54.611833+0000 mgr.y (mgr.24661) 2093 : cluster [DBG] pgmap v4022: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 879 KiB/s rd, 5.0 KiB/s wr, 1.19k op/s 2024-09-13T00:46:56.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:56 smithi136 bash[76371]: cluster 2024-09-13T00:46:55.918045+0000 mon.a (mon.0) 4122 : cluster [DBG] osdmap e2370: 8 total, 8 up, 8 in 2024-09-13T00:46:56.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:56 smithi136 bash[75164]: cluster 2024-09-13T00:46:54.611833+0000 mgr.y (mgr.24661) 2093 : cluster [DBG] pgmap v4022: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 879 KiB/s rd, 5.0 KiB/s wr, 1.19k op/s 2024-09-13T00:46:56.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:56 smithi136 bash[75164]: cluster 2024-09-13T00:46:55.918045+0000 mon.a (mon.0) 4122 : cluster [DBG] osdmap e2370: 8 total, 8 up, 8 in 2024-09-13T00:46:58.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:57 smithi136 bash[76371]: cluster 2024-09-13T00:46:56.612894+0000 mgr.y (mgr.24661) 2094 : cluster [DBG] pgmap v4025: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 3.5 KiB/s rd, 0 B/s wr, 6 op/s 2024-09-13T00:46:58.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:57 smithi136 bash[76371]: cluster 2024-09-13T00:46:56.921797+0000 mon.a (mon.0) 4123 : cluster [DBG] osdmap e2371: 8 total, 8 up, 8 in 2024-09-13T00:46:58.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:57 smithi136 bash[76371]: audit 2024-09-13T00:46:57.282020+0000 mon.a (mon.0) 4124 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:46:58.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:57 smithi136 bash[76371]: audit 2024-09-13T00:46:57.652504+0000 mon.a (mon.0) 4125 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:46:58.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:57 smithi136 bash[76371]: audit 2024-09-13T00:46:57.657189+0000 mon.a (mon.0) 4126 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:46:58.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:57 smithi136 bash[75164]: cluster 2024-09-13T00:46:56.612894+0000 mgr.y (mgr.24661) 2094 : cluster [DBG] pgmap v4025: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 3.5 KiB/s rd, 0 B/s wr, 6 op/s 2024-09-13T00:46:58.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:57 smithi136 bash[75164]: cluster 2024-09-13T00:46:56.921797+0000 mon.a (mon.0) 4123 : cluster [DBG] osdmap e2371: 8 total, 8 up, 8 in 2024-09-13T00:46:58.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:57 smithi136 bash[75164]: audit 2024-09-13T00:46:57.282020+0000 mon.a (mon.0) 4124 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:46:58.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:57 smithi136 bash[75164]: audit 2024-09-13T00:46:57.652504+0000 mon.a (mon.0) 4125 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:46:58.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:57 smithi136 bash[75164]: audit 2024-09-13T00:46:57.657189+0000 mon.a (mon.0) 4126 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:46:58.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:57 smithi181 bash[67587]: cluster 2024-09-13T00:46:56.612894+0000 mgr.y (mgr.24661) 2094 : cluster [DBG] pgmap v4025: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 3.5 KiB/s rd, 0 B/s wr, 6 op/s 2024-09-13T00:46:58.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:57 smithi181 bash[67587]: cluster 2024-09-13T00:46:56.921797+0000 mon.a (mon.0) 4123 : cluster [DBG] osdmap e2371: 8 total, 8 up, 8 in 2024-09-13T00:46:58.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:57 smithi181 bash[67587]: audit 2024-09-13T00:46:57.282020+0000 mon.a (mon.0) 4124 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:46:58.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:57 smithi181 bash[67587]: audit 2024-09-13T00:46:57.652504+0000 mon.a (mon.0) 4125 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:46:58.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:57 smithi181 bash[67587]: audit 2024-09-13T00:46:57.657189+0000 mon.a (mon.0) 4126 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:46:59.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:58 smithi136 bash[75164]: cluster 2024-09-13T00:46:57.921180+0000 mon.a (mon.0) 4127 : cluster [DBG] osdmap e2372: 8 total, 8 up, 8 in 2024-09-13T00:46:59.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:58 smithi136 bash[75164]: audit 2024-09-13T00:46:58.086081+0000 mon.a (mon.0) 4128 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:46:59.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:58 smithi136 bash[75164]: audit 2024-09-13T00:46:58.087238+0000 mon.a (mon.0) 4129 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:46:59.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:58 smithi136 bash[75164]: audit 2024-09-13T00:46:58.094374+0000 mon.a (mon.0) 4130 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:46:59.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:58 smithi136 bash[76371]: cluster 2024-09-13T00:46:57.921180+0000 mon.a (mon.0) 4127 : cluster [DBG] osdmap e2372: 8 total, 8 up, 8 in 2024-09-13T00:46:59.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:58 smithi136 bash[76371]: audit 2024-09-13T00:46:58.086081+0000 mon.a (mon.0) 4128 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:46:59.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:58 smithi136 bash[76371]: audit 2024-09-13T00:46:58.087238+0000 mon.a (mon.0) 4129 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:46:59.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:58 smithi136 bash[76371]: audit 2024-09-13T00:46:58.094374+0000 mon.a (mon.0) 4130 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:46:59.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:58 smithi181 bash[67587]: cluster 2024-09-13T00:46:57.921180+0000 mon.a (mon.0) 4127 : cluster [DBG] osdmap e2372: 8 total, 8 up, 8 in 2024-09-13T00:46:59.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:58 smithi181 bash[67587]: audit 2024-09-13T00:46:58.086081+0000 mon.a (mon.0) 4128 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:46:59.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:58 smithi181 bash[67587]: audit 2024-09-13T00:46:58.087238+0000 mon.a (mon.0) 4129 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:46:59.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:58 smithi181 bash[67587]: audit 2024-09-13T00:46:58.094374+0000 mon.a (mon.0) 4130 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:46:59.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:46:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:46:59] "GET /metrics HTTP/1.1" 200 36746 "" "Prometheus/2.43.0" 2024-09-13T00:47:00.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:59 smithi136 bash[75164]: cluster 2024-09-13T00:46:58.613756+0000 mgr.y (mgr.24661) 2095 : cluster [DBG] pgmap v4028: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 844 KiB/s rd, 5.0 KiB/s wr, 1.14k op/s 2024-09-13T00:47:00.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:59 smithi136 bash[75164]: cluster 2024-09-13T00:46:58.941339+0000 mon.a (mon.0) 4131 : cluster [DBG] osdmap e2373: 8 total, 8 up, 8 in 2024-09-13T00:47:00.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:46:59 smithi136 bash[75164]: cluster 2024-09-13T00:46:59.917953+0000 mon.a (mon.0) 4132 : cluster [DBG] osdmap e2374: 8 total, 8 up, 8 in 2024-09-13T00:47:00.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:59 smithi136 bash[76371]: cluster 2024-09-13T00:46:58.613756+0000 mgr.y (mgr.24661) 2095 : cluster [DBG] pgmap v4028: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 844 KiB/s rd, 5.0 KiB/s wr, 1.14k op/s 2024-09-13T00:47:00.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:59 smithi136 bash[76371]: cluster 2024-09-13T00:46:58.941339+0000 mon.a (mon.0) 4131 : cluster [DBG] osdmap e2373: 8 total, 8 up, 8 in 2024-09-13T00:47:00.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:46:59 smithi136 bash[76371]: cluster 2024-09-13T00:46:59.917953+0000 mon.a (mon.0) 4132 : cluster [DBG] osdmap e2374: 8 total, 8 up, 8 in 2024-09-13T00:47:00.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:59 smithi181 bash[67587]: cluster 2024-09-13T00:46:58.613756+0000 mgr.y (mgr.24661) 2095 : cluster [DBG] pgmap v4028: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 844 KiB/s rd, 5.0 KiB/s wr, 1.14k op/s 2024-09-13T00:47:00.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:59 smithi181 bash[67587]: cluster 2024-09-13T00:46:58.941339+0000 mon.a (mon.0) 4131 : cluster [DBG] osdmap e2373: 8 total, 8 up, 8 in 2024-09-13T00:47:00.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:46:59 smithi181 bash[67587]: cluster 2024-09-13T00:46:59.917953+0000 mon.a (mon.0) 4132 : cluster [DBG] osdmap e2374: 8 total, 8 up, 8 in 2024-09-13T00:47:02.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:01 smithi136 bash[75164]: cluster 2024-09-13T00:47:00.614600+0000 mgr.y (mgr.24661) 2096 : cluster [DBG] pgmap v4031: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 844 KiB/s rd, 5.0 KiB/s wr, 1.14k op/s 2024-09-13T00:47:02.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:01 smithi136 bash[75164]: cluster 2024-09-13T00:47:00.958168+0000 mon.a (mon.0) 4133 : cluster [DBG] osdmap e2375: 8 total, 8 up, 8 in 2024-09-13T00:47:02.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:01 smithi136 bash[76371]: cluster 2024-09-13T00:47:00.614600+0000 mgr.y (mgr.24661) 2096 : cluster [DBG] pgmap v4031: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 844 KiB/s rd, 5.0 KiB/s wr, 1.14k op/s 2024-09-13T00:47:02.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:01 smithi136 bash[76371]: cluster 2024-09-13T00:47:00.958168+0000 mon.a (mon.0) 4133 : cluster [DBG] osdmap e2375: 8 total, 8 up, 8 in 2024-09-13T00:47:02.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:01 smithi181 bash[67587]: cluster 2024-09-13T00:47:00.614600+0000 mgr.y (mgr.24661) 2096 : cluster [DBG] pgmap v4031: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 844 KiB/s rd, 5.0 KiB/s wr, 1.14k op/s 2024-09-13T00:47:02.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:01 smithi181 bash[67587]: cluster 2024-09-13T00:47:00.958168+0000 mon.a (mon.0) 4133 : cluster [DBG] osdmap e2375: 8 total, 8 up, 8 in 2024-09-13T00:47:02.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:47:02 smithi181 bash[95505]: ts=2024-09-13T00:47:02.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:47:03.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:02 smithi136 bash[75164]: cluster 2024-09-13T00:47:01.976434+0000 mon.a (mon.0) 4134 : cluster [DBG] osdmap e2376: 8 total, 8 up, 8 in 2024-09-13T00:47:03.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:02 smithi136 bash[76371]: cluster 2024-09-13T00:47:01.976434+0000 mon.a (mon.0) 4134 : cluster [DBG] osdmap e2376: 8 total, 8 up, 8 in 2024-09-13T00:47:03.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:02 smithi181 bash[67587]: cluster 2024-09-13T00:47:01.976434+0000 mon.a (mon.0) 4134 : cluster [DBG] osdmap e2376: 8 total, 8 up, 8 in 2024-09-13T00:47:04.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:03 smithi136 bash[75164]: cluster 2024-09-13T00:47:02.615239+0000 mgr.y (mgr.24661) 2097 : cluster [DBG] pgmap v4034: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 812 KiB/s rd, 5.0 KiB/s wr, 1.10k op/s 2024-09-13T00:47:04.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:03 smithi136 bash[75164]: cluster 2024-09-13T00:47:02.986690+0000 mon.a (mon.0) 4135 : cluster [DBG] osdmap e2377: 8 total, 8 up, 8 in 2024-09-13T00:47:04.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:03 smithi136 bash[76371]: cluster 2024-09-13T00:47:02.615239+0000 mgr.y (mgr.24661) 2097 : cluster [DBG] pgmap v4034: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 812 KiB/s rd, 5.0 KiB/s wr, 1.10k op/s 2024-09-13T00:47:04.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:03 smithi136 bash[76371]: cluster 2024-09-13T00:47:02.986690+0000 mon.a (mon.0) 4135 : cluster [DBG] osdmap e2377: 8 total, 8 up, 8 in 2024-09-13T00:47:04.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:03 smithi181 bash[67587]: cluster 2024-09-13T00:47:02.615239+0000 mgr.y (mgr.24661) 2097 : cluster [DBG] pgmap v4034: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 812 KiB/s rd, 5.0 KiB/s wr, 1.10k op/s 2024-09-13T00:47:04.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:03 smithi181 bash[67587]: cluster 2024-09-13T00:47:02.986690+0000 mon.a (mon.0) 4135 : cluster [DBG] osdmap e2377: 8 total, 8 up, 8 in 2024-09-13T00:47:05.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:04 smithi136 bash[76371]: cluster 2024-09-13T00:47:03.994403+0000 mon.a (mon.0) 4136 : cluster [DBG] osdmap e2378: 8 total, 8 up, 8 in 2024-09-13T00:47:05.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:04 smithi136 bash[76371]: cluster 2024-09-13T00:47:04.921191+0000 mon.a (mon.0) 4137 : cluster [DBG] osdmap e2379: 8 total, 8 up, 8 in 2024-09-13T00:47:05.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:04 smithi136 bash[75164]: cluster 2024-09-13T00:47:03.994403+0000 mon.a (mon.0) 4136 : cluster [DBG] osdmap e2378: 8 total, 8 up, 8 in 2024-09-13T00:47:05.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:04 smithi136 bash[75164]: cluster 2024-09-13T00:47:04.921191+0000 mon.a (mon.0) 4137 : cluster [DBG] osdmap e2379: 8 total, 8 up, 8 in 2024-09-13T00:47:05.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:04 smithi181 bash[67587]: cluster 2024-09-13T00:47:03.994403+0000 mon.a (mon.0) 4136 : cluster [DBG] osdmap e2378: 8 total, 8 up, 8 in 2024-09-13T00:47:05.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:04 smithi181 bash[67587]: cluster 2024-09-13T00:47:04.921191+0000 mon.a (mon.0) 4137 : cluster [DBG] osdmap e2379: 8 total, 8 up, 8 in 2024-09-13T00:47:06.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:05 smithi136 bash[76371]: cluster 2024-09-13T00:47:04.615913+0000 mgr.y (mgr.24661) 2098 : cluster [DBG] pgmap v4037: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 812 KiB/s rd, 5.0 KiB/s wr, 1.10k op/s 2024-09-13T00:47:06.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:05 smithi136 bash[75164]: cluster 2024-09-13T00:47:04.615913+0000 mgr.y (mgr.24661) 2098 : cluster [DBG] pgmap v4037: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 812 KiB/s rd, 5.0 KiB/s wr, 1.10k op/s 2024-09-13T00:47:06.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:05 smithi181 bash[67587]: cluster 2024-09-13T00:47:04.615913+0000 mgr.y (mgr.24661) 2098 : cluster [DBG] pgmap v4037: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 812 KiB/s rd, 5.0 KiB/s wr, 1.10k op/s 2024-09-13T00:47:07.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:07 smithi181 bash[67587]: cluster 2024-09-13T00:47:06.010978+0000 mon.a (mon.0) 4138 : cluster [DBG] osdmap e2380: 8 total, 8 up, 8 in 2024-09-13T00:47:07.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:07 smithi136 bash[75164]: cluster 2024-09-13T00:47:06.010978+0000 mon.a (mon.0) 4138 : cluster [DBG] osdmap e2380: 8 total, 8 up, 8 in 2024-09-13T00:47:07.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:07 smithi136 bash[76371]: cluster 2024-09-13T00:47:06.010978+0000 mon.a (mon.0) 4138 : cluster [DBG] osdmap e2380: 8 total, 8 up, 8 in 2024-09-13T00:47:08.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:08 smithi181 bash[67587]: cluster 2024-09-13T00:47:06.616856+0000 mgr.y (mgr.24661) 2099 : cluster [DBG] pgmap v4040: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 2.5 KiB/s rd, 0 B/s wr, 4 op/s 2024-09-13T00:47:08.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:08 smithi181 bash[67587]: cluster 2024-09-13T00:47:07.021235+0000 mon.a (mon.0) 4139 : cluster [DBG] osdmap e2381: 8 total, 8 up, 8 in 2024-09-13T00:47:08.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:08 smithi136 bash[75164]: cluster 2024-09-13T00:47:06.616856+0000 mgr.y (mgr.24661) 2099 : cluster [DBG] pgmap v4040: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 2.5 KiB/s rd, 0 B/s wr, 4 op/s 2024-09-13T00:47:08.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:08 smithi136 bash[75164]: cluster 2024-09-13T00:47:07.021235+0000 mon.a (mon.0) 4139 : cluster [DBG] osdmap e2381: 8 total, 8 up, 8 in 2024-09-13T00:47:08.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:08 smithi136 bash[76371]: cluster 2024-09-13T00:47:06.616856+0000 mgr.y (mgr.24661) 2099 : cluster [DBG] pgmap v4040: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 2.5 KiB/s rd, 0 B/s wr, 4 op/s 2024-09-13T00:47:08.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:08 smithi136 bash[76371]: cluster 2024-09-13T00:47:07.021235+0000 mon.a (mon.0) 4139 : cluster [DBG] osdmap e2381: 8 total, 8 up, 8 in 2024-09-13T00:47:09.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:09 smithi181 bash[67587]: cluster 2024-09-13T00:47:08.030138+0000 mon.a (mon.0) 4140 : cluster [DBG] osdmap e2382: 8 total, 8 up, 8 in 2024-09-13T00:47:09.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:09 smithi181 bash[67587]: audit 2024-09-13T00:47:08.849892+0000 mon.a (mon.0) 4141 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:47:09.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:09 smithi136 bash[75164]: cluster 2024-09-13T00:47:08.030138+0000 mon.a (mon.0) 4140 : cluster [DBG] osdmap e2382: 8 total, 8 up, 8 in 2024-09-13T00:47:09.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:09 smithi136 bash[75164]: audit 2024-09-13T00:47:08.849892+0000 mon.a (mon.0) 4141 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:47:09.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:09 smithi136 bash[76371]: cluster 2024-09-13T00:47:08.030138+0000 mon.a (mon.0) 4140 : cluster [DBG] osdmap e2382: 8 total, 8 up, 8 in 2024-09-13T00:47:09.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:09 smithi136 bash[76371]: audit 2024-09-13T00:47:08.849892+0000 mon.a (mon.0) 4141 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:47:09.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:47:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:47:09] "GET /metrics HTTP/1.1" 200 36744 "" "Prometheus/2.43.0" 2024-09-13T00:47:10.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:10 smithi181 bash[67587]: cluster 2024-09-13T00:47:08.617717+0000 mgr.y (mgr.24661) 2100 : cluster [DBG] pgmap v4043: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 780 KiB/s rd, 5.0 KiB/s wr, 1.06k op/s 2024-09-13T00:47:10.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:10 smithi181 bash[67587]: cluster 2024-09-13T00:47:09.044367+0000 mon.a (mon.0) 4142 : cluster [DBG] osdmap e2383: 8 total, 8 up, 8 in 2024-09-13T00:47:10.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:10 smithi181 bash[67587]: cluster 2024-09-13T00:47:09.923295+0000 mon.a (mon.0) 4143 : cluster [DBG] osdmap e2384: 8 total, 8 up, 8 in 2024-09-13T00:47:10.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:10 smithi136 bash[75164]: cluster 2024-09-13T00:47:08.617717+0000 mgr.y (mgr.24661) 2100 : cluster [DBG] pgmap v4043: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 780 KiB/s rd, 5.0 KiB/s wr, 1.06k op/s 2024-09-13T00:47:10.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:10 smithi136 bash[75164]: cluster 2024-09-13T00:47:09.044367+0000 mon.a (mon.0) 4142 : cluster [DBG] osdmap e2383: 8 total, 8 up, 8 in 2024-09-13T00:47:10.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:10 smithi136 bash[75164]: cluster 2024-09-13T00:47:09.923295+0000 mon.a (mon.0) 4143 : cluster [DBG] osdmap e2384: 8 total, 8 up, 8 in 2024-09-13T00:47:10.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:10 smithi136 bash[76371]: cluster 2024-09-13T00:47:08.617717+0000 mgr.y (mgr.24661) 2100 : cluster [DBG] pgmap v4043: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 780 KiB/s rd, 5.0 KiB/s wr, 1.06k op/s 2024-09-13T00:47:10.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:10 smithi136 bash[76371]: cluster 2024-09-13T00:47:09.044367+0000 mon.a (mon.0) 4142 : cluster [DBG] osdmap e2383: 8 total, 8 up, 8 in 2024-09-13T00:47:10.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:10 smithi136 bash[76371]: cluster 2024-09-13T00:47:09.923295+0000 mon.a (mon.0) 4143 : cluster [DBG] osdmap e2384: 8 total, 8 up, 8 in 2024-09-13T00:47:12.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:12 smithi181 bash[67587]: cluster 2024-09-13T00:47:10.618593+0000 mgr.y (mgr.24661) 2101 : cluster [DBG] pgmap v4046: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 780 KiB/s rd, 5.0 KiB/s wr, 1.06k op/s 2024-09-13T00:47:12.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:12 smithi181 bash[67587]: cluster 2024-09-13T00:47:11.052511+0000 mon.a (mon.0) 4144 : cluster [DBG] osdmap e2385: 8 total, 8 up, 8 in 2024-09-13T00:47:12.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:12 smithi136 bash[75164]: cluster 2024-09-13T00:47:10.618593+0000 mgr.y (mgr.24661) 2101 : cluster [DBG] pgmap v4046: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 780 KiB/s rd, 5.0 KiB/s wr, 1.06k op/s 2024-09-13T00:47:12.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:12 smithi136 bash[75164]: cluster 2024-09-13T00:47:11.052511+0000 mon.a (mon.0) 4144 : cluster [DBG] osdmap e2385: 8 total, 8 up, 8 in 2024-09-13T00:47:12.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:12 smithi136 bash[76371]: cluster 2024-09-13T00:47:10.618593+0000 mgr.y (mgr.24661) 2101 : cluster [DBG] pgmap v4046: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 780 KiB/s rd, 5.0 KiB/s wr, 1.06k op/s 2024-09-13T00:47:12.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:12 smithi136 bash[76371]: cluster 2024-09-13T00:47:11.052511+0000 mon.a (mon.0) 4144 : cluster [DBG] osdmap e2385: 8 total, 8 up, 8 in 2024-09-13T00:47:12.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:47:12 smithi181 bash[95505]: ts=2024-09-13T00:47:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:47:13.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:13 smithi181 bash[67587]: cluster 2024-09-13T00:47:12.057227+0000 mon.a (mon.0) 4145 : cluster [DBG] osdmap e2386: 8 total, 8 up, 8 in 2024-09-13T00:47:13.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:13 smithi136 bash[75164]: cluster 2024-09-13T00:47:12.057227+0000 mon.a (mon.0) 4145 : cluster [DBG] osdmap e2386: 8 total, 8 up, 8 in 2024-09-13T00:47:13.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:13 smithi136 bash[76371]: cluster 2024-09-13T00:47:12.057227+0000 mon.a (mon.0) 4145 : cluster [DBG] osdmap e2386: 8 total, 8 up, 8 in 2024-09-13T00:47:14.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:14 smithi181 bash[67587]: cluster 2024-09-13T00:47:12.619274+0000 mgr.y (mgr.24661) 2102 : cluster [DBG] pgmap v4049: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 746 KiB/s rd, 5.0 KiB/s wr, 1.01k op/s 2024-09-13T00:47:14.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:14 smithi181 bash[67587]: cluster 2024-09-13T00:47:13.082914+0000 mon.a (mon.0) 4146 : cluster [DBG] osdmap e2387: 8 total, 8 up, 8 in 2024-09-13T00:47:14.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:14 smithi136 bash[75164]: cluster 2024-09-13T00:47:12.619274+0000 mgr.y (mgr.24661) 2102 : cluster [DBG] pgmap v4049: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 746 KiB/s rd, 5.0 KiB/s wr, 1.01k op/s 2024-09-13T00:47:14.511 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:14 smithi136 bash[75164]: cluster 2024-09-13T00:47:13.082914+0000 mon.a (mon.0) 4146 : cluster [DBG] osdmap e2387: 8 total, 8 up, 8 in 2024-09-13T00:47:14.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:14 smithi136 bash[76371]: cluster 2024-09-13T00:47:12.619274+0000 mgr.y (mgr.24661) 2102 : cluster [DBG] pgmap v4049: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 746 KiB/s rd, 5.0 KiB/s wr, 1.01k op/s 2024-09-13T00:47:14.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:14 smithi136 bash[76371]: cluster 2024-09-13T00:47:13.082914+0000 mon.a (mon.0) 4146 : cluster [DBG] osdmap e2387: 8 total, 8 up, 8 in 2024-09-13T00:47:15.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:15 smithi181 bash[67587]: cluster 2024-09-13T00:47:14.090669+0000 mon.a (mon.0) 4147 : cluster [DBG] osdmap e2388: 8 total, 8 up, 8 in 2024-09-13T00:47:15.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:15 smithi181 bash[67587]: cluster 2024-09-13T00:47:14.927235+0000 mon.a (mon.0) 4148 : cluster [DBG] osdmap e2389: 8 total, 8 up, 8 in 2024-09-13T00:47:15.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:15 smithi136 bash[75164]: cluster 2024-09-13T00:47:14.090669+0000 mon.a (mon.0) 4147 : cluster [DBG] osdmap e2388: 8 total, 8 up, 8 in 2024-09-13T00:47:15.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:15 smithi136 bash[75164]: cluster 2024-09-13T00:47:14.927235+0000 mon.a (mon.0) 4148 : cluster [DBG] osdmap e2389: 8 total, 8 up, 8 in 2024-09-13T00:47:15.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:15 smithi136 bash[76371]: cluster 2024-09-13T00:47:14.090669+0000 mon.a (mon.0) 4147 : cluster [DBG] osdmap e2388: 8 total, 8 up, 8 in 2024-09-13T00:47:15.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:15 smithi136 bash[76371]: cluster 2024-09-13T00:47:14.927235+0000 mon.a (mon.0) 4148 : cluster [DBG] osdmap e2389: 8 total, 8 up, 8 in 2024-09-13T00:47:16.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:16 smithi136 bash[75164]: cluster 2024-09-13T00:47:14.620125+0000 mgr.y (mgr.24661) 2103 : cluster [DBG] pgmap v4052: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 747 KiB/s rd, 5.0 KiB/s wr, 1.01k op/s 2024-09-13T00:47:16.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:16 smithi136 bash[75164]: cluster 2024-09-13T00:47:15.928202+0000 mon.a (mon.0) 4149 : cluster [DBG] osdmap e2390: 8 total, 8 up, 8 in 2024-09-13T00:47:16.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:16 smithi136 bash[76371]: cluster 2024-09-13T00:47:14.620125+0000 mgr.y (mgr.24661) 2103 : cluster [DBG] pgmap v4052: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 747 KiB/s rd, 5.0 KiB/s wr, 1.01k op/s 2024-09-13T00:47:16.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:16 smithi136 bash[76371]: cluster 2024-09-13T00:47:15.928202+0000 mon.a (mon.0) 4149 : cluster [DBG] osdmap e2390: 8 total, 8 up, 8 in 2024-09-13T00:47:16.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:16 smithi181 bash[67587]: cluster 2024-09-13T00:47:14.620125+0000 mgr.y (mgr.24661) 2103 : cluster [DBG] pgmap v4052: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 747 KiB/s rd, 5.0 KiB/s wr, 1.01k op/s 2024-09-13T00:47:16.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:16 smithi181 bash[67587]: cluster 2024-09-13T00:47:15.928202+0000 mon.a (mon.0) 4149 : cluster [DBG] osdmap e2390: 8 total, 8 up, 8 in 2024-09-13T00:47:18.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:17 smithi136 bash[76371]: cluster 2024-09-13T00:47:16.621203+0000 mgr.y (mgr.24661) 2104 : cluster [DBG] pgmap v4055: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 3.7 KiB/s rd, 0 B/s wr, 7 op/s 2024-09-13T00:47:18.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:17 smithi136 bash[76371]: cluster 2024-09-13T00:47:16.932944+0000 mon.a (mon.0) 4150 : cluster [DBG] osdmap e2391: 8 total, 8 up, 8 in 2024-09-13T00:47:18.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:17 smithi136 bash[75164]: cluster 2024-09-13T00:47:16.621203+0000 mgr.y (mgr.24661) 2104 : cluster [DBG] pgmap v4055: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 3.7 KiB/s rd, 0 B/s wr, 7 op/s 2024-09-13T00:47:18.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:17 smithi136 bash[75164]: cluster 2024-09-13T00:47:16.932944+0000 mon.a (mon.0) 4150 : cluster [DBG] osdmap e2391: 8 total, 8 up, 8 in 2024-09-13T00:47:18.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:17 smithi181 bash[67587]: cluster 2024-09-13T00:47:16.621203+0000 mgr.y (mgr.24661) 2104 : cluster [DBG] pgmap v4055: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 3.7 KiB/s rd, 0 B/s wr, 7 op/s 2024-09-13T00:47:18.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:17 smithi181 bash[67587]: cluster 2024-09-13T00:47:16.932944+0000 mon.a (mon.0) 4150 : cluster [DBG] osdmap e2391: 8 total, 8 up, 8 in 2024-09-13T00:47:19.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:18 smithi136 bash[75164]: cluster 2024-09-13T00:47:17.934978+0000 mon.a (mon.0) 4151 : cluster [DBG] osdmap e2392: 8 total, 8 up, 8 in 2024-09-13T00:47:19.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:18 smithi136 bash[76371]: cluster 2024-09-13T00:47:17.934978+0000 mon.a (mon.0) 4151 : cluster [DBG] osdmap e2392: 8 total, 8 up, 8 in 2024-09-13T00:47:19.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:18 smithi181 bash[67587]: cluster 2024-09-13T00:47:17.934978+0000 mon.a (mon.0) 4151 : cluster [DBG] osdmap e2392: 8 total, 8 up, 8 in 2024-09-13T00:47:19.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:47:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:47:19] "GET /metrics HTTP/1.1" 200 36744 "" "Prometheus/2.43.0" 2024-09-13T00:47:20.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:19 smithi136 bash[75164]: cluster 2024-09-13T00:47:18.622135+0000 mgr.y (mgr.24661) 2105 : cluster [DBG] pgmap v4058: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 715 KiB/s rd, 5.0 KiB/s wr, 967 op/s 2024-09-13T00:47:20.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:19 smithi136 bash[75164]: cluster 2024-09-13T00:47:18.958092+0000 mon.a (mon.0) 4152 : cluster [DBG] osdmap e2393: 8 total, 8 up, 8 in 2024-09-13T00:47:20.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:19 smithi136 bash[75164]: cluster 2024-09-13T00:47:19.928057+0000 mon.a (mon.0) 4153 : cluster [DBG] osdmap e2394: 8 total, 8 up, 8 in 2024-09-13T00:47:20.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:19 smithi136 bash[76371]: cluster 2024-09-13T00:47:18.622135+0000 mgr.y (mgr.24661) 2105 : cluster [DBG] pgmap v4058: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 715 KiB/s rd, 5.0 KiB/s wr, 967 op/s 2024-09-13T00:47:20.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:19 smithi136 bash[76371]: cluster 2024-09-13T00:47:18.958092+0000 mon.a (mon.0) 4152 : cluster [DBG] osdmap e2393: 8 total, 8 up, 8 in 2024-09-13T00:47:20.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:19 smithi136 bash[76371]: cluster 2024-09-13T00:47:19.928057+0000 mon.a (mon.0) 4153 : cluster [DBG] osdmap e2394: 8 total, 8 up, 8 in 2024-09-13T00:47:20.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:19 smithi181 bash[67587]: cluster 2024-09-13T00:47:18.622135+0000 mgr.y (mgr.24661) 2105 : cluster [DBG] pgmap v4058: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 715 KiB/s rd, 5.0 KiB/s wr, 967 op/s 2024-09-13T00:47:20.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:19 smithi181 bash[67587]: cluster 2024-09-13T00:47:18.958092+0000 mon.a (mon.0) 4152 : cluster [DBG] osdmap e2393: 8 total, 8 up, 8 in 2024-09-13T00:47:20.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:19 smithi181 bash[67587]: cluster 2024-09-13T00:47:19.928057+0000 mon.a (mon.0) 4153 : cluster [DBG] osdmap e2394: 8 total, 8 up, 8 in 2024-09-13T00:47:22.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:21 smithi136 bash[75164]: cluster 2024-09-13T00:47:20.623010+0000 mgr.y (mgr.24661) 2106 : cluster [DBG] pgmap v4061: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 714 KiB/s rd, 5.0 KiB/s wr, 967 op/s 2024-09-13T00:47:22.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:21 smithi136 bash[75164]: cluster 2024-09-13T00:47:20.968847+0000 mon.a (mon.0) 4154 : cluster [DBG] osdmap e2395: 8 total, 8 up, 8 in 2024-09-13T00:47:22.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:21 smithi136 bash[76371]: cluster 2024-09-13T00:47:20.623010+0000 mgr.y (mgr.24661) 2106 : cluster [DBG] pgmap v4061: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 714 KiB/s rd, 5.0 KiB/s wr, 967 op/s 2024-09-13T00:47:22.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:21 smithi136 bash[76371]: cluster 2024-09-13T00:47:20.968847+0000 mon.a (mon.0) 4154 : cluster [DBG] osdmap e2395: 8 total, 8 up, 8 in 2024-09-13T00:47:22.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:21 smithi181 bash[67587]: cluster 2024-09-13T00:47:20.623010+0000 mgr.y (mgr.24661) 2106 : cluster [DBG] pgmap v4061: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 714 KiB/s rd, 5.0 KiB/s wr, 967 op/s 2024-09-13T00:47:22.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:21 smithi181 bash[67587]: cluster 2024-09-13T00:47:20.968847+0000 mon.a (mon.0) 4154 : cluster [DBG] osdmap e2395: 8 total, 8 up, 8 in 2024-09-13T00:47:22.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:47:22 smithi181 bash[95505]: ts=2024-09-13T00:47:22.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:47:23.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:22 smithi136 bash[75164]: cluster 2024-09-13T00:47:21.972185+0000 mon.a (mon.0) 4155 : cluster [DBG] osdmap e2396: 8 total, 8 up, 8 in 2024-09-13T00:47:23.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:22 smithi136 bash[76371]: cluster 2024-09-13T00:47:21.972185+0000 mon.a (mon.0) 4155 : cluster [DBG] osdmap e2396: 8 total, 8 up, 8 in 2024-09-13T00:47:23.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:22 smithi181 bash[67587]: cluster 2024-09-13T00:47:21.972185+0000 mon.a (mon.0) 4155 : cluster [DBG] osdmap e2396: 8 total, 8 up, 8 in 2024-09-13T00:47:24.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:24 smithi181 bash[67587]: cluster 2024-09-13T00:47:22.623734+0000 mgr.y (mgr.24661) 2107 : cluster [DBG] pgmap v4064: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 682 KiB/s rd, 5.0 KiB/s wr, 923 op/s 2024-09-13T00:47:24.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:24 smithi181 bash[67587]: cluster 2024-09-13T00:47:23.001392+0000 mon.a (mon.0) 4156 : cluster [DBG] osdmap e2397: 8 total, 8 up, 8 in 2024-09-13T00:47:24.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:24 smithi181 bash[67587]: audit 2024-09-13T00:47:23.850457+0000 mon.a (mon.0) 4157 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:47:24.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:24 smithi136 bash[75164]: cluster 2024-09-13T00:47:22.623734+0000 mgr.y (mgr.24661) 2107 : cluster [DBG] pgmap v4064: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 682 KiB/s rd, 5.0 KiB/s wr, 923 op/s 2024-09-13T00:47:24.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:24 smithi136 bash[75164]: cluster 2024-09-13T00:47:23.001392+0000 mon.a (mon.0) 4156 : cluster [DBG] osdmap e2397: 8 total, 8 up, 8 in 2024-09-13T00:47:24.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:24 smithi136 bash[75164]: audit 2024-09-13T00:47:23.850457+0000 mon.a (mon.0) 4157 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:47:24.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:24 smithi136 bash[76371]: cluster 2024-09-13T00:47:22.623734+0000 mgr.y (mgr.24661) 2107 : cluster [DBG] pgmap v4064: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 682 KiB/s rd, 5.0 KiB/s wr, 923 op/s 2024-09-13T00:47:24.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:24 smithi136 bash[76371]: cluster 2024-09-13T00:47:23.001392+0000 mon.a (mon.0) 4156 : cluster [DBG] osdmap e2397: 8 total, 8 up, 8 in 2024-09-13T00:47:24.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:24 smithi136 bash[76371]: audit 2024-09-13T00:47:23.850457+0000 mon.a (mon.0) 4157 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:47:25.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:25 smithi181 bash[67587]: cluster 2024-09-13T00:47:24.004598+0000 mon.a (mon.0) 4158 : cluster [DBG] osdmap e2398: 8 total, 8 up, 8 in 2024-09-13T00:47:25.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:25 smithi181 bash[67587]: cluster 2024-09-13T00:47:24.930514+0000 mon.a (mon.0) 4159 : cluster [DBG] osdmap e2399: 8 total, 8 up, 8 in 2024-09-13T00:47:25.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:25 smithi136 bash[75164]: cluster 2024-09-13T00:47:24.004598+0000 mon.a (mon.0) 4158 : cluster [DBG] osdmap e2398: 8 total, 8 up, 8 in 2024-09-13T00:47:25.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:25 smithi136 bash[75164]: cluster 2024-09-13T00:47:24.930514+0000 mon.a (mon.0) 4159 : cluster [DBG] osdmap e2399: 8 total, 8 up, 8 in 2024-09-13T00:47:25.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:25 smithi136 bash[76371]: cluster 2024-09-13T00:47:24.004598+0000 mon.a (mon.0) 4158 : cluster [DBG] osdmap e2398: 8 total, 8 up, 8 in 2024-09-13T00:47:25.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:25 smithi136 bash[76371]: cluster 2024-09-13T00:47:24.930514+0000 mon.a (mon.0) 4159 : cluster [DBG] osdmap e2399: 8 total, 8 up, 8 in 2024-09-13T00:47:26.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:26 smithi181 bash[67587]: cluster 2024-09-13T00:47:24.624412+0000 mgr.y (mgr.24661) 2108 : cluster [DBG] pgmap v4067: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 682 KiB/s rd, 5.0 KiB/s wr, 922 op/s 2024-09-13T00:47:26.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:26 smithi136 bash[76371]: cluster 2024-09-13T00:47:24.624412+0000 mgr.y (mgr.24661) 2108 : cluster [DBG] pgmap v4067: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 682 KiB/s rd, 5.0 KiB/s wr, 922 op/s 2024-09-13T00:47:26.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:26 smithi136 bash[75164]: cluster 2024-09-13T00:47:24.624412+0000 mgr.y (mgr.24661) 2108 : cluster [DBG] pgmap v4067: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 682 KiB/s rd, 5.0 KiB/s wr, 922 op/s 2024-09-13T00:47:27.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:27 smithi181 bash[67587]: cluster 2024-09-13T00:47:26.038053+0000 mon.a (mon.0) 4160 : cluster [DBG] osdmap e2400: 8 total, 8 up, 8 in 2024-09-13T00:47:27.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:27 smithi136 bash[75164]: cluster 2024-09-13T00:47:26.038053+0000 mon.a (mon.0) 4160 : cluster [DBG] osdmap e2400: 8 total, 8 up, 8 in 2024-09-13T00:47:27.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:27 smithi136 bash[76371]: cluster 2024-09-13T00:47:26.038053+0000 mon.a (mon.0) 4160 : cluster [DBG] osdmap e2400: 8 total, 8 up, 8 in 2024-09-13T00:47:28.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:28 smithi181 bash[67587]: cluster 2024-09-13T00:47:26.625215+0000 mgr.y (mgr.24661) 2109 : cluster [DBG] pgmap v4070: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 2.7 KiB/s rd, 0 B/s wr, 4 op/s 2024-09-13T00:47:28.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:28 smithi181 bash[67587]: cluster 2024-09-13T00:47:27.050060+0000 mon.a (mon.0) 4161 : cluster [DBG] osdmap e2401: 8 total, 8 up, 8 in 2024-09-13T00:47:28.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:28 smithi136 bash[75164]: cluster 2024-09-13T00:47:26.625215+0000 mgr.y (mgr.24661) 2109 : cluster [DBG] pgmap v4070: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 2.7 KiB/s rd, 0 B/s wr, 4 op/s 2024-09-13T00:47:28.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:28 smithi136 bash[75164]: cluster 2024-09-13T00:47:27.050060+0000 mon.a (mon.0) 4161 : cluster [DBG] osdmap e2401: 8 total, 8 up, 8 in 2024-09-13T00:47:28.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:28 smithi136 bash[76371]: cluster 2024-09-13T00:47:26.625215+0000 mgr.y (mgr.24661) 2109 : cluster [DBG] pgmap v4070: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 2.7 KiB/s rd, 0 B/s wr, 4 op/s 2024-09-13T00:47:28.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:28 smithi136 bash[76371]: cluster 2024-09-13T00:47:27.050060+0000 mon.a (mon.0) 4161 : cluster [DBG] osdmap e2401: 8 total, 8 up, 8 in 2024-09-13T00:47:29.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:29 smithi181 bash[67587]: cluster 2024-09-13T00:47:28.052236+0000 mon.a (mon.0) 4162 : cluster [DBG] osdmap e2402: 8 total, 8 up, 8 in 2024-09-13T00:47:29.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:29 smithi136 bash[75164]: cluster 2024-09-13T00:47:28.052236+0000 mon.a (mon.0) 4162 : cluster [DBG] osdmap e2402: 8 total, 8 up, 8 in 2024-09-13T00:47:29.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:29 smithi136 bash[76371]: cluster 2024-09-13T00:47:28.052236+0000 mon.a (mon.0) 4162 : cluster [DBG] osdmap e2402: 8 total, 8 up, 8 in 2024-09-13T00:47:29.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:47:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:47:29] "GET /metrics HTTP/1.1" 200 36747 "" "Prometheus/2.43.0" 2024-09-13T00:47:30.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:30 smithi181 bash[67587]: cluster 2024-09-13T00:47:28.626102+0000 mgr.y (mgr.24661) 2110 : cluster [DBG] pgmap v4073: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 648 KiB/s rd, 5.0 KiB/s wr, 878 op/s 2024-09-13T00:47:30.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:30 smithi181 bash[67587]: cluster 2024-09-13T00:47:29.076264+0000 mon.a (mon.0) 4163 : cluster [DBG] osdmap e2403: 8 total, 8 up, 8 in 2024-09-13T00:47:30.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:30 smithi181 bash[67587]: cluster 2024-09-13T00:47:29.933575+0000 mon.a (mon.0) 4164 : cluster [DBG] osdmap e2404: 8 total, 8 up, 8 in 2024-09-13T00:47:30.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:30 smithi136 bash[75164]: cluster 2024-09-13T00:47:28.626102+0000 mgr.y (mgr.24661) 2110 : cluster [DBG] pgmap v4073: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 648 KiB/s rd, 5.0 KiB/s wr, 878 op/s 2024-09-13T00:47:30.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:30 smithi136 bash[75164]: cluster 2024-09-13T00:47:29.076264+0000 mon.a (mon.0) 4163 : cluster [DBG] osdmap e2403: 8 total, 8 up, 8 in 2024-09-13T00:47:30.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:30 smithi136 bash[75164]: cluster 2024-09-13T00:47:29.933575+0000 mon.a (mon.0) 4164 : cluster [DBG] osdmap e2404: 8 total, 8 up, 8 in 2024-09-13T00:47:30.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:30 smithi136 bash[76371]: cluster 2024-09-13T00:47:28.626102+0000 mgr.y (mgr.24661) 2110 : cluster [DBG] pgmap v4073: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 648 KiB/s rd, 5.0 KiB/s wr, 878 op/s 2024-09-13T00:47:30.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:30 smithi136 bash[76371]: cluster 2024-09-13T00:47:29.076264+0000 mon.a (mon.0) 4163 : cluster [DBG] osdmap e2403: 8 total, 8 up, 8 in 2024-09-13T00:47:30.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:30 smithi136 bash[76371]: cluster 2024-09-13T00:47:29.933575+0000 mon.a (mon.0) 4164 : cluster [DBG] osdmap e2404: 8 total, 8 up, 8 in 2024-09-13T00:47:32.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:31 smithi136 bash[76371]: cluster 2024-09-13T00:47:30.627051+0000 mgr.y (mgr.24661) 2111 : cluster [DBG] pgmap v4076: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 649 KiB/s rd, 5.0 KiB/s wr, 879 op/s 2024-09-13T00:47:32.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:31 smithi136 bash[76371]: cluster 2024-09-13T00:47:30.936169+0000 mon.a (mon.0) 4165 : cluster [DBG] osdmap e2405: 8 total, 8 up, 8 in 2024-09-13T00:47:32.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:31 smithi136 bash[75164]: cluster 2024-09-13T00:47:30.627051+0000 mgr.y (mgr.24661) 2111 : cluster [DBG] pgmap v4076: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 649 KiB/s rd, 5.0 KiB/s wr, 879 op/s 2024-09-13T00:47:32.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:31 smithi136 bash[75164]: cluster 2024-09-13T00:47:30.936169+0000 mon.a (mon.0) 4165 : cluster [DBG] osdmap e2405: 8 total, 8 up, 8 in 2024-09-13T00:47:32.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:31 smithi181 bash[67587]: cluster 2024-09-13T00:47:30.627051+0000 mgr.y (mgr.24661) 2111 : cluster [DBG] pgmap v4076: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 649 KiB/s rd, 5.0 KiB/s wr, 879 op/s 2024-09-13T00:47:32.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:31 smithi181 bash[67587]: cluster 2024-09-13T00:47:30.936169+0000 mon.a (mon.0) 4165 : cluster [DBG] osdmap e2405: 8 total, 8 up, 8 in 2024-09-13T00:47:32.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:47:32 smithi181 bash[95505]: ts=2024-09-13T00:47:32.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:47:33.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:32 smithi136 bash[76371]: cluster 2024-09-13T00:47:31.945432+0000 mon.a (mon.0) 4166 : cluster [DBG] osdmap e2406: 8 total, 8 up, 8 in 2024-09-13T00:47:33.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:32 smithi136 bash[75164]: cluster 2024-09-13T00:47:31.945432+0000 mon.a (mon.0) 4166 : cluster [DBG] osdmap e2406: 8 total, 8 up, 8 in 2024-09-13T00:47:33.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:32 smithi181 bash[67587]: cluster 2024-09-13T00:47:31.945432+0000 mon.a (mon.0) 4166 : cluster [DBG] osdmap e2406: 8 total, 8 up, 8 in 2024-09-13T00:47:34.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:33 smithi136 bash[75164]: cluster 2024-09-13T00:47:32.627785+0000 mgr.y (mgr.24661) 2112 : cluster [DBG] pgmap v4079: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 616 KiB/s rd, 5.0 KiB/s wr, 836 op/s 2024-09-13T00:47:34.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:33 smithi136 bash[75164]: cluster 2024-09-13T00:47:32.957876+0000 mon.a (mon.0) 4167 : cluster [DBG] osdmap e2407: 8 total, 8 up, 8 in 2024-09-13T00:47:34.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:33 smithi136 bash[76371]: cluster 2024-09-13T00:47:32.627785+0000 mgr.y (mgr.24661) 2112 : cluster [DBG] pgmap v4079: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 616 KiB/s rd, 5.0 KiB/s wr, 836 op/s 2024-09-13T00:47:34.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:33 smithi136 bash[76371]: cluster 2024-09-13T00:47:32.957876+0000 mon.a (mon.0) 4167 : cluster [DBG] osdmap e2407: 8 total, 8 up, 8 in 2024-09-13T00:47:34.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:33 smithi181 bash[67587]: cluster 2024-09-13T00:47:32.627785+0000 mgr.y (mgr.24661) 2112 : cluster [DBG] pgmap v4079: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 616 KiB/s rd, 5.0 KiB/s wr, 836 op/s 2024-09-13T00:47:34.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:33 smithi181 bash[67587]: cluster 2024-09-13T00:47:32.957876+0000 mon.a (mon.0) 4167 : cluster [DBG] osdmap e2407: 8 total, 8 up, 8 in 2024-09-13T00:47:35.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:34 smithi136 bash[75164]: cluster 2024-09-13T00:47:33.964390+0000 mon.a (mon.0) 4168 : cluster [DBG] osdmap e2408: 8 total, 8 up, 8 in 2024-09-13T00:47:35.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:34 smithi136 bash[75164]: cluster 2024-09-13T00:47:34.936700+0000 mon.a (mon.0) 4169 : cluster [DBG] osdmap e2409: 8 total, 8 up, 8 in 2024-09-13T00:47:35.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:34 smithi136 bash[76371]: cluster 2024-09-13T00:47:33.964390+0000 mon.a (mon.0) 4168 : cluster [DBG] osdmap e2408: 8 total, 8 up, 8 in 2024-09-13T00:47:35.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:34 smithi136 bash[76371]: cluster 2024-09-13T00:47:34.936700+0000 mon.a (mon.0) 4169 : cluster [DBG] osdmap e2409: 8 total, 8 up, 8 in 2024-09-13T00:47:35.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:34 smithi181 bash[67587]: cluster 2024-09-13T00:47:33.964390+0000 mon.a (mon.0) 4168 : cluster [DBG] osdmap e2408: 8 total, 8 up, 8 in 2024-09-13T00:47:35.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:34 smithi181 bash[67587]: cluster 2024-09-13T00:47:34.936700+0000 mon.a (mon.0) 4169 : cluster [DBG] osdmap e2409: 8 total, 8 up, 8 in 2024-09-13T00:47:36.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:35 smithi136 bash[75164]: cluster 2024-09-13T00:47:34.628578+0000 mgr.y (mgr.24661) 2113 : cluster [DBG] pgmap v4082: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 616 KiB/s rd, 5.0 KiB/s wr, 834 op/s 2024-09-13T00:47:36.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:35 smithi136 bash[76371]: cluster 2024-09-13T00:47:34.628578+0000 mgr.y (mgr.24661) 2113 : cluster [DBG] pgmap v4082: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 616 KiB/s rd, 5.0 KiB/s wr, 834 op/s 2024-09-13T00:47:36.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:35 smithi181 bash[67587]: cluster 2024-09-13T00:47:34.628578+0000 mgr.y (mgr.24661) 2113 : cluster [DBG] pgmap v4082: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 616 KiB/s rd, 5.0 KiB/s wr, 834 op/s 2024-09-13T00:47:37.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:37 smithi136 bash[75164]: cluster 2024-09-13T00:47:36.001833+0000 mon.a (mon.0) 4170 : cluster [DBG] osdmap e2410: 8 total, 8 up, 8 in 2024-09-13T00:47:37.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:37 smithi136 bash[76371]: cluster 2024-09-13T00:47:36.001833+0000 mon.a (mon.0) 4170 : cluster [DBG] osdmap e2410: 8 total, 8 up, 8 in 2024-09-13T00:47:37.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:37 smithi181 bash[67587]: cluster 2024-09-13T00:47:36.001833+0000 mon.a (mon.0) 4170 : cluster [DBG] osdmap e2410: 8 total, 8 up, 8 in 2024-09-13T00:47:38.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:38 smithi136 bash[75164]: cluster 2024-09-13T00:47:36.629417+0000 mgr.y (mgr.24661) 2114 : cluster [DBG] pgmap v4085: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 3.0 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T00:47:38.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:38 smithi136 bash[75164]: cluster 2024-09-13T00:47:37.010300+0000 mon.a (mon.0) 4171 : cluster [DBG] osdmap e2411: 8 total, 8 up, 8 in 2024-09-13T00:47:38.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:38 smithi136 bash[76371]: cluster 2024-09-13T00:47:36.629417+0000 mgr.y (mgr.24661) 2114 : cluster [DBG] pgmap v4085: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 3.0 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T00:47:38.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:38 smithi136 bash[76371]: cluster 2024-09-13T00:47:37.010300+0000 mon.a (mon.0) 4171 : cluster [DBG] osdmap e2411: 8 total, 8 up, 8 in 2024-09-13T00:47:38.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:38 smithi181 bash[67587]: cluster 2024-09-13T00:47:36.629417+0000 mgr.y (mgr.24661) 2114 : cluster [DBG] pgmap v4085: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 3.0 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T00:47:38.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:38 smithi181 bash[67587]: cluster 2024-09-13T00:47:37.010300+0000 mon.a (mon.0) 4171 : cluster [DBG] osdmap e2411: 8 total, 8 up, 8 in 2024-09-13T00:47:39.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:39 smithi181 bash[67587]: cluster 2024-09-13T00:47:38.019231+0000 mon.a (mon.0) 4172 : cluster [DBG] osdmap e2412: 8 total, 8 up, 8 in 2024-09-13T00:47:39.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:39 smithi181 bash[67587]: audit 2024-09-13T00:47:38.850880+0000 mon.a (mon.0) 4173 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:47:39.412 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:39 smithi136 bash[76371]: cluster 2024-09-13T00:47:38.019231+0000 mon.a (mon.0) 4172 : cluster [DBG] osdmap e2412: 8 total, 8 up, 8 in 2024-09-13T00:47:39.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:39 smithi136 bash[76371]: audit 2024-09-13T00:47:38.850880+0000 mon.a (mon.0) 4173 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:47:39.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:39 smithi136 bash[75164]: cluster 2024-09-13T00:47:38.019231+0000 mon.a (mon.0) 4172 : cluster [DBG] osdmap e2412: 8 total, 8 up, 8 in 2024-09-13T00:47:39.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:39 smithi136 bash[75164]: audit 2024-09-13T00:47:38.850880+0000 mon.a (mon.0) 4173 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:47:39.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:47:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:47:39] "GET /metrics HTTP/1.1" 200 36743 "" "Prometheus/2.43.0" 2024-09-13T00:47:40.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:40 smithi181 bash[67587]: cluster 2024-09-13T00:47:38.630303+0000 mgr.y (mgr.24661) 2115 : cluster [DBG] pgmap v4088: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 586 KiB/s rd, 5.0 KiB/s wr, 795 op/s 2024-09-13T00:47:40.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:40 smithi181 bash[67587]: cluster 2024-09-13T00:47:39.029332+0000 mon.a (mon.0) 4174 : cluster [DBG] osdmap e2413: 8 total, 8 up, 8 in 2024-09-13T00:47:40.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:40 smithi181 bash[67587]: cluster 2024-09-13T00:47:39.940811+0000 mon.a (mon.0) 4175 : cluster [DBG] osdmap e2414: 8 total, 8 up, 8 in 2024-09-13T00:47:40.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:40 smithi136 bash[75164]: cluster 2024-09-13T00:47:38.630303+0000 mgr.y (mgr.24661) 2115 : cluster [DBG] pgmap v4088: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 586 KiB/s rd, 5.0 KiB/s wr, 795 op/s 2024-09-13T00:47:40.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:40 smithi136 bash[75164]: cluster 2024-09-13T00:47:39.029332+0000 mon.a (mon.0) 4174 : cluster [DBG] osdmap e2413: 8 total, 8 up, 8 in 2024-09-13T00:47:40.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:40 smithi136 bash[75164]: cluster 2024-09-13T00:47:39.940811+0000 mon.a (mon.0) 4175 : cluster [DBG] osdmap e2414: 8 total, 8 up, 8 in 2024-09-13T00:47:40.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:40 smithi136 bash[76371]: cluster 2024-09-13T00:47:38.630303+0000 mgr.y (mgr.24661) 2115 : cluster [DBG] pgmap v4088: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 586 KiB/s rd, 5.0 KiB/s wr, 795 op/s 2024-09-13T00:47:40.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:40 smithi136 bash[76371]: cluster 2024-09-13T00:47:39.029332+0000 mon.a (mon.0) 4174 : cluster [DBG] osdmap e2413: 8 total, 8 up, 8 in 2024-09-13T00:47:40.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:40 smithi136 bash[76371]: cluster 2024-09-13T00:47:39.940811+0000 mon.a (mon.0) 4175 : cluster [DBG] osdmap e2414: 8 total, 8 up, 8 in 2024-09-13T00:47:42.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:42 smithi181 bash[67587]: cluster 2024-09-13T00:47:40.631291+0000 mgr.y (mgr.24661) 2116 : cluster [DBG] pgmap v4091: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 584 KiB/s rd, 5.0 KiB/s wr, 792 op/s 2024-09-13T00:47:42.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:42 smithi181 bash[67587]: cluster 2024-09-13T00:47:41.037269+0000 mon.a (mon.0) 4176 : cluster [DBG] osdmap e2415: 8 total, 8 up, 8 in 2024-09-13T00:47:42.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:42 smithi136 bash[75164]: cluster 2024-09-13T00:47:40.631291+0000 mgr.y (mgr.24661) 2116 : cluster [DBG] pgmap v4091: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 584 KiB/s rd, 5.0 KiB/s wr, 792 op/s 2024-09-13T00:47:42.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:42 smithi136 bash[75164]: cluster 2024-09-13T00:47:41.037269+0000 mon.a (mon.0) 4176 : cluster [DBG] osdmap e2415: 8 total, 8 up, 8 in 2024-09-13T00:47:42.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:42 smithi136 bash[76371]: cluster 2024-09-13T00:47:40.631291+0000 mgr.y (mgr.24661) 2116 : cluster [DBG] pgmap v4091: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 584 KiB/s rd, 5.0 KiB/s wr, 792 op/s 2024-09-13T00:47:42.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:42 smithi136 bash[76371]: cluster 2024-09-13T00:47:41.037269+0000 mon.a (mon.0) 4176 : cluster [DBG] osdmap e2415: 8 total, 8 up, 8 in 2024-09-13T00:47:42.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:47:42 smithi181 bash[95505]: ts=2024-09-13T00:47:42.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:47:43.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:43 smithi181 bash[67587]: cluster 2024-09-13T00:47:42.052092+0000 mon.a (mon.0) 4177 : cluster [DBG] osdmap e2416: 8 total, 8 up, 8 in 2024-09-13T00:47:43.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:43 smithi136 bash[75164]: cluster 2024-09-13T00:47:42.052092+0000 mon.a (mon.0) 4177 : cluster [DBG] osdmap e2416: 8 total, 8 up, 8 in 2024-09-13T00:47:43.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:43 smithi136 bash[76371]: cluster 2024-09-13T00:47:42.052092+0000 mon.a (mon.0) 4177 : cluster [DBG] osdmap e2416: 8 total, 8 up, 8 in 2024-09-13T00:47:44.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:44 smithi181 bash[67587]: cluster 2024-09-13T00:47:42.632003+0000 mgr.y (mgr.24661) 2117 : cluster [DBG] pgmap v4094: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 550 KiB/s rd, 5.0 KiB/s wr, 747 op/s 2024-09-13T00:47:44.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:44 smithi181 bash[67587]: cluster 2024-09-13T00:47:43.060512+0000 mon.a (mon.0) 4178 : cluster [DBG] osdmap e2417: 8 total, 8 up, 8 in 2024-09-13T00:47:44.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:44 smithi136 bash[76371]: cluster 2024-09-13T00:47:42.632003+0000 mgr.y (mgr.24661) 2117 : cluster [DBG] pgmap v4094: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 550 KiB/s rd, 5.0 KiB/s wr, 747 op/s 2024-09-13T00:47:44.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:44 smithi136 bash[76371]: cluster 2024-09-13T00:47:43.060512+0000 mon.a (mon.0) 4178 : cluster [DBG] osdmap e2417: 8 total, 8 up, 8 in 2024-09-13T00:47:44.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:44 smithi136 bash[75164]: cluster 2024-09-13T00:47:42.632003+0000 mgr.y (mgr.24661) 2117 : cluster [DBG] pgmap v4094: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 550 KiB/s rd, 5.0 KiB/s wr, 747 op/s 2024-09-13T00:47:44.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:44 smithi136 bash[75164]: cluster 2024-09-13T00:47:43.060512+0000 mon.a (mon.0) 4178 : cluster [DBG] osdmap e2417: 8 total, 8 up, 8 in 2024-09-13T00:47:45.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:45 smithi181 bash[67587]: cluster 2024-09-13T00:47:44.062807+0000 mon.a (mon.0) 4179 : cluster [DBG] osdmap e2418: 8 total, 8 up, 8 in 2024-09-13T00:47:45.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:45 smithi181 bash[67587]: cluster 2024-09-13T00:47:44.944498+0000 mon.a (mon.0) 4180 : cluster [DBG] osdmap e2419: 8 total, 8 up, 8 in 2024-09-13T00:47:45.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:45 smithi136 bash[75164]: cluster 2024-09-13T00:47:44.062807+0000 mon.a (mon.0) 4179 : cluster [DBG] osdmap e2418: 8 total, 8 up, 8 in 2024-09-13T00:47:45.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:45 smithi136 bash[75164]: cluster 2024-09-13T00:47:44.944498+0000 mon.a (mon.0) 4180 : cluster [DBG] osdmap e2419: 8 total, 8 up, 8 in 2024-09-13T00:47:45.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:45 smithi136 bash[76371]: cluster 2024-09-13T00:47:44.062807+0000 mon.a (mon.0) 4179 : cluster [DBG] osdmap e2418: 8 total, 8 up, 8 in 2024-09-13T00:47:45.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:45 smithi136 bash[76371]: cluster 2024-09-13T00:47:44.944498+0000 mon.a (mon.0) 4180 : cluster [DBG] osdmap e2419: 8 total, 8 up, 8 in 2024-09-13T00:47:46.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:46 smithi181 bash[67587]: cluster 2024-09-13T00:47:44.632764+0000 mgr.y (mgr.24661) 2118 : cluster [DBG] pgmap v4097: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 552 KiB/s rd, 5.0 KiB/s wr, 748 op/s 2024-09-13T00:47:46.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:46 smithi181 bash[67587]: cluster 2024-09-13T00:47:45.945924+0000 mon.a (mon.0) 4181 : cluster [DBG] osdmap e2420: 8 total, 8 up, 8 in 2024-09-13T00:47:46.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:46 smithi136 bash[75164]: cluster 2024-09-13T00:47:44.632764+0000 mgr.y (mgr.24661) 2118 : cluster [DBG] pgmap v4097: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 552 KiB/s rd, 5.0 KiB/s wr, 748 op/s 2024-09-13T00:47:46.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:46 smithi136 bash[75164]: cluster 2024-09-13T00:47:45.945924+0000 mon.a (mon.0) 4181 : cluster [DBG] osdmap e2420: 8 total, 8 up, 8 in 2024-09-13T00:47:46.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:46 smithi136 bash[76371]: cluster 2024-09-13T00:47:44.632764+0000 mgr.y (mgr.24661) 2118 : cluster [DBG] pgmap v4097: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 552 KiB/s rd, 5.0 KiB/s wr, 748 op/s 2024-09-13T00:47:46.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:46 smithi136 bash[76371]: cluster 2024-09-13T00:47:45.945924+0000 mon.a (mon.0) 4181 : cluster [DBG] osdmap e2420: 8 total, 8 up, 8 in 2024-09-13T00:47:48.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:47 smithi136 bash[76371]: cluster 2024-09-13T00:47:46.633618+0000 mgr.y (mgr.24661) 2119 : cluster [DBG] pgmap v4100: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 2.7 KiB/s rd, 0 B/s wr, 4 op/s 2024-09-13T00:47:48.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:47 smithi136 bash[76371]: cluster 2024-09-13T00:47:46.949478+0000 mon.a (mon.0) 4182 : cluster [DBG] osdmap e2421: 8 total, 8 up, 8 in 2024-09-13T00:47:48.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:47 smithi136 bash[75164]: cluster 2024-09-13T00:47:46.633618+0000 mgr.y (mgr.24661) 2119 : cluster [DBG] pgmap v4100: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 2.7 KiB/s rd, 0 B/s wr, 4 op/s 2024-09-13T00:47:48.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:47 smithi136 bash[75164]: cluster 2024-09-13T00:47:46.949478+0000 mon.a (mon.0) 4182 : cluster [DBG] osdmap e2421: 8 total, 8 up, 8 in 2024-09-13T00:47:48.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:47 smithi181 bash[67587]: cluster 2024-09-13T00:47:46.633618+0000 mgr.y (mgr.24661) 2119 : cluster [DBG] pgmap v4100: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 2.7 KiB/s rd, 0 B/s wr, 4 op/s 2024-09-13T00:47:48.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:47 smithi181 bash[67587]: cluster 2024-09-13T00:47:46.949478+0000 mon.a (mon.0) 4182 : cluster [DBG] osdmap e2421: 8 total, 8 up, 8 in 2024-09-13T00:47:49.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:48 smithi136 bash[75164]: cluster 2024-09-13T00:47:47.962439+0000 mon.a (mon.0) 4183 : cluster [DBG] osdmap e2422: 8 total, 8 up, 8 in 2024-09-13T00:47:49.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:48 smithi136 bash[76371]: cluster 2024-09-13T00:47:47.962439+0000 mon.a (mon.0) 4183 : cluster [DBG] osdmap e2422: 8 total, 8 up, 8 in 2024-09-13T00:47:49.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:48 smithi181 bash[67587]: cluster 2024-09-13T00:47:47.962439+0000 mon.a (mon.0) 4183 : cluster [DBG] osdmap e2422: 8 total, 8 up, 8 in 2024-09-13T00:47:49.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:47:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:47:49] "GET /metrics HTTP/1.1" 200 36743 "" "Prometheus/2.43.0" 2024-09-13T00:47:50.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:49 smithi136 bash[75164]: cluster 2024-09-13T00:47:48.634577+0000 mgr.y (mgr.24661) 2120 : cluster [DBG] pgmap v4103: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 521 KiB/s rd, 5.0 KiB/s wr, 706 op/s 2024-09-13T00:47:50.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:49 smithi136 bash[75164]: cluster 2024-09-13T00:47:48.971325+0000 mon.a (mon.0) 4184 : cluster [DBG] osdmap e2423: 8 total, 8 up, 8 in 2024-09-13T00:47:50.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:49 smithi136 bash[75164]: cluster 2024-09-13T00:47:49.945215+0000 mon.a (mon.0) 4185 : cluster [DBG] osdmap e2424: 8 total, 8 up, 8 in 2024-09-13T00:47:50.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:49 smithi136 bash[76371]: cluster 2024-09-13T00:47:48.634577+0000 mgr.y (mgr.24661) 2120 : cluster [DBG] pgmap v4103: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 521 KiB/s rd, 5.0 KiB/s wr, 706 op/s 2024-09-13T00:47:50.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:49 smithi136 bash[76371]: cluster 2024-09-13T00:47:48.971325+0000 mon.a (mon.0) 4184 : cluster [DBG] osdmap e2423: 8 total, 8 up, 8 in 2024-09-13T00:47:50.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:49 smithi136 bash[76371]: cluster 2024-09-13T00:47:49.945215+0000 mon.a (mon.0) 4185 : cluster [DBG] osdmap e2424: 8 total, 8 up, 8 in 2024-09-13T00:47:50.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:49 smithi181 bash[67587]: cluster 2024-09-13T00:47:48.634577+0000 mgr.y (mgr.24661) 2120 : cluster [DBG] pgmap v4103: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 521 KiB/s rd, 5.0 KiB/s wr, 706 op/s 2024-09-13T00:47:50.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:49 smithi181 bash[67587]: cluster 2024-09-13T00:47:48.971325+0000 mon.a (mon.0) 4184 : cluster [DBG] osdmap e2423: 8 total, 8 up, 8 in 2024-09-13T00:47:50.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:49 smithi181 bash[67587]: cluster 2024-09-13T00:47:49.945215+0000 mon.a (mon.0) 4185 : cluster [DBG] osdmap e2424: 8 total, 8 up, 8 in 2024-09-13T00:47:52.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:51 smithi136 bash[75164]: cluster 2024-09-13T00:47:50.635594+0000 mgr.y (mgr.24661) 2121 : cluster [DBG] pgmap v4106: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 520 KiB/s rd, 5.0 KiB/s wr, 705 op/s 2024-09-13T00:47:52.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:51 smithi136 bash[75164]: cluster 2024-09-13T00:47:50.980918+0000 mon.a (mon.0) 4186 : cluster [DBG] osdmap e2425: 8 total, 8 up, 8 in 2024-09-13T00:47:52.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:51 smithi136 bash[76371]: cluster 2024-09-13T00:47:50.635594+0000 mgr.y (mgr.24661) 2121 : cluster [DBG] pgmap v4106: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 520 KiB/s rd, 5.0 KiB/s wr, 705 op/s 2024-09-13T00:47:52.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:51 smithi136 bash[76371]: cluster 2024-09-13T00:47:50.980918+0000 mon.a (mon.0) 4186 : cluster [DBG] osdmap e2425: 8 total, 8 up, 8 in 2024-09-13T00:47:52.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:51 smithi181 bash[67587]: cluster 2024-09-13T00:47:50.635594+0000 mgr.y (mgr.24661) 2121 : cluster [DBG] pgmap v4106: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 520 KiB/s rd, 5.0 KiB/s wr, 705 op/s 2024-09-13T00:47:52.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:51 smithi181 bash[67587]: cluster 2024-09-13T00:47:50.980918+0000 mon.a (mon.0) 4186 : cluster [DBG] osdmap e2425: 8 total, 8 up, 8 in 2024-09-13T00:47:52.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:47:52 smithi181 bash[95505]: ts=2024-09-13T00:47:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:47:53.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:53 smithi136 bash[75164]: cluster 2024-09-13T00:47:51.983162+0000 mon.a (mon.0) 4187 : cluster [DBG] osdmap e2426: 8 total, 8 up, 8 in 2024-09-13T00:47:53.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:53 smithi136 bash[76371]: cluster 2024-09-13T00:47:51.983162+0000 mon.a (mon.0) 4187 : cluster [DBG] osdmap e2426: 8 total, 8 up, 8 in 2024-09-13T00:47:53.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:53 smithi181 bash[67587]: cluster 2024-09-13T00:47:51.983162+0000 mon.a (mon.0) 4187 : cluster [DBG] osdmap e2426: 8 total, 8 up, 8 in 2024-09-13T00:47:54.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:54 smithi136 bash[75164]: cluster 2024-09-13T00:47:52.636307+0000 mgr.y (mgr.24661) 2122 : cluster [DBG] pgmap v4109: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 485 KiB/s rd, 5.0 KiB/s wr, 657 op/s 2024-09-13T00:47:54.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:54 smithi136 bash[75164]: cluster 2024-09-13T00:47:53.012592+0000 mon.a (mon.0) 4188 : cluster [DBG] osdmap e2427: 8 total, 8 up, 8 in 2024-09-13T00:47:54.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:54 smithi136 bash[75164]: audit 2024-09-13T00:47:53.851212+0000 mon.a (mon.0) 4189 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:47:54.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:54 smithi136 bash[76371]: cluster 2024-09-13T00:47:52.636307+0000 mgr.y (mgr.24661) 2122 : cluster [DBG] pgmap v4109: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 485 KiB/s rd, 5.0 KiB/s wr, 657 op/s 2024-09-13T00:47:54.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:54 smithi136 bash[76371]: cluster 2024-09-13T00:47:53.012592+0000 mon.a (mon.0) 4188 : cluster [DBG] osdmap e2427: 8 total, 8 up, 8 in 2024-09-13T00:47:54.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:54 smithi136 bash[76371]: audit 2024-09-13T00:47:53.851212+0000 mon.a (mon.0) 4189 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:47:54.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:54 smithi181 bash[67587]: cluster 2024-09-13T00:47:52.636307+0000 mgr.y (mgr.24661) 2122 : cluster [DBG] pgmap v4109: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 485 KiB/s rd, 5.0 KiB/s wr, 657 op/s 2024-09-13T00:47:54.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:54 smithi181 bash[67587]: cluster 2024-09-13T00:47:53.012592+0000 mon.a (mon.0) 4188 : cluster [DBG] osdmap e2427: 8 total, 8 up, 8 in 2024-09-13T00:47:54.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:54 smithi181 bash[67587]: audit 2024-09-13T00:47:53.851212+0000 mon.a (mon.0) 4189 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:47:55.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:55 smithi181 bash[67587]: cluster 2024-09-13T00:47:54.018306+0000 mon.a (mon.0) 4190 : cluster [DBG] osdmap e2428: 8 total, 8 up, 8 in 2024-09-13T00:47:55.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:55 smithi181 bash[67587]: cluster 2024-09-13T00:47:54.947467+0000 mon.a (mon.0) 4191 : cluster [DBG] osdmap e2429: 8 total, 8 up, 8 in 2024-09-13T00:47:55.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:55 smithi136 bash[75164]: cluster 2024-09-13T00:47:54.018306+0000 mon.a (mon.0) 4190 : cluster [DBG] osdmap e2428: 8 total, 8 up, 8 in 2024-09-13T00:47:55.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:55 smithi136 bash[75164]: cluster 2024-09-13T00:47:54.947467+0000 mon.a (mon.0) 4191 : cluster [DBG] osdmap e2429: 8 total, 8 up, 8 in 2024-09-13T00:47:55.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:55 smithi136 bash[76371]: cluster 2024-09-13T00:47:54.018306+0000 mon.a (mon.0) 4190 : cluster [DBG] osdmap e2428: 8 total, 8 up, 8 in 2024-09-13T00:47:55.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:55 smithi136 bash[76371]: cluster 2024-09-13T00:47:54.947467+0000 mon.a (mon.0) 4191 : cluster [DBG] osdmap e2429: 8 total, 8 up, 8 in 2024-09-13T00:47:56.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:56 smithi181 bash[67587]: cluster 2024-09-13T00:47:54.637081+0000 mgr.y (mgr.24661) 2123 : cluster [DBG] pgmap v4112: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 487 KiB/s rd, 5.0 KiB/s wr, 660 op/s 2024-09-13T00:47:56.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:56 smithi136 bash[75164]: cluster 2024-09-13T00:47:54.637081+0000 mgr.y (mgr.24661) 2123 : cluster [DBG] pgmap v4112: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 487 KiB/s rd, 5.0 KiB/s wr, 660 op/s 2024-09-13T00:47:56.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:56 smithi136 bash[76371]: cluster 2024-09-13T00:47:54.637081+0000 mgr.y (mgr.24661) 2123 : cluster [DBG] pgmap v4112: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 487 KiB/s rd, 5.0 KiB/s wr, 660 op/s 2024-09-13T00:47:57.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:57 smithi181 bash[67587]: cluster 2024-09-13T00:47:56.037215+0000 mon.a (mon.0) 4192 : cluster [DBG] osdmap e2430: 8 total, 8 up, 8 in 2024-09-13T00:47:57.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:57 smithi136 bash[75164]: cluster 2024-09-13T00:47:56.037215+0000 mon.a (mon.0) 4192 : cluster [DBG] osdmap e2430: 8 total, 8 up, 8 in 2024-09-13T00:47:57.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:57 smithi136 bash[76371]: cluster 2024-09-13T00:47:56.037215+0000 mon.a (mon.0) 4192 : cluster [DBG] osdmap e2430: 8 total, 8 up, 8 in 2024-09-13T00:47:58.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:58 smithi181 bash[67587]: cluster 2024-09-13T00:47:56.637911+0000 mgr.y (mgr.24661) 2124 : cluster [DBG] pgmap v4115: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 4.0 KiB/s rd, 0 B/s wr, 7 op/s 2024-09-13T00:47:58.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:58 smithi181 bash[67587]: cluster 2024-09-13T00:47:57.047408+0000 mon.a (mon.0) 4193 : cluster [DBG] osdmap e2431: 8 total, 8 up, 8 in 2024-09-13T00:47:58.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:58 smithi136 bash[75164]: cluster 2024-09-13T00:47:56.637911+0000 mgr.y (mgr.24661) 2124 : cluster [DBG] pgmap v4115: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 4.0 KiB/s rd, 0 B/s wr, 7 op/s 2024-09-13T00:47:58.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:58 smithi136 bash[75164]: cluster 2024-09-13T00:47:57.047408+0000 mon.a (mon.0) 4193 : cluster [DBG] osdmap e2431: 8 total, 8 up, 8 in 2024-09-13T00:47:58.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:58 smithi136 bash[76371]: cluster 2024-09-13T00:47:56.637911+0000 mgr.y (mgr.24661) 2124 : cluster [DBG] pgmap v4115: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 4.0 KiB/s rd, 0 B/s wr, 7 op/s 2024-09-13T00:47:58.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:58 smithi136 bash[76371]: cluster 2024-09-13T00:47:57.047408+0000 mon.a (mon.0) 4193 : cluster [DBG] osdmap e2431: 8 total, 8 up, 8 in 2024-09-13T00:47:59.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:59 smithi181 bash[67587]: cluster 2024-09-13T00:47:58.048812+0000 mon.a (mon.0) 4194 : cluster [DBG] osdmap e2432: 8 total, 8 up, 8 in 2024-09-13T00:47:59.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:59 smithi181 bash[67587]: audit 2024-09-13T00:47:58.221591+0000 mon.a (mon.0) 4195 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:47:59.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:59 smithi181 bash[67587]: audit 2024-09-13T00:47:58.593366+0000 mon.a (mon.0) 4196 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:47:59.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:59 smithi181 bash[67587]: audit 2024-09-13T00:47:58.603131+0000 mon.a (mon.0) 4197 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:47:59.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:59 smithi181 bash[67587]: audit 2024-09-13T00:47:59.018651+0000 mon.a (mon.0) 4198 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:47:59.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:59 smithi181 bash[67587]: audit 2024-09-13T00:47:59.020187+0000 mon.a (mon.0) 4199 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:47:59.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:47:59 smithi181 bash[67587]: audit 2024-09-13T00:47:59.029958+0000 mon.a (mon.0) 4200 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:47:59.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:59 smithi136 bash[76371]: cluster 2024-09-13T00:47:58.048812+0000 mon.a (mon.0) 4194 : cluster [DBG] osdmap e2432: 8 total, 8 up, 8 in 2024-09-13T00:47:59.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:59 smithi136 bash[76371]: audit 2024-09-13T00:47:58.221591+0000 mon.a (mon.0) 4195 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:47:59.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:59 smithi136 bash[76371]: audit 2024-09-13T00:47:58.593366+0000 mon.a (mon.0) 4196 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:47:59.414 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:59 smithi136 bash[76371]: audit 2024-09-13T00:47:58.603131+0000 mon.a (mon.0) 4197 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:47:59.414 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:59 smithi136 bash[76371]: audit 2024-09-13T00:47:59.018651+0000 mon.a (mon.0) 4198 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:47:59.414 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:59 smithi136 bash[76371]: audit 2024-09-13T00:47:59.020187+0000 mon.a (mon.0) 4199 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:47:59.414 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:47:59 smithi136 bash[76371]: audit 2024-09-13T00:47:59.029958+0000 mon.a (mon.0) 4200 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:47:59.414 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:59 smithi136 bash[75164]: cluster 2024-09-13T00:47:58.048812+0000 mon.a (mon.0) 4194 : cluster [DBG] osdmap e2432: 8 total, 8 up, 8 in 2024-09-13T00:47:59.414 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:59 smithi136 bash[75164]: audit 2024-09-13T00:47:58.221591+0000 mon.a (mon.0) 4195 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:47:59.414 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:59 smithi136 bash[75164]: audit 2024-09-13T00:47:58.593366+0000 mon.a (mon.0) 4196 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:47:59.414 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:59 smithi136 bash[75164]: audit 2024-09-13T00:47:58.603131+0000 mon.a (mon.0) 4197 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:47:59.414 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:59 smithi136 bash[75164]: audit 2024-09-13T00:47:59.018651+0000 mon.a (mon.0) 4198 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:47:59.414 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:59 smithi136 bash[75164]: audit 2024-09-13T00:47:59.020187+0000 mon.a (mon.0) 4199 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:47:59.414 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:47:59 smithi136 bash[75164]: audit 2024-09-13T00:47:59.029958+0000 mon.a (mon.0) 4200 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:47:59.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:47:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:47:59] "GET /metrics HTTP/1.1" 200 36744 "" "Prometheus/2.43.0" 2024-09-13T00:48:00.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:00 smithi181 bash[67587]: cluster 2024-09-13T00:47:58.638499+0000 mgr.y (mgr.24661) 2125 : cluster [DBG] pgmap v4118: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 454 KiB/s rd, 5.0 KiB/s wr, 614 op/s 2024-09-13T00:48:00.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:00 smithi181 bash[67587]: cluster 2024-09-13T00:47:59.071659+0000 mon.a (mon.0) 4201 : cluster [DBG] osdmap e2433: 8 total, 8 up, 8 in 2024-09-13T00:48:00.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:00 smithi181 bash[67587]: cluster 2024-09-13T00:47:59.950536+0000 mon.a (mon.0) 4202 : cluster [DBG] osdmap e2434: 8 total, 8 up, 8 in 2024-09-13T00:48:00.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:00 smithi136 bash[75164]: cluster 2024-09-13T00:47:58.638499+0000 mgr.y (mgr.24661) 2125 : cluster [DBG] pgmap v4118: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 454 KiB/s rd, 5.0 KiB/s wr, 614 op/s 2024-09-13T00:48:00.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:00 smithi136 bash[75164]: cluster 2024-09-13T00:47:59.071659+0000 mon.a (mon.0) 4201 : cluster [DBG] osdmap e2433: 8 total, 8 up, 8 in 2024-09-13T00:48:00.511 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:00 smithi136 bash[75164]: cluster 2024-09-13T00:47:59.950536+0000 mon.a (mon.0) 4202 : cluster [DBG] osdmap e2434: 8 total, 8 up, 8 in 2024-09-13T00:48:00.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:00 smithi136 bash[76371]: cluster 2024-09-13T00:47:58.638499+0000 mgr.y (mgr.24661) 2125 : cluster [DBG] pgmap v4118: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 454 KiB/s rd, 5.0 KiB/s wr, 614 op/s 2024-09-13T00:48:00.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:00 smithi136 bash[76371]: cluster 2024-09-13T00:47:59.071659+0000 mon.a (mon.0) 4201 : cluster [DBG] osdmap e2433: 8 total, 8 up, 8 in 2024-09-13T00:48:00.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:00 smithi136 bash[76371]: cluster 2024-09-13T00:47:59.950536+0000 mon.a (mon.0) 4202 : cluster [DBG] osdmap e2434: 8 total, 8 up, 8 in 2024-09-13T00:48:02.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:01 smithi136 bash[75164]: cluster 2024-09-13T00:48:00.639550+0000 mgr.y (mgr.24661) 2126 : cluster [DBG] pgmap v4121: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 454 KiB/s rd, 5.0 KiB/s wr, 616 op/s 2024-09-13T00:48:02.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:01 smithi136 bash[75164]: cluster 2024-09-13T00:48:00.952571+0000 mon.a (mon.0) 4203 : cluster [DBG] osdmap e2435: 8 total, 8 up, 8 in 2024-09-13T00:48:02.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:01 smithi136 bash[76371]: cluster 2024-09-13T00:48:00.639550+0000 mgr.y (mgr.24661) 2126 : cluster [DBG] pgmap v4121: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 454 KiB/s rd, 5.0 KiB/s wr, 616 op/s 2024-09-13T00:48:02.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:01 smithi136 bash[76371]: cluster 2024-09-13T00:48:00.952571+0000 mon.a (mon.0) 4203 : cluster [DBG] osdmap e2435: 8 total, 8 up, 8 in 2024-09-13T00:48:02.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:01 smithi181 bash[67587]: cluster 2024-09-13T00:48:00.639550+0000 mgr.y (mgr.24661) 2126 : cluster [DBG] pgmap v4121: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 454 KiB/s rd, 5.0 KiB/s wr, 616 op/s 2024-09-13T00:48:02.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:01 smithi181 bash[67587]: cluster 2024-09-13T00:48:00.952571+0000 mon.a (mon.0) 4203 : cluster [DBG] osdmap e2435: 8 total, 8 up, 8 in 2024-09-13T00:48:02.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:48:02 smithi181 bash[95505]: ts=2024-09-13T00:48:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:48:03.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:02 smithi136 bash[75164]: cluster 2024-09-13T00:48:01.962719+0000 mon.a (mon.0) 4204 : cluster [DBG] osdmap e2436: 8 total, 8 up, 8 in 2024-09-13T00:48:03.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:02 smithi136 bash[76371]: cluster 2024-09-13T00:48:01.962719+0000 mon.a (mon.0) 4204 : cluster [DBG] osdmap e2436: 8 total, 8 up, 8 in 2024-09-13T00:48:03.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:02 smithi181 bash[67587]: cluster 2024-09-13T00:48:01.962719+0000 mon.a (mon.0) 4204 : cluster [DBG] osdmap e2436: 8 total, 8 up, 8 in 2024-09-13T00:48:04.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:03 smithi136 bash[75164]: cluster 2024-09-13T00:48:02.640292+0000 mgr.y (mgr.24661) 2127 : cluster [DBG] pgmap v4124: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 420 KiB/s rd, 5.0 KiB/s wr, 570 op/s 2024-09-13T00:48:04.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:03 smithi136 bash[75164]: cluster 2024-09-13T00:48:02.973890+0000 mon.a (mon.0) 4205 : cluster [DBG] osdmap e2437: 8 total, 8 up, 8 in 2024-09-13T00:48:04.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:03 smithi136 bash[76371]: cluster 2024-09-13T00:48:02.640292+0000 mgr.y (mgr.24661) 2127 : cluster [DBG] pgmap v4124: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 420 KiB/s rd, 5.0 KiB/s wr, 570 op/s 2024-09-13T00:48:04.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:03 smithi136 bash[76371]: cluster 2024-09-13T00:48:02.973890+0000 mon.a (mon.0) 4205 : cluster [DBG] osdmap e2437: 8 total, 8 up, 8 in 2024-09-13T00:48:04.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:03 smithi181 bash[67587]: cluster 2024-09-13T00:48:02.640292+0000 mgr.y (mgr.24661) 2127 : cluster [DBG] pgmap v4124: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 420 KiB/s rd, 5.0 KiB/s wr, 570 op/s 2024-09-13T00:48:04.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:03 smithi181 bash[67587]: cluster 2024-09-13T00:48:02.973890+0000 mon.a (mon.0) 4205 : cluster [DBG] osdmap e2437: 8 total, 8 up, 8 in 2024-09-13T00:48:05.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:04 smithi136 bash[75164]: cluster 2024-09-13T00:48:03.975845+0000 mon.a (mon.0) 4206 : cluster [DBG] osdmap e2438: 8 total, 8 up, 8 in 2024-09-13T00:48:05.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:04 smithi136 bash[75164]: cluster 2024-09-13T00:48:04.951389+0000 mon.a (mon.0) 4207 : cluster [DBG] osdmap e2439: 8 total, 8 up, 8 in 2024-09-13T00:48:05.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:04 smithi136 bash[76371]: cluster 2024-09-13T00:48:03.975845+0000 mon.a (mon.0) 4206 : cluster [DBG] osdmap e2438: 8 total, 8 up, 8 in 2024-09-13T00:48:05.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:04 smithi136 bash[76371]: cluster 2024-09-13T00:48:04.951389+0000 mon.a (mon.0) 4207 : cluster [DBG] osdmap e2439: 8 total, 8 up, 8 in 2024-09-13T00:48:05.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:04 smithi181 bash[67587]: cluster 2024-09-13T00:48:03.975845+0000 mon.a (mon.0) 4206 : cluster [DBG] osdmap e2438: 8 total, 8 up, 8 in 2024-09-13T00:48:05.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:04 smithi181 bash[67587]: cluster 2024-09-13T00:48:04.951389+0000 mon.a (mon.0) 4207 : cluster [DBG] osdmap e2439: 8 total, 8 up, 8 in 2024-09-13T00:48:06.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:06 smithi181 bash[67587]: cluster 2024-09-13T00:48:04.640996+0000 mgr.y (mgr.24661) 2128 : cluster [DBG] pgmap v4127: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 420 KiB/s rd, 5.0 KiB/s wr, 567 op/s 2024-09-13T00:48:06.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:06 smithi136 bash[75164]: cluster 2024-09-13T00:48:04.640996+0000 mgr.y (mgr.24661) 2128 : cluster [DBG] pgmap v4127: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 420 KiB/s rd, 5.0 KiB/s wr, 567 op/s 2024-09-13T00:48:06.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:06 smithi136 bash[76371]: cluster 2024-09-13T00:48:04.640996+0000 mgr.y (mgr.24661) 2128 : cluster [DBG] pgmap v4127: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 420 KiB/s rd, 5.0 KiB/s wr, 567 op/s 2024-09-13T00:48:07.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:07 smithi181 bash[67587]: cluster 2024-09-13T00:48:06.006318+0000 mon.a (mon.0) 4208 : cluster [DBG] osdmap e2440: 8 total, 8 up, 8 in 2024-09-13T00:48:07.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:07 smithi136 bash[75164]: cluster 2024-09-13T00:48:06.006318+0000 mon.a (mon.0) 4208 : cluster [DBG] osdmap e2440: 8 total, 8 up, 8 in 2024-09-13T00:48:07.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:07 smithi136 bash[76371]: cluster 2024-09-13T00:48:06.006318+0000 mon.a (mon.0) 4208 : cluster [DBG] osdmap e2440: 8 total, 8 up, 8 in 2024-09-13T00:48:08.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:08 smithi181 bash[67587]: cluster 2024-09-13T00:48:06.641764+0000 mgr.y (mgr.24661) 2129 : cluster [DBG] pgmap v4130: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 2.5 KiB/s rd, 0 B/s wr, 4 op/s 2024-09-13T00:48:08.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:08 smithi181 bash[67587]: cluster 2024-09-13T00:48:07.025308+0000 mon.a (mon.0) 4209 : cluster [DBG] osdmap e2441: 8 total, 8 up, 8 in 2024-09-13T00:48:08.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:08 smithi136 bash[75164]: cluster 2024-09-13T00:48:06.641764+0000 mgr.y (mgr.24661) 2129 : cluster [DBG] pgmap v4130: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 2.5 KiB/s rd, 0 B/s wr, 4 op/s 2024-09-13T00:48:08.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:08 smithi136 bash[75164]: cluster 2024-09-13T00:48:07.025308+0000 mon.a (mon.0) 4209 : cluster [DBG] osdmap e2441: 8 total, 8 up, 8 in 2024-09-13T00:48:08.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:08 smithi136 bash[76371]: cluster 2024-09-13T00:48:06.641764+0000 mgr.y (mgr.24661) 2129 : cluster [DBG] pgmap v4130: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 2.5 KiB/s rd, 0 B/s wr, 4 op/s 2024-09-13T00:48:08.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:08 smithi136 bash[76371]: cluster 2024-09-13T00:48:07.025308+0000 mon.a (mon.0) 4209 : cluster [DBG] osdmap e2441: 8 total, 8 up, 8 in 2024-09-13T00:48:09.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:09 smithi181 bash[67587]: cluster 2024-09-13T00:48:08.049110+0000 mon.a (mon.0) 4210 : cluster [DBG] osdmap e2442: 8 total, 8 up, 8 in 2024-09-13T00:48:09.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:09 smithi181 bash[67587]: audit 2024-09-13T00:48:08.851604+0000 mon.a (mon.0) 4211 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:48:09.412 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:09 smithi136 bash[75164]: cluster 2024-09-13T00:48:08.049110+0000 mon.a (mon.0) 4210 : cluster [DBG] osdmap e2442: 8 total, 8 up, 8 in 2024-09-13T00:48:09.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:09 smithi136 bash[75164]: audit 2024-09-13T00:48:08.851604+0000 mon.a (mon.0) 4211 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:48:09.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:09 smithi136 bash[76371]: cluster 2024-09-13T00:48:08.049110+0000 mon.a (mon.0) 4210 : cluster [DBG] osdmap e2442: 8 total, 8 up, 8 in 2024-09-13T00:48:09.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:09 smithi136 bash[76371]: audit 2024-09-13T00:48:08.851604+0000 mon.a (mon.0) 4211 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:48:09.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:48:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:48:09] "GET /metrics HTTP/1.1" 200 36746 "" "Prometheus/2.43.0" 2024-09-13T00:48:10.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:10 smithi181 bash[67587]: cluster 2024-09-13T00:48:08.642644+0000 mgr.y (mgr.24661) 2130 : cluster [DBG] pgmap v4133: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 390 KiB/s rd, 5.0 KiB/s wr, 527 op/s 2024-09-13T00:48:10.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:10 smithi181 bash[67587]: cluster 2024-09-13T00:48:09.076549+0000 mon.a (mon.0) 4212 : cluster [DBG] osdmap e2443: 8 total, 8 up, 8 in 2024-09-13T00:48:10.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:10 smithi181 bash[67587]: cluster 2024-09-13T00:48:09.955379+0000 mon.a (mon.0) 4213 : cluster [DBG] osdmap e2444: 8 total, 8 up, 8 in 2024-09-13T00:48:10.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:10 smithi136 bash[75164]: cluster 2024-09-13T00:48:08.642644+0000 mgr.y (mgr.24661) 2130 : cluster [DBG] pgmap v4133: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 390 KiB/s rd, 5.0 KiB/s wr, 527 op/s 2024-09-13T00:48:10.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:10 smithi136 bash[75164]: cluster 2024-09-13T00:48:09.076549+0000 mon.a (mon.0) 4212 : cluster [DBG] osdmap e2443: 8 total, 8 up, 8 in 2024-09-13T00:48:10.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:10 smithi136 bash[75164]: cluster 2024-09-13T00:48:09.955379+0000 mon.a (mon.0) 4213 : cluster [DBG] osdmap e2444: 8 total, 8 up, 8 in 2024-09-13T00:48:10.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:10 smithi136 bash[76371]: cluster 2024-09-13T00:48:08.642644+0000 mgr.y (mgr.24661) 2130 : cluster [DBG] pgmap v4133: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 390 KiB/s rd, 5.0 KiB/s wr, 527 op/s 2024-09-13T00:48:10.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:10 smithi136 bash[76371]: cluster 2024-09-13T00:48:09.076549+0000 mon.a (mon.0) 4212 : cluster [DBG] osdmap e2443: 8 total, 8 up, 8 in 2024-09-13T00:48:10.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:10 smithi136 bash[76371]: cluster 2024-09-13T00:48:09.955379+0000 mon.a (mon.0) 4213 : cluster [DBG] osdmap e2444: 8 total, 8 up, 8 in 2024-09-13T00:48:12.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:11 smithi136 bash[75164]: cluster 2024-09-13T00:48:10.643645+0000 mgr.y (mgr.24661) 2131 : cluster [DBG] pgmap v4136: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 390 KiB/s rd, 5.0 KiB/s wr, 528 op/s 2024-09-13T00:48:12.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:11 smithi136 bash[75164]: cluster 2024-09-13T00:48:10.958884+0000 mon.a (mon.0) 4214 : cluster [DBG] osdmap e2445: 8 total, 8 up, 8 in 2024-09-13T00:48:12.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:11 smithi136 bash[76371]: cluster 2024-09-13T00:48:10.643645+0000 mgr.y (mgr.24661) 2131 : cluster [DBG] pgmap v4136: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 390 KiB/s rd, 5.0 KiB/s wr, 528 op/s 2024-09-13T00:48:12.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:11 smithi136 bash[76371]: cluster 2024-09-13T00:48:10.958884+0000 mon.a (mon.0) 4214 : cluster [DBG] osdmap e2445: 8 total, 8 up, 8 in 2024-09-13T00:48:12.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:11 smithi181 bash[67587]: cluster 2024-09-13T00:48:10.643645+0000 mgr.y (mgr.24661) 2131 : cluster [DBG] pgmap v4136: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 390 KiB/s rd, 5.0 KiB/s wr, 528 op/s 2024-09-13T00:48:12.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:11 smithi181 bash[67587]: cluster 2024-09-13T00:48:10.958884+0000 mon.a (mon.0) 4214 : cluster [DBG] osdmap e2445: 8 total, 8 up, 8 in 2024-09-13T00:48:12.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:48:12 smithi181 bash[95505]: ts=2024-09-13T00:48:12.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:48:13.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:12 smithi136 bash[75164]: cluster 2024-09-13T00:48:11.970219+0000 mon.a (mon.0) 4215 : cluster [DBG] osdmap e2446: 8 total, 8 up, 8 in 2024-09-13T00:48:13.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:12 smithi136 bash[76371]: cluster 2024-09-13T00:48:11.970219+0000 mon.a (mon.0) 4215 : cluster [DBG] osdmap e2446: 8 total, 8 up, 8 in 2024-09-13T00:48:13.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:12 smithi181 bash[67587]: cluster 2024-09-13T00:48:11.970219+0000 mon.a (mon.0) 4215 : cluster [DBG] osdmap e2446: 8 total, 8 up, 8 in 2024-09-13T00:48:14.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:13 smithi136 bash[75164]: cluster 2024-09-13T00:48:12.644357+0000 mgr.y (mgr.24661) 2132 : cluster [DBG] pgmap v4139: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 354 KiB/s rd, 5.0 KiB/s wr, 479 op/s 2024-09-13T00:48:14.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:13 smithi136 bash[75164]: cluster 2024-09-13T00:48:12.979869+0000 mon.a (mon.0) 4216 : cluster [DBG] osdmap e2447: 8 total, 8 up, 8 in 2024-09-13T00:48:14.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:13 smithi136 bash[76371]: cluster 2024-09-13T00:48:12.644357+0000 mgr.y (mgr.24661) 2132 : cluster [DBG] pgmap v4139: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 354 KiB/s rd, 5.0 KiB/s wr, 479 op/s 2024-09-13T00:48:14.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:13 smithi136 bash[76371]: cluster 2024-09-13T00:48:12.979869+0000 mon.a (mon.0) 4216 : cluster [DBG] osdmap e2447: 8 total, 8 up, 8 in 2024-09-13T00:48:14.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:13 smithi181 bash[67587]: cluster 2024-09-13T00:48:12.644357+0000 mgr.y (mgr.24661) 2132 : cluster [DBG] pgmap v4139: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 354 KiB/s rd, 5.0 KiB/s wr, 479 op/s 2024-09-13T00:48:14.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:13 smithi181 bash[67587]: cluster 2024-09-13T00:48:12.979869+0000 mon.a (mon.0) 4216 : cluster [DBG] osdmap e2447: 8 total, 8 up, 8 in 2024-09-13T00:48:15.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:14 smithi136 bash[75164]: cluster 2024-09-13T00:48:13.987709+0000 mon.a (mon.0) 4217 : cluster [DBG] osdmap e2448: 8 total, 8 up, 8 in 2024-09-13T00:48:15.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:14 smithi136 bash[75164]: cluster 2024-09-13T00:48:14.957533+0000 mon.a (mon.0) 4218 : cluster [DBG] osdmap e2449: 8 total, 8 up, 8 in 2024-09-13T00:48:15.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:15 smithi136 bash[76371]: cluster 2024-09-13T00:48:13.987709+0000 mon.a (mon.0) 4217 : cluster [DBG] osdmap e2448: 8 total, 8 up, 8 in 2024-09-13T00:48:15.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:15 smithi136 bash[76371]: cluster 2024-09-13T00:48:14.957533+0000 mon.a (mon.0) 4218 : cluster [DBG] osdmap e2449: 8 total, 8 up, 8 in 2024-09-13T00:48:15.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:15 smithi181 bash[67587]: cluster 2024-09-13T00:48:13.987709+0000 mon.a (mon.0) 4217 : cluster [DBG] osdmap e2448: 8 total, 8 up, 8 in 2024-09-13T00:48:15.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:15 smithi181 bash[67587]: cluster 2024-09-13T00:48:14.957533+0000 mon.a (mon.0) 4218 : cluster [DBG] osdmap e2449: 8 total, 8 up, 8 in 2024-09-13T00:48:16.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:16 smithi181 bash[67587]: cluster 2024-09-13T00:48:14.645083+0000 mgr.y (mgr.24661) 2133 : cluster [DBG] pgmap v4142: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 355 KiB/s rd, 5.0 KiB/s wr, 481 op/s 2024-09-13T00:48:16.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:16 smithi136 bash[76371]: cluster 2024-09-13T00:48:14.645083+0000 mgr.y (mgr.24661) 2133 : cluster [DBG] pgmap v4142: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 355 KiB/s rd, 5.0 KiB/s wr, 481 op/s 2024-09-13T00:48:16.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:16 smithi136 bash[75164]: cluster 2024-09-13T00:48:14.645083+0000 mgr.y (mgr.24661) 2133 : cluster [DBG] pgmap v4142: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 355 KiB/s rd, 5.0 KiB/s wr, 481 op/s 2024-09-13T00:48:17.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:17 smithi181 bash[67587]: cluster 2024-09-13T00:48:16.010172+0000 mon.a (mon.0) 4219 : cluster [DBG] osdmap e2450: 8 total, 8 up, 8 in 2024-09-13T00:48:17.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:17 smithi136 bash[75164]: cluster 2024-09-13T00:48:16.010172+0000 mon.a (mon.0) 4219 : cluster [DBG] osdmap e2450: 8 total, 8 up, 8 in 2024-09-13T00:48:17.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:17 smithi136 bash[76371]: cluster 2024-09-13T00:48:16.010172+0000 mon.a (mon.0) 4219 : cluster [DBG] osdmap e2450: 8 total, 8 up, 8 in 2024-09-13T00:48:18.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:18 smithi181 bash[67587]: cluster 2024-09-13T00:48:16.645870+0000 mgr.y (mgr.24661) 2134 : cluster [DBG] pgmap v4145: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 3.0 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T00:48:18.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:18 smithi181 bash[67587]: cluster 2024-09-13T00:48:17.034581+0000 mon.a (mon.0) 4220 : cluster [DBG] osdmap e2451: 8 total, 8 up, 8 in 2024-09-13T00:48:18.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:18 smithi136 bash[75164]: cluster 2024-09-13T00:48:16.645870+0000 mgr.y (mgr.24661) 2134 : cluster [DBG] pgmap v4145: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 3.0 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T00:48:18.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:18 smithi136 bash[75164]: cluster 2024-09-13T00:48:17.034581+0000 mon.a (mon.0) 4220 : cluster [DBG] osdmap e2451: 8 total, 8 up, 8 in 2024-09-13T00:48:18.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:18 smithi136 bash[76371]: cluster 2024-09-13T00:48:16.645870+0000 mgr.y (mgr.24661) 2134 : cluster [DBG] pgmap v4145: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 3.0 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T00:48:18.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:18 smithi136 bash[76371]: cluster 2024-09-13T00:48:17.034581+0000 mon.a (mon.0) 4220 : cluster [DBG] osdmap e2451: 8 total, 8 up, 8 in 2024-09-13T00:48:19.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:19 smithi181 bash[67587]: cluster 2024-09-13T00:48:18.041875+0000 mon.a (mon.0) 4221 : cluster [DBG] osdmap e2452: 8 total, 8 up, 8 in 2024-09-13T00:48:19.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:19 smithi136 bash[75164]: cluster 2024-09-13T00:48:18.041875+0000 mon.a (mon.0) 4221 : cluster [DBG] osdmap e2452: 8 total, 8 up, 8 in 2024-09-13T00:48:19.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:19 smithi136 bash[76371]: cluster 2024-09-13T00:48:18.041875+0000 mon.a (mon.0) 4221 : cluster [DBG] osdmap e2452: 8 total, 8 up, 8 in 2024-09-13T00:48:19.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:48:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:48:19] "GET /metrics HTTP/1.1" 200 36746 "" "Prometheus/2.43.0" 2024-09-13T00:48:20.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:20 smithi181 bash[67587]: cluster 2024-09-13T00:48:18.646748+0000 mgr.y (mgr.24661) 2135 : cluster [DBG] pgmap v4148: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 324 KiB/s rd, 5.0 KiB/s wr, 438 op/s 2024-09-13T00:48:20.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:20 smithi181 bash[67587]: cluster 2024-09-13T00:48:19.051227+0000 mon.a (mon.0) 4222 : cluster [DBG] osdmap e2453: 8 total, 8 up, 8 in 2024-09-13T00:48:20.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:20 smithi181 bash[67587]: cluster 2024-09-13T00:48:19.962522+0000 mon.a (mon.0) 4223 : cluster [DBG] osdmap e2454: 8 total, 8 up, 8 in 2024-09-13T00:48:20.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:20 smithi136 bash[75164]: cluster 2024-09-13T00:48:18.646748+0000 mgr.y (mgr.24661) 2135 : cluster [DBG] pgmap v4148: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 324 KiB/s rd, 5.0 KiB/s wr, 438 op/s 2024-09-13T00:48:20.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:20 smithi136 bash[75164]: cluster 2024-09-13T00:48:19.051227+0000 mon.a (mon.0) 4222 : cluster [DBG] osdmap e2453: 8 total, 8 up, 8 in 2024-09-13T00:48:20.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:20 smithi136 bash[75164]: cluster 2024-09-13T00:48:19.962522+0000 mon.a (mon.0) 4223 : cluster [DBG] osdmap e2454: 8 total, 8 up, 8 in 2024-09-13T00:48:20.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:20 smithi136 bash[76371]: cluster 2024-09-13T00:48:18.646748+0000 mgr.y (mgr.24661) 2135 : cluster [DBG] pgmap v4148: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 324 KiB/s rd, 5.0 KiB/s wr, 438 op/s 2024-09-13T00:48:20.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:20 smithi136 bash[76371]: cluster 2024-09-13T00:48:19.051227+0000 mon.a (mon.0) 4222 : cluster [DBG] osdmap e2453: 8 total, 8 up, 8 in 2024-09-13T00:48:20.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:20 smithi136 bash[76371]: cluster 2024-09-13T00:48:19.962522+0000 mon.a (mon.0) 4223 : cluster [DBG] osdmap e2454: 8 total, 8 up, 8 in 2024-09-13T00:48:22.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:22 smithi181 bash[67587]: cluster 2024-09-13T00:48:20.647669+0000 mgr.y (mgr.24661) 2136 : cluster [DBG] pgmap v4151: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 325 KiB/s rd, 5.0 KiB/s wr, 440 op/s 2024-09-13T00:48:22.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:22 smithi181 bash[67587]: cluster 2024-09-13T00:48:21.063420+0000 mon.a (mon.0) 4224 : cluster [DBG] osdmap e2455: 8 total, 8 up, 8 in 2024-09-13T00:48:22.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:22 smithi136 bash[75164]: cluster 2024-09-13T00:48:20.647669+0000 mgr.y (mgr.24661) 2136 : cluster [DBG] pgmap v4151: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 325 KiB/s rd, 5.0 KiB/s wr, 440 op/s 2024-09-13T00:48:22.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:22 smithi136 bash[75164]: cluster 2024-09-13T00:48:21.063420+0000 mon.a (mon.0) 4224 : cluster [DBG] osdmap e2455: 8 total, 8 up, 8 in 2024-09-13T00:48:22.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:22 smithi136 bash[76371]: cluster 2024-09-13T00:48:20.647669+0000 mgr.y (mgr.24661) 2136 : cluster [DBG] pgmap v4151: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 325 KiB/s rd, 5.0 KiB/s wr, 440 op/s 2024-09-13T00:48:22.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:22 smithi136 bash[76371]: cluster 2024-09-13T00:48:21.063420+0000 mon.a (mon.0) 4224 : cluster [DBG] osdmap e2455: 8 total, 8 up, 8 in 2024-09-13T00:48:22.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:48:22 smithi181 bash[95505]: ts=2024-09-13T00:48:22.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:48:23.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:23 smithi181 bash[67587]: cluster 2024-09-13T00:48:22.077159+0000 mon.a (mon.0) 4225 : cluster [DBG] osdmap e2456: 8 total, 8 up, 8 in 2024-09-13T00:48:23.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:23 smithi136 bash[75164]: cluster 2024-09-13T00:48:22.077159+0000 mon.a (mon.0) 4225 : cluster [DBG] osdmap e2456: 8 total, 8 up, 8 in 2024-09-13T00:48:23.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:23 smithi136 bash[76371]: cluster 2024-09-13T00:48:22.077159+0000 mon.a (mon.0) 4225 : cluster [DBG] osdmap e2456: 8 total, 8 up, 8 in 2024-09-13T00:48:24.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:24 smithi181 bash[67587]: cluster 2024-09-13T00:48:22.648317+0000 mgr.y (mgr.24661) 2137 : cluster [DBG] pgmap v4154: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 290 KiB/s rd, 5.0 KiB/s wr, 394 op/s 2024-09-13T00:48:24.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:24 smithi181 bash[67587]: cluster 2024-09-13T00:48:23.084741+0000 mon.a (mon.0) 4226 : cluster [DBG] osdmap e2457: 8 total, 8 up, 8 in 2024-09-13T00:48:24.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:24 smithi181 bash[67587]: audit 2024-09-13T00:48:23.851625+0000 mon.a (mon.0) 4227 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:48:24.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:24 smithi136 bash[75164]: cluster 2024-09-13T00:48:22.648317+0000 mgr.y (mgr.24661) 2137 : cluster [DBG] pgmap v4154: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 290 KiB/s rd, 5.0 KiB/s wr, 394 op/s 2024-09-13T00:48:24.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:24 smithi136 bash[75164]: cluster 2024-09-13T00:48:23.084741+0000 mon.a (mon.0) 4226 : cluster [DBG] osdmap e2457: 8 total, 8 up, 8 in 2024-09-13T00:48:24.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:24 smithi136 bash[75164]: audit 2024-09-13T00:48:23.851625+0000 mon.a (mon.0) 4227 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:48:24.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:24 smithi136 bash[76371]: cluster 2024-09-13T00:48:22.648317+0000 mgr.y (mgr.24661) 2137 : cluster [DBG] pgmap v4154: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 290 KiB/s rd, 5.0 KiB/s wr, 394 op/s 2024-09-13T00:48:24.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:24 smithi136 bash[76371]: cluster 2024-09-13T00:48:23.084741+0000 mon.a (mon.0) 4226 : cluster [DBG] osdmap e2457: 8 total, 8 up, 8 in 2024-09-13T00:48:24.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:24 smithi136 bash[76371]: audit 2024-09-13T00:48:23.851625+0000 mon.a (mon.0) 4227 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:48:25.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:25 smithi136 bash[75164]: cluster 2024-09-13T00:48:24.085285+0000 mon.a (mon.0) 4228 : cluster [DBG] osdmap e2458: 8 total, 8 up, 8 in 2024-09-13T00:48:25.511 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:25 smithi136 bash[75164]: cluster 2024-09-13T00:48:24.963991+0000 mon.a (mon.0) 4229 : cluster [DBG] osdmap e2459: 8 total, 8 up, 8 in 2024-09-13T00:48:25.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:25 smithi136 bash[76371]: cluster 2024-09-13T00:48:24.085285+0000 mon.a (mon.0) 4228 : cluster [DBG] osdmap e2458: 8 total, 8 up, 8 in 2024-09-13T00:48:25.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:25 smithi136 bash[76371]: cluster 2024-09-13T00:48:24.963991+0000 mon.a (mon.0) 4229 : cluster [DBG] osdmap e2459: 8 total, 8 up, 8 in 2024-09-13T00:48:25.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:25 smithi181 bash[67587]: cluster 2024-09-13T00:48:24.085285+0000 mon.a (mon.0) 4228 : cluster [DBG] osdmap e2458: 8 total, 8 up, 8 in 2024-09-13T00:48:25.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:25 smithi181 bash[67587]: cluster 2024-09-13T00:48:24.963991+0000 mon.a (mon.0) 4229 : cluster [DBG] osdmap e2459: 8 total, 8 up, 8 in 2024-09-13T00:48:26.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:26 smithi136 bash[75164]: cluster 2024-09-13T00:48:24.649121+0000 mgr.y (mgr.24661) 2138 : cluster [DBG] pgmap v4157: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 290 KiB/s rd, 5.0 KiB/s wr, 393 op/s 2024-09-13T00:48:26.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:26 smithi136 bash[75164]: cluster 2024-09-13T00:48:25.970534+0000 mon.a (mon.0) 4230 : cluster [DBG] osdmap e2460: 8 total, 8 up, 8 in 2024-09-13T00:48:26.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:26 smithi136 bash[76371]: cluster 2024-09-13T00:48:24.649121+0000 mgr.y (mgr.24661) 2138 : cluster [DBG] pgmap v4157: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 290 KiB/s rd, 5.0 KiB/s wr, 393 op/s 2024-09-13T00:48:26.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:26 smithi136 bash[76371]: cluster 2024-09-13T00:48:25.970534+0000 mon.a (mon.0) 4230 : cluster [DBG] osdmap e2460: 8 total, 8 up, 8 in 2024-09-13T00:48:26.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:26 smithi181 bash[67587]: cluster 2024-09-13T00:48:24.649121+0000 mgr.y (mgr.24661) 2138 : cluster [DBG] pgmap v4157: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 290 KiB/s rd, 5.0 KiB/s wr, 393 op/s 2024-09-13T00:48:26.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:26 smithi181 bash[67587]: cluster 2024-09-13T00:48:25.970534+0000 mon.a (mon.0) 4230 : cluster [DBG] osdmap e2460: 8 total, 8 up, 8 in 2024-09-13T00:48:28.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:27 smithi136 bash[75164]: cluster 2024-09-13T00:48:26.649890+0000 mgr.y (mgr.24661) 2139 : cluster [DBG] pgmap v4160: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 3.2 KiB/s rd, 0 B/s wr, 6 op/s 2024-09-13T00:48:28.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:27 smithi136 bash[75164]: cluster 2024-09-13T00:48:26.970450+0000 mon.a (mon.0) 4231 : cluster [DBG] osdmap e2461: 8 total, 8 up, 8 in 2024-09-13T00:48:28.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:27 smithi136 bash[76371]: cluster 2024-09-13T00:48:26.649890+0000 mgr.y (mgr.24661) 2139 : cluster [DBG] pgmap v4160: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 3.2 KiB/s rd, 0 B/s wr, 6 op/s 2024-09-13T00:48:28.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:27 smithi136 bash[76371]: cluster 2024-09-13T00:48:26.970450+0000 mon.a (mon.0) 4231 : cluster [DBG] osdmap e2461: 8 total, 8 up, 8 in 2024-09-13T00:48:28.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:27 smithi181 bash[67587]: cluster 2024-09-13T00:48:26.649890+0000 mgr.y (mgr.24661) 2139 : cluster [DBG] pgmap v4160: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 3.2 KiB/s rd, 0 B/s wr, 6 op/s 2024-09-13T00:48:28.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:27 smithi181 bash[67587]: cluster 2024-09-13T00:48:26.970450+0000 mon.a (mon.0) 4231 : cluster [DBG] osdmap e2461: 8 total, 8 up, 8 in 2024-09-13T00:48:29.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:28 smithi136 bash[75164]: cluster 2024-09-13T00:48:27.979410+0000 mon.a (mon.0) 4232 : cluster [DBG] osdmap e2462: 8 total, 8 up, 8 in 2024-09-13T00:48:29.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:28 smithi136 bash[76371]: cluster 2024-09-13T00:48:27.979410+0000 mon.a (mon.0) 4232 : cluster [DBG] osdmap e2462: 8 total, 8 up, 8 in 2024-09-13T00:48:29.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:28 smithi181 bash[67587]: cluster 2024-09-13T00:48:27.979410+0000 mon.a (mon.0) 4232 : cluster [DBG] osdmap e2462: 8 total, 8 up, 8 in 2024-09-13T00:48:29.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:48:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:48:29] "GET /metrics HTTP/1.1" 200 36746 "" "Prometheus/2.43.0" 2024-09-13T00:48:30.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:29 smithi136 bash[75164]: cluster 2024-09-13T00:48:28.650787+0000 mgr.y (mgr.24661) 2140 : cluster [DBG] pgmap v4163: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 259 KiB/s rd, 5.0 KiB/s wr, 353 op/s 2024-09-13T00:48:30.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:29 smithi136 bash[75164]: cluster 2024-09-13T00:48:28.989353+0000 mon.a (mon.0) 4233 : cluster [DBG] osdmap e2463: 8 total, 8 up, 8 in 2024-09-13T00:48:30.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:29 smithi136 bash[75164]: cluster 2024-09-13T00:48:29.966734+0000 mon.a (mon.0) 4234 : cluster [DBG] osdmap e2464: 8 total, 8 up, 8 in 2024-09-13T00:48:30.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:29 smithi136 bash[76371]: cluster 2024-09-13T00:48:28.650787+0000 mgr.y (mgr.24661) 2140 : cluster [DBG] pgmap v4163: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 259 KiB/s rd, 5.0 KiB/s wr, 353 op/s 2024-09-13T00:48:30.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:29 smithi136 bash[76371]: cluster 2024-09-13T00:48:28.989353+0000 mon.a (mon.0) 4233 : cluster [DBG] osdmap e2463: 8 total, 8 up, 8 in 2024-09-13T00:48:30.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:29 smithi136 bash[76371]: cluster 2024-09-13T00:48:29.966734+0000 mon.a (mon.0) 4234 : cluster [DBG] osdmap e2464: 8 total, 8 up, 8 in 2024-09-13T00:48:30.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:29 smithi181 bash[67587]: cluster 2024-09-13T00:48:28.650787+0000 mgr.y (mgr.24661) 2140 : cluster [DBG] pgmap v4163: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 259 KiB/s rd, 5.0 KiB/s wr, 353 op/s 2024-09-13T00:48:30.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:29 smithi181 bash[67587]: cluster 2024-09-13T00:48:28.989353+0000 mon.a (mon.0) 4233 : cluster [DBG] osdmap e2463: 8 total, 8 up, 8 in 2024-09-13T00:48:30.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:29 smithi181 bash[67587]: cluster 2024-09-13T00:48:29.966734+0000 mon.a (mon.0) 4234 : cluster [DBG] osdmap e2464: 8 total, 8 up, 8 in 2024-09-13T00:48:32.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:32 smithi136 bash[75164]: cluster 2024-09-13T00:48:30.651778+0000 mgr.y (mgr.24661) 2141 : cluster [DBG] pgmap v4166: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 259 KiB/s rd, 5.0 KiB/s wr, 353 op/s 2024-09-13T00:48:32.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:32 smithi136 bash[75164]: cluster 2024-09-13T00:48:31.004603+0000 mon.a (mon.0) 4235 : cluster [DBG] osdmap e2465: 8 total, 8 up, 8 in 2024-09-13T00:48:32.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:32 smithi136 bash[76371]: cluster 2024-09-13T00:48:30.651778+0000 mgr.y (mgr.24661) 2141 : cluster [DBG] pgmap v4166: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 259 KiB/s rd, 5.0 KiB/s wr, 353 op/s 2024-09-13T00:48:32.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:32 smithi136 bash[76371]: cluster 2024-09-13T00:48:31.004603+0000 mon.a (mon.0) 4235 : cluster [DBG] osdmap e2465: 8 total, 8 up, 8 in 2024-09-13T00:48:32.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:32 smithi181 bash[67587]: cluster 2024-09-13T00:48:30.651778+0000 mgr.y (mgr.24661) 2141 : cluster [DBG] pgmap v4166: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 259 KiB/s rd, 5.0 KiB/s wr, 353 op/s 2024-09-13T00:48:32.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:32 smithi181 bash[67587]: cluster 2024-09-13T00:48:31.004603+0000 mon.a (mon.0) 4235 : cluster [DBG] osdmap e2465: 8 total, 8 up, 8 in 2024-09-13T00:48:32.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:48:32 smithi181 bash[95505]: ts=2024-09-13T00:48:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:48:33.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:33 smithi181 bash[67587]: cluster 2024-09-13T00:48:32.015644+0000 mon.a (mon.0) 4236 : cluster [DBG] osdmap e2466: 8 total, 8 up, 8 in 2024-09-13T00:48:33.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:33 smithi136 bash[76371]: cluster 2024-09-13T00:48:32.015644+0000 mon.a (mon.0) 4236 : cluster [DBG] osdmap e2466: 8 total, 8 up, 8 in 2024-09-13T00:48:33.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:33 smithi136 bash[75164]: cluster 2024-09-13T00:48:32.015644+0000 mon.a (mon.0) 4236 : cluster [DBG] osdmap e2466: 8 total, 8 up, 8 in 2024-09-13T00:48:34.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:34 smithi181 bash[67587]: cluster 2024-09-13T00:48:32.652470+0000 mgr.y (mgr.24661) 2142 : cluster [DBG] pgmap v4169: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 223 KiB/s rd, 5.0 KiB/s wr, 306 op/s 2024-09-13T00:48:34.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:34 smithi181 bash[67587]: cluster 2024-09-13T00:48:33.026077+0000 mon.a (mon.0) 4237 : cluster [DBG] osdmap e2467: 8 total, 8 up, 8 in 2024-09-13T00:48:34.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:34 smithi136 bash[75164]: cluster 2024-09-13T00:48:32.652470+0000 mgr.y (mgr.24661) 2142 : cluster [DBG] pgmap v4169: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 223 KiB/s rd, 5.0 KiB/s wr, 306 op/s 2024-09-13T00:48:34.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:34 smithi136 bash[75164]: cluster 2024-09-13T00:48:33.026077+0000 mon.a (mon.0) 4237 : cluster [DBG] osdmap e2467: 8 total, 8 up, 8 in 2024-09-13T00:48:34.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:34 smithi136 bash[76371]: cluster 2024-09-13T00:48:32.652470+0000 mgr.y (mgr.24661) 2142 : cluster [DBG] pgmap v4169: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 223 KiB/s rd, 5.0 KiB/s wr, 306 op/s 2024-09-13T00:48:34.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:34 smithi136 bash[76371]: cluster 2024-09-13T00:48:33.026077+0000 mon.a (mon.0) 4237 : cluster [DBG] osdmap e2467: 8 total, 8 up, 8 in 2024-09-13T00:48:35.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:35 smithi181 bash[67587]: cluster 2024-09-13T00:48:34.027188+0000 mon.a (mon.0) 4238 : cluster [DBG] osdmap e2468: 8 total, 8 up, 8 in 2024-09-13T00:48:35.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:35 smithi181 bash[67587]: cluster 2024-09-13T00:48:34.969289+0000 mon.a (mon.0) 4239 : cluster [DBG] osdmap e2469: 8 total, 8 up, 8 in 2024-09-13T00:48:35.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:35 smithi136 bash[75164]: cluster 2024-09-13T00:48:34.027188+0000 mon.a (mon.0) 4238 : cluster [DBG] osdmap e2468: 8 total, 8 up, 8 in 2024-09-13T00:48:35.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:35 smithi136 bash[75164]: cluster 2024-09-13T00:48:34.969289+0000 mon.a (mon.0) 4239 : cluster [DBG] osdmap e2469: 8 total, 8 up, 8 in 2024-09-13T00:48:35.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:35 smithi136 bash[76371]: cluster 2024-09-13T00:48:34.027188+0000 mon.a (mon.0) 4238 : cluster [DBG] osdmap e2468: 8 total, 8 up, 8 in 2024-09-13T00:48:35.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:35 smithi136 bash[76371]: cluster 2024-09-13T00:48:34.969289+0000 mon.a (mon.0) 4239 : cluster [DBG] osdmap e2469: 8 total, 8 up, 8 in 2024-09-13T00:48:36.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:36 smithi181 bash[67587]: cluster 2024-09-13T00:48:34.653153+0000 mgr.y (mgr.24661) 2143 : cluster [DBG] pgmap v4172: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 223 KiB/s rd, 5.0 KiB/s wr, 302 op/s 2024-09-13T00:48:36.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:36 smithi136 bash[75164]: cluster 2024-09-13T00:48:34.653153+0000 mgr.y (mgr.24661) 2143 : cluster [DBG] pgmap v4172: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 223 KiB/s rd, 5.0 KiB/s wr, 302 op/s 2024-09-13T00:48:36.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:36 smithi136 bash[76371]: cluster 2024-09-13T00:48:34.653153+0000 mgr.y (mgr.24661) 2143 : cluster [DBG] pgmap v4172: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 223 KiB/s rd, 5.0 KiB/s wr, 302 op/s 2024-09-13T00:48:37.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:37 smithi181 bash[67587]: cluster 2024-09-13T00:48:36.056796+0000 mon.a (mon.0) 4240 : cluster [DBG] osdmap e2470: 8 total, 8 up, 8 in 2024-09-13T00:48:37.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:37 smithi136 bash[75164]: cluster 2024-09-13T00:48:36.056796+0000 mon.a (mon.0) 4240 : cluster [DBG] osdmap e2470: 8 total, 8 up, 8 in 2024-09-13T00:48:37.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:37 smithi136 bash[76371]: cluster 2024-09-13T00:48:36.056796+0000 mon.a (mon.0) 4240 : cluster [DBG] osdmap e2470: 8 total, 8 up, 8 in 2024-09-13T00:48:38.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:38 smithi181 bash[67587]: cluster 2024-09-13T00:48:36.653960+0000 mgr.y (mgr.24661) 2144 : cluster [DBG] pgmap v4175: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 3.5 KiB/s rd, 0 B/s wr, 6 op/s 2024-09-13T00:48:38.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:38 smithi181 bash[67587]: cluster 2024-09-13T00:48:37.080946+0000 mon.a (mon.0) 4241 : cluster [DBG] osdmap e2471: 8 total, 8 up, 8 in 2024-09-13T00:48:38.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:38 smithi136 bash[75164]: cluster 2024-09-13T00:48:36.653960+0000 mgr.y (mgr.24661) 2144 : cluster [DBG] pgmap v4175: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 3.5 KiB/s rd, 0 B/s wr, 6 op/s 2024-09-13T00:48:38.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:38 smithi136 bash[75164]: cluster 2024-09-13T00:48:37.080946+0000 mon.a (mon.0) 4241 : cluster [DBG] osdmap e2471: 8 total, 8 up, 8 in 2024-09-13T00:48:38.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:38 smithi136 bash[76371]: cluster 2024-09-13T00:48:36.653960+0000 mgr.y (mgr.24661) 2144 : cluster [DBG] pgmap v4175: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 3.5 KiB/s rd, 0 B/s wr, 6 op/s 2024-09-13T00:48:38.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:38 smithi136 bash[76371]: cluster 2024-09-13T00:48:37.080946+0000 mon.a (mon.0) 4241 : cluster [DBG] osdmap e2471: 8 total, 8 up, 8 in 2024-09-13T00:48:39.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:39 smithi181 bash[67587]: cluster 2024-09-13T00:48:38.086692+0000 mon.a (mon.0) 4242 : cluster [DBG] osdmap e2472: 8 total, 8 up, 8 in 2024-09-13T00:48:39.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:39 smithi181 bash[67587]: audit 2024-09-13T00:48:38.852227+0000 mon.a (mon.0) 4243 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:48:39.412 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:39 smithi136 bash[75164]: cluster 2024-09-13T00:48:38.086692+0000 mon.a (mon.0) 4242 : cluster [DBG] osdmap e2472: 8 total, 8 up, 8 in 2024-09-13T00:48:39.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:39 smithi136 bash[75164]: audit 2024-09-13T00:48:38.852227+0000 mon.a (mon.0) 4243 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:48:39.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:39 smithi136 bash[76371]: cluster 2024-09-13T00:48:38.086692+0000 mon.a (mon.0) 4242 : cluster [DBG] osdmap e2472: 8 total, 8 up, 8 in 2024-09-13T00:48:39.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:39 smithi136 bash[76371]: audit 2024-09-13T00:48:38.852227+0000 mon.a (mon.0) 4243 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:48:39.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:48:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:48:39] "GET /metrics HTTP/1.1" 200 36745 "" "Prometheus/2.43.0" 2024-09-13T00:48:40.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:40 smithi181 bash[67587]: cluster 2024-09-13T00:48:38.654841+0000 mgr.y (mgr.24661) 2145 : cluster [DBG] pgmap v4178: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 193 KiB/s rd, 5.0 KiB/s wr, 264 op/s 2024-09-13T00:48:40.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:40 smithi181 bash[67587]: cluster 2024-09-13T00:48:39.097996+0000 mon.a (mon.0) 4244 : cluster [DBG] osdmap e2473: 8 total, 8 up, 8 in 2024-09-13T00:48:40.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:40 smithi181 bash[67587]: cluster 2024-09-13T00:48:39.973556+0000 mon.a (mon.0) 4245 : cluster [DBG] osdmap e2474: 8 total, 8 up, 8 in 2024-09-13T00:48:40.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:40 smithi136 bash[75164]: cluster 2024-09-13T00:48:38.654841+0000 mgr.y (mgr.24661) 2145 : cluster [DBG] pgmap v4178: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 193 KiB/s rd, 5.0 KiB/s wr, 264 op/s 2024-09-13T00:48:40.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:40 smithi136 bash[75164]: cluster 2024-09-13T00:48:39.097996+0000 mon.a (mon.0) 4244 : cluster [DBG] osdmap e2473: 8 total, 8 up, 8 in 2024-09-13T00:48:40.511 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:40 smithi136 bash[75164]: cluster 2024-09-13T00:48:39.973556+0000 mon.a (mon.0) 4245 : cluster [DBG] osdmap e2474: 8 total, 8 up, 8 in 2024-09-13T00:48:40.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:40 smithi136 bash[76371]: cluster 2024-09-13T00:48:38.654841+0000 mgr.y (mgr.24661) 2145 : cluster [DBG] pgmap v4178: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 193 KiB/s rd, 5.0 KiB/s wr, 264 op/s 2024-09-13T00:48:40.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:40 smithi136 bash[76371]: cluster 2024-09-13T00:48:39.097996+0000 mon.a (mon.0) 4244 : cluster [DBG] osdmap e2473: 8 total, 8 up, 8 in 2024-09-13T00:48:40.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:40 smithi136 bash[76371]: cluster 2024-09-13T00:48:39.973556+0000 mon.a (mon.0) 4245 : cluster [DBG] osdmap e2474: 8 total, 8 up, 8 in 2024-09-13T00:48:42.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:41 smithi136 bash[76371]: cluster 2024-09-13T00:48:40.655859+0000 mgr.y (mgr.24661) 2146 : cluster [DBG] pgmap v4181: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 193 KiB/s rd, 5.0 KiB/s wr, 263 op/s 2024-09-13T00:48:42.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:41 smithi136 bash[76371]: cluster 2024-09-13T00:48:40.975118+0000 mon.a (mon.0) 4246 : cluster [DBG] osdmap e2475: 8 total, 8 up, 8 in 2024-09-13T00:48:42.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:41 smithi136 bash[75164]: cluster 2024-09-13T00:48:40.655859+0000 mgr.y (mgr.24661) 2146 : cluster [DBG] pgmap v4181: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 193 KiB/s rd, 5.0 KiB/s wr, 263 op/s 2024-09-13T00:48:42.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:41 smithi136 bash[75164]: cluster 2024-09-13T00:48:40.975118+0000 mon.a (mon.0) 4246 : cluster [DBG] osdmap e2475: 8 total, 8 up, 8 in 2024-09-13T00:48:42.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:41 smithi181 bash[67587]: cluster 2024-09-13T00:48:40.655859+0000 mgr.y (mgr.24661) 2146 : cluster [DBG] pgmap v4181: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 193 KiB/s rd, 5.0 KiB/s wr, 263 op/s 2024-09-13T00:48:42.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:41 smithi181 bash[67587]: cluster 2024-09-13T00:48:40.975118+0000 mon.a (mon.0) 4246 : cluster [DBG] osdmap e2475: 8 total, 8 up, 8 in 2024-09-13T00:48:42.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:48:42 smithi181 bash[95505]: ts=2024-09-13T00:48:42.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:48:43.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:43 smithi136 bash[75164]: cluster 2024-09-13T00:48:41.975781+0000 mon.a (mon.0) 4247 : cluster [DBG] osdmap e2476: 8 total, 8 up, 8 in 2024-09-13T00:48:43.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:42 smithi136 bash[76371]: cluster 2024-09-13T00:48:41.975781+0000 mon.a (mon.0) 4247 : cluster [DBG] osdmap e2476: 8 total, 8 up, 8 in 2024-09-13T00:48:43.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:42 smithi181 bash[67587]: cluster 2024-09-13T00:48:41.975781+0000 mon.a (mon.0) 4247 : cluster [DBG] osdmap e2476: 8 total, 8 up, 8 in 2024-09-13T00:48:44.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:44 smithi181 bash[67587]: cluster 2024-09-13T00:48:42.656548+0000 mgr.y (mgr.24661) 2147 : cluster [DBG] pgmap v4184: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 159 KiB/s rd, 5.0 KiB/s wr, 218 op/s 2024-09-13T00:48:44.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:44 smithi181 bash[67587]: cluster 2024-09-13T00:48:43.010080+0000 mon.a (mon.0) 4248 : cluster [DBG] osdmap e2477: 8 total, 8 up, 8 in 2024-09-13T00:48:44.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:44 smithi136 bash[75164]: cluster 2024-09-13T00:48:42.656548+0000 mgr.y (mgr.24661) 2147 : cluster [DBG] pgmap v4184: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 159 KiB/s rd, 5.0 KiB/s wr, 218 op/s 2024-09-13T00:48:44.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:44 smithi136 bash[75164]: cluster 2024-09-13T00:48:43.010080+0000 mon.a (mon.0) 4248 : cluster [DBG] osdmap e2477: 8 total, 8 up, 8 in 2024-09-13T00:48:44.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:44 smithi136 bash[76371]: cluster 2024-09-13T00:48:42.656548+0000 mgr.y (mgr.24661) 2147 : cluster [DBG] pgmap v4184: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 159 KiB/s rd, 5.0 KiB/s wr, 218 op/s 2024-09-13T00:48:44.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:44 smithi136 bash[76371]: cluster 2024-09-13T00:48:43.010080+0000 mon.a (mon.0) 4248 : cluster [DBG] osdmap e2477: 8 total, 8 up, 8 in 2024-09-13T00:48:45.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:45 smithi181 bash[67587]: cluster 2024-09-13T00:48:44.008452+0000 mon.a (mon.0) 4249 : cluster [DBG] osdmap e2478: 8 total, 8 up, 8 in 2024-09-13T00:48:45.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:45 smithi181 bash[67587]: cluster 2024-09-13T00:48:44.972577+0000 mon.a (mon.0) 4250 : cluster [DBG] osdmap e2479: 8 total, 8 up, 8 in 2024-09-13T00:48:45.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:45 smithi136 bash[75164]: cluster 2024-09-13T00:48:44.008452+0000 mon.a (mon.0) 4249 : cluster [DBG] osdmap e2478: 8 total, 8 up, 8 in 2024-09-13T00:48:45.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:45 smithi136 bash[75164]: cluster 2024-09-13T00:48:44.972577+0000 mon.a (mon.0) 4250 : cluster [DBG] osdmap e2479: 8 total, 8 up, 8 in 2024-09-13T00:48:45.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:45 smithi136 bash[76371]: cluster 2024-09-13T00:48:44.008452+0000 mon.a (mon.0) 4249 : cluster [DBG] osdmap e2478: 8 total, 8 up, 8 in 2024-09-13T00:48:45.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:45 smithi136 bash[76371]: cluster 2024-09-13T00:48:44.972577+0000 mon.a (mon.0) 4250 : cluster [DBG] osdmap e2479: 8 total, 8 up, 8 in 2024-09-13T00:48:46.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:46 smithi181 bash[67587]: cluster 2024-09-13T00:48:44.657296+0000 mgr.y (mgr.24661) 2148 : cluster [DBG] pgmap v4187: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 160 KiB/s rd, 5.0 KiB/s wr, 218 op/s 2024-09-13T00:48:46.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:46 smithi136 bash[75164]: cluster 2024-09-13T00:48:44.657296+0000 mgr.y (mgr.24661) 2148 : cluster [DBG] pgmap v4187: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 160 KiB/s rd, 5.0 KiB/s wr, 218 op/s 2024-09-13T00:48:46.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:46 smithi136 bash[76371]: cluster 2024-09-13T00:48:44.657296+0000 mgr.y (mgr.24661) 2148 : cluster [DBG] pgmap v4187: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 160 KiB/s rd, 5.0 KiB/s wr, 218 op/s 2024-09-13T00:48:47.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:47 smithi181 bash[67587]: cluster 2024-09-13T00:48:46.039828+0000 mon.a (mon.0) 4251 : cluster [DBG] osdmap e2480: 8 total, 8 up, 8 in 2024-09-13T00:48:47.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:47 smithi136 bash[75164]: cluster 2024-09-13T00:48:46.039828+0000 mon.a (mon.0) 4251 : cluster [DBG] osdmap e2480: 8 total, 8 up, 8 in 2024-09-13T00:48:47.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:47 smithi136 bash[76371]: cluster 2024-09-13T00:48:46.039828+0000 mon.a (mon.0) 4251 : cluster [DBG] osdmap e2480: 8 total, 8 up, 8 in 2024-09-13T00:48:48.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:48 smithi181 bash[67587]: cluster 2024-09-13T00:48:46.658160+0000 mgr.y (mgr.24661) 2149 : cluster [DBG] pgmap v4190: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 4.2 KiB/s rd, 0 B/s wr, 8 op/s 2024-09-13T00:48:48.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:48 smithi181 bash[67587]: cluster 2024-09-13T00:48:47.051338+0000 mon.a (mon.0) 4252 : cluster [DBG] osdmap e2481: 8 total, 8 up, 8 in 2024-09-13T00:48:48.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:48 smithi136 bash[75164]: cluster 2024-09-13T00:48:46.658160+0000 mgr.y (mgr.24661) 2149 : cluster [DBG] pgmap v4190: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 4.2 KiB/s rd, 0 B/s wr, 8 op/s 2024-09-13T00:48:48.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:48 smithi136 bash[75164]: cluster 2024-09-13T00:48:47.051338+0000 mon.a (mon.0) 4252 : cluster [DBG] osdmap e2481: 8 total, 8 up, 8 in 2024-09-13T00:48:48.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:48 smithi136 bash[76371]: cluster 2024-09-13T00:48:46.658160+0000 mgr.y (mgr.24661) 2149 : cluster [DBG] pgmap v4190: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 4.2 KiB/s rd, 0 B/s wr, 8 op/s 2024-09-13T00:48:48.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:48 smithi136 bash[76371]: cluster 2024-09-13T00:48:47.051338+0000 mon.a (mon.0) 4252 : cluster [DBG] osdmap e2481: 8 total, 8 up, 8 in 2024-09-13T00:48:49.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:49 smithi181 bash[67587]: cluster 2024-09-13T00:48:48.051740+0000 mon.a (mon.0) 4253 : cluster [DBG] osdmap e2482: 8 total, 8 up, 8 in 2024-09-13T00:48:49.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:49 smithi136 bash[76371]: cluster 2024-09-13T00:48:48.051740+0000 mon.a (mon.0) 4253 : cluster [DBG] osdmap e2482: 8 total, 8 up, 8 in 2024-09-13T00:48:49.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:49 smithi136 bash[75164]: cluster 2024-09-13T00:48:48.051740+0000 mon.a (mon.0) 4253 : cluster [DBG] osdmap e2482: 8 total, 8 up, 8 in 2024-09-13T00:48:49.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:48:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:48:49] "GET /metrics HTTP/1.1" 200 36745 "" "Prometheus/2.43.0" 2024-09-13T00:48:50.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:50 smithi181 bash[67587]: cluster 2024-09-13T00:48:48.659085+0000 mgr.y (mgr.24661) 2150 : cluster [DBG] pgmap v4193: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 128 KiB/s rd, 5.0 KiB/s wr, 176 op/s 2024-09-13T00:48:50.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:50 smithi181 bash[67587]: cluster 2024-09-13T00:48:49.076352+0000 mon.a (mon.0) 4254 : cluster [DBG] osdmap e2483: 8 total, 8 up, 8 in 2024-09-13T00:48:50.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:50 smithi181 bash[67587]: cluster 2024-09-13T00:48:49.976896+0000 mon.a (mon.0) 4255 : cluster [DBG] osdmap e2484: 8 total, 8 up, 8 in 2024-09-13T00:48:50.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:50 smithi136 bash[75164]: cluster 2024-09-13T00:48:48.659085+0000 mgr.y (mgr.24661) 2150 : cluster [DBG] pgmap v4193: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 128 KiB/s rd, 5.0 KiB/s wr, 176 op/s 2024-09-13T00:48:50.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:50 smithi136 bash[75164]: cluster 2024-09-13T00:48:49.076352+0000 mon.a (mon.0) 4254 : cluster [DBG] osdmap e2483: 8 total, 8 up, 8 in 2024-09-13T00:48:50.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:50 smithi136 bash[75164]: cluster 2024-09-13T00:48:49.976896+0000 mon.a (mon.0) 4255 : cluster [DBG] osdmap e2484: 8 total, 8 up, 8 in 2024-09-13T00:48:50.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:50 smithi136 bash[76371]: cluster 2024-09-13T00:48:48.659085+0000 mgr.y (mgr.24661) 2150 : cluster [DBG] pgmap v4193: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 128 KiB/s rd, 5.0 KiB/s wr, 176 op/s 2024-09-13T00:48:50.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:50 smithi136 bash[76371]: cluster 2024-09-13T00:48:49.076352+0000 mon.a (mon.0) 4254 : cluster [DBG] osdmap e2483: 8 total, 8 up, 8 in 2024-09-13T00:48:50.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:50 smithi136 bash[76371]: cluster 2024-09-13T00:48:49.976896+0000 mon.a (mon.0) 4255 : cluster [DBG] osdmap e2484: 8 total, 8 up, 8 in 2024-09-13T00:48:52.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:51 smithi136 bash[75164]: cluster 2024-09-13T00:48:50.660102+0000 mgr.y (mgr.24661) 2151 : cluster [DBG] pgmap v4196: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 128 KiB/s rd, 5.0 KiB/s wr, 174 op/s 2024-09-13T00:48:52.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:51 smithi136 bash[75164]: cluster 2024-09-13T00:48:50.978347+0000 mon.a (mon.0) 4256 : cluster [DBG] osdmap e2485: 8 total, 8 up, 8 in 2024-09-13T00:48:52.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:51 smithi136 bash[76371]: cluster 2024-09-13T00:48:50.660102+0000 mgr.y (mgr.24661) 2151 : cluster [DBG] pgmap v4196: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 128 KiB/s rd, 5.0 KiB/s wr, 174 op/s 2024-09-13T00:48:52.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:51 smithi136 bash[76371]: cluster 2024-09-13T00:48:50.978347+0000 mon.a (mon.0) 4256 : cluster [DBG] osdmap e2485: 8 total, 8 up, 8 in 2024-09-13T00:48:52.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:51 smithi181 bash[67587]: cluster 2024-09-13T00:48:50.660102+0000 mgr.y (mgr.24661) 2151 : cluster [DBG] pgmap v4196: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 128 KiB/s rd, 5.0 KiB/s wr, 174 op/s 2024-09-13T00:48:52.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:51 smithi181 bash[67587]: cluster 2024-09-13T00:48:50.978347+0000 mon.a (mon.0) 4256 : cluster [DBG] osdmap e2485: 8 total, 8 up, 8 in 2024-09-13T00:48:52.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:48:52 smithi181 bash[95505]: ts=2024-09-13T00:48:52.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:48:53.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:52 smithi136 bash[75164]: cluster 2024-09-13T00:48:51.990180+0000 mon.a (mon.0) 4257 : cluster [DBG] osdmap e2486: 8 total, 8 up, 8 in 2024-09-13T00:48:53.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:52 smithi136 bash[76371]: cluster 2024-09-13T00:48:51.990180+0000 mon.a (mon.0) 4257 : cluster [DBG] osdmap e2486: 8 total, 8 up, 8 in 2024-09-13T00:48:53.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:52 smithi181 bash[67587]: cluster 2024-09-13T00:48:51.990180+0000 mon.a (mon.0) 4257 : cluster [DBG] osdmap e2486: 8 total, 8 up, 8 in 2024-09-13T00:48:54.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:53 smithi136 bash[75164]: cluster 2024-09-13T00:48:52.660810+0000 mgr.y (mgr.24661) 2152 : cluster [DBG] pgmap v4199: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 95 KiB/s rd, 5.0 KiB/s wr, 131 op/s 2024-09-13T00:48:54.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:53 smithi136 bash[75164]: cluster 2024-09-13T00:48:52.999876+0000 mon.a (mon.0) 4258 : cluster [DBG] osdmap e2487: 8 total, 8 up, 8 in 2024-09-13T00:48:54.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:53 smithi136 bash[75164]: audit 2024-09-13T00:48:53.852800+0000 mon.a (mon.0) 4259 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:48:54.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:53 smithi136 bash[76371]: cluster 2024-09-13T00:48:52.660810+0000 mgr.y (mgr.24661) 2152 : cluster [DBG] pgmap v4199: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 95 KiB/s rd, 5.0 KiB/s wr, 131 op/s 2024-09-13T00:48:54.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:53 smithi136 bash[76371]: cluster 2024-09-13T00:48:52.999876+0000 mon.a (mon.0) 4258 : cluster [DBG] osdmap e2487: 8 total, 8 up, 8 in 2024-09-13T00:48:54.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:53 smithi136 bash[76371]: audit 2024-09-13T00:48:53.852800+0000 mon.a (mon.0) 4259 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:48:54.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:53 smithi181 bash[67587]: cluster 2024-09-13T00:48:52.660810+0000 mgr.y (mgr.24661) 2152 : cluster [DBG] pgmap v4199: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 95 KiB/s rd, 5.0 KiB/s wr, 131 op/s 2024-09-13T00:48:54.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:53 smithi181 bash[67587]: cluster 2024-09-13T00:48:52.999876+0000 mon.a (mon.0) 4258 : cluster [DBG] osdmap e2487: 8 total, 8 up, 8 in 2024-09-13T00:48:54.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:53 smithi181 bash[67587]: audit 2024-09-13T00:48:53.852800+0000 mon.a (mon.0) 4259 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:48:55.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:55 smithi181 bash[67587]: cluster 2024-09-13T00:48:54.010286+0000 mon.a (mon.0) 4260 : cluster [DBG] osdmap e2488: 8 total, 8 up, 8 in 2024-09-13T00:48:55.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:55 smithi181 bash[67587]: cluster 2024-09-13T00:48:54.978778+0000 mon.a (mon.0) 4261 : cluster [DBG] osdmap e2489: 8 total, 8 up, 8 in 2024-09-13T00:48:55.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:55 smithi136 bash[75164]: cluster 2024-09-13T00:48:54.010286+0000 mon.a (mon.0) 4260 : cluster [DBG] osdmap e2488: 8 total, 8 up, 8 in 2024-09-13T00:48:55.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:55 smithi136 bash[75164]: cluster 2024-09-13T00:48:54.978778+0000 mon.a (mon.0) 4261 : cluster [DBG] osdmap e2489: 8 total, 8 up, 8 in 2024-09-13T00:48:55.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:55 smithi136 bash[76371]: cluster 2024-09-13T00:48:54.010286+0000 mon.a (mon.0) 4260 : cluster [DBG] osdmap e2488: 8 total, 8 up, 8 in 2024-09-13T00:48:55.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:55 smithi136 bash[76371]: cluster 2024-09-13T00:48:54.978778+0000 mon.a (mon.0) 4261 : cluster [DBG] osdmap e2489: 8 total, 8 up, 8 in 2024-09-13T00:48:56.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:56 smithi181 bash[67587]: cluster 2024-09-13T00:48:54.661469+0000 mgr.y (mgr.24661) 2153 : cluster [DBG] pgmap v4202: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 95 KiB/s rd, 5.0 KiB/s wr, 131 op/s 2024-09-13T00:48:56.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:56 smithi136 bash[75164]: cluster 2024-09-13T00:48:54.661469+0000 mgr.y (mgr.24661) 2153 : cluster [DBG] pgmap v4202: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 95 KiB/s rd, 5.0 KiB/s wr, 131 op/s 2024-09-13T00:48:56.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:56 smithi136 bash[76371]: cluster 2024-09-13T00:48:54.661469+0000 mgr.y (mgr.24661) 2153 : cluster [DBG] pgmap v4202: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 95 KiB/s rd, 5.0 KiB/s wr, 131 op/s 2024-09-13T00:48:57.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:57 smithi181 bash[67587]: cluster 2024-09-13T00:48:56.031265+0000 mon.a (mon.0) 4262 : cluster [DBG] osdmap e2490: 8 total, 8 up, 8 in 2024-09-13T00:48:57.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:57 smithi136 bash[75164]: cluster 2024-09-13T00:48:56.031265+0000 mon.a (mon.0) 4262 : cluster [DBG] osdmap e2490: 8 total, 8 up, 8 in 2024-09-13T00:48:57.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:57 smithi136 bash[76371]: cluster 2024-09-13T00:48:56.031265+0000 mon.a (mon.0) 4262 : cluster [DBG] osdmap e2490: 8 total, 8 up, 8 in 2024-09-13T00:48:58.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:58 smithi181 bash[67587]: cluster 2024-09-13T00:48:56.662342+0000 mgr.y (mgr.24661) 2154 : cluster [DBG] pgmap v4205: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 3.7 KiB/s rd, 0 B/s wr, 7 op/s 2024-09-13T00:48:58.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:58 smithi181 bash[67587]: cluster 2024-09-13T00:48:57.055905+0000 mon.a (mon.0) 4263 : cluster [DBG] osdmap e2491: 8 total, 8 up, 8 in 2024-09-13T00:48:58.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:58 smithi136 bash[75164]: cluster 2024-09-13T00:48:56.662342+0000 mgr.y (mgr.24661) 2154 : cluster [DBG] pgmap v4205: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 3.7 KiB/s rd, 0 B/s wr, 7 op/s 2024-09-13T00:48:58.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:58 smithi136 bash[75164]: cluster 2024-09-13T00:48:57.055905+0000 mon.a (mon.0) 4263 : cluster [DBG] osdmap e2491: 8 total, 8 up, 8 in 2024-09-13T00:48:58.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:58 smithi136 bash[76371]: cluster 2024-09-13T00:48:56.662342+0000 mgr.y (mgr.24661) 2154 : cluster [DBG] pgmap v4205: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 3.7 KiB/s rd, 0 B/s wr, 7 op/s 2024-09-13T00:48:58.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:58 smithi136 bash[76371]: cluster 2024-09-13T00:48:57.055905+0000 mon.a (mon.0) 4263 : cluster [DBG] osdmap e2491: 8 total, 8 up, 8 in 2024-09-13T00:48:59.250 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_many_snaps PASSED [ 47%] 2024-09-13T00:48:59.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:48:59 smithi181 bash[67587]: cluster 2024-09-13T00:48:58.055069+0000 mon.a (mon.0) 4264 : cluster [DBG] osdmap e2492: 8 total, 8 up, 8 in 2024-09-13T00:48:59.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:48:59 smithi136 bash[76371]: cluster 2024-09-13T00:48:58.055069+0000 mon.a (mon.0) 4264 : cluster [DBG] osdmap e2492: 8 total, 8 up, 8 in 2024-09-13T00:48:59.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:48:59 smithi136 bash[75164]: cluster 2024-09-13T00:48:58.055069+0000 mon.a (mon.0) 4264 : cluster [DBG] osdmap e2492: 8 total, 8 up, 8 in 2024-09-13T00:48:59.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:48:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:48:59] "GET /metrics HTTP/1.1" 200 36743 "" "Prometheus/2.43.0" 2024-09-13T00:49:00.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:00 smithi181 bash[67587]: cluster 2024-09-13T00:48:58.663268+0000 mgr.y (mgr.24661) 2155 : cluster [DBG] pgmap v4208: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 64 KiB/s rd, 5.0 KiB/s wr, 89 op/s 2024-09-13T00:49:00.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:00 smithi181 bash[67587]: cluster 2024-09-13T00:48:59.081656+0000 mon.a (mon.0) 4265 : cluster [DBG] osdmap e2493: 8 total, 8 up, 8 in 2024-09-13T00:49:00.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:00 smithi181 bash[67587]: audit 2024-09-13T00:48:59.143542+0000 mon.a (mon.0) 4266 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:49:00.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:00 smithi181 bash[67587]: audit 2024-09-13T00:48:59.682266+0000 mon.a (mon.0) 4267 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:49:00.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:00 smithi181 bash[67587]: audit 2024-09-13T00:48:59.683718+0000 mon.a (mon.0) 4268 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:49:00.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:00 smithi181 bash[67587]: audit 2024-09-13T00:48:59.690908+0000 mon.a (mon.0) 4269 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:49:00.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:00 smithi181 bash[67587]: cluster 2024-09-13T00:48:59.983133+0000 mon.a (mon.0) 4270 : cluster [DBG] osdmap e2494: 8 total, 8 up, 8 in 2024-09-13T00:49:00.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:00 smithi136 bash[76371]: cluster 2024-09-13T00:48:58.663268+0000 mgr.y (mgr.24661) 2155 : cluster [DBG] pgmap v4208: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 64 KiB/s rd, 5.0 KiB/s wr, 89 op/s 2024-09-13T00:49:00.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:00 smithi136 bash[76371]: cluster 2024-09-13T00:48:59.081656+0000 mon.a (mon.0) 4265 : cluster [DBG] osdmap e2493: 8 total, 8 up, 8 in 2024-09-13T00:49:00.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:00 smithi136 bash[76371]: audit 2024-09-13T00:48:59.143542+0000 mon.a (mon.0) 4266 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:49:00.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:00 smithi136 bash[76371]: audit 2024-09-13T00:48:59.682266+0000 mon.a (mon.0) 4267 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:49:00.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:00 smithi136 bash[76371]: audit 2024-09-13T00:48:59.683718+0000 mon.a (mon.0) 4268 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:49:00.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:00 smithi136 bash[76371]: audit 2024-09-13T00:48:59.690908+0000 mon.a (mon.0) 4269 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:49:00.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:00 smithi136 bash[76371]: cluster 2024-09-13T00:48:59.983133+0000 mon.a (mon.0) 4270 : cluster [DBG] osdmap e2494: 8 total, 8 up, 8 in 2024-09-13T00:49:00.511 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:00 smithi136 bash[75164]: cluster 2024-09-13T00:48:58.663268+0000 mgr.y (mgr.24661) 2155 : cluster [DBG] pgmap v4208: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 64 KiB/s rd, 5.0 KiB/s wr, 89 op/s 2024-09-13T00:49:00.511 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:00 smithi136 bash[75164]: cluster 2024-09-13T00:48:59.081656+0000 mon.a (mon.0) 4265 : cluster [DBG] osdmap e2493: 8 total, 8 up, 8 in 2024-09-13T00:49:00.511 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:00 smithi136 bash[75164]: audit 2024-09-13T00:48:59.143542+0000 mon.a (mon.0) 4266 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:49:00.511 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:00 smithi136 bash[75164]: audit 2024-09-13T00:48:59.682266+0000 mon.a (mon.0) 4267 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:49:00.511 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:00 smithi136 bash[75164]: audit 2024-09-13T00:48:59.683718+0000 mon.a (mon.0) 4268 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:49:00.511 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:00 smithi136 bash[75164]: audit 2024-09-13T00:48:59.690908+0000 mon.a (mon.0) 4269 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:49:00.511 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:00 smithi136 bash[75164]: cluster 2024-09-13T00:48:59.983133+0000 mon.a (mon.0) 4270 : cluster [DBG] osdmap e2494: 8 total, 8 up, 8 in 2024-09-13T00:49:01.205 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_set_snap_deleted PASSED [ 48%] 2024-09-13T00:49:02.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:01 smithi136 bash[76371]: cluster 2024-09-13T00:49:00.664525+0000 mgr.y (mgr.24661) 2156 : cluster [DBG] pgmap v4211: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 80 KiB/s rd, 8.5 KiB/s wr, 113 op/s 2024-09-13T00:49:02.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:01 smithi136 bash[76371]: cluster 2024-09-13T00:49:01.002257+0000 mon.a (mon.0) 4271 : cluster [DBG] osdmap e2495: 8 total, 8 up, 8 in 2024-09-13T00:49:02.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:01 smithi136 bash[75164]: cluster 2024-09-13T00:49:00.664525+0000 mgr.y (mgr.24661) 2156 : cluster [DBG] pgmap v4211: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 80 KiB/s rd, 8.5 KiB/s wr, 113 op/s 2024-09-13T00:49:02.261 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:01 smithi136 bash[75164]: cluster 2024-09-13T00:49:01.002257+0000 mon.a (mon.0) 4271 : cluster [DBG] osdmap e2495: 8 total, 8 up, 8 in 2024-09-13T00:49:02.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:01 smithi181 bash[67587]: cluster 2024-09-13T00:49:00.664525+0000 mgr.y (mgr.24661) 2156 : cluster [DBG] pgmap v4211: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 80 KiB/s rd, 8.5 KiB/s wr, 113 op/s 2024-09-13T00:49:02.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:01 smithi181 bash[67587]: cluster 2024-09-13T00:49:01.002257+0000 mon.a (mon.0) 4271 : cluster [DBG] osdmap e2495: 8 total, 8 up, 8 in 2024-09-13T00:49:02.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:49:02 smithi181 bash[95505]: ts=2024-09-13T00:49:02.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:49:03.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:03 smithi136 bash[75164]: cluster 2024-09-13T00:49:02.105892+0000 mon.a (mon.0) 4272 : cluster [DBG] osdmap e2496: 8 total, 8 up, 8 in 2024-09-13T00:49:03.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:03 smithi136 bash[76371]: cluster 2024-09-13T00:49:02.105892+0000 mon.a (mon.0) 4272 : cluster [DBG] osdmap e2496: 8 total, 8 up, 8 in 2024-09-13T00:49:03.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:03 smithi181 bash[67587]: cluster 2024-09-13T00:49:02.105892+0000 mon.a (mon.0) 4272 : cluster [DBG] osdmap e2496: 8 total, 8 up, 8 in 2024-09-13T00:49:04.512 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:04 smithi136 bash[76371]: cluster 2024-09-13T00:49:02.665252+0000 mgr.y (mgr.24661) 2157 : cluster [DBG] pgmap v4214: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 58 KiB/s rd, 7.5 KiB/s wr, 82 op/s 2024-09-13T00:49:04.512 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:04 smithi136 bash[76371]: cluster 2024-09-13T00:49:03.117967+0000 mon.a (mon.0) 4273 : cluster [DBG] osdmap e2497: 8 total, 8 up, 8 in 2024-09-13T00:49:04.513 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:04 smithi136 bash[75164]: cluster 2024-09-13T00:49:02.665252+0000 mgr.y (mgr.24661) 2157 : cluster [DBG] pgmap v4214: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 58 KiB/s rd, 7.5 KiB/s wr, 82 op/s 2024-09-13T00:49:04.513 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:04 smithi136 bash[75164]: cluster 2024-09-13T00:49:03.117967+0000 mon.a (mon.0) 4273 : cluster [DBG] osdmap e2497: 8 total, 8 up, 8 in 2024-09-13T00:49:04.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:04 smithi181 bash[67587]: cluster 2024-09-13T00:49:02.665252+0000 mgr.y (mgr.24661) 2157 : cluster [DBG] pgmap v4214: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 58 KiB/s rd, 7.5 KiB/s wr, 82 op/s 2024-09-13T00:49:04.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:04 smithi181 bash[67587]: cluster 2024-09-13T00:49:03.117967+0000 mon.a (mon.0) 4273 : cluster [DBG] osdmap e2497: 8 total, 8 up, 8 in 2024-09-13T00:49:05.209 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_set_snap_recreated PASSED [ 48%] 2024-09-13T00:49:05.446 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_lock_unlock PASSED [ 49%] 2024-09-13T00:49:05.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:05 smithi136 bash[76371]: cluster 2024-09-13T00:49:04.123448+0000 mon.a (mon.0) 4274 : cluster [DBG] osdmap e2498: 8 total, 8 up, 8 in 2024-09-13T00:49:05.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:05 smithi136 bash[76371]: cluster 2024-09-13T00:49:04.985931+0000 mon.a (mon.0) 4275 : cluster [DBG] osdmap e2499: 8 total, 8 up, 8 in 2024-09-13T00:49:05.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:05 smithi136 bash[75164]: cluster 2024-09-13T00:49:04.123448+0000 mon.a (mon.0) 4274 : cluster [DBG] osdmap e2498: 8 total, 8 up, 8 in 2024-09-13T00:49:05.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:05 smithi136 bash[75164]: cluster 2024-09-13T00:49:04.985931+0000 mon.a (mon.0) 4275 : cluster [DBG] osdmap e2499: 8 total, 8 up, 8 in 2024-09-13T00:49:05.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:05 smithi181 bash[67587]: cluster 2024-09-13T00:49:04.123448+0000 mon.a (mon.0) 4274 : cluster [DBG] osdmap e2498: 8 total, 8 up, 8 in 2024-09-13T00:49:05.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:05 smithi181 bash[67587]: cluster 2024-09-13T00:49:04.985931+0000 mon.a (mon.0) 4275 : cluster [DBG] osdmap e2499: 8 total, 8 up, 8 in 2024-09-13T00:49:05.848 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_list_lockers PASSED [ 50%] 2024-09-13T00:49:06.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:06 smithi136 bash[75164]: cluster 2024-09-13T00:49:04.665932+0000 mgr.y (mgr.24661) 2158 : cluster [DBG] pgmap v4217: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 48 KiB/s rd, 4.7 KiB/s wr, 67 op/s 2024-09-13T00:49:06.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:06 smithi136 bash[76371]: cluster 2024-09-13T00:49:04.665932+0000 mgr.y (mgr.24661) 2158 : cluster [DBG] pgmap v4217: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 48 KiB/s rd, 4.7 KiB/s wr, 67 op/s 2024-09-13T00:49:06.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:06 smithi181 bash[67587]: cluster 2024-09-13T00:49:04.665932+0000 mgr.y (mgr.24661) 2158 : cluster [DBG] pgmap v4217: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 48 KiB/s rd, 4.7 KiB/s wr, 67 op/s 2024-09-13T00:49:07.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:07 smithi136 bash[75164]: cluster 2024-09-13T00:49:06.134101+0000 mon.a (mon.0) 4276 : cluster [DBG] osdmap e2500: 8 total, 8 up, 8 in 2024-09-13T00:49:07.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:07 smithi136 bash[76371]: cluster 2024-09-13T00:49:06.134101+0000 mon.a (mon.0) 4276 : cluster [DBG] osdmap e2500: 8 total, 8 up, 8 in 2024-09-13T00:49:07.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:07 smithi181 bash[67587]: cluster 2024-09-13T00:49:06.134101+0000 mon.a (mon.0) 4276 : cluster [DBG] osdmap e2500: 8 total, 8 up, 8 in 2024-09-13T00:49:08.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:08 smithi136 bash[75164]: cluster 2024-09-13T00:49:06.666734+0000 mgr.y (mgr.24661) 2159 : cluster [DBG] pgmap v4220: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 86 KiB/s rd, 12 KiB/s wr, 127 op/s 2024-09-13T00:49:08.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:08 smithi136 bash[75164]: cluster 2024-09-13T00:49:07.161847+0000 mon.a (mon.0) 4277 : cluster [DBG] osdmap e2501: 8 total, 8 up, 8 in 2024-09-13T00:49:08.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:08 smithi136 bash[76371]: cluster 2024-09-13T00:49:06.666734+0000 mgr.y (mgr.24661) 2159 : cluster [DBG] pgmap v4220: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 86 KiB/s rd, 12 KiB/s wr, 127 op/s 2024-09-13T00:49:08.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:08 smithi136 bash[76371]: cluster 2024-09-13T00:49:07.161847+0000 mon.a (mon.0) 4277 : cluster [DBG] osdmap e2501: 8 total, 8 up, 8 in 2024-09-13T00:49:08.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:08 smithi181 bash[67587]: cluster 2024-09-13T00:49:06.666734+0000 mgr.y (mgr.24661) 2159 : cluster [DBG] pgmap v4220: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 86 KiB/s rd, 12 KiB/s wr, 127 op/s 2024-09-13T00:49:08.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:08 smithi181 bash[67587]: cluster 2024-09-13T00:49:07.161847+0000 mon.a (mon.0) 4277 : cluster [DBG] osdmap e2501: 8 total, 8 up, 8 in 2024-09-13T00:49:09.385 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_diff_iterate PASSED [ 50%] 2024-09-13T00:49:09.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:09 smithi136 bash[76371]: cluster 2024-09-13T00:49:08.166985+0000 mon.a (mon.0) 4278 : cluster [DBG] osdmap e2502: 8 total, 8 up, 8 in 2024-09-13T00:49:09.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:09 smithi136 bash[76371]: audit 2024-09-13T00:49:08.853136+0000 mon.a (mon.0) 4279 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:49:09.510 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:49:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:49:09] "GET /metrics HTTP/1.1" 200 36741 "" "Prometheus/2.43.0" 2024-09-13T00:49:09.511 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:09 smithi136 bash[75164]: cluster 2024-09-13T00:49:08.166985+0000 mon.a (mon.0) 4278 : cluster [DBG] osdmap e2502: 8 total, 8 up, 8 in 2024-09-13T00:49:09.511 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:09 smithi136 bash[75164]: audit 2024-09-13T00:49:08.853136+0000 mon.a (mon.0) 4279 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:49:09.593 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_aio_read PASSED [ 51%] 2024-09-13T00:49:09.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:09 smithi181 bash[67587]: cluster 2024-09-13T00:49:08.166985+0000 mon.a (mon.0) 4278 : cluster [DBG] osdmap e2502: 8 total, 8 up, 8 in 2024-09-13T00:49:09.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:09 smithi181 bash[67587]: audit 2024-09-13T00:49:08.853136+0000 mon.a (mon.0) 4279 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:49:09.830 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_aio_write PASSED [ 51%] 2024-09-13T00:49:10.062 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_aio_discard PASSED [ 52%] 2024-09-13T00:49:10.290 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_aio_write_zeroes PASSED [ 52%] 2024-09-13T00:49:10.508 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_aio_flush PASSED [ 53%] 2024-09-13T00:49:10.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:10 smithi136 bash[76371]: cluster 2024-09-13T00:49:08.667747+0000 mgr.y (mgr.24661) 2160 : cluster [DBG] pgmap v4223: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 126 KiB/s rd, 13 KiB/s wr, 177 op/s 2024-09-13T00:49:10.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:10 smithi136 bash[76371]: cluster 2024-09-13T00:49:09.179387+0000 mon.a (mon.0) 4280 : cluster [DBG] osdmap e2503: 8 total, 8 up, 8 in 2024-09-13T00:49:10.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:10 smithi136 bash[76371]: cluster 2024-09-13T00:49:09.990680+0000 mon.a (mon.0) 4281 : cluster [DBG] osdmap e2504: 8 total, 8 up, 8 in 2024-09-13T00:49:10.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:10 smithi136 bash[75164]: cluster 2024-09-13T00:49:08.667747+0000 mgr.y (mgr.24661) 2160 : cluster [DBG] pgmap v4223: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 126 KiB/s rd, 13 KiB/s wr, 177 op/s 2024-09-13T00:49:10.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:10 smithi136 bash[75164]: cluster 2024-09-13T00:49:09.179387+0000 mon.a (mon.0) 4280 : cluster [DBG] osdmap e2503: 8 total, 8 up, 8 in 2024-09-13T00:49:10.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:10 smithi136 bash[75164]: cluster 2024-09-13T00:49:09.990680+0000 mon.a (mon.0) 4281 : cluster [DBG] osdmap e2504: 8 total, 8 up, 8 in 2024-09-13T00:49:10.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:10 smithi181 bash[67587]: cluster 2024-09-13T00:49:08.667747+0000 mgr.y (mgr.24661) 2160 : cluster [DBG] pgmap v4223: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 126 KiB/s rd, 13 KiB/s wr, 177 op/s 2024-09-13T00:49:10.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:10 smithi181 bash[67587]: cluster 2024-09-13T00:49:09.179387+0000 mon.a (mon.0) 4280 : cluster [DBG] osdmap e2503: 8 total, 8 up, 8 in 2024-09-13T00:49:10.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:10 smithi181 bash[67587]: cluster 2024-09-13T00:49:09.990680+0000 mon.a (mon.0) 4281 : cluster [DBG] osdmap e2504: 8 total, 8 up, 8 in 2024-09-13T00:49:12.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:12 smithi136 bash[75164]: cluster 2024-09-13T00:49:10.668816+0000 mgr.y (mgr.24661) 2161 : cluster [DBG] pgmap v4226: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 213 KiB/s rd, 7.0 KiB/s wr, 276 op/s 2024-09-13T00:49:12.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:12 smithi136 bash[76371]: cluster 2024-09-13T00:49:10.668816+0000 mgr.y (mgr.24661) 2161 : cluster [DBG] pgmap v4226: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 213 KiB/s rd, 7.0 KiB/s wr, 276 op/s 2024-09-13T00:49:12.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:12 smithi181 bash[67587]: cluster 2024-09-13T00:49:10.668816+0000 mgr.y (mgr.24661) 2161 : cluster [DBG] pgmap v4226: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 213 KiB/s rd, 7.0 KiB/s wr, 276 op/s 2024-09-13T00:49:12.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:49:12 smithi181 bash[95505]: ts=2024-09-13T00:49:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:49:12.639 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_metadata PASSED [ 53%] 2024-09-13T00:49:12.846 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_watchers_list PASSED [ 54%] 2024-09-13T00:49:13.141 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_config_list PASSED [ 54%] 2024-09-13T00:49:13.415 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_image_config_set_and_get_and_remove PASSED [ 55%] 2024-09-13T00:49:13.628 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_sparsify PASSED [ 56%] 2024-09-13T00:49:14.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:14 smithi136 bash[75164]: cluster 2024-09-13T00:49:12.669850+0000 mgr.y (mgr.24661) 2162 : cluster [DBG] pgmap v4227: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 227 KiB/s rd, 11 KiB/s wr, 294 op/s 2024-09-13T00:49:14.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:14 smithi136 bash[76371]: cluster 2024-09-13T00:49:12.669850+0000 mgr.y (mgr.24661) 2162 : cluster [DBG] pgmap v4227: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 227 KiB/s rd, 11 KiB/s wr, 294 op/s 2024-09-13T00:49:14.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:14 smithi181 bash[67587]: cluster 2024-09-13T00:49:12.669850+0000 mgr.y (mgr.24661) 2162 : cluster [DBG] pgmap v4227: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 227 KiB/s rd, 11 KiB/s wr, 294 op/s 2024-09-13T00:49:16.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:15 smithi136 bash[76371]: cluster 2024-09-13T00:49:14.670464+0000 mgr.y (mgr.24661) 2163 : cluster [DBG] pgmap v4228: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 266 KiB/s rd, 14 KiB/s wr, 349 op/s 2024-09-13T00:49:16.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:15 smithi136 bash[76371]: cluster 2024-09-13T00:49:14.986904+0000 mon.a (mon.0) 4282 : cluster [DBG] osdmap e2505: 8 total, 8 up, 8 in 2024-09-13T00:49:16.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:15 smithi136 bash[75164]: cluster 2024-09-13T00:49:14.670464+0000 mgr.y (mgr.24661) 2163 : cluster [DBG] pgmap v4228: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 266 KiB/s rd, 14 KiB/s wr, 349 op/s 2024-09-13T00:49:16.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:15 smithi136 bash[75164]: cluster 2024-09-13T00:49:14.986904+0000 mon.a (mon.0) 4282 : cluster [DBG] osdmap e2505: 8 total, 8 up, 8 in 2024-09-13T00:49:16.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:15 smithi181 bash[67587]: cluster 2024-09-13T00:49:14.670464+0000 mgr.y (mgr.24661) 2163 : cluster [DBG] pgmap v4228: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 266 KiB/s rd, 14 KiB/s wr, 349 op/s 2024-09-13T00:49:16.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:15 smithi181 bash[67587]: cluster 2024-09-13T00:49:14.986904+0000 mon.a (mon.0) 4282 : cluster [DBG] osdmap e2505: 8 total, 8 up, 8 in 2024-09-13T00:49:18.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:17 smithi136 bash[75164]: cluster 2024-09-13T00:49:16.671069+0000 mgr.y (mgr.24661) 2164 : cluster [DBG] pgmap v4230: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 1.7 MiB/s rd, 34 KiB/s wr, 717 op/s 2024-09-13T00:49:18.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:17 smithi136 bash[76371]: cluster 2024-09-13T00:49:16.671069+0000 mgr.y (mgr.24661) 2164 : cluster [DBG] pgmap v4230: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 1.7 MiB/s rd, 34 KiB/s wr, 717 op/s 2024-09-13T00:49:18.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:17 smithi181 bash[67587]: cluster 2024-09-13T00:49:16.671069+0000 mgr.y (mgr.24661) 2164 : cluster [DBG] pgmap v4230: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 1.7 MiB/s rd, 34 KiB/s wr, 717 op/s 2024-09-13T00:49:19.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:49:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:49:19] "GET /metrics HTTP/1.1" 200 36741 "" "Prometheus/2.43.0" 2024-09-13T00:49:20.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:19 smithi136 bash[75164]: cluster 2024-09-13T00:49:18.672047+0000 mgr.y (mgr.24661) 2165 : cluster [DBG] pgmap v4231: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 1.5 MiB/s rd, 30 KiB/s wr, 655 op/s 2024-09-13T00:49:20.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:19 smithi136 bash[76371]: cluster 2024-09-13T00:49:18.672047+0000 mgr.y (mgr.24661) 2165 : cluster [DBG] pgmap v4231: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 1.5 MiB/s rd, 30 KiB/s wr, 655 op/s 2024-09-13T00:49:20.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:19 smithi181 bash[67587]: cluster 2024-09-13T00:49:18.672047+0000 mgr.y (mgr.24661) 2165 : cluster [DBG] pgmap v4231: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 1.5 MiB/s rd, 30 KiB/s wr, 655 op/s 2024-09-13T00:49:21.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:20 smithi136 bash[75164]: cluster 2024-09-13T00:49:19.992601+0000 mon.a (mon.0) 4283 : cluster [DBG] osdmap e2506: 8 total, 8 up, 8 in 2024-09-13T00:49:21.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:20 smithi136 bash[76371]: cluster 2024-09-13T00:49:19.992601+0000 mon.a (mon.0) 4283 : cluster [DBG] osdmap e2506: 8 total, 8 up, 8 in 2024-09-13T00:49:21.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:20 smithi181 bash[67587]: cluster 2024-09-13T00:49:19.992601+0000 mon.a (mon.0) 4283 : cluster [DBG] osdmap e2506: 8 total, 8 up, 8 in 2024-09-13T00:49:21.946 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_encryption_luks1 PASSED [ 56%] 2024-09-13T00:49:22.260 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:22 smithi136 bash[76371]: cluster 2024-09-13T00:49:20.672594+0000 mgr.y (mgr.24661) 2166 : cluster [DBG] pgmap v4233: 137 pgs: 137 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 1.6 MiB/s rd, 538 KiB/s wr, 555 op/s 2024-09-13T00:49:22.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:22 smithi136 bash[75164]: cluster 2024-09-13T00:49:20.672594+0000 mgr.y (mgr.24661) 2166 : cluster [DBG] pgmap v4233: 137 pgs: 137 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 1.6 MiB/s rd, 538 KiB/s wr, 555 op/s 2024-09-13T00:49:22.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:22 smithi181 bash[67587]: cluster 2024-09-13T00:49:20.672594+0000 mgr.y (mgr.24661) 2166 : cluster [DBG] pgmap v4233: 137 pgs: 137 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 1.6 MiB/s rd, 538 KiB/s wr, 555 op/s 2024-09-13T00:49:22.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:49:22 smithi181 bash[95505]: ts=2024-09-13T00:49:22.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:49:24.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:24 smithi181 bash[67587]: cluster 2024-09-13T00:49:22.673485+0000 mgr.y (mgr.24661) 2167 : cluster [DBG] pgmap v4234: 137 pgs: 137 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 1.5 MiB/s rd, 534 KiB/s wr, 451 op/s 2024-09-13T00:49:24.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:24 smithi181 bash[67587]: audit 2024-09-13T00:49:23.853340+0000 mon.a (mon.0) 4284 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:49:24.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:24 smithi136 bash[75164]: cluster 2024-09-13T00:49:22.673485+0000 mgr.y (mgr.24661) 2167 : cluster [DBG] pgmap v4234: 137 pgs: 137 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 1.5 MiB/s rd, 534 KiB/s wr, 451 op/s 2024-09-13T00:49:24.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:24 smithi136 bash[75164]: audit 2024-09-13T00:49:23.853340+0000 mon.a (mon.0) 4284 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:49:24.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:24 smithi136 bash[76371]: cluster 2024-09-13T00:49:22.673485+0000 mgr.y (mgr.24661) 2167 : cluster [DBG] pgmap v4234: 137 pgs: 137 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 1.5 MiB/s rd, 534 KiB/s wr, 451 op/s 2024-09-13T00:49:24.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:24 smithi136 bash[76371]: audit 2024-09-13T00:49:23.853340+0000 mon.a (mon.0) 4284 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:49:26.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:26 smithi181 bash[67587]: cluster 2024-09-13T00:49:24.674187+0000 mgr.y (mgr.24661) 2168 : cluster [DBG] pgmap v4235: 137 pgs: 137 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 1.2 MiB/s rd, 441 KiB/s wr, 370 op/s 2024-09-13T00:49:26.512 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:26 smithi136 bash[75164]: cluster 2024-09-13T00:49:24.674187+0000 mgr.y (mgr.24661) 2168 : cluster [DBG] pgmap v4235: 137 pgs: 137 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 1.2 MiB/s rd, 441 KiB/s wr, 370 op/s 2024-09-13T00:49:26.513 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:26 smithi136 bash[76371]: cluster 2024-09-13T00:49:24.674187+0000 mgr.y (mgr.24661) 2168 : cluster [DBG] pgmap v4235: 137 pgs: 137 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 1.2 MiB/s rd, 441 KiB/s wr, 370 op/s 2024-09-13T00:49:28.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:28 smithi181 bash[67587]: cluster 2024-09-13T00:49:26.674996+0000 mgr.y (mgr.24661) 2169 : cluster [DBG] pgmap v4236: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 116 KiB/s rd, 412 KiB/s wr, 76 op/s 2024-09-13T00:49:28.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:28 smithi136 bash[75164]: cluster 2024-09-13T00:49:26.674996+0000 mgr.y (mgr.24661) 2169 : cluster [DBG] pgmap v4236: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 116 KiB/s rd, 412 KiB/s wr, 76 op/s 2024-09-13T00:49:28.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:28 smithi136 bash[76371]: cluster 2024-09-13T00:49:26.674996+0000 mgr.y (mgr.24661) 2169 : cluster [DBG] pgmap v4236: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 116 KiB/s rd, 412 KiB/s wr, 76 op/s 2024-09-13T00:49:29.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:49:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:49:29] "GET /metrics HTTP/1.1" 200 36742 "" "Prometheus/2.43.0" 2024-09-13T00:49:30.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:30 smithi181 bash[67587]: cluster 2024-09-13T00:49:28.675736+0000 mgr.y (mgr.24661) 2170 : cluster [DBG] pgmap v4237: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 83 KiB/s rd, 412 KiB/s wr, 34 op/s 2024-09-13T00:49:30.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:30 smithi136 bash[75164]: cluster 2024-09-13T00:49:28.675736+0000 mgr.y (mgr.24661) 2170 : cluster [DBG] pgmap v4237: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 83 KiB/s rd, 412 KiB/s wr, 34 op/s 2024-09-13T00:49:30.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:30 smithi136 bash[76371]: cluster 2024-09-13T00:49:28.675736+0000 mgr.y (mgr.24661) 2170 : cluster [DBG] pgmap v4237: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 83 KiB/s rd, 412 KiB/s wr, 34 op/s 2024-09-13T00:49:32.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:32 smithi181 bash[67587]: cluster 2024-09-13T00:49:30.676443+0000 mgr.y (mgr.24661) 2171 : cluster [DBG] pgmap v4238: 137 pgs: 137 active+clean; 17 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 109 KiB/s rd, 1.5 MiB/s wr, 39 op/s 2024-09-13T00:49:32.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:32 smithi136 bash[75164]: cluster 2024-09-13T00:49:30.676443+0000 mgr.y (mgr.24661) 2171 : cluster [DBG] pgmap v4238: 137 pgs: 137 active+clean; 17 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 109 KiB/s rd, 1.5 MiB/s wr, 39 op/s 2024-09-13T00:49:32.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:32 smithi136 bash[76371]: cluster 2024-09-13T00:49:30.676443+0000 mgr.y (mgr.24661) 2171 : cluster [DBG] pgmap v4238: 137 pgs: 137 active+clean; 17 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 109 KiB/s rd, 1.5 MiB/s wr, 39 op/s 2024-09-13T00:49:32.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:49:32 smithi181 bash[95505]: ts=2024-09-13T00:49:32.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:49:33.421 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_encryption_luks2 PASSED [ 57%] 2024-09-13T00:49:33.645 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImageId::test_read PASSED [ 57%] 2024-09-13T00:49:33.889 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImageId::test_write PASSED [ 58%] 2024-09-13T00:49:34.145 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestImageId::test_resize PASSED [ 58%] 2024-09-13T00:49:34.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:34 smithi181 bash[67587]: cluster 2024-09-13T00:49:32.677165+0000 mgr.y (mgr.24661) 2172 : cluster [DBG] pgmap v4239: 137 pgs: 137 active+clean; 21 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 44 KiB/s rd, 1.3 MiB/s wr, 27 op/s 2024-09-13T00:49:34.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:34 smithi136 bash[75164]: cluster 2024-09-13T00:49:32.677165+0000 mgr.y (mgr.24661) 2172 : cluster [DBG] pgmap v4239: 137 pgs: 137 active+clean; 21 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 44 KiB/s rd, 1.3 MiB/s wr, 27 op/s 2024-09-13T00:49:34.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:34 smithi136 bash[76371]: cluster 2024-09-13T00:49:32.677165+0000 mgr.y (mgr.24661) 2172 : cluster [DBG] pgmap v4239: 137 pgs: 137 active+clean; 21 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 44 KiB/s rd, 1.3 MiB/s wr, 27 op/s 2024-09-13T00:49:36.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:36 smithi181 bash[67587]: cluster 2024-09-13T00:49:34.678182+0000 mgr.y (mgr.24661) 2173 : cluster [DBG] pgmap v4240: 137 pgs: 137 active+clean; 21 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 44 KiB/s rd, 1.3 MiB/s wr, 27 op/s 2024-09-13T00:49:36.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:36 smithi181 bash[67587]: cluster 2024-09-13T00:49:35.044308+0000 mon.a (mon.0) 4285 : cluster [DBG] osdmap e2507: 8 total, 8 up, 8 in 2024-09-13T00:49:36.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:36 smithi136 bash[75164]: cluster 2024-09-13T00:49:34.678182+0000 mgr.y (mgr.24661) 2173 : cluster [DBG] pgmap v4240: 137 pgs: 137 active+clean; 21 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 44 KiB/s rd, 1.3 MiB/s wr, 27 op/s 2024-09-13T00:49:36.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:36 smithi136 bash[75164]: cluster 2024-09-13T00:49:35.044308+0000 mon.a (mon.0) 4285 : cluster [DBG] osdmap e2507: 8 total, 8 up, 8 in 2024-09-13T00:49:36.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:36 smithi136 bash[76371]: cluster 2024-09-13T00:49:34.678182+0000 mgr.y (mgr.24661) 2173 : cluster [DBG] pgmap v4240: 137 pgs: 137 active+clean; 21 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 44 KiB/s rd, 1.3 MiB/s wr, 27 op/s 2024-09-13T00:49:36.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:36 smithi136 bash[76371]: cluster 2024-09-13T00:49:35.044308+0000 mon.a (mon.0) 4285 : cluster [DBG] osdmap e2507: 8 total, 8 up, 8 in 2024-09-13T00:49:37.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:37 smithi181 bash[67587]: cluster 2024-09-13T00:49:36.055104+0000 mon.a (mon.0) 4286 : cluster [DBG] osdmap e2508: 8 total, 8 up, 8 in 2024-09-13T00:49:37.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:37 smithi136 bash[75164]: cluster 2024-09-13T00:49:36.055104+0000 mon.a (mon.0) 4286 : cluster [DBG] osdmap e2508: 8 total, 8 up, 8 in 2024-09-13T00:49:37.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:37 smithi136 bash[76371]: cluster 2024-09-13T00:49:36.055104+0000 mon.a (mon.0) 4286 : cluster [DBG] osdmap e2508: 8 total, 8 up, 8 in 2024-09-13T00:49:38.293 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestClone::test_with_params PASSED [ 59%] 2024-09-13T00:49:38.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:38 smithi181 bash[67587]: cluster 2024-09-13T00:49:36.678933+0000 mgr.y (mgr.24661) 2174 : cluster [DBG] pgmap v4243: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 143 KiB/s rd, 2.0 MiB/s wr, 99 op/s 2024-09-13T00:49:38.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:38 smithi181 bash[67587]: cluster 2024-09-13T00:49:37.066145+0000 mon.a (mon.0) 4287 : cluster [DBG] osdmap e2509: 8 total, 8 up, 8 in 2024-09-13T00:49:38.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:38 smithi136 bash[75164]: cluster 2024-09-13T00:49:36.678933+0000 mgr.y (mgr.24661) 2174 : cluster [DBG] pgmap v4243: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 143 KiB/s rd, 2.0 MiB/s wr, 99 op/s 2024-09-13T00:49:38.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:38 smithi136 bash[75164]: cluster 2024-09-13T00:49:37.066145+0000 mon.a (mon.0) 4287 : cluster [DBG] osdmap e2509: 8 total, 8 up, 8 in 2024-09-13T00:49:38.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:38 smithi136 bash[76371]: cluster 2024-09-13T00:49:36.678933+0000 mgr.y (mgr.24661) 2174 : cluster [DBG] pgmap v4243: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 143 KiB/s rd, 2.0 MiB/s wr, 99 op/s 2024-09-13T00:49:38.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:38 smithi136 bash[76371]: cluster 2024-09-13T00:49:37.066145+0000 mon.a (mon.0) 4287 : cluster [DBG] osdmap e2509: 8 total, 8 up, 8 in 2024-09-13T00:49:39.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:39 smithi181 bash[67587]: cluster 2024-09-13T00:49:38.067015+0000 mon.a (mon.0) 4288 : cluster [DBG] osdmap e2510: 8 total, 8 up, 8 in 2024-09-13T00:49:39.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:39 smithi181 bash[67587]: audit 2024-09-13T00:49:38.853771+0000 mon.a (mon.0) 4289 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:49:39.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:39 smithi136 bash[75164]: cluster 2024-09-13T00:49:38.067015+0000 mon.a (mon.0) 4288 : cluster [DBG] osdmap e2510: 8 total, 8 up, 8 in 2024-09-13T00:49:39.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:39 smithi136 bash[75164]: audit 2024-09-13T00:49:38.853771+0000 mon.a (mon.0) 4289 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:49:39.414 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:39 smithi136 bash[76371]: cluster 2024-09-13T00:49:38.067015+0000 mon.a (mon.0) 4288 : cluster [DBG] osdmap e2510: 8 total, 8 up, 8 in 2024-09-13T00:49:39.414 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:39 smithi136 bash[76371]: audit 2024-09-13T00:49:38.853771+0000 mon.a (mon.0) 4289 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:49:39.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:49:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:49:39] "GET /metrics HTTP/1.1" 200 36746 "" "Prometheus/2.43.0" 2024-09-13T00:49:40.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:40 smithi181 bash[67587]: cluster 2024-09-13T00:49:38.679796+0000 mgr.y (mgr.24661) 2175 : cluster [DBG] pgmap v4246: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 315 KiB/s rd, 15 KiB/s wr, 326 op/s 2024-09-13T00:49:40.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:40 smithi181 bash[67587]: cluster 2024-09-13T00:49:39.085838+0000 mon.a (mon.0) 4290 : cluster [DBG] osdmap e2511: 8 total, 8 up, 8 in 2024-09-13T00:49:40.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:40 smithi136 bash[75164]: cluster 2024-09-13T00:49:38.679796+0000 mgr.y (mgr.24661) 2175 : cluster [DBG] pgmap v4246: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 315 KiB/s rd, 15 KiB/s wr, 326 op/s 2024-09-13T00:49:40.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:40 smithi136 bash[75164]: cluster 2024-09-13T00:49:39.085838+0000 mon.a (mon.0) 4290 : cluster [DBG] osdmap e2511: 8 total, 8 up, 8 in 2024-09-13T00:49:40.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:40 smithi136 bash[76371]: cluster 2024-09-13T00:49:38.679796+0000 mgr.y (mgr.24661) 2175 : cluster [DBG] pgmap v4246: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 315 KiB/s rd, 15 KiB/s wr, 326 op/s 2024-09-13T00:49:40.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:40 smithi136 bash[76371]: cluster 2024-09-13T00:49:39.085838+0000 mon.a (mon.0) 4290 : cluster [DBG] osdmap e2511: 8 total, 8 up, 8 in 2024-09-13T00:49:41.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:41 smithi136 bash[75164]: cluster 2024-09-13T00:49:40.095801+0000 mon.a (mon.0) 4291 : cluster [DBG] osdmap e2512: 8 total, 8 up, 8 in 2024-09-13T00:49:41.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:41 smithi136 bash[76371]: cluster 2024-09-13T00:49:40.095801+0000 mon.a (mon.0) 4291 : cluster [DBG] osdmap e2512: 8 total, 8 up, 8 in 2024-09-13T00:49:41.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:41 smithi181 bash[67587]: cluster 2024-09-13T00:49:40.095801+0000 mon.a (mon.0) 4291 : cluster [DBG] osdmap e2512: 8 total, 8 up, 8 in 2024-09-13T00:49:42.309 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestClone::test_with_params2 PASSED [ 59%] 2024-09-13T00:49:42.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:42 smithi136 bash[75164]: cluster 2024-09-13T00:49:40.680798+0000 mgr.y (mgr.24661) 2176 : cluster [DBG] pgmap v4249: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 214 KiB/s rd, 10 KiB/s wr, 282 op/s 2024-09-13T00:49:42.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:42 smithi136 bash[75164]: cluster 2024-09-13T00:49:41.105008+0000 mon.a (mon.0) 4292 : cluster [DBG] osdmap e2513: 8 total, 8 up, 8 in 2024-09-13T00:49:42.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:42 smithi136 bash[76371]: cluster 2024-09-13T00:49:40.680798+0000 mgr.y (mgr.24661) 2176 : cluster [DBG] pgmap v4249: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 214 KiB/s rd, 10 KiB/s wr, 282 op/s 2024-09-13T00:49:42.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:42 smithi136 bash[76371]: cluster 2024-09-13T00:49:41.105008+0000 mon.a (mon.0) 4292 : cluster [DBG] osdmap e2513: 8 total, 8 up, 8 in 2024-09-13T00:49:42.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:49:42 smithi181 bash[95505]: ts=2024-09-13T00:49:42.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:49:42.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:42 smithi181 bash[67587]: cluster 2024-09-13T00:49:40.680798+0000 mgr.y (mgr.24661) 2176 : cluster [DBG] pgmap v4249: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 214 KiB/s rd, 10 KiB/s wr, 282 op/s 2024-09-13T00:49:42.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:42 smithi181 bash[67587]: cluster 2024-09-13T00:49:41.105008+0000 mon.a (mon.0) 4292 : cluster [DBG] osdmap e2513: 8 total, 8 up, 8 in 2024-09-13T00:49:43.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:43 smithi136 bash[75164]: cluster 2024-09-13T00:49:42.106687+0000 mon.a (mon.0) 4293 : cluster [DBG] osdmap e2514: 8 total, 8 up, 8 in 2024-09-13T00:49:43.541 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:43 smithi136 bash[76371]: cluster 2024-09-13T00:49:42.106687+0000 mon.a (mon.0) 4293 : cluster [DBG] osdmap e2514: 8 total, 8 up, 8 in 2024-09-13T00:49:43.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:43 smithi181 bash[67587]: cluster 2024-09-13T00:49:42.106687+0000 mon.a (mon.0) 4293 : cluster [DBG] osdmap e2514: 8 total, 8 up, 8 in 2024-09-13T00:49:44.365 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestClone::test_with_params3 SKIPPED [ 60%] 2024-09-13T00:49:44.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:44 smithi136 bash[75164]: cluster 2024-09-13T00:49:42.681481+0000 mgr.y (mgr.24661) 2177 : cluster [DBG] pgmap v4252: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 136 KiB/s rd, 12 KiB/s wr, 189 op/s 2024-09-13T00:49:44.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:44 smithi136 bash[75164]: cluster 2024-09-13T00:49:43.126265+0000 mon.a (mon.0) 4294 : cluster [DBG] osdmap e2515: 8 total, 8 up, 8 in 2024-09-13T00:49:44.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:44 smithi136 bash[76371]: cluster 2024-09-13T00:49:42.681481+0000 mgr.y (mgr.24661) 2177 : cluster [DBG] pgmap v4252: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 136 KiB/s rd, 12 KiB/s wr, 189 op/s 2024-09-13T00:49:44.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:44 smithi136 bash[76371]: cluster 2024-09-13T00:49:43.126265+0000 mon.a (mon.0) 4294 : cluster [DBG] osdmap e2515: 8 total, 8 up, 8 in 2024-09-13T00:49:44.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:44 smithi181 bash[67587]: cluster 2024-09-13T00:49:42.681481+0000 mgr.y (mgr.24661) 2177 : cluster [DBG] pgmap v4252: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 136 KiB/s rd, 12 KiB/s wr, 189 op/s 2024-09-13T00:49:44.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:44 smithi181 bash[67587]: cluster 2024-09-13T00:49:43.126265+0000 mon.a (mon.0) 4294 : cluster [DBG] osdmap e2515: 8 total, 8 up, 8 in 2024-09-13T00:49:45.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:45 smithi136 bash[75164]: cluster 2024-09-13T00:49:44.136647+0000 mon.a (mon.0) 4295 : cluster [DBG] osdmap e2516: 8 total, 8 up, 8 in 2024-09-13T00:49:45.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:45 smithi136 bash[75164]: cluster 2024-09-13T00:49:45.001966+0000 mon.a (mon.0) 4296 : cluster [DBG] osdmap e2517: 8 total, 8 up, 8 in 2024-09-13T00:49:45.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:45 smithi136 bash[76371]: cluster 2024-09-13T00:49:44.136647+0000 mon.a (mon.0) 4295 : cluster [DBG] osdmap e2516: 8 total, 8 up, 8 in 2024-09-13T00:49:45.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:45 smithi136 bash[76371]: cluster 2024-09-13T00:49:45.001966+0000 mon.a (mon.0) 4296 : cluster [DBG] osdmap e2517: 8 total, 8 up, 8 in 2024-09-13T00:49:45.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:45 smithi181 bash[67587]: cluster 2024-09-13T00:49:44.136647+0000 mon.a (mon.0) 4295 : cluster [DBG] osdmap e2516: 8 total, 8 up, 8 in 2024-09-13T00:49:45.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:45 smithi181 bash[67587]: cluster 2024-09-13T00:49:45.001966+0000 mon.a (mon.0) 4296 : cluster [DBG] osdmap e2517: 8 total, 8 up, 8 in 2024-09-13T00:49:46.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:46 smithi136 bash[75164]: cluster 2024-09-13T00:49:44.682184+0000 mgr.y (mgr.24661) 2178 : cluster [DBG] pgmap v4255: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 179 KiB/s rd, 13 KiB/s wr, 245 op/s 2024-09-13T00:49:46.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:46 smithi136 bash[76371]: cluster 2024-09-13T00:49:44.682184+0000 mgr.y (mgr.24661) 2178 : cluster [DBG] pgmap v4255: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 179 KiB/s rd, 13 KiB/s wr, 245 op/s 2024-09-13T00:49:46.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:46 smithi181 bash[67587]: cluster 2024-09-13T00:49:44.682184+0000 mgr.y (mgr.24661) 2178 : cluster [DBG] pgmap v4255: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 179 KiB/s rd, 13 KiB/s wr, 245 op/s 2024-09-13T00:49:47.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:47 smithi136 bash[75164]: cluster 2024-09-13T00:49:46.155783+0000 mon.a (mon.0) 4297 : cluster [DBG] osdmap e2518: 8 total, 8 up, 8 in 2024-09-13T00:49:47.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:47 smithi136 bash[76371]: cluster 2024-09-13T00:49:46.155783+0000 mon.a (mon.0) 4297 : cluster [DBG] osdmap e2518: 8 total, 8 up, 8 in 2024-09-13T00:49:47.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:47 smithi181 bash[67587]: cluster 2024-09-13T00:49:46.155783+0000 mon.a (mon.0) 4297 : cluster [DBG] osdmap e2518: 8 total, 8 up, 8 in 2024-09-13T00:49:48.403 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestClone::test_stripe_unit_and_count PASSED [ 60%] 2024-09-13T00:49:48.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:48 smithi136 bash[75164]: cluster 2024-09-13T00:49:46.683003+0000 mgr.y (mgr.24661) 2179 : cluster [DBG] pgmap v4258: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 311 KiB/s rd, 21 KiB/s wr, 420 op/s 2024-09-13T00:49:48.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:48 smithi136 bash[75164]: cluster 2024-09-13T00:49:47.169446+0000 mon.a (mon.0) 4298 : cluster [DBG] osdmap e2519: 8 total, 8 up, 8 in 2024-09-13T00:49:48.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:48 smithi136 bash[76371]: cluster 2024-09-13T00:49:46.683003+0000 mgr.y (mgr.24661) 2179 : cluster [DBG] pgmap v4258: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 311 KiB/s rd, 21 KiB/s wr, 420 op/s 2024-09-13T00:49:48.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:48 smithi136 bash[76371]: cluster 2024-09-13T00:49:47.169446+0000 mon.a (mon.0) 4298 : cluster [DBG] osdmap e2519: 8 total, 8 up, 8 in 2024-09-13T00:49:48.600 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:48 smithi181 bash[67587]: cluster 2024-09-13T00:49:46.683003+0000 mgr.y (mgr.24661) 2179 : cluster [DBG] pgmap v4258: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 311 KiB/s rd, 21 KiB/s wr, 420 op/s 2024-09-13T00:49:48.600 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:48 smithi181 bash[67587]: cluster 2024-09-13T00:49:47.169446+0000 mon.a (mon.0) 4298 : cluster [DBG] osdmap e2519: 8 total, 8 up, 8 in 2024-09-13T00:49:49.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:49 smithi136 bash[75164]: cluster 2024-09-13T00:49:48.180364+0000 mon.a (mon.0) 4299 : cluster [DBG] osdmap e2520: 8 total, 8 up, 8 in 2024-09-13T00:49:49.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:49 smithi136 bash[76371]: cluster 2024-09-13T00:49:48.180364+0000 mon.a (mon.0) 4299 : cluster [DBG] osdmap e2520: 8 total, 8 up, 8 in 2024-09-13T00:49:49.511 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:49:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:49:49] "GET /metrics HTTP/1.1" 200 36746 "" "Prometheus/2.43.0" 2024-09-13T00:49:49.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:49 smithi181 bash[67587]: cluster 2024-09-13T00:49:48.180364+0000 mon.a (mon.0) 4299 : cluster [DBG] osdmap e2520: 8 total, 8 up, 8 in 2024-09-13T00:49:50.216 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestClone::test_clone_format FAILED [ 61%] 2024-09-13T00:49:50.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:50 smithi136 bash[75164]: cluster 2024-09-13T00:49:48.683910+0000 mgr.y (mgr.24661) 2180 : cluster [DBG] pgmap v4261: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 237 KiB/s rd, 18 KiB/s wr, 322 op/s 2024-09-13T00:49:50.511 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:50 smithi136 bash[75164]: cluster 2024-09-13T00:49:49.186413+0000 mon.a (mon.0) 4300 : cluster [DBG] osdmap e2521: 8 total, 8 up, 8 in 2024-09-13T00:49:50.511 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:50 smithi136 bash[75164]: cluster 2024-09-13T00:49:50.007453+0000 mon.a (mon.0) 4301 : cluster [DBG] osdmap e2522: 8 total, 8 up, 8 in 2024-09-13T00:49:50.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:50 smithi136 bash[76371]: cluster 2024-09-13T00:49:48.683910+0000 mgr.y (mgr.24661) 2180 : cluster [DBG] pgmap v4261: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 237 KiB/s rd, 18 KiB/s wr, 322 op/s 2024-09-13T00:49:50.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:50 smithi136 bash[76371]: cluster 2024-09-13T00:49:49.186413+0000 mon.a (mon.0) 4300 : cluster [DBG] osdmap e2521: 8 total, 8 up, 8 in 2024-09-13T00:49:50.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:50 smithi136 bash[76371]: cluster 2024-09-13T00:49:50.007453+0000 mon.a (mon.0) 4301 : cluster [DBG] osdmap e2522: 8 total, 8 up, 8 in 2024-09-13T00:49:50.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:50 smithi181 bash[67587]: cluster 2024-09-13T00:49:48.683910+0000 mgr.y (mgr.24661) 2180 : cluster [DBG] pgmap v4261: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 237 KiB/s rd, 18 KiB/s wr, 322 op/s 2024-09-13T00:49:50.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:50 smithi181 bash[67587]: cluster 2024-09-13T00:49:49.186413+0000 mon.a (mon.0) 4300 : cluster [DBG] osdmap e2521: 8 total, 8 up, 8 in 2024-09-13T00:49:50.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:50 smithi181 bash[67587]: cluster 2024-09-13T00:49:50.007453+0000 mon.a (mon.0) 4301 : cluster [DBG] osdmap e2522: 8 total, 8 up, 8 in 2024-09-13T00:49:52.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:52 smithi136 bash[75164]: cluster 2024-09-13T00:49:50.685050+0000 mgr.y (mgr.24661) 2181 : cluster [DBG] pgmap v4264: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 217 KiB/s rd, 13 KiB/s wr, 298 op/s 2024-09-13T00:49:52.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:52 smithi136 bash[75164]: cluster 2024-09-13T00:49:51.195496+0000 mon.a (mon.0) 4302 : cluster [DBG] osdmap e2523: 8 total, 8 up, 8 in 2024-09-13T00:49:52.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:52 smithi136 bash[76371]: cluster 2024-09-13T00:49:50.685050+0000 mgr.y (mgr.24661) 2181 : cluster [DBG] pgmap v4264: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 217 KiB/s rd, 13 KiB/s wr, 298 op/s 2024-09-13T00:49:52.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:52 smithi136 bash[76371]: cluster 2024-09-13T00:49:51.195496+0000 mon.a (mon.0) 4302 : cluster [DBG] osdmap e2523: 8 total, 8 up, 8 in 2024-09-13T00:49:52.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:52 smithi181 bash[67587]: cluster 2024-09-13T00:49:50.685050+0000 mgr.y (mgr.24661) 2181 : cluster [DBG] pgmap v4264: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 217 KiB/s rd, 13 KiB/s wr, 298 op/s 2024-09-13T00:49:52.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:52 smithi181 bash[67587]: cluster 2024-09-13T00:49:51.195496+0000 mon.a (mon.0) 4302 : cluster [DBG] osdmap e2523: 8 total, 8 up, 8 in 2024-09-13T00:49:52.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:49:52 smithi181 bash[95505]: ts=2024-09-13T00:49:52.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:49:53.424 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestClone::test_unprotected FAILED [ 62%] 2024-09-13T00:49:53.424 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestClone::test_unprotected ERROR [ 62%] 2024-09-13T00:49:53.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:53 smithi136 bash[75164]: cluster 2024-09-13T00:49:52.205761+0000 mon.a (mon.0) 4303 : cluster [DBG] osdmap e2524: 8 total, 8 up, 8 in 2024-09-13T00:49:53.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:53 smithi136 bash[76371]: cluster 2024-09-13T00:49:52.205761+0000 mon.a (mon.0) 4303 : cluster [DBG] osdmap e2524: 8 total, 8 up, 8 in 2024-09-13T00:49:53.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:53 smithi181 bash[67587]: cluster 2024-09-13T00:49:52.205761+0000 mon.a (mon.0) 4303 : cluster [DBG] osdmap e2524: 8 total, 8 up, 8 in 2024-09-13T00:49:54.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:54 smithi136 bash[75164]: cluster 2024-09-13T00:49:52.685908+0000 mgr.y (mgr.24661) 2182 : cluster [DBG] pgmap v4267: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 248 KiB/s rd, 16 KiB/s wr, 336 op/s 2024-09-13T00:49:54.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:54 smithi136 bash[75164]: cluster 2024-09-13T00:49:53.215865+0000 mon.a (mon.0) 4304 : cluster [DBG] osdmap e2525: 8 total, 8 up, 8 in 2024-09-13T00:49:54.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:54 smithi136 bash[75164]: audit 2024-09-13T00:49:53.854328+0000 mon.a (mon.0) 4305 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:49:54.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:54 smithi136 bash[76371]: cluster 2024-09-13T00:49:52.685908+0000 mgr.y (mgr.24661) 2182 : cluster [DBG] pgmap v4267: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 248 KiB/s rd, 16 KiB/s wr, 336 op/s 2024-09-13T00:49:54.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:54 smithi136 bash[76371]: cluster 2024-09-13T00:49:53.215865+0000 mon.a (mon.0) 4304 : cluster [DBG] osdmap e2525: 8 total, 8 up, 8 in 2024-09-13T00:49:54.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:54 smithi136 bash[76371]: audit 2024-09-13T00:49:53.854328+0000 mon.a (mon.0) 4305 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:49:54.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:54 smithi181 bash[67587]: cluster 2024-09-13T00:49:52.685908+0000 mgr.y (mgr.24661) 2182 : cluster [DBG] pgmap v4267: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 248 KiB/s rd, 16 KiB/s wr, 336 op/s 2024-09-13T00:49:54.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:54 smithi181 bash[67587]: cluster 2024-09-13T00:49:53.215865+0000 mon.a (mon.0) 4304 : cluster [DBG] osdmap e2525: 8 total, 8 up, 8 in 2024-09-13T00:49:54.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:54 smithi181 bash[67587]: audit 2024-09-13T00:49:53.854328+0000 mon.a (mon.0) 4305 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:49:55.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:55 smithi136 bash[75164]: cluster 2024-09-13T00:49:54.230434+0000 mon.a (mon.0) 4306 : cluster [DBG] osdmap e2526: 8 total, 8 up, 8 in 2024-09-13T00:49:55.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:55 smithi136 bash[75164]: cluster 2024-09-13T00:49:55.014681+0000 mon.a (mon.0) 4307 : cluster [DBG] osdmap e2527: 8 total, 8 up, 8 in 2024-09-13T00:49:55.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:55 smithi136 bash[75164]: audit 2024-09-13T00:49:55.017494+0000 mon.c (mon.2) 148 : audit [INF] from='client.? 172.21.15.136:0/1130477352' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rbd-api-smithi136-117597-2","app": "rbd"}]: dispatch 2024-09-13T00:49:55.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:55 smithi136 bash[75164]: audit 2024-09-13T00:49:55.019212+0000 mon.a (mon.0) 4308 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rbd-api-smithi136-117597-2","app": "rbd"}]: dispatch 2024-09-13T00:49:55.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:55 smithi136 bash[76371]: cluster 2024-09-13T00:49:54.230434+0000 mon.a (mon.0) 4306 : cluster [DBG] osdmap e2526: 8 total, 8 up, 8 in 2024-09-13T00:49:55.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:55 smithi136 bash[76371]: cluster 2024-09-13T00:49:55.014681+0000 mon.a (mon.0) 4307 : cluster [DBG] osdmap e2527: 8 total, 8 up, 8 in 2024-09-13T00:49:55.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:55 smithi136 bash[76371]: audit 2024-09-13T00:49:55.017494+0000 mon.c (mon.2) 148 : audit [INF] from='client.? 172.21.15.136:0/1130477352' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rbd-api-smithi136-117597-2","app": "rbd"}]: dispatch 2024-09-13T00:49:55.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:55 smithi136 bash[76371]: audit 2024-09-13T00:49:55.019212+0000 mon.a (mon.0) 4308 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rbd-api-smithi136-117597-2","app": "rbd"}]: dispatch 2024-09-13T00:49:55.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:55 smithi181 bash[67587]: cluster 2024-09-13T00:49:54.230434+0000 mon.a (mon.0) 4306 : cluster [DBG] osdmap e2526: 8 total, 8 up, 8 in 2024-09-13T00:49:55.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:55 smithi181 bash[67587]: cluster 2024-09-13T00:49:55.014681+0000 mon.a (mon.0) 4307 : cluster [DBG] osdmap e2527: 8 total, 8 up, 8 in 2024-09-13T00:49:55.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:55 smithi181 bash[67587]: audit 2024-09-13T00:49:55.017494+0000 mon.c (mon.2) 148 : audit [INF] from='client.? 172.21.15.136:0/1130477352' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rbd-api-smithi136-117597-2","app": "rbd"}]: dispatch 2024-09-13T00:49:55.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:55 smithi181 bash[67587]: audit 2024-09-13T00:49:55.019212+0000 mon.a (mon.0) 4308 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rbd-api-smithi136-117597-2","app": "rbd"}]: dispatch 2024-09-13T00:49:56.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:56 smithi136 bash[75164]: cluster 2024-09-13T00:49:54.686605+0000 mgr.y (mgr.24661) 2183 : cluster [DBG] pgmap v4270: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 209 KiB/s rd, 14 KiB/s wr, 276 op/s 2024-09-13T00:49:56.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:56 smithi136 bash[75164]: audit 2024-09-13T00:49:56.015085+0000 mon.a (mon.0) 4309 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rbd-api-smithi136-117597-2","app": "rbd"}]': finished 2024-09-13T00:49:56.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:56 smithi136 bash[75164]: cluster 2024-09-13T00:49:56.016054+0000 mon.a (mon.0) 4310 : cluster [DBG] osdmap e2528: 8 total, 8 up, 8 in 2024-09-13T00:49:56.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:56 smithi136 bash[76371]: cluster 2024-09-13T00:49:54.686605+0000 mgr.y (mgr.24661) 2183 : cluster [DBG] pgmap v4270: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 209 KiB/s rd, 14 KiB/s wr, 276 op/s 2024-09-13T00:49:56.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:56 smithi136 bash[76371]: audit 2024-09-13T00:49:56.015085+0000 mon.a (mon.0) 4309 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rbd-api-smithi136-117597-2","app": "rbd"}]': finished 2024-09-13T00:49:56.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:56 smithi136 bash[76371]: cluster 2024-09-13T00:49:56.016054+0000 mon.a (mon.0) 4310 : cluster [DBG] osdmap e2528: 8 total, 8 up, 8 in 2024-09-13T00:49:56.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:56 smithi181 bash[67587]: cluster 2024-09-13T00:49:54.686605+0000 mgr.y (mgr.24661) 2183 : cluster [DBG] pgmap v4270: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 209 KiB/s rd, 14 KiB/s wr, 276 op/s 2024-09-13T00:49:56.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:56 smithi181 bash[67587]: audit 2024-09-13T00:49:56.015085+0000 mon.a (mon.0) 4309 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rbd-api-smithi136-117597-2","app": "rbd"}]': finished 2024-09-13T00:49:56.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:56 smithi181 bash[67587]: cluster 2024-09-13T00:49:56.016054+0000 mon.a (mon.0) 4310 : cluster [DBG] osdmap e2528: 8 total, 8 up, 8 in 2024-09-13T00:49:58.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:58 smithi136 bash[75164]: cluster 2024-09-13T00:49:56.687476+0000 mgr.y (mgr.24661) 2184 : cluster [DBG] pgmap v4273: 169 pgs: 8 creating+peering, 18 unknown, 143 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 133 KiB/s rd, 11 KiB/s wr, 180 op/s 2024-09-13T00:49:58.260 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:58 smithi136 bash[75164]: cluster 2024-09-13T00:49:57.016286+0000 mon.a (mon.0) 4311 : cluster [DBG] osdmap e2529: 8 total, 8 up, 8 in 2024-09-13T00:49:58.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:58 smithi136 bash[76371]: cluster 2024-09-13T00:49:56.687476+0000 mgr.y (mgr.24661) 2184 : cluster [DBG] pgmap v4273: 169 pgs: 8 creating+peering, 18 unknown, 143 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 133 KiB/s rd, 11 KiB/s wr, 180 op/s 2024-09-13T00:49:58.261 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:58 smithi136 bash[76371]: cluster 2024-09-13T00:49:57.016286+0000 mon.a (mon.0) 4311 : cluster [DBG] osdmap e2529: 8 total, 8 up, 8 in 2024-09-13T00:49:58.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:58 smithi181 bash[67587]: cluster 2024-09-13T00:49:56.687476+0000 mgr.y (mgr.24661) 2184 : cluster [DBG] pgmap v4273: 169 pgs: 8 creating+peering, 18 unknown, 143 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 133 KiB/s rd, 11 KiB/s wr, 180 op/s 2024-09-13T00:49:58.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:58 smithi181 bash[67587]: cluster 2024-09-13T00:49:57.016286+0000 mon.a (mon.0) 4311 : cluster [DBG] osdmap e2529: 8 total, 8 up, 8 in 2024-09-13T00:49:59.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:49:59 smithi181 bash[67587]: cluster 2024-09-13T00:49:58.030650+0000 mon.a (mon.0) 4312 : cluster [DBG] osdmap e2530: 8 total, 8 up, 8 in 2024-09-13T00:49:59.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:49:59 smithi136 bash[75164]: cluster 2024-09-13T00:49:58.030650+0000 mon.a (mon.0) 4312 : cluster [DBG] osdmap e2530: 8 total, 8 up, 8 in 2024-09-13T00:49:59.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:49:59 smithi136 bash[76371]: cluster 2024-09-13T00:49:58.030650+0000 mon.a (mon.0) 4312 : cluster [DBG] osdmap e2530: 8 total, 8 up, 8 in 2024-09-13T00:49:59.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:49:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:49:59] "GET /metrics HTTP/1.1" 200 36743 "" "Prometheus/2.43.0" 2024-09-13T00:50:00.170 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestClone::test_unprotect_with_children PASSED [ 62%] 2024-09-13T00:50:00.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:00 smithi181 bash[67587]: cluster 2024-09-13T00:49:58.688414+0000 mgr.y (mgr.24661) 2185 : cluster [DBG] pgmap v4276: 169 pgs: 8 creating+peering, 2 unknown, 159 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 95 KiB/s rd, 8.5 KiB/s wr, 132 op/s 2024-09-13T00:50:00.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:00 smithi181 bash[67587]: cluster 2024-09-13T00:49:59.038214+0000 mon.a (mon.0) 4313 : cluster [DBG] osdmap e2531: 8 total, 8 up, 8 in 2024-09-13T00:50:00.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:00 smithi181 bash[67587]: audit 2024-09-13T00:49:59.813136+0000 mon.a (mon.0) 4314 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:50:00.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:00 smithi181 bash[67587]: cluster 2024-09-13T00:50:00.000176+0000 mon.a (mon.0) 4315 : cluster [INF] overall HEALTH_OK 2024-09-13T00:50:00.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:00 smithi181 bash[67587]: cluster 2024-09-13T00:50:00.010777+0000 mon.a (mon.0) 4316 : cluster [DBG] osdmap e2532: 8 total, 8 up, 8 in 2024-09-13T00:50:00.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:00 smithi136 bash[75164]: cluster 2024-09-13T00:49:58.688414+0000 mgr.y (mgr.24661) 2185 : cluster [DBG] pgmap v4276: 169 pgs: 8 creating+peering, 2 unknown, 159 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 95 KiB/s rd, 8.5 KiB/s wr, 132 op/s 2024-09-13T00:50:00.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:00 smithi136 bash[75164]: cluster 2024-09-13T00:49:59.038214+0000 mon.a (mon.0) 4313 : cluster [DBG] osdmap e2531: 8 total, 8 up, 8 in 2024-09-13T00:50:00.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:00 smithi136 bash[75164]: audit 2024-09-13T00:49:59.813136+0000 mon.a (mon.0) 4314 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:50:00.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:00 smithi136 bash[75164]: cluster 2024-09-13T00:50:00.000176+0000 mon.a (mon.0) 4315 : cluster [INF] overall HEALTH_OK 2024-09-13T00:50:00.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:00 smithi136 bash[75164]: cluster 2024-09-13T00:50:00.010777+0000 mon.a (mon.0) 4316 : cluster [DBG] osdmap e2532: 8 total, 8 up, 8 in 2024-09-13T00:50:00.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:00 smithi136 bash[76371]: cluster 2024-09-13T00:49:58.688414+0000 mgr.y (mgr.24661) 2185 : cluster [DBG] pgmap v4276: 169 pgs: 8 creating+peering, 2 unknown, 159 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 95 KiB/s rd, 8.5 KiB/s wr, 132 op/s 2024-09-13T00:50:00.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:00 smithi136 bash[76371]: cluster 2024-09-13T00:49:59.038214+0000 mon.a (mon.0) 4313 : cluster [DBG] osdmap e2531: 8 total, 8 up, 8 in 2024-09-13T00:50:00.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:00 smithi136 bash[76371]: audit 2024-09-13T00:49:59.813136+0000 mon.a (mon.0) 4314 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:50:00.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:00 smithi136 bash[76371]: cluster 2024-09-13T00:50:00.000176+0000 mon.a (mon.0) 4315 : cluster [INF] overall HEALTH_OK 2024-09-13T00:50:00.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:00 smithi136 bash[76371]: cluster 2024-09-13T00:50:00.010777+0000 mon.a (mon.0) 4316 : cluster [DBG] osdmap e2532: 8 total, 8 up, 8 in 2024-09-13T00:50:02.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:01 smithi136 bash[75164]: audit 2024-09-13T00:50:00.625634+0000 mon.a (mon.0) 4317 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:50:02.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:01 smithi136 bash[75164]: audit 2024-09-13T00:50:00.633445+0000 mon.a (mon.0) 4318 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:50:02.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:01 smithi136 bash[75164]: audit 2024-09-13T00:50:00.635137+0000 mon.a (mon.0) 4319 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:50:02.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:01 smithi136 bash[75164]: audit 2024-09-13T00:50:00.636059+0000 mon.a (mon.0) 4320 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:50:02.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:01 smithi136 bash[75164]: audit 2024-09-13T00:50:00.646226+0000 mon.a (mon.0) 4321 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:50:02.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:01 smithi136 bash[75164]: cluster 2024-09-13T00:50:00.689086+0000 mgr.y (mgr.24661) 2186 : cluster [DBG] pgmap v4279: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 91 KiB/s rd, 3.5 KiB/s wr, 120 op/s 2024-09-13T00:50:02.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:01 smithi136 bash[75164]: cluster 2024-09-13T00:50:01.040421+0000 mon.a (mon.0) 4322 : cluster [DBG] osdmap e2533: 8 total, 8 up, 8 in 2024-09-13T00:50:02.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:01 smithi136 bash[76371]: audit 2024-09-13T00:50:00.625634+0000 mon.a (mon.0) 4317 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:50:02.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:01 smithi136 bash[76371]: audit 2024-09-13T00:50:00.633445+0000 mon.a (mon.0) 4318 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:50:02.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:01 smithi136 bash[76371]: audit 2024-09-13T00:50:00.635137+0000 mon.a (mon.0) 4319 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:50:02.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:01 smithi136 bash[76371]: audit 2024-09-13T00:50:00.636059+0000 mon.a (mon.0) 4320 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:50:02.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:01 smithi136 bash[76371]: audit 2024-09-13T00:50:00.646226+0000 mon.a (mon.0) 4321 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:50:02.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:01 smithi136 bash[76371]: cluster 2024-09-13T00:50:00.689086+0000 mgr.y (mgr.24661) 2186 : cluster [DBG] pgmap v4279: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 91 KiB/s rd, 3.5 KiB/s wr, 120 op/s 2024-09-13T00:50:02.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:01 smithi136 bash[76371]: cluster 2024-09-13T00:50:01.040421+0000 mon.a (mon.0) 4322 : cluster [DBG] osdmap e2533: 8 total, 8 up, 8 in 2024-09-13T00:50:02.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:01 smithi181 bash[67587]: audit 2024-09-13T00:50:00.625634+0000 mon.a (mon.0) 4317 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:50:02.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:01 smithi181 bash[67587]: audit 2024-09-13T00:50:00.633445+0000 mon.a (mon.0) 4318 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:50:02.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:01 smithi181 bash[67587]: audit 2024-09-13T00:50:00.635137+0000 mon.a (mon.0) 4319 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:50:02.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:01 smithi181 bash[67587]: audit 2024-09-13T00:50:00.636059+0000 mon.a (mon.0) 4320 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:50:02.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:01 smithi181 bash[67587]: audit 2024-09-13T00:50:00.646226+0000 mon.a (mon.0) 4321 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:50:02.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:01 smithi181 bash[67587]: cluster 2024-09-13T00:50:00.689086+0000 mgr.y (mgr.24661) 2186 : cluster [DBG] pgmap v4279: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 91 KiB/s rd, 3.5 KiB/s wr, 120 op/s 2024-09-13T00:50:02.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:01 smithi181 bash[67587]: cluster 2024-09-13T00:50:01.040421+0000 mon.a (mon.0) 4322 : cluster [DBG] osdmap e2533: 8 total, 8 up, 8 in 2024-09-13T00:50:02.254 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestClone::test_stat PASSED [ 63%] 2024-09-13T00:50:02.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:50:02 smithi181 bash[95505]: ts=2024-09-13T00:50:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:50:03.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:03 smithi181 bash[67587]: cluster 2024-09-13T00:50:02.044455+0000 mon.a (mon.0) 4323 : cluster [DBG] osdmap e2534: 8 total, 8 up, 8 in 2024-09-13T00:50:03.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:03 smithi136 bash[75164]: cluster 2024-09-13T00:50:02.044455+0000 mon.a (mon.0) 4323 : cluster [DBG] osdmap e2534: 8 total, 8 up, 8 in 2024-09-13T00:50:03.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:03 smithi136 bash[76371]: cluster 2024-09-13T00:50:02.044455+0000 mon.a (mon.0) 4323 : cluster [DBG] osdmap e2534: 8 total, 8 up, 8 in 2024-09-13T00:50:04.280 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestClone::test_resize_stat PASSED [ 63%] 2024-09-13T00:50:04.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:04 smithi181 bash[67587]: cluster 2024-09-13T00:50:02.689703+0000 mgr.y (mgr.24661) 2187 : cluster [DBG] pgmap v4282: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 31 KiB/s rd, 2.5 KiB/s wr, 43 op/s 2024-09-13T00:50:04.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:04 smithi181 bash[67587]: cluster 2024-09-13T00:50:03.058409+0000 mon.a (mon.0) 4324 : cluster [DBG] osdmap e2535: 8 total, 8 up, 8 in 2024-09-13T00:50:04.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:04 smithi136 bash[75164]: cluster 2024-09-13T00:50:02.689703+0000 mgr.y (mgr.24661) 2187 : cluster [DBG] pgmap v4282: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 31 KiB/s rd, 2.5 KiB/s wr, 43 op/s 2024-09-13T00:50:04.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:04 smithi136 bash[75164]: cluster 2024-09-13T00:50:03.058409+0000 mon.a (mon.0) 4324 : cluster [DBG] osdmap e2535: 8 total, 8 up, 8 in 2024-09-13T00:50:04.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:04 smithi136 bash[76371]: cluster 2024-09-13T00:50:02.689703+0000 mgr.y (mgr.24661) 2187 : cluster [DBG] pgmap v4282: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 31 KiB/s rd, 2.5 KiB/s wr, 43 op/s 2024-09-13T00:50:04.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:04 smithi136 bash[76371]: cluster 2024-09-13T00:50:03.058409+0000 mon.a (mon.0) 4324 : cluster [DBG] osdmap e2535: 8 total, 8 up, 8 in 2024-09-13T00:50:05.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:05 smithi181 bash[67587]: cluster 2024-09-13T00:50:04.066177+0000 mon.a (mon.0) 4325 : cluster [DBG] osdmap e2536: 8 total, 8 up, 8 in 2024-09-13T00:50:05.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:05 smithi136 bash[75164]: cluster 2024-09-13T00:50:04.066177+0000 mon.a (mon.0) 4325 : cluster [DBG] osdmap e2536: 8 total, 8 up, 8 in 2024-09-13T00:50:05.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:05 smithi136 bash[76371]: cluster 2024-09-13T00:50:04.066177+0000 mon.a (mon.0) 4325 : cluster [DBG] osdmap e2536: 8 total, 8 up, 8 in 2024-09-13T00:50:06.285 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestClone::test_resize_io PASSED [ 64%] 2024-09-13T00:50:06.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:06 smithi181 bash[67587]: cluster 2024-09-13T00:50:04.690392+0000 mgr.y (mgr.24661) 2188 : cluster [DBG] pgmap v4285: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 57 KiB/s rd, 7.7 KiB/s wr, 85 op/s 2024-09-13T00:50:06.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:06 smithi181 bash[67587]: cluster 2024-09-13T00:50:05.073652+0000 mon.a (mon.0) 4326 : cluster [DBG] osdmap e2537: 8 total, 8 up, 8 in 2024-09-13T00:50:06.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:06 smithi136 bash[75164]: cluster 2024-09-13T00:50:04.690392+0000 mgr.y (mgr.24661) 2188 : cluster [DBG] pgmap v4285: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 57 KiB/s rd, 7.7 KiB/s wr, 85 op/s 2024-09-13T00:50:06.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:06 smithi136 bash[75164]: cluster 2024-09-13T00:50:05.073652+0000 mon.a (mon.0) 4326 : cluster [DBG] osdmap e2537: 8 total, 8 up, 8 in 2024-09-13T00:50:06.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:06 smithi136 bash[76371]: cluster 2024-09-13T00:50:04.690392+0000 mgr.y (mgr.24661) 2188 : cluster [DBG] pgmap v4285: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 57 KiB/s rd, 7.7 KiB/s wr, 85 op/s 2024-09-13T00:50:06.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:06 smithi136 bash[76371]: cluster 2024-09-13T00:50:05.073652+0000 mon.a (mon.0) 4326 : cluster [DBG] osdmap e2537: 8 total, 8 up, 8 in 2024-09-13T00:50:07.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:07 smithi181 bash[67587]: cluster 2024-09-13T00:50:06.084133+0000 mon.a (mon.0) 4327 : cluster [DBG] osdmap e2538: 8 total, 8 up, 8 in 2024-09-13T00:50:07.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:07 smithi136 bash[75164]: cluster 2024-09-13T00:50:06.084133+0000 mon.a (mon.0) 4327 : cluster [DBG] osdmap e2538: 8 total, 8 up, 8 in 2024-09-13T00:50:07.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:07 smithi136 bash[76371]: cluster 2024-09-13T00:50:06.084133+0000 mon.a (mon.0) 4327 : cluster [DBG] osdmap e2538: 8 total, 8 up, 8 in 2024-09-13T00:50:08.320 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestClone::test_read PASSED [ 64%] 2024-09-13T00:50:08.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:08 smithi181 bash[67587]: cluster 2024-09-13T00:50:06.691167+0000 mgr.y (mgr.24661) 2189 : cluster [DBG] pgmap v4288: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 265 KiB/s rd, 20 KiB/s wr, 355 op/s 2024-09-13T00:50:08.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:08 smithi181 bash[67587]: cluster 2024-09-13T00:50:07.094328+0000 mon.a (mon.0) 4328 : cluster [DBG] osdmap e2539: 8 total, 8 up, 8 in 2024-09-13T00:50:08.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:08 smithi136 bash[75164]: cluster 2024-09-13T00:50:06.691167+0000 mgr.y (mgr.24661) 2189 : cluster [DBG] pgmap v4288: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 265 KiB/s rd, 20 KiB/s wr, 355 op/s 2024-09-13T00:50:08.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:08 smithi136 bash[75164]: cluster 2024-09-13T00:50:07.094328+0000 mon.a (mon.0) 4328 : cluster [DBG] osdmap e2539: 8 total, 8 up, 8 in 2024-09-13T00:50:08.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:08 smithi136 bash[76371]: cluster 2024-09-13T00:50:06.691167+0000 mgr.y (mgr.24661) 2189 : cluster [DBG] pgmap v4288: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 265 KiB/s rd, 20 KiB/s wr, 355 op/s 2024-09-13T00:50:08.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:08 smithi136 bash[76371]: cluster 2024-09-13T00:50:07.094328+0000 mon.a (mon.0) 4328 : cluster [DBG] osdmap e2539: 8 total, 8 up, 8 in 2024-09-13T00:50:09.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:09 smithi136 bash[76371]: cluster 2024-09-13T00:50:08.102089+0000 mon.a (mon.0) 4329 : cluster [DBG] osdmap e2540: 8 total, 8 up, 8 in 2024-09-13T00:50:09.413 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:09 smithi136 bash[76371]: audit 2024-09-13T00:50:08.854745+0000 mon.a (mon.0) 4330 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:50:09.413 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:09 smithi136 bash[75164]: cluster 2024-09-13T00:50:08.102089+0000 mon.a (mon.0) 4329 : cluster [DBG] osdmap e2540: 8 total, 8 up, 8 in 2024-09-13T00:50:09.414 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:09 smithi136 bash[75164]: audit 2024-09-13T00:50:08.854745+0000 mon.a (mon.0) 4330 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:50:09.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:09 smithi181 bash[67587]: cluster 2024-09-13T00:50:08.102089+0000 mon.a (mon.0) 4329 : cluster [DBG] osdmap e2540: 8 total, 8 up, 8 in 2024-09-13T00:50:09.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:09 smithi181 bash[67587]: audit 2024-09-13T00:50:08.854745+0000 mon.a (mon.0) 4330 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:50:09.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:50:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:50:09] "GET /metrics HTTP/1.1" 200 36742 "" "Prometheus/2.43.0" 2024-09-13T00:50:10.260 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestClone::test_write PASSED [ 65%] 2024-09-13T00:50:10.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:10 smithi136 bash[75164]: cluster 2024-09-13T00:50:08.692201+0000 mgr.y (mgr.24661) 2190 : cluster [DBG] pgmap v4291: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 369 KiB/s rd, 27 KiB/s wr, 492 op/s 2024-09-13T00:50:10.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:10 smithi136 bash[75164]: cluster 2024-09-13T00:50:09.115820+0000 mon.a (mon.0) 4331 : cluster [DBG] osdmap e2541: 8 total, 8 up, 8 in 2024-09-13T00:50:10.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:10 smithi136 bash[75164]: cluster 2024-09-13T00:50:10.032886+0000 mon.a (mon.0) 4332 : cluster [DBG] osdmap e2542: 8 total, 8 up, 8 in 2024-09-13T00:50:10.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:10 smithi136 bash[76371]: cluster 2024-09-13T00:50:08.692201+0000 mgr.y (mgr.24661) 2190 : cluster [DBG] pgmap v4291: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 369 KiB/s rd, 27 KiB/s wr, 492 op/s 2024-09-13T00:50:10.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:10 smithi136 bash[76371]: cluster 2024-09-13T00:50:09.115820+0000 mon.a (mon.0) 4331 : cluster [DBG] osdmap e2541: 8 total, 8 up, 8 in 2024-09-13T00:50:10.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:10 smithi136 bash[76371]: cluster 2024-09-13T00:50:10.032886+0000 mon.a (mon.0) 4332 : cluster [DBG] osdmap e2542: 8 total, 8 up, 8 in 2024-09-13T00:50:10.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:10 smithi181 bash[67587]: cluster 2024-09-13T00:50:08.692201+0000 mgr.y (mgr.24661) 2190 : cluster [DBG] pgmap v4291: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 369 KiB/s rd, 27 KiB/s wr, 492 op/s 2024-09-13T00:50:10.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:10 smithi181 bash[67587]: cluster 2024-09-13T00:50:09.115820+0000 mon.a (mon.0) 4331 : cluster [DBG] osdmap e2541: 8 total, 8 up, 8 in 2024-09-13T00:50:10.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:10 smithi181 bash[67587]: cluster 2024-09-13T00:50:10.032886+0000 mon.a (mon.0) 4332 : cluster [DBG] osdmap e2542: 8 total, 8 up, 8 in 2024-09-13T00:50:12.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:12 smithi136 bash[76371]: cluster 2024-09-13T00:50:10.693223+0000 mgr.y (mgr.24661) 2191 : cluster [DBG] pgmap v4294: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 272 KiB/s rd, 20 KiB/s wr, 368 op/s 2024-09-13T00:50:12.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:12 smithi136 bash[76371]: cluster 2024-09-13T00:50:11.130654+0000 mon.a (mon.0) 4333 : cluster [DBG] osdmap e2543: 8 total, 8 up, 8 in 2024-09-13T00:50:12.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:12 smithi136 bash[75164]: cluster 2024-09-13T00:50:10.693223+0000 mgr.y (mgr.24661) 2191 : cluster [DBG] pgmap v4294: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 272 KiB/s rd, 20 KiB/s wr, 368 op/s 2024-09-13T00:50:12.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:12 smithi136 bash[75164]: cluster 2024-09-13T00:50:11.130654+0000 mon.a (mon.0) 4333 : cluster [DBG] osdmap e2543: 8 total, 8 up, 8 in 2024-09-13T00:50:12.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:12 smithi181 bash[67587]: cluster 2024-09-13T00:50:10.693223+0000 mgr.y (mgr.24661) 2191 : cluster [DBG] pgmap v4294: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 272 KiB/s rd, 20 KiB/s wr, 368 op/s 2024-09-13T00:50:12.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:12 smithi181 bash[67587]: cluster 2024-09-13T00:50:11.130654+0000 mon.a (mon.0) 4333 : cluster [DBG] osdmap e2543: 8 total, 8 up, 8 in 2024-09-13T00:50:12.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:50:12 smithi181 bash[95505]: ts=2024-09-13T00:50:12.317Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:50:14.098 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 13 00:50:13 smithi181 bash[98820]: logger=cleanup t=2024-09-13T00:50:13.814149669Z level=info msg="Completed cleanup jobs" duration=120.024213ms 2024-09-13T00:50:14.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:14 smithi136 bash[76371]: cluster 2024-09-13T00:50:12.694149+0000 mgr.y (mgr.24661) 2192 : cluster [DBG] pgmap v4296: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 225 KiB/s rd, 22 KiB/s wr, 320 op/s 2024-09-13T00:50:14.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:14 smithi136 bash[75164]: cluster 2024-09-13T00:50:12.694149+0000 mgr.y (mgr.24661) 2192 : cluster [DBG] pgmap v4296: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 225 KiB/s rd, 22 KiB/s wr, 320 op/s 2024-09-13T00:50:14.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:14 smithi181 bash[67587]: cluster 2024-09-13T00:50:12.694149+0000 mgr.y (mgr.24661) 2192 : cluster [DBG] pgmap v4296: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 225 KiB/s rd, 22 KiB/s wr, 320 op/s 2024-09-13T00:50:16.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:16 smithi181 bash[67587]: cluster 2024-09-13T00:50:14.695176+0000 mgr.y (mgr.24661) 2193 : cluster [DBG] pgmap v4297: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 353 KiB/s rd, 17 KiB/s wr, 454 op/s 2024-09-13T00:50:16.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:16 smithi181 bash[67587]: cluster 2024-09-13T00:50:15.038307+0000 mon.a (mon.0) 4334 : cluster [DBG] osdmap e2544: 8 total, 8 up, 8 in 2024-09-13T00:50:16.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:16 smithi136 bash[75164]: cluster 2024-09-13T00:50:14.695176+0000 mgr.y (mgr.24661) 2193 : cluster [DBG] pgmap v4297: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 353 KiB/s rd, 17 KiB/s wr, 454 op/s 2024-09-13T00:50:16.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:16 smithi136 bash[75164]: cluster 2024-09-13T00:50:15.038307+0000 mon.a (mon.0) 4334 : cluster [DBG] osdmap e2544: 8 total, 8 up, 8 in 2024-09-13T00:50:16.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:16 smithi136 bash[76371]: cluster 2024-09-13T00:50:14.695176+0000 mgr.y (mgr.24661) 2193 : cluster [DBG] pgmap v4297: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 353 KiB/s rd, 17 KiB/s wr, 454 op/s 2024-09-13T00:50:16.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:16 smithi136 bash[76371]: cluster 2024-09-13T00:50:15.038307+0000 mon.a (mon.0) 4334 : cluster [DBG] osdmap e2544: 8 total, 8 up, 8 in 2024-09-13T00:50:18.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:18 smithi181 bash[67587]: cluster 2024-09-13T00:50:16.696363+0000 mgr.y (mgr.24661) 2194 : cluster [DBG] pgmap v4299: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 637 KiB/s rd, 20 KiB/s wr, 776 op/s 2024-09-13T00:50:18.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:18 smithi136 bash[76371]: cluster 2024-09-13T00:50:16.696363+0000 mgr.y (mgr.24661) 2194 : cluster [DBG] pgmap v4299: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 637 KiB/s rd, 20 KiB/s wr, 776 op/s 2024-09-13T00:50:18.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:18 smithi136 bash[75164]: cluster 2024-09-13T00:50:16.696363+0000 mgr.y (mgr.24661) 2194 : cluster [DBG] pgmap v4299: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 637 KiB/s rd, 20 KiB/s wr, 776 op/s 2024-09-13T00:50:19.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:50:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:50:19] "GET /metrics HTTP/1.1" 200 36742 "" "Prometheus/2.43.0" 2024-09-13T00:50:20.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:20 smithi181 bash[67587]: cluster 2024-09-13T00:50:18.697528+0000 mgr.y (mgr.24661) 2195 : cluster [DBG] pgmap v4300: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 836 KiB/s rd, 16 KiB/s wr, 996 op/s 2024-09-13T00:50:20.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:20 smithi136 bash[75164]: cluster 2024-09-13T00:50:18.697528+0000 mgr.y (mgr.24661) 2195 : cluster [DBG] pgmap v4300: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 836 KiB/s rd, 16 KiB/s wr, 996 op/s 2024-09-13T00:50:20.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:20 smithi136 bash[76371]: cluster 2024-09-13T00:50:18.697528+0000 mgr.y (mgr.24661) 2195 : cluster [DBG] pgmap v4300: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 836 KiB/s rd, 16 KiB/s wr, 996 op/s 2024-09-13T00:50:21.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:21 smithi181 bash[67587]: cluster 2024-09-13T00:50:20.037231+0000 mon.a (mon.0) 4335 : cluster [DBG] osdmap e2545: 8 total, 8 up, 8 in 2024-09-13T00:50:21.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:21 smithi136 bash[75164]: cluster 2024-09-13T00:50:20.037231+0000 mon.a (mon.0) 4335 : cluster [DBG] osdmap e2545: 8 total, 8 up, 8 in 2024-09-13T00:50:21.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:21 smithi136 bash[76371]: cluster 2024-09-13T00:50:20.037231+0000 mon.a (mon.0) 4335 : cluster [DBG] osdmap e2545: 8 total, 8 up, 8 in 2024-09-13T00:50:22.291 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestClone::test_list_children PASSED [ 65%] 2024-09-13T00:50:22.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:22 smithi181 bash[67587]: cluster 2024-09-13T00:50:20.698511+0000 mgr.y (mgr.24661) 2196 : cluster [DBG] pgmap v4302: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 1.0 MiB/s rd, 12 KiB/s wr, 1.22k op/s 2024-09-13T00:50:22.348 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:50:22 smithi181 bash[95505]: ts=2024-09-13T00:50:22.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:50:22.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:22 smithi136 bash[75164]: cluster 2024-09-13T00:50:20.698511+0000 mgr.y (mgr.24661) 2196 : cluster [DBG] pgmap v4302: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 1.0 MiB/s rd, 12 KiB/s wr, 1.22k op/s 2024-09-13T00:50:22.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:22 smithi136 bash[76371]: cluster 2024-09-13T00:50:20.698511+0000 mgr.y (mgr.24661) 2196 : cluster [DBG] pgmap v4302: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 1.0 MiB/s rd, 12 KiB/s wr, 1.22k op/s 2024-09-13T00:50:23.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:23 smithi181 bash[67587]: cluster 2024-09-13T00:50:22.066245+0000 mon.a (mon.0) 4336 : cluster [DBG] osdmap e2546: 8 total, 8 up, 8 in 2024-09-13T00:50:23.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:23 smithi136 bash[75164]: cluster 2024-09-13T00:50:22.066245+0000 mon.a (mon.0) 4336 : cluster [DBG] osdmap e2546: 8 total, 8 up, 8 in 2024-09-13T00:50:23.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:23 smithi136 bash[76371]: cluster 2024-09-13T00:50:22.066245+0000 mon.a (mon.0) 4336 : cluster [DBG] osdmap e2546: 8 total, 8 up, 8 in 2024-09-13T00:50:24.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:24 smithi181 bash[67587]: cluster 2024-09-13T00:50:22.699244+0000 mgr.y (mgr.24661) 2197 : cluster [DBG] pgmap v4304: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 1.1 MiB/s rd, 12 KiB/s wr, 1.27k op/s 2024-09-13T00:50:24.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:24 smithi181 bash[67587]: cluster 2024-09-13T00:50:23.096052+0000 mon.a (mon.0) 4337 : cluster [DBG] osdmap e2547: 8 total, 8 up, 8 in 2024-09-13T00:50:24.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:24 smithi181 bash[67587]: audit 2024-09-13T00:50:23.855125+0000 mon.a (mon.0) 4338 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:50:24.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:24 smithi136 bash[75164]: cluster 2024-09-13T00:50:22.699244+0000 mgr.y (mgr.24661) 2197 : cluster [DBG] pgmap v4304: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 1.1 MiB/s rd, 12 KiB/s wr, 1.27k op/s 2024-09-13T00:50:24.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:24 smithi136 bash[75164]: cluster 2024-09-13T00:50:23.096052+0000 mon.a (mon.0) 4337 : cluster [DBG] osdmap e2547: 8 total, 8 up, 8 in 2024-09-13T00:50:24.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:24 smithi136 bash[75164]: audit 2024-09-13T00:50:23.855125+0000 mon.a (mon.0) 4338 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:50:24.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:24 smithi136 bash[76371]: cluster 2024-09-13T00:50:22.699244+0000 mgr.y (mgr.24661) 2197 : cluster [DBG] pgmap v4304: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 1.1 MiB/s rd, 12 KiB/s wr, 1.27k op/s 2024-09-13T00:50:24.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:24 smithi136 bash[76371]: cluster 2024-09-13T00:50:23.096052+0000 mon.a (mon.0) 4337 : cluster [DBG] osdmap e2547: 8 total, 8 up, 8 in 2024-09-13T00:50:24.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:24 smithi136 bash[76371]: audit 2024-09-13T00:50:23.855125+0000 mon.a (mon.0) 4338 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:50:25.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:25 smithi136 bash[76371]: cluster 2024-09-13T00:50:24.106543+0000 mon.a (mon.0) 4339 : cluster [DBG] osdmap e2548: 8 total, 8 up, 8 in 2024-09-13T00:50:25.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:25 smithi136 bash[75164]: cluster 2024-09-13T00:50:24.106543+0000 mon.a (mon.0) 4339 : cluster [DBG] osdmap e2548: 8 total, 8 up, 8 in 2024-09-13T00:50:25.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:25 smithi181 bash[67587]: cluster 2024-09-13T00:50:24.106543+0000 mon.a (mon.0) 4339 : cluster [DBG] osdmap e2548: 8 total, 8 up, 8 in 2024-09-13T00:50:26.356 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestClone::test_flatten_errors PASSED [ 66%] 2024-09-13T00:50:26.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:26 smithi136 bash[75164]: cluster 2024-09-13T00:50:24.699912+0000 mgr.y (mgr.24661) 2198 : cluster [DBG] pgmap v4307: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 997 KiB/s rd, 11 KiB/s wr, 1.18k op/s 2024-09-13T00:50:26.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:26 smithi136 bash[75164]: cluster 2024-09-13T00:50:25.115982+0000 mon.a (mon.0) 4340 : cluster [DBG] osdmap e2549: 8 total, 8 up, 8 in 2024-09-13T00:50:26.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:26 smithi136 bash[76371]: cluster 2024-09-13T00:50:24.699912+0000 mgr.y (mgr.24661) 2198 : cluster [DBG] pgmap v4307: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 997 KiB/s rd, 11 KiB/s wr, 1.18k op/s 2024-09-13T00:50:26.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:26 smithi136 bash[76371]: cluster 2024-09-13T00:50:25.115982+0000 mon.a (mon.0) 4340 : cluster [DBG] osdmap e2549: 8 total, 8 up, 8 in 2024-09-13T00:50:26.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:26 smithi181 bash[67587]: cluster 2024-09-13T00:50:24.699912+0000 mgr.y (mgr.24661) 2198 : cluster [DBG] pgmap v4307: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 997 KiB/s rd, 11 KiB/s wr, 1.18k op/s 2024-09-13T00:50:26.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:26 smithi181 bash[67587]: cluster 2024-09-13T00:50:25.115982+0000 mon.a (mon.0) 4340 : cluster [DBG] osdmap e2549: 8 total, 8 up, 8 in 2024-09-13T00:50:27.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:27 smithi136 bash[75164]: cluster 2024-09-13T00:50:26.128109+0000 mon.a (mon.0) 4341 : cluster [DBG] osdmap e2550: 8 total, 8 up, 8 in 2024-09-13T00:50:27.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:27 smithi136 bash[76371]: cluster 2024-09-13T00:50:26.128109+0000 mon.a (mon.0) 4341 : cluster [DBG] osdmap e2550: 8 total, 8 up, 8 in 2024-09-13T00:50:27.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:27 smithi181 bash[67587]: cluster 2024-09-13T00:50:26.128109+0000 mon.a (mon.0) 4341 : cluster [DBG] osdmap e2550: 8 total, 8 up, 8 in 2024-09-13T00:50:28.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:28 smithi136 bash[75164]: cluster 2024-09-13T00:50:26.700730+0000 mgr.y (mgr.24661) 2199 : cluster [DBG] pgmap v4310: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 528 KiB/s rd, 14 KiB/s wr, 660 op/s 2024-09-13T00:50:28.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:28 smithi136 bash[75164]: cluster 2024-09-13T00:50:27.135581+0000 mon.a (mon.0) 4342 : cluster [DBG] osdmap e2551: 8 total, 8 up, 8 in 2024-09-13T00:50:28.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:28 smithi136 bash[76371]: cluster 2024-09-13T00:50:26.700730+0000 mgr.y (mgr.24661) 2199 : cluster [DBG] pgmap v4310: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 528 KiB/s rd, 14 KiB/s wr, 660 op/s 2024-09-13T00:50:28.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:28 smithi136 bash[76371]: cluster 2024-09-13T00:50:27.135581+0000 mon.a (mon.0) 4342 : cluster [DBG] osdmap e2551: 8 total, 8 up, 8 in 2024-09-13T00:50:28.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:28 smithi181 bash[67587]: cluster 2024-09-13T00:50:26.700730+0000 mgr.y (mgr.24661) 2199 : cluster [DBG] pgmap v4310: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 528 KiB/s rd, 14 KiB/s wr, 660 op/s 2024-09-13T00:50:28.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:28 smithi181 bash[67587]: cluster 2024-09-13T00:50:27.135581+0000 mon.a (mon.0) 4342 : cluster [DBG] osdmap e2551: 8 total, 8 up, 8 in 2024-09-13T00:50:29.355 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestClone::test_flatten_basic PASSED [ 67%] 2024-09-13T00:50:29.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:50:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:50:29] "GET /metrics HTTP/1.1" 200 36745 "" "Prometheus/2.43.0" 2024-09-13T00:50:30.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:30 smithi136 bash[75164]: cluster 2024-09-13T00:50:28.701822+0000 mgr.y (mgr.24661) 2200 : cluster [DBG] pgmap v4312: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 228 KiB/s rd, 14 KiB/s wr, 306 op/s 2024-09-13T00:50:30.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:30 smithi136 bash[75164]: cluster 2024-09-13T00:50:29.144557+0000 mon.a (mon.0) 4343 : cluster [DBG] osdmap e2552: 8 total, 8 up, 8 in 2024-09-13T00:50:30.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:30 smithi136 bash[75164]: cluster 2024-09-13T00:50:30.041238+0000 mon.a (mon.0) 4344 : cluster [DBG] osdmap e2553: 8 total, 8 up, 8 in 2024-09-13T00:50:30.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:30 smithi136 bash[76371]: cluster 2024-09-13T00:50:28.701822+0000 mgr.y (mgr.24661) 2200 : cluster [DBG] pgmap v4312: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 228 KiB/s rd, 14 KiB/s wr, 306 op/s 2024-09-13T00:50:30.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:30 smithi136 bash[76371]: cluster 2024-09-13T00:50:29.144557+0000 mon.a (mon.0) 4343 : cluster [DBG] osdmap e2552: 8 total, 8 up, 8 in 2024-09-13T00:50:30.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:30 smithi136 bash[76371]: cluster 2024-09-13T00:50:30.041238+0000 mon.a (mon.0) 4344 : cluster [DBG] osdmap e2553: 8 total, 8 up, 8 in 2024-09-13T00:50:30.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:30 smithi181 bash[67587]: cluster 2024-09-13T00:50:28.701822+0000 mgr.y (mgr.24661) 2200 : cluster [DBG] pgmap v4312: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 228 KiB/s rd, 14 KiB/s wr, 306 op/s 2024-09-13T00:50:30.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:30 smithi181 bash[67587]: cluster 2024-09-13T00:50:29.144557+0000 mon.a (mon.0) 4343 : cluster [DBG] osdmap e2552: 8 total, 8 up, 8 in 2024-09-13T00:50:30.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:30 smithi181 bash[67587]: cluster 2024-09-13T00:50:30.041238+0000 mon.a (mon.0) 4344 : cluster [DBG] osdmap e2553: 8 total, 8 up, 8 in 2024-09-13T00:50:32.381 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestClone::test_flatten_smaller_order PASSED [ 67%] 2024-09-13T00:50:32.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:32 smithi136 bash[75164]: cluster 2024-09-13T00:50:30.702914+0000 mgr.y (mgr.24661) 2201 : cluster [DBG] pgmap v4315: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 241 KiB/s rd, 17 KiB/s wr, 326 op/s 2024-09-13T00:50:32.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:32 smithi136 bash[76371]: cluster 2024-09-13T00:50:30.702914+0000 mgr.y (mgr.24661) 2201 : cluster [DBG] pgmap v4315: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 241 KiB/s rd, 17 KiB/s wr, 326 op/s 2024-09-13T00:50:32.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:32 smithi181 bash[67587]: cluster 2024-09-13T00:50:30.702914+0000 mgr.y (mgr.24661) 2201 : cluster [DBG] pgmap v4315: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 241 KiB/s rd, 17 KiB/s wr, 326 op/s 2024-09-13T00:50:32.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:50:32 smithi181 bash[95505]: ts=2024-09-13T00:50:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:50:33.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:33 smithi136 bash[75164]: cluster 2024-09-13T00:50:32.151717+0000 mon.a (mon.0) 4345 : cluster [DBG] osdmap e2554: 8 total, 8 up, 8 in 2024-09-13T00:50:33.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:33 smithi136 bash[76371]: cluster 2024-09-13T00:50:32.151717+0000 mon.a (mon.0) 4345 : cluster [DBG] osdmap e2554: 8 total, 8 up, 8 in 2024-09-13T00:50:33.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:33 smithi181 bash[67587]: cluster 2024-09-13T00:50:32.151717+0000 mon.a (mon.0) 4345 : cluster [DBG] osdmap e2554: 8 total, 8 up, 8 in 2024-09-13T00:50:34.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:34 smithi136 bash[75164]: cluster 2024-09-13T00:50:32.703635+0000 mgr.y (mgr.24661) 2202 : cluster [DBG] pgmap v4317: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 325 KiB/s rd, 20 KiB/s wr, 425 op/s 2024-09-13T00:50:34.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:34 smithi136 bash[75164]: cluster 2024-09-13T00:50:33.179606+0000 mon.a (mon.0) 4346 : cluster [DBG] osdmap e2555: 8 total, 8 up, 8 in 2024-09-13T00:50:34.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:34 smithi136 bash[76371]: cluster 2024-09-13T00:50:32.703635+0000 mgr.y (mgr.24661) 2202 : cluster [DBG] pgmap v4317: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 325 KiB/s rd, 20 KiB/s wr, 425 op/s 2024-09-13T00:50:34.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:34 smithi136 bash[76371]: cluster 2024-09-13T00:50:33.179606+0000 mon.a (mon.0) 4346 : cluster [DBG] osdmap e2555: 8 total, 8 up, 8 in 2024-09-13T00:50:34.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:34 smithi181 bash[67587]: cluster 2024-09-13T00:50:32.703635+0000 mgr.y (mgr.24661) 2202 : cluster [DBG] pgmap v4317: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 325 KiB/s rd, 20 KiB/s wr, 425 op/s 2024-09-13T00:50:34.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:34 smithi181 bash[67587]: cluster 2024-09-13T00:50:33.179606+0000 mon.a (mon.0) 4346 : cluster [DBG] osdmap e2555: 8 total, 8 up, 8 in 2024-09-13T00:50:35.268 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestClone::test_flatten_larger_order PASSED [ 68%] 2024-09-13T00:50:36.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:36 smithi181 bash[67587]: cluster 2024-09-13T00:50:34.704582+0000 mgr.y (mgr.24661) 2203 : cluster [DBG] pgmap v4319: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 336 KiB/s rd, 24 KiB/s wr, 448 op/s 2024-09-13T00:50:36.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:36 smithi181 bash[67587]: cluster 2024-09-13T00:50:35.046663+0000 mon.a (mon.0) 4347 : cluster [DBG] osdmap e2556: 8 total, 8 up, 8 in 2024-09-13T00:50:36.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:36 smithi136 bash[76371]: cluster 2024-09-13T00:50:34.704582+0000 mgr.y (mgr.24661) 2203 : cluster [DBG] pgmap v4319: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 336 KiB/s rd, 24 KiB/s wr, 448 op/s 2024-09-13T00:50:36.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:36 smithi136 bash[76371]: cluster 2024-09-13T00:50:35.046663+0000 mon.a (mon.0) 4347 : cluster [DBG] osdmap e2556: 8 total, 8 up, 8 in 2024-09-13T00:50:36.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:36 smithi136 bash[75164]: cluster 2024-09-13T00:50:34.704582+0000 mgr.y (mgr.24661) 2203 : cluster [DBG] pgmap v4319: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 336 KiB/s rd, 24 KiB/s wr, 448 op/s 2024-09-13T00:50:36.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:36 smithi136 bash[75164]: cluster 2024-09-13T00:50:35.046663+0000 mon.a (mon.0) 4347 : cluster [DBG] osdmap e2556: 8 total, 8 up, 8 in 2024-09-13T00:50:37.401 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestClone::test_flatten_drops_cache PASSED [ 68%] 2024-09-13T00:50:37.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:37 smithi136 bash[75164]: cluster 2024-09-13T00:50:36.180038+0000 mon.a (mon.0) 4348 : cluster [DBG] osdmap e2557: 8 total, 8 up, 8 in 2024-09-13T00:50:37.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:37 smithi136 bash[76371]: cluster 2024-09-13T00:50:36.180038+0000 mon.a (mon.0) 4348 : cluster [DBG] osdmap e2557: 8 total, 8 up, 8 in 2024-09-13T00:50:37.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:37 smithi181 bash[67587]: cluster 2024-09-13T00:50:36.180038+0000 mon.a (mon.0) 4348 : cluster [DBG] osdmap e2557: 8 total, 8 up, 8 in 2024-09-13T00:50:38.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:38 smithi136 bash[75164]: cluster 2024-09-13T00:50:36.705602+0000 mgr.y (mgr.24661) 2204 : cluster [DBG] pgmap v4322: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 476 KiB/s rd, 34 KiB/s wr, 647 op/s 2024-09-13T00:50:38.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:38 smithi136 bash[75164]: cluster 2024-09-13T00:50:37.193218+0000 mon.a (mon.0) 4349 : cluster [DBG] osdmap e2558: 8 total, 8 up, 8 in 2024-09-13T00:50:38.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:38 smithi136 bash[76371]: cluster 2024-09-13T00:50:36.705602+0000 mgr.y (mgr.24661) 2204 : cluster [DBG] pgmap v4322: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 476 KiB/s rd, 34 KiB/s wr, 647 op/s 2024-09-13T00:50:38.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:38 smithi136 bash[76371]: cluster 2024-09-13T00:50:37.193218+0000 mon.a (mon.0) 4349 : cluster [DBG] osdmap e2558: 8 total, 8 up, 8 in 2024-09-13T00:50:38.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:38 smithi181 bash[67587]: cluster 2024-09-13T00:50:36.705602+0000 mgr.y (mgr.24661) 2204 : cluster [DBG] pgmap v4322: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 476 KiB/s rd, 34 KiB/s wr, 647 op/s 2024-09-13T00:50:38.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:38 smithi181 bash[67587]: cluster 2024-09-13T00:50:37.193218+0000 mon.a (mon.0) 4349 : cluster [DBG] osdmap e2558: 8 total, 8 up, 8 in 2024-09-13T00:50:39.510 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:50:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:50:39] "GET /metrics HTTP/1.1" 200 36743 "" "Prometheus/2.43.0" 2024-09-13T00:50:39.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:39 smithi136 bash[75164]: cluster 2024-09-13T00:50:38.196761+0000 mon.a (mon.0) 4350 : cluster [DBG] osdmap e2559: 8 total, 8 up, 8 in 2024-09-13T00:50:39.511 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:39 smithi136 bash[75164]: audit 2024-09-13T00:50:38.855497+0000 mon.a (mon.0) 4351 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:50:39.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:39 smithi136 bash[76371]: cluster 2024-09-13T00:50:38.196761+0000 mon.a (mon.0) 4350 : cluster [DBG] osdmap e2559: 8 total, 8 up, 8 in 2024-09-13T00:50:39.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:39 smithi136 bash[76371]: audit 2024-09-13T00:50:38.855497+0000 mon.a (mon.0) 4351 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:50:39.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:39 smithi181 bash[67587]: cluster 2024-09-13T00:50:38.196761+0000 mon.a (mon.0) 4350 : cluster [DBG] osdmap e2559: 8 total, 8 up, 8 in 2024-09-13T00:50:39.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:39 smithi181 bash[67587]: audit 2024-09-13T00:50:38.855497+0000 mon.a (mon.0) 4351 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:50:40.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:40 smithi136 bash[75164]: cluster 2024-09-13T00:50:38.706479+0000 mgr.y (mgr.24661) 2205 : cluster [DBG] pgmap v4325: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 625 KiB/s rd, 40 KiB/s wr, 839 op/s 2024-09-13T00:50:40.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:40 smithi136 bash[75164]: cluster 2024-09-13T00:50:39.224424+0000 mon.a (mon.0) 4352 : cluster [DBG] osdmap e2560: 8 total, 8 up, 8 in 2024-09-13T00:50:40.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:40 smithi136 bash[75164]: cluster 2024-09-13T00:50:40.047366+0000 mon.a (mon.0) 4353 : cluster [DBG] osdmap e2561: 8 total, 8 up, 8 in 2024-09-13T00:50:40.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:40 smithi136 bash[76371]: cluster 2024-09-13T00:50:38.706479+0000 mgr.y (mgr.24661) 2205 : cluster [DBG] pgmap v4325: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 625 KiB/s rd, 40 KiB/s wr, 839 op/s 2024-09-13T00:50:40.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:40 smithi136 bash[76371]: cluster 2024-09-13T00:50:39.224424+0000 mon.a (mon.0) 4352 : cluster [DBG] osdmap e2560: 8 total, 8 up, 8 in 2024-09-13T00:50:40.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:40 smithi136 bash[76371]: cluster 2024-09-13T00:50:40.047366+0000 mon.a (mon.0) 4353 : cluster [DBG] osdmap e2561: 8 total, 8 up, 8 in 2024-09-13T00:50:40.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:40 smithi181 bash[67587]: cluster 2024-09-13T00:50:38.706479+0000 mgr.y (mgr.24661) 2205 : cluster [DBG] pgmap v4325: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 625 KiB/s rd, 40 KiB/s wr, 839 op/s 2024-09-13T00:50:40.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:40 smithi181 bash[67587]: cluster 2024-09-13T00:50:39.224424+0000 mon.a (mon.0) 4352 : cluster [DBG] osdmap e2560: 8 total, 8 up, 8 in 2024-09-13T00:50:40.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:40 smithi181 bash[67587]: cluster 2024-09-13T00:50:40.047366+0000 mon.a (mon.0) 4353 : cluster [DBG] osdmap e2561: 8 total, 8 up, 8 in 2024-09-13T00:50:41.444 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestClone::test_flatten_multi_level PASSED [ 69%] 2024-09-13T00:50:42.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:42 smithi136 bash[75164]: cluster 2024-09-13T00:50:40.707480+0000 mgr.y (mgr.24661) 2206 : cluster [DBG] pgmap v4328: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 255 KiB/s rd, 16 KiB/s wr, 346 op/s 2024-09-13T00:50:42.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:42 smithi136 bash[75164]: cluster 2024-09-13T00:50:41.228478+0000 mon.a (mon.0) 4354 : cluster [DBG] osdmap e2562: 8 total, 8 up, 8 in 2024-09-13T00:50:42.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:42 smithi136 bash[76371]: cluster 2024-09-13T00:50:40.707480+0000 mgr.y (mgr.24661) 2206 : cluster [DBG] pgmap v4328: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 255 KiB/s rd, 16 KiB/s wr, 346 op/s 2024-09-13T00:50:42.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:42 smithi136 bash[76371]: cluster 2024-09-13T00:50:41.228478+0000 mon.a (mon.0) 4354 : cluster [DBG] osdmap e2562: 8 total, 8 up, 8 in 2024-09-13T00:50:42.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:42 smithi181 bash[67587]: cluster 2024-09-13T00:50:40.707480+0000 mgr.y (mgr.24661) 2206 : cluster [DBG] pgmap v4328: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 255 KiB/s rd, 16 KiB/s wr, 346 op/s 2024-09-13T00:50:42.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:42 smithi181 bash[67587]: cluster 2024-09-13T00:50:41.228478+0000 mon.a (mon.0) 4354 : cluster [DBG] osdmap e2562: 8 total, 8 up, 8 in 2024-09-13T00:50:42.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:50:42 smithi181 bash[95505]: ts=2024-09-13T00:50:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:50:43.461 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestClone::test_flatten_with_progress PASSED [ 69%] 2024-09-13T00:50:43.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:43 smithi136 bash[75164]: cluster 2024-09-13T00:50:42.244817+0000 mon.a (mon.0) 4355 : cluster [DBG] osdmap e2563: 8 total, 8 up, 8 in 2024-09-13T00:50:43.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:43 smithi136 bash[76371]: cluster 2024-09-13T00:50:42.244817+0000 mon.a (mon.0) 4355 : cluster [DBG] osdmap e2563: 8 total, 8 up, 8 in 2024-09-13T00:50:43.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:43 smithi181 bash[67587]: cluster 2024-09-13T00:50:42.244817+0000 mon.a (mon.0) 4355 : cluster [DBG] osdmap e2563: 8 total, 8 up, 8 in 2024-09-13T00:50:44.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:44 smithi136 bash[75164]: cluster 2024-09-13T00:50:42.708170+0000 mgr.y (mgr.24661) 2207 : cluster [DBG] pgmap v4331: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 142 KiB/s rd, 16 KiB/s wr, 200 op/s 2024-09-13T00:50:44.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:44 smithi136 bash[75164]: cluster 2024-09-13T00:50:43.253152+0000 mon.a (mon.0) 4356 : cluster [DBG] osdmap e2564: 8 total, 8 up, 8 in 2024-09-13T00:50:44.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:44 smithi136 bash[76371]: cluster 2024-09-13T00:50:42.708170+0000 mgr.y (mgr.24661) 2207 : cluster [DBG] pgmap v4331: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 142 KiB/s rd, 16 KiB/s wr, 200 op/s 2024-09-13T00:50:44.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:44 smithi136 bash[76371]: cluster 2024-09-13T00:50:43.253152+0000 mon.a (mon.0) 4356 : cluster [DBG] osdmap e2564: 8 total, 8 up, 8 in 2024-09-13T00:50:44.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:44 smithi181 bash[67587]: cluster 2024-09-13T00:50:42.708170+0000 mgr.y (mgr.24661) 2207 : cluster [DBG] pgmap v4331: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 142 KiB/s rd, 16 KiB/s wr, 200 op/s 2024-09-13T00:50:44.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:44 smithi181 bash[67587]: cluster 2024-09-13T00:50:43.253152+0000 mon.a (mon.0) 4356 : cluster [DBG] osdmap e2564: 8 total, 8 up, 8 in 2024-09-13T00:50:45.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:45 smithi181 bash[67587]: cluster 2024-09-13T00:50:44.263294+0000 mon.a (mon.0) 4357 : cluster [DBG] osdmap e2565: 8 total, 8 up, 8 in 2024-09-13T00:50:45.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:45 smithi181 bash[67587]: cluster 2024-09-13T00:50:45.048609+0000 mon.a (mon.0) 4358 : cluster [DBG] osdmap e2566: 8 total, 8 up, 8 in 2024-09-13T00:50:45.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:45 smithi136 bash[76371]: cluster 2024-09-13T00:50:44.263294+0000 mon.a (mon.0) 4357 : cluster [DBG] osdmap e2565: 8 total, 8 up, 8 in 2024-09-13T00:50:45.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:45 smithi136 bash[76371]: cluster 2024-09-13T00:50:45.048609+0000 mon.a (mon.0) 4358 : cluster [DBG] osdmap e2566: 8 total, 8 up, 8 in 2024-09-13T00:50:45.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:45 smithi136 bash[75164]: cluster 2024-09-13T00:50:44.263294+0000 mon.a (mon.0) 4357 : cluster [DBG] osdmap e2565: 8 total, 8 up, 8 in 2024-09-13T00:50:45.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:45 smithi136 bash[75164]: cluster 2024-09-13T00:50:45.048609+0000 mon.a (mon.0) 4358 : cluster [DBG] osdmap e2566: 8 total, 8 up, 8 in 2024-09-13T00:50:46.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:46 smithi181 bash[67587]: cluster 2024-09-13T00:50:44.708962+0000 mgr.y (mgr.24661) 2208 : cluster [DBG] pgmap v4334: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 309 KiB/s rd, 20 KiB/s wr, 408 op/s 2024-09-13T00:50:46.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:46 smithi136 bash[75164]: cluster 2024-09-13T00:50:44.708962+0000 mgr.y (mgr.24661) 2208 : cluster [DBG] pgmap v4334: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 309 KiB/s rd, 20 KiB/s wr, 408 op/s 2024-09-13T00:50:46.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:46 smithi136 bash[76371]: cluster 2024-09-13T00:50:44.708962+0000 mgr.y (mgr.24661) 2208 : cluster [DBG] pgmap v4334: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 309 KiB/s rd, 20 KiB/s wr, 408 op/s 2024-09-13T00:50:47.523 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestClone::test_resize_flatten_multi_level PASSED [ 70%] 2024-09-13T00:50:47.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:47 smithi181 bash[67587]: cluster 2024-09-13T00:50:46.285656+0000 mon.a (mon.0) 4359 : cluster [DBG] osdmap e2567: 8 total, 8 up, 8 in 2024-09-13T00:50:47.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:47 smithi136 bash[75164]: cluster 2024-09-13T00:50:46.285656+0000 mon.a (mon.0) 4359 : cluster [DBG] osdmap e2567: 8 total, 8 up, 8 in 2024-09-13T00:50:47.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:47 smithi136 bash[76371]: cluster 2024-09-13T00:50:46.285656+0000 mon.a (mon.0) 4359 : cluster [DBG] osdmap e2567: 8 total, 8 up, 8 in 2024-09-13T00:50:48.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:48 smithi181 bash[67587]: cluster 2024-09-13T00:50:46.709811+0000 mgr.y (mgr.24661) 2209 : cluster [DBG] pgmap v4337: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 397 KiB/s rd, 29 KiB/s wr, 536 op/s 2024-09-13T00:50:48.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:48 smithi181 bash[67587]: cluster 2024-09-13T00:50:47.295446+0000 mon.a (mon.0) 4360 : cluster [DBG] osdmap e2568: 8 total, 8 up, 8 in 2024-09-13T00:50:48.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:48 smithi136 bash[75164]: cluster 2024-09-13T00:50:46.709811+0000 mgr.y (mgr.24661) 2209 : cluster [DBG] pgmap v4337: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 397 KiB/s rd, 29 KiB/s wr, 536 op/s 2024-09-13T00:50:48.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:48 smithi136 bash[75164]: cluster 2024-09-13T00:50:47.295446+0000 mon.a (mon.0) 4360 : cluster [DBG] osdmap e2568: 8 total, 8 up, 8 in 2024-09-13T00:50:48.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:48 smithi136 bash[76371]: cluster 2024-09-13T00:50:46.709811+0000 mgr.y (mgr.24661) 2209 : cluster [DBG] pgmap v4337: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 397 KiB/s rd, 29 KiB/s wr, 536 op/s 2024-09-13T00:50:48.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:48 smithi136 bash[76371]: cluster 2024-09-13T00:50:47.295446+0000 mon.a (mon.0) 4360 : cluster [DBG] osdmap e2568: 8 total, 8 up, 8 in 2024-09-13T00:50:49.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:49 smithi181 bash[67587]: cluster 2024-09-13T00:50:48.302989+0000 mon.a (mon.0) 4361 : cluster [DBG] osdmap e2569: 8 total, 8 up, 8 in 2024-09-13T00:50:49.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:49 smithi136 bash[75164]: cluster 2024-09-13T00:50:48.302989+0000 mon.a (mon.0) 4361 : cluster [DBG] osdmap e2569: 8 total, 8 up, 8 in 2024-09-13T00:50:49.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:50:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:50:49] "GET /metrics HTTP/1.1" 200 36743 "" "Prometheus/2.43.0" 2024-09-13T00:50:49.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:49 smithi136 bash[76371]: cluster 2024-09-13T00:50:48.302989+0000 mon.a (mon.0) 4361 : cluster [DBG] osdmap e2569: 8 total, 8 up, 8 in 2024-09-13T00:50:50.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:50 smithi181 bash[67587]: cluster 2024-09-13T00:50:48.710757+0000 mgr.y (mgr.24661) 2210 : cluster [DBG] pgmap v4340: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 328 KiB/s rd, 25 KiB/s wr, 444 op/s 2024-09-13T00:50:50.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:50 smithi181 bash[67587]: cluster 2024-09-13T00:50:49.314026+0000 mon.a (mon.0) 4362 : cluster [DBG] osdmap e2570: 8 total, 8 up, 8 in 2024-09-13T00:50:50.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:50 smithi181 bash[67587]: cluster 2024-09-13T00:50:50.053430+0000 mon.a (mon.0) 4363 : cluster [DBG] osdmap e2571: 8 total, 8 up, 8 in 2024-09-13T00:50:50.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:50 smithi136 bash[75164]: cluster 2024-09-13T00:50:48.710757+0000 mgr.y (mgr.24661) 2210 : cluster [DBG] pgmap v4340: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 328 KiB/s rd, 25 KiB/s wr, 444 op/s 2024-09-13T00:50:50.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:50 smithi136 bash[75164]: cluster 2024-09-13T00:50:49.314026+0000 mon.a (mon.0) 4362 : cluster [DBG] osdmap e2570: 8 total, 8 up, 8 in 2024-09-13T00:50:50.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:50 smithi136 bash[75164]: cluster 2024-09-13T00:50:50.053430+0000 mon.a (mon.0) 4363 : cluster [DBG] osdmap e2571: 8 total, 8 up, 8 in 2024-09-13T00:50:50.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:50 smithi136 bash[76371]: cluster 2024-09-13T00:50:48.710757+0000 mgr.y (mgr.24661) 2210 : cluster [DBG] pgmap v4340: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 328 KiB/s rd, 25 KiB/s wr, 444 op/s 2024-09-13T00:50:50.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:50 smithi136 bash[76371]: cluster 2024-09-13T00:50:49.314026+0000 mon.a (mon.0) 4362 : cluster [DBG] osdmap e2570: 8 total, 8 up, 8 in 2024-09-13T00:50:50.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:50 smithi136 bash[76371]: cluster 2024-09-13T00:50:50.053430+0000 mon.a (mon.0) 4363 : cluster [DBG] osdmap e2571: 8 total, 8 up, 8 in 2024-09-13T00:50:51.523 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestClone::test_trash_snapshot PASSED [ 70%] 2024-09-13T00:50:51.809 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestExclusiveLock::test_ownership PASSED [ 71%] 2024-09-13T00:50:52.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:52 smithi181 bash[67587]: cluster 2024-09-13T00:50:50.711733+0000 mgr.y (mgr.24661) 2211 : cluster [DBG] pgmap v4343: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 203 KiB/s rd, 14 KiB/s wr, 276 op/s 2024-09-13T00:50:52.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:52 smithi181 bash[67587]: cluster 2024-09-13T00:50:51.324219+0000 mon.a (mon.0) 4364 : cluster [DBG] osdmap e2572: 8 total, 8 up, 8 in 2024-09-13T00:50:52.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:50:52 smithi181 bash[95505]: ts=2024-09-13T00:50:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:50:52.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:52 smithi136 bash[75164]: cluster 2024-09-13T00:50:50.711733+0000 mgr.y (mgr.24661) 2211 : cluster [DBG] pgmap v4343: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 203 KiB/s rd, 14 KiB/s wr, 276 op/s 2024-09-13T00:50:52.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:52 smithi136 bash[75164]: cluster 2024-09-13T00:50:51.324219+0000 mon.a (mon.0) 4364 : cluster [DBG] osdmap e2572: 8 total, 8 up, 8 in 2024-09-13T00:50:52.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:52 smithi136 bash[76371]: cluster 2024-09-13T00:50:50.711733+0000 mgr.y (mgr.24661) 2211 : cluster [DBG] pgmap v4343: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 203 KiB/s rd, 14 KiB/s wr, 276 op/s 2024-09-13T00:50:52.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:52 smithi136 bash[76371]: cluster 2024-09-13T00:50:51.324219+0000 mon.a (mon.0) 4364 : cluster [DBG] osdmap e2572: 8 total, 8 up, 8 in 2024-09-13T00:50:53.564 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestExclusiveLock::test_snapshot_leadership PASSED [ 71%] 2024-09-13T00:50:53.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:53 smithi181 bash[67587]: cluster 2024-09-13T00:50:52.329324+0000 mon.a (mon.0) 4365 : cluster [DBG] osdmap e2573: 8 total, 8 up, 8 in 2024-09-13T00:50:53.718 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestExclusiveLock::test_read_only_leadership PASSED [ 72%] 2024-09-13T00:50:53.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:53 smithi136 bash[75164]: cluster 2024-09-13T00:50:52.329324+0000 mon.a (mon.0) 4365 : cluster [DBG] osdmap e2573: 8 total, 8 up, 8 in 2024-09-13T00:50:53.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:53 smithi136 bash[76371]: cluster 2024-09-13T00:50:52.329324+0000 mon.a (mon.0) 4365 : cluster [DBG] osdmap e2573: 8 total, 8 up, 8 in 2024-09-13T00:50:54.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:54 smithi136 bash[75164]: cluster 2024-09-13T00:50:52.712470+0000 mgr.y (mgr.24661) 2212 : cluster [DBG] pgmap v4346: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 78 KiB/s rd, 10 KiB/s wr, 121 op/s 2024-09-13T00:50:54.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:54 smithi136 bash[75164]: cluster 2024-09-13T00:50:53.350191+0000 mon.a (mon.0) 4366 : cluster [DBG] osdmap e2574: 8 total, 8 up, 8 in 2024-09-13T00:50:54.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:54 smithi136 bash[75164]: audit 2024-09-13T00:50:53.855975+0000 mon.a (mon.0) 4367 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:50:54.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:54 smithi136 bash[76371]: cluster 2024-09-13T00:50:52.712470+0000 mgr.y (mgr.24661) 2212 : cluster [DBG] pgmap v4346: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 78 KiB/s rd, 10 KiB/s wr, 121 op/s 2024-09-13T00:50:54.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:54 smithi136 bash[76371]: cluster 2024-09-13T00:50:53.350191+0000 mon.a (mon.0) 4366 : cluster [DBG] osdmap e2574: 8 total, 8 up, 8 in 2024-09-13T00:50:54.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:54 smithi136 bash[76371]: audit 2024-09-13T00:50:53.855975+0000 mon.a (mon.0) 4367 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:50:54.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:54 smithi181 bash[67587]: cluster 2024-09-13T00:50:52.712470+0000 mgr.y (mgr.24661) 2212 : cluster [DBG] pgmap v4346: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 78 KiB/s rd, 10 KiB/s wr, 121 op/s 2024-09-13T00:50:54.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:54 smithi181 bash[67587]: cluster 2024-09-13T00:50:53.350191+0000 mon.a (mon.0) 4366 : cluster [DBG] osdmap e2574: 8 total, 8 up, 8 in 2024-09-13T00:50:54.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:54 smithi181 bash[67587]: audit 2024-09-13T00:50:53.855975+0000 mon.a (mon.0) 4367 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:50:55.273 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestExclusiveLock::test_follower_flatten PASSED [ 73%] 2024-09-13T00:50:55.599 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestExclusiveLock::test_follower_resize PASSED [ 73%] 2024-09-13T00:50:55.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:55 smithi136 bash[76371]: cluster 2024-09-13T00:50:54.350834+0000 mon.a (mon.0) 4368 : cluster [DBG] osdmap e2575: 8 total, 8 up, 8 in 2024-09-13T00:50:55.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:55 smithi136 bash[76371]: cluster 2024-09-13T00:50:55.055967+0000 mon.a (mon.0) 4369 : cluster [DBG] osdmap e2576: 8 total, 8 up, 8 in 2024-09-13T00:50:55.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:55 smithi136 bash[75164]: cluster 2024-09-13T00:50:54.350834+0000 mon.a (mon.0) 4368 : cluster [DBG] osdmap e2575: 8 total, 8 up, 8 in 2024-09-13T00:50:55.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:55 smithi136 bash[75164]: cluster 2024-09-13T00:50:55.055967+0000 mon.a (mon.0) 4369 : cluster [DBG] osdmap e2576: 8 total, 8 up, 8 in 2024-09-13T00:50:55.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:55 smithi181 bash[67587]: cluster 2024-09-13T00:50:54.350834+0000 mon.a (mon.0) 4368 : cluster [DBG] osdmap e2575: 8 total, 8 up, 8 in 2024-09-13T00:50:55.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:55 smithi181 bash[67587]: cluster 2024-09-13T00:50:55.055967+0000 mon.a (mon.0) 4369 : cluster [DBG] osdmap e2576: 8 total, 8 up, 8 in 2024-09-13T00:50:56.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:56 smithi136 bash[75164]: cluster 2024-09-13T00:50:54.713197+0000 mgr.y (mgr.24661) 2213 : cluster [DBG] pgmap v4349: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 200 KiB/s rd, 12 KiB/s wr, 266 op/s 2024-09-13T00:50:56.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:56 smithi136 bash[76371]: cluster 2024-09-13T00:50:54.713197+0000 mgr.y (mgr.24661) 2213 : cluster [DBG] pgmap v4349: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 200 KiB/s rd, 12 KiB/s wr, 266 op/s 2024-09-13T00:50:56.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:56 smithi181 bash[67587]: cluster 2024-09-13T00:50:54.713197+0000 mgr.y (mgr.24661) 2213 : cluster [DBG] pgmap v4349: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 200 KiB/s rd, 12 KiB/s wr, 266 op/s 2024-09-13T00:50:57.616 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestExclusiveLock::test_follower_snap_create PASSED [ 74%] 2024-09-13T00:50:57.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:57 smithi136 bash[75164]: cluster 2024-09-13T00:50:56.377727+0000 mon.a (mon.0) 4370 : cluster [DBG] osdmap e2577: 8 total, 8 up, 8 in 2024-09-13T00:50:57.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:57 smithi136 bash[76371]: cluster 2024-09-13T00:50:56.377727+0000 mon.a (mon.0) 4370 : cluster [DBG] osdmap e2577: 8 total, 8 up, 8 in 2024-09-13T00:50:57.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:57 smithi181 bash[67587]: cluster 2024-09-13T00:50:56.377727+0000 mon.a (mon.0) 4370 : cluster [DBG] osdmap e2577: 8 total, 8 up, 8 in 2024-09-13T00:50:58.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:58 smithi136 bash[75164]: cluster 2024-09-13T00:50:56.714047+0000 mgr.y (mgr.24661) 2214 : cluster [DBG] pgmap v4352: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 306 KiB/s rd, 20 KiB/s wr, 406 op/s 2024-09-13T00:50:58.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:58 smithi136 bash[75164]: cluster 2024-09-13T00:50:57.402402+0000 mon.a (mon.0) 4371 : cluster [DBG] osdmap e2578: 8 total, 8 up, 8 in 2024-09-13T00:50:58.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:58 smithi136 bash[76371]: cluster 2024-09-13T00:50:56.714047+0000 mgr.y (mgr.24661) 2214 : cluster [DBG] pgmap v4352: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 306 KiB/s rd, 20 KiB/s wr, 406 op/s 2024-09-13T00:50:58.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:58 smithi136 bash[76371]: cluster 2024-09-13T00:50:57.402402+0000 mon.a (mon.0) 4371 : cluster [DBG] osdmap e2578: 8 total, 8 up, 8 in 2024-09-13T00:50:58.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:58 smithi181 bash[67587]: cluster 2024-09-13T00:50:56.714047+0000 mgr.y (mgr.24661) 2214 : cluster [DBG] pgmap v4352: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 306 KiB/s rd, 20 KiB/s wr, 406 op/s 2024-09-13T00:50:58.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:58 smithi181 bash[67587]: cluster 2024-09-13T00:50:57.402402+0000 mon.a (mon.0) 4371 : cluster [DBG] osdmap e2578: 8 total, 8 up, 8 in 2024-09-13T00:50:59.635 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestExclusiveLock::test_follower_snap_rollback PASSED [ 74%] 2024-09-13T00:50:59.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:50:59 smithi136 bash[75164]: cluster 2024-09-13T00:50:58.406175+0000 mon.a (mon.0) 4372 : cluster [DBG] osdmap e2579: 8 total, 8 up, 8 in 2024-09-13T00:50:59.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:50:59 smithi136 bash[76371]: cluster 2024-09-13T00:50:58.406175+0000 mon.a (mon.0) 4372 : cluster [DBG] osdmap e2579: 8 total, 8 up, 8 in 2024-09-13T00:50:59.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:50:59 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:50:59] "GET /metrics HTTP/1.1" 200 36742 "" "Prometheus/2.43.0" 2024-09-13T00:50:59.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:50:59 smithi181 bash[67587]: cluster 2024-09-13T00:50:58.406175+0000 mon.a (mon.0) 4372 : cluster [DBG] osdmap e2579: 8 total, 8 up, 8 in 2024-09-13T00:50:59.928 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestExclusiveLock::test_follower_discard PASSED [ 75%] 2024-09-13T00:51:00.178 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestExclusiveLock::test_follower_write PASSED [ 75%] 2024-09-13T00:51:00.378 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestExclusiveLock::test_acquire_release_lock PASSED [ 76%] 2024-09-13T00:51:00.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:00 smithi136 bash[75164]: cluster 2024-09-13T00:50:58.714996+0000 mgr.y (mgr.24661) 2215 : cluster [DBG] pgmap v4355: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 305 KiB/s rd, 22 KiB/s wr, 417 op/s 2024-09-13T00:51:00.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:00 smithi136 bash[75164]: cluster 2024-09-13T00:50:59.429445+0000 mon.a (mon.0) 4373 : cluster [DBG] osdmap e2580: 8 total, 8 up, 8 in 2024-09-13T00:51:00.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:00 smithi136 bash[75164]: cluster 2024-09-13T00:51:00.051927+0000 mon.a (mon.0) 4374 : cluster [DBG] osdmap e2581: 8 total, 8 up, 8 in 2024-09-13T00:51:00.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:00 smithi136 bash[76371]: cluster 2024-09-13T00:50:58.714996+0000 mgr.y (mgr.24661) 2215 : cluster [DBG] pgmap v4355: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 305 KiB/s rd, 22 KiB/s wr, 417 op/s 2024-09-13T00:51:00.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:00 smithi136 bash[76371]: cluster 2024-09-13T00:50:59.429445+0000 mon.a (mon.0) 4373 : cluster [DBG] osdmap e2580: 8 total, 8 up, 8 in 2024-09-13T00:51:00.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:00 smithi136 bash[76371]: cluster 2024-09-13T00:51:00.051927+0000 mon.a (mon.0) 4374 : cluster [DBG] osdmap e2581: 8 total, 8 up, 8 in 2024-09-13T00:51:00.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:00 smithi181 bash[67587]: cluster 2024-09-13T00:50:58.714996+0000 mgr.y (mgr.24661) 2215 : cluster [DBG] pgmap v4355: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 305 KiB/s rd, 22 KiB/s wr, 417 op/s 2024-09-13T00:51:00.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:00 smithi181 bash[67587]: cluster 2024-09-13T00:50:59.429445+0000 mon.a (mon.0) 4373 : cluster [DBG] osdmap e2580: 8 total, 8 up, 8 in 2024-09-13T00:51:00.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:00 smithi181 bash[67587]: cluster 2024-09-13T00:51:00.051927+0000 mon.a (mon.0) 4374 : cluster [DBG] osdmap e2581: 8 total, 8 up, 8 in 2024-09-13T00:51:01.634 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestExclusiveLock::test_break_lock PASSED [ 76%] 2024-09-13T00:51:01.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:01 smithi136 bash[75164]: audit 2024-09-13T00:51:00.466784+0000 mon.a (mon.0) 4375 : audit [INF] from='client.? 172.21.15.136:0/205774720' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.136:0/2231517774"}]: dispatch 2024-09-13T00:51:01.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:01 smithi136 bash[75164]: audit 2024-09-13T00:51:00.751286+0000 mon.a (mon.0) 4376 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:51:01.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:01 smithi136 bash[75164]: audit 2024-09-13T00:51:01.271281+0000 mon.a (mon.0) 4377 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:51:01.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:01 smithi136 bash[75164]: audit 2024-09-13T00:51:01.272471+0000 mon.a (mon.0) 4378 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:51:01.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:01 smithi136 bash[75164]: audit 2024-09-13T00:51:01.281996+0000 mon.a (mon.0) 4379 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:51:01.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:01 smithi136 bash[76371]: audit 2024-09-13T00:51:00.466784+0000 mon.a (mon.0) 4375 : audit [INF] from='client.? 172.21.15.136:0/205774720' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.136:0/2231517774"}]: dispatch 2024-09-13T00:51:01.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:01 smithi136 bash[76371]: audit 2024-09-13T00:51:00.751286+0000 mon.a (mon.0) 4376 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:51:01.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:01 smithi136 bash[76371]: audit 2024-09-13T00:51:01.271281+0000 mon.a (mon.0) 4377 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:51:01.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:01 smithi136 bash[76371]: audit 2024-09-13T00:51:01.272471+0000 mon.a (mon.0) 4378 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:51:01.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:01 smithi136 bash[76371]: audit 2024-09-13T00:51:01.281996+0000 mon.a (mon.0) 4379 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:51:01.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:01 smithi181 bash[67587]: audit 2024-09-13T00:51:00.466784+0000 mon.a (mon.0) 4375 : audit [INF] from='client.? 172.21.15.136:0/205774720' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.136:0/2231517774"}]: dispatch 2024-09-13T00:51:01.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:01 smithi181 bash[67587]: audit 2024-09-13T00:51:00.751286+0000 mon.a (mon.0) 4376 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:51:01.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:01 smithi181 bash[67587]: audit 2024-09-13T00:51:01.271281+0000 mon.a (mon.0) 4377 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:51:01.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:01 smithi181 bash[67587]: audit 2024-09-13T00:51:01.272471+0000 mon.a (mon.0) 4378 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:51:01.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:01 smithi181 bash[67587]: audit 2024-09-13T00:51:01.281996+0000 mon.a (mon.0) 4379 : audit [INF] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' 2024-09-13T00:51:01.866 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestMirroring::test_uuid PASSED [ 77%] 2024-09-13T00:51:02.117 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestMirroring::test_site_name PASSED [ 78%] 2024-09-13T00:51:02.432 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestMirroring::test_mirror_peer_bootstrap PASSED [ 78%] 2024-09-13T00:51:02.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:51:02 smithi181 bash[95505]: ts=2024-09-13T00:51:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:51:02.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:02 smithi181 bash[67587]: cluster 2024-09-13T00:51:00.716024+0000 mgr.y (mgr.24661) 2216 : cluster [DBG] pgmap v4358: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 310 KiB/s rd, 16 KiB/s wr, 417 op/s 2024-09-13T00:51:02.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:02 smithi181 bash[67587]: audit 2024-09-13T00:51:01.439736+0000 mon.a (mon.0) 4380 : audit [INF] from='client.? 172.21.15.136:0/205774720' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.136:0/2231517774"}]': finished 2024-09-13T00:51:02.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:02 smithi181 bash[67587]: cluster 2024-09-13T00:51:01.439899+0000 mon.a (mon.0) 4381 : cluster [DBG] osdmap e2582: 8 total, 8 up, 8 in 2024-09-13T00:51:02.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:02 smithi181 bash[67587]: audit 2024-09-13T00:51:01.935676+0000 mon.a (mon.0) 4382 : audit [INF] from='client.? ' entity='client.admin' 2024-09-13T00:51:02.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:02 smithi181 bash[67587]: audit 2024-09-13T00:51:01.936932+0000 mon.c (mon.2) 149 : audit [DBG] from='client.? 172.21.15.136:0/1130477352' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-09-13T00:51:02.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:02 smithi181 bash[67587]: audit 2024-09-13T00:51:01.937764+0000 mon.c (mon.2) 150 : audit [INF] from='client.? 172.21.15.136:0/1130477352' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]: dispatch 2024-09-13T00:51:02.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:02 smithi181 bash[67587]: audit 2024-09-13T00:51:01.938334+0000 mon.a (mon.0) 4383 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]: dispatch 2024-09-13T00:51:02.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:02 smithi181 bash[67587]: audit 2024-09-13T00:51:01.944315+0000 mon.a (mon.0) 4384 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]': finished 2024-09-13T00:51:02.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:02 smithi181 bash[67587]: audit 2024-09-13T00:51:01.945786+0000 mon.c (mon.2) 151 : audit [DBG] from='client.? 172.21.15.136:0/1130477352' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-09-13T00:51:02.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:02 smithi181 bash[67587]: audit 2024-09-13T00:51:02.218135+0000 mon.c (mon.2) 152 : audit [DBG] from='client.? 172.21.15.136:0/1130477352' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer_client_id"}]: dispatch 2024-09-13T00:51:02.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:02 smithi181 bash[67587]: audit 2024-09-13T00:51:02.219045+0000 mon.c (mon.2) 153 : audit [INF] from='client.? 172.21.15.136:0/1130477352' entity='client.admin' cmd=[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]: dispatch 2024-09-13T00:51:02.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:02 smithi181 bash[67587]: audit 2024-09-13T00:51:02.219760+0000 mon.a (mon.0) 4385 : audit [INF] from='client.? ' entity='client.admin' cmd=[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]: dispatch 2024-09-13T00:51:02.600 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:02 smithi181 bash[67587]: audit 2024-09-13T00:51:02.221345+0000 mon.c (mon.2) 154 : audit [DBG] from='client.? 172.21.15.136:0/1130477352' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-09-13T00:51:02.723 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestMirroring::test_mirror_peer PASSED [ 79%] 2024-09-13T00:51:02.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:02 smithi136 bash[76371]: cluster 2024-09-13T00:51:00.716024+0000 mgr.y (mgr.24661) 2216 : cluster [DBG] pgmap v4358: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 310 KiB/s rd, 16 KiB/s wr, 417 op/s 2024-09-13T00:51:02.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:02 smithi136 bash[76371]: audit 2024-09-13T00:51:01.439736+0000 mon.a (mon.0) 4380 : audit [INF] from='client.? 172.21.15.136:0/205774720' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.136:0/2231517774"}]': finished 2024-09-13T00:51:02.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:02 smithi136 bash[76371]: cluster 2024-09-13T00:51:01.439899+0000 mon.a (mon.0) 4381 : cluster [DBG] osdmap e2582: 8 total, 8 up, 8 in 2024-09-13T00:51:02.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:02 smithi136 bash[76371]: audit 2024-09-13T00:51:01.935676+0000 mon.a (mon.0) 4382 : audit [INF] from='client.? ' entity='client.admin' 2024-09-13T00:51:02.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:02 smithi136 bash[76371]: audit 2024-09-13T00:51:01.936932+0000 mon.c (mon.2) 149 : audit [DBG] from='client.? 172.21.15.136:0/1130477352' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-09-13T00:51:02.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:02 smithi136 bash[76371]: audit 2024-09-13T00:51:01.937764+0000 mon.c (mon.2) 150 : audit [INF] from='client.? 172.21.15.136:0/1130477352' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]: dispatch 2024-09-13T00:51:02.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:02 smithi136 bash[76371]: audit 2024-09-13T00:51:01.938334+0000 mon.a (mon.0) 4383 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]: dispatch 2024-09-13T00:51:02.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:02 smithi136 bash[76371]: audit 2024-09-13T00:51:01.944315+0000 mon.a (mon.0) 4384 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]': finished 2024-09-13T00:51:02.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:02 smithi136 bash[76371]: audit 2024-09-13T00:51:01.945786+0000 mon.c (mon.2) 151 : audit [DBG] from='client.? 172.21.15.136:0/1130477352' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-09-13T00:51:02.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:02 smithi136 bash[76371]: audit 2024-09-13T00:51:02.218135+0000 mon.c (mon.2) 152 : audit [DBG] from='client.? 172.21.15.136:0/1130477352' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer_client_id"}]: dispatch 2024-09-13T00:51:02.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:02 smithi136 bash[76371]: audit 2024-09-13T00:51:02.219045+0000 mon.c (mon.2) 153 : audit [INF] from='client.? 172.21.15.136:0/1130477352' entity='client.admin' cmd=[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]: dispatch 2024-09-13T00:51:02.762 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:02 smithi136 bash[76371]: audit 2024-09-13T00:51:02.219760+0000 mon.a (mon.0) 4385 : audit [INF] from='client.? ' entity='client.admin' cmd=[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]: dispatch 2024-09-13T00:51:02.762 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:02 smithi136 bash[76371]: audit 2024-09-13T00:51:02.221345+0000 mon.c (mon.2) 154 : audit [DBG] from='client.? 172.21.15.136:0/1130477352' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-09-13T00:51:02.762 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:02 smithi136 bash[75164]: cluster 2024-09-13T00:51:00.716024+0000 mgr.y (mgr.24661) 2216 : cluster [DBG] pgmap v4358: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 310 KiB/s rd, 16 KiB/s wr, 417 op/s 2024-09-13T00:51:02.762 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:02 smithi136 bash[75164]: audit 2024-09-13T00:51:01.439736+0000 mon.a (mon.0) 4380 : audit [INF] from='client.? 172.21.15.136:0/205774720' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.136:0/2231517774"}]': finished 2024-09-13T00:51:02.762 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:02 smithi136 bash[75164]: cluster 2024-09-13T00:51:01.439899+0000 mon.a (mon.0) 4381 : cluster [DBG] osdmap e2582: 8 total, 8 up, 8 in 2024-09-13T00:51:02.762 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:02 smithi136 bash[75164]: audit 2024-09-13T00:51:01.935676+0000 mon.a (mon.0) 4382 : audit [INF] from='client.? ' entity='client.admin' 2024-09-13T00:51:02.762 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:02 smithi136 bash[75164]: audit 2024-09-13T00:51:01.936932+0000 mon.c (mon.2) 149 : audit [DBG] from='client.? 172.21.15.136:0/1130477352' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-09-13T00:51:02.762 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:02 smithi136 bash[75164]: audit 2024-09-13T00:51:01.937764+0000 mon.c (mon.2) 150 : audit [INF] from='client.? 172.21.15.136:0/1130477352' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]: dispatch 2024-09-13T00:51:02.762 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:02 smithi136 bash[75164]: audit 2024-09-13T00:51:01.938334+0000 mon.a (mon.0) 4383 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]: dispatch 2024-09-13T00:51:02.763 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:02 smithi136 bash[75164]: audit 2024-09-13T00:51:01.944315+0000 mon.a (mon.0) 4384 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]': finished 2024-09-13T00:51:02.763 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:02 smithi136 bash[75164]: audit 2024-09-13T00:51:01.945786+0000 mon.c (mon.2) 151 : audit [DBG] from='client.? 172.21.15.136:0/1130477352' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-09-13T00:51:02.763 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:02 smithi136 bash[75164]: audit 2024-09-13T00:51:02.218135+0000 mon.c (mon.2) 152 : audit [DBG] from='client.? 172.21.15.136:0/1130477352' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer_client_id"}]: dispatch 2024-09-13T00:51:02.763 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:02 smithi136 bash[75164]: audit 2024-09-13T00:51:02.219045+0000 mon.c (mon.2) 153 : audit [INF] from='client.? 172.21.15.136:0/1130477352' entity='client.admin' cmd=[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]: dispatch 2024-09-13T00:51:02.763 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:02 smithi136 bash[75164]: audit 2024-09-13T00:51:02.219760+0000 mon.a (mon.0) 4385 : audit [INF] from='client.? ' entity='client.admin' cmd=[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]: dispatch 2024-09-13T00:51:02.763 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:02 smithi136 bash[75164]: audit 2024-09-13T00:51:02.221345+0000 mon.c (mon.2) 154 : audit [DBG] from='client.? 172.21.15.136:0/1130477352' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-09-13T00:51:02.966 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestMirroring::test_mirror_image SKIPPED [ 79%] 2024-09-13T00:51:03.211 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestMirroring::test_mirror_image_status SKIPPED [ 80%] 2024-09-13T00:51:03.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:03 smithi181 bash[67587]: audit 2024-09-13T00:51:02.528572+0000 mon.a (mon.0) 4386 : audit [INF] from='client.? ' entity='client.admin' 2024-09-13T00:51:03.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:03 smithi181 bash[67587]: audit 2024-09-13T00:51:02.529658+0000 mon.c (mon.2) 155 : audit [DBG] from='client.? 172.21.15.136:0/1130477352' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer/113/d7a1a3d1-c873-4d8a-a781-5a49a294a994"}]: dispatch 2024-09-13T00:51:03.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:03 smithi181 bash[67587]: audit 2024-09-13T00:51:02.530498+0000 mon.c (mon.2) 156 : audit [INF] from='client.? 172.21.15.136:0/1130477352' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/113/d7a1a3d1-c873-4d8a-a781-5a49a294a994"}]: dispatch 2024-09-13T00:51:03.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:03 smithi181 bash[67587]: audit 2024-09-13T00:51:02.531075+0000 mon.a (mon.0) 4387 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/113/d7a1a3d1-c873-4d8a-a781-5a49a294a994"}]: dispatch 2024-09-13T00:51:03.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:03 smithi181 bash[67587]: audit 2024-09-13T00:51:02.538782+0000 mon.a (mon.0) 4388 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/113/d7a1a3d1-c873-4d8a-a781-5a49a294a994"}]': finished 2024-09-13T00:51:04.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:03 smithi136 bash[75164]: audit 2024-09-13T00:51:02.528572+0000 mon.a (mon.0) 4386 : audit [INF] from='client.? ' entity='client.admin' 2024-09-13T00:51:04.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:03 smithi136 bash[75164]: audit 2024-09-13T00:51:02.529658+0000 mon.c (mon.2) 155 : audit [DBG] from='client.? 172.21.15.136:0/1130477352' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer/113/d7a1a3d1-c873-4d8a-a781-5a49a294a994"}]: dispatch 2024-09-13T00:51:04.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:03 smithi136 bash[75164]: audit 2024-09-13T00:51:02.530498+0000 mon.c (mon.2) 156 : audit [INF] from='client.? 172.21.15.136:0/1130477352' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/113/d7a1a3d1-c873-4d8a-a781-5a49a294a994"}]: dispatch 2024-09-13T00:51:04.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:03 smithi136 bash[75164]: audit 2024-09-13T00:51:02.531075+0000 mon.a (mon.0) 4387 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/113/d7a1a3d1-c873-4d8a-a781-5a49a294a994"}]: dispatch 2024-09-13T00:51:04.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:03 smithi136 bash[75164]: audit 2024-09-13T00:51:02.538782+0000 mon.a (mon.0) 4388 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/113/d7a1a3d1-c873-4d8a-a781-5a49a294a994"}]': finished 2024-09-13T00:51:04.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:03 smithi136 bash[76371]: audit 2024-09-13T00:51:02.528572+0000 mon.a (mon.0) 4386 : audit [INF] from='client.? ' entity='client.admin' 2024-09-13T00:51:04.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:03 smithi136 bash[76371]: audit 2024-09-13T00:51:02.529658+0000 mon.c (mon.2) 155 : audit [DBG] from='client.? 172.21.15.136:0/1130477352' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer/113/d7a1a3d1-c873-4d8a-a781-5a49a294a994"}]: dispatch 2024-09-13T00:51:04.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:03 smithi136 bash[76371]: audit 2024-09-13T00:51:02.530498+0000 mon.c (mon.2) 156 : audit [INF] from='client.? 172.21.15.136:0/1130477352' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/113/d7a1a3d1-c873-4d8a-a781-5a49a294a994"}]: dispatch 2024-09-13T00:51:04.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:03 smithi136 bash[76371]: audit 2024-09-13T00:51:02.531075+0000 mon.a (mon.0) 4387 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/113/d7a1a3d1-c873-4d8a-a781-5a49a294a994"}]: dispatch 2024-09-13T00:51:04.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:03 smithi136 bash[76371]: audit 2024-09-13T00:51:02.538782+0000 mon.a (mon.0) 4388 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/113/d7a1a3d1-c873-4d8a-a781-5a49a294a994"}]': finished 2024-09-13T00:51:04.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:04 smithi181 bash[67587]: cluster 2024-09-13T00:51:02.716948+0000 mgr.y (mgr.24661) 2217 : cluster [DBG] pgmap v4360: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 304 KiB/s rd, 17 KiB/s wr, 408 op/s 2024-09-13T00:51:04.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:04 smithi181 bash[67587]: cluster 2024-09-13T00:51:03.552607+0000 mon.a (mon.0) 4389 : cluster [DBG] osdmap e2583: 8 total, 8 up, 8 in 2024-09-13T00:51:05.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:04 smithi136 bash[76371]: cluster 2024-09-13T00:51:02.716948+0000 mgr.y (mgr.24661) 2217 : cluster [DBG] pgmap v4360: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 304 KiB/s rd, 17 KiB/s wr, 408 op/s 2024-09-13T00:51:05.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:04 smithi136 bash[76371]: cluster 2024-09-13T00:51:03.552607+0000 mon.a (mon.0) 4389 : cluster [DBG] osdmap e2583: 8 total, 8 up, 8 in 2024-09-13T00:51:05.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:04 smithi136 bash[75164]: cluster 2024-09-13T00:51:02.716948+0000 mgr.y (mgr.24661) 2217 : cluster [DBG] pgmap v4360: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 304 KiB/s rd, 17 KiB/s wr, 408 op/s 2024-09-13T00:51:05.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:04 smithi136 bash[75164]: cluster 2024-09-13T00:51:03.552607+0000 mon.a (mon.0) 4389 : cluster [DBG] osdmap e2583: 8 total, 8 up, 8 in 2024-09-13T00:51:05.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:05 smithi181 bash[67587]: cluster 2024-09-13T00:51:04.556051+0000 mon.a (mon.0) 4390 : cluster [DBG] osdmap e2584: 8 total, 8 up, 8 in 2024-09-13T00:51:05.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:05 smithi181 bash[67587]: cluster 2024-09-13T00:51:05.058363+0000 mon.a (mon.0) 4391 : cluster [DBG] osdmap e2585: 8 total, 8 up, 8 in 2024-09-13T00:51:05.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:05 smithi181 bash[67587]: audit 2024-09-13T00:51:05.128286+0000 mon.c (mon.2) 157 : audit [INF] from='client.? 172.21.15.136:0/1130477352' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/113/e95154ca-9f05-455f-b788-90c4bd6fc574"}]: dispatch 2024-09-13T00:51:05.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:05 smithi181 bash[67587]: audit 2024-09-13T00:51:05.129120+0000 mon.a (mon.0) 4392 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/113/e95154ca-9f05-455f-b788-90c4bd6fc574"}]: dispatch 2024-09-13T00:51:05.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:05 smithi181 bash[67587]: audit 2024-09-13T00:51:05.135731+0000 mon.a (mon.0) 4393 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/113/e95154ca-9f05-455f-b788-90c4bd6fc574"}]': finished 2024-09-13T00:51:05.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:05 smithi181 bash[67587]: audit 2024-09-13T00:51:05.215079+0000 mon.c (mon.2) 158 : audit [INF] from='client.? 172.21.15.136:0/1130477352' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/113/ebdbdd77-3d12-4bcf-871d-241ac808709e"}]: dispatch 2024-09-13T00:51:05.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:05 smithi181 bash[67587]: audit 2024-09-13T00:51:05.215787+0000 mon.a (mon.0) 4394 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/113/ebdbdd77-3d12-4bcf-871d-241ac808709e"}]: dispatch 2024-09-13T00:51:05.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:05 smithi181 bash[67587]: audit 2024-09-13T00:51:05.222223+0000 mon.a (mon.0) 4395 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/113/ebdbdd77-3d12-4bcf-871d-241ac808709e"}]': finished 2024-09-13T00:51:06.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:05 smithi136 bash[76371]: cluster 2024-09-13T00:51:04.556051+0000 mon.a (mon.0) 4390 : cluster [DBG] osdmap e2584: 8 total, 8 up, 8 in 2024-09-13T00:51:06.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:05 smithi136 bash[76371]: cluster 2024-09-13T00:51:05.058363+0000 mon.a (mon.0) 4391 : cluster [DBG] osdmap e2585: 8 total, 8 up, 8 in 2024-09-13T00:51:06.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:05 smithi136 bash[76371]: audit 2024-09-13T00:51:05.128286+0000 mon.c (mon.2) 157 : audit [INF] from='client.? 172.21.15.136:0/1130477352' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/113/e95154ca-9f05-455f-b788-90c4bd6fc574"}]: dispatch 2024-09-13T00:51:06.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:05 smithi136 bash[76371]: audit 2024-09-13T00:51:05.129120+0000 mon.a (mon.0) 4392 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/113/e95154ca-9f05-455f-b788-90c4bd6fc574"}]: dispatch 2024-09-13T00:51:06.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:05 smithi136 bash[76371]: audit 2024-09-13T00:51:05.135731+0000 mon.a (mon.0) 4393 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/113/e95154ca-9f05-455f-b788-90c4bd6fc574"}]': finished 2024-09-13T00:51:06.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:05 smithi136 bash[76371]: audit 2024-09-13T00:51:05.215079+0000 mon.c (mon.2) 158 : audit [INF] from='client.? 172.21.15.136:0/1130477352' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/113/ebdbdd77-3d12-4bcf-871d-241ac808709e"}]: dispatch 2024-09-13T00:51:06.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:05 smithi136 bash[76371]: audit 2024-09-13T00:51:05.215787+0000 mon.a (mon.0) 4394 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/113/ebdbdd77-3d12-4bcf-871d-241ac808709e"}]: dispatch 2024-09-13T00:51:06.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:05 smithi136 bash[76371]: audit 2024-09-13T00:51:05.222223+0000 mon.a (mon.0) 4395 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/113/ebdbdd77-3d12-4bcf-871d-241ac808709e"}]': finished 2024-09-13T00:51:06.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:05 smithi136 bash[75164]: cluster 2024-09-13T00:51:04.556051+0000 mon.a (mon.0) 4390 : cluster [DBG] osdmap e2584: 8 total, 8 up, 8 in 2024-09-13T00:51:06.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:05 smithi136 bash[75164]: cluster 2024-09-13T00:51:05.058363+0000 mon.a (mon.0) 4391 : cluster [DBG] osdmap e2585: 8 total, 8 up, 8 in 2024-09-13T00:51:06.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:05 smithi136 bash[75164]: audit 2024-09-13T00:51:05.128286+0000 mon.c (mon.2) 157 : audit [INF] from='client.? 172.21.15.136:0/1130477352' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/113/e95154ca-9f05-455f-b788-90c4bd6fc574"}]: dispatch 2024-09-13T00:51:06.011 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:05 smithi136 bash[75164]: audit 2024-09-13T00:51:05.129120+0000 mon.a (mon.0) 4392 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/113/e95154ca-9f05-455f-b788-90c4bd6fc574"}]: dispatch 2024-09-13T00:51:06.012 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:05 smithi136 bash[75164]: audit 2024-09-13T00:51:05.135731+0000 mon.a (mon.0) 4393 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/113/e95154ca-9f05-455f-b788-90c4bd6fc574"}]': finished 2024-09-13T00:51:06.012 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:05 smithi136 bash[75164]: audit 2024-09-13T00:51:05.215079+0000 mon.c (mon.2) 158 : audit [INF] from='client.? 172.21.15.136:0/1130477352' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/113/ebdbdd77-3d12-4bcf-871d-241ac808709e"}]: dispatch 2024-09-13T00:51:06.012 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:05 smithi136 bash[75164]: audit 2024-09-13T00:51:05.215787+0000 mon.a (mon.0) 4394 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/113/ebdbdd77-3d12-4bcf-871d-241ac808709e"}]: dispatch 2024-09-13T00:51:06.012 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:05 smithi136 bash[75164]: audit 2024-09-13T00:51:05.222223+0000 mon.a (mon.0) 4395 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/113/ebdbdd77-3d12-4bcf-871d-241ac808709e"}]': finished 2024-09-13T00:51:06.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:06 smithi181 bash[67587]: cluster 2024-09-13T00:51:04.717749+0000 mgr.y (mgr.24661) 2218 : cluster [DBG] pgmap v4363: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 205 KiB/s rd, 14 KiB/s wr, 285 op/s 2024-09-13T00:51:06.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:06 smithi181 bash[67587]: cluster 2024-09-13T00:51:06.227841+0000 mon.a (mon.0) 4396 : cluster [DBG] osdmap e2586: 8 total, 8 up, 8 in 2024-09-13T00:51:07.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:06 smithi136 bash[75164]: cluster 2024-09-13T00:51:04.717749+0000 mgr.y (mgr.24661) 2218 : cluster [DBG] pgmap v4363: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 205 KiB/s rd, 14 KiB/s wr, 285 op/s 2024-09-13T00:51:07.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:06 smithi136 bash[75164]: cluster 2024-09-13T00:51:06.227841+0000 mon.a (mon.0) 4396 : cluster [DBG] osdmap e2586: 8 total, 8 up, 8 in 2024-09-13T00:51:07.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:06 smithi136 bash[76371]: cluster 2024-09-13T00:51:04.717749+0000 mgr.y (mgr.24661) 2218 : cluster [DBG] pgmap v4363: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 205 KiB/s rd, 14 KiB/s wr, 285 op/s 2024-09-13T00:51:07.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:06 smithi136 bash[76371]: cluster 2024-09-13T00:51:06.227841+0000 mon.a (mon.0) 4396 : cluster [DBG] osdmap e2586: 8 total, 8 up, 8 in 2024-09-13T00:51:08.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:08 smithi136 bash[75164]: cluster 2024-09-13T00:51:06.718548+0000 mgr.y (mgr.24661) 2219 : cluster [DBG] pgmap v4366: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 356 KiB/s rd, 23 KiB/s wr, 492 op/s 2024-09-13T00:51:08.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:08 smithi136 bash[75164]: cluster 2024-09-13T00:51:07.230323+0000 mon.a (mon.0) 4397 : cluster [DBG] osdmap e2587: 8 total, 8 up, 8 in 2024-09-13T00:51:08.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:08 smithi136 bash[76371]: cluster 2024-09-13T00:51:06.718548+0000 mgr.y (mgr.24661) 2219 : cluster [DBG] pgmap v4366: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 356 KiB/s rd, 23 KiB/s wr, 492 op/s 2024-09-13T00:51:08.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:08 smithi136 bash[76371]: cluster 2024-09-13T00:51:07.230323+0000 mon.a (mon.0) 4397 : cluster [DBG] osdmap e2587: 8 total, 8 up, 8 in 2024-09-13T00:51:08.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:08 smithi181 bash[67587]: cluster 2024-09-13T00:51:06.718548+0000 mgr.y (mgr.24661) 2219 : cluster [DBG] pgmap v4366: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 356 KiB/s rd, 23 KiB/s wr, 492 op/s 2024-09-13T00:51:08.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:08 smithi181 bash[67587]: cluster 2024-09-13T00:51:07.230323+0000 mon.a (mon.0) 4397 : cluster [DBG] osdmap e2587: 8 total, 8 up, 8 in 2024-09-13T00:51:09.510 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:51:09 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:51:09] "GET /metrics HTTP/1.1" 200 36746 "" "Prometheus/2.43.0" 2024-09-13T00:51:09.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:09 smithi136 bash[75164]: cluster 2024-09-13T00:51:08.235816+0000 mon.a (mon.0) 4398 : cluster [DBG] osdmap e2588: 8 total, 8 up, 8 in 2024-09-13T00:51:09.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:09 smithi136 bash[75164]: audit 2024-09-13T00:51:08.856378+0000 mon.a (mon.0) 4399 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:51:09.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:09 smithi136 bash[76371]: cluster 2024-09-13T00:51:08.235816+0000 mon.a (mon.0) 4398 : cluster [DBG] osdmap e2588: 8 total, 8 up, 8 in 2024-09-13T00:51:09.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:09 smithi136 bash[76371]: audit 2024-09-13T00:51:08.856378+0000 mon.a (mon.0) 4399 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:51:09.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:09 smithi181 bash[67587]: cluster 2024-09-13T00:51:08.235816+0000 mon.a (mon.0) 4398 : cluster [DBG] osdmap e2588: 8 total, 8 up, 8 in 2024-09-13T00:51:09.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:09 smithi181 bash[67587]: audit 2024-09-13T00:51:08.856378+0000 mon.a (mon.0) 4399 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:51:10.214 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestMirroring::test_mirror_image_create_snapshot PASSED [ 80%] 2024-09-13T00:51:10.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:10 smithi136 bash[75164]: cluster 2024-09-13T00:51:08.719426+0000 mgr.y (mgr.24661) 2220 : cluster [DBG] pgmap v4369: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 312 KiB/s rd, 26 KiB/s wr, 426 op/s 2024-09-13T00:51:10.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:10 smithi136 bash[75164]: cluster 2024-09-13T00:51:09.258970+0000 mon.a (mon.0) 4400 : cluster [DBG] osdmap e2589: 8 total, 8 up, 8 in 2024-09-13T00:51:10.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:10 smithi136 bash[75164]: cluster 2024-09-13T00:51:10.061714+0000 mon.a (mon.0) 4401 : cluster [DBG] osdmap e2590: 8 total, 8 up, 8 in 2024-09-13T00:51:10.510 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:10 smithi136 bash[76371]: cluster 2024-09-13T00:51:08.719426+0000 mgr.y (mgr.24661) 2220 : cluster [DBG] pgmap v4369: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 312 KiB/s rd, 26 KiB/s wr, 426 op/s 2024-09-13T00:51:10.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:10 smithi136 bash[76371]: cluster 2024-09-13T00:51:09.258970+0000 mon.a (mon.0) 4400 : cluster [DBG] osdmap e2589: 8 total, 8 up, 8 in 2024-09-13T00:51:10.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:10 smithi136 bash[76371]: cluster 2024-09-13T00:51:10.061714+0000 mon.a (mon.0) 4401 : cluster [DBG] osdmap e2590: 8 total, 8 up, 8 in 2024-09-13T00:51:10.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:10 smithi181 bash[67587]: cluster 2024-09-13T00:51:08.719426+0000 mgr.y (mgr.24661) 2220 : cluster [DBG] pgmap v4369: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 312 KiB/s rd, 26 KiB/s wr, 426 op/s 2024-09-13T00:51:10.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:10 smithi181 bash[67587]: cluster 2024-09-13T00:51:09.258970+0000 mon.a (mon.0) 4400 : cluster [DBG] osdmap e2589: 8 total, 8 up, 8 in 2024-09-13T00:51:10.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:10 smithi181 bash[67587]: cluster 2024-09-13T00:51:10.061714+0000 mon.a (mon.0) 4401 : cluster [DBG] osdmap e2590: 8 total, 8 up, 8 in 2024-09-13T00:51:12.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:12 smithi181 bash[67587]: cluster 2024-09-13T00:51:10.720406+0000 mgr.y (mgr.24661) 2221 : cluster [DBG] pgmap v4372: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 69 KiB/s rd, 10 KiB/s wr, 101 op/s 2024-09-13T00:51:12.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:12 smithi181 bash[67587]: cluster 2024-09-13T00:51:11.265869+0000 mon.a (mon.0) 4402 : cluster [DBG] osdmap e2591: 8 total, 8 up, 8 in 2024-09-13T00:51:12.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:51:12 smithi181 bash[95505]: ts=2024-09-13T00:51:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:51:12.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:12 smithi136 bash[75164]: cluster 2024-09-13T00:51:10.720406+0000 mgr.y (mgr.24661) 2221 : cluster [DBG] pgmap v4372: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 69 KiB/s rd, 10 KiB/s wr, 101 op/s 2024-09-13T00:51:12.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:12 smithi136 bash[75164]: cluster 2024-09-13T00:51:11.265869+0000 mon.a (mon.0) 4402 : cluster [DBG] osdmap e2591: 8 total, 8 up, 8 in 2024-09-13T00:51:12.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:12 smithi136 bash[76371]: cluster 2024-09-13T00:51:10.720406+0000 mgr.y (mgr.24661) 2221 : cluster [DBG] pgmap v4372: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 69 KiB/s rd, 10 KiB/s wr, 101 op/s 2024-09-13T00:51:12.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:12 smithi136 bash[76371]: cluster 2024-09-13T00:51:11.265869+0000 mon.a (mon.0) 4402 : cluster [DBG] osdmap e2591: 8 total, 8 up, 8 in 2024-09-13T00:51:13.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:13 smithi181 bash[67587]: cluster 2024-09-13T00:51:12.275418+0000 mon.a (mon.0) 4403 : cluster [DBG] osdmap e2592: 8 total, 8 up, 8 in 2024-09-13T00:51:13.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:13 smithi181 bash[67587]: audit 2024-09-13T00:51:12.321858+0000 mon.c (mon.2) 159 : audit [INF] from='client.? 172.21.15.136:0/1130477352' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/113/b1ac07bd-32dc-446f-9c3a-b8f2888799b4"}]: dispatch 2024-09-13T00:51:13.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:13 smithi181 bash[67587]: audit 2024-09-13T00:51:12.322476+0000 mon.a (mon.0) 4404 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/113/b1ac07bd-32dc-446f-9c3a-b8f2888799b4"}]: dispatch 2024-09-13T00:51:13.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:13 smithi181 bash[67587]: audit 2024-09-13T00:51:12.328988+0000 mon.a (mon.0) 4405 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/113/b1ac07bd-32dc-446f-9c3a-b8f2888799b4"}]': finished 2024-09-13T00:51:13.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:13 smithi136 bash[75164]: cluster 2024-09-13T00:51:12.275418+0000 mon.a (mon.0) 4403 : cluster [DBG] osdmap e2592: 8 total, 8 up, 8 in 2024-09-13T00:51:13.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:13 smithi136 bash[75164]: audit 2024-09-13T00:51:12.321858+0000 mon.c (mon.2) 159 : audit [INF] from='client.? 172.21.15.136:0/1130477352' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/113/b1ac07bd-32dc-446f-9c3a-b8f2888799b4"}]: dispatch 2024-09-13T00:51:13.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:13 smithi136 bash[75164]: audit 2024-09-13T00:51:12.322476+0000 mon.a (mon.0) 4404 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/113/b1ac07bd-32dc-446f-9c3a-b8f2888799b4"}]: dispatch 2024-09-13T00:51:13.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:13 smithi136 bash[75164]: audit 2024-09-13T00:51:12.328988+0000 mon.a (mon.0) 4405 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/113/b1ac07bd-32dc-446f-9c3a-b8f2888799b4"}]': finished 2024-09-13T00:51:13.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:13 smithi136 bash[76371]: cluster 2024-09-13T00:51:12.275418+0000 mon.a (mon.0) 4403 : cluster [DBG] osdmap e2592: 8 total, 8 up, 8 in 2024-09-13T00:51:13.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:13 smithi136 bash[76371]: audit 2024-09-13T00:51:12.321858+0000 mon.c (mon.2) 159 : audit [INF] from='client.? 172.21.15.136:0/1130477352' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/113/b1ac07bd-32dc-446f-9c3a-b8f2888799b4"}]: dispatch 2024-09-13T00:51:13.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:13 smithi136 bash[76371]: audit 2024-09-13T00:51:12.322476+0000 mon.a (mon.0) 4404 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/113/b1ac07bd-32dc-446f-9c3a-b8f2888799b4"}]: dispatch 2024-09-13T00:51:13.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:13 smithi136 bash[76371]: audit 2024-09-13T00:51:12.328988+0000 mon.a (mon.0) 4405 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/113/b1ac07bd-32dc-446f-9c3a-b8f2888799b4"}]': finished 2024-09-13T00:51:14.496 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestMirroring::test_aio_mirror_image_create_snapshot PASSED [ 81%] 2024-09-13T00:51:14.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:14 smithi181 bash[67587]: cluster 2024-09-13T00:51:12.721078+0000 mgr.y (mgr.24661) 2222 : cluster [DBG] pgmap v4375: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 86 KiB/s rd, 8.7 KiB/s wr, 122 op/s 2024-09-13T00:51:14.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:14 smithi181 bash[67587]: cluster 2024-09-13T00:51:13.339099+0000 mon.a (mon.0) 4406 : cluster [DBG] osdmap e2593: 8 total, 8 up, 8 in 2024-09-13T00:51:14.676 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestTrash::test_move PASSED [ 81%] 2024-09-13T00:51:14.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:14 smithi136 bash[75164]: cluster 2024-09-13T00:51:12.721078+0000 mgr.y (mgr.24661) 2222 : cluster [DBG] pgmap v4375: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 86 KiB/s rd, 8.7 KiB/s wr, 122 op/s 2024-09-13T00:51:14.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:14 smithi136 bash[75164]: cluster 2024-09-13T00:51:13.339099+0000 mon.a (mon.0) 4406 : cluster [DBG] osdmap e2593: 8 total, 8 up, 8 in 2024-09-13T00:51:14.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:14 smithi136 bash[76371]: cluster 2024-09-13T00:51:12.721078+0000 mgr.y (mgr.24661) 2222 : cluster [DBG] pgmap v4375: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 86 KiB/s rd, 8.7 KiB/s wr, 122 op/s 2024-09-13T00:51:14.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:14 smithi136 bash[76371]: cluster 2024-09-13T00:51:13.339099+0000 mon.a (mon.0) 4406 : cluster [DBG] osdmap e2593: 8 total, 8 up, 8 in 2024-09-13T00:51:15.014 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestTrash::test_purge PASSED [ 82%] 2024-09-13T00:51:15.195 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestTrash::test_remove_denied PASSED [ 82%] 2024-09-13T00:51:15.323 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestTrash::test_remove PASSED [ 83%] 2024-09-13T00:51:15.438 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestTrash::test_remove_with_progress PASSED [ 84%] 2024-09-13T00:51:15.587 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestTrash::test_get PASSED [ 84%] 2024-09-13T00:51:15.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:15 smithi136 bash[75164]: cluster 2024-09-13T00:51:14.351791+0000 mon.a (mon.0) 4407 : cluster [DBG] osdmap e2594: 8 total, 8 up, 8 in 2024-09-13T00:51:15.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:15 smithi136 bash[75164]: cluster 2024-09-13T00:51:15.060518+0000 mon.a (mon.0) 4408 : cluster [DBG] osdmap e2595: 8 total, 8 up, 8 in 2024-09-13T00:51:15.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:15 smithi136 bash[76371]: cluster 2024-09-13T00:51:14.351791+0000 mon.a (mon.0) 4407 : cluster [DBG] osdmap e2594: 8 total, 8 up, 8 in 2024-09-13T00:51:15.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:15 smithi136 bash[76371]: cluster 2024-09-13T00:51:15.060518+0000 mon.a (mon.0) 4408 : cluster [DBG] osdmap e2595: 8 total, 8 up, 8 in 2024-09-13T00:51:15.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:15 smithi181 bash[67587]: cluster 2024-09-13T00:51:14.351791+0000 mon.a (mon.0) 4407 : cluster [DBG] osdmap e2594: 8 total, 8 up, 8 in 2024-09-13T00:51:15.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:15 smithi181 bash[67587]: cluster 2024-09-13T00:51:15.060518+0000 mon.a (mon.0) 4408 : cluster [DBG] osdmap e2595: 8 total, 8 up, 8 in 2024-09-13T00:51:15.897 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestTrash::test_list PASSED [ 85%] 2024-09-13T00:51:16.143 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestTrash::test_restore PASSED [ 85%] 2024-09-13T00:51:16.155 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::test_create_group PASSED [ 86%] 2024-09-13T00:51:16.175 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::test_rename_group PASSED [ 86%] 2024-09-13T00:51:16.177 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::test_list_groups_empty PASSED [ 87%] 2024-09-13T00:51:16.192 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::test_list_groups PASSED [ 87%] 2024-09-13T00:51:16.206 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::test_list_groups_after_removed PASSED [ 88%] 2024-09-13T00:51:16.436 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestGroups::test_group_image_add PASSED [ 89%] 2024-09-13T00:51:16.668 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestGroups::test_group_image_list_empty PASSED [ 89%] 2024-09-13T00:51:16.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:16 smithi136 bash[75164]: cluster 2024-09-13T00:51:14.721507+0000 mgr.y (mgr.24661) 2223 : cluster [DBG] pgmap v4378: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 81 KiB/s rd, 8.7 KiB/s wr, 109 op/s 2024-09-13T00:51:16.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:16 smithi136 bash[76371]: cluster 2024-09-13T00:51:14.721507+0000 mgr.y (mgr.24661) 2223 : cluster [DBG] pgmap v4378: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 81 KiB/s rd, 8.7 KiB/s wr, 109 op/s 2024-09-13T00:51:16.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:16 smithi181 bash[67587]: cluster 2024-09-13T00:51:14.721507+0000 mgr.y (mgr.24661) 2223 : cluster [DBG] pgmap v4378: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 81 KiB/s rd, 8.7 KiB/s wr, 109 op/s 2024-09-13T00:51:16.946 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestGroups::test_group_image_list PASSED [ 90%] 2024-09-13T00:51:17.290 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestGroups::test_group_image_list_move_to_trash PASSED [ 90%] 2024-09-13T00:51:18.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:18 smithi136 bash[75164]: cluster 2024-09-13T00:51:16.722855+0000 mgr.y (mgr.24661) 2224 : cluster [DBG] pgmap v4380: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 128 KiB/s rd, 15 KiB/s wr, 180 op/s 2024-09-13T00:51:18.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:18 smithi136 bash[76371]: cluster 2024-09-13T00:51:16.722855+0000 mgr.y (mgr.24661) 2224 : cluster [DBG] pgmap v4380: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 128 KiB/s rd, 15 KiB/s wr, 180 op/s 2024-09-13T00:51:18.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:18 smithi181 bash[67587]: cluster 2024-09-13T00:51:16.722855+0000 mgr.y (mgr.24661) 2224 : cluster [DBG] pgmap v4380: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 128 KiB/s rd, 15 KiB/s wr, 180 op/s 2024-09-13T00:51:19.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:51:19 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:51:19] "GET /metrics HTTP/1.1" 200 36746 "" "Prometheus/2.43.0" 2024-09-13T00:51:20.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:20 smithi136 bash[76371]: cluster 2024-09-13T00:51:18.724116+0000 mgr.y (mgr.24661) 2225 : cluster [DBG] pgmap v4381: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 217 KiB/s rd, 23 KiB/s wr, 320 op/s 2024-09-13T00:51:20.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:20 smithi136 bash[76371]: cluster 2024-09-13T00:51:20.070452+0000 mon.a (mon.0) 4409 : cluster [DBG] osdmap e2596: 8 total, 8 up, 8 in 2024-09-13T00:51:20.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:20 smithi136 bash[75164]: cluster 2024-09-13T00:51:18.724116+0000 mgr.y (mgr.24661) 2225 : cluster [DBG] pgmap v4381: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 217 KiB/s rd, 23 KiB/s wr, 320 op/s 2024-09-13T00:51:20.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:20 smithi136 bash[75164]: cluster 2024-09-13T00:51:20.070452+0000 mon.a (mon.0) 4409 : cluster [DBG] osdmap e2596: 8 total, 8 up, 8 in 2024-09-13T00:51:20.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:20 smithi181 bash[67587]: cluster 2024-09-13T00:51:18.724116+0000 mgr.y (mgr.24661) 2225 : cluster [DBG] pgmap v4381: 137 pgs: 137 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 217 KiB/s rd, 23 KiB/s wr, 320 op/s 2024-09-13T00:51:20.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:20 smithi181 bash[67587]: cluster 2024-09-13T00:51:20.070452+0000 mon.a (mon.0) 4409 : cluster [DBG] osdmap e2596: 8 total, 8 up, 8 in 2024-09-13T00:51:21.765 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestGroups::test_group_image_many_images PASSED [ 91%] 2024-09-13T00:51:22.075 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestGroups::test_group_image_remove PASSED [ 91%] 2024-09-13T00:51:22.330 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestGroups::test_group_snap PASSED [ 92%] 2024-09-13T00:51:22.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:22 smithi181 bash[67587]: cluster 2024-09-13T00:51:20.725389+0000 mgr.y (mgr.24661) 2226 : cluster [DBG] pgmap v4383: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 362 KiB/s rd, 48 KiB/s wr, 543 op/s 2024-09-13T00:51:22.598 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:51:22 smithi181 bash[95505]: ts=2024-09-13T00:51:22.317Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:51:22.612 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestGroups::test_group_snap_flags PASSED [ 92%] 2024-09-13T00:51:22.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:22 smithi136 bash[75164]: cluster 2024-09-13T00:51:20.725389+0000 mgr.y (mgr.24661) 2226 : cluster [DBG] pgmap v4383: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 362 KiB/s rd, 48 KiB/s wr, 543 op/s 2024-09-13T00:51:22.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:22 smithi136 bash[76371]: cluster 2024-09-13T00:51:20.725389+0000 mgr.y (mgr.24661) 2226 : cluster [DBG] pgmap v4383: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 362 KiB/s rd, 48 KiB/s wr, 543 op/s 2024-09-13T00:51:23.046 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestGroups::test_group_snap_list_many PASSED [ 93%] 2024-09-13T00:51:24.613 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestGroups::test_group_snap_namespace PASSED [ 93%] 2024-09-13T00:51:24.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:24 smithi136 bash[75164]: cluster 2024-09-13T00:51:22.726608+0000 mgr.y (mgr.24661) 2227 : cluster [DBG] pgmap v4384: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 433 KiB/s rd, 54 KiB/s wr, 637 op/s 2024-09-13T00:51:24.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:24 smithi136 bash[75164]: cluster 2024-09-13T00:51:23.381392+0000 mon.a (mon.0) 4410 : cluster [DBG] osdmap e2597: 8 total, 8 up, 8 in 2024-09-13T00:51:24.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:24 smithi136 bash[75164]: audit 2024-09-13T00:51:23.856829+0000 mon.a (mon.0) 4411 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:51:24.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:24 smithi136 bash[76371]: cluster 2024-09-13T00:51:22.726608+0000 mgr.y (mgr.24661) 2227 : cluster [DBG] pgmap v4384: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 433 KiB/s rd, 54 KiB/s wr, 637 op/s 2024-09-13T00:51:24.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:24 smithi136 bash[76371]: cluster 2024-09-13T00:51:23.381392+0000 mon.a (mon.0) 4410 : cluster [DBG] osdmap e2597: 8 total, 8 up, 8 in 2024-09-13T00:51:24.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:24 smithi136 bash[76371]: audit 2024-09-13T00:51:23.856829+0000 mon.a (mon.0) 4411 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:51:24.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:24 smithi181 bash[67587]: cluster 2024-09-13T00:51:22.726608+0000 mgr.y (mgr.24661) 2227 : cluster [DBG] pgmap v4384: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 433 KiB/s rd, 54 KiB/s wr, 637 op/s 2024-09-13T00:51:24.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:24 smithi181 bash[67587]: cluster 2024-09-13T00:51:23.381392+0000 mon.a (mon.0) 4410 : cluster [DBG] osdmap e2597: 8 total, 8 up, 8 in 2024-09-13T00:51:24.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:24 smithi181 bash[67587]: audit 2024-09-13T00:51:23.856829+0000 mon.a (mon.0) 4411 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:51:24.875 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestGroups::test_group_snap_rename PASSED [ 94%] 2024-09-13T00:51:25.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:25 smithi136 bash[75164]: cluster 2024-09-13T00:51:24.391638+0000 mon.a (mon.0) 4412 : cluster [DBG] osdmap e2598: 8 total, 8 up, 8 in 2024-09-13T00:51:25.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:25 smithi136 bash[76371]: cluster 2024-09-13T00:51:24.391638+0000 mon.a (mon.0) 4412 : cluster [DBG] osdmap e2598: 8 total, 8 up, 8 in 2024-09-13T00:51:25.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:25 smithi181 bash[67587]: cluster 2024-09-13T00:51:24.391638+0000 mon.a (mon.0) 4412 : cluster [DBG] osdmap e2598: 8 total, 8 up, 8 in 2024-09-13T00:51:26.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:26 smithi136 bash[75164]: cluster 2024-09-13T00:51:24.727350+0000 mgr.y (mgr.24661) 2228 : cluster [DBG] pgmap v4387: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 494 KiB/s rd, 53 KiB/s wr, 714 op/s 2024-09-13T00:51:26.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:26 smithi136 bash[75164]: cluster 2024-09-13T00:51:25.407765+0000 mon.a (mon.0) 4413 : cluster [DBG] osdmap e2599: 8 total, 8 up, 8 in 2024-09-13T00:51:26.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:26 smithi136 bash[76371]: cluster 2024-09-13T00:51:24.727350+0000 mgr.y (mgr.24661) 2228 : cluster [DBG] pgmap v4387: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 494 KiB/s rd, 53 KiB/s wr, 714 op/s 2024-09-13T00:51:26.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:26 smithi136 bash[76371]: cluster 2024-09-13T00:51:25.407765+0000 mon.a (mon.0) 4413 : cluster [DBG] osdmap e2599: 8 total, 8 up, 8 in 2024-09-13T00:51:26.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:26 smithi181 bash[67587]: cluster 2024-09-13T00:51:24.727350+0000 mgr.y (mgr.24661) 2228 : cluster [DBG] pgmap v4387: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 494 KiB/s rd, 53 KiB/s wr, 714 op/s 2024-09-13T00:51:26.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:26 smithi181 bash[67587]: cluster 2024-09-13T00:51:25.407765+0000 mon.a (mon.0) 4413 : cluster [DBG] osdmap e2599: 8 total, 8 up, 8 in 2024-09-13T00:51:27.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:27 smithi136 bash[76371]: cluster 2024-09-13T00:51:26.415707+0000 mon.a (mon.0) 4414 : cluster [DBG] osdmap e2600: 8 total, 8 up, 8 in 2024-09-13T00:51:27.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:27 smithi136 bash[75164]: cluster 2024-09-13T00:51:26.415707+0000 mon.a (mon.0) 4414 : cluster [DBG] osdmap e2600: 8 total, 8 up, 8 in 2024-09-13T00:51:27.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:27 smithi181 bash[67587]: cluster 2024-09-13T00:51:26.415707+0000 mon.a (mon.0) 4414 : cluster [DBG] osdmap e2600: 8 total, 8 up, 8 in 2024-09-13T00:51:28.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:28 smithi136 bash[75164]: cluster 2024-09-13T00:51:26.728299+0000 mgr.y (mgr.24661) 2229 : cluster [DBG] pgmap v4390: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 520 KiB/s rd, 41 KiB/s wr, 729 op/s 2024-09-13T00:51:28.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:28 smithi136 bash[75164]: cluster 2024-09-13T00:51:27.427543+0000 mon.a (mon.0) 4415 : cluster [DBG] osdmap e2601: 8 total, 8 up, 8 in 2024-09-13T00:51:28.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:28 smithi136 bash[76371]: cluster 2024-09-13T00:51:26.728299+0000 mgr.y (mgr.24661) 2229 : cluster [DBG] pgmap v4390: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 520 KiB/s rd, 41 KiB/s wr, 729 op/s 2024-09-13T00:51:28.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:28 smithi136 bash[76371]: cluster 2024-09-13T00:51:27.427543+0000 mon.a (mon.0) 4415 : cluster [DBG] osdmap e2601: 8 total, 8 up, 8 in 2024-09-13T00:51:28.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:28 smithi181 bash[67587]: cluster 2024-09-13T00:51:26.728299+0000 mgr.y (mgr.24661) 2229 : cluster [DBG] pgmap v4390: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 520 KiB/s rd, 41 KiB/s wr, 729 op/s 2024-09-13T00:51:28.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:28 smithi181 bash[67587]: cluster 2024-09-13T00:51:27.427543+0000 mon.a (mon.0) 4415 : cluster [DBG] osdmap e2601: 8 total, 8 up, 8 in 2024-09-13T00:51:29.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:29 smithi136 bash[75164]: cluster 2024-09-13T00:51:28.441374+0000 mon.a (mon.0) 4416 : cluster [DBG] osdmap e2602: 8 total, 8 up, 8 in 2024-09-13T00:51:29.761 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:51:29 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:51:29] "GET /metrics HTTP/1.1" 200 36744 "" "Prometheus/2.43.0" 2024-09-13T00:51:29.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:29 smithi136 bash[76371]: cluster 2024-09-13T00:51:28.441374+0000 mon.a (mon.0) 4416 : cluster [DBG] osdmap e2602: 8 total, 8 up, 8 in 2024-09-13T00:51:29.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:29 smithi181 bash[67587]: cluster 2024-09-13T00:51:28.441374+0000 mon.a (mon.0) 4416 : cluster [DBG] osdmap e2602: 8 total, 8 up, 8 in 2024-09-13T00:51:30.604 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestGroups::test_group_snap_rollback FAILED [ 95%] 2024-09-13T00:51:30.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:30 smithi136 bash[75164]: cluster 2024-09-13T00:51:28.729242+0000 mgr.y (mgr.24661) 2230 : cluster [DBG] pgmap v4393: 137 pgs: 7 active+clean+snaptrim_wait, 130 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 454 KiB/s rd, 46 KiB/s wr, 633 op/s 2024-09-13T00:51:30.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:30 smithi136 bash[75164]: cluster 2024-09-13T00:51:29.451268+0000 mon.a (mon.0) 4417 : cluster [DBG] osdmap e2603: 8 total, 8 up, 8 in 2024-09-13T00:51:30.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:30 smithi136 bash[75164]: cluster 2024-09-13T00:51:30.071503+0000 mon.a (mon.0) 4418 : cluster [DBG] osdmap e2604: 8 total, 8 up, 8 in 2024-09-13T00:51:30.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:30 smithi136 bash[76371]: cluster 2024-09-13T00:51:28.729242+0000 mgr.y (mgr.24661) 2230 : cluster [DBG] pgmap v4393: 137 pgs: 7 active+clean+snaptrim_wait, 130 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 454 KiB/s rd, 46 KiB/s wr, 633 op/s 2024-09-13T00:51:30.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:30 smithi136 bash[76371]: cluster 2024-09-13T00:51:29.451268+0000 mon.a (mon.0) 4417 : cluster [DBG] osdmap e2603: 8 total, 8 up, 8 in 2024-09-13T00:51:30.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:30 smithi136 bash[76371]: cluster 2024-09-13T00:51:30.071503+0000 mon.a (mon.0) 4418 : cluster [DBG] osdmap e2604: 8 total, 8 up, 8 in 2024-09-13T00:51:30.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:30 smithi181 bash[67587]: cluster 2024-09-13T00:51:28.729242+0000 mgr.y (mgr.24661) 2230 : cluster [DBG] pgmap v4393: 137 pgs: 7 active+clean+snaptrim_wait, 130 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 454 KiB/s rd, 46 KiB/s wr, 633 op/s 2024-09-13T00:51:30.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:30 smithi181 bash[67587]: cluster 2024-09-13T00:51:29.451268+0000 mon.a (mon.0) 4417 : cluster [DBG] osdmap e2603: 8 total, 8 up, 8 in 2024-09-13T00:51:30.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:30 smithi181 bash[67587]: cluster 2024-09-13T00:51:30.071503+0000 mon.a (mon.0) 4418 : cluster [DBG] osdmap e2604: 8 total, 8 up, 8 in 2024-09-13T00:51:31.371 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestMigration::test_migration PASSED [ 95%] 2024-09-13T00:51:31.617 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestMigration::test_migration_clone_v1_to_v1 FAILED [ 96%] 2024-09-13T00:51:32.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:32 smithi181 bash[67587]: cluster 2024-09-13T00:51:30.730231+0000 mgr.y (mgr.24661) 2231 : cluster [DBG] pgmap v4396: 137 pgs: 7 active+clean+snaptrim_wait, 130 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 358 KiB/s rd, 22 KiB/s wr, 471 op/s 2024-09-13T00:51:32.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:32 smithi181 bash[67587]: cluster 2024-09-13T00:51:31.456469+0000 mon.a (mon.0) 4419 : cluster [DBG] osdmap e2605: 8 total, 8 up, 8 in 2024-09-13T00:51:32.599 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:51:32 smithi181 bash[95505]: ts=2024-09-13T00:51:32.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:51:32.611 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestMigration::test_migration_clone_v1_to_v2 FAILED [ 96%] 2024-09-13T00:51:32.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:32 smithi136 bash[75164]: cluster 2024-09-13T00:51:30.730231+0000 mgr.y (mgr.24661) 2231 : cluster [DBG] pgmap v4396: 137 pgs: 7 active+clean+snaptrim_wait, 130 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 358 KiB/s rd, 22 KiB/s wr, 471 op/s 2024-09-13T00:51:32.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:32 smithi136 bash[75164]: cluster 2024-09-13T00:51:31.456469+0000 mon.a (mon.0) 4419 : cluster [DBG] osdmap e2605: 8 total, 8 up, 8 in 2024-09-13T00:51:32.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:32 smithi136 bash[76371]: cluster 2024-09-13T00:51:30.730231+0000 mgr.y (mgr.24661) 2231 : cluster [DBG] pgmap v4396: 137 pgs: 7 active+clean+snaptrim_wait, 130 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 358 KiB/s rd, 22 KiB/s wr, 471 op/s 2024-09-13T00:51:32.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:32 smithi136 bash[76371]: cluster 2024-09-13T00:51:31.456469+0000 mon.a (mon.0) 4419 : cluster [DBG] osdmap e2605: 8 total, 8 up, 8 in 2024-09-13T00:51:33.626 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestMigration::test_migration_clone_v2_to_v1 FAILED [ 97%] 2024-09-13T00:51:33.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:33 smithi136 bash[76371]: cluster 2024-09-13T00:51:32.471706+0000 mon.a (mon.0) 4420 : cluster [DBG] osdmap e2606: 8 total, 8 up, 8 in 2024-09-13T00:51:33.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:33 smithi136 bash[75164]: cluster 2024-09-13T00:51:32.471706+0000 mon.a (mon.0) 4420 : cluster [DBG] osdmap e2606: 8 total, 8 up, 8 in 2024-09-13T00:51:33.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:33 smithi181 bash[67587]: cluster 2024-09-13T00:51:32.471706+0000 mon.a (mon.0) 4420 : cluster [DBG] osdmap e2606: 8 total, 8 up, 8 in 2024-09-13T00:51:34.614 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestMigration::test_migration_clone_v2_to_v2 FAILED [ 97%] 2024-09-13T00:51:34.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:34 smithi136 bash[75164]: cluster 2024-09-13T00:51:32.730942+0000 mgr.y (mgr.24661) 2232 : cluster [DBG] pgmap v4399: 137 pgs: 7 active+clean+snaptrim_wait, 130 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 293 KiB/s rd, 21 KiB/s wr, 394 op/s 2024-09-13T00:51:34.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:34 smithi136 bash[75164]: cluster 2024-09-13T00:51:33.481653+0000 mon.a (mon.0) 4421 : cluster [DBG] osdmap e2607: 8 total, 8 up, 8 in 2024-09-13T00:51:34.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:34 smithi136 bash[76371]: cluster 2024-09-13T00:51:32.730942+0000 mgr.y (mgr.24661) 2232 : cluster [DBG] pgmap v4399: 137 pgs: 7 active+clean+snaptrim_wait, 130 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 293 KiB/s rd, 21 KiB/s wr, 394 op/s 2024-09-13T00:51:34.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:34 smithi136 bash[76371]: cluster 2024-09-13T00:51:33.481653+0000 mon.a (mon.0) 4421 : cluster [DBG] osdmap e2607: 8 total, 8 up, 8 in 2024-09-13T00:51:34.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:34 smithi181 bash[67587]: cluster 2024-09-13T00:51:32.730942+0000 mgr.y (mgr.24661) 2232 : cluster [DBG] pgmap v4399: 137 pgs: 7 active+clean+snaptrim_wait, 130 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 293 KiB/s rd, 21 KiB/s wr, 394 op/s 2024-09-13T00:51:34.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:34 smithi181 bash[67587]: cluster 2024-09-13T00:51:33.481653+0000 mon.a (mon.0) 4421 : cluster [DBG] osdmap e2607: 8 total, 8 up, 8 in 2024-09-13T00:51:35.760 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:35 smithi136 bash[76371]: cluster 2024-09-13T00:51:34.483868+0000 mon.a (mon.0) 4422 : cluster [DBG] osdmap e2608: 8 total, 8 up, 8 in 2024-09-13T00:51:35.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:35 smithi136 bash[75164]: cluster 2024-09-13T00:51:34.483868+0000 mon.a (mon.0) 4422 : cluster [DBG] osdmap e2608: 8 total, 8 up, 8 in 2024-09-13T00:51:35.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:35 smithi181 bash[67587]: cluster 2024-09-13T00:51:34.483868+0000 mon.a (mon.0) 4422 : cluster [DBG] osdmap e2608: 8 total, 8 up, 8 in 2024-09-13T00:51:36.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:36 smithi181 bash[67587]: cluster 2024-09-13T00:51:34.731635+0000 mgr.y (mgr.24661) 2233 : cluster [DBG] pgmap v4402: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 190 KiB/s rd, 14 KiB/s wr, 257 op/s 2024-09-13T00:51:36.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:36 smithi181 bash[67587]: cluster 2024-09-13T00:51:35.512901+0000 mon.a (mon.0) 4423 : cluster [DBG] osdmap e2609: 8 total, 8 up, 8 in 2024-09-13T00:51:37.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:36 smithi136 bash[75164]: cluster 2024-09-13T00:51:34.731635+0000 mgr.y (mgr.24661) 2233 : cluster [DBG] pgmap v4402: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 190 KiB/s rd, 14 KiB/s wr, 257 op/s 2024-09-13T00:51:37.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:36 smithi136 bash[75164]: cluster 2024-09-13T00:51:35.512901+0000 mon.a (mon.0) 4423 : cluster [DBG] osdmap e2609: 8 total, 8 up, 8 in 2024-09-13T00:51:37.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:36 smithi136 bash[76371]: cluster 2024-09-13T00:51:34.731635+0000 mgr.y (mgr.24661) 2233 : cluster [DBG] pgmap v4402: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 190 KiB/s rd, 14 KiB/s wr, 257 op/s 2024-09-13T00:51:37.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:36 smithi136 bash[76371]: cluster 2024-09-13T00:51:35.512901+0000 mon.a (mon.0) 4423 : cluster [DBG] osdmap e2609: 8 total, 8 up, 8 in 2024-09-13T00:51:37.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:37 smithi181 bash[67587]: cluster 2024-09-13T00:51:36.513931+0000 mon.a (mon.0) 4424 : cluster [DBG] osdmap e2610: 8 total, 8 up, 8 in 2024-09-13T00:51:38.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:37 smithi136 bash[75164]: cluster 2024-09-13T00:51:36.513931+0000 mon.a (mon.0) 4424 : cluster [DBG] osdmap e2610: 8 total, 8 up, 8 in 2024-09-13T00:51:38.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:37 smithi136 bash[76371]: cluster 2024-09-13T00:51:36.513931+0000 mon.a (mon.0) 4424 : cluster [DBG] osdmap e2610: 8 total, 8 up, 8 in 2024-09-13T00:51:38.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:38 smithi181 bash[67587]: cluster 2024-09-13T00:51:36.732752+0000 mgr.y (mgr.24661) 2234 : cluster [DBG] pgmap v4405: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 194 KiB/s rd, 12 KiB/s wr, 258 op/s 2024-09-13T00:51:38.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:38 smithi181 bash[67587]: cluster 2024-09-13T00:51:37.535119+0000 mon.a (mon.0) 4425 : cluster [DBG] osdmap e2611: 8 total, 8 up, 8 in 2024-09-13T00:51:38.904 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestMigration::test_migration_import PASSED [ 98%] 2024-09-13T00:51:39.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:38 smithi136 bash[75164]: cluster 2024-09-13T00:51:36.732752+0000 mgr.y (mgr.24661) 2234 : cluster [DBG] pgmap v4405: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 194 KiB/s rd, 12 KiB/s wr, 258 op/s 2024-09-13T00:51:39.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:38 smithi136 bash[75164]: cluster 2024-09-13T00:51:37.535119+0000 mon.a (mon.0) 4425 : cluster [DBG] osdmap e2611: 8 total, 8 up, 8 in 2024-09-13T00:51:39.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:38 smithi136 bash[76371]: cluster 2024-09-13T00:51:36.732752+0000 mgr.y (mgr.24661) 2234 : cluster [DBG] pgmap v4405: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 194 KiB/s rd, 12 KiB/s wr, 258 op/s 2024-09-13T00:51:39.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:38 smithi136 bash[76371]: cluster 2024-09-13T00:51:37.535119+0000 mon.a (mon.0) 4425 : cluster [DBG] osdmap e2611: 8 total, 8 up, 8 in 2024-09-13T00:51:39.515 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestMigration::test_migration_with_progress PASSED [ 98%] 2024-09-13T00:51:39.760 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:51:39 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:51:39] "GET /metrics HTTP/1.1" 200 36744 "" "Prometheus/2.43.0" 2024-09-13T00:51:39.760 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:39 smithi136 bash[75164]: cluster 2024-09-13T00:51:38.543673+0000 mon.a (mon.0) 4426 : cluster [DBG] osdmap e2612: 8 total, 8 up, 8 in 2024-09-13T00:51:39.761 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:39 smithi136 bash[75164]: audit 2024-09-13T00:51:38.857266+0000 mon.a (mon.0) 4427 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:51:39.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:39 smithi136 bash[76371]: cluster 2024-09-13T00:51:38.543673+0000 mon.a (mon.0) 4426 : cluster [DBG] osdmap e2612: 8 total, 8 up, 8 in 2024-09-13T00:51:39.761 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:39 smithi136 bash[76371]: audit 2024-09-13T00:51:38.857266+0000 mon.a (mon.0) 4427 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:51:39.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:39 smithi181 bash[67587]: cluster 2024-09-13T00:51:38.543673+0000 mon.a (mon.0) 4426 : cluster [DBG] osdmap e2612: 8 total, 8 up, 8 in 2024-09-13T00:51:39.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:39 smithi181 bash[67587]: audit 2024-09-13T00:51:38.857266+0000 mon.a (mon.0) 4427 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:51:40.016 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestMigration::test_migrate_abort PASSED [ 99%] 2024-09-13T00:51:40.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:40 smithi181 bash[67587]: cluster 2024-09-13T00:51:38.733639+0000 mgr.y (mgr.24661) 2235 : cluster [DBG] pgmap v4408: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 218 KiB/s rd, 13 KiB/s wr, 281 op/s 2024-09-13T00:51:40.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:40 smithi181 bash[67587]: cluster 2024-09-13T00:51:40.074904+0000 mon.a (mon.0) 4428 : cluster [DBG] osdmap e2613: 8 total, 8 up, 8 in 2024-09-13T00:51:41.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:40 smithi136 bash[75164]: cluster 2024-09-13T00:51:38.733639+0000 mgr.y (mgr.24661) 2235 : cluster [DBG] pgmap v4408: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 218 KiB/s rd, 13 KiB/s wr, 281 op/s 2024-09-13T00:51:41.010 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:40 smithi136 bash[75164]: cluster 2024-09-13T00:51:40.074904+0000 mon.a (mon.0) 4428 : cluster [DBG] osdmap e2613: 8 total, 8 up, 8 in 2024-09-13T00:51:41.010 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:40 smithi136 bash[76371]: cluster 2024-09-13T00:51:38.733639+0000 mgr.y (mgr.24661) 2235 : cluster [DBG] pgmap v4408: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 218 KiB/s rd, 13 KiB/s wr, 281 op/s 2024-09-13T00:51:41.011 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:40 smithi136 bash[76371]: cluster 2024-09-13T00:51:40.074904+0000 mon.a (mon.0) 4428 : cluster [DBG] osdmap e2613: 8 total, 8 up, 8 in 2024-09-13T00:51:41.098 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py::TestMigration::test_migrate_abort_with_progress PASSED [100%] 2024-09-13T00:51:41.099 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.099 INFO:tasks.workunit.client.0.smithi136.stdout:==================================== ERRORS ==================================== 2024-09-13T00:51:41.099 INFO:tasks.workunit.client.0.smithi136.stdout:_________ ERROR at teardown of TestImage.test_deep_copy_clone_v1_to_v1 _________ 2024-09-13T00:51:41.099 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.099 INFO:tasks.workunit.client.0.smithi136.stdout:self = 2024-09-13T00:51:41.099 INFO:tasks.workunit.client.0.smithi136.stdout:method = > 2024-09-13T00:51:41.099 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.099 INFO:tasks.workunit.client.0.smithi136.stdout: def teardown_method(self, method): 2024-09-13T00:51:41.099 INFO:tasks.workunit.client.0.smithi136.stdout: self.image.close() 2024-09-13T00:51:41.099 INFO:tasks.workunit.client.0.smithi136.stdout:> remove_image() 2024-09-13T00:51:41.100 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.100 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py:584: 2024-09-13T00:51:41.100 INFO:tasks.workunit.client.0.smithi136.stdout:_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 2024-09-13T00:51:41.100 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py:121: in remove_image 2024-09-13T00:51:41.100 INFO:tasks.workunit.client.0.smithi136.stdout: RBD().remove(ioctx, image_name) 2024-09-13T00:51:41.100 INFO:tasks.workunit.client.0.smithi136.stdout:_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 2024-09-13T00:51:41.100 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.100 INFO:tasks.workunit.client.0.smithi136.stdout:> ??? 2024-09-13T00:51:41.100 INFO:tasks.workunit.client.0.smithi136.stdout:E rbd.ImageHasSnapshots: [errno 39] RBD image has snapshots (error removing image) 2024-09-13T00:51:41.100 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.100 INFO:tasks.workunit.client.0.smithi136.stdout:rbd.pyx:767: ImageHasSnapshots 2024-09-13T00:51:41.100 INFO:tasks.workunit.client.0.smithi136.stdout:_________ ERROR at teardown of TestImage.test_deep_copy_clone_v1_to_v2 _________ 2024-09-13T00:51:41.100 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.101 INFO:tasks.workunit.client.0.smithi136.stdout:self = 2024-09-13T00:51:41.101 INFO:tasks.workunit.client.0.smithi136.stdout:method = > 2024-09-13T00:51:41.101 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.101 INFO:tasks.workunit.client.0.smithi136.stdout: def teardown_method(self, method): 2024-09-13T00:51:41.101 INFO:tasks.workunit.client.0.smithi136.stdout: self.image.close() 2024-09-13T00:51:41.101 INFO:tasks.workunit.client.0.smithi136.stdout:> remove_image() 2024-09-13T00:51:41.101 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.101 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py:584: 2024-09-13T00:51:41.101 INFO:tasks.workunit.client.0.smithi136.stdout:_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 2024-09-13T00:51:41.101 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py:121: in remove_image 2024-09-13T00:51:41.101 INFO:tasks.workunit.client.0.smithi136.stdout: RBD().remove(ioctx, image_name) 2024-09-13T00:51:41.101 INFO:tasks.workunit.client.0.smithi136.stdout:_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 2024-09-13T00:51:41.102 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.102 INFO:tasks.workunit.client.0.smithi136.stdout:> ??? 2024-09-13T00:51:41.102 INFO:tasks.workunit.client.0.smithi136.stdout:E rbd.ImageHasSnapshots: [errno 39] RBD image has snapshots (error removing image) 2024-09-13T00:51:41.102 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.102 INFO:tasks.workunit.client.0.smithi136.stdout:rbd.pyx:767: ImageHasSnapshots 2024-09-13T00:51:41.102 INFO:tasks.workunit.client.0.smithi136.stdout:_________ ERROR at teardown of TestImage.test_deep_copy_clone_v2_to_v1 _________ 2024-09-13T00:51:41.102 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.102 INFO:tasks.workunit.client.0.smithi136.stdout:self = 2024-09-13T00:51:41.102 INFO:tasks.workunit.client.0.smithi136.stdout:method = > 2024-09-13T00:51:41.102 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.102 INFO:tasks.workunit.client.0.smithi136.stdout: def teardown_method(self, method): 2024-09-13T00:51:41.102 INFO:tasks.workunit.client.0.smithi136.stdout: self.image.close() 2024-09-13T00:51:41.102 INFO:tasks.workunit.client.0.smithi136.stdout:> remove_image() 2024-09-13T00:51:41.102 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.103 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py:584: 2024-09-13T00:51:41.103 INFO:tasks.workunit.client.0.smithi136.stdout:_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 2024-09-13T00:51:41.103 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py:121: in remove_image 2024-09-13T00:51:41.103 INFO:tasks.workunit.client.0.smithi136.stdout: RBD().remove(ioctx, image_name) 2024-09-13T00:51:41.103 INFO:tasks.workunit.client.0.smithi136.stdout:_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 2024-09-13T00:51:41.103 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.103 INFO:tasks.workunit.client.0.smithi136.stdout:> ??? 2024-09-13T00:51:41.103 INFO:tasks.workunit.client.0.smithi136.stdout:E rbd.ImageHasSnapshots: [errno 39] RBD image has snapshots (error removing image) 2024-09-13T00:51:41.103 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.103 INFO:tasks.workunit.client.0.smithi136.stdout:rbd.pyx:767: ImageHasSnapshots 2024-09-13T00:51:41.103 INFO:tasks.workunit.client.0.smithi136.stdout:_________ ERROR at teardown of TestImage.test_deep_copy_clone_v2_to_v2 _________ 2024-09-13T00:51:41.103 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.104 INFO:tasks.workunit.client.0.smithi136.stdout:self = 2024-09-13T00:51:41.104 INFO:tasks.workunit.client.0.smithi136.stdout:method = > 2024-09-13T00:51:41.104 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.104 INFO:tasks.workunit.client.0.smithi136.stdout: def teardown_method(self, method): 2024-09-13T00:51:41.104 INFO:tasks.workunit.client.0.smithi136.stdout: self.image.close() 2024-09-13T00:51:41.104 INFO:tasks.workunit.client.0.smithi136.stdout:> remove_image() 2024-09-13T00:51:41.104 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.104 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py:584: 2024-09-13T00:51:41.104 INFO:tasks.workunit.client.0.smithi136.stdout:_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 2024-09-13T00:51:41.104 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py:121: in remove_image 2024-09-13T00:51:41.104 INFO:tasks.workunit.client.0.smithi136.stdout: RBD().remove(ioctx, image_name) 2024-09-13T00:51:41.105 INFO:tasks.workunit.client.0.smithi136.stdout:_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 2024-09-13T00:51:41.105 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.105 INFO:tasks.workunit.client.0.smithi136.stdout:> ??? 2024-09-13T00:51:41.105 INFO:tasks.workunit.client.0.smithi136.stdout:E rbd.ImageHasSnapshots: [errno 39] RBD image has snapshots (error removing image) 2024-09-13T00:51:41.105 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.105 INFO:tasks.workunit.client.0.smithi136.stdout:rbd.pyx:767: ImageHasSnapshots 2024-09-13T00:51:41.105 INFO:tasks.workunit.client.0.smithi136.stdout:________ ERROR at teardown of TestImage.test_deep_copy_clone_v1_flatten ________ 2024-09-13T00:51:41.105 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.105 INFO:tasks.workunit.client.0.smithi136.stdout:self = 2024-09-13T00:51:41.105 INFO:tasks.workunit.client.0.smithi136.stdout:method = > 2024-09-13T00:51:41.105 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.106 INFO:tasks.workunit.client.0.smithi136.stdout: def teardown_method(self, method): 2024-09-13T00:51:41.106 INFO:tasks.workunit.client.0.smithi136.stdout: self.image.close() 2024-09-13T00:51:41.106 INFO:tasks.workunit.client.0.smithi136.stdout:> remove_image() 2024-09-13T00:51:41.106 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.106 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py:584: 2024-09-13T00:51:41.106 INFO:tasks.workunit.client.0.smithi136.stdout:_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 2024-09-13T00:51:41.106 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py:121: in remove_image 2024-09-13T00:51:41.106 INFO:tasks.workunit.client.0.smithi136.stdout: RBD().remove(ioctx, image_name) 2024-09-13T00:51:41.106 INFO:tasks.workunit.client.0.smithi136.stdout:_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 2024-09-13T00:51:41.106 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.106 INFO:tasks.workunit.client.0.smithi136.stdout:> ??? 2024-09-13T00:51:41.106 INFO:tasks.workunit.client.0.smithi136.stdout:E rbd.ImageHasSnapshots: [errno 39] RBD image has snapshots (error removing image) 2024-09-13T00:51:41.106 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.106 INFO:tasks.workunit.client.0.smithi136.stdout:rbd.pyx:767: ImageHasSnapshots 2024-09-13T00:51:41.107 INFO:tasks.workunit.client.0.smithi136.stdout:________ ERROR at teardown of TestImage.test_deep_copy_clone_v2_flatten ________ 2024-09-13T00:51:41.107 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.107 INFO:tasks.workunit.client.0.smithi136.stdout:self = 2024-09-13T00:51:41.107 INFO:tasks.workunit.client.0.smithi136.stdout:method = > 2024-09-13T00:51:41.107 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.107 INFO:tasks.workunit.client.0.smithi136.stdout: def teardown_method(self, method): 2024-09-13T00:51:41.107 INFO:tasks.workunit.client.0.smithi136.stdout: self.image.close() 2024-09-13T00:51:41.107 INFO:tasks.workunit.client.0.smithi136.stdout:> remove_image() 2024-09-13T00:51:41.107 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.107 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py:584: 2024-09-13T00:51:41.107 INFO:tasks.workunit.client.0.smithi136.stdout:_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 2024-09-13T00:51:41.107 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py:121: in remove_image 2024-09-13T00:51:41.108 INFO:tasks.workunit.client.0.smithi136.stdout: RBD().remove(ioctx, image_name) 2024-09-13T00:51:41.108 INFO:tasks.workunit.client.0.smithi136.stdout:_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 2024-09-13T00:51:41.108 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.108 INFO:tasks.workunit.client.0.smithi136.stdout:> ??? 2024-09-13T00:51:41.108 INFO:tasks.workunit.client.0.smithi136.stdout:E rbd.ImageHasSnapshots: [errno 39] RBD image has snapshots (error removing image) 2024-09-13T00:51:41.108 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.108 INFO:tasks.workunit.client.0.smithi136.stdout:rbd.pyx:767: ImageHasSnapshots 2024-09-13T00:51:41.108 INFO:tasks.workunit.client.0.smithi136.stdout:_______________ ERROR at teardown of TestClone.test_unprotected ________________ 2024-09-13T00:51:41.108 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.108 INFO:tasks.workunit.client.0.smithi136.stdout:self = 2024-09-13T00:51:41.108 INFO:tasks.workunit.client.0.smithi136.stdout:method = > 2024-09-13T00:51:41.108 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.109 INFO:tasks.workunit.client.0.smithi136.stdout: def teardown_method(self, method): 2024-09-13T00:51:41.109 INFO:tasks.workunit.client.0.smithi136.stdout: global ioctx 2024-09-13T00:51:41.109 INFO:tasks.workunit.client.0.smithi136.stdout: self.clone.close() 2024-09-13T00:51:41.109 INFO:tasks.workunit.client.0.smithi136.stdout: self.rbd.remove(ioctx, self.clone_name) 2024-09-13T00:51:41.109 INFO:tasks.workunit.client.0.smithi136.stdout: self.image.unprotect_snap('snap1') 2024-09-13T00:51:41.109 INFO:tasks.workunit.client.0.smithi136.stdout: self.image.remove_snap('snap1') 2024-09-13T00:51:41.109 INFO:tasks.workunit.client.0.smithi136.stdout: self.image.close() 2024-09-13T00:51:41.109 INFO:tasks.workunit.client.0.smithi136.stdout:> remove_image() 2024-09-13T00:51:41.109 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.109 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py:1632: 2024-09-13T00:51:41.109 INFO:tasks.workunit.client.0.smithi136.stdout:_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 2024-09-13T00:51:41.109 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py:121: in remove_image 2024-09-13T00:51:41.109 INFO:tasks.workunit.client.0.smithi136.stdout: RBD().remove(ioctx, image_name) 2024-09-13T00:51:41.109 INFO:tasks.workunit.client.0.smithi136.stdout:_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 2024-09-13T00:51:41.110 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.110 INFO:tasks.workunit.client.0.smithi136.stdout:> ??? 2024-09-13T00:51:41.110 INFO:tasks.workunit.client.0.smithi136.stdout:E rbd.ImageHasSnapshots: [errno 39] RBD image has snapshots (error removing image) 2024-09-13T00:51:41.110 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.110 INFO:tasks.workunit.client.0.smithi136.stdout:rbd.pyx:767: ImageHasSnapshots 2024-09-13T00:51:41.110 INFO:tasks.workunit.client.0.smithi136.stdout:=================================== FAILURES =================================== 2024-09-13T00:51:41.110 INFO:tasks.workunit.client.0.smithi136.stdout:___________________ TestImage.test_deep_copy_clone_v1_to_v1 ____________________ 2024-09-13T00:51:41.110 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.110 INFO:tasks.workunit.client.0.smithi136.stdout:self = 2024-09-13T00:51:41.110 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.110 INFO:tasks.workunit.client.0.smithi136.stdout: @require_features([RBD_FEATURE_LAYERING]) 2024-09-13T00:51:41.110 INFO:tasks.workunit.client.0.smithi136.stdout: def test_deep_copy_clone_v1_to_v1(self): 2024-09-13T00:51:41.110 INFO:tasks.workunit.client.0.smithi136.stdout: self.image.write(b'a' * 256, 0) 2024-09-13T00:51:41.111 INFO:tasks.workunit.client.0.smithi136.stdout: self.image.create_snap('snap1') 2024-09-13T00:51:41.111 INFO:tasks.workunit.client.0.smithi136.stdout: self.image.write(b'b' * 256, 0) 2024-09-13T00:51:41.111 INFO:tasks.workunit.client.0.smithi136.stdout: self.image.protect_snap('snap1') 2024-09-13T00:51:41.111 INFO:tasks.workunit.client.0.smithi136.stdout: clone_name = get_temp_image_name() 2024-09-13T00:51:41.111 INFO:tasks.workunit.client.0.smithi136.stdout: dst_name = get_temp_image_name() 2024-09-13T00:51:41.111 INFO:tasks.workunit.client.0.smithi136.stdout:> self.rbd.clone(ioctx, image_name, 'snap1', ioctx, clone_name, features, 2024-09-13T00:51:41.111 INFO:tasks.workunit.client.0.smithi136.stdout: clone_format=1) 2024-09-13T00:51:41.111 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.111 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py:823: 2024-09-13T00:51:41.111 INFO:tasks.workunit.client.0.smithi136.stdout:_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 2024-09-13T00:51:41.111 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.111 INFO:tasks.workunit.client.0.smithi136.stdout:> ??? 2024-09-13T00:51:41.111 INFO:tasks.workunit.client.0.smithi136.stdout:E TypeError: clone() got an unexpected keyword argument 'clone_format' 2024-09-13T00:51:41.111 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.111 INFO:tasks.workunit.client.0.smithi136.stdout:rbd.pyx:631: TypeError 2024-09-13T00:51:41.112 INFO:tasks.workunit.client.0.smithi136.stdout:___________________ TestImage.test_deep_copy_clone_v1_to_v2 ____________________ 2024-09-13T00:51:41.112 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.112 INFO:tasks.workunit.client.0.smithi136.stdout:self = 2024-09-13T00:51:41.112 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.112 INFO:tasks.workunit.client.0.smithi136.stdout: @require_features([RBD_FEATURE_LAYERING]) 2024-09-13T00:51:41.112 INFO:tasks.workunit.client.0.smithi136.stdout: def test_deep_copy_clone_v1_to_v2(self): 2024-09-13T00:51:41.112 INFO:tasks.workunit.client.0.smithi136.stdout: self.image.write(b'a' * 256, 0) 2024-09-13T00:51:41.112 INFO:tasks.workunit.client.0.smithi136.stdout: self.image.create_snap('snap1') 2024-09-13T00:51:41.112 INFO:tasks.workunit.client.0.smithi136.stdout: self.image.write(b'b' * 256, 0) 2024-09-13T00:51:41.112 INFO:tasks.workunit.client.0.smithi136.stdout: self.image.protect_snap('snap1') 2024-09-13T00:51:41.112 INFO:tasks.workunit.client.0.smithi136.stdout: clone_name = get_temp_image_name() 2024-09-13T00:51:41.112 INFO:tasks.workunit.client.0.smithi136.stdout: dst_name = get_temp_image_name() 2024-09-13T00:51:41.112 INFO:tasks.workunit.client.0.smithi136.stdout:> self.rbd.clone(ioctx, image_name, 'snap1', ioctx, clone_name, features, 2024-09-13T00:51:41.112 INFO:tasks.workunit.client.0.smithi136.stdout: clone_format=1) 2024-09-13T00:51:41.112 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.113 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py:854: 2024-09-13T00:51:41.113 INFO:tasks.workunit.client.0.smithi136.stdout:_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 2024-09-13T00:51:41.113 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.113 INFO:tasks.workunit.client.0.smithi136.stdout:> ??? 2024-09-13T00:51:41.113 INFO:tasks.workunit.client.0.smithi136.stdout:E TypeError: clone() got an unexpected keyword argument 'clone_format' 2024-09-13T00:51:41.113 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.113 INFO:tasks.workunit.client.0.smithi136.stdout:rbd.pyx:631: TypeError 2024-09-13T00:51:41.113 INFO:tasks.workunit.client.0.smithi136.stdout:___________________ TestImage.test_deep_copy_clone_v2_to_v1 ____________________ 2024-09-13T00:51:41.113 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.113 INFO:tasks.workunit.client.0.smithi136.stdout:self = 2024-09-13T00:51:41.113 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.113 INFO:tasks.workunit.client.0.smithi136.stdout: @require_features([RBD_FEATURE_LAYERING]) 2024-09-13T00:51:41.114 INFO:tasks.workunit.client.0.smithi136.stdout: def test_deep_copy_clone_v2_to_v1(self): 2024-09-13T00:51:41.114 INFO:tasks.workunit.client.0.smithi136.stdout: self.image.write(b'a' * 256, 0) 2024-09-13T00:51:41.114 INFO:tasks.workunit.client.0.smithi136.stdout: self.image.create_snap('snap1') 2024-09-13T00:51:41.114 INFO:tasks.workunit.client.0.smithi136.stdout: self.image.write(b'b' * 256, 0) 2024-09-13T00:51:41.114 INFO:tasks.workunit.client.0.smithi136.stdout: self.image.protect_snap('snap1') 2024-09-13T00:51:41.114 INFO:tasks.workunit.client.0.smithi136.stdout: clone_name = get_temp_image_name() 2024-09-13T00:51:41.114 INFO:tasks.workunit.client.0.smithi136.stdout: dst_name = get_temp_image_name() 2024-09-13T00:51:41.114 INFO:tasks.workunit.client.0.smithi136.stdout:> self.rbd.clone(ioctx, image_name, 'snap1', ioctx, clone_name, features, 2024-09-13T00:51:41.114 INFO:tasks.workunit.client.0.smithi136.stdout: clone_format=2) 2024-09-13T00:51:41.114 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.115 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py:885: 2024-09-13T00:51:41.115 INFO:tasks.workunit.client.0.smithi136.stdout:_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 2024-09-13T00:51:41.115 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.115 INFO:tasks.workunit.client.0.smithi136.stdout:> ??? 2024-09-13T00:51:41.115 INFO:tasks.workunit.client.0.smithi136.stdout:E TypeError: clone() got an unexpected keyword argument 'clone_format' 2024-09-13T00:51:41.115 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.115 INFO:tasks.workunit.client.0.smithi136.stdout:rbd.pyx:631: TypeError 2024-09-13T00:51:41.115 INFO:tasks.workunit.client.0.smithi136.stdout:___________________ TestImage.test_deep_copy_clone_v2_to_v2 ____________________ 2024-09-13T00:51:41.116 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.116 INFO:tasks.workunit.client.0.smithi136.stdout:self = 2024-09-13T00:51:41.116 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.116 INFO:tasks.workunit.client.0.smithi136.stdout: @require_features([RBD_FEATURE_LAYERING]) 2024-09-13T00:51:41.116 INFO:tasks.workunit.client.0.smithi136.stdout: def test_deep_copy_clone_v2_to_v2(self): 2024-09-13T00:51:41.116 INFO:tasks.workunit.client.0.smithi136.stdout: self.image.write(b'a' * 256, 0) 2024-09-13T00:51:41.116 INFO:tasks.workunit.client.0.smithi136.stdout: self.image.create_snap('snap1') 2024-09-13T00:51:41.116 INFO:tasks.workunit.client.0.smithi136.stdout: self.image.write(b'b' * 256, 0) 2024-09-13T00:51:41.116 INFO:tasks.workunit.client.0.smithi136.stdout: clone_name = get_temp_image_name() 2024-09-13T00:51:41.116 INFO:tasks.workunit.client.0.smithi136.stdout: dst_name = get_temp_image_name() 2024-09-13T00:51:41.116 INFO:tasks.workunit.client.0.smithi136.stdout:> self.rbd.clone(ioctx, image_name, 'snap1', ioctx, clone_name, features, 2024-09-13T00:51:41.116 INFO:tasks.workunit.client.0.smithi136.stdout: clone_format=2) 2024-09-13T00:51:41.116 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.116 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py:915: 2024-09-13T00:51:41.117 INFO:tasks.workunit.client.0.smithi136.stdout:_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 2024-09-13T00:51:41.117 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.117 INFO:tasks.workunit.client.0.smithi136.stdout:> ??? 2024-09-13T00:51:41.117 INFO:tasks.workunit.client.0.smithi136.stdout:E TypeError: clone() got an unexpected keyword argument 'clone_format' 2024-09-13T00:51:41.117 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.117 INFO:tasks.workunit.client.0.smithi136.stdout:rbd.pyx:631: TypeError 2024-09-13T00:51:41.117 INFO:tasks.workunit.client.0.smithi136.stdout:__________________ TestImage.test_deep_copy_clone_v1_flatten ___________________ 2024-09-13T00:51:41.117 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.117 INFO:tasks.workunit.client.0.smithi136.stdout:self = 2024-09-13T00:51:41.117 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.117 INFO:tasks.workunit.client.0.smithi136.stdout: @require_features([RBD_FEATURE_LAYERING]) 2024-09-13T00:51:41.117 INFO:tasks.workunit.client.0.smithi136.stdout: def test_deep_copy_clone_v1_flatten(self): 2024-09-13T00:51:41.117 INFO:tasks.workunit.client.0.smithi136.stdout: self.image.write(b'a' * 256, 0) 2024-09-13T00:51:41.117 INFO:tasks.workunit.client.0.smithi136.stdout: self.image.create_snap('snap1') 2024-09-13T00:51:41.117 INFO:tasks.workunit.client.0.smithi136.stdout: self.image.write(b'b' * 256, 0) 2024-09-13T00:51:41.118 INFO:tasks.workunit.client.0.smithi136.stdout: self.image.protect_snap('snap1') 2024-09-13T00:51:41.118 INFO:tasks.workunit.client.0.smithi136.stdout: clone_name = get_temp_image_name() 2024-09-13T00:51:41.118 INFO:tasks.workunit.client.0.smithi136.stdout: dst_name = get_temp_image_name() 2024-09-13T00:51:41.118 INFO:tasks.workunit.client.0.smithi136.stdout:> self.rbd.clone(ioctx, image_name, 'snap1', ioctx, clone_name, features, 2024-09-13T00:51:41.118 INFO:tasks.workunit.client.0.smithi136.stdout: clone_format=1) 2024-09-13T00:51:41.118 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.118 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py:945: 2024-09-13T00:51:41.118 INFO:tasks.workunit.client.0.smithi136.stdout:_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 2024-09-13T00:51:41.118 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.118 INFO:tasks.workunit.client.0.smithi136.stdout:> ??? 2024-09-13T00:51:41.118 INFO:tasks.workunit.client.0.smithi136.stdout:E TypeError: clone() got an unexpected keyword argument 'clone_format' 2024-09-13T00:51:41.118 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.118 INFO:tasks.workunit.client.0.smithi136.stdout:rbd.pyx:631: TypeError 2024-09-13T00:51:41.118 INFO:tasks.workunit.client.0.smithi136.stdout:__________________ TestImage.test_deep_copy_clone_v2_flatten ___________________ 2024-09-13T00:51:41.118 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.119 INFO:tasks.workunit.client.0.smithi136.stdout:self = 2024-09-13T00:51:41.119 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.119 INFO:tasks.workunit.client.0.smithi136.stdout: @require_features([RBD_FEATURE_LAYERING]) 2024-09-13T00:51:41.119 INFO:tasks.workunit.client.0.smithi136.stdout: def test_deep_copy_clone_v2_flatten(self): 2024-09-13T00:51:41.119 INFO:tasks.workunit.client.0.smithi136.stdout: self.image.write(b'a' * 256, 0) 2024-09-13T00:51:41.119 INFO:tasks.workunit.client.0.smithi136.stdout: self.image.create_snap('snap1') 2024-09-13T00:51:41.119 INFO:tasks.workunit.client.0.smithi136.stdout: self.image.write(b'b' * 256, 0) 2024-09-13T00:51:41.119 INFO:tasks.workunit.client.0.smithi136.stdout: clone_name = get_temp_image_name() 2024-09-13T00:51:41.119 INFO:tasks.workunit.client.0.smithi136.stdout: dst_name = get_temp_image_name() 2024-09-13T00:51:41.119 INFO:tasks.workunit.client.0.smithi136.stdout:> self.rbd.clone(ioctx, image_name, 'snap1', ioctx, clone_name, features, 2024-09-13T00:51:41.119 INFO:tasks.workunit.client.0.smithi136.stdout: clone_format=2) 2024-09-13T00:51:41.119 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.119 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py:975: 2024-09-13T00:51:41.119 INFO:tasks.workunit.client.0.smithi136.stdout:_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 2024-09-13T00:51:41.119 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.119 INFO:tasks.workunit.client.0.smithi136.stdout:> ??? 2024-09-13T00:51:41.120 INFO:tasks.workunit.client.0.smithi136.stdout:E TypeError: clone() got an unexpected keyword argument 'clone_format' 2024-09-13T00:51:41.120 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.120 INFO:tasks.workunit.client.0.smithi136.stdout:rbd.pyx:631: TypeError 2024-09-13T00:51:41.120 INFO:tasks.workunit.client.0.smithi136.stdout:_________________________ TestClone.test_clone_format __________________________ 2024-09-13T00:51:41.120 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.120 INFO:tasks.workunit.client.0.smithi136.stdout:self = 2024-09-13T00:51:41.120 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.120 INFO:tasks.workunit.client.0.smithi136.stdout: def test_clone_format(self): 2024-09-13T00:51:41.120 INFO:tasks.workunit.client.0.smithi136.stdout: clone_name2 = get_temp_image_name() 2024-09-13T00:51:41.120 INFO:tasks.workunit.client.0.smithi136.stdout:> self.rbd.clone(ioctx, image_name, 'snap1', ioctx, clone_name2, 2024-09-13T00:51:41.120 INFO:tasks.workunit.client.0.smithi136.stdout: features, clone_format=1) 2024-09-13T00:51:41.120 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.120 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py:1696: 2024-09-13T00:51:41.121 INFO:tasks.workunit.client.0.smithi136.stdout:_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 2024-09-13T00:51:41.121 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.121 INFO:tasks.workunit.client.0.smithi136.stdout:> ??? 2024-09-13T00:51:41.121 INFO:tasks.workunit.client.0.smithi136.stdout:E TypeError: clone() got an unexpected keyword argument 'clone_format' 2024-09-13T00:51:41.121 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.121 INFO:tasks.workunit.client.0.smithi136.stdout:rbd.pyx:631: TypeError 2024-09-13T00:51:41.121 INFO:tasks.workunit.client.0.smithi136.stdout:__________________________ TestClone.test_unprotected __________________________ 2024-09-13T00:51:41.121 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.121 INFO:tasks.workunit.client.0.smithi136.stdout:self = 2024-09-13T00:51:41.121 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.121 INFO:tasks.workunit.client.0.smithi136.stdout: def test_unprotected(self): 2024-09-13T00:51:41.121 INFO:tasks.workunit.client.0.smithi136.stdout: self.image.create_snap('snap2') 2024-09-13T00:51:41.121 INFO:tasks.workunit.client.0.smithi136.stdout: clone_name2 = get_temp_image_name() 2024-09-13T00:51:41.122 INFO:tasks.workunit.client.0.smithi136.stdout: # clone format 1 requires a protected snapshot 2024-09-13T00:51:41.122 INFO:tasks.workunit.client.0.smithi136.stdout:> assert_raises(InvalidArgument, self.rbd.clone, ioctx, image_name, 2024-09-13T00:51:41.122 INFO:tasks.workunit.client.0.smithi136.stdout: 'snap2', ioctx, clone_name2, features, clone_format=1) 2024-09-13T00:51:41.122 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.122 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py:1712: 2024-09-13T00:51:41.122 INFO:tasks.workunit.client.0.smithi136.stdout:_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 2024-09-13T00:51:41.122 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/assertions.py:18: in assert_raises 2024-09-13T00:51:41.122 INFO:tasks.workunit.client.0.smithi136.stdout: callableObj(*args, **kwargs) 2024-09-13T00:51:41.122 INFO:tasks.workunit.client.0.smithi136.stdout:_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 2024-09-13T00:51:41.122 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.122 INFO:tasks.workunit.client.0.smithi136.stdout:> ??? 2024-09-13T00:51:41.122 INFO:tasks.workunit.client.0.smithi136.stdout:E TypeError: clone() got an unexpected keyword argument 'clone_format' 2024-09-13T00:51:41.122 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.123 INFO:tasks.workunit.client.0.smithi136.stdout:rbd.pyx:631: TypeError 2024-09-13T00:51:41.123 INFO:tasks.workunit.client.0.smithi136.stdout:_____________________ TestGroups.test_group_snap_rollback ______________________ 2024-09-13T00:51:41.123 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.123 INFO:tasks.workunit.client.0.smithi136.stdout:self = 2024-09-13T00:51:41.123 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.123 INFO:tasks.workunit.client.0.smithi136.stdout: def test_group_snap_rollback(self): 2024-09-13T00:51:41.123 INFO:tasks.workunit.client.0.smithi136.stdout: for _ in range(1, 3): 2024-09-13T00:51:41.123 INFO:tasks.workunit.client.0.smithi136.stdout: create_image() 2024-09-13T00:51:41.123 INFO:tasks.workunit.client.0.smithi136.stdout: self.image_names.append(image_name) 2024-09-13T00:51:41.123 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.123 INFO:tasks.workunit.client.0.smithi136.stdout: with Image(ioctx, self.image_names[0]) as image: 2024-09-13T00:51:41.123 INFO:tasks.workunit.client.0.smithi136.stdout: image.write(b'1' * 256, 0) 2024-09-13T00:51:41.123 INFO:tasks.workunit.client.0.smithi136.stdout: with Image(ioctx, self.image_names[1]) as image: 2024-09-13T00:51:41.124 INFO:tasks.workunit.client.0.smithi136.stdout: image.write(b'2' * 256, 0) 2024-09-13T00:51:41.124 INFO:tasks.workunit.client.0.smithi136.stdout: with Image(ioctx, self.image_names[2]) as image: 2024-09-13T00:51:41.124 INFO:tasks.workunit.client.0.smithi136.stdout: image.write(b'3' * 256, 0) 2024-09-13T00:51:41.124 INFO:tasks.workunit.client.0.smithi136.stdout: self.group.add_image(ioctx, self.image_names[0]) 2024-09-13T00:51:41.124 INFO:tasks.workunit.client.0.smithi136.stdout: snap_name1 = get_temp_snap_name() 2024-09-13T00:51:41.124 INFO:tasks.workunit.client.0.smithi136.stdout: self.group.create_snap(snap_name1) 2024-09-13T00:51:41.124 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.124 INFO:tasks.workunit.client.0.smithi136.stdout: with Image(ioctx, self.image_names[0]) as image: 2024-09-13T00:51:41.124 INFO:tasks.workunit.client.0.smithi136.stdout: image.write(b'4' * 256, 0) 2024-09-13T00:51:41.124 INFO:tasks.workunit.client.0.smithi136.stdout: with Image(ioctx, self.image_names[1]) as image: 2024-09-13T00:51:41.124 INFO:tasks.workunit.client.0.smithi136.stdout: image.write(b'5' * 256, 0) 2024-09-13T00:51:41.124 INFO:tasks.workunit.client.0.smithi136.stdout: with Image(ioctx, self.image_names[2]) as image: 2024-09-13T00:51:41.125 INFO:tasks.workunit.client.0.smithi136.stdout: image.write(b'6' * 256, 0) 2024-09-13T00:51:41.125 INFO:tasks.workunit.client.0.smithi136.stdout: self.group.add_image(ioctx, self.image_names[1]) 2024-09-13T00:51:41.125 INFO:tasks.workunit.client.0.smithi136.stdout: snap_name2 = get_temp_snap_name() 2024-09-13T00:51:41.125 INFO:tasks.workunit.client.0.smithi136.stdout: self.group.create_snap(snap_name2) 2024-09-13T00:51:41.125 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.125 INFO:tasks.workunit.client.0.smithi136.stdout: with Image(ioctx, self.image_names[0]) as image: 2024-09-13T00:51:41.125 INFO:tasks.workunit.client.0.smithi136.stdout: image.write(b'7' * 256, 0) 2024-09-13T00:51:41.125 INFO:tasks.workunit.client.0.smithi136.stdout: with Image(ioctx, self.image_names[1]) as image: 2024-09-13T00:51:41.125 INFO:tasks.workunit.client.0.smithi136.stdout: image.write(b'8' * 256, 0) 2024-09-13T00:51:41.125 INFO:tasks.workunit.client.0.smithi136.stdout: with Image(ioctx, self.image_names[2]) as image: 2024-09-13T00:51:41.125 INFO:tasks.workunit.client.0.smithi136.stdout: image.write(b'9' * 256, 0) 2024-09-13T00:51:41.125 INFO:tasks.workunit.client.0.smithi136.stdout: self.group.add_image(ioctx, self.image_names[2]) 2024-09-13T00:51:41.125 INFO:tasks.workunit.client.0.smithi136.stdout: snap_name3 = get_temp_snap_name() 2024-09-13T00:51:41.126 INFO:tasks.workunit.client.0.smithi136.stdout: self.group.create_snap(snap_name3) 2024-09-13T00:51:41.126 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.126 INFO:tasks.workunit.client.0.smithi136.stdout: with Image(ioctx, self.image_names[0]) as image: 2024-09-13T00:51:41.126 INFO:tasks.workunit.client.0.smithi136.stdout: image.write(b'a' * 256, 0) 2024-09-13T00:51:41.126 INFO:tasks.workunit.client.0.smithi136.stdout: with Image(ioctx, self.image_names[1]) as image: 2024-09-13T00:51:41.126 INFO:tasks.workunit.client.0.smithi136.stdout: image.write(b'b' * 256, 0) 2024-09-13T00:51:41.126 INFO:tasks.workunit.client.0.smithi136.stdout: with Image(ioctx, self.image_names[2]) as image: 2024-09-13T00:51:41.126 INFO:tasks.workunit.client.0.smithi136.stdout: image.write(b'c' * 256, 0) 2024-09-13T00:51:41.126 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.126 INFO:tasks.workunit.client.0.smithi136.stdout: for i in range(0, 3): 2024-09-13T00:51:41.126 INFO:tasks.workunit.client.0.smithi136.stdout: self.group.remove_image(ioctx, self.image_names[i]) 2024-09-13T00:51:41.126 INFO:tasks.workunit.client.0.smithi136.stdout: with Image(ioctx, self.image_names[0]) as image: 2024-09-13T00:51:41.126 INFO:tasks.workunit.client.0.smithi136.stdout: image_snaps = list(image.list_snaps()) 2024-09-13T00:51:41.127 INFO:tasks.workunit.client.0.smithi136.stdout: assert [s['namespace'] for s in image_snaps] == [RBD_SNAP_NAMESPACE_TYPE_GROUP, 2024-09-13T00:51:41.127 INFO:tasks.workunit.client.0.smithi136.stdout: RBD_SNAP_NAMESPACE_TYPE_GROUP, 2024-09-13T00:51:41.127 INFO:tasks.workunit.client.0.smithi136.stdout: RBD_SNAP_NAMESPACE_TYPE_GROUP] 2024-09-13T00:51:41.127 INFO:tasks.workunit.client.0.smithi136.stdout: with Image(ioctx, self.image_names[1]) as image: 2024-09-13T00:51:41.127 INFO:tasks.workunit.client.0.smithi136.stdout: image_snaps = list(image.list_snaps()) 2024-09-13T00:51:41.127 INFO:tasks.workunit.client.0.smithi136.stdout: assert [s['namespace'] for s in image_snaps] == [RBD_SNAP_NAMESPACE_TYPE_GROUP, 2024-09-13T00:51:41.127 INFO:tasks.workunit.client.0.smithi136.stdout: RBD_SNAP_NAMESPACE_TYPE_GROUP] 2024-09-13T00:51:41.127 INFO:tasks.workunit.client.0.smithi136.stdout: with Image(ioctx, self.image_names[2]) as image: 2024-09-13T00:51:41.127 INFO:tasks.workunit.client.0.smithi136.stdout: image_snaps = list(image.list_snaps()) 2024-09-13T00:51:41.127 INFO:tasks.workunit.client.0.smithi136.stdout: assert [s['namespace'] for s in image_snaps] == [RBD_SNAP_NAMESPACE_TYPE_GROUP] 2024-09-13T00:51:41.127 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.128 INFO:tasks.workunit.client.0.smithi136.stdout: # group = [] 2024-09-13T00:51:41.128 INFO:tasks.workunit.client.0.smithi136.stdout:> assert_raises(InvalidArgument, self.group.rollback_to_snap, snap_name1) 2024-09-13T00:51:41.128 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.128 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py:2878: 2024-09-13T00:51:41.128 INFO:tasks.workunit.client.0.smithi136.stdout:_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 2024-09-13T00:51:41.128 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.128 INFO:tasks.workunit.client.0.smithi136.stdout:excClass = 2024-09-13T00:51:41.128 INFO:tasks.workunit.client.0.smithi136.stdout:callableObj = 2024-09-13T00:51:41.128 INFO:tasks.workunit.client.0.smithi136.stdout:args = ('snap33',), kwargs = {}, excName = 'InvalidArgument' 2024-09-13T00:51:41.128 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.128 INFO:tasks.workunit.client.0.smithi136.stdout: def assert_raises(excClass, callableObj, *args, **kwargs): 2024-09-13T00:51:41.128 INFO:tasks.workunit.client.0.smithi136.stdout: """ 2024-09-13T00:51:41.128 INFO:tasks.workunit.client.0.smithi136.stdout: Like unittest.TestCase.assertRaises, but returns the exception. 2024-09-13T00:51:41.129 INFO:tasks.workunit.client.0.smithi136.stdout: """ 2024-09-13T00:51:41.129 INFO:tasks.workunit.client.0.smithi136.stdout: try: 2024-09-13T00:51:41.129 INFO:tasks.workunit.client.0.smithi136.stdout: callableObj(*args, **kwargs) 2024-09-13T00:51:41.129 INFO:tasks.workunit.client.0.smithi136.stdout: except excClass as e: 2024-09-13T00:51:41.129 INFO:tasks.workunit.client.0.smithi136.stdout: return e 2024-09-13T00:51:41.129 INFO:tasks.workunit.client.0.smithi136.stdout: else: 2024-09-13T00:51:41.129 INFO:tasks.workunit.client.0.smithi136.stdout: if hasattr(excClass, '__name__'): 2024-09-13T00:51:41.129 INFO:tasks.workunit.client.0.smithi136.stdout: excName = excClass.__name__ 2024-09-13T00:51:41.129 INFO:tasks.workunit.client.0.smithi136.stdout: else: 2024-09-13T00:51:41.129 INFO:tasks.workunit.client.0.smithi136.stdout: excName = str(excClass) 2024-09-13T00:51:41.129 INFO:tasks.workunit.client.0.smithi136.stdout:> raise AssertionError("%s not raised" % excName) 2024-09-13T00:51:41.129 INFO:tasks.workunit.client.0.smithi136.stdout:E AssertionError: InvalidArgument not raised 2024-09-13T00:51:41.129 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.130 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/assertions.py:26: AssertionError 2024-09-13T00:51:41.130 INFO:tasks.workunit.client.0.smithi136.stdout:_________________ TestMigration.test_migration_clone_v1_to_v1 __________________ 2024-09-13T00:51:41.130 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.130 INFO:tasks.workunit.client.0.smithi136.stdout:self = 2024-09-13T00:51:41.130 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.130 INFO:tasks.workunit.client.0.smithi136.stdout: @require_features([RBD_FEATURE_LAYERING]) 2024-09-13T00:51:41.130 INFO:tasks.workunit.client.0.smithi136.stdout: def test_migration_clone_v1_to_v1(self): 2024-09-13T00:51:41.130 INFO:tasks.workunit.client.0.smithi136.stdout: create_image() 2024-09-13T00:51:41.130 INFO:tasks.workunit.client.0.smithi136.stdout: with Image(ioctx, image_name) as image: 2024-09-13T00:51:41.130 INFO:tasks.workunit.client.0.smithi136.stdout: image_id = image.id() 2024-09-13T00:51:41.130 INFO:tasks.workunit.client.0.smithi136.stdout: image.create_snap('snap1') 2024-09-13T00:51:41.130 INFO:tasks.workunit.client.0.smithi136.stdout: image.protect_snap('snap1') 2024-09-13T00:51:41.130 INFO:tasks.workunit.client.0.smithi136.stdout: clone_name = get_temp_image_name() 2024-09-13T00:51:41.131 INFO:tasks.workunit.client.0.smithi136.stdout:> RBD().clone(ioctx, image_name, 'snap1', ioctx, clone_name, features, 2024-09-13T00:51:41.131 INFO:tasks.workunit.client.0.smithi136.stdout: clone_format=1) 2024-09-13T00:51:41.131 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.131 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py:3031: 2024-09-13T00:51:41.131 INFO:tasks.workunit.client.0.smithi136.stdout:_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 2024-09-13T00:51:41.131 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.131 INFO:tasks.workunit.client.0.smithi136.stdout:> ??? 2024-09-13T00:51:41.131 INFO:tasks.workunit.client.0.smithi136.stdout:E TypeError: clone() got an unexpected keyword argument 'clone_format' 2024-09-13T00:51:41.131 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.131 INFO:tasks.workunit.client.0.smithi136.stdout:rbd.pyx:631: TypeError 2024-09-13T00:51:41.131 INFO:tasks.workunit.client.0.smithi136.stdout:_________________ TestMigration.test_migration_clone_v1_to_v2 __________________ 2024-09-13T00:51:41.131 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.131 INFO:tasks.workunit.client.0.smithi136.stdout:self = 2024-09-13T00:51:41.132 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.132 INFO:tasks.workunit.client.0.smithi136.stdout: @require_features([RBD_FEATURE_LAYERING]) 2024-09-13T00:51:41.132 INFO:tasks.workunit.client.0.smithi136.stdout: def test_migration_clone_v1_to_v2(self): 2024-09-13T00:51:41.132 INFO:tasks.workunit.client.0.smithi136.stdout: create_image() 2024-09-13T00:51:41.132 INFO:tasks.workunit.client.0.smithi136.stdout: with Image(ioctx, image_name) as image: 2024-09-13T00:51:41.132 INFO:tasks.workunit.client.0.smithi136.stdout: image_id = image.id() 2024-09-13T00:51:41.132 INFO:tasks.workunit.client.0.smithi136.stdout: image.create_snap('snap1') 2024-09-13T00:51:41.132 INFO:tasks.workunit.client.0.smithi136.stdout: image.protect_snap('snap1') 2024-09-13T00:51:41.132 INFO:tasks.workunit.client.0.smithi136.stdout: clone_name = get_temp_image_name() 2024-09-13T00:51:41.132 INFO:tasks.workunit.client.0.smithi136.stdout:> RBD().clone(ioctx, image_name, 'snap1', ioctx, clone_name, features, 2024-09-13T00:51:41.132 INFO:tasks.workunit.client.0.smithi136.stdout: clone_format=1) 2024-09-13T00:51:41.132 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.132 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py:3060: 2024-09-13T00:51:41.132 INFO:tasks.workunit.client.0.smithi136.stdout:_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 2024-09-13T00:51:41.133 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.133 INFO:tasks.workunit.client.0.smithi136.stdout:> ??? 2024-09-13T00:51:41.133 INFO:tasks.workunit.client.0.smithi136.stdout:E TypeError: clone() got an unexpected keyword argument 'clone_format' 2024-09-13T00:51:41.133 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.133 INFO:tasks.workunit.client.0.smithi136.stdout:rbd.pyx:631: TypeError 2024-09-13T00:51:41.133 INFO:tasks.workunit.client.0.smithi136.stdout:_________________ TestMigration.test_migration_clone_v2_to_v1 __________________ 2024-09-13T00:51:41.133 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.133 INFO:tasks.workunit.client.0.smithi136.stdout:self = 2024-09-13T00:51:41.133 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.133 INFO:tasks.workunit.client.0.smithi136.stdout: @require_features([RBD_FEATURE_LAYERING]) 2024-09-13T00:51:41.133 INFO:tasks.workunit.client.0.smithi136.stdout: def test_migration_clone_v2_to_v1(self): 2024-09-13T00:51:41.133 INFO:tasks.workunit.client.0.smithi136.stdout: create_image() 2024-09-13T00:51:41.133 INFO:tasks.workunit.client.0.smithi136.stdout: with Image(ioctx, image_name) as image: 2024-09-13T00:51:41.134 INFO:tasks.workunit.client.0.smithi136.stdout: image_id = image.id() 2024-09-13T00:51:41.134 INFO:tasks.workunit.client.0.smithi136.stdout: image.create_snap('snap1') 2024-09-13T00:51:41.134 INFO:tasks.workunit.client.0.smithi136.stdout: image.protect_snap('snap1') 2024-09-13T00:51:41.134 INFO:tasks.workunit.client.0.smithi136.stdout: clone_name = get_temp_image_name() 2024-09-13T00:51:41.134 INFO:tasks.workunit.client.0.smithi136.stdout:> RBD().clone(ioctx, image_name, 'snap1', ioctx, clone_name, features, 2024-09-13T00:51:41.134 INFO:tasks.workunit.client.0.smithi136.stdout: clone_format=2) 2024-09-13T00:51:41.134 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.134 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py:3089: 2024-09-13T00:51:41.134 INFO:tasks.workunit.client.0.smithi136.stdout:_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 2024-09-13T00:51:41.134 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.134 INFO:tasks.workunit.client.0.smithi136.stdout:> ??? 2024-09-13T00:51:41.134 INFO:tasks.workunit.client.0.smithi136.stdout:E TypeError: clone() got an unexpected keyword argument 'clone_format' 2024-09-13T00:51:41.134 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.134 INFO:tasks.workunit.client.0.smithi136.stdout:rbd.pyx:631: TypeError 2024-09-13T00:51:41.135 INFO:tasks.workunit.client.0.smithi136.stdout:_________________ TestMigration.test_migration_clone_v2_to_v2 __________________ 2024-09-13T00:51:41.135 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.135 INFO:tasks.workunit.client.0.smithi136.stdout:self = 2024-09-13T00:51:41.135 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.135 INFO:tasks.workunit.client.0.smithi136.stdout: @require_features([RBD_FEATURE_LAYERING]) 2024-09-13T00:51:41.135 INFO:tasks.workunit.client.0.smithi136.stdout: def test_migration_clone_v2_to_v2(self): 2024-09-13T00:51:41.135 INFO:tasks.workunit.client.0.smithi136.stdout: create_image() 2024-09-13T00:51:41.135 INFO:tasks.workunit.client.0.smithi136.stdout: with Image(ioctx, image_name) as image: 2024-09-13T00:51:41.135 INFO:tasks.workunit.client.0.smithi136.stdout: image_id = image.id() 2024-09-13T00:51:41.135 INFO:tasks.workunit.client.0.smithi136.stdout: image.create_snap('snap1') 2024-09-13T00:51:41.135 INFO:tasks.workunit.client.0.smithi136.stdout: clone_name = get_temp_image_name() 2024-09-13T00:51:41.135 INFO:tasks.workunit.client.0.smithi136.stdout:> RBD().clone(ioctx, image_name, 'snap1', ioctx, clone_name, features, 2024-09-13T00:51:41.135 INFO:tasks.workunit.client.0.smithi136.stdout: clone_format=2) 2024-09-13T00:51:41.136 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.136 INFO:tasks.workunit.client.0.smithi136.stdout:../../../clone.client.0/src/test/pybind/test_rbd.py:3117: 2024-09-13T00:51:41.136 INFO:tasks.workunit.client.0.smithi136.stdout:_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 2024-09-13T00:51:41.136 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.136 INFO:tasks.workunit.client.0.smithi136.stdout:> ??? 2024-09-13T00:51:41.136 INFO:tasks.workunit.client.0.smithi136.stdout:E TypeError: clone() got an unexpected keyword argument 'clone_format' 2024-09-13T00:51:41.136 INFO:tasks.workunit.client.0.smithi136.stdout: 2024-09-13T00:51:41.136 INFO:tasks.workunit.client.0.smithi136.stdout:rbd.pyx:631: TypeError 2024-09-13T00:51:41.136 INFO:tasks.workunit.client.0.smithi136.stdout:=========================== short test summary info ============================ 2024-09-13T00:51:41.136 INFO:tasks.workunit.client.0.smithi136.stdout:FAILED ../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_deep_copy_clone_v1_to_v1 2024-09-13T00:51:41.136 INFO:tasks.workunit.client.0.smithi136.stdout:FAILED ../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_deep_copy_clone_v1_to_v2 2024-09-13T00:51:41.136 INFO:tasks.workunit.client.0.smithi136.stdout:FAILED ../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_deep_copy_clone_v2_to_v1 2024-09-13T00:51:41.137 INFO:tasks.workunit.client.0.smithi136.stdout:FAILED ../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_deep_copy_clone_v2_to_v2 2024-09-13T00:51:41.137 INFO:tasks.workunit.client.0.smithi136.stdout:FAILED ../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_deep_copy_clone_v1_flatten 2024-09-13T00:51:41.137 INFO:tasks.workunit.client.0.smithi136.stdout:FAILED ../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_deep_copy_clone_v2_flatten 2024-09-13T00:51:41.137 INFO:tasks.workunit.client.0.smithi136.stdout:FAILED ../../../clone.client.0/src/test/pybind/test_rbd.py::TestClone::test_clone_format 2024-09-13T00:51:41.137 INFO:tasks.workunit.client.0.smithi136.stdout:FAILED ../../../clone.client.0/src/test/pybind/test_rbd.py::TestClone::test_unprotected 2024-09-13T00:51:41.137 INFO:tasks.workunit.client.0.smithi136.stdout:FAILED ../../../clone.client.0/src/test/pybind/test_rbd.py::TestGroups::test_group_snap_rollback 2024-09-13T00:51:41.137 INFO:tasks.workunit.client.0.smithi136.stdout:FAILED ../../../clone.client.0/src/test/pybind/test_rbd.py::TestMigration::test_migration_clone_v1_to_v1 2024-09-13T00:51:41.137 INFO:tasks.workunit.client.0.smithi136.stdout:FAILED ../../../clone.client.0/src/test/pybind/test_rbd.py::TestMigration::test_migration_clone_v1_to_v2 2024-09-13T00:51:41.137 INFO:tasks.workunit.client.0.smithi136.stdout:FAILED ../../../clone.client.0/src/test/pybind/test_rbd.py::TestMigration::test_migration_clone_v2_to_v1 2024-09-13T00:51:41.137 INFO:tasks.workunit.client.0.smithi136.stdout:FAILED ../../../clone.client.0/src/test/pybind/test_rbd.py::TestMigration::test_migration_clone_v2_to_v2 2024-09-13T00:51:41.137 INFO:tasks.workunit.client.0.smithi136.stdout:ERROR ../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_deep_copy_clone_v1_to_v1 2024-09-13T00:51:41.137 INFO:tasks.workunit.client.0.smithi136.stdout:ERROR ../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_deep_copy_clone_v1_to_v2 2024-09-13T00:51:41.138 INFO:tasks.workunit.client.0.smithi136.stdout:ERROR ../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_deep_copy_clone_v2_to_v1 2024-09-13T00:51:41.138 INFO:tasks.workunit.client.0.smithi136.stdout:ERROR ../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_deep_copy_clone_v2_to_v2 2024-09-13T00:51:41.138 INFO:tasks.workunit.client.0.smithi136.stdout:ERROR ../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_deep_copy_clone_v1_flatten 2024-09-13T00:51:41.138 INFO:tasks.workunit.client.0.smithi136.stdout:ERROR ../../../clone.client.0/src/test/pybind/test_rbd.py::TestImage::test_deep_copy_clone_v2_flatten 2024-09-13T00:51:41.138 INFO:tasks.workunit.client.0.smithi136.stdout:ERROR ../../../clone.client.0/src/test/pybind/test_rbd.py::TestClone::test_unprotected 2024-09-13T00:51:41.138 INFO:tasks.workunit.client.0.smithi136.stdout:======= 13 failed, 161 passed, 8 skipped, 7 errors in 665.86s (0:11:05) ======== 2024-09-13T00:51:41.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T00:51:41.160 INFO:tasks.workunit:Stopping ['rbd/test_librbd_python.sh'] on client.0... 2024-09-13T00:51:41.160 DEBUG:teuthology.orchestra.run.smithi136:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2024-09-13T00:51:41.425 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 48, in task with parallel.parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/full_sequential.py", line 30, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_7a853b1706d076d58e0da7a5d3b9b9eb36276880/qa/tasks/workunit.py", line 126, in task with parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_7a853b1706d076d58e0da7a5d3b9b9eb36276880/qa/tasks/workunit.py", line 424, in _run_tests remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed (workunit test rbd/test_librbd_python.sh) on smithi136 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_FEATURES=61 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/test_librbd_python.sh' 2024-09-13T00:51:41.671 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=3ff6a9c762404a5db8bc7718de80ef7c Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 48, in task with parallel.parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/full_sequential.py", line 30, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_7a853b1706d076d58e0da7a5d3b9b9eb36276880/qa/tasks/workunit.py", line 126, in task with parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_7a853b1706d076d58e0da7a5d3b9b9eb36276880/qa/tasks/workunit.py", line 424, in _run_tests remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed (workunit test rbd/test_librbd_python.sh) on smithi136 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_FEATURES=61 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/test_librbd_python.sh' 2024-09-13T00:51:41.674 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-09-13T00:51:41.685 INFO:tasks.cephadm:Teardown begin 2024-09-13T00:51:41.685 INFO:tasks.cephfs.filesystem:Destroying file system cephfs and related pools 2024-09-13T00:51:41.685 DEBUG:teuthology.orchestra.run.smithi136:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-09-13T00:51:42.030 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-13T00:51:42.031 INFO:teuthology.orchestra.run.smithi136.stdout:{"epoch":15,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":54211,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.181:6833/3688985057","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6832","nonce":3688985057},{"type":"v1","addr":"172.21.15.181:6833","nonce":3688985057}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":15}],"filesystems":[{"mdsmap":{"epoch":14,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-09-12T23:33:29.192396+0000","modified":"2024-09-12T23:48:24.200564+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":644,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":1,"in":[0],"up":{"mds_0":44318},"failed":[],"damaged":[],"stopped":[],"info":{"gid_44318":{"gid":44318,"name":"a","rank":0,"incarnation":11,"state":"up:active","state_seq":9,"addr":"172.21.15.136:6835/496731293","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6834","nonce":496731293},{"type":"v1","addr":"172.21.15.136:6835","nonce":496731293}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2024-09-13T00:51:42.031 INFO:teuthology.orchestra.run.smithi136.stderr:dumped fsmap epoch 15 2024-09-13T00:51:42.050 DEBUG:teuthology.orchestra.run.smithi136:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-09-13T00:51:42.348 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:51:42 smithi181 bash[95505]: ts=2024-09-13T00:51:42.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:51:42.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:42 smithi181 bash[67587]: cluster 2024-09-13T00:51:40.734610+0000 mgr.y (mgr.24661) 2236 : cluster [DBG] pgmap v4410: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 270 KiB/s rd, 13 KiB/s wr, 347 op/s 2024-09-13T00:51:42.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:42 smithi181 bash[67587]: cluster 2024-09-13T00:51:41.077014+0000 mon.a (mon.0) 4429 : cluster [DBG] osdmap e2614: 8 total, 8 up, 8 in 2024-09-13T00:51:42.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:42 smithi181 bash[67587]: audit 2024-09-13T00:51:42.030401+0000 mon.c (mon.2) 160 : audit [DBG] from='client.? 172.21.15.136:0/1182210815' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-13T00:51:42.385 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-13T00:51:42.386 INFO:teuthology.orchestra.run.smithi136.stdout:{"epoch":15,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":54211,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.181:6833/3688985057","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6832","nonce":3688985057},{"type":"v1","addr":"172.21.15.181:6833","nonce":3688985057}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":15}],"filesystems":[{"mdsmap":{"epoch":14,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-09-12T23:33:29.192396+0000","modified":"2024-09-12T23:48:24.200564+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":644,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":1,"in":[0],"up":{"mds_0":44318},"failed":[],"damaged":[],"stopped":[],"info":{"gid_44318":{"gid":44318,"name":"a","rank":0,"incarnation":11,"state":"up:active","state_seq":9,"addr":"172.21.15.136:6835/496731293","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6834","nonce":496731293},{"type":"v1","addr":"172.21.15.136:6835","nonce":496731293}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2024-09-13T00:51:42.386 INFO:teuthology.orchestra.run.smithi136.stderr:dumped fsmap epoch 15 2024-09-13T00:51:42.403 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:42 smithi136 bash[75164]: cluster 2024-09-13T00:51:40.734610+0000 mgr.y (mgr.24661) 2236 : cluster [DBG] pgmap v4410: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 270 KiB/s rd, 13 KiB/s wr, 347 op/s 2024-09-13T00:51:42.403 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:42 smithi136 bash[75164]: cluster 2024-09-13T00:51:41.077014+0000 mon.a (mon.0) 4429 : cluster [DBG] osdmap e2614: 8 total, 8 up, 8 in 2024-09-13T00:51:42.404 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:42 smithi136 bash[75164]: audit 2024-09-13T00:51:42.030401+0000 mon.c (mon.2) 160 : audit [DBG] from='client.? 172.21.15.136:0/1182210815' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-13T00:51:42.404 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:42 smithi136 bash[76371]: cluster 2024-09-13T00:51:40.734610+0000 mgr.y (mgr.24661) 2236 : cluster [DBG] pgmap v4410: 137 pgs: 137 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 270 KiB/s rd, 13 KiB/s wr, 347 op/s 2024-09-13T00:51:42.404 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:42 smithi136 bash[76371]: cluster 2024-09-13T00:51:41.077014+0000 mon.a (mon.0) 4429 : cluster [DBG] osdmap e2614: 8 total, 8 up, 8 in 2024-09-13T00:51:42.404 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:42 smithi136 bash[76371]: audit 2024-09-13T00:51:42.030401+0000 mon.c (mon.2) 160 : audit [DBG] from='client.? 172.21.15.136:0/1182210815' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-13T00:51:42.405 DEBUG:teuthology.orchestra.run.smithi136:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-13T00:51:42.712 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-13T00:51:42.713 INFO:teuthology.orchestra.run.smithi136.stdout:{"epoch":2614,"fsid":"da58a47c-715d-11ef-bceb-c7b262605968","created":"2024-09-12T23:23:29.000926+0000","modified":"2024-09-13T00:51:41.067816+0000","last_up_change":"2024-09-12T23:47:42.111391+0000","last_in_change":"2024-09-12T23:32:18.995528+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":37,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":114,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-12T23:28:33.956985+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-09-12T23:33:27.478434+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-09-12T23:33:28.566349+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"54","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"rbd","create_time":"2024-09-12T23:34:48.404257+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"1134","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":1134,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":3,"score_stable":3,"optimal_score":1,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"87aedf04-695e-4a8f-80c3-ffd18883a4d5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":391,"up_from":405,"up_thru":2527,"down_at":392,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6802","nonce":1696528169},{"type":"v1","addr":"172.21.15.136:6803","nonce":1696528169}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6804","nonce":1696528169},{"type":"v1","addr":"172.21.15.136:6805","nonce":1696528169}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6808","nonce":1696528169},{"type":"v1","addr":"172.21.15.136:6809","nonce":1696528169}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6806","nonce":1696528169},{"type":"v1","addr":"172.21.15.136:6807","nonce":1696528169}]},"public_addr":"172.21.15.136:6803/1696528169","cluster_addr":"172.21.15.136:6805/1696528169","heartbeat_back_addr":"172.21.15.136:6809/1696528169","heartbeat_front_addr":"172.21.15.136:6807/1696528169","state":["exists","up"]},{"osd":1,"uuid":"850d4f98-0640-4151-bd45-a351ca41d54e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":419,"up_from":433,"up_thru":2527,"down_at":420,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6810","nonce":3078861551},{"type":"v1","addr":"172.21.15.136:6811","nonce":3078861551}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6812","nonce":3078861551},{"type":"v1","addr":"172.21.15.136:6813","nonce":3078861551}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6816","nonce":3078861551},{"type":"v1","addr":"172.21.15.136:6817","nonce":3078861551}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6814","nonce":3078861551},{"type":"v1","addr":"172.21.15.136:6815","nonce":3078861551}]},"public_addr":"172.21.15.136:6811/3078861551","cluster_addr":"172.21.15.136:6813/3078861551","heartbeat_back_addr":"172.21.15.136:6817/3078861551","heartbeat_front_addr":"172.21.15.136:6815/3078861551","state":["exists","up"]},{"osd":2,"uuid":"bde707d6-e027-4b63-9853-fd59212eea1e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":449,"up_from":463,"up_thru":2527,"down_at":450,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6818","nonce":439422470},{"type":"v1","addr":"172.21.15.136:6819","nonce":439422470}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6820","nonce":439422470},{"type":"v1","addr":"172.21.15.136:6821","nonce":439422470}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6824","nonce":439422470},{"type":"v1","addr":"172.21.15.136:6825","nonce":439422470}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6822","nonce":439422470},{"type":"v1","addr":"172.21.15.136:6823","nonce":439422470}]},"public_addr":"172.21.15.136:6819/439422470","cluster_addr":"172.21.15.136:6821/439422470","heartbeat_back_addr":"172.21.15.136:6825/439422470","heartbeat_front_addr":"172.21.15.136:6823/439422470","state":["exists","up"]},{"osd":3,"uuid":"d61f8357-b89b-4c05-b366-65a9f788064d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":478,"up_from":492,"up_thru":2527,"down_at":479,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6826","nonce":2156253618},{"type":"v1","addr":"172.21.15.136:6827","nonce":2156253618}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6828","nonce":2156253618},{"type":"v1","addr":"172.21.15.136:6829","nonce":2156253618}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6832","nonce":2156253618},{"type":"v1","addr":"172.21.15.136:6833","nonce":2156253618}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6830","nonce":2156253618},{"type":"v1","addr":"172.21.15.136:6831","nonce":2156253618}]},"public_addr":"172.21.15.136:6827/2156253618","cluster_addr":"172.21.15.136:6829/2156253618","heartbeat_back_addr":"172.21.15.136:6833/2156253618","heartbeat_front_addr":"172.21.15.136:6831/2156253618","state":["exists","up"]},{"osd":4,"uuid":"98b0e81c-824e-4dbc-9faa-2de9f199af1b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":30,"last_clean_end":507,"up_from":520,"up_thru":2527,"down_at":508,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6800","nonce":2759874318},{"type":"v1","addr":"172.21.15.181:6801","nonce":2759874318}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6802","nonce":2759874318},{"type":"v1","addr":"172.21.15.181:6803","nonce":2759874318}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6806","nonce":2759874318},{"type":"v1","addr":"172.21.15.181:6807","nonce":2759874318}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6804","nonce":2759874318},{"type":"v1","addr":"172.21.15.181:6805","nonce":2759874318}]},"public_addr":"172.21.15.181:6801/2759874318","cluster_addr":"172.21.15.181:6803/2759874318","heartbeat_back_addr":"172.21.15.181:6807/2759874318","heartbeat_front_addr":"172.21.15.181:6805/2759874318","state":["exists","up"]},{"osd":5,"uuid":"5c6d2f69-7f9c-4689-95b0-bd4eb858956d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":36,"last_clean_end":534,"up_from":548,"up_thru":2527,"down_at":535,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6808","nonce":3863825756},{"type":"v1","addr":"172.21.15.181:6809","nonce":3863825756}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6810","nonce":3863825756},{"type":"v1","addr":"172.21.15.181:6811","nonce":3863825756}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6814","nonce":3863825756},{"type":"v1","addr":"172.21.15.181:6815","nonce":3863825756}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6812","nonce":3863825756},{"type":"v1","addr":"172.21.15.181:6813","nonce":3863825756}]},"public_addr":"172.21.15.181:6809/3863825756","cluster_addr":"172.21.15.181:6811/3863825756","heartbeat_back_addr":"172.21.15.181:6815/3863825756","heartbeat_front_addr":"172.21.15.181:6813/3863825756","state":["exists","up"]},{"osd":6,"uuid":"532ee052-af15-433a-8e93-b0b642aaf85b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":42,"last_clean_end":563,"up_from":575,"up_thru":2527,"down_at":564,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6816","nonce":1329407811},{"type":"v1","addr":"172.21.15.181:6817","nonce":1329407811}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6818","nonce":1329407811},{"type":"v1","addr":"172.21.15.181:6819","nonce":1329407811}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6822","nonce":1329407811},{"type":"v1","addr":"172.21.15.181:6823","nonce":1329407811}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6820","nonce":1329407811},{"type":"v1","addr":"172.21.15.181:6821","nonce":1329407811}]},"public_addr":"172.21.15.181:6817/1329407811","cluster_addr":"172.21.15.181:6819/1329407811","heartbeat_back_addr":"172.21.15.181:6823/1329407811","heartbeat_front_addr":"172.21.15.181:6821/1329407811","state":["exists","up"]},{"osd":7,"uuid":"d2ff80f0-6ef0-4098-877c-a891d48c687d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":48,"last_clean_end":591,"up_from":605,"up_thru":2527,"down_at":592,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6824","nonce":2237860309},{"type":"v1","addr":"172.21.15.181:6825","nonce":2237860309}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6826","nonce":2237860309},{"type":"v1","addr":"172.21.15.181:6827","nonce":2237860309}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6830","nonce":2237860309},{"type":"v1","addr":"172.21.15.181:6831","nonce":2237860309}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6828","nonce":2237860309},{"type":"v1","addr":"172.21.15.181:6829","nonce":2237860309}]},"public_addr":"172.21.15.181:6825/2237860309","cluster_addr":"172.21.15.181:6827/2237860309","heartbeat_back_addr":"172.21.15.181:6831/2237860309","heartbeat_front_addr":"172.21.15.181:6829/2237860309","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:26:50.657580+0000","dead_epoch":391},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:27:40.536476+0000","dead_epoch":419},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:28:31.755012+0000","dead_epoch":449},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:29:23.585451+0000","dead_epoch":478},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:30:11.888258+0000","dead_epoch":507},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:31:01.812765+0000","dead_epoch":534},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:31:52.166696+0000","dead_epoch":563},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-12T23:32:42.696525+0000","dead_epoch":590}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.3","mappings":[{"from":7,"to":0}]},{"pgid":"2.a","mappings":[{"from":7,"to":0}]},{"pgid":"2.b","mappings":[{"from":7,"to":2}]},{"pgid":"2.1a","mappings":[{"from":7,"to":0}]},{"pgid":"2.1d","mappings":[{"from":7,"to":4}]},{"pgid":"2.1f","mappings":[{"from":7,"to":2}]},{"pgid":"3.9","mappings":[{"from":4,"to":1}]},{"pgid":"3.a","mappings":[{"from":4,"to":0}]},{"pgid":"3.10","mappings":[{"from":0,"to":1}]},{"pgid":"3.11","mappings":[{"from":7,"to":0}]},{"pgid":"3.14","mappings":[{"from":4,"to":2}]},{"pgid":"3.30","mappings":[{"from":5,"to":2}]},{"pgid":"3.31","mappings":[{"from":4,"to":2}]},{"pgid":"3.37","mappings":[{"from":5,"to":1}]},{"pgid":"3.3e","mappings":[{"from":6,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.136:0/2437398169":"2024-09-13T01:31:26.284421+0000","172.21.15.136:0/1549678324":"2024-09-13T23:23:54.847045+0000","172.21.15.136:0/129641967":"2024-09-13T23:34:07.892265+0000","172.21.15.181:6832/2077070284":"2024-09-13T23:48:21.160950+0000","172.21.15.136:0/4113402351":"2024-09-13T23:40:07.531346+0000","172.21.15.181:0/1736879849":"2024-09-13T23:41:07.408939+0000","172.21.15.136:6801/3939504613":"2024-09-13T23:40:07.531346+0000","172.21.15.136:6800/3939504613":"2024-09-13T23:40:07.531346+0000","172.21.15.136:0/3346741202":"2024-09-13T23:40:07.531346+0000","172.21.15.136:6800/1810367175":"2024-09-13T23:34:07.892265+0000","172.21.15.136:0/3924275900":"2024-09-13T23:34:07.892265+0000","172.21.15.136:6800/821792295":"2024-09-13T23:42:08.612521+0000","172.21.15.136:0/1211818073":"2024-09-13T01:36:34.122188+0000","172.21.15.136:0/895274419":"2024-09-13T01:31:25.304748+0000","172.21.15.181:6835/1989969453":"2024-09-13T23:41:07.408939+0000","172.21.15.181:6834/1989969453":"2024-09-13T23:41:07.408939+0000","172.21.15.136:0/3161590810":"2024-09-13T23:42:08.612521+0000","172.21.15.136:0/1890850023":"2024-09-13T23:42:08.612521+0000","172.21.15.181:0/3047046473":"2024-09-13T23:41:07.408939+0000","172.21.15.181:0/1149753131":"2024-09-13T23:41:07.408939+0000","172.21.15.136:0/1370177551":"2024-09-13T23:42:08.612521+0000","172.21.15.136:0/1913282058":"2024-09-13T23:40:07.531346+0000","172.21.15.136:6800/2894181186":"2024-09-13T23:23:54.847045+0000","172.21.15.136:0/1964283041":"2024-09-13T23:24:25.346823+0000","172.21.15.181:0/2898677511":"2024-09-13T23:41:07.408939+0000","172.21.15.136:6800/3594272622":"2024-09-13T23:24:25.346823+0000","172.21.15.136:0/2528576426":"2024-09-13T23:40:07.531346+0000","172.21.15.136:6801/2894181186":"2024-09-13T23:23:54.847045+0000","172.21.15.181:0/912737797":"2024-09-13T23:41:07.408939+0000","172.21.15.181:6833/2077070284":"2024-09-13T23:48:21.160950+0000","172.21.15.136:0/1675562321":"2024-09-13T23:40:07.531346+0000","172.21.15.136:0/320185649":"2024-09-13T23:42:08.612521+0000","172.21.15.181:0/3092067856":"2024-09-13T23:41:07.408939+0000","172.21.15.136:0/3642432358":"2024-09-13T23:34:07.892265+0000","172.21.15.136:0/942965045":"2024-09-13T23:42:08.612521+0000","172.21.15.136:6801/821792295":"2024-09-13T23:42:08.612521+0000","172.21.15.136:0/3376519841":"2024-09-13T23:23:54.847045+0000","172.21.15.136:6801/1810367175":"2024-09-13T23:34:07.892265+0000","172.21.15.136:0/513809002":"2024-09-13T23:34:07.892265+0000","172.21.15.136:0/2231517774":"2024-09-13T01:51:00.466854+0000","172.21.15.136:0/3910468981":"2024-09-13T23:23:54.847045+0000","172.21.15.136:0/4260962972":"2024-09-13T23:24:25.346823+0000","172.21.15.136:0/3094962794":"2024-09-13T23:42:08.612521+0000","172.21.15.136:6801/3594272622":"2024-09-13T23:24:25.346823+0000","172.21.15.136:0/3143991445":"2024-09-13T23:42:08.612521+0000","172.21.15.181:0/1647856294":"2024-09-13T23:41:07.408939+0000","172.21.15.136:0/209539205":"2024-09-13T23:24:25.346823+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"},"teuthologyprofile":{"crush-device-class":"","crush-failure-domain":"osd","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van","w":"8"}},"removed_snaps_queue":[{"pool":60,"snaps":[{"begin":2,"length":1}]},{"pool":63,"snaps":[{"begin":1,"length":2}]},{"pool":68,"snaps":[{"begin":1,"length":2}]},{"pool":77,"snaps":[{"begin":2,"length":1}]},{"pool":78,"snaps":[{"begin":2,"length":1}]},{"pool":79,"snaps":[{"begin":2,"length":1}]},{"pool":80,"snaps":[{"begin":2,"length":1}]},{"pool":81,"snaps":[{"begin":2,"length":1}]},{"pool":82,"snaps":[{"begin":2,"length":1}]},{"pool":85,"snaps":[{"begin":6,"length":1},{"begin":8,"length":1},{"begin":10,"length":1},{"begin":12,"length":1},{"begin":14,"length":1}]},{"pool":87,"snaps":[{"begin":62,"length":2},{"begin":71,"length":3}]},{"pool":88,"snaps":[{"begin":24,"length":2},{"begin":31,"length":1}]},{"pool":89,"snaps":[{"begin":2,"length":1}]},{"pool":111,"snaps":[{"begin":9,"length":5}]},{"pool":112,"snaps":[{"begin":9,"length":5}]},{"pool":113,"snaps":[{"begin":595,"length":2}]},{"pool":114,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T00:51:42.732 DEBUG:teuthology.orchestra.run.smithi136:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs fail cephfs 2024-09-13T00:51:43.076 INFO:teuthology.orchestra.run.smithi136.stderr:cephfs marked not joinable; MDS cannot join the cluster. All MDS ranks marked failed. 2024-09-13T00:51:43.099 DEBUG:teuthology.orchestra.run.smithi136:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs rm cephfs --yes-i-really-mean-it 2024-09-13T00:51:43.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:43 smithi181 bash[67587]: audit 2024-09-13T00:51:42.385235+0000 mon.a (mon.0) 4430 : audit [DBG] from='client.? 172.21.15.136:0/912672868' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-13T00:51:43.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:43 smithi181 bash[67587]: audit 2024-09-13T00:51:42.707247+0000 mon.c (mon.2) 161 : audit [DBG] from='client.? 172.21.15.136:0/2639364842' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T00:51:43.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:43 smithi181 bash[67587]: audit 2024-09-13T00:51:43.058643+0000 mon.c (mon.2) 162 : audit [INF] from='client.? 172.21.15.136:0/1914867600' entity='client.admin' cmd=[{"prefix": "fs fail", "fs_name": "cephfs"}]: dispatch 2024-09-13T00:51:43.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:43 smithi181 bash[67587]: audit 2024-09-13T00:51:43.059421+0000 mon.a (mon.0) 4431 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "fs fail", "fs_name": "cephfs"}]: dispatch 2024-09-13T00:51:43.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:43 smithi181 bash[67587]: cluster 2024-09-13T00:51:43.060381+0000 mon.a (mon.0) 4432 : cluster [WRN] Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-13T00:51:43.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:43 smithi181 bash[67587]: cluster 2024-09-13T00:51:43.060407+0000 mon.a (mon.0) 4433 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T00:51:43.350 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:43 smithi181 bash[67587]: cluster 2024-09-13T00:51:43.064982+0000 mon.a (mon.0) 4434 : cluster [DBG] osdmap e2615: 8 total, 8 up, 8 in 2024-09-13T00:51:43.510 INFO:journalctl@ceph.mds.a.smithi136.stdout:Sep 13 00:51:43 smithi136 bash[97471]: ignoring --setuser ceph since I am not root 2024-09-13T00:51:43.510 INFO:journalctl@ceph.mds.a.smithi136.stdout:Sep 13 00:51:43 smithi136 bash[97471]: ignoring --setgroup ceph since I am not root 2024-09-13T00:51:43.511 INFO:journalctl@ceph.mds.a.smithi136.stdout:Sep 13 00:51:43 smithi136 bash[97471]: starting mds.a at 2024-09-13T00:51:43.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:43 smithi136 bash[76371]: audit 2024-09-13T00:51:42.385235+0000 mon.a (mon.0) 4430 : audit [DBG] from='client.? 172.21.15.136:0/912672868' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-13T00:51:43.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:43 smithi136 bash[76371]: audit 2024-09-13T00:51:42.707247+0000 mon.c (mon.2) 161 : audit [DBG] from='client.? 172.21.15.136:0/2639364842' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T00:51:43.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:43 smithi136 bash[76371]: audit 2024-09-13T00:51:43.058643+0000 mon.c (mon.2) 162 : audit [INF] from='client.? 172.21.15.136:0/1914867600' entity='client.admin' cmd=[{"prefix": "fs fail", "fs_name": "cephfs"}]: dispatch 2024-09-13T00:51:43.512 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:43 smithi136 bash[76371]: audit 2024-09-13T00:51:43.059421+0000 mon.a (mon.0) 4431 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "fs fail", "fs_name": "cephfs"}]: dispatch 2024-09-13T00:51:43.512 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:43 smithi136 bash[76371]: cluster 2024-09-13T00:51:43.060381+0000 mon.a (mon.0) 4432 : cluster [WRN] Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-13T00:51:43.512 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:43 smithi136 bash[76371]: cluster 2024-09-13T00:51:43.060407+0000 mon.a (mon.0) 4433 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T00:51:43.512 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:43 smithi136 bash[76371]: cluster 2024-09-13T00:51:43.064982+0000 mon.a (mon.0) 4434 : cluster [DBG] osdmap e2615: 8 total, 8 up, 8 in 2024-09-13T00:51:43.512 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:43 smithi136 bash[75164]: debug 2024-09-13T00:51:43.055+0000 7f6a844f2640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T00:51:43.512 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:43 smithi136 bash[75164]: audit 2024-09-13T00:51:42.385235+0000 mon.a (mon.0) 4430 : audit [DBG] from='client.? 172.21.15.136:0/912672868' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-13T00:51:43.513 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:43 smithi136 bash[75164]: audit 2024-09-13T00:51:42.707247+0000 mon.c (mon.2) 161 : audit [DBG] from='client.? 172.21.15.136:0/2639364842' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T00:51:43.513 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:43 smithi136 bash[75164]: audit 2024-09-13T00:51:43.058643+0000 mon.c (mon.2) 162 : audit [INF] from='client.? 172.21.15.136:0/1914867600' entity='client.admin' cmd=[{"prefix": "fs fail", "fs_name": "cephfs"}]: dispatch 2024-09-13T00:51:43.513 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:43 smithi136 bash[75164]: audit 2024-09-13T00:51:43.059421+0000 mon.a (mon.0) 4431 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "fs fail", "fs_name": "cephfs"}]: dispatch 2024-09-13T00:51:43.513 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:43 smithi136 bash[75164]: cluster 2024-09-13T00:51:43.060381+0000 mon.a (mon.0) 4432 : cluster [WRN] Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-13T00:51:43.513 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:43 smithi136 bash[75164]: cluster 2024-09-13T00:51:43.060407+0000 mon.a (mon.0) 4433 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T00:51:43.513 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:43 smithi136 bash[75164]: cluster 2024-09-13T00:51:43.064982+0000 mon.a (mon.0) 4434 : cluster [DBG] osdmap e2615: 8 total, 8 up, 8 in 2024-09-13T00:51:44.122 DEBUG:teuthology.orchestra.run.smithi136:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool rm cephfs_metadata cephfs_metadata --yes-i-really-really-mean-it 2024-09-13T00:51:44.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:44 smithi181 bash[67587]: cluster 2024-09-13T00:51:42.735390+0000 mgr.y (mgr.24661) 2237 : cluster [DBG] pgmap v4412: 105 pgs: 105 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:51:44.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:44 smithi181 bash[67587]: audit 2024-09-13T00:51:43.073447+0000 mon.a (mon.0) 4435 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "fs fail", "fs_name": "cephfs"}]': finished 2024-09-13T00:51:44.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:44 smithi181 bash[67587]: cluster 2024-09-13T00:51:43.073520+0000 mon.a (mon.0) 4436 : cluster [DBG] fsmap cephfs:0/1 1 up:standby, 1 failed 2024-09-13T00:51:44.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:44 smithi181 bash[67587]: audit 2024-09-13T00:51:43.431440+0000 mon.a (mon.0) 4437 : audit [INF] from='client.? 172.21.15.136:0/2996147228' entity='client.admin' cmd=[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:51:44.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:44 smithi136 bash[75164]: cluster 2024-09-13T00:51:42.735390+0000 mgr.y (mgr.24661) 2237 : cluster [DBG] pgmap v4412: 105 pgs: 105 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:51:44.510 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:44 smithi136 bash[75164]: audit 2024-09-13T00:51:43.073447+0000 mon.a (mon.0) 4435 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "fs fail", "fs_name": "cephfs"}]': finished 2024-09-13T00:51:44.511 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:44 smithi136 bash[75164]: cluster 2024-09-13T00:51:43.073520+0000 mon.a (mon.0) 4436 : cluster [DBG] fsmap cephfs:0/1 1 up:standby, 1 failed 2024-09-13T00:51:44.511 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:44 smithi136 bash[75164]: audit 2024-09-13T00:51:43.431440+0000 mon.a (mon.0) 4437 : audit [INF] from='client.? 172.21.15.136:0/2996147228' entity='client.admin' cmd=[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:51:44.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:44 smithi136 bash[76371]: cluster 2024-09-13T00:51:42.735390+0000 mgr.y (mgr.24661) 2237 : cluster [DBG] pgmap v4412: 105 pgs: 105 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:51:44.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:44 smithi136 bash[76371]: audit 2024-09-13T00:51:43.073447+0000 mon.a (mon.0) 4435 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "fs fail", "fs_name": "cephfs"}]': finished 2024-09-13T00:51:44.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:44 smithi136 bash[76371]: cluster 2024-09-13T00:51:43.073520+0000 mon.a (mon.0) 4436 : cluster [DBG] fsmap cephfs:0/1 1 up:standby, 1 failed 2024-09-13T00:51:44.511 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:44 smithi136 bash[76371]: audit 2024-09-13T00:51:43.431440+0000 mon.a (mon.0) 4437 : audit [INF] from='client.? 172.21.15.136:0/2996147228' entity='client.admin' cmd=[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:51:45.107 INFO:teuthology.orchestra.run.smithi136.stderr:pool 'cephfs_metadata' removed 2024-09-13T00:51:45.135 DEBUG:teuthology.orchestra.run.smithi136:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool rm cephfs_data cephfs_data --yes-i-really-really-mean-it 2024-09-13T00:51:45.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:45 smithi181 bash[67587]: cluster 2024-09-13T00:51:44.082723+0000 mon.a (mon.0) 4438 : cluster [INF] Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-13T00:51:45.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:45 smithi181 bash[67587]: cluster 2024-09-13T00:51:44.082772+0000 mon.a (mon.0) 4439 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-13T00:51:45.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:45 smithi181 bash[67587]: cluster 2024-09-13T00:51:44.082801+0000 mon.a (mon.0) 4440 : cluster [INF] Cluster is now healthy 2024-09-13T00:51:45.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:45 smithi181 bash[67587]: cluster 2024-09-13T00:51:44.096596+0000 mon.a (mon.0) 4441 : cluster [DBG] mds.? [v2:172.21.15.136:6834/2484689419,v1:172.21.15.136:6835/2484689419] up:boot 2024-09-13T00:51:45.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:45 smithi181 bash[67587]: audit 2024-09-13T00:51:44.096730+0000 mon.a (mon.0) 4442 : audit [INF] from='client.? 172.21.15.136:0/2996147228' entity='client.admin' cmd='[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]': finished 2024-09-13T00:51:45.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:45 smithi181 bash[67587]: cluster 2024-09-13T00:51:44.096916+0000 mon.a (mon.0) 4443 : cluster [DBG] fsmap 2 up:standby 2024-09-13T00:51:45.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:45 smithi181 bash[67587]: audit 2024-09-13T00:51:44.097180+0000 mon.a (mon.0) 4444 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-13T00:51:45.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:45 smithi181 bash[67587]: audit 2024-09-13T00:51:44.421126+0000 mon.a (mon.0) 4445 : audit [INF] from='client.? 172.21.15.136:0/996570163' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]: dispatch 2024-09-13T00:51:45.511 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:45 smithi136 bash[75164]: cluster 2024-09-13T00:51:44.082723+0000 mon.a (mon.0) 4438 : cluster [INF] Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-13T00:51:45.512 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:45 smithi136 bash[75164]: cluster 2024-09-13T00:51:44.082772+0000 mon.a (mon.0) 4439 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-13T00:51:45.512 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:45 smithi136 bash[75164]: cluster 2024-09-13T00:51:44.082801+0000 mon.a (mon.0) 4440 : cluster [INF] Cluster is now healthy 2024-09-13T00:51:45.512 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:45 smithi136 bash[75164]: cluster 2024-09-13T00:51:44.096596+0000 mon.a (mon.0) 4441 : cluster [DBG] mds.? [v2:172.21.15.136:6834/2484689419,v1:172.21.15.136:6835/2484689419] up:boot 2024-09-13T00:51:45.512 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:45 smithi136 bash[75164]: audit 2024-09-13T00:51:44.096730+0000 mon.a (mon.0) 4442 : audit [INF] from='client.? 172.21.15.136:0/2996147228' entity='client.admin' cmd='[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]': finished 2024-09-13T00:51:45.512 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:45 smithi136 bash[75164]: cluster 2024-09-13T00:51:44.096916+0000 mon.a (mon.0) 4443 : cluster [DBG] fsmap 2 up:standby 2024-09-13T00:51:45.512 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:45 smithi136 bash[75164]: audit 2024-09-13T00:51:44.097180+0000 mon.a (mon.0) 4444 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-13T00:51:45.512 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:45 smithi136 bash[75164]: audit 2024-09-13T00:51:44.421126+0000 mon.a (mon.0) 4445 : audit [INF] from='client.? 172.21.15.136:0/996570163' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]: dispatch 2024-09-13T00:51:45.513 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:45 smithi136 bash[76371]: cluster 2024-09-13T00:51:44.082723+0000 mon.a (mon.0) 4438 : cluster [INF] Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-13T00:51:45.513 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:45 smithi136 bash[76371]: cluster 2024-09-13T00:51:44.082772+0000 mon.a (mon.0) 4439 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-13T00:51:45.513 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:45 smithi136 bash[76371]: cluster 2024-09-13T00:51:44.082801+0000 mon.a (mon.0) 4440 : cluster [INF] Cluster is now healthy 2024-09-13T00:51:45.513 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:45 smithi136 bash[76371]: cluster 2024-09-13T00:51:44.096596+0000 mon.a (mon.0) 4441 : cluster [DBG] mds.? [v2:172.21.15.136:6834/2484689419,v1:172.21.15.136:6835/2484689419] up:boot 2024-09-13T00:51:45.513 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:45 smithi136 bash[76371]: audit 2024-09-13T00:51:44.096730+0000 mon.a (mon.0) 4442 : audit [INF] from='client.? 172.21.15.136:0/2996147228' entity='client.admin' cmd='[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]': finished 2024-09-13T00:51:45.513 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:45 smithi136 bash[76371]: cluster 2024-09-13T00:51:44.096916+0000 mon.a (mon.0) 4443 : cluster [DBG] fsmap 2 up:standby 2024-09-13T00:51:45.514 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:45 smithi136 bash[76371]: audit 2024-09-13T00:51:44.097180+0000 mon.a (mon.0) 4444 : audit [DBG] from='mgr.24661 172.21.15.136:0/418574901' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-13T00:51:45.514 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:45 smithi136 bash[76371]: audit 2024-09-13T00:51:44.421126+0000 mon.a (mon.0) 4445 : audit [INF] from='client.? 172.21.15.136:0/996570163' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]: dispatch 2024-09-13T00:51:46.120 INFO:teuthology.orchestra.run.smithi136.stderr:pool 'cephfs_data' removed 2024-09-13T00:51:46.150 DEBUG:teuthology.orchestra.run.smithi136:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-13T00:51:46.160 DEBUG:teuthology.orchestra.run.smithi181:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-13T00:51:46.174 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-09-13T00:51:46.174 DEBUG:teuthology.orchestra.run.smithi136:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-13T00:51:46.209 DEBUG:teuthology.orchestra.run.smithi181:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-13T00:51:46.223 INFO:tasks.cephadm:Stopping all daemons... 2024-09-13T00:51:46.223 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-09-13T00:51:46.223 DEBUG:teuthology.orchestra.run.smithi136:> sudo systemctl stop ceph-da58a47c-715d-11ef-bceb-c7b262605968@mon.a 2024-09-13T00:51:46.373 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:46 smithi136 bash[76371]: cluster 2024-09-13T00:51:44.736181+0000 mgr.y (mgr.24661) 2238 : cluster [DBG] pgmap v4414: 105 pgs: 105 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:51:46.373 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:46 smithi136 bash[76371]: audit 2024-09-13T00:51:45.107243+0000 mon.a (mon.0) 4446 : audit [INF] from='client.? 172.21.15.136:0/996570163' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T00:51:46.373 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:46 smithi136 bash[76371]: cluster 2024-09-13T00:51:45.107407+0000 mon.a (mon.0) 4447 : cluster [DBG] osdmap e2616: 8 total, 8 up, 8 in 2024-09-13T00:51:46.373 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:46 smithi136 bash[76371]: audit 2024-09-13T00:51:45.428858+0000 mon.a (mon.0) 4448 : audit [INF] from='client.? 172.21.15.136:0/4236858191' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]: dispatch 2024-09-13T00:51:46.374 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:46 smithi136 bash[75164]: cluster 2024-09-13T00:51:44.736181+0000 mgr.y (mgr.24661) 2238 : cluster [DBG] pgmap v4414: 105 pgs: 105 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:51:46.374 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:46 smithi136 bash[75164]: audit 2024-09-13T00:51:45.107243+0000 mon.a (mon.0) 4446 : audit [INF] from='client.? 172.21.15.136:0/996570163' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T00:51:46.374 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:46 smithi136 bash[75164]: cluster 2024-09-13T00:51:45.107407+0000 mon.a (mon.0) 4447 : cluster [DBG] osdmap e2616: 8 total, 8 up, 8 in 2024-09-13T00:51:46.374 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:46 smithi136 bash[75164]: audit 2024-09-13T00:51:45.428858+0000 mon.a (mon.0) 4448 : audit [INF] from='client.? 172.21.15.136:0/4236858191' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]: dispatch 2024-09-13T00:51:46.374 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:46 smithi136 systemd[1]: Stopping Ceph mon.a for da58a47c-715d-11ef-bceb-c7b262605968... 2024-09-13T00:51:46.374 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:46 smithi136 bash[75164]: debug 2024-09-13T00:51:46.371+0000 7f6a89cfd640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-09-13T00:51:46.428 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:46 smithi136 bash[75164]: debug 2024-09-13T00:51:46.371+0000 7f6a89cfd640 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-09-13T00:51:46.428 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:51:46 smithi136 bash[68787]: [13/Sep/2024:00:51:46] ENGINE Bus STOPPING 2024-09-13T00:51:46.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:46 smithi181 bash[67587]: cluster 2024-09-13T00:51:44.736181+0000 mgr.y (mgr.24661) 2238 : cluster [DBG] pgmap v4414: 105 pgs: 105 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:51:46.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:46 smithi181 bash[67587]: audit 2024-09-13T00:51:45.107243+0000 mon.a (mon.0) 4446 : audit [INF] from='client.? 172.21.15.136:0/996570163' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T00:51:46.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:46 smithi181 bash[67587]: cluster 2024-09-13T00:51:45.107407+0000 mon.a (mon.0) 4447 : cluster [DBG] osdmap e2616: 8 total, 8 up, 8 in 2024-09-13T00:51:46.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:46 smithi181 bash[67587]: audit 2024-09-13T00:51:45.428858+0000 mon.a (mon.0) 4448 : audit [INF] from='client.? 172.21.15.136:0/4236858191' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]: dispatch 2024-09-13T00:51:46.735 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:51:46 smithi136 bash[68787]: [13/Sep/2024:00:51:46] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T00:51:46.735 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:51:46 smithi136 bash[68787]: [13/Sep/2024:00:51:46] ENGINE Bus STOPPED 2024-09-13T00:51:46.735 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:51:46 smithi136 bash[68787]: [13/Sep/2024:00:51:46] ENGINE Bus STARTING 2024-09-13T00:51:46.735 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:51:46 smithi136 bash[68787]: [13/Sep/2024:00:51:46] ENGINE Serving on http://:::9283 2024-09-13T00:51:46.736 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:51:46 smithi136 bash[68787]: [13/Sep/2024:00:51:46] ENGINE Bus STARTED 2024-09-13T00:51:48.187 INFO:journalctl@ceph.mon.a.smithi136.stdout:Sep 13 00:51:47 smithi136 bash[119621]: ceph-da58a47c-715d-11ef-bceb-c7b262605968-mon-a 2024-09-13T00:51:48.192 DEBUG:teuthology.orchestra.run.smithi136:> sudo pkill -f 'journalctl -f -n 0 -u ceph-da58a47c-715d-11ef-bceb-c7b262605968@mon.a.service' 2024-09-13T00:51:48.216 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T00:51:48.217 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-09-13T00:51:48.217 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-09-13T00:51:48.217 DEBUG:teuthology.orchestra.run.smithi136:> sudo systemctl stop ceph-da58a47c-715d-11ef-bceb-c7b262605968@mon.c 2024-09-13T00:51:48.495 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:51:48 smithi136 bash[68787]: [13/Sep/2024:00:51:48] ENGINE Bus STOPPING 2024-09-13T00:51:48.495 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:48 smithi136 systemd[1]: Stopping Ceph mon.c for da58a47c-715d-11ef-bceb-c7b262605968... 2024-09-13T00:51:48.495 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:48 smithi136 bash[76371]: debug 2024-09-13T00:51:48.391+0000 7f502ce8d640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-09-13T00:51:48.495 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:48 smithi136 bash[76371]: debug 2024-09-13T00:51:48.391+0000 7f502ce8d640 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2024-09-13T00:51:48.753 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:51:48 smithi136 bash[68787]: [13/Sep/2024:00:51:48] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T00:51:48.754 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:51:48 smithi136 bash[68787]: [13/Sep/2024:00:51:48] ENGINE Bus STOPPED 2024-09-13T00:51:48.754 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:51:48 smithi136 bash[68787]: [13/Sep/2024:00:51:48] ENGINE Bus STARTING 2024-09-13T00:51:48.754 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:51:48 smithi136 bash[68787]: [13/Sep/2024:00:51:48] ENGINE Serving on http://:::9283 2024-09-13T00:51:48.754 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:51:48 smithi136 bash[68787]: [13/Sep/2024:00:51:48] ENGINE Bus STARTED 2024-09-13T00:51:49.412 DEBUG:teuthology.orchestra.run.smithi136:> sudo pkill -f 'journalctl -f -n 0 -u ceph-da58a47c-715d-11ef-bceb-c7b262605968@mon.c.service' 2024-09-13T00:51:49.420 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:51:49 smithi136 bash[68787]: ::ffff:172.21.15.181 - - [13/Sep/2024:00:51:49] "GET /metrics HTTP/1.1" 200 36744 "" "Prometheus/2.43.0" 2024-09-13T00:51:49.420 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:49 smithi136 bash[119763]: ceph-da58a47c-715d-11ef-bceb-c7b262605968-mon-c 2024-09-13T00:51:49.421 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:49 smithi136 bash[119837]: Error response from daemon: No such container: ceph-da58a47c-715d-11ef-bceb-c7b262605968-mon-c 2024-09-13T00:51:49.421 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:49 smithi136 systemd[1]: ceph-da58a47c-715d-11ef-bceb-c7b262605968@mon.c.service: Deactivated successfully. 2024-09-13T00:51:49.421 INFO:journalctl@ceph.mon.c.smithi136.stdout:Sep 13 00:51:49 smithi136 systemd[1]: Stopped Ceph mon.c for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-13T00:51:49.435 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T00:51:49.435 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-09-13T00:51:49.436 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-09-13T00:51:49.436 DEBUG:teuthology.orchestra.run.smithi181:> sudo systemctl stop ceph-da58a47c-715d-11ef-bceb-c7b262605968@mon.b 2024-09-13T00:51:49.733 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:49 smithi181 systemd[1]: Stopping Ceph mon.b for da58a47c-715d-11ef-bceb-c7b262605968... 2024-09-13T00:51:49.734 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:49 smithi181 bash[67587]: debug 2024-09-13T00:51:49.592+0000 7fba258ff640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-09-13T00:51:49.734 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:49 smithi181 bash[67587]: debug 2024-09-13T00:51:49.592+0000 7fba258ff640 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2024-09-13T00:51:50.629 DEBUG:teuthology.orchestra.run.smithi181:> sudo pkill -f 'journalctl -f -n 0 -u ceph-da58a47c-715d-11ef-bceb-c7b262605968@mon.b.service' 2024-09-13T00:51:50.638 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:50 smithi181 bash[109935]: ceph-da58a47c-715d-11ef-bceb-c7b262605968-mon-b 2024-09-13T00:51:50.638 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:50 smithi181 bash[110000]: Error response from daemon: No such container: ceph-da58a47c-715d-11ef-bceb-c7b262605968-mon-b 2024-09-13T00:51:50.638 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:50 smithi181 systemd[1]: ceph-da58a47c-715d-11ef-bceb-c7b262605968@mon.b.service: Deactivated successfully. 2024-09-13T00:51:50.638 INFO:journalctl@ceph.mon.b.smithi181.stdout:Sep 13 00:51:50 smithi181 systemd[1]: Stopped Ceph mon.b for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-13T00:51:50.653 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T00:51:50.654 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-09-13T00:51:50.654 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-09-13T00:51:50.654 DEBUG:teuthology.orchestra.run.smithi136:> sudo systemctl stop ceph-da58a47c-715d-11ef-bceb-c7b262605968@mgr.y 2024-09-13T00:51:51.010 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:51:50 smithi136 systemd[1]: Stopping Ceph mgr.y for da58a47c-715d-11ef-bceb-c7b262605968... 2024-09-13T00:51:51.010 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:51:50 smithi136 bash[68787]: debug 2024-09-13T00:51:50.803+0000 7f5ea79e2640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-13T00:51:51.010 INFO:journalctl@ceph.mgr.y.smithi136.stdout:Sep 13 00:51:50 smithi136 bash[68787]: debug 2024-09-13T00:51:50.803+0000 7f5ea79e2640 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2024-09-13T00:51:51.656 DEBUG:teuthology.orchestra.run.smithi136:> sudo pkill -f 'journalctl -f -n 0 -u ceph-da58a47c-715d-11ef-bceb-c7b262605968@mgr.y.service' 2024-09-13T00:51:51.682 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T00:51:51.683 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-09-13T00:51:51.683 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-09-13T00:51:51.683 DEBUG:teuthology.orchestra.run.smithi181:> sudo systemctl stop ceph-da58a47c-715d-11ef-bceb-c7b262605968@mgr.x 2024-09-13T00:51:51.952 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Sep 13 00:51:51 smithi181 systemd[1]: Stopping Ceph mgr.x for da58a47c-715d-11ef-bceb-c7b262605968... 2024-09-13T00:51:52.539 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:51:52 smithi181 bash[95505]: ts=2024-09-13T00:51:52.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"da58a47c-715d-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:51:52.740 DEBUG:teuthology.orchestra.run.smithi181:> sudo pkill -f 'journalctl -f -n 0 -u ceph-da58a47c-715d-11ef-bceb-c7b262605968@mgr.x.service' 2024-09-13T00:51:52.762 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T00:51:52.762 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-09-13T00:51:52.763 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-09-13T00:51:52.763 DEBUG:teuthology.orchestra.run.smithi136:> sudo systemctl stop ceph-da58a47c-715d-11ef-bceb-c7b262605968@osd.0 2024-09-13T00:51:53.260 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 13 00:51:52 smithi136 systemd[1]: Stopping Ceph osd.0 for da58a47c-715d-11ef-bceb-c7b262605968... 2024-09-13T00:51:53.261 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 13 00:51:52 smithi136 bash[80792]: debug 2024-09-13T00:51:52.995+0000 7f046aab5640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-13T00:51:53.261 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 13 00:51:52 smithi136 bash[80792]: debug 2024-09-13T00:51:52.995+0000 7f046aab5640 -1 osd.0 2617 *** Got signal Terminated *** 2024-09-13T00:51:53.261 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 13 00:51:52 smithi136 bash[80792]: debug 2024-09-13T00:51:52.995+0000 7f046aab5640 -1 osd.0 2617 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T00:51:59.260 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 13 00:51:58 smithi136 bash[120006]: ceph-da58a47c-715d-11ef-bceb-c7b262605968-osd-0 2024-09-13T00:51:59.261 INFO:journalctl@ceph.osd.0.smithi136.stdout:Sep 13 00:51:59 smithi136 bash[122286]: Error response from daemon: No such container: ceph-da58a47c-715d-11ef-bceb-c7b262605968-osd-0 2024-09-13T00:52:01.095 DEBUG:teuthology.orchestra.run.smithi136:> sudo pkill -f 'journalctl -f -n 0 -u ceph-da58a47c-715d-11ef-bceb-c7b262605968@osd.0.service' 2024-09-13T00:52:01.123 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T00:52:01.124 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-09-13T00:52:01.124 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-09-13T00:52:01.124 DEBUG:teuthology.orchestra.run.smithi136:> sudo systemctl stop ceph-da58a47c-715d-11ef-bceb-c7b262605968@osd.1 2024-09-13T00:52:01.510 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 13 00:52:01 smithi136 systemd[1]: Stopping Ceph osd.1 for da58a47c-715d-11ef-bceb-c7b262605968... 2024-09-13T00:52:01.511 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 13 00:52:01 smithi136 bash[84852]: debug 2024-09-13T00:52:01.303+0000 7f8a233c0640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-13T00:52:01.511 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 13 00:52:01 smithi136 bash[84852]: debug 2024-09-13T00:52:01.303+0000 7f8a233c0640 -1 osd.1 2617 *** Got signal Terminated *** 2024-09-13T00:52:01.511 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 13 00:52:01 smithi136 bash[84852]: debug 2024-09-13T00:52:01.303+0000 7f8a233c0640 -1 osd.1 2617 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T00:52:07.826 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 13 00:52:07 smithi136 bash[122419]: ceph-da58a47c-715d-11ef-bceb-c7b262605968-osd-1 2024-09-13T00:52:07.826 INFO:journalctl@ceph.osd.1.smithi136.stdout:Sep 13 00:52:07 smithi136 bash[122488]: Error response from daemon: No such container: ceph-da58a47c-715d-11ef-bceb-c7b262605968-osd-1 2024-09-13T00:52:09.521 DEBUG:teuthology.orchestra.run.smithi136:> sudo pkill -f 'journalctl -f -n 0 -u ceph-da58a47c-715d-11ef-bceb-c7b262605968@osd.1.service' 2024-09-13T00:52:09.550 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T00:52:09.550 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-09-13T00:52:09.551 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-09-13T00:52:09.551 DEBUG:teuthology.orchestra.run.smithi136:> sudo systemctl stop ceph-da58a47c-715d-11ef-bceb-c7b262605968@osd.2 2024-09-13T00:52:10.010 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 13 00:52:09 smithi136 systemd[1]: Stopping Ceph osd.2 for da58a47c-715d-11ef-bceb-c7b262605968... 2024-09-13T00:52:10.010 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 13 00:52:09 smithi136 bash[89273]: debug 2024-09-13T00:52:09.746+0000 7faed52f9640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-13T00:52:10.011 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 13 00:52:09 smithi136 bash[89273]: debug 2024-09-13T00:52:09.746+0000 7faed52f9640 -1 osd.2 2617 *** Got signal Terminated *** 2024-09-13T00:52:10.011 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 13 00:52:09 smithi136 bash[89273]: debug 2024-09-13T00:52:09.746+0000 7faed52f9640 -1 osd.2 2617 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T00:52:16.260 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 13 00:52:15 smithi136 bash[122617]: ceph-da58a47c-715d-11ef-bceb-c7b262605968-osd-2 2024-09-13T00:52:16.260 INFO:journalctl@ceph.osd.2.smithi136.stdout:Sep 13 00:52:16 smithi136 bash[122679]: Error response from daemon: No such container: ceph-da58a47c-715d-11ef-bceb-c7b262605968-osd-2 2024-09-13T00:52:17.943 DEBUG:teuthology.orchestra.run.smithi136:> sudo pkill -f 'journalctl -f -n 0 -u ceph-da58a47c-715d-11ef-bceb-c7b262605968@osd.2.service' 2024-09-13T00:52:17.971 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T00:52:17.971 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-09-13T00:52:17.972 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-09-13T00:52:17.972 DEBUG:teuthology.orchestra.run.smithi136:> sudo systemctl stop ceph-da58a47c-715d-11ef-bceb-c7b262605968@osd.3 2024-09-13T00:52:18.260 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 13 00:52:18 smithi136 systemd[1]: Stopping Ceph osd.3 for da58a47c-715d-11ef-bceb-c7b262605968... 2024-09-13T00:52:18.261 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 13 00:52:18 smithi136 bash[93456]: debug 2024-09-13T00:52:18.146+0000 7f243df15640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-13T00:52:18.261 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 13 00:52:18 smithi136 bash[93456]: debug 2024-09-13T00:52:18.146+0000 7f243df15640 -1 osd.3 2617 *** Got signal Terminated *** 2024-09-13T00:52:18.261 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 13 00:52:18 smithi136 bash[93456]: debug 2024-09-13T00:52:18.146+0000 7f243df15640 -1 osd.3 2617 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T00:52:19.598 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:19 smithi181 bash[80534]: debug 2024-09-13T00:52:19.295+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:55.496698+0000 front 2024-09-13T00:51:55.496655+0000 (oldest deadline 2024-09-13T00:52:18.996611+0000) 2024-09-13T00:52:20.598 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:20 smithi181 bash[80534]: debug 2024-09-13T00:52:20.283+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:55.496698+0000 front 2024-09-13T00:51:55.496655+0000 (oldest deadline 2024-09-13T00:52:18.996611+0000) 2024-09-13T00:52:20.598 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:20 smithi181 bash[84920]: debug 2024-09-13T00:52:20.207+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:56.711218+0000 front 2024-09-13T00:51:56.710814+0000 (oldest deadline 2024-09-13T00:52:20.210630+0000) 2024-09-13T00:52:21.180 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 00:52:20 smithi181 bash[72192]: debug 2024-09-13T00:52:20.887+0000 7f7a3651f640 -1 osd.4 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:56.581301+0000 front 2024-09-13T00:51:56.581410+0000 (oldest deadline 2024-09-13T00:52:20.080941+0000) 2024-09-13T00:52:21.598 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:21 smithi181 bash[80534]: debug 2024-09-13T00:52:21.315+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:55.496698+0000 front 2024-09-13T00:51:55.496655+0000 (oldest deadline 2024-09-13T00:52:18.996611+0000) 2024-09-13T00:52:21.599 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:21 smithi181 bash[84920]: debug 2024-09-13T00:52:21.179+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:56.711218+0000 front 2024-09-13T00:51:56.710814+0000 (oldest deadline 2024-09-13T00:52:20.210630+0000) 2024-09-13T00:52:22.203 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 00:52:21 smithi181 bash[72192]: debug 2024-09-13T00:52:21.883+0000 7f7a3651f640 -1 osd.4 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:56.581301+0000 front 2024-09-13T00:51:56.581410+0000 (oldest deadline 2024-09-13T00:52:20.080941+0000) 2024-09-13T00:52:22.598 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:22 smithi181 bash[80534]: debug 2024-09-13T00:52:22.347+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:55.496698+0000 front 2024-09-13T00:51:55.496655+0000 (oldest deadline 2024-09-13T00:52:18.996611+0000) 2024-09-13T00:52:22.598 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:22 smithi181 bash[84920]: debug 2024-09-13T00:52:22.203+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:56.711218+0000 front 2024-09-13T00:51:56.710814+0000 (oldest deadline 2024-09-13T00:52:20.210630+0000) 2024-09-13T00:52:23.010 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 13 00:52:22 smithi136 bash[93456]: debug 2024-09-13T00:52:22.686+0000 7f2436521640 -1 osd.3 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:56.705693+0000 front 2024-09-13T00:51:56.705492+0000 (oldest deadline 2024-09-13T00:52:22.605554+0000) 2024-09-13T00:52:23.098 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 00:52:22 smithi181 bash[76307]: debug 2024-09-13T00:52:22.731+0000 7fbb26e65640 -1 osd.5 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:57.353663+0000 front 2024-09-13T00:51:57.353572+0000 (oldest deadline 2024-09-13T00:52:22.653641+0000) 2024-09-13T00:52:23.099 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 00:52:22 smithi181 bash[72192]: debug 2024-09-13T00:52:22.891+0000 7f7a3651f640 -1 osd.4 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:56.581301+0000 front 2024-09-13T00:51:56.581410+0000 (oldest deadline 2024-09-13T00:52:20.080941+0000) 2024-09-13T00:52:23.099 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:52:22 smithi181 bash[95505]: ts=2024-09-13T00:52:22.761Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.136:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.136:8765: connect: connection refused" 2024-09-13T00:52:23.100 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:52:22 smithi181 bash[95505]: ts=2024-09-13T00:52:22.764Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.136:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.136:8765: connect: connection refused" 2024-09-13T00:52:23.100 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:52:22 smithi181 bash[95505]: ts=2024-09-13T00:52:22.768Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.136:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.136:8765: connect: connection refused" 2024-09-13T00:52:23.100 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:52:22 smithi181 bash[95505]: ts=2024-09-13T00:52:22.771Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.136:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.136:8765: connect: connection refused" 2024-09-13T00:52:23.100 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:52:22 smithi181 bash[95505]: ts=2024-09-13T00:52:22.771Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.136:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.136:8765: connect: connection refused" 2024-09-13T00:52:23.598 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:23 smithi181 bash[80534]: debug 2024-09-13T00:52:23.383+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:55.496698+0000 front 2024-09-13T00:51:55.496655+0000 (oldest deadline 2024-09-13T00:52:18.996611+0000) 2024-09-13T00:52:23.598 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:23 smithi181 bash[84920]: debug 2024-09-13T00:52:23.171+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:56.711218+0000 front 2024-09-13T00:51:56.710814+0000 (oldest deadline 2024-09-13T00:52:20.210630+0000) 2024-09-13T00:52:24.098 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 00:52:23 smithi181 bash[72192]: debug 2024-09-13T00:52:23.915+0000 7f7a3651f640 -1 osd.4 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:56.581301+0000 front 2024-09-13T00:51:56.581410+0000 (oldest deadline 2024-09-13T00:52:20.080941+0000) 2024-09-13T00:52:24.099 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 00:52:23 smithi181 bash[76307]: debug 2024-09-13T00:52:23.691+0000 7fbb26e65640 -1 osd.5 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:57.353663+0000 front 2024-09-13T00:51:57.353572+0000 (oldest deadline 2024-09-13T00:52:22.653641+0000) 2024-09-13T00:52:24.510 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 13 00:52:24 smithi136 bash[122807]: ceph-da58a47c-715d-11ef-bceb-c7b262605968-osd-3 2024-09-13T00:52:24.510 INFO:journalctl@ceph.osd.3.smithi136.stdout:Sep 13 00:52:24 smithi136 bash[122868]: Error response from daemon: No such container: ceph-da58a47c-715d-11ef-bceb-c7b262605968-osd-3 2024-09-13T00:52:24.598 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:24 smithi181 bash[80534]: debug 2024-09-13T00:52:24.331+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:55.496698+0000 front 2024-09-13T00:51:55.496655+0000 (oldest deadline 2024-09-13T00:52:18.996611+0000) 2024-09-13T00:52:24.598 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:24 smithi181 bash[84920]: debug 2024-09-13T00:52:24.159+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:56.711218+0000 front 2024-09-13T00:51:56.710814+0000 (oldest deadline 2024-09-13T00:52:20.210630+0000) 2024-09-13T00:52:25.098 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 00:52:24 smithi181 bash[72192]: debug 2024-09-13T00:52:24.939+0000 7f7a3651f640 -1 osd.4 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:56.581301+0000 front 2024-09-13T00:51:56.581410+0000 (oldest deadline 2024-09-13T00:52:20.080941+0000) 2024-09-13T00:52:25.098 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 00:52:24 smithi181 bash[76307]: debug 2024-09-13T00:52:24.711+0000 7fbb26e65640 -1 osd.5 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:57.353663+0000 front 2024-09-13T00:51:57.353572+0000 (oldest deadline 2024-09-13T00:52:22.653641+0000) 2024-09-13T00:52:25.598 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:25 smithi181 bash[80534]: debug 2024-09-13T00:52:25.307+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:55.496698+0000 front 2024-09-13T00:51:55.496655+0000 (oldest deadline 2024-09-13T00:52:18.996611+0000) 2024-09-13T00:52:25.598 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:25 smithi181 bash[84920]: debug 2024-09-13T00:52:25.195+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:56.711218+0000 front 2024-09-13T00:51:56.710814+0000 (oldest deadline 2024-09-13T00:52:20.210630+0000) 2024-09-13T00:52:25.963 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 00:52:25 smithi181 bash[76307]: debug 2024-09-13T00:52:25.675+0000 7fbb26e65640 -1 osd.5 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:57.353663+0000 front 2024-09-13T00:51:57.353572+0000 (oldest deadline 2024-09-13T00:52:22.653641+0000) 2024-09-13T00:52:26.216 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 00:52:25 smithi181 bash[72192]: debug 2024-09-13T00:52:25.963+0000 7f7a3651f640 -1 osd.4 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:56.581301+0000 front 2024-09-13T00:51:56.581410+0000 (oldest deadline 2024-09-13T00:52:20.080941+0000) 2024-09-13T00:52:26.311 DEBUG:teuthology.orchestra.run.smithi136:> sudo pkill -f 'journalctl -f -n 0 -u ceph-da58a47c-715d-11ef-bceb-c7b262605968@osd.3.service' 2024-09-13T00:52:26.337 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T00:52:26.337 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-09-13T00:52:26.337 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-09-13T00:52:26.337 DEBUG:teuthology.orchestra.run.smithi181:> sudo systemctl stop ceph-da58a47c-715d-11ef-bceb-c7b262605968@osd.4 2024-09-13T00:52:26.493 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 00:52:26 smithi181 systemd[1]: Stopping Ceph osd.4 for da58a47c-715d-11ef-bceb-c7b262605968... 2024-09-13T00:52:26.493 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:26 smithi181 bash[80534]: debug 2024-09-13T00:52:26.343+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:55.496698+0000 front 2024-09-13T00:51:55.496655+0000 (oldest deadline 2024-09-13T00:52:18.996611+0000) 2024-09-13T00:52:26.494 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:26 smithi181 bash[84920]: debug 2024-09-13T00:52:26.215+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:56.711218+0000 front 2024-09-13T00:51:56.710814+0000 (oldest deadline 2024-09-13T00:52:20.210630+0000) 2024-09-13T00:52:26.631 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 00:52:26 smithi181 bash[72192]: debug 2024-09-13T00:52:26.491+0000 7f7a39f0b640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-13T00:52:26.631 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 00:52:26 smithi181 bash[72192]: debug 2024-09-13T00:52:26.491+0000 7f7a39f0b640 -1 osd.4 2617 *** Got signal Terminated *** 2024-09-13T00:52:26.631 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 00:52:26 smithi181 bash[72192]: debug 2024-09-13T00:52:26.491+0000 7f7a39f0b640 -1 osd.4 2617 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T00:52:26.848 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 00:52:26 smithi181 bash[76307]: debug 2024-09-13T00:52:26.627+0000 7fbb26e65640 -1 osd.5 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:57.353663+0000 front 2024-09-13T00:51:57.353572+0000 (oldest deadline 2024-09-13T00:52:22.653641+0000) 2024-09-13T00:52:27.348 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 00:52:26 smithi181 bash[72192]: debug 2024-09-13T00:52:26.983+0000 7f7a3651f640 -1 osd.4 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:56.581301+0000 front 2024-09-13T00:51:56.581410+0000 (oldest deadline 2024-09-13T00:52:20.080941+0000) 2024-09-13T00:52:27.349 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:27 smithi181 bash[84920]: debug 2024-09-13T00:52:27.223+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:56.711218+0000 front 2024-09-13T00:51:56.710814+0000 (oldest deadline 2024-09-13T00:52:20.210630+0000) 2024-09-13T00:52:27.349 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:27 smithi181 bash[84920]: debug 2024-09-13T00:52:27.223+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:01.912156+0000 front 2024-09-13T00:52:01.912324+0000 (oldest deadline 2024-09-13T00:52:27.212136+0000) 2024-09-13T00:52:27.625 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:27 smithi181 bash[80534]: debug 2024-09-13T00:52:27.371+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:55.496698+0000 front 2024-09-13T00:51:55.496655+0000 (oldest deadline 2024-09-13T00:52:18.996611+0000) 2024-09-13T00:52:27.957 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 00:52:27 smithi181 bash[76307]: debug 2024-09-13T00:52:27.623+0000 7fbb26e65640 -1 osd.5 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:57.353663+0000 front 2024-09-13T00:51:57.353572+0000 (oldest deadline 2024-09-13T00:52:22.653641+0000) 2024-09-13T00:52:28.259 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 00:52:27 smithi181 bash[72192]: debug 2024-09-13T00:52:27.955+0000 7f7a3651f640 -1 osd.4 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:56.581301+0000 front 2024-09-13T00:51:56.581410+0000 (oldest deadline 2024-09-13T00:52:20.080941+0000) 2024-09-13T00:52:28.598 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:28 smithi181 bash[80534]: debug 2024-09-13T00:52:28.399+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:55.496698+0000 front 2024-09-13T00:51:55.496655+0000 (oldest deadline 2024-09-13T00:52:18.996611+0000) 2024-09-13T00:52:28.599 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:28 smithi181 bash[84920]: debug 2024-09-13T00:52:28.259+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:56.711218+0000 front 2024-09-13T00:51:56.710814+0000 (oldest deadline 2024-09-13T00:52:20.210630+0000) 2024-09-13T00:52:28.599 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:28 smithi181 bash[84920]: debug 2024-09-13T00:52:28.259+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:01.912156+0000 front 2024-09-13T00:52:01.912324+0000 (oldest deadline 2024-09-13T00:52:27.212136+0000) 2024-09-13T00:52:28.932 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 00:52:28 smithi181 bash[76307]: debug 2024-09-13T00:52:28.603+0000 7fbb26e65640 -1 osd.5 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:57.353663+0000 front 2024-09-13T00:51:57.353572+0000 (oldest deadline 2024-09-13T00:52:22.653641+0000) 2024-09-13T00:52:29.262 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 00:52:28 smithi181 bash[72192]: debug 2024-09-13T00:52:28.931+0000 7f7a3651f640 -1 osd.4 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:56.581301+0000 front 2024-09-13T00:51:56.581410+0000 (oldest deadline 2024-09-13T00:52:20.080941+0000) 2024-09-13T00:52:29.598 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:29 smithi181 bash[80534]: debug 2024-09-13T00:52:29.383+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:55.496698+0000 front 2024-09-13T00:51:55.496655+0000 (oldest deadline 2024-09-13T00:52:18.996611+0000) 2024-09-13T00:52:29.598 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:29 smithi181 bash[84920]: debug 2024-09-13T00:52:29.259+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:56.711218+0000 front 2024-09-13T00:51:56.710814+0000 (oldest deadline 2024-09-13T00:52:20.210630+0000) 2024-09-13T00:52:29.599 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:29 smithi181 bash[84920]: debug 2024-09-13T00:52:29.259+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:01.912156+0000 front 2024-09-13T00:52:01.912324+0000 (oldest deadline 2024-09-13T00:52:27.212136+0000) 2024-09-13T00:52:29.974 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 00:52:29 smithi181 bash[76307]: debug 2024-09-13T00:52:29.615+0000 7fbb26e65640 -1 osd.5 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:57.353663+0000 front 2024-09-13T00:51:57.353572+0000 (oldest deadline 2024-09-13T00:52:22.653641+0000) 2024-09-13T00:52:30.300 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 00:52:29 smithi181 bash[72192]: debug 2024-09-13T00:52:29.971+0000 7f7a3651f640 -1 osd.4 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:56.581301+0000 front 2024-09-13T00:51:56.581410+0000 (oldest deadline 2024-09-13T00:52:20.080941+0000) 2024-09-13T00:52:30.598 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:30 smithi181 bash[80534]: debug 2024-09-13T00:52:30.343+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:55.496698+0000 front 2024-09-13T00:51:55.496655+0000 (oldest deadline 2024-09-13T00:52:18.996611+0000) 2024-09-13T00:52:30.598 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:30 smithi181 bash[80534]: debug 2024-09-13T00:52:30.343+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:05.899543+0000 front 2024-09-13T00:52:05.899798+0000 (oldest deadline 2024-09-13T00:52:29.399407+0000) 2024-09-13T00:52:30.599 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:30 smithi181 bash[84920]: debug 2024-09-13T00:52:30.299+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:56.711218+0000 front 2024-09-13T00:51:56.710814+0000 (oldest deadline 2024-09-13T00:52:20.210630+0000) 2024-09-13T00:52:30.599 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:30 smithi181 bash[84920]: debug 2024-09-13T00:52:30.299+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:01.912156+0000 front 2024-09-13T00:52:01.912324+0000 (oldest deadline 2024-09-13T00:52:27.212136+0000) 2024-09-13T00:52:31.022 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 00:52:30 smithi181 bash[76307]: debug 2024-09-13T00:52:30.607+0000 7fbb26e65640 -1 osd.5 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:57.353663+0000 front 2024-09-13T00:51:57.353572+0000 (oldest deadline 2024-09-13T00:52:22.653641+0000) 2024-09-13T00:52:31.277 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 00:52:31 smithi181 bash[72192]: debug 2024-09-13T00:52:31.019+0000 7f7a3651f640 -1 osd.4 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:56.581301+0000 front 2024-09-13T00:51:56.581410+0000 (oldest deadline 2024-09-13T00:52:20.080941+0000) 2024-09-13T00:52:31.277 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 00:52:31 smithi181 bash[72192]: debug 2024-09-13T00:52:31.019+0000 7f7a3651f640 -1 osd.4 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:05.382393+0000 front 2024-09-13T00:52:05.382217+0000 (oldest deadline 2024-09-13T00:52:30.583187+0000) 2024-09-13T00:52:31.598 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:31 smithi181 bash[80534]: debug 2024-09-13T00:52:31.359+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:55.496698+0000 front 2024-09-13T00:51:55.496655+0000 (oldest deadline 2024-09-13T00:52:18.996611+0000) 2024-09-13T00:52:31.598 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:31 smithi181 bash[80534]: debug 2024-09-13T00:52:31.359+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:05.899543+0000 front 2024-09-13T00:52:05.899798+0000 (oldest deadline 2024-09-13T00:52:29.399407+0000) 2024-09-13T00:52:31.599 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:31 smithi181 bash[84920]: debug 2024-09-13T00:52:31.275+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:56.711218+0000 front 2024-09-13T00:51:56.710814+0000 (oldest deadline 2024-09-13T00:52:20.210630+0000) 2024-09-13T00:52:31.599 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:31 smithi181 bash[84920]: debug 2024-09-13T00:52:31.275+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:01.912156+0000 front 2024-09-13T00:52:01.912324+0000 (oldest deadline 2024-09-13T00:52:27.212136+0000) 2024-09-13T00:52:32.098 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 00:52:31 smithi181 bash[76307]: debug 2024-09-13T00:52:31.619+0000 7fbb26e65640 -1 osd.5 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:57.353663+0000 front 2024-09-13T00:51:57.353572+0000 (oldest deadline 2024-09-13T00:52:22.653641+0000) 2024-09-13T00:52:32.098 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 00:52:31 smithi181 bash[76307]: debug 2024-09-13T00:52:31.619+0000 7fbb26e65640 -1 osd.5 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:02.654684+0000 front 2024-09-13T00:52:02.654437+0000 (oldest deadline 2024-09-13T00:52:31.455000+0000) 2024-09-13T00:52:32.598 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:32 smithi181 bash[80534]: debug 2024-09-13T00:52:32.311+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:55.496698+0000 front 2024-09-13T00:51:55.496655+0000 (oldest deadline 2024-09-13T00:52:18.996611+0000) 2024-09-13T00:52:32.598 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:32 smithi181 bash[80534]: debug 2024-09-13T00:52:32.311+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:05.899543+0000 front 2024-09-13T00:52:05.899798+0000 (oldest deadline 2024-09-13T00:52:29.399407+0000) 2024-09-13T00:52:32.599 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:32 smithi181 bash[84920]: debug 2024-09-13T00:52:32.279+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:56.711218+0000 front 2024-09-13T00:51:56.710814+0000 (oldest deadline 2024-09-13T00:52:20.210630+0000) 2024-09-13T00:52:32.599 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:32 smithi181 bash[84920]: debug 2024-09-13T00:52:32.279+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:01.912156+0000 front 2024-09-13T00:52:01.912324+0000 (oldest deadline 2024-09-13T00:52:27.212136+0000) 2024-09-13T00:52:32.975 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 00:52:32 smithi181 bash[110143]: ceph-da58a47c-715d-11ef-bceb-c7b262605968-osd-4 2024-09-13T00:52:32.975 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 00:52:32 smithi181 bash[110205]: Error response from daemon: No such container: ceph-da58a47c-715d-11ef-bceb-c7b262605968-osd-4 2024-09-13T00:52:32.976 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 00:52:32 smithi181 bash[76307]: debug 2024-09-13T00:52:32.611+0000 7fbb26e65640 -1 osd.5 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:57.353663+0000 front 2024-09-13T00:51:57.353572+0000 (oldest deadline 2024-09-13T00:52:22.653641+0000) 2024-09-13T00:52:32.976 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 00:52:32 smithi181 bash[76307]: debug 2024-09-13T00:52:32.611+0000 7fbb26e65640 -1 osd.5 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:02.654684+0000 front 2024-09-13T00:52:02.654437+0000 (oldest deadline 2024-09-13T00:52:31.455000+0000) 2024-09-13T00:52:33.597 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:33 smithi181 bash[80534]: debug 2024-09-13T00:52:33.267+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:55.496698+0000 front 2024-09-13T00:51:55.496655+0000 (oldest deadline 2024-09-13T00:52:18.996611+0000) 2024-09-13T00:52:33.598 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:33 smithi181 bash[80534]: debug 2024-09-13T00:52:33.267+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:05.899543+0000 front 2024-09-13T00:52:05.899798+0000 (oldest deadline 2024-09-13T00:52:29.399407+0000) 2024-09-13T00:52:33.598 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:33 smithi181 bash[84920]: debug 2024-09-13T00:52:33.323+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:56.711218+0000 front 2024-09-13T00:51:56.710814+0000 (oldest deadline 2024-09-13T00:52:20.210630+0000) 2024-09-13T00:52:33.598 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:33 smithi181 bash[84920]: debug 2024-09-13T00:52:33.323+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:01.912156+0000 front 2024-09-13T00:52:01.912324+0000 (oldest deadline 2024-09-13T00:52:27.212136+0000) 2024-09-13T00:52:33.848 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 00:52:33 smithi181 bash[76307]: debug 2024-09-13T00:52:33.595+0000 7fbb26e65640 -1 osd.5 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:57.353663+0000 front 2024-09-13T00:51:57.353572+0000 (oldest deadline 2024-09-13T00:52:22.653641+0000) 2024-09-13T00:52:33.848 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 00:52:33 smithi181 bash[76307]: debug 2024-09-13T00:52:33.595+0000 7fbb26e65640 -1 osd.5 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:02.654684+0000 front 2024-09-13T00:52:02.654437+0000 (oldest deadline 2024-09-13T00:52:31.455000+0000) 2024-09-13T00:52:34.348 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:34 smithi181 bash[80534]: debug 2024-09-13T00:52:34.235+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:55.496698+0000 front 2024-09-13T00:51:55.496655+0000 (oldest deadline 2024-09-13T00:52:18.996611+0000) 2024-09-13T00:52:34.348 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:34 smithi181 bash[80534]: debug 2024-09-13T00:52:34.235+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:05.899543+0000 front 2024-09-13T00:52:05.899798+0000 (oldest deadline 2024-09-13T00:52:29.399407+0000) 2024-09-13T00:52:34.349 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:34 smithi181 bash[84920]: debug 2024-09-13T00:52:34.275+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:56.711218+0000 front 2024-09-13T00:51:56.710814+0000 (oldest deadline 2024-09-13T00:52:20.210630+0000) 2024-09-13T00:52:34.349 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:34 smithi181 bash[84920]: debug 2024-09-13T00:52:34.275+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:01.912156+0000 front 2024-09-13T00:52:01.912324+0000 (oldest deadline 2024-09-13T00:52:27.212136+0000) 2024-09-13T00:52:34.848 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 00:52:34 smithi181 bash[76307]: debug 2024-09-13T00:52:34.575+0000 7fbb26e65640 -1 osd.5 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:57.353663+0000 front 2024-09-13T00:51:57.353572+0000 (oldest deadline 2024-09-13T00:52:22.653641+0000) 2024-09-13T00:52:34.849 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 00:52:34 smithi181 bash[76307]: debug 2024-09-13T00:52:34.575+0000 7fbb26e65640 -1 osd.5 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:02.654684+0000 front 2024-09-13T00:52:02.654437+0000 (oldest deadline 2024-09-13T00:52:31.455000+0000) 2024-09-13T00:52:35.037 DEBUG:teuthology.orchestra.run.smithi181:> sudo pkill -f 'journalctl -f -n 0 -u ceph-da58a47c-715d-11ef-bceb-c7b262605968@osd.4.service' 2024-09-13T00:52:35.064 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T00:52:35.065 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-09-13T00:52:35.065 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-09-13T00:52:35.065 DEBUG:teuthology.orchestra.run.smithi181:> sudo systemctl stop ceph-da58a47c-715d-11ef-bceb-c7b262605968@osd.5 2024-09-13T00:52:35.348 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:35 smithi181 bash[80534]: debug 2024-09-13T00:52:35.255+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:55.496698+0000 front 2024-09-13T00:51:55.496655+0000 (oldest deadline 2024-09-13T00:52:18.996611+0000) 2024-09-13T00:52:35.349 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:35 smithi181 bash[80534]: debug 2024-09-13T00:52:35.255+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:05.899543+0000 front 2024-09-13T00:52:05.899798+0000 (oldest deadline 2024-09-13T00:52:29.399407+0000) 2024-09-13T00:52:35.349 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:35 smithi181 bash[84920]: debug 2024-09-13T00:52:35.279+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:56.711218+0000 front 2024-09-13T00:51:56.710814+0000 (oldest deadline 2024-09-13T00:52:20.210630+0000) 2024-09-13T00:52:35.349 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:35 smithi181 bash[84920]: debug 2024-09-13T00:52:35.279+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:01.912156+0000 front 2024-09-13T00:52:01.912324+0000 (oldest deadline 2024-09-13T00:52:27.212136+0000) 2024-09-13T00:52:35.350 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 00:52:35 smithi181 systemd[1]: Stopping Ceph osd.5 for da58a47c-715d-11ef-bceb-c7b262605968... 2024-09-13T00:52:35.350 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 00:52:35 smithi181 bash[76307]: debug 2024-09-13T00:52:35.223+0000 7fbb2a851640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-13T00:52:35.350 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 00:52:35 smithi181 bash[76307]: debug 2024-09-13T00:52:35.223+0000 7fbb2a851640 -1 osd.5 2617 *** Got signal Terminated *** 2024-09-13T00:52:35.350 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 00:52:35 smithi181 bash[76307]: debug 2024-09-13T00:52:35.223+0000 7fbb2a851640 -1 osd.5 2617 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T00:52:36.098 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 00:52:35 smithi181 bash[76307]: debug 2024-09-13T00:52:35.603+0000 7fbb26e65640 -1 osd.5 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:57.353663+0000 front 2024-09-13T00:51:57.353572+0000 (oldest deadline 2024-09-13T00:52:22.653641+0000) 2024-09-13T00:52:36.098 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 00:52:35 smithi181 bash[76307]: debug 2024-09-13T00:52:35.603+0000 7fbb26e65640 -1 osd.5 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:02.654684+0000 front 2024-09-13T00:52:02.654437+0000 (oldest deadline 2024-09-13T00:52:31.455000+0000) 2024-09-13T00:52:36.556 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:36 smithi181 bash[80534]: debug 2024-09-13T00:52:36.287+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:55.496698+0000 front 2024-09-13T00:51:55.496655+0000 (oldest deadline 2024-09-13T00:52:18.996611+0000) 2024-09-13T00:52:36.557 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:36 smithi181 bash[80534]: debug 2024-09-13T00:52:36.287+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:05.899543+0000 front 2024-09-13T00:52:05.899798+0000 (oldest deadline 2024-09-13T00:52:29.399407+0000) 2024-09-13T00:52:36.557 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:36 smithi181 bash[80534]: debug 2024-09-13T00:52:36.287+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6822 osd.2 since back 2024-09-13T00:52:14.001454+0000 front 2024-09-13T00:52:14.001488+0000 (oldest deadline 2024-09-13T00:52:35.701317+0000) 2024-09-13T00:52:36.557 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:36 smithi181 bash[84920]: debug 2024-09-13T00:52:36.319+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:56.711218+0000 front 2024-09-13T00:51:56.710814+0000 (oldest deadline 2024-09-13T00:52:20.210630+0000) 2024-09-13T00:52:36.557 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:36 smithi181 bash[84920]: debug 2024-09-13T00:52:36.319+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:01.912156+0000 front 2024-09-13T00:52:01.912324+0000 (oldest deadline 2024-09-13T00:52:27.212136+0000) 2024-09-13T00:52:36.558 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:36 smithi181 bash[84920]: debug 2024-09-13T00:52:36.319+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6822 osd.2 since back 2024-09-13T00:52:12.513546+0000 front 2024-09-13T00:52:12.513849+0000 (oldest deadline 2024-09-13T00:52:36.013356+0000) 2024-09-13T00:52:36.848 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 00:52:36 smithi181 bash[76307]: debug 2024-09-13T00:52:36.555+0000 7fbb26e65640 -1 osd.5 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:57.353663+0000 front 2024-09-13T00:51:57.353572+0000 (oldest deadline 2024-09-13T00:52:22.653641+0000) 2024-09-13T00:52:36.848 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 00:52:36 smithi181 bash[76307]: debug 2024-09-13T00:52:36.555+0000 7fbb26e65640 -1 osd.5 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:02.654684+0000 front 2024-09-13T00:52:02.654437+0000 (oldest deadline 2024-09-13T00:52:31.455000+0000) 2024-09-13T00:52:37.531 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:37 smithi181 bash[80534]: debug 2024-09-13T00:52:37.247+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:55.496698+0000 front 2024-09-13T00:51:55.496655+0000 (oldest deadline 2024-09-13T00:52:18.996611+0000) 2024-09-13T00:52:37.531 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:37 smithi181 bash[80534]: debug 2024-09-13T00:52:37.247+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:05.899543+0000 front 2024-09-13T00:52:05.899798+0000 (oldest deadline 2024-09-13T00:52:29.399407+0000) 2024-09-13T00:52:37.531 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:37 smithi181 bash[80534]: debug 2024-09-13T00:52:37.247+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6822 osd.2 since back 2024-09-13T00:52:14.001454+0000 front 2024-09-13T00:52:14.001488+0000 (oldest deadline 2024-09-13T00:52:35.701317+0000) 2024-09-13T00:52:37.531 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:37 smithi181 bash[84920]: debug 2024-09-13T00:52:37.323+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:56.711218+0000 front 2024-09-13T00:51:56.710814+0000 (oldest deadline 2024-09-13T00:52:20.210630+0000) 2024-09-13T00:52:37.532 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:37 smithi181 bash[84920]: debug 2024-09-13T00:52:37.323+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:01.912156+0000 front 2024-09-13T00:52:01.912324+0000 (oldest deadline 2024-09-13T00:52:27.212136+0000) 2024-09-13T00:52:37.532 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:37 smithi181 bash[84920]: debug 2024-09-13T00:52:37.323+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6822 osd.2 since back 2024-09-13T00:52:12.513546+0000 front 2024-09-13T00:52:12.513849+0000 (oldest deadline 2024-09-13T00:52:36.013356+0000) 2024-09-13T00:52:37.848 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 00:52:37 smithi181 bash[76307]: debug 2024-09-13T00:52:37.531+0000 7fbb26e65640 -1 osd.5 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:57.353663+0000 front 2024-09-13T00:51:57.353572+0000 (oldest deadline 2024-09-13T00:52:22.653641+0000) 2024-09-13T00:52:37.848 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 00:52:37 smithi181 bash[76307]: debug 2024-09-13T00:52:37.531+0000 7fbb26e65640 -1 osd.5 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:02.654684+0000 front 2024-09-13T00:52:02.654437+0000 (oldest deadline 2024-09-13T00:52:31.455000+0000) 2024-09-13T00:52:37.849 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 00:52:37 smithi181 bash[76307]: debug 2024-09-13T00:52:37.531+0000 7fbb26e65640 -1 osd.5 2617 heartbeat_check: no reply from 172.21.15.136:6822 osd.2 since back 2024-09-13T00:52:11.455891+0000 front 2024-09-13T00:52:11.456006+0000 (oldest deadline 2024-09-13T00:52:37.355785+0000) 2024-09-13T00:52:38.554 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:38 smithi181 bash[80534]: debug 2024-09-13T00:52:38.259+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:55.496698+0000 front 2024-09-13T00:51:55.496655+0000 (oldest deadline 2024-09-13T00:52:18.996611+0000) 2024-09-13T00:52:38.554 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:38 smithi181 bash[80534]: debug 2024-09-13T00:52:38.259+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:05.899543+0000 front 2024-09-13T00:52:05.899798+0000 (oldest deadline 2024-09-13T00:52:29.399407+0000) 2024-09-13T00:52:38.554 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:38 smithi181 bash[80534]: debug 2024-09-13T00:52:38.259+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6822 osd.2 since back 2024-09-13T00:52:14.001454+0000 front 2024-09-13T00:52:14.001488+0000 (oldest deadline 2024-09-13T00:52:35.701317+0000) 2024-09-13T00:52:38.554 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:38 smithi181 bash[84920]: debug 2024-09-13T00:52:38.311+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:56.711218+0000 front 2024-09-13T00:51:56.710814+0000 (oldest deadline 2024-09-13T00:52:20.210630+0000) 2024-09-13T00:52:38.555 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:38 smithi181 bash[84920]: debug 2024-09-13T00:52:38.311+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:01.912156+0000 front 2024-09-13T00:52:01.912324+0000 (oldest deadline 2024-09-13T00:52:27.212136+0000) 2024-09-13T00:52:38.555 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:38 smithi181 bash[84920]: debug 2024-09-13T00:52:38.311+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6822 osd.2 since back 2024-09-13T00:52:12.513546+0000 front 2024-09-13T00:52:12.513849+0000 (oldest deadline 2024-09-13T00:52:36.013356+0000) 2024-09-13T00:52:38.848 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 00:52:38 smithi181 bash[76307]: debug 2024-09-13T00:52:38.551+0000 7fbb26e65640 -1 osd.5 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:57.353663+0000 front 2024-09-13T00:51:57.353572+0000 (oldest deadline 2024-09-13T00:52:22.653641+0000) 2024-09-13T00:52:38.848 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 00:52:38 smithi181 bash[76307]: debug 2024-09-13T00:52:38.551+0000 7fbb26e65640 -1 osd.5 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:02.654684+0000 front 2024-09-13T00:52:02.654437+0000 (oldest deadline 2024-09-13T00:52:31.455000+0000) 2024-09-13T00:52:38.848 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 00:52:38 smithi181 bash[76307]: debug 2024-09-13T00:52:38.551+0000 7fbb26e65640 -1 osd.5 2617 heartbeat_check: no reply from 172.21.15.136:6822 osd.2 since back 2024-09-13T00:52:11.455891+0000 front 2024-09-13T00:52:11.456006+0000 (oldest deadline 2024-09-13T00:52:37.355785+0000) 2024-09-13T00:52:39.537 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:39 smithi181 bash[80534]: debug 2024-09-13T00:52:39.211+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:55.496698+0000 front 2024-09-13T00:51:55.496655+0000 (oldest deadline 2024-09-13T00:52:18.996611+0000) 2024-09-13T00:52:39.538 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:39 smithi181 bash[80534]: debug 2024-09-13T00:52:39.211+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:05.899543+0000 front 2024-09-13T00:52:05.899798+0000 (oldest deadline 2024-09-13T00:52:29.399407+0000) 2024-09-13T00:52:39.538 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:39 smithi181 bash[80534]: debug 2024-09-13T00:52:39.211+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6822 osd.2 since back 2024-09-13T00:52:14.001454+0000 front 2024-09-13T00:52:14.001488+0000 (oldest deadline 2024-09-13T00:52:35.701317+0000) 2024-09-13T00:52:39.538 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:39 smithi181 bash[84920]: debug 2024-09-13T00:52:39.307+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:56.711218+0000 front 2024-09-13T00:51:56.710814+0000 (oldest deadline 2024-09-13T00:52:20.210630+0000) 2024-09-13T00:52:39.538 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:39 smithi181 bash[84920]: debug 2024-09-13T00:52:39.307+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:01.912156+0000 front 2024-09-13T00:52:01.912324+0000 (oldest deadline 2024-09-13T00:52:27.212136+0000) 2024-09-13T00:52:39.538 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:39 smithi181 bash[84920]: debug 2024-09-13T00:52:39.307+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6822 osd.2 since back 2024-09-13T00:52:12.513546+0000 front 2024-09-13T00:52:12.513849+0000 (oldest deadline 2024-09-13T00:52:36.013356+0000) 2024-09-13T00:52:39.848 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 00:52:39 smithi181 bash[76307]: debug 2024-09-13T00:52:39.535+0000 7fbb26e65640 -1 osd.5 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:57.353663+0000 front 2024-09-13T00:51:57.353572+0000 (oldest deadline 2024-09-13T00:52:22.653641+0000) 2024-09-13T00:52:39.848 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 00:52:39 smithi181 bash[76307]: debug 2024-09-13T00:52:39.535+0000 7fbb26e65640 -1 osd.5 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:02.654684+0000 front 2024-09-13T00:52:02.654437+0000 (oldest deadline 2024-09-13T00:52:31.455000+0000) 2024-09-13T00:52:39.848 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 00:52:39 smithi181 bash[76307]: debug 2024-09-13T00:52:39.535+0000 7fbb26e65640 -1 osd.5 2617 heartbeat_check: no reply from 172.21.15.136:6822 osd.2 since back 2024-09-13T00:52:11.455891+0000 front 2024-09-13T00:52:11.456006+0000 (oldest deadline 2024-09-13T00:52:37.355785+0000) 2024-09-13T00:52:40.598 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:40 smithi181 bash[80534]: debug 2024-09-13T00:52:40.247+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:55.496698+0000 front 2024-09-13T00:51:55.496655+0000 (oldest deadline 2024-09-13T00:52:18.996611+0000) 2024-09-13T00:52:40.598 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:40 smithi181 bash[80534]: debug 2024-09-13T00:52:40.247+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:05.899543+0000 front 2024-09-13T00:52:05.899798+0000 (oldest deadline 2024-09-13T00:52:29.399407+0000) 2024-09-13T00:52:40.599 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:40 smithi181 bash[80534]: debug 2024-09-13T00:52:40.247+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6822 osd.2 since back 2024-09-13T00:52:14.001454+0000 front 2024-09-13T00:52:14.001488+0000 (oldest deadline 2024-09-13T00:52:35.701317+0000) 2024-09-13T00:52:40.599 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:40 smithi181 bash[84920]: debug 2024-09-13T00:52:40.263+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:56.711218+0000 front 2024-09-13T00:51:56.710814+0000 (oldest deadline 2024-09-13T00:52:20.210630+0000) 2024-09-13T00:52:40.599 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:40 smithi181 bash[84920]: debug 2024-09-13T00:52:40.263+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:01.912156+0000 front 2024-09-13T00:52:01.912324+0000 (oldest deadline 2024-09-13T00:52:27.212136+0000) 2024-09-13T00:52:40.599 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:40 smithi181 bash[84920]: debug 2024-09-13T00:52:40.263+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6822 osd.2 since back 2024-09-13T00:52:12.513546+0000 front 2024-09-13T00:52:12.513849+0000 (oldest deadline 2024-09-13T00:52:36.013356+0000) 2024-09-13T00:52:41.585 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 00:52:41 smithi181 bash[110335]: ceph-da58a47c-715d-11ef-bceb-c7b262605968-osd-5 2024-09-13T00:52:41.585 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:41 smithi181 bash[84920]: debug 2024-09-13T00:52:41.291+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:56.711218+0000 front 2024-09-13T00:51:56.710814+0000 (oldest deadline 2024-09-13T00:52:20.210630+0000) 2024-09-13T00:52:41.585 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:41 smithi181 bash[84920]: debug 2024-09-13T00:52:41.291+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:01.912156+0000 front 2024-09-13T00:52:01.912324+0000 (oldest deadline 2024-09-13T00:52:27.212136+0000) 2024-09-13T00:52:41.586 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:41 smithi181 bash[84920]: debug 2024-09-13T00:52:41.291+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6822 osd.2 since back 2024-09-13T00:52:12.513546+0000 front 2024-09-13T00:52:12.513849+0000 (oldest deadline 2024-09-13T00:52:36.013356+0000) 2024-09-13T00:52:41.586 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:41 smithi181 bash[80534]: debug 2024-09-13T00:52:41.215+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:55.496698+0000 front 2024-09-13T00:51:55.496655+0000 (oldest deadline 2024-09-13T00:52:18.996611+0000) 2024-09-13T00:52:41.586 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:41 smithi181 bash[80534]: debug 2024-09-13T00:52:41.215+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:05.899543+0000 front 2024-09-13T00:52:05.899798+0000 (oldest deadline 2024-09-13T00:52:29.399407+0000) 2024-09-13T00:52:41.586 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:41 smithi181 bash[80534]: debug 2024-09-13T00:52:41.215+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6822 osd.2 since back 2024-09-13T00:52:14.001454+0000 front 2024-09-13T00:52:14.001488+0000 (oldest deadline 2024-09-13T00:52:35.701317+0000) 2024-09-13T00:52:41.848 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 00:52:41 smithi181 bash[110401]: Error response from daemon: No such container: ceph-da58a47c-715d-11ef-bceb-c7b262605968-osd-5 2024-09-13T00:52:42.598 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:42 smithi181 bash[80534]: debug 2024-09-13T00:52:42.175+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:55.496698+0000 front 2024-09-13T00:51:55.496655+0000 (oldest deadline 2024-09-13T00:52:18.996611+0000) 2024-09-13T00:52:42.598 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:42 smithi181 bash[80534]: debug 2024-09-13T00:52:42.175+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:05.899543+0000 front 2024-09-13T00:52:05.899798+0000 (oldest deadline 2024-09-13T00:52:29.399407+0000) 2024-09-13T00:52:42.599 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:42 smithi181 bash[80534]: debug 2024-09-13T00:52:42.175+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6822 osd.2 since back 2024-09-13T00:52:14.001454+0000 front 2024-09-13T00:52:14.001488+0000 (oldest deadline 2024-09-13T00:52:35.701317+0000) 2024-09-13T00:52:42.599 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:42 smithi181 bash[84920]: debug 2024-09-13T00:52:42.307+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:56.711218+0000 front 2024-09-13T00:51:56.710814+0000 (oldest deadline 2024-09-13T00:52:20.210630+0000) 2024-09-13T00:52:42.599 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:42 smithi181 bash[84920]: debug 2024-09-13T00:52:42.307+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:01.912156+0000 front 2024-09-13T00:52:01.912324+0000 (oldest deadline 2024-09-13T00:52:27.212136+0000) 2024-09-13T00:52:42.599 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:42 smithi181 bash[84920]: debug 2024-09-13T00:52:42.307+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6822 osd.2 since back 2024-09-13T00:52:12.513546+0000 front 2024-09-13T00:52:12.513849+0000 (oldest deadline 2024-09-13T00:52:36.013356+0000) 2024-09-13T00:52:43.313 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:43 smithi181 bash[80534]: debug 2024-09-13T00:52:43.131+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:55.496698+0000 front 2024-09-13T00:51:55.496655+0000 (oldest deadline 2024-09-13T00:52:18.996611+0000) 2024-09-13T00:52:43.313 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:43 smithi181 bash[80534]: debug 2024-09-13T00:52:43.131+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:05.899543+0000 front 2024-09-13T00:52:05.899798+0000 (oldest deadline 2024-09-13T00:52:29.399407+0000) 2024-09-13T00:52:43.313 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:43 smithi181 bash[80534]: debug 2024-09-13T00:52:43.131+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6822 osd.2 since back 2024-09-13T00:52:14.001454+0000 front 2024-09-13T00:52:14.001488+0000 (oldest deadline 2024-09-13T00:52:35.701317+0000) 2024-09-13T00:52:43.598 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:43 smithi181 bash[84920]: debug 2024-09-13T00:52:43.311+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:56.711218+0000 front 2024-09-13T00:51:56.710814+0000 (oldest deadline 2024-09-13T00:52:20.210630+0000) 2024-09-13T00:52:43.598 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:43 smithi181 bash[84920]: debug 2024-09-13T00:52:43.311+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:01.912156+0000 front 2024-09-13T00:52:01.912324+0000 (oldest deadline 2024-09-13T00:52:27.212136+0000) 2024-09-13T00:52:43.599 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:43 smithi181 bash[84920]: debug 2024-09-13T00:52:43.311+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6822 osd.2 since back 2024-09-13T00:52:12.513546+0000 front 2024-09-13T00:52:12.513849+0000 (oldest deadline 2024-09-13T00:52:36.013356+0000) 2024-09-13T00:52:43.656 DEBUG:teuthology.orchestra.run.smithi181:> sudo pkill -f 'journalctl -f -n 0 -u ceph-da58a47c-715d-11ef-bceb-c7b262605968@osd.5.service' 2024-09-13T00:52:43.684 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T00:52:43.685 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-09-13T00:52:43.685 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-09-13T00:52:43.685 DEBUG:teuthology.orchestra.run.smithi181:> sudo systemctl stop ceph-da58a47c-715d-11ef-bceb-c7b262605968@osd.6 2024-09-13T00:52:44.098 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:43 smithi181 systemd[1]: Stopping Ceph osd.6 for da58a47c-715d-11ef-bceb-c7b262605968... 2024-09-13T00:52:44.098 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:43 smithi181 bash[80534]: debug 2024-09-13T00:52:43.839+0000 7ff97ea4a640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-13T00:52:44.099 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:43 smithi181 bash[80534]: debug 2024-09-13T00:52:43.839+0000 7ff97ea4a640 -1 osd.6 2617 *** Got signal Terminated *** 2024-09-13T00:52:44.099 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:43 smithi181 bash[80534]: debug 2024-09-13T00:52:43.839+0000 7ff97ea4a640 -1 osd.6 2617 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T00:52:44.598 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:44 smithi181 bash[80534]: debug 2024-09-13T00:52:44.151+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:55.496698+0000 front 2024-09-13T00:51:55.496655+0000 (oldest deadline 2024-09-13T00:52:18.996611+0000) 2024-09-13T00:52:44.598 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:44 smithi181 bash[80534]: debug 2024-09-13T00:52:44.151+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:05.899543+0000 front 2024-09-13T00:52:05.899798+0000 (oldest deadline 2024-09-13T00:52:29.399407+0000) 2024-09-13T00:52:44.599 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:44 smithi181 bash[80534]: debug 2024-09-13T00:52:44.151+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6822 osd.2 since back 2024-09-13T00:52:14.001454+0000 front 2024-09-13T00:52:14.001488+0000 (oldest deadline 2024-09-13T00:52:35.701317+0000) 2024-09-13T00:52:44.599 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:44 smithi181 bash[80534]: debug 2024-09-13T00:52:44.151+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6830 osd.3 since back 2024-09-13T00:52:20.402881+0000 front 2024-09-13T00:52:20.402854+0000 (oldest deadline 2024-09-13T00:52:43.902703+0000) 2024-09-13T00:52:44.599 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:44 smithi181 bash[84920]: debug 2024-09-13T00:52:44.307+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:56.711218+0000 front 2024-09-13T00:51:56.710814+0000 (oldest deadline 2024-09-13T00:52:20.210630+0000) 2024-09-13T00:52:44.599 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:44 smithi181 bash[84920]: debug 2024-09-13T00:52:44.307+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:01.912156+0000 front 2024-09-13T00:52:01.912324+0000 (oldest deadline 2024-09-13T00:52:27.212136+0000) 2024-09-13T00:52:44.599 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:44 smithi181 bash[84920]: debug 2024-09-13T00:52:44.307+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6822 osd.2 since back 2024-09-13T00:52:12.513546+0000 front 2024-09-13T00:52:12.513849+0000 (oldest deadline 2024-09-13T00:52:36.013356+0000) 2024-09-13T00:52:45.598 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:45 smithi181 bash[80534]: debug 2024-09-13T00:52:45.115+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:55.496698+0000 front 2024-09-13T00:51:55.496655+0000 (oldest deadline 2024-09-13T00:52:18.996611+0000) 2024-09-13T00:52:45.598 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:45 smithi181 bash[80534]: debug 2024-09-13T00:52:45.115+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:05.899543+0000 front 2024-09-13T00:52:05.899798+0000 (oldest deadline 2024-09-13T00:52:29.399407+0000) 2024-09-13T00:52:45.599 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:45 smithi181 bash[80534]: debug 2024-09-13T00:52:45.115+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6822 osd.2 since back 2024-09-13T00:52:14.001454+0000 front 2024-09-13T00:52:14.001488+0000 (oldest deadline 2024-09-13T00:52:35.701317+0000) 2024-09-13T00:52:45.599 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:45 smithi181 bash[80534]: debug 2024-09-13T00:52:45.115+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6830 osd.3 since back 2024-09-13T00:52:20.402881+0000 front 2024-09-13T00:52:20.402854+0000 (oldest deadline 2024-09-13T00:52:43.902703+0000) 2024-09-13T00:52:45.599 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:45 smithi181 bash[84920]: debug 2024-09-13T00:52:45.347+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:56.711218+0000 front 2024-09-13T00:51:56.710814+0000 (oldest deadline 2024-09-13T00:52:20.210630+0000) 2024-09-13T00:52:45.599 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:45 smithi181 bash[84920]: debug 2024-09-13T00:52:45.347+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:01.912156+0000 front 2024-09-13T00:52:01.912324+0000 (oldest deadline 2024-09-13T00:52:27.212136+0000) 2024-09-13T00:52:45.599 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:45 smithi181 bash[84920]: debug 2024-09-13T00:52:45.347+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6822 osd.2 since back 2024-09-13T00:52:12.513546+0000 front 2024-09-13T00:52:12.513849+0000 (oldest deadline 2024-09-13T00:52:36.013356+0000) 2024-09-13T00:52:45.600 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:45 smithi181 bash[84920]: debug 2024-09-13T00:52:45.347+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6830 osd.3 since back 2024-09-13T00:52:23.015232+0000 front 2024-09-13T00:52:23.015273+0000 (oldest deadline 2024-09-13T00:52:44.714868+0000) 2024-09-13T00:52:46.598 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:46 smithi181 bash[80534]: debug 2024-09-13T00:52:46.127+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:55.496698+0000 front 2024-09-13T00:51:55.496655+0000 (oldest deadline 2024-09-13T00:52:18.996611+0000) 2024-09-13T00:52:46.598 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:46 smithi181 bash[80534]: debug 2024-09-13T00:52:46.127+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:05.899543+0000 front 2024-09-13T00:52:05.899798+0000 (oldest deadline 2024-09-13T00:52:29.399407+0000) 2024-09-13T00:52:46.599 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:46 smithi181 bash[80534]: debug 2024-09-13T00:52:46.127+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6822 osd.2 since back 2024-09-13T00:52:14.001454+0000 front 2024-09-13T00:52:14.001488+0000 (oldest deadline 2024-09-13T00:52:35.701317+0000) 2024-09-13T00:52:46.599 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:46 smithi181 bash[80534]: debug 2024-09-13T00:52:46.127+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6830 osd.3 since back 2024-09-13T00:52:20.402881+0000 front 2024-09-13T00:52:20.402854+0000 (oldest deadline 2024-09-13T00:52:43.902703+0000) 2024-09-13T00:52:46.599 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:46 smithi181 bash[84920]: debug 2024-09-13T00:52:46.327+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:56.711218+0000 front 2024-09-13T00:51:56.710814+0000 (oldest deadline 2024-09-13T00:52:20.210630+0000) 2024-09-13T00:52:46.599 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:46 smithi181 bash[84920]: debug 2024-09-13T00:52:46.327+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:01.912156+0000 front 2024-09-13T00:52:01.912324+0000 (oldest deadline 2024-09-13T00:52:27.212136+0000) 2024-09-13T00:52:46.599 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:46 smithi181 bash[84920]: debug 2024-09-13T00:52:46.327+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6822 osd.2 since back 2024-09-13T00:52:12.513546+0000 front 2024-09-13T00:52:12.513849+0000 (oldest deadline 2024-09-13T00:52:36.013356+0000) 2024-09-13T00:52:46.599 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:46 smithi181 bash[84920]: debug 2024-09-13T00:52:46.327+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6830 osd.3 since back 2024-09-13T00:52:23.015232+0000 front 2024-09-13T00:52:23.015273+0000 (oldest deadline 2024-09-13T00:52:44.714868+0000) 2024-09-13T00:52:47.598 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:47 smithi181 bash[80534]: debug 2024-09-13T00:52:47.115+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:55.496698+0000 front 2024-09-13T00:51:55.496655+0000 (oldest deadline 2024-09-13T00:52:18.996611+0000) 2024-09-13T00:52:47.599 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:47 smithi181 bash[80534]: debug 2024-09-13T00:52:47.115+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:05.899543+0000 front 2024-09-13T00:52:05.899798+0000 (oldest deadline 2024-09-13T00:52:29.399407+0000) 2024-09-13T00:52:47.599 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:47 smithi181 bash[80534]: debug 2024-09-13T00:52:47.115+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6822 osd.2 since back 2024-09-13T00:52:14.001454+0000 front 2024-09-13T00:52:14.001488+0000 (oldest deadline 2024-09-13T00:52:35.701317+0000) 2024-09-13T00:52:47.599 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:47 smithi181 bash[80534]: debug 2024-09-13T00:52:47.115+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6830 osd.3 since back 2024-09-13T00:52:20.402881+0000 front 2024-09-13T00:52:20.402854+0000 (oldest deadline 2024-09-13T00:52:43.902703+0000) 2024-09-13T00:52:47.600 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:47 smithi181 bash[84920]: debug 2024-09-13T00:52:47.335+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:56.711218+0000 front 2024-09-13T00:51:56.710814+0000 (oldest deadline 2024-09-13T00:52:20.210630+0000) 2024-09-13T00:52:47.600 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:47 smithi181 bash[84920]: debug 2024-09-13T00:52:47.335+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:01.912156+0000 front 2024-09-13T00:52:01.912324+0000 (oldest deadline 2024-09-13T00:52:27.212136+0000) 2024-09-13T00:52:47.600 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:47 smithi181 bash[84920]: debug 2024-09-13T00:52:47.335+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6822 osd.2 since back 2024-09-13T00:52:12.513546+0000 front 2024-09-13T00:52:12.513849+0000 (oldest deadline 2024-09-13T00:52:36.013356+0000) 2024-09-13T00:52:47.600 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:47 smithi181 bash[84920]: debug 2024-09-13T00:52:47.335+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6830 osd.3 since back 2024-09-13T00:52:23.015232+0000 front 2024-09-13T00:52:23.015273+0000 (oldest deadline 2024-09-13T00:52:44.714868+0000) 2024-09-13T00:52:48.348 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:48 smithi181 bash[80534]: debug 2024-09-13T00:52:48.083+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:55.496698+0000 front 2024-09-13T00:51:55.496655+0000 (oldest deadline 2024-09-13T00:52:18.996611+0000) 2024-09-13T00:52:48.348 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:48 smithi181 bash[80534]: debug 2024-09-13T00:52:48.083+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:05.899543+0000 front 2024-09-13T00:52:05.899798+0000 (oldest deadline 2024-09-13T00:52:29.399407+0000) 2024-09-13T00:52:48.349 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:48 smithi181 bash[80534]: debug 2024-09-13T00:52:48.083+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6822 osd.2 since back 2024-09-13T00:52:14.001454+0000 front 2024-09-13T00:52:14.001488+0000 (oldest deadline 2024-09-13T00:52:35.701317+0000) 2024-09-13T00:52:48.349 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:48 smithi181 bash[80534]: debug 2024-09-13T00:52:48.083+0000 7ff977056640 -1 osd.6 2617 heartbeat_check: no reply from 172.21.15.136:6830 osd.3 since back 2024-09-13T00:52:20.402881+0000 front 2024-09-13T00:52:20.402854+0000 (oldest deadline 2024-09-13T00:52:43.902703+0000) 2024-09-13T00:52:48.349 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:48 smithi181 bash[84920]: debug 2024-09-13T00:52:48.307+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:56.711218+0000 front 2024-09-13T00:51:56.710814+0000 (oldest deadline 2024-09-13T00:52:20.210630+0000) 2024-09-13T00:52:48.349 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:48 smithi181 bash[84920]: debug 2024-09-13T00:52:48.307+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:01.912156+0000 front 2024-09-13T00:52:01.912324+0000 (oldest deadline 2024-09-13T00:52:27.212136+0000) 2024-09-13T00:52:48.349 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:48 smithi181 bash[84920]: debug 2024-09-13T00:52:48.307+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6822 osd.2 since back 2024-09-13T00:52:12.513546+0000 front 2024-09-13T00:52:12.513849+0000 (oldest deadline 2024-09-13T00:52:36.013356+0000) 2024-09-13T00:52:48.350 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:48 smithi181 bash[84920]: debug 2024-09-13T00:52:48.307+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6830 osd.3 since back 2024-09-13T00:52:23.015232+0000 front 2024-09-13T00:52:23.015273+0000 (oldest deadline 2024-09-13T00:52:44.714868+0000) 2024-09-13T00:52:49.598 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:49 smithi181 bash[84920]: debug 2024-09-13T00:52:49.295+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:56.711218+0000 front 2024-09-13T00:51:56.710814+0000 (oldest deadline 2024-09-13T00:52:20.210630+0000) 2024-09-13T00:52:49.598 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:49 smithi181 bash[84920]: debug 2024-09-13T00:52:49.295+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:01.912156+0000 front 2024-09-13T00:52:01.912324+0000 (oldest deadline 2024-09-13T00:52:27.212136+0000) 2024-09-13T00:52:49.599 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:49 smithi181 bash[84920]: debug 2024-09-13T00:52:49.295+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6822 osd.2 since back 2024-09-13T00:52:12.513546+0000 front 2024-09-13T00:52:12.513849+0000 (oldest deadline 2024-09-13T00:52:36.013356+0000) 2024-09-13T00:52:49.599 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:49 smithi181 bash[84920]: debug 2024-09-13T00:52:49.295+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6830 osd.3 since back 2024-09-13T00:52:23.015232+0000 front 2024-09-13T00:52:23.015273+0000 (oldest deadline 2024-09-13T00:52:44.714868+0000) 2024-09-13T00:52:50.348 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:50 smithi181 bash[110534]: ceph-da58a47c-715d-11ef-bceb-c7b262605968-osd-6 2024-09-13T00:52:50.349 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 00:52:50 smithi181 bash[110595]: Error response from daemon: No such container: ceph-da58a47c-715d-11ef-bceb-c7b262605968-osd-6 2024-09-13T00:52:50.349 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:50 smithi181 bash[84920]: debug 2024-09-13T00:52:50.283+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:56.711218+0000 front 2024-09-13T00:51:56.710814+0000 (oldest deadline 2024-09-13T00:52:20.210630+0000) 2024-09-13T00:52:50.349 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:50 smithi181 bash[84920]: debug 2024-09-13T00:52:50.283+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:01.912156+0000 front 2024-09-13T00:52:01.912324+0000 (oldest deadline 2024-09-13T00:52:27.212136+0000) 2024-09-13T00:52:50.349 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:50 smithi181 bash[84920]: debug 2024-09-13T00:52:50.283+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6822 osd.2 since back 2024-09-13T00:52:12.513546+0000 front 2024-09-13T00:52:12.513849+0000 (oldest deadline 2024-09-13T00:52:36.013356+0000) 2024-09-13T00:52:50.349 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:50 smithi181 bash[84920]: debug 2024-09-13T00:52:50.283+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6830 osd.3 since back 2024-09-13T00:52:23.015232+0000 front 2024-09-13T00:52:23.015273+0000 (oldest deadline 2024-09-13T00:52:44.714868+0000) 2024-09-13T00:52:51.546 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:51 smithi181 bash[84920]: debug 2024-09-13T00:52:51.267+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:56.711218+0000 front 2024-09-13T00:51:56.710814+0000 (oldest deadline 2024-09-13T00:52:20.210630+0000) 2024-09-13T00:52:51.546 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:51 smithi181 bash[84920]: debug 2024-09-13T00:52:51.267+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:01.912156+0000 front 2024-09-13T00:52:01.912324+0000 (oldest deadline 2024-09-13T00:52:27.212136+0000) 2024-09-13T00:52:51.546 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:51 smithi181 bash[84920]: debug 2024-09-13T00:52:51.267+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6822 osd.2 since back 2024-09-13T00:52:12.513546+0000 front 2024-09-13T00:52:12.513849+0000 (oldest deadline 2024-09-13T00:52:36.013356+0000) 2024-09-13T00:52:51.546 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:51 smithi181 bash[84920]: debug 2024-09-13T00:52:51.267+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6830 osd.3 since back 2024-09-13T00:52:23.015232+0000 front 2024-09-13T00:52:23.015273+0000 (oldest deadline 2024-09-13T00:52:44.714868+0000) 2024-09-13T00:52:52.166 DEBUG:teuthology.orchestra.run.smithi181:> sudo pkill -f 'journalctl -f -n 0 -u ceph-da58a47c-715d-11ef-bceb-c7b262605968@osd.6.service' 2024-09-13T00:52:52.193 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T00:52:52.193 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-09-13T00:52:52.194 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-09-13T00:52:52.194 DEBUG:teuthology.orchestra.run.smithi181:> sudo systemctl stop ceph-da58a47c-715d-11ef-bceb-c7b262605968@osd.7 2024-09-13T00:52:52.598 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:52 smithi181 systemd[1]: Stopping Ceph osd.7 for da58a47c-715d-11ef-bceb-c7b262605968... 2024-09-13T00:52:52.599 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:52 smithi181 bash[84920]: debug 2024-09-13T00:52:52.291+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:56.711218+0000 front 2024-09-13T00:51:56.710814+0000 (oldest deadline 2024-09-13T00:52:20.210630+0000) 2024-09-13T00:52:52.599 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:52 smithi181 bash[84920]: debug 2024-09-13T00:52:52.291+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:01.912156+0000 front 2024-09-13T00:52:01.912324+0000 (oldest deadline 2024-09-13T00:52:27.212136+0000) 2024-09-13T00:52:52.599 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:52 smithi181 bash[84920]: debug 2024-09-13T00:52:52.291+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6822 osd.2 since back 2024-09-13T00:52:12.513546+0000 front 2024-09-13T00:52:12.513849+0000 (oldest deadline 2024-09-13T00:52:36.013356+0000) 2024-09-13T00:52:52.599 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:52 smithi181 bash[84920]: debug 2024-09-13T00:52:52.291+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6830 osd.3 since back 2024-09-13T00:52:23.015232+0000 front 2024-09-13T00:52:23.015273+0000 (oldest deadline 2024-09-13T00:52:44.714868+0000) 2024-09-13T00:52:52.599 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:52 smithi181 bash[84920]: debug 2024-09-13T00:52:52.351+0000 7f2f04a86640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-13T00:52:52.600 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:52 smithi181 bash[84920]: debug 2024-09-13T00:52:52.351+0000 7f2f04a86640 -1 osd.7 2617 *** Got signal Terminated *** 2024-09-13T00:52:52.600 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:52 smithi181 bash[84920]: debug 2024-09-13T00:52:52.351+0000 7f2f04a86640 -1 osd.7 2617 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T00:52:53.598 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:53 smithi181 bash[84920]: debug 2024-09-13T00:52:53.279+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:56.711218+0000 front 2024-09-13T00:51:56.710814+0000 (oldest deadline 2024-09-13T00:52:20.210630+0000) 2024-09-13T00:52:53.598 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:53 smithi181 bash[84920]: debug 2024-09-13T00:52:53.279+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:01.912156+0000 front 2024-09-13T00:52:01.912324+0000 (oldest deadline 2024-09-13T00:52:27.212136+0000) 2024-09-13T00:52:53.599 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:53 smithi181 bash[84920]: debug 2024-09-13T00:52:53.279+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6822 osd.2 since back 2024-09-13T00:52:12.513546+0000 front 2024-09-13T00:52:12.513849+0000 (oldest deadline 2024-09-13T00:52:36.013356+0000) 2024-09-13T00:52:53.599 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:53 smithi181 bash[84920]: debug 2024-09-13T00:52:53.279+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6830 osd.3 since back 2024-09-13T00:52:23.015232+0000 front 2024-09-13T00:52:23.015273+0000 (oldest deadline 2024-09-13T00:52:44.714868+0000) 2024-09-13T00:52:53.599 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:53 smithi181 bash[84920]: debug 2024-09-13T00:52:53.279+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.181:6804 osd.4 since back 2024-09-13T00:52:28.816259+0000 front 2024-09-13T00:52:28.816298+0000 (oldest deadline 2024-09-13T00:52:52.916043+0000) 2024-09-13T00:52:54.598 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:54 smithi181 bash[84920]: debug 2024-09-13T00:52:54.287+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:56.711218+0000 front 2024-09-13T00:51:56.710814+0000 (oldest deadline 2024-09-13T00:52:20.210630+0000) 2024-09-13T00:52:54.599 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:54 smithi181 bash[84920]: debug 2024-09-13T00:52:54.287+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:01.912156+0000 front 2024-09-13T00:52:01.912324+0000 (oldest deadline 2024-09-13T00:52:27.212136+0000) 2024-09-13T00:52:54.599 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:54 smithi181 bash[84920]: debug 2024-09-13T00:52:54.287+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6822 osd.2 since back 2024-09-13T00:52:12.513546+0000 front 2024-09-13T00:52:12.513849+0000 (oldest deadline 2024-09-13T00:52:36.013356+0000) 2024-09-13T00:52:54.599 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:54 smithi181 bash[84920]: debug 2024-09-13T00:52:54.287+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6830 osd.3 since back 2024-09-13T00:52:23.015232+0000 front 2024-09-13T00:52:23.015273+0000 (oldest deadline 2024-09-13T00:52:44.714868+0000) 2024-09-13T00:52:54.599 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:54 smithi181 bash[84920]: debug 2024-09-13T00:52:54.287+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.181:6804 osd.4 since back 2024-09-13T00:52:28.816259+0000 front 2024-09-13T00:52:28.816298+0000 (oldest deadline 2024-09-13T00:52:52.916043+0000) 2024-09-13T00:52:55.598 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:55 smithi181 bash[84920]: debug 2024-09-13T00:52:55.319+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:56.711218+0000 front 2024-09-13T00:51:56.710814+0000 (oldest deadline 2024-09-13T00:52:20.210630+0000) 2024-09-13T00:52:55.599 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:55 smithi181 bash[84920]: debug 2024-09-13T00:52:55.319+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:01.912156+0000 front 2024-09-13T00:52:01.912324+0000 (oldest deadline 2024-09-13T00:52:27.212136+0000) 2024-09-13T00:52:55.599 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:55 smithi181 bash[84920]: debug 2024-09-13T00:52:55.319+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6822 osd.2 since back 2024-09-13T00:52:12.513546+0000 front 2024-09-13T00:52:12.513849+0000 (oldest deadline 2024-09-13T00:52:36.013356+0000) 2024-09-13T00:52:55.599 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:55 smithi181 bash[84920]: debug 2024-09-13T00:52:55.319+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6830 osd.3 since back 2024-09-13T00:52:23.015232+0000 front 2024-09-13T00:52:23.015273+0000 (oldest deadline 2024-09-13T00:52:44.714868+0000) 2024-09-13T00:52:55.599 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:55 smithi181 bash[84920]: debug 2024-09-13T00:52:55.319+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.181:6804 osd.4 since back 2024-09-13T00:52:28.816259+0000 front 2024-09-13T00:52:28.816298+0000 (oldest deadline 2024-09-13T00:52:52.916043+0000) 2024-09-13T00:52:56.848 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:56 smithi181 bash[84920]: debug 2024-09-13T00:52:56.355+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6806 osd.0 since back 2024-09-13T00:51:56.711218+0000 front 2024-09-13T00:51:56.710814+0000 (oldest deadline 2024-09-13T00:52:20.210630+0000) 2024-09-13T00:52:56.849 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:56 smithi181 bash[84920]: debug 2024-09-13T00:52:56.355+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6814 osd.1 since back 2024-09-13T00:52:01.912156+0000 front 2024-09-13T00:52:01.912324+0000 (oldest deadline 2024-09-13T00:52:27.212136+0000) 2024-09-13T00:52:56.849 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:56 smithi181 bash[84920]: debug 2024-09-13T00:52:56.355+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6822 osd.2 since back 2024-09-13T00:52:12.513546+0000 front 2024-09-13T00:52:12.513849+0000 (oldest deadline 2024-09-13T00:52:36.013356+0000) 2024-09-13T00:52:56.849 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:56 smithi181 bash[84920]: debug 2024-09-13T00:52:56.355+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.136:6830 osd.3 since back 2024-09-13T00:52:23.015232+0000 front 2024-09-13T00:52:23.015273+0000 (oldest deadline 2024-09-13T00:52:44.714868+0000) 2024-09-13T00:52:56.849 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:56 smithi181 bash[84920]: debug 2024-09-13T00:52:56.355+0000 7f2efd092640 -1 osd.7 2617 heartbeat_check: no reply from 172.21.15.181:6804 osd.4 since back 2024-09-13T00:52:28.816259+0000 front 2024-09-13T00:52:28.816298+0000 (oldest deadline 2024-09-13T00:52:52.916043+0000) 2024-09-13T00:52:58.669 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:58 smithi181 bash[110726]: ceph-da58a47c-715d-11ef-bceb-c7b262605968-osd-7 2024-09-13T00:52:58.669 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 00:52:58 smithi181 bash[110792]: Error response from daemon: No such container: ceph-da58a47c-715d-11ef-bceb-c7b262605968-osd-7 2024-09-13T00:53:00.525 DEBUG:teuthology.orchestra.run.smithi181:> sudo pkill -f 'journalctl -f -n 0 -u ceph-da58a47c-715d-11ef-bceb-c7b262605968@osd.7.service' 2024-09-13T00:53:00.552 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T00:53:00.553 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-09-13T00:53:00.553 INFO:tasks.cephadm.mds.a:Stopping mds.a... 2024-09-13T00:53:00.553 DEBUG:teuthology.orchestra.run.smithi136:> sudo systemctl stop ceph-da58a47c-715d-11ef-bceb-c7b262605968@mds.a 2024-09-13T00:53:01.010 INFO:journalctl@ceph.mds.a.smithi136.stdout:Sep 13 00:53:00 smithi136 systemd[1]: Stopping Ceph mds.a for da58a47c-715d-11ef-bceb-c7b262605968... 2024-09-13T00:53:01.010 INFO:journalctl@ceph.mds.a.smithi136.stdout:Sep 13 00:53:00 smithi136 bash[97471]: debug 2024-09-13T00:53:00.689+0000 7fe593db8640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mds -n mds.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-13T00:53:01.011 INFO:journalctl@ceph.mds.a.smithi136.stdout:Sep 13 00:53:00 smithi136 bash[97471]: debug 2024-09-13T00:53:00.689+0000 7fe593db8640 -1 mds.a *** got signal Terminated *** 2024-09-13T00:53:03.846 DEBUG:teuthology.orchestra.run.smithi136:> sudo pkill -f 'journalctl -f -n 0 -u ceph-da58a47c-715d-11ef-bceb-c7b262605968@mds.a.service' 2024-09-13T00:53:03.873 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T00:53:03.873 INFO:tasks.cephadm.mds.a:Stopped mds.a 2024-09-13T00:53:03.874 INFO:tasks.cephadm.mds.b:Stopping mds.b... 2024-09-13T00:53:03.874 DEBUG:teuthology.orchestra.run.smithi181:> sudo systemctl stop ceph-da58a47c-715d-11ef-bceb-c7b262605968@mds.b 2024-09-13T00:53:04.348 INFO:journalctl@ceph.mds.b.smithi181.stdout:Sep 13 00:53:03 smithi181 systemd[1]: Stopping Ceph mds.b for da58a47c-715d-11ef-bceb-c7b262605968... 2024-09-13T00:53:04.349 INFO:journalctl@ceph.mds.b.smithi181.stdout:Sep 13 00:53:04 smithi181 bash[88772]: debug 2024-09-13T00:53:04.011+0000 7f1c8e7e0640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mds -n mds.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-13T00:53:04.349 INFO:journalctl@ceph.mds.b.smithi181.stdout:Sep 13 00:53:04 smithi181 bash[88772]: debug 2024-09-13T00:53:04.011+0000 7f1c8e7e0640 -1 mds.b *** got signal Terminated *** 2024-09-13T00:53:06.055 DEBUG:teuthology.orchestra.run.smithi181:> sudo pkill -f 'journalctl -f -n 0 -u ceph-da58a47c-715d-11ef-bceb-c7b262605968@mds.b.service' 2024-09-13T00:53:06.080 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T00:53:06.081 INFO:tasks.cephadm.mds.b:Stopped mds.b 2024-09-13T00:53:06.081 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-09-13T00:53:06.081 DEBUG:teuthology.orchestra.run.smithi181:> sudo systemctl stop ceph-da58a47c-715d-11ef-bceb-c7b262605968@prometheus.a 2024-09-13T00:53:06.348 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:53:06 smithi181 systemd[1]: Stopping Ceph prometheus.a for da58a47c-715d-11ef-bceb-c7b262605968... 2024-09-13T00:53:06.348 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:53:06 smithi181 bash[95505]: ts=2024-09-13T00:53:06.238Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-13T00:53:06.349 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:53:06 smithi181 bash[95505]: ts=2024-09-13T00:53:06.238Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-13T00:53:06.349 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:53:06 smithi181 bash[95505]: ts=2024-09-13T00:53:06.239Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-13T00:53:06.349 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:53:06 smithi181 bash[95505]: ts=2024-09-13T00:53:06.239Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-13T00:53:06.349 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:53:06 smithi181 bash[95505]: ts=2024-09-13T00:53:06.239Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-13T00:53:06.349 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:53:06 smithi181 bash[95505]: ts=2024-09-13T00:53:06.239Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-13T00:53:06.349 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:53:06 smithi181 bash[95505]: ts=2024-09-13T00:53:06.239Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-13T00:53:06.349 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:53:06 smithi181 bash[95505]: ts=2024-09-13T00:53:06.239Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-13T00:53:06.350 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:53:06 smithi181 bash[95505]: ts=2024-09-13T00:53:06.241Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-13T00:53:06.350 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:53:06 smithi181 bash[95505]: ts=2024-09-13T00:53:06.245Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-13T00:53:06.350 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:53:06 smithi181 bash[95505]: ts=2024-09-13T00:53:06.245Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-13T00:53:06.350 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Sep 13 00:53:06 smithi181 bash[95505]: ts=2024-09-13T00:53:06.245Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-13T00:53:06.998 DEBUG:teuthology.orchestra.run.smithi181:> sudo pkill -f 'journalctl -f -n 0 -u ceph-da58a47c-715d-11ef-bceb-c7b262605968@prometheus.a.service' 2024-09-13T00:53:07.022 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T00:53:07.022 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-09-13T00:53:07.022 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid da58a47c-715d-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-13T00:53:11.889 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 13 00:53:11 smithi136 bash[104600]: ts=2024-09-13T00:53:11.625Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2024-09-13T00:53:11.890 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 13 00:53:11 smithi136 bash[104600]: ts=2024-09-13T00:53:11.627Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi181.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.181:8443: connect: connection refused" 2024-09-13T00:53:12.199 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 13 00:53:11 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:53:12.200 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 13 00:53:12 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:53:12.200 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 13 00:53:11 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:53:12.200 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 13 00:53:12 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:53:12.469 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 13 00:53:12 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:53:12.470 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 13 00:53:12 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:53:12.760 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 13 00:53:12 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:53:12.760 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 13 00:53:12 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:53:13.084 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 13 00:53:12 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:53:13.084 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 13 00:53:13 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:53:13.084 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 13 00:53:12 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:53:13.084 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 13 00:53:13 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:53:13.333 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 13 00:53:13 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:53:13.334 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 13 00:53:13 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:53:13.687 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 13 00:53:13 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:53:13.688 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 13 00:53:13 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:53:13.688 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 13 00:53:13 smithi136 systemd[1]: Stopping Ceph node-exporter.a for da58a47c-715d-11ef-bceb-c7b262605968... 2024-09-13T00:53:14.586 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 13 00:53:14 smithi136 bash[123387]: ceph-da58a47c-715d-11ef-bceb-c7b262605968-node-exporter-a 2024-09-13T00:53:14.587 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 13 00:53:14 smithi136 systemd[1]: ceph-da58a47c-715d-11ef-bceb-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-09-13T00:53:14.587 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 13 00:53:14 smithi136 bash[123475]: Error response from daemon: No such container: ceph-da58a47c-715d-11ef-bceb-c7b262605968-node-exporter-a 2024-09-13T00:53:14.587 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 13 00:53:14 smithi136 systemd[1]: ceph-da58a47c-715d-11ef-bceb-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-09-13T00:53:14.587 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 13 00:53:14 smithi136 systemd[1]: Stopped Ceph node-exporter.a for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-13T00:53:14.843 INFO:journalctl@ceph.node-exporter.a.smithi136.stdout:Sep 13 00:53:14 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:53:14.843 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 13 00:53:14 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:53:14.844 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 13 00:53:14 smithi136 systemd[1]: Stopping Ceph alertmanager.a for da58a47c-715d-11ef-bceb-c7b262605968... 2024-09-13T00:53:14.844 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 13 00:53:14 smithi136 bash[104600]: ts=2024-09-13T00:53:14.798Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-13T00:53:15.584 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 13 00:53:15 smithi136 bash[123534]: ceph-da58a47c-715d-11ef-bceb-c7b262605968-alertmanager-a 2024-09-13T00:53:15.584 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 13 00:53:15 smithi136 bash[123592]: Error response from daemon: No such container: ceph-da58a47c-715d-11ef-bceb-c7b262605968-alertmanager-a 2024-09-13T00:53:15.584 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 13 00:53:15 smithi136 systemd[1]: ceph-da58a47c-715d-11ef-bceb-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-09-13T00:53:15.585 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 13 00:53:15 smithi136 systemd[1]: Stopped Ceph alertmanager.a for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-13T00:53:16.002 INFO:journalctl@ceph.alertmanager.a.smithi136.stdout:Sep 13 00:53:15 smithi136 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:53:16.266 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid da58a47c-715d-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-13T00:53:21.293 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 13 00:53:21 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:53:21.294 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 13 00:53:21 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:53:21.548 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 13 00:53:21 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:53:21.548 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 13 00:53:21 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:53:21.870 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 13 00:53:21 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:53:21.870 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 13 00:53:21 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:53:21.870 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 13 00:53:21 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:53:22.166 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 13 00:53:22 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:53:22.166 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 13 00:53:21 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:53:22.166 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 13 00:53:22 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:53:22.430 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 13 00:53:22 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:53:22.431 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 13 00:53:22 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:53:22.772 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 13 00:53:22 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:53:22.772 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 13 00:53:22 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:53:23.098 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 13 00:53:22 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:53:23.098 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 13 00:53:22 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:53:23.099 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 13 00:53:22 smithi181 systemd[1]: Stopping Ceph node-exporter.b for da58a47c-715d-11ef-bceb-c7b262605968... 2024-09-13T00:53:23.848 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 13 00:53:23 smithi181 bash[111378]: ceph-da58a47c-715d-11ef-bceb-c7b262605968-node-exporter-b 2024-09-13T00:53:23.848 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 13 00:53:23 smithi181 systemd[1]: ceph-da58a47c-715d-11ef-bceb-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-09-13T00:53:23.848 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 13 00:53:23 smithi181 bash[111439]: Error response from daemon: No such container: ceph-da58a47c-715d-11ef-bceb-c7b262605968-node-exporter-b 2024-09-13T00:53:23.849 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 13 00:53:23 smithi181 systemd[1]: ceph-da58a47c-715d-11ef-bceb-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-09-13T00:53:23.849 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 13 00:53:23 smithi181 systemd[1]: Stopped Ceph node-exporter.b for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-13T00:53:24.147 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Sep 13 00:53:23 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:53:24.148 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 13 00:53:23 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:53:24.148 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 13 00:53:23 smithi181 systemd[1]: Stopping Ceph grafana.a for da58a47c-715d-11ef-bceb-c7b262605968... 2024-09-13T00:53:24.848 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 13 00:53:24 smithi181 bash[111501]: ceph-da58a47c-715d-11ef-bceb-c7b262605968-grafana-a 2024-09-13T00:53:24.848 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 13 00:53:24 smithi181 systemd[1]: ceph-da58a47c-715d-11ef-bceb-c7b262605968@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-09-13T00:53:24.848 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 13 00:53:24 smithi181 bash[111562]: Error response from daemon: No such container: ceph-da58a47c-715d-11ef-bceb-c7b262605968-grafana-a 2024-09-13T00:53:24.849 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 13 00:53:24 smithi181 systemd[1]: ceph-da58a47c-715d-11ef-bceb-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-09-13T00:53:24.849 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 13 00:53:24 smithi181 systemd[1]: Stopped Ceph grafana.a for da58a47c-715d-11ef-bceb-c7b262605968. 2024-09-13T00:53:25.184 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Sep 13 00:53:24 smithi181 systemd[1]: /etc/systemd/system/ceph-da58a47c-715d-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:53:25.425 DEBUG:teuthology.orchestra.run.smithi136:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-13T00:53:25.439 DEBUG:teuthology.orchestra.run.smithi181:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-13T00:53:25.452 INFO:tasks.cephadm:Archiving crash dumps... 2024-09-13T00:53:25.455 DEBUG:teuthology.misc:Transferring archived files from smithi136:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-09-11_01:16:05-upgrade:quincy-x-reef-distro-default-smithi/7901063/remote/smithi136/crash 2024-09-13T00:53:25.455 DEBUG:teuthology.orchestra.run.smithi136:> sudo tar c -f - -C /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/crash -- . 2024-09-13T00:53:25.496 INFO:teuthology.orchestra.run.smithi136.stderr:tar: /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/crash: Cannot open: No such file or directory 2024-09-13T00:53:25.496 INFO:teuthology.orchestra.run.smithi136.stderr:tar: Error is not recoverable: exiting now 2024-09-13T00:53:25.498 DEBUG:teuthology.misc:Transferring archived files from smithi181:/var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-09-11_01:16:05-upgrade:quincy-x-reef-distro-default-smithi/7901063/remote/smithi181/crash 2024-09-13T00:53:25.499 DEBUG:teuthology.orchestra.run.smithi181:> sudo tar c -f - -C /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/crash -- . 2024-09-13T00:53:25.509 INFO:teuthology.orchestra.run.smithi181.stderr:tar: /var/lib/ceph/da58a47c-715d-11ef-bceb-c7b262605968/crash: Cannot open: No such file or directory 2024-09-13T00:53:25.509 INFO:teuthology.orchestra.run.smithi181.stderr:tar: Error is not recoverable: exiting now 2024-09-13T00:53:25.510 INFO:tasks.cephadm:Checking cluster log for badness... 2024-09-13T00:53:25.510 DEBUG:teuthology.orchestra.run.smithi136:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/da58a47c-715d-11ef-bceb-c7b262605968/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-09-13T00:53:25.551 INFO:teuthology.orchestra.run.smithi136.stderr:grep: /var/log/ceph/da58a47c-715d-11ef-bceb-c7b262605968/ceph.log: No such file or directory 2024-09-13T00:53:25.553 INFO:tasks.cephadm:Compressing logs... 2024-09-13T00:53:25.553 DEBUG:teuthology.orchestra.run.smithi136:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-13T00:53:25.597 DEBUG:teuthology.orchestra.run.smithi181:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-13T00:53:25.605 INFO:teuthology.orchestra.run.smithi136.stderr:find: '/var/log/rbd-target-api': No such file or directory 2024-09-13T00:53:25.609 INFO:teuthology.orchestra.run.smithi181.stderr:find: '/var/log/rbd-target-api': No such file or directory 2024-09-13T00:55:17.463 INFO:tasks.cephadm:Archiving logs... 2024-09-13T00:55:17.464 DEBUG:teuthology.misc:Transferring archived files from smithi136:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-11_01:16:05-upgrade:quincy-x-reef-distro-default-smithi/7901063/remote/smithi136/log 2024-09-13T00:55:17.465 DEBUG:teuthology.orchestra.run.smithi136:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-13T00:55:28.009 DEBUG:teuthology.misc:Transferring archived files from smithi181:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-11_01:16:05-upgrade:quincy-x-reef-distro-default-smithi/7901063/remote/smithi181/log 2024-09-13T00:55:28.010 DEBUG:teuthology.orchestra.run.smithi181:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-13T00:55:35.891 INFO:tasks.cephadm:Removing cluster... 2024-09-13T00:55:35.891 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid da58a47c-715d-11ef-bceb-c7b262605968 --force 2024-09-13T00:55:37.539 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid da58a47c-715d-11ef-bceb-c7b262605968 --force 2024-09-13T00:55:39.117 INFO:tasks.cephadm:Removing cephadm ... 2024-09-13T00:55:39.118 DEBUG:teuthology.orchestra.run.smithi136:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-13T00:55:39.124 DEBUG:teuthology.orchestra.run.smithi181:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-13T00:55:39.129 INFO:tasks.cephadm:Teardown complete 2024-09-13T00:55:39.129 DEBUG:teuthology.run_tasks:Unwinding manager install 2024-09-13T00:55:39.145 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2024-09-13T00:55:39.146 DEBUG:teuthology.orchestra.run.smithi136:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-09-13T00:55:39.170 DEBUG:teuthology.orchestra.run.smithi181:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-09-13T00:55:39.233 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-09-13T00:55:39.243 INFO:teuthology.task.clock:Checking final clock skew... 2024-09-13T00:55:39.243 DEBUG:teuthology.orchestra.run.smithi136:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-13T00:55:39.246 DEBUG:teuthology.orchestra.run.smithi181:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-13T00:55:39.259 INFO:teuthology.orchestra.run.smithi136.stdout: remote refid st t when poll reach delay offset jitter 2024-09-13T00:55:39.260 INFO:teuthology.orchestra.run.smithi136.stdout:============================================================================== 2024-09-13T00:55:39.260 INFO:teuthology.orchestra.run.smithi136.stdout:*hv01.front.sepi 67.205.162.81 3 u 17 64 377 0.093 -0.607 0.089 2024-09-13T00:55:39.260 INFO:teuthology.orchestra.run.smithi136.stdout:+hv02.front.sepi 63.231.80.2 3 u 5 64 377 0.125 -0.155 0.099 2024-09-13T00:55:39.260 INFO:teuthology.orchestra.run.smithi136.stdout:+hv03.front.sepi 74.6.168.72 3 u 60 64 377 0.105 -0.599 0.079 2024-09-13T00:55:39.260 INFO:teuthology.orchestra.run.smithi136.stdout: hv04.front.sepi .INIT. 16 u - 1024 0 0.000 +0.000 0.000 2024-09-13T00:55:39.262 INFO:teuthology.orchestra.run.smithi181.stdout: remote refid st t when poll reach delay offset jitter 2024-09-13T00:55:39.262 INFO:teuthology.orchestra.run.smithi181.stdout:============================================================================== 2024-09-13T00:55:39.263 INFO:teuthology.orchestra.run.smithi181.stdout:*hv01.front.sepi 67.205.162.81 3 u 39 128 377 0.221 -1.772 0.292 2024-09-13T00:55:39.263 INFO:teuthology.orchestra.run.smithi181.stdout:+hv02.front.sepi 63.231.80.2 3 u 114 128 377 0.187 -1.029 0.248 2024-09-13T00:55:39.263 INFO:teuthology.orchestra.run.smithi181.stdout:+hv03.front.sepi 74.6.168.72 3 u 3 128 377 0.208 -1.546 0.206 2024-09-13T00:55:39.263 INFO:teuthology.orchestra.run.smithi181.stdout: hv04.front.sepi .INIT. 16 u - 1024 0 0.000 +0.000 0.000 2024-09-13T00:55:39.264 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-09-13T00:55:39.288 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-09-13T00:55:39.289 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-09-13T00:55:39.357 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-09-13T00:55:39.407 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-09-13T00:55:39.454 INFO:teuthology.task.internal:Duration was 6153.819750 seconds 2024-09-13T00:55:39.454 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-09-13T00:55:39.522 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-09-13T00:55:39.522 DEBUG:teuthology.orchestra.run.smithi136:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-13T00:55:39.525 DEBUG:teuthology.orchestra.run.smithi181:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-13T00:55:39.563 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-09-13T00:55:39.563 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi136.front.sepia.ceph.com 2024-09-13T00:55:39.563 DEBUG:teuthology.orchestra.run.smithi136:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-13T00:55:39.608 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi181.front.sepia.ceph.com 2024-09-13T00:55:39.609 DEBUG:teuthology.orchestra.run.smithi181:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-13T00:55:39.620 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-09-13T00:55:39.621 DEBUG:teuthology.orchestra.run.smithi136:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-13T00:55:39.657 DEBUG:teuthology.orchestra.run.smithi181:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-13T00:55:39.753 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-09-13T00:55:39.753 DEBUG:teuthology.orchestra.run.smithi136:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-13T00:55:39.755 DEBUG:teuthology.orchestra.run.smithi181:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-13T00:55:40.046 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-09-13T00:55:40.056 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-09-13T00:55:40.057 DEBUG:teuthology.orchestra.run.smithi136:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-13T00:55:40.069 DEBUG:teuthology.orchestra.run.smithi181:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-13T00:55:40.083 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-09-13T00:55:40.094 DEBUG:teuthology.orchestra.run.smithi136:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-13T00:55:40.113 DEBUG:teuthology.orchestra.run.smithi181:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-13T00:55:40.121 INFO:teuthology.orchestra.run.smithi136.stdout:kernel.core_pattern = core 2024-09-13T00:55:40.136 INFO:teuthology.orchestra.run.smithi181.stdout:kernel.core_pattern = core 2024-09-13T00:55:40.148 DEBUG:teuthology.orchestra.run.smithi136:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-13T00:55:40.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T00:55:40.180 DEBUG:teuthology.orchestra.run.smithi181:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-13T00:55:40.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T00:55:40.194 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-09-13T00:55:40.205 INFO:teuthology.task.internal:Transferring archived files... 2024-09-13T00:55:40.206 DEBUG:teuthology.misc:Transferring archived files from smithi136:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-11_01:16:05-upgrade:quincy-x-reef-distro-default-smithi/7901063/remote/smithi136 2024-09-13T00:55:40.206 DEBUG:teuthology.orchestra.run.smithi136:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-13T00:55:40.255 DEBUG:teuthology.misc:Transferring archived files from smithi181:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-11_01:16:05-upgrade:quincy-x-reef-distro-default-smithi/7901063/remote/smithi181 2024-09-13T00:55:40.255 DEBUG:teuthology.orchestra.run.smithi181:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-13T00:55:40.284 INFO:teuthology.task.internal:Removing archive directory... 2024-09-13T00:55:40.285 DEBUG:teuthology.orchestra.run.smithi136:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-13T00:55:40.293 DEBUG:teuthology.orchestra.run.smithi181:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-13T00:55:40.327 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-09-13T00:55:40.339 INFO:teuthology.task.internal:Not uploading archives. 2024-09-13T00:55:40.339 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-09-13T00:55:40.363 INFO:teuthology.task.internal:Tidying up after the test... 2024-09-13T00:55:40.364 DEBUG:teuthology.orchestra.run.smithi136:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-13T00:55:40.366 DEBUG:teuthology.orchestra.run.smithi181:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-13T00:55:40.370 INFO:teuthology.orchestra.run.smithi136.stdout: 658646 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Sep 13 00:55 /home/ubuntu/cephtest 2024-09-13T00:55:40.371 INFO:teuthology.orchestra.run.smithi136.stdout: 1062873 4 drwxrwxr-x 3 ubuntu ubuntu 4096 Sep 13 00:40 /home/ubuntu/cephtest/mnt.0 2024-09-13T00:55:40.371 INFO:teuthology.orchestra.run.smithi136.stdout: 1062886 4 drwxrwxr-x 3 ubuntu ubuntu 4096 Sep 13 00:40 /home/ubuntu/cephtest/mnt.0/client.0 2024-09-13T00:55:40.371 INFO:teuthology.orchestra.run.smithi136.stdout: 1076839 4 drwxrwxr-x 2 ubuntu ubuntu 4096 Sep 13 00:40 /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-13T00:55:40.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T00:55:40.373 INFO:teuthology.orchestra.run.smithi136.stderr:rmdir: failed to remove '/home/ubuntu/cephtest': Directory not empty 2024-09-13T00:55:40.373 ERROR:teuthology.run_tasks:Manager failed: internal.base Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 160, in run_tasks suppress = manager.__exit__(*exc_info) File "/usr/lib/python3.10/contextlib.py", line 142, in __exit__ next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/internal/__init__.py", line 53, in base run.wait( File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 479, in wait proc.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi136 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2024-09-13T00:55:40.381 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-09-13T00:55:40.415 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-09-13T00:55:40.473 DEBUG:teuthology.run_tasks:Exception was not quenched, exiting: CommandFailedError: Command failed on smithi136 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2024-09-13T00:55:40.478 INFO:teuthology.run:Summary data: description: upgrade:quincy-x/parallel/{0-random-distro$/{ubuntu_22.04} 0-start 1-tasks mon_election/classic upgrade-sequence workload/{ec-rados-default rados_api rados_loadgenbig rbd_import_export test_rbd_api test_rbd_python}} duration: 6153.8197503089905 failure_reason: 'Command failed (workunit test rbd/test_librbd_python.sh) on smithi136 with status 1: ''mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_FEATURES=61 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/test_librbd_python.sh''' flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=3ff6a9c762404a5db8bc7718de80ef7c status: fail success: false 2024-09-13T00:55:40.478 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-13T00:55:40.487 INFO:teuthology.orchestra.run.smithi181.stdout: 658637 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 13 00:55 /home/ubuntu/cephtest 2024-09-13T00:55:40.577 INFO:teuthology.run:FAIL